From 2bd30b8dc75b823dd7bbef9d7bc68e1fd7ecee5c Mon Sep 17 00:00:00 2001 From: Sang-Heon Jeon Date: Mon, 24 Aug 2026 01:18:14 +0900 Subject: coccinelle: alloc_cast: drop removed allocators - pci_alloc_consistent() and pci_zalloc_consistent() were removed by commit 7968778914e5 ("PCI: Remove the deprecated "pci-dma-compat.h" API") - kmem_alloc() was removed by commit f078d4ea8276 ("xfs: convert kmem_alloc() to kmalloc()") - kmem_zalloc() was removed by commit 10634530f7ba ("xfs: convert kmem_zalloc() to kzalloc()") - kmem_zone_alloc() and kmem_zone_zalloc() were removed by commit bae633a4a283 ("xfs: remove xfs_zone_{alloc,zalloc} helpers") So drop them from the rules. No functional change. Signed-off-by: Sang-Heon Jeon Signed-off-by: Julia Lawall --- fs/9p/Kconfig | 45 + fs/9p/Makefile | 17 + fs/9p/acl.c | 336 + fs/9p/acl.h | 54 + fs/9p/cache.c | 76 + fs/9p/cache.h | 26 + fs/9p/fid.c | 316 + fs/9p/fid.h | 61 + fs/9p/v9fs.c | 786 ++ fs/9p/v9fs.h | 269 + fs/9p/v9fs_vfs.h | 93 + fs/9p/vfs_addr.c | 201 + fs/9p/vfs_dentry.c | 228 + fs/9p/vfs_dir.c | 266 + fs/9p/vfs_file.c | 534 + fs/9p/vfs_inode.c | 1402 +++ fs/9p/vfs_inode_dotl.c | 994 ++ fs/9p/vfs_super.c | 366 + fs/9p/xattr.c | 193 + fs/9p/xattr.h | 25 + fs/Kconfig | 424 + fs/Kconfig.binfmt | 210 + fs/Makefile | 130 + fs/adfs/Kconfig | 29 + fs/adfs/Makefile | 8 + fs/adfs/adfs.h | 230 + fs/adfs/dir.c | 458 + fs/adfs/dir_f.c | 324 + fs/adfs/dir_f.h | 61 + fs/adfs/dir_fplus.c | 287 + fs/adfs/dir_fplus.h | 42 + fs/adfs/file.c | 36 + fs/adfs/inode.c | 373 + fs/adfs/map.c | 407 + fs/adfs/super.c | 502 + fs/affs/Changes | 343 + fs/affs/Kconfig | 24 + fs/affs/Makefile | 10 + fs/affs/affs.h | 322 + fs/affs/amigaffs.c | 543 + fs/affs/amigaffs.h | 149 + fs/affs/bitmap.c | 365 + fs/affs/dir.c | 180 + fs/affs/file.c | 1017 ++ fs/affs/inode.c | 417 + fs/affs/namei.c | 582 ++ fs/affs/super.c | 684 ++ fs/affs/symlink.c | 75 + fs/afs/Kconfig | 44 + fs/afs/Makefile | 46 + fs/afs/addr_list.c | 414 + fs/afs/addr_prefs.c | 533 + fs/afs/afs.h | 206 + fs/afs/afs_cm.h | 29 + fs/afs/afs_fs.h | 62 + fs/afs/afs_vl.h | 138 + fs/afs/callback.c | 262 + fs/afs/cell.c | 939 ++ fs/afs/cm_security.c | 341 + fs/afs/cmservice.c | 620 ++ fs/afs/dir.c | 2240 ++++ fs/afs/dir_edit.c | 648 ++ fs/afs/dir_search.c | 227 + fs/afs/dir_silly.c | 289 + fs/afs/dynroot.c | 406 + fs/afs/file.c | 596 ++ fs/afs/flock.c | 877 ++ fs/afs/fs_operation.c | 377 + fs/afs/fs_probe.c | 539 + fs/afs/fsclient.c | 2137 ++++ fs/afs/inode.c | 853 ++ fs/afs/internal.h | 1970 ++++ fs/afs/main.c | 237 + fs/afs/misc.c | 208 + fs/afs/mntpt.c | 226 + fs/afs/proc.c | 786 ++ fs/afs/protocol_afs.h | 15 + fs/afs/protocol_uae.h | 132 + fs/afs/protocol_yfs.h | 179 + fs/afs/rotate.c | 763 ++ fs/afs/rxrpc.c | 996 ++ fs/afs/security.c | 520 + fs/afs/server.c | 631 ++ fs/afs/server_list.c | 249 + fs/afs/super.c | 771 ++ fs/afs/symlink.c | 278 + fs/afs/validation.c | 490 + fs/afs/vl_alias.c | 340 + fs/afs/vl_list.c | 354 + fs/afs/vl_probe.c | 310 + fs/afs/vl_rotate.c | 399 + fs/afs/vlclient.c | 793 ++ fs/afs/volume.c | 472 + fs/afs/write.c | 308 + fs/afs/xattr.c | 363 + fs/afs/xdr_fs.h | 116 + fs/afs/yfsclient.c | 2232 ++++ fs/aio.c | 2498 +++++ fs/anon_inodes.c | 358 + fs/attr.c | 567 + fs/autofs/Kconfig | 20 + fs/autofs/Makefile | 8 + fs/autofs/autofs_i.h | 306 + fs/autofs/dev-ioctl.c | 810 ++ fs/autofs/expire.c | 620 ++ fs/autofs/init.c | 42 + fs/autofs/inode.c | 455 + fs/autofs/root.c | 931 ++ fs/autofs/symlink.c | 26 + fs/autofs/waitq.c | 513 + fs/backing-file.c | 366 + fs/bad_inode.c | 251 + fs/befs/ChangeLog | 417 + fs/befs/Kconfig | 28 + fs/befs/Makefile | 8 + fs/befs/befs.h | 150 + fs/befs/befs_fs_types.h | 256 + fs/befs/btree.c | 783 ++ fs/befs/btree.h | 12 + fs/befs/datastream.c | 525 + fs/befs/datastream.h | 20 + fs/befs/debug.c | 262 + fs/befs/endian.h | 126 + fs/befs/inode.c | 55 + fs/befs/inode.h | 7 + fs/befs/io.c | 61 + fs/befs/io.h | 6 + fs/befs/linuxvfs.c | 1030 ++ fs/befs/super.c | 118 + fs/befs/super.h | 6 + fs/bfs/Kconfig | 21 + fs/bfs/Makefile | 8 + fs/bfs/bfs.h | 69 + fs/bfs/dir.c | 356 + fs/bfs/file.c | 203 + fs/bfs/inode.c | 538 + fs/binfmt_elf.c | 2144 ++++ fs/binfmt_elf_fdpic.c | 1672 +++ fs/binfmt_flat.c | 946 ++ fs/binfmt_misc.c | 1695 +++ fs/binfmt_misc_bpf.c | 434 + fs/binfmt_script.c | 159 + fs/bpf_fs_kfuncs.c | 503 + fs/btrfs/Kconfig | 118 + fs/btrfs/Makefile | 51 + fs/btrfs/accessors.c | 125 + fs/btrfs/accessors.h | 1050 ++ fs/btrfs/acl.c | 124 + fs/btrfs/acl.h | 40 + fs/btrfs/async-thread.c | 372 + fs/btrfs/async-thread.h | 51 + fs/btrfs/backref.c | 3697 +++++++ fs/btrfs/backref.h | 469 + fs/btrfs/bio.c | 1072 ++ fs/btrfs/bio.h | 133 + fs/btrfs/block-group.c | 5027 +++++++++ fs/btrfs/block-group.h | 429 + fs/btrfs/block-rsv.c | 627 ++ fs/btrfs/block-rsv.h | 145 + fs/btrfs/btrfs_inode.h | 643 ++ fs/btrfs/compression.c | 1652 +++ fs/btrfs/compression.h | 183 + fs/btrfs/ctree.c | 5091 +++++++++ fs/btrfs/ctree.h | 746 ++ fs/btrfs/defrag.c | 1506 +++ fs/btrfs/defrag.h | 31 + fs/btrfs/delalloc-space.c | 539 + fs/btrfs/delalloc-space.h | 31 + fs/btrfs/delayed-inode.c | 2228 ++++ fs/btrfs/delayed-inode.h | 257 + fs/btrfs/delayed-ref.c | 1406 +++ fs/btrfs/delayed-ref.h | 459 + fs/btrfs/dev-replace.c | 1372 +++ fs/btrfs/dev-replace.h | 41 + fs/btrfs/dir-item.c | 432 + fs/btrfs/dir-item.h | 57 + fs/btrfs/direct-io.c | 1164 +++ fs/btrfs/direct-io.h | 16 + fs/btrfs/discard.c | 828 ++ fs/btrfs/discard.h | 40 + fs/btrfs/disk-io.c | 5097 +++++++++ fs/btrfs/disk-io.h | 133 + fs/btrfs/export.c | 307 + fs/btrfs/export.h | 29 + fs/btrfs/extent-io-tree.c | 2071 ++++ fs/btrfs/extent-io-tree.h | 244 + fs/btrfs/extent-tree.c | 6938 ++++++++++++ fs/btrfs/extent-tree.h | 169 + fs/btrfs/extent_io.c | 5124 +++++++++ fs/btrfs/extent_io.h | 454 + fs/btrfs/extent_map.c | 1396 +++ fs/btrfs/extent_map.h | 195 + fs/btrfs/fiemap.c | 928 ++ fs/btrfs/fiemap.h | 11 + fs/btrfs/file-item.c | 1433 +++ fs/btrfs/file-item.h | 87 + fs/btrfs/file.c | 3863 +++++++ fs/btrfs/file.h | 51 + fs/btrfs/free-space-cache.c | 4390 ++++++++ fs/btrfs/free-space-cache.h | 174 + fs/btrfs/free-space-tree.c | 1852 ++++ fs/btrfs/free-space-tree.h | 62 + fs/btrfs/fs.c | 344 + fs/btrfs/fs.h | 1263 +++ fs/btrfs/inode-item.c | 733 ++ fs/btrfs/inode-item.h | 116 + fs/btrfs/inode.c | 10862 +++++++++++++++++++ fs/btrfs/ioctl.c | 5704 ++++++++++ fs/btrfs/ioctl.h | 29 + fs/btrfs/locking.c | 383 + fs/btrfs/locking.h | 251 + fs/btrfs/lru_cache.c | 166 + fs/btrfs/lru_cache.h | 71 + fs/btrfs/lzo.c | 604 ++ fs/btrfs/messages.c | 299 + fs/btrfs/messages.h | 207 + fs/btrfs/misc.h | 227 + fs/btrfs/ordered-data.c | 1371 +++ fs/btrfs/ordered-data.h | 234 + fs/btrfs/orphan.c | 48 + fs/btrfs/orphan.h | 16 + fs/btrfs/print-tree.c | 639 ++ fs/btrfs/print-tree.h | 21 + fs/btrfs/props.c | 485 + fs/btrfs/props.h | 31 + fs/btrfs/qgroup.c | 4993 +++++++++ fs/btrfs/qgroup.h | 421 + fs/btrfs/raid-stripe-tree.c | 511 + fs/btrfs/raid-stripe-tree.h | 60 + fs/btrfs/raid56.c | 3035 ++++++ fs/btrfs/raid56.h | 291 + fs/btrfs/ref-verify.c | 1025 ++ fs/btrfs/ref-verify.h | 58 + fs/btrfs/reflink.c | 983 ++ fs/btrfs/reflink.h | 14 + fs/btrfs/relocation.c | 6283 +++++++++++ fs/btrfs/relocation.h | 52 + fs/btrfs/root-tree.c | 528 + fs/btrfs/root-tree.h | 42 + fs/btrfs/scrub.c | 3293 ++++++ fs/btrfs/scrub.h | 22 + fs/btrfs/send.c | 8297 +++++++++++++++ fs/btrfs/send.h | 187 + fs/btrfs/space-info.c | 2267 ++++ fs/btrfs/space-info.h | 355 + fs/btrfs/subpage.c | 993 ++ fs/btrfs/subpage.h | 225 + fs/btrfs/super.c | 2744 +++++ fs/btrfs/super.h | 38 + fs/btrfs/sysfs.c | 2702 +++++ fs/btrfs/sysfs.h | 57 + fs/btrfs/tests/btrfs-tests.c | 314 + fs/btrfs/tests/btrfs-tests.h | 83 + fs/btrfs/tests/chunk-allocation-tests.c | 476 + fs/btrfs/tests/delayed-refs-tests.c | 1016 ++ fs/btrfs/tests/extent-buffer-tests.c | 223 + fs/btrfs/tests/extent-io-tests.c | 830 ++ fs/btrfs/tests/extent-map-tests.c | 1202 +++ fs/btrfs/tests/free-space-tests.c | 1059 ++ fs/btrfs/tests/free-space-tree-tests.c | 586 ++ fs/btrfs/tests/inode-tests.c | 1095 ++ fs/btrfs/tests/qgroup-tests.c | 550 + fs/btrfs/tests/raid-stripe-tree-tests.c | 1161 ++ fs/btrfs/tests/zoned-tests.c | 675 ++ fs/btrfs/transaction.c | 2768 +++++ fs/btrfs/transaction.h | 348 + fs/btrfs/tree-checker.c | 2742 +++++ fs/btrfs/tree-checker.h | 83 + fs/btrfs/tree-log.c | 8201 +++++++++++++++ fs/btrfs/tree-log.h | 107 + fs/btrfs/tree-mod-log.c | 1146 ++ fs/btrfs/tree-mod-log.h | 58 + fs/btrfs/ulist.c | 298 + fs/btrfs/ulist.h | 77 + fs/btrfs/uuid-tree.c | 566 + fs/btrfs/uuid-tree.h | 21 + fs/btrfs/verity.c | 806 ++ fs/btrfs/verity.h | 35 + fs/btrfs/volumes.c | 9034 ++++++++++++++++ fs/btrfs/volumes.h | 953 ++ fs/btrfs/xattr.c | 527 + fs/btrfs/xattr.h | 31 + fs/btrfs/zlib.c | 514 + fs/btrfs/zoned.c | 3286 ++++++ fs/btrfs/zoned.h | 435 + fs/btrfs/zstd.c | 736 ++ fs/buffer.c | 3005 ++++++ fs/cachefiles/Kconfig | 28 + fs/cachefiles/Makefile | 20 + fs/cachefiles/cache.c | 428 + fs/cachefiles/daemon.c | 745 ++ fs/cachefiles/error_inject.c | 36 + fs/cachefiles/interface.c | 453 + fs/cachefiles/internal.h | 408 + fs/cachefiles/io.c | 738 ++ fs/cachefiles/key.c | 137 + fs/cachefiles/main.c | 93 + fs/cachefiles/namei.c | 824 ++ fs/cachefiles/security.c | 112 + fs/cachefiles/volume.c | 141 + fs/cachefiles/xattr.c | 304 + fs/ceph/Kconfig | 49 + fs/ceph/Makefile | 15 + fs/ceph/acl.c | 259 + fs/ceph/addr.c | 2644 +++++ fs/ceph/cache.c | 112 + fs/ceph/cache.h | 117 + fs/ceph/caps.c | 5126 +++++++++ fs/ceph/ceph_frag.c | 23 + fs/ceph/crypto.c | 605 ++ fs/ceph/crypto.h | 272 + fs/ceph/debugfs.c | 745 ++ fs/ceph/dir.c | 2279 ++++ fs/ceph/export.c | 618 ++ fs/ceph/file.c | 3260 ++++++ fs/ceph/inode.c | 3258 ++++++ fs/ceph/io.c | 213 + fs/ceph/io.h | 14 + fs/ceph/ioctl.c | 438 + fs/ceph/ioctl.h | 101 + fs/ceph/locks.c | 528 + fs/ceph/mds_client.c | 7367 +++++++++++++ fs/ceph/mds_client.h | 733 ++ fs/ceph/mdsmap.c | 462 + fs/ceph/mdsmap.h | 80 + fs/ceph/metric.c | 533 + fs/ceph/metric.h | 281 + fs/ceph/quota.c | 564 + fs/ceph/snap.c | 1343 +++ fs/ceph/strings.c | 130 + fs/ceph/subvolume_metrics.c | 416 + fs/ceph/subvolume_metrics.h | 97 + fs/ceph/super.c | 1716 +++ fs/ceph/super.h | 1481 +++ fs/ceph/util.c | 100 + fs/ceph/xattr.c | 1485 +++ fs/char_dev.c | 689 ++ fs/coda/Kconfig | 22 + fs/coda/Makefile | 14 + fs/coda/cache.c | 121 + fs/coda/cnode.c | 179 + fs/coda/coda_cache.h | 23 + fs/coda/coda_fs_i.h | 59 + fs/coda/coda_int.h | 31 + fs/coda/coda_linux.c | 208 + fs/coda/coda_linux.h | 95 + fs/coda/coda_psdev.h | 95 + fs/coda/dir.c | 578 + fs/coda/file.c | 331 + fs/coda/inode.c | 402 + fs/coda/pioctl.c | 88 + fs/coda/psdev.c | 438 + fs/coda/symlink.c | 40 + fs/coda/sysctl.c | 53 + fs/coda/upcall.c | 961 ++ fs/compat_binfmt_elf.c | 154 + fs/configfs/Kconfig | 11 + fs/configfs/Makefile | 8 + fs/configfs/configfs_internal.h | 158 + fs/configfs/dir.c | 1980 ++++ fs/configfs/file.c | 481 + fs/configfs/inode.c | 194 + fs/configfs/item.c | 190 + fs/configfs/mount.c | 179 + fs/configfs/symlink.c | 262 + fs/coredump.c | 1778 ++++ fs/cramfs/Kconfig | 54 + fs/cramfs/Makefile | 8 + fs/cramfs/README | 107 + fs/cramfs/inode.c | 1012 ++ fs/cramfs/internal.h | 4 + fs/cramfs/uncompress.c | 80 + fs/crypto/Kconfig | 40 + fs/crypto/Makefile | 13 + fs/crypto/block.c | 415 + fs/crypto/crypto.c | 346 + fs/crypto/fname.c | 514 + fs/crypto/fscrypt_private.h | 783 ++ fs/crypto/hkdf.c | 100 + fs/crypto/hooks.c | 473 + fs/crypto/keyring.c | 1238 +++ fs/crypto/keysetup.c | 850 ++ fs/crypto/keysetup_v1.c | 291 + fs/crypto/policy.c | 899 ++ fs/d_path.c | 450 + fs/dax.c | 2278 ++++ fs/dcache.c | 3525 +++++++ fs/debugfs/Makefile | 5 + fs/debugfs/file.c | 1436 +++ fs/debugfs/inode.c | 932 ++ fs/debugfs/internal.h | 58 + fs/devpts/Makefile | 8 + fs/devpts/inode.c | 572 + fs/direct-io.c | 1316 +++ fs/dlm/Kconfig | 16 + fs/dlm/Makefile | 21 + fs/dlm/ast.c | 275 + fs/dlm/ast.h | 29 + fs/dlm/config.c | 1053 ++ fs/dlm/config.h | 60 + fs/dlm/debug_fs.c | 825 ++ fs/dlm/dir.c | 400 + fs/dlm/dir.h | 24 + fs/dlm/dlm_internal.h | 833 ++ fs/dlm/lock.c | 6338 +++++++++++ fs/dlm/lock.h | 85 + fs/dlm/lockspace.c | 841 ++ fs/dlm/lockspace.h | 37 + fs/dlm/lowcomms.c | 1987 ++++ fs/dlm/lowcomms.h | 55 + fs/dlm/lvb_table.h | 16 + fs/dlm/main.c | 105 + fs/dlm/member.c | 772 ++ fs/dlm/member.h | 31 + fs/dlm/memory.c | 179 + fs/dlm/memory.h | 33 + fs/dlm/midcomms.c | 1514 +++ fs/dlm/midcomms.h | 42 + fs/dlm/plock.c | 630 ++ fs/dlm/rcom.c | 691 ++ fs/dlm/rcom.h | 26 + fs/dlm/recover.c | 917 ++ fs/dlm/recover.h | 32 + fs/dlm/recoverd.c | 468 + fs/dlm/recoverd.h | 21 + fs/dlm/requestqueue.c | 149 + fs/dlm/requestqueue.h | 21 + fs/dlm/user.c | 989 ++ fs/dlm/user.h | 17 + fs/dlm/util.c | 66 + fs/dlm/util.h | 18 + fs/drop_caches.c | 98 + fs/ecryptfs/Kconfig | 23 + fs/ecryptfs/Makefile | 11 + fs/ecryptfs/crypto.c | 1975 ++++ fs/ecryptfs/debug.c | 89 + fs/ecryptfs/dentry.c | 76 + fs/ecryptfs/ecryptfs_kernel.h | 697 ++ fs/ecryptfs/file.c | 442 + fs/ecryptfs/inode.c | 1183 +++ fs/ecryptfs/keystore.c | 2516 +++++ fs/ecryptfs/kthread.c | 159 + fs/ecryptfs/main.c | 895 ++ fs/ecryptfs/messaging.c | 461 + fs/ecryptfs/miscdev.c | 499 + fs/ecryptfs/mmap.c | 514 + fs/ecryptfs/read_write.c | 263 + fs/ecryptfs/super.c | 181 + fs/efivarfs/Kconfig | 14 + fs/efivarfs/Makefile | 8 + fs/efivarfs/file.c | 154 + fs/efivarfs/inode.c | 192 + fs/efivarfs/internal.h | 67 + fs/efivarfs/super.c | 571 + fs/efivarfs/vars.c | 633 ++ fs/erofs/Kconfig | 212 + fs/erofs/Makefile | 12 + fs/erofs/compress.h | 89 + fs/erofs/data.c | 534 + fs/erofs/decompressor.c | 517 + fs/erofs/decompressor_crypto.c | 182 + fs/erofs/decompressor_deflate.c | 203 + fs/erofs/decompressor_lzma.c | 236 + fs/erofs/decompressor_zstd.c | 220 + fs/erofs/dir.c | 134 + fs/erofs/erofs_fs.h | 469 + fs/erofs/fileio.c | 217 + fs/erofs/inode.c | 399 + fs/erofs/internal.h | 539 + fs/erofs/ishare.c | 214 + fs/erofs/namei.c | 224 + fs/erofs/super.c | 1146 ++ fs/erofs/sysfs.c | 302 + fs/erofs/xattr.c | 649 ++ fs/erofs/xattr.h | 36 + fs/erofs/zdata.c | 1935 ++++ fs/erofs/zmap.c | 838 ++ fs/erofs/zutil.c | 316 + fs/eventfd.c | 423 + fs/eventpoll.c | 3030 ++++++ fs/exec.c | 2098 ++++ fs/exfat/Kconfig | 24 + fs/exfat/Makefile | 8 + fs/exfat/balloc.c | 406 + fs/exfat/cache.c | 357 + fs/exfat/dir.c | 1381 +++ fs/exfat/exfat_fs.h | 704 ++ fs/exfat/exfat_raw.h | 197 + fs/exfat/fatent.c | 585 ++ fs/exfat/file.c | 1083 ++ fs/exfat/inode.c | 459 + fs/exfat/iomap.c | 279 + fs/exfat/iomap.h | 18 + fs/exfat/misc.c | 207 + fs/exfat/namei.c | 1351 +++ fs/exfat/nls.c | 809 ++ fs/exfat/super.c | 943 ++ fs/exportfs/Makefile | 7 + fs/exportfs/expfs.c | 610 ++ fs/ext2/Kconfig | 50 + fs/ext2/Makefile | 16 + fs/ext2/acl.c | 276 + fs/ext2/acl.h | 73 + fs/ext2/balloc.c | 1535 +++ fs/ext2/dir.c | 739 ++ fs/ext2/ext2.h | 815 ++ fs/ext2/file.c | 217 + fs/ext2/ialloc.c | 673 ++ fs/ext2/inode.c | 1652 +++ fs/ext2/ioctl.c | 159 + fs/ext2/namei.c | 434 + fs/ext2/super.c | 1701 +++ fs/ext2/symlink.c | 36 + fs/ext2/trace.c | 6 + fs/ext2/trace.h | 94 + fs/ext2/xattr.c | 1055 ++ fs/ext2/xattr.h | 116 + fs/ext2/xattr_security.c | 59 + fs/ext2/xattr_trusted.c | 43 + fs/ext2/xattr_user.c | 50 + fs/ext4/.kunitconfig | 3 + fs/ext4/Kconfig | 92 + fs/ext4/Makefile | 21 + fs/ext4/acl.c | 304 + fs/ext4/acl.h | 74 + fs/ext4/balloc.c | 1007 ++ fs/ext4/bitmap.c | 99 + fs/ext4/block_validity.c | 370 + fs/ext4/crypto.c | 248 + fs/ext4/dir.c | 710 ++ fs/ext4/ext4.h | 4078 ++++++++ fs/ext4/ext4_extents.h | 280 + fs/ext4/ext4_jbd2.c | 433 + fs/ext4/ext4_jbd2.h | 457 + fs/ext4/extents-test.c | 1064 ++ fs/ext4/extents.c | 6420 ++++++++++++ fs/ext4/extents_status.c | 2392 +++++ fs/ext4/extents_status.h | 254 + fs/ext4/fast_commit.c | 2766 +++++ fs/ext4/fast_commit.h | 189 + fs/ext4/file.c | 1068 ++ fs/ext4/fsmap.c | 792 ++ fs/ext4/fsmap.h | 56 + fs/ext4/fsync.c | 176 + fs/ext4/hash-test.c | 567 + fs/ext4/hash.c | 347 + fs/ext4/ialloc.c | 1632 +++ fs/ext4/indirect.c | 1474 +++ fs/ext4/inline.c | 2018 ++++ fs/ext4/inode-test.c | 283 + fs/ext4/inode.c | 6970 ++++++++++++ fs/ext4/ioctl.c | 2028 ++++ fs/ext4/mballoc-test.c | 1008 ++ fs/ext4/mballoc.c | 7284 +++++++++++++ fs/ext4/mballoc.h | 303 + fs/ext4/migrate.c | 685 ++ fs/ext4/mmp.c | 403 + fs/ext4/move_extent.c | 658 ++ fs/ext4/namei.c | 4251 ++++++++ fs/ext4/orphan.c | 662 ++ fs/ext4/page-io.c | 549 + fs/ext4/readpage.c | 386 + fs/ext4/resize.c | 2192 ++++ fs/ext4/super.c | 7641 ++++++++++++++ fs/ext4/symlink.c | 136 + fs/ext4/sysfs.c | 698 ++ fs/ext4/truncate.h | 52 + fs/ext4/verity.c | 391 + fs/ext4/xattr.c | 3238 ++++++ fs/ext4/xattr.h | 231 + fs/ext4/xattr_hurd.c | 52 + fs/ext4/xattr_security.c | 66 + fs/ext4/xattr_trusted.c | 47 + fs/ext4/xattr_user.c | 50 + fs/f2fs/Kconfig | 152 + fs/f2fs/Makefile | 12 + fs/f2fs/acl.c | 455 + fs/f2fs/acl.h | 51 + fs/f2fs/checkpoint.c | 2207 ++++ fs/f2fs/compress.c | 2090 ++++ fs/f2fs/data.c | 4585 ++++++++ fs/f2fs/debug.c | 864 ++ fs/f2fs/dir.c | 1148 ++ fs/f2fs/extent_cache.c | 1253 +++ fs/f2fs/f2fs.h | 5160 +++++++++ fs/f2fs/file.c | 5558 ++++++++++ fs/f2fs/gc.c | 2461 +++++ fs/f2fs/gc.h | 202 + fs/f2fs/hash.c | 137 + fs/f2fs/inline.c | 860 ++ fs/f2fs/inode.c | 1081 ++ fs/f2fs/iostat.c | 351 + fs/f2fs/iostat.h | 89 + fs/f2fs/namei.c | 1402 +++ fs/f2fs/node.c | 3583 +++++++ fs/f2fs/node.h | 433 + fs/f2fs/recovery.c | 970 ++ fs/f2fs/segment.c | 5901 +++++++++++ fs/f2fs/segment.h | 1030 ++ fs/f2fs/shrinker.c | 246 + fs/f2fs/super.c | 5775 ++++++++++ fs/f2fs/sysfs.c | 2112 ++++ fs/f2fs/verity.c | 289 + fs/f2fs/xattr.c | 851 ++ fs/f2fs/xattr.h | 169 + fs/failfs.c | 166 + fs/fat/.kunitconfig | 5 + fs/fat/Kconfig | 131 + fs/fat/Makefile | 14 + fs/fat/cache.c | 381 + fs/fat/dir.c | 1467 +++ fs/fat/fat.h | 492 + fs/fat/fat_test.c | 395 + fs/fat/fatent.c | 853 ++ fs/fat/file.c | 614 ++ fs/fat/inode.c | 2025 ++++ fs/fat/misc.c | 367 + fs/fat/namei_msdos.c | 720 ++ fs/fat/namei_vfat.c | 1261 +++ fs/fat/nfs.c | 301 + fs/fcntl.c | 1183 +++ fs/fhandle.c | 474 + fs/file.c | 1531 +++ fs/file_attr.c | 486 + fs/file_table.c | 667 ++ fs/filesystems.c | 413 + fs/fs-writeback.c | 3102 ++++++ fs/fs_context.c | 568 + fs/fs_dirent.c | 105 + fs/fs_parser.c | 390 + fs/fs_pin.c | 97 + fs/fs_struct.c | 252 + fs/fserror.c | 194 + fs/fsopen.c | 491 + fs/fuse/Kconfig | 78 + fs/fuse/Makefile | 22 + fs/fuse/acl.c | 161 + fs/fuse/args.h | 65 + fs/fuse/backing.c | 180 + fs/fuse/control.c | 366 + fs/fuse/cuse.c | 663 ++ fs/fuse/dax.c | 1356 +++ fs/fuse/dev.c | 2430 +++++ fs/fuse/dev.h | 110 + fs/fuse/dev_uring.c | 1456 +++ fs/fuse/dev_uring_i.h | 202 + fs/fuse/dir.c | 2497 +++++ fs/fuse/file.c | 3120 ++++++ fs/fuse/fuse_dev_i.h | 422 + fs/fuse/fuse_i.h | 1332 +++ fs/fuse/fuse_trace.h | 154 + fs/fuse/inode.c | 2189 ++++ fs/fuse/ioctl.c | 577 + fs/fuse/iomode.c | 275 + fs/fuse/notify.c | 446 + fs/fuse/passthrough.c | 197 + fs/fuse/poll.c | 141 + fs/fuse/readdir.c | 656 ++ fs/fuse/req.c | 99 + fs/fuse/req_timeout.c | 148 + fs/fuse/sysctl.c | 65 + fs/fuse/sysctl.h | 9 + fs/fuse/trace.c | 13 + fs/fuse/virtio_fs.c | 1854 ++++ fs/fuse/xattr.c | 214 + fs/gfs2/Kconfig | 36 + fs/gfs2/Makefile | 11 + fs/gfs2/acl.c | 148 + fs/gfs2/acl.h | 19 + fs/gfs2/aops.c | 743 ++ fs/gfs2/aops.h | 14 + fs/gfs2/bmap.c | 2530 +++++ fs/gfs2/bmap.h | 71 + fs/gfs2/dentry.c | 103 + fs/gfs2/dir.c | 2187 ++++ fs/gfs2/dir.h | 82 + fs/gfs2/export.c | 194 + fs/gfs2/file.c | 1651 +++ fs/gfs2/gfs2.h | 23 + fs/gfs2/glock.c | 2664 +++++ fs/gfs2/glock.h | 309 + fs/gfs2/glops.c | 710 ++ fs/gfs2/glops.h | 28 + fs/gfs2/incore.h | 883 ++ fs/gfs2/inode.c | 2338 +++++ fs/gfs2/inode.h | 135 + fs/gfs2/lock_dlm.c | 1459 +++ fs/gfs2/log.c | 1385 +++ fs/gfs2/log.h | 63 + fs/gfs2/lops.c | 1117 ++ fs/gfs2/lops.h | 91 + fs/gfs2/main.c | 261 + fs/gfs2/meta_io.c | 516 + fs/gfs2/meta_io.h | 78 + fs/gfs2/ops_fstype.c | 1817 ++++ fs/gfs2/quota.c | 1829 ++++ fs/gfs2/quota.h | 68 + fs/gfs2/recovery.c | 594 ++ fs/gfs2/recovery.h | 36 + fs/gfs2/rgrp.c | 2769 +++++ fs/gfs2/rgrp.h | 97 + fs/gfs2/super.c | 1584 +++ fs/gfs2/super.h | 66 + fs/gfs2/sys.c | 795 ++ fs/gfs2/sys.h | 22 + fs/gfs2/trace_gfs2.h | 647 ++ fs/gfs2/trans.c | 362 + fs/gfs2/trans.h | 52 + fs/gfs2/util.c | 436 + fs/gfs2/util.h | 199 + fs/gfs2/xattr.c | 1512 +++ fs/gfs2/xattr.h | 63 + fs/hfs/.kunitconfig | 7 + fs/hfs/Kconfig | 30 + fs/hfs/Makefile | 13 + fs/hfs/attr.c | 153 + fs/hfs/bfind.c | 246 + fs/hfs/bitmap.c | 243 + fs/hfs/bnode.c | 542 + fs/hfs/brec.c | 548 + fs/hfs/btree.c | 544 + fs/hfs/btree.h | 171 + fs/hfs/catalog.c | 536 + fs/hfs/dir.c | 331 + fs/hfs/extent.c | 552 + fs/hfs/hfs.h | 21 + fs/hfs/hfs_fs.h | 319 + fs/hfs/inode.c | 752 ++ fs/hfs/mdb.c | 462 + fs/hfs/part_tbl.c | 117 + fs/hfs/string.c | 119 + fs/hfs/string_test.c | 133 + fs/hfs/super.c | 503 + fs/hfs/sysdep.c | 50 + fs/hfs/trans.c | 150 + fs/hfsplus/.kunitconfig | 8 + fs/hfsplus/Kconfig | 31 + fs/hfsplus/Makefile | 13 + fs/hfsplus/attributes.c | 499 + fs/hfsplus/bfind.c | 343 + fs/hfsplus/bitmap.c | 246 + fs/hfsplus/bnode.c | 720 ++ fs/hfsplus/brec.c | 563 + fs/hfsplus/btree.c | 679 ++ fs/hfsplus/catalog.c | 540 + fs/hfsplus/dir.c | 643 ++ fs/hfsplus/extents.c | 636 ++ fs/hfsplus/hfsplus_fs.h | 706 ++ fs/hfsplus/hfsplus_raw.h | 20 + fs/hfsplus/inode.c | 842 ++ fs/hfsplus/ioctl.c | 70 + fs/hfsplus/options.c | 169 + fs/hfsplus/part_tbl.c | 157 + fs/hfsplus/super.c | 803 ++ fs/hfsplus/tables.c | 3246 ++++++ fs/hfsplus/unicode.c | 628 ++ fs/hfsplus/unicode_test.c | 1595 +++ fs/hfsplus/wrapper.c | 240 + fs/hfsplus/xattr.c | 1036 ++ fs/hfsplus/xattr.h | 41 + fs/hfsplus/xattr_security.c | 79 + fs/hfsplus/xattr_trusted.c | 38 + fs/hfsplus/xattr_user.c | 38 + fs/hostfs/Makefile | 13 + fs/hostfs/hostfs.h | 73 + fs/hostfs/hostfs_kern.c | 1096 ++ fs/hostfs/hostfs_user.c | 421 + fs/hostfs/hostfs_user_exp.c | 28 + fs/hpfs/Kconfig | 17 + fs/hpfs/Makefile | 9 + fs/hpfs/alloc.c | 582 ++ fs/hpfs/anode.c | 498 + fs/hpfs/buffer.c | 232 + fs/hpfs/dentry.c | 62 + fs/hpfs/dir.c | 331 + fs/hpfs/dnode.c | 1095 ++ fs/hpfs/ea.c | 368 + fs/hpfs/file.c | 276 + fs/hpfs/hpfs.h | 579 + fs/hpfs/hpfs_fn.h | 379 + fs/hpfs/inode.c | 318 + fs/hpfs/map.c | 335 + fs/hpfs/name.c | 114 + fs/hpfs/namei.c | 621 ++ fs/hpfs/super.c | 789 ++ fs/hugetlbfs/Makefile | 8 + fs/hugetlbfs/inode.c | 1628 +++ fs/init.c | 218 + fs/inode.c | 3116 ++++++ fs/internal.h | 369 + fs/ioctl.c | 702 ++ fs/iomap/Makefile | 19 + fs/iomap/bio.c | 189 + fs/iomap/buffered-io.c | 2102 ++++ fs/iomap/direct-io.c | 1088 ++ fs/iomap/fiemap.c | 120 + fs/iomap/internal.h | 35 + fs/iomap/ioend.c | 536 + fs/iomap/iter.c | 134 + fs/iomap/seek.c | 100 + fs/iomap/swapfile.c | 192 + fs/iomap/trace.c | 12 + fs/iomap/trace.h | 333 + fs/isofs/Kconfig | 41 + fs/isofs/Makefile | 10 + fs/isofs/compress.c | 386 + fs/isofs/dir.c | 280 + fs/isofs/export.c | 201 + fs/isofs/inode.c | 1637 +++ fs/isofs/isofs.h | 209 + fs/isofs/joliet.c | 68 + fs/isofs/namei.c | 166 + fs/isofs/rock.c | 830 ++ fs/isofs/rock.h | 119 + fs/isofs/util.c | 86 + fs/isofs/zisofs.h | 16 + fs/jbd2/Kconfig | 34 + fs/jbd2/Makefile | 8 + fs/jbd2/checkpoint.c | 731 ++ fs/jbd2/commit.c | 1186 +++ fs/jbd2/journal.c | 3066 ++++++ fs/jbd2/recovery.c | 996 ++ fs/jbd2/revoke.c | 745 ++ fs/jbd2/transaction.c | 2836 +++++ fs/jffs2/Kconfig | 186 + fs/jffs2/LICENCE | 30 + fs/jffs2/Makefile | 22 + fs/jffs2/README.Locking | 169 + fs/jffs2/acl.c | 312 + fs/jffs2/acl.h | 43 + fs/jffs2/background.c | 165 + fs/jffs2/build.c | 434 + fs/jffs2/compr.c | 426 + fs/jffs2/compr.h | 119 + fs/jffs2/compr_lzo.c | 110 + fs/jffs2/compr_rtime.c | 136 + fs/jffs2/compr_rubin.c | 447 + fs/jffs2/compr_zlib.c | 222 + fs/jffs2/debug.c | 866 ++ fs/jffs2/debug.h | 276 + fs/jffs2/dir.c | 889 ++ fs/jffs2/erase.c | 487 + fs/jffs2/file.c | 336 + fs/jffs2/fs.c | 742 ++ fs/jffs2/gc.c | 1405 +++ fs/jffs2/ioctl.c | 22 + fs/jffs2/jffs2_fs_i.h | 56 + fs/jffs2/jffs2_fs_sb.h | 163 + fs/jffs2/malloc.c | 299 + fs/jffs2/nodelist.c | 755 ++ fs/jffs2/nodelist.h | 484 + fs/jffs2/nodemgmt.c | 890 ++ fs/jffs2/os-linux.h | 197 + fs/jffs2/read.c | 228 + fs/jffs2/readinode.c | 1447 +++ fs/jffs2/scan.c | 1184 +++ fs/jffs2/security.c | 73 + fs/jffs2/summary.c | 882 ++ fs/jffs2/summary.h | 213 + fs/jffs2/super.c | 443 + fs/jffs2/symlink.c | 19 + fs/jffs2/wbuf.c | 1350 +++ fs/jffs2/write.c | 723 ++ fs/jffs2/writev.c | 51 + fs/jffs2/xattr.c | 1353 +++ fs/jffs2/xattr.h | 127 + fs/jffs2/xattr_trusted.c | 47 + fs/jffs2/xattr_user.c | 41 + fs/jfs/Kconfig | 51 + fs/jfs/Makefile | 15 + fs/jfs/acl.c | 153 + fs/jfs/file.c | 158 + fs/jfs/inode.c | 422 + fs/jfs/ioctl.c | 145 + fs/jfs/jfs_acl.h | 24 + fs/jfs/jfs_btree.h | 159 + fs/jfs/jfs_debug.c | 80 + fs/jfs/jfs_debug.h | 109 + fs/jfs/jfs_dinode.h | 169 + fs/jfs/jfs_discard.c | 114 + fs/jfs/jfs_discard.h | 13 + fs/jfs/jfs_dmap.c | 4182 ++++++++ fs/jfs/jfs_dmap.h | 303 + fs/jfs/jfs_dtree.c | 4483 ++++++++ fs/jfs/jfs_dtree.h | 260 + fs/jfs/jfs_extent.c | 398 + fs/jfs/jfs_extent.h | 16 + fs/jfs/jfs_filsys.h | 276 + fs/jfs/jfs_imap.c | 3178 ++++++ fs/jfs/jfs_imap.h | 162 + fs/jfs/jfs_incore.h | 226 + fs/jfs/jfs_inode.c | 134 + fs/jfs/jfs_inode.h | 41 + fs/jfs/jfs_lock.h | 39 + fs/jfs/jfs_logmgr.c | 2485 +++++ fs/jfs/jfs_logmgr.h | 507 + fs/jfs/jfs_metapage.c | 953 ++ fs/jfs/jfs_metapage.h | 142 + fs/jfs/jfs_mount.c | 502 + fs/jfs/jfs_superblock.h | 109 + fs/jfs/jfs_txnmgr.c | 3021 ++++++ fs/jfs/jfs_txnmgr.h | 298 + fs/jfs/jfs_types.h | 157 + fs/jfs/jfs_umount.c | 163 + fs/jfs/jfs_unicode.c | 125 + fs/jfs/jfs_unicode.h | 136 + fs/jfs/jfs_xattr.h | 62 + fs/jfs/jfs_xtree.c | 2930 ++++++ fs/jfs/jfs_xtree.h | 113 + fs/jfs/namei.c | 1621 +++ fs/jfs/resize.c | 536 + fs/jfs/super.c | 1069 ++ fs/jfs/symlink.c | 22 + fs/jfs/xattr.c | 1055 ++ fs/kernel_read_file.c | 180 + fs/kernfs/Kconfig | 8 + fs/kernfs/Makefile | 6 + fs/kernfs/dir.c | 2015 ++++ fs/kernfs/file.c | 1085 ++ fs/kernfs/inode.c | 406 + fs/kernfs/kernfs-internal.h | 236 + fs/kernfs/mount.c | 472 + fs/kernfs/symlink.c | 155 + fs/libfs.c | 2317 ++++ fs/lockd/Makefile | 52 + fs/lockd/clnt4xdr.c | 582 ++ fs/lockd/clntlock.c | 297 + fs/lockd/clntproc.c | 889 ++ fs/lockd/clntxdr.c | 614 ++ fs/lockd/host.c | 723 ++ fs/lockd/lockd.h | 483 + fs/lockd/mon.c | 581 + fs/lockd/netlink.c | 45 + fs/lockd/netlink.h | 20 + fs/lockd/netns.h | 25 + fs/lockd/nlm.h | 56 + fs/lockd/nlm3xdr_gen.c | 686 ++ fs/lockd/nlm3xdr_gen.h | 30 + fs/lockd/nlm4xdr_gen.c | 739 ++ fs/lockd/nlm4xdr_gen.h | 30 + fs/lockd/procfs.c | 92 + fs/lockd/procfs.h | 27 + fs/lockd/share.h | 39 + fs/lockd/svc.c | 791 ++ fs/lockd/svc4proc.c | 1432 +++ fs/lockd/svclock.c | 1042 ++ fs/lockd/svcproc.c | 1453 +++ fs/lockd/svcshare.c | 151 + fs/lockd/svcsubs.c | 578 + fs/lockd/trace.c | 3 + fs/lockd/trace.h | 107 + fs/lockd/xdr.h | 85 + fs/locks.c | 3163 ++++++ fs/mbcache.c | 444 + fs/minix/Kconfig | 27 + fs/minix/Makefile | 8 + fs/minix/bitmap.c | 269 + fs/minix/dir.c | 458 + fs/minix/file.c | 54 + fs/minix/inode.c | 789 ++ fs/minix/itree_common.c | 374 + fs/minix/itree_v1.c | 67 + fs/minix/itree_v2.c | 75 + fs/minix/minix.h | 180 + fs/minix/namei.c | 291 + fs/mnt_idmapping.c | 389 + fs/mount.h | 278 + fs/mpage.c | 693 ++ fs/namei.c | 6581 ++++++++++++ fs/namespace.c | 6566 ++++++++++++ fs/netfs/Kconfig | 60 + fs/netfs/Makefile | 36 + fs/netfs/buffered_read.c | 840 ++ fs/netfs/buffered_write.c | 616 ++ fs/netfs/direct_read.c | 255 + fs/netfs/direct_write.c | 387 + fs/netfs/fscache_cache.c | 429 + fs/netfs/fscache_cookie.c | 1184 +++ fs/netfs/fscache_internal.h | 14 + fs/netfs/fscache_io.c | 290 + fs/netfs/fscache_main.c | 109 + fs/netfs/fscache_proc.c | 47 + fs/netfs/fscache_stats.c | 103 + fs/netfs/fscache_volume.c | 532 + fs/netfs/internal.h | 553 + fs/netfs/iterator.c | 308 + fs/netfs/locking.c | 300 + fs/netfs/main.c | 183 + fs/netfs/misc.c | 565 + fs/netfs/objects.c | 260 + fs/netfs/read_collect.c | 608 ++ fs/netfs/read_pgpriv2.c | 233 + fs/netfs/read_retry.c | 312 + fs/netfs/read_single.c | 191 + fs/netfs/rolling_buffer.c | 226 + fs/netfs/stats.c | 103 + fs/netfs/write_collect.c | 525 + fs/netfs/write_issue.c | 906 ++ fs/netfs/write_retry.c | 238 + fs/nfs/Kconfig | 217 + fs/nfs/Makefile | 38 + fs/nfs/blocklayout/Makefile | 7 + fs/nfs/blocklayout/blocklayout.c | 1047 ++ fs/nfs/blocklayout/blocklayout.h | 204 + fs/nfs/blocklayout/dev.c | 599 ++ fs/nfs/blocklayout/extent_tree.c | 726 ++ fs/nfs/blocklayout/rpc_pipefs.c | 270 + fs/nfs/cache_lib.c | 158 + fs/nfs/cache_lib.h | 32 + fs/nfs/callback.c | 355 + fs/nfs/callback.h | 204 + fs/nfs/callback_proc.c | 756 ++ fs/nfs/callback_xdr.c | 1111 ++ fs/nfs/client.c | 1542 +++ fs/nfs/delegation.c | 1619 +++ fs/nfs/delegation.h | 141 + fs/nfs/dir.c | 3433 ++++++ fs/nfs/direct.c | 1112 ++ fs/nfs/dns_resolve.c | 480 + fs/nfs/dns_resolve.h | 37 + fs/nfs/export.c | 166 + fs/nfs/file.c | 982 ++ fs/nfs/filelayout/Makefile | 6 + fs/nfs/filelayout/filelayout.c | 1155 ++ fs/nfs/filelayout/filelayout.h | 118 + fs/nfs/filelayout/filelayoutdev.c | 305 + fs/nfs/flexfilelayout/Makefile | 6 + fs/nfs/flexfilelayout/flexfilelayout.c | 3069 ++++++ fs/nfs/flexfilelayout/flexfilelayout.h | 269 + fs/nfs/flexfilelayout/flexfilelayoutdev.c | 644 ++ fs/nfs/fs_context.c | 1793 ++++ fs/nfs/fscache.c | 387 + fs/nfs/fscache.h | 202 + fs/nfs/getroot.c | 133 + fs/nfs/inode.c | 2793 +++++ fs/nfs/internal.h | 1012 ++ fs/nfs/io.c | 204 + fs/nfs/iostat.h | 61 + fs/nfs/localio.c | 1120 ++ fs/nfs/mount_clnt.c | 471 + fs/nfs/namespace.c | 386 + fs/nfs/netns.h | 48 + fs/nfs/nfs.h | 30 + fs/nfs/nfs2super.c | 33 + fs/nfs/nfs2xdr.c | 1088 ++ fs/nfs/nfs3_fs.h | 37 + fs/nfs/nfs3acl.c | 343 + fs/nfs/nfs3client.c | 144 + fs/nfs/nfs3proc.c | 1128 ++ fs/nfs/nfs3super.c | 34 + fs/nfs/nfs3xdr.c | 2516 +++++ fs/nfs/nfs40.h | 19 + fs/nfs/nfs40client.c | 245 + fs/nfs/nfs40proc.c | 395 + fs/nfs/nfs42.h | 69 + fs/nfs/nfs42proc.c | 1687 +++ fs/nfs/nfs42xattr.c | 1067 ++ fs/nfs/nfs42xdr.c | 1830 ++++ fs/nfs/nfs4_fs.h | 682 ++ fs/nfs/nfs4client.c | 1203 +++ fs/nfs/nfs4file.c | 459 + fs/nfs/nfs4getroot.c | 39 + fs/nfs/nfs4idmap.c | 793 ++ fs/nfs/nfs4idmap.h | 68 + fs/nfs/nfs4namespace.c | 564 + fs/nfs/nfs4proc.c | 10797 +++++++++++++++++++ fs/nfs/nfs4renewd.c | 160 + fs/nfs/nfs4session.c | 653 ++ fs/nfs/nfs4session.h | 158 + fs/nfs/nfs4state.c | 2713 +++++ fs/nfs/nfs4super.c | 317 + fs/nfs/nfs4sysctl.c | 52 + fs/nfs/nfs4trace.c | 40 + fs/nfs/nfs4trace.h | 2859 +++++ fs/nfs/nfs4xdr.c | 7786 ++++++++++++++ fs/nfs/nfsroot.c | 316 + fs/nfs/nfstrace.c | 15 + fs/nfs/nfstrace.h | 1986 ++++ fs/nfs/pagelist.c | 1496 +++ fs/nfs/pnfs.c | 3535 +++++++ fs/nfs/pnfs.h | 928 ++ fs/nfs/pnfs_dev.c | 381 + fs/nfs/pnfs_nfs.c | 1225 +++ fs/nfs/proc.c | 781 ++ fs/nfs/read.c | 499 + fs/nfs/super.c | 1463 +++ fs/nfs/symlink.c | 80 + fs/nfs/sysctl.c | 46 + fs/nfs/sysfs.c | 474 + fs/nfs/sysfs.h | 33 + fs/nfs/unlink.c | 532 + fs/nfs/write.c | 2212 ++++ fs/nfs_common/Makefile | 16 + fs/nfs_common/common.c | 200 + fs/nfs_common/grace.c | 145 + fs/nfs_common/localio_trace.c | 10 + fs/nfs_common/localio_trace.h | 56 + fs/nfs_common/nfs_ssc.c | 91 + fs/nfs_common/nfsacl.c | 423 + fs/nfs_common/nfslocalio.c | 387 + fs/nfsd/Kconfig | 197 + fs/nfsd/Makefile | 49 + fs/nfsd/acl.h | 54 + fs/nfsd/attr4.h | 162 + fs/nfsd/auth.c | 108 + fs/nfsd/auth.h | 23 + fs/nfsd/blocklayout.c | 526 + fs/nfsd/blocklayoutxdr.c | 295 + fs/nfsd/blocklayoutxdr.h | 76 + fs/nfsd/cache.h | 92 + fs/nfsd/current_stateid.h | 41 + fs/nfsd/debugfs.c | 147 + fs/nfsd/export.c | 2252 ++++ fs/nfsd/export.h | 139 + fs/nfsd/filecache.c | 1565 +++ fs/nfsd/filecache.h | 92 + fs/nfsd/flexfilelayout.c | 144 + fs/nfsd/flexfilelayoutxdr.c | 130 + fs/nfsd/flexfilelayoutxdr.h | 50 + fs/nfsd/idmap.h | 60 + fs/nfsd/localio.c | 220 + fs/nfsd/lockd.c | 112 + fs/nfsd/netlink.c | 250 + fs/nfsd/netlink.h | 55 + fs/nfsd/netns.h | 268 + fs/nfsd/nfs2acl.c | 397 + fs/nfsd/nfs3acl.c | 288 + fs/nfsd/nfs3proc.c | 1117 ++ fs/nfsd/nfs3xdr.c | 1356 +++ fs/nfsd/nfs4acl.c | 884 ++ fs/nfsd/nfs4callback.c | 2018 ++++ fs/nfsd/nfs4idmap.c | 733 ++ fs/nfsd/nfs4layouts.c | 967 ++ fs/nfsd/nfs4proc.c | 4273 ++++++++ fs/nfsd/nfs4recover.c | 2060 ++++ fs/nfsd/nfs4state.c | 10554 +++++++++++++++++++ fs/nfsd/nfs4xdr.c | 6810 ++++++++++++ fs/nfsd/nfs4xdr_gen.c | 1159 ++ fs/nfsd/nfs4xdr_gen.h | 53 + fs/nfsd/nfscache.c | 669 ++ fs/nfsd/nfsctl.c | 2942 ++++++ fs/nfsd/nfsd.h | 392 + fs/nfsd/nfsfh.c | 972 ++ fs/nfsd/nfsfh.h | 342 + fs/nfsd/nfsproc.c | 854 ++ fs/nfsd/nfssvc.c | 1095 ++ fs/nfsd/nfsxdr.c | 677 ++ fs/nfsd/pnfs.h | 110 + fs/nfsd/state.h | 991 ++ fs/nfsd/stats.c | 88 + fs/nfsd/stats.h | 140 + fs/nfsd/trace.c | 4 + fs/nfsd/trace.h | 2780 +++++ fs/nfsd/vfs.c | 2998 ++++++ fs/nfsd/vfs.h | 171 + fs/nfsd/xdr.h | 178 + fs/nfsd/xdr3.h | 315 + fs/nfsd/xdr4.h | 1098 ++ fs/nfsd/xdr4cb.h | 93 + fs/nilfs2/Kconfig | 27 + fs/nilfs2/Makefile | 6 + fs/nilfs2/alloc.c | 958 ++ fs/nilfs2/alloc.h | 99 + fs/nilfs2/bmap.c | 568 + fs/nilfs2/bmap.h | 283 + fs/nilfs2/btnode.c | 356 + fs/nilfs2/btnode.h | 47 + fs/nilfs2/btree.c | 2485 +++++ fs/nilfs2/btree.h | 65 + fs/nilfs2/cpfile.c | 1184 +++ fs/nilfs2/cpfile.h | 36 + fs/nilfs2/dat.c | 541 + fs/nilfs2/dat.h | 47 + fs/nilfs2/dir.c | 665 ++ fs/nilfs2/direct.c | 387 + fs/nilfs2/direct.h | 28 + fs/nilfs2/export.h | 26 + fs/nilfs2/file.c | 165 + fs/nilfs2/gcinode.c | 184 + fs/nilfs2/ifile.c | 215 + fs/nilfs2/ifile.h | 45 + fs/nilfs2/inode.c | 1229 +++ fs/nilfs2/ioctl.c | 1415 +++ fs/nilfs2/mdt.c | 678 ++ fs/nilfs2/mdt.h | 115 + fs/nilfs2/namei.c | 594 ++ fs/nilfs2/nilfs.h | 415 + fs/nilfs2/page.c | 579 + fs/nilfs2/page.h | 55 + fs/nilfs2/recovery.c | 1016 ++ fs/nilfs2/segbuf.c | 472 + fs/nilfs2/segbuf.h | 172 + fs/nilfs2/segment.c | 2929 ++++++ fs/nilfs2/segment.h | 244 + fs/nilfs2/sufile.c | 1273 +++ fs/nilfs2/sufile.h | 162 + fs/nilfs2/super.c | 1413 +++ fs/nilfs2/sysfs.c | 1140 ++ fs/nilfs2/sysfs.h | 167 + fs/nilfs2/the_nilfs.c | 945 ++ fs/nilfs2/the_nilfs.h | 376 + fs/nls/Kconfig | 623 ++ fs/nls/Makefile | 57 + fs/nls/mac-celtic.c | 602 ++ fs/nls/mac-centeuro.c | 532 + fs/nls/mac-croatian.c | 602 ++ fs/nls/mac-cyrillic.c | 497 + fs/nls/mac-gaelic.c | 567 + fs/nls/mac-greek.c | 497 + fs/nls/mac-iceland.c | 602 ++ fs/nls/mac-inuit.c | 532 + fs/nls/mac-roman.c | 637 ++ fs/nls/mac-romanian.c | 602 ++ fs/nls/mac-turkish.c | 602 ++ fs/nls/nls_ascii.c | 167 + fs/nls/nls_base.c | 560 + fs/nls/nls_cp1250.c | 347 + fs/nls/nls_cp1251.c | 302 + fs/nls/nls_cp1255.c | 385 + fs/nls/nls_cp437.c | 388 + fs/nls/nls_cp737.c | 351 + fs/nls/nls_cp775.c | 320 + fs/nls/nls_cp850.c | 316 + fs/nls/nls_cp852.c | 338 + fs/nls/nls_cp855.c | 300 + fs/nls/nls_cp857.c | 302 + fs/nls/nls_cp860.c | 365 + fs/nls/nls_cp861.c | 388 + fs/nls/nls_cp862.c | 422 + fs/nls/nls_cp863.c | 382 + fs/nls/nls_cp864.c | 408 + fs/nls/nls_cp865.c | 388 + fs/nls/nls_cp866.c | 306 + fs/nls/nls_cp869.c | 316 + fs/nls/nls_cp874.c | 276 + fs/nls/nls_cp932.c | 7934 ++++++++++++++ fs/nls/nls_cp936.c | 11112 ++++++++++++++++++++ fs/nls/nls_cp949.c | 13947 +++++++++++++++++++++++++ fs/nls/nls_cp950.c | 9483 +++++++++++++++++ fs/nls/nls_euc-jp.c | 581 + fs/nls/nls_iso8859-1.c | 258 + fs/nls/nls_iso8859-13.c | 286 + fs/nls/nls_iso8859-14.c | 342 + fs/nls/nls_iso8859-15.c | 308 + fs/nls/nls_iso8859-2.c | 309 + fs/nls/nls_iso8859-3.c | 309 + fs/nls/nls_iso8859-4.c | 309 + fs/nls/nls_iso8859-5.c | 273 + fs/nls/nls_iso8859-6.c | 264 + fs/nls/nls_iso8859-7.c | 318 + fs/nls/nls_iso8859-9.c | 273 + fs/nls/nls_koi8-r.c | 324 + fs/nls/nls_koi8-ru.c | 83 + fs/nls/nls_koi8-u.c | 331 + fs/nls/nls_ucs2_data.h | 15 + fs/nls/nls_ucs2_utils.c | 147 + fs/nls/nls_ucs2_utils.h | 285 + fs/nls/nls_utf8.c | 68 + fs/notify/Kconfig | 7 + fs/notify/Makefile | 7 + fs/notify/dnotify/Kconfig | 12 + fs/notify/dnotify/Makefile | 2 + fs/notify/dnotify/dnotify.c | 413 + fs/notify/fanotify/Kconfig | 26 + fs/notify/fanotify/Makefile | 2 + fs/notify/fanotify/fanotify.c | 1121 ++ fs/notify/fanotify/fanotify.h | 557 + fs/notify/fanotify/fanotify_user.c | 2204 ++++ fs/notify/fdinfo.c | 153 + fs/notify/fdinfo.h | 28 + fs/notify/fsnotify.c | 733 ++ fs/notify/fsnotify.h | 114 + fs/notify/group.c | 160 + fs/notify/inotify/Kconfig | 17 + fs/notify/inotify/Makefile | 2 + fs/notify/inotify/inotify.h | 69 + fs/notify/inotify/inotify_fsnotify.c | 209 + fs/notify/inotify/inotify_user.c | 848 ++ fs/notify/mark.c | 1194 +++ fs/notify/notification.c | 193 + fs/nsfs.c | 713 ++ fs/ntfs/Kconfig | 48 + fs/ntfs/Makefile | 10 + fs/ntfs/aops.c | 318 + fs/ntfs/attrib.c | 5721 ++++++++++ fs/ntfs/attrib.h | 168 + fs/ntfs/attrlist.c | 302 + fs/ntfs/attrlist.h | 20 + fs/ntfs/bdev-io.c | 120 + fs/ntfs/bitmap.c | 290 + fs/ntfs/bitmap.h | 100 + fs/ntfs/collate.c | 146 + fs/ntfs/collate.h | 36 + fs/ntfs/compress.c | 1578 +++ fs/ntfs/debug.c | 171 + fs/ntfs/debug.h | 63 + fs/ntfs/dir.c | 1206 +++ fs/ntfs/dir.h | 32 + fs/ntfs/ea.c | 957 ++ fs/ntfs/ea.h | 30 + fs/ntfs/file.c | 1189 +++ fs/ntfs/index.c | 2233 ++++ fs/ntfs/index.h | 119 + fs/ntfs/inode.c | 3815 +++++++ fs/ntfs/inode.h | 360 + fs/ntfs/iomap.c | 831 ++ fs/ntfs/iomap.h | 23 + fs/ntfs/layout.h | 2365 +++++ fs/ntfs/lcnalloc.c | 1049 ++ fs/ntfs/lcnalloc.h | 134 + fs/ntfs/logfile.c | 780 ++ fs/ntfs/logfile.h | 245 + fs/ntfs/malloc.h | 77 + fs/ntfs/mft.c | 2845 +++++ fs/ntfs/mft.h | 91 + fs/ntfs/mst.c | 194 + fs/ntfs/namei.c | 1682 +++ fs/ntfs/ntfs.h | 294 + fs/ntfs/object_id.c | 158 + fs/ntfs/object_id.h | 14 + fs/ntfs/reparse.c | 964 ++ fs/ntfs/reparse.h | 24 + fs/ntfs/runlist.c | 2093 ++++ fs/ntfs/runlist.h | 97 + fs/ntfs/super.c | 2714 +++++ fs/ntfs/sysctl.c | 54 + fs/ntfs/sysctl.h | 26 + fs/ntfs/time.h | 87 + fs/ntfs/unistr.c | 477 + fs/ntfs/upcase.c | 70 + fs/ntfs/volume.h | 296 + fs/ntfs3/Kconfig | 49 + fs/ntfs3/Makefile | 56 + fs/ntfs3/attrib.c | 2812 +++++ fs/ntfs3/attrlist.c | 439 + fs/ntfs3/bitfunc.c | 128 + fs/ntfs3/bitmap.c | 1564 +++ fs/ntfs3/debug.h | 55 + fs/ntfs3/dir.c | 747 ++ fs/ntfs3/file.c | 1603 +++ fs/ntfs3/frecord.c | 3451 ++++++ fs/ntfs3/fslog.c | 5408 ++++++++++ fs/ntfs3/fsntfs.c | 2717 +++++ fs/ntfs3/index.c | 2835 +++++ fs/ntfs3/inode.c | 2329 +++++ fs/ntfs3/lib/decompress_common.c | 319 + fs/ntfs3/lib/decompress_common.h | 343 + fs/ntfs3/lib/lib.h | 32 + fs/ntfs3/lib/lzx_decompress.c | 669 ++ fs/ntfs3/lib/xpress_decompress.c | 142 + fs/ntfs3/lznt.c | 458 + fs/ntfs3/namei.c | 564 + fs/ntfs3/ntfs.h | 1235 +++ fs/ntfs3/ntfs_fs.h | 1283 +++ fs/ntfs3/record.c | 690 ++ fs/ntfs3/run.c | 1423 +++ fs/ntfs3/super.c | 2025 ++++ fs/ntfs3/upcase.c | 116 + fs/ntfs3/xattr.c | 1075 ++ fs/nullfs.c | 66 + fs/ocfs2/Kconfig | 79 + fs/ocfs2/Makefile | 53 + fs/ocfs2/acl.c | 452 + fs/ocfs2/acl.h | 39 + fs/ocfs2/alloc.c | 7741 ++++++++++++++ fs/ocfs2/alloc.h | 308 + fs/ocfs2/aops.c | 2479 +++++ fs/ocfs2/aops.h | 75 + fs/ocfs2/blockcheck.c | 605 ++ fs/ocfs2/blockcheck.h | 94 + fs/ocfs2/buffer_head_io.c | 456 + fs/ocfs2/buffer_head_io.h | 58 + fs/ocfs2/cluster/Makefile | 5 + fs/ocfs2/cluster/heartbeat.c | 2755 +++++ fs/ocfs2/cluster/heartbeat.h | 76 + fs/ocfs2/cluster/masklog.c | 174 + fs/ocfs2/cluster/masklog.h | 196 + fs/ocfs2/cluster/netdebug.c | 506 + fs/ocfs2/cluster/nodemanager.c | 889 ++ fs/ocfs2/cluster/nodemanager.h | 74 + fs/ocfs2/cluster/ocfs2_heartbeat.h | 22 + fs/ocfs2/cluster/ocfs2_nodemanager.h | 28 + fs/ocfs2/cluster/quorum.c | 326 + fs/ocfs2/cluster/quorum.h | 19 + fs/ocfs2/cluster/sys.c | 65 + fs/ocfs2/cluster/sys.h | 16 + fs/ocfs2/cluster/tcp.c | 2274 ++++ fs/ocfs2/cluster/tcp.h | 139 + fs/ocfs2/cluster/tcp_internal.h | 226 + fs/ocfs2/dcache.c | 459 + fs/ocfs2/dcache.h | 43 + fs/ocfs2/dir.c | 4575 ++++++++ fs/ocfs2/dir.h | 100 + fs/ocfs2/dlm/Makefile | 5 + fs/ocfs2/dlm/dlmapi.h | 201 + fs/ocfs2/dlm/dlmast.c | 475 + fs/ocfs2/dlm/dlmcommon.h | 1132 ++ fs/ocfs2/dlm/dlmconvert.c | 555 + fs/ocfs2/dlm/dlmconvert.h | 18 + fs/ocfs2/dlm/dlmdebug.c | 844 ++ fs/ocfs2/dlm/dlmdebug.h | 49 + fs/ocfs2/dlm/dlmdomain.c | 2371 +++++ fs/ocfs2/dlm/dlmdomain.h | 41 + fs/ocfs2/dlm/dlmlock.c | 743 ++ fs/ocfs2/dlm/dlmmaster.c | 3557 +++++++ fs/ocfs2/dlm/dlmrecovery.c | 2962 ++++++ fs/ocfs2/dlm/dlmthread.c | 809 ++ fs/ocfs2/dlm/dlmunlock.c | 695 ++ fs/ocfs2/dlmfs/Makefile | 4 + fs/ocfs2/dlmfs/dlmfs.c | 636 ++ fs/ocfs2/dlmfs/userdlm.c | 682 ++ fs/ocfs2/dlmfs/userdlm.h | 95 + fs/ocfs2/dlmglue.c | 4485 ++++++++ fs/ocfs2/dlmglue.h | 207 + fs/ocfs2/export.c | 285 + fs/ocfs2/export.h | 17 + fs/ocfs2/extent_map.c | 1047 ++ fs/ocfs2/extent_map.h | 80 + fs/ocfs2/file.c | 2898 +++++ fs/ocfs2/file.h | 74 + fs/ocfs2/filecheck.c | 509 + fs/ocfs2/filecheck.h | 64 + fs/ocfs2/heartbeat.c | 101 + fs/ocfs2/heartbeat.h | 29 + fs/ocfs2/inode.c | 2035 ++++ fs/ocfs2/inode.h | 173 + fs/ocfs2/ioctl.c | 1003 ++ fs/ocfs2/ioctl.h | 20 + fs/ocfs2/journal.c | 2480 +++++ fs/ocfs2/journal.h | 613 ++ fs/ocfs2/localalloc.c | 1318 +++ fs/ocfs2/localalloc.h | 52 + fs/ocfs2/locks.c | 125 + fs/ocfs2/locks.h | 16 + fs/ocfs2/mmap.c | 177 + fs/ocfs2/mmap.h | 7 + fs/ocfs2/move_extents.c | 1094 ++ fs/ocfs2/move_extents.h | 12 + fs/ocfs2/namei.c | 2956 ++++++ fs/ocfs2/namei.h | 38 + fs/ocfs2/ocfs1_fs_compat.h | 94 + fs/ocfs2/ocfs2.h | 991 ++ fs/ocfs2/ocfs2_fs.h | 1616 +++ fs/ocfs2/ocfs2_ioctl.h | 224 + fs/ocfs2/ocfs2_lockid.h | 117 + fs/ocfs2/ocfs2_lockingver.h | 22 + fs/ocfs2/ocfs2_trace.h | 2764 +++++ fs/ocfs2/quota.h | 123 + fs/ocfs2/quota_global.c | 1051 ++ fs/ocfs2/quota_local.c | 1318 +++ fs/ocfs2/refcounttree.c | 4834 +++++++++ fs/ocfs2/refcounttree.h | 127 + fs/ocfs2/reservations.c | 824 ++ fs/ocfs2/reservations.h | 144 + fs/ocfs2/resize.c | 598 ++ fs/ocfs2/resize.h | 16 + fs/ocfs2/slot_map.c | 542 + fs/ocfs2/slot_map.h | 28 + fs/ocfs2/stack_o2cb.c | 439 + fs/ocfs2/stack_user.c | 1102 ++ fs/ocfs2/stackglue.c | 701 ++ fs/ocfs2/stackglue.h | 293 + fs/ocfs2/suballoc.c | 3325 ++++++ fs/ocfs2/suballoc.h | 226 + fs/ocfs2/super.c | 2573 +++++ fs/ocfs2/super.h | 34 + fs/ocfs2/symlink.c | 90 + fs/ocfs2/symlink.h | 26 + fs/ocfs2/sysfile.c | 164 + fs/ocfs2/sysfile.h | 17 + fs/ocfs2/uptodate.c | 621 ++ fs/ocfs2/uptodate.h | 68 + fs/ocfs2/xattr.c | 7650 ++++++++++++++ fs/ocfs2/xattr.h | 93 + fs/omfs/Kconfig | 15 + fs/omfs/Makefile | 5 + fs/omfs/bitmap.c | 194 + fs/omfs/dir.c | 461 + fs/omfs/file.c | 380 + fs/omfs/inode.c | 664 ++ fs/omfs/omfs.h | 69 + fs/omfs/omfs_fs.h | 83 + fs/open.c | 1634 +++ fs/openpromfs/Makefile | 8 + fs/openpromfs/inode.c | 475 + fs/orangefs/Kconfig | 7 + fs/orangefs/Makefile | 11 + fs/orangefs/acl.c | 153 + fs/orangefs/dcache.c | 139 + fs/orangefs/devorangefs-req.c | 815 ++ fs/orangefs/dir.c | 398 + fs/orangefs/downcall.h | 137 + fs/orangefs/file.c | 587 ++ fs/orangefs/inode.c | 1167 +++ fs/orangefs/namei.c | 431 + fs/orangefs/orangefs-bufmap.c | 836 ++ fs/orangefs/orangefs-bufmap.h | 36 + fs/orangefs/orangefs-cache.c | 168 + fs/orangefs/orangefs-debug.h | 46 + fs/orangefs/orangefs-debugfs.c | 1050 ++ fs/orangefs/orangefs-debugfs.h | 7 + fs/orangefs/orangefs-dev-proto.h | 61 + fs/orangefs/orangefs-kernel.h | 484 + fs/orangefs/orangefs-mod.c | 232 + fs/orangefs/orangefs-sysfs.c | 1316 +++ fs/orangefs/orangefs-sysfs.h | 2 + fs/orangefs/orangefs-utils.c | 560 + fs/orangefs/protocol.h | 362 + fs/orangefs/super.c | 664 ++ fs/orangefs/symlink.c | 19 + fs/orangefs/upcall.h | 260 + fs/orangefs/waitqueue.c | 373 + fs/orangefs/xattr.c | 561 + fs/overlayfs/Kconfig | 136 + fs/overlayfs/Makefile | 9 + fs/overlayfs/copy_up.c | 1291 +++ fs/overlayfs/dir.c | 1495 +++ fs/overlayfs/export.c | 868 ++ fs/overlayfs/file.c | 666 ++ fs/overlayfs/inode.c | 1314 +++ fs/overlayfs/namei.c | 1483 +++ fs/overlayfs/overlayfs.h | 955 ++ fs/overlayfs/ovl_entry.h | 193 + fs/overlayfs/params.c | 1104 ++ fs/overlayfs/params.h | 44 + fs/overlayfs/readdir.c | 1326 +++ fs/overlayfs/super.c | 1623 +++ fs/overlayfs/util.c | 1518 +++ fs/overlayfs/xattrs.c | 262 + fs/pidfs.c | 1195 +++ fs/pipe.c | 1633 +++ fs/pnode.c | 687 ++ fs/pnode.h | 65 + fs/posix_acl.c | 1303 +++ fs/proc/Kconfig | 128 + fs/proc/Makefile | 38 + fs/proc/array.c | 822 ++ fs/proc/base.c | 3990 +++++++ fs/proc/bootconfig.c | 102 + fs/proc/cmdline.c | 24 + fs/proc/consoles.c | 116 + fs/proc/cpuinfo.c | 28 + fs/proc/devices.c | 64 + fs/proc/fd.c | 407 + fs/proc/fd.h | 21 + fs/proc/generic.c | 856 ++ fs/proc/inode.c | 680 ++ fs/proc/internal.h | 437 + fs/proc/interrupts.c | 42 + fs/proc/kcore.c | 742 ++ fs/proc/kmsg.c | 63 + fs/proc/loadavg.c | 37 + fs/proc/meminfo.c | 188 + fs/proc/namespaces.c | 196 + fs/proc/nommu.c | 116 + fs/proc/page.c | 289 + fs/proc/proc_net.c | 426 + fs/proc/proc_sysctl.c | 1725 +++ fs/proc/proc_tty.c | 177 + fs/proc/root.c | 468 + fs/proc/self.c | 66 + fs/proc/softirqs.c | 37 + fs/proc/stat.c | 180 + fs/proc/task_mmu.c | 3631 +++++++ fs/proc/task_nommu.c | 296 + fs/proc/thread_self.c | 65 + fs/proc/uptime.c | 45 + fs/proc/util.c | 24 + fs/proc/version.c | 27 + fs/proc/vmcore.c | 1756 ++++ fs/proc_namespace.c | 337 + fs/pstore/Kconfig | 191 + fs/pstore/Makefile | 20 + fs/pstore/blk.c | 361 + fs/pstore/ftrace.c | 230 + fs/pstore/inode.c | 528 + fs/pstore/internal.h | 54 + fs/pstore/platform.c | 766 ++ fs/pstore/pmsg.c | 94 + fs/pstore/ram.c | 999 ++ fs/pstore/ram_core.c | 642 ++ fs/pstore/ram_internal.h | 98 + fs/pstore/zone.c | 1475 +++ fs/qnx4/Kconfig | 16 + fs/qnx4/Makefile | 8 + fs/qnx4/bitmap.c | 45 + fs/qnx4/dir.c | 81 + fs/qnx4/inode.c | 436 + fs/qnx4/namei.c | 115 + fs/qnx4/qnx4.h | 106 + fs/qnx6/Kconfig | 28 + fs/qnx6/Makefile | 9 + fs/qnx6/dir.c | 284 + fs/qnx6/inode.c | 694 ++ fs/qnx6/namei.c | 37 + fs/qnx6/qnx6.h | 129 + fs/qnx6/super_mmi.c | 149 + fs/quota/Kconfig | 60 + fs/quota/Makefile | 7 + fs/quota/compat.h | 34 + fs/quota/dquot.c | 2998 ++++++ fs/quota/kqid.c | 133 + fs/quota/netlink.c | 103 + fs/quota/quota.c | 1010 ++ fs/quota/quota_tree.c | 853 ++ fs/quota/quota_tree.h | 26 + fs/quota/quota_v1.c | 249 + fs/quota/quota_v2.c | 455 + fs/quota/quotaio_v1.h | 36 + fs/quota/quotaio_v2.h | 76 + fs/ramfs/Makefile | 10 + fs/ramfs/file-mmu.c | 55 + fs/ramfs/file-nommu.c | 273 + fs/ramfs/inode.c | 329 + fs/ramfs/internal.h | 9 + fs/read_write.c | 1822 ++++ fs/readdir.c | 573 + fs/remap_range.c | 567 + fs/resctrl/Kconfig | 39 + fs/resctrl/Makefile | 6 + fs/resctrl/ctrlmondata.c | 1075 ++ fs/resctrl/internal.h | 520 + fs/resctrl/monitor.c | 1985 ++++ fs/resctrl/monitor_trace.h | 33 + fs/resctrl/pseudo_lock.c | 1099 ++ fs/resctrl/rdtgroup.c | 4921 +++++++++ fs/romfs/Kconfig | 64 + fs/romfs/Makefile | 13 + fs/romfs/internal.h | 43 + fs/romfs/mmap-nommu.c | 85 + fs/romfs/storage.c | 287 + fs/romfs/super.c | 672 ++ fs/select.c | 1438 +++ fs/seq_file.c | 1143 ++ fs/signalfd.c | 351 + fs/smb/Kconfig | 29 + fs/smb/Makefile | 6 + fs/smb/client/.gitignore | 4 + fs/smb/client/Kconfig | 232 + fs/smb/client/Makefile | 83 + fs/smb/client/asn1.c | 65 + fs/smb/client/cached_dir.c | 875 ++ fs/smb/client/cached_dir.h | 96 + fs/smb/client/cifs_debug.c | 1297 +++ fs/smb/client/cifs_debug.h | 159 + fs/smb/client/cifs_fs_sb.h | 76 + fs/smb/client/cifs_ioctl.h | 124 + fs/smb/client/cifs_spnego.c | 252 + fs/smb/client/cifs_spnego.h | 34 + fs/smb/client/cifs_spnego_negtokeninit.asn1 | 40 + fs/smb/client/cifs_swn.c | 876 ++ fs/smb/client/cifs_swn.h | 52 + fs/smb/client/cifs_unicode.c | 631 ++ fs/smb/client/cifs_unicode.h | 86 + fs/smb/client/cifsacl.c | 2002 ++++ fs/smb/client/cifsacl.h | 110 + fs/smb/client/cifsencrypt.c | 514 + fs/smb/client/cifsfs.c | 2225 ++++ fs/smb/client/cifsfs.h | 171 + fs/smb/client/cifsglob.h | 2404 +++++ fs/smb/client/cifspdu.h | 12 + fs/smb/client/cifsproto.h | 615 ++ fs/smb/client/cifsroot.c | 94 + fs/smb/client/cifssmb.c | 6411 ++++++++++++ fs/smb/client/compress.c | 372 + fs/smb/client/compress.h | 50 + fs/smb/client/connect.c | 4542 ++++++++ fs/smb/client/dfs.c | 467 + fs/smb/client/dfs.h | 199 + fs/smb/client/dfs_cache.c | 1356 +++ fs/smb/client/dfs_cache.h | 110 + fs/smb/client/dir.c | 1212 +++ fs/smb/client/dns_resolve.c | 96 + fs/smb/client/dns_resolve.h | 37 + fs/smb/client/export.c | 53 + fs/smb/client/file.c | 3518 +++++++ fs/smb/client/fs_context.c | 2258 ++++ fs/smb/client/fs_context.h | 431 + fs/smb/client/fscache.c | 168 + fs/smb/client/fscache.h | 106 + fs/smb/client/gen_smb1_mapping | 124 + fs/smb/client/gen_smb2_mapping | 86 + fs/smb/client/inode.c | 3559 +++++++ fs/smb/client/ioctl.c | 723 ++ fs/smb/client/link.c | 642 ++ fs/smb/client/misc.c | 1112 ++ fs/smb/client/namespace.c | 298 + fs/smb/client/netlink.c | 94 + fs/smb/client/netlink.h | 16 + fs/smb/client/netmisc.c | 212 + fs/smb/client/nterr.h | 575 + fs/smb/client/ntlmssp.h | 158 + fs/smb/client/readdir.c | 1235 +++ fs/smb/client/reparse.c | 1289 +++ fs/smb/client/reparse.h | 142 + fs/smb/client/rfc1002pdu.h | 61 + fs/smb/client/sess.c | 1039 ++ fs/smb/client/smb1debug.c | 25 + fs/smb/client/smb1encrypt.c | 140 + fs/smb/client/smb1maperror.c | 282 + fs/smb/client/smb1maperror_test.c | 78 + fs/smb/client/smb1misc.c | 189 + fs/smb/client/smb1ops.c | 1498 +++ fs/smb/client/smb1pdu.h | 2340 +++++ fs/smb/client/smb1proto.h | 350 + fs/smb/client/smb1session.c | 995 ++ fs/smb/client/smb1transport.c | 584 ++ fs/smb/client/smb2file.c | 467 + fs/smb/client/smb2glob.h | 61 + fs/smb/client/smb2inode.c | 1581 +++ fs/smb/client/smb2maperror.c | 126 + fs/smb/client/smb2maperror_test.c | 50 + fs/smb/client/smb2misc.c | 957 ++ fs/smb/client/smb2ops.c | 6120 +++++++++++ fs/smb/client/smb2pdu.c | 6465 ++++++++++++ fs/smb/client/smb2pdu.h | 315 + fs/smb/client/smb2proto.h | 278 + fs/smb/client/smb2transport.c | 816 ++ fs/smb/client/smbdirect.c | 561 + fs/smb/client/smbdirect.h | 67 + fs/smb/client/smbencrypt.c | 83 + fs/smb/client/smberr.h | 298 + fs/smb/client/trace.c | 10 + fs/smb/client/trace.h | 1970 ++++ fs/smb/client/transport.c | 1291 +++ fs/smb/client/unc.c | 69 + fs/smb/client/winucase.c | 649 ++ fs/smb/client/xattr.c | 535 + fs/smb/common/Makefile | 9 + fs/smb/common/cifs_md4.c | 198 + fs/smb/common/compress/compress.c | 404 + fs/smb/common/compress/compress.h | 29 + fs/smb/common/compress/lz77.c | 461 + fs/smb/common/compress/lz77.h | 46 + fs/smb/common/fscc.h | 604 ++ fs/smb/common/md4.h | 27 + fs/smb/common/smb1pdu.h | 56 + fs/smb/common/smb2pdu.h | 1836 ++++ fs/smb/common/smb2status.h | 1790 ++++ fs/smb/common/smbacl.h | 122 + fs/smb/common/smbfsctl.h | 175 + fs/smb/common/smbglob.h | 70 + fs/smb/server/Kconfig | 73 + fs/smb/server/Makefile | 21 + fs/smb/server/asn1.c | 241 + fs/smb/server/asn1.h | 21 + fs/smb/server/auth.c | 1137 ++ fs/smb/server/auth.h | 74 + fs/smb/server/compress.c | 260 + fs/smb/server/compress.h | 17 + fs/smb/server/connection.c | 860 ++ fs/smb/server/connection.h | 320 + fs/smb/server/crypto_ctx.c | 176 + fs/smb/server/crypto_ctx.h | 32 + fs/smb/server/glob.h | 49 + fs/smb/server/ksmbd_netlink.h | 443 + fs/smb/server/ksmbd_spnego_negtokeninit.asn1 | 39 + fs/smb/server/ksmbd_spnego_negtokentarg.asn1 | 26 + fs/smb/server/ksmbd_work.c | 196 + fs/smb/server/ksmbd_work.h | 163 + fs/smb/server/mgmt/ksmbd_ida.c | 38 + fs/smb/server/mgmt/ksmbd_ida.h | 34 + fs/smb/server/mgmt/share_config.c | 321 + fs/smb/server/mgmt/share_config.h | 102 + fs/smb/server/mgmt/tree_connect.c | 180 + fs/smb/server/mgmt/tree_connect.h | 64 + fs/smb/server/mgmt/user_config.c | 112 + fs/smb/server/mgmt/user_config.h | 70 + fs/smb/server/mgmt/user_session.c | 780 ++ fs/smb/server/mgmt/user_session.h | 123 + fs/smb/server/misc.c | 363 + fs/smb/server/misc.h | 63 + fs/smb/server/ndr.c | 514 + fs/smb/server/ndr.h | 22 + fs/smb/server/ntlmssp.h | 169 + fs/smb/server/oplock.c | 2503 +++++ fs/smb/server/oplock.h | 147 + fs/smb/server/proc.c | 270 + fs/smb/server/server.c | 760 ++ fs/smb/server/server.h | 77 + fs/smb/server/smb2misc.c | 564 + fs/smb/server/smb2ops.c | 343 + fs/smb/server/smb2pdu.c | 12348 ++++++++++++++++++++++ fs/smb/server/smb2pdu.h | 465 + fs/smb/server/smb_common.c | 879 ++ fs/smb/server/smb_common.h | 209 + fs/smb/server/smbacl.c | 1735 +++ fs/smb/server/smbacl.h | 132 + fs/smb/server/stats.h | 124 + fs/smb/server/transport_ipc.c | 1025 ++ fs/smb/server/transport_ipc.h | 47 + fs/smb/server/transport_rdma.c | 554 + fs/smb/server/transport_rdma.h | 32 + fs/smb/server/transport_tcp.c | 703 ++ fs/smb/server/transport_tcp.h | 15 + fs/smb/server/unicode.c | 454 + fs/smb/server/unicode.h | 41 + fs/smb/server/vfs.c | 2411 +++++ fs/smb/server/vfs.h | 184 + fs/smb/server/vfs_cache.c | 1995 ++++ fs/smb/server/vfs_cache.h | 263 + fs/smb/server/xattr.h | 122 + fs/smb/smbdirect/Kconfig | 9 + fs/smb/smbdirect/Makefile | 18 + fs/smb/smbdirect/accept.c | 868 ++ fs/smb/smbdirect/connect.c | 926 ++ fs/smb/smbdirect/connection.c | 2190 ++++ fs/smb/smbdirect/debug.c | 88 + fs/smb/smbdirect/devices.c | 277 + fs/smb/smbdirect/internal.h | 141 + fs/smb/smbdirect/listen.c | 308 + fs/smb/smbdirect/main.c | 121 + fs/smb/smbdirect/mr.c | 496 + fs/smb/smbdirect/pdu.h | 59 + fs/smb/smbdirect/rw.c | 255 + fs/smb/smbdirect/socket.c | 791 ++ fs/smb/smbdirect/socket.h | 832 ++ fs/splice.c | 1994 ++++ fs/squashfs/Kconfig | 287 + fs/squashfs/Makefile | 19 + fs/squashfs/block.c | 386 + fs/squashfs/cache.c | 557 + fs/squashfs/decompressor.c | 141 + fs/squashfs/decompressor.h | 54 + fs/squashfs/decompressor_multi.c | 203 + fs/squashfs/decompressor_multi_percpu.c | 110 + fs/squashfs/decompressor_single.c | 91 + fs/squashfs/dir.c | 225 + fs/squashfs/export.c | 180 + fs/squashfs/file.c | 781 ++ fs/squashfs/file_cache.c | 36 + fs/squashfs/file_direct.c | 124 + fs/squashfs/fragment.c | 89 + fs/squashfs/id.c | 115 + fs/squashfs/inode.c | 456 + fs/squashfs/lz4_wrapper.c | 146 + fs/squashfs/lzo_wrapper.c | 123 + fs/squashfs/namei.c | 232 + fs/squashfs/page_actor.c | 141 + fs/squashfs/page_actor.h | 59 + fs/squashfs/squashfs.h | 120 + fs/squashfs/squashfs_fs.h | 453 + fs/squashfs/squashfs_fs_i.h | 41 + fs/squashfs/squashfs_fs_sb.h | 82 + fs/squashfs/super.c | 706 ++ fs/squashfs/symlink.c | 109 + fs/squashfs/xattr.c | 271 + fs/squashfs/xattr.h | 41 + fs/squashfs/xattr_id.c | 132 + fs/squashfs/xz_wrapper.c | 196 + fs/squashfs/zlib_wrapper.c | 137 + fs/squashfs/zstd_wrapper.c | 154 + fs/stack.c | 76 + fs/stat.c | 968 ++ fs/statfs.c | 405 + fs/super.c | 2465 +++++ fs/sync.c | 380 + fs/sysctls.c | 37 + fs/sysfs/Kconfig | 25 + fs/sysfs/Makefile | 6 + fs/sysfs/dir.c | 161 + fs/sysfs/file.c | 821 ++ fs/sysfs/group.c | 638 ++ fs/sysfs/mount.c | 103 + fs/sysfs/symlink.c | 199 + fs/sysfs/sysfs.h | 43 + fs/tests/binfmt_elf_kunit.c | 64 + fs/tests/exec_kunit.c | 132 + fs/timerfd.c | 621 ++ fs/tracefs/Makefile | 6 + fs/tracefs/event_inode.c | 928 ++ fs/tracefs/inode.c | 809 ++ fs/tracefs/internal.h | 82 + fs/ubifs/Kconfig | 101 + fs/ubifs/Makefile | 11 + fs/ubifs/auth.c | 530 + fs/ubifs/budget.c | 714 ++ fs/ubifs/commit.c | 733 ++ fs/ubifs/compress.c | 369 + fs/ubifs/crypto.c | 97 + fs/ubifs/debug.c | 3052 ++++++ fs/ubifs/debug.h | 305 + fs/ubifs/dir.c | 1772 ++++ fs/ubifs/file.c | 1666 +++ fs/ubifs/find.c | 963 ++ fs/ubifs/gc.c | 1015 ++ fs/ubifs/io.c | 1262 +++ fs/ubifs/ioctl.c | 230 + fs/ubifs/journal.c | 2059 ++++ fs/ubifs/key.h | 543 + fs/ubifs/log.c | 762 ++ fs/ubifs/lprops.c | 1307 +++ fs/ubifs/lpt.c | 2435 +++++ fs/ubifs/lpt_commit.c | 1995 ++++ fs/ubifs/master.c | 476 + fs/ubifs/misc.c | 69 + fs/ubifs/misc.h | 289 + fs/ubifs/orphan.c | 947 ++ fs/ubifs/recovery.c | 1586 +++ fs/ubifs/replay.c | 1252 +++ fs/ubifs/sb.c | 956 ++ fs/ubifs/scan.c | 366 + fs/ubifs/shrinker.c | 319 + fs/ubifs/super.c | 2515 +++++ fs/ubifs/sysfs.c | 156 + fs/ubifs/tnc.c | 3575 +++++++ fs/ubifs/tnc_commit.c | 1112 ++ fs/ubifs/tnc_misc.c | 517 + fs/ubifs/ubifs-media.h | 855 ++ fs/ubifs/ubifs.h | 2159 ++++ fs/ubifs/xattr.c | 706 ++ fs/udf/Kconfig | 19 + fs/udf/Makefile | 10 + fs/udf/balloc.c | 744 ++ fs/udf/dir.c | 162 + fs/udf/directory.c | 543 + fs/udf/ecma_167.h | 816 ++ fs/udf/file.c | 256 + fs/udf/ialloc.c | 113 + fs/udf/inode.c | 2493 +++++ fs/udf/lowlevel.c | 62 + fs/udf/misc.c | 289 + fs/udf/namei.c | 1028 ++ fs/udf/osta_udf.h | 305 + fs/udf/partition.c | 340 + fs/udf/super.c | 2584 +++++ fs/udf/symlink.c | 173 + fs/udf/truncate.c | 293 + fs/udf/udf_i.h | 65 + fs/udf/udf_sb.h | 183 + fs/udf/udfdecl.h | 262 + fs/udf/udfend.h | 78 + fs/udf/udftime.c | 89 + fs/udf/unicode.c | 399 + fs/ufs/Kconfig | 45 + fs/ufs/Makefile | 10 + fs/ufs/balloc.c | 936 ++ fs/ufs/cylinder.c | 207 + fs/ufs/dir.c | 658 ++ fs/ufs/file.c | 48 + fs/ufs/ialloc.c | 355 + fs/ufs/inode.c | 1222 +++ fs/ufs/namei.c | 329 + fs/ufs/super.c | 1495 +++ fs/ufs/swab.h | 116 + fs/ufs/ufs.h | 164 + fs/ufs/ufs_fs.h | 958 ++ fs/ufs/util.c | 223 + fs/ufs/util.h | 597 ++ fs/unicode/.gitignore | 3 + fs/unicode/Kconfig | 16 + fs/unicode/Makefile | 41 + fs/unicode/README.utf8data | 57 + fs/unicode/mkutf8data.c | 3434 ++++++ fs/unicode/tests/.kunitconfig | 3 + fs/unicode/tests/utf8_kunit.c | 297 + fs/unicode/utf8-core.c | 222 + fs/unicode/utf8-norm.c | 594 ++ fs/unicode/utf8data.c_shipped | 4124 ++++++++ fs/unicode/utf8n.h | 83 + fs/utimes.c | 294 + fs/vboxsf/Kconfig | 10 + fs/vboxsf/Makefile | 5 + fs/vboxsf/dir.c | 481 + fs/vboxsf/file.c | 395 + fs/vboxsf/shfl_hostintf.h | 901 ++ fs/vboxsf/super.c | 484 + fs/vboxsf/utils.c | 599 ++ fs/vboxsf/vboxsf_wrappers.c | 371 + fs/vboxsf/vfsmod.h | 152 + fs/verity/Kconfig | 49 + fs/verity/Makefile | 12 + fs/verity/enable.c | 413 + fs/verity/fsverity_private.h | 171 + fs/verity/hash_algs.c | 188 + fs/verity/init.c | 70 + fs/verity/measure.c | 191 + fs/verity/open.c | 425 + fs/verity/pagecache.c | 80 + fs/verity/read_metadata.c | 204 + fs/verity/signature.c | 138 + fs/verity/verify.c | 499 + fs/xattr.c | 1761 ++++ fs/xfs/Kconfig | 243 + fs/xfs/Makefile | 256 + fs/xfs/libxfs/xfs_ag.c | 995 ++ fs/xfs/libxfs/xfs_ag.h | 362 + fs/xfs/libxfs/xfs_ag_resv.c | 418 + fs/xfs/libxfs/xfs_ag_resv.h | 36 + fs/xfs/libxfs/xfs_alloc.c | 4192 ++++++++ fs/xfs/libxfs/xfs_alloc.h | 290 + fs/xfs/libxfs/xfs_alloc_btree.c | 619 ++ fs/xfs/libxfs/xfs_alloc_btree.h | 69 + fs/xfs/libxfs/xfs_attr.c | 1661 +++ fs/xfs/libxfs/xfs_attr.h | 657 ++ fs/xfs/libxfs/xfs_attr_leaf.c | 3161 ++++++ fs/xfs/libxfs/xfs_attr_leaf.h | 115 + fs/xfs/libxfs/xfs_attr_remote.c | 738 ++ fs/xfs/libxfs/xfs_attr_remote.h | 26 + fs/xfs/libxfs/xfs_attr_sf.h | 58 + fs/xfs/libxfs/xfs_bit.c | 106 + fs/xfs/libxfs/xfs_bit.h | 75 + fs/xfs/libxfs/xfs_bmap.c | 6264 +++++++++++ fs/xfs/libxfs/xfs_bmap.h | 303 + fs/xfs/libxfs/xfs_bmap_btree.c | 860 ++ fs/xfs/libxfs/xfs_bmap_btree.h | 204 + fs/xfs/libxfs/xfs_btree.c | 5650 ++++++++++ fs/xfs/libxfs/xfs_btree.h | 715 ++ fs/xfs/libxfs/xfs_btree_mem.c | 346 + fs/xfs/libxfs/xfs_btree_mem.h | 75 + fs/xfs/libxfs/xfs_btree_staging.c | 819 ++ fs/xfs/libxfs/xfs_btree_staging.h | 130 + fs/xfs/libxfs/xfs_cksum.h | 82 + fs/xfs/libxfs/xfs_da_btree.c | 2912 ++++++ fs/xfs/libxfs/xfs_da_btree.h | 247 + fs/xfs/libxfs/xfs_da_format.h | 907 ++ fs/xfs/libxfs/xfs_defer.c | 1245 +++ fs/xfs/libxfs/xfs_defer.h | 165 + fs/xfs/libxfs/xfs_dir2.c | 1411 +++ fs/xfs/libxfs/xfs_dir2.h | 358 + fs/xfs/libxfs/xfs_dir2_block.c | 1281 +++ fs/xfs/libxfs/xfs_dir2_data.c | 1255 +++ fs/xfs/libxfs/xfs_dir2_leaf.c | 1881 ++++ fs/xfs/libxfs/xfs_dir2_node.c | 2348 +++++ fs/xfs/libxfs/xfs_dir2_priv.h | 218 + fs/xfs/libxfs/xfs_dir2_sf.c | 1268 +++ fs/xfs/libxfs/xfs_dquot_buf.c | 506 + fs/xfs/libxfs/xfs_errortag.h | 143 + fs/xfs/libxfs/xfs_exchmaps.c | 1247 +++ fs/xfs/libxfs/xfs_exchmaps.h | 124 + fs/xfs/libxfs/xfs_format.h | 2030 ++++ fs/xfs/libxfs/xfs_fs.h | 1301 +++ fs/xfs/libxfs/xfs_group.c | 230 + fs/xfs/libxfs/xfs_group.h | 192 + fs/xfs/libxfs/xfs_health.h | 297 + fs/xfs/libxfs/xfs_ialloc.c | 3185 ++++++ fs/xfs/libxfs/xfs_ialloc.h | 118 + fs/xfs/libxfs/xfs_ialloc_btree.c | 838 ++ fs/xfs/libxfs/xfs_ialloc_btree.h | 79 + fs/xfs/libxfs/xfs_iext_tree.c | 1067 ++ fs/xfs/libxfs/xfs_inode_buf.c | 975 ++ fs/xfs/libxfs/xfs_inode_buf.h | 56 + fs/xfs/libxfs/xfs_inode_fork.c | 778 ++ fs/xfs/libxfs/xfs_inode_fork.h | 274 + fs/xfs/libxfs/xfs_inode_util.c | 742 ++ fs/xfs/libxfs/xfs_inode_util.h | 62 + fs/xfs/libxfs/xfs_log_format.h | 1086 ++ fs/xfs/libxfs/xfs_log_recover.h | 174 + fs/xfs/libxfs/xfs_log_rlimit.c | 251 + fs/xfs/libxfs/xfs_metadir.c | 501 + fs/xfs/libxfs/xfs_metadir.h | 49 + fs/xfs/libxfs/xfs_metafile.c | 327 + fs/xfs/libxfs/xfs_metafile.h | 44 + fs/xfs/libxfs/xfs_ondisk.h | 326 + fs/xfs/libxfs/xfs_parent.c | 380 + fs/xfs/libxfs/xfs_parent.h | 110 + fs/xfs/libxfs/xfs_quota_defs.h | 187 + fs/xfs/libxfs/xfs_refcount.c | 2244 ++++ fs/xfs/libxfs/xfs_refcount.h | 146 + fs/xfs/libxfs/xfs_refcount_btree.c | 543 + fs/xfs/libxfs/xfs_refcount_btree.h | 72 + fs/xfs/libxfs/xfs_rmap.c | 3142 ++++++ fs/xfs/libxfs/xfs_rmap.h | 280 + fs/xfs/libxfs/xfs_rmap_btree.c | 872 ++ fs/xfs/libxfs/xfs_rmap_btree.h | 72 + fs/xfs/libxfs/xfs_rtbitmap.c | 1504 +++ fs/xfs/libxfs/xfs_rtbitmap.h | 453 + fs/xfs/libxfs/xfs_rtgroup.c | 745 ++ fs/xfs/libxfs/xfs_rtgroup.h | 389 + fs/xfs/libxfs/xfs_rtrefcount_btree.c | 757 ++ fs/xfs/libxfs/xfs_rtrefcount_btree.h | 189 + fs/xfs/libxfs/xfs_rtrmap_btree.c | 1033 ++ fs/xfs/libxfs/xfs_rtrmap_btree.h | 212 + fs/xfs/libxfs/xfs_sb.c | 1755 ++++ fs/xfs/libxfs/xfs_sb.h | 49 + fs/xfs/libxfs/xfs_shared.h | 257 + fs/xfs/libxfs/xfs_symlink_remote.c | 432 + fs/xfs/libxfs/xfs_symlink_remote.h | 28 + fs/xfs/libxfs/xfs_trans_inode.c | 130 + fs/xfs/libxfs/xfs_trans_resv.c | 1559 +++ fs/xfs/libxfs/xfs_trans_resv.h | 130 + fs/xfs/libxfs/xfs_trans_space.c | 121 + fs/xfs/libxfs/xfs_trans_space.h | 124 + fs/xfs/libxfs/xfs_types.c | 256 + fs/xfs/libxfs/xfs_types.h | 287 + fs/xfs/libxfs/xfs_zones.c | 116 + fs/xfs/libxfs/xfs_zones.h | 44 + fs/xfs/scrub/agb_bitmap.c | 103 + fs/xfs/scrub/agb_bitmap.h | 73 + fs/xfs/scrub/agheader.c | 1109 ++ fs/xfs/scrub/agheader_repair.c | 1906 ++++ fs/xfs/scrub/agino_bitmap.h | 49 + fs/xfs/scrub/alloc.c | 195 + fs/xfs/scrub/alloc_repair.c | 946 ++ fs/xfs/scrub/attr.c | 686 ++ fs/xfs/scrub/attr.h | 31 + fs/xfs/scrub/attr_repair.c | 1656 +++ fs/xfs/scrub/attr_repair.h | 15 + fs/xfs/scrub/bitmap.c | 584 ++ fs/xfs/scrub/bitmap.h | 67 + fs/xfs/scrub/bmap.c | 1192 +++ fs/xfs/scrub/bmap_repair.c | 992 ++ fs/xfs/scrub/btree.c | 807 ++ fs/xfs/scrub/btree.h | 70 + fs/xfs/scrub/common.c | 1756 ++++ fs/xfs/scrub/common.h | 276 + fs/xfs/scrub/cow_repair.c | 743 ++ fs/xfs/scrub/dab_bitmap.h | 37 + fs/xfs/scrub/dabtree.c | 628 ++ fs/xfs/scrub/dabtree.h | 46 + fs/xfs/scrub/dir.c | 1183 +++ fs/xfs/scrub/dir_repair.c | 1963 ++++ fs/xfs/scrub/dirtree.c | 1008 ++ fs/xfs/scrub/dirtree.h | 168 + fs/xfs/scrub/dirtree_repair.c | 835 ++ fs/xfs/scrub/dqiterate.c | 211 + fs/xfs/scrub/findparent.c | 469 + fs/xfs/scrub/findparent.h | 56 + fs/xfs/scrub/fsb_bitmap.h | 37 + fs/xfs/scrub/fscounters.c | 621 ++ fs/xfs/scrub/fscounters.h | 21 + fs/xfs/scrub/fscounters_repair.c | 85 + fs/xfs/scrub/health.c | 337 + fs/xfs/scrub/health.h | 17 + fs/xfs/scrub/ialloc.c | 796 ++ fs/xfs/scrub/ialloc_repair.c | 885 ++ fs/xfs/scrub/ino_bitmap.h | 37 + fs/xfs/scrub/inode.c | 841 ++ fs/xfs/scrub/inode_repair.c | 2093 ++++ fs/xfs/scrub/iscan.c | 826 ++ fs/xfs/scrub/iscan.h | 100 + fs/xfs/scrub/listxattr.c | 320 + fs/xfs/scrub/listxattr.h | 19 + fs/xfs/scrub/metapath.c | 677 ++ fs/xfs/scrub/newbt.c | 611 ++ fs/xfs/scrub/newbt.h | 76 + fs/xfs/scrub/nlinks.c | 1073 ++ fs/xfs/scrub/nlinks.h | 109 + fs/xfs/scrub/nlinks_repair.c | 354 + fs/xfs/scrub/off_bitmap.h | 37 + fs/xfs/scrub/orphanage.c | 631 ++ fs/xfs/scrub/orphanage.h | 86 + fs/xfs/scrub/parent.c | 952 ++ fs/xfs/scrub/parent_repair.c | 1622 +++ fs/xfs/scrub/quota.c | 345 + fs/xfs/scrub/quota.h | 36 + fs/xfs/scrub/quota_repair.c | 566 + fs/xfs/scrub/quotacheck.c | 860 ++ fs/xfs/scrub/quotacheck.h | 76 + fs/xfs/scrub/quotacheck_repair.c | 248 + fs/xfs/scrub/rcbag.c | 307 + fs/xfs/scrub/rcbag.h | 28 + fs/xfs/scrub/rcbag_btree.c | 352 + fs/xfs/scrub/rcbag_btree.h | 81 + fs/xfs/scrub/readdir.c | 421 + fs/xfs/scrub/readdir.h | 22 + fs/xfs/scrub/reap.c | 1694 +++ fs/xfs/scrub/reap.h | 47 + fs/xfs/scrub/refcount.c | 632 ++ fs/xfs/scrub/refcount_repair.c | 741 ++ fs/xfs/scrub/repair.c | 1410 +++ fs/xfs/scrub/repair.h | 283 + fs/xfs/scrub/rgb_bitmap.h | 37 + fs/xfs/scrub/rgsuper.c | 100 + fs/xfs/scrub/rmap.c | 648 ++ fs/xfs/scrub/rmap_repair.c | 1734 +++ fs/xfs/scrub/rtb_bitmap.h | 37 + fs/xfs/scrub/rtbitmap.c | 299 + fs/xfs/scrub/rtbitmap.h | 77 + fs/xfs/scrub/rtbitmap_repair.c | 643 ++ fs/xfs/scrub/rtrefcount.c | 660 ++ fs/xfs/scrub/rtrefcount_repair.c | 752 ++ fs/xfs/scrub/rtrmap.c | 328 + fs/xfs/scrub/rtrmap_repair.c | 978 ++ fs/xfs/scrub/rtsummary.c | 369 + fs/xfs/scrub/rtsummary.h | 37 + fs/xfs/scrub/rtsummary_repair.c | 186 + fs/xfs/scrub/scrub.c | 1012 ++ fs/xfs/scrub/scrub.h | 350 + fs/xfs/scrub/stats.c | 415 + fs/xfs/scrub/stats.h | 59 + fs/xfs/scrub/symlink.c | 91 + fs/xfs/scrub/symlink_repair.c | 510 + fs/xfs/scrub/tempexch.h | 22 + fs/xfs/scrub/tempfile.c | 981 ++ fs/xfs/scrub/tempfile.h | 51 + fs/xfs/scrub/trace.c | 61 + fs/xfs/scrub/trace.h | 3849 +++++++ fs/xfs/scrub/xfarray.c | 1072 ++ fs/xfs/scrub/xfarray.h | 193 + fs/xfs/scrub/xfblob.c | 168 + fs/xfs/scrub/xfblob.h | 50 + fs/xfs/scrub/xfile.c | 325 + fs/xfs/scrub/xfile.h | 35 + fs/xfs/scrub/xfs_scrub.h | 17 + fs/xfs/xfs_acl.c | 294 + fs/xfs/xfs_acl.h | 31 + fs/xfs/xfs_aops.c | 880 ++ fs/xfs/xfs_aops.h | 15 + fs/xfs/xfs_attr_inactive.c | 409 + fs/xfs/xfs_attr_item.c | 1207 +++ fs/xfs/xfs_attr_item.h | 64 + fs/xfs/xfs_attr_list.c | 599 ++ fs/xfs/xfs_bio_io.c | 53 + fs/xfs/xfs_bmap_item.c | 720 ++ fs/xfs/xfs_bmap_item.h | 78 + fs/xfs/xfs_bmap_util.c | 1865 ++++ fs/xfs/xfs_bmap_util.h | 93 + fs/xfs/xfs_buf.c | 2108 ++++ fs/xfs/xfs_buf.h | 374 + fs/xfs/xfs_buf_item.c | 1087 ++ fs/xfs/xfs_buf_item.h | 71 + fs/xfs/xfs_buf_item_recover.c | 1238 +++ fs/xfs/xfs_buf_mem.c | 240 + fs/xfs/xfs_buf_mem.h | 32 + fs/xfs/xfs_dahash_test.c | 673 ++ fs/xfs/xfs_dahash_test.h | 12 + fs/xfs/xfs_dir2_readdir.c | 556 + fs/xfs/xfs_discard.c | 886 ++ fs/xfs/xfs_discard.h | 12 + fs/xfs/xfs_dquot.c | 1552 +++ fs/xfs/xfs_dquot.h | 243 + fs/xfs/xfs_dquot_item.c | 278 + fs/xfs/xfs_dquot_item.h | 28 + fs/xfs/xfs_dquot_item_recover.c | 215 + fs/xfs/xfs_drain.c | 152 + fs/xfs/xfs_drain.h | 87 + fs/xfs/xfs_error.c | 385 + fs/xfs/xfs_error.h | 99 + fs/xfs/xfs_exchmaps_item.c | 611 ++ fs/xfs/xfs_exchmaps_item.h | 64 + fs/xfs/xfs_exchrange.c | 923 ++ fs/xfs/xfs_exchrange.h | 52 + fs/xfs/xfs_export.c | 249 + fs/xfs/xfs_export.h | 62 + fs/xfs/xfs_extent_busy.c | 727 ++ fs/xfs/xfs_extent_busy.h | 79 + fs/xfs/xfs_extfree_item.c | 1029 ++ fs/xfs/xfs_extfree_item.h | 100 + fs/xfs/xfs_file.c | 2212 ++++ fs/xfs/xfs_file.h | 15 + fs/xfs/xfs_filestream.c | 396 + fs/xfs/xfs_filestream.h | 28 + fs/xfs/xfs_fsmap.c | 1428 +++ fs/xfs/xfs_fsmap.h | 48 + fs/xfs/xfs_fsops.c | 590 ++ fs/xfs/xfs_fsops.h | 18 + fs/xfs/xfs_globals.c | 55 + fs/xfs/xfs_handle.c | 922 ++ fs/xfs/xfs_handle.h | 33 + fs/xfs/xfs_health.c | 744 ++ fs/xfs/xfs_healthmon.c | 1260 +++ fs/xfs/xfs_healthmon.h | 184 + fs/xfs/xfs_hooks.c | 52 + fs/xfs/xfs_hooks.h | 65 + fs/xfs/xfs_icache.c | 2356 +++++ fs/xfs/xfs_icache.h | 84 + fs/xfs/xfs_icreate_item.c | 260 + fs/xfs/xfs_icreate_item.h | 22 + fs/xfs/xfs_inode.c | 3077 ++++++ fs/xfs/xfs_inode.h | 676 ++ fs/xfs/xfs_inode_item.c | 1247 +++ fs/xfs/xfs_inode_item.h | 62 + fs/xfs/xfs_inode_item_recover.c | 603 ++ fs/xfs/xfs_ioctl.c | 1472 +++ fs/xfs/xfs_ioctl.h | 44 + fs/xfs/xfs_ioctl32.c | 518 + fs/xfs/xfs_ioctl32.h | 193 + fs/xfs/xfs_iomap.c | 2410 +++++ fs/xfs/xfs_iomap.h | 66 + fs/xfs/xfs_iops.c | 1513 +++ fs/xfs/xfs_iops.h | 26 + fs/xfs/xfs_itable.c | 484 + fs/xfs/xfs_itable.h | 69 + fs/xfs/xfs_iunlink_item.c | 177 + fs/xfs/xfs_iunlink_item.h | 27 + fs/xfs/xfs_iwalk.c | 754 ++ fs/xfs/xfs_iwalk.h | 44 + fs/xfs/xfs_log.c | 3477 ++++++ fs/xfs/xfs_log.h | 130 + fs/xfs/xfs_log_cil.c | 2069 ++++ fs/xfs/xfs_log_priv.h | 747 ++ fs/xfs/xfs_log_recover.c | 3587 +++++++ fs/xfs/xfs_message.c | 160 + fs/xfs/xfs_message.h | 101 + fs/xfs/xfs_mount.c | 1699 +++ fs/xfs/xfs_mount.h | 833 ++ fs/xfs/xfs_mru_cache.c | 541 + fs/xfs/xfs_mru_cache.h | 34 + fs/xfs/xfs_notify_failure.c | 397 + fs/xfs/xfs_notify_failure.h | 11 + fs/xfs/xfs_platform.h | 303 + fs/xfs/xfs_pnfs.c | 357 + fs/xfs/xfs_pnfs.h | 20 + fs/xfs/xfs_pwork.c | 121 + fs/xfs/xfs_pwork.h | 59 + fs/xfs/xfs_qm.c | 2137 ++++ fs/xfs/xfs_qm.h | 191 + fs/xfs/xfs_qm_bhv.c | 192 + fs/xfs/xfs_qm_syscalls.c | 546 + fs/xfs/xfs_quota.h | 230 + fs/xfs/xfs_quotaops.c | 289 + fs/xfs/xfs_refcount_item.c | 862 ++ fs/xfs/xfs_refcount_item.h | 82 + fs/xfs/xfs_reflink.c | 1964 ++++ fs/xfs/xfs_reflink.h | 73 + fs/xfs/xfs_rmap_item.c | 891 ++ fs/xfs/xfs_rmap_item.h | 81 + fs/xfs/xfs_rtalloc.c | 2223 ++++ fs/xfs/xfs_rtalloc.h | 80 + fs/xfs/xfs_stats.c | 179 + fs/xfs/xfs_stats.h | 230 + fs/xfs/xfs_super.c | 2774 +++++ fs/xfs/xfs_super.h | 104 + fs/xfs/xfs_symlink.c | 362 + fs/xfs/xfs_symlink.h | 16 + fs/xfs/xfs_sysctl.c | 197 + fs/xfs/xfs_sysctl.h | 100 + fs/xfs/xfs_sysfs.c | 898 ++ fs/xfs/xfs_sysfs.h | 60 + fs/xfs/xfs_trace.c | 65 + fs/xfs/xfs_trace.h | 6488 ++++++++++++ fs/xfs/xfs_trans.c | 1444 +++ fs/xfs/xfs_trans.h | 283 + fs/xfs/xfs_trans_ail.c | 980 ++ fs/xfs/xfs_trans_buf.c | 793 ++ fs/xfs/xfs_trans_dquot.c | 1065 ++ fs/xfs/xfs_trans_priv.h | 170 + fs/xfs/xfs_verify_media.c | 443 + fs/xfs/xfs_verify_media.h | 13 + fs/xfs/xfs_xattr.c | 347 + fs/xfs/xfs_xattr.h | 14 + fs/xfs/xfs_zone_alloc.c | 1447 +++ fs/xfs/xfs_zone_alloc.h | 75 + fs/xfs/xfs_zone_gc.c | 1244 +++ fs/xfs/xfs_zone_info.c | 110 + fs/xfs/xfs_zone_priv.h | 112 + fs/xfs/xfs_zone_space_resv.c | 262 + fs/zonefs/Kconfig | 11 + fs/zonefs/Makefile | 6 + fs/zonefs/file.c | 878 ++ fs/zonefs/super.c | 1476 +++ fs/zonefs/sysfs.c | 135 + fs/zonefs/trace.h | 106 + fs/zonefs/zonefs.h | 289 + 2345 files changed, 1657958 insertions(+) create mode 100644 fs/9p/Kconfig create mode 100644 fs/9p/Makefile create mode 100644 fs/9p/acl.c create mode 100644 fs/9p/acl.h create mode 100644 fs/9p/cache.c create mode 100644 fs/9p/cache.h create mode 100644 fs/9p/fid.c create mode 100644 fs/9p/fid.h create mode 100644 fs/9p/v9fs.c create mode 100644 fs/9p/v9fs.h create mode 100644 fs/9p/v9fs_vfs.h create mode 100644 fs/9p/vfs_addr.c create mode 100644 fs/9p/vfs_dentry.c create mode 100644 fs/9p/vfs_dir.c create mode 100644 fs/9p/vfs_file.c create mode 100644 fs/9p/vfs_inode.c create mode 100644 fs/9p/vfs_inode_dotl.c create mode 100644 fs/9p/vfs_super.c create mode 100644 fs/9p/xattr.c create mode 100644 fs/9p/xattr.h create mode 100644 fs/Kconfig create mode 100644 fs/Kconfig.binfmt create mode 100644 fs/Makefile create mode 100644 fs/adfs/Kconfig create mode 100644 fs/adfs/Makefile create mode 100644 fs/adfs/adfs.h create mode 100644 fs/adfs/dir.c create mode 100644 fs/adfs/dir_f.c create mode 100644 fs/adfs/dir_f.h create mode 100644 fs/adfs/dir_fplus.c create mode 100644 fs/adfs/dir_fplus.h create mode 100644 fs/adfs/file.c create mode 100644 fs/adfs/inode.c create mode 100644 fs/adfs/map.c create mode 100644 fs/adfs/super.c create mode 100644 fs/affs/Changes create mode 100644 fs/affs/Kconfig create mode 100644 fs/affs/Makefile create mode 100644 fs/affs/affs.h create mode 100644 fs/affs/amigaffs.c create mode 100644 fs/affs/amigaffs.h create mode 100644 fs/affs/bitmap.c create mode 100644 fs/affs/dir.c create mode 100644 fs/affs/file.c create mode 100644 fs/affs/inode.c create mode 100644 fs/affs/namei.c create mode 100644 fs/affs/super.c create mode 100644 fs/affs/symlink.c create mode 100644 fs/afs/Kconfig create mode 100644 fs/afs/Makefile create mode 100644 fs/afs/addr_list.c create mode 100644 fs/afs/addr_prefs.c create mode 100644 fs/afs/afs.h create mode 100644 fs/afs/afs_cm.h create mode 100644 fs/afs/afs_fs.h create mode 100644 fs/afs/afs_vl.h create mode 100644 fs/afs/callback.c create mode 100644 fs/afs/cell.c create mode 100644 fs/afs/cm_security.c create mode 100644 fs/afs/cmservice.c create mode 100644 fs/afs/dir.c create mode 100644 fs/afs/dir_edit.c create mode 100644 fs/afs/dir_search.c create mode 100644 fs/afs/dir_silly.c create mode 100644 fs/afs/dynroot.c create mode 100644 fs/afs/file.c create mode 100644 fs/afs/flock.c create mode 100644 fs/afs/fs_operation.c create mode 100644 fs/afs/fs_probe.c create mode 100644 fs/afs/fsclient.c create mode 100644 fs/afs/inode.c create mode 100644 fs/afs/internal.h create mode 100644 fs/afs/main.c create mode 100644 fs/afs/misc.c create mode 100644 fs/afs/mntpt.c create mode 100644 fs/afs/proc.c create mode 100644 fs/afs/protocol_afs.h create mode 100644 fs/afs/protocol_uae.h create mode 100644 fs/afs/protocol_yfs.h create mode 100644 fs/afs/rotate.c create mode 100644 fs/afs/rxrpc.c create mode 100644 fs/afs/security.c create mode 100644 fs/afs/server.c create mode 100644 fs/afs/server_list.c create mode 100644 fs/afs/super.c create mode 100644 fs/afs/symlink.c create mode 100644 fs/afs/validation.c create mode 100644 fs/afs/vl_alias.c create mode 100644 fs/afs/vl_list.c create mode 100644 fs/afs/vl_probe.c create mode 100644 fs/afs/vl_rotate.c create mode 100644 fs/afs/vlclient.c create mode 100644 fs/afs/volume.c create mode 100644 fs/afs/write.c create mode 100644 fs/afs/xattr.c create mode 100644 fs/afs/xdr_fs.h create mode 100644 fs/afs/yfsclient.c create mode 100644 fs/aio.c create mode 100644 fs/anon_inodes.c create mode 100644 fs/attr.c create mode 100644 fs/autofs/Kconfig create mode 100644 fs/autofs/Makefile create mode 100644 fs/autofs/autofs_i.h create mode 100644 fs/autofs/dev-ioctl.c create mode 100644 fs/autofs/expire.c create mode 100644 fs/autofs/init.c create mode 100644 fs/autofs/inode.c create mode 100644 fs/autofs/root.c create mode 100644 fs/autofs/symlink.c create mode 100644 fs/autofs/waitq.c create mode 100644 fs/backing-file.c create mode 100644 fs/bad_inode.c create mode 100644 fs/befs/ChangeLog create mode 100644 fs/befs/Kconfig create mode 100644 fs/befs/Makefile create mode 100644 fs/befs/befs.h create mode 100644 fs/befs/befs_fs_types.h create mode 100644 fs/befs/btree.c create mode 100644 fs/befs/btree.h create mode 100644 fs/befs/datastream.c create mode 100644 fs/befs/datastream.h create mode 100644 fs/befs/debug.c create mode 100644 fs/befs/endian.h create mode 100644 fs/befs/inode.c create mode 100644 fs/befs/inode.h create mode 100644 fs/befs/io.c create mode 100644 fs/befs/io.h create mode 100644 fs/befs/linuxvfs.c create mode 100644 fs/befs/super.c create mode 100644 fs/befs/super.h create mode 100644 fs/bfs/Kconfig create mode 100644 fs/bfs/Makefile create mode 100644 fs/bfs/bfs.h create mode 100644 fs/bfs/dir.c create mode 100644 fs/bfs/file.c create mode 100644 fs/bfs/inode.c create mode 100644 fs/binfmt_elf.c create mode 100644 fs/binfmt_elf_fdpic.c create mode 100644 fs/binfmt_flat.c create mode 100644 fs/binfmt_misc.c create mode 100644 fs/binfmt_misc_bpf.c create mode 100644 fs/binfmt_script.c create mode 100644 fs/bpf_fs_kfuncs.c create mode 100644 fs/btrfs/Kconfig create mode 100644 fs/btrfs/Makefile create mode 100644 fs/btrfs/accessors.c create mode 100644 fs/btrfs/accessors.h create mode 100644 fs/btrfs/acl.c create mode 100644 fs/btrfs/acl.h create mode 100644 fs/btrfs/async-thread.c create mode 100644 fs/btrfs/async-thread.h create mode 100644 fs/btrfs/backref.c create mode 100644 fs/btrfs/backref.h create mode 100644 fs/btrfs/bio.c create mode 100644 fs/btrfs/bio.h create mode 100644 fs/btrfs/block-group.c create mode 100644 fs/btrfs/block-group.h create mode 100644 fs/btrfs/block-rsv.c create mode 100644 fs/btrfs/block-rsv.h create mode 100644 fs/btrfs/btrfs_inode.h create mode 100644 fs/btrfs/compression.c create mode 100644 fs/btrfs/compression.h create mode 100644 fs/btrfs/ctree.c create mode 100644 fs/btrfs/ctree.h create mode 100644 fs/btrfs/defrag.c create mode 100644 fs/btrfs/defrag.h create mode 100644 fs/btrfs/delalloc-space.c create mode 100644 fs/btrfs/delalloc-space.h create mode 100644 fs/btrfs/delayed-inode.c create mode 100644 fs/btrfs/delayed-inode.h create mode 100644 fs/btrfs/delayed-ref.c create mode 100644 fs/btrfs/delayed-ref.h create mode 100644 fs/btrfs/dev-replace.c create mode 100644 fs/btrfs/dev-replace.h create mode 100644 fs/btrfs/dir-item.c create mode 100644 fs/btrfs/dir-item.h create mode 100644 fs/btrfs/direct-io.c create mode 100644 fs/btrfs/direct-io.h create mode 100644 fs/btrfs/discard.c create mode 100644 fs/btrfs/discard.h create mode 100644 fs/btrfs/disk-io.c create mode 100644 fs/btrfs/disk-io.h create mode 100644 fs/btrfs/export.c create mode 100644 fs/btrfs/export.h create mode 100644 fs/btrfs/extent-io-tree.c create mode 100644 fs/btrfs/extent-io-tree.h create mode 100644 fs/btrfs/extent-tree.c create mode 100644 fs/btrfs/extent-tree.h create mode 100644 fs/btrfs/extent_io.c create mode 100644 fs/btrfs/extent_io.h create mode 100644 fs/btrfs/extent_map.c create mode 100644 fs/btrfs/extent_map.h create mode 100644 fs/btrfs/fiemap.c create mode 100644 fs/btrfs/fiemap.h create mode 100644 fs/btrfs/file-item.c create mode 100644 fs/btrfs/file-item.h create mode 100644 fs/btrfs/file.c create mode 100644 fs/btrfs/file.h create mode 100644 fs/btrfs/free-space-cache.c create mode 100644 fs/btrfs/free-space-cache.h create mode 100644 fs/btrfs/free-space-tree.c create mode 100644 fs/btrfs/free-space-tree.h create mode 100644 fs/btrfs/fs.c create mode 100644 fs/btrfs/fs.h create mode 100644 fs/btrfs/inode-item.c create mode 100644 fs/btrfs/inode-item.h create mode 100644 fs/btrfs/inode.c create mode 100644 fs/btrfs/ioctl.c create mode 100644 fs/btrfs/ioctl.h create mode 100644 fs/btrfs/locking.c create mode 100644 fs/btrfs/locking.h create mode 100644 fs/btrfs/lru_cache.c create mode 100644 fs/btrfs/lru_cache.h create mode 100644 fs/btrfs/lzo.c create mode 100644 fs/btrfs/messages.c create mode 100644 fs/btrfs/messages.h create mode 100644 fs/btrfs/misc.h create mode 100644 fs/btrfs/ordered-data.c create mode 100644 fs/btrfs/ordered-data.h create mode 100644 fs/btrfs/orphan.c create mode 100644 fs/btrfs/orphan.h create mode 100644 fs/btrfs/print-tree.c create mode 100644 fs/btrfs/print-tree.h create mode 100644 fs/btrfs/props.c create mode 100644 fs/btrfs/props.h create mode 100644 fs/btrfs/qgroup.c create mode 100644 fs/btrfs/qgroup.h create mode 100644 fs/btrfs/raid-stripe-tree.c create mode 100644 fs/btrfs/raid-stripe-tree.h create mode 100644 fs/btrfs/raid56.c create mode 100644 fs/btrfs/raid56.h create mode 100644 fs/btrfs/ref-verify.c create mode 100644 fs/btrfs/ref-verify.h create mode 100644 fs/btrfs/reflink.c create mode 100644 fs/btrfs/reflink.h create mode 100644 fs/btrfs/relocation.c create mode 100644 fs/btrfs/relocation.h create mode 100644 fs/btrfs/root-tree.c create mode 100644 fs/btrfs/root-tree.h create mode 100644 fs/btrfs/scrub.c create mode 100644 fs/btrfs/scrub.h create mode 100644 fs/btrfs/send.c create mode 100644 fs/btrfs/send.h create mode 100644 fs/btrfs/space-info.c create mode 100644 fs/btrfs/space-info.h create mode 100644 fs/btrfs/subpage.c create mode 100644 fs/btrfs/subpage.h create mode 100644 fs/btrfs/super.c create mode 100644 fs/btrfs/super.h create mode 100644 fs/btrfs/sysfs.c create mode 100644 fs/btrfs/sysfs.h create mode 100644 fs/btrfs/tests/btrfs-tests.c create mode 100644 fs/btrfs/tests/btrfs-tests.h create mode 100644 fs/btrfs/tests/chunk-allocation-tests.c create mode 100644 fs/btrfs/tests/delayed-refs-tests.c create mode 100644 fs/btrfs/tests/extent-buffer-tests.c create mode 100644 fs/btrfs/tests/extent-io-tests.c create mode 100644 fs/btrfs/tests/extent-map-tests.c create mode 100644 fs/btrfs/tests/free-space-tests.c create mode 100644 fs/btrfs/tests/free-space-tree-tests.c create mode 100644 fs/btrfs/tests/inode-tests.c create mode 100644 fs/btrfs/tests/qgroup-tests.c create mode 100644 fs/btrfs/tests/raid-stripe-tree-tests.c create mode 100644 fs/btrfs/tests/zoned-tests.c create mode 100644 fs/btrfs/transaction.c create mode 100644 fs/btrfs/transaction.h create mode 100644 fs/btrfs/tree-checker.c create mode 100644 fs/btrfs/tree-checker.h create mode 100644 fs/btrfs/tree-log.c create mode 100644 fs/btrfs/tree-log.h create mode 100644 fs/btrfs/tree-mod-log.c create mode 100644 fs/btrfs/tree-mod-log.h create mode 100644 fs/btrfs/ulist.c create mode 100644 fs/btrfs/ulist.h create mode 100644 fs/btrfs/uuid-tree.c create mode 100644 fs/btrfs/uuid-tree.h create mode 100644 fs/btrfs/verity.c create mode 100644 fs/btrfs/verity.h create mode 100644 fs/btrfs/volumes.c create mode 100644 fs/btrfs/volumes.h create mode 100644 fs/btrfs/xattr.c create mode 100644 fs/btrfs/xattr.h create mode 100644 fs/btrfs/zlib.c create mode 100644 fs/btrfs/zoned.c create mode 100644 fs/btrfs/zoned.h create mode 100644 fs/btrfs/zstd.c create mode 100644 fs/buffer.c create mode 100644 fs/cachefiles/Kconfig create mode 100644 fs/cachefiles/Makefile create mode 100644 fs/cachefiles/cache.c create mode 100644 fs/cachefiles/daemon.c create mode 100644 fs/cachefiles/error_inject.c create mode 100644 fs/cachefiles/interface.c create mode 100644 fs/cachefiles/internal.h create mode 100644 fs/cachefiles/io.c create mode 100644 fs/cachefiles/key.c create mode 100644 fs/cachefiles/main.c create mode 100644 fs/cachefiles/namei.c create mode 100644 fs/cachefiles/security.c create mode 100644 fs/cachefiles/volume.c create mode 100644 fs/cachefiles/xattr.c create mode 100644 fs/ceph/Kconfig create mode 100644 fs/ceph/Makefile create mode 100644 fs/ceph/acl.c create mode 100644 fs/ceph/addr.c create mode 100644 fs/ceph/cache.c create mode 100644 fs/ceph/cache.h create mode 100644 fs/ceph/caps.c create mode 100644 fs/ceph/ceph_frag.c create mode 100644 fs/ceph/crypto.c create mode 100644 fs/ceph/crypto.h create mode 100644 fs/ceph/debugfs.c create mode 100644 fs/ceph/dir.c create mode 100644 fs/ceph/export.c create mode 100644 fs/ceph/file.c create mode 100644 fs/ceph/inode.c create mode 100644 fs/ceph/io.c create mode 100644 fs/ceph/io.h create mode 100644 fs/ceph/ioctl.c create mode 100644 fs/ceph/ioctl.h create mode 100644 fs/ceph/locks.c create mode 100644 fs/ceph/mds_client.c create mode 100644 fs/ceph/mds_client.h create mode 100644 fs/ceph/mdsmap.c create mode 100644 fs/ceph/mdsmap.h create mode 100644 fs/ceph/metric.c create mode 100644 fs/ceph/metric.h create mode 100644 fs/ceph/quota.c create mode 100644 fs/ceph/snap.c create mode 100644 fs/ceph/strings.c create mode 100644 fs/ceph/subvolume_metrics.c create mode 100644 fs/ceph/subvolume_metrics.h create mode 100644 fs/ceph/super.c create mode 100644 fs/ceph/super.h create mode 100644 fs/ceph/util.c create mode 100644 fs/ceph/xattr.c create mode 100644 fs/char_dev.c create mode 100644 fs/coda/Kconfig create mode 100644 fs/coda/Makefile create mode 100644 fs/coda/cache.c create mode 100644 fs/coda/cnode.c create mode 100644 fs/coda/coda_cache.h create mode 100644 fs/coda/coda_fs_i.h create mode 100644 fs/coda/coda_int.h create mode 100644 fs/coda/coda_linux.c create mode 100644 fs/coda/coda_linux.h create mode 100644 fs/coda/coda_psdev.h create mode 100644 fs/coda/dir.c create mode 100644 fs/coda/file.c create mode 100644 fs/coda/inode.c create mode 100644 fs/coda/pioctl.c create mode 100644 fs/coda/psdev.c create mode 100644 fs/coda/symlink.c create mode 100644 fs/coda/sysctl.c create mode 100644 fs/coda/upcall.c create mode 100644 fs/compat_binfmt_elf.c create mode 100644 fs/configfs/Kconfig create mode 100644 fs/configfs/Makefile create mode 100644 fs/configfs/configfs_internal.h create mode 100644 fs/configfs/dir.c create mode 100644 fs/configfs/file.c create mode 100644 fs/configfs/inode.c create mode 100644 fs/configfs/item.c create mode 100644 fs/configfs/mount.c create mode 100644 fs/configfs/symlink.c create mode 100644 fs/coredump.c create mode 100644 fs/cramfs/Kconfig create mode 100644 fs/cramfs/Makefile create mode 100644 fs/cramfs/README create mode 100644 fs/cramfs/inode.c create mode 100644 fs/cramfs/internal.h create mode 100644 fs/cramfs/uncompress.c create mode 100644 fs/crypto/Kconfig create mode 100644 fs/crypto/Makefile create mode 100644 fs/crypto/block.c create mode 100644 fs/crypto/crypto.c create mode 100644 fs/crypto/fname.c create mode 100644 fs/crypto/fscrypt_private.h create mode 100644 fs/crypto/hkdf.c create mode 100644 fs/crypto/hooks.c create mode 100644 fs/crypto/keyring.c create mode 100644 fs/crypto/keysetup.c create mode 100644 fs/crypto/keysetup_v1.c create mode 100644 fs/crypto/policy.c create mode 100644 fs/d_path.c create mode 100644 fs/dax.c create mode 100644 fs/dcache.c create mode 100644 fs/debugfs/Makefile create mode 100644 fs/debugfs/file.c create mode 100644 fs/debugfs/inode.c create mode 100644 fs/debugfs/internal.h create mode 100644 fs/devpts/Makefile create mode 100644 fs/devpts/inode.c create mode 100644 fs/direct-io.c create mode 100644 fs/dlm/Kconfig create mode 100644 fs/dlm/Makefile create mode 100644 fs/dlm/ast.c create mode 100644 fs/dlm/ast.h create mode 100644 fs/dlm/config.c create mode 100644 fs/dlm/config.h create mode 100644 fs/dlm/debug_fs.c create mode 100644 fs/dlm/dir.c create mode 100644 fs/dlm/dir.h create mode 100644 fs/dlm/dlm_internal.h create mode 100644 fs/dlm/lock.c create mode 100644 fs/dlm/lock.h create mode 100644 fs/dlm/lockspace.c create mode 100644 fs/dlm/lockspace.h create mode 100644 fs/dlm/lowcomms.c create mode 100644 fs/dlm/lowcomms.h create mode 100644 fs/dlm/lvb_table.h create mode 100644 fs/dlm/main.c create mode 100644 fs/dlm/member.c create mode 100644 fs/dlm/member.h create mode 100644 fs/dlm/memory.c create mode 100644 fs/dlm/memory.h create mode 100644 fs/dlm/midcomms.c create mode 100644 fs/dlm/midcomms.h create mode 100644 fs/dlm/plock.c create mode 100644 fs/dlm/rcom.c create mode 100644 fs/dlm/rcom.h create mode 100644 fs/dlm/recover.c create mode 100644 fs/dlm/recover.h create mode 100644 fs/dlm/recoverd.c create mode 100644 fs/dlm/recoverd.h create mode 100644 fs/dlm/requestqueue.c create mode 100644 fs/dlm/requestqueue.h create mode 100644 fs/dlm/user.c create mode 100644 fs/dlm/user.h create mode 100644 fs/dlm/util.c create mode 100644 fs/dlm/util.h create mode 100644 fs/drop_caches.c create mode 100644 fs/ecryptfs/Kconfig create mode 100644 fs/ecryptfs/Makefile create mode 100644 fs/ecryptfs/crypto.c create mode 100644 fs/ecryptfs/debug.c create mode 100644 fs/ecryptfs/dentry.c create mode 100644 fs/ecryptfs/ecryptfs_kernel.h create mode 100644 fs/ecryptfs/file.c create mode 100644 fs/ecryptfs/inode.c create mode 100644 fs/ecryptfs/keystore.c create mode 100644 fs/ecryptfs/kthread.c create mode 100644 fs/ecryptfs/main.c create mode 100644 fs/ecryptfs/messaging.c create mode 100644 fs/ecryptfs/miscdev.c create mode 100644 fs/ecryptfs/mmap.c create mode 100644 fs/ecryptfs/read_write.c create mode 100644 fs/ecryptfs/super.c create mode 100644 fs/efivarfs/Kconfig create mode 100644 fs/efivarfs/Makefile create mode 100644 fs/efivarfs/file.c create mode 100644 fs/efivarfs/inode.c create mode 100644 fs/efivarfs/internal.h create mode 100644 fs/efivarfs/super.c create mode 100644 fs/efivarfs/vars.c create mode 100644 fs/erofs/Kconfig create mode 100644 fs/erofs/Makefile create mode 100644 fs/erofs/compress.h create mode 100644 fs/erofs/data.c create mode 100644 fs/erofs/decompressor.c create mode 100644 fs/erofs/decompressor_crypto.c create mode 100644 fs/erofs/decompressor_deflate.c create mode 100644 fs/erofs/decompressor_lzma.c create mode 100644 fs/erofs/decompressor_zstd.c create mode 100644 fs/erofs/dir.c create mode 100644 fs/erofs/erofs_fs.h create mode 100644 fs/erofs/fileio.c create mode 100644 fs/erofs/inode.c create mode 100644 fs/erofs/internal.h create mode 100644 fs/erofs/ishare.c create mode 100644 fs/erofs/namei.c create mode 100644 fs/erofs/super.c create mode 100644 fs/erofs/sysfs.c create mode 100644 fs/erofs/xattr.c create mode 100644 fs/erofs/xattr.h create mode 100644 fs/erofs/zdata.c create mode 100644 fs/erofs/zmap.c create mode 100644 fs/erofs/zutil.c create mode 100644 fs/eventfd.c create mode 100644 fs/eventpoll.c create mode 100644 fs/exec.c create mode 100644 fs/exfat/Kconfig create mode 100644 fs/exfat/Makefile create mode 100644 fs/exfat/balloc.c create mode 100644 fs/exfat/cache.c create mode 100644 fs/exfat/dir.c create mode 100644 fs/exfat/exfat_fs.h create mode 100644 fs/exfat/exfat_raw.h create mode 100644 fs/exfat/fatent.c create mode 100644 fs/exfat/file.c create mode 100644 fs/exfat/inode.c create mode 100644 fs/exfat/iomap.c create mode 100644 fs/exfat/iomap.h create mode 100644 fs/exfat/misc.c create mode 100644 fs/exfat/namei.c create mode 100644 fs/exfat/nls.c create mode 100644 fs/exfat/super.c create mode 100644 fs/exportfs/Makefile create mode 100644 fs/exportfs/expfs.c create mode 100644 fs/ext2/Kconfig create mode 100644 fs/ext2/Makefile create mode 100644 fs/ext2/acl.c create mode 100644 fs/ext2/acl.h create mode 100644 fs/ext2/balloc.c create mode 100644 fs/ext2/dir.c create mode 100644 fs/ext2/ext2.h create mode 100644 fs/ext2/file.c create mode 100644 fs/ext2/ialloc.c create mode 100644 fs/ext2/inode.c create mode 100644 fs/ext2/ioctl.c create mode 100644 fs/ext2/namei.c create mode 100644 fs/ext2/super.c create mode 100644 fs/ext2/symlink.c create mode 100644 fs/ext2/trace.c create mode 100644 fs/ext2/trace.h create mode 100644 fs/ext2/xattr.c create mode 100644 fs/ext2/xattr.h create mode 100644 fs/ext2/xattr_security.c create mode 100644 fs/ext2/xattr_trusted.c create mode 100644 fs/ext2/xattr_user.c create mode 100644 fs/ext4/.kunitconfig create mode 100644 fs/ext4/Kconfig create mode 100644 fs/ext4/Makefile create mode 100644 fs/ext4/acl.c create mode 100644 fs/ext4/acl.h create mode 100644 fs/ext4/balloc.c create mode 100644 fs/ext4/bitmap.c create mode 100644 fs/ext4/block_validity.c create mode 100644 fs/ext4/crypto.c create mode 100644 fs/ext4/dir.c create mode 100644 fs/ext4/ext4.h create mode 100644 fs/ext4/ext4_extents.h create mode 100644 fs/ext4/ext4_jbd2.c create mode 100644 fs/ext4/ext4_jbd2.h create mode 100644 fs/ext4/extents-test.c create mode 100644 fs/ext4/extents.c create mode 100644 fs/ext4/extents_status.c create mode 100644 fs/ext4/extents_status.h create mode 100644 fs/ext4/fast_commit.c create mode 100644 fs/ext4/fast_commit.h create mode 100644 fs/ext4/file.c create mode 100644 fs/ext4/fsmap.c create mode 100644 fs/ext4/fsmap.h create mode 100644 fs/ext4/fsync.c create mode 100644 fs/ext4/hash-test.c create mode 100644 fs/ext4/hash.c create mode 100644 fs/ext4/ialloc.c create mode 100644 fs/ext4/indirect.c create mode 100644 fs/ext4/inline.c create mode 100644 fs/ext4/inode-test.c create mode 100644 fs/ext4/inode.c create mode 100644 fs/ext4/ioctl.c create mode 100644 fs/ext4/mballoc-test.c create mode 100644 fs/ext4/mballoc.c create mode 100644 fs/ext4/mballoc.h create mode 100644 fs/ext4/migrate.c create mode 100644 fs/ext4/mmp.c create mode 100644 fs/ext4/move_extent.c create mode 100644 fs/ext4/namei.c create mode 100644 fs/ext4/orphan.c create mode 100644 fs/ext4/page-io.c create mode 100644 fs/ext4/readpage.c create mode 100644 fs/ext4/resize.c create mode 100644 fs/ext4/super.c create mode 100644 fs/ext4/symlink.c create mode 100644 fs/ext4/sysfs.c create mode 100644 fs/ext4/truncate.h create mode 100644 fs/ext4/verity.c create mode 100644 fs/ext4/xattr.c create mode 100644 fs/ext4/xattr.h create mode 100644 fs/ext4/xattr_hurd.c create mode 100644 fs/ext4/xattr_security.c create mode 100644 fs/ext4/xattr_trusted.c create mode 100644 fs/ext4/xattr_user.c create mode 100644 fs/f2fs/Kconfig create mode 100644 fs/f2fs/Makefile create mode 100644 fs/f2fs/acl.c create mode 100644 fs/f2fs/acl.h create mode 100644 fs/f2fs/checkpoint.c create mode 100644 fs/f2fs/compress.c create mode 100644 fs/f2fs/data.c create mode 100644 fs/f2fs/debug.c create mode 100644 fs/f2fs/dir.c create mode 100644 fs/f2fs/extent_cache.c create mode 100644 fs/f2fs/f2fs.h create mode 100644 fs/f2fs/file.c create mode 100644 fs/f2fs/gc.c create mode 100644 fs/f2fs/gc.h create mode 100644 fs/f2fs/hash.c create mode 100644 fs/f2fs/inline.c create mode 100644 fs/f2fs/inode.c create mode 100644 fs/f2fs/iostat.c create mode 100644 fs/f2fs/iostat.h create mode 100644 fs/f2fs/namei.c create mode 100644 fs/f2fs/node.c create mode 100644 fs/f2fs/node.h create mode 100644 fs/f2fs/recovery.c create mode 100644 fs/f2fs/segment.c create mode 100644 fs/f2fs/segment.h create mode 100644 fs/f2fs/shrinker.c create mode 100644 fs/f2fs/super.c create mode 100644 fs/f2fs/sysfs.c create mode 100644 fs/f2fs/verity.c create mode 100644 fs/f2fs/xattr.c create mode 100644 fs/f2fs/xattr.h create mode 100644 fs/failfs.c create mode 100644 fs/fat/.kunitconfig create mode 100644 fs/fat/Kconfig create mode 100644 fs/fat/Makefile create mode 100644 fs/fat/cache.c create mode 100644 fs/fat/dir.c create mode 100644 fs/fat/fat.h create mode 100644 fs/fat/fat_test.c create mode 100644 fs/fat/fatent.c create mode 100644 fs/fat/file.c create mode 100644 fs/fat/inode.c create mode 100644 fs/fat/misc.c create mode 100644 fs/fat/namei_msdos.c create mode 100644 fs/fat/namei_vfat.c create mode 100644 fs/fat/nfs.c create mode 100644 fs/fcntl.c create mode 100644 fs/fhandle.c create mode 100644 fs/file.c create mode 100644 fs/file_attr.c create mode 100644 fs/file_table.c create mode 100644 fs/filesystems.c create mode 100644 fs/fs-writeback.c create mode 100644 fs/fs_context.c create mode 100644 fs/fs_dirent.c create mode 100644 fs/fs_parser.c create mode 100644 fs/fs_pin.c create mode 100644 fs/fs_struct.c create mode 100644 fs/fserror.c create mode 100644 fs/fsopen.c create mode 100644 fs/fuse/Kconfig create mode 100644 fs/fuse/Makefile create mode 100644 fs/fuse/acl.c create mode 100644 fs/fuse/args.h create mode 100644 fs/fuse/backing.c create mode 100644 fs/fuse/control.c create mode 100644 fs/fuse/cuse.c create mode 100644 fs/fuse/dax.c create mode 100644 fs/fuse/dev.c create mode 100644 fs/fuse/dev.h create mode 100644 fs/fuse/dev_uring.c create mode 100644 fs/fuse/dev_uring_i.h create mode 100644 fs/fuse/dir.c create mode 100644 fs/fuse/file.c create mode 100644 fs/fuse/fuse_dev_i.h create mode 100644 fs/fuse/fuse_i.h create mode 100644 fs/fuse/fuse_trace.h create mode 100644 fs/fuse/inode.c create mode 100644 fs/fuse/ioctl.c create mode 100644 fs/fuse/iomode.c create mode 100644 fs/fuse/notify.c create mode 100644 fs/fuse/passthrough.c create mode 100644 fs/fuse/poll.c create mode 100644 fs/fuse/readdir.c create mode 100644 fs/fuse/req.c create mode 100644 fs/fuse/req_timeout.c create mode 100644 fs/fuse/sysctl.c create mode 100644 fs/fuse/sysctl.h create mode 100644 fs/fuse/trace.c create mode 100644 fs/fuse/virtio_fs.c create mode 100644 fs/fuse/xattr.c create mode 100644 fs/gfs2/Kconfig create mode 100644 fs/gfs2/Makefile create mode 100644 fs/gfs2/acl.c create mode 100644 fs/gfs2/acl.h create mode 100644 fs/gfs2/aops.c create mode 100644 fs/gfs2/aops.h create mode 100644 fs/gfs2/bmap.c create mode 100644 fs/gfs2/bmap.h create mode 100644 fs/gfs2/dentry.c create mode 100644 fs/gfs2/dir.c create mode 100644 fs/gfs2/dir.h create mode 100644 fs/gfs2/export.c create mode 100644 fs/gfs2/file.c create mode 100644 fs/gfs2/gfs2.h create mode 100644 fs/gfs2/glock.c create mode 100644 fs/gfs2/glock.h create mode 100644 fs/gfs2/glops.c create mode 100644 fs/gfs2/glops.h create mode 100644 fs/gfs2/incore.h create mode 100644 fs/gfs2/inode.c create mode 100644 fs/gfs2/inode.h create mode 100644 fs/gfs2/lock_dlm.c create mode 100644 fs/gfs2/log.c create mode 100644 fs/gfs2/log.h create mode 100644 fs/gfs2/lops.c create mode 100644 fs/gfs2/lops.h create mode 100644 fs/gfs2/main.c create mode 100644 fs/gfs2/meta_io.c create mode 100644 fs/gfs2/meta_io.h create mode 100644 fs/gfs2/ops_fstype.c create mode 100644 fs/gfs2/quota.c create mode 100644 fs/gfs2/quota.h create mode 100644 fs/gfs2/recovery.c create mode 100644 fs/gfs2/recovery.h create mode 100644 fs/gfs2/rgrp.c create mode 100644 fs/gfs2/rgrp.h create mode 100644 fs/gfs2/super.c create mode 100644 fs/gfs2/super.h create mode 100644 fs/gfs2/sys.c create mode 100644 fs/gfs2/sys.h create mode 100644 fs/gfs2/trace_gfs2.h create mode 100644 fs/gfs2/trans.c create mode 100644 fs/gfs2/trans.h create mode 100644 fs/gfs2/util.c create mode 100644 fs/gfs2/util.h create mode 100644 fs/gfs2/xattr.c create mode 100644 fs/gfs2/xattr.h create mode 100644 fs/hfs/.kunitconfig create mode 100644 fs/hfs/Kconfig create mode 100644 fs/hfs/Makefile create mode 100644 fs/hfs/attr.c create mode 100644 fs/hfs/bfind.c create mode 100644 fs/hfs/bitmap.c create mode 100644 fs/hfs/bnode.c create mode 100644 fs/hfs/brec.c create mode 100644 fs/hfs/btree.c create mode 100644 fs/hfs/btree.h create mode 100644 fs/hfs/catalog.c create mode 100644 fs/hfs/dir.c create mode 100644 fs/hfs/extent.c create mode 100644 fs/hfs/hfs.h create mode 100644 fs/hfs/hfs_fs.h create mode 100644 fs/hfs/inode.c create mode 100644 fs/hfs/mdb.c create mode 100644 fs/hfs/part_tbl.c create mode 100644 fs/hfs/string.c create mode 100644 fs/hfs/string_test.c create mode 100644 fs/hfs/super.c create mode 100644 fs/hfs/sysdep.c create mode 100644 fs/hfs/trans.c create mode 100644 fs/hfsplus/.kunitconfig create mode 100644 fs/hfsplus/Kconfig create mode 100644 fs/hfsplus/Makefile create mode 100644 fs/hfsplus/attributes.c create mode 100644 fs/hfsplus/bfind.c create mode 100644 fs/hfsplus/bitmap.c create mode 100644 fs/hfsplus/bnode.c create mode 100644 fs/hfsplus/brec.c create mode 100644 fs/hfsplus/btree.c create mode 100644 fs/hfsplus/catalog.c create mode 100644 fs/hfsplus/dir.c create mode 100644 fs/hfsplus/extents.c create mode 100644 fs/hfsplus/hfsplus_fs.h create mode 100644 fs/hfsplus/hfsplus_raw.h create mode 100644 fs/hfsplus/inode.c create mode 100644 fs/hfsplus/ioctl.c create mode 100644 fs/hfsplus/options.c create mode 100644 fs/hfsplus/part_tbl.c create mode 100644 fs/hfsplus/super.c create mode 100644 fs/hfsplus/tables.c create mode 100644 fs/hfsplus/unicode.c create mode 100644 fs/hfsplus/unicode_test.c create mode 100644 fs/hfsplus/wrapper.c create mode 100644 fs/hfsplus/xattr.c create mode 100644 fs/hfsplus/xattr.h create mode 100644 fs/hfsplus/xattr_security.c create mode 100644 fs/hfsplus/xattr_trusted.c create mode 100644 fs/hfsplus/xattr_user.c create mode 100644 fs/hostfs/Makefile create mode 100644 fs/hostfs/hostfs.h create mode 100644 fs/hostfs/hostfs_kern.c create mode 100644 fs/hostfs/hostfs_user.c create mode 100644 fs/hostfs/hostfs_user_exp.c create mode 100644 fs/hpfs/Kconfig create mode 100644 fs/hpfs/Makefile create mode 100644 fs/hpfs/alloc.c create mode 100644 fs/hpfs/anode.c create mode 100644 fs/hpfs/buffer.c create mode 100644 fs/hpfs/dentry.c create mode 100644 fs/hpfs/dir.c create mode 100644 fs/hpfs/dnode.c create mode 100644 fs/hpfs/ea.c create mode 100644 fs/hpfs/file.c create mode 100644 fs/hpfs/hpfs.h create mode 100644 fs/hpfs/hpfs_fn.h create mode 100644 fs/hpfs/inode.c create mode 100644 fs/hpfs/map.c create mode 100644 fs/hpfs/name.c create mode 100644 fs/hpfs/namei.c create mode 100644 fs/hpfs/super.c create mode 100644 fs/hugetlbfs/Makefile create mode 100644 fs/hugetlbfs/inode.c create mode 100644 fs/init.c create mode 100644 fs/inode.c create mode 100644 fs/internal.h create mode 100644 fs/ioctl.c create mode 100644 fs/iomap/Makefile create mode 100644 fs/iomap/bio.c create mode 100644 fs/iomap/buffered-io.c create mode 100644 fs/iomap/direct-io.c create mode 100644 fs/iomap/fiemap.c create mode 100644 fs/iomap/internal.h create mode 100644 fs/iomap/ioend.c create mode 100644 fs/iomap/iter.c create mode 100644 fs/iomap/seek.c create mode 100644 fs/iomap/swapfile.c create mode 100644 fs/iomap/trace.c create mode 100644 fs/iomap/trace.h create mode 100644 fs/isofs/Kconfig create mode 100644 fs/isofs/Makefile create mode 100644 fs/isofs/compress.c create mode 100644 fs/isofs/dir.c create mode 100644 fs/isofs/export.c create mode 100644 fs/isofs/inode.c create mode 100644 fs/isofs/isofs.h create mode 100644 fs/isofs/joliet.c create mode 100644 fs/isofs/namei.c create mode 100644 fs/isofs/rock.c create mode 100644 fs/isofs/rock.h create mode 100644 fs/isofs/util.c create mode 100644 fs/isofs/zisofs.h create mode 100644 fs/jbd2/Kconfig create mode 100644 fs/jbd2/Makefile create mode 100644 fs/jbd2/checkpoint.c create mode 100644 fs/jbd2/commit.c create mode 100644 fs/jbd2/journal.c create mode 100644 fs/jbd2/recovery.c create mode 100644 fs/jbd2/revoke.c create mode 100644 fs/jbd2/transaction.c create mode 100644 fs/jffs2/Kconfig create mode 100644 fs/jffs2/LICENCE create mode 100644 fs/jffs2/Makefile create mode 100644 fs/jffs2/README.Locking create mode 100644 fs/jffs2/acl.c create mode 100644 fs/jffs2/acl.h create mode 100644 fs/jffs2/background.c create mode 100644 fs/jffs2/build.c create mode 100644 fs/jffs2/compr.c create mode 100644 fs/jffs2/compr.h create mode 100644 fs/jffs2/compr_lzo.c create mode 100644 fs/jffs2/compr_rtime.c create mode 100644 fs/jffs2/compr_rubin.c create mode 100644 fs/jffs2/compr_zlib.c create mode 100644 fs/jffs2/debug.c create mode 100644 fs/jffs2/debug.h create mode 100644 fs/jffs2/dir.c create mode 100644 fs/jffs2/erase.c create mode 100644 fs/jffs2/file.c create mode 100644 fs/jffs2/fs.c create mode 100644 fs/jffs2/gc.c create mode 100644 fs/jffs2/ioctl.c create mode 100644 fs/jffs2/jffs2_fs_i.h create mode 100644 fs/jffs2/jffs2_fs_sb.h create mode 100644 fs/jffs2/malloc.c create mode 100644 fs/jffs2/nodelist.c create mode 100644 fs/jffs2/nodelist.h create mode 100644 fs/jffs2/nodemgmt.c create mode 100644 fs/jffs2/os-linux.h create mode 100644 fs/jffs2/read.c create mode 100644 fs/jffs2/readinode.c create mode 100644 fs/jffs2/scan.c create mode 100644 fs/jffs2/security.c create mode 100644 fs/jffs2/summary.c create mode 100644 fs/jffs2/summary.h create mode 100644 fs/jffs2/super.c create mode 100644 fs/jffs2/symlink.c create mode 100644 fs/jffs2/wbuf.c create mode 100644 fs/jffs2/write.c create mode 100644 fs/jffs2/writev.c create mode 100644 fs/jffs2/xattr.c create mode 100644 fs/jffs2/xattr.h create mode 100644 fs/jffs2/xattr_trusted.c create mode 100644 fs/jffs2/xattr_user.c create mode 100644 fs/jfs/Kconfig create mode 100644 fs/jfs/Makefile create mode 100644 fs/jfs/acl.c create mode 100644 fs/jfs/file.c create mode 100644 fs/jfs/inode.c create mode 100644 fs/jfs/ioctl.c create mode 100644 fs/jfs/jfs_acl.h create mode 100644 fs/jfs/jfs_btree.h create mode 100644 fs/jfs/jfs_debug.c create mode 100644 fs/jfs/jfs_debug.h create mode 100644 fs/jfs/jfs_dinode.h create mode 100644 fs/jfs/jfs_discard.c create mode 100644 fs/jfs/jfs_discard.h create mode 100644 fs/jfs/jfs_dmap.c create mode 100644 fs/jfs/jfs_dmap.h create mode 100644 fs/jfs/jfs_dtree.c create mode 100644 fs/jfs/jfs_dtree.h create mode 100644 fs/jfs/jfs_extent.c create mode 100644 fs/jfs/jfs_extent.h create mode 100644 fs/jfs/jfs_filsys.h create mode 100644 fs/jfs/jfs_imap.c create mode 100644 fs/jfs/jfs_imap.h create mode 100644 fs/jfs/jfs_incore.h create mode 100644 fs/jfs/jfs_inode.c create mode 100644 fs/jfs/jfs_inode.h create mode 100644 fs/jfs/jfs_lock.h create mode 100644 fs/jfs/jfs_logmgr.c create mode 100644 fs/jfs/jfs_logmgr.h create mode 100644 fs/jfs/jfs_metapage.c create mode 100644 fs/jfs/jfs_metapage.h create mode 100644 fs/jfs/jfs_mount.c create mode 100644 fs/jfs/jfs_superblock.h create mode 100644 fs/jfs/jfs_txnmgr.c create mode 100644 fs/jfs/jfs_txnmgr.h create mode 100644 fs/jfs/jfs_types.h create mode 100644 fs/jfs/jfs_umount.c create mode 100644 fs/jfs/jfs_unicode.c create mode 100644 fs/jfs/jfs_unicode.h create mode 100644 fs/jfs/jfs_xattr.h create mode 100644 fs/jfs/jfs_xtree.c create mode 100644 fs/jfs/jfs_xtree.h create mode 100644 fs/jfs/namei.c create mode 100644 fs/jfs/resize.c create mode 100644 fs/jfs/super.c create mode 100644 fs/jfs/symlink.c create mode 100644 fs/jfs/xattr.c create mode 100644 fs/kernel_read_file.c create mode 100644 fs/kernfs/Kconfig create mode 100644 fs/kernfs/Makefile create mode 100644 fs/kernfs/dir.c create mode 100644 fs/kernfs/file.c create mode 100644 fs/kernfs/inode.c create mode 100644 fs/kernfs/kernfs-internal.h create mode 100644 fs/kernfs/mount.c create mode 100644 fs/kernfs/symlink.c create mode 100644 fs/libfs.c create mode 100644 fs/lockd/Makefile create mode 100644 fs/lockd/clnt4xdr.c create mode 100644 fs/lockd/clntlock.c create mode 100644 fs/lockd/clntproc.c create mode 100644 fs/lockd/clntxdr.c create mode 100644 fs/lockd/host.c create mode 100644 fs/lockd/lockd.h create mode 100644 fs/lockd/mon.c create mode 100644 fs/lockd/netlink.c create mode 100644 fs/lockd/netlink.h create mode 100644 fs/lockd/netns.h create mode 100644 fs/lockd/nlm.h create mode 100644 fs/lockd/nlm3xdr_gen.c create mode 100644 fs/lockd/nlm3xdr_gen.h create mode 100644 fs/lockd/nlm4xdr_gen.c create mode 100644 fs/lockd/nlm4xdr_gen.h create mode 100644 fs/lockd/procfs.c create mode 100644 fs/lockd/procfs.h create mode 100644 fs/lockd/share.h create mode 100644 fs/lockd/svc.c create mode 100644 fs/lockd/svc4proc.c create mode 100644 fs/lockd/svclock.c create mode 100644 fs/lockd/svcproc.c create mode 100644 fs/lockd/svcshare.c create mode 100644 fs/lockd/svcsubs.c create mode 100644 fs/lockd/trace.c create mode 100644 fs/lockd/trace.h create mode 100644 fs/lockd/xdr.h create mode 100644 fs/locks.c create mode 100644 fs/mbcache.c create mode 100644 fs/minix/Kconfig create mode 100644 fs/minix/Makefile create mode 100644 fs/minix/bitmap.c create mode 100644 fs/minix/dir.c create mode 100644 fs/minix/file.c create mode 100644 fs/minix/inode.c create mode 100644 fs/minix/itree_common.c create mode 100644 fs/minix/itree_v1.c create mode 100644 fs/minix/itree_v2.c create mode 100644 fs/minix/minix.h create mode 100644 fs/minix/namei.c create mode 100644 fs/mnt_idmapping.c create mode 100644 fs/mount.h create mode 100644 fs/mpage.c create mode 100644 fs/namei.c create mode 100644 fs/namespace.c create mode 100644 fs/netfs/Kconfig create mode 100644 fs/netfs/Makefile create mode 100644 fs/netfs/buffered_read.c create mode 100644 fs/netfs/buffered_write.c create mode 100644 fs/netfs/direct_read.c create mode 100644 fs/netfs/direct_write.c create mode 100644 fs/netfs/fscache_cache.c create mode 100644 fs/netfs/fscache_cookie.c create mode 100644 fs/netfs/fscache_internal.h create mode 100644 fs/netfs/fscache_io.c create mode 100644 fs/netfs/fscache_main.c create mode 100644 fs/netfs/fscache_proc.c create mode 100644 fs/netfs/fscache_stats.c create mode 100644 fs/netfs/fscache_volume.c create mode 100644 fs/netfs/internal.h create mode 100644 fs/netfs/iterator.c create mode 100644 fs/netfs/locking.c create mode 100644 fs/netfs/main.c create mode 100644 fs/netfs/misc.c create mode 100644 fs/netfs/objects.c create mode 100644 fs/netfs/read_collect.c create mode 100644 fs/netfs/read_pgpriv2.c create mode 100644 fs/netfs/read_retry.c create mode 100644 fs/netfs/read_single.c create mode 100644 fs/netfs/rolling_buffer.c create mode 100644 fs/netfs/stats.c create mode 100644 fs/netfs/write_collect.c create mode 100644 fs/netfs/write_issue.c create mode 100644 fs/netfs/write_retry.c create mode 100644 fs/nfs/Kconfig create mode 100644 fs/nfs/Makefile create mode 100644 fs/nfs/blocklayout/Makefile create mode 100644 fs/nfs/blocklayout/blocklayout.c create mode 100644 fs/nfs/blocklayout/blocklayout.h create mode 100644 fs/nfs/blocklayout/dev.c create mode 100644 fs/nfs/blocklayout/extent_tree.c create mode 100644 fs/nfs/blocklayout/rpc_pipefs.c create mode 100644 fs/nfs/cache_lib.c create mode 100644 fs/nfs/cache_lib.h create mode 100644 fs/nfs/callback.c create mode 100644 fs/nfs/callback.h create mode 100644 fs/nfs/callback_proc.c create mode 100644 fs/nfs/callback_xdr.c create mode 100644 fs/nfs/client.c create mode 100644 fs/nfs/delegation.c create mode 100644 fs/nfs/delegation.h create mode 100644 fs/nfs/dir.c create mode 100644 fs/nfs/direct.c create mode 100644 fs/nfs/dns_resolve.c create mode 100644 fs/nfs/dns_resolve.h create mode 100644 fs/nfs/export.c create mode 100644 fs/nfs/file.c create mode 100644 fs/nfs/filelayout/Makefile create mode 100644 fs/nfs/filelayout/filelayout.c create mode 100644 fs/nfs/filelayout/filelayout.h create mode 100644 fs/nfs/filelayout/filelayoutdev.c create mode 100644 fs/nfs/flexfilelayout/Makefile create mode 100644 fs/nfs/flexfilelayout/flexfilelayout.c create mode 100644 fs/nfs/flexfilelayout/flexfilelayout.h create mode 100644 fs/nfs/flexfilelayout/flexfilelayoutdev.c create mode 100644 fs/nfs/fs_context.c create mode 100644 fs/nfs/fscache.c create mode 100644 fs/nfs/fscache.h create mode 100644 fs/nfs/getroot.c create mode 100644 fs/nfs/inode.c create mode 100644 fs/nfs/internal.h create mode 100644 fs/nfs/io.c create mode 100644 fs/nfs/iostat.h create mode 100644 fs/nfs/localio.c create mode 100644 fs/nfs/mount_clnt.c create mode 100644 fs/nfs/namespace.c create mode 100644 fs/nfs/netns.h create mode 100644 fs/nfs/nfs.h create mode 100644 fs/nfs/nfs2super.c create mode 100644 fs/nfs/nfs2xdr.c create mode 100644 fs/nfs/nfs3_fs.h create mode 100644 fs/nfs/nfs3acl.c create mode 100644 fs/nfs/nfs3client.c create mode 100644 fs/nfs/nfs3proc.c create mode 100644 fs/nfs/nfs3super.c create mode 100644 fs/nfs/nfs3xdr.c create mode 100644 fs/nfs/nfs40.h create mode 100644 fs/nfs/nfs40client.c create mode 100644 fs/nfs/nfs40proc.c create mode 100644 fs/nfs/nfs42.h create mode 100644 fs/nfs/nfs42proc.c create mode 100644 fs/nfs/nfs42xattr.c create mode 100644 fs/nfs/nfs42xdr.c create mode 100644 fs/nfs/nfs4_fs.h create mode 100644 fs/nfs/nfs4client.c create mode 100644 fs/nfs/nfs4file.c create mode 100644 fs/nfs/nfs4getroot.c create mode 100644 fs/nfs/nfs4idmap.c create mode 100644 fs/nfs/nfs4idmap.h create mode 100644 fs/nfs/nfs4namespace.c create mode 100644 fs/nfs/nfs4proc.c create mode 100644 fs/nfs/nfs4renewd.c create mode 100644 fs/nfs/nfs4session.c create mode 100644 fs/nfs/nfs4session.h create mode 100644 fs/nfs/nfs4state.c create mode 100644 fs/nfs/nfs4super.c create mode 100644 fs/nfs/nfs4sysctl.c create mode 100644 fs/nfs/nfs4trace.c create mode 100644 fs/nfs/nfs4trace.h create mode 100644 fs/nfs/nfs4xdr.c create mode 100644 fs/nfs/nfsroot.c create mode 100644 fs/nfs/nfstrace.c create mode 100644 fs/nfs/nfstrace.h create mode 100644 fs/nfs/pagelist.c create mode 100644 fs/nfs/pnfs.c create mode 100644 fs/nfs/pnfs.h create mode 100644 fs/nfs/pnfs_dev.c create mode 100644 fs/nfs/pnfs_nfs.c create mode 100644 fs/nfs/proc.c create mode 100644 fs/nfs/read.c create mode 100644 fs/nfs/super.c create mode 100644 fs/nfs/symlink.c create mode 100644 fs/nfs/sysctl.c create mode 100644 fs/nfs/sysfs.c create mode 100644 fs/nfs/sysfs.h create mode 100644 fs/nfs/unlink.c create mode 100644 fs/nfs/write.c create mode 100644 fs/nfs_common/Makefile create mode 100644 fs/nfs_common/common.c create mode 100644 fs/nfs_common/grace.c create mode 100644 fs/nfs_common/localio_trace.c create mode 100644 fs/nfs_common/localio_trace.h create mode 100644 fs/nfs_common/nfs_ssc.c create mode 100644 fs/nfs_common/nfsacl.c create mode 100644 fs/nfs_common/nfslocalio.c create mode 100644 fs/nfsd/Kconfig create mode 100644 fs/nfsd/Makefile create mode 100644 fs/nfsd/acl.h create mode 100644 fs/nfsd/attr4.h create mode 100644 fs/nfsd/auth.c create mode 100644 fs/nfsd/auth.h create mode 100644 fs/nfsd/blocklayout.c create mode 100644 fs/nfsd/blocklayoutxdr.c create mode 100644 fs/nfsd/blocklayoutxdr.h create mode 100644 fs/nfsd/cache.h create mode 100644 fs/nfsd/current_stateid.h create mode 100644 fs/nfsd/debugfs.c create mode 100644 fs/nfsd/export.c create mode 100644 fs/nfsd/export.h create mode 100644 fs/nfsd/filecache.c create mode 100644 fs/nfsd/filecache.h create mode 100644 fs/nfsd/flexfilelayout.c create mode 100644 fs/nfsd/flexfilelayoutxdr.c create mode 100644 fs/nfsd/flexfilelayoutxdr.h create mode 100644 fs/nfsd/idmap.h create mode 100644 fs/nfsd/localio.c create mode 100644 fs/nfsd/lockd.c create mode 100644 fs/nfsd/netlink.c create mode 100644 fs/nfsd/netlink.h create mode 100644 fs/nfsd/netns.h create mode 100644 fs/nfsd/nfs2acl.c create mode 100644 fs/nfsd/nfs3acl.c create mode 100644 fs/nfsd/nfs3proc.c create mode 100644 fs/nfsd/nfs3xdr.c create mode 100644 fs/nfsd/nfs4acl.c create mode 100644 fs/nfsd/nfs4callback.c create mode 100644 fs/nfsd/nfs4idmap.c create mode 100644 fs/nfsd/nfs4layouts.c create mode 100644 fs/nfsd/nfs4proc.c create mode 100644 fs/nfsd/nfs4recover.c create mode 100644 fs/nfsd/nfs4state.c create mode 100644 fs/nfsd/nfs4xdr.c create mode 100644 fs/nfsd/nfs4xdr_gen.c create mode 100644 fs/nfsd/nfs4xdr_gen.h create mode 100644 fs/nfsd/nfscache.c create mode 100644 fs/nfsd/nfsctl.c create mode 100644 fs/nfsd/nfsd.h create mode 100644 fs/nfsd/nfsfh.c create mode 100644 fs/nfsd/nfsfh.h create mode 100644 fs/nfsd/nfsproc.c create mode 100644 fs/nfsd/nfssvc.c create mode 100644 fs/nfsd/nfsxdr.c create mode 100644 fs/nfsd/pnfs.h create mode 100644 fs/nfsd/state.h create mode 100644 fs/nfsd/stats.c create mode 100644 fs/nfsd/stats.h create mode 100644 fs/nfsd/trace.c create mode 100644 fs/nfsd/trace.h create mode 100644 fs/nfsd/vfs.c create mode 100644 fs/nfsd/vfs.h create mode 100644 fs/nfsd/xdr.h create mode 100644 fs/nfsd/xdr3.h create mode 100644 fs/nfsd/xdr4.h create mode 100644 fs/nfsd/xdr4cb.h create mode 100644 fs/nilfs2/Kconfig create mode 100644 fs/nilfs2/Makefile create mode 100644 fs/nilfs2/alloc.c create mode 100644 fs/nilfs2/alloc.h create mode 100644 fs/nilfs2/bmap.c create mode 100644 fs/nilfs2/bmap.h create mode 100644 fs/nilfs2/btnode.c create mode 100644 fs/nilfs2/btnode.h create mode 100644 fs/nilfs2/btree.c create mode 100644 fs/nilfs2/btree.h create mode 100644 fs/nilfs2/cpfile.c create mode 100644 fs/nilfs2/cpfile.h create mode 100644 fs/nilfs2/dat.c create mode 100644 fs/nilfs2/dat.h create mode 100644 fs/nilfs2/dir.c create mode 100644 fs/nilfs2/direct.c create mode 100644 fs/nilfs2/direct.h create mode 100644 fs/nilfs2/export.h create mode 100644 fs/nilfs2/file.c create mode 100644 fs/nilfs2/gcinode.c create mode 100644 fs/nilfs2/ifile.c create mode 100644 fs/nilfs2/ifile.h create mode 100644 fs/nilfs2/inode.c create mode 100644 fs/nilfs2/ioctl.c create mode 100644 fs/nilfs2/mdt.c create mode 100644 fs/nilfs2/mdt.h create mode 100644 fs/nilfs2/namei.c create mode 100644 fs/nilfs2/nilfs.h create mode 100644 fs/nilfs2/page.c create mode 100644 fs/nilfs2/page.h create mode 100644 fs/nilfs2/recovery.c create mode 100644 fs/nilfs2/segbuf.c create mode 100644 fs/nilfs2/segbuf.h create mode 100644 fs/nilfs2/segment.c create mode 100644 fs/nilfs2/segment.h create mode 100644 fs/nilfs2/sufile.c create mode 100644 fs/nilfs2/sufile.h create mode 100644 fs/nilfs2/super.c create mode 100644 fs/nilfs2/sysfs.c create mode 100644 fs/nilfs2/sysfs.h create mode 100644 fs/nilfs2/the_nilfs.c create mode 100644 fs/nilfs2/the_nilfs.h create mode 100644 fs/nls/Kconfig create mode 100644 fs/nls/Makefile create mode 100644 fs/nls/mac-celtic.c create mode 100644 fs/nls/mac-centeuro.c create mode 100644 fs/nls/mac-croatian.c create mode 100644 fs/nls/mac-cyrillic.c create mode 100644 fs/nls/mac-gaelic.c create mode 100644 fs/nls/mac-greek.c create mode 100644 fs/nls/mac-iceland.c create mode 100644 fs/nls/mac-inuit.c create mode 100644 fs/nls/mac-roman.c create mode 100644 fs/nls/mac-romanian.c create mode 100644 fs/nls/mac-turkish.c create mode 100644 fs/nls/nls_ascii.c create mode 100644 fs/nls/nls_base.c create mode 100644 fs/nls/nls_cp1250.c create mode 100644 fs/nls/nls_cp1251.c create mode 100644 fs/nls/nls_cp1255.c create mode 100644 fs/nls/nls_cp437.c create mode 100644 fs/nls/nls_cp737.c create mode 100644 fs/nls/nls_cp775.c create mode 100644 fs/nls/nls_cp850.c create mode 100644 fs/nls/nls_cp852.c create mode 100644 fs/nls/nls_cp855.c create mode 100644 fs/nls/nls_cp857.c create mode 100644 fs/nls/nls_cp860.c create mode 100644 fs/nls/nls_cp861.c create mode 100644 fs/nls/nls_cp862.c create mode 100644 fs/nls/nls_cp863.c create mode 100644 fs/nls/nls_cp864.c create mode 100644 fs/nls/nls_cp865.c create mode 100644 fs/nls/nls_cp866.c create mode 100644 fs/nls/nls_cp869.c create mode 100644 fs/nls/nls_cp874.c create mode 100644 fs/nls/nls_cp932.c create mode 100644 fs/nls/nls_cp936.c create mode 100644 fs/nls/nls_cp949.c create mode 100644 fs/nls/nls_cp950.c create mode 100644 fs/nls/nls_euc-jp.c create mode 100644 fs/nls/nls_iso8859-1.c create mode 100644 fs/nls/nls_iso8859-13.c create mode 100644 fs/nls/nls_iso8859-14.c create mode 100644 fs/nls/nls_iso8859-15.c create mode 100644 fs/nls/nls_iso8859-2.c create mode 100644 fs/nls/nls_iso8859-3.c create mode 100644 fs/nls/nls_iso8859-4.c create mode 100644 fs/nls/nls_iso8859-5.c create mode 100644 fs/nls/nls_iso8859-6.c create mode 100644 fs/nls/nls_iso8859-7.c create mode 100644 fs/nls/nls_iso8859-9.c create mode 100644 fs/nls/nls_koi8-r.c create mode 100644 fs/nls/nls_koi8-ru.c create mode 100644 fs/nls/nls_koi8-u.c create mode 100644 fs/nls/nls_ucs2_data.h create mode 100644 fs/nls/nls_ucs2_utils.c create mode 100644 fs/nls/nls_ucs2_utils.h create mode 100644 fs/nls/nls_utf8.c create mode 100644 fs/notify/Kconfig create mode 100644 fs/notify/Makefile create mode 100644 fs/notify/dnotify/Kconfig create mode 100644 fs/notify/dnotify/Makefile create mode 100644 fs/notify/dnotify/dnotify.c create mode 100644 fs/notify/fanotify/Kconfig create mode 100644 fs/notify/fanotify/Makefile create mode 100644 fs/notify/fanotify/fanotify.c create mode 100644 fs/notify/fanotify/fanotify.h create mode 100644 fs/notify/fanotify/fanotify_user.c create mode 100644 fs/notify/fdinfo.c create mode 100644 fs/notify/fdinfo.h create mode 100644 fs/notify/fsnotify.c create mode 100644 fs/notify/fsnotify.h create mode 100644 fs/notify/group.c create mode 100644 fs/notify/inotify/Kconfig create mode 100644 fs/notify/inotify/Makefile create mode 100644 fs/notify/inotify/inotify.h create mode 100644 fs/notify/inotify/inotify_fsnotify.c create mode 100644 fs/notify/inotify/inotify_user.c create mode 100644 fs/notify/mark.c create mode 100644 fs/notify/notification.c create mode 100644 fs/nsfs.c create mode 100644 fs/ntfs/Kconfig create mode 100644 fs/ntfs/Makefile create mode 100644 fs/ntfs/aops.c create mode 100644 fs/ntfs/attrib.c create mode 100644 fs/ntfs/attrib.h create mode 100644 fs/ntfs/attrlist.c create mode 100644 fs/ntfs/attrlist.h create mode 100644 fs/ntfs/bdev-io.c create mode 100644 fs/ntfs/bitmap.c create mode 100644 fs/ntfs/bitmap.h create mode 100644 fs/ntfs/collate.c create mode 100644 fs/ntfs/collate.h create mode 100644 fs/ntfs/compress.c create mode 100644 fs/ntfs/debug.c create mode 100644 fs/ntfs/debug.h create mode 100644 fs/ntfs/dir.c create mode 100644 fs/ntfs/dir.h create mode 100644 fs/ntfs/ea.c create mode 100644 fs/ntfs/ea.h create mode 100644 fs/ntfs/file.c create mode 100644 fs/ntfs/index.c create mode 100644 fs/ntfs/index.h create mode 100644 fs/ntfs/inode.c create mode 100644 fs/ntfs/inode.h create mode 100644 fs/ntfs/iomap.c create mode 100644 fs/ntfs/iomap.h create mode 100644 fs/ntfs/layout.h create mode 100644 fs/ntfs/lcnalloc.c create mode 100644 fs/ntfs/lcnalloc.h create mode 100644 fs/ntfs/logfile.c create mode 100644 fs/ntfs/logfile.h create mode 100644 fs/ntfs/malloc.h create mode 100644 fs/ntfs/mft.c create mode 100644 fs/ntfs/mft.h create mode 100644 fs/ntfs/mst.c create mode 100644 fs/ntfs/namei.c create mode 100644 fs/ntfs/ntfs.h create mode 100644 fs/ntfs/object_id.c create mode 100644 fs/ntfs/object_id.h create mode 100644 fs/ntfs/reparse.c create mode 100644 fs/ntfs/reparse.h create mode 100644 fs/ntfs/runlist.c create mode 100644 fs/ntfs/runlist.h create mode 100644 fs/ntfs/super.c create mode 100644 fs/ntfs/sysctl.c create mode 100644 fs/ntfs/sysctl.h create mode 100644 fs/ntfs/time.h create mode 100644 fs/ntfs/unistr.c create mode 100644 fs/ntfs/upcase.c create mode 100644 fs/ntfs/volume.h create mode 100644 fs/ntfs3/Kconfig create mode 100644 fs/ntfs3/Makefile create mode 100644 fs/ntfs3/attrib.c create mode 100644 fs/ntfs3/attrlist.c create mode 100644 fs/ntfs3/bitfunc.c create mode 100644 fs/ntfs3/bitmap.c create mode 100644 fs/ntfs3/debug.h create mode 100644 fs/ntfs3/dir.c create mode 100644 fs/ntfs3/file.c create mode 100644 fs/ntfs3/frecord.c create mode 100644 fs/ntfs3/fslog.c create mode 100644 fs/ntfs3/fsntfs.c create mode 100644 fs/ntfs3/index.c create mode 100644 fs/ntfs3/inode.c create mode 100644 fs/ntfs3/lib/decompress_common.c create mode 100644 fs/ntfs3/lib/decompress_common.h create mode 100644 fs/ntfs3/lib/lib.h create mode 100644 fs/ntfs3/lib/lzx_decompress.c create mode 100644 fs/ntfs3/lib/xpress_decompress.c create mode 100644 fs/ntfs3/lznt.c create mode 100644 fs/ntfs3/namei.c create mode 100644 fs/ntfs3/ntfs.h create mode 100644 fs/ntfs3/ntfs_fs.h create mode 100644 fs/ntfs3/record.c create mode 100644 fs/ntfs3/run.c create mode 100644 fs/ntfs3/super.c create mode 100644 fs/ntfs3/upcase.c create mode 100644 fs/ntfs3/xattr.c create mode 100644 fs/nullfs.c create mode 100644 fs/ocfs2/Kconfig create mode 100644 fs/ocfs2/Makefile create mode 100644 fs/ocfs2/acl.c create mode 100644 fs/ocfs2/acl.h create mode 100644 fs/ocfs2/alloc.c create mode 100644 fs/ocfs2/alloc.h create mode 100644 fs/ocfs2/aops.c create mode 100644 fs/ocfs2/aops.h create mode 100644 fs/ocfs2/blockcheck.c create mode 100644 fs/ocfs2/blockcheck.h create mode 100644 fs/ocfs2/buffer_head_io.c create mode 100644 fs/ocfs2/buffer_head_io.h create mode 100644 fs/ocfs2/cluster/Makefile create mode 100644 fs/ocfs2/cluster/heartbeat.c create mode 100644 fs/ocfs2/cluster/heartbeat.h create mode 100644 fs/ocfs2/cluster/masklog.c create mode 100644 fs/ocfs2/cluster/masklog.h create mode 100644 fs/ocfs2/cluster/netdebug.c create mode 100644 fs/ocfs2/cluster/nodemanager.c create mode 100644 fs/ocfs2/cluster/nodemanager.h create mode 100644 fs/ocfs2/cluster/ocfs2_heartbeat.h create mode 100644 fs/ocfs2/cluster/ocfs2_nodemanager.h create mode 100644 fs/ocfs2/cluster/quorum.c create mode 100644 fs/ocfs2/cluster/quorum.h create mode 100644 fs/ocfs2/cluster/sys.c create mode 100644 fs/ocfs2/cluster/sys.h create mode 100644 fs/ocfs2/cluster/tcp.c create mode 100644 fs/ocfs2/cluster/tcp.h create mode 100644 fs/ocfs2/cluster/tcp_internal.h create mode 100644 fs/ocfs2/dcache.c create mode 100644 fs/ocfs2/dcache.h create mode 100644 fs/ocfs2/dir.c create mode 100644 fs/ocfs2/dir.h create mode 100644 fs/ocfs2/dlm/Makefile create mode 100644 fs/ocfs2/dlm/dlmapi.h create mode 100644 fs/ocfs2/dlm/dlmast.c create mode 100644 fs/ocfs2/dlm/dlmcommon.h create mode 100644 fs/ocfs2/dlm/dlmconvert.c create mode 100644 fs/ocfs2/dlm/dlmconvert.h create mode 100644 fs/ocfs2/dlm/dlmdebug.c create mode 100644 fs/ocfs2/dlm/dlmdebug.h create mode 100644 fs/ocfs2/dlm/dlmdomain.c create mode 100644 fs/ocfs2/dlm/dlmdomain.h create mode 100644 fs/ocfs2/dlm/dlmlock.c create mode 100644 fs/ocfs2/dlm/dlmmaster.c create mode 100644 fs/ocfs2/dlm/dlmrecovery.c create mode 100644 fs/ocfs2/dlm/dlmthread.c create mode 100644 fs/ocfs2/dlm/dlmunlock.c create mode 100644 fs/ocfs2/dlmfs/Makefile create mode 100644 fs/ocfs2/dlmfs/dlmfs.c create mode 100644 fs/ocfs2/dlmfs/userdlm.c create mode 100644 fs/ocfs2/dlmfs/userdlm.h create mode 100644 fs/ocfs2/dlmglue.c create mode 100644 fs/ocfs2/dlmglue.h create mode 100644 fs/ocfs2/export.c create mode 100644 fs/ocfs2/export.h create mode 100644 fs/ocfs2/extent_map.c create mode 100644 fs/ocfs2/extent_map.h create mode 100644 fs/ocfs2/file.c create mode 100644 fs/ocfs2/file.h create mode 100644 fs/ocfs2/filecheck.c create mode 100644 fs/ocfs2/filecheck.h create mode 100644 fs/ocfs2/heartbeat.c create mode 100644 fs/ocfs2/heartbeat.h create mode 100644 fs/ocfs2/inode.c create mode 100644 fs/ocfs2/inode.h create mode 100644 fs/ocfs2/ioctl.c create mode 100644 fs/ocfs2/ioctl.h create mode 100644 fs/ocfs2/journal.c create mode 100644 fs/ocfs2/journal.h create mode 100644 fs/ocfs2/localalloc.c create mode 100644 fs/ocfs2/localalloc.h create mode 100644 fs/ocfs2/locks.c create mode 100644 fs/ocfs2/locks.h create mode 100644 fs/ocfs2/mmap.c create mode 100644 fs/ocfs2/mmap.h create mode 100644 fs/ocfs2/move_extents.c create mode 100644 fs/ocfs2/move_extents.h create mode 100644 fs/ocfs2/namei.c create mode 100644 fs/ocfs2/namei.h create mode 100644 fs/ocfs2/ocfs1_fs_compat.h create mode 100644 fs/ocfs2/ocfs2.h create mode 100644 fs/ocfs2/ocfs2_fs.h create mode 100644 fs/ocfs2/ocfs2_ioctl.h create mode 100644 fs/ocfs2/ocfs2_lockid.h create mode 100644 fs/ocfs2/ocfs2_lockingver.h create mode 100644 fs/ocfs2/ocfs2_trace.h create mode 100644 fs/ocfs2/quota.h create mode 100644 fs/ocfs2/quota_global.c create mode 100644 fs/ocfs2/quota_local.c create mode 100644 fs/ocfs2/refcounttree.c create mode 100644 fs/ocfs2/refcounttree.h create mode 100644 fs/ocfs2/reservations.c create mode 100644 fs/ocfs2/reservations.h create mode 100644 fs/ocfs2/resize.c create mode 100644 fs/ocfs2/resize.h create mode 100644 fs/ocfs2/slot_map.c create mode 100644 fs/ocfs2/slot_map.h create mode 100644 fs/ocfs2/stack_o2cb.c create mode 100644 fs/ocfs2/stack_user.c create mode 100644 fs/ocfs2/stackglue.c create mode 100644 fs/ocfs2/stackglue.h create mode 100644 fs/ocfs2/suballoc.c create mode 100644 fs/ocfs2/suballoc.h create mode 100644 fs/ocfs2/super.c create mode 100644 fs/ocfs2/super.h create mode 100644 fs/ocfs2/symlink.c create mode 100644 fs/ocfs2/symlink.h create mode 100644 fs/ocfs2/sysfile.c create mode 100644 fs/ocfs2/sysfile.h create mode 100644 fs/ocfs2/uptodate.c create mode 100644 fs/ocfs2/uptodate.h create mode 100644 fs/ocfs2/xattr.c create mode 100644 fs/ocfs2/xattr.h create mode 100644 fs/omfs/Kconfig create mode 100644 fs/omfs/Makefile create mode 100644 fs/omfs/bitmap.c create mode 100644 fs/omfs/dir.c create mode 100644 fs/omfs/file.c create mode 100644 fs/omfs/inode.c create mode 100644 fs/omfs/omfs.h create mode 100644 fs/omfs/omfs_fs.h create mode 100644 fs/open.c create mode 100644 fs/openpromfs/Makefile create mode 100644 fs/openpromfs/inode.c create mode 100644 fs/orangefs/Kconfig create mode 100644 fs/orangefs/Makefile create mode 100644 fs/orangefs/acl.c create mode 100644 fs/orangefs/dcache.c create mode 100644 fs/orangefs/devorangefs-req.c create mode 100644 fs/orangefs/dir.c create mode 100644 fs/orangefs/downcall.h create mode 100644 fs/orangefs/file.c create mode 100644 fs/orangefs/inode.c create mode 100644 fs/orangefs/namei.c create mode 100644 fs/orangefs/orangefs-bufmap.c create mode 100644 fs/orangefs/orangefs-bufmap.h create mode 100644 fs/orangefs/orangefs-cache.c create mode 100644 fs/orangefs/orangefs-debug.h create mode 100644 fs/orangefs/orangefs-debugfs.c create mode 100644 fs/orangefs/orangefs-debugfs.h create mode 100644 fs/orangefs/orangefs-dev-proto.h create mode 100644 fs/orangefs/orangefs-kernel.h create mode 100644 fs/orangefs/orangefs-mod.c create mode 100644 fs/orangefs/orangefs-sysfs.c create mode 100644 fs/orangefs/orangefs-sysfs.h create mode 100644 fs/orangefs/orangefs-utils.c create mode 100644 fs/orangefs/protocol.h create mode 100644 fs/orangefs/super.c create mode 100644 fs/orangefs/symlink.c create mode 100644 fs/orangefs/upcall.h create mode 100644 fs/orangefs/waitqueue.c create mode 100644 fs/orangefs/xattr.c create mode 100644 fs/overlayfs/Kconfig create mode 100644 fs/overlayfs/Makefile create mode 100644 fs/overlayfs/copy_up.c create mode 100644 fs/overlayfs/dir.c create mode 100644 fs/overlayfs/export.c create mode 100644 fs/overlayfs/file.c create mode 100644 fs/overlayfs/inode.c create mode 100644 fs/overlayfs/namei.c create mode 100644 fs/overlayfs/overlayfs.h create mode 100644 fs/overlayfs/ovl_entry.h create mode 100644 fs/overlayfs/params.c create mode 100644 fs/overlayfs/params.h create mode 100644 fs/overlayfs/readdir.c create mode 100644 fs/overlayfs/super.c create mode 100644 fs/overlayfs/util.c create mode 100644 fs/overlayfs/xattrs.c create mode 100644 fs/pidfs.c create mode 100644 fs/pipe.c create mode 100644 fs/pnode.c create mode 100644 fs/pnode.h create mode 100644 fs/posix_acl.c create mode 100644 fs/proc/Kconfig create mode 100644 fs/proc/Makefile create mode 100644 fs/proc/array.c create mode 100644 fs/proc/base.c create mode 100644 fs/proc/bootconfig.c create mode 100644 fs/proc/cmdline.c create mode 100644 fs/proc/consoles.c create mode 100644 fs/proc/cpuinfo.c create mode 100644 fs/proc/devices.c create mode 100644 fs/proc/fd.c create mode 100644 fs/proc/fd.h create mode 100644 fs/proc/generic.c create mode 100644 fs/proc/inode.c create mode 100644 fs/proc/internal.h create mode 100644 fs/proc/interrupts.c create mode 100644 fs/proc/kcore.c create mode 100644 fs/proc/kmsg.c create mode 100644 fs/proc/loadavg.c create mode 100644 fs/proc/meminfo.c create mode 100644 fs/proc/namespaces.c create mode 100644 fs/proc/nommu.c create mode 100644 fs/proc/page.c create mode 100644 fs/proc/proc_net.c create mode 100644 fs/proc/proc_sysctl.c create mode 100644 fs/proc/proc_tty.c create mode 100644 fs/proc/root.c create mode 100644 fs/proc/self.c create mode 100644 fs/proc/softirqs.c create mode 100644 fs/proc/stat.c create mode 100644 fs/proc/task_mmu.c create mode 100644 fs/proc/task_nommu.c create mode 100644 fs/proc/thread_self.c create mode 100644 fs/proc/uptime.c create mode 100644 fs/proc/util.c create mode 100644 fs/proc/version.c create mode 100644 fs/proc/vmcore.c create mode 100644 fs/proc_namespace.c create mode 100644 fs/pstore/Kconfig create mode 100644 fs/pstore/Makefile create mode 100644 fs/pstore/blk.c create mode 100644 fs/pstore/ftrace.c create mode 100644 fs/pstore/inode.c create mode 100644 fs/pstore/internal.h create mode 100644 fs/pstore/platform.c create mode 100644 fs/pstore/pmsg.c create mode 100644 fs/pstore/ram.c create mode 100644 fs/pstore/ram_core.c create mode 100644 fs/pstore/ram_internal.h create mode 100644 fs/pstore/zone.c create mode 100644 fs/qnx4/Kconfig create mode 100644 fs/qnx4/Makefile create mode 100644 fs/qnx4/bitmap.c create mode 100644 fs/qnx4/dir.c create mode 100644 fs/qnx4/inode.c create mode 100644 fs/qnx4/namei.c create mode 100644 fs/qnx4/qnx4.h create mode 100644 fs/qnx6/Kconfig create mode 100644 fs/qnx6/Makefile create mode 100644 fs/qnx6/dir.c create mode 100644 fs/qnx6/inode.c create mode 100644 fs/qnx6/namei.c create mode 100644 fs/qnx6/qnx6.h create mode 100644 fs/qnx6/super_mmi.c create mode 100644 fs/quota/Kconfig create mode 100644 fs/quota/Makefile create mode 100644 fs/quota/compat.h create mode 100644 fs/quota/dquot.c create mode 100644 fs/quota/kqid.c create mode 100644 fs/quota/netlink.c create mode 100644 fs/quota/quota.c create mode 100644 fs/quota/quota_tree.c create mode 100644 fs/quota/quota_tree.h create mode 100644 fs/quota/quota_v1.c create mode 100644 fs/quota/quota_v2.c create mode 100644 fs/quota/quotaio_v1.h create mode 100644 fs/quota/quotaio_v2.h create mode 100644 fs/ramfs/Makefile create mode 100644 fs/ramfs/file-mmu.c create mode 100644 fs/ramfs/file-nommu.c create mode 100644 fs/ramfs/inode.c create mode 100644 fs/ramfs/internal.h create mode 100644 fs/read_write.c create mode 100644 fs/readdir.c create mode 100644 fs/remap_range.c create mode 100644 fs/resctrl/Kconfig create mode 100644 fs/resctrl/Makefile create mode 100644 fs/resctrl/ctrlmondata.c create mode 100644 fs/resctrl/internal.h create mode 100644 fs/resctrl/monitor.c create mode 100644 fs/resctrl/monitor_trace.h create mode 100644 fs/resctrl/pseudo_lock.c create mode 100644 fs/resctrl/rdtgroup.c create mode 100644 fs/romfs/Kconfig create mode 100644 fs/romfs/Makefile create mode 100644 fs/romfs/internal.h create mode 100644 fs/romfs/mmap-nommu.c create mode 100644 fs/romfs/storage.c create mode 100644 fs/romfs/super.c create mode 100644 fs/select.c create mode 100644 fs/seq_file.c create mode 100644 fs/signalfd.c create mode 100644 fs/smb/Kconfig create mode 100644 fs/smb/Makefile create mode 100644 fs/smb/client/.gitignore create mode 100644 fs/smb/client/Kconfig create mode 100644 fs/smb/client/Makefile create mode 100644 fs/smb/client/asn1.c create mode 100644 fs/smb/client/cached_dir.c create mode 100644 fs/smb/client/cached_dir.h create mode 100644 fs/smb/client/cifs_debug.c create mode 100644 fs/smb/client/cifs_debug.h create mode 100644 fs/smb/client/cifs_fs_sb.h create mode 100644 fs/smb/client/cifs_ioctl.h create mode 100644 fs/smb/client/cifs_spnego.c create mode 100644 fs/smb/client/cifs_spnego.h create mode 100644 fs/smb/client/cifs_spnego_negtokeninit.asn1 create mode 100644 fs/smb/client/cifs_swn.c create mode 100644 fs/smb/client/cifs_swn.h create mode 100644 fs/smb/client/cifs_unicode.c create mode 100644 fs/smb/client/cifs_unicode.h create mode 100644 fs/smb/client/cifsacl.c create mode 100644 fs/smb/client/cifsacl.h create mode 100644 fs/smb/client/cifsencrypt.c create mode 100644 fs/smb/client/cifsfs.c create mode 100644 fs/smb/client/cifsfs.h create mode 100644 fs/smb/client/cifsglob.h create mode 100644 fs/smb/client/cifspdu.h create mode 100644 fs/smb/client/cifsproto.h create mode 100644 fs/smb/client/cifsroot.c create mode 100644 fs/smb/client/cifssmb.c create mode 100644 fs/smb/client/compress.c create mode 100644 fs/smb/client/compress.h create mode 100644 fs/smb/client/connect.c create mode 100644 fs/smb/client/dfs.c create mode 100644 fs/smb/client/dfs.h create mode 100644 fs/smb/client/dfs_cache.c create mode 100644 fs/smb/client/dfs_cache.h create mode 100644 fs/smb/client/dir.c create mode 100644 fs/smb/client/dns_resolve.c create mode 100644 fs/smb/client/dns_resolve.h create mode 100644 fs/smb/client/export.c create mode 100644 fs/smb/client/file.c create mode 100644 fs/smb/client/fs_context.c create mode 100644 fs/smb/client/fs_context.h create mode 100644 fs/smb/client/fscache.c create mode 100644 fs/smb/client/fscache.h create mode 100644 fs/smb/client/gen_smb1_mapping create mode 100644 fs/smb/client/gen_smb2_mapping create mode 100644 fs/smb/client/inode.c create mode 100644 fs/smb/client/ioctl.c create mode 100644 fs/smb/client/link.c create mode 100644 fs/smb/client/misc.c create mode 100644 fs/smb/client/namespace.c create mode 100644 fs/smb/client/netlink.c create mode 100644 fs/smb/client/netlink.h create mode 100644 fs/smb/client/netmisc.c create mode 100644 fs/smb/client/nterr.h create mode 100644 fs/smb/client/ntlmssp.h create mode 100644 fs/smb/client/readdir.c create mode 100644 fs/smb/client/reparse.c create mode 100644 fs/smb/client/reparse.h create mode 100644 fs/smb/client/rfc1002pdu.h create mode 100644 fs/smb/client/sess.c create mode 100644 fs/smb/client/smb1debug.c create mode 100644 fs/smb/client/smb1encrypt.c create mode 100644 fs/smb/client/smb1maperror.c create mode 100644 fs/smb/client/smb1maperror_test.c create mode 100644 fs/smb/client/smb1misc.c create mode 100644 fs/smb/client/smb1ops.c create mode 100644 fs/smb/client/smb1pdu.h create mode 100644 fs/smb/client/smb1proto.h create mode 100644 fs/smb/client/smb1session.c create mode 100644 fs/smb/client/smb1transport.c create mode 100644 fs/smb/client/smb2file.c create mode 100644 fs/smb/client/smb2glob.h create mode 100644 fs/smb/client/smb2inode.c create mode 100644 fs/smb/client/smb2maperror.c create mode 100644 fs/smb/client/smb2maperror_test.c create mode 100644 fs/smb/client/smb2misc.c create mode 100644 fs/smb/client/smb2ops.c create mode 100644 fs/smb/client/smb2pdu.c create mode 100644 fs/smb/client/smb2pdu.h create mode 100644 fs/smb/client/smb2proto.h create mode 100644 fs/smb/client/smb2transport.c create mode 100644 fs/smb/client/smbdirect.c create mode 100644 fs/smb/client/smbdirect.h create mode 100644 fs/smb/client/smbencrypt.c create mode 100644 fs/smb/client/smberr.h create mode 100644 fs/smb/client/trace.c create mode 100644 fs/smb/client/trace.h create mode 100644 fs/smb/client/transport.c create mode 100644 fs/smb/client/unc.c create mode 100644 fs/smb/client/winucase.c create mode 100644 fs/smb/client/xattr.c create mode 100644 fs/smb/common/Makefile create mode 100644 fs/smb/common/cifs_md4.c create mode 100644 fs/smb/common/compress/compress.c create mode 100644 fs/smb/common/compress/compress.h create mode 100644 fs/smb/common/compress/lz77.c create mode 100644 fs/smb/common/compress/lz77.h create mode 100644 fs/smb/common/fscc.h create mode 100644 fs/smb/common/md4.h create mode 100644 fs/smb/common/smb1pdu.h create mode 100644 fs/smb/common/smb2pdu.h create mode 100644 fs/smb/common/smb2status.h create mode 100644 fs/smb/common/smbacl.h create mode 100644 fs/smb/common/smbfsctl.h create mode 100644 fs/smb/common/smbglob.h create mode 100644 fs/smb/server/Kconfig create mode 100644 fs/smb/server/Makefile create mode 100644 fs/smb/server/asn1.c create mode 100644 fs/smb/server/asn1.h create mode 100644 fs/smb/server/auth.c create mode 100644 fs/smb/server/auth.h create mode 100644 fs/smb/server/compress.c create mode 100644 fs/smb/server/compress.h create mode 100644 fs/smb/server/connection.c create mode 100644 fs/smb/server/connection.h create mode 100644 fs/smb/server/crypto_ctx.c create mode 100644 fs/smb/server/crypto_ctx.h create mode 100644 fs/smb/server/glob.h create mode 100644 fs/smb/server/ksmbd_netlink.h create mode 100644 fs/smb/server/ksmbd_spnego_negtokeninit.asn1 create mode 100644 fs/smb/server/ksmbd_spnego_negtokentarg.asn1 create mode 100644 fs/smb/server/ksmbd_work.c create mode 100644 fs/smb/server/ksmbd_work.h create mode 100644 fs/smb/server/mgmt/ksmbd_ida.c create mode 100644 fs/smb/server/mgmt/ksmbd_ida.h create mode 100644 fs/smb/server/mgmt/share_config.c create mode 100644 fs/smb/server/mgmt/share_config.h create mode 100644 fs/smb/server/mgmt/tree_connect.c create mode 100644 fs/smb/server/mgmt/tree_connect.h create mode 100644 fs/smb/server/mgmt/user_config.c create mode 100644 fs/smb/server/mgmt/user_config.h create mode 100644 fs/smb/server/mgmt/user_session.c create mode 100644 fs/smb/server/mgmt/user_session.h create mode 100644 fs/smb/server/misc.c create mode 100644 fs/smb/server/misc.h create mode 100644 fs/smb/server/ndr.c create mode 100644 fs/smb/server/ndr.h create mode 100644 fs/smb/server/ntlmssp.h create mode 100644 fs/smb/server/oplock.c create mode 100644 fs/smb/server/oplock.h create mode 100644 fs/smb/server/proc.c create mode 100644 fs/smb/server/server.c create mode 100644 fs/smb/server/server.h create mode 100644 fs/smb/server/smb2misc.c create mode 100644 fs/smb/server/smb2ops.c create mode 100644 fs/smb/server/smb2pdu.c create mode 100644 fs/smb/server/smb2pdu.h create mode 100644 fs/smb/server/smb_common.c create mode 100644 fs/smb/server/smb_common.h create mode 100644 fs/smb/server/smbacl.c create mode 100644 fs/smb/server/smbacl.h create mode 100644 fs/smb/server/stats.h create mode 100644 fs/smb/server/transport_ipc.c create mode 100644 fs/smb/server/transport_ipc.h create mode 100644 fs/smb/server/transport_rdma.c create mode 100644 fs/smb/server/transport_rdma.h create mode 100644 fs/smb/server/transport_tcp.c create mode 100644 fs/smb/server/transport_tcp.h create mode 100644 fs/smb/server/unicode.c create mode 100644 fs/smb/server/unicode.h create mode 100644 fs/smb/server/vfs.c create mode 100644 fs/smb/server/vfs.h create mode 100644 fs/smb/server/vfs_cache.c create mode 100644 fs/smb/server/vfs_cache.h create mode 100644 fs/smb/server/xattr.h create mode 100644 fs/smb/smbdirect/Kconfig create mode 100644 fs/smb/smbdirect/Makefile create mode 100644 fs/smb/smbdirect/accept.c create mode 100644 fs/smb/smbdirect/connect.c create mode 100644 fs/smb/smbdirect/connection.c create mode 100644 fs/smb/smbdirect/debug.c create mode 100644 fs/smb/smbdirect/devices.c create mode 100644 fs/smb/smbdirect/internal.h create mode 100644 fs/smb/smbdirect/listen.c create mode 100644 fs/smb/smbdirect/main.c create mode 100644 fs/smb/smbdirect/mr.c create mode 100644 fs/smb/smbdirect/pdu.h create mode 100644 fs/smb/smbdirect/rw.c create mode 100644 fs/smb/smbdirect/socket.c create mode 100644 fs/smb/smbdirect/socket.h create mode 100644 fs/splice.c create mode 100644 fs/squashfs/Kconfig create mode 100644 fs/squashfs/Makefile create mode 100644 fs/squashfs/block.c create mode 100644 fs/squashfs/cache.c create mode 100644 fs/squashfs/decompressor.c create mode 100644 fs/squashfs/decompressor.h create mode 100644 fs/squashfs/decompressor_multi.c create mode 100644 fs/squashfs/decompressor_multi_percpu.c create mode 100644 fs/squashfs/decompressor_single.c create mode 100644 fs/squashfs/dir.c create mode 100644 fs/squashfs/export.c create mode 100644 fs/squashfs/file.c create mode 100644 fs/squashfs/file_cache.c create mode 100644 fs/squashfs/file_direct.c create mode 100644 fs/squashfs/fragment.c create mode 100644 fs/squashfs/id.c create mode 100644 fs/squashfs/inode.c create mode 100644 fs/squashfs/lz4_wrapper.c create mode 100644 fs/squashfs/lzo_wrapper.c create mode 100644 fs/squashfs/namei.c create mode 100644 fs/squashfs/page_actor.c create mode 100644 fs/squashfs/page_actor.h create mode 100644 fs/squashfs/squashfs.h create mode 100644 fs/squashfs/squashfs_fs.h create mode 100644 fs/squashfs/squashfs_fs_i.h create mode 100644 fs/squashfs/squashfs_fs_sb.h create mode 100644 fs/squashfs/super.c create mode 100644 fs/squashfs/symlink.c create mode 100644 fs/squashfs/xattr.c create mode 100644 fs/squashfs/xattr.h create mode 100644 fs/squashfs/xattr_id.c create mode 100644 fs/squashfs/xz_wrapper.c create mode 100644 fs/squashfs/zlib_wrapper.c create mode 100644 fs/squashfs/zstd_wrapper.c create mode 100644 fs/stack.c create mode 100644 fs/stat.c create mode 100644 fs/statfs.c create mode 100644 fs/super.c create mode 100644 fs/sync.c create mode 100644 fs/sysctls.c create mode 100644 fs/sysfs/Kconfig create mode 100644 fs/sysfs/Makefile create mode 100644 fs/sysfs/dir.c create mode 100644 fs/sysfs/file.c create mode 100644 fs/sysfs/group.c create mode 100644 fs/sysfs/mount.c create mode 100644 fs/sysfs/symlink.c create mode 100644 fs/sysfs/sysfs.h create mode 100644 fs/tests/binfmt_elf_kunit.c create mode 100644 fs/tests/exec_kunit.c create mode 100644 fs/timerfd.c create mode 100644 fs/tracefs/Makefile create mode 100644 fs/tracefs/event_inode.c create mode 100644 fs/tracefs/inode.c create mode 100644 fs/tracefs/internal.h create mode 100644 fs/ubifs/Kconfig create mode 100644 fs/ubifs/Makefile create mode 100644 fs/ubifs/auth.c create mode 100644 fs/ubifs/budget.c create mode 100644 fs/ubifs/commit.c create mode 100644 fs/ubifs/compress.c create mode 100644 fs/ubifs/crypto.c create mode 100644 fs/ubifs/debug.c create mode 100644 fs/ubifs/debug.h create mode 100644 fs/ubifs/dir.c create mode 100644 fs/ubifs/file.c create mode 100644 fs/ubifs/find.c create mode 100644 fs/ubifs/gc.c create mode 100644 fs/ubifs/io.c create mode 100644 fs/ubifs/ioctl.c create mode 100644 fs/ubifs/journal.c create mode 100644 fs/ubifs/key.h create mode 100644 fs/ubifs/log.c create mode 100644 fs/ubifs/lprops.c create mode 100644 fs/ubifs/lpt.c create mode 100644 fs/ubifs/lpt_commit.c create mode 100644 fs/ubifs/master.c create mode 100644 fs/ubifs/misc.c create mode 100644 fs/ubifs/misc.h create mode 100644 fs/ubifs/orphan.c create mode 100644 fs/ubifs/recovery.c create mode 100644 fs/ubifs/replay.c create mode 100644 fs/ubifs/sb.c create mode 100644 fs/ubifs/scan.c create mode 100644 fs/ubifs/shrinker.c create mode 100644 fs/ubifs/super.c create mode 100644 fs/ubifs/sysfs.c create mode 100644 fs/ubifs/tnc.c create mode 100644 fs/ubifs/tnc_commit.c create mode 100644 fs/ubifs/tnc_misc.c create mode 100644 fs/ubifs/ubifs-media.h create mode 100644 fs/ubifs/ubifs.h create mode 100644 fs/ubifs/xattr.c create mode 100644 fs/udf/Kconfig create mode 100644 fs/udf/Makefile create mode 100644 fs/udf/balloc.c create mode 100644 fs/udf/dir.c create mode 100644 fs/udf/directory.c create mode 100644 fs/udf/ecma_167.h create mode 100644 fs/udf/file.c create mode 100644 fs/udf/ialloc.c create mode 100644 fs/udf/inode.c create mode 100644 fs/udf/lowlevel.c create mode 100644 fs/udf/misc.c create mode 100644 fs/udf/namei.c create mode 100644 fs/udf/osta_udf.h create mode 100644 fs/udf/partition.c create mode 100644 fs/udf/super.c create mode 100644 fs/udf/symlink.c create mode 100644 fs/udf/truncate.c create mode 100644 fs/udf/udf_i.h create mode 100644 fs/udf/udf_sb.h create mode 100644 fs/udf/udfdecl.h create mode 100644 fs/udf/udfend.h create mode 100644 fs/udf/udftime.c create mode 100644 fs/udf/unicode.c create mode 100644 fs/ufs/Kconfig create mode 100644 fs/ufs/Makefile create mode 100644 fs/ufs/balloc.c create mode 100644 fs/ufs/cylinder.c create mode 100644 fs/ufs/dir.c create mode 100644 fs/ufs/file.c create mode 100644 fs/ufs/ialloc.c create mode 100644 fs/ufs/inode.c create mode 100644 fs/ufs/namei.c create mode 100644 fs/ufs/super.c create mode 100644 fs/ufs/swab.h create mode 100644 fs/ufs/ufs.h create mode 100644 fs/ufs/ufs_fs.h create mode 100644 fs/ufs/util.c create mode 100644 fs/ufs/util.h create mode 100644 fs/unicode/.gitignore create mode 100644 fs/unicode/Kconfig create mode 100644 fs/unicode/Makefile create mode 100644 fs/unicode/README.utf8data create mode 100644 fs/unicode/mkutf8data.c create mode 100644 fs/unicode/tests/.kunitconfig create mode 100644 fs/unicode/tests/utf8_kunit.c create mode 100644 fs/unicode/utf8-core.c create mode 100644 fs/unicode/utf8-norm.c create mode 100644 fs/unicode/utf8data.c_shipped create mode 100644 fs/unicode/utf8n.h create mode 100644 fs/utimes.c create mode 100644 fs/vboxsf/Kconfig create mode 100644 fs/vboxsf/Makefile create mode 100644 fs/vboxsf/dir.c create mode 100644 fs/vboxsf/file.c create mode 100644 fs/vboxsf/shfl_hostintf.h create mode 100644 fs/vboxsf/super.c create mode 100644 fs/vboxsf/utils.c create mode 100644 fs/vboxsf/vboxsf_wrappers.c create mode 100644 fs/vboxsf/vfsmod.h create mode 100644 fs/verity/Kconfig create mode 100644 fs/verity/Makefile create mode 100644 fs/verity/enable.c create mode 100644 fs/verity/fsverity_private.h create mode 100644 fs/verity/hash_algs.c create mode 100644 fs/verity/init.c create mode 100644 fs/verity/measure.c create mode 100644 fs/verity/open.c create mode 100644 fs/verity/pagecache.c create mode 100644 fs/verity/read_metadata.c create mode 100644 fs/verity/signature.c create mode 100644 fs/verity/verify.c create mode 100644 fs/xattr.c create mode 100644 fs/xfs/Kconfig create mode 100644 fs/xfs/Makefile create mode 100644 fs/xfs/libxfs/xfs_ag.c create mode 100644 fs/xfs/libxfs/xfs_ag.h create mode 100644 fs/xfs/libxfs/xfs_ag_resv.c create mode 100644 fs/xfs/libxfs/xfs_ag_resv.h create mode 100644 fs/xfs/libxfs/xfs_alloc.c create mode 100644 fs/xfs/libxfs/xfs_alloc.h create mode 100644 fs/xfs/libxfs/xfs_alloc_btree.c create mode 100644 fs/xfs/libxfs/xfs_alloc_btree.h create mode 100644 fs/xfs/libxfs/xfs_attr.c create mode 100644 fs/xfs/libxfs/xfs_attr.h create mode 100644 fs/xfs/libxfs/xfs_attr_leaf.c create mode 100644 fs/xfs/libxfs/xfs_attr_leaf.h create mode 100644 fs/xfs/libxfs/xfs_attr_remote.c create mode 100644 fs/xfs/libxfs/xfs_attr_remote.h create mode 100644 fs/xfs/libxfs/xfs_attr_sf.h create mode 100644 fs/xfs/libxfs/xfs_bit.c create mode 100644 fs/xfs/libxfs/xfs_bit.h create mode 100644 fs/xfs/libxfs/xfs_bmap.c create mode 100644 fs/xfs/libxfs/xfs_bmap.h create mode 100644 fs/xfs/libxfs/xfs_bmap_btree.c create mode 100644 fs/xfs/libxfs/xfs_bmap_btree.h create mode 100644 fs/xfs/libxfs/xfs_btree.c create mode 100644 fs/xfs/libxfs/xfs_btree.h create mode 100644 fs/xfs/libxfs/xfs_btree_mem.c create mode 100644 fs/xfs/libxfs/xfs_btree_mem.h create mode 100644 fs/xfs/libxfs/xfs_btree_staging.c create mode 100644 fs/xfs/libxfs/xfs_btree_staging.h create mode 100644 fs/xfs/libxfs/xfs_cksum.h create mode 100644 fs/xfs/libxfs/xfs_da_btree.c create mode 100644 fs/xfs/libxfs/xfs_da_btree.h create mode 100644 fs/xfs/libxfs/xfs_da_format.h create mode 100644 fs/xfs/libxfs/xfs_defer.c create mode 100644 fs/xfs/libxfs/xfs_defer.h create mode 100644 fs/xfs/libxfs/xfs_dir2.c create mode 100644 fs/xfs/libxfs/xfs_dir2.h create mode 100644 fs/xfs/libxfs/xfs_dir2_block.c create mode 100644 fs/xfs/libxfs/xfs_dir2_data.c create mode 100644 fs/xfs/libxfs/xfs_dir2_leaf.c create mode 100644 fs/xfs/libxfs/xfs_dir2_node.c create mode 100644 fs/xfs/libxfs/xfs_dir2_priv.h create mode 100644 fs/xfs/libxfs/xfs_dir2_sf.c create mode 100644 fs/xfs/libxfs/xfs_dquot_buf.c create mode 100644 fs/xfs/libxfs/xfs_errortag.h create mode 100644 fs/xfs/libxfs/xfs_exchmaps.c create mode 100644 fs/xfs/libxfs/xfs_exchmaps.h create mode 100644 fs/xfs/libxfs/xfs_format.h create mode 100644 fs/xfs/libxfs/xfs_fs.h create mode 100644 fs/xfs/libxfs/xfs_group.c create mode 100644 fs/xfs/libxfs/xfs_group.h create mode 100644 fs/xfs/libxfs/xfs_health.h create mode 100644 fs/xfs/libxfs/xfs_ialloc.c create mode 100644 fs/xfs/libxfs/xfs_ialloc.h create mode 100644 fs/xfs/libxfs/xfs_ialloc_btree.c create mode 100644 fs/xfs/libxfs/xfs_ialloc_btree.h create mode 100644 fs/xfs/libxfs/xfs_iext_tree.c create mode 100644 fs/xfs/libxfs/xfs_inode_buf.c create mode 100644 fs/xfs/libxfs/xfs_inode_buf.h create mode 100644 fs/xfs/libxfs/xfs_inode_fork.c create mode 100644 fs/xfs/libxfs/xfs_inode_fork.h create mode 100644 fs/xfs/libxfs/xfs_inode_util.c create mode 100644 fs/xfs/libxfs/xfs_inode_util.h create mode 100644 fs/xfs/libxfs/xfs_log_format.h create mode 100644 fs/xfs/libxfs/xfs_log_recover.h create mode 100644 fs/xfs/libxfs/xfs_log_rlimit.c create mode 100644 fs/xfs/libxfs/xfs_metadir.c create mode 100644 fs/xfs/libxfs/xfs_metadir.h create mode 100644 fs/xfs/libxfs/xfs_metafile.c create mode 100644 fs/xfs/libxfs/xfs_metafile.h create mode 100644 fs/xfs/libxfs/xfs_ondisk.h create mode 100644 fs/xfs/libxfs/xfs_parent.c create mode 100644 fs/xfs/libxfs/xfs_parent.h create mode 100644 fs/xfs/libxfs/xfs_quota_defs.h create mode 100644 fs/xfs/libxfs/xfs_refcount.c create mode 100644 fs/xfs/libxfs/xfs_refcount.h create mode 100644 fs/xfs/libxfs/xfs_refcount_btree.c create mode 100644 fs/xfs/libxfs/xfs_refcount_btree.h create mode 100644 fs/xfs/libxfs/xfs_rmap.c create mode 100644 fs/xfs/libxfs/xfs_rmap.h create mode 100644 fs/xfs/libxfs/xfs_rmap_btree.c create mode 100644 fs/xfs/libxfs/xfs_rmap_btree.h create mode 100644 fs/xfs/libxfs/xfs_rtbitmap.c create mode 100644 fs/xfs/libxfs/xfs_rtbitmap.h create mode 100644 fs/xfs/libxfs/xfs_rtgroup.c create mode 100644 fs/xfs/libxfs/xfs_rtgroup.h create mode 100644 fs/xfs/libxfs/xfs_rtrefcount_btree.c create mode 100644 fs/xfs/libxfs/xfs_rtrefcount_btree.h create mode 100644 fs/xfs/libxfs/xfs_rtrmap_btree.c create mode 100644 fs/xfs/libxfs/xfs_rtrmap_btree.h create mode 100644 fs/xfs/libxfs/xfs_sb.c create mode 100644 fs/xfs/libxfs/xfs_sb.h create mode 100644 fs/xfs/libxfs/xfs_shared.h create mode 100644 fs/xfs/libxfs/xfs_symlink_remote.c create mode 100644 fs/xfs/libxfs/xfs_symlink_remote.h create mode 100644 fs/xfs/libxfs/xfs_trans_inode.c create mode 100644 fs/xfs/libxfs/xfs_trans_resv.c create mode 100644 fs/xfs/libxfs/xfs_trans_resv.h create mode 100644 fs/xfs/libxfs/xfs_trans_space.c create mode 100644 fs/xfs/libxfs/xfs_trans_space.h create mode 100644 fs/xfs/libxfs/xfs_types.c create mode 100644 fs/xfs/libxfs/xfs_types.h create mode 100644 fs/xfs/libxfs/xfs_zones.c create mode 100644 fs/xfs/libxfs/xfs_zones.h create mode 100644 fs/xfs/scrub/agb_bitmap.c create mode 100644 fs/xfs/scrub/agb_bitmap.h create mode 100644 fs/xfs/scrub/agheader.c create mode 100644 fs/xfs/scrub/agheader_repair.c create mode 100644 fs/xfs/scrub/agino_bitmap.h create mode 100644 fs/xfs/scrub/alloc.c create mode 100644 fs/xfs/scrub/alloc_repair.c create mode 100644 fs/xfs/scrub/attr.c create mode 100644 fs/xfs/scrub/attr.h create mode 100644 fs/xfs/scrub/attr_repair.c create mode 100644 fs/xfs/scrub/attr_repair.h create mode 100644 fs/xfs/scrub/bitmap.c create mode 100644 fs/xfs/scrub/bitmap.h create mode 100644 fs/xfs/scrub/bmap.c create mode 100644 fs/xfs/scrub/bmap_repair.c create mode 100644 fs/xfs/scrub/btree.c create mode 100644 fs/xfs/scrub/btree.h create mode 100644 fs/xfs/scrub/common.c create mode 100644 fs/xfs/scrub/common.h create mode 100644 fs/xfs/scrub/cow_repair.c create mode 100644 fs/xfs/scrub/dab_bitmap.h create mode 100644 fs/xfs/scrub/dabtree.c create mode 100644 fs/xfs/scrub/dabtree.h create mode 100644 fs/xfs/scrub/dir.c create mode 100644 fs/xfs/scrub/dir_repair.c create mode 100644 fs/xfs/scrub/dirtree.c create mode 100644 fs/xfs/scrub/dirtree.h create mode 100644 fs/xfs/scrub/dirtree_repair.c create mode 100644 fs/xfs/scrub/dqiterate.c create mode 100644 fs/xfs/scrub/findparent.c create mode 100644 fs/xfs/scrub/findparent.h create mode 100644 fs/xfs/scrub/fsb_bitmap.h create mode 100644 fs/xfs/scrub/fscounters.c create mode 100644 fs/xfs/scrub/fscounters.h create mode 100644 fs/xfs/scrub/fscounters_repair.c create mode 100644 fs/xfs/scrub/health.c create mode 100644 fs/xfs/scrub/health.h create mode 100644 fs/xfs/scrub/ialloc.c create mode 100644 fs/xfs/scrub/ialloc_repair.c create mode 100644 fs/xfs/scrub/ino_bitmap.h create mode 100644 fs/xfs/scrub/inode.c create mode 100644 fs/xfs/scrub/inode_repair.c create mode 100644 fs/xfs/scrub/iscan.c create mode 100644 fs/xfs/scrub/iscan.h create mode 100644 fs/xfs/scrub/listxattr.c create mode 100644 fs/xfs/scrub/listxattr.h create mode 100644 fs/xfs/scrub/metapath.c create mode 100644 fs/xfs/scrub/newbt.c create mode 100644 fs/xfs/scrub/newbt.h create mode 100644 fs/xfs/scrub/nlinks.c create mode 100644 fs/xfs/scrub/nlinks.h create mode 100644 fs/xfs/scrub/nlinks_repair.c create mode 100644 fs/xfs/scrub/off_bitmap.h create mode 100644 fs/xfs/scrub/orphanage.c create mode 100644 fs/xfs/scrub/orphanage.h create mode 100644 fs/xfs/scrub/parent.c create mode 100644 fs/xfs/scrub/parent_repair.c create mode 100644 fs/xfs/scrub/quota.c create mode 100644 fs/xfs/scrub/quota.h create mode 100644 fs/xfs/scrub/quota_repair.c create mode 100644 fs/xfs/scrub/quotacheck.c create mode 100644 fs/xfs/scrub/quotacheck.h create mode 100644 fs/xfs/scrub/quotacheck_repair.c create mode 100644 fs/xfs/scrub/rcbag.c create mode 100644 fs/xfs/scrub/rcbag.h create mode 100644 fs/xfs/scrub/rcbag_btree.c create mode 100644 fs/xfs/scrub/rcbag_btree.h create mode 100644 fs/xfs/scrub/readdir.c create mode 100644 fs/xfs/scrub/readdir.h create mode 100644 fs/xfs/scrub/reap.c create mode 100644 fs/xfs/scrub/reap.h create mode 100644 fs/xfs/scrub/refcount.c create mode 100644 fs/xfs/scrub/refcount_repair.c create mode 100644 fs/xfs/scrub/repair.c create mode 100644 fs/xfs/scrub/repair.h create mode 100644 fs/xfs/scrub/rgb_bitmap.h create mode 100644 fs/xfs/scrub/rgsuper.c create mode 100644 fs/xfs/scrub/rmap.c create mode 100644 fs/xfs/scrub/rmap_repair.c create mode 100644 fs/xfs/scrub/rtb_bitmap.h create mode 100644 fs/xfs/scrub/rtbitmap.c create mode 100644 fs/xfs/scrub/rtbitmap.h create mode 100644 fs/xfs/scrub/rtbitmap_repair.c create mode 100644 fs/xfs/scrub/rtrefcount.c create mode 100644 fs/xfs/scrub/rtrefcount_repair.c create mode 100644 fs/xfs/scrub/rtrmap.c create mode 100644 fs/xfs/scrub/rtrmap_repair.c create mode 100644 fs/xfs/scrub/rtsummary.c create mode 100644 fs/xfs/scrub/rtsummary.h create mode 100644 fs/xfs/scrub/rtsummary_repair.c create mode 100644 fs/xfs/scrub/scrub.c create mode 100644 fs/xfs/scrub/scrub.h create mode 100644 fs/xfs/scrub/stats.c create mode 100644 fs/xfs/scrub/stats.h create mode 100644 fs/xfs/scrub/symlink.c create mode 100644 fs/xfs/scrub/symlink_repair.c create mode 100644 fs/xfs/scrub/tempexch.h create mode 100644 fs/xfs/scrub/tempfile.c create mode 100644 fs/xfs/scrub/tempfile.h create mode 100644 fs/xfs/scrub/trace.c create mode 100644 fs/xfs/scrub/trace.h create mode 100644 fs/xfs/scrub/xfarray.c create mode 100644 fs/xfs/scrub/xfarray.h create mode 100644 fs/xfs/scrub/xfblob.c create mode 100644 fs/xfs/scrub/xfblob.h create mode 100644 fs/xfs/scrub/xfile.c create mode 100644 fs/xfs/scrub/xfile.h create mode 100644 fs/xfs/scrub/xfs_scrub.h create mode 100644 fs/xfs/xfs_acl.c create mode 100644 fs/xfs/xfs_acl.h create mode 100644 fs/xfs/xfs_aops.c create mode 100644 fs/xfs/xfs_aops.h create mode 100644 fs/xfs/xfs_attr_inactive.c create mode 100644 fs/xfs/xfs_attr_item.c create mode 100644 fs/xfs/xfs_attr_item.h create mode 100644 fs/xfs/xfs_attr_list.c create mode 100644 fs/xfs/xfs_bio_io.c create mode 100644 fs/xfs/xfs_bmap_item.c create mode 100644 fs/xfs/xfs_bmap_item.h create mode 100644 fs/xfs/xfs_bmap_util.c create mode 100644 fs/xfs/xfs_bmap_util.h create mode 100644 fs/xfs/xfs_buf.c create mode 100644 fs/xfs/xfs_buf.h create mode 100644 fs/xfs/xfs_buf_item.c create mode 100644 fs/xfs/xfs_buf_item.h create mode 100644 fs/xfs/xfs_buf_item_recover.c create mode 100644 fs/xfs/xfs_buf_mem.c create mode 100644 fs/xfs/xfs_buf_mem.h create mode 100644 fs/xfs/xfs_dahash_test.c create mode 100644 fs/xfs/xfs_dahash_test.h create mode 100644 fs/xfs/xfs_dir2_readdir.c create mode 100644 fs/xfs/xfs_discard.c create mode 100644 fs/xfs/xfs_discard.h create mode 100644 fs/xfs/xfs_dquot.c create mode 100644 fs/xfs/xfs_dquot.h create mode 100644 fs/xfs/xfs_dquot_item.c create mode 100644 fs/xfs/xfs_dquot_item.h create mode 100644 fs/xfs/xfs_dquot_item_recover.c create mode 100644 fs/xfs/xfs_drain.c create mode 100644 fs/xfs/xfs_drain.h create mode 100644 fs/xfs/xfs_error.c create mode 100644 fs/xfs/xfs_error.h create mode 100644 fs/xfs/xfs_exchmaps_item.c create mode 100644 fs/xfs/xfs_exchmaps_item.h create mode 100644 fs/xfs/xfs_exchrange.c create mode 100644 fs/xfs/xfs_exchrange.h create mode 100644 fs/xfs/xfs_export.c create mode 100644 fs/xfs/xfs_export.h create mode 100644 fs/xfs/xfs_extent_busy.c create mode 100644 fs/xfs/xfs_extent_busy.h create mode 100644 fs/xfs/xfs_extfree_item.c create mode 100644 fs/xfs/xfs_extfree_item.h create mode 100644 fs/xfs/xfs_file.c create mode 100644 fs/xfs/xfs_file.h create mode 100644 fs/xfs/xfs_filestream.c create mode 100644 fs/xfs/xfs_filestream.h create mode 100644 fs/xfs/xfs_fsmap.c create mode 100644 fs/xfs/xfs_fsmap.h create mode 100644 fs/xfs/xfs_fsops.c create mode 100644 fs/xfs/xfs_fsops.h create mode 100644 fs/xfs/xfs_globals.c create mode 100644 fs/xfs/xfs_handle.c create mode 100644 fs/xfs/xfs_handle.h create mode 100644 fs/xfs/xfs_health.c create mode 100644 fs/xfs/xfs_healthmon.c create mode 100644 fs/xfs/xfs_healthmon.h create mode 100644 fs/xfs/xfs_hooks.c create mode 100644 fs/xfs/xfs_hooks.h create mode 100644 fs/xfs/xfs_icache.c create mode 100644 fs/xfs/xfs_icache.h create mode 100644 fs/xfs/xfs_icreate_item.c create mode 100644 fs/xfs/xfs_icreate_item.h create mode 100644 fs/xfs/xfs_inode.c create mode 100644 fs/xfs/xfs_inode.h create mode 100644 fs/xfs/xfs_inode_item.c create mode 100644 fs/xfs/xfs_inode_item.h create mode 100644 fs/xfs/xfs_inode_item_recover.c create mode 100644 fs/xfs/xfs_ioctl.c create mode 100644 fs/xfs/xfs_ioctl.h create mode 100644 fs/xfs/xfs_ioctl32.c create mode 100644 fs/xfs/xfs_ioctl32.h create mode 100644 fs/xfs/xfs_iomap.c create mode 100644 fs/xfs/xfs_iomap.h create mode 100644 fs/xfs/xfs_iops.c create mode 100644 fs/xfs/xfs_iops.h create mode 100644 fs/xfs/xfs_itable.c create mode 100644 fs/xfs/xfs_itable.h create mode 100644 fs/xfs/xfs_iunlink_item.c create mode 100644 fs/xfs/xfs_iunlink_item.h create mode 100644 fs/xfs/xfs_iwalk.c create mode 100644 fs/xfs/xfs_iwalk.h create mode 100644 fs/xfs/xfs_log.c create mode 100644 fs/xfs/xfs_log.h create mode 100644 fs/xfs/xfs_log_cil.c create mode 100644 fs/xfs/xfs_log_priv.h create mode 100644 fs/xfs/xfs_log_recover.c create mode 100644 fs/xfs/xfs_message.c create mode 100644 fs/xfs/xfs_message.h create mode 100644 fs/xfs/xfs_mount.c create mode 100644 fs/xfs/xfs_mount.h create mode 100644 fs/xfs/xfs_mru_cache.c create mode 100644 fs/xfs/xfs_mru_cache.h create mode 100644 fs/xfs/xfs_notify_failure.c create mode 100644 fs/xfs/xfs_notify_failure.h create mode 100644 fs/xfs/xfs_platform.h create mode 100644 fs/xfs/xfs_pnfs.c create mode 100644 fs/xfs/xfs_pnfs.h create mode 100644 fs/xfs/xfs_pwork.c create mode 100644 fs/xfs/xfs_pwork.h create mode 100644 fs/xfs/xfs_qm.c create mode 100644 fs/xfs/xfs_qm.h create mode 100644 fs/xfs/xfs_qm_bhv.c create mode 100644 fs/xfs/xfs_qm_syscalls.c create mode 100644 fs/xfs/xfs_quota.h create mode 100644 fs/xfs/xfs_quotaops.c create mode 100644 fs/xfs/xfs_refcount_item.c create mode 100644 fs/xfs/xfs_refcount_item.h create mode 100644 fs/xfs/xfs_reflink.c create mode 100644 fs/xfs/xfs_reflink.h create mode 100644 fs/xfs/xfs_rmap_item.c create mode 100644 fs/xfs/xfs_rmap_item.h create mode 100644 fs/xfs/xfs_rtalloc.c create mode 100644 fs/xfs/xfs_rtalloc.h create mode 100644 fs/xfs/xfs_stats.c create mode 100644 fs/xfs/xfs_stats.h create mode 100644 fs/xfs/xfs_super.c create mode 100644 fs/xfs/xfs_super.h create mode 100644 fs/xfs/xfs_symlink.c create mode 100644 fs/xfs/xfs_symlink.h create mode 100644 fs/xfs/xfs_sysctl.c create mode 100644 fs/xfs/xfs_sysctl.h create mode 100644 fs/xfs/xfs_sysfs.c create mode 100644 fs/xfs/xfs_sysfs.h create mode 100644 fs/xfs/xfs_trace.c create mode 100644 fs/xfs/xfs_trace.h create mode 100644 fs/xfs/xfs_trans.c create mode 100644 fs/xfs/xfs_trans.h create mode 100644 fs/xfs/xfs_trans_ail.c create mode 100644 fs/xfs/xfs_trans_buf.c create mode 100644 fs/xfs/xfs_trans_dquot.c create mode 100644 fs/xfs/xfs_trans_priv.h create mode 100644 fs/xfs/xfs_verify_media.c create mode 100644 fs/xfs/xfs_verify_media.h create mode 100644 fs/xfs/xfs_xattr.c create mode 100644 fs/xfs/xfs_xattr.h create mode 100644 fs/xfs/xfs_zone_alloc.c create mode 100644 fs/xfs/xfs_zone_alloc.h create mode 100644 fs/xfs/xfs_zone_gc.c create mode 100644 fs/xfs/xfs_zone_info.c create mode 100644 fs/xfs/xfs_zone_priv.h create mode 100644 fs/xfs/xfs_zone_space_resv.c create mode 100644 fs/zonefs/Kconfig create mode 100644 fs/zonefs/Makefile create mode 100644 fs/zonefs/file.c create mode 100644 fs/zonefs/super.c create mode 100644 fs/zonefs/sysfs.c create mode 100644 fs/zonefs/trace.h create mode 100644 fs/zonefs/zonefs.h (limited to 'fs') diff --git a/fs/9p/Kconfig b/fs/9p/Kconfig new file mode 100644 index 000000000..0c63df574 --- /dev/null +++ b/fs/9p/Kconfig @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: GPL-2.0-only +config 9P_FS + tristate "Plan 9 Resource Sharing Support (9P2000)" + depends on NET_9P + select NETFS_SUPPORT + help + If you say Y here, you will get experimental support for + Plan 9 resource sharing via the 9P2000 protocol. + + See for more information. + + If unsure, say N. + +if 9P_FS +config 9P_FSCACHE + bool "Enable 9P client caching support" + depends on 9P_FS=m && FSCACHE || 9P_FS=y && FSCACHE=y + help + Choose Y here to enable persistent, read-only local + caching support for 9p clients using FS-Cache + + +config 9P_FS_POSIX_ACL + bool "9P POSIX Access Control Lists" + select FS_POSIX_ACL + help + POSIX Access Control Lists (ACLs) support permissions for users and + groups beyond the owner/group/world scheme. + + If you don't know what Access Control Lists are, say N + +endif + + +config 9P_FS_SECURITY + bool "9P Security Labels" + depends on 9P_FS + help + Security labels support alternative access control models + implemented by security modules like SELinux. This option + enables an extended attribute handler for file security + labels in the 9P filesystem. + + If you are not using a security module that requires using + extended attributes for file security labels, say N. diff --git a/fs/9p/Makefile b/fs/9p/Makefile new file mode 100644 index 000000000..e7800a5c7 --- /dev/null +++ b/fs/9p/Makefile @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_9P_FS) := 9p.o + +9p-objs := \ + vfs_super.o \ + vfs_inode.o \ + vfs_inode_dotl.o \ + vfs_addr.o \ + vfs_file.o \ + vfs_dir.o \ + vfs_dentry.o \ + v9fs.o \ + fid.o \ + xattr.o + +9p-$(CONFIG_9P_FSCACHE) += cache.o +9p-$(CONFIG_9P_FS_POSIX_ACL) += acl.o diff --git a/fs/9p/acl.c b/fs/9p/acl.c new file mode 100644 index 000000000..ae7e7cf75 --- /dev/null +++ b/fs/9p/acl.c @@ -0,0 +1,336 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * Copyright IBM Corporation, 2010 + * Author Aneesh Kumar K.V + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "xattr.h" +#include "acl.h" +#include "v9fs.h" +#include "v9fs_vfs.h" +#include "fid.h" + +static struct posix_acl *v9fs_fid_get_acl(struct p9_fid *fid, const char *name) +{ + ssize_t size; + void *value = NULL; + struct posix_acl *acl = NULL; + + size = v9fs_fid_xattr_get(fid, name, NULL, 0); + if (size < 0) + return ERR_PTR(size); + if (size == 0) + return ERR_PTR(-ENODATA); + + value = kzalloc(size, GFP_NOFS); + if (!value) + return ERR_PTR(-ENOMEM); + + size = v9fs_fid_xattr_get(fid, name, value, size); + if (size < 0) + acl = ERR_PTR(size); + else if (size == 0) + acl = ERR_PTR(-ENODATA); + else + acl = posix_acl_from_xattr(&init_user_ns, value, size); + kfree(value); + return acl; +} + +static struct posix_acl *v9fs_acl_get(struct dentry *dentry, const char *name) +{ + struct p9_fid *fid; + struct posix_acl *acl = NULL; + + fid = v9fs_fid_lookup(dentry); + if (IS_ERR(fid)) + return ERR_CAST(fid); + + acl = v9fs_fid_get_acl(fid, name); + p9_fid_put(fid); + return acl; +} + +static struct posix_acl *__v9fs_get_acl(struct p9_fid *fid, const char *name) +{ + int retval; + struct posix_acl *acl = NULL; + + acl = v9fs_fid_get_acl(fid, name); + if (!IS_ERR(acl)) + return acl; + + retval = PTR_ERR(acl); + if (retval == -ENODATA || retval == -ENOSYS || retval == -EOPNOTSUPP) + return NULL; + + /* map everything else to -EIO */ + return ERR_PTR(-EIO); +} + +int v9fs_get_acl(struct inode *inode, struct p9_fid *fid) +{ + int retval = 0; + struct posix_acl *pacl, *dacl; + struct v9fs_session_info *v9ses; + + v9ses = v9fs_inode2v9ses(inode); + if (((v9ses->flags & V9FS_ACCESS_MASK) != V9FS_ACCESS_CLIENT) || + ((v9ses->flags & V9FS_ACL_MASK) != V9FS_POSIX_ACL)) { + set_cached_acl(inode, ACL_TYPE_DEFAULT, NULL); + set_cached_acl(inode, ACL_TYPE_ACCESS, NULL); + return 0; + } + /* get the default/access acl values and cache them */ + dacl = __v9fs_get_acl(fid, XATTR_NAME_POSIX_ACL_DEFAULT); + pacl = __v9fs_get_acl(fid, XATTR_NAME_POSIX_ACL_ACCESS); + + if (!IS_ERR(dacl) && !IS_ERR(pacl)) { + set_cached_acl(inode, ACL_TYPE_DEFAULT, dacl); + set_cached_acl(inode, ACL_TYPE_ACCESS, pacl); + } else + retval = -EIO; + + if (!IS_ERR(dacl)) + posix_acl_release(dacl); + + if (!IS_ERR(pacl)) + posix_acl_release(pacl); + + return retval; +} + +static struct posix_acl *v9fs_get_cached_acl(struct inode *inode, int type) +{ + struct posix_acl *acl; + /* + * 9p Always cache the acl value when + * instantiating the inode (v9fs_inode_from_fid) + */ + acl = get_cached_acl(inode, type); + BUG_ON(is_uncached_acl(acl)); + return acl; +} + +struct posix_acl *v9fs_iop_get_inode_acl(struct inode *inode, int type, bool rcu) +{ + struct v9fs_session_info *v9ses; + + if (rcu) + return ERR_PTR(-ECHILD); + + v9ses = v9fs_inode2v9ses(inode); + if (((v9ses->flags & V9FS_ACCESS_MASK) != V9FS_ACCESS_CLIENT) || + ((v9ses->flags & V9FS_ACL_MASK) != V9FS_POSIX_ACL)) { + /* + * On access = client and acl = on mode get the acl + * values from the server + */ + return NULL; + } + return v9fs_get_cached_acl(inode, type); + +} + +struct posix_acl *v9fs_iop_get_acl(struct mnt_idmap *idmap, + struct dentry *dentry, int type) +{ + struct v9fs_session_info *v9ses; + + v9ses = v9fs_dentry2v9ses(dentry); + /* We allow set/get/list of acl when access=client is not specified. */ + if ((v9ses->flags & V9FS_ACCESS_MASK) != V9FS_ACCESS_CLIENT) + return v9fs_acl_get(dentry, posix_acl_xattr_name(type)); + return v9fs_get_cached_acl(d_inode(dentry), type); +} + +int v9fs_iop_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type) +{ + int retval; + size_t size = 0; + void *value = NULL; + const char *acl_name; + struct v9fs_session_info *v9ses; + struct inode *inode = d_inode(dentry); + + if (acl) { + retval = posix_acl_valid(inode->i_sb->s_user_ns, acl); + if (retval) + goto err_out; + + value = posix_acl_to_xattr(&init_user_ns, acl, &size, GFP_NOFS); + if (!value) { + retval = -ENOMEM; + goto err_out; + } + } + + /* + * set the attribute on the remote. Without even looking at the + * xattr value. We leave it to the server to validate + */ + acl_name = posix_acl_xattr_name(type); + v9ses = v9fs_dentry2v9ses(dentry); + if ((v9ses->flags & V9FS_ACCESS_MASK) != V9FS_ACCESS_CLIENT) { + retval = v9fs_xattr_set(dentry, acl_name, value, size, 0); + goto err_out; + } + + if (S_ISLNK(inode->i_mode)) { + retval = -EOPNOTSUPP; + goto err_out; + } + + if (!inode_owner_or_capable(&nop_mnt_idmap, inode)) { + retval = -EPERM; + goto err_out; + } + + switch (type) { + case ACL_TYPE_ACCESS: + if (acl) { + struct iattr iattr = {}; + struct posix_acl *acl_mode = acl; + + retval = posix_acl_update_mode(&nop_mnt_idmap, inode, + &iattr.ia_mode, + &acl_mode); + if (retval) + goto err_out; + if (!acl_mode) { + /* + * ACL can be represented by the mode bits. + * So don't update ACL below. + */ + kfree(value); + value = NULL; + size = 0; + } + iattr.ia_valid = ATTR_MODE; + /* + * FIXME should we update ctime ? + * What is the following setxattr update the mode ? + */ + v9fs_vfs_setattr_dotl(&nop_mnt_idmap, dentry, &iattr); + } + break; + case ACL_TYPE_DEFAULT: + if (!S_ISDIR(inode->i_mode)) { + retval = acl ? -EINVAL : 0; + goto err_out; + } + break; + } + + retval = v9fs_xattr_set(dentry, acl_name, value, size, 0); + if (!retval) + set_cached_acl(inode, type, acl); + +err_out: + kfree(value); + return retval; +} + +static int v9fs_set_acl(struct p9_fid *fid, int type, struct posix_acl *acl) +{ + int retval; + char *name; + size_t size; + void *buffer; + + if (!acl) + return 0; + + /* Set a setxattr request to server */ + buffer = posix_acl_to_xattr(&init_user_ns, acl, &size, GFP_KERNEL); + if (!buffer) + return -ENOMEM; + + switch (type) { + case ACL_TYPE_ACCESS: + name = XATTR_NAME_POSIX_ACL_ACCESS; + break; + case ACL_TYPE_DEFAULT: + name = XATTR_NAME_POSIX_ACL_DEFAULT; + break; + default: + BUG(); + } + retval = v9fs_fid_xattr_set(fid, name, buffer, size, 0); + kfree(buffer); + return retval; +} + +int v9fs_acl_chmod(struct inode *inode, struct p9_fid *fid) +{ + int retval = 0; + struct posix_acl *acl; + + if (S_ISLNK(inode->i_mode)) + return -EOPNOTSUPP; + acl = v9fs_get_cached_acl(inode, ACL_TYPE_ACCESS); + if (acl) { + retval = __posix_acl_chmod(&acl, GFP_KERNEL, inode->i_mode); + if (retval) + return retval; + set_cached_acl(inode, ACL_TYPE_ACCESS, acl); + retval = v9fs_set_acl(fid, ACL_TYPE_ACCESS, acl); + posix_acl_release(acl); + } + return retval; +} + +int v9fs_set_create_acl(struct inode *inode, struct p9_fid *fid, + struct posix_acl *dacl, struct posix_acl *acl) +{ + set_cached_acl(inode, ACL_TYPE_DEFAULT, dacl); + set_cached_acl(inode, ACL_TYPE_ACCESS, acl); + v9fs_set_acl(fid, ACL_TYPE_DEFAULT, dacl); + v9fs_set_acl(fid, ACL_TYPE_ACCESS, acl); + return 0; +} + +void v9fs_put_acl(struct posix_acl *dacl, + struct posix_acl *acl) +{ + posix_acl_release(dacl); + posix_acl_release(acl); +} + +int v9fs_acl_mode(struct inode *dir, umode_t *modep, + struct posix_acl **dpacl, struct posix_acl **pacl) +{ + int retval = 0; + umode_t mode = *modep; + struct posix_acl *acl = NULL; + + if (!S_ISLNK(mode)) { + acl = v9fs_get_cached_acl(dir, ACL_TYPE_DEFAULT); + if (IS_ERR(acl)) + return PTR_ERR(acl); + if (!acl) + mode &= ~current_umask(); + } + if (acl) { + if (S_ISDIR(mode)) + *dpacl = posix_acl_dup(acl); + retval = __posix_acl_create(&acl, GFP_NOFS, &mode); + if (retval < 0) + return retval; + if (retval > 0) + *pacl = acl; + else + posix_acl_release(acl); + } + *modep = mode; + return 0; +} diff --git a/fs/9p/acl.h b/fs/9p/acl.h new file mode 100644 index 000000000..333cfcc28 --- /dev/null +++ b/fs/9p/acl.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +/* + * Copyright IBM Corporation, 2010 + * Author Aneesh Kumar K.V + */ +#ifndef FS_9P_ACL_H +#define FS_9P_ACL_H + +#ifdef CONFIG_9P_FS_POSIX_ACL +int v9fs_get_acl(struct inode *inode, struct p9_fid *fid); +struct posix_acl *v9fs_iop_get_inode_acl(struct inode *inode, int type, + bool rcu); +struct posix_acl *v9fs_iop_get_acl(struct mnt_idmap *idmap, + struct dentry *dentry, int type); +int v9fs_iop_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type); +int v9fs_acl_chmod(struct inode *inode, struct p9_fid *fid); +int v9fs_set_create_acl(struct inode *inode, struct p9_fid *fid, + struct posix_acl *dacl, struct posix_acl *acl); +int v9fs_acl_mode(struct inode *dir, umode_t *modep, + struct posix_acl **dpacl, struct posix_acl **pacl); +void v9fs_put_acl(struct posix_acl *dacl, struct posix_acl *acl); +#else +#define v9fs_iop_get_inode_acl NULL +#define v9fs_iop_get_acl NULL +#define v9fs_iop_set_acl NULL +static inline int v9fs_get_acl(struct inode *inode, struct p9_fid *fid) +{ + return 0; +} +static inline int v9fs_acl_chmod(struct inode *inode, struct p9_fid *fid) +{ + return 0; +} +static inline int v9fs_set_create_acl(struct inode *inode, + struct p9_fid *fid, + struct posix_acl *dacl, + struct posix_acl *acl) +{ + return 0; +} +static inline void v9fs_put_acl(struct posix_acl *dacl, + struct posix_acl *acl) +{ +} +static inline int v9fs_acl_mode(struct inode *dir, umode_t *modep, + struct posix_acl **dpacl, + struct posix_acl **pacl) +{ + return 0; +} + +#endif +#endif /* FS_9P_XATTR_H */ diff --git a/fs/9p/cache.c b/fs/9p/cache.c new file mode 100644 index 000000000..12c0ae29f --- /dev/null +++ b/fs/9p/cache.c @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * V9FS cache definitions. + * + * Copyright (C) 2009 by Abhishek Kulkarni + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "v9fs.h" +#include "cache.h" + +int v9fs_cache_session_get_cookie(struct v9fs_session_info *v9ses, + const char *dev_name) +{ + struct fscache_volume *vcookie; + char *name, *p; + + name = kasprintf(GFP_KERNEL, "9p,%s,%s", + dev_name, v9ses->cachetag ?: v9ses->aname); + if (!name) + return -ENOMEM; + + for (p = name; *p; p++) + if (*p == '/') + *p = ';'; + + vcookie = fscache_acquire_volume(name, NULL, NULL, 0); + p9_debug(P9_DEBUG_FSC, "session %p get volume %p (%s)\n", + v9ses, vcookie, name); + if (IS_ERR(vcookie)) { + if (vcookie != ERR_PTR(-EBUSY)) { + kfree(name); + return PTR_ERR(vcookie); + } + pr_err("Cache volume key already in use (%s)\n", name); + vcookie = NULL; + } + v9ses->fscache = vcookie; + kfree(name); + return 0; +} + +void v9fs_cache_inode_get_cookie(struct inode *inode) +{ + struct v9fs_inode *v9inode = V9FS_I(inode); + struct v9fs_session_info *v9ses; + __le32 version; + __le64 path; + + if (!S_ISREG(inode->i_mode)) + return; + if (WARN_ON(v9fs_inode_cookie(v9inode))) + return; + + version = cpu_to_le32(v9inode->qid.version); + path = cpu_to_le64(v9inode->qid.path); + v9ses = v9fs_inode2v9ses(inode); + v9inode->netfs.cache = + fscache_acquire_cookie(v9fs_session_cache(v9ses), + 0, + &path, sizeof(path), + &version, sizeof(version), + i_size_read(&v9inode->netfs.inode)); + if (v9inode->netfs.cache) + mapping_set_release_always(inode->i_mapping); + + p9_debug(P9_DEBUG_FSC, "inode %p get cookie %p\n", + inode, v9fs_inode_cookie(v9inode)); +} diff --git a/fs/9p/cache.h b/fs/9p/cache.h new file mode 100644 index 000000000..ee1b6b06a --- /dev/null +++ b/fs/9p/cache.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * V9FS cache definitions. + * + * Copyright (C) 2009 by Abhishek Kulkarni + */ + +#ifndef _9P_CACHE_H +#define _9P_CACHE_H + +#ifdef CONFIG_9P_FSCACHE +#include + +extern int v9fs_cache_session_get_cookie(struct v9fs_session_info *v9ses, + const char *dev_name); + +extern void v9fs_cache_inode_get_cookie(struct inode *inode); + +#else /* CONFIG_9P_FSCACHE */ + +static inline void v9fs_cache_inode_get_cookie(struct inode *inode) +{ +} + +#endif /* CONFIG_9P_FSCACHE */ +#endif /* _9P_CACHE_H */ diff --git a/fs/9p/fid.c b/fs/9p/fid.c new file mode 100644 index 000000000..76242d450 --- /dev/null +++ b/fs/9p/fid.c @@ -0,0 +1,316 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * V9FS FID Management + * + * Copyright (C) 2007 by Latchesar Ionkov + * Copyright (C) 2005, 2006 by Eric Van Hensbergen + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "v9fs.h" +#include "v9fs_vfs.h" +#include "fid.h" + +static inline void __add_fid(struct dentry *dentry, struct p9_fid *fid) +{ + struct v9fs_dentry *v9fs_dentry = to_v9fs_dentry(dentry); + + hlist_add_head(&fid->dlist, &v9fs_dentry->head); +} + + +/** + * v9fs_fid_add - add a fid to a dentry + * @dentry: dentry that the fid is being added to + * @pfid: fid to add, NULLed out + * + */ +void v9fs_fid_add(struct dentry *dentry, struct p9_fid **pfid) +{ + struct p9_fid *fid = *pfid; + + spin_lock(&dentry->d_lock); + __add_fid(dentry, fid); + spin_unlock(&dentry->d_lock); + + *pfid = NULL; +} + +static bool v9fs_is_writeable(int mode) +{ + if (mode & (P9_OWRITE|P9_ORDWR)) + return true; + else + return false; +} + +/** + * v9fs_fid_find_inode - search for an open fid off of the inode list + * @inode: return a fid pointing to a specific inode + * @want_writeable: only consider fids which are writeable + * @uid: return a fid belonging to the specified user + * @any: ignore uid as a selection criteria + * + */ +struct p9_fid *v9fs_fid_find_inode(struct inode *inode, bool want_writeable, + kuid_t uid, bool any) +{ + struct hlist_head *h; + struct p9_fid *fid, *ret = NULL; + + p9_debug(P9_DEBUG_VFS, " inode: %p\n", inode); + + spin_lock(&inode->i_lock); + h = (struct hlist_head *)&inode->i_private; + hlist_for_each_entry(fid, h, ilist) { + if (any || uid_eq(fid->uid, uid)) { + if (want_writeable && !v9fs_is_writeable(fid->mode)) { + p9_debug(P9_DEBUG_VFS, " mode: %x not writeable?\n", + fid->mode); + continue; + } + p9_fid_get(fid); + ret = fid; + break; + } + } + spin_unlock(&inode->i_lock); + return ret; +} + +/** + * v9fs_open_fid_add - add an open fid to an inode + * @inode: inode that the fid is being added to + * @pfid: fid to add, NULLed out + * + */ + +void v9fs_open_fid_add(struct inode *inode, struct p9_fid **pfid) +{ + struct p9_fid *fid = *pfid; + + spin_lock(&inode->i_lock); + hlist_add_head(&fid->ilist, (struct hlist_head *)&inode->i_private); + spin_unlock(&inode->i_lock); + + *pfid = NULL; +} + + +/** + * v9fs_fid_find - retrieve a fid that belongs to the specified uid + * @dentry: dentry to look for fid in + * @uid: return fid that belongs to the specified user + * @any: if non-zero, return any fid associated with the dentry + * + */ + +static struct p9_fid *v9fs_fid_find(struct dentry *dentry, kuid_t uid, int any) +{ + struct v9fs_dentry *v9fs_dentry = to_v9fs_dentry(dentry); + struct p9_fid *fid, *ret; + + p9_debug(P9_DEBUG_VFS, " dentry: %pd (%p) uid %d any %d\n", + dentry, dentry, from_kuid(&init_user_ns, uid), + any); + ret = NULL; + /* we'll recheck under lock if there's anything to look in */ + if (!hlist_empty(&v9fs_dentry->head)) { + spin_lock(&dentry->d_lock); + hlist_for_each_entry(fid, &v9fs_dentry->head, dlist) { + if (any || uid_eq(fid->uid, uid)) { + ret = fid; + p9_fid_get(ret); + break; + } + } + spin_unlock(&dentry->d_lock); + } + if (!ret && dentry->d_inode) + ret = v9fs_fid_find_inode(dentry->d_inode, false, uid, any); + + return ret; +} + +/* + * We need to hold v9ses->rename_sem as long as we hold references + * to returned path array. Array element contain pointers to + * dentry names. + */ +static int build_path_from_dentry(struct v9fs_session_info *v9ses, + struct dentry *dentry, const unsigned char ***names) +{ + int n = 0, i; + const unsigned char **wnames; + struct dentry *ds; + + for (ds = dentry; !IS_ROOT(ds); ds = ds->d_parent) + n++; + + wnames = kmalloc_array(n, sizeof(char *), GFP_KERNEL); + if (!wnames) + goto err_out; + + for (ds = dentry, i = (n-1); i >= 0; i--, ds = ds->d_parent) + wnames[i] = ds->d_name.name; + + *names = wnames; + return n; +err_out: + return -ENOMEM; +} + +static struct p9_fid *v9fs_fid_lookup_with_uid(struct dentry *dentry, + kuid_t uid, int any) +{ + struct dentry *ds; + const unsigned char **wnames, *uname; + int i, n, l, access; + struct v9fs_session_info *v9ses; + struct p9_fid *fid, *root_fid, *old_fid; + + v9ses = v9fs_dentry2v9ses(dentry); + access = v9ses->flags & V9FS_ACCESS_MASK; + fid = v9fs_fid_find(dentry, uid, any); + if (fid) + return fid; + /* + * we don't have a matching fid. To do a TWALK we need + * parent fid. We need to prevent rename when we want to + * look at the parent. + */ + down_read(&v9ses->rename_sem); + ds = dentry->d_parent; + fid = v9fs_fid_find(ds, uid, any); + if (fid) { + /* Found the parent fid do a lookup with that */ + old_fid = fid; + + fid = p9_client_walk(old_fid, 1, &dentry->d_name.name, 1); + p9_fid_put(old_fid); + goto fid_out; + } + up_read(&v9ses->rename_sem); + + /* start from the root and try to do a lookup */ + root_fid = v9fs_fid_find(dentry->d_sb->s_root, uid, any); + if (!root_fid) { + /* the user is not attached to the fs yet */ + if (access == V9FS_ACCESS_SINGLE) + return ERR_PTR(-EPERM); + + if (v9fs_proto_dotu(v9ses) || v9fs_proto_dotl(v9ses)) + uname = NULL; + else + uname = v9ses->uname; + + fid = p9_client_attach(v9ses->clnt, NULL, uname, uid, + v9ses->aname); + if (IS_ERR(fid)) + return fid; + + root_fid = p9_fid_get(fid); + v9fs_fid_add(dentry->d_sb->s_root, &fid); + } + /* If we are root ourself just return that */ + if (dentry->d_sb->s_root == dentry) + return root_fid; + + /* + * Do a multipath walk with attached root. + * When walking parent we need to make sure we + * don't have a parallel rename happening + */ + down_read(&v9ses->rename_sem); + n = build_path_from_dentry(v9ses, dentry, &wnames); + if (n < 0) { + fid = ERR_PTR(n); + goto err_out; + } + fid = root_fid; + old_fid = root_fid; + i = 0; + while (i < n) { + l = min(n - i, P9_MAXWELEM); + /* + * We need to hold rename lock when doing a multipath + * walk to ensure none of the path components change + */ + fid = p9_client_walk(old_fid, l, &wnames[i], + old_fid == root_fid /* clone */); + /* non-cloning walk will return the same fid */ + if (fid != old_fid) { + p9_fid_put(old_fid); + old_fid = fid; + } + if (IS_ERR(fid)) { + kfree(wnames); + goto err_out; + } + i += l; + } + kfree(wnames); +fid_out: + if (!IS_ERR(fid)) { + spin_lock(&dentry->d_lock); + if (d_unhashed(dentry)) { + spin_unlock(&dentry->d_lock); + p9_fid_put(fid); + fid = ERR_PTR(-ENOENT); + } else { + __add_fid(dentry, fid); + p9_fid_get(fid); + spin_unlock(&dentry->d_lock); + } + } +err_out: + up_read(&v9ses->rename_sem); + return fid; +} + +/** + * v9fs_fid_lookup - lookup for a fid, try to walk if not found + * @dentry: dentry to look for fid in + * + * Look for a fid in the specified dentry for the current user. + * If no fid is found, try to create one walking from a fid from the parent + * dentry (if it has one), or the root dentry. If the user haven't accessed + * the fs yet, attach now and walk from the root. + */ + +struct p9_fid *v9fs_fid_lookup(struct dentry *dentry) +{ + kuid_t uid; + int any, access; + struct v9fs_session_info *v9ses; + + v9ses = v9fs_dentry2v9ses(dentry); + access = v9ses->flags & V9FS_ACCESS_MASK; + switch (access) { + case V9FS_ACCESS_SINGLE: + case V9FS_ACCESS_USER: + case V9FS_ACCESS_CLIENT: + uid = current_fsuid(); + any = 0; + break; + + case V9FS_ACCESS_ANY: + uid = v9ses->uid; + any = 1; + break; + + default: + uid = INVALID_UID; + any = 0; + break; + } + return v9fs_fid_lookup_with_uid(dentry, uid, any); +} + diff --git a/fs/9p/fid.h b/fs/9p/fid.h new file mode 100644 index 000000000..0d6138bee --- /dev/null +++ b/fs/9p/fid.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * V9FS FID Management + * + * Copyright (C) 2005 by Eric Van Hensbergen + */ +#ifndef FS_9P_FID_H +#define FS_9P_FID_H +#include +#include "v9fs.h" + +struct p9_fid *v9fs_fid_find_inode(struct inode *inode, bool want_writeable, + kuid_t uid, bool any); +struct p9_fid *v9fs_fid_lookup(struct dentry *dentry); +static inline struct p9_fid *v9fs_parent_fid(struct dentry *dentry) +{ + return v9fs_fid_lookup(dentry->d_parent); +} +void v9fs_fid_add(struct dentry *dentry, struct p9_fid **fid); +void v9fs_open_fid_add(struct inode *inode, struct p9_fid **fid); +static inline struct p9_fid *clone_fid(struct p9_fid *fid) +{ + return IS_ERR(fid) ? fid : p9_client_walk(fid, 0, NULL, 1); +} +static inline struct p9_fid *v9fs_fid_clone(struct dentry *dentry) +{ + struct p9_fid *fid, *nfid; + + fid = v9fs_fid_lookup(dentry); + if (!fid || IS_ERR(fid)) + return fid; + + nfid = clone_fid(fid); + p9_fid_put(fid); + return nfid; +} +/** + * v9fs_fid_addmodes - add cache flags to fid mode (for client use only) + * @fid: fid to augment + * @s_flags: session info mount flags + * @s_cache: session info cache flags + * @f_flags: unix open flags + * + * make sure mode reflects flags of underlying mounts + * also qid.version == 0 reflects a synthetic or legacy file system + * NOTE: these are set after open so only reflect 9p client not + * underlying file system on server. + */ +static inline void v9fs_fid_add_modes(struct p9_fid *fid, unsigned int s_flags, + unsigned int s_cache, unsigned int f_flags) +{ + if ((!s_cache) || + ((fid->qid.version == 0) && !(s_flags & V9FS_IGNORE_QV)) || + (s_flags & V9FS_DIRECT_IO) || (f_flags & O_DIRECT)) { + fid->mode |= P9L_DIRECT; /* no read or write cache */ + } else if ((!(s_cache & CACHE_WRITEBACK)) || + (f_flags & O_DSYNC) || (s_flags & V9FS_SYNC)) { + fid->mode |= P9L_NOWRITECACHE; + } +} +#endif diff --git a/fs/9p/v9fs.c b/fs/9p/v9fs.c new file mode 100644 index 000000000..274c51571 --- /dev/null +++ b/fs/9p/v9fs.c @@ -0,0 +1,786 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file contains functions assisting in mapping VFS to 9P2000 + * + * Copyright (C) 2004-2008 by Eric Van Hensbergen + * Copyright (C) 2002 by Ron Minnich + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "v9fs.h" +#include "v9fs_vfs.h" +#include "cache.h" + +/* cache=loose default negative dentry retention time is 24hours */ +#define CACHE_LOOSE_NDENTRY_TIMEOUT_DEFAULT (24 * 60 * 60 * 1000) + +static DEFINE_SPINLOCK(v9fs_sessionlist_lock); +static LIST_HEAD(v9fs_sessionlist); +struct kmem_cache *v9fs_inode_cache; + +/* + * Option Parsing (code inspired by NFS code) + * NOTE: each transport will parse its own options + */ + +enum { + /* Mount-point source, we need to handle this explicitly because + * the code below accepts unknown args and the vfs layer only handles + * source if we rejected it as EINVAL */ + Opt_source, + /* Options that take integer arguments */ + Opt_debug, Opt_dfltuid, Opt_dfltgid, Opt_afid, Opt_negtimeout, + /* String options */ + Opt_uname, Opt_remotename, Opt_cache, Opt_cachetag, + /* Options that take no arguments */ + Opt_nodevmap, Opt_noxattr, Opt_directio, Opt_ignoreqv, + /* Access options */ + Opt_access, Opt_posixacl, + /* Lock timeout option */ + Opt_locktimeout, + + /* Client options */ + Opt_msize, Opt_trans, Opt_legacy, Opt_version, + + /* fd transport options */ + /* Options that take integer arguments */ + Opt_rfdno, Opt_wfdno, + /* Options that take no arguments */ + + /* rdma transport options */ + /* Options that take integer arguments */ + Opt_rq_depth, Opt_sq_depth, Opt_timeout, + + /* Options for both fd and rdma transports */ + Opt_port, Opt_privport, +}; + +static const struct constant_table p9_versions[] = { + { "9p2000", p9_proto_legacy }, + { "9p2000.u", p9_proto_2000u }, + { "9p2000.L", p9_proto_2000L }, + {} +}; + +/* + * This structure contains all parameters used for the core code, + * the client, and all the transports. + */ +const struct fs_parameter_spec v9fs_param_spec[] = { + fsparam_string ("source", Opt_source), + fsparam_u32hex ("debug", Opt_debug), + fsparam_uid ("dfltuid", Opt_dfltuid), + fsparam_gid ("dfltgid", Opt_dfltgid), + fsparam_u32 ("afid", Opt_afid), + fsparam_string ("uname", Opt_uname), + fsparam_string ("aname", Opt_remotename), + fsparam_flag ("nodevmap", Opt_nodevmap), + fsparam_flag ("noxattr", Opt_noxattr), + fsparam_flag ("directio", Opt_directio), + fsparam_flag ("ignoreqv", Opt_ignoreqv), + fsparam_string ("cache", Opt_cache), + fsparam_string ("cachetag", Opt_cachetag), + fsparam_string ("access", Opt_access), + fsparam_flag ("posixacl", Opt_posixacl), + fsparam_u32 ("locktimeout", Opt_locktimeout), + fsparam_s32 ("negtimeout", Opt_negtimeout), + + /* client options */ + fsparam_u32 ("msize", Opt_msize), + fsparam_flag ("noextend", Opt_legacy), + fsparam_string ("trans", Opt_trans), + fsparam_enum ("version", Opt_version, p9_versions), + + /* fd transport options */ + fsparam_u32 ("rfdno", Opt_rfdno), + fsparam_u32 ("wfdno", Opt_wfdno), + + /* rdma transport options */ + fsparam_u32 ("sq", Opt_sq_depth), + fsparam_u32 ("rq", Opt_rq_depth), + fsparam_u32 ("timeout", Opt_timeout), + + /* fd and rdma transprt options */ + fsparam_u32 ("port", Opt_port), + fsparam_flag ("privport", Opt_privport), + {} +}; + +/* Interpret mount options for cache mode */ +static int get_cache_mode(char *s) +{ + int version = -EINVAL; + + if (!strcmp(s, "loose")) { + version = CACHE_SC_LOOSE; + p9_debug(P9_DEBUG_9P, "Cache mode: loose\n"); + } else if (!strcmp(s, "fscache")) { + version = CACHE_SC_FSCACHE; + p9_debug(P9_DEBUG_9P, "Cache mode: fscache\n"); + } else if (!strcmp(s, "mmap")) { + version = CACHE_SC_MMAP; + p9_debug(P9_DEBUG_9P, "Cache mode: mmap\n"); + } else if (!strcmp(s, "readahead")) { + version = CACHE_SC_READAHEAD; + p9_debug(P9_DEBUG_9P, "Cache mode: readahead\n"); + } else if (!strcmp(s, "none")) { + version = CACHE_SC_NONE; + p9_debug(P9_DEBUG_9P, "Cache mode: none\n"); + } else if (kstrtoint(s, 0, &version) != 0) { + version = -EINVAL; + pr_info("Unknown Cache mode or invalid value %s\n", s); + } + return version; +} + +/* + * Display the mount options in /proc/mounts. + */ +int v9fs_show_options(struct seq_file *m, struct dentry *root) +{ + struct v9fs_session_info *v9ses = root->d_sb->s_fs_info; + + if (v9ses->debug) + seq_printf(m, ",debug=%#x", v9ses->debug); + if (!uid_eq(v9ses->dfltuid, V9FS_DEFUID)) + seq_printf(m, ",dfltuid=%u", + from_kuid_munged(&init_user_ns, v9ses->dfltuid)); + if (!gid_eq(v9ses->dfltgid, V9FS_DEFGID)) + seq_printf(m, ",dfltgid=%u", + from_kgid_munged(&init_user_ns, v9ses->dfltgid)); + if (v9ses->afid != ~0) + seq_printf(m, ",afid=%u", v9ses->afid); + if (v9ses->flags & V9FS_NDENTRY_TIMEOUT_SET) + seq_printf(m, ",negtimeout=%d", + (int)v9ses->ndentry_timeout_ms); + if (strcmp(v9ses->uname, V9FS_DEFUSER) != 0) + seq_printf(m, ",uname=%s", v9ses->uname); + if (strcmp(v9ses->aname, V9FS_DEFANAME) != 0) + seq_printf(m, ",aname=%s", v9ses->aname); + if (v9ses->nodev) + seq_puts(m, ",nodevmap"); + if (v9ses->cache) + seq_printf(m, ",cache=%#x", v9ses->cache); +#ifdef CONFIG_9P_FSCACHE + if (v9ses->cachetag && (v9ses->cache & CACHE_FSCACHE)) + seq_printf(m, ",cachetag=%s", v9ses->cachetag); +#endif + + switch (v9ses->flags & V9FS_ACCESS_MASK) { + case V9FS_ACCESS_USER: + seq_puts(m, ",access=user"); + break; + case V9FS_ACCESS_ANY: + seq_puts(m, ",access=any"); + break; + case V9FS_ACCESS_CLIENT: + seq_puts(m, ",access=client"); + break; + case V9FS_ACCESS_SINGLE: + seq_printf(m, ",access=%u", + from_kuid_munged(&init_user_ns, v9ses->uid)); + break; + } + + if (v9ses->flags & V9FS_IGNORE_QV) + seq_puts(m, ",ignoreqv"); + if (v9ses->flags & V9FS_DIRECT_IO) + seq_puts(m, ",directio"); + if (v9ses->flags & V9FS_POSIX_ACL) + seq_puts(m, ",posixacl"); + + if (v9ses->flags & V9FS_NO_XATTR) + seq_puts(m, ",noxattr"); + + return p9_show_client_options(m, v9ses->clnt); +} + +/** + * v9fs_parse_param - parse a mount option into the filesystem context + * @fc: the filesystem context + * @param: the parameter to parse + * + * Return 0 upon success, -ERRNO upon failure. + */ +int v9fs_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct v9fs_context *ctx = fc->fs_private; + struct fs_parse_result result; + char *s; + int r; + int opt; + struct p9_client_opts *clnt = &ctx->client_opts; + struct p9_fd_opts *fd_opts = &ctx->fd_opts; + struct p9_rdma_opts *rdma_opts = &ctx->rdma_opts; + struct p9_session_opts *session_opts = &ctx->session_opts; + + opt = fs_parse(fc, v9fs_param_spec, param, &result); + if (opt < 0) { + /* + * We might like to report bad mount options here, but + * traditionally 9p has ignored unknown mount options + */ + if (opt == -ENOPARAM) + return 0; + + return opt; + } + + switch (opt) { + case Opt_source: + if (fc->source) { + pr_info("p9: multiple sources not supported\n"); + return -EINVAL; + } + fc->source = param->string; + param->string = NULL; + break; + case Opt_debug: + session_opts->debug = result.uint_32; +#ifdef CONFIG_NET_9P_DEBUG + p9_debug_level = result.uint_32; +#endif + break; + + case Opt_dfltuid: + session_opts->dfltuid = result.uid; + break; + case Opt_dfltgid: + session_opts->dfltgid = result.gid; + break; + case Opt_afid: + session_opts->afid = result.uint_32; + break; + case Opt_uname: + kfree(session_opts->uname); + session_opts->uname = param->string; + param->string = NULL; + break; + case Opt_remotename: + kfree(session_opts->aname); + session_opts->aname = param->string; + param->string = NULL; + break; + case Opt_nodevmap: + session_opts->nodev = 1; + break; + case Opt_noxattr: + session_opts->flags |= V9FS_NO_XATTR; + break; + case Opt_directio: + session_opts->flags |= V9FS_DIRECT_IO; + break; + case Opt_ignoreqv: + session_opts->flags |= V9FS_IGNORE_QV; + break; + case Opt_cachetag: +#ifdef CONFIG_9P_FSCACHE + kfree(session_opts->cachetag); + session_opts->cachetag = param->string; + param->string = NULL; +#endif + break; + case Opt_cache: + r = get_cache_mode(param->string); + if (r < 0) + return r; + session_opts->cache = r; + break; + case Opt_access: + s = param->string; + session_opts->flags &= ~V9FS_ACCESS_MASK; + if (strcmp(s, "user") == 0) { + session_opts->flags |= V9FS_ACCESS_USER; + } else if (strcmp(s, "any") == 0) { + session_opts->flags |= V9FS_ACCESS_ANY; + } else if (strcmp(s, "client") == 0) { + session_opts->flags |= V9FS_ACCESS_CLIENT; + } else { + uid_t uid; + + session_opts->flags |= V9FS_ACCESS_SINGLE; + r = kstrtouint(s, 10, &uid); + if (r) { + pr_info("Unknown access argument %s: %d\n", + param->string, r); + return r; + } + session_opts->uid = make_kuid(current_user_ns(), uid); + if (!uid_valid(session_opts->uid)) { + pr_info("Unknown uid %s\n", s); + return -EINVAL; + } + } + break; + + case Opt_posixacl: +#ifdef CONFIG_9P_FS_POSIX_ACL + session_opts->flags |= V9FS_POSIX_ACL; +#else + p9_debug(P9_DEBUG_ERROR, + "Not defined CONFIG_9P_FS_POSIX_ACL. Ignoring posixacl option\n"); +#endif + break; + + case Opt_locktimeout: + if (result.uint_32 < 1) { + p9_debug(P9_DEBUG_ERROR, + "locktimeout must be a greater than zero integer.\n"); + return -EINVAL; + } + session_opts->session_lock_timeout = (long)result.uint_32 * HZ; + break; + + case Opt_negtimeout: + session_opts->flags |= V9FS_NDENTRY_TIMEOUT_SET; + if (result.int_32 < 0) { + session_opts->ndentry_timeout_ms = + NDENTRY_TIMEOUT_NEVER; + } else { + session_opts->ndentry_timeout_ms = result.int_32; + } + break; + + /* Options for client */ + case Opt_msize: + if (result.uint_32 < 4096) { + p9_debug(P9_DEBUG_ERROR, "msize should be at least 4k\n"); + return -EINVAL; + } + if (result.uint_32 > INT_MAX) { + p9_debug(P9_DEBUG_ERROR, "msize too big\n"); + return -EINVAL; + } + clnt->msize = result.uint_32; + break; + case Opt_trans: + v9fs_put_trans(clnt->trans_mod); + clnt->trans_mod = v9fs_get_trans_by_name(param->string); + if (!clnt->trans_mod) { + pr_info("Could not find request transport: %s\n", + param->string); + return -EINVAL; + } + break; + case Opt_legacy: + clnt->proto_version = p9_proto_legacy; + break; + case Opt_version: + clnt->proto_version = result.uint_32; + p9_debug(P9_DEBUG_9P, "Protocol version: %s\n", param->string); + break; + /* Options for fd transport */ + case Opt_rfdno: + fd_opts->rfd = result.uint_32; + break; + case Opt_wfdno: + fd_opts->wfd = result.uint_32; + break; + /* Options for rdma transport */ + case Opt_sq_depth: + rdma_opts->sq_depth = result.uint_32; + break; + case Opt_rq_depth: + rdma_opts->rq_depth = result.uint_32; + break; + case Opt_timeout: + rdma_opts->timeout = result.uint_32; + break; + /* Options for both fd and rdma transports */ + case Opt_port: + fd_opts->port = result.uint_32; + rdma_opts->port = result.uint_32; + break; + case Opt_privport: + fd_opts->privport = true; + rdma_opts->port = true; + break; + } + + return 0; +} + +static void v9fs_apply_options(struct v9fs_session_info *v9ses, + struct fs_context *fc) +{ + struct v9fs_context *ctx = fc->fs_private; + + v9ses->debug = ctx->session_opts.debug; + v9ses->dfltuid = ctx->session_opts.dfltuid; + v9ses->dfltgid = ctx->session_opts.dfltgid; + v9ses->afid = ctx->session_opts.afid; + v9ses->uname = ctx->session_opts.uname; + ctx->session_opts.uname = NULL; + v9ses->aname = ctx->session_opts.aname; + ctx->session_opts.aname = NULL; + v9ses->nodev = ctx->session_opts.nodev; + /* + * Note that we must |= flags here as session_init already + * set basic flags. This adds in flags from parsed options. + * Default access flags must be cleared if session options + * changes them to avoid mangling the setting. + */ + if (ctx->session_opts.flags & V9FS_ACCESS_MASK) + v9ses->flags &= ~V9FS_ACCESS_MASK; + v9ses->flags |= ctx->session_opts.flags; +#ifdef CONFIG_9P_FSCACHE + v9ses->cachetag = ctx->session_opts.cachetag; + ctx->session_opts.cachetag = NULL; +#endif + v9ses->cache = ctx->session_opts.cache; + v9ses->uid = ctx->session_opts.uid; + v9ses->session_lock_timeout = ctx->session_opts.session_lock_timeout; + v9ses->ndentry_timeout_ms = ctx->session_opts.ndentry_timeout_ms; + + /* If negative dentry timeout has not been overridden set default for + * cache=loose + */ + if (!(v9ses->flags & V9FS_NDENTRY_TIMEOUT_SET) && + (v9ses->cache & CACHE_LOOSE)) + v9ses->ndentry_timeout_ms = CACHE_LOOSE_NDENTRY_TIMEOUT_DEFAULT; +} + +/** + * v9fs_session_init - initialize session + * @v9ses: session information structure + * @fc: the filesystem mount context + * + */ + +struct p9_fid *v9fs_session_init(struct v9fs_session_info *v9ses, + struct fs_context *fc) +{ + struct p9_fid *fid; + int rc = -ENOMEM; + + init_rwsem(&v9ses->rename_sem); + + v9ses->clnt = p9_client_create(fc); + if (IS_ERR(v9ses->clnt)) { + rc = PTR_ERR(v9ses->clnt); + p9_debug(P9_DEBUG_ERROR, "problem initializing 9p client\n"); + goto err_names; + } + + /* + * Initialize flags on the real v9ses. v9fs_apply_options below + * will |= the additional flags from parsed options. + */ + v9ses->flags = V9FS_ACCESS_USER; + + if (p9_is_proto_dotl(v9ses->clnt)) { + v9ses->flags = V9FS_ACCESS_CLIENT; + v9ses->flags |= V9FS_PROTO_2000L; + } else if (p9_is_proto_dotu(v9ses->clnt)) { + v9ses->flags |= V9FS_PROTO_2000U; + } + + v9fs_apply_options(v9ses, fc); + + v9ses->maxdata = v9ses->clnt->msize - P9_IOHDRSZ; + + if (!v9fs_proto_dotl(v9ses) && + ((v9ses->flags & V9FS_ACCESS_MASK) == V9FS_ACCESS_CLIENT)) { + /* + * We support ACCESS_CLIENT only for dotl. + * Fall back to ACCESS_USER + */ + v9ses->flags &= ~V9FS_ACCESS_MASK; + v9ses->flags |= V9FS_ACCESS_USER; + } + /* FIXME: for legacy mode, fall back to V9FS_ACCESS_ANY */ + if (!(v9fs_proto_dotu(v9ses) || v9fs_proto_dotl(v9ses)) && + ((v9ses->flags&V9FS_ACCESS_MASK) == V9FS_ACCESS_USER)) { + + v9ses->flags &= ~V9FS_ACCESS_MASK; + v9ses->flags |= V9FS_ACCESS_ANY; + v9ses->uid = INVALID_UID; + } + if (!v9fs_proto_dotl(v9ses) || + !((v9ses->flags & V9FS_ACCESS_MASK) == V9FS_ACCESS_CLIENT)) { + /* + * We support ACL checks on client only if the protocol is + * 9P2000.L and access is V9FS_ACCESS_CLIENT. + */ + v9ses->flags &= ~V9FS_ACL_MASK; + } + + fid = p9_client_attach(v9ses->clnt, NULL, v9ses->uname, INVALID_UID, + v9ses->aname); + if (IS_ERR(fid)) { + rc = PTR_ERR(fid); + p9_debug(P9_DEBUG_ERROR, "cannot attach\n"); + goto err_clnt; + } + + if ((v9ses->flags & V9FS_ACCESS_MASK) == V9FS_ACCESS_SINGLE) + fid->uid = v9ses->uid; + else + fid->uid = INVALID_UID; + +#ifdef CONFIG_9P_FSCACHE + /* register the session for caching */ + if (v9ses->cache & CACHE_FSCACHE) { + rc = v9fs_cache_session_get_cookie(v9ses, fc->source); + if (rc < 0) + goto err_clnt; + } +#endif + spin_lock(&v9fs_sessionlist_lock); + list_add(&v9ses->slist, &v9fs_sessionlist); + spin_unlock(&v9fs_sessionlist_lock); + + return fid; + +err_clnt: +#ifdef CONFIG_9P_FSCACHE + kfree(v9ses->cachetag); +#endif + p9_client_destroy(v9ses->clnt); +err_names: + kfree(v9ses->uname); + kfree(v9ses->aname); + return ERR_PTR(rc); +} + +/** + * v9fs_session_close - shutdown a session + * @v9ses: session information structure + * + */ + +void v9fs_session_close(struct v9fs_session_info *v9ses) +{ + if (v9ses->clnt) { + p9_client_destroy(v9ses->clnt); + v9ses->clnt = NULL; + } + +#ifdef CONFIG_9P_FSCACHE + fscache_relinquish_volume(v9fs_session_cache(v9ses), NULL, false); + kfree(v9ses->cachetag); +#endif + kfree(v9ses->uname); + kfree(v9ses->aname); + + spin_lock(&v9fs_sessionlist_lock); + list_del(&v9ses->slist); + spin_unlock(&v9fs_sessionlist_lock); +} + +/** + * v9fs_session_cancel - terminate a session + * @v9ses: session to terminate + * + * mark transport as disconnected and cancel all pending requests. + */ + +void v9fs_session_cancel(struct v9fs_session_info *v9ses) +{ + p9_debug(P9_DEBUG_ERROR, "cancel session %p\n", v9ses); + p9_client_disconnect(v9ses->clnt); +} + +/** + * v9fs_session_begin_cancel - Begin terminate of a session + * @v9ses: session to terminate + * + * After this call we don't allow any request other than clunk. + */ + +void v9fs_session_begin_cancel(struct v9fs_session_info *v9ses) +{ + p9_debug(P9_DEBUG_ERROR, "begin cancel session %p\n", v9ses); + p9_client_begin_disconnect(v9ses->clnt); +} + +static struct kobject *v9fs_kobj; + +#ifdef CONFIG_9P_FSCACHE +/* + * List caches associated with a session + */ +static ssize_t caches_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + ssize_t n = 0, count = 0, limit = PAGE_SIZE; + struct v9fs_session_info *v9ses; + + spin_lock(&v9fs_sessionlist_lock); + list_for_each_entry(v9ses, &v9fs_sessionlist, slist) { + if (v9ses->cachetag) { + n = snprintf(buf + count, limit, "%s\n", v9ses->cachetag); + if (n < 0) { + count = n; + break; + } + + count += n; + limit -= n; + } + } + + spin_unlock(&v9fs_sessionlist_lock); + return count; +} + +static struct kobj_attribute v9fs_attr_cache = __ATTR_RO(caches); +#endif /* CONFIG_9P_FSCACHE */ + +static struct attribute *v9fs_attrs[] = { +#ifdef CONFIG_9P_FSCACHE + &v9fs_attr_cache.attr, +#endif + NULL, +}; + +static const struct attribute_group v9fs_attr_group = { + .attrs = v9fs_attrs, +}; + +/** + * v9fs_sysfs_init - Initialize the v9fs sysfs interface + * + */ + +static int __init v9fs_sysfs_init(void) +{ + int ret; + + v9fs_kobj = kobject_create_and_add("9p", fs_kobj); + if (!v9fs_kobj) + return -ENOMEM; + + ret = sysfs_create_group(v9fs_kobj, &v9fs_attr_group); + if (ret) { + kobject_put(v9fs_kobj); + return ret; + } + + return 0; +} + +/** + * v9fs_sysfs_cleanup - Unregister the v9fs sysfs interface + * + */ + +static void v9fs_sysfs_cleanup(void) +{ + sysfs_remove_group(v9fs_kobj, &v9fs_attr_group); + kobject_put(v9fs_kobj); +} + +static void v9fs_inode_init_once(void *foo) +{ + struct v9fs_inode *v9inode = (struct v9fs_inode *)foo; + + memset(&v9inode->qid, 0, sizeof(v9inode->qid)); + inode_init_once(&v9inode->netfs.inode); +} + +/** + * v9fs_init_inode_cache - initialize a cache for 9P + * Returns 0 on success. + */ +static int v9fs_init_inode_cache(void) +{ + v9fs_inode_cache = kmem_cache_create("v9fs_inode_cache", + sizeof(struct v9fs_inode), + 0, (SLAB_RECLAIM_ACCOUNT| + SLAB_ACCOUNT), + v9fs_inode_init_once); + if (!v9fs_inode_cache) + return -ENOMEM; + + return 0; +} + +/** + * v9fs_destroy_inode_cache - destroy the cache of 9P inode + * + */ +static void v9fs_destroy_inode_cache(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(v9fs_inode_cache); +} + +/** + * init_v9fs - Initialize module + * + */ + +static int __init init_v9fs(void) +{ + int err; + + pr_info("Installing v9fs 9p2000 file system support\n"); + /* TODO: Setup list of registered transport modules */ + + err = v9fs_init_inode_cache(); + if (err < 0) { + pr_err("Failed to register v9fs for caching\n"); + return err; + } + + err = v9fs_sysfs_init(); + if (err < 0) { + pr_err("Failed to register with sysfs\n"); + goto out_cache; + } + err = register_filesystem(&v9fs_fs_type); + if (err < 0) { + pr_err("Failed to register filesystem\n"); + goto out_sysfs_cleanup; + } + + return 0; + +out_sysfs_cleanup: + v9fs_sysfs_cleanup(); + +out_cache: + v9fs_destroy_inode_cache(); + + return err; +} + +/** + * exit_v9fs - shutdown module + * + */ + +static void __exit exit_v9fs(void) +{ + v9fs_sysfs_cleanup(); + v9fs_destroy_inode_cache(); + unregister_filesystem(&v9fs_fs_type); +} + +module_init(init_v9fs) +module_exit(exit_v9fs) + +MODULE_AUTHOR("Latchesar Ionkov "); +MODULE_AUTHOR("Eric Van Hensbergen "); +MODULE_AUTHOR("Ron Minnich "); +MODULE_DESCRIPTION("9P Client File System"); +MODULE_LICENSE("GPL"); diff --git a/fs/9p/v9fs.h b/fs/9p/v9fs.h new file mode 100644 index 000000000..a462bcbfc --- /dev/null +++ b/fs/9p/v9fs.h @@ -0,0 +1,269 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * V9FS definitions. + * + * Copyright (C) 2004-2008 by Eric Van Hensbergen + * Copyright (C) 2002 by Ron Minnich + */ +#ifndef FS_9P_V9FS_H +#define FS_9P_V9FS_H + +#include +#include +#include +#include +#include + +/** + * enum p9_session_flags - option flags for each 9P session + * @V9FS_PROTO_2000U: whether or not to use 9P2000.u extensions + * @V9FS_PROTO_2000L: whether or not to use 9P2000.l extensions + * @V9FS_ACCESS_SINGLE: only the mounting user can access the hierarchy + * @V9FS_ACCESS_USER: a new attach will be issued for every user (default) + * @V9FS_ACCESS_CLIENT: Just like user, but access check is performed on client. + * @V9FS_ACCESS_ANY: use a single attach for all users + * @V9FS_ACCESS_MASK: bit mask of different ACCESS options + * @V9FS_POSIX_ACL: POSIX ACLs are enforced + * @V9FS_NDENTRY_TIMEOUT_SET: Has negative dentry timeout retention time been + * overridden by negtimeout mount option + * + * Session flags reflect options selected by users at mount time + */ +#define V9FS_ACCESS_ANY (V9FS_ACCESS_SINGLE | \ + V9FS_ACCESS_USER | \ + V9FS_ACCESS_CLIENT) +#define V9FS_ACCESS_MASK V9FS_ACCESS_ANY +#define V9FS_ACL_MASK V9FS_POSIX_ACL + +enum p9_session_flags { + V9FS_PROTO_2000U = 0x01, + V9FS_PROTO_2000L = 0x02, + V9FS_ACCESS_SINGLE = 0x04, + V9FS_ACCESS_USER = 0x08, + V9FS_ACCESS_CLIENT = 0x10, + V9FS_POSIX_ACL = 0x20, + V9FS_NO_XATTR = 0x40, + V9FS_IGNORE_QV = 0x80, /* ignore qid.version for cache hints */ + V9FS_DIRECT_IO = 0x100, + V9FS_SYNC = 0x200, + V9FS_NDENTRY_TIMEOUT_SET = 0x400, +}; + +/** + * enum p9_cache_shortcuts - human readable cache preferences + * @CACHE_SC_NONE: disable all caches + * @CACHE_SC_READAHEAD: only provide caching for readahead + * @CACHE_SC_MMAP: provide caching to enable mmap + * @CACHE_SC_LOOSE: non-coherent caching for files and meta data + * @CACHE_SC_FSCACHE: persistent non-coherent caching for files and meta-data + * + */ + +enum p9_cache_shortcuts { + CACHE_SC_NONE = 0b00000000, + CACHE_SC_READAHEAD = 0b00000001, + CACHE_SC_MMAP = 0b00000101, + CACHE_SC_LOOSE = 0b00001111, + CACHE_SC_FSCACHE = 0b10001111, +}; + +/** + * enum p9_cache_bits - possible values of ->cache + * @CACHE_NONE: caches disabled + * @CACHE_FILE: file caching (open to close) + * @CACHE_META: meta-data and directory caching + * @CACHE_WRITEBACK: write-back caching for files + * @CACHE_LOOSE: don't check cache consistency + * @CACHE_FSCACHE: local persistent caches + * + */ + +enum p9_cache_bits { + CACHE_NONE = 0b00000000, + CACHE_FILE = 0b00000001, + CACHE_META = 0b00000010, + CACHE_WRITEBACK = 0b00000100, + CACHE_LOOSE = 0b00001000, + CACHE_FSCACHE = 0b10000000, +}; + +/** + * struct v9fs_session_info - per-instance session information + * @flags: session options of type &p9_session_flags + * @nodev: set to 1 to disable device mapping + * @debug: debug level + * @afid: authentication handle + * @cache: cache mode of type &p9_cache_bits + * @ndentry_timeout: Negative dentry lookup cache retention time in ms + * @cachetag: the tag of the cache associated with this session + * @fscache: session cookie associated with FS-Cache + * @uname: string user name to mount hierarchy as + * @aname: mount specifier for remote hierarchy + * @maxdata: maximum data to be sent/recvd per protocol message + * @dfltuid: default numeric userid to mount hierarchy as + * @dfltgid: default numeric groupid to mount hierarchy as + * @uid: if %V9FS_ACCESS_SINGLE, the numeric uid which mounted the hierarchy + * @clnt: reference to 9P network client instantiated for this session + * @slist: reference to list of registered 9p sessions + * @ndentry_timeout_ms: Negative dentry caching retention time + * + * This structure holds state for each session instance established during + * a sys_mount() . + * + * Bugs: there seems to be a lot of state which could be condensed and/or + * removed. + */ + +struct v9fs_session_info { + /* options */ + unsigned int flags; + unsigned char nodev; + unsigned short debug; + unsigned int afid; + unsigned int cache; + unsigned int ndentry_timeout_ms; +#ifdef CONFIG_9P_FSCACHE + char *cachetag; + struct fscache_volume *fscache; +#endif + + char *uname; /* user name to mount as */ + char *aname; /* name of remote hierarchy being mounted */ + unsigned int maxdata; /* max data for client interface */ + kuid_t dfltuid; /* default uid/muid for legacy support */ + kgid_t dfltgid; /* default gid for legacy support */ + kuid_t uid; /* if ACCESS_SINGLE, the uid that has access */ + struct p9_client *clnt; /* 9p client */ + struct list_head slist; /* list of sessions registered with v9fs */ + struct rw_semaphore rename_sem; + long session_lock_timeout; /* retry interval for blocking locks */ +}; + +#define NDENTRY_TIMEOUT_NEVER (-1U) + +/* cache_validity flags */ +#define V9FS_INO_INVALID_ATTR 0x01 + +struct v9fs_inode { + struct netfs_inode netfs; /* Netfslib context and vfs inode */ + struct p9_qid qid; + unsigned int cache_validity; + struct mutex v_mutex; +}; + +static inline struct v9fs_inode *V9FS_I(const struct inode *inode) +{ + return container_of(inode, struct v9fs_inode, netfs.inode); +} + +static inline struct fscache_cookie *v9fs_inode_cookie(struct v9fs_inode *v9inode) +{ +#ifdef CONFIG_9P_FSCACHE + return netfs_i_cookie(&v9inode->netfs); +#else + return NULL; +#endif +} + +static inline struct fscache_volume *v9fs_session_cache(struct v9fs_session_info *v9ses) +{ +#ifdef CONFIG_9P_FSCACHE + return v9ses->fscache; +#else + return NULL; +#endif +} + +extern const struct fs_parameter_spec v9fs_param_spec[]; + +extern int v9fs_parse_param(struct fs_context *fc, struct fs_parameter *param); +extern int v9fs_show_options(struct seq_file *m, struct dentry *root); + +struct p9_fid *v9fs_session_init(struct v9fs_session_info *v9ses, + struct fs_context *fc); +extern void v9fs_session_close(struct v9fs_session_info *v9ses); +extern void v9fs_session_cancel(struct v9fs_session_info *v9ses); +extern void v9fs_session_begin_cancel(struct v9fs_session_info *v9ses); +extern struct dentry *v9fs_vfs_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags); +extern int v9fs_vfs_unlink(struct inode *i, struct dentry *d); +extern int v9fs_vfs_rmdir(struct inode *i, struct dentry *d); +extern int v9fs_vfs_rename(struct mnt_idmap *idmap, + struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry, + unsigned int flags); +extern struct inode *v9fs_inode_from_fid(struct v9fs_session_info *v9ses, + struct p9_fid *fid, + struct super_block *sb, int new); +extern const struct inode_operations v9fs_dir_inode_operations_dotl; +extern const struct inode_operations v9fs_file_inode_operations_dotl; +extern const struct inode_operations v9fs_symlink_inode_operations_dotl; +extern const struct netfs_request_ops v9fs_req_ops; +extern struct inode *v9fs_inode_from_fid_dotl(struct v9fs_session_info *v9ses, + struct p9_fid *fid, + struct super_block *sb, int new); + +/* other default globals */ +#define V9FS_PORT 564 +#define V9FS_DEFUSER "nobody" +#define V9FS_DEFANAME "" +#define V9FS_DEFUID KUIDT_INIT(-2) +#define V9FS_DEFGID KGIDT_INIT(-2) + +static inline struct v9fs_session_info *v9fs_inode2v9ses(struct inode *inode) +{ + return inode->i_sb->s_fs_info; +} + +static inline struct v9fs_session_info *v9fs_dentry2v9ses(const struct dentry *dentry) +{ + return dentry->d_sb->s_fs_info; +} + +static inline int v9fs_proto_dotu(struct v9fs_session_info *v9ses) +{ + return v9ses->flags & V9FS_PROTO_2000U; +} + +static inline int v9fs_proto_dotl(struct v9fs_session_info *v9ses) +{ + return v9ses->flags & V9FS_PROTO_2000L; +} + +/** + * v9fs_get_inode_from_fid - Helper routine to populate an inode by + * issuing a attribute request + * @v9ses: session information + * @fid: fid to issue attribute request for + * @sb: superblock on which to create inode + * + */ +static inline struct inode * +v9fs_get_inode_from_fid(struct v9fs_session_info *v9ses, struct p9_fid *fid, + struct super_block *sb) +{ + if (v9fs_proto_dotl(v9ses)) + return v9fs_inode_from_fid_dotl(v9ses, fid, sb, 0); + else + return v9fs_inode_from_fid(v9ses, fid, sb, 0); +} + +/** + * v9fs_get_new_inode_from_fid - Helper routine to populate an inode by + * issuing a attribute request + * @v9ses: session information + * @fid: fid to issue attribute request for + * @sb: superblock on which to create inode + * + */ +static inline struct inode * +v9fs_get_new_inode_from_fid(struct v9fs_session_info *v9ses, struct p9_fid *fid, + struct super_block *sb) +{ + if (v9fs_proto_dotl(v9ses)) + return v9fs_inode_from_fid_dotl(v9ses, fid, sb, 1); + else + return v9fs_inode_from_fid(v9ses, fid, sb, 1); +} + +#endif diff --git a/fs/9p/v9fs_vfs.h b/fs/9p/v9fs_vfs.h new file mode 100644 index 000000000..1856d91f8 --- /dev/null +++ b/fs/9p/v9fs_vfs.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * V9FS VFS extensions. + * + * Copyright (C) 2004 by Eric Van Hensbergen + * Copyright (C) 2002 by Ron Minnich + */ +#ifndef FS_9P_V9FS_VFS_H +#define FS_9P_V9FS_VFS_H + +/* plan9 semantics are that created files are implicitly opened. + * But linux semantics are that you call create, then open. + * the plan9 approach is superior as it provides an atomic + * open. + * we track the create fid here. When the file is opened, if fidopen is + * non-zero, we use the fid and can skip some steps. + * there may be a better way to do this, but I don't know it. + * one BAD way is to clunk the fid on create, then open it again: + * you lose the atomicity of file open + */ + +/* special case: + * unlink calls remove, which is an implicit clunk. So we have to track + * that kind of thing so that we don't try to clunk a dead fid. + */ +#define P9_LOCK_TIMEOUT (30*HZ) + +/* flags for v9fs_stat2inode() & v9fs_stat2inode_dotl() */ +#define V9FS_STAT2INODE_KEEP_ISIZE 1 + +/** + * struct v9fs_dentry - v9fs specific dentry data + * @head: List of fid associated with this dentry + * @expire_time: Lookup cache expiration time for negative dentries + * @rcu: used by kfree_rcu to schedule clean up job + */ +struct v9fs_dentry { + struct hlist_head head; + u64 expire_time; + struct rcu_head rcu; +}; +#define to_v9fs_dentry(d) ((struct v9fs_dentry *)((d)->d_fsdata)) + +extern struct file_system_type v9fs_fs_type; +extern const struct address_space_operations v9fs_addr_operations; +extern const struct file_operations v9fs_file_operations; +extern const struct file_operations v9fs_file_operations_dotl; +extern const struct file_operations v9fs_dir_operations; +extern const struct file_operations v9fs_dir_operations_dotl; +extern const struct dentry_operations v9fs_dentry_operations; +extern void v9fs_ndentry_refresh_timeout(struct dentry *dentry); +extern void v9fs_dentry_fid_remove(struct dentry *dentry); +extern const struct dentry_operations v9fs_cached_dentry_operations; +extern struct kmem_cache *v9fs_inode_cache; + +struct inode *v9fs_alloc_inode(struct super_block *sb); +void v9fs_free_inode(struct inode *inode); +void v9fs_set_netfs_context(struct inode *inode); +int v9fs_init_inode(struct v9fs_session_info *v9ses, + struct inode *inode, umode_t mode, dev_t rdev); +void v9fs_evict_inode(struct inode *inode); +#if (BITS_PER_LONG == 32) +#define QID2INO(q) ((ino_t) (((q)->path+2) ^ (((q)->path) >> 32))) +#else +#define QID2INO(q) ((ino_t) ((q)->path+2)) +#endif + +void v9fs_stat2inode(struct p9_wstat *stat, struct inode *inode, + struct super_block *sb, unsigned int flags); +void v9fs_stat2inode_dotl(struct p9_stat_dotl *stat, struct inode *inode, + unsigned int flags); +int v9fs_dir_release(struct inode *inode, struct file *filp); +int v9fs_file_open(struct inode *inode, struct file *file); +int v9fs_uflags2omode(int uflags, int extended); + +void v9fs_blank_wstat(struct p9_wstat *wstat); +int v9fs_vfs_setattr_dotl(struct mnt_idmap *idmap, + struct dentry *dentry, struct iattr *iattr); +int v9fs_file_fsync_dotl(struct file *filp, loff_t start, loff_t end, + int datasync); +int v9fs_refresh_inode(struct p9_fid *fid, struct inode *inode); +int v9fs_refresh_inode_dotl(struct p9_fid *fid, struct inode *inode); +static inline void v9fs_invalidate_inode_attr(struct inode *inode) +{ + struct v9fs_inode *v9inode; + + v9inode = V9FS_I(inode); + v9inode->cache_validity |= V9FS_INO_INVALID_ATTR; +} + +int v9fs_open_to_dotl_flags(int flags); + +#endif diff --git a/fs/9p/vfs_addr.c b/fs/9p/vfs_addr.c new file mode 100644 index 000000000..1ac0b3dcc --- /dev/null +++ b/fs/9p/vfs_addr.c @@ -0,0 +1,201 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file contians vfs address (mmap) ops for 9P2000. + * + * Copyright (C) 2005 by Eric Van Hensbergen + * Copyright (C) 2002 by Ron Minnich + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "v9fs.h" +#include "v9fs_vfs.h" +#include "cache.h" +#include "fid.h" + +/* + * Writeback calls this when it finds a folio that needs uploading. This isn't + * called if writeback only has copy-to-cache to deal with. + */ +static void v9fs_begin_writeback(struct netfs_io_request *wreq) +{ + struct p9_fid *fid; + + fid = v9fs_fid_find_inode(wreq->inode, true, INVALID_UID, true); + if (!fid) { + WARN_ONCE(1, "folio expected an open fid inode->i_ino=%llx\n", + wreq->inode->i_ino); + return; + } + + wreq->wsize = fid->clnt->msize - P9_IOHDRSZ; + if (fid->iounit) + wreq->wsize = min(wreq->wsize, fid->iounit); + wreq->netfs_priv = fid; + wreq->io_streams[0].avail = true; +} + +/* + * Issue a subrequest to write to the server. + */ +static void v9fs_issue_write(struct netfs_io_subrequest *subreq) +{ + struct p9_fid *fid = subreq->rreq->netfs_priv; + int err, len; + + len = p9_client_write(fid, subreq->start, &subreq->io_iter, &err); + if (len > 0) + __set_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); + netfs_write_subrequest_terminated(subreq, len ?: err); +} + +/** + * v9fs_issue_read - Issue a read from 9P + * @subreq: The read to make + */ +static void v9fs_issue_read(struct netfs_io_subrequest *subreq) +{ + struct netfs_io_request *rreq = subreq->rreq; + struct p9_fid *fid = rreq->netfs_priv; + char *target; + unsigned long long pos = subreq->start + subreq->transferred; + int total = 0, err, len, n; + + if (S_ISLNK(rreq->inode->i_mode)) { + /* p9_client_readlink() must not be called for legacy protocols + * 9p2000 or 9p2000.u. + */ + BUG_ON(!p9_is_proto_dotl(fid->clnt)); + if (WARN_ON_ONCE(pos)) { + /* reading a link at a non null offset should + * not happen + */ + err = -EIO; + goto fill_subreq; + } + err = p9_client_readlink(fid, &target); + if (err != 0) + goto fill_subreq; + len = strlen(target); + n = copy_to_iter(target, len, &subreq->io_iter); + kfree(target); + total = n; + } else { + total = p9_client_read(fid, pos, &subreq->io_iter, &err); + } + +fill_subreq: + /* if we just extended the file size, any portion not in + * cache won't be on server and is zeroes */ + if (subreq->rreq->origin != NETFS_UNBUFFERED_READ && + subreq->rreq->origin != NETFS_DIO_READ) + __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags); + if (pos + total >= i_size_read(rreq->inode)) + __set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags); + if (!err && total) { + subreq->transferred += total; + __set_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); + } + + subreq->error = err; + netfs_read_subreq_terminated(subreq); +} + +/** + * v9fs_init_request - Initialise a request + * @rreq: The read request + * @file: The file being read from + */ +static int v9fs_init_request(struct netfs_io_request *rreq, struct file *file) +{ + struct p9_fid *fid; + struct dentry *dentry; + bool writing = (rreq->origin == NETFS_READ_FOR_WRITE || + rreq->origin == NETFS_WRITETHROUGH || + rreq->origin == NETFS_UNBUFFERED_WRITE || + rreq->origin == NETFS_DIO_WRITE); + + if (rreq->origin == NETFS_WRITEBACK) + return 0; /* We don't get the write handle until we find we + * have actually dirty data and not just + * copy-to-cache data. + */ + + if (file) { + fid = file->private_data; + if (!fid) + goto no_fid; + p9_fid_get(fid); + } else if (S_ISLNK(rreq->inode->i_mode)) { + dentry = d_find_any_alias(rreq->inode); + if (!dentry) + goto no_fid; + fid = v9fs_fid_lookup(dentry); + dput(dentry); + if (IS_ERR(fid)) + goto no_fid; + } else { + fid = v9fs_fid_find_inode(rreq->inode, writing, INVALID_UID, true); + if (!fid) + goto no_fid; + } + + rreq->wsize = fid->clnt->msize - P9_IOHDRSZ; + if (fid->iounit) + rreq->wsize = min(rreq->wsize, fid->iounit); + + /* we might need to read from a fid that was opened write-only + * for read-modify-write of page cache, use the writeback fid + * for that */ + WARN_ON(rreq->origin == NETFS_READ_FOR_WRITE && !(fid->mode & P9_ORDWR)); + rreq->netfs_priv = fid; + return 0; + +no_fid: + WARN_ONCE(1, "folio expected an open fid inode->i_ino=%llx\n", + rreq->inode->i_ino); + return -EINVAL; +} + +/** + * v9fs_free_request - Cleanup request initialized by v9fs_init_rreq + * @rreq: The I/O request to clean up + */ +static void v9fs_free_request(struct netfs_io_request *rreq) +{ + struct p9_fid *fid = rreq->netfs_priv; + + p9_fid_put(fid); +} + +const struct netfs_request_ops v9fs_req_ops = { + .init_request = v9fs_init_request, + .free_request = v9fs_free_request, + .issue_read = v9fs_issue_read, + .begin_writeback = v9fs_begin_writeback, + .issue_write = v9fs_issue_write, +}; + +const struct address_space_operations v9fs_addr_operations = { + .read_folio = netfs_read_folio, + .readahead = netfs_readahead, + .dirty_folio = netfs_dirty_folio, + .release_folio = netfs_release_folio, + .invalidate_folio = netfs_invalidate_folio, + .direct_IO = noop_direct_IO, + .writepages = netfs_writepages, + .migrate_folio = filemap_migrate_folio, +}; diff --git a/fs/9p/vfs_dentry.c b/fs/9p/vfs_dentry.c new file mode 100644 index 000000000..e549e2226 --- /dev/null +++ b/fs/9p/vfs_dentry.c @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file contians vfs dentry ops for the 9P2000 protocol. + * + * Copyright (C) 2004 by Eric Van Hensbergen + * Copyright (C) 2002 by Ron Minnich + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "v9fs.h" +#include "v9fs_vfs.h" +#include "fid.h" + +/** + * v9fs_ndentry_is_expired - Check if negative dentry lookup has expired + * + * This should be called to know if a negative dentry should be removed from + * cache. + * + * @dentry: dentry in question + * + */ +static bool v9fs_ndentry_is_expired(struct dentry const *dentry) +{ + struct v9fs_session_info *v9ses = v9fs_dentry2v9ses(dentry); + struct v9fs_dentry *v9fs_dentry = to_v9fs_dentry(dentry); + + if (v9ses->ndentry_timeout_ms == NDENTRY_TIMEOUT_NEVER) + return false; + + return time_before_eq64(v9fs_dentry->expire_time, get_jiffies_64()); +} + +/** + * v9fs_ndentry_refresh_timeout - Refresh negative dentry lookup cache timeout + * + * This should be called when a look up yields a negative entry. + * + * @dentry: dentry in question + * + */ +void v9fs_ndentry_refresh_timeout(struct dentry *dentry) +{ + struct v9fs_session_info *v9ses = v9fs_dentry2v9ses(dentry); + struct v9fs_dentry *v9fs_dentry = to_v9fs_dentry(dentry); + + if (v9ses->ndentry_timeout_ms == NDENTRY_TIMEOUT_NEVER) + return; + + v9fs_dentry->expire_time = get_jiffies_64() + + msecs_to_jiffies(v9ses->ndentry_timeout_ms); +} + +/** + * v9fs_cached_dentry_delete - called when dentry refcount equals 0 + * @dentry: dentry in question + * + */ +static int v9fs_cached_dentry_delete(const struct dentry *dentry) +{ + p9_debug(P9_DEBUG_VFS, " dentry: %pd (%p)\n", + dentry, dentry); + + if (!d_really_is_negative(dentry)) + return 0; + + return v9fs_ndentry_is_expired(dentry); +} + +static void __v9fs_dentry_fid_remove(struct dentry *dentry) +{ + struct v9fs_dentry *v9fs_dentry = to_v9fs_dentry(dentry); + struct hlist_node *p, *n; + struct hlist_head head; + + p9_debug(P9_DEBUG_VFS, " dentry: %pd (%p)\n", + dentry, dentry); + + spin_lock(&dentry->d_lock); + hlist_move_list(&v9fs_dentry->head, &head); + spin_unlock(&dentry->d_lock); + + hlist_for_each_safe(p, n, &head) + p9_fid_put(hlist_entry(p, struct p9_fid, dlist)); +} + +/** + * v9fs_dentry_fid_remove - Release all dentry's fids + * @dentry: dentry in question + * + */ +void v9fs_dentry_fid_remove(struct dentry *dentry) +{ + __v9fs_dentry_fid_remove(dentry); +} + +/** + * v9fs_dentry_init - Initialize v9fs dentry data + * @dentry: dentry in question + * + */ +static int v9fs_dentry_init(struct dentry *dentry) +{ + struct v9fs_dentry *v9fs_dentry = kzalloc(sizeof(*v9fs_dentry), + GFP_KERNEL); + + if (!v9fs_dentry) + return -ENOMEM; + + INIT_HLIST_HEAD(&v9fs_dentry->head); + dentry->d_fsdata = (void *)v9fs_dentry; + return 0; +} + +/** + * v9fs_dentry_release - called when dentry is going to be freed + * @dentry: dentry that is being released + * + */ +static void v9fs_dentry_release(struct dentry *dentry) +{ + struct v9fs_dentry *v9fs_dentry = to_v9fs_dentry(dentry); + + __v9fs_dentry_fid_remove(dentry); + kfree_rcu(v9fs_dentry, rcu); +} + +static int __v9fs_lookup_revalidate(struct dentry *dentry, unsigned int flags) +{ + struct p9_fid *fid; + struct inode *inode; + struct v9fs_inode *v9inode; + + if (flags & LOOKUP_RCU) + return -ECHILD; + + inode = d_inode(dentry); + if (!inode) + return !v9fs_ndentry_is_expired(dentry); + + v9inode = V9FS_I(inode); + if (v9inode->cache_validity & V9FS_INO_INVALID_ATTR) { + int retval; + struct v9fs_session_info *v9ses; + + fid = v9fs_fid_lookup(dentry); + if (IS_ERR(fid)) { + p9_debug( + P9_DEBUG_VFS, + "v9fs_fid_lookup: dentry = %pd (%p), got error %pe\n", + dentry, dentry, fid); + return PTR_ERR(fid); + } + + v9ses = v9fs_inode2v9ses(inode); + if (v9fs_proto_dotl(v9ses)) + retval = v9fs_refresh_inode_dotl(fid, inode); + else + retval = v9fs_refresh_inode(fid, inode); + p9_fid_put(fid); + + if (retval == -ENOENT) { + p9_debug(P9_DEBUG_VFS, "dentry: %pd (%p) invalidated due to ENOENT\n", + dentry, dentry); + return 0; + } + if (v9inode->cache_validity & V9FS_INO_INVALID_ATTR) { + p9_debug(P9_DEBUG_VFS, "dentry: %pd (%p) invalidated due to type change\n", + dentry, dentry); + return 0; + } + if (retval < 0) { + p9_debug(P9_DEBUG_VFS, + "refresh inode: dentry = %pd (%p), got error %pe\n", + dentry, dentry, ERR_PTR(retval)); + return retval; + } + } + p9_debug(P9_DEBUG_VFS, "dentry: %pd (%p) is valid\n", dentry, dentry); + return 1; +} + +static int v9fs_lookup_revalidate(struct inode *dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + return __v9fs_lookup_revalidate(dentry, flags); +} + +static bool v9fs_dentry_unalias_trylock(const struct dentry *dentry) +{ + struct v9fs_session_info *v9ses = v9fs_dentry2v9ses(dentry); + return down_write_trylock(&v9ses->rename_sem); +} + +static void v9fs_dentry_unalias_unlock(const struct dentry *dentry) +{ + struct v9fs_session_info *v9ses = v9fs_dentry2v9ses(dentry); + up_write(&v9ses->rename_sem); +} + +const struct dentry_operations v9fs_cached_dentry_operations = { + .d_revalidate = v9fs_lookup_revalidate, + .d_weak_revalidate = __v9fs_lookup_revalidate, + .d_delete = v9fs_cached_dentry_delete, + .d_init = v9fs_dentry_init, + .d_release = v9fs_dentry_release, + .d_unalias_trylock = v9fs_dentry_unalias_trylock, + .d_unalias_unlock = v9fs_dentry_unalias_unlock, +}; + +const struct dentry_operations v9fs_dentry_operations = { + .d_init = v9fs_dentry_init, + .d_release = v9fs_dentry_release, + .d_unalias_trylock = v9fs_dentry_unalias_trylock, + .d_unalias_unlock = v9fs_dentry_unalias_unlock, +}; diff --git a/fs/9p/vfs_dir.c b/fs/9p/vfs_dir.c new file mode 100644 index 000000000..323f85352 --- /dev/null +++ b/fs/9p/vfs_dir.c @@ -0,0 +1,266 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file contains vfs directory ops for the 9P2000 protocol. + * + * Copyright (C) 2004 by Eric Van Hensbergen + * Copyright (C) 2002 by Ron Minnich + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "v9fs.h" +#include "v9fs_vfs.h" +#include "fid.h" + +/** + * struct p9_rdir - readdir accounting + * @head: start offset of current dirread buffer + * @tail: end offset of current dirread buffer + * @offset: file position the data at @head corresponds to + * @buf: dirread buffer + * + * private structure for keeping track of readdir + * allocated on demand + */ + +struct p9_rdir { + int head; + int tail; + loff_t offset; + uint8_t buf[]; +}; + +/** + * dt_type - return file type + * @mistat: mistat structure + * + */ + +static inline int dt_type(struct p9_wstat *mistat) +{ + unsigned long perm = mistat->mode; + int rettype = DT_REG; + + if (perm & P9_DMDIR) + rettype = DT_DIR; + if (perm & P9_DMSYMLINK) + rettype = DT_LNK; + + return rettype; +} + +/** + * v9fs_alloc_rdir_buf - Allocate buffer used for read and readdir + * @filp: opened file structure + * @buflen: Length in bytes of buffer to allocate + * + */ + +static struct p9_rdir *v9fs_alloc_rdir_buf(struct file *filp, int buflen) +{ + struct p9_fid *fid = filp->private_data; + + if (!fid->rdir) + fid->rdir = kvzalloc(sizeof(struct p9_rdir) + buflen, GFP_KERNEL); + return fid->rdir; +} + +/** + * v9fs_dir_readdir - iterate through a directory + * @file: opened file structure + * @ctx: actor we feed the entries to + * + */ + +static int v9fs_dir_readdir(struct file *file, struct dir_context *ctx) +{ + bool over; + struct p9_wstat st; + int err = 0; + struct p9_fid *fid; + int buflen; + struct p9_rdir *rdir; + struct kvec kvec; + + p9_debug(P9_DEBUG_VFS, "name %pD\n", file); + fid = file->private_data; + + buflen = fid->clnt->msize - P9_IOHDRSZ; + + rdir = v9fs_alloc_rdir_buf(file, buflen); + if (!rdir) + return -ENOMEM; + kvec.iov_base = rdir->buf; + kvec.iov_len = buflen; + + if (rdir->head < rdir->tail && rdir->offset != ctx->pos) + rdir->head = rdir->tail = 0; + + while (1) { + if (rdir->tail == rdir->head) { + struct iov_iter to; + int n; + + iov_iter_kvec(&to, ITER_DEST, &kvec, 1, buflen); + n = p9_client_read(file->private_data, ctx->pos, &to, + &err); + if (err) + return err; + if (n == 0) + return 0; + + rdir->head = 0; + rdir->tail = n; + rdir->offset = ctx->pos; + } + while (rdir->head < rdir->tail) { + err = p9stat_read(fid->clnt, rdir->buf + rdir->head, + rdir->tail - rdir->head, &st); + if (err <= 0) { + p9_debug(P9_DEBUG_VFS, "returned %d\n", err); + return -EIO; + } + + over = !dir_emit(ctx, st.name, strlen(st.name), + QID2INO(&st.qid), dt_type(&st)); + p9stat_free(&st); + if (over) + return 0; + + rdir->head += err; + ctx->pos += err; + rdir->offset = ctx->pos; + } + } +} + +/** + * v9fs_dir_readdir_dotl - iterate through a directory + * @file: opened file structure + * @ctx: actor we feed the entries to + * + */ +static int v9fs_dir_readdir_dotl(struct file *file, struct dir_context *ctx) +{ + int err = 0; + struct p9_fid *fid; + int buflen; + struct p9_rdir *rdir; + struct p9_dirent curdirent; + + p9_debug(P9_DEBUG_VFS, "name %pD\n", file); + fid = file->private_data; + + buflen = fid->clnt->msize - P9_READDIRHDRSZ; + + rdir = v9fs_alloc_rdir_buf(file, buflen); + if (!rdir) + return -ENOMEM; + + if (rdir->head < rdir->tail && rdir->offset != ctx->pos) + rdir->head = rdir->tail = 0; + + while (1) { + if (rdir->tail == rdir->head) { + err = p9_client_readdir(fid, rdir->buf, buflen, + ctx->pos); + if (err <= 0) + return err; + + rdir->head = 0; + rdir->tail = err; + rdir->offset = ctx->pos; + } + + while (rdir->head < rdir->tail) { + + err = p9dirent_read(fid->clnt, rdir->buf + rdir->head, + rdir->tail - rdir->head, + &curdirent); + if (err < 0) { + p9_debug(P9_DEBUG_VFS, "returned %d\n", err); + return -EIO; + } + + if (!dir_emit(ctx, curdirent.d_name, + strlen(curdirent.d_name), + QID2INO(&curdirent.qid), + curdirent.d_type)) + return 0; + + ctx->pos = curdirent.d_off; + rdir->head += err; + rdir->offset = ctx->pos; + } + } +} + + +/** + * v9fs_dir_release - close a directory or a file + * @inode: inode of the directory or file + * @filp: file pointer to a directory or file + * + */ + +int v9fs_dir_release(struct inode *inode, struct file *filp) +{ + struct v9fs_inode *v9inode = V9FS_I(inode); + struct p9_fid *fid; + __le32 version; + loff_t i_size; + int retval = 0, put_err; + + fid = filp->private_data; + p9_debug(P9_DEBUG_VFS, "inode: %p filp: %p fid: %d\n", + inode, filp, fid ? fid->fid : -1); + + if (fid) { + if ((S_ISREG(inode->i_mode)) && (filp->f_mode & FMODE_WRITE)) + retval = filemap_fdatawrite(inode->i_mapping); + + spin_lock(&inode->i_lock); + hlist_del(&fid->ilist); + spin_unlock(&inode->i_lock); + put_err = p9_fid_put(fid); + retval = retval < 0 ? retval : put_err; + } + + if ((filp->f_mode & FMODE_WRITE)) { + version = cpu_to_le32(v9inode->qid.version); + i_size = i_size_read(inode); + fscache_unuse_cookie(v9fs_inode_cookie(v9inode), + &version, &i_size); + } else { + fscache_unuse_cookie(v9fs_inode_cookie(v9inode), NULL, NULL); + } + return retval; +} + +const struct file_operations v9fs_dir_operations = { + .read = generic_read_dir, + .llseek = generic_file_llseek, + .iterate_shared = v9fs_dir_readdir, + .open = v9fs_file_open, + .release = v9fs_dir_release, +}; + +const struct file_operations v9fs_dir_operations_dotl = { + .read = generic_read_dir, + .llseek = generic_file_llseek, + .iterate_shared = v9fs_dir_readdir_dotl, + .open = v9fs_file_open, + .release = v9fs_dir_release, + .fsync = v9fs_file_fsync_dotl, +}; diff --git a/fs/9p/vfs_file.c b/fs/9p/vfs_file.c new file mode 100644 index 000000000..cddfb4b7c --- /dev/null +++ b/fs/9p/vfs_file.c @@ -0,0 +1,534 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file contians vfs file ops for 9P2000. + * + * Copyright (C) 2004 by Eric Van Hensbergen + * Copyright (C) 2002 by Ron Minnich + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "v9fs.h" +#include "v9fs_vfs.h" +#include "fid.h" +#include "cache.h" + +static const struct vm_operations_struct v9fs_mmap_file_vm_ops; + +/** + * v9fs_file_open - open a file (or directory) + * @inode: inode to be opened + * @file: file being opened + * + */ + +int v9fs_file_open(struct inode *inode, struct file *file) +{ + int err; + struct v9fs_session_info *v9ses; + struct p9_fid *fid; + int omode; + int o_append; + + p9_debug(P9_DEBUG_VFS, "inode: %p file: %p\n", inode, file); + v9ses = v9fs_inode2v9ses(inode); + if (v9fs_proto_dotl(v9ses)) { + omode = v9fs_open_to_dotl_flags(file->f_flags); + o_append = P9_DOTL_APPEND; + } else { + omode = v9fs_uflags2omode(file->f_flags, + v9fs_proto_dotu(v9ses)); + o_append = P9_OAPPEND; + } + fid = file->private_data; + if (!fid) { + fid = v9fs_fid_clone(file_dentry(file)); + if (IS_ERR(fid)) + return PTR_ERR(fid); + + if ((v9ses->cache & CACHE_WRITEBACK) && (omode & P9_OWRITE)) { + int writeback_omode = (omode & ~(P9_OWRITE | o_append)) | P9_ORDWR; + + p9_debug(P9_DEBUG_CACHE, "write-only file with writeback enabled, try opening O_RDWR\n"); + + err = p9_client_open(fid, writeback_omode); + if (err < 0) { + p9_debug(P9_DEBUG_CACHE, "could not open O_RDWR, disabling caches\n"); + err = p9_client_open(fid, omode); + fid->mode |= P9L_DIRECT; + } + } else { + err = p9_client_open(fid, omode); + } + if (err < 0) { + p9_fid_put(fid); + return err; + } + if ((file->f_flags & O_APPEND) && + (!v9fs_proto_dotu(v9ses) && !v9fs_proto_dotl(v9ses))) + generic_file_llseek(file, 0, SEEK_END); + + file->private_data = fid; + } + +#ifdef CONFIG_9P_FSCACHE + if (v9ses->cache & CACHE_FSCACHE) + fscache_use_cookie(v9fs_inode_cookie(V9FS_I(inode)), + file->f_mode & FMODE_WRITE); +#endif + v9fs_fid_add_modes(fid, v9ses->flags, v9ses->cache, file->f_flags); + v9fs_open_fid_add(inode, &fid); + return 0; +} + +/** + * v9fs_file_lock - lock a file (or directory) + * @filp: file to be locked + * @cmd: lock command + * @fl: file lock structure + * + * Bugs: this looks like a local only lock, we should extend into 9P + * by using open exclusive + */ + +static int v9fs_file_lock(struct file *filp, int cmd, struct file_lock *fl) +{ + struct inode *inode = file_inode(filp); + + p9_debug(P9_DEBUG_VFS, "filp: %p lock: %p\n", filp, fl); + + if ((IS_SETLK(cmd) || IS_SETLKW(cmd)) && fl->c.flc_type != F_UNLCK) { + filemap_write_and_wait(inode->i_mapping); + invalidate_mapping_pages(&inode->i_data, 0, -1); + } + + return 0; +} + +static int v9fs_file_do_lock(struct file *filp, int cmd, struct file_lock *fl) +{ + struct p9_flock flock; + struct p9_fid *fid; + uint8_t status = P9_LOCK_ERROR; + int res = 0; + struct v9fs_session_info *v9ses; + + fid = filp->private_data; + BUG_ON(fid == NULL); + + BUG_ON((fl->c.flc_flags & FL_POSIX) != FL_POSIX); + + res = locks_lock_file_wait(filp, fl); + if (res < 0) + goto out; + + /* convert posix lock to p9 tlock args */ + memset(&flock, 0, sizeof(flock)); + /* map the lock type */ + switch (fl->c.flc_type) { + case F_RDLCK: + flock.type = P9_LOCK_TYPE_RDLCK; + break; + case F_WRLCK: + flock.type = P9_LOCK_TYPE_WRLCK; + break; + case F_UNLCK: + flock.type = P9_LOCK_TYPE_UNLCK; + break; + } + flock.start = fl->fl_start; + if (fl->fl_end == OFFSET_MAX) + flock.length = 0; + else + flock.length = fl->fl_end - fl->fl_start + 1; + flock.proc_id = fl->c.flc_pid; + flock.client_id = fid->clnt->name; + if (IS_SETLKW(cmd)) + flock.flags = P9_LOCK_FLAGS_BLOCK; + + v9ses = v9fs_inode2v9ses(file_inode(filp)); + + /* + * if its a blocked request and we get P9_LOCK_BLOCKED as the status + * for lock request, keep on trying + */ + for (;;) { + res = p9_client_lock_dotl(fid, &flock, &status); + if (res < 0) + goto out_unlock; + + if (status != P9_LOCK_BLOCKED) + break; + if (status == P9_LOCK_BLOCKED && !IS_SETLKW(cmd)) + break; + if (schedule_timeout_interruptible(v9ses->session_lock_timeout) + != 0) + break; + /* + * p9_client_lock_dotl overwrites flock.client_id with the + * server message, free and reuse the client name + */ + if (flock.client_id != fid->clnt->name) { + kfree(flock.client_id); + flock.client_id = fid->clnt->name; + } + } + + /* map 9p status to VFS status */ + switch (status) { + case P9_LOCK_SUCCESS: + res = 0; + break; + case P9_LOCK_BLOCKED: + res = -EAGAIN; + break; + default: + p9_debug(P9_DEBUG_ERROR, "unknown lock status code: %d\n", status); + fallthrough; + case P9_LOCK_ERROR: + case P9_LOCK_GRACE: + res = -ENOLCK; + break; + } + +out_unlock: + /* + * incase server returned error for lock request, revert + * it locally + */ + if (res < 0 && fl->c.flc_type != F_UNLCK) { + unsigned char type = fl->c.flc_type; + + fl->c.flc_type = F_UNLCK; + /* Even if this fails we want to return the remote error */ + locks_lock_file_wait(filp, fl); + fl->c.flc_type = type; + } + if (flock.client_id != fid->clnt->name) + kfree(flock.client_id); +out: + return res; +} + +static int v9fs_file_getlock(struct file *filp, struct file_lock *fl) +{ + struct p9_getlock glock; + struct p9_fid *fid; + int res = 0; + + fid = filp->private_data; + BUG_ON(fid == NULL); + + posix_test_lock(filp, fl); + /* + * if we have a conflicting lock locally, no need to validate + * with server + */ + if (fl->c.flc_type != F_UNLCK) + return res; + + /* convert posix lock to p9 tgetlock args */ + memset(&glock, 0, sizeof(glock)); + glock.type = P9_LOCK_TYPE_UNLCK; + glock.start = fl->fl_start; + if (fl->fl_end == OFFSET_MAX) + glock.length = 0; + else + glock.length = fl->fl_end - fl->fl_start + 1; + glock.proc_id = fl->c.flc_pid; + glock.client_id = fid->clnt->name; + + res = p9_client_getlock_dotl(fid, &glock); + if (res < 0) + goto out; + /* map 9p lock type to os lock type */ + switch (glock.type) { + case P9_LOCK_TYPE_RDLCK: + fl->c.flc_type = F_RDLCK; + break; + case P9_LOCK_TYPE_WRLCK: + fl->c.flc_type = F_WRLCK; + break; + case P9_LOCK_TYPE_UNLCK: + fl->c.flc_type = F_UNLCK; + break; + } + if (glock.type != P9_LOCK_TYPE_UNLCK) { + fl->fl_start = glock.start; + if (glock.length == 0) + fl->fl_end = OFFSET_MAX; + else + fl->fl_end = glock.start + glock.length - 1; + fl->c.flc_pid = -glock.proc_id; + } +out: + if (glock.client_id != fid->clnt->name) + kfree(glock.client_id); + return res; +} + +/** + * v9fs_file_lock_dotl - lock a file (or directory) + * @filp: file to be locked + * @cmd: lock command + * @fl: file lock structure + * + */ + +static int v9fs_file_lock_dotl(struct file *filp, int cmd, struct file_lock *fl) +{ + struct inode *inode = file_inode(filp); + int ret = -ENOLCK; + + p9_debug(P9_DEBUG_VFS, "filp: %p cmd:%d lock: %p name: %pD\n", + filp, cmd, fl, filp); + + if ((IS_SETLK(cmd) || IS_SETLKW(cmd)) && fl->c.flc_type != F_UNLCK) { + filemap_write_and_wait(inode->i_mapping); + invalidate_mapping_pages(&inode->i_data, 0, -1); + } + + if (IS_SETLK(cmd) || IS_SETLKW(cmd)) + ret = v9fs_file_do_lock(filp, cmd, fl); + else if (IS_GETLK(cmd)) + ret = v9fs_file_getlock(filp, fl); + else + ret = -EINVAL; + return ret; +} + +/** + * v9fs_file_flock_dotl - lock a file + * @filp: file to be locked + * @cmd: lock command + * @fl: file lock structure + * + */ + +static int v9fs_file_flock_dotl(struct file *filp, int cmd, + struct file_lock *fl) +{ + struct inode *inode = file_inode(filp); + int ret = -ENOLCK; + + p9_debug(P9_DEBUG_VFS, "filp: %p cmd:%d lock: %p name: %pD\n", + filp, cmd, fl, filp); + + if (!(fl->c.flc_flags & FL_FLOCK)) + goto out_err; + + if ((IS_SETLK(cmd) || IS_SETLKW(cmd)) && fl->c.flc_type != F_UNLCK) { + filemap_write_and_wait(inode->i_mapping); + invalidate_mapping_pages(&inode->i_data, 0, -1); + } + /* Convert flock to posix lock */ + fl->c.flc_flags |= FL_POSIX; + fl->c.flc_flags ^= FL_FLOCK; + + if (IS_SETLK(cmd) | IS_SETLKW(cmd)) + ret = v9fs_file_do_lock(filp, cmd, fl); + else + ret = -EINVAL; +out_err: + return ret; +} + +/** + * v9fs_file_read_iter - read from a file + * @iocb: The operation parameters + * @to: The buffer to read into + * + */ +static ssize_t +v9fs_file_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct p9_fid *fid = iocb->ki_filp->private_data; + + p9_debug(P9_DEBUG_VFS, "fid %d count %zu offset %lld\n", + fid->fid, iov_iter_count(to), iocb->ki_pos); + + if (fid->mode & P9L_DIRECT) + return netfs_unbuffered_read_iter(iocb, to); + + p9_debug(P9_DEBUG_VFS, "(cached)\n"); + return netfs_file_read_iter(iocb, to); +} + +/* + * v9fs_file_splice_read - splice-read from a file + * @in: The 9p file to read from + * @ppos: Where to find/update the file position + * @pipe: The pipe to splice into + * @len: The maximum amount of data to splice + * @flags: SPLICE_F_* flags + */ +static ssize_t v9fs_file_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, + size_t len, unsigned int flags) +{ + struct p9_fid *fid = in->private_data; + + p9_debug(P9_DEBUG_VFS, "fid %d count %zu offset %lld\n", + fid->fid, len, *ppos); + + if (fid->mode & P9L_DIRECT) + return copy_splice_read(in, ppos, pipe, len, flags); + return filemap_splice_read(in, ppos, pipe, len, flags); +} + +/** + * v9fs_file_write_iter - write to a file + * @iocb: The operation parameters + * @from: The data to write + * + */ +static ssize_t +v9fs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct p9_fid *fid = file->private_data; + + p9_debug(P9_DEBUG_VFS, "fid %d\n", fid->fid); + + if (fid->mode & (P9L_DIRECT | P9L_NOWRITECACHE)) + return netfs_unbuffered_write_iter(iocb, from); + + p9_debug(P9_DEBUG_CACHE, "(cached)\n"); + return netfs_file_write_iter(iocb, from); +} + +static int v9fs_file_fsync(struct file *filp, loff_t start, loff_t end, + int datasync) +{ + struct p9_fid *fid; + struct inode *inode = filp->f_mapping->host; + struct p9_wstat wstat; + int retval; + + retval = file_write_and_wait_range(filp, start, end); + if (retval) + return retval; + + inode_lock(inode); + p9_debug(P9_DEBUG_VFS, "filp %p datasync %x\n", filp, datasync); + + fid = filp->private_data; + v9fs_blank_wstat(&wstat); + + retval = p9_client_wstat(fid, &wstat); + inode_unlock(inode); + + return retval; +} + +int v9fs_file_fsync_dotl(struct file *filp, loff_t start, loff_t end, + int datasync) +{ + struct p9_fid *fid; + struct inode *inode = filp->f_mapping->host; + int retval; + + retval = file_write_and_wait_range(filp, start, end); + if (retval) + return retval; + + inode_lock(inode); + p9_debug(P9_DEBUG_VFS, "filp %p datasync %x\n", filp, datasync); + + fid = filp->private_data; + + retval = p9_client_fsync(fid, datasync); + inode_unlock(inode); + + return retval; +} + +static int +v9fs_file_mmap_prepare(struct vm_area_desc *desc) +{ + int retval; + struct file *filp = desc->file; + struct inode *inode = file_inode(filp); + struct v9fs_session_info *v9ses = v9fs_inode2v9ses(inode); + + p9_debug(P9_DEBUG_MMAP, "filp :%p\n", filp); + + if (!(v9ses->cache & CACHE_WRITEBACK)) { + p9_debug(P9_DEBUG_CACHE, "(read-only mmap mode)"); + return generic_file_readonly_mmap_prepare(desc); + } + + retval = generic_file_mmap_prepare(desc); + if (!retval) + desc->vm_ops = &v9fs_mmap_file_vm_ops; + + return retval; +} + +static vm_fault_t +v9fs_vm_page_mkwrite(struct vm_fault *vmf) +{ + return netfs_page_mkwrite(vmf, NULL); +} + +static void v9fs_mmap_vm_close(struct vm_area_struct *vma) +{ + if (!(vma->vm_flags & VM_SHARED)) + return; + + p9_debug(P9_DEBUG_VFS, "9p VMA close, %p, flushing", vma); + + filemap_fdatawrite_range(file_inode(vma->vm_file)->i_mapping, + (loff_t)vma->vm_pgoff * PAGE_SIZE, + (loff_t)vma->vm_pgoff * PAGE_SIZE + + (vma->vm_end - vma->vm_start - 1)); +} + +static const struct vm_operations_struct v9fs_mmap_file_vm_ops = { + .close = v9fs_mmap_vm_close, + .fault = filemap_fault, + .map_pages = filemap_map_pages, + .page_mkwrite = v9fs_vm_page_mkwrite, +}; + +const struct file_operations v9fs_file_operations = { + .llseek = generic_file_llseek, + .read_iter = v9fs_file_read_iter, + .write_iter = v9fs_file_write_iter, + .open = v9fs_file_open, + .release = v9fs_dir_release, + .lock = v9fs_file_lock, + .mmap_prepare = generic_file_readonly_mmap_prepare, + .splice_read = v9fs_file_splice_read, + .splice_write = iter_file_splice_write, + .fsync = v9fs_file_fsync, +}; + +const struct file_operations v9fs_file_operations_dotl = { + .llseek = generic_file_llseek, + .read_iter = v9fs_file_read_iter, + .write_iter = v9fs_file_write_iter, + .open = v9fs_file_open, + .release = v9fs_dir_release, + .lock = v9fs_file_lock_dotl, + .flock = v9fs_file_flock_dotl, + .mmap_prepare = v9fs_file_mmap_prepare, + .splice_read = v9fs_file_splice_read, + .splice_write = iter_file_splice_write, + .fsync = v9fs_file_fsync_dotl, +}; diff --git a/fs/9p/vfs_inode.c b/fs/9p/vfs_inode.c new file mode 100644 index 000000000..3829554ca --- /dev/null +++ b/fs/9p/vfs_inode.c @@ -0,0 +1,1402 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file contains vfs inode ops for the 9P2000 protocol. + * + * Copyright (C) 2004 by Eric Van Hensbergen + * Copyright (C) 2002 by Ron Minnich + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "v9fs.h" +#include "v9fs_vfs.h" +#include "fid.h" +#include "cache.h" +#include "xattr.h" +#include "acl.h" + +static const struct inode_operations v9fs_dir_inode_operations; +static const struct inode_operations v9fs_dir_inode_operations_dotu; +static const struct inode_operations v9fs_file_inode_operations; +static const struct inode_operations v9fs_symlink_inode_operations; + +/** + * unixmode2p9mode - convert unix mode bits to plan 9 + * @v9ses: v9fs session information + * @mode: mode to convert + * + */ + +static u32 unixmode2p9mode(struct v9fs_session_info *v9ses, umode_t mode) +{ + int res; + + res = mode & 0777; + if (S_ISDIR(mode)) + res |= P9_DMDIR; + if (v9fs_proto_dotu(v9ses)) { + if (v9ses->nodev == 0) { + if (S_ISSOCK(mode)) + res |= P9_DMSOCKET; + if (S_ISFIFO(mode)) + res |= P9_DMNAMEDPIPE; + if (S_ISBLK(mode)) + res |= P9_DMDEVICE; + if (S_ISCHR(mode)) + res |= P9_DMDEVICE; + } + + if ((mode & S_ISUID) == S_ISUID) + res |= P9_DMSETUID; + if ((mode & S_ISGID) == S_ISGID) + res |= P9_DMSETGID; + if ((mode & S_ISVTX) == S_ISVTX) + res |= P9_DMSETVTX; + } + return res; +} + +/** + * p9mode2perm- convert plan9 mode bits to unix permission bits + * @v9ses: v9fs session information + * @stat: p9_wstat from which mode need to be derived + * + */ +static int p9mode2perm(struct v9fs_session_info *v9ses, + struct p9_wstat *stat) +{ + int res; + int mode = stat->mode; + + res = mode & 0777; /* S_IRWXUGO */ + if (v9fs_proto_dotu(v9ses)) { + if ((mode & P9_DMSETUID) == P9_DMSETUID) + res |= S_ISUID; + + if ((mode & P9_DMSETGID) == P9_DMSETGID) + res |= S_ISGID; + + if ((mode & P9_DMSETVTX) == P9_DMSETVTX) + res |= S_ISVTX; + } + return res; +} + +/** + * p9mode2unixmode- convert plan9 mode bits to unix mode bits + * @v9ses: v9fs session information + * @stat: p9_wstat from which mode need to be derived + * @rdev: major number, minor number in case of device files. + * + */ +static umode_t p9mode2unixmode(struct v9fs_session_info *v9ses, + struct p9_wstat *stat, dev_t *rdev) +{ + int res, r; + u32 mode = stat->mode; + + *rdev = 0; + res = p9mode2perm(v9ses, stat); + + if ((mode & P9_DMDIR) == P9_DMDIR) + res |= S_IFDIR; + else if ((mode & P9_DMSYMLINK) && (v9fs_proto_dotu(v9ses))) + res |= S_IFLNK; + else if ((mode & P9_DMSOCKET) && (v9fs_proto_dotu(v9ses)) + && (v9ses->nodev == 0)) + res |= S_IFSOCK; + else if ((mode & P9_DMNAMEDPIPE) && (v9fs_proto_dotu(v9ses)) + && (v9ses->nodev == 0)) + res |= S_IFIFO; + else if ((mode & P9_DMDEVICE) && (v9fs_proto_dotu(v9ses)) + && (v9ses->nodev == 0)) { + char type = 0; + int major = -1, minor = -1; + + r = sscanf(stat->extension, "%c %i %i", &type, &major, &minor); + if (r != 3) { + p9_debug(P9_DEBUG_ERROR, + "invalid device string, umode will be bogus: %s\n", + stat->extension); + return res; + } + switch (type) { + case 'c': + res |= S_IFCHR; + break; + case 'b': + res |= S_IFBLK; + break; + default: + p9_debug(P9_DEBUG_ERROR, "Unknown special type %c %s\n", + type, stat->extension); + } + *rdev = MKDEV(major, minor); + } else + res |= S_IFREG; + + return res; +} + +/** + * v9fs_uflags2omode- convert posix open flags to plan 9 mode bits + * @uflags: flags to convert + * @extended: if .u extensions are active + */ + +int v9fs_uflags2omode(int uflags, int extended) +{ + int ret; + + switch (uflags&3) { + default: + case O_RDONLY: + ret = P9_OREAD; + break; + + case O_WRONLY: + ret = P9_OWRITE; + break; + + case O_RDWR: + ret = P9_ORDWR; + break; + } + + if (uflags & O_TRUNC) + ret |= P9_OTRUNC; + + if (extended) { + if (uflags & O_EXCL) + ret |= P9_OEXCL; + + if (uflags & O_APPEND) + ret |= P9_OAPPEND; + } + + return ret; +} + +/** + * v9fs_blank_wstat - helper function to setup a 9P stat structure + * @wstat: structure to initialize + * + */ + +void +v9fs_blank_wstat(struct p9_wstat *wstat) +{ + wstat->type = ~0; + wstat->dev = ~0; + wstat->qid.type = ~0; + wstat->qid.version = ~0; + *((long long *)&wstat->qid.path) = ~0; + wstat->mode = ~0; + wstat->atime = ~0; + wstat->mtime = ~0; + wstat->length = ~0; + wstat->name = NULL; + wstat->uid = NULL; + wstat->gid = NULL; + wstat->muid = NULL; + wstat->n_uid = INVALID_UID; + wstat->n_gid = INVALID_GID; + wstat->n_muid = INVALID_UID; + wstat->extension = NULL; +} + +/** + * v9fs_alloc_inode - helper function to allocate an inode + * @sb: The superblock to allocate the inode from + */ +struct inode *v9fs_alloc_inode(struct super_block *sb) +{ + struct v9fs_inode *v9inode; + + v9inode = alloc_inode_sb(sb, v9fs_inode_cache, GFP_KERNEL); + if (!v9inode) + return NULL; + v9inode->cache_validity = 0; + mutex_init(&v9inode->v_mutex); + return &v9inode->netfs.inode; +} + +/** + * v9fs_free_inode - destroy an inode + * @inode: The inode to be freed + */ + +void v9fs_free_inode(struct inode *inode) +{ + kmem_cache_free(v9fs_inode_cache, V9FS_I(inode)); +} + +/* + * Set parameters for the netfs library + */ +void v9fs_set_netfs_context(struct inode *inode) +{ + struct v9fs_inode *v9inode = V9FS_I(inode); + netfs_inode_init(&v9inode->netfs, &v9fs_req_ops, true); +} + +int v9fs_init_inode(struct v9fs_session_info *v9ses, + struct inode *inode, umode_t mode, dev_t rdev) +{ + int err = 0; + + inode_init_owner(&nop_mnt_idmap, inode, NULL, mode); + inode->i_blocks = 0; + inode->i_rdev = rdev; + simple_inode_init_ts(inode); + inode->i_mapping->a_ops = &v9fs_addr_operations; + inode->i_private = NULL; + + switch (mode & S_IFMT) { + case S_IFIFO: + case S_IFBLK: + case S_IFCHR: + case S_IFSOCK: + if (v9fs_proto_dotl(v9ses)) { + inode->i_op = &v9fs_file_inode_operations_dotl; + } else if (v9fs_proto_dotu(v9ses)) { + inode->i_op = &v9fs_file_inode_operations; + } else { + p9_debug(P9_DEBUG_ERROR, + "special files without extended mode\n"); + err = -EINVAL; + goto error; + } + init_special_inode(inode, inode->i_mode, inode->i_rdev); + break; + case S_IFREG: + if (v9fs_proto_dotl(v9ses)) { + inode->i_op = &v9fs_file_inode_operations_dotl; + inode->i_fop = &v9fs_file_operations_dotl; + } else { + inode->i_op = &v9fs_file_inode_operations; + inode->i_fop = &v9fs_file_operations; + } + + break; + case S_IFLNK: + if (!v9fs_proto_dotu(v9ses) && !v9fs_proto_dotl(v9ses)) { + p9_debug(P9_DEBUG_ERROR, + "extended modes used with legacy protocol\n"); + err = -EINVAL; + goto error; + } + + if (v9fs_proto_dotl(v9ses)) { + inode->i_op = &v9fs_symlink_inode_operations_dotl; + inode_nohighmem(inode); + } else { + inode->i_op = &v9fs_symlink_inode_operations; + } + + break; + case S_IFDIR: + inc_nlink(inode); + if (v9fs_proto_dotl(v9ses)) + inode->i_op = &v9fs_dir_inode_operations_dotl; + else if (v9fs_proto_dotu(v9ses)) + inode->i_op = &v9fs_dir_inode_operations_dotu; + else + inode->i_op = &v9fs_dir_inode_operations; + + if (v9fs_proto_dotl(v9ses)) + inode->i_fop = &v9fs_dir_operations_dotl; + else + inode->i_fop = &v9fs_dir_operations; + + break; + default: + p9_debug(P9_DEBUG_ERROR, "BAD mode 0x%hx S_IFMT 0x%x\n", + mode, mode & S_IFMT); + err = -EINVAL; + goto error; + } +error: + return err; + +} + +/** + * v9fs_evict_inode - Remove an inode from the inode cache + * @inode: inode to release + * + */ +void v9fs_evict_inode(struct inode *inode) +{ + struct v9fs_inode __maybe_unused *v9inode = V9FS_I(inode); + __le32 __maybe_unused version; + + if (!is_bad_inode(inode)) { + netfs_wait_for_outstanding_io(inode); + truncate_inode_pages_final(&inode->i_data); + + version = cpu_to_le32(v9inode->qid.version); + netfs_clear_inode_writeback(inode, &version); + + clear_inode(inode); + filemap_fdatawrite(&inode->i_data); + +#ifdef CONFIG_9P_FSCACHE + if (v9fs_inode_cookie(v9inode)) + fscache_relinquish_cookie(v9fs_inode_cookie(v9inode), false); +#endif + } else + clear_inode(inode); +} + +static int v9fs_test_inode(struct inode *inode, void *data) +{ + int umode; + dev_t rdev; + struct v9fs_inode *v9inode = V9FS_I(inode); + struct p9_wstat *st = (struct p9_wstat *)data; + struct v9fs_session_info *v9ses = v9fs_inode2v9ses(inode); + + umode = p9mode2unixmode(v9ses, st, &rdev); + /* don't match inode of different type */ + if (inode_wrong_type(inode, umode)) + return 0; + + /* compare qid details */ + if (memcmp(&v9inode->qid.version, + &st->qid.version, sizeof(v9inode->qid.version))) + return 0; + + if (v9inode->qid.type != st->qid.type) + return 0; + + if (v9inode->qid.path != st->qid.path) + return 0; + return 1; +} + +static int v9fs_test_new_inode(struct inode *inode, void *data) +{ + return 0; +} + +static int v9fs_set_inode(struct inode *inode, void *data) +{ + struct v9fs_inode *v9inode = V9FS_I(inode); + struct p9_wstat *st = (struct p9_wstat *)data; + + memcpy(&v9inode->qid, &st->qid, sizeof(st->qid)); + return 0; +} + +static struct inode *v9fs_qid_iget(struct super_block *sb, + struct p9_qid *qid, + struct p9_wstat *st, + int new) +{ + dev_t rdev; + int retval; + umode_t umode; + struct inode *inode; + struct v9fs_session_info *v9ses = sb->s_fs_info; + int (*test)(struct inode *inode, void *data); + + if (new) + test = v9fs_test_new_inode; + else + test = v9fs_test_inode; + + inode = iget5_locked(sb, QID2INO(qid), test, v9fs_set_inode, st); + if (!inode) + return ERR_PTR(-ENOMEM); + if (!(inode_state_read_once(inode) & I_NEW)) + return inode; + /* + * initialize the inode with the stat info + * FIXME!! we may need support for stale inodes + * later. + */ + inode->i_ino = QID2INO(qid); + umode = p9mode2unixmode(v9ses, st, &rdev); + retval = v9fs_init_inode(v9ses, inode, umode, rdev); + if (retval) + goto error; + + v9fs_stat2inode(st, inode, sb, 0); + v9fs_set_netfs_context(inode); + v9fs_cache_inode_get_cookie(inode); + unlock_new_inode(inode); + return inode; +error: + iget_failed(inode); + return ERR_PTR(retval); + +} + +struct inode * +v9fs_inode_from_fid(struct v9fs_session_info *v9ses, struct p9_fid *fid, + struct super_block *sb, int new) +{ + struct p9_wstat *st; + struct inode *inode = NULL; + + st = p9_client_stat(fid); + if (IS_ERR(st)) + return ERR_CAST(st); + + inode = v9fs_qid_iget(sb, &st->qid, st, new); + p9stat_free(st); + kfree(st); + return inode; +} + +/** + * v9fs_at_to_dotl_flags- convert Linux specific AT flags to + * plan 9 AT flag. + * @flags: flags to convert + */ +static int v9fs_at_to_dotl_flags(int flags) +{ + int rflags = 0; + + if (flags & AT_REMOVEDIR) + rflags |= P9_DOTL_AT_REMOVEDIR; + + return rflags; +} + +/** + * v9fs_dec_count - helper functon to drop i_nlink. + * + * If a directory had nlink <= 2 (including . and ..), then we should not drop + * the link count, which indicates the underlying exported fs doesn't maintain + * nlink accurately. e.g. + * - overlayfs sets nlink to 1 for merged dir + * - ext4 (with dir_nlink feature enabled) sets nlink to 1 if a dir has more + * than EXT4_LINK_MAX (65000) links. + * + * In cacheless mode the server is the source of truth for nlink and the + * inode is going away immediately, so locally adjusting i_nlink buys + * nothing and races with concurrent metadata fetches that may already + * have observed the post-unlink value (nlink == 0). + * + * @inode: inode whose nlink is being dropped + */ +static void v9fs_dec_count(struct inode *inode) +{ + struct v9fs_session_info *v9ses = v9fs_inode2v9ses(inode); + + if (!(v9ses->cache & (CACHE_META | CACHE_LOOSE))) + return; + if (!S_ISDIR(inode->i_mode) || inode->i_nlink > 2) + drop_nlink(inode); +} + +/** + * v9fs_remove - helper function to remove files and directories + * @dir: directory inode that is being deleted + * @dentry: dentry that is being deleted + * @flags: removing a directory + * + */ + +static int v9fs_remove(struct inode *dir, struct dentry *dentry, int flags) +{ + struct inode *inode; + int retval = -EOPNOTSUPP; + struct p9_fid *v9fid, *dfid; + struct v9fs_session_info *v9ses; + + p9_debug(P9_DEBUG_VFS, "inode: %p dentry: %p rmdir: %x\n", + dir, dentry, flags); + + v9ses = v9fs_inode2v9ses(dir); + inode = d_inode(dentry); + dfid = v9fs_parent_fid(dentry); + if (IS_ERR(dfid)) { + retval = PTR_ERR(dfid); + p9_debug(P9_DEBUG_VFS, "fid lookup failed %d\n", retval); + return retval; + } + if (v9fs_proto_dotl(v9ses)) + retval = p9_client_unlinkat(dfid, dentry->d_name.name, + v9fs_at_to_dotl_flags(flags)); + p9_fid_put(dfid); + if (retval == -EOPNOTSUPP) { + /* Try the one based on path */ + v9fid = v9fs_fid_clone(dentry); + if (IS_ERR(v9fid)) + return PTR_ERR(v9fid); + retval = p9_client_remove(v9fid); + } + if (!retval) { + /* + * directories on unlink should have zero + * link count + */ + if (flags & AT_REMOVEDIR) { + clear_nlink(inode); + v9fs_dec_count(dir); + } else + v9fs_dec_count(inode); + + v9fs_invalidate_inode_attr(inode); + v9fs_invalidate_inode_attr(dir); + + /* invalidate all fids associated with dentry */ + /* NOTE: This will not include open fids */ + v9fs_dentry_fid_remove(dentry); + } + return retval; +} + +/** + * v9fs_create - Create a file + * @v9ses: session information + * @dir: directory that dentry is being created in + * @dentry: dentry that is being created + * @extension: 9p2000.u extension string to support devices, etc. + * @perm: create permissions + * @mode: open mode + * + */ +static struct p9_fid * +v9fs_create(struct v9fs_session_info *v9ses, struct inode *dir, + struct dentry *dentry, char *extension, u32 perm, u8 mode) +{ + int err; + const unsigned char *name; + struct p9_fid *dfid, *ofid = NULL, *fid = NULL; + struct inode *inode; + + p9_debug(P9_DEBUG_VFS, "name %pd\n", dentry); + + name = dentry->d_name.name; + dfid = v9fs_parent_fid(dentry); + if (IS_ERR(dfid)) { + err = PTR_ERR(dfid); + p9_debug(P9_DEBUG_VFS, "fid lookup failed %d\n", err); + return ERR_PTR(err); + } + + /* clone a fid to use for creation */ + ofid = clone_fid(dfid); + if (IS_ERR(ofid)) { + err = PTR_ERR(ofid); + p9_debug(P9_DEBUG_VFS, "p9_client_walk failed %d\n", err); + goto error; + } + + err = p9_client_fcreate(ofid, name, perm, mode, extension); + if (err < 0) { + p9_debug(P9_DEBUG_VFS, "p9_client_fcreate failed %d\n", err); + goto error; + } + + if (!(perm & P9_DMLINK)) { + /* now walk from the parent so we can get unopened fid */ + fid = p9_client_walk(dfid, 1, &name, 1); + if (IS_ERR(fid)) { + err = PTR_ERR(fid); + p9_debug(P9_DEBUG_VFS, + "p9_client_walk failed %d\n", err); + goto error; + } + /* + * instantiate inode and assign the unopened fid to the dentry + */ + inode = v9fs_get_new_inode_from_fid(v9ses, fid, dir->i_sb); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + p9_debug(P9_DEBUG_VFS, + "inode creation failed %d\n", err); + goto error; + } + v9fs_fid_add(dentry, &fid); + d_instantiate(dentry, inode); + } + p9_fid_put(dfid); + return ofid; +error: + p9_fid_put(dfid); + p9_fid_put(ofid); + p9_fid_put(fid); + return ERR_PTR(err); +} + +/** + * v9fs_vfs_create - VFS hook to create a regular file + * @idmap: idmap of the mount + * @dir: The parent directory + * @dentry: The name of file to be created + * @mode: The UNIX file mode to set + * + * open(.., O_CREAT) is handled in v9fs_vfs_atomic_open(). This is only called + * for mknod(2). + * + */ + +static int +v9fs_vfs_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct v9fs_session_info *v9ses = v9fs_inode2v9ses(dir); + u32 perm = unixmode2p9mode(v9ses, mode); + struct p9_fid *fid; + + /* P9_OEXCL? */ + fid = v9fs_create(v9ses, dir, dentry, NULL, perm, P9_ORDWR); + if (IS_ERR(fid)) + return PTR_ERR(fid); + + v9fs_invalidate_inode_attr(dir); + p9_fid_put(fid); + + return 0; +} + +/** + * v9fs_vfs_mkdir - VFS mkdir hook to create a directory + * @idmap: idmap of the mount + * @dir: inode that is being unlinked + * @dentry: dentry that is being unlinked + * @mode: mode for new directory + * + */ + +static struct dentry *v9fs_vfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + u32 perm; + struct p9_fid *fid; + struct v9fs_session_info *v9ses; + + p9_debug(P9_DEBUG_VFS, "name %pd\n", dentry); + v9ses = v9fs_inode2v9ses(dir); + perm = unixmode2p9mode(v9ses, mode); + fid = v9fs_create(v9ses, dir, dentry, NULL, perm, P9_OREAD); + if (IS_ERR(fid)) + return ERR_CAST(fid); + inc_nlink(dir); + v9fs_invalidate_inode_attr(dir); + p9_fid_put(fid); + return NULL; +} + +/** + * v9fs_vfs_lookup - VFS lookup hook to "walk" to a new inode + * @dir: inode that is being walked from + * @dentry: dentry that is being walked to? + * @flags: lookup flags (unused) + * + */ + +struct dentry *v9fs_vfs_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + struct dentry *res; + struct v9fs_session_info *v9ses; + struct p9_fid *dfid, *fid; + struct inode *inode; + const unsigned char *name; + + p9_debug(P9_DEBUG_VFS, "dir: %p dentry: (%pd) %p flags: %x\n", + dir, dentry, dentry, flags); + + if (dentry->d_name.len > NAME_MAX) + return ERR_PTR(-ENAMETOOLONG); + + v9ses = v9fs_inode2v9ses(dir); + /* We can walk d_parent because we hold the dir->i_mutex */ + dfid = v9fs_parent_fid(dentry); + if (IS_ERR(dfid)) + return ERR_CAST(dfid); + + /* + * Make sure we don't use a wrong inode due to parallel + * unlink. For cached mode create calls request for new + * inode. But with cache disabled, lookup should do this. + */ + name = dentry->d_name.name; + fid = p9_client_walk(dfid, 1, &name, 1); + p9_fid_put(dfid); + if (fid == ERR_PTR(-ENOENT)) { + inode = NULL; + v9fs_ndentry_refresh_timeout(dentry); + } else if (IS_ERR(fid)) { + inode = ERR_CAST(fid); + } else if (v9ses->cache & (CACHE_META|CACHE_LOOSE)) { + inode = v9fs_get_inode_from_fid(v9ses, fid, dir->i_sb); + } else { + inode = v9fs_get_new_inode_from_fid(v9ses, fid, dir->i_sb); + } + /* + * If we had a rename on the server and a parallel lookup + * for the new name, then make sure we instantiate with + * the new name. ie look up for a/b, while on server somebody + * moved b under k and client parallely did a lookup for + * k/b. + */ + res = d_splice_alias(inode, dentry); + if (!IS_ERR(fid)) { + if (!res) + v9fs_fid_add(dentry, &fid); + else if (!IS_ERR(res)) + v9fs_fid_add(res, &fid); + else + p9_fid_put(fid); + } + return res; +} + +static int +v9fs_vfs_atomic_open(struct inode *dir, struct dentry *dentry, + struct file *file, unsigned int flags, umode_t mode) +{ + int err; + u32 perm; + struct v9fs_inode __maybe_unused *v9inode; + struct v9fs_session_info *v9ses; + struct p9_fid *fid; + struct inode *inode; + int p9_omode; + + if (d_in_lookup(dentry)) { + struct dentry *res = v9fs_vfs_lookup(dir, dentry, 0); + if (res || d_really_is_positive(dentry)) + return finish_no_open(file, res); + } + + /* Only creates */ + if (!(flags & O_CREAT)) + return finish_no_open(file, NULL); + + v9ses = v9fs_inode2v9ses(dir); + perm = unixmode2p9mode(v9ses, mode); + p9_omode = v9fs_uflags2omode(flags, v9fs_proto_dotu(v9ses)); + + if ((v9ses->cache & CACHE_WRITEBACK) && (p9_omode & P9_OWRITE)) { + p9_omode = (p9_omode & ~(P9_OWRITE | P9_OAPPEND)) | P9_ORDWR; + p9_debug(P9_DEBUG_CACHE, + "write-only file with writeback enabled, creating w/ O_RDWR\n"); + } + fid = v9fs_create(v9ses, dir, dentry, NULL, perm, p9_omode); + if (IS_ERR(fid)) + return PTR_ERR(fid); + + v9fs_invalidate_inode_attr(dir); + inode = d_inode(dentry); + v9inode = V9FS_I(inode); + err = finish_open(file, dentry, generic_file_open); + if (unlikely(err)) { + p9_fid_put(fid); + return err; + } + + file->private_data = fid; +#ifdef CONFIG_9P_FSCACHE + if (v9ses->cache & CACHE_FSCACHE) + fscache_use_cookie(v9fs_inode_cookie(v9inode), + file->f_mode & FMODE_WRITE); +#endif + + v9fs_fid_add_modes(fid, v9ses->flags, v9ses->cache, file->f_flags); + v9fs_open_fid_add(inode, &fid); + + file->f_mode |= FMODE_CREATED; + return 0; +} + +/** + * v9fs_vfs_unlink - VFS unlink hook to delete an inode + * @i: inode that is being unlinked + * @d: dentry that is being unlinked + * + */ + +int v9fs_vfs_unlink(struct inode *i, struct dentry *d) +{ + return v9fs_remove(i, d, 0); +} + +/** + * v9fs_vfs_rmdir - VFS unlink hook to delete a directory + * @i: inode that is being unlinked + * @d: dentry that is being unlinked + * + */ + +int v9fs_vfs_rmdir(struct inode *i, struct dentry *d) +{ + return v9fs_remove(i, d, AT_REMOVEDIR); +} + +/** + * v9fs_vfs_rename - VFS hook to rename an inode + * @idmap: The idmap of the mount + * @old_dir: old dir inode + * @old_dentry: old dentry + * @new_dir: new dir inode + * @new_dentry: new dentry + * @flags: RENAME_* flags + * + */ + +int +v9fs_vfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) +{ + int retval; + struct inode *old_inode; + struct inode *new_inode; + struct v9fs_session_info *v9ses; + struct p9_fid *oldfid = NULL, *dfid = NULL; + struct p9_fid *olddirfid = NULL; + struct p9_fid *newdirfid = NULL; + struct p9_wstat wstat; + + if (flags) + return -EINVAL; + + p9_debug(P9_DEBUG_VFS, "\n"); + old_inode = d_inode(old_dentry); + new_inode = d_inode(new_dentry); + v9ses = v9fs_inode2v9ses(old_inode); + oldfid = v9fs_fid_lookup(old_dentry); + if (IS_ERR(oldfid)) + return PTR_ERR(oldfid); + + dfid = v9fs_parent_fid(old_dentry); + olddirfid = clone_fid(dfid); + p9_fid_put(dfid); + dfid = NULL; + + if (IS_ERR(olddirfid)) { + retval = PTR_ERR(olddirfid); + goto error; + } + + dfid = v9fs_parent_fid(new_dentry); + newdirfid = clone_fid(dfid); + p9_fid_put(dfid); + dfid = NULL; + + if (IS_ERR(newdirfid)) { + retval = PTR_ERR(newdirfid); + goto error; + } + + down_write(&v9ses->rename_sem); + if (v9fs_proto_dotl(v9ses)) { + retval = p9_client_renameat(olddirfid, old_dentry->d_name.name, + newdirfid, new_dentry->d_name.name); + if (retval == -EOPNOTSUPP) + retval = p9_client_rename(oldfid, newdirfid, + new_dentry->d_name.name); + if (retval != -EOPNOTSUPP) + goto error_locked; + } + if (old_dentry->d_parent != new_dentry->d_parent) { + /* + * 9P .u can only handle file rename in the same directory + */ + + p9_debug(P9_DEBUG_ERROR, "old dir and new dir are different\n"); + retval = -EXDEV; + goto error_locked; + } + v9fs_blank_wstat(&wstat); + wstat.muid = v9ses->uname; + wstat.name = new_dentry->d_name.name; + retval = p9_client_wstat(oldfid, &wstat); + +error_locked: + if (!retval) { + if (new_inode) { + if (S_ISDIR(new_inode->i_mode)) + clear_nlink(new_inode); + else + v9fs_dec_count(new_inode); + } + if (S_ISDIR(old_inode->i_mode)) { + if (!new_inode) + inc_nlink(new_dir); + v9fs_dec_count(old_dir); + } + v9fs_invalidate_inode_attr(old_inode); + v9fs_invalidate_inode_attr(old_dir); + v9fs_invalidate_inode_attr(new_dir); + + /* successful rename */ + d_move(old_dentry, new_dentry); + } + up_write(&v9ses->rename_sem); + +error: + p9_fid_put(newdirfid); + p9_fid_put(olddirfid); + p9_fid_put(oldfid); + return retval; +} + +/** + * v9fs_vfs_getattr - retrieve file metadata + * @idmap: idmap of the mount + * @path: Object to query + * @stat: metadata structure to populate + * @request_mask: Mask of STATX_xxx flags indicating the caller's interests + * @flags: AT_STATX_xxx setting + * + */ + +static int +v9fs_vfs_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags) +{ + struct dentry *dentry = path->dentry; + struct inode *inode = d_inode(dentry); + struct v9fs_session_info *v9ses; + struct p9_fid *fid; + struct p9_wstat *st; + + p9_debug(P9_DEBUG_VFS, "dentry: %p\n", dentry); + v9ses = v9fs_dentry2v9ses(dentry); + if (v9ses->cache & (CACHE_META|CACHE_LOOSE)) { + generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat); + return 0; + } else if (v9ses->cache & CACHE_WRITEBACK) { + if (S_ISREG(inode->i_mode)) { + int retval = filemap_fdatawrite(inode->i_mapping); + + if (retval) + p9_debug(P9_DEBUG_ERROR, + "flushing writeback during getattr returned %d\n", retval); + } + } + fid = v9fs_fid_lookup(dentry); + if (IS_ERR(fid)) + return PTR_ERR(fid); + + st = p9_client_stat(fid); + p9_fid_put(fid); + if (IS_ERR(st)) + return PTR_ERR(st); + + v9fs_stat2inode(st, d_inode(dentry), dentry->d_sb, 0); + generic_fillattr(&nop_mnt_idmap, request_mask, d_inode(dentry), stat); + + p9stat_free(st); + kfree(st); + return 0; +} + +/** + * v9fs_vfs_setattr - set file metadata + * @idmap: idmap of the mount + * @dentry: file whose metadata to set + * @iattr: metadata assignment structure + * + */ + +static int v9fs_vfs_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, struct iattr *iattr) +{ + int retval, use_dentry = 0; + struct inode *inode = d_inode(dentry); + struct v9fs_session_info *v9ses; + struct p9_fid *fid = NULL; + struct p9_wstat wstat; + + p9_debug(P9_DEBUG_VFS, "\n"); + retval = setattr_prepare(&nop_mnt_idmap, dentry, iattr); + if (retval) + return retval; + + v9ses = v9fs_dentry2v9ses(dentry); + if (iattr->ia_valid & ATTR_FILE) { + fid = iattr->ia_file->private_data; + WARN_ON(!fid); + } + if (!fid) { + fid = v9fs_fid_lookup(dentry); + use_dentry = 1; + } + if (IS_ERR(fid)) + return PTR_ERR(fid); + + v9fs_blank_wstat(&wstat); + if (iattr->ia_valid & ATTR_MODE) + wstat.mode = unixmode2p9mode(v9ses, iattr->ia_mode); + + if (iattr->ia_valid & ATTR_MTIME) + wstat.mtime = iattr->ia_mtime.tv_sec; + + if (iattr->ia_valid & ATTR_ATIME) + wstat.atime = iattr->ia_atime.tv_sec; + + if (iattr->ia_valid & ATTR_SIZE) + wstat.length = iattr->ia_size; + + if (v9fs_proto_dotu(v9ses)) { + if (iattr->ia_valid & ATTR_UID) + wstat.n_uid = iattr->ia_uid; + + if (iattr->ia_valid & ATTR_GID) + wstat.n_gid = iattr->ia_gid; + } + + /* Write all dirty data */ + if (d_is_reg(dentry)) { + retval = filemap_fdatawrite(inode->i_mapping); + if (retval) + p9_debug(P9_DEBUG_ERROR, + "flushing writeback during setattr returned %d\n", retval); + } + + retval = p9_client_wstat(fid, &wstat); + + if (use_dentry) + p9_fid_put(fid); + + if (retval < 0) + return retval; + + if ((iattr->ia_valid & ATTR_SIZE) && + iattr->ia_size != i_size_read(inode)) { + truncate_setsize(inode, iattr->ia_size); + netfs_resize_file(netfs_inode(inode), iattr->ia_size, true); + +#ifdef CONFIG_9P_FSCACHE + if (v9ses->cache & CACHE_FSCACHE) { + struct v9fs_inode *v9inode = V9FS_I(inode); + + fscache_resize_cookie(v9fs_inode_cookie(v9inode), iattr->ia_size); + } +#endif + } + + v9fs_invalidate_inode_attr(inode); + + setattr_copy(&nop_mnt_idmap, inode, iattr); + mark_inode_dirty(inode); + return 0; +} + +/** + * v9fs_stat2inode - populate an inode structure with mistat info + * @stat: Plan 9 metadata (mistat) structure + * @inode: inode to populate + * @sb: superblock of filesystem + * @flags: control flags (e.g. V9FS_STAT2INODE_KEEP_ISIZE) + * + */ + +void +v9fs_stat2inode(struct p9_wstat *stat, struct inode *inode, + struct super_block *sb, unsigned int flags) +{ + umode_t mode; + struct v9fs_session_info *v9ses = sb->s_fs_info; + struct v9fs_inode *v9inode = V9FS_I(inode); + + inode_set_atime(inode, stat->atime, 0); + inode_set_mtime(inode, stat->mtime, 0); + inode_set_ctime(inode, stat->mtime, 0); + + inode->i_uid = v9ses->dfltuid; + inode->i_gid = v9ses->dfltgid; + + if (v9fs_proto_dotu(v9ses)) { + inode->i_uid = stat->n_uid; + inode->i_gid = stat->n_gid; + } + if ((S_ISREG(inode->i_mode)) || (S_ISDIR(inode->i_mode))) { + if (v9fs_proto_dotu(v9ses)) { + unsigned int i_nlink; + /* + * Hadlink support got added later to the .u extension. + * So there can be a server out there that doesn't + * support this even with .u extension. That would + * just leave us with stat->extension being an empty + * string, though. + */ + /* HARDLINKCOUNT %u */ + if (sscanf(stat->extension, + " HARDLINKCOUNT %u", &i_nlink) == 1) + set_nlink(inode, i_nlink); + } + } + mode = p9mode2perm(v9ses, stat); + mode |= inode->i_mode & ~S_IALLUGO; + inode->i_mode = mode; + + spin_lock(&inode->i_lock); + netfs_write_remote_i_size(inode, stat->length); + if (!(flags & V9FS_STAT2INODE_KEEP_ISIZE)) + i_size_write(inode, stat->length); + /* not real number of blocks, but 512 byte ones ... */ + inode->i_blocks = (stat->length + 512 - 1) >> 9; + spin_unlock(&inode->i_lock); + v9inode->cache_validity &= ~V9FS_INO_INVALID_ATTR; +} + +/** + * v9fs_vfs_get_link - follow a symlink path + * @dentry: dentry for symlink + * @inode: inode for symlink + * @done: delayed call for when we are done with the return value + */ + +static const char *v9fs_vfs_get_link(struct dentry *dentry, + struct inode *inode, + struct delayed_call *done) +{ + struct v9fs_session_info *v9ses; + struct p9_fid *fid; + struct p9_wstat *st; + char *res; + + if (!dentry) + return ERR_PTR(-ECHILD); + + v9ses = v9fs_dentry2v9ses(dentry); + if (!v9fs_proto_dotu(v9ses)) + return ERR_PTR(-EBADF); + + p9_debug(P9_DEBUG_VFS, "%pd\n", dentry); + fid = v9fs_fid_lookup(dentry); + + if (IS_ERR(fid)) + return ERR_CAST(fid); + + st = p9_client_stat(fid); + p9_fid_put(fid); + if (IS_ERR(st)) + return ERR_CAST(st); + + if (!(st->mode & P9_DMSYMLINK)) { + p9stat_free(st); + kfree(st); + return ERR_PTR(-EINVAL); + } + res = st->extension; + st->extension = NULL; + if (strlen(res) >= PATH_MAX) + res[PATH_MAX - 1] = '\0'; + + p9stat_free(st); + kfree(st); + set_delayed_call(done, kfree_link, res); + return res; +} + +/** + * v9fs_vfs_mkspecial - create a special file + * @dir: inode to create special file in + * @dentry: dentry to create + * @perm: mode to create special file + * @extension: 9p2000.u format extension string representing special file + * + */ + +static int v9fs_vfs_mkspecial(struct inode *dir, struct dentry *dentry, + u32 perm, const char *extension) +{ + struct p9_fid *fid; + struct v9fs_session_info *v9ses; + + v9ses = v9fs_inode2v9ses(dir); + if (!v9fs_proto_dotu(v9ses)) { + p9_debug(P9_DEBUG_ERROR, "not extended\n"); + return -EPERM; + } + + fid = v9fs_create(v9ses, dir, dentry, (char *) extension, perm, + P9_OREAD); + if (IS_ERR(fid)) + return PTR_ERR(fid); + + v9fs_invalidate_inode_attr(dir); + p9_fid_put(fid); + return 0; +} + +/** + * v9fs_vfs_symlink - helper function to create symlinks + * @idmap: idmap of the mount + * @dir: directory inode containing symlink + * @dentry: dentry for symlink + * @symname: symlink data + * + * See Also: 9P2000.u RFC for more information + * + */ + +static int +v9fs_vfs_symlink(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const char *symname) +{ + p9_debug(P9_DEBUG_VFS, " %llu,%pd,%s\n", + dir->i_ino, dentry, symname); + + return v9fs_vfs_mkspecial(dir, dentry, P9_DMSYMLINK, symname); +} + +#define U32_MAX_DIGITS 10 + +/** + * v9fs_vfs_link - create a hardlink + * @old_dentry: dentry for file to link to + * @dir: inode destination for new link + * @dentry: dentry for link + * + */ + +static int +v9fs_vfs_link(struct dentry *old_dentry, struct inode *dir, + struct dentry *dentry) +{ + int retval; + char name[1 + U32_MAX_DIGITS + 2]; /* sign + number + \n + \0 */ + struct p9_fid *oldfid; + + p9_debug(P9_DEBUG_VFS, " %llu,%pd,%pd\n", + dir->i_ino, dentry, old_dentry); + + oldfid = v9fs_fid_clone(old_dentry); + if (IS_ERR(oldfid)) + return PTR_ERR(oldfid); + + sprintf(name, "%d\n", oldfid->fid); + retval = v9fs_vfs_mkspecial(dir, dentry, P9_DMLINK, name); + if (!retval) { + v9fs_refresh_inode(oldfid, d_inode(old_dentry)); + v9fs_invalidate_inode_attr(dir); + } + p9_fid_put(oldfid); + return retval; +} + +/** + * v9fs_vfs_mknod - create a special file + * @idmap: idmap of the mount + * @dir: inode destination for new link + * @dentry: dentry for file + * @mode: mode for creation + * @rdev: device associated with special file + * + */ + +static int +v9fs_vfs_mknod(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) +{ + struct v9fs_session_info *v9ses = v9fs_inode2v9ses(dir); + int retval; + char name[2 + U32_MAX_DIGITS + 1 + U32_MAX_DIGITS + 1]; + u32 perm; + + p9_debug(P9_DEBUG_VFS, " %llu,%pd mode: %x MAJOR: %u MINOR: %u\n", + dir->i_ino, dentry, mode, + MAJOR(rdev), MINOR(rdev)); + + /* build extension */ + if (S_ISBLK(mode)) + sprintf(name, "b %u %u", MAJOR(rdev), MINOR(rdev)); + else if (S_ISCHR(mode)) + sprintf(name, "c %u %u", MAJOR(rdev), MINOR(rdev)); + else + *name = 0; + + perm = unixmode2p9mode(v9ses, mode); + retval = v9fs_vfs_mkspecial(dir, dentry, perm, name); + + return retval; +} + +int v9fs_refresh_inode(struct p9_fid *fid, struct inode *inode) +{ + int umode; + dev_t rdev; + struct p9_wstat *st; + struct v9fs_session_info *v9ses; + unsigned int flags; + + v9ses = v9fs_inode2v9ses(inode); + st = p9_client_stat(fid); + if (IS_ERR(st)) + return PTR_ERR(st); + /* + * Don't update inode if the file type is different + */ + umode = p9mode2unixmode(v9ses, st, &rdev); + if (inode_wrong_type(inode, umode)) + goto out; + + /* + * We don't want to refresh inode->i_size, + * because we may have cached data + */ + flags = (v9ses->cache & CACHE_LOOSE) ? + V9FS_STAT2INODE_KEEP_ISIZE : 0; + v9fs_stat2inode(st, inode, inode->i_sb, flags); +out: + p9stat_free(st); + kfree(st); + return 0; +} + +static const struct inode_operations v9fs_dir_inode_operations_dotu = { + .create = v9fs_vfs_create, + .lookup = v9fs_vfs_lookup, + .atomic_open = v9fs_vfs_atomic_open, + .symlink = v9fs_vfs_symlink, + .link = v9fs_vfs_link, + .unlink = v9fs_vfs_unlink, + .mkdir = v9fs_vfs_mkdir, + .rmdir = v9fs_vfs_rmdir, + .mknod = v9fs_vfs_mknod, + .rename = v9fs_vfs_rename, + .getattr = v9fs_vfs_getattr, + .setattr = v9fs_vfs_setattr, +}; + +static const struct inode_operations v9fs_dir_inode_operations = { + .create = v9fs_vfs_create, + .lookup = v9fs_vfs_lookup, + .atomic_open = v9fs_vfs_atomic_open, + .unlink = v9fs_vfs_unlink, + .mkdir = v9fs_vfs_mkdir, + .rmdir = v9fs_vfs_rmdir, + .mknod = v9fs_vfs_mknod, + .rename = v9fs_vfs_rename, + .getattr = v9fs_vfs_getattr, + .setattr = v9fs_vfs_setattr, +}; + +static const struct inode_operations v9fs_file_inode_operations = { + .getattr = v9fs_vfs_getattr, + .setattr = v9fs_vfs_setattr, +}; + +static const struct inode_operations v9fs_symlink_inode_operations = { + .get_link = v9fs_vfs_get_link, + .getattr = v9fs_vfs_getattr, + .setattr = v9fs_vfs_setattr, +}; diff --git a/fs/9p/vfs_inode_dotl.c b/fs/9p/vfs_inode_dotl.c new file mode 100644 index 000000000..116b29e95 --- /dev/null +++ b/fs/9p/vfs_inode_dotl.c @@ -0,0 +1,994 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file contains vfs inode ops for the 9P2000.L protocol. + * + * Copyright (C) 2004 by Eric Van Hensbergen + * Copyright (C) 2002 by Ron Minnich + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "v9fs.h" +#include "v9fs_vfs.h" +#include "fid.h" +#include "cache.h" +#include "xattr.h" +#include "acl.h" + +static int +v9fs_vfs_mknod_dotl(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t omode, dev_t rdev); + +/** + * v9fs_get_fsgid_for_create - Helper function to get the gid for a new object + * @dir_inode: The directory inode + * + * Helper function to get the gid for creating a + * new file system object. This checks the S_ISGID to determine the owning + * group of the new file system object. + */ + +static kgid_t v9fs_get_fsgid_for_create(struct inode *dir_inode) +{ + BUG_ON(dir_inode == NULL); + + if (dir_inode->i_mode & S_ISGID) { + /* set_gid bit is set.*/ + return dir_inode->i_gid; + } + return current_fsgid(); +} + +static int v9fs_test_inode_dotl(struct inode *inode, void *data) +{ + struct v9fs_inode *v9inode = V9FS_I(inode); + struct p9_stat_dotl *st = (struct p9_stat_dotl *)data; + + /* don't match inode of different type */ + if (inode_wrong_type(inode, st->st_mode)) + return 0; + + if (inode->i_generation != st->st_gen) + return 0; + + /* compare qid details */ + if (memcmp(&v9inode->qid.version, + &st->qid.version, sizeof(v9inode->qid.version))) + return 0; + + if (v9inode->qid.type != st->qid.type) + return 0; + + if (v9inode->qid.path != st->qid.path) + return 0; + return 1; +} + +/* Always get a new inode */ +static int v9fs_test_new_inode_dotl(struct inode *inode, void *data) +{ + return 0; +} + +static int v9fs_set_inode_dotl(struct inode *inode, void *data) +{ + struct v9fs_inode *v9inode = V9FS_I(inode); + struct p9_stat_dotl *st = (struct p9_stat_dotl *)data; + + memcpy(&v9inode->qid, &st->qid, sizeof(st->qid)); + inode->i_generation = st->st_gen; + return 0; +} + +static struct inode *v9fs_qid_iget_dotl(struct super_block *sb, + struct p9_qid *qid, + struct p9_fid *fid, + struct p9_stat_dotl *st, + int new) +{ + int retval; + struct inode *inode; + struct v9fs_session_info *v9ses = sb->s_fs_info; + int (*test)(struct inode *inode, void *data); + + if (new) + test = v9fs_test_new_inode_dotl; + else + test = v9fs_test_inode_dotl; + + inode = iget5_locked(sb, QID2INO(qid), test, v9fs_set_inode_dotl, st); + if (!inode) + return ERR_PTR(-ENOMEM); + if (!(inode_state_read_once(inode) & I_NEW)) + return inode; + /* + * initialize the inode with the stat info + * FIXME!! we may need support for stale inodes + * later. + */ + inode->i_ino = QID2INO(qid); + retval = v9fs_init_inode(v9ses, inode, + st->st_mode, new_decode_dev(st->st_rdev)); + if (retval) + goto error; + + v9fs_stat2inode_dotl(st, inode, 0); + v9fs_set_netfs_context(inode); + v9fs_cache_inode_get_cookie(inode); + retval = v9fs_get_acl(inode, fid); + if (retval) + goto error; + + unlock_new_inode(inode); + return inode; +error: + iget_failed(inode); + return ERR_PTR(retval); + +} + +struct inode * +v9fs_inode_from_fid_dotl(struct v9fs_session_info *v9ses, struct p9_fid *fid, + struct super_block *sb, int new) +{ + struct p9_stat_dotl *st; + struct inode *inode = NULL; + + st = p9_client_getattr_dotl(fid, P9_STATS_BASIC | P9_STATS_GEN); + if (IS_ERR(st)) + return ERR_CAST(st); + + inode = v9fs_qid_iget_dotl(sb, &st->qid, fid, st, new); + kfree(st); + return inode; +} + +struct dotl_openflag_map { + int open_flag; + int dotl_flag; +}; + +static int v9fs_mapped_dotl_flags(int flags) +{ + int i; + int rflags = 0; + struct dotl_openflag_map dotl_oflag_map[] = { + { O_CREAT, P9_DOTL_CREATE }, + { O_EXCL, P9_DOTL_EXCL }, + { O_NOCTTY, P9_DOTL_NOCTTY }, + { O_APPEND, P9_DOTL_APPEND }, + { O_NONBLOCK, P9_DOTL_NONBLOCK }, + { O_DSYNC, P9_DOTL_DSYNC }, + { FASYNC, P9_DOTL_FASYNC }, + { O_DIRECT, P9_DOTL_DIRECT }, + { O_LARGEFILE, P9_DOTL_LARGEFILE }, + { O_DIRECTORY, P9_DOTL_DIRECTORY }, + { O_NOFOLLOW, P9_DOTL_NOFOLLOW }, + { O_NOATIME, P9_DOTL_NOATIME }, + { O_CLOEXEC, P9_DOTL_CLOEXEC }, + { O_SYNC, P9_DOTL_SYNC}, + }; + for (i = 0; i < ARRAY_SIZE(dotl_oflag_map); i++) { + if (flags & dotl_oflag_map[i].open_flag) + rflags |= dotl_oflag_map[i].dotl_flag; + } + return rflags; +} + +/** + * v9fs_open_to_dotl_flags- convert Linux specific open flags to + * plan 9 open flag. + * @flags: flags to convert + */ +int v9fs_open_to_dotl_flags(int flags) +{ + int rflags = 0; + + /* + * We have same bits for P9_DOTL_READONLY, P9_DOTL_WRONLY + * and P9_DOTL_NOACCESS + */ + rflags |= flags & O_ACCMODE; + rflags |= v9fs_mapped_dotl_flags(flags); + + return rflags; +} + +/** + * v9fs_vfs_create_dotl - VFS hook to create files for 9P2000.L protocol. + * @idmap: The user namespace of the mount + * @dir: directory inode that is being created + * @dentry: dentry that is being deleted + * @omode: create permissions + * + */ +static int +v9fs_vfs_create_dotl(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t omode) +{ + return v9fs_vfs_mknod_dotl(idmap, dir, dentry, omode, 0); +} + +static int +v9fs_vfs_atomic_open_dotl(struct inode *dir, struct dentry *dentry, + struct file *file, unsigned int flags, umode_t omode) +{ + int err = 0; + kgid_t gid; + umode_t mode; + int p9_omode = v9fs_open_to_dotl_flags(flags); + const unsigned char *name = NULL; + struct p9_qid qid; + struct inode *inode; + struct p9_fid *fid = NULL; + struct p9_fid *dfid = NULL, *ofid = NULL; + struct v9fs_session_info *v9ses; + struct posix_acl *pacl = NULL, *dacl = NULL; + + if (d_in_lookup(dentry)) { + struct dentry *res = v9fs_vfs_lookup(dir, dentry, 0); + if (res || d_really_is_positive(dentry)) + return finish_no_open(file, res); + } + + /* Only creates */ + if (!(flags & O_CREAT)) + return finish_no_open(file, NULL); + + v9ses = v9fs_inode2v9ses(dir); + + name = dentry->d_name.name; + p9_debug(P9_DEBUG_VFS, "name:%s flags:0x%x mode:0x%x\n", + name, flags, omode); + + dfid = v9fs_parent_fid(dentry); + if (IS_ERR(dfid)) { + err = PTR_ERR(dfid); + p9_debug(P9_DEBUG_VFS, "fid lookup failed %d\n", err); + goto out; + } + + /* clone a fid to use for creation */ + ofid = clone_fid(dfid); + if (IS_ERR(ofid)) { + err = PTR_ERR(ofid); + p9_debug(P9_DEBUG_VFS, "p9_client_walk failed %d\n", err); + goto out; + } + + gid = v9fs_get_fsgid_for_create(dir); + + mode = omode; + /* Update mode based on ACL value */ + err = v9fs_acl_mode(dir, &mode, &dacl, &pacl); + if (err) { + p9_debug(P9_DEBUG_VFS, "Failed to get acl values in create %d\n", + err); + goto out; + } + + if ((v9ses->cache & CACHE_WRITEBACK) && (p9_omode & P9_OWRITE)) { + p9_omode = (p9_omode & ~(P9_OWRITE | P9_DOTL_APPEND)) | P9_ORDWR; + p9_debug(P9_DEBUG_CACHE, + "write-only file with writeback enabled, creating w/ O_RDWR\n"); + } + err = p9_client_create_dotl(ofid, name, p9_omode, mode, gid, &qid); + if (err < 0) { + p9_debug(P9_DEBUG_VFS, "p9_client_open_dotl failed in create %d\n", + err); + goto out; + } + v9fs_invalidate_inode_attr(dir); + + /* instantiate inode and assign the unopened fid to the dentry */ + fid = p9_client_walk(dfid, 1, &name, 1); + if (IS_ERR(fid)) { + err = PTR_ERR(fid); + p9_debug(P9_DEBUG_VFS, "p9_client_walk failed %d\n", err); + goto out; + } + inode = v9fs_get_new_inode_from_fid(v9ses, fid, dir->i_sb); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + p9_debug(P9_DEBUG_VFS, "inode creation failed %d\n", err); + goto out; + } + /* Now set the ACL based on the default value */ + v9fs_set_create_acl(inode, fid, dacl, pacl); + + v9fs_fid_add(dentry, &fid); + d_instantiate(dentry, inode); + + /* Since we are opening a file, assign the open fid to the file */ + err = finish_open(file, dentry, generic_file_open); + if (err) + goto out; + file->private_data = ofid; +#ifdef CONFIG_9P_FSCACHE + if (v9ses->cache & CACHE_FSCACHE) { + struct v9fs_inode *v9inode = V9FS_I(inode); + fscache_use_cookie(v9fs_inode_cookie(v9inode), + file->f_mode & FMODE_WRITE); + } +#endif + v9fs_fid_add_modes(ofid, v9ses->flags, v9ses->cache, flags); + v9fs_open_fid_add(inode, &ofid); + file->f_mode |= FMODE_CREATED; +out: + p9_fid_put(dfid); + p9_fid_put(ofid); + p9_fid_put(fid); + v9fs_put_acl(dacl, pacl); + return err; +} + +/** + * v9fs_vfs_mkdir_dotl - VFS mkdir hook to create a directory + * @idmap: The idmap of the mount + * @dir: inode that is being unlinked + * @dentry: dentry that is being unlinked + * @omode: mode for new directory + * + */ + +static struct dentry *v9fs_vfs_mkdir_dotl(struct mnt_idmap *idmap, + struct inode *dir, struct dentry *dentry, + umode_t omode) +{ + int err = 0; + struct v9fs_session_info *v9ses; + struct p9_fid *fid = NULL, *dfid = NULL; + kgid_t gid; + const unsigned char *name; + umode_t mode; + struct inode *inode; + struct p9_qid qid; + struct posix_acl *dacl = NULL, *pacl = NULL; + + p9_debug(P9_DEBUG_VFS, "name %pd\n", dentry); + v9ses = v9fs_inode2v9ses(dir); + + if (dir->i_mode & S_ISGID) + omode |= S_ISGID; + + dfid = v9fs_parent_fid(dentry); + if (IS_ERR(dfid)) { + err = PTR_ERR(dfid); + p9_debug(P9_DEBUG_VFS, "fid lookup failed %d\n", err); + goto error; + } + + gid = v9fs_get_fsgid_for_create(dir); + mode = omode; + /* Update mode based on ACL value */ + err = v9fs_acl_mode(dir, &mode, &dacl, &pacl); + if (err) { + p9_debug(P9_DEBUG_VFS, "Failed to get acl values in mkdir %d\n", + err); + goto error; + } + name = dentry->d_name.name; + err = p9_client_mkdir_dotl(dfid, name, mode, gid, &qid); + if (err < 0) + goto error; + fid = p9_client_walk(dfid, 1, &name, 1); + if (IS_ERR(fid)) { + err = PTR_ERR(fid); + p9_debug(P9_DEBUG_VFS, "p9_client_walk failed %d\n", + err); + goto error; + } + + /* instantiate inode and assign the unopened fid to the dentry */ + inode = v9fs_get_new_inode_from_fid(v9ses, fid, dir->i_sb); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + p9_debug(P9_DEBUG_VFS, "inode creation failed %d\n", + err); + goto error; + } + v9fs_set_create_acl(inode, fid, dacl, pacl); + v9fs_fid_add(dentry, &fid); + d_instantiate(dentry, inode); + err = 0; + inc_nlink(dir); + v9fs_invalidate_inode_attr(dir); +error: + p9_fid_put(fid); + v9fs_put_acl(dacl, pacl); + p9_fid_put(dfid); + return err ? ERR_PTR(err) : NULL; +} + +static int +v9fs_vfs_getattr_dotl(struct mnt_idmap *idmap, + const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int flags) +{ + struct dentry *dentry = path->dentry; + struct v9fs_session_info *v9ses; + struct p9_fid *fid; + struct inode *inode = d_inode(dentry); + struct p9_stat_dotl *st; + + p9_debug(P9_DEBUG_VFS, "dentry: %p\n", dentry); + v9ses = v9fs_dentry2v9ses(dentry); + if (v9ses->cache & (CACHE_META|CACHE_LOOSE)) { + generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat); + return 0; + } else if (v9ses->cache) { + if (S_ISREG(inode->i_mode)) { + int retval = filemap_fdatawrite(inode->i_mapping); + + if (retval) + p9_debug(P9_DEBUG_ERROR, + "flushing writeback during getattr returned %d\n", retval); + } + } + fid = v9fs_fid_lookup(dentry); + if (IS_ERR(fid)) + return PTR_ERR(fid); + + /* Ask for all the fields in stat structure. Server will return + * whatever it supports + */ + + st = p9_client_getattr_dotl(fid, P9_STATS_ALL); + p9_fid_put(fid); + if (IS_ERR(st)) + return PTR_ERR(st); + + v9fs_stat2inode_dotl(st, d_inode(dentry), 0); + generic_fillattr(&nop_mnt_idmap, request_mask, d_inode(dentry), stat); + /* Change block size to what the server returned */ + stat->blksize = st->st_blksize; + + kfree(st); + return 0; +} + +/* + * Attribute flags. + */ +#define P9_ATTR_MODE (1 << 0) +#define P9_ATTR_UID (1 << 1) +#define P9_ATTR_GID (1 << 2) +#define P9_ATTR_SIZE (1 << 3) +#define P9_ATTR_ATIME (1 << 4) +#define P9_ATTR_MTIME (1 << 5) +#define P9_ATTR_CTIME (1 << 6) +#define P9_ATTR_ATIME_SET (1 << 7) +#define P9_ATTR_MTIME_SET (1 << 8) + +struct dotl_iattr_map { + int iattr_valid; + int p9_iattr_valid; +}; + +static int v9fs_mapped_iattr_valid(int iattr_valid) +{ + int i; + int p9_iattr_valid = 0; + struct dotl_iattr_map dotl_iattr_map[] = { + { ATTR_MODE, P9_ATTR_MODE }, + { ATTR_UID, P9_ATTR_UID }, + { ATTR_GID, P9_ATTR_GID }, + { ATTR_SIZE, P9_ATTR_SIZE }, + { ATTR_ATIME, P9_ATTR_ATIME }, + { ATTR_MTIME, P9_ATTR_MTIME }, + { ATTR_CTIME, P9_ATTR_CTIME }, + { ATTR_ATIME_SET, P9_ATTR_ATIME_SET }, + { ATTR_MTIME_SET, P9_ATTR_MTIME_SET }, + }; + for (i = 0; i < ARRAY_SIZE(dotl_iattr_map); i++) { + if (iattr_valid & dotl_iattr_map[i].iattr_valid) + p9_iattr_valid |= dotl_iattr_map[i].p9_iattr_valid; + } + return p9_iattr_valid; +} + +/** + * v9fs_vfs_setattr_dotl - set file metadata + * @idmap: idmap of the mount + * @dentry: file whose metadata to set + * @iattr: metadata assignment structure + * + */ + +int v9fs_vfs_setattr_dotl(struct mnt_idmap *idmap, + struct dentry *dentry, struct iattr *iattr) +{ + int retval, use_dentry = 0; + struct inode *inode = d_inode(dentry); + struct v9fs_session_info __maybe_unused *v9ses; + struct p9_fid *fid = NULL; + struct p9_iattr_dotl p9attr = { + .uid = INVALID_UID, + .gid = INVALID_GID, + }; + + p9_debug(P9_DEBUG_VFS, "\n"); + + retval = setattr_prepare(&nop_mnt_idmap, dentry, iattr); + if (retval) + return retval; + + v9ses = v9fs_dentry2v9ses(dentry); + + p9attr.valid = v9fs_mapped_iattr_valid(iattr->ia_valid); + if (iattr->ia_valid & ATTR_MODE) + p9attr.mode = iattr->ia_mode; + if (iattr->ia_valid & ATTR_UID) + p9attr.uid = iattr->ia_uid; + if (iattr->ia_valid & ATTR_GID) + p9attr.gid = iattr->ia_gid; + if (iattr->ia_valid & ATTR_SIZE) + p9attr.size = iattr->ia_size; + if (iattr->ia_valid & ATTR_ATIME_SET) { + p9attr.atime_sec = iattr->ia_atime.tv_sec; + p9attr.atime_nsec = iattr->ia_atime.tv_nsec; + } + if (iattr->ia_valid & ATTR_MTIME_SET) { + p9attr.mtime_sec = iattr->ia_mtime.tv_sec; + p9attr.mtime_nsec = iattr->ia_mtime.tv_nsec; + } + + if (iattr->ia_valid & ATTR_FILE) { + fid = iattr->ia_file->private_data; + WARN_ON(!fid); + } + if (!fid) { + fid = v9fs_fid_lookup(dentry); + use_dentry = 1; + } + if (IS_ERR(fid)) + return PTR_ERR(fid); + + /* Write all dirty data */ + if (S_ISREG(inode->i_mode)) { + retval = filemap_fdatawrite(inode->i_mapping); + if (retval < 0) + p9_debug(P9_DEBUG_ERROR, + "Flushing file prior to setattr failed: %d\n", retval); + } + + retval = p9_client_setattr(fid, &p9attr); + if (retval < 0) { + if (use_dentry) + p9_fid_put(fid); + return retval; + } + + if ((iattr->ia_valid & ATTR_SIZE) && iattr->ia_size != + i_size_read(inode)) { + truncate_setsize(inode, iattr->ia_size); + netfs_resize_file(netfs_inode(inode), iattr->ia_size, true); + +#ifdef CONFIG_9P_FSCACHE + if (v9ses->cache & CACHE_FSCACHE) + fscache_resize_cookie(v9fs_inode_cookie(V9FS_I(inode)), + iattr->ia_size); +#endif + } + + v9fs_invalidate_inode_attr(inode); + setattr_copy(&nop_mnt_idmap, inode, iattr); + mark_inode_dirty(inode); + if (iattr->ia_valid & ATTR_MODE) { + /* We also want to update ACL when we update mode bits */ + retval = v9fs_acl_chmod(inode, fid); + if (retval < 0) { + if (use_dentry) + p9_fid_put(fid); + return retval; + } + } + if (use_dentry) + p9_fid_put(fid); + + return 0; +} + +/** + * v9fs_stat2inode_dotl - populate an inode structure with stat info + * @stat: stat structure + * @inode: inode to populate + * @flags: ctrl flags (e.g. V9FS_STAT2INODE_KEEP_ISIZE) + * + */ + +void +v9fs_stat2inode_dotl(struct p9_stat_dotl *stat, struct inode *inode, + unsigned int flags) +{ + umode_t mode; + struct v9fs_inode *v9inode = V9FS_I(inode); + + if ((stat->st_result_mask & P9_STATS_BASIC) == P9_STATS_BASIC) { + inode_set_atime(inode, stat->st_atime_sec, + stat->st_atime_nsec); + inode_set_mtime(inode, stat->st_mtime_sec, + stat->st_mtime_nsec); + inode_set_ctime(inode, stat->st_ctime_sec, + stat->st_ctime_nsec); + inode->i_uid = stat->st_uid; + inode->i_gid = stat->st_gid; + set_nlink(inode, stat->st_nlink); + + mode = stat->st_mode & S_IALLUGO; + mode |= inode->i_mode & ~S_IALLUGO; + inode->i_mode = mode; + + spin_lock(&inode->i_lock); + netfs_write_remote_i_size(inode, stat->st_size); + if (!(flags & V9FS_STAT2INODE_KEEP_ISIZE)) + i_size_write(inode, stat->st_size); + inode->i_blocks = stat->st_blocks; + spin_unlock(&inode->i_lock); + } else { + if (stat->st_result_mask & P9_STATS_ATIME) { + inode_set_atime(inode, stat->st_atime_sec, + stat->st_atime_nsec); + } + if (stat->st_result_mask & P9_STATS_MTIME) { + inode_set_mtime(inode, stat->st_mtime_sec, + stat->st_mtime_nsec); + } + if (stat->st_result_mask & P9_STATS_CTIME) { + inode_set_ctime(inode, stat->st_ctime_sec, + stat->st_ctime_nsec); + } + if (stat->st_result_mask & P9_STATS_UID) + inode->i_uid = stat->st_uid; + if (stat->st_result_mask & P9_STATS_GID) + inode->i_gid = stat->st_gid; + if (stat->st_result_mask & P9_STATS_NLINK) + set_nlink(inode, stat->st_nlink); + if (stat->st_result_mask & P9_STATS_MODE) { + mode = stat->st_mode & S_IALLUGO; + mode |= inode->i_mode & ~S_IALLUGO; + inode->i_mode = mode; + } + spin_lock(&inode->i_lock); + if (!(flags & V9FS_STAT2INODE_KEEP_ISIZE) && + stat->st_result_mask & P9_STATS_SIZE) { + netfs_write_remote_i_size(inode, stat->st_size); + i_size_write(inode, stat->st_size); + } + if (stat->st_result_mask & P9_STATS_BLOCKS) + inode->i_blocks = stat->st_blocks; + spin_unlock(&inode->i_lock); + } + if (stat->st_result_mask & P9_STATS_GEN) + inode->i_generation = stat->st_gen; + + /* Currently we don't support P9_STATS_BTIME and P9_STATS_DATA_VERSION + * because the inode structure does not have fields for them. + */ + v9inode->cache_validity &= ~V9FS_INO_INVALID_ATTR; +} + +static int +v9fs_vfs_symlink_dotl(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const char *symname) +{ + int err; + kgid_t gid; + const unsigned char *name; + struct v9fs_session_info *v9ses; + struct p9_qid qid; + struct p9_fid *dfid; + struct p9_fid *fid = NULL; + struct inode *inode; + + name = dentry->d_name.name; + p9_debug(P9_DEBUG_VFS, "%llu,%s,%s\n", dir->i_ino, name, symname); + + dfid = v9fs_parent_fid(dentry); + if (IS_ERR(dfid)) { + err = PTR_ERR(dfid); + p9_debug(P9_DEBUG_VFS, "fid lookup failed %d\n", err); + return err; + } + + gid = v9fs_get_fsgid_for_create(dir); + + /* Server doesn't alter fid on TSYMLINK. Hence no need to clone it. */ + err = p9_client_symlink(dfid, name, symname, gid, &qid); + + if (err < 0) { + p9_debug(P9_DEBUG_VFS, "p9_client_symlink failed %d\n", err); + goto error; + } + + v9fs_invalidate_inode_attr(dir); + + /* instantiate inode and assign the unopened fid to the dentry */ + fid = p9_client_walk(dfid, 1, &name, 1); + if (IS_ERR(fid)) { + err = PTR_ERR(fid); + p9_debug(P9_DEBUG_VFS, "p9_client_walk failed %d\n", + err); + goto error; + } + + v9ses = v9fs_inode2v9ses(dir); + inode = v9fs_get_new_inode_from_fid(v9ses, fid, dir->i_sb); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + p9_debug(P9_DEBUG_VFS, "inode creation failed %d\n", + err); + goto error; + } + v9fs_fid_add(dentry, &fid); + d_instantiate(dentry, inode); + err = 0; +error: + p9_fid_put(fid); + p9_fid_put(dfid); + return err; +} + +/** + * v9fs_vfs_link_dotl - create a hardlink for dotl + * @old_dentry: dentry for file to link to + * @dir: inode destination for new link + * @dentry: dentry for link + * + */ + +static int +v9fs_vfs_link_dotl(struct dentry *old_dentry, struct inode *dir, + struct dentry *dentry) +{ + int err; + struct p9_fid *dfid, *oldfid; + struct v9fs_session_info *v9ses; + + p9_debug(P9_DEBUG_VFS, "dir ino: %llu, old_name: %pd, new_name: %pd\n", + dir->i_ino, old_dentry, dentry); + + v9ses = v9fs_inode2v9ses(dir); + dfid = v9fs_parent_fid(dentry); + if (IS_ERR(dfid)) + return PTR_ERR(dfid); + + oldfid = v9fs_fid_lookup(old_dentry); + if (IS_ERR(oldfid)) { + p9_fid_put(dfid); + return PTR_ERR(oldfid); + } + + err = p9_client_link(dfid, oldfid, dentry->d_name.name); + + p9_fid_put(dfid); + p9_fid_put(oldfid); + if (err < 0) { + p9_debug(P9_DEBUG_VFS, "p9_client_link failed %d\n", err); + return err; + } + + v9fs_invalidate_inode_attr(dir); + if (v9ses->cache & (CACHE_META|CACHE_LOOSE)) { + /* Get the latest stat info from server. */ + struct p9_fid *fid; + + fid = v9fs_fid_lookup(old_dentry); + if (IS_ERR(fid)) + return PTR_ERR(fid); + + v9fs_refresh_inode_dotl(fid, d_inode(old_dentry)); + p9_fid_put(fid); + } + ihold(d_inode(old_dentry)); + d_instantiate(dentry, d_inode(old_dentry)); + + return err; +} + +/** + * v9fs_vfs_mknod_dotl - create a special file + * @idmap: The idmap of the mount + * @dir: inode destination for new link + * @dentry: dentry for file + * @omode: mode for creation + * @rdev: device associated with special file + * + */ +static int +v9fs_vfs_mknod_dotl(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t omode, dev_t rdev) +{ + int err; + kgid_t gid; + const unsigned char *name; + umode_t mode; + struct v9fs_session_info *v9ses; + struct p9_fid *fid = NULL, *dfid = NULL; + struct inode *inode; + struct p9_qid qid; + struct posix_acl *dacl = NULL, *pacl = NULL; + + p9_debug(P9_DEBUG_VFS, " %llu,%pd mode: %x MAJOR: %u MINOR: %u\n", + dir->i_ino, dentry, omode, + MAJOR(rdev), MINOR(rdev)); + + v9ses = v9fs_inode2v9ses(dir); + dfid = v9fs_parent_fid(dentry); + if (IS_ERR(dfid)) { + err = PTR_ERR(dfid); + p9_debug(P9_DEBUG_VFS, "fid lookup failed %d\n", err); + goto error; + } + + gid = v9fs_get_fsgid_for_create(dir); + mode = omode; + /* Update mode based on ACL value */ + err = v9fs_acl_mode(dir, &mode, &dacl, &pacl); + if (err) { + p9_debug(P9_DEBUG_VFS, "Failed to get acl values in mknod %d\n", + err); + goto error; + } + name = dentry->d_name.name; + + err = p9_client_mknod_dotl(dfid, name, mode, rdev, gid, &qid); + if (err < 0) + goto error; + + v9fs_invalidate_inode_attr(dir); + fid = p9_client_walk(dfid, 1, &name, 1); + if (IS_ERR(fid)) { + err = PTR_ERR(fid); + p9_debug(P9_DEBUG_VFS, "p9_client_walk failed %d\n", + err); + goto error; + } + inode = v9fs_get_new_inode_from_fid(v9ses, fid, dir->i_sb); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + p9_debug(P9_DEBUG_VFS, "inode creation failed %d\n", + err); + goto error; + } + v9fs_set_create_acl(inode, fid, dacl, pacl); + v9fs_fid_add(dentry, &fid); + d_instantiate(dentry, inode); + err = 0; +error: + p9_fid_put(fid); + v9fs_put_acl(dacl, pacl); + p9_fid_put(dfid); + + return err; +} + +/** + * v9fs_vfs_get_link_nocache_dotl - Resolve a symlink directly. + * + * To be used when symlink caching is not enabled. + * + * @dentry: dentry for symlink + * @inode: inode for symlink + * @done: destructor for return value + */ +static const char * +v9fs_vfs_get_link_nocache_dotl(struct dentry *dentry, + struct inode *inode, + struct delayed_call *done) +{ + struct p9_fid *fid; + char *target; + int retval; + + if (!dentry) + return ERR_PTR(-ECHILD); + + p9_debug(P9_DEBUG_VFS, "%pd\n", dentry); + + fid = v9fs_fid_lookup(dentry); + if (IS_ERR(fid)) + return ERR_CAST(fid); + retval = p9_client_readlink(fid, &target); + p9_fid_put(fid); + if (retval) + return ERR_PTR(retval); + set_delayed_call(done, kfree_link, target); + return target; +} + +/** + * v9fs_vfs_get_link_dotl - follow a symlink path + * @dentry: dentry for symlink + * @inode: inode for symlink + * @done: destructor for return value + */ +static const char * +v9fs_vfs_get_link_dotl(struct dentry *dentry, + struct inode *inode, + struct delayed_call *done) +{ + struct v9fs_session_info *v9ses; + + v9ses = v9fs_inode2v9ses(inode); + if (v9ses->cache & (CACHE_META|CACHE_LOOSE)) + return page_get_link(dentry, inode, done); + + return v9fs_vfs_get_link_nocache_dotl(dentry, inode, done); +} + +int v9fs_refresh_inode_dotl(struct p9_fid *fid, struct inode *inode) +{ + struct p9_stat_dotl *st; + struct v9fs_session_info *v9ses; + unsigned int flags; + + v9ses = v9fs_inode2v9ses(inode); + st = p9_client_getattr_dotl(fid, P9_STATS_ALL); + if (IS_ERR(st)) + return PTR_ERR(st); + /* + * Don't update inode if the file type is different + */ + if (inode_wrong_type(inode, st->st_mode)) + goto out; + + /* + * We don't want to refresh inode->i_size, + * because we may have cached data + */ + flags = (v9ses->cache & CACHE_LOOSE) ? + V9FS_STAT2INODE_KEEP_ISIZE : 0; + v9fs_stat2inode_dotl(st, inode, flags); +out: + kfree(st); + return 0; +} + +const struct inode_operations v9fs_dir_inode_operations_dotl = { + .create = v9fs_vfs_create_dotl, + .atomic_open = v9fs_vfs_atomic_open_dotl, + .lookup = v9fs_vfs_lookup, + .link = v9fs_vfs_link_dotl, + .symlink = v9fs_vfs_symlink_dotl, + .unlink = v9fs_vfs_unlink, + .mkdir = v9fs_vfs_mkdir_dotl, + .rmdir = v9fs_vfs_rmdir, + .mknod = v9fs_vfs_mknod_dotl, + .rename = v9fs_vfs_rename, + .getattr = v9fs_vfs_getattr_dotl, + .setattr = v9fs_vfs_setattr_dotl, + .listxattr = v9fs_listxattr, + .get_inode_acl = v9fs_iop_get_inode_acl, + .get_acl = v9fs_iop_get_acl, + .set_acl = v9fs_iop_set_acl, +}; + +const struct inode_operations v9fs_file_inode_operations_dotl = { + .getattr = v9fs_vfs_getattr_dotl, + .setattr = v9fs_vfs_setattr_dotl, + .listxattr = v9fs_listxattr, + .get_inode_acl = v9fs_iop_get_inode_acl, + .get_acl = v9fs_iop_get_acl, + .set_acl = v9fs_iop_set_acl, +}; + +const struct inode_operations v9fs_symlink_inode_operations_dotl = { + .get_link = v9fs_vfs_get_link_dotl, + .getattr = v9fs_vfs_getattr_dotl, + .setattr = v9fs_vfs_setattr_dotl, + .listxattr = v9fs_listxattr, +}; diff --git a/fs/9p/vfs_super.c b/fs/9p/vfs_super.c new file mode 100644 index 000000000..94d6b02c2 --- /dev/null +++ b/fs/9p/vfs_super.c @@ -0,0 +1,366 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * + * Copyright (C) 2004 by Eric Van Hensbergen + * Copyright (C) 2002 by Ron Minnich + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "v9fs.h" +#include "v9fs_vfs.h" +#include "fid.h" +#include "xattr.h" +#include "acl.h" + +static const struct super_operations v9fs_super_ops, v9fs_super_ops_dotl; + +static int v9fs_fill_super(struct super_block *sb) +{ + int ret; + struct v9fs_session_info *v9ses = v9ses = sb->s_fs_info; + + sb->s_maxbytes = MAX_LFS_FILESIZE; + sb->s_blocksize_bits = fls(v9ses->maxdata - 1); + sb->s_blocksize = 1 << sb->s_blocksize_bits; + sb->s_magic = V9FS_MAGIC; + if (v9fs_proto_dotl(v9ses)) { + sb->s_op = &v9fs_super_ops_dotl; + if (!(v9ses->flags & V9FS_NO_XATTR)) + sb->s_xattr = v9fs_xattr_handlers; + } else { + sb->s_op = &v9fs_super_ops; + sb->s_time_max = U32_MAX; + } + + sb->s_time_min = 0; + + ret = super_setup_bdi(sb); + if (ret) + return ret; + + if (!v9ses->cache) { + sb->s_bdi->ra_pages = 0; + sb->s_bdi->io_pages = 0; + } else { + sb->s_bdi->ra_pages = v9ses->maxdata >> PAGE_SHIFT; + sb->s_bdi->io_pages = v9ses->maxdata >> PAGE_SHIFT; + } + + sb->s_flags |= SB_ACTIVE; + +#ifdef CONFIG_9P_FS_POSIX_ACL + if ((v9ses->flags & V9FS_ACL_MASK) == V9FS_POSIX_ACL) + sb->s_flags |= SB_POSIXACL; +#endif + + return 0; +} + +/** + * v9fs_get_tree - create the mountable root and superblock + * @fc: the filesystem context + * + */ + +static int v9fs_get_tree(struct fs_context *fc) +{ + struct super_block *sb = NULL; + struct inode *inode = NULL; + struct dentry *root = NULL; + struct v9fs_session_info *v9ses = NULL; + struct p9_fid *fid; + int retval = 0; + + p9_debug(P9_DEBUG_VFS, "\n"); + + v9ses = kzalloc_obj(struct v9fs_session_info); + if (!v9ses) + return -ENOMEM; + + fid = v9fs_session_init(v9ses, fc); + if (IS_ERR(fid)) { + retval = PTR_ERR(fid); + goto free_session; + } + + fc->s_fs_info = v9ses; + sb = sget_fc(fc, NULL, set_anon_super_fc); + if (IS_ERR(sb)) { + retval = PTR_ERR(sb); + goto clunk_fid; + } + retval = v9fs_fill_super(sb); + if (retval) + goto release_sb; + + if (v9ses->cache & (CACHE_META|CACHE_LOOSE)) { + set_default_d_op(sb, &v9fs_cached_dentry_operations); + } else { + set_default_d_op(sb, &v9fs_dentry_operations); + sb->s_d_flags |= DCACHE_DONTCACHE; + } + + inode = v9fs_get_new_inode_from_fid(v9ses, fid, sb); + if (IS_ERR(inode)) { + retval = PTR_ERR(inode); + goto release_sb; + } + + root = d_make_root(inode); + if (!root) { + retval = -ENOMEM; + goto release_sb; + } + sb->s_root = root; + retval = v9fs_get_acl(inode, fid); + if (retval) + goto release_sb; + v9fs_fid_add(root, &fid); + + p9_debug(P9_DEBUG_VFS, " simple set mount, return 0\n"); + fc->root = dget(sb->s_root); + return 0; + +clunk_fid: + p9_fid_put(fid); + v9fs_session_close(v9ses); +free_session: + kfree(v9ses); + return retval; + +release_sb: + /* + * we will do the session_close and root dentry release + * in the below call. But we need to clunk fid, because we haven't + * attached the fid to dentry so it won't get clunked + * automatically. + */ + p9_fid_put(fid); + deactivate_locked_super(sb); + return retval; +} + +/** + * v9fs_kill_super - Kill Superblock + * @s: superblock + * + */ + +static void v9fs_kill_super(struct super_block *s) +{ + struct v9fs_session_info *v9ses = s->s_fs_info; + + p9_debug(P9_DEBUG_VFS, " %p\n", s); + + kill_anon_super(s); + + v9fs_session_cancel(v9ses); + v9fs_session_close(v9ses); + kfree(v9ses); + s->s_fs_info = NULL; + p9_debug(P9_DEBUG_VFS, "exiting kill_super\n"); +} + +static void +v9fs_umount_begin(struct super_block *sb) +{ + struct v9fs_session_info *v9ses; + + v9ses = sb->s_fs_info; + v9fs_session_begin_cancel(v9ses); +} + +static int v9fs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct v9fs_session_info *v9ses; + struct p9_fid *fid; + struct p9_rstatfs rs; + int res; + + fid = v9fs_fid_lookup(dentry); + if (IS_ERR(fid)) { + res = PTR_ERR(fid); + goto done; + } + + v9ses = v9fs_dentry2v9ses(dentry); + if (v9fs_proto_dotl(v9ses)) { + res = p9_client_statfs(fid, &rs); + if (res == 0) { + buf->f_type = rs.type; + buf->f_bsize = rs.bsize; + buf->f_blocks = rs.blocks; + buf->f_bfree = rs.bfree; + buf->f_bavail = rs.bavail; + buf->f_files = rs.files; + buf->f_ffree = rs.ffree; + buf->f_fsid = u64_to_fsid(rs.fsid); + buf->f_namelen = rs.namelen; + } + if (res != -ENOSYS) + goto done; + } + res = simple_statfs(dentry, buf); +done: + p9_fid_put(fid); + return res; +} + +static int v9fs_drop_inode(struct inode *inode) +{ + struct v9fs_session_info *v9ses; + + v9ses = v9fs_inode2v9ses(inode); + if (v9ses->cache & (CACHE_META|CACHE_LOOSE)) + return inode_generic_drop(inode); + /* + * in case of non cached mode always drop the + * inode because we want the inode attribute + * to always match that on the server. + */ + return 1; +} + +static int v9fs_write_inode(struct inode *inode, + struct writeback_control *wbc) +{ + /* + * send an fsync request to server irrespective of + * wbc->sync_mode. + */ + p9_debug(P9_DEBUG_VFS, "%s: inode %p\n", __func__, inode); + return netfs_unpin_writeback(inode, wbc); +} + +static int v9fs_write_inode_dotl(struct inode *inode, + struct writeback_control *wbc) +{ + + p9_debug(P9_DEBUG_VFS, "%s: inode %p\n", __func__, inode); + + return netfs_unpin_writeback(inode, wbc); +} + +static const struct super_operations v9fs_super_ops = { + .alloc_inode = v9fs_alloc_inode, + .free_inode = v9fs_free_inode, + .statfs = simple_statfs, + .drop_inode = v9fs_drop_inode, + .evict_inode = v9fs_evict_inode, + .show_options = v9fs_show_options, + .umount_begin = v9fs_umount_begin, + .write_inode = v9fs_write_inode, +}; + +static const struct super_operations v9fs_super_ops_dotl = { + .alloc_inode = v9fs_alloc_inode, + .free_inode = v9fs_free_inode, + .statfs = v9fs_statfs, + .drop_inode = v9fs_drop_inode, + .evict_inode = v9fs_evict_inode, + .show_options = v9fs_show_options, + .umount_begin = v9fs_umount_begin, + .write_inode = v9fs_write_inode_dotl, +}; + +static void v9fs_free_fc(struct fs_context *fc) +{ + struct v9fs_context *ctx = fc->fs_private; + + if (!ctx) + return; + + /* These should be NULL by now but guard against leaks */ + kfree(ctx->session_opts.uname); + kfree(ctx->session_opts.aname); +#ifdef CONFIG_9P_FSCACHE + kfree(ctx->session_opts.cachetag); +#endif + if (ctx->client_opts.trans_mod) + v9fs_put_trans(ctx->client_opts.trans_mod); + kfree(ctx); +} + +static const struct fs_context_operations v9fs_context_ops = { + .parse_param = v9fs_parse_param, + .get_tree = v9fs_get_tree, + .free = v9fs_free_fc, +}; + +static int v9fs_init_fs_context(struct fs_context *fc) +{ + struct v9fs_context *ctx; + + ctx = kzalloc_obj(*ctx); + if (!ctx) + return -ENOMEM; + + fc->ops = &v9fs_context_ops; + fc->fs_private = ctx; + + /* initialize core options */ + ctx->session_opts.afid = ~0; + ctx->session_opts.cache = CACHE_NONE; + ctx->session_opts.session_lock_timeout = P9_LOCK_TIMEOUT; + ctx->session_opts.uname = kstrdup(V9FS_DEFUSER, GFP_KERNEL); + if (!ctx->session_opts.uname) + goto error; + + ctx->session_opts.aname = kstrdup(V9FS_DEFANAME, GFP_KERNEL); + if (!ctx->session_opts.aname) + goto error; + + ctx->session_opts.uid = INVALID_UID; + ctx->session_opts.dfltuid = V9FS_DEFUID; + ctx->session_opts.dfltgid = V9FS_DEFGID; + ctx->session_opts.ndentry_timeout_ms = 0; + + /* initialize client options */ + ctx->client_opts.proto_version = p9_proto_2000L; + ctx->client_opts.msize = DEFAULT_MSIZE; + + /* initialize fd transport options */ + ctx->fd_opts.port = P9_FD_PORT; + ctx->fd_opts.rfd = ~0; + ctx->fd_opts.wfd = ~0; + ctx->fd_opts.privport = false; + + /* initialize rdma transport options */ + ctx->rdma_opts.port = P9_RDMA_PORT; + ctx->rdma_opts.sq_depth = P9_RDMA_SQ_DEPTH; + ctx->rdma_opts.rq_depth = P9_RDMA_RQ_DEPTH; + ctx->rdma_opts.timeout = P9_RDMA_TIMEOUT; + ctx->rdma_opts.privport = false; + + return 0; +error: + fc->need_free = 1; + return -ENOMEM; +} + +struct file_system_type v9fs_fs_type = { + .name = "9p", + .kill_sb = v9fs_kill_super, + .owner = THIS_MODULE, + .fs_flags = FS_RENAME_DOES_D_MOVE, + .init_fs_context = v9fs_init_fs_context, + .parameters = v9fs_param_spec, +}; +MODULE_ALIAS_FS("9p"); diff --git a/fs/9p/xattr.c b/fs/9p/xattr.c new file mode 100644 index 000000000..8604e3377 --- /dev/null +++ b/fs/9p/xattr.c @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * Copyright IBM Corporation, 2010 + * Author Aneesh Kumar K.V + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "fid.h" +#include "xattr.h" + +ssize_t v9fs_fid_xattr_get(struct p9_fid *fid, const char *name, + void *buffer, size_t buffer_size) +{ + ssize_t retval; + u64 attr_size; + struct p9_fid *attr_fid; + struct kvec kvec = {.iov_base = buffer, .iov_len = buffer_size}; + struct iov_iter to; + int err; + + iov_iter_kvec(&to, ITER_DEST, &kvec, 1, buffer_size); + + attr_fid = p9_client_xattrwalk(fid, name, &attr_size); + if (IS_ERR(attr_fid)) { + retval = PTR_ERR(attr_fid); + p9_debug(P9_DEBUG_VFS, "p9_client_attrwalk failed %zd\n", + retval); + return retval; + } + if (attr_size > buffer_size) { + if (buffer_size) + retval = -ERANGE; + else if (attr_size > SSIZE_MAX) + retval = -EOVERFLOW; + else /* request to get the attr_size */ + retval = attr_size; + } else { + iov_iter_truncate(&to, attr_size); + retval = p9_client_read(attr_fid, 0, &to, &err); + if (err) + retval = err; + } + p9_fid_put(attr_fid); + return retval; +} + + +/* + * v9fs_xattr_get() + * + * Copy an extended attribute into the buffer + * provided, or compute the buffer size required. + * Buffer is NULL to compute the size of the buffer required. + * + * Returns a negative error number on failure, or the number of bytes + * used / required on success. + */ +ssize_t v9fs_xattr_get(struct dentry *dentry, const char *name, + void *buffer, size_t buffer_size) +{ + struct p9_fid *fid; + int ret; + + p9_debug(P9_DEBUG_VFS, "name = '%s' value_len = %zu\n", + name, buffer_size); + fid = v9fs_fid_lookup(dentry); + if (IS_ERR(fid)) + return PTR_ERR(fid); + ret = v9fs_fid_xattr_get(fid, name, buffer, buffer_size); + p9_fid_put(fid); + + return ret; +} + +/* + * v9fs_xattr_set() + * + * Create, replace or remove an extended attribute for this inode. Buffer + * is NULL to remove an existing extended attribute, and non-NULL to + * either replace an existing extended attribute, or create a new extended + * attribute. The flags XATTR_REPLACE and XATTR_CREATE + * specify that an extended attribute must exist and must not exist + * previous to the call, respectively. + * + * Returns 0, or a negative error number on failure. + */ +int v9fs_xattr_set(struct dentry *dentry, const char *name, + const void *value, size_t value_len, int flags) +{ + int ret; + struct p9_fid *fid; + + fid = v9fs_fid_lookup(dentry); + if (IS_ERR(fid)) + return PTR_ERR(fid); + ret = v9fs_fid_xattr_set(fid, name, value, value_len, flags); + p9_fid_put(fid); + return ret; +} + +int v9fs_fid_xattr_set(struct p9_fid *fid, const char *name, + const void *value, size_t value_len, int flags) +{ + struct kvec kvec = {.iov_base = (void *)value, .iov_len = value_len}; + struct iov_iter from; + int retval, err; + + iov_iter_kvec(&from, ITER_SOURCE, &kvec, 1, value_len); + + p9_debug(P9_DEBUG_VFS, "name = %s value_len = %zu flags = %d\n", + name, value_len, flags); + + /* Clone it */ + fid = clone_fid(fid); + if (IS_ERR(fid)) + return PTR_ERR(fid); + + /* + * On success fid points to xattr + */ + retval = p9_client_xattrcreate(fid, name, value_len, flags); + if (retval < 0) + p9_debug(P9_DEBUG_VFS, "p9_client_xattrcreate failed %d\n", + retval); + else + p9_client_write(fid, 0, &from, &retval); + err = p9_fid_put(fid); + if (!retval && err) + retval = err; + return retval; +} + +ssize_t v9fs_listxattr(struct dentry *dentry, char *buffer, size_t buffer_size) +{ + /* Txattrwalk with an empty string lists xattrs instead */ + return v9fs_xattr_get(dentry, "", buffer, buffer_size); +} + +static int v9fs_xattr_handler_get(const struct xattr_handler *handler, + struct dentry *dentry, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + const char *full_name = xattr_full_name(handler, name); + + return v9fs_xattr_get(dentry, full_name, buffer, size); +} + +static int v9fs_xattr_handler_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *dentry, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +{ + const char *full_name = xattr_full_name(handler, name); + + return v9fs_xattr_set(dentry, full_name, value, size, flags); +} + +static const struct xattr_handler v9fs_xattr_user_handler = { + .prefix = XATTR_USER_PREFIX, + .get = v9fs_xattr_handler_get, + .set = v9fs_xattr_handler_set, +}; + +static const struct xattr_handler v9fs_xattr_trusted_handler = { + .prefix = XATTR_TRUSTED_PREFIX, + .get = v9fs_xattr_handler_get, + .set = v9fs_xattr_handler_set, +}; + +#ifdef CONFIG_9P_FS_SECURITY +static const struct xattr_handler v9fs_xattr_security_handler = { + .prefix = XATTR_SECURITY_PREFIX, + .get = v9fs_xattr_handler_get, + .set = v9fs_xattr_handler_set, +}; +#endif + +const struct xattr_handler * const v9fs_xattr_handlers[] = { + &v9fs_xattr_user_handler, + &v9fs_xattr_trusted_handler, +#ifdef CONFIG_9P_FS_SECURITY + &v9fs_xattr_security_handler, +#endif + NULL +}; diff --git a/fs/9p/xattr.h b/fs/9p/xattr.h new file mode 100644 index 000000000..3ad5a8023 --- /dev/null +++ b/fs/9p/xattr.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +/* + * Copyright IBM Corporation, 2010 + * Author Aneesh Kumar K.V + */ +#ifndef FS_9P_XATTR_H +#define FS_9P_XATTR_H + +#include +#include +#include + +extern const struct xattr_handler * const v9fs_xattr_handlers[]; + +ssize_t v9fs_fid_xattr_get(struct p9_fid *fid, const char *name, + void *buffer, size_t buffer_size); +ssize_t v9fs_xattr_get(struct dentry *dentry, const char *name, + void *buffer, size_t buffer_size); +int v9fs_fid_xattr_set(struct p9_fid *fid, const char *name, + const void *value, size_t value_len, int flags); +int v9fs_xattr_set(struct dentry *dentry, const char *name, + const void *value, size_t value_len, int flags); +ssize_t v9fs_listxattr(struct dentry *dentry, char *buffer, + size_t buffer_size); +#endif /* FS_9P_XATTR_H */ diff --git a/fs/Kconfig b/fs/Kconfig new file mode 100644 index 000000000..e05917adc --- /dev/null +++ b/fs/Kconfig @@ -0,0 +1,424 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# File system configuration +# + +menu "File systems" + +# Use unaligned word dcache accesses +config DCACHE_WORD_ACCESS + bool + +config VALIDATE_FS_PARSER + bool "Validate filesystem parameter description" + help + Enable this to perform validation of the parameter description for a + filesystem when it is registered. + +config FS_IOMAP + bool + +# Stackable filesystems +config FS_STACK + bool + +config BUFFER_HEAD + bool + +# old blockdev_direct_IO implementation. Use iomap for new code instead +config LEGACY_DIRECT_IO + depends on BUFFER_HEAD + bool + +if BLOCK + +source "fs/ext2/Kconfig" +source "fs/ext4/Kconfig" +source "fs/jbd2/Kconfig" + +config FS_MBCACHE +# Meta block cache for Extended Attributes (ext2/ext3/ext4) + tristate + default y if EXT2_FS=y && EXT2_FS_XATTR + default y if EXT4_FS=y + default m if EXT2_FS_XATTR || EXT4_FS + +source "fs/jfs/Kconfig" + +source "fs/xfs/Kconfig" +source "fs/gfs2/Kconfig" +source "fs/ocfs2/Kconfig" +source "fs/btrfs/Kconfig" +source "fs/nilfs2/Kconfig" +source "fs/f2fs/Kconfig" +source "fs/zonefs/Kconfig" + +endif # BLOCK + +config FS_DAX + bool "File system based Direct Access (DAX) support" + depends on MMU + depends on ZONE_DEVICE + select FS_IOMAP + select DAX + help + Direct Access (DAX) can be used on memory-backed block devices. + If the block device supports DAX and the filesystem supports DAX, + then you can avoid using the pagecache to buffer I/Os. Turning + on this option will compile in support for DAX. + + For a DAX device to support file system access it needs to have + struct pages. For the nfit based NVDIMMs this can be enabled + using the ndctl utility: + + # ndctl create-namespace --force --reconfig=namespace0.0 \ + --mode=fsdax --map=mem + + See the 'create-namespace' man page for details on the overhead of + --map=mem: + https://docs.pmem.io/ndctl-user-guide/ndctl-man-pages/ndctl-create-namespace + + For ndctl to work CONFIG_DEV_DAX needs to be enabled as well. For most + file systems DAX support needs to be manually enabled globally or + per-inode using a mount option as well. See the file documentation in + Documentation/filesystems/dax.rst for details. + + If you do not have a block device that is capable of using this, + or if unsure, say N. Saying Y will increase the size of the kernel + by about 5kB. + +config FS_DAX_PMD + bool + default FS_DAX + depends on FS_DAX + depends on ZONE_DEVICE + depends on TRANSPARENT_HUGEPAGE + +# Posix ACL utility routines +# +# Note: Posix ACLs can be implemented without these helpers. Never use +# this symbol for ifdefs in core code. +# +config FS_POSIX_ACL + def_bool n + +config EXPORTFS + tristate + +config EXPORTFS_BLOCK_OPS + bool "Enable filesystem export operations for block IO" + help + This option enables the export operations for a filesystem to support + external block IO. + +config FILE_LOCKING + bool "Enable POSIX file locking API" if EXPERT + default y + help + This option enables standard file locking support, required + for filesystems like NFS and for the flock() system + call. Disabling this option saves about 11k. + +source "fs/crypto/Kconfig" + +source "fs/verity/Kconfig" + +source "fs/notify/Kconfig" + +source "fs/quota/Kconfig" + +source "fs/autofs/Kconfig" +source "fs/fuse/Kconfig" +source "fs/overlayfs/Kconfig" + +menu "Caches" + +source "fs/netfs/Kconfig" +source "fs/cachefiles/Kconfig" + +endmenu + +if BLOCK +menu "CD-ROM/DVD Filesystems" + +source "fs/isofs/Kconfig" +source "fs/udf/Kconfig" + +endmenu +endif # BLOCK + +if BLOCK +menu "DOS/FAT/EXFAT/NT Filesystems" + +source "fs/fat/Kconfig" +source "fs/exfat/Kconfig" +source "fs/ntfs/Kconfig" +source "fs/ntfs3/Kconfig" + +endmenu +endif # BLOCK + +menu "Pseudo filesystems" + +source "fs/proc/Kconfig" +source "fs/kernfs/Kconfig" +source "fs/sysfs/Kconfig" + +config TMPFS + bool "Tmpfs virtual memory file system support (former shm fs)" + depends on SHMEM + select MEMFD_CREATE + help + Tmpfs is a file system which keeps all files in virtual memory. + + Everything in tmpfs is temporary in the sense that no files will be + created on your hard drive. The files live in memory and swap + space. If you unmount a tmpfs instance, everything stored therein is + lost. + + See for details. + +config TMPFS_POSIX_ACL + bool "Tmpfs POSIX Access Control Lists" + depends on TMPFS + select TMPFS_XATTR + select FS_POSIX_ACL + help + POSIX Access Control Lists (ACLs) support additional access rights + for users and groups beyond the standard owner/group/world scheme, + and this option selects support for ACLs specifically for tmpfs + filesystems. + + If you've selected TMPFS, it's possible that you'll also need + this option as there are a number of Linux distros that require + POSIX ACL support under /dev for certain features to work properly. + For example, some distros need this feature for ALSA-related /dev + files for sound to work properly. In short, if you're not sure, + say Y. + +config TMPFS_XATTR + bool "Tmpfs extended attributes" + depends on TMPFS + default n + help + Extended attributes are name:value pairs associated with inodes by + the kernel or by users (see the attr(5) manual page for details). + + This enables support for the trusted.*, security.* and user.* + namespaces. + + You need this for POSIX ACL support on tmpfs. + + If unsure, say N. + +config TMPFS_INODE64 + bool "Use 64-bit ino_t by default in tmpfs" + depends on TMPFS && 64BIT + default n + help + tmpfs has historically used only inode numbers as wide as an unsigned + int. In some cases this can cause wraparound, potentially resulting + in multiple files with the same inode number on a single device. This + option makes tmpfs use the full width of ino_t by default, without + needing to specify the inode64 option when mounting. + + But if a long-lived tmpfs is to be accessed by 32-bit applications so + ancient that opening a file larger than 2GiB fails with EINVAL, then + the INODE64 config option and inode64 mount option risk operations + failing with EOVERFLOW once 33-bit inode numbers are reached. + + To override this configured default, use the inode32 or inode64 + option when mounting. + + If unsure, say N. + +config TMPFS_QUOTA + bool "Tmpfs quota support" + depends on TMPFS + select QUOTA + help + Quota support allows to set per user and group limits for tmpfs + usage. Say Y to enable quota support. Once enabled you can control + user and group quota enforcement with quota, usrquota and grpquota + mount options. + + If unsure, say N. + +config ARCH_SUPPORTS_HUGETLBFS + def_bool n + +menuconfig HUGETLBFS + bool "HugeTLB file system support" + depends on ARCH_SUPPORTS_HUGETLBFS + select MEMFD_CREATE + select PADATA if SMP + help + hugetlbfs is a filesystem backing for HugeTLB pages, based on + ramfs. For architectures that support it, say Y here and read + for details. + + If unsure, say N. + +if HUGETLBFS +config HUGETLB_PAGE_OPTIMIZE_VMEMMAP_DEFAULT_ON + bool "HugeTLB Vmemmap Optimization (HVO) defaults to on" + default n + depends on HUGETLB_PAGE_OPTIMIZE_VMEMMAP + help + The HugeTLB Vmemmap Optimization (HVO) defaults to off. Say Y here to + enable HVO by default. It can be disabled via hugetlb_free_vmemmap=off + (boot command line) or hugetlb_optimize_vmemmap (sysctl). +endif # HUGETLBFS + +config HUGETLB_PAGE + def_bool HUGETLBFS + select XARRAY_MULTI + +config HUGETLB_PAGE_OPTIMIZE_VMEMMAP + def_bool HUGETLB_PAGE + depends on ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP + depends on SPARSEMEM_VMEMMAP + select SPARSEMEM_VMEMMAP_PREINIT if ARCH_WANT_HUGETLB_VMEMMAP_PREINIT + +config HUGETLB_PMD_PAGE_TABLE_SHARING + def_bool HUGETLB_PAGE + depends on ARCH_WANT_HUGE_PMD_SHARE && SPLIT_PMD_PTLOCKS + +config ARCH_HAS_GIGANTIC_PAGE + bool + +source "fs/configfs/Kconfig" +source "fs/efivarfs/Kconfig" + +endmenu + +menuconfig MISC_FILESYSTEMS + bool "Miscellaneous filesystems" + default y + help + Say Y here to get to see options for various miscellaneous + filesystems, such as filesystems that came from other + operating systems. + + This option alone does not add any kernel code. + + If you say N, all options in this submenu will be skipped and + disabled; if unsure, say Y here. + +if MISC_FILESYSTEMS + +source "fs/orangefs/Kconfig" +source "fs/adfs/Kconfig" +source "fs/affs/Kconfig" +source "fs/ecryptfs/Kconfig" +source "fs/hfs/Kconfig" +source "fs/hfsplus/Kconfig" +source "fs/befs/Kconfig" +source "fs/bfs/Kconfig" +source "fs/jffs2/Kconfig" +# UBIFS File system configuration +source "fs/ubifs/Kconfig" +source "fs/cramfs/Kconfig" +source "fs/squashfs/Kconfig" +source "fs/minix/Kconfig" +source "fs/omfs/Kconfig" +source "fs/hpfs/Kconfig" +source "fs/qnx4/Kconfig" +source "fs/qnx6/Kconfig" +source "fs/resctrl/Kconfig" +source "fs/romfs/Kconfig" +source "fs/pstore/Kconfig" +source "fs/ufs/Kconfig" +source "fs/erofs/Kconfig" +source "fs/vboxsf/Kconfig" + +endif # MISC_FILESYSTEMS + +menuconfig NETWORK_FILESYSTEMS + bool "Network File Systems" + default y + depends on NET + help + Say Y here to get to see options for network filesystems and + filesystem-related networking code, such as NFS daemon and + RPCSEC security modules. + + This option alone does not add any kernel code. + + If you say N, all options in this submenu will be skipped and + disabled; if unsure, say Y here. + +if NETWORK_FILESYSTEMS + +source "fs/nfs/Kconfig" +source "fs/nfsd/Kconfig" + +config GRACE_PERIOD + tristate + +config LOCKD + tristate + depends on FILE_LOCKING + select CRC32 + select GRACE_PERIOD + +config LOCKD_V4 + bool + depends on NFSD || NFS_V3 + depends on FILE_LOCKING + default y + +config NFS_ACL_SUPPORT + tristate + select FS_POSIX_ACL + +config NFS_COMMON + bool + depends on NFSD || NFS_FS || LOCKD + default y + +config NFS_COMMON_LOCALIO_SUPPORT + tristate + depends on NFS_LOCALIO + default y if NFSD=y || NFS_FS=y + default m if NFSD=m && NFS_FS=m + select SUNRPC + +config NFS_LOCALIO + bool "NFS client and server support for LOCALIO auxiliary protocol" + depends on NFSD && NFS_FS + select NFS_COMMON_LOCALIO_SUPPORT + default n + help + Some NFS servers support an auxiliary NFS LOCALIO protocol + that is not an official part of the NFS protocol. + + This option enables support for the LOCALIO protocol in the + kernel's NFS server and client. Enable this to permit local + NFS clients to bypass the network when issuing reads and + writes to the local NFS server. + + If unsure, say N. + +config NFS_V4_2_SSC_HELPER + bool + default y if NFS_V4_2 + +source "net/sunrpc/Kconfig" +source "fs/ceph/Kconfig" + +source "fs/smb/Kconfig" +source "fs/coda/Kconfig" +source "fs/afs/Kconfig" +source "fs/9p/Kconfig" + +endif # NETWORK_FILESYSTEMS + +source "fs/nls/Kconfig" +source "fs/dlm/Kconfig" +source "fs/unicode/Kconfig" + +config IO_WQ + bool + +endmenu diff --git a/fs/Kconfig.binfmt b/fs/Kconfig.binfmt new file mode 100644 index 000000000..daeac4889 --- /dev/null +++ b/fs/Kconfig.binfmt @@ -0,0 +1,210 @@ +# SPDX-License-Identifier: GPL-2.0-only + +menu "Executable file formats" + +config BINFMT_ELF + bool "Kernel support for ELF binaries" + depends on MMU + select ELFCORE + default y + help + ELF (Executable and Linkable Format) is a format for libraries and + executables used across different architectures and operating + systems. Saying Y here will enable your kernel to run ELF binaries + and enlarge it by about 13 KB. ELF support under Linux has now all + but replaced the traditional Linux a.out formats (QMAGIC and ZMAGIC) + because it is portable (this does *not* mean that you will be able + to run executables from different architectures or operating systems + however) and makes building run-time libraries very easy. Many new + executables are distributed solely in ELF format. You definitely + want to say Y here. + + Information about ELF is contained in the ELF HOWTO available from + . + + If you find that after upgrading from Linux kernel 1.2 and saying Y + here, you still can't run any ELF binaries (they just crash), then + you'll have to install the newest ELF runtime libraries, including + ld.so (check the file for location and + latest version). + +config BINFMT_ELF_KUNIT_TEST + bool "Build KUnit tests for ELF binary support" if !KUNIT_ALL_TESTS + depends on KUNIT=y && BINFMT_ELF=y + default KUNIT_ALL_TESTS + help + This builds the ELF loader KUnit tests, which try to gather + prior bug fixes into a regression test collection. This is really + only needed for debugging. Note that with CONFIG_COMPAT=y, the + compat_binfmt_elf KUnit test is also created. + +config COMPAT_BINFMT_ELF + def_bool y + depends on COMPAT && BINFMT_ELF + select ELFCORE + +config ARCH_BINFMT_ELF_STATE + bool + +config ARCH_BINFMT_ELF_EXTRA_PHDRS + bool + +config ARCH_HAVE_ELF_PROT + bool + +config ARCH_USE_GNU_PROPERTY + bool + +config BINFMT_ELF_FDPIC + bool "Kernel support for FDPIC ELF binaries" + default y if !BINFMT_ELF + depends on ARM || ((M68K || RISCV || SUPERH || XTENSA) && !MMU) + select ELFCORE + help + ELF FDPIC binaries are based on ELF, but allow the individual load + segments of a binary to be located in memory independently of each + other. This makes this format ideal for use in environments where no + MMU is available as it still permits text segments to be shared, + even if data segments are not. + + It is also possible to run FDPIC ELF binaries on MMU linux also. + +config ELFCORE + bool + help + This option enables kernel/elfcore.o. + +config CORE_DUMP_DEFAULT_ELF_HEADERS + bool "Write ELF core dumps with partial segments" + default y + depends on BINFMT_ELF && ELF_CORE + help + ELF core dump files describe each memory mapping of the crashed + process, and can contain or omit the memory contents of each one. + The contents of an unmodified text mapping are omitted by default. + + For an unmodified text mapping of an ELF object, including just + the first page of the file in a core dump makes it possible to + identify the build ID bits in the file, without paying the i/o + cost and disk space to dump all the text. However, versions of + GDB before 6.7 are confused by ELF core dump files in this format. + + The core dump behavior can be controlled per process using + the /proc/PID/coredump_filter pseudo-file; this setting is + inherited. See Documentation/filesystems/proc.rst for details. + + This config option changes the default setting of coredump_filter + seen at boot time. If unsure, say Y. + +config BINFMT_SCRIPT + tristate "Kernel support for scripts starting with #!" + default y + help + Say Y here if you want to execute interpreted scripts starting with + #! followed by the path to an interpreter. + + You can build this support as a module; however, until that module + gets loaded, you cannot run scripts. Thus, if you want to load this + module from an initramfs, the portion of the initramfs before loading + this module must consist of compiled binaries only. + + Most systems will not boot if you say M or N here. If unsure, say Y. + +config ARCH_HAS_BINFMT_FLAT + bool + +config BINFMT_FLAT + bool "Kernel support for flat binaries" + depends on ARCH_HAS_BINFMT_FLAT + help + Support uClinux FLAT format binaries. + +config BINFMT_FLAT_ARGVP_ENVP_ON_STACK + bool + +config BINFMT_FLAT_OLD_ALWAYS_RAM + bool + +config BINFMT_FLAT_NO_DATA_START_OFFSET + bool + +config BINFMT_FLAT_OLD + bool "Enable support for very old legacy flat binaries" + depends on BINFMT_FLAT + help + Support decade old uClinux FLAT format binaries. Unless you know + you have some of those say N here. + +config BINFMT_ZFLAT + bool "Enable ZFLAT support" + depends on BINFMT_FLAT + select ZLIB_INFLATE + help + Support FLAT format compressed binaries + +config BINFMT_MISC + tristate "Kernel support for MISC binaries" + help + If you say Y here, it will be possible to plug wrapper-driven binary + formats into the kernel. You will like this especially when you use + programs that need an interpreter to run like Java, Python, .NET or + Emacs-Lisp. It's also useful if you often run DOS executables under + the Linux DOS emulator DOSEMU (read the DOSEMU-HOWTO, available from + ). Once you have + registered such a binary class with the kernel, you can start one of + those programs simply by typing in its name at a shell prompt; Linux + will automatically feed it to the correct interpreter. + + You can do other nice things, too. Read the file + to learn how to use this + feature, for information about how + to include Java support. and for + information about how to include Mono-based .NET support. + + To use binfmt_misc, you will need to mount it: + mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc + + You may say M here for module support and later load the module when + you have use for it; the module is called binfmt_misc. If you + don't know what to answer at this point, say Y. + +config BINFMT_MISC_BPF + bool "BPF-selected interpreters for misc binaries" + depends on BINFMT_MISC=y + depends on BPF_SYSCALL && BPF_JIT && DEBUG_INFO_BTF + help + Allow binfmt_misc binary type handlers to be implemented as bpf + struct_ops programs. Instead of matching a fixed magic and + redirecting to a fixed interpreter recorded at registration time + such handlers match binaries programmatically and compute the + interpreter to use per binary, e.g. relative to the location of + the binary itself. + + If you don't know what to answer at this point, say N. + +config COREDUMP + bool "Enable core dump support" if EXPERT + default y + help + This option enables support for performing core dumps. You almost + certainly want to say Y here. Not necessary on systems that never + need debugging or only ever run flawless code. + +config EXEC_KUNIT_TEST + bool "Build execve tests" if !KUNIT_ALL_TESTS + depends on KUNIT=y + default KUNIT_ALL_TESTS + help + This builds the exec KUnit tests, which tests boundary conditions + of various aspects of the exec internals. + +config ARCH_HAS_ELF_CORE_EFLAGS + bool + depends on BINFMT_ELF && ELF_CORE + default n + help + Select this option if the architecture makes use of the e_flags + field in the ELF header to store ABI or other architecture-specific + information that should be preserved in core dumps. + +endmenu diff --git a/fs/Makefile b/fs/Makefile new file mode 100644 index 000000000..055dfc23d --- /dev/null +++ b/fs/Makefile @@ -0,0 +1,130 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the Linux filesystems. +# +# 14 Sep 2000, Christoph Hellwig +# Rewritten to use lists instead of if-statements. +# + + +obj-y := open.o read_write.o file_table.o super.o \ + char_dev.o stat.o exec.o pipe.o namei.o fcntl.o \ + ioctl.o readdir.o select.o dcache.o inode.o \ + attr.o bad_inode.o file.o filesystems.o namespace.o \ + seq_file.o xattr.o libfs.o fs-writeback.o \ + pnode.o splice.o sync.o utimes.o d_path.o \ + stack.o fs_struct.o statfs.o fs_pin.o nsfs.o \ + fs_dirent.o fs_context.o fs_parser.o fsopen.o init.o \ + kernel_read_file.o mnt_idmapping.o remap_range.o pidfs.o \ + file_attr.o fserror.o nullfs.o failfs.o + +obj-$(CONFIG_BUFFER_HEAD) += buffer.o mpage.o +obj-$(CONFIG_PROC_FS) += proc_namespace.o +obj-$(CONFIG_LEGACY_DIRECT_IO) += direct-io.o +obj-y += notify/ +obj-$(CONFIG_EPOLL) += eventpoll.o +obj-y += anon_inodes.o +obj-$(CONFIG_SIGNALFD) += signalfd.o +obj-$(CONFIG_TIMERFD) += timerfd.o +obj-$(CONFIG_EVENTFD) += eventfd.o +obj-$(CONFIG_AIO) += aio.o +obj-$(CONFIG_FS_DAX) += dax.o +obj-$(CONFIG_FS_ENCRYPTION) += crypto/ +obj-$(CONFIG_FS_VERITY) += verity/ +obj-$(CONFIG_FILE_LOCKING) += locks.o +obj-$(CONFIG_BINFMT_MISC) += binfmt_misc.o +obj-$(CONFIG_BINFMT_MISC_BPF) += binfmt_misc_bpf.o +obj-$(CONFIG_BINFMT_SCRIPT) += binfmt_script.o +obj-$(CONFIG_BINFMT_ELF) += binfmt_elf.o +obj-$(CONFIG_COMPAT_BINFMT_ELF) += compat_binfmt_elf.o +obj-$(CONFIG_BINFMT_ELF_FDPIC) += binfmt_elf_fdpic.o +obj-$(CONFIG_BINFMT_FLAT) += binfmt_flat.o + +obj-$(CONFIG_FS_STACK) += backing-file.o +obj-$(CONFIG_FS_MBCACHE) += mbcache.o +obj-$(CONFIG_FS_POSIX_ACL) += posix_acl.o +obj-$(CONFIG_NFS_COMMON) += nfs_common/ +obj-$(CONFIG_COREDUMP) += coredump.o +obj-$(CONFIG_SYSCTL) += drop_caches.o sysctls.o + +obj-$(CONFIG_FHANDLE) += fhandle.o +obj-y += iomap/ + +obj-y += quota/ + +obj-$(CONFIG_PROC_FS) += proc/ +obj-$(CONFIG_KERNFS) += kernfs/ +obj-$(CONFIG_SYSFS) += sysfs/ +obj-$(CONFIG_CONFIGFS_FS) += configfs/ +obj-y += devpts/ + +obj-$(CONFIG_DLM) += dlm/ + +# Do not add any filesystems before this line +obj-$(CONFIG_NETFS_SUPPORT) += netfs/ +obj-$(CONFIG_EXT4_FS) += ext4/ +# We place ext4 before ext2 so that clean ext3 root fs's do NOT mount using the +# ext2 driver, which doesn't know about journalling! Explicitly request ext2 +# by giving the rootfstype= parameter. +obj-$(CONFIG_EXT2_FS) += ext2/ +obj-$(CONFIG_JBD2) += jbd2/ +obj-$(CONFIG_CRAMFS) += cramfs/ +obj-$(CONFIG_SQUASHFS) += squashfs/ +obj-y += ramfs/ +obj-$(CONFIG_HUGETLBFS) += hugetlbfs/ +obj-$(CONFIG_CODA_FS) += coda/ +obj-$(CONFIG_MINIX_FS) += minix/ +obj-$(CONFIG_FAT_FS) += fat/ +obj-$(CONFIG_EXFAT_FS) += exfat/ +obj-$(CONFIG_BFS_FS) += bfs/ +obj-$(CONFIG_ISO9660_FS) += isofs/ +obj-$(CONFIG_HFSPLUS_FS) += hfsplus/ # Before hfs to find wrapped HFS+ +obj-$(CONFIG_HFS_FS) += hfs/ +obj-$(CONFIG_ECRYPT_FS) += ecryptfs/ +obj-$(CONFIG_NFS_FS) += nfs/ +obj-$(CONFIG_EXPORTFS) += exportfs/ +obj-$(CONFIG_NFSD) += nfsd/ +obj-$(CONFIG_LOCKD) += lockd/ +obj-$(CONFIG_NLS) += nls/ +obj-y += unicode/ +obj-$(CONFIG_SMBFS) += smb/ +obj-$(CONFIG_HPFS_FS) += hpfs/ +obj-$(CONFIG_NTFS_FS) += ntfs/ +obj-$(CONFIG_NTFS3_FS) += ntfs3/ +obj-$(CONFIG_UFS_FS) += ufs/ +obj-$(CONFIG_JFFS2_FS) += jffs2/ +obj-$(CONFIG_UBIFS_FS) += ubifs/ +obj-$(CONFIG_AFFS_FS) += affs/ +obj-$(CONFIG_ROMFS_FS) += romfs/ +obj-$(CONFIG_QNX4FS_FS) += qnx4/ +obj-$(CONFIG_QNX6FS_FS) += qnx6/ +obj-$(CONFIG_AUTOFS_FS) += autofs/ +obj-$(CONFIG_ADFS_FS) += adfs/ +obj-$(CONFIG_FUSE_FS) += fuse/ +obj-$(CONFIG_OVERLAY_FS) += overlayfs/ +obj-$(CONFIG_ORANGEFS_FS) += orangefs/ +obj-$(CONFIG_UDF_FS) += udf/ +obj-$(CONFIG_SUN_OPENPROMFS) += openpromfs/ +obj-$(CONFIG_OMFS_FS) += omfs/ +obj-$(CONFIG_JFS_FS) += jfs/ +obj-$(CONFIG_XFS_FS) += xfs/ +obj-$(CONFIG_9P_FS) += 9p/ +obj-$(CONFIG_AFS_FS) += afs/ +obj-$(CONFIG_NILFS2_FS) += nilfs2/ +obj-$(CONFIG_BEFS_FS) += befs/ +obj-y += hostfs/ +obj-$(CONFIG_CACHEFILES) += cachefiles/ +obj-$(CONFIG_DEBUG_FS) += debugfs/ +obj-$(CONFIG_TRACING) += tracefs/ +obj-$(CONFIG_OCFS2_FS) += ocfs2/ +obj-$(CONFIG_BTRFS_FS) += btrfs/ +obj-$(CONFIG_GFS2_FS) += gfs2/ +obj-$(CONFIG_F2FS_FS) += f2fs/ +obj-$(CONFIG_CEPH_FS) += ceph/ +obj-$(CONFIG_PSTORE) += pstore/ +obj-$(CONFIG_EFIVAR_FS) += efivarfs/ +obj-$(CONFIG_EROFS_FS) += erofs/ +obj-$(CONFIG_VBOXSF_FS) += vboxsf/ +obj-$(CONFIG_ZONEFS_FS) += zonefs/ +obj-$(CONFIG_BPF_LSM) += bpf_fs_kfuncs.o +obj-$(CONFIG_RESCTRL_FS) += resctrl/ diff --git a/fs/adfs/Kconfig b/fs/adfs/Kconfig new file mode 100644 index 000000000..1b97058f0 --- /dev/null +++ b/fs/adfs/Kconfig @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: GPL-2.0-only +config ADFS_FS + tristate "ADFS file system support" + depends on BLOCK + select BUFFER_HEAD + help + The Acorn Disc Filing System is the standard file system of the + RiscOS operating system which runs on Acorn's ARM-based Risc PC + systems and the Acorn Archimedes range of machines. If you say Y + here, Linux will be able to read from ADFS partitions on hard drives + and from ADFS-formatted floppy discs. If you also want to be able to + write to those devices, say Y to "ADFS write support" below. + + The ADFS partition should be the first partition (i.e., + /dev/[hs]d?1) on each of your drives. Please read the file + for further details. + + To compile this code as a module, choose M here: the module will be + called adfs. + + If unsure, say N. + +config ADFS_FS_RW + bool "ADFS write support (DANGEROUS)" + depends on ADFS_FS + help + If you say Y here, you will be able to write to ADFS partitions on + hard drives and ADFS-formatted floppy disks. This is experimental + codes, so if you're unsure, say N. diff --git a/fs/adfs/Makefile b/fs/adfs/Makefile new file mode 100644 index 000000000..cf7de6ece --- /dev/null +++ b/fs/adfs/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the linux adfs filesystem routines. +# + +obj-$(CONFIG_ADFS_FS) += adfs.o + +adfs-objs := dir.o dir_f.o dir_fplus.o file.o inode.o map.o super.o diff --git a/fs/adfs/adfs.h b/fs/adfs/adfs.h new file mode 100644 index 000000000..0d32b7cd9 --- /dev/null +++ b/fs/adfs/adfs.h @@ -0,0 +1,230 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include +#include + +/* Internal data structures for ADFS */ + +#define ADFS_FREE_FRAG 0 +#define ADFS_BAD_FRAG 1 +#define ADFS_ROOT_FRAG 2 + +#define ADFS_FILETYPE_NONE ((u16)~0) + +/* RISC OS 12-bit filetype is stored in load_address[19:8] */ +static inline u16 adfs_filetype(u32 loadaddr) +{ + return (loadaddr & 0xfff00000) == 0xfff00000 ? + (loadaddr >> 8) & 0xfff : ADFS_FILETYPE_NONE; +} + +#define ADFS_NDA_OWNER_READ (1 << 0) +#define ADFS_NDA_OWNER_WRITE (1 << 1) +#define ADFS_NDA_LOCKED (1 << 2) +#define ADFS_NDA_DIRECTORY (1 << 3) +#define ADFS_NDA_EXECUTE (1 << 4) +#define ADFS_NDA_PUBLIC_READ (1 << 5) +#define ADFS_NDA_PUBLIC_WRITE (1 << 6) + +/* + * adfs file system inode data in memory + */ +struct adfs_inode_info { + loff_t mmu_private; + __u32 parent_id; /* parent indirect disc address */ + __u32 indaddr; /* object indirect disc address */ + __u32 loadaddr; /* RISC OS load address */ + __u32 execaddr; /* RISC OS exec address */ + unsigned int attr; /* RISC OS permissions */ + struct inode vfs_inode; +}; + +static inline struct adfs_inode_info *ADFS_I(struct inode *inode) +{ + return container_of(inode, struct adfs_inode_info, vfs_inode); +} + +static inline bool adfs_inode_is_stamped(struct inode *inode) +{ + return (ADFS_I(inode)->loadaddr & 0xfff00000) == 0xfff00000; +} + +/* + * Forward-declare this + */ +struct adfs_discmap; +struct adfs_dir_ops; + +/* + * ADFS file system superblock data in memory + */ +struct adfs_sb_info { + union { struct { + struct adfs_discmap *s_map; /* bh list containing map */ + const struct adfs_dir_ops *s_dir; /* directory operations */ + }; + struct rcu_head rcu; /* used only at shutdown time */ + }; + kuid_t s_uid; /* owner uid */ + kgid_t s_gid; /* owner gid */ + umode_t s_owner_mask; /* ADFS owner perm -> unix perm */ + umode_t s_other_mask; /* ADFS other perm -> unix perm */ + int s_ftsuffix; /* ,xyz hex filetype suffix option */ + + __u32 s_ids_per_zone; /* max. no ids in one zone */ + __u32 s_idlen; /* length of ID in map */ + __u32 s_map_size; /* sector size of a map */ + signed int s_map2blk; /* shift left by this for map->sector*/ + unsigned int s_log2sharesize;/* log2 share size */ + unsigned int s_namelen; /* maximum number of characters in name */ +}; + +static inline struct adfs_sb_info *ADFS_SB(struct super_block *sb) +{ + return sb->s_fs_info; +} + +/* + * Directory handling + */ +struct adfs_dir { + struct super_block *sb; + + int nr_buffers; + struct buffer_head *bh[4]; + struct buffer_head **bhs; + + unsigned int pos; + __u32 parent_id; + + union { + struct adfs_dirheader *dirhead; + struct adfs_bigdirheader *bighead; + }; + union { + struct adfs_newdirtail *newtail; + struct adfs_bigdirtail *bigtail; + }; +}; + +/* + * This is the overall maximum name length + */ +#define ADFS_MAX_NAME_LEN (256 + 4) /* +4 for ,xyz hex filetype suffix */ +struct object_info { + __u32 parent_id; /* parent object id */ + __u32 indaddr; /* indirect disc addr */ + __u32 loadaddr; /* load address */ + __u32 execaddr; /* execution address */ + __u32 size; /* size */ + __u8 attr; /* RISC OS attributes */ + unsigned int name_len; /* name length */ + char name[ADFS_MAX_NAME_LEN];/* file name */ +}; + +struct adfs_dir_ops { + int (*read)(struct super_block *sb, unsigned int indaddr, + unsigned int size, struct adfs_dir *dir); + int (*iterate)(struct adfs_dir *dir, struct dir_context *ctx); + int (*setpos)(struct adfs_dir *dir, unsigned int fpos); + int (*getnext)(struct adfs_dir *dir, struct object_info *obj); + int (*update)(struct adfs_dir *dir, struct object_info *obj); + int (*create)(struct adfs_dir *dir, struct object_info *obj); + int (*remove)(struct adfs_dir *dir, struct object_info *obj); + int (*commit)(struct adfs_dir *dir); +}; + +struct adfs_discmap { + struct buffer_head *dm_bh; + __u32 dm_startblk; + unsigned int dm_startbit; + unsigned int dm_endbit; +}; + +/* Inode stuff */ +struct inode *adfs_iget(struct super_block *sb, struct object_info *obj); +int adfs_write_inode(struct inode *inode, struct writeback_control *wbc); +int adfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr); + +/* map.c */ +int adfs_map_lookup(struct super_block *sb, u32 frag_id, unsigned int offset); +void adfs_map_statfs(struct super_block *sb, struct kstatfs *buf); +struct adfs_discmap *adfs_read_map(struct super_block *sb, struct adfs_discrecord *dr); +void adfs_free_map(struct super_block *sb); + +/* Misc */ +__printf(3, 4) +void __adfs_error(struct super_block *sb, const char *function, + const char *fmt, ...); +#define adfs_error(sb, fmt...) __adfs_error(sb, __func__, fmt) +void adfs_msg(struct super_block *sb, const char *pfx, const char *fmt, ...); + +/* super.c */ + +/* + * Inodes and file operations + */ + +/* dir_*.c */ +extern const struct inode_operations adfs_dir_inode_operations; +extern const struct file_operations adfs_dir_operations; +extern const struct dentry_operations adfs_dentry_operations; +extern const struct adfs_dir_ops adfs_f_dir_ops; +extern const struct adfs_dir_ops adfs_fplus_dir_ops; + +int adfs_dir_copyfrom(void *dst, struct adfs_dir *dir, unsigned int offset, + size_t len); +int adfs_dir_copyto(struct adfs_dir *dir, unsigned int offset, const void *src, + size_t len); +void adfs_dir_relse(struct adfs_dir *dir); +int adfs_dir_read_buffers(struct super_block *sb, u32 indaddr, + unsigned int size, struct adfs_dir *dir); +void adfs_object_fixup(struct adfs_dir *dir, struct object_info *obj); +extern int adfs_dir_update(struct super_block *sb, struct object_info *obj, + int wait); + +/* file.c */ +extern const struct inode_operations adfs_file_inode_operations; +extern const struct file_operations adfs_file_operations; + +static inline __u32 signed_asl(__u32 val, signed int shift) +{ + if (shift >= 0) + val <<= shift; + else + val >>= -shift; + return val; +} + +/* + * Calculate the address of a block in an object given the block offset + * and the object identity. + * + * The root directory ID should always be looked up in the map [3.4] + */ +static inline int __adfs_block_map(struct super_block *sb, u32 indaddr, + unsigned int block) +{ + if (indaddr & 255) { + unsigned int off; + + off = (indaddr & 255) - 1; + block += off << ADFS_SB(sb)->s_log2sharesize; + } + + return adfs_map_lookup(sb, indaddr >> 8, block); +} + +/* Return the disc record from the map */ +static inline +struct adfs_discrecord *adfs_map_discrecord(struct adfs_discmap *dm) +{ + return (void *)(dm[0].dm_bh->b_data + 4); +} + +static inline u64 adfs_disc_size(const struct adfs_discrecord *dr) +{ + return (u64)le32_to_cpu(dr->disc_size_high) << 32 | + le32_to_cpu(dr->disc_size); +} diff --git a/fs/adfs/dir.c b/fs/adfs/dir.c new file mode 100644 index 000000000..11afa9e15 --- /dev/null +++ b/fs/adfs/dir.c @@ -0,0 +1,458 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/adfs/dir.c + * + * Copyright (C) 1999-2000 Russell King + * + * Common directory handling for ADFS + */ +#include +#include +#include "adfs.h" + +/* + * For future. This should probably be per-directory. + */ +static DECLARE_RWSEM(adfs_dir_rwsem); + +int adfs_dir_copyfrom(void *dst, struct adfs_dir *dir, unsigned int offset, + size_t len) +{ + struct super_block *sb = dir->sb; + unsigned int index, remain; + + index = offset >> sb->s_blocksize_bits; + offset &= sb->s_blocksize - 1; + remain = sb->s_blocksize - offset; + if (index + (remain < len) >= dir->nr_buffers) + return -EINVAL; + + if (remain < len) { + memcpy(dst, dir->bhs[index]->b_data + offset, remain); + dst += remain; + len -= remain; + index += 1; + offset = 0; + } + + memcpy(dst, dir->bhs[index]->b_data + offset, len); + + return 0; +} + +int adfs_dir_copyto(struct adfs_dir *dir, unsigned int offset, const void *src, + size_t len) +{ + struct super_block *sb = dir->sb; + unsigned int index, remain; + + index = offset >> sb->s_blocksize_bits; + offset &= sb->s_blocksize - 1; + remain = sb->s_blocksize - offset; + if (index + (remain < len) >= dir->nr_buffers) + return -EINVAL; + + if (remain < len) { + memcpy(dir->bhs[index]->b_data + offset, src, remain); + src += remain; + len -= remain; + index += 1; + offset = 0; + } + + memcpy(dir->bhs[index]->b_data + offset, src, len); + + return 0; +} + +static void __adfs_dir_cleanup(struct adfs_dir *dir) +{ + dir->nr_buffers = 0; + + if (dir->bhs != dir->bh) + kfree(dir->bhs); + dir->bhs = NULL; + dir->sb = NULL; +} + +void adfs_dir_relse(struct adfs_dir *dir) +{ + unsigned int i; + + for (i = 0; i < dir->nr_buffers; i++) + brelse(dir->bhs[i]); + + __adfs_dir_cleanup(dir); +} + +static void adfs_dir_forget(struct adfs_dir *dir) +{ + unsigned int i; + + for (i = 0; i < dir->nr_buffers; i++) + bforget(dir->bhs[i]); + + __adfs_dir_cleanup(dir); +} + +int adfs_dir_read_buffers(struct super_block *sb, u32 indaddr, + unsigned int size, struct adfs_dir *dir) +{ + struct buffer_head **bhs; + unsigned int i, num; + int block; + + num = ALIGN(size, sb->s_blocksize) >> sb->s_blocksize_bits; + if (num > ARRAY_SIZE(dir->bh)) { + /* We only allow one extension */ + if (dir->bhs != dir->bh) + return -EINVAL; + + bhs = kzalloc_objs(*bhs, num); + if (!bhs) + return -ENOMEM; + + if (dir->nr_buffers) + memcpy(bhs, dir->bhs, dir->nr_buffers * sizeof(*bhs)); + + dir->bhs = bhs; + } + + for (i = dir->nr_buffers; i < num; i++) { + block = __adfs_block_map(sb, indaddr, i); + if (!block) { + adfs_error(sb, "dir %06x has a hole at offset %u", + indaddr, i); + goto error; + } + + dir->bhs[i] = sb_bread(sb, block); + if (!dir->bhs[i]) { + adfs_error(sb, + "dir %06x failed read at offset %u, mapped block 0x%08x", + indaddr, i, block); + goto error; + } + + dir->nr_buffers++; + } + return 0; + +error: + adfs_dir_relse(dir); + + return -EIO; +} + +static int adfs_dir_read(struct super_block *sb, u32 indaddr, + unsigned int size, struct adfs_dir *dir) +{ + dir->sb = sb; + dir->bhs = dir->bh; + dir->nr_buffers = 0; + + return ADFS_SB(sb)->s_dir->read(sb, indaddr, size, dir); +} + +static int adfs_dir_read_inode(struct super_block *sb, struct inode *inode, + struct adfs_dir *dir) +{ + int ret; + + ret = adfs_dir_read(sb, ADFS_I(inode)->indaddr, inode->i_size, dir); + if (ret) + return ret; + + if (ADFS_I(inode)->parent_id != dir->parent_id) { + adfs_error(sb, + "parent directory id changed under me! (%06x but got %06x)\n", + ADFS_I(inode)->parent_id, dir->parent_id); + adfs_dir_relse(dir); + ret = -EIO; + } + + return ret; +} + +static void adfs_dir_mark_dirty(struct adfs_dir *dir) +{ + unsigned int i; + + /* Mark the buffers dirty */ + for (i = 0; i < dir->nr_buffers; i++) + mark_buffer_dirty(dir->bhs[i]); +} + +static int adfs_dir_sync(struct adfs_dir *dir) +{ + int err = 0; + int i; + + for (i = dir->nr_buffers - 1; i >= 0; i--) { + struct buffer_head *bh = dir->bhs[i]; + sync_dirty_buffer(bh); + if (buffer_req(bh) && !buffer_uptodate(bh)) + err = -EIO; + } + + return err; +} + +void adfs_object_fixup(struct adfs_dir *dir, struct object_info *obj) +{ + unsigned int dots, i; + + /* + * RISC OS allows the use of '/' in directory entry names, so we need + * to fix these up. '/' is typically used for FAT compatibility to + * represent '.', so do the same conversion here. In any case, '.' + * will never be in a RISC OS name since it is used as the pathname + * separator. Handle the case where we may generate a '.' or '..' + * name, replacing the first character with '^' (the RISC OS "parent + * directory" character.) + */ + for (i = dots = 0; i < obj->name_len; i++) + if (obj->name[i] == '/') { + obj->name[i] = '.'; + dots++; + } + + if (obj->name_len <= 2 && dots == obj->name_len) + obj->name[0] = '^'; + + /* + * If the object is a file, and the user requested the ,xyz hex + * filetype suffix to the name, check the filetype and append. + */ + if (!(obj->attr & ADFS_NDA_DIRECTORY) && ADFS_SB(dir->sb)->s_ftsuffix) { + u16 filetype = adfs_filetype(obj->loadaddr); + + if (filetype != ADFS_FILETYPE_NONE) { + obj->name[obj->name_len++] = ','; + obj->name[obj->name_len++] = hex_asc_lo(filetype >> 8); + obj->name[obj->name_len++] = hex_asc_lo(filetype >> 4); + obj->name[obj->name_len++] = hex_asc_lo(filetype >> 0); + } + } +} + +static int adfs_iterate(struct file *file, struct dir_context *ctx) +{ + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + const struct adfs_dir_ops *ops = ADFS_SB(sb)->s_dir; + struct adfs_dir dir; + int ret; + + down_read(&adfs_dir_rwsem); + ret = adfs_dir_read_inode(sb, inode, &dir); + if (ret) + goto unlock; + + if (ctx->pos == 0) { + if (!dir_emit_dot(file, ctx)) + goto unlock_relse; + ctx->pos = 1; + } + if (ctx->pos == 1) { + if (!dir_emit(ctx, "..", 2, dir.parent_id, DT_DIR)) + goto unlock_relse; + ctx->pos = 2; + } + + ret = ops->iterate(&dir, ctx); + +unlock_relse: + up_read(&adfs_dir_rwsem); + adfs_dir_relse(&dir); + return ret; + +unlock: + up_read(&adfs_dir_rwsem); + return ret; +} + +int +adfs_dir_update(struct super_block *sb, struct object_info *obj, int wait) +{ + const struct adfs_dir_ops *ops = ADFS_SB(sb)->s_dir; + struct adfs_dir dir; + int ret; + + if (!IS_ENABLED(CONFIG_ADFS_FS_RW)) + return -EINVAL; + + if (!ops->update) + return -EINVAL; + + down_write(&adfs_dir_rwsem); + ret = adfs_dir_read(sb, obj->parent_id, 0, &dir); + if (ret) + goto unlock; + + ret = ops->update(&dir, obj); + if (ret) + goto forget; + + ret = ops->commit(&dir); + if (ret) + goto forget; + up_write(&adfs_dir_rwsem); + + adfs_dir_mark_dirty(&dir); + + if (wait) + ret = adfs_dir_sync(&dir); + + adfs_dir_relse(&dir); + return ret; + + /* + * If the updated failed because the entry wasn't found, we can + * just release the buffers. If it was any other error, forget + * the dirtied buffers so they aren't written back to the media. + */ +forget: + if (ret == -ENOENT) + adfs_dir_relse(&dir); + else + adfs_dir_forget(&dir); +unlock: + up_write(&adfs_dir_rwsem); + + return ret; +} + +static unsigned char adfs_tolower(unsigned char c) +{ + if (c >= 'A' && c <= 'Z') + c += 'a' - 'A'; + return c; +} + +static int __adfs_compare(const unsigned char *qstr, u32 qlen, + const char *str, u32 len) +{ + u32 i; + + if (qlen != len) + return 1; + + for (i = 0; i < qlen; i++) + if (adfs_tolower(qstr[i]) != adfs_tolower(str[i])) + return 1; + + return 0; +} + +static int adfs_dir_lookup_byname(struct inode *inode, const struct qstr *qstr, + struct object_info *obj) +{ + struct super_block *sb = inode->i_sb; + const struct adfs_dir_ops *ops = ADFS_SB(sb)->s_dir; + const unsigned char *name; + struct adfs_dir dir; + u32 name_len; + int ret; + + down_read(&adfs_dir_rwsem); + ret = adfs_dir_read_inode(sb, inode, &dir); + if (ret) + goto unlock; + + ret = ops->setpos(&dir, 0); + if (ret) + goto unlock_relse; + + ret = -ENOENT; + name = qstr->name; + name_len = qstr->len; + while (ops->getnext(&dir, obj) == 0) { + if (!__adfs_compare(name, name_len, obj->name, obj->name_len)) { + ret = 0; + break; + } + } + obj->parent_id = ADFS_I(inode)->indaddr; + +unlock_relse: + up_read(&adfs_dir_rwsem); + adfs_dir_relse(&dir); + return ret; + +unlock: + up_read(&adfs_dir_rwsem); + return ret; +} + +const struct file_operations adfs_dir_operations = { + .read = generic_read_dir, + .llseek = generic_file_llseek, + .iterate_shared = adfs_iterate, + .fsync = simple_fsync, +}; + +static int +adfs_hash(const struct dentry *parent, struct qstr *qstr) +{ + const unsigned char *name; + unsigned long hash; + u32 len; + + if (qstr->len > ADFS_SB(parent->d_sb)->s_namelen) + return -ENAMETOOLONG; + + len = qstr->len; + name = qstr->name; + hash = init_name_hash(parent); + while (len--) + hash = partial_name_hash(adfs_tolower(*name++), hash); + qstr->hash = end_name_hash(hash); + + return 0; +} + +/* + * Compare two names, taking note of the name length + * requirements of the underlying filesystem. + */ +static int adfs_compare(const struct dentry *dentry, unsigned int len, + const char *str, const struct qstr *qstr) +{ + return __adfs_compare(qstr->name, qstr->len, str, len); +} + +const struct dentry_operations adfs_dentry_operations = { + .d_hash = adfs_hash, + .d_compare = adfs_compare, +}; + +static struct dentry * +adfs_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) +{ + struct inode *inode = NULL; + struct object_info obj; + int error; + + error = adfs_dir_lookup_byname(dir, &dentry->d_name, &obj); + if (error == 0) { + /* + * This only returns NULL if get_empty_inode + * fails. + */ + inode = adfs_iget(dir->i_sb, &obj); + if (!inode) + inode = ERR_PTR(-EACCES); + } else if (error != -ENOENT) { + inode = ERR_PTR(error); + } + return d_splice_alias(inode, dentry); +} + +/* + * directories can handle most operations... + */ +const struct inode_operations adfs_dir_inode_operations = { + .lookup = adfs_lookup, + .setattr = adfs_setattr, +}; diff --git a/fs/adfs/dir_f.c b/fs/adfs/dir_f.c new file mode 100644 index 000000000..05e963402 --- /dev/null +++ b/fs/adfs/dir_f.c @@ -0,0 +1,324 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/adfs/dir_f.c + * + * Copyright (C) 1997-1999 Russell King + * + * E and F format directory handling + */ +#include "adfs.h" +#include "dir_f.h" + +/* + * Read an (unaligned) value of length 1..4 bytes + */ +static inline unsigned int adfs_readval(unsigned char *p, int len) +{ + unsigned int val = 0; + + switch (len) { + case 4: val |= p[3] << 24; + fallthrough; + case 3: val |= p[2] << 16; + fallthrough; + case 2: val |= p[1] << 8; + fallthrough; + default: val |= p[0]; + } + return val; +} + +static inline void adfs_writeval(unsigned char *p, int len, unsigned int val) +{ + switch (len) { + case 4: p[3] = val >> 24; + fallthrough; + case 3: p[2] = val >> 16; + fallthrough; + case 2: p[1] = val >> 8; + fallthrough; + default: p[0] = val; + } +} + +#define ror13(v) ((v >> 13) | (v << 19)) + +#define dir_u8(idx) \ + ({ int _buf = idx >> blocksize_bits; \ + int _off = idx - (_buf << blocksize_bits);\ + *(u8 *)(bh[_buf]->b_data + _off); \ + }) + +#define dir_u32(idx) \ + ({ int _buf = idx >> blocksize_bits; \ + int _off = idx - (_buf << blocksize_bits);\ + *(__le32 *)(bh[_buf]->b_data + _off); \ + }) + +#define bufoff(_bh,_idx) \ + ({ int _buf = _idx >> blocksize_bits; \ + int _off = _idx - (_buf << blocksize_bits);\ + (void *)(_bh[_buf]->b_data + _off); \ + }) + +/* + * There are some algorithms that are nice in + * assembler, but a bitch in C... This is one + * of them. + */ +static u8 +adfs_dir_checkbyte(const struct adfs_dir *dir) +{ + struct buffer_head * const *bh = dir->bh; + const int blocksize_bits = dir->sb->s_blocksize_bits; + union { __le32 *ptr32; u8 *ptr8; } ptr, end; + u32 dircheck = 0; + int last = 5 - 26; + int i = 0; + + /* + * Accumulate each word up to the last whole + * word of the last directory entry. This + * can spread across several buffer heads. + */ + do { + last += 26; + do { + dircheck = le32_to_cpu(dir_u32(i)) ^ ror13(dircheck); + + i += sizeof(u32); + } while (i < (last & ~3)); + } while (dir_u8(last) != 0); + + /* + * Accumulate the last few bytes. These + * bytes will be within the same bh. + */ + if (i != last) { + ptr.ptr8 = bufoff(bh, i); + end.ptr8 = ptr.ptr8 + last - i; + + do { + dircheck = *ptr.ptr8++ ^ ror13(dircheck); + } while (ptr.ptr8 < end.ptr8); + } + + /* + * The directory tail is in the final bh + * Note that contary to the RISC OS PRMs, + * the first few bytes are NOT included + * in the check. All bytes are in the + * same bh. + */ + ptr.ptr8 = bufoff(bh, 2008); + end.ptr8 = ptr.ptr8 + 36; + + do { + __le32 v = *ptr.ptr32++; + dircheck = le32_to_cpu(v) ^ ror13(dircheck); + } while (ptr.ptr32 < end.ptr32); + + return (dircheck ^ (dircheck >> 8) ^ (dircheck >> 16) ^ (dircheck >> 24)) & 0xff; +} + +static int adfs_f_validate(struct adfs_dir *dir) +{ + struct adfs_dirheader *head = dir->dirhead; + struct adfs_newdirtail *tail = dir->newtail; + + if (head->startmasseq != tail->endmasseq || + tail->dirlastmask || tail->reserved[0] || tail->reserved[1] || + (memcmp(&head->startname, "Nick", 4) && + memcmp(&head->startname, "Hugo", 4)) || + memcmp(&head->startname, &tail->endname, 4) || + adfs_dir_checkbyte(dir) != tail->dircheckbyte) + return -EIO; + + return 0; +} + +/* Read and check that a directory is valid */ +static int adfs_f_read(struct super_block *sb, u32 indaddr, unsigned int size, + struct adfs_dir *dir) +{ + const unsigned int blocksize_bits = sb->s_blocksize_bits; + int ret; + + if (size && size != ADFS_NEWDIR_SIZE) + return -EIO; + + ret = adfs_dir_read_buffers(sb, indaddr, ADFS_NEWDIR_SIZE, dir); + if (ret) + return ret; + + dir->dirhead = bufoff(dir->bh, 0); + dir->newtail = bufoff(dir->bh, 2007); + + if (adfs_f_validate(dir)) + goto bad_dir; + + dir->parent_id = adfs_readval(dir->newtail->dirparent, 3); + + return 0; + +bad_dir: + adfs_error(sb, "dir %06x is corrupted", indaddr); + adfs_dir_relse(dir); + + return -EIO; +} + +/* + * convert a disk-based directory entry to a Linux ADFS directory entry + */ +static inline void +adfs_dir2obj(struct adfs_dir *dir, struct object_info *obj, + struct adfs_direntry *de) +{ + unsigned int name_len; + + for (name_len = 0; name_len < ADFS_F_NAME_LEN; name_len++) { + if (de->dirobname[name_len] < ' ') + break; + + obj->name[name_len] = de->dirobname[name_len]; + } + + obj->name_len = name_len; + obj->indaddr = adfs_readval(de->dirinddiscadd, 3); + obj->loadaddr = adfs_readval(de->dirload, 4); + obj->execaddr = adfs_readval(de->direxec, 4); + obj->size = adfs_readval(de->dirlen, 4); + obj->attr = de->newdiratts; + + adfs_object_fixup(dir, obj); +} + +/* + * convert a Linux ADFS directory entry to a disk-based directory entry + */ +static inline void +adfs_obj2dir(struct adfs_direntry *de, struct object_info *obj) +{ + adfs_writeval(de->dirinddiscadd, 3, obj->indaddr); + adfs_writeval(de->dirload, 4, obj->loadaddr); + adfs_writeval(de->direxec, 4, obj->execaddr); + adfs_writeval(de->dirlen, 4, obj->size); + de->newdiratts = obj->attr; +} + +/* + * get a directory entry. Note that the caller is responsible + * for holding the relevant locks. + */ +static int +__adfs_dir_get(struct adfs_dir *dir, int pos, struct object_info *obj) +{ + struct adfs_direntry de; + int ret; + + ret = adfs_dir_copyfrom(&de, dir, pos, 26); + if (ret) + return ret; + + if (!de.dirobname[0]) + return -ENOENT; + + adfs_dir2obj(dir, obj, &de); + + return 0; +} + +static int +adfs_f_setpos(struct adfs_dir *dir, unsigned int fpos) +{ + if (fpos >= ADFS_NUM_DIR_ENTRIES) + return -ENOENT; + + dir->pos = 5 + fpos * 26; + return 0; +} + +static int +adfs_f_getnext(struct adfs_dir *dir, struct object_info *obj) +{ + unsigned int ret; + + ret = __adfs_dir_get(dir, dir->pos, obj); + if (ret == 0) + dir->pos += 26; + + return ret; +} + +static int adfs_f_iterate(struct adfs_dir *dir, struct dir_context *ctx) +{ + struct object_info obj; + int pos = 5 + (ctx->pos - 2) * 26; + + while (ctx->pos < 2 + ADFS_NUM_DIR_ENTRIES) { + if (__adfs_dir_get(dir, pos, &obj)) + break; + if (!dir_emit(ctx, obj.name, obj.name_len, + obj.indaddr, DT_UNKNOWN)) + break; + pos += 26; + ctx->pos++; + } + return 0; +} + +static int adfs_f_update(struct adfs_dir *dir, struct object_info *obj) +{ + struct adfs_direntry de; + int offset, ret; + + offset = 5 - (int)sizeof(de); + + do { + offset += sizeof(de); + ret = adfs_dir_copyfrom(&de, dir, offset, sizeof(de)); + if (ret) { + adfs_error(dir->sb, "error reading directory entry"); + return -ENOENT; + } + if (!de.dirobname[0]) { + adfs_error(dir->sb, "unable to locate entry to update"); + return -ENOENT; + } + } while (adfs_readval(de.dirinddiscadd, 3) != obj->indaddr); + + /* Update the directory entry with the new object state */ + adfs_obj2dir(&de, obj); + + /* Write the directory entry back to the directory */ + return adfs_dir_copyto(dir, offset, &de, 26); +} + +static int adfs_f_commit(struct adfs_dir *dir) +{ + int ret; + + /* Increment directory sequence number */ + dir->dirhead->startmasseq += 1; + dir->newtail->endmasseq += 1; + + /* Update directory check byte */ + dir->newtail->dircheckbyte = adfs_dir_checkbyte(dir); + + /* Make sure the directory still validates correctly */ + ret = adfs_f_validate(dir); + if (ret) + adfs_msg(dir->sb, KERN_ERR, "error: update broke directory"); + + return ret; +} + +const struct adfs_dir_ops adfs_f_dir_ops = { + .read = adfs_f_read, + .iterate = adfs_f_iterate, + .setpos = adfs_f_setpos, + .getnext = adfs_f_getnext, + .update = adfs_f_update, + .commit = adfs_f_commit, +}; diff --git a/fs/adfs/dir_f.h b/fs/adfs/dir_f.h new file mode 100644 index 000000000..4e6c53d59 --- /dev/null +++ b/fs/adfs/dir_f.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * linux/fs/adfs/dir_f.h + * + * Copyright (C) 1999 Russell King + * + * Structures of directories on the F format disk + */ +#ifndef ADFS_DIR_F_H +#define ADFS_DIR_F_H + +/* + * Directory header + */ +struct adfs_dirheader { + __u8 startmasseq; + __u8 startname[4]; +} __attribute__((packed)); + +#define ADFS_NEWDIR_SIZE 2048 +#define ADFS_NUM_DIR_ENTRIES 77 + +/* + * Directory entries + */ +struct adfs_direntry { +#define ADFS_F_NAME_LEN 10 + char dirobname[ADFS_F_NAME_LEN]; + __u8 dirload[4]; + __u8 direxec[4]; + __u8 dirlen[4]; + __u8 dirinddiscadd[3]; + __u8 newdiratts; +} __attribute__((packed)); + +/* + * Directory tail + */ +struct adfs_olddirtail { + __u8 dirlastmask; + char dirname[10]; + __u8 dirparent[3]; + char dirtitle[19]; + __u8 reserved[14]; + __u8 endmasseq; + __u8 endname[4]; + __u8 dircheckbyte; +} __attribute__((packed)); + +struct adfs_newdirtail { + __u8 dirlastmask; + __u8 reserved[2]; + __u8 dirparent[3]; + char dirtitle[19]; + char dirname[10]; + __u8 endmasseq; + __u8 endname[4]; + __u8 dircheckbyte; +} __attribute__((packed)); + +#endif diff --git a/fs/adfs/dir_fplus.c b/fs/adfs/dir_fplus.c new file mode 100644 index 000000000..4a1592401 --- /dev/null +++ b/fs/adfs/dir_fplus.c @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/adfs/dir_fplus.c + * + * Copyright (C) 1997-1999 Russell King + */ +#include "adfs.h" +#include "dir_fplus.h" + +/* Return the byte offset to directory entry pos */ +static unsigned int adfs_fplus_offset(const struct adfs_bigdirheader *h, + unsigned int pos) +{ + return offsetof(struct adfs_bigdirheader, bigdirname) + + ALIGN(le32_to_cpu(h->bigdirnamelen), 4) + + pos * sizeof(struct adfs_bigdirentry); +} + +static int adfs_fplus_validate_header(const struct adfs_bigdirheader *h) +{ + unsigned int size = le32_to_cpu(h->bigdirsize); + unsigned int len; + + if (h->bigdirversion[0] != 0 || h->bigdirversion[1] != 0 || + h->bigdirversion[2] != 0 || + h->bigdirstartname != cpu_to_le32(BIGDIRSTARTNAME) || + !size || size & 2047 || size > SZ_4M) + return -EIO; + + size -= sizeof(struct adfs_bigdirtail) + + offsetof(struct adfs_bigdirheader, bigdirname); + + /* Check that bigdirnamelen fits within the directory */ + len = ALIGN(le32_to_cpu(h->bigdirnamelen), 4); + if (len > size) + return -EIO; + + size -= len; + + /* Check that bigdirnamesize fits within the directory */ + len = le32_to_cpu(h->bigdirnamesize); + if (len > size) + return -EIO; + + size -= len; + + /* + * Avoid division, we know that absolute maximum number of entries + * can not be so large to cause overflow of the multiplication below. + */ + len = le32_to_cpu(h->bigdirentries); + if (len > SZ_4M / sizeof(struct adfs_bigdirentry) || + len * sizeof(struct adfs_bigdirentry) > size) + return -EIO; + + return 0; +} + +static int adfs_fplus_validate_tail(const struct adfs_bigdirheader *h, + const struct adfs_bigdirtail *t) +{ + if (t->bigdirendname != cpu_to_le32(BIGDIRENDNAME) || + t->bigdirendmasseq != h->startmasseq || + t->reserved[0] != 0 || t->reserved[1] != 0) + return -EIO; + + return 0; +} + +static u8 adfs_fplus_checkbyte(struct adfs_dir *dir) +{ + struct adfs_bigdirheader *h = dir->bighead; + struct adfs_bigdirtail *t = dir->bigtail; + unsigned int end, bs, bi, i; + __le32 *bp; + u32 dircheck; + + end = adfs_fplus_offset(h, le32_to_cpu(h->bigdirentries)) + + le32_to_cpu(h->bigdirnamesize); + + /* Accumulate the contents of the header, entries and names */ + for (dircheck = 0, bi = 0; end; bi++) { + bp = (void *)dir->bhs[bi]->b_data; + bs = dir->bhs[bi]->b_size; + if (bs > end) + bs = end; + + for (i = 0; i < bs; i += sizeof(u32)) + dircheck = ror32(dircheck, 13) ^ le32_to_cpup(bp++); + + end -= bs; + } + + /* Accumulate the contents of the tail except for the check byte */ + dircheck = ror32(dircheck, 13) ^ le32_to_cpu(t->bigdirendname); + dircheck = ror32(dircheck, 13) ^ t->bigdirendmasseq; + dircheck = ror32(dircheck, 13) ^ t->reserved[0]; + dircheck = ror32(dircheck, 13) ^ t->reserved[1]; + + return dircheck ^ dircheck >> 8 ^ dircheck >> 16 ^ dircheck >> 24; +} + +static int adfs_fplus_read(struct super_block *sb, u32 indaddr, + unsigned int size, struct adfs_dir *dir) +{ + struct adfs_bigdirheader *h; + struct adfs_bigdirtail *t; + unsigned int dirsize; + int ret; + + /* Read first buffer */ + ret = adfs_dir_read_buffers(sb, indaddr, sb->s_blocksize, dir); + if (ret) + return ret; + + dir->bighead = h = (void *)dir->bhs[0]->b_data; + ret = adfs_fplus_validate_header(h); + if (ret) { + adfs_error(sb, "dir %06x has malformed header", indaddr); + goto out; + } + + dirsize = le32_to_cpu(h->bigdirsize); + if (size && dirsize != size) { + adfs_msg(sb, KERN_WARNING, + "dir %06x header size %X does not match directory size %X", + indaddr, dirsize, size); + } + + /* Read remaining buffers */ + ret = adfs_dir_read_buffers(sb, indaddr, dirsize, dir); + if (ret) + return ret; + + dir->bigtail = t = (struct adfs_bigdirtail *) + (dir->bhs[dir->nr_buffers - 1]->b_data + (sb->s_blocksize - 8)); + + ret = adfs_fplus_validate_tail(h, t); + if (ret) { + adfs_error(sb, "dir %06x has malformed tail", indaddr); + goto out; + } + + if (adfs_fplus_checkbyte(dir) != t->bigdircheckbyte) { + adfs_error(sb, "dir %06x checkbyte mismatch\n", indaddr); + goto out; + } + + dir->parent_id = le32_to_cpu(h->bigdirparent); + return 0; + +out: + adfs_dir_relse(dir); + + return ret; +} + +static int +adfs_fplus_setpos(struct adfs_dir *dir, unsigned int fpos) +{ + int ret = -ENOENT; + + if (fpos <= le32_to_cpu(dir->bighead->bigdirentries)) { + dir->pos = fpos; + ret = 0; + } + + return ret; +} + +static int +adfs_fplus_getnext(struct adfs_dir *dir, struct object_info *obj) +{ + struct adfs_bigdirheader *h = dir->bighead; + struct adfs_bigdirentry bde; + unsigned int offset; + int ret; + + if (dir->pos >= le32_to_cpu(h->bigdirentries)) + return -ENOENT; + + offset = adfs_fplus_offset(h, dir->pos); + + ret = adfs_dir_copyfrom(&bde, dir, offset, + sizeof(struct adfs_bigdirentry)); + if (ret) + return ret; + + obj->loadaddr = le32_to_cpu(bde.bigdirload); + obj->execaddr = le32_to_cpu(bde.bigdirexec); + obj->size = le32_to_cpu(bde.bigdirlen); + obj->indaddr = le32_to_cpu(bde.bigdirindaddr); + obj->attr = le32_to_cpu(bde.bigdirattr); + obj->name_len = le32_to_cpu(bde.bigdirobnamelen); + + offset = adfs_fplus_offset(h, le32_to_cpu(h->bigdirentries)); + offset += le32_to_cpu(bde.bigdirobnameptr); + + ret = adfs_dir_copyfrom(obj->name, dir, offset, obj->name_len); + if (ret) + return ret; + + adfs_object_fixup(dir, obj); + + dir->pos += 1; + + return 0; +} + +static int adfs_fplus_iterate(struct adfs_dir *dir, struct dir_context *ctx) +{ + struct object_info obj; + + if ((ctx->pos - 2) >> 32) + return 0; + + if (adfs_fplus_setpos(dir, ctx->pos - 2)) + return 0; + + while (!adfs_fplus_getnext(dir, &obj)) { + if (!dir_emit(ctx, obj.name, obj.name_len, + obj.indaddr, DT_UNKNOWN)) + break; + ctx->pos++; + } + + return 0; +} + +static int adfs_fplus_update(struct adfs_dir *dir, struct object_info *obj) +{ + struct adfs_bigdirheader *h = dir->bighead; + struct adfs_bigdirentry bde; + int offset, end, ret; + + offset = adfs_fplus_offset(h, 0) - sizeof(bde); + end = adfs_fplus_offset(h, le32_to_cpu(h->bigdirentries)); + + do { + offset += sizeof(bde); + if (offset >= end) { + adfs_error(dir->sb, "unable to locate entry to update"); + return -ENOENT; + } + ret = adfs_dir_copyfrom(&bde, dir, offset, sizeof(bde)); + if (ret) { + adfs_error(dir->sb, "error reading directory entry"); + return -ENOENT; + } + } while (le32_to_cpu(bde.bigdirindaddr) != obj->indaddr); + + bde.bigdirload = cpu_to_le32(obj->loadaddr); + bde.bigdirexec = cpu_to_le32(obj->execaddr); + bde.bigdirlen = cpu_to_le32(obj->size); + bde.bigdirindaddr = cpu_to_le32(obj->indaddr); + bde.bigdirattr = cpu_to_le32(obj->attr); + + return adfs_dir_copyto(dir, offset, &bde, sizeof(bde)); +} + +static int adfs_fplus_commit(struct adfs_dir *dir) +{ + int ret; + + /* Increment directory sequence number */ + dir->bighead->startmasseq += 1; + dir->bigtail->bigdirendmasseq += 1; + + /* Update directory check byte */ + dir->bigtail->bigdircheckbyte = adfs_fplus_checkbyte(dir); + + /* Make sure the directory still validates correctly */ + ret = adfs_fplus_validate_header(dir->bighead); + if (ret == 0) + ret = adfs_fplus_validate_tail(dir->bighead, dir->bigtail); + + return ret; +} + +const struct adfs_dir_ops adfs_fplus_dir_ops = { + .read = adfs_fplus_read, + .iterate = adfs_fplus_iterate, + .setpos = adfs_fplus_setpos, + .getnext = adfs_fplus_getnext, + .update = adfs_fplus_update, + .commit = adfs_fplus_commit, +}; diff --git a/fs/adfs/dir_fplus.h b/fs/adfs/dir_fplus.h new file mode 100644 index 000000000..d729b1591 --- /dev/null +++ b/fs/adfs/dir_fplus.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * linux/fs/adfs/dir_fplus.h + * + * Copyright (C) 1999 Russell King + * + * Structures of directories on the F+ format disk + */ + +#define ADFS_FPLUS_NAME_LEN 255 + +#define BIGDIRSTARTNAME ('S' | 'B' << 8 | 'P' << 16 | 'r' << 24) +#define BIGDIRENDNAME ('o' | 'v' << 8 | 'e' << 16 | 'n' << 24) + +struct adfs_bigdirheader { + __u8 startmasseq; + __u8 bigdirversion[3]; + __le32 bigdirstartname; + __le32 bigdirnamelen; + __le32 bigdirsize; + __le32 bigdirentries; + __le32 bigdirnamesize; + __le32 bigdirparent; + char bigdirname[1]; +} __attribute__((packed, aligned(4))); + +struct adfs_bigdirentry { + __le32 bigdirload; + __le32 bigdirexec; + __le32 bigdirlen; + __le32 bigdirindaddr; + __le32 bigdirattr; + __le32 bigdirobnamelen; + __le32 bigdirobnameptr; +} __attribute__((packed, aligned(4))); + +struct adfs_bigdirtail { + __le32 bigdirendname; + __u8 bigdirendmasseq; + __u8 reserved[2]; + __u8 bigdircheckbyte; +} __attribute__((packed, aligned(4))); diff --git a/fs/adfs/file.c b/fs/adfs/file.c new file mode 100644 index 000000000..d056d48c6 --- /dev/null +++ b/fs/adfs/file.c @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/adfs/file.c + * + * Copyright (C) 1997-1999 Russell King + * from: + * + * linux/fs/ext2/file.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * from + * + * linux/fs/minix/file.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * adfs regular file handling primitives + */ +#include "adfs.h" + +const struct file_operations adfs_file_operations = { + .llseek = generic_file_llseek, + .read_iter = generic_file_read_iter, + .mmap_prepare = generic_file_mmap_prepare, + .fsync = simple_fsync, + .write_iter = generic_file_write_iter, + .splice_read = filemap_splice_read, +}; + +const struct inode_operations adfs_file_inode_operations = { + .setattr = adfs_setattr, +}; diff --git a/fs/adfs/inode.c b/fs/adfs/inode.c new file mode 100644 index 000000000..4ac442d0a --- /dev/null +++ b/fs/adfs/inode.c @@ -0,0 +1,373 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/adfs/inode.c + * + * Copyright (C) 1997-1999 Russell King + */ +#include +#include +#include +#include "adfs.h" + +/* + * Lookup/Create a block at offset 'block' into 'inode'. We currently do + * not support creation of new blocks, so we return -EIO for this case. + */ +static int +adfs_get_block(struct inode *inode, sector_t block, struct buffer_head *bh, + int create) +{ + if (!create) { + if (block >= inode->i_blocks) + goto abort_toobig; + + block = __adfs_block_map(inode->i_sb, ADFS_I(inode)->indaddr, + block); + if (block) + map_bh(bh, inode->i_sb, block); + return 0; + } + /* don't support allocation of blocks yet */ + return -EIO; + +abort_toobig: + return 0; +} + +static int adfs_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + return mpage_writepages(mapping, wbc, adfs_get_block); +} + +static int adfs_read_folio(struct file *file, struct folio *folio) +{ + return block_read_full_folio(folio, adfs_get_block); +} + +static void adfs_write_failed(struct address_space *mapping, loff_t to) +{ + struct inode *inode = mapping->host; + + if (to > inode->i_size) + truncate_pagecache(inode, inode->i_size); +} + +static int adfs_write_begin(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, + struct folio **foliop, void **fsdata) +{ + int ret; + + ret = cont_write_begin(iocb, mapping, pos, len, foliop, fsdata, + adfs_get_block, + &ADFS_I(mapping->host)->mmu_private); + if (unlikely(ret)) + adfs_write_failed(mapping, pos + len); + + return ret; +} + +static sector_t _adfs_bmap(struct address_space *mapping, sector_t block) +{ + return generic_block_bmap(mapping, block, adfs_get_block); +} + +static const struct address_space_operations adfs_aops = { + .dirty_folio = block_dirty_folio, + .invalidate_folio = block_invalidate_folio, + .read_folio = adfs_read_folio, + .writepages = adfs_writepages, + .write_begin = adfs_write_begin, + .write_end = generic_write_end, + .migrate_folio = buffer_migrate_folio, + .bmap = _adfs_bmap, +}; + +/* + * Convert ADFS attributes and filetype to Linux permission. + */ +static umode_t +adfs_atts2mode(struct super_block *sb, struct inode *inode) +{ + unsigned int attr = ADFS_I(inode)->attr; + umode_t mode, rmask; + struct adfs_sb_info *asb = ADFS_SB(sb); + + if (attr & ADFS_NDA_DIRECTORY) { + mode = S_IRUGO & asb->s_owner_mask; + return S_IFDIR | S_IXUGO | mode; + } + + switch (adfs_filetype(ADFS_I(inode)->loadaddr)) { + case 0xfc0: /* LinkFS */ + return S_IFLNK|S_IRWXUGO; + + case 0xfe6: /* UnixExec */ + rmask = S_IRUGO | S_IXUGO; + break; + + default: + rmask = S_IRUGO; + } + + mode = S_IFREG; + + if (attr & ADFS_NDA_OWNER_READ) + mode |= rmask & asb->s_owner_mask; + + if (attr & ADFS_NDA_OWNER_WRITE) + mode |= S_IWUGO & asb->s_owner_mask; + + if (attr & ADFS_NDA_PUBLIC_READ) + mode |= rmask & asb->s_other_mask; + + if (attr & ADFS_NDA_PUBLIC_WRITE) + mode |= S_IWUGO & asb->s_other_mask; + return mode; +} + +/* + * Convert Linux permission to ADFS attribute. We try to do the reverse + * of atts2mode, but there is not a 1:1 translation. + */ +static int adfs_mode2atts(struct super_block *sb, struct inode *inode, + umode_t ia_mode) +{ + struct adfs_sb_info *asb = ADFS_SB(sb); + umode_t mode; + int attr; + + /* FIXME: should we be able to alter a link? */ + if (S_ISLNK(inode->i_mode)) + return ADFS_I(inode)->attr; + + /* Directories do not have read/write permissions on the media */ + if (S_ISDIR(inode->i_mode)) + return ADFS_NDA_DIRECTORY; + + attr = 0; + mode = ia_mode & asb->s_owner_mask; + if (mode & S_IRUGO) + attr |= ADFS_NDA_OWNER_READ; + if (mode & S_IWUGO) + attr |= ADFS_NDA_OWNER_WRITE; + + mode = ia_mode & asb->s_other_mask; + mode &= ~asb->s_owner_mask; + if (mode & S_IRUGO) + attr |= ADFS_NDA_PUBLIC_READ; + if (mode & S_IWUGO) + attr |= ADFS_NDA_PUBLIC_WRITE; + + return attr; +} + +static const s64 nsec_unix_epoch_diff_risc_os_epoch = 2208988800000000000LL; + +/* + * Convert an ADFS time to Unix time. ADFS has a 40-bit centi-second time + * referenced to 1 Jan 1900 (til 2248) so we need to discard 2208988800 seconds + * of time to convert from RISC OS epoch to Unix epoch. + */ +static void +adfs_adfs2unix_time(struct timespec64 *tv, struct inode *inode) +{ + unsigned int high, low; + /* 01 Jan 1970 00:00:00 (Unix epoch) as nanoseconds since + * 01 Jan 1900 00:00:00 (RISC OS epoch) + */ + s64 nsec; + + if (!adfs_inode_is_stamped(inode)) + goto cur_time; + + high = ADFS_I(inode)->loadaddr & 0xFF; /* top 8 bits of timestamp */ + low = ADFS_I(inode)->execaddr; /* bottom 32 bits of timestamp */ + + /* convert 40-bit centi-seconds to 32-bit seconds + * going via nanoseconds to retain precision + */ + nsec = (((s64) high << 32) | (s64) low) * 10000000; /* cs to ns */ + + /* Files dated pre 01 Jan 1970 00:00:00. */ + if (nsec < nsec_unix_epoch_diff_risc_os_epoch) + goto too_early; + + /* convert from RISC OS to Unix epoch */ + nsec -= nsec_unix_epoch_diff_risc_os_epoch; + + *tv = ns_to_timespec64(nsec); + return; + + cur_time: + *tv = current_time(inode); + return; + + too_early: + tv->tv_sec = tv->tv_nsec = 0; + return; +} + +/* Convert an Unix time to ADFS time for an entry that is already stamped. */ +static void adfs_unix2adfs_time(struct inode *inode, + const struct timespec64 *ts) +{ + s64 cs, nsec = timespec64_to_ns(ts); + + /* convert from Unix to RISC OS epoch */ + nsec += nsec_unix_epoch_diff_risc_os_epoch; + + /* convert from nanoseconds to centiseconds */ + cs = div_s64(nsec, 10000000); + + cs = clamp_t(s64, cs, 0, 0xffffffffff); + + ADFS_I(inode)->loadaddr &= ~0xff; + ADFS_I(inode)->loadaddr |= (cs >> 32) & 0xff; + ADFS_I(inode)->execaddr = cs; +} + +/* + * Fill in the inode information from the object information. + * + * Note that this is an inode-less filesystem, so we can't use the inode + * number to reference the metadata on the media. Instead, we use the + * inode number to hold the object ID, which in turn will tell us where + * the data is held. We also save the parent object ID, and with these + * two, we can locate the metadata. + * + * This does mean that we rely on an objects parent remaining the same at + * all times - we cannot cope with a cross-directory rename (yet). + */ +struct inode * +adfs_iget(struct super_block *sb, struct object_info *obj) +{ + struct inode *inode; + struct timespec64 ts; + + inode = new_inode(sb); + if (!inode) + goto out; + + inode->i_uid = ADFS_SB(sb)->s_uid; + inode->i_gid = ADFS_SB(sb)->s_gid; + inode->i_ino = obj->indaddr; + inode->i_size = obj->size; + set_nlink(inode, 2); + inode->i_blocks = (inode->i_size + sb->s_blocksize - 1) >> + sb->s_blocksize_bits; + + /* + * we need to save the parent directory ID so that + * write_inode can update the directory information + * for this file. This will need special handling + * for cross-directory renames. + */ + ADFS_I(inode)->parent_id = obj->parent_id; + ADFS_I(inode)->indaddr = obj->indaddr; + ADFS_I(inode)->loadaddr = obj->loadaddr; + ADFS_I(inode)->execaddr = obj->execaddr; + ADFS_I(inode)->attr = obj->attr; + + inode->i_mode = adfs_atts2mode(sb, inode); + adfs_adfs2unix_time(&ts, inode); + inode_set_atime_to_ts(inode, ts); + inode_set_mtime_to_ts(inode, ts); + inode_set_ctime_to_ts(inode, ts); + + if (S_ISDIR(inode->i_mode)) { + inode->i_op = &adfs_dir_inode_operations; + inode->i_fop = &adfs_dir_operations; + } else if (S_ISREG(inode->i_mode)) { + inode->i_op = &adfs_file_inode_operations; + inode->i_fop = &adfs_file_operations; + inode->i_mapping->a_ops = &adfs_aops; + ADFS_I(inode)->mmu_private = inode->i_size; + } + + inode_fake_hash(inode); + +out: + return inode; +} + +/* + * Validate and convert a changed access mode/time to their ADFS equivalents. + * adfs_write_inode will actually write the information back to the directory + * later. + */ +int +adfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr) +{ + struct inode *inode = d_inode(dentry); + struct super_block *sb = inode->i_sb; + unsigned int ia_valid = attr->ia_valid; + int error; + + error = setattr_prepare(&nop_mnt_idmap, dentry, attr); + + /* + * we can't change the UID or GID of any file - + * we have a global UID/GID in the superblock + */ + if ((ia_valid & ATTR_UID && !uid_eq(attr->ia_uid, ADFS_SB(sb)->s_uid)) || + (ia_valid & ATTR_GID && !gid_eq(attr->ia_gid, ADFS_SB(sb)->s_gid))) + error = -EPERM; + + if (error) + goto out; + + /* XXX: this is missing some actual on-disk truncation.. */ + if (ia_valid & ATTR_SIZE) + truncate_setsize(inode, attr->ia_size); + + if (ia_valid & ATTR_MTIME && adfs_inode_is_stamped(inode)) { + adfs_unix2adfs_time(inode, &attr->ia_mtime); + adfs_adfs2unix_time(&attr->ia_mtime, inode); + inode_set_mtime_to_ts(inode, attr->ia_mtime); + } + + /* + * FIXME: should we make these == to i_mtime since we don't + * have the ability to represent them in our filesystem? + */ + if (ia_valid & ATTR_ATIME) + inode_set_atime_to_ts(inode, attr->ia_atime); + if (ia_valid & ATTR_CTIME) + inode_set_ctime_to_ts(inode, attr->ia_ctime); + if (ia_valid & ATTR_MODE) { + ADFS_I(inode)->attr = adfs_mode2atts(sb, inode, attr->ia_mode); + inode->i_mode = adfs_atts2mode(sb, inode); + } + + /* + * FIXME: should we be marking this inode dirty even if + * we don't have any metadata to write back? + */ + if (ia_valid & (ATTR_SIZE | ATTR_MTIME | ATTR_MODE)) + mark_inode_dirty(inode); +out: + return error; +} + +/* + * write an existing inode back to the directory, and therefore the disk. + * The adfs-specific inode data has already been updated by * adfs_setattr(). + */ +int adfs_write_inode(struct inode *inode, struct writeback_control *wbc) +{ + struct super_block *sb = inode->i_sb; + struct object_info obj; + + obj.indaddr = ADFS_I(inode)->indaddr; + obj.name_len = 0; + obj.parent_id = ADFS_I(inode)->parent_id; + obj.loadaddr = ADFS_I(inode)->loadaddr; + obj.execaddr = ADFS_I(inode)->execaddr; + obj.attr = ADFS_I(inode)->attr; + obj.size = inode->i_size; + + return adfs_dir_update(sb, &obj, wbc->sync_mode == WB_SYNC_ALL); +} diff --git a/fs/adfs/map.c b/fs/adfs/map.c new file mode 100644 index 000000000..9d535a2ca --- /dev/null +++ b/fs/adfs/map.c @@ -0,0 +1,407 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/adfs/map.c + * + * Copyright (C) 1997-2002 Russell King + */ +#include +#include +#include +#include "adfs.h" + +/* + * The ADFS map is basically a set of sectors. Each sector is called a + * zone which contains a bitstream made up of variable sized fragments. + * Each bit refers to a set of bytes in the filesystem, defined by + * log2bpmb. This may be larger or smaller than the sector size, but + * the overall size it describes will always be a round number of + * sectors. A fragment id is always idlen bits long. + * + * < idlen > < n > <1> + * +---------+-------//---------+---+ + * | frag id | 0000....000000 | 1 | + * +---------+-------//---------+---+ + * + * The physical disk space used by a fragment is taken from the start of + * the fragment id up to and including the '1' bit - ie, idlen + n + 1 + * bits. + * + * A fragment id can be repeated multiple times in the whole map for + * large or fragmented files. The first map zone a fragment starts in + * is given by fragment id / ids_per_zone - this allows objects to start + * from any zone on the disk. + * + * Free space is described by a linked list of fragments. Each free + * fragment describes free space in the same way as the other fragments, + * however, the frag id specifies an offset (in map bits) from the end + * of this fragment to the start of the next free fragment. + * + * Objects stored on the disk are allocated object ids (we use these as + * our inode numbers.) Object ids contain a fragment id and an optional + * offset. This allows a directory fragment to contain small files + * associated with that directory. + */ + +/* + * For the future... + */ +static DEFINE_RWLOCK(adfs_map_lock); + +/* + * This is fun. We need to load up to 19 bits from the map at an + * arbitrary bit alignment. (We're limited to 19 bits by F+ version 2). + */ +#define GET_FRAG_ID(_map,_start,_idmask) \ + ({ \ + unsigned char *_m = _map + (_start >> 3); \ + u32 _frag = get_unaligned_le32(_m); \ + _frag >>= (_start & 7); \ + _frag & _idmask; \ + }) + +/* + * return the map bit offset of the fragment frag_id in the zone dm. + * Note that the loop is optimised for best asm code - look at the + * output of: + * gcc -D__KERNEL__ -O2 -I../../include -o - -S map.c + */ +static int lookup_zone(const struct adfs_discmap *dm, const unsigned int idlen, + const u32 frag_id, unsigned int *offset) +{ + const unsigned int endbit = dm->dm_endbit; + const u32 idmask = (1 << idlen) - 1; + unsigned char *map = dm->dm_bh->b_data; + unsigned int start = dm->dm_startbit; + unsigned int freelink, fragend; + u32 frag; + + frag = GET_FRAG_ID(map, 8, idmask & 0x7fff); + freelink = frag ? 8 + frag : 0; + + do { + frag = GET_FRAG_ID(map, start, idmask); + + fragend = find_next_bit_le(map, endbit, start + idlen); + if (fragend >= endbit) + goto error; + + if (start == freelink) { + freelink += frag & 0x7fff; + } else if (frag == frag_id) { + unsigned int length = fragend + 1 - start; + + if (*offset < length) + return start + *offset; + *offset -= length; + } + + start = fragend + 1; + } while (start < endbit); + return -1; + +error: + printk(KERN_ERR "adfs: oversized fragment 0x%x at 0x%x-0x%x\n", + frag, start, fragend); + return -1; +} + +/* + * Scan the free space map, for this zone, calculating the total + * number of map bits in each free space fragment. + * + * Note: idmask is limited to 15 bits [3.2] + */ +static unsigned int +scan_free_map(struct adfs_sb_info *asb, struct adfs_discmap *dm) +{ + const unsigned int endbit = dm->dm_endbit; + const unsigned int idlen = asb->s_idlen; + const unsigned int frag_idlen = idlen <= 15 ? idlen : 15; + const u32 idmask = (1 << frag_idlen) - 1; + unsigned char *map = dm->dm_bh->b_data; + unsigned int start = 8, fragend; + u32 frag; + unsigned long total = 0; + + /* + * get fragment id + */ + frag = GET_FRAG_ID(map, start, idmask); + + /* + * If the freelink is null, then no free fragments + * exist in this zone. + */ + if (frag == 0) + return 0; + + do { + start += frag; + + frag = GET_FRAG_ID(map, start, idmask); + + fragend = find_next_bit_le(map, endbit, start + idlen); + if (fragend >= endbit) + goto error; + + total += fragend + 1 - start; + } while (frag >= idlen + 1); + + if (frag != 0) + printk(KERN_ERR "adfs: undersized free fragment\n"); + + return total; +error: + printk(KERN_ERR "adfs: oversized free fragment\n"); + return 0; +} + +static int scan_map(struct adfs_sb_info *asb, unsigned int zone, + const u32 frag_id, unsigned int mapoff) +{ + const unsigned int idlen = asb->s_idlen; + struct adfs_discmap *dm, *dm_end; + int result; + + dm = asb->s_map + zone; + zone = asb->s_map_size; + dm_end = asb->s_map + zone; + + do { + result = lookup_zone(dm, idlen, frag_id, &mapoff); + + if (result != -1) + goto found; + + dm ++; + if (dm == dm_end) + dm = asb->s_map; + } while (--zone > 0); + + return -1; +found: + result -= dm->dm_startbit; + result += dm->dm_startblk; + + return result; +} + +/* + * calculate the amount of free blocks in the map. + * + * n=1 + * total_free = E(free_in_zone_n) + * nzones + */ +void adfs_map_statfs(struct super_block *sb, struct kstatfs *buf) +{ + struct adfs_sb_info *asb = ADFS_SB(sb); + struct adfs_discrecord *dr = adfs_map_discrecord(asb->s_map); + struct adfs_discmap *dm; + unsigned int total = 0; + unsigned int zone; + + dm = asb->s_map; + zone = asb->s_map_size; + + do { + total += scan_free_map(asb, dm++); + } while (--zone > 0); + + buf->f_blocks = adfs_disc_size(dr) >> sb->s_blocksize_bits; + buf->f_files = asb->s_ids_per_zone * asb->s_map_size; + buf->f_bavail = + buf->f_bfree = signed_asl(total, asb->s_map2blk); +} + +int adfs_map_lookup(struct super_block *sb, u32 frag_id, unsigned int offset) +{ + struct adfs_sb_info *asb = ADFS_SB(sb); + unsigned int zone, mapoff; + int result; + + /* + * map & root fragment is special - it starts in the center of the + * disk. The other fragments start at zone (frag / ids_per_zone) + */ + if (frag_id == ADFS_ROOT_FRAG) + zone = asb->s_map_size >> 1; + else + zone = frag_id / asb->s_ids_per_zone; + + if (zone >= asb->s_map_size) + goto bad_fragment; + + /* Convert sector offset to map offset */ + mapoff = signed_asl(offset, -asb->s_map2blk); + + read_lock(&adfs_map_lock); + result = scan_map(asb, zone, frag_id, mapoff); + read_unlock(&adfs_map_lock); + + if (result > 0) { + unsigned int secoff; + + /* Calculate sector offset into map block */ + secoff = offset - signed_asl(mapoff, asb->s_map2blk); + return secoff + signed_asl(result, asb->s_map2blk); + } + + adfs_error(sb, "fragment 0x%04x at offset %d not found in map", + frag_id, offset); + return 0; + +bad_fragment: + adfs_error(sb, "invalid fragment 0x%04x (zone = %d, max = %d)", + frag_id, zone, asb->s_map_size); + return 0; +} + +static unsigned char adfs_calczonecheck(struct super_block *sb, unsigned char *map) +{ + unsigned int v0, v1, v2, v3; + int i; + + v0 = v1 = v2 = v3 = 0; + for (i = sb->s_blocksize - 4; i; i -= 4) { + v0 += map[i] + (v3 >> 8); + v3 &= 0xff; + v1 += map[i + 1] + (v0 >> 8); + v0 &= 0xff; + v2 += map[i + 2] + (v1 >> 8); + v1 &= 0xff; + v3 += map[i + 3] + (v2 >> 8); + v2 &= 0xff; + } + v0 += v3 >> 8; + v1 += map[1] + (v0 >> 8); + v2 += map[2] + (v1 >> 8); + v3 += map[3] + (v2 >> 8); + + return v0 ^ v1 ^ v2 ^ v3; +} + +static int adfs_checkmap(struct super_block *sb, struct adfs_discmap *dm) +{ + unsigned char crosscheck = 0, zonecheck = 1; + int i; + + for (i = 0; i < ADFS_SB(sb)->s_map_size; i++) { + unsigned char *map; + + map = dm[i].dm_bh->b_data; + + if (adfs_calczonecheck(sb, map) != map[0]) { + adfs_error(sb, "zone %d fails zonecheck", i); + zonecheck = 0; + } + crosscheck ^= map[3]; + } + if (crosscheck != 0xff) + adfs_error(sb, "crosscheck != 0xff"); + return crosscheck == 0xff && zonecheck; +} + +/* + * Layout the map - the first zone contains a copy of the disc record, + * and the last zone must be limited to the size of the filesystem. + */ +static void adfs_map_layout(struct adfs_discmap *dm, unsigned int nzones, + struct adfs_discrecord *dr) +{ + unsigned int zone, zone_size; + u64 size; + + zone_size = (8 << dr->log2secsize) - le16_to_cpu(dr->zone_spare); + + dm[0].dm_bh = NULL; + dm[0].dm_startblk = 0; + dm[0].dm_startbit = 32 + ADFS_DR_SIZE_BITS; + dm[0].dm_endbit = 32 + zone_size; + + for (zone = 1; zone < nzones; zone++) { + dm[zone].dm_bh = NULL; + dm[zone].dm_startblk = zone * zone_size - ADFS_DR_SIZE_BITS; + dm[zone].dm_startbit = 32; + dm[zone].dm_endbit = 32 + zone_size; + } + + size = adfs_disc_size(dr) >> dr->log2bpmb; + size -= (nzones - 1) * zone_size - ADFS_DR_SIZE_BITS; + dm[nzones - 1].dm_endbit = 32 + size; +} + +static int adfs_map_read(struct adfs_discmap *dm, struct super_block *sb, + unsigned int map_addr, unsigned int nzones) +{ + unsigned int zone; + + for (zone = 0; zone < nzones; zone++) { + dm[zone].dm_bh = sb_bread(sb, map_addr + zone); + if (!dm[zone].dm_bh) + return -EIO; + } + + return 0; +} + +static void adfs_map_relse(struct adfs_discmap *dm, unsigned int nzones) +{ + unsigned int zone; + + for (zone = 0; zone < nzones; zone++) + brelse(dm[zone].dm_bh); +} + +struct adfs_discmap *adfs_read_map(struct super_block *sb, struct adfs_discrecord *dr) +{ + struct adfs_sb_info *asb = ADFS_SB(sb); + struct adfs_discmap *dm; + unsigned int map_addr, zone_size, nzones; + int ret; + + nzones = dr->nzones | dr->nzones_high << 8; + zone_size = (8 << dr->log2secsize) - le16_to_cpu(dr->zone_spare); + + asb->s_idlen = dr->idlen; + asb->s_map_size = nzones; + asb->s_map2blk = dr->log2bpmb - dr->log2secsize; + asb->s_log2sharesize = dr->log2sharesize; + asb->s_ids_per_zone = zone_size / (asb->s_idlen + 1); + + map_addr = (nzones >> 1) * zone_size - + ((nzones > 1) ? ADFS_DR_SIZE_BITS : 0); + map_addr = signed_asl(map_addr, asb->s_map2blk); + + dm = kmalloc_objs(*dm, nzones); + if (dm == NULL) { + adfs_error(sb, "not enough memory"); + return ERR_PTR(-ENOMEM); + } + + adfs_map_layout(dm, nzones, dr); + + ret = adfs_map_read(dm, sb, map_addr, nzones); + if (ret) { + adfs_error(sb, "unable to read map"); + goto error_free; + } + + if (adfs_checkmap(sb, dm)) + return dm; + + adfs_error(sb, "map corrupted"); + +error_free: + adfs_map_relse(dm, nzones); + kfree(dm); + return ERR_PTR(-EIO); +} + +void adfs_free_map(struct super_block *sb) +{ + struct adfs_sb_info *asb = ADFS_SB(sb); + + adfs_map_relse(asb->s_map, asb->s_map_size); + kfree(asb->s_map); +} diff --git a/fs/adfs/super.c b/fs/adfs/super.c new file mode 100644 index 000000000..a4cd0a515 --- /dev/null +++ b/fs/adfs/super.c @@ -0,0 +1,502 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/adfs/super.c + * + * Copyright (C) 1997-1999 Russell King + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "adfs.h" +#include "dir_f.h" +#include "dir_fplus.h" + +#define ADFS_SB_FLAGS SB_NOATIME + +#define ADFS_DEFAULT_OWNER_MASK S_IRWXU +#define ADFS_DEFAULT_OTHER_MASK (S_IRWXG | S_IRWXO) + +void __adfs_error(struct super_block *sb, const char *function, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + + printk(KERN_CRIT "ADFS-fs error (device %s)%s%s: %pV\n", + sb->s_id, function ? ": " : "", + function ? function : "", &vaf); + + va_end(args); +} + +void adfs_msg(struct super_block *sb, const char *pfx, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + printk("%sADFS-fs (%s): %pV\n", pfx, sb->s_id, &vaf); + va_end(args); +} + +static int adfs_checkdiscrecord(struct adfs_discrecord *dr) +{ + unsigned int max_idlen; + int i; + + /* sector size must be 256, 512 or 1024 bytes */ + if (dr->log2secsize != 8 && + dr->log2secsize != 9 && + dr->log2secsize != 10) + return 1; + + /* idlen must be at least log2secsize + 3 */ + if (dr->idlen < dr->log2secsize + 3) + return 1; + + /* we cannot have such a large disc that we + * are unable to represent sector offsets in + * 32 bits. This works out at 2.0 TB. + */ + if (le32_to_cpu(dr->disc_size_high) >> dr->log2secsize) + return 1; + + /* + * Maximum idlen is limited to 16 bits for new directories by + * the three-byte storage of an indirect disc address. For + * big directories, idlen must be no greater than 19 v2 [1.0] + */ + max_idlen = dr->format_version ? 19 : 16; + if (dr->idlen > max_idlen) + return 1; + + /* reserved bytes should be zero */ + for (i = 0; i < sizeof(dr->unused52); i++) + if (dr->unused52[i] != 0) + return 1; + + return 0; +} + +static void adfs_put_super(struct super_block *sb) +{ + struct adfs_sb_info *asb = ADFS_SB(sb); + + adfs_free_map(sb); + kfree_rcu(asb, rcu); +} + +static int adfs_show_options(struct seq_file *seq, struct dentry *root) +{ + struct adfs_sb_info *asb = ADFS_SB(root->d_sb); + + if (!uid_eq(asb->s_uid, GLOBAL_ROOT_UID)) + seq_printf(seq, ",uid=%u", from_kuid_munged(&init_user_ns, asb->s_uid)); + if (!gid_eq(asb->s_gid, GLOBAL_ROOT_GID)) + seq_printf(seq, ",gid=%u", from_kgid_munged(&init_user_ns, asb->s_gid)); + if (asb->s_owner_mask != ADFS_DEFAULT_OWNER_MASK) + seq_printf(seq, ",ownmask=%o", asb->s_owner_mask); + if (asb->s_other_mask != ADFS_DEFAULT_OTHER_MASK) + seq_printf(seq, ",othmask=%o", asb->s_other_mask); + if (asb->s_ftsuffix != 0) + seq_printf(seq, ",ftsuffix=%u", asb->s_ftsuffix); + + return 0; +} + +enum {Opt_uid, Opt_gid, Opt_ownmask, Opt_othmask, Opt_ftsuffix}; + +static const struct fs_parameter_spec adfs_param_spec[] = { + fsparam_uid ("uid", Opt_uid), + fsparam_gid ("gid", Opt_gid), + fsparam_u32oct ("ownmask", Opt_ownmask), + fsparam_u32oct ("othmask", Opt_othmask), + fsparam_u32 ("ftsuffix", Opt_ftsuffix), + {} +}; + +static int adfs_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct adfs_sb_info *asb = fc->s_fs_info; + struct fs_parse_result result; + int opt; + + opt = fs_parse(fc, adfs_param_spec, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_uid: + asb->s_uid = result.uid; + break; + case Opt_gid: + asb->s_gid = result.gid; + break; + case Opt_ownmask: + asb->s_owner_mask = result.uint_32; + break; + case Opt_othmask: + asb->s_other_mask = result.uint_32; + break; + case Opt_ftsuffix: + asb->s_ftsuffix = result.uint_32; + break; + default: + return -EINVAL; + } + return 0; +} + +static int adfs_reconfigure(struct fs_context *fc) +{ + struct adfs_sb_info *new_asb = fc->s_fs_info; + struct adfs_sb_info *asb = ADFS_SB(fc->root->d_sb); + + sync_filesystem(fc->root->d_sb); + fc->sb_flags |= ADFS_SB_FLAGS; + + /* Structure copy newly parsed options */ + *asb = *new_asb; + + return 0; +} + +static int adfs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + struct adfs_sb_info *sbi = ADFS_SB(sb); + u64 id = huge_encode_dev(sb->s_bdev->bd_dev); + + adfs_map_statfs(sb, buf); + + buf->f_type = ADFS_SUPER_MAGIC; + buf->f_namelen = sbi->s_namelen; + buf->f_bsize = sb->s_blocksize; + buf->f_ffree = (long)(buf->f_bfree * buf->f_files) / (long)buf->f_blocks; + buf->f_fsid = u64_to_fsid(id); + + return 0; +} + +static struct kmem_cache *adfs_inode_cachep; + +static struct inode *adfs_alloc_inode(struct super_block *sb) +{ + struct adfs_inode_info *ei; + ei = alloc_inode_sb(sb, adfs_inode_cachep, GFP_KERNEL); + if (!ei) + return NULL; + return &ei->vfs_inode; +} + +static void adfs_free_inode(struct inode *inode) +{ + kmem_cache_free(adfs_inode_cachep, ADFS_I(inode)); +} + +static int adfs_drop_inode(struct inode *inode) +{ + /* always drop inodes if we are read-only */ + return !IS_ENABLED(CONFIG_ADFS_FS_RW) || IS_RDONLY(inode); +} + +static void init_once(void *foo) +{ + struct adfs_inode_info *ei = (struct adfs_inode_info *) foo; + + inode_init_once(&ei->vfs_inode); +} + +static int __init init_inodecache(void) +{ + adfs_inode_cachep = kmem_cache_create("adfs_inode_cache", + sizeof(struct adfs_inode_info), + 0, (SLAB_RECLAIM_ACCOUNT| + SLAB_ACCOUNT), + init_once); + if (adfs_inode_cachep == NULL) + return -ENOMEM; + return 0; +} + +static void destroy_inodecache(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(adfs_inode_cachep); +} + +static const struct super_operations adfs_sops = { + .alloc_inode = adfs_alloc_inode, + .free_inode = adfs_free_inode, + .drop_inode = adfs_drop_inode, + .write_inode = adfs_write_inode, + .put_super = adfs_put_super, + .statfs = adfs_statfs, + .show_options = adfs_show_options, +}; + +static int adfs_probe(struct super_block *sb, unsigned int offset, int silent, + int (*validate)(struct super_block *sb, + struct buffer_head *bh, + struct adfs_discrecord **bhp)) +{ + struct adfs_sb_info *asb = ADFS_SB(sb); + struct adfs_discrecord *dr; + struct buffer_head *bh; + unsigned int blocksize = BLOCK_SIZE; + int ret, try; + + for (try = 0; try < 2; try++) { + /* try to set the requested block size */ + if (sb->s_blocksize != blocksize && + !sb_set_blocksize(sb, blocksize)) { + if (!silent) + adfs_msg(sb, KERN_ERR, + "error: unsupported blocksize"); + return -EINVAL; + } + + /* read the buffer */ + bh = sb_bread(sb, offset >> sb->s_blocksize_bits); + if (!bh) { + adfs_msg(sb, KERN_ERR, + "error: unable to read block %u, try %d", + offset >> sb->s_blocksize_bits, try); + return -EIO; + } + + /* validate it */ + ret = validate(sb, bh, &dr); + if (ret) { + brelse(bh); + return ret; + } + + /* does the block size match the filesystem block size? */ + blocksize = 1 << dr->log2secsize; + if (sb->s_blocksize == blocksize) { + asb->s_map = adfs_read_map(sb, dr); + brelse(bh); + return PTR_ERR_OR_ZERO(asb->s_map); + } + + brelse(bh); + } + + return -EIO; +} + +static int adfs_validate_bblk(struct super_block *sb, struct buffer_head *bh, + struct adfs_discrecord **drp) +{ + struct adfs_discrecord *dr; + unsigned char *b_data; + + b_data = bh->b_data + (ADFS_DISCRECORD % sb->s_blocksize); + if (adfs_checkbblk(b_data)) + return -EILSEQ; + + /* Do some sanity checks on the ADFS disc record */ + dr = (struct adfs_discrecord *)(b_data + ADFS_DR_OFFSET); + if (adfs_checkdiscrecord(dr)) + return -EILSEQ; + + if ((dr->nzones | dr->nzones_high << 8) == 0) + return -EILSEQ; + + *drp = dr; + return 0; +} + +static int adfs_validate_dr0(struct super_block *sb, struct buffer_head *bh, + struct adfs_discrecord **drp) +{ + struct adfs_discrecord *dr; + + /* Do some sanity checks on the ADFS disc record */ + dr = (struct adfs_discrecord *)(bh->b_data + 4); + if (adfs_checkdiscrecord(dr) || dr->nzones_high || dr->nzones != 1) + return -EILSEQ; + + *drp = dr; + return 0; +} + +static int adfs_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct adfs_discrecord *dr; + struct object_info root_obj; + struct adfs_sb_info *asb = sb->s_fs_info; + struct inode *root; + int ret = -EINVAL; + int silent = fc->sb_flags & SB_SILENT; + + sb->s_flags |= ADFS_SB_FLAGS; + + sb->s_fs_info = asb; + sb->s_magic = ADFS_SUPER_MAGIC; + sb->s_time_gran = 10000000; + + /* Try to probe the filesystem boot block */ + ret = adfs_probe(sb, ADFS_DISCRECORD, 1, adfs_validate_bblk); + if (ret == -EILSEQ) + ret = adfs_probe(sb, 0, silent, adfs_validate_dr0); + if (ret == -EILSEQ) { + if (!silent) + adfs_msg(sb, KERN_ERR, + "error: can't find an ADFS filesystem on dev %s.", + sb->s_id); + ret = -EINVAL; + } + if (ret) + goto error; + + /* set up enough so that we can read an inode */ + sb->s_op = &adfs_sops; + + dr = adfs_map_discrecord(asb->s_map); + + root_obj.parent_id = root_obj.indaddr = le32_to_cpu(dr->root); + root_obj.name_len = 0; + /* Set root object date as 01 Jan 1987 00:00:00 */ + root_obj.loadaddr = 0xfff0003f; + root_obj.execaddr = 0xec22c000; + root_obj.size = ADFS_NEWDIR_SIZE; + root_obj.attr = ADFS_NDA_DIRECTORY | ADFS_NDA_OWNER_READ | + ADFS_NDA_OWNER_WRITE | ADFS_NDA_PUBLIC_READ; + + /* + * If this is a F+ disk with variable length directories, + * get the root_size from the disc record. + */ + if (dr->format_version) { + root_obj.size = le32_to_cpu(dr->root_size); + asb->s_dir = &adfs_fplus_dir_ops; + asb->s_namelen = ADFS_FPLUS_NAME_LEN; + } else { + asb->s_dir = &adfs_f_dir_ops; + asb->s_namelen = ADFS_F_NAME_LEN; + } + /* + * ,xyz hex filetype suffix may be added by driver + * to files that have valid RISC OS filetype + */ + if (asb->s_ftsuffix) + asb->s_namelen += 4; + + set_default_d_op(sb, &adfs_dentry_operations); + root = adfs_iget(sb, &root_obj); + sb->s_root = d_make_root(root); + if (!sb->s_root) { + adfs_free_map(sb); + adfs_error(sb, "get root inode failed\n"); + ret = -EIO; + goto error; + } + return 0; + +error: + sb->s_fs_info = NULL; + kfree(asb); + return ret; +} + +static int adfs_get_tree(struct fs_context *fc) +{ + return get_tree_bdev(fc, adfs_fill_super); +} + +static void adfs_free_fc(struct fs_context *fc) +{ + struct adfs_context *asb = fc->s_fs_info; + + kfree(asb); +} + +static const struct fs_context_operations adfs_context_ops = { + .parse_param = adfs_parse_param, + .get_tree = adfs_get_tree, + .reconfigure = adfs_reconfigure, + .free = adfs_free_fc, +}; + +static int adfs_init_fs_context(struct fs_context *fc) +{ + struct adfs_sb_info *asb; + + asb = kzalloc_obj(struct adfs_sb_info); + if (!asb) + return -ENOMEM; + + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE) { + struct super_block *sb = fc->root->d_sb; + struct adfs_sb_info *old_asb = ADFS_SB(sb); + + /* structure copy existing options before parsing */ + *asb = *old_asb; + } else { + /* set default options */ + asb->s_uid = GLOBAL_ROOT_UID; + asb->s_gid = GLOBAL_ROOT_GID; + asb->s_owner_mask = ADFS_DEFAULT_OWNER_MASK; + asb->s_other_mask = ADFS_DEFAULT_OTHER_MASK; + asb->s_ftsuffix = 0; + } + + fc->ops = &adfs_context_ops; + fc->s_fs_info = asb; + + return 0; +} + +static struct file_system_type adfs_fs_type = { + .owner = THIS_MODULE, + .name = "adfs", + .kill_sb = kill_block_super, + .fs_flags = FS_REQUIRES_DEV, + .init_fs_context = adfs_init_fs_context, + .parameters = adfs_param_spec, +}; +MODULE_ALIAS_FS("adfs"); + +static int __init init_adfs_fs(void) +{ + int err = init_inodecache(); + if (err) + goto out1; + err = register_filesystem(&adfs_fs_type); + if (err) + goto out; + return 0; +out: + destroy_inodecache(); +out1: + return err; +} + +static void __exit exit_adfs_fs(void) +{ + unregister_filesystem(&adfs_fs_type); + destroy_inodecache(); +} + +module_init(init_adfs_fs) +module_exit(exit_adfs_fs) +MODULE_DESCRIPTION("Acorn Disc Filing System"); +MODULE_LICENSE("GPL"); diff --git a/fs/affs/Changes b/fs/affs/Changes new file mode 100644 index 000000000..b41c2c979 --- /dev/null +++ b/fs/affs/Changes @@ -0,0 +1,343 @@ +(Note: I consider version numbers as cheap. That means +that I do not like numbers like 0.1 and the like for +things that can be used since quite some time. But +then, 3.1 doesn't mean 'perfectly stable', too.) + +Known bugs: +----------- + +- Doesn't work on the alpha. The only 64/32-bit + problem that I'm aware of (pointer/int conversion + in readdir()) gives compiler warnings but is + apparently not causing the failure, as directory + reads basically work (but all files are of size 0). + Alas, I've got no alpha to debug. :-( + +- The partition checker (drivers/block/genhd.c) + doesn't work with devices which have 256 byte + blocks (some very old SCSI drives). + +- The feature to automatically make the fs clean + might leave a trashed file system with the + bitmap flag set valid. + +- When a file is truncated to a size that is not + a multiple of the blocksize, the rest of the + last allocated block is not cleared. Well, + this fs never claimed to be Posix conformant. + +Please direct bug reports to: zippel@linux-m68k.org + +Version 3.20 +------------ +- kill kernel lock +- fix for a possible bitmap corruption + +Version 3.19 +------------ + +- sizeof changes from Kernel Janitor Project +- several bug fixes found with fsx + +Version 3.18 +------------ + +- change to global min macro + warning fixes +- add module tags + +Version 3.17 +------------ + +- locking fixes +- wrong sign in __affs_hash_dentry +- remove unnecessary check in affs_new_inode +- enable international mode for dircache fs + +Version 3.16 +------------ + +- use mark_buffer_dirty_inode instead of mark_buffer_dirty. +- introduce affs_lock_{link|dir|ext}. + +Version 3.15 +------------ + +- disable link to directories until we can properly support them. +- locking fixes for link creation/removal. + +Version 3.14 +------------ + +- correctly cut off long file names for compares +- correctly initialize s_last_bmap + +Version 3.13 +------------ + +Major cleanup for 2.4 [Roman Zippel] +- new extended block handling +- new bitmap allocation functions +- locking should be safe for the future +- cleanup of some interfaces + +Version 3.12 +------------ + +more 2.4 fixes: [Roman Zippel] +- s_lock changes +- increased getblock mess +- clear meta blocks + +Version 3.11 +------------ + +- Converted to use 2.3.x page cache [Dave Jones] +- Corruption in truncate() bugfix [Ken Tyler ] + +Version 3.10 +------------ + +- Changed partition checker to allow devices + with physical blocks != 512 bytes. + +- The partition checker now also ignores the + word at 0xd0 that Windows likes to write to. + +Version 3.9 +----------- + +- Moved cleanup from release_file() to put_inode(). + This makes the first one obsolete. + +- truncate() zeroes the unused remainder of a + partially used last block when a file is truncated. + It also marks the inode dirty now (which is not + really necessary as notify_change() will do + it anyway). + +- Added a few comments, fixed some typos (and + introduced some new ones), made the debug messages + more consistent. Changed a bad example in the + doc file (affs.txt). + +- Sets the NOEXEC flag in read_super() for old file + systems, since you can't run programs on them. + +Version 3.8 +----------- +Bill Hawes kindly reviewed the affs and sent me the +patches he did. They're marked (BH). Thanks, Bill! + +- Cleanup of error handling in read_super(). + Didn't release all resources in case of an + error. (BH) + +- put_inode() releases the ext cache only if it's + no longer needed. (BH) + +- One set of dentry callbacks is enough. (BH) + +- Cleanup of error handling in namei.c. (BH) + +- Cleanup of error handling in file.c. (BH) + +- The original blocksize of the device is + restored when the fs is unmounted. (BH) + +- getblock() did not invalidate the key cache + when it allocated a new block. + +- Removed some unnecessary locks as Bill + suggested. + +- Simplified match_name(), changed all hashing + and case insensitive name comparisons to use + uppercase. This makes the tolower() routines + obsolete. + +- Added mount option 'mufs' to force muFS + uid/gid interpretation. + +- File mode changes were not updated on disk. + This was fixed before, but somehow got lost. + +Version 3.7 +----------- + +- Added dentry callbacks to allow the dcache to + operate case insensitive and length ignorant + like the affs itself. + +- getblock() didn't update the lastblock field in the + inode if the fs was not an OFS. This bug only shows + up if a file was enlarged via truncate() and there + was not enough space. + +- Remove some more superfluous code left over from + the old link days ... + +- Fixed some oversights which were in patch 2.1.78. + +- Fixed a few typos. + +Version 3.6 +----------- + +- dentry changes. (Thanks to Jes Sorensen for his help.) + +- Fixed bug in balloc(): Superblock was not set dirty after + the bitmap was changed, so the bitmap wasn't sync'd. + +- Fixed nasty bug in find_new_zone(): If the current + zone number was zero, the loop didn't terminate, + causing a solid lock-up. + +- Removed support for old-style directory reads. + +- Fixed bug in add_entry(): When doing a sorted insert, + the pointer to the next entry in the hash chain wasn't + correctly byte-swapped. Since most of the users of the + affs use it on a 68k, they didn't notice. But why did + I not find this during my tests? + +- Fixed some oversights (version wasn't updated on some + directory changes). + +- Handling of hard links rewritten. To the VFS + they appear now as normal Unix links. They are + now resolved only once in lookup(). The backside + is that unlink(), rename() and rmdir() have to + be smart about them, but the result is worth the + effort. This also led to some code cleanup. + +- Changed name type to unsigned char; the test for + invalid filenames didn't work correctly. + (Thanks to Michael Krause for pointing at this.) + +- Changed mapping of executable flag. + +- Changed all network byte-order macros to the + recommended ones. + +- Added a remount function, so attempts to remount + a dircache filesystem or one with errors read/write + can be trapped. Previously, ro remounts didn't + flush the super block, and rw remounts didn't + create allocation zones ... + +- Call shrink_dcache_parent() in rmdir(). + (Thanks to Bill Hawes.) + +- Permission checks in unlink(). + +- Allow mounting of volumes with superfluous + bitmap pointers read only, also allows them + to be remounted read/write. + +- Owner/Group defaults now to the fs user (i.e. + the one that mounted it) instead of root. This + obsoletes the mount options uid and gid. + +- Argument to volume option could overflow the + name buffer. It is now silently truncated to + 30 characters. (Damn it! This kind of bug + is too embarrassing.) + +- Split inode.c into 2 files, the superblock + routines desperately wanted their own file. + +- truncate() didn't allocate an extension block + cache. If a file was extended by means of + truncate(), this led to an Oops. + +- fsuser is now checked last. + +- rename() will not ignore changes in filename + casing any more (though mv(1) still won't allow + you to do "mv oldname OldName"). + +Version 3.5 +----------- + +- Extension block caches are now allocated on + demand instead of when a file is opened, as + files can be read and written without opening + them (e. g. the loopback device does this). + +- Removed an unused function. + +Version 3.4 +----------- + +- Hash chains are now sorted by block numbers. + (Thanks to Kars de Jong for finding this.) +- Removed all unnecessary external symbols. + +Version 3.3 +----------- + +- Tried to make all types 'correct' and consistent. +- Errors and warnings are now reported via a + function. They are all prefixed by a severity + and have the same appearance: + "AFFS: : " + (There's one exception to this, as in that function + is no pointer to the super block available.) +- The filesystem is remounted read-only after an + error. +- The names of newly created filesystem objects are + now checked for validity. +- Minor cleanups in comments. +- Added this Changes file. At last! + +Version 3.2 +----------- + +- Extension block cache: Reading/writing of huge files + (several MB) is much faster (of course the added + overhead slows down opening, but this is hardly + noticeable). +- The same get_block()-routine can now be used for + both OFS and FFS. +- The super block is now searched in the block that + was calculated and in the one following. This + should remedy the round-off error introduced by + the 1-k blocks that Linux uses. +- Minor changes to adhere to the new VFS interface. +- The number of used blocks is now also calculated + if the filesystem is mounted read-only. +- Prefixed some constants with AFFS_ to avoid name + clashes. +- Removed 'EXPERIMENTAL' status. + +Version 3.1 +----------- + +- Fixed a nasty bug which didn't allow read-only + mounts. +- Allow dir-cache filesystems to be mounted + read only. +- OFS support. +- Several other changes I just cannot remember + any more. + +Version 3.0 +----------- + +- Almost complete rewrite for the new VFS + interface in Linux 1.3. +- Write support. +- Support for hard and symbolic links. +- Lots of things I remember even less ... + +Version 2.0 +----------- + +- Fixed a few things to get it compiled. +- Automatic root block calculation. +- Partition checker for genhd.c + +======================================== + +Let's just call Ray Burr's original affs +'Version 1.0'. diff --git a/fs/affs/Kconfig b/fs/affs/Kconfig new file mode 100644 index 000000000..1ae432d26 --- /dev/null +++ b/fs/affs/Kconfig @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: GPL-2.0-only +config AFFS_FS + tristate "Amiga FFS file system support" + depends on BLOCK + select BUFFER_HEAD + select LEGACY_DIRECT_IO + help + The Fast File System (FFS) is the common file system used on hard + disks by Amiga(tm) systems since AmigaOS Version 1.3 (34.20). Say Y + if you want to be able to read and write files from and to an Amiga + FFS partition on your hard drive. Amiga floppies however cannot be + read with this driver due to an incompatibility of the floppy + controller used in an Amiga and the standard floppy controller in + PCs and workstations. Read + and . + + With this driver you can also mount disk files used by Bernd + Schmidt's Un*X Amiga Emulator + (). + If you want to do this, you will also need to say Y or M to "Loop + device support", above. + + To compile this file system support as a module, choose M here: the + module will be called affs. If unsure, say N. diff --git a/fs/affs/Makefile b/fs/affs/Makefile new file mode 100644 index 000000000..f2c811429 --- /dev/null +++ b/fs/affs/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the Linux affs filesystem routines. +# + +#ccflags-y := -DDEBUG=1 + +obj-$(CONFIG_AFFS_FS) += affs.o + +affs-objs := super.o namei.o inode.o file.o dir.o amigaffs.o bitmap.o symlink.o diff --git a/fs/affs/affs.h b/fs/affs/affs.h new file mode 100644 index 000000000..d1c506c1f --- /dev/null +++ b/fs/affs/affs.h @@ -0,0 +1,322 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifdef pr_fmt +#undef pr_fmt +#endif + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include "amigaffs.h" +#include +#include + +/* Ugly macros make the code more pretty. */ + +#define AFFS_BLOCK(sb, bh, blk) (AFFS_HEAD(bh)->table[AFFS_SB(sb)->s_hashsize-1-(blk)]) + +#define AFFS_HEAD(bh) ((struct affs_head *)(bh)->b_data) +#define AFFS_TAIL(sb, bh) ((struct affs_tail *)((bh)->b_data+(sb)->s_blocksize-sizeof(struct affs_tail))) +#define AFFS_ROOT_HEAD(bh) ((struct affs_root_head *)(bh)->b_data) +#define AFFS_ROOT_TAIL(sb, bh) ((struct affs_root_tail *)((bh)->b_data+(sb)->s_blocksize-sizeof(struct affs_root_tail))) +#define AFFS_DATA_HEAD(bh) ((struct affs_data_head *)(bh)->b_data) +#define AFFS_DATA(bh) (((struct affs_data_head *)(bh)->b_data)->data) + +#define AFFS_CACHE_SIZE PAGE_SIZE + +#define AFFS_LC_SIZE (AFFS_CACHE_SIZE/sizeof(u32)/2) +#define AFFS_AC_SIZE (AFFS_CACHE_SIZE/sizeof(struct affs_ext_key)/2) +#define AFFS_AC_MASK (AFFS_AC_SIZE-1) + +#define AFFSNAMEMAX 30U + +struct affs_ext_key { + u32 ext; /* idx of the extended block */ + u32 key; /* block number */ +}; + +/* + * affs fs inode data in memory + */ +struct affs_inode_info { + atomic_t i_opencnt; + struct mutex i_link_lock; /* Protects internal inode access. */ + struct mutex i_ext_lock; /* Protects internal inode access. */ +#define i_hash_lock i_ext_lock + u32 i_blkcnt; /* block count */ + u32 i_extcnt; /* extended block count */ + u32 *i_lc; /* linear cache of extended blocks */ + u32 i_lc_size; + u32 i_lc_shift; + u32 i_lc_mask; + struct affs_ext_key *i_ac; /* associative cache of extended blocks */ + u32 i_ext_last; /* last accessed extended block */ + struct buffer_head *i_ext_bh; /* bh of last extended block */ + loff_t mmu_private; + u32 i_protect; /* unused attribute bits */ + u32 i_lastalloc; /* last allocated block */ + int i_pa_cnt; /* number of preallocated blocks */ + struct inode vfs_inode; +}; + +/* short cut to get to the affs specific inode data */ +static inline struct affs_inode_info *AFFS_I(struct inode *inode) +{ + return container_of(inode, struct affs_inode_info, vfs_inode); +} + +/* + * super-block data in memory + * + * Block numbers are adjusted for their actual size + * + */ + +struct affs_bm_info { + u32 bm_key; /* Disk block number */ + u32 bm_free; /* Free blocks in here */ +}; + +struct affs_sb_info { + int s_partition_size; /* Partition size in blocks. */ + int s_reserved; /* Number of reserved blocks. */ + //u32 s_blksize; /* Initial device blksize */ + u32 s_data_blksize; /* size of the data block w/o header */ + u32 s_root_block; /* FFS root block number. */ + int s_hashsize; /* Size of hash table. */ + unsigned long s_flags; /* See below. */ + kuid_t s_uid; /* uid to override */ + kgid_t s_gid; /* gid to override */ + umode_t s_mode; /* mode to override */ + struct buffer_head *s_root_bh; /* Cached root block. */ + struct mutex s_bmlock; /* Protects bitmap access. */ + struct affs_bm_info *s_bitmap; /* Bitmap infos. */ + u32 s_bmap_count; /* # of bitmap blocks. */ + u32 s_bmap_bits; /* # of bits in one bitmap blocks */ + u32 s_last_bmap; + struct buffer_head *s_bmap_bh; + char *s_prefix; /* Prefix for volumes and assigns. */ + char s_volume[32]; /* Volume prefix for absolute symlinks. */ + spinlock_t symlink_lock; /* protects the previous two */ + struct super_block *sb; /* the VFS superblock object */ + int work_queued; /* non-zero delayed work is queued */ + struct delayed_work sb_work; /* superblock flush delayed work */ + spinlock_t work_lock; /* protects sb_work and work_queued */ + struct rcu_head rcu; +}; + +#define AFFS_MOUNT_SF_INTL 0x0001 /* International filesystem. */ +#define AFFS_MOUNT_SF_BM_VALID 0x0002 /* Bitmap is valid. */ +#define AFFS_MOUNT_SF_IMMUTABLE 0x0004 /* Protection bits cannot be changed */ +#define AFFS_MOUNT_SF_QUIET 0x0008 /* chmod errors will be not reported */ +#define AFFS_MOUNT_SF_SETUID 0x0010 /* Ignore Amiga uid */ +#define AFFS_MOUNT_SF_SETGID 0x0020 /* Ignore Amiga gid */ +#define AFFS_MOUNT_SF_SETMODE 0x0040 /* Ignore Amiga protection bits */ +#define AFFS_MOUNT_SF_MUFS 0x0100 /* Use MUFS uid/gid mapping */ +#define AFFS_MOUNT_SF_OFS 0x0200 /* Old filesystem */ +#define AFFS_MOUNT_SF_PREFIX 0x0400 /* Buffer for prefix is allocated */ +#define AFFS_MOUNT_SF_VERBOSE 0x0800 /* Talk about fs when mounting */ +#define AFFS_MOUNT_SF_NO_TRUNCATE 0x1000 /* Don't truncate filenames */ + +#define affs_clear_opt(o, opt) (o &= ~AFFS_MOUNT_##opt) +#define affs_set_opt(o, opt) (o |= AFFS_MOUNT_##opt) +#define affs_test_opt(o, opt) ((o) & AFFS_MOUNT_##opt) + +/* short cut to get to the affs specific sb data */ +static inline struct affs_sb_info *AFFS_SB(struct super_block *sb) +{ + return sb->s_fs_info; +} + +void affs_mark_sb_dirty(struct super_block *sb); + +/* amigaffs.c */ + +extern int affs_insert_hash(struct inode *inode, struct buffer_head *bh); +extern int affs_remove_hash(struct inode *dir, struct buffer_head *rem_bh); +extern int affs_remove_header(struct dentry *dentry); +extern u32 affs_checksum_block(struct super_block *sb, struct buffer_head *bh); +extern void affs_fix_checksum(struct super_block *sb, struct buffer_head *bh); +extern void affs_secs_to_datestamp(time64_t secs, struct affs_date *ds); +extern umode_t affs_prot_to_mode(u32 prot); +extern void affs_mode_to_prot(struct inode *inode); +__printf(3, 4) +extern void affs_error(struct super_block *sb, const char *function, + const char *fmt, ...); +__printf(3, 4) +extern void affs_warning(struct super_block *sb, const char *function, + const char *fmt, ...); +extern bool affs_nofilenametruncate(const struct dentry *dentry); +extern int affs_check_name(const unsigned char *name, int len, + bool notruncate); +extern int affs_copy_name(unsigned char *bstr, struct dentry *dentry); + +/* bitmap. c */ + +extern u32 affs_count_free_blocks(struct super_block *s); +extern void affs_free_block(struct super_block *sb, u32 block); +extern u32 affs_alloc_block(struct inode *inode, u32 goal); +extern int affs_init_bitmap(struct super_block *sb, int *flags); +extern void affs_free_bitmap(struct super_block *sb); + +/* namei.c */ + +extern const struct export_operations affs_export_ops; +extern int affs_hash_name(struct super_block *sb, const u8 *name, unsigned int len); +extern struct dentry *affs_lookup(struct inode *dir, struct dentry *dentry, unsigned int); +extern int affs_unlink(struct inode *dir, struct dentry *dentry); +extern int affs_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode); +extern struct dentry *affs_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode); +extern int affs_rmdir(struct inode *dir, struct dentry *dentry); +extern int affs_link(struct dentry *olddentry, struct inode *dir, + struct dentry *dentry); +extern int affs_symlink(struct mnt_idmap *idmap, + struct inode *dir, struct dentry *dentry, + const char *symname); +extern int affs_rename2(struct mnt_idmap *idmap, + struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry, + unsigned int flags); + +/* inode.c */ + +extern struct inode *affs_new_inode(struct inode *dir); +extern int affs_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, struct iattr *attr); +extern void affs_evict_inode(struct inode *inode); +extern struct inode *affs_iget(struct super_block *sb, + unsigned long ino); +extern int affs_write_inode(struct inode *inode, + struct writeback_control *wbc); +extern int affs_add_entry(struct inode *dir, struct inode *inode, + struct dentry *dentry, s32 type); + +/* file.c */ + +void affs_free_prealloc(struct inode *inode); +extern void affs_truncate(struct inode *); +int affs_file_fsync(struct file *, loff_t, loff_t, int); + +/* dir.c */ + +extern void affs_dir_truncate(struct inode *); + +/* jump tables */ + +extern const struct inode_operations affs_file_inode_operations; +extern const struct inode_operations affs_dir_inode_operations; +extern const struct inode_operations affs_symlink_inode_operations; +extern const struct file_operations affs_file_operations; +extern const struct file_operations affs_file_operations_ofs; +extern const struct file_operations affs_dir_operations; +extern const struct address_space_operations affs_symlink_aops; +extern const struct address_space_operations affs_aops; +extern const struct address_space_operations affs_aops_ofs; + +extern const struct dentry_operations affs_dentry_operations; +extern const struct dentry_operations affs_intl_dentry_operations; + +static inline bool affs_validblock(struct super_block *sb, int block) +{ + return(block >= AFFS_SB(sb)->s_reserved && + block < AFFS_SB(sb)->s_partition_size); +} + +static inline struct buffer_head * +affs_bread(struct super_block *sb, int block) +{ + pr_debug("%s: %d\n", __func__, block); + if (affs_validblock(sb, block)) + return sb_bread(sb, block); + return NULL; +} +static inline struct buffer_head * +affs_getblk(struct super_block *sb, int block) +{ + pr_debug("%s: %d\n", __func__, block); + if (affs_validblock(sb, block)) + return sb_getblk(sb, block); + return NULL; +} +static inline struct buffer_head * +affs_getzeroblk(struct super_block *sb, int block) +{ + struct buffer_head *bh; + pr_debug("%s: %d\n", __func__, block); + if (affs_validblock(sb, block)) { + bh = sb_getblk(sb, block); + lock_buffer(bh); + memset(bh->b_data, 0 , sb->s_blocksize); + set_buffer_uptodate(bh); + unlock_buffer(bh); + return bh; + } + return NULL; +} +static inline struct buffer_head * +affs_getemptyblk(struct super_block *sb, int block) +{ + struct buffer_head *bh; + pr_debug("%s: %d\n", __func__, block); + if (affs_validblock(sb, block)) { + bh = sb_getblk(sb, block); + wait_on_buffer(bh); + set_buffer_uptodate(bh); + return bh; + } + return NULL; +} +static inline void +affs_brelse(struct buffer_head *bh) +{ + if (bh) + pr_debug("%s: %lld\n", __func__, (long long) bh->b_blocknr); + brelse(bh); +} + +static inline void +affs_adjust_checksum(struct buffer_head *bh, u32 val) +{ + u32 tmp = be32_to_cpu(((__be32 *)bh->b_data)[5]); + ((__be32 *)bh->b_data)[5] = cpu_to_be32(tmp - val); +} +static inline void +affs_adjust_bitmapchecksum(struct buffer_head *bh, u32 val) +{ + u32 tmp = be32_to_cpu(((__be32 *)bh->b_data)[0]); + ((__be32 *)bh->b_data)[0] = cpu_to_be32(tmp - val); +} + +static inline void +affs_lock_link(struct inode *inode) +{ + mutex_lock(&AFFS_I(inode)->i_link_lock); +} +static inline void +affs_unlock_link(struct inode *inode) +{ + mutex_unlock(&AFFS_I(inode)->i_link_lock); +} +static inline void +affs_lock_dir(struct inode *inode) +{ + mutex_lock_nested(&AFFS_I(inode)->i_hash_lock, SINGLE_DEPTH_NESTING); +} +static inline void +affs_unlock_dir(struct inode *inode) +{ + mutex_unlock(&AFFS_I(inode)->i_hash_lock); +} +static inline void +affs_lock_ext(struct inode *inode) +{ + mutex_lock(&AFFS_I(inode)->i_ext_lock); +} +static inline void +affs_unlock_ext(struct inode *inode) +{ + mutex_unlock(&AFFS_I(inode)->i_ext_lock); +} diff --git a/fs/affs/amigaffs.c b/fs/affs/amigaffs.c new file mode 100644 index 000000000..6cc0fc9a4 --- /dev/null +++ b/fs/affs/amigaffs.c @@ -0,0 +1,543 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/affs/amigaffs.c + * + * (c) 1996 Hans-Joachim Widmaier - Rewritten + * + * (C) 1993 Ray Burr - Amiga FFS filesystem. + * + * Please send bug reports to: hjw@zvw.de + */ + +#include +#include +#include "affs.h" + +/* + * Functions for accessing Amiga-FFS structures. + */ + + +/* Insert a header block bh into the directory dir + * caller must hold AFFS_DIR->i_hash_lock! + */ + +int +affs_insert_hash(struct inode *dir, struct buffer_head *bh) +{ + struct super_block *sb = dir->i_sb; + struct buffer_head *dir_bh; + u32 ino, hash_ino; + int offset; + + ino = bh->b_blocknr; + offset = affs_hash_name(sb, AFFS_TAIL(sb, bh)->name + 1, AFFS_TAIL(sb, bh)->name[0]); + + pr_debug("%s(dir=%llu, ino=%d)\n", __func__, dir->i_ino, ino); + + dir_bh = affs_bread(sb, dir->i_ino); + if (!dir_bh) + return -EIO; + + hash_ino = be32_to_cpu(AFFS_HEAD(dir_bh)->table[offset]); + while (hash_ino) { + affs_brelse(dir_bh); + dir_bh = affs_bread(sb, hash_ino); + if (!dir_bh) + return -EIO; + hash_ino = be32_to_cpu(AFFS_TAIL(sb, dir_bh)->hash_chain); + } + AFFS_TAIL(sb, bh)->parent = cpu_to_be32(dir->i_ino); + AFFS_TAIL(sb, bh)->hash_chain = 0; + affs_fix_checksum(sb, bh); + + if (dir->i_ino == dir_bh->b_blocknr) + AFFS_HEAD(dir_bh)->table[offset] = cpu_to_be32(ino); + else + AFFS_TAIL(sb, dir_bh)->hash_chain = cpu_to_be32(ino); + + affs_adjust_checksum(dir_bh, ino); + mark_buffer_dirty(dir_bh); + affs_brelse(dir_bh); + + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + inode_inc_iversion(dir); + mark_inode_dirty(dir); + + return 0; +} + +/* Remove a header block from its directory. + * caller must hold AFFS_DIR->i_hash_lock! + */ + +int +affs_remove_hash(struct inode *dir, struct buffer_head *rem_bh) +{ + struct super_block *sb; + struct buffer_head *bh; + u32 rem_ino, hash_ino; + __be32 ino; + int offset, retval; + + sb = dir->i_sb; + rem_ino = rem_bh->b_blocknr; + offset = affs_hash_name(sb, AFFS_TAIL(sb, rem_bh)->name+1, AFFS_TAIL(sb, rem_bh)->name[0]); + pr_debug("%s(dir=%llu, ino=%d, hashval=%d)\n", __func__, dir->i_ino, + rem_ino, offset); + + bh = affs_bread(sb, dir->i_ino); + if (!bh) + return -EIO; + + retval = -ENOENT; + hash_ino = be32_to_cpu(AFFS_HEAD(bh)->table[offset]); + while (hash_ino) { + if (hash_ino == rem_ino) { + ino = AFFS_TAIL(sb, rem_bh)->hash_chain; + if (dir->i_ino == bh->b_blocknr) + AFFS_HEAD(bh)->table[offset] = ino; + else + AFFS_TAIL(sb, bh)->hash_chain = ino; + affs_adjust_checksum(bh, be32_to_cpu(ino) - hash_ino); + mark_buffer_dirty(bh); + AFFS_TAIL(sb, rem_bh)->parent = 0; + retval = 0; + break; + } + affs_brelse(bh); + bh = affs_bread(sb, hash_ino); + if (!bh) + return -EIO; + hash_ino = be32_to_cpu(AFFS_TAIL(sb, bh)->hash_chain); + } + + affs_brelse(bh); + + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + inode_inc_iversion(dir); + mark_inode_dirty(dir); + + return retval; +} + +static void +affs_fix_dcache(struct inode *inode, u32 entry_ino) +{ + struct dentry *dentry; + spin_lock(&inode->i_lock); + for_each_alias(dentry, inode) { + if (entry_ino == (u32)(long)dentry->d_fsdata) { + dentry->d_fsdata = (void *)(unsigned long)inode->i_ino; + break; + } + } + spin_unlock(&inode->i_lock); +} + + +/* Remove header from link chain */ + +static int +affs_remove_link(struct dentry *dentry) +{ + struct inode *dir, *inode = d_inode(dentry); + struct super_block *sb = inode->i_sb; + struct buffer_head *bh, *link_bh = NULL; + u32 link_ino, ino; + int retval; + + pr_debug("%s(key=%llu)\n", __func__, inode->i_ino); + retval = -EIO; + bh = affs_bread(sb, inode->i_ino); + if (!bh) + goto done; + + link_ino = (u32)(long)dentry->d_fsdata; + if (inode->i_ino == link_ino) { + /* we can't remove the head of the link, as its blocknr is still used as ino, + * so we remove the block of the first link instead. + */ + link_ino = be32_to_cpu(AFFS_TAIL(sb, bh)->link_chain); + link_bh = affs_bread(sb, link_ino); + if (!link_bh) + goto done; + + dir = affs_iget(sb, be32_to_cpu(AFFS_TAIL(sb, link_bh)->parent)); + if (IS_ERR(dir)) { + retval = PTR_ERR(dir); + goto done; + } + + affs_lock_dir(dir); + /* + * if there's a dentry for that block, make it + * refer to inode itself. + */ + affs_fix_dcache(inode, link_ino); + retval = affs_remove_hash(dir, link_bh); + if (retval) { + affs_unlock_dir(dir); + goto done; + } + mark_buffer_dirty(link_bh); + + memcpy(AFFS_TAIL(sb, bh)->name, AFFS_TAIL(sb, link_bh)->name, 32); + retval = affs_insert_hash(dir, bh); + if (retval) { + affs_unlock_dir(dir); + goto done; + } + mark_buffer_dirty(bh); + + affs_unlock_dir(dir); + iput(dir); + } else { + link_bh = affs_bread(sb, link_ino); + if (!link_bh) + goto done; + } + + while ((ino = be32_to_cpu(AFFS_TAIL(sb, bh)->link_chain)) != 0) { + if (ino == link_ino) { + __be32 ino2 = AFFS_TAIL(sb, link_bh)->link_chain; + AFFS_TAIL(sb, bh)->link_chain = ino2; + affs_adjust_checksum(bh, be32_to_cpu(ino2) - link_ino); + mark_buffer_dirty(bh); + retval = 0; + /* Fix the link count, if bh is a normal header block without links */ + switch (be32_to_cpu(AFFS_TAIL(sb, bh)->stype)) { + case ST_LINKDIR: + case ST_LINKFILE: + break; + default: + if (!AFFS_TAIL(sb, bh)->link_chain) + set_nlink(inode, 1); + } + affs_free_block(sb, link_ino); + goto done; + } + affs_brelse(bh); + bh = affs_bread(sb, ino); + if (!bh) + goto done; + } + retval = -ENOENT; +done: + affs_brelse(link_bh); + affs_brelse(bh); + return retval; +} + + +static int +affs_empty_dir(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + struct buffer_head *bh; + int retval, size; + + retval = -EIO; + bh = affs_bread(sb, inode->i_ino); + if (!bh) + goto done; + + retval = -ENOTEMPTY; + for (size = AFFS_SB(sb)->s_hashsize - 1; size >= 0; size--) + if (AFFS_HEAD(bh)->table[size]) + goto not_empty; + retval = 0; +not_empty: + affs_brelse(bh); +done: + return retval; +} + + +/* Remove a filesystem object. If the object to be removed has + * links to it, one of the links must be changed to inherit + * the file or directory. As above, any inode will do. + * The buffer will not be freed. If the header is a link, the + * block will be marked as free. + * This function returns a negative error number in case of + * an error, else 0 if the inode is to be deleted or 1 if not. + */ + +int +affs_remove_header(struct dentry *dentry) +{ + struct super_block *sb; + struct inode *inode, *dir; + struct buffer_head *bh = NULL; + int retval; + + dir = d_inode(dentry->d_parent); + sb = dir->i_sb; + + retval = -ENOENT; + inode = d_inode(dentry); + if (!inode) + goto done; + + pr_debug("%s(key=%llu)\n", __func__, inode->i_ino); + retval = -EIO; + bh = affs_bread(sb, (u32)(long)dentry->d_fsdata); + if (!bh) + goto done; + + affs_lock_link(inode); + affs_lock_dir(dir); + switch (be32_to_cpu(AFFS_TAIL(sb, bh)->stype)) { + case ST_USERDIR: + /* if we ever want to support links to dirs + * i_hash_lock of the inode must only be + * taken after some checks + */ + affs_lock_dir(inode); + retval = affs_empty_dir(inode); + affs_unlock_dir(inode); + if (retval) + goto done_unlock; + break; + default: + break; + } + + retval = affs_remove_hash(dir, bh); + if (retval) + goto done_unlock; + mark_buffer_dirty(bh); + + affs_unlock_dir(dir); + + if (inode->i_nlink > 1) + retval = affs_remove_link(dentry); + else + clear_nlink(inode); + affs_unlock_link(inode); + inode_set_ctime_current(inode); + mark_inode_dirty(inode); + +done: + affs_brelse(bh); + return retval; + +done_unlock: + affs_unlock_dir(dir); + affs_unlock_link(inode); + goto done; +} + +/* Checksum a block, do various consistency checks and optionally return + the blocks type number. DATA points to the block. If their pointers + are non-null, *PTYPE and *STYPE are set to the primary and secondary + block types respectively, *HASHSIZE is set to the size of the hashtable + (which lets us calculate the block size). + Returns non-zero if the block is not consistent. */ + +u32 +affs_checksum_block(struct super_block *sb, struct buffer_head *bh) +{ + __be32 *ptr = (__be32 *)bh->b_data; + u32 sum; + int bsize; + + sum = 0; + for (bsize = sb->s_blocksize / sizeof(__be32); bsize > 0; bsize--) + sum += be32_to_cpu(*ptr++); + return sum; +} + +/* + * Calculate the checksum of a disk block and store it + * at the indicated position. + */ + +void +affs_fix_checksum(struct super_block *sb, struct buffer_head *bh) +{ + int cnt = sb->s_blocksize / sizeof(__be32); + __be32 *ptr = (__be32 *)bh->b_data; + u32 checksum; + __be32 *checksumptr; + + checksumptr = ptr + 5; + *checksumptr = 0; + for (checksum = 0; cnt > 0; ptr++, cnt--) + checksum += be32_to_cpu(*ptr); + *checksumptr = cpu_to_be32(-checksum); +} + +void +affs_secs_to_datestamp(time64_t secs, struct affs_date *ds) +{ + u32 days; + u32 minute; + s32 rem; + + secs -= sys_tz.tz_minuteswest * 60 + AFFS_EPOCH_DELTA; + if (secs < 0) + secs = 0; + days = div_s64_rem(secs, 86400, &rem); + minute = rem / 60; + rem -= minute * 60; + + ds->days = cpu_to_be32(days); + ds->mins = cpu_to_be32(minute); + ds->ticks = cpu_to_be32(rem * 50); +} + +umode_t +affs_prot_to_mode(u32 prot) +{ + umode_t mode = 0; + + if (!(prot & FIBF_NOWRITE)) + mode |= 0200; + if (!(prot & FIBF_NOREAD)) + mode |= 0400; + if (!(prot & FIBF_NOEXECUTE)) + mode |= 0100; + if (prot & FIBF_GRP_WRITE) + mode |= 0020; + if (prot & FIBF_GRP_READ) + mode |= 0040; + if (prot & FIBF_GRP_EXECUTE) + mode |= 0010; + if (prot & FIBF_OTR_WRITE) + mode |= 0002; + if (prot & FIBF_OTR_READ) + mode |= 0004; + if (prot & FIBF_OTR_EXECUTE) + mode |= 0001; + + return mode; +} + +void +affs_mode_to_prot(struct inode *inode) +{ + u32 prot = AFFS_I(inode)->i_protect; + umode_t mode = inode->i_mode; + + /* + * First, clear all RWED bits for owner, group, other. + * Then, recalculate them afresh. + * + * We'll always clear the delete-inhibit bit for the owner, as that is + * the classic single-user mode AmigaOS protection bit and we need to + * stay compatible with all scenarios. + * + * Since multi-user AmigaOS is an extension, we'll only set the + * delete-allow bit if any of the other bits in the same user class + * (group/other) are used. + */ + prot &= ~(FIBF_NOEXECUTE | FIBF_NOREAD + | FIBF_NOWRITE | FIBF_NODELETE + | FIBF_GRP_EXECUTE | FIBF_GRP_READ + | FIBF_GRP_WRITE | FIBF_GRP_DELETE + | FIBF_OTR_EXECUTE | FIBF_OTR_READ + | FIBF_OTR_WRITE | FIBF_OTR_DELETE); + + /* Classic single-user AmigaOS flags. These are inverted. */ + if (!(mode & 0100)) + prot |= FIBF_NOEXECUTE; + if (!(mode & 0400)) + prot |= FIBF_NOREAD; + if (!(mode & 0200)) + prot |= FIBF_NOWRITE; + + /* Multi-user extended flags. Not inverted. */ + if (mode & 0010) + prot |= FIBF_GRP_EXECUTE; + if (mode & 0040) + prot |= FIBF_GRP_READ; + if (mode & 0020) + prot |= FIBF_GRP_WRITE; + if (mode & 0070) + prot |= FIBF_GRP_DELETE; + + if (mode & 0001) + prot |= FIBF_OTR_EXECUTE; + if (mode & 0004) + prot |= FIBF_OTR_READ; + if (mode & 0002) + prot |= FIBF_OTR_WRITE; + if (mode & 0007) + prot |= FIBF_OTR_DELETE; + + AFFS_I(inode)->i_protect = prot; +} + +void +affs_error(struct super_block *sb, const char *function, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + pr_crit("error (device %s): %s(): %pV\n", sb->s_id, function, &vaf); + if (!sb_rdonly(sb)) + pr_warn("Remounting filesystem read-only\n"); + sb->s_flags |= SB_RDONLY; + va_end(args); +} + +void +affs_warning(struct super_block *sb, const char *function, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + pr_warn("(device %s): %s(): %pV\n", sb->s_id, function, &vaf); + va_end(args); +} + +bool +affs_nofilenametruncate(const struct dentry *dentry) +{ + return affs_test_opt(AFFS_SB(dentry->d_sb)->s_flags, SF_NO_TRUNCATE); +} + +/* Check if the name is valid for a affs object. */ + +int +affs_check_name(const unsigned char *name, int len, bool notruncate) +{ + int i; + + if (len > AFFSNAMEMAX) { + if (notruncate) + return -ENAMETOOLONG; + len = AFFSNAMEMAX; + } + for (i = 0; i < len; i++) { + if (name[i] < ' ' || name[i] == ':' + || (name[i] > 0x7e && name[i] < 0xa0)) + return -EINVAL; + } + + return 0; +} + +/* This function copies name to bstr, with at most 30 + * characters length. The bstr will be prepended by + * a length byte. + * NOTE: The name will must be already checked by + * affs_check_name()! + */ + +int +affs_copy_name(unsigned char *bstr, struct dentry *dentry) +{ + u32 len = min(dentry->d_name.len, AFFSNAMEMAX); + + *bstr++ = len; + memcpy(bstr, dentry->d_name.name, len); + return len; +} diff --git a/fs/affs/amigaffs.h b/fs/affs/amigaffs.h new file mode 100644 index 000000000..da3217ab6 --- /dev/null +++ b/fs/affs/amigaffs.h @@ -0,0 +1,149 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef AMIGAFFS_H +#define AMIGAFFS_H + +#include +#include + +#define FS_OFS 0x444F5300 +#define FS_FFS 0x444F5301 +#define FS_INTLOFS 0x444F5302 +#define FS_INTLFFS 0x444F5303 +#define FS_DCOFS 0x444F5304 +#define FS_DCFFS 0x444F5305 +#define MUFS_FS 0x6d754653 /* 'muFS' */ +#define MUFS_OFS 0x6d754600 /* 'muF\0' */ +#define MUFS_FFS 0x6d754601 /* 'muF\1' */ +#define MUFS_INTLOFS 0x6d754602 /* 'muF\2' */ +#define MUFS_INTLFFS 0x6d754603 /* 'muF\3' */ +#define MUFS_DCOFS 0x6d754604 /* 'muF\4' */ +#define MUFS_DCFFS 0x6d754605 /* 'muF\5' */ + +#define T_SHORT 2 +#define T_LIST 16 +#define T_DATA 8 + +#define ST_LINKFILE -4 +#define ST_FILE -3 +#define ST_ROOT 1 +#define ST_USERDIR 2 +#define ST_SOFTLINK 3 +#define ST_LINKDIR 4 + +#define AFFS_ROOT_BMAPS 25 + +/* Seconds since Amiga epoch of 1978/01/01 to UNIX */ +#define AFFS_EPOCH_DELTA ((8 * 365 + 2) * 86400LL) + +struct affs_date { + __be32 days; + __be32 mins; + __be32 ticks; +}; + +struct affs_short_date { + __be16 days; + __be16 mins; + __be16 ticks; +}; + +struct affs_root_head { + __be32 ptype; + /* The following fields are not used, but kept as documentation. */ + __be32 spare1; + __be32 spare2; + __be32 hash_size; + __be32 spare3; + __be32 checksum; + __be32 hashtable[]; +}; + +struct affs_root_tail { + __be32 bm_flag; + __be32 bm_blk[AFFS_ROOT_BMAPS]; + __be32 bm_ext; + struct affs_date root_change; + u8 disk_name[32]; + __be32 spare1; + __be32 spare2; + struct affs_date disk_change; + struct affs_date disk_create; + __be32 spare3; + __be32 spare4; + __be32 dcache; + __be32 stype; +}; + +struct affs_head { + __be32 ptype; + __be32 key; + __be32 block_count; + __be32 spare1; + __be32 first_data; + __be32 checksum; + __be32 table[]; +}; + +struct affs_tail { + __be32 spare1; + __be16 uid; + __be16 gid; + __be32 protect; + __be32 size; + u8 comment[92]; + struct affs_date change; + u8 name[32]; + __be32 spare2; + __be32 original; + __be32 link_chain; + __be32 spare[5]; + __be32 hash_chain; + __be32 parent; + __be32 extension; + __be32 stype; +}; + +struct slink_front +{ + __be32 ptype; + __be32 key; + __be32 spare1[3]; + __be32 checksum; + u8 symname[]; /* depends on block size */ +}; + +struct affs_data_head +{ + __be32 ptype; + __be32 key; + __be32 sequence; + __be32 size; + __be32 next; + __be32 checksum; + u8 data[]; /* depends on block size */ +}; + +/* Permission bits */ + +#define FIBF_OTR_READ 0x8000 +#define FIBF_OTR_WRITE 0x4000 +#define FIBF_OTR_EXECUTE 0x2000 +#define FIBF_OTR_DELETE 0x1000 +#define FIBF_GRP_READ 0x0800 +#define FIBF_GRP_WRITE 0x0400 +#define FIBF_GRP_EXECUTE 0x0200 +#define FIBF_GRP_DELETE 0x0100 + +#define FIBF_HIDDEN 0x0080 +#define FIBF_SCRIPT 0x0040 +#define FIBF_PURE 0x0020 /* no use under linux */ +#define FIBF_ARCHIVED 0x0010 /* never set, always cleared on write */ +#define FIBF_NOREAD 0x0008 /* 0 means allowed */ +#define FIBF_NOWRITE 0x0004 /* 0 means allowed */ +#define FIBF_NOEXECUTE 0x0002 /* 0 means allowed, ignored under linux */ +#define FIBF_NODELETE 0x0001 /* 0 means allowed */ + +#define FIBF_OWNER 0x000F /* Bits pertaining to owner */ +#define FIBF_MASK 0xEE0E /* Bits modified by Linux */ + +#endif diff --git a/fs/affs/bitmap.c b/fs/affs/bitmap.c new file mode 100644 index 000000000..40bc4ce6a --- /dev/null +++ b/fs/affs/bitmap.c @@ -0,0 +1,365 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/affs/bitmap.c + * + * (c) 1996 Hans-Joachim Widmaier + * + * bitmap.c contains the code that handles all bitmap related stuff - + * block allocation, deallocation, calculation of free space. + */ + +#include +#include "affs.h" + +u32 +affs_count_free_blocks(struct super_block *sb) +{ + struct affs_bm_info *bm; + u32 free; + int i; + + pr_debug("%s()\n", __func__); + + if (sb_rdonly(sb)) + return 0; + + mutex_lock(&AFFS_SB(sb)->s_bmlock); + + bm = AFFS_SB(sb)->s_bitmap; + free = 0; + for (i = AFFS_SB(sb)->s_bmap_count; i > 0; bm++, i--) + free += bm->bm_free; + + mutex_unlock(&AFFS_SB(sb)->s_bmlock); + + return free; +} + +void +affs_free_block(struct super_block *sb, u32 block) +{ + struct affs_sb_info *sbi = AFFS_SB(sb); + struct affs_bm_info *bm; + struct buffer_head *bh; + u32 blk, bmap, bit, mask, tmp; + __be32 *data; + + pr_debug("%s(%u)\n", __func__, block); + + if (block > sbi->s_partition_size) + goto err_range; + + blk = block - sbi->s_reserved; + bmap = blk / sbi->s_bmap_bits; + bit = blk % sbi->s_bmap_bits; + bm = &sbi->s_bitmap[bmap]; + + mutex_lock(&sbi->s_bmlock); + + bh = sbi->s_bmap_bh; + if (sbi->s_last_bmap != bmap) { + affs_brelse(bh); + bh = affs_bread(sb, bm->bm_key); + if (!bh) + goto err_bh_read; + sbi->s_bmap_bh = bh; + sbi->s_last_bmap = bmap; + } + + mask = 1 << (bit & 31); + data = (__be32 *)bh->b_data + bit / 32 + 1; + + /* mark block free */ + tmp = be32_to_cpu(*data); + if (tmp & mask) + goto err_free; + *data = cpu_to_be32(tmp | mask); + + /* fix checksum */ + tmp = be32_to_cpu(*(__be32 *)bh->b_data); + *(__be32 *)bh->b_data = cpu_to_be32(tmp - mask); + + mark_buffer_dirty(bh); + affs_mark_sb_dirty(sb); + bm->bm_free++; + + mutex_unlock(&sbi->s_bmlock); + return; + +err_free: + affs_warning(sb,"affs_free_block","Trying to free block %u which is already free", block); + mutex_unlock(&sbi->s_bmlock); + return; + +err_bh_read: + affs_error(sb,"affs_free_block","Cannot read bitmap block %u", bm->bm_key); + sbi->s_bmap_bh = NULL; + sbi->s_last_bmap = ~0; + mutex_unlock(&sbi->s_bmlock); + return; + +err_range: + affs_error(sb, "affs_free_block","Block %u outside partition", block); +} + +/* + * Allocate a block in the given allocation zone. + * Since we have to byte-swap the bitmap on little-endian + * machines, this is rather expensive. Therefore we will + * preallocate up to 16 blocks from the same word, if + * possible. We are not doing preallocations in the + * header zone, though. + */ + +u32 +affs_alloc_block(struct inode *inode, u32 goal) +{ + struct super_block *sb; + struct affs_sb_info *sbi; + struct affs_bm_info *bm; + struct buffer_head *bh; + __be32 *data, *enddata; + u32 blk, bmap, bit, mask, mask2, tmp; + int i; + + sb = inode->i_sb; + sbi = AFFS_SB(sb); + + pr_debug("balloc(inode=%llu,goal=%u): ", inode->i_ino, goal); + + if (AFFS_I(inode)->i_pa_cnt) { + pr_debug("%d\n", AFFS_I(inode)->i_lastalloc+1); + AFFS_I(inode)->i_pa_cnt--; + return ++AFFS_I(inode)->i_lastalloc; + } + + if (!goal || goal > sbi->s_partition_size) { + if (goal) + affs_warning(sb, "affs_balloc", "invalid goal %d", goal); + //if (!AFFS_I(inode)->i_last_block) + // affs_warning(sb, "affs_balloc", "no last alloc block"); + goal = sbi->s_reserved; + } + + blk = goal - sbi->s_reserved; + bmap = blk / sbi->s_bmap_bits; + bm = &sbi->s_bitmap[bmap]; + + mutex_lock(&sbi->s_bmlock); + + if (bm->bm_free) + goto find_bmap_bit; + +find_bmap: + /* search for the next bmap buffer with free bits */ + i = sbi->s_bmap_count; + do { + if (--i < 0) + goto err_full; + bmap++; + bm++; + if (bmap < sbi->s_bmap_count) + continue; + /* restart search at zero */ + bmap = 0; + bm = sbi->s_bitmap; + } while (!bm->bm_free); + blk = bmap * sbi->s_bmap_bits; + +find_bmap_bit: + + bh = sbi->s_bmap_bh; + if (sbi->s_last_bmap != bmap) { + affs_brelse(bh); + bh = affs_bread(sb, bm->bm_key); + if (!bh) + goto err_bh_read; + sbi->s_bmap_bh = bh; + sbi->s_last_bmap = bmap; + } + + /* find an unused block in this bitmap block */ + bit = blk % sbi->s_bmap_bits; + data = (__be32 *)bh->b_data + bit / 32 + 1; + enddata = (__be32 *)((u8 *)bh->b_data + sb->s_blocksize); + mask = ~0UL << (bit & 31); + blk &= ~31UL; + + tmp = be32_to_cpu(*data); + if (tmp & mask) + goto find_bit; + + /* scan the rest of the buffer */ + do { + blk += 32; + if (++data >= enddata) + /* didn't find something, can only happen + * if scan didn't start at 0, try next bmap + */ + goto find_bmap; + } while (!*data); + tmp = be32_to_cpu(*data); + mask = ~0; + +find_bit: + /* finally look for a free bit in the word */ + bit = ffs(tmp & mask) - 1; + blk += bit + sbi->s_reserved; + mask2 = mask = 1 << (bit & 31); + AFFS_I(inode)->i_lastalloc = blk; + + /* prealloc as much as possible within this word */ + while ((mask2 <<= 1)) { + if (!(tmp & mask2)) + break; + AFFS_I(inode)->i_pa_cnt++; + mask |= mask2; + } + bm->bm_free -= AFFS_I(inode)->i_pa_cnt + 1; + + *data = cpu_to_be32(tmp & ~mask); + + /* fix checksum */ + tmp = be32_to_cpu(*(__be32 *)bh->b_data); + *(__be32 *)bh->b_data = cpu_to_be32(tmp + mask); + + mark_buffer_dirty(bh); + affs_mark_sb_dirty(sb); + + mutex_unlock(&sbi->s_bmlock); + + pr_debug("%d\n", blk); + return blk; + +err_bh_read: + affs_error(sb,"affs_read_block","Cannot read bitmap block %u", bm->bm_key); + sbi->s_bmap_bh = NULL; + sbi->s_last_bmap = ~0; +err_full: + mutex_unlock(&sbi->s_bmlock); + pr_debug("failed\n"); + return 0; +} + +int affs_init_bitmap(struct super_block *sb, int *flags) +{ + struct affs_bm_info *bm; + struct buffer_head *bmap_bh = NULL, *bh = NULL; + __be32 *bmap_blk; + u32 size, blk, end, offset, mask; + int i, res = 0; + struct affs_sb_info *sbi = AFFS_SB(sb); + + if (*flags & SB_RDONLY) + return 0; + + if (!AFFS_ROOT_TAIL(sb, sbi->s_root_bh)->bm_flag) { + pr_notice("Bitmap invalid - mounting %s read only\n", sb->s_id); + *flags |= SB_RDONLY; + return 0; + } + + sbi->s_last_bmap = ~0; + sbi->s_bmap_bh = NULL; + sbi->s_bmap_bits = sb->s_blocksize * 8 - 32; + sbi->s_bmap_count = (sbi->s_partition_size - sbi->s_reserved + + sbi->s_bmap_bits - 1) / sbi->s_bmap_bits; + size = sbi->s_bmap_count * sizeof(*bm); + bm = sbi->s_bitmap = kzalloc(size, GFP_KERNEL); + if (!sbi->s_bitmap) { + pr_err("Bitmap allocation failed\n"); + return -ENOMEM; + } + + bmap_blk = (__be32 *)sbi->s_root_bh->b_data; + blk = sb->s_blocksize / 4 - 49; + end = blk + 25; + + for (i = sbi->s_bmap_count; i > 0; bm++, i--) { + affs_brelse(bh); + + bm->bm_key = be32_to_cpu(bmap_blk[blk]); + bh = affs_bread(sb, bm->bm_key); + if (!bh) { + pr_err("Cannot read bitmap\n"); + res = -EIO; + goto out; + } + if (affs_checksum_block(sb, bh)) { + pr_warn("Bitmap %u invalid - mounting %s read only.\n", + bm->bm_key, sb->s_id); + *flags |= SB_RDONLY; + goto out; + } + pr_debug("read bitmap block %d: %d\n", blk, bm->bm_key); + bm->bm_free = memweight(bh->b_data + 4, sb->s_blocksize - 4); + + /* Don't try read the extension if this is the last block, + * but we also need the right bm pointer below + */ + if (++blk < end || i == 1) + continue; + if (bmap_bh) + affs_brelse(bmap_bh); + bmap_bh = affs_bread(sb, be32_to_cpu(bmap_blk[blk])); + if (!bmap_bh) { + pr_err("Cannot read bitmap extension\n"); + res = -EIO; + goto out; + } + bmap_blk = (__be32 *)bmap_bh->b_data; + blk = 0; + end = sb->s_blocksize / 4 - 1; + } + + offset = (sbi->s_partition_size - sbi->s_reserved) % sbi->s_bmap_bits; + mask = ~(0xFFFFFFFFU << (offset & 31)); + pr_debug("last word: %d %d %d\n", offset, offset / 32 + 1, mask); + offset = offset / 32 + 1; + + if (mask) { + u32 old, new; + + /* Mark unused bits in the last word as allocated */ + old = be32_to_cpu(((__be32 *)bh->b_data)[offset]); + new = old & mask; + //if (old != new) { + ((__be32 *)bh->b_data)[offset] = cpu_to_be32(new); + /* fix checksum */ + //new -= old; + //old = be32_to_cpu(*(__be32 *)bh->b_data); + //*(__be32 *)bh->b_data = cpu_to_be32(old - new); + //mark_buffer_dirty(bh); + //} + /* correct offset for the bitmap count below */ + //offset++; + } + while (++offset < sb->s_blocksize / 4) + ((__be32 *)bh->b_data)[offset] = 0; + ((__be32 *)bh->b_data)[0] = 0; + ((__be32 *)bh->b_data)[0] = cpu_to_be32(-affs_checksum_block(sb, bh)); + mark_buffer_dirty(bh); + + /* recalculate bitmap count for last block */ + bm--; + bm->bm_free = memweight(bh->b_data + 4, sb->s_blocksize - 4); + +out: + affs_brelse(bh); + affs_brelse(bmap_bh); + return res; +} + +void affs_free_bitmap(struct super_block *sb) +{ + struct affs_sb_info *sbi = AFFS_SB(sb); + + if (!sbi->s_bitmap) + return; + + affs_brelse(sbi->s_bmap_bh); + sbi->s_bmap_bh = NULL; + sbi->s_last_bmap = ~0; + kfree(sbi->s_bitmap); + sbi->s_bitmap = NULL; +} diff --git a/fs/affs/dir.c b/fs/affs/dir.c new file mode 100644 index 000000000..d2b897ebb --- /dev/null +++ b/fs/affs/dir.c @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/affs/dir.c + * + * (c) 1996 Hans-Joachim Widmaier - Rewritten + * + * (C) 1993 Ray Burr - Modified for Amiga FFS filesystem. + * + * (C) 1992 Eric Youngdale Modified for ISO 9660 filesystem. + * + * (C) 1991 Linus Torvalds - minix filesystem + * + * affs directory handling functions + * + */ + +#include +#include +#include "affs.h" + +struct affs_dir_data { + unsigned long ino; + u64 cookie; +}; + +static int affs_readdir(struct file *, struct dir_context *); + +static loff_t affs_dir_llseek(struct file *file, loff_t offset, int whence) +{ + struct affs_dir_data *data = file->private_data; + + return generic_llseek_cookie(file, offset, whence, &data->cookie); +} + +static int affs_dir_open(struct inode *inode, struct file *file) +{ + struct affs_dir_data *data; + + data = kzalloc_obj(struct affs_dir_data); + if (!data) + return -ENOMEM; + file->private_data = data; + return 0; +} + +static int affs_dir_release(struct inode *inode, struct file *file) +{ + kfree(file->private_data); + return 0; +} + +const struct file_operations affs_dir_operations = { + .open = affs_dir_open, + .read = generic_read_dir, + .llseek = affs_dir_llseek, + .iterate_shared = affs_readdir, + .fsync = affs_file_fsync, + .release = affs_dir_release, + .setlease = generic_setlease, +}; + +/* + * directories can handle most operations... + */ +const struct inode_operations affs_dir_inode_operations = { + .create = affs_create, + .lookup = affs_lookup, + .link = affs_link, + .unlink = affs_unlink, + .symlink = affs_symlink, + .mkdir = affs_mkdir, + .rmdir = affs_rmdir, + .rename = affs_rename2, + .setattr = affs_setattr, +}; + +static int +affs_readdir(struct file *file, struct dir_context *ctx) +{ + struct inode *inode = file_inode(file); + struct affs_dir_data *data = file->private_data; + struct super_block *sb = inode->i_sb; + struct buffer_head *dir_bh = NULL; + struct buffer_head *fh_bh = NULL; + unsigned char *name; + int namelen; + u32 i; + int hash_pos; + int chain_pos; + u32 ino; + int error = 0; + + pr_debug("%s(ino=%llu,f_pos=%llx)\n", __func__, inode->i_ino, ctx->pos); + + if (ctx->pos < 2) { + data->ino = 0; + if (!dir_emit_dots(file, ctx)) + return 0; + } + + affs_lock_dir(inode); + chain_pos = (ctx->pos - 2) & 0xffff; + hash_pos = (ctx->pos - 2) >> 16; + if (chain_pos == 0xffff) { + affs_warning(sb, "readdir", "More than 65535 entries in chain"); + chain_pos = 0; + hash_pos++; + ctx->pos = ((hash_pos << 16) | chain_pos) + 2; + } + dir_bh = affs_bread(sb, inode->i_ino); + if (!dir_bh) + goto out_unlock_dir; + + /* If the directory hasn't changed since the last call to readdir(), + * we can jump directly to where we left off. + */ + ino = data->ino; + if (ino && inode_eq_iversion(inode, data->cookie)) { + pr_debug("readdir() left off=%d\n", ino); + goto inside; + } + if (hash_pos >= AFFS_SB(sb)->s_hashsize) + goto done; + + ino = be32_to_cpu(AFFS_HEAD(dir_bh)->table[hash_pos]); + for (i = 0; ino && i < chain_pos; i++) { + fh_bh = affs_bread(sb, ino); + if (!fh_bh) { + affs_error(sb, "readdir","Cannot read block %d", i); + error = -EIO; + goto out_brelse_dir; + } + ino = be32_to_cpu(AFFS_TAIL(sb, fh_bh)->hash_chain); + affs_brelse(fh_bh); + fh_bh = NULL; + } + if (ino) + goto inside; + hash_pos++; + + for (; hash_pos < AFFS_SB(sb)->s_hashsize; hash_pos++) { + ino = be32_to_cpu(AFFS_HEAD(dir_bh)->table[hash_pos]); + if (!ino) + continue; + ctx->pos = (hash_pos << 16) + 2; +inside: + do { + fh_bh = affs_bread(sb, ino); + if (!fh_bh) { + affs_error(sb, "readdir", + "Cannot read block %d", ino); + break; + } + + namelen = min(AFFS_TAIL(sb, fh_bh)->name[0], + (u8)AFFSNAMEMAX); + name = AFFS_TAIL(sb, fh_bh)->name + 1; + pr_debug("readdir(): dir_emit(\"%.*s\", ino=%u), hash=%d, f_pos=%llx\n", + namelen, name, ino, hash_pos, ctx->pos); + + if (!dir_emit(ctx, name, namelen, ino, DT_UNKNOWN)) + goto done; + ctx->pos++; + ino = be32_to_cpu(AFFS_TAIL(sb, fh_bh)->hash_chain); + affs_brelse(fh_bh); + fh_bh = NULL; + } while (ino); + } +done: + data->cookie = inode_query_iversion(inode); + data->ino = ino; + affs_brelse(fh_bh); + +out_brelse_dir: + affs_brelse(dir_bh); + +out_unlock_dir: + affs_unlock_dir(inode); + return error; +} diff --git a/fs/affs/file.c b/fs/affs/file.c new file mode 100644 index 000000000..23e088a7e --- /dev/null +++ b/fs/affs/file.c @@ -0,0 +1,1017 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/affs/file.c + * + * (c) 1996 Hans-Joachim Widmaier - Rewritten + * + * (C) 1993 Ray Burr - Modified for Amiga FFS filesystem. + * + * (C) 1992 Eric Youngdale Modified for ISO 9660 filesystem. + * + * (C) 1991 Linus Torvalds - minix filesystem + * + * affs regular file handling primitives + */ + +#include +#include +#include +#include +#include "affs.h" + +static struct buffer_head *affs_get_extblock_slow(struct inode *inode, u32 ext); + +static int +affs_file_open(struct inode *inode, struct file *filp) +{ + pr_debug("open(%llu,%d)\n", + inode->i_ino, atomic_read(&AFFS_I(inode)->i_opencnt)); + atomic_inc(&AFFS_I(inode)->i_opencnt); + return 0; +} + +static int +affs_file_release(struct inode *inode, struct file *filp) +{ + pr_debug("release(%llu, %d)\n", + inode->i_ino, atomic_read(&AFFS_I(inode)->i_opencnt)); + + if (atomic_dec_and_test(&AFFS_I(inode)->i_opencnt)) { + inode_lock(inode); + if (inode->i_size != AFFS_I(inode)->mmu_private) + affs_truncate(inode); + affs_free_prealloc(inode); + inode_unlock(inode); + } + + return 0; +} + +static int +affs_grow_extcache(struct inode *inode, u32 lc_idx) +{ + struct super_block *sb = inode->i_sb; + struct buffer_head *bh; + u32 lc_max; + int i, j, key; + + if (!AFFS_I(inode)->i_lc) { + char *ptr = (char *)get_zeroed_page(GFP_NOFS); + if (!ptr) + return -ENOMEM; + AFFS_I(inode)->i_lc = (u32 *)ptr; + AFFS_I(inode)->i_ac = (struct affs_ext_key *)(ptr + AFFS_CACHE_SIZE / 2); + } + + lc_max = AFFS_LC_SIZE << AFFS_I(inode)->i_lc_shift; + + if (AFFS_I(inode)->i_extcnt > lc_max) { + u32 lc_shift, lc_mask, tmp, off; + + /* need to recalculate linear cache, start from old size */ + lc_shift = AFFS_I(inode)->i_lc_shift; + tmp = (AFFS_I(inode)->i_extcnt / AFFS_LC_SIZE) >> lc_shift; + for (; tmp; tmp >>= 1) + lc_shift++; + lc_mask = (1 << lc_shift) - 1; + + /* fix idx and old size to new shift */ + lc_idx >>= (lc_shift - AFFS_I(inode)->i_lc_shift); + AFFS_I(inode)->i_lc_size >>= (lc_shift - AFFS_I(inode)->i_lc_shift); + + /* first shrink old cache to make more space */ + off = 1 << (lc_shift - AFFS_I(inode)->i_lc_shift); + for (i = 1, j = off; j < AFFS_LC_SIZE; i++, j += off) + AFFS_I(inode)->i_ac[i] = AFFS_I(inode)->i_ac[j]; + + AFFS_I(inode)->i_lc_shift = lc_shift; + AFFS_I(inode)->i_lc_mask = lc_mask; + } + + /* fill cache to the needed index */ + i = AFFS_I(inode)->i_lc_size; + AFFS_I(inode)->i_lc_size = lc_idx + 1; + for (; i <= lc_idx; i++) { + if (!i) { + AFFS_I(inode)->i_lc[0] = inode->i_ino; + continue; + } + key = AFFS_I(inode)->i_lc[i - 1]; + j = AFFS_I(inode)->i_lc_mask + 1; + // unlock cache + for (; j > 0; j--) { + bh = affs_bread(sb, key); + if (!bh) + goto err; + key = be32_to_cpu(AFFS_TAIL(sb, bh)->extension); + affs_brelse(bh); + } + // lock cache + AFFS_I(inode)->i_lc[i] = key; + } + + return 0; + +err: + // lock cache + return -EIO; +} + +static struct buffer_head * +affs_alloc_extblock(struct inode *inode, struct buffer_head *bh, u32 ext) +{ + struct super_block *sb = inode->i_sb; + struct buffer_head *new_bh; + u32 blocknr, tmp; + + blocknr = affs_alloc_block(inode, bh->b_blocknr); + if (!blocknr) + return ERR_PTR(-ENOSPC); + + new_bh = affs_getzeroblk(sb, blocknr); + if (!new_bh) { + affs_free_block(sb, blocknr); + return ERR_PTR(-EIO); + } + + AFFS_HEAD(new_bh)->ptype = cpu_to_be32(T_LIST); + AFFS_HEAD(new_bh)->key = cpu_to_be32(blocknr); + AFFS_TAIL(sb, new_bh)->stype = cpu_to_be32(ST_FILE); + AFFS_TAIL(sb, new_bh)->parent = cpu_to_be32(inode->i_ino); + affs_fix_checksum(sb, new_bh); + + mark_buffer_dirty(new_bh); + + tmp = be32_to_cpu(AFFS_TAIL(sb, bh)->extension); + if (tmp) + affs_warning(sb, "alloc_ext", "previous extension set (%x)", tmp); + AFFS_TAIL(sb, bh)->extension = cpu_to_be32(blocknr); + affs_adjust_checksum(bh, blocknr - tmp); + mark_buffer_dirty(bh); + + AFFS_I(inode)->i_extcnt++; + mark_inode_dirty(inode); + + return new_bh; +} + +static inline struct buffer_head * +affs_get_extblock(struct inode *inode, u32 ext) +{ + /* inline the simplest case: same extended block as last time */ + struct buffer_head *bh = AFFS_I(inode)->i_ext_bh; + if (ext == AFFS_I(inode)->i_ext_last) + get_bh(bh); + else + /* we have to do more (not inlined) */ + bh = affs_get_extblock_slow(inode, ext); + + return bh; +} + +static struct buffer_head * +affs_get_extblock_slow(struct inode *inode, u32 ext) +{ + struct super_block *sb = inode->i_sb; + struct buffer_head *bh; + u32 ext_key; + u32 lc_idx, lc_off, ac_idx; + u32 tmp, idx; + + if (ext == AFFS_I(inode)->i_ext_last + 1) { + /* read the next extended block from the current one */ + bh = AFFS_I(inode)->i_ext_bh; + ext_key = be32_to_cpu(AFFS_TAIL(sb, bh)->extension); + if (ext < AFFS_I(inode)->i_extcnt) + goto read_ext; + BUG_ON(ext > AFFS_I(inode)->i_extcnt); + bh = affs_alloc_extblock(inode, bh, ext); + if (IS_ERR(bh)) + return bh; + goto store_ext; + } + + if (ext == 0) { + /* we seek back to the file header block */ + ext_key = inode->i_ino; + goto read_ext; + } + + if (ext >= AFFS_I(inode)->i_extcnt) { + struct buffer_head *prev_bh; + + /* allocate a new extended block */ + BUG_ON(ext > AFFS_I(inode)->i_extcnt); + + /* get previous extended block */ + prev_bh = affs_get_extblock(inode, ext - 1); + if (IS_ERR(prev_bh)) + return prev_bh; + bh = affs_alloc_extblock(inode, prev_bh, ext); + affs_brelse(prev_bh); + if (IS_ERR(bh)) + return bh; + goto store_ext; + } + +again: + /* check if there is an extended cache and whether it's large enough */ + lc_idx = ext >> AFFS_I(inode)->i_lc_shift; + lc_off = ext & AFFS_I(inode)->i_lc_mask; + + if (lc_idx >= AFFS_I(inode)->i_lc_size) { + int err; + + err = affs_grow_extcache(inode, lc_idx); + if (err) + return ERR_PTR(err); + goto again; + } + + /* every n'th key we find in the linear cache */ + if (!lc_off) { + ext_key = AFFS_I(inode)->i_lc[lc_idx]; + goto read_ext; + } + + /* maybe it's still in the associative cache */ + ac_idx = (ext - lc_idx - 1) & AFFS_AC_MASK; + if (AFFS_I(inode)->i_ac[ac_idx].ext == ext) { + ext_key = AFFS_I(inode)->i_ac[ac_idx].key; + goto read_ext; + } + + /* try to find one of the previous extended blocks */ + tmp = ext; + idx = ac_idx; + while (--tmp, --lc_off > 0) { + idx = (idx - 1) & AFFS_AC_MASK; + if (AFFS_I(inode)->i_ac[idx].ext == tmp) { + ext_key = AFFS_I(inode)->i_ac[idx].key; + goto find_ext; + } + } + + /* fall back to the linear cache */ + ext_key = AFFS_I(inode)->i_lc[lc_idx]; +find_ext: + /* read all extended blocks until we find the one we need */ + //unlock cache + do { + bh = affs_bread(sb, ext_key); + if (!bh) + goto err_bread; + ext_key = be32_to_cpu(AFFS_TAIL(sb, bh)->extension); + affs_brelse(bh); + tmp++; + } while (tmp < ext); + //lock cache + + /* store it in the associative cache */ + // recalculate ac_idx? + AFFS_I(inode)->i_ac[ac_idx].ext = ext; + AFFS_I(inode)->i_ac[ac_idx].key = ext_key; + +read_ext: + /* finally read the right extended block */ + //unlock cache + bh = affs_bread(sb, ext_key); + if (!bh) + goto err_bread; + //lock cache + +store_ext: + /* release old cached extended block and store the new one */ + affs_brelse(AFFS_I(inode)->i_ext_bh); + AFFS_I(inode)->i_ext_last = ext; + AFFS_I(inode)->i_ext_bh = bh; + get_bh(bh); + + return bh; + +err_bread: + affs_brelse(bh); + return ERR_PTR(-EIO); +} + +static int +affs_get_block(struct inode *inode, sector_t block, struct buffer_head *bh_result, int create) +{ + struct super_block *sb = inode->i_sb; + struct buffer_head *ext_bh; + u32 ext; + + pr_debug("%s(%llu, %llu)\n", __func__, inode->i_ino, + (unsigned long long)block); + + BUG_ON(block > (sector_t)0x7fffffffUL); + + if (block >= AFFS_I(inode)->i_blkcnt) { + if (block > AFFS_I(inode)->i_blkcnt || !create) + goto err_big; + } else + create = 0; + + //lock cache + affs_lock_ext(inode); + + ext = (u32)block / AFFS_SB(sb)->s_hashsize; + block -= ext * AFFS_SB(sb)->s_hashsize; + ext_bh = affs_get_extblock(inode, ext); + if (IS_ERR(ext_bh)) + goto err_ext; + map_bh(bh_result, sb, (sector_t)be32_to_cpu(AFFS_BLOCK(sb, ext_bh, block))); + + if (create) { + u32 blocknr = affs_alloc_block(inode, ext_bh->b_blocknr); + if (!blocknr) + goto err_alloc; + set_buffer_new(bh_result); + AFFS_I(inode)->mmu_private += AFFS_SB(sb)->s_data_blksize; + AFFS_I(inode)->i_blkcnt++; + + /* store new block */ + if (bh_result->b_blocknr) + affs_warning(sb, "get_block", + "block already set (%llx)", + (unsigned long long)bh_result->b_blocknr); + AFFS_BLOCK(sb, ext_bh, block) = cpu_to_be32(blocknr); + AFFS_HEAD(ext_bh)->block_count = cpu_to_be32(block + 1); + affs_adjust_checksum(ext_bh, blocknr - bh_result->b_blocknr + 1); + bh_result->b_blocknr = blocknr; + + if (!block) { + /* insert first block into header block */ + u32 tmp = be32_to_cpu(AFFS_HEAD(ext_bh)->first_data); + if (tmp) + affs_warning(sb, "get_block", "first block already set (%d)", tmp); + AFFS_HEAD(ext_bh)->first_data = cpu_to_be32(blocknr); + affs_adjust_checksum(ext_bh, blocknr - tmp); + } + } + + affs_brelse(ext_bh); + //unlock cache + affs_unlock_ext(inode); + return 0; + +err_big: + affs_error(inode->i_sb, "get_block", "strange block request %llu", + (unsigned long long)block); + return -EIO; +err_ext: + // unlock cache + affs_unlock_ext(inode); + return PTR_ERR(ext_bh); +err_alloc: + brelse(ext_bh); + clear_buffer_mapped(bh_result); + bh_result->b_bdev = NULL; + // unlock cache + affs_unlock_ext(inode); + return -ENOSPC; +} + +static int affs_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + return mpage_writepages(mapping, wbc, affs_get_block); +} + +static int affs_read_folio(struct file *file, struct folio *folio) +{ + return block_read_full_folio(folio, affs_get_block); +} + +static void affs_write_failed(struct address_space *mapping, loff_t to) +{ + struct inode *inode = mapping->host; + + if (to > inode->i_size) { + truncate_pagecache(inode, inode->i_size); + affs_truncate(inode); + } +} + +static ssize_t +affs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) +{ + struct file *file = iocb->ki_filp; + struct address_space *mapping = file->f_mapping; + struct inode *inode = mapping->host; + size_t count = iov_iter_count(iter); + loff_t offset = iocb->ki_pos; + ssize_t ret; + + if (iov_iter_rw(iter) == WRITE) { + loff_t size = offset + count; + + if (AFFS_I(inode)->mmu_private < size) + return 0; + } + + ret = blockdev_direct_IO(iocb, inode, iter, affs_get_block); + if (ret < 0 && iov_iter_rw(iter) == WRITE) + affs_write_failed(mapping, offset + count); + return ret; +} + +static int affs_write_begin(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, + struct folio **foliop, void **fsdata) +{ + int ret; + + ret = cont_write_begin(iocb, mapping, pos, len, foliop, fsdata, + affs_get_block, + &AFFS_I(mapping->host)->mmu_private); + if (unlikely(ret)) + affs_write_failed(mapping, pos + len); + + return ret; +} + +static int affs_write_end(const struct kiocb *iocb, + struct address_space *mapping, loff_t pos, + unsigned int len, unsigned int copied, + struct folio *folio, void *fsdata) +{ + struct inode *inode = mapping->host; + int ret; + + ret = generic_write_end(iocb, mapping, pos, len, copied, folio, fsdata); + + /* Clear Archived bit on file writes, as AmigaOS would do */ + if (AFFS_I(inode)->i_protect & FIBF_ARCHIVED) { + AFFS_I(inode)->i_protect &= ~FIBF_ARCHIVED; + mark_inode_dirty(inode); + } + + return ret; +} + +static sector_t _affs_bmap(struct address_space *mapping, sector_t block) +{ + return generic_block_bmap(mapping,block,affs_get_block); +} + +const struct address_space_operations affs_aops = { + .dirty_folio = block_dirty_folio, + .invalidate_folio = block_invalidate_folio, + .read_folio = affs_read_folio, + .writepages = affs_writepages, + .write_begin = affs_write_begin, + .write_end = affs_write_end, + .direct_IO = affs_direct_IO, + .migrate_folio = buffer_migrate_folio, + .bmap = _affs_bmap +}; + +static inline struct buffer_head * +affs_bread_ino(struct inode *inode, int block, int create) +{ + struct buffer_head *bh, tmp_bh; + int err; + + tmp_bh.b_state = 0; + err = affs_get_block(inode, block, &tmp_bh, create); + if (!err) { + bh = affs_bread(inode->i_sb, tmp_bh.b_blocknr); + if (bh) { + bh->b_state |= tmp_bh.b_state; + return bh; + } + err = -EIO; + } + return ERR_PTR(err); +} + +static inline struct buffer_head * +affs_getzeroblk_ino(struct inode *inode, int block) +{ + struct buffer_head *bh, tmp_bh; + int err; + + tmp_bh.b_state = 0; + err = affs_get_block(inode, block, &tmp_bh, 1); + if (!err) { + bh = affs_getzeroblk(inode->i_sb, tmp_bh.b_blocknr); + if (bh) { + bh->b_state |= tmp_bh.b_state; + return bh; + } + err = -EIO; + } + return ERR_PTR(err); +} + +static inline struct buffer_head * +affs_getemptyblk_ino(struct inode *inode, int block) +{ + struct buffer_head *bh, tmp_bh; + int err; + + tmp_bh.b_state = 0; + err = affs_get_block(inode, block, &tmp_bh, 1); + if (!err) { + bh = affs_getemptyblk(inode->i_sb, tmp_bh.b_blocknr); + if (bh) { + bh->b_state |= tmp_bh.b_state; + return bh; + } + err = -EIO; + } + return ERR_PTR(err); +} + +static int affs_do_read_folio_ofs(struct folio *folio, size_t to, int create) +{ + struct inode *inode = folio->mapping->host; + struct super_block *sb = inode->i_sb; + struct buffer_head *bh; + size_t pos = 0; + size_t bidx, boff, bsize; + u32 tmp; + + pr_debug("%s(%llu, %ld, 0, %zu)\n", __func__, inode->i_ino, + folio->index, to); + BUG_ON(to > folio_size(folio)); + bsize = AFFS_SB(sb)->s_data_blksize; + tmp = folio_pos(folio); + bidx = tmp / bsize; + boff = tmp % bsize; + + while (pos < to) { + bh = affs_bread_ino(inode, bidx, create); + if (IS_ERR(bh)) + return PTR_ERR(bh); + tmp = min(bsize - boff, to - pos); + BUG_ON(pos + tmp > to || tmp > bsize); + memcpy_to_folio(folio, pos, AFFS_DATA(bh) + boff, tmp); + affs_brelse(bh); + bidx++; + pos += tmp; + boff = 0; + } + return 0; +} + +static int +affs_extent_file_ofs(struct inode *inode, u32 newsize) +{ + struct super_block *sb = inode->i_sb; + struct buffer_head *bh, *prev_bh; + u32 bidx, boff; + u32 size, bsize; + u32 tmp; + + pr_debug("%s(%llu, %d)\n", __func__, inode->i_ino, newsize); + bsize = AFFS_SB(sb)->s_data_blksize; + bh = NULL; + size = AFFS_I(inode)->mmu_private; + bidx = size / bsize; + boff = size % bsize; + if (boff) { + bh = affs_bread_ino(inode, bidx, 0); + if (IS_ERR(bh)) + return PTR_ERR(bh); + tmp = min(bsize - boff, newsize - size); + BUG_ON(boff + tmp > bsize || tmp > bsize); + memset(AFFS_DATA(bh) + boff, 0, tmp); + be32_add_cpu(&AFFS_DATA_HEAD(bh)->size, tmp); + affs_fix_checksum(sb, bh); + mark_buffer_dirty(bh); + size += tmp; + bidx++; + } else if (bidx) { + bh = affs_bread_ino(inode, bidx - 1, 0); + if (IS_ERR(bh)) + return PTR_ERR(bh); + } + + while (size < newsize) { + prev_bh = bh; + bh = affs_getzeroblk_ino(inode, bidx); + if (IS_ERR(bh)) + goto out; + tmp = min(bsize, newsize - size); + BUG_ON(tmp > bsize); + AFFS_DATA_HEAD(bh)->ptype = cpu_to_be32(T_DATA); + AFFS_DATA_HEAD(bh)->key = cpu_to_be32(inode->i_ino); + AFFS_DATA_HEAD(bh)->sequence = cpu_to_be32(bidx + 1); + AFFS_DATA_HEAD(bh)->size = cpu_to_be32(tmp); + affs_fix_checksum(sb, bh); + bh->b_state &= ~(1UL << BH_New); + mark_buffer_dirty(bh); + if (prev_bh) { + u32 tmp_next = be32_to_cpu(AFFS_DATA_HEAD(prev_bh)->next); + + if (tmp_next) + affs_warning(sb, "extent_file_ofs", + "next block already set for %d (%d)", + bidx, tmp_next); + AFFS_DATA_HEAD(prev_bh)->next = cpu_to_be32(bh->b_blocknr); + affs_adjust_checksum(prev_bh, bh->b_blocknr - tmp_next); + mark_buffer_dirty(prev_bh); + affs_brelse(prev_bh); + } + size += bsize; + bidx++; + } + affs_brelse(bh); + inode->i_size = AFFS_I(inode)->mmu_private = newsize; + return 0; + +out: + inode->i_size = AFFS_I(inode)->mmu_private = newsize; + return PTR_ERR(bh); +} + +static int affs_read_folio_ofs(struct file *file, struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + size_t to; + int err; + + pr_debug("%s(%llu, %ld)\n", __func__, inode->i_ino, folio->index); + to = folio_size(folio); + if (folio_pos(folio) + to > inode->i_size) { + to = inode->i_size - folio_pos(folio); + folio_zero_segment(folio, to, folio_size(folio)); + } + + err = affs_do_read_folio_ofs(folio, to, 0); + if (!err) + folio_mark_uptodate(folio); + folio_unlock(folio); + return err; +} + +static int affs_write_begin_ofs(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, + struct folio **foliop, void **fsdata) +{ + struct inode *inode = mapping->host; + struct folio *folio; + pgoff_t index; + int err = 0; + + pr_debug("%s(%llu, %llu, %llu)\n", __func__, inode->i_ino, pos, + pos + len); + if (pos > AFFS_I(inode)->mmu_private) { + /* XXX: this probably leaves a too-big i_size in case of + * failure. Should really be updating i_size at write_end time + */ + err = affs_extent_file_ofs(inode, pos); + if (err) + return err; + } + + index = pos >> PAGE_SHIFT; + folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN, + mapping_gfp_mask(mapping)); + if (IS_ERR(folio)) + return PTR_ERR(folio); + *foliop = folio; + + if (folio_test_uptodate(folio)) + return 0; + + /* XXX: inefficient but safe in the face of short writes */ + err = affs_do_read_folio_ofs(folio, folio_size(folio), 1); + if (err) { + folio_unlock(folio); + folio_put(folio); + } + return err; +} + +static int affs_write_end_ofs(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct folio *folio, void *fsdata) +{ + struct inode *inode = mapping->host; + struct super_block *sb = inode->i_sb; + struct buffer_head *bh, *prev_bh; + char *data; + u32 bidx, boff, bsize; + unsigned from, to; + u32 tmp; + int written; + + from = pos & (PAGE_SIZE - 1); + to = from + len; + /* + * XXX: not sure if this can handle short copies (len < copied), but + * we don't have to, because the folio should always be uptodate here, + * due to write_begin. + */ + + pr_debug("%s(%llu, %llu, %llu)\n", __func__, inode->i_ino, pos, + pos + len); + bsize = AFFS_SB(sb)->s_data_blksize; + data = folio_address(folio); + + bh = NULL; + written = 0; + tmp = (folio->index << PAGE_SHIFT) + from; + bidx = tmp / bsize; + boff = tmp % bsize; + if (boff) { + bh = affs_bread_ino(inode, bidx, 0); + if (IS_ERR(bh)) { + written = PTR_ERR(bh); + goto err_first_bh; + } + tmp = min(bsize - boff, to - from); + BUG_ON(boff + tmp > bsize || tmp > bsize); + memcpy(AFFS_DATA(bh) + boff, data + from, tmp); + AFFS_DATA_HEAD(bh)->size = cpu_to_be32( + max(boff + tmp, be32_to_cpu(AFFS_DATA_HEAD(bh)->size))); + affs_fix_checksum(sb, bh); + mark_buffer_dirty(bh); + written += tmp; + from += tmp; + bidx++; + } else if (bidx) { + bh = affs_bread_ino(inode, bidx - 1, 0); + if (IS_ERR(bh)) { + written = PTR_ERR(bh); + goto err_first_bh; + } + } + while (from + bsize <= to) { + prev_bh = bh; + bh = affs_getemptyblk_ino(inode, bidx); + if (IS_ERR(bh)) + goto err_bh; + memcpy(AFFS_DATA(bh), data + from, bsize); + if (buffer_new(bh)) { + AFFS_DATA_HEAD(bh)->ptype = cpu_to_be32(T_DATA); + AFFS_DATA_HEAD(bh)->key = cpu_to_be32(inode->i_ino); + AFFS_DATA_HEAD(bh)->sequence = cpu_to_be32(bidx + 1); + AFFS_DATA_HEAD(bh)->size = cpu_to_be32(bsize); + AFFS_DATA_HEAD(bh)->next = 0; + bh->b_state &= ~(1UL << BH_New); + if (prev_bh) { + u32 tmp_next = be32_to_cpu(AFFS_DATA_HEAD(prev_bh)->next); + + if (tmp_next) + affs_warning(sb, "commit_write_ofs", + "next block already set for %d (%d)", + bidx, tmp_next); + AFFS_DATA_HEAD(prev_bh)->next = cpu_to_be32(bh->b_blocknr); + affs_adjust_checksum(prev_bh, bh->b_blocknr - tmp_next); + mark_buffer_dirty(prev_bh); + } + } + affs_brelse(prev_bh); + affs_fix_checksum(sb, bh); + mark_buffer_dirty(bh); + written += bsize; + from += bsize; + bidx++; + } + if (from < to) { + prev_bh = bh; + bh = affs_bread_ino(inode, bidx, 1); + if (IS_ERR(bh)) + goto err_bh; + tmp = min(bsize, to - from); + BUG_ON(tmp > bsize); + memcpy(AFFS_DATA(bh), data + from, tmp); + if (buffer_new(bh)) { + AFFS_DATA_HEAD(bh)->ptype = cpu_to_be32(T_DATA); + AFFS_DATA_HEAD(bh)->key = cpu_to_be32(inode->i_ino); + AFFS_DATA_HEAD(bh)->sequence = cpu_to_be32(bidx + 1); + AFFS_DATA_HEAD(bh)->size = cpu_to_be32(tmp); + AFFS_DATA_HEAD(bh)->next = 0; + bh->b_state &= ~(1UL << BH_New); + if (prev_bh) { + u32 tmp_next = be32_to_cpu(AFFS_DATA_HEAD(prev_bh)->next); + + if (tmp_next) + affs_warning(sb, "commit_write_ofs", + "next block already set for %d (%d)", + bidx, tmp_next); + AFFS_DATA_HEAD(prev_bh)->next = cpu_to_be32(bh->b_blocknr); + affs_adjust_checksum(prev_bh, bh->b_blocknr - tmp_next); + mark_buffer_dirty(prev_bh); + } + } else if (be32_to_cpu(AFFS_DATA_HEAD(bh)->size) < tmp) + AFFS_DATA_HEAD(bh)->size = cpu_to_be32(tmp); + affs_brelse(prev_bh); + affs_fix_checksum(sb, bh); + mark_buffer_dirty(bh); + written += tmp; + from += tmp; + bidx++; + } + folio_mark_uptodate(folio); + +done: + affs_brelse(bh); + tmp = (folio->index << PAGE_SHIFT) + from; + if (tmp > inode->i_size) + inode->i_size = AFFS_I(inode)->mmu_private = tmp; + + /* Clear Archived bit on file writes, as AmigaOS would do */ + if (AFFS_I(inode)->i_protect & FIBF_ARCHIVED) { + AFFS_I(inode)->i_protect &= ~FIBF_ARCHIVED; + mark_inode_dirty(inode); + } + +err_first_bh: + folio_unlock(folio); + folio_put(folio); + + return written; + +err_bh: + bh = prev_bh; + if (!written) + written = PTR_ERR(bh); + goto done; +} + +const struct address_space_operations affs_aops_ofs = { + .dirty_folio = block_dirty_folio, + .invalidate_folio = block_invalidate_folio, + .read_folio = affs_read_folio_ofs, + //.writepages = affs_writepages_ofs, + .write_begin = affs_write_begin_ofs, + .write_end = affs_write_end_ofs, + .migrate_folio = filemap_migrate_folio, +}; + +/* Free any preallocated blocks. */ + +void +affs_free_prealloc(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + + pr_debug("free_prealloc(ino=%llu)\n", inode->i_ino); + + while (AFFS_I(inode)->i_pa_cnt) { + AFFS_I(inode)->i_pa_cnt--; + affs_free_block(sb, ++AFFS_I(inode)->i_lastalloc); + } +} + +/* Truncate (or enlarge) a file to the requested size. */ + +void +affs_truncate(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + u32 ext, ext_key; + u32 last_blk, blkcnt, blk; + u32 size; + struct buffer_head *ext_bh; + int i; + + pr_debug("truncate(inode=%llu, oldsize=%llu, newsize=%llu)\n", + inode->i_ino, AFFS_I(inode)->mmu_private, inode->i_size); + + last_blk = 0; + ext = 0; + if (inode->i_size) { + last_blk = ((u32)inode->i_size - 1) / AFFS_SB(sb)->s_data_blksize; + ext = last_blk / AFFS_SB(sb)->s_hashsize; + } + + if (inode->i_size > AFFS_I(inode)->mmu_private) { + struct address_space *mapping = inode->i_mapping; + struct folio *folio; + void *fsdata = NULL; + loff_t isize = inode->i_size; + int res; + + res = mapping->a_ops->write_begin(NULL, mapping, isize, 0, &folio, &fsdata); + if (!res) + res = mapping->a_ops->write_end(NULL, mapping, isize, 0, 0, folio, fsdata); + else + inode->i_size = AFFS_I(inode)->mmu_private; + mark_inode_dirty(inode); + return; + } else if (inode->i_size == AFFS_I(inode)->mmu_private) + return; + + // lock cache + ext_bh = affs_get_extblock(inode, ext); + if (IS_ERR(ext_bh)) { + affs_warning(sb, "truncate", + "unexpected read error for ext block %u (%ld)", + ext, PTR_ERR(ext_bh)); + return; + } + if (AFFS_I(inode)->i_lc) { + /* clear linear cache */ + i = (ext + 1) >> AFFS_I(inode)->i_lc_shift; + if (AFFS_I(inode)->i_lc_size > i) { + AFFS_I(inode)->i_lc_size = i; + for (; i < AFFS_LC_SIZE; i++) + AFFS_I(inode)->i_lc[i] = 0; + } + /* clear associative cache */ + for (i = 0; i < AFFS_AC_SIZE; i++) + if (AFFS_I(inode)->i_ac[i].ext >= ext) + AFFS_I(inode)->i_ac[i].ext = 0; + } + ext_key = be32_to_cpu(AFFS_TAIL(sb, ext_bh)->extension); + + blkcnt = AFFS_I(inode)->i_blkcnt; + i = 0; + blk = last_blk; + if (inode->i_size) { + i = last_blk % AFFS_SB(sb)->s_hashsize + 1; + blk++; + } else + AFFS_HEAD(ext_bh)->first_data = 0; + AFFS_HEAD(ext_bh)->block_count = cpu_to_be32(i); + size = AFFS_SB(sb)->s_hashsize; + if (size > blkcnt - blk + i) + size = blkcnt - blk + i; + for (; i < size; i++, blk++) { + affs_free_block(sb, be32_to_cpu(AFFS_BLOCK(sb, ext_bh, i))); + AFFS_BLOCK(sb, ext_bh, i) = 0; + } + AFFS_TAIL(sb, ext_bh)->extension = 0; + affs_fix_checksum(sb, ext_bh); + mark_buffer_dirty(ext_bh); + affs_brelse(ext_bh); + + if (inode->i_size) { + AFFS_I(inode)->i_blkcnt = last_blk + 1; + AFFS_I(inode)->i_extcnt = ext + 1; + if (affs_test_opt(AFFS_SB(sb)->s_flags, SF_OFS)) { + struct buffer_head *bh = affs_bread_ino(inode, last_blk, 0); + u32 tmp; + if (IS_ERR(bh)) { + affs_warning(sb, "truncate", + "unexpected read error for last block %u (%ld)", + ext, PTR_ERR(bh)); + return; + } + tmp = be32_to_cpu(AFFS_DATA_HEAD(bh)->next); + AFFS_DATA_HEAD(bh)->next = 0; + affs_adjust_checksum(bh, -tmp); + affs_brelse(bh); + } + } else { + AFFS_I(inode)->i_blkcnt = 0; + AFFS_I(inode)->i_extcnt = 1; + } + AFFS_I(inode)->mmu_private = inode->i_size; + // unlock cache + + while (ext_key) { + ext_bh = affs_bread(sb, ext_key); + size = AFFS_SB(sb)->s_hashsize; + if (size > blkcnt - blk) + size = blkcnt - blk; + for (i = 0; i < size; i++, blk++) + affs_free_block(sb, be32_to_cpu(AFFS_BLOCK(sb, ext_bh, i))); + affs_free_block(sb, ext_key); + ext_key = be32_to_cpu(AFFS_TAIL(sb, ext_bh)->extension); + affs_brelse(ext_bh); + } + affs_free_prealloc(inode); +} + +int affs_file_fsync(struct file *filp, loff_t start, loff_t end, int datasync) +{ + struct inode *inode = filp->f_mapping->host; + int ret, err; + + err = file_write_and_wait_range(filp, start, end); + if (err) + return err; + + inode_lock(inode); + ret = write_inode_now(inode, 0); + err = sync_blockdev(inode->i_sb->s_bdev); + if (!ret) + ret = err; + inode_unlock(inode); + return ret; +} +const struct file_operations affs_file_operations = { + .llseek = generic_file_llseek, + .read_iter = generic_file_read_iter, + .write_iter = generic_file_write_iter, + .mmap_prepare = generic_file_mmap_prepare, + .open = affs_file_open, + .release = affs_file_release, + .fsync = affs_file_fsync, + .splice_read = filemap_splice_read, + .setlease = generic_setlease, +}; + +const struct inode_operations affs_file_inode_operations = { + .setattr = affs_setattr, +}; diff --git a/fs/affs/inode.c b/fs/affs/inode.c new file mode 100644 index 000000000..d4a3f381c --- /dev/null +++ b/fs/affs/inode.c @@ -0,0 +1,417 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/affs/inode.c + * + * (c) 1996 Hans-Joachim Widmaier - Rewritten + * + * (C) 1993 Ray Burr - Modified for Amiga FFS filesystem. + * + * (C) 1992 Eric Youngdale Modified for ISO9660 filesystem. + * + * (C) 1991 Linus Torvalds - minix filesystem + */ +#include +#include +#include +#include "affs.h" + +struct inode *affs_iget(struct super_block *sb, unsigned long ino) +{ + struct affs_sb_info *sbi = AFFS_SB(sb); + struct buffer_head *bh; + struct affs_tail *tail; + struct inode *inode; + u32 block; + u32 size; + u32 prot; + u16 id; + + inode = iget_locked(sb, ino); + if (!inode) + return ERR_PTR(-ENOMEM); + if (!(inode_state_read_once(inode) & I_NEW)) + return inode; + + pr_debug("affs_iget(%llu)\n", inode->i_ino); + + block = inode->i_ino; + bh = affs_bread(sb, block); + if (!bh) { + affs_warning(sb, "read_inode", "Cannot read block %d", block); + goto bad_inode; + } + if (affs_checksum_block(sb, bh) || be32_to_cpu(AFFS_HEAD(bh)->ptype) != T_SHORT) { + affs_warning(sb,"read_inode", + "Checksum or type (ptype=%d) error on inode %d", + AFFS_HEAD(bh)->ptype, block); + goto bad_inode; + } + + tail = AFFS_TAIL(sb, bh); + prot = be32_to_cpu(tail->protect); + + inode->i_size = 0; + set_nlink(inode, 1); + inode->i_mode = 0; + AFFS_I(inode)->i_extcnt = 1; + AFFS_I(inode)->i_ext_last = ~1; + AFFS_I(inode)->i_protect = prot; + atomic_set(&AFFS_I(inode)->i_opencnt, 0); + AFFS_I(inode)->i_blkcnt = 0; + AFFS_I(inode)->i_lc = NULL; + AFFS_I(inode)->i_lc_size = 0; + AFFS_I(inode)->i_lc_shift = 0; + AFFS_I(inode)->i_lc_mask = 0; + AFFS_I(inode)->i_ac = NULL; + AFFS_I(inode)->i_ext_bh = NULL; + AFFS_I(inode)->mmu_private = 0; + AFFS_I(inode)->i_lastalloc = 0; + AFFS_I(inode)->i_pa_cnt = 0; + + if (affs_test_opt(sbi->s_flags, SF_SETMODE)) + inode->i_mode = sbi->s_mode; + else + inode->i_mode = affs_prot_to_mode(prot); + + id = be16_to_cpu(tail->uid); + if (id == 0 || affs_test_opt(sbi->s_flags, SF_SETUID)) + inode->i_uid = sbi->s_uid; + else if (id == 0xFFFF && affs_test_opt(sbi->s_flags, SF_MUFS)) + i_uid_write(inode, 0); + else + i_uid_write(inode, id); + + id = be16_to_cpu(tail->gid); + if (id == 0 || affs_test_opt(sbi->s_flags, SF_SETGID)) + inode->i_gid = sbi->s_gid; + else if (id == 0xFFFF && affs_test_opt(sbi->s_flags, SF_MUFS)) + i_gid_write(inode, 0); + else + i_gid_write(inode, id); + + switch (be32_to_cpu(tail->stype)) { + case ST_ROOT: + inode->i_uid = sbi->s_uid; + inode->i_gid = sbi->s_gid; + fallthrough; + case ST_USERDIR: + if (be32_to_cpu(tail->stype) == ST_USERDIR || + affs_test_opt(sbi->s_flags, SF_SETMODE)) { + if (inode->i_mode & S_IRUSR) + inode->i_mode |= S_IXUSR; + if (inode->i_mode & S_IRGRP) + inode->i_mode |= S_IXGRP; + if (inode->i_mode & S_IROTH) + inode->i_mode |= S_IXOTH; + inode->i_mode |= S_IFDIR; + } else + inode->i_mode = S_IRUGO | S_IXUGO | S_IWUSR | S_IFDIR; + /* Maybe it should be controlled by mount parameter? */ + //inode->i_mode |= S_ISVTX; + inode->i_op = &affs_dir_inode_operations; + inode->i_fop = &affs_dir_operations; + break; + case ST_LINKDIR: +#if 0 + affs_warning(sb, "read_inode", "inode is LINKDIR"); + goto bad_inode; +#else + inode->i_mode |= S_IFDIR; + /* ... and leave ->i_op and ->i_fop pointing to empty */ + break; +#endif + case ST_LINKFILE: + affs_warning(sb, "read_inode", "inode is LINKFILE"); + goto bad_inode; + case ST_FILE: + size = be32_to_cpu(tail->size); + inode->i_mode |= S_IFREG; + AFFS_I(inode)->mmu_private = inode->i_size = size; + if (inode->i_size) { + AFFS_I(inode)->i_blkcnt = (size - 1) / + sbi->s_data_blksize + 1; + AFFS_I(inode)->i_extcnt = (AFFS_I(inode)->i_blkcnt - 1) / + sbi->s_hashsize + 1; + } + if (tail->link_chain) + set_nlink(inode, 2); + inode->i_mapping->a_ops = affs_test_opt(sbi->s_flags, SF_OFS) ? + &affs_aops_ofs : &affs_aops; + inode->i_op = &affs_file_inode_operations; + inode->i_fop = &affs_file_operations; + break; + case ST_SOFTLINK: + inode->i_size = strlen((char *)AFFS_HEAD(bh)->table); + inode->i_mode |= S_IFLNK; + inode_nohighmem(inode); + inode->i_op = &affs_symlink_inode_operations; + inode->i_data.a_ops = &affs_symlink_aops; + break; + } + + inode_set_mtime(inode, + inode_set_atime(inode, inode_set_ctime(inode, (be32_to_cpu(tail->change.days) * 86400LL + be32_to_cpu(tail->change.mins) * 60 + be32_to_cpu(tail->change.ticks) / 50 + AFFS_EPOCH_DELTA) + sys_tz.tz_minuteswest * 60, 0).tv_sec, 0).tv_sec, + 0); + affs_brelse(bh); + unlock_new_inode(inode); + return inode; + +bad_inode: + affs_brelse(bh); + iget_failed(inode); + return ERR_PTR(-EIO); +} + +int +affs_write_inode(struct inode *inode, struct writeback_control *wbc) +{ + struct super_block *sb = inode->i_sb; + struct buffer_head *bh; + struct affs_tail *tail; + uid_t uid; + gid_t gid; + + pr_debug("write_inode(%llu)\n", inode->i_ino); + + if (!inode->i_nlink) + // possibly free block + return 0; + bh = affs_bread(sb, inode->i_ino); + if (!bh) { + affs_error(sb, "write_inode", "Cannot read block %llu", inode->i_ino); + return -EIO; + } + tail = AFFS_TAIL(sb, bh); + if (tail->stype == cpu_to_be32(ST_ROOT)) { + affs_secs_to_datestamp(inode_get_mtime_sec(inode), + &AFFS_ROOT_TAIL(sb, bh)->root_change); + } else { + tail->protect = cpu_to_be32(AFFS_I(inode)->i_protect); + tail->size = cpu_to_be32(inode->i_size); + affs_secs_to_datestamp(inode_get_mtime_sec(inode), + &tail->change); + if (!(inode->i_ino == AFFS_SB(sb)->s_root_block)) { + uid = i_uid_read(inode); + gid = i_gid_read(inode); + if (affs_test_opt(AFFS_SB(sb)->s_flags, SF_MUFS)) { + if (uid == 0 || uid == 0xFFFF) + uid = uid ^ ~0; + if (gid == 0 || gid == 0xFFFF) + gid = gid ^ ~0; + } + if (!affs_test_opt(AFFS_SB(sb)->s_flags, SF_SETUID)) + tail->uid = cpu_to_be16(uid); + if (!affs_test_opt(AFFS_SB(sb)->s_flags, SF_SETGID)) + tail->gid = cpu_to_be16(gid); + } + } + affs_fix_checksum(sb, bh); + mark_buffer_dirty(bh); + affs_brelse(bh); + affs_free_prealloc(inode); + return 0; +} + +int +affs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr) +{ + struct inode *inode = d_inode(dentry); + int error; + + pr_debug("setattr(%llu,0x%x)\n", inode->i_ino, attr->ia_valid); + + error = setattr_prepare(&nop_mnt_idmap, dentry, attr); + if (error) + goto out; + + if (((attr->ia_valid & ATTR_UID) && + affs_test_opt(AFFS_SB(inode->i_sb)->s_flags, SF_SETUID)) || + ((attr->ia_valid & ATTR_GID) && + affs_test_opt(AFFS_SB(inode->i_sb)->s_flags, SF_SETGID)) || + ((attr->ia_valid & ATTR_MODE) && + (AFFS_SB(inode->i_sb)->s_flags & + (AFFS_MOUNT_SF_SETMODE | AFFS_MOUNT_SF_IMMUTABLE)))) { + if (!affs_test_opt(AFFS_SB(inode->i_sb)->s_flags, SF_QUIET)) + error = -EPERM; + goto out; + } + + if ((attr->ia_valid & ATTR_SIZE) && + attr->ia_size != i_size_read(inode)) { + error = inode_newsize_ok(inode, attr->ia_size); + if (error) + return error; + + truncate_setsize(inode, attr->ia_size); + affs_truncate(inode); + } + + setattr_copy(&nop_mnt_idmap, inode, attr); + mark_inode_dirty(inode); + + if (attr->ia_valid & ATTR_MODE) + affs_mode_to_prot(inode); +out: + return error; +} + +void +affs_evict_inode(struct inode *inode) +{ + unsigned long cache_page; + pr_debug("evict_inode(ino=%llu, nlink=%u)\n", + inode->i_ino, inode->i_nlink); + truncate_inode_pages_final(&inode->i_data); + + if (!inode->i_nlink) { + inode->i_size = 0; + affs_truncate(inode); + } + + clear_inode(inode); + affs_free_prealloc(inode); + cache_page = (unsigned long)AFFS_I(inode)->i_lc; + if (cache_page) { + pr_debug("freeing ext cache\n"); + AFFS_I(inode)->i_lc = NULL; + AFFS_I(inode)->i_ac = NULL; + free_page(cache_page); + } + affs_brelse(AFFS_I(inode)->i_ext_bh); + AFFS_I(inode)->i_ext_last = ~1; + AFFS_I(inode)->i_ext_bh = NULL; + + if (!inode->i_nlink) + affs_free_block(inode->i_sb, inode->i_ino); +} + +struct inode * +affs_new_inode(struct inode *dir) +{ + struct super_block *sb = dir->i_sb; + struct inode *inode; + u32 block; + struct buffer_head *bh; + + if (!(inode = new_inode(sb))) + goto err_inode; + + if (!(block = affs_alloc_block(dir, dir->i_ino))) + goto err_block; + + bh = affs_getzeroblk(sb, block); + if (!bh) + goto err_bh; + mark_buffer_dirty(bh); + affs_brelse(bh); + + inode->i_uid = current_fsuid(); + inode->i_gid = current_fsgid(); + inode->i_ino = block; + set_nlink(inode, 1); + simple_inode_init_ts(inode); + atomic_set(&AFFS_I(inode)->i_opencnt, 0); + AFFS_I(inode)->i_blkcnt = 0; + AFFS_I(inode)->i_lc = NULL; + AFFS_I(inode)->i_lc_size = 0; + AFFS_I(inode)->i_lc_shift = 0; + AFFS_I(inode)->i_lc_mask = 0; + AFFS_I(inode)->i_ac = NULL; + AFFS_I(inode)->i_ext_bh = NULL; + AFFS_I(inode)->mmu_private = 0; + AFFS_I(inode)->i_protect = 0; + AFFS_I(inode)->i_lastalloc = 0; + AFFS_I(inode)->i_pa_cnt = 0; + AFFS_I(inode)->i_extcnt = 1; + AFFS_I(inode)->i_ext_last = ~1; + + insert_inode_hash(inode); + + return inode; + +err_bh: + affs_free_block(sb, block); +err_block: + iput(inode); +err_inode: + return NULL; +} + +/* + * Add an entry to a directory. Create the header block + * and insert it into the hash table. + */ + +int +affs_add_entry(struct inode *dir, struct inode *inode, struct dentry *dentry, s32 type) +{ + struct super_block *sb = dir->i_sb; + struct buffer_head *inode_bh = NULL; + struct buffer_head *bh; + u32 block = 0; + int retval; + + pr_debug("%s(dir=%llu, inode=%llu, \"%pd\", type=%d)\n", __func__, + dir->i_ino, inode->i_ino, dentry, type); + + retval = -EIO; + bh = affs_bread(sb, inode->i_ino); + if (!bh) + goto done; + + affs_lock_link(inode); + switch (type) { + case ST_LINKFILE: + case ST_LINKDIR: + retval = -ENOSPC; + block = affs_alloc_block(dir, dir->i_ino); + if (!block) + goto err; + retval = -EIO; + inode_bh = bh; + bh = affs_getzeroblk(sb, block); + if (!bh) + goto err; + break; + default: + break; + } + + AFFS_HEAD(bh)->ptype = cpu_to_be32(T_SHORT); + AFFS_HEAD(bh)->key = cpu_to_be32(bh->b_blocknr); + affs_copy_name(AFFS_TAIL(sb, bh)->name, dentry); + AFFS_TAIL(sb, bh)->stype = cpu_to_be32(type); + AFFS_TAIL(sb, bh)->parent = cpu_to_be32(dir->i_ino); + + if (inode_bh) { + __be32 chain; + chain = AFFS_TAIL(sb, inode_bh)->link_chain; + AFFS_TAIL(sb, bh)->original = cpu_to_be32(inode->i_ino); + AFFS_TAIL(sb, bh)->link_chain = chain; + AFFS_TAIL(sb, inode_bh)->link_chain = cpu_to_be32(block); + affs_adjust_checksum(inode_bh, block - be32_to_cpu(chain)); + mark_buffer_dirty(inode_bh); + set_nlink(inode, 2); + ihold(inode); + } + affs_fix_checksum(sb, bh); + mark_buffer_dirty(bh); + dentry->d_fsdata = (void *)(long)bh->b_blocknr; + + affs_lock_dir(dir); + retval = affs_insert_hash(dir, bh); + mark_buffer_dirty(bh); + affs_unlock_dir(dir); + affs_unlock_link(inode); + + d_instantiate(dentry, inode); +done: + affs_brelse(inode_bh); + affs_brelse(bh); + return retval; +err: + if (block) + affs_free_block(sb, block); + affs_unlock_link(inode); + goto done; +} diff --git a/fs/affs/namei.c b/fs/affs/namei.c new file mode 100644 index 000000000..6cb52efaf --- /dev/null +++ b/fs/affs/namei.c @@ -0,0 +1,582 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/affs/namei.c + * + * (c) 1996 Hans-Joachim Widmaier - Rewritten + * + * (C) 1993 Ray Burr - Modified for Amiga FFS filesystem. + * + * (C) 1991 Linus Torvalds - minix filesystem + */ + +#include "affs.h" +#include + +typedef int (*toupper_t)(int); + +/* Simple toupper() for DOS\1 */ + +static int +affs_toupper(int ch) +{ + return ch >= 'a' && ch <= 'z' ? ch -= ('a' - 'A') : ch; +} + +/* International toupper() for DOS\3 ("international") */ + +static int +affs_intl_toupper(int ch) +{ + return (ch >= 'a' && ch <= 'z') || (ch >= 0xE0 + && ch <= 0xFE && ch != 0xF7) ? + ch - ('a' - 'A') : ch; +} + +static inline toupper_t +affs_get_toupper(struct super_block *sb) +{ + return affs_test_opt(AFFS_SB(sb)->s_flags, SF_INTL) ? + affs_intl_toupper : affs_toupper; +} + +/* + * Note: the dentry argument is the parent dentry. + */ +static inline int +__affs_hash_dentry(const struct dentry *dentry, struct qstr *qstr, toupper_t fn, bool notruncate) +{ + const u8 *name = qstr->name; + unsigned long hash; + int retval; + u32 len; + + retval = affs_check_name(qstr->name, qstr->len, notruncate); + if (retval) + return retval; + + hash = init_name_hash(dentry); + len = min(qstr->len, AFFSNAMEMAX); + for (; len > 0; name++, len--) + hash = partial_name_hash(fn(*name), hash); + qstr->hash = end_name_hash(hash); + + return 0; +} + +static int +affs_hash_dentry(const struct dentry *dentry, struct qstr *qstr) +{ + return __affs_hash_dentry(dentry, qstr, affs_toupper, + affs_nofilenametruncate(dentry)); + +} + +static int +affs_intl_hash_dentry(const struct dentry *dentry, struct qstr *qstr) +{ + return __affs_hash_dentry(dentry, qstr, affs_intl_toupper, + affs_nofilenametruncate(dentry)); + +} + +static inline int __affs_compare_dentry(unsigned int len, + const char *str, const struct qstr *name, toupper_t fn, + bool notruncate) +{ + const u8 *aname = str; + const u8 *bname = name->name; + + /* + * 'str' is the name of an already existing dentry, so the name + * must be valid. 'name' must be validated first. + */ + + if (affs_check_name(name->name, name->len, notruncate)) + return 1; + + /* + * If the names are longer than the allowed 30 chars, + * the excess is ignored, so their length may differ. + */ + if (len >= AFFSNAMEMAX) { + if (name->len < AFFSNAMEMAX) + return 1; + len = AFFSNAMEMAX; + } else if (len != name->len) + return 1; + + for (; len > 0; len--) + if (fn(*aname++) != fn(*bname++)) + return 1; + + return 0; +} + +static int +affs_compare_dentry(const struct dentry *dentry, + unsigned int len, const char *str, const struct qstr *name) +{ + + return __affs_compare_dentry(len, str, name, affs_toupper, + affs_nofilenametruncate(dentry)); +} + +static int +affs_intl_compare_dentry(const struct dentry *dentry, + unsigned int len, const char *str, const struct qstr *name) +{ + return __affs_compare_dentry(len, str, name, affs_intl_toupper, + affs_nofilenametruncate(dentry)); + +} + +/* + * NOTE! unlike strncmp, affs_match returns 1 for success, 0 for failure. + */ + +static inline int +affs_match(struct dentry *dentry, const u8 *name2, toupper_t fn) +{ + const u8 *name = dentry->d_name.name; + int len = dentry->d_name.len; + + if (len >= AFFSNAMEMAX) { + if (*name2 < AFFSNAMEMAX) + return 0; + len = AFFSNAMEMAX; + } else if (len != *name2) + return 0; + + for (name2++; len > 0; len--) + if (fn(*name++) != fn(*name2++)) + return 0; + return 1; +} + +int +affs_hash_name(struct super_block *sb, const u8 *name, unsigned int len) +{ + toupper_t fn = affs_get_toupper(sb); + u32 hash; + + hash = len = min(len, AFFSNAMEMAX); + for (; len > 0; len--) + hash = (hash * 13 + fn(*name++)) & 0x7ff; + + return hash % AFFS_SB(sb)->s_hashsize; +} + +static struct buffer_head * +affs_find_entry(struct inode *dir, struct dentry *dentry) +{ + struct super_block *sb = dir->i_sb; + struct buffer_head *bh; + toupper_t fn = affs_get_toupper(sb); + u32 key; + + pr_debug("%s(\"%pd\")\n", __func__, dentry); + + bh = affs_bread(sb, dir->i_ino); + if (!bh) + return ERR_PTR(-EIO); + + key = be32_to_cpu(AFFS_HEAD(bh)->table[affs_hash_name(sb, dentry->d_name.name, dentry->d_name.len)]); + + for (;;) { + affs_brelse(bh); + if (key == 0) + return NULL; + bh = affs_bread(sb, key); + if (!bh) + return ERR_PTR(-EIO); + if (affs_match(dentry, AFFS_TAIL(sb, bh)->name, fn)) + return bh; + key = be32_to_cpu(AFFS_TAIL(sb, bh)->hash_chain); + } +} + +struct dentry * +affs_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) +{ + struct super_block *sb = dir->i_sb; + struct buffer_head *bh; + struct inode *inode = NULL; + struct dentry *res; + + pr_debug("%s(\"%pd\")\n", __func__, dentry); + + affs_lock_dir(dir); + bh = affs_find_entry(dir, dentry); + if (IS_ERR(bh)) { + affs_unlock_dir(dir); + return ERR_CAST(bh); + } + if (bh) { + u32 ino = bh->b_blocknr; + + /* store the real header ino in d_fsdata for faster lookups */ + dentry->d_fsdata = (void *)(long)ino; + switch (be32_to_cpu(AFFS_TAIL(sb, bh)->stype)) { + //link to dirs disabled + //case ST_LINKDIR: + case ST_LINKFILE: + ino = be32_to_cpu(AFFS_TAIL(sb, bh)->original); + } + affs_brelse(bh); + inode = affs_iget(sb, ino); + } + res = d_splice_alias(inode, dentry); + if (!IS_ERR_OR_NULL(res)) + res->d_fsdata = dentry->d_fsdata; + affs_unlock_dir(dir); + return res; +} + +int +affs_unlink(struct inode *dir, struct dentry *dentry) +{ + pr_debug("%s(dir=%llu, %llu \"%pd\")\n", __func__, dir->i_ino, + d_inode(dentry)->i_ino, dentry); + + return affs_remove_header(dentry); +} + +int +affs_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct super_block *sb = dir->i_sb; + struct inode *inode; + int error; + + pr_debug("%s(%llu,\"%pd\",0%ho)\n", + __func__, dir->i_ino, dentry, mode); + + inode = affs_new_inode(dir); + if (!inode) + return -ENOSPC; + + inode->i_mode = mode; + affs_mode_to_prot(inode); + mark_inode_dirty(inode); + + inode->i_op = &affs_file_inode_operations; + inode->i_fop = &affs_file_operations; + inode->i_mapping->a_ops = affs_test_opt(AFFS_SB(sb)->s_flags, SF_OFS) ? + &affs_aops_ofs : &affs_aops; + error = affs_add_entry(dir, inode, dentry, ST_FILE); + if (error) { + clear_nlink(inode); + iput(inode); + return error; + } + return 0; +} + +struct dentry * +affs_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct inode *inode; + int error; + + pr_debug("%s(%llu,\"%pd\",0%ho)\n", + __func__, dir->i_ino, dentry, mode); + + inode = affs_new_inode(dir); + if (!inode) + return ERR_PTR(-ENOSPC); + + inode->i_mode = mode; + affs_mode_to_prot(inode); + + inode->i_op = &affs_dir_inode_operations; + inode->i_fop = &affs_dir_operations; + + error = affs_add_entry(dir, inode, dentry, ST_USERDIR); + if (error) { + clear_nlink(inode); + mark_inode_dirty(inode); + iput(inode); + return ERR_PTR(error); + } + return NULL; +} + +int +affs_rmdir(struct inode *dir, struct dentry *dentry) +{ + pr_debug("%s(dir=%llu, %llu \"%pd\")\n", __func__, dir->i_ino, + d_inode(dentry)->i_ino, dentry); + + return affs_remove_header(dentry); +} + +int +affs_symlink(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const char *symname) +{ + struct super_block *sb = dir->i_sb; + struct buffer_head *bh; + struct inode *inode; + char *p; + int i, maxlen, error; + char c, lc; + + pr_debug("%s(%llu,\"%pd\" -> \"%s\")\n", + __func__, dir->i_ino, dentry, symname); + + maxlen = AFFS_SB(sb)->s_hashsize * sizeof(u32) - 1; + inode = affs_new_inode(dir); + if (!inode) + return -ENOSPC; + + inode->i_op = &affs_symlink_inode_operations; + inode_nohighmem(inode); + inode->i_data.a_ops = &affs_symlink_aops; + inode->i_mode = S_IFLNK | 0777; + affs_mode_to_prot(inode); + + error = -EIO; + bh = affs_bread(sb, inode->i_ino); + if (!bh) + goto err; + i = 0; + p = (char *)AFFS_HEAD(bh)->table; + lc = '/'; + if (*symname == '/') { + struct affs_sb_info *sbi = AFFS_SB(sb); + while (*symname == '/') + symname++; + spin_lock(&sbi->symlink_lock); + while (sbi->s_volume[i]) /* Cannot overflow */ + *p++ = sbi->s_volume[i++]; + spin_unlock(&sbi->symlink_lock); + } + while (i < maxlen && (c = *symname++)) { + if (c == '.' && lc == '/' && *symname == '.' && symname[1] == '/') { + *p++ = '/'; + i++; + symname += 2; + lc = '/'; + } else if (c == '.' && lc == '/' && *symname == '/') { + symname++; + lc = '/'; + } else { + *p++ = c; + lc = c; + i++; + } + if (lc == '/') + while (*symname == '/') + symname++; + } + *p = 0; + inode->i_size = i + 1; + mark_buffer_dirty(bh); + affs_brelse(bh); + mark_inode_dirty(inode); + + error = affs_add_entry(dir, inode, dentry, ST_SOFTLINK); + if (error) + goto err; + + return 0; + +err: + clear_nlink(inode); + mark_inode_dirty(inode); + iput(inode); + return error; +} + +int +affs_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry) +{ + struct inode *inode = d_inode(old_dentry); + + pr_debug("%s(%llu, %llu, \"%pd\")\n", __func__, inode->i_ino, dir->i_ino, + dentry); + + return affs_add_entry(dir, inode, dentry, ST_LINKFILE); +} + +static int +affs_rename(struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry) +{ + struct super_block *sb = old_dir->i_sb; + struct buffer_head *bh = NULL; + int retval; + + retval = affs_check_name(new_dentry->d_name.name, + new_dentry->d_name.len, + affs_nofilenametruncate(old_dentry)); + + if (retval) + return retval; + + /* Unlink destination if it already exists */ + if (d_really_is_positive(new_dentry)) { + retval = affs_remove_header(new_dentry); + if (retval) + return retval; + } + + bh = affs_bread(sb, d_inode(old_dentry)->i_ino); + if (!bh) + return -EIO; + + /* Remove header from its parent directory. */ + affs_lock_dir(old_dir); + retval = affs_remove_hash(old_dir, bh); + affs_unlock_dir(old_dir); + if (retval) + goto done; + + /* And insert it into the new directory with the new name. */ + affs_copy_name(AFFS_TAIL(sb, bh)->name, new_dentry); + affs_fix_checksum(sb, bh); + affs_lock_dir(new_dir); + retval = affs_insert_hash(new_dir, bh); + affs_unlock_dir(new_dir); + /* TODO: move it back to old_dir, if error? */ + +done: + mark_buffer_dirty(bh); + affs_brelse(bh); + return retval; +} + +static int +affs_xrename(struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry) +{ + + struct super_block *sb = old_dir->i_sb; + struct buffer_head *bh_old = NULL; + struct buffer_head *bh_new = NULL; + int retval; + + bh_old = affs_bread(sb, d_inode(old_dentry)->i_ino); + if (!bh_old) + return -EIO; + + bh_new = affs_bread(sb, d_inode(new_dentry)->i_ino); + if (!bh_new) { + affs_brelse(bh_old); + return -EIO; + } + + /* Remove old header from its parent directory. */ + affs_lock_dir(old_dir); + retval = affs_remove_hash(old_dir, bh_old); + affs_unlock_dir(old_dir); + if (retval) + goto done; + + /* Remove new header from its parent directory. */ + affs_lock_dir(new_dir); + retval = affs_remove_hash(new_dir, bh_new); + affs_unlock_dir(new_dir); + if (retval) + goto done; + + /* Insert old into the new directory with the new name. */ + affs_copy_name(AFFS_TAIL(sb, bh_old)->name, new_dentry); + affs_fix_checksum(sb, bh_old); + affs_lock_dir(new_dir); + retval = affs_insert_hash(new_dir, bh_old); + affs_unlock_dir(new_dir); + + /* Insert new into the old directory with the old name. */ + affs_copy_name(AFFS_TAIL(sb, bh_new)->name, old_dentry); + affs_fix_checksum(sb, bh_new); + affs_lock_dir(old_dir); + retval = affs_insert_hash(old_dir, bh_new); + affs_unlock_dir(old_dir); +done: + mark_buffer_dirty(bh_old); + mark_buffer_dirty(bh_new); + affs_brelse(bh_old); + affs_brelse(bh_new); + return retval; +} + +int affs_rename2(struct mnt_idmap *idmap, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) +{ + + if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE)) + return -EINVAL; + + pr_debug("%s(old=%llu,\"%pd\" to new=%llu,\"%pd\")\n", __func__, + old_dir->i_ino, old_dentry, new_dir->i_ino, new_dentry); + + if (flags & RENAME_EXCHANGE) + return affs_xrename(old_dir, old_dentry, new_dir, new_dentry); + + return affs_rename(old_dir, old_dentry, new_dir, new_dentry); +} + +static struct dentry *affs_get_parent(struct dentry *child) +{ + struct inode *parent; + struct buffer_head *bh; + + bh = affs_bread(child->d_sb, d_inode(child)->i_ino); + if (!bh) + return ERR_PTR(-EIO); + + parent = affs_iget(child->d_sb, + be32_to_cpu(AFFS_TAIL(child->d_sb, bh)->parent)); + brelse(bh); + return d_obtain_alias(parent); +} + +static struct inode *affs_nfs_get_inode(struct super_block *sb, u64 ino, + u32 generation) +{ + struct inode *inode; + + if (!affs_validblock(sb, ino)) + return ERR_PTR(-ESTALE); + + inode = affs_iget(sb, ino); + if (IS_ERR(inode)) + return ERR_CAST(inode); + + return inode; +} + +static struct dentry *affs_fh_to_dentry(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + return generic_fh_to_dentry(sb, fid, fh_len, fh_type, + affs_nfs_get_inode); +} + +static struct dentry *affs_fh_to_parent(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + return generic_fh_to_parent(sb, fid, fh_len, fh_type, + affs_nfs_get_inode); +} + +const struct export_operations affs_export_ops = { + .encode_fh = generic_encode_ino32_fh, + .fh_to_dentry = affs_fh_to_dentry, + .fh_to_parent = affs_fh_to_parent, + .get_parent = affs_get_parent, +}; + +const struct dentry_operations affs_dentry_operations = { + .d_hash = affs_hash_dentry, + .d_compare = affs_compare_dentry, +}; + +const struct dentry_operations affs_intl_dentry_operations = { + .d_hash = affs_intl_hash_dentry, + .d_compare = affs_intl_compare_dentry, +}; diff --git a/fs/affs/super.c b/fs/affs/super.c new file mode 100644 index 000000000..ed8225b3c --- /dev/null +++ b/fs/affs/super.c @@ -0,0 +1,684 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/affs/inode.c + * + * (c) 1996 Hans-Joachim Widmaier - Rewritten + * + * (C) 1993 Ray Burr - Modified for Amiga FFS filesystem. + * + * (C) 1992 Eric Youngdale Modified for ISO 9660 filesystem. + * + * (C) 1991 Linus Torvalds - minix filesystem + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "affs.h" + +static int affs_statfs(struct dentry *dentry, struct kstatfs *buf); +static int affs_show_options(struct seq_file *m, struct dentry *root); + +static void +affs_commit_super(struct super_block *sb, int wait) +{ + struct affs_sb_info *sbi = AFFS_SB(sb); + struct buffer_head *bh = sbi->s_root_bh; + struct affs_root_tail *tail = AFFS_ROOT_TAIL(sb, bh); + + lock_buffer(bh); + affs_secs_to_datestamp(ktime_get_real_seconds(), &tail->disk_change); + affs_fix_checksum(sb, bh); + unlock_buffer(bh); + + mark_buffer_dirty(bh); + if (wait) + sync_dirty_buffer(bh); +} + +static void +affs_put_super(struct super_block *sb) +{ + struct affs_sb_info *sbi = AFFS_SB(sb); + pr_debug("%s()\n", __func__); + + cancel_delayed_work_sync(&sbi->sb_work); +} + +static int +affs_sync_fs(struct super_block *sb, int wait) +{ + affs_commit_super(sb, wait); + return 0; +} + +static void flush_superblock(struct work_struct *work) +{ + struct affs_sb_info *sbi; + struct super_block *sb; + + sbi = container_of(work, struct affs_sb_info, sb_work.work); + sb = sbi->sb; + + spin_lock(&sbi->work_lock); + sbi->work_queued = 0; + spin_unlock(&sbi->work_lock); + + affs_commit_super(sb, 1); +} + +void affs_mark_sb_dirty(struct super_block *sb) +{ + struct affs_sb_info *sbi = AFFS_SB(sb); + unsigned long delay; + + if (sb_rdonly(sb)) + return; + + spin_lock(&sbi->work_lock); + if (!sbi->work_queued) { + delay = msecs_to_jiffies(dirty_writeback_interval * 10); + queue_delayed_work(system_dfl_long_wq, &sbi->sb_work, delay); + sbi->work_queued = 1; + } + spin_unlock(&sbi->work_lock); +} + +static struct kmem_cache * affs_inode_cachep; + +static struct inode *affs_alloc_inode(struct super_block *sb) +{ + struct affs_inode_info *i; + + i = alloc_inode_sb(sb, affs_inode_cachep, GFP_KERNEL); + if (!i) + return NULL; + + inode_set_iversion(&i->vfs_inode, 1); + i->i_lc = NULL; + i->i_ext_bh = NULL; + i->i_pa_cnt = 0; + + return &i->vfs_inode; +} + +static void affs_free_inode(struct inode *inode) +{ + kmem_cache_free(affs_inode_cachep, AFFS_I(inode)); +} + +static void init_once(void *foo) +{ + struct affs_inode_info *ei = (struct affs_inode_info *) foo; + + mutex_init(&ei->i_link_lock); + mutex_init(&ei->i_ext_lock); + inode_init_once(&ei->vfs_inode); +} + +static int __init init_inodecache(void) +{ + affs_inode_cachep = kmem_cache_create("affs_inode_cache", + sizeof(struct affs_inode_info), + 0, (SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT), + init_once); + if (affs_inode_cachep == NULL) + return -ENOMEM; + return 0; +} + +static void destroy_inodecache(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(affs_inode_cachep); +} + +static const struct super_operations affs_sops = { + .alloc_inode = affs_alloc_inode, + .free_inode = affs_free_inode, + .write_inode = affs_write_inode, + .evict_inode = affs_evict_inode, + .put_super = affs_put_super, + .sync_fs = affs_sync_fs, + .statfs = affs_statfs, + .show_options = affs_show_options, +}; + +enum { + Opt_bs, Opt_mode, Opt_mufs, Opt_notruncate, Opt_prefix, Opt_protect, + Opt_reserved, Opt_root, Opt_setgid, Opt_setuid, + Opt_verbose, Opt_volume, Opt_ignore, +}; + +struct affs_context { + kuid_t uid; /* uid to override */ + kgid_t gid; /* gid to override */ + unsigned int mode; /* mode to override */ + unsigned int reserved; /* Number of reserved blocks */ + int root_block; /* FFS root block number */ + int blocksize; /* Initial device blksize */ + char *prefix; /* Prefix for volumes and assigns */ + char volume[32]; /* Vol. prefix for absolute symlinks */ + unsigned long mount_flags; /* Options */ +}; + +static const struct fs_parameter_spec affs_param_spec[] = { + fsparam_u32 ("bs", Opt_bs), + fsparam_u32oct ("mode", Opt_mode), + fsparam_flag ("mufs", Opt_mufs), + fsparam_flag ("nofilenametruncate", Opt_notruncate), + fsparam_string ("prefix", Opt_prefix), + fsparam_flag ("protect", Opt_protect), + fsparam_u32 ("reserved", Opt_reserved), + fsparam_u32 ("root", Opt_root), + fsparam_gid ("setgid", Opt_setgid), + fsparam_uid ("setuid", Opt_setuid), + fsparam_flag ("verbose", Opt_verbose), + fsparam_string ("volume", Opt_volume), + fsparam_flag ("grpquota", Opt_ignore), + fsparam_flag ("noquota", Opt_ignore), + fsparam_flag ("quota", Opt_ignore), + fsparam_flag ("usrquota", Opt_ignore), + {}, +}; + +static int affs_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct affs_context *ctx = fc->fs_private; + struct fs_parse_result result; + int n; + int opt; + + opt = fs_parse(fc, affs_param_spec, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_bs: + n = result.uint_32; + if (n != 512 && n != 1024 && n != 2048 + && n != 4096) { + pr_warn("Invalid blocksize (512, 1024, 2048, 4096 allowed)\n"); + return -EINVAL; + } + ctx->blocksize = n; + break; + case Opt_mode: + ctx->mode = result.uint_32 & 0777; + affs_set_opt(ctx->mount_flags, SF_SETMODE); + break; + case Opt_mufs: + affs_set_opt(ctx->mount_flags, SF_MUFS); + break; + case Opt_notruncate: + affs_set_opt(ctx->mount_flags, SF_NO_TRUNCATE); + break; + case Opt_prefix: + kfree(ctx->prefix); + ctx->prefix = param->string; + param->string = NULL; + affs_set_opt(ctx->mount_flags, SF_PREFIX); + break; + case Opt_protect: + affs_set_opt(ctx->mount_flags, SF_IMMUTABLE); + break; + case Opt_reserved: + ctx->reserved = result.uint_32; + break; + case Opt_root: + ctx->root_block = result.uint_32; + break; + case Opt_setgid: + ctx->gid = result.gid; + affs_set_opt(ctx->mount_flags, SF_SETGID); + break; + case Opt_setuid: + ctx->uid = result.uid; + affs_set_opt(ctx->mount_flags, SF_SETUID); + break; + case Opt_verbose: + affs_set_opt(ctx->mount_flags, SF_VERBOSE); + break; + case Opt_volume: + strscpy(ctx->volume, param->string, 32); + break; + case Opt_ignore: + /* Silently ignore the quota options */ + break; + default: + return -EINVAL; + } + return 0; +} + +static int affs_show_options(struct seq_file *m, struct dentry *root) +{ + struct super_block *sb = root->d_sb; + struct affs_sb_info *sbi = AFFS_SB(sb); + + if (sb->s_blocksize) + seq_printf(m, ",bs=%lu", sb->s_blocksize); + if (affs_test_opt(sbi->s_flags, SF_SETMODE)) + seq_printf(m, ",mode=%o", sbi->s_mode); + if (affs_test_opt(sbi->s_flags, SF_MUFS)) + seq_puts(m, ",mufs"); + if (affs_test_opt(sbi->s_flags, SF_NO_TRUNCATE)) + seq_puts(m, ",nofilenametruncate"); + if (affs_test_opt(sbi->s_flags, SF_PREFIX)) + seq_printf(m, ",prefix=%s", sbi->s_prefix); + if (affs_test_opt(sbi->s_flags, SF_IMMUTABLE)) + seq_puts(m, ",protect"); + if (sbi->s_reserved != 2) + seq_printf(m, ",reserved=%u", sbi->s_reserved); + if (sbi->s_root_block != (sbi->s_reserved + sbi->s_partition_size - 1) / 2) + seq_printf(m, ",root=%u", sbi->s_root_block); + if (affs_test_opt(sbi->s_flags, SF_SETGID)) + seq_printf(m, ",setgid=%u", + from_kgid_munged(&init_user_ns, sbi->s_gid)); + if (affs_test_opt(sbi->s_flags, SF_SETUID)) + seq_printf(m, ",setuid=%u", + from_kuid_munged(&init_user_ns, sbi->s_uid)); + if (affs_test_opt(sbi->s_flags, SF_VERBOSE)) + seq_puts(m, ",verbose"); + if (sbi->s_volume[0]) + seq_printf(m, ",volume=%s", sbi->s_volume); + return 0; +} + +/* This function definitely needs to be split up. Some fine day I'll + * hopefully have the guts to do so. Until then: sorry for the mess. + */ + +static int affs_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct affs_sb_info *sbi; + struct affs_context *ctx = fc->fs_private; + struct buffer_head *root_bh = NULL; + struct buffer_head *boot_bh; + struct inode *root_inode = NULL; + int silent = fc->sb_flags & SB_SILENT; + int size, blocksize; + u32 chksum; + int num_bm; + int i, j; + int tmp_flags; /* fix remount prototype... */ + u8 sig[4]; + int ret; + + sb->s_magic = AFFS_SUPER_MAGIC; + sb->s_op = &affs_sops; + sb->s_flags |= SB_NODIRATIME; + + sb->s_time_gran = NSEC_PER_SEC; + sb->s_time_min = sys_tz.tz_minuteswest * 60 + AFFS_EPOCH_DELTA; + sb->s_time_max = 86400LL * U32_MAX + 86400 + sb->s_time_min; + + sbi = kzalloc_obj(struct affs_sb_info); + if (!sbi) + return -ENOMEM; + + sb->s_fs_info = sbi; + sbi->sb = sb; + mutex_init(&sbi->s_bmlock); + spin_lock_init(&sbi->symlink_lock); + spin_lock_init(&sbi->work_lock); + INIT_DELAYED_WORK(&sbi->sb_work, flush_superblock); + + sbi->s_flags = ctx->mount_flags; + sbi->s_mode = ctx->mode; + sbi->s_uid = ctx->uid; + sbi->s_gid = ctx->gid; + sbi->s_reserved = ctx->reserved; + sbi->s_prefix = ctx->prefix; + ctx->prefix = NULL; + memcpy(sbi->s_volume, ctx->volume, 32); + + /* N.B. after this point s_prefix must be released */ + + /* Get the size of the device in 512-byte blocks. + * If we later see that the partition uses bigger + * blocks, we will have to change it. + */ + + size = bdev_nr_sectors(sb->s_bdev); + pr_debug("initial blocksize=%d, #blocks=%d\n", 512, size); + + if (!sb_set_blocksize(sb, PAGE_SIZE)) + return -EINVAL; + /* Try to find root block. Its location depends on the block size. */ + + i = bdev_logical_block_size(sb->s_bdev); + j = PAGE_SIZE; + blocksize = ctx->blocksize; + if (blocksize > 0) { + i = j = blocksize; + size = size / (blocksize / 512); + } + + for (blocksize = i; blocksize <= j; blocksize <<= 1, size >>= 1) { + sbi->s_root_block = ctx->root_block; + if (ctx->root_block < 0) + sbi->s_root_block = (ctx->reserved + size - 1) / 2; + pr_debug("setting blocksize to %d\n", blocksize); + if (!sb_set_blocksize(sb, blocksize)) + return -EINVAL; + sbi->s_partition_size = size; + + /* The root block location that was calculated above is not + * correct if the partition size is an odd number of 512- + * byte blocks, which will be rounded down to a number of + * 1024-byte blocks, and if there were an even number of + * reserved blocks. Ideally, all partition checkers should + * report the real number of blocks of the real blocksize, + * but since this just cannot be done, we have to try to + * find the root block anyways. In the above case, it is one + * block behind the calculated one. So we check this one, too. + */ + for (num_bm = 0; num_bm < 2; num_bm++) { + pr_debug("Dev %s, trying root=%u, bs=%d, " + "size=%d, reserved=%d\n", + sb->s_id, + sbi->s_root_block + num_bm, + ctx->blocksize, size, ctx->reserved); + root_bh = affs_bread(sb, sbi->s_root_block + num_bm); + if (!root_bh) + continue; + if (!affs_checksum_block(sb, root_bh) && + be32_to_cpu(AFFS_ROOT_HEAD(root_bh)->ptype) == T_SHORT && + be32_to_cpu(AFFS_ROOT_TAIL(sb, root_bh)->stype) == ST_ROOT) { + sbi->s_hashsize = blocksize / 4 - 56; + sbi->s_root_block += num_bm; + goto got_root; + } + affs_brelse(root_bh); + root_bh = NULL; + } + } + if (!silent) + pr_err("No valid root block on device %s\n", sb->s_id); + return -EINVAL; + + /* N.B. after this point bh must be released */ +got_root: + /* Keep super block in cache */ + sbi->s_root_bh = root_bh; + ctx->root_block = sbi->s_root_block; + + /* Find out which kind of FS we have */ + boot_bh = sb_bread(sb, 0); + if (!boot_bh) { + pr_err("Cannot read boot block\n"); + return -EINVAL; + } + memcpy(sig, boot_bh->b_data, 4); + brelse(boot_bh); + chksum = be32_to_cpu(*(__be32 *)sig); + + /* Dircache filesystems are compatible with non-dircache ones + * when reading. As long as they aren't supported, writing is + * not recommended. + */ + if ((chksum == FS_DCFFS || chksum == MUFS_DCFFS || chksum == FS_DCOFS + || chksum == MUFS_DCOFS) && !sb_rdonly(sb)) { + pr_notice("Dircache FS - mounting %s read only\n", sb->s_id); + sb->s_flags |= SB_RDONLY; + } + switch (chksum) { + case MUFS_FS: + case MUFS_INTLFFS: + case MUFS_DCFFS: + affs_set_opt(sbi->s_flags, SF_MUFS); + fallthrough; + case FS_INTLFFS: + case FS_DCFFS: + affs_set_opt(sbi->s_flags, SF_INTL); + break; + case MUFS_FFS: + affs_set_opt(sbi->s_flags, SF_MUFS); + break; + case FS_FFS: + break; + case MUFS_OFS: + affs_set_opt(sbi->s_flags, SF_MUFS); + fallthrough; + case FS_OFS: + affs_set_opt(sbi->s_flags, SF_OFS); + sb->s_flags |= SB_NOEXEC; + break; + case MUFS_DCOFS: + case MUFS_INTLOFS: + affs_set_opt(sbi->s_flags, SF_MUFS); + fallthrough; + case FS_DCOFS: + case FS_INTLOFS: + affs_set_opt(sbi->s_flags, SF_INTL); + affs_set_opt(sbi->s_flags, SF_OFS); + sb->s_flags |= SB_NOEXEC; + break; + default: + pr_err("Unknown filesystem on device %s: %08X\n", + sb->s_id, chksum); + return -EINVAL; + } + + if (affs_test_opt(ctx->mount_flags, SF_VERBOSE)) { + u8 len = AFFS_ROOT_TAIL(sb, root_bh)->disk_name[0]; + pr_notice("Mounting volume \"%.*s\": Type=%.3s\\%c, Blocksize=%d\n", + len > 31 ? 31 : len, + AFFS_ROOT_TAIL(sb, root_bh)->disk_name + 1, + sig, sig[3] + '0', blocksize); + } + + sb->s_flags |= SB_NODEV | SB_NOSUID; + + sbi->s_data_blksize = sb->s_blocksize; + if (affs_test_opt(sbi->s_flags, SF_OFS)) + sbi->s_data_blksize -= 24; + + tmp_flags = sb->s_flags; + ret = affs_init_bitmap(sb, &tmp_flags); + if (ret) + return ret; + sb->s_flags = tmp_flags; + + /* set up enough so that it can read an inode */ + + root_inode = affs_iget(sb, ctx->root_block); + if (IS_ERR(root_inode)) + return PTR_ERR(root_inode); + + if (affs_test_opt(AFFS_SB(sb)->s_flags, SF_INTL)) + set_default_d_op(sb, &affs_intl_dentry_operations); + else + set_default_d_op(sb, &affs_dentry_operations); + + sb->s_root = d_make_root(root_inode); + if (!sb->s_root) { + pr_err("AFFS: Get root inode failed\n"); + return -ENOMEM; + } + + sb->s_export_op = &affs_export_ops; + pr_debug("s_flags=%lX\n", sb->s_flags); + return 0; +} + +static int affs_reconfigure(struct fs_context *fc) +{ + struct super_block *sb = fc->root->d_sb; + struct affs_context *ctx = fc->fs_private; + struct affs_sb_info *sbi = AFFS_SB(sb); + int res = 0; + + sync_filesystem(sb); + fc->sb_flags |= SB_NODIRATIME; + + flush_delayed_work(&sbi->sb_work); + + /* + * NB: Historically, only mount_flags, mode, uid, gic, prefix, + * and volume are accepted during remount. + */ + sbi->s_flags = ctx->mount_flags; + sbi->s_mode = ctx->mode; + sbi->s_uid = ctx->uid; + sbi->s_gid = ctx->gid; + /* protect against readers */ + spin_lock(&sbi->symlink_lock); + if (ctx->prefix) { + kfree(sbi->s_prefix); + sbi->s_prefix = ctx->prefix; + ctx->prefix = NULL; + } + memcpy(sbi->s_volume, ctx->volume, 32); + spin_unlock(&sbi->symlink_lock); + + if ((bool)(fc->sb_flags & SB_RDONLY) == sb_rdonly(sb)) + return 0; + + if (fc->sb_flags & SB_RDONLY) + affs_free_bitmap(sb); + else + res = affs_init_bitmap(sb, &fc->sb_flags); + + return res; +} + +static int +affs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + int free; + u64 id = huge_encode_dev(sb->s_bdev->bd_dev); + + pr_debug("%s() partsize=%d, reserved=%d\n", + __func__, AFFS_SB(sb)->s_partition_size, + AFFS_SB(sb)->s_reserved); + + free = affs_count_free_blocks(sb); + buf->f_type = AFFS_SUPER_MAGIC; + buf->f_bsize = sb->s_blocksize; + buf->f_blocks = AFFS_SB(sb)->s_partition_size - AFFS_SB(sb)->s_reserved; + buf->f_bfree = free; + buf->f_bavail = free; + buf->f_fsid = u64_to_fsid(id); + buf->f_namelen = AFFSNAMEMAX; + return 0; +} + +static int affs_get_tree(struct fs_context *fc) +{ + return get_tree_bdev(fc, affs_fill_super); +} + +static void affs_kill_sb(struct super_block *sb) +{ + struct affs_sb_info *sbi = AFFS_SB(sb); + kill_block_super(sb); + if (sbi) { + affs_free_bitmap(sb); + affs_brelse(sbi->s_root_bh); + kfree(sbi->s_prefix); + mutex_destroy(&sbi->s_bmlock); + kfree_rcu(sbi, rcu); + } +} + +static void affs_free_fc(struct fs_context *fc) +{ + struct affs_context *ctx = fc->fs_private; + + kfree(ctx->prefix); + kfree(ctx); +} + +static const struct fs_context_operations affs_context_ops = { + .parse_param = affs_parse_param, + .get_tree = affs_get_tree, + .reconfigure = affs_reconfigure, + .free = affs_free_fc, +}; + +static int affs_init_fs_context(struct fs_context *fc) +{ + struct affs_context *ctx; + + ctx = kzalloc_obj(struct affs_context); + if (!ctx) + return -ENOMEM; + + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE) { + struct super_block *sb = fc->root->d_sb; + struct affs_sb_info *sbi = AFFS_SB(sb); + + /* + * NB: historically, no options other than volume were + * preserved across a remount unless they were explicitly + * passed in. + */ + memcpy(ctx->volume, sbi->s_volume, 32); + } else { + ctx->uid = current_uid(); + ctx->gid = current_gid(); + ctx->reserved = 2; + ctx->root_block = -1; + ctx->blocksize = -1; + ctx->volume[0] = ':'; + } + + fc->ops = &affs_context_ops; + fc->fs_private = ctx; + + return 0; +} + +static struct file_system_type affs_fs_type = { + .owner = THIS_MODULE, + .name = "affs", + .kill_sb = affs_kill_sb, + .fs_flags = FS_REQUIRES_DEV, + .init_fs_context = affs_init_fs_context, + .parameters = affs_param_spec, +}; +MODULE_ALIAS_FS("affs"); + +static int __init init_affs_fs(void) +{ + int err = init_inodecache(); + if (err) + goto out1; + err = register_filesystem(&affs_fs_type); + if (err) + goto out; + return 0; +out: + destroy_inodecache(); +out1: + return err; +} + +static void __exit exit_affs_fs(void) +{ + unregister_filesystem(&affs_fs_type); + destroy_inodecache(); +} + +MODULE_DESCRIPTION("Amiga filesystem support for Linux"); +MODULE_LICENSE("GPL"); + +module_init(init_affs_fs) +module_exit(exit_affs_fs) diff --git a/fs/affs/symlink.c b/fs/affs/symlink.c new file mode 100644 index 000000000..4994abb97 --- /dev/null +++ b/fs/affs/symlink.c @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/affs/symlink.c + * + * 1995 Hans-Joachim Widmaier - Modified for affs. + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * affs symlink handling code + */ + +#include "affs.h" + +static int affs_symlink_read_folio(struct file *file, struct folio *folio) +{ + struct buffer_head *bh; + struct inode *inode = folio->mapping->host; + char *link = folio_address(folio); + struct slink_front *lf; + int i, j; + char c; + char lc; + + pr_debug("get_link(ino=%llu)\n", inode->i_ino); + + bh = affs_bread(inode->i_sb, inode->i_ino); + if (!bh) + goto fail; + i = 0; + j = 0; + lf = (struct slink_front *)bh->b_data; + lc = 0; + + if (strchr(lf->symname,':')) { /* Handle assign or volume name */ + struct affs_sb_info *sbi = AFFS_SB(inode->i_sb); + char *pf; + spin_lock(&sbi->symlink_lock); + pf = sbi->s_prefix ? sbi->s_prefix : "/"; + while (i < 1023 && (c = pf[i])) + link[i++] = c; + spin_unlock(&sbi->symlink_lock); + while (i < 1023 && lf->symname[j] != ':') + link[i++] = lf->symname[j++]; + if (i < 1023) + link[i++] = '/'; + j++; + lc = '/'; + } + while (i < 1023 && (c = lf->symname[j])) { + if (c == '/' && lc == '/' && i < 1020) { /* parent dir */ + link[i++] = '.'; + link[i++] = '.'; + } + link[i++] = c; + lc = c; + j++; + } + link[i] = '\0'; + affs_brelse(bh); + folio_mark_uptodate(folio); + folio_unlock(folio); + return 0; +fail: + folio_unlock(folio); + return -EIO; +} + +const struct address_space_operations affs_symlink_aops = { + .read_folio = affs_symlink_read_folio, +}; + +const struct inode_operations affs_symlink_inode_operations = { + .get_link = page_get_link, + .setattr = affs_setattr, +}; diff --git a/fs/afs/Kconfig b/fs/afs/Kconfig new file mode 100644 index 000000000..682bd8ec2 --- /dev/null +++ b/fs/afs/Kconfig @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: GPL-2.0-only +config AFS_FS + tristate "Andrew File System support (AFS)" + depends on INET + select AF_RXRPC + select DNS_RESOLVER + select NETFS_SUPPORT + select CRYPTO_KRB5 + help + If you say Y here, you will get an experimental Andrew File System + driver. It currently only supports unsecured read-only AFS access. + + See for more information. + + If unsure, say N. + +config AFS_DEBUG + bool "AFS dynamic debugging" + depends on AFS_FS + help + Say Y here to make runtime controllable debugging messages appear. + + See for more information. + + If unsure, say N. + +config AFS_FSCACHE + bool "Provide AFS client caching support" + depends on AFS_FS=m && FSCACHE || AFS_FS=y && FSCACHE=y + help + Say Y here if you want AFS data to be cached locally on disk through + the generic filesystem cache manager + +config AFS_DEBUG_CURSOR + bool "AFS server cursor debugging" + depends on AFS_FS + help + Say Y here to cause the contents of a server cursor to be dumped to + the dmesg log if the server rotation algorithm fails to successfully + contact a server. + + See for more information. + + If unsure, say N. diff --git a/fs/afs/Makefile b/fs/afs/Makefile new file mode 100644 index 000000000..0d8f1982d --- /dev/null +++ b/fs/afs/Makefile @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for Red Hat Linux AFS client. +# + +kafs-y := \ + addr_list.o \ + addr_prefs.o \ + callback.o \ + cell.o \ + cm_security.o \ + cmservice.o \ + dir.o \ + dir_edit.o \ + dir_search.o \ + dir_silly.o \ + dynroot.o \ + file.o \ + flock.o \ + fsclient.o \ + fs_operation.o \ + fs_probe.o \ + inode.o \ + main.o \ + misc.o \ + mntpt.o \ + rotate.o \ + rxrpc.o \ + security.o \ + server.o \ + server_list.o \ + super.o \ + symlink.o \ + validation.o \ + vlclient.o \ + vl_alias.o \ + vl_list.o \ + vl_probe.o \ + vl_rotate.o \ + volume.o \ + write.o \ + xattr.o \ + yfsclient.o + +kafs-$(CONFIG_PROC_FS) += proc.o +obj-$(CONFIG_AFS_FS) := kafs.o diff --git a/fs/afs/addr_list.c b/fs/afs/addr_list.c new file mode 100644 index 000000000..63bf096b7 --- /dev/null +++ b/fs/afs/addr_list.c @@ -0,0 +1,414 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Server address list management + * + * Copyright (C) 2017 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include "internal.h" +#include "afs_fs.h" + +static void afs_free_addrlist(struct rcu_head *rcu) +{ + struct afs_addr_list *alist = container_of(rcu, struct afs_addr_list, rcu); + unsigned int i; + + for (i = 0; i < alist->nr_addrs; i++) + rxrpc_kernel_put_peer(alist->addrs[i].peer); + trace_afs_alist(alist->debug_id, refcount_read(&alist->usage), afs_alist_trace_free); + kfree(alist); +} + +/* + * Release an address list. + */ +void afs_put_addrlist(struct afs_addr_list *alist, enum afs_alist_trace reason) +{ + unsigned int debug_id; + bool dead; + int r; + + if (!alist) + return; + debug_id = alist->debug_id; + dead = __refcount_dec_and_test(&alist->usage, &r); + trace_afs_alist(debug_id, r - 1, reason); + if (dead) + call_rcu(&alist->rcu, afs_free_addrlist); +} + +struct afs_addr_list *afs_get_addrlist(struct afs_addr_list *alist, enum afs_alist_trace reason) +{ + int r; + + if (alist) { + __refcount_inc(&alist->usage, &r); + trace_afs_alist(alist->debug_id, r + 1, reason); + } + return alist; +} + +/* + * Allocate an address list. + */ +struct afs_addr_list *afs_alloc_addrlist(unsigned int nr) +{ + struct afs_addr_list *alist; + static atomic_t debug_id; + + _enter("%u", nr); + + if (nr > AFS_MAX_ADDRESSES) + nr = AFS_MAX_ADDRESSES; + + alist = kzalloc_flex(*alist, addrs, nr); + if (!alist) + return NULL; + + refcount_set(&alist->usage, 1); + alist->max_addrs = nr; + alist->debug_id = atomic_inc_return(&debug_id); + trace_afs_alist(alist->debug_id, 1, afs_alist_trace_alloc); + return alist; +} + +/* + * Parse a text string consisting of delimited addresses. + */ +struct afs_vlserver_list *afs_parse_text_addrs(struct afs_net *net, + const char *text, size_t len, + char delim, + unsigned short service, + unsigned short port) +{ + struct afs_vlserver_list *vllist; + struct afs_addr_list *alist; + const char *p, *end = text + len; + const char *problem; + unsigned int nr = 0; + int ret = -ENOMEM; + + _enter("%*.*s,%c", (int)len, (int)len, text, delim); + + if (!len) { + _leave(" = -EDESTADDRREQ [empty]"); + return ERR_PTR(-EDESTADDRREQ); + } + + if (delim == ':' && (memchr(text, ',', len) || !memchr(text, '.', len))) + delim = ','; + + /* Count the addresses */ + p = text; + do { + if (!*p) { + problem = "nul"; + goto inval; + } + if (*p == delim) + continue; + nr++; + if (*p == '[') { + p++; + if (p == end) { + problem = "brace1"; + goto inval; + } + p = memchr(p, ']', end - p); + if (!p) { + problem = "brace2"; + goto inval; + } + p++; + if (p >= end) + break; + } + + p = memchr(p, delim, end - p); + if (!p) + break; + p++; + } while (p < end); + + _debug("%u/%u addresses", nr, AFS_MAX_ADDRESSES); + + vllist = afs_alloc_vlserver_list(1); + if (!vllist) + return ERR_PTR(-ENOMEM); + + vllist->nr_servers = 1; + vllist->servers[0].server = afs_alloc_vlserver("", 7, AFS_VL_PORT); + if (!vllist->servers[0].server) + goto error_vl; + + alist = afs_alloc_addrlist(nr); + if (!alist) + goto error; + + /* Extract the addresses */ + p = text; + do { + const char *q, *stop; + unsigned int xport = port; + __be32 x[4]; + int family; + + if (*p == delim) { + p++; + continue; + } + + if (*p == '[') { + p++; + q = memchr(p, ']', end - p); + } else { + for (q = p; q < end; q++) + if (*q == '+' || *q == delim) + break; + } + + if (in4_pton(p, q - p, (u8 *)&x[0], -1, &stop)) { + family = AF_INET; + } else if (in6_pton(p, q - p, (u8 *)x, -1, &stop)) { + family = AF_INET6; + } else { + problem = "family"; + goto bad_address; + } + + p = q; + if (stop != p) { + problem = "nostop"; + goto bad_address; + } + + if (q < end && *q == ']') + p++; + + if (p < end) { + if (*p == '+') { + /* Port number specification "+1234" */ + xport = 0; + p++; + if (p >= end || !isdigit(*p)) { + problem = "port"; + goto bad_address; + } + do { + xport *= 10; + xport += *p - '0'; + if (xport > 65535) { + problem = "pval"; + goto bad_address; + } + p++; + } while (p < end && isdigit(*p)); + } else if (*p == delim) { + p++; + } else { + problem = "weird"; + goto bad_address; + } + } + + if (family == AF_INET) + ret = afs_merge_fs_addr4(net, alist, x[0], xport); + else + ret = afs_merge_fs_addr6(net, alist, x, xport); + if (ret < 0) + goto error; + + } while (p < end); + + rcu_assign_pointer(vllist->servers[0].server->addresses, alist); + _leave(" = [nr %u]", alist->nr_addrs); + return vllist; + +inval: + _leave(" = -EINVAL [%s %zu %*.*s]", + problem, p - text, (int)len, (int)len, text); + return ERR_PTR(-EINVAL); +bad_address: + _leave(" = -EINVAL [%s %zu %*.*s]", + problem, p - text, (int)len, (int)len, text); + ret = -EINVAL; +error: + afs_put_addrlist(alist, afs_alist_trace_put_parse_error); +error_vl: + afs_put_vlserverlist(net, vllist); + return ERR_PTR(ret); +} + +/* + * Perform a DNS query for VL servers and build a up an address list. + */ +struct afs_vlserver_list *afs_dns_query(struct afs_cell *cell, time64_t *_expiry) +{ + struct afs_vlserver_list *vllist; + char *result = NULL; + int ret; + + _enter("%s", cell->name); + + ret = dns_query(cell->net->net, "afsdb", cell->name, cell->name_len, + "srv=1", &result, _expiry, true); + if (ret < 0) { + _leave(" = %d [dns]", ret); + return ERR_PTR(ret); + } + + if (*_expiry == 0) + *_expiry = ktime_get_real_seconds() + 60; + + if (ret > 1 && result[0] == 0) + vllist = afs_extract_vlserver_list(cell, result, ret); + else + vllist = afs_parse_text_addrs(cell->net, result, ret, ',', + VL_SERVICE, AFS_VL_PORT); + kfree(result); + if (IS_ERR(vllist) && vllist != ERR_PTR(-ENOMEM)) + pr_err("Failed to parse DNS data %ld\n", PTR_ERR(vllist)); + + return vllist; +} + +/* + * Merge an IPv4 entry into a fileserver address list. + */ +int afs_merge_fs_addr4(struct afs_net *net, struct afs_addr_list *alist, + __be32 xdr, u16 port) +{ + struct sockaddr_rxrpc srx; + struct rxrpc_peer *peer; + int i; + + if (alist->nr_addrs >= alist->max_addrs) + return 0; + + srx.srx_family = AF_RXRPC; + srx.transport_type = SOCK_DGRAM; + srx.transport_len = sizeof(srx.transport.sin); + srx.transport.sin.sin_family = AF_INET; + srx.transport.sin.sin_port = htons(port); + srx.transport.sin.sin_addr.s_addr = xdr; + + peer = rxrpc_kernel_lookup_peer(net->socket, &srx, GFP_KERNEL); + if (IS_ERR(peer)) + return PTR_ERR(peer); + + for (i = 0; i < alist->nr_ipv4; i++) { + if (peer == alist->addrs[i].peer) { + rxrpc_kernel_put_peer(peer); + return 0; + } + if (peer <= alist->addrs[i].peer) + break; + } + + if (i < alist->nr_addrs) + memmove(alist->addrs + i + 1, + alist->addrs + i, + sizeof(alist->addrs[0]) * (alist->nr_addrs - i)); + + alist->addrs[i].peer = peer; + alist->nr_ipv4++; + alist->nr_addrs++; + return 0; +} + +/* + * Merge an IPv6 entry into a fileserver address list. + */ +int afs_merge_fs_addr6(struct afs_net *net, struct afs_addr_list *alist, + __be32 *xdr, u16 port) +{ + struct sockaddr_rxrpc srx; + struct rxrpc_peer *peer; + int i; + + if (alist->nr_addrs >= alist->max_addrs) + return 0; + + srx.srx_family = AF_RXRPC; + srx.transport_type = SOCK_DGRAM; + srx.transport_len = sizeof(srx.transport.sin6); + srx.transport.sin6.sin6_family = AF_INET6; + srx.transport.sin6.sin6_port = htons(port); + memcpy(&srx.transport.sin6.sin6_addr, xdr, 16); + + peer = rxrpc_kernel_lookup_peer(net->socket, &srx, GFP_KERNEL); + if (IS_ERR(peer)) + return PTR_ERR(peer); + + for (i = alist->nr_ipv4; i < alist->nr_addrs; i++) { + if (peer == alist->addrs[i].peer) { + rxrpc_kernel_put_peer(peer); + return 0; + } + if (peer <= alist->addrs[i].peer) + break; + } + + if (i < alist->nr_addrs) + memmove(alist->addrs + i + 1, + alist->addrs + i, + sizeof(alist->addrs[0]) * (alist->nr_addrs - i)); + alist->addrs[i].peer = peer; + alist->nr_addrs++; + return 0; +} + +/* + * Set the app data on the rxrpc peers an address list points to + */ +void afs_set_peer_appdata(struct afs_server *server, + struct afs_addr_list *old_alist, + struct afs_addr_list *new_alist) +{ + unsigned long data = (unsigned long)server; + int n = 0, o = 0; + + if (!old_alist) { + /* New server. Just set all. */ + for (; n < new_alist->nr_addrs; n++) + rxrpc_kernel_set_peer_data(new_alist->addrs[n].peer, data); + return; + } + if (!new_alist) { + /* Dead server. Just remove all. */ + for (; o < old_alist->nr_addrs; o++) + rxrpc_kernel_set_peer_data(old_alist->addrs[o].peer, 0); + return; + } + + /* Walk through the two lists simultaneously, setting new peers and + * clearing old ones. The two lists are ordered by pointer to peer + * record. + */ + while (n < new_alist->nr_addrs && o < old_alist->nr_addrs) { + struct rxrpc_peer *pn = new_alist->addrs[n].peer; + struct rxrpc_peer *po = old_alist->addrs[o].peer; + + if (pn == po) + continue; + if (pn < po) { + rxrpc_kernel_set_peer_data(pn, data); + n++; + } else { + rxrpc_kernel_set_peer_data(po, 0); + o++; + } + } + + if (n < new_alist->nr_addrs) + for (; n < new_alist->nr_addrs; n++) + rxrpc_kernel_set_peer_data(new_alist->addrs[n].peer, data); + if (o < old_alist->nr_addrs) + for (; o < old_alist->nr_addrs; o++) + rxrpc_kernel_set_peer_data(old_alist->addrs[o].peer, 0); +} diff --git a/fs/afs/addr_prefs.c b/fs/afs/addr_prefs.c new file mode 100644 index 000000000..e8c0640f1 --- /dev/null +++ b/fs/afs/addr_prefs.c @@ -0,0 +1,533 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Address preferences management + * + * Copyright (C) 2023 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": addr_prefs: " fmt +#include +#include +#include +#include +#include +#include "internal.h" + +static inline struct afs_net *afs_seq2net_single(struct seq_file *m) +{ + return afs_net(seq_file_single_net(m)); +} + +/* + * Split a NUL-terminated string up to the first newline around spaces. The + * source string will be modified to have NUL-terminations inserted. + */ +static int afs_split_string(char **pbuf, char *strv[], unsigned int maxstrv) +{ + unsigned int count = 0; + char *p = *pbuf; + + maxstrv--; /* Allow for terminal NULL */ + for (;;) { + /* Skip over spaces */ + while (isspace(*p)) { + if (*p == '\n') { + p++; + break; + } + p++; + } + if (!*p) + break; + + /* Mark start of word */ + if (count >= maxstrv) { + pr_warn("Too many elements in string\n"); + return -EINVAL; + } + strv[count++] = p; + + /* Skip over word */ + while (!isspace(*p) && *p) + p++; + if (!*p) + break; + + /* Mark end of word */ + if (*p == '\n') { + *p++ = 0; + break; + } + *p++ = 0; + } + + *pbuf = p; + strv[count] = NULL; + return count; +} + +/* + * Parse an address with an optional subnet mask. + */ +static int afs_parse_address(char *p, struct afs_addr_preference *pref) +{ + const char *stop; + unsigned long mask, tmp; + char *end = p + strlen(p); + bool bracket = false; + + if (*p == '[') { + p++; + bracket = true; + } + +#if 0 + if (*p == '[') { + p++; + q = memchr(p, ']', end - p); + if (!q) { + pr_warn("Can't find closing ']'\n"); + return -EINVAL; + } + } else { + for (q = p; q < end; q++) + if (*q == '/') + break; + } +#endif + + if (in4_pton(p, end - p, (u8 *)&pref->ipv4_addr, -1, &stop)) { + pref->family = AF_INET; + mask = 32; + } else if (in6_pton(p, end - p, (u8 *)&pref->ipv6_addr, -1, &stop)) { + pref->family = AF_INET6; + mask = 128; + } else { + pr_warn("Can't determine address family\n"); + return -EINVAL; + } + + p = (char *)stop; + if (bracket) { + if (*p != ']') { + pr_warn("Can't find closing ']'\n"); + return -EINVAL; + } + p++; + } + + if (*p == '/') { + p++; + tmp = simple_strtoul(p, &p, 10); + if (tmp > mask) { + pr_warn("Subnet mask too large\n"); + return -EINVAL; + } + if (tmp == 0) { + pr_warn("Subnet mask too small\n"); + return -EINVAL; + } + mask = tmp; + } + + if (*p) { + pr_warn("Invalid address\n"); + return -EINVAL; + } + + pref->subnet_mask = mask; + return 0; +} + +enum cmp_ret { + CONTINUE_SEARCH, + INSERT_HERE, + EXACT_MATCH, + SUBNET_MATCH, +}; + +/* + * See if a candidate address matches a listed address. + */ +static enum cmp_ret afs_cmp_address_pref(const struct afs_addr_preference *a, + const struct afs_addr_preference *b) +{ + int subnet = min(a->subnet_mask, b->subnet_mask); + const __be32 *pa, *pb; + u32 mask, na, nb; + int diff; + + if (a->family != b->family) + return INSERT_HERE; + + switch (a->family) { + case AF_INET6: + pa = a->ipv6_addr.s6_addr32; + pb = b->ipv6_addr.s6_addr32; + break; + case AF_INET: + pa = &a->ipv4_addr.s_addr; + pb = &b->ipv4_addr.s_addr; + break; + } + + while (subnet > 32) { + diff = ntohl(*pa++) - ntohl(*pb++); + if (diff < 0) + return INSERT_HERE; /* a 0) + return CONTINUE_SEARCH; /* a>b */ + subnet -= 32; + } + + if (subnet == 0) + return EXACT_MATCH; + + mask = 0xffffffffU << (32 - subnet); + na = ntohl(*pa); + nb = ntohl(*pb); + diff = (na & mask) - (nb & mask); + //kdebug("diff %08x %08x %08x %d", na, nb, mask, diff); + if (diff < 0) + return INSERT_HERE; /* a 0) + return CONTINUE_SEARCH; /* a>b */ + if (a->subnet_mask == b->subnet_mask) + return EXACT_MATCH; + if (a->subnet_mask > b->subnet_mask) + return SUBNET_MATCH; /* a binds tighter than b */ + return CONTINUE_SEARCH; /* b binds tighter than a */ +} + +/* + * Insert an address preference. + */ +static int afs_insert_address_pref(struct afs_addr_preference_list **_preflist, + struct afs_addr_preference *pref, + int index) +{ + struct afs_addr_preference_list *preflist = *_preflist, *old = preflist; + size_t size, max_prefs; + + _enter("{%u/%u/%u},%u", preflist->ipv6_off, preflist->nr, preflist->max_prefs, index); + + if (preflist->nr == 255) + return -ENOSPC; + if (preflist->nr >= preflist->max_prefs) { + max_prefs = preflist->max_prefs + 1; + size = struct_size(preflist, prefs, max_prefs); + size = roundup_pow_of_two(size); + max_prefs = min_t(size_t, (size - sizeof(*preflist)) / sizeof(*pref), 255); + preflist = kmalloc(size, GFP_KERNEL); + if (!preflist) + return -ENOMEM; + *preflist = **_preflist; + preflist->max_prefs = max_prefs; + *_preflist = preflist; + + if (index < preflist->nr) + memcpy(preflist->prefs + index + 1, old->prefs + index, + sizeof(*pref) * (preflist->nr - index)); + if (index > 0) + memcpy(preflist->prefs, old->prefs, sizeof(*pref) * index); + } else { + if (index < preflist->nr) + memmove(preflist->prefs + index + 1, preflist->prefs + index, + sizeof(*pref) * (preflist->nr - index)); + } + + preflist->prefs[index] = *pref; + preflist->nr++; + if (pref->family == AF_INET) + preflist->ipv6_off++; + return 0; +} + +/* + * Add an address preference. + * echo "add [/] " >/proc/fs/afs/addr_prefs + */ +static int afs_add_address_pref(struct afs_net *net, struct afs_addr_preference_list **_preflist, + int argc, char **argv) +{ + struct afs_addr_preference_list *preflist = *_preflist; + struct afs_addr_preference pref; + enum cmp_ret cmp; + int ret, i, stop; + + if (argc != 3) { + pr_warn("Wrong number of params\n"); + return -EINVAL; + } + + if (strcmp(argv[0], "udp") != 0) { + pr_warn("Unsupported protocol\n"); + return -EINVAL; + } + + ret = afs_parse_address(argv[1], &pref); + if (ret < 0) + return ret; + + ret = kstrtou16(argv[2], 10, &pref.prio); + if (ret < 0) { + pr_warn("Invalid priority\n"); + return ret; + } + + if (pref.family == AF_INET) { + i = 0; + stop = preflist->ipv6_off; + } else { + i = preflist->ipv6_off; + stop = preflist->nr; + } + + for (; i < stop; i++) { + cmp = afs_cmp_address_pref(&pref, &preflist->prefs[i]); + switch (cmp) { + case CONTINUE_SEARCH: + continue; + case INSERT_HERE: + case SUBNET_MATCH: + return afs_insert_address_pref(_preflist, &pref, i); + case EXACT_MATCH: + preflist->prefs[i].prio = pref.prio; + return 0; + } + } + + return afs_insert_address_pref(_preflist, &pref, i); +} + +/* + * Delete an address preference. + */ +static int afs_delete_address_pref(struct afs_addr_preference_list **_preflist, + int index) +{ + struct afs_addr_preference_list *preflist = *_preflist; + + _enter("{%u/%u/%u},%u", preflist->ipv6_off, preflist->nr, preflist->max_prefs, index); + + if (preflist->nr == 0) + return -ENOENT; + + if (index < preflist->nr - 1) + memmove(preflist->prefs + index, preflist->prefs + index + 1, + sizeof(preflist->prefs[0]) * (preflist->nr - index - 1)); + + if (index < preflist->ipv6_off) + preflist->ipv6_off--; + preflist->nr--; + return 0; +} + +/* + * Delete an address preference. + * echo "del [/]" >/proc/fs/afs/addr_prefs + */ +static int afs_del_address_pref(struct afs_net *net, struct afs_addr_preference_list **_preflist, + int argc, char **argv) +{ + struct afs_addr_preference_list *preflist = *_preflist; + struct afs_addr_preference pref; + enum cmp_ret cmp; + int ret, i, stop; + + if (argc != 2) { + pr_warn("Wrong number of params\n"); + return -EINVAL; + } + + if (strcmp(argv[0], "udp") != 0) { + pr_warn("Unsupported protocol\n"); + return -EINVAL; + } + + ret = afs_parse_address(argv[1], &pref); + if (ret < 0) + return ret; + + if (pref.family == AF_INET) { + i = 0; + stop = preflist->ipv6_off; + } else { + i = preflist->ipv6_off; + stop = preflist->nr; + } + + for (; i < stop; i++) { + cmp = afs_cmp_address_pref(&pref, &preflist->prefs[i]); + switch (cmp) { + case CONTINUE_SEARCH: + continue; + case INSERT_HERE: + case SUBNET_MATCH: + return 0; + case EXACT_MATCH: + return afs_delete_address_pref(_preflist, i); + } + } + + return -ENOANO; +} + +/* + * Handle writes to /proc/fs/afs/addr_prefs + */ +int afs_proc_addr_prefs_write(struct file *file, char *buf, size_t size) +{ + struct afs_addr_preference_list *preflist, *old; + struct seq_file *m = file->private_data; + struct afs_net *net = afs_seq2net_single(m); + size_t psize; + char *argv[5]; + int ret, argc, max_prefs; + + inode_lock(file_inode(file)); + + /* Allocate a candidate new list and initialise it from the old. */ + old = rcu_dereference_protected(net->address_prefs, + lockdep_is_held(&file_inode(file)->i_rwsem)); + + if (old) + max_prefs = old->nr + 1; + else + max_prefs = 1; + + psize = struct_size(old, prefs, max_prefs); + psize = roundup_pow_of_two(psize); + max_prefs = min_t(size_t, (psize - sizeof(*old)) / sizeof(old->prefs[0]), 255); + + ret = -ENOMEM; + preflist = kmalloc_flex(*preflist, prefs, max_prefs); + if (!preflist) + goto done; + + if (old) + memcpy(preflist, old, struct_size(preflist, prefs, old->nr)); + else + memset(preflist, 0, sizeof(*preflist)); + preflist->max_prefs = max_prefs; + + do { + argc = afs_split_string(&buf, argv, ARRAY_SIZE(argv)); + if (argc < 0) { + ret = argc; + goto done; + } + if (argc < 2) + goto inval; + + if (strcmp(argv[0], "add") == 0) + ret = afs_add_address_pref(net, &preflist, argc - 1, argv + 1); + else if (strcmp(argv[0], "del") == 0) + ret = afs_del_address_pref(net, &preflist, argc - 1, argv + 1); + else + goto inval; + if (ret < 0) + goto done; + } while (*buf); + + preflist->version++; + rcu_assign_pointer(net->address_prefs, preflist); + /* Store prefs before version */ + smp_store_release(&net->address_pref_version, preflist->version); + kfree_rcu(old, rcu); + preflist = NULL; + ret = 0; + +done: + kfree(preflist); + inode_unlock(file_inode(file)); + _leave(" = %d", ret); + return ret; + +inval: + pr_warn("Invalid Command\n"); + ret = -EINVAL; + goto done; +} + +/* + * Mark the priorities on an address list if the address preferences table has + * changed. The caller must hold the RCU read lock. + */ +void afs_get_address_preferences_rcu(struct afs_net *net, struct afs_addr_list *alist) +{ + const struct afs_addr_preference_list *preflist = + rcu_dereference(net->address_prefs); + const struct sockaddr_in6 *sin6; + const struct sockaddr_in *sin; + const struct sockaddr *sa; + struct afs_addr_preference test; + enum cmp_ret cmp; + int i, j; + + if (!preflist || !preflist->nr || !alist->nr_addrs || + smp_load_acquire(&alist->addr_pref_version) == preflist->version) + return; + + test.family = AF_INET; + test.subnet_mask = 32; + test.prio = 0; + for (i = 0; i < alist->nr_ipv4; i++) { + sa = rxrpc_kernel_remote_addr(alist->addrs[i].peer); + sin = (const struct sockaddr_in *)sa; + test.ipv4_addr = sin->sin_addr; + for (j = 0; j < preflist->ipv6_off; j++) { + cmp = afs_cmp_address_pref(&test, &preflist->prefs[j]); + switch (cmp) { + case CONTINUE_SEARCH: + continue; + case INSERT_HERE: + break; + case EXACT_MATCH: + case SUBNET_MATCH: + WRITE_ONCE(alist->addrs[i].prio, preflist->prefs[j].prio); + break; + } + } + } + + test.family = AF_INET6; + test.subnet_mask = 128; + test.prio = 0; + for (; i < alist->nr_addrs; i++) { + sa = rxrpc_kernel_remote_addr(alist->addrs[i].peer); + sin6 = (const struct sockaddr_in6 *)sa; + test.ipv6_addr = sin6->sin6_addr; + for (j = preflist->ipv6_off; j < preflist->nr; j++) { + cmp = afs_cmp_address_pref(&test, &preflist->prefs[j]); + switch (cmp) { + case CONTINUE_SEARCH: + continue; + case INSERT_HERE: + break; + case EXACT_MATCH: + case SUBNET_MATCH: + WRITE_ONCE(alist->addrs[i].prio, preflist->prefs[j].prio); + break; + } + } + } + + smp_store_release(&alist->addr_pref_version, preflist->version); +} + +/* + * Mark the priorities on an address list if the address preferences table has + * changed. Avoid taking the RCU read lock if we can. + */ +void afs_get_address_preferences(struct afs_net *net, struct afs_addr_list *alist) +{ + if (!net->address_prefs || + /* Load version before prefs */ + smp_load_acquire(&net->address_pref_version) == alist->addr_pref_version) + return; + + rcu_read_lock(); + afs_get_address_preferences_rcu(net, alist); + rcu_read_unlock(); +} diff --git a/fs/afs/afs.h b/fs/afs/afs.h new file mode 100644 index 000000000..ec3db00bd --- /dev/null +++ b/fs/afs/afs.h @@ -0,0 +1,206 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* AFS common types + * + * Copyright (C) 2002, 2007 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#ifndef AFS_H +#define AFS_H + +#include + +#define AFS_MAXCELLNAME 253 /* Maximum length of a cell name (DNS limited) */ +#define AFS_MAXVOLNAME 64 /* Maximum length of a volume name */ +#define AFS_MAXNSERVERS 8 /* Maximum servers in a basic volume record */ +#define AFS_NMAXNSERVERS 13 /* Maximum servers in a N/U-class volume record */ +#define AFS_MAXTYPES 3 /* Maximum number of volume types */ +#define AFSNAMEMAX 256 /* Maximum length of a filename plus NUL */ +#define AFSPATHMAX 1024 /* Maximum length of a pathname plus NUL */ +#define AFSOPAQUEMAX 1024 /* Maximum length of an opaque field */ + +#define AFS_VL_MAX_LIFESPAN 120 +#define AFS_PROBE_MAX_LIFESPAN 30 + +typedef u64 afs_volid_t; +typedef u64 afs_vnodeid_t; +typedef u64 afs_dataversion_t; + +typedef enum { + AFSVL_RWVOL, /* read/write volume */ + AFSVL_ROVOL, /* read-only volume */ + AFSVL_BACKVOL, /* backup volume */ +} __attribute__((packed)) afs_voltype_t; + +typedef enum { + AFS_FTYPE_INVALID = 0, + AFS_FTYPE_FILE = 1, + AFS_FTYPE_DIR = 2, + AFS_FTYPE_SYMLINK = 3, +} afs_file_type_t; + +typedef enum { + AFS_LOCK_READ = 0, /* read lock request */ + AFS_LOCK_WRITE = 1, /* write lock request */ +} afs_lock_type_t; + +#define AFS_LOCKWAIT (5 * 60) /* time until a lock times out (seconds) */ + +/* + * AFS file identifier + */ +struct afs_fid { + afs_volid_t vid; /* volume ID */ + afs_vnodeid_t vnode; /* Lower 64-bits of file index within volume */ + u32 vnode_hi; /* Upper 32-bits of file index */ + u32 unique; /* unique ID number (file index version) */ +}; + +/* + * AFS callback notification + */ +typedef enum { + AFSCM_CB_UNTYPED = 0, /* no type set on CB break */ + AFSCM_CB_EXCLUSIVE = 1, /* CB exclusive to CM [not implemented] */ + AFSCM_CB_SHARED = 2, /* CB shared by other CM's */ + AFSCM_CB_DROPPED = 3, /* CB promise cancelled by file server */ +} afs_callback_type_t; + +struct afs_callback { + time64_t expires_at; /* Time at which expires */ + //unsigned version; /* Callback version */ + //afs_callback_type_t type; /* Type of callback */ +}; + +struct afs_callback_break { + struct afs_fid fid; /* File identifier */ + //struct afs_callback cb; /* Callback details */ +}; + +#define AFSCBMAX 50 /* maximum callbacks transferred per bulk op */ + +struct afs_uuid { + __be32 time_low; /* low part of timestamp */ + __be16 time_mid; /* mid part of timestamp */ + __be16 time_hi_and_version; /* high part of timestamp and version */ + __s8 clock_seq_hi_and_reserved; /* clock seq hi and variant */ + __s8 clock_seq_low; /* clock seq low */ + __s8 node[6]; /* spatially unique node ID (MAC addr) */ +}; + +/* + * AFS volume information + */ +struct afs_volume_info { + afs_volid_t vid; /* volume ID */ + afs_voltype_t type; /* type of this volume */ + afs_volid_t type_vids[5]; /* volume ID's for possible types for this vol */ + + /* list of fileservers serving this volume */ + size_t nservers; /* number of entries used in servers[] */ + struct { + struct in_addr addr; /* fileserver address */ + } servers[8]; +}; + +/* + * AFS security ACE access mask + */ +typedef u32 afs_access_t; +#define AFS_ACE_READ 0x00000001U /* - permission to read a file/dir */ +#define AFS_ACE_WRITE 0x00000002U /* - permission to write/chmod a file */ +#define AFS_ACE_INSERT 0x00000004U /* - permission to create dirent in a dir */ +#define AFS_ACE_LOOKUP 0x00000008U /* - permission to lookup a file/dir in a dir */ +#define AFS_ACE_DELETE 0x00000010U /* - permission to delete a dirent from a dir */ +#define AFS_ACE_LOCK 0x00000020U /* - permission to lock a file */ +#define AFS_ACE_ADMINISTER 0x00000040U /* - permission to change ACL */ +#define AFS_ACE_USER_A 0x01000000U /* - 'A' user-defined permission */ +#define AFS_ACE_USER_B 0x02000000U /* - 'B' user-defined permission */ +#define AFS_ACE_USER_C 0x04000000U /* - 'C' user-defined permission */ +#define AFS_ACE_USER_D 0x08000000U /* - 'D' user-defined permission */ +#define AFS_ACE_USER_E 0x10000000U /* - 'E' user-defined permission */ +#define AFS_ACE_USER_F 0x20000000U /* - 'F' user-defined permission */ +#define AFS_ACE_USER_G 0x40000000U /* - 'G' user-defined permission */ +#define AFS_ACE_USER_H 0x80000000U /* - 'H' user-defined permission */ + +/* + * AFS file status information + */ +struct afs_file_status { + u64 size; /* file size */ + afs_dataversion_t data_version; /* current data version */ + struct timespec64 mtime_client; /* Last time client changed data */ + struct timespec64 mtime_server; /* Last time server changed data */ + s64 author; /* author ID */ + s64 owner; /* owner ID */ + s64 group; /* group ID */ + afs_access_t caller_access; /* access rights for authenticated caller */ + afs_access_t anon_access; /* access rights for unauthenticated caller */ + umode_t mode; /* UNIX mode */ + afs_file_type_t type; /* file type */ + u32 nlink; /* link count */ + s32 lock_count; /* file lock count (0=UNLK -1=WRLCK +ve=#RDLCK */ + u32 abort_code; /* Abort if bulk-fetching this failed */ +}; + +struct afs_status_cb { + struct afs_file_status status; + struct afs_callback callback; + bool have_status; /* True if status record was retrieved */ + bool have_cb; /* True if cb record was retrieved */ + bool have_error; /* True if status.abort_code indicates an error */ +}; + +/* + * AFS file status change request + */ + +#define AFS_SET_MTIME 0x01 /* set the mtime */ +#define AFS_SET_OWNER 0x02 /* set the owner ID */ +#define AFS_SET_GROUP 0x04 /* set the group ID (unsupported?) */ +#define AFS_SET_MODE 0x08 /* set the UNIX mode */ +#define AFS_SET_SEG_SIZE 0x10 /* set the segment size (unsupported) */ + +/* + * AFS volume synchronisation information + */ +struct afs_volsync { + time64_t creation; /* Volume creation time (or TIME64_MIN) */ + time64_t update; /* Volume update time (or TIME64_MIN) */ +}; + +/* + * AFS volume status record + */ +struct afs_volume_status { + afs_volid_t vid; /* volume ID */ + afs_volid_t parent_id; /* parent volume ID */ + u8 online; /* true if volume currently online and available */ + u8 in_service; /* true if volume currently in service */ + u8 blessed; /* same as in_service */ + u8 needs_salvage; /* true if consistency checking required */ + u32 type; /* volume type (afs_voltype_t) */ + u64 min_quota; /* minimum space set aside (blocks) */ + u64 max_quota; /* maximum space this volume may occupy (blocks) */ + u64 blocks_in_use; /* space this volume currently occupies (blocks) */ + u64 part_blocks_avail; /* space available in volume's partition */ + u64 part_max_blocks; /* size of volume's partition */ + s64 vol_copy_date; + s64 vol_backup_date; +}; + +#define AFS_BLOCK_SIZE 1024 + +/* + * XDR encoding of UUID in AFS. + */ +struct afs_uuid__xdr { + __be32 time_low; + __be32 time_mid; + __be32 time_hi_and_version; + __be32 clock_seq_hi_and_reserved; + __be32 clock_seq_low; + __be32 node[6]; +}; + +#endif /* AFS_H */ diff --git a/fs/afs/afs_cm.h b/fs/afs/afs_cm.h new file mode 100644 index 000000000..565cbe0a8 --- /dev/null +++ b/fs/afs/afs_cm.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* AFS Cache Manager definitions + * + * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#ifndef AFS_CM_H +#define AFS_CM_H + +#define AFS_CM_PORT 7001 /* AFS file server port */ +#define CM_SERVICE 1 /* AFS File Service ID */ + +enum AFS_CM_Operations { + CBCallBack = 204, /* break callback promises */ + CBInitCallBackState = 205, /* initialise callback state */ + CBProbe = 206, /* probe client */ + CBGetLock = 207, /* get contents of CM lock table */ + CBGetCE = 208, /* get cache file description */ + CBGetXStatsVersion = 209, /* get version of extended statistics */ + CBGetXStats = 210, /* get contents of extended statistics data */ + CBInitCallBackState3 = 213, /* initialise callback state, version 3 */ + CBProbeUuid = 214, /* check the client hasn't rebooted */ + CBTellMeAboutYourself = 65538, /* get client capabilities */ +}; + +#define AFS_CAP_ERROR_TRANSLATION 0x1 + +#endif /* AFS_FS_H */ diff --git a/fs/afs/afs_fs.h b/fs/afs/afs_fs.h new file mode 100644 index 000000000..20ab344ba --- /dev/null +++ b/fs/afs/afs_fs.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* AFS File Service definitions + * + * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#ifndef AFS_FS_H +#define AFS_FS_H + +#define AFS_FS_PORT 7000 /* AFS file server port */ +#define FS_SERVICE 1 /* AFS File Service ID */ + +enum AFS_FS_Operations { + FSFETCHDATA = 130, /* AFS Fetch file data */ + FSFETCHACL = 131, /* AFS Fetch file ACL */ + FSFETCHSTATUS = 132, /* AFS Fetch file status */ + FSSTOREDATA = 133, /* AFS Store file data */ + FSSTOREACL = 134, /* AFS Store file ACL */ + FSSTORESTATUS = 135, /* AFS Store file status */ + FSREMOVEFILE = 136, /* AFS Remove a file */ + FSCREATEFILE = 137, /* AFS Create a file */ + FSRENAME = 138, /* AFS Rename or move a file or directory */ + FSSYMLINK = 139, /* AFS Create a symbolic link */ + FSLINK = 140, /* AFS Create a hard link */ + FSMAKEDIR = 141, /* AFS Create a directory */ + FSREMOVEDIR = 142, /* AFS Remove a directory */ + FSGIVEUPCALLBACKS = 147, /* AFS Discard callback promises */ + FSGETVOLUMEINFO = 148, /* AFS Get information about a volume */ + FSGETVOLUMESTATUS = 149, /* AFS Get volume status information */ + FSGETROOTVOLUME = 151, /* AFS Get root volume name */ + FSBULKSTATUS = 155, /* AFS Fetch multiple file statuses */ + FSSETLOCK = 156, /* AFS Request a file lock */ + FSEXTENDLOCK = 157, /* AFS Extend a file lock */ + FSRELEASELOCK = 158, /* AFS Release a file lock */ + FSLOOKUP = 161, /* AFS lookup file in directory */ + FSINLINEBULKSTATUS = 65536, /* AFS Fetch multiple file statuses with inline errors */ + FSFETCHDATA64 = 65537, /* AFS Fetch file data */ + FSSTOREDATA64 = 65538, /* AFS Store file data */ + FSGIVEUPALLCALLBACKS = 65539, /* AFS Give up all outstanding callbacks on a server */ + FSGETCAPABILITIES = 65540, /* Probe and get the capabilities of a fileserver */ +}; + +enum AFS_FS_Errors { + VRESTARTING = -100, /* Server is restarting */ + VSALVAGE = 101, /* volume needs salvaging */ + VNOVNODE = 102, /* no such file/dir (vnode) */ + VNOVOL = 103, /* no such volume or volume unavailable */ + VVOLEXISTS = 104, /* volume name already exists */ + VNOSERVICE = 105, /* volume not currently in service */ + VOFFLINE = 106, /* volume is currently offline (more info available [VVL-spec]) */ + VONLINE = 107, /* volume is already online */ + VDISKFULL = 108, /* disk partition is full */ + VOVERQUOTA = 109, /* volume's maximum quota exceeded */ + VBUSY = 110, /* volume is temporarily unavailable */ + VMOVED = 111, /* volume moved to new server - ask this FS where */ + VIO = 112, /* I/O error in volume */ + VSALVAGING = 113, /* Volume is being salvaged */ + VRESTRICTED = 120, /* Volume is restricted from using */ +}; + +#endif /* AFS_FS_H */ diff --git a/fs/afs/afs_vl.h b/fs/afs/afs_vl.h new file mode 100644 index 000000000..b835e25a2 --- /dev/null +++ b/fs/afs/afs_vl.h @@ -0,0 +1,138 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* AFS Volume Location Service client interface + * + * Copyright (C) 2002, 2007 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#ifndef AFS_VL_H +#define AFS_VL_H + +#include "afs.h" + +#define AFS_VL_PORT 7003 /* volume location service port */ +#define VL_SERVICE 52 /* RxRPC service ID for the Volume Location service */ +#define YFS_VL_SERVICE 2503 /* Service ID for AuriStor upgraded VL service */ +#define YFS_VL_MAXCELLNAME 256 /* Maximum length of a cell name in YFS protocol */ + +enum AFSVL_Operations { + VLGETENTRYBYID = 503, /* AFS Get VLDB entry by ID */ + VLGETENTRYBYNAME = 504, /* AFS Get VLDB entry by name */ + VLPROBE = 514, /* AFS probe VL service */ + VLGETENTRYBYIDU = 526, /* AFS Get VLDB entry by ID (UUID-variant) */ + VLGETENTRYBYNAMEU = 527, /* AFS Get VLDB entry by name (UUID-variant) */ + VLGETADDRSU = 533, /* AFS Get addrs for fileserver */ + YVLGETENDPOINTS = 64002, /* YFS Get endpoints for file/volume server */ + YVLGETCELLNAME = 64014, /* YFS Get actual cell name */ + VLGETCAPABILITIES = 65537, /* AFS Get server capabilities */ +}; + +enum AFSVL_Errors { + AFSVL_IDEXIST = 363520, /* Volume Id entry exists in vl database */ + AFSVL_IO = 363521, /* I/O related error */ + AFSVL_NAMEEXIST = 363522, /* Volume name entry exists in vl database */ + AFSVL_CREATEFAIL = 363523, /* Internal creation failure */ + AFSVL_NOENT = 363524, /* No such entry */ + AFSVL_EMPTY = 363525, /* Vl database is empty */ + AFSVL_ENTDELETED = 363526, /* Entry is deleted (soft delete) */ + AFSVL_BADNAME = 363527, /* Volume name is illegal */ + AFSVL_BADINDEX = 363528, /* Index is out of range */ + AFSVL_BADVOLTYPE = 363529, /* Bad volume type */ + AFSVL_BADSERVER = 363530, /* Illegal server number (out of range) */ + AFSVL_BADPARTITION = 363531, /* Bad partition number */ + AFSVL_REPSFULL = 363532, /* Run out of space for Replication sites */ + AFSVL_NOREPSERVER = 363533, /* No such Replication server site exists */ + AFSVL_DUPREPSERVER = 363534, /* Replication site already exists */ + AFSVL_RWNOTFOUND = 363535, /* Parent R/W entry not found */ + AFSVL_BADREFCOUNT = 363536, /* Illegal Reference Count number */ + AFSVL_SIZEEXCEEDED = 363537, /* Vl size for attributes exceeded */ + AFSVL_BADENTRY = 363538, /* Bad incoming vl entry */ + AFSVL_BADVOLIDBUMP = 363539, /* Illegal max volid increment */ + AFSVL_IDALREADYHASHED = 363540, /* RO/BACK id already hashed */ + AFSVL_ENTRYLOCKED = 363541, /* Vl entry is already locked */ + AFSVL_BADVOLOPER = 363542, /* Bad volume operation code */ + AFSVL_BADRELLOCKTYPE = 363543, /* Bad release lock type */ + AFSVL_RERELEASE = 363544, /* Status report: last release was aborted */ + AFSVL_BADSERVERFLAG = 363545, /* Invalid replication site server flag */ + AFSVL_PERM = 363546, /* No permission access */ + AFSVL_NOMEM = 363547, /* malloc/realloc failed to alloc enough memory */ +}; + +enum { + YFS_SERVER_INDEX = 0, + YFS_SERVER_UUID = 1, + YFS_SERVER_ENDPOINT = 2, +}; + +enum { + YFS_ENDPOINT_IPV4 = 0, + YFS_ENDPOINT_IPV6 = 1, +}; + +#define YFS_MAXENDPOINTS 16 + +/* + * maps to "struct vldbentry" in vvl-spec.pdf + */ +struct afs_vldbentry { + char name[65]; /* name of volume (with NUL char) */ + afs_voltype_t type; /* volume type */ + unsigned num_servers; /* num servers that hold instances of this vol */ + unsigned clone_id; /* cloning ID */ + + unsigned flags; +#define AFS_VLF_RWEXISTS 0x1000 /* R/W volume exists */ +#define AFS_VLF_ROEXISTS 0x2000 /* R/O volume exists */ +#define AFS_VLF_BACKEXISTS 0x4000 /* backup volume exists */ + + afs_volid_t volume_ids[3]; /* volume IDs */ + + struct { + struct in_addr addr; /* server address */ + unsigned partition; /* partition ID on this server */ + unsigned flags; /* server specific flags */ +#define AFS_VLSF_NEWREPSITE 0x0001 /* Ignore all 'non-new' servers */ +#define AFS_VLSF_ROVOL 0x0002 /* this server holds a R/O instance of the volume */ +#define AFS_VLSF_RWVOL 0x0004 /* this server holds a R/W instance of the volume */ +#define AFS_VLSF_BACKVOL 0x0008 /* this server holds a backup instance of the volume */ +#define AFS_VLSF_UUID 0x0010 /* This server is referred to by its UUID */ +#define AFS_VLSF_DONTUSE 0x0020 /* This server ref should be ignored */ + } servers[8]; +}; + +#define AFS_VLDB_MAXNAMELEN 65 + + +struct afs_ListAddrByAttributes__xdr { + __be32 Mask; +#define AFS_VLADDR_IPADDR 0x1 /* Match by ->ipaddr */ +#define AFS_VLADDR_INDEX 0x2 /* Match by ->index */ +#define AFS_VLADDR_UUID 0x4 /* Match by ->uuid */ + __be32 ipaddr; + __be32 index; + __be32 spare; + struct afs_uuid__xdr uuid; +}; + +struct afs_uvldbentry__xdr { + __be32 name[AFS_VLDB_MAXNAMELEN]; + __be32 nServers; + struct afs_uuid__xdr serverNumber[AFS_NMAXNSERVERS]; + __be32 serverUnique[AFS_NMAXNSERVERS]; + __be32 serverPartition[AFS_NMAXNSERVERS]; + __be32 serverFlags[AFS_NMAXNSERVERS]; + __be32 volumeId[AFS_MAXTYPES]; + __be32 cloneId; + __be32 flags; + __be32 spares1; + __be32 spares2; + __be32 spares3; + __be32 spares4; + __be32 spares5; + __be32 spares6; + __be32 spares7; + __be32 spares8; + __be32 spares9; +}; + +#endif /* AFS_VL_H */ diff --git a/fs/afs/callback.c b/fs/afs/callback.c new file mode 100644 index 000000000..61354003c --- /dev/null +++ b/fs/afs/callback.c @@ -0,0 +1,262 @@ +/* + * Copyright (c) 2002, 2007 Red Hat, Inc. All rights reserved. + * + * This software may be freely redistributed under the terms of the + * GNU General Public License. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + * Authors: David Woodhouse + * David Howells + * + */ + +#include +#include +#include +#include +#include +#include "internal.h" + +/* + * Handle invalidation of an mmap'd file. We invalidate all the PTEs referring + * to the pages in this file's pagecache, forcing the kernel to go through + * ->fault() or ->page_mkwrite() - at which point we can handle invalidation + * more fully. + */ +void afs_invalidate_mmap_work(struct work_struct *work) +{ + struct afs_vnode *vnode = container_of(work, struct afs_vnode, cb_work); + + unmap_mapping_pages(vnode->netfs.inode.i_mapping, 0, 0, false); +} + +static void afs_volume_init_callback(struct afs_volume *volume) +{ + struct afs_vnode *vnode; + + down_read(&volume->open_mmaps_lock); + + list_for_each_entry(vnode, &volume->open_mmaps, cb_mmap_link) { + if (vnode->cb_v_check != atomic_read(&volume->cb_v_break)) { + afs_clear_cb_promise(vnode, afs_cb_promise_clear_vol_init_cb); + queue_work(system_dfl_wq, &vnode->cb_work); + } + } + + up_read(&volume->open_mmaps_lock); +} + +/* + * Allow the fileserver to request callback state (re-)initialisation. + * Unfortunately, UUIDs are not guaranteed unique. + */ +void afs_init_callback_state(struct afs_server *server) +{ + struct afs_server_entry *se; + + down_read(&server->cell->vs_lock); + + list_for_each_entry(se, &server->volumes, slink) { + se->cb_expires_at = AFS_NO_CB_PROMISE; + se->volume->cb_expires_at = AFS_NO_CB_PROMISE; + trace_afs_cb_v_break(se->volume->vid, atomic_read(&se->volume->cb_v_break), + afs_cb_break_for_s_reinit); + if (!list_empty(&se->volume->open_mmaps)) + afs_volume_init_callback(se->volume); + } + + up_read(&server->cell->vs_lock); +} + +/* + * actually break a callback + */ +void __afs_break_callback(struct afs_vnode *vnode, enum afs_cb_break_reason reason) +{ + _enter(""); + + clear_bit(AFS_VNODE_NEW_CONTENT, &vnode->flags); + if (afs_clear_cb_promise(vnode, afs_cb_promise_clear_cb_break)) { + vnode->cb_break++; + vnode->cb_v_check = atomic_read(&vnode->volume->cb_v_break); + afs_clear_permits(vnode); + + if (vnode->lock_state == AFS_VNODE_LOCK_WAITING_FOR_CB) + afs_lock_may_be_available(vnode); + + if (reason != afs_cb_break_for_deleted && + vnode->status.type == AFS_FTYPE_FILE && + atomic_read(&vnode->cb_nr_mmap)) + queue_work(system_dfl_wq, &vnode->cb_work); + + trace_afs_cb_break(&vnode->fid, vnode->cb_break, reason, true); + } else { + trace_afs_cb_break(&vnode->fid, vnode->cb_break, reason, false); + } +} + +void afs_break_callback(struct afs_vnode *vnode, enum afs_cb_break_reason reason) +{ + write_seqlock(&vnode->cb_lock); + __afs_break_callback(vnode, reason); + write_sequnlock(&vnode->cb_lock); +} + +/* + * Look up a volume by volume ID under RCU conditions. + */ +static struct afs_volume *afs_lookup_volume_rcu(struct afs_cell *cell, + afs_volid_t vid) +{ + struct afs_volume *volume = NULL; + struct rb_node *p; + + scoped_seqlock_read(&cell->volume_lock, ss_lock) { + /* Unfortunately, rbtree walking doesn't give reliable results + * under just the RCU read lock, so we have to check for + * changes. + */ + p = rcu_dereference_raw(cell->volumes.rb_node); + while (p) { + volume = rb_entry(p, struct afs_volume, cell_node); + + if (volume->vid < vid) + p = rcu_dereference_raw(p->rb_left); + else if (volume->vid > vid) + p = rcu_dereference_raw(p->rb_right); + else + break; + volume = NULL; + } + + if (volume && afs_try_get_volume(volume, afs_volume_trace_get_callback)) + break; + volume = NULL; + } + + return volume; +} + +/* + * Allow the fileserver to break callbacks at the volume-level. This is + * typically done when, for example, a R/W volume is snapshotted to a R/O + * volume (the only way to change an R/O volume). It may also, however, happen + * when a volserver takes control of a volume (offlining it, moving it, etc.). + * + * Every file in that volume will need to be reevaluated. + */ +static void afs_break_volume_callback(struct afs_server *server, + struct afs_volume *volume) + __releases(RCU) +{ + struct afs_server_list *slist = rcu_dereference(volume->servers); + unsigned int i, cb_v_break; + + write_lock(&volume->cb_v_break_lock); + + for (i = 0; i < slist->nr_servers; i++) + if (slist->servers[i].server == server) + slist->servers[i].cb_expires_at = AFS_NO_CB_PROMISE; + volume->cb_expires_at = AFS_NO_CB_PROMISE; + + cb_v_break = atomic_inc_return_release(&volume->cb_v_break); + trace_afs_cb_v_break(volume->vid, cb_v_break, afs_cb_break_for_volume_callback); + + write_unlock(&volume->cb_v_break_lock); + rcu_read_unlock(); + + if (!list_empty(&volume->open_mmaps)) + afs_volume_init_callback(volume); +} + +/* + * allow the fileserver to explicitly break one callback + * - happens when + * - the backing file is changed + * - a lock is released + */ +static void afs_break_one_callback(struct afs_server *server, + struct afs_volume *volume, + struct afs_fid *fid) +{ + struct super_block *sb; + struct afs_vnode *vnode; + struct inode *inode; + + /* See if we can find a matching inode - even an I_NEW inode needs to + * be marked as it can have its callback broken before we finish + * setting up the local inode. + */ + sb = rcu_dereference(volume->sb); + if (!sb) + return; + + inode = find_inode_rcu(sb, fid->vnode, afs_ilookup5_test_by_fid, fid); + if (inode) { + vnode = AFS_FS_I(inode); + afs_break_callback(vnode, afs_cb_break_for_callback); + } else { + trace_afs_cb_miss(fid, afs_cb_break_for_callback); + } +} + +static void afs_break_some_callbacks(struct afs_server *server, + struct afs_callback_break *cbb, + size_t *_count) +{ + struct afs_callback_break *residue = cbb; + struct afs_volume *volume; + afs_volid_t vid = cbb->fid.vid; + size_t i; + + rcu_read_lock(); + volume = afs_lookup_volume_rcu(server->cell, vid); + if (!volume) { + /* Ignore breaks on unknown volumes. */ + rcu_read_unlock(); + *_count = 0; + } else if (cbb->fid.vnode == 0 && cbb->fid.unique == 0) { + afs_break_volume_callback(server, volume); + *_count -= 1; + if (*_count) + memmove(cbb, cbb + 1, sizeof(*cbb) * *_count); + } else { + /* TODO: Find all matching volumes if we couldn't match the server and + * break them anyway. + */ + + for (i = *_count; i > 0; cbb++, i--) { + if (cbb->fid.vid == vid) { + _debug("- Fid { vl=%08llx n=%llu u=%u }", + cbb->fid.vid, + cbb->fid.vnode, + cbb->fid.unique); + --*_count; + if (volume) + afs_break_one_callback(server, volume, &cbb->fid); + } else { + *residue++ = *cbb; + } + } + rcu_read_unlock(); + } + + afs_put_volume(volume, afs_volume_trace_put_callback); +} + +/* + * allow the fileserver to break callback promises + */ +void afs_break_callbacks(struct afs_server *server, size_t count, + struct afs_callback_break *callbacks) +{ + _enter("%p,%zu,", server, count); + + ASSERT(server != NULL); + + while (count > 0) + afs_break_some_callbacks(server, callbacks, &count); +} diff --git a/fs/afs/cell.c b/fs/afs/cell.c new file mode 100644 index 000000000..47a264576 --- /dev/null +++ b/fs/afs/cell.c @@ -0,0 +1,939 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* AFS cell and server record management + * + * Copyright (C) 2002, 2017 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +static unsigned __read_mostly afs_cell_gc_delay = 10; +static unsigned __read_mostly afs_cell_min_ttl = 10 * 60; +static unsigned __read_mostly afs_cell_max_ttl = 24 * 60 * 60; +static atomic_t cell_debug_id; + +static void afs_cell_timer(struct timer_list *timer); +static void afs_destroy_cell_work(struct work_struct *work); +static void afs_manage_cell_work(struct work_struct *work); + +static void afs_dec_cells_outstanding(struct afs_net *net) +{ + if (atomic_dec_and_test(&net->cells_outstanding)) + wake_up_var(&net->cells_outstanding); +} + +static void afs_set_cell_state(struct afs_cell *cell, enum afs_cell_state state) +{ + smp_store_release(&cell->state, state); /* Commit cell changes before state */ + smp_wmb(); /* Set cell state before task state */ + wake_up_var(&cell->state); +} + +/* + * Look up and get an activation reference on a cell record. The caller must + * hold net->cells_lock at least read-locked. + */ +static struct afs_cell *afs_find_cell_locked(struct afs_net *net, + const char *name, unsigned int namesz, + enum afs_cell_trace reason) +{ + struct afs_cell *cell = NULL; + struct rb_node *p; + int n; + + _enter("%*.*s", namesz, namesz, name); + + if (name && namesz == 0) + return ERR_PTR(-EINVAL); + if (namesz > AFS_MAXCELLNAME) + return ERR_PTR(-ENAMETOOLONG); + + if (!name) { + cell = rcu_dereference_protected(net->ws_cell, + lockdep_is_held(&net->cells_lock)); + if (!cell) + return ERR_PTR(-EDESTADDRREQ); + goto found; + } + + p = net->cells.rb_node; + while (p) { + cell = rb_entry(p, struct afs_cell, net_node); + + n = strncasecmp(cell->name, name, + min_t(size_t, cell->name_len, namesz)); + if (n == 0) + n = cell->name_len - namesz; + if (n < 0) + p = p->rb_left; + else if (n > 0) + p = p->rb_right; + else + goto found; + } + + return ERR_PTR(-ENOENT); + +found: + return afs_use_cell(cell, reason); +} + +/* + * Look up and get an activation reference on a cell record. + */ +struct afs_cell *afs_find_cell(struct afs_net *net, + const char *name, unsigned int namesz, + enum afs_cell_trace reason) +{ + struct afs_cell *cell; + + down_read(&net->cells_lock); + cell = afs_find_cell_locked(net, name, namesz, reason); + up_read(&net->cells_lock); + return cell; +} + +/* + * Set up a cell record and fill in its name, VL server address list and + * allocate an anonymous key + */ +static struct afs_cell *afs_alloc_cell(struct afs_net *net, + const char *name, unsigned int namelen, + const char *addresses) +{ + struct afs_vlserver_list *vllist = NULL; + struct afs_cell *cell; + int i, ret; + + ASSERT(name); + if (namelen == 0) + return ERR_PTR(-EINVAL); + if (namelen > AFS_MAXCELLNAME) { + _leave(" = -ENAMETOOLONG"); + return ERR_PTR(-ENAMETOOLONG); + } + + /* Prohibit cell names that contain unprintable chars, '/' and '@' or + * that begin with a dot. This also precludes "@cell". + */ + if (name[0] == '.') + return ERR_PTR(-EINVAL); + for (i = 0; i < namelen; i++) { + char ch = name[i]; + if (!isprint(ch) || ch == '/' || ch == '@') + return ERR_PTR(-EINVAL); + } + + _enter("%*.*s,%s", namelen, namelen, name, addresses); + + cell = kzalloc_obj(struct afs_cell); + if (!cell) { + _leave(" = -ENOMEM"); + return ERR_PTR(-ENOMEM); + } + + /* Allocate the cell name and the key name in one go. */ + cell->name = kmalloc(1 + namelen + 1 + + 4 + namelen + 1, GFP_KERNEL); + if (!cell->name) { + kfree(cell); + return ERR_PTR(-ENOMEM); + } + + cell->name[0] = '.'; + cell->name++; + cell->name_len = namelen; + for (i = 0; i < namelen; i++) + cell->name[i] = tolower(name[i]); + cell->name[i++] = 0; + + cell->key_desc = cell->name + i; + memcpy(cell->key_desc, "afs@", 4); + memcpy(cell->key_desc + 4, cell->name, cell->name_len + 1); + + cell->net = net; + refcount_set(&cell->ref, 1); + atomic_set(&cell->active, 0); + INIT_WORK(&cell->destroyer, afs_destroy_cell_work); + INIT_WORK(&cell->manager, afs_manage_cell_work); + timer_setup(&cell->management_timer, afs_cell_timer, 0); + init_rwsem(&cell->vs_lock); + cell->volumes = RB_ROOT; + INIT_HLIST_HEAD(&cell->proc_volumes); + seqlock_init(&cell->volume_lock); + cell->fs_servers = RB_ROOT; + init_rwsem(&cell->fs_lock); + rwlock_init(&cell->vl_servers_lock); + cell->flags = (1 << AFS_CELL_FL_CHECK_ALIAS); + + /* Provide a VL server list, filling it in if we were given a list of + * addresses to use. + */ + if (addresses) { + vllist = afs_parse_text_addrs(net, + addresses, strlen(addresses), ':', + VL_SERVICE, AFS_VL_PORT); + if (IS_ERR(vllist)) { + ret = PTR_ERR(vllist); + vllist = NULL; + goto parse_failed; + } + + vllist->source = DNS_RECORD_FROM_CONFIG; + vllist->status = DNS_LOOKUP_NOT_DONE; + cell->dns_expiry = TIME64_MAX; + } else { + ret = -ENOMEM; + vllist = afs_alloc_vlserver_list(0); + if (!vllist) + goto error; + vllist->source = DNS_RECORD_UNAVAILABLE; + vllist->status = DNS_LOOKUP_NOT_DONE; + cell->dns_expiry = ktime_get_real_seconds(); + } + + rcu_assign_pointer(cell->vl_servers, vllist); + + cell->dns_source = vllist->source; + cell->dns_status = vllist->status; + smp_store_release(&cell->dns_lookup_count, 1); /* vs source/status */ + atomic_inc(&net->cells_outstanding); + cell->debug_id = atomic_inc_return(&cell_debug_id); + + trace_afs_cell(cell->debug_id, 1, 0, afs_cell_trace_alloc); + + _leave(" = %p", cell); + return cell; + +parse_failed: + if (ret == -EINVAL) + printk(KERN_ERR "kAFS: bad VL server IP address\n"); +error: + afs_put_vlserverlist(cell->net, vllist); + kfree(cell->name - 1); + kfree(cell); + _leave(" = %d", ret); + return ERR_PTR(ret); +} + +/* + * afs_lookup_cell - Look up or create a cell record. + * @net: The network namespace + * @name: The name of the cell. + * @namesz: The strlen of the cell name. + * @vllist: A colon/comma separated list of numeric IP addresses or NULL. + * @reason: The reason we're doing the lookup + * @trace: The reason to be logged if the lookup is successful. + * + * Look up a cell record by name and query the DNS for VL server addresses if + * needed. Note that that actual DNS query is punted off to the manager thread + * so that this function can return immediately if interrupted whilst allowing + * cell records to be shared even if not yet fully constructed. + */ +struct afs_cell *afs_lookup_cell(struct afs_net *net, + const char *name, unsigned int namesz, + const char *vllist, + enum afs_lookup_cell_for reason, + enum afs_cell_trace trace) +{ + struct afs_cell *cell, *candidate, *cursor; + struct rb_node *parent, **pp; + enum afs_cell_state state; + int ret, n; + + _enter("%s,%s,%u", name, vllist, reason); + + if (reason != AFS_LOOKUP_CELL_PRELOAD) { + cell = afs_find_cell(net, name, namesz, trace); + if (!IS_ERR(cell)) { + if (reason == AFS_LOOKUP_CELL_DYNROOT) + goto no_wait; + if (cell->state == AFS_CELL_SETTING_UP || + cell->state == AFS_CELL_UNLOOKED) + goto lookup_cell; + goto wait_for_cell; + } + } + + /* Assume we're probably going to create a cell and preallocate and + * mostly set up a candidate record. We can then use this to stash the + * name, the net namespace and VL server addresses. + * + * We also want to do this before we hold any locks as it may involve + * upcalling to userspace to make DNS queries. + */ + candidate = afs_alloc_cell(net, name, namesz, vllist); + if (IS_ERR(candidate)) { + _leave(" = %ld", PTR_ERR(candidate)); + return candidate; + } + + /* Find the insertion point and check to see if someone else added a + * cell whilst we were allocating. + */ + down_write(&net->cells_lock); + + pp = &net->cells.rb_node; + parent = NULL; + while (*pp) { + parent = *pp; + cursor = rb_entry(parent, struct afs_cell, net_node); + + n = strncasecmp(cursor->name, name, + min_t(size_t, cursor->name_len, namesz)); + if (n == 0) + n = cursor->name_len - namesz; + if (n < 0) + pp = &(*pp)->rb_left; + else if (n > 0) + pp = &(*pp)->rb_right; + else + goto cell_already_exists; + } + + ret = idr_alloc_cyclic(&net->cells_dyn_ino, candidate, + 2, INT_MAX / 2, GFP_KERNEL); + if (ret < 0) + goto cant_alloc_ino; + candidate->dynroot_ino = ret; + set_bit(AFS_CELL_FL_HAVE_INO, &candidate->flags); + + cell = candidate; + candidate = NULL; + afs_use_cell(cell, trace); + rb_link_node_rcu(&cell->net_node, parent, pp); + rb_insert_color(&cell->net_node, &net->cells); + up_write(&net->cells_lock); + +lookup_cell: + if (reason != AFS_LOOKUP_CELL_PRELOAD && + reason != AFS_LOOKUP_CELL_ROOTCELL) { + set_bit(AFS_CELL_FL_DO_LOOKUP, &cell->flags); + afs_queue_cell(cell, afs_cell_trace_queue_new); + } + +wait_for_cell: + state = smp_load_acquire(&cell->state); /* vs error */ + switch (state) { + case AFS_CELL_ACTIVE: + case AFS_CELL_DEAD: + break; + case AFS_CELL_UNLOOKED: + default: + if (reason == AFS_LOOKUP_CELL_PRELOAD || + reason == AFS_LOOKUP_CELL_ROOTCELL) + break; + _debug("wait_for_cell"); + afs_see_cell(cell, afs_cell_trace_wait); + wait_var_event(&cell->state, + ({ + state = smp_load_acquire(&cell->state); /* vs error */ + state == AFS_CELL_ACTIVE || state == AFS_CELL_DEAD; + })); + _debug("waited_for_cell %d %d", cell->state, cell->error); + } + +no_wait: + /* Check the state obtained from the wait check. */ + state = smp_load_acquire(&cell->state); /* vs error */ + if (state == AFS_CELL_DEAD) { + ret = cell->error; + goto error; + } + if (state == AFS_CELL_ACTIVE) { + switch (cell->dns_status) { + case DNS_LOOKUP_NOT_DONE: + if (cell->dns_source == DNS_RECORD_FROM_CONFIG) { + ret = 0; + break; + } + fallthrough; + default: + ret = -EIO; + goto error; + case DNS_LOOKUP_GOOD: + case DNS_LOOKUP_GOOD_WITH_BAD: + ret = 0; + break; + case DNS_LOOKUP_GOT_NOT_FOUND: + ret = -ENOENT; + goto error; + case DNS_LOOKUP_BAD: + ret = -EREMOTEIO; + goto error; + case DNS_LOOKUP_GOT_LOCAL_FAILURE: + case DNS_LOOKUP_GOT_TEMP_FAILURE: + case DNS_LOOKUP_GOT_NS_FAILURE: + ret = -EDESTADDRREQ; + goto error; + } + } + + _leave(" = %p [cell]", cell); + return cell; + +cant_alloc_ino: + up_write(&net->cells_lock); + afs_put_cell(candidate, afs_cell_trace_put_candidate); + goto error_noput; + +cell_already_exists: + _debug("cell exists"); + cell = cursor; + if (reason == AFS_LOOKUP_CELL_PRELOAD) { + ret = -EEXIST; + } else { + afs_use_cell(cursor, trace); + ret = 0; + } + up_write(&net->cells_lock); + if (candidate) + afs_put_cell(candidate, afs_cell_trace_put_candidate); + if (ret == 0) + goto wait_for_cell; + goto error_noput; +error: + afs_unuse_cell(cell, afs_cell_trace_unuse_lookup_error); +error_noput: + _leave(" = %d [error]", ret); + return ERR_PTR(ret); +} + +/* + * set the root cell information + * - can be called with a module parameter string + * - can be called from a write to /proc/fs/afs/rootcell + */ +int afs_cell_init(struct afs_net *net, const char *rootcell) +{ + struct afs_cell *old_root, *new_root; + const char *cp, *vllist; + size_t len; + + _enter(""); + + if (!rootcell) { + /* module is loaded with no parameters, or built statically. + * - in the future we might initialize cell DB here. + */ + _leave(" = 0 [no root]"); + return 0; + } + + cp = strchr(rootcell, ':'); + if (!cp) { + _debug("kAFS: no VL server IP addresses specified"); + vllist = NULL; + len = strlen(rootcell); + } else { + vllist = cp + 1; + len = cp - rootcell; + } + + if (len == 0 || !rootcell[0] || rootcell[0] == '.' || rootcell[len - 1] == '.') + return -EINVAL; + if (memchr(rootcell, '/', len)) + return -EINVAL; + cp = strstr(rootcell, ".."); + if (cp && cp < rootcell + len) + return -EINVAL; + + /* allocate a cell record for the root/workstation cell */ + new_root = afs_lookup_cell(net, rootcell, len, vllist, + AFS_LOOKUP_CELL_ROOTCELL, + afs_cell_trace_use_lookup_ws); + if (IS_ERR(new_root)) { + _leave(" = %ld", PTR_ERR(new_root)); + return PTR_ERR(new_root); + } + + if (!test_and_set_bit(AFS_CELL_FL_NO_GC, &new_root->flags)) + afs_use_cell(new_root, afs_cell_trace_use_pin); + + /* install the new cell */ + down_write(&net->cells_lock); + old_root = rcu_replace_pointer(net->ws_cell, new_root, + lockdep_is_held(&net->cells_lock)); + up_write(&net->cells_lock); + + afs_unuse_cell(old_root, afs_cell_trace_unuse_ws); + _leave(" = 0"); + return 0; +} + +/* + * Update a cell's VL server address list from the DNS. + */ +static int afs_update_cell(struct afs_cell *cell) +{ + struct afs_vlserver_list *vllist, *old = NULL, *p; + unsigned int min_ttl = READ_ONCE(afs_cell_min_ttl); + unsigned int max_ttl = READ_ONCE(afs_cell_max_ttl); + time64_t now, expiry = 0; + int ret = 0; + + _enter("%s", cell->name); + + vllist = afs_dns_query(cell, &expiry); + if (IS_ERR(vllist)) { + ret = PTR_ERR(vllist); + + _debug("%s: fail %d", cell->name, ret); + if (ret == -ENOMEM) + goto out_wake; + + vllist = afs_alloc_vlserver_list(0); + if (!vllist) { + if (ret >= 0) + ret = -ENOMEM; + goto out_wake; + } + + switch (ret) { + case -ENODATA: + case -EDESTADDRREQ: + vllist->status = DNS_LOOKUP_GOT_NOT_FOUND; + break; + case -EAGAIN: + case -ECONNREFUSED: + vllist->status = DNS_LOOKUP_GOT_TEMP_FAILURE; + break; + default: + vllist->status = DNS_LOOKUP_GOT_LOCAL_FAILURE; + break; + } + } + + _debug("%s: got list %d %d", cell->name, vllist->source, vllist->status); + cell->dns_status = vllist->status; + + now = ktime_get_real_seconds(); + if (min_ttl > max_ttl) + max_ttl = min_ttl; + if (expiry < now + min_ttl) + expiry = now + min_ttl; + else if (expiry > now + max_ttl) + expiry = now + max_ttl; + + _debug("%s: status %d", cell->name, vllist->status); + if (vllist->source == DNS_RECORD_UNAVAILABLE) { + switch (vllist->status) { + case DNS_LOOKUP_GOT_NOT_FOUND: + /* The DNS said that the cell does not exist or there + * weren't any addresses to be had. + */ + cell->dns_expiry = expiry; + break; + + case DNS_LOOKUP_BAD: + case DNS_LOOKUP_GOT_LOCAL_FAILURE: + case DNS_LOOKUP_GOT_TEMP_FAILURE: + case DNS_LOOKUP_GOT_NS_FAILURE: + default: + cell->dns_expiry = now + 10; + break; + } + } else { + cell->dns_expiry = expiry; + } + + /* Replace the VL server list if the new record has servers or the old + * record doesn't. + */ + write_lock(&cell->vl_servers_lock); + p = rcu_dereference_protected(cell->vl_servers, true); + if (vllist->nr_servers > 0 || p->nr_servers == 0) { + rcu_assign_pointer(cell->vl_servers, vllist); + cell->dns_source = vllist->source; + old = p; + } else { + old = vllist; + } + write_unlock(&cell->vl_servers_lock); + afs_put_vlserverlist(cell->net, old); + +out_wake: + smp_store_release(&cell->dns_lookup_count, + cell->dns_lookup_count + 1); /* vs source/status */ + wake_up_var(&cell->dns_lookup_count); + _leave(" = %d", ret); + return ret; +} + +/* + * Destroy a cell record + */ +static void afs_cell_destroy(struct rcu_head *rcu) +{ + struct afs_cell *cell = container_of(rcu, struct afs_cell, rcu); + struct afs_net *net = cell->net; + int r; + + _enter("%p{%s}", cell, cell->name); + + r = refcount_read(&cell->ref); + ASSERTCMP(r, ==, 0); + trace_afs_cell(cell->debug_id, r, atomic_read(&cell->active), afs_cell_trace_free); + + afs_put_vlserverlist(net, rcu_access_pointer(cell->vl_servers)); + afs_unuse_cell(cell->alias_of, afs_cell_trace_unuse_alias); + key_put(cell->anonymous_key); + kfree(cell->name - 1); + kfree(cell); + + afs_dec_cells_outstanding(net); + _leave(" [destroyed]"); +} + +static void afs_destroy_cell_work(struct work_struct *work) +{ + struct afs_cell *cell = container_of(work, struct afs_cell, destroyer); + + afs_see_cell(cell, afs_cell_trace_destroy); + timer_delete_sync(&cell->management_timer); + cancel_work_sync(&cell->manager); + + if (test_bit(AFS_CELL_FL_HAVE_INO, &cell->flags)) { + down_write(&cell->net->cells_lock); + idr_remove(&cell->net->cells_dyn_ino, cell->dynroot_ino); + up_write(&cell->net->cells_lock); + } + + call_rcu(&cell->rcu, afs_cell_destroy); +} + +/* + * Get a reference on a cell record. + */ +struct afs_cell *afs_get_cell(struct afs_cell *cell, enum afs_cell_trace reason) +{ + int r; + + __refcount_inc(&cell->ref, &r); + trace_afs_cell(cell->debug_id, r + 1, atomic_read(&cell->active), reason); + return cell; +} + +/* + * Drop a reference on a cell record. + */ +void afs_put_cell(struct afs_cell *cell, enum afs_cell_trace reason) +{ + if (cell) { + unsigned int debug_id = cell->debug_id; + unsigned int a; + bool zero; + int r; + + a = atomic_read(&cell->active); + zero = __refcount_dec_and_test(&cell->ref, &r); + trace_afs_cell(debug_id, r - 1, a, reason); + if (zero) { + a = atomic_read(&cell->active); + WARN(a != 0, "Cell active count %u > 0\n", a); + WARN_ON(!queue_work(afs_wq, &cell->destroyer)); + } + } +} + +/* + * Note a cell becoming more active. + */ +struct afs_cell *afs_use_cell(struct afs_cell *cell, enum afs_cell_trace reason) +{ + int r, a; + + __refcount_inc(&cell->ref, &r); + a = atomic_inc_return(&cell->active); + trace_afs_cell(cell->debug_id, r + 1, a, reason); + return cell; +} + +/* + * Record a cell becoming less active. When the active counter reaches 1, it + * is scheduled for destruction, but may get reactivated. + */ +void afs_unuse_cell(struct afs_cell *cell, enum afs_cell_trace reason) +{ + unsigned int debug_id; + time64_t now, expire_delay; + bool zero; + int r, a; + + if (!cell) + return; + + _enter("%s", cell->name); + + now = ktime_get_real_seconds(); + cell->last_inactive = now; + expire_delay = 0; + if (cell->vl_servers->nr_servers) + expire_delay = afs_cell_gc_delay; + + debug_id = cell->debug_id; + a = atomic_dec_return(&cell->active); + if (!a) + /* 'cell' may now be garbage collected. */ + afs_set_cell_timer(cell, expire_delay); + + zero = __refcount_dec_and_test(&cell->ref, &r); + trace_afs_cell(debug_id, r - 1, a, reason); + if (zero) + WARN_ON(!queue_work(afs_wq, &cell->destroyer)); +} + +/* + * Note that a cell has been seen. + */ +void afs_see_cell(struct afs_cell *cell, enum afs_cell_trace reason) +{ + int r, a; + + r = refcount_read(&cell->ref); + a = atomic_read(&cell->active); + trace_afs_cell(cell->debug_id, r, a, reason); +} + +/* + * Queue a cell for management, giving the workqueue a ref to hold. + */ +void afs_queue_cell(struct afs_cell *cell, enum afs_cell_trace reason) +{ + queue_work(afs_wq, &cell->manager); +} + +/* + * Cell-specific management timer. + */ +static void afs_cell_timer(struct timer_list *timer) +{ + struct afs_cell *cell = container_of(timer, struct afs_cell, management_timer); + + afs_see_cell(cell, afs_cell_trace_see_mgmt_timer); + if (refcount_read(&cell->ref) > 0 && cell->net->live) + queue_work(afs_wq, &cell->manager); +} + +/* + * Set/reduce the cell timer. + */ +void afs_set_cell_timer(struct afs_cell *cell, unsigned int delay_secs) +{ + timer_reduce(&cell->management_timer, jiffies + delay_secs * HZ); +} + +/* + * Activate a cell. + */ +static int afs_activate_cell(struct afs_net *net, struct afs_cell *cell) +{ + struct hlist_node **p; + struct afs_cell *pcell; + int ret; + + ret = afs_proc_cell_setup(cell); + if (ret < 0) + return ret; + + mutex_lock(&net->proc_cells_lock); + for (p = &net->proc_cells.first; *p; p = &(*p)->next) { + pcell = hlist_entry(*p, struct afs_cell, proc_link); + if (strcmp(cell->name, pcell->name) < 0) + break; + } + + cell->proc_link.pprev = p; + cell->proc_link.next = *p; + rcu_assign_pointer(*p, &cell->proc_link.next); + if (cell->proc_link.next) + cell->proc_link.next->pprev = &cell->proc_link.next; + + mutex_unlock(&net->proc_cells_lock); + return 0; +} + +/* + * Deactivate a cell. + */ +static void afs_deactivate_cell(struct afs_net *net, struct afs_cell *cell) +{ + _enter("%s", cell->name); + + afs_proc_cell_remove(cell); + + mutex_lock(&net->proc_cells_lock); + if (!hlist_unhashed(&cell->proc_link)) + hlist_del_rcu(&cell->proc_link); + mutex_unlock(&net->proc_cells_lock); + + _leave(""); +} + +static bool afs_has_cell_expired(struct afs_cell *cell, time64_t *_next_manage) +{ + const struct afs_vlserver_list *vllist; + time64_t expire_at = cell->last_inactive; + time64_t now = ktime_get_real_seconds(); + + if (atomic_read(&cell->active)) + return false; + if (!cell->net->live) + return true; + + vllist = rcu_dereference_protected(cell->vl_servers, true); + if (vllist && vllist->nr_servers > 0) + expire_at += afs_cell_gc_delay; + + if (expire_at <= now) + return true; + if (expire_at < *_next_manage) + *_next_manage = expire_at; + return false; +} + +/* + * Manage a cell record, initialising and destroying it, maintaining its DNS + * records. + */ +static bool afs_manage_cell(struct afs_cell *cell) +{ + struct afs_net *net = cell->net; + time64_t next_manage = TIME64_MAX; + int ret; + + _enter("%s", cell->name); + + _debug("state %u", cell->state); + switch (cell->state) { + case AFS_CELL_SETTING_UP: + goto set_up_cell; + case AFS_CELL_UNLOOKED: + case AFS_CELL_ACTIVE: + goto cell_is_active; + case AFS_CELL_REMOVING: + WARN_ON_ONCE(1); + return false; + case AFS_CELL_DEAD: + return false; + default: + _debug("bad state %u", cell->state); + WARN_ON_ONCE(1); /* Unhandled state */ + return false; + } + +set_up_cell: + ret = afs_activate_cell(net, cell); + if (ret < 0) { + cell->error = ret; + goto remove_cell; + } + + afs_set_cell_state(cell, AFS_CELL_UNLOOKED); + +cell_is_active: + if (afs_has_cell_expired(cell, &next_manage)) + goto remove_cell; + + if (test_and_clear_bit(AFS_CELL_FL_DO_LOOKUP, &cell->flags)) { + ret = afs_update_cell(cell); + if (ret < 0) + cell->error = ret; + if (cell->state == AFS_CELL_UNLOOKED) + afs_set_cell_state(cell, AFS_CELL_ACTIVE); + } + + if (next_manage < TIME64_MAX && cell->net->live) { + time64_t now = ktime_get_real_seconds(); + + if (next_manage - now <= 0) + afs_queue_cell(cell, afs_cell_trace_queue_again); + else + afs_set_cell_timer(cell, next_manage - now); + } + _leave(" [done %u]", cell->state); + return false; + +remove_cell: + down_write(&net->cells_lock); + + if (atomic_read(&cell->active)) { + up_write(&net->cells_lock); + goto cell_is_active; + } + + /* Make sure that the expiring server records are going to see the fact + * that the cell is caput. + */ + afs_set_cell_state(cell, AFS_CELL_REMOVING); + + afs_deactivate_cell(net, cell); + afs_purge_servers(cell); + + rb_erase(&cell->net_node, &net->cells); + afs_see_cell(cell, afs_cell_trace_unuse_delete); + up_write(&net->cells_lock); + + /* The root volume is pinning the cell */ + afs_put_volume(cell->root_volume, afs_volume_trace_put_cell_root); + cell->root_volume = NULL; + + afs_set_cell_state(cell, AFS_CELL_DEAD); + return true; +} + +static void afs_manage_cell_work(struct work_struct *work) +{ + struct afs_cell *cell = container_of(work, struct afs_cell, manager); + bool final_put; + + afs_see_cell(cell, afs_cell_trace_manage); + final_put = afs_manage_cell(cell); + afs_see_cell(cell, afs_cell_trace_managed); + if (final_put) + afs_put_cell(cell, afs_cell_trace_put_final); +} + +/* + * Purge in-memory cell database. + */ +void afs_cell_purge(struct afs_net *net) +{ + struct afs_cell *ws; + struct rb_node *cursor; + + _enter(""); + + down_write(&net->cells_lock); + ws = rcu_replace_pointer(net->ws_cell, NULL, + lockdep_is_held(&net->cells_lock)); + up_write(&net->cells_lock); + afs_unuse_cell(ws, afs_cell_trace_unuse_ws); + + _debug("kick cells"); + down_read(&net->cells_lock); + for (cursor = rb_first(&net->cells); cursor; cursor = rb_next(cursor)) { + struct afs_cell *cell = rb_entry(cursor, struct afs_cell, net_node); + + afs_see_cell(cell, afs_cell_trace_purge); + + if (test_and_clear_bit(AFS_CELL_FL_NO_GC, &cell->flags)) + afs_unuse_cell(cell, afs_cell_trace_unuse_pin); + + afs_queue_cell(cell, afs_cell_trace_queue_purge); + } + up_read(&net->cells_lock); + + _debug("wait"); + wait_var_event(&net->cells_outstanding, + !atomic_read(&net->cells_outstanding)); + _leave(""); +} diff --git a/fs/afs/cm_security.c b/fs/afs/cm_security.c new file mode 100644 index 000000000..103168c70 --- /dev/null +++ b/fs/afs/cm_security.c @@ -0,0 +1,341 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Cache manager security. + * + * Copyright (C) 2025 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include "internal.h" +#include "afs_cm.h" +#include "afs_fs.h" +#include "protocol_yfs.h" +#define RXRPC_TRACE_ONLY_DEFINE_ENUMS +#include + +#define RXGK_SERVER_ENC_TOKEN 1036U // 0x40c +#define xdr_round_up(x) (round_up((x), sizeof(__be32))) +#define xdr_len_object(x) (4 + round_up((x), sizeof(__be32))) + +#ifdef CONFIG_RXGK +static int afs_create_yfs_cm_token(struct sk_buff *challenge, + struct afs_server *server); +#endif + +/* + * Respond to an RxGK challenge, adding appdata. + */ +static int afs_respond_to_challenge(struct sk_buff *challenge) +{ +#ifdef CONFIG_RXGK + struct krb5_buffer appdata = {}; + struct afs_server *server; +#endif + struct rxrpc_peer *peer; + unsigned long peer_data; + u16 service_id; + u8 security_index; + + rxrpc_kernel_query_challenge(challenge, &peer, &peer_data, + &service_id, &security_index); + + _enter("%u,%u", service_id, security_index); + + switch (service_id) { + /* We don't send CM_SERVICE RPCs, so don't expect a challenge + * therefrom. + */ + case FS_SERVICE: + case VL_SERVICE: + case YFS_FS_SERVICE: + case YFS_VL_SERVICE: + break; + default: + pr_warn("Can't respond to unknown challenge %u:%u", + service_id, security_index); + return rxrpc_kernel_reject_challenge(challenge, RX_USER_ABORT, -EPROTO, + afs_abort_unsupported_sec_class); + } + + switch (security_index) { +#ifdef CONFIG_RXKAD + case RXRPC_SECURITY_RXKAD: + return rxkad_kernel_respond_to_challenge(challenge); +#endif + +#ifdef CONFIG_RXGK + case RXRPC_SECURITY_RXGK: + return rxgk_kernel_respond_to_challenge(challenge, &appdata); + + case RXRPC_SECURITY_YFS_RXGK: + switch (service_id) { + case FS_SERVICE: + case YFS_FS_SERVICE: + server = (struct afs_server *)peer_data; + if (!server->cm_rxgk_appdata.data) { + mutex_lock(&server->cm_token_lock); + if (!server->cm_rxgk_appdata.data) + afs_create_yfs_cm_token(challenge, server); + mutex_unlock(&server->cm_token_lock); + } + if (server->cm_rxgk_appdata.data) + appdata = server->cm_rxgk_appdata; + break; + } + return rxgk_kernel_respond_to_challenge(challenge, &appdata); +#endif + + default: + return rxrpc_kernel_reject_challenge(challenge, RX_USER_ABORT, -EPROTO, + afs_abort_unsupported_sec_class); + } +} + +/* + * Process the OOB message queue, processing challenge packets. + */ +void afs_process_oob_queue(struct work_struct *work) +{ + struct afs_net *net = container_of(work, struct afs_net, rx_oob_work); + struct sk_buff *oob; + enum rxrpc_oob_type type; + + while (READ_ONCE(net->live) && + (oob = rxrpc_kernel_dequeue_oob(net->socket, &type))) { + switch (type) { + case RXRPC_OOB_CHALLENGE: + afs_respond_to_challenge(oob); + break; + } + rxrpc_kernel_free_oob(oob); + } +} + +#ifdef CONFIG_RXGK +/* + * Create a securities keyring for the cache manager and attach a key to it for + * the RxGK tokens we want to use to secure the callback connection back from + * the fileserver. + */ +int afs_create_token_key(struct afs_net *net, struct socket *socket) +{ + const struct krb5_enctype *krb5; + struct key *ring; + key_ref_t key; + char K0[32], *desc; + int ret; + + ring = keyring_alloc("kafs", + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, current_cred(), + KEY_POS_SEARCH | KEY_POS_WRITE | + KEY_USR_VIEW | KEY_USR_READ | KEY_USR_SEARCH, + KEY_ALLOC_NOT_IN_QUOTA, + NULL, NULL); + if (IS_ERR(ring)) + return PTR_ERR(ring); + + ret = rxrpc_sock_set_security_keyring(socket->sk, ring); + if (ret < 0) + goto out; + + ret = -ENOPKG; + krb5 = crypto_krb5_find_enctype(KRB5_ENCTYPE_AES128_CTS_HMAC_SHA1_96); + if (!krb5) + goto out; + + if (WARN_ON_ONCE(krb5->key_len > sizeof(K0))) + goto out; + + ret = -ENOMEM; + desc = kasprintf(GFP_KERNEL, "%u:%u:%u:%u", + YFS_CM_SERVICE, RXRPC_SECURITY_YFS_RXGK, 1, krb5->etype); + if (!desc) + goto out; + + wait_for_random_bytes(); + get_random_bytes(K0, krb5->key_len); + + key = key_create(make_key_ref(ring, true), + "rxrpc_s", desc, + K0, krb5->key_len, + KEY_POS_VIEW | KEY_POS_READ | KEY_POS_SEARCH | KEY_USR_VIEW, + KEY_ALLOC_NOT_IN_QUOTA); + kfree(desc); + if (IS_ERR(key)) { + ret = PTR_ERR(key); + goto out; + } + + net->fs_cm_token_key = key_ref_to_ptr(key); + ret = 0; +out: + key_put(ring); + return ret; +} + +/* + * Create an YFS RxGK GSS token to use as a ticket to the specified fileserver. + */ +static int afs_create_yfs_cm_token(struct sk_buff *challenge, + struct afs_server *server) +{ + const struct krb5_enctype *conn_krb5, *token_krb5; + const struct krb5_buffer *token_key; + struct crypto_aead *aead; + struct scatterlist sg; + struct afs_net *net = server->cell->net; + const struct key *key = net->fs_cm_token_key; + size_t keysize, uuidsize, authsize, toksize, encsize, contsize, adatasize, offset; + __be32 caps[1] = { + [0] = htonl(AFS_CAP_ERROR_TRANSLATION), + }; + __be32 *xdr; + void *appdata, *K0, *encbase; + u32 enctype; + int ret; + + if (!key) + return -ENOKEY; + + /* Assume that the fileserver is happy to use the same encoding type as + * we were told to use by the token obtained by the user. + */ + enctype = rxgk_kernel_query_challenge(challenge); + + conn_krb5 = crypto_krb5_find_enctype(enctype); + if (!conn_krb5) + return -ENOPKG; + token_krb5 = key->payload.data[0]; + token_key = (const struct krb5_buffer *)&key->payload.data[2]; + + /* struct rxgk_key { + * afs_uint32 enctype; + * opaque key<>; + * }; + */ + keysize = 4 + xdr_len_object(conn_krb5->key_len); + + /* struct RXGK_AuthName { + * afs_int32 kind; + * opaque data; + * opaque display; + * }; + */ + uuidsize = sizeof(server->uuid); + authsize = 4 + xdr_len_object(uuidsize) + xdr_len_object(0); + + /* struct RXGK_Token { + * rxgk_key K0; + * RXGK_Level level; + * rxgkTime starttime; + * afs_int32 lifetime; + * afs_int32 bytelife; + * rxgkTime expirationtime; + * struct RXGK_AuthName identities<>; + * }; + */ + toksize = keysize + 8 + 4 + 4 + 8 + xdr_len_object(authsize); + + offset = 0; + encsize = crypto_krb5_how_much_buffer(token_krb5, KRB5_ENCRYPT_MODE, toksize, &offset); + + /* struct RXGK_TokenContainer { + * afs_int32 kvno; + * afs_int32 enctype; + * opaque encrypted_token<>; + * }; + */ + contsize = 4 + 4 + xdr_len_object(encsize); + + /* struct YFSAppData { + * opr_uuid initiatorUuid; + * opr_uuid acceptorUuid; + * Capabilities caps; + * afs_int32 enctype; + * opaque callbackKey<>; + * opaque callbackToken<>; + * }; + */ + adatasize = 16 + 16 + + xdr_len_object(sizeof(caps)) + + 4 + + xdr_len_object(conn_krb5->key_len) + + xdr_len_object(contsize); + + ret = -ENOMEM; + appdata = kzalloc(adatasize, GFP_KERNEL); + if (!appdata) + goto out; + xdr = appdata; + + memcpy(xdr, &net->uuid, 16); /* appdata.initiatorUuid */ + xdr += 16 / 4; + memcpy(xdr, &server->uuid, 16); /* appdata.acceptorUuid */ + xdr += 16 / 4; + *xdr++ = htonl(ARRAY_SIZE(caps)); /* appdata.caps.len */ + memcpy(xdr, &caps, sizeof(caps)); /* appdata.caps */ + xdr += ARRAY_SIZE(caps); + *xdr++ = htonl(conn_krb5->etype); /* appdata.enctype */ + + *xdr++ = htonl(conn_krb5->key_len); /* appdata.callbackKey.len */ + K0 = xdr; + get_random_bytes(K0, conn_krb5->key_len); /* appdata.callbackKey.data */ + xdr += xdr_round_up(conn_krb5->key_len) / 4; + + *xdr++ = htonl(contsize); /* appdata.callbackToken.len */ + *xdr++ = htonl(1); /* cont.kvno */ + *xdr++ = htonl(token_krb5->etype); /* cont.enctype */ + *xdr++ = htonl(encsize); /* cont.encrypted_token.len */ + + encbase = xdr; + xdr += offset / 4; + *xdr++ = htonl(conn_krb5->etype); /* token.K0.enctype */ + *xdr++ = htonl(conn_krb5->key_len); /* token.K0.key.len */ + memcpy(xdr, K0, conn_krb5->key_len); /* token.K0.key.data */ + xdr += xdr_round_up(conn_krb5->key_len) / 4; + + *xdr++ = htonl(RXRPC_SECURITY_ENCRYPT); /* token.level */ + *xdr++ = htonl(0); /* token.starttime */ + *xdr++ = htonl(0); /* " */ + *xdr++ = htonl(0); /* token.lifetime */ + *xdr++ = htonl(0); /* token.bytelife */ + *xdr++ = htonl(0); /* token.expirationtime */ + *xdr++ = htonl(0); /* " */ + *xdr++ = htonl(1); /* token.identities.count */ + *xdr++ = htonl(0); /* token.identities[0].kind */ + *xdr++ = htonl(uuidsize); /* token.identities[0].data.len */ + memcpy(xdr, &server->uuid, uuidsize); + xdr += xdr_round_up(uuidsize) / 4; + *xdr++ = htonl(0); /* token.identities[0].display.len */ + + xdr = encbase + xdr_round_up(encsize); + + if ((unsigned long)xdr - (unsigned long)appdata != adatasize) + pr_err("Appdata size incorrect %lx != %zx\n", + (unsigned long)xdr - (unsigned long)appdata, adatasize); + + aead = crypto_krb5_prepare_encryption(token_krb5, token_key, RXGK_SERVER_ENC_TOKEN, + GFP_KERNEL); + if (IS_ERR(aead)) { + ret = PTR_ERR(aead); + goto out_token; + } + + sg_init_one(&sg, encbase, encsize); + ret = crypto_krb5_encrypt(token_krb5, aead, &sg, 1, encsize, offset, toksize, false); + if (ret < 0) + goto out_aead; + + server->cm_rxgk_appdata.len = adatasize; + server->cm_rxgk_appdata.data = appdata; + appdata = NULL; + +out_aead: + crypto_free_aead(aead); +out_token: + kfree(appdata); +out: + return ret; +} +#endif /* CONFIG_RXGK */ diff --git a/fs/afs/cmservice.c b/fs/afs/cmservice.c new file mode 100644 index 000000000..db394f101 --- /dev/null +++ b/fs/afs/cmservice.c @@ -0,0 +1,620 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* AFS Cache Manager Service + * + * Copyright (C) 2002 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include "internal.h" +#include "afs_cm.h" +#include "protocol_yfs.h" +#define RXRPC_TRACE_ONLY_DEFINE_ENUMS +#include + +static int afs_deliver_cb_init_call_back_state(struct afs_call *); +static int afs_deliver_cb_init_call_back_state3(struct afs_call *); +static int afs_deliver_cb_probe(struct afs_call *); +static int afs_deliver_cb_callback(struct afs_call *); +static int afs_deliver_cb_probe_uuid(struct afs_call *); +static int afs_deliver_cb_tell_me_about_yourself(struct afs_call *); +static void afs_cm_destructor(struct afs_call *); +static void SRXAFSCB_CallBack(struct work_struct *); +static void SRXAFSCB_InitCallBackState(struct work_struct *); +static void SRXAFSCB_Probe(struct work_struct *); +static void SRXAFSCB_ProbeUuid(struct work_struct *); +static void SRXAFSCB_TellMeAboutYourself(struct work_struct *); + +static int afs_deliver_yfs_cb_callback(struct afs_call *); + +/* + * CB.CallBack operation type + */ +static const struct afs_call_type afs_SRXCBCallBack = { + .name = "CB.CallBack", + .deliver = afs_deliver_cb_callback, + .destructor = afs_cm_destructor, + .work = SRXAFSCB_CallBack, +}; + +/* + * CB.InitCallBackState operation type + */ +static const struct afs_call_type afs_SRXCBInitCallBackState = { + .name = "CB.InitCallBackState", + .deliver = afs_deliver_cb_init_call_back_state, + .destructor = afs_cm_destructor, + .work = SRXAFSCB_InitCallBackState, +}; + +/* + * CB.InitCallBackState3 operation type + */ +static const struct afs_call_type afs_SRXCBInitCallBackState3 = { + .name = "CB.InitCallBackState3", + .deliver = afs_deliver_cb_init_call_back_state3, + .destructor = afs_cm_destructor, + .work = SRXAFSCB_InitCallBackState, +}; + +/* + * CB.Probe operation type + */ +static const struct afs_call_type afs_SRXCBProbe = { + .name = "CB.Probe", + .deliver = afs_deliver_cb_probe, + .destructor = afs_cm_destructor, + .work = SRXAFSCB_Probe, +}; + +/* + * CB.ProbeUuid operation type + */ +static const struct afs_call_type afs_SRXCBProbeUuid = { + .name = "CB.ProbeUuid", + .deliver = afs_deliver_cb_probe_uuid, + .destructor = afs_cm_destructor, + .work = SRXAFSCB_ProbeUuid, +}; + +/* + * CB.TellMeAboutYourself operation type + */ +static const struct afs_call_type afs_SRXCBTellMeAboutYourself = { + .name = "CB.TellMeAboutYourself", + .deliver = afs_deliver_cb_tell_me_about_yourself, + .destructor = afs_cm_destructor, + .work = SRXAFSCB_TellMeAboutYourself, +}; + +/* + * YFS CB.CallBack operation type + */ +static const struct afs_call_type afs_SRXYFSCB_CallBack = { + .name = "YFSCB.CallBack", + .deliver = afs_deliver_yfs_cb_callback, + .destructor = afs_cm_destructor, + .work = SRXAFSCB_CallBack, +}; + +/* + * route an incoming cache manager call + * - return T if supported, F if not + */ +bool afs_cm_incoming_call(struct afs_call *call) +{ + _enter("{%u, CB.OP %u}", call->service_id, call->operation_ID); + + switch (call->operation_ID) { + case CBCallBack: + call->type = &afs_SRXCBCallBack; + return true; + case CBInitCallBackState: + call->type = &afs_SRXCBInitCallBackState; + return true; + case CBInitCallBackState3: + call->type = &afs_SRXCBInitCallBackState3; + return true; + case CBProbe: + call->type = &afs_SRXCBProbe; + return true; + case CBProbeUuid: + call->type = &afs_SRXCBProbeUuid; + return true; + case CBTellMeAboutYourself: + call->type = &afs_SRXCBTellMeAboutYourself; + return true; + case YFSCBCallBack: + if (call->service_id != YFS_CM_SERVICE) + return false; + call->type = &afs_SRXYFSCB_CallBack; + return true; + default: + return false; + } +} + +/* + * Clean up a cache manager call. + */ +static void afs_cm_destructor(struct afs_call *call) +{ + kfree(call->buffer); + call->buffer = NULL; +} + +/* + * Abort a service call from within an action function. + */ +static void afs_abort_service_call(struct afs_call *call, u32 abort_code, int error, + enum rxrpc_abort_reason why) +{ + rxrpc_kernel_abort_call(call->net->socket, call->rxcall, + abort_code, error, why); + afs_set_call_complete(call, error, 0); +} + +/* + * The server supplied a list of callbacks that it wanted to break. + */ +static void SRXAFSCB_CallBack(struct work_struct *work) +{ + struct afs_call *call = container_of(work, struct afs_call, work); + + _enter(""); + + /* We need to break the callbacks before sending the reply as the + * server holds up change visibility till it receives our reply so as + * to maintain cache coherency. + */ + if (call->server) { + trace_afs_server(call->server->debug_id, + refcount_read(&call->server->ref), + atomic_read(&call->server->active), + afs_server_trace_callback); + afs_break_callbacks(call->server, call->count, call->request); + } + + afs_send_empty_reply(call); + afs_put_call(call); + _leave(""); +} + +/* + * deliver request data to a CB.CallBack call + */ +static int afs_deliver_cb_callback(struct afs_call *call) +{ + struct afs_callback_break *cb; + __be32 *bp; + int ret, loop; + + _enter("{%u}", call->unmarshall); + + switch (call->unmarshall) { + case 0: + afs_extract_to_tmp(call); + call->unmarshall++; + + /* extract the FID array and its count in two steps */ + fallthrough; + case 1: + _debug("extract FID count"); + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + call->count = ntohl(call->tmp); + _debug("FID count: %u", call->count); + if (call->count > AFSCBMAX) + return afs_protocol_error(call, afs_eproto_cb_fid_count); + + call->buffer = kmalloc(array3_size(call->count, 3, 4), + GFP_KERNEL); + if (!call->buffer) + return -ENOMEM; + afs_extract_to_buf(call, call->count * 3 * 4); + call->unmarshall++; + + fallthrough; + case 2: + _debug("extract FID array"); + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + _debug("unmarshall FID array"); + call->request = kzalloc_objs(struct afs_callback_break, + call->count); + if (!call->request) + return -ENOMEM; + + cb = call->request; + bp = call->buffer; + for (loop = call->count; loop > 0; loop--, cb++) { + cb->fid.vid = ntohl(*bp++); + cb->fid.vnode = ntohl(*bp++); + cb->fid.unique = ntohl(*bp++); + } + + afs_extract_to_tmp(call); + call->unmarshall++; + + /* extract the callback array and its count in two steps */ + fallthrough; + case 3: + _debug("extract CB count"); + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + call->count2 = ntohl(call->tmp); + _debug("CB count: %u", call->count2); + if (call->count2 != call->count && call->count2 != 0) + return afs_protocol_error(call, afs_eproto_cb_count); + call->iter = &call->def_iter; + iov_iter_discard(&call->def_iter, ITER_DEST, call->count2 * 3 * 4); + call->unmarshall++; + + fallthrough; + case 4: + _debug("extract discard %zu/%u", + iov_iter_count(call->iter), call->count2 * 3 * 4); + + ret = afs_extract_data(call, false); + if (ret < 0) + return ret; + + call->unmarshall++; + fallthrough; + + case 5: + break; + } + + if (!afs_check_call_state(call, AFS_CALL_SV_REPLYING)) + return afs_io_error(call, afs_io_error_cm_reply); + return 0; +} + +/* + * allow the fileserver to request callback state (re-)initialisation + */ +static void SRXAFSCB_InitCallBackState(struct work_struct *work) +{ + struct afs_call *call = container_of(work, struct afs_call, work); + + _enter("{%p}", call->server); + + if (call->server) + afs_init_callback_state(call->server); + afs_send_empty_reply(call); + afs_put_call(call); + _leave(""); +} + +/* + * deliver request data to a CB.InitCallBackState call + */ +static int afs_deliver_cb_init_call_back_state(struct afs_call *call) +{ + _enter(""); + + afs_extract_discard(call, 0); + return afs_extract_data(call, false); +} + +/* + * deliver request data to a CB.InitCallBackState3 call + */ +static int afs_deliver_cb_init_call_back_state3(struct afs_call *call) +{ + struct afs_uuid *r; + unsigned loop; + __be32 *b; + int ret; + + _enter("{%u}", call->unmarshall); + + switch (call->unmarshall) { + case 0: + call->buffer = kmalloc_array(11, sizeof(__be32), GFP_KERNEL); + if (!call->buffer) + return -ENOMEM; + afs_extract_to_buf(call, 11 * sizeof(__be32)); + call->unmarshall++; + + fallthrough; + case 1: + _debug("extract UUID"); + ret = afs_extract_data(call, false); + switch (ret) { + case 0: break; + default: return ret; + } + + _debug("unmarshall UUID"); + call->request = kmalloc_obj(struct afs_uuid); + if (!call->request) + return -ENOMEM; + + b = call->buffer; + r = call->request; + r->time_low = b[0]; + r->time_mid = htons(ntohl(b[1])); + r->time_hi_and_version = htons(ntohl(b[2])); + r->clock_seq_hi_and_reserved = ntohl(b[3]); + r->clock_seq_low = ntohl(b[4]); + + for (loop = 0; loop < 6; loop++) + r->node[loop] = ntohl(b[loop + 5]); + + call->unmarshall++; + fallthrough; + + case 2: + break; + } + + if (!afs_check_call_state(call, AFS_CALL_SV_REPLYING)) + return afs_io_error(call, afs_io_error_cm_reply); + + if (!call->server) { + trace_afs_cm_no_server_u(call, call->request); + return 0; + } + + if (memcmp(call->request, &call->server->_uuid, sizeof(call->server->_uuid)) != 0) { + pr_notice("Callback UUID does not match fileserver UUID\n"); + trace_afs_cm_no_server_u(call, call->request); + return 0; + } + + return 0; +} + +/* + * allow the fileserver to see if the cache manager is still alive + */ +static void SRXAFSCB_Probe(struct work_struct *work) +{ + struct afs_call *call = container_of(work, struct afs_call, work); + + _enter(""); + afs_send_empty_reply(call); + afs_put_call(call); + _leave(""); +} + +/* + * deliver request data to a CB.Probe call + */ +static int afs_deliver_cb_probe(struct afs_call *call) +{ + int ret; + + _enter(""); + + afs_extract_discard(call, 0); + ret = afs_extract_data(call, false); + if (ret < 0) + return ret; + + if (!afs_check_call_state(call, AFS_CALL_SV_REPLYING)) + return afs_io_error(call, afs_io_error_cm_reply); + return 0; +} + +/* + * Allow the fileserver to quickly find out if the cache manager has been + * rebooted. + */ +static void SRXAFSCB_ProbeUuid(struct work_struct *work) +{ + struct afs_call *call = container_of(work, struct afs_call, work); + struct afs_uuid *r = call->request; + + _enter(""); + + if (memcmp(r, &call->net->uuid, sizeof(call->net->uuid)) == 0) + afs_send_empty_reply(call); + else + afs_abort_service_call(call, 1, 1, afs_abort_probeuuid_negative); + + afs_put_call(call); + _leave(""); +} + +/* + * deliver request data to a CB.ProbeUuid call + */ +static int afs_deliver_cb_probe_uuid(struct afs_call *call) +{ + struct afs_uuid *r; + unsigned loop; + __be32 *b; + int ret; + + _enter("{%u}", call->unmarshall); + + switch (call->unmarshall) { + case 0: + call->buffer = kmalloc_array(11, sizeof(__be32), GFP_KERNEL); + if (!call->buffer) + return -ENOMEM; + afs_extract_to_buf(call, 11 * sizeof(__be32)); + call->unmarshall++; + + fallthrough; + case 1: + _debug("extract UUID"); + ret = afs_extract_data(call, false); + switch (ret) { + case 0: break; + default: return ret; + } + + _debug("unmarshall UUID"); + call->request = kmalloc_obj(struct afs_uuid); + if (!call->request) + return -ENOMEM; + + b = call->buffer; + r = call->request; + r->time_low = b[0]; + r->time_mid = htons(ntohl(b[1])); + r->time_hi_and_version = htons(ntohl(b[2])); + r->clock_seq_hi_and_reserved = ntohl(b[3]); + r->clock_seq_low = ntohl(b[4]); + + for (loop = 0; loop < 6; loop++) + r->node[loop] = ntohl(b[loop + 5]); + + call->unmarshall++; + fallthrough; + + case 2: + break; + } + + if (!afs_check_call_state(call, AFS_CALL_SV_REPLYING)) + return afs_io_error(call, afs_io_error_cm_reply); + return 0; +} + +/* + * allow the fileserver to ask about the cache manager's capabilities + */ +static void SRXAFSCB_TellMeAboutYourself(struct work_struct *work) +{ + struct afs_call *call = container_of(work, struct afs_call, work); + int loop; + + struct { + struct /* InterfaceAddr */ { + __be32 nifs; + __be32 uuid[11]; + __be32 ifaddr[32]; + __be32 netmask[32]; + __be32 mtu[32]; + } ia; + struct /* Capabilities */ { + __be32 capcount; + __be32 caps[1]; + } cap; + } reply; + + _enter(""); + + memset(&reply, 0, sizeof(reply)); + + reply.ia.uuid[0] = call->net->uuid.time_low; + reply.ia.uuid[1] = htonl(ntohs(call->net->uuid.time_mid)); + reply.ia.uuid[2] = htonl(ntohs(call->net->uuid.time_hi_and_version)); + reply.ia.uuid[3] = htonl((s8) call->net->uuid.clock_seq_hi_and_reserved); + reply.ia.uuid[4] = htonl((s8) call->net->uuid.clock_seq_low); + for (loop = 0; loop < 6; loop++) + reply.ia.uuid[loop + 5] = htonl((s8) call->net->uuid.node[loop]); + + reply.cap.capcount = htonl(1); + reply.cap.caps[0] = htonl(AFS_CAP_ERROR_TRANSLATION); + afs_send_simple_reply(call, &reply, sizeof(reply)); + afs_put_call(call); + _leave(""); +} + +/* + * deliver request data to a CB.TellMeAboutYourself call + */ +static int afs_deliver_cb_tell_me_about_yourself(struct afs_call *call) +{ + int ret; + + _enter(""); + + afs_extract_discard(call, 0); + ret = afs_extract_data(call, false); + if (ret < 0) + return ret; + + if (!afs_check_call_state(call, AFS_CALL_SV_REPLYING)) + return afs_io_error(call, afs_io_error_cm_reply); + return 0; +} + +/* + * deliver request data to a YFS CB.CallBack call + */ +static int afs_deliver_yfs_cb_callback(struct afs_call *call) +{ + struct afs_callback_break *cb; + struct yfs_xdr_YFSFid *bp; + size_t size; + int ret, loop; + + _enter("{%u}", call->unmarshall); + + switch (call->unmarshall) { + case 0: + afs_extract_to_tmp(call); + call->unmarshall++; + + /* extract the FID array and its count in two steps */ + fallthrough; + case 1: + _debug("extract FID count"); + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + call->count = ntohl(call->tmp); + _debug("FID count: %u", call->count); + if (call->count > YFSCBMAX) + return afs_protocol_error(call, afs_eproto_cb_fid_count); + + size = array_size(call->count, sizeof(struct yfs_xdr_YFSFid)); + call->buffer = kmalloc(size, GFP_KERNEL); + if (!call->buffer) + return -ENOMEM; + afs_extract_to_buf(call, size); + call->unmarshall++; + + fallthrough; + case 2: + _debug("extract FID array"); + ret = afs_extract_data(call, false); + if (ret < 0) + return ret; + + _debug("unmarshall FID array"); + call->request = kzalloc_objs(struct afs_callback_break, + call->count); + if (!call->request) + return -ENOMEM; + + cb = call->request; + bp = call->buffer; + for (loop = call->count; loop > 0; loop--, cb++) { + cb->fid.vid = xdr_to_u64(bp->volume); + cb->fid.vnode = xdr_to_u64(bp->vnode.lo); + cb->fid.vnode_hi = ntohl(bp->vnode.hi); + cb->fid.unique = ntohl(bp->vnode.unique); + bp++; + } + + afs_extract_to_tmp(call); + call->unmarshall++; + fallthrough; + + case 3: + break; + } + + if (!afs_check_call_state(call, AFS_CALL_SV_REPLYING)) + return afs_io_error(call, afs_io_error_cm_reply); + return 0; +} diff --git a/fs/afs/dir.c b/fs/afs/dir.c new file mode 100644 index 000000000..81565366d --- /dev/null +++ b/fs/afs/dir.c @@ -0,0 +1,2240 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* dir.c: AFS filesystem directory handling + * + * Copyright (C) 2002, 2018 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" +#include "afs_fs.h" +#include "xdr_fs.h" + +static struct dentry *afs_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags); +static int afs_dir_open(struct inode *inode, struct file *file); +static int afs_readdir(struct file *file, struct dir_context *ctx); +static int afs_d_revalidate(struct inode *dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags); +static int afs_d_delete(const struct dentry *dentry); +static void afs_d_iput(struct dentry *dentry, struct inode *inode); +static bool afs_lookup_one_filldir(struct dir_context *ctx, const char *name, int nlen, + u64 ino, u32 uniquifier); +#define AFS_LOOKUP_ONE ((filldir_t)0x123UL) +static bool afs_lookup_filldir(struct dir_context *ctx, const char *name, int nlen, + u64 ino, u32 uniquifier); +#define AFS_LOOKUP ((filldir_t)0x137UL) +static int afs_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode); +static struct dentry *afs_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode); +static int afs_rmdir(struct inode *dir, struct dentry *dentry); +static int afs_unlink(struct inode *dir, struct dentry *dentry); +static int afs_link(struct dentry *from, struct inode *dir, + struct dentry *dentry); +static int afs_symlink(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const char *content); +static int afs_rename(struct mnt_idmap *idmap, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags); +static int afs_dir_writepages(struct address_space *mapping, + struct writeback_control *wbc); + +const struct file_operations afs_dir_file_operations = { + .open = afs_dir_open, + .release = afs_release, + .iterate_shared = afs_readdir, + .lock = afs_lock, + .llseek = generic_file_llseek, +}; + +const struct inode_operations afs_dir_inode_operations = { + .create = afs_create, + .lookup = afs_lookup, + .link = afs_link, + .unlink = afs_unlink, + .symlink = afs_symlink, + .mkdir = afs_mkdir, + .rmdir = afs_rmdir, + .rename = afs_rename, + .permission = afs_permission, + .getattr = afs_getattr, + .setattr = afs_setattr, +}; + +const struct address_space_operations afs_dir_aops = { + .writepages = afs_dir_writepages, +}; + +const struct dentry_operations afs_fs_dentry_operations = { + .d_revalidate = afs_d_revalidate, + .d_delete = afs_d_delete, + .d_release = afs_d_release, + .d_automount = afs_d_automount, + .d_iput = afs_d_iput, +}; + +struct afs_lookup_one_cookie { + struct dir_context ctx; + struct qstr name; + bool found; + struct afs_fid fid; +}; + +struct afs_lookup_cookie { + struct dir_context ctx; + struct qstr name; + unsigned short nr_fids; + struct afs_fid fids[50]; +}; + +static void afs_dir_unuse_cookie(struct afs_vnode *dvnode, int ret) +{ + if (ret == 0) { + struct afs_vnode_cache_aux aux; + loff_t i_size = i_size_read(&dvnode->netfs.inode); + + afs_set_cache_aux(dvnode, &aux); + fscache_unuse_cookie(afs_vnode_cache(dvnode), &aux, &i_size); + } else { + fscache_unuse_cookie(afs_vnode_cache(dvnode), NULL, NULL); + } +} + +/* + * Iterate through a kmapped directory segment, dumping a summary of + * the contents. + */ +static size_t afs_dir_dump_step(void *iter_base, size_t progress, size_t len, + void *priv, void *priv2) +{ + do { + union afs_xdr_dir_block *block = iter_base; + + pr_warn("[%05zx] %32phN\n", progress, block); + iter_base += AFS_DIR_BLOCK_SIZE; + progress += AFS_DIR_BLOCK_SIZE; + len -= AFS_DIR_BLOCK_SIZE; + } while (len > 0); + + return len; +} + +/* + * Dump the contents of a directory. + */ +static void afs_dir_dump(struct afs_vnode *dvnode) +{ + struct iov_iter iter; + unsigned long long i_size = i_size_read(&dvnode->netfs.inode); + + pr_warn("DIR %llx:%llx is=%llx\n", + dvnode->fid.vid, dvnode->fid.vnode, i_size); + + iov_iter_folio_queue(&iter, ITER_SOURCE, dvnode->directory, 0, 0, i_size); + iterate_folioq(&iter, iov_iter_count(&iter), NULL, NULL, + afs_dir_dump_step); +} + +/* + * check that a directory folio is valid + */ +static bool afs_dir_check_block(struct afs_vnode *dvnode, size_t progress, + union afs_xdr_dir_block *block) +{ + if (block->hdr.magic != AFS_DIR_MAGIC) { + pr_warn("%s(%llx): [%zx] bad magic %04x\n", + __func__, dvnode->netfs.inode.i_ino, + progress, ntohs(block->hdr.magic)); + trace_afs_dir_check_failed(dvnode, progress); + trace_afs_file_error(dvnode, -EIO, afs_file_error_dir_bad_magic); + return false; + } + + /* Make sure each block is NUL terminated so we can reasonably + * use string functions on it. The filenames in the folio + * *should* be NUL-terminated anyway. + */ + ((u8 *)block)[AFS_DIR_BLOCK_SIZE - 1] = 0; + afs_stat_v(dvnode, n_read_dir); + return true; +} + +/* + * Iterate through a kmapped directory segment, checking the content. + */ +static size_t afs_dir_check_step(void *iter_base, size_t progress, size_t len, + void *priv, void *priv2) +{ + struct afs_vnode *dvnode = priv; + + if (WARN_ON_ONCE(progress % AFS_DIR_BLOCK_SIZE || + len % AFS_DIR_BLOCK_SIZE)) + return len; + + do { + if (!afs_dir_check_block(dvnode, progress, iter_base)) + break; + iter_base += AFS_DIR_BLOCK_SIZE; + len -= AFS_DIR_BLOCK_SIZE; + } while (len > 0); + + return len; +} + +/* + * Check all the blocks in a directory. + */ +static int afs_dir_check(struct afs_vnode *dvnode) +{ + struct iov_iter iter; + unsigned long long i_size = i_size_read(&dvnode->netfs.inode); + size_t checked = 0; + + if (unlikely(!i_size)) + return 0; + + iov_iter_folio_queue(&iter, ITER_SOURCE, dvnode->directory, 0, 0, i_size); + checked = iterate_folioq(&iter, iov_iter_count(&iter), dvnode, NULL, + afs_dir_check_step); + if (checked != i_size) { + afs_dir_dump(dvnode); + return -EIO; + } + return 0; +} + +/* + * open an AFS directory file + */ +static int afs_dir_open(struct inode *inode, struct file *file) +{ + _enter("{%llu}", inode->i_ino); + + BUILD_BUG_ON(sizeof(union afs_xdr_dir_block) != 2048); + BUILD_BUG_ON(sizeof(union afs_xdr_dirent) != 32); + + if (test_bit(AFS_VNODE_DELETED, &AFS_FS_I(inode)->flags)) + return -ENOENT; + + return afs_open(inode, file); +} + +/* + * Read a file in a single download. + */ +static ssize_t afs_do_read_single(struct afs_vnode *dvnode, struct file *file) +{ + struct iov_iter iter; + ssize_t ret; + loff_t i_size; + + i_size = i_size_read(&dvnode->netfs.inode); + if (i_size < AFS_DIR_BLOCK_SIZE) + return afs_bad(dvnode, afs_file_error_dir_small); + if (i_size > AFS_DIR_BLOCK_SIZE * 1024) { + trace_afs_file_error(dvnode, -EFBIG, afs_file_error_dir_big); + return -EFBIG; + } + + /* Expand the storage. TODO: Shrink the storage too. */ + if (dvnode->directory_size < i_size) { + size_t cur_size = dvnode->directory_size; + + ret = netfs_alloc_folioq_buffer(NULL, + &dvnode->directory, &cur_size, i_size, + mapping_gfp_mask(dvnode->netfs.inode.i_mapping)); + dvnode->directory_size = cur_size; + if (ret < 0) + return ret; + } + + iov_iter_folio_queue(&iter, ITER_DEST, dvnode->directory, 0, 0, dvnode->directory_size); + + /* AFS requires us to perform the read of a directory synchronously as + * a single unit to avoid issues with the directory contents being + * changed between reads. + */ + ret = netfs_read_single(&dvnode->netfs.inode, file, &iter); + if (ret >= 0) { + i_size = i_size_read(&dvnode->netfs.inode); + if (i_size > ret) { + /* The content has grown, so we need to expand the + * buffer. + */ + ret = -ESTALE; + } else { + int ret2 = afs_dir_check(dvnode); + + if (ret2 < 0) + ret = ret2; + } + } + + return ret; +} + +static ssize_t afs_read_single(struct afs_vnode *dvnode, struct file *file) +{ + ssize_t ret; + + fscache_use_cookie(afs_vnode_cache(dvnode), false); + ret = afs_do_read_single(dvnode, file); + fscache_unuse_cookie(afs_vnode_cache(dvnode), NULL, NULL); + return ret; +} + +/* + * Read the directory into a folio_queue buffer in one go, scrubbing the + * previous contents. We return -ESTALE if the caller needs to call us again. + */ +ssize_t afs_read_dir(struct afs_vnode *dvnode, struct file *file) + __acquires(&dvnode->validate_lock) +{ + ssize_t ret; + loff_t i_size; + + i_size = i_size_read(&dvnode->netfs.inode); + + ret = -ERESTARTSYS; + if (down_read_killable(&dvnode->validate_lock) < 0) + goto error; + + /* We only need to reread the data if it became invalid - or if we + * haven't read it yet. + */ + if (test_bit(AFS_VNODE_DIR_VALID, &dvnode->flags) && + test_bit(AFS_VNODE_DIR_READ, &dvnode->flags)) { + ret = i_size; + goto valid; + } + + up_read(&dvnode->validate_lock); + if (down_write_killable(&dvnode->validate_lock) < 0) + goto error; + + if (!test_bit(AFS_VNODE_DIR_VALID, &dvnode->flags)) + afs_invalidate_cache(dvnode, 0); + + if (!test_bit(AFS_VNODE_DIR_VALID, &dvnode->flags) || + !test_bit(AFS_VNODE_DIR_READ, &dvnode->flags)) { + trace_afs_reload_dir(dvnode); + ret = afs_read_single(dvnode, file); + if (ret < 0) + goto error_unlock; + + // TODO: Trim excess pages + + set_bit(AFS_VNODE_DIR_VALID, &dvnode->flags); + set_bit(AFS_VNODE_DIR_READ, &dvnode->flags); + } else { + ret = i_size; + } + + downgrade_write(&dvnode->validate_lock); +valid: + return ret; + +error_unlock: + up_write(&dvnode->validate_lock); +error: + _leave(" = %zd", ret); + return ret; +} + +/* + * deal with one block in an AFS directory + */ +static int afs_dir_iterate_block(struct afs_vnode *dvnode, + struct dir_context *ctx, + union afs_xdr_dir_block *block) +{ + union afs_xdr_dirent *dire; + unsigned int blknum, base, hdr, pos, next, nr_slots; + size_t nlen; + int tmp; + + blknum = ctx->pos / AFS_DIR_BLOCK_SIZE; + base = blknum * AFS_DIR_SLOTS_PER_BLOCK; + hdr = (blknum == 0 ? AFS_DIR_RESV_BLOCKS0 : AFS_DIR_RESV_BLOCKS); + pos = DIV_ROUND_UP(ctx->pos, AFS_DIR_DIRENT_SIZE) - base; + + _enter("%llx,%x", ctx->pos, blknum); + + /* walk through the block, an entry at a time */ + for (unsigned int slot = hdr; slot < AFS_DIR_SLOTS_PER_BLOCK; slot = next) { + /* skip entries marked unused in the bitmap */ + if (!(block->hdr.bitmap[slot / 8] & + (1 << (slot % 8)))) { + _debug("ENT[%x]: Unused", base + slot); + next = slot + 1; + if (next >= pos) + ctx->pos = (base + next) * sizeof(union afs_xdr_dirent); + continue; + } + + /* got a valid entry */ + dire = &block->dirents[slot]; + nlen = strnlen(dire->u.name, + (unsigned long)(block + 1) - (unsigned long)dire->u.name - 1); + if (nlen > AFSNAMEMAX - 1) { + _debug("ENT[%x]: Name too long (len %zx)", + base + slot, nlen); + return afs_bad(dvnode, afs_file_error_dir_name_too_long); + } + + _debug("ENT[%x]: %s %zx \"%s\"", + base + slot, (slot < pos ? "skip" : "fill"), + nlen, dire->u.name); + + nr_slots = afs_dir_calc_slots(nlen); + next = slot + nr_slots; + if (next > AFS_DIR_SLOTS_PER_BLOCK) { + _debug("ENT[%x]: extends beyond end dir block (len %zx)", + base + slot, nlen); + return afs_bad(dvnode, afs_file_error_dir_over_end); + } + + /* Check that the name-extension dirents are all allocated */ + for (tmp = 1; tmp < nr_slots; tmp++) { + unsigned int xslot = slot + tmp; + + if (!(block->hdr.bitmap[xslot / 8] & (1 << (xslot % 8)))) { + _debug("ENT[%x]: Unmarked extension (%x/%x)", + base + slot, tmp, nr_slots); + return afs_bad(dvnode, afs_file_error_dir_unmarked_ext); + } + } + + /* skip if starts before the current position */ + if (slot < pos) { + if (next > pos) + ctx->pos = (base + next) * sizeof(union afs_xdr_dirent); + continue; + } + + /* found the next entry */ + if (ctx->actor == AFS_LOOKUP) { + if (!afs_lookup_filldir(ctx, dire->u.name, nlen, + ntohl(dire->u.vnode), + ntohl(dire->u.unique))) + return 0; + } else if (ctx->actor == AFS_LOOKUP_ONE) { + if (!afs_lookup_one_filldir(ctx, dire->u.name, nlen, + ntohl(dire->u.vnode), + ntohl(dire->u.unique))) + return 0; + } else if (!dir_emit(ctx, dire->u.name, nlen, + ntohl(dire->u.vnode), DT_UNKNOWN)) { + _leave(" = 0 [full]"); + return 0; + } + + ctx->pos = (base + next) * sizeof(union afs_xdr_dirent); + } + + _leave(" = 1 [more]"); + return 1; +} + +struct afs_dir_iteration_ctx { + struct dir_context *dir_ctx; + int error; +}; + +/* + * Iterate through a kmapped directory segment. + */ +static size_t afs_dir_iterate_step(void *iter_base, size_t progress, size_t len, + void *priv, void *priv2) +{ + struct afs_dir_iteration_ctx *ctx = priv2; + struct afs_vnode *dvnode = priv; + int ret; + + if (WARN_ON_ONCE(progress % AFS_DIR_BLOCK_SIZE || + len % AFS_DIR_BLOCK_SIZE)) { + pr_err("Mis-iteration prog=%zx len=%zx\n", + progress % AFS_DIR_BLOCK_SIZE, + len % AFS_DIR_BLOCK_SIZE); + return len; + } + + do { + ret = afs_dir_iterate_block(dvnode, ctx->dir_ctx, iter_base); + if (ret != 1) + break; + + ctx->dir_ctx->pos = round_up(ctx->dir_ctx->pos, AFS_DIR_BLOCK_SIZE); + iter_base += AFS_DIR_BLOCK_SIZE; + len -= AFS_DIR_BLOCK_SIZE; + } while (len > 0); + + return len; +} + +/* + * Iterate through the directory folios. + */ +static int afs_dir_iterate_contents(struct inode *dir, struct dir_context *dir_ctx) +{ + struct afs_dir_iteration_ctx ctx = { .dir_ctx = dir_ctx }; + struct afs_vnode *dvnode = AFS_FS_I(dir); + struct iov_iter iter; + unsigned long long i_size = i_size_read(dir); + + /* Round the file position up to the next entry boundary */ + dir_ctx->pos = round_up(dir_ctx->pos, sizeof(union afs_xdr_dirent)); + + if (i_size <= 0 || dir_ctx->pos >= i_size) + return 0; + + iov_iter_folio_queue(&iter, ITER_SOURCE, dvnode->directory, 0, 0, i_size); + iov_iter_advance(&iter, round_down(dir_ctx->pos, AFS_DIR_BLOCK_SIZE)); + + iterate_folioq(&iter, iov_iter_count(&iter), dvnode, &ctx, + afs_dir_iterate_step); + + if (ctx.error == -ESTALE) + afs_invalidate_dir(dvnode, afs_dir_invalid_iter_stale); + return ctx.error; +} + +/* + * iterate through the data blob that lists the contents of an AFS directory + */ +static int afs_dir_iterate(struct inode *dir, struct dir_context *ctx, + struct file *file, afs_dataversion_t *_dir_version) +{ + struct afs_vnode *dvnode = AFS_FS_I(dir); + int retry_limit = 100; + int ret; + + _enter("{%llu},%llx,,", dir->i_ino, ctx->pos); + + do { + if (--retry_limit < 0) { + pr_warn("afs_read_dir(): Too many retries\n"); + ret = -ESTALE; + break; + } + ret = afs_read_dir(dvnode, file); + if (ret < 0) { + if (ret != -ESTALE) + break; + if (test_bit(AFS_VNODE_DELETED, &AFS_FS_I(dir)->flags)) { + ret = -ESTALE; + break; + } + continue; + } + *_dir_version = inode_peek_iversion_raw(dir); + + ret = afs_dir_iterate_contents(dir, ctx); + up_read(&dvnode->validate_lock); + } while (ret == -ESTALE); + + _leave(" = %d", ret); + return ret; +} + +/* + * read an AFS directory + */ +static int afs_readdir(struct file *file, struct dir_context *ctx) +{ + afs_dataversion_t dir_version; + + ctx->dt_flags_mask = UINT_MAX; + return afs_dir_iterate(file_inode(file), ctx, file, &dir_version); +} + +/* + * Search the directory for a single name + * - if afs_dir_iterate_block() spots this function, it'll pass the FID + * uniquifier through dtype + */ +static bool afs_lookup_one_filldir(struct dir_context *ctx, const char *name, + int nlen, u64 ino, u32 uniquifier) +{ + struct afs_lookup_one_cookie *cookie = + container_of(ctx, struct afs_lookup_one_cookie, ctx); + + _enter("{%s,%u},%s,%u,,%llu,%u", + cookie->name.name, cookie->name.len, name, nlen, + (unsigned long long) ino, uniquifier); + + /* insanity checks first */ + BUILD_BUG_ON(sizeof(union afs_xdr_dir_block) != 2048); + BUILD_BUG_ON(sizeof(union afs_xdr_dirent) != 32); + + if (cookie->name.len != nlen || + memcmp(cookie->name.name, name, nlen) != 0) { + _leave(" = true [keep looking]"); + return true; + } + + cookie->fid.vnode = ino; + cookie->fid.unique = uniquifier; + cookie->found = 1; + + _leave(" = false [found]"); + return false; +} + +/* + * Do a lookup of a single name in a directory + * - just returns the FID the dentry name maps to if found + */ +static int afs_do_lookup_one(struct inode *dir, const struct qstr *name, + struct afs_fid *fid, + afs_dataversion_t *_dir_version) +{ + struct afs_super_info *as = dir->i_sb->s_fs_info; + struct afs_lookup_one_cookie cookie = { + .ctx.actor = AFS_LOOKUP_ONE, + .name = *name, + .fid.vid = as->volume->vid + }; + int ret; + + _enter("{%llu},{%.*s},", dir->i_ino, name->len, name->name); + + /* search the directory */ + ret = afs_dir_iterate(dir, &cookie.ctx, NULL, _dir_version); + if (ret < 0) { + _leave(" = %d [iter]", ret); + return ret; + } + + if (!cookie.found) { + _leave(" = -ENOENT [not found]"); + return -ENOENT; + } + + *fid = cookie.fid; + _leave(" = 0 { vn=%llu u=%u }", fid->vnode, fid->unique); + return 0; +} + +/* + * search the directory for a name + * - if afs_dir_iterate_block() spots this function, it'll pass the FID + * uniquifier through dtype + */ +static bool afs_lookup_filldir(struct dir_context *ctx, const char *name, + int nlen, u64 ino, u32 uniquifier) +{ + struct afs_lookup_cookie *cookie = + container_of(ctx, struct afs_lookup_cookie, ctx); + + _enter("{%s,%u},%s,%u,,%llu,%u", + cookie->name.name, cookie->name.len, name, nlen, + (unsigned long long) ino, uniquifier); + + /* insanity checks first */ + BUILD_BUG_ON(sizeof(union afs_xdr_dir_block) != 2048); + BUILD_BUG_ON(sizeof(union afs_xdr_dirent) != 32); + + if (cookie->nr_fids < 50) { + cookie->fids[cookie->nr_fids].vnode = ino; + cookie->fids[cookie->nr_fids].unique = uniquifier; + cookie->nr_fids++; + } + + return cookie->nr_fids < 50; +} + +/* + * Deal with the result of a successful lookup operation. Turn all the files + * into inodes and save the first one - which is the one we actually want. + */ +static void afs_do_lookup_success(struct afs_operation *op) +{ + struct afs_vnode_param *vp; + struct afs_vnode *vnode; + struct inode *inode; + u32 abort_code; + int i; + + _enter(""); + + for (i = 0; i < op->nr_files; i++) { + switch (i) { + case 0: + vp = &op->file[0]; + abort_code = vp->scb.status.abort_code; + if (abort_code != 0) { + op->call_abort_code = abort_code; + afs_op_set_error(op, afs_abort_to_error(abort_code)); + op->cumul_error.abort_code = abort_code; + } + break; + + case 1: + vp = &op->file[1]; + break; + + default: + vp = &op->more_files[i - 2]; + break; + } + + if (vp->scb.status.abort_code) + trace_afs_bulkstat_error(op, &vp->fid, i, vp->scb.status.abort_code); + if (!vp->scb.have_status && !vp->scb.have_error) + continue; + + _debug("do [%u]", i); + if (vp->vnode) { + if (!test_bit(AFS_VNODE_UNSET, &vp->vnode->flags)) + afs_vnode_commit_status(op, vp); + } else if (vp->scb.status.abort_code == 0) { + inode = afs_iget(op, vp); + if (!IS_ERR(inode)) { + vnode = AFS_FS_I(inode); + afs_cache_permit(vnode, op->key, + 0 /* Assume vnode->cb_break is 0 */ + + op->cb_v_break, + &vp->scb); + vp->vnode = vnode; + vp->put_vnode = true; + } + } else { + _debug("- abort %d %llx:%llx.%x", + vp->scb.status.abort_code, + vp->fid.vid, vp->fid.vnode, vp->fid.unique); + } + } + + _leave(""); +} + +static const struct afs_operation_ops afs_inline_bulk_status_operation = { + .issue_afs_rpc = afs_fs_inline_bulk_status, + .issue_yfs_rpc = yfs_fs_inline_bulk_status, + .success = afs_do_lookup_success, +}; + +static const struct afs_operation_ops afs_lookup_fetch_status_operation = { + .issue_afs_rpc = afs_fs_fetch_status, + .issue_yfs_rpc = yfs_fs_fetch_status, + .success = afs_do_lookup_success, + .aborted = afs_check_for_remote_deletion, +}; + +/* + * See if we know that the server we expect to use doesn't support + * FS.InlineBulkStatus. + */ +static bool afs_server_supports_ibulk(struct afs_vnode *dvnode) +{ + struct afs_server_list *slist; + struct afs_volume *volume = dvnode->volume; + struct afs_server *server; + bool ret = true; + int i; + + if (!test_bit(AFS_VOLUME_MAYBE_NO_IBULK, &volume->flags)) + return true; + + rcu_read_lock(); + slist = rcu_dereference(volume->servers); + + for (i = 0; i < slist->nr_servers; i++) { + server = slist->servers[i].server; + if (server == dvnode->cb_server) { + if (test_bit(AFS_SERVER_FL_NO_IBULK, &server->flags)) + ret = false; + break; + } + } + + rcu_read_unlock(); + return ret; +} + +/* + * Do a lookup in a directory. We make use of bulk lookup to query a slew of + * files in one go and create inodes for them. The inode of the file we were + * asked for is returned. + */ +static struct inode *afs_do_lookup(struct inode *dir, struct dentry *dentry) +{ + struct afs_lookup_cookie *cookie; + struct afs_vnode_param *vp; + struct afs_operation *op; + struct afs_vnode *dvnode = AFS_FS_I(dir), *vnode; + struct inode *inode = NULL, *ti; + afs_dataversion_t data_version = READ_ONCE(dvnode->status.data_version); + bool supports_ibulk, isnew; + long ret; + int i; + + _enter("{%llu},%p{%pd},", dir->i_ino, dentry, dentry); + + cookie = kzalloc_obj(struct afs_lookup_cookie); + if (!cookie) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < ARRAY_SIZE(cookie->fids); i++) + cookie->fids[i].vid = dvnode->fid.vid; + cookie->ctx.actor = AFS_LOOKUP; + cookie->name = dentry->d_name; + cookie->nr_fids = 2; /* slot 1 is saved for the fid we actually want + * and slot 0 for the directory */ + + /* Search the directory for the named entry using the hash table... */ + ret = afs_dir_search(dvnode, &dentry->d_name, &cookie->fids[1], &data_version); + if (ret < 0) + goto out; + + supports_ibulk = afs_server_supports_ibulk(dvnode); + if (supports_ibulk) { + /* ...then scan linearly from that point for entries to lookup-ahead. */ + cookie->ctx.pos = (ret + 1) * AFS_DIR_DIRENT_SIZE; + afs_dir_iterate(dir, &cookie->ctx, NULL, &data_version); + } + + dentry->d_fsdata = (void *)(unsigned long)data_version; + + /* Check to see if we already have an inode for the primary fid. */ + inode = ilookup5(dir->i_sb, cookie->fids[1].vnode, + afs_ilookup5_test_by_fid, &cookie->fids[1]); + if (inode) + goto out; /* We do */ + + /* Okay, we didn't find it. We need to query the server - and whilst + * we're doing that, we're going to attempt to look up a bunch of other + * vnodes also. + */ + op = afs_alloc_operation(NULL, dvnode->volume); + if (IS_ERR(op)) { + ret = PTR_ERR(op); + goto out; + } + + afs_op_set_vnode(op, 0, dvnode); + afs_op_set_fid(op, 1, &cookie->fids[1]); + + op->nr_files = cookie->nr_fids; + _debug("nr_files %u", op->nr_files); + + /* Need space for examining all the selected files */ + if (op->nr_files > 2) { + op->more_files = kvzalloc_objs(struct afs_vnode_param, + op->nr_files - 2); + if (!op->more_files) { + afs_op_nomem(op); + goto out_op; + } + + for (i = 2; i < op->nr_files; i++) { + vp = &op->more_files[i - 2]; + vp->fid = cookie->fids[i]; + + /* Find any inodes that already exist and get their + * callback counters. + */ + ti = ilookup5_nowait(dir->i_sb, vp->fid.vnode, + afs_ilookup5_test_by_fid, &vp->fid, &isnew); + if (!IS_ERR_OR_NULL(ti)) { + vnode = AFS_FS_I(ti); + vp->dv_before = vnode->status.data_version; + vp->cb_break_before = afs_calc_vnode_cb_break(vnode); + vp->vnode = vnode; + vp->put_vnode = true; + vp->speculative = true; /* vnode not locked */ + } + } + } + + /* Try FS.InlineBulkStatus first. Abort codes for the individual + * lookups contained therein are stored in the reply without aborting + * the whole operation. + */ + afs_op_set_error(op, -ENOTSUPP); + if (supports_ibulk) { + op->ops = &afs_inline_bulk_status_operation; + afs_begin_vnode_operation(op); + afs_wait_for_operation(op); + } + + if (afs_op_error(op) == -ENOTSUPP) { + /* We could try FS.BulkStatus next, but this aborts the entire + * op if any of the lookups fails - so, for the moment, revert + * to FS.FetchStatus for op->file[1]. + */ + op->fetch_status.which = 1; + op->ops = &afs_lookup_fetch_status_operation; + afs_begin_vnode_operation(op); + afs_wait_for_operation(op); + } + +out_op: + if (!afs_op_error(op)) { + if (op->file[1].scb.status.abort_code) { + afs_op_accumulate_error(op, -ECONNABORTED, + op->file[1].scb.status.abort_code); + } else { + inode = &op->file[1].vnode->netfs.inode; + op->file[1].vnode = NULL; + } + } + + if (op->file[0].scb.have_status) + dentry->d_fsdata = (void *)(unsigned long)op->file[0].scb.status.data_version; + else + dentry->d_fsdata = (void *)(unsigned long)op->file[0].dv_before; + ret = afs_put_operation(op); +out: + kfree(cookie); + _leave(""); + return inode ?: ERR_PTR(ret); +} + +/* + * Look up an entry in a directory with @sys substitution. + */ +static struct dentry *afs_lookup_atsys(struct inode *dir, struct dentry *dentry) +{ + struct afs_sysnames *subs; + struct afs_net *net = afs_i2net(dir); + struct dentry *ret; + char *buf, *p, *name; + int len, i; + + _enter(""); + + ret = ERR_PTR(-ENOMEM); + p = buf = kmalloc(AFSNAMEMAX, GFP_KERNEL); + if (!buf) + goto out_p; + if (dentry->d_name.len > 4) { + memcpy(p, dentry->d_name.name, dentry->d_name.len - 4); + p += dentry->d_name.len - 4; + } + + /* There is an ordered list of substitutes that we have to try. */ + read_lock(&net->sysnames_lock); + subs = net->sysnames; + refcount_inc(&subs->usage); + read_unlock(&net->sysnames_lock); + + for (i = 0; i < subs->nr; i++) { + name = subs->subs[i]; + len = dentry->d_name.len - 4 + strlen(name); + if (len >= AFSNAMEMAX) { + ret = ERR_PTR(-ENAMETOOLONG); + goto out_s; + } + + strcpy(p, name); + ret = lookup_noperm(&QSTR(buf), dentry->d_parent); + if (IS_ERR(ret) || d_is_positive(ret)) + goto out_s; + dput(ret); + } + + /* We don't want to d_add() the @sys dentry here as we don't want to + * the cached dentry to hide changes to the sysnames list. + */ + ret = NULL; +out_s: + afs_put_sysnames(subs); + kfree(buf); +out_p: + return ret; +} + +/* + * look up an entry in a directory + */ +static struct dentry *afs_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + struct afs_vnode *dvnode = AFS_FS_I(dir); + struct afs_fid fid = {}; + struct inode *inode; + struct dentry *d; + int ret; + + _enter("{%llx:%llu},%p{%pd},", + dvnode->fid.vid, dvnode->fid.vnode, dentry, dentry); + + ASSERTCMP(d_inode(dentry), ==, NULL); + + if (dentry->d_name.len >= AFSNAMEMAX) { + _leave(" = -ENAMETOOLONG"); + return ERR_PTR(-ENAMETOOLONG); + } + + if (test_bit(AFS_VNODE_DELETED, &dvnode->flags)) { + _leave(" = -ESTALE"); + return ERR_PTR(-ESTALE); + } + + ret = afs_validate(dvnode, NULL); + if (ret < 0) { + afs_dir_unuse_cookie(dvnode, ret); + _leave(" = %d [val]", ret); + return ERR_PTR(ret); + } + + if (dentry->d_name.len >= 4 && + dentry->d_name.name[dentry->d_name.len - 4] == '@' && + dentry->d_name.name[dentry->d_name.len - 3] == 's' && + dentry->d_name.name[dentry->d_name.len - 2] == 'y' && + dentry->d_name.name[dentry->d_name.len - 1] == 's') + return afs_lookup_atsys(dir, dentry); + + afs_stat_v(dvnode, n_lookup); + inode = afs_do_lookup(dir, dentry); + if (inode == ERR_PTR(-ENOENT)) + inode = NULL; + else if (!IS_ERR_OR_NULL(inode)) + fid = AFS_FS_I(inode)->fid; + + _debug("splice %p", dentry->d_inode); + d = d_splice_alias(inode, dentry); + if (!IS_ERR_OR_NULL(d)) { + d->d_fsdata = dentry->d_fsdata; + trace_afs_lookup(dvnode, &d->d_name, &fid); + } else { + trace_afs_lookup(dvnode, &dentry->d_name, &fid); + } + _leave(""); + return d; +} + +/* + * Check the validity of a dentry under RCU conditions. + */ +static int afs_d_revalidate_rcu(struct afs_vnode *dvnode, struct dentry *dentry) +{ + long dir_version, de_version; + + _enter("%p", dentry); + + if (test_bit(AFS_VNODE_DELETED, &dvnode->flags)) + return -ECHILD; + + if (!afs_check_validity(dvnode)) + return -ECHILD; + + /* We only need to invalidate a dentry if the server's copy changed + * behind our back. If we made the change, it's no problem. Note that + * on a 32-bit system, we only have 32 bits in the dentry to store the + * version. + */ + dir_version = (long)READ_ONCE(dvnode->status.data_version); + de_version = (long)READ_ONCE(dentry->d_fsdata); + if (de_version != dir_version) { + dir_version = (long)READ_ONCE(dvnode->invalid_before); + if (de_version - dir_version < 0) + return -ECHILD; + } + + return 1; /* Still valid */ +} + +/* + * check that a dentry lookup hit has found a valid entry + * - NOTE! the hit can be a negative hit too, so we can't assume we have an + * inode + */ +static int afs_d_revalidate(struct inode *parent_dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + struct afs_vnode *vnode, *dir = AFS_FS_I(parent_dir); + struct afs_fid fid; + struct inode *inode; + struct key *key; + afs_dataversion_t dir_version, invalid_before; + long de_version; + int ret; + + if (flags & LOOKUP_RCU) + return afs_d_revalidate_rcu(dir, dentry); + + if (d_really_is_positive(dentry)) { + vnode = AFS_FS_I(d_inode(dentry)); + _enter("{v={%llx:%llu} n=%pd fl=%lx},", + vnode->fid.vid, vnode->fid.vnode, dentry, + vnode->flags); + } else { + _enter("{neg n=%pd}", dentry); + } + + key = afs_request_key(AFS_FS_S(dentry->d_sb)->volume->cell); + if (IS_ERR(key)) + key = NULL; + + /* validate the parent directory */ + ret = afs_validate(dir, key); + if (ret == -ERESTARTSYS) { + key_put(key); + return ret; + } + + if (test_bit(AFS_VNODE_DELETED, &dir->flags)) { + _debug("%pd: parent dir deleted", dentry); + goto not_found; + } + + /* We only need to invalidate a dentry if the server's copy changed + * behind our back. If we made the change, it's no problem. Note that + * on a 32-bit system, we only have 32 bits in the dentry to store the + * version. + */ + dir_version = dir->status.data_version; + de_version = (long)dentry->d_fsdata; + if (de_version == (long)dir_version) + goto out_valid_noupdate; + + invalid_before = dir->invalid_before; + if (de_version - (long)invalid_before >= 0) + goto out_valid; + + _debug("dir modified"); + afs_stat_v(dir, n_reval); + + /* search the directory for this vnode */ + ret = afs_do_lookup_one(&dir->netfs.inode, name, &fid, &dir_version); + switch (ret) { + case 0: + /* the filename maps to something */ + if (d_really_is_negative(dentry)) + goto not_found; + inode = d_inode(dentry); + if (is_bad_inode(inode)) { + printk("kAFS: afs_d_revalidate: %pd2 has bad inode\n", + dentry); + goto not_found; + } + + vnode = AFS_FS_I(inode); + + /* if the vnode ID has changed, then the dirent points to a + * different file */ + if (fid.vnode != vnode->fid.vnode) { + _debug("%pd: dirent changed [%llu != %llu]", + dentry, fid.vnode, + vnode->fid.vnode); + goto not_found; + } + + /* if the vnode ID uniqifier has changed, then the file has + * been deleted and replaced, and the original vnode ID has + * been reused */ + if (fid.unique != vnode->fid.unique) { + _debug("%pd: file deleted (uq %u -> %u I:%u)", + dentry, fid.unique, + vnode->fid.unique, + vnode->netfs.inode.i_generation); + goto not_found; + } + goto out_valid; + + case -ENOENT: + /* the filename is unknown */ + _debug("%pd: dirent not found", dentry); + if (d_really_is_positive(dentry)) + goto not_found; + goto out_valid; + + default: + _debug("failed to iterate parent %pd2: %d", dentry, ret); + goto not_found; + } + +out_valid: + dentry->d_fsdata = (void *)(unsigned long)dir_version; +out_valid_noupdate: + key_put(key); + _leave(" = 1 [valid]"); + return 1; + +not_found: + _debug("dropping dentry %pd2", dentry); + key_put(key); + + _leave(" = 0 [bad]"); + return 0; +} + +/* + * allow the VFS to enquire as to whether a dentry should be unhashed (mustn't + * sleep) + * - called from dput() when d_count is going to 0. + * - return 1 to request dentry be unhashed, 0 otherwise + */ +static int afs_d_delete(const struct dentry *dentry) +{ + _enter("%pd", dentry); + + if (dentry->d_flags & DCACHE_NFSFS_RENAMED) + goto zap; + + if (d_really_is_positive(dentry) && + (test_bit(AFS_VNODE_DELETED, &AFS_FS_I(d_inode(dentry))->flags) || + test_bit(AFS_VNODE_PSEUDODIR, &AFS_FS_I(d_inode(dentry))->flags))) + goto zap; + + _leave(" = 0 [keep]"); + return 0; + +zap: + _leave(" = 1 [zap]"); + return 1; +} + +/* + * Clean up sillyrename files on dentry removal. + */ +static void afs_d_iput(struct dentry *dentry, struct inode *inode) +{ + if (dentry->d_flags & DCACHE_NFSFS_RENAMED) + afs_silly_iput(dentry, inode); + iput(inode); +} + +/* + * handle dentry release + */ +void afs_d_release(struct dentry *dentry) +{ + _enter("%pd", dentry); +} + +void afs_check_for_remote_deletion(struct afs_operation *op) +{ + struct afs_vnode *vnode = op->file[0].vnode; + + switch (afs_op_abort_code(op)) { + case VNOVNODE: + set_bit(AFS_VNODE_DELETED, &vnode->flags); + clear_nlink(&vnode->netfs.inode); + afs_break_callback(vnode, afs_cb_break_for_deleted); + } +} + +/* + * Create a new inode for create/mkdir/symlink + */ +static void afs_vnode_new_inode(struct afs_operation *op) +{ + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_vnode_param *vp = &op->file[1]; + struct afs_vnode *vnode; + struct inode *inode; + + _enter(""); + + ASSERTCMP(afs_op_error(op), ==, 0); + + inode = afs_iget(op, vp); + if (IS_ERR(inode)) { + /* ENOMEM or EINTR at a really inconvenient time - just abandon + * the new directory on the server. + */ + afs_op_accumulate_error(op, PTR_ERR(inode), 0); + return; + } + + vnode = AFS_FS_I(inode); + set_bit(AFS_VNODE_NEW_CONTENT, &vnode->flags); + if (S_ISDIR(inode->i_mode)) + afs_mkdir_init_dir(vnode, dvp->vnode); + else if (S_ISLNK(inode->i_mode)) + afs_init_new_symlink(vnode, op); + if (!afs_op_error(op)) + afs_cache_permit(vnode, op->key, vnode->cb_break, &vp->scb); + d_instantiate(op->dentry, inode); +} + +static void afs_create_success(struct afs_operation *op) +{ + _enter("op=%08x", op->debug_id); + op->ctime = op->file[0].scb.status.mtime_client; + afs_vnode_commit_status(op, &op->file[0]); + afs_update_dentry_version(op, &op->file[0], op->dentry); + afs_vnode_new_inode(op); +} + +static void afs_create_edit_dir(struct afs_operation *op) +{ + struct netfs_cache_resources cres = {}; + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_vnode_param *vp = &op->file[1]; + struct afs_vnode *dvnode = dvp->vnode; + + _enter("op=%08x", op->debug_id); + + fscache_begin_write_operation(&cres, afs_vnode_cache(dvnode)); + down_write(&dvnode->validate_lock); + if (test_bit(AFS_VNODE_DIR_VALID, &dvnode->flags) && + dvnode->status.data_version == dvp->dv_before + dvp->dv_delta) + afs_edit_dir_add(dvnode, &op->dentry->d_name, &vp->fid, + op->create.reason); + up_write(&dvnode->validate_lock); + fscache_end_operation(&cres); +} + +static void afs_create_put(struct afs_operation *op) +{ + _enter("op=%08x", op->debug_id); + + if (afs_op_error(op)) + d_drop(op->dentry); +} + +static const struct afs_operation_ops afs_mkdir_operation = { + .issue_afs_rpc = afs_fs_make_dir, + .issue_yfs_rpc = yfs_fs_make_dir, + .success = afs_create_success, + .aborted = afs_check_for_remote_deletion, + .edit_dir = afs_create_edit_dir, + .put = afs_create_put, +}; + +/* + * create a directory on an AFS filesystem + */ +static struct dentry *afs_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct afs_operation *op; + struct afs_vnode *dvnode = AFS_FS_I(dir); + int ret; + + _enter("{%llx:%llu},{%pd},%ho", + dvnode->fid.vid, dvnode->fid.vnode, dentry, mode); + + op = afs_alloc_operation(NULL, dvnode->volume); + if (IS_ERR(op)) { + d_drop(dentry); + return ERR_CAST(op); + } + + fscache_use_cookie(afs_vnode_cache(dvnode), true); + + afs_op_set_vnode(op, 0, dvnode); + op->file[0].dv_delta = 1; + op->file[0].modification = true; + op->file[0].update_ctime = true; + op->dentry = dentry; + op->create.mode = mode; + op->create.reason = afs_edit_dir_for_mkdir; + op->mtime = current_time(dir); + op->ops = &afs_mkdir_operation; + ret = afs_do_sync_operation(op); + afs_dir_unuse_cookie(dvnode, ret); + return ERR_PTR(ret); +} + +/* + * Remove a subdir from a directory. + */ +static void afs_dir_remove_subdir(struct dentry *dentry) +{ + if (d_really_is_positive(dentry)) { + struct afs_vnode *vnode = AFS_FS_I(d_inode(dentry)); + + clear_nlink(&vnode->netfs.inode); + set_bit(AFS_VNODE_DELETED, &vnode->flags); + afs_clear_cb_promise(vnode, afs_cb_promise_clear_rmdir); + afs_invalidate_dir(vnode, afs_dir_invalid_subdir_removed); + } +} + +static void afs_rmdir_success(struct afs_operation *op) +{ + _enter("op=%08x", op->debug_id); + op->ctime = op->file[0].scb.status.mtime_client; + afs_vnode_commit_status(op, &op->file[0]); + afs_update_dentry_version(op, &op->file[0], op->dentry); +} + +static void afs_rmdir_edit_dir(struct afs_operation *op) +{ + struct netfs_cache_resources cres = {}; + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_vnode *dvnode = dvp->vnode; + + _enter("op=%08x", op->debug_id); + afs_dir_remove_subdir(op->dentry); + + fscache_begin_write_operation(&cres, afs_vnode_cache(dvnode)); + down_write(&dvnode->validate_lock); + if (test_bit(AFS_VNODE_DIR_VALID, &dvnode->flags) && + dvnode->status.data_version == dvp->dv_before + dvp->dv_delta) + afs_edit_dir_remove(dvnode, &op->dentry->d_name, + afs_edit_dir_for_rmdir); + up_write(&dvnode->validate_lock); + fscache_end_operation(&cres); +} + +static void afs_rmdir_put(struct afs_operation *op) +{ + _enter("op=%08x", op->debug_id); + if (op->file[1].vnode) + up_write(&op->file[1].vnode->rmdir_lock); +} + +static const struct afs_operation_ops afs_rmdir_operation = { + .issue_afs_rpc = afs_fs_remove_dir, + .issue_yfs_rpc = yfs_fs_remove_dir, + .success = afs_rmdir_success, + .aborted = afs_check_for_remote_deletion, + .edit_dir = afs_rmdir_edit_dir, + .put = afs_rmdir_put, +}; + +/* + * remove a directory from an AFS filesystem + */ +static int afs_rmdir(struct inode *dir, struct dentry *dentry) +{ + struct afs_operation *op; + struct afs_vnode *dvnode = AFS_FS_I(dir), *vnode = NULL; + int ret; + + _enter("{%llx:%llu},{%pd}", + dvnode->fid.vid, dvnode->fid.vnode, dentry); + + op = afs_alloc_operation(NULL, dvnode->volume); + if (IS_ERR(op)) + return PTR_ERR(op); + + fscache_use_cookie(afs_vnode_cache(dvnode), true); + + afs_op_set_vnode(op, 0, dvnode); + op->file[0].dv_delta = 1; + op->file[0].modification = true; + op->file[0].update_ctime = true; + + op->dentry = dentry; + op->ops = &afs_rmdir_operation; + + /* Try to make sure we have a callback promise on the victim. */ + if (d_really_is_positive(dentry)) { + vnode = AFS_FS_I(d_inode(dentry)); + ret = afs_validate(vnode, op->key); + if (ret < 0) + goto error; + } + + if (vnode) { + ret = down_write_killable(&vnode->rmdir_lock); + if (ret < 0) + goto error; + op->file[1].vnode = vnode; + } + + ret = afs_do_sync_operation(op); + + /* Not all systems that can host afs servers have ENOTEMPTY. */ + if (ret == -EEXIST) + ret = -ENOTEMPTY; +out: + afs_dir_unuse_cookie(dvnode, ret); + return ret; + +error: + ret = afs_put_operation(op); + goto out; +} + +/* + * Remove a link to a file or symlink from a directory. + * + * If the file was not deleted due to excess hard links, the fileserver will + * break the callback promise on the file - if it had one - before it returns + * to us, and if it was deleted, it won't + * + * However, if we didn't have a callback promise outstanding, or it was + * outstanding on a different server, then it won't break it either... + */ +static void afs_dir_remove_link(struct afs_operation *op) +{ + struct afs_vnode *dvnode = op->file[0].vnode; + struct afs_vnode *vnode = op->file[1].vnode; + struct dentry *dentry = op->dentry; + int ret; + + if (afs_op_error(op) || + (op->file[1].scb.have_status && op->file[1].scb.have_error)) + return; + if (d_really_is_positive(dentry)) + return; + + if (test_bit(AFS_VNODE_DELETED, &vnode->flags)) { + /* Already done */ + } else if (test_bit(AFS_VNODE_DIR_VALID, &dvnode->flags)) { + write_seqlock(&vnode->cb_lock); + drop_nlink(&vnode->netfs.inode); + if (vnode->netfs.inode.i_nlink == 0) { + set_bit(AFS_VNODE_DELETED, &vnode->flags); + __afs_break_callback(vnode, afs_cb_break_for_unlink); + } + write_sequnlock(&vnode->cb_lock); + } else { + afs_break_callback(vnode, afs_cb_break_for_unlink); + + if (test_bit(AFS_VNODE_DELETED, &vnode->flags)) + _debug("AFS_VNODE_DELETED"); + + ret = afs_validate(vnode, op->key); + if (ret != -ESTALE) + afs_op_set_error(op, ret); + } + + _debug("nlink %d [val %d]", vnode->netfs.inode.i_nlink, afs_op_error(op)); +} + +static void afs_unlink_success(struct afs_operation *op) +{ + _enter("op=%08x", op->debug_id); + op->ctime = op->file[0].scb.status.mtime_client; + afs_check_dir_conflict(op, &op->file[0]); + afs_vnode_commit_status(op, &op->file[0]); + afs_vnode_commit_status(op, &op->file[1]); + afs_update_dentry_version(op, &op->file[0], op->dentry); + afs_dir_remove_link(op); +} + +static void afs_unlink_edit_dir(struct afs_operation *op) +{ + struct netfs_cache_resources cres = {}; + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_vnode *dvnode = dvp->vnode; + + _enter("op=%08x", op->debug_id); + fscache_begin_write_operation(&cres, afs_vnode_cache(dvnode)); + down_write(&dvnode->validate_lock); + if (test_bit(AFS_VNODE_DIR_VALID, &dvnode->flags) && + dvnode->status.data_version == dvp->dv_before + dvp->dv_delta) + afs_edit_dir_remove(dvnode, &op->dentry->d_name, + afs_edit_dir_for_unlink); + up_write(&dvnode->validate_lock); + fscache_end_operation(&cres); +} + +static void afs_unlink_put(struct afs_operation *op) +{ + _enter("op=%08x", op->debug_id); + if (op->unlink.need_rehash && afs_op_error(op) < 0 && afs_op_error(op) != -ENOENT) + d_rehash(op->dentry); +} + +static const struct afs_operation_ops afs_unlink_operation = { + .issue_afs_rpc = afs_fs_remove_file, + .issue_yfs_rpc = yfs_fs_remove_file, + .success = afs_unlink_success, + .aborted = afs_check_for_remote_deletion, + .edit_dir = afs_unlink_edit_dir, + .put = afs_unlink_put, +}; + +/* + * Remove a file or symlink from an AFS filesystem. + */ +static int afs_unlink(struct inode *dir, struct dentry *dentry) +{ + struct afs_operation *op; + struct afs_vnode *dvnode = AFS_FS_I(dir); + struct afs_vnode *vnode = AFS_FS_I(d_inode(dentry)); + int ret; + + _enter("{%llx:%llu},{%pd}", + dvnode->fid.vid, dvnode->fid.vnode, dentry); + + if (dentry->d_name.len >= AFSNAMEMAX) + return -ENAMETOOLONG; + + op = afs_alloc_operation(NULL, dvnode->volume); + if (IS_ERR(op)) + return PTR_ERR(op); + + fscache_use_cookie(afs_vnode_cache(dvnode), true); + + afs_op_set_vnode(op, 0, dvnode); + op->file[0].dv_delta = 1; + op->file[0].modification = true; + op->file[0].update_ctime = true; + + /* Try to make sure we have a callback promise on the victim. */ + ret = afs_validate(vnode, op->key); + if (ret < 0) { + afs_op_set_error(op, ret); + goto error; + } + + spin_lock(&dentry->d_lock); + if (d_count(dentry) > 1) { + spin_unlock(&dentry->d_lock); + /* Start asynchronous writeout of the inode */ + write_inode_now(d_inode(dentry), 0); + afs_op_set_error(op, afs_sillyrename(dvnode, vnode, dentry, op->key)); + goto error; + } + if (!d_unhashed(dentry)) { + /* Prevent a race with RCU lookup. */ + __d_drop(dentry); + op->unlink.need_rehash = true; + } + spin_unlock(&dentry->d_lock); + + op->file[1].vnode = vnode; + op->file[1].update_ctime = true; + op->file[1].op_unlinked = true; + op->dentry = dentry; + op->ops = &afs_unlink_operation; + afs_begin_vnode_operation(op); + afs_wait_for_operation(op); + + /* If there was a conflict with a third party, check the status of the + * unlinked vnode. + */ + if (afs_op_error(op) == 0 && (op->flags & AFS_OPERATION_DIR_CONFLICT)) { + op->file[1].update_ctime = false; + op->fetch_status.which = 1; + op->ops = &afs_fetch_status_operation; + afs_begin_vnode_operation(op); + afs_wait_for_operation(op); + } + +error: + ret = afs_put_operation(op); + afs_dir_unuse_cookie(dvnode, ret); + return ret; +} + +static const struct afs_operation_ops afs_create_operation = { + .issue_afs_rpc = afs_fs_create_file, + .issue_yfs_rpc = yfs_fs_create_file, + .success = afs_create_success, + .aborted = afs_check_for_remote_deletion, + .edit_dir = afs_create_edit_dir, + .put = afs_create_put, +}; + +/* + * create a regular file on an AFS filesystem + */ +static int afs_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct afs_operation *op; + struct afs_vnode *dvnode = AFS_FS_I(dir); + int ret = -ENAMETOOLONG; + + _enter("{%llx:%llu},{%pd},%ho", + dvnode->fid.vid, dvnode->fid.vnode, dentry, mode); + + if (dentry->d_name.len >= AFSNAMEMAX) + goto error; + + op = afs_alloc_operation(NULL, dvnode->volume); + if (IS_ERR(op)) { + ret = PTR_ERR(op); + goto error; + } + + fscache_use_cookie(afs_vnode_cache(dvnode), true); + + afs_op_set_vnode(op, 0, dvnode); + op->file[0].dv_delta = 1; + op->file[0].modification = true; + op->file[0].update_ctime = true; + + op->dentry = dentry; + op->create.mode = S_IFREG | mode; + op->create.reason = afs_edit_dir_for_create; + op->mtime = current_time(dir); + op->ops = &afs_create_operation; + ret = afs_do_sync_operation(op); + afs_dir_unuse_cookie(dvnode, ret); + return ret; + +error: + d_drop(dentry); + _leave(" = %d", ret); + return ret; +} + +static void afs_link_success(struct afs_operation *op) +{ + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_vnode_param *vp = &op->file[1]; + + _enter("op=%08x", op->debug_id); + op->ctime = dvp->scb.status.mtime_client; + afs_vnode_commit_status(op, dvp); + afs_vnode_commit_status(op, vp); + afs_update_dentry_version(op, dvp, op->dentry); + if (op->dentry_2->d_parent == op->dentry->d_parent) + afs_update_dentry_version(op, dvp, op->dentry_2); + ihold(&vp->vnode->netfs.inode); + d_instantiate(op->dentry, &vp->vnode->netfs.inode); +} + +static void afs_link_put(struct afs_operation *op) +{ + _enter("op=%08x", op->debug_id); + if (afs_op_error(op)) + d_drop(op->dentry); +} + +static const struct afs_operation_ops afs_link_operation = { + .issue_afs_rpc = afs_fs_link, + .issue_yfs_rpc = yfs_fs_link, + .success = afs_link_success, + .aborted = afs_check_for_remote_deletion, + .edit_dir = afs_create_edit_dir, + .put = afs_link_put, +}; + +/* + * create a hard link between files in an AFS filesystem + */ +static int afs_link(struct dentry *from, struct inode *dir, + struct dentry *dentry) +{ + struct afs_operation *op; + struct afs_vnode *dvnode = AFS_FS_I(dir); + struct afs_vnode *vnode = AFS_FS_I(d_inode(from)); + int ret = -ENAMETOOLONG; + + _enter("{%llx:%llu},{%llx:%llu},{%pd}", + vnode->fid.vid, vnode->fid.vnode, + dvnode->fid.vid, dvnode->fid.vnode, + dentry); + + if (dentry->d_name.len >= AFSNAMEMAX) + goto error; + + op = afs_alloc_operation(NULL, dvnode->volume); + if (IS_ERR(op)) { + ret = PTR_ERR(op); + goto error; + } + + fscache_use_cookie(afs_vnode_cache(dvnode), true); + + ret = afs_validate(vnode, op->key); + if (ret < 0) + goto error_op; + + afs_op_set_vnode(op, 0, dvnode); + afs_op_set_vnode(op, 1, vnode); + op->file[0].dv_delta = 1; + op->file[0].modification = true; + op->file[0].update_ctime = true; + op->file[1].update_ctime = true; + + op->dentry = dentry; + op->dentry_2 = from; + op->ops = &afs_link_operation; + op->create.reason = afs_edit_dir_for_link; + ret = afs_do_sync_operation(op); + afs_dir_unuse_cookie(dvnode, ret); + return ret; + +error_op: + afs_put_operation(op); + afs_dir_unuse_cookie(dvnode, ret); +error: + d_drop(dentry); + _leave(" = %d", ret); + return ret; +} + +static void afs_symlink_put(struct afs_operation *op) +{ + kfree(op->create.symlink); + op->create.symlink = NULL; + afs_create_put(op); +} + +static const struct afs_operation_ops afs_symlink_operation = { + .issue_afs_rpc = afs_fs_symlink, + .issue_yfs_rpc = yfs_fs_symlink, + .success = afs_create_success, + .aborted = afs_check_for_remote_deletion, + .edit_dir = afs_create_edit_dir, + .put = afs_symlink_put, +}; + +/* + * create a symlink in an AFS filesystem + */ +static int afs_symlink(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const char *content) +{ + struct afs_operation *op; + struct afs_symlink *symlink; + struct afs_vnode *dvnode = AFS_FS_I(dir); + size_t clen = strlen(content); + int ret; + + _enter("{%llx:%llu},{%pd},%s", + dvnode->fid.vid, dvnode->fid.vnode, dentry, + content); + + ret = -ENAMETOOLONG; + if (dentry->d_name.len >= AFSNAMEMAX) + goto error; + + ret = -EINVAL; + if (clen >= AFSPATHMAX) + goto error; + + ret = -ENOMEM; + symlink = kmalloc_flex(struct afs_symlink, content, clen + 1, GFP_KERNEL); + if (!symlink) + goto error; + refcount_set(&symlink->ref, 1); + memcpy(symlink->content, content, clen + 1); + + op = afs_alloc_operation(NULL, dvnode->volume); + if (IS_ERR(op)) { + ret = PTR_ERR(op); + kfree(symlink); + goto error; + } + + fscache_use_cookie(afs_vnode_cache(dvnode), true); + + afs_op_set_vnode(op, 0, dvnode); + op->file[0].dv_delta = 1; + + op->dentry = dentry; + op->ops = &afs_symlink_operation; + op->create.reason = afs_edit_dir_for_symlink; + op->create.symlink = symlink; + op->mtime = current_time(dir); + ret = afs_do_sync_operation(op); + afs_dir_unuse_cookie(dvnode, ret); + return ret; + +error: + d_drop(dentry); + _leave(" = %d", ret); + return ret; +} + +static void afs_rename_success(struct afs_operation *op) +{ + struct afs_vnode *vnode = op->more_files[0].vnode; + struct afs_vnode *new_vnode = op->more_files[1].vnode; + + _enter("op=%08x", op->debug_id); + + op->ctime = op->file[0].scb.status.mtime_client; + afs_check_dir_conflict(op, &op->file[1]); + afs_vnode_commit_status(op, &op->file[0]); + if (op->file[1].vnode != op->file[0].vnode) { + op->ctime = op->file[1].scb.status.mtime_client; + afs_vnode_commit_status(op, &op->file[1]); + } + if (op->more_files[0].scb.have_status) + afs_vnode_commit_status(op, &op->more_files[0]); + if (op->more_files[1].scb.have_status) + afs_vnode_commit_status(op, &op->more_files[1]); + + /* If we're moving a subdir between dirs, we need to update + * its DV counter too as the ".." will be altered. + */ + if (op->file[0].vnode != op->file[1].vnode) { + if (S_ISDIR(vnode->netfs.inode.i_mode)) { + u64 new_dv; + + write_seqlock(&vnode->cb_lock); + + new_dv = vnode->status.data_version + 1; + trace_afs_set_dv(vnode, new_dv); + vnode->status.data_version = new_dv; + inode_set_iversion_raw(&vnode->netfs.inode, new_dv); + + write_sequnlock(&vnode->cb_lock); + } + + if ((op->rename.rename_flags & RENAME_EXCHANGE) && + S_ISDIR(new_vnode->netfs.inode.i_mode)) { + u64 new_dv; + + write_seqlock(&new_vnode->cb_lock); + + new_dv = new_vnode->status.data_version + 1; + new_vnode->status.data_version = new_dv; + inode_set_iversion_raw(&new_vnode->netfs.inode, new_dv); + + write_sequnlock(&new_vnode->cb_lock); + } + } +} + +static void afs_rename_edit_dir(struct afs_operation *op) +{ + struct netfs_cache_resources orig_cres = {}, new_cres = {}; + struct afs_vnode_param *orig_dvp = &op->file[0]; + struct afs_vnode_param *new_dvp = &op->file[1]; + struct afs_vnode *orig_dvnode = orig_dvp->vnode; + struct afs_vnode *new_dvnode = new_dvp->vnode; + struct afs_vnode *vnode = AFS_FS_I(d_inode(op->dentry)); + struct dentry *old_dentry = op->dentry; + struct dentry *new_dentry = op->dentry_2; + struct inode *new_inode; + + _enter("op=%08x", op->debug_id); + + if (op->rename.rehash) { + d_rehash(op->rename.rehash); + op->rename.rehash = NULL; + } + + fscache_begin_write_operation(&orig_cres, afs_vnode_cache(orig_dvnode)); + if (new_dvnode != orig_dvnode) + fscache_begin_write_operation(&new_cres, afs_vnode_cache(new_dvnode)); + + down_write(&orig_dvnode->validate_lock); + if (test_bit(AFS_VNODE_DIR_VALID, &orig_dvnode->flags) && + orig_dvnode->status.data_version == orig_dvp->dv_before + orig_dvp->dv_delta) + afs_edit_dir_remove(orig_dvnode, &old_dentry->d_name, + afs_edit_dir_for_rename_0); + + if (new_dvnode != orig_dvnode) { + up_write(&orig_dvnode->validate_lock); + down_write(&new_dvnode->validate_lock); + } + + if (test_bit(AFS_VNODE_DIR_VALID, &new_dvnode->flags) && + new_dvnode->status.data_version == new_dvp->dv_before + new_dvp->dv_delta) { + if (!op->rename.new_negative) + afs_edit_dir_remove(new_dvnode, &new_dentry->d_name, + afs_edit_dir_for_rename_1); + + afs_edit_dir_add(new_dvnode, &new_dentry->d_name, + &vnode->fid, afs_edit_dir_for_rename_2); + } + + if (S_ISDIR(vnode->netfs.inode.i_mode) && + new_dvnode != orig_dvnode && + test_bit(AFS_VNODE_DIR_VALID, &vnode->flags)) + afs_edit_dir_update(vnode, &dotdot_name, new_dvnode, + afs_edit_dir_for_rename_sub); + + new_inode = d_inode(new_dentry); + if (new_inode) { + spin_lock(&new_inode->i_lock); + if (S_ISDIR(new_inode->i_mode)) + clear_nlink(new_inode); + else if (new_inode->i_nlink > 0) + drop_nlink(new_inode); + spin_unlock(&new_inode->i_lock); + } + + /* Now we can update d_fsdata on the dentries to reflect their + * new parent's data_version. + */ + afs_update_dentry_version(op, new_dvp, op->dentry); + afs_update_dentry_version(op, new_dvp, op->dentry_2); + + d_move(old_dentry, new_dentry); + + up_write(&new_dvnode->validate_lock); + fscache_end_operation(&orig_cres); + if (new_dvnode != orig_dvnode) + fscache_end_operation(&new_cres); +} + +static void afs_rename_exchange_edit_dir(struct afs_operation *op) +{ + struct afs_vnode_param *orig_dvp = &op->file[0]; + struct afs_vnode_param *new_dvp = &op->file[1]; + struct afs_vnode *orig_dvnode = orig_dvp->vnode; + struct afs_vnode *new_dvnode = new_dvp->vnode; + struct afs_vnode *old_vnode = op->more_files[0].vnode; + struct afs_vnode *new_vnode = op->more_files[1].vnode; + struct dentry *old_dentry = op->dentry; + struct dentry *new_dentry = op->dentry_2; + + _enter("op=%08x", op->debug_id); + + if (new_dvnode == orig_dvnode) { + down_write(&orig_dvnode->validate_lock); + if (test_bit(AFS_VNODE_DIR_VALID, &orig_dvnode->flags) && + orig_dvnode->status.data_version == orig_dvp->dv_before + orig_dvp->dv_delta) { + afs_edit_dir_update(orig_dvnode, &old_dentry->d_name, + new_vnode, afs_edit_dir_for_rename_0); + afs_edit_dir_update(orig_dvnode, &new_dentry->d_name, + old_vnode, afs_edit_dir_for_rename_1); + } + + d_exchange(old_dentry, new_dentry); + up_write(&orig_dvnode->validate_lock); + } else { + down_write(&orig_dvnode->validate_lock); + if (test_bit(AFS_VNODE_DIR_VALID, &orig_dvnode->flags) && + orig_dvnode->status.data_version == orig_dvp->dv_before + orig_dvp->dv_delta) + afs_edit_dir_update(orig_dvnode, &old_dentry->d_name, + new_vnode, afs_edit_dir_for_rename_0); + + up_write(&orig_dvnode->validate_lock); + down_write(&new_dvnode->validate_lock); + + if (test_bit(AFS_VNODE_DIR_VALID, &new_dvnode->flags) && + new_dvnode->status.data_version == new_dvp->dv_before + new_dvp->dv_delta) + afs_edit_dir_update(new_dvnode, &new_dentry->d_name, + old_vnode, afs_edit_dir_for_rename_1); + + if (S_ISDIR(old_vnode->netfs.inode.i_mode) && + test_bit(AFS_VNODE_DIR_VALID, &old_vnode->flags)) + afs_edit_dir_update(old_vnode, &dotdot_name, new_dvnode, + afs_edit_dir_for_rename_sub); + + if (S_ISDIR(new_vnode->netfs.inode.i_mode) && + test_bit(AFS_VNODE_DIR_VALID, &new_vnode->flags)) + afs_edit_dir_update(new_vnode, &dotdot_name, orig_dvnode, + afs_edit_dir_for_rename_sub); + + /* Now we can update d_fsdata on the dentries to reflect their + * new parents' data_version. + */ + afs_update_dentry_version(op, new_dvp, old_dentry); + afs_update_dentry_version(op, orig_dvp, new_dentry); + + d_exchange(old_dentry, new_dentry); + up_write(&new_dvnode->validate_lock); + } +} + +static void afs_rename_put(struct afs_operation *op) +{ + _enter("op=%08x", op->debug_id); + if (op->rename.rehash) + d_rehash(op->rename.rehash); + dput(op->rename.tmp); + if (afs_op_error(op)) + d_rehash(op->dentry); +} + +static const struct afs_operation_ops afs_rename_operation = { + .issue_afs_rpc = afs_fs_rename, + .issue_yfs_rpc = yfs_fs_rename, + .success = afs_rename_success, + .edit_dir = afs_rename_edit_dir, + .put = afs_rename_put, +}; + +#if 0 /* Autoswitched in yfs_fs_rename_replace(). */ +static const struct afs_operation_ops afs_rename_replace_operation = { + .issue_afs_rpc = NULL, + .issue_yfs_rpc = yfs_fs_rename_replace, + .success = afs_rename_success, + .edit_dir = afs_rename_edit_dir, + .put = afs_rename_put, +}; +#endif + +static const struct afs_operation_ops afs_rename_noreplace_operation = { + .issue_afs_rpc = NULL, + .issue_yfs_rpc = yfs_fs_rename_noreplace, + .success = afs_rename_success, + .edit_dir = afs_rename_edit_dir, + .put = afs_rename_put, +}; + +static const struct afs_operation_ops afs_rename_exchange_operation = { + .issue_afs_rpc = NULL, + .issue_yfs_rpc = yfs_fs_rename_exchange, + .success = afs_rename_success, + .edit_dir = afs_rename_exchange_edit_dir, + .put = afs_rename_put, +}; + +/* + * rename a file in an AFS filesystem and/or move it between directories + */ +static int afs_rename(struct mnt_idmap *idmap, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) +{ + struct afs_operation *op; + struct afs_vnode *orig_dvnode, *new_dvnode, *vnode, *new_vnode = NULL; + int ret; + + if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE)) + return -EINVAL; + + /* Don't allow silly-rename files be moved around. */ + if (old_dentry->d_flags & DCACHE_NFSFS_RENAMED) + return -EINVAL; + + vnode = AFS_FS_I(d_inode(old_dentry)); + orig_dvnode = AFS_FS_I(old_dir); + new_dvnode = AFS_FS_I(new_dir); + if (d_is_positive(new_dentry)) + new_vnode = AFS_FS_I(d_inode(new_dentry)); + + _enter("{%llx:%llu},{%llx:%llu},{%llx:%llu},{%pd}", + orig_dvnode->fid.vid, orig_dvnode->fid.vnode, + vnode->fid.vid, vnode->fid.vnode, + new_dvnode->fid.vid, new_dvnode->fid.vnode, + new_dentry); + + op = afs_alloc_operation(NULL, orig_dvnode->volume); + if (IS_ERR(op)) + return PTR_ERR(op); + + fscache_use_cookie(afs_vnode_cache(orig_dvnode), true); + if (new_dvnode != orig_dvnode) + fscache_use_cookie(afs_vnode_cache(new_dvnode), true); + + ret = afs_validate(vnode, op->key); + afs_op_set_error(op, ret); + if (ret < 0) + goto error; + + ret = -ENOMEM; + op->more_files = kvzalloc_objs(struct afs_vnode_param, 2); + if (!op->more_files) + goto error; + + afs_op_set_vnode(op, 0, orig_dvnode); + afs_op_set_vnode(op, 1, new_dvnode); /* May be same as orig_dvnode */ + op->file[0].dv_delta = 1; + op->file[1].dv_delta = 1; + op->file[0].modification = true; + op->file[1].modification = true; + op->file[0].update_ctime = true; + op->file[1].update_ctime = true; + op->more_files[0].vnode = vnode; + op->more_files[0].speculative = true; + op->more_files[1].vnode = new_vnode; + op->more_files[1].speculative = true; + op->nr_files = 4; + + op->dentry = old_dentry; + op->dentry_2 = new_dentry; + op->rename.rename_flags = flags; + op->rename.new_negative = d_is_negative(new_dentry); + + if (flags & RENAME_NOREPLACE) { + op->ops = &afs_rename_noreplace_operation; + } else if (flags & RENAME_EXCHANGE) { + op->ops = &afs_rename_exchange_operation; + d_drop(new_dentry); + } else { + /* If we might displace the target, we might need to do silly + * rename. + */ + op->ops = &afs_rename_operation; + + /* For non-directories, check whether the target is busy and if + * so, make a copy of the dentry and then do a silly-rename. + * If the silly-rename succeeds, the copied dentry is hashed + * and becomes the new target. + */ + if (d_is_positive(new_dentry) && !d_is_dir(new_dentry)) { + /* To prevent any new references to the target during + * the rename, we unhash the dentry in advance. + */ + if (!d_unhashed(new_dentry)) { + d_drop(new_dentry); + op->rename.rehash = new_dentry; + } + + if (d_count(new_dentry) > 2) { + /* copy the target dentry's name */ + op->rename.tmp = d_alloc(new_dentry->d_parent, + &new_dentry->d_name); + if (!op->rename.tmp) { + afs_op_nomem(op); + goto error; + } + + ret = afs_sillyrename(new_dvnode, + AFS_FS_I(d_inode(new_dentry)), + new_dentry, op->key); + if (ret) { + afs_op_set_error(op, ret); + goto error; + } + + op->dentry_2 = op->rename.tmp; + op->rename.rehash = NULL; + op->rename.new_negative = true; + } + } + } + + /* This bit is potentially nasty as there's a potential race with + * afs_d_revalidate{,_rcu}(). We have to change d_fsdata on the dentry + * to reflect it's new parent's new data_version after the op, but + * d_revalidate may see old_dentry between the op having taken place + * and the version being updated. + * + * So drop the old_dentry for now to make other threads go through + * lookup instead - which we hold a lock against. + */ + d_drop(old_dentry); + + ret = afs_do_sync_operation(op); + if (ret == -ENOTSUPP) + ret = -EINVAL; +out: + afs_dir_unuse_cookie(orig_dvnode, ret); + if (new_dvnode != orig_dvnode) + afs_dir_unuse_cookie(new_dvnode, ret); + return ret; + +error: + ret = afs_put_operation(op); + goto out; +} + +/* + * Write the directory contents to the cache as a single blob. + */ +static int afs_dir_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct afs_vnode *dvnode = AFS_FS_I(mapping->host); + struct iov_iter iter; + int ret = 0; + + /* Need to lock to prevent the folio queue and folios from being thrown + * away. + */ + if (!down_read_trylock(&dvnode->validate_lock)) { + if (wbc->sync_mode == WB_SYNC_NONE) { + /* The VFS will have undirtied the inode. */ + netfs_single_mark_inode_dirty(&dvnode->netfs.inode); + return 0; + } + down_read(&dvnode->validate_lock); + } + + if (test_bit(AFS_VNODE_DIR_VALID, &dvnode->flags)) { + iov_iter_folio_queue(&iter, ITER_SOURCE, dvnode->directory, 0, 0, + i_size_read(&dvnode->netfs.inode)); + ret = netfs_writeback_single(mapping, wbc, &iter); + if (ret == 1) + ret = 0; /* Skipped write due to lock conflict. */ + } + + up_read(&dvnode->validate_lock); + return ret; +} diff --git a/fs/afs/dir_edit.c b/fs/afs/dir_edit.c new file mode 100644 index 000000000..3ead36a07 --- /dev/null +++ b/fs/afs/dir_edit.c @@ -0,0 +1,648 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* AFS filesystem directory editing + * + * Copyright (C) 2018 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include "internal.h" +#include "xdr_fs.h" + +/* + * Find a number of contiguous clear bits in a directory block bitmask. + * + * There are 64 slots, which means we can load the entire bitmap into a + * variable. The first bit doesn't count as it corresponds to the block header + * slot. nr_slots is between 1 and 9. + */ +static int afs_find_contig_bits(union afs_xdr_dir_block *block, unsigned int nr_slots) +{ + u64 bitmap; + u32 mask; + int bit, n; + + bitmap = (u64)block->hdr.bitmap[0] << 0 * 8; + bitmap |= (u64)block->hdr.bitmap[1] << 1 * 8; + bitmap |= (u64)block->hdr.bitmap[2] << 2 * 8; + bitmap |= (u64)block->hdr.bitmap[3] << 3 * 8; + bitmap |= (u64)block->hdr.bitmap[4] << 4 * 8; + bitmap |= (u64)block->hdr.bitmap[5] << 5 * 8; + bitmap |= (u64)block->hdr.bitmap[6] << 6 * 8; + bitmap |= (u64)block->hdr.bitmap[7] << 7 * 8; + bitmap >>= 1; /* The first entry is metadata */ + bit = 1; + mask = (1 << nr_slots) - 1; + + do { + if (sizeof(unsigned long) == 8) + n = ffz(bitmap); + else + n = ((u32)bitmap) != 0 ? + ffz((u32)bitmap) : + ffz((u32)(bitmap >> 32)) + 32; + bitmap >>= n; + bit += n; + + if ((bitmap & mask) == 0) { + if (bit > 64 - nr_slots) + return -1; + return bit; + } + + n = __ffs(bitmap); + bitmap >>= n; + bit += n; + } while (bitmap); + + return -1; +} + +/* + * Set a number of contiguous bits in the directory block bitmap. + */ +static void afs_set_contig_bits(union afs_xdr_dir_block *block, + int bit, unsigned int nr_slots) +{ + u64 mask; + + mask = (1 << nr_slots) - 1; + mask <<= bit; + + block->hdr.bitmap[0] |= (u8)(mask >> 0 * 8); + block->hdr.bitmap[1] |= (u8)(mask >> 1 * 8); + block->hdr.bitmap[2] |= (u8)(mask >> 2 * 8); + block->hdr.bitmap[3] |= (u8)(mask >> 3 * 8); + block->hdr.bitmap[4] |= (u8)(mask >> 4 * 8); + block->hdr.bitmap[5] |= (u8)(mask >> 5 * 8); + block->hdr.bitmap[6] |= (u8)(mask >> 6 * 8); + block->hdr.bitmap[7] |= (u8)(mask >> 7 * 8); +} + +/* + * Clear a number of contiguous bits in the directory block bitmap. + */ +static void afs_clear_contig_bits(union afs_xdr_dir_block *block, + int bit, unsigned int nr_slots) +{ + u64 mask; + + mask = (1 << nr_slots) - 1; + mask <<= bit; + + block->hdr.bitmap[0] &= ~(u8)(mask >> 0 * 8); + block->hdr.bitmap[1] &= ~(u8)(mask >> 1 * 8); + block->hdr.bitmap[2] &= ~(u8)(mask >> 2 * 8); + block->hdr.bitmap[3] &= ~(u8)(mask >> 3 * 8); + block->hdr.bitmap[4] &= ~(u8)(mask >> 4 * 8); + block->hdr.bitmap[5] &= ~(u8)(mask >> 5 * 8); + block->hdr.bitmap[6] &= ~(u8)(mask >> 6 * 8); + block->hdr.bitmap[7] &= ~(u8)(mask >> 7 * 8); +} + +/* + * Get a specific block, extending the directory storage to cover it as needed. + */ +static union afs_xdr_dir_block *afs_dir_get_block(struct afs_dir_iter *iter, size_t block) +{ + struct folio_queue *fq; + struct afs_vnode *dvnode = iter->dvnode; + struct folio *folio; + size_t blpos = block * AFS_DIR_BLOCK_SIZE; + size_t blend = (block + 1) * AFS_DIR_BLOCK_SIZE, fpos = iter->fpos; + int ret; + + if (dvnode->directory_size < blend) { + size_t cur_size = dvnode->directory_size; + + ret = netfs_alloc_folioq_buffer( + NULL, &dvnode->directory, &cur_size, blend, + mapping_gfp_mask(dvnode->netfs.inode.i_mapping)); + dvnode->directory_size = cur_size; + if (ret < 0) + goto fail; + } + + fq = iter->fq; + if (!fq) + fq = dvnode->directory; + + /* Search the folio queue for the folio containing the block... */ + for (; fq; fq = fq->next) { + for (int s = iter->fq_slot; s < folioq_count(fq); s++) { + size_t fsize = folioq_folio_size(fq, s); + + if (blend <= fpos + fsize) { + /* ... and then return the mapped block. */ + folio = folioq_folio(fq, s); + if (WARN_ON_ONCE(folio_pos(folio) != fpos)) + goto fail; + iter->fq = fq; + iter->fq_slot = s; + iter->fpos = fpos; + return kmap_local_folio(folio, blpos - fpos); + } + fpos += fsize; + } + iter->fq_slot = 0; + } + +fail: + iter->fq = NULL; + iter->fq_slot = 0; + afs_invalidate_dir(dvnode, afs_dir_invalid_edit_get_block); + return NULL; +} + +/* + * Scan a directory block looking for a dirent of the right name. + */ +static int afs_dir_scan_block(const union afs_xdr_dir_block *block, const struct qstr *name, + unsigned int blocknum) +{ + const union afs_xdr_dirent *de; + u64 bitmap; + int d, len, n; + + _enter(""); + + bitmap = (u64)block->hdr.bitmap[0] << 0 * 8; + bitmap |= (u64)block->hdr.bitmap[1] << 1 * 8; + bitmap |= (u64)block->hdr.bitmap[2] << 2 * 8; + bitmap |= (u64)block->hdr.bitmap[3] << 3 * 8; + bitmap |= (u64)block->hdr.bitmap[4] << 4 * 8; + bitmap |= (u64)block->hdr.bitmap[5] << 5 * 8; + bitmap |= (u64)block->hdr.bitmap[6] << 6 * 8; + bitmap |= (u64)block->hdr.bitmap[7] << 7 * 8; + + for (d = (blocknum == 0 ? AFS_DIR_RESV_BLOCKS0 : AFS_DIR_RESV_BLOCKS); + d < AFS_DIR_SLOTS_PER_BLOCK; + d++) { + if (!((bitmap >> d) & 1)) + continue; + de = &block->dirents[d]; + if (de->u.valid != 1) + continue; + + /* The block was NUL-terminated by afs_dir_check_page(). */ + len = strlen(de->u.name); + if (len == name->len && + memcmp(de->u.name, name->name, name->len) == 0) + return d; + + n = round_up(12 + len + 1 + 4, AFS_DIR_DIRENT_SIZE); + n /= AFS_DIR_DIRENT_SIZE; + d += n - 1; + } + + return -1; +} + +/* + * Initialise a new directory block. Note that block 0 is special and contains + * some extra metadata. + */ +static void afs_edit_init_block(union afs_xdr_dir_block *meta, + union afs_xdr_dir_block *block, int block_num) +{ + memset(block, 0, sizeof(*block)); + block->hdr.npages = htons(1); + block->hdr.magic = AFS_DIR_MAGIC; + block->hdr.bitmap[0] = 1; + + if (block_num == 0) { + block->hdr.bitmap[0] = 0xff; + block->hdr.bitmap[1] = 0x1f; + memset(block->meta.alloc_ctrs, + AFS_DIR_SLOTS_PER_BLOCK, + sizeof(block->meta.alloc_ctrs)); + meta->meta.alloc_ctrs[0] = + AFS_DIR_SLOTS_PER_BLOCK - AFS_DIR_RESV_BLOCKS0; + } + + if (block_num < AFS_DIR_BLOCKS_WITH_CTR) + meta->meta.alloc_ctrs[block_num] = + AFS_DIR_SLOTS_PER_BLOCK - AFS_DIR_RESV_BLOCKS; +} + +/* + * Edit a directory's file data to add a new directory entry. Doing this after + * create, mkdir, symlink, link or rename if the data version number is + * incremented by exactly one avoids the need to re-download the entire + * directory contents. + * + * The caller must hold the inode locked. + */ +void afs_edit_dir_add(struct afs_vnode *vnode, + const struct qstr *name, struct afs_fid *new_fid, + enum afs_edit_dir_reason why) +{ + union afs_xdr_dir_block *meta, *block; + union afs_xdr_dirent *de; + struct afs_dir_iter iter = { .dvnode = vnode }; + unsigned int nr_blocks, b, entry; + loff_t i_size; + int slot; + + _enter(",,{%d,%s},", name->len, name->name); + + i_size = i_size_read(&vnode->netfs.inode); + if (i_size > AFS_DIR_BLOCK_SIZE * AFS_DIR_MAX_BLOCKS || + (i_size & (AFS_DIR_BLOCK_SIZE - 1))) { + afs_invalidate_dir(vnode, afs_dir_invalid_edit_add_bad_size); + return; + } + + meta = afs_dir_get_block(&iter, 0); + if (!meta) + return; + + /* Work out how many slots we're going to need. */ + iter.nr_slots = afs_dir_calc_slots(name->len); + + if (i_size == 0) + goto new_directory; + nr_blocks = i_size / AFS_DIR_BLOCK_SIZE; + + /* Find a block that has sufficient slots available. Each folio + * contains two or more directory blocks. + */ + for (b = 0; b < nr_blocks + 1; b++) { + /* If the directory extended into a new folio, then we need to + * tack a new folio on the end. + */ + if (nr_blocks >= AFS_DIR_MAX_BLOCKS) + goto error_too_many_blocks; + + /* Lower dir blocks have a counter in the header we can check. */ + if (b < AFS_DIR_BLOCKS_WITH_CTR && + meta->meta.alloc_ctrs[b] < iter.nr_slots) + continue; + + block = afs_dir_get_block(&iter, b); + if (!block) + goto error; + + /* Abandon the edit if we got a callback break. */ + if (!test_bit(AFS_VNODE_DIR_VALID, &vnode->flags)) + goto already_invalidated; + + _debug("block %u: %2u %3u %u", + b, + (b < AFS_DIR_BLOCKS_WITH_CTR) ? meta->meta.alloc_ctrs[b] : 99, + ntohs(block->hdr.npages), + ntohs(block->hdr.magic)); + + /* Initialise the block if necessary. */ + if (b == nr_blocks) { + _debug("init %u", b); + afs_edit_init_block(meta, block, b); + afs_set_i_size(vnode, (b + 1) * AFS_DIR_BLOCK_SIZE); + } + + /* We need to try and find one or more consecutive slots to + * hold the entry. + */ + slot = afs_find_contig_bits(block, iter.nr_slots); + if (slot >= 0) { + _debug("slot %u", slot); + goto found_space; + } + + kunmap_local(block); + } + + /* There are no spare slots of sufficient size, yet the operation + * succeeded. Download the directory again. + */ + trace_afs_edit_dir(vnode, why, afs_edit_dir_create_nospc, 0, 0, 0, 0, name->name); + afs_invalidate_dir(vnode, afs_dir_invalid_edit_add_no_slots); + goto out_unmap; + +new_directory: + afs_edit_init_block(meta, meta, 0); + i_size = AFS_DIR_BLOCK_SIZE; + afs_set_i_size(vnode, i_size); + slot = AFS_DIR_RESV_BLOCKS0; + block = afs_dir_get_block(&iter, 0); + nr_blocks = 1; + b = 0; + +found_space: + /* Set the dirent slot. */ + trace_afs_edit_dir(vnode, why, afs_edit_dir_create, b, slot, + new_fid->vnode, new_fid->unique, name->name); + de = &block->dirents[slot]; + de->u.valid = 1; + de->u.unused[0] = 0; + de->u.hash_next = 0; // TODO: Really need to maintain this + de->u.vnode = htonl(new_fid->vnode); + de->u.unique = htonl(new_fid->unique); + memcpy(de->u.name, name->name, name->len + 1); + de->u.name[name->len] = 0; + + /* Adjust the bitmap. */ + afs_set_contig_bits(block, slot, iter.nr_slots); + + /* Adjust the allocation counter. */ + if (b < AFS_DIR_BLOCKS_WITH_CTR) + meta->meta.alloc_ctrs[b] -= iter.nr_slots; + + /* Adjust the hash chain. */ + entry = b * AFS_DIR_SLOTS_PER_BLOCK + slot; + iter.bucket = afs_dir_hash_name(name); + de->u.hash_next = meta->meta.hashtable[iter.bucket]; + meta->meta.hashtable[iter.bucket] = htons(entry); + kunmap_local(block); + + inode_inc_iversion_raw(&vnode->netfs.inode); + afs_stat_v(vnode, n_dir_cr); + _debug("Insert %s in %u[%u]", name->name, b, slot); + + netfs_single_mark_inode_dirty(&vnode->netfs.inode); + +out_unmap: + kunmap_local(meta); + _leave(""); + return; + +already_invalidated: + trace_afs_edit_dir(vnode, why, afs_edit_dir_create_inval, 0, 0, 0, 0, name->name); + kunmap_local(block); + goto out_unmap; + +error_too_many_blocks: + afs_invalidate_dir(vnode, afs_dir_invalid_edit_add_too_many_blocks); +error: + trace_afs_edit_dir(vnode, why, afs_edit_dir_create_error, 0, 0, 0, 0, name->name); + goto out_unmap; +} + +/* + * Edit a directory's file data to remove a new directory entry. Doing this + * after unlink, rmdir or rename if the data version number is incremented by + * exactly one avoids the need to re-download the entire directory contents. + * + * The caller must hold the inode locked. + */ +void afs_edit_dir_remove(struct afs_vnode *vnode, + const struct qstr *name, enum afs_edit_dir_reason why) +{ + union afs_xdr_dir_block *meta, *block, *pblock; + union afs_xdr_dirent *de, *pde; + struct afs_dir_iter iter = { .dvnode = vnode }; + struct afs_fid fid; + unsigned int b, slot, entry; + loff_t i_size; + __be16 next; + int found; + + _enter(",,{%d,%s},", name->len, name->name); + + i_size = i_size_read(&vnode->netfs.inode); + if (i_size < AFS_DIR_BLOCK_SIZE || + i_size > AFS_DIR_BLOCK_SIZE * AFS_DIR_MAX_BLOCKS || + (i_size & (AFS_DIR_BLOCK_SIZE - 1))) { + afs_invalidate_dir(vnode, afs_dir_invalid_edit_rem_bad_size); + return; + } + + if (!afs_dir_init_iter(&iter, name)) + return; + + meta = afs_dir_get_block(&iter, 0); + if (!meta) + return; + + /* Find the entry in the blob. */ + found = afs_dir_search_bucket(&iter, name, &fid); + if (found < 0) { + /* Didn't find the dirent to clobber. Re-download. */ + trace_afs_edit_dir(vnode, why, afs_edit_dir_delete_noent, + 0, 0, 0, 0, name->name); + afs_invalidate_dir(vnode, afs_dir_invalid_edit_rem_wrong_name); + goto out_unmap; + } + + entry = found; + b = entry / AFS_DIR_SLOTS_PER_BLOCK; + slot = entry % AFS_DIR_SLOTS_PER_BLOCK; + + block = afs_dir_find_block(&iter, b); + if (!block) + goto error; + if (!test_bit(AFS_VNODE_DIR_VALID, &vnode->flags)) + goto already_invalidated; + + /* Check and clear the entry. */ + de = &block->dirents[slot]; + if (de->u.valid != 1) + goto error_unmap; + + trace_afs_edit_dir(vnode, why, afs_edit_dir_delete, b, slot, + ntohl(de->u.vnode), ntohl(de->u.unique), + name->name); + + /* Adjust the bitmap. */ + afs_clear_contig_bits(block, slot, iter.nr_slots); + + /* Adjust the allocation counter. */ + if (b < AFS_DIR_BLOCKS_WITH_CTR) + meta->meta.alloc_ctrs[b] += iter.nr_slots; + + /* Clear the constituent entries. */ + next = de->u.hash_next; + memset(de, 0, sizeof(*de) * iter.nr_slots); + kunmap_local(block); + + /* Adjust the hash chain: if iter->prev_entry is 0, the hashtable head + * index is previous; otherwise it's slot number of the previous entry. + */ + if (!iter.prev_entry) { + __be16 prev_next = meta->meta.hashtable[iter.bucket]; + + if (unlikely(prev_next != htons(entry))) { + pr_warn("%llx:%llx:%x: not head of chain b=%x p=%x,%x e=%x %*s", + vnode->fid.vid, vnode->fid.vnode, vnode->fid.unique, + iter.bucket, iter.prev_entry, prev_next, entry, + name->len, name->name); + goto error; + } + meta->meta.hashtable[iter.bucket] = next; + } else { + unsigned int pb = iter.prev_entry / AFS_DIR_SLOTS_PER_BLOCK; + unsigned int ps = iter.prev_entry % AFS_DIR_SLOTS_PER_BLOCK; + __be16 prev_next; + + pblock = afs_dir_find_block(&iter, pb); + if (!pblock) + goto error; + pde = &pblock->dirents[ps]; + prev_next = pde->u.hash_next; + if (prev_next != htons(entry)) { + kunmap_local(pblock); + pr_warn("%llx:%llx:%x: not prev in chain b=%x p=%x,%x e=%x %*s", + vnode->fid.vid, vnode->fid.vnode, vnode->fid.unique, + iter.bucket, iter.prev_entry, prev_next, entry, + name->len, name->name); + goto error; + } + pde->u.hash_next = next; + kunmap_local(pblock); + } + + netfs_single_mark_inode_dirty(&vnode->netfs.inode); + + inode_set_iversion_raw(&vnode->netfs.inode, vnode->status.data_version); + afs_stat_v(vnode, n_dir_rm); + _debug("Remove %s from %u[%u]", name->name, b, slot); + +out_unmap: + kunmap_local(meta); + _leave(""); + return; + +already_invalidated: + kunmap_local(block); + trace_afs_edit_dir(vnode, why, afs_edit_dir_delete_inval, + 0, 0, 0, 0, name->name); + goto out_unmap; + +error_unmap: + kunmap_local(block); +error: + trace_afs_edit_dir(vnode, why, afs_edit_dir_delete_error, + 0, 0, 0, 0, name->name); + goto out_unmap; +} + +/* + * Edit an entry in a directory to update the vnode it refers to. This is also + * used to update the ".." entry in a directory. + */ +void afs_edit_dir_update(struct afs_vnode *vnode, const struct qstr *name, + struct afs_vnode *new_dvnode, enum afs_edit_dir_reason why) +{ + union afs_xdr_dir_block *block; + union afs_xdr_dirent *de; + struct afs_dir_iter iter = { .dvnode = vnode }; + unsigned int nr_blocks, b; + loff_t i_size; + int slot; + + _enter(""); + + i_size = i_size_read(&vnode->netfs.inode); + if (i_size < AFS_DIR_BLOCK_SIZE) { + afs_invalidate_dir(vnode, afs_dir_invalid_edit_upd_bad_size); + return; + } + + nr_blocks = i_size / AFS_DIR_BLOCK_SIZE; + + /* Find a block that has sufficient slots available. Each folio + * contains two or more directory blocks. + */ + for (b = 0; b < nr_blocks; b++) { + block = afs_dir_get_block(&iter, b); + if (!block) + goto error; + + /* Abandon the edit if we got a callback break. */ + if (!test_bit(AFS_VNODE_DIR_VALID, &vnode->flags)) + goto already_invalidated; + + slot = afs_dir_scan_block(block, name, b); + if (slot >= 0) + goto found_dirent; + + kunmap_local(block); + } + + /* Didn't find the dirent to clobber. Download the directory again. */ + trace_afs_edit_dir(vnode, why, afs_edit_dir_update_nodd, + 0, 0, 0, 0, name->name); + afs_invalidate_dir(vnode, afs_dir_invalid_edit_upd_no_dd); + goto out; + +found_dirent: + de = &block->dirents[slot]; + de->u.vnode = htonl(new_dvnode->fid.vnode); + de->u.unique = htonl(new_dvnode->fid.unique); + + trace_afs_edit_dir(vnode, why, afs_edit_dir_update_dd, b, slot, + ntohl(de->u.vnode), ntohl(de->u.unique), name->name); + + kunmap_local(block); + netfs_single_mark_inode_dirty(&vnode->netfs.inode); + inode_set_iversion_raw(&vnode->netfs.inode, vnode->status.data_version); + +out: + _leave(""); + return; + +already_invalidated: + kunmap_local(block); + trace_afs_edit_dir(vnode, why, afs_edit_dir_update_inval, + 0, 0, 0, 0, name->name); + goto out; + +error: + trace_afs_edit_dir(vnode, why, afs_edit_dir_update_error, + 0, 0, 0, 0, name->name); + goto out; +} + +/* + * Initialise a new directory. We need to fill in the "." and ".." entries. + */ +void afs_mkdir_init_dir(struct afs_vnode *dvnode, struct afs_vnode *parent_dvnode) +{ + union afs_xdr_dir_block *meta; + struct afs_dir_iter iter = { .dvnode = dvnode }; + union afs_xdr_dirent *de; + unsigned int slot = AFS_DIR_RESV_BLOCKS0; + loff_t i_size; + + i_size = i_size_read(&dvnode->netfs.inode); + if (i_size != AFS_DIR_BLOCK_SIZE) { + afs_invalidate_dir(dvnode, afs_dir_invalid_edit_add_bad_size); + return; + } + + meta = afs_dir_get_block(&iter, 0); + if (!meta) + return; + + afs_edit_init_block(meta, meta, 0); + + de = &meta->dirents[slot]; + de->u.valid = 1; + de->u.vnode = htonl(dvnode->fid.vnode); + de->u.unique = htonl(dvnode->fid.unique); + memcpy(de->u.name, ".", 2); + trace_afs_edit_dir(dvnode, afs_edit_dir_for_mkdir, afs_edit_dir_mkdir, 0, slot, + dvnode->fid.vnode, dvnode->fid.unique, "."); + slot++; + + de = &meta->dirents[slot]; + de->u.valid = 1; + de->u.vnode = htonl(parent_dvnode->fid.vnode); + de->u.unique = htonl(parent_dvnode->fid.unique); + memcpy(de->u.name, "..", 3); + trace_afs_edit_dir(dvnode, afs_edit_dir_for_mkdir, afs_edit_dir_mkdir, 0, slot, + parent_dvnode->fid.vnode, parent_dvnode->fid.unique, ".."); + + afs_set_contig_bits(meta, AFS_DIR_RESV_BLOCKS0, 2); + meta->meta.alloc_ctrs[0] -= 2; + kunmap_local(meta); + + netfs_single_mark_inode_dirty(&dvnode->netfs.inode); + set_bit(AFS_VNODE_DIR_VALID, &dvnode->flags); + set_bit(AFS_VNODE_DIR_READ, &dvnode->flags); +} diff --git a/fs/afs/dir_search.c b/fs/afs/dir_search.c new file mode 100644 index 000000000..104411c06 --- /dev/null +++ b/fs/afs/dir_search.c @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Search a directory's hash table. + * + * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + * + * https://tools.ietf.org/html/draft-keiser-afs3-directory-object-00 + */ + +#include +#include +#include +#include +#include "internal.h" +#include "afs_fs.h" +#include "xdr_fs.h" + +/* + * Calculate the name hash. + */ +unsigned int afs_dir_hash_name(const struct qstr *name) +{ + const unsigned char *p = name->name; + unsigned int hash = 0, i; + int bucket; + + for (i = 0; i < name->len; i++) + hash = (hash * 173) + p[i]; + bucket = hash & (AFS_DIR_HASHTBL_SIZE - 1); + if (hash > INT_MAX) { + bucket = AFS_DIR_HASHTBL_SIZE - bucket; + bucket &= (AFS_DIR_HASHTBL_SIZE - 1); + } + return bucket; +} + +/* + * Reset a directory iterator. + */ +static bool afs_dir_reset_iter(struct afs_dir_iter *iter) +{ + unsigned long long i_size = i_size_read(&iter->dvnode->netfs.inode); + unsigned int nblocks; + + /* Work out the maximum number of steps we can take. */ + nblocks = umin(i_size / AFS_DIR_BLOCK_SIZE, AFS_DIR_MAX_BLOCKS); + if (!nblocks) + return false; + iter->loop_check = nblocks * (AFS_DIR_SLOTS_PER_BLOCK - AFS_DIR_RESV_BLOCKS); + iter->prev_entry = 0; /* Hash head is previous */ + return true; +} + +/* + * Initialise a directory iterator for looking up a name. + */ +bool afs_dir_init_iter(struct afs_dir_iter *iter, const struct qstr *name) +{ + iter->nr_slots = afs_dir_calc_slots(name->len); + iter->bucket = afs_dir_hash_name(name); + return afs_dir_reset_iter(iter); +} + +/* + * Get a specific block. + */ +union afs_xdr_dir_block *afs_dir_find_block(struct afs_dir_iter *iter, size_t block) +{ + struct folio_queue *fq = iter->fq; + struct afs_vnode *dvnode = iter->dvnode; + struct folio *folio; + size_t blpos = block * AFS_DIR_BLOCK_SIZE; + size_t blend = (block + 1) * AFS_DIR_BLOCK_SIZE, fpos = iter->fpos; + int slot = iter->fq_slot; + + _enter("%zx,%d", block, slot); + + if (iter->block) { + kunmap_local(iter->block); + iter->block = NULL; + } + + if (dvnode->directory_size < blend) + goto fail; + + if (!fq || blpos < fpos) { + fq = dvnode->directory; + slot = 0; + fpos = 0; + } + + /* Search the folio queue for the folio containing the block... */ + for (; fq; fq = fq->next) { + for (; slot < folioq_count(fq); slot++) { + size_t fsize = folioq_folio_size(fq, slot); + + if (blend <= fpos + fsize) { + /* ... and then return the mapped block. */ + folio = folioq_folio(fq, slot); + if (WARN_ON_ONCE(folio_pos(folio) != fpos)) + goto fail; + iter->fq = fq; + iter->fq_slot = slot; + iter->fpos = fpos; + iter->block = kmap_local_folio(folio, blpos - fpos); + return iter->block; + } + fpos += fsize; + } + slot = 0; + } + +fail: + iter->fq = NULL; + iter->fq_slot = 0; + afs_invalidate_dir(dvnode, afs_dir_invalid_edit_get_block); + return NULL; +} + +/* + * Search through a directory bucket. + */ +int afs_dir_search_bucket(struct afs_dir_iter *iter, const struct qstr *name, + struct afs_fid *_fid) +{ + const union afs_xdr_dir_block *meta; + unsigned int entry; + int ret = -ESTALE; + + meta = afs_dir_find_block(iter, 0); + if (!meta) + return -ESTALE; + + entry = ntohs(meta->meta.hashtable[iter->bucket & (AFS_DIR_HASHTBL_SIZE - 1)]); + _enter("%x,%x", iter->bucket, entry); + + while (entry) { + const union afs_xdr_dir_block *block; + const union afs_xdr_dirent *dire; + unsigned int blnum = entry / AFS_DIR_SLOTS_PER_BLOCK; + unsigned int slot = entry % AFS_DIR_SLOTS_PER_BLOCK; + unsigned int resv = (blnum == 0 ? AFS_DIR_RESV_BLOCKS0 : AFS_DIR_RESV_BLOCKS); + + _debug("search %x", entry); + + if (slot < resv) { + kdebug("slot out of range h=%x rs=%2x sl=%2x-%2x", + iter->bucket, resv, slot, slot + iter->nr_slots - 1); + goto bad; + } + + block = afs_dir_find_block(iter, blnum); + if (!block) + goto bad; + dire = &block->dirents[slot]; + + if (slot + iter->nr_slots <= AFS_DIR_SLOTS_PER_BLOCK && + memcmp(dire->u.name, name->name, name->len) == 0 && + dire->u.name[name->len] == '\0') { + _fid->vnode = ntohl(dire->u.vnode); + _fid->unique = ntohl(dire->u.unique); + ret = entry; + goto found; + } + + iter->prev_entry = entry; + entry = ntohs(dire->u.hash_next); + if (!--iter->loop_check) { + kdebug("dir chain loop h=%x", iter->bucket); + goto bad; + } + } + + ret = -ENOENT; +found: + if (iter->block) { + kunmap_local(iter->block); + iter->block = NULL; + } + +bad: + if (ret == -ESTALE) + afs_invalidate_dir(iter->dvnode, afs_dir_invalid_iter_stale); + _leave(" = %d", ret); + return ret; +} + +/* + * Search the appropriate hash chain in the contents of an AFS directory. + */ +int afs_dir_search(struct afs_vnode *dvnode, const struct qstr *name, + struct afs_fid *_fid, afs_dataversion_t *_dir_version) +{ + struct afs_dir_iter iter = { .dvnode = dvnode, }; + int ret, retry_limit = 3; + + _enter("{%llu},,,", dvnode->netfs.inode.i_ino); + + if (!afs_dir_init_iter(&iter, name)) + return -ENOENT; + do { + if (--retry_limit < 0) { + pr_warn("afs_read_dir(): Too many retries\n"); + ret = -ESTALE; + break; + } + ret = afs_read_dir(dvnode, NULL); + if (ret < 0) { + if (ret != -ESTALE) + break; + if (test_bit(AFS_VNODE_DELETED, &dvnode->flags)) { + ret = -ESTALE; + break; + } + continue; + } + *_dir_version = inode_peek_iversion_raw(&dvnode->netfs.inode); + + ret = afs_dir_search_bucket(&iter, name, _fid); + up_read(&dvnode->validate_lock); + if (ret == -ESTALE) + afs_dir_reset_iter(&iter); + } while (ret == -ESTALE); + + _leave(" = %d", ret); + return ret; +} diff --git a/fs/afs/dir_silly.c b/fs/afs/dir_silly.c new file mode 100644 index 000000000..982bb6ec1 --- /dev/null +++ b/fs/afs/dir_silly.c @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* AFS silly rename handling + * + * Copyright (C) 2019 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + * - Derived from NFS's sillyrename. + */ + +#include +#include +#include +#include +#include "internal.h" + +static void afs_silly_rename_success(struct afs_operation *op) +{ + _enter("op=%08x", op->debug_id); + + afs_check_dir_conflict(op, &op->file[0]); + afs_vnode_commit_status(op, &op->file[0]); +} + +static void afs_silly_rename_edit_dir(struct afs_operation *op) +{ + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_vnode *dvnode = dvp->vnode; + struct afs_vnode *vnode = AFS_FS_I(d_inode(op->dentry)); + struct dentry *old = op->dentry; + struct dentry *new = op->dentry_2; + + spin_lock(&old->d_lock); + old->d_flags |= DCACHE_NFSFS_RENAMED; + spin_unlock(&old->d_lock); + if (dvnode->silly_key != op->key) { + key_put(dvnode->silly_key); + dvnode->silly_key = key_get(op->key); + } + + down_write(&dvnode->validate_lock); + if (test_bit(AFS_VNODE_DIR_VALID, &dvnode->flags) && + dvnode->status.data_version == dvp->dv_before + dvp->dv_delta) { + afs_edit_dir_remove(dvnode, &old->d_name, + afs_edit_dir_for_silly_0); + afs_edit_dir_add(dvnode, &new->d_name, + &vnode->fid, afs_edit_dir_for_silly_1); + } + up_write(&dvnode->validate_lock); +} + +static const struct afs_operation_ops afs_silly_rename_operation = { + .issue_afs_rpc = afs_fs_rename, + .issue_yfs_rpc = yfs_fs_rename, + .success = afs_silly_rename_success, + .edit_dir = afs_silly_rename_edit_dir, +}; + +/* + * Actually perform the silly rename step. + */ +static int afs_do_silly_rename(struct afs_vnode *dvnode, struct afs_vnode *vnode, + struct dentry *old, struct dentry *new, + struct key *key) +{ + struct afs_operation *op; + + _enter("%pd,%pd", old, new); + + op = afs_alloc_operation(key, dvnode->volume); + if (IS_ERR(op)) + return PTR_ERR(op); + + op->more_files = kvzalloc_objs(struct afs_vnode_param, 2); + if (!op->more_files) { + afs_put_operation(op); + return -ENOMEM; + } + + afs_op_set_vnode(op, 0, dvnode); + afs_op_set_vnode(op, 1, dvnode); + op->file[0].dv_delta = 1; + op->file[1].dv_delta = 1; + op->file[0].modification = true; + op->file[1].modification = true; + op->file[0].update_ctime = true; + op->file[1].update_ctime = true; + op->more_files[0].vnode = AFS_FS_I(d_inode(old)); + op->more_files[0].speculative = true; + op->more_files[1].vnode = AFS_FS_I(d_inode(new)); + op->more_files[1].speculative = true; + op->nr_files = 4; + + op->dentry = old; + op->dentry_2 = new; + op->ops = &afs_silly_rename_operation; + + trace_afs_silly_rename(vnode, false); + return afs_do_sync_operation(op); +} + +/* + * Perform silly-rename of a dentry. + * + * AFS is stateless and the server doesn't know when the client is holding a + * file open. To prevent application problems when a file is unlinked while + * it's still open, the client performs a "silly-rename". That is, it renames + * the file to a hidden file in the same directory, and only performs the + * unlink once the last reference to it is put. + * + * The final cleanup is done during dentry_iput. + */ +int afs_sillyrename(struct afs_vnode *dvnode, struct afs_vnode *vnode, + struct dentry *dentry, struct key *key) +{ + static unsigned int sillycounter; + struct dentry *sdentry = NULL; + unsigned char silly[16]; + int ret = -EBUSY; + + _enter(""); + + /* We don't allow a dentry to be silly-renamed twice. */ + if (dentry->d_flags & DCACHE_NFSFS_RENAMED) + return -EBUSY; + + sdentry = NULL; + do { + dput(sdentry); + sillycounter++; + + /* Create a silly name. Note that the ".__afs" prefix is + * understood by the salvager and must not be changed. + */ + scnprintf(silly, sizeof(silly), ".__afs%04X", sillycounter); + sdentry = lookup_noperm(&QSTR(silly), dentry->d_parent); + + /* N.B. Better to return EBUSY here ... it could be dangerous + * to delete the file while it's in use. + */ + if (IS_ERR(sdentry)) + goto out; + } while (!d_is_negative(sdentry)); + + ihold(&vnode->netfs.inode); + + ret = afs_do_silly_rename(dvnode, vnode, dentry, sdentry, key); + switch (ret) { + case 0: + /* The rename succeeded. */ + set_bit(AFS_VNODE_SILLY_DELETED, &vnode->flags); + d_move(dentry, sdentry); + break; + case -ERESTARTSYS: + /* The result of the rename is unknown. Play it safe by forcing + * a new lookup. + */ + d_drop(dentry); + d_drop(sdentry); + } + + iput(&vnode->netfs.inode); + dput(sdentry); +out: + _leave(" = %d", ret); + return ret; +} + +static void afs_silly_unlink_success(struct afs_operation *op) +{ + _enter("op=%08x", op->debug_id); + afs_check_dir_conflict(op, &op->file[0]); + afs_vnode_commit_status(op, &op->file[0]); + afs_vnode_commit_status(op, &op->file[1]); + afs_update_dentry_version(op, &op->file[0], op->dentry); +} + +static void afs_silly_unlink_edit_dir(struct afs_operation *op) +{ + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_vnode *dvnode = dvp->vnode; + + _enter("op=%08x", op->debug_id); + down_write(&dvnode->validate_lock); + if (test_bit(AFS_VNODE_DIR_VALID, &dvnode->flags) && + dvnode->status.data_version == dvp->dv_before + dvp->dv_delta) + afs_edit_dir_remove(dvnode, &op->dentry->d_name, + afs_edit_dir_for_unlink); + up_write(&dvnode->validate_lock); +} + +static const struct afs_operation_ops afs_silly_unlink_operation = { + .issue_afs_rpc = afs_fs_remove_file, + .issue_yfs_rpc = yfs_fs_remove_file, + .success = afs_silly_unlink_success, + .aborted = afs_check_for_remote_deletion, + .edit_dir = afs_silly_unlink_edit_dir, +}; + +/* + * Tell the server to remove a sillyrename file. + */ +static int afs_do_silly_unlink(struct afs_vnode *dvnode, struct afs_vnode *vnode, + struct dentry *dentry, struct key *key) +{ + struct afs_operation *op; + + _enter(""); + + op = afs_alloc_operation(NULL, dvnode->volume); + if (IS_ERR(op)) + return PTR_ERR(op); + + afs_op_set_vnode(op, 0, dvnode); + afs_op_set_vnode(op, 1, vnode); + op->file[0].dv_delta = 1; + op->file[0].modification = true; + op->file[0].update_ctime = true; + op->file[1].op_unlinked = true; + op->file[1].update_ctime = true; + + op->dentry = dentry; + op->ops = &afs_silly_unlink_operation; + + trace_afs_silly_rename(vnode, true); + afs_begin_vnode_operation(op); + afs_wait_for_operation(op); + + /* If there was a conflict with a third party, check the status of the + * unlinked vnode. + */ + if (op->cumul_error.error == 0 && (op->flags & AFS_OPERATION_DIR_CONFLICT)) { + op->file[1].update_ctime = false; + op->fetch_status.which = 1; + op->ops = &afs_fetch_status_operation; + afs_begin_vnode_operation(op); + afs_wait_for_operation(op); + } + + return afs_put_operation(op); +} + +/* + * Remove sillyrename file on iput. + */ +int afs_silly_iput(struct dentry *dentry, struct inode *inode) +{ + struct afs_vnode *dvnode = AFS_FS_I(d_inode(dentry->d_parent)); + struct afs_vnode *vnode = AFS_FS_I(inode); + struct dentry *alias; + int ret; + + _enter("%p{%pd},%llx", dentry, dentry, vnode->fid.vnode); + + down_read(&dvnode->rmdir_lock); + + alias = d_alloc_parallel(dentry->d_parent, &dentry->d_name); + if (IS_ERR(alias)) { + up_read(&dvnode->rmdir_lock); + return 0; + } + + if (!d_in_lookup(alias)) { + /* We raced with lookup... See if we need to transfer the + * sillyrename information to the aliased dentry. + */ + ret = 0; + spin_lock(&alias->d_lock); + if (d_really_is_positive(alias) && + !(alias->d_flags & DCACHE_NFSFS_RENAMED)) { + alias->d_flags |= DCACHE_NFSFS_RENAMED; + ret = 1; + } + spin_unlock(&alias->d_lock); + up_read(&dvnode->rmdir_lock); + dput(alias); + return ret; + } + + /* Stop lock-release from complaining. */ + spin_lock(&vnode->lock); + vnode->lock_state = AFS_VNODE_LOCK_DELETED; + trace_afs_flock_ev(vnode, NULL, afs_flock_silly_delete, 0); + spin_unlock(&vnode->lock); + + afs_do_silly_unlink(dvnode, vnode, dentry, dvnode->silly_key); + up_read(&dvnode->rmdir_lock); + d_lookup_done(alias); + dput(alias); + return 1; +} diff --git a/fs/afs/dynroot.c b/fs/afs/dynroot.c new file mode 100644 index 000000000..6e3c8c691 --- /dev/null +++ b/fs/afs/dynroot.c @@ -0,0 +1,406 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* AFS dynamic root handling + * + * Copyright (C) 2018 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include "internal.h" + +#define AFS_MIN_DYNROOT_CELL_INO 4 /* Allow for ., .., @cell, .@cell */ +#define AFS_MAX_DYNROOT_CELL_INO ((unsigned int)INT_MAX) + +static struct dentry *afs_lookup_atcell(struct inode *dir, struct dentry *dentry, ino_t ino); + +/* + * iget5() comparator for inode created by autocell operations + */ +static int afs_iget5_pseudo_test(struct inode *inode, void *opaque) +{ + struct afs_fid *fid = opaque; + + return inode->i_ino == fid->vnode; +} + +/* + * iget5() inode initialiser + */ +static int afs_iget5_pseudo_set(struct inode *inode, void *opaque) +{ + struct afs_super_info *as = AFS_FS_S(inode->i_sb); + struct afs_vnode *vnode = AFS_FS_I(inode); + struct afs_fid *fid = opaque; + + vnode->volume = as->volume; + vnode->fid = *fid; + inode->i_ino = fid->vnode; + inode->i_generation = fid->unique; + return 0; +} + +/* + * Create an inode for an autocell dynamic automount dir. + */ +static struct inode *afs_iget_pseudo_dir(struct super_block *sb, ino_t ino) +{ + struct afs_vnode *vnode; + struct inode *inode; + struct afs_fid fid = { .vnode = ino, .unique = 1, }; + + _enter(""); + + inode = iget5_locked(sb, fid.vnode, + afs_iget5_pseudo_test, afs_iget5_pseudo_set, &fid); + if (!inode) { + _leave(" = -ENOMEM"); + return ERR_PTR(-ENOMEM); + } + + _debug("GOT INODE %p { ino=%llu, vl=%llx, vn=%llx, u=%x }", + inode, inode->i_ino, fid.vid, fid.vnode, fid.unique); + + vnode = AFS_FS_I(inode); + + if (inode_state_read_once(inode) & I_NEW) { + netfs_inode_init(&vnode->netfs, NULL, false); + simple_inode_init_ts(inode); + set_nlink(inode, 2); + inode->i_size = 0; + inode->i_mode = S_IFDIR | 0555; + inode->i_op = &afs_autocell_inode_operations; + inode->i_uid = GLOBAL_ROOT_UID; + inode->i_gid = GLOBAL_ROOT_GID; + inode->i_blocks = 0; + inode->i_generation = 0; + inode->i_flags |= S_AUTOMOUNT | S_NOATIME; + + set_bit(AFS_VNODE_PSEUDODIR, &vnode->flags); + set_bit(AFS_VNODE_MOUNTPOINT, &vnode->flags); + + unlock_new_inode(inode); + } + _leave(" = %p", inode); + return inode; +} + +/* + * Try to automount the mountpoint with pseudo directory, if the autocell + * option is set. + */ +static struct dentry *afs_dynroot_lookup_cell(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + struct afs_cell *cell = NULL; + struct afs_net *net = afs_d2net(dentry); + struct inode *inode = NULL; + const char *name = dentry->d_name.name; + size_t len = dentry->d_name.len; + bool dotted = false; + int ret = -ENOENT; + + /* Names prefixed with a dot are R/W mounts. */ + if (name[0] == '.') { + name++; + len--; + dotted = true; + } + + cell = afs_lookup_cell(net, name, len, NULL, + AFS_LOOKUP_CELL_DYNROOT, + afs_cell_trace_use_lookup_dynroot); + if (IS_ERR(cell)) { + ret = PTR_ERR(cell); + goto out_no_cell; + } + + inode = afs_iget_pseudo_dir(dir->i_sb, cell->dynroot_ino * 2 + dotted); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + goto out; + } + + dentry->d_fsdata = cell; + return d_splice_alias(inode, dentry); + +out: + afs_unuse_cell(cell, afs_cell_trace_unuse_lookup_dynroot); +out_no_cell: + if (!inode) + return d_splice_alias(inode, dentry); + return ret == -ENOENT ? NULL : ERR_PTR(ret); +} + +/* + * Look up an entry in a dynroot directory. + */ +static struct dentry *afs_dynroot_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + _enter("%pd", dentry); + + if (flags & LOOKUP_CREATE) + return ERR_PTR(-EOPNOTSUPP); + + if (dentry->d_name.len >= AFSNAMEMAX) { + _leave(" = -ENAMETOOLONG"); + return ERR_PTR(-ENAMETOOLONG); + } + + if (dentry->d_name.len == 5 && + memcmp(dentry->d_name.name, "@cell", 5) == 0) + return afs_lookup_atcell(dir, dentry, 2); + + if (dentry->d_name.len == 6 && + memcmp(dentry->d_name.name, ".@cell", 6) == 0) + return afs_lookup_atcell(dir, dentry, 3); + + return afs_dynroot_lookup_cell(dir, dentry, flags); +} + +const struct inode_operations afs_dynroot_inode_operations = { + .lookup = afs_dynroot_lookup, +}; + +static void afs_dynroot_d_release(struct dentry *dentry) +{ + struct afs_cell *cell = dentry->d_fsdata; + + afs_unuse_cell(cell, afs_cell_trace_unuse_dynroot_mntpt); +} + +/* + * Keep @cell symlink dentries around, but only keep cell autodirs when they're + * being used. + */ +static int afs_dynroot_delete_dentry(const struct dentry *dentry) +{ + const struct qstr *name = &dentry->d_name; + + if (name->len == 5 && memcmp(name->name, "@cell", 5) == 0) + return 0; + if (name->len == 6 && memcmp(name->name, ".@cell", 6) == 0) + return 0; + return 1; +} + +const struct dentry_operations afs_dynroot_dentry_operations = { + .d_delete = afs_dynroot_delete_dentry, + .d_release = afs_dynroot_d_release, + .d_automount = afs_d_automount, +}; + +static void afs_atcell_delayed_put_cell(void *arg) +{ + struct afs_cell *cell = arg; + + afs_put_cell(cell, afs_cell_trace_put_atcell); +} + +/* + * Read @cell or .@cell symlinks. + */ +static const char *afs_atcell_get_link(struct dentry *dentry, struct inode *inode, + struct delayed_call *done) +{ + struct afs_vnode *vnode = AFS_FS_I(inode); + struct afs_cell *cell; + struct afs_net *net = afs_i2net(inode); + const char *name; + bool dotted = vnode->fid.vnode == 3; + + if (!rcu_access_pointer(net->ws_cell)) + return ERR_PTR(-ENOENT); + + if (!dentry) { + /* We're in RCU-pathwalk. */ + cell = rcu_dereference(net->ws_cell); + if (dotted) + name = cell->name - 1; + else + name = cell->name; + /* Shouldn't need to set a delayed call. */ + return name; + } + + down_read(&net->cells_lock); + + cell = rcu_dereference_protected(net->ws_cell, lockdep_is_held(&net->cells_lock)); + if (dotted) + name = cell->name - 1; + else + name = cell->name; + afs_get_cell(cell, afs_cell_trace_get_atcell); + set_delayed_call(done, afs_atcell_delayed_put_cell, cell); + + up_read(&net->cells_lock); + return name; +} + +static const struct inode_operations afs_atcell_inode_operations = { + .get_link = afs_atcell_get_link, +}; + +/* + * Create an inode for the @cell or .@cell symlinks. + */ +static struct dentry *afs_lookup_atcell(struct inode *dir, struct dentry *dentry, ino_t ino) +{ + struct afs_vnode *vnode; + struct inode *inode; + struct afs_fid fid = { .vnode = ino, .unique = 1, }; + + inode = iget5_locked(dir->i_sb, fid.vnode, + afs_iget5_pseudo_test, afs_iget5_pseudo_set, &fid); + if (!inode) + return ERR_PTR(-ENOMEM); + + vnode = AFS_FS_I(inode); + + if (inode_state_read_once(inode) & I_NEW) { + netfs_inode_init(&vnode->netfs, NULL, false); + simple_inode_init_ts(inode); + set_nlink(inode, 1); + inode->i_size = 0; + inode->i_mode = S_IFLNK | 0555; + inode->i_op = &afs_atcell_inode_operations; + inode->i_uid = GLOBAL_ROOT_UID; + inode->i_gid = GLOBAL_ROOT_GID; + inode->i_blocks = 0; + inode->i_generation = 0; + inode->i_flags |= S_NOATIME; + + unlock_new_inode(inode); + } + return d_splice_alias(inode, dentry); +} + +/* + * Transcribe the cell database into readdir content under net->cells_lock. + * Each cell produces two entries, one prefixed with a dot and one not. + */ +static int afs_dynroot_readdir_cells(struct afs_net *net, struct dir_context *ctx) +{ + const struct afs_cell *cell; + loff_t newpos; + + _enter("%llu", ctx->pos); + + for (;;) { + unsigned int ix = ctx->pos >> 1; + + cell = idr_get_next(&net->cells_dyn_ino, &ix); + if (!cell) + return 0; + if (READ_ONCE(cell->state) == AFS_CELL_REMOVING || + READ_ONCE(cell->state) == AFS_CELL_DEAD) { + ctx->pos += 2; + ctx->pos &= ~1; + continue; + } + + newpos = ix << 1; + if (newpos > ctx->pos) + ctx->pos = newpos; + + _debug("pos %llu -> cell %u", ctx->pos, cell->dynroot_ino); + + if ((ctx->pos & 1) == 0) { + if (!dir_emit(ctx, cell->name, cell->name_len, + cell->dynroot_ino, DT_DIR)) + return 0; + ctx->pos++; + } + if ((ctx->pos & 1) == 1) { + if (!dir_emit(ctx, cell->name - 1, cell->name_len + 1, + cell->dynroot_ino + 1, DT_DIR)) + return 0; + ctx->pos++; + } + } + return 0; +} + +/* + * Read the AFS dynamic root directory. This produces a list of cellnames, + * dotted and undotted, along with @cell and .@cell links if configured. + */ +static int afs_dynroot_readdir(struct file *file, struct dir_context *ctx) +{ + struct afs_net *net = afs_d2net(file->f_path.dentry); + int ret = 0; + + if (!dir_emit_dots(file, ctx)) + return 0; + + if (ctx->pos == 2) { + if (rcu_access_pointer(net->ws_cell) && + !dir_emit(ctx, "@cell", 5, 2, DT_LNK)) + return 0; + ctx->pos = 3; + } + if (ctx->pos == 3) { + if (rcu_access_pointer(net->ws_cell) && + !dir_emit(ctx, ".@cell", 6, 3, DT_LNK)) + return 0; + ctx->pos = 4; + } + + if ((unsigned long long)ctx->pos <= AFS_MAX_DYNROOT_CELL_INO) { + down_read(&net->cells_lock); + ret = afs_dynroot_readdir_cells(net, ctx); + up_read(&net->cells_lock); + } + return ret; +} + +static const struct file_operations afs_dynroot_file_operations = { + .llseek = generic_file_llseek, + .read = generic_read_dir, + .iterate_shared = afs_dynroot_readdir, + .fsync = noop_fsync, +}; + +/* + * Create an inode for a dynamic root directory. + */ +struct inode *afs_dynroot_iget_root(struct super_block *sb) +{ + struct afs_super_info *as = AFS_FS_S(sb); + struct afs_vnode *vnode; + struct inode *inode; + struct afs_fid fid = { .vid = 0, .vnode = 1, .unique = 1,}; + + if (as->volume) + fid.vid = as->volume->vid; + + inode = iget5_locked(sb, fid.vnode, + afs_iget5_pseudo_test, afs_iget5_pseudo_set, &fid); + if (!inode) + return ERR_PTR(-ENOMEM); + + vnode = AFS_FS_I(inode); + + /* there shouldn't be an existing inode */ + if (inode_state_read_once(inode) & I_NEW) { + netfs_inode_init(&vnode->netfs, NULL, false); + simple_inode_init_ts(inode); + set_nlink(inode, 2); + inode->i_size = 0; + inode->i_mode = S_IFDIR | 0555; + inode->i_op = &afs_dynroot_inode_operations; + inode->i_fop = &afs_dynroot_file_operations; + inode->i_uid = GLOBAL_ROOT_UID; + inode->i_gid = GLOBAL_ROOT_GID; + inode->i_blocks = 0; + inode->i_generation = 0; + inode->i_flags |= S_NOATIME; + + set_bit(AFS_VNODE_PSEUDODIR, &vnode->flags); + unlock_new_inode(inode); + } + _leave(" = %p", inode); + return inode; +} diff --git a/fs/afs/file.c b/fs/afs/file.c new file mode 100644 index 000000000..0467742bf --- /dev/null +++ b/fs/afs/file.c @@ -0,0 +1,596 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* AFS filesystem file handling + * + * Copyright (C) 2002, 2007 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +static int afs_file_mmap_prepare(struct vm_area_desc *desc); + +static ssize_t afs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter); +static ssize_t afs_file_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, + size_t len, unsigned int flags); +static void afs_vm_open(struct vm_area_struct *area); +static void afs_vm_close(struct vm_area_struct *area); +static vm_fault_t afs_vm_map_pages(struct vm_fault *vmf, pgoff_t start_pgoff, pgoff_t end_pgoff); +static int afs_mapped(unsigned long start, unsigned long end, pgoff_t pgoff, + const struct file *file, void **vm_private_data); + +const struct file_operations afs_file_operations = { + .open = afs_open, + .release = afs_release, + .llseek = generic_file_llseek, + .read_iter = afs_file_read_iter, + .write_iter = netfs_file_write_iter, + .mmap_prepare = afs_file_mmap_prepare, + .splice_read = afs_file_splice_read, + .splice_write = iter_file_splice_write, + .fsync = afs_fsync, + .lock = afs_lock, + .flock = afs_flock, +}; + +const struct inode_operations afs_file_inode_operations = { + .getattr = afs_getattr, + .setattr = afs_setattr, + .permission = afs_permission, +}; + +const struct address_space_operations afs_file_aops = { + .direct_IO = noop_direct_IO, + .read_folio = netfs_read_folio, + .readahead = netfs_readahead, + .dirty_folio = netfs_dirty_folio, + .release_folio = netfs_release_folio, + .invalidate_folio = netfs_invalidate_folio, + .migrate_folio = filemap_migrate_folio, + .writepages = afs_writepages, +}; + +static const struct vm_operations_struct afs_vm_ops = { + .mapped = afs_mapped, + .open = afs_vm_open, + .close = afs_vm_close, + .fault = filemap_fault, + .map_pages = afs_vm_map_pages, + .page_mkwrite = afs_page_mkwrite, +}; + +/* + * Discard a pin on a writeback key. + */ +void afs_put_wb_key(struct afs_wb_key *wbk) +{ + if (wbk && refcount_dec_and_test(&wbk->usage)) { + key_put(wbk->key); + kfree(wbk); + } +} + +/* + * Cache key for writeback. + */ +int afs_cache_wb_key(struct afs_vnode *vnode, struct afs_file *af) +{ + struct afs_wb_key *wbk, *p; + + wbk = kzalloc_obj(struct afs_wb_key); + if (!wbk) + return -ENOMEM; + refcount_set(&wbk->usage, 2); + wbk->key = af->key; + + spin_lock(&vnode->wb_lock); + list_for_each_entry(p, &vnode->wb_keys, vnode_link) { + if (p->key == wbk->key) + goto found; + } + + key_get(wbk->key); + list_add_tail(&wbk->vnode_link, &vnode->wb_keys); + spin_unlock(&vnode->wb_lock); + af->wb = wbk; + return 0; + +found: + refcount_inc(&p->usage); + spin_unlock(&vnode->wb_lock); + af->wb = p; + kfree(wbk); + return 0; +} + +/* + * open an AFS file or directory and attach a key to it + */ +int afs_open(struct inode *inode, struct file *file) +{ + struct afs_vnode *vnode = AFS_FS_I(inode); + struct afs_file *af; + struct key *key; + int ret; + + _enter("{%llx:%llu},", vnode->fid.vid, vnode->fid.vnode); + + key = afs_request_key(vnode->volume->cell); + if (IS_ERR(key)) { + ret = PTR_ERR(key); + goto error; + } + + af = kzalloc_obj(*af); + if (!af) { + ret = -ENOMEM; + goto error_key; + } + af->key = key; + + ret = afs_validate(vnode, key); + if (ret < 0) + goto error_af; + + if (file->f_mode & FMODE_WRITE) { + ret = afs_cache_wb_key(vnode, af); + if (ret < 0) + goto error_af; + } + + if (file->f_flags & O_TRUNC) + set_bit(AFS_VNODE_NEW_CONTENT, &vnode->flags); + + fscache_use_cookie(afs_vnode_cache(vnode), file->f_mode & FMODE_WRITE); + + file->private_data = af; + _leave(" = 0"); + return 0; + +error_af: + kfree(af); +error_key: + key_put(key); +error: + _leave(" = %d", ret); + return ret; +} + +/* + * release an AFS file or directory and discard its key + */ +int afs_release(struct inode *inode, struct file *file) +{ + struct afs_vnode_cache_aux aux; + struct afs_vnode *vnode = AFS_FS_I(inode); + struct afs_file *af = file->private_data; + loff_t i_size; + int ret = 0; + + _enter("{%llx:%llu},", vnode->fid.vid, vnode->fid.vnode); + + if ((file->f_mode & FMODE_WRITE)) + ret = vfs_fsync(file, 0); + + file->private_data = NULL; + if (af->wb) + afs_put_wb_key(af->wb); + + if ((file->f_mode & FMODE_WRITE)) { + i_size = i_size_read(&vnode->netfs.inode); + afs_set_cache_aux(vnode, &aux); + fscache_unuse_cookie(afs_vnode_cache(vnode), &aux, &i_size); + } else { + fscache_unuse_cookie(afs_vnode_cache(vnode), NULL, NULL); + } + + key_put(af->key); + kfree(af); + afs_prune_wb_keys(vnode); + _leave(" = %d", ret); + return ret; +} + +static void afs_fetch_data_notify(struct afs_operation *op) +{ + struct netfs_io_subrequest *subreq = op->fetch.subreq; + + subreq->error = afs_op_error(op); + netfs_read_subreq_terminated(subreq); +} + +static void afs_fetch_data_success(struct afs_operation *op) +{ + struct afs_vnode *vnode = op->file[0].vnode; + + _enter("op=%08x", op->debug_id); + afs_vnode_commit_status(op, &op->file[0]); + afs_stat_v(vnode, n_fetches); + atomic_long_add(op->fetch.subreq->transferred, &op->net->n_fetch_bytes); + afs_fetch_data_notify(op); +} + +static void afs_fetch_data_aborted(struct afs_operation *op) +{ + afs_check_for_remote_deletion(op); + afs_fetch_data_notify(op); +} + +const struct afs_operation_ops afs_fetch_data_operation = { + .issue_afs_rpc = afs_fs_fetch_data, + .issue_yfs_rpc = yfs_fs_fetch_data, + .success = afs_fetch_data_success, + .aborted = afs_fetch_data_aborted, + .failed = afs_fetch_data_notify, +}; + +static void afs_issue_read_call(struct afs_operation *op) +{ + op->call_responded = false; + op->call_error = 0; + op->call_abort_code = 0; + if (test_bit(AFS_SERVER_FL_IS_YFS, &op->server->flags)) + yfs_fs_fetch_data(op); + else + afs_fs_fetch_data(op); +} + +static void afs_end_read(struct afs_operation *op) +{ + if (op->call_responded && op->server) + set_bit(AFS_SERVER_FL_RESPONDING, &op->server->flags); + + if (!afs_op_error(op)) + afs_fetch_data_success(op); + else if (op->cumul_error.aborted) + afs_fetch_data_aborted(op); + else + afs_fetch_data_notify(op); + + afs_end_vnode_operation(op); + afs_put_operation(op); +} + +/* + * Perform I/O processing on an asynchronous call. The work item carries a ref + * to the call struct that we either need to release or to pass on. + */ +static void afs_read_receive(struct afs_call *call) +{ + struct afs_operation *op = call->op; + enum afs_call_state state; + + _enter(""); + + state = READ_ONCE(call->state); + if (state == AFS_CALL_COMPLETE) + return; + trace_afs_read_recv(op, call); + + while (state < AFS_CALL_COMPLETE && READ_ONCE(call->need_attention)) { + WRITE_ONCE(call->need_attention, false); + afs_deliver_to_call(call); + state = READ_ONCE(call->state); + } + + if (state < AFS_CALL_COMPLETE) { + netfs_read_subreq_progress(op->fetch.subreq); + if (rxrpc_kernel_check_life(call->net->socket, call->rxcall)) + return; + /* rxrpc terminated the call. */ + afs_set_call_complete(call, call->error, call->abort_code); + } + + op->call_abort_code = call->abort_code; + op->call_error = call->error; + op->call_responded = call->responded; + op->call = NULL; + call->op = NULL; + afs_put_call(call); + + /* If the call failed, then we need to crank the server rotation + * handle and try the next. + */ + if (afs_select_fileserver(op)) { + afs_issue_read_call(op); + return; + } + + afs_end_read(op); +} + +void afs_fetch_data_async_rx(struct work_struct *work) +{ + struct afs_call *call = container_of(work, struct afs_call, async_work); + + afs_read_receive(call); + afs_put_call(call); +} + +void afs_fetch_data_immediate_cancel(struct afs_call *call) +{ + if (call->async) { + afs_get_call(call, afs_call_trace_wake); + if (!queue_work(afs_async_calls, &call->async_work)) + afs_deferred_put_call(call); + flush_work(&call->async_work); + } +} + +/* + * Fetch file data from the volume. + */ +static void afs_issue_read(struct netfs_io_subrequest *subreq) +{ + struct afs_operation *op; + struct afs_vnode *vnode = AFS_FS_I(subreq->rreq->inode); + struct key *key = subreq->rreq->netfs_priv; + + _enter("%s{%llx:%llu.%u},%x,,,", + vnode->volume->name, + vnode->fid.vid, + vnode->fid.vnode, + vnode->fid.unique, + key_serial(key)); + + op = afs_alloc_operation(key, vnode->volume); + if (IS_ERR(op)) { + subreq->error = PTR_ERR(op); + netfs_read_subreq_terminated(subreq); + return; + } + + afs_op_set_vnode(op, 0, vnode); + + op->fetch.subreq = subreq; + op->ops = &afs_fetch_data_operation; + + trace_netfs_sreq(subreq, netfs_sreq_trace_submit); + + if (subreq->rreq->origin == NETFS_READAHEAD || + subreq->rreq->iocb) { + op->flags |= AFS_OPERATION_ASYNC; + + if (!afs_begin_vnode_operation(op)) { + subreq->error = afs_put_operation(op); + netfs_read_subreq_terminated(subreq); + return; + } + + if (!afs_select_fileserver(op)) { + afs_end_read(op); + return; + } + + afs_issue_read_call(op); + } else { + afs_do_sync_operation(op); + } +} + +static int afs_init_request(struct netfs_io_request *rreq, struct file *file) +{ + struct afs_vnode *vnode = AFS_FS_I(rreq->inode); + + if (file) + rreq->netfs_priv = key_get(afs_file_key(file)); + rreq->rsize = 256 * 1024; + rreq->wsize = 256 * 1024 * 1024; + + switch (rreq->origin) { + case NETFS_READ_SINGLE: + if (!file) { + struct key *key = afs_request_key(vnode->volume->cell); + + if (IS_ERR(key)) + return PTR_ERR(key); + rreq->netfs_priv = key; + } + break; + case NETFS_WRITEBACK: + case NETFS_WRITETHROUGH: + case NETFS_UNBUFFERED_WRITE: + case NETFS_DIO_WRITE: + if (S_ISREG(rreq->inode->i_mode)) + rreq->io_streams[0].avail = true; + break; + case NETFS_WRITEBACK_SINGLE: + default: + break; + } + return 0; +} + +static int afs_check_write_begin(struct file *file, loff_t pos, unsigned len, + struct folio **foliop, void **_fsdata) +{ + struct afs_vnode *vnode = AFS_FS_I(file_inode(file)); + + return test_bit(AFS_VNODE_DELETED, &vnode->flags) ? -ESTALE : 0; +} + +static void afs_free_request(struct netfs_io_request *rreq) +{ + key_put(rreq->netfs_priv); + afs_put_wb_key(rreq->netfs_priv2); +} + +/* + * Set the file size and block count, taking ->cb_lock and ->i_lock to maintain + * coherency and prevent 64-bit tearing on 32-bit arches. + * + * Also, estimate the number of 512 bytes blocks used, rounded up to nearest 1K + * for consistency with other AFS clients. + */ +void afs_set_i_size(struct afs_vnode *vnode, loff_t new_i_size) +{ + struct inode *inode = &vnode->netfs.inode; + loff_t i_size; + + write_seqlock(&vnode->cb_lock); + spin_lock(&inode->i_lock); + i_size = i_size_read(inode); + if (new_i_size > i_size) { + i_size_write(inode, new_i_size); + inode_set_bytes(inode, round_up(new_i_size, 1024)); + } + spin_unlock(&inode->i_lock); + write_sequnlock(&vnode->cb_lock); + fscache_update_cookie(afs_vnode_cache(vnode), NULL, &new_i_size); +} + +static void afs_update_i_size(struct inode *inode, loff_t new_i_size) +{ + afs_set_i_size(AFS_FS_I(inode), new_i_size); +} + +static void afs_netfs_invalidate_cache(struct netfs_io_request *wreq) +{ + struct afs_vnode *vnode = AFS_FS_I(wreq->inode); + + afs_invalidate_cache(vnode, 0); +} + +const struct netfs_request_ops afs_req_ops = { + .init_request = afs_init_request, + .free_request = afs_free_request, + .check_write_begin = afs_check_write_begin, + .issue_read = afs_issue_read, + .update_i_size = afs_update_i_size, + .invalidate_cache = afs_netfs_invalidate_cache, + .begin_writeback = afs_begin_writeback, + .prepare_write = afs_prepare_write, + .issue_write = afs_issue_write, + .retry_request = afs_retry_request, +}; + +static void afs_add_open_mmap(struct afs_vnode *vnode) +{ + if (atomic_inc_return(&vnode->cb_nr_mmap) == 1) { + down_write(&vnode->volume->open_mmaps_lock); + + if (list_empty(&vnode->cb_mmap_link)) + list_add_tail(&vnode->cb_mmap_link, &vnode->volume->open_mmaps); + + up_write(&vnode->volume->open_mmaps_lock); + } +} + +static void afs_drop_open_mmap(struct afs_vnode *vnode) +{ + if (atomic_add_unless(&vnode->cb_nr_mmap, -1, 1)) + return; + + down_write(&vnode->volume->open_mmaps_lock); + + read_seqlock_excl(&vnode->cb_lock); + // the only place where ->cb_nr_mmap may hit 0 + // see __afs_break_callback() for the other side... + if (atomic_dec_and_test(&vnode->cb_nr_mmap)) + list_del_init(&vnode->cb_mmap_link); + read_sequnlock_excl(&vnode->cb_lock); + + up_write(&vnode->volume->open_mmaps_lock); + flush_work(&vnode->cb_work); +} + +/* + * Handle setting up a memory mapping on an AFS file. + */ +static int afs_file_mmap_prepare(struct vm_area_desc *desc) +{ + int ret; + + ret = generic_file_mmap_prepare(desc); + if (ret) + return ret; + + desc->vm_ops = &afs_vm_ops; + return ret; +} + +static int afs_mapped(unsigned long start, unsigned long end, pgoff_t pgoff, + const struct file *file, void **vm_private_data) +{ + struct afs_vnode *vnode = AFS_FS_I(file_inode(file)); + + afs_add_open_mmap(vnode); + return 0; +} + +static void afs_vm_open(struct vm_area_struct *vma) +{ + struct file *file = vma->vm_file; + struct afs_vnode *vnode = AFS_FS_I(file_inode(file)); + + afs_add_open_mmap(vnode); +} + +static void afs_vm_close(struct vm_area_struct *vma) +{ + struct file *file = vma->vm_file; + struct afs_vnode *vnode = AFS_FS_I(file_inode(file)); + + afs_drop_open_mmap(vnode); +} + +static vm_fault_t afs_vm_map_pages(struct vm_fault *vmf, pgoff_t start_pgoff, pgoff_t end_pgoff) +{ + struct file *file = vmf->vma->vm_file; + struct afs_vnode *vnode = AFS_FS_I(file_inode(file)); + + if (afs_check_validity(vnode)) + return filemap_map_pages(vmf, start_pgoff, end_pgoff); + return 0; +} + +static ssize_t afs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) +{ + struct inode *inode = file_inode(iocb->ki_filp); + struct afs_vnode *vnode = AFS_FS_I(inode); + struct afs_file *af = iocb->ki_filp->private_data; + ssize_t ret; + + if (iocb->ki_flags & IOCB_DIRECT) + return netfs_unbuffered_read_iter(iocb, iter); + + ret = netfs_start_io_read(inode); + if (ret < 0) + return ret; + ret = afs_validate(vnode, af->key); + if (ret == 0) + ret = filemap_read(iocb, iter, 0); + netfs_end_io_read(inode); + return ret; +} + +static ssize_t afs_file_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, + size_t len, unsigned int flags) +{ + struct inode *inode = file_inode(in); + struct afs_vnode *vnode = AFS_FS_I(inode); + struct afs_file *af = in->private_data; + ssize_t ret; + + ret = netfs_start_io_read(inode); + if (ret < 0) + return ret; + ret = afs_validate(vnode, af->key); + if (ret == 0) + ret = filemap_splice_read(in, ppos, pipe, len, flags); + netfs_end_io_read(inode); + return ret; +} diff --git a/fs/afs/flock.c b/fs/afs/flock.c new file mode 100644 index 000000000..f0e96a350 --- /dev/null +++ b/fs/afs/flock.c @@ -0,0 +1,877 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* AFS file locking support + * + * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include "internal.h" + +#define AFS_LOCK_GRANTED 0 +#define AFS_LOCK_PENDING 1 +#define AFS_LOCK_YOUR_TRY 2 + +struct workqueue_struct *afs_lock_manager; + +static void afs_next_locker(struct afs_vnode *vnode, int error); +static void afs_fl_copy_lock(struct file_lock *new, struct file_lock *fl); +static void afs_fl_release_private(struct file_lock *fl); + +static const struct file_lock_operations afs_lock_ops = { + .fl_copy_lock = afs_fl_copy_lock, + .fl_release_private = afs_fl_release_private, +}; + +static inline void afs_set_lock_state(struct afs_vnode *vnode, enum afs_lock_state state) +{ + _debug("STATE %u -> %u", vnode->lock_state, state); + vnode->lock_state = state; +} + +static atomic_t afs_file_lock_debug_id; + +/* + * if the callback is broken on this vnode, then the lock may now be available + */ +void afs_lock_may_be_available(struct afs_vnode *vnode) +{ + _enter("{%llx:%llu}", vnode->fid.vid, vnode->fid.vnode); + + spin_lock(&vnode->lock); + if (vnode->lock_state == AFS_VNODE_LOCK_WAITING_FOR_CB) + afs_next_locker(vnode, 0); + trace_afs_flock_ev(vnode, NULL, afs_flock_callback_break, 0); + spin_unlock(&vnode->lock); +} + +/* + * the lock will time out in 5 minutes unless we extend it, so schedule + * extension in a bit less than that time + */ +static void afs_schedule_lock_extension(struct afs_vnode *vnode) +{ + ktime_t expires_at, now, duration; + u64 duration_j; + + expires_at = ktime_add_ms(vnode->locked_at, AFS_LOCKWAIT * 1000 / 2); + now = ktime_get_real(); + duration = ktime_sub(expires_at, now); + if (duration <= 0) + duration_j = 0; + else + duration_j = nsecs_to_jiffies(ktime_to_ns(duration)); + + queue_delayed_work(afs_lock_manager, &vnode->lock_work, duration_j); +} + +/* + * In the case of successful completion of a lock operation, record the time + * the reply appeared and start the lock extension timer. + */ +void afs_lock_op_done(struct afs_call *call) +{ + struct afs_operation *op = call->op; + struct afs_vnode *vnode = op->file[0].vnode; + + if (call->error == 0) { + spin_lock(&vnode->lock); + trace_afs_flock_ev(vnode, NULL, afs_flock_timestamp, 0); + vnode->locked_at = call->issue_time; + afs_schedule_lock_extension(vnode); + spin_unlock(&vnode->lock); + } +} + +/* + * grant one or more locks (readlocks are allowed to jump the queue if the + * first lock in the queue is itself a readlock) + * - the caller must hold the vnode lock + */ +static void afs_grant_locks(struct afs_vnode *vnode) +{ + struct file_lock *p, *_p; + bool exclusive = (vnode->lock_type == AFS_LOCK_WRITE); + + list_for_each_entry_safe(p, _p, &vnode->pending_locks, fl_u.afs.link) { + if (!exclusive && lock_is_write(p)) + continue; + + list_move_tail(&p->fl_u.afs.link, &vnode->granted_locks); + p->fl_u.afs.state = AFS_LOCK_GRANTED; + trace_afs_flock_op(vnode, p, afs_flock_op_grant); + locks_wake_up(p); + } +} + +/* + * If an error is specified, reject every pending lock that matches the + * authentication and type of the lock we failed to get. If there are any + * remaining lockers, try to wake up one of them to have a go. + */ +static void afs_next_locker(struct afs_vnode *vnode, int error) +{ + struct file_lock *p, *_p, *next = NULL; + struct key *key = vnode->lock_key; + unsigned int type = F_RDLCK; + + _enter(""); + + if (vnode->lock_type == AFS_LOCK_WRITE) + type = F_WRLCK; + + list_for_each_entry_safe(p, _p, &vnode->pending_locks, fl_u.afs.link) { + if (error && + p->c.flc_type == type && + afs_file_key(p->c.flc_file) == key) { + list_del_init(&p->fl_u.afs.link); + p->fl_u.afs.state = error; + locks_wake_up(p); + } + + /* Select the next locker to hand off to. */ + if (next && (lock_is_write(next) || lock_is_read(p))) + continue; + next = p; + } + + vnode->lock_key = NULL; + key_put(key); + + if (next) { + afs_set_lock_state(vnode, AFS_VNODE_LOCK_SETTING); + next->fl_u.afs.state = AFS_LOCK_YOUR_TRY; + trace_afs_flock_op(vnode, next, afs_flock_op_wake); + locks_wake_up(next); + } else { + afs_set_lock_state(vnode, AFS_VNODE_LOCK_NONE); + trace_afs_flock_ev(vnode, NULL, afs_flock_no_lockers, 0); + } + + _leave(""); +} + +/* + * Kill off all waiters in the the pending lock queue due to the vnode being + * deleted. + */ +static void afs_kill_lockers_enoent(struct afs_vnode *vnode) +{ + struct file_lock *p; + + afs_set_lock_state(vnode, AFS_VNODE_LOCK_DELETED); + + while (!list_empty(&vnode->pending_locks)) { + p = list_entry(vnode->pending_locks.next, + struct file_lock, fl_u.afs.link); + list_del_init(&p->fl_u.afs.link); + p->fl_u.afs.state = -ENOENT; + locks_wake_up(p); + } + + key_put(vnode->lock_key); + vnode->lock_key = NULL; +} + +static void afs_lock_success(struct afs_operation *op) +{ + _enter("op=%08x", op->debug_id); + afs_vnode_commit_status(op, &op->file[0]); +} + +static const struct afs_operation_ops afs_set_lock_operation = { + .issue_afs_rpc = afs_fs_set_lock, + .issue_yfs_rpc = yfs_fs_set_lock, + .success = afs_lock_success, + .aborted = afs_check_for_remote_deletion, +}; + +/* + * Get a lock on a file + */ +static int afs_set_lock(struct afs_vnode *vnode, struct key *key, + afs_lock_type_t type) +{ + struct afs_operation *op; + + _enter("%s{%llx:%llu.%u},%x,%u", + vnode->volume->name, + vnode->fid.vid, + vnode->fid.vnode, + vnode->fid.unique, + key_serial(key), type); + + op = afs_alloc_operation(key, vnode->volume); + if (IS_ERR(op)) + return PTR_ERR(op); + + afs_op_set_vnode(op, 0, vnode); + + op->lock.type = type; + op->ops = &afs_set_lock_operation; + return afs_do_sync_operation(op); +} + +static const struct afs_operation_ops afs_extend_lock_operation = { + .issue_afs_rpc = afs_fs_extend_lock, + .issue_yfs_rpc = yfs_fs_extend_lock, + .success = afs_lock_success, +}; + +/* + * Extend a lock on a file + */ +static int afs_extend_lock(struct afs_vnode *vnode, struct key *key) +{ + struct afs_operation *op; + + _enter("%s{%llx:%llu.%u},%x", + vnode->volume->name, + vnode->fid.vid, + vnode->fid.vnode, + vnode->fid.unique, + key_serial(key)); + + op = afs_alloc_operation(key, vnode->volume); + if (IS_ERR(op)) + return PTR_ERR(op); + + afs_op_set_vnode(op, 0, vnode); + + op->flags |= AFS_OPERATION_UNINTR; + op->ops = &afs_extend_lock_operation; + return afs_do_sync_operation(op); +} + +static const struct afs_operation_ops afs_release_lock_operation = { + .issue_afs_rpc = afs_fs_release_lock, + .issue_yfs_rpc = yfs_fs_release_lock, + .success = afs_lock_success, +}; + +/* + * Release a lock on a file + */ +static int afs_release_lock(struct afs_vnode *vnode, struct key *key) +{ + struct afs_operation *op; + + _enter("%s{%llx:%llu.%u},%x", + vnode->volume->name, + vnode->fid.vid, + vnode->fid.vnode, + vnode->fid.unique, + key_serial(key)); + + op = afs_alloc_operation(key, vnode->volume); + if (IS_ERR(op)) + return PTR_ERR(op); + + afs_op_set_vnode(op, 0, vnode); + + op->flags |= AFS_OPERATION_UNINTR; + op->ops = &afs_release_lock_operation; + return afs_do_sync_operation(op); +} + +/* + * do work for a lock, including: + * - probing for a lock we're waiting on but didn't get immediately + * - extending a lock that's close to timing out + */ +void afs_lock_work(struct work_struct *work) +{ + struct afs_vnode *vnode = + container_of(work, struct afs_vnode, lock_work.work); + struct key *key; + int ret; + + _enter("{%llx:%llu}", vnode->fid.vid, vnode->fid.vnode); + + spin_lock(&vnode->lock); + +again: + _debug("wstate %u for %p", vnode->lock_state, vnode); + switch (vnode->lock_state) { + case AFS_VNODE_LOCK_NEED_UNLOCK: + afs_set_lock_state(vnode, AFS_VNODE_LOCK_UNLOCKING); + trace_afs_flock_ev(vnode, NULL, afs_flock_work_unlocking, 0); + spin_unlock(&vnode->lock); + + /* attempt to release the server lock; if it fails, we just + * wait 5 minutes and it'll expire anyway */ + ret = afs_release_lock(vnode, vnode->lock_key); + if (ret < 0 && vnode->lock_state != AFS_VNODE_LOCK_DELETED) { + trace_afs_flock_ev(vnode, NULL, afs_flock_release_fail, + ret); + printk(KERN_WARNING "AFS:" + " Failed to release lock on {%llx:%llx} error %d\n", + vnode->fid.vid, vnode->fid.vnode, ret); + } + + spin_lock(&vnode->lock); + if (ret == -ENOENT) + afs_kill_lockers_enoent(vnode); + else + afs_next_locker(vnode, 0); + spin_unlock(&vnode->lock); + return; + + /* If we've already got a lock, then it must be time to extend that + * lock as AFS locks time out after 5 minutes. + */ + case AFS_VNODE_LOCK_GRANTED: + _debug("extend"); + + ASSERT(!list_empty(&vnode->granted_locks)); + + key = key_get(vnode->lock_key); + afs_set_lock_state(vnode, AFS_VNODE_LOCK_EXTENDING); + trace_afs_flock_ev(vnode, NULL, afs_flock_work_extending, 0); + spin_unlock(&vnode->lock); + + ret = afs_extend_lock(vnode, key); /* RPC */ + key_put(key); + + if (ret < 0) { + trace_afs_flock_ev(vnode, NULL, afs_flock_extend_fail, + ret); + pr_warn("AFS: Failed to extend lock on {%llx:%llx} error %d\n", + vnode->fid.vid, vnode->fid.vnode, ret); + } + + spin_lock(&vnode->lock); + + if (ret == -ENOENT) { + afs_kill_lockers_enoent(vnode); + spin_unlock(&vnode->lock); + return; + } + + if (vnode->lock_state != AFS_VNODE_LOCK_EXTENDING) + goto again; + afs_set_lock_state(vnode, AFS_VNODE_LOCK_GRANTED); + + if (ret != 0) + queue_delayed_work(afs_lock_manager, &vnode->lock_work, + HZ * 10); + spin_unlock(&vnode->lock); + _leave(" [ext]"); + return; + + /* If we're waiting for a callback to indicate lock release, we can't + * actually rely on this, so need to recheck at regular intervals. The + * problem is that the server might not notify us if the lock just + * expires (say because a client died) rather than being explicitly + * released. + */ + case AFS_VNODE_LOCK_WAITING_FOR_CB: + _debug("retry"); + afs_next_locker(vnode, 0); + spin_unlock(&vnode->lock); + return; + + case AFS_VNODE_LOCK_DELETED: + afs_kill_lockers_enoent(vnode); + spin_unlock(&vnode->lock); + return; + + default: + /* Looks like a lock request was withdrawn. */ + spin_unlock(&vnode->lock); + _leave(" [no]"); + return; + } +} + +/* + * pass responsibility for the unlocking of a vnode on the server to the + * manager thread, lest a pending signal in the calling thread interrupt + * AF_RXRPC + * - the caller must hold the vnode lock + */ +static void afs_defer_unlock(struct afs_vnode *vnode) +{ + _enter("%u", vnode->lock_state); + + if (list_empty(&vnode->granted_locks) && + (vnode->lock_state == AFS_VNODE_LOCK_GRANTED || + vnode->lock_state == AFS_VNODE_LOCK_EXTENDING)) { + cancel_delayed_work(&vnode->lock_work); + + afs_set_lock_state(vnode, AFS_VNODE_LOCK_NEED_UNLOCK); + trace_afs_flock_ev(vnode, NULL, afs_flock_defer_unlock, 0); + queue_delayed_work(afs_lock_manager, &vnode->lock_work, 0); + } +} + +/* + * Check that our view of the file metadata is up to date and check to see + * whether we think that we have a locking permit. + */ +static int afs_do_setlk_check(struct afs_vnode *vnode, struct key *key, + enum afs_flock_mode mode, afs_lock_type_t type) +{ + afs_access_t access; + int ret; + + /* Make sure we've got a callback on this file and that our view of the + * data version is up to date. + */ + ret = afs_validate(vnode, key); + if (ret < 0) + return ret; + + /* Check the permission set to see if we're actually going to be + * allowed to get a lock on this file. + */ + ret = afs_check_permit(vnode, key, &access); + if (ret < 0) + return ret; + + /* At a rough estimation, you need LOCK, WRITE or INSERT perm to + * read-lock a file and WRITE or INSERT perm to write-lock a file. + * + * We can't rely on the server to do this for us since if we want to + * share a read lock that we already have, we won't go the server. + */ + if (type == AFS_LOCK_READ) { + if (!(access & (AFS_ACE_INSERT | AFS_ACE_WRITE | AFS_ACE_LOCK))) + return -EACCES; + } else { + if (!(access & (AFS_ACE_INSERT | AFS_ACE_WRITE))) + return -EACCES; + } + + return 0; +} + +/* + * request a lock on a file on the server + */ +static int afs_do_setlk(struct file *file, struct file_lock *fl) +{ + struct inode *inode = file_inode(file); + struct afs_vnode *vnode = AFS_FS_I(inode); + enum afs_flock_mode mode = AFS_FS_S(inode->i_sb)->flock_mode; + afs_lock_type_t type; + struct key *key = afs_file_key(file); + bool partial, no_server_lock = false; + int ret; + + if (mode == afs_flock_mode_unset) + mode = afs_flock_mode_openafs; + + _enter("{%llx:%llu},%llu-%llu,%u,%u", + vnode->fid.vid, vnode->fid.vnode, + fl->fl_start, fl->fl_end, fl->c.flc_type, mode); + + fl->fl_ops = &afs_lock_ops; + INIT_LIST_HEAD(&fl->fl_u.afs.link); + fl->fl_u.afs.state = AFS_LOCK_PENDING; + + partial = (fl->fl_start != 0 || fl->fl_end != OFFSET_MAX); + type = lock_is_read(fl) ? AFS_LOCK_READ : AFS_LOCK_WRITE; + if (mode == afs_flock_mode_write && partial) + type = AFS_LOCK_WRITE; + + ret = afs_do_setlk_check(vnode, key, mode, type); + if (ret < 0) + return ret; + + trace_afs_flock_op(vnode, fl, afs_flock_op_set_lock); + + /* AFS3 protocol only supports full-file locks and doesn't provide any + * method of upgrade/downgrade, so we need to emulate for partial-file + * locks. + * + * The OpenAFS client only gets a server lock for a full-file lock and + * keeps partial-file locks local. Allow this behaviour to be emulated + * (as the default). + */ + if (mode == afs_flock_mode_local || + (partial && mode == afs_flock_mode_openafs)) { + no_server_lock = true; + goto skip_server_lock; + } + + spin_lock(&vnode->lock); + list_add_tail(&fl->fl_u.afs.link, &vnode->pending_locks); + + ret = -ENOENT; + if (vnode->lock_state == AFS_VNODE_LOCK_DELETED) + goto error_unlock; + + /* If we've already got a lock on the server then try to move to having + * the VFS grant the requested lock. Note that this means that other + * clients may get starved out. + */ + _debug("try %u", vnode->lock_state); + if (vnode->lock_state == AFS_VNODE_LOCK_GRANTED) { + if (type == AFS_LOCK_READ) { + _debug("instant readlock"); + list_move_tail(&fl->fl_u.afs.link, &vnode->granted_locks); + fl->fl_u.afs.state = AFS_LOCK_GRANTED; + goto vnode_is_locked_u; + } + + if (vnode->lock_type == AFS_LOCK_WRITE) { + _debug("instant writelock"); + list_move_tail(&fl->fl_u.afs.link, &vnode->granted_locks); + fl->fl_u.afs.state = AFS_LOCK_GRANTED; + goto vnode_is_locked_u; + } + } + + if (vnode->lock_state == AFS_VNODE_LOCK_NONE && + !(fl->c.flc_flags & FL_SLEEP)) { + ret = -EAGAIN; + if (type == AFS_LOCK_READ) { + if (vnode->status.lock_count == -1) + goto lock_is_contended; /* Write locked */ + } else { + if (vnode->status.lock_count != 0) + goto lock_is_contended; /* Locked */ + } + } + + if (vnode->lock_state != AFS_VNODE_LOCK_NONE) + goto need_to_wait; + +try_to_lock: + /* We don't have a lock on this vnode and we aren't currently waiting + * for one either, so ask the server for a lock. + * + * Note that we need to be careful if we get interrupted by a signal + * after dispatching the request as we may still get the lock, even + * though we don't wait for the reply (it's not too bad a problem - the + * lock will expire in 5 mins anyway). + */ + trace_afs_flock_ev(vnode, fl, afs_flock_try_to_lock, 0); + vnode->lock_key = key_get(key); + vnode->lock_type = type; + afs_set_lock_state(vnode, AFS_VNODE_LOCK_SETTING); + spin_unlock(&vnode->lock); + + ret = afs_set_lock(vnode, key, type); /* RPC */ + + spin_lock(&vnode->lock); + switch (ret) { + case -EKEYREJECTED: + case -EKEYEXPIRED: + case -EKEYREVOKED: + case -EPERM: + case -EACCES: + fl->fl_u.afs.state = ret; + trace_afs_flock_ev(vnode, fl, afs_flock_fail_perm, ret); + list_del_init(&fl->fl_u.afs.link); + afs_next_locker(vnode, ret); + goto error_unlock; + + case -ENOENT: + fl->fl_u.afs.state = ret; + trace_afs_flock_ev(vnode, fl, afs_flock_fail_other, ret); + list_del_init(&fl->fl_u.afs.link); + afs_kill_lockers_enoent(vnode); + goto error_unlock; + + default: + fl->fl_u.afs.state = ret; + trace_afs_flock_ev(vnode, fl, afs_flock_fail_other, ret); + list_del_init(&fl->fl_u.afs.link); + afs_next_locker(vnode, 0); + goto error_unlock; + + case -EWOULDBLOCK: + /* The server doesn't have a lock-waiting queue, so the client + * will have to retry. The server will break the outstanding + * callbacks on a file when a lock is released. + */ + ASSERT(list_empty(&vnode->granted_locks)); + ASSERTCMP(vnode->pending_locks.next, ==, &fl->fl_u.afs.link); + goto lock_is_contended; + + case 0: + afs_set_lock_state(vnode, AFS_VNODE_LOCK_GRANTED); + trace_afs_flock_ev(vnode, fl, afs_flock_acquired, type); + afs_grant_locks(vnode); + goto vnode_is_locked_u; + } + +vnode_is_locked_u: + spin_unlock(&vnode->lock); +vnode_is_locked: + /* the lock has been granted by the server... */ + ASSERTCMP(fl->fl_u.afs.state, ==, AFS_LOCK_GRANTED); + +skip_server_lock: + /* ... but the VFS still needs to distribute access on this client. */ + trace_afs_flock_ev(vnode, fl, afs_flock_vfs_locking, 0); + ret = locks_lock_file_wait(file, fl); + trace_afs_flock_ev(vnode, fl, afs_flock_vfs_lock, ret); + if (ret < 0) + goto vfs_rejected_lock; + + /* Again, make sure we've got a callback on this file and, again, make + * sure that our view of the data version is up to date (we ignore + * errors incurred here and deal with the consequences elsewhere). + */ + afs_validate(vnode, key); + _leave(" = 0"); + return 0; + +lock_is_contended: + if (!(fl->c.flc_flags & FL_SLEEP)) { + list_del_init(&fl->fl_u.afs.link); + afs_next_locker(vnode, 0); + ret = -EAGAIN; + goto error_unlock; + } + + afs_set_lock_state(vnode, AFS_VNODE_LOCK_WAITING_FOR_CB); + trace_afs_flock_ev(vnode, fl, afs_flock_would_block, ret); + queue_delayed_work(afs_lock_manager, &vnode->lock_work, HZ * 5); + +need_to_wait: + /* We're going to have to wait. Either this client doesn't have a lock + * on the server yet and we need to wait for a callback to occur, or + * the client does have a lock on the server, but it's shared and we + * need an exclusive lock. + */ + spin_unlock(&vnode->lock); + + trace_afs_flock_ev(vnode, fl, afs_flock_waiting, 0); + ret = wait_event_interruptible(fl->c.flc_wait, + fl->fl_u.afs.state != AFS_LOCK_PENDING); + trace_afs_flock_ev(vnode, fl, afs_flock_waited, ret); + + if (fl->fl_u.afs.state >= 0 && fl->fl_u.afs.state != AFS_LOCK_GRANTED) { + spin_lock(&vnode->lock); + + switch (fl->fl_u.afs.state) { + case AFS_LOCK_YOUR_TRY: + fl->fl_u.afs.state = AFS_LOCK_PENDING; + goto try_to_lock; + case AFS_LOCK_PENDING: + if (ret > 0) { + /* We need to retry the lock. We may not be + * notified by the server if it just expired + * rather than being released. + */ + ASSERTCMP(vnode->lock_state, ==, AFS_VNODE_LOCK_WAITING_FOR_CB); + afs_set_lock_state(vnode, AFS_VNODE_LOCK_SETTING); + fl->fl_u.afs.state = AFS_LOCK_PENDING; + goto try_to_lock; + } + goto error_unlock; + case AFS_LOCK_GRANTED: + default: + break; + } + + spin_unlock(&vnode->lock); + } + + if (fl->fl_u.afs.state == AFS_LOCK_GRANTED) + goto vnode_is_locked; + ret = fl->fl_u.afs.state; + goto error; + +vfs_rejected_lock: + /* The VFS rejected the lock we just obtained, so we have to discard + * what we just got. We defer this to the lock manager work item to + * deal with. + */ + _debug("vfs refused %d", ret); + if (no_server_lock) + goto error; + spin_lock(&vnode->lock); + list_del_init(&fl->fl_u.afs.link); + afs_defer_unlock(vnode); + +error_unlock: + spin_unlock(&vnode->lock); +error: + _leave(" = %d", ret); + return ret; +} + +/* + * unlock on a file on the server + */ +static int afs_do_unlk(struct file *file, struct file_lock *fl) +{ + struct afs_vnode *vnode = AFS_FS_I(file_inode(file)); + int ret; + + _enter("{%llx:%llu},%u", vnode->fid.vid, vnode->fid.vnode, + fl->c.flc_type); + + trace_afs_flock_op(vnode, fl, afs_flock_op_unlock); + + /* Flush all pending writes before doing anything with locks. */ + vfs_fsync(file, 0); + + ret = locks_lock_file_wait(file, fl); + _leave(" = %d [%u]", ret, vnode->lock_state); + return ret; +} + +/* + * return information about a lock we currently hold, if indeed we hold one + */ +static int afs_do_getlk(struct file *file, struct file_lock *fl) +{ + struct afs_vnode *vnode = AFS_FS_I(file_inode(file)); + struct key *key = afs_file_key(file); + int ret, lock_count; + + _enter(""); + + if (vnode->lock_state == AFS_VNODE_LOCK_DELETED) + return -ENOENT; + + fl->c.flc_type = F_UNLCK; + + /* check local lock records first */ + posix_test_lock(file, fl); + if (lock_is_unlock(fl)) { + /* no local locks; consult the server */ + ret = afs_fetch_status(vnode, key, false, NULL); + if (ret < 0) + goto error; + + lock_count = READ_ONCE(vnode->status.lock_count); + if (lock_count != 0) { + if (lock_count > 0) + fl->c.flc_type = F_RDLCK; + else + fl->c.flc_type = F_WRLCK; + fl->fl_start = 0; + fl->fl_end = OFFSET_MAX; + fl->c.flc_pid = 0; + } + } + + ret = 0; +error: + _leave(" = %d [%hd]", ret, fl->c.flc_type); + return ret; +} + +/* + * manage POSIX locks on a file + */ +int afs_lock(struct file *file, int cmd, struct file_lock *fl) +{ + struct afs_vnode *vnode = AFS_FS_I(file_inode(file)); + enum afs_flock_operation op; + int ret; + + _enter("{%llx:%llu},%d,{t=%x,fl=%x,r=%Ld:%Ld}", + vnode->fid.vid, vnode->fid.vnode, cmd, + fl->c.flc_type, fl->c.flc_flags, + (long long) fl->fl_start, (long long) fl->fl_end); + + if (IS_GETLK(cmd)) + return afs_do_getlk(file, fl); + + fl->fl_u.afs.debug_id = atomic_inc_return(&afs_file_lock_debug_id); + trace_afs_flock_op(vnode, fl, afs_flock_op_lock); + + if (lock_is_unlock(fl)) + ret = afs_do_unlk(file, fl); + else + ret = afs_do_setlk(file, fl); + + switch (ret) { + case 0: op = afs_flock_op_return_ok; break; + case -EAGAIN: op = afs_flock_op_return_eagain; break; + case -EDEADLK: op = afs_flock_op_return_edeadlk; break; + default: op = afs_flock_op_return_error; break; + } + trace_afs_flock_op(vnode, fl, op); + return ret; +} + +/* + * manage FLOCK locks on a file + */ +int afs_flock(struct file *file, int cmd, struct file_lock *fl) +{ + struct afs_vnode *vnode = AFS_FS_I(file_inode(file)); + enum afs_flock_operation op; + int ret; + + _enter("{%llx:%llu},%d,{t=%x,fl=%x}", + vnode->fid.vid, vnode->fid.vnode, cmd, + fl->c.flc_type, fl->c.flc_flags); + + /* + * No BSD flocks over NFS allowed. + * Note: we could try to fake a POSIX lock request here by + * using ((u32) filp | 0x80000000) or some such as the pid. + * Not sure whether that would be unique, though, or whether + * that would break in other places. + */ + if (!(fl->c.flc_flags & FL_FLOCK)) + return -ENOLCK; + + fl->fl_u.afs.debug_id = atomic_inc_return(&afs_file_lock_debug_id); + trace_afs_flock_op(vnode, fl, afs_flock_op_flock); + + /* we're simulating flock() locks using posix locks on the server */ + if (lock_is_unlock(fl)) + ret = afs_do_unlk(file, fl); + else + ret = afs_do_setlk(file, fl); + + switch (ret) { + case 0: op = afs_flock_op_return_ok; break; + case -EAGAIN: op = afs_flock_op_return_eagain; break; + case -EDEADLK: op = afs_flock_op_return_edeadlk; break; + default: op = afs_flock_op_return_error; break; + } + trace_afs_flock_op(vnode, fl, op); + return ret; +} + +/* + * the POSIX lock management core VFS code copies the lock record and adds the + * copy into its own list, so we need to add that copy to the vnode's lock + * queue in the same place as the original (which will be deleted shortly + * after) + */ +static void afs_fl_copy_lock(struct file_lock *new, struct file_lock *fl) +{ + struct afs_vnode *vnode = AFS_FS_I(file_inode(fl->c.flc_file)); + + _enter(""); + + new->fl_u.afs.debug_id = atomic_inc_return(&afs_file_lock_debug_id); + + spin_lock(&vnode->lock); + trace_afs_flock_op(vnode, new, afs_flock_op_copy_lock); + list_add(&new->fl_u.afs.link, &fl->fl_u.afs.link); + spin_unlock(&vnode->lock); +} + +/* + * need to remove this lock from the vnode queue when it's removed from the + * VFS's list + */ +static void afs_fl_release_private(struct file_lock *fl) +{ + struct afs_vnode *vnode = AFS_FS_I(file_inode(fl->c.flc_file)); + + _enter(""); + + spin_lock(&vnode->lock); + + trace_afs_flock_op(vnode, fl, afs_flock_op_release_lock); + list_del_init(&fl->fl_u.afs.link); + if (list_empty(&vnode->granted_locks)) + afs_defer_unlock(vnode); + + _debug("state %u for %p", vnode->lock_state, vnode); + spin_unlock(&vnode->lock); +} diff --git a/fs/afs/fs_operation.c b/fs/afs/fs_operation.c new file mode 100644 index 000000000..20801b295 --- /dev/null +++ b/fs/afs/fs_operation.c @@ -0,0 +1,377 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Fileserver-directed operation handling. + * + * Copyright (C) 2020 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include "internal.h" + +static atomic_t afs_operation_debug_counter; + +/* + * Create an operation against a volume. + */ +struct afs_operation *afs_alloc_operation(struct key *key, struct afs_volume *volume) +{ + struct afs_operation *op; + + _enter(""); + + op = kzalloc_obj(*op); + if (!op) + return ERR_PTR(-ENOMEM); + + if (!key) { + key = afs_request_key(volume->cell); + if (IS_ERR(key)) { + kfree(op); + return ERR_CAST(key); + } + } else { + key_get(key); + } + + op->key = key; + op->volume = afs_get_volume(volume, afs_volume_trace_get_new_op); + op->net = volume->cell->net; + op->cb_v_break = atomic_read(&volume->cb_v_break); + op->pre_volsync.creation = volume->creation_time; + op->pre_volsync.update = volume->update_time; + op->debug_id = atomic_inc_return(&afs_operation_debug_counter); + op->nr_iterations = -1; + afs_op_set_error(op, -EDESTADDRREQ); + + _leave(" = [op=%08x]", op->debug_id); + return op; +} + +struct afs_io_locker { + struct list_head link; + struct task_struct *task; + unsigned long have_lock; +}; + +/* + * Unlock the I/O lock on a vnode. + */ +static void afs_unlock_for_io(struct afs_vnode *vnode) +{ + struct afs_io_locker *locker; + + spin_lock(&vnode->lock); + locker = list_first_entry_or_null(&vnode->io_lock_waiters, + struct afs_io_locker, link); + if (locker) { + list_del(&locker->link); + smp_store_release(&locker->have_lock, 1); /* The unlock barrier. */ + smp_mb__after_atomic(); /* Store have_lock before task state */ + wake_up_process(locker->task); + } else { + clear_bit(AFS_VNODE_IO_LOCK, &vnode->flags); + } + spin_unlock(&vnode->lock); +} + +/* + * Lock the I/O lock on a vnode uninterruptibly. We can't use an ordinary + * mutex as lockdep will complain if we unlock it in the wrong thread. + */ +static void afs_lock_for_io(struct afs_vnode *vnode) +{ + struct afs_io_locker myself = { .task = current, }; + + spin_lock(&vnode->lock); + + if (!test_and_set_bit(AFS_VNODE_IO_LOCK, &vnode->flags)) { + spin_unlock(&vnode->lock); + return; + } + + list_add_tail(&myself.link, &vnode->io_lock_waiters); + spin_unlock(&vnode->lock); + + for (;;) { + set_current_state(TASK_UNINTERRUPTIBLE); + if (smp_load_acquire(&myself.have_lock)) /* The lock barrier */ + break; + schedule(); + } + __set_current_state(TASK_RUNNING); +} + +/* + * Lock the I/O lock on a vnode interruptibly. We can't use an ordinary mutex + * as lockdep will complain if we unlock it in the wrong thread. + */ +static int afs_lock_for_io_interruptible(struct afs_vnode *vnode) +{ + struct afs_io_locker myself = { .task = current, }; + int ret = 0; + + spin_lock(&vnode->lock); + + if (!test_and_set_bit(AFS_VNODE_IO_LOCK, &vnode->flags)) { + spin_unlock(&vnode->lock); + return 0; + } + + list_add_tail(&myself.link, &vnode->io_lock_waiters); + spin_unlock(&vnode->lock); + + for (;;) { + set_current_state(TASK_INTERRUPTIBLE); + if (smp_load_acquire(&myself.have_lock) || /* The lock barrier */ + signal_pending(current)) + break; + schedule(); + } + __set_current_state(TASK_RUNNING); + + /* If we got a signal, try to transfer the lock onto the next + * waiter. + */ + if (unlikely(signal_pending(current))) { + spin_lock(&vnode->lock); + if (myself.have_lock) { + spin_unlock(&vnode->lock); + afs_unlock_for_io(vnode); + } else { + list_del(&myself.link); + spin_unlock(&vnode->lock); + } + ret = -ERESTARTSYS; + } + return ret; +} + +/* + * Lock the vnode(s) being operated upon. + */ +static bool afs_get_io_locks(struct afs_operation *op) +{ + struct afs_vnode *vnode = op->file[0].vnode; + struct afs_vnode *vnode2 = op->file[1].vnode; + + _enter(""); + + if (op->flags & AFS_OPERATION_UNINTR) { + afs_lock_for_io(vnode); + op->flags |= AFS_OPERATION_LOCK_0; + _leave(" = t [1]"); + return true; + } + + if (!vnode2 || !op->file[1].need_io_lock || vnode == vnode2) + vnode2 = NULL; + + if (vnode2 > vnode) + swap(vnode, vnode2); + + if (afs_lock_for_io_interruptible(vnode) < 0) { + afs_op_set_error(op, -ERESTARTSYS); + op->flags |= AFS_OPERATION_STOP; + _leave(" = f [I 0]"); + return false; + } + op->flags |= AFS_OPERATION_LOCK_0; + + if (vnode2) { + if (afs_lock_for_io_interruptible(vnode2) < 0) { + afs_op_set_error(op, -ERESTARTSYS); + op->flags |= AFS_OPERATION_STOP; + afs_unlock_for_io(vnode); + op->flags &= ~AFS_OPERATION_LOCK_0; + _leave(" = f [I 1]"); + return false; + } + op->flags |= AFS_OPERATION_LOCK_1; + } + + _leave(" = t [2]"); + return true; +} + +static void afs_drop_io_locks(struct afs_operation *op) +{ + struct afs_vnode *vnode = op->file[0].vnode; + struct afs_vnode *vnode2 = op->file[1].vnode; + + _enter(""); + + if (op->flags & AFS_OPERATION_LOCK_1) + afs_unlock_for_io(vnode2); + if (op->flags & AFS_OPERATION_LOCK_0) + afs_unlock_for_io(vnode); +} + +static void afs_prepare_vnode(struct afs_operation *op, struct afs_vnode_param *vp, + unsigned int index) +{ + struct afs_vnode *vnode = vp->vnode; + + if (vnode) { + vp->fid = vnode->fid; + vp->dv_before = vnode->status.data_version; + vp->cb_break_before = afs_calc_vnode_cb_break(vnode); + if (vnode->lock_state != AFS_VNODE_LOCK_NONE) + op->flags |= AFS_OPERATION_CUR_ONLY; + if (vp->modification) + set_bit(AFS_VNODE_MODIFYING, &vnode->flags); + } + + if (vp->fid.vnode) + _debug("PREP[%u] {%llx:%llu.%u}", + index, vp->fid.vid, vp->fid.vnode, vp->fid.unique); +} + +/* + * Begin an operation on the fileserver. + * + * Fileserver operations are serialised on the server by vnode, so we serialise + * them here also using the io_lock. + */ +bool afs_begin_vnode_operation(struct afs_operation *op) +{ + struct afs_vnode *vnode = op->file[0].vnode; + + ASSERT(vnode); + + _enter(""); + + if (op->file[0].need_io_lock) + if (!afs_get_io_locks(op)) + return false; + + afs_prepare_vnode(op, &op->file[0], 0); + afs_prepare_vnode(op, &op->file[1], 1); + op->cb_v_break = atomic_read(&op->volume->cb_v_break); + _leave(" = true"); + return true; +} + +/* + * Tidy up a filesystem cursor and unlock the vnode. + */ +void afs_end_vnode_operation(struct afs_operation *op) +{ + _enter(""); + + switch (afs_op_error(op)) { + case -EDESTADDRREQ: + case -EADDRNOTAVAIL: + case -ENETUNREACH: + case -EHOSTUNREACH: + afs_dump_edestaddrreq(op); + break; + } + + afs_drop_io_locks(op); +} + +/* + * Wait for an in-progress operation to complete. + */ +void afs_wait_for_operation(struct afs_operation *op) +{ + _enter(""); + + while (afs_select_fileserver(op)) { + op->call_responded = false; + op->call_error = 0; + op->call_abort_code = 0; + if (test_bit(AFS_SERVER_FL_IS_YFS, &op->server->flags) && + op->ops->issue_yfs_rpc) + op->ops->issue_yfs_rpc(op); + else if (op->ops->issue_afs_rpc) + op->ops->issue_afs_rpc(op); + else + op->call_error = -ENOTSUPP; + + if (op->call) { + afs_wait_for_call_to_complete(op->call); + op->call_abort_code = op->call->abort_code; + op->call_error = op->call->error; + op->call_responded = op->call->responded; + afs_put_call(op->call); + } + } + + if (op->call_responded && op->server) + set_bit(AFS_SERVER_FL_RESPONDING, &op->server->flags); + + if (!afs_op_error(op)) { + _debug("success"); + op->ops->success(op); + } else if (op->cumul_error.aborted) { + if (op->ops->aborted) + op->ops->aborted(op); + } else { + if (op->ops->failed) + op->ops->failed(op); + } + + afs_end_vnode_operation(op); + + if (!afs_op_error(op) && op->ops->edit_dir) { + _debug("edit_dir"); + op->ops->edit_dir(op); + } + _leave(""); +} + +/* + * Dispose of an operation. + */ +int afs_put_operation(struct afs_operation *op) +{ + struct afs_addr_list *alist; + int i, ret = afs_op_error(op); + + _enter("op=%08x,%d", op->debug_id, ret); + + if (op->ops && op->ops->put) + op->ops->put(op); + if (op->file[0].modification) + clear_bit(AFS_VNODE_MODIFYING, &op->file[0].vnode->flags); + if (op->file[1].modification && op->file[1].vnode != op->file[0].vnode) + clear_bit(AFS_VNODE_MODIFYING, &op->file[1].vnode->flags); + if (op->file[0].put_vnode) + iput(&op->file[0].vnode->netfs.inode); + if (op->file[1].put_vnode) + iput(&op->file[1].vnode->netfs.inode); + + if (op->more_files) { + for (i = 0; i < op->nr_files - 2; i++) + if (op->more_files[i].put_vnode) + iput(&op->more_files[i].vnode->netfs.inode); + kvfree(op->more_files); + } + + if (op->estate) { + alist = op->estate->addresses; + if (alist) { + if (op->call_responded && + op->addr_index != alist->preferred && + test_bit(alist->preferred, &op->addr_tried)) + WRITE_ONCE(alist->preferred, op->addr_index); + } + } + + afs_clear_server_states(op); + afs_put_serverlist(op->net, op->server_list); + afs_put_volume(op->volume, afs_volume_trace_put_put_op); + key_put(op->key); + kfree(op); + return ret; +} + +int afs_do_sync_operation(struct afs_operation *op) +{ + afs_begin_vnode_operation(op); + afs_wait_for_operation(op); + return afs_put_operation(op); +} diff --git a/fs/afs/fs_probe.c b/fs/afs/fs_probe.c new file mode 100644 index 000000000..a91ad1938 --- /dev/null +++ b/fs/afs/fs_probe.c @@ -0,0 +1,539 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* AFS fileserver probing + * + * Copyright (C) 2018, 2020 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include "afs_fs.h" +#include "internal.h" +#include "protocol_afs.h" +#include "protocol_yfs.h" + +static unsigned int afs_fs_probe_fast_poll_interval = 30 * HZ; +static unsigned int afs_fs_probe_slow_poll_interval = 5 * 60 * HZ; + +struct afs_endpoint_state *afs_get_endpoint_state(struct afs_endpoint_state *estate, + enum afs_estate_trace where) +{ + if (estate) { + int r; + + __refcount_inc(&estate->ref, &r); + trace_afs_estate(estate->server_id, estate->probe_seq, r, where); + } + return estate; +} + +static void afs_endpoint_state_rcu(struct rcu_head *rcu) +{ + struct afs_endpoint_state *estate = container_of(rcu, struct afs_endpoint_state, rcu); + + trace_afs_estate(estate->server_id, estate->probe_seq, refcount_read(&estate->ref), + afs_estate_trace_free); + afs_put_addrlist(estate->addresses, afs_alist_trace_put_estate); + kfree(estate); +} + +void afs_put_endpoint_state(struct afs_endpoint_state *estate, enum afs_estate_trace where) +{ + if (estate) { + unsigned int server_id = estate->server_id, probe_seq = estate->probe_seq; + bool dead; + int r; + + dead = __refcount_dec_and_test(&estate->ref, &r); + trace_afs_estate(server_id, probe_seq, r, where); + if (dead) + call_rcu(&estate->rcu, afs_endpoint_state_rcu); + } +} + +/* + * Start the probe polling timer. We have to supply it with an inc on the + * outstanding server count. + */ +static void afs_schedule_fs_probe(struct afs_net *net, + struct afs_server *server, bool fast) +{ + unsigned long atj; + + if (!net->live) + return; + + atj = server->probed_at; + atj += fast ? afs_fs_probe_fast_poll_interval : afs_fs_probe_slow_poll_interval; + + afs_inc_servers_outstanding(net); + if (timer_reduce(&net->fs_probe_timer, atj)) + afs_dec_servers_outstanding(net); +} + +/* + * Handle the completion of a set of probes. + */ +static void afs_finished_fs_probe(struct afs_net *net, struct afs_server *server, + struct afs_endpoint_state *estate) +{ + bool responded = test_bit(AFS_ESTATE_RESPONDED, &estate->flags); + + write_seqlock(&net->fs_lock); + if (responded) { + list_add_tail(&server->probe_link, &net->fs_probe_slow); + } else { + server->rtt = UINT_MAX; + clear_bit(AFS_SERVER_FL_RESPONDING, &server->flags); + list_add_tail(&server->probe_link, &net->fs_probe_fast); + } + + write_sequnlock(&net->fs_lock); + + afs_schedule_fs_probe(net, server, !responded); +} + +/* + * Handle the completion of a probe. + */ +static void afs_done_one_fs_probe(struct afs_net *net, struct afs_server *server, + struct afs_endpoint_state *estate) +{ + _enter(""); + + if (atomic_dec_and_test(&estate->nr_probing)) + afs_finished_fs_probe(net, server, estate); + + wake_up_all(&server->probe_wq); +} + +/* + * Handle inability to send a probe due to ENOMEM when trying to allocate a + * call struct. + */ +static void afs_fs_probe_not_done(struct afs_net *net, + struct afs_server *server, + struct afs_endpoint_state *estate, + int index) +{ + _enter(""); + + trace_afs_io_error(0, -ENOMEM, afs_io_error_fs_probe_fail); + spin_lock(&server->probe_lock); + + set_bit(AFS_ESTATE_LOCAL_FAILURE, &estate->flags); + if (estate->error == 0) + estate->error = -ENOMEM; + + set_bit(index, &estate->failed_set); + + spin_unlock(&server->probe_lock); + return afs_done_one_fs_probe(net, server, estate); +} + +/* + * Process the result of probing a fileserver. This is called after successful + * or failed delivery of an FS.GetCapabilities operation. + */ +void afs_fileserver_probe_result(struct afs_call *call) +{ + struct afs_endpoint_state *estate = call->probe; + struct afs_addr_list *alist = estate->addresses; + struct afs_address *addr = &alist->addrs[call->probe_index]; + struct afs_server *server = call->server; + unsigned int index = call->probe_index; + unsigned int rtt_us = -1, cap0; + int ret = call->error; + + _enter("%pU,%u", &server->uuid, index); + + WRITE_ONCE(addr->last_error, ret); + + spin_lock(&server->probe_lock); + + switch (ret) { + case 0: + estate->error = 0; + goto responded; + case -ECONNABORTED: + if (!test_bit(AFS_ESTATE_RESPONDED, &estate->flags)) { + estate->abort_code = call->abort_code; + estate->error = ret; + } + goto responded; + case -ENOMEM: + case -ENONET: + clear_bit(index, &estate->responsive_set); + set_bit(AFS_ESTATE_LOCAL_FAILURE, &estate->flags); + trace_afs_io_error(call->debug_id, ret, afs_io_error_fs_probe_fail); + goto out; + case -ECONNRESET: /* Responded, but call expired. */ + case -ERFKILL: + case -EADDRNOTAVAIL: + case -ENETUNREACH: + case -EHOSTUNREACH: + case -EHOSTDOWN: + case -ECONNREFUSED: + case -ETIMEDOUT: + case -ETIME: + default: + clear_bit(index, &estate->responsive_set); + set_bit(index, &estate->failed_set); + if (!test_bit(AFS_ESTATE_RESPONDED, &estate->flags) && + (estate->error == 0 || + estate->error == -ETIMEDOUT || + estate->error == -ETIME)) + estate->error = ret; + trace_afs_io_error(call->debug_id, ret, afs_io_error_fs_probe_fail); + goto out; + } + +responded: + clear_bit(index, &estate->failed_set); + + if (call->service_id == YFS_FS_SERVICE) { + set_bit(AFS_ESTATE_IS_YFS, &estate->flags); + set_bit(AFS_SERVER_FL_IS_YFS, &server->flags); + server->service_id = call->service_id; + } else { + set_bit(AFS_ESTATE_NOT_YFS, &estate->flags); + if (!test_bit(AFS_ESTATE_IS_YFS, &estate->flags)) { + clear_bit(AFS_SERVER_FL_IS_YFS, &server->flags); + server->service_id = call->service_id; + } + cap0 = ntohl(call->tmp); + if (cap0 & AFS3_VICED_CAPABILITY_64BITFILES) + set_bit(AFS_SERVER_FL_HAS_FS64, &server->flags); + else + clear_bit(AFS_SERVER_FL_HAS_FS64, &server->flags); + } + + rtt_us = rxrpc_kernel_get_srtt(addr->peer); + if (rtt_us < estate->rtt) { + estate->rtt = rtt_us; + server->rtt = rtt_us; + alist->preferred = index; + } + + smp_wmb(); /* Set rtt before responded. */ + set_bit(AFS_ESTATE_RESPONDED, &estate->flags); + set_bit(index, &estate->responsive_set); + set_bit(AFS_SERVER_FL_RESPONDING, &server->flags); +out: + spin_unlock(&server->probe_lock); + + trace_afs_fs_probe(server, false, estate, index, call->error, call->abort_code, rtt_us); + _debug("probe[%x] %pU [%u] %pISpc rtt=%d ret=%d", + estate->probe_seq, &server->uuid, index, + rxrpc_kernel_remote_addr(alist->addrs[index].peer), + rtt_us, ret); + + return afs_done_one_fs_probe(call->net, server, estate); +} + +/* + * Probe all of a fileserver's addresses to find out the best route and to + * query its capabilities. + */ +int afs_fs_probe_fileserver(struct afs_net *net, struct afs_server *server, + struct afs_addr_list *new_alist, struct key *key) +{ + struct afs_endpoint_state *estate, *old; + struct afs_addr_list *old_alist = NULL, *alist; + unsigned long unprobed; + + _enter("%pU", &server->uuid); + + estate = kzalloc_obj(*estate); + if (!estate) + return -ENOMEM; + + refcount_set(&estate->ref, 2); + estate->server_id = server->debug_id; + estate->rtt = UINT_MAX; + + write_lock(&server->fs_lock); + + old = rcu_dereference_protected(server->endpoint_state, + lockdep_is_held(&server->fs_lock)); + if (old) { + estate->responsive_set = old->responsive_set; + if (!new_alist) + new_alist = old->addresses; + } + + if (old_alist != new_alist) + afs_set_peer_appdata(server, old_alist, new_alist); + + estate->addresses = afs_get_addrlist(new_alist, afs_alist_trace_get_estate); + alist = estate->addresses; + estate->probe_seq = ++server->probe_counter; + atomic_set(&estate->nr_probing, alist->nr_addrs); + + if (new_alist) + server->addr_version = new_alist->version; + rcu_assign_pointer(server->endpoint_state, estate); + write_unlock(&server->fs_lock); + if (old) + set_bit(AFS_ESTATE_SUPERSEDED, &old->flags); + + trace_afs_estate(estate->server_id, estate->probe_seq, refcount_read(&estate->ref), + afs_estate_trace_alloc_probe); + + afs_get_address_preferences(net, new_alist); + + server->probed_at = jiffies; + unprobed = (1UL << alist->nr_addrs) - 1; + while (unprobed) { + unsigned int index = 0, i; + int best_prio = -1; + + for (i = 0; i < alist->nr_addrs; i++) { + if (test_bit(i, &unprobed) && + alist->addrs[i].prio > best_prio) { + index = i; + best_prio = alist->addrs[i].prio; + } + } + __clear_bit(index, &unprobed); + + trace_afs_fs_probe(server, true, estate, index, 0, 0, 0); + if (!afs_fs_get_capabilities(net, server, estate, index, key)) + afs_fs_probe_not_done(net, server, estate, index); + } + + afs_put_endpoint_state(old, afs_estate_trace_put_probe); + afs_put_endpoint_state(estate, afs_estate_trace_put_probe); + return 0; +} + +/* + * Wait for the first as-yet untried fileserver to respond, for the probe state + * to be superseded or for all probes to finish. + */ +int afs_wait_for_fs_probes(struct afs_operation *op, struct afs_server_state *states, bool intr) +{ + struct afs_endpoint_state *estate; + struct afs_server_list *slist = op->server_list; + bool still_probing = true; + int ret = 0, i; + + _enter("%u", slist->nr_servers); + + for (i = 0; i < slist->nr_servers; i++) { + estate = states[i].endpoint_state; + if (test_bit(AFS_ESTATE_SUPERSEDED, &estate->flags)) + return 2; + if (atomic_read(&estate->nr_probing)) + still_probing = true; + if (estate->responsive_set & states[i].untried_addrs) + return 1; + } + if (!still_probing) + return 0; + + for (i = 0; i < slist->nr_servers; i++) + add_wait_queue(&slist->servers[i].server->probe_wq, &states[i].probe_waiter); + + for (;;) { + still_probing = false; + + set_current_state(intr ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE); + for (i = 0; i < slist->nr_servers; i++) { + estate = states[i].endpoint_state; + if (test_bit(AFS_ESTATE_SUPERSEDED, &estate->flags)) { + ret = 2; + goto stop; + } + if (atomic_read(&estate->nr_probing)) + still_probing = true; + if (estate->responsive_set & states[i].untried_addrs) { + ret = 1; + goto stop; + } + } + + if (!still_probing || signal_pending(current)) + goto stop; + schedule(); + } + +stop: + set_current_state(TASK_RUNNING); + + for (i = 0; i < slist->nr_servers; i++) + remove_wait_queue(&slist->servers[i].server->probe_wq, &states[i].probe_waiter); + + if (!ret && signal_pending(current)) + ret = -ERESTARTSYS; + return ret; +} + +/* + * Probe timer. We have an increment on fs_outstanding that we need to pass + * along to the work item. + */ +void afs_fs_probe_timer(struct timer_list *timer) +{ + struct afs_net *net = container_of(timer, struct afs_net, fs_probe_timer); + + if (!net->live || !queue_work(afs_wq, &net->fs_prober)) + afs_dec_servers_outstanding(net); +} + +/* + * Dispatch a probe to a server. + */ +static void afs_dispatch_fs_probe(struct afs_net *net, struct afs_server *server) + __releases(&net->fs_lock) +{ + struct key *key = NULL; + + /* We remove it from the queues here - it will be added back to + * one of the queues on the completion of the probe. + */ + list_del_init(&server->probe_link); + + afs_get_server(server, afs_server_trace_get_probe); + write_sequnlock(&net->fs_lock); + + afs_fs_probe_fileserver(net, server, NULL, key); + afs_put_server(net, server, afs_server_trace_put_probe); +} + +/* + * Probe a server immediately without waiting for its due time to come + * round. This is used when all of the addresses have been tried. + */ +void afs_probe_fileserver(struct afs_net *net, struct afs_server *server) +{ + write_seqlock(&net->fs_lock); + if (!list_empty(&server->probe_link)) + return afs_dispatch_fs_probe(net, server); + write_sequnlock(&net->fs_lock); +} + +/* + * Probe dispatcher to regularly dispatch probes to keep NAT alive. + */ +void afs_fs_probe_dispatcher(struct work_struct *work) +{ + struct afs_net *net = container_of(work, struct afs_net, fs_prober); + struct afs_server *fast, *slow, *server; + unsigned long nowj, timer_at, poll_at; + bool first_pass = true, set_timer = false; + + if (!net->live) { + afs_dec_servers_outstanding(net); + return; + } + + _enter(""); + + if (list_empty(&net->fs_probe_fast) && list_empty(&net->fs_probe_slow)) { + afs_dec_servers_outstanding(net); + _leave(" [none]"); + return; + } + +again: + write_seqlock(&net->fs_lock); + + fast = slow = server = NULL; + nowj = jiffies; + timer_at = nowj + MAX_JIFFY_OFFSET; + + if (!list_empty(&net->fs_probe_fast)) { + fast = list_first_entry(&net->fs_probe_fast, struct afs_server, probe_link); + poll_at = fast->probed_at + afs_fs_probe_fast_poll_interval; + if (time_before(nowj, poll_at)) { + timer_at = poll_at; + set_timer = true; + fast = NULL; + } + } + + if (!list_empty(&net->fs_probe_slow)) { + slow = list_first_entry(&net->fs_probe_slow, struct afs_server, probe_link); + poll_at = slow->probed_at + afs_fs_probe_slow_poll_interval; + if (time_before(nowj, poll_at)) { + if (time_before(poll_at, timer_at)) + timer_at = poll_at; + set_timer = true; + slow = NULL; + } + } + + server = fast ?: slow; + if (server) + _debug("probe %pU", &server->uuid); + + if (server && (first_pass || !need_resched())) { + afs_dispatch_fs_probe(net, server); + first_pass = false; + goto again; + } + + write_sequnlock(&net->fs_lock); + + if (server) { + if (!queue_work(afs_wq, &net->fs_prober)) + afs_dec_servers_outstanding(net); + _leave(" [requeue]"); + } else if (set_timer) { + if (timer_reduce(&net->fs_probe_timer, timer_at)) + afs_dec_servers_outstanding(net); + _leave(" [timer]"); + } else { + afs_dec_servers_outstanding(net); + _leave(" [quiesce]"); + } +} + +/* + * Wait for a probe on a particular fileserver to complete for 2s. + */ +int afs_wait_for_one_fs_probe(struct afs_server *server, struct afs_endpoint_state *estate, + unsigned long exclude, bool is_intr) +{ + struct wait_queue_entry wait; + unsigned long timo = 2 * HZ; + + if (atomic_read(&estate->nr_probing) == 0) + goto dont_wait; + + init_wait_entry(&wait, 0); + for (;;) { + prepare_to_wait_event(&server->probe_wq, &wait, + is_intr ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE); + if (timo == 0 || + test_bit(AFS_ESTATE_SUPERSEDED, &estate->flags) || + (estate->responsive_set & ~exclude) || + atomic_read(&estate->nr_probing) == 0 || + (is_intr && signal_pending(current))) + break; + timo = schedule_timeout(timo); + } + + finish_wait(&server->probe_wq, &wait); + +dont_wait: + if (test_bit(AFS_ESTATE_SUPERSEDED, &estate->flags)) + return 0; + if (estate->responsive_set & ~exclude) + return 1; + if (is_intr && signal_pending(current)) + return -ERESTARTSYS; + if (timo == 0) + return -ETIME; + return -EDESTADDRREQ; +} + +/* + * Clean up the probing when the namespace is killed off. + */ +void afs_fs_probe_cleanup(struct afs_net *net) +{ + if (timer_delete_sync(&net->fs_probe_timer)) + afs_dec_servers_outstanding(net); +} diff --git a/fs/afs/fsclient.c b/fs/afs/fsclient.c new file mode 100644 index 000000000..1a3f186a6 --- /dev/null +++ b/fs/afs/fsclient.c @@ -0,0 +1,2137 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* AFS File Server client stubs + * + * Copyright (C) 2002, 2007 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include "internal.h" +#include "afs_fs.h" +#include "xdr_fs.h" + +/* + * decode an AFSFid block + */ +static void xdr_decode_AFSFid(const __be32 **_bp, struct afs_fid *fid) +{ + const __be32 *bp = *_bp; + + fid->vid = ntohl(*bp++); + fid->vnode = ntohl(*bp++); + fid->unique = ntohl(*bp++); + *_bp = bp; +} + +/* + * Dump a bad file status record. + */ +static void xdr_dump_bad(const __be32 *bp) +{ + __be32 x[4]; + int i; + + pr_notice("AFS XDR: Bad status record\n"); + for (i = 0; i < 5 * 4 * 4; i += 16) { + memcpy(x, bp, 16); + bp += 4; + pr_notice("%03x: %08x %08x %08x %08x\n", + i, ntohl(x[0]), ntohl(x[1]), ntohl(x[2]), ntohl(x[3])); + } + + memcpy(x, bp, 4); + pr_notice("0x50: %08x\n", ntohl(x[0])); +} + +/* + * decode an AFSFetchStatus block + */ +static void xdr_decode_AFSFetchStatus(const __be32 **_bp, + struct afs_call *call, + struct afs_status_cb *scb) +{ + const struct afs_xdr_AFSFetchStatus *xdr = (const void *)*_bp; + struct afs_file_status *status = &scb->status; + bool inline_error = (call->operation_ID == afs_FS_InlineBulkStatus); + u64 data_version, size; + u32 type, abort_code; + + abort_code = ntohl(xdr->abort_code); + + if (xdr->if_version != htonl(AFS_FSTATUS_VERSION)) { + if (xdr->if_version == htonl(0) && + abort_code != 0 && + inline_error) { + /* The OpenAFS fileserver has a bug in FS.InlineBulkStatus + * whereby it doesn't set the interface version in the error + * case. + */ + status->abort_code = abort_code; + scb->have_error = true; + goto advance; + } + + pr_warn("Unknown AFSFetchStatus version %u\n", ntohl(xdr->if_version)); + goto bad; + } + + if (abort_code != 0 && inline_error) { + status->abort_code = abort_code; + scb->have_error = true; + goto advance; + } + + type = ntohl(xdr->type); + switch (type) { + case AFS_FTYPE_FILE: + case AFS_FTYPE_DIR: + case AFS_FTYPE_SYMLINK: + status->type = type; + break; + default: + goto bad; + } + + status->nlink = ntohl(xdr->nlink); + status->author = ntohl(xdr->author); + status->owner = ntohl(xdr->owner); + status->caller_access = ntohl(xdr->caller_access); /* Ticket dependent */ + status->anon_access = ntohl(xdr->anon_access); + status->mode = ntohl(xdr->mode) & S_IALLUGO; + status->group = ntohl(xdr->group); + status->lock_count = ntohl(xdr->lock_count); + + status->mtime_client.tv_sec = ntohl(xdr->mtime_client); + status->mtime_client.tv_nsec = 0; + status->mtime_server.tv_sec = ntohl(xdr->mtime_server); + status->mtime_server.tv_nsec = 0; + + size = (u64)ntohl(xdr->size_lo); + size |= (u64)ntohl(xdr->size_hi) << 32; + status->size = size; + + data_version = (u64)ntohl(xdr->data_version_lo); + data_version |= (u64)ntohl(xdr->data_version_hi) << 32; + status->data_version = data_version; + scb->have_status = true; +advance: + *_bp = (const void *)*_bp + sizeof(*xdr); + return; + +bad: + xdr_dump_bad(*_bp); + afs_protocol_error(call, afs_eproto_bad_status); + goto advance; +} + +static time64_t xdr_decode_expiry(struct afs_call *call, u32 expiry) +{ + return ktime_divns(call->issue_time, NSEC_PER_SEC) + expiry; +} + +static void xdr_decode_AFSCallBack(const __be32 **_bp, + struct afs_call *call, + struct afs_status_cb *scb) +{ + struct afs_callback *cb = &scb->callback; + const __be32 *bp = *_bp; + + bp++; /* version */ + cb->expires_at = xdr_decode_expiry(call, ntohl(*bp++)); + bp++; /* type */ + scb->have_cb = true; + *_bp = bp; +} + +/* + * decode an AFSVolSync block + */ +static void xdr_decode_AFSVolSync(const __be32 **_bp, + struct afs_volsync *volsync) +{ + const __be32 *bp = *_bp; + u32 creation; + + creation = ntohl(*bp++); + bp++; /* spare2 */ + bp++; /* spare3 */ + bp++; /* spare4 */ + bp++; /* spare5 */ + bp++; /* spare6 */ + *_bp = bp; + + if (volsync) + volsync->creation = creation; +} + +/* + * encode the requested attributes into an AFSStoreStatus block + */ +static void xdr_encode_AFS_StoreStatus(__be32 **_bp, struct iattr *attr) +{ + __be32 *bp = *_bp; + u32 mask = 0, mtime = 0, owner = 0, group = 0, mode = 0; + + mask = 0; + if (attr->ia_valid & ATTR_MTIME) { + mask |= AFS_SET_MTIME; + mtime = attr->ia_mtime.tv_sec; + } + + if (attr->ia_valid & ATTR_UID) { + mask |= AFS_SET_OWNER; + owner = from_kuid(&init_user_ns, attr->ia_uid); + } + + if (attr->ia_valid & ATTR_GID) { + mask |= AFS_SET_GROUP; + group = from_kgid(&init_user_ns, attr->ia_gid); + } + + if (attr->ia_valid & ATTR_MODE) { + mask |= AFS_SET_MODE; + mode = attr->ia_mode & S_IALLUGO; + } + + *bp++ = htonl(mask); + *bp++ = htonl(mtime); + *bp++ = htonl(owner); + *bp++ = htonl(group); + *bp++ = htonl(mode); + *bp++ = 0; /* segment size */ + *_bp = bp; +} + +/* + * decode an AFSFetchVolumeStatus block + */ +static void xdr_decode_AFSFetchVolumeStatus(const __be32 **_bp, + struct afs_volume_status *vs) +{ + const __be32 *bp = *_bp; + + vs->vid = ntohl(*bp++); + vs->parent_id = ntohl(*bp++); + vs->online = ntohl(*bp++); + vs->in_service = ntohl(*bp++); + vs->blessed = ntohl(*bp++); + vs->needs_salvage = ntohl(*bp++); + vs->type = ntohl(*bp++); + vs->min_quota = ntohl(*bp++); + vs->max_quota = ntohl(*bp++); + vs->blocks_in_use = ntohl(*bp++); + vs->part_blocks_avail = ntohl(*bp++); + vs->part_max_blocks = ntohl(*bp++); + vs->vol_copy_date = 0; + vs->vol_backup_date = 0; + *_bp = bp; +} + +/* + * deliver reply data to an FS.FetchStatus + */ +static int afs_deliver_fs_fetch_status(struct afs_call *call) +{ + struct afs_operation *op = call->op; + struct afs_vnode_param *vp = &op->file[op->fetch_status.which]; + const __be32 *bp; + int ret; + + ret = afs_transfer_reply(call); + if (ret < 0) + return ret; + + /* unmarshall the reply once we've received all of it */ + bp = call->buffer; + xdr_decode_AFSFetchStatus(&bp, call, &vp->scb); + xdr_decode_AFSCallBack(&bp, call, &vp->scb); + xdr_decode_AFSVolSync(&bp, &op->volsync); + + _leave(" = 0 [done]"); + return 0; +} + +/* + * FS.FetchStatus operation type + */ +static const struct afs_call_type afs_RXFSFetchStatus = { + .name = "FS.FetchStatus", + .op = afs_FS_FetchStatus, + .deliver = afs_deliver_fs_fetch_status, + .destructor = afs_flat_call_destructor, +}; + +/* + * fetch the status information for a file + */ +void afs_fs_fetch_status(struct afs_operation *op) +{ + struct afs_vnode_param *vp = &op->file[op->fetch_status.which]; + struct afs_call *call; + __be32 *bp; + + _enter(",%x,{%llx:%llu},,", + key_serial(op->key), vp->fid.vid, vp->fid.vnode); + + call = afs_alloc_flat_call(op->net, &afs_RXFSFetchStatus, + 16, (21 + 3 + 6) * 4); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + bp[0] = htonl(FSFETCHSTATUS); + bp[1] = htonl(vp->fid.vid); + bp[2] = htonl(vp->fid.vnode); + bp[3] = htonl(vp->fid.unique); + + call->fid = vp->fid; + trace_afs_make_fs_call(call, &vp->fid); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * deliver reply data to an FS.FetchData + */ +static int afs_deliver_fs_fetch_data(struct afs_call *call) +{ + struct afs_operation *op = call->op; + struct netfs_io_subrequest *subreq = op->fetch.subreq; + struct afs_vnode_param *vp = &op->file[0]; + const __be32 *bp; + size_t count_before; + int ret; + + _enter("{%u,%zu,%zu/%llu}", + call->unmarshall, call->iov_len, iov_iter_count(call->iter), + call->remaining); + + switch (call->unmarshall) { + case 0: + call->remaining = 0; + call->unmarshall++; + if (call->operation_ID == FSFETCHDATA64) { + afs_extract_to_tmp64(call); + } else { + call->tmp_u = htonl(0); + afs_extract_to_tmp(call); + } + fallthrough; + + /* Extract the returned data length into ->remaining. + * This may indicate more or less data than was + * requested will be returned. + */ + case 1: + _debug("extract data length"); + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + call->remaining = be64_to_cpu(call->tmp64); + _debug("DATA length: %llu", call->remaining); + + if (call->remaining == 0) + goto no_more_data; + + call->iter = &subreq->io_iter; + call->iov_len = umin(call->remaining, subreq->len - subreq->transferred); + call->unmarshall++; + fallthrough; + + /* extract the returned data */ + case 2: + count_before = call->iov_len; + _debug("extract data %zu/%llu", count_before, call->remaining); + + ret = afs_extract_data(call, true); + subreq->transferred += count_before - call->iov_len; + call->remaining -= count_before - call->iov_len; + if (ret < 0) + return ret; + + call->iter = &call->def_iter; + if (call->remaining) + goto no_more_data; + + /* Discard any excess data the server gave us */ + afs_extract_discard(call, call->remaining); + call->unmarshall = 3; + fallthrough; + + case 3: + _debug("extract discard %zu/%llu", + iov_iter_count(call->iter), call->remaining); + + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + no_more_data: + call->unmarshall = 4; + afs_extract_to_buf(call, (21 + 3 + 6) * 4); + fallthrough; + + /* extract the metadata */ + case 4: + ret = afs_extract_data(call, false); + if (ret < 0) + return ret; + + bp = call->buffer; + xdr_decode_AFSFetchStatus(&bp, call, &vp->scb); + xdr_decode_AFSCallBack(&bp, call, &vp->scb); + xdr_decode_AFSVolSync(&bp, &op->volsync); + + if (subreq->start + subreq->transferred >= vp->scb.status.size) + __set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags); + + call->unmarshall++; + fallthrough; + + case 5: + break; + } + + _leave(" = 0 [done]"); + return 0; +} + +/* + * FS.FetchData operation type + */ +static const struct afs_call_type afs_RXFSFetchData = { + .name = "FS.FetchData", + .op = afs_FS_FetchData, + .async_rx = afs_fetch_data_async_rx, + .deliver = afs_deliver_fs_fetch_data, + .immediate_cancel = afs_fetch_data_immediate_cancel, + .destructor = afs_flat_call_destructor, +}; + +static const struct afs_call_type afs_RXFSFetchData64 = { + .name = "FS.FetchData64", + .op = afs_FS_FetchData64, + .async_rx = afs_fetch_data_async_rx, + .deliver = afs_deliver_fs_fetch_data, + .immediate_cancel = afs_fetch_data_immediate_cancel, + .destructor = afs_flat_call_destructor, +}; + +/* + * fetch data from a very large file + */ +static void afs_fs_fetch_data64(struct afs_operation *op) +{ + struct netfs_io_subrequest *subreq = op->fetch.subreq; + struct afs_vnode_param *vp = &op->file[0]; + struct afs_call *call; + __be32 *bp; + + _enter(""); + + call = afs_alloc_flat_call(op->net, &afs_RXFSFetchData64, 32, (21 + 3 + 6) * 4); + if (!call) + return afs_op_nomem(op); + + if (op->flags & AFS_OPERATION_ASYNC) + call->async = true; + + /* marshall the parameters */ + bp = call->request; + bp[0] = htonl(FSFETCHDATA64); + bp[1] = htonl(vp->fid.vid); + bp[2] = htonl(vp->fid.vnode); + bp[3] = htonl(vp->fid.unique); + bp[4] = htonl(upper_32_bits(subreq->start + subreq->transferred)); + bp[5] = htonl(lower_32_bits(subreq->start + subreq->transferred)); + bp[6] = 0; + bp[7] = htonl(lower_32_bits(subreq->len - subreq->transferred)); + + call->fid = vp->fid; + trace_afs_make_fs_call(call, &vp->fid); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * fetch data from a file + */ +void afs_fs_fetch_data(struct afs_operation *op) +{ + struct netfs_io_subrequest *subreq = op->fetch.subreq; + struct afs_vnode_param *vp = &op->file[0]; + struct afs_call *call; + __be32 *bp; + + if (test_bit(AFS_SERVER_FL_HAS_FS64, &op->server->flags)) + return afs_fs_fetch_data64(op); + + _enter(""); + + call = afs_alloc_flat_call(op->net, &afs_RXFSFetchData, 24, (21 + 3 + 6) * 4); + if (!call) + return afs_op_nomem(op); + + if (op->flags & AFS_OPERATION_ASYNC) + call->async = true; + + /* marshall the parameters */ + bp = call->request; + bp[0] = htonl(FSFETCHDATA); + bp[1] = htonl(vp->fid.vid); + bp[2] = htonl(vp->fid.vnode); + bp[3] = htonl(vp->fid.unique); + bp[4] = htonl(lower_32_bits(subreq->start + subreq->transferred)); + bp[5] = htonl(lower_32_bits(subreq->len - subreq->transferred)); + + call->fid = vp->fid; + trace_afs_make_fs_call(call, &vp->fid); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * deliver reply data to an FS.CreateFile or an FS.MakeDir + */ +static int afs_deliver_fs_create_vnode(struct afs_call *call) +{ + struct afs_operation *op = call->op; + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_vnode_param *vp = &op->file[1]; + const __be32 *bp; + int ret; + + ret = afs_transfer_reply(call); + if (ret < 0) + return ret; + + /* unmarshall the reply once we've received all of it */ + bp = call->buffer; + xdr_decode_AFSFid(&bp, &op->file[1].fid); + xdr_decode_AFSFetchStatus(&bp, call, &vp->scb); + xdr_decode_AFSFetchStatus(&bp, call, &dvp->scb); + xdr_decode_AFSCallBack(&bp, call, &vp->scb); + xdr_decode_AFSVolSync(&bp, &op->volsync); + + _leave(" = 0 [done]"); + return 0; +} + +/* + * FS.CreateFile and FS.MakeDir operation type + */ +static const struct afs_call_type afs_RXFSCreateFile = { + .name = "FS.CreateFile", + .op = afs_FS_CreateFile, + .deliver = afs_deliver_fs_create_vnode, + .destructor = afs_flat_call_destructor, +}; + +/* + * Create a file. + */ +void afs_fs_create_file(struct afs_operation *op) +{ + const struct qstr *name = &op->dentry->d_name; + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_call *call; + size_t namesz, reqsz, padsz; + __be32 *bp; + + _enter(""); + + namesz = name->len; + padsz = (4 - (namesz & 3)) & 3; + reqsz = (5 * 4) + namesz + padsz + (6 * 4); + + call = afs_alloc_flat_call(op->net, &afs_RXFSCreateFile, + reqsz, (3 + 21 + 21 + 3 + 6) * 4); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + *bp++ = htonl(FSCREATEFILE); + *bp++ = htonl(dvp->fid.vid); + *bp++ = htonl(dvp->fid.vnode); + *bp++ = htonl(dvp->fid.unique); + *bp++ = htonl(namesz); + memcpy(bp, name->name, namesz); + bp = (void *) bp + namesz; + if (padsz > 0) { + memset(bp, 0, padsz); + bp = (void *) bp + padsz; + } + *bp++ = htonl(AFS_SET_MODE | AFS_SET_MTIME); + *bp++ = htonl(op->mtime.tv_sec); /* mtime */ + *bp++ = 0; /* owner */ + *bp++ = 0; /* group */ + *bp++ = htonl(op->create.mode & S_IALLUGO); /* unix mode */ + *bp++ = 0; /* segment size */ + + call->fid = dvp->fid; + trace_afs_make_fs_call1(call, &dvp->fid, name); + afs_make_op_call(op, call, GFP_NOFS); +} + +static const struct afs_call_type afs_RXFSMakeDir = { + .name = "FS.MakeDir", + .op = afs_FS_MakeDir, + .deliver = afs_deliver_fs_create_vnode, + .destructor = afs_flat_call_destructor, +}; + +/* + * Create a new directory + */ +void afs_fs_make_dir(struct afs_operation *op) +{ + const struct qstr *name = &op->dentry->d_name; + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_call *call; + size_t namesz, reqsz, padsz; + __be32 *bp; + + _enter(""); + + namesz = name->len; + padsz = (4 - (namesz & 3)) & 3; + reqsz = (5 * 4) + namesz + padsz + (6 * 4); + + call = afs_alloc_flat_call(op->net, &afs_RXFSMakeDir, + reqsz, (3 + 21 + 21 + 3 + 6) * 4); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + *bp++ = htonl(FSMAKEDIR); + *bp++ = htonl(dvp->fid.vid); + *bp++ = htonl(dvp->fid.vnode); + *bp++ = htonl(dvp->fid.unique); + *bp++ = htonl(namesz); + memcpy(bp, name->name, namesz); + bp = (void *) bp + namesz; + if (padsz > 0) { + memset(bp, 0, padsz); + bp = (void *) bp + padsz; + } + *bp++ = htonl(AFS_SET_MODE | AFS_SET_MTIME); + *bp++ = htonl(op->mtime.tv_sec); /* mtime */ + *bp++ = 0; /* owner */ + *bp++ = 0; /* group */ + *bp++ = htonl(op->create.mode & S_IALLUGO); /* unix mode */ + *bp++ = 0; /* segment size */ + + call->fid = dvp->fid; + trace_afs_make_fs_call1(call, &dvp->fid, name); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * Deliver reply data to any operation that returns status and volume sync. + */ +static int afs_deliver_fs_file_status_and_vol(struct afs_call *call) +{ + struct afs_operation *op = call->op; + struct afs_vnode_param *vp = &op->file[0]; + const __be32 *bp; + int ret; + + ret = afs_transfer_reply(call); + if (ret < 0) + return ret; + + /* unmarshall the reply once we've received all of it */ + bp = call->buffer; + xdr_decode_AFSFetchStatus(&bp, call, &vp->scb); + xdr_decode_AFSVolSync(&bp, &op->volsync); + + _leave(" = 0 [done]"); + return 0; +} + +/* + * FS.RemoveFile operation type + */ +static const struct afs_call_type afs_RXFSRemoveFile = { + .name = "FS.RemoveFile", + .op = afs_FS_RemoveFile, + .deliver = afs_deliver_fs_file_status_and_vol, + .destructor = afs_flat_call_destructor, +}; + +/* + * Remove a file. + */ +void afs_fs_remove_file(struct afs_operation *op) +{ + const struct qstr *name = &op->dentry->d_name; + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_call *call; + size_t namesz, reqsz, padsz; + __be32 *bp; + + _enter(""); + + namesz = name->len; + padsz = (4 - (namesz & 3)) & 3; + reqsz = (5 * 4) + namesz + padsz; + + call = afs_alloc_flat_call(op->net, &afs_RXFSRemoveFile, + reqsz, (21 + 6) * 4); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + *bp++ = htonl(FSREMOVEFILE); + *bp++ = htonl(dvp->fid.vid); + *bp++ = htonl(dvp->fid.vnode); + *bp++ = htonl(dvp->fid.unique); + *bp++ = htonl(namesz); + memcpy(bp, name->name, namesz); + bp = (void *) bp + namesz; + if (padsz > 0) { + memset(bp, 0, padsz); + bp = (void *) bp + padsz; + } + + call->fid = dvp->fid; + trace_afs_make_fs_call1(call, &dvp->fid, name); + afs_make_op_call(op, call, GFP_NOFS); +} + +static const struct afs_call_type afs_RXFSRemoveDir = { + .name = "FS.RemoveDir", + .op = afs_FS_RemoveDir, + .deliver = afs_deliver_fs_file_status_and_vol, + .destructor = afs_flat_call_destructor, +}; + +/* + * Remove a directory. + */ +void afs_fs_remove_dir(struct afs_operation *op) +{ + const struct qstr *name = &op->dentry->d_name; + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_call *call; + size_t namesz, reqsz, padsz; + __be32 *bp; + + _enter(""); + + namesz = name->len; + padsz = (4 - (namesz & 3)) & 3; + reqsz = (5 * 4) + namesz + padsz; + + call = afs_alloc_flat_call(op->net, &afs_RXFSRemoveDir, + reqsz, (21 + 6) * 4); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + *bp++ = htonl(FSREMOVEDIR); + *bp++ = htonl(dvp->fid.vid); + *bp++ = htonl(dvp->fid.vnode); + *bp++ = htonl(dvp->fid.unique); + *bp++ = htonl(namesz); + memcpy(bp, name->name, namesz); + bp = (void *) bp + namesz; + if (padsz > 0) { + memset(bp, 0, padsz); + bp = (void *) bp + padsz; + } + + call->fid = dvp->fid; + trace_afs_make_fs_call1(call, &dvp->fid, name); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * deliver reply data to an FS.Link + */ +static int afs_deliver_fs_link(struct afs_call *call) +{ + struct afs_operation *op = call->op; + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_vnode_param *vp = &op->file[1]; + const __be32 *bp; + int ret; + + _enter("{%u}", call->unmarshall); + + ret = afs_transfer_reply(call); + if (ret < 0) + return ret; + + /* unmarshall the reply once we've received all of it */ + bp = call->buffer; + xdr_decode_AFSFetchStatus(&bp, call, &vp->scb); + xdr_decode_AFSFetchStatus(&bp, call, &dvp->scb); + xdr_decode_AFSVolSync(&bp, &op->volsync); + + _leave(" = 0 [done]"); + return 0; +} + +/* + * FS.Link operation type + */ +static const struct afs_call_type afs_RXFSLink = { + .name = "FS.Link", + .op = afs_FS_Link, + .deliver = afs_deliver_fs_link, + .destructor = afs_flat_call_destructor, +}; + +/* + * make a hard link + */ +void afs_fs_link(struct afs_operation *op) +{ + const struct qstr *name = &op->dentry->d_name; + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_vnode_param *vp = &op->file[1]; + struct afs_call *call; + size_t namesz, reqsz, padsz; + __be32 *bp; + + _enter(""); + + namesz = name->len; + padsz = (4 - (namesz & 3)) & 3; + reqsz = (5 * 4) + namesz + padsz + (3 * 4); + + call = afs_alloc_flat_call(op->net, &afs_RXFSLink, reqsz, (21 + 21 + 6) * 4); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + *bp++ = htonl(FSLINK); + *bp++ = htonl(dvp->fid.vid); + *bp++ = htonl(dvp->fid.vnode); + *bp++ = htonl(dvp->fid.unique); + *bp++ = htonl(namesz); + memcpy(bp, name->name, namesz); + bp = (void *) bp + namesz; + if (padsz > 0) { + memset(bp, 0, padsz); + bp = (void *) bp + padsz; + } + *bp++ = htonl(vp->fid.vid); + *bp++ = htonl(vp->fid.vnode); + *bp++ = htonl(vp->fid.unique); + + call->fid = vp->fid; + trace_afs_make_fs_call1(call, &vp->fid, name); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * deliver reply data to an FS.Symlink + */ +static int afs_deliver_fs_symlink(struct afs_call *call) +{ + struct afs_operation *op = call->op; + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_vnode_param *vp = &op->file[1]; + const __be32 *bp; + int ret; + + _enter("{%u}", call->unmarshall); + + ret = afs_transfer_reply(call); + if (ret < 0) + return ret; + + /* unmarshall the reply once we've received all of it */ + bp = call->buffer; + xdr_decode_AFSFid(&bp, &vp->fid); + xdr_decode_AFSFetchStatus(&bp, call, &vp->scb); + xdr_decode_AFSFetchStatus(&bp, call, &dvp->scb); + xdr_decode_AFSVolSync(&bp, &op->volsync); + + _leave(" = 0 [done]"); + return 0; +} + +/* + * FS.Symlink operation type + */ +static const struct afs_call_type afs_RXFSSymlink = { + .name = "FS.Symlink", + .op = afs_FS_Symlink, + .deliver = afs_deliver_fs_symlink, + .destructor = afs_flat_call_destructor, +}; + +/* + * create a symbolic link + */ +void afs_fs_symlink(struct afs_operation *op) +{ + const struct qstr *name = &op->dentry->d_name; + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_call *call; + size_t namesz, reqsz, padsz, c_namesz, c_padsz; + __be32 *bp; + + _enter(""); + + namesz = name->len; + padsz = (4 - (namesz & 3)) & 3; + + c_namesz = strlen(op->create.symlink->content); + c_padsz = (4 - (c_namesz & 3)) & 3; + + reqsz = (6 * 4) + namesz + padsz + c_namesz + c_padsz + (6 * 4); + + call = afs_alloc_flat_call(op->net, &afs_RXFSSymlink, reqsz, + (3 + 21 + 21 + 6) * 4); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + *bp++ = htonl(FSSYMLINK); + *bp++ = htonl(dvp->fid.vid); + *bp++ = htonl(dvp->fid.vnode); + *bp++ = htonl(dvp->fid.unique); + *bp++ = htonl(namesz); + memcpy(bp, name->name, namesz); + bp = (void *) bp + namesz; + if (padsz > 0) { + memset(bp, 0, padsz); + bp = (void *) bp + padsz; + } + *bp++ = htonl(c_namesz); + memcpy(bp, op->create.symlink->content, c_namesz); + bp = (void *) bp + c_namesz; + if (c_padsz > 0) { + memset(bp, 0, c_padsz); + bp = (void *) bp + c_padsz; + } + *bp++ = htonl(AFS_SET_MODE | AFS_SET_MTIME); + *bp++ = htonl(op->mtime.tv_sec); /* mtime */ + *bp++ = 0; /* owner */ + *bp++ = 0; /* group */ + *bp++ = htonl(S_IRWXUGO); /* unix mode */ + *bp++ = 0; /* segment size */ + + call->fid = dvp->fid; + trace_afs_make_fs_call1(call, &dvp->fid, name); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * deliver reply data to an FS.Rename + */ +static int afs_deliver_fs_rename(struct afs_call *call) +{ + struct afs_operation *op = call->op; + struct afs_vnode_param *orig_dvp = &op->file[0]; + struct afs_vnode_param *new_dvp = &op->file[1]; + const __be32 *bp; + int ret; + + ret = afs_transfer_reply(call); + if (ret < 0) + return ret; + + bp = call->buffer; + /* If the two dirs are the same, we have two copies of the same status + * report, so we just decode it twice. + */ + xdr_decode_AFSFetchStatus(&bp, call, &orig_dvp->scb); + xdr_decode_AFSFetchStatus(&bp, call, &new_dvp->scb); + xdr_decode_AFSVolSync(&bp, &op->volsync); + + _leave(" = 0 [done]"); + return 0; +} + +/* + * FS.Rename operation type + */ +static const struct afs_call_type afs_RXFSRename = { + .name = "FS.Rename", + .op = afs_FS_Rename, + .deliver = afs_deliver_fs_rename, + .destructor = afs_flat_call_destructor, +}; + +/* + * Rename/move a file or directory. + */ +void afs_fs_rename(struct afs_operation *op) +{ + struct afs_vnode_param *orig_dvp = &op->file[0]; + struct afs_vnode_param *new_dvp = &op->file[1]; + const struct qstr *orig_name = &op->dentry->d_name; + const struct qstr *new_name = &op->dentry_2->d_name; + struct afs_call *call; + size_t reqsz, o_namesz, o_padsz, n_namesz, n_padsz; + __be32 *bp; + + _enter(""); + + o_namesz = orig_name->len; + o_padsz = (4 - (o_namesz & 3)) & 3; + + n_namesz = new_name->len; + n_padsz = (4 - (n_namesz & 3)) & 3; + + reqsz = (4 * 4) + + 4 + o_namesz + o_padsz + + (3 * 4) + + 4 + n_namesz + n_padsz; + + call = afs_alloc_flat_call(op->net, &afs_RXFSRename, reqsz, (21 + 21 + 6) * 4); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + *bp++ = htonl(FSRENAME); + *bp++ = htonl(orig_dvp->fid.vid); + *bp++ = htonl(orig_dvp->fid.vnode); + *bp++ = htonl(orig_dvp->fid.unique); + *bp++ = htonl(o_namesz); + memcpy(bp, orig_name->name, o_namesz); + bp = (void *) bp + o_namesz; + if (o_padsz > 0) { + memset(bp, 0, o_padsz); + bp = (void *) bp + o_padsz; + } + + *bp++ = htonl(new_dvp->fid.vid); + *bp++ = htonl(new_dvp->fid.vnode); + *bp++ = htonl(new_dvp->fid.unique); + *bp++ = htonl(n_namesz); + memcpy(bp, new_name->name, n_namesz); + bp = (void *) bp + n_namesz; + if (n_padsz > 0) { + memset(bp, 0, n_padsz); + bp = (void *) bp + n_padsz; + } + + call->fid = orig_dvp->fid; + trace_afs_make_fs_call2(call, &orig_dvp->fid, orig_name, new_name); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * Deliver reply data to FS.StoreData or FS.StoreStatus + */ +static int afs_deliver_fs_store_data(struct afs_call *call) +{ + struct afs_operation *op = call->op; + struct afs_vnode_param *vp = &op->file[0]; + const __be32 *bp; + int ret; + + _enter(""); + + ret = afs_transfer_reply(call); + if (ret < 0) + return ret; + + /* unmarshall the reply once we've received all of it */ + bp = call->buffer; + xdr_decode_AFSFetchStatus(&bp, call, &vp->scb); + xdr_decode_AFSVolSync(&bp, &op->volsync); + + _leave(" = 0 [done]"); + return 0; +} + +/* + * FS.StoreData operation type + */ +static const struct afs_call_type afs_RXFSStoreData = { + .name = "FS.StoreData", + .op = afs_FS_StoreData, + .deliver = afs_deliver_fs_store_data, + .destructor = afs_flat_call_destructor, +}; + +static const struct afs_call_type afs_RXFSStoreData64 = { + .name = "FS.StoreData64", + .op = afs_FS_StoreData64, + .deliver = afs_deliver_fs_store_data, + .destructor = afs_flat_call_destructor, +}; + +/* + * store a set of pages to a very large file + */ +static void afs_fs_store_data64(struct afs_operation *op) +{ + struct afs_vnode_param *vp = &op->file[0]; + struct afs_call *call; + __be32 *bp; + + _enter(",%x,{%llx:%llu},,", + key_serial(op->key), vp->fid.vid, vp->fid.vnode); + + call = afs_alloc_flat_call(op->net, &afs_RXFSStoreData64, + (4 + 6 + 3 * 2) * 4, + (21 + 6) * 4); + if (!call) + return afs_op_nomem(op); + + call->write_iter = op->store.write_iter; + + /* marshall the parameters */ + bp = call->request; + *bp++ = htonl(FSSTOREDATA64); + *bp++ = htonl(vp->fid.vid); + *bp++ = htonl(vp->fid.vnode); + *bp++ = htonl(vp->fid.unique); + + *bp++ = htonl(AFS_SET_MTIME); /* mask */ + *bp++ = htonl(op->mtime.tv_sec); /* mtime */ + *bp++ = 0; /* owner */ + *bp++ = 0; /* group */ + *bp++ = 0; /* unix mode */ + *bp++ = 0; /* segment size */ + + *bp++ = htonl(upper_32_bits(op->store.pos)); + *bp++ = htonl(lower_32_bits(op->store.pos)); + *bp++ = htonl(upper_32_bits(op->store.size)); + *bp++ = htonl(lower_32_bits(op->store.size)); + *bp++ = htonl(upper_32_bits(op->store.i_size)); + *bp++ = htonl(lower_32_bits(op->store.i_size)); + + call->fid = vp->fid; + trace_afs_make_fs_call(call, &vp->fid); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * Write data to a file on the server. + */ +void afs_fs_store_data(struct afs_operation *op) +{ + struct afs_vnode_param *vp = &op->file[0]; + struct afs_call *call; + __be32 *bp; + + _enter(",%x,{%llx:%llu},,", + key_serial(op->key), vp->fid.vid, vp->fid.vnode); + + _debug("size %llx, at %llx, i_size %llx", + (unsigned long long)op->store.size, + (unsigned long long)op->store.pos, + (unsigned long long)op->store.i_size); + + if (test_bit(AFS_SERVER_FL_HAS_FS64, &op->server->flags)) + return afs_fs_store_data64(op); + + call = afs_alloc_flat_call(op->net, &afs_RXFSStoreData, + (4 + 6 + 3) * 4, + (21 + 6) * 4); + if (!call) + return afs_op_nomem(op); + + call->write_iter = op->store.write_iter; + + /* marshall the parameters */ + bp = call->request; + *bp++ = htonl(FSSTOREDATA); + *bp++ = htonl(vp->fid.vid); + *bp++ = htonl(vp->fid.vnode); + *bp++ = htonl(vp->fid.unique); + + *bp++ = htonl(AFS_SET_MTIME); /* mask */ + *bp++ = htonl(op->mtime.tv_sec); /* mtime */ + *bp++ = 0; /* owner */ + *bp++ = 0; /* group */ + *bp++ = 0; /* unix mode */ + *bp++ = 0; /* segment size */ + + *bp++ = htonl(lower_32_bits(op->store.pos)); + *bp++ = htonl(lower_32_bits(op->store.size)); + *bp++ = htonl(lower_32_bits(op->store.i_size)); + + call->fid = vp->fid; + trace_afs_make_fs_call(call, &vp->fid); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * FS.StoreStatus operation type + */ +static const struct afs_call_type afs_RXFSStoreStatus = { + .name = "FS.StoreStatus", + .op = afs_FS_StoreStatus, + .deliver = afs_deliver_fs_store_data, + .destructor = afs_flat_call_destructor, +}; + +static const struct afs_call_type afs_RXFSStoreData_as_Status = { + .name = "FS.StoreData", + .op = afs_FS_StoreData, + .deliver = afs_deliver_fs_store_data, + .destructor = afs_flat_call_destructor, +}; + +static const struct afs_call_type afs_RXFSStoreData64_as_Status = { + .name = "FS.StoreData64", + .op = afs_FS_StoreData64, + .deliver = afs_deliver_fs_store_data, + .destructor = afs_flat_call_destructor, +}; + +/* + * set the attributes on a very large file, using FS.StoreData rather than + * FS.StoreStatus so as to alter the file size also + */ +static void afs_fs_setattr_size64(struct afs_operation *op) +{ + struct afs_vnode_param *vp = &op->file[0]; + struct afs_call *call; + struct iattr *attr = op->setattr.attr; + __be32 *bp; + + _enter(",%x,{%llx:%llu},,", + key_serial(op->key), vp->fid.vid, vp->fid.vnode); + + ASSERT(attr->ia_valid & ATTR_SIZE); + + call = afs_alloc_flat_call(op->net, &afs_RXFSStoreData64_as_Status, + (4 + 6 + 3 * 2) * 4, + (21 + 6) * 4); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + *bp++ = htonl(FSSTOREDATA64); + *bp++ = htonl(vp->fid.vid); + *bp++ = htonl(vp->fid.vnode); + *bp++ = htonl(vp->fid.unique); + + xdr_encode_AFS_StoreStatus(&bp, attr); + + *bp++ = htonl(upper_32_bits(attr->ia_size)); /* position of start of write */ + *bp++ = htonl(lower_32_bits(attr->ia_size)); + *bp++ = 0; /* size of write */ + *bp++ = 0; + *bp++ = htonl(upper_32_bits(attr->ia_size)); /* new file length */ + *bp++ = htonl(lower_32_bits(attr->ia_size)); + + call->fid = vp->fid; + trace_afs_make_fs_call(call, &vp->fid); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * set the attributes on a file, using FS.StoreData rather than FS.StoreStatus + * so as to alter the file size also + */ +static void afs_fs_setattr_size(struct afs_operation *op) +{ + struct afs_vnode_param *vp = &op->file[0]; + struct afs_call *call; + struct iattr *attr = op->setattr.attr; + __be32 *bp; + + _enter(",%x,{%llx:%llu},,", + key_serial(op->key), vp->fid.vid, vp->fid.vnode); + + ASSERT(attr->ia_valid & ATTR_SIZE); + if (test_bit(AFS_SERVER_FL_HAS_FS64, &op->server->flags)) + return afs_fs_setattr_size64(op); + + call = afs_alloc_flat_call(op->net, &afs_RXFSStoreData_as_Status, + (4 + 6 + 3) * 4, + (21 + 6) * 4); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + *bp++ = htonl(FSSTOREDATA); + *bp++ = htonl(vp->fid.vid); + *bp++ = htonl(vp->fid.vnode); + *bp++ = htonl(vp->fid.unique); + + xdr_encode_AFS_StoreStatus(&bp, attr); + + *bp++ = htonl(attr->ia_size); /* position of start of write */ + *bp++ = 0; /* size of write */ + *bp++ = htonl(attr->ia_size); /* new file length */ + + call->fid = vp->fid; + trace_afs_make_fs_call(call, &vp->fid); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * set the attributes on a file, using FS.StoreData if there's a change in file + * size, and FS.StoreStatus otherwise + */ +void afs_fs_setattr(struct afs_operation *op) +{ + struct afs_vnode_param *vp = &op->file[0]; + struct afs_call *call; + struct iattr *attr = op->setattr.attr; + __be32 *bp; + + if (attr->ia_valid & ATTR_SIZE) + return afs_fs_setattr_size(op); + + _enter(",%x,{%llx:%llu},,", + key_serial(op->key), vp->fid.vid, vp->fid.vnode); + + call = afs_alloc_flat_call(op->net, &afs_RXFSStoreStatus, + (4 + 6) * 4, + (21 + 6) * 4); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + *bp++ = htonl(FSSTORESTATUS); + *bp++ = htonl(vp->fid.vid); + *bp++ = htonl(vp->fid.vnode); + *bp++ = htonl(vp->fid.unique); + + xdr_encode_AFS_StoreStatus(&bp, op->setattr.attr); + + call->fid = vp->fid; + trace_afs_make_fs_call(call, &vp->fid); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * deliver reply data to an FS.GetVolumeStatus + */ +static int afs_deliver_fs_get_volume_status(struct afs_call *call) +{ + struct afs_operation *op = call->op; + const __be32 *bp; + char *p; + u32 size; + int ret; + + _enter("{%u}", call->unmarshall); + + switch (call->unmarshall) { + case 0: + call->unmarshall++; + afs_extract_to_buf(call, 12 * 4); + fallthrough; + + /* extract the returned status record */ + case 1: + _debug("extract status"); + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + bp = call->buffer; + xdr_decode_AFSFetchVolumeStatus(&bp, &op->volstatus.vs); + call->unmarshall++; + afs_extract_to_tmp(call); + fallthrough; + + /* extract the volume name length */ + case 2: + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + call->count = ntohl(call->tmp); + _debug("volname length: %u", call->count); + if (call->count >= AFSNAMEMAX) + return afs_protocol_error(call, afs_eproto_volname_len); + size = (call->count + 3) & ~3; /* It's padded */ + afs_extract_to_buf(call, size); + call->unmarshall++; + fallthrough; + + /* extract the volume name */ + case 3: + _debug("extract volname"); + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + p = call->buffer; + p[call->count] = 0; + _debug("volname '%s'", p); + afs_extract_to_tmp(call); + call->unmarshall++; + fallthrough; + + /* extract the offline message length */ + case 4: + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + call->count = ntohl(call->tmp); + _debug("offline msg length: %u", call->count); + if (call->count >= AFSNAMEMAX) + return afs_protocol_error(call, afs_eproto_offline_msg_len); + size = (call->count + 3) & ~3; /* It's padded */ + afs_extract_to_buf(call, size); + call->unmarshall++; + fallthrough; + + /* extract the offline message */ + case 5: + _debug("extract offline"); + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + p = call->buffer; + p[call->count] = 0; + _debug("offline '%s'", p); + + afs_extract_to_tmp(call); + call->unmarshall++; + fallthrough; + + /* extract the message of the day length */ + case 6: + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + call->count = ntohl(call->tmp); + _debug("motd length: %u", call->count); + if (call->count >= AFSNAMEMAX) + return afs_protocol_error(call, afs_eproto_motd_len); + size = (call->count + 3) & ~3; /* It's padded */ + afs_extract_to_buf(call, size); + call->unmarshall++; + fallthrough; + + /* extract the message of the day */ + case 7: + _debug("extract motd"); + ret = afs_extract_data(call, false); + if (ret < 0) + return ret; + + p = call->buffer; + p[call->count] = 0; + _debug("motd '%s'", p); + + call->unmarshall++; + fallthrough; + + case 8: + break; + } + + _leave(" = 0 [done]"); + return 0; +} + +/* + * FS.GetVolumeStatus operation type + */ +static const struct afs_call_type afs_RXFSGetVolumeStatus = { + .name = "FS.GetVolumeStatus", + .op = afs_FS_GetVolumeStatus, + .deliver = afs_deliver_fs_get_volume_status, + .destructor = afs_flat_call_destructor, +}; + +/* + * fetch the status of a volume + */ +void afs_fs_get_volume_status(struct afs_operation *op) +{ + struct afs_vnode_param *vp = &op->file[0]; + struct afs_call *call; + __be32 *bp; + + _enter(""); + + call = afs_alloc_flat_call(op->net, &afs_RXFSGetVolumeStatus, 2 * 4, + max(12 * 4, AFSOPAQUEMAX + 1)); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + bp[0] = htonl(FSGETVOLUMESTATUS); + bp[1] = htonl(vp->fid.vid); + + call->fid = vp->fid; + trace_afs_make_fs_call(call, &vp->fid); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * deliver reply data to an FS.SetLock, FS.ExtendLock or FS.ReleaseLock + */ +static int afs_deliver_fs_xxxx_lock(struct afs_call *call) +{ + struct afs_operation *op = call->op; + const __be32 *bp; + int ret; + + _enter("{%u}", call->unmarshall); + + ret = afs_transfer_reply(call); + if (ret < 0) + return ret; + + /* unmarshall the reply once we've received all of it */ + bp = call->buffer; + xdr_decode_AFSVolSync(&bp, &op->volsync); + + _leave(" = 0 [done]"); + return 0; +} + +/* + * FS.SetLock operation type + */ +static const struct afs_call_type afs_RXFSSetLock = { + .name = "FS.SetLock", + .op = afs_FS_SetLock, + .deliver = afs_deliver_fs_xxxx_lock, + .done = afs_lock_op_done, + .destructor = afs_flat_call_destructor, +}; + +/* + * FS.ExtendLock operation type + */ +static const struct afs_call_type afs_RXFSExtendLock = { + .name = "FS.ExtendLock", + .op = afs_FS_ExtendLock, + .deliver = afs_deliver_fs_xxxx_lock, + .done = afs_lock_op_done, + .destructor = afs_flat_call_destructor, +}; + +/* + * FS.ReleaseLock operation type + */ +static const struct afs_call_type afs_RXFSReleaseLock = { + .name = "FS.ReleaseLock", + .op = afs_FS_ReleaseLock, + .deliver = afs_deliver_fs_xxxx_lock, + .destructor = afs_flat_call_destructor, +}; + +/* + * Set a lock on a file + */ +void afs_fs_set_lock(struct afs_operation *op) +{ + struct afs_vnode_param *vp = &op->file[0]; + struct afs_call *call; + __be32 *bp; + + _enter(""); + + call = afs_alloc_flat_call(op->net, &afs_RXFSSetLock, 5 * 4, 6 * 4); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + *bp++ = htonl(FSSETLOCK); + *bp++ = htonl(vp->fid.vid); + *bp++ = htonl(vp->fid.vnode); + *bp++ = htonl(vp->fid.unique); + *bp++ = htonl(op->lock.type); + + call->fid = vp->fid; + trace_afs_make_fs_calli(call, &vp->fid, op->lock.type); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * extend a lock on a file + */ +void afs_fs_extend_lock(struct afs_operation *op) +{ + struct afs_vnode_param *vp = &op->file[0]; + struct afs_call *call; + __be32 *bp; + + _enter(""); + + call = afs_alloc_flat_call(op->net, &afs_RXFSExtendLock, 4 * 4, 6 * 4); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + *bp++ = htonl(FSEXTENDLOCK); + *bp++ = htonl(vp->fid.vid); + *bp++ = htonl(vp->fid.vnode); + *bp++ = htonl(vp->fid.unique); + + call->fid = vp->fid; + trace_afs_make_fs_call(call, &vp->fid); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * release a lock on a file + */ +void afs_fs_release_lock(struct afs_operation *op) +{ + struct afs_vnode_param *vp = &op->file[0]; + struct afs_call *call; + __be32 *bp; + + _enter(""); + + call = afs_alloc_flat_call(op->net, &afs_RXFSReleaseLock, 4 * 4, 6 * 4); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + *bp++ = htonl(FSRELEASELOCK); + *bp++ = htonl(vp->fid.vid); + *bp++ = htonl(vp->fid.vnode); + *bp++ = htonl(vp->fid.unique); + + call->fid = vp->fid; + trace_afs_make_fs_call(call, &vp->fid); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * Deliver reply data to an FS.GiveUpAllCallBacks operation. + */ +static int afs_deliver_fs_give_up_all_callbacks(struct afs_call *call) +{ + return afs_transfer_reply(call); +} + +/* + * FS.GiveUpAllCallBacks operation type + */ +static const struct afs_call_type afs_RXFSGiveUpAllCallBacks = { + .name = "FS.GiveUpAllCallBacks", + .op = afs_FS_GiveUpAllCallBacks, + .deliver = afs_deliver_fs_give_up_all_callbacks, + .destructor = afs_flat_call_destructor, +}; + +/* + * Flush all the callbacks we have on a server. + */ +int afs_fs_give_up_all_callbacks(struct afs_net *net, struct afs_server *server, + struct afs_address *addr, struct key *key) +{ + struct afs_call *call; + __be32 *bp; + int ret; + + _enter(""); + + call = afs_alloc_flat_call(net, &afs_RXFSGiveUpAllCallBacks, 1 * 4, 0); + if (!call) + return -ENOMEM; + + call->key = key; + call->peer = rxrpc_kernel_get_peer(addr->peer); + call->service_id = server->service_id; + + /* marshall the parameters */ + bp = call->request; + *bp++ = htonl(FSGIVEUPALLCALLBACKS); + + call->server = afs_use_server(server, false, afs_server_trace_use_give_up_cb); + afs_make_call(call, GFP_NOFS); + afs_wait_for_call_to_complete(call); + ret = call->error; + if (call->responded) + set_bit(AFS_SERVER_FL_RESPONDING, &server->flags); + afs_put_call(call); + return ret; +} + +/* + * Deliver reply data to an FS.GetCapabilities operation. + */ +static int afs_deliver_fs_get_capabilities(struct afs_call *call) +{ + u32 count; + int ret; + + _enter("{%u,%zu}", call->unmarshall, iov_iter_count(call->iter)); + + switch (call->unmarshall) { + case 0: + afs_extract_to_tmp(call); + call->unmarshall++; + fallthrough; + + /* Extract the capabilities word count */ + case 1: + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + count = ntohl(call->tmp); + call->count = count; + call->count2 = count; + if (count == 0) { + call->unmarshall = 4; + call->tmp = 0; + break; + } + + /* Extract the first word of the capabilities to call->tmp */ + afs_extract_to_tmp(call); + call->unmarshall++; + fallthrough; + + case 2: + ret = afs_extract_data(call, false); + if (ret < 0) + return ret; + + afs_extract_discard(call, (count - 1) * sizeof(__be32)); + call->unmarshall++; + fallthrough; + + /* Extract remaining capabilities words */ + case 3: + ret = afs_extract_data(call, false); + if (ret < 0) + return ret; + + call->unmarshall++; + break; + } + + _leave(" = 0 [done]"); + return 0; +} + +static void afs_fs_get_capabilities_destructor(struct afs_call *call) +{ + afs_put_endpoint_state(call->probe, afs_estate_trace_put_getcaps); + afs_flat_call_destructor(call); +} + +/* + * FS.GetCapabilities operation type + */ +static const struct afs_call_type afs_RXFSGetCapabilities = { + .name = "FS.GetCapabilities", + .op = afs_FS_GetCapabilities, + .deliver = afs_deliver_fs_get_capabilities, + .done = afs_fileserver_probe_result, + .immediate_cancel = afs_fileserver_probe_result, + .destructor = afs_fs_get_capabilities_destructor, +}; + +/* + * Probe a fileserver for the capabilities that it supports. This RPC can + * reply with up to 196 words. The operation is asynchronous and if we managed + * to allocate a call, true is returned the result is delivered through the + * ->done() - otherwise we return false to indicate we didn't even try. + */ +bool afs_fs_get_capabilities(struct afs_net *net, struct afs_server *server, + struct afs_endpoint_state *estate, unsigned int addr_index, + struct key *key) +{ + struct afs_call *call; + __be32 *bp; + + _enter(""); + + call = afs_alloc_flat_call(net, &afs_RXFSGetCapabilities, 1 * 4, 16 * 4); + if (!call) + return false; + + call->key = key; + call->server = afs_use_server(server, false, afs_server_trace_use_get_caps); + call->peer = rxrpc_kernel_get_peer(estate->addresses->addrs[addr_index].peer); + call->probe = afs_get_endpoint_state(estate, afs_estate_trace_get_getcaps); + call->probe_index = addr_index; + call->service_id = server->service_id; + call->upgrade = true; + call->async = true; + call->max_lifespan = AFS_PROBE_MAX_LIFESPAN; + + /* marshall the parameters */ + bp = call->request; + *bp++ = htonl(FSGETCAPABILITIES); + + trace_afs_make_fs_call(call, NULL); + afs_make_call(call, GFP_NOFS); + afs_put_call(call); + return true; +} + +/* + * Deliver reply data to an FS.InlineBulkStatus call + */ +static int afs_deliver_fs_inline_bulk_status(struct afs_call *call) +{ + struct afs_operation *op = call->op; + struct afs_status_cb *scb; + const __be32 *bp; + u32 tmp; + int ret; + + _enter("{%u}", call->unmarshall); + + switch (call->unmarshall) { + case 0: + afs_extract_to_tmp(call); + call->unmarshall++; + fallthrough; + + /* Extract the file status count and array in two steps */ + case 1: + _debug("extract status count"); + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + tmp = ntohl(call->tmp); + _debug("status count: %u/%u", tmp, op->nr_files); + if (tmp != op->nr_files) + return afs_protocol_error(call, afs_eproto_ibulkst_count); + + call->count = 0; + call->unmarshall++; + more_counts: + afs_extract_to_buf(call, 21 * sizeof(__be32)); + fallthrough; + + case 2: + _debug("extract status array %u", call->count); + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + switch (call->count) { + case 0: + scb = &op->file[0].scb; + break; + case 1: + scb = &op->file[1].scb; + break; + default: + scb = &op->more_files[call->count - 2].scb; + break; + } + + bp = call->buffer; + xdr_decode_AFSFetchStatus(&bp, call, scb); + + call->count++; + if (call->count < op->nr_files) + goto more_counts; + + call->count = 0; + call->unmarshall++; + afs_extract_to_tmp(call); + fallthrough; + + /* Extract the callback count and array in two steps */ + case 3: + _debug("extract CB count"); + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + tmp = ntohl(call->tmp); + _debug("CB count: %u", tmp); + if (tmp != op->nr_files) + return afs_protocol_error(call, afs_eproto_ibulkst_cb_count); + call->count = 0; + call->unmarshall++; + more_cbs: + afs_extract_to_buf(call, 3 * sizeof(__be32)); + fallthrough; + + case 4: + _debug("extract CB array"); + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + _debug("unmarshall CB array"); + switch (call->count) { + case 0: + scb = &op->file[0].scb; + break; + case 1: + scb = &op->file[1].scb; + break; + default: + scb = &op->more_files[call->count - 2].scb; + break; + } + + bp = call->buffer; + xdr_decode_AFSCallBack(&bp, call, scb); + call->count++; + if (call->count < op->nr_files) + goto more_cbs; + + afs_extract_to_buf(call, 6 * sizeof(__be32)); + call->unmarshall++; + fallthrough; + + case 5: + ret = afs_extract_data(call, false); + if (ret < 0) + return ret; + + bp = call->buffer; + /* Unfortunately, prior to OpenAFS-1.6, volsync here is filled + * with rubbish. + */ + xdr_decode_AFSVolSync(&bp, NULL); + + call->unmarshall++; + fallthrough; + + case 6: + break; + } + + _leave(" = 0 [done]"); + return 0; +} + +static void afs_done_fs_inline_bulk_status(struct afs_call *call) +{ + if (call->error == -ECONNABORTED && + call->abort_code == RX_INVALID_OPERATION) { + set_bit(AFS_SERVER_FL_NO_IBULK, &call->server->flags); + if (call->op) + set_bit(AFS_VOLUME_MAYBE_NO_IBULK, &call->op->volume->flags); + } +} + +/* + * FS.InlineBulkStatus operation type + */ +static const struct afs_call_type afs_RXFSInlineBulkStatus = { + .name = "FS.InlineBulkStatus", + .op = afs_FS_InlineBulkStatus, + .deliver = afs_deliver_fs_inline_bulk_status, + .done = afs_done_fs_inline_bulk_status, + .destructor = afs_flat_call_destructor, +}; + +/* + * Fetch the status information for up to 50 files + */ +void afs_fs_inline_bulk_status(struct afs_operation *op) +{ + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_vnode_param *vp = &op->file[1]; + struct afs_call *call; + __be32 *bp; + int i; + + if (test_bit(AFS_SERVER_FL_NO_IBULK, &op->server->flags)) { + afs_op_set_error(op, -ENOTSUPP); + return; + } + + _enter(",%x,{%llx:%llu},%u", + key_serial(op->key), vp->fid.vid, vp->fid.vnode, op->nr_files); + + call = afs_alloc_flat_call(op->net, &afs_RXFSInlineBulkStatus, + (2 + op->nr_files * 3) * 4, + 21 * 4); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + *bp++ = htonl(FSINLINEBULKSTATUS); + *bp++ = htonl(op->nr_files); + *bp++ = htonl(dvp->fid.vid); + *bp++ = htonl(dvp->fid.vnode); + *bp++ = htonl(dvp->fid.unique); + *bp++ = htonl(vp->fid.vid); + *bp++ = htonl(vp->fid.vnode); + *bp++ = htonl(vp->fid.unique); + for (i = 0; i < op->nr_files - 2; i++) { + *bp++ = htonl(op->more_files[i].fid.vid); + *bp++ = htonl(op->more_files[i].fid.vnode); + *bp++ = htonl(op->more_files[i].fid.unique); + } + + call->fid = vp->fid; + trace_afs_make_fs_call(call, &vp->fid); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * deliver reply data to an FS.FetchACL + */ +static int afs_deliver_fs_fetch_acl(struct afs_call *call) +{ + struct afs_operation *op = call->op; + struct afs_vnode_param *vp = &op->file[0]; + struct afs_acl *acl; + const __be32 *bp; + unsigned int size; + int ret; + + _enter("{%u}", call->unmarshall); + + switch (call->unmarshall) { + case 0: + afs_extract_to_tmp(call); + call->unmarshall++; + fallthrough; + + /* extract the returned data length */ + case 1: + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + size = call->count2 = ntohl(call->tmp); + size = round_up(size, 4); + + acl = kmalloc_flex(*acl, data, size); + if (!acl) + return -ENOMEM; + op->acl = acl; + acl->size = call->count2; + afs_extract_begin(call, acl->data, size); + call->unmarshall++; + fallthrough; + + /* extract the returned data */ + case 2: + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + afs_extract_to_buf(call, (21 + 6) * 4); + call->unmarshall++; + fallthrough; + + /* extract the metadata */ + case 3: + ret = afs_extract_data(call, false); + if (ret < 0) + return ret; + + bp = call->buffer; + xdr_decode_AFSFetchStatus(&bp, call, &vp->scb); + xdr_decode_AFSVolSync(&bp, &op->volsync); + + call->unmarshall++; + fallthrough; + + case 4: + break; + } + + _leave(" = 0 [done]"); + return 0; +} + +/* + * FS.FetchACL operation type + */ +static const struct afs_call_type afs_RXFSFetchACL = { + .name = "FS.FetchACL", + .op = afs_FS_FetchACL, + .deliver = afs_deliver_fs_fetch_acl, +}; + +/* + * Fetch the ACL for a file. + */ +void afs_fs_fetch_acl(struct afs_operation *op) +{ + struct afs_vnode_param *vp = &op->file[0]; + struct afs_call *call; + __be32 *bp; + + _enter(",%x,{%llx:%llu},,", + key_serial(op->key), vp->fid.vid, vp->fid.vnode); + + call = afs_alloc_flat_call(op->net, &afs_RXFSFetchACL, 16, (21 + 6) * 4); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + bp[0] = htonl(FSFETCHACL); + bp[1] = htonl(vp->fid.vid); + bp[2] = htonl(vp->fid.vnode); + bp[3] = htonl(vp->fid.unique); + + call->fid = vp->fid; + trace_afs_make_fs_call(call, &vp->fid); + afs_make_op_call(op, call, GFP_KERNEL); +} + +/* + * FS.StoreACL operation type + */ +static const struct afs_call_type afs_RXFSStoreACL = { + .name = "FS.StoreACL", + .op = afs_FS_StoreACL, + .deliver = afs_deliver_fs_file_status_and_vol, + .destructor = afs_flat_call_destructor, +}; + +/* + * Fetch the ACL for a file. + */ +void afs_fs_store_acl(struct afs_operation *op) +{ + struct afs_vnode_param *vp = &op->file[0]; + struct afs_call *call; + const struct afs_acl *acl = op->acl; + size_t size; + __be32 *bp; + + _enter(",%x,{%llx:%llu},,", + key_serial(op->key), vp->fid.vid, vp->fid.vnode); + + size = round_up(acl->size, 4); + call = afs_alloc_flat_call(op->net, &afs_RXFSStoreACL, + 5 * 4 + size, (21 + 6) * 4); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + bp[0] = htonl(FSSTOREACL); + bp[1] = htonl(vp->fid.vid); + bp[2] = htonl(vp->fid.vnode); + bp[3] = htonl(vp->fid.unique); + bp[4] = htonl(acl->size); + memcpy(&bp[5], acl->data, acl->size); + if (acl->size != size) + memset((void *)&bp[5] + acl->size, 0, size - acl->size); + + call->fid = vp->fid; + trace_afs_make_fs_call(call, &vp->fid); + afs_make_op_call(op, call, GFP_KERNEL); +} diff --git a/fs/afs/inode.c b/fs/afs/inode.c new file mode 100644 index 000000000..14f39a9be --- /dev/null +++ b/fs/afs/inode.c @@ -0,0 +1,853 @@ +/* + * Copyright (c) 2002 Red Hat, Inc. All rights reserved. + * + * This software may be freely redistributed under the terms of the + * GNU General Public License. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + * Authors: David Woodhouse + * David Howells + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" +#include "afs_fs.h" + +static noinline void dump_vnode(struct afs_vnode *vnode, struct afs_vnode *parent_vnode) +{ + static unsigned long once_only; + + pr_warn("kAFS: AFS vnode with undefined type %u\n", vnode->status.type); + pr_warn("kAFS: A=%d m=%o s=%llx v=%llx\n", + vnode->status.abort_code, + vnode->status.mode, + vnode->status.size, + vnode->status.data_version); + pr_warn("kAFS: vnode %llx:%llx:%x\n", + vnode->fid.vid, + vnode->fid.vnode, + vnode->fid.unique); + if (parent_vnode) + pr_warn("kAFS: dir %llx:%llx:%x\n", + parent_vnode->fid.vid, + parent_vnode->fid.vnode, + parent_vnode->fid.unique); + + if (!test_and_set_bit(0, &once_only)) + dump_stack(); +} + +/* + * Set parameters for the netfs library + */ +static void afs_set_netfs_context(struct afs_vnode *vnode, bool is_file) +{ + netfs_inode_init(&vnode->netfs, &afs_req_ops, is_file); +} + +/* + * Initialise an inode from the vnode status. + */ +static int afs_inode_init_from_status(struct afs_operation *op, + struct afs_vnode_param *vp, + struct afs_vnode *vnode) +{ + struct afs_file_status *status = &vp->scb.status; + struct inode *inode = AFS_VNODE_TO_I(vnode); + struct timespec64 t; + + _enter("{%llx:%llu.%u} %s", + vp->fid.vid, vp->fid.vnode, vp->fid.unique, + op->type ? op->type->name : "???"); + + _debug("FS: ft=%d lk=%d sz=%llu ver=%Lu mod=%hu", + status->type, + status->nlink, + (unsigned long long) status->size, + status->data_version, + status->mode); + + write_seqlock(&vnode->cb_lock); + + vnode->cb_v_check = op->cb_v_break; + vnode->status = *status; + + t = status->mtime_client; + inode_set_ctime_to_ts(inode, t); + inode_set_mtime_to_ts(inode, t); + inode_set_atime_to_ts(inode, t); + inode->i_flags |= S_NOATIME; + inode->i_uid = make_kuid(&init_user_ns, status->owner); + inode->i_gid = make_kgid(&init_user_ns, status->group); + set_nlink(&vnode->netfs.inode, status->nlink); + + i_size_write(inode, status->size); + inode_set_bytes(inode, status->size); + afs_set_netfs_context(vnode, status->type == AFS_FTYPE_FILE); + + switch (status->type) { + case AFS_FTYPE_FILE: + inode->i_mode = S_IFREG | (status->mode & S_IALLUGO); + inode->i_op = &afs_file_inode_operations; + inode->i_fop = &afs_file_operations; + inode->i_mapping->a_ops = &afs_file_aops; + mapping_set_large_folios(inode->i_mapping); + break; + case AFS_FTYPE_DIR: + inode->i_mode = S_IFDIR | (status->mode & S_IALLUGO); + inode->i_op = &afs_dir_inode_operations; + inode->i_fop = &afs_dir_file_operations; + inode->i_mapping->a_ops = &afs_dir_aops; + __set_bit(NETFS_ICTX_SINGLE_NO_UPLOAD, &vnode->netfs.flags); + /* Assume locally cached directory data will be valid. */ + __set_bit(AFS_VNODE_DIR_VALID, &vnode->flags); + break; + case AFS_FTYPE_SYMLINK: + /* Symlinks with a mode of 0644 are actually mountpoints. */ + if ((status->mode & 0777) == 0644) { + inode->i_flags |= S_AUTOMOUNT; + + set_bit(AFS_VNODE_MOUNTPOINT, &vnode->flags); + + inode->i_mode = S_IFDIR | 0555; + inode->i_op = &afs_mntpt_inode_operations; + inode->i_fop = &afs_mntpt_file_operations; + } else { + inode->i_mode = S_IFLNK | status->mode; + inode->i_op = &afs_symlink_inode_operations; + } + inode->i_mapping->a_ops = &afs_symlink_aops; + inode_nohighmem(inode); + break; + default: + dump_vnode(vnode, op->file[0].vnode != vnode ? op->file[0].vnode : NULL); + write_sequnlock(&vnode->cb_lock); + return afs_protocol_error(NULL, afs_eproto_file_type); + } + + vnode->invalid_before = status->data_version; + trace_afs_set_dv(vnode, status->data_version); + inode_set_iversion_raw(&vnode->netfs.inode, status->data_version); + + if (!vp->scb.have_cb) { + /* it's a symlink we just created (the fileserver + * didn't give us a callback) */ + afs_clear_cb_promise(vnode, afs_cb_promise_set_new_symlink); + } else { + vnode->cb_server = op->server; + afs_set_cb_promise(vnode, vp->scb.callback.expires_at, + afs_cb_promise_set_new_inode); + } + + write_sequnlock(&vnode->cb_lock); + return 0; +} + +/* + * Update the core inode struct from a returned status record. + */ +static void afs_apply_status(struct afs_operation *op, + struct afs_vnode_param *vp) +{ + struct afs_file_status *status = &vp->scb.status; + struct afs_vnode *vnode = vp->vnode; + struct netfs_inode *ictx = &vnode->netfs; + struct inode *inode = &ictx->inode; + struct timespec64 t; + umode_t mode; + bool unexpected_jump = false; + bool data_changed = false; + bool change_size = vp->set_size; + + _enter("{%llx:%llu.%u} %s", + vp->fid.vid, vp->fid.vnode, vp->fid.unique, + op->type ? op->type->name : "???"); + + BUG_ON(test_bit(AFS_VNODE_UNSET, &vnode->flags)); + + if (status->type != vnode->status.type) { + pr_warn("Vnode %llx:%llx:%x changed type %u to %u\n", + vnode->fid.vid, + vnode->fid.vnode, + vnode->fid.unique, + status->type, vnode->status.type); + afs_protocol_error(NULL, afs_eproto_bad_status); + return; + } + + if (status->nlink != vnode->status.nlink) + set_nlink(inode, status->nlink); + + if (status->owner != vnode->status.owner) + inode->i_uid = make_kuid(&init_user_ns, status->owner); + + if (status->group != vnode->status.group) + inode->i_gid = make_kgid(&init_user_ns, status->group); + + if (status->mode != vnode->status.mode) { + mode = inode->i_mode; + mode &= ~S_IALLUGO; + mode |= status->mode & S_IALLUGO; + WRITE_ONCE(inode->i_mode, mode); + } + + t = status->mtime_client; + inode_set_mtime_to_ts(inode, t); + if (vp->update_ctime) + inode_set_ctime_to_ts(inode, op->ctime); + + if (vnode->status.data_version != status->data_version) { + trace_afs_set_dv(vnode, status->data_version); + data_changed = true; + } + + vnode->status = *status; + + if (vp->dv_before + vp->dv_delta != status->data_version) { + trace_afs_dv_mismatch(vnode, vp->dv_before, vp->dv_delta, + status->data_version); + + if (vnode->cb_ro_snapshot == atomic_read(&vnode->volume->cb_ro_snapshot) && + atomic64_read(&vnode->cb_expires_at) != AFS_NO_CB_PROMISE) + pr_warn("kAFS: vnode modified {%llx:%llu} %llx->%llx %s (op=%x)\n", + vnode->fid.vid, vnode->fid.vnode, + (unsigned long long)vp->dv_before + vp->dv_delta, + (unsigned long long)status->data_version, + op->type ? op->type->name : "???", + op->debug_id); + + vnode->invalid_before = status->data_version; + if (vnode->status.type == AFS_FTYPE_DIR) + afs_invalidate_dir(vnode, afs_dir_invalid_dv_mismatch); + else + set_bit(AFS_VNODE_ZAP_DATA, &vnode->flags); + change_size = true; + data_changed = true; + unexpected_jump = true; + } else if (vnode->status.type == AFS_FTYPE_DIR) { + /* Expected directory change is handled elsewhere so + * that we can locally edit the directory and save on a + * download. + */ + if (test_bit(AFS_VNODE_DIR_VALID, &vnode->flags)) + data_changed = false; + change_size = true; + } + + if (data_changed) { + unsigned long long zero_point, size = status->size; + + inode_set_iversion_raw(inode, status->data_version); + + /* Only update the size if the data version jumped. If the + * file is being modified locally, then we might have our own + * idea of what the size should be that's not the same as + * what's on the server. + */ + spin_lock(&inode->i_lock); + + if (change_size || size > i_size_read(inode)) { + /* We can read the sizes directly as we hold i_lock. */ + zero_point = ictx->_zero_point; + + if (unexpected_jump) + zero_point = size; + netfs_write_sizes(inode, size, size, zero_point); + inode_set_bytes(inode, size); + inode_set_ctime_to_ts(inode, t); + inode_set_atime_to_ts(inode, t); + } else { + netfs_write_remote_i_size(inode, size); + } + spin_unlock(&inode->i_lock); + + if (op->ops == &afs_fetch_data_operation) + op->fetch.subreq->rreq->i_size = size; + } +} + +/* + * Apply a callback to a vnode. + */ +static void afs_apply_callback(struct afs_operation *op, + struct afs_vnode_param *vp) +{ + struct afs_callback *cb = &vp->scb.callback; + struct afs_vnode *vnode = vp->vnode; + + if (!afs_cb_is_broken(vp->cb_break_before, vnode)) { + if (op->volume->type == AFSVL_RWVOL) + vnode->cb_server = op->server; + afs_set_cb_promise(vnode, cb->expires_at, afs_cb_promise_set_apply_cb); + } +} + +/* + * Apply the received status and callback to an inode all in the same critical + * section to avoid races with afs_validate(). + */ +void afs_vnode_commit_status(struct afs_operation *op, struct afs_vnode_param *vp) +{ + struct afs_vnode *vnode = vp->vnode; + + _enter(""); + + write_seqlock(&vnode->cb_lock); + + if (vp->scb.have_error) { + /* A YFS server will return this from RemoveFile2 and AFS and + * YFS will return this from InlineBulkStatus. + */ + if (vp->scb.status.abort_code == VNOVNODE) { + set_bit(AFS_VNODE_DELETED, &vnode->flags); + clear_nlink(&vnode->netfs.inode); + __afs_break_callback(vnode, afs_cb_break_for_deleted); + op->flags &= ~AFS_OPERATION_DIR_CONFLICT; + } + } else if (vp->scb.have_status) { + if (vp->speculative && + (test_bit(AFS_VNODE_MODIFYING, &vnode->flags) || + vp->dv_before != vnode->status.data_version)) + /* Ignore the result of a speculative bulk status fetch + * if it splits around a modification op, thereby + * appearing to regress the data version. + */ + goto out; + afs_apply_status(op, vp); + if (vp->scb.have_cb) + afs_apply_callback(op, vp); + } else if (vp->op_unlinked && !(op->flags & AFS_OPERATION_DIR_CONFLICT)) { + drop_nlink(&vnode->netfs.inode); + if (vnode->netfs.inode.i_nlink == 0) { + set_bit(AFS_VNODE_DELETED, &vnode->flags); + __afs_break_callback(vnode, afs_cb_break_for_deleted); + } + } + +out: + write_sequnlock(&vnode->cb_lock); + + if (vp->scb.have_status) + afs_cache_permit(vnode, op->key, vp->cb_break_before, &vp->scb); +} + +static void afs_fetch_status_success(struct afs_operation *op) +{ + struct afs_vnode_param *vp = &op->file[op->fetch_status.which]; + struct afs_vnode *vnode = vp->vnode; + int ret; + + if (inode_state_read_once(&vnode->netfs.inode) & I_NEW) { + ret = afs_inode_init_from_status(op, vp, vnode); + afs_op_set_error(op, ret); + if (ret == 0) + afs_cache_permit(vnode, op->key, vp->cb_break_before, &vp->scb); + } else { + afs_vnode_commit_status(op, vp); + } +} + +const struct afs_operation_ops afs_fetch_status_operation = { + .issue_afs_rpc = afs_fs_fetch_status, + .issue_yfs_rpc = yfs_fs_fetch_status, + .success = afs_fetch_status_success, + .aborted = afs_check_for_remote_deletion, +}; + +/* + * Fetch file status from the volume. + */ +int afs_fetch_status(struct afs_vnode *vnode, struct key *key, bool is_new, + afs_access_t *_caller_access) +{ + struct afs_operation *op; + + _enter("%s,{%llx:%llu.%u,S=%lx}", + vnode->volume->name, + vnode->fid.vid, vnode->fid.vnode, vnode->fid.unique, + vnode->flags); + + op = afs_alloc_operation(key, vnode->volume); + if (IS_ERR(op)) + return PTR_ERR(op); + + afs_op_set_vnode(op, 0, vnode); + + op->nr_files = 1; + op->ops = &afs_fetch_status_operation; + afs_begin_vnode_operation(op); + afs_wait_for_operation(op); + + if (_caller_access) + *_caller_access = op->file[0].scb.status.caller_access; + return afs_put_operation(op); +} + +/* + * ilookup() comparator + */ +int afs_ilookup5_test_by_fid(struct inode *inode, void *opaque) +{ + struct afs_vnode *vnode = AFS_FS_I(inode); + struct afs_fid *fid = opaque; + + return (fid->vnode == vnode->fid.vnode && + fid->vnode_hi == vnode->fid.vnode_hi && + fid->unique == vnode->fid.unique); +} + +/* + * iget5() comparator + */ +static int afs_iget5_test(struct inode *inode, void *opaque) +{ + struct afs_vnode_param *vp = opaque; + //struct afs_vnode *vnode = AFS_FS_I(inode); + + return afs_ilookup5_test_by_fid(inode, &vp->fid); +} + +/* + * iget5() inode initialiser + */ +static int afs_iget5_set(struct inode *inode, void *opaque) +{ + struct afs_vnode_param *vp = opaque; + struct afs_super_info *as = AFS_FS_S(inode->i_sb); + struct afs_vnode *vnode = AFS_FS_I(inode); + + vnode->volume = as->volume; + vnode->fid = vp->fid; + + /* YFS supports 96-bit vnode IDs, but Linux only supports + * 64-bit inode numbers. + */ + inode->i_ino = vnode->fid.vnode; + inode->i_generation = vnode->fid.unique; + return 0; +} + +/* + * Get a cache cookie for an inode. + */ +static void afs_get_inode_cache(struct afs_vnode *vnode) +{ +#ifdef CONFIG_AFS_FSCACHE + struct { + __be32 vnode_id; + __be32 unique; + __be32 vnode_id_ext[2]; /* Allow for a 96-bit key */ + } __packed key; + struct afs_vnode_cache_aux aux; + + if (vnode->status.type != AFS_FTYPE_FILE && + vnode->status.type != AFS_FTYPE_DIR && + vnode->status.type != AFS_FTYPE_SYMLINK) { + vnode->netfs.cache = NULL; + return; + } + + key.vnode_id = htonl(vnode->fid.vnode); + key.unique = htonl(vnode->fid.unique); + key.vnode_id_ext[0] = htonl(vnode->fid.vnode >> 32); + key.vnode_id_ext[1] = htonl(vnode->fid.vnode_hi); + afs_set_cache_aux(vnode, &aux); + + afs_vnode_set_cache(vnode, + fscache_acquire_cookie( + vnode->volume->cache, + vnode->status.type == AFS_FTYPE_FILE ? + 0 : FSCACHE_ADV_SINGLE_CHUNK, + &key, sizeof(key), + &aux, sizeof(aux), + i_size_read(&vnode->netfs.inode))); +#endif +} + +/* + * inode retrieval + */ +struct inode *afs_iget(struct afs_operation *op, struct afs_vnode_param *vp) +{ + struct afs_vnode_param *dvp = &op->file[0]; + struct super_block *sb = dvp->vnode->netfs.inode.i_sb; + struct afs_vnode *vnode; + struct inode *inode; + int ret; + + _enter(",{%llx:%llu.%u},,", vp->fid.vid, vp->fid.vnode, vp->fid.unique); + + inode = iget5_locked(sb, vp->fid.vnode, afs_iget5_test, afs_iget5_set, vp); + if (!inode) { + _leave(" = -ENOMEM"); + return ERR_PTR(-ENOMEM); + } + + vnode = AFS_FS_I(inode); + + _debug("GOT INODE %p { vl=%llx vn=%llx, u=%x }", + inode, vnode->fid.vid, vnode->fid.vnode, vnode->fid.unique); + + /* deal with an existing inode */ + if (!(inode_state_read_once(inode) & I_NEW)) { + _leave(" = %p", inode); + return inode; + } + + ret = afs_inode_init_from_status(op, vp, vnode); + if (ret < 0) + goto bad_inode; + + afs_get_inode_cache(vnode); + + /* success */ + clear_bit(AFS_VNODE_UNSET, &vnode->flags); + unlock_new_inode(inode); + _leave(" = %p", inode); + return inode; + + /* failure */ +bad_inode: + iget_failed(inode); + _leave(" = %d [bad]", ret); + return ERR_PTR(ret); +} + +static int afs_iget5_set_root(struct inode *inode, void *opaque) +{ + struct afs_super_info *as = AFS_FS_S(inode->i_sb); + struct afs_vnode *vnode = AFS_FS_I(inode); + + vnode->volume = as->volume; + vnode->fid.vid = as->volume->vid; + vnode->fid.vnode = 1; + vnode->fid.unique = 1; + inode->i_ino = 1; + inode->i_generation = 1; + return 0; +} + +/* + * Set up the root inode for a volume. This is always vnode 1, unique 1 within + * the volume. + */ +struct inode *afs_root_iget(struct super_block *sb, struct key *key) +{ + struct afs_super_info *as = AFS_FS_S(sb); + struct afs_operation *op; + struct afs_vnode *vnode; + struct inode *inode; + int ret; + + _enter(",{%llx},,", as->volume->vid); + + inode = iget5_locked(sb, 1, NULL, afs_iget5_set_root, NULL); + if (!inode) { + _leave(" = -ENOMEM"); + return ERR_PTR(-ENOMEM); + } + + _debug("GOT ROOT INODE %p { vl=%llx }", inode, as->volume->vid); + + BUG_ON(!(inode_state_read_once(inode) & I_NEW)); + + vnode = AFS_FS_I(inode); + vnode->cb_v_check = atomic_read(&as->volume->cb_v_break); + + op = afs_alloc_operation(key, as->volume); + if (IS_ERR(op)) { + ret = PTR_ERR(op); + goto error; + } + + afs_op_set_vnode(op, 0, vnode); + + op->nr_files = 1; + op->ops = &afs_fetch_status_operation; + ret = afs_do_sync_operation(op); + if (ret < 0) + goto error; + + afs_get_inode_cache(vnode); + + clear_bit(AFS_VNODE_UNSET, &vnode->flags); + unlock_new_inode(inode); + _leave(" = %p", inode); + return inode; + +error: + iget_failed(inode); + _leave(" = %d [bad]", ret); + return ERR_PTR(ret); +} + +/* + * read the attributes of an inode + */ +int afs_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int query_flags) +{ + struct inode *inode = d_inode(path->dentry); + struct afs_vnode *vnode = AFS_FS_I(inode); + struct key *key; + int ret, seq; + + _enter("{ ino=%llu v=%u }", inode->i_ino, inode->i_generation); + + if (vnode->volume && + !(query_flags & AT_STATX_DONT_SYNC) && + atomic64_read(&vnode->cb_expires_at) == AFS_NO_CB_PROMISE) { + key = afs_request_key(vnode->volume->cell); + if (IS_ERR(key)) + return PTR_ERR(key); + ret = afs_validate(vnode, key); + key_put(key); + if (ret < 0) + return ret; + } + + do { + seq = read_seqbegin(&vnode->cb_lock); + generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat); + if (test_bit(AFS_VNODE_SILLY_DELETED, &vnode->flags) && + stat->nlink > 0) + stat->nlink -= 1; + + /* Lie about the size of directories. We maintain a locally + * edited copy and may make different allocation decisions on + * it, but we need to give userspace the server's size. + */ + if (S_ISDIR(inode->i_mode)) + stat->size = netfs_read_remote_i_size(inode); + } while (read_seqretry(&vnode->cb_lock, seq)); + + return 0; +} + +/* + * discard an AFS inode + */ +int afs_drop_inode(struct inode *inode) +{ + _enter(""); + + if (test_bit(AFS_VNODE_PSEUDODIR, &AFS_FS_I(inode)->flags)) + return inode_just_drop(inode); + else + return inode_generic_drop(inode); +} + +/* + * clear an AFS inode + */ +void afs_evict_inode(struct inode *inode) +{ + struct afs_vnode_cache_aux aux; + struct afs_super_info *sbi = AFS_FS_S(inode->i_sb); + struct afs_vnode *vnode = AFS_FS_I(inode); + + _enter("{%llx:%llu.%d}", + vnode->fid.vid, + vnode->fid.vnode, + vnode->fid.unique); + + _debug("CLEAR INODE %p", inode); + + ASSERTCMP(inode->i_ino, ==, vnode->fid.vnode); + + if ((S_ISDIR(inode->i_mode) || + S_ISLNK(inode->i_mode)) && + (inode_state_read_once(inode) & I_DIRTY) && + !sbi->dyn_root) { + struct writeback_control wbc = { + .sync_mode = WB_SYNC_ALL, + .for_sync = true, + .range_end = LLONG_MAX, + }; + + inode->i_mapping->a_ops->writepages(inode->i_mapping, &wbc); + } + + flush_delayed_work(&vnode->lock_work); + netfs_wait_for_outstanding_io(inode); + truncate_inode_pages_final(&inode->i_data); + netfs_free_folioq_buffer(vnode->directory); + if (vnode->symlink) + afs_evict_symlink(vnode); + + afs_set_cache_aux(vnode, &aux); + netfs_clear_inode_writeback(inode, &aux); + clear_inode(inode); + + while (!list_empty(&vnode->wb_keys)) { + struct afs_wb_key *wbk = list_entry(vnode->wb_keys.next, + struct afs_wb_key, vnode_link); + list_del(&wbk->vnode_link); + afs_put_wb_key(wbk); + } + + fscache_relinquish_cookie(afs_vnode_cache(vnode), + test_bit(AFS_VNODE_DELETED, &vnode->flags)); + + afs_prune_wb_keys(vnode); + afs_put_permits(rcu_access_pointer(vnode->permit_cache)); + key_put(vnode->silly_key); + vnode->silly_key = NULL; + key_put(vnode->lock_key); + vnode->lock_key = NULL; + _leave(""); +} + +static void afs_setattr_success(struct afs_operation *op) +{ + struct afs_vnode_param *vp = &op->file[0]; + struct inode *inode = &vp->vnode->netfs.inode; + loff_t old_i_size = i_size_read(inode); + + op->setattr.old_i_size = old_i_size; + afs_vnode_commit_status(op, vp); + /* inode->i_size has now been changed. */ + + if (op->setattr.attr->ia_valid & ATTR_SIZE) { + loff_t size = op->setattr.attr->ia_size; + if (size > old_i_size) + pagecache_isize_extended(inode, old_i_size, size); + } +} + +static void afs_setattr_edit_file(struct afs_operation *op) +{ + struct afs_vnode_param *vp = &op->file[0]; + struct afs_vnode *vnode = vp->vnode; + struct inode *inode = &vnode->netfs.inode; + + if (op->setattr.attr->ia_valid & ATTR_SIZE) { + loff_t size = op->setattr.attr->ia_size; + loff_t old = op->setattr.old_i_size; + + /* Note: inode->i_size was updated by afs_apply_status() inside + * the I/O and callback locks. + */ + + if (size != old) { + truncate_pagecache(inode, size); + netfs_resize_file(&vnode->netfs, size, true); + fscache_resize_cookie(afs_vnode_cache(vnode), size); + } + } +} + +static const struct afs_operation_ops afs_setattr_operation = { + .issue_afs_rpc = afs_fs_setattr, + .issue_yfs_rpc = yfs_fs_setattr, + .success = afs_setattr_success, + .edit_dir = afs_setattr_edit_file, +}; + +/* + * set the attributes of an inode + */ +int afs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +{ + const unsigned int supported = + ATTR_SIZE | ATTR_MODE | ATTR_UID | ATTR_GID | + ATTR_MTIME | ATTR_MTIME_SET | ATTR_TIMES_SET | ATTR_TOUCH; + struct afs_operation *op; + struct afs_vnode *vnode = AFS_FS_I(d_inode(dentry)); + struct inode *inode = &vnode->netfs.inode; + loff_t i_size; + int ret; + + _enter("{%llx:%llu},{n=%pd},%x", + vnode->fid.vid, vnode->fid.vnode, dentry, + attr->ia_valid); + + if (!(attr->ia_valid & supported)) { + _leave(" = 0 [unsupported]"); + return 0; + } + + i_size = i_size_read(inode); + if (attr->ia_valid & ATTR_SIZE) { + if (!S_ISREG(inode->i_mode)) + return -EISDIR; + + ret = inode_newsize_ok(inode, attr->ia_size); + if (ret) + return ret; + + if (attr->ia_size == i_size) + attr->ia_valid &= ~ATTR_SIZE; + } + + fscache_use_cookie(afs_vnode_cache(vnode), true); + + /* Prevent any new writebacks from starting whilst we do this. */ + down_write(&vnode->validate_lock); + + if ((attr->ia_valid & ATTR_SIZE) && S_ISREG(inode->i_mode)) { + loff_t size = attr->ia_size; + + /* Wait for any outstanding writes to the server to complete */ + loff_t from = min(size, i_size); + loff_t to = max(size, i_size); + ret = filemap_fdatawait_range(inode->i_mapping, from, to); + if (ret < 0) + goto out_unlock; + + /* Don't talk to the server if we're just shortening in-memory + * writes that haven't gone to the server yet. + */ + if (!(attr->ia_valid & (supported & ~ATTR_SIZE & ~ATTR_MTIME)) && + attr->ia_size < i_size && + attr->ia_size > netfs_read_remote_i_size(inode)) { + truncate_setsize(inode, attr->ia_size); + netfs_resize_file(&vnode->netfs, size, false); + fscache_resize_cookie(afs_vnode_cache(vnode), + attr->ia_size); + ret = 0; + goto out_unlock; + } + } + + op = afs_alloc_operation(((attr->ia_valid & ATTR_FILE) ? + afs_file_key(attr->ia_file) : NULL), + vnode->volume); + if (IS_ERR(op)) { + ret = PTR_ERR(op); + goto out_unlock; + } + + afs_op_set_vnode(op, 0, vnode); + op->setattr.attr = attr; + + if (attr->ia_valid & ATTR_SIZE) { + op->file[0].dv_delta = 1; + op->file[0].set_size = true; + } + op->ctime = attr->ia_ctime; + op->file[0].update_ctime = 1; + op->file[0].modification = true; + + op->ops = &afs_setattr_operation; + ret = afs_do_sync_operation(op); + +out_unlock: + up_write(&vnode->validate_lock); + fscache_unuse_cookie(afs_vnode_cache(vnode), NULL, NULL); + _leave(" = %d", ret); + return ret; +} diff --git a/fs/afs/internal.h b/fs/afs/internal.h new file mode 100644 index 000000000..290873bac --- /dev/null +++ b/fs/afs/internal.h @@ -0,0 +1,1970 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* internal AFS stuff + * + * Copyright (C) 2002, 2007 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "afs.h" +#include "afs_vl.h" + +#define AFS_CELL_MAX_ADDRS 15 + +struct afs_call; +struct afs_vnode; +struct afs_server_probe; + +/* + * Partial file-locking emulation mode. (The problem being that AFS3 only + * allows whole-file locks and no upgrading/downgrading). + */ +enum afs_flock_mode { + afs_flock_mode_unset, + afs_flock_mode_local, /* Local locking only */ + afs_flock_mode_openafs, /* Don't get server lock for a partial lock */ + afs_flock_mode_strict, /* Always get a server lock for a partial lock */ + afs_flock_mode_write, /* Get an exclusive server lock for a partial lock */ +}; + +struct afs_fs_context { + bool force; /* T to force cell type */ + bool autocell; /* T if set auto mount operation */ + bool dyn_root; /* T if dynamic root */ + bool no_cell; /* T if the source is "none" (for dynroot) */ + enum afs_flock_mode flock_mode; /* Partial file-locking emulation mode */ + afs_voltype_t type; /* type of volume requested */ + unsigned int volnamesz; /* size of volume name */ + const char *volname; /* name of volume to mount */ + struct afs_net *net; /* the AFS net namespace stuff */ + struct afs_cell *cell; /* cell in which to find volume */ + struct afs_volume *volume; /* volume record */ + struct key *key; /* key to use for secure mounting */ +}; + +enum afs_call_state { + AFS_CALL_CL_REQUESTING, /* Client: Request is being sent */ + AFS_CALL_CL_AWAIT_REPLY, /* Client: Awaiting reply */ + AFS_CALL_CL_PROC_REPLY, /* Client: rxrpc call complete; processing reply */ + AFS_CALL_SV_AWAIT_OP_ID, /* Server: Awaiting op ID */ + AFS_CALL_SV_AWAIT_REQUEST, /* Server: Awaiting request data */ + AFS_CALL_SV_REPLYING, /* Server: Replying */ + AFS_CALL_SV_AWAIT_ACK, /* Server: Awaiting final ACK */ + AFS_CALL_COMPLETE, /* Completed or failed */ +}; + +/* + * Address preferences. + */ +struct afs_addr_preference { + union { + struct in_addr ipv4_addr; /* AF_INET address to compare against */ + struct in6_addr ipv6_addr; /* AF_INET6 address to compare against */ + }; + sa_family_t family; /* Which address to use */ + u16 prio; /* Priority */ + u8 subnet_mask; /* How many bits to compare */ +}; + +struct afs_addr_preference_list { + struct rcu_head rcu; + u16 version; /* Incremented when prefs list changes */ + u8 ipv6_off; /* Offset of IPv6 addresses */ + u8 nr; /* Number of addresses in total */ + u8 max_prefs; /* Number of prefs allocated */ + struct afs_addr_preference prefs[] __counted_by(max_prefs); +}; + +struct afs_address { + struct rxrpc_peer *peer; + short last_error; /* Last error from this address */ + u16 prio; /* Address priority */ +}; + +/* + * List of server addresses. + */ +struct afs_addr_list { + struct rcu_head rcu; + refcount_t usage; + u32 version; /* Version */ + unsigned int debug_id; + unsigned int addr_pref_version; /* Version of address preference list */ + unsigned char max_addrs; + unsigned char nr_addrs; + unsigned char preferred; /* Preferred address */ + unsigned char nr_ipv4; /* Number of IPv4 addresses */ + enum dns_record_source source:8; + enum dns_lookup_status status:8; + unsigned long probe_failed; /* Mask of addrs that failed locally/ICMP */ + unsigned long responded; /* Mask of addrs that responded */ + struct afs_address addrs[] __counted_by(max_addrs); +#define AFS_MAX_ADDRESSES ((unsigned int)(sizeof(unsigned long) * 8)) +}; + +/* + * a record of an in-progress RxRPC call + */ +struct afs_call { + const struct afs_call_type *type; /* type of call */ + wait_queue_head_t waitq; /* processes awaiting completion */ + struct work_struct async_work; /* async I/O processor */ + struct work_struct work; /* actual work processor */ + struct work_struct free_work; /* Deferred free processor */ + struct rxrpc_call *rxcall; /* RxRPC call handle */ + struct rxrpc_peer *peer; /* Remote endpoint */ + struct key *key; /* security for this call */ + struct afs_net *net; /* The network namespace */ + struct afs_server *server; /* The fileserver record if fs op (pins ref) */ + struct afs_vlserver *vlserver; /* The vlserver record if vl op */ + void *request; /* request data (first part) */ + size_t iov_len; /* Size of *iter to be used */ + struct iov_iter def_iter; /* Default buffer/data iterator */ + struct iov_iter *write_iter; /* Iterator defining write to be made */ + struct iov_iter *iter; /* Iterator currently in use */ + union { /* Convenience for ->def_iter */ + struct kvec kvec[1]; + struct bio_vec bvec[1]; + }; + void *buffer; /* reply receive buffer */ + union { + struct afs_endpoint_state *probe; + struct afs_addr_list *vl_probe; + struct afs_addr_list *ret_alist; + struct afs_vldb_entry *ret_vldb; + char *ret_str; + }; + struct afs_fid fid; /* Primary vnode ID (or all zeroes) */ + unsigned char probe_index; /* Address in ->probe_alist */ + struct afs_operation *op; + unsigned int server_index; + refcount_t ref; + enum afs_call_state state; + spinlock_t state_lock; + int error; /* error code */ + u32 abort_code; /* Remote abort ID or 0 */ + unsigned long long remaining; /* How much is left to receive */ + unsigned int max_lifespan; /* Maximum lifespan in secs to set if not 0 */ + unsigned request_size; /* size of request data */ + unsigned reply_max; /* maximum size of reply */ + unsigned count2; /* count used in unmarshalling */ + unsigned char unmarshall; /* unmarshalling phase */ + bool drop_ref; /* T if need to drop ref for incoming call */ + bool need_attention; /* T if RxRPC poked us */ + bool async; /* T if asynchronous */ + bool upgrade; /* T to request service upgrade */ + bool intr; /* T if interruptible */ + bool unmarshalling_error; /* T if an unmarshalling error occurred */ + bool responded; /* Got a response from the call (may be abort) */ + u8 security_ix; /* Security class */ + u16 service_id; /* Actual service ID (after upgrade) */ + unsigned int debug_id; /* Trace ID */ + u32 enctype; /* Security encoding type */ + u32 operation_ID; /* operation ID for an incoming call */ + u32 count; /* count for use in unmarshalling */ + union { /* place to extract temporary data */ + struct { + __be32 tmp_u; + __be32 tmp; + } __attribute__((packed)); + __be64 tmp64; + }; + ktime_t issue_time; /* Time of issue of operation */ +}; + +struct afs_call_type { + const char *name; + unsigned int op; /* Really enum afs_fs_operation */ + + /* deliver request or reply data to an call + * - returning an error will cause the call to be aborted + */ + int (*deliver)(struct afs_call *call); + + /* clean up a call */ + void (*destructor)(struct afs_call *call); + + /* Async receive processing function */ + void (*async_rx)(struct work_struct *work); + + /* Work function */ + void (*work)(struct work_struct *work); + + /* Call done function (gets called immediately on success or failure) */ + void (*done)(struct afs_call *call); + + /* Handle a call being immediately cancelled. */ + void (*immediate_cancel)(struct afs_call *call); +}; + +/* + * Key available for writeback on a file. + */ +struct afs_wb_key { + refcount_t usage; + struct key *key; + struct list_head vnode_link; /* Link in vnode->wb_keys */ +}; + +/* + * AFS open file information record. Pointed to by file->private_data. + */ +struct afs_file { + struct key *key; /* The key this file was opened with */ + struct afs_wb_key *wb; /* Writeback key record for this file */ +}; + +static inline struct key *afs_file_key(struct file *file) +{ + struct afs_file *af = file->private_data; + + return af->key; +} + +/* + * AFS superblock private data + * - there's one superblock per volume + */ +struct afs_super_info { + struct net *net_ns; /* Network namespace */ + struct afs_cell *cell; /* The cell in which the volume resides */ + struct afs_volume *volume; /* volume record */ + enum afs_flock_mode flock_mode:8; /* File locking emulation mode */ + bool dyn_root; /* True if dynamic root */ +}; + +static inline struct afs_super_info *AFS_FS_S(struct super_block *sb) +{ + return sb->s_fs_info; +} + +extern struct file_system_type afs_fs_type; + +/* + * Set of substitutes for @sys. + */ +struct afs_sysnames { +#define AFS_NR_SYSNAME 16 + char *subs[AFS_NR_SYSNAME]; + refcount_t usage; + unsigned short nr; + char blank[1]; +}; + +/* + * AFS network namespace record. + */ +struct afs_net { + struct net *net; /* Backpointer to the owning net namespace */ + struct afs_uuid uuid; + bool live; /* F if this namespace is being removed */ + + /* AF_RXRPC I/O stuff */ + struct socket *socket; + struct afs_call *spare_incoming_call; + struct work_struct charge_preallocation_work; + struct work_struct rx_oob_work; + struct mutex socket_mutex; + atomic_t nr_outstanding_calls; + atomic_t nr_superblocks; + + /* Cell database */ + struct rb_root cells; + struct idr cells_dyn_ino; /* cell->dynroot_ino mapping */ + struct afs_cell __rcu *ws_cell; + atomic_t cells_outstanding; + struct rw_semaphore cells_lock; + struct mutex cells_alias_lock; + + struct mutex proc_cells_lock; + struct hlist_head proc_cells; + + /* Known servers. Theoretically each fileserver can only be in one + * cell, but in practice, people create aliases and subsets and there's + * no easy way to distinguish them. + */ + seqlock_t fs_lock; /* For fs_probe_*, fs_proc */ + struct list_head fs_probe_fast; /* List of afs_server to probe at 30s intervals */ + struct list_head fs_probe_slow; /* List of afs_server to probe at 5m intervals */ + struct hlist_head fs_proc; /* procfs servers list */ + + struct key *fs_cm_token_key; /* Key for creating CM tokens */ + struct work_struct fs_prober; + struct timer_list fs_probe_timer; + atomic_t servers_outstanding; + + /* File locking renewal management */ + struct mutex lock_manager_mutex; + + /* Misc */ + struct super_block *dynroot_sb; /* Dynamic root mount superblock */ + struct proc_dir_entry *proc_afs; /* /proc/net/afs directory */ + struct afs_sysnames *sysnames; + rwlock_t sysnames_lock; + struct afs_addr_preference_list __rcu *address_prefs; + u16 address_pref_version; + + /* Statistics counters */ + atomic_t n_lookup; /* Number of lookups done */ + atomic_t n_reval; /* Number of dentries needing revalidation */ + atomic_t n_inval; /* Number of invalidations by the server */ + atomic_t n_relpg; /* Number of invalidations by release_folio */ + atomic_t n_read_dir; /* Number of directory pages read */ + atomic_t n_dir_cr; /* Number of directory entry creation edits */ + atomic_t n_dir_rm; /* Number of directory entry removal edits */ + atomic_t n_stores; /* Number of store ops */ + atomic_long_t n_store_bytes; /* Number of bytes stored */ + atomic_long_t n_fetch_bytes; /* Number of bytes fetched */ + atomic_t n_fetches; /* Number of data fetch ops */ +}; + +extern const char afs_init_sysname[]; + +enum afs_cell_state { + AFS_CELL_SETTING_UP, + AFS_CELL_UNLOOKED, + AFS_CELL_ACTIVE, + AFS_CELL_REMOVING, + AFS_CELL_DEAD, +}; + +/* + * AFS cell record. + * + * This is a tricky concept to get right as it is possible to create aliases + * simply by pointing AFSDB/SRV records for two names at the same set of VL + * servers; it is also possible to do things like setting up two sets of VL + * servers, one of which provides a superset of the volumes provided by the + * other (for internal/external division, for example). + * + * Cells only exist in the sense that (a) a cell's name maps to a set of VL + * servers and (b) a cell's name is used by the client to select the key to use + * for authentication and encryption. The cell name is not typically used in + * the protocol. + * + * Two cells are determined to be aliases if they have an explicit alias (YFS + * only), share any VL servers in common or have at least one volume in common. + * "In common" means that the address list of the VL servers or the fileservers + * share at least one endpoint. + */ +struct afs_cell { + union { + struct rcu_head rcu; + struct rb_node net_node; /* Node in net->cells */ + }; + struct afs_net *net; + struct afs_cell *alias_of; /* The cell this is an alias of */ + struct afs_volume *root_volume; /* The root.cell volume if there is one */ + struct key *anonymous_key; /* anonymous user key for this cell */ + struct work_struct destroyer; /* Destroyer for cell */ + struct work_struct manager; /* Manager for init/deinit/dns */ + struct timer_list management_timer; /* General management timer */ + struct hlist_node proc_link; /* /proc cell list link */ + time64_t dns_expiry; /* Time AFSDB/SRV record expires */ + time64_t last_inactive; /* Time of last drop of usage count */ + refcount_t ref; /* Struct refcount */ + atomic_t active; /* Active usage counter */ + unsigned long flags; +#define AFS_CELL_FL_NO_GC 0 /* The cell was added manually, don't auto-gc */ +#define AFS_CELL_FL_DO_LOOKUP 1 /* DNS lookup requested */ +#define AFS_CELL_FL_CHECK_ALIAS 2 /* Need to check for aliases */ +#define AFS_CELL_FL_HAVE_INO 3 /* Have dynroot_ino */ + enum afs_cell_state state; + short error; + enum dns_record_source dns_source:8; /* Latest source of data from lookup */ + enum dns_lookup_status dns_status:8; /* Latest status of data from lookup */ + unsigned int dns_lookup_count; /* Counter of DNS lookups */ + unsigned int debug_id; + unsigned int dynroot_ino; /* Inode numbers for dynroot (a pair) */ + + /* The volumes belonging to this cell */ + struct rw_semaphore vs_lock; /* Lock for server->volumes */ + struct rb_root volumes; /* Tree of volumes on this server */ + struct hlist_head proc_volumes; /* procfs volume list */ + seqlock_t volume_lock; /* For volumes */ + + /* Active fileserver interaction state. */ + struct rb_root fs_servers; /* afs_server (by server UUID) */ + struct rw_semaphore fs_lock; /* For fs_servers */ + + /* VL server list. */ + rwlock_t vl_servers_lock; /* Lock on vl_servers */ + struct afs_vlserver_list __rcu *vl_servers; + + u8 name_len; /* Length of name */ + char *name; /* Cell name, case-flattened and NUL-padded */ + char *key_desc; /* Authentication key description */ +}; + +/* + * Volume Location server record. + */ +struct afs_vlserver { + struct rcu_head rcu; + struct afs_addr_list __rcu *addresses; /* List of addresses for this VL server */ + unsigned long flags; +#define AFS_VLSERVER_FL_PROBED 0 /* The VL server has been probed */ +#define AFS_VLSERVER_FL_PROBING 1 /* VL server is being probed */ +#define AFS_VLSERVER_FL_IS_YFS 2 /* Server is YFS not AFS */ +#define AFS_VLSERVER_FL_RESPONDING 3 /* VL server is responding */ + rwlock_t lock; /* Lock on addresses */ + refcount_t ref; + unsigned int rtt; /* Server's current RTT in uS */ + unsigned int debug_id; + + /* Probe state */ + wait_queue_head_t probe_wq; + atomic_t probe_outstanding; + spinlock_t probe_lock; + struct { + unsigned int rtt; /* Best RTT in uS (or UINT_MAX) */ + u32 abort_code; + short error; + unsigned short flags; +#define AFS_VLSERVER_PROBE_RESPONDED 0x01 /* At least once response (may be abort) */ +#define AFS_VLSERVER_PROBE_IS_YFS 0x02 /* The peer appears to be YFS */ +#define AFS_VLSERVER_PROBE_NOT_YFS 0x04 /* The peer appears not to be YFS */ +#define AFS_VLSERVER_PROBE_LOCAL_FAILURE 0x08 /* A local failure prevented a probe */ + } probe; + + u16 service_id; /* Service ID we're using */ + u16 port; + u16 name_len; /* Length of name */ + char name[]; /* Server name, case-flattened */ +}; + +/* + * Weighted list of Volume Location servers. + */ +struct afs_vlserver_entry { + u16 priority; /* Preference (as SRV) */ + u16 weight; /* Weight (as SRV) */ + enum dns_record_source source:8; + enum dns_lookup_status status:8; + struct afs_vlserver *server; +}; + +struct afs_vlserver_list { + struct rcu_head rcu; + refcount_t ref; + u8 nr_servers; + u8 index; /* Server currently in use */ + u8 preferred; /* Preferred server */ + enum dns_record_source source:8; + enum dns_lookup_status status:8; + rwlock_t lock; + struct afs_vlserver_entry servers[]; +}; + +/* + * Cached VLDB entry. + * + * This is pointed to by cell->vldb_entries, indexed by name. + */ +struct afs_vldb_entry { + afs_volid_t vid[3]; /* Volume IDs for R/W, R/O and Bak volumes */ + + unsigned long flags; +#define AFS_VLDB_HAS_RW 0 /* - R/W volume exists */ +#define AFS_VLDB_HAS_RO 1 /* - R/O volume exists */ +#define AFS_VLDB_HAS_BAK 2 /* - Backup volume exists */ +#define AFS_VLDB_QUERY_VALID 3 /* - Record is valid */ +#define AFS_VLDB_QUERY_ERROR 4 /* - VL server returned error */ + + uuid_t fs_server[AFS_NMAXNSERVERS]; + u32 addr_version[AFS_NMAXNSERVERS]; /* Registration change counters */ + u8 fs_mask[AFS_NMAXNSERVERS]; +#define AFS_VOL_VTM_RW 0x01 /* R/W version of the volume is available (on this server) */ +#define AFS_VOL_VTM_RO 0x02 /* R/O version of the volume is available (on this server) */ +#define AFS_VOL_VTM_BAK 0x04 /* backup version of the volume is available (on this server) */ + u8 vlsf_flags[AFS_NMAXNSERVERS]; + short error; + u8 nr_servers; /* Number of server records */ + u8 name_len; + u8 name[AFS_MAXVOLNAME + 1]; /* NUL-padded volume name */ +}; + +/* + * Fileserver endpoint state. The records the addresses of a fileserver's + * endpoints and the state and result of a round of probing on them. This + * allows the rotation algorithm to access those results without them being + * erased by a subsequent round of probing. + */ +struct afs_endpoint_state { + struct rcu_head rcu; + struct afs_addr_list *addresses; /* The addresses being probed */ + unsigned long responsive_set; /* Bitset of responsive endpoints */ + unsigned long failed_set; /* Bitset of endpoints we failed to probe */ + refcount_t ref; + unsigned int server_id; /* Debug ID of server */ + unsigned int probe_seq; /* Probe sequence (from server::probe_counter) */ + atomic_t nr_probing; /* Number of outstanding probes */ + unsigned int rtt; /* Best RTT in uS (or UINT_MAX) */ + s32 abort_code; + short error; + unsigned long flags; +#define AFS_ESTATE_RESPONDED 0 /* Set if the server responded */ +#define AFS_ESTATE_SUPERSEDED 1 /* Set if this record has been superseded */ +#define AFS_ESTATE_IS_YFS 2 /* Set if probe upgraded to YFS */ +#define AFS_ESTATE_NOT_YFS 3 /* Set if probe didn't upgrade to YFS */ +#define AFS_ESTATE_LOCAL_FAILURE 4 /* Set if there was a local failure (eg. ENOMEM) */ +}; + +/* + * Record of fileserver with which we're actively communicating. + */ +struct afs_server { + struct rcu_head rcu; + union { + uuid_t uuid; /* Server ID */ + struct afs_uuid _uuid; + }; + + struct afs_cell *cell; /* Cell to which belongs (pins ref) */ + struct rb_node uuid_rb; /* Link in cell->fs_servers */ + struct list_head probe_link; /* Link in net->fs_probe_* */ + struct hlist_node proc_link; /* Link in net->fs_proc */ + struct list_head volumes; /* RCU list of afs_server_entry objects */ + struct work_struct destroyer; /* Work item to try and destroy a server */ + struct timer_list timer; /* Management timer */ + struct mutex cm_token_lock; /* Lock governing creation of appdata */ + struct krb5_buffer cm_rxgk_appdata; /* Appdata to be included in RESPONSE packet */ + time64_t unuse_time; /* Time at which last unused */ + unsigned long flags; +#define AFS_SERVER_FL_RESPONDING 0 /* The server is responding */ +#define AFS_SERVER_FL_UPDATING 1 +#define AFS_SERVER_FL_NEEDS_UPDATE 2 /* Fileserver address list is out of date */ +#define AFS_SERVER_FL_UNCREATED 3 /* The record needs creating */ +#define AFS_SERVER_FL_CREATING 4 /* The record is being created */ +#define AFS_SERVER_FL_EXPIRED 5 /* The record has expired */ +#define AFS_SERVER_FL_NOT_FOUND 6 /* VL server says no such server */ +#define AFS_SERVER_FL_VL_FAIL 7 /* Failed to access VL server */ +#define AFS_SERVER_FL_MAY_HAVE_CB 8 /* May have callbacks on this fileserver */ +#define AFS_SERVER_FL_IS_YFS 16 /* Server is YFS not AFS */ +#define AFS_SERVER_FL_NO_IBULK 17 /* Fileserver doesn't support FS.InlineBulkStatus */ +#define AFS_SERVER_FL_NO_RM2 18 /* Fileserver doesn't support YFS.RemoveFile2 */ +#define AFS_SERVER_FL_HAS_FS64 19 /* Fileserver supports FS.{Fetch,Store}Data64 */ +#define AFS_SERVER_FL_NO_RENAME2 20 /* YFS Fileserver doesn't support enhanced rename */ + refcount_t ref; /* Object refcount */ + atomic_t active; /* Active user count */ + u32 addr_version; /* Address list version */ + u16 service_id; /* Service ID we're using. */ + short create_error; /* Creation error */ + unsigned int rtt; /* Server's current RTT in uS */ + unsigned int debug_id; /* Debugging ID for traces */ + + /* file service access */ + rwlock_t fs_lock; /* access lock */ + + /* Probe state */ + struct afs_endpoint_state __rcu *endpoint_state; /* Latest endpoint/probe state */ + unsigned long probed_at; /* Time last probe was dispatched (jiffies) */ + wait_queue_head_t probe_wq; + unsigned int probe_counter; /* Number of probes issued */ + spinlock_t probe_lock; +}; + +enum afs_ro_replicating { + AFS_RO_NOT_REPLICATING, /* Not doing replication */ + AFS_RO_REPLICATING_USE_OLD, /* Replicating; use old version */ + AFS_RO_REPLICATING_USE_NEW, /* Replicating; switch to new version */ +} __mode(byte); + +/* + * Replaceable volume server list. + */ +struct afs_server_entry { + struct afs_server *server; + struct afs_volume *volume; + struct list_head slink; /* Link in server->volumes */ + time64_t cb_expires_at; /* Time at which volume-level callback expires */ + unsigned long flags; +#define AFS_SE_EXCLUDED 0 /* Set if server is to be excluded in rotation */ +#define AFS_SE_VOLUME_OFFLINE 1 /* Set if volume offline notice given */ +#define AFS_SE_VOLUME_BUSY 2 /* Set if volume busy notice given */ +}; + +struct afs_server_list { + struct rcu_head rcu; + refcount_t usage; + bool attached; /* T if attached to servers */ + enum afs_ro_replicating ro_replicating; /* RW->RO update (probably) in progress */ + unsigned char nr_servers; + unsigned short vnovol_mask; /* Servers to be skipped due to VNOVOL */ + unsigned int seq; /* Set to ->servers_seq when installed */ + rwlock_t lock; + struct afs_server_entry servers[]; +}; + +/* + * Live AFS volume management. + */ +struct afs_volume { + struct rcu_head rcu; + afs_volid_t vid; /* The volume ID of this volume */ + afs_volid_t vids[AFS_MAXTYPES]; /* All associated volume IDs */ + refcount_t ref; + unsigned int debug_id; /* Debugging ID for traces */ + time64_t update_at; /* Time at which to next update */ + struct afs_cell *cell; /* Cell to which belongs (pins ref) */ + struct rb_node cell_node; /* Link in cell->volumes */ + struct hlist_node proc_link; /* Link in cell->proc_volumes */ + struct super_block __rcu *sb; /* Superblock on which inodes reside */ + struct work_struct destructor; /* Deferred destructor */ + unsigned long flags; +#define AFS_VOLUME_NEEDS_UPDATE 0 /* - T if an update needs performing */ +#define AFS_VOLUME_UPDATING 1 /* - T if an update is in progress */ +#define AFS_VOLUME_WAIT 2 /* - T if users must wait for update */ +#define AFS_VOLUME_DELETED 3 /* - T if volume appears deleted */ +#define AFS_VOLUME_MAYBE_NO_IBULK 4 /* - T if some servers don't have InlineBulkStatus */ +#define AFS_VOLUME_RM_TREE 5 /* - Set if volume removed from cell->volumes */ +#ifdef CONFIG_AFS_FSCACHE + struct fscache_volume *cache; /* Caching cookie */ +#endif + struct afs_server_list __rcu *servers; /* List of servers on which volume resides */ + rwlock_t servers_lock; /* Lock for ->servers */ + unsigned int servers_seq; /* Incremented each time ->servers changes */ + + /* RO release tracking */ + struct mutex volsync_lock; /* Time/state evaluation lock */ + time64_t creation_time; /* Volume creation time (or TIME64_MIN) */ + time64_t update_time; /* Volume update time (or TIME64_MIN) */ + + /* Callback management */ + struct mutex cb_check_lock; /* Lock to control race to check after v_break */ + time64_t cb_expires_at; /* Earliest volume callback expiry time */ + atomic_t cb_ro_snapshot; /* RO volume update-from-snapshot counter */ + atomic_t cb_v_break; /* Volume-break event counter. */ + atomic_t cb_v_check; /* Volume-break has-been-checked counter. */ + atomic_t cb_scrub; /* Scrub-all-data event counter. */ + rwlock_t cb_v_break_lock; + struct rw_semaphore open_mmaps_lock; + struct list_head open_mmaps; /* List of vnodes that are mmapped */ + + afs_voltype_t type; /* type of volume */ + char type_force; /* force volume type (suppress R/O -> R/W) */ + u8 name_len; + u8 name[AFS_MAXVOLNAME + 1]; /* NUL-padded volume name */ +}; + +enum afs_lock_state { + AFS_VNODE_LOCK_NONE, /* The vnode has no lock on the server */ + AFS_VNODE_LOCK_WAITING_FOR_CB, /* We're waiting for the server to break the callback */ + AFS_VNODE_LOCK_SETTING, /* We're asking the server for a lock */ + AFS_VNODE_LOCK_GRANTED, /* We have a lock on the server */ + AFS_VNODE_LOCK_EXTENDING, /* We're extending a lock on the server */ + AFS_VNODE_LOCK_NEED_UNLOCK, /* We need to unlock on the server */ + AFS_VNODE_LOCK_UNLOCKING, /* We're telling the server to unlock */ + AFS_VNODE_LOCK_DELETED, /* The vnode has been deleted whilst we have a lock */ +}; + +/* + * AFS inode private data. + * + * Note that afs_alloc_inode() *must* reset anything that could incorrectly + * leak from one inode to another. + */ +struct afs_vnode { + struct netfs_inode netfs; /* Netfslib context and vfs inode */ + struct afs_volume *volume; /* volume on which vnode resides */ + struct afs_fid fid; /* the file identifier for this inode */ + struct afs_file_status status; /* AFS status info for this file */ + afs_dataversion_t invalid_before; /* Child dentries are invalid before this */ + struct afs_permits __rcu *permit_cache; /* cache of permits so far obtained */ + struct list_head io_lock_waiters; /* Threads waiting for the I/O lock */ + struct rw_semaphore validate_lock; /* lock for validating this vnode */ + struct rw_semaphore rmdir_lock; /* Lock for rmdir vs sillyrename */ + struct key *silly_key; /* Silly rename key */ + spinlock_t wb_lock; /* lock for wb_keys */ + spinlock_t lock; /* waitqueue/flags lock */ + unsigned long flags; +#define AFS_VNODE_IO_LOCK 0 /* Set if the I/O serialisation lock is held */ +#define AFS_VNODE_UNSET 1 /* set if vnode attributes not yet set */ +#define AFS_VNODE_DIR_VALID 2 /* Set if dir contents are valid */ +#define AFS_VNODE_ZAP_DATA 3 /* set if vnode's data should be invalidated */ +#define AFS_VNODE_DELETED 4 /* set if vnode deleted on server */ +#define AFS_VNODE_MOUNTPOINT 5 /* set if vnode is a mountpoint symlink */ +#define AFS_VNODE_PSEUDODIR 7 /* set if Vnode is a pseudo directory */ +#define AFS_VNODE_NEW_CONTENT 8 /* Set if file has new content (create/trunc-0) */ +#define AFS_VNODE_SILLY_DELETED 9 /* Set if file has been silly-deleted */ +#define AFS_VNODE_MODIFYING 10 /* Set if we're performing a modification op */ +#define AFS_VNODE_DIR_READ 11 /* Set if we've read a dir's contents */ + + struct folio_queue *directory; /* Directory contents */ + struct afs_symlink __rcu *symlink; /* Symlink content */ + struct list_head wb_keys; /* List of keys available for writeback */ + struct list_head pending_locks; /* locks waiting to be granted */ + struct list_head granted_locks; /* locks granted on this file */ + struct delayed_work lock_work; /* work to be done in locking */ + struct key *lock_key; /* Key to be used in lock ops */ + ktime_t locked_at; /* Time at which lock obtained */ + enum afs_lock_state lock_state : 8; + afs_lock_type_t lock_type : 8; + unsigned int directory_size; /* Amount of space in ->directory */ + + /* outstanding callback notification on this file */ + struct work_struct cb_work; /* Work for mmap'd files */ + struct list_head cb_mmap_link; /* Link in cell->fs_open_mmaps */ + void *cb_server; /* Server with callback/filelock */ + atomic_t cb_nr_mmap; /* Number of mmaps */ + unsigned int cb_ro_snapshot; /* RO volume release counter on ->volume */ + unsigned int cb_scrub; /* Scrub counter on ->volume */ + unsigned int cb_break; /* Break counter on vnode */ + unsigned int cb_v_check; /* Break check counter on ->volume */ + seqlock_t cb_lock; /* Lock for ->cb_server, ->status, ->cb_*break */ + + atomic64_t cb_expires_at; /* time at which callback expires */ +#define AFS_NO_CB_PROMISE TIME64_MIN +}; + +static inline struct fscache_cookie *afs_vnode_cache(struct afs_vnode *vnode) +{ +#ifdef CONFIG_AFS_FSCACHE + return netfs_i_cookie(&vnode->netfs); +#else + return NULL; +#endif +} + +static inline void afs_vnode_set_cache(struct afs_vnode *vnode, + struct fscache_cookie *cookie) +{ +#ifdef CONFIG_AFS_FSCACHE + vnode->netfs.cache = cookie; +#endif +} + +/* + * cached security record for one user's attempt to access a vnode + */ +struct afs_permit { + struct key *key; /* RxRPC ticket holding a security context */ + afs_access_t access; /* CallerAccess value for this key */ +}; + +/* + * Immutable cache of CallerAccess records from attempts to access vnodes. + * These may be shared between multiple vnodes. + */ +struct afs_permits { + struct rcu_head rcu; + struct hlist_node hash_node; /* Link in hash */ + unsigned long h; /* Hash value for this permit list */ + refcount_t usage; + unsigned short nr_permits; /* Number of records */ + bool invalidated; /* Invalidated due to key change */ + struct afs_permit permits[] __counted_by(nr_permits); /* List of permits sorted by key pointer */ +}; + +/* + * Copy of symlink content for normal use. + */ +struct afs_symlink { + struct rcu_head rcu; + refcount_t ref; + char content[]; +}; + +/* + * Error prioritisation and accumulation. + */ +struct afs_error { + s32 abort_code; /* Cumulative abort code */ + short error; /* Cumulative error */ + bool responded; /* T if server responded */ + bool aborted; /* T if ->error is from an abort */ +}; + +/* + * Cursor for iterating over a set of volume location servers. + */ +struct afs_vl_cursor { + struct afs_cell *cell; /* The cell we're querying */ + struct afs_vlserver_list *server_list; /* Current server list (pins ref) */ + struct afs_vlserver *server; /* Server on which this resides */ + struct afs_addr_list *alist; /* Current address list (pins ref) */ + struct key *key; /* Key for the server */ + unsigned long untried_servers; /* Bitmask of untried servers */ + unsigned long addr_tried; /* Tried addresses */ + struct afs_error cumul_error; /* Cumulative error */ + unsigned int debug_id; + s32 call_abort_code; + short call_error; /* Error from single call */ + short server_index; /* Current server */ + signed char addr_index; /* Current address */ + unsigned short flags; +#define AFS_VL_CURSOR_STOP 0x0001 /* Set to cease iteration */ +#define AFS_VL_CURSOR_RETRY 0x0002 /* Set to do a retry */ +#define AFS_VL_CURSOR_RETRIED 0x0004 /* Set if started a retry */ + short nr_iterations; /* Number of server iterations */ + bool call_responded; /* T if the current address responded */ +}; + +/* + * Fileserver state tracking for an operation. An array of these is kept, + * indexed by server index. + */ +struct afs_server_state { + /* Tracking of fileserver probe state. Other operations may interfere + * by probing a fileserver when accessing other volumes. + */ + unsigned int probe_seq; + unsigned long untried_addrs; /* Addresses we haven't tried yet */ + struct wait_queue_entry probe_waiter; + struct afs_endpoint_state *endpoint_state; /* Endpoint state being monitored */ +}; + +/* + * Fileserver operation methods. + */ +struct afs_operation_ops { + void (*issue_afs_rpc)(struct afs_operation *op); + void (*issue_yfs_rpc)(struct afs_operation *op); + void (*success)(struct afs_operation *op); + void (*aborted)(struct afs_operation *op); + void (*failed)(struct afs_operation *op); + void (*edit_dir)(struct afs_operation *op); + void (*put)(struct afs_operation *op); +}; + +struct afs_vnode_param { + struct afs_vnode *vnode; + struct afs_fid fid; /* Fid to access */ + struct afs_status_cb scb; /* Returned status and callback promise */ + afs_dataversion_t dv_before; /* Data version before the call */ + unsigned int cb_break_before; /* cb_break before the call */ + u8 dv_delta; /* Expected change in data version */ + bool put_vnode:1; /* T if we have a ref on the vnode */ + bool need_io_lock:1; /* T if we need the I/O lock on this */ + bool update_ctime:1; /* Need to update the ctime */ + bool set_size:1; /* Must update i_size */ + bool op_unlinked:1; /* True if file was unlinked by op */ + bool speculative:1; /* T if speculative status fetch (no vnode lock) */ + bool modification:1; /* Set if the content gets modified */ +}; + +/* + * Fileserver operation wrapper, handling server and address rotation + * asynchronously. May make simultaneous calls to multiple servers. + */ +struct afs_operation { + struct afs_net *net; /* Network namespace */ + struct key *key; /* Key for the cell */ + const struct afs_call_type *type; /* Type of call done */ + const struct afs_operation_ops *ops; + + /* Parameters/results for the operation */ + struct afs_volume *volume; /* Volume being accessed */ + struct afs_vnode_param file[2]; + struct afs_vnode_param *more_files; + struct afs_volsync pre_volsync; /* Volsync before op */ + struct afs_volsync volsync; /* Volsync returned by op */ + struct dentry *dentry; /* Dentry to be altered */ + struct dentry *dentry_2; /* Second dentry to be altered */ + struct timespec64 mtime; /* Modification time to record */ + struct timespec64 ctime; /* Change time to set */ + struct afs_error cumul_error; /* Cumulative error */ + short nr_files; /* Number of entries in file[], more_files */ + unsigned int debug_id; + + unsigned int cb_v_break; /* Volume break counter before op */ + + union { + struct { + int which; /* Which ->file[] to fetch for */ + } fetch_status; + struct { + int reason; /* enum afs_edit_dir_reason */ + mode_t mode; + struct afs_symlink *symlink; + } create; + struct { + bool need_rehash; + } unlink; + struct { + struct dentry *rehash; + struct dentry *tmp; + unsigned int rename_flags; + bool new_negative; + } rename; + struct { + struct netfs_io_subrequest *subreq; + } fetch; + struct { + afs_lock_type_t type; + } lock; + struct { + struct iov_iter *write_iter; + loff_t pos; + loff_t size; + loff_t i_size; + } store; + struct { + struct iattr *attr; + loff_t old_i_size; + } setattr; + struct afs_acl *acl; + struct yfs_acl *yacl; + struct { + struct afs_volume_status vs; + struct kstatfs *buf; + } volstatus; + }; + + /* Fileserver iteration state */ + struct afs_server_list *server_list; /* Current server list (pins ref) */ + struct afs_server *server; /* Server we're using (ref pinned by server_list) */ + struct afs_endpoint_state *estate; /* Current endpoint state (doesn't pin ref) */ + struct afs_server_state *server_states; /* States of the servers involved */ + struct afs_call *call; + unsigned long untried_servers; /* Bitmask of untried servers */ + unsigned long addr_tried; /* Tried addresses */ + s32 call_abort_code; /* Abort code from single call */ + short call_error; /* Error from single call */ + short server_index; /* Current server */ + short nr_iterations; /* Number of server iterations */ + signed char addr_index; /* Current address */ + bool call_responded; /* T if the current address responded */ + + unsigned int flags; +#define AFS_OPERATION_STOP 0x0001 /* Set to cease iteration */ +#define AFS_OPERATION_VBUSY 0x0002 /* Set if seen VBUSY */ +#define AFS_OPERATION_VMOVED 0x0004 /* Set if seen VMOVED */ +#define AFS_OPERATION_VNOVOL 0x0008 /* Set if seen VNOVOL */ +#define AFS_OPERATION_CUR_ONLY 0x0010 /* Set if current server only (file lock held) */ +#define AFS_OPERATION_NO_VSLEEP 0x0020 /* Set to prevent sleep on VBUSY, VOFFLINE, ... */ +#define AFS_OPERATION_UNINTR 0x0040 /* Set if op is uninterruptible */ +#define AFS_OPERATION_DOWNGRADE 0x0080 /* Set to retry with downgraded opcode */ +#define AFS_OPERATION_LOCK_0 0x0100 /* Set if have io_lock on file[0] */ +#define AFS_OPERATION_LOCK_1 0x0200 /* Set if have io_lock on file[1] */ +#define AFS_OPERATION_TRIED_ALL 0x0400 /* Set if we've tried all the fileservers */ +#define AFS_OPERATION_RETRY_SERVER 0x0800 /* Set if we should retry the current server */ +#define AFS_OPERATION_DIR_CONFLICT 0x1000 /* Set if we detected a 3rd-party dir change */ +#define AFS_OPERATION_ASYNC 0x2000 /* Set if should run asynchronously */ +}; + +/* + * Cache auxiliary data. + */ +struct afs_vnode_cache_aux { + __be64 data_version; +} __packed; + +static inline void afs_set_cache_aux(struct afs_vnode *vnode, + struct afs_vnode_cache_aux *aux) +{ + aux->data_version = cpu_to_be64(vnode->status.data_version); +} + +static inline void afs_invalidate_cache(struct afs_vnode *vnode, unsigned int flags) +{ + struct afs_vnode_cache_aux aux; + + afs_set_cache_aux(vnode, &aux); + fscache_invalidate(afs_vnode_cache(vnode), &aux, + i_size_read(&vnode->netfs.inode), flags); +} + +/* + * Directory iteration management. + */ +struct afs_dir_iter { + struct afs_vnode *dvnode; + union afs_xdr_dir_block *block; + struct folio_queue *fq; + unsigned int fpos; + int fq_slot; + unsigned int loop_check; + u8 nr_slots; + u8 bucket; + unsigned int prev_entry; +}; + +#include + +/*****************************************************************************/ +/* + * addr_list.c + */ +struct afs_addr_list *afs_get_addrlist(struct afs_addr_list *alist, enum afs_alist_trace reason); +extern struct afs_addr_list *afs_alloc_addrlist(unsigned int nr); +extern void afs_put_addrlist(struct afs_addr_list *alist, enum afs_alist_trace reason); +extern struct afs_vlserver_list *afs_parse_text_addrs(struct afs_net *, + const char *, size_t, char, + unsigned short, unsigned short); +bool afs_addr_list_same(const struct afs_addr_list *a, + const struct afs_addr_list *b); +extern struct afs_vlserver_list *afs_dns_query(struct afs_cell *, time64_t *); + +extern int afs_merge_fs_addr4(struct afs_net *net, struct afs_addr_list *addr, + __be32 xdr, u16 port); +extern int afs_merge_fs_addr6(struct afs_net *net, struct afs_addr_list *addr, + __be32 *xdr, u16 port); +void afs_set_peer_appdata(struct afs_server *server, + struct afs_addr_list *old_alist, + struct afs_addr_list *new_alist); + +/* + * addr_prefs.c + */ +int afs_proc_addr_prefs_write(struct file *file, char *buf, size_t size); +void afs_get_address_preferences_rcu(struct afs_net *net, struct afs_addr_list *alist); +void afs_get_address_preferences(struct afs_net *net, struct afs_addr_list *alist); + +/* + * callback.c + */ +extern void afs_invalidate_mmap_work(struct work_struct *); +extern void afs_init_callback_state(struct afs_server *); +extern void __afs_break_callback(struct afs_vnode *, enum afs_cb_break_reason); +extern void afs_break_callback(struct afs_vnode *, enum afs_cb_break_reason); +extern void afs_break_callbacks(struct afs_server *, size_t, struct afs_callback_break *); + +static inline unsigned int afs_calc_vnode_cb_break(struct afs_vnode *vnode) +{ + return vnode->cb_break + vnode->cb_ro_snapshot + vnode->cb_scrub; +} + +static inline bool afs_cb_is_broken(unsigned int cb_break, + const struct afs_vnode *vnode) +{ + return cb_break != (vnode->cb_break + + atomic_read(&vnode->volume->cb_ro_snapshot) + + atomic_read(&vnode->volume->cb_scrub)); +} + +/* + * cell.c + */ +extern int afs_cell_init(struct afs_net *, const char *); +extern struct afs_cell *afs_find_cell(struct afs_net *, const char *, unsigned, + enum afs_cell_trace); +enum afs_lookup_cell_for { + AFS_LOOKUP_CELL_DYNROOT, + AFS_LOOKUP_CELL_MOUNTPOINT, + AFS_LOOKUP_CELL_DIRECT_MOUNT, + AFS_LOOKUP_CELL_PRELOAD, + AFS_LOOKUP_CELL_ROOTCELL, + AFS_LOOKUP_CELL_ALIAS_CHECK, +}; +struct afs_cell *afs_lookup_cell(struct afs_net *net, + const char *name, unsigned int namesz, + const char *vllist, + enum afs_lookup_cell_for reason, + enum afs_cell_trace trace); +extern struct afs_cell *afs_use_cell(struct afs_cell *, enum afs_cell_trace); +void afs_unuse_cell(struct afs_cell *cell, enum afs_cell_trace reason); +extern struct afs_cell *afs_get_cell(struct afs_cell *, enum afs_cell_trace); +extern void afs_see_cell(struct afs_cell *, enum afs_cell_trace); +extern void afs_put_cell(struct afs_cell *, enum afs_cell_trace); +extern void afs_queue_cell(struct afs_cell *, enum afs_cell_trace); +void afs_set_cell_timer(struct afs_cell *cell, unsigned int delay_secs); +extern void __net_exit afs_cell_purge(struct afs_net *); + +/* + * cmservice.c + */ +extern bool afs_cm_incoming_call(struct afs_call *); + +/* + * cm_security.c + */ +void afs_process_oob_queue(struct work_struct *work); +#ifdef CONFIG_RXGK +int afs_create_token_key(struct afs_net *net, struct socket *socket); +#else +static inline int afs_create_token_key(struct afs_net *net, struct socket *socket) +{ + return 0; +} +#endif + +/* + * dir.c + */ +extern const struct file_operations afs_dir_file_operations; +extern const struct inode_operations afs_dir_inode_operations; +extern const struct address_space_operations afs_dir_aops; +extern const struct dentry_operations afs_fs_dentry_operations; + +ssize_t afs_read_dir(struct afs_vnode *dvnode, struct file *file) + __acquires(&dvnode->validate_lock); +extern void afs_d_release(struct dentry *); +extern void afs_check_for_remote_deletion(struct afs_operation *); + +/* + * dir_edit.c + */ +extern void afs_edit_dir_add(struct afs_vnode *, const struct qstr *, struct afs_fid *, + enum afs_edit_dir_reason); +extern void afs_edit_dir_remove(struct afs_vnode *, const struct qstr *, enum afs_edit_dir_reason); +void afs_edit_dir_update(struct afs_vnode *vnode, const struct qstr *name, + struct afs_vnode *new_dvnode, enum afs_edit_dir_reason why); +void afs_mkdir_init_dir(struct afs_vnode *dvnode, struct afs_vnode *parent_vnode); + +/* + * dir_search.c + */ +unsigned int afs_dir_hash_name(const struct qstr *name); +bool afs_dir_init_iter(struct afs_dir_iter *iter, const struct qstr *name); +union afs_xdr_dir_block *afs_dir_find_block(struct afs_dir_iter *iter, size_t block); +int afs_dir_search_bucket(struct afs_dir_iter *iter, const struct qstr *name, + struct afs_fid *_fid); +int afs_dir_search(struct afs_vnode *dvnode, const struct qstr *name, + struct afs_fid *_fid, afs_dataversion_t *_dir_version); + +/* + * dir_silly.c + */ +extern int afs_sillyrename(struct afs_vnode *, struct afs_vnode *, + struct dentry *, struct key *); +extern int afs_silly_iput(struct dentry *, struct inode *); + +/* + * dynroot.c + */ +extern const struct inode_operations afs_dynroot_inode_operations; +extern const struct dentry_operations afs_dynroot_dentry_operations; + +struct inode *afs_dynroot_iget_root(struct super_block *sb); + +/* + * file.c + */ +extern const struct address_space_operations afs_file_aops; +extern const struct inode_operations afs_file_inode_operations; +extern const struct file_operations afs_file_operations; +extern const struct afs_operation_ops afs_fetch_data_operation; +extern const struct netfs_request_ops afs_req_ops; + +extern int afs_cache_wb_key(struct afs_vnode *, struct afs_file *); +extern void afs_put_wb_key(struct afs_wb_key *); +extern int afs_open(struct inode *, struct file *); +extern int afs_release(struct inode *, struct file *); +void afs_fetch_data_async_rx(struct work_struct *work); +void afs_fetch_data_immediate_cancel(struct afs_call *call); +void afs_set_i_size(struct afs_vnode *vnode, loff_t new_i_size); + +/* + * flock.c + */ +extern struct workqueue_struct *afs_lock_manager; + +extern void afs_lock_op_done(struct afs_call *); +extern void afs_lock_work(struct work_struct *); +extern void afs_lock_may_be_available(struct afs_vnode *); +extern int afs_lock(struct file *, int, struct file_lock *); +extern int afs_flock(struct file *, int, struct file_lock *); + +/* + * fsclient.c + */ +extern void afs_fs_fetch_status(struct afs_operation *); +extern void afs_fs_fetch_data(struct afs_operation *); +extern void afs_fs_create_file(struct afs_operation *); +extern void afs_fs_make_dir(struct afs_operation *); +extern void afs_fs_remove_file(struct afs_operation *); +extern void afs_fs_remove_dir(struct afs_operation *); +extern void afs_fs_link(struct afs_operation *); +extern void afs_fs_symlink(struct afs_operation *); +extern void afs_fs_rename(struct afs_operation *); +extern void afs_fs_store_data(struct afs_operation *); +extern void afs_fs_setattr(struct afs_operation *); +extern void afs_fs_get_volume_status(struct afs_operation *); +extern void afs_fs_set_lock(struct afs_operation *); +extern void afs_fs_extend_lock(struct afs_operation *); +extern void afs_fs_release_lock(struct afs_operation *); +int afs_fs_give_up_all_callbacks(struct afs_net *net, struct afs_server *server, + struct afs_address *addr, struct key *key); +bool afs_fs_get_capabilities(struct afs_net *net, struct afs_server *server, + struct afs_endpoint_state *estate, unsigned int addr_index, + struct key *key); +extern void afs_fs_inline_bulk_status(struct afs_operation *); + +struct afs_acl { + u32 size; + u8 data[] __counted_by(size); +}; + +extern void afs_fs_fetch_acl(struct afs_operation *); +extern void afs_fs_store_acl(struct afs_operation *); + +/* + * fs_operation.c + */ +extern struct afs_operation *afs_alloc_operation(struct key *, struct afs_volume *); +extern int afs_put_operation(struct afs_operation *); +extern bool afs_begin_vnode_operation(struct afs_operation *); +extern void afs_end_vnode_operation(struct afs_operation *op); +extern void afs_wait_for_operation(struct afs_operation *); +extern int afs_do_sync_operation(struct afs_operation *); + +static inline void afs_op_set_vnode(struct afs_operation *op, unsigned int n, + struct afs_vnode *vnode) +{ + op->file[n].vnode = vnode; + op->file[n].need_io_lock = true; +} + +static inline void afs_op_set_fid(struct afs_operation *op, unsigned int n, + const struct afs_fid *fid) +{ + op->file[n].fid = *fid; +} + +/* + * fs_probe.c + */ +struct afs_endpoint_state *afs_get_endpoint_state(struct afs_endpoint_state *estate, + enum afs_estate_trace where); +void afs_put_endpoint_state(struct afs_endpoint_state *estate, enum afs_estate_trace where); +extern void afs_fileserver_probe_result(struct afs_call *); +int afs_fs_probe_fileserver(struct afs_net *net, struct afs_server *server, + struct afs_addr_list *new_alist, struct key *key); +int afs_wait_for_fs_probes(struct afs_operation *op, struct afs_server_state *states, bool intr); +extern void afs_probe_fileserver(struct afs_net *, struct afs_server *); +extern void afs_fs_probe_dispatcher(struct work_struct *); +int afs_wait_for_one_fs_probe(struct afs_server *server, struct afs_endpoint_state *estate, + unsigned long exclude, bool is_intr); +extern void afs_fs_probe_cleanup(struct afs_net *); + +/* + * inode.c + */ +extern const struct afs_operation_ops afs_fetch_status_operation; + +extern void afs_vnode_commit_status(struct afs_operation *, struct afs_vnode_param *); +extern int afs_fetch_status(struct afs_vnode *, struct key *, bool, afs_access_t *); +extern int afs_ilookup5_test_by_fid(struct inode *, void *); +extern struct inode *afs_iget(struct afs_operation *, struct afs_vnode_param *); +extern struct inode *afs_root_iget(struct super_block *, struct key *); +extern int afs_getattr(struct mnt_idmap *idmap, const struct path *, + struct kstat *, u32, unsigned int); +extern int afs_setattr(struct mnt_idmap *idmap, struct dentry *, struct iattr *); +extern void afs_evict_inode(struct inode *); +extern int afs_drop_inode(struct inode *); + +/* + * main.c + */ +extern struct workqueue_struct *afs_wq; +extern int afs_net_id; + +static inline struct afs_net *afs_net(struct net *net) +{ + return net_generic(net, afs_net_id); +} + +static inline struct afs_net *afs_sb2net(struct super_block *sb) +{ + return afs_net(AFS_FS_S(sb)->net_ns); +} + +static inline struct afs_net *afs_d2net(struct dentry *dentry) +{ + return afs_sb2net(dentry->d_sb); +} + +static inline struct afs_net *afs_i2net(struct inode *inode) +{ + return afs_sb2net(inode->i_sb); +} + +static inline struct afs_net *afs_v2net(struct afs_vnode *vnode) +{ + return afs_i2net(&vnode->netfs.inode); +} + +static inline struct afs_net *afs_sock2net(struct sock *sk) +{ + return net_generic(sock_net(sk), afs_net_id); +} + +static inline void __afs_stat(atomic_t *s) +{ + atomic_inc(s); +} + +#define afs_stat_v(vnode, n) __afs_stat(&afs_v2net(vnode)->n) + +/* + * misc.c + */ +extern int afs_abort_to_error(u32); +extern void afs_prioritise_error(struct afs_error *, int, u32); + +static inline void afs_op_nomem(struct afs_operation *op) +{ + op->cumul_error.error = -ENOMEM; +} + +static inline int afs_op_error(const struct afs_operation *op) +{ + return op->cumul_error.error; +} + +static inline s32 afs_op_abort_code(const struct afs_operation *op) +{ + return op->cumul_error.abort_code; +} + +static inline int afs_op_set_error(struct afs_operation *op, int error) +{ + return op->cumul_error.error = error; +} + +static inline void afs_op_accumulate_error(struct afs_operation *op, int error, s32 abort_code) +{ + afs_prioritise_error(&op->cumul_error, error, abort_code); +} + +/* + * mntpt.c + */ +extern const struct inode_operations afs_mntpt_inode_operations; +extern const struct inode_operations afs_autocell_inode_operations; +extern const struct file_operations afs_mntpt_file_operations; + +extern struct vfsmount *afs_d_automount(struct path *); +extern void afs_mntpt_kill_timer(void); + +/* + * proc.c + */ +#ifdef CONFIG_PROC_FS +extern int __net_init afs_proc_init(struct afs_net *); +extern void __net_exit afs_proc_cleanup(struct afs_net *); +extern int afs_proc_cell_setup(struct afs_cell *); +extern void afs_proc_cell_remove(struct afs_cell *); +extern void afs_put_sysnames(struct afs_sysnames *); +#else +static inline int afs_proc_init(struct afs_net *net) { return 0; } +static inline void afs_proc_cleanup(struct afs_net *net) {} +static inline int afs_proc_cell_setup(struct afs_cell *cell) { return 0; } +static inline void afs_proc_cell_remove(struct afs_cell *cell) {} +static inline void afs_put_sysnames(struct afs_sysnames *sysnames) {} +#endif + +/* + * rotate.c + */ +void afs_clear_server_states(struct afs_operation *op); +extern bool afs_select_fileserver(struct afs_operation *); +extern void afs_dump_edestaddrreq(const struct afs_operation *); + +/* + * rxrpc.c + */ +extern struct workqueue_struct *afs_async_calls; + +extern int __net_init afs_open_socket(struct afs_net *); +extern void __net_exit afs_close_socket(struct afs_net *); +extern void afs_charge_preallocation(struct work_struct *); +extern void afs_put_call(struct afs_call *); +void afs_deferred_put_call(struct afs_call *call); +void afs_make_call(struct afs_call *call, gfp_t gfp); +void afs_deliver_to_call(struct afs_call *call); +void afs_wait_for_call_to_complete(struct afs_call *call); +extern struct afs_call *afs_alloc_flat_call(struct afs_net *, + const struct afs_call_type *, + size_t, size_t); +extern void afs_flat_call_destructor(struct afs_call *); +extern void afs_send_empty_reply(struct afs_call *); +extern void afs_send_simple_reply(struct afs_call *, const void *, size_t); +extern int afs_extract_data(struct afs_call *, bool); +extern int afs_protocol_error(struct afs_call *, enum afs_eproto_cause); + +static inline struct afs_call *afs_get_call(struct afs_call *call, + enum afs_call_trace why) +{ + int r; + + __refcount_inc(&call->ref, &r); + + trace_afs_call(call->debug_id, why, r + 1, + atomic_read(&call->net->nr_outstanding_calls), + __builtin_return_address(0)); + return call; +} + +static inline void afs_see_call(struct afs_call *call, enum afs_call_trace why) +{ + int r = refcount_read(&call->ref); + + trace_afs_call(call->debug_id, why, r, + atomic_read(&call->net->nr_outstanding_calls), + __builtin_return_address(0)); +} + +static inline void afs_make_op_call(struct afs_operation *op, struct afs_call *call, + gfp_t gfp) +{ + struct afs_addr_list *alist = op->estate->addresses; + + op->call = afs_get_call(call, afs_call_trace_get); + op->type = call->type; + call->op = op; + call->key = op->key; + call->intr = !(op->flags & AFS_OPERATION_UNINTR); + call->peer = rxrpc_kernel_get_peer(alist->addrs[op->addr_index].peer); + call->service_id = op->server->service_id; + afs_make_call(call, gfp); + afs_put_call(call); +} + +static inline void afs_extract_begin(struct afs_call *call, void *buf, size_t size) +{ + call->iov_len = size; + call->kvec[0].iov_base = buf; + call->kvec[0].iov_len = size; + iov_iter_kvec(&call->def_iter, ITER_DEST, call->kvec, 1, size); +} + +static inline void afs_extract_to_tmp(struct afs_call *call) +{ + call->iov_len = sizeof(call->tmp); + afs_extract_begin(call, &call->tmp, sizeof(call->tmp)); +} + +static inline void afs_extract_to_tmp64(struct afs_call *call) +{ + call->iov_len = sizeof(call->tmp64); + afs_extract_begin(call, &call->tmp64, sizeof(call->tmp64)); +} + +static inline void afs_extract_discard(struct afs_call *call, size_t size) +{ + call->iov_len = size; + iov_iter_discard(&call->def_iter, ITER_DEST, size); +} + +static inline void afs_extract_to_buf(struct afs_call *call, size_t size) +{ + call->iov_len = size; + afs_extract_begin(call, call->buffer, size); +} + +static inline int afs_transfer_reply(struct afs_call *call) +{ + return afs_extract_data(call, false); +} + +static inline bool afs_check_call_state(struct afs_call *call, + enum afs_call_state state) +{ + return READ_ONCE(call->state) == state; +} + +static inline bool afs_set_call_state(struct afs_call *call, + enum afs_call_state from, + enum afs_call_state to) +{ + bool ok = false; + + spin_lock_bh(&call->state_lock); + if (call->state == from) { + call->state = to; + trace_afs_call_state(call, from, to, 0, 0); + ok = true; + } + spin_unlock_bh(&call->state_lock); + return ok; +} + +static inline void afs_set_call_complete(struct afs_call *call, + int error, u32 remote_abort) +{ + enum afs_call_state state; + bool ok = false; + + spin_lock_bh(&call->state_lock); + state = call->state; + if (state != AFS_CALL_COMPLETE) { + call->abort_code = remote_abort; + call->error = error; + call->state = AFS_CALL_COMPLETE; + trace_afs_call_state(call, state, AFS_CALL_COMPLETE, + error, remote_abort); + ok = true; + } + spin_unlock_bh(&call->state_lock); + if (ok) { + trace_afs_call_done(call); + + /* Asynchronous calls have two refs to release - one from the alloc and + * one queued with the work item - and we can't just deallocate the + * call because the work item may be queued again. + */ + if (call->drop_ref) + afs_put_call(call); + } +} + +/* + * security.c + */ +extern void afs_put_permits(struct afs_permits *); +extern void afs_clear_permits(struct afs_vnode *); +extern void afs_cache_permit(struct afs_vnode *, struct key *, unsigned int, + struct afs_status_cb *); +extern struct key *afs_request_key(struct afs_cell *); +extern struct key *afs_request_key_rcu(struct afs_cell *); +extern int afs_check_permit(struct afs_vnode *, struct key *, afs_access_t *); +extern int afs_permission(struct mnt_idmap *, struct inode *, int); +extern void __exit afs_clean_up_permit_cache(void); + +/* + * server.c + */ +extern spinlock_t afs_server_peer_lock; + +struct afs_server *afs_find_server(const struct rxrpc_peer *peer); +extern struct afs_server *afs_lookup_server(struct afs_cell *, struct key *, const uuid_t *, u32); +extern struct afs_server *afs_get_server(struct afs_server *, enum afs_server_trace); +struct afs_server *afs_use_server(struct afs_server *server, bool activate, + enum afs_server_trace reason); +void afs_unuse_server(struct afs_net *net, struct afs_server *server, + enum afs_server_trace reason); +void afs_unuse_server_notime(struct afs_net *net, struct afs_server *server, + enum afs_server_trace reason); +extern void afs_put_server(struct afs_net *, struct afs_server *, enum afs_server_trace); +void afs_purge_servers(struct afs_cell *cell); +extern void afs_fs_probe_timer(struct timer_list *); +void __net_exit afs_wait_for_servers(struct afs_net *net); +bool afs_check_server_record(struct afs_operation *op, struct afs_server *server, struct key *key); + +static inline void afs_see_server(struct afs_server *server, enum afs_server_trace trace) +{ + int r = refcount_read(&server->ref); + int a = atomic_read(&server->active); + + trace_afs_server(server->debug_id, r, a, trace); + +} + +static inline void afs_inc_servers_outstanding(struct afs_net *net) +{ + atomic_inc(&net->servers_outstanding); +} + +static inline void afs_dec_servers_outstanding(struct afs_net *net) +{ + if (atomic_dec_and_test(&net->servers_outstanding)) + wake_up_var(&net->servers_outstanding); +} + +static inline bool afs_is_probing_server(struct afs_server *server) +{ + return list_empty(&server->probe_link); +} + +/* + * server_list.c + */ +static inline struct afs_server_list *afs_get_serverlist(struct afs_server_list *slist) +{ + refcount_inc(&slist->usage); + return slist; +} + +extern void afs_put_serverlist(struct afs_net *, struct afs_server_list *); +struct afs_server_list *afs_alloc_server_list(struct afs_volume *volume, + struct key *key, + struct afs_vldb_entry *vldb); +extern bool afs_annotate_server_list(struct afs_server_list *, struct afs_server_list *); +void afs_attach_volume_to_servers(struct afs_volume *volume, struct afs_server_list *slist); +void afs_reattach_volume_to_servers(struct afs_volume *volume, struct afs_server_list *slist, + struct afs_server_list *old); +void afs_detach_volume_from_servers(struct afs_volume *volume, struct afs_server_list *slist); + +/* + * super.c + */ +extern int __init afs_fs_init(void); +extern void afs_fs_exit(void); + +/* + * symlink.c + */ +extern const struct inode_operations afs_symlink_inode_operations; +extern const struct address_space_operations afs_symlink_aops; + +void afs_invalidate_symlink(struct afs_vnode *vnode); +void afs_evict_symlink(struct afs_vnode *vnode); +void afs_init_new_symlink(struct afs_vnode *vnode, struct afs_operation *op); +const char *afs_get_link(struct dentry *dentry, struct inode *inode, + struct delayed_call *callback); +int afs_readlink(struct dentry *dentry, char __user *buffer, int buflen); +int afs_symlink_writepages(struct address_space *mapping, + struct writeback_control *wbc); + +/* + * validation.c + */ +bool afs_check_validity(const struct afs_vnode *vnode); +int afs_update_volume_state(struct afs_operation *op); +int afs_validate(struct afs_vnode *vnode, struct key *key); + +/* + * vlclient.c + */ +extern struct afs_vldb_entry *afs_vl_get_entry_by_name_u(struct afs_vl_cursor *, + const char *, int); +extern struct afs_addr_list *afs_vl_get_addrs_u(struct afs_vl_cursor *, const uuid_t *); +struct afs_call *afs_vl_get_capabilities(struct afs_net *net, + struct afs_addr_list *alist, + unsigned int addr_index, + struct key *key, + struct afs_vlserver *server, + unsigned int server_index); +extern struct afs_addr_list *afs_yfsvl_get_endpoints(struct afs_vl_cursor *, const uuid_t *); +extern char *afs_yfsvl_get_cell_name(struct afs_vl_cursor *); + +/* + * vl_alias.c + */ +extern int afs_cell_detect_alias(struct afs_cell *, struct key *); + +/* + * vl_probe.c + */ +extern void afs_vlserver_probe_result(struct afs_call *); +extern int afs_send_vl_probes(struct afs_net *, struct key *, struct afs_vlserver_list *); +extern int afs_wait_for_vl_probes(struct afs_vlserver_list *, unsigned long); + +/* + * vl_rotate.c + */ +extern bool afs_begin_vlserver_operation(struct afs_vl_cursor *, + struct afs_cell *, struct key *); +extern bool afs_select_vlserver(struct afs_vl_cursor *); +extern bool afs_select_current_vlserver(struct afs_vl_cursor *); +extern int afs_end_vlserver_operation(struct afs_vl_cursor *); + +/* + * vlserver_list.c + */ +static inline struct afs_vlserver *afs_get_vlserver(struct afs_vlserver *vlserver) +{ + refcount_inc(&vlserver->ref); + return vlserver; +} + +static inline struct afs_vlserver_list *afs_get_vlserverlist(struct afs_vlserver_list *vllist) +{ + if (vllist) + refcount_inc(&vllist->ref); + return vllist; +} + +extern struct afs_vlserver *afs_alloc_vlserver(const char *, size_t, unsigned short); +extern void afs_put_vlserver(struct afs_net *, struct afs_vlserver *); +extern struct afs_vlserver_list *afs_alloc_vlserver_list(unsigned int); +extern void afs_put_vlserverlist(struct afs_net *, struct afs_vlserver_list *); +extern struct afs_vlserver_list *afs_extract_vlserver_list(struct afs_cell *, + const void *, size_t); + +/* + * volume.c + */ +extern struct afs_volume *afs_create_volume(struct afs_fs_context *); +extern int afs_activate_volume(struct afs_volume *); +extern void afs_deactivate_volume(struct afs_volume *); +bool afs_try_get_volume(struct afs_volume *volume, enum afs_volume_trace reason); +extern struct afs_volume *afs_get_volume(struct afs_volume *, enum afs_volume_trace); +void afs_put_volume(struct afs_volume *volume, enum afs_volume_trace reason); +extern int afs_check_volume_status(struct afs_volume *, struct afs_operation *); + +/* + * write.c + */ +void afs_prepare_write(struct netfs_io_subrequest *subreq); +void afs_issue_write(struct netfs_io_subrequest *subreq); +void afs_begin_writeback(struct netfs_io_request *wreq); +void afs_retry_request(struct netfs_io_request *wreq, struct netfs_io_stream *stream); +extern int afs_writepages(struct address_space *, struct writeback_control *); +extern int afs_fsync(struct file *, loff_t, loff_t, int); +extern vm_fault_t afs_page_mkwrite(struct vm_fault *vmf); +extern void afs_prune_wb_keys(struct afs_vnode *); + +/* + * xattr.c + */ +extern const struct xattr_handler * const afs_xattr_handlers[]; + +/* + * yfsclient.c + */ +extern void yfs_fs_fetch_data(struct afs_operation *); +extern void yfs_fs_create_file(struct afs_operation *); +extern void yfs_fs_make_dir(struct afs_operation *); +extern void yfs_fs_remove_file2(struct afs_operation *); +extern void yfs_fs_remove_file(struct afs_operation *); +extern void yfs_fs_remove_dir(struct afs_operation *); +extern void yfs_fs_link(struct afs_operation *); +extern void yfs_fs_symlink(struct afs_operation *); +extern void yfs_fs_rename(struct afs_operation *); +void yfs_fs_rename_replace(struct afs_operation *op); +void yfs_fs_rename_noreplace(struct afs_operation *op); +void yfs_fs_rename_exchange(struct afs_operation *op); +extern void yfs_fs_store_data(struct afs_operation *); +extern void yfs_fs_setattr(struct afs_operation *); +extern void yfs_fs_get_volume_status(struct afs_operation *); +extern void yfs_fs_set_lock(struct afs_operation *); +extern void yfs_fs_extend_lock(struct afs_operation *); +extern void yfs_fs_release_lock(struct afs_operation *); +extern void yfs_fs_fetch_status(struct afs_operation *); +extern void yfs_fs_inline_bulk_status(struct afs_operation *); + +struct yfs_acl { + struct afs_acl *acl; /* Dir/file/symlink ACL */ + struct afs_acl *vol_acl; /* Whole volume ACL */ + u32 inherit_flag; /* True if ACL is inherited from parent dir */ + u32 num_cleaned; /* Number of ACEs removed due to subject removal */ + unsigned int flags; +#define YFS_ACL_WANT_ACL 0x01 /* Set if caller wants ->acl */ +#define YFS_ACL_WANT_VOL_ACL 0x02 /* Set if caller wants ->vol_acl */ +}; + +extern void yfs_free_opaque_acl(struct yfs_acl *); +extern void yfs_fs_fetch_opaque_acl(struct afs_operation *); +extern void yfs_fs_store_opaque_acl2(struct afs_operation *); + +/* + * Miscellaneous inline functions. + */ +static inline struct afs_vnode *AFS_FS_I(struct inode *inode) +{ + return container_of(inode, struct afs_vnode, netfs.inode); +} + +static inline struct inode *AFS_VNODE_TO_I(struct afs_vnode *vnode) +{ + return &vnode->netfs.inode; +} + +/* + * Note that a dentry got changed. We need to set d_fsdata to the data version + * number derived from the result of the operation. It doesn't matter if + * d_fsdata goes backwards as we'll just revalidate. + */ +static inline void afs_update_dentry_version(struct afs_operation *op, + struct afs_vnode_param *dir_vp, + struct dentry *dentry) +{ + if (!op->cumul_error.error) + dentry->d_fsdata = + (void *)(unsigned long)dir_vp->scb.status.data_version; +} + +/* + * Check for a conflicting operation on a directory that we just unlinked from. + * If someone managed to sneak a link or an unlink in on the file we just + * unlinked, we won't be able to trust nlink on an AFS file (but not YFS). + */ +static inline void afs_check_dir_conflict(struct afs_operation *op, + struct afs_vnode_param *dvp) +{ + if (dvp->dv_before + dvp->dv_delta != dvp->scb.status.data_version) + op->flags |= AFS_OPERATION_DIR_CONFLICT; +} + +static inline int afs_io_error(struct afs_call *call, enum afs_io_error where) +{ + trace_afs_io_error(call->debug_id, -EIO, where); + return -EIO; +} + +static inline int afs_bad(struct afs_vnode *vnode, enum afs_file_error where) +{ + trace_afs_file_error(vnode, -EIO, where); + return -EIO; +} + +/* + * Set the callback promise on a vnode. + */ +static inline void afs_set_cb_promise(struct afs_vnode *vnode, time64_t expires_at, + enum afs_cb_promise_trace trace) +{ + atomic64_set(&vnode->cb_expires_at, expires_at); + trace_afs_cb_promise(vnode, trace); +} + +/* + * Clear the callback promise on a vnode, returning true if it was promised. + */ +static inline bool afs_clear_cb_promise(struct afs_vnode *vnode, + enum afs_cb_promise_trace trace) +{ + trace_afs_cb_promise(vnode, trace); + return atomic64_xchg(&vnode->cb_expires_at, AFS_NO_CB_PROMISE) != AFS_NO_CB_PROMISE; +} + +/* + * Mark a directory as being invalid. + */ +static inline void afs_invalidate_dir(struct afs_vnode *dvnode, + enum afs_dir_invalid_trace trace) +{ + if (test_and_clear_bit(AFS_VNODE_DIR_VALID, &dvnode->flags)) { + trace_afs_dir_invalid(dvnode, trace); + afs_stat_v(dvnode, n_inval); + } +} + +/*****************************************************************************/ +/* + * debug tracing + */ +extern unsigned afs_debug; + +#define dbgprintk(FMT,...) \ + printk("[%-6.6s] "FMT"\n", current->comm ,##__VA_ARGS__) + +#define kenter(FMT,...) dbgprintk("==> %s("FMT")",__func__ ,##__VA_ARGS__) +#define kleave(FMT,...) dbgprintk("<== %s()"FMT"",__func__ ,##__VA_ARGS__) +#define kdebug(FMT,...) dbgprintk(" "FMT ,##__VA_ARGS__) + + +#if defined(__KDEBUG) +#define _enter(FMT,...) kenter(FMT,##__VA_ARGS__) +#define _leave(FMT,...) kleave(FMT,##__VA_ARGS__) +#define _debug(FMT,...) kdebug(FMT,##__VA_ARGS__) + +#elif defined(CONFIG_AFS_DEBUG) +#define AFS_DEBUG_KENTER 0x01 +#define AFS_DEBUG_KLEAVE 0x02 +#define AFS_DEBUG_KDEBUG 0x04 + +#define _enter(FMT,...) \ +do { \ + if (unlikely(afs_debug & AFS_DEBUG_KENTER)) \ + kenter(FMT,##__VA_ARGS__); \ +} while (0) + +#define _leave(FMT,...) \ +do { \ + if (unlikely(afs_debug & AFS_DEBUG_KLEAVE)) \ + kleave(FMT,##__VA_ARGS__); \ +} while (0) + +#define _debug(FMT,...) \ +do { \ + if (unlikely(afs_debug & AFS_DEBUG_KDEBUG)) \ + kdebug(FMT,##__VA_ARGS__); \ +} while (0) + +#else +#define _enter(FMT,...) no_printk("==> %s("FMT")",__func__ ,##__VA_ARGS__) +#define _leave(FMT,...) no_printk("<== %s()"FMT"",__func__ ,##__VA_ARGS__) +#define _debug(FMT,...) no_printk(" "FMT ,##__VA_ARGS__) +#endif + +/* + * debug assertion checking + */ +#if 1 // defined(__KDEBUGALL) + +#define ASSERT(X) \ +do { \ + if (unlikely(!(X))) { \ + printk(KERN_ERR "\n"); \ + printk(KERN_ERR "AFS: Assertion failed\n"); \ + BUG(); \ + } \ +} while(0) + +#define ASSERTCMP(X, OP, Y) \ +do { \ + if (unlikely(!((X) OP (Y)))) { \ + printk(KERN_ERR "\n"); \ + printk(KERN_ERR "AFS: Assertion failed\n"); \ + printk(KERN_ERR "%lu " #OP " %lu is false\n", \ + (unsigned long)(X), (unsigned long)(Y)); \ + printk(KERN_ERR "0x%lx " #OP " 0x%lx is false\n", \ + (unsigned long)(X), (unsigned long)(Y)); \ + BUG(); \ + } \ +} while(0) + +#define ASSERTRANGE(L, OP1, N, OP2, H) \ +do { \ + if (unlikely(!((L) OP1 (N)) || !((N) OP2 (H)))) { \ + printk(KERN_ERR "\n"); \ + printk(KERN_ERR "AFS: Assertion failed\n"); \ + printk(KERN_ERR "%lu "#OP1" %lu "#OP2" %lu is false\n", \ + (unsigned long)(L), (unsigned long)(N), \ + (unsigned long)(H)); \ + printk(KERN_ERR "0x%lx "#OP1" 0x%lx "#OP2" 0x%lx is false\n", \ + (unsigned long)(L), (unsigned long)(N), \ + (unsigned long)(H)); \ + BUG(); \ + } \ +} while(0) + +#define ASSERTIF(C, X) \ +do { \ + if (unlikely((C) && !(X))) { \ + printk(KERN_ERR "\n"); \ + printk(KERN_ERR "AFS: Assertion failed\n"); \ + BUG(); \ + } \ +} while(0) + +#define ASSERTIFCMP(C, X, OP, Y) \ +do { \ + if (unlikely((C) && !((X) OP (Y)))) { \ + printk(KERN_ERR "\n"); \ + printk(KERN_ERR "AFS: Assertion failed\n"); \ + printk(KERN_ERR "%lu " #OP " %lu is false\n", \ + (unsigned long)(X), (unsigned long)(Y)); \ + printk(KERN_ERR "0x%lx " #OP " 0x%lx is false\n", \ + (unsigned long)(X), (unsigned long)(Y)); \ + BUG(); \ + } \ +} while(0) + +#else + +#define ASSERT(X) \ +do { \ +} while(0) + +#define ASSERTCMP(X, OP, Y) \ +do { \ +} while(0) + +#define ASSERTRANGE(L, OP1, N, OP2, H) \ +do { \ +} while(0) + +#define ASSERTIF(C, X) \ +do { \ +} while(0) + +#define ASSERTIFCMP(C, X, OP, Y) \ +do { \ +} while(0) + +#endif /* __KDEBUGALL */ diff --git a/fs/afs/main.c b/fs/afs/main.c new file mode 100644 index 000000000..7a883c599 --- /dev/null +++ b/fs/afs/main.c @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* AFS client file system + * + * Copyright (C) 2002,5 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include +#define CREATE_TRACE_POINTS +#include "internal.h" + +MODULE_DESCRIPTION("AFS Client File System"); +MODULE_AUTHOR("Red Hat, Inc."); +MODULE_LICENSE("GPL"); + +unsigned afs_debug; +module_param_named(debug, afs_debug, uint, S_IWUSR | S_IRUGO); +MODULE_PARM_DESC(debug, "AFS debugging mask"); + +static char *rootcell; + +module_param(rootcell, charp, 0); +MODULE_PARM_DESC(rootcell, "root AFS cell name and VL server IP addr list"); + +struct workqueue_struct *afs_wq; +static struct proc_dir_entry *afs_proc_symlink; + +#if defined(CONFIG_ALPHA) +const char afs_init_sysname[] = "alpha_linux26"; +#elif defined(CONFIG_X86_64) +const char afs_init_sysname[] = "amd64_linux26"; +#elif defined(CONFIG_ARM) +const char afs_init_sysname[] = "arm_linux26"; +#elif defined(CONFIG_ARM64) +const char afs_init_sysname[] = "aarch64_linux26"; +#elif defined(CONFIG_X86_32) +const char afs_init_sysname[] = "i386_linux26"; +#elif defined(CONFIG_PPC64) +const char afs_init_sysname[] = "ppc64_linux26"; +#elif defined(CONFIG_PPC32) +const char afs_init_sysname[] = "ppc_linux26"; +#elif defined(CONFIG_S390) +#ifdef CONFIG_64BIT +const char afs_init_sysname[] = "s390x_linux26"; +#else +const char afs_init_sysname[] = "s390_linux26"; +#endif +#elif defined(CONFIG_SPARC64) +const char afs_init_sysname[] = "sparc64_linux26"; +#elif defined(CONFIG_SPARC32) +const char afs_init_sysname[] = "sparc_linux26"; +#else +const char afs_init_sysname[] = "unknown_linux26"; +#endif + +/* + * Initialise an AFS network namespace record. + */ +static int __net_init afs_net_init(struct net *net_ns) +{ + struct afs_sysnames *sysnames; + struct afs_net *net = afs_net(net_ns); + int ret; + + net->net = net_ns; + net->live = true; + generate_random_uuid((unsigned char *)&net->uuid); + + INIT_WORK(&net->charge_preallocation_work, afs_charge_preallocation); + INIT_WORK(&net->rx_oob_work, afs_process_oob_queue); + mutex_init(&net->socket_mutex); + + net->cells = RB_ROOT; + idr_init(&net->cells_dyn_ino); + init_rwsem(&net->cells_lock); + mutex_init(&net->cells_alias_lock); + mutex_init(&net->proc_cells_lock); + INIT_HLIST_HEAD(&net->proc_cells); + + seqlock_init(&net->fs_lock); + INIT_LIST_HEAD(&net->fs_probe_fast); + INIT_LIST_HEAD(&net->fs_probe_slow); + INIT_HLIST_HEAD(&net->fs_proc); + + INIT_WORK(&net->fs_prober, afs_fs_probe_dispatcher); + timer_setup(&net->fs_probe_timer, afs_fs_probe_timer, 0); + atomic_set(&net->servers_outstanding, 1); + + ret = -ENOMEM; + sysnames = kzalloc_obj(*sysnames); + if (!sysnames) + goto error_sysnames; + sysnames->subs[0] = (char *)&afs_init_sysname; + sysnames->nr = 1; + refcount_set(&sysnames->usage, 1); + net->sysnames = sysnames; + rwlock_init(&net->sysnames_lock); + + /* Register the /proc stuff */ + ret = afs_proc_init(net); + if (ret < 0) + goto error_proc; + + /* Initialise the cell DB */ + ret = afs_cell_init(net, rootcell); + if (ret < 0) + goto error_cell_init; + + /* Create the RxRPC transport */ + ret = afs_open_socket(net); + if (ret < 0) + goto error_open_socket; + + return 0; + +error_open_socket: + net->live = false; + afs_fs_probe_cleanup(net); + afs_cell_purge(net); + afs_wait_for_servers(net); +error_cell_init: + net->live = false; + afs_proc_cleanup(net); +error_proc: + afs_put_sysnames(net->sysnames); +error_sysnames: + idr_destroy(&net->cells_dyn_ino); + net->live = false; + return ret; +} + +/* + * Clean up and destroy an AFS network namespace record. + */ +static void __net_exit afs_net_exit(struct net *net_ns) +{ + struct afs_net *net = afs_net(net_ns); + + net->live = false; + afs_fs_probe_cleanup(net); + afs_cell_purge(net); + afs_wait_for_servers(net); + afs_close_socket(net); + afs_proc_cleanup(net); + afs_put_sysnames(net->sysnames); + idr_destroy(&net->cells_dyn_ino); + kfree_rcu(rcu_access_pointer(net->address_prefs), rcu); +} + +static struct pernet_operations afs_net_ops = { + .init = afs_net_init, + .exit = afs_net_exit, + .id = &afs_net_id, + .size = sizeof(struct afs_net), +}; + +/* + * initialise the AFS client FS module + */ +static int __init afs_init(void) +{ + int ret = -ENOMEM; + + printk(KERN_INFO "kAFS: Red Hat AFS client v0.1 registering.\n"); + + afs_wq = alloc_workqueue("afs", WQ_PERCPU, 0); + if (!afs_wq) + goto error_afs_wq; + afs_async_calls = alloc_workqueue("kafsd", WQ_MEM_RECLAIM | WQ_UNBOUND, 0); + if (!afs_async_calls) + goto error_async; + afs_lock_manager = alloc_workqueue("kafs_lockd", WQ_MEM_RECLAIM | WQ_PERCPU, 0); + if (!afs_lock_manager) + goto error_lockmgr; + + ret = register_pernet_device(&afs_net_ops); + if (ret < 0) + goto error_net; + + /* register the filesystems */ + ret = afs_fs_init(); + if (ret < 0) + goto error_fs; + + afs_proc_symlink = proc_symlink("fs/afs", NULL, "../self/net/afs"); + if (!afs_proc_symlink) { + ret = -ENOMEM; + goto error_proc; + } + + return ret; + +error_proc: + afs_fs_exit(); +error_fs: + unregister_pernet_device(&afs_net_ops); +error_net: + destroy_workqueue(afs_lock_manager); +error_lockmgr: + destroy_workqueue(afs_async_calls); +error_async: + destroy_workqueue(afs_wq); +error_afs_wq: + rcu_barrier(); + printk(KERN_ERR "kAFS: failed to register: %d\n", ret); + return ret; +} + +/* XXX late_initcall is kludgy, but the only alternative seems to create + * a transport upon the first mount, which is worse. Or is it? + */ +late_initcall(afs_init); /* must be called after net/ to create socket */ + +/* + * clean up on module removal + */ +static void __exit afs_exit(void) +{ + printk(KERN_INFO "kAFS: Red Hat AFS client v0.1 unregistering.\n"); + + proc_remove(afs_proc_symlink); + afs_fs_exit(); + unregister_pernet_device(&afs_net_ops); + destroy_workqueue(afs_lock_manager); + destroy_workqueue(afs_async_calls); + destroy_workqueue(afs_wq); + afs_clean_up_permit_cache(); + rcu_barrier(); +} + +module_exit(afs_exit); diff --git a/fs/afs/misc.c b/fs/afs/misc.c new file mode 100644 index 000000000..c8a7f2660 --- /dev/null +++ b/fs/afs/misc.c @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* miscellaneous bits + * + * Copyright (C) 2002, 2007 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include "internal.h" +#include "afs_fs.h" +#include "protocol_uae.h" + +/* + * convert an AFS abort code to a Linux error number + */ +int afs_abort_to_error(u32 abort_code) +{ + switch (abort_code) { + /* Low errno codes inserted into abort namespace */ + case 13: return -EACCES; + case 27: return -EFBIG; + case 30: return -EROFS; + + /* VICE "special error" codes; 101 - 111 */ + case VSALVAGE: return -EIO; + case VNOVNODE: return -ENOENT; + case VNOVOL: return -ENOMEDIUM; + case VVOLEXISTS: return -EEXIST; + case VNOSERVICE: return -EIO; + case VOFFLINE: return -ENOENT; + case VONLINE: return -EEXIST; + case VDISKFULL: return -ENOSPC; + case VOVERQUOTA: return -EDQUOT; + case VBUSY: return -EBUSY; + case VMOVED: return -ENXIO; + + /* Volume Location server errors */ + case AFSVL_IDEXIST: return -EEXIST; + case AFSVL_IO: return -EREMOTEIO; + case AFSVL_NAMEEXIST: return -EEXIST; + case AFSVL_CREATEFAIL: return -EREMOTEIO; + case AFSVL_NOENT: return -ENOMEDIUM; + case AFSVL_EMPTY: return -ENOMEDIUM; + case AFSVL_ENTDELETED: return -ENOMEDIUM; + case AFSVL_BADNAME: return -EINVAL; + case AFSVL_BADINDEX: return -EINVAL; + case AFSVL_BADVOLTYPE: return -EINVAL; + case AFSVL_BADSERVER: return -EINVAL; + case AFSVL_BADPARTITION: return -EINVAL; + case AFSVL_REPSFULL: return -EFBIG; + case AFSVL_NOREPSERVER: return -ENOENT; + case AFSVL_DUPREPSERVER: return -EEXIST; + case AFSVL_RWNOTFOUND: return -ENOENT; + case AFSVL_BADREFCOUNT: return -EINVAL; + case AFSVL_SIZEEXCEEDED: return -EINVAL; + case AFSVL_BADENTRY: return -EINVAL; + case AFSVL_BADVOLIDBUMP: return -EINVAL; + case AFSVL_IDALREADYHASHED: return -EINVAL; + case AFSVL_ENTRYLOCKED: return -EBUSY; + case AFSVL_BADVOLOPER: return -EBADRQC; + case AFSVL_BADRELLOCKTYPE: return -EINVAL; + case AFSVL_RERELEASE: return -EREMOTEIO; + case AFSVL_BADSERVERFLAG: return -EINVAL; + case AFSVL_PERM: return -EACCES; + case AFSVL_NOMEM: return -EREMOTEIO; + + /* Unified AFS error table */ + case UAEPERM: return -EPERM; + case UAENOENT: return -ENOENT; + case UAEAGAIN: return -EAGAIN; + case UAEACCES: return -EACCES; + case UAEBUSY: return -EBUSY; + case UAEEXIST: return -EEXIST; + case UAENOTDIR: return -ENOTDIR; + case UAEISDIR: return -EISDIR; + case UAEFBIG: return -EFBIG; + case UAENOSPC: return -ENOSPC; + case UAEROFS: return -EROFS; + case UAEMLINK: return -EMLINK; + case UAEDEADLK: return -EDEADLK; + case UAENAMETOOLONG: return -ENAMETOOLONG; + case UAENOLCK: return -ENOLCK; + case UAENOTEMPTY: return -ENOTEMPTY; + case UAELOOP: return -ELOOP; + case UAEOVERFLOW: return -EOVERFLOW; + case UAENOMEDIUM: return -ENOMEDIUM; + case UAEDQUOT: return -EDQUOT; + + /* RXKAD abort codes; from include/rxrpc/packet.h. ET "RXK" == 0x1260B00 */ + case RXKADINCONSISTENCY: return -EPROTO; + case RXKADPACKETSHORT: return -EPROTO; + case RXKADLEVELFAIL: return -EKEYREJECTED; + case RXKADTICKETLEN: return -EKEYREJECTED; + case RXKADOUTOFSEQUENCE: return -EPROTO; + case RXKADNOAUTH: return -EKEYREJECTED; + case RXKADBADKEY: return -EKEYREJECTED; + case RXKADBADTICKET: return -EKEYREJECTED; + case RXKADUNKNOWNKEY: return -EKEYREJECTED; + case RXKADEXPIRED: return -EKEYEXPIRED; + case RXKADSEALEDINCON: return -EKEYREJECTED; + case RXKADDATALEN: return -EKEYREJECTED; + case RXKADILLEGALLEVEL: return -EKEYREJECTED; + + case RXGK_INCONSISTENCY: return -EPROTO; + case RXGK_PACKETSHORT: return -EPROTO; + case RXGK_BADCHALLENGE: return -EPROTO; + case RXGK_SEALEDINCON: return -EKEYREJECTED; + case RXGK_NOTAUTH: return -EKEYREJECTED; + case RXGK_EXPIRED: return -EKEYEXPIRED; + case RXGK_BADLEVEL: return -EKEYREJECTED; + case RXGK_BADKEYNO: return -EKEYREJECTED; + case RXGK_NOTRXGK: return -EKEYREJECTED; + case RXGK_UNSUPPORTED: return -EKEYREJECTED; + case RXGK_GSSERROR: return -EKEYREJECTED; +#ifdef RXGK_BADETYPE + case RXGK_BADETYPE: return -ENOPKG; +#endif +#ifdef RXGK_BADTOKEN + case RXGK_BADTOKEN: return -EKEYREJECTED; +#endif +#ifdef RXGK_BADETYPE + case RXGK_DATALEN: return -EPROTO; +#endif +#ifdef RXGK_BADQOP + case RXGK_BADQOP: return -EKEYREJECTED; +#endif + + case KRB5_PROG_KEYTYPE_NOSUPP: return -ENOPKG; + + case RXGEN_OPCODE: return -ENOTSUPP; + case RX_INVALID_OPERATION: return -ENOTSUPP; + + default: return -EREMOTEIO; + } +} + +/* + * Select the error to report from a set of errors. + */ +void afs_prioritise_error(struct afs_error *e, int error, u32 abort_code) +{ + switch (error) { + case 0: + e->aborted = false; + e->error = 0; + return; + default: + if (e->error == -ETIMEDOUT || + e->error == -ETIME) + return; + fallthrough; + case -ETIMEDOUT: + case -ETIME: + if (e->error == -ENOMEM || + e->error == -ENONET) + return; + fallthrough; + case -ENOMEM: + case -ENONET: + if (e->error == -ERFKILL) + return; + fallthrough; + case -ERFKILL: + if (e->error == -EADDRNOTAVAIL) + return; + fallthrough; + case -EADDRNOTAVAIL: + if (e->error == -ENETUNREACH) + return; + fallthrough; + case -ENETUNREACH: + if (e->error == -EHOSTUNREACH) + return; + fallthrough; + case -EHOSTUNREACH: + if (e->error == -EHOSTDOWN) + return; + fallthrough; + case -EHOSTDOWN: + if (e->error == -ECONNREFUSED) + return; + fallthrough; + case -ECONNREFUSED: + if (e->error == -ECONNRESET) + return; + fallthrough; + case -ECONNRESET: /* Responded, but call expired. */ + if (e->responded) + return; + e->error = error; + e->aborted = false; + return; + + case -ECONNABORTED: + e->error = afs_abort_to_error(abort_code); + e->aborted = true; + e->responded = true; + return; + case -ENETRESET: /* Responded, but we seem to have changed address */ + e->aborted = false; + e->responded = true; + e->error = error; + return; + } +} diff --git a/fs/afs/mntpt.c b/fs/afs/mntpt.c new file mode 100644 index 000000000..57c204a3c --- /dev/null +++ b/fs/afs/mntpt.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* mountpoint management + * + * Copyright (C) 2002 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + + +static struct dentry *afs_mntpt_lookup(struct inode *dir, + struct dentry *dentry, + unsigned int flags); +static int afs_mntpt_open(struct inode *inode, struct file *file); +static void afs_mntpt_expiry_timed_out(struct work_struct *work); + +const struct file_operations afs_mntpt_file_operations = { + .open = afs_mntpt_open, + .llseek = noop_llseek, +}; + +const struct inode_operations afs_mntpt_inode_operations = { + .lookup = afs_mntpt_lookup, + .readlink = afs_readlink, + .getattr = afs_getattr, +}; + +const struct inode_operations afs_autocell_inode_operations = { + .getattr = afs_getattr, +}; + +static LIST_HEAD(afs_vfsmounts); +static DECLARE_DELAYED_WORK(afs_mntpt_expiry_timer, afs_mntpt_expiry_timed_out); + +static unsigned long afs_mntpt_expiry_timeout = 10 * 60; + +static const char afs_root_volume[] = "root.cell"; + +/* + * no valid lookup procedure on this sort of dir + */ +static struct dentry *afs_mntpt_lookup(struct inode *dir, + struct dentry *dentry, + unsigned int flags) +{ + _enter("%p,%p{%pd2}", dir, dentry, dentry); + return ERR_PTR(-EREMOTE); +} + +/* + * no valid open procedure on this sort of dir + */ +static int afs_mntpt_open(struct inode *inode, struct file *file) +{ + _enter("%p,%p{%pD2}", inode, file, file); + return -EREMOTE; +} + +/* + * Set the parameters for the proposed superblock. + */ +static int afs_mntpt_set_params(struct fs_context *fc, struct dentry *mntpt) +{ + struct afs_fs_context *ctx = fc->fs_private; + struct afs_super_info *src_as = AFS_FS_S(mntpt->d_sb); + struct afs_vnode *vnode = AFS_FS_I(d_inode(mntpt)); + struct afs_cell *cell; + const char *p; + int ret; + + if (fc->net_ns != src_as->net_ns) { + put_net(fc->net_ns); + fc->net_ns = get_net(src_as->net_ns); + } + + if (src_as->volume && src_as->volume->type == AFSVL_RWVOL) { + ctx->type = AFSVL_RWVOL; + ctx->force = true; + } + if (ctx->cell) { + afs_unuse_cell(ctx->cell, afs_cell_trace_unuse_mntpt); + ctx->cell = NULL; + } + if (test_bit(AFS_VNODE_PSEUDODIR, &vnode->flags)) { + /* if the directory is a pseudo directory, use the d_name */ + unsigned size = mntpt->d_name.len; + + if (size < 2) + return -ENOENT; + + p = mntpt->d_name.name; + if (mntpt->d_name.name[0] == '.') { + size--; + p++; + ctx->type = AFSVL_RWVOL; + ctx->force = true; + } + if (size > AFS_MAXCELLNAME) + return -ENAMETOOLONG; + + cell = afs_lookup_cell(ctx->net, p, size, NULL, + AFS_LOOKUP_CELL_MOUNTPOINT, + afs_cell_trace_use_lookup_mntpt); + if (IS_ERR(cell)) { + pr_err("kAFS: unable to lookup cell '%pd'\n", mntpt); + return PTR_ERR(cell); + } + ctx->cell = cell; + + ctx->volname = afs_root_volume; + ctx->volnamesz = sizeof(afs_root_volume) - 1; + } else { + /* read the contents of the AFS special symlink */ + DEFINE_DELAYED_CALL(cleanup); + const char *content; + loff_t size = i_size_read(d_inode(mntpt)); + + if (src_as->cell) + ctx->cell = afs_use_cell(src_as->cell, afs_cell_trace_use_mntpt); + + if (size < 2 || size > PAGE_SIZE - 1) + return -EINVAL; + + content = afs_get_link(mntpt, d_inode(mntpt), &cleanup); + if (IS_ERR(content)) { + do_delayed_call(&cleanup); + return PTR_ERR(content); + } + + ret = -EINVAL; + if (content[size - 1] == '.') + ret = vfs_parse_fs_qstr(fc, "source", + &QSTR_LEN(content, size - 1)); + do_delayed_call(&cleanup); + if (ret < 0) + return ret; + + /* Don't cross a backup volume mountpoint from a backup volume */ + if (src_as->volume && src_as->volume->type == AFSVL_BACKVOL && + ctx->type == AFSVL_BACKVOL) + return -ENODEV; + } + + return 0; +} + +/* + * create a vfsmount to be automounted + */ +static struct vfsmount *afs_mntpt_do_automount(struct dentry *mntpt) +{ + struct fs_context *fc; + struct vfsmount *mnt; + int ret; + + BUG_ON(!d_inode(mntpt)); + + fc = fs_context_for_submount(&afs_fs_type, mntpt); + if (IS_ERR(fc)) + return ERR_CAST(fc); + + ret = afs_mntpt_set_params(fc, mntpt); + if (!ret) + mnt = fc_mount(fc); + else + mnt = ERR_PTR(ret); + + put_fs_context(fc); + return mnt; +} + +/* + * handle an automount point + */ +struct vfsmount *afs_d_automount(struct path *path) +{ + struct vfsmount *newmnt; + + _enter("{%pd}", path->dentry); + + newmnt = afs_mntpt_do_automount(path->dentry); + if (IS_ERR(newmnt)) + return newmnt; + + mnt_set_expiry(newmnt, &afs_vfsmounts); + queue_delayed_work(afs_wq, &afs_mntpt_expiry_timer, + afs_mntpt_expiry_timeout * HZ); + _leave(" = %p", newmnt); + return newmnt; +} + +/* + * handle mountpoint expiry timer going off + */ +static void afs_mntpt_expiry_timed_out(struct work_struct *work) +{ + _enter(""); + + if (!list_empty(&afs_vfsmounts)) { + mark_mounts_for_expiry(&afs_vfsmounts); + queue_delayed_work(afs_wq, &afs_mntpt_expiry_timer, + afs_mntpt_expiry_timeout * HZ); + } + + _leave(""); +} + +/* + * kill the AFS mountpoint timer if it's still running + */ +void afs_mntpt_kill_timer(void) +{ + _enter(""); + + ASSERT(list_empty(&afs_vfsmounts)); + cancel_delayed_work_sync(&afs_mntpt_expiry_timer); +} diff --git a/fs/afs/proc.c b/fs/afs/proc.c new file mode 100644 index 000000000..a9cc5cdb4 --- /dev/null +++ b/fs/afs/proc.c @@ -0,0 +1,786 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* /proc interface for AFS + * + * Copyright (C) 2002 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include "internal.h" + +struct afs_vl_seq_net_private { + struct seq_net_private seq; /* Must be first */ + struct afs_vlserver_list *vllist; +}; + +static inline struct afs_net *afs_seq2net(struct seq_file *m) +{ + return afs_net(seq_file_net(m)); +} + +static inline struct afs_net *afs_seq2net_single(struct seq_file *m) +{ + return afs_net(seq_file_single_net(m)); +} + +/* + * Display the list of cells known to the namespace. + */ +static int afs_proc_cells_show(struct seq_file *m, void *v) +{ + struct afs_vlserver_list *vllist; + struct afs_cell *cell; + + if (v == SEQ_START_TOKEN) { + /* display header on line 1 */ + seq_puts(m, "USE ACT TTL SV ST NAME\n"); + return 0; + } + + cell = list_entry(v, struct afs_cell, proc_link); + vllist = rcu_dereference(cell->vl_servers); + + /* display one cell per line on subsequent lines */ + seq_printf(m, "%3u %3u %6lld %2u %2u %s\n", + refcount_read(&cell->ref), + atomic_read(&cell->active), + cell->dns_expiry - ktime_get_real_seconds(), + vllist ? vllist->nr_servers : 0, + cell->state, + cell->name); + return 0; +} + +static void *afs_proc_cells_start(struct seq_file *m, loff_t *_pos) + __acquires(rcu) +{ + rcu_read_lock(); + return seq_hlist_start_head_rcu(&afs_seq2net(m)->proc_cells, *_pos); +} + +static void *afs_proc_cells_next(struct seq_file *m, void *v, loff_t *pos) +{ + return seq_hlist_next_rcu(v, &afs_seq2net(m)->proc_cells, pos); +} + +static void afs_proc_cells_stop(struct seq_file *m, void *v) + __releases(rcu) +{ + rcu_read_unlock(); +} + +static const struct seq_operations afs_proc_cells_ops = { + .start = afs_proc_cells_start, + .next = afs_proc_cells_next, + .stop = afs_proc_cells_stop, + .show = afs_proc_cells_show, +}; + +/* + * handle writes to /proc/fs/afs/cells + * - to add cells: echo "add [:][:]" + */ +static int afs_proc_cells_write(struct file *file, char *buf, size_t size) +{ + struct seq_file *m = file->private_data; + struct afs_net *net = afs_seq2net(m); + char *name, *args; + int ret; + + /* trim to first NL */ + name = memchr(buf, '\n', size); + if (name) + *name = 0; + + /* split into command, name and argslist */ + name = strchr(buf, ' '); + if (!name) + goto inval; + do { + *name++ = 0; + } while(*name == ' '); + if (!*name) + goto inval; + + args = strchr(name, ' '); + if (args) { + do { + *args++ = 0; + } while(*args == ' '); + if (!*args) + goto inval; + } + + /* determine command to perform */ + _debug("cmd=%s name=%s args=%s", buf, name, args); + + if (strcmp(buf, "add") == 0) { + struct afs_cell *cell; + + cell = afs_lookup_cell(net, name, strlen(name), args, + AFS_LOOKUP_CELL_PRELOAD, + afs_cell_trace_use_lookup_add); + if (IS_ERR(cell)) { + ret = PTR_ERR(cell); + goto done; + } + + if (test_and_set_bit(AFS_CELL_FL_NO_GC, &cell->flags)) + afs_unuse_cell(cell, afs_cell_trace_unuse_no_pin); + } else { + goto inval; + } + + ret = 0; + +done: + _leave(" = %d", ret); + return ret; + +inval: + ret = -EINVAL; + printk("kAFS: Invalid Command on /proc/fs/afs/cells file\n"); + goto done; +} + +/* + * Display the list of addr_prefs known to the namespace. + */ +static int afs_proc_addr_prefs_show(struct seq_file *m, void *v) +{ + struct afs_addr_preference_list *preflist; + struct afs_addr_preference *pref; + struct afs_net *net = afs_seq2net_single(m); + union { + struct sockaddr_in sin; + struct sockaddr_in6 sin6; + } addr; + unsigned int i; + char buf[44]; /* Maximum ipv6 + max subnet is 43 */ + + rcu_read_lock(); + preflist = rcu_dereference(net->address_prefs); + + if (!preflist) { + seq_puts(m, "NO PREFS\n"); + goto out; + } + + seq_printf(m, "PROT SUBNET PRIOR (v=%u n=%u/%u/%u)\n", + preflist->version, preflist->ipv6_off, preflist->nr, preflist->max_prefs); + + memset(&addr, 0, sizeof(addr)); + + for (i = 0; i < preflist->nr; i++) { + pref = &preflist->prefs[i]; + + addr.sin.sin_family = pref->family; + if (pref->family == AF_INET) { + memcpy(&addr.sin.sin_addr, &pref->ipv4_addr, + sizeof(addr.sin.sin_addr)); + snprintf(buf, sizeof(buf), "%pISc/%u", &addr.sin, pref->subnet_mask); + seq_printf(m, "UDP %-43.43s %5u\n", buf, pref->prio); + } else { + memcpy(&addr.sin6.sin6_addr, &pref->ipv6_addr, + sizeof(addr.sin6.sin6_addr)); + snprintf(buf, sizeof(buf), "%pISc/%u", &addr.sin6, pref->subnet_mask); + seq_printf(m, "UDP %-43.43s %5u\n", buf, pref->prio); + } + } + +out: + rcu_read_unlock(); + return 0; +} + +/* + * Display the name of the current workstation cell. + */ +static int afs_proc_rootcell_show(struct seq_file *m, void *v) +{ + struct afs_cell *cell; + struct afs_net *net; + + net = afs_seq2net_single(m); + down_read(&net->cells_lock); + cell = rcu_dereference_protected(net->ws_cell, lockdep_is_held(&net->cells_lock)); + if (cell) + seq_printf(m, "%s\n", cell->name); + up_read(&net->cells_lock); + return 0; +} + +/* + * Set the current workstation cell and optionally supply its list of volume + * location servers. + * + * echo "cell.name:192.168.231.14" >/proc/fs/afs/rootcell + */ +static int afs_proc_rootcell_write(struct file *file, char *buf, size_t size) +{ + struct seq_file *m = file->private_data; + struct afs_net *net = afs_seq2net_single(m); + char *s; + int ret; + + ret = -EINVAL; + if (buf[0] == '.') + goto out; + if (memchr(buf, '/', size)) + goto out; + + /* trim to first NL */ + s = memchr(buf, '\n', size); + if (s) + *s = 0; + + /* determine command to perform */ + _debug("rootcell=%s", buf); + + ret = -EEXIST; + inode_lock(file_inode(file)); + if (!rcu_access_pointer(net->ws_cell)) + ret = afs_cell_init(net, buf); + else + printk("busy\n"); + inode_unlock(file_inode(file)); + +out: + _leave(" = %d", ret); + return ret; +} + +static const char afs_vol_types[3][3] = { + [AFSVL_RWVOL] = "RW", + [AFSVL_ROVOL] = "RO", + [AFSVL_BACKVOL] = "BK", +}; + +/* + * Display the list of volumes known to a cell. + */ +static int afs_proc_cell_volumes_show(struct seq_file *m, void *v) +{ + struct afs_volume *vol = hlist_entry(v, struct afs_volume, proc_link); + + /* Display header on line 1 */ + if (v == SEQ_START_TOKEN) { + seq_puts(m, "USE VID TY NAME\n"); + return 0; + } + + seq_printf(m, "%3d %08llx %s %s\n", + refcount_read(&vol->ref), vol->vid, + afs_vol_types[vol->type], + vol->name); + + return 0; +} + +static void *afs_proc_cell_volumes_start(struct seq_file *m, loff_t *_pos) + __acquires(cell->proc_lock) +{ + struct afs_cell *cell = pde_data(file_inode(m->file)); + + rcu_read_lock(); + return seq_hlist_start_head_rcu(&cell->proc_volumes, *_pos); +} + +static void *afs_proc_cell_volumes_next(struct seq_file *m, void *v, + loff_t *_pos) +{ + struct afs_cell *cell = pde_data(file_inode(m->file)); + + return seq_hlist_next_rcu(v, &cell->proc_volumes, _pos); +} + +static void afs_proc_cell_volumes_stop(struct seq_file *m, void *v) + __releases(cell->proc_lock) +{ + rcu_read_unlock(); +} + +static const struct seq_operations afs_proc_cell_volumes_ops = { + .start = afs_proc_cell_volumes_start, + .next = afs_proc_cell_volumes_next, + .stop = afs_proc_cell_volumes_stop, + .show = afs_proc_cell_volumes_show, +}; + +static const char *const dns_record_sources[NR__dns_record_source + 1] = { + [DNS_RECORD_UNAVAILABLE] = "unav", + [DNS_RECORD_FROM_CONFIG] = "cfg", + [DNS_RECORD_FROM_DNS_A] = "A", + [DNS_RECORD_FROM_DNS_AFSDB] = "AFSDB", + [DNS_RECORD_FROM_DNS_SRV] = "SRV", + [DNS_RECORD_FROM_NSS] = "nss", + [NR__dns_record_source] = "[weird]" +}; + +static const char *const dns_lookup_statuses[NR__dns_lookup_status + 1] = { + [DNS_LOOKUP_NOT_DONE] = "no-lookup", + [DNS_LOOKUP_GOOD] = "good", + [DNS_LOOKUP_GOOD_WITH_BAD] = "good/bad", + [DNS_LOOKUP_BAD] = "bad", + [DNS_LOOKUP_GOT_NOT_FOUND] = "not-found", + [DNS_LOOKUP_GOT_LOCAL_FAILURE] = "local-failure", + [DNS_LOOKUP_GOT_TEMP_FAILURE] = "temp-failure", + [DNS_LOOKUP_GOT_NS_FAILURE] = "ns-failure", + [NR__dns_lookup_status] = "[weird]" +}; + +/* + * Display the list of Volume Location servers we're using for a cell. + */ +static int afs_proc_cell_vlservers_show(struct seq_file *m, void *v) +{ + const struct afs_vl_seq_net_private *priv = m->private; + const struct afs_vlserver_list *vllist = priv->vllist; + const struct afs_vlserver_entry *entry; + const struct afs_vlserver *vlserver; + const struct afs_addr_list *alist; + int i; + + if (v == SEQ_START_TOKEN) { + seq_printf(m, "# source %s, status %s\n", + dns_record_sources[vllist ? vllist->source : 0], + dns_lookup_statuses[vllist ? vllist->status : 0]); + return 0; + } + + entry = v; + vlserver = entry->server; + alist = rcu_dereference(vlserver->addresses); + + seq_printf(m, "%s [p=%hu w=%hu s=%s,%s]:\n", + vlserver->name, entry->priority, entry->weight, + dns_record_sources[alist ? alist->source : entry->source], + dns_lookup_statuses[alist ? alist->status : entry->status]); + if (alist) { + for (i = 0; i < alist->nr_addrs; i++) + seq_printf(m, " %c %pISpc\n", + alist->preferred == i ? '>' : '-', + rxrpc_kernel_remote_addr(alist->addrs[i].peer)); + } + seq_printf(m, " info: fl=%lx rtt=%d\n", vlserver->flags, vlserver->rtt); + seq_printf(m, " probe: fl=%x e=%d ac=%d out=%d\n", + vlserver->probe.flags, vlserver->probe.error, + vlserver->probe.abort_code, + atomic_read(&vlserver->probe_outstanding)); + return 0; +} + +static void *afs_proc_cell_vlservers_start(struct seq_file *m, loff_t *_pos) + __acquires(rcu) +{ + struct afs_vl_seq_net_private *priv = m->private; + struct afs_vlserver_list *vllist; + struct afs_cell *cell = pde_data(file_inode(m->file)); + loff_t pos = *_pos; + + rcu_read_lock(); + + vllist = rcu_dereference(cell->vl_servers); + priv->vllist = vllist; + + if (pos < 0) + *_pos = pos = 0; + if (pos == 0) + return SEQ_START_TOKEN; + + if (pos - 1 >= vllist->nr_servers) + return NULL; + + return &vllist->servers[pos - 1]; +} + +static void *afs_proc_cell_vlservers_next(struct seq_file *m, void *v, + loff_t *_pos) +{ + struct afs_vl_seq_net_private *priv = m->private; + struct afs_vlserver_list *vllist = priv->vllist; + loff_t pos; + + pos = *_pos; + pos++; + *_pos = pos; + if (!vllist || pos - 1 >= vllist->nr_servers) + return NULL; + + return &vllist->servers[pos - 1]; +} + +static void afs_proc_cell_vlservers_stop(struct seq_file *m, void *v) + __releases(rcu) +{ + rcu_read_unlock(); +} + +static const struct seq_operations afs_proc_cell_vlservers_ops = { + .start = afs_proc_cell_vlservers_start, + .next = afs_proc_cell_vlservers_next, + .stop = afs_proc_cell_vlservers_stop, + .show = afs_proc_cell_vlservers_show, +}; + +/* + * Display the list of fileservers we're using within a namespace. + */ +static int afs_proc_servers_show(struct seq_file *m, void *v) +{ + struct afs_endpoint_state *estate; + struct afs_addr_list *alist; + struct afs_server *server; + unsigned long failed; + int i; + + if (v == SEQ_START_TOKEN) { + seq_puts(m, "UUID REF ACT CELL\n"); + return 0; + } + + server = list_entry(v, struct afs_server, proc_link); + seq_printf(m, "%pU %3d %3d %s\n", + &server->uuid, + refcount_read(&server->ref), + atomic_read(&server->active), + server->cell->name); + seq_printf(m, " - info: fl=%lx rtt=%u\n", + server->flags, server->rtt); + seq_printf(m, " - probe: last=%d\n", + (int)(jiffies - server->probed_at) / HZ); + + estate = rcu_dereference(server->endpoint_state); + if (!estate) + goto out; + failed = estate->failed_set; + seq_printf(m, " - ESTATE pq=%x np=%u rsp=%lx f=%lx\n", + estate->probe_seq, atomic_read(&estate->nr_probing), + estate->responsive_set, estate->failed_set); + + alist = estate->addresses; + seq_printf(m, " - ALIST v=%u ap=%u\n", + alist->version, alist->addr_pref_version); + for (i = 0; i < alist->nr_addrs; i++) { + const struct afs_address *addr = &alist->addrs[i]; + + seq_printf(m, " [%x] %pISpc%s rtt=%d err=%d p=%u\n", + i, rxrpc_kernel_remote_addr(addr->peer), + alist->preferred == i ? "*" : + test_bit(i, &failed) ? "!" : "", + rxrpc_kernel_get_srtt(addr->peer), + addr->last_error, addr->prio); + } + +out: + return 0; +} + +static void *afs_proc_servers_start(struct seq_file *m, loff_t *_pos) + __acquires(rcu) +{ + rcu_read_lock(); + return seq_hlist_start_head_rcu(&afs_seq2net(m)->fs_proc, *_pos); +} + +static void *afs_proc_servers_next(struct seq_file *m, void *v, loff_t *_pos) +{ + return seq_hlist_next_rcu(v, &afs_seq2net(m)->fs_proc, _pos); +} + +static void afs_proc_servers_stop(struct seq_file *m, void *v) + __releases(rcu) +{ + rcu_read_unlock(); +} + +static const struct seq_operations afs_proc_servers_ops = { + .start = afs_proc_servers_start, + .next = afs_proc_servers_next, + .stop = afs_proc_servers_stop, + .show = afs_proc_servers_show, +}; + +/* + * Display the list of strings that may be substituted for the @sys pathname + * macro. + */ +static int afs_proc_sysname_show(struct seq_file *m, void *v) +{ + struct afs_net *net = afs_seq2net(m); + struct afs_sysnames *sysnames = net->sysnames; + unsigned int i = (unsigned long)v - 1; + + if (i < sysnames->nr) + seq_printf(m, "%s\n", sysnames->subs[i]); + return 0; +} + +static void *afs_proc_sysname_start(struct seq_file *m, loff_t *pos) + __acquires(&net->sysnames_lock) +{ + struct afs_net *net = afs_seq2net(m); + struct afs_sysnames *names; + + read_lock(&net->sysnames_lock); + + names = net->sysnames; + if (*pos >= names->nr) + return NULL; + return (void *)(unsigned long)(*pos + 1); +} + +static void *afs_proc_sysname_next(struct seq_file *m, void *v, loff_t *pos) +{ + struct afs_net *net = afs_seq2net(m); + struct afs_sysnames *names = net->sysnames; + + *pos += 1; + if (*pos >= names->nr) + return NULL; + return (void *)(unsigned long)(*pos + 1); +} + +static void afs_proc_sysname_stop(struct seq_file *m, void *v) + __releases(&net->sysnames_lock) +{ + struct afs_net *net = afs_seq2net(m); + + read_unlock(&net->sysnames_lock); +} + +static const struct seq_operations afs_proc_sysname_ops = { + .start = afs_proc_sysname_start, + .next = afs_proc_sysname_next, + .stop = afs_proc_sysname_stop, + .show = afs_proc_sysname_show, +}; + +/* + * Allow the @sys substitution to be configured. + */ +static int afs_proc_sysname_write(struct file *file, char *buf, size_t size) +{ + struct afs_sysnames *sysnames, *kill; + struct seq_file *m = file->private_data; + struct afs_net *net = afs_seq2net(m); + char *s, *p, *sub; + int ret, len; + + sysnames = kzalloc_obj(*sysnames); + if (!sysnames) + return -ENOMEM; + refcount_set(&sysnames->usage, 1); + kill = sysnames; + + p = buf; + while ((s = strsep(&p, " \t\n"))) { + len = strlen(s); + if (len == 0) + continue; + ret = -ENAMETOOLONG; + if (len >= AFSNAMEMAX) + goto error; + + if (len >= 4 && + s[len - 4] == '@' && + s[len - 3] == 's' && + s[len - 2] == 'y' && + s[len - 1] == 's') + /* Protect against recursion */ + goto invalid; + + if (s[0] == '.' && + (len < 2 || (len == 2 && s[1] == '.'))) + goto invalid; + + if (memchr(s, '/', len)) + goto invalid; + + ret = -EFBIG; + if (sysnames->nr >= AFS_NR_SYSNAME) + goto out; + + if (strcmp(s, afs_init_sysname) == 0) { + sub = (char *)afs_init_sysname; + } else { + ret = -ENOMEM; + sub = kmemdup(s, len + 1, GFP_KERNEL); + if (!sub) + goto out; + } + + sysnames->subs[sysnames->nr] = sub; + sysnames->nr++; + } + + if (sysnames->nr == 0) { + sysnames->subs[0] = sysnames->blank; + sysnames->nr++; + } + + write_lock(&net->sysnames_lock); + kill = net->sysnames; + net->sysnames = sysnames; + write_unlock(&net->sysnames_lock); + ret = 0; +out: + afs_put_sysnames(kill); + return ret; + +invalid: + ret = -EINVAL; +error: + goto out; +} + +void afs_put_sysnames(struct afs_sysnames *sysnames) +{ + int i; + + if (sysnames && refcount_dec_and_test(&sysnames->usage)) { + for (i = 0; i < sysnames->nr; i++) + if (sysnames->subs[i] != afs_init_sysname && + sysnames->subs[i] != sysnames->blank) + kfree(sysnames->subs[i]); + kfree(sysnames); + } +} + +/* + * Display general per-net namespace statistics + */ +static int afs_proc_stats_show(struct seq_file *m, void *v) +{ + struct afs_net *net = afs_seq2net_single(m); + + seq_puts(m, "kAFS statistics\n"); + + seq_printf(m, "dir-mgmt: look=%u reval=%u inval=%u relpg=%u\n", + atomic_read(&net->n_lookup), + atomic_read(&net->n_reval), + atomic_read(&net->n_inval), + atomic_read(&net->n_relpg)); + + seq_printf(m, "dir-data: rdpg=%u\n", + atomic_read(&net->n_read_dir)); + + seq_printf(m, "dir-edit: cr=%u rm=%u\n", + atomic_read(&net->n_dir_cr), + atomic_read(&net->n_dir_rm)); + + seq_printf(m, "file-rd : n=%u nb=%lu\n", + atomic_read(&net->n_fetches), + atomic_long_read(&net->n_fetch_bytes)); + seq_printf(m, "file-wr : n=%u nb=%lu\n", + atomic_read(&net->n_stores), + atomic_long_read(&net->n_store_bytes)); + return 0; +} + +/* + * initialise /proc/fs/afs// + */ +int afs_proc_cell_setup(struct afs_cell *cell) +{ + struct proc_dir_entry *dir; + struct afs_net *net = cell->net; + + _enter("%p{%s},%p", cell, cell->name, net->proc_afs); + + dir = proc_net_mkdir(net->net, cell->name, net->proc_afs); + if (!dir) + goto error_dir; + + if (!proc_create_net_data("vlservers", 0444, dir, + &afs_proc_cell_vlservers_ops, + sizeof(struct afs_vl_seq_net_private), + cell) || + !proc_create_net_data("volumes", 0444, dir, + &afs_proc_cell_volumes_ops, + sizeof(struct seq_net_private), + cell)) + goto error_tree; + + _leave(" = 0"); + return 0; + +error_tree: + remove_proc_subtree(cell->name, net->proc_afs); +error_dir: + _leave(" = -ENOMEM"); + return -ENOMEM; +} + +/* + * remove /proc/fs/afs// + */ +void afs_proc_cell_remove(struct afs_cell *cell) +{ + struct afs_net *net = cell->net; + + _enter(""); + remove_proc_subtree(cell->name, net->proc_afs); + _leave(""); +} + +/* + * initialise the /proc/fs/afs/ directory + */ +int afs_proc_init(struct afs_net *net) +{ + struct proc_dir_entry *p; + + _enter(""); + + p = proc_net_mkdir(net->net, "afs", net->net->proc_net); + if (!p) + goto error_dir; + + if (!proc_create_net_data_write("cells", 0644, p, + &afs_proc_cells_ops, + afs_proc_cells_write, + sizeof(struct seq_net_private), + NULL) || + !proc_create_net_single_write("rootcell", 0644, p, + afs_proc_rootcell_show, + afs_proc_rootcell_write, + NULL) || + !proc_create_net("servers", 0444, p, &afs_proc_servers_ops, + sizeof(struct seq_net_private)) || + !proc_create_net_single("stats", 0444, p, afs_proc_stats_show, NULL) || + !proc_create_net_data_write("sysname", 0644, p, + &afs_proc_sysname_ops, + afs_proc_sysname_write, + sizeof(struct seq_net_private), + NULL) || + !proc_create_net_single_write("addr_prefs", 0644, p, + afs_proc_addr_prefs_show, + afs_proc_addr_prefs_write, + NULL)) + goto error_tree; + + net->proc_afs = p; + _leave(" = 0"); + return 0; + +error_tree: + proc_remove(p); +error_dir: + _leave(" = -ENOMEM"); + return -ENOMEM; +} + +/* + * clean up the /proc/fs/afs/ directory + */ +void afs_proc_cleanup(struct afs_net *net) +{ + proc_remove(net->proc_afs); + net->proc_afs = NULL; +} diff --git a/fs/afs/protocol_afs.h b/fs/afs/protocol_afs.h new file mode 100644 index 000000000..0c39358c8 --- /dev/null +++ b/fs/afs/protocol_afs.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* AFS protocol bits + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + + +#define AFSCAPABILITIESMAX 196 /* Maximum number of words in a capability set */ + +/* AFS3 Fileserver capabilities word 0 */ +#define AFS3_VICED_CAPABILITY_ERRORTRANS 0x0001 /* Uses UAE errors */ +#define AFS3_VICED_CAPABILITY_64BITFILES 0x0002 /* FetchData64 & StoreData64 supported */ +#define AFS3_VICED_CAPABILITY_WRITELOCKACL 0x0004 /* Can lock a file even without lock perm */ +#define AFS3_VICED_CAPABILITY_SANEACLS 0x0008 /* ACLs reviewed for sanity - don't use */ diff --git a/fs/afs/protocol_uae.h b/fs/afs/protocol_uae.h new file mode 100644 index 000000000..1b3d1060b --- /dev/null +++ b/fs/afs/protocol_uae.h @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Universal AFS Error codes (UAE). + * + * Copyright (C) 2003, Daria Phoebe Brashear + * Copyright (C) 2018 Red Hat, Inc. All Rights Reserved. + */ + +enum { + UAEPERM = 0x2f6df00, /* Operation not permitted */ + UAENOENT = 0x2f6df01, /* No such file or directory */ + UAESRCH = 0x2f6df02, /* No such process */ + UAEINTR = 0x2f6df03, /* Interrupted system call */ + UAEIO = 0x2f6df04, /* I/O error */ + UAENXIO = 0x2f6df05, /* No such device or address */ + UAE2BIG = 0x2f6df06, /* Arg list too long */ + UAENOEXEC = 0x2f6df07, /* Exec format error */ + UAEBADF = 0x2f6df08, /* Bad file number */ + UAECHILD = 0x2f6df09, /* No child processes */ + UAEAGAIN = 0x2f6df0a, /* Try again */ + UAENOMEM = 0x2f6df0b, /* Out of memory */ + UAEACCES = 0x2f6df0c, /* Permission denied */ + UAEFAULT = 0x2f6df0d, /* Bad address */ + UAENOTBLK = 0x2f6df0e, /* Block device required */ + UAEBUSY = 0x2f6df0f, /* Device or resource busy */ + UAEEXIST = 0x2f6df10, /* File exists */ + UAEXDEV = 0x2f6df11, /* Cross-device link */ + UAENODEV = 0x2f6df12, /* No such device */ + UAENOTDIR = 0x2f6df13, /* Not a directory */ + UAEISDIR = 0x2f6df14, /* Is a directory */ + UAEINVAL = 0x2f6df15, /* Invalid argument */ + UAENFILE = 0x2f6df16, /* File table overflow */ + UAEMFILE = 0x2f6df17, /* Too many open files */ + UAENOTTY = 0x2f6df18, /* Not a typewriter */ + UAETXTBSY = 0x2f6df19, /* Text file busy */ + UAEFBIG = 0x2f6df1a, /* File too large */ + UAENOSPC = 0x2f6df1b, /* No space left on device */ + UAESPIPE = 0x2f6df1c, /* Illegal seek */ + UAEROFS = 0x2f6df1d, /* Read-only file system */ + UAEMLINK = 0x2f6df1e, /* Too many links */ + UAEPIPE = 0x2f6df1f, /* Broken pipe */ + UAEDOM = 0x2f6df20, /* Math argument out of domain of func */ + UAERANGE = 0x2f6df21, /* Math result not representable */ + UAEDEADLK = 0x2f6df22, /* Resource deadlock would occur */ + UAENAMETOOLONG = 0x2f6df23, /* File name too long */ + UAENOLCK = 0x2f6df24, /* No record locks available */ + UAENOSYS = 0x2f6df25, /* Function not implemented */ + UAENOTEMPTY = 0x2f6df26, /* Directory not empty */ + UAELOOP = 0x2f6df27, /* Too many symbolic links encountered */ + UAEWOULDBLOCK = 0x2f6df28, /* Operation would block */ + UAENOMSG = 0x2f6df29, /* No message of desired type */ + UAEIDRM = 0x2f6df2a, /* Identifier removed */ + UAECHRNG = 0x2f6df2b, /* Channel number out of range */ + UAEL2NSYNC = 0x2f6df2c, /* Level 2 not synchronized */ + UAEL3HLT = 0x2f6df2d, /* Level 3 halted */ + UAEL3RST = 0x2f6df2e, /* Level 3 reset */ + UAELNRNG = 0x2f6df2f, /* Link number out of range */ + UAEUNATCH = 0x2f6df30, /* Protocol driver not attached */ + UAENOCSI = 0x2f6df31, /* No CSI structure available */ + UAEL2HLT = 0x2f6df32, /* Level 2 halted */ + UAEBADE = 0x2f6df33, /* Invalid exchange */ + UAEBADR = 0x2f6df34, /* Invalid request descriptor */ + UAEXFULL = 0x2f6df35, /* Exchange full */ + UAENOANO = 0x2f6df36, /* No anode */ + UAEBADRQC = 0x2f6df37, /* Invalid request code */ + UAEBADSLT = 0x2f6df38, /* Invalid slot */ + UAEBFONT = 0x2f6df39, /* Bad font file format */ + UAENOSTR = 0x2f6df3a, /* Device not a stream */ + UAENODATA = 0x2f6df3b, /* No data available */ + UAETIME = 0x2f6df3c, /* Timer expired */ + UAENOSR = 0x2f6df3d, /* Out of streams resources */ + UAENONET = 0x2f6df3e, /* Machine is not on the network */ + UAENOPKG = 0x2f6df3f, /* Package not installed */ + UAEREMOTE = 0x2f6df40, /* Object is remote */ + UAENOLINK = 0x2f6df41, /* Link has been severed */ + UAEADV = 0x2f6df42, /* Advertise error */ + UAESRMNT = 0x2f6df43, /* Srmount error */ + UAECOMM = 0x2f6df44, /* Communication error on send */ + UAEPROTO = 0x2f6df45, /* Protocol error */ + UAEMULTIHOP = 0x2f6df46, /* Multihop attempted */ + UAEDOTDOT = 0x2f6df47, /* RFS specific error */ + UAEBADMSG = 0x2f6df48, /* Not a data message */ + UAEOVERFLOW = 0x2f6df49, /* Value too large for defined data type */ + UAENOTUNIQ = 0x2f6df4a, /* Name not unique on network */ + UAEBADFD = 0x2f6df4b, /* File descriptor in bad state */ + UAEREMCHG = 0x2f6df4c, /* Remote address changed */ + UAELIBACC = 0x2f6df4d, /* Can not access a needed shared library */ + UAELIBBAD = 0x2f6df4e, /* Accessing a corrupted shared library */ + UAELIBSCN = 0x2f6df4f, /* .lib section in a.out corrupted */ + UAELIBMAX = 0x2f6df50, /* Attempting to link in too many shared libraries */ + UAELIBEXEC = 0x2f6df51, /* Cannot exec a shared library directly */ + UAEILSEQ = 0x2f6df52, /* Illegal byte sequence */ + UAERESTART = 0x2f6df53, /* Interrupted system call should be restarted */ + UAESTRPIPE = 0x2f6df54, /* Streams pipe error */ + UAEUSERS = 0x2f6df55, /* Too many users */ + UAENOTSOCK = 0x2f6df56, /* Socket operation on non-socket */ + UAEDESTADDRREQ = 0x2f6df57, /* Destination address required */ + UAEMSGSIZE = 0x2f6df58, /* Message too long */ + UAEPROTOTYPE = 0x2f6df59, /* Protocol wrong type for socket */ + UAENOPROTOOPT = 0x2f6df5a, /* Protocol not available */ + UAEPROTONOSUPPORT = 0x2f6df5b, /* Protocol not supported */ + UAESOCKTNOSUPPORT = 0x2f6df5c, /* Socket type not supported */ + UAEOPNOTSUPP = 0x2f6df5d, /* Operation not supported on transport endpoint */ + UAEPFNOSUPPORT = 0x2f6df5e, /* Protocol family not supported */ + UAEAFNOSUPPORT = 0x2f6df5f, /* Address family not supported by protocol */ + UAEADDRINUSE = 0x2f6df60, /* Address already in use */ + UAEADDRNOTAVAIL = 0x2f6df61, /* Cannot assign requested address */ + UAENETDOWN = 0x2f6df62, /* Network is down */ + UAENETUNREACH = 0x2f6df63, /* Network is unreachable */ + UAENETRESET = 0x2f6df64, /* Network dropped connection because of reset */ + UAECONNABORTED = 0x2f6df65, /* Software caused connection abort */ + UAECONNRESET = 0x2f6df66, /* Connection reset by peer */ + UAENOBUFS = 0x2f6df67, /* No buffer space available */ + UAEISCONN = 0x2f6df68, /* Transport endpoint is already connected */ + UAENOTCONN = 0x2f6df69, /* Transport endpoint is not connected */ + UAESHUTDOWN = 0x2f6df6a, /* Cannot send after transport endpoint shutdown */ + UAETOOMANYREFS = 0x2f6df6b, /* Too many references: cannot splice */ + UAETIMEDOUT = 0x2f6df6c, /* Connection timed out */ + UAECONNREFUSED = 0x2f6df6d, /* Connection refused */ + UAEHOSTDOWN = 0x2f6df6e, /* Host is down */ + UAEHOSTUNREACH = 0x2f6df6f, /* No route to host */ + UAEALREADY = 0x2f6df70, /* Operation already in progress */ + UAEINPROGRESS = 0x2f6df71, /* Operation now in progress */ + UAESTALE = 0x2f6df72, /* Stale NFS file handle */ + UAEUCLEAN = 0x2f6df73, /* Structure needs cleaning */ + UAENOTNAM = 0x2f6df74, /* Not a XENIX named type file */ + UAENAVAIL = 0x2f6df75, /* No XENIX semaphores available */ + UAEISNAM = 0x2f6df76, /* Is a named type file */ + UAEREMOTEIO = 0x2f6df77, /* Remote I/O error */ + UAEDQUOT = 0x2f6df78, /* Quota exceeded */ + UAENOMEDIUM = 0x2f6df79, /* No medium found */ + UAEMEDIUMTYPE = 0x2f6df7a, /* Wrong medium type */ +}; diff --git a/fs/afs/protocol_yfs.h b/fs/afs/protocol_yfs.h new file mode 100644 index 000000000..b2f06c191 --- /dev/null +++ b/fs/afs/protocol_yfs.h @@ -0,0 +1,179 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* YFS protocol bits + * + * Copyright (C) 2018 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define YFS_FS_SERVICE 2500 +#define YFS_CM_SERVICE 2501 + +#define YFSCBMAX 1024 + +enum YFS_CM_Operations { + YFSCBProbe = 206, /* probe client */ + YFSCBGetLock = 207, /* get contents of CM lock table */ + YFSCBXStatsVersion = 209, /* get version of extended statistics */ + YFSCBGetXStats = 210, /* get contents of extended statistics data */ + YFSCBInitCallBackState3 = 213, /* initialise callback state, version 3 */ + YFSCBProbeUuid = 214, /* check the client hasn't rebooted */ + YFSCBGetServerPrefs = 215, + YFSCBGetCellServDV = 216, + YFSCBGetLocalCell = 217, + YFSCBGetCacheConfig = 218, + YFSCBGetCellByNum = 65537, + YFSCBTellMeAboutYourself = 65538, /* get client capabilities */ + YFSCBCallBack = 64204, +}; + +enum YFS_FS_Operations { + YFSFETCHACL = 64131, /* YFS Fetch file AFS3 ACL */ + YFSFETCHSTATUS = 64132, /* YFS Fetch file status */ + YFSSTOREACL = 64134, /* YFS Store file AFS3 ACL */ + YFSSTORESTATUS = 64135, /* YFS Store file status */ + YFSREMOVEFILE = 64136, /* YFS Remove a file */ + YFSCREATEFILE = 64137, /* YFS Create a file */ + YFSRENAME = 64138, /* YFS Rename or move a file or directory */ + YFSSYMLINK = 64139, /* YFS Create a symbolic link */ + YFSLINK = 64140, /* YFS Create a hard link */ + YFSMAKEDIR = 64141, /* YFS Create a directory */ + YFSREMOVEDIR = 64142, /* YFS Remove a directory */ + YFSGETVOLUMESTATUS = 64149, /* YFS Get volume status information */ + YFSSETVOLUMESTATUS = 64150, /* YFS Set volume status information */ + YFSSETLOCK = 64156, /* YFS Request a file lock */ + YFSEXTENDLOCK = 64157, /* YFS Extend a file lock */ + YFSRELEASELOCK = 64158, /* YFS Release a file lock */ + YFSLOOKUP = 64161, /* YFS lookup file in directory */ + YFSFLUSHCPS = 64165, + YFSFETCHOPAQUEACL = 64168, /* YFS Fetch file YFS ACL */ + YFSWHOAMI = 64170, + YFSREMOVEACL = 64171, + YFSREMOVEFILE2 = 64173, + YFSSTOREOPAQUEACL2 = 64174, + YFSRENAME_REPLACE = 64176, + YFSRENAME_NOREPLACE = 64177, + YFSRENAME_EXCHANGE = 64187, + YFSINLINEBULKSTATUS = 64536, /* YFS Fetch multiple file statuses with errors */ + YFSFETCHDATA64 = 64537, /* YFS Fetch file data */ + YFSSTOREDATA64 = 64538, /* YFS Store file data */ + YFSUPDATESYMLINK = 64540, +}; + +struct yfs_xdr_u64 { + __be32 msw; + __be32 lsw; +} __packed; + +static inline u64 xdr_to_u64(const struct yfs_xdr_u64 x) +{ + return ((u64)ntohl(x.msw) << 32) | ntohl(x.lsw); +} + +static inline struct yfs_xdr_u64 u64_to_xdr(const u64 x) +{ + return (struct yfs_xdr_u64){ .msw = htonl(x >> 32), .lsw = htonl(x) }; +} + +struct yfs_xdr_vnode { + struct yfs_xdr_u64 lo; + __be32 hi; + __be32 unique; +} __packed; + +struct yfs_xdr_YFSFid { + struct yfs_xdr_u64 volume; + struct yfs_xdr_vnode vnode; +} __packed; + + +struct yfs_xdr_YFSFetchStatus { + __be32 type; + __be32 nlink; + struct yfs_xdr_u64 size; + struct yfs_xdr_u64 data_version; + struct yfs_xdr_u64 author; + struct yfs_xdr_u64 owner; + struct yfs_xdr_u64 group; + __be32 mode; + __be32 caller_access; + __be32 anon_access; + struct yfs_xdr_vnode parent; + __be32 data_access_protocol; + struct yfs_xdr_u64 mtime_client; + struct yfs_xdr_u64 mtime_server; + __be32 lock_count; + __be32 abort_code; +} __packed; + +struct yfs_xdr_YFSCallBack { + __be32 version; + struct yfs_xdr_u64 expiration_time; + __be32 type; +} __packed; + +struct yfs_xdr_YFSStoreStatus { + __be32 mask; + __be32 mode; + struct yfs_xdr_u64 mtime_client; + struct yfs_xdr_u64 owner; + struct yfs_xdr_u64 group; +} __packed; + +struct yfs_xdr_RPCFlags { + __be32 rpc_flags; +} __packed; + +struct yfs_xdr_YFSVolSync { + struct yfs_xdr_u64 vol_creation_date; + struct yfs_xdr_u64 vol_update_date; + struct yfs_xdr_u64 max_quota; + struct yfs_xdr_u64 blocks_in_use; + struct yfs_xdr_u64 blocks_avail; +} __packed; + +enum yfs_volume_type { + yfs_volume_type_ro = 0, + yfs_volume_type_rw = 1, +}; + +#define yfs_FVSOnline 0x1 +#define yfs_FVSInservice 0x2 +#define yfs_FVSBlessed 0x4 +#define yfs_FVSNeedsSalvage 0x8 + +struct yfs_xdr_YFSFetchVolumeStatus { + struct yfs_xdr_u64 vid; + struct yfs_xdr_u64 parent_id; + __be32 flags; + __be32 type; + struct yfs_xdr_u64 max_quota; + struct yfs_xdr_u64 blocks_in_use; + struct yfs_xdr_u64 part_blocks_avail; + struct yfs_xdr_u64 part_max_blocks; + struct yfs_xdr_u64 vol_copy_date; + struct yfs_xdr_u64 vol_backup_date; +} __packed; + +struct yfs_xdr_YFSStoreVolumeStatus { + __be32 mask; + struct yfs_xdr_u64 min_quota; + struct yfs_xdr_u64 max_quota; + struct yfs_xdr_u64 file_quota; +} __packed; + +enum yfs_lock_type { + yfs_LockNone = -1, + yfs_LockRead = 0, + yfs_LockWrite = 1, + yfs_LockExtend = 2, + yfs_LockRelease = 3, + yfs_LockMandatoryRead = 0x100, + yfs_LockMandatoryWrite = 0x101, + yfs_LockMandatoryExtend = 0x102, +}; + +/* RXYFS Viced Capability Flags */ +#define YFS_VICED_CAPABILITY_ERRORTRANS 0x0001 /* Deprecated v0.195 */ +#define YFS_VICED_CAPABILITY_64BITFILES 0x0002 /* Deprecated v0.195 */ +#define YFS_VICED_CAPABILITY_WRITELOCKACL 0x0004 /* Can lock a file even without lock perm */ +#define YFS_VICED_CAPABILITY_SANEACLS 0x0008 /* Deprecated v0.195 */ diff --git a/fs/afs/rotate.c b/fs/afs/rotate.c new file mode 100644 index 000000000..490b433e4 --- /dev/null +++ b/fs/afs/rotate.c @@ -0,0 +1,763 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Handle fileserver selection and rotation. + * + * Copyright (C) 2017 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include "internal.h" +#include "afs_fs.h" +#include "protocol_uae.h" + +void afs_clear_server_states(struct afs_operation *op) +{ + unsigned int i; + + if (op->server_states) { + for (i = 0; i < op->server_list->nr_servers; i++) + afs_put_endpoint_state(op->server_states[i].endpoint_state, + afs_estate_trace_put_server_state); + kfree(op->server_states); + } +} + +/* + * Begin iteration through a server list, starting with the vnode's last used + * server if possible, or the last recorded good server if not. + */ +static bool afs_start_fs_iteration(struct afs_operation *op, + struct afs_vnode *vnode) +{ + struct afs_server *server; + void *cb_server; + int i; + + trace_afs_rotate(op, afs_rotate_trace_start, 0); + + read_lock(&op->volume->servers_lock); + op->server_list = afs_get_serverlist( + rcu_dereference_protected(op->volume->servers, + lockdep_is_held(&op->volume->servers_lock))); + read_unlock(&op->volume->servers_lock); + + op->server_states = kzalloc_objs(op->server_states[0], + op->server_list->nr_servers); + if (!op->server_states) { + afs_op_nomem(op); + trace_afs_rotate(op, afs_rotate_trace_nomem, 0); + return false; + } + + rcu_read_lock(); + for (i = 0; i < op->server_list->nr_servers; i++) { + struct afs_endpoint_state *estate; + struct afs_server_state *s = &op->server_states[i]; + + server = op->server_list->servers[i].server; + estate = rcu_dereference(server->endpoint_state); + s->endpoint_state = afs_get_endpoint_state(estate, + afs_estate_trace_get_server_state); + s->probe_seq = estate->probe_seq; + s->untried_addrs = (1UL << estate->addresses->nr_addrs) - 1; + init_waitqueue_entry(&s->probe_waiter, current); + afs_get_address_preferences(op->net, estate->addresses); + } + rcu_read_unlock(); + + + op->untried_servers = (1UL << op->server_list->nr_servers) - 1; + op->server_index = -1; + + cb_server = vnode->cb_server; + if (cb_server) { + /* See if the vnode's preferred record is still available */ + for (i = 0; i < op->server_list->nr_servers; i++) { + server = op->server_list->servers[i].server; + if (server == cb_server) { + op->server_index = i; + goto found_interest; + } + } + + /* If we have a lock outstanding on a server that's no longer + * serving this vnode, then we can't switch to another server + * and have to return an error. + */ + if (op->flags & AFS_OPERATION_CUR_ONLY) { + afs_op_set_error(op, -ESTALE); + trace_afs_rotate(op, afs_rotate_trace_stale_lock, 0); + return false; + } + + /* Note that the callback promise is effectively broken */ + write_seqlock(&vnode->cb_lock); + ASSERTCMP(cb_server, ==, vnode->cb_server); + vnode->cb_server = NULL; + if (afs_clear_cb_promise(vnode, afs_cb_promise_clear_rotate_server)) + vnode->cb_break++; + write_sequnlock(&vnode->cb_lock); + } + +found_interest: + return true; +} + +/* + * Post volume busy note. + */ +static void afs_busy(struct afs_operation *op, u32 abort_code) +{ + const char *m; + + switch (abort_code) { + case VOFFLINE: m = "offline"; break; + case VRESTARTING: m = "restarting"; break; + case VSALVAGING: m = "being salvaged"; break; + default: m = "busy"; break; + } + + pr_notice("kAFS: Volume %llu '%s' on server %pU is %s\n", + op->volume->vid, op->volume->name, &op->server->uuid, m); +} + +/* + * Sleep and retry the operation to the same fileserver. + */ +static bool afs_sleep_and_retry(struct afs_operation *op) +{ + trace_afs_rotate(op, afs_rotate_trace_busy_sleep, 0); + if (!(op->flags & AFS_OPERATION_UNINTR)) { + msleep_interruptible(1000); + if (signal_pending(current)) { + afs_op_set_error(op, -ERESTARTSYS); + return false; + } + } else { + msleep(1000); + } + + return true; +} + +/* + * Select the fileserver to use. May be called multiple times to rotate + * through the fileservers. + */ +bool afs_select_fileserver(struct afs_operation *op) +{ + struct afs_addr_list *alist; + struct afs_server *server; + struct afs_vnode *vnode = op->file[0].vnode; + unsigned long set, failed; + s32 abort_code = op->call_abort_code; + int best_prio = 0; + int error = op->call_error, addr_index, i, j; + + op->nr_iterations++; + + _enter("OP=%x+%x,%llx,%u{%lx},%u{%lx},%d,%d", + op->debug_id, op->nr_iterations, op->volume->vid, + op->server_index, op->untried_servers, + op->addr_index, op->addr_tried, + error, abort_code); + + if (op->flags & AFS_OPERATION_STOP) { + trace_afs_rotate(op, afs_rotate_trace_stopped, 0); + _leave(" = f [stopped]"); + return false; + } + + if (op->nr_iterations == 0) + goto start; + + WRITE_ONCE(op->estate->addresses->addrs[op->addr_index].last_error, error); + trace_afs_rotate(op, afs_rotate_trace_iter, op->call_error); + + /* Evaluate the result of the previous operation, if there was one. */ + switch (op->call_error) { + case 0: + clear_bit(AFS_SE_VOLUME_OFFLINE, + &op->server_list->servers[op->server_index].flags); + clear_bit(AFS_SE_VOLUME_BUSY, + &op->server_list->servers[op->server_index].flags); + op->cumul_error.responded = true; + + /* We succeeded, but we may need to redo the op from another + * server if we're looking at a set of RO volumes where some of + * the servers have not yet been brought up to date lest we + * regress the data. We only switch to the new version once + * >=50% of the servers are updated. + */ + error = afs_update_volume_state(op); + if (error != 0) { + if (error == 1) { + afs_sleep_and_retry(op); + goto restart_from_beginning; + } + afs_op_set_error(op, error); + goto failed; + } + fallthrough; + default: + /* Success or local failure. Stop. */ + afs_op_set_error(op, error); + op->flags |= AFS_OPERATION_STOP; + trace_afs_rotate(op, afs_rotate_trace_stop, error); + _leave(" = f [okay/local %d]", error); + return false; + + case -ECONNABORTED: + /* The far side rejected the operation on some grounds. This + * might involve the server being busy or the volume having been moved. + * + * Note that various V* errors should not be sent to a cache manager + * by a fileserver as they should be translated to more modern UAE* + * errors instead. IBM AFS and OpenAFS fileservers, however, do leak + * these abort codes. + */ + trace_afs_rotate(op, afs_rotate_trace_aborted, abort_code); + op->cumul_error.responded = true; + switch (abort_code) { + case VNOVOL: + /* This fileserver doesn't know about the volume. + * - May indicate that the VL is wrong - retry once and compare + * the results. + * - May indicate that the fileserver couldn't attach to the vol. + * - The volume might have been temporarily removed so that it can + * be replaced by a volume restore. "vos" might have ended one + * transaction and has yet to create the next. + * - The volume might not be blessed or might not be in-service + * (administrative action). + */ + if (op->flags & AFS_OPERATION_VNOVOL) { + afs_op_accumulate_error(op, -EREMOTEIO, abort_code); + goto next_server; + } + + write_lock(&op->volume->servers_lock); + op->server_list->vnovol_mask |= 1 << op->server_index; + write_unlock(&op->volume->servers_lock); + + set_bit(AFS_VOLUME_NEEDS_UPDATE, &op->volume->flags); + error = afs_check_volume_status(op->volume, op); + if (error < 0) { + afs_op_set_error(op, error); + goto failed; + } + + if (test_bit(AFS_VOLUME_DELETED, &op->volume->flags)) { + afs_op_set_error(op, -ENOMEDIUM); + goto failed; + } + + /* If the server list didn't change, then assume that + * it's the fileserver having trouble. + */ + if (rcu_access_pointer(op->volume->servers) == op->server_list) { + afs_op_accumulate_error(op, -EREMOTEIO, abort_code); + goto next_server; + } + + /* Try again */ + op->flags |= AFS_OPERATION_VNOVOL; + _leave(" = t [vnovol]"); + return true; + + case VVOLEXISTS: + case VONLINE: + /* These should not be returned from the fileserver. */ + pr_warn("Fileserver returned unexpected abort %d\n", + abort_code); + afs_op_accumulate_error(op, -EREMOTEIO, abort_code); + goto next_server; + + case VNOSERVICE: + /* Prior to AFS 3.2 VNOSERVICE was returned from the fileserver + * if the volume was neither in-service nor administratively + * blessed. All usage was replaced by VNOVOL because AFS 3.1 and + * earlier cache managers did not handle VNOSERVICE and assumed + * it was the client OSes errno 105. + * + * Starting with OpenAFS 1.4.8 VNOSERVICE was repurposed as the + * fileserver idle dead time error which was sent in place of + * RX_CALL_TIMEOUT (-3). The error was intended to be sent if the + * fileserver took too long to send a reply to the client. + * RX_CALL_TIMEOUT would have caused the cache manager to mark the + * server down whereas VNOSERVICE since AFS 3.2 would cause cache + * manager to temporarily (up to 15 minutes) mark the volume + * instance as unusable. + * + * The idle dead logic resulted in cache inconsistency since a + * state changing call that the cache manager assumed was dead + * could still be processed to completion by the fileserver. This + * logic was removed in OpenAFS 1.8.0 and VNOSERVICE is no longer + * returned. However, many 1.4.8 through 1.6.24 fileservers are + * still in existence. + * + * AuriStorFS fileservers have never returned VNOSERVICE. + * + * VNOSERVICE should be treated as an alias for RX_CALL_TIMEOUT. + */ + case RX_CALL_TIMEOUT: + afs_op_accumulate_error(op, -ETIMEDOUT, abort_code); + goto next_server; + + case VSALVAGING: /* This error should not be leaked to cache managers + * but is from OpenAFS demand attach fileservers. + * It should be treated as an alias for VOFFLINE. + */ + case VSALVAGE: /* VSALVAGE should be treated as a synonym of VOFFLINE */ + case VOFFLINE: + /* The volume is in use by the volserver or another volume utility + * for an operation that might alter the contents. The volume is + * expected to come back but it might take a long time (could be + * days). + */ + if (!test_and_set_bit(AFS_SE_VOLUME_OFFLINE, + &op->server_list->servers[op->server_index].flags)) { + afs_busy(op, abort_code); + clear_bit(AFS_SE_VOLUME_BUSY, + &op->server_list->servers[op->server_index].flags); + } + if (op->flags & AFS_OPERATION_NO_VSLEEP) { + afs_op_set_error(op, -EADV); + goto failed; + } + goto busy; + + case VRESTARTING: /* The fileserver is either shutting down or starting up. */ + case VBUSY: + /* The volume is in use by the volserver or another volume + * utility for an operation that is not expected to alter the + * contents of the volume. VBUSY does not need to be returned + * for a ROVOL or BACKVOL bound to an ITBusy volserver + * transaction. The fileserver is permitted to continue serving + * content from ROVOLs and BACKVOLs during an ITBusy transaction + * because the content will not change. However, many fileserver + * releases do return VBUSY for ROVOL and BACKVOL instances under + * many circumstances. + * + * Retry after going round all the servers unless we have a file + * lock we need to maintain. + */ + if (op->flags & AFS_OPERATION_NO_VSLEEP) { + afs_op_set_error(op, -EBUSY); + goto failed; + } + if (!test_and_set_bit(AFS_SE_VOLUME_BUSY, + &op->server_list->servers[op->server_index].flags)) { + afs_busy(op, abort_code); + clear_bit(AFS_SE_VOLUME_OFFLINE, + &op->server_list->servers[op->server_index].flags); + } + busy: + if (op->flags & AFS_OPERATION_CUR_ONLY) { + if (!afs_sleep_and_retry(op)) + goto failed; + + /* Retry with same server & address */ + _leave(" = t [vbusy]"); + return true; + } + + op->flags |= AFS_OPERATION_VBUSY; + goto next_server; + + case VMOVED: + /* The volume migrated to another server. We consider + * consider all locks and callbacks broken and request + * an update from the VLDB. + * + * We also limit the number of VMOVED hops we will + * honour, just in case someone sets up a loop. + */ + if (op->flags & AFS_OPERATION_VMOVED) { + afs_op_set_error(op, -EREMOTEIO); + goto failed; + } + op->flags |= AFS_OPERATION_VMOVED; + + set_bit(AFS_VOLUME_WAIT, &op->volume->flags); + set_bit(AFS_VOLUME_NEEDS_UPDATE, &op->volume->flags); + error = afs_check_volume_status(op->volume, op); + if (error < 0) { + afs_op_set_error(op, error); + goto failed; + } + + /* If the server list didn't change, then the VLDB is + * out of sync with the fileservers. This is hopefully + * a temporary condition, however, so we don't want to + * permanently block access to the file. + * + * TODO: Try other fileservers if we can. + * + * TODO: Retry a few times with sleeps. + */ + if (rcu_access_pointer(op->volume->servers) == op->server_list) { + afs_op_accumulate_error(op, -ENOMEDIUM, abort_code); + goto failed; + } + + goto restart_from_beginning; + + case UAEIO: + case VIO: + afs_op_accumulate_error(op, -EREMOTEIO, abort_code); + if (op->volume->type != AFSVL_RWVOL) + goto next_server; + goto failed; + + case VDISKFULL: + case UAENOSPC: + /* The partition is full. Only applies to RWVOLs. + * Translate locally and return ENOSPC. + * No replicas to failover to. + */ + afs_op_set_error(op, -ENOSPC); + goto failed_but_online; + + case VOVERQUOTA: + case UAEDQUOT: + /* Volume is full. Only applies to RWVOLs. + * Translate locally and return EDQUOT. + * No replicas to failover to. + */ + afs_op_set_error(op, -EDQUOT); + goto failed_but_online; + + case RX_INVALID_OPERATION: + case RXGEN_OPCODE: + /* Handle downgrading to an older operation. */ + afs_op_set_error(op, -ENOTSUPP); + if (op->flags & AFS_OPERATION_DOWNGRADE) { + op->flags &= ~AFS_OPERATION_DOWNGRADE; + goto go_again; + } + goto failed_but_online; + + default: + afs_op_accumulate_error(op, error, abort_code); + failed_but_online: + clear_bit(AFS_SE_VOLUME_OFFLINE, + &op->server_list->servers[op->server_index].flags); + clear_bit(AFS_SE_VOLUME_BUSY, + &op->server_list->servers[op->server_index].flags); + goto failed; + } + + case -ETIMEDOUT: + case -ETIME: + if (afs_op_error(op) != -EDESTADDRREQ) + goto iterate_address; + fallthrough; + case -ERFKILL: + case -EADDRNOTAVAIL: + case -ENETUNREACH: + case -EHOSTUNREACH: + case -EHOSTDOWN: + case -ECONNREFUSED: + _debug("no conn"); + afs_op_accumulate_error(op, error, 0); + goto iterate_address; + + case -ENETRESET: + pr_warn("kAFS: Peer reset %s (op=%x)\n", + op->type ? op->type->name : "???", op->debug_id); + fallthrough; + case -ECONNRESET: + _debug("call reset"); + afs_op_set_error(op, error); + goto failed; + } + +restart_from_beginning: + trace_afs_rotate(op, afs_rotate_trace_restart, 0); + _debug("restart"); + op->estate = NULL; + op->server = NULL; + afs_clear_server_states(op); + op->server_states = NULL; + afs_put_serverlist(op->net, op->server_list); + op->server_list = NULL; +start: + _debug("start"); + ASSERTCMP(op->estate, ==, NULL); + /* See if we need to do an update of the volume record. Note that the + * volume may have moved or even have been deleted. + */ + error = afs_check_volume_status(op->volume, op); + trace_afs_rotate(op, afs_rotate_trace_check_vol_status, error); + if (error < 0) { + afs_op_set_error(op, error); + goto failed; + } + + if (!afs_start_fs_iteration(op, vnode)) + goto failed; + + _debug("__ VOL %llx __", op->volume->vid); + +pick_server: + _debug("pick [%lx]", op->untried_servers); + ASSERTCMP(op->estate, ==, NULL); + + error = afs_wait_for_fs_probes(op, op->server_states, + !(op->flags & AFS_OPERATION_UNINTR)); + switch (error) { + case 0: /* No untried responsive servers and no outstanding probes */ + trace_afs_rotate(op, afs_rotate_trace_probe_none, 0); + goto no_more_servers; + case 1: /* Got a response */ + trace_afs_rotate(op, afs_rotate_trace_probe_response, 0); + break; + case 2: /* Probe data superseded */ + trace_afs_rotate(op, afs_rotate_trace_probe_superseded, 0); + goto restart_from_beginning; + default: + trace_afs_rotate(op, afs_rotate_trace_probe_error, error); + afs_op_set_error(op, error); + goto failed; + } + + /* Pick the untried server with the highest priority untried endpoint. + * If we have outstanding callbacks, we stick with the server we're + * already using if we can. + */ + if (op->server) { + _debug("server %u", op->server_index); + if (test_bit(op->server_index, &op->untried_servers)) + goto selected_server; + op->server = NULL; + _debug("no server"); + } + + rcu_read_lock(); + op->server_index = -1; + best_prio = -1; + for (i = 0; i < op->server_list->nr_servers; i++) { + struct afs_endpoint_state *es; + struct afs_server_entry *se = &op->server_list->servers[i]; + struct afs_addr_list *sal; + struct afs_server *s = se->server; + + if (!test_bit(i, &op->untried_servers) || + test_bit(AFS_SE_EXCLUDED, &se->flags) || + !test_bit(AFS_SERVER_FL_RESPONDING, &s->flags)) + continue; + es = op->server_states[i].endpoint_state; + sal = es->addresses; + + afs_get_address_preferences_rcu(op->net, sal); + for (j = 0; j < sal->nr_addrs; j++) { + if (es->failed_set & (1 << j)) + continue; + if (!sal->addrs[j].peer) + continue; + if (sal->addrs[j].prio > best_prio) { + op->server_index = i; + best_prio = sal->addrs[j].prio; + } + } + } + rcu_read_unlock(); + + if (op->server_index == -1) + goto no_more_servers; + +selected_server: + trace_afs_rotate(op, afs_rotate_trace_selected_server, best_prio); + _debug("use %d prio %u", op->server_index, best_prio); + __clear_bit(op->server_index, &op->untried_servers); + + /* We're starting on a different fileserver from the list. We need to + * check it, create a callback intercept, find its address list and + * probe its capabilities before we use it. + */ + ASSERTCMP(op->estate, ==, NULL); + server = op->server_list->servers[op->server_index].server; + + if (!afs_check_server_record(op, server, op->key)) + goto failed; + + _debug("USING SERVER: %pU", &server->uuid); + + op->flags |= AFS_OPERATION_RETRY_SERVER; + op->server = server; + if (vnode->cb_server != server) { + vnode->cb_server = server; + vnode->cb_v_check = atomic_read(&vnode->volume->cb_v_break); + afs_clear_cb_promise(vnode, afs_cb_promise_clear_server_change); + } + +retry_server: + op->addr_tried = 0; + op->addr_index = -1; + +iterate_address: + /* Iterate over the current server's address list to try and find an + * address on which it will respond to us. + */ + op->estate = op->server_states[op->server_index].endpoint_state; + set = READ_ONCE(op->estate->responsive_set); + failed = READ_ONCE(op->estate->failed_set); + _debug("iterate ES=%x rs=%lx fs=%lx", op->estate->probe_seq, set, failed); + set &= ~(failed | op->addr_tried); + trace_afs_rotate(op, afs_rotate_trace_iterate_addr, set); + if (!set) + goto wait_for_more_probe_results; + + alist = op->estate->addresses; + best_prio = -1; + addr_index = 0; + for (i = 0; i < alist->nr_addrs; i++) { + if (!(set & (1 << i))) + continue; + if (alist->addrs[i].prio > best_prio) { + addr_index = i; + best_prio = alist->addrs[i].prio; + } + } + + alist->preferred = addr_index; + + op->addr_index = addr_index; + set_bit(addr_index, &op->addr_tried); + + _debug("address [%u] %u/%u %pISp", + op->server_index, addr_index, alist->nr_addrs, + rxrpc_kernel_remote_addr(alist->addrs[op->addr_index].peer)); +go_again: + op->volsync.creation = TIME64_MIN; + op->volsync.update = TIME64_MIN; + op->call_responded = false; + _leave(" = t"); + return true; + +wait_for_more_probe_results: + error = afs_wait_for_one_fs_probe(op->server, op->estate, op->addr_tried, + !(op->flags & AFS_OPERATION_UNINTR)); + if (error == 1) + goto iterate_address; + if (!error) + goto restart_from_beginning; + + /* We've now had a failure to respond on all of a server's addresses - + * immediately probe them again and consider retrying the server. + */ + trace_afs_rotate(op, afs_rotate_trace_probe_fileserver, 0); + afs_probe_fileserver(op->net, op->server); + if (op->flags & AFS_OPERATION_RETRY_SERVER) { + error = afs_wait_for_one_fs_probe(op->server, op->estate, op->addr_tried, + !(op->flags & AFS_OPERATION_UNINTR)); + switch (error) { + case 1: + op->flags &= ~AFS_OPERATION_RETRY_SERVER; + trace_afs_rotate(op, afs_rotate_trace_retry_server, 1); + goto retry_server; + case 0: + trace_afs_rotate(op, afs_rotate_trace_retry_server, 0); + goto restart_from_beginning; + case -ERESTARTSYS: + afs_op_set_error(op, error); + goto failed; + case -ETIME: + case -EDESTADDRREQ: + goto next_server; + } + } + +next_server: + trace_afs_rotate(op, afs_rotate_trace_next_server, 0); + _debug("next"); + op->estate = NULL; + goto pick_server; + +no_more_servers: + /* That's all the servers poked to no good effect. Try again if some + * of them were busy. + */ + trace_afs_rotate(op, afs_rotate_trace_no_more_servers, 0); + if (op->flags & AFS_OPERATION_VBUSY) { + afs_sleep_and_retry(op); + op->flags &= ~AFS_OPERATION_VBUSY; + goto restart_from_beginning; + } + + rcu_read_lock(); + for (i = 0; i < op->server_list->nr_servers; i++) { + struct afs_endpoint_state *estate; + + estate = op->server_states[i].endpoint_state; + error = READ_ONCE(estate->error); + if (error < 0) + afs_op_accumulate_error(op, error, estate->abort_code); + } + rcu_read_unlock(); + +failed: + trace_afs_rotate(op, afs_rotate_trace_failed, 0); + op->flags |= AFS_OPERATION_STOP; + op->estate = NULL; + _leave(" = f [failed %d]", afs_op_error(op)); + return false; +} + +/* + * Dump cursor state in the case of the error being EDESTADDRREQ. + */ +void afs_dump_edestaddrreq(const struct afs_operation *op) +{ + static int count; + int i; + + if (!IS_ENABLED(CONFIG_AFS_DEBUG_CURSOR) || count > 3) + return; + count++; + + rcu_read_lock(); + + pr_notice("EDESTADDR occurred\n"); + pr_notice("OP: cbb=%x cbb2=%x fl=%x err=%hd\n", + op->file[0].cb_break_before, + op->file[1].cb_break_before, op->flags, op->cumul_error.error); + pr_notice("OP: ut=%lx ix=%d ni=%u\n", + op->untried_servers, op->server_index, op->nr_iterations); + pr_notice("OP: call er=%d ac=%d r=%u\n", + op->call_error, op->call_abort_code, op->call_responded); + + if (op->server_list) { + const struct afs_server_list *sl = op->server_list; + + pr_notice("FC: SL nr=%u vnov=%hx\n", + sl->nr_servers, sl->vnovol_mask); + for (i = 0; i < sl->nr_servers; i++) { + const struct afs_server *s = sl->servers[i].server; + const struct afs_endpoint_state *e = + rcu_dereference(s->endpoint_state); + const struct afs_addr_list *a = e->addresses; + + pr_notice("FC: server fl=%lx av=%u %pU\n", + s->flags, s->addr_version, &s->uuid); + pr_notice("FC: - pq=%x R=%lx F=%lx\n", + e->probe_seq, e->responsive_set, e->failed_set); + if (a) { + pr_notice("FC: - av=%u nr=%u/%u/%u pr=%u\n", + a->version, + a->nr_ipv4, a->nr_addrs, a->max_addrs, + a->preferred); + if (a == e->addresses) + pr_notice("FC: - current\n"); + } + } + } + + pr_notice("AC: t=%lx ax=%d\n", op->addr_tried, op->addr_index); + rcu_read_unlock(); +} diff --git a/fs/afs/rxrpc.c b/fs/afs/rxrpc.c new file mode 100644 index 000000000..d82916657 --- /dev/null +++ b/fs/afs/rxrpc.c @@ -0,0 +1,996 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Maintain an RxRPC server socket to do AFS communications through + * + * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include + +#include +#include +#include "internal.h" +#include "afs_cm.h" +#include "protocol_yfs.h" +#define RXRPC_TRACE_ONLY_DEFINE_ENUMS +#include + +struct workqueue_struct *afs_async_calls; + +static void afs_deferred_free_worker(struct work_struct *work); +static void afs_wake_up_call_waiter(struct sock *, struct rxrpc_call *, unsigned long); +static void afs_wake_up_async_call(struct sock *, struct rxrpc_call *, unsigned long); +static void afs_process_async_call(struct work_struct *); +static void afs_rx_new_call(struct sock *, struct rxrpc_call *, unsigned long); +static void afs_rx_discard_new_call(struct rxrpc_call *, unsigned long); +static void afs_rx_attach(struct rxrpc_call *rxcall, unsigned long user_call_ID); +static void afs_rx_notify_oob(struct sock *sk, struct sk_buff *oob); +static int afs_deliver_cm_op_id(struct afs_call *); + +static const struct rxrpc_kernel_ops afs_rxrpc_callback_ops = { + .notify_new_call = afs_rx_new_call, + .discard_new_call = afs_rx_discard_new_call, + .user_attach_call = afs_rx_attach, + .notify_oob = afs_rx_notify_oob, +}; + +/* asynchronous incoming call initial processing */ +static const struct afs_call_type afs_RXCMxxxx = { + .name = "CB.xxxx", + .deliver = afs_deliver_cm_op_id, +}; + +/* + * open an RxRPC socket and bind it to be a server for callback notifications + * - the socket is left in blocking mode and non-blocking ops use MSG_DONTWAIT + */ +int afs_open_socket(struct afs_net *net) +{ + struct sockaddr_rxrpc srx; + struct socket *socket; + int ret; + + _enter(""); + + ret = sock_create_kern(net->net, AF_RXRPC, SOCK_DGRAM, PF_INET6, &socket); + if (ret < 0) + goto error_1; + + socket->sk->sk_allocation = GFP_NOFS; + socket->sk->sk_user_data = net; + + /* bind the callback manager's address to make this a server socket */ + memset(&srx, 0, sizeof(srx)); + srx.srx_family = AF_RXRPC; + srx.srx_service = CM_SERVICE; + srx.transport_type = SOCK_DGRAM; + srx.transport_len = sizeof(srx.transport.sin6); + srx.transport.sin6.sin6_family = AF_INET6; + srx.transport.sin6.sin6_port = htons(AFS_CM_PORT); + + ret = rxrpc_sock_set_min_security_level(socket->sk, + RXRPC_SECURITY_ENCRYPT); + if (ret < 0) + goto error_2; + + ret = rxrpc_sock_set_manage_response(socket->sk, true); + if (ret < 0) + goto error_2; + + ret = afs_create_token_key(net, socket); + if (ret < 0) + pr_err("Couldn't create RxGK CM key: %d\n", ret); + + ret = kernel_bind(socket, (struct sockaddr_unsized *) &srx, sizeof(srx)); + if (ret == -EADDRINUSE) { + srx.transport.sin6.sin6_port = 0; + ret = kernel_bind(socket, (struct sockaddr_unsized *) &srx, sizeof(srx)); + } + if (ret < 0) + goto error_2; + + srx.srx_service = YFS_CM_SERVICE; + ret = kernel_bind(socket, (struct sockaddr_unsized *) &srx, sizeof(srx)); + if (ret < 0) + goto error_2; + + /* Ideally, we'd turn on service upgrade here, but we can't because + * OpenAFS is buggy and leaks the userStatus field from packet to + * packet and between FS packets and CB packets - so if we try to do an + * upgrade on an FS packet, OpenAFS will leak that into the CB packet + * it sends back to us. + */ + + rxrpc_kernel_set_notifications(socket, &afs_rxrpc_callback_ops); + + ret = kernel_listen(socket, INT_MAX); + if (ret < 0) + goto error_2; + + net->socket = socket; + afs_charge_preallocation(&net->charge_preallocation_work); + _leave(" = 0"); + return 0; + +error_2: + sock_release(socket); +error_1: + _leave(" = %d", ret); + return ret; +} + +/* + * close the RxRPC socket AFS was using + */ +void afs_close_socket(struct afs_net *net) +{ + _enter(""); + + cancel_work_sync(&net->charge_preallocation_work); + cancel_work_sync(&net->rx_oob_work); + /* Future work items should now see ->live is false. */ + + kernel_listen(net->socket, 0); + + /* Make sure work items are no longer running. */ + flush_workqueue(afs_async_calls); + cancel_work_sync(&net->charge_preallocation_work); + + if (net->spare_incoming_call) { + afs_put_call(net->spare_incoming_call); + net->spare_incoming_call = NULL; + } + + _debug("outstanding %u", atomic_read(&net->nr_outstanding_calls)); + wait_var_event(&net->nr_outstanding_calls, + !atomic_read(&net->nr_outstanding_calls)); + _debug("no outstanding calls"); + + kernel_sock_shutdown(net->socket, SHUT_RDWR); + flush_workqueue(afs_async_calls); + cancel_work_sync(&net->rx_oob_work); + net->socket->sk->sk_user_data = NULL; + sock_release(net->socket); + key_put(net->fs_cm_token_key); + + _debug("dework"); + _leave(""); +} + +/* + * Allocate a call. + */ +static struct afs_call *afs_alloc_call(struct afs_net *net, + const struct afs_call_type *type, + gfp_t gfp) +{ + struct afs_call *call; + int o; + + call = kzalloc_obj(*call, gfp); + if (!call) + return NULL; + + call->type = type; + call->net = net; + call->debug_id = atomic_inc_return(&rxrpc_debug_id); + refcount_set(&call->ref, 1); + INIT_WORK(&call->async_work, type->async_rx ?: afs_process_async_call); + INIT_WORK(&call->work, call->type->work); + INIT_WORK(&call->free_work, afs_deferred_free_worker); + init_waitqueue_head(&call->waitq); + spin_lock_init(&call->state_lock); + call->iter = &call->def_iter; + + o = atomic_inc_return(&net->nr_outstanding_calls); + trace_afs_call(call->debug_id, afs_call_trace_alloc, 1, o, + __builtin_return_address(0)); + return call; +} + +static void afs_free_call(struct afs_call *call) +{ + struct afs_net *net = call->net; + int o; + + ASSERT(!work_pending(&call->async_work)); + + rxrpc_kernel_put_peer(call->peer); + + if (call->rxcall) { + rxrpc_kernel_shutdown_call(net->socket, call->rxcall); + rxrpc_kernel_put_call(net->socket, call->rxcall); + call->rxcall = NULL; + } + if (call->type->destructor) + call->type->destructor(call); + + afs_unuse_server_notime(call->net, call->server, afs_server_trace_unuse_call); + kfree(call->request); + + o = atomic_read(&net->nr_outstanding_calls); + trace_afs_call(call->debug_id, afs_call_trace_free, 0, o, + __builtin_return_address(0)); + kfree(call); + + o = atomic_dec_return(&net->nr_outstanding_calls); + if (o == 0) + wake_up_var(&net->nr_outstanding_calls); +} + +/* + * Dispose of a reference on a call. + */ +void afs_put_call(struct afs_call *call) +{ + struct afs_net *net = call->net; + unsigned int debug_id = call->debug_id; + bool zero; + int r, o; + + zero = __refcount_dec_and_test(&call->ref, &r); + o = atomic_read(&net->nr_outstanding_calls); + trace_afs_call(debug_id, afs_call_trace_put, r - 1, o, + __builtin_return_address(0)); + if (zero) + afs_free_call(call); +} + +static void afs_deferred_free_worker(struct work_struct *work) +{ + struct afs_call *call = container_of(work, struct afs_call, free_work); + + afs_free_call(call); +} + +/* + * Dispose of a reference on a call, deferring the cleanup to a workqueue + * to avoid lock recursion. + */ +void afs_deferred_put_call(struct afs_call *call) +{ + struct afs_net *net = call->net; + unsigned int debug_id = call->debug_id; + bool zero; + int r, o; + + zero = __refcount_dec_and_test(&call->ref, &r); + o = atomic_read(&net->nr_outstanding_calls); + trace_afs_call(debug_id, afs_call_trace_put, r - 1, o, + __builtin_return_address(0)); + if (zero) + schedule_work(&call->free_work); +} + +/* + * Queue the call for actual work. + */ +static void afs_queue_call_work(struct afs_call *call) +{ + if (call->type->work) { + afs_get_call(call, afs_call_trace_work); + if (!queue_work(afs_wq, &call->work)) + afs_put_call(call); + } +} + +/* + * allocate a call with flat request and reply buffers + */ +struct afs_call *afs_alloc_flat_call(struct afs_net *net, + const struct afs_call_type *type, + size_t request_size, size_t reply_max) +{ + struct afs_call *call; + + call = afs_alloc_call(net, type, GFP_NOFS); + if (!call) + goto nomem_call; + + if (request_size) { + call->request_size = request_size; + call->request = kmalloc(request_size, GFP_NOFS); + if (!call->request) + goto nomem_free; + } + + if (reply_max) { + call->reply_max = reply_max; + call->buffer = kmalloc(reply_max, GFP_NOFS); + if (!call->buffer) + goto nomem_free; + } + + afs_extract_to_buf(call, call->reply_max); + call->operation_ID = type->op; + init_waitqueue_head(&call->waitq); + return call; + +nomem_free: + afs_put_call(call); +nomem_call: + return NULL; +} + +/* + * clean up a call with flat buffer + */ +void afs_flat_call_destructor(struct afs_call *call) +{ + _enter(""); + + kfree(call->request); + call->request = NULL; + kfree(call->buffer); + call->buffer = NULL; +} + +/* + * Advance the AFS call state when the RxRPC call ends the transmit phase. + */ +static void afs_notify_end_request_tx(struct sock *sock, + struct rxrpc_call *rxcall, + unsigned long call_user_ID) +{ + struct afs_call *call = (struct afs_call *)call_user_ID; + + afs_set_call_state(call, AFS_CALL_CL_REQUESTING, AFS_CALL_CL_AWAIT_REPLY); +} + +/* + * Initiate a call and synchronously queue up the parameters for dispatch. Any + * error is stored into the call struct, which the caller must check for. + */ +void afs_make_call(struct afs_call *call, gfp_t gfp) +{ + struct rxrpc_call *rxcall; + struct msghdr msg; + struct kvec iov[1]; + size_t len; + s64 tx_total_len; + int ret; + + _enter(",{%pISp+%u},", rxrpc_kernel_remote_addr(call->peer), call->service_id); + + ASSERT(call->type != NULL); + ASSERT(call->type->name != NULL); + + _debug("____MAKE %p{%s,%x} [%d]____", + call, call->type->name, key_serial(call->key), + atomic_read(&call->net->nr_outstanding_calls)); + + trace_afs_make_call(call); + + /* Work out the length we're going to transmit. This is awkward for + * calls such as FS.StoreData where there's an extra injection of data + * after the initial fixed part. + */ + tx_total_len = call->request_size; + if (call->write_iter) + tx_total_len += iov_iter_count(call->write_iter); + + /* If the call is going to be asynchronous, we need an extra ref for + * the call to hold itself so the caller need not hang on to its ref. + */ + if (call->async) { + afs_get_call(call, afs_call_trace_get); + call->drop_ref = true; + } + + /* create a call */ + rxcall = rxrpc_kernel_begin_call(call->net->socket, call->peer, call->key, + (unsigned long)call, + tx_total_len, + call->max_lifespan, + gfp, + (call->async ? + afs_wake_up_async_call : + afs_wake_up_call_waiter), + call->service_id, + call->upgrade, + (call->intr ? RXRPC_PREINTERRUPTIBLE : + RXRPC_UNINTERRUPTIBLE), + call->debug_id); + if (IS_ERR(rxcall)) { + ret = PTR_ERR(rxcall); + call->error = ret; + goto error_kill_call; + } + + call->rxcall = rxcall; + call->issue_time = ktime_get_real(); + + /* send the request */ + iov[0].iov_base = call->request; + iov[0].iov_len = call->request_size; + + msg.msg_name = NULL; + msg.msg_namelen = 0; + iov_iter_kvec(&msg.msg_iter, ITER_SOURCE, iov, 1, call->request_size); + msg.msg_control = NULL; + msg.msg_controllen = 0; + msg.msg_flags = MSG_WAITALL | (call->write_iter ? MSG_MORE : 0); + + ret = rxrpc_kernel_send_data(call->net->socket, rxcall, + &msg, call->request_size, + afs_notify_end_request_tx); + if (ret < 0) + goto error_do_abort; + + if (call->write_iter) { + msg.msg_iter = *call->write_iter; + msg.msg_flags &= ~MSG_MORE; + trace_afs_send_data(call, &msg); + + ret = rxrpc_kernel_send_data(call->net->socket, + call->rxcall, &msg, + iov_iter_count(&msg.msg_iter), + afs_notify_end_request_tx); + *call->write_iter = msg.msg_iter; + + trace_afs_sent_data(call, &msg, ret); + if (ret < 0) + goto error_do_abort; + } + + /* Note that at this point, we may have received the reply or an abort + * - and an asynchronous call may already have completed. + * + * afs_wait_for_call_to_complete(call) + * must be called to synchronously clean up. + */ + return; + +error_do_abort: + if (ret != -ECONNABORTED) + rxrpc_kernel_abort_call(call->net->socket, rxcall, + RX_USER_ABORT, ret, + afs_abort_send_data_error); + if (call->async) { + afs_see_call(call, afs_call_trace_async_abort); + return; + } + + if (ret == -ECONNABORTED) { + len = 0; + iov_iter_kvec(&msg.msg_iter, ITER_DEST, NULL, 0, 0); + rxrpc_kernel_recv_data(call->net->socket, rxcall, + &msg.msg_iter, &len, false, + &call->abort_code, &call->service_id); + call->responded = true; + } + call->error = ret; + trace_afs_call_done(call); +error_kill_call: + if (call->async) + afs_see_call(call, afs_call_trace_async_kill); + if (call->type->immediate_cancel) + call->type->immediate_cancel(call); + + /* We need to dispose of the extra ref we grabbed for an async call. + * The call, however, might be queued on afs_async_calls and we need to + * make sure we don't get any more notifications that might requeue it. + */ + if (call->rxcall) + rxrpc_kernel_shutdown_call(call->net->socket, call->rxcall); + if (call->async) { + if (cancel_work_sync(&call->async_work)) + afs_put_call(call); + afs_set_call_complete(call, ret, 0); + } + + call->error = ret; + call->state = AFS_CALL_COMPLETE; + _leave(" = %d", ret); +} + +/* + * Log remote abort codes that indicate that we have a protocol disagreement + * with the server. + */ +static void afs_log_error(struct afs_call *call, s32 remote_abort) +{ + static int max = 0; + const char *msg; + int m; + + switch (remote_abort) { + case RX_EOF: msg = "unexpected EOF"; break; + case RXGEN_CC_MARSHAL: msg = "client marshalling"; break; + case RXGEN_CC_UNMARSHAL: msg = "client unmarshalling"; break; + case RXGEN_SS_MARSHAL: msg = "server marshalling"; break; + case RXGEN_SS_UNMARSHAL: msg = "server unmarshalling"; break; + case RXGEN_DECODE: msg = "opcode decode"; break; + case RXGEN_SS_XDRFREE: msg = "server XDR cleanup"; break; + case RXGEN_CC_XDRFREE: msg = "client XDR cleanup"; break; + case -32: msg = "insufficient data"; break; + default: + return; + } + + m = max; + if (m < 3) { + max = m + 1; + pr_notice("kAFS: Peer reported %s failure on %s [%pISp]\n", + msg, call->type->name, + rxrpc_kernel_remote_addr(call->peer)); + } +} + +/* + * deliver messages to a call + */ +void afs_deliver_to_call(struct afs_call *call) +{ + enum afs_call_state state; + size_t len; + u32 abort_code, remote_abort = 0; + int ret; + + _enter("%s", call->type->name); + + while (state = READ_ONCE(call->state), + state == AFS_CALL_CL_AWAIT_REPLY || + state == AFS_CALL_SV_AWAIT_OP_ID || + state == AFS_CALL_SV_AWAIT_REQUEST || + state == AFS_CALL_SV_AWAIT_ACK + ) { + if (state == AFS_CALL_SV_AWAIT_ACK) { + len = 0; + iov_iter_kvec(&call->def_iter, ITER_DEST, NULL, 0, 0); + ret = rxrpc_kernel_recv_data(call->net->socket, + call->rxcall, &call->def_iter, + &len, false, &remote_abort, + &call->service_id); + trace_afs_receive_data(call, &call->def_iter, false, ret); + + if (ret == -EINPROGRESS || ret == -EAGAIN) + return; + if (ret < 0 || ret == 1) { + if (ret == 1) + ret = 0; + goto call_complete; + } + return; + } + + ret = call->type->deliver(call); + state = READ_ONCE(call->state); + if (ret == 0 && call->unmarshalling_error) + ret = -EBADMSG; + switch (ret) { + case 0: + call->responded = true; + afs_queue_call_work(call); + if (state == AFS_CALL_CL_PROC_REPLY) { + if (call->op) + set_bit(AFS_SERVER_FL_MAY_HAVE_CB, + &call->op->server->flags); + goto call_complete; + } + ASSERTCMP(state, >, AFS_CALL_CL_PROC_REPLY); + goto done; + case -EINPROGRESS: + case -EAGAIN: + goto out; + case -ECONNABORTED: + ASSERTCMP(state, ==, AFS_CALL_COMPLETE); + call->responded = true; + afs_log_error(call, call->abort_code); + goto done; + case -ENOTSUPP: + call->responded = true; + abort_code = RXGEN_OPCODE; + rxrpc_kernel_abort_call(call->net->socket, call->rxcall, + abort_code, ret, + afs_abort_op_not_supported); + goto local_abort; + case -EIO: + pr_err("kAFS: Call %u in bad state %u\n", + call->debug_id, state); + fallthrough; + case -ENODATA: + case -EBADMSG: + case -EMSGSIZE: + case -ENOMEM: + case -EFAULT: + abort_code = RXGEN_CC_UNMARSHAL; + if (state != AFS_CALL_CL_AWAIT_REPLY) + abort_code = RXGEN_SS_UNMARSHAL; + rxrpc_kernel_abort_call(call->net->socket, call->rxcall, + abort_code, ret, + afs_abort_unmarshal_error); + goto local_abort; + default: + abort_code = RX_CALL_DEAD; + rxrpc_kernel_abort_call(call->net->socket, call->rxcall, + abort_code, ret, + afs_abort_general_error); + goto local_abort; + } + } + +done: + if (call->type->done) + call->type->done(call); +out: + _leave(""); + return; + +local_abort: + abort_code = 0; +call_complete: + afs_set_call_complete(call, ret, remote_abort); + goto done; +} + +/* + * Wait synchronously for a call to complete. + */ +void afs_wait_for_call_to_complete(struct afs_call *call) +{ + bool rxrpc_complete = false; + + _enter(""); + + if (!afs_check_call_state(call, AFS_CALL_COMPLETE)) { + DECLARE_WAITQUEUE(myself, current); + + add_wait_queue(&call->waitq, &myself); + for (;;) { + set_current_state(TASK_UNINTERRUPTIBLE); + + /* deliver any messages that are in the queue */ + if (!afs_check_call_state(call, AFS_CALL_COMPLETE) && + call->need_attention) { + call->need_attention = false; + __set_current_state(TASK_RUNNING); + afs_deliver_to_call(call); + continue; + } + + if (afs_check_call_state(call, AFS_CALL_COMPLETE)) + break; + + if (!rxrpc_kernel_check_life(call->net->socket, call->rxcall)) { + /* rxrpc terminated the call. */ + rxrpc_complete = true; + break; + } + + schedule(); + } + + remove_wait_queue(&call->waitq, &myself); + __set_current_state(TASK_RUNNING); + } + + if (!afs_check_call_state(call, AFS_CALL_COMPLETE)) { + if (rxrpc_complete) { + afs_set_call_complete(call, call->error, call->abort_code); + } else { + /* Kill off the call if it's still live. */ + _debug("call interrupted"); + if (rxrpc_kernel_abort_call(call->net->socket, call->rxcall, + RX_USER_ABORT, -EINTR, + afs_abort_interrupted)) + afs_set_call_complete(call, -EINTR, 0); + } + } +} + +/* + * wake up a waiting call + */ +static void afs_wake_up_call_waiter(struct sock *sk, struct rxrpc_call *rxcall, + unsigned long call_user_ID) +{ + struct afs_call *call = (struct afs_call *)call_user_ID; + + call->need_attention = true; + wake_up(&call->waitq); +} + +/* + * Wake up an asynchronous call. The caller is holding the call notify + * spinlock around this, so we can't call afs_put_call(). + */ +static void afs_wake_up_async_call(struct sock *sk, struct rxrpc_call *rxcall, + unsigned long call_user_ID) +{ + struct afs_call *call = (struct afs_call *)call_user_ID; + int r; + + trace_afs_notify_call(rxcall, call); + call->need_attention = true; + + if (__refcount_inc_not_zero(&call->ref, &r)) { + trace_afs_call(call->debug_id, afs_call_trace_wake, r + 1, + atomic_read(&call->net->nr_outstanding_calls), + __builtin_return_address(0)); + + if (!queue_work(afs_async_calls, &call->async_work)) + afs_deferred_put_call(call); + } +} + +/* + * Perform I/O processing on an asynchronous call. The work item carries a ref + * to the call struct that we either need to release or to pass on. + */ +static void afs_process_async_call(struct work_struct *work) +{ + struct afs_call *call = container_of(work, struct afs_call, async_work); + + _enter(""); + + if (call->state < AFS_CALL_COMPLETE && call->need_attention) { + call->need_attention = false; + afs_deliver_to_call(call); + } + + afs_put_call(call); + _leave(""); +} + +static void afs_rx_attach(struct rxrpc_call *rxcall, unsigned long user_call_ID) +{ + struct afs_call *call = (struct afs_call *)user_call_ID; + + call->rxcall = rxcall; +} + +/* + * Charge the incoming call preallocation. + */ +void afs_charge_preallocation(struct work_struct *work) +{ + struct afs_net *net = + container_of(work, struct afs_net, charge_preallocation_work); + struct afs_call *call = net->spare_incoming_call; + + while (READ_ONCE(net->live)) { + if (!call) { + call = afs_alloc_call(net, &afs_RXCMxxxx, GFP_KERNEL); + if (!call) + break; + + call->drop_ref = true; + call->async = true; + call->state = AFS_CALL_SV_AWAIT_OP_ID; + init_waitqueue_head(&call->waitq); + afs_extract_to_tmp(call); + } + + if (rxrpc_kernel_charge_accept(net->socket, + afs_wake_up_async_call, + (unsigned long)call, + GFP_KERNEL, + call->debug_id) < 0) + break; + call = NULL; + } + net->spare_incoming_call = call; +} + +/* + * Discard a preallocated call when a socket is shut down. + */ +static void afs_rx_discard_new_call(struct rxrpc_call *rxcall, + unsigned long user_call_ID) +{ + struct afs_call *call = (struct afs_call *)user_call_ID; + + call->rxcall = NULL; + afs_put_call(call); +} + +/* + * Notification of an incoming call. + */ +static void afs_rx_new_call(struct sock *sk, struct rxrpc_call *rxcall, + unsigned long user_call_ID) +{ + struct afs_call *call = (struct afs_call *)user_call_ID; + struct afs_net *net = afs_sock2net(sk); + + call->peer = rxrpc_kernel_get_call_peer(sk->sk_socket, call->rxcall); + call->server = afs_find_server(call->peer); + if (!call->server) + trace_afs_cm_no_server(call, rxrpc_kernel_remote_srx(call->peer)); + + if (net->live) + queue_work(afs_wq, &net->charge_preallocation_work); +} + +/* + * Grab the operation ID from an incoming cache manager call. The socket + * buffer is discarded on error or if we don't yet have sufficient data. + */ +static int afs_deliver_cm_op_id(struct afs_call *call) +{ + int ret; + + _enter("{%zu}", iov_iter_count(call->iter)); + + /* the operation ID forms the first four bytes of the request data */ + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + call->operation_ID = ntohl(call->tmp); + afs_set_call_state(call, AFS_CALL_SV_AWAIT_OP_ID, AFS_CALL_SV_AWAIT_REQUEST); + + /* ask the cache manager to route the call (it'll change the call type + * if successful) */ + if (!afs_cm_incoming_call(call)) + return -ENOTSUPP; + + call->security_ix = rxrpc_kernel_query_call_security(call->rxcall, + &call->service_id, + &call->enctype); + + trace_afs_cb_call(call); + call->work.func = call->type->work; + + /* pass responsibility for the remainder of this message off to the + * cache manager op */ + return call->type->deliver(call); +} + +/* + * Advance the AFS call state when an RxRPC service call ends the transmit + * phase. + */ +static void afs_notify_end_reply_tx(struct sock *sock, + struct rxrpc_call *rxcall, + unsigned long call_user_ID) +{ + struct afs_call *call = (struct afs_call *)call_user_ID; + + afs_set_call_state(call, AFS_CALL_SV_REPLYING, AFS_CALL_SV_AWAIT_ACK); +} + +/* + * send an empty reply + */ +void afs_send_empty_reply(struct afs_call *call) +{ + struct afs_net *net = call->net; + struct msghdr msg; + + _enter(""); + + rxrpc_kernel_set_tx_length(net->socket, call->rxcall, 0); + + msg.msg_name = NULL; + msg.msg_namelen = 0; + iov_iter_kvec(&msg.msg_iter, ITER_SOURCE, NULL, 0, 0); + msg.msg_control = NULL; + msg.msg_controllen = 0; + msg.msg_flags = 0; + + switch (rxrpc_kernel_send_data(net->socket, call->rxcall, &msg, 0, + afs_notify_end_reply_tx)) { + case 0: + _leave(" [replied]"); + return; + + case -ENOMEM: + _debug("oom"); + rxrpc_kernel_abort_call(net->socket, call->rxcall, + RXGEN_SS_MARSHAL, -ENOMEM, + afs_abort_oom); + fallthrough; + default: + _leave(" [error]"); + return; + } +} + +/* + * send a simple reply + */ +void afs_send_simple_reply(struct afs_call *call, const void *buf, size_t len) +{ + struct afs_net *net = call->net; + struct msghdr msg; + struct kvec iov[1]; + int n; + + _enter(""); + + rxrpc_kernel_set_tx_length(net->socket, call->rxcall, len); + + iov[0].iov_base = (void *) buf; + iov[0].iov_len = len; + msg.msg_name = NULL; + msg.msg_namelen = 0; + iov_iter_kvec(&msg.msg_iter, ITER_SOURCE, iov, 1, len); + msg.msg_control = NULL; + msg.msg_controllen = 0; + msg.msg_flags = 0; + + n = rxrpc_kernel_send_data(net->socket, call->rxcall, &msg, len, + afs_notify_end_reply_tx); + if (n >= 0) { + /* Success */ + _leave(" [replied]"); + return; + } + + if (n == -ENOMEM) { + _debug("oom"); + rxrpc_kernel_abort_call(net->socket, call->rxcall, + RXGEN_SS_MARSHAL, -ENOMEM, + afs_abort_oom); + } + _leave(" [error]"); +} + +/* + * Extract a piece of data from the received data socket buffers. + */ +int afs_extract_data(struct afs_call *call, bool want_more) +{ + struct afs_net *net = call->net; + struct iov_iter *iter = call->iter; + enum afs_call_state state; + u32 remote_abort = 0; + int ret; + + _enter("{%s,%zu,%zu},%d", + call->type->name, call->iov_len, iov_iter_count(iter), want_more); + + ret = rxrpc_kernel_recv_data(net->socket, call->rxcall, iter, + &call->iov_len, want_more, &remote_abort, + &call->service_id); + trace_afs_receive_data(call, call->iter, want_more, ret); + if (ret == 0 || ret == -EAGAIN) + return ret; + + state = READ_ONCE(call->state); + if (ret == 1) { + switch (state) { + case AFS_CALL_CL_AWAIT_REPLY: + afs_set_call_state(call, state, AFS_CALL_CL_PROC_REPLY); + break; + case AFS_CALL_SV_AWAIT_REQUEST: + afs_set_call_state(call, state, AFS_CALL_SV_REPLYING); + break; + case AFS_CALL_COMPLETE: + kdebug("prem complete %d", call->error); + return afs_io_error(call, afs_io_error_extract); + default: + break; + } + return 0; + } + + afs_set_call_complete(call, ret, remote_abort); + return ret; +} + +/* + * Log protocol error production. + */ +noinline int afs_protocol_error(struct afs_call *call, + enum afs_eproto_cause cause) +{ + trace_afs_protocol_error(call, cause); + if (call) + call->unmarshalling_error = true; + return -EBADMSG; +} + +/* + * Wake up OOB notification processing. + */ +static void afs_rx_notify_oob(struct sock *sk, struct sk_buff *oob) +{ + struct afs_net *net = sk->sk_user_data; + + if (READ_ONCE(net->live)) + queue_work(afs_wq, &net->rx_oob_work); +} diff --git a/fs/afs/security.c b/fs/afs/security.c new file mode 100644 index 000000000..6d00d62a6 --- /dev/null +++ b/fs/afs/security.c @@ -0,0 +1,520 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* AFS security handling + * + * Copyright (C) 2007, 2017 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +static DEFINE_HASHTABLE(afs_permits_cache, 10); +static DEFINE_SPINLOCK(afs_permits_lock); +static DEFINE_MUTEX(afs_key_lock); + +/* + * Allocate a key to use as a placeholder for anonymous user security. + */ +static int afs_alloc_anon_key(struct afs_cell *cell) +{ + struct key *key; + + mutex_lock(&afs_key_lock); + key = cell->anonymous_key; + if (!key) { + key = rxrpc_get_null_key(cell->key_desc); + if (!IS_ERR(key)) + cell->anonymous_key = key; + } + mutex_unlock(&afs_key_lock); + + if (IS_ERR(key)) + return PTR_ERR(key); + + _debug("anon key %p{%x}", + cell->anonymous_key, key_serial(cell->anonymous_key)); + return 0; +} + +/* + * get a key + */ +struct key *afs_request_key(struct afs_cell *cell) +{ + struct key *key; + int ret; + + _enter("{%s}", cell->key_desc); + + _debug("key %s", cell->key_desc); + key = request_key_net(&key_type_rxrpc, cell->key_desc, + cell->net->net, NULL); + if (IS_ERR(key)) { + if (PTR_ERR(key) != -ENOKEY) { + _leave(" = %ld", PTR_ERR(key)); + return key; + } + + if (!cell->anonymous_key) { + ret = afs_alloc_anon_key(cell); + if (ret < 0) + return ERR_PTR(ret); + } + + /* act as anonymous user */ + _leave(" = {%x} [anon]", key_serial(cell->anonymous_key)); + return key_get(cell->anonymous_key); + } else { + /* act as authorised user */ + _leave(" = {%x} [auth]", key_serial(key)); + return key; + } +} + +/* + * Get a key when pathwalk is in rcuwalk mode. + */ +struct key *afs_request_key_rcu(struct afs_cell *cell) +{ + struct key *key; + + _enter("{%s}", cell->key_desc); + + _debug("key %s", cell->key_desc); + key = request_key_net_rcu(&key_type_rxrpc, cell->key_desc, + cell->net->net); + if (IS_ERR(key)) { + if (PTR_ERR(key) != -ENOKEY) { + _leave(" = %ld", PTR_ERR(key)); + return key; + } + + /* act as anonymous user */ + if (!cell->anonymous_key) + return NULL; /* Need to allocate */ + _leave(" = {%x} [anon]", key_serial(cell->anonymous_key)); + return key_get(cell->anonymous_key); + } else { + /* act as authorised user */ + _leave(" = {%x} [auth]", key_serial(key)); + return key; + } +} + +/* + * Dispose of a list of permits. + */ +static void afs_permits_rcu(struct rcu_head *rcu) +{ + struct afs_permits *permits = + container_of(rcu, struct afs_permits, rcu); + int i; + + for (i = 0; i < permits->nr_permits; i++) + key_put(permits->permits[i].key); + kfree(permits); +} + +/* + * Discard a permission cache. + */ +void afs_put_permits(struct afs_permits *permits) +{ + if (permits && refcount_dec_and_test(&permits->usage)) { + spin_lock(&afs_permits_lock); + hash_del_rcu(&permits->hash_node); + spin_unlock(&afs_permits_lock); + call_rcu(&permits->rcu, afs_permits_rcu); + } +} + +/* + * Clear a permit cache on callback break. + */ +void afs_clear_permits(struct afs_vnode *vnode) +{ + struct afs_permits *permits; + + spin_lock(&vnode->lock); + permits = rcu_dereference_protected(vnode->permit_cache, + lockdep_is_held(&vnode->lock)); + RCU_INIT_POINTER(vnode->permit_cache, NULL); + spin_unlock(&vnode->lock); + + afs_put_permits(permits); +} + +/* + * Hash a list of permits. Use simple addition to make it easy to add an extra + * one at an as-yet indeterminate position in the list. + */ +static void afs_hash_permits(struct afs_permits *permits) +{ + unsigned long h = permits->nr_permits; + int i; + + for (i = 0; i < permits->nr_permits; i++) { + h += (unsigned long)permits->permits[i].key / sizeof(void *); + h += permits->permits[i].access; + } + + permits->h = h; +} + +/* + * Cache the CallerAccess result obtained from doing a fileserver operation + * that returned a vnode status for a particular key. If a callback break + * occurs whilst the operation was in progress then we have to ditch the cache + * as the ACL *may* have changed. + */ +void afs_cache_permit(struct afs_vnode *vnode, struct key *key, + unsigned int cb_break, struct afs_status_cb *scb) +{ + struct afs_permits *permits, *xpermits, *replacement, *zap, *new = NULL; + afs_access_t caller_access = scb->status.caller_access; + size_t size = 0; + bool changed = false; + int i, j; + + _enter("{%llx:%llu},%x,%x", + vnode->fid.vid, vnode->fid.vnode, key_serial(key), caller_access); + + rcu_read_lock(); + + /* Check for the common case first: We got back the same access as last + * time we tried and already have it recorded. + */ + permits = rcu_dereference(vnode->permit_cache); + if (permits) { + if (!permits->invalidated) { + for (i = 0; i < permits->nr_permits; i++) { + if (permits->permits[i].key < key) + continue; + if (permits->permits[i].key > key) + break; + if (permits->permits[i].access != caller_access) { + changed = true; + break; + } + + if (afs_cb_is_broken(cb_break, vnode)) { + changed = true; + break; + } + + /* The cache is still good. */ + rcu_read_unlock(); + return; + } + } + + changed |= permits->invalidated; + size = permits->nr_permits; + + /* If this set of permits is now wrong, clear the permits + * pointer so that no one tries to use the stale information. + */ + if (changed) { + spin_lock(&vnode->lock); + if (permits != rcu_access_pointer(vnode->permit_cache)) + goto someone_else_changed_it_unlock; + RCU_INIT_POINTER(vnode->permit_cache, NULL); + spin_unlock(&vnode->lock); + + afs_put_permits(permits); + permits = NULL; + size = 0; + } + } + + if (afs_cb_is_broken(cb_break, vnode)) + goto someone_else_changed_it; + + /* We need a ref on any permits list we want to copy as we'll have to + * drop the lock to do memory allocation. + */ + if (permits && !refcount_inc_not_zero(&permits->usage)) + goto someone_else_changed_it; + + rcu_read_unlock(); + + /* Speculatively create a new list with the revised permission set. We + * discard this if we find an extant match already in the hash, but + * it's easier to compare with memcmp this way. + * + * We fill in the key pointers at this time, but we don't get the refs + * yet. + */ + size++; + new = kzalloc_flex(*new, permits, size, GFP_NOFS); + if (!new) + goto out_put; + + refcount_set(&new->usage, 1); + new->nr_permits = size; + i = j = 0; + if (permits) { + for (i = 0; i < permits->nr_permits; i++) { + if (j == i && permits->permits[i].key > key) { + new->permits[j].key = key; + new->permits[j].access = caller_access; + j++; + } + new->permits[j].key = permits->permits[i].key; + new->permits[j].access = permits->permits[i].access; + j++; + } + } + + if (j == i) { + new->permits[j].key = key; + new->permits[j].access = caller_access; + } + + afs_hash_permits(new); + + /* Now see if the permit list we want is actually already available */ + spin_lock(&afs_permits_lock); + + hash_for_each_possible(afs_permits_cache, xpermits, hash_node, new->h) { + if (xpermits->h != new->h || + xpermits->invalidated || + xpermits->nr_permits != new->nr_permits || + memcmp(xpermits->permits, new->permits, + new->nr_permits * sizeof(struct afs_permit)) != 0) + continue; + + if (refcount_inc_not_zero(&xpermits->usage)) { + replacement = xpermits; + goto found; + } + + break; + } + + for (i = 0; i < new->nr_permits; i++) + key_get(new->permits[i].key); + hash_add_rcu(afs_permits_cache, &new->hash_node, new->h); + replacement = new; + new = NULL; + +found: + spin_unlock(&afs_permits_lock); + + kfree(new); + + rcu_read_lock(); + spin_lock(&vnode->lock); + zap = rcu_access_pointer(vnode->permit_cache); + if (!afs_cb_is_broken(cb_break, vnode) && zap == permits) + rcu_assign_pointer(vnode->permit_cache, replacement); + else + zap = replacement; + spin_unlock(&vnode->lock); + rcu_read_unlock(); + afs_put_permits(zap); +out_put: + afs_put_permits(permits); + return; + +someone_else_changed_it_unlock: + spin_unlock(&vnode->lock); +someone_else_changed_it: + /* Someone else changed the cache under us - don't recheck at this + * time. + */ + rcu_read_unlock(); + return; +} + +static bool afs_check_permit_rcu(struct afs_vnode *vnode, struct key *key, + afs_access_t *_access) +{ + const struct afs_permits *permits; + int i; + + _enter("{%llx:%llu},%x", + vnode->fid.vid, vnode->fid.vnode, key_serial(key)); + + /* check the permits to see if we've got one yet */ + if (key == vnode->volume->cell->anonymous_key) { + *_access = vnode->status.anon_access; + _leave(" = t [anon %x]", *_access); + return true; + } + + permits = rcu_dereference(vnode->permit_cache); + if (permits) { + for (i = 0; i < permits->nr_permits; i++) { + if (permits->permits[i].key < key) + continue; + if (permits->permits[i].key > key) + break; + + *_access = permits->permits[i].access; + _leave(" = %u [perm %x]", !permits->invalidated, *_access); + return !permits->invalidated; + } + } + + _leave(" = f"); + return false; +} + +/* + * check with the fileserver to see if the directory or parent directory is + * permitted to be accessed with this authorisation, and if so, what access it + * is granted + */ +int afs_check_permit(struct afs_vnode *vnode, struct key *key, + afs_access_t *_access) +{ + struct afs_permits *permits; + bool valid = false; + int i, ret; + + _enter("{%llx:%llu},%x", + vnode->fid.vid, vnode->fid.vnode, key_serial(key)); + + /* check the permits to see if we've got one yet */ + if (key == vnode->volume->cell->anonymous_key) { + _debug("anon"); + *_access = vnode->status.anon_access; + valid = true; + } else { + rcu_read_lock(); + permits = rcu_dereference(vnode->permit_cache); + if (permits) { + for (i = 0; i < permits->nr_permits; i++) { + if (permits->permits[i].key < key) + continue; + if (permits->permits[i].key > key) + break; + + *_access = permits->permits[i].access; + valid = !permits->invalidated; + break; + } + } + rcu_read_unlock(); + } + + if (!valid) { + /* Check the status on the file we're actually interested in + * (the post-processing will cache the result). + */ + _debug("no valid permit"); + + ret = afs_fetch_status(vnode, key, false, _access); + if (ret < 0) { + *_access = 0; + _leave(" = %d", ret); + return ret; + } + } + + _leave(" = 0 [access %x]", *_access); + return 0; +} + +/* + * check the permissions on an AFS file + * - AFS ACLs are attached to directories only, and a file is controlled by its + * parent directory's ACL + */ +int afs_permission(struct mnt_idmap *idmap, struct inode *inode, + int mask) +{ + struct afs_vnode *vnode = AFS_FS_I(inode); + afs_access_t access; + struct key *key; + int ret = 0; + + _enter("{{%llx:%llu},%lx},%x,", + vnode->fid.vid, vnode->fid.vnode, vnode->flags, mask); + + if (mask & MAY_NOT_BLOCK) { + key = afs_request_key_rcu(vnode->volume->cell); + if (IS_ERR_OR_NULL(key)) + return -ECHILD; + + ret = -ECHILD; + if (!afs_check_validity(vnode) || + !afs_check_permit_rcu(vnode, key, &access)) + goto error; + } else { + key = afs_request_key(vnode->volume->cell); + if (IS_ERR(key)) { + _leave(" = %ld [key]", PTR_ERR(key)); + return PTR_ERR(key); + } + + ret = afs_validate(vnode, key); + if (ret < 0) + goto error; + + /* check the permits to see if we've got one yet */ + ret = afs_check_permit(vnode, key, &access); + if (ret < 0) + goto error; + } + + /* interpret the access mask */ + _debug("REQ %x ACC %x on %s", + mask, access, S_ISDIR(inode->i_mode) ? "dir" : "file"); + + ret = 0; + if (S_ISDIR(inode->i_mode)) { + if (mask & (MAY_EXEC | MAY_READ | MAY_CHDIR)) { + if (!(access & AFS_ACE_LOOKUP)) + goto permission_denied; + } + if (mask & MAY_WRITE) { + if (!(access & (AFS_ACE_DELETE | /* rmdir, unlink, rename from */ + AFS_ACE_INSERT))) /* create, mkdir, symlink, rename to */ + goto permission_denied; + } + } else { + if (!(access & AFS_ACE_LOOKUP)) + goto permission_denied; + if ((mask & MAY_EXEC) && !(inode->i_mode & S_IXUSR)) + goto permission_denied; + if (mask & (MAY_EXEC | MAY_READ)) { + if (!(access & AFS_ACE_READ)) + goto permission_denied; + if (!(inode->i_mode & S_IRUSR)) + goto permission_denied; + } else if (mask & MAY_WRITE) { + if (!(access & AFS_ACE_WRITE)) + goto permission_denied; + if (!(inode->i_mode & S_IWUSR)) + goto permission_denied; + } + } + + key_put(key); + _leave(" = %d", ret); + return ret; + +permission_denied: + ret = -EACCES; +error: + key_put(key); + _leave(" = %d", ret); + return ret; +} + +void __exit afs_clean_up_permit_cache(void) +{ + int i; + + for (i = 0; i < HASH_SIZE(afs_permits_cache); i++) + WARN_ON_ONCE(!hlist_empty(&afs_permits_cache[i])); + +} diff --git a/fs/afs/server.c b/fs/afs/server.c new file mode 100644 index 000000000..0fe162ea2 --- /dev/null +++ b/fs/afs/server.c @@ -0,0 +1,631 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* AFS server record management + * + * Copyright (C) 2002, 2007 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include "afs_fs.h" +#include "internal.h" +#include "protocol_yfs.h" + +static unsigned afs_server_gc_delay = 10; /* Server record timeout in seconds */ +static atomic_t afs_server_debug_id; + +static void __afs_put_server(struct afs_net *, struct afs_server *); +static void afs_server_timer(struct timer_list *timer); +static void afs_server_destroyer(struct work_struct *work); + +/* + * Find a server by one of its addresses. + */ +struct afs_server *afs_find_server(const struct rxrpc_peer *peer) +{ + struct afs_server *server = (struct afs_server *)rxrpc_kernel_get_peer_data(peer); + + if (!server) + return NULL; + return afs_use_server(server, false, afs_server_trace_use_cm_call); +} + +/* + * Look up a server by its UUID and mark it active. The caller must hold + * cell->fs_lock. + */ +static struct afs_server *afs_find_server_by_uuid(struct afs_cell *cell, const uuid_t *uuid) +{ + struct afs_server *server; + struct rb_node *p; + int diff; + + _enter("%pU", uuid); + + p = cell->fs_servers.rb_node; + while (p) { + server = rb_entry(p, struct afs_server, uuid_rb); + + diff = memcmp(uuid, &server->uuid, sizeof(*uuid)); + if (diff < 0) { + p = p->rb_left; + } else if (diff > 0) { + p = p->rb_right; + } else { + if (test_bit(AFS_SERVER_FL_UNCREATED, &server->flags)) + return NULL; /* Need a write lock */ + afs_use_server(server, true, afs_server_trace_use_by_uuid); + return server; + } + } + + return NULL; +} + +/* + * Install a server record in the cell tree. The caller must hold an exclusive + * lock on cell->fs_lock. + */ +static struct afs_server *afs_install_server(struct afs_cell *cell, + struct afs_server **candidate) +{ + struct afs_server *server; + struct afs_net *net = cell->net; + struct rb_node **pp, *p; + int diff; + + _enter("%p", candidate); + + /* Firstly install the server in the UUID lookup tree */ + pp = &cell->fs_servers.rb_node; + p = NULL; + while (*pp) { + p = *pp; + _debug("- consider %p", p); + server = rb_entry(p, struct afs_server, uuid_rb); + diff = memcmp(&(*candidate)->uuid, &server->uuid, sizeof(uuid_t)); + if (diff < 0) + pp = &(*pp)->rb_left; + else if (diff > 0) + pp = &(*pp)->rb_right; + else + goto exists; + } + + server = *candidate; + *candidate = NULL; + rb_link_node(&server->uuid_rb, p, pp); + rb_insert_color(&server->uuid_rb, &cell->fs_servers); + write_seqlock(&net->fs_lock); + hlist_add_head_rcu(&server->proc_link, &net->fs_proc); + write_sequnlock(&net->fs_lock); + + afs_get_cell(cell, afs_cell_trace_get_server); + +exists: + afs_use_server(server, true, afs_server_trace_use_install); + return server; +} + +/* + * Allocate a new server record and mark it as active but uncreated. + */ +static struct afs_server *afs_alloc_server(struct afs_cell *cell, const uuid_t *uuid) +{ + struct afs_server *server; + struct afs_net *net = cell->net; + + _enter(""); + + server = kzalloc_obj(struct afs_server); + if (!server) + return NULL; + + refcount_set(&server->ref, 1); + atomic_set(&server->active, 0); + __set_bit(AFS_SERVER_FL_UNCREATED, &server->flags); + server->debug_id = atomic_inc_return(&afs_server_debug_id); + server->uuid = *uuid; + rwlock_init(&server->fs_lock); + INIT_WORK(&server->destroyer, &afs_server_destroyer); + timer_setup(&server->timer, afs_server_timer, 0); + INIT_LIST_HEAD(&server->volumes); + init_waitqueue_head(&server->probe_wq); + mutex_init(&server->cm_token_lock); + INIT_LIST_HEAD(&server->probe_link); + INIT_HLIST_NODE(&server->proc_link); + spin_lock_init(&server->probe_lock); + server->cell = cell; + server->rtt = UINT_MAX; + server->service_id = FS_SERVICE; + server->probe_counter = 1; + server->probed_at = jiffies - LONG_MAX / 2; + + afs_inc_servers_outstanding(net); + _leave(" = %p", server); + return server; +} + +/* + * Look up an address record for a server + */ +static struct afs_addr_list *afs_vl_lookup_addrs(struct afs_server *server, + struct key *key) +{ + struct afs_vl_cursor vc; + struct afs_addr_list *alist = NULL; + int ret; + + ret = -ERESTARTSYS; + if (afs_begin_vlserver_operation(&vc, server->cell, key)) { + while (afs_select_vlserver(&vc)) { + if (test_bit(AFS_VLSERVER_FL_IS_YFS, &vc.server->flags)) + alist = afs_yfsvl_get_endpoints(&vc, &server->uuid); + else + alist = afs_vl_get_addrs_u(&vc, &server->uuid); + } + + ret = afs_end_vlserver_operation(&vc); + } + + return ret < 0 ? ERR_PTR(ret) : alist; +} + +/* + * Get or create a fileserver record and return it with an active-use count on + * it. + */ +struct afs_server *afs_lookup_server(struct afs_cell *cell, struct key *key, + const uuid_t *uuid, u32 addr_version) +{ + struct afs_addr_list *alist = NULL; + struct afs_server *server, *candidate = NULL; + bool creating = false; + int ret; + + _enter("%p,%pU", cell->net, uuid); + + down_read(&cell->fs_lock); + server = afs_find_server_by_uuid(cell, uuid); + /* Won't see servers marked uncreated. */ + up_read(&cell->fs_lock); + + if (server) { + timer_delete_sync(&server->timer); + if (test_bit(AFS_SERVER_FL_CREATING, &server->flags)) + goto wait_for_creation; + if (server->addr_version != addr_version) + set_bit(AFS_SERVER_FL_NEEDS_UPDATE, &server->flags); + return server; + } + + candidate = afs_alloc_server(cell, uuid); + if (!candidate) { + afs_put_addrlist(alist, afs_alist_trace_put_server_oom); + return ERR_PTR(-ENOMEM); + } + + down_write(&cell->fs_lock); + server = afs_install_server(cell, &candidate); + if (test_bit(AFS_SERVER_FL_CREATING, &server->flags)) { + /* We need to wait for creation to complete. */ + up_write(&cell->fs_lock); + goto wait_for_creation; + } + if (test_bit(AFS_SERVER_FL_UNCREATED, &server->flags)) { + set_bit(AFS_SERVER_FL_CREATING, &server->flags); + clear_bit(AFS_SERVER_FL_UNCREATED, &server->flags); + creating = true; + } + up_write(&cell->fs_lock); + timer_delete_sync(&server->timer); + + /* If we get to create the server, we look up the addresses and then + * immediately dispatch an asynchronous probe to each interface on the + * fileserver. This will make sure the repeat-probing service is + * started. + */ + if (creating) { + alist = afs_vl_lookup_addrs(server, key); + if (IS_ERR(alist)) { + ret = PTR_ERR(alist); + goto create_failed; + } + + ret = afs_fs_probe_fileserver(cell->net, server, alist, key); + if (ret) + goto create_failed; + + clear_and_wake_up_bit(AFS_SERVER_FL_CREATING, &server->flags); + } + +out: + afs_put_addrlist(alist, afs_alist_trace_put_server_create); + if (candidate) { + kfree(rcu_access_pointer(server->endpoint_state)); + kfree(candidate); + afs_dec_servers_outstanding(cell->net); + } + return server ?: ERR_PTR(ret); + +wait_for_creation: + afs_see_server(server, afs_server_trace_wait_create); + wait_on_bit(&server->flags, AFS_SERVER_FL_CREATING, TASK_UNINTERRUPTIBLE); + if (test_bit_acquire(AFS_SERVER_FL_UNCREATED, &server->flags)) { + /* Barrier: read flag before error */ + ret = READ_ONCE(server->create_error); + afs_put_server(cell->net, server, afs_server_trace_unuse_create_fail); + server = NULL; + goto out; + } + + ret = 0; + goto out; + +create_failed: + down_write(&cell->fs_lock); + + WRITE_ONCE(server->create_error, ret); + smp_wmb(); /* Barrier: set error before flag. */ + set_bit(AFS_SERVER_FL_UNCREATED, &server->flags); + + clear_and_wake_up_bit(AFS_SERVER_FL_CREATING, &server->flags); + + if (test_bit(AFS_SERVER_FL_UNCREATED, &server->flags)) { + clear_bit(AFS_SERVER_FL_UNCREATED, &server->flags); + creating = true; + } + afs_unuse_server(cell->net, server, afs_server_trace_unuse_create_fail); + server = NULL; + + up_write(&cell->fs_lock); + goto out; +} + +/* + * Set/reduce a server's timer. + */ +static void afs_set_server_timer(struct afs_server *server, unsigned int delay_secs) +{ + mod_timer(&server->timer, jiffies + delay_secs * HZ); +} + +/* + * Get a reference on a server object. + */ +struct afs_server *afs_get_server(struct afs_server *server, + enum afs_server_trace reason) +{ + unsigned int a; + int r; + + __refcount_inc(&server->ref, &r); + a = atomic_read(&server->active); + trace_afs_server(server->debug_id, r + 1, a, reason); + return server; +} + +/* + * Get an active count on a server object and maybe remove from the inactive + * list. + */ +struct afs_server *afs_use_server(struct afs_server *server, bool activate, + enum afs_server_trace reason) +{ + unsigned int a; + int r; + + __refcount_inc(&server->ref, &r); + a = atomic_inc_return(&server->active); + if (a == 1 && activate && + !test_bit(AFS_SERVER_FL_EXPIRED, &server->flags)) + timer_delete(&server->timer); + + trace_afs_server(server->debug_id, r + 1, a, reason); + return server; +} + +/* + * Release a reference on a server record. + */ +void afs_put_server(struct afs_net *net, struct afs_server *server, + enum afs_server_trace reason) +{ + unsigned int a, debug_id; + bool zero; + int r; + + if (!server) + return; + + debug_id = server->debug_id; + a = atomic_read(&server->active); + zero = __refcount_dec_and_test(&server->ref, &r); + trace_afs_server(debug_id, r - 1, a, reason); + if (unlikely(zero)) + __afs_put_server(net, server); +} + +/* + * Drop an active count on a server object without updating the last-unused + * time. + */ +void afs_unuse_server_notime(struct afs_net *net, struct afs_server *server, + enum afs_server_trace reason) +{ + if (!server) + return; + + if (atomic_dec_and_test(&server->active)) { + if (test_bit(AFS_SERVER_FL_EXPIRED, &server->flags) || + READ_ONCE(server->cell->state) >= AFS_CELL_REMOVING) + schedule_work(&server->destroyer); + } + + afs_put_server(net, server, reason); +} + +/* + * Drop an active count on a server object. + */ +void afs_unuse_server(struct afs_net *net, struct afs_server *server, + enum afs_server_trace reason) +{ + if (!server) + return; + + if (atomic_dec_and_test(&server->active)) { + if (!test_bit(AFS_SERVER_FL_EXPIRED, &server->flags) && + READ_ONCE(server->cell->state) < AFS_CELL_REMOVING) { + time64_t unuse_time = ktime_get_real_seconds(); + + server->unuse_time = unuse_time; + afs_set_server_timer(server, afs_server_gc_delay); + } else { + schedule_work(&server->destroyer); + } + } + + afs_put_server(net, server, reason); +} + +static void afs_server_rcu(struct rcu_head *rcu) +{ + struct afs_server *server = container_of(rcu, struct afs_server, rcu); + + trace_afs_server(server->debug_id, refcount_read(&server->ref), + atomic_read(&server->active), afs_server_trace_free); + afs_put_endpoint_state(rcu_access_pointer(server->endpoint_state), + afs_estate_trace_put_server); + afs_put_cell(server->cell, afs_cell_trace_put_server); + kfree(server->cm_rxgk_appdata.data); + kfree(server); +} + +static void __afs_put_server(struct afs_net *net, struct afs_server *server) +{ + call_rcu(&server->rcu, afs_server_rcu); + afs_dec_servers_outstanding(net); +} + +static void afs_give_up_callbacks(struct afs_net *net, struct afs_server *server) +{ + struct afs_endpoint_state *estate = rcu_access_pointer(server->endpoint_state); + struct afs_addr_list *alist = estate->addresses; + + afs_fs_give_up_all_callbacks(net, server, &alist->addrs[alist->preferred], NULL); +} + +/* + * Check to see if the server record has expired. + */ +static bool afs_has_server_expired(const struct afs_server *server) +{ + time64_t expires_at; + + if (atomic_read(&server->active)) + return false; + + if (server->cell->net->live || + server->cell->state >= AFS_CELL_REMOVING) { + trace_afs_server(server->debug_id, refcount_read(&server->ref), + 0, afs_server_trace_purging); + return true; + } + + expires_at = server->unuse_time; + if (!test_bit(AFS_SERVER_FL_VL_FAIL, &server->flags) && + !test_bit(AFS_SERVER_FL_NOT_FOUND, &server->flags)) + expires_at += afs_server_gc_delay; + + return ktime_get_real_seconds() > expires_at; +} + +/* + * Remove a server record from it's parent cell's database. + */ +static bool afs_remove_server_from_cell(struct afs_server *server) +{ + struct afs_cell *cell = server->cell; + + down_write(&cell->fs_lock); + + if (!afs_has_server_expired(server)) { + up_write(&cell->fs_lock); + return false; + } + + set_bit(AFS_SERVER_FL_EXPIRED, &server->flags); + _debug("expire %pU %u", &server->uuid, atomic_read(&server->active)); + afs_see_server(server, afs_server_trace_see_expired); + rb_erase(&server->uuid_rb, &cell->fs_servers); + up_write(&cell->fs_lock); + return true; +} + +static void afs_server_destroyer(struct work_struct *work) +{ + struct afs_endpoint_state *estate; + struct afs_server *server = container_of(work, struct afs_server, destroyer); + struct afs_net *net = server->cell->net; + + afs_see_server(server, afs_server_trace_see_destroyer); + + if (test_bit(AFS_SERVER_FL_EXPIRED, &server->flags)) + return; + + if (!afs_remove_server_from_cell(server)) + return; + + timer_shutdown_sync(&server->timer); + cancel_work(&server->destroyer); + + if (test_bit(AFS_SERVER_FL_MAY_HAVE_CB, &server->flags)) + afs_give_up_callbacks(net, server); + + /* Unbind the rxrpc_peer records from the server. */ + estate = rcu_access_pointer(server->endpoint_state); + if (estate) + afs_set_peer_appdata(server, estate->addresses, NULL); + + write_seqlock(&net->fs_lock); + list_del_init(&server->probe_link); + if (!hlist_unhashed(&server->proc_link)) + hlist_del_rcu(&server->proc_link); + write_sequnlock(&net->fs_lock); + + afs_put_server(net, server, afs_server_trace_destroy); +} + +static void afs_server_timer(struct timer_list *timer) +{ + struct afs_server *server = container_of(timer, struct afs_server, timer); + + afs_see_server(server, afs_server_trace_see_timer); + if (!test_bit(AFS_SERVER_FL_EXPIRED, &server->flags)) + schedule_work(&server->destroyer); +} + +/* + * Wake up all the servers in a cell so that they can purge themselves. + */ +void afs_purge_servers(struct afs_cell *cell) +{ + struct afs_server *server; + struct rb_node *rb; + + down_read(&cell->fs_lock); + for (rb = rb_first(&cell->fs_servers); rb; rb = rb_next(rb)) { + server = rb_entry(rb, struct afs_server, uuid_rb); + afs_see_server(server, afs_server_trace_see_purge); + schedule_work(&server->destroyer); + } + up_read(&cell->fs_lock); +} + +/* + * Wait for outstanding servers. + */ +void afs_wait_for_servers(struct afs_net *net) +{ + _enter(""); + + atomic_dec(&net->servers_outstanding); + wait_var_event(&net->servers_outstanding, + !atomic_read(&net->servers_outstanding)); + _leave(""); +} + +/* + * Get an update for a server's address list. + */ +static noinline bool afs_update_server_record(struct afs_operation *op, + struct afs_server *server, + struct key *key) +{ + struct afs_endpoint_state *estate; + struct afs_addr_list *alist; + bool has_addrs; + + _enter(""); + + trace_afs_server(server->debug_id, refcount_read(&server->ref), + atomic_read(&server->active), + afs_server_trace_update); + + alist = afs_vl_lookup_addrs(server, op->key); + if (IS_ERR(alist)) { + rcu_read_lock(); + estate = rcu_dereference(server->endpoint_state); + has_addrs = estate->addresses; + rcu_read_unlock(); + + if ((PTR_ERR(alist) == -ERESTARTSYS || + PTR_ERR(alist) == -EINTR) && + (op->flags & AFS_OPERATION_UNINTR) && + has_addrs) { + _leave(" = t [intr]"); + return true; + } + afs_op_set_error(op, PTR_ERR(alist)); + _leave(" = f [%d]", afs_op_error(op)); + return false; + } + + if (server->addr_version != alist->version) + afs_fs_probe_fileserver(op->net, server, alist, key); + + afs_put_addrlist(alist, afs_alist_trace_put_server_update); + _leave(" = t"); + return true; +} + +/* + * See if a server's address list needs updating. + */ +bool afs_check_server_record(struct afs_operation *op, struct afs_server *server, + struct key *key) +{ + bool success; + int ret, retries = 0; + + _enter(""); + + ASSERT(server); + +retry: + if (test_bit(AFS_SERVER_FL_UPDATING, &server->flags)) + goto wait; + if (test_bit(AFS_SERVER_FL_NEEDS_UPDATE, &server->flags)) + goto update; + _leave(" = t [good]"); + return true; + +update: + if (!test_and_set_bit_lock(AFS_SERVER_FL_UPDATING, &server->flags)) { + clear_bit(AFS_SERVER_FL_NEEDS_UPDATE, &server->flags); + success = afs_update_server_record(op, server, key); + clear_bit_unlock(AFS_SERVER_FL_UPDATING, &server->flags); + wake_up_bit(&server->flags, AFS_SERVER_FL_UPDATING); + _leave(" = %d", success); + return success; + } + +wait: + ret = wait_on_bit(&server->flags, AFS_SERVER_FL_UPDATING, + (op->flags & AFS_OPERATION_UNINTR) ? + TASK_UNINTERRUPTIBLE : TASK_INTERRUPTIBLE); + if (ret == -ERESTARTSYS) { + afs_op_set_error(op, ret); + _leave(" = f [intr]"); + return false; + } + + retries++; + if (retries == 4) { + _leave(" = f [stale]"); + ret = -ESTALE; + return false; + } + goto retry; +} diff --git a/fs/afs/server_list.c b/fs/afs/server_list.c new file mode 100644 index 000000000..4178d39b2 --- /dev/null +++ b/fs/afs/server_list.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* AFS fileserver list management. + * + * Copyright (C) 2017 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include "internal.h" + +void afs_put_serverlist(struct afs_net *net, struct afs_server_list *slist) +{ + int i; + + if (slist && refcount_dec_and_test(&slist->usage)) { + for (i = 0; i < slist->nr_servers; i++) + afs_unuse_server(net, slist->servers[i].server, + afs_server_trace_unuse_slist); + kfree_rcu(slist, rcu); + } +} + +/* + * Build a server list from a VLDB record. + */ +struct afs_server_list *afs_alloc_server_list(struct afs_volume *volume, + struct key *key, + struct afs_vldb_entry *vldb) +{ + struct afs_server_list *slist; + struct afs_server *server; + unsigned int type_mask = 1 << volume->type; + bool use_newrepsites = false; + int ret = -ENOMEM, nr_servers = 0, newrep = 0, i, j, usable = 0; + + /* Work out if we're going to restrict to NEWREPSITE-marked servers or + * not. If at least one site is marked as NEWREPSITE, then it's likely + * that "vos release" is busy updating RO sites. We cut over from one + * to the other when >=50% of the sites have been updated. Sites that + * are in the process of being updated are marked DONTUSE. + */ + for (i = 0; i < vldb->nr_servers; i++) { + if (!(vldb->fs_mask[i] & type_mask)) + continue; + nr_servers++; + if (vldb->vlsf_flags[i] & AFS_VLSF_DONTUSE) + continue; + usable++; + if (vldb->vlsf_flags[i] & AFS_VLSF_NEWREPSITE) + newrep++; + } + + slist = kzalloc_flex(*slist, servers, nr_servers); + if (!slist) + goto error; + + if (newrep) { + if (newrep < usable / 2) { + slist->ro_replicating = AFS_RO_REPLICATING_USE_OLD; + } else { + slist->ro_replicating = AFS_RO_REPLICATING_USE_NEW; + use_newrepsites = true; + } + } + + refcount_set(&slist->usage, 1); + rwlock_init(&slist->lock); + + /* Make sure a records exists for each server in the list. */ + for (i = 0; i < vldb->nr_servers; i++) { + unsigned long se_flags = 0; + bool newrepsite = vldb->vlsf_flags[i] & AFS_VLSF_NEWREPSITE; + + if (!(vldb->fs_mask[i] & type_mask)) + continue; + if (vldb->vlsf_flags[i] & AFS_VLSF_DONTUSE) + __set_bit(AFS_SE_EXCLUDED, &se_flags); + if (newrep && (newrepsite ^ use_newrepsites)) + __set_bit(AFS_SE_EXCLUDED, &se_flags); + + server = afs_lookup_server(volume->cell, key, &vldb->fs_server[i], + vldb->addr_version[i]); + if (IS_ERR(server)) { + ret = PTR_ERR(server); + if (ret == -ENOENT || + ret == -ENOMEDIUM) + continue; + goto error_2; + } + + /* Insertion-sort by UUID */ + for (j = 0; j < slist->nr_servers; j++) + if (memcmp(&slist->servers[j].server->uuid, + &server->uuid, + sizeof(server->uuid)) >= 0) + break; + if (j < slist->nr_servers) { + if (slist->servers[j].server == server) { + afs_unuse_server_notime(volume->cell->net, server, + afs_server_trace_unuse_slist_isort); + continue; + } + + memmove(slist->servers + j + 1, + slist->servers + j, + (slist->nr_servers - j) * sizeof(struct afs_server_entry)); + } + + slist->servers[j].server = server; + slist->servers[j].volume = volume; + slist->servers[j].flags = se_flags; + slist->servers[j].cb_expires_at = AFS_NO_CB_PROMISE; + slist->nr_servers++; + } + + if (slist->nr_servers == 0) { + ret = -EDESTADDRREQ; + goto error_2; + } + + return slist; + +error_2: + afs_put_serverlist(volume->cell->net, slist); +error: + return ERR_PTR(ret); +} + +/* + * Copy the annotations from an old server list to its potential replacement. + */ +bool afs_annotate_server_list(struct afs_server_list *new, + struct afs_server_list *old) +{ + unsigned long mask = 1UL << AFS_SE_EXCLUDED; + int i; + + if (old->nr_servers != new->nr_servers || + old->ro_replicating != new->ro_replicating) + goto changed; + + for (i = 0; i < old->nr_servers; i++) { + if (old->servers[i].server != new->servers[i].server) + goto changed; + if ((old->servers[i].flags & mask) != (new->servers[i].flags & mask)) + goto changed; + } + return false; +changed: + return true; +} + +/* + * Attach a volume to the servers it is going to use. + */ +void afs_attach_volume_to_servers(struct afs_volume *volume, struct afs_server_list *slist) +{ + struct afs_server_entry *se, *pe; + struct afs_server *server; + struct list_head *p; + unsigned int i; + + down_write(&volume->cell->vs_lock); + + for (i = 0; i < slist->nr_servers; i++) { + se = &slist->servers[i]; + server = se->server; + + list_for_each(p, &server->volumes) { + pe = list_entry(p, struct afs_server_entry, slink); + if (volume->vid <= pe->volume->vid) + break; + } + list_add_tail(&se->slink, p); + } + + slist->attached = true; + up_write(&volume->cell->vs_lock); +} + +/* + * Reattach a volume to the servers it is going to use when server list is + * replaced. We try to switch the attachment points to avoid rewalking the + * lists. + */ +void afs_reattach_volume_to_servers(struct afs_volume *volume, struct afs_server_list *new, + struct afs_server_list *old) +{ + unsigned int n = 0, o = 0; + + down_write(&volume->cell->vs_lock); + + while (n < new->nr_servers || o < old->nr_servers) { + struct afs_server_entry *pn = n < new->nr_servers ? &new->servers[n] : NULL; + struct afs_server_entry *po = o < old->nr_servers ? &old->servers[o] : NULL; + struct afs_server_entry *s; + struct list_head *p; + int diff; + + if (pn && po && pn->server == po->server) { + pn->cb_expires_at = po->cb_expires_at; + list_replace(&po->slink, &pn->slink); + n++; + o++; + continue; + } + + if (pn && po) + diff = memcmp(&pn->server->uuid, &po->server->uuid, + sizeof(pn->server->uuid)); + else + diff = pn ? -1 : 1; + + if (diff < 0) { + list_for_each(p, &pn->server->volumes) { + s = list_entry(p, struct afs_server_entry, slink); + if (volume->vid <= s->volume->vid) + break; + } + list_add_tail(&pn->slink, p); + n++; + } else { + list_del(&po->slink); + o++; + } + } + + up_write(&volume->cell->vs_lock); +} + +/* + * Detach a volume from the servers it has been using. + */ +void afs_detach_volume_from_servers(struct afs_volume *volume, struct afs_server_list *slist) +{ + unsigned int i; + + if (!slist->attached) + return; + + down_write(&volume->cell->vs_lock); + + for (i = 0; i < slist->nr_servers; i++) + list_del(&slist->servers[i].slink); + + slist->attached = false; + up_write(&volume->cell->vs_lock); +} diff --git a/fs/afs/super.c b/fs/afs/super.c new file mode 100644 index 000000000..82bb71382 --- /dev/null +++ b/fs/afs/super.c @@ -0,0 +1,771 @@ +/* AFS superblock handling + * + * Copyright (c) 2002, 2007, 2018 Red Hat, Inc. All rights reserved. + * + * This software may be freely redistributed under the terms of the + * GNU General Public License. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + * Authors: David Howells + * David Woodhouse + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +static void afs_i_init_once(void *foo); +static void afs_kill_super(struct super_block *sb); +static struct inode *afs_alloc_inode(struct super_block *sb); +static void afs_destroy_inode(struct inode *inode); +static void afs_free_inode(struct inode *inode); +static int afs_statfs(struct dentry *dentry, struct kstatfs *buf); +static int afs_show_devname(struct seq_file *m, struct dentry *root); +static int afs_show_options(struct seq_file *m, struct dentry *root); +static int afs_init_fs_context(struct fs_context *fc); +static const struct fs_parameter_spec afs_fs_parameters[]; + +struct file_system_type afs_fs_type = { + .owner = THIS_MODULE, + .name = "afs", + .init_fs_context = afs_init_fs_context, + .parameters = afs_fs_parameters, + .kill_sb = afs_kill_super, + .fs_flags = FS_RENAME_DOES_D_MOVE, +}; +MODULE_ALIAS_FS("afs"); + +int afs_net_id; + +static const struct super_operations afs_super_ops = { + .statfs = afs_statfs, + .alloc_inode = afs_alloc_inode, + .write_inode = netfs_unpin_writeback, + .drop_inode = afs_drop_inode, + .destroy_inode = afs_destroy_inode, + .free_inode = afs_free_inode, + .evict_inode = afs_evict_inode, + .show_devname = afs_show_devname, + .show_options = afs_show_options, +}; + +static struct kmem_cache *afs_inode_cachep; +static atomic_t afs_count_active_inodes; + +enum afs_param { + Opt_autocell, + Opt_dyn, + Opt_flock, + Opt_source, +}; + +static const struct constant_table afs_param_flock[] = { + {"local", afs_flock_mode_local }, + {"openafs", afs_flock_mode_openafs }, + {"strict", afs_flock_mode_strict }, + {"write", afs_flock_mode_write }, + {} +}; + +static const struct fs_parameter_spec afs_fs_parameters[] = { + fsparam_flag ("autocell", Opt_autocell), + fsparam_flag ("dyn", Opt_dyn), + fsparam_enum ("flock", Opt_flock, afs_param_flock), + fsparam_string("source", Opt_source), + {} +}; + +/* + * initialise the filesystem + */ +int __init afs_fs_init(void) +{ + int ret; + + _enter(""); + + /* create ourselves an inode cache */ + atomic_set(&afs_count_active_inodes, 0); + + ret = -ENOMEM; + afs_inode_cachep = kmem_cache_create("afs_inode_cache", + sizeof(struct afs_vnode), + 0, + SLAB_HWCACHE_ALIGN|SLAB_ACCOUNT, + afs_i_init_once); + if (!afs_inode_cachep) { + printk(KERN_NOTICE "kAFS: Failed to allocate inode cache\n"); + return ret; + } + + /* now export our filesystem to lesser mortals */ + ret = register_filesystem(&afs_fs_type); + if (ret < 0) { + kmem_cache_destroy(afs_inode_cachep); + _leave(" = %d", ret); + return ret; + } + + _leave(" = 0"); + return 0; +} + +/* + * clean up the filesystem + */ +void afs_fs_exit(void) +{ + _enter(""); + + afs_mntpt_kill_timer(); + unregister_filesystem(&afs_fs_type); + + if (atomic_read(&afs_count_active_inodes) != 0) { + printk("kAFS: %d active inode objects still present\n", + atomic_read(&afs_count_active_inodes)); + BUG(); + } + + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(afs_inode_cachep); + _leave(""); +} + +/* + * Display the mount device name in /proc/mounts. + */ +static int afs_show_devname(struct seq_file *m, struct dentry *root) +{ + struct afs_super_info *as = AFS_FS_S(root->d_sb); + struct afs_volume *volume = as->volume; + struct afs_cell *cell = as->cell; + const char *suf = ""; + char pref = '%'; + + if (as->dyn_root) { + seq_puts(m, "none"); + return 0; + } + + switch (volume->type) { + case AFSVL_RWVOL: + break; + case AFSVL_ROVOL: + pref = '#'; + if (volume->type_force) + suf = ".readonly"; + break; + case AFSVL_BACKVOL: + pref = '#'; + suf = ".backup"; + break; + } + + seq_printf(m, "%c%s:%s%s", pref, cell->name, volume->name, suf); + return 0; +} + +/* + * Display the mount options in /proc/mounts. + */ +static int afs_show_options(struct seq_file *m, struct dentry *root) +{ + struct afs_super_info *as = AFS_FS_S(root->d_sb); + const char *p = NULL; + + if (as->dyn_root) + seq_puts(m, ",dyn"); + switch (as->flock_mode) { + case afs_flock_mode_unset: break; + case afs_flock_mode_local: p = "local"; break; + case afs_flock_mode_openafs: p = "openafs"; break; + case afs_flock_mode_strict: p = "strict"; break; + case afs_flock_mode_write: p = "write"; break; + } + if (p) + seq_printf(m, ",flock=%s", p); + + return 0; +} + +/* + * Parse the source name to get cell name, volume name, volume type and R/W + * selector. + * + * This can be one of the following: + * "%[cell:]volume[.]" R/W volume + * "#[cell:]volume[.]" R/O or R/W volume (R/O parent), + * or R/W (R/W parent) volume + * "%[cell:]volume.readonly" R/O volume + * "#[cell:]volume.readonly" R/O volume + * "%[cell:]volume.backup" Backup volume + * "#[cell:]volume.backup" Backup volume + */ +static int afs_parse_source(struct fs_context *fc, struct fs_parameter *param) +{ + struct afs_fs_context *ctx = fc->fs_private; + struct afs_cell *cell; + const char *cellname, *suffix, *name = param->string; + int cellnamesz; + + _enter(",%s", name); + + if (fc->source) + return invalf(fc, "kAFS: Multiple sources not supported"); + + if (!name) { + printk(KERN_ERR "kAFS: no volume name specified\n"); + return -EINVAL; + } + + if ((name[0] != '%' && name[0] != '#') || !name[1]) { + /* To use dynroot, we don't want to have to provide a source */ + if (strcmp(name, "none") == 0) { + ctx->no_cell = true; + return 0; + } + printk(KERN_ERR "kAFS: unparsable volume name\n"); + return -EINVAL; + } + + /* determine the type of volume we're looking for */ + if (name[0] == '%') { + ctx->type = AFSVL_RWVOL; + ctx->force = true; + } + name++; + + /* split the cell name out if there is one */ + ctx->volname = strchr(name, ':'); + if (ctx->volname) { + cellname = name; + cellnamesz = ctx->volname - name; + ctx->volname++; + } else { + ctx->volname = name; + cellname = NULL; + cellnamesz = 0; + } + + /* the volume type is further affected by a possible suffix */ + suffix = strrchr(ctx->volname, '.'); + if (suffix) { + if (strcmp(suffix, ".readonly") == 0) { + ctx->type = AFSVL_ROVOL; + ctx->force = true; + } else if (strcmp(suffix, ".backup") == 0) { + ctx->type = AFSVL_BACKVOL; + ctx->force = true; + } else if (suffix[1] == 0) { + } else { + suffix = NULL; + } + } + + ctx->volnamesz = suffix ? + suffix - ctx->volname : strlen(ctx->volname); + + _debug("cell %*.*s [%p]", + cellnamesz, cellnamesz, cellname ?: "", ctx->cell); + + /* lookup the cell record */ + if (cellname) { + cell = afs_lookup_cell(ctx->net, cellname, cellnamesz, + NULL, AFS_LOOKUP_CELL_DIRECT_MOUNT, + afs_cell_trace_use_lookup_mount); + if (IS_ERR(cell)) { + pr_err("kAFS: unable to lookup cell '%*.*s'\n", + cellnamesz, cellnamesz, cellname ?: ""); + return PTR_ERR(cell); + } + afs_unuse_cell(ctx->cell, afs_cell_trace_unuse_parse); + afs_see_cell(cell, afs_cell_trace_see_source); + ctx->cell = cell; + } + + _debug("CELL:%s [%p] VOLUME:%*.*s SUFFIX:%s TYPE:%d%s", + ctx->cell->name, ctx->cell, + ctx->volnamesz, ctx->volnamesz, ctx->volname, + suffix ?: "-", ctx->type, ctx->force ? " FORCE" : ""); + + fc->source = param->string; + param->string = NULL; + return 0; +} + +/* + * Parse a single mount parameter. + */ +static int afs_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct fs_parse_result result; + struct afs_fs_context *ctx = fc->fs_private; + int opt; + + opt = fs_parse(fc, afs_fs_parameters, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_source: + return afs_parse_source(fc, param); + + case Opt_autocell: + ctx->autocell = true; + break; + + case Opt_dyn: + ctx->dyn_root = true; + break; + + case Opt_flock: + ctx->flock_mode = result.uint_32; + break; + + default: + return -EINVAL; + } + + _leave(" = 0"); + return 0; +} + +/* + * Validate the options, get the cell key and look up the volume. + */ +static int afs_validate_fc(struct fs_context *fc) +{ + struct afs_fs_context *ctx = fc->fs_private; + struct afs_volume *volume; + struct afs_cell *cell; + struct key *key; + int ret; + + if (!ctx->dyn_root) { + if (ctx->no_cell) { + pr_warn("kAFS: Can only specify source 'none' with -o dyn\n"); + return -EINVAL; + } + + if (!ctx->cell) { + pr_warn("kAFS: No cell specified\n"); + return -EDESTADDRREQ; + } + + reget_key: + /* We try to do the mount securely. */ + key = afs_request_key(ctx->cell); + if (IS_ERR(key)) + return PTR_ERR(key); + + ctx->key = key; + + if (ctx->volume) { + afs_put_volume(ctx->volume, afs_volume_trace_put_validate_fc); + ctx->volume = NULL; + } + + if (test_bit(AFS_CELL_FL_CHECK_ALIAS, &ctx->cell->flags)) { + ret = afs_cell_detect_alias(ctx->cell, key); + if (ret < 0) + return ret; + if (ret == 1) { + _debug("switch to alias"); + key_put(ctx->key); + ctx->key = NULL; + cell = afs_use_cell(ctx->cell->alias_of, + afs_cell_trace_use_fc_alias); + afs_unuse_cell(ctx->cell, afs_cell_trace_unuse_fc); + ctx->cell = cell; + goto reget_key; + } + } + + volume = afs_create_volume(ctx); + if (IS_ERR(volume)) + return PTR_ERR(volume); + + ctx->volume = volume; + if (volume->type != AFSVL_RWVOL) { + ctx->flock_mode = afs_flock_mode_local; + fc->sb_flags |= SB_RDONLY; + } + } + + return 0; +} + +/* + * check a superblock to see if it's the one we're looking for + */ +static int afs_test_super(struct super_block *sb, struct fs_context *fc) +{ + struct afs_fs_context *ctx = fc->fs_private; + struct afs_super_info *as = AFS_FS_S(sb); + + return (as->net_ns == fc->net_ns && + as->volume && + as->volume->vid == ctx->volume->vid && + as->cell == ctx->cell && + !as->dyn_root); +} + +static int afs_dynroot_test_super(struct super_block *sb, struct fs_context *fc) +{ + struct afs_super_info *as = AFS_FS_S(sb); + + return (as->net_ns == fc->net_ns && + as->dyn_root); +} + +static int afs_set_super(struct super_block *sb, struct fs_context *fc) +{ + return set_anon_super(sb, NULL); +} + +/* + * fill in the superblock + */ +static int afs_fill_super(struct super_block *sb, struct afs_fs_context *ctx) +{ + struct afs_super_info *as = AFS_FS_S(sb); + struct inode *inode = NULL; + int ret; + + _enter(""); + + /* fill in the superblock */ + sb->s_blocksize = PAGE_SIZE; + sb->s_blocksize_bits = PAGE_SHIFT; + sb->s_maxbytes = MAX_LFS_FILESIZE; + sb->s_magic = AFS_FS_MAGIC; + sb->s_op = &afs_super_ops; + if (!as->dyn_root) + sb->s_xattr = afs_xattr_handlers; + ret = super_setup_bdi(sb); + if (ret) + return ret; + + /* allocate the root inode and dentry */ + if (as->dyn_root) { + inode = afs_dynroot_iget_root(sb); + } else { + sprintf(sb->s_id, "%llu", as->volume->vid); + afs_activate_volume(as->volume); + inode = afs_root_iget(sb, ctx->key); + } + + if (IS_ERR(inode)) + return PTR_ERR(inode); + + ret = -ENOMEM; + sb->s_root = d_make_root(inode); + if (!sb->s_root) + goto error; + + if (as->dyn_root) { + set_default_d_op(sb, &afs_dynroot_dentry_operations); + } else { + set_default_d_op(sb, &afs_fs_dentry_operations); + rcu_assign_pointer(as->volume->sb, sb); + } + + _leave(" = 0"); + return 0; + +error: + _leave(" = %d", ret); + return ret; +} + +static struct afs_super_info *afs_alloc_sbi(struct fs_context *fc) +{ + struct afs_fs_context *ctx = fc->fs_private; + struct afs_super_info *as; + + as = kzalloc_obj(struct afs_super_info); + if (as) { + as->net_ns = get_net(fc->net_ns); + as->flock_mode = ctx->flock_mode; + if (ctx->dyn_root) { + as->dyn_root = true; + } else { + as->cell = afs_use_cell(ctx->cell, afs_cell_trace_use_sbi); + as->volume = afs_get_volume(ctx->volume, + afs_volume_trace_get_alloc_sbi); + } + } + return as; +} + +static void afs_destroy_sbi(struct afs_super_info *as) +{ + if (as) { + afs_put_volume(as->volume, afs_volume_trace_put_destroy_sbi); + afs_unuse_cell(as->cell, afs_cell_trace_unuse_sbi); + put_net(as->net_ns); + kfree(as); + } +} + +static void afs_kill_super(struct super_block *sb) +{ + struct afs_super_info *as = AFS_FS_S(sb); + + /* Clear the callback interests (which will do ilookup5) before + * deactivating the superblock. + */ + if (as->volume) + rcu_assign_pointer(as->volume->sb, NULL); + kill_anon_super(sb); + if (as->volume) + afs_deactivate_volume(as->volume); + afs_destroy_sbi(as); +} + +/* + * Get an AFS superblock and root directory. + */ +static int afs_get_tree(struct fs_context *fc) +{ + struct afs_fs_context *ctx = fc->fs_private; + struct super_block *sb; + struct afs_super_info *as; + int ret; + + ret = afs_validate_fc(fc); + if (ret) + goto error; + + _enter(""); + + /* allocate a superblock info record */ + ret = -ENOMEM; + as = afs_alloc_sbi(fc); + if (!as) + goto error; + fc->s_fs_info = as; + + /* allocate a deviceless superblock */ + sb = sget_fc(fc, + as->dyn_root ? afs_dynroot_test_super : afs_test_super, + afs_set_super); + if (IS_ERR(sb)) { + ret = PTR_ERR(sb); + goto error; + } + + if (!sb->s_root) { + /* initial superblock/root creation */ + _debug("create"); + ret = afs_fill_super(sb, ctx); + if (ret < 0) + goto error_sb; + sb->s_flags |= SB_ACTIVE; + } else { + _debug("reuse"); + ASSERTCMP(sb->s_flags, &, SB_ACTIVE); + } + + fc->root = dget(sb->s_root); + if (!ctx->dyn_root) + trace_afs_get_tree(as->cell, as->volume); + _leave(" = 0 [%p]", sb); + return 0; + +error_sb: + deactivate_locked_super(sb); +error: + _leave(" = %d", ret); + return ret; +} + +static void afs_free_fc(struct fs_context *fc) +{ + struct afs_fs_context *ctx = fc->fs_private; + + afs_destroy_sbi(fc->s_fs_info); + afs_put_volume(ctx->volume, afs_volume_trace_put_free_fc); + afs_unuse_cell(ctx->cell, afs_cell_trace_unuse_fc); + key_put(ctx->key); + kfree(ctx); +} + +static const struct fs_context_operations afs_context_ops = { + .free = afs_free_fc, + .parse_param = afs_parse_param, + .get_tree = afs_get_tree, +}; + +/* + * Set up the filesystem mount context. + */ +static int afs_init_fs_context(struct fs_context *fc) +{ + struct afs_fs_context *ctx; + struct afs_cell *cell; + + ctx = kzalloc_obj(struct afs_fs_context); + if (!ctx) + return -ENOMEM; + + ctx->type = AFSVL_ROVOL; + ctx->net = afs_net(fc->net_ns); + + /* Default to the workstation cell. */ + cell = afs_find_cell(ctx->net, NULL, 0, afs_cell_trace_use_fc); + if (IS_ERR(cell)) + cell = NULL; + ctx->cell = cell; + + fc->fs_private = ctx; + fc->ops = &afs_context_ops; + return 0; +} + +/* + * Initialise an inode cache slab element prior to any use. Note that + * afs_alloc_inode() *must* reset anything that could incorrectly leak from one + * inode to another. + */ +static void afs_i_init_once(void *_vnode) +{ + struct afs_vnode *vnode = _vnode; + + memset(vnode, 0, sizeof(*vnode)); + inode_init_once(&vnode->netfs.inode); + INIT_LIST_HEAD(&vnode->io_lock_waiters); + init_rwsem(&vnode->validate_lock); + spin_lock_init(&vnode->wb_lock); + spin_lock_init(&vnode->lock); + INIT_LIST_HEAD(&vnode->wb_keys); + INIT_LIST_HEAD(&vnode->pending_locks); + INIT_LIST_HEAD(&vnode->granted_locks); + INIT_LIST_HEAD(&vnode->cb_mmap_link); + seqlock_init(&vnode->cb_lock); +} + +/* + * allocate an AFS inode struct from our slab cache + */ +static struct inode *afs_alloc_inode(struct super_block *sb) +{ + struct afs_vnode *vnode; + + vnode = alloc_inode_sb(sb, afs_inode_cachep, GFP_KERNEL); + if (!vnode) + return NULL; + + atomic_inc(&afs_count_active_inodes); + + /* Reset anything that shouldn't leak from one inode to the next. */ + memset(&vnode->fid, 0, sizeof(vnode->fid)); + memset(&vnode->status, 0, sizeof(vnode->status)); + afs_vnode_set_cache(vnode, NULL); + + vnode->volume = NULL; + vnode->lock_key = NULL; + vnode->permit_cache = NULL; + vnode->directory = NULL; + vnode->directory_size = 0; + + vnode->flags = 1 << AFS_VNODE_UNSET; + vnode->lock_state = AFS_VNODE_LOCK_NONE; + + init_rwsem(&vnode->rmdir_lock); + INIT_WORK(&vnode->cb_work, afs_invalidate_mmap_work); + INIT_DELAYED_WORK(&vnode->lock_work, afs_lock_work); + + _leave(" = %p", &vnode->netfs.inode); + return &vnode->netfs.inode; +} + +static void afs_free_inode(struct inode *inode) +{ + kmem_cache_free(afs_inode_cachep, AFS_FS_I(inode)); +} + +/* + * destroy an AFS inode struct + */ +static void afs_destroy_inode(struct inode *inode) +{ + struct afs_vnode *vnode = AFS_FS_I(inode); + + _enter("%p{%llx:%llu}", inode, vnode->fid.vid, vnode->fid.vnode); + + _debug("DESTROY INODE %p", inode); + + atomic_dec(&afs_count_active_inodes); +} + +static void afs_get_volume_status_success(struct afs_operation *op) +{ + struct afs_volume_status *vs = &op->volstatus.vs; + struct kstatfs *buf = op->volstatus.buf; + + if (vs->max_quota == 0) + buf->f_blocks = vs->part_max_blocks; + else + buf->f_blocks = vs->max_quota; + + if (buf->f_blocks > vs->blocks_in_use) + buf->f_bavail = buf->f_bfree = + buf->f_blocks - vs->blocks_in_use; +} + +static const struct afs_operation_ops afs_get_volume_status_operation = { + .issue_afs_rpc = afs_fs_get_volume_status, + .issue_yfs_rpc = yfs_fs_get_volume_status, + .success = afs_get_volume_status_success, +}; + +/* + * return information about an AFS volume + */ +static int afs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct afs_super_info *as = AFS_FS_S(dentry->d_sb); + struct afs_operation *op; + struct afs_vnode *vnode = AFS_FS_I(d_inode(dentry)); + + buf->f_type = dentry->d_sb->s_magic; + buf->f_bsize = AFS_BLOCK_SIZE; + buf->f_namelen = AFSNAMEMAX - 1; + + if (as->dyn_root) { + buf->f_blocks = 1; + buf->f_bavail = 0; + buf->f_bfree = 0; + return 0; + } + + op = afs_alloc_operation(NULL, as->volume); + if (IS_ERR(op)) + return PTR_ERR(op); + + afs_op_set_vnode(op, 0, vnode); + op->nr_files = 1; + op->volstatus.buf = buf; + op->ops = &afs_get_volume_status_operation; + return afs_do_sync_operation(op); +} diff --git a/fs/afs/symlink.c b/fs/afs/symlink.c new file mode 100644 index 000000000..16b4823cb --- /dev/null +++ b/fs/afs/symlink.c @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* AFS filesystem symbolic link handling + * + * Copyright (C) 2026 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include "internal.h" + +static void afs_put_symlink(struct afs_symlink *symlink) +{ + if (refcount_dec_and_test(&symlink->ref)) + kfree_rcu(symlink, rcu); +} + +static void afs_replace_symlink(struct afs_vnode *vnode, struct afs_symlink *symlink) +{ + struct afs_symlink *old; + + old = rcu_replace_pointer(vnode->symlink, symlink, + lockdep_is_held(&vnode->validate_lock)); + if (old) + afs_put_symlink(old); +} + +/* + * In the event that a third-party update of a symlink occurs, dispose of the + * copy of the old contents. Called under ->validate_lock. + */ +void afs_invalidate_symlink(struct afs_vnode *vnode) +{ + afs_replace_symlink(vnode, NULL); +} + +/* + * Dispose of a symlink copy during inode deletion. + */ +void afs_evict_symlink(struct afs_vnode *vnode) +{ + struct afs_symlink *old; + + old = rcu_replace_pointer(vnode->symlink, NULL, true); + if (old) + afs_put_symlink(old); + +} + +/* + * Set up a locally created symlink inode for immediate write to the cache. + */ +void afs_init_new_symlink(struct afs_vnode *vnode, struct afs_operation *op) +{ + struct afs_symlink *symlink = op->create.symlink; + size_t dsize = 0; + size_t size = strlen(symlink->content) + 1; + char *p; + + rcu_assign_pointer(vnode->symlink, symlink); + op->create.symlink = NULL; + + if (!fscache_cookie_enabled(netfs_i_cookie(&vnode->netfs))) + return; + + if (netfs_alloc_folioq_buffer(NULL, &vnode->directory, &dsize, size, + mapping_gfp_mask(vnode->netfs.inode.i_mapping)) < 0) + return; + + vnode->directory_size = dsize; + p = kmap_local_folio(folioq_folio(vnode->directory, 0), 0); + memcpy(p, symlink->content, size); + kunmap_local(p); + netfs_single_mark_inode_dirty(&vnode->netfs.inode); +} + +/* + * Read a symlink in a single download. + */ +static ssize_t afs_do_read_symlink(struct afs_vnode *vnode) +{ + struct afs_symlink *symlink; + struct iov_iter iter; + ssize_t ret; + loff_t i_size; + + i_size = i_size_read(&vnode->netfs.inode); + if (i_size > PAGE_SIZE - 1) { + trace_afs_file_error(vnode, -EFBIG, afs_file_error_dir_big); + return -EFBIG; + } + + if (!vnode->directory) { + size_t cur_size = 0; + + ret = netfs_alloc_folioq_buffer(NULL, + &vnode->directory, &cur_size, PAGE_SIZE, + mapping_gfp_mask(vnode->netfs.inode.i_mapping)); + vnode->directory_size = PAGE_SIZE - 1; + if (ret < 0) + return ret; + } + + iov_iter_folio_queue(&iter, ITER_DEST, vnode->directory, 0, 0, PAGE_SIZE); + + /* AFS requires us to perform the read of a symlink as a single unit to + * avoid issues with the content being changed between reads. + */ + ret = netfs_read_single(&vnode->netfs.inode, NULL, &iter); + if (ret >= 0) { + i_size = ret; + if (i_size > PAGE_SIZE - 1) { + trace_afs_file_error(vnode, -EFBIG, afs_file_error_dir_big); + return -EFBIG; + } + vnode->directory_size = i_size; + + /* Copy the symlink. */ + symlink = kmalloc_flex(struct afs_symlink, content, i_size + 1, + GFP_KERNEL); + if (!symlink) + return -ENOMEM; + + refcount_set(&symlink->ref, 1); + symlink->content[i_size] = 0; + + const char *s = kmap_local_folio(folioq_folio(vnode->directory, 0), 0); + + memcpy(symlink->content, s, i_size); + kunmap_local(s); + + afs_replace_symlink(vnode, symlink); + } + + if (!fscache_cookie_enabled(netfs_i_cookie(&vnode->netfs))) { + netfs_free_folioq_buffer(vnode->directory); + vnode->directory = NULL; + vnode->directory_size = 0; + } + + return ret; +} + +static ssize_t afs_read_symlink(struct afs_vnode *vnode) +{ + ssize_t ret; + + fscache_use_cookie(afs_vnode_cache(vnode), false); + ret = afs_do_read_symlink(vnode); + fscache_unuse_cookie(afs_vnode_cache(vnode), NULL, NULL); + return ret; +} + +static void afs_put_link(void *arg) +{ + afs_put_symlink(arg); +} + +const char *afs_get_link(struct dentry *dentry, struct inode *inode, + struct delayed_call *callback) +{ + struct afs_symlink *symlink; + struct afs_vnode *vnode = AFS_FS_I(inode); + ssize_t ret; + + if (!dentry) { + /* RCU pathwalk. */ + symlink = rcu_dereference(vnode->symlink); + if (!symlink || !afs_check_validity(vnode)) + return ERR_PTR(-ECHILD); + set_delayed_call(callback, NULL, NULL); + return symlink->content; + } + + if (vnode->symlink) { + ret = afs_validate(vnode, NULL); + if (ret < 0) + return ERR_PTR(ret); + + down_read(&vnode->validate_lock); + if (vnode->symlink) + goto good; + up_read(&vnode->validate_lock); + } + + if (down_write_killable(&vnode->validate_lock) < 0) + return ERR_PTR(-ERESTARTSYS); + if (!vnode->symlink) { + ret = afs_read_symlink(vnode); + if (ret < 0) { + up_write(&vnode->validate_lock); + return ERR_PTR(ret); + } + } + + downgrade_write(&vnode->validate_lock); + +good: + symlink = rcu_dereference_protected(vnode->symlink, + lockdep_is_held(&vnode->validate_lock)); + refcount_inc(&symlink->ref); + up_read(&vnode->validate_lock); + + set_delayed_call(callback, afs_put_link, symlink); + return symlink->content; +} + +int afs_readlink(struct dentry *dentry, char __user *buffer, int buflen) +{ + DEFINE_DELAYED_CALL(done); + const char *content; + int len; + + content = afs_get_link(dentry, d_inode(dentry), &done); + if (IS_ERR(content)) { + do_delayed_call(&done); + return PTR_ERR(content); + } + + len = umin(strlen(content), buflen); + if (copy_to_user(buffer, content, len)) + len = -EFAULT; + do_delayed_call(&done); + return len; +} + +/* + * Write the symlink contents to the cache as a single blob. We then throw + * away the page we used to receive it. + */ +int afs_symlink_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct afs_vnode *vnode = AFS_FS_I(mapping->host); + struct iov_iter iter; + int ret = 0; + + if (!down_read_trylock(&vnode->validate_lock)) { + if (wbc->sync_mode == WB_SYNC_NONE) { + /* The VFS will have undirtied the inode. */ + netfs_single_mark_inode_dirty(&vnode->netfs.inode); + return 0; + } + down_read(&vnode->validate_lock); + } + + if (vnode->directory && + atomic64_read(&vnode->cb_expires_at) != AFS_NO_CB_PROMISE) { + iov_iter_folio_queue(&iter, ITER_SOURCE, vnode->directory, 0, 0, + i_size_read(&vnode->netfs.inode)); + ret = netfs_writeback_single(mapping, wbc, &iter); + } + + if (ret == 0) { + netfs_wb_begin(&vnode->netfs, false); + netfs_free_folioq_buffer(vnode->directory); + vnode->directory = NULL; + vnode->directory_size = 0; + netfs_wb_end(&vnode->netfs); + } else if (ret == 1) { + ret = 0; /* Skipped write due to lock conflict. */ + } + + up_read(&vnode->validate_lock); + return ret; +} + +const struct inode_operations afs_symlink_inode_operations = { + .get_link = afs_get_link, + .readlink = afs_readlink, +}; + +const struct address_space_operations afs_symlink_aops = { + .writepages = afs_symlink_writepages, +}; diff --git a/fs/afs/validation.c b/fs/afs/validation.c new file mode 100644 index 000000000..e997563af --- /dev/null +++ b/fs/afs/validation.c @@ -0,0 +1,490 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* vnode and volume validity verification. + * + * Copyright (C) 2023 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include "internal.h" + +/* + * Data validation is managed through a number of mechanisms from the server: + * + * (1) On first contact with a server (such as if it has just been rebooted), + * the server sends us a CB.InitCallBackState* request. + * + * (2) On a RW volume, in response to certain vnode (inode)-accessing RPC + * calls, the server maintains a time-limited per-vnode promise that it + * will send us a CB.CallBack request if a third party alters the vnodes + * accessed. + * + * Note that a vnode-level callbacks may also be sent for other reasons, + * such as filelock release. + * + * (3) On a RO (or Backup) volume, in response to certain vnode-accessing RPC + * calls, each server maintains a time-limited per-volume promise that it + * will send us a CB.CallBack request if the RO volume is updated to a + * snapshot of the RW volume ("vos release"). This is an atomic event + * that cuts over all instances of the RO volume across multiple servers + * simultaneously. + * + * Note that a volume-level callbacks may also be sent for other reasons, + * such as the volumeserver taking over control of the volume from the + * fileserver. + * + * Note also that each server maintains an independent time limit on an + * independent callback. + * + * (4) Certain RPC calls include a volume information record "VolSync" in + * their reply. This contains a creation date for the volume that should + * remain unchanged for a RW volume (but will be changed if the volume is + * restored from backup) or will be bumped to the time of snapshotting + * when a RO volume is released. + * + * In order to track this events, the following are provided: + * + * ->cb_v_break. A counter of events that might mean that the contents of + * a volume have been altered since we last checked a vnode. + * + * ->cb_v_check. A counter of the number of events that we've sent a + * query to the server for. Everything's up to date if this equals + * cb_v_break. + * + * ->cb_scrub. A counter of the number of regression events for which we + * have to completely wipe the cache. + * + * ->cb_ro_snapshot. A counter of the number of times that we've + * recognised that a RO volume has been updated. + * + * ->cb_break. A counter of events that might mean that the contents of a + * vnode have been altered. + * + * ->cb_expires_at. The time at which the callback promise expires or + * AFS_NO_CB_PROMISE if we have no promise. + * + * The way we manage things is: + * + * (1) When a volume-level CB.CallBack occurs, we increment ->cb_v_break on + * the volume and reset ->cb_expires_at (ie. set AFS_NO_CB_PROMISE) on the + * volume and volume's server record. + * + * (2) When a CB.InitCallBackState occurs, we treat this as a volume-level + * callback break on all the volumes that have been using that volume + * (ie. increment ->cb_v_break and reset ->cb_expires_at). + * + * (3) When a vnode-level CB.CallBack occurs, we increment ->cb_break on the + * vnode and reset its ->cb_expires_at. If the vnode is mmapped, we also + * dispatch a work item to unmap all PTEs to the vnode's pagecache to + * force reentry to the filesystem for revalidation. + * + * (4) When entering the filesystem, we call afs_validate() to check the + * validity of a vnode. This first checks to see if ->cb_v_check and + * ->cb_v_break match, and if they don't, we lock volume->cb_check_lock + * exclusively and perform an FS.FetchStatus on the vnode. + * + * After checking the volume, we check the vnode. If there's a mismatch + * between the volume counters and the vnode's mirrors of those counters, + * we lock vnode->validate_lock and issue an FS.FetchStatus on the vnode. + * + * (5) When the reply from FS.FetchStatus arrives, the VolSync record is + * parsed: + * + * (A) If the Creation timestamp has changed on a RW volume or regressed + * on a RO volume, we try to increment ->cb_scrub; if it advances on a + * RO volume, we assume "vos release" happened and try to increment + * ->cb_ro_snapshot. + * + * (B) If the Update timestamp has regressed, we try to increment + * ->cb_scrub. + * + * Note that in both of these cases, we only do the increment if we can + * cmpxchg the value of the timestamp from the value we noted before the + * op. This tries to prevent parallel ops from fighting one another. + * + * volume->cb_v_check is then set to ->cb_v_break. + * + * (6) The AFSCallBack record included in the FS.FetchStatus reply is also + * parsed and used to set the promise in ->cb_expires_at for the vnode, + * the volume and the volume's server record. + * + * (7) If ->cb_scrub is seen to have advanced, we invalidate the pagecache for + * the vnode. + */ + +/* + * Check the validity of a vnode/inode and its parent volume. + */ +bool afs_check_validity(const struct afs_vnode *vnode) +{ + const struct afs_volume *volume = vnode->volume; + enum afs_vnode_invalid_trace trace = afs_vnode_valid_trace; + time64_t cb_expires_at = atomic64_read(&vnode->cb_expires_at); + time64_t deadline = ktime_get_real_seconds() + 10; + + if (test_bit(AFS_VNODE_DELETED, &vnode->flags)) + return true; + + if (atomic_read(&volume->cb_v_check) != atomic_read(&volume->cb_v_break)) + trace = afs_vnode_invalid_trace_cb_v_break; + else if (cb_expires_at == AFS_NO_CB_PROMISE) + trace = afs_vnode_invalid_trace_no_cb_promise; + else if (cb_expires_at <= deadline) + trace = afs_vnode_invalid_trace_expired; + else if (volume->cb_expires_at <= deadline) + trace = afs_vnode_invalid_trace_vol_expired; + else if (vnode->cb_ro_snapshot != atomic_read(&volume->cb_ro_snapshot)) + trace = afs_vnode_invalid_trace_cb_ro_snapshot; + else if (vnode->cb_scrub != atomic_read(&volume->cb_scrub)) + trace = afs_vnode_invalid_trace_cb_scrub; + else if (test_bit(AFS_VNODE_ZAP_DATA, &vnode->flags)) + trace = afs_vnode_invalid_trace_zap_data; + else + return true; + trace_afs_vnode_invalid(vnode, trace); + return false; +} + +/* + * See if the server we've just talked to is currently excluded. + */ +static bool __afs_is_server_excluded(struct afs_operation *op, struct afs_volume *volume) +{ + const struct afs_server_entry *se; + const struct afs_server_list *slist; + bool is_excluded = true; + int i; + + rcu_read_lock(); + + slist = rcu_dereference(volume->servers); + for (i = 0; i < slist->nr_servers; i++) { + se = &slist->servers[i]; + if (op->server == se->server) { + is_excluded = test_bit(AFS_SE_EXCLUDED, &se->flags); + break; + } + } + + rcu_read_unlock(); + return is_excluded; +} + +/* + * Update the volume's server list when the creation time changes and see if + * the server we've just talked to is currently excluded. + */ +static int afs_is_server_excluded(struct afs_operation *op, struct afs_volume *volume) +{ + int ret; + + if (__afs_is_server_excluded(op, volume)) + return 1; + + set_bit(AFS_VOLUME_NEEDS_UPDATE, &volume->flags); + ret = afs_check_volume_status(op->volume, op); + if (ret < 0) + return ret; + + return __afs_is_server_excluded(op, volume); +} + +/* + * Handle a change to the volume creation time in the VolSync record. + */ +static int afs_update_volume_creation_time(struct afs_operation *op, struct afs_volume *volume) +{ + unsigned int snap; + time64_t cur = volume->creation_time; + time64_t old = op->pre_volsync.creation; + time64_t new = op->volsync.creation; + int ret; + + _enter("%llx,%llx,%llx->%llx", volume->vid, cur, old, new); + + if (cur == TIME64_MIN) { + volume->creation_time = new; + return 0; + } + + if (new == cur) + return 0; + + /* Try to advance the creation timestamp from what we had before the + * operation to what we got back from the server. This should + * hopefully ensure that in a race between multiple operations only one + * of them will do this. + */ + if (cur != old) + return 0; + + /* If the creation time changes in an unexpected way, we need to scrub + * our caches. For a RW vol, this will only change if the volume is + * restored from a backup; for a RO/Backup vol, this will advance when + * the volume is updated to a new snapshot (eg. "vos release"). + */ + if (volume->type == AFSVL_RWVOL) + goto regressed; + if (volume->type == AFSVL_BACKVOL) { + if (new < old) + goto regressed; + goto advance; + } + + /* We have an RO volume, we need to query the VL server and look at the + * server flags to see if RW->RO replication is in progress. + */ + ret = afs_is_server_excluded(op, volume); + if (ret < 0) + return ret; + if (ret > 0) { + snap = atomic_read(&volume->cb_ro_snapshot); + trace_afs_cb_v_break(volume->vid, snap, afs_cb_break_volume_excluded); + return ret; + } + +advance: + snap = atomic_inc_return(&volume->cb_ro_snapshot); + trace_afs_cb_v_break(volume->vid, snap, afs_cb_break_for_vos_release); + volume->creation_time = new; + return 0; + +regressed: + atomic_inc(&volume->cb_scrub); + trace_afs_cb_v_break(volume->vid, 0, afs_cb_break_for_creation_regress); + volume->creation_time = new; + return 0; +} + +/* + * Handle a change to the volume update time in the VolSync record. + */ +static void afs_update_volume_update_time(struct afs_operation *op, struct afs_volume *volume) +{ + enum afs_cb_break_reason reason = afs_cb_break_no_break; + time64_t cur = volume->update_time; + time64_t old = op->pre_volsync.update; + time64_t new = op->volsync.update; + + _enter("%llx,%llx,%llx->%llx", volume->vid, cur, old, new); + + if (cur == TIME64_MIN) { + volume->update_time = new; + return; + } + + if (new == cur) + return; + + /* If the volume update time changes in an unexpected way, we need to + * scrub our caches. For a RW vol, this will advance on every + * modification op; for a RO/Backup vol, this will advance when the + * volume is updated to a new snapshot (eg. "vos release"). + */ + if (new < old) + reason = afs_cb_break_for_update_regress; + + /* Try to advance the update timestamp from what we had before the + * operation to what we got back from the server. This should + * hopefully ensure that in a race between multiple operations only one + * of them will do this. + */ + if (cur == old) { + if (reason == afs_cb_break_for_update_regress) { + atomic_inc(&volume->cb_scrub); + trace_afs_cb_v_break(volume->vid, 0, reason); + } + volume->update_time = new; + } +} + +static int afs_update_volume_times(struct afs_operation *op, struct afs_volume *volume) +{ + int ret = 0; + + if (likely(op->volsync.creation == volume->creation_time && + op->volsync.update == volume->update_time)) + return 0; + + mutex_lock(&volume->volsync_lock); + if (op->volsync.creation != volume->creation_time) { + ret = afs_update_volume_creation_time(op, volume); + if (ret < 0) + goto out; + } + if (op->volsync.update != volume->update_time) + afs_update_volume_update_time(op, volume); +out: + mutex_unlock(&volume->volsync_lock); + return ret; +} + +/* + * Update the state of a volume, including recording the expiration time of the + * callback promise. Returns 1 to redo the operation from the start. + */ +int afs_update_volume_state(struct afs_operation *op) +{ + struct afs_server_list *slist = op->server_list; + struct afs_server_entry *se = &slist->servers[op->server_index]; + struct afs_callback *cb = &op->file[0].scb.callback; + struct afs_volume *volume = op->volume; + unsigned int cb_v_break = atomic_read(&volume->cb_v_break); + unsigned int cb_v_check = atomic_read(&volume->cb_v_check); + int ret; + + _enter("%llx", op->volume->vid); + + if (op->volsync.creation != TIME64_MIN || op->volsync.update != TIME64_MIN) { + ret = afs_update_volume_times(op, volume); + if (ret != 0) { + _leave(" = %d", ret); + return ret; + } + } + + if (op->cb_v_break == cb_v_break && + (op->file[0].scb.have_cb || op->file[1].scb.have_cb)) { + time64_t expires_at = cb->expires_at; + + if (!op->file[0].scb.have_cb) + expires_at = op->file[1].scb.callback.expires_at; + + se->cb_expires_at = expires_at; + volume->cb_expires_at = expires_at; + } + if (cb_v_check < op->cb_v_break) + atomic_cmpxchg(&volume->cb_v_check, cb_v_check, op->cb_v_break); + return 0; +} + +/* + * mark the data attached to an inode as obsolete due to a write on the server + * - might also want to ditch all the outstanding writes and dirty pages + */ +static void afs_zap_data(struct afs_vnode *vnode) +{ + _enter("{%llx:%llu}", vnode->fid.vid, vnode->fid.vnode); + + afs_invalidate_cache(vnode, 0); + + /* nuke all the non-dirty pages that aren't locked, mapped or being + * written back in a regular file and completely discard the pages in a + * directory or symlink */ + if (S_ISREG(vnode->netfs.inode.i_mode)) + filemap_invalidate_inode(&vnode->netfs.inode, true, 0, LLONG_MAX); + else + filemap_invalidate_inode(&vnode->netfs.inode, false, 0, LLONG_MAX); +} + +/* + * validate a vnode/inode + * - there are several things we need to check + * - parent dir data changes (rm, rmdir, rename, mkdir, create, link, + * symlink) + * - parent dir metadata changed (security changes) + * - dentry data changed (write, truncate) + * - dentry metadata changed (security changes) + */ +int afs_validate(struct afs_vnode *vnode, struct key *key) +{ + struct afs_volume *volume = vnode->volume; + unsigned int cb_ro_snapshot, cb_scrub; + time64_t deadline = ktime_get_real_seconds() + 10; + bool zap = false, locked_vol = false; + int ret; + + _enter("{v={%llx:%llu} fl=%lx},%x", + vnode->fid.vid, vnode->fid.vnode, vnode->flags, + key_serial(key)); + + if (afs_check_validity(vnode)) + return test_bit(AFS_VNODE_DELETED, &vnode->flags) ? -ESTALE : 0; + + ret = down_write_killable(&vnode->validate_lock); + if (ret < 0) + goto error; + + if (test_bit(AFS_VNODE_DELETED, &vnode->flags)) { + ret = -ESTALE; + goto error_unlock; + } + + /* Validate a volume after the v_break has changed or the volume + * callback expired. We only want to do this once per volume per + * v_break change. The actual work will be done when parsing the + * status fetch reply. + */ + if (volume->cb_expires_at <= deadline || + atomic_read(&volume->cb_v_check) != atomic_read(&volume->cb_v_break)) { + ret = mutex_lock_interruptible(&volume->cb_check_lock); + if (ret < 0) + goto error_unlock; + locked_vol = true; + } + + cb_ro_snapshot = atomic_read(&volume->cb_ro_snapshot); + cb_scrub = atomic_read(&volume->cb_scrub); + if (vnode->cb_ro_snapshot != cb_ro_snapshot || + vnode->cb_scrub != cb_scrub) + unmap_mapping_pages(vnode->netfs.inode.i_mapping, 0, 0, false); + + if (vnode->cb_ro_snapshot != cb_ro_snapshot || + vnode->cb_scrub != cb_scrub || + volume->cb_expires_at <= deadline || + atomic_read(&volume->cb_v_check) != atomic_read(&volume->cb_v_break) || + atomic64_read(&vnode->cb_expires_at) <= deadline + ) { + ret = afs_fetch_status(vnode, key, false, NULL); + if (ret < 0) { + if (ret == -ENOENT) { + set_bit(AFS_VNODE_DELETED, &vnode->flags); + ret = -ESTALE; + } + goto error_unlock; + } + + _debug("new promise [fl=%lx]", vnode->flags); + } + + /* We can drop the volume lock now as. */ + if (locked_vol) { + mutex_unlock(&volume->cb_check_lock); + locked_vol = false; + } + + cb_ro_snapshot = atomic_read(&volume->cb_ro_snapshot); + cb_scrub = atomic_read(&volume->cb_scrub); + _debug("vnode inval %x==%x %x==%x", + vnode->cb_ro_snapshot, cb_ro_snapshot, + vnode->cb_scrub, cb_scrub); + if (vnode->cb_scrub != cb_scrub) + zap = true; + vnode->cb_ro_snapshot = cb_ro_snapshot; + vnode->cb_scrub = cb_scrub; + + /* If the vnode's data version number changed then its contents are + * different. Note that afs_apply_status() doesn't set ZAP_DATA on + * directories. + */ + zap |= test_and_clear_bit(AFS_VNODE_ZAP_DATA, &vnode->flags); + if (zap) { + if (S_ISREG(vnode->netfs.inode.i_mode)) + afs_zap_data(vnode); + else if (S_ISLNK(vnode->netfs.inode.i_mode)) + afs_invalidate_symlink(vnode); + } + up_write(&vnode->validate_lock); + _leave(" = 0"); + return 0; + +error_unlock: + if (locked_vol) + mutex_unlock(&volume->cb_check_lock); + up_write(&vnode->validate_lock); +error: + _leave(" = %d", ret); + return ret; +} diff --git a/fs/afs/vl_alias.c b/fs/afs/vl_alias.c new file mode 100644 index 000000000..fc9676abd --- /dev/null +++ b/fs/afs/vl_alias.c @@ -0,0 +1,340 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* AFS cell alias detection + * + * Copyright (C) 2020 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include "internal.h" + +/* + * Sample a volume. + */ +static struct afs_volume *afs_sample_volume(struct afs_cell *cell, struct key *key, + const char *name, unsigned int namelen) +{ + struct afs_volume *volume; + struct afs_fs_context fc = { + .type = 0, /* Explicitly leave it to the VLDB */ + .volnamesz = namelen, + .volname = name, + .net = cell->net, + .cell = cell, + .key = key, /* This might need to be something */ + }; + + volume = afs_create_volume(&fc); + _leave(" = %p", volume); + return volume; +} + +/* + * Compare the address lists of a pair of fileservers. + */ +static int afs_compare_fs_alists(const struct afs_server *server_a, + const struct afs_server *server_b) +{ + const struct afs_addr_list *la, *lb; + int a = 0, b = 0, addr_matches = 0; + + la = rcu_dereference(server_a->endpoint_state)->addresses; + lb = rcu_dereference(server_b->endpoint_state)->addresses; + + while (a < la->nr_addrs && b < lb->nr_addrs) { + unsigned long pa = (unsigned long)la->addrs[a].peer; + unsigned long pb = (unsigned long)lb->addrs[b].peer; + long diff = pa - pb; + + if (diff < 0) { + a++; + } else if (diff > 0) { + b++; + } else { + addr_matches++; + a++; + b++; + } + } + + return addr_matches; +} + +/* + * Compare the fileserver lists of two volumes. The server lists are sorted in + * order of ascending UUID. + */ +static int afs_compare_volume_slists(const struct afs_volume *vol_a, + const struct afs_volume *vol_b) +{ + const struct afs_server_list *la, *lb; + int i, a = 0, b = 0, uuid_matches = 0, addr_matches = 0; + + la = rcu_dereference(vol_a->servers); + lb = rcu_dereference(vol_b->servers); + + for (i = 0; i < AFS_MAXTYPES; i++) + if (vol_a->vids[i] != vol_b->vids[i]) + return 0; + + while (a < la->nr_servers && b < lb->nr_servers) { + const struct afs_server *server_a = la->servers[a].server; + const struct afs_server *server_b = lb->servers[b].server; + int diff = memcmp(&server_a->uuid, &server_b->uuid, sizeof(uuid_t)); + + if (diff < 0) { + a++; + } else if (diff > 0) { + b++; + } else { + uuid_matches++; + addr_matches += afs_compare_fs_alists(server_a, server_b); + a++; + b++; + } + } + + _leave(" = %d [um %d]", addr_matches, uuid_matches); + return addr_matches; +} + +/* + * Compare root.cell volumes. + */ +static int afs_compare_cell_roots(struct afs_cell *cell) +{ + struct afs_cell *p; + + _enter(""); + + rcu_read_lock(); + + hlist_for_each_entry_rcu(p, &cell->net->proc_cells, proc_link) { + if (p == cell || p->alias_of) + continue; + if (!p->root_volume) + continue; /* Ignore cells that don't have a root.cell volume. */ + + if (afs_compare_volume_slists(cell->root_volume, p->root_volume) != 0) + goto is_alias; + } + + rcu_read_unlock(); + _leave(" = 0"); + return 0; + +is_alias: + rcu_read_unlock(); + cell->alias_of = afs_use_cell(p, afs_cell_trace_use_alias); + return 1; +} + +/* + * Query the new cell for a volume from a cell we're already using. + */ +static int afs_query_for_alias_one(struct afs_cell *cell, struct key *key, + struct afs_cell *p) +{ + struct afs_volume *volume, *pvol = NULL; + int ret; + + /* Arbitrarily pick a volume from the list. */ + read_seqlock_excl(&p->volume_lock); + if (!RB_EMPTY_ROOT(&p->volumes)) + pvol = afs_get_volume(rb_entry(p->volumes.rb_node, + struct afs_volume, cell_node), + afs_volume_trace_get_query_alias); + read_sequnlock_excl(&p->volume_lock); + if (!pvol) + return 0; + + _enter("%s:%s", cell->name, pvol->name); + + /* And see if it's in the new cell. */ + volume = afs_sample_volume(cell, key, pvol->name, pvol->name_len); + if (IS_ERR(volume)) { + afs_put_volume(pvol, afs_volume_trace_put_query_alias); + if (PTR_ERR(volume) != -ENOMEDIUM) + return PTR_ERR(volume); + /* That volume is not in the new cell, so not an alias */ + return 0; + } + + /* The new cell has a like-named volume also - compare volume ID, + * server and address lists. + */ + ret = 0; + if (pvol->vid == volume->vid) { + rcu_read_lock(); + if (afs_compare_volume_slists(volume, pvol)) + ret = 1; + rcu_read_unlock(); + } + + afs_put_volume(volume, afs_volume_trace_put_query_alias); + afs_put_volume(pvol, afs_volume_trace_put_query_alias); + return ret; +} + +/* + * Query the new cell for volumes we know exist in cells we're already using. + */ +static int afs_query_for_alias(struct afs_cell *cell, struct key *key) +{ + struct afs_cell *p; + + _enter("%s", cell->name); + + if (mutex_lock_interruptible(&cell->net->proc_cells_lock) < 0) + return -ERESTARTSYS; + + hlist_for_each_entry(p, &cell->net->proc_cells, proc_link) { + if (p == cell || p->alias_of) + continue; + if (RB_EMPTY_ROOT(&p->volumes)) + continue; + if (p->root_volume) + continue; /* Ignore cells that have a root.cell volume. */ + afs_use_cell(p, afs_cell_trace_use_check_alias); + mutex_unlock(&cell->net->proc_cells_lock); + + if (afs_query_for_alias_one(cell, key, p) != 0) + goto is_alias; + + if (mutex_lock_interruptible(&cell->net->proc_cells_lock) < 0) { + afs_unuse_cell(p, afs_cell_trace_unuse_check_alias); + return -ERESTARTSYS; + } + + afs_unuse_cell(p, afs_cell_trace_unuse_check_alias); + } + + mutex_unlock(&cell->net->proc_cells_lock); + _leave(" = 0"); + return 0; + +is_alias: + cell->alias_of = p; /* Transfer our ref */ + return 1; +} + +/* + * Look up a VLDB record for a volume. + */ +static char *afs_vl_get_cell_name(struct afs_cell *cell, struct key *key) +{ + struct afs_vl_cursor vc; + char *cell_name = ERR_PTR(-EDESTADDRREQ); + bool skipped = false, not_skipped = false; + int ret; + + if (!afs_begin_vlserver_operation(&vc, cell, key)) + return ERR_PTR(-ERESTARTSYS); + + while (afs_select_vlserver(&vc)) { + if (!test_bit(AFS_VLSERVER_FL_IS_YFS, &vc.server->flags)) { + vc.call_error = -EOPNOTSUPP; + skipped = true; + continue; + } + not_skipped = true; + cell_name = afs_yfsvl_get_cell_name(&vc); + } + + ret = afs_end_vlserver_operation(&vc); + if (skipped && !not_skipped) + ret = -EOPNOTSUPP; + return ret < 0 ? ERR_PTR(ret) : cell_name; +} + +static int yfs_check_canonical_cell_name(struct afs_cell *cell, struct key *key) +{ + struct afs_cell *master; + size_t name_len; + char *cell_name; + + cell_name = afs_vl_get_cell_name(cell, key); + if (IS_ERR(cell_name)) + return PTR_ERR(cell_name); + + if (strcmp(cell_name, cell->name) == 0) { + kfree(cell_name); + return 0; + } + + name_len = strlen(cell_name); + if (!name_len || name_len > AFS_MAXCELLNAME) + master = ERR_PTR(-EOPNOTSUPP); + else + master = afs_lookup_cell(cell->net, cell_name, name_len, NULL, + AFS_LOOKUP_CELL_ALIAS_CHECK, + afs_cell_trace_use_lookup_canonical); + kfree(cell_name); + if (IS_ERR(master)) + return PTR_ERR(master); + + cell->alias_of = master; /* Transfer our ref */ + return 1; +} + +static int afs_do_cell_detect_alias(struct afs_cell *cell, struct key *key) +{ + struct afs_volume *root_volume; + int ret; + + _enter("%s", cell->name); + + ret = yfs_check_canonical_cell_name(cell, key); + if (ret != -EOPNOTSUPP) + return ret; + + /* Try and get the root.cell volume for comparison with other cells */ + root_volume = afs_sample_volume(cell, key, "root.cell", 9); + if (!IS_ERR(root_volume)) { + cell->root_volume = root_volume; + return afs_compare_cell_roots(cell); + } + + if (PTR_ERR(root_volume) != -ENOMEDIUM) + return PTR_ERR(root_volume); + + /* Okay, this cell doesn't have an root.cell volume. We need to + * locate some other random volume and use that to check. + */ + return afs_query_for_alias(cell, key); +} + +/* + * Check to see if a new cell is an alias of a cell we already have. At this + * point we have the cell's volume server list. + * + * Returns 0 if we didn't detect an alias, 1 if we found an alias and an error + * if we had problems gathering the data required. In the case the we did + * detect an alias, cell->alias_of is set to point to the assumed master. + */ +int afs_cell_detect_alias(struct afs_cell *cell, struct key *key) +{ + struct afs_net *net = cell->net; + int ret; + + if (mutex_lock_interruptible(&net->cells_alias_lock) < 0) + return -ERESTARTSYS; + + if (test_bit(AFS_CELL_FL_CHECK_ALIAS, &cell->flags)) { + ret = afs_do_cell_detect_alias(cell, key); + if (ret >= 0) + clear_bit_unlock(AFS_CELL_FL_CHECK_ALIAS, &cell->flags); + } else { + ret = cell->alias_of ? 1 : 0; + } + + mutex_unlock(&net->cells_alias_lock); + + if (ret == 1) + pr_notice("kAFS: Cell %s is an alias of %s\n", + cell->name, cell->alias_of->name); + return ret; +} diff --git a/fs/afs/vl_list.c b/fs/afs/vl_list.c new file mode 100644 index 000000000..c1dac5dbe --- /dev/null +++ b/fs/afs/vl_list.c @@ -0,0 +1,354 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* AFS vlserver list management. + * + * Copyright (C) 2018 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include "internal.h" + +struct afs_vlserver *afs_alloc_vlserver(const char *name, size_t name_len, + unsigned short port) +{ + struct afs_vlserver *vlserver; + static atomic_t debug_ids; + + vlserver = kzalloc_flex(*vlserver, name, name_len + 1); + if (vlserver) { + refcount_set(&vlserver->ref, 1); + rwlock_init(&vlserver->lock); + init_waitqueue_head(&vlserver->probe_wq); + spin_lock_init(&vlserver->probe_lock); + vlserver->debug_id = atomic_inc_return(&debug_ids); + vlserver->rtt = UINT_MAX; + vlserver->name_len = name_len; + vlserver->service_id = VL_SERVICE; + vlserver->port = port; + memcpy(vlserver->name, name, name_len); + } + return vlserver; +} + +static void afs_vlserver_rcu(struct rcu_head *rcu) +{ + struct afs_vlserver *vlserver = container_of(rcu, struct afs_vlserver, rcu); + + afs_put_addrlist(rcu_access_pointer(vlserver->addresses), + afs_alist_trace_put_vlserver); + kfree_rcu(vlserver, rcu); +} + +void afs_put_vlserver(struct afs_net *net, struct afs_vlserver *vlserver) +{ + if (vlserver && + refcount_dec_and_test(&vlserver->ref)) + call_rcu(&vlserver->rcu, afs_vlserver_rcu); +} + +struct afs_vlserver_list *afs_alloc_vlserver_list(unsigned int nr_servers) +{ + struct afs_vlserver_list *vllist; + + vllist = kzalloc_flex(*vllist, servers, nr_servers); + if (vllist) { + refcount_set(&vllist->ref, 1); + rwlock_init(&vllist->lock); + } + + return vllist; +} + +void afs_put_vlserverlist(struct afs_net *net, struct afs_vlserver_list *vllist) +{ + if (vllist) { + if (refcount_dec_and_test(&vllist->ref)) { + int i; + + for (i = 0; i < vllist->nr_servers; i++) { + afs_put_vlserver(net, vllist->servers[i].server); + } + kfree_rcu(vllist, rcu); + } + } +} + +static u16 afs_extract_le16(const u8 **_b) +{ + u16 val; + + val = (u16)*(*_b)++ << 0; + val |= (u16)*(*_b)++ << 8; + return val; +} + +/* + * Build a VL server address list from a DNS queried server list. + */ +static struct afs_addr_list *afs_extract_vl_addrs(struct afs_net *net, + const u8 **_b, const u8 *end, + u8 nr_addrs, u16 port) +{ + struct afs_addr_list *alist; + const u8 *b = *_b; + int ret; + + alist = afs_alloc_addrlist(nr_addrs); + if (!alist) + return ERR_PTR(-ENOMEM); + if (nr_addrs == 0) + return alist; + + for (; nr_addrs > 0 && end - b >= nr_addrs; nr_addrs--) { + struct dns_server_list_v1_address hdr; + __be32 x[4]; + + hdr.address_type = *b++; + + switch (hdr.address_type) { + case DNS_ADDRESS_IS_IPV4: + if (end - b < 4) { + _leave(" = -EINVAL [short inet]"); + ret = -EINVAL; + goto error; + } + memcpy(x, b, 4); + ret = afs_merge_fs_addr4(net, alist, x[0], port); + if (ret < 0) + goto error; + b += 4; + break; + + case DNS_ADDRESS_IS_IPV6: + if (end - b < 16) { + _leave(" = -EINVAL [short inet6]"); + ret = -EINVAL; + goto error; + } + memcpy(x, b, 16); + ret = afs_merge_fs_addr6(net, alist, x, port); + if (ret < 0) + goto error; + b += 16; + break; + + default: + _leave(" = -EADDRNOTAVAIL [unknown af %u]", + hdr.address_type); + ret = -EADDRNOTAVAIL; + goto error; + } + } + + /* Start with IPv6 if available. */ + if (alist->nr_ipv4 < alist->nr_addrs) + alist->preferred = alist->nr_ipv4; + + *_b = b; + return alist; + +error: + *_b = b; + afs_put_addrlist(alist, afs_alist_trace_put_parse_error); + return ERR_PTR(ret); +} + +/* + * Build a VL server list from a DNS queried server list. + */ +struct afs_vlserver_list *afs_extract_vlserver_list(struct afs_cell *cell, + const void *buffer, + size_t buffer_size) +{ + const struct dns_server_list_v1_header *hdr = buffer; + struct dns_server_list_v1_server bs; + struct afs_vlserver_list *vllist, *previous; + struct afs_addr_list *addrs; + struct afs_vlserver *server; + const u8 *b = buffer, *end = buffer + buffer_size; + int ret = -ENOMEM, nr_servers, i, j; + + _enter(""); + + /* Check that it's a server list, v1 */ + if (end - b < sizeof(*hdr) || + hdr->hdr.content != DNS_PAYLOAD_IS_SERVER_LIST || + hdr->hdr.version != 1) { + pr_notice("kAFS: Got DNS record [%u,%u] len %zu\n", + hdr->hdr.content, hdr->hdr.version, end - b); + ret = -EDESTADDRREQ; + goto dump; + } + + nr_servers = hdr->nr_servers; + + vllist = afs_alloc_vlserver_list(nr_servers); + if (!vllist) + return ERR_PTR(-ENOMEM); + + vllist->source = (hdr->source < NR__dns_record_source) ? + hdr->source : NR__dns_record_source; + vllist->status = (hdr->status < NR__dns_lookup_status) ? + hdr->status : NR__dns_lookup_status; + + read_lock(&cell->vl_servers_lock); + previous = afs_get_vlserverlist( + rcu_dereference_protected(cell->vl_servers, + lockdep_is_held(&cell->vl_servers_lock))); + read_unlock(&cell->vl_servers_lock); + + b += sizeof(*hdr); + while (end - b >= sizeof(bs)) { + int nlen; + + bs.name_len = afs_extract_le16(&b); + bs.priority = afs_extract_le16(&b); + bs.weight = afs_extract_le16(&b); + bs.port = afs_extract_le16(&b); + bs.source = *b++; + bs.status = *b++; + bs.protocol = *b++; + bs.nr_addrs = *b++; + + nlen = min3(bs.name_len, end - b, 255); + + _debug("extract %u %u %u %u %u %u %*.*s", + bs.name_len, bs.priority, bs.weight, + bs.port, bs.protocol, bs.nr_addrs, + bs.name_len, nlen, b); + + if (end - b < bs.name_len) + break; + + ret = -EPROTONOSUPPORT; + if (bs.protocol == DNS_SERVER_PROTOCOL_UNSPECIFIED) { + bs.protocol = DNS_SERVER_PROTOCOL_UDP; + } else if (bs.protocol != DNS_SERVER_PROTOCOL_UDP) { + _leave(" = [proto %u]", bs.protocol); + goto error; + } + + if (bs.port == 0) + bs.port = AFS_VL_PORT; + if (bs.source > NR__dns_record_source) + bs.source = NR__dns_record_source; + if (bs.status > NR__dns_lookup_status) + bs.status = NR__dns_lookup_status; + + /* See if we can update an old server record */ + server = NULL; + for (i = 0; i < previous->nr_servers; i++) { + struct afs_vlserver *p = previous->servers[i].server; + + if (p->name_len == bs.name_len && + p->port == bs.port && + strncasecmp(b, p->name, bs.name_len) == 0) { + server = afs_get_vlserver(p); + break; + } + } + + if (!server) { + ret = -ENOMEM; + server = afs_alloc_vlserver(b, bs.name_len, bs.port); + if (!server) + goto error; + } + + b += bs.name_len; + + /* Extract the addresses - note that we can't skip this as we + * have to advance the payload pointer. + */ + addrs = afs_extract_vl_addrs(cell->net, &b, end, bs.nr_addrs, bs.port); + if (IS_ERR(addrs)) { + ret = PTR_ERR(addrs); + goto error_2; + } + + if (vllist->nr_servers >= nr_servers) { + _debug("skip %u >= %u", vllist->nr_servers, nr_servers); + afs_put_addrlist(addrs, afs_alist_trace_put_parse_empty); + afs_put_vlserver(cell->net, server); + continue; + } + + addrs->source = bs.source; + addrs->status = bs.status; + + if (addrs->nr_addrs == 0) { + afs_put_addrlist(addrs, afs_alist_trace_put_parse_empty); + if (!rcu_access_pointer(server->addresses)) { + afs_put_vlserver(cell->net, server); + continue; + } + } else { + struct afs_addr_list *old = addrs; + + write_lock(&server->lock); + old = rcu_replace_pointer(server->addresses, old, + lockdep_is_held(&server->lock)); + write_unlock(&server->lock); + afs_put_addrlist(old, afs_alist_trace_put_vlserver_old); + } + + /* Check for duplicates in the server list */ + for (j = 0; j < vllist->nr_servers; j++) { + struct afs_vlserver *s = vllist->servers[j].server; + + if (s->name_len == server->name_len && + s->port == server->port && + strncasecmp(s->name, server->name, server->name_len) == 0) { + afs_put_vlserver(cell->net, server); + server = NULL; + break; + } + } + if (!server) + continue; + + /* Insertion-sort by priority and weight */ + for (j = 0; j < vllist->nr_servers; j++) { + if (bs.priority < vllist->servers[j].priority) + break; /* Lower preferable */ + if (bs.priority == vllist->servers[j].priority && + bs.weight > vllist->servers[j].weight) + break; /* Higher preferable */ + } + + if (j < vllist->nr_servers) { + memmove(vllist->servers + j + 1, + vllist->servers + j, + (vllist->nr_servers - j) * sizeof(struct afs_vlserver_entry)); + } + + clear_bit(AFS_VLSERVER_FL_PROBED, &server->flags); + + vllist->servers[j].priority = bs.priority; + vllist->servers[j].weight = bs.weight; + vllist->servers[j].server = server; + vllist->nr_servers++; + } + + if (b != end) { + _debug("parse error %zd", b - end); + goto error; + } + + afs_put_vlserverlist(cell->net, previous); + _leave(" = ok [%u]", vllist->nr_servers); + return vllist; + +error_2: + afs_put_vlserver(cell->net, server); +error: + afs_put_vlserverlist(cell->net, vllist); + afs_put_vlserverlist(cell->net, previous); +dump: + if (ret != -ENOMEM) { + printk(KERN_DEBUG "DNS: at %zu\n", (const void *)b - buffer); + print_hex_dump_bytes("DNS: ", DUMP_PREFIX_NONE, buffer, buffer_size); + } + return ERR_PTR(ret); +} diff --git a/fs/afs/vl_probe.c b/fs/afs/vl_probe.c new file mode 100644 index 000000000..3d2e0c925 --- /dev/null +++ b/fs/afs/vl_probe.c @@ -0,0 +1,310 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* AFS vlserver probing + * + * Copyright (C) 2018 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include "afs_fs.h" +#include "internal.h" +#include "protocol_yfs.h" + + +/* + * Handle the completion of a set of probes. + */ +static void afs_finished_vl_probe(struct afs_vlserver *server) +{ + if (!(server->probe.flags & AFS_VLSERVER_PROBE_RESPONDED)) { + server->rtt = UINT_MAX; + clear_bit(AFS_VLSERVER_FL_RESPONDING, &server->flags); + } + + clear_bit_unlock(AFS_VLSERVER_FL_PROBING, &server->flags); + wake_up_bit(&server->flags, AFS_VLSERVER_FL_PROBING); +} + +/* + * Handle the completion of a probe RPC call. + */ +static void afs_done_one_vl_probe(struct afs_vlserver *server, bool wake_up) +{ + if (atomic_dec_and_test(&server->probe_outstanding)) { + afs_finished_vl_probe(server); + wake_up = true; + } + + if (wake_up) + wake_up_all(&server->probe_wq); +} + +/* + * Process the result of probing a vlserver. This is called after successful + * or failed delivery of an VL.GetCapabilities operation. + */ +void afs_vlserver_probe_result(struct afs_call *call) +{ + struct afs_addr_list *alist = call->vl_probe; + struct afs_vlserver *server = call->vlserver; + struct afs_address *addr = &alist->addrs[call->probe_index]; + unsigned int server_index = call->server_index; + unsigned int rtt_us = 0; + unsigned int index = call->probe_index; + bool have_result = false; + int ret = call->error; + + _enter("%s,%u,%u,%d,%d", server->name, server_index, index, ret, call->abort_code); + + spin_lock(&server->probe_lock); + + switch (ret) { + case 0: + server->probe.error = 0; + goto responded; + case -ECONNABORTED: + if (!(server->probe.flags & AFS_VLSERVER_PROBE_RESPONDED)) { + server->probe.abort_code = call->abort_code; + server->probe.error = ret; + } + goto responded; + case -ENOMEM: + case -ENONET: + case -EKEYEXPIRED: + case -EKEYREVOKED: + case -EKEYREJECTED: + server->probe.flags |= AFS_VLSERVER_PROBE_LOCAL_FAILURE; + if (server->probe.error == 0) + server->probe.error = ret; + trace_afs_io_error(call->debug_id, ret, afs_io_error_vl_probe_fail); + goto out; + case -ECONNRESET: /* Responded, but call expired. */ + case -ERFKILL: + case -EADDRNOTAVAIL: + case -ENETUNREACH: + case -EHOSTUNREACH: + case -EHOSTDOWN: + case -ECONNREFUSED: + case -ETIMEDOUT: + case -ETIME: + default: + clear_bit(index, &alist->responded); + set_bit(index, &alist->probe_failed); + if (!(server->probe.flags & AFS_VLSERVER_PROBE_RESPONDED) && + (server->probe.error == 0 || + server->probe.error == -ETIMEDOUT || + server->probe.error == -ETIME)) + server->probe.error = ret; + trace_afs_io_error(call->debug_id, ret, afs_io_error_vl_probe_fail); + goto out; + } + +responded: + set_bit(index, &alist->responded); + clear_bit(index, &alist->probe_failed); + + if (call->service_id == YFS_VL_SERVICE) { + server->probe.flags |= AFS_VLSERVER_PROBE_IS_YFS; + set_bit(AFS_VLSERVER_FL_IS_YFS, &server->flags); + server->service_id = call->service_id; + } else { + server->probe.flags |= AFS_VLSERVER_PROBE_NOT_YFS; + if (!(server->probe.flags & AFS_VLSERVER_PROBE_IS_YFS)) { + clear_bit(AFS_VLSERVER_FL_IS_YFS, &server->flags); + server->service_id = call->service_id; + } + } + + rtt_us = rxrpc_kernel_get_srtt(addr->peer); + if (rtt_us < server->probe.rtt) { + server->probe.rtt = rtt_us; + server->rtt = rtt_us; + alist->preferred = index; + } + + smp_wmb(); /* Set rtt before responded. */ + server->probe.flags |= AFS_VLSERVER_PROBE_RESPONDED; + set_bit(AFS_VLSERVER_FL_PROBED, &server->flags); + set_bit(AFS_VLSERVER_FL_RESPONDING, &server->flags); + have_result = true; +out: + spin_unlock(&server->probe_lock); + + trace_afs_vl_probe(server, false, alist, index, call->error, call->abort_code, rtt_us); + _debug("probe [%u][%u] %pISpc rtt=%d ret=%d", + server_index, index, rxrpc_kernel_remote_addr(addr->peer), + rtt_us, ret); + + afs_done_one_vl_probe(server, have_result); +} + +/* + * Probe all of a vlserver's addresses to find out the best route and to + * query its capabilities. + */ +static bool afs_do_probe_vlserver(struct afs_net *net, + struct afs_vlserver *server, + struct key *key, + unsigned int server_index, + struct afs_error *_e) +{ + struct afs_addr_list *alist; + struct afs_call *call; + unsigned long unprobed; + unsigned int index, i; + bool in_progress = false; + int best_prio; + + _enter("%s", server->name); + + read_lock(&server->lock); + alist = rcu_dereference_protected(server->addresses, + lockdep_is_held(&server->lock)); + afs_get_addrlist(alist, afs_alist_trace_get_vlprobe); + read_unlock(&server->lock); + + atomic_set(&server->probe_outstanding, alist->nr_addrs); + memset(&server->probe, 0, sizeof(server->probe)); + server->probe.rtt = UINT_MAX; + + unprobed = (1UL << alist->nr_addrs) - 1; + while (unprobed) { + best_prio = -1; + index = 0; + for (i = 0; i < alist->nr_addrs; i++) { + if (test_bit(i, &unprobed) && + alist->addrs[i].prio > best_prio) { + index = i; + best_prio = alist->addrs[i].prio; + } + } + __clear_bit(index, &unprobed); + + trace_afs_vl_probe(server, true, alist, index, 0, 0, 0); + call = afs_vl_get_capabilities(net, alist, index, key, server, + server_index); + if (!IS_ERR(call)) { + afs_prioritise_error(_e, call->error, call->abort_code); + afs_put_call(call); + in_progress = true; + } else { + afs_prioritise_error(_e, PTR_ERR(call), 0); + afs_done_one_vl_probe(server, false); + } + } + + afs_put_addrlist(alist, afs_alist_trace_put_vlprobe); + return in_progress; +} + +/* + * Send off probes to all unprobed servers. + */ +int afs_send_vl_probes(struct afs_net *net, struct key *key, + struct afs_vlserver_list *vllist) +{ + struct afs_vlserver *server; + struct afs_error e = {}; + bool in_progress = false; + int i; + + for (i = 0; i < vllist->nr_servers; i++) { + server = vllist->servers[i].server; + if (test_bit(AFS_VLSERVER_FL_PROBED, &server->flags)) + continue; + + if (!test_and_set_bit_lock(AFS_VLSERVER_FL_PROBING, &server->flags) && + afs_do_probe_vlserver(net, server, key, i, &e)) + in_progress = true; + } + + return in_progress ? 0 : e.error; +} + +/* + * Wait for the first as-yet untried server to respond. + */ +int afs_wait_for_vl_probes(struct afs_vlserver_list *vllist, + unsigned long untried) +{ + struct wait_queue_entry *waits; + struct afs_vlserver *server; + unsigned int rtt = UINT_MAX, rtt_s; + bool have_responders = false; + int pref = -1, i; + + _enter("%u,%lx", vllist->nr_servers, untried); + + /* Only wait for servers that have a probe outstanding. */ + for (i = 0; i < vllist->nr_servers; i++) { + if (test_bit(i, &untried)) { + server = vllist->servers[i].server; + if (!test_bit(AFS_VLSERVER_FL_PROBING, &server->flags)) + __clear_bit(i, &untried); + if (server->probe.flags & AFS_VLSERVER_PROBE_RESPONDED) + have_responders = true; + } + } + if (have_responders || !untried) + return 0; + + waits = kmalloc(array_size(vllist->nr_servers, sizeof(*waits)), GFP_KERNEL); + if (!waits) + return -ENOMEM; + + for (i = 0; i < vllist->nr_servers; i++) { + if (test_bit(i, &untried)) { + server = vllist->servers[i].server; + init_waitqueue_entry(&waits[i], current); + add_wait_queue(&server->probe_wq, &waits[i]); + } + } + + for (;;) { + bool still_probing = false; + + set_current_state(TASK_INTERRUPTIBLE); + for (i = 0; i < vllist->nr_servers; i++) { + if (test_bit(i, &untried)) { + server = vllist->servers[i].server; + if (server->probe.flags & AFS_VLSERVER_PROBE_RESPONDED) + goto stop; + if (test_bit(AFS_VLSERVER_FL_PROBING, &server->flags)) + still_probing = true; + } + } + + if (!still_probing || signal_pending(current)) + goto stop; + schedule(); + } + +stop: + set_current_state(TASK_RUNNING); + + for (i = 0; i < vllist->nr_servers; i++) { + if (test_bit(i, &untried)) { + server = vllist->servers[i].server; + rtt_s = READ_ONCE(server->rtt); + if (test_bit(AFS_VLSERVER_FL_RESPONDING, &server->flags) && + rtt_s < rtt) { + pref = i; + rtt = rtt_s; + } + + remove_wait_queue(&server->probe_wq, &waits[i]); + } + } + + kfree(waits); + + if (pref == -1 && signal_pending(current)) + return -ERESTARTSYS; + + if (pref >= 0) + vllist->preferred = pref; + + _leave(" = 0 [%u]", pref); + return 0; +} diff --git a/fs/afs/vl_rotate.c b/fs/afs/vl_rotate.c new file mode 100644 index 000000000..6ad9688d8 --- /dev/null +++ b/fs/afs/vl_rotate.c @@ -0,0 +1,399 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Handle vlserver selection and rotation. + * + * Copyright (C) 2018 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include "internal.h" +#include "afs_vl.h" + +/* + * Begin an operation on a volume location server. + */ +bool afs_begin_vlserver_operation(struct afs_vl_cursor *vc, struct afs_cell *cell, + struct key *key) +{ + static atomic_t debug_ids; + + memset(vc, 0, sizeof(*vc)); + vc->cell = cell; + vc->key = key; + vc->cumul_error.error = -EDESTADDRREQ; + vc->nr_iterations = -1; + + if (signal_pending(current)) { + vc->cumul_error.error = -EINTR; + vc->flags |= AFS_VL_CURSOR_STOP; + return false; + } + + vc->debug_id = atomic_inc_return(&debug_ids); + return true; +} + +/* + * Begin iteration through a server list, starting with the last used server if + * possible, or the last recorded good server if not. + */ +static bool afs_start_vl_iteration(struct afs_vl_cursor *vc) +{ + struct afs_cell *cell = vc->cell; + unsigned int dns_lookup_count; + + if (cell->dns_source == DNS_RECORD_UNAVAILABLE || + cell->dns_expiry <= ktime_get_real_seconds()) { + dns_lookup_count = smp_load_acquire(&cell->dns_lookup_count); + set_bit(AFS_CELL_FL_DO_LOOKUP, &cell->flags); + afs_queue_cell(cell, afs_cell_trace_queue_dns); + + if (cell->dns_source == DNS_RECORD_UNAVAILABLE) { + if (wait_var_event_interruptible( + &cell->dns_lookup_count, + smp_load_acquire(&cell->dns_lookup_count) + != dns_lookup_count) < 0) { + vc->cumul_error.error = -ERESTARTSYS; + return false; + } + } + + /* Status load is ordered after lookup counter load */ + if (cell->dns_status == DNS_LOOKUP_GOT_NOT_FOUND) { + pr_warn("No record of cell %s\n", cell->name); + vc->cumul_error.error = -ENOENT; + return false; + } + + if (cell->dns_source == DNS_RECORD_UNAVAILABLE) { + vc->cumul_error.error = -EDESTADDRREQ; + return false; + } + } + + read_lock(&cell->vl_servers_lock); + vc->server_list = afs_get_vlserverlist( + rcu_dereference_protected(cell->vl_servers, + lockdep_is_held(&cell->vl_servers_lock))); + read_unlock(&cell->vl_servers_lock); + if (!vc->server_list->nr_servers) + return false; + + vc->untried_servers = (1UL << vc->server_list->nr_servers) - 1; + vc->server_index = -1; + return true; +} + +/* + * Select the vlserver to use. May be called multiple times to rotate + * through the vlservers. + */ +bool afs_select_vlserver(struct afs_vl_cursor *vc) +{ + struct afs_addr_list *alist = vc->alist; + struct afs_vlserver *vlserver; + unsigned long set, failed; + unsigned int rtt; + s32 abort_code = vc->call_abort_code; + int error = vc->call_error, i; + + vc->nr_iterations++; + + _enter("VC=%x+%x,%d{%lx},%d{%lx},%d,%d", + vc->debug_id, vc->nr_iterations, vc->server_index, vc->untried_servers, + vc->addr_index, vc->addr_tried, + error, abort_code); + + if (vc->flags & AFS_VL_CURSOR_STOP) { + _leave(" = f [stopped]"); + return false; + } + + if (vc->nr_iterations == 0) + goto start; + + WRITE_ONCE(alist->addrs[vc->addr_index].last_error, error); + + /* Evaluate the result of the previous operation, if there was one. */ + switch (error) { + default: + case 0: + /* Success or local failure. Stop. */ + vc->cumul_error.error = error; + vc->flags |= AFS_VL_CURSOR_STOP; + _leave(" = f [okay/local %d]", vc->cumul_error.error); + return false; + + case -ECONNABORTED: + /* The far side rejected the operation on some grounds. This + * might involve the server being busy or the volume having been moved. + */ + switch (abort_code) { + case AFSVL_IO: + case AFSVL_BADVOLOPER: + case AFSVL_NOMEM: + /* The server went weird. */ + afs_prioritise_error(&vc->cumul_error, -EREMOTEIO, abort_code); + //write_lock(&vc->cell->vl_servers_lock); + //vc->server_list->weird_mask |= 1 << vc->server_index; + //write_unlock(&vc->cell->vl_servers_lock); + goto next_server; + + default: + afs_prioritise_error(&vc->cumul_error, error, abort_code); + goto failed; + } + + case -ERFKILL: + case -EADDRNOTAVAIL: + case -ENETUNREACH: + case -EHOSTUNREACH: + case -EHOSTDOWN: + case -ECONNREFUSED: + case -ETIMEDOUT: + case -ETIME: + _debug("no conn %d", error); + afs_prioritise_error(&vc->cumul_error, error, 0); + goto iterate_address; + + case -ECONNRESET: + _debug("call reset"); + afs_prioritise_error(&vc->cumul_error, error, 0); + vc->flags |= AFS_VL_CURSOR_RETRY; + goto next_server; + + case -EOPNOTSUPP: + _debug("notsupp"); + goto next_server; + } + +restart_from_beginning: + _debug("restart"); + if (vc->call_responded && + vc->addr_index != vc->alist->preferred && + test_bit(alist->preferred, &vc->addr_tried)) + WRITE_ONCE(alist->preferred, vc->addr_index); + afs_put_addrlist(alist, afs_alist_trace_put_vlrotate_restart); + alist = vc->alist = NULL; + + afs_put_vlserverlist(vc->cell->net, vc->server_list); + vc->server_list = NULL; + if (vc->flags & AFS_VL_CURSOR_RETRIED) + goto failed; + vc->flags |= AFS_VL_CURSOR_RETRIED; +start: + _debug("start"); + ASSERTCMP(alist, ==, NULL); + + if (!afs_start_vl_iteration(vc)) + goto failed; + + error = afs_send_vl_probes(vc->cell->net, vc->key, vc->server_list); + if (error < 0) { + afs_prioritise_error(&vc->cumul_error, error, 0); + goto failed; + } + +pick_server: + _debug("pick [%lx]", vc->untried_servers); + ASSERTCMP(alist, ==, NULL); + + error = afs_wait_for_vl_probes(vc->server_list, vc->untried_servers); + if (error < 0) { + afs_prioritise_error(&vc->cumul_error, error, 0); + goto failed; + } + + /* Pick the untried server with the lowest RTT. */ + vc->server_index = vc->server_list->preferred; + if (test_bit(vc->server_index, &vc->untried_servers)) + goto selected_server; + + vc->server_index = -1; + rtt = UINT_MAX; + for (i = 0; i < vc->server_list->nr_servers; i++) { + struct afs_vlserver *s = vc->server_list->servers[i].server; + + if (!test_bit(i, &vc->untried_servers) || + !test_bit(AFS_VLSERVER_FL_RESPONDING, &s->flags)) + continue; + if (s->probe.rtt <= rtt) { + vc->server_index = i; + rtt = s->probe.rtt; + } + } + + if (vc->server_index == -1) + goto no_more_servers; + +selected_server: + _debug("use %d", vc->server_index); + __clear_bit(vc->server_index, &vc->untried_servers); + + /* We're starting on a different vlserver from the list. We need to + * check it, find its address list and probe its capabilities before we + * use it. + */ + vlserver = vc->server_list->servers[vc->server_index].server; + vc->server = vlserver; + + _debug("USING VLSERVER: %s", vlserver->name); + + read_lock(&vlserver->lock); + alist = rcu_dereference_protected(vlserver->addresses, + lockdep_is_held(&vlserver->lock)); + vc->alist = afs_get_addrlist(alist, afs_alist_trace_get_vlrotate_set); + read_unlock(&vlserver->lock); + + vc->addr_tried = 0; + vc->addr_index = -1; + +iterate_address: + /* Iterate over the current server's address list to try and find an + * address on which it will respond to us. + */ + set = READ_ONCE(alist->responded); + failed = READ_ONCE(alist->probe_failed); + vc->addr_index = READ_ONCE(alist->preferred); + + _debug("%lx-%lx-%lx,%d", set, failed, vc->addr_tried, vc->addr_index); + + set &= ~(failed | vc->addr_tried); + + if (!set) + goto next_server; + + if (!test_bit(vc->addr_index, &set)) + vc->addr_index = __ffs(set); + + set_bit(vc->addr_index, &vc->addr_tried); + vc->alist = alist; + + _debug("VL address %d/%d", vc->addr_index, alist->nr_addrs); + + vc->call_responded = false; + _leave(" = t %pISpc", rxrpc_kernel_remote_addr(alist->addrs[vc->addr_index].peer)); + return true; + +next_server: + _debug("next"); + ASSERT(alist); + if (vc->call_responded && + vc->addr_index != alist->preferred && + test_bit(alist->preferred, &vc->addr_tried)) + WRITE_ONCE(alist->preferred, vc->addr_index); + afs_put_addrlist(alist, afs_alist_trace_put_vlrotate_next); + alist = vc->alist = NULL; + goto pick_server; + +no_more_servers: + /* That's all the servers poked to no good effect. Try again if some + * of them were busy. + */ + if (vc->flags & AFS_VL_CURSOR_RETRY) + goto restart_from_beginning; + + for (i = 0; i < vc->server_list->nr_servers; i++) { + struct afs_vlserver *s = vc->server_list->servers[i].server; + + if (test_bit(AFS_VLSERVER_FL_RESPONDING, &s->flags)) + vc->cumul_error.responded = true; + afs_prioritise_error(&vc->cumul_error, READ_ONCE(s->probe.error), + s->probe.abort_code); + } + +failed: + if (alist) { + if (vc->call_responded && + vc->addr_index != alist->preferred && + test_bit(alist->preferred, &vc->addr_tried)) + WRITE_ONCE(alist->preferred, vc->addr_index); + afs_put_addrlist(alist, afs_alist_trace_put_vlrotate_fail); + alist = vc->alist = NULL; + } + vc->flags |= AFS_VL_CURSOR_STOP; + _leave(" = f [failed %d]", vc->cumul_error.error); + return false; +} + +/* + * Dump cursor state in the case of the error being EDESTADDRREQ. + */ +static void afs_vl_dump_edestaddrreq(const struct afs_vl_cursor *vc) +{ + struct afs_cell *cell = vc->cell; + static int count; + int i; + + if (!IS_ENABLED(CONFIG_AFS_DEBUG_CURSOR) || count > 3) + return; + count++; + + rcu_read_lock(); + pr_notice("EDESTADDR occurred\n"); + pr_notice("CELL: %s err=%d\n", cell->name, cell->error); + pr_notice("DNS: src=%u st=%u lc=%x\n", + cell->dns_source, cell->dns_status, cell->dns_lookup_count); + pr_notice("VC: ut=%lx ix=%u ni=%hu fl=%hx err=%hd\n", + vc->untried_servers, vc->server_index, vc->nr_iterations, + vc->flags, vc->cumul_error.error); + pr_notice("VC: call er=%d ac=%d r=%u\n", + vc->call_error, vc->call_abort_code, vc->call_responded); + + if (vc->server_list) { + const struct afs_vlserver_list *sl = vc->server_list; + pr_notice("VC: SL nr=%u ix=%u\n", + sl->nr_servers, sl->index); + for (i = 0; i < sl->nr_servers; i++) { + const struct afs_vlserver *s = sl->servers[i].server; + pr_notice("VC: server %s+%hu fl=%lx E=%hd\n", + s->name, s->port, s->flags, s->probe.error); + if (s->addresses) { + const struct afs_addr_list *a = + rcu_dereference(s->addresses); + pr_notice("VC: - nr=%u/%u/%u pf=%u\n", + a->nr_ipv4, a->nr_addrs, a->max_addrs, + a->preferred); + pr_notice("VC: - R=%lx F=%lx\n", + a->responded, a->probe_failed); + if (a == vc->alist) + pr_notice("VC: - current\n"); + } + } + } + + pr_notice("AC: t=%lx ax=%u\n", vc->addr_tried, vc->addr_index); + rcu_read_unlock(); +} + +/* + * Tidy up a volume location server cursor and unlock the vnode. + */ +int afs_end_vlserver_operation(struct afs_vl_cursor *vc) +{ + struct afs_net *net = vc->cell->net; + + _enter("VC=%x+%x", vc->debug_id, vc->nr_iterations); + + switch (vc->cumul_error.error) { + case -EDESTADDRREQ: + case -EADDRNOTAVAIL: + case -ENETUNREACH: + case -EHOSTUNREACH: + afs_vl_dump_edestaddrreq(vc); + break; + } + + if (vc->alist) { + if (vc->call_responded && + vc->addr_index != vc->alist->preferred && + test_bit(vc->alist->preferred, &vc->addr_tried)) + WRITE_ONCE(vc->alist->preferred, vc->addr_index); + afs_put_addrlist(vc->alist, afs_alist_trace_put_vlrotate_end); + vc->alist = NULL; + } + afs_put_vlserverlist(net, vc->server_list); + return vc->cumul_error.error; +} diff --git a/fs/afs/vlclient.c b/fs/afs/vlclient.c new file mode 100644 index 000000000..a40b43464 --- /dev/null +++ b/fs/afs/vlclient.c @@ -0,0 +1,793 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* AFS Volume Location Service client + * + * Copyright (C) 2002 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include "afs_fs.h" +#include "internal.h" + +/* + * Deliver reply data to a VL.GetEntryByNameU call. + */ +static int afs_deliver_vl_get_entry_by_name_u(struct afs_call *call) +{ + struct afs_uvldbentry__xdr *uvldb; + struct afs_vldb_entry *entry; + u32 nr_servers, vlflags; + int i, ret; + + _enter(""); + + ret = afs_transfer_reply(call); + if (ret < 0) + return ret; + + /* unmarshall the reply once we've received all of it */ + uvldb = call->buffer; + entry = call->ret_vldb; + + nr_servers = ntohl(uvldb->nServers); + if (nr_servers > AFS_NMAXNSERVERS) + nr_servers = AFS_NMAXNSERVERS; + + for (i = 0; i < ARRAY_SIZE(uvldb->name) - 1; i++) + entry->name[i] = (u8)ntohl(uvldb->name[i]); + entry->name[i] = 0; + entry->name_len = strlen(entry->name); + + vlflags = ntohl(uvldb->flags); + for (i = 0; i < nr_servers; i++) { + struct afs_uuid__xdr *xdr; + struct afs_uuid *uuid; + u32 tmp = ntohl(uvldb->serverFlags[i]); + int j; + int n = entry->nr_servers; + + if (tmp & AFS_VLSF_RWVOL) { + entry->fs_mask[n] |= AFS_VOL_VTM_RW; + if (vlflags & AFS_VLF_BACKEXISTS) + entry->fs_mask[n] |= AFS_VOL_VTM_BAK; + } + if (tmp & AFS_VLSF_ROVOL) + entry->fs_mask[n] |= AFS_VOL_VTM_RO; + if (!entry->fs_mask[n]) + continue; + + xdr = &uvldb->serverNumber[i]; + uuid = (struct afs_uuid *)&entry->fs_server[n]; + uuid->time_low = xdr->time_low; + uuid->time_mid = htons(ntohl(xdr->time_mid)); + uuid->time_hi_and_version = htons(ntohl(xdr->time_hi_and_version)); + uuid->clock_seq_hi_and_reserved = (u8)ntohl(xdr->clock_seq_hi_and_reserved); + uuid->clock_seq_low = (u8)ntohl(xdr->clock_seq_low); + for (j = 0; j < 6; j++) + uuid->node[j] = (u8)ntohl(xdr->node[j]); + + entry->vlsf_flags[n] = tmp; + entry->addr_version[n] = ntohl(uvldb->serverUnique[i]); + entry->nr_servers++; + } + + for (i = 0; i < AFS_MAXTYPES; i++) + entry->vid[i] = ntohl(uvldb->volumeId[i]); + + if (vlflags & AFS_VLF_RWEXISTS) + __set_bit(AFS_VLDB_HAS_RW, &entry->flags); + if (vlflags & AFS_VLF_ROEXISTS) + __set_bit(AFS_VLDB_HAS_RO, &entry->flags); + if (vlflags & AFS_VLF_BACKEXISTS) + __set_bit(AFS_VLDB_HAS_BAK, &entry->flags); + + if (!(vlflags & (AFS_VLF_RWEXISTS | AFS_VLF_ROEXISTS | AFS_VLF_BACKEXISTS))) { + entry->error = -ENOMEDIUM; + __set_bit(AFS_VLDB_QUERY_ERROR, &entry->flags); + } + + __set_bit(AFS_VLDB_QUERY_VALID, &entry->flags); + _leave(" = 0 [done]"); + return 0; +} + +/* + * VL.GetEntryByNameU operation type. + */ +static const struct afs_call_type afs_RXVLGetEntryByNameU = { + .name = "VL.GetEntryByNameU", + .op = afs_VL_GetEntryByNameU, + .deliver = afs_deliver_vl_get_entry_by_name_u, + .destructor = afs_flat_call_destructor, +}; + +/* + * Dispatch a get volume entry by name or ID operation (uuid variant). If the + * volname is a decimal number then it's a volume ID not a volume name. + */ +struct afs_vldb_entry *afs_vl_get_entry_by_name_u(struct afs_vl_cursor *vc, + const char *volname, + int volnamesz) +{ + struct afs_vldb_entry *entry; + struct afs_call *call; + struct afs_net *net = vc->cell->net; + size_t reqsz, padsz; + __be32 *bp; + + _enter(""); + + padsz = (4 - (volnamesz & 3)) & 3; + reqsz = 8 + volnamesz + padsz; + + entry = kzalloc_obj(struct afs_vldb_entry); + if (!entry) + return ERR_PTR(-ENOMEM); + + call = afs_alloc_flat_call(net, &afs_RXVLGetEntryByNameU, reqsz, + sizeof(struct afs_uvldbentry__xdr)); + if (!call) { + kfree(entry); + return ERR_PTR(-ENOMEM); + } + + call->key = vc->key; + call->ret_vldb = entry; + call->max_lifespan = AFS_VL_MAX_LIFESPAN; + call->peer = rxrpc_kernel_get_peer(vc->alist->addrs[vc->addr_index].peer); + call->service_id = vc->server->service_id; + + /* Marshall the parameters */ + bp = call->request; + *bp++ = htonl(VLGETENTRYBYNAMEU); + *bp++ = htonl(volnamesz); + memcpy(bp, volname, volnamesz); + if (padsz > 0) + memset((void *)bp + volnamesz, 0, padsz); + + trace_afs_make_vl_call(call); + afs_make_call(call, GFP_KERNEL); + afs_wait_for_call_to_complete(call); + vc->call_abort_code = call->abort_code; + vc->call_error = call->error; + vc->call_responded = call->responded; + afs_put_call(call); + if (vc->call_error) { + kfree(entry); + return ERR_PTR(vc->call_error); + } + return entry; +} + +/* + * Deliver reply data to a VL.GetAddrsU call. + * + * GetAddrsU(IN ListAddrByAttributes *inaddr, + * OUT afsUUID *uuidp1, + * OUT uint32_t *uniquifier, + * OUT uint32_t *nentries, + * OUT bulkaddrs *blkaddrs); + */ +static int afs_deliver_vl_get_addrs_u(struct afs_call *call) +{ + struct afs_addr_list *alist; + __be32 *bp; + u32 uniquifier, nentries, count; + int i, ret; + + _enter("{%u,%zu/%u}", + call->unmarshall, iov_iter_count(call->iter), call->count); + + switch (call->unmarshall) { + case 0: + afs_extract_to_buf(call, + sizeof(struct afs_uuid__xdr) + 3 * sizeof(__be32)); + call->unmarshall++; + + /* Extract the returned uuid, uniquifier, nentries and + * blkaddrs size */ + fallthrough; + case 1: + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + bp = call->buffer + sizeof(struct afs_uuid__xdr); + uniquifier = ntohl(*bp++); + nentries = ntohl(*bp++); + count = ntohl(*bp); + + nentries = min(nentries, count); + alist = afs_alloc_addrlist(nentries); + if (!alist) + return -ENOMEM; + alist->version = uniquifier; + call->ret_alist = alist; + call->count = count; + call->count2 = nentries; + call->unmarshall++; + + more_entries: + count = min(call->count, 4U); + afs_extract_to_buf(call, count * sizeof(__be32)); + + fallthrough; /* and extract entries */ + case 2: + ret = afs_extract_data(call, call->count > 4); + if (ret < 0) + return ret; + + alist = call->ret_alist; + bp = call->buffer; + count = min(call->count, 4U); + for (i = 0; i < count; i++) { + if (alist->nr_addrs < call->count2) { + ret = afs_merge_fs_addr4(call->net, alist, *bp++, AFS_FS_PORT); + if (ret < 0) + return ret; + } + } + + call->count -= count; + if (call->count > 0) + goto more_entries; + call->unmarshall++; + break; + } + + _leave(" = 0 [done]"); + return 0; +} + +/* + * VL.GetAddrsU operation type. + */ +static const struct afs_call_type afs_RXVLGetAddrsU = { + .name = "VL.GetAddrsU", + .op = afs_VL_GetAddrsU, + .deliver = afs_deliver_vl_get_addrs_u, + .destructor = afs_flat_call_destructor, +}; + +/* + * Dispatch an operation to get the addresses for a server, where the server is + * nominated by UUID. + */ +struct afs_addr_list *afs_vl_get_addrs_u(struct afs_vl_cursor *vc, + const uuid_t *uuid) +{ + struct afs_ListAddrByAttributes__xdr *r; + struct afs_addr_list *alist; + const struct afs_uuid *u = (const struct afs_uuid *)uuid; + struct afs_call *call; + struct afs_net *net = vc->cell->net; + __be32 *bp; + int i; + + _enter(""); + + call = afs_alloc_flat_call(net, &afs_RXVLGetAddrsU, + sizeof(__be32) + sizeof(struct afs_ListAddrByAttributes__xdr), + sizeof(struct afs_uuid__xdr) + 3 * sizeof(__be32)); + if (!call) + return ERR_PTR(-ENOMEM); + + call->key = vc->key; + call->ret_alist = NULL; + call->max_lifespan = AFS_VL_MAX_LIFESPAN; + call->peer = rxrpc_kernel_get_peer(vc->alist->addrs[vc->addr_index].peer); + call->service_id = vc->server->service_id; + + /* Marshall the parameters */ + bp = call->request; + *bp++ = htonl(VLGETADDRSU); + r = (struct afs_ListAddrByAttributes__xdr *)bp; + r->Mask = htonl(AFS_VLADDR_UUID); + r->ipaddr = 0; + r->index = 0; + r->spare = 0; + r->uuid.time_low = u->time_low; + r->uuid.time_mid = htonl(ntohs(u->time_mid)); + r->uuid.time_hi_and_version = htonl(ntohs(u->time_hi_and_version)); + r->uuid.clock_seq_hi_and_reserved = htonl(u->clock_seq_hi_and_reserved); + r->uuid.clock_seq_low = htonl(u->clock_seq_low); + for (i = 0; i < 6; i++) + r->uuid.node[i] = htonl(u->node[i]); + + trace_afs_make_vl_call(call); + afs_make_call(call, GFP_KERNEL); + afs_wait_for_call_to_complete(call); + vc->call_abort_code = call->abort_code; + vc->call_error = call->error; + vc->call_responded = call->responded; + alist = call->ret_alist; + afs_put_call(call); + if (vc->call_error) { + afs_put_addrlist(alist, afs_alist_trace_put_getaddru); + return ERR_PTR(vc->call_error); + } + return alist; +} + +/* + * Deliver reply data to an VL.GetCapabilities operation. + */ +static int afs_deliver_vl_get_capabilities(struct afs_call *call) +{ + u32 count; + int ret; + + _enter("{%u,%zu/%u}", + call->unmarshall, iov_iter_count(call->iter), call->count); + + switch (call->unmarshall) { + case 0: + afs_extract_to_tmp(call); + call->unmarshall++; + + fallthrough; /* and extract the capabilities word count */ + case 1: + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + count = ntohl(call->tmp); + call->count = count; + call->count2 = count; + + call->unmarshall++; + afs_extract_discard(call, count * sizeof(__be32)); + + fallthrough; /* and extract capabilities words */ + case 2: + ret = afs_extract_data(call, false); + if (ret < 0) + return ret; + + /* TODO: Examine capabilities */ + + call->unmarshall++; + break; + } + + _leave(" = 0 [done]"); + return 0; +} + +static void afs_destroy_vl_get_capabilities(struct afs_call *call) +{ + afs_put_addrlist(call->vl_probe, afs_alist_trace_put_vlgetcaps); + afs_put_vlserver(call->net, call->vlserver); + afs_flat_call_destructor(call); +} + +/* + * VL.GetCapabilities operation type + */ +static const struct afs_call_type afs_RXVLGetCapabilities = { + .name = "VL.GetCapabilities", + .op = afs_VL_GetCapabilities, + .deliver = afs_deliver_vl_get_capabilities, + .immediate_cancel = afs_vlserver_probe_result, + .done = afs_vlserver_probe_result, + .destructor = afs_destroy_vl_get_capabilities, +}; + +/* + * Probe a volume server for the capabilities that it supports. This can + * return up to 196 words. + * + * We use this to probe for service upgrade to determine what the server at the + * other end supports. + */ +struct afs_call *afs_vl_get_capabilities(struct afs_net *net, + struct afs_addr_list *alist, + unsigned int addr_index, + struct key *key, + struct afs_vlserver *server, + unsigned int server_index) +{ + struct afs_call *call; + __be32 *bp; + + _enter(""); + + call = afs_alloc_flat_call(net, &afs_RXVLGetCapabilities, 1 * 4, 16 * 4); + if (!call) + return ERR_PTR(-ENOMEM); + + call->key = key; + call->vlserver = afs_get_vlserver(server); + call->server_index = server_index; + call->peer = rxrpc_kernel_get_peer(alist->addrs[addr_index].peer); + call->vl_probe = afs_get_addrlist(alist, afs_alist_trace_get_vlgetcaps); + call->probe_index = addr_index; + call->service_id = server->service_id; + call->upgrade = true; + call->async = true; + call->max_lifespan = AFS_PROBE_MAX_LIFESPAN; + + /* marshall the parameters */ + bp = call->request; + *bp++ = htonl(VLGETCAPABILITIES); + + /* Can't take a ref on server */ + trace_afs_make_vl_call(call); + afs_make_call(call, GFP_KERNEL); + return call; +} + +/* + * Deliver reply data to a YFSVL.GetEndpoints call. + * + * GetEndpoints(IN yfsServerAttributes *attr, + * OUT opr_uuid *uuid, + * OUT afs_int32 *uniquifier, + * OUT endpoints *fsEndpoints, + * OUT endpoints *volEndpoints) + */ +static int afs_deliver_yfsvl_get_endpoints(struct afs_call *call) +{ + struct afs_addr_list *alist; + __be32 *bp; + u32 uniquifier, size; + int ret; + + _enter("{%u,%zu,%u}", + call->unmarshall, iov_iter_count(call->iter), call->count2); + + switch (call->unmarshall) { + case 0: + afs_extract_to_buf(call, sizeof(uuid_t) + 3 * sizeof(__be32)); + call->unmarshall = 1; + + /* Extract the returned uuid, uniquifier, fsEndpoints count and + * either the first fsEndpoint type or the volEndpoints + * count if there are no fsEndpoints. */ + fallthrough; + case 1: + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + bp = call->buffer + sizeof(uuid_t); + uniquifier = ntohl(*bp++); + call->count = ntohl(*bp++); + call->count2 = ntohl(*bp); /* Type or next count */ + + if (call->count > YFS_MAXENDPOINTS) + return afs_protocol_error(call, afs_eproto_yvl_fsendpt_num); + + alist = afs_alloc_addrlist(call->count); + if (!alist) + return -ENOMEM; + alist->version = uniquifier; + call->ret_alist = alist; + + if (call->count == 0) + goto extract_volendpoints; + + next_fsendpoint: + switch (call->count2) { + case YFS_ENDPOINT_IPV4: + size = sizeof(__be32) * (1 + 1 + 1); + break; + case YFS_ENDPOINT_IPV6: + size = sizeof(__be32) * (1 + 4 + 1); + break; + default: + return afs_protocol_error(call, afs_eproto_yvl_fsendpt_type); + } + + size += sizeof(__be32); + afs_extract_to_buf(call, size); + call->unmarshall = 2; + + fallthrough; /* and extract fsEndpoints[] entries */ + case 2: + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + alist = call->ret_alist; + bp = call->buffer; + switch (call->count2) { + case YFS_ENDPOINT_IPV4: + if (ntohl(bp[0]) != sizeof(__be32) * 2) + return afs_protocol_error( + call, afs_eproto_yvl_fsendpt4_len); + ret = afs_merge_fs_addr4(call->net, alist, bp[1], ntohl(bp[2])); + if (ret < 0) + return ret; + bp += 3; + break; + case YFS_ENDPOINT_IPV6: + if (ntohl(bp[0]) != sizeof(__be32) * 5) + return afs_protocol_error( + call, afs_eproto_yvl_fsendpt6_len); + ret = afs_merge_fs_addr6(call->net, alist, bp + 1, ntohl(bp[5])); + if (ret < 0) + return ret; + bp += 6; + break; + default: + return afs_protocol_error(call, afs_eproto_yvl_fsendpt_type); + } + + /* Got either the type of the next entry or the count of + * volEndpoints if no more fsEndpoints. + */ + call->count2 = ntohl(*bp++); + + call->count--; + if (call->count > 0) + goto next_fsendpoint; + + extract_volendpoints: + /* Extract the list of volEndpoints. */ + call->count = call->count2; + if (!call->count) + goto end; + if (call->count > YFS_MAXENDPOINTS) + return afs_protocol_error(call, afs_eproto_yvl_vlendpt_type); + + afs_extract_to_buf(call, 1 * sizeof(__be32)); + call->unmarshall = 3; + + /* Extract the type of volEndpoints[0]. Normally we would + * extract the type of the next endpoint when we extract the + * data of the current one, but this is the first... + */ + fallthrough; + case 3: + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + bp = call->buffer; + + next_volendpoint: + call->count2 = ntohl(*bp++); + switch (call->count2) { + case YFS_ENDPOINT_IPV4: + size = sizeof(__be32) * (1 + 1 + 1); + break; + case YFS_ENDPOINT_IPV6: + size = sizeof(__be32) * (1 + 4 + 1); + break; + default: + return afs_protocol_error(call, afs_eproto_yvl_vlendpt_type); + } + + if (call->count > 1) + size += sizeof(__be32); /* Get next type too */ + afs_extract_to_buf(call, size); + call->unmarshall = 4; + + fallthrough; /* and extract volEndpoints[] entries */ + case 4: + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + bp = call->buffer; + switch (call->count2) { + case YFS_ENDPOINT_IPV4: + if (ntohl(bp[0]) != sizeof(__be32) * 2) + return afs_protocol_error( + call, afs_eproto_yvl_vlendpt4_len); + bp += 3; + break; + case YFS_ENDPOINT_IPV6: + if (ntohl(bp[0]) != sizeof(__be32) * 5) + return afs_protocol_error( + call, afs_eproto_yvl_vlendpt6_len); + bp += 6; + break; + default: + return afs_protocol_error(call, afs_eproto_yvl_vlendpt_type); + } + + /* Got either the type of the next entry or the count of + * volEndpoints if no more fsEndpoints. + */ + call->count--; + if (call->count > 0) + goto next_volendpoint; + + end: + afs_extract_discard(call, 0); + call->unmarshall = 5; + + fallthrough; /* Done */ + case 5: + ret = afs_extract_data(call, false); + if (ret < 0) + return ret; + call->unmarshall = 6; + fallthrough; + + case 6: + break; + } + + _leave(" = 0 [done]"); + return 0; +} + +/* + * YFSVL.GetEndpoints operation type. + */ +static const struct afs_call_type afs_YFSVLGetEndpoints = { + .name = "YFSVL.GetEndpoints", + .op = afs_YFSVL_GetEndpoints, + .deliver = afs_deliver_yfsvl_get_endpoints, + .destructor = afs_flat_call_destructor, +}; + +/* + * Dispatch an operation to get the addresses for a server, where the server is + * nominated by UUID. + */ +struct afs_addr_list *afs_yfsvl_get_endpoints(struct afs_vl_cursor *vc, + const uuid_t *uuid) +{ + struct afs_addr_list *alist; + struct afs_call *call; + struct afs_net *net = vc->cell->net; + __be32 *bp; + + _enter(""); + + call = afs_alloc_flat_call(net, &afs_YFSVLGetEndpoints, + sizeof(__be32) * 2 + sizeof(*uuid), + sizeof(struct in6_addr) + sizeof(__be32) * 3); + if (!call) + return ERR_PTR(-ENOMEM); + + call->key = vc->key; + call->ret_alist = NULL; + call->max_lifespan = AFS_VL_MAX_LIFESPAN; + call->peer = rxrpc_kernel_get_peer(vc->alist->addrs[vc->addr_index].peer); + call->service_id = vc->server->service_id; + + /* Marshall the parameters */ + bp = call->request; + *bp++ = htonl(YVLGETENDPOINTS); + *bp++ = htonl(YFS_SERVER_UUID); + memcpy(bp, uuid, sizeof(*uuid)); /* Type opr_uuid */ + + trace_afs_make_vl_call(call); + afs_make_call(call, GFP_KERNEL); + afs_wait_for_call_to_complete(call); + vc->call_abort_code = call->abort_code; + vc->call_error = call->error; + vc->call_responded = call->responded; + alist = call->ret_alist; + afs_put_call(call); + if (vc->call_error) { + afs_put_addrlist(alist, afs_alist_trace_put_getaddru); + return ERR_PTR(vc->call_error); + } + return alist; +} + +/* + * Deliver reply data to a YFSVL.GetCellName operation. + */ +static int afs_deliver_yfsvl_get_cell_name(struct afs_call *call) +{ + char *cell_name; + u32 namesz, paddedsz; + int ret; + + _enter("{%u,%zu/%u}", + call->unmarshall, iov_iter_count(call->iter), call->count); + + switch (call->unmarshall) { + case 0: + afs_extract_to_tmp(call); + call->unmarshall++; + + fallthrough; /* and extract the cell name length */ + case 1: + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + namesz = ntohl(call->tmp); + if (namesz > YFS_VL_MAXCELLNAME) + return afs_protocol_error(call, afs_eproto_cellname_len); + paddedsz = (namesz + 3) & ~3; + call->count = namesz; + call->count2 = paddedsz - namesz; + + cell_name = kmalloc(namesz + 1, GFP_KERNEL); + if (!cell_name) + return -ENOMEM; + cell_name[namesz] = 0; + call->ret_str = cell_name; + + afs_extract_begin(call, cell_name, namesz); + call->unmarshall++; + + fallthrough; /* and extract cell name */ + case 2: + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + afs_extract_discard(call, call->count2); + call->unmarshall++; + + fallthrough; /* and extract padding */ + case 3: + ret = afs_extract_data(call, false); + if (ret < 0) + return ret; + + call->unmarshall++; + break; + } + + _leave(" = 0 [done]"); + return 0; +} + +/* + * VL.GetCapabilities operation type + */ +static const struct afs_call_type afs_YFSVLGetCellName = { + .name = "YFSVL.GetCellName", + .op = afs_YFSVL_GetCellName, + .deliver = afs_deliver_yfsvl_get_cell_name, + .destructor = afs_flat_call_destructor, +}; + +/* + * Probe a volume server for the capabilities that it supports. This can + * return up to 196 words. + * + * We use this to probe for service upgrade to determine what the server at the + * other end supports. + */ +char *afs_yfsvl_get_cell_name(struct afs_vl_cursor *vc) +{ + struct afs_call *call; + struct afs_net *net = vc->cell->net; + __be32 *bp; + char *cellname; + + _enter(""); + + call = afs_alloc_flat_call(net, &afs_YFSVLGetCellName, 1 * 4, 0); + if (!call) + return ERR_PTR(-ENOMEM); + + call->key = vc->key; + call->ret_str = NULL; + call->max_lifespan = AFS_VL_MAX_LIFESPAN; + call->peer = rxrpc_kernel_get_peer(vc->alist->addrs[vc->addr_index].peer); + call->service_id = vc->server->service_id; + + /* marshall the parameters */ + bp = call->request; + *bp++ = htonl(YVLGETCELLNAME); + + /* Can't take a ref on server */ + trace_afs_make_vl_call(call); + afs_make_call(call, GFP_KERNEL); + afs_wait_for_call_to_complete(call); + vc->call_abort_code = call->abort_code; + vc->call_error = call->error; + vc->call_responded = call->responded; + cellname = call->ret_str; + afs_put_call(call); + if (vc->call_error) { + kfree(cellname); + return ERR_PTR(vc->call_error); + } + return cellname; +} diff --git a/fs/afs/volume.c b/fs/afs/volume.c new file mode 100644 index 000000000..4f79d25ec --- /dev/null +++ b/fs/afs/volume.c @@ -0,0 +1,472 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* AFS volume management + * + * Copyright (C) 2002, 2007 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include "internal.h" + +static unsigned __read_mostly afs_volume_record_life = 60 * 60; +static atomic_t afs_volume_debug_id; + +static void afs_destroy_volume(struct work_struct *work); + +/* + * Insert a volume into a cell. If there's an existing volume record, that is + * returned instead with a ref held. + */ +static struct afs_volume *afs_insert_volume_into_cell(struct afs_cell *cell, + struct afs_volume *volume) +{ + struct afs_volume *p; + struct rb_node *parent = NULL, **pp; + + write_seqlock(&cell->volume_lock); + + pp = &cell->volumes.rb_node; + while (*pp) { + parent = *pp; + p = rb_entry(parent, struct afs_volume, cell_node); + if (p->vid < volume->vid) { + pp = &(*pp)->rb_left; + } else if (p->vid > volume->vid) { + pp = &(*pp)->rb_right; + } else { + if (afs_try_get_volume(p, afs_volume_trace_get_cell_insert)) { + volume = p; + goto found; + } + + set_bit(AFS_VOLUME_RM_TREE, &p->flags); + rb_replace_node_rcu(&p->cell_node, &volume->cell_node, &cell->volumes); + } + } + + rb_link_node_rcu(&volume->cell_node, parent, pp); + rb_insert_color(&volume->cell_node, &cell->volumes); + hlist_add_head_rcu(&volume->proc_link, &cell->proc_volumes); + +found: + write_sequnlock(&cell->volume_lock); + return volume; + +} + +static void afs_remove_volume_from_cell(struct afs_volume *volume) +{ + struct afs_cell *cell = volume->cell; + + if (!hlist_unhashed(&volume->proc_link)) { + trace_afs_volume(volume->debug_id, volume->vid, refcount_read(&volume->ref), + afs_volume_trace_remove); + write_seqlock(&cell->volume_lock); + hlist_del_rcu(&volume->proc_link); + if (!test_and_set_bit(AFS_VOLUME_RM_TREE, &volume->flags)) + rb_erase(&volume->cell_node, &cell->volumes); + write_sequnlock(&cell->volume_lock); + } +} + +/* + * Allocate a volume record and load it up from a vldb record. + */ +static struct afs_volume *afs_alloc_volume(struct afs_fs_context *params, + struct afs_vldb_entry *vldb, + struct afs_server_list **_slist) +{ + struct afs_server_list *slist; + struct afs_volume *volume; + int ret = -ENOMEM, i; + + volume = kzalloc_obj(struct afs_volume); + if (!volume) + goto error_0; + + volume->debug_id = atomic_inc_return(&afs_volume_debug_id); + volume->vid = vldb->vid[params->type]; + volume->update_at = ktime_get_real_seconds() + afs_volume_record_life; + volume->cell = afs_get_cell(params->cell, afs_cell_trace_get_vol); + volume->type = params->type; + volume->type_force = params->force; + volume->name_len = vldb->name_len; + volume->creation_time = TIME64_MIN; + volume->update_time = TIME64_MIN; + + refcount_set(&volume->ref, 1); + INIT_HLIST_NODE(&volume->proc_link); + INIT_WORK(&volume->destructor, afs_destroy_volume); + rwlock_init(&volume->servers_lock); + mutex_init(&volume->volsync_lock); + mutex_init(&volume->cb_check_lock); + rwlock_init(&volume->cb_v_break_lock); + INIT_LIST_HEAD(&volume->open_mmaps); + init_rwsem(&volume->open_mmaps_lock); + memcpy(volume->name, vldb->name, vldb->name_len + 1); + + for (i = 0; i < AFS_MAXTYPES; i++) + volume->vids[i] = vldb->vid[i]; + + slist = afs_alloc_server_list(volume, params->key, vldb); + if (IS_ERR(slist)) { + ret = PTR_ERR(slist); + goto error_1; + } + + *_slist = slist; + rcu_assign_pointer(volume->servers, slist); + trace_afs_volume(volume->debug_id, volume->vid, 1, afs_volume_trace_alloc); + return volume; + +error_1: + afs_put_cell(volume->cell, afs_cell_trace_put_vol); + kfree(volume); +error_0: + return ERR_PTR(ret); +} + +/* + * Look up or allocate a volume record. + */ +static struct afs_volume *afs_lookup_volume(struct afs_fs_context *params, + struct afs_vldb_entry *vldb) +{ + struct afs_server_list *slist; + struct afs_volume *candidate, *volume; + + candidate = afs_alloc_volume(params, vldb, &slist); + if (IS_ERR(candidate)) + return candidate; + + volume = afs_insert_volume_into_cell(params->cell, candidate); + if (volume == candidate) + afs_attach_volume_to_servers(volume, slist); + else + afs_put_volume(candidate, afs_volume_trace_put_cell_dup); + return volume; +} + +/* + * Look up a VLDB record for a volume. + */ +static struct afs_vldb_entry *afs_vl_lookup_vldb(struct afs_cell *cell, + struct key *key, + const char *volname, + size_t volnamesz) +{ + struct afs_vldb_entry *vldb = ERR_PTR(-EDESTADDRREQ); + struct afs_vl_cursor vc; + int ret; + + if (!afs_begin_vlserver_operation(&vc, cell, key)) + return ERR_PTR(-ERESTARTSYS); + + while (afs_select_vlserver(&vc)) { + vldb = afs_vl_get_entry_by_name_u(&vc, volname, volnamesz); + } + + ret = afs_end_vlserver_operation(&vc); + return ret < 0 ? ERR_PTR(ret) : vldb; +} + +/* + * Look up a volume in the VL server and create a candidate volume record for + * it. + * + * The volume name can be one of the following: + * "%[cell:]volume[.]" R/W volume + * "#[cell:]volume[.]" R/O or R/W volume (rwparent=0), + * or R/W (rwparent=1) volume + * "%[cell:]volume.readonly" R/O volume + * "#[cell:]volume.readonly" R/O volume + * "%[cell:]volume.backup" Backup volume + * "#[cell:]volume.backup" Backup volume + * + * The cell name is optional, and defaults to the current cell. + * + * See "The Rules of Mount Point Traversal" in Chapter 5 of the AFS SysAdmin + * Guide + * - Rule 1: Explicit type suffix forces access of that type or nothing + * (no suffix, then use Rule 2 & 3) + * - Rule 2: If parent volume is R/O, then mount R/O volume by preference, R/W + * if not available + * - Rule 3: If parent volume is R/W, then only mount R/W volume unless + * explicitly told otherwise + */ +struct afs_volume *afs_create_volume(struct afs_fs_context *params) +{ + struct afs_vldb_entry *vldb; + struct afs_volume *volume; + unsigned long type_mask = 1UL << params->type; + + vldb = afs_vl_lookup_vldb(params->cell, params->key, + params->volname, params->volnamesz); + if (IS_ERR(vldb)) + return ERR_CAST(vldb); + + if (test_bit(AFS_VLDB_QUERY_ERROR, &vldb->flags)) { + volume = ERR_PTR(vldb->error); + goto error; + } + + /* Make the final decision on the type we want */ + volume = ERR_PTR(-ENOMEDIUM); + if (params->force) { + if (!(vldb->flags & type_mask)) + goto error; + } else if (test_bit(AFS_VLDB_HAS_RO, &vldb->flags)) { + params->type = AFSVL_ROVOL; + } else if (test_bit(AFS_VLDB_HAS_RW, &vldb->flags)) { + params->type = AFSVL_RWVOL; + } else { + goto error; + } + + volume = afs_lookup_volume(params, vldb); + +error: + kfree(vldb); + return volume; +} + +/* + * Destroy a volume record + */ +static void afs_destroy_volume(struct work_struct *work) +{ + struct afs_volume *volume = container_of(work, struct afs_volume, destructor); + struct afs_server_list *slist = rcu_access_pointer(volume->servers); + + _enter("%p", volume); + +#ifdef CONFIG_AFS_FSCACHE + ASSERTCMP(volume->cache, ==, NULL); +#endif + + afs_detach_volume_from_servers(volume, slist); + afs_remove_volume_from_cell(volume); + afs_put_serverlist(volume->cell->net, slist); + afs_put_cell(volume->cell, afs_cell_trace_put_vol); + trace_afs_volume(volume->debug_id, volume->vid, refcount_read(&volume->ref), + afs_volume_trace_free); + kfree_rcu(volume, rcu); + + _leave(" [destroyed]"); +} + +/* + * Try to get a reference on a volume record. + */ +bool afs_try_get_volume(struct afs_volume *volume, enum afs_volume_trace reason) +{ + int r; + + if (__refcount_inc_not_zero(&volume->ref, &r)) { + trace_afs_volume(volume->debug_id, volume->vid, r + 1, reason); + return true; + } + return false; +} + +/* + * Get a reference on a volume record. + */ +struct afs_volume *afs_get_volume(struct afs_volume *volume, + enum afs_volume_trace reason) +{ + if (volume) { + int r; + + __refcount_inc(&volume->ref, &r); + trace_afs_volume(volume->debug_id, volume->vid, r + 1, reason); + } + return volume; +} + + +/* + * Drop a reference on a volume record. + */ +void afs_put_volume(struct afs_volume *volume, enum afs_volume_trace reason) +{ + if (volume) { + unsigned int debug_id = volume->debug_id; + afs_volid_t vid = volume->vid; + bool zero; + int r; + + zero = __refcount_dec_and_test(&volume->ref, &r); + trace_afs_volume(debug_id, vid, r - 1, reason); + if (zero) + schedule_work(&volume->destructor); + } +} + +/* + * Activate a volume. + */ +int afs_activate_volume(struct afs_volume *volume) +{ +#ifdef CONFIG_AFS_FSCACHE + struct fscache_volume *vcookie; + char *name; + + name = kasprintf(GFP_KERNEL, "afs,%s,%llx", + volume->cell->name, volume->vid); + if (!name) + return -ENOMEM; + + vcookie = fscache_acquire_volume(name, NULL, NULL, 0); + if (IS_ERR(vcookie)) { + if (vcookie != ERR_PTR(-EBUSY)) { + kfree(name); + return PTR_ERR(vcookie); + } + pr_err("AFS: Cache volume key already in use (%s)\n", name); + vcookie = NULL; + } + volume->cache = vcookie; + kfree(name); +#endif + return 0; +} + +/* + * Deactivate a volume. + */ +void afs_deactivate_volume(struct afs_volume *volume) +{ + _enter("%s", volume->name); + +#ifdef CONFIG_AFS_FSCACHE + fscache_relinquish_volume(volume->cache, NULL, + test_bit(AFS_VOLUME_DELETED, &volume->flags)); + volume->cache = NULL; +#endif + + _leave(""); +} + +/* + * Query the VL service to update the volume status. + */ +static int afs_update_volume_status(struct afs_volume *volume, struct key *key) +{ + struct afs_server_list *new, *old, *discard; + struct afs_vldb_entry *vldb; + char idbuf[24]; + int ret, idsz; + + _enter(""); + + /* We look up an ID by passing it as a decimal string in the + * operation's name parameter. + */ + idsz = snprintf(idbuf, sizeof(idbuf), "%llu", volume->vid); + + vldb = afs_vl_lookup_vldb(volume->cell, key, idbuf, idsz); + if (IS_ERR(vldb)) { + ret = PTR_ERR(vldb); + goto error; + } + + /* See if the volume got renamed. */ + if (vldb->name_len != volume->name_len || + memcmp(vldb->name, volume->name, vldb->name_len) != 0) { + /* TODO: Use RCU'd string. */ + memcpy(volume->name, vldb->name, AFS_MAXVOLNAME); + volume->name_len = vldb->name_len; + } + + /* See if the volume's server list got updated. */ + new = afs_alloc_server_list(volume, key, vldb); + if (IS_ERR(new)) { + ret = PTR_ERR(new); + goto error_vldb; + } + + write_lock(&volume->servers_lock); + + discard = new; + old = rcu_dereference_protected(volume->servers, + lockdep_is_held(&volume->servers_lock)); + if (afs_annotate_server_list(new, old)) { + new->seq = volume->servers_seq + 1; + rcu_assign_pointer(volume->servers, new); + smp_wmb(); + volume->servers_seq++; + discard = old; + } + + /* Check more often if replication is ongoing. */ + if (new->ro_replicating) + volume->update_at = ktime_get_real_seconds() + 10 * 60; + else + volume->update_at = ktime_get_real_seconds() + afs_volume_record_life; + write_unlock(&volume->servers_lock); + + if (discard == old) + afs_reattach_volume_to_servers(volume, new, old); + afs_put_serverlist(volume->cell->net, discard); + ret = 0; +error_vldb: + kfree(vldb); +error: + _leave(" = %d", ret); + return ret; +} + +/* + * Make sure the volume record is up to date. + */ +int afs_check_volume_status(struct afs_volume *volume, struct afs_operation *op) +{ + int ret, retries = 0; + + _enter(""); + +retry: + if (test_bit(AFS_VOLUME_WAIT, &volume->flags)) + goto wait; + if (volume->update_at <= ktime_get_real_seconds() || + test_bit(AFS_VOLUME_NEEDS_UPDATE, &volume->flags)) + goto update; + _leave(" = 0"); + return 0; + +update: + if (!test_and_set_bit_lock(AFS_VOLUME_UPDATING, &volume->flags)) { + clear_bit(AFS_VOLUME_NEEDS_UPDATE, &volume->flags); + ret = afs_update_volume_status(volume, op->key); + if (ret < 0) + set_bit(AFS_VOLUME_NEEDS_UPDATE, &volume->flags); + clear_bit_unlock(AFS_VOLUME_WAIT, &volume->flags); + clear_bit_unlock(AFS_VOLUME_UPDATING, &volume->flags); + wake_up_bit(&volume->flags, AFS_VOLUME_WAIT); + _leave(" = %d", ret); + return ret; + } + +wait: + if (!test_bit(AFS_VOLUME_WAIT, &volume->flags)) { + _leave(" = 0 [no wait]"); + return 0; + } + + ret = wait_on_bit(&volume->flags, AFS_VOLUME_WAIT, + (op->flags & AFS_OPERATION_UNINTR) ? + TASK_UNINTERRUPTIBLE : TASK_INTERRUPTIBLE); + if (ret == -ERESTARTSYS) { + _leave(" = %d", ret); + return ret; + } + + retries++; + if (retries == 4) { + _leave(" = -ESTALE"); + return -ESTALE; + } + goto retry; +} diff --git a/fs/afs/write.c b/fs/afs/write.c new file mode 100644 index 000000000..7f34b9397 --- /dev/null +++ b/fs/afs/write.c @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* handling of writes to regular files and writing back to the server + * + * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +/* + * completion of write to server + */ +static void afs_pages_written_back(struct afs_vnode *vnode, loff_t start, unsigned int len) +{ + _enter("{%llx:%llu},{%x @%llx}", + vnode->fid.vid, vnode->fid.vnode, len, start); + + afs_prune_wb_keys(vnode); + _leave(""); +} + +/* + * Find a key to use for the writeback. We cached the keys used to author the + * writes on the vnode. wreq->netfs_priv2 will contain the last writeback key + * record used or NULL and we need to start from there if it's set. + * wreq->netfs_priv will be set to the key itself or NULL. + */ +static void afs_get_writeback_key(struct netfs_io_request *wreq) +{ + struct afs_wb_key *wbk, *old = wreq->netfs_priv2; + struct afs_vnode *vnode = AFS_FS_I(wreq->inode); + + key_put(wreq->netfs_priv); + wreq->netfs_priv = NULL; + wreq->netfs_priv2 = NULL; + + spin_lock(&vnode->wb_lock); + if (old) + wbk = list_next_entry(old, vnode_link); + else + wbk = list_first_entry(&vnode->wb_keys, struct afs_wb_key, vnode_link); + + list_for_each_entry_from(wbk, &vnode->wb_keys, vnode_link) { + _debug("wbk %u", key_serial(wbk->key)); + if (key_validate(wbk->key) == 0) { + refcount_inc(&wbk->usage); + wreq->netfs_priv = key_get(wbk->key); + wreq->netfs_priv2 = wbk; + _debug("USE WB KEY %u", key_serial(wbk->key)); + break; + } + } + + spin_unlock(&vnode->wb_lock); + + afs_put_wb_key(old); +} + +static void afs_store_data_success(struct afs_operation *op) +{ + struct afs_vnode *vnode = op->file[0].vnode; + + op->ctime = op->file[0].scb.status.mtime_client; + afs_vnode_commit_status(op, &op->file[0]); + if (!afs_op_error(op)) { + afs_pages_written_back(vnode, op->store.pos, op->store.size); + afs_stat_v(vnode, n_stores); + atomic_long_add(op->store.size, &afs_v2net(vnode)->n_store_bytes); + } +} + +static const struct afs_operation_ops afs_store_data_operation = { + .issue_afs_rpc = afs_fs_store_data, + .issue_yfs_rpc = yfs_fs_store_data, + .success = afs_store_data_success, +}; + +/* + * Prepare a subrequest to write to the server. This sets the max_len + * parameter. + */ +void afs_prepare_write(struct netfs_io_subrequest *subreq) +{ + struct netfs_io_stream *stream = &subreq->rreq->io_streams[subreq->stream_nr]; + + //if (test_bit(NETFS_SREQ_RETRYING, &subreq->flags)) + // subreq->max_len = 512 * 1024; + //else + stream->sreq_max_len = 256 * 1024 * 1024; +} + +/* + * Issue a subrequest to write to the server. + */ +static void afs_issue_write_worker(struct work_struct *work) +{ + struct netfs_io_subrequest *subreq = container_of(work, struct netfs_io_subrequest, work); + struct netfs_io_request *wreq = subreq->rreq; + struct afs_operation *op; + struct afs_vnode *vnode = AFS_FS_I(wreq->inode); + unsigned long long pos = subreq->start + subreq->transferred; + size_t len = subreq->len - subreq->transferred; + int ret = -ENOKEY; + + _enter("R=%x[%x],%s{%llx:%llu.%u},%llx,%zx", + wreq->debug_id, subreq->debug_index, + vnode->volume->name, + vnode->fid.vid, + vnode->fid.vnode, + vnode->fid.unique, + pos, len); + +#if 0 // Error injection + if (subreq->debug_index == 3) + return netfs_write_subrequest_terminated(subreq, -ENOANO); + + if (!subreq->retry_count) { + set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); + return netfs_write_subrequest_terminated(subreq, -EAGAIN); + } +#endif + + op = afs_alloc_operation(wreq->netfs_priv, vnode->volume); + if (IS_ERR(op)) + return netfs_write_subrequest_terminated(subreq, -EAGAIN); + + afs_op_set_vnode(op, 0, vnode); + op->file[0].dv_delta = 1; + op->file[0].modification = true; + op->store.pos = pos; + op->store.size = len; + op->flags |= AFS_OPERATION_UNINTR; + op->ops = &afs_store_data_operation; + + afs_begin_vnode_operation(op); + + op->store.write_iter = &subreq->io_iter; + op->store.i_size = umax(pos + len, netfs_read_remote_i_size(&vnode->netfs.inode)); + op->mtime = inode_get_mtime(&vnode->netfs.inode); + + afs_wait_for_operation(op); + ret = afs_put_operation(op); + switch (ret) { + case 0: + __set_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); + break; + case -EACCES: + case -EPERM: + case -ENOKEY: + case -EKEYEXPIRED: + case -EKEYREJECTED: + case -EKEYREVOKED: + /* If there are more keys we can try, use the retry algorithm + * to rotate the keys. + */ + if (wreq->netfs_priv2) + set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); + break; + } + + netfs_write_subrequest_terminated(subreq, ret < 0 ? ret : subreq->len); +} + +void afs_issue_write(struct netfs_io_subrequest *subreq) +{ + subreq->work.func = afs_issue_write_worker; + if (!queue_work(system_dfl_wq, &subreq->work)) + WARN_ON_ONCE(1); +} + +/* + * Writeback calls this when it finds a folio that needs uploading. This isn't + * called if writeback only has copy-to-cache to deal with. + */ +void afs_begin_writeback(struct netfs_io_request *wreq) +{ + if (S_ISREG(wreq->inode->i_mode)) + afs_get_writeback_key(wreq); +} + +/* + * Prepare to retry the writes in request. Use this to try rotating the + * available writeback keys. + */ +void afs_retry_request(struct netfs_io_request *wreq, struct netfs_io_stream *stream) +{ + struct netfs_io_subrequest *subreq = + list_first_entry(&stream->subrequests, + struct netfs_io_subrequest, rreq_link); + + switch (wreq->origin) { + case NETFS_READAHEAD: + case NETFS_READPAGE: + case NETFS_READ_GAPS: + case NETFS_READ_SINGLE: + case NETFS_READ_FOR_WRITE: + case NETFS_UNBUFFERED_READ: + case NETFS_DIO_READ: + return; + default: + break; + } + + switch (subreq->error) { + case -EACCES: + case -EPERM: + case -ENOKEY: + case -EKEYEXPIRED: + case -EKEYREJECTED: + case -EKEYREVOKED: + afs_get_writeback_key(wreq); + if (!wreq->netfs_priv) + stream->failed = true; + break; + } +} + +/* + * write some of the pending data back to the server + */ +int afs_writepages(struct address_space *mapping, struct writeback_control *wbc) +{ + struct afs_vnode *vnode = AFS_FS_I(mapping->host); + int ret; + + /* We have to be careful as we can end up racing with setattr() + * truncating the pagecache since the caller doesn't take a lock here + * to prevent it. + */ + if (wbc->sync_mode == WB_SYNC_ALL) + down_read(&vnode->validate_lock); + else if (!down_read_trylock(&vnode->validate_lock)) + return 0; + + ret = netfs_writepages(mapping, wbc); + up_read(&vnode->validate_lock); + return ret; +} + +/* + * flush any dirty pages for this process, and check for write errors. + * - the return status from this call provides a reliable indication of + * whether any write errors occurred for this process. + */ +int afs_fsync(struct file *file, loff_t start, loff_t end, int datasync) +{ + struct afs_vnode *vnode = AFS_FS_I(file_inode(file)); + struct afs_file *af = file->private_data; + int ret; + + _enter("{%llx:%llu},{n=%pD},%d", + vnode->fid.vid, vnode->fid.vnode, file, + datasync); + + ret = afs_validate(vnode, af->key); + if (ret < 0) + return ret; + + return file_write_and_wait_range(file, start, end); +} + +/* + * notification that a previously read-only page is about to become writable + * - if it returns an error, the caller will deliver a bus error signal + */ +vm_fault_t afs_page_mkwrite(struct vm_fault *vmf) +{ + struct file *file = vmf->vma->vm_file; + + if (afs_validate(AFS_FS_I(file_inode(file)), afs_file_key(file)) < 0) + return VM_FAULT_SIGBUS; + return netfs_page_mkwrite(vmf, NULL); +} + +/* + * Prune the keys cached for writeback. The caller must hold vnode->wb_lock. + */ +void afs_prune_wb_keys(struct afs_vnode *vnode) +{ + LIST_HEAD(graveyard); + struct afs_wb_key *wbk, *tmp; + + /* Discard unused keys */ + spin_lock(&vnode->wb_lock); + + if (!mapping_tagged(&vnode->netfs.inode.i_data, PAGECACHE_TAG_WRITEBACK) && + !mapping_tagged(&vnode->netfs.inode.i_data, PAGECACHE_TAG_DIRTY)) { + list_for_each_entry_safe(wbk, tmp, &vnode->wb_keys, vnode_link) { + if (refcount_read(&wbk->usage) == 1) + list_move(&wbk->vnode_link, &graveyard); + } + } + + spin_unlock(&vnode->wb_lock); + + while (!list_empty(&graveyard)) { + wbk = list_entry(graveyard.next, struct afs_wb_key, vnode_link); + list_del(&wbk->vnode_link); + afs_put_wb_key(wbk); + } +} diff --git a/fs/afs/xattr.c b/fs/afs/xattr.c new file mode 100644 index 000000000..3770ed236 --- /dev/null +++ b/fs/afs/xattr.c @@ -0,0 +1,363 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Extended attribute handling for AFS. We use xattrs to get and set metadata + * instead of providing pioctl(). + * + * Copyright (C) 2017 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include "internal.h" + +/* + * Deal with the result of a successful fetch ACL operation. + */ +static void afs_acl_success(struct afs_operation *op) +{ + afs_vnode_commit_status(op, &op->file[0]); +} + +static void afs_acl_put(struct afs_operation *op) +{ + kfree(op->acl); +} + +static const struct afs_operation_ops afs_fetch_acl_operation = { + .issue_afs_rpc = afs_fs_fetch_acl, + .success = afs_acl_success, + .put = afs_acl_put, +}; + +/* + * Get a file's ACL. + */ +static int afs_xattr_get_acl(const struct xattr_handler *handler, + struct dentry *dentry, + struct inode *inode, const char *name, + void *buffer, size_t size) +{ + struct afs_operation *op; + struct afs_vnode *vnode = AFS_FS_I(inode); + struct afs_acl *acl = NULL; + int ret; + + op = afs_alloc_operation(NULL, vnode->volume); + if (IS_ERR(op)) + return -ENOMEM; + + afs_op_set_vnode(op, 0, vnode); + op->ops = &afs_fetch_acl_operation; + + afs_begin_vnode_operation(op); + afs_wait_for_operation(op); + acl = op->acl; + op->acl = NULL; + ret = afs_put_operation(op); + + if (ret == 0) { + ret = acl->size; + if (size > 0) { + if (acl->size <= size) + memcpy(buffer, acl->data, acl->size); + else + ret = -ERANGE; + } + } + + kfree(acl); + return ret; +} + +static bool afs_make_acl(struct afs_operation *op, + const void *buffer, size_t size) +{ + struct afs_acl *acl; + + acl = kmalloc_flex(*acl, data, size); + if (!acl) { + afs_op_nomem(op); + return false; + } + + acl->size = size; + memcpy(acl->data, buffer, size); + op->acl = acl; + return true; +} + +static const struct afs_operation_ops afs_store_acl_operation = { + .issue_afs_rpc = afs_fs_store_acl, + .success = afs_acl_success, + .put = afs_acl_put, +}; + +/* + * Set a file's AFS3 ACL. + */ +static int afs_xattr_set_acl(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *dentry, + struct inode *inode, const char *name, + const void *buffer, size_t size, int flags) +{ + struct afs_operation *op; + struct afs_vnode *vnode = AFS_FS_I(inode); + + if (flags == XATTR_CREATE) + return -EINVAL; + + op = afs_alloc_operation(NULL, vnode->volume); + if (IS_ERR(op)) + return -ENOMEM; + + afs_op_set_vnode(op, 0, vnode); + if (!afs_make_acl(op, buffer, size)) + return afs_put_operation(op); + + op->ops = &afs_store_acl_operation; + return afs_do_sync_operation(op); +} + +static const struct xattr_handler afs_xattr_afs_acl_handler = { + .name = "afs.acl", + .get = afs_xattr_get_acl, + .set = afs_xattr_set_acl, +}; + +static const struct afs_operation_ops yfs_fetch_opaque_acl_operation = { + .issue_yfs_rpc = yfs_fs_fetch_opaque_acl, + .success = afs_acl_success, + /* Don't free op->yacl in .put here */ +}; + +/* + * Get a file's YFS ACL. + */ +static int afs_xattr_get_yfs(const struct xattr_handler *handler, + struct dentry *dentry, + struct inode *inode, const char *name, + void *buffer, size_t size) +{ + struct afs_operation *op; + struct afs_vnode *vnode = AFS_FS_I(inode); + struct yfs_acl *yacl = NULL; + char buf[16], *data; + int which = 0, dsize, ret = -ENOMEM; + + if (strcmp(name, "acl") == 0) + which = 0; + else if (strcmp(name, "acl_inherited") == 0) + which = 1; + else if (strcmp(name, "acl_num_cleaned") == 0) + which = 2; + else if (strcmp(name, "vol_acl") == 0) + which = 3; + else + return -EOPNOTSUPP; + + yacl = kzalloc_obj(struct yfs_acl); + if (!yacl) + goto error; + + if (which == 0) + yacl->flags |= YFS_ACL_WANT_ACL; + else if (which == 3) + yacl->flags |= YFS_ACL_WANT_VOL_ACL; + + op = afs_alloc_operation(NULL, vnode->volume); + if (IS_ERR(op)) + goto error_yacl; + + afs_op_set_vnode(op, 0, vnode); + op->yacl = yacl; + op->ops = &yfs_fetch_opaque_acl_operation; + + afs_begin_vnode_operation(op); + afs_wait_for_operation(op); + ret = afs_put_operation(op); + + if (ret == 0) { + switch (which) { + case 0: + data = yacl->acl->data; + dsize = yacl->acl->size; + break; + case 1: + data = buf; + dsize = scnprintf(buf, sizeof(buf), "%u", yacl->inherit_flag); + break; + case 2: + data = buf; + dsize = scnprintf(buf, sizeof(buf), "%u", yacl->num_cleaned); + break; + case 3: + data = yacl->vol_acl->data; + dsize = yacl->vol_acl->size; + break; + default: + ret = -EOPNOTSUPP; + goto error_yacl; + } + + ret = dsize; + if (size > 0) { + if (dsize <= size) + memcpy(buffer, data, dsize); + else + ret = -ERANGE; + } + } else if (ret == -ENOTSUPP) { + ret = -ENODATA; + } + +error_yacl: + yfs_free_opaque_acl(yacl); +error: + return ret; +} + +static const struct afs_operation_ops yfs_store_opaque_acl2_operation = { + .issue_yfs_rpc = yfs_fs_store_opaque_acl2, + .success = afs_acl_success, + .put = afs_acl_put, +}; + +/* + * Set a file's YFS ACL. + */ +static int afs_xattr_set_yfs(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *dentry, + struct inode *inode, const char *name, + const void *buffer, size_t size, int flags) +{ + struct afs_operation *op; + struct afs_vnode *vnode = AFS_FS_I(inode); + int ret; + + if (flags == XATTR_CREATE || + strcmp(name, "acl") != 0) + return -EINVAL; + + op = afs_alloc_operation(NULL, vnode->volume); + if (IS_ERR(op)) + return -ENOMEM; + + afs_op_set_vnode(op, 0, vnode); + if (!afs_make_acl(op, buffer, size)) + return afs_put_operation(op); + + op->ops = &yfs_store_opaque_acl2_operation; + ret = afs_do_sync_operation(op); + if (ret == -ENOTSUPP) + ret = -ENODATA; + return ret; +} + +static const struct xattr_handler afs_xattr_yfs_handler = { + .prefix = "afs.yfs.", + .get = afs_xattr_get_yfs, + .set = afs_xattr_set_yfs, +}; + +/* + * Get the name of the cell on which a file resides. + */ +static int afs_xattr_get_cell(const struct xattr_handler *handler, + struct dentry *dentry, + struct inode *inode, const char *name, + void *buffer, size_t size) +{ + struct afs_vnode *vnode = AFS_FS_I(inode); + struct afs_cell *cell = vnode->volume->cell; + size_t namelen; + + namelen = cell->name_len; + if (size == 0) + return namelen; + if (namelen > size) + return -ERANGE; + memcpy(buffer, cell->name, namelen); + return namelen; +} + +static const struct xattr_handler afs_xattr_afs_cell_handler = { + .name = "afs.cell", + .get = afs_xattr_get_cell, +}; + +/* + * Get the volume ID, vnode ID and vnode uniquifier of a file as a sequence of + * hex numbers separated by colons. + */ +static int afs_xattr_get_fid(const struct xattr_handler *handler, + struct dentry *dentry, + struct inode *inode, const char *name, + void *buffer, size_t size) +{ + struct afs_vnode *vnode = AFS_FS_I(inode); + char text[16 + 1 + 24 + 1 + 8 + 1]; + size_t len; + + /* The volume ID is 64-bit, the vnode ID is 96-bit and the + * uniquifier is 32-bit. + */ + len = scnprintf(text, sizeof(text), "%llx:", vnode->fid.vid); + if (vnode->fid.vnode_hi) + len += scnprintf(text + len, sizeof(text) - len, "%x%016llx", + vnode->fid.vnode_hi, vnode->fid.vnode); + else + len += scnprintf(text + len, sizeof(text) - len, "%llx", + vnode->fid.vnode); + len += scnprintf(text + len, sizeof(text) - len, ":%x", + vnode->fid.unique); + + if (size == 0) + return len; + if (len > size) + return -ERANGE; + memcpy(buffer, text, len); + return len; +} + +static const struct xattr_handler afs_xattr_afs_fid_handler = { + .name = "afs.fid", + .get = afs_xattr_get_fid, +}; + +/* + * Get the name of the volume on which a file resides. + */ +static int afs_xattr_get_volume(const struct xattr_handler *handler, + struct dentry *dentry, + struct inode *inode, const char *name, + void *buffer, size_t size) +{ + struct afs_vnode *vnode = AFS_FS_I(inode); + const char *volname = vnode->volume->name; + size_t namelen; + + namelen = strlen(volname); + if (size == 0) + return namelen; + if (namelen > size) + return -ERANGE; + memcpy(buffer, volname, namelen); + return namelen; +} + +static const struct xattr_handler afs_xattr_afs_volume_handler = { + .name = "afs.volume", + .get = afs_xattr_get_volume, +}; + +const struct xattr_handler * const afs_xattr_handlers[] = { + &afs_xattr_afs_acl_handler, + &afs_xattr_afs_cell_handler, + &afs_xattr_afs_fid_handler, + &afs_xattr_afs_volume_handler, + &afs_xattr_yfs_handler, /* afs.yfs. prefix */ + NULL +}; diff --git a/fs/afs/xdr_fs.h b/fs/afs/xdr_fs.h new file mode 100644 index 000000000..cc5f143d2 --- /dev/null +++ b/fs/afs/xdr_fs.h @@ -0,0 +1,116 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* AFS fileserver XDR types + * + * Copyright (C) 2018 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#ifndef XDR_FS_H +#define XDR_FS_H + +struct afs_xdr_AFSFetchStatus { + __be32 if_version; +#define AFS_FSTATUS_VERSION 1 + __be32 type; + __be32 nlink; + __be32 size_lo; + __be32 data_version_lo; + __be32 author; + __be32 owner; + __be32 caller_access; + __be32 anon_access; + __be32 mode; + __be32 parent_vnode; + __be32 parent_unique; + __be32 seg_size; + __be32 mtime_client; + __be32 mtime_server; + __be32 group; + __be32 sync_counter; + __be32 data_version_hi; + __be32 lock_count; + __be32 size_hi; + __be32 abort_code; +} __packed; + +#define AFS_DIR_HASHTBL_SIZE 128 +#define AFS_DIR_DIRENT_SIZE 32 +#define AFS_DIR_SLOTS_PER_BLOCK 64 +#define AFS_DIR_BLOCK_SIZE 2048 +#define AFS_DIR_BLOCKS_PER_PAGE (PAGE_SIZE / AFS_DIR_BLOCK_SIZE) +#define AFS_DIR_MAX_SLOTS 65536 +#define AFS_DIR_BLOCKS_WITH_CTR 128 +#define AFS_DIR_MAX_BLOCKS 1023 +#define AFS_DIR_RESV_BLOCKS 1 +#define AFS_DIR_RESV_BLOCKS0 13 + +/* + * Directory entry structure. + */ +union afs_xdr_dirent { + struct { + u8 valid; + u8 unused[1]; + __be16 hash_next; + __be32 vnode; + __be32 unique; + u8 name[]; + /* When determining the number of dirent slots needed to + * represent a directory entry, name should be assumed to be 16 + * bytes, due to a now-standardised (mis)calculation, but it is + * in fact 20 bytes in size. afs_dir_calc_slots() should be + * used for this. + * + * For names longer than (16 or) 20 bytes, extra slots should + * be annexed to this one using the extended_name format. + */ + } u; + u8 extended_name[32]; +} __packed; + +/* + * Directory block header (one at the beginning of every 2048-byte block). + */ +struct afs_xdr_dir_hdr { + __be16 npages; + __be16 magic; +#define AFS_DIR_MAGIC htons(1234) + u8 reserved; + u8 bitmap[8]; + u8 pad[19]; +} __packed; + +/* + * Directory block layout + */ +union afs_xdr_dir_block { + struct afs_xdr_dir_hdr hdr; + + struct { + struct afs_xdr_dir_hdr hdr; + u8 alloc_ctrs[AFS_DIR_BLOCKS_WITH_CTR]; + __be16 hashtable[AFS_DIR_HASHTBL_SIZE]; + } meta; + + union afs_xdr_dirent dirents[AFS_DIR_SLOTS_PER_BLOCK]; +} __packed; + +/* + * Directory layout on a linux VM page. + */ +struct afs_xdr_dir_page { + union afs_xdr_dir_block blocks[AFS_DIR_BLOCKS_PER_PAGE]; +}; + +/* + * Calculate the number of dirent slots required for any given name length. + * The calculation is made assuming the part of the name in the first slot is + * 16 bytes, rather than 20, but this miscalculation is now standardised. + */ +static inline unsigned int afs_dir_calc_slots(size_t name_len) +{ + name_len++; /* NUL-terminated */ + return 1 + ((name_len + 15) / AFS_DIR_DIRENT_SIZE); +} + +#endif /* XDR_FS_H */ diff --git a/fs/afs/yfsclient.c b/fs/afs/yfsclient.c new file mode 100644 index 000000000..d94117973 --- /dev/null +++ b/fs/afs/yfsclient.c @@ -0,0 +1,2232 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* YFS File Server client stubs + * + * Copyright (C) 2018 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include "internal.h" +#include "afs_fs.h" +#include "xdr_fs.h" +#include "protocol_yfs.h" + +#define xdr_size(x) (sizeof(*x) / sizeof(__be32)) + +static void xdr_decode_YFSFid(const __be32 **_bp, struct afs_fid *fid) +{ + const struct yfs_xdr_YFSFid *x = (const void *)*_bp; + + fid->vid = xdr_to_u64(x->volume); + fid->vnode = xdr_to_u64(x->vnode.lo); + fid->vnode_hi = ntohl(x->vnode.hi); + fid->unique = ntohl(x->vnode.unique); + *_bp += xdr_size(x); +} + +static __be32 *xdr_encode_u32(__be32 *bp, u32 n) +{ + *bp++ = htonl(n); + return bp; +} + +static __be32 *xdr_encode_u64(__be32 *bp, u64 n) +{ + struct yfs_xdr_u64 *x = (void *)bp; + + *x = u64_to_xdr(n); + return bp + xdr_size(x); +} + +static __be32 *xdr_encode_YFSFid(__be32 *bp, struct afs_fid *fid) +{ + struct yfs_xdr_YFSFid *x = (void *)bp; + + x->volume = u64_to_xdr(fid->vid); + x->vnode.lo = u64_to_xdr(fid->vnode); + x->vnode.hi = htonl(fid->vnode_hi); + x->vnode.unique = htonl(fid->unique); + return bp + xdr_size(x); +} + +static size_t xdr_strlen(unsigned int len) +{ + return sizeof(__be32) + round_up(len, sizeof(__be32)); +} + +static __be32 *xdr_encode_string(__be32 *bp, const char *p, unsigned int len) +{ + bp = xdr_encode_u32(bp, len); + bp = memcpy(bp, p, len); + if (len & 3) { + unsigned int pad = 4 - (len & 3); + + memset((u8 *)bp + len, 0, pad); + len += pad; + } + + return bp + len / sizeof(__be32); +} + +static __be32 *xdr_encode_name(__be32 *bp, const struct qstr *p) +{ + return xdr_encode_string(bp, p->name, p->len); +} + +static s64 linux_to_yfs_time(const struct timespec64 *t) +{ + /* Convert to 100ns intervals. */ + return (u64)t->tv_sec * 10000000 + t->tv_nsec/100; +} + +static __be32 *xdr_encode_YFSStoreStatus(__be32 *bp, mode_t *mode, + const struct timespec64 *t) +{ + struct yfs_xdr_YFSStoreStatus *x = (void *)bp; + mode_t masked_mode = mode ? *mode & S_IALLUGO : 0; + s64 mtime = linux_to_yfs_time(t); + u32 mask = AFS_SET_MTIME; + + mask |= mode ? AFS_SET_MODE : 0; + + x->mask = htonl(mask); + x->mode = htonl(masked_mode); + x->mtime_client = u64_to_xdr(mtime); + x->owner = u64_to_xdr(0); + x->group = u64_to_xdr(0); + return bp + xdr_size(x); +} + +/* + * Convert a signed 100ns-resolution 64-bit time into a timespec. + */ +static struct timespec64 yfs_time_to_linux(s64 t) +{ + struct timespec64 ts; + u64 abs_t; + + /* + * Unfortunately can not use normal 64 bit division on 32 bit arch, but + * the alternative, do_div, does not work with negative numbers so have + * to special case them + */ + if (t < 0) { + abs_t = -t; + ts.tv_nsec = (time64_t)(do_div(abs_t, 10000000) * 100); + ts.tv_nsec = -ts.tv_nsec; + ts.tv_sec = -abs_t; + } else { + abs_t = t; + ts.tv_nsec = (time64_t)do_div(abs_t, 10000000) * 100; + ts.tv_sec = abs_t; + } + + return ts; +} + +static struct timespec64 xdr_to_time(const struct yfs_xdr_u64 xdr) +{ + s64 t = xdr_to_u64(xdr); + + return yfs_time_to_linux(t); +} + +static void yfs_check_req(struct afs_call *call, __be32 *bp) +{ + size_t len = (void *)bp - call->request; + + if (len > call->request_size) + pr_err("kAFS: %s: Request buffer overflow (%zu>%u)\n", + call->type->name, len, call->request_size); + else if (len < call->request_size) + pr_warn("kAFS: %s: Request buffer underflow (%zu<%u)\n", + call->type->name, len, call->request_size); +} + +/* + * Dump a bad file status record. + */ +static void xdr_dump_bad(const __be32 *bp) +{ + __be32 x[4]; + int i; + + pr_notice("YFS XDR: Bad status record\n"); + for (i = 0; i < 6 * 4 * 4; i += 16) { + memcpy(x, bp, 16); + bp += 4; + pr_notice("%03x: %08x %08x %08x %08x\n", + i, ntohl(x[0]), ntohl(x[1]), ntohl(x[2]), ntohl(x[3])); + } + + memcpy(x, bp, 8); + pr_notice("0x60: %08x %08x\n", ntohl(x[0]), ntohl(x[1])); +} + +/* + * Decode a YFSFetchStatus block + */ +static void xdr_decode_YFSFetchStatus(const __be32 **_bp, + struct afs_call *call, + struct afs_status_cb *scb) +{ + const struct yfs_xdr_YFSFetchStatus *xdr = (const void *)*_bp; + struct afs_file_status *status = &scb->status; + u32 type; + + status->abort_code = ntohl(xdr->abort_code); + if (status->abort_code != 0) { + if (status->abort_code == VNOVNODE) + status->nlink = 0; + scb->have_error = true; + goto advance; + } + + type = ntohl(xdr->type); + switch (type) { + case AFS_FTYPE_FILE: + case AFS_FTYPE_DIR: + case AFS_FTYPE_SYMLINK: + status->type = type; + break; + default: + goto bad; + } + + status->nlink = ntohl(xdr->nlink); + status->author = xdr_to_u64(xdr->author); + status->owner = xdr_to_u64(xdr->owner); + status->caller_access = ntohl(xdr->caller_access); /* Ticket dependent */ + status->anon_access = ntohl(xdr->anon_access); + status->mode = ntohl(xdr->mode) & S_IALLUGO; + status->group = xdr_to_u64(xdr->group); + status->lock_count = ntohl(xdr->lock_count); + + status->mtime_client = xdr_to_time(xdr->mtime_client); + status->mtime_server = xdr_to_time(xdr->mtime_server); + status->size = xdr_to_u64(xdr->size); + status->data_version = xdr_to_u64(xdr->data_version); + scb->have_status = true; +advance: + *_bp += xdr_size(xdr); + return; + +bad: + xdr_dump_bad(*_bp); + afs_protocol_error(call, afs_eproto_bad_status); + goto advance; +} + +/* + * Decode a YFSCallBack block + */ +static void xdr_decode_YFSCallBack(const __be32 **_bp, + struct afs_call *call, + struct afs_status_cb *scb) +{ + struct yfs_xdr_YFSCallBack *x = (void *)*_bp; + struct afs_callback *cb = &scb->callback; + ktime_t cb_expiry; + + cb_expiry = ktime_add(call->issue_time, xdr_to_u64(x->expiration_time) * 100); + cb->expires_at = ktime_divns(cb_expiry, NSEC_PER_SEC); + scb->have_cb = true; + *_bp += xdr_size(x); +} + +/* + * Decode a YFSVolSync block + */ +static void xdr_decode_YFSVolSync(const __be32 **_bp, + struct afs_volsync *volsync) +{ + struct yfs_xdr_YFSVolSync *x = (void *)*_bp; + u64 creation, update; + + if (volsync) { + creation = xdr_to_u64(x->vol_creation_date); + do_div(creation, 10 * 1000 * 1000); + volsync->creation = creation; + update = xdr_to_u64(x->vol_update_date); + do_div(update, 10 * 1000 * 1000); + volsync->update = update; + } + + *_bp += xdr_size(x); +} + +/* + * Encode the requested attributes into a YFSStoreStatus block + */ +static __be32 *xdr_encode_YFS_StoreStatus(__be32 *bp, struct iattr *attr) +{ + struct yfs_xdr_YFSStoreStatus *x = (void *)bp; + s64 mtime = 0, owner = 0, group = 0; + u32 mask = 0, mode = 0; + + mask = 0; + if (attr->ia_valid & ATTR_MTIME) { + mask |= AFS_SET_MTIME; + mtime = linux_to_yfs_time(&attr->ia_mtime); + } + + if (attr->ia_valid & ATTR_UID) { + mask |= AFS_SET_OWNER; + owner = from_kuid(&init_user_ns, attr->ia_uid); + } + + if (attr->ia_valid & ATTR_GID) { + mask |= AFS_SET_GROUP; + group = from_kgid(&init_user_ns, attr->ia_gid); + } + + if (attr->ia_valid & ATTR_MODE) { + mask |= AFS_SET_MODE; + mode = attr->ia_mode & S_IALLUGO; + } + + x->mask = htonl(mask); + x->mode = htonl(mode); + x->mtime_client = u64_to_xdr(mtime); + x->owner = u64_to_xdr(owner); + x->group = u64_to_xdr(group); + return bp + xdr_size(x); +} + +/* + * Decode a YFSFetchVolumeStatus block. + */ +static void xdr_decode_YFSFetchVolumeStatus(const __be32 **_bp, + struct afs_volume_status *vs) +{ + const struct yfs_xdr_YFSFetchVolumeStatus *x = (const void *)*_bp; + u32 flags; + + vs->vid = xdr_to_u64(x->vid); + vs->parent_id = xdr_to_u64(x->parent_id); + flags = ntohl(x->flags); + vs->online = flags & yfs_FVSOnline; + vs->in_service = flags & yfs_FVSInservice; + vs->blessed = flags & yfs_FVSBlessed; + vs->needs_salvage = flags & yfs_FVSNeedsSalvage; + vs->type = ntohl(x->type); + vs->min_quota = 0; + vs->max_quota = xdr_to_u64(x->max_quota); + vs->blocks_in_use = xdr_to_u64(x->blocks_in_use); + vs->part_blocks_avail = xdr_to_u64(x->part_blocks_avail); + vs->part_max_blocks = xdr_to_u64(x->part_max_blocks); + vs->vol_copy_date = xdr_to_u64(x->vol_copy_date); + vs->vol_backup_date = xdr_to_u64(x->vol_backup_date); + *_bp += sizeof(*x) / sizeof(__be32); +} + +/* + * Deliver reply data to operations that just return a file status and a volume + * sync record. + */ +static int yfs_deliver_status_and_volsync(struct afs_call *call) +{ + struct afs_operation *op = call->op; + const __be32 *bp; + int ret; + + ret = afs_transfer_reply(call); + if (ret < 0) + return ret; + + bp = call->buffer; + xdr_decode_YFSFetchStatus(&bp, call, &op->file[0].scb); + xdr_decode_YFSVolSync(&bp, &op->volsync); + + _leave(" = 0 [done]"); + return 0; +} + +/* + * Deliver reply data to an YFS.FetchData64. + */ +static int yfs_deliver_fs_fetch_data64(struct afs_call *call) +{ + struct afs_operation *op = call->op; + struct netfs_io_subrequest *subreq = op->fetch.subreq; + struct afs_vnode_param *vp = &op->file[0]; + const __be32 *bp; + size_t count_before; + int ret; + + _enter("{%u,%zu, %zu/%llu}", + call->unmarshall, call->iov_len, iov_iter_count(call->iter), + call->remaining); + + switch (call->unmarshall) { + case 0: + call->remaining = 0; + afs_extract_to_tmp64(call); + call->unmarshall++; + fallthrough; + + /* Extract the returned data length into ->actual_len. This + * may indicate more or less data than was requested will be + * returned. + */ + case 1: + _debug("extract data length"); + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + call->remaining = be64_to_cpu(call->tmp64); + _debug("DATA length: %llu", call->remaining); + + if (call->remaining == 0) + goto no_more_data; + + call->iter = &subreq->io_iter; + call->iov_len = min(call->remaining, subreq->len - subreq->transferred); + call->unmarshall++; + fallthrough; + + /* extract the returned data */ + case 2: + count_before = call->iov_len; + _debug("extract data %zu/%llu", count_before, call->remaining); + + ret = afs_extract_data(call, true); + subreq->transferred += count_before - call->iov_len; + if (ret < 0) + return ret; + + call->iter = &call->def_iter; + if (call->remaining) + goto no_more_data; + + /* Discard any excess data the server gave us */ + afs_extract_discard(call, call->remaining); + call->unmarshall = 3; + fallthrough; + + case 3: + _debug("extract discard %zu/%llu", + iov_iter_count(call->iter), call->remaining); + + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + no_more_data: + call->unmarshall = 4; + afs_extract_to_buf(call, + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSCallBack) + + sizeof(struct yfs_xdr_YFSVolSync)); + fallthrough; + + /* extract the metadata */ + case 4: + ret = afs_extract_data(call, false); + if (ret < 0) + return ret; + + bp = call->buffer; + xdr_decode_YFSFetchStatus(&bp, call, &vp->scb); + xdr_decode_YFSCallBack(&bp, call, &vp->scb); + xdr_decode_YFSVolSync(&bp, &op->volsync); + + if (subreq->start + subreq->transferred >= vp->scb.status.size) + __set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags); + + call->unmarshall++; + fallthrough; + + case 5: + break; + } + + _leave(" = 0 [done]"); + return 0; +} + +/* + * YFS.FetchData64 operation type + */ +static const struct afs_call_type yfs_RXYFSFetchData64 = { + .name = "YFS.FetchData64", + .op = yfs_FS_FetchData64, + .async_rx = afs_fetch_data_async_rx, + .deliver = yfs_deliver_fs_fetch_data64, + .immediate_cancel = afs_fetch_data_immediate_cancel, + .destructor = afs_flat_call_destructor, +}; + +/* + * Fetch data from a file. + */ +void yfs_fs_fetch_data(struct afs_operation *op) +{ + struct netfs_io_subrequest *subreq = op->fetch.subreq; + struct afs_vnode_param *vp = &op->file[0]; + struct afs_call *call; + __be32 *bp; + + _enter(",%x,{%llx:%llu},%llx,%zx", + key_serial(op->key), vp->fid.vid, vp->fid.vnode, + subreq->start + subreq->transferred, + subreq->len - subreq->transferred); + + call = afs_alloc_flat_call(op->net, &yfs_RXYFSFetchData64, + sizeof(__be32) * 2 + + sizeof(struct yfs_xdr_YFSFid) + + sizeof(struct yfs_xdr_u64) * 2, + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSCallBack) + + sizeof(struct yfs_xdr_YFSVolSync)); + if (!call) + return afs_op_nomem(op); + + if (op->flags & AFS_OPERATION_ASYNC) + call->async = true; + + /* marshall the parameters */ + bp = call->request; + bp = xdr_encode_u32(bp, YFSFETCHDATA64); + bp = xdr_encode_u32(bp, 0); /* RPC flags */ + bp = xdr_encode_YFSFid(bp, &vp->fid); + bp = xdr_encode_u64(bp, subreq->start + subreq->transferred); + bp = xdr_encode_u64(bp, subreq->len - subreq->transferred); + yfs_check_req(call, bp); + + call->fid = vp->fid; + trace_afs_make_fs_call(call, &vp->fid); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * Deliver reply data for YFS.CreateFile or YFS.MakeDir. + */ +static int yfs_deliver_fs_create_vnode(struct afs_call *call) +{ + struct afs_operation *op = call->op; + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_vnode_param *vp = &op->file[1]; + const __be32 *bp; + int ret; + + _enter("{%u}", call->unmarshall); + + ret = afs_transfer_reply(call); + if (ret < 0) + return ret; + + /* unmarshall the reply once we've received all of it */ + bp = call->buffer; + xdr_decode_YFSFid(&bp, &op->file[1].fid); + xdr_decode_YFSFetchStatus(&bp, call, &vp->scb); + xdr_decode_YFSFetchStatus(&bp, call, &dvp->scb); + xdr_decode_YFSCallBack(&bp, call, &vp->scb); + xdr_decode_YFSVolSync(&bp, &op->volsync); + + _leave(" = 0 [done]"); + return 0; +} + +/* + * FS.CreateFile and FS.MakeDir operation type + */ +static const struct afs_call_type afs_RXFSCreateFile = { + .name = "YFS.CreateFile", + .op = yfs_FS_CreateFile, + .deliver = yfs_deliver_fs_create_vnode, + .destructor = afs_flat_call_destructor, +}; + +/* + * Create a file. + */ +void yfs_fs_create_file(struct afs_operation *op) +{ + const struct qstr *name = &op->dentry->d_name; + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_call *call; + size_t reqsz, rplsz; + __be32 *bp; + + _enter(""); + + reqsz = (sizeof(__be32) + + sizeof(__be32) + + sizeof(struct yfs_xdr_YFSFid) + + xdr_strlen(name->len) + + sizeof(struct yfs_xdr_YFSStoreStatus) + + sizeof(__be32)); + rplsz = (sizeof(struct yfs_xdr_YFSFid) + + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSCallBack) + + sizeof(struct yfs_xdr_YFSVolSync)); + + call = afs_alloc_flat_call(op->net, &afs_RXFSCreateFile, reqsz, rplsz); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + bp = xdr_encode_u32(bp, YFSCREATEFILE); + bp = xdr_encode_u32(bp, 0); /* RPC flags */ + bp = xdr_encode_YFSFid(bp, &dvp->fid); + bp = xdr_encode_name(bp, name); + bp = xdr_encode_YFSStoreStatus(bp, &op->create.mode, &op->mtime); + bp = xdr_encode_u32(bp, yfs_LockNone); /* ViceLockType */ + yfs_check_req(call, bp); + + call->fid = dvp->fid; + trace_afs_make_fs_call1(call, &dvp->fid, name); + afs_make_op_call(op, call, GFP_NOFS); +} + +static const struct afs_call_type yfs_RXFSMakeDir = { + .name = "YFS.MakeDir", + .op = yfs_FS_MakeDir, + .deliver = yfs_deliver_fs_create_vnode, + .destructor = afs_flat_call_destructor, +}; + +/* + * Make a directory. + */ +void yfs_fs_make_dir(struct afs_operation *op) +{ + const struct qstr *name = &op->dentry->d_name; + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_call *call; + size_t reqsz, rplsz; + __be32 *bp; + + _enter(""); + + reqsz = (sizeof(__be32) + + sizeof(struct yfs_xdr_RPCFlags) + + sizeof(struct yfs_xdr_YFSFid) + + xdr_strlen(name->len) + + sizeof(struct yfs_xdr_YFSStoreStatus)); + rplsz = (sizeof(struct yfs_xdr_YFSFid) + + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSCallBack) + + sizeof(struct yfs_xdr_YFSVolSync)); + + call = afs_alloc_flat_call(op->net, &yfs_RXFSMakeDir, reqsz, rplsz); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + bp = xdr_encode_u32(bp, YFSMAKEDIR); + bp = xdr_encode_u32(bp, 0); /* RPC flags */ + bp = xdr_encode_YFSFid(bp, &dvp->fid); + bp = xdr_encode_name(bp, name); + bp = xdr_encode_YFSStoreStatus(bp, &op->create.mode, &op->mtime); + yfs_check_req(call, bp); + + call->fid = dvp->fid; + trace_afs_make_fs_call1(call, &dvp->fid, name); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * Deliver reply data to a YFS.RemoveFile2 operation. + */ +static int yfs_deliver_fs_remove_file2(struct afs_call *call) +{ + struct afs_operation *op = call->op; + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_vnode_param *vp = &op->file[1]; + struct afs_fid fid; + const __be32 *bp; + int ret; + + _enter("{%u}", call->unmarshall); + + ret = afs_transfer_reply(call); + if (ret < 0) + return ret; + + bp = call->buffer; + xdr_decode_YFSFetchStatus(&bp, call, &dvp->scb); + xdr_decode_YFSFid(&bp, &fid); + xdr_decode_YFSFetchStatus(&bp, call, &vp->scb); + /* Was deleted if vnode->status.abort_code == VNOVNODE. */ + + xdr_decode_YFSVolSync(&bp, &op->volsync); + return 0; +} + +static void yfs_done_fs_remove_file2(struct afs_call *call) +{ + if (call->error == -ECONNABORTED && + (call->abort_code == RX_INVALID_OPERATION || + call->abort_code == RXGEN_OPCODE)) { + set_bit(AFS_SERVER_FL_NO_RM2, &call->op->server->flags); + call->op->flags |= AFS_OPERATION_DOWNGRADE; + } +} + +/* + * YFS.RemoveFile2 operation type. + */ +static const struct afs_call_type yfs_RXYFSRemoveFile2 = { + .name = "YFS.RemoveFile2", + .op = yfs_FS_RemoveFile2, + .deliver = yfs_deliver_fs_remove_file2, + .done = yfs_done_fs_remove_file2, + .destructor = afs_flat_call_destructor, +}; + +/* + * Remove a file and retrieve new file status. + */ +void yfs_fs_remove_file2(struct afs_operation *op) +{ + struct afs_vnode_param *dvp = &op->file[0]; + const struct qstr *name = &op->dentry->d_name; + struct afs_call *call; + __be32 *bp; + + _enter(""); + + call = afs_alloc_flat_call(op->net, &yfs_RXYFSRemoveFile2, + sizeof(__be32) + + sizeof(struct yfs_xdr_RPCFlags) + + sizeof(struct yfs_xdr_YFSFid) + + xdr_strlen(name->len), + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSFid) + + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSVolSync)); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + bp = xdr_encode_u32(bp, YFSREMOVEFILE2); + bp = xdr_encode_u32(bp, 0); /* RPC flags */ + bp = xdr_encode_YFSFid(bp, &dvp->fid); + bp = xdr_encode_name(bp, name); + yfs_check_req(call, bp); + + call->fid = dvp->fid; + trace_afs_make_fs_call1(call, &dvp->fid, name); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * Deliver reply data to a YFS.RemoveFile or YFS.RemoveDir operation. + */ +static int yfs_deliver_fs_remove(struct afs_call *call) +{ + struct afs_operation *op = call->op; + struct afs_vnode_param *dvp = &op->file[0]; + const __be32 *bp; + int ret; + + _enter("{%u}", call->unmarshall); + + ret = afs_transfer_reply(call); + if (ret < 0) + return ret; + + bp = call->buffer; + xdr_decode_YFSFetchStatus(&bp, call, &dvp->scb); + xdr_decode_YFSVolSync(&bp, &op->volsync); + return 0; +} + +/* + * FS.RemoveDir and FS.RemoveFile operation types. + */ +static const struct afs_call_type yfs_RXYFSRemoveFile = { + .name = "YFS.RemoveFile", + .op = yfs_FS_RemoveFile, + .deliver = yfs_deliver_fs_remove, + .destructor = afs_flat_call_destructor, +}; + +/* + * Remove a file. + */ +void yfs_fs_remove_file(struct afs_operation *op) +{ + const struct qstr *name = &op->dentry->d_name; + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_call *call; + __be32 *bp; + + _enter(""); + + if (!test_bit(AFS_SERVER_FL_NO_RM2, &op->server->flags)) + return yfs_fs_remove_file2(op); + + call = afs_alloc_flat_call(op->net, &yfs_RXYFSRemoveFile, + sizeof(__be32) + + sizeof(struct yfs_xdr_RPCFlags) + + sizeof(struct yfs_xdr_YFSFid) + + xdr_strlen(name->len), + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSVolSync)); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + bp = xdr_encode_u32(bp, YFSREMOVEFILE); + bp = xdr_encode_u32(bp, 0); /* RPC flags */ + bp = xdr_encode_YFSFid(bp, &dvp->fid); + bp = xdr_encode_name(bp, name); + yfs_check_req(call, bp); + + call->fid = dvp->fid; + trace_afs_make_fs_call1(call, &dvp->fid, name); + afs_make_op_call(op, call, GFP_NOFS); +} + +static const struct afs_call_type yfs_RXYFSRemoveDir = { + .name = "YFS.RemoveDir", + .op = yfs_FS_RemoveDir, + .deliver = yfs_deliver_fs_remove, + .destructor = afs_flat_call_destructor, +}; + +/* + * Remove a directory. + */ +void yfs_fs_remove_dir(struct afs_operation *op) +{ + const struct qstr *name = &op->dentry->d_name; + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_call *call; + __be32 *bp; + + _enter(""); + + call = afs_alloc_flat_call(op->net, &yfs_RXYFSRemoveDir, + sizeof(__be32) + + sizeof(struct yfs_xdr_RPCFlags) + + sizeof(struct yfs_xdr_YFSFid) + + xdr_strlen(name->len), + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSVolSync)); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + bp = xdr_encode_u32(bp, YFSREMOVEDIR); + bp = xdr_encode_u32(bp, 0); /* RPC flags */ + bp = xdr_encode_YFSFid(bp, &dvp->fid); + bp = xdr_encode_name(bp, name); + yfs_check_req(call, bp); + + call->fid = dvp->fid; + trace_afs_make_fs_call1(call, &dvp->fid, name); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * Deliver reply data to a YFS.Link operation. + */ +static int yfs_deliver_fs_link(struct afs_call *call) +{ + struct afs_operation *op = call->op; + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_vnode_param *vp = &op->file[1]; + const __be32 *bp; + int ret; + + _enter("{%u}", call->unmarshall); + + ret = afs_transfer_reply(call); + if (ret < 0) + return ret; + + bp = call->buffer; + xdr_decode_YFSFetchStatus(&bp, call, &vp->scb); + xdr_decode_YFSFetchStatus(&bp, call, &dvp->scb); + xdr_decode_YFSVolSync(&bp, &op->volsync); + _leave(" = 0 [done]"); + return 0; +} + +/* + * YFS.Link operation type. + */ +static const struct afs_call_type yfs_RXYFSLink = { + .name = "YFS.Link", + .op = yfs_FS_Link, + .deliver = yfs_deliver_fs_link, + .destructor = afs_flat_call_destructor, +}; + +/* + * Make a hard link. + */ +void yfs_fs_link(struct afs_operation *op) +{ + const struct qstr *name = &op->dentry->d_name; + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_vnode_param *vp = &op->file[1]; + struct afs_call *call; + __be32 *bp; + + _enter(""); + + call = afs_alloc_flat_call(op->net, &yfs_RXYFSLink, + sizeof(__be32) + + sizeof(struct yfs_xdr_RPCFlags) + + sizeof(struct yfs_xdr_YFSFid) + + xdr_strlen(name->len) + + sizeof(struct yfs_xdr_YFSFid), + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSVolSync)); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + bp = xdr_encode_u32(bp, YFSLINK); + bp = xdr_encode_u32(bp, 0); /* RPC flags */ + bp = xdr_encode_YFSFid(bp, &dvp->fid); + bp = xdr_encode_name(bp, name); + bp = xdr_encode_YFSFid(bp, &vp->fid); + yfs_check_req(call, bp); + + call->fid = vp->fid; + trace_afs_make_fs_call1(call, &vp->fid, name); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * Deliver reply data to a YFS.Symlink operation. + */ +static int yfs_deliver_fs_symlink(struct afs_call *call) +{ + struct afs_operation *op = call->op; + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_vnode_param *vp = &op->file[1]; + const __be32 *bp; + int ret; + + _enter("{%u}", call->unmarshall); + + ret = afs_transfer_reply(call); + if (ret < 0) + return ret; + + /* unmarshall the reply once we've received all of it */ + bp = call->buffer; + xdr_decode_YFSFid(&bp, &vp->fid); + xdr_decode_YFSFetchStatus(&bp, call, &vp->scb); + xdr_decode_YFSFetchStatus(&bp, call, &dvp->scb); + xdr_decode_YFSVolSync(&bp, &op->volsync); + + _leave(" = 0 [done]"); + return 0; +} + +/* + * YFS.Symlink operation type + */ +static const struct afs_call_type yfs_RXYFSSymlink = { + .name = "YFS.Symlink", + .op = yfs_FS_Symlink, + .deliver = yfs_deliver_fs_symlink, + .destructor = afs_flat_call_destructor, +}; + +/* + * Create a symbolic link. + */ +void yfs_fs_symlink(struct afs_operation *op) +{ + const struct qstr *name = &op->dentry->d_name; + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_call *call; + size_t contents_sz; + mode_t mode = 0777; + __be32 *bp; + + _enter(""); + + contents_sz = strlen(op->create.symlink->content); + call = afs_alloc_flat_call(op->net, &yfs_RXYFSSymlink, + sizeof(__be32) + + sizeof(struct yfs_xdr_RPCFlags) + + sizeof(struct yfs_xdr_YFSFid) + + xdr_strlen(name->len) + + xdr_strlen(contents_sz) + + sizeof(struct yfs_xdr_YFSStoreStatus), + sizeof(struct yfs_xdr_YFSFid) + + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSVolSync)); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + bp = xdr_encode_u32(bp, YFSSYMLINK); + bp = xdr_encode_u32(bp, 0); /* RPC flags */ + bp = xdr_encode_YFSFid(bp, &dvp->fid); + bp = xdr_encode_name(bp, name); + bp = xdr_encode_string(bp, op->create.symlink->content, contents_sz); + bp = xdr_encode_YFSStoreStatus(bp, &mode, &op->mtime); + yfs_check_req(call, bp); + + call->fid = dvp->fid; + trace_afs_make_fs_call1(call, &dvp->fid, name); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * Deliver reply data to a YFS.Rename operation. + */ +static int yfs_deliver_fs_rename(struct afs_call *call) +{ + struct afs_operation *op = call->op; + struct afs_vnode_param *orig_dvp = &op->file[0]; + struct afs_vnode_param *new_dvp = &op->file[1]; + const __be32 *bp; + int ret; + + _enter("{%u}", call->unmarshall); + + ret = afs_transfer_reply(call); + if (ret < 0) + return ret; + + bp = call->buffer; + /* If the two dirs are the same, we have two copies of the same status + * report, so we just decode it twice. + */ + xdr_decode_YFSFetchStatus(&bp, call, &orig_dvp->scb); + xdr_decode_YFSFetchStatus(&bp, call, &new_dvp->scb); + xdr_decode_YFSVolSync(&bp, &op->volsync); + _leave(" = 0 [done]"); + return 0; +} + +/* + * YFS.Rename operation type + */ +static const struct afs_call_type yfs_RXYFSRename = { + .name = "FS.Rename", + .op = yfs_FS_Rename, + .deliver = yfs_deliver_fs_rename, + .destructor = afs_flat_call_destructor, +}; + +/* + * Rename a file or directory. + */ +void yfs_fs_rename(struct afs_operation *op) +{ + struct afs_vnode_param *orig_dvp = &op->file[0]; + struct afs_vnode_param *new_dvp = &op->file[1]; + const struct qstr *orig_name = &op->dentry->d_name; + const struct qstr *new_name = &op->dentry_2->d_name; + struct afs_call *call; + __be32 *bp; + + _enter(""); + + if (!test_bit(AFS_SERVER_FL_NO_RENAME2, &op->server->flags)) + return yfs_fs_rename_replace(op); + + call = afs_alloc_flat_call(op->net, &yfs_RXYFSRename, + sizeof(__be32) + + sizeof(struct yfs_xdr_RPCFlags) + + sizeof(struct yfs_xdr_YFSFid) + + xdr_strlen(orig_name->len) + + sizeof(struct yfs_xdr_YFSFid) + + xdr_strlen(new_name->len), + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSVolSync)); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + bp = xdr_encode_u32(bp, YFSRENAME); + bp = xdr_encode_u32(bp, 0); /* RPC flags */ + bp = xdr_encode_YFSFid(bp, &orig_dvp->fid); + bp = xdr_encode_name(bp, orig_name); + bp = xdr_encode_YFSFid(bp, &new_dvp->fid); + bp = xdr_encode_name(bp, new_name); + yfs_check_req(call, bp); + + call->fid = orig_dvp->fid; + trace_afs_make_fs_call2(call, &orig_dvp->fid, orig_name, new_name); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * Deliver reply data to a YFS.Rename_NoReplace operation. This does not + * return the status of a displaced target inode as there cannot be one. + */ +static int yfs_deliver_fs_rename_1(struct afs_call *call) +{ + struct afs_operation *op = call->op; + struct afs_vnode_param *orig_dvp = &op->file[0]; + struct afs_vnode_param *new_dvp = &op->file[1]; + struct afs_vnode_param *old_vp = &op->more_files[0]; + const __be32 *bp; + int ret; + + _enter("{%u}", call->unmarshall); + + ret = afs_transfer_reply(call); + if (ret < 0) + return ret; + + bp = call->buffer; + /* If the two dirs are the same, we have two copies of the same status + * report, so we just decode it twice. + */ + xdr_decode_YFSFetchStatus(&bp, call, &orig_dvp->scb); + xdr_decode_YFSFid(&bp, &old_vp->fid); + xdr_decode_YFSFetchStatus(&bp, call, &old_vp->scb); + xdr_decode_YFSFetchStatus(&bp, call, &new_dvp->scb); + xdr_decode_YFSVolSync(&bp, &op->volsync); + _leave(" = 0 [done]"); + return 0; +} + +/* + * Deliver reply data to a YFS.Rename_Replace or a YFS.Rename_Exchange + * operation. These return the status of the displaced target inode if there + * was one. + */ +static int yfs_deliver_fs_rename_2(struct afs_call *call) +{ + struct afs_operation *op = call->op; + struct afs_vnode_param *orig_dvp = &op->file[0]; + struct afs_vnode_param *new_dvp = &op->file[1]; + struct afs_vnode_param *old_vp = &op->more_files[0]; + struct afs_vnode_param *new_vp = &op->more_files[1]; + const __be32 *bp; + int ret; + + _enter("{%u}", call->unmarshall); + + ret = afs_transfer_reply(call); + if (ret < 0) + return ret; + + bp = call->buffer; + /* If the two dirs are the same, we have two copies of the same status + * report, so we just decode it twice. + */ + xdr_decode_YFSFetchStatus(&bp, call, &orig_dvp->scb); + xdr_decode_YFSFid(&bp, &old_vp->fid); + xdr_decode_YFSFetchStatus(&bp, call, &old_vp->scb); + xdr_decode_YFSFetchStatus(&bp, call, &new_dvp->scb); + xdr_decode_YFSFid(&bp, &new_vp->fid); + xdr_decode_YFSFetchStatus(&bp, call, &new_vp->scb); + xdr_decode_YFSVolSync(&bp, &op->volsync); + _leave(" = 0 [done]"); + return 0; +} + +static void yfs_done_fs_rename_replace(struct afs_call *call) +{ + if (call->error == -ECONNABORTED && + (call->abort_code == RX_INVALID_OPERATION || + call->abort_code == RXGEN_OPCODE)) { + set_bit(AFS_SERVER_FL_NO_RENAME2, &call->op->server->flags); + call->op->flags |= AFS_OPERATION_DOWNGRADE; + } +} + +/* + * YFS.Rename_Replace operation type + */ +static const struct afs_call_type yfs_RXYFSRename_Replace = { + .name = "FS.Rename_Replace", + .op = yfs_FS_Rename_Replace, + .deliver = yfs_deliver_fs_rename_2, + .done = yfs_done_fs_rename_replace, + .destructor = afs_flat_call_destructor, +}; + +/* + * YFS.Rename_NoReplace operation type + */ +static const struct afs_call_type yfs_RXYFSRename_NoReplace = { + .name = "FS.Rename_NoReplace", + .op = yfs_FS_Rename_NoReplace, + .deliver = yfs_deliver_fs_rename_1, + .destructor = afs_flat_call_destructor, +}; + +/* + * YFS.Rename_Exchange operation type + */ +static const struct afs_call_type yfs_RXYFSRename_Exchange = { + .name = "FS.Rename_Exchange", + .op = yfs_FS_Rename_Exchange, + .deliver = yfs_deliver_fs_rename_2, + .destructor = afs_flat_call_destructor, +}; + +/* + * Rename a file or directory, replacing the target if it exists. The status + * of a displaced target is returned. + */ +void yfs_fs_rename_replace(struct afs_operation *op) +{ + struct afs_vnode_param *orig_dvp = &op->file[0]; + struct afs_vnode_param *new_dvp = &op->file[1]; + const struct qstr *orig_name = &op->dentry->d_name; + const struct qstr *new_name = &op->dentry_2->d_name; + struct afs_call *call; + __be32 *bp; + + _enter(""); + + call = afs_alloc_flat_call(op->net, &yfs_RXYFSRename_Replace, + sizeof(__be32) + + sizeof(struct yfs_xdr_RPCFlags) + + sizeof(struct yfs_xdr_YFSFid) + + xdr_strlen(orig_name->len) + + sizeof(struct yfs_xdr_YFSFid) + + xdr_strlen(new_name->len), + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSFid) + + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSFid) + + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSVolSync)); + if (!call) + return afs_op_nomem(op); + + /* Marshall the parameters. */ + bp = call->request; + bp = xdr_encode_u32(bp, YFSRENAME_REPLACE); + bp = xdr_encode_u32(bp, 0); /* RPC flags */ + bp = xdr_encode_YFSFid(bp, &orig_dvp->fid); + bp = xdr_encode_name(bp, orig_name); + bp = xdr_encode_YFSFid(bp, &new_dvp->fid); + bp = xdr_encode_name(bp, new_name); + yfs_check_req(call, bp); + + call->fid = orig_dvp->fid; + trace_afs_make_fs_call2(call, &orig_dvp->fid, orig_name, new_name); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * Rename a file or directory, failing if the target dirent exists. + */ +void yfs_fs_rename_noreplace(struct afs_operation *op) +{ + struct afs_vnode_param *orig_dvp = &op->file[0]; + struct afs_vnode_param *new_dvp = &op->file[1]; + const struct qstr *orig_name = &op->dentry->d_name; + const struct qstr *new_name = &op->dentry_2->d_name; + struct afs_call *call; + __be32 *bp; + + _enter(""); + + call = afs_alloc_flat_call(op->net, &yfs_RXYFSRename_NoReplace, + sizeof(__be32) + + sizeof(struct yfs_xdr_RPCFlags) + + sizeof(struct yfs_xdr_YFSFid) + + xdr_strlen(orig_name->len) + + sizeof(struct yfs_xdr_YFSFid) + + xdr_strlen(new_name->len), + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSFid) + + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSVolSync)); + if (!call) + return afs_op_nomem(op); + + /* Marshall the parameters. */ + bp = call->request; + bp = xdr_encode_u32(bp, YFSRENAME_NOREPLACE); + bp = xdr_encode_u32(bp, 0); /* RPC flags */ + bp = xdr_encode_YFSFid(bp, &orig_dvp->fid); + bp = xdr_encode_name(bp, orig_name); + bp = xdr_encode_YFSFid(bp, &new_dvp->fid); + bp = xdr_encode_name(bp, new_name); + yfs_check_req(call, bp); + + call->fid = orig_dvp->fid; + trace_afs_make_fs_call2(call, &orig_dvp->fid, orig_name, new_name); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * Exchange a pair of files directories. + */ +void yfs_fs_rename_exchange(struct afs_operation *op) +{ + struct afs_vnode_param *orig_dvp = &op->file[0]; + struct afs_vnode_param *new_dvp = &op->file[1]; + const struct qstr *orig_name = &op->dentry->d_name; + const struct qstr *new_name = &op->dentry_2->d_name; + struct afs_call *call; + __be32 *bp; + + _enter(""); + + call = afs_alloc_flat_call(op->net, &yfs_RXYFSRename_Exchange, + sizeof(__be32) + + sizeof(struct yfs_xdr_RPCFlags) + + sizeof(struct yfs_xdr_YFSFid) + + xdr_strlen(orig_name->len) + + sizeof(struct yfs_xdr_YFSFid) + + xdr_strlen(new_name->len), + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSFid) + + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSFid) + + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSVolSync)); + if (!call) + return afs_op_nomem(op); + + /* Marshall the parameters. */ + bp = call->request; + bp = xdr_encode_u32(bp, YFSRENAME_EXCHANGE); + bp = xdr_encode_u32(bp, 0); /* RPC flags */ + bp = xdr_encode_YFSFid(bp, &orig_dvp->fid); + bp = xdr_encode_name(bp, orig_name); + bp = xdr_encode_YFSFid(bp, &new_dvp->fid); + bp = xdr_encode_name(bp, new_name); + yfs_check_req(call, bp); + + call->fid = orig_dvp->fid; + trace_afs_make_fs_call2(call, &orig_dvp->fid, orig_name, new_name); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * YFS.StoreData64 operation type. + */ +static const struct afs_call_type yfs_RXYFSStoreData64 = { + .name = "YFS.StoreData64", + .op = yfs_FS_StoreData64, + .deliver = yfs_deliver_status_and_volsync, + .destructor = afs_flat_call_destructor, +}; + +/* + * Store a set of pages to a large file. + */ +void yfs_fs_store_data(struct afs_operation *op) +{ + struct afs_vnode_param *vp = &op->file[0]; + struct afs_call *call; + __be32 *bp; + + _enter(",%x,{%llx:%llu},,", + key_serial(op->key), vp->fid.vid, vp->fid.vnode); + + _debug("size %llx, at %llx, i_size %llx", + (unsigned long long)op->store.size, + (unsigned long long)op->store.pos, + (unsigned long long)op->store.i_size); + + call = afs_alloc_flat_call(op->net, &yfs_RXYFSStoreData64, + sizeof(__be32) + + sizeof(__be32) + + sizeof(struct yfs_xdr_YFSFid) + + sizeof(struct yfs_xdr_YFSStoreStatus) + + sizeof(struct yfs_xdr_u64) * 3, + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSVolSync)); + if (!call) + return afs_op_nomem(op); + + call->write_iter = op->store.write_iter; + + /* marshall the parameters */ + bp = call->request; + bp = xdr_encode_u32(bp, YFSSTOREDATA64); + bp = xdr_encode_u32(bp, 0); /* RPC flags */ + bp = xdr_encode_YFSFid(bp, &vp->fid); + bp = xdr_encode_YFSStoreStatus(bp, NULL, &op->mtime); + bp = xdr_encode_u64(bp, op->store.pos); + bp = xdr_encode_u64(bp, op->store.size); + bp = xdr_encode_u64(bp, op->store.i_size); + yfs_check_req(call, bp); + + call->fid = vp->fid; + trace_afs_make_fs_call(call, &vp->fid); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * YFS.StoreStatus operation type + */ +static const struct afs_call_type yfs_RXYFSStoreStatus = { + .name = "YFS.StoreStatus", + .op = yfs_FS_StoreStatus, + .deliver = yfs_deliver_status_and_volsync, + .destructor = afs_flat_call_destructor, +}; + +static const struct afs_call_type yfs_RXYFSStoreData64_as_Status = { + .name = "YFS.StoreData64", + .op = yfs_FS_StoreData64, + .deliver = yfs_deliver_status_and_volsync, + .destructor = afs_flat_call_destructor, +}; + +/* + * Set the attributes on a file, using YFS.StoreData64 rather than + * YFS.StoreStatus so as to alter the file size also. + */ +static void yfs_fs_setattr_size(struct afs_operation *op) +{ + struct afs_vnode_param *vp = &op->file[0]; + struct afs_call *call; + struct iattr *attr = op->setattr.attr; + __be32 *bp; + + _enter(",%x,{%llx:%llu},,", + key_serial(op->key), vp->fid.vid, vp->fid.vnode); + + call = afs_alloc_flat_call(op->net, &yfs_RXYFSStoreData64_as_Status, + sizeof(__be32) * 2 + + sizeof(struct yfs_xdr_YFSFid) + + sizeof(struct yfs_xdr_YFSStoreStatus) + + sizeof(struct yfs_xdr_u64) * 3, + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSVolSync)); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + bp = xdr_encode_u32(bp, YFSSTOREDATA64); + bp = xdr_encode_u32(bp, 0); /* RPC flags */ + bp = xdr_encode_YFSFid(bp, &vp->fid); + bp = xdr_encode_YFS_StoreStatus(bp, attr); + bp = xdr_encode_u64(bp, attr->ia_size); /* position of start of write */ + bp = xdr_encode_u64(bp, 0); /* size of write */ + bp = xdr_encode_u64(bp, attr->ia_size); /* new file length */ + yfs_check_req(call, bp); + + call->fid = vp->fid; + trace_afs_make_fs_call(call, &vp->fid); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * Set the attributes on a file, using YFS.StoreData64 if there's a change in + * file size, and YFS.StoreStatus otherwise. + */ +void yfs_fs_setattr(struct afs_operation *op) +{ + struct afs_vnode_param *vp = &op->file[0]; + struct afs_call *call; + struct iattr *attr = op->setattr.attr; + __be32 *bp; + + if (attr->ia_valid & ATTR_SIZE) + return yfs_fs_setattr_size(op); + + _enter(",%x,{%llx:%llu},,", + key_serial(op->key), vp->fid.vid, vp->fid.vnode); + + call = afs_alloc_flat_call(op->net, &yfs_RXYFSStoreStatus, + sizeof(__be32) * 2 + + sizeof(struct yfs_xdr_YFSFid) + + sizeof(struct yfs_xdr_YFSStoreStatus), + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSVolSync)); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + bp = xdr_encode_u32(bp, YFSSTORESTATUS); + bp = xdr_encode_u32(bp, 0); /* RPC flags */ + bp = xdr_encode_YFSFid(bp, &vp->fid); + bp = xdr_encode_YFS_StoreStatus(bp, attr); + yfs_check_req(call, bp); + + call->fid = vp->fid; + trace_afs_make_fs_call(call, &vp->fid); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * Deliver reply data to a YFS.GetVolumeStatus operation. + */ +static int yfs_deliver_fs_get_volume_status(struct afs_call *call) +{ + struct afs_operation *op = call->op; + const __be32 *bp; + char *p; + u32 size; + int ret; + + _enter("{%u}", call->unmarshall); + + switch (call->unmarshall) { + case 0: + call->unmarshall++; + afs_extract_to_buf(call, sizeof(struct yfs_xdr_YFSFetchVolumeStatus)); + fallthrough; + + /* extract the returned status record */ + case 1: + _debug("extract status"); + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + bp = call->buffer; + xdr_decode_YFSFetchVolumeStatus(&bp, &op->volstatus.vs); + call->unmarshall++; + afs_extract_to_tmp(call); + fallthrough; + + /* extract the volume name length */ + case 2: + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + call->count = ntohl(call->tmp); + _debug("volname length: %u", call->count); + if (call->count >= AFSNAMEMAX) + return afs_protocol_error(call, afs_eproto_volname_len); + size = (call->count + 3) & ~3; /* It's padded */ + afs_extract_to_buf(call, size); + call->unmarshall++; + fallthrough; + + /* extract the volume name */ + case 3: + _debug("extract volname"); + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + p = call->buffer; + p[call->count] = 0; + _debug("volname '%s'", p); + afs_extract_to_tmp(call); + call->unmarshall++; + fallthrough; + + /* extract the offline message length */ + case 4: + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + call->count = ntohl(call->tmp); + _debug("offline msg length: %u", call->count); + if (call->count >= AFSNAMEMAX) + return afs_protocol_error(call, afs_eproto_offline_msg_len); + size = (call->count + 3) & ~3; /* It's padded */ + afs_extract_to_buf(call, size); + call->unmarshall++; + fallthrough; + + /* extract the offline message */ + case 5: + _debug("extract offline"); + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + p = call->buffer; + p[call->count] = 0; + _debug("offline '%s'", p); + + afs_extract_to_tmp(call); + call->unmarshall++; + fallthrough; + + /* extract the message of the day length */ + case 6: + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + call->count = ntohl(call->tmp); + _debug("motd length: %u", call->count); + if (call->count >= AFSNAMEMAX) + return afs_protocol_error(call, afs_eproto_motd_len); + size = (call->count + 3) & ~3; /* It's padded */ + afs_extract_to_buf(call, size); + call->unmarshall++; + fallthrough; + + /* extract the message of the day */ + case 7: + _debug("extract motd"); + ret = afs_extract_data(call, false); + if (ret < 0) + return ret; + + p = call->buffer; + p[call->count] = 0; + _debug("motd '%s'", p); + + call->unmarshall++; + fallthrough; + + case 8: + break; + } + + _leave(" = 0 [done]"); + return 0; +} + +/* + * YFS.GetVolumeStatus operation type + */ +static const struct afs_call_type yfs_RXYFSGetVolumeStatus = { + .name = "YFS.GetVolumeStatus", + .op = yfs_FS_GetVolumeStatus, + .deliver = yfs_deliver_fs_get_volume_status, + .destructor = afs_flat_call_destructor, +}; + +/* + * fetch the status of a volume + */ +void yfs_fs_get_volume_status(struct afs_operation *op) +{ + struct afs_vnode_param *vp = &op->file[0]; + struct afs_call *call; + __be32 *bp; + + _enter(""); + + call = afs_alloc_flat_call(op->net, &yfs_RXYFSGetVolumeStatus, + sizeof(__be32) * 2 + + sizeof(struct yfs_xdr_u64), + max_t(size_t, + sizeof(struct yfs_xdr_YFSFetchVolumeStatus) + + sizeof(__be32), + AFSOPAQUEMAX + 1)); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + bp = xdr_encode_u32(bp, YFSGETVOLUMESTATUS); + bp = xdr_encode_u32(bp, 0); /* RPC flags */ + bp = xdr_encode_u64(bp, vp->fid.vid); + yfs_check_req(call, bp); + + call->fid = vp->fid; + trace_afs_make_fs_call(call, &vp->fid); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * YFS.SetLock operation type + */ +static const struct afs_call_type yfs_RXYFSSetLock = { + .name = "YFS.SetLock", + .op = yfs_FS_SetLock, + .deliver = yfs_deliver_status_and_volsync, + .done = afs_lock_op_done, + .destructor = afs_flat_call_destructor, +}; + +/* + * YFS.ExtendLock operation type + */ +static const struct afs_call_type yfs_RXYFSExtendLock = { + .name = "YFS.ExtendLock", + .op = yfs_FS_ExtendLock, + .deliver = yfs_deliver_status_and_volsync, + .done = afs_lock_op_done, + .destructor = afs_flat_call_destructor, +}; + +/* + * YFS.ReleaseLock operation type + */ +static const struct afs_call_type yfs_RXYFSReleaseLock = { + .name = "YFS.ReleaseLock", + .op = yfs_FS_ReleaseLock, + .deliver = yfs_deliver_status_and_volsync, + .destructor = afs_flat_call_destructor, +}; + +/* + * Set a lock on a file + */ +void yfs_fs_set_lock(struct afs_operation *op) +{ + struct afs_vnode_param *vp = &op->file[0]; + struct afs_call *call; + __be32 *bp; + + _enter(""); + + call = afs_alloc_flat_call(op->net, &yfs_RXYFSSetLock, + sizeof(__be32) * 2 + + sizeof(struct yfs_xdr_YFSFid) + + sizeof(__be32), + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSVolSync)); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + bp = xdr_encode_u32(bp, YFSSETLOCK); + bp = xdr_encode_u32(bp, 0); /* RPC flags */ + bp = xdr_encode_YFSFid(bp, &vp->fid); + bp = xdr_encode_u32(bp, op->lock.type); + yfs_check_req(call, bp); + + call->fid = vp->fid; + trace_afs_make_fs_calli(call, &vp->fid, op->lock.type); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * extend a lock on a file + */ +void yfs_fs_extend_lock(struct afs_operation *op) +{ + struct afs_vnode_param *vp = &op->file[0]; + struct afs_call *call; + __be32 *bp; + + _enter(""); + + call = afs_alloc_flat_call(op->net, &yfs_RXYFSExtendLock, + sizeof(__be32) * 2 + + sizeof(struct yfs_xdr_YFSFid), + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSVolSync)); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + bp = xdr_encode_u32(bp, YFSEXTENDLOCK); + bp = xdr_encode_u32(bp, 0); /* RPC flags */ + bp = xdr_encode_YFSFid(bp, &vp->fid); + yfs_check_req(call, bp); + + call->fid = vp->fid; + trace_afs_make_fs_call(call, &vp->fid); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * release a lock on a file + */ +void yfs_fs_release_lock(struct afs_operation *op) +{ + struct afs_vnode_param *vp = &op->file[0]; + struct afs_call *call; + __be32 *bp; + + _enter(""); + + call = afs_alloc_flat_call(op->net, &yfs_RXYFSReleaseLock, + sizeof(__be32) * 2 + + sizeof(struct yfs_xdr_YFSFid), + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSVolSync)); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + bp = xdr_encode_u32(bp, YFSRELEASELOCK); + bp = xdr_encode_u32(bp, 0); /* RPC flags */ + bp = xdr_encode_YFSFid(bp, &vp->fid); + yfs_check_req(call, bp); + + call->fid = vp->fid; + trace_afs_make_fs_call(call, &vp->fid); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * Deliver a reply to YFS.FetchStatus + */ +static int yfs_deliver_fs_fetch_status(struct afs_call *call) +{ + struct afs_operation *op = call->op; + struct afs_vnode_param *vp = &op->file[op->fetch_status.which]; + const __be32 *bp; + int ret; + + ret = afs_transfer_reply(call); + if (ret < 0) + return ret; + + /* unmarshall the reply once we've received all of it */ + bp = call->buffer; + xdr_decode_YFSFetchStatus(&bp, call, &vp->scb); + xdr_decode_YFSCallBack(&bp, call, &vp->scb); + xdr_decode_YFSVolSync(&bp, &op->volsync); + + _leave(" = 0 [done]"); + return 0; +} + +/* + * YFS.FetchStatus operation type + */ +static const struct afs_call_type yfs_RXYFSFetchStatus = { + .name = "YFS.FetchStatus", + .op = yfs_FS_FetchStatus, + .deliver = yfs_deliver_fs_fetch_status, + .destructor = afs_flat_call_destructor, +}; + +/* + * Fetch the status information for a fid without needing a vnode handle. + */ +void yfs_fs_fetch_status(struct afs_operation *op) +{ + struct afs_vnode_param *vp = &op->file[op->fetch_status.which]; + struct afs_call *call; + __be32 *bp; + + _enter(",%x,{%llx:%llu},,", + key_serial(op->key), vp->fid.vid, vp->fid.vnode); + + call = afs_alloc_flat_call(op->net, &yfs_RXYFSFetchStatus, + sizeof(__be32) * 2 + + sizeof(struct yfs_xdr_YFSFid), + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSCallBack) + + sizeof(struct yfs_xdr_YFSVolSync)); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + bp = xdr_encode_u32(bp, YFSFETCHSTATUS); + bp = xdr_encode_u32(bp, 0); /* RPC flags */ + bp = xdr_encode_YFSFid(bp, &vp->fid); + yfs_check_req(call, bp); + + call->fid = vp->fid; + trace_afs_make_fs_call(call, &vp->fid); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * Deliver reply data to an YFS.InlineBulkStatus call + */ +static int yfs_deliver_fs_inline_bulk_status(struct afs_call *call) +{ + struct afs_operation *op = call->op; + struct afs_status_cb *scb; + const __be32 *bp; + u32 tmp; + int ret; + + _enter("{%u}", call->unmarshall); + + switch (call->unmarshall) { + case 0: + afs_extract_to_tmp(call); + call->unmarshall++; + fallthrough; + + /* Extract the file status count and array in two steps */ + case 1: + _debug("extract status count"); + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + tmp = ntohl(call->tmp); + _debug("status count: %u/%u", tmp, op->nr_files); + if (tmp != op->nr_files) + return afs_protocol_error(call, afs_eproto_ibulkst_count); + + call->count = 0; + call->unmarshall++; + more_counts: + afs_extract_to_buf(call, sizeof(struct yfs_xdr_YFSFetchStatus)); + fallthrough; + + case 2: + _debug("extract status array %u", call->count); + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + switch (call->count) { + case 0: + scb = &op->file[0].scb; + break; + case 1: + scb = &op->file[1].scb; + break; + default: + scb = &op->more_files[call->count - 2].scb; + break; + } + + bp = call->buffer; + xdr_decode_YFSFetchStatus(&bp, call, scb); + + call->count++; + if (call->count < op->nr_files) + goto more_counts; + + call->count = 0; + call->unmarshall++; + afs_extract_to_tmp(call); + fallthrough; + + /* Extract the callback count and array in two steps */ + case 3: + _debug("extract CB count"); + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + tmp = ntohl(call->tmp); + _debug("CB count: %u", tmp); + if (tmp != op->nr_files) + return afs_protocol_error(call, afs_eproto_ibulkst_cb_count); + call->count = 0; + call->unmarshall++; + more_cbs: + afs_extract_to_buf(call, sizeof(struct yfs_xdr_YFSCallBack)); + fallthrough; + + case 4: + _debug("extract CB array"); + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + _debug("unmarshall CB array"); + switch (call->count) { + case 0: + scb = &op->file[0].scb; + break; + case 1: + scb = &op->file[1].scb; + break; + default: + scb = &op->more_files[call->count - 2].scb; + break; + } + + bp = call->buffer; + xdr_decode_YFSCallBack(&bp, call, scb); + call->count++; + if (call->count < op->nr_files) + goto more_cbs; + + afs_extract_to_buf(call, sizeof(struct yfs_xdr_YFSVolSync)); + call->unmarshall++; + fallthrough; + + case 5: + ret = afs_extract_data(call, false); + if (ret < 0) + return ret; + + bp = call->buffer; + xdr_decode_YFSVolSync(&bp, &op->volsync); + + call->unmarshall++; + fallthrough; + + case 6: + break; + } + + _leave(" = 0 [done]"); + return 0; +} + +/* + * FS.InlineBulkStatus operation type + */ +static const struct afs_call_type yfs_RXYFSInlineBulkStatus = { + .name = "YFS.InlineBulkStatus", + .op = yfs_FS_InlineBulkStatus, + .deliver = yfs_deliver_fs_inline_bulk_status, + .destructor = afs_flat_call_destructor, +}; + +/* + * Fetch the status information for up to 1024 files + */ +void yfs_fs_inline_bulk_status(struct afs_operation *op) +{ + struct afs_vnode_param *dvp = &op->file[0]; + struct afs_vnode_param *vp = &op->file[1]; + struct afs_call *call; + __be32 *bp; + int i; + + _enter(",%x,{%llx:%llu},%u", + key_serial(op->key), vp->fid.vid, vp->fid.vnode, op->nr_files); + + call = afs_alloc_flat_call(op->net, &yfs_RXYFSInlineBulkStatus, + sizeof(__be32) + + sizeof(__be32) + + sizeof(__be32) + + sizeof(struct yfs_xdr_YFSFid) * op->nr_files, + sizeof(struct yfs_xdr_YFSFetchStatus)); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + bp = xdr_encode_u32(bp, YFSINLINEBULKSTATUS); + bp = xdr_encode_u32(bp, 0); /* RPCFlags */ + bp = xdr_encode_u32(bp, op->nr_files); + bp = xdr_encode_YFSFid(bp, &dvp->fid); + bp = xdr_encode_YFSFid(bp, &vp->fid); + for (i = 0; i < op->nr_files - 2; i++) + bp = xdr_encode_YFSFid(bp, &op->more_files[i].fid); + yfs_check_req(call, bp); + + call->fid = vp->fid; + trace_afs_make_fs_call(call, &vp->fid); + afs_make_op_call(op, call, GFP_NOFS); +} + +/* + * Deliver reply data to an YFS.FetchOpaqueACL. + */ +static int yfs_deliver_fs_fetch_opaque_acl(struct afs_call *call) +{ + struct afs_operation *op = call->op; + struct afs_vnode_param *vp = &op->file[0]; + struct yfs_acl *yacl = op->yacl; + struct afs_acl *acl; + const __be32 *bp; + unsigned int size; + int ret; + + _enter("{%u}", call->unmarshall); + + switch (call->unmarshall) { + case 0: + afs_extract_to_tmp(call); + call->unmarshall++; + fallthrough; + + /* Extract the file ACL length */ + case 1: + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + size = call->count2 = ntohl(call->tmp); + size = round_up(size, 4); + + if (yacl->flags & YFS_ACL_WANT_ACL) { + acl = kmalloc_flex(*acl, data, size); + if (!acl) + return -ENOMEM; + yacl->acl = acl; + acl->size = call->count2; + afs_extract_begin(call, acl->data, size); + } else { + afs_extract_discard(call, size); + } + call->unmarshall++; + fallthrough; + + /* Extract the file ACL */ + case 2: + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + afs_extract_to_tmp(call); + call->unmarshall++; + fallthrough; + + /* Extract the volume ACL length */ + case 3: + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + size = call->count2 = ntohl(call->tmp); + size = round_up(size, 4); + + if (yacl->flags & YFS_ACL_WANT_VOL_ACL) { + acl = kmalloc_flex(*acl, data, size); + if (!acl) + return -ENOMEM; + yacl->vol_acl = acl; + acl->size = call->count2; + afs_extract_begin(call, acl->data, size); + } else { + afs_extract_discard(call, size); + } + call->unmarshall++; + fallthrough; + + /* Extract the volume ACL */ + case 4: + ret = afs_extract_data(call, true); + if (ret < 0) + return ret; + + afs_extract_to_buf(call, + sizeof(__be32) * 2 + + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSVolSync)); + call->unmarshall++; + fallthrough; + + /* extract the metadata */ + case 5: + ret = afs_extract_data(call, false); + if (ret < 0) + return ret; + + bp = call->buffer; + yacl->inherit_flag = ntohl(*bp++); + yacl->num_cleaned = ntohl(*bp++); + xdr_decode_YFSFetchStatus(&bp, call, &vp->scb); + xdr_decode_YFSVolSync(&bp, &op->volsync); + + call->unmarshall++; + fallthrough; + + case 6: + break; + } + + _leave(" = 0 [done]"); + return 0; +} + +void yfs_free_opaque_acl(struct yfs_acl *yacl) +{ + if (yacl) { + kfree(yacl->acl); + kfree(yacl->vol_acl); + kfree(yacl); + } +} + +/* + * YFS.FetchOpaqueACL operation type + */ +static const struct afs_call_type yfs_RXYFSFetchOpaqueACL = { + .name = "YFS.FetchOpaqueACL", + .op = yfs_FS_FetchOpaqueACL, + .deliver = yfs_deliver_fs_fetch_opaque_acl, + .destructor = afs_flat_call_destructor, +}; + +/* + * Fetch the YFS advanced ACLs for a file. + */ +void yfs_fs_fetch_opaque_acl(struct afs_operation *op) +{ + struct afs_vnode_param *vp = &op->file[0]; + struct afs_call *call; + __be32 *bp; + + _enter(",%x,{%llx:%llu},,", + key_serial(op->key), vp->fid.vid, vp->fid.vnode); + + call = afs_alloc_flat_call(op->net, &yfs_RXYFSFetchOpaqueACL, + sizeof(__be32) * 2 + + sizeof(struct yfs_xdr_YFSFid), + sizeof(__be32) * 2 + + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSVolSync)); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + bp = xdr_encode_u32(bp, YFSFETCHOPAQUEACL); + bp = xdr_encode_u32(bp, 0); /* RPC flags */ + bp = xdr_encode_YFSFid(bp, &vp->fid); + yfs_check_req(call, bp); + + call->fid = vp->fid; + trace_afs_make_fs_call(call, &vp->fid); + afs_make_op_call(op, call, GFP_KERNEL); +} + +/* + * YFS.StoreOpaqueACL2 operation type + */ +static const struct afs_call_type yfs_RXYFSStoreOpaqueACL2 = { + .name = "YFS.StoreOpaqueACL2", + .op = yfs_FS_StoreOpaqueACL2, + .deliver = yfs_deliver_status_and_volsync, + .destructor = afs_flat_call_destructor, +}; + +/* + * Fetch the YFS ACL for a file. + */ +void yfs_fs_store_opaque_acl2(struct afs_operation *op) +{ + struct afs_vnode_param *vp = &op->file[0]; + struct afs_call *call; + struct afs_acl *acl = op->acl; + size_t size; + __be32 *bp; + + _enter(",%x,{%llx:%llu},,", + key_serial(op->key), vp->fid.vid, vp->fid.vnode); + + size = round_up(acl->size, 4); + call = afs_alloc_flat_call(op->net, &yfs_RXYFSStoreOpaqueACL2, + sizeof(__be32) * 2 + + sizeof(struct yfs_xdr_YFSFid) + + sizeof(__be32) + size, + sizeof(struct yfs_xdr_YFSFetchStatus) + + sizeof(struct yfs_xdr_YFSVolSync)); + if (!call) + return afs_op_nomem(op); + + /* marshall the parameters */ + bp = call->request; + bp = xdr_encode_u32(bp, YFSSTOREOPAQUEACL2); + bp = xdr_encode_u32(bp, 0); /* RPC flags */ + bp = xdr_encode_YFSFid(bp, &vp->fid); + bp = xdr_encode_u32(bp, acl->size); + memcpy(bp, acl->data, acl->size); + if (acl->size != size) + memset((void *)bp + acl->size, 0, size - acl->size); + bp += size / sizeof(__be32); + yfs_check_req(call, bp); + + call->fid = vp->fid; + trace_afs_make_fs_call(call, &vp->fid); + afs_make_op_call(op, call, GFP_KERNEL); +} diff --git a/fs/aio.c b/fs/aio.c new file mode 100644 index 000000000..d78acc69f --- /dev/null +++ b/fs/aio.c @@ -0,0 +1,2498 @@ +/* + * An async IO implementation for Linux + * Written by Benjamin LaHaise + * + * Implements an efficient asynchronous io interface. + * + * Copyright 2000, 2001, 2002 Red Hat, Inc. All Rights Reserved. + * Copyright 2018 Christoph Hellwig. + * + * See ../COPYING for licensing terms. + */ +#define pr_fmt(fmt) "%s: " fmt, __func__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "internal.h" + +#define KIOCB_KEY 0 + +#define AIO_RING_MAGIC 0xa10a10a1 +#define AIO_RING_COMPAT_FEATURES 1 +#define AIO_RING_INCOMPAT_FEATURES 0 +struct aio_ring { + unsigned id; /* kernel internal index number */ + unsigned nr; /* number of io_events */ + unsigned head; /* Written to by userland or under ring_lock + * mutex by aio_read_events_ring(). */ + unsigned tail; + + unsigned magic; + unsigned compat_features; + unsigned incompat_features; + unsigned header_length; /* size of aio_ring */ + + + struct io_event io_events[]; +}; /* 128 bytes + ring size */ + +/* + * Plugging is meant to work with larger batches of IOs. If we don't + * have more than the below, then don't bother setting up a plug. + */ +#define AIO_PLUG_THRESHOLD 2 + +#define AIO_RING_PAGES 8 + +struct kioctx_table { + struct rcu_head rcu; + unsigned nr; + struct kioctx __rcu *table[] __counted_by(nr); +}; + +struct kioctx_cpu { + unsigned reqs_available; +}; + +struct ctx_rq_wait { + struct completion comp; + atomic_t count; +}; + +struct kioctx { + struct percpu_ref users; + atomic_t dead; + + struct percpu_ref reqs; + + unsigned long user_id; + + struct kioctx_cpu __percpu *cpu; + + /* + * For percpu reqs_available, number of slots we move to/from global + * counter at a time: + */ + unsigned req_batch; + /* + * This is what userspace passed to io_setup(), it's not used for + * anything but counting against the global max_reqs quota. + * + * The real limit is nr_events - 1, which will be larger (see + * aio_setup_ring()) + */ + unsigned max_reqs; + + /* Size of ringbuffer, in units of struct io_event */ + unsigned nr_events; + + unsigned long mmap_base; + unsigned long mmap_size; + + struct folio **ring_folios; + long nr_pages; + + struct rcu_work free_rwork; /* see free_ioctx() */ + + /* + * signals when all in-flight requests are done + */ + struct ctx_rq_wait *rq_wait; + + struct { + /* + * This counts the number of available slots in the ringbuffer, + * so we avoid overflowing it: it's decremented (if positive) + * when allocating a kiocb and incremented when the resulting + * io_event is pulled off the ringbuffer. + * + * We batch accesses to it with a percpu version. + */ + atomic_t reqs_available; + } ____cacheline_aligned_in_smp; + + struct { + spinlock_t ctx_lock; + struct list_head active_reqs; /* used for cancellation */ + } ____cacheline_aligned_in_smp; + + struct { + struct mutex ring_lock; + wait_queue_head_t wait; + } ____cacheline_aligned_in_smp; + + struct { + unsigned tail; + unsigned completed_events; + spinlock_t completion_lock; + } ____cacheline_aligned_in_smp; + + struct folio *internal_folios[AIO_RING_PAGES]; + struct file *aio_ring_file; + + unsigned id; +}; + +/* + * First field must be the file pointer in all the + * iocb unions! See also 'struct kiocb' in + */ +struct fsync_iocb { + struct file *file; + struct work_struct work; + bool datasync; + struct cred *creds; +}; + +struct poll_iocb { + struct file *file; + struct wait_queue_head *head; + __poll_t events; + bool cancelled; + bool work_scheduled; + bool work_need_resched; + struct wait_queue_entry wait; + struct work_struct work; +}; + +/* + * NOTE! Each of the iocb union members has the file pointer + * as the first entry in their struct definition. So you can + * access the file pointer through any of the sub-structs, + * or directly as just 'ki_filp' in this struct. + */ +struct aio_kiocb { + union { + struct file *ki_filp; + struct kiocb rw; + struct fsync_iocb fsync; + struct poll_iocb poll; + }; + + struct kioctx *ki_ctx; + kiocb_cancel_fn *ki_cancel; + + struct io_event ki_res; + + struct list_head ki_list; /* the aio core uses this + * for cancellation */ + refcount_t ki_refcnt; + + /* + * If the aio_resfd field of the userspace iocb is not zero, + * this is the underlying eventfd context to deliver events to. + */ + struct eventfd_ctx *ki_eventfd; +}; + +struct aio_inode_info { + struct inode vfs_inode; + spinlock_t migrate_lock; + struct kioctx *ctx; +}; + +static inline struct aio_inode_info *AIO_I(struct inode *inode) +{ + return container_of(inode, struct aio_inode_info, vfs_inode); +} + +/*------ sysctl variables----*/ +static DEFINE_SPINLOCK(aio_nr_lock); +static unsigned long aio_nr; /* current system wide number of aio requests */ +static unsigned long aio_max_nr = 0x10000; /* system wide maximum number of aio requests */ +/*----end sysctl variables---*/ +#ifdef CONFIG_SYSCTL +static const struct ctl_table aio_sysctls[] = { + { + .procname = "aio-nr", + .data = &aio_nr, + .maxlen = sizeof(aio_nr), + .mode = 0444, + .proc_handler = proc_doulongvec_minmax, + }, + { + .procname = "aio-max-nr", + .data = &aio_max_nr, + .maxlen = sizeof(aio_max_nr), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + }, +}; + +static void __init aio_sysctl_init(void) +{ + register_sysctl_init("fs", aio_sysctls); +} +#else +#define aio_sysctl_init() do { } while (0) +#endif + +static struct kmem_cache *kiocb_cachep; +static struct kmem_cache *kioctx_cachep; +static struct kmem_cache *aio_inode_cachep; + +static struct vfsmount *aio_mnt; + +static const struct file_operations aio_ring_fops; +static const struct address_space_operations aio_ctx_aops; + +static struct file *aio_private_file(struct kioctx *ctx, loff_t nr_pages) +{ + struct file *file; + struct inode *inode = alloc_anon_inode(aio_mnt->mnt_sb); + + if (IS_ERR(inode)) + return ERR_CAST(inode); + + inode->i_mapping->a_ops = &aio_ctx_aops; + AIO_I(inode)->ctx = ctx; + inode->i_size = PAGE_SIZE * nr_pages; + + file = alloc_file_pseudo(inode, aio_mnt, "[aio]", + O_RDWR, &aio_ring_fops); + if (IS_ERR(file)) + iput(inode); + return file; +} + +static struct inode *aio_alloc_inode(struct super_block *sb) +{ + struct aio_inode_info *ai; + + ai = alloc_inode_sb(sb, aio_inode_cachep, GFP_KERNEL); + if (!ai) + return NULL; + ai->ctx = NULL; + + return &ai->vfs_inode; +} + +static void aio_free_inode(struct inode *inode) +{ + kmem_cache_free(aio_inode_cachep, AIO_I(inode)); +} + +static const struct super_operations aio_super_operations = { + .alloc_inode = aio_alloc_inode, + .free_inode = aio_free_inode, + .statfs = simple_statfs, +}; + +static int aio_init_fs_context(struct fs_context *fc) +{ + struct pseudo_fs_context *pfc; + + pfc = init_pseudo(fc, AIO_RING_MAGIC); + if (!pfc) + return -ENOMEM; + pfc->ops = &aio_super_operations; + return 0; +} + +static void init_once(void *obj) +{ + struct aio_inode_info *ai = obj; + + inode_init_once(&ai->vfs_inode); + spin_lock_init(&ai->migrate_lock); +} + +/* aio_setup + * Creates the slab caches used by the aio routines, panic on + * failure as this is done early during the boot sequence. + */ +static int __init aio_setup(void) +{ + static struct file_system_type aio_fs = { + .name = "aio", + .init_fs_context = aio_init_fs_context, + .kill_sb = kill_anon_super, + }; + + aio_inode_cachep = kmem_cache_create("aio_inode_cache", + sizeof(struct aio_inode_info), 0, + (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_ACCOUNT), + init_once); + aio_mnt = kern_mount(&aio_fs); + if (IS_ERR(aio_mnt)) + panic("Failed to create aio fs mount."); + + kiocb_cachep = KMEM_CACHE(aio_kiocb, SLAB_HWCACHE_ALIGN|SLAB_PANIC); + kioctx_cachep = KMEM_CACHE(kioctx,SLAB_HWCACHE_ALIGN|SLAB_PANIC); + aio_sysctl_init(); + return 0; +} +__initcall(aio_setup); + +static void put_aio_ring_file(struct kioctx *ctx) +{ + struct file *aio_ring_file = ctx->aio_ring_file; + + if (aio_ring_file) { + struct inode *inode = file_inode(aio_ring_file); + + truncate_setsize(inode, 0); + + /* Prevent further access to the kioctx from migratepages */ + spin_lock(&AIO_I(inode)->migrate_lock); + AIO_I(inode)->ctx = NULL; + ctx->aio_ring_file = NULL; + spin_unlock(&AIO_I(inode)->migrate_lock); + + fput(aio_ring_file); + } +} + +static void aio_free_ring(struct kioctx *ctx) +{ + int i; + + /* Disconnect the kiotx from the ring file. This prevents future + * accesses to the kioctx from page migration. + */ + put_aio_ring_file(ctx); + + for (i = 0; i < ctx->nr_pages; i++) { + struct folio *folio = ctx->ring_folios[i]; + + if (!folio) + continue; + + pr_debug("pid(%d) [%d] folio->count=%d\n", current->pid, i, + folio_ref_count(folio)); + ctx->ring_folios[i] = NULL; + folio_put(folio); + } + + if (ctx->ring_folios && ctx->ring_folios != ctx->internal_folios) { + kfree(ctx->ring_folios); + ctx->ring_folios = NULL; + } +} + +static int aio_ring_mremap(struct vm_area_struct *vma) +{ + struct file *file = vma->vm_file; + struct mm_struct *mm = vma->vm_mm; + struct kioctx_table *table; + int i, res = -EINVAL; + + spin_lock(&mm->ioctx_lock); + rcu_read_lock(); + table = rcu_dereference(mm->ioctx_table); + if (!table) + goto out_unlock; + + for (i = 0; i < table->nr; i++) { + struct kioctx *ctx; + + ctx = rcu_dereference(table->table[i]); + if (ctx && ctx->aio_ring_file == file) { + if (!atomic_read(&ctx->dead) && + (ctx->mmap_size == (vma->vm_end - vma->vm_start))) { + ctx->user_id = ctx->mmap_base = vma->vm_start; + res = 0; + } + break; + } + } + +out_unlock: + rcu_read_unlock(); + spin_unlock(&mm->ioctx_lock); + return res; +} + +static const struct vm_operations_struct aio_ring_vm_ops = { + .mremap = aio_ring_mremap, +#if IS_ENABLED(CONFIG_MMU) + .fault = filemap_fault, + .map_pages = filemap_map_pages, + .page_mkwrite = filemap_page_mkwrite, +#endif +}; + +static int aio_ring_mmap_prepare(struct vm_area_desc *desc) +{ + vma_desc_set_flags(desc, VMA_DONTEXPAND_BIT, VMA_DONTCOPY_BIT); + desc->vm_ops = &aio_ring_vm_ops; + return 0; +} + +static const struct file_operations aio_ring_fops = { + .mmap_prepare = aio_ring_mmap_prepare, +}; + +#if IS_ENABLED(CONFIG_MIGRATION) +static int aio_migrate_folio(struct address_space *mapping, struct folio *dst, + struct folio *src, enum migrate_mode mode) +{ + struct kioctx *ctx; + struct aio_inode_info *ai = AIO_I(mapping->host); + unsigned long flags; + pgoff_t idx; + int rc = 0; + + /* ai->migrate_lock here protects against the kioctx teardown. */ + spin_lock(&ai->migrate_lock); + ctx = ai->ctx; + if (!ctx) { + rc = -EINVAL; + goto out; + } + + /* The ring_lock mutex. The prevents aio_read_events() from writing + * to the ring's head, and prevents page migration from mucking in + * a partially initialized kiotx. + */ + if (!mutex_trylock(&ctx->ring_lock)) { + rc = -EAGAIN; + goto out; + } + + idx = src->index; + if (idx < (pgoff_t)ctx->nr_pages) { + /* Make sure the old folio hasn't already been changed */ + if (ctx->ring_folios[idx] != src) + rc = -EAGAIN; + } else + rc = -EINVAL; + + if (rc != 0) + goto out_unlock; + + /* Writeback must be complete */ + BUG_ON(folio_test_writeback(src)); + folio_get(dst); + + rc = folio_migrate_mapping(mapping, dst, src, 1); + if (rc) { + folio_put(dst); + goto out_unlock; + } + + /* Take completion_lock to prevent other writes to the ring buffer + * while the old folio is copied to the new. This prevents new + * events from being lost. + */ + spin_lock_irqsave(&ctx->completion_lock, flags); + folio_copy(dst, src); + folio_migrate_flags(dst, src); + BUG_ON(ctx->ring_folios[idx] != src); + ctx->ring_folios[idx] = dst; + spin_unlock_irqrestore(&ctx->completion_lock, flags); + + /* The old folio is no longer accessible. */ + folio_put(src); + +out_unlock: + mutex_unlock(&ctx->ring_lock); +out: + spin_unlock(&ai->migrate_lock); + return rc; +} +#else +#define aio_migrate_folio NULL +#endif + +static const struct address_space_operations aio_ctx_aops = { + .dirty_folio = noop_dirty_folio, + .migrate_folio = aio_migrate_folio, +}; + +static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events) +{ + struct aio_ring *ring; + struct mm_struct *mm = current->mm; + unsigned long size, unused; + int nr_pages; + int i; + struct file *file; + + /* Compensate for the ring buffer's head/tail overlap entry */ + nr_events += 2; /* 1 is required, 2 for good luck */ + + size = sizeof(struct aio_ring); + size += sizeof(struct io_event) * nr_events; + + nr_pages = PFN_UP(size); + if (nr_pages < 0) + return -EINVAL; + + file = aio_private_file(ctx, nr_pages); + if (IS_ERR(file)) { + ctx->aio_ring_file = NULL; + return -ENOMEM; + } + + ctx->aio_ring_file = file; + nr_events = (PAGE_SIZE * nr_pages - sizeof(struct aio_ring)) + / sizeof(struct io_event); + + ctx->ring_folios = ctx->internal_folios; + if (nr_pages > AIO_RING_PAGES) { + ctx->ring_folios = kzalloc_objs(struct folio *, nr_pages); + if (!ctx->ring_folios) { + put_aio_ring_file(ctx); + return -ENOMEM; + } + } + + for (i = 0; i < nr_pages; i++) { + struct folio *folio; + + folio = __filemap_get_folio(file->f_mapping, i, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, + GFP_USER | __GFP_ZERO); + if (IS_ERR(folio)) + break; + + pr_debug("pid(%d) [%d] folio->count=%d\n", current->pid, i, + folio_ref_count(folio)); + folio_end_read(folio, true); + + ctx->ring_folios[i] = folio; + } + ctx->nr_pages = i; + + if (unlikely(i != nr_pages)) { + aio_free_ring(ctx); + return -ENOMEM; + } + + ctx->mmap_size = nr_pages * PAGE_SIZE; + pr_debug("attempting mmap of %lu bytes\n", ctx->mmap_size); + + if (mmap_write_lock_killable(mm)) { + ctx->mmap_size = 0; + aio_free_ring(ctx); + return -EINTR; + } + + ctx->mmap_base = do_mmap(ctx->aio_ring_file, 0, ctx->mmap_size, + PROT_READ | PROT_WRITE, + MAP_SHARED, EMPTY_VMA_FLAGS, 0, &unused, NULL); + mmap_write_unlock(mm); + if (IS_ERR((void *)ctx->mmap_base)) { + ctx->mmap_size = 0; + aio_free_ring(ctx); + return -ENOMEM; + } + + pr_debug("mmap address: 0x%08lx\n", ctx->mmap_base); + + ctx->user_id = ctx->mmap_base; + ctx->nr_events = nr_events; /* trusted copy */ + + ring = folio_address(ctx->ring_folios[0]); + ring->nr = nr_events; /* user copy */ + ring->id = ~0U; + ring->head = ring->tail = 0; + ring->magic = AIO_RING_MAGIC; + ring->compat_features = AIO_RING_COMPAT_FEATURES; + ring->incompat_features = AIO_RING_INCOMPAT_FEATURES; + ring->header_length = sizeof(struct aio_ring); + flush_dcache_folio(ctx->ring_folios[0]); + + return 0; +} + +#define AIO_EVENTS_PER_PAGE (PAGE_SIZE / sizeof(struct io_event)) +#define AIO_EVENTS_FIRST_PAGE ((PAGE_SIZE - sizeof(struct aio_ring)) / sizeof(struct io_event)) +#define AIO_EVENTS_OFFSET (AIO_EVENTS_PER_PAGE - AIO_EVENTS_FIRST_PAGE) + +void kiocb_set_cancel_fn(struct kiocb *iocb, kiocb_cancel_fn *cancel) +{ + struct aio_kiocb *req; + struct kioctx *ctx; + unsigned long flags; + + /* + * kiocb didn't come from aio or is neither a read nor a write, hence + * ignore it. + */ + if (!(iocb->ki_flags & IOCB_AIO_RW)) + return; + + req = container_of(iocb, struct aio_kiocb, rw); + + if (WARN_ON_ONCE(!list_empty(&req->ki_list))) + return; + + ctx = req->ki_ctx; + + spin_lock_irqsave(&ctx->ctx_lock, flags); + list_add_tail(&req->ki_list, &ctx->active_reqs); + req->ki_cancel = cancel; + spin_unlock_irqrestore(&ctx->ctx_lock, flags); +} +EXPORT_SYMBOL(kiocb_set_cancel_fn); + +/* + * free_ioctx() should be RCU delayed to synchronize against the RCU + * protected lookup_ioctx() and also needs process context to call + * aio_free_ring(). Use rcu_work. + */ +static void free_ioctx(struct work_struct *work) +{ + struct kioctx *ctx = container_of(to_rcu_work(work), struct kioctx, + free_rwork); + pr_debug("freeing %p\n", ctx); + + aio_free_ring(ctx); + free_percpu(ctx->cpu); + percpu_ref_exit(&ctx->reqs); + percpu_ref_exit(&ctx->users); + kmem_cache_free(kioctx_cachep, ctx); +} + +static void free_ioctx_reqs(struct percpu_ref *ref) +{ + struct kioctx *ctx = container_of(ref, struct kioctx, reqs); + + /* At this point we know that there are no any in-flight requests */ + if (ctx->rq_wait && atomic_dec_and_test(&ctx->rq_wait->count)) + complete(&ctx->rq_wait->comp); + + /* Synchronize against RCU protected table->table[] dereferences */ + INIT_RCU_WORK(&ctx->free_rwork, free_ioctx); + queue_rcu_work(system_percpu_wq, &ctx->free_rwork); +} + +/* + * When this function runs, the kioctx has been removed from the "hash table" + * and ctx->users has dropped to 0, so we know no more kiocbs can be submitted - + * now it's safe to cancel any that need to be. + */ +static void free_ioctx_users(struct percpu_ref *ref) +{ + struct kioctx *ctx = container_of(ref, struct kioctx, users); + struct aio_kiocb *req; + + spin_lock_irq(&ctx->ctx_lock); + + while (!list_empty(&ctx->active_reqs)) { + req = list_first_entry(&ctx->active_reqs, + struct aio_kiocb, ki_list); + req->ki_cancel(&req->rw); + list_del_init(&req->ki_list); + } + + spin_unlock_irq(&ctx->ctx_lock); + + percpu_ref_kill(&ctx->reqs); + percpu_ref_put(&ctx->reqs); +} + +static int ioctx_add_table(struct kioctx *ctx, struct mm_struct *mm) +{ + unsigned i, new_nr; + struct kioctx_table *table, *old; + struct aio_ring *ring; + + spin_lock(&mm->ioctx_lock); + table = rcu_dereference_raw(mm->ioctx_table); + + while (1) { + if (table) + for (i = 0; i < table->nr; i++) + if (!rcu_access_pointer(table->table[i])) { + ctx->id = i; + rcu_assign_pointer(table->table[i], ctx); + spin_unlock(&mm->ioctx_lock); + + /* While kioctx setup is in progress, + * we are protected from page migration + * changes ring_folios by ->ring_lock. + */ + ring = folio_address(ctx->ring_folios[0]); + ring->id = ctx->id; + return 0; + } + + new_nr = (table ? table->nr : 1) * 4; + spin_unlock(&mm->ioctx_lock); + + table = kzalloc_flex(*table, table, new_nr); + if (!table) + return -ENOMEM; + + table->nr = new_nr; + + spin_lock(&mm->ioctx_lock); + old = rcu_dereference_raw(mm->ioctx_table); + + if (!old) { + rcu_assign_pointer(mm->ioctx_table, table); + } else if (table->nr > old->nr) { + memcpy(table->table, old->table, + old->nr * sizeof(struct kioctx *)); + + rcu_assign_pointer(mm->ioctx_table, table); + kfree_rcu(old, rcu); + } else { + kfree(table); + table = old; + } + } +} + +static void aio_nr_sub(unsigned nr) +{ + spin_lock(&aio_nr_lock); + if (WARN_ON(aio_nr - nr > aio_nr)) + aio_nr = 0; + else + aio_nr -= nr; + spin_unlock(&aio_nr_lock); +} + +/* ioctx_alloc + * Allocates and initializes an ioctx. Returns an ERR_PTR if it failed. + */ +static struct kioctx *ioctx_alloc(unsigned nr_events) +{ + struct mm_struct *mm = current->mm; + struct kioctx *ctx; + int err = -ENOMEM; + + /* + * Store the original nr_events -- what userspace passed to io_setup(), + * for counting against the global limit -- before it changes. + */ + unsigned int max_reqs = nr_events; + + /* + * We keep track of the number of available ringbuffer slots, to prevent + * overflow (reqs_available), and we also use percpu counters for this. + * + * So since up to half the slots might be on other cpu's percpu counters + * and unavailable, double nr_events so userspace sees what they + * expected: additionally, we move req_batch slots to/from percpu + * counters at a time, so make sure that isn't 0: + */ + nr_events = max(nr_events, num_possible_cpus() * 4); + nr_events *= 2; + + /* Prevent overflows */ + if (nr_events > (0x10000000U / sizeof(struct io_event))) { + pr_debug("ENOMEM: nr_events too high\n"); + return ERR_PTR(-EINVAL); + } + + if (!nr_events || (unsigned long)max_reqs > aio_max_nr) + return ERR_PTR(-EAGAIN); + + ctx = kmem_cache_zalloc(kioctx_cachep, GFP_KERNEL); + if (!ctx) + return ERR_PTR(-ENOMEM); + + ctx->max_reqs = max_reqs; + + spin_lock_init(&ctx->ctx_lock); + spin_lock_init(&ctx->completion_lock); + mutex_init(&ctx->ring_lock); + /* Protect against page migration throughout kiotx setup by keeping + * the ring_lock mutex held until setup is complete. */ + mutex_lock(&ctx->ring_lock); + init_waitqueue_head(&ctx->wait); + + INIT_LIST_HEAD(&ctx->active_reqs); + + if (percpu_ref_init(&ctx->users, free_ioctx_users, 0, GFP_KERNEL)) + goto err; + + if (percpu_ref_init(&ctx->reqs, free_ioctx_reqs, 0, GFP_KERNEL)) + goto err; + + ctx->cpu = alloc_percpu(struct kioctx_cpu); + if (!ctx->cpu) + goto err; + + err = aio_setup_ring(ctx, nr_events); + if (err < 0) + goto err; + + atomic_set(&ctx->reqs_available, ctx->nr_events - 1); + ctx->req_batch = (ctx->nr_events - 1) / (num_possible_cpus() * 4); + if (ctx->req_batch < 1) + ctx->req_batch = 1; + + /* limit the number of system wide aios */ + spin_lock(&aio_nr_lock); + if (aio_nr + ctx->max_reqs > aio_max_nr || + aio_nr + ctx->max_reqs < aio_nr) { + spin_unlock(&aio_nr_lock); + err = -EAGAIN; + goto err_ctx; + } + aio_nr += ctx->max_reqs; + spin_unlock(&aio_nr_lock); + + percpu_ref_get(&ctx->users); /* io_setup() will drop this ref */ + percpu_ref_get(&ctx->reqs); /* free_ioctx_users() will drop this */ + + err = ioctx_add_table(ctx, mm); + if (err) + goto err_cleanup; + + /* Release the ring_lock mutex now that all setup is complete. */ + mutex_unlock(&ctx->ring_lock); + + pr_debug("allocated ioctx %p[%ld]: mm=%p mask=0x%x\n", + ctx, ctx->user_id, mm, ctx->nr_events); + return ctx; + +err_cleanup: + aio_nr_sub(ctx->max_reqs); +err_ctx: + atomic_set(&ctx->dead, 1); + if (ctx->mmap_size) + vm_munmap(ctx->mmap_base, ctx->mmap_size); + aio_free_ring(ctx); +err: + mutex_unlock(&ctx->ring_lock); + free_percpu(ctx->cpu); + percpu_ref_exit(&ctx->reqs); + percpu_ref_exit(&ctx->users); + kmem_cache_free(kioctx_cachep, ctx); + pr_debug("error allocating ioctx %d\n", err); + return ERR_PTR(err); +} + +/* kill_ioctx + * Cancels all outstanding aio requests on an aio context. Used + * when the processes owning a context have all exited to encourage + * the rapid destruction of the kioctx. + */ +static int kill_ioctx(struct mm_struct *mm, struct kioctx *ctx, + struct ctx_rq_wait *wait) +{ + struct kioctx_table *table; + + spin_lock(&mm->ioctx_lock); + if (atomic_xchg(&ctx->dead, 1)) { + spin_unlock(&mm->ioctx_lock); + return -EINVAL; + } + + table = rcu_dereference_raw(mm->ioctx_table); + WARN_ON(ctx != rcu_access_pointer(table->table[ctx->id])); + RCU_INIT_POINTER(table->table[ctx->id], NULL); + spin_unlock(&mm->ioctx_lock); + + /* free_ioctx_reqs() will do the necessary RCU synchronization */ + wake_up_all(&ctx->wait); + + /* + * It'd be more correct to do this in free_ioctx(), after all + * the outstanding kiocbs have finished - but by then io_destroy + * has already returned, so io_setup() could potentially return + * -EAGAIN with no ioctxs actually in use (as far as userspace + * could tell). + */ + aio_nr_sub(ctx->max_reqs); + + if (ctx->mmap_size) + vm_munmap(ctx->mmap_base, ctx->mmap_size); + + ctx->rq_wait = wait; + percpu_ref_kill(&ctx->users); + return 0; +} + +/* + * exit_aio: called when the last user of mm goes away. At this point, there is + * no way for any new requests to be submited or any of the io_* syscalls to be + * called on the context. + * + * There may be outstanding kiocbs, but free_ioctx() will explicitly wait on + * them. + */ +void exit_aio(struct mm_struct *mm) +{ + struct kioctx_table *table = rcu_dereference_raw(mm->ioctx_table); + struct ctx_rq_wait wait; + int i, skipped; + + if (!table) + return; + + atomic_set(&wait.count, table->nr); + init_completion(&wait.comp); + + skipped = 0; + for (i = 0; i < table->nr; ++i) { + struct kioctx *ctx = + rcu_dereference_protected(table->table[i], true); + + if (!ctx) { + skipped++; + continue; + } + + /* + * We don't need to bother with munmap() here - exit_mmap(mm) + * is coming and it'll unmap everything. And we simply can't, + * this is not necessarily our ->mm. + * Since kill_ioctx() uses non-zero ->mmap_size as indicator + * that it needs to unmap the area, just set it to 0. + */ + ctx->mmap_size = 0; + kill_ioctx(mm, ctx, &wait); + } + + if (!atomic_sub_and_test(skipped, &wait.count)) { + /* Wait until all IO for the context are done. */ + wait_for_completion(&wait.comp); + } + + RCU_INIT_POINTER(mm->ioctx_table, NULL); + kfree(table); +} + +static void put_reqs_available(struct kioctx *ctx, unsigned nr) +{ + struct kioctx_cpu *kcpu; + unsigned long flags; + + local_irq_save(flags); + kcpu = this_cpu_ptr(ctx->cpu); + kcpu->reqs_available += nr; + + while (kcpu->reqs_available >= ctx->req_batch * 2) { + kcpu->reqs_available -= ctx->req_batch; + atomic_add(ctx->req_batch, &ctx->reqs_available); + } + + local_irq_restore(flags); +} + +static bool __get_reqs_available(struct kioctx *ctx) +{ + struct kioctx_cpu *kcpu; + bool ret = false; + unsigned long flags; + + local_irq_save(flags); + kcpu = this_cpu_ptr(ctx->cpu); + if (!kcpu->reqs_available) { + int avail = atomic_read(&ctx->reqs_available); + + do { + if (avail < ctx->req_batch) + goto out; + } while (!atomic_try_cmpxchg(&ctx->reqs_available, + &avail, avail - ctx->req_batch)); + + kcpu->reqs_available += ctx->req_batch; + } + + ret = true; + kcpu->reqs_available--; +out: + local_irq_restore(flags); + return ret; +} + +/* refill_reqs_available + * Updates the reqs_available reference counts used for tracking the + * number of free slots in the completion ring. This can be called + * from aio_complete() (to optimistically update reqs_available) or + * from aio_get_req() (the we're out of events case). It must be + * called holding ctx->completion_lock. + */ +static void refill_reqs_available(struct kioctx *ctx, unsigned head, + unsigned tail) +{ + unsigned events_in_ring, completed; + + /* Clamp head since userland can write to it. */ + head %= ctx->nr_events; + if (head <= tail) + events_in_ring = tail - head; + else + events_in_ring = ctx->nr_events - (head - tail); + + completed = ctx->completed_events; + if (events_in_ring < completed) + completed -= events_in_ring; + else + completed = 0; + + if (!completed) + return; + + ctx->completed_events -= completed; + put_reqs_available(ctx, completed); +} + +/* user_refill_reqs_available + * Called to refill reqs_available when aio_get_req() encounters an + * out of space in the completion ring. + */ +static void user_refill_reqs_available(struct kioctx *ctx) +{ + spin_lock_irq(&ctx->completion_lock); + if (ctx->completed_events) { + struct aio_ring *ring; + unsigned head; + + /* Access of ring->head may race with aio_read_events_ring() + * here, but that's okay since whether we read the old version + * or the new version, and either will be valid. The important + * part is that head cannot pass tail since we prevent + * aio_complete() from updating tail by holding + * ctx->completion_lock. Even if head is invalid, the check + * against ctx->completed_events below will make sure we do the + * safe/right thing. + */ + ring = folio_address(ctx->ring_folios[0]); + head = ring->head; + + refill_reqs_available(ctx, head, ctx->tail); + } + + spin_unlock_irq(&ctx->completion_lock); +} + +static bool get_reqs_available(struct kioctx *ctx) +{ + if (__get_reqs_available(ctx)) + return true; + user_refill_reqs_available(ctx); + return __get_reqs_available(ctx); +} + +/* aio_get_req + * Allocate a slot for an aio request. + * Returns NULL if no requests are free. + * + * The refcount is initialized to 2 - one for the async op completion, + * one for the synchronous code that does this. + */ +static inline struct aio_kiocb *aio_get_req(struct kioctx *ctx) +{ + struct aio_kiocb *req; + + req = kmem_cache_alloc(kiocb_cachep, GFP_KERNEL); + if (unlikely(!req)) + return NULL; + + if (unlikely(!get_reqs_available(ctx))) { + kmem_cache_free(kiocb_cachep, req); + return NULL; + } + + percpu_ref_get(&ctx->reqs); + req->ki_ctx = ctx; + INIT_LIST_HEAD(&req->ki_list); + refcount_set(&req->ki_refcnt, 2); + req->ki_eventfd = NULL; + return req; +} + +static struct kioctx *lookup_ioctx(unsigned long ctx_id) +{ + struct aio_ring __user *ring = (void __user *)ctx_id; + struct mm_struct *mm = current->mm; + struct kioctx *ctx, *ret = NULL; + struct kioctx_table *table; + unsigned id; + + if (get_user(id, &ring->id)) + return NULL; + + rcu_read_lock(); + table = rcu_dereference(mm->ioctx_table); + + if (!table || id >= table->nr) + goto out; + + id = array_index_nospec(id, table->nr); + ctx = rcu_dereference(table->table[id]); + if (ctx && ctx->user_id == ctx_id) { + if (percpu_ref_tryget_live(&ctx->users)) + ret = ctx; + } +out: + rcu_read_unlock(); + return ret; +} + +static inline void iocb_destroy(struct aio_kiocb *iocb) +{ + if (iocb->ki_eventfd) + eventfd_ctx_put(iocb->ki_eventfd); + if (iocb->ki_filp) + fput(iocb->ki_filp); + percpu_ref_put(&iocb->ki_ctx->reqs); + kmem_cache_free(kiocb_cachep, iocb); +} + +struct aio_waiter { + struct wait_queue_entry w; + size_t min_nr; +}; + +/* aio_complete + * Called when the io request on the given iocb is complete. + */ +static void aio_complete(struct aio_kiocb *iocb) +{ + struct kioctx *ctx = iocb->ki_ctx; + struct aio_ring *ring; + struct io_event *ev_page, *event; + unsigned tail, pos, head, avail; + unsigned long flags; + + /* + * Add a completion event to the ring buffer. Must be done holding + * ctx->completion_lock to prevent other code from messing with the tail + * pointer since we might be called from irq context. + */ + spin_lock_irqsave(&ctx->completion_lock, flags); + + tail = ctx->tail; + pos = tail + AIO_EVENTS_OFFSET; + + if (++tail >= ctx->nr_events) + tail = 0; + + ev_page = folio_address(ctx->ring_folios[pos / AIO_EVENTS_PER_PAGE]); + event = ev_page + pos % AIO_EVENTS_PER_PAGE; + + *event = iocb->ki_res; + + flush_dcache_folio(ctx->ring_folios[pos / AIO_EVENTS_PER_PAGE]); + + pr_debug("%p[%u]: %p: %p %Lx %Lx %Lx\n", ctx, tail, iocb, + (void __user *)(unsigned long)iocb->ki_res.obj, + iocb->ki_res.data, iocb->ki_res.res, iocb->ki_res.res2); + + /* after flagging the request as done, we + * must never even look at it again + */ + smp_wmb(); /* make event visible before updating tail */ + + ctx->tail = tail; + + ring = folio_address(ctx->ring_folios[0]); + head = ring->head; + ring->tail = tail; + flush_dcache_folio(ctx->ring_folios[0]); + + ctx->completed_events++; + if (ctx->completed_events > 1) + refill_reqs_available(ctx, head, tail); + + avail = tail > head + ? tail - head + : tail + ctx->nr_events - head; + spin_unlock_irqrestore(&ctx->completion_lock, flags); + + pr_debug("added to ring %p at [%u]\n", iocb, tail); + + /* + * Check if the user asked us to deliver the result through an + * eventfd. The eventfd_signal() function is safe to be called + * from IRQ context. + */ + if (iocb->ki_eventfd) + eventfd_signal(iocb->ki_eventfd); + + /* + * We have to order our ring_info tail store above and test + * of the wait list below outside the wait lock. This is + * like in wake_up_bit() where clearing a bit has to be + * ordered with the unlocked test. + */ + smp_mb(); + + if (waitqueue_active(&ctx->wait)) { + struct aio_waiter *curr, *next; + unsigned long flags; + + spin_lock_irqsave(&ctx->wait.lock, flags); + list_for_each_entry_safe(curr, next, &ctx->wait.head, w.entry) + if (avail >= curr->min_nr) { + wake_up_process(curr->w.private); + list_del_init_careful(&curr->w.entry); + } + spin_unlock_irqrestore(&ctx->wait.lock, flags); + } +} + +static inline void iocb_put(struct aio_kiocb *iocb) +{ + if (refcount_dec_and_test(&iocb->ki_refcnt)) { + aio_complete(iocb); + iocb_destroy(iocb); + } +} + +/* aio_read_events_ring + * Pull an event off of the ioctx's event ring. Returns the number of + * events fetched + */ +static long aio_read_events_ring(struct kioctx *ctx, + struct io_event __user *event, long nr) +{ + struct aio_ring *ring; + unsigned head, tail, pos; + long ret = 0; + int copy_ret; + + /* + * The mutex can block and wake us up and that will cause + * wait_event_interruptible_hrtimeout() to schedule without sleeping + * and repeat. This should be rare enough that it doesn't cause + * peformance issues. See the comment in read_events() for more detail. + */ + sched_annotate_sleep(); + mutex_lock(&ctx->ring_lock); + + /* Access to ->ring_folios here is protected by ctx->ring_lock. */ + ring = folio_address(ctx->ring_folios[0]); + head = ring->head; + tail = ring->tail; + + /* + * Ensure that once we've read the current tail pointer, that + * we also see the events that were stored up to the tail. + */ + smp_rmb(); + + pr_debug("h%u t%u m%u\n", head, tail, ctx->nr_events); + + if (head == tail) + goto out; + + head %= ctx->nr_events; + tail %= ctx->nr_events; + + while (ret < nr) { + long avail; + struct io_event *ev; + struct folio *folio; + + avail = (head <= tail ? tail : ctx->nr_events) - head; + if (head == tail) + break; + + pos = head + AIO_EVENTS_OFFSET; + folio = ctx->ring_folios[pos / AIO_EVENTS_PER_PAGE]; + pos %= AIO_EVENTS_PER_PAGE; + + avail = min(avail, nr - ret); + avail = min_t(long, avail, AIO_EVENTS_PER_PAGE - pos); + + ev = folio_address(folio); + copy_ret = copy_to_user(event + ret, ev + pos, + sizeof(*ev) * avail); + + if (unlikely(copy_ret)) { + ret = -EFAULT; + goto out; + } + + ret += avail; + head += avail; + head %= ctx->nr_events; + } + + ring = folio_address(ctx->ring_folios[0]); + ring->head = head; + flush_dcache_folio(ctx->ring_folios[0]); + + pr_debug("%li h%u t%u\n", ret, head, tail); +out: + mutex_unlock(&ctx->ring_lock); + + return ret; +} + +static bool aio_read_events(struct kioctx *ctx, long min_nr, long nr, + struct io_event __user *event, long *i) +{ + long ret = aio_read_events_ring(ctx, event + *i, nr - *i); + + if (ret > 0) + *i += ret; + + if (unlikely(atomic_read(&ctx->dead))) + ret = -EINVAL; + + if (!*i) + *i = ret; + + return ret < 0 || *i >= min_nr; +} + +static long read_events(struct kioctx *ctx, long min_nr, long nr, + struct io_event __user *event, + ktime_t until) +{ + struct hrtimer_sleeper t; + struct aio_waiter w; + long ret = 0, ret2 = 0; + + /* + * Note that aio_read_events() is being called as the conditional - i.e. + * we're calling it after prepare_to_wait() has set task state to + * TASK_INTERRUPTIBLE. + * + * But aio_read_events() can block, and if it blocks it's going to flip + * the task state back to TASK_RUNNING. + * + * This should be ok, provided it doesn't flip the state back to + * TASK_RUNNING and return 0 too much - that causes us to spin. That + * will only happen if the mutex_lock() call blocks, and we then find + * the ringbuffer empty. So in practice we should be ok, but it's + * something to be aware of when touching this code. + */ + aio_read_events(ctx, min_nr, nr, event, &ret); + if (until == 0 || ret < 0 || ret >= min_nr) + return ret; + + hrtimer_setup_sleeper_on_stack(&t, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + if (until != KTIME_MAX) { + hrtimer_set_expires_range_ns(&t.timer, until, current->timer_slack_ns); + hrtimer_sleeper_start_expires(&t, HRTIMER_MODE_REL); + } + + init_wait(&w.w); + + while (1) { + unsigned long nr_got = ret; + + w.min_nr = min_nr - ret; + + ret2 = prepare_to_wait_event(&ctx->wait, &w.w, TASK_INTERRUPTIBLE); + if (!ret2 && !t.task) + ret2 = -ETIME; + + if (aio_read_events(ctx, min_nr, nr, event, &ret) || ret2) + break; + + if (nr_got == ret) + schedule(); + } + + finish_wait(&ctx->wait, &w.w); + hrtimer_cancel(&t.timer); + destroy_hrtimer_on_stack(&t.timer); + + return ret; +} + +/* sys_io_setup: + * Create an aio_context capable of receiving at least nr_events. + * ctxp must not point to an aio_context that already exists, and + * must be initialized to 0 prior to the call. On successful + * creation of the aio_context, *ctxp is filled in with the resulting + * handle. May fail with -EINVAL if *ctxp is not initialized, + * if the specified nr_events exceeds internal limits. May fail + * with -EAGAIN if the specified nr_events exceeds the user's limit + * of available events. May fail with -ENOMEM if insufficient kernel + * resources are available. May fail with -EFAULT if an invalid + * pointer is passed for ctxp. Will fail with -ENOSYS if not + * implemented. + */ +SYSCALL_DEFINE2(io_setup, unsigned, nr_events, aio_context_t __user *, ctxp) +{ + struct kioctx *ioctx = NULL; + unsigned long ctx; + long ret; + + ret = get_user(ctx, ctxp); + if (unlikely(ret)) + goto out; + + ret = -EINVAL; + if (unlikely(ctx || nr_events == 0)) { + pr_debug("EINVAL: ctx %lu nr_events %u\n", + ctx, nr_events); + goto out; + } + + ioctx = ioctx_alloc(nr_events); + ret = PTR_ERR(ioctx); + if (!IS_ERR(ioctx)) { + ret = put_user(ioctx->user_id, ctxp); + if (ret) + kill_ioctx(current->mm, ioctx, NULL); + percpu_ref_put(&ioctx->users); + } + +out: + return ret; +} + +#ifdef CONFIG_COMPAT +COMPAT_SYSCALL_DEFINE2(io_setup, unsigned, nr_events, u32 __user *, ctx32p) +{ + struct kioctx *ioctx = NULL; + unsigned long ctx; + long ret; + + ret = get_user(ctx, ctx32p); + if (unlikely(ret)) + goto out; + + ret = -EINVAL; + if (unlikely(ctx || nr_events == 0)) { + pr_debug("EINVAL: ctx %lu nr_events %u\n", + ctx, nr_events); + goto out; + } + + ioctx = ioctx_alloc(nr_events); + ret = PTR_ERR(ioctx); + if (!IS_ERR(ioctx)) { + /* truncating is ok because it's a user address */ + ret = put_user((u32)ioctx->user_id, ctx32p); + if (ret) + kill_ioctx(current->mm, ioctx, NULL); + percpu_ref_put(&ioctx->users); + } + +out: + return ret; +} +#endif + +/* sys_io_destroy: + * Destroy the aio_context specified. May cancel any outstanding + * AIOs and block on completion. Will fail with -ENOSYS if not + * implemented. May fail with -EINVAL if the context pointed to + * is invalid. + */ +SYSCALL_DEFINE1(io_destroy, aio_context_t, ctx) +{ + struct kioctx *ioctx = lookup_ioctx(ctx); + if (likely(NULL != ioctx)) { + struct ctx_rq_wait wait; + int ret; + + init_completion(&wait.comp); + atomic_set(&wait.count, 1); + + /* Pass requests_done to kill_ioctx() where it can be set + * in a thread-safe way. If we try to set it here then we have + * a race condition if two io_destroy() called simultaneously. + */ + ret = kill_ioctx(current->mm, ioctx, &wait); + percpu_ref_put(&ioctx->users); + + /* Wait until all IO for the context are done. Otherwise kernel + * keep using user-space buffers even if user thinks the context + * is destroyed. + */ + if (!ret) + wait_for_completion(&wait.comp); + + return ret; + } + pr_debug("EINVAL: invalid context id\n"); + return -EINVAL; +} + +static void aio_remove_iocb(struct aio_kiocb *iocb) +{ + struct kioctx *ctx = iocb->ki_ctx; + unsigned long flags; + + spin_lock_irqsave(&ctx->ctx_lock, flags); + list_del(&iocb->ki_list); + spin_unlock_irqrestore(&ctx->ctx_lock, flags); +} + +static void aio_complete_rw(struct kiocb *kiocb, long res) +{ + struct aio_kiocb *iocb = container_of(kiocb, struct aio_kiocb, rw); + + if (!list_empty_careful(&iocb->ki_list)) + aio_remove_iocb(iocb); + + if (kiocb->ki_flags & IOCB_WRITE) { + struct inode *inode = file_inode(kiocb->ki_filp); + + if (S_ISREG(inode->i_mode)) + kiocb_end_write(kiocb); + } + + iocb->ki_res.res = res; + iocb->ki_res.res2 = 0; + iocb_put(iocb); +} + +static int aio_prep_rw(struct kiocb *req, const struct iocb *iocb, int rw_type) +{ + int ret; + + req->ki_write_stream = 0; + req->ki_complete = aio_complete_rw; + req->private = NULL; + req->ki_pos = iocb->aio_offset; + req->ki_flags = req->ki_filp->f_iocb_flags | IOCB_AIO_RW; + if (iocb->aio_flags & IOCB_FLAG_RESFD) + req->ki_flags |= IOCB_EVENTFD; + if (iocb->aio_flags & IOCB_FLAG_IOPRIO) { + /* + * If the IOCB_FLAG_IOPRIO flag of aio_flags is set, then + * aio_reqprio is interpreted as an I/O scheduling + * class and priority. + */ + ret = ioprio_check_cap(iocb->aio_reqprio); + if (ret) { + pr_debug("aio ioprio check cap error: %d\n", ret); + return ret; + } + + req->ki_ioprio = iocb->aio_reqprio; + } else + req->ki_ioprio = get_current_ioprio(); + + ret = kiocb_set_rw_flags(req, iocb->aio_rw_flags, rw_type); + if (unlikely(ret)) + return ret; + + req->ki_flags &= ~IOCB_HIPRI; /* no one is going to poll for this I/O */ + return 0; +} + +static ssize_t aio_setup_rw(int rw, const struct iocb *iocb, + struct iovec **iovec, bool vectored, bool compat, + struct iov_iter *iter) +{ + void __user *buf = (void __user *)(uintptr_t)iocb->aio_buf; + size_t len = iocb->aio_nbytes; + + if (!vectored) { + ssize_t ret = import_ubuf(rw, buf, len, iter); + *iovec = NULL; + return ret; + } + + return __import_iovec(rw, buf, len, UIO_FASTIOV, iovec, iter, compat); +} + +static inline void aio_rw_done(struct kiocb *req, ssize_t ret) +{ + switch (ret) { + case -EIOCBQUEUED: + break; + case -ERESTARTSYS: + case -ERESTARTNOINTR: + case -ERESTARTNOHAND: + case -ERESTART_RESTARTBLOCK: + /* + * There's no easy way to restart the syscall since other AIO's + * may be already running. Just fail this IO with EINTR. + */ + ret = -EINTR; + fallthrough; + default: + req->ki_complete(req, ret); + } +} + +static int aio_read(struct kiocb *req, const struct iocb *iocb, + bool vectored, bool compat) +{ + struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; + struct iov_iter iter; + struct file *file; + int ret; + + ret = aio_prep_rw(req, iocb, READ); + if (ret) + return ret; + file = req->ki_filp; + if (unlikely(!(file->f_mode & FMODE_READ))) + return -EBADF; + if (unlikely(!file->f_op->read_iter)) + return -EINVAL; + + ret = aio_setup_rw(ITER_DEST, iocb, &iovec, vectored, compat, &iter); + if (ret < 0) + return ret; + ret = rw_verify_area(READ, file, &req->ki_pos, iov_iter_count(&iter)); + if (!ret) + aio_rw_done(req, file->f_op->read_iter(req, &iter)); + kfree(iovec); + return ret; +} + +static int aio_write(struct kiocb *req, const struct iocb *iocb, + bool vectored, bool compat) +{ + struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; + struct iov_iter iter; + struct file *file; + int ret; + + ret = aio_prep_rw(req, iocb, WRITE); + if (ret) + return ret; + file = req->ki_filp; + + if (unlikely(!(file->f_mode & FMODE_WRITE))) + return -EBADF; + if (unlikely(!file->f_op->write_iter)) + return -EINVAL; + + ret = aio_setup_rw(ITER_SOURCE, iocb, &iovec, vectored, compat, &iter); + if (ret < 0) + return ret; + ret = rw_verify_area(WRITE, file, &req->ki_pos, iov_iter_count(&iter)); + if (!ret) { + if (S_ISREG(file_inode(file)->i_mode)) + kiocb_start_write(req); + req->ki_flags |= IOCB_WRITE; + aio_rw_done(req, file->f_op->write_iter(req, &iter)); + } + kfree(iovec); + return ret; +} + +static void aio_fsync_work(struct work_struct *work) +{ + struct aio_kiocb *iocb = container_of(work, struct aio_kiocb, fsync.work); + + scoped_with_creds(iocb->fsync.creds) + iocb->ki_res.res = vfs_fsync(iocb->fsync.file, iocb->fsync.datasync); + + put_cred(iocb->fsync.creds); + iocb_put(iocb); +} + +static int aio_fsync(struct fsync_iocb *req, const struct iocb *iocb, + bool datasync) +{ + if (unlikely(iocb->aio_buf || iocb->aio_offset || iocb->aio_nbytes || + iocb->aio_rw_flags)) + return -EINVAL; + + if (unlikely(!req->file->f_op->fsync)) + return -EINVAL; + + req->creds = prepare_creds(); + if (!req->creds) + return -ENOMEM; + + req->datasync = datasync; + INIT_WORK(&req->work, aio_fsync_work); + schedule_work(&req->work); + return 0; +} + +static void aio_poll_put_work(struct work_struct *work) +{ + struct poll_iocb *req = container_of(work, struct poll_iocb, work); + struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll); + + iocb_put(iocb); +} + +/* + * Safely lock the waitqueue which the request is on, synchronizing with the + * case where the ->poll() provider decides to free its waitqueue early. + * + * Returns true on success, meaning that req->head->lock was locked, req->wait + * is on req->head, and an RCU read lock was taken. Returns false if the + * request was already removed from its waitqueue (which might no longer exist). + */ +static bool poll_iocb_lock_wq(struct poll_iocb *req) +{ + wait_queue_head_t *head; + + /* + * While we hold the waitqueue lock and the waitqueue is nonempty, + * wake_up_pollfree() will wait for us. However, taking the waitqueue + * lock in the first place can race with the waitqueue being freed. + * + * We solve this as eventpoll does: by taking advantage of the fact that + * all users of wake_up_pollfree() will RCU-delay the actual free. If + * we enter rcu_read_lock() and see that the pointer to the queue is + * non-NULL, we can then lock it without the memory being freed out from + * under us, then check whether the request is still on the queue. + * + * Keep holding rcu_read_lock() as long as we hold the queue lock, in + * case the caller deletes the entry from the queue, leaving it empty. + * In that case, only RCU prevents the queue memory from being freed. + */ + rcu_read_lock(); + head = smp_load_acquire(&req->head); + if (head) { + spin_lock(&head->lock); + if (!list_empty(&req->wait.entry)) + return true; + spin_unlock(&head->lock); + } + rcu_read_unlock(); + return false; +} + +static void poll_iocb_unlock_wq(struct poll_iocb *req) +{ + spin_unlock(&req->head->lock); + rcu_read_unlock(); +} + +static void aio_poll_complete_work(struct work_struct *work) +{ + struct poll_iocb *req = container_of(work, struct poll_iocb, work); + struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll); + struct poll_table_struct pt = { ._key = req->events }; + struct kioctx *ctx = iocb->ki_ctx; + __poll_t mask = 0; + + if (!READ_ONCE(req->cancelled)) + mask = vfs_poll(req->file, &pt) & req->events; + + /* + * Note that ->ki_cancel callers also delete iocb from active_reqs after + * calling ->ki_cancel. We need the ctx_lock roundtrip here to + * synchronize with them. In the cancellation case the list_del_init + * itself is not actually needed, but harmless so we keep it in to + * avoid further branches in the fast path. + */ + spin_lock_irq(&ctx->ctx_lock); + if (poll_iocb_lock_wq(req)) { + if (!mask && !READ_ONCE(req->cancelled)) { + /* + * The request isn't actually ready to be completed yet. + * Reschedule completion if another wakeup came in. + */ + if (req->work_need_resched) { + schedule_work(&req->work); + req->work_need_resched = false; + } else { + req->work_scheduled = false; + } + poll_iocb_unlock_wq(req); + spin_unlock_irq(&ctx->ctx_lock); + return; + } + list_del_init(&req->wait.entry); + poll_iocb_unlock_wq(req); + } /* else, POLLFREE has freed the waitqueue, so we must complete */ + list_del_init(&iocb->ki_list); + iocb->ki_res.res = mangle_poll(mask); + spin_unlock_irq(&ctx->ctx_lock); + + iocb_put(iocb); +} + +/* assumes we are called with irqs disabled */ +static int aio_poll_cancel(struct kiocb *iocb) +{ + struct aio_kiocb *aiocb = container_of(iocb, struct aio_kiocb, rw); + struct poll_iocb *req = &aiocb->poll; + + if (poll_iocb_lock_wq(req)) { + WRITE_ONCE(req->cancelled, true); + if (!req->work_scheduled) { + schedule_work(&aiocb->poll.work); + req->work_scheduled = true; + } + poll_iocb_unlock_wq(req); + } /* else, the request was force-cancelled by POLLFREE already */ + + return 0; +} + +static int aio_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, + void *key) +{ + struct poll_iocb *req = container_of(wait, struct poll_iocb, wait); + struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll); + __poll_t mask = key_to_poll(key); + unsigned long flags; + + /* for instances that support it check for an event match first: */ + if (mask && !(mask & req->events)) + return 0; + + /* + * Complete the request inline if possible. This requires that three + * conditions be met: + * 1. An event mask must have been passed. If a plain wakeup was done + * instead, then mask == 0 and we have to call vfs_poll() to get + * the events, so inline completion isn't possible. + * 2. The completion work must not have already been scheduled. + * 3. ctx_lock must not be busy. We have to use trylock because we + * already hold the waitqueue lock, so this inverts the normal + * locking order. Use irqsave/irqrestore because not all + * filesystems (e.g. fuse) call this function with IRQs disabled, + * yet IRQs have to be disabled before ctx_lock is obtained. + */ + if (mask && !req->work_scheduled && + spin_trylock_irqsave(&iocb->ki_ctx->ctx_lock, flags)) { + struct kioctx *ctx = iocb->ki_ctx; + + list_del_init(&req->wait.entry); + list_del(&iocb->ki_list); + iocb->ki_res.res = mangle_poll(mask); + if (iocb->ki_eventfd && !eventfd_signal_allowed()) { + iocb = NULL; + INIT_WORK(&req->work, aio_poll_put_work); + schedule_work(&req->work); + } + spin_unlock_irqrestore(&ctx->ctx_lock, flags); + if (iocb) + iocb_put(iocb); + } else { + /* + * Schedule the completion work if needed. If it was already + * scheduled, record that another wakeup came in. + * + * Don't remove the request from the waitqueue here, as it might + * not actually be complete yet (we won't know until vfs_poll() + * is called), and we must not miss any wakeups. POLLFREE is an + * exception to this; see below. + */ + if (req->work_scheduled) { + req->work_need_resched = true; + } else { + schedule_work(&req->work); + req->work_scheduled = true; + } + + /* + * If the waitqueue is being freed early but we can't complete + * the request inline, we have to tear down the request as best + * we can. That means immediately removing the request from its + * waitqueue and preventing all further accesses to the + * waitqueue via the request. We also need to schedule the + * completion work (done above). Also mark the request as + * cancelled, to potentially skip an unneeded call to ->poll(). + */ + if (mask & POLLFREE) { + WRITE_ONCE(req->cancelled, true); + list_del_init(&req->wait.entry); + + /* + * Careful: this *must* be the last step, since as soon + * as req->head is NULL'ed out, the request can be + * completed and freed, since aio_poll_complete_work() + * will no longer need to take the waitqueue lock. + */ + smp_store_release(&req->head, NULL); + } + } + return 1; +} + +struct aio_poll_table { + struct poll_table_struct pt; + struct aio_kiocb *iocb; + bool queued; + int error; +}; + +static void +aio_poll_queue_proc(struct file *file, struct wait_queue_head *head, + struct poll_table_struct *p) +{ + struct aio_poll_table *pt = container_of(p, struct aio_poll_table, pt); + + /* multiple wait queues per file are not supported */ + if (unlikely(pt->queued)) { + pt->error = -EINVAL; + return; + } + + pt->queued = true; + pt->error = 0; + pt->iocb->poll.head = head; + add_wait_queue(head, &pt->iocb->poll.wait); +} + +static int aio_poll(struct aio_kiocb *aiocb, const struct iocb *iocb) +{ + struct kioctx *ctx = aiocb->ki_ctx; + struct poll_iocb *req = &aiocb->poll; + struct aio_poll_table apt; + bool cancel = false; + __poll_t mask; + + /* reject any unknown events outside the normal event mask. */ + if ((u16)iocb->aio_buf != iocb->aio_buf) + return -EINVAL; + /* reject fields that are not defined for poll */ + if (iocb->aio_offset || iocb->aio_nbytes || iocb->aio_rw_flags) + return -EINVAL; + + INIT_WORK(&req->work, aio_poll_complete_work); + req->events = demangle_poll(iocb->aio_buf) | EPOLLERR | EPOLLHUP; + + req->head = NULL; + req->cancelled = false; + req->work_scheduled = false; + req->work_need_resched = false; + + apt.pt._qproc = aio_poll_queue_proc; + apt.pt._key = req->events; + apt.iocb = aiocb; + apt.queued = false; + apt.error = -EINVAL; /* same as no support for IOCB_CMD_POLL */ + + /* initialized the list so that we can do list_empty checks */ + INIT_LIST_HEAD(&req->wait.entry); + init_waitqueue_func_entry(&req->wait, aio_poll_wake); + + mask = vfs_poll(req->file, &apt.pt) & req->events; + spin_lock_irq(&ctx->ctx_lock); + if (likely(apt.queued)) { + bool on_queue = poll_iocb_lock_wq(req); + + if (!on_queue || req->work_scheduled) { + /* + * aio_poll_wake() already either scheduled the async + * completion work, or completed the request inline. + */ + if (apt.error) /* unsupported case: multiple queues */ + cancel = true; + apt.error = 0; + mask = 0; + } + if (mask || apt.error) { + /* Steal to complete synchronously. */ + list_del_init(&req->wait.entry); + } else if (cancel) { + /* Cancel if possible (may be too late though). */ + WRITE_ONCE(req->cancelled, true); + } else if (on_queue) { + /* + * Actually waiting for an event, so add the request to + * active_reqs so that it can be cancelled if needed. + */ + list_add_tail(&aiocb->ki_list, &ctx->active_reqs); + aiocb->ki_cancel = aio_poll_cancel; + } + if (on_queue) + poll_iocb_unlock_wq(req); + } + if (mask) { /* no async, we'd stolen it */ + aiocb->ki_res.res = mangle_poll(mask); + apt.error = 0; + } + spin_unlock_irq(&ctx->ctx_lock); + if (mask) + iocb_put(aiocb); + return apt.error; +} + +static int __io_submit_one(struct kioctx *ctx, const struct iocb *iocb, + struct iocb __user *user_iocb, struct aio_kiocb *req, + bool compat) +{ + req->ki_filp = fget(iocb->aio_fildes); + if (unlikely(!req->ki_filp)) + return -EBADF; + + if (iocb->aio_flags & IOCB_FLAG_RESFD) { + struct eventfd_ctx *eventfd; + /* + * If the IOCB_FLAG_RESFD flag of aio_flags is set, get an + * instance of the file* now. The file descriptor must be + * an eventfd() fd, and will be signaled for each completed + * event using the eventfd_signal() function. + */ + eventfd = eventfd_ctx_fdget(iocb->aio_resfd); + if (IS_ERR(eventfd)) + return PTR_ERR(eventfd); + + req->ki_eventfd = eventfd; + } + + if (unlikely(put_user(KIOCB_KEY, &user_iocb->aio_key))) { + pr_debug("EFAULT: aio_key\n"); + return -EFAULT; + } + + req->ki_res.obj = (u64)(unsigned long)user_iocb; + req->ki_res.data = iocb->aio_data; + req->ki_res.res = 0; + req->ki_res.res2 = 0; + + switch (iocb->aio_lio_opcode) { + case IOCB_CMD_PREAD: + return aio_read(&req->rw, iocb, false, compat); + case IOCB_CMD_PWRITE: + return aio_write(&req->rw, iocb, false, compat); + case IOCB_CMD_PREADV: + return aio_read(&req->rw, iocb, true, compat); + case IOCB_CMD_PWRITEV: + return aio_write(&req->rw, iocb, true, compat); + case IOCB_CMD_FSYNC: + return aio_fsync(&req->fsync, iocb, false); + case IOCB_CMD_FDSYNC: + return aio_fsync(&req->fsync, iocb, true); + case IOCB_CMD_POLL: + return aio_poll(req, iocb); + default: + pr_debug("invalid aio operation %d\n", iocb->aio_lio_opcode); + return -EINVAL; + } +} + +static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, + bool compat) +{ + struct aio_kiocb *req; + struct iocb iocb; + int err; + + if (unlikely(copy_from_user(&iocb, user_iocb, sizeof(iocb)))) + return -EFAULT; + + /* enforce forwards compatibility on users */ + if (unlikely(iocb.aio_reserved2)) { + pr_debug("EINVAL: reserve field set\n"); + return -EINVAL; + } + + /* prevent overflows */ + if (unlikely( + (iocb.aio_buf != (unsigned long)iocb.aio_buf) || + (iocb.aio_nbytes != (size_t)iocb.aio_nbytes) || + ((ssize_t)iocb.aio_nbytes < 0) + )) { + pr_debug("EINVAL: overflow check\n"); + return -EINVAL; + } + + req = aio_get_req(ctx); + if (unlikely(!req)) + return -EAGAIN; + + err = __io_submit_one(ctx, &iocb, user_iocb, req, compat); + + /* Done with the synchronous reference */ + iocb_put(req); + + /* + * If err is 0, we'd either done aio_complete() ourselves or have + * arranged for that to be done asynchronously. Anything non-zero + * means that we need to destroy req ourselves. + */ + if (unlikely(err)) { + iocb_destroy(req); + put_reqs_available(ctx, 1); + } + return err; +} + +/* sys_io_submit: + * Queue the nr iocbs pointed to by iocbpp for processing. Returns + * the number of iocbs queued. May return -EINVAL if the aio_context + * specified by ctx_id is invalid, if nr is < 0, if the iocb at + * *iocbpp[0] is not properly initialized, if the operation specified + * is invalid for the file descriptor in the iocb. May fail with + * -EFAULT if any of the data structures point to invalid data. May + * fail with -EBADF if the file descriptor specified in the first + * iocb is invalid. May fail with -EAGAIN if insufficient resources + * are available to queue any iocbs. Will return 0 if nr is 0. Will + * fail with -ENOSYS if not implemented. + */ +SYSCALL_DEFINE3(io_submit, aio_context_t, ctx_id, long, nr, + struct iocb __user * __user *, iocbpp) +{ + struct kioctx *ctx; + long ret = 0; + int i = 0; + struct blk_plug plug; + + if (unlikely(nr < 0)) + return -EINVAL; + + ctx = lookup_ioctx(ctx_id); + if (unlikely(!ctx)) { + pr_debug("EINVAL: invalid context id\n"); + return -EINVAL; + } + + if (nr > ctx->nr_events) + nr = ctx->nr_events; + + if (nr > AIO_PLUG_THRESHOLD) + blk_start_plug(&plug); + for (i = 0; i < nr; i++) { + struct iocb __user *user_iocb; + + if (unlikely(get_user(user_iocb, iocbpp + i))) { + ret = -EFAULT; + break; + } + + ret = io_submit_one(ctx, user_iocb, false); + if (ret) + break; + } + if (nr > AIO_PLUG_THRESHOLD) + blk_finish_plug(&plug); + + percpu_ref_put(&ctx->users); + return i ? i : ret; +} + +#ifdef CONFIG_COMPAT +COMPAT_SYSCALL_DEFINE3(io_submit, compat_aio_context_t, ctx_id, + int, nr, compat_uptr_t __user *, iocbpp) +{ + struct kioctx *ctx; + long ret = 0; + int i = 0; + struct blk_plug plug; + + if (unlikely(nr < 0)) + return -EINVAL; + + ctx = lookup_ioctx(ctx_id); + if (unlikely(!ctx)) { + pr_debug("EINVAL: invalid context id\n"); + return -EINVAL; + } + + if (nr > ctx->nr_events) + nr = ctx->nr_events; + + if (nr > AIO_PLUG_THRESHOLD) + blk_start_plug(&plug); + for (i = 0; i < nr; i++) { + compat_uptr_t user_iocb; + + if (unlikely(get_user(user_iocb, iocbpp + i))) { + ret = -EFAULT; + break; + } + + ret = io_submit_one(ctx, compat_ptr(user_iocb), true); + if (ret) + break; + } + if (nr > AIO_PLUG_THRESHOLD) + blk_finish_plug(&plug); + + percpu_ref_put(&ctx->users); + return i ? i : ret; +} +#endif + +/* sys_io_cancel: + * Attempts to cancel an iocb previously passed to io_submit. If + * the operation is successfully cancelled, the resulting event is + * copied into the memory pointed to by result without being placed + * into the completion queue and 0 is returned. May fail with + * -EFAULT if any of the data structures pointed to are invalid. + * May fail with -EINVAL if aio_context specified by ctx_id is + * invalid. May fail with -EAGAIN if the iocb specified was not + * cancelled. Will fail with -ENOSYS if not implemented. + */ +SYSCALL_DEFINE3(io_cancel, aio_context_t, ctx_id, struct iocb __user *, iocb, + struct io_event __user *, result) +{ + struct kioctx *ctx; + struct aio_kiocb *kiocb; + int ret = -EINVAL; + u32 key; + u64 obj = (u64)(unsigned long)iocb; + + if (unlikely(get_user(key, &iocb->aio_key))) + return -EFAULT; + if (unlikely(key != KIOCB_KEY)) + return -EINVAL; + + ctx = lookup_ioctx(ctx_id); + if (unlikely(!ctx)) + return -EINVAL; + + spin_lock_irq(&ctx->ctx_lock); + list_for_each_entry(kiocb, &ctx->active_reqs, ki_list) { + if (kiocb->ki_res.obj == obj) { + ret = kiocb->ki_cancel(&kiocb->rw); + list_del_init(&kiocb->ki_list); + break; + } + } + spin_unlock_irq(&ctx->ctx_lock); + + if (!ret) { + /* + * The result argument is no longer used - the io_event is + * always delivered via the ring buffer. -EINPROGRESS indicates + * cancellation is progress: + */ + ret = -EINPROGRESS; + } + + percpu_ref_put(&ctx->users); + + return ret; +} + +static long do_io_getevents(aio_context_t ctx_id, + long min_nr, + long nr, + struct io_event __user *events, + struct timespec64 *ts) +{ + ktime_t until = ts ? timespec64_to_ktime(*ts) : KTIME_MAX; + struct kioctx *ioctx = lookup_ioctx(ctx_id); + long ret = -EINVAL; + + if (likely(ioctx)) { + if (likely(min_nr <= nr && min_nr >= 0)) + ret = read_events(ioctx, min_nr, nr, events, until); + percpu_ref_put(&ioctx->users); + } + + return ret; +} + +/* io_getevents: + * Attempts to read at least min_nr events and up to nr events from + * the completion queue for the aio_context specified by ctx_id. If + * it succeeds, the number of read events is returned. May fail with + * -EINVAL if ctx_id is invalid, if min_nr is out of range, if nr is + * out of range, if timeout is out of range. May fail with -EFAULT + * if any of the memory specified is invalid. May return 0 or + * < min_nr if the timeout specified by timeout has elapsed + * before sufficient events are available, where timeout == NULL + * specifies an infinite timeout. Note that the timeout pointed to by + * timeout is relative. Will fail with -ENOSYS if not implemented. + */ +#ifdef CONFIG_64BIT + +SYSCALL_DEFINE5(io_getevents, aio_context_t, ctx_id, + long, min_nr, + long, nr, + struct io_event __user *, events, + struct __kernel_timespec __user *, timeout) +{ + struct timespec64 ts; + int ret; + + if (timeout && unlikely(get_timespec64(&ts, timeout))) + return -EFAULT; + + ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL); + if (!ret && signal_pending(current)) + ret = -EINTR; + return ret; +} + +#endif + +struct __aio_sigset { + const sigset_t __user *sigmask; + size_t sigsetsize; +}; + +SYSCALL_DEFINE6(io_pgetevents, + aio_context_t, ctx_id, + long, min_nr, + long, nr, + struct io_event __user *, events, + struct __kernel_timespec __user *, timeout, + const struct __aio_sigset __user *, usig) +{ + struct __aio_sigset ksig = { NULL, }; + struct timespec64 ts; + bool interrupted; + int ret; + + if (timeout && unlikely(get_timespec64(&ts, timeout))) + return -EFAULT; + + if (usig && copy_from_user(&ksig, usig, sizeof(ksig))) + return -EFAULT; + + ret = set_user_sigmask(ksig.sigmask, ksig.sigsetsize); + if (ret) + return ret; + + ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL); + + interrupted = signal_pending(current); + restore_saved_sigmask_unless(interrupted); + if (interrupted && !ret) + ret = -ERESTARTNOHAND; + + return ret; +} + +#if defined(CONFIG_COMPAT_32BIT_TIME) && !defined(CONFIG_64BIT) + +SYSCALL_DEFINE6(io_pgetevents_time32, + aio_context_t, ctx_id, + long, min_nr, + long, nr, + struct io_event __user *, events, + struct old_timespec32 __user *, timeout, + const struct __aio_sigset __user *, usig) +{ + struct __aio_sigset ksig = { NULL, }; + struct timespec64 ts; + bool interrupted; + int ret; + + if (timeout && unlikely(get_old_timespec32(&ts, timeout))) + return -EFAULT; + + if (usig && copy_from_user(&ksig, usig, sizeof(ksig))) + return -EFAULT; + + + ret = set_user_sigmask(ksig.sigmask, ksig.sigsetsize); + if (ret) + return ret; + + ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL); + + interrupted = signal_pending(current); + restore_saved_sigmask_unless(interrupted); + if (interrupted && !ret) + ret = -ERESTARTNOHAND; + + return ret; +} + +#endif + +#if defined(CONFIG_COMPAT_32BIT_TIME) + +SYSCALL_DEFINE5(io_getevents_time32, __u32, ctx_id, + __s32, min_nr, + __s32, nr, + struct io_event __user *, events, + struct old_timespec32 __user *, timeout) +{ + struct timespec64 t; + int ret; + + if (timeout && get_old_timespec32(&t, timeout)) + return -EFAULT; + + ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL); + if (!ret && signal_pending(current)) + ret = -EINTR; + return ret; +} + +#endif + +#ifdef CONFIG_COMPAT + +struct __compat_aio_sigset { + compat_uptr_t sigmask; + compat_size_t sigsetsize; +}; + +#if defined(CONFIG_COMPAT_32BIT_TIME) + +COMPAT_SYSCALL_DEFINE6(io_pgetevents, + compat_aio_context_t, ctx_id, + compat_long_t, min_nr, + compat_long_t, nr, + struct io_event __user *, events, + struct old_timespec32 __user *, timeout, + const struct __compat_aio_sigset __user *, usig) +{ + struct __compat_aio_sigset ksig = { 0, }; + struct timespec64 t; + bool interrupted; + int ret; + + if (timeout && get_old_timespec32(&t, timeout)) + return -EFAULT; + + if (usig && copy_from_user(&ksig, usig, sizeof(ksig))) + return -EFAULT; + + ret = set_compat_user_sigmask(compat_ptr(ksig.sigmask), ksig.sigsetsize); + if (ret) + return ret; + + ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL); + + interrupted = signal_pending(current); + restore_saved_sigmask_unless(interrupted); + if (interrupted && !ret) + ret = -ERESTARTNOHAND; + + return ret; +} + +#endif + +COMPAT_SYSCALL_DEFINE6(io_pgetevents_time64, + compat_aio_context_t, ctx_id, + compat_long_t, min_nr, + compat_long_t, nr, + struct io_event __user *, events, + struct __kernel_timespec __user *, timeout, + const struct __compat_aio_sigset __user *, usig) +{ + struct __compat_aio_sigset ksig = { 0, }; + struct timespec64 t; + bool interrupted; + int ret; + + if (timeout && get_timespec64(&t, timeout)) + return -EFAULT; + + if (usig && copy_from_user(&ksig, usig, sizeof(ksig))) + return -EFAULT; + + ret = set_compat_user_sigmask(compat_ptr(ksig.sigmask), ksig.sigsetsize); + if (ret) + return ret; + + ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL); + + interrupted = signal_pending(current); + restore_saved_sigmask_unless(interrupted); + if (interrupted && !ret) + ret = -ERESTARTNOHAND; + + return ret; +} +#endif diff --git a/fs/anon_inodes.c b/fs/anon_inodes.c new file mode 100644 index 000000000..a7b9b948e --- /dev/null +++ b/fs/anon_inodes.c @@ -0,0 +1,358 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * fs/anon_inodes.c + * + * Copyright (C) 2007 Davide Libenzi + * + * Thanks to Arnd Bergmann for code review and suggestions. + * More changes for Thomas Gleixner suggestions. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "internal.h" + +static struct vfsmount *anon_inode_mnt __ro_after_init; +static struct inode *anon_inode_inode __ro_after_init; + +/* + * User space expects anonymous inodes to have no file type in st_mode. + * + * In particular, 'lsof' has this legacy logic: + * + * type = s->st_mode & S_IFMT; + * switch (type) { + * ... + * case 0: + * if (!strcmp(p, "anon_inode")) + * Lf->ntype = Ntype = N_ANON_INODE; + * + * to detect our old anon_inode logic. + * + * Rather than mess with our internal sane inode data, just fix it + * up here in getattr() by masking off the format bits. + */ +int anon_inode_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, + unsigned int query_flags) +{ + struct inode *inode = d_inode(path->dentry); + + generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat); + stat->mode &= ~S_IFMT; + return 0; +} + +int anon_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +{ + return -EOPNOTSUPP; +} + +static const struct inode_operations anon_inode_operations = { + .getattr = anon_inode_getattr, + .setattr = anon_inode_setattr, +}; + +/* + * anon_inodefs_dname() is called from d_path(). + */ +static char *anon_inodefs_dname(struct dentry *dentry, char *buffer, int buflen) +{ + return dynamic_dname(buffer, buflen, "anon_inode:%s", + dentry->d_name.name); +} + +static const struct dentry_operations anon_inodefs_dentry_operations = { + .d_dname = anon_inodefs_dname, +}; + +static int anon_inodefs_init_fs_context(struct fs_context *fc) +{ + struct pseudo_fs_context *ctx = init_pseudo(fc, ANON_INODE_FS_MAGIC); + if (!ctx) + return -ENOMEM; + ctx->dops = &anon_inodefs_dentry_operations; + return 0; +} + +static struct file_system_type anon_inode_fs_type = { + .name = "anon_inodefs", + .init_fs_context = anon_inodefs_init_fs_context, + .kill_sb = kill_anon_super, +}; + +/** + * anon_inode_make_secure_inode - allocate an anonymous inode with security context + * @sb: [in] Superblock to allocate from + * @name: [in] Name of the class of the newfile (e.g., "secretmem") + * @context_inode: + * [in] Optional parent inode for security inheritance + * + * The function ensures proper security initialization through the LSM hook + * security_inode_init_security_anon(). + * + * Return: Pointer to new inode on success, ERR_PTR on failure. + */ +struct inode *anon_inode_make_secure_inode(struct super_block *sb, const char *name, + const struct inode *context_inode) +{ + struct inode *inode; + int error; + + inode = alloc_anon_inode(sb); + if (IS_ERR(inode)) + return inode; + inode->i_flags &= ~S_PRIVATE; + inode->i_op = &anon_inode_operations; + error = security_inode_init_security_anon(inode, &QSTR(name), + context_inode); + if (error) { + iput(inode); + return ERR_PTR(error); + } + return inode; +} +EXPORT_SYMBOL_FOR_MODULES(anon_inode_make_secure_inode, "kvm"); + +static struct file *__anon_inode_getfile(const char *name, + const struct file_operations *fops, + void *priv, int flags, + const struct inode *context_inode, + bool make_inode) +{ + struct inode *inode; + struct file *file; + + if (fops->owner && !try_module_get(fops->owner)) + return ERR_PTR(-ENOENT); + + if (make_inode) { + inode = anon_inode_make_secure_inode(anon_inode_mnt->mnt_sb, + name, context_inode); + if (IS_ERR(inode)) { + file = ERR_CAST(inode); + goto err; + } + } else { + inode = anon_inode_inode; + if (IS_ERR(inode)) { + file = ERR_PTR(-ENODEV); + goto err; + } + /* + * We know the anon_inode inode count is always + * greater than zero, so ihold() is safe. + */ + ihold(inode); + } + + file = alloc_file_pseudo(inode, anon_inode_mnt, name, + flags & (O_ACCMODE | O_NONBLOCK), fops); + if (IS_ERR(file)) + goto err_iput; + + file->f_mapping = inode->i_mapping; + + file->private_data = priv; + + return file; + +err_iput: + iput(inode); +err: + module_put(fops->owner); + return file; +} + +/** + * anon_inode_getfile - creates a new file instance by hooking it up to an + * anonymous inode, and a dentry that describe the "class" + * of the file + * + * @name: [in] name of the "class" of the new file + * @fops: [in] file operations for the new file + * @priv: [in] private data for the new file (will be file's private_data) + * @flags: [in] flags + * + * Creates a new file by hooking it on a single inode. This is useful for files + * that do not need to have a full-fledged inode in order to operate correctly. + * All the files created with anon_inode_getfile() will share a single inode, + * hence saving memory and avoiding code duplication for the file/inode/dentry + * setup. Returns the newly created file* or an error pointer. + */ +struct file *anon_inode_getfile(const char *name, + const struct file_operations *fops, + void *priv, int flags) +{ + return __anon_inode_getfile(name, fops, priv, flags, NULL, false); +} +EXPORT_SYMBOL_GPL(anon_inode_getfile); + +/** + * anon_inode_getfile_fmode - creates a new file instance by hooking it up to an + * anonymous inode, and a dentry that describe the "class" + * of the file + * + * @name: [in] name of the "class" of the new file + * @fops: [in] file operations for the new file + * @priv: [in] private data for the new file (will be file's private_data) + * @flags: [in] flags + * @f_mode: [in] fmode + * + * Creates a new file by hooking it on a single inode. This is useful for files + * that do not need to have a full-fledged inode in order to operate correctly. + * All the files created with anon_inode_getfile() will share a single inode, + * hence saving memory and avoiding code duplication for the file/inode/dentry + * setup. Allows setting the fmode. Returns the newly created file* or an error + * pointer. + */ +struct file *anon_inode_getfile_fmode(const char *name, + const struct file_operations *fops, + void *priv, int flags, fmode_t f_mode) +{ + struct file *file; + + file = __anon_inode_getfile(name, fops, priv, flags, NULL, false); + if (!IS_ERR(file)) + file->f_mode |= f_mode; + + return file; +} +EXPORT_SYMBOL_GPL(anon_inode_getfile_fmode); + +/** + * anon_inode_create_getfile - Like anon_inode_getfile(), but creates a new + * !S_PRIVATE anon inode rather than reuse the + * singleton anon inode and calls the + * inode_init_security_anon() LSM hook. + * + * @name: [in] name of the "class" of the new file + * @fops: [in] file operations for the new file + * @priv: [in] private data for the new file (will be file's private_data) + * @flags: [in] flags + * @context_inode: + * [in] the logical relationship with the new inode (optional) + * + * Create a new anonymous inode and file pair. This can be done for two + * reasons: + * + * - for the inode to have its own security context, so that LSMs can enforce + * policy on the inode's creation; + * + * - if the caller needs a unique inode, for example in order to customize + * the size returned by fstat() + * + * The LSM may use @context_inode in inode_init_security_anon(), but a + * reference to it is not held. + * + * Returns the newly created file* or an error pointer. + */ +struct file *anon_inode_create_getfile(const char *name, + const struct file_operations *fops, + void *priv, int flags, + const struct inode *context_inode) +{ + return __anon_inode_getfile(name, fops, priv, flags, + context_inode, true); +} +EXPORT_SYMBOL_GPL(anon_inode_create_getfile); + +static int __anon_inode_getfd(const char *name, + const struct file_operations *fops, + void *priv, int flags, + const struct inode *context_inode, + bool make_inode) +{ + return FD_ADD(flags, __anon_inode_getfile(name, fops, priv, flags, + context_inode, make_inode)); +} + +/** + * anon_inode_getfd - creates a new file instance by hooking it up to + * an anonymous inode and a dentry that describe + * the "class" of the file + * + * @name: [in] name of the "class" of the new file + * @fops: [in] file operations for the new file + * @priv: [in] private data for the new file (will be file's private_data) + * @flags: [in] flags + * + * Creates a new file by hooking it on a single inode. This is + * useful for files that do not need to have a full-fledged inode in + * order to operate correctly. All the files created with + * anon_inode_getfd() will use the same singleton inode, reducing + * memory use and avoiding code duplication for the file/inode/dentry + * setup. Returns a newly created file descriptor or an error code. + */ +int anon_inode_getfd(const char *name, const struct file_operations *fops, + void *priv, int flags) +{ + return __anon_inode_getfd(name, fops, priv, flags, NULL, false); +} +EXPORT_SYMBOL_GPL(anon_inode_getfd); + +/** + * anon_inode_create_getfd - Like anon_inode_getfd(), but creates a new + * !S_PRIVATE anon inode rather than reuse the singleton anon inode, and calls + * the inode_init_security_anon() LSM hook. + * + * @name: [in] name of the "class" of the new file + * @fops: [in] file operations for the new file + * @priv: [in] private data for the new file (will be file's private_data) + * @flags: [in] flags + * @context_inode: + * [in] the logical relationship with the new inode (optional) + * + * Create a new anonymous inode and file pair. This can be done for two + * reasons: + * + * - for the inode to have its own security context, so that LSMs can enforce + * policy on the inode's creation; + * + * - if the caller needs a unique inode, for example in order to customize + * the size returned by fstat() + * + * The LSM may use @context_inode in inode_init_security_anon(), but a + * reference to it is not held. + * + * Returns a newly created file descriptor or an error code. + */ +int anon_inode_create_getfd(const char *name, const struct file_operations *fops, + void *priv, int flags, + const struct inode *context_inode) +{ + return __anon_inode_getfd(name, fops, priv, flags, context_inode, true); +} + + +static int __init anon_inode_init(void) +{ + anon_inode_mnt = kern_mount(&anon_inode_fs_type); + if (IS_ERR(anon_inode_mnt)) + panic("anon_inode_init() kernel mount failed (%ld)\n", PTR_ERR(anon_inode_mnt)); + + anon_inode_inode = alloc_anon_inode(anon_inode_mnt->mnt_sb); + if (IS_ERR(anon_inode_inode)) + panic("anon_inode_init() inode allocation failed (%ld)\n", PTR_ERR(anon_inode_inode)); + anon_inode_inode->i_op = &anon_inode_operations; + + return 0; +} + +fs_initcall(anon_inode_init); + diff --git a/fs/attr.c b/fs/attr.c new file mode 100644 index 000000000..71888ac90 --- /dev/null +++ b/fs/attr.c @@ -0,0 +1,567 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/attr.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * changes by Thomas Schoebel-Theuer + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * setattr_should_drop_sgid - determine whether the setgid bit needs to be + * removed + * @idmap: idmap of the mount @inode was found from + * @inode: inode to check + * + * This function determines whether the setgid bit needs to be removed. + * We retain backwards compatibility and require setgid bit to be removed + * unconditionally if S_IXGRP is set. Otherwise we have the exact same + * requirements as setattr_prepare() and setattr_copy(). + * + * Return: ATTR_KILL_SGID if setgid bit needs to be removed, 0 otherwise. + */ +int setattr_should_drop_sgid(struct mnt_idmap *idmap, + const struct inode *inode) +{ + umode_t mode = inode->i_mode; + + if (!(mode & S_ISGID)) + return 0; + if (mode & S_IXGRP) + return ATTR_KILL_SGID; + if (!in_group_or_capable(idmap, inode, i_gid_into_vfsgid(idmap, inode))) + return ATTR_KILL_SGID; + return 0; +} +EXPORT_SYMBOL(setattr_should_drop_sgid); + +/** + * setattr_should_drop_suidgid - determine whether the set{g,u}id bit + * needs to be dropped + * @idmap: idmap of the mount @inode was found from + * @inode: inode to check + * + * This function determines whether the set{g,u}id bits need to be removed. + * If the setuid bit needs to be removed ATTR_KILL_SUID is returned. If the + * setgid bit needs to be removed ATTR_KILL_SGID is returned. If both + * set{g,u}id bits need to be removed the corresponding mask of both flags is + * returned. + * + * Return: A mask of ATTR_KILL_S{G,U}ID indicating which - if any - setid bits + * to remove, 0 otherwise. + */ +int setattr_should_drop_suidgid(struct mnt_idmap *idmap, + struct inode *inode) +{ + umode_t mode = inode->i_mode; + int kill = 0; + + /* suid always must be killed */ + if (unlikely(mode & S_ISUID)) + kill = ATTR_KILL_SUID; + + kill |= setattr_should_drop_sgid(idmap, inode); + + if (unlikely(kill && !capable(CAP_FSETID) && S_ISREG(mode))) + return kill; + + return 0; +} +EXPORT_SYMBOL(setattr_should_drop_suidgid); + +/** + * chown_ok - verify permissions to chown inode + * @idmap: idmap of the mount @inode was found from + * @inode: inode to check permissions on + * @ia_vfsuid: uid to chown @inode to + * + * If the inode has been found through an idmapped mount the idmap of + * the vfsmount must be passed through @idmap. This function will then + * take care to map the inode according to @idmap before checking + * permissions. On non-idmapped mounts or if permission checking is to be + * performed on the raw inode simply pass @nop_mnt_idmap. + */ +static bool chown_ok(struct mnt_idmap *idmap, + const struct inode *inode, vfsuid_t ia_vfsuid) +{ + vfsuid_t vfsuid = i_uid_into_vfsuid(idmap, inode); + if (vfsuid_eq_kuid(vfsuid, current_fsuid()) && + vfsuid_eq(ia_vfsuid, vfsuid)) + return true; + if (capable_wrt_inode_uidgid(idmap, inode, CAP_CHOWN)) + return true; + if (!vfsuid_valid(vfsuid) && + ns_capable(inode->i_sb->s_user_ns, CAP_CHOWN)) + return true; + return false; +} + +/** + * chgrp_ok - verify permissions to chgrp inode + * @idmap: idmap of the mount @inode was found from + * @inode: inode to check permissions on + * @ia_vfsgid: gid to chown @inode to + * + * If the inode has been found through an idmapped mount the idmap of + * the vfsmount must be passed through @idmap. This function will then + * take care to map the inode according to @idmap before checking + * permissions. On non-idmapped mounts or if permission checking is to be + * performed on the raw inode simply pass @nop_mnt_idmap. + */ +static bool chgrp_ok(struct mnt_idmap *idmap, + const struct inode *inode, vfsgid_t ia_vfsgid) +{ + vfsgid_t vfsgid = i_gid_into_vfsgid(idmap, inode); + vfsuid_t vfsuid = i_uid_into_vfsuid(idmap, inode); + if (vfsuid_eq_kuid(vfsuid, current_fsuid())) { + if (vfsgid_eq(ia_vfsgid, vfsgid)) + return true; + if (vfsgid_in_group_p(ia_vfsgid)) + return true; + } + if (capable_wrt_inode_uidgid(idmap, inode, CAP_CHOWN)) + return true; + if (!vfsgid_valid(vfsgid) && + ns_capable(inode->i_sb->s_user_ns, CAP_CHOWN)) + return true; + return false; +} + +/** + * setattr_prepare - check if attribute changes to a dentry are allowed + * @idmap: idmap of the mount the inode was found from + * @dentry: dentry to check + * @attr: attributes to change + * + * Check if we are allowed to change the attributes contained in @attr + * in the given dentry. This includes the normal unix access permission + * checks, as well as checks for rlimits and others. The function also clears + * SGID bit from mode if user is not allowed to set it. Also file capabilities + * and IMA extended attributes are cleared if ATTR_KILL_PRIV is set. + * + * If the inode has been found through an idmapped mount the idmap of + * the vfsmount must be passed through @idmap. This function will then + * take care to map the inode according to @idmap before checking + * permissions. On non-idmapped mounts or if permission checking is to be + * performed on the raw inode simply pass @nop_mnt_idmap. + * + * Should be called as the first thing in ->setattr implementations, + * possibly after taking additional locks. + */ +int setattr_prepare(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +{ + struct inode *inode = d_inode(dentry); + unsigned int ia_valid = attr->ia_valid; + + /* + * First check size constraints. These can't be overridden using + * ATTR_FORCE. + */ + if (ia_valid & ATTR_SIZE) { + int error; + + /* + * Verity files are immutable, so deny truncates. This isn't + * covered by the open-time check because sys_truncate() takes a + * path, not an open file. + */ + if (IS_VERITY(inode)) + return -EPERM; + + error = inode_newsize_ok(inode, attr->ia_size); + if (error) + return error; + } + + /* If force is set do it anyway. */ + if (ia_valid & ATTR_FORCE) + goto kill_priv; + + /* Make sure a caller can chown. */ + if ((ia_valid & ATTR_UID) && + !chown_ok(idmap, inode, attr->ia_vfsuid)) + return -EPERM; + + /* Make sure caller can chgrp. */ + if ((ia_valid & ATTR_GID) && + !chgrp_ok(idmap, inode, attr->ia_vfsgid)) + return -EPERM; + + /* Make sure a caller can chmod. */ + if (ia_valid & ATTR_MODE) { + vfsgid_t vfsgid; + + if (!inode_owner_or_capable(idmap, inode)) + return -EPERM; + + if (ia_valid & ATTR_GID) + vfsgid = attr->ia_vfsgid; + else + vfsgid = i_gid_into_vfsgid(idmap, inode); + + /* Also check the setgid bit! */ + if (!in_group_or_capable(idmap, inode, vfsgid)) + attr->ia_mode &= ~S_ISGID; + } + + /* Check for setting the inode time. */ + if (ia_valid & (ATTR_MTIME_SET | ATTR_ATIME_SET | ATTR_TIMES_SET)) { + if (!inode_owner_or_capable(idmap, inode)) + return -EPERM; + } + +kill_priv: + /* User has permission for the change */ + if (ia_valid & ATTR_KILL_PRIV) { + int error; + + error = security_inode_killpriv(idmap, dentry); + if (error) + return error; + } + + return 0; +} +EXPORT_SYMBOL(setattr_prepare); + +/** + * inode_newsize_ok - may this inode be truncated to a given size + * @inode: the inode to be truncated + * @offset: the new size to assign to the inode + * + * inode_newsize_ok must be called with i_rwsem held exclusively. + * + * inode_newsize_ok will check filesystem limits and ulimits to check that the + * new inode size is within limits. inode_newsize_ok will also send SIGXFSZ + * when necessary. Caller must not proceed with inode size change if failure is + * returned. @inode must be a file (not directory), with appropriate + * permissions to allow truncate (inode_newsize_ok does NOT check these + * conditions). + * + * Return: 0 on success, -ve errno on failure + */ +int inode_newsize_ok(const struct inode *inode, loff_t offset) +{ + if (offset < 0) + return -EINVAL; + if (inode->i_size < offset) { + unsigned long limit; + + limit = rlimit(RLIMIT_FSIZE); + if (limit != RLIM_INFINITY && offset > limit) + goto out_sig; + if (offset > inode->i_sb->s_maxbytes) + goto out_big; + } else { + /* + * truncation of in-use swapfiles is disallowed - it would + * cause subsequent swapout to scribble on the now-freed + * blocks. + */ + if (IS_SWAPFILE(inode)) + return -ETXTBSY; + } + + return 0; +out_sig: + send_sig(SIGXFSZ, current, 0); +out_big: + return -EFBIG; +} +EXPORT_SYMBOL(inode_newsize_ok); + +/** + * setattr_copy_mgtime - update timestamps for mgtime inodes + * @inode: inode timestamps to be updated + * @attr: attrs for the update + * + * With multigrain timestamps, take more care to prevent races when + * updating the ctime. Always update the ctime to the very latest using + * the standard mechanism, and use that to populate the atime and mtime + * appropriately (unless those are being set to specific values). + */ +static void setattr_copy_mgtime(struct inode *inode, const struct iattr *attr) +{ + unsigned int ia_valid = attr->ia_valid; + struct timespec64 now; + + if (ia_valid & ATTR_CTIME_SET) + now = inode_set_ctime_deleg(inode, attr->ia_ctime); + else if (ia_valid & ATTR_CTIME) + now = inode_set_ctime_current(inode); + else + now = current_time(inode); + + if (ia_valid & ATTR_ATIME_SET) + inode_set_atime_to_ts(inode, attr->ia_atime); + else if (ia_valid & ATTR_ATIME) + inode_set_atime_to_ts(inode, now); + + if (ia_valid & ATTR_MTIME_SET) + inode_set_mtime_to_ts(inode, attr->ia_mtime); + else if (ia_valid & ATTR_MTIME) + inode_set_mtime_to_ts(inode, now); +} + +/** + * setattr_copy - copy simple metadata updates into the generic inode + * @idmap: idmap of the mount the inode was found from + * @inode: the inode to be updated + * @attr: the new attributes + * + * setattr_copy must be called with i_rwsem held exclusively. + * + * setattr_copy updates the inode's metadata with that specified + * in attr on idmapped mounts. Necessary permission checks to determine + * whether or not the S_ISGID property needs to be removed are performed with + * the correct idmapped mount permission helpers. + * Noticeably missing is inode size update, which is more complex + * as it requires pagecache updates. + * + * If the inode has been found through an idmapped mount the idmap of + * the vfsmount must be passed through @idmap. This function will then + * take care to map the inode according to @idmap before checking + * permissions. On non-idmapped mounts or if permission checking is to be + * performed on the raw inode simply pass @nop_mnt_idmap. + * + * The inode is not marked as dirty after this operation. The rationale is + * that for "simple" filesystems, the struct inode is the inode storage. + * The caller is free to mark the inode dirty afterwards if needed. + */ +void setattr_copy(struct mnt_idmap *idmap, struct inode *inode, + const struct iattr *attr) +{ + unsigned int ia_valid = attr->ia_valid; + + i_uid_update(idmap, attr, inode); + i_gid_update(idmap, attr, inode); + if (ia_valid & ATTR_MODE) { + umode_t mode = attr->ia_mode; + if (!in_group_or_capable(idmap, inode, + i_gid_into_vfsgid(idmap, inode))) + mode &= ~S_ISGID; + inode->i_mode = mode; + } + + if (is_mgtime(inode)) + return setattr_copy_mgtime(inode, attr); + + if (ia_valid & ATTR_ATIME) + inode_set_atime_to_ts(inode, attr->ia_atime); + if (ia_valid & ATTR_MTIME) + inode_set_mtime_to_ts(inode, attr->ia_mtime); + + if (ia_valid & ATTR_CTIME_SET) + inode_set_ctime_deleg(inode, attr->ia_ctime); + else if (ia_valid & ATTR_CTIME) + inode_set_ctime_to_ts(inode, attr->ia_ctime); +} +EXPORT_SYMBOL(setattr_copy); + +int may_setattr(struct mnt_idmap *idmap, struct inode *inode, + unsigned int ia_valid) +{ + int error; + + if (ia_valid & (ATTR_MODE | ATTR_UID | ATTR_GID | ATTR_TIMES_SET)) { + if (IS_IMMUTABLE(inode) || IS_APPEND(inode)) + return -EPERM; + } + + /* + * If utimes(2) and friends are called with times == NULL (or both + * times are UTIME_NOW), then we need to check for write permission + */ + if (ia_valid & ATTR_TOUCH) { + if (IS_IMMUTABLE(inode)) + return -EPERM; + + if (!inode_owner_or_capable(idmap, inode)) { + error = inode_permission(idmap, inode, MAY_WRITE); + if (error) + return error; + } + } + return 0; +} +EXPORT_SYMBOL(may_setattr); + +/** + * notify_change - modify attributes of a filesystem object + * @idmap: idmap of the mount the inode was found from + * @dentry: object affected + * @attr: new attributes + * @delegated_inode: returns inode, if the inode is delegated + * + * The caller must hold the i_rwsem exclusively on the affected object. + * + * If notify_change discovers a delegation in need of breaking, + * it will return -EWOULDBLOCK and return a reference to the inode in + * delegated_inode. The caller should then break the delegation and + * retry. Because breaking a delegation may take a long time, the + * caller should drop the i_rwsem before doing so. + * + * Alternatively, a caller may pass NULL for delegated_inode. This may + * be appropriate for callers that expect the underlying filesystem not + * to be NFS exported. Also, passing NULL is fine for callers holding + * the file open for write, as there can be no conflicting delegation in + * that case. + * + * If the inode has been found through an idmapped mount the idmap of + * the vfsmount must be passed through @idmap. This function will then + * take care to map the inode according to @idmap before checking + * permissions. On non-idmapped mounts or if permission checking is to be + * performed on the raw inode simply pass @nop_mnt_idmap. + */ +int notify_change(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr, struct delegated_inode *delegated_inode) +{ + struct inode *inode = dentry->d_inode; + umode_t mode = inode->i_mode; + int error; + struct timespec64 now; + unsigned int ia_valid = attr->ia_valid; + + WARN_ON_ONCE(!inode_is_locked(inode)); + + error = may_setattr(idmap, inode, ia_valid); + if (error) + return error; + + if ((ia_valid & ATTR_MODE)) { + /* + * Don't allow changing the mode of symlinks: + * + * (1) The vfs doesn't take the mode of symlinks into account + * during permission checking. + * (2) This has never worked correctly. Most major filesystems + * did return EOPNOTSUPP due to interactions with POSIX ACLs + * but did still updated the mode of the symlink. + * This inconsistency led system call wrapper providers such + * as libc to block changing the mode of symlinks with + * EOPNOTSUPP already. + * (3) To even do this in the first place one would have to use + * specific file descriptors and quite some effort. + */ + if (S_ISLNK(inode->i_mode)) + return -EOPNOTSUPP; + + /* Flag setting protected by i_rwsem */ + if (is_sxid(attr->ia_mode)) + inode->i_flags &= ~S_NOSEC; + } + + now = current_time(inode); + + if (ia_valid & ATTR_ATIME_SET) + attr->ia_atime = timestamp_truncate(attr->ia_atime, inode); + else + attr->ia_atime = now; + if (ia_valid & ATTR_CTIME_SET) + attr->ia_ctime = timestamp_truncate(attr->ia_ctime, inode); + else + attr->ia_ctime = now; + if (ia_valid & ATTR_MTIME_SET) + attr->ia_mtime = timestamp_truncate(attr->ia_mtime, inode); + else + attr->ia_mtime = now; + + if (ia_valid & ATTR_KILL_PRIV) { + error = security_inode_need_killpriv(dentry); + if (error < 0) + return error; + if (error == 0) + ia_valid = attr->ia_valid &= ~ATTR_KILL_PRIV; + } + + /* + * We now pass ATTR_KILL_S*ID to the lower level setattr function so + * that the function has the ability to reinterpret a mode change + * that's due to these bits. This adds an implicit restriction that + * no function will ever call notify_change with both ATTR_MODE and + * ATTR_KILL_S*ID set. + */ + if ((ia_valid & (ATTR_KILL_SUID|ATTR_KILL_SGID)) && + (ia_valid & ATTR_MODE)) + BUG(); + + if (ia_valid & ATTR_KILL_SUID) { + if (mode & S_ISUID) { + ia_valid = attr->ia_valid |= ATTR_MODE; + attr->ia_mode = (inode->i_mode & ~S_ISUID); + } + } + if (ia_valid & ATTR_KILL_SGID) { + if (mode & S_ISGID) { + if (!(ia_valid & ATTR_MODE)) { + ia_valid = attr->ia_valid |= ATTR_MODE; + attr->ia_mode = inode->i_mode; + } + attr->ia_mode &= ~S_ISGID; + } + } + if (!(attr->ia_valid & ~(ATTR_KILL_SUID | ATTR_KILL_SGID))) + return 0; + + /* + * Verify that uid/gid changes are valid in the target + * namespace of the superblock. + */ + if (ia_valid & ATTR_UID && + !vfsuid_has_fsmapping(idmap, inode->i_sb->s_user_ns, + attr->ia_vfsuid)) + return -EOVERFLOW; + if (ia_valid & ATTR_GID && + !vfsgid_has_fsmapping(idmap, inode->i_sb->s_user_ns, + attr->ia_vfsgid)) + return -EOVERFLOW; + + /* Don't allow modifications of files with invalid uids or + * gids unless those uids & gids are being made valid. + */ + if (!(ia_valid & ATTR_UID) && + !vfsuid_valid(i_uid_into_vfsuid(idmap, inode))) + return -EOVERFLOW; + if (!(ia_valid & ATTR_GID) && + !vfsgid_valid(i_gid_into_vfsgid(idmap, inode))) + return -EOVERFLOW; + + error = security_inode_setattr(idmap, dentry, attr); + if (error) + return error; + + /* + * If ATTR_DELEG is set, then these attributes are being set on + * behalf of the holder of a write delegation. We want to avoid + * breaking the delegation in this case. + */ + if (!(ia_valid & ATTR_DELEG)) { + error = try_break_deleg(inode, 0, delegated_inode); + if (error) + return error; + } + + if (inode->i_op->setattr) + error = inode->i_op->setattr(idmap, dentry, attr); + else + error = simple_setattr(idmap, dentry, attr); + + if (!error) { + fsnotify_change(dentry, ia_valid); + security_inode_post_setattr(idmap, dentry, ia_valid); + } + + return error; +} +EXPORT_SYMBOL(notify_change); diff --git a/fs/autofs/Kconfig b/fs/autofs/Kconfig new file mode 100644 index 000000000..54c12d948 --- /dev/null +++ b/fs/autofs/Kconfig @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: GPL-2.0-only +config AUTOFS_FS + tristate "Kernel automounter support (supports v3, v4 and v5)" + help + The automounter is a tool to automatically mount remote file systems + on demand. This implementation is partially kernel-based to reduce + overhead in the already-mounted case; this is unlike the BSD + automounter (amd), which is a pure user space daemon. + + To use the automounter you need the user-space tools from + ; you also want + to answer Y to "NFS file system support", below. + + To compile this support as a module, choose M here: the module will be + called autofs. + + If you are not a part of a fairly large, distributed network or + don't have a laptop which needs to dynamically reconfigure to the + local network, you probably do not need an automounter, and can say + N here. diff --git a/fs/autofs/Makefile b/fs/autofs/Makefile new file mode 100644 index 000000000..495ac542e --- /dev/null +++ b/fs/autofs/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the linux autofs-filesystem routines. +# + +obj-$(CONFIG_AUTOFS_FS) += autofs4.o + +autofs4-objs := init.o inode.o root.o symlink.o waitq.o expire.o dev-ioctl.o diff --git a/fs/autofs/autofs_i.h b/fs/autofs/autofs_i.h new file mode 100644 index 000000000..4fd555528 --- /dev/null +++ b/fs/autofs/autofs_i.h @@ -0,0 +1,306 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright 1997-1998 Transmeta Corporation - All Rights Reserved + * Copyright 2005-2006 Ian Kent + */ + +/* Internal header file for autofs */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../mount.h" +#include + + +/* This is the range of ioctl() numbers we claim as ours */ +#define AUTOFS_IOC_FIRST AUTOFS_IOC_READY +#define AUTOFS_IOC_COUNT 32 + +#define AUTOFS_DEV_IOCTL_IOC_FIRST (AUTOFS_DEV_IOCTL_VERSION) +#define AUTOFS_DEV_IOCTL_IOC_COUNT \ + (AUTOFS_DEV_IOCTL_ISMOUNTPOINT_CMD - AUTOFS_DEV_IOCTL_VERSION_CMD) + +#ifdef pr_fmt +#undef pr_fmt +#endif +#define pr_fmt(fmt) KBUILD_MODNAME ":pid:%d:%s: " fmt, current->pid, __func__ + +extern struct file_system_type autofs_fs_type; + +/* + * Unified info structure. This is pointed to by both the dentry and + * inode structures. Each file in the filesystem has an instance of this + * structure. It holds a reference to the dentry, so dentries are never + * flushed while the file exists. All name lookups are dealt with at the + * dentry level, although the filesystem can interfere in the validation + * process. Readdir is implemented by traversing the dentry lists. + */ +struct autofs_info { + struct dentry *dentry; + int flags; + + struct completion expire_complete; + + struct list_head active; + + struct list_head expiring; + + struct autofs_sb_info *sbi; + unsigned long exp_timeout; + unsigned long last_used; + int count; + + kuid_t uid; + kgid_t gid; + struct rcu_head rcu; +}; + +#define AUTOFS_INF_EXPIRING (1<<0) /* dentry in the process of expiring */ +#define AUTOFS_INF_WANT_EXPIRE (1<<1) /* the dentry is being considered + * for expiry, so RCU_walk is + * not permitted. If it progresses to + * actual expiry attempt, the flag is + * not cleared when EXPIRING is set - + * in that case it gets cleared only + * when it comes to clearing EXPIRING. + */ +#define AUTOFS_INF_PENDING (1<<2) /* dentry pending mount */ + +#define AUTOFS_INF_EXPIRE_SET (1<<3) /* per-dentry expire timeout set for + this mount point. + */ +struct autofs_wait_queue { + wait_queue_head_t queue; + struct autofs_wait_queue *next; + autofs_wqt_t wait_queue_token; + /* We use the following to see what we are waiting for */ + struct qstr name; + u32 offset; + u32 dev; + u64 ino; + kuid_t uid; + kgid_t gid; + pid_t pid; + pid_t tgid; + /* This is for status reporting upon return */ + int status; + unsigned int wait_ctr; +}; + +#define AUTOFS_SBI_MAGIC 0x6d4a556d + +#define AUTOFS_SBI_CATATONIC 0x0001 +#define AUTOFS_SBI_STRICTEXPIRE 0x0002 +#define AUTOFS_SBI_IGNORE 0x0004 + +struct autofs_sb_info { + u32 magic; + int pipefd; + struct file *pipe; + struct pid *oz_pgrp; + u64 mnt_ns_id; + int version; + int sub_version; + int min_proto; + int max_proto; + unsigned int flags; + unsigned long exp_timeout; + unsigned int type; + struct super_block *sb; + struct mutex wq_mutex; + struct mutex pipe_mutex; + spinlock_t fs_lock; + struct autofs_wait_queue *queues; /* Wait queue pointer */ + spinlock_t lookup_lock; + struct list_head active_list; + struct list_head expiring_list; + struct rcu_head rcu; +}; + +static inline struct autofs_sb_info *autofs_sbi(struct super_block *sb) +{ + return (struct autofs_sb_info *)(sb->s_fs_info); +} + +static inline struct autofs_info *autofs_dentry_ino(struct dentry *dentry) +{ + return (struct autofs_info *)(dentry->d_fsdata); +} + +/* autofs_oz_mode(): do we see the man behind the curtain? (The + * processes which do manipulations for us in user space sees the raw + * filesystem without "magic".) + */ +static inline int autofs_oz_mode(struct autofs_sb_info *sbi) +{ + return ((sbi->flags & AUTOFS_SBI_CATATONIC) || + task_pgrp(current) == sbi->oz_pgrp); +} + +static inline bool autofs_empty(struct autofs_info *ino) +{ + return ino->count < 2; +} + +struct inode *autofs_get_inode(struct super_block *, umode_t); +void autofs_free_ino(struct autofs_info *); + +/* Expiration */ +int is_autofs_dentry(struct dentry *); +int autofs_expire_wait(const struct path *path, int rcu_walk); +int autofs_expire_run(struct super_block *, struct vfsmount *, + struct autofs_sb_info *, + struct autofs_packet_expire __user *); +int autofs_do_expire_multi(struct super_block *sb, struct vfsmount *mnt, + struct autofs_sb_info *sbi, unsigned int how); +int autofs_expire_multi(struct super_block *, struct vfsmount *, + struct autofs_sb_info *, int __user *); + +/* Device node initialization */ + +int autofs_dev_ioctl_init(void); +void autofs_dev_ioctl_exit(void); + +/* Operations structures */ + +extern const struct inode_operations autofs_symlink_inode_operations; +extern const struct inode_operations autofs_dir_inode_operations; +extern const struct file_operations autofs_dir_operations; +extern const struct file_operations autofs_root_operations; +extern const struct dentry_operations autofs_dentry_operations; + +/* VFS automount flags management functions */ +static inline void __managed_dentry_set_managed(struct dentry *dentry) +{ + dentry->d_flags |= (DCACHE_NEED_AUTOMOUNT|DCACHE_MANAGE_TRANSIT); +} + +static inline void managed_dentry_set_managed(struct dentry *dentry) +{ + spin_lock(&dentry->d_lock); + __managed_dentry_set_managed(dentry); + spin_unlock(&dentry->d_lock); +} + +static inline void __managed_dentry_clear_managed(struct dentry *dentry) +{ + dentry->d_flags &= ~(DCACHE_NEED_AUTOMOUNT|DCACHE_MANAGE_TRANSIT); +} + +static inline void managed_dentry_clear_managed(struct dentry *dentry) +{ + spin_lock(&dentry->d_lock); + __managed_dentry_clear_managed(dentry); + spin_unlock(&dentry->d_lock); +} + +/* Initializing function */ + +extern const struct fs_parameter_spec autofs_param_specs[]; +int autofs_init_fs_context(struct fs_context *fc); +struct autofs_info *autofs_new_ino(struct autofs_sb_info *); +void autofs_clean_ino(struct autofs_info *); + +static inline int autofs_check_pipe(struct file *pipe) +{ + if (pipe->f_mode & FMODE_PATH) + return -EINVAL; + if (!(pipe->f_mode & FMODE_CAN_WRITE)) + return -EINVAL; + if (!S_ISFIFO(file_inode(pipe)->i_mode)) + return -EINVAL; + return 0; +} + +static inline void autofs_set_packet_pipe_flags(struct file *pipe) +{ + /* We want a packet pipe */ + pipe->f_flags |= O_DIRECT; + /* We don't expect -EAGAIN */ + pipe->f_flags &= ~O_NONBLOCK; +} + +static inline int autofs_prepare_pipe(struct file *pipe) +{ + int ret = autofs_check_pipe(pipe); + if (ret < 0) + return ret; + autofs_set_packet_pipe_flags(pipe); + return 0; +} + +/* Queue management functions */ + +int autofs_wait(struct autofs_sb_info *, + const struct path *, enum autofs_notify); +int autofs_wait_release(struct autofs_sb_info *, autofs_wqt_t, int); +void autofs_catatonic_mode(struct autofs_sb_info *); + +static inline u32 autofs_get_dev(struct autofs_sb_info *sbi) +{ + return new_encode_dev(sbi->sb->s_dev); +} + +static inline u64 autofs_get_ino(struct autofs_sb_info *sbi) +{ + return d_inode(sbi->sb->s_root)->i_ino; +} + +static inline void __autofs_add_expiring(struct dentry *dentry) +{ + struct autofs_sb_info *sbi = autofs_sbi(dentry->d_sb); + struct autofs_info *ino = autofs_dentry_ino(dentry); + + if (ino) { + if (list_empty(&ino->expiring)) + list_add(&ino->expiring, &sbi->expiring_list); + } +} + +static inline void autofs_add_expiring(struct dentry *dentry) +{ + struct autofs_sb_info *sbi = autofs_sbi(dentry->d_sb); + struct autofs_info *ino = autofs_dentry_ino(dentry); + + if (ino) { + spin_lock(&sbi->lookup_lock); + if (list_empty(&ino->expiring)) + list_add(&ino->expiring, &sbi->expiring_list); + spin_unlock(&sbi->lookup_lock); + } +} + +static inline void autofs_del_expiring(struct dentry *dentry) +{ + struct autofs_sb_info *sbi = autofs_sbi(dentry->d_sb); + struct autofs_info *ino = autofs_dentry_ino(dentry); + + if (ino) { + spin_lock(&sbi->lookup_lock); + if (!list_empty(&ino->expiring)) + list_del_init(&ino->expiring); + spin_unlock(&sbi->lookup_lock); + } +} + +void autofs_kill_sb(struct super_block *); diff --git a/fs/autofs/dev-ioctl.c b/fs/autofs/dev-ioctl.c new file mode 100644 index 000000000..6743b3b64 --- /dev/null +++ b/fs/autofs/dev-ioctl.c @@ -0,0 +1,810 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 2008 Red Hat, Inc. All rights reserved. + * Copyright 2008 Ian Kent + */ + +#include +#include +#include +#include +#include +#include + +#include "autofs_i.h" + +/* + * This module implements an interface for routing autofs ioctl control + * commands via a miscellaneous device file. + * + * The alternate interface is needed because we need to be able open + * an ioctl file descriptor on an autofs mount that may be covered by + * another mount. This situation arises when starting automount(8) + * or other user space daemon which uses direct mounts or offset + * mounts (used for autofs lazy mount/umount of nested mount trees), + * which have been left busy at service shutdown. + */ + +typedef int (*ioctl_fn)(struct file *, struct autofs_sb_info *, + struct autofs_dev_ioctl *); + +static int check_name(const char *name) +{ + if (!strchr(name, '/')) + return -EINVAL; + return 0; +} + +/* + * Check a string doesn't overrun the chunk of + * memory we copied from user land. + */ +static int invalid_str(char *str, size_t size) +{ + if (memchr(str, 0, size)) + return 0; + return -EINVAL; +} + +/* + * Check that the user compiled against correct version of autofs + * misc device code. + * + * As well as checking the version compatibility this always copies + * the kernel interface version out. + */ +static int check_dev_ioctl_version(int cmd, struct autofs_dev_ioctl *param) +{ + int err = 0; + + if ((param->ver_major != AUTOFS_DEV_IOCTL_VERSION_MAJOR) || + (param->ver_minor > AUTOFS_DEV_IOCTL_VERSION_MINOR)) { + pr_warn("ioctl control interface version mismatch: " + "kernel(%u.%u), user(%u.%u), cmd(0x%08x)\n", + AUTOFS_DEV_IOCTL_VERSION_MAJOR, + AUTOFS_DEV_IOCTL_VERSION_MINOR, + param->ver_major, param->ver_minor, cmd); + err = -EINVAL; + } + + /* Fill in the kernel version. */ + param->ver_major = AUTOFS_DEV_IOCTL_VERSION_MAJOR; + param->ver_minor = AUTOFS_DEV_IOCTL_VERSION_MINOR; + + return err; +} + +/* + * Copy parameter control struct, including a possible path allocated + * at the end of the struct. + */ +static struct autofs_dev_ioctl * +copy_dev_ioctl(struct autofs_dev_ioctl __user *in) +{ + struct autofs_dev_ioctl tmp, *res; + + if (copy_from_user(&tmp, in, AUTOFS_DEV_IOCTL_SIZE)) + return ERR_PTR(-EFAULT); + + if (tmp.size < AUTOFS_DEV_IOCTL_SIZE) + return ERR_PTR(-EINVAL); + + if (tmp.size > AUTOFS_DEV_IOCTL_SIZE + PATH_MAX) + return ERR_PTR(-ENAMETOOLONG); + + res = memdup_user(in, tmp.size); + if (!IS_ERR(res)) + res->size = tmp.size; + + return res; +} + +static inline void free_dev_ioctl(struct autofs_dev_ioctl *param) +{ + kfree(param); +} + +/* + * Check sanity of parameter control fields and if a path is present + * check that it is terminated and contains at least one "/". + */ +static int validate_dev_ioctl(int cmd, struct autofs_dev_ioctl *param) +{ + unsigned int inr = _IOC_NR(cmd); + int err; + + err = check_dev_ioctl_version(cmd, param); + if (err) { + pr_warn("invalid device control module version " + "supplied for cmd(0x%08x)\n", cmd); + goto out; + } + + if (param->size > AUTOFS_DEV_IOCTL_SIZE) { + err = invalid_str(param->path, param->size - AUTOFS_DEV_IOCTL_SIZE); + if (err) { + pr_warn( + "path string terminator missing for cmd(0x%08x)\n", + cmd); + goto out; + } + + /* Setting the per-dentry expire timeout requires a trailing + * path component, ie. no '/', so invert the logic of the + * check_name() return for AUTOFS_DEV_IOCTL_TIMEOUT_CMD. + */ + err = check_name(param->path); + if (inr == AUTOFS_DEV_IOCTL_TIMEOUT_CMD) + err = err ? 0 : -EINVAL; + if (err) { + pr_warn("invalid path supplied for cmd(0x%08x)\n", + cmd); + goto out; + } + } else { + if (inr == AUTOFS_DEV_IOCTL_OPENMOUNT_CMD || + inr == AUTOFS_DEV_IOCTL_REQUESTER_CMD || + inr == AUTOFS_DEV_IOCTL_ISMOUNTPOINT_CMD) { + err = -EINVAL; + goto out; + } + } + + err = 0; +out: + return err; +} + +/* Return autofs dev ioctl version */ +static int autofs_dev_ioctl_version(struct file *fp, + struct autofs_sb_info *sbi, + struct autofs_dev_ioctl *param) +{ + /* This should have already been set. */ + param->ver_major = AUTOFS_DEV_IOCTL_VERSION_MAJOR; + param->ver_minor = AUTOFS_DEV_IOCTL_VERSION_MINOR; + return 0; +} + +/* Return autofs module protocol version */ +static int autofs_dev_ioctl_protover(struct file *fp, + struct autofs_sb_info *sbi, + struct autofs_dev_ioctl *param) +{ + param->protover.version = sbi->version; + return 0; +} + +/* Return autofs module protocol sub version */ +static int autofs_dev_ioctl_protosubver(struct file *fp, + struct autofs_sb_info *sbi, + struct autofs_dev_ioctl *param) +{ + param->protosubver.sub_version = sbi->sub_version; + return 0; +} + +/* Find the topmost mount satisfying test() */ +static int find_autofs_mount(const char *pathname, + struct path *res, + int test(const struct path *path, void *data), + void *data) +{ + struct path path; + int err; + + err = kern_path(pathname, LOOKUP_MOUNTPOINT, &path); + if (err) + return err; + err = -ENOENT; + while (path.dentry == path.mnt->mnt_root) { + if (path.dentry->d_sb->s_magic == AUTOFS_SUPER_MAGIC) { + if (test(&path, data)) { + path_get(&path); + *res = path; + err = 0; + break; + } + } + if (!follow_up(&path)) + break; + } + path_put(&path); + return err; +} + +static int test_by_dev(const struct path *path, void *p) +{ + return path->dentry->d_sb->s_dev == *(dev_t *)p; +} + +static int test_by_type(const struct path *path, void *p) +{ + struct autofs_info *ino = autofs_dentry_ino(path->dentry); + + return ino && ino->sbi->type & *(unsigned *)p; +} + +/* + * Open a file descriptor on the autofs mount point corresponding + * to the given path and device number (aka. new_encode_dev(sb->s_dev)). + */ +static int autofs_dev_ioctl_open_mountpoint(const char *name, dev_t devid) +{ + struct path path __free(path_put) = {}; + int err; + + err = find_autofs_mount(name, &path, test_by_dev, &devid); + if (err) + return err; + + return FD_ADD(O_CLOEXEC, dentry_open(&path, O_RDONLY, current_cred())); +} + +/* Open a file descriptor on an autofs mount point */ +static int autofs_dev_ioctl_openmount(struct file *fp, + struct autofs_sb_info *sbi, + struct autofs_dev_ioctl *param) +{ + const char *path; + dev_t devid; + int err, fd; + + /* param->path has been checked in validate_dev_ioctl() */ + + if (!param->openmount.devid) + return -EINVAL; + + param->ioctlfd = -1; + + path = param->path; + devid = new_decode_dev(param->openmount.devid); + + err = 0; + fd = autofs_dev_ioctl_open_mountpoint(path, devid); + if (unlikely(fd < 0)) { + err = fd; + goto out; + } + + param->ioctlfd = fd; +out: + return err; +} + +/* Close file descriptor allocated above (user can also use close(2)). */ +static int autofs_dev_ioctl_closemount(struct file *fp, + struct autofs_sb_info *sbi, + struct autofs_dev_ioctl *param) +{ + return close_fd(param->ioctlfd); +} + +/* + * Send "ready" status for an existing wait (either a mount or an expire + * request). + */ +static int autofs_dev_ioctl_ready(struct file *fp, + struct autofs_sb_info *sbi, + struct autofs_dev_ioctl *param) +{ + autofs_wqt_t token; + + token = (autofs_wqt_t) param->ready.token; + return autofs_wait_release(sbi, token, 0); +} + +/* + * Send "fail" status for an existing wait (either a mount or an expire + * request). + */ +static int autofs_dev_ioctl_fail(struct file *fp, + struct autofs_sb_info *sbi, + struct autofs_dev_ioctl *param) +{ + autofs_wqt_t token; + int status; + + token = (autofs_wqt_t) param->fail.token; + status = param->fail.status < 0 ? param->fail.status : -ENOENT; + return autofs_wait_release(sbi, token, status); +} + +/* + * Set the pipe fd for kernel communication to the daemon. + * + * Normally this is set at mount using an option but if we + * are reconnecting to a busy mount then we need to use this + * to tell the autofs mount about the new kernel pipe fd. In + * order to protect mounts against incorrectly setting the + * pipefd we also require that the autofs mount be catatonic. + * + * This also sets the process group id used to identify the + * controlling process (eg. the owning automount(8) daemon). + */ +static int autofs_dev_ioctl_setpipefd(struct file *fp, + struct autofs_sb_info *sbi, + struct autofs_dev_ioctl *param) +{ + int pipefd; + int err = 0; + struct pid *new_pid = NULL; + + if (param->setpipefd.pipefd == -1) + return -EINVAL; + + pipefd = param->setpipefd.pipefd; + + mutex_lock(&sbi->wq_mutex); + if (!(sbi->flags & AUTOFS_SBI_CATATONIC)) { + mutex_unlock(&sbi->wq_mutex); + return -EBUSY; + } else { + struct file *pipe; + + new_pid = get_task_pid(current, PIDTYPE_PGID); + + if (ns_of_pid(new_pid) != ns_of_pid(sbi->oz_pgrp)) { + pr_warn("not allowed to change PID namespace\n"); + err = -EINVAL; + goto out; + } + + pipe = fget(pipefd); + if (!pipe) { + err = -EBADF; + goto out; + } + if (autofs_prepare_pipe(pipe) < 0) { + err = -EPIPE; + fput(pipe); + goto out; + } + swap(sbi->oz_pgrp, new_pid); + sbi->pipefd = pipefd; + sbi->pipe = pipe; + sbi->mnt_ns_id = to_ns_common(current->nsproxy->mnt_ns)->ns_id; + sbi->flags &= ~AUTOFS_SBI_CATATONIC; + } +out: + put_pid(new_pid); + mutex_unlock(&sbi->wq_mutex); + return err; +} + +/* + * Make the autofs mount point catatonic, no longer responsive to + * mount requests. Also closes the kernel pipe file descriptor. + */ +static int autofs_dev_ioctl_catatonic(struct file *fp, + struct autofs_sb_info *sbi, + struct autofs_dev_ioctl *param) +{ + autofs_catatonic_mode(sbi); + return 0; +} + +/* + * Set the autofs mount expire timeout. + * + * There are two places an expire timeout can be set, in the autofs + * super block info. (this is all that's needed for direct and offset + * mounts because there's a distinct mount corresponding to each of + * these) and per-dentry within within the dentry info. If a per-dentry + * timeout is set it will override the expire timeout set in the parent + * autofs super block info. + * + * If setting the autofs super block expire timeout the autofs_dev_ioctl + * size field will be equal to the autofs_dev_ioctl structure size. If + * setting the per-dentry expire timeout the mount point name is passed + * in the autofs_dev_ioctl path field and the size field updated to + * reflect this. + * + * Setting the autofs mount expire timeout sets the timeout in the super + * block info. struct. Setting the per-dentry timeout does a little more. + * If the timeout is equal to -1 the per-dentry timeout (and flag) is + * cleared which reverts to using the super block timeout, otherwise if + * timeout is 0 the timeout is set to this value and the flag is left + * set which disables expiration for the mount point, lastly the flag + * and the timeout are set enabling the dentry to use this timeout. + */ +static int autofs_dev_ioctl_timeout(struct file *fp, + struct autofs_sb_info *sbi, + struct autofs_dev_ioctl *param) +{ + unsigned long timeout = param->timeout.timeout; + + /* If setting the expire timeout for an individual indirect + * mount point dentry the mount trailing component path is + * placed in param->path and param->size adjusted to account + * for it otherwise param->size it is set to the structure + * size. + */ + if (param->size == AUTOFS_DEV_IOCTL_SIZE) { + param->timeout.timeout = sbi->exp_timeout / HZ; + sbi->exp_timeout = timeout * HZ; + } else { + struct dentry *base = fp->f_path.dentry; + int path_len = param->size - AUTOFS_DEV_IOCTL_SIZE - 1; + struct dentry *dentry; + struct autofs_info *ino; + + if (!autofs_type_indirect(sbi->type)) + return -EINVAL; + + dentry = try_lookup_noperm(&QSTR_LEN(param->path, path_len), + base); + if (IS_ERR_OR_NULL(dentry)) + return dentry ? PTR_ERR(dentry) : -ENOENT; + ino = autofs_dentry_ino(dentry); + if (!ino) { + dput(dentry); + return -ENOENT; + } + + if (ino->exp_timeout && ino->flags & AUTOFS_INF_EXPIRE_SET) + param->timeout.timeout = ino->exp_timeout / HZ; + else + param->timeout.timeout = sbi->exp_timeout / HZ; + + if (timeout == -1) { + /* Revert to using the super block timeout */ + ino->flags &= ~AUTOFS_INF_EXPIRE_SET; + ino->exp_timeout = 0; + } else { + /* Set the dentry expire flag and timeout. + * + * If timeout is 0 it will prevent the expire + * of this particular automount. + */ + ino->flags |= AUTOFS_INF_EXPIRE_SET; + ino->exp_timeout = timeout * HZ; + } + + /* An expire timeout greater than the superblock timeout + * could be a problem at shutdown but the super block + * timeout itself can change so all we can really do is + * warn the user. + */ + if (ino->flags & AUTOFS_INF_EXPIRE_SET && + ino->exp_timeout > sbi->exp_timeout) + pr_warn("per-mount expire timeout is greater than " + "the parent autofs mount timeout which could " + "prevent shutdown\n"); + + dput(dentry); + } + + return 0; +} + +/* + * Return the uid and gid of the last request for the mount + * + * When reconstructing an autofs mount tree with active mounts + * we need to re-connect to mounts that may have used the original + * process uid and gid (or string variations of them) for mount + * lookups within the map entry. + */ +static int autofs_dev_ioctl_requester(struct file *fp, + struct autofs_sb_info *sbi, + struct autofs_dev_ioctl *param) +{ + struct autofs_info *ino; + struct path path; + dev_t devid; + int err = -ENOENT; + + /* param->path has been checked in validate_dev_ioctl() */ + + devid = sbi->sb->s_dev; + + param->requester.uid = param->requester.gid = -1; + + err = find_autofs_mount(param->path, &path, test_by_dev, &devid); + if (err) + goto out; + + ino = autofs_dentry_ino(path.dentry); + if (ino) { + err = 0; + autofs_expire_wait(&path, 0); + spin_lock(&sbi->fs_lock); + param->requester.uid = + from_kuid_munged(current_user_ns(), ino->uid); + param->requester.gid = + from_kgid_munged(current_user_ns(), ino->gid); + spin_unlock(&sbi->fs_lock); + } + path_put(&path); +out: + return err; +} + +/* + * Call repeatedly until it returns -EAGAIN, meaning there's nothing + * more that can be done. + */ +static int autofs_dev_ioctl_expire(struct file *fp, + struct autofs_sb_info *sbi, + struct autofs_dev_ioctl *param) +{ + struct vfsmount *mnt; + int how; + + how = param->expire.how; + mnt = fp->f_path.mnt; + + return autofs_do_expire_multi(sbi->sb, mnt, sbi, how); +} + +/* Check if autofs mount point is in use */ +static int autofs_dev_ioctl_askumount(struct file *fp, + struct autofs_sb_info *sbi, + struct autofs_dev_ioctl *param) +{ + param->askumount.may_umount = 0; + if (may_umount(fp->f_path.mnt)) + param->askumount.may_umount = 1; + return 0; +} + +/* + * Check if the given path is a mountpoint. + * + * If we are supplied with the file descriptor of an autofs + * mount we're looking for a specific mount. In this case + * the path is considered a mountpoint if it is itself a + * mountpoint or contains a mount, such as a multi-mount + * without a root mount. In this case we return 1 if the + * path is a mount point and the super magic of the covering + * mount if there is one or 0 if it isn't a mountpoint. + * + * If we aren't supplied with a file descriptor then we + * lookup the path and check if it is the root of a mount. + * If a type is given we are looking for a particular autofs + * mount and if we don't find a match we return fail. If the + * located path is the root of a mount we return 1 along with + * the super magic of the mount or 0 otherwise. + * + * In both cases the device number (as returned by + * new_encode_dev()) is also returned. + */ +static int autofs_dev_ioctl_ismountpoint(struct file *fp, + struct autofs_sb_info *sbi, + struct autofs_dev_ioctl *param) +{ + struct path path; + const char *name; + unsigned int type; + unsigned int devid, magic; + int err = -ENOENT; + + /* param->path has been checked in validate_dev_ioctl() */ + + name = param->path; + type = param->ismountpoint.in.type; + + param->ismountpoint.out.devid = devid = 0; + param->ismountpoint.out.magic = magic = 0; + + if (!fp || param->ioctlfd == -1) { + if (autofs_type_any(type)) + err = kern_path(name, LOOKUP_FOLLOW | LOOKUP_MOUNTPOINT, + &path); + else + err = find_autofs_mount(name, &path, + test_by_type, &type); + if (err) + goto out; + devid = new_encode_dev(path.dentry->d_sb->s_dev); + err = 0; + if (path.mnt->mnt_root == path.dentry) { + err = 1; + magic = path.dentry->d_sb->s_magic; + } + } else { + dev_t dev = sbi->sb->s_dev; + + err = find_autofs_mount(name, &path, test_by_dev, &dev); + if (err) + goto out; + + devid = new_encode_dev(dev); + + err = path_has_submounts(&path); + + if (follow_down_one(&path)) + magic = path.dentry->d_sb->s_magic; + } + + param->ismountpoint.out.devid = devid; + param->ismountpoint.out.magic = magic; + path_put(&path); +out: + return err; +} + +/* + * Our range of ioctl numbers isn't 0 based so we need to shift + * the array index by _IOC_NR(AUTOFS_CTL_IOC_FIRST) for the table + * lookup. + */ +#define cmd_idx(cmd) (cmd - _IOC_NR(AUTOFS_DEV_IOCTL_IOC_FIRST)) + +static ioctl_fn lookup_dev_ioctl(unsigned int cmd) +{ + static const ioctl_fn _ioctls[] = { + autofs_dev_ioctl_version, + autofs_dev_ioctl_protover, + autofs_dev_ioctl_protosubver, + autofs_dev_ioctl_openmount, + autofs_dev_ioctl_closemount, + autofs_dev_ioctl_ready, + autofs_dev_ioctl_fail, + autofs_dev_ioctl_setpipefd, + autofs_dev_ioctl_catatonic, + autofs_dev_ioctl_timeout, + autofs_dev_ioctl_requester, + autofs_dev_ioctl_expire, + autofs_dev_ioctl_askumount, + autofs_dev_ioctl_ismountpoint, + }; + unsigned int idx = cmd_idx(cmd); + + if (idx >= ARRAY_SIZE(_ioctls)) + return NULL; + idx = array_index_nospec(idx, ARRAY_SIZE(_ioctls)); + return _ioctls[idx]; +} + +/* ioctl dispatcher */ +static int _autofs_dev_ioctl(unsigned int command, + struct autofs_dev_ioctl __user *user) +{ + struct autofs_dev_ioctl *param; + struct file *fp; + struct autofs_sb_info *sbi; + unsigned int cmd_first, cmd; + ioctl_fn fn = NULL; + int err = 0; + + cmd_first = _IOC_NR(AUTOFS_DEV_IOCTL_IOC_FIRST); + cmd = _IOC_NR(command); + + if (_IOC_TYPE(command) != _IOC_TYPE(AUTOFS_DEV_IOCTL_IOC_FIRST) || + cmd - cmd_first > AUTOFS_DEV_IOCTL_IOC_COUNT) { + return -ENOTTY; + } + + /* Only root can use ioctls other than AUTOFS_DEV_IOCTL_VERSION_CMD + * and AUTOFS_DEV_IOCTL_ISMOUNTPOINT_CMD + */ + if (cmd != AUTOFS_DEV_IOCTL_VERSION_CMD && + cmd != AUTOFS_DEV_IOCTL_ISMOUNTPOINT_CMD && + !capable(CAP_SYS_ADMIN)) + return -EPERM; + + /* Copy the parameters into kernel space. */ + param = copy_dev_ioctl(user); + if (IS_ERR(param)) + return PTR_ERR(param); + + err = validate_dev_ioctl(command, param); + if (err) + goto out; + + fn = lookup_dev_ioctl(cmd); + if (!fn) { + pr_warn("unknown command 0x%08x\n", command); + err = -ENOTTY; + goto out; + } + + fp = NULL; + sbi = NULL; + + /* + * For obvious reasons the openmount can't have a file + * descriptor yet. We don't take a reference to the + * file during close to allow for immediate release, + * and the same for retrieving ioctl version. + */ + if (cmd != AUTOFS_DEV_IOCTL_VERSION_CMD && + cmd != AUTOFS_DEV_IOCTL_OPENMOUNT_CMD && + cmd != AUTOFS_DEV_IOCTL_CLOSEMOUNT_CMD) { + struct super_block *sb; + + fp = fget(param->ioctlfd); + if (!fp) { + if (cmd == AUTOFS_DEV_IOCTL_ISMOUNTPOINT_CMD) + goto cont; + err = -EBADF; + goto out; + } + + sb = file_inode(fp)->i_sb; + if (sb->s_type != &autofs_fs_type) { + err = -EINVAL; + fput(fp); + goto out; + } + sbi = autofs_sbi(sb); + + /* + * Admin needs to be able to set the mount catatonic in + * order to be able to perform the re-open. + */ + if (!autofs_oz_mode(sbi) && + cmd != AUTOFS_DEV_IOCTL_CATATONIC_CMD) { + err = -EACCES; + fput(fp); + goto out; + } + } +cont: + err = fn(fp, sbi, param); + + if (fp) + fput(fp); + if (err >= 0 && copy_to_user(user, param, AUTOFS_DEV_IOCTL_SIZE)) + err = -EFAULT; +out: + free_dev_ioctl(param); + return err; +} + +static long autofs_dev_ioctl(struct file *file, unsigned int command, + unsigned long u) +{ + int err; + + err = _autofs_dev_ioctl(command, (struct autofs_dev_ioctl __user *) u); + return (long) err; +} + +#ifdef CONFIG_COMPAT +static long autofs_dev_ioctl_compat(struct file *file, unsigned int command, + unsigned long u) +{ + return autofs_dev_ioctl(file, command, (unsigned long) compat_ptr(u)); +} +#else +#define autofs_dev_ioctl_compat NULL +#endif + +static const struct file_operations _dev_ioctl_fops = { + .unlocked_ioctl = autofs_dev_ioctl, + .compat_ioctl = autofs_dev_ioctl_compat, + .owner = THIS_MODULE, + .llseek = noop_llseek, +}; + +static struct miscdevice _autofs_dev_ioctl_misc = { + .minor = AUTOFS_MINOR, + .name = AUTOFS_DEVICE_NAME, + .fops = &_dev_ioctl_fops, + .mode = 0644, +}; + +MODULE_ALIAS_MISCDEV(AUTOFS_MINOR); +MODULE_ALIAS("devname:autofs"); + +/* Register/deregister misc character device */ +int __init autofs_dev_ioctl_init(void) +{ + int r; + + r = misc_register(&_autofs_dev_ioctl_misc); + if (r) { + pr_err("misc_register failed for control device\n"); + return r; + } + + return 0; +} + +void autofs_dev_ioctl_exit(void) +{ + misc_deregister(&_autofs_dev_ioctl_misc); +} diff --git a/fs/autofs/expire.c b/fs/autofs/expire.c new file mode 100644 index 000000000..5c2d459e1 --- /dev/null +++ b/fs/autofs/expire.c @@ -0,0 +1,620 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 1997-1998 Transmeta Corporation -- All Rights Reserved + * Copyright 1999-2000 Jeremy Fitzhardinge + * Copyright 2001-2006 Ian Kent + */ + +#include "autofs_i.h" + +/* Check if a dentry can be expired */ +static inline int autofs_can_expire(struct dentry *dentry, + unsigned long timeout, unsigned int how) +{ + struct autofs_info *ino = autofs_dentry_ino(dentry); + + /* dentry in the process of being deleted */ + if (ino == NULL) + return 0; + + if (!(how & AUTOFS_EXP_IMMEDIATE)) { + /* Too young to die */ + if (!timeout || time_after(ino->last_used + timeout, jiffies)) + return 0; + } + return 1; +} + +/* Check a mount point for busyness */ +static int autofs_mount_busy(struct vfsmount *mnt, + struct dentry *dentry, unsigned int how) +{ + struct dentry *top = dentry; + struct path path = {.mnt = mnt, .dentry = dentry}; + int status = 1; + + pr_debug("dentry %p %pd\n", dentry, dentry); + + path_get(&path); + + if (!follow_down_one(&path)) + goto done; + + if (is_autofs_dentry(path.dentry)) { + struct autofs_sb_info *sbi = autofs_sbi(path.dentry->d_sb); + + /* This is an autofs submount, we can't expire it */ + if (autofs_type_indirect(sbi->type)) + goto done; + } + + /* Not a submount, has a forced expire been requested */ + if (how & AUTOFS_EXP_FORCED) { + status = 0; + goto done; + } + + /* Update the expiry counter if fs is busy */ + if (!may_umount_tree(path.mnt)) { + struct autofs_info *ino; + + ino = autofs_dentry_ino(top); + ino->last_used = jiffies; + goto done; + } + + status = 0; +done: + pr_debug("returning = %d\n", status); + path_put(&path); + return status; +} + +/* p->d_lock held */ +static struct dentry *positive_after(struct dentry *p, struct dentry *child) +{ + child = child ? d_next_sibling(child) : d_first_child(p); + + hlist_for_each_entry_from(child, d_sib) { + spin_lock_nested(&child->d_lock, DENTRY_D_LOCK_NESTED); + if (simple_positive(child)) { + dget_dlock(child); + spin_unlock(&child->d_lock); + return child; + } + spin_unlock(&child->d_lock); + } + + return NULL; +} + +/* + * Calculate and dget next entry in the subdirs list under root. + */ +static struct dentry *get_next_positive_subdir(struct dentry *prev, + struct dentry *root) +{ + struct autofs_sb_info *sbi = autofs_sbi(root->d_sb); + struct dentry *q; + + spin_lock(&sbi->lookup_lock); + spin_lock(&root->d_lock); + q = positive_after(root, prev); + spin_unlock(&root->d_lock); + spin_unlock(&sbi->lookup_lock); + dput(prev); + return q; +} + +/* + * Calculate and dget next entry in top down tree traversal. + */ +static struct dentry *get_next_positive_dentry(struct dentry *prev, + struct dentry *root) +{ + struct autofs_sb_info *sbi = autofs_sbi(root->d_sb); + struct dentry *p = prev, *ret = NULL, *d = NULL; + + if (prev == NULL) + return dget(root); + + spin_lock(&sbi->lookup_lock); + spin_lock(&p->d_lock); + while (1) { + struct dentry *parent; + + ret = positive_after(p, d); + if (ret || p == root) + break; + parent = p->d_parent; + spin_unlock(&p->d_lock); + spin_lock(&parent->d_lock); + d = p; + p = parent; + } + spin_unlock(&p->d_lock); + spin_unlock(&sbi->lookup_lock); + dput(prev); + return ret; +} + +/* + * Check a direct mount point for busyness. + * Direct mounts have similar expiry semantics to tree mounts. + * The tree is not busy iff no mountpoints are busy and there are no + * autofs submounts. + */ +static int autofs_direct_busy(struct vfsmount *mnt, + struct dentry *top, + unsigned long timeout, + unsigned int how) +{ + pr_debug("top %p %pd\n", top, top); + + /* Forced expire, user space handles busy mounts */ + if (how & AUTOFS_EXP_FORCED) + return 0; + + /* If it's busy update the expiry counters */ + if (!may_umount_tree(mnt)) { + struct autofs_info *ino; + + ino = autofs_dentry_ino(top); + if (ino) + ino->last_used = jiffies; + return 1; + } + + /* Timeout of a direct mount is determined by its top dentry */ + if (!autofs_can_expire(top, timeout, how)) + return 1; + + return 0; +} + +/* + * Check a directory tree of mount points for busyness + * The tree is not busy iff no mountpoints are busy + */ +static int autofs_tree_busy(struct vfsmount *mnt, + struct dentry *top, + unsigned long timeout, + unsigned int how) +{ + struct autofs_info *top_ino = autofs_dentry_ino(top); + struct dentry *p; + + pr_debug("top %p %pd\n", top, top); + + /* Negative dentry - give up */ + if (!simple_positive(top)) + return 1; + + p = NULL; + while ((p = get_next_positive_dentry(p, top))) { + pr_debug("dentry %p %pd\n", p, p); + + /* + * Is someone visiting anywhere in the subtree ? + * If there's no mount we need to check the usage + * count for the autofs dentry. + * If the fs is busy update the expiry counter. + */ + if (d_mountpoint(p)) { + if (autofs_mount_busy(mnt, p, how)) { + top_ino->last_used = jiffies; + dput(p); + return 1; + } + } else { + struct autofs_info *ino = autofs_dentry_ino(p); + unsigned int ino_count = READ_ONCE(ino->count); + + /* allow for dget above and top is already dgot */ + if (p == top) + ino_count += 2; + else + ino_count++; + + if (d_count(p) > ino_count) { + top_ino->last_used = jiffies; + dput(p); + return 1; + } + } + } + + /* Forced expire, user space handles busy mounts */ + if (how & AUTOFS_EXP_FORCED) + return 0; + + /* Timeout of a tree mount is ultimately determined by its top dentry */ + if (!autofs_can_expire(top, timeout, how)) + return 1; + + return 0; +} + +static struct dentry *autofs_check_leaves(struct vfsmount *mnt, + struct dentry *parent, + unsigned long timeout, + unsigned int how) +{ + struct dentry *p; + + pr_debug("parent %p %pd\n", parent, parent); + + p = NULL; + while ((p = get_next_positive_dentry(p, parent))) { + pr_debug("dentry %p %pd\n", p, p); + + if (d_mountpoint(p)) { + /* Can we umount this guy */ + if (autofs_mount_busy(mnt, p, how)) + continue; + + /* This isn't a submount so if a forced expire + * has been requested, user space handles busy + * mounts */ + if (how & AUTOFS_EXP_FORCED) + return p; + + /* Can we expire this guy */ + if (autofs_can_expire(p, timeout, how)) + return p; + } + } + return NULL; +} + +/* Check if we can expire a direct mount (possibly a tree) */ +static struct dentry *autofs_expire_direct(struct super_block *sb, + struct vfsmount *mnt, + struct autofs_sb_info *sbi, + unsigned int how) +{ + struct dentry *root = dget(sb->s_root); + struct autofs_info *ino; + unsigned long timeout; + + if (!root) + return NULL; + + timeout = sbi->exp_timeout; + + if (!autofs_direct_busy(mnt, root, timeout, how)) { + spin_lock(&sbi->fs_lock); + ino = autofs_dentry_ino(root); + /* No point expiring a pending mount */ + if (ino->flags & AUTOFS_INF_PENDING) { + spin_unlock(&sbi->fs_lock); + goto out; + } + ino->flags |= AUTOFS_INF_WANT_EXPIRE; + spin_unlock(&sbi->fs_lock); + synchronize_rcu(); + if (!autofs_direct_busy(mnt, root, timeout, how)) { + spin_lock(&sbi->fs_lock); + ino->flags |= AUTOFS_INF_EXPIRING; + init_completion(&ino->expire_complete); + spin_unlock(&sbi->fs_lock); + return root; + } + spin_lock(&sbi->fs_lock); + ino->flags &= ~AUTOFS_INF_WANT_EXPIRE; + spin_unlock(&sbi->fs_lock); + } +out: + dput(root); + + return NULL; +} + +/* Check if 'dentry' should expire, or return a nearby + * dentry that is suitable. + * If returned dentry is different from arg dentry, + * then a dget() reference was taken, else not. + */ +static struct dentry *should_expire(struct dentry *dentry, + struct vfsmount *mnt, + unsigned long timeout, + unsigned int how) +{ + struct autofs_info *ino = autofs_dentry_ino(dentry); + unsigned int ino_count; + + /* No point expiring a pending mount */ + if (ino->flags & AUTOFS_INF_PENDING) + return NULL; + + /* + * Case 1: (i) indirect mount or top level pseudo direct mount + * (autofs-4.1). + * (ii) indirect mount with offset mount, check the "/" + * offset (autofs-5.0+). + */ + if (d_mountpoint(dentry)) { + pr_debug("checking mountpoint %p %pd\n", dentry, dentry); + + /* Can we umount this guy */ + if (autofs_mount_busy(mnt, dentry, how)) + return NULL; + + /* This isn't a submount so if a forced expire + * has been requested, user space handles busy + * mounts */ + if (how & AUTOFS_EXP_FORCED) + return dentry; + + /* Can we expire this guy */ + if (autofs_can_expire(dentry, timeout, how)) + return dentry; + return NULL; + } + + if (d_is_symlink(dentry)) { + pr_debug("checking symlink %p %pd\n", dentry, dentry); + + /* Forced expire, user space handles busy mounts */ + if (how & AUTOFS_EXP_FORCED) + return dentry; + + /* + * A symlink can't be "busy" in the usual sense so + * just check last used for expire timeout. + */ + if (autofs_can_expire(dentry, timeout, how)) + return dentry; + return NULL; + } + + if (autofs_empty(ino)) + return NULL; + + /* Case 2: tree mount, expire iff entire tree is not busy */ + if (!(how & AUTOFS_EXP_LEAVES)) { + /* Not a forced expire? */ + if (!(how & AUTOFS_EXP_FORCED)) { + /* ref-walk currently on this dentry? */ + ino_count = READ_ONCE(ino->count) + 1; + if (d_count(dentry) > ino_count) + return NULL; + } + + if (!autofs_tree_busy(mnt, dentry, timeout, how)) + return dentry; + /* + * Case 3: pseudo direct mount, expire individual leaves + * (autofs-4.1). + */ + } else { + struct dentry *expired; + + /* Not a forced expire? */ + if (!(how & AUTOFS_EXP_FORCED)) { + /* ref-walk currently on this dentry? */ + ino_count = READ_ONCE(ino->count) + 1; + if (d_count(dentry) > ino_count) + return NULL; + } + + expired = autofs_check_leaves(mnt, dentry, timeout, how); + if (expired) { + if (expired == dentry) + dput(dentry); + return expired; + } + } + return NULL; +} + +/* + * Find an eligible tree to time-out + * A tree is eligible if :- + * - it is unused by any user process + * - it has been unused for exp_timeout time + */ +static struct dentry *autofs_expire_indirect(struct super_block *sb, + struct vfsmount *mnt, + struct autofs_sb_info *sbi, + unsigned int how) +{ + unsigned long timeout; + struct dentry *root = sb->s_root; + struct dentry *dentry; + struct dentry *expired; + struct dentry *found; + struct autofs_info *ino; + + if (!root) + return NULL; + + dentry = NULL; + while ((dentry = get_next_positive_subdir(dentry, root))) { + spin_lock(&sbi->fs_lock); + ino = autofs_dentry_ino(dentry); + if (ino->flags & AUTOFS_INF_WANT_EXPIRE) { + spin_unlock(&sbi->fs_lock); + continue; + } + spin_unlock(&sbi->fs_lock); + + if (ino->flags & AUTOFS_INF_EXPIRE_SET) + timeout = ino->exp_timeout; + else + timeout = sbi->exp_timeout; + + expired = should_expire(dentry, mnt, timeout, how); + if (!expired) + continue; + + spin_lock(&sbi->fs_lock); + ino = autofs_dentry_ino(expired); + ino->flags |= AUTOFS_INF_WANT_EXPIRE; + spin_unlock(&sbi->fs_lock); + synchronize_rcu(); + + /* Make sure a reference is not taken on found if + * things have changed. + */ + how &= ~AUTOFS_EXP_LEAVES; + found = should_expire(expired, mnt, timeout, how); + if (found != expired) { // something has changed, continue + dput(found); + goto next; + } + + if (expired != dentry) + dput(dentry); + + spin_lock(&sbi->fs_lock); + goto found; +next: + spin_lock(&sbi->fs_lock); + ino->flags &= ~AUTOFS_INF_WANT_EXPIRE; + spin_unlock(&sbi->fs_lock); + if (expired != dentry) + dput(expired); + } + return NULL; + +found: + pr_debug("returning %p %pd\n", expired, expired); + ino->flags |= AUTOFS_INF_EXPIRING; + init_completion(&ino->expire_complete); + spin_unlock(&sbi->fs_lock); + return expired; +} + +int autofs_expire_wait(const struct path *path, int rcu_walk) +{ + struct dentry *dentry = path->dentry; + struct autofs_sb_info *sbi = autofs_sbi(dentry->d_sb); + struct autofs_info *ino = autofs_dentry_ino(dentry); + int status; + int state; + + /* Block on any pending expire */ + if (!(ino->flags & AUTOFS_INF_WANT_EXPIRE)) + return 0; + if (rcu_walk) + return -ECHILD; + +retry: + spin_lock(&sbi->fs_lock); + state = ino->flags & (AUTOFS_INF_WANT_EXPIRE | AUTOFS_INF_EXPIRING); + if (state == AUTOFS_INF_WANT_EXPIRE) { + spin_unlock(&sbi->fs_lock); + /* + * Possibly being selected for expire, wait until + * it's selected or not. + */ + schedule_timeout_uninterruptible(HZ/10); + goto retry; + } + if (state & AUTOFS_INF_EXPIRING) { + spin_unlock(&sbi->fs_lock); + + pr_debug("waiting for expire %p name=%pd\n", dentry, dentry); + + status = autofs_wait(sbi, path, NFY_NONE); + wait_for_completion(&ino->expire_complete); + + pr_debug("expire done status=%d\n", status); + + if (d_unhashed(dentry)) + return -EAGAIN; + + return status; + } + spin_unlock(&sbi->fs_lock); + + return 0; +} + +/* Perform an expiry operation */ +int autofs_expire_run(struct super_block *sb, + struct vfsmount *mnt, + struct autofs_sb_info *sbi, + struct autofs_packet_expire __user *pkt_p) +{ + struct autofs_packet_expire pkt; + struct autofs_info *ino; + struct dentry *dentry; + int ret = 0; + + memset(&pkt, 0, sizeof(pkt)); + + pkt.hdr.proto_version = sbi->version; + pkt.hdr.type = autofs_ptype_expire; + + dentry = autofs_expire_indirect(sb, mnt, sbi, 0); + if (!dentry) + return -EAGAIN; + + pkt.len = dentry->d_name.len; + memcpy(pkt.name, dentry->d_name.name, pkt.len); + pkt.name[pkt.len] = '\0'; + + if (copy_to_user(pkt_p, &pkt, sizeof(struct autofs_packet_expire))) + ret = -EFAULT; + + spin_lock(&sbi->fs_lock); + ino = autofs_dentry_ino(dentry); + /* avoid rapid-fire expire attempts if expiry fails */ + ino->last_used = jiffies; + ino->flags &= ~(AUTOFS_INF_EXPIRING|AUTOFS_INF_WANT_EXPIRE); + complete_all(&ino->expire_complete); + spin_unlock(&sbi->fs_lock); + + dput(dentry); + + return ret; +} + +int autofs_do_expire_multi(struct super_block *sb, struct vfsmount *mnt, + struct autofs_sb_info *sbi, unsigned int how) +{ + struct dentry *dentry; + int ret = -EAGAIN; + + if (autofs_type_trigger(sbi->type)) + dentry = autofs_expire_direct(sb, mnt, sbi, how); + else + dentry = autofs_expire_indirect(sb, mnt, sbi, how); + + if (dentry) { + struct autofs_info *ino = autofs_dentry_ino(dentry); + const struct path path = { .mnt = mnt, .dentry = dentry }; + + /* This is synchronous because it makes the daemon a + * little easier + */ + ret = autofs_wait(sbi, &path, NFY_EXPIRE); + + spin_lock(&sbi->fs_lock); + /* avoid rapid-fire expire attempts if expiry fails */ + ino->last_used = jiffies; + ino->flags &= ~(AUTOFS_INF_EXPIRING|AUTOFS_INF_WANT_EXPIRE); + complete_all(&ino->expire_complete); + spin_unlock(&sbi->fs_lock); + dput(dentry); + } + + return ret; +} + +/* + * Call repeatedly until it returns -EAGAIN, meaning there's nothing + * more to be done. + */ +int autofs_expire_multi(struct super_block *sb, struct vfsmount *mnt, + struct autofs_sb_info *sbi, int __user *arg) +{ + unsigned int how = 0; + + if (arg && get_user(how, arg)) + return -EFAULT; + + return autofs_do_expire_multi(sb, mnt, sbi, how); +} diff --git a/fs/autofs/init.c b/fs/autofs/init.c new file mode 100644 index 000000000..1d644a35f --- /dev/null +++ b/fs/autofs/init.c @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 1997-1998 Transmeta Corporation -- All Rights Reserved + */ + +#include +#include +#include "autofs_i.h" + +struct file_system_type autofs_fs_type = { + .owner = THIS_MODULE, + .name = "autofs", + .init_fs_context = autofs_init_fs_context, + .parameters = autofs_param_specs, + .kill_sb = autofs_kill_sb, +}; +MODULE_ALIAS_FS("autofs"); +MODULE_ALIAS("autofs"); + +static int __init init_autofs_fs(void) +{ + int err; + + autofs_dev_ioctl_init(); + + err = register_filesystem(&autofs_fs_type); + if (err) + autofs_dev_ioctl_exit(); + + return err; +} + +static void __exit exit_autofs_fs(void) +{ + autofs_dev_ioctl_exit(); + unregister_filesystem(&autofs_fs_type); +} + +module_init(init_autofs_fs) +module_exit(exit_autofs_fs) +MODULE_DESCRIPTION("Kernel automounter support"); +MODULE_LICENSE("GPL"); diff --git a/fs/autofs/inode.c b/fs/autofs/inode.c new file mode 100644 index 000000000..c1e210cec --- /dev/null +++ b/fs/autofs/inode.c @@ -0,0 +1,455 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 1997-1998 Transmeta Corporation -- All Rights Reserved + * Copyright 2005-2006 Ian Kent + */ + +#include +#include + +#include "autofs_i.h" + +struct autofs_info *autofs_new_ino(struct autofs_sb_info *sbi) +{ + struct autofs_info *ino; + + ino = kzalloc_obj(*ino); + if (ino) { + INIT_LIST_HEAD(&ino->active); + INIT_LIST_HEAD(&ino->expiring); + ino->last_used = jiffies; + ino->sbi = sbi; + ino->exp_timeout = -1; + ino->count = 1; + } + return ino; +} + +void autofs_clean_ino(struct autofs_info *ino) +{ + ino->uid = GLOBAL_ROOT_UID; + ino->gid = GLOBAL_ROOT_GID; + ino->exp_timeout = -1; + ino->last_used = jiffies; +} + +void autofs_free_ino(struct autofs_info *ino) +{ + kfree_rcu(ino, rcu); +} + +void autofs_kill_sb(struct super_block *sb) +{ + struct autofs_sb_info *sbi = autofs_sbi(sb); + + /* + * In the event of a failure in get_sb_nodev the superblock + * info is not present so nothing else has been setup, so + * just call kill_anon_super when we are called from + * deactivate_super. + */ + if (sbi) { + /* Free wait queues, close pipe */ + autofs_catatonic_mode(sbi); + put_pid(sbi->oz_pgrp); + } + + pr_debug("shutting down\n"); + kill_anon_super(sb); + if (sbi) + kfree_rcu(sbi, rcu); +} + +static int autofs_show_options(struct seq_file *m, struct dentry *root) +{ + struct autofs_sb_info *sbi = autofs_sbi(root->d_sb); + struct inode *root_inode = d_inode(root->d_sb->s_root); + + if (!sbi) + return 0; + + seq_printf(m, ",fd=%d", sbi->pipefd); + if (!uid_eq(root_inode->i_uid, GLOBAL_ROOT_UID)) + seq_printf(m, ",uid=%u", + from_kuid_munged(&init_user_ns, root_inode->i_uid)); + if (!gid_eq(root_inode->i_gid, GLOBAL_ROOT_GID)) + seq_printf(m, ",gid=%u", + from_kgid_munged(&init_user_ns, root_inode->i_gid)); + seq_printf(m, ",pgrp=%d", pid_vnr(sbi->oz_pgrp)); + seq_printf(m, ",timeout=%lu", sbi->exp_timeout/HZ); + seq_printf(m, ",minproto=%d", sbi->min_proto); + seq_printf(m, ",maxproto=%d", sbi->max_proto); + + if (autofs_type_offset(sbi->type)) + seq_puts(m, ",offset"); + else if (autofs_type_direct(sbi->type)) + seq_puts(m, ",direct"); + else + seq_puts(m, ",indirect"); + if (sbi->flags & AUTOFS_SBI_STRICTEXPIRE) + seq_puts(m, ",strictexpire"); + if (sbi->flags & AUTOFS_SBI_IGNORE) + seq_puts(m, ",ignore"); +#ifdef CONFIG_CHECKPOINT_RESTORE + if (sbi->pipe) + seq_printf(m, ",pipe_ino=%llu", file_inode(sbi->pipe)->i_ino); + else + seq_puts(m, ",pipe_ino=-1"); +#endif + return 0; +} + +static void autofs_evict_inode(struct inode *inode) +{ + clear_inode(inode); + kfree(inode->i_private); +} + +static const struct super_operations autofs_sops = { + .statfs = simple_statfs, + .show_options = autofs_show_options, + .evict_inode = autofs_evict_inode, +}; + +enum { + Opt_direct, + Opt_fd, + Opt_gid, + Opt_ignore, + Opt_indirect, + Opt_maxproto, + Opt_minproto, + Opt_offset, + Opt_pgrp, + Opt_strictexpire, + Opt_uid, +}; + +const struct fs_parameter_spec autofs_param_specs[] = { + fsparam_flag ("direct", Opt_direct), + fsparam_fd ("fd", Opt_fd), + fsparam_gid ("gid", Opt_gid), + fsparam_flag ("ignore", Opt_ignore), + fsparam_flag ("indirect", Opt_indirect), + fsparam_u32 ("maxproto", Opt_maxproto), + fsparam_u32 ("minproto", Opt_minproto), + fsparam_flag ("offset", Opt_offset), + fsparam_u32 ("pgrp", Opt_pgrp), + fsparam_flag ("strictexpire", Opt_strictexpire), + fsparam_uid ("uid", Opt_uid), + {} +}; + +struct autofs_fs_context { + kuid_t uid; + kgid_t gid; + int pgrp; + bool pgrp_set; +}; + +/* + * Open the fd. We do it here rather than in get_tree so that it's done in the + * context of the system call that passed the data and not the one that + * triggered the superblock creation, lest the fd gets reassigned. + */ +static int autofs_parse_fd(struct fs_context *fc, struct autofs_sb_info *sbi, + struct fs_parameter *param, + struct fs_parse_result *result) +{ + struct file *pipe; + int ret; + + if (param->type == fs_value_is_file) { + /* came through the new api */ + pipe = param->file; + param->file = NULL; + } else { + pipe = fget(result->uint_32); + } + if (!pipe) { + errorf(fc, "could not open pipe file descriptor"); + return -EBADF; + } + + ret = autofs_check_pipe(pipe); + if (ret < 0) { + errorf(fc, "Invalid/unusable pipe"); + fput(pipe); + return -EBADF; + } + + autofs_set_packet_pipe_flags(pipe); + + if (sbi->pipe) + fput(sbi->pipe); + + sbi->pipefd = result->uint_32; + sbi->pipe = pipe; + + return 0; +} + +static int autofs_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct autofs_fs_context *ctx = fc->fs_private; + struct autofs_sb_info *sbi = fc->s_fs_info; + struct fs_parse_result result; + int opt; + + opt = fs_parse(fc, autofs_param_specs, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_fd: + return autofs_parse_fd(fc, sbi, param, &result); + case Opt_uid: + ctx->uid = result.uid; + break; + case Opt_gid: + ctx->gid = result.gid; + break; + case Opt_pgrp: + ctx->pgrp = result.uint_32; + ctx->pgrp_set = true; + break; + case Opt_minproto: + sbi->min_proto = result.uint_32; + break; + case Opt_maxproto: + sbi->max_proto = result.uint_32; + break; + case Opt_indirect: + set_autofs_type_indirect(&sbi->type); + break; + case Opt_direct: + set_autofs_type_direct(&sbi->type); + break; + case Opt_offset: + set_autofs_type_offset(&sbi->type); + break; + case Opt_strictexpire: + sbi->flags |= AUTOFS_SBI_STRICTEXPIRE; + break; + case Opt_ignore: + sbi->flags |= AUTOFS_SBI_IGNORE; + } + + return 0; +} + +static struct autofs_sb_info *autofs_alloc_sbi(void) +{ + struct autofs_sb_info *sbi; + + sbi = kzalloc_obj(*sbi); + if (!sbi) + return NULL; + + sbi->magic = AUTOFS_SBI_MAGIC; + sbi->flags = AUTOFS_SBI_CATATONIC; + sbi->min_proto = AUTOFS_MIN_PROTO_VERSION; + sbi->max_proto = AUTOFS_MAX_PROTO_VERSION; + sbi->pipefd = -1; + sbi->mnt_ns_id = to_ns_common(current->nsproxy->mnt_ns)->ns_id; + + set_autofs_type_indirect(&sbi->type); + mutex_init(&sbi->wq_mutex); + mutex_init(&sbi->pipe_mutex); + spin_lock_init(&sbi->fs_lock); + spin_lock_init(&sbi->lookup_lock); + INIT_LIST_HEAD(&sbi->active_list); + INIT_LIST_HEAD(&sbi->expiring_list); + + return sbi; +} + +static int autofs_validate_protocol(struct fs_context *fc) +{ + struct autofs_sb_info *sbi = fc->s_fs_info; + + /* Test versions first */ + if (sbi->max_proto < AUTOFS_MIN_PROTO_VERSION || + sbi->min_proto > AUTOFS_MAX_PROTO_VERSION) { + errorf(fc, "kernel does not match daemon version " + "daemon (%d, %d) kernel (%d, %d)\n", + sbi->min_proto, sbi->max_proto, + AUTOFS_MIN_PROTO_VERSION, AUTOFS_MAX_PROTO_VERSION); + return -EINVAL; + } + + /* Establish highest kernel protocol version */ + if (sbi->max_proto > AUTOFS_MAX_PROTO_VERSION) + sbi->version = AUTOFS_MAX_PROTO_VERSION; + else + sbi->version = sbi->max_proto; + + switch (sbi->version) { + case 4: + sbi->sub_version = 7; + break; + case 5: + sbi->sub_version = AUTOFS_PROTO_SUBVERSION; + break; + default: + sbi->sub_version = 0; + } + + return 0; +} + +static int autofs_fill_super(struct super_block *s, struct fs_context *fc) +{ + struct autofs_fs_context *ctx = fc->fs_private; + struct autofs_sb_info *sbi = s->s_fs_info; + struct inode *root_inode; + struct autofs_info *ino; + + pr_debug("starting up, sbi = %p\n", sbi); + + sbi->sb = s; + s->s_blocksize = 1024; + s->s_blocksize_bits = 10; + s->s_magic = AUTOFS_SUPER_MAGIC; + s->s_op = &autofs_sops; + set_default_d_op(s, &autofs_dentry_operations); + s->s_time_gran = 1; + + /* + * Get the root inode and dentry, but defer checking for errors. + */ + ino = autofs_new_ino(sbi); + if (!ino) + return -ENOMEM; + + root_inode = autofs_get_inode(s, S_IFDIR | 0755); + if (!root_inode) + return -ENOMEM; + + root_inode->i_uid = ctx->uid; + root_inode->i_gid = ctx->gid; + root_inode->i_fop = &autofs_root_operations; + root_inode->i_op = &autofs_dir_inode_operations; + + s->s_root = d_make_root(root_inode); + if (unlikely(!s->s_root)) { + autofs_free_ino(ino); + return -ENOMEM; + } + s->s_root->d_fsdata = ino; + + if (ctx->pgrp_set) { + sbi->oz_pgrp = find_get_pid(ctx->pgrp); + if (!sbi->oz_pgrp) + return invalf(fc, "Could not find process group %d", + ctx->pgrp); + } else + sbi->oz_pgrp = get_task_pid(current, PIDTYPE_PGID); + + if (autofs_type_trigger(sbi->type)) + /* s->s_root won't be contended so there's little to + * be gained by not taking the d_lock when setting + * d_flags, even when a lot mounts are being done. + */ + managed_dentry_set_managed(s->s_root); + + pr_debug("pipe fd = %d, pgrp = %u\n", + sbi->pipefd, pid_nr(sbi->oz_pgrp)); + + sbi->flags &= ~AUTOFS_SBI_CATATONIC; + return 0; +} + +/* + * Validate the parameters and then request a superblock. + */ +static int autofs_get_tree(struct fs_context *fc) +{ + struct autofs_sb_info *sbi = fc->s_fs_info; + int ret; + + ret = autofs_validate_protocol(fc); + if (ret) + return ret; + + if (sbi->pipefd < 0) + return invalf(fc, "No control pipe specified"); + + return get_tree_nodev(fc, autofs_fill_super); +} + +static void autofs_free_fc(struct fs_context *fc) +{ + struct autofs_fs_context *ctx = fc->fs_private; + struct autofs_sb_info *sbi = fc->s_fs_info; + + if (sbi) { + if (sbi->pipe) + fput(sbi->pipe); + kfree(sbi); + } + kfree(ctx); +} + +static const struct fs_context_operations autofs_context_ops = { + .free = autofs_free_fc, + .parse_param = autofs_parse_param, + .get_tree = autofs_get_tree, +}; + +/* + * Set up the filesystem mount context. + */ +int autofs_init_fs_context(struct fs_context *fc) +{ + struct autofs_fs_context *ctx; + struct autofs_sb_info *sbi; + + ctx = kzalloc_obj(struct autofs_fs_context); + if (!ctx) + goto nomem; + + ctx->uid = current_uid(); + ctx->gid = current_gid(); + + sbi = autofs_alloc_sbi(); + if (!sbi) + goto nomem_ctx; + + fc->fs_private = ctx; + fc->s_fs_info = sbi; + fc->ops = &autofs_context_ops; + return 0; + +nomem_ctx: + kfree(ctx); +nomem: + return -ENOMEM; +} + +struct inode *autofs_get_inode(struct super_block *sb, umode_t mode) +{ + struct inode *inode = new_inode(sb); + + if (inode == NULL) + return NULL; + + inode->i_mode = mode; + if (sb->s_root) { + inode->i_uid = d_inode(sb->s_root)->i_uid; + inode->i_gid = d_inode(sb->s_root)->i_gid; + } + simple_inode_init_ts(inode); + inode->i_ino = get_next_ino(); + + if (S_ISDIR(mode)) { + set_nlink(inode, 2); + inode->i_op = &autofs_dir_inode_operations; + inode->i_fop = &autofs_dir_operations; + } else if (S_ISLNK(mode)) { + inode->i_op = &autofs_symlink_inode_operations; + } else + WARN_ON(1); + + return inode; +} diff --git a/fs/autofs/root.c b/fs/autofs/root.c new file mode 100644 index 000000000..b36439f45 --- /dev/null +++ b/fs/autofs/root.c @@ -0,0 +1,931 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 1997-1998 Transmeta Corporation -- All Rights Reserved + * Copyright 1999-2000 Jeremy Fitzhardinge + * Copyright 2001-2006 Ian Kent + */ + +#include +#include +#include + +#include "autofs_i.h" + +static int autofs_dir_permission(struct mnt_idmap *, struct inode *, int); +static int autofs_dir_symlink(struct mnt_idmap *, struct inode *, + struct dentry *, const char *); +static int autofs_dir_unlink(struct inode *, struct dentry *); +static int autofs_dir_rmdir(struct inode *, struct dentry *); +static struct dentry *autofs_dir_mkdir(struct mnt_idmap *, struct inode *, + struct dentry *, umode_t); +static long autofs_root_ioctl(struct file *, unsigned int, unsigned long); +#ifdef CONFIG_COMPAT +static long autofs_root_compat_ioctl(struct file *, + unsigned int, unsigned long); +#endif +static int autofs_dir_open(struct inode *inode, struct file *file); +static struct dentry *autofs_lookup(struct inode *, + struct dentry *, unsigned int); +static struct vfsmount *autofs_d_automount(struct path *); +static int autofs_d_manage(const struct path *, bool); +static void autofs_dentry_release(struct dentry *); + +const struct file_operations autofs_root_operations = { + .open = dcache_dir_open, + .release = dcache_dir_close, + .read = generic_read_dir, + .iterate_shared = dcache_readdir, + .llseek = dcache_dir_lseek, + .unlocked_ioctl = autofs_root_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = autofs_root_compat_ioctl, +#endif +}; + +const struct file_operations autofs_dir_operations = { + .open = autofs_dir_open, + .release = dcache_dir_close, + .read = generic_read_dir, + .iterate_shared = dcache_readdir, + .llseek = dcache_dir_lseek, +}; + +const struct inode_operations autofs_dir_inode_operations = { + .lookup = autofs_lookup, + .permission = autofs_dir_permission, + .unlink = autofs_dir_unlink, + .symlink = autofs_dir_symlink, + .mkdir = autofs_dir_mkdir, + .rmdir = autofs_dir_rmdir, +}; + +const struct dentry_operations autofs_dentry_operations = { + .d_automount = autofs_d_automount, + .d_manage = autofs_d_manage, + .d_release = autofs_dentry_release, +}; + +static void autofs_del_active(struct dentry *dentry) +{ + struct autofs_sb_info *sbi = autofs_sbi(dentry->d_sb); + struct autofs_info *ino; + + ino = autofs_dentry_ino(dentry); + spin_lock(&sbi->lookup_lock); + list_del_init(&ino->active); + spin_unlock(&sbi->lookup_lock); +} + +static int autofs_dir_open(struct inode *inode, struct file *file) +{ + struct dentry *dentry = file->f_path.dentry; + struct autofs_sb_info *sbi = autofs_sbi(dentry->d_sb); + struct autofs_info *ino = autofs_dentry_ino(dentry); + + pr_debug("file=%p dentry=%p %pd\n", file, dentry, dentry); + + if (autofs_oz_mode(sbi)) + goto out; + + /* + * An empty directory in an autofs file system is always a + * mount point. The daemon must have failed to mount this + * during lookup so it doesn't exist. This can happen, for + * example, if user space returns an incorrect status for a + * mount request. Otherwise we're doing a readdir on the + * autofs file system so just let the libfs routines handle + * it. + */ + spin_lock(&sbi->lookup_lock); + if (!path_is_mountpoint(&file->f_path) && autofs_empty(ino)) { + spin_unlock(&sbi->lookup_lock); + return -ENOENT; + } + spin_unlock(&sbi->lookup_lock); + +out: + return dcache_dir_open(inode, file); +} + +static void autofs_dentry_release(struct dentry *de) +{ + struct autofs_info *ino = autofs_dentry_ino(de); + struct autofs_sb_info *sbi = autofs_sbi(de->d_sb); + + pr_debug("releasing %p\n", de); + + if (!ino) + return; + + if (sbi) { + spin_lock(&sbi->lookup_lock); + if (!list_empty(&ino->active)) + list_del(&ino->active); + if (!list_empty(&ino->expiring)) + list_del(&ino->expiring); + spin_unlock(&sbi->lookup_lock); + } + + autofs_free_ino(ino); +} + +static struct dentry *autofs_lookup_active(struct dentry *dentry) +{ + struct autofs_sb_info *sbi = autofs_sbi(dentry->d_sb); + struct dentry *parent = dentry->d_parent; + const struct qstr *name = &dentry->d_name; + unsigned int len = name->len; + unsigned int hash = name->hash; + const unsigned char *str = name->name; + struct list_head *p, *head; + + head = &sbi->active_list; + if (list_empty(head)) + return NULL; + spin_lock(&sbi->lookup_lock); + list_for_each(p, head) { + struct autofs_info *ino; + struct dentry *active; + const struct qstr *qstr; + + ino = list_entry(p, struct autofs_info, active); + active = ino->dentry; + + spin_lock(&active->d_lock); + + /* Already gone? */ + if ((int) d_count(active) <= 0) + goto next; + + qstr = &active->d_name; + + if (active->d_name.hash != hash) + goto next; + if (active->d_parent != parent) + goto next; + + if (qstr->len != len) + goto next; + if (memcmp(qstr->name, str, len)) + goto next; + + if (d_unhashed(active)) { + dget_dlock(active); + spin_unlock(&active->d_lock); + spin_unlock(&sbi->lookup_lock); + return active; + } +next: + spin_unlock(&active->d_lock); + } + spin_unlock(&sbi->lookup_lock); + + return NULL; +} + +static struct dentry *autofs_lookup_expiring(struct dentry *dentry, + bool rcu_walk) +{ + struct autofs_sb_info *sbi = autofs_sbi(dentry->d_sb); + struct dentry *parent = dentry->d_parent; + const struct qstr *name = &dentry->d_name; + unsigned int len = name->len; + unsigned int hash = name->hash; + const unsigned char *str = name->name; + struct list_head *p, *head; + + head = &sbi->expiring_list; + if (list_empty(head)) + return NULL; + spin_lock(&sbi->lookup_lock); + list_for_each(p, head) { + struct autofs_info *ino; + struct dentry *expiring; + const struct qstr *qstr; + + if (rcu_walk) { + spin_unlock(&sbi->lookup_lock); + return ERR_PTR(-ECHILD); + } + + ino = list_entry(p, struct autofs_info, expiring); + expiring = ino->dentry; + + spin_lock(&expiring->d_lock); + + /* We've already been dentry_iput or unlinked */ + if (d_really_is_negative(expiring)) + goto next; + + qstr = &expiring->d_name; + + if (expiring->d_name.hash != hash) + goto next; + if (expiring->d_parent != parent) + goto next; + + if (qstr->len != len) + goto next; + if (memcmp(qstr->name, str, len)) + goto next; + + if (d_unhashed(expiring)) { + dget_dlock(expiring); + spin_unlock(&expiring->d_lock); + spin_unlock(&sbi->lookup_lock); + return expiring; + } +next: + spin_unlock(&expiring->d_lock); + } + spin_unlock(&sbi->lookup_lock); + + return NULL; +} + +static int autofs_mount_wait(const struct path *path, bool rcu_walk) +{ + struct autofs_sb_info *sbi = autofs_sbi(path->dentry->d_sb); + struct autofs_info *ino = autofs_dentry_ino(path->dentry); + int status = 0; + + if (ino->flags & AUTOFS_INF_PENDING) { + if (rcu_walk) + return -ECHILD; + pr_debug("waiting for mount name=%pd\n", path->dentry); + status = autofs_wait(sbi, path, NFY_MOUNT); + pr_debug("mount wait done status=%d\n", status); + ino->last_used = jiffies; + return status; + } + if (!(sbi->flags & AUTOFS_SBI_STRICTEXPIRE)) + ino->last_used = jiffies; + return status; +} + +static int do_expire_wait(const struct path *path, bool rcu_walk) +{ + struct dentry *dentry = path->dentry; + struct dentry *expiring; + + expiring = autofs_lookup_expiring(dentry, rcu_walk); + if (IS_ERR(expiring)) + return PTR_ERR(expiring); + if (!expiring) + return autofs_expire_wait(path, rcu_walk); + else { + const struct path this = { .mnt = path->mnt, .dentry = expiring }; + /* + * If we are racing with expire the request might not + * be quite complete, but the directory has been removed + * so it must have been successful, just wait for it. + */ + autofs_expire_wait(&this, 0); + autofs_del_expiring(expiring); + dput(expiring); + } + return 0; +} + +static struct dentry *autofs_mountpoint_changed(struct path *path) +{ + struct dentry *dentry = path->dentry; + struct autofs_sb_info *sbi = autofs_sbi(dentry->d_sb); + + /* If this is an indirect mount the dentry could have gone away + * and a new one created. + * + * This is unusual and I can't remember the case for which it + * was originally added now. But an example of how this can + * happen is an autofs indirect mount that has the "browse" + * option set and also has the "symlink" option in the autofs + * map entry. In this case the daemon will remove the browse + * directory and create a symlink as the mount leaving the + * struct path stale. + * + * Another not so obvious case is when a mount in an autofs + * indirect mount that uses the "nobrowse" option is being + * expired at the same time as a path walk. If the mount has + * been umounted but the mount point directory seen before + * becoming unhashed (during a lockless path walk) when a stat + * family system call is made the mount won't be re-mounted as + * it should. In this case the mount point that's been removed + * (by the daemon) will be stale and the a new mount point + * dentry created. + */ + if (autofs_type_indirect(sbi->type) && d_unhashed(dentry)) { + struct dentry *parent = dentry->d_parent; + struct autofs_info *ino; + struct dentry *new; + + new = d_lookup(parent, &dentry->d_name); + if (!new) + return NULL; + ino = autofs_dentry_ino(new); + ino->last_used = jiffies; + dput(path->dentry); + path->dentry = new; + } + return path->dentry; +} + +static struct vfsmount *autofs_d_automount(struct path *path) +{ + struct dentry *dentry = path->dentry; + struct autofs_sb_info *sbi = autofs_sbi(dentry->d_sb); + struct autofs_info *ino = autofs_dentry_ino(dentry); + int status; + + pr_debug("dentry=%p %pd\n", dentry, dentry); + + /* The daemon never triggers a mount. */ + if (autofs_oz_mode(sbi)) + return NULL; + + /* Refuse to trigger mount if current namespace is not the owner + * and the mount is propagation private. + */ + if (sbi->mnt_ns_id != to_ns_common(current->nsproxy->mnt_ns)->ns_id) { + if (vfsmount_to_propagation_flags(path->mnt) & MS_PRIVATE) + return ERR_PTR(-EPERM); + } + + /* + * If an expire request is pending everyone must wait. + * If the expire fails we're still mounted so continue + * the follow and return. A return of -EAGAIN (which only + * happens with indirect mounts) means the expire completed + * and the directory was removed, so just go ahead and try + * the mount. + */ + status = do_expire_wait(path, 0); + if (status && status != -EAGAIN) + return NULL; + + /* Callback to the daemon to perform the mount or wait */ + spin_lock(&sbi->fs_lock); + if (ino->flags & AUTOFS_INF_PENDING) { + spin_unlock(&sbi->fs_lock); + status = autofs_mount_wait(path, 0); + if (status) + return ERR_PTR(status); + goto done; + } + + /* + * If the dentry is a symlink it's equivalent to a directory + * having path_is_mountpoint() true, so there's no need to call + * back to the daemon. + */ + if (d_really_is_positive(dentry) && d_is_symlink(dentry)) { + spin_unlock(&sbi->fs_lock); + goto done; + } + + if (!path_is_mountpoint(path)) { + /* + * It's possible that user space hasn't removed directories + * after umounting a rootless multi-mount, although it + * should. For v5 path_has_submounts() is sufficient to + * handle this because the leaves of the directory tree under + * the mount never trigger mounts themselves (they have an + * autofs trigger mount mounted on them). But v4 pseudo direct + * mounts do need the leaves to trigger mounts. In this case + * we have no choice but to use the autofs_empty() check and + * require user space behave. + */ + if (sbi->version > 4) { + if (path_has_submounts(path)) { + spin_unlock(&sbi->fs_lock); + goto done; + } + } else { + if (!autofs_empty(ino)) { + spin_unlock(&sbi->fs_lock); + goto done; + } + } + ino->flags |= AUTOFS_INF_PENDING; + spin_unlock(&sbi->fs_lock); + status = autofs_mount_wait(path, 0); + spin_lock(&sbi->fs_lock); + ino->flags &= ~AUTOFS_INF_PENDING; + if (status) { + spin_unlock(&sbi->fs_lock); + return ERR_PTR(status); + } + } + spin_unlock(&sbi->fs_lock); +done: + /* Mount succeeded, check if we ended up with a new dentry */ + dentry = autofs_mountpoint_changed(path); + if (!dentry) + return ERR_PTR(-ENOENT); + + return NULL; +} + +static int autofs_d_manage(const struct path *path, bool rcu_walk) +{ + struct dentry *dentry = path->dentry; + struct autofs_sb_info *sbi = autofs_sbi(dentry->d_sb); + struct autofs_info *ino = autofs_dentry_ino(dentry); + int status; + + pr_debug("dentry=%p %pd\n", dentry, dentry); + + /* The daemon never waits. */ + if (autofs_oz_mode(sbi)) { + if (!path_is_mountpoint(path)) + return -EISDIR; + return 0; + } + + /* Wait for pending expires */ + if (do_expire_wait(path, rcu_walk) == -ECHILD) + return -ECHILD; + + /* + * This dentry may be under construction so wait on mount + * completion. + */ + status = autofs_mount_wait(path, rcu_walk); + if (status) + return status; + + if (rcu_walk) { + /* We don't need fs_lock in rcu_walk mode, + * just testing 'AUTOFS_INF_WANT_EXPIRE' is enough. + * + * We only return -EISDIR when certain this isn't + * a mount-trap. + */ + struct inode *inode; + + if (ino->flags & AUTOFS_INF_WANT_EXPIRE) + return 0; + if (path_is_mountpoint(path)) + return 0; + inode = d_inode_rcu(dentry); + if (inode && S_ISLNK(inode->i_mode)) + return -EISDIR; + if (!autofs_empty(ino)) + return -EISDIR; + return 0; + } + + spin_lock(&sbi->fs_lock); + /* + * If the dentry has been selected for expire while we slept + * on the lock then it might go away. We'll deal with that in + * ->d_automount() and wait on a new mount if the expire + * succeeds or return here if it doesn't (since there's no + * mount to follow with a rootless multi-mount). + */ + if (!(ino->flags & AUTOFS_INF_EXPIRING)) { + /* + * Any needed mounting has been completed and the path + * updated so check if this is a rootless multi-mount so + * we can avoid needless calls ->d_automount() and avoid + * an incorrect ELOOP error return. + */ + if ((!path_is_mountpoint(path) && !autofs_empty(ino)) || + (d_really_is_positive(dentry) && d_is_symlink(dentry))) + status = -EISDIR; + } + spin_unlock(&sbi->fs_lock); + + return status; +} + +/* Lookups in the root directory */ +static struct dentry *autofs_lookup(struct inode *dir, + struct dentry *dentry, unsigned int flags) +{ + struct autofs_sb_info *sbi; + struct autofs_info *ino; + struct dentry *active; + + pr_debug("name = %pd\n", dentry); + + /* File name too long to exist */ + if (dentry->d_name.len > NAME_MAX) + return ERR_PTR(-ENAMETOOLONG); + + sbi = autofs_sbi(dir->i_sb); + + pr_debug("pid = %u, pgrp = %u, catatonic = %d, oz_mode = %d\n", + current->pid, task_pgrp_nr(current), + sbi->flags & AUTOFS_SBI_CATATONIC, + autofs_oz_mode(sbi)); + + active = autofs_lookup_active(dentry); + if (active) + return active; + else { + /* + * A dentry that is not within the root can never trigger a + * mount operation, unless the directory already exists, so we + * can return fail immediately. The daemon however does need + * to create directories within the file system. + */ + if (!autofs_oz_mode(sbi) && !IS_ROOT(dentry->d_parent)) + return ERR_PTR(-ENOENT); + + ino = autofs_new_ino(sbi); + if (!ino) + return ERR_PTR(-ENOMEM); + + spin_lock(&sbi->lookup_lock); + spin_lock(&dentry->d_lock); + /* Mark entries in the root as mount triggers */ + if (IS_ROOT(dentry->d_parent) && + autofs_type_indirect(sbi->type)) + __managed_dentry_set_managed(dentry); + dentry->d_fsdata = ino; + ino->dentry = dentry; + + list_add(&ino->active, &sbi->active_list); + spin_unlock(&sbi->lookup_lock); + spin_unlock(&dentry->d_lock); + } + return NULL; +} + +static int autofs_dir_permission(struct mnt_idmap *idmap, + struct inode *inode, int mask) +{ + if (mask & MAY_WRITE) { + struct autofs_sb_info *sbi = autofs_sbi(inode->i_sb); + + if (!autofs_oz_mode(sbi)) + return -EACCES; + + /* autofs_oz_mode() needs to allow path walks when the + * autofs mount is catatonic but the state of an autofs + * file system needs to be preserved over restarts. + */ + if (sbi->flags & AUTOFS_SBI_CATATONIC) + return -EACCES; + } + + return generic_permission(idmap, inode, mask); +} + +static int autofs_dir_symlink(struct mnt_idmap *idmap, + struct inode *dir, struct dentry *dentry, + const char *symname) +{ + struct autofs_info *ino = autofs_dentry_ino(dentry); + struct autofs_info *p_ino; + struct inode *inode; + char *cp; + + pr_debug("%s <- %pd\n", symname, dentry); + + BUG_ON(!ino); + + autofs_clean_ino(ino); + + autofs_del_active(dentry); + + cp = kstrdup(symname, GFP_KERNEL); + if (!cp) + return -ENOMEM; + + inode = autofs_get_inode(dir->i_sb, S_IFLNK | 0555); + if (!inode) { + kfree(cp); + return -ENOMEM; + } + inode->i_private = cp; + inode->i_size = strlen(cp); + + d_make_persistent(dentry, inode); + p_ino = autofs_dentry_ino(dentry->d_parent); + p_ino->count++; + + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + + return 0; +} + +/* + * NOTE! + * + * Normal filesystems would do a "d_delete()" to tell the VFS dcache + * that the file no longer exists. However, doing that means that the + * VFS layer can turn the dentry into a negative dentry. We don't want + * this, because the unlink is probably the result of an expire. + * We simply d_drop it and add it to a expiring list in the super block, + * which allows the dentry lookup to check for an incomplete expire. + * + * If a process is blocked on the dentry waiting for the expire to finish, + * it will invalidate the dentry and try to mount with a new one. + * + * Also see autofs_dir_rmdir().. + */ +static int autofs_dir_unlink(struct inode *dir, struct dentry *dentry) +{ + struct autofs_sb_info *sbi = autofs_sbi(dir->i_sb); + struct autofs_info *p_ino; + + p_ino = autofs_dentry_ino(dentry->d_parent); + p_ino->count--; + d_make_discardable(dentry); + + d_inode(dentry)->i_size = 0; + clear_nlink(d_inode(dentry)); + + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + + spin_lock(&sbi->lookup_lock); + __autofs_add_expiring(dentry); + d_drop(dentry); + spin_unlock(&sbi->lookup_lock); + + return 0; +} + +/* + * Version 4 of autofs provides a pseudo direct mount implementation + * that relies on directories at the leaves of a directory tree under + * an indirect mount to trigger mounts. To allow for this we need to + * set the DMANAGED_AUTOMOUNT and DMANAGED_TRANSIT flags on the leaves + * of the directory tree. There is no need to clear the automount flag + * following a mount or restore it after an expire because these mounts + * are always covered. However, it is necessary to ensure that these + * flags are clear on non-empty directories to avoid unnecessary calls + * during path walks. + */ +static void autofs_set_leaf_automount_flags(struct dentry *dentry) +{ + struct dentry *parent; + + /* root and dentrys in the root are already handled */ + if (IS_ROOT(dentry->d_parent)) + return; + + managed_dentry_set_managed(dentry); + + parent = dentry->d_parent; + /* only consider parents below dentrys in the root */ + if (IS_ROOT(parent->d_parent)) + return; + managed_dentry_clear_managed(parent); +} + +static void autofs_clear_leaf_automount_flags(struct dentry *dentry) +{ + struct dentry *parent; + + /* flags for dentrys in the root are handled elsewhere */ + if (IS_ROOT(dentry->d_parent)) + return; + + managed_dentry_clear_managed(dentry); + + parent = dentry->d_parent; + /* only consider parents below dentrys in the root */ + if (IS_ROOT(parent->d_parent)) + return; + if (autofs_dentry_ino(parent)->count == 2) + managed_dentry_set_managed(parent); +} + +static int autofs_dir_rmdir(struct inode *dir, struct dentry *dentry) +{ + struct autofs_sb_info *sbi = autofs_sbi(dir->i_sb); + struct autofs_info *ino = autofs_dentry_ino(dentry); + struct autofs_info *p_ino; + + pr_debug("dentry %p, removing %pd\n", dentry, dentry); + + if (ino->count != 1) + return -ENOTEMPTY; + + spin_lock(&sbi->lookup_lock); + __autofs_add_expiring(dentry); + d_drop(dentry); + spin_unlock(&sbi->lookup_lock); + + if (sbi->version < 5) + autofs_clear_leaf_automount_flags(dentry); + + p_ino = autofs_dentry_ino(dentry->d_parent); + p_ino->count--; + d_make_discardable(dentry); + d_inode(dentry)->i_size = 0; + clear_nlink(d_inode(dentry)); + + if (dir->i_nlink) + drop_nlink(dir); + + return 0; +} + +static struct dentry *autofs_dir_mkdir(struct mnt_idmap *idmap, + struct inode *dir, struct dentry *dentry, + umode_t mode) +{ + struct autofs_sb_info *sbi = autofs_sbi(dir->i_sb); + struct autofs_info *ino = autofs_dentry_ino(dentry); + struct autofs_info *p_ino; + struct inode *inode; + + pr_debug("dentry %p, creating %pd\n", dentry, dentry); + + BUG_ON(!ino); + + autofs_clean_ino(ino); + + autofs_del_active(dentry); + + inode = autofs_get_inode(dir->i_sb, mode); + if (!inode) + return ERR_PTR(-ENOMEM); + + if (sbi->version < 5) + autofs_set_leaf_automount_flags(dentry); + + d_make_persistent(dentry, inode); + p_ino = autofs_dentry_ino(dentry->d_parent); + p_ino->count++; + inc_nlink(dir); + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + + return NULL; +} + +/* Get/set timeout ioctl() operation */ +#ifdef CONFIG_COMPAT +static inline int autofs_compat_get_set_timeout(struct autofs_sb_info *sbi, + compat_ulong_t __user *p) +{ + unsigned long ntimeout; + int rv; + + rv = get_user(ntimeout, p); + if (rv) + goto error; + + rv = put_user(sbi->exp_timeout/HZ, p); + if (rv) + goto error; + + if (ntimeout > UINT_MAX/HZ) + sbi->exp_timeout = 0; + else + sbi->exp_timeout = ntimeout * HZ; + + return 0; +error: + return rv; +} +#endif + +static inline int autofs_get_set_timeout(struct autofs_sb_info *sbi, + unsigned long __user *p) +{ + unsigned long ntimeout; + int rv; + + rv = get_user(ntimeout, p); + if (rv) + goto error; + + rv = put_user(sbi->exp_timeout/HZ, p); + if (rv) + goto error; + + if (ntimeout > ULONG_MAX/HZ) + sbi->exp_timeout = 0; + else + sbi->exp_timeout = ntimeout * HZ; + + return 0; +error: + return rv; +} + +/* Return protocol version */ +static inline int autofs_get_protover(struct autofs_sb_info *sbi, + int __user *p) +{ + return put_user(sbi->version, p); +} + +/* Return protocol sub version */ +static inline int autofs_get_protosubver(struct autofs_sb_info *sbi, + int __user *p) +{ + return put_user(sbi->sub_version, p); +} + +/* +* Tells the daemon whether it can umount the autofs mount. +*/ +static inline int autofs_ask_umount(struct vfsmount *mnt, int __user *p) +{ + int status = 0; + + if (may_umount(mnt)) + status = 1; + + pr_debug("may umount %d\n", status); + + status = put_user(status, p); + + return status; +} + +/* Identify autofs_dentries - this is so we can tell if there's + * an extra dentry refcount or not. We only hold a refcount on the + * dentry if its non-negative (ie, d_inode != NULL) + */ +int is_autofs_dentry(struct dentry *dentry) +{ + return dentry && d_really_is_positive(dentry) && + dentry->d_op == &autofs_dentry_operations && + dentry->d_fsdata != NULL; +} + +/* + * ioctl()'s on the root directory is the chief method for the daemon to + * generate kernel reactions + */ +static int autofs_root_ioctl_unlocked(struct inode *inode, struct file *filp, + unsigned int cmd, unsigned long arg) +{ + struct autofs_sb_info *sbi = autofs_sbi(inode->i_sb); + void __user *p = (void __user *)arg; + + pr_debug("cmd = 0x%08x, arg = 0x%08lx, sbi = %p, pgrp = %u\n", + cmd, arg, sbi, task_pgrp_nr(current)); + + if (_IOC_TYPE(cmd) != _IOC_TYPE(AUTOFS_IOC_FIRST) || + _IOC_NR(cmd) - _IOC_NR(AUTOFS_IOC_FIRST) >= AUTOFS_IOC_COUNT) + return -ENOTTY; + + if (!autofs_oz_mode(sbi) && !capable(CAP_SYS_ADMIN)) + return -EPERM; + + switch (cmd) { + case AUTOFS_IOC_READY: /* Wait queue: go ahead and retry */ + return autofs_wait_release(sbi, (autofs_wqt_t) arg, 0); + case AUTOFS_IOC_FAIL: /* Wait queue: fail with ENOENT */ + return autofs_wait_release(sbi, (autofs_wqt_t) arg, -ENOENT); + case AUTOFS_IOC_CATATONIC: /* Enter catatonic mode (daemon shutdown) */ + autofs_catatonic_mode(sbi); + return 0; + case AUTOFS_IOC_PROTOVER: /* Get protocol version */ + return autofs_get_protover(sbi, p); + case AUTOFS_IOC_PROTOSUBVER: /* Get protocol sub version */ + return autofs_get_protosubver(sbi, p); + case AUTOFS_IOC_SETTIMEOUT: + return autofs_get_set_timeout(sbi, p); +#ifdef CONFIG_COMPAT + case AUTOFS_IOC_SETTIMEOUT32: + return autofs_compat_get_set_timeout(sbi, p); +#endif + + case AUTOFS_IOC_ASKUMOUNT: + return autofs_ask_umount(filp->f_path.mnt, p); + + /* return a single thing to expire */ + case AUTOFS_IOC_EXPIRE: + return autofs_expire_run(inode->i_sb, filp->f_path.mnt, sbi, p); + /* same as above, but can send multiple expires through pipe */ + case AUTOFS_IOC_EXPIRE_MULTI: + return autofs_expire_multi(inode->i_sb, + filp->f_path.mnt, sbi, p); + + default: + return -EINVAL; + } +} + +static long autofs_root_ioctl(struct file *filp, + unsigned int cmd, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + + return autofs_root_ioctl_unlocked(inode, filp, cmd, arg); +} + +#ifdef CONFIG_COMPAT +static long autofs_root_compat_ioctl(struct file *filp, + unsigned int cmd, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + int ret; + + if (cmd == AUTOFS_IOC_READY || cmd == AUTOFS_IOC_FAIL) + ret = autofs_root_ioctl_unlocked(inode, filp, cmd, arg); + else + ret = autofs_root_ioctl_unlocked(inode, filp, cmd, + (unsigned long) compat_ptr(arg)); + + return ret; +} +#endif diff --git a/fs/autofs/symlink.c b/fs/autofs/symlink.c new file mode 100644 index 000000000..7ac67dc76 --- /dev/null +++ b/fs/autofs/symlink.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 1997-1998 Transmeta Corporation -- All Rights Reserved + */ + +#include "autofs_i.h" + +static const char *autofs_get_link(struct dentry *dentry, + struct inode *inode, + struct delayed_call *done) +{ + struct autofs_sb_info *sbi; + struct autofs_info *ino; + + if (!dentry) + return ERR_PTR(-ECHILD); + sbi = autofs_sbi(dentry->d_sb); + ino = autofs_dentry_ino(dentry); + if (ino && !autofs_oz_mode(sbi)) + ino->last_used = jiffies; + return d_inode(dentry)->i_private; +} + +const struct inode_operations autofs_symlink_inode_operations = { + .get_link = autofs_get_link +}; diff --git a/fs/autofs/waitq.c b/fs/autofs/waitq.c new file mode 100644 index 000000000..d46241342 --- /dev/null +++ b/fs/autofs/waitq.c @@ -0,0 +1,513 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 1997-1998 Transmeta Corporation -- All Rights Reserved + * Copyright 2001-2006 Ian Kent + */ + +#include +#include "autofs_i.h" + +/* We make this a static variable rather than a part of the superblock; it + * is better if we don't reassign numbers easily even across filesystems + */ +static autofs_wqt_t autofs_next_wait_queue = 1; + +void autofs_catatonic_mode(struct autofs_sb_info *sbi) +{ + struct autofs_wait_queue *wq, *nwq; + + mutex_lock(&sbi->wq_mutex); + if (sbi->flags & AUTOFS_SBI_CATATONIC) { + mutex_unlock(&sbi->wq_mutex); + return; + } + + pr_debug("entering catatonic mode\n"); + + sbi->flags |= AUTOFS_SBI_CATATONIC; + wq = sbi->queues; + sbi->queues = NULL; /* Erase all wait queues */ + while (wq) { + nwq = wq->next; + wq->status = -ENOENT; /* Magic is gone - report failure */ + kfree(wq->name.name - wq->offset); + wq->name.name = NULL; + wake_up(&wq->queue); + if (!--wq->wait_ctr) + kfree(wq); + wq = nwq; + } + fput(sbi->pipe); /* Close the pipe */ + sbi->pipe = NULL; + sbi->pipefd = -1; + mutex_unlock(&sbi->wq_mutex); +} + +static int autofs_write(struct autofs_sb_info *sbi, + struct file *file, const void *addr, int bytes) +{ + unsigned long sigpipe, flags; + const char *data = (const char *)addr; + ssize_t wr = 0; + + sigpipe = sigismember(¤t->pending.signal, SIGPIPE); + + mutex_lock(&sbi->pipe_mutex); + while (bytes) { + wr = __kernel_write(file, data, bytes, NULL); + if (wr <= 0) + break; + data += wr; + bytes -= wr; + } + mutex_unlock(&sbi->pipe_mutex); + + /* Keep the currently executing process from receiving a + * SIGPIPE unless it was already supposed to get one + */ + if (wr == -EPIPE && !sigpipe) { + spin_lock_irqsave(¤t->sighand->siglock, flags); + sigdelset(¤t->pending.signal, SIGPIPE); + recalc_sigpending(); + spin_unlock_irqrestore(¤t->sighand->siglock, flags); + } + + /* if 'wr' returned 0 (impossible) we assume -EIO (safe) */ + return bytes == 0 ? 0 : wr < 0 ? wr : -EIO; +} + +static void autofs_notify_daemon(struct autofs_sb_info *sbi, + struct autofs_wait_queue *wq, + int type) +{ + union { + struct autofs_packet_hdr hdr; + union autofs_packet_union v4_pkt; + union autofs_v5_packet_union v5_pkt; + } pkt; + struct file *pipe = NULL; + size_t pktsz; + int ret; + + pr_debug("wait id = 0x%08lx, name = %.*s, type=%d\n", + (unsigned long) wq->wait_queue_token, + wq->name.len, wq->name.name, type); + + memset(&pkt, 0, sizeof(pkt)); /* For security reasons */ + + pkt.hdr.proto_version = sbi->version; + pkt.hdr.type = type; + + switch (type) { + /* Kernel protocol v4 missing and expire packets */ + case autofs_ptype_missing: + { + struct autofs_packet_missing *mp = &pkt.v4_pkt.missing; + + pktsz = sizeof(*mp); + + mp->wait_queue_token = wq->wait_queue_token; + mp->len = wq->name.len; + memcpy(mp->name, wq->name.name, wq->name.len); + mp->name[wq->name.len] = '\0'; + break; + } + case autofs_ptype_expire_multi: + { + struct autofs_packet_expire_multi *ep = + &pkt.v4_pkt.expire_multi; + + pktsz = sizeof(*ep); + + ep->wait_queue_token = wq->wait_queue_token; + ep->len = wq->name.len; + memcpy(ep->name, wq->name.name, wq->name.len); + ep->name[wq->name.len] = '\0'; + break; + } + /* + * Kernel protocol v5 packet for handling indirect and direct + * mount missing and expire requests + */ + case autofs_ptype_missing_indirect: + case autofs_ptype_expire_indirect: + case autofs_ptype_missing_direct: + case autofs_ptype_expire_direct: + { + struct autofs_v5_packet *packet = &pkt.v5_pkt.v5_packet; + struct user_namespace *user_ns = sbi->pipe->f_cred->user_ns; + + pktsz = sizeof(*packet); + + packet->wait_queue_token = wq->wait_queue_token; + packet->len = wq->name.len; + memcpy(packet->name, wq->name.name, wq->name.len); + packet->name[wq->name.len] = '\0'; + packet->dev = wq->dev; + packet->ino = wq->ino; + packet->uid = from_kuid_munged(user_ns, wq->uid); + packet->gid = from_kgid_munged(user_ns, wq->gid); + packet->pid = wq->pid; + packet->tgid = wq->tgid; + break; + } + default: + pr_warn("bad type %d!\n", type); + mutex_unlock(&sbi->wq_mutex); + return; + } + + pipe = get_file(sbi->pipe); + + mutex_unlock(&sbi->wq_mutex); + + switch (ret = autofs_write(sbi, pipe, &pkt, pktsz)) { + case 0: + break; + case -ENOMEM: + case -ERESTARTSYS: + /* Just fail this one */ + autofs_wait_release(sbi, wq->wait_queue_token, ret); + break; + default: + autofs_catatonic_mode(sbi); + break; + } + fput(pipe); +} + +static struct autofs_wait_queue * +autofs_find_wait(struct autofs_sb_info *sbi, const struct qstr *qstr) +{ + struct autofs_wait_queue *wq; + + for (wq = sbi->queues; wq; wq = wq->next) { + if (wq->name.hash == qstr->hash && + wq->name.len == qstr->len && + wq->name.name && + !memcmp(wq->name.name, qstr->name, qstr->len)) + break; + } + return wq; +} + +/* + * Check if we have a valid request. + * Returns + * 1 if the request should continue. + * In this case we can return an autofs_wait_queue entry if one is + * found or NULL to idicate a new wait needs to be created. + * 0 or a negative errno if the request shouldn't continue. + */ +static int validate_request(struct autofs_wait_queue **wait, + struct autofs_sb_info *sbi, + const struct qstr *qstr, + const struct path *path, enum autofs_notify notify) +{ + struct dentry *dentry = path->dentry; + struct autofs_wait_queue *wq; + struct autofs_info *ino; + + if (sbi->flags & AUTOFS_SBI_CATATONIC) + return -ENOENT; + + /* Wait in progress, continue; */ + wq = autofs_find_wait(sbi, qstr); + if (wq) { + *wait = wq; + return 1; + } + + *wait = NULL; + + /* If we don't yet have any info this is a new request */ + ino = autofs_dentry_ino(dentry); + if (!ino) + return 1; + + /* + * If we've been asked to wait on an existing expire (NFY_NONE) + * but there is no wait in the queue ... + */ + if (notify == NFY_NONE) { + /* + * Either we've betean the pending expire to post it's + * wait or it finished while we waited on the mutex. + * So we need to wait till either, the wait appears + * or the expire finishes. + */ + + while (ino->flags & AUTOFS_INF_EXPIRING) { + mutex_unlock(&sbi->wq_mutex); + schedule_timeout_interruptible(HZ/10); + if (mutex_lock_interruptible(&sbi->wq_mutex)) + return -EINTR; + + if (sbi->flags & AUTOFS_SBI_CATATONIC) + return -ENOENT; + + wq = autofs_find_wait(sbi, qstr); + if (wq) { + *wait = wq; + return 1; + } + } + + /* + * Not ideal but the status has already gone. Of the two + * cases where we wait on NFY_NONE neither depend on the + * return status of the wait. + */ + return 0; + } + + /* + * If we've been asked to trigger a mount and the request + * completed while we waited on the mutex ... + */ + if (notify == NFY_MOUNT) { + struct dentry *new = NULL; + struct path this; + int valid = 1; + + /* + * If the dentry was successfully mounted while we slept + * on the wait queue mutex we can return success. If it + * isn't mounted (doesn't have submounts for the case of + * a multi-mount with no mount at it's base) we can + * continue on and create a new request. + */ + if (!IS_ROOT(dentry)) { + if (d_unhashed(dentry) && + d_really_is_positive(dentry)) { + struct dentry *parent = dentry->d_parent; + + new = d_lookup(parent, &dentry->d_name); + if (new) + dentry = new; + } + } + this.mnt = path->mnt; + this.dentry = dentry; + if (path_has_submounts(&this)) + valid = 0; + + if (new) + dput(new); + return valid; + } + + return 1; +} + +int autofs_wait(struct autofs_sb_info *sbi, + const struct path *path, enum autofs_notify notify) +{ + struct dentry *dentry = path->dentry; + struct autofs_wait_queue *wq; + struct qstr qstr; + char *name; + int status, ret, type; + unsigned int offset = 0; + pid_t pid; + pid_t tgid; + + /* In catatonic mode, we don't wait for nobody */ + if (sbi->flags & AUTOFS_SBI_CATATONIC) + return -ENOENT; + + /* + * Try translating pids to the namespace of the daemon. + * + * Zero means failure: we are in an unrelated pid namespace. + */ + pid = task_pid_nr_ns(current, ns_of_pid(sbi->oz_pgrp)); + tgid = task_tgid_nr_ns(current, ns_of_pid(sbi->oz_pgrp)); + if (pid == 0 || tgid == 0) + return -ENOENT; + + if (d_really_is_negative(dentry)) { + /* + * A wait for a negative dentry is invalid for certain + * cases. A direct or offset mount "always" has its mount + * point directory created and so the request dentry must + * be positive or the map key doesn't exist. The situation + * is very similar for indirect mounts except only dentrys + * in the root of the autofs file system may be negative. + */ + if (autofs_type_trigger(sbi->type)) + return -ENOENT; + else if (!IS_ROOT(dentry->d_parent)) + return -ENOENT; + } + + name = kmalloc(NAME_MAX + 1, GFP_KERNEL); + if (!name) + return -ENOMEM; + + /* If this is a direct mount request create a dummy name */ + if (IS_ROOT(dentry) && autofs_type_trigger(sbi->type)) { + qstr.name = name; + qstr.len = sprintf(name, "%p", dentry); + } else { + char *p = dentry_path_raw(dentry, name, NAME_MAX); + if (IS_ERR(p)) { + kfree(name); + return -ENOENT; + } + qstr.name = ++p; // skip the leading slash + qstr.len = strlen(p); + offset = p - name; + } + qstr.hash = full_name_hash(dentry, qstr.name, qstr.len); + + if (mutex_lock_interruptible(&sbi->wq_mutex)) { + kfree(name); + return -EINTR; + } + + ret = validate_request(&wq, sbi, &qstr, path, notify); + if (ret <= 0) { + if (ret != -EINTR) + mutex_unlock(&sbi->wq_mutex); + kfree(name); + return ret; + } + + if (!wq) { + /* Create a new wait queue */ + wq = kmalloc_obj(struct autofs_wait_queue); + if (!wq) { + kfree(name); + mutex_unlock(&sbi->wq_mutex); + return -ENOMEM; + } + + wq->wait_queue_token = autofs_next_wait_queue; + if (++autofs_next_wait_queue == 0) + autofs_next_wait_queue = 1; + wq->next = sbi->queues; + sbi->queues = wq; + init_waitqueue_head(&wq->queue); + memcpy(&wq->name, &qstr, sizeof(struct qstr)); + wq->offset = offset; + wq->dev = autofs_get_dev(sbi); + wq->ino = autofs_get_ino(sbi); + wq->uid = current_uid(); + wq->gid = current_gid(); + wq->pid = pid; + wq->tgid = tgid; + wq->status = -EINTR; /* Status return if interrupted */ + wq->wait_ctr = 2; + + if (sbi->version < 5) { + if (notify == NFY_MOUNT) + type = autofs_ptype_missing; + else + type = autofs_ptype_expire_multi; + } else { + if (notify == NFY_MOUNT) + type = autofs_type_trigger(sbi->type) ? + autofs_ptype_missing_direct : + autofs_ptype_missing_indirect; + else + type = autofs_type_trigger(sbi->type) ? + autofs_ptype_expire_direct : + autofs_ptype_expire_indirect; + } + + pr_debug("new wait id = 0x%08lx, name = %.*s, nfy=%d\n", + (unsigned long) wq->wait_queue_token, wq->name.len, + wq->name.name, notify); + + /* + * autofs_notify_daemon() may block; it will unlock ->wq_mutex + */ + autofs_notify_daemon(sbi, wq, type); + } else { + wq->wait_ctr++; + pr_debug("existing wait id = 0x%08lx, name = %.*s, nfy=%d\n", + (unsigned long) wq->wait_queue_token, wq->name.len, + wq->name.name, notify); + mutex_unlock(&sbi->wq_mutex); + kfree(name); + } + + /* + * wq->name.name is NULL iff the lock is already released + * or the mount has been made catatonic. + */ + wait_event_killable(wq->queue, wq->name.name == NULL); + status = wq->status; + + /* + * For direct and offset mounts we need to track the requester's + * uid and gid in the dentry info struct. This is so it can be + * supplied, on request, by the misc device ioctl interface. + * This is needed during daemon resatart when reconnecting + * to existing, active, autofs mounts. The uid and gid (and + * related string values) may be used for macro substitution + * in autofs mount maps. + */ + if (!status) { + struct autofs_info *ino; + struct dentry *de = NULL; + + /* direct mount or browsable map */ + ino = autofs_dentry_ino(dentry); + if (!ino) { + /* If not lookup actual dentry used */ + de = d_lookup(dentry->d_parent, &dentry->d_name); + if (de) + ino = autofs_dentry_ino(de); + } + + /* Set mount requester */ + if (ino) { + spin_lock(&sbi->fs_lock); + ino->uid = wq->uid; + ino->gid = wq->gid; + spin_unlock(&sbi->fs_lock); + } + + if (de) + dput(de); + } + + /* Are we the last process to need status? */ + mutex_lock(&sbi->wq_mutex); + if (!--wq->wait_ctr) + kfree(wq); + mutex_unlock(&sbi->wq_mutex); + + return status; +} + + +int autofs_wait_release(struct autofs_sb_info *sbi, + autofs_wqt_t wait_queue_token, int status) +{ + struct autofs_wait_queue *wq, **wql; + + mutex_lock(&sbi->wq_mutex); + for (wql = &sbi->queues; (wq = *wql) != NULL; wql = &wq->next) { + if (wq->wait_queue_token == wait_queue_token) + break; + } + + if (!wq) { + mutex_unlock(&sbi->wq_mutex); + return -EINVAL; + } + + *wql = wq->next; /* Unlink from chain */ + kfree(wq->name.name - wq->offset); + wq->name.name = NULL; /* Do not wait on this queue */ + wq->status = status; + wake_up(&wq->queue); + if (!--wq->wait_ctr) + kfree(wq); + mutex_unlock(&sbi->wq_mutex); + + return 0; +} diff --git a/fs/backing-file.c b/fs/backing-file.c new file mode 100644 index 000000000..cc101143f --- /dev/null +++ b/fs/backing-file.c @@ -0,0 +1,366 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Common helpers for stackable filesystems and backing files. + * + * Forked from fs/overlayfs/file.c. + * + * Copyright (C) 2017 Red Hat, Inc. + * Copyright (C) 2023 CTERA Networks. + */ + +#include +#include +#include +#include +#include + +#include "internal.h" + +/** + * backing_file_open - open a backing file for kernel internal use + * @user_file: file the user requested to open + * @flags: open flags + * @real_path: path of the backing file + * @cred: credentials for open + * + * Open a backing file for a stackable filesystem (e.g., overlayfs). + * @user_file->f_path may be on the stackable filesystem and @real_path + * on the underlying filesystem. In this case, we want to be able to + * return the path of the stackable filesystem. This is done by + * embedding the returned file into a container structure that also + * stores the stacked file's path, which can be retrieved using + * backing_file_user_path(). + */ +struct file *backing_file_open(const struct file *user_file, int flags, + const struct path *real_path, + const struct cred *cred) +{ + const struct path *user_path = file_user_path(user_file); + struct file *f; + int error; + + f = alloc_empty_backing_file(flags, cred, user_file); + if (IS_ERR(f)) + return f; + + path_get(user_path); + backing_file_set_user_path(f, user_path); + error = vfs_open(real_path, f); + if (error) { + fput(f); + f = ERR_PTR(error); + } + + return f; +} +EXPORT_SYMBOL_GPL(backing_file_open); + +struct file *backing_tmpfile_open(const struct file *user_file, int flags, + const struct path *real_parentpath, + umode_t mode, const struct cred *cred) +{ + struct mnt_idmap *real_idmap = mnt_idmap(real_parentpath->mnt); + const struct path *user_path = &user_file->f_path; + struct file *f; + int error; + + f = alloc_empty_backing_file(flags, cred, user_file); + if (IS_ERR(f)) + return f; + + path_get(user_path); + backing_file_set_user_path(f, user_path); + error = vfs_tmpfile(real_idmap, real_parentpath, f, mode); + if (error) { + fput(f); + f = ERR_PTR(error); + } + return f; +} +EXPORT_SYMBOL(backing_tmpfile_open); + +struct backing_aio { + struct kiocb iocb; + refcount_t ref; + struct kiocb *orig_iocb; + /* used for aio completion */ + void (*end_write)(struct kiocb *iocb, ssize_t); + struct work_struct work; + long res; +}; + +static struct kmem_cache *backing_aio_cachep; + +#define BACKING_IOCB_MASK \ + (IOCB_NOWAIT | IOCB_HIPRI | IOCB_DSYNC | IOCB_SYNC | IOCB_APPEND) + +static rwf_t iocb_to_rw_flags(int flags) +{ + return (__force rwf_t)(flags & BACKING_IOCB_MASK); +} + +static void backing_aio_put(struct backing_aio *aio) +{ + if (refcount_dec_and_test(&aio->ref)) { + fput(aio->iocb.ki_filp); + kmem_cache_free(backing_aio_cachep, aio); + } +} + +static void backing_aio_cleanup(struct backing_aio *aio, long res) +{ + struct kiocb *iocb = &aio->iocb; + struct kiocb *orig_iocb = aio->orig_iocb; + + orig_iocb->ki_pos = iocb->ki_pos; + if (aio->end_write) + aio->end_write(orig_iocb, res); + + backing_aio_put(aio); +} + +static void backing_aio_rw_complete(struct kiocb *iocb, long res) +{ + struct backing_aio *aio = container_of(iocb, struct backing_aio, iocb); + struct kiocb *orig_iocb = aio->orig_iocb; + + if (iocb->ki_flags & IOCB_WRITE) + kiocb_end_write(iocb); + + backing_aio_cleanup(aio, res); + orig_iocb->ki_complete(orig_iocb, res); +} + +static void backing_aio_complete_work(struct work_struct *work) +{ + struct backing_aio *aio = container_of(work, struct backing_aio, work); + + backing_aio_rw_complete(&aio->iocb, aio->res); +} + +static void backing_aio_queue_completion(struct kiocb *iocb, long res) +{ + struct backing_aio *aio = container_of(iocb, struct backing_aio, iocb); + + /* + * Punt to a work queue to serialize updates of mtime/size. + */ + aio->res = res; + INIT_WORK(&aio->work, backing_aio_complete_work); + queue_work(file_inode(aio->orig_iocb->ki_filp)->i_sb->s_dio_done_wq, + &aio->work); +} + +static int backing_aio_init_wq(struct kiocb *iocb) +{ + struct super_block *sb = file_inode(iocb->ki_filp)->i_sb; + + if (sb->s_dio_done_wq) + return 0; + + return sb_init_dio_done_wq(sb); +} + +static int do_backing_file_read_iter(struct file *file, struct iov_iter *iter, + struct kiocb *iocb, int flags) +{ + struct backing_aio *aio = NULL; + int ret; + + if (is_sync_kiocb(iocb)) { + rwf_t rwf = iocb_to_rw_flags(flags); + + return vfs_iter_read(file, iter, &iocb->ki_pos, rwf); + } + + aio = kmem_cache_zalloc(backing_aio_cachep, GFP_KERNEL); + if (!aio) + return -ENOMEM; + + aio->orig_iocb = iocb; + kiocb_clone(&aio->iocb, iocb, get_file(file)); + aio->iocb.ki_complete = backing_aio_rw_complete; + refcount_set(&aio->ref, 2); + ret = vfs_iocb_iter_read(file, &aio->iocb, iter); + backing_aio_put(aio); + if (ret != -EIOCBQUEUED) + backing_aio_cleanup(aio, ret); + return ret; +} + +ssize_t backing_file_read_iter(struct file *file, struct iov_iter *iter, + struct kiocb *iocb, int flags, + struct backing_file_ctx *ctx) +{ + ssize_t ret; + + if (WARN_ON_ONCE(!(file->f_mode & FMODE_BACKING))) + return -EIO; + + if (!iov_iter_count(iter)) + return 0; + + if (iocb->ki_flags & IOCB_DIRECT && + !(file->f_mode & FMODE_CAN_ODIRECT)) + return -EINVAL; + + scoped_with_creds(ctx->cred) + ret = do_backing_file_read_iter(file, iter, iocb, flags); + + if (ctx->accessed) + ctx->accessed(iocb->ki_filp); + + return ret; +} +EXPORT_SYMBOL_GPL(backing_file_read_iter); + +static int do_backing_file_write_iter(struct file *file, struct iov_iter *iter, + struct kiocb *iocb, int flags, + void (*end_write)(struct kiocb *, ssize_t)) +{ + struct backing_aio *aio; + int ret; + + if (is_sync_kiocb(iocb)) { + rwf_t rwf = iocb_to_rw_flags(flags); + + ret = vfs_iter_write(file, iter, &iocb->ki_pos, rwf); + if (end_write) + end_write(iocb, ret); + return ret; + } + + ret = backing_aio_init_wq(iocb); + if (ret) + return ret; + + aio = kmem_cache_zalloc(backing_aio_cachep, GFP_KERNEL); + if (!aio) + return -ENOMEM; + + aio->orig_iocb = iocb; + aio->end_write = end_write; + kiocb_clone(&aio->iocb, iocb, get_file(file)); + aio->iocb.ki_flags = flags; + aio->iocb.ki_complete = backing_aio_queue_completion; + refcount_set(&aio->ref, 2); + ret = vfs_iocb_iter_write(file, &aio->iocb, iter); + backing_aio_put(aio); + if (ret != -EIOCBQUEUED) + backing_aio_cleanup(aio, ret); + return ret; +} + +ssize_t backing_file_write_iter(struct file *file, struct iov_iter *iter, + struct kiocb *iocb, int flags, + struct backing_file_ctx *ctx) +{ + ssize_t ret; + + if (WARN_ON_ONCE(!(file->f_mode & FMODE_BACKING))) + return -EIO; + + if (!iov_iter_count(iter)) + return 0; + + ret = file_remove_privs(iocb->ki_filp); + if (ret) + return ret; + + if (iocb->ki_flags & IOCB_DIRECT && + !(file->f_mode & FMODE_CAN_ODIRECT)) + return -EINVAL; + + scoped_with_creds(ctx->cred) + return do_backing_file_write_iter(file, iter, iocb, flags, ctx->end_write); +} +EXPORT_SYMBOL_GPL(backing_file_write_iter); + +ssize_t backing_file_splice_read(struct file *in, struct kiocb *iocb, + struct pipe_inode_info *pipe, size_t len, + unsigned int flags, + struct backing_file_ctx *ctx) +{ + ssize_t ret; + + if (WARN_ON_ONCE(!(in->f_mode & FMODE_BACKING))) + return -EIO; + + scoped_with_creds(ctx->cred) + ret = vfs_splice_read(in, &iocb->ki_pos, pipe, len, flags); + + if (ctx->accessed) + ctx->accessed(iocb->ki_filp); + + return ret; +} +EXPORT_SYMBOL_GPL(backing_file_splice_read); + +ssize_t backing_file_splice_write(struct pipe_inode_info *pipe, + struct file *out, struct kiocb *iocb, + size_t len, unsigned int flags, + struct backing_file_ctx *ctx) +{ + ssize_t ret; + + if (WARN_ON_ONCE(!(out->f_mode & FMODE_BACKING))) + return -EIO; + + if (!out->f_op->splice_write) + return -EINVAL; + + ret = file_remove_privs(iocb->ki_filp); + if (ret) + return ret; + + scoped_with_creds(ctx->cred) { + file_start_write(out); + ret = out->f_op->splice_write(pipe, out, &iocb->ki_pos, len, flags); + file_end_write(out); + } + + if (ctx->end_write) + ctx->end_write(iocb, ret); + + return ret; +} +EXPORT_SYMBOL_GPL(backing_file_splice_write); + +int backing_file_mmap(struct file *file, struct vm_area_struct *vma, + struct backing_file_ctx *ctx) +{ + struct file *user_file = vma->vm_file; + int ret; + + if (WARN_ON_ONCE(!(file->f_mode & FMODE_BACKING))) + return -EIO; + + if (!can_mmap_file(file)) + return -ENODEV; + + vma_set_file(vma, file); + + scoped_with_creds(ctx->cred) { + ret = security_mmap_backing_file(vma, file, user_file); + if (ret) + return ret; + + ret = vfs_mmap(vma->vm_file, vma); + } + + if (ctx->accessed) + ctx->accessed(user_file); + + return ret; +} +EXPORT_SYMBOL_GPL(backing_file_mmap); + +static int __init backing_aio_init(void) +{ + backing_aio_cachep = KMEM_CACHE(backing_aio, SLAB_HWCACHE_ALIGN); + if (!backing_aio_cachep) + return -ENOMEM; + + return 0; +} +fs_initcall(backing_aio_init); diff --git a/fs/bad_inode.c b/fs/bad_inode.c new file mode 100644 index 000000000..486c40f73 --- /dev/null +++ b/fs/bad_inode.c @@ -0,0 +1,251 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/bad_inode.c + * + * Copyright (C) 1997, Stephen Tweedie + * + * Provide stub functions for unreadable inodes + * + * Fabian Frederick : August 2003 - All file operations assigned to EIO + */ + +#include +#include +#include +#include +#include +#include +#include + +static int bad_file_open(struct inode *inode, struct file *filp) +{ + return -EIO; +} + +static const struct file_operations bad_file_ops = +{ + .open = bad_file_open, +}; + +static int bad_inode_create(struct mnt_idmap *idmap, + struct inode *dir, struct dentry *dentry, + umode_t mode) +{ + return -EIO; +} + +static struct dentry *bad_inode_lookup(struct inode *dir, + struct dentry *dentry, unsigned int flags) +{ + return ERR_PTR(-EIO); +} + +static int bad_inode_link (struct dentry *old_dentry, struct inode *dir, + struct dentry *dentry) +{ + return -EIO; +} + +static int bad_inode_unlink(struct inode *dir, struct dentry *dentry) +{ + return -EIO; +} + +static int bad_inode_symlink(struct mnt_idmap *idmap, + struct inode *dir, struct dentry *dentry, + const char *symname) +{ + return -EIO; +} + +static struct dentry *bad_inode_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + return ERR_PTR(-EIO); +} + +static int bad_inode_rmdir (struct inode *dir, struct dentry *dentry) +{ + return -EIO; +} + +static int bad_inode_mknod(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) +{ + return -EIO; +} + +static int bad_inode_rename2(struct mnt_idmap *idmap, + struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry, + unsigned int flags) +{ + return -EIO; +} + +static int bad_inode_readlink(struct dentry *dentry, char __user *buffer, + int buflen) +{ + return -EIO; +} + +static int bad_inode_permission(struct mnt_idmap *idmap, + struct inode *inode, int mask) +{ + return -EIO; +} + +static int bad_inode_getattr(struct mnt_idmap *idmap, + const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int query_flags) +{ + return -EIO; +} + +static int bad_inode_setattr(struct mnt_idmap *idmap, + struct dentry *direntry, struct iattr *attrs) +{ + return -EIO; +} + +static ssize_t bad_inode_listxattr(struct dentry *dentry, char *buffer, + size_t buffer_size) +{ + return -EIO; +} + +static const char *bad_inode_get_link(struct dentry *dentry, + struct inode *inode, + struct delayed_call *done) +{ + return ERR_PTR(-EIO); +} + +static struct posix_acl *bad_inode_get_acl(struct inode *inode, int type, bool rcu) +{ + return ERR_PTR(-EIO); +} + +static int bad_inode_fiemap(struct inode *inode, + struct fiemap_extent_info *fieinfo, u64 start, + u64 len) +{ + return -EIO; +} + +static int bad_inode_update_time(struct inode *inode, enum fs_update_time type, + unsigned int flags) +{ + return -EIO; +} + +static int bad_inode_atomic_open(struct inode *inode, struct dentry *dentry, + struct file *file, unsigned int open_flag, + umode_t create_mode) +{ + return -EIO; +} + +static int bad_inode_tmpfile(struct mnt_idmap *idmap, + struct inode *inode, struct file *file, + umode_t mode) +{ + return -EIO; +} + +static int bad_inode_set_acl(struct mnt_idmap *idmap, + struct dentry *dentry, struct posix_acl *acl, + int type) +{ + return -EIO; +} + +static const struct inode_operations bad_inode_ops = +{ + .create = bad_inode_create, + .lookup = bad_inode_lookup, + .link = bad_inode_link, + .unlink = bad_inode_unlink, + .symlink = bad_inode_symlink, + .mkdir = bad_inode_mkdir, + .rmdir = bad_inode_rmdir, + .mknod = bad_inode_mknod, + .rename = bad_inode_rename2, + .readlink = bad_inode_readlink, + .permission = bad_inode_permission, + .getattr = bad_inode_getattr, + .setattr = bad_inode_setattr, + .listxattr = bad_inode_listxattr, + .get_link = bad_inode_get_link, + .get_inode_acl = bad_inode_get_acl, + .fiemap = bad_inode_fiemap, + .update_time = bad_inode_update_time, + .atomic_open = bad_inode_atomic_open, + .tmpfile = bad_inode_tmpfile, + .set_acl = bad_inode_set_acl, +}; + + +/* + * When a filesystem is unable to read an inode due to an I/O error in + * its read_inode() function, it can call make_bad_inode() to return a + * set of stubs which will return EIO errors as required. + * + * We only need to do limited initialisation: all other fields are + * preinitialised to zero automatically. + */ + +/** + * make_bad_inode - mark an inode bad due to an I/O error + * @inode: Inode to mark bad + * + * When an inode cannot be read due to a media or remote network + * failure this function makes the inode "bad" and causes I/O operations + * on it to fail from this point on. + */ + +void make_bad_inode(struct inode *inode) +{ + remove_inode_hash(inode); + + inode->i_mode = S_IFREG; + simple_inode_init_ts(inode); + inode->i_op = &bad_inode_ops; + inode->i_opflags &= ~IOP_XATTR; + inode->i_fop = &bad_file_ops; +} +EXPORT_SYMBOL(make_bad_inode); + +/* + * This tests whether an inode has been flagged as bad. The test uses + * &bad_inode_ops to cover the case of invalidated inodes as well as + * those created by make_bad_inode() above. + */ + +/** + * is_bad_inode - is an inode errored + * @inode: inode to test + * + * Returns true if the inode in question has been marked as bad. + */ + +bool is_bad_inode(struct inode *inode) +{ + return (inode->i_op == &bad_inode_ops); +} + +EXPORT_SYMBOL(is_bad_inode); + +/** + * iget_failed - Mark an under-construction inode as dead and release it + * @inode: The inode to discard + * + * Mark an under-construction inode as dead and release it. + */ +void iget_failed(struct inode *inode) +{ + make_bad_inode(inode); + unlock_new_inode(inode); + iput(inode); +} +EXPORT_SYMBOL(iget_failed); diff --git a/fs/befs/ChangeLog b/fs/befs/ChangeLog new file mode 100644 index 000000000..aff7eec8f --- /dev/null +++ b/fs/befs/ChangeLog @@ -0,0 +1,417 @@ +Version 0.92 (2002-03-29) +========== +* Minor cleanup. Ran Lindent on the sources. + +Version 0.92 (2002-03-27) +========== +* Fixed module makefile problem. It was not compiling all the correct + source files! +* Removed duplicated function definition +* Fixed potential null pointer dereference when reporting an error + +Version 0.91 (2002-03-26) +========== +* Oy! Fixed stupid bug that would cause an unresolved symbol error. + Thanks to Laszlo Boszormenyi for pointing this out to me. + +Version 0.9 (2002-03-14) +========== +* Added Sergey S. Kostyliov's patch to eliminate memcpy() overhead + from b+tree operations. Changes the befs_read_datastream() interface. + +* Segregated the functions that interface directly with the linux vfs + interface into their own file called linuxvfs.c. [WD] + +Version 0.64 (2002-02-07) +========== +* Did the string comparison really right this time (btree.c) [WD] + +* Fixed up some places where I assumed that a long int could hold + a pointer value. (btree.c) [WD] + +* Andrew Farnham pointed out that the module + wouldn't work on older (<2.4.10) kernels due to an unresolved symbol. + This is bad, since 2.4.9 is still the current RedHat kernel. I added + a workaround for this problem (compatibility.h) [WD] + +* Sergey S. Kostyliov made befs_find_key() use a binary search to find + keys within btree nodes, rather than the linear search we were using + before. (btree.c) [Sergey S. Kostyliov ] + +* Made a debian package of the source for use with kernel-package. [WD] + + +Version 0.63 (2002-01-31) +========== +* Fixed bug in befs_find_brun_indirect() that would result in the wrong + block being read. It was introduced when adding byteswapping in + 0.61. (datastream.c) [WD] + +* Fixed a longstanding bug in befs_find_key() that would result in it + finding the first key that is a substring of the string it is searching + for. For example, this would cause files in the same directory with + names like file1 and file2 to mysteriously be duplicates of each other + (because they have the same inode number). Many thanks to Pavel Roskin + for reporting this serious bug!!! + (btree.c) [WD] + +* Added support for long symlinks, after Axel Dorfler explained up how + they work. I had forgotten all about them. (inode.c, symlink.c) [WD] + +* Documentation improvements in source. [WD] + +* Makefile fix for independent module when CONFIG_MODVERSION is set in + kernel config [Pavel Roskin ] + +* Compile warning fix for namei.c. [Sergey S. Kostyliov ] + + +Version 0.62 +========== +* Fixed makefile for module install [WD] + + +Version 0.61 (2002-01-20) +========== +* Made functions in endian.h to do the correct byteswapping, no matter + the arch. [WD] + +* Abbandoned silly checks for a NULL superblock pointer in debug.c. [WD] + +* Misc code cleanups. Also cleanup of this changelog file. [WD] + +* Added byteswapping to all metadata reads from disk. + Uses the functions from endian.h [WD] + +* Remove the typedef of struct super_block to vfs_sb, as it offended + certain peoples' aesthetic sense. [WD] + +* Ditto with the befs_read_block() interface. [WD] + + +Version 0.6 (2001-12-15) +========== +* Cleanup of NLS functions (util.c) [WD] + +* Make directory lookup/read use the NLS if an iocharset is provided. [WD] + +* Fixed stupid bug where specifying the uid or gid mount options as '0' + would result in the filesystem using the on-disk uid and gid. [WD] + +* Added mount option to control debug printing. + The option is, simply enough, 'debug'. + (super.c, debug.c) [WD] + +* Removed notion of btree handle from btree.c. It was unnecessary, as the + linux VFS doesn't allow us to keep any state between calls. Updated + dir.c, namei.c befs_fs.h to account for it. [WD] + +* Improved handleing of overflow nodes when listing directories. + Now works for overflow nodes hanging off of nodes other than the root + node. This is the cleaner solution to Brent Miszalaski's problem. [WD] + +* Added new debug/warning/error print functions in debug.c. + More flexible. Will soon be controllable at mount time + (see TODO). [WD] + +* Rewrote datastream position lookups. + (datastream.c) [WD] + +* Moved the TODO list to its own file. + + +Version 0.50 (2001-11-13) +========== +* Added workaround for mis-understanding of the nature of the b+trees used + in directories. A cleaner solution will come after I've thought about it + for a while. Thanks to Brent Miszalaski for finding and reporting this bug. + (btree.c) [WD] + +* Minor cleanups + +* Added test for "impossible" condition of empty internal nodes in + seekleaf() in btree.c [WD] + +* Implemented the abstracted read_block() in io.c [WD] + +* Cleaned up the inode validation in inode.c [WD] + +* Anton Altaparmakov figured out (by asking Linus :) ) what was causing the + hanging disk io problem. It turns out you need to have the sync_pages + callback defined in your address_space_ops, even if it just uses the + default linux-supplied implementation. Fixed. Works now. + (file.c) [WD] + +* Anton Altaparmakov and Christoph Hellwig alerted me to the fact that + filesystem code should be using GFP_NOFS instead of GFP_KERNEL as the + priority parameter to kmalloc(). Fixed. + (datastream.c, btree.c super.c inode.c) [WD] + +* Anton also told me that the blocksize is not allowed to be larger than + the page size in linux, which is 4k i386. Oops. Added a test for + (blocksize > PAGE_SIZE), and refuse to mount in that case. What this + practically means is that 8k blocksize volumes won't work without a major + restructuring of the driver (or an alpha or other 64bit hardware). [WD] + +* Cleaned up the befs_count_blocks() function. Much smarter now. + And somewhat smaller too. [WD] + +* Made inode allocations use a slab cache + (super.c inode.c) [WD] + +* Moved the freeing of the private inode section from put_inode() to + clear_inode(). This fixes a potential free twice type bug. Put_inode() + can be called multiple times for each inode struct. [WD] + +* Converted all non vfs-callback functions to use befs_sb_info as the + superblock type, rather than struct super_block. This is for + portablity. [WD] + +* Fixed a couple of compile warnings due to use of malloc.h, when slab.h + is the new way. (inode.c, super.c) [WD] + +* Fixed erronous includes of linux/befs_fs_i.h and linux/befs_fs_sb.h + in inode.c [WD] + +Version 0.45 (2001-10-29) +========== +* Added functions to get the private superblock and inode structures from + their enclosing public structures. Switched all references to the + private portions to use them. (many files) [WD] + +* Made read_super and read_inode allocate the private portions of those + structures into the generic pointer fields of the public structures + with kmalloc(). put_super and put_inode free them. This allows us not + to have to touch the definitions of the public structures in + include/linux/fs.h. Also, befs_inode_info is huge (because of the + symlink string). (super.c, inode.c, befs_fs.h) [WD] + +* Fixed a thinko that was corrupting file reads after the first block_run + is done being read. (datastream.c) [WD] + +* Removed fsync() hooks, since a read-only filesystem doesn't need them. + [Christoph Hellwig]. + +* Fixed befs_readlink() (symlink.c) [Christoph Hellwig]. + +* Removed all the Read-Write stuff. I'll redo it when it is time to add + write support (various files) [WD]. + +* Removed prototypes for functions who's definitions have been removed + (befs_fs.h) [WD]. + + +Version 0.4 (2001-10-28) +========== +* Made it an option to use the old non-pagecache befs_file_read() for + testing purposes. (fs/Config.in) + +* Fixed unused variable warnings when compiling without debugging. + +* Fixed a bug where the inode and super_block didn't get their blockbits + fields set (inode.c and super.c). + +* Release patch version 11. AKA befs-driver version 0.4. + +* Thats right. New versioning scheme. + I've done some serious testing on it now (on my box anyhow), and it + seems stable and not outragously slow. Existing features are more-or-less + correct (see TODO list). But it isn't 1.0 yet. I think 0.4 gives me some + headroom before the big 1.0. + + +2001-10-26 +========== +* Fixed date format in this file. Was I smoking crack? + +* Removed old datastream code from file.c, since it is nolonger used. + +* Generic_read_file() is now used to read regular file data. + It doesn't chew up the buffer cache (it does page io instead), and seems + to be about as fast (even though it has to look up each file block + indivdualy). And it knows about doing readahead, which is a major plus. + So it does i/o in much larger chunks. It is the correct linux way. It + uses befs_get_block() by way of befs_readpage() to find the disk offsets + of blocks, which in turn calls befs_fpos2brun() in datastream.c to do + the hard work of finding the disk block number. + +* Changed method of checking for a dirty filesystem in befs_read_super + (super.c). Now we check to see if log_start and log_end differ. If so, + the journal needs to be replayed, and the filesystem cannot be mounted. + +* Fixed an extra instance of MOD_DEC_USE_COUNT in super.c + +* Fixed a problem with reading the superblock on devices with large sector + sizes (such as cdroms) on linux 2.4.10 and up. + +2001-10-24 +========== +* Fix nasty bug in converting block numbers to struct befs_inode_addr. + Subtle, because the old version was only sometimes wrong. + Probably responsible for lots of problems. (inode.c) + +* Fix bug with reading an empty directory. (btree.c and dir.c) + +* This one looks good. Release patch version 10 + +2001-10-23 +========== +* Added btree searching function. + +* Use befs_btree_find in befs_lookup (namei.c) + +* Additional comments in btree.c + +2001-10-22 +========== +* Added B+tree reading functions (in btree.c). + Made befs_readdir() use them them instead of the cruft in index.c. + +2001-09-11 +========== +* Converted befs_read_file() to use the new datastream code. + +* Finally updated the README file. + +* Added many comments. + +* Posted version 6 + +* Removed byte-order conversion code. + I have no intention of supporting it, and it was very ugly. + Flow control with #ifdef (ugh). Maybe I'll redo it once + native byteorder works 100%. + +2001-09-10 +========== +* Finished implementing read_datastream() + +* made befs_read_brun() more general + Supports an offset to start at and a max bytes to read + Added a wrapper function to give the old call + +2001-09-30 +========== +* Discovered that the datastream handleing code in file.c is quite deficient + in several respects. For one thing, it doesn't deal with indirect blocks + +* Rewrote datastream handleing. + +* Created io.c, for io related functions. + Previously, the befs_bread() funtions lived in file.c + Created the befs_read_brun() function. + + +2001-09-07 +========== +* Made a function to actually count the number of fs blocks used by a file. + And helper functions. + (fs/befs/inode.c) + +2001-09-05 +========== +* Fixed a misunderstanding of the inode fields. + This fixed the problmem with wrong file sizes from du and others. + The i_blocks field of the inode struct is not the number of blocks for the + inode, it is the number of blocks for the file. Also, i_blksize is not + necessarily the size of the inode, although in practice it works out. + Changed to blocksize of filesystem. + (fs/befs/inode.c) + +* Permanently removed code that had been provisionally ifdefed out of befs_fs.h + +* Since we don't support access time, make that field zero, instead of + copying m_time. + (fs/befs/inode.c) + +* Added sanity check for inode reading + Make sure inode we got was the one we asked for. + (fs/befs/inode.c) + +* Code cleanup + Local pointers to commonly used structures in inode.c. + Got rid of abominations befs_iaddr2inode() and befs_inode2ino(). + Replaced with single function iaddr2blockno(). + (fs/befs/super.c) (fs/befs/inode.c) + +2001-09-01 +========== +* Fixed the problem with statfs where it would always claim the disk was + half full, due to improper understanding of the statfs fields. + (fs/befs/super.c) + +* Posted verion 4 of the patch + +2001-09-01 +========== +* Changed the macros in befs_fs.h to inline functions. + More readable. Typesafe. Better + (include/linux/befs_fs.h) + +* Moved type definitions from befs_fs.h to a new file, befs_fs_types.h + Because befs_fs_i.h and befs_fs_sb.h were including befs_fs.h for the + typedefs, and they are inlcuded in , which has definitions + that I want the inline functions in befs_fs.h to be able to see. Nasty + circularity. + (include/linux/befs_fs.h) + +2001-08-30 +========== +* Cleaned up some wording. + +* Added additional consitency checks on mount + Check block_size agrees with block_shift + Check flags == BEFS_CLEAN + (fs/befs/super.c) + +* Tell the kernel to only mount befs read-only. + By setting the SB_RDONLY flag in befs_read_super(). + Not that it was possible to write before. But now the kernel won't even try. + (fs/befs/super.c) + +* Got rid of kernel warning on mount. + The kernel doesn't like it if you call set_blocksize() on a device when + you have some of its blocks open. Moved the second set_blocksize() to the + very end of befs_read_super(), after we are done with the disk superblock. + (fs/befs/super.c) + +* Fixed wrong number of args bug in befs_dump_inode + (fs/befs/debug.c) + +* Solved lots of type mismatches in kprint()s + (everwhere) + +2001-08-27 +========== +* Cleaned up the fs/Config.in entries a bit, now slightly more descriptive. + +* BeFS depends on NLS, so I made activating BeFS enable the NLS questions + (fs/nls/Config.in) + +* Added Configure.help entries for CONFIG_BEFS_FS and CONFIG_DEBUG_BEFS + (currently at fs/befs/Kconfig) + +2001-08-?? +========== +* Removed superblock locking calls in befs_read_super(). In 2.4, the VFS + hands us a super_block struct that is already locked. + +2001-08-13 +========== +* Will Dyson is now attempting to maintain this module + Makoto Kato is original author.Daniel Berlin + also did some work on it (fixing it up for the later 2.3.x kernels, IIRC). + +* Fixed compile errors on 2.4.1 kernel (WD) + Resolve rejected patches + Accommodate changed NLS interface (util.h) + Needed to include in most files + Makefile changes + fs/Config.in changes + +* Tried to niceify the code using the ext2 fs as a guide + Declare befs_fs_type using the DECLARE_FSTYPE_DEV() macro + +* Made it a configure option to turn on debugging (fs/Config.in) + +* Compiles on 2.4.7 diff --git a/fs/befs/Kconfig b/fs/befs/Kconfig new file mode 100644 index 000000000..5fcfc4024 --- /dev/null +++ b/fs/befs/Kconfig @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: GPL-2.0-only +config BEFS_FS + tristate "BeOS file system (BeFS) support (read only)" + depends on BLOCK + select BUFFER_HEAD + select NLS + help + The BeOS File System (BeFS) is the native file system of Be, Inc's + BeOS. Notable features include support for arbitrary attributes + on files and directories, and database-like indices on selected + attributes. (Also note that this driver doesn't make those features + available at this time). It is a 64 bit filesystem, so it supports + extremely large volumes and files. + + If you use this filesystem, you should also say Y to at least one + of the NLS (native language support) options below. + + If you don't know what this is about, say N. + + To compile this as a module, choose M here: the module will be + called befs. + +config BEFS_DEBUG + bool "Debug BeFS" + depends on BEFS_FS + help + If you say Y here, you can use the 'debug' mount option to enable + debugging output from the driver. diff --git a/fs/befs/Makefile b/fs/befs/Makefile new file mode 100644 index 000000000..6c9c3cbc5 --- /dev/null +++ b/fs/befs/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the linux BeOS filesystem routines. +# + +obj-$(CONFIG_BEFS_FS) += befs.o +ccflags-$(CONFIG_BEFS_DEBUG) += -DDEBUG +befs-objs := datastream.o btree.o super.o inode.o debug.o io.o linuxvfs.o diff --git a/fs/befs/befs.h b/fs/befs/befs.h new file mode 100644 index 000000000..7cd472456 --- /dev/null +++ b/fs/befs/befs.h @@ -0,0 +1,150 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * befs.h + * + * Copyright (C) 2001-2002 Will Dyson + * Copyright (C) 1999 Makoto Kato (m_kato@ga2.so-net.ne.jp) + */ + +#ifndef _LINUX_BEFS_H +#define _LINUX_BEFS_H + +#include "befs_fs_types.h" + +/* used in debug.c */ +#define BEFS_VERSION "0.9.3" + + +typedef u64 befs_blocknr_t; +/* + * BeFS in memory structures + */ + +struct befs_mount_options { + kgid_t gid; + kuid_t uid; + int use_gid; + int use_uid; + int debug; + char *iocharset; +}; + +struct befs_sb_info { + u32 magic1; + u32 block_size; + u32 block_shift; + int byte_order; + befs_off_t num_blocks; + befs_off_t used_blocks; + u32 inode_size; + u32 magic2; + + /* Allocation group information */ + u32 blocks_per_ag; + u32 ag_shift; + u32 num_ags; + + /* State of the superblock */ + u32 flags; + + /* Journal log entry */ + befs_block_run log_blocks; + befs_off_t log_start; + befs_off_t log_end; + + befs_inode_addr root_dir; + befs_inode_addr indices; + u32 magic3; + + struct befs_mount_options mount_opts; + struct nls_table *nls; +}; + +struct befs_inode_info { + u32 i_flags; + u32 i_type; + + befs_inode_addr i_inode_num; + befs_inode_addr i_parent; + befs_inode_addr i_attribute; + + union { + befs_data_stream ds; + char symlink[BEFS_SYMLINK_LEN]; + } i_data; + + struct inode vfs_inode; +}; + +enum befs_err { + BEFS_OK, + BEFS_ERR, + BEFS_BAD_INODE, + BEFS_BT_END, + BEFS_BT_EMPTY, + BEFS_BT_MATCH, + BEFS_BT_OVERFLOW, + BEFS_BT_NOT_FOUND +}; + + +/****************************/ +/* debug.c */ +__printf(2, 3) +void befs_error(const struct super_block *sb, const char *fmt, ...); +__printf(2, 3) +void befs_warning(const struct super_block *sb, const char *fmt, ...); +__printf(2, 3) +void befs_debug(const struct super_block *sb, const char *fmt, ...); + +void befs_dump_super_block(const struct super_block *sb, befs_super_block *); +void befs_dump_inode(const struct super_block *sb, befs_inode *); +void befs_dump_index_entry(const struct super_block *sb, befs_disk_btree_super *); +void befs_dump_index_node(const struct super_block *sb, befs_btree_nodehead *); +/****************************/ + + +/* Gets a pointer to the private portion of the super_block + * structure from the public part + */ +static inline struct befs_sb_info * +BEFS_SB(const struct super_block *super) +{ + return (struct befs_sb_info *) super->s_fs_info; +} + +static inline struct befs_inode_info * +BEFS_I(const struct inode *inode) +{ + return container_of(inode, struct befs_inode_info, vfs_inode); +} + +static inline befs_blocknr_t +iaddr2blockno(struct super_block *sb, const befs_inode_addr *iaddr) +{ + return ((iaddr->allocation_group << BEFS_SB(sb)->ag_shift) + + iaddr->start); +} + +static inline befs_inode_addr +blockno2iaddr(struct super_block *sb, befs_blocknr_t blockno) +{ + befs_inode_addr iaddr; + + iaddr.allocation_group = blockno >> BEFS_SB(sb)->ag_shift; + iaddr.start = + blockno - (iaddr.allocation_group << BEFS_SB(sb)->ag_shift); + iaddr.len = 1; + + return iaddr; +} + +static inline unsigned int +befs_iaddrs_per_block(struct super_block *sb) +{ + return BEFS_SB(sb)->block_size / sizeof(befs_disk_inode_addr); +} + +#include "endian.h" + +#endif /* _LINUX_BEFS_H */ diff --git a/fs/befs/befs_fs_types.h b/fs/befs/befs_fs_types.h new file mode 100644 index 000000000..8019fde81 --- /dev/null +++ b/fs/befs/befs_fs_types.h @@ -0,0 +1,256 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * fs/befs/befs_fs_types.h + * + * Copyright (C) 2001 Will Dyson (will@cs.earlham.edu) + * + * + * + * from linux/include/linux/befs_fs.h + * + * Copyright (C) 1999 Makoto Kato (m_kato@ga2.so-net.ne.jp) + * + */ + +#ifndef _LINUX_BEFS_FS_TYPES +#define _LINUX_BEFS_FS_TYPES + +#ifdef __KERNEL__ +#include +#endif /*__KERNEL__*/ + +#define PACKED __attribute__ ((__packed__)) + +/* + * Max name lengths of BFS + */ + +#define BEFS_NAME_LEN 255 + +#define BEFS_SYMLINK_LEN 144 +#define BEFS_NUM_DIRECT_BLOCKS 12 +#define B_OS_NAME_LENGTH 32 + +/* The datastream blocks mapped by the double-indirect + * block are always 4 fs blocks long. + * This eliminates the need for linear searches among + * the potentially huge number of indirect blocks + * + * Err. Should that be 4 fs blocks or 4k??? + * It matters on large blocksize volumes + */ +#define BEFS_DBLINDIR_BRUN_LEN 4 + +/* + * Flags of superblock + */ + +enum super_flags { + BEFS_BYTESEX_BE, + BEFS_BYTESEX_LE, + BEFS_CLEAN = 0x434c454e, + BEFS_DIRTY = 0x44495254, + BEFS_SUPER_MAGIC1 = 0x42465331, /* BFS1 */ + BEFS_SUPER_MAGIC2 = 0xdd121031, + BEFS_SUPER_MAGIC3 = 0x15b6830e, +}; + +#define BEFS_BYTEORDER_NATIVE 0x42494745 +#define BEFS_BYTEORDER_NATIVE_LE ((__force fs32)cpu_to_le32(BEFS_BYTEORDER_NATIVE)) +#define BEFS_BYTEORDER_NATIVE_BE ((__force fs32)cpu_to_be32(BEFS_BYTEORDER_NATIVE)) + +#define BEFS_SUPER_MAGIC BEFS_SUPER_MAGIC1 +#define BEFS_SUPER_MAGIC1_LE ((__force fs32)cpu_to_le32(BEFS_SUPER_MAGIC1)) +#define BEFS_SUPER_MAGIC1_BE ((__force fs32)cpu_to_be32(BEFS_SUPER_MAGIC1)) + +/* + * Flags of inode + */ + +#define BEFS_INODE_MAGIC1 0x3bbe0ad9 + +enum inode_flags { + BEFS_INODE_IN_USE = 0x00000001, + BEFS_ATTR_INODE = 0x00000004, + BEFS_INODE_LOGGED = 0x00000008, + BEFS_INODE_DELETED = 0x00000010, + BEFS_LONG_SYMLINK = 0x00000040, + BEFS_PERMANENT_FLAG = 0x0000ffff, + BEFS_INODE_NO_CREATE = 0x00010000, + BEFS_INODE_WAS_WRITTEN = 0x00020000, + BEFS_NO_TRANSACTION = 0x00040000, +}; +/* + * On-Disk datastructures of BeFS + */ + +typedef u64 __bitwise fs64; +typedef u32 __bitwise fs32; +typedef u16 __bitwise fs16; + +typedef u64 befs_off_t; +typedef fs64 befs_time_t; + +/* Block runs */ +typedef struct { + fs32 allocation_group; + fs16 start; + fs16 len; +} PACKED befs_disk_block_run; + +typedef struct { + u32 allocation_group; + u16 start; + u16 len; +} PACKED befs_block_run; + +typedef befs_disk_block_run befs_disk_inode_addr; +typedef befs_block_run befs_inode_addr; + +/* + * The Superblock Structure + */ +typedef struct { + char name[B_OS_NAME_LENGTH]; + fs32 magic1; + fs32 fs_byte_order; + + fs32 block_size; + fs32 block_shift; + + fs64 num_blocks; + fs64 used_blocks; + + fs32 inode_size; + + fs32 magic2; + fs32 blocks_per_ag; + fs32 ag_shift; + fs32 num_ags; + + fs32 flags; + + befs_disk_block_run log_blocks; + fs64 log_start; + fs64 log_end; + + fs32 magic3; + befs_disk_inode_addr root_dir; + befs_disk_inode_addr indices; + +} PACKED befs_super_block; + +/* + * Note: the indirect and dbl_indir block_runs may + * be longer than one block! + */ +typedef struct { + befs_disk_block_run direct[BEFS_NUM_DIRECT_BLOCKS]; + fs64 max_direct_range; + befs_disk_block_run indirect; + fs64 max_indirect_range; + befs_disk_block_run double_indirect; + fs64 max_double_indirect_range; + fs64 size; +} PACKED befs_disk_data_stream; + +typedef struct { + befs_block_run direct[BEFS_NUM_DIRECT_BLOCKS]; + befs_off_t max_direct_range; + befs_block_run indirect; + befs_off_t max_indirect_range; + befs_block_run double_indirect; + befs_off_t max_double_indirect_range; + befs_off_t size; +} PACKED befs_data_stream; + +/* Attribute */ +typedef struct { + fs32 type; + fs16 name_size; + fs16 data_size; + char name[1]; +} PACKED befs_small_data; + +/* Inode structure */ +typedef struct { + fs32 magic1; + befs_disk_inode_addr inode_num; + fs32 uid; + fs32 gid; + fs32 mode; + fs32 flags; + befs_time_t create_time; + befs_time_t last_modified_time; + befs_disk_inode_addr parent; + befs_disk_inode_addr attributes; + fs32 type; + + fs32 inode_size; + fs32 etc; /* not use */ + + union { + befs_disk_data_stream datastream; + char symlink[BEFS_SYMLINK_LEN]; + } data; + + fs32 pad[4]; /* not use */ + befs_small_data small_data[1]; +} PACKED befs_inode; + +/* + * B+tree superblock + */ + +#define BEFS_BTREE_MAGIC 0x69f6c2e8 + +enum btree_types { + BTREE_STRING_TYPE = 0, + BTREE_INT32_TYPE = 1, + BTREE_UINT32_TYPE = 2, + BTREE_INT64_TYPE = 3, + BTREE_UINT64_TYPE = 4, + BTREE_FLOAT_TYPE = 5, + BTREE_DOUBLE_TYPE = 6 +}; + +typedef struct { + fs32 magic; + fs32 node_size; + fs32 max_depth; + fs32 data_type; + fs64 root_node_ptr; + fs64 free_node_ptr; + fs64 max_size; +} PACKED befs_disk_btree_super; + +typedef struct { + u32 magic; + u32 node_size; + u32 max_depth; + u32 data_type; + befs_off_t root_node_ptr; + befs_off_t free_node_ptr; + befs_off_t max_size; +} PACKED befs_btree_super; + +/* + * Header structure of each btree node + */ +typedef struct { + fs64 left; + fs64 right; + fs64 overflow; + fs16 all_key_count; + fs16 all_key_length; +} PACKED befs_btree_nodehead; + +typedef struct { + befs_off_t left; + befs_off_t right; + befs_off_t overflow; + u16 all_key_count; + u16 all_key_length; +} PACKED befs_host_btree_nodehead; + +#endif /* _LINUX_BEFS_FS_TYPES */ diff --git a/fs/befs/btree.c b/fs/befs/btree.c new file mode 100644 index 000000000..aa24f1dac --- /dev/null +++ b/fs/befs/btree.c @@ -0,0 +1,783 @@ +/* + * linux/fs/befs/btree.c + * + * Copyright (C) 2001-2002 Will Dyson + * + * Licensed under the GNU GPL. See the file COPYING for details. + * + * 2002-02-05: Sergey S. Kostyliov added binary search within + * btree nodes. + * + * Many thanks to: + * + * Dominic Giampaolo, author of "Practical File System + * Design with the Be File System", for such a helpful book. + * + * Marcus J. Ranum, author of the b+tree package in + * comp.sources.misc volume 10. This code is not copied from that + * work, but it is partially based on it. + * + * Makoto Kato, author of the original BeFS for linux filesystem + * driver. + */ + +#include +#include +#include +#include +#include + +#include "befs.h" +#include "btree.h" +#include "datastream.h" + +/* + * The btree functions in this file are built on top of the + * datastream.c interface, which is in turn built on top of the + * io.c interface. + */ + +/* Befs B+tree structure: + * + * The first thing in the tree is the tree superblock. It tells you + * all kinds of useful things about the tree, like where the rootnode + * is located, and the size of the nodes (always 1024 with current version + * of BeOS). + * + * The rest of the tree consists of a series of nodes. Nodes contain a header + * (struct befs_btree_nodehead), the packed key data, an array of shorts + * containing the ending offsets for each of the keys, and an array of + * befs_off_t values. In interior nodes, the keys are the ending keys for + * the childnode they point to, and the values are offsets into the + * datastream containing the tree. + */ + +/* Note: + * + * The book states 2 confusing things about befs b+trees. First, + * it states that the overflow field of node headers is used by internal nodes + * to point to another node that "effectively continues this one". Here is what + * I believe that means. Each key in internal nodes points to another node that + * contains key values less than itself. Inspection reveals that the last key + * in the internal node is not the last key in the index. Keys that are + * greater than the last key in the internal node go into the overflow node. + * I imagine there is a performance reason for this. + * + * Second, it states that the header of a btree node is sufficient to + * distinguish internal nodes from leaf nodes. Without saying exactly how. + * After figuring out the first, it becomes obvious that internal nodes have + * overflow nodes and leafnodes do not. + */ + +/* + * Currently, this code is only good for directory B+trees. + * In order to be used for other BFS indexes, it needs to be extended to handle + * duplicate keys and non-string keytypes (int32, int64, float, double). + */ + +/* + * In memory structure of each btree node + */ +struct befs_btree_node { + befs_host_btree_nodehead head; /* head of node converted to cpu byteorder */ + struct buffer_head *bh; + befs_btree_nodehead *od_node; /* on disk node */ +}; + +/* local constants */ +static const befs_off_t BEFS_BT_INVAL = 0xffffffffffffffffULL; + +/* local functions */ +static int befs_btree_seekleaf(struct super_block *sb, const befs_data_stream *ds, + befs_btree_super * bt_super, + struct befs_btree_node *this_node, + befs_off_t * node_off); + +static int befs_bt_read_super(struct super_block *sb, const befs_data_stream *ds, + befs_btree_super * sup); + +static int befs_bt_read_node(struct super_block *sb, const befs_data_stream *ds, + struct befs_btree_node *node, + befs_off_t node_off); + +static int befs_leafnode(struct befs_btree_node *node); + +static fs16 *befs_bt_keylen_index(struct befs_btree_node *node); + +static fs64 *befs_bt_valarray(struct befs_btree_node *node); + +static char *befs_bt_keydata(struct befs_btree_node *node); + +static int befs_find_key(struct super_block *sb, + struct befs_btree_node *node, + const char *findkey, befs_off_t * value); + +static char *befs_bt_get_key(struct super_block *sb, + struct befs_btree_node *node, + int index, u16 * keylen); + +static int befs_compare_strings(const void *key1, int keylen1, + const void *key2, int keylen2); + +/** + * befs_bt_read_super() - read in btree superblock convert to cpu byteorder + * @sb: Filesystem superblock + * @ds: Datastream to read from + * @sup: Buffer in which to place the btree superblock + * + * Calls befs_read_datastream to read in the btree superblock and + * makes sure it is in cpu byteorder, byteswapping if necessary. + * Return: BEFS_OK on success and if *@sup contains the btree superblock in cpu + * byte order. Otherwise return BEFS_ERR on error. + */ +static int +befs_bt_read_super(struct super_block *sb, const befs_data_stream *ds, + befs_btree_super * sup) +{ + struct buffer_head *bh; + befs_disk_btree_super *od_sup; + + befs_debug(sb, "---> %s", __func__); + + bh = befs_read_datastream(sb, ds, 0, NULL); + + if (!bh) { + befs_error(sb, "Couldn't read index header."); + goto error; + } + od_sup = (befs_disk_btree_super *) bh->b_data; + befs_dump_index_entry(sb, od_sup); + + sup->magic = fs32_to_cpu(sb, od_sup->magic); + sup->node_size = fs32_to_cpu(sb, od_sup->node_size); + sup->max_depth = fs32_to_cpu(sb, od_sup->max_depth); + sup->data_type = fs32_to_cpu(sb, od_sup->data_type); + sup->root_node_ptr = fs64_to_cpu(sb, od_sup->root_node_ptr); + + brelse(bh); + if (sup->magic != BEFS_BTREE_MAGIC) { + befs_error(sb, "Index header has bad magic."); + goto error; + } + + befs_debug(sb, "<--- %s", __func__); + return BEFS_OK; + + error: + befs_debug(sb, "<--- %s ERROR", __func__); + return BEFS_ERR; +} + +/** + * befs_bt_read_node - read in btree node and convert to cpu byteorder + * @sb: Filesystem superblock + * @ds: Datastream to read from + * @node: Buffer in which to place the btree node + * @node_off: Starting offset (in bytes) of the node in @ds + * + * Calls befs_read_datastream to read in the indicated btree node and + * makes sure its header fields are in cpu byteorder, byteswapping if + * necessary. + * Note: node->bh must be NULL when this function is called the first time. + * Don't forget brelse(node->bh) after last call. + * + * On success, returns BEFS_OK and *@node contains the btree node that + * starts at @node_off, with the node->head fields in cpu byte order. + * + * On failure, BEFS_ERR is returned. + */ + +static int +befs_bt_read_node(struct super_block *sb, const befs_data_stream *ds, + struct befs_btree_node *node, befs_off_t node_off) +{ + uint off = 0; + + befs_debug(sb, "---> %s", __func__); + + if (node->bh) + brelse(node->bh); + + node->bh = befs_read_datastream(sb, ds, node_off, &off); + if (!node->bh) { + befs_error(sb, "%s failed to read " + "node at %llu", __func__, node_off); + befs_debug(sb, "<--- %s ERROR", __func__); + + return BEFS_ERR; + } + node->od_node = + (befs_btree_nodehead *) ((void *) node->bh->b_data + off); + + befs_dump_index_node(sb, node->od_node); + + node->head.left = fs64_to_cpu(sb, node->od_node->left); + node->head.right = fs64_to_cpu(sb, node->od_node->right); + node->head.overflow = fs64_to_cpu(sb, node->od_node->overflow); + node->head.all_key_count = + fs16_to_cpu(sb, node->od_node->all_key_count); + node->head.all_key_length = + fs16_to_cpu(sb, node->od_node->all_key_length); + + befs_debug(sb, "<--- %s", __func__); + return BEFS_OK; +} + +/** + * befs_btree_find - Find a key in a befs B+tree + * @sb: Filesystem superblock + * @ds: Datastream containing btree + * @key: Key string to lookup in btree + * @value: Value stored with @key + * + * On success, returns BEFS_OK and sets *@value to the value stored + * with @key (usually the disk block number of an inode). + * + * On failure, returns BEFS_ERR or BEFS_BT_NOT_FOUND. + * + * Algorithm: + * Read the superblock and rootnode of the b+tree. + * Drill down through the interior nodes using befs_find_key(). + * Once at the correct leaf node, use befs_find_key() again to get the + * actual value stored with the key. + */ +int +befs_btree_find(struct super_block *sb, const befs_data_stream *ds, + const char *key, befs_off_t * value) +{ + struct befs_btree_node *this_node; + befs_btree_super bt_super; + befs_off_t node_off; + int res; + + befs_debug(sb, "---> %s Key: %s", __func__, key); + + if (befs_bt_read_super(sb, ds, &bt_super) != BEFS_OK) { + befs_error(sb, + "befs_btree_find() failed to read index superblock"); + goto error; + } + + this_node = kmalloc_obj(struct befs_btree_node, GFP_NOFS); + if (!this_node) { + befs_error(sb, "befs_btree_find() failed to allocate %zu " + "bytes of memory", sizeof(struct befs_btree_node)); + goto error; + } + + this_node->bh = NULL; + + /* read in root node */ + node_off = bt_super.root_node_ptr; + if (befs_bt_read_node(sb, ds, this_node, node_off) != BEFS_OK) { + befs_error(sb, "befs_btree_find() failed to read " + "node at %llu", node_off); + goto error_alloc; + } + + while (!befs_leafnode(this_node)) { + res = befs_find_key(sb, this_node, key, &node_off); + /* if no key set, try the overflow node */ + if (res == BEFS_BT_OVERFLOW) + node_off = this_node->head.overflow; + if (befs_bt_read_node(sb, ds, this_node, node_off) != BEFS_OK) { + befs_error(sb, "befs_btree_find() failed to read " + "node at %llu", node_off); + goto error_alloc; + } + } + + /* at a leaf node now, check if it is correct */ + res = befs_find_key(sb, this_node, key, value); + + brelse(this_node->bh); + kfree(this_node); + + if (res != BEFS_BT_MATCH) { + befs_error(sb, "<--- %s Key %s not found", __func__, key); + befs_debug(sb, "<--- %s ERROR", __func__); + *value = 0; + return BEFS_BT_NOT_FOUND; + } + befs_debug(sb, "<--- %s Found key %s, value %llu", __func__, + key, *value); + return BEFS_OK; + + error_alloc: + kfree(this_node); + error: + *value = 0; + befs_debug(sb, "<--- %s ERROR", __func__); + return BEFS_ERR; +} + +/** + * befs_find_key - Search for a key within a node + * @sb: Filesystem superblock + * @node: Node to find the key within + * @findkey: Keystring to search for + * @value: If key is found, the value stored with the key is put here + * + * Finds exact match if one exists, and returns BEFS_BT_MATCH. + * If there is no match and node's value array is too small for key, return + * BEFS_BT_OVERFLOW. + * If no match and node should countain this key, return BEFS_BT_NOT_FOUND. + * + * Uses binary search instead of a linear. + */ +static int +befs_find_key(struct super_block *sb, struct befs_btree_node *node, + const char *findkey, befs_off_t * value) +{ + int first, last, mid; + int eq; + u16 keylen; + int findkey_len; + char *thiskey; + fs64 *valarray; + + befs_debug(sb, "---> %s %s", __func__, findkey); + + findkey_len = strlen(findkey); + + /* if node can not contain key, just skip this node */ + last = node->head.all_key_count - 1; + thiskey = befs_bt_get_key(sb, node, last, &keylen); + + eq = befs_compare_strings(thiskey, keylen, findkey, findkey_len); + if (eq < 0) { + befs_debug(sb, "<--- node can't contain %s", findkey); + return BEFS_BT_OVERFLOW; + } + + valarray = befs_bt_valarray(node); + + /* simple binary search */ + first = 0; + mid = 0; + while (last >= first) { + mid = (last + first) / 2; + befs_debug(sb, "first: %d, last: %d, mid: %d", first, last, + mid); + thiskey = befs_bt_get_key(sb, node, mid, &keylen); + eq = befs_compare_strings(thiskey, keylen, findkey, + findkey_len); + + if (eq == 0) { + befs_debug(sb, "<--- %s found %s at %d", + __func__, thiskey, mid); + + *value = fs64_to_cpu(sb, valarray[mid]); + return BEFS_BT_MATCH; + } + if (eq > 0) + last = mid - 1; + else + first = mid + 1; + } + + /* return an existing value so caller can arrive to a leaf node */ + if (eq < 0) + *value = fs64_to_cpu(sb, valarray[mid + 1]); + else + *value = fs64_to_cpu(sb, valarray[mid]); + befs_error(sb, "<--- %s %s not found", __func__, findkey); + befs_debug(sb, "<--- %s ERROR", __func__); + return BEFS_BT_NOT_FOUND; +} + +/** + * befs_btree_read - Traverse leafnodes of a btree + * @sb: Filesystem superblock + * @ds: Datastream containing btree + * @key_no: Key number (alphabetical order) of key to read + * @bufsize: Size of the buffer to return key in + * @keybuf: Pointer to a buffer to put the key in + * @keysize: Length of the returned key + * @value: Value stored with the returned key + * + * Here's how it works: Key_no is the index of the key/value pair to + * return in keybuf/value. + * Bufsize is the size of keybuf (BEFS_NAME_LEN+1 is a good size). Keysize is + * the number of characters in the key (just a convenience). + * + * Algorithm: + * Get the first leafnode of the tree. See if the requested key is in that + * node. If not, follow the node->right link to the next leafnode. Repeat + * until the (key_no)th key is found or the tree is out of keys. + */ +int +befs_btree_read(struct super_block *sb, const befs_data_stream *ds, + loff_t key_no, size_t bufsize, char *keybuf, size_t * keysize, + befs_off_t * value) +{ + struct befs_btree_node *this_node; + befs_btree_super bt_super; + befs_off_t node_off; + int cur_key; + fs64 *valarray; + char *keystart; + u16 keylen; + int res; + + uint key_sum = 0; + + befs_debug(sb, "---> %s", __func__); + + if (befs_bt_read_super(sb, ds, &bt_super) != BEFS_OK) { + befs_error(sb, + "befs_btree_read() failed to read index superblock"); + goto error; + } + + this_node = kmalloc_obj(struct befs_btree_node, GFP_NOFS); + if (this_node == NULL) { + befs_error(sb, "befs_btree_read() failed to allocate %zu " + "bytes of memory", sizeof(struct befs_btree_node)); + goto error; + } + + node_off = bt_super.root_node_ptr; + this_node->bh = NULL; + + /* seeks down to first leafnode, reads it into this_node */ + res = befs_btree_seekleaf(sb, ds, &bt_super, this_node, &node_off); + if (res == BEFS_BT_EMPTY) { + brelse(this_node->bh); + kfree(this_node); + *value = 0; + *keysize = 0; + befs_debug(sb, "<--- %s Tree is EMPTY", __func__); + return BEFS_BT_EMPTY; + } else if (res == BEFS_ERR) { + goto error_alloc; + } + + /* find the leaf node containing the key_no key */ + + while (key_sum + this_node->head.all_key_count <= key_no) { + + /* no more nodes to look in: key_no is too large */ + if (this_node->head.right == BEFS_BT_INVAL) { + *keysize = 0; + *value = 0; + befs_debug(sb, + "<--- %s END of keys at %llu", __func__, + (unsigned long long) + key_sum + this_node->head.all_key_count); + brelse(this_node->bh); + kfree(this_node); + return BEFS_BT_END; + } + + key_sum += this_node->head.all_key_count; + node_off = this_node->head.right; + + if (befs_bt_read_node(sb, ds, this_node, node_off) != BEFS_OK) { + befs_error(sb, "%s failed to read node at %llu", + __func__, (unsigned long long)node_off); + goto error_alloc; + } + } + + /* how many keys into this_node is key_no */ + cur_key = key_no - key_sum; + + /* get pointers to datastructures within the node body */ + valarray = befs_bt_valarray(this_node); + + keystart = befs_bt_get_key(sb, this_node, cur_key, &keylen); + + befs_debug(sb, "Read [%llu,%d]: keysize %d", + (long long unsigned int)node_off, (int)cur_key, + (int)keylen); + + if (bufsize < keylen + 1) { + befs_error(sb, "%s keybuf too small (%zu) " + "for key of size %d", __func__, bufsize, keylen); + brelse(this_node->bh); + goto error_alloc; + } + + strscpy(keybuf, keystart, keylen + 1); + *value = fs64_to_cpu(sb, valarray[cur_key]); + *keysize = keylen; + + befs_debug(sb, "Read [%llu,%d]: Key \"%.*s\", Value %llu", node_off, + cur_key, keylen, keybuf, *value); + + brelse(this_node->bh); + kfree(this_node); + + befs_debug(sb, "<--- %s", __func__); + + return BEFS_OK; + + error_alloc: + kfree(this_node); + + error: + *keysize = 0; + *value = 0; + befs_debug(sb, "<--- %s ERROR", __func__); + return BEFS_ERR; +} + +/** + * befs_btree_seekleaf - Find the first leafnode in the btree + * @sb: Filesystem superblock + * @ds: Datastream containing btree + * @bt_super: Pointer to the superblock of the btree + * @this_node: Buffer to return the leafnode in + * @node_off: Pointer to offset of current node within datastream. Modified + * by the function. + * + * Helper function for btree traverse. Moves the current position to the + * start of the first leaf node. + * + * Also checks for an empty tree. If there are no keys, returns BEFS_BT_EMPTY. + */ +static int +befs_btree_seekleaf(struct super_block *sb, const befs_data_stream *ds, + befs_btree_super *bt_super, + struct befs_btree_node *this_node, + befs_off_t * node_off) +{ + + befs_debug(sb, "---> %s", __func__); + + if (befs_bt_read_node(sb, ds, this_node, *node_off) != BEFS_OK) { + befs_error(sb, "%s failed to read " + "node at %llu", __func__, *node_off); + goto error; + } + befs_debug(sb, "Seekleaf to root node %llu", *node_off); + + if (this_node->head.all_key_count == 0 && befs_leafnode(this_node)) { + befs_debug(sb, "<--- %s Tree is EMPTY", __func__); + return BEFS_BT_EMPTY; + } + + while (!befs_leafnode(this_node)) { + + if (this_node->head.all_key_count == 0) { + befs_debug(sb, "%s encountered " + "an empty interior node: %llu. Using Overflow " + "node: %llu", __func__, *node_off, + this_node->head.overflow); + *node_off = this_node->head.overflow; + } else { + fs64 *valarray = befs_bt_valarray(this_node); + *node_off = fs64_to_cpu(sb, valarray[0]); + } + if (befs_bt_read_node(sb, ds, this_node, *node_off) != BEFS_OK) { + befs_error(sb, "%s failed to read " + "node at %llu", __func__, *node_off); + goto error; + } + + befs_debug(sb, "Seekleaf to child node %llu", *node_off); + } + befs_debug(sb, "Node %llu is a leaf node", *node_off); + + return BEFS_OK; + + error: + befs_debug(sb, "<--- %s ERROR", __func__); + return BEFS_ERR; +} + +/** + * befs_leafnode - Determine if the btree node is a leaf node or an + * interior node + * @node: Pointer to node structure to test + * + * Return 1 if leaf, 0 if interior + */ +static int +befs_leafnode(struct befs_btree_node *node) +{ + /* all interior nodes (and only interior nodes) have an overflow node */ + if (node->head.overflow == BEFS_BT_INVAL) + return 1; + else + return 0; +} + +/** + * befs_bt_keylen_index - Finds start of keylen index in a node + * @node: Pointer to the node structure to find the keylen index within + * + * Returns a pointer to the start of the key length index array + * of the B+tree node *@node + * + * "The length of all the keys in the node is added to the size of the + * header and then rounded up to a multiple of four to get the beginning + * of the key length index" (p.88, practical filesystem design). + * + * Except that rounding up to 8 works, and rounding up to 4 doesn't. + */ +static fs16 * +befs_bt_keylen_index(struct befs_btree_node *node) +{ + const int keylen_align = 8; + unsigned long int off = + (sizeof (befs_btree_nodehead) + node->head.all_key_length); + ulong tmp = off % keylen_align; + + if (tmp) + off += keylen_align - tmp; + + return (fs16 *) ((void *) node->od_node + off); +} + +/** + * befs_bt_valarray - Finds the start of value array in a node + * @node: Pointer to the node structure to find the value array within + * + * Returns a pointer to the start of the value array + * of the node pointed to by the node header + */ +static fs64 * +befs_bt_valarray(struct befs_btree_node *node) +{ + void *keylen_index_start = (void *) befs_bt_keylen_index(node); + size_t keylen_index_size = node->head.all_key_count * sizeof (fs16); + + return (fs64 *) (keylen_index_start + keylen_index_size); +} + +/** + * befs_bt_keydata - Finds start of keydata array in a node + * @node: Pointer to the node structure to find the keydata array within + * + * Returns a pointer to the start of the keydata array + * of the node pointed to by the node header + */ +static char * +befs_bt_keydata(struct befs_btree_node *node) +{ + return (char *) ((void *) node->od_node + sizeof (befs_btree_nodehead)); +} + +/** + * befs_bt_get_key - returns a pointer to the start of a key + * @sb: filesystem superblock + * @node: node in which to look for the key + * @index: the index of the key to get + * @keylen: modified to be the length of the key at @index + * + * Returns a valid pointer into @node on success. + * Returns NULL on failure (bad input) and sets *@keylen = 0 + */ +static char * +befs_bt_get_key(struct super_block *sb, struct befs_btree_node *node, + int index, u16 * keylen) +{ + int prev_key_end; + char *keystart; + fs16 *keylen_index; + + if (index < 0 || index > node->head.all_key_count) { + *keylen = 0; + return NULL; + } + + keystart = befs_bt_keydata(node); + keylen_index = befs_bt_keylen_index(node); + + if (index == 0) + prev_key_end = 0; + else + prev_key_end = fs16_to_cpu(sb, keylen_index[index - 1]); + + *keylen = fs16_to_cpu(sb, keylen_index[index]) - prev_key_end; + + return keystart + prev_key_end; +} + +/** + * befs_compare_strings - compare two strings + * @key1: pointer to the first key to be compared + * @keylen1: length in bytes of key1 + * @key2: pointer to the second key to be compared + * @keylen2: length in bytes of key2 + * + * Returns 0 if @key1 and @key2 are equal. + * Returns >0 if @key1 is greater. + * Returns <0 if @key2 is greater. + */ +static int +befs_compare_strings(const void *key1, int keylen1, + const void *key2, int keylen2) +{ + int len = min_t(int, keylen1, keylen2); + int result = strncmp(key1, key2, len); + if (result == 0) + result = keylen1 - keylen2; + return result; +} + +/* These will be used for non-string keyed btrees */ +#if 0 +static int +btree_compare_int32(cont void *key1, int keylen1, const void *key2, int keylen2) +{ + return *(int32_t *) key1 - *(int32_t *) key2; +} + +static int +btree_compare_uint32(cont void *key1, int keylen1, + const void *key2, int keylen2) +{ + if (*(u_int32_t *) key1 == *(u_int32_t *) key2) + return 0; + else if (*(u_int32_t *) key1 > *(u_int32_t *) key2) + return 1; + + return -1; +} +static int +btree_compare_int64(cont void *key1, int keylen1, const void *key2, int keylen2) +{ + if (*(int64_t *) key1 == *(int64_t *) key2) + return 0; + else if (*(int64_t *) key1 > *(int64_t *) key2) + return 1; + + return -1; +} + +static int +btree_compare_uint64(cont void *key1, int keylen1, + const void *key2, int keylen2) +{ + if (*(u_int64_t *) key1 == *(u_int64_t *) key2) + return 0; + else if (*(u_int64_t *) key1 > *(u_int64_t *) key2) + return 1; + + return -1; +} + +static int +btree_compare_float(cont void *key1, int keylen1, const void *key2, int keylen2) +{ + float result = *(float *) key1 - *(float *) key2; + if (result == 0.0f) + return 0; + + return (result < 0.0f) ? -1 : 1; +} + +static int +btree_compare_double(cont void *key1, int keylen1, + const void *key2, int keylen2) +{ + double result = *(double *) key1 - *(double *) key2; + if (result == 0.0) + return 0; + + return (result < 0.0) ? -1 : 1; +} +#endif //0 diff --git a/fs/befs/btree.h b/fs/befs/btree.h new file mode 100644 index 000000000..a253a6276 --- /dev/null +++ b/fs/befs/btree.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * btree.h + * + */ + +int befs_btree_find(struct super_block *sb, const befs_data_stream *ds, + const char *key, befs_off_t *value); + +int befs_btree_read(struct super_block *sb, const befs_data_stream *ds, + loff_t key_no, size_t bufsize, char *keybuf, + size_t *keysize, befs_off_t *value); diff --git a/fs/befs/datastream.c b/fs/befs/datastream.c new file mode 100644 index 000000000..97719a7c7 --- /dev/null +++ b/fs/befs/datastream.c @@ -0,0 +1,525 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/befs/datastream.c + * + * Copyright (C) 2001 Will Dyson + * + * Based on portions of file.c by Makoto Kato + * + * Many thanks to Dominic Giampaolo, author of "Practical File System + * Design with the Be File System", for such a helpful book. + * + */ + +#include +#include +#include + +#include "befs.h" +#include "datastream.h" +#include "io.h" + +const befs_inode_addr BAD_IADDR = { 0, 0, 0 }; + +static int befs_find_brun_direct(struct super_block *sb, + const befs_data_stream *data, + befs_blocknr_t blockno, befs_block_run *run); + +static int befs_find_brun_indirect(struct super_block *sb, + const befs_data_stream *data, + befs_blocknr_t blockno, + befs_block_run *run); + +static int befs_find_brun_dblindirect(struct super_block *sb, + const befs_data_stream *data, + befs_blocknr_t blockno, + befs_block_run *run); + +/** + * befs_read_datastream - get buffer_head containing data, starting from pos. + * @sb: Filesystem superblock + * @ds: datastream to find data with + * @pos: start of data + * @off: offset of data in buffer_head->b_data + * + * Returns pointer to buffer_head containing data starting with offset @off, + * if you don't need to know offset just set @off = NULL. + */ +struct buffer_head * +befs_read_datastream(struct super_block *sb, const befs_data_stream *ds, + befs_off_t pos, uint *off) +{ + struct buffer_head *bh; + befs_block_run run; + befs_blocknr_t block; /* block coresponding to pos */ + + befs_debug(sb, "---> %s %llu", __func__, pos); + block = pos >> BEFS_SB(sb)->block_shift; + if (off) + *off = pos - (block << BEFS_SB(sb)->block_shift); + + if (befs_fblock2brun(sb, ds, block, &run) != BEFS_OK) { + befs_error(sb, "BeFS: Error finding disk addr of block %lu", + (unsigned long)block); + befs_debug(sb, "<--- %s ERROR", __func__); + return NULL; + } + bh = befs_bread_iaddr(sb, run); + if (!bh) { + befs_error(sb, "BeFS: Error reading block %lu from datastream", + (unsigned long)block); + return NULL; + } + + befs_debug(sb, "<--- %s read data, starting at %llu", __func__, pos); + + return bh; +} + +/** + * befs_fblock2brun - give back block run for fblock + * @sb: the superblock + * @data: datastream to read from + * @fblock: the blocknumber with the file position to find + * @run: The found run is passed back through this pointer + * + * Takes a file position and gives back a brun who's starting block + * is block number fblock of the file. + * + * Returns BEFS_OK or BEFS_ERR. + * + * Calls specialized functions for each of the three possible + * datastream regions. + */ +int +befs_fblock2brun(struct super_block *sb, const befs_data_stream *data, + befs_blocknr_t fblock, befs_block_run *run) +{ + int err; + befs_off_t pos = fblock << BEFS_SB(sb)->block_shift; + + if (pos < data->max_direct_range) { + err = befs_find_brun_direct(sb, data, fblock, run); + + } else if (pos < data->max_indirect_range) { + err = befs_find_brun_indirect(sb, data, fblock, run); + + } else if (pos < data->max_double_indirect_range) { + err = befs_find_brun_dblindirect(sb, data, fblock, run); + + } else { + befs_error(sb, + "befs_fblock2brun() was asked to find block %lu, " + "which is not mapped by the datastream\n", + (unsigned long)fblock); + err = BEFS_ERR; + } + return err; +} + +/** + * befs_read_lsmylink - read long symlink from datastream. + * @sb: Filesystem superblock + * @ds: Datastream to read from + * @buff: Buffer in which to place long symlink data + * @len: Length of the long symlink in bytes + * + * Returns the number of bytes read + */ +size_t +befs_read_lsymlink(struct super_block *sb, const befs_data_stream *ds, + void *buff, befs_off_t len) +{ + befs_off_t bytes_read = 0; /* bytes readed */ + u16 plen; + struct buffer_head *bh; + + befs_debug(sb, "---> %s length: %llu", __func__, len); + + while (bytes_read < len) { + bh = befs_read_datastream(sb, ds, bytes_read, NULL); + if (!bh) { + befs_error(sb, "BeFS: Error reading datastream block " + "starting from %llu", bytes_read); + befs_debug(sb, "<--- %s ERROR", __func__); + return bytes_read; + + } + plen = ((bytes_read + BEFS_SB(sb)->block_size) < len) ? + BEFS_SB(sb)->block_size : len - bytes_read; + memcpy(buff + bytes_read, bh->b_data, plen); + brelse(bh); + bytes_read += plen; + } + + befs_debug(sb, "<--- %s read %u bytes", __func__, (unsigned int) + bytes_read); + return bytes_read; +} + +/** + * befs_count_blocks - blocks used by a file + * @sb: Filesystem superblock + * @ds: Datastream of the file + * + * Counts the number of fs blocks that the file represented by + * inode occupies on the filesystem, counting both regular file + * data and filesystem metadata (and eventually attribute data + * when we support attributes) +*/ + +befs_blocknr_t +befs_count_blocks(struct super_block *sb, const befs_data_stream *ds) +{ + befs_blocknr_t blocks; + befs_blocknr_t datablocks; /* File data blocks */ + befs_blocknr_t metablocks; /* FS metadata blocks */ + struct befs_sb_info *befs_sb = BEFS_SB(sb); + + befs_debug(sb, "---> %s", __func__); + + datablocks = ds->size >> befs_sb->block_shift; + if (ds->size & (befs_sb->block_size - 1)) + datablocks += 1; + + metablocks = 1; /* Start with 1 block for inode */ + + /* Size of indirect block */ + if (ds->size > ds->max_direct_range) + metablocks += ds->indirect.len; + + /* + * Double indir block, plus all the indirect blocks it maps. + * In the double-indirect range, all block runs of data are + * BEFS_DBLINDIR_BRUN_LEN blocks long. Therefore, we know + * how many data block runs are in the double-indirect region, + * and from that we know how many indirect blocks it takes to + * map them. We assume that the indirect blocks are also + * BEFS_DBLINDIR_BRUN_LEN blocks long. + */ + if (ds->size > ds->max_indirect_range && ds->max_indirect_range != 0) { + uint dbl_bytes; + uint dbl_bruns; + uint indirblocks; + + dbl_bytes = + ds->max_double_indirect_range - ds->max_indirect_range; + dbl_bruns = + dbl_bytes / (befs_sb->block_size * BEFS_DBLINDIR_BRUN_LEN); + indirblocks = dbl_bruns / befs_iaddrs_per_block(sb); + + metablocks += ds->double_indirect.len; + metablocks += indirblocks; + } + + blocks = datablocks + metablocks; + befs_debug(sb, "<--- %s %u blocks", __func__, (unsigned int)blocks); + + return blocks; +} + +/** + * befs_find_brun_direct - find a direct block run in the datastream + * @sb: the superblock + * @data: the datastream + * @blockno: the blocknumber to find + * @run: The found run is passed back through this pointer + * + * Finds the block run that starts at file block number blockno + * in the file represented by the datastream data, if that + * blockno is in the direct region of the datastream. + * + * Return value is BEFS_OK if the blockrun is found, BEFS_ERR + * otherwise. + * + * Algorithm: + * Linear search. Checks each element of array[] to see if it + * contains the blockno-th filesystem block. This is necessary + * because the block runs map variable amounts of data. Simply + * keeps a count of the number of blocks searched so far (sum), + * incrementing this by the length of each block run as we come + * across it. Adds sum to *count before returning (this is so + * you can search multiple arrays that are logicaly one array, + * as in the indirect region code). + * + * When/if blockno is found, if blockno is inside of a block + * run as stored on disk, we offset the start and length members + * of the block run, so that blockno is the start and len is + * still valid (the run ends in the same place). + */ +static int +befs_find_brun_direct(struct super_block *sb, const befs_data_stream *data, + befs_blocknr_t blockno, befs_block_run *run) +{ + int i; + const befs_block_run *array = data->direct; + befs_blocknr_t sum; + + befs_debug(sb, "---> %s, find %lu", __func__, (unsigned long)blockno); + + for (i = 0, sum = 0; i < BEFS_NUM_DIRECT_BLOCKS; + sum += array[i].len, i++) { + if (blockno >= sum && blockno < sum + (array[i].len)) { + int offset = blockno - sum; + + run->allocation_group = array[i].allocation_group; + run->start = array[i].start + offset; + run->len = array[i].len - offset; + + befs_debug(sb, "---> %s, " + "found %lu at direct[%d]", __func__, + (unsigned long)blockno, i); + return BEFS_OK; + } + } + + befs_error(sb, "%s failed to find file block %lu", __func__, + (unsigned long)blockno); + befs_debug(sb, "---> %s ERROR", __func__); + return BEFS_ERR; +} + +/** + * befs_find_brun_indirect - find a block run in the datastream + * @sb: the superblock + * @data: the datastream + * @blockno: the blocknumber to find + * @run: The found run is passed back through this pointer + * + * Finds the block run that starts at file block number blockno + * in the file represented by the datastream data, if that + * blockno is in the indirect region of the datastream. + * + * Return value is BEFS_OK if the blockrun is found, BEFS_ERR + * otherwise. + * + * Algorithm: + * For each block in the indirect run of the datastream, read + * it in and search through it for search_blk. + * + * XXX: + * Really should check to make sure blockno is inside indirect + * region. + */ +static int +befs_find_brun_indirect(struct super_block *sb, + const befs_data_stream *data, + befs_blocknr_t blockno, + befs_block_run *run) +{ + int i, j; + befs_blocknr_t sum = 0; + befs_blocknr_t indir_start_blk; + befs_blocknr_t search_blk; + struct buffer_head *indirblock; + befs_disk_block_run *array; + + befs_block_run indirect = data->indirect; + befs_blocknr_t indirblockno = iaddr2blockno(sb, &indirect); + int arraylen = befs_iaddrs_per_block(sb); + + befs_debug(sb, "---> %s, find %lu", __func__, (unsigned long)blockno); + + indir_start_blk = data->max_direct_range >> BEFS_SB(sb)->block_shift; + search_blk = blockno - indir_start_blk; + + /* Examine blocks of the indirect run one at a time */ + for (i = 0; i < indirect.len; i++) { + indirblock = sb_bread(sb, indirblockno + i); + if (indirblock == NULL) { + befs_error(sb, "---> %s failed to read " + "disk block %lu from the indirect brun", + __func__, (unsigned long)indirblockno + i); + befs_debug(sb, "<--- %s ERROR", __func__); + return BEFS_ERR; + } + + array = (befs_disk_block_run *) indirblock->b_data; + + for (j = 0; j < arraylen; ++j) { + int len = fs16_to_cpu(sb, array[j].len); + + if (search_blk >= sum && search_blk < sum + len) { + int offset = search_blk - sum; + run->allocation_group = + fs32_to_cpu(sb, array[j].allocation_group); + run->start = + fs16_to_cpu(sb, array[j].start) + offset; + run->len = + fs16_to_cpu(sb, array[j].len) - offset; + + brelse(indirblock); + befs_debug(sb, + "<--- %s found file block " + "%lu at indirect[%d]", __func__, + (unsigned long)blockno, + j + (i * arraylen)); + return BEFS_OK; + } + sum += len; + } + + brelse(indirblock); + } + + /* Only fallthrough is an error */ + befs_error(sb, "BeFS: %s failed to find " + "file block %lu", __func__, (unsigned long)blockno); + + befs_debug(sb, "<--- %s ERROR", __func__); + return BEFS_ERR; +} + +/** + * befs_find_brun_dblindirect - find a block run in the datastream + * @sb: the superblock + * @data: the datastream + * @blockno: the blocknumber to find + * @run: The found run is passed back through this pointer + * + * Finds the block run that starts at file block number blockno + * in the file represented by the datastream data, if that + * blockno is in the double-indirect region of the datastream. + * + * Return value is BEFS_OK if the blockrun is found, BEFS_ERR + * otherwise. + * + * Algorithm: + * The block runs in the double-indirect region are different. + * They are always allocated 4 fs blocks at a time, so each + * block run maps a constant amount of file data. This means + * that we can directly calculate how many block runs into the + * double-indirect region we need to go to get to the one that + * maps a particular filesystem block. + * + * We do this in two stages. First we calculate which of the + * inode addresses in the double-indirect block will point us + * to the indirect block that contains the mapping for the data, + * then we calculate which of the inode addresses in that + * indirect block maps the data block we are after. + * + * Oh, and once we've done that, we actually read in the blocks + * that contain the inode addresses we calculated above. Even + * though the double-indirect run may be several blocks long, + * we can calculate which of those blocks will contain the index + * we are after and only read that one. We then follow it to + * the indirect block and perform a similar process to find + * the actual block run that maps the data block we are interested + * in. + * + * Then we offset the run as in befs_find_brun_array() and we are + * done. + */ +static int +befs_find_brun_dblindirect(struct super_block *sb, + const befs_data_stream *data, + befs_blocknr_t blockno, + befs_block_run *run) +{ + int dblindir_indx; + int indir_indx; + int offset; + int dbl_which_block; + int which_block; + int dbl_block_indx; + int block_indx; + off_t dblindir_leftover; + befs_blocknr_t blockno_at_run_start; + struct buffer_head *dbl_indir_block; + struct buffer_head *indir_block; + befs_block_run indir_run; + befs_disk_inode_addr *iaddr_array; + + befs_blocknr_t indir_start_blk = + data->max_indirect_range >> BEFS_SB(sb)->block_shift; + + off_t dbl_indir_off = blockno - indir_start_blk; + + /* number of data blocks mapped by each of the iaddrs in + * the indirect block pointed to by the double indirect block + */ + size_t iblklen = BEFS_DBLINDIR_BRUN_LEN; + + /* number of data blocks mapped by each of the iaddrs in + * the double indirect block + */ + size_t diblklen = iblklen * befs_iaddrs_per_block(sb) + * BEFS_DBLINDIR_BRUN_LEN; + + befs_debug(sb, "---> %s find %lu", __func__, (unsigned long)blockno); + + /* First, discover which of the double_indir->indir blocks + * contains pos. Then figure out how much of pos that + * accounted for. Then discover which of the iaddrs in + * the indirect block contains pos. + */ + + dblindir_indx = dbl_indir_off / diblklen; + dblindir_leftover = dbl_indir_off % diblklen; + indir_indx = dblindir_leftover / diblklen; + + /* Read double indirect block */ + dbl_which_block = dblindir_indx / befs_iaddrs_per_block(sb); + if (dbl_which_block > data->double_indirect.len) { + befs_error(sb, "The double-indirect index calculated by " + "%s, %d, is outside the range " + "of the double-indirect block", __func__, + dblindir_indx); + return BEFS_ERR; + } + + dbl_indir_block = + sb_bread(sb, iaddr2blockno(sb, &data->double_indirect) + + dbl_which_block); + if (dbl_indir_block == NULL) { + befs_error(sb, "%s couldn't read the " + "double-indirect block at blockno %lu", __func__, + (unsigned long) + iaddr2blockno(sb, &data->double_indirect) + + dbl_which_block); + return BEFS_ERR; + } + + dbl_block_indx = + dblindir_indx - (dbl_which_block * befs_iaddrs_per_block(sb)); + iaddr_array = (befs_disk_inode_addr *) dbl_indir_block->b_data; + indir_run = fsrun_to_cpu(sb, iaddr_array[dbl_block_indx]); + brelse(dbl_indir_block); + + /* Read indirect block */ + which_block = indir_indx / befs_iaddrs_per_block(sb); + if (which_block > indir_run.len) { + befs_error(sb, "The indirect index calculated by " + "%s, %d, is outside the range " + "of the indirect block", __func__, indir_indx); + return BEFS_ERR; + } + + indir_block = + sb_bread(sb, iaddr2blockno(sb, &indir_run) + which_block); + if (indir_block == NULL) { + befs_error(sb, "%s couldn't read the indirect block " + "at blockno %lu", __func__, (unsigned long) + iaddr2blockno(sb, &indir_run) + which_block); + return BEFS_ERR; + } + + block_indx = indir_indx - (which_block * befs_iaddrs_per_block(sb)); + iaddr_array = (befs_disk_inode_addr *) indir_block->b_data; + *run = fsrun_to_cpu(sb, iaddr_array[block_indx]); + brelse(indir_block); + + blockno_at_run_start = indir_start_blk; + blockno_at_run_start += diblklen * dblindir_indx; + blockno_at_run_start += iblklen * indir_indx; + offset = blockno - blockno_at_run_start; + + run->start += offset; + run->len -= offset; + + befs_debug(sb, "Found file block %lu in double_indirect[%d][%d]," + " double_indirect_leftover = %lu", (unsigned long) + blockno, dblindir_indx, indir_indx, dblindir_leftover); + + return BEFS_OK; +} diff --git a/fs/befs/datastream.h b/fs/befs/datastream.h new file mode 100644 index 000000000..39b1d4766 --- /dev/null +++ b/fs/befs/datastream.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * datastream.h + * + */ + +struct buffer_head *befs_read_datastream(struct super_block *sb, + const befs_data_stream *ds, + befs_off_t pos, uint *off); + +int befs_fblock2brun(struct super_block *sb, const befs_data_stream *data, + befs_blocknr_t fblock, befs_block_run *run); + +size_t befs_read_lsymlink(struct super_block *sb, const befs_data_stream *data, + void *buff, befs_off_t len); + +befs_blocknr_t befs_count_blocks(struct super_block *sb, + const befs_data_stream *ds); + +extern const befs_inode_addr BAD_IADDR; diff --git a/fs/befs/debug.c b/fs/befs/debug.c new file mode 100644 index 000000000..02fa66fb8 --- /dev/null +++ b/fs/befs/debug.c @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/befs/debug.c + * + * Copyright (C) 2001 Will Dyson (will_dyson at pobox.com) + * + * With help from the ntfs-tng driver by Anton Altparmakov + * + * Copyright (C) 1999 Makoto Kato (m_kato@ga2.so-net.ne.jp) + * + * debug functions + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#ifdef __KERNEL__ + +#include +#include +#include +#include +#include +#include + +#endif /* __KERNEL__ */ + +#include "befs.h" + +void +befs_error(const struct super_block *sb, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + pr_err("(%s): %pV\n", sb->s_id, &vaf); + va_end(args); +} + +void +befs_warning(const struct super_block *sb, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + pr_warn("(%s): %pV\n", sb->s_id, &vaf); + va_end(args); +} + +void +befs_debug(const struct super_block *sb, const char *fmt, ...) +{ +#ifdef CONFIG_BEFS_DEBUG + + struct va_format vaf; + va_list args; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + pr_debug("(%s): %pV\n", sb->s_id, &vaf); + va_end(args); + +#endif //CONFIG_BEFS_DEBUG +} + +void +befs_dump_inode(const struct super_block *sb, befs_inode *inode) +{ +#ifdef CONFIG_BEFS_DEBUG + + befs_block_run tmp_run; + + befs_debug(sb, "befs_inode information"); + + befs_debug(sb, " magic1 %08x", fs32_to_cpu(sb, inode->magic1)); + + tmp_run = fsrun_to_cpu(sb, inode->inode_num); + befs_debug(sb, " inode_num %u, %hu, %hu", + tmp_run.allocation_group, tmp_run.start, tmp_run.len); + + befs_debug(sb, " uid %u", fs32_to_cpu(sb, inode->uid)); + befs_debug(sb, " gid %u", fs32_to_cpu(sb, inode->gid)); + befs_debug(sb, " mode %08x", fs32_to_cpu(sb, inode->mode)); + befs_debug(sb, " flags %08x", fs32_to_cpu(sb, inode->flags)); + befs_debug(sb, " create_time %llu", + fs64_to_cpu(sb, inode->create_time)); + befs_debug(sb, " last_modified_time %llu", + fs64_to_cpu(sb, inode->last_modified_time)); + + tmp_run = fsrun_to_cpu(sb, inode->parent); + befs_debug(sb, " parent [%u, %hu, %hu]", + tmp_run.allocation_group, tmp_run.start, tmp_run.len); + + tmp_run = fsrun_to_cpu(sb, inode->attributes); + befs_debug(sb, " attributes [%u, %hu, %hu]", + tmp_run.allocation_group, tmp_run.start, tmp_run.len); + + befs_debug(sb, " type %08x", fs32_to_cpu(sb, inode->type)); + befs_debug(sb, " inode_size %u", fs32_to_cpu(sb, inode->inode_size)); + + if (S_ISLNK(fs32_to_cpu(sb, inode->mode))) { + befs_debug(sb, " Symbolic link [%s]", inode->data.symlink); + } else { + int i; + + for (i = 0; i < BEFS_NUM_DIRECT_BLOCKS; i++) { + tmp_run = + fsrun_to_cpu(sb, inode->data.datastream.direct[i]); + befs_debug(sb, " direct %d [%u, %hu, %hu]", i, + tmp_run.allocation_group, tmp_run.start, + tmp_run.len); + } + befs_debug(sb, " max_direct_range %llu", + fs64_to_cpu(sb, + inode->data.datastream. + max_direct_range)); + + tmp_run = fsrun_to_cpu(sb, inode->data.datastream.indirect); + befs_debug(sb, " indirect [%u, %hu, %hu]", + tmp_run.allocation_group, + tmp_run.start, tmp_run.len); + + befs_debug(sb, " max_indirect_range %llu", + fs64_to_cpu(sb, + inode->data.datastream. + max_indirect_range)); + + tmp_run = + fsrun_to_cpu(sb, inode->data.datastream.double_indirect); + befs_debug(sb, " double indirect [%u, %hu, %hu]", + tmp_run.allocation_group, tmp_run.start, + tmp_run.len); + + befs_debug(sb, " max_double_indirect_range %llu", + fs64_to_cpu(sb, + inode->data.datastream. + max_double_indirect_range)); + + befs_debug(sb, " size %llu", + fs64_to_cpu(sb, inode->data.datastream.size)); + } + +#endif //CONFIG_BEFS_DEBUG +} + +/* + * Display super block structure for debug. + */ + +void +befs_dump_super_block(const struct super_block *sb, befs_super_block *sup) +{ +#ifdef CONFIG_BEFS_DEBUG + + befs_block_run tmp_run; + + befs_debug(sb, "befs_super_block information"); + + befs_debug(sb, " name %s", sup->name); + befs_debug(sb, " magic1 %08x", fs32_to_cpu(sb, sup->magic1)); + befs_debug(sb, " fs_byte_order %08x", + fs32_to_cpu(sb, sup->fs_byte_order)); + + befs_debug(sb, " block_size %u", fs32_to_cpu(sb, sup->block_size)); + befs_debug(sb, " block_shift %u", fs32_to_cpu(sb, sup->block_shift)); + + befs_debug(sb, " num_blocks %llu", fs64_to_cpu(sb, sup->num_blocks)); + befs_debug(sb, " used_blocks %llu", fs64_to_cpu(sb, sup->used_blocks)); + befs_debug(sb, " inode_size %u", fs32_to_cpu(sb, sup->inode_size)); + + befs_debug(sb, " magic2 %08x", fs32_to_cpu(sb, sup->magic2)); + befs_debug(sb, " blocks_per_ag %u", + fs32_to_cpu(sb, sup->blocks_per_ag)); + befs_debug(sb, " ag_shift %u", fs32_to_cpu(sb, sup->ag_shift)); + befs_debug(sb, " num_ags %u", fs32_to_cpu(sb, sup->num_ags)); + + befs_debug(sb, " flags %08x", fs32_to_cpu(sb, sup->flags)); + + tmp_run = fsrun_to_cpu(sb, sup->log_blocks); + befs_debug(sb, " log_blocks %u, %hu, %hu", + tmp_run.allocation_group, tmp_run.start, tmp_run.len); + + befs_debug(sb, " log_start %lld", fs64_to_cpu(sb, sup->log_start)); + befs_debug(sb, " log_end %lld", fs64_to_cpu(sb, sup->log_end)); + + befs_debug(sb, " magic3 %08x", fs32_to_cpu(sb, sup->magic3)); + + tmp_run = fsrun_to_cpu(sb, sup->root_dir); + befs_debug(sb, " root_dir %u, %hu, %hu", + tmp_run.allocation_group, tmp_run.start, tmp_run.len); + + tmp_run = fsrun_to_cpu(sb, sup->indices); + befs_debug(sb, " indices %u, %hu, %hu", + tmp_run.allocation_group, tmp_run.start, tmp_run.len); + +#endif //CONFIG_BEFS_DEBUG +} + +#if 0 +/* unused */ +void +befs_dump_small_data(const struct super_block *sb, befs_small_data *sd) +{ +} + +/* unused */ +void +befs_dump_run(const struct super_block *sb, befs_disk_block_run run) +{ +#ifdef CONFIG_BEFS_DEBUG + + befs_block_run n = fsrun_to_cpu(sb, run); + + befs_debug(sb, "[%u, %hu, %hu]", n.allocation_group, n.start, n.len); + +#endif //CONFIG_BEFS_DEBUG +} +#endif /* 0 */ + +void +befs_dump_index_entry(const struct super_block *sb, + befs_disk_btree_super *super) +{ +#ifdef CONFIG_BEFS_DEBUG + + befs_debug(sb, "Btree super structure"); + befs_debug(sb, " magic %08x", fs32_to_cpu(sb, super->magic)); + befs_debug(sb, " node_size %u", fs32_to_cpu(sb, super->node_size)); + befs_debug(sb, " max_depth %08x", fs32_to_cpu(sb, super->max_depth)); + + befs_debug(sb, " data_type %08x", fs32_to_cpu(sb, super->data_type)); + befs_debug(sb, " root_node_pointer %016LX", + fs64_to_cpu(sb, super->root_node_ptr)); + befs_debug(sb, " free_node_pointer %016LX", + fs64_to_cpu(sb, super->free_node_ptr)); + befs_debug(sb, " maximum size %016LX", + fs64_to_cpu(sb, super->max_size)); + +#endif //CONFIG_BEFS_DEBUG +} + +void +befs_dump_index_node(const struct super_block *sb, befs_btree_nodehead *node) +{ +#ifdef CONFIG_BEFS_DEBUG + + befs_debug(sb, "Btree node structure"); + befs_debug(sb, " left %016LX", fs64_to_cpu(sb, node->left)); + befs_debug(sb, " right %016LX", fs64_to_cpu(sb, node->right)); + befs_debug(sb, " overflow %016LX", fs64_to_cpu(sb, node->overflow)); + befs_debug(sb, " all_key_count %hu", + fs16_to_cpu(sb, node->all_key_count)); + befs_debug(sb, " all_key_length %hu", + fs16_to_cpu(sb, node->all_key_length)); + +#endif //CONFIG_BEFS_DEBUG +} diff --git a/fs/befs/endian.h b/fs/befs/endian.h new file mode 100644 index 000000000..bb55a54c2 --- /dev/null +++ b/fs/befs/endian.h @@ -0,0 +1,126 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * linux/fs/befs/endian.h + * + * Copyright (C) 2001 Will Dyson + * + * Partially based on similar funtions in the sysv driver. + */ + +#ifndef LINUX_BEFS_ENDIAN +#define LINUX_BEFS_ENDIAN + +#include + +static inline u64 +fs64_to_cpu(const struct super_block *sb, fs64 n) +{ + if (BEFS_SB(sb)->byte_order == BEFS_BYTESEX_LE) + return le64_to_cpu((__force __le64)n); + else + return be64_to_cpu((__force __be64)n); +} + +static inline fs64 +cpu_to_fs64(const struct super_block *sb, u64 n) +{ + if (BEFS_SB(sb)->byte_order == BEFS_BYTESEX_LE) + return (__force fs64)cpu_to_le64(n); + else + return (__force fs64)cpu_to_be64(n); +} + +static inline u32 +fs32_to_cpu(const struct super_block *sb, fs32 n) +{ + if (BEFS_SB(sb)->byte_order == BEFS_BYTESEX_LE) + return le32_to_cpu((__force __le32)n); + else + return be32_to_cpu((__force __be32)n); +} + +static inline fs32 +cpu_to_fs32(const struct super_block *sb, u32 n) +{ + if (BEFS_SB(sb)->byte_order == BEFS_BYTESEX_LE) + return (__force fs32)cpu_to_le32(n); + else + return (__force fs32)cpu_to_be32(n); +} + +static inline u16 +fs16_to_cpu(const struct super_block *sb, fs16 n) +{ + if (BEFS_SB(sb)->byte_order == BEFS_BYTESEX_LE) + return le16_to_cpu((__force __le16)n); + else + return be16_to_cpu((__force __be16)n); +} + +static inline fs16 +cpu_to_fs16(const struct super_block *sb, u16 n) +{ + if (BEFS_SB(sb)->byte_order == BEFS_BYTESEX_LE) + return (__force fs16)cpu_to_le16(n); + else + return (__force fs16)cpu_to_be16(n); +} + +/* Composite types below here */ + +static inline befs_block_run +fsrun_to_cpu(const struct super_block *sb, befs_disk_block_run n) +{ + befs_block_run run; + + if (BEFS_SB(sb)->byte_order == BEFS_BYTESEX_LE) { + run.allocation_group = le32_to_cpu((__force __le32)n.allocation_group); + run.start = le16_to_cpu((__force __le16)n.start); + run.len = le16_to_cpu((__force __le16)n.len); + } else { + run.allocation_group = be32_to_cpu((__force __be32)n.allocation_group); + run.start = be16_to_cpu((__force __be16)n.start); + run.len = be16_to_cpu((__force __be16)n.len); + } + return run; +} + +static inline befs_disk_block_run +cpu_to_fsrun(const struct super_block *sb, befs_block_run n) +{ + befs_disk_block_run run; + + if (BEFS_SB(sb)->byte_order == BEFS_BYTESEX_LE) { + run.allocation_group = cpu_to_le32(n.allocation_group); + run.start = cpu_to_le16(n.start); + run.len = cpu_to_le16(n.len); + } else { + run.allocation_group = cpu_to_be32(n.allocation_group); + run.start = cpu_to_be16(n.start); + run.len = cpu_to_be16(n.len); + } + return run; +} + +static inline befs_data_stream +fsds_to_cpu(const struct super_block *sb, const befs_disk_data_stream *n) +{ + befs_data_stream data; + int i; + + for (i = 0; i < BEFS_NUM_DIRECT_BLOCKS; ++i) + data.direct[i] = fsrun_to_cpu(sb, n->direct[i]); + + data.max_direct_range = fs64_to_cpu(sb, n->max_direct_range); + data.indirect = fsrun_to_cpu(sb, n->indirect); + data.max_indirect_range = fs64_to_cpu(sb, n->max_indirect_range); + data.double_indirect = fsrun_to_cpu(sb, n->double_indirect); + data.max_double_indirect_range = fs64_to_cpu(sb, + n-> + max_double_indirect_range); + data.size = fs64_to_cpu(sb, n->size); + + return data; +} + +#endif //LINUX_BEFS_ENDIAN diff --git a/fs/befs/inode.c b/fs/befs/inode.c new file mode 100644 index 000000000..791b46a6f --- /dev/null +++ b/fs/befs/inode.c @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * inode.c + * + * Copyright (C) 2001 Will Dyson + */ + +#include + +#include "befs.h" +#include "inode.h" + +/* + * Validates the correctness of the befs inode + * Returns BEFS_OK if the inode should be used, otherwise + * returns BEFS_BAD_INODE + */ +int +befs_check_inode(struct super_block *sb, befs_inode *raw_inode, + befs_blocknr_t inode) +{ + u32 magic1 = fs32_to_cpu(sb, raw_inode->magic1); + befs_inode_addr ino_num = fsrun_to_cpu(sb, raw_inode->inode_num); + u32 flags = fs32_to_cpu(sb, raw_inode->flags); + + /* check magic header. */ + if (magic1 != BEFS_INODE_MAGIC1) { + befs_error(sb, + "Inode has a bad magic header - inode = %lu", + (unsigned long)inode); + return BEFS_BAD_INODE; + } + + /* + * Sanity check2: inodes store their own block address. Check it. + */ + if (inode != iaddr2blockno(sb, &ino_num)) { + befs_error(sb, "inode blocknr field disagrees with vfs " + "VFS: %lu, Inode %lu", (unsigned long) + inode, (unsigned long)iaddr2blockno(sb, &ino_num)); + return BEFS_BAD_INODE; + } + + /* + * check flag + */ + + if (!(flags & BEFS_INODE_IN_USE)) { + befs_error(sb, "inode is not used - inode = %lu", + (unsigned long)inode); + return BEFS_BAD_INODE; + } + + return BEFS_OK; +} diff --git a/fs/befs/inode.h b/fs/befs/inode.h new file mode 100644 index 000000000..2219e412f --- /dev/null +++ b/fs/befs/inode.h @@ -0,0 +1,7 @@ +/* + * inode.h + * + */ + +int befs_check_inode(struct super_block *sb, befs_inode *raw_inode, + befs_blocknr_t inode); diff --git a/fs/befs/io.c b/fs/befs/io.c new file mode 100644 index 000000000..2caf50a4a --- /dev/null +++ b/fs/befs/io.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/befs/io.c + * + * Copyright (C) 2001 Will Dyson + +#include "befs.h" +#include "io.h" + +/* + * Converts befs notion of disk addr to a disk offset and uses + * linux kernel function sb_bread() to get the buffer containing + * the offset. + */ + +struct buffer_head * +befs_bread_iaddr(struct super_block *sb, befs_inode_addr iaddr) +{ + struct buffer_head *bh; + befs_blocknr_t block; + struct befs_sb_info *befs_sb = BEFS_SB(sb); + + befs_debug(sb, "---> Enter %s " + "[%u, %hu, %hu]", __func__, iaddr.allocation_group, + iaddr.start, iaddr.len); + + if (iaddr.allocation_group > befs_sb->num_ags) { + befs_error(sb, "BEFS: Invalid allocation group %u, max is %u", + iaddr.allocation_group, befs_sb->num_ags); + goto error; + } + + block = iaddr2blockno(sb, &iaddr); + + befs_debug(sb, "%s: offset = %lu", __func__, (unsigned long)block); + + bh = sb_bread(sb, block); + + if (bh == NULL) { + befs_error(sb, "Failed to read block %lu", + (unsigned long)block); + goto error; + } + + befs_debug(sb, "<--- %s", __func__); + return bh; + +error: + befs_debug(sb, "<--- %s ERROR", __func__); + return NULL; +} diff --git a/fs/befs/io.h b/fs/befs/io.h new file mode 100644 index 000000000..9b3e1967c --- /dev/null +++ b/fs/befs/io.h @@ -0,0 +1,6 @@ +/* + * io.h + */ + +struct buffer_head *befs_bread_iaddr(struct super_block *sb, + befs_inode_addr iaddr); diff --git a/fs/befs/linuxvfs.c b/fs/befs/linuxvfs.c new file mode 100644 index 000000000..ee0cbae52 --- /dev/null +++ b/fs/befs/linuxvfs.c @@ -0,0 +1,1030 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/befs/linuxvfs.c + * + * Copyright (C) 2001 Will Dyson +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "befs.h" +#include "btree.h" +#include "inode.h" +#include "datastream.h" +#include "super.h" +#include "io.h" + +MODULE_DESCRIPTION("BeOS File System (BeFS) driver"); +MODULE_AUTHOR("Will Dyson"); +MODULE_LICENSE("GPL"); + +/* The units the vfs expects inode->i_blocks to be in */ +#define VFS_BLOCK_SIZE 512 + +static int befs_readdir(struct file *, struct dir_context *); +static int befs_get_block(struct inode *, sector_t, struct buffer_head *, int); +static int befs_read_folio(struct file *file, struct folio *folio); +static sector_t befs_bmap(struct address_space *mapping, sector_t block); +static struct dentry *befs_lookup(struct inode *, struct dentry *, + unsigned int); +static struct inode *befs_iget(struct super_block *, unsigned long); +static struct inode *befs_alloc_inode(struct super_block *sb); +static void befs_free_inode(struct inode *inode); +static void befs_destroy_inodecache(void); +static int befs_symlink_read_folio(struct file *, struct folio *); +static int befs_utf2nls(struct super_block *sb, const char *in, int in_len, + char **out, int *out_len); +static int befs_nls2utf(struct super_block *sb, const char *in, int in_len, + char **out, int *out_len); +static void befs_put_super(struct super_block *); +static int befs_statfs(struct dentry *, struct kstatfs *); +static int befs_show_options(struct seq_file *, struct dentry *); +static struct dentry *befs_fh_to_dentry(struct super_block *sb, + struct fid *fid, int fh_len, int fh_type); +static struct dentry *befs_fh_to_parent(struct super_block *sb, + struct fid *fid, int fh_len, int fh_type); +static struct dentry *befs_get_parent(struct dentry *child); +static void befs_free_fc(struct fs_context *fc); + +static const struct super_operations befs_sops = { + .alloc_inode = befs_alloc_inode, /* allocate a new inode */ + .free_inode = befs_free_inode, /* deallocate an inode */ + .put_super = befs_put_super, /* uninit super */ + .statfs = befs_statfs, /* statfs */ + .show_options = befs_show_options, +}; + +/* slab cache for befs_inode_info objects */ +static struct kmem_cache *befs_inode_cachep; + +static const struct file_operations befs_dir_operations = { + .read = generic_read_dir, + .iterate_shared = befs_readdir, + .llseek = generic_file_llseek, + .setlease = generic_setlease, +}; + +static const struct inode_operations befs_dir_inode_operations = { + .lookup = befs_lookup, +}; + +static const struct address_space_operations befs_aops = { + .read_folio = befs_read_folio, + .bmap = befs_bmap, +}; + +static const struct address_space_operations befs_symlink_aops = { + .read_folio = befs_symlink_read_folio, +}; + +static const struct export_operations befs_export_operations = { + .encode_fh = generic_encode_ino32_fh, + .fh_to_dentry = befs_fh_to_dentry, + .fh_to_parent = befs_fh_to_parent, + .get_parent = befs_get_parent, +}; + +/* + * Called by generic_file_read() to read a folio of data + * + * In turn, simply calls a generic block read function and + * passes it the address of befs_get_block, for mapping file + * positions to disk blocks. + */ +static int befs_read_folio(struct file *file, struct folio *folio) +{ + return block_read_full_folio(folio, befs_get_block); +} + +static sector_t +befs_bmap(struct address_space *mapping, sector_t block) +{ + return generic_block_bmap(mapping, block, befs_get_block); +} + +/* + * Generic function to map a file position (block) to a + * disk offset (passed back in bh_result). + * + * Used by many higher level functions. + * + * Calls befs_fblock2brun() in datastream.c to do the real work. + */ + +static int +befs_get_block(struct inode *inode, sector_t block, + struct buffer_head *bh_result, int create) +{ + struct super_block *sb = inode->i_sb; + befs_data_stream *ds = &BEFS_I(inode)->i_data.ds; + befs_block_run run = BAD_IADDR; + int res; + ulong disk_off; + + befs_debug(sb, "---> befs_get_block() for inode %llu, block %ld", + inode->i_ino, (long)block); + if (create) { + befs_error(sb, "befs_get_block() was asked to write to " + "block %ld in inode %llu", (long)block, + inode->i_ino); + return -EPERM; + } + + res = befs_fblock2brun(sb, ds, block, &run); + if (res != BEFS_OK) { + befs_error(sb, + "<--- %s for inode %llu, block %ld ERROR", + __func__, inode->i_ino, + (long)block); + return -EFBIG; + } + + disk_off = (ulong) iaddr2blockno(sb, &run); + + map_bh(bh_result, inode->i_sb, disk_off); + + befs_debug(sb, "<--- %s for inode %llu, block %ld, disk address %lu", + __func__, inode->i_ino, (long)block, + (unsigned long)disk_off); + + return 0; +} + +static struct dentry * +befs_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) +{ + struct inode *inode; + struct super_block *sb = dir->i_sb; + const befs_data_stream *ds = &BEFS_I(dir)->i_data.ds; + befs_off_t offset; + int ret; + int utfnamelen; + char *utfname; + const char *name = dentry->d_name.name; + + befs_debug(sb, "---> %s name %pd inode %llu", __func__, + dentry, dir->i_ino); + + /* Convert to UTF-8 */ + if (BEFS_SB(sb)->nls) { + ret = + befs_nls2utf(sb, name, strlen(name), &utfname, &utfnamelen); + if (ret < 0) { + befs_debug(sb, "<--- %s ERROR", __func__); + return ERR_PTR(ret); + } + ret = befs_btree_find(sb, ds, utfname, &offset); + kfree(utfname); + + } else { + ret = befs_btree_find(sb, ds, name, &offset); + } + + if (ret == BEFS_BT_NOT_FOUND) { + befs_debug(sb, "<--- %s %pd not found", __func__, dentry); + inode = NULL; + } else if (ret != BEFS_OK || offset == 0) { + befs_error(sb, "<--- %s Error", __func__); + inode = ERR_PTR(-ENODATA); + } else { + inode = befs_iget(dir->i_sb, (ino_t) offset); + } + befs_debug(sb, "<--- %s", __func__); + + return d_splice_alias(inode, dentry); +} + +static int +befs_readdir(struct file *file, struct dir_context *ctx) +{ + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + const befs_data_stream *ds = &BEFS_I(inode)->i_data.ds; + befs_off_t value; + int result; + size_t keysize; + char keybuf[BEFS_NAME_LEN + 1]; + + befs_debug(sb, "---> %s name %pD, inode %llu, ctx->pos %lld", + __func__, file, inode->i_ino, ctx->pos); + + while (1) { + result = befs_btree_read(sb, ds, ctx->pos, BEFS_NAME_LEN + 1, + keybuf, &keysize, &value); + + if (result == BEFS_ERR) { + befs_debug(sb, "<--- %s ERROR", __func__); + befs_error(sb, "IO error reading %pD (inode %llu)", + file, inode->i_ino); + return -EIO; + + } else if (result == BEFS_BT_END) { + befs_debug(sb, "<--- %s END", __func__); + return 0; + + } else if (result == BEFS_BT_EMPTY) { + befs_debug(sb, "<--- %s Empty directory", __func__); + return 0; + } + + /* Convert to NLS */ + if (BEFS_SB(sb)->nls) { + char *nlsname; + int nlsnamelen; + + result = + befs_utf2nls(sb, keybuf, keysize, &nlsname, + &nlsnamelen); + if (result < 0) { + befs_debug(sb, "<--- %s ERROR", __func__); + return result; + } + if (!dir_emit(ctx, nlsname, nlsnamelen, + (ino_t) value, DT_UNKNOWN)) { + kfree(nlsname); + return 0; + } + kfree(nlsname); + } else { + if (!dir_emit(ctx, keybuf, keysize, + (ino_t) value, DT_UNKNOWN)) + return 0; + } + ctx->pos++; + } +} + +static struct inode * +befs_alloc_inode(struct super_block *sb) +{ + struct befs_inode_info *bi; + + bi = alloc_inode_sb(sb, befs_inode_cachep, GFP_KERNEL); + if (!bi) + return NULL; + return &bi->vfs_inode; +} + +static void befs_free_inode(struct inode *inode) +{ + kmem_cache_free(befs_inode_cachep, BEFS_I(inode)); +} + +static void init_once(void *foo) +{ + struct befs_inode_info *bi = (struct befs_inode_info *) foo; + + inode_init_once(&bi->vfs_inode); +} + +static struct inode *befs_iget(struct super_block *sb, unsigned long ino) +{ + struct buffer_head *bh; + befs_inode *raw_inode; + struct befs_sb_info *befs_sb = BEFS_SB(sb); + struct befs_inode_info *befs_ino; + struct inode *inode; + + befs_debug(sb, "---> %s inode = %lu", __func__, ino); + + inode = iget_locked(sb, ino); + if (!inode) + return ERR_PTR(-ENOMEM); + if (!(inode_state_read_once(inode) & I_NEW)) + return inode; + + befs_ino = BEFS_I(inode); + + /* convert from vfs's inode number to befs's inode number */ + befs_ino->i_inode_num = blockno2iaddr(sb, inode->i_ino); + + befs_debug(sb, " real inode number [%u, %hu, %hu]", + befs_ino->i_inode_num.allocation_group, + befs_ino->i_inode_num.start, befs_ino->i_inode_num.len); + + bh = sb_bread(sb, inode->i_ino); + if (!bh) { + befs_error(sb, "unable to read inode block - " + "inode = %llu", inode->i_ino); + goto unacquire_none; + } + + raw_inode = (befs_inode *) bh->b_data; + + befs_dump_inode(sb, raw_inode); + + if (befs_check_inode(sb, raw_inode, inode->i_ino) != BEFS_OK) { + befs_error(sb, "Bad inode: %llu", inode->i_ino); + goto unacquire_bh; + } + + inode->i_mode = (umode_t) fs32_to_cpu(sb, raw_inode->mode); + + /* + * set uid and gid. But since current BeOS is single user OS, so + * you can change by "uid" or "gid" options. + */ + + inode->i_uid = befs_sb->mount_opts.use_uid ? + befs_sb->mount_opts.uid : + make_kuid(&init_user_ns, fs32_to_cpu(sb, raw_inode->uid)); + inode->i_gid = befs_sb->mount_opts.use_gid ? + befs_sb->mount_opts.gid : + make_kgid(&init_user_ns, fs32_to_cpu(sb, raw_inode->gid)); + + set_nlink(inode, 1); + + /* + * BEFS's time is 64 bits, but current VFS is 32 bits... + * BEFS don't have access time. Nor inode change time. VFS + * doesn't have creation time. + * Also, the lower 16 bits of the last_modified_time and + * create_time are just a counter to help ensure uniqueness + * for indexing purposes. (PFD, page 54) + */ + + inode_set_mtime(inode, + fs64_to_cpu(sb, raw_inode->last_modified_time) >> 16, + 0);/* lower 16 bits are not a time */ + inode_set_ctime_to_ts(inode, inode_get_mtime(inode)); + inode_set_atime_to_ts(inode, inode_get_mtime(inode)); + + befs_ino->i_inode_num = fsrun_to_cpu(sb, raw_inode->inode_num); + befs_ino->i_parent = fsrun_to_cpu(sb, raw_inode->parent); + befs_ino->i_attribute = fsrun_to_cpu(sb, raw_inode->attributes); + befs_ino->i_flags = fs32_to_cpu(sb, raw_inode->flags); + + if (S_ISLNK(inode->i_mode) && !(befs_ino->i_flags & BEFS_LONG_SYMLINK)){ + inode->i_size = 0; + inode->i_blocks = befs_sb->block_size / VFS_BLOCK_SIZE; + strscpy(befs_ino->i_data.symlink, raw_inode->data.symlink, + BEFS_SYMLINK_LEN); + } else { + int num_blks; + + befs_ino->i_data.ds = + fsds_to_cpu(sb, &raw_inode->data.datastream); + + num_blks = befs_count_blocks(sb, &befs_ino->i_data.ds); + inode->i_blocks = + num_blks * (befs_sb->block_size / VFS_BLOCK_SIZE); + inode->i_size = befs_ino->i_data.ds.size; + } + + inode->i_mapping->a_ops = &befs_aops; + + if (S_ISREG(inode->i_mode)) { + inode->i_fop = &generic_ro_fops; + } else if (S_ISDIR(inode->i_mode)) { + inode->i_op = &befs_dir_inode_operations; + inode->i_fop = &befs_dir_operations; + } else if (S_ISLNK(inode->i_mode)) { + if (befs_ino->i_flags & BEFS_LONG_SYMLINK) { + inode->i_op = &page_symlink_inode_operations; + inode_nohighmem(inode); + inode->i_mapping->a_ops = &befs_symlink_aops; + } else { + inode->i_link = befs_ino->i_data.symlink; + inode->i_op = &simple_symlink_inode_operations; + } + } else { + befs_error(sb, "Inode %llu is not a regular file, " + "directory or symlink. THAT IS WRONG! BeFS has no " + "on disk special files", inode->i_ino); + goto unacquire_bh; + } + + brelse(bh); + befs_debug(sb, "<--- %s", __func__); + unlock_new_inode(inode); + return inode; + +unacquire_bh: + brelse(bh); + +unacquire_none: + iget_failed(inode); + befs_debug(sb, "<--- %s - Bad inode", __func__); + return ERR_PTR(-EIO); +} + +/* Initialize the inode cache. Called at fs setup. + * + * Taken from NFS implementation by Al Viro. + */ +static int __init +befs_init_inodecache(void) +{ + befs_inode_cachep = kmem_cache_create_usercopy("befs_inode_cache", + sizeof(struct befs_inode_info), 0, + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, + offsetof(struct befs_inode_info, + i_data.symlink), + sizeof_field(struct befs_inode_info, + i_data.symlink), + init_once); + if (befs_inode_cachep == NULL) + return -ENOMEM; + + return 0; +} + +/* Called at fs teardown. + * + * Taken from NFS implementation by Al Viro. + */ +static void +befs_destroy_inodecache(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(befs_inode_cachep); +} + +/* + * The inode of symbolic link is different to data stream. + * The data stream become link name. Unless the LONG_SYMLINK + * flag is set. + */ +static int befs_symlink_read_folio(struct file *unused, struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + struct super_block *sb = inode->i_sb; + struct befs_inode_info *befs_ino = BEFS_I(inode); + befs_data_stream *data = &befs_ino->i_data.ds; + befs_off_t len = data->size; + char *link = folio_address(folio); + int err = -EIO; + + if (len == 0 || len > PAGE_SIZE) { + befs_error(sb, "Long symlink with illegal length"); + goto fail; + } + befs_debug(sb, "Follow long symlink"); + + if (befs_read_lsymlink(sb, data, link, len) != len) { + befs_error(sb, "Failed to read entire long symlink"); + goto fail; + } + link[len - 1] = '\0'; + err = 0; +fail: + folio_end_read(folio, err == 0); + return err; +} + +/* + * UTF-8 to NLS charset convert routine + * + * Uses uni2char() / char2uni() rather than the nls tables directly + */ +static int +befs_utf2nls(struct super_block *sb, const char *in, + int in_len, char **out, int *out_len) +{ + struct nls_table *nls = BEFS_SB(sb)->nls; + int i, o; + unicode_t uni; + int unilen, utflen; + char *result; + /* The utf8->nls conversion won't make the final nls string bigger + * than the utf one, but if the string is pure ascii they'll have the + * same width and an extra char is needed to save the additional \0 + */ + int maxlen = in_len + 1; + + befs_debug(sb, "---> %s", __func__); + + if (!nls) { + befs_error(sb, "%s called with no NLS table loaded", __func__); + return -EINVAL; + } + + *out = result = kmalloc(maxlen, GFP_NOFS); + if (!*out) + return -ENOMEM; + + for (i = o = 0; i < in_len; i += utflen, o += unilen) { + + /* convert from UTF-8 to Unicode */ + utflen = utf8_to_utf32(&in[i], in_len - i, &uni); + if (utflen < 0) + goto conv_err; + + /* convert from Unicode to nls */ + if (uni > MAX_WCHAR_T) + goto conv_err; + unilen = nls->uni2char(uni, &result[o], in_len - o); + if (unilen < 0) + goto conv_err; + } + result[o] = '\0'; + *out_len = o; + + befs_debug(sb, "<--- %s", __func__); + + return o; + +conv_err: + befs_error(sb, "Name using character set %s contains a character that " + "cannot be converted to unicode.", nls->charset); + befs_debug(sb, "<--- %s", __func__); + kfree(result); + return -EILSEQ; +} + +/** + * befs_nls2utf - Convert NLS string to utf8 encodeing + * @sb: Superblock + * @in: Input string buffer in NLS format + * @in_len: Length of input string in bytes + * @out: The output string in UTF-8 format + * @out_len: Length of the output buffer + * + * Converts input string @in, which is in the format of the loaded NLS map, + * into a utf8 string. + * + * The destination string @out is allocated by this function and the caller is + * responsible for freeing it with kfree() + * + * On return, *@out_len is the length of @out in bytes. + * + * On success, the return value is the number of utf8 characters written to + * the output buffer @out. + * + * On Failure, a negative number coresponding to the error code is returned. + */ + +static int +befs_nls2utf(struct super_block *sb, const char *in, + int in_len, char **out, int *out_len) +{ + struct nls_table *nls = BEFS_SB(sb)->nls; + int i, o; + wchar_t uni; + int unilen, utflen; + char *result; + /* + * There are nls characters that will translate to 3-chars-wide UTF-8 + * characters, an additional byte is needed to save the final \0 + * in special cases + */ + int maxlen = (3 * in_len) + 1; + + befs_debug(sb, "---> %s\n", __func__); + + if (!nls) { + befs_error(sb, "%s called with no NLS table loaded.", + __func__); + return -EINVAL; + } + + *out = result = kmalloc(maxlen, GFP_NOFS); + if (!*out) { + *out_len = 0; + return -ENOMEM; + } + + for (i = o = 0; i < in_len; i += unilen, o += utflen) { + + /* convert from nls to unicode */ + unilen = nls->char2uni(&in[i], in_len - i, &uni); + if (unilen < 0) + goto conv_err; + + /* convert from unicode to UTF-8 */ + utflen = utf32_to_utf8(uni, &result[o], 3); + if (utflen <= 0) + goto conv_err; + } + + result[o] = '\0'; + *out_len = o; + + befs_debug(sb, "<--- %s", __func__); + + return i; + +conv_err: + befs_error(sb, "Name using character set %s contains a character that " + "cannot be converted to unicode.", nls->charset); + befs_debug(sb, "<--- %s", __func__); + kfree(result); + return -EILSEQ; +} + +static struct inode *befs_nfs_get_inode(struct super_block *sb, uint64_t ino, + uint32_t generation) +{ + /* No need to handle i_generation */ + return befs_iget(sb, ino); +} + +/* + * Map a NFS file handle to a corresponding dentry + */ +static struct dentry *befs_fh_to_dentry(struct super_block *sb, + struct fid *fid, int fh_len, int fh_type) +{ + return generic_fh_to_dentry(sb, fid, fh_len, fh_type, + befs_nfs_get_inode); +} + +/* + * Find the parent for a file specified by NFS handle + */ +static struct dentry *befs_fh_to_parent(struct super_block *sb, + struct fid *fid, int fh_len, int fh_type) +{ + return generic_fh_to_parent(sb, fid, fh_len, fh_type, + befs_nfs_get_inode); +} + +static struct dentry *befs_get_parent(struct dentry *child) +{ + struct inode *parent; + struct befs_inode_info *befs_ino = BEFS_I(d_inode(child)); + + parent = befs_iget(child->d_sb, + (unsigned long)befs_ino->i_parent.start); + return d_obtain_alias(parent); +} + +enum { + Opt_uid, Opt_gid, Opt_charset, Opt_debug, +}; + +static const struct fs_parameter_spec befs_param_spec[] = { + fsparam_uid ("uid", Opt_uid), + fsparam_gid ("gid", Opt_gid), + fsparam_string ("iocharset", Opt_charset), + fsparam_flag ("debug", Opt_debug), + {} +}; + +static int +befs_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct befs_mount_options *opts = fc->fs_private; + int token; + struct fs_parse_result result; + + /* befs ignores all options on remount */ + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE) + return 0; + + token = fs_parse(fc, befs_param_spec, param, &result); + if (token < 0) + return token; + + switch (token) { + case Opt_uid: + opts->uid = result.uid; + opts->use_uid = 1; + break; + case Opt_gid: + opts->gid = result.gid; + opts->use_gid = 1; + break; + case Opt_charset: + kfree(opts->iocharset); + opts->iocharset = param->string; + param->string = NULL; + break; + case Opt_debug: + opts->debug = 1; + break; + default: + return -EINVAL; + } + return 0; +} + +static int befs_show_options(struct seq_file *m, struct dentry *root) +{ + struct befs_sb_info *befs_sb = BEFS_SB(root->d_sb); + struct befs_mount_options *opts = &befs_sb->mount_opts; + + if (!uid_eq(opts->uid, GLOBAL_ROOT_UID)) + seq_printf(m, ",uid=%u", + from_kuid_munged(&init_user_ns, opts->uid)); + if (!gid_eq(opts->gid, GLOBAL_ROOT_GID)) + seq_printf(m, ",gid=%u", + from_kgid_munged(&init_user_ns, opts->gid)); + if (opts->iocharset) + seq_printf(m, ",charset=%s", opts->iocharset); + if (opts->debug) + seq_puts(m, ",debug"); + return 0; +} + +/* This function has the responsibiltiy of getting the + * filesystem ready for unmounting. + * Basically, we free everything that we allocated in + * befs_read_inode + */ +static void +befs_put_super(struct super_block *sb) +{ + kfree(BEFS_SB(sb)->mount_opts.iocharset); + BEFS_SB(sb)->mount_opts.iocharset = NULL; + unload_nls(BEFS_SB(sb)->nls); + kfree(sb->s_fs_info); + sb->s_fs_info = NULL; +} + +/* + * Copy the parsed options into the sbi mount_options member + */ +static void +befs_set_options(struct befs_sb_info *sbi, struct befs_mount_options *opts) +{ + sbi->mount_opts.uid = opts->uid; + sbi->mount_opts.gid = opts->gid; + sbi->mount_opts.use_uid = opts->use_uid; + sbi->mount_opts.use_gid = opts->use_gid; + sbi->mount_opts.debug = opts->debug; + sbi->mount_opts.iocharset = opts->iocharset; + opts->iocharset = NULL; +} + +/* Allocate private field of the superblock, fill it. + * + * Finish filling the public superblock fields + * Make the root directory + * Load a set of NLS translations if needed. + */ +static int +befs_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct buffer_head *bh; + struct befs_sb_info *befs_sb; + befs_super_block *disk_sb; + struct inode *root; + long ret = -EINVAL; + const unsigned long sb_block = 0; + const off_t x86_sb_off = 512; + int blocksize; + struct befs_mount_options *parsed_opts = fc->fs_private; + int silent = fc->sb_flags & SB_SILENT; + + sb->s_fs_info = kzalloc_obj(*befs_sb); + if (sb->s_fs_info == NULL) + goto unacquire_none; + + befs_sb = BEFS_SB(sb); + + befs_set_options(befs_sb, parsed_opts); + + befs_debug(sb, "---> %s", __func__); + + if (!sb_rdonly(sb)) { + befs_warning(sb, + "No write support. Marking filesystem read-only"); + sb->s_flags |= SB_RDONLY; + } + + /* + * Set dummy blocksize to read super block. + * Will be set to real fs blocksize later. + * + * Linux 2.4.10 and later refuse to read blocks smaller than + * the logical block size for the device. But we also need to read at + * least 1k to get the second 512 bytes of the volume. + */ + blocksize = sb_min_blocksize(sb, 1024); + if (!blocksize) { + if (!silent) + befs_error(sb, "unable to set blocksize"); + goto unacquire_priv_sbp; + } + + bh = sb_bread(sb, sb_block); + if (!bh) { + if (!silent) + befs_error(sb, "unable to read superblock"); + goto unacquire_priv_sbp; + } + + /* account for offset of super block on x86 */ + disk_sb = (befs_super_block *) bh->b_data; + if ((disk_sb->magic1 == BEFS_SUPER_MAGIC1_LE) || + (disk_sb->magic1 == BEFS_SUPER_MAGIC1_BE)) { + befs_debug(sb, "Using PPC superblock location"); + } else { + befs_debug(sb, "Using x86 superblock location"); + disk_sb = + (befs_super_block *) ((void *) bh->b_data + x86_sb_off); + } + + if ((befs_load_sb(sb, disk_sb) != BEFS_OK) || + (befs_check_sb(sb) != BEFS_OK)) + goto unacquire_bh; + + befs_dump_super_block(sb, disk_sb); + + brelse(bh); + + if (befs_sb->num_blocks > ~((sector_t)0)) { + if (!silent) + befs_error(sb, "blocks count: %llu is larger than the host can use", + befs_sb->num_blocks); + goto unacquire_priv_sbp; + } + + /* + * set up enough so that it can read an inode + * Fill in kernel superblock fields from private sb + */ + sb->s_magic = BEFS_SUPER_MAGIC; + /* Set real blocksize of fs */ + if (!sb_set_blocksize(sb, (ulong) befs_sb->block_size)) + goto unacquire_priv_sbp; + sb->s_op = &befs_sops; + sb->s_export_op = &befs_export_operations; + sb->s_time_min = 0; + sb->s_time_max = 0xffffffffffffll; + root = befs_iget(sb, iaddr2blockno(sb, &(befs_sb->root_dir))); + if (IS_ERR(root)) { + ret = PTR_ERR(root); + goto unacquire_priv_sbp; + } + sb->s_root = d_make_root(root); + if (!sb->s_root) { + if (!silent) + befs_error(sb, "get root inode failed"); + goto unacquire_priv_sbp; + } + + /* load nls library */ + if (befs_sb->mount_opts.iocharset) { + befs_debug(sb, "Loading nls: %s", + befs_sb->mount_opts.iocharset); + befs_sb->nls = load_nls(befs_sb->mount_opts.iocharset); + if (!befs_sb->nls) { + befs_warning(sb, "Cannot load nls %s" + " loading default nls", + befs_sb->mount_opts.iocharset); + befs_sb->nls = load_nls_default(); + } + /* load default nls if none is specified in mount options */ + } else { + befs_debug(sb, "Loading default nls"); + befs_sb->nls = load_nls_default(); + } + + return 0; + +unacquire_bh: + brelse(bh); + +unacquire_priv_sbp: + kfree(befs_sb->mount_opts.iocharset); + kfree(sb->s_fs_info); + sb->s_fs_info = NULL; + +unacquire_none: + return ret; +} + +static int +befs_reconfigure(struct fs_context *fc) +{ + sync_filesystem(fc->root->d_sb); + if (!(fc->sb_flags & SB_RDONLY)) + return -EINVAL; + return 0; +} + +static int +befs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + u64 id = huge_encode_dev(sb->s_bdev->bd_dev); + + befs_debug(sb, "---> %s", __func__); + + buf->f_type = BEFS_SUPER_MAGIC; + buf->f_bsize = sb->s_blocksize; + buf->f_blocks = BEFS_SB(sb)->num_blocks; + buf->f_bfree = BEFS_SB(sb)->num_blocks - BEFS_SB(sb)->used_blocks; + buf->f_bavail = buf->f_bfree; + buf->f_files = 0; /* UNKNOWN */ + buf->f_ffree = 0; /* UNKNOWN */ + buf->f_fsid = u64_to_fsid(id); + buf->f_namelen = BEFS_NAME_LEN; + + befs_debug(sb, "<--- %s", __func__); + + return 0; +} + +static int befs_get_tree(struct fs_context *fc) +{ + return get_tree_bdev(fc, befs_fill_super); +} + +static const struct fs_context_operations befs_context_ops = { + .parse_param = befs_parse_param, + .get_tree = befs_get_tree, + .reconfigure = befs_reconfigure, + .free = befs_free_fc, +}; + +static int befs_init_fs_context(struct fs_context *fc) +{ + struct befs_mount_options *opts; + + opts = kzalloc_obj(*opts); + if (!opts) + return -ENOMEM; + + /* Initialize options */ + opts->uid = GLOBAL_ROOT_UID; + opts->gid = GLOBAL_ROOT_GID; + + fc->fs_private = opts; + fc->ops = &befs_context_ops; + + return 0; +} + +static void befs_free_fc(struct fs_context *fc) +{ + struct befs_mount_options *opts = fc->fs_private; + + kfree(opts->iocharset); + kfree(fc->fs_private); +} + +static struct file_system_type befs_fs_type = { + .owner = THIS_MODULE, + .name = "befs", + .kill_sb = kill_block_super, + .fs_flags = FS_REQUIRES_DEV, + .init_fs_context = befs_init_fs_context, + .parameters = befs_param_spec, +}; +MODULE_ALIAS_FS("befs"); + +static int __init +init_befs_fs(void) +{ + int err; + + pr_info("version: %s\n", BEFS_VERSION); + + err = befs_init_inodecache(); + if (err) + goto unacquire_none; + + err = register_filesystem(&befs_fs_type); + if (err) + goto unacquire_inodecache; + + return 0; + +unacquire_inodecache: + befs_destroy_inodecache(); + +unacquire_none: + return err; +} + +static void __exit +exit_befs_fs(void) +{ + befs_destroy_inodecache(); + + unregister_filesystem(&befs_fs_type); +} + +/* + * Macros that typecheck the init and exit functions, + * ensures that they are called at init and cleanup, + * and eliminates warnings about unused functions. + */ +module_init(init_befs_fs) +module_exit(exit_befs_fs) diff --git a/fs/befs/super.c b/fs/befs/super.c new file mode 100644 index 000000000..7c50025c9 --- /dev/null +++ b/fs/befs/super.c @@ -0,0 +1,118 @@ +/* + * super.c + * + * Copyright (C) 2001-2002 Will Dyson + * + * Licensed under the GNU GPL. See the file COPYING for details. + * + */ + +#include +#include /* for PAGE_SIZE */ + +#include "befs.h" +#include "super.h" + +/* + * befs_load_sb -- Read from disk and properly byteswap all the fields + * of the befs superblock + */ +int +befs_load_sb(struct super_block *sb, befs_super_block *disk_sb) +{ + struct befs_sb_info *befs_sb = BEFS_SB(sb); + + /* Check the byte order of the filesystem */ + if (disk_sb->fs_byte_order == BEFS_BYTEORDER_NATIVE_LE) + befs_sb->byte_order = BEFS_BYTESEX_LE; + else if (disk_sb->fs_byte_order == BEFS_BYTEORDER_NATIVE_BE) + befs_sb->byte_order = BEFS_BYTESEX_BE; + + befs_sb->magic1 = fs32_to_cpu(sb, disk_sb->magic1); + befs_sb->magic2 = fs32_to_cpu(sb, disk_sb->magic2); + befs_sb->magic3 = fs32_to_cpu(sb, disk_sb->magic3); + befs_sb->block_size = fs32_to_cpu(sb, disk_sb->block_size); + befs_sb->block_shift = fs32_to_cpu(sb, disk_sb->block_shift); + befs_sb->num_blocks = fs64_to_cpu(sb, disk_sb->num_blocks); + befs_sb->used_blocks = fs64_to_cpu(sb, disk_sb->used_blocks); + befs_sb->inode_size = fs32_to_cpu(sb, disk_sb->inode_size); + + befs_sb->blocks_per_ag = fs32_to_cpu(sb, disk_sb->blocks_per_ag); + befs_sb->ag_shift = fs32_to_cpu(sb, disk_sb->ag_shift); + befs_sb->num_ags = fs32_to_cpu(sb, disk_sb->num_ags); + + befs_sb->flags = fs32_to_cpu(sb, disk_sb->flags); + + befs_sb->log_blocks = fsrun_to_cpu(sb, disk_sb->log_blocks); + befs_sb->log_start = fs64_to_cpu(sb, disk_sb->log_start); + befs_sb->log_end = fs64_to_cpu(sb, disk_sb->log_end); + + befs_sb->root_dir = fsrun_to_cpu(sb, disk_sb->root_dir); + befs_sb->indices = fsrun_to_cpu(sb, disk_sb->indices); + befs_sb->nls = NULL; + + return BEFS_OK; +} + +int +befs_check_sb(struct super_block *sb) +{ + struct befs_sb_info *befs_sb = BEFS_SB(sb); + + /* Check magic headers of super block */ + if ((befs_sb->magic1 != BEFS_SUPER_MAGIC1) + || (befs_sb->magic2 != BEFS_SUPER_MAGIC2) + || (befs_sb->magic3 != BEFS_SUPER_MAGIC3)) { + befs_error(sb, "invalid magic header"); + return BEFS_ERR; + } + + /* + * Check blocksize of BEFS. + * + * Blocksize of BEFS is 1024, 2048, 4096 or 8192. + */ + + if ((befs_sb->block_size != 1024) + && (befs_sb->block_size != 2048) + && (befs_sb->block_size != 4096) + && (befs_sb->block_size != 8192)) { + befs_error(sb, "invalid blocksize: %u", befs_sb->block_size); + return BEFS_ERR; + } + + if (befs_sb->block_size > PAGE_SIZE) { + befs_error(sb, "blocksize(%u) cannot be larger " + "than system pagesize(%lu)", befs_sb->block_size, + PAGE_SIZE); + return BEFS_ERR; + } + + /* + * block_shift and block_size encode the same information + * in different ways as a consistency check. + */ + + if ((1 << befs_sb->block_shift) != befs_sb->block_size) { + befs_error(sb, "block_shift disagrees with block_size. " + "Corruption likely."); + return BEFS_ERR; + } + + + /* ag_shift also encodes the same information as blocks_per_ag in a + * different way, non-fatal consistency check + */ + if ((1 << befs_sb->ag_shift) != befs_sb->blocks_per_ag) + befs_error(sb, "ag_shift disagrees with blocks_per_ag."); + + if (befs_sb->log_start != befs_sb->log_end || + befs_sb->flags == BEFS_DIRTY) { + befs_error(sb, "Filesystem not clean! There are blocks in the " + "journal. You must boot into BeOS and mount this " + "volume to make it clean."); + return BEFS_ERR; + } + + return BEFS_OK; +} diff --git a/fs/befs/super.h b/fs/befs/super.h new file mode 100644 index 000000000..ec1df30a7 --- /dev/null +++ b/fs/befs/super.h @@ -0,0 +1,6 @@ +/* + * super.h + */ + +int befs_load_sb(struct super_block *sb, befs_super_block *disk_sb); +int befs_check_sb(struct super_block *sb); diff --git a/fs/bfs/Kconfig b/fs/bfs/Kconfig new file mode 100644 index 000000000..8e7ef866b --- /dev/null +++ b/fs/bfs/Kconfig @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: GPL-2.0-only +config BFS_FS + tristate "BFS file system support" + depends on BLOCK + select BUFFER_HEAD + help + Boot File System (BFS) is a file system used under SCO UnixWare to + allow the bootloader access to the kernel image and other important + files during the boot process. It is usually mounted under /stand + and corresponds to the slice marked as "STAND" in the UnixWare + partition. You should say Y if you want to read or write the files + on your /stand slice from within Linux. You then also need to say Y + to "UnixWare slices support", below. More information about the BFS + file system is contained in the file + . + + If you don't know what this is about, say N. + + To compile this as a module, choose M here: the module will be called + bfs. Note that the file system of your root partition (the one + containing the directory /) cannot be compiled as a module. diff --git a/fs/bfs/Makefile b/fs/bfs/Makefile new file mode 100644 index 000000000..2b6bc5eb4 --- /dev/null +++ b/fs/bfs/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for BFS filesystem. +# + +obj-$(CONFIG_BFS_FS) += bfs.o + +bfs-objs := inode.o file.o dir.o diff --git a/fs/bfs/bfs.h b/fs/bfs/bfs.h new file mode 100644 index 000000000..b08afe733 --- /dev/null +++ b/fs/bfs/bfs.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * fs/bfs/bfs.h + * Copyright (C) 1999-2018 Tigran Aivazian + */ +#ifndef _FS_BFS_BFS_H +#define _FS_BFS_BFS_H + +#include + +/* In theory BFS supports up to 512 inodes, numbered from 2 (for /) up to 513 inclusive. + In actual fact, attempting to create the 512th inode (i.e. inode No. 513 or file No. 511) + will fail with ENOSPC in bfs_add_entry(): the root directory cannot contain so many entries, counting '..'. + So, mkfs.bfs(8) should really limit its -N option to 511 and not 512. For now, we just print a warning + if a filesystem is mounted with such "impossible to fill up" number of inodes */ +#define BFS_MAX_LASTI 513 + +/* + * BFS file system in-core superblock info + */ +struct bfs_sb_info { + unsigned long si_blocks; + unsigned long si_freeb; + unsigned long si_freei; + unsigned long si_lf_eblk; + unsigned long si_lasti; + DECLARE_BITMAP(si_imap, BFS_MAX_LASTI+1); + struct mutex bfs_lock; +}; + +/* + * BFS file system in-core inode info + */ +struct bfs_inode_info { + unsigned long i_dsk_ino; /* inode number from the disk, can be 0 */ + unsigned long i_sblock; + unsigned long i_eblock; + struct mapping_metadata_bhs i_metadata_bhs; + struct inode vfs_inode; +}; + +static inline struct bfs_sb_info *BFS_SB(struct super_block *sb) +{ + return sb->s_fs_info; +} + +static inline struct bfs_inode_info *BFS_I(struct inode *inode) +{ + return container_of(inode, struct bfs_inode_info, vfs_inode); +} + + +#define printf(format, args...) \ + printk(KERN_ERR "BFS-fs: %s(): " format, __func__, ## args) + +/* inode.c */ +extern struct inode *bfs_iget(struct super_block *sb, unsigned long ino); +extern void bfs_dump_imap(const char *, struct super_block *); + +/* file.c */ +extern const struct inode_operations bfs_file_inops; +extern const struct file_operations bfs_file_operations; +extern const struct address_space_operations bfs_aops; + +/* dir.c */ +extern const struct inode_operations bfs_dir_inops; +extern const struct file_operations bfs_dir_operations; + +#endif /* _FS_BFS_BFS_H */ diff --git a/fs/bfs/dir.c b/fs/bfs/dir.c new file mode 100644 index 000000000..91a4871fa --- /dev/null +++ b/fs/bfs/dir.c @@ -0,0 +1,356 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/bfs/dir.c + * BFS directory operations. + * Copyright (C) 1999-2018 Tigran Aivazian + * Made endianness-clean by Andrew Stribblehill 2005 + */ + +#include +#include +#include +#include +#include +#include "bfs.h" + +#undef DEBUG + +#ifdef DEBUG +#define dprintf(x...) printf(x) +#else +#define dprintf(x...) +#endif + +static int bfs_add_entry(struct inode *dir, const struct qstr *child, int ino); +static struct buffer_head *bfs_find_entry(struct inode *dir, + const struct qstr *child, + struct bfs_dirent **res_dir); + +static int bfs_readdir(struct file *f, struct dir_context *ctx) +{ + struct inode *dir = file_inode(f); + struct buffer_head *bh; + struct bfs_dirent *de; + unsigned int offset; + int block; + + if (ctx->pos & (BFS_DIRENT_SIZE - 1)) { + printf("Bad f_pos=%08lx for %s:%08llx\n", + (unsigned long)ctx->pos, + dir->i_sb->s_id, dir->i_ino); + return -EINVAL; + } + + while (ctx->pos < dir->i_size) { + offset = ctx->pos & (BFS_BSIZE - 1); + block = BFS_I(dir)->i_sblock + (ctx->pos >> BFS_BSIZE_BITS); + bh = sb_bread(dir->i_sb, block); + if (!bh) { + ctx->pos += BFS_BSIZE - offset; + continue; + } + do { + de = (struct bfs_dirent *)(bh->b_data + offset); + if (de->ino) { + int size = strnlen(de->name, BFS_NAMELEN); + if (!dir_emit(ctx, de->name, size, + le16_to_cpu(de->ino), + DT_UNKNOWN)) { + brelse(bh); + return 0; + } + } + offset += BFS_DIRENT_SIZE; + ctx->pos += BFS_DIRENT_SIZE; + } while ((offset < BFS_BSIZE) && (ctx->pos < dir->i_size)); + brelse(bh); + } + return 0; +} + +const struct file_operations bfs_dir_operations = { + .read = generic_read_dir, + .iterate_shared = bfs_readdir, + .fsync = simple_fsync, + .llseek = generic_file_llseek, +}; + +static int bfs_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + int err; + struct inode *inode; + struct super_block *s = dir->i_sb; + struct bfs_sb_info *info = BFS_SB(s); + unsigned long ino; + + inode = new_inode(s); + if (!inode) + return -ENOMEM; + mutex_lock(&info->bfs_lock); + ino = find_first_zero_bit(info->si_imap, info->si_lasti + 1); + if (ino > info->si_lasti) { + mutex_unlock(&info->bfs_lock); + iput(inode); + return -ENOSPC; + } + set_bit(ino, info->si_imap); + info->si_freei--; + inode_init_owner(&nop_mnt_idmap, inode, dir, mode); + simple_inode_init_ts(inode); + inode->i_blocks = 0; + inode->i_op = &bfs_file_inops; + inode->i_fop = &bfs_file_operations; + inode->i_mapping->a_ops = &bfs_aops; + inode->i_ino = ino; + BFS_I(inode)->i_dsk_ino = ino; + BFS_I(inode)->i_sblock = 0; + BFS_I(inode)->i_eblock = 0; + insert_inode_hash(inode); + mark_inode_dirty(inode); + bfs_dump_imap("create", s); + + err = bfs_add_entry(dir, &dentry->d_name, inode->i_ino); + if (err) { + inode_dec_link_count(inode); + mutex_unlock(&info->bfs_lock); + iput(inode); + return err; + } + mutex_unlock(&info->bfs_lock); + d_instantiate(dentry, inode); + return 0; +} + +static struct dentry *bfs_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + struct inode *inode = NULL; + struct buffer_head *bh; + struct bfs_dirent *de; + struct bfs_sb_info *info = BFS_SB(dir->i_sb); + + if (dentry->d_name.len > BFS_NAMELEN) + return ERR_PTR(-ENAMETOOLONG); + + mutex_lock(&info->bfs_lock); + bh = bfs_find_entry(dir, &dentry->d_name, &de); + if (bh) { + unsigned long ino = (unsigned long)le16_to_cpu(de->ino); + brelse(bh); + inode = bfs_iget(dir->i_sb, ino); + } + mutex_unlock(&info->bfs_lock); + return d_splice_alias(inode, dentry); +} + +static int bfs_link(struct dentry *old, struct inode *dir, + struct dentry *new) +{ + struct inode *inode = d_inode(old); + struct bfs_sb_info *info = BFS_SB(inode->i_sb); + int err; + + mutex_lock(&info->bfs_lock); + err = bfs_add_entry(dir, &new->d_name, inode->i_ino); + if (err) { + mutex_unlock(&info->bfs_lock); + return err; + } + inc_nlink(inode); + inode_set_ctime_current(inode); + mark_inode_dirty(inode); + ihold(inode); + d_instantiate(new, inode); + mutex_unlock(&info->bfs_lock); + return 0; +} + +static int bfs_unlink(struct inode *dir, struct dentry *dentry) +{ + int error = -ENOENT; + struct inode *inode = d_inode(dentry); + struct buffer_head *bh; + struct bfs_dirent *de; + struct bfs_sb_info *info = BFS_SB(inode->i_sb); + + mutex_lock(&info->bfs_lock); + bh = bfs_find_entry(dir, &dentry->d_name, &de); + if (!bh || (le16_to_cpu(de->ino) != inode->i_ino)) + goto out_brelse; + + if (!inode->i_nlink) { + printf("unlinking non-existent file %s:%llu (nlink=%d)\n", + inode->i_sb->s_id, inode->i_ino, + inode->i_nlink); + set_nlink(inode, 1); + } + de->ino = 0; + mmb_mark_buffer_dirty(bh, &BFS_I(dir)->i_metadata_bhs); + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + mark_inode_dirty(dir); + inode_set_ctime_to_ts(inode, inode_get_ctime(dir)); + inode_dec_link_count(inode); + error = 0; + +out_brelse: + brelse(bh); + mutex_unlock(&info->bfs_lock); + return error; +} + +static int bfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) +{ + struct inode *old_inode, *new_inode; + struct buffer_head *old_bh, *new_bh; + struct bfs_dirent *old_de, *new_de; + struct bfs_sb_info *info; + int error = -ENOENT; + + if (flags & ~RENAME_NOREPLACE) + return -EINVAL; + + old_bh = new_bh = NULL; + old_inode = d_inode(old_dentry); + if (S_ISDIR(old_inode->i_mode)) + return -EINVAL; + + info = BFS_SB(old_inode->i_sb); + + mutex_lock(&info->bfs_lock); + old_bh = bfs_find_entry(old_dir, &old_dentry->d_name, &old_de); + + if (!old_bh || (le16_to_cpu(old_de->ino) != old_inode->i_ino)) + goto end_rename; + + error = -EPERM; + new_inode = d_inode(new_dentry); + new_bh = bfs_find_entry(new_dir, &new_dentry->d_name, &new_de); + + if (new_bh && !new_inode) { + brelse(new_bh); + new_bh = NULL; + } + if (!new_bh) { + error = bfs_add_entry(new_dir, &new_dentry->d_name, + old_inode->i_ino); + if (error) + goto end_rename; + } + old_de->ino = 0; + inode_set_mtime_to_ts(old_dir, inode_set_ctime_current(old_dir)); + mark_inode_dirty(old_dir); + if (new_inode) { + inode_set_ctime_current(new_inode); + inode_dec_link_count(new_inode); + } + mmb_mark_buffer_dirty(old_bh, &BFS_I(old_dir)->i_metadata_bhs); + error = 0; + +end_rename: + mutex_unlock(&info->bfs_lock); + brelse(old_bh); + brelse(new_bh); + return error; +} + +const struct inode_operations bfs_dir_inops = { + .create = bfs_create, + .lookup = bfs_lookup, + .link = bfs_link, + .unlink = bfs_unlink, + .rename = bfs_rename, +}; + +static int bfs_add_entry(struct inode *dir, const struct qstr *child, int ino) +{ + const unsigned char *name = child->name; + int namelen = child->len; + struct buffer_head *bh; + struct bfs_dirent *de; + int block, sblock, eblock, off, pos; + int i; + + dprintf("name=%s, namelen=%d\n", name, namelen); + + sblock = BFS_I(dir)->i_sblock; + eblock = BFS_I(dir)->i_eblock; + for (block = sblock; block <= eblock; block++) { + bh = sb_bread(dir->i_sb, block); + if (!bh) + return -EIO; + for (off = 0; off < BFS_BSIZE; off += BFS_DIRENT_SIZE) { + de = (struct bfs_dirent *)(bh->b_data + off); + if (!de->ino) { + pos = (block - sblock) * BFS_BSIZE + off; + if (pos >= dir->i_size) { + dir->i_size += BFS_DIRENT_SIZE; + inode_set_ctime_current(dir); + } + inode_set_mtime_to_ts(dir, + inode_set_ctime_current(dir)); + mark_inode_dirty(dir); + de->ino = cpu_to_le16((u16)ino); + for (i = 0; i < BFS_NAMELEN; i++) + de->name[i] = + (i < namelen) ? name[i] : 0; + mmb_mark_buffer_dirty(bh, + &BFS_I(dir)->i_metadata_bhs); + brelse(bh); + return 0; + } + } + brelse(bh); + } + return -ENOSPC; +} + +static inline int bfs_namecmp(int len, const unsigned char *name, + const char *buffer) +{ + if ((len < BFS_NAMELEN) && buffer[len]) + return 0; + return !memcmp(name, buffer, len); +} + +static struct buffer_head *bfs_find_entry(struct inode *dir, + const struct qstr *child, + struct bfs_dirent **res_dir) +{ + unsigned long block = 0, offset = 0; + struct buffer_head *bh = NULL; + struct bfs_dirent *de; + const unsigned char *name = child->name; + int namelen = child->len; + + *res_dir = NULL; + if (namelen > BFS_NAMELEN) + return NULL; + + while (block * BFS_BSIZE + offset < dir->i_size) { + if (!bh) { + bh = sb_bread(dir->i_sb, BFS_I(dir)->i_sblock + block); + if (!bh) { + block++; + continue; + } + } + de = (struct bfs_dirent *)(bh->b_data + offset); + offset += BFS_DIRENT_SIZE; + if (le16_to_cpu(de->ino) && + bfs_namecmp(namelen, name, de->name)) { + *res_dir = de; + return bh; + } + if (offset < bh->b_size) + continue; + brelse(bh); + bh = NULL; + offset = 0; + block++; + } + brelse(bh); + return NULL; +} diff --git a/fs/bfs/file.c b/fs/bfs/file.c new file mode 100644 index 000000000..d33d6bde9 --- /dev/null +++ b/fs/bfs/file.c @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/bfs/file.c + * BFS file operations. + * Copyright (C) 1999-2018 Tigran Aivazian + * + * Make the file block allocation algorithm understand the size + * of the underlying block device. + * Copyright (C) 2007 Dmitri Vorobiev + * + */ + +#include +#include +#include +#include "bfs.h" + +#undef DEBUG + +#ifdef DEBUG +#define dprintf(x...) printf(x) +#else +#define dprintf(x...) +#endif + +const struct file_operations bfs_file_operations = { + .llseek = generic_file_llseek, + .read_iter = generic_file_read_iter, + .write_iter = generic_file_write_iter, + .mmap_prepare = generic_file_mmap_prepare, + .splice_read = filemap_splice_read, +}; + +static int bfs_move_block(unsigned long from, unsigned long to, + struct super_block *sb) +{ + struct buffer_head *bh, *new; + + bh = sb_bread(sb, from); + if (!bh) + return -EIO; + new = sb_getblk(sb, to); + memcpy(new->b_data, bh->b_data, bh->b_size); + mark_buffer_dirty(new); + bforget(bh); + brelse(new); + return 0; +} + +static int bfs_move_blocks(struct super_block *sb, unsigned long start, + unsigned long end, unsigned long where) +{ + unsigned long i; + + dprintf("%08lx-%08lx->%08lx\n", start, end, where); + for (i = start; i <= end; i++) + if(bfs_move_block(i, where + i, sb)) { + dprintf("failed to move block %08lx -> %08lx\n", i, + where + i); + return -EIO; + } + return 0; +} + +static int bfs_get_block(struct inode *inode, sector_t block, + struct buffer_head *bh_result, int create) +{ + unsigned long phys; + int err; + struct super_block *sb = inode->i_sb; + struct bfs_sb_info *info = BFS_SB(sb); + struct bfs_inode_info *bi = BFS_I(inode); + + phys = bi->i_sblock + block; + if (!create) { + if (phys <= bi->i_eblock) { + dprintf("c=%d, b=%08lx, phys=%09lx (granted)\n", + create, (unsigned long)block, phys); + map_bh(bh_result, sb, phys); + } + return 0; + } + + /* + * If the file is not empty and the requested block is within the + * range of blocks allocated for this file, we can grant it. + */ + if (bi->i_sblock && (phys <= bi->i_eblock)) { + dprintf("c=%d, b=%08lx, phys=%08lx (interim block granted)\n", + create, (unsigned long)block, phys); + map_bh(bh_result, sb, phys); + return 0; + } + + /* The file will be extended, so let's see if there is enough space. */ + if (phys >= info->si_blocks) + return -ENOSPC; + + /* The rest has to be protected against itself. */ + mutex_lock(&info->bfs_lock); + + /* + * If the last data block for this file is the last allocated + * block, we can extend the file trivially, without moving it + * anywhere. + */ + if (bi->i_eblock == info->si_lf_eblk) { + dprintf("c=%d, b=%08lx, phys=%08lx (simple extension)\n", + create, (unsigned long)block, phys); + map_bh(bh_result, sb, phys); + info->si_freeb -= phys - bi->i_eblock; + info->si_lf_eblk = bi->i_eblock = phys; + mark_inode_dirty(inode); + err = 0; + goto out; + } + + /* Ok, we have to move this entire file to the next free block. */ + phys = info->si_lf_eblk + 1; + if (phys + block >= info->si_blocks) { + err = -ENOSPC; + goto out; + } + + if (bi->i_sblock) { + err = bfs_move_blocks(inode->i_sb, bi->i_sblock, + bi->i_eblock, phys); + if (err) { + dprintf("failed to move ino=%08lx -> fs corruption\n", + inode->i_ino); + goto out; + } + } else + err = 0; + + dprintf("c=%d, b=%08lx, phys=%08lx (moved)\n", + create, (unsigned long)block, phys); + bi->i_sblock = phys; + phys += block; + info->si_lf_eblk = bi->i_eblock = phys; + + /* + * This assumes nothing can write the inode back while we are here + * and thus update inode->i_blocks! (XXX) + */ + info->si_freeb -= bi->i_eblock - bi->i_sblock + 1 - inode->i_blocks; + mark_inode_dirty(inode); + map_bh(bh_result, sb, phys); +out: + mutex_unlock(&info->bfs_lock); + return err; +} + +static int bfs_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + return mpage_writepages(mapping, wbc, bfs_get_block); +} + +static int bfs_read_folio(struct file *file, struct folio *folio) +{ + return block_read_full_folio(folio, bfs_get_block); +} + +static void bfs_write_failed(struct address_space *mapping, loff_t to) +{ + struct inode *inode = mapping->host; + + if (to > inode->i_size) + truncate_pagecache(inode, inode->i_size); +} + +static int bfs_write_begin(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, + struct folio **foliop, void **fsdata) +{ + int ret; + + ret = block_write_begin(mapping, pos, len, foliop, bfs_get_block); + if (unlikely(ret)) + bfs_write_failed(mapping, pos + len); + + return ret; +} + +static sector_t bfs_bmap(struct address_space *mapping, sector_t block) +{ + return generic_block_bmap(mapping, block, bfs_get_block); +} + +const struct address_space_operations bfs_aops = { + .dirty_folio = block_dirty_folio, + .invalidate_folio = block_invalidate_folio, + .read_folio = bfs_read_folio, + .writepages = bfs_writepages, + .write_begin = bfs_write_begin, + .write_end = generic_write_end, + .migrate_folio = buffer_migrate_folio, + .bmap = bfs_bmap, +}; + +const struct inode_operations bfs_file_inops; diff --git a/fs/bfs/inode.c b/fs/bfs/inode.c new file mode 100644 index 000000000..06e3a848b --- /dev/null +++ b/fs/bfs/inode.c @@ -0,0 +1,538 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * fs/bfs/inode.c + * BFS superblock and inode operations. + * Copyright (C) 1999-2018 Tigran Aivazian + * From fs/minix, Copyright (C) 1991, 1992 Linus Torvalds. + * Made endianness-clean by Andrew Stribblehill , 2005. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "bfs.h" + +MODULE_AUTHOR("Tigran Aivazian "); +MODULE_DESCRIPTION("SCO UnixWare BFS filesystem for Linux"); +MODULE_LICENSE("GPL"); + +#undef DEBUG + +#ifdef DEBUG +#define dprintf(x...) printf(x) +#else +#define dprintf(x...) +#endif + +struct inode *bfs_iget(struct super_block *sb, unsigned long ino) +{ + struct bfs_inode *di; + struct inode *inode; + struct buffer_head *bh; + int block, off; + + inode = iget_locked(sb, ino); + if (!inode) + return ERR_PTR(-ENOMEM); + if (!(inode_state_read_once(inode) & I_NEW)) + return inode; + + if ((ino < BFS_ROOT_INO) || (ino > BFS_SB(inode->i_sb)->si_lasti)) { + printf("Bad inode number %s:%08lx\n", inode->i_sb->s_id, ino); + goto error; + } + + block = (ino - BFS_ROOT_INO) / BFS_INODES_PER_BLOCK + 1; + bh = sb_bread(inode->i_sb, block); + if (!bh) { + printf("Unable to read inode %s:%08lx\n", inode->i_sb->s_id, + ino); + goto error; + } + + off = (ino - BFS_ROOT_INO) % BFS_INODES_PER_BLOCK; + di = (struct bfs_inode *)bh->b_data + off; + + /* + * https://martin.hinner.info/fs/bfs/bfs-structure.html explains that + * BFS in SCO UnixWare environment used only lower 9 bits of di->i_mode + * value. This means that, although bfs_write_inode() saves whole + * inode->i_mode bits (which include S_IFMT bits and S_IS{UID,GID,VTX} + * bits), middle 7 bits of di->i_mode value can be garbage when these + * bits were not saved by bfs_write_inode(). + * Since we can't tell whether middle 7 bits are garbage, use only + * lower 12 bits (i.e. tolerate S_IS{UID,GID,VTX} bits possibly being + * garbage) and reconstruct S_IFMT bits for Linux environment from + * di->i_vtype value. + */ + inode->i_mode = 0x00000FFF & le32_to_cpu(di->i_mode); + if (le32_to_cpu(di->i_vtype) == BFS_VDIR) { + inode->i_mode |= S_IFDIR; + inode->i_op = &bfs_dir_inops; + inode->i_fop = &bfs_dir_operations; + } else if (le32_to_cpu(di->i_vtype) == BFS_VREG) { + inode->i_mode |= S_IFREG; + inode->i_op = &bfs_file_inops; + inode->i_fop = &bfs_file_operations; + inode->i_mapping->a_ops = &bfs_aops; + } else { + brelse(bh); + printf("Unknown vtype=%u %s:%08lx\n", + le32_to_cpu(di->i_vtype), inode->i_sb->s_id, ino); + goto error; + } + + BFS_I(inode)->i_sblock = le32_to_cpu(di->i_sblock); + BFS_I(inode)->i_eblock = le32_to_cpu(di->i_eblock); + BFS_I(inode)->i_dsk_ino = le16_to_cpu(di->i_ino); + i_uid_write(inode, le32_to_cpu(di->i_uid)); + i_gid_write(inode, le32_to_cpu(di->i_gid)); + set_nlink(inode, le32_to_cpu(di->i_nlink)); + inode->i_size = BFS_FILESIZE(di); + inode->i_blocks = BFS_FILEBLOCKS(di); + inode_set_atime(inode, le32_to_cpu(di->i_atime), 0); + inode_set_mtime(inode, le32_to_cpu(di->i_mtime), 0); + inode_set_ctime(inode, le32_to_cpu(di->i_ctime), 0); + + brelse(bh); + unlock_new_inode(inode); + return inode; + +error: + iget_failed(inode); + return ERR_PTR(-EIO); +} + +static struct bfs_inode *find_inode(struct super_block *sb, u16 ino, struct buffer_head **p) +{ + if ((ino < BFS_ROOT_INO) || (ino > BFS_SB(sb)->si_lasti)) { + printf("Bad inode number %s:%08x\n", sb->s_id, ino); + return ERR_PTR(-EIO); + } + + ino -= BFS_ROOT_INO; + + *p = sb_bread(sb, 1 + ino / BFS_INODES_PER_BLOCK); + if (!*p) { + printf("Unable to read inode %s:%08x\n", sb->s_id, ino); + return ERR_PTR(-EIO); + } + + return (struct bfs_inode *)(*p)->b_data + ino % BFS_INODES_PER_BLOCK; +} + +static int bfs_write_inode(struct inode *inode, struct writeback_control *wbc) +{ + struct bfs_sb_info *info = BFS_SB(inode->i_sb); + unsigned int ino = (u16)inode->i_ino; + unsigned long i_sblock; + struct bfs_inode *di; + struct buffer_head *bh; + + dprintf("ino=%08x\n", ino); + + di = find_inode(inode->i_sb, ino, &bh); + if (IS_ERR(di)) + return PTR_ERR(di); + + mutex_lock(&info->bfs_lock); + + if (ino == BFS_ROOT_INO) + di->i_vtype = cpu_to_le32(BFS_VDIR); + else + di->i_vtype = cpu_to_le32(BFS_VREG); + + di->i_ino = cpu_to_le16(ino); + di->i_mode = cpu_to_le32(inode->i_mode); + di->i_uid = cpu_to_le32(i_uid_read(inode)); + di->i_gid = cpu_to_le32(i_gid_read(inode)); + di->i_nlink = cpu_to_le32(inode->i_nlink); + di->i_atime = cpu_to_le32(inode_get_atime_sec(inode)); + di->i_mtime = cpu_to_le32(inode_get_mtime_sec(inode)); + di->i_ctime = cpu_to_le32(inode_get_ctime_sec(inode)); + i_sblock = BFS_I(inode)->i_sblock; + di->i_sblock = cpu_to_le32(i_sblock); + di->i_eblock = cpu_to_le32(BFS_I(inode)->i_eblock); + di->i_eoffset = cpu_to_le32(i_sblock * BFS_BSIZE + inode->i_size - 1); + + mark_buffer_dirty(bh); + brelse(bh); + mutex_unlock(&info->bfs_lock); + set_inode_metadata_writeback(inode); + return 0; +} + +static int bfs_sync_inode_metadata(struct inode *inode, + struct writeback_control *wbc) +{ + int err = 0; + struct bfs_inode *di; + struct buffer_head *bh; + + di = find_inode(inode->i_sb, (u16)inode->i_ino, &bh); + if (IS_ERR(di)) + return PTR_ERR(di); + + sync_dirty_buffer(bh); + if (buffer_write_io_error(bh)) { + err = -EIO; + goto out; + } + err = mmb_sync(&BFS_I(inode)->i_metadata_bhs); +out: + brelse(bh); + return err; +} + +static void bfs_evict_inode(struct inode *inode) +{ + unsigned long ino = inode->i_ino; + struct bfs_inode *di; + struct buffer_head *bh; + struct super_block *s = inode->i_sb; + struct bfs_sb_info *info = BFS_SB(s); + struct bfs_inode_info *bi = BFS_I(inode); + + dprintf("ino=%08lx\n", ino); + + truncate_inode_pages_final(&inode->i_data); + if (inode->i_nlink) + mmb_sync(&BFS_I(inode)->i_metadata_bhs); + mmb_invalidate(&BFS_I(inode)->i_metadata_bhs); + clear_inode(inode); + + if (inode->i_nlink) + return; + + di = find_inode(s, inode->i_ino, &bh); + if (IS_ERR(di)) + return; + + mutex_lock(&info->bfs_lock); + /* clear on-disk inode */ + memset(di, 0, sizeof(struct bfs_inode)); + mark_buffer_dirty(bh); + brelse(bh); + + if (bi->i_dsk_ino) { + if (bi->i_sblock) + info->si_freeb += bi->i_eblock + 1 - bi->i_sblock; + info->si_freei++; + clear_bit(ino, info->si_imap); + bfs_dump_imap("evict_inode", s); + } + + /* + * If this was the last file, make the previous block + * "last block of the last file" even if there is no + * real file there, saves us 1 gap. + */ + if (info->si_lf_eblk == bi->i_eblock) + info->si_lf_eblk = bi->i_sblock - 1; + mutex_unlock(&info->bfs_lock); +} + +static void bfs_put_super(struct super_block *s) +{ + struct bfs_sb_info *info = BFS_SB(s); + + if (!info) + return; + + mutex_destroy(&info->bfs_lock); + kfree(info); + s->s_fs_info = NULL; +} + +static int bfs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *s = dentry->d_sb; + struct bfs_sb_info *info = BFS_SB(s); + u64 id = huge_encode_dev(s->s_bdev->bd_dev); + buf->f_type = BFS_MAGIC; + buf->f_bsize = s->s_blocksize; + buf->f_blocks = info->si_blocks; + buf->f_bfree = buf->f_bavail = info->si_freeb; + buf->f_files = info->si_lasti + 1 - BFS_ROOT_INO; + buf->f_ffree = info->si_freei; + buf->f_fsid = u64_to_fsid(id); + buf->f_namelen = BFS_NAMELEN; + return 0; +} + +static struct kmem_cache *bfs_inode_cachep; + +static struct inode *bfs_alloc_inode(struct super_block *sb) +{ + struct bfs_inode_info *bi; + bi = alloc_inode_sb(sb, bfs_inode_cachep, GFP_KERNEL); + if (!bi) + return NULL; + mmb_init(&bi->i_metadata_bhs, &bi->vfs_inode.i_data); + + return &bi->vfs_inode; +} + +static void bfs_free_inode(struct inode *inode) +{ + kmem_cache_free(bfs_inode_cachep, BFS_I(inode)); +} + +static void init_once(void *foo) +{ + struct bfs_inode_info *bi = foo; + + inode_init_once(&bi->vfs_inode); +} + +static int __init init_inodecache(void) +{ + bfs_inode_cachep = kmem_cache_create("bfs_inode_cache", + sizeof(struct bfs_inode_info), + 0, (SLAB_RECLAIM_ACCOUNT| + SLAB_ACCOUNT), + init_once); + if (bfs_inode_cachep == NULL) + return -ENOMEM; + return 0; +} + +static void destroy_inodecache(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(bfs_inode_cachep); +} + +static const struct super_operations bfs_sops = { + .alloc_inode = bfs_alloc_inode, + .free_inode = bfs_free_inode, + .write_inode = bfs_write_inode, + .sync_inode_metadata = bfs_sync_inode_metadata, + .evict_inode = bfs_evict_inode, + .put_super = bfs_put_super, + .statfs = bfs_statfs, +}; + +void bfs_dump_imap(const char *prefix, struct super_block *s) +{ +#ifdef DEBUG + int i; + char *tmpbuf = kzalloc(PAGE_SIZE, GFP_KERNEL); + + if (!tmpbuf) + return; + for (i = BFS_SB(s)->si_lasti; i >= 0; i--) { + if (i > PAGE_SIZE - 100) break; + if (test_bit(i, BFS_SB(s)->si_imap)) + strcat(tmpbuf, "1"); + else + strcat(tmpbuf, "0"); + } + printf("%s: lasti=%08lx <%s>\n", prefix, BFS_SB(s)->si_lasti, tmpbuf); + kfree(tmpbuf); +#endif +} + +static int bfs_fill_super(struct super_block *s, struct fs_context *fc) +{ + struct buffer_head *bh, *sbh; + struct bfs_super_block *bfs_sb; + struct inode *inode; + unsigned i; + struct bfs_sb_info *info; + int ret = -EINVAL; + unsigned long i_sblock, i_eblock, i_eoff, s_size; + int silent = fc->sb_flags & SB_SILENT; + + info = kzalloc_obj(*info); + if (!info) + return -ENOMEM; + mutex_init(&info->bfs_lock); + s->s_fs_info = info; + s->s_time_min = 0; + s->s_time_max = U32_MAX; + + if (!sb_set_blocksize(s, BFS_BSIZE)) + goto out; + + sbh = sb_bread(s, 0); + if (!sbh) + goto out; + bfs_sb = (struct bfs_super_block *)sbh->b_data; + if (le32_to_cpu(bfs_sb->s_magic) != BFS_MAGIC) { + if (!silent) + printf("No BFS filesystem on %s (magic=%08x)\n", s->s_id, le32_to_cpu(bfs_sb->s_magic)); + goto out1; + } + if (BFS_UNCLEAN(bfs_sb, s) && !silent) + printf("%s is unclean, continuing\n", s->s_id); + + s->s_magic = BFS_MAGIC; + + if (le32_to_cpu(bfs_sb->s_start) > le32_to_cpu(bfs_sb->s_end) || + le32_to_cpu(bfs_sb->s_start) < sizeof(struct bfs_super_block) + sizeof(struct bfs_dirent)) { + printf("Superblock is corrupted on %s\n", s->s_id); + goto out1; + } + + info->si_lasti = (le32_to_cpu(bfs_sb->s_start) - BFS_BSIZE) / sizeof(struct bfs_inode) + BFS_ROOT_INO - 1; + if (info->si_lasti == BFS_MAX_LASTI) + printf("NOTE: filesystem %s was created with 512 inodes, the real maximum is 511, mounting anyway\n", s->s_id); + else if (info->si_lasti > BFS_MAX_LASTI) { + printf("Impossible last inode number %lu > %d on %s\n", info->si_lasti, BFS_MAX_LASTI, s->s_id); + goto out1; + } + for (i = 0; i < BFS_ROOT_INO; i++) + set_bit(i, info->si_imap); + + s->s_op = &bfs_sops; + inode = bfs_iget(s, BFS_ROOT_INO); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + goto out1; + } + s->s_root = d_make_root(inode); + if (!s->s_root) { + ret = -ENOMEM; + goto out1; + } + + info->si_blocks = (le32_to_cpu(bfs_sb->s_end) + 1) >> BFS_BSIZE_BITS; + info->si_freeb = (le32_to_cpu(bfs_sb->s_end) + 1 - le32_to_cpu(bfs_sb->s_start)) >> BFS_BSIZE_BITS; + info->si_freei = 0; + info->si_lf_eblk = 0; + + /* can we read the last block? */ + bh = sb_bread(s, info->si_blocks - 1); + if (!bh) { + printf("Last block not available on %s: %lu\n", s->s_id, info->si_blocks - 1); + ret = -EIO; + goto out2; + } + brelse(bh); + + bh = NULL; + for (i = BFS_ROOT_INO; i <= info->si_lasti; i++) { + struct bfs_inode *di; + int block = (i - BFS_ROOT_INO) / BFS_INODES_PER_BLOCK + 1; + int off = (i - BFS_ROOT_INO) % BFS_INODES_PER_BLOCK; + unsigned long eblock; + + if (!off) { + brelse(bh); + bh = sb_bread(s, block); + } + + if (!bh) + continue; + + di = (struct bfs_inode *)bh->b_data + off; + + /* test if filesystem is not corrupted */ + + i_eoff = le32_to_cpu(di->i_eoffset); + i_sblock = le32_to_cpu(di->i_sblock); + i_eblock = le32_to_cpu(di->i_eblock); + s_size = le32_to_cpu(bfs_sb->s_end); + + if (i_sblock > info->si_blocks || + i_eblock > info->si_blocks || + i_sblock > i_eblock || + (i_eoff != le32_to_cpu(-1) && i_eoff > s_size) || + i_sblock * BFS_BSIZE > i_eoff) { + + printf("Inode 0x%08x corrupted on %s\n", i, s->s_id); + + brelse(bh); + ret = -EIO; + goto out2; + } + + if (!di->i_ino) { + info->si_freei++; + continue; + } + set_bit(i, info->si_imap); + info->si_freeb -= BFS_FILEBLOCKS(di); + + eblock = le32_to_cpu(di->i_eblock); + if (eblock > info->si_lf_eblk) + info->si_lf_eblk = eblock; + } + brelse(bh); + brelse(sbh); + bfs_dump_imap("fill_super", s); + return 0; + +out2: + dput(s->s_root); + s->s_root = NULL; +out1: + brelse(sbh); +out: + mutex_destroy(&info->bfs_lock); + kfree(info); + s->s_fs_info = NULL; + return ret; +} + +static int bfs_get_tree(struct fs_context *fc) +{ + return get_tree_bdev(fc, bfs_fill_super); +} + +static const struct fs_context_operations bfs_context_ops = { + .get_tree = bfs_get_tree, +}; + +static int bfs_init_fs_context(struct fs_context *fc) +{ + fc->ops = &bfs_context_ops; + + return 0; +} + +static struct file_system_type bfs_fs_type = { + .owner = THIS_MODULE, + .name = "bfs", + .init_fs_context = bfs_init_fs_context, + .kill_sb = kill_block_super, + .fs_flags = FS_REQUIRES_DEV, +}; +MODULE_ALIAS_FS("bfs"); + +static int __init init_bfs_fs(void) +{ + int err = init_inodecache(); + if (err) + goto out1; + err = register_filesystem(&bfs_fs_type); + if (err) + goto out; + return 0; +out: + destroy_inodecache(); +out1: + return err; +} + +static void __exit exit_bfs_fs(void) +{ + unregister_filesystem(&bfs_fs_type); + destroy_inodecache(); +} + +module_init(init_bfs_fs) +module_exit(exit_bfs_fs) diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c new file mode 100644 index 000000000..06d0df105 --- /dev/null +++ b/fs/binfmt_elf.c @@ -0,0 +1,2144 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/binfmt_elf.c + * + * These are the functions used to load ELF format executables as used + * on SVr4 machines. Information on the format may be found in the book + * "UNIX SYSTEM V RELEASE 4 Programmers Guide: Ansi C and Programming Support + * Tools". + * + * Copyright 1993, 1994: Eric Youngdale (ericy@cais.com). + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef ELF_COMPAT +#define ELF_COMPAT 0 +#endif + +#ifndef user_long_t +#define user_long_t long +#endif +#ifndef user_siginfo_t +#define user_siginfo_t siginfo_t +#endif + +/* That's for binfmt_elf_fdpic to deal with */ +#ifndef elf_check_fdpic +#define elf_check_fdpic(ex) false +#endif + +static int load_elf_binary(struct linux_binprm *bprm); + +/* + * If we don't support core dumping, then supply a NULL so we + * don't even try. + */ +#ifdef CONFIG_ELF_CORE +static int elf_core_dump(struct coredump_params *cprm); +#else +#define elf_core_dump NULL +#endif + +#if ELF_EXEC_PAGESIZE > PAGE_SIZE +#define ELF_MIN_ALIGN ELF_EXEC_PAGESIZE +#else +#define ELF_MIN_ALIGN PAGE_SIZE +#endif + +#ifndef ELF_CORE_EFLAGS +#define ELF_CORE_EFLAGS 0 +#endif + +#define ELF_PAGESTART(_v) ((_v) & ~(int)(ELF_MIN_ALIGN-1)) +#define ELF_PAGEOFFSET(_v) ((_v) & (ELF_MIN_ALIGN-1)) +#define ELF_PAGEALIGN(_v) (((_v) + ELF_MIN_ALIGN - 1) & ~(ELF_MIN_ALIGN - 1)) + +static struct linux_binfmt elf_format = { + .module = THIS_MODULE, + .load_binary = load_elf_binary, +#ifdef CONFIG_COREDUMP + .core_dump = elf_core_dump, + .min_coredump = ELF_EXEC_PAGESIZE, +#endif +}; + +#define BAD_ADDR(x) (unlikely((unsigned long)(x) >= TASK_SIZE)) + +static inline void elf_coredump_set_mm_eflags(struct mm_struct *mm, u32 flags) +{ +#ifdef CONFIG_ARCH_HAS_ELF_CORE_EFLAGS + mm->saved_e_flags = flags; +#endif +} + +static inline u32 elf_coredump_get_mm_eflags(struct mm_struct *mm, u32 flags) +{ +#ifdef CONFIG_ARCH_HAS_ELF_CORE_EFLAGS + flags = mm->saved_e_flags; +#endif + return flags; +} + +/* + * We need to explicitly zero any trailing portion of the page that follows + * p_filesz when it ends before the page ends (e.g. bss), otherwise this + * memory will contain the junk from the file that should not be present. + */ +static int padzero(unsigned long address) +{ + unsigned long nbyte; + + nbyte = ELF_PAGEOFFSET(address); + if (nbyte) { + nbyte = ELF_MIN_ALIGN - nbyte; + if (clear_user((void __user *)address, nbyte)) + return -EFAULT; + } + return 0; +} + +/* Let's use some macros to make this stack manipulation a little clearer */ +#ifdef CONFIG_STACK_GROWSUP +#define STACK_ADD(sp, items) ((elf_addr_t __user *)(sp) + (items)) +#define STACK_ROUND(sp, items) \ + ((15 + (unsigned long) ((sp) + (items))) &~ 15UL) +#define STACK_ALLOC(sp, len) ({ \ + elf_addr_t __user *old_sp = (elf_addr_t __user *)sp; sp += len; \ + old_sp; }) +#else +#define STACK_ADD(sp, items) ((elf_addr_t __user *)(sp) - (items)) +#define STACK_ROUND(sp, items) \ + (((unsigned long) (sp - items)) &~ 15UL) +#define STACK_ALLOC(sp, len) (sp -= len) +#endif + +#ifndef ELF_BASE_PLATFORM +/* + * AT_BASE_PLATFORM indicates the "real" hardware/microarchitecture. + * If the arch defines ELF_BASE_PLATFORM (in asm/elf.h), the value + * will be copied to the user stack in the same manner as AT_PLATFORM. + */ +#define ELF_BASE_PLATFORM NULL +#endif + +static int +create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec, + unsigned long interp_load_addr, + unsigned long e_entry, unsigned long phdr_addr) +{ + struct mm_struct *mm = current->mm; + unsigned long p = bprm->p; + int argc = bprm->argc; + int envc = bprm->envc; + elf_addr_t __user *sp; + elf_addr_t __user *u_platform; + elf_addr_t __user *u_base_platform; + elf_addr_t __user *u_rand_bytes; + const char *k_platform = ELF_PLATFORM; + const char *k_base_platform = ELF_BASE_PLATFORM; + unsigned char k_rand_bytes[16]; + int items; + elf_addr_t *elf_info; + int ei_index; + const struct cred *cred = current_cred(); + struct vm_area_struct *vma; + + /* + * In some cases (e.g. Hyper-Threading), we want to avoid L1 + * evictions by the processes running on the same package. One + * thing we can do is to shuffle the initial stack for them. + */ + + p = arch_align_stack(p); + + /* + * If this architecture has a platform capability string, copy it + * to userspace. In some cases (Sparc), this info is impossible + * for userspace to get any other way, in others (i386) it is + * merely difficult. + */ + u_platform = NULL; + if (k_platform) { + size_t len = strlen(k_platform) + 1; + + u_platform = (elf_addr_t __user *)STACK_ALLOC(p, len); + if (copy_to_user(u_platform, k_platform, len)) + return -EFAULT; + } + + /* + * If this architecture has a "base" platform capability + * string, copy it to userspace. + */ + u_base_platform = NULL; + if (k_base_platform) { + size_t len = strlen(k_base_platform) + 1; + + u_base_platform = (elf_addr_t __user *)STACK_ALLOC(p, len); + if (copy_to_user(u_base_platform, k_base_platform, len)) + return -EFAULT; + } + + /* + * Generate 16 random bytes for userspace PRNG seeding. + */ + get_random_bytes(k_rand_bytes, sizeof(k_rand_bytes)); + u_rand_bytes = (elf_addr_t __user *) + STACK_ALLOC(p, sizeof(k_rand_bytes)); + if (copy_to_user(u_rand_bytes, k_rand_bytes, sizeof(k_rand_bytes))) + return -EFAULT; + + /* Create the ELF interpreter info */ + elf_info = (elf_addr_t *)mm->saved_auxv; + /* update AT_VECTOR_SIZE_BASE if the number of NEW_AUX_ENT() changes */ +#define NEW_AUX_ENT(id, val) \ + do { \ + *elf_info++ = id; \ + *elf_info++ = val; \ + } while (0) + +#ifdef ARCH_DLINFO + /* + * ARCH_DLINFO must come first so PPC can do its special alignment of + * AUXV. + * update AT_VECTOR_SIZE_ARCH if the number of NEW_AUX_ENT() in + * ARCH_DLINFO changes + */ + ARCH_DLINFO; +#endif + NEW_AUX_ENT(AT_HWCAP, ELF_HWCAP); + NEW_AUX_ENT(AT_PAGESZ, ELF_EXEC_PAGESIZE); + NEW_AUX_ENT(AT_CLKTCK, CLOCKS_PER_SEC); + NEW_AUX_ENT(AT_PHDR, phdr_addr); + NEW_AUX_ENT(AT_PHENT, sizeof(struct elf_phdr)); + NEW_AUX_ENT(AT_PHNUM, exec->e_phnum); + NEW_AUX_ENT(AT_BASE, interp_load_addr); + NEW_AUX_ENT(AT_FLAGS, bprm_at_flags(bprm)); + NEW_AUX_ENT(AT_ENTRY, e_entry); + NEW_AUX_ENT(AT_UID, from_kuid_munged(cred->user_ns, cred->uid)); + NEW_AUX_ENT(AT_EUID, from_kuid_munged(cred->user_ns, cred->euid)); + NEW_AUX_ENT(AT_GID, from_kgid_munged(cred->user_ns, cred->gid)); + NEW_AUX_ENT(AT_EGID, from_kgid_munged(cred->user_ns, cred->egid)); + NEW_AUX_ENT(AT_SECURE, bprm->secureexec); + NEW_AUX_ENT(AT_RANDOM, (elf_addr_t)(unsigned long)u_rand_bytes); +#ifdef ELF_HWCAP2 + NEW_AUX_ENT(AT_HWCAP2, ELF_HWCAP2); +#endif +#ifdef ELF_HWCAP3 + NEW_AUX_ENT(AT_HWCAP3, ELF_HWCAP3); +#endif +#ifdef ELF_HWCAP4 + NEW_AUX_ENT(AT_HWCAP4, ELF_HWCAP4); +#endif + NEW_AUX_ENT(AT_EXECFN, bprm->exec); + if (k_platform) { + NEW_AUX_ENT(AT_PLATFORM, + (elf_addr_t)(unsigned long)u_platform); + } + if (k_base_platform) { + NEW_AUX_ENT(AT_BASE_PLATFORM, + (elf_addr_t)(unsigned long)u_base_platform); + } + if (bprm->have_execfd) { + NEW_AUX_ENT(AT_EXECFD, bprm->execfd); + } +#ifdef CONFIG_RSEQ + NEW_AUX_ENT(AT_RSEQ_FEATURE_SIZE, offsetof(struct rseq, end)); + NEW_AUX_ENT(AT_RSEQ_ALIGN, rseq_alloc_align()); +#endif +#undef NEW_AUX_ENT + /* AT_NULL is zero; clear the rest too */ + memset(elf_info, 0, (char *)mm->saved_auxv + + sizeof(mm->saved_auxv) - (char *)elf_info); + + /* And advance past the AT_NULL entry. */ + elf_info += 2; + + ei_index = elf_info - (elf_addr_t *)mm->saved_auxv; + sp = STACK_ADD(p, ei_index); + + items = (argc + 1) + (envc + 1) + 1; + bprm->p = STACK_ROUND(sp, items); + + /* Point sp at the lowest address on the stack */ +#ifdef CONFIG_STACK_GROWSUP + sp = (elf_addr_t __user *)bprm->p - items - ei_index; + bprm->exec = (unsigned long)sp; /* XXX: PARISC HACK */ +#else + sp = (elf_addr_t __user *)bprm->p; +#endif + + + /* + * Grow the stack manually; some architectures have a limit on how + * far ahead a user-space access may be in order to grow the stack. + */ + if (mmap_write_lock_killable(mm)) + return -EINTR; + vma = find_extend_vma_locked(mm, bprm->p); + mmap_write_unlock(mm); + if (!vma) + return -EFAULT; + + /* Now, let's put argc (and argv, envp if appropriate) on the stack */ + if (put_user(argc, sp++)) + return -EFAULT; + + /* Populate list of argv pointers back to argv strings. */ + p = mm->arg_end = mm->arg_start; + while (argc-- > 0) { + size_t len; + if (put_user((elf_addr_t)p, sp++)) + return -EFAULT; + len = strnlen_user((void __user *)p, MAX_ARG_STRLEN); + if (!len || len > MAX_ARG_STRLEN) + return -EINVAL; + p += len; + } + if (put_user(0, sp++)) + return -EFAULT; + mm->arg_end = p; + + /* Populate list of envp pointers back to envp strings. */ + mm->env_end = mm->env_start = p; + while (envc-- > 0) { + size_t len; + if (put_user((elf_addr_t)p, sp++)) + return -EFAULT; + len = strnlen_user((void __user *)p, MAX_ARG_STRLEN); + if (!len || len > MAX_ARG_STRLEN) + return -EINVAL; + p += len; + } + if (put_user(0, sp++)) + return -EFAULT; + mm->env_end = p; + + /* Put the elf_info on the stack in the right place. */ + if (copy_to_user(sp, mm->saved_auxv, ei_index * sizeof(elf_addr_t))) + return -EFAULT; + return 0; +} + +/* + * Map "eppnt->p_filesz" bytes from "filep" offset "eppnt->p_offset" + * into memory at "addr". (Note that p_filesz is rounded up to the + * next page, so any extra bytes from the file must be wiped.) + */ +static unsigned long elf_map(struct file *filep, unsigned long addr, + const struct elf_phdr *eppnt, int prot, int type, + unsigned long total_size) +{ + unsigned long map_addr; + unsigned long size = eppnt->p_filesz + ELF_PAGEOFFSET(eppnt->p_vaddr); + unsigned long off = eppnt->p_offset - ELF_PAGEOFFSET(eppnt->p_vaddr); + addr = ELF_PAGESTART(addr); + size = ELF_PAGEALIGN(size); + + /* mmap() will return -EINVAL if given a zero size, but a + * segment with zero filesize is perfectly valid */ + if (!size) + return addr; + + /* + * total_size is the size of the ELF (interpreter) image. + * The _first_ mmap needs to know the full size, otherwise + * randomization might put this image into an overlapping + * position with the ELF binary image. (since size < total_size) + * So we first map the 'big' image - and unmap the remainder at + * the end. (which unmap is needed for ELF images with holes.) + */ + if (total_size) { + total_size = ELF_PAGEALIGN(total_size); + map_addr = vm_mmap(filep, addr, total_size, prot, type, off); + if (!BAD_ADDR(map_addr)) + vm_munmap(map_addr+size, total_size-size); + } else + map_addr = vm_mmap(filep, addr, size, prot, type, off); + + if ((type & MAP_FIXED_NOREPLACE) && + PTR_ERR((void *)map_addr) == -EEXIST) + pr_info("%d (%s): Uhuuh, elf segment at %px requested but the memory is mapped already\n", + task_pid_nr(current), current->comm, (void *)addr); + + return(map_addr); +} + +/* + * Map "eppnt->p_filesz" bytes from "filep" offset "eppnt->p_offset" + * into memory at "addr". Memory from "p_filesz" through "p_memsz" + * rounded up to the next page is zeroed. + */ +static unsigned long elf_load(struct file *filep, unsigned long addr, + const struct elf_phdr *eppnt, int prot, int type, + unsigned long total_size) +{ + unsigned long zero_start, zero_end; + unsigned long map_addr; + + if (eppnt->p_filesz) { + map_addr = elf_map(filep, addr, eppnt, prot, type, total_size); + if (BAD_ADDR(map_addr)) + return map_addr; + if (eppnt->p_memsz > eppnt->p_filesz) { + zero_start = map_addr + ELF_PAGEOFFSET(eppnt->p_vaddr) + + eppnt->p_filesz; + zero_end = map_addr + ELF_PAGEOFFSET(eppnt->p_vaddr) + + eppnt->p_memsz; + + /* + * Zero the end of the last mapped page but ignore + * any errors if the segment isn't writable. + */ + if (padzero(zero_start) && (prot & PROT_WRITE)) + return -EFAULT; + } + } else { + map_addr = zero_start = ELF_PAGESTART(addr); + zero_end = zero_start + ELF_PAGEOFFSET(eppnt->p_vaddr) + + eppnt->p_memsz; + } + if (eppnt->p_memsz > eppnt->p_filesz) { + /* + * Map the last of the segment. + * If the header is requesting these pages to be + * executable, honour that (ppc32 needs this). + */ + int error; + + zero_start = ELF_PAGEALIGN(zero_start); + zero_end = ELF_PAGEALIGN(zero_end); + + error = vm_brk_flags(zero_start, zero_end - zero_start, + prot & PROT_EXEC); + if (error) + map_addr = error; + } + return map_addr; +} + +static unsigned long total_mapping_size(const struct elf_phdr *phdr, int nr) +{ + elf_addr_t min_addr = -1; + elf_addr_t max_addr = 0; + bool pt_load = false; + int i; + + for (i = 0; i < nr; i++) { + if (phdr[i].p_type == PT_LOAD) { + min_addr = min(min_addr, ELF_PAGESTART(phdr[i].p_vaddr)); + max_addr = max(max_addr, phdr[i].p_vaddr + phdr[i].p_memsz); + pt_load = true; + } + } + return pt_load ? (max_addr - min_addr) : 0; +} + +static int elf_read(struct file *file, void *buf, size_t len, loff_t pos) +{ + ssize_t rv; + + rv = kernel_read(file, buf, len, &pos); + if (unlikely(rv != len)) { + return (rv < 0) ? rv : -EIO; + } + return 0; +} + +static unsigned long maximum_alignment(struct elf_phdr *cmds, int nr) +{ + unsigned long alignment = 0; + int i; + + for (i = 0; i < nr; i++) { + if (cmds[i].p_type == PT_LOAD) { + unsigned long p_align = cmds[i].p_align; + + /* skip non-power of two alignments as invalid */ + if (!is_power_of_2(p_align)) + continue; + alignment = max(alignment, p_align); + } + } + + /* ensure we align to at least one page */ + return ELF_PAGEALIGN(alignment); +} + +/** + * load_elf_phdrs() - load ELF program headers + * @elf_ex: ELF header of the binary whose program headers should be loaded + * @elf_file: the opened ELF binary file + * + * Loads ELF program headers from the binary file elf_file, which has the ELF + * header pointed to by elf_ex, into a newly allocated array. The caller is + * responsible for freeing the allocated data. Returns NULL upon failure. + */ +static struct elf_phdr *load_elf_phdrs(const struct elfhdr *elf_ex, + struct file *elf_file) +{ + struct elf_phdr *elf_phdata = NULL; + int retval = -1; + unsigned int size; + + /* + * If the size of this structure has changed, then punt, since + * we will be doing the wrong thing. + */ + if (elf_ex->e_phentsize != sizeof(struct elf_phdr)) + goto out; + + /* Sanity check the number of program headers... */ + /* ...and their total size. */ + size = sizeof(struct elf_phdr) * elf_ex->e_phnum; + if (size == 0 || size > 65536) + goto out; + + elf_phdata = kmalloc(size, GFP_KERNEL); + if (!elf_phdata) + goto out; + + /* Read in the program headers */ + retval = elf_read(elf_file, elf_phdata, size, elf_ex->e_phoff); + +out: + if (retval) { + kfree(elf_phdata); + elf_phdata = NULL; + } + return elf_phdata; +} + +#ifndef CONFIG_ARCH_BINFMT_ELF_STATE + +/** + * struct arch_elf_state - arch-specific ELF loading state + * + * This structure is used to preserve architecture specific data during + * the loading of an ELF file, throughout the checking of architecture + * specific ELF headers & through to the point where the ELF load is + * known to be proceeding (ie. SET_PERSONALITY). + * + * This implementation is a dummy for architectures which require no + * specific state. + */ +struct arch_elf_state { +}; + +#define INIT_ARCH_ELF_STATE {} + +/** + * arch_elf_pt_proc() - check a PT_LOPROC..PT_HIPROC ELF program header + * @ehdr: The main ELF header + * @phdr: The program header to check + * @elf: The open ELF file + * @is_interp: True if the phdr is from the interpreter of the ELF being + * loaded, else false. + * @state: Architecture-specific state preserved throughout the process + * of loading the ELF. + * + * Inspects the program header phdr to validate its correctness and/or + * suitability for the system. Called once per ELF program header in the + * range PT_LOPROC to PT_HIPROC, for both the ELF being loaded and its + * interpreter. + * + * Return: Zero to proceed with the ELF load, non-zero to fail the ELF load + * with that return code. + */ +static inline int arch_elf_pt_proc(struct elfhdr *ehdr, + struct elf_phdr *phdr, + struct file *elf, bool is_interp, + struct arch_elf_state *state) +{ + /* Dummy implementation, always proceed */ + return 0; +} + +/** + * arch_check_elf() - check an ELF executable + * @ehdr: The main ELF header + * @has_interp: True if the ELF has an interpreter, else false. + * @interp_ehdr: The interpreter's ELF header + * @state: Architecture-specific state preserved throughout the process + * of loading the ELF. + * + * Provides a final opportunity for architecture code to reject the loading + * of the ELF & cause an exec syscall to return an error. This is called after + * all program headers to be checked by arch_elf_pt_proc have been. + * + * Return: Zero to proceed with the ELF load, non-zero to fail the ELF load + * with that return code. + */ +static inline int arch_check_elf(struct elfhdr *ehdr, bool has_interp, + struct elfhdr *interp_ehdr, + struct arch_elf_state *state) +{ + /* Dummy implementation, always proceed */ + return 0; +} + +#endif /* !CONFIG_ARCH_BINFMT_ELF_STATE */ + +static inline int make_prot(u32 p_flags, struct arch_elf_state *arch_state, + bool has_interp, bool is_interp) +{ + int prot = 0; + + if (p_flags & PF_R) + prot |= PROT_READ; + if (p_flags & PF_W) + prot |= PROT_WRITE; + if (p_flags & PF_X) + prot |= PROT_EXEC; + + return arch_elf_adjust_prot(prot, arch_state, has_interp, is_interp); +} + +/* This is much more generalized than the library routine read function, + so we keep this separate. Technically the library read function + is only provided so that we can read a.out libraries that have + an ELF header */ + +static unsigned long load_elf_interp(struct elfhdr *interp_elf_ex, + struct file *interpreter, + unsigned long no_base, struct elf_phdr *interp_elf_phdata, + struct arch_elf_state *arch_state) +{ + struct elf_phdr *eppnt; + unsigned long load_addr = 0; + int load_addr_set = 0; + unsigned long error = ~0UL; + unsigned long total_size; + int i; + + /* First of all, some simple consistency checks */ + if (interp_elf_ex->e_type != ET_EXEC && + interp_elf_ex->e_type != ET_DYN) + goto out; + if (!elf_check_arch(interp_elf_ex) || + elf_check_fdpic(interp_elf_ex)) + goto out; + if (!can_mmap_file(interpreter)) + goto out; + + total_size = total_mapping_size(interp_elf_phdata, + interp_elf_ex->e_phnum); + if (!total_size) { + error = -EINVAL; + goto out; + } + + eppnt = interp_elf_phdata; + for (i = 0; i < interp_elf_ex->e_phnum; i++, eppnt++) { + if (eppnt->p_type == PT_LOAD) { + int elf_type = MAP_PRIVATE; + int elf_prot = make_prot(eppnt->p_flags, arch_state, + true, true); + unsigned long vaddr = 0; + unsigned long k, map_addr; + + vaddr = eppnt->p_vaddr; + if (interp_elf_ex->e_type == ET_EXEC || load_addr_set) + elf_type |= MAP_FIXED; + else if (no_base && interp_elf_ex->e_type == ET_DYN) + load_addr = -vaddr; + + map_addr = elf_load(interpreter, load_addr + vaddr, + eppnt, elf_prot, elf_type, total_size); + total_size = 0; + error = map_addr; + if (BAD_ADDR(map_addr)) + goto out; + + if (!load_addr_set && + interp_elf_ex->e_type == ET_DYN) { + load_addr = map_addr - ELF_PAGESTART(vaddr); + load_addr_set = 1; + } + + /* + * Check to see if the section's size will overflow the + * allowed task size. Note that p_filesz must always be + * <= p_memsize so it's only necessary to check p_memsz. + */ + k = load_addr + eppnt->p_vaddr; + if (BAD_ADDR(k) || + eppnt->p_filesz > eppnt->p_memsz || + eppnt->p_memsz > TASK_SIZE || + TASK_SIZE - eppnt->p_memsz < k) { + error = -ENOMEM; + goto out; + } + } + } + + error = load_addr; +out: + return error; +} + +/* + * These are the functions used to load ELF style executables and shared + * libraries. There is no binary dependent code anywhere else. + */ + +static int parse_elf_property(const char *data, size_t *off, size_t datasz, + struct arch_elf_state *arch, + bool have_prev_type, u32 *prev_type) +{ + size_t o, step; + const struct gnu_property *pr; + int ret; + + if (*off == datasz) + return -ENOENT; + + if (WARN_ON_ONCE(*off > datasz || *off % ELF_GNU_PROPERTY_ALIGN)) + return -EIO; + o = *off; + datasz -= *off; + + if (datasz < sizeof(*pr)) + return -ENOEXEC; + pr = (const struct gnu_property *)(data + o); + o += sizeof(*pr); + datasz -= sizeof(*pr); + + if (pr->pr_datasz > datasz) + return -ENOEXEC; + + WARN_ON_ONCE(o % ELF_GNU_PROPERTY_ALIGN); + step = round_up(pr->pr_datasz, ELF_GNU_PROPERTY_ALIGN); + if (step > datasz) + return -ENOEXEC; + + /* Properties are supposed to be unique and sorted on pr_type: */ + if (have_prev_type && pr->pr_type <= *prev_type) + return -ENOEXEC; + *prev_type = pr->pr_type; + + ret = arch_parse_elf_property(pr->pr_type, data + o, + pr->pr_datasz, ELF_COMPAT, arch); + if (ret) + return ret; + + *off = o + step; + return 0; +} + +#define NOTE_DATA_SZ SZ_1K +#define NOTE_NAME_SZ (sizeof(NN_GNU_PROPERTY_TYPE_0)) + +static int parse_elf_properties(struct file *f, const struct elf_phdr *phdr, + struct arch_elf_state *arch) +{ + union { + struct elf_note nhdr; + char data[NOTE_DATA_SZ]; + } note; + loff_t pos; + ssize_t n; + size_t off, datasz; + int ret; + bool have_prev_type; + u32 prev_type; + + if (!IS_ENABLED(CONFIG_ARCH_USE_GNU_PROPERTY) || !phdr) + return 0; + + /* load_elf_binary() shouldn't call us unless this is true... */ + if (WARN_ON_ONCE(phdr->p_type != PT_GNU_PROPERTY)) + return -ENOEXEC; + + /* If the properties are crazy large, that's too bad (for now): */ + if (phdr->p_filesz > sizeof(note)) + return -ENOEXEC; + + pos = phdr->p_offset; + n = kernel_read(f, ¬e, phdr->p_filesz, &pos); + + BUILD_BUG_ON(sizeof(note) < sizeof(note.nhdr) + NOTE_NAME_SZ); + if (n < 0 || n < sizeof(note.nhdr) + NOTE_NAME_SZ) + return -EIO; + + if (note.nhdr.n_type != NT_GNU_PROPERTY_TYPE_0 || + note.nhdr.n_namesz != NOTE_NAME_SZ || + strncmp(note.data + sizeof(note.nhdr), + NN_GNU_PROPERTY_TYPE_0, n - sizeof(note.nhdr))) + return -ENOEXEC; + + off = round_up(sizeof(note.nhdr) + NOTE_NAME_SZ, + ELF_GNU_PROPERTY_ALIGN); + if (off > n) + return -ENOEXEC; + + if (note.nhdr.n_descsz > n - off) + return -ENOEXEC; + datasz = off + note.nhdr.n_descsz; + + have_prev_type = false; + do { + ret = parse_elf_property(note.data, &off, datasz, arch, + have_prev_type, &prev_type); + have_prev_type = true; + } while (!ret); + + return ret == -ENOENT ? 0 : ret; +} + +static int load_elf_binary(struct linux_binprm *bprm) +{ + struct file *interpreter = NULL; /* to shut gcc up */ + unsigned long load_bias = 0, phdr_addr = 0; + int first_pt_load = 1; + unsigned long error; + struct elf_phdr *elf_ppnt, *elf_phdata, *interp_elf_phdata = NULL; + struct elf_phdr *elf_property_phdata = NULL; + unsigned long elf_brk; + bool brk_moved = false; + int retval, i; + unsigned long elf_entry; + unsigned long e_entry; + unsigned long interp_load_addr = 0; + unsigned long start_code, end_code, start_data, end_data; + unsigned long reloc_func_desc __maybe_unused = 0; + int executable_stack = EXSTACK_DEFAULT; + struct elfhdr *elf_ex = (struct elfhdr *)bprm->buf; + struct elfhdr *interp_elf_ex = NULL; + struct arch_elf_state arch_state = INIT_ARCH_ELF_STATE; + struct mm_struct *mm; + struct pt_regs *regs; + + retval = -ENOEXEC; + /* First of all, some simple consistency checks */ + if (memcmp(elf_ex->e_ident, ELFMAG, SELFMAG) != 0) + goto out; + + if (elf_ex->e_type != ET_EXEC && elf_ex->e_type != ET_DYN) + goto out; + if (!elf_check_arch(elf_ex)) + goto out; + if (elf_check_fdpic(elf_ex)) + goto out; + if (!can_mmap_file(bprm->file)) + goto out; + + elf_phdata = load_elf_phdrs(elf_ex, bprm->file); + if (!elf_phdata) + goto out; + + elf_ppnt = elf_phdata; + for (i = 0; i < elf_ex->e_phnum; i++, elf_ppnt++) { + char *elf_interpreter; + + if (elf_ppnt->p_type == PT_GNU_PROPERTY) { + elf_property_phdata = elf_ppnt; + continue; + } + + if (elf_ppnt->p_type != PT_INTERP) + continue; + + /* + * This is the program interpreter used for shared libraries - + * for now assume that this is an a.out format binary. + */ + retval = -ENOEXEC; + if (elf_ppnt->p_filesz > PATH_MAX || elf_ppnt->p_filesz < 2) + goto out_free_ph; + + retval = -ENOMEM; + elf_interpreter = kmalloc(elf_ppnt->p_filesz, GFP_KERNEL); + if (!elf_interpreter) + goto out_free_ph; + + retval = elf_read(bprm->file, elf_interpreter, elf_ppnt->p_filesz, + elf_ppnt->p_offset); + if (retval < 0) + goto out_free_interp; + /* make sure path is NULL terminated */ + retval = -ENOEXEC; + if (elf_interpreter[elf_ppnt->p_filesz - 1] != '\0') + goto out_free_interp; + + interpreter = bprm_open_interpreter(bprm, elf_interpreter); + kfree(elf_interpreter); + retval = PTR_ERR(interpreter); + if (IS_ERR(interpreter)) + goto out_free_ph; + + /* + * If the binary is not readable then enforce mm->dumpable = 0 + * regardless of the interpreter's permissions. + */ + would_dump(bprm, interpreter); + + interp_elf_ex = kmalloc_obj(*interp_elf_ex); + if (!interp_elf_ex) { + retval = -ENOMEM; + goto out_free_file; + } + + /* Get the exec headers */ + retval = elf_read(interpreter, interp_elf_ex, + sizeof(*interp_elf_ex), 0); + if (retval < 0) + goto out_free_dentry; + + break; + +out_free_interp: + kfree(elf_interpreter); + goto out_free_ph; + } + + /* No PT_INTERP to substitute for: the override does not apply. */ + bprm_drop_loader(bprm); + + elf_ppnt = elf_phdata; + for (i = 0; i < elf_ex->e_phnum; i++, elf_ppnt++) + switch (elf_ppnt->p_type) { + case PT_GNU_STACK: + if (elf_ppnt->p_flags & PF_X) + executable_stack = EXSTACK_ENABLE_X; + else + executable_stack = EXSTACK_DISABLE_X; + break; + + case PT_LOPROC ... PT_HIPROC: + retval = arch_elf_pt_proc(elf_ex, elf_ppnt, + bprm->file, false, + &arch_state); + if (retval) + goto out_free_dentry; + break; + } + + /* Some simple consistency checks for the interpreter */ + if (interpreter) { + retval = -ELIBBAD; + /* Not an ELF interpreter */ + if (memcmp(interp_elf_ex->e_ident, ELFMAG, SELFMAG) != 0) + goto out_free_dentry; + /* Verify the interpreter has a valid arch */ + if (!elf_check_arch(interp_elf_ex) || + elf_check_fdpic(interp_elf_ex)) + goto out_free_dentry; + + /* Load the interpreter program headers */ + interp_elf_phdata = load_elf_phdrs(interp_elf_ex, + interpreter); + if (!interp_elf_phdata) + goto out_free_dentry; + + /* Pass PT_LOPROC..PT_HIPROC headers to arch code */ + elf_property_phdata = NULL; + elf_ppnt = interp_elf_phdata; + for (i = 0; i < interp_elf_ex->e_phnum; i++, elf_ppnt++) + switch (elf_ppnt->p_type) { + case PT_GNU_PROPERTY: + elf_property_phdata = elf_ppnt; + break; + + case PT_LOPROC ... PT_HIPROC: + retval = arch_elf_pt_proc(interp_elf_ex, + elf_ppnt, interpreter, + true, &arch_state); + if (retval) + goto out_free_dentry; + break; + } + } + + retval = parse_elf_properties(interpreter ?: bprm->file, + elf_property_phdata, &arch_state); + if (retval) + goto out_free_dentry; + + /* + * Allow arch code to reject the ELF at this point, whilst it's + * still possible to return an error to the code that invoked + * the exec syscall. + */ + retval = arch_check_elf(elf_ex, + !!interpreter, interp_elf_ex, + &arch_state); + if (retval) + goto out_free_dentry; + + /* Flush all traces of the currently running executable */ + retval = begin_new_exec(bprm); + if (retval) + goto out_free_dentry; + + /* Do this immediately, since STACK_TOP as used in setup_arg_pages + may depend on the personality. */ + SET_PERSONALITY2(*elf_ex, &arch_state); + if (elf_read_implies_exec(*elf_ex, executable_stack)) + current->personality |= READ_IMPLIES_EXEC; + + const int snapshot_randomize_va_space = READ_ONCE(randomize_va_space); + if (!(current->personality & ADDR_NO_RANDOMIZE) && snapshot_randomize_va_space) + current->flags |= PF_RANDOMIZE; + + setup_new_exec(bprm); + + /* Do this so that we can load the interpreter, if need be. We will + change some of these later */ + retval = setup_arg_pages(bprm, randomize_stack_top(STACK_TOP), + executable_stack); + if (retval < 0) + goto out_free_dentry; + + elf_brk = 0; + + start_code = ~0UL; + end_code = 0; + start_data = 0; + end_data = 0; + + /* Now we do a little grungy work by mmapping the ELF image into + the correct location in memory. */ + for(i = 0, elf_ppnt = elf_phdata; + i < elf_ex->e_phnum; i++, elf_ppnt++) { + int elf_prot, elf_flags; + unsigned long k, vaddr; + unsigned long total_size = 0; + unsigned long alignment; + + if (elf_ppnt->p_type != PT_LOAD) + continue; + + elf_prot = make_prot(elf_ppnt->p_flags, &arch_state, + !!interpreter, false); + + elf_flags = MAP_PRIVATE; + + vaddr = elf_ppnt->p_vaddr; + /* + * The first time through the loop, first_pt_load is true: + * layout will be calculated. Once set, use MAP_FIXED since + * we know we've already safely mapped the entire region with + * MAP_FIXED_NOREPLACE in the once-per-binary logic following. + */ + if (!first_pt_load) { + elf_flags |= MAP_FIXED; + } else if (elf_ex->e_type == ET_EXEC) { + /* + * This logic is run once for the first LOAD Program + * Header for ET_EXEC binaries. No special handling + * is needed. + */ + elf_flags |= MAP_FIXED_NOREPLACE; + } else if (elf_ex->e_type == ET_DYN) { + /* + * This logic is run once for the first LOAD Program + * Header for ET_DYN binaries to calculate the + * randomization (load_bias) for all the LOAD + * Program Headers. + */ + + /* + * Calculate the entire size of the ELF mapping + * (total_size), used for the initial mapping, + * due to load_addr_set which is set to true later + * once the initial mapping is performed. + * + * Note that this is only sensible when the LOAD + * segments are contiguous (or overlapping). If + * used for LOADs that are far apart, this would + * cause the holes between LOADs to be mapped, + * running the risk of having the mapping fail, + * as it would be larger than the ELF file itself. + * + * As a result, only ET_DYN does this, since + * some ET_EXEC (e.g. ia64) may have large virtual + * memory holes between LOADs. + * + */ + total_size = total_mapping_size(elf_phdata, + elf_ex->e_phnum); + if (!total_size) { + retval = -EINVAL; + goto out_free_dentry; + } + + /* Calculate any requested alignment. */ + alignment = maximum_alignment(elf_phdata, elf_ex->e_phnum); + + /** + * DOC: PIE handling + * + * There are effectively two types of ET_DYN ELF + * binaries: programs (i.e. PIE: ET_DYN with + * PT_INTERP) and loaders (i.e. static PIE: ET_DYN + * without PT_INTERP, usually the ELF interpreter + * itself). Loaders must be loaded away from programs + * since the program may otherwise collide with the + * loader (especially for ET_EXEC which does not have + * a randomized position). + * + * For example, to handle invocations of + * "./ld.so someprog" to test out a new version of + * the loader, the subsequent program that the + * loader loads must avoid the loader itself, so + * they cannot share the same load range. Sufficient + * room for the brk must be allocated with the + * loader as well, since brk must be available with + * the loader. + * + * Therefore, programs are loaded offset from + * ELF_ET_DYN_BASE and loaders are loaded into the + * independently randomized mmap region (0 load_bias + * without MAP_FIXED nor MAP_FIXED_NOREPLACE). + * + * See below for "brk" handling details, which is + * also affected by program vs loader and ASLR. + */ + if (interpreter) { + /* On ET_DYN with PT_INTERP, we do the ASLR. */ + load_bias = ELF_ET_DYN_BASE; + if (current->flags & PF_RANDOMIZE) + load_bias += arch_mmap_rnd(); + /* Adjust alignment as requested. */ + if (alignment) + load_bias &= ~(alignment - 1); + elf_flags |= MAP_FIXED_NOREPLACE; + } else { + /* + * For ET_DYN without PT_INTERP, we rely on + * the architectures's (potentially ASLR) mmap + * base address (via a load_bias of 0). + * + * When a large alignment is requested, we + * must do the allocation at address "0" right + * now to discover where things will load so + * that we can adjust the resulting alignment. + * In this case (load_bias != 0), we can use + * MAP_FIXED_NOREPLACE to make sure the mapping + * doesn't collide with anything. + */ + if (alignment > ELF_MIN_ALIGN) { + load_bias = elf_load(bprm->file, 0, elf_ppnt, + elf_prot, elf_flags, total_size); + if (BAD_ADDR(load_bias)) { + retval = IS_ERR_VALUE(load_bias) ? + PTR_ERR((void*)load_bias) : -EINVAL; + goto out_free_dentry; + } + vm_munmap(load_bias, total_size); + /* Adjust alignment as requested. */ + if (alignment) + load_bias &= ~(alignment - 1); + elf_flags |= MAP_FIXED_NOREPLACE; + } else + load_bias = 0; + } + + /* + * Since load_bias is used for all subsequent loading + * calculations, we must lower it by the first vaddr + * so that the remaining calculations based on the + * ELF vaddrs will be correctly offset. The result + * is then page aligned. + */ + load_bias = ELF_PAGESTART(load_bias - vaddr); + } + + error = elf_load(bprm->file, load_bias + vaddr, elf_ppnt, + elf_prot, elf_flags, total_size); + if (BAD_ADDR(error)) { + retval = IS_ERR_VALUE(error) ? + PTR_ERR((void*)error) : -EINVAL; + goto out_free_dentry; + } + + if (first_pt_load) { + first_pt_load = 0; + if (elf_ex->e_type == ET_DYN) { + load_bias += error - + ELF_PAGESTART(load_bias + vaddr); + reloc_func_desc = load_bias; + } + } + + /* + * Figure out which segment in the file contains the Program + * Header table, and map to the associated memory address. + */ + if (elf_ppnt->p_offset <= elf_ex->e_phoff && + elf_ex->e_phoff < elf_ppnt->p_offset + elf_ppnt->p_filesz) { + phdr_addr = elf_ex->e_phoff - elf_ppnt->p_offset + + elf_ppnt->p_vaddr; + } + + k = elf_ppnt->p_vaddr; + if ((elf_ppnt->p_flags & PF_X) && k < start_code) + start_code = k; + if (start_data < k) + start_data = k; + + /* + * Check to see if the section's size will overflow the + * allowed task size. Note that p_filesz must always be + * <= p_memsz so it is only necessary to check p_memsz. + */ + if (BAD_ADDR(k) || elf_ppnt->p_filesz > elf_ppnt->p_memsz || + elf_ppnt->p_memsz > TASK_SIZE || + TASK_SIZE - elf_ppnt->p_memsz < k) { + /* set_brk can never work. Avoid overflows. */ + retval = -EINVAL; + goto out_free_dentry; + } + + k = elf_ppnt->p_vaddr + elf_ppnt->p_filesz; + + if ((elf_ppnt->p_flags & PF_X) && end_code < k) + end_code = k; + if (end_data < k) + end_data = k; + k = elf_ppnt->p_vaddr + elf_ppnt->p_memsz; + if (k > elf_brk) + elf_brk = k; + } + + e_entry = elf_ex->e_entry + load_bias; + phdr_addr += load_bias; + elf_brk += load_bias; + start_code += load_bias; + end_code += load_bias; + start_data += load_bias; + end_data += load_bias; + + if (interpreter) { + elf_entry = load_elf_interp(interp_elf_ex, + interpreter, + load_bias, interp_elf_phdata, + &arch_state); + if (!IS_ERR_VALUE(elf_entry)) { + /* + * load_elf_interp() returns relocation + * adjustment + */ + interp_load_addr = elf_entry; + elf_entry += interp_elf_ex->e_entry; + } + if (BAD_ADDR(elf_entry)) { + retval = IS_ERR_VALUE(elf_entry) ? + (int)elf_entry : -EINVAL; + goto out_free_dentry; + } + reloc_func_desc = interp_load_addr; + + exe_file_allow_write_access(interpreter); + fput(interpreter); + + kfree(interp_elf_ex); + kfree(interp_elf_phdata); + } else { + elf_entry = e_entry; + if (BAD_ADDR(elf_entry)) { + retval = -EINVAL; + goto out_free_dentry; + } + } + + kfree(elf_phdata); + + set_binfmt(&elf_format); + +#ifdef ARCH_HAS_SETUP_ADDITIONAL_PAGES + retval = ARCH_SETUP_ADDITIONAL_PAGES(bprm, elf_ex, !!interpreter); + if (retval < 0) + goto out; +#endif /* ARCH_HAS_SETUP_ADDITIONAL_PAGES */ + + retval = create_elf_tables(bprm, elf_ex, interp_load_addr, + e_entry, phdr_addr); + if (retval < 0) + goto out; + + mm = current->mm; + mm->end_code = end_code; + mm->start_code = start_code; + mm->start_data = start_data; + mm->end_data = end_data; + mm->start_stack = bprm->p; + + elf_coredump_set_mm_eflags(mm, elf_ex->e_flags); + + /** + * DOC: "brk" handling + * + * For architectures with ELF randomization, when executing a + * loader directly (i.e. static PIE: ET_DYN without PT_INTERP), + * move the brk area out of the mmap region and into the unused + * ELF_ET_DYN_BASE region. Since "brk" grows up it may collide + * early with the stack growing down or other regions being put + * into the mmap region by the kernel (e.g. vdso). + * + * In the CONFIG_COMPAT_BRK case, though, everything is turned + * off because we're not allowed to move the brk at all. + */ + if (!IS_ENABLED(CONFIG_COMPAT_BRK) && + IS_ENABLED(CONFIG_ARCH_HAS_ELF_RANDOMIZE) && + elf_ex->e_type == ET_DYN && !interpreter) { + elf_brk = ELF_ET_DYN_BASE; + /* This counts as moving the brk, so let brk(2) know. */ + brk_moved = true; + } + mm->start_brk = mm->brk = ELF_PAGEALIGN(elf_brk); + + if ((current->flags & PF_RANDOMIZE) && snapshot_randomize_va_space > 1) { + /* + * If we didn't move the brk to ELF_ET_DYN_BASE (above), + * leave a gap between .bss and brk. + */ + if (!brk_moved) + mm->brk = mm->start_brk = mm->brk + PAGE_SIZE; + + mm->brk = mm->start_brk = arch_randomize_brk(mm); + brk_moved = true; + } + +#ifdef compat_brk_randomized + if (brk_moved) + current->brk_randomized = 1; +#endif + + if (current->personality & MMAP_PAGE_ZERO) { + /* Why this, you ask??? Well SVr4 maps page 0 as read-only, + and some applications "depend" upon this behavior. + Since we do not have the power to recompile these, we + emulate the SVr4 behavior. Sigh. */ + error = vm_mmap(NULL, 0, PAGE_SIZE, PROT_READ | PROT_EXEC, + MAP_FIXED | MAP_PRIVATE, 0); + if (!error) + mseal_mmap_page_zero(); + } + + regs = current_pt_regs(); +#ifdef ELF_PLAT_INIT + /* + * The ABI may specify that certain registers be set up in special + * ways (on i386 %edx is the address of a DT_FINI function, for + * example. In addition, it may also specify (eg, PowerPC64 ELF) + * that the e_entry field is the address of the function descriptor + * for the startup routine, rather than the address of the startup + * routine itself. This macro performs whatever initialization to + * the regs structure is required as well as any relocations to the + * function descriptor entries when executing dynamically links apps. + */ + ELF_PLAT_INIT(regs, reloc_func_desc); +#endif + + finalize_exec(bprm); + START_THREAD(elf_ex, regs, elf_entry, bprm->p); + retval = 0; +out: + return retval; + + /* error cleanup */ +out_free_dentry: + kfree(interp_elf_ex); + kfree(interp_elf_phdata); +out_free_file: + exe_file_allow_write_access(interpreter); + if (interpreter) + fput(interpreter); +out_free_ph: + kfree(elf_phdata); + goto out; +} + +#ifdef CONFIG_ELF_CORE +/* + * ELF core dumper + * + * Modelled on fs/exec.c:aout_core_dump() + * Jeremy Fitzhardinge + */ + +/* An ELF note in memory */ +struct memelfnote +{ + const char *name; + int type; + unsigned int datasz; + void *data; +}; + +static int notesize(struct memelfnote *en) +{ + int sz; + + sz = sizeof(struct elf_note); + sz += roundup(strlen(en->name) + 1, 4); + sz += roundup(en->datasz, 4); + + return sz; +} + +static int writenote(struct memelfnote *men, struct coredump_params *cprm) +{ + struct elf_note en; + en.n_namesz = strlen(men->name) + 1; + en.n_descsz = men->datasz; + en.n_type = men->type; + + return dump_emit(cprm, &en, sizeof(en)) && + dump_emit(cprm, men->name, en.n_namesz) && dump_align(cprm, 4) && + dump_emit(cprm, men->data, men->datasz) && dump_align(cprm, 4); +} + +static void fill_elf_header(struct elfhdr *elf, int segs, + u16 machine, u32 flags) +{ + memset(elf, 0, sizeof(*elf)); + + memcpy(elf->e_ident, ELFMAG, SELFMAG); + elf->e_ident[EI_CLASS] = ELF_CLASS; + elf->e_ident[EI_DATA] = ELF_DATA; + elf->e_ident[EI_VERSION] = EV_CURRENT; + elf->e_ident[EI_OSABI] = ELF_OSABI; + + elf->e_type = ET_CORE; + elf->e_machine = machine; + elf->e_version = EV_CURRENT; + elf->e_phoff = sizeof(struct elfhdr); + elf->e_flags = flags; + elf->e_ehsize = sizeof(struct elfhdr); + elf->e_phentsize = sizeof(struct elf_phdr); + elf->e_phnum = segs; +} + +static void fill_elf_note_phdr(struct elf_phdr *phdr, int sz, loff_t offset) +{ + phdr->p_type = PT_NOTE; + phdr->p_offset = offset; + phdr->p_vaddr = 0; + phdr->p_paddr = 0; + phdr->p_filesz = sz; + phdr->p_memsz = 0; + phdr->p_flags = 0; + phdr->p_align = 4; +} + +static void __fill_note(struct memelfnote *note, const char *name, int type, + unsigned int sz, void *data) +{ + note->name = name; + note->type = type; + note->datasz = sz; + note->data = data; +} + +#define fill_note(note, type, sz, data) \ + __fill_note(note, NN_ ## type, NT_ ## type, sz, data) + +/* + * fill up all the fields in prstatus from the given task struct, except + * registers which need to be filled up separately. + */ +static void fill_prstatus(struct elf_prstatus_common *prstatus, + struct task_struct *p, long signr) +{ + prstatus->pr_info.si_signo = prstatus->pr_cursig = signr; + prstatus->pr_sigpend = p->pending.signal.sig[0]; + prstatus->pr_sighold = p->blocked.sig[0]; + rcu_read_lock(); + prstatus->pr_ppid = task_pid_vnr(rcu_dereference(p->real_parent)); + rcu_read_unlock(); + prstatus->pr_pid = task_pid_vnr(p); + prstatus->pr_pgrp = task_pgrp_vnr(p); + prstatus->pr_sid = task_session_vnr(p); + if (thread_group_leader(p)) { + struct task_cputime cputime; + + /* + * This is the record for the group leader. It shows the + * group-wide total, not its individual thread total. + */ + thread_group_cputime(p, &cputime); + prstatus->pr_utime = ns_to_kernel_old_timeval(cputime.utime); + prstatus->pr_stime = ns_to_kernel_old_timeval(cputime.stime); + } else { + u64 utime, stime; + + task_cputime(p, &utime, &stime); + prstatus->pr_utime = ns_to_kernel_old_timeval(utime); + prstatus->pr_stime = ns_to_kernel_old_timeval(stime); + } + + prstatus->pr_cutime = ns_to_kernel_old_timeval(p->signal->cutime); + prstatus->pr_cstime = ns_to_kernel_old_timeval(p->signal->cstime); +} + +static int fill_psinfo(struct elf_prpsinfo *psinfo, struct task_struct *p, + struct mm_struct *mm) +{ + const struct cred *cred; + unsigned int i, len; + unsigned int state; + + /* first copy the parameters from user space */ + memset(psinfo, 0, sizeof(struct elf_prpsinfo)); + + len = mm->arg_end - mm->arg_start; + if (len >= ELF_PRARGSZ) + len = ELF_PRARGSZ-1; + if (copy_from_user(&psinfo->pr_psargs, + (const char __user *)mm->arg_start, len)) + return -EFAULT; + for(i = 0; i < len; i++) + if (psinfo->pr_psargs[i] == 0) + psinfo->pr_psargs[i] = ' '; + psinfo->pr_psargs[len] = 0; + + rcu_read_lock(); + psinfo->pr_ppid = task_pid_vnr(rcu_dereference(p->real_parent)); + rcu_read_unlock(); + psinfo->pr_pid = task_pid_vnr(p); + psinfo->pr_pgrp = task_pgrp_vnr(p); + psinfo->pr_sid = task_session_vnr(p); + + state = READ_ONCE(p->__state); + i = state ? ffz(~state) + 1 : 0; + psinfo->pr_state = i; + psinfo->pr_sname = (i > 5) ? '.' : "RSDTZW"[i]; + psinfo->pr_zomb = psinfo->pr_sname == 'Z'; + psinfo->pr_nice = task_nice(p); + psinfo->pr_flag = p->flags; + rcu_read_lock(); + cred = __task_cred(p); + SET_UID(psinfo->pr_uid, from_kuid_munged(cred->user_ns, cred->uid)); + SET_GID(psinfo->pr_gid, from_kgid_munged(cred->user_ns, cred->gid)); + rcu_read_unlock(); + get_task_comm(psinfo->pr_fname, p); + + return 0; +} + +static void fill_auxv_note(struct memelfnote *note, struct mm_struct *mm) +{ + elf_addr_t *auxv = (elf_addr_t *) mm->saved_auxv; + int i = 0; + do + i += 2; + while (auxv[i - 2] != AT_NULL); + fill_note(note, AUXV, i * sizeof(elf_addr_t), auxv); +} + +static void fill_siginfo_note(struct memelfnote *note, user_siginfo_t *csigdata, + const kernel_siginfo_t *siginfo) +{ + copy_siginfo_to_external(csigdata, siginfo); + fill_note(note, SIGINFO, sizeof(*csigdata), csigdata); +} + +/* + * Format of NT_FILE note: + * + * long count -- how many files are mapped + * long page_size -- units for file_ofs + * array of [COUNT] elements of + * long start + * long end + * long file_ofs + * followed by COUNT filenames in ASCII: "FILE1" NUL "FILE2" NUL... + */ +static int fill_files_note(struct memelfnote *note, struct coredump_params *cprm) +{ + unsigned count, size, names_ofs, remaining, n; + user_long_t *data; + user_long_t *start_end_ofs; + char *name_base, *name_curpos; + int i; + + /* *Estimated* file count and total data size needed */ + count = cprm->vma_count; + if (count > UINT_MAX / 64) + return -EINVAL; + size = count * 64; + + names_ofs = (2 + 3 * count) * sizeof(data[0]); + alloc: + /* paranoia check */ + if (size >= core_file_note_size_limit) { + pr_warn_once("coredump Note size too large: %u (does kernel.core_file_note_size_limit sysctl need adjustment?\n", + size); + return -EINVAL; + } + size = round_up(size, PAGE_SIZE); + /* + * "size" can be 0 here legitimately. + * Let it ENOMEM and omit NT_FILE section which will be empty anyway. + */ + data = kvmalloc(size, GFP_KERNEL); + if (ZERO_OR_NULL_PTR(data)) + return -ENOMEM; + + start_end_ofs = data + 2; + name_base = name_curpos = ((char *)data) + names_ofs; + remaining = size - names_ofs; + count = 0; + for (i = 0; i < cprm->vma_count; i++) { + struct core_vma_metadata *m = &cprm->vma_meta[i]; + struct file *file; + const char *filename; + + file = m->file; + if (!file) + continue; + filename = file_path(file, name_curpos, remaining); + if (IS_ERR(filename)) { + if (PTR_ERR(filename) == -ENAMETOOLONG) { + kvfree(data); + size = size * 5 / 4; + goto alloc; + } + continue; + } + + /* file_path() fills at the end, move name down */ + /* n = strlen(filename) + 1: */ + n = (name_curpos + remaining) - filename; + remaining = filename - name_curpos; + memmove(name_curpos, filename, n); + name_curpos += n; + + *start_end_ofs++ = m->start; + *start_end_ofs++ = m->end; + *start_end_ofs++ = m->pgoff; + count++; + } + + /* Now we know exact count of files, can store it */ + data[0] = count; + data[1] = PAGE_SIZE; + /* + * Count usually is less than mm->map_count, + * we need to move filenames down. + */ + n = cprm->vma_count - count; + if (n != 0) { + unsigned shift_bytes = n * 3 * sizeof(data[0]); + memmove(name_base - shift_bytes, name_base, + name_curpos - name_base); + name_curpos -= shift_bytes; + } + + size = name_curpos - (char *)data; + fill_note(note, FILE, size, data); + return 0; +} + +#include + +struct elf_thread_core_info { + struct elf_thread_core_info *next; + struct task_struct *task; + struct elf_prstatus prstatus; + struct memelfnote notes[]; +}; + +struct elf_note_info { + struct elf_thread_core_info *thread; + struct memelfnote psinfo; + struct memelfnote signote; + struct memelfnote auxv; + struct memelfnote files; + user_siginfo_t csigdata; + size_t size; + int thread_notes; +}; + +#ifdef CORE_DUMP_USE_REGSET +/* + * When a regset has a writeback hook, we call it on each thread before + * dumping user memory. On register window machines, this makes sure the + * user memory backing the register data is up to date before we read it. + */ +static void do_thread_regset_writeback(struct task_struct *task, + const struct user_regset *regset) +{ + if (regset->writeback) + regset->writeback(task, regset, 1); +} + +#ifndef PRSTATUS_SIZE +#define PRSTATUS_SIZE sizeof(struct elf_prstatus) +#endif + +#ifndef SET_PR_FPVALID +#define SET_PR_FPVALID(S) ((S)->pr_fpvalid = 1) +#endif + +static int fill_thread_core_info(struct elf_thread_core_info *t, + const struct user_regset_view *view, + long signr, struct elf_note_info *info) +{ + unsigned int note_iter, view_iter; + + /* + * NT_PRSTATUS is the one special case, because the regset data + * goes into the pr_reg field inside the note contents, rather + * than being the whole note contents. We fill the regset in here. + * We assume that regset 0 is NT_PRSTATUS. + */ + fill_prstatus(&t->prstatus.common, t->task, signr); + regset_get(t->task, &view->regsets[0], + sizeof(t->prstatus.pr_reg), &t->prstatus.pr_reg); + + fill_note(&t->notes[0], PRSTATUS, PRSTATUS_SIZE, &t->prstatus); + info->size += notesize(&t->notes[0]); + + do_thread_regset_writeback(t->task, &view->regsets[0]); + + /* + * Each other regset might generate a note too. For each regset + * that has no core_note_type or is inactive, skip it. + */ + note_iter = 1; + for (view_iter = 1; view_iter < view->n; ++view_iter) { + const struct user_regset *regset = &view->regsets[view_iter]; + int note_type = regset->core_note_type; + const char *note_name = regset->core_note_name; + bool is_fpreg = note_type == NT_PRFPREG; + void *data; + int ret; + + do_thread_regset_writeback(t->task, regset); + if (!note_type) // not for coredumps + continue; + if (regset->active && regset->active(t->task, regset) <= 0) + continue; + + ret = regset_get_alloc(t->task, regset, ~0U, &data); + if (ret < 0) + continue; + + if (WARN_ON_ONCE(note_iter >= info->thread_notes)) + break; + + if (is_fpreg) + SET_PR_FPVALID(&t->prstatus); + + /* There should be a note name, but if not, guess: */ + if (WARN_ON_ONCE(!note_name)) + note_name = "LINUX"; + else + /* Warn on non-legacy-compatible names, for now. */ + WARN_ON_ONCE(strcmp(note_name, + is_fpreg ? "CORE" : "LINUX")); + + __fill_note(&t->notes[note_iter], note_name, note_type, + ret, data); + + info->size += notesize(&t->notes[note_iter]); + note_iter++; + } + + return 1; +} +#else +static int fill_thread_core_info(struct elf_thread_core_info *t, + const struct user_regset_view *view, + long signr, struct elf_note_info *info) +{ + struct task_struct *p = t->task; + elf_fpregset_t *fpu; + + fill_prstatus(&t->prstatus.common, p, signr); + elf_core_copy_task_regs(p, &t->prstatus.pr_reg); + + fill_note(&t->notes[0], PRSTATUS, sizeof(t->prstatus), &t->prstatus); + info->size += notesize(&t->notes[0]); + + fpu = kzalloc_obj(elf_fpregset_t); + if (!fpu || !elf_core_copy_task_fpregs(p, fpu)) { + kfree(fpu); + return 1; + } + + t->prstatus.pr_fpvalid = 1; + fill_note(&t->notes[1], PRFPREG, sizeof(*fpu), fpu); + info->size += notesize(&t->notes[1]); + + return 1; +} +#endif + +static int fill_note_info(struct elfhdr *elf, int phdrs, + struct elf_note_info *info, + struct coredump_params *cprm) +{ + struct task_struct *dump_task = current; + const struct user_regset_view *view; + struct elf_thread_core_info *t; + struct elf_prpsinfo *psinfo; + struct core_thread *ct; + u16 machine; + u32 flags; + + psinfo = kmalloc_obj(*psinfo); + if (!psinfo) + return 0; + fill_note(&info->psinfo, PRPSINFO, sizeof(*psinfo), psinfo); + +#ifdef CORE_DUMP_USE_REGSET + view = task_user_regset_view(dump_task); + + /* + * Figure out how many notes we're going to need for each thread. + */ + info->thread_notes = 0; + for (int i = 0; i < view->n; ++i) + if (view->regsets[i].core_note_type != 0) + ++info->thread_notes; + + /* + * Sanity check. We rely on regset 0 being in NT_PRSTATUS, + * since it is our one special case. + */ + if (unlikely(info->thread_notes == 0) || + unlikely(view->regsets[0].core_note_type != NT_PRSTATUS)) { + WARN_ON(1); + return 0; + } + + machine = view->e_machine; + flags = view->e_flags; +#else + view = NULL; + info->thread_notes = 2; + machine = ELF_ARCH; + flags = ELF_CORE_EFLAGS; +#endif + + /* + * Override ELF e_flags with value taken from process, + * if arch needs that. + */ + flags = elf_coredump_get_mm_eflags(dump_task->mm, flags); + + /* + * Initialize the ELF file header. + */ + fill_elf_header(elf, phdrs, machine, flags); + + /* + * Allocate a structure for each thread. + */ + info->thread = kzalloc_flex(*info->thread, notes, info->thread_notes); + if (unlikely(!info->thread)) + return 0; + + info->thread->task = dump_task; + for (ct = dump_task->signal->core_state->dumper.next; ct; ct = ct->next) { + t = kzalloc_flex(*t, notes, info->thread_notes); + if (unlikely(!t)) + return 0; + + t->task = ct->task; + t->next = info->thread->next; + info->thread->next = t; + } + + /* + * Now fill in each thread's information. + */ + for (t = info->thread; t != NULL; t = t->next) + if (!fill_thread_core_info(t, view, cprm->siginfo->si_signo, info)) + return 0; + + /* + * Fill in the two process-wide notes. + */ + fill_psinfo(psinfo, dump_task->group_leader, dump_task->mm); + info->size += notesize(&info->psinfo); + + fill_siginfo_note(&info->signote, &info->csigdata, cprm->siginfo); + info->size += notesize(&info->signote); + + fill_auxv_note(&info->auxv, current->mm); + info->size += notesize(&info->auxv); + + if (fill_files_note(&info->files, cprm) == 0) + info->size += notesize(&info->files); + + return 1; +} + +/* + * Write all the notes for each thread. When writing the first thread, the + * process-wide notes are interleaved after the first thread-specific note. + */ +static int write_note_info(struct elf_note_info *info, + struct coredump_params *cprm) +{ + bool first = true; + struct elf_thread_core_info *t = info->thread; + + do { + int i; + + if (!writenote(&t->notes[0], cprm)) + return 0; + + if (first && !writenote(&info->psinfo, cprm)) + return 0; + if (first && !writenote(&info->signote, cprm)) + return 0; + if (first && !writenote(&info->auxv, cprm)) + return 0; + if (first && info->files.data && + !writenote(&info->files, cprm)) + return 0; + + for (i = 1; i < info->thread_notes; ++i) + if (t->notes[i].data && + !writenote(&t->notes[i], cprm)) + return 0; + + first = false; + t = t->next; + } while (t); + + return 1; +} + +static void free_note_info(struct elf_note_info *info) +{ + struct elf_thread_core_info *threads = info->thread; + while (threads) { + unsigned int i; + struct elf_thread_core_info *t = threads; + threads = t->next; + WARN_ON(t->notes[0].data && t->notes[0].data != &t->prstatus); + for (i = 1; i < info->thread_notes; ++i) + kvfree(t->notes[i].data); + kfree(t); + } + kfree(info->psinfo.data); + kvfree(info->files.data); +} + +static void fill_extnum_info(struct elfhdr *elf, struct elf_shdr *shdr4extnum, + elf_addr_t e_shoff, int segs) +{ + elf->e_shoff = e_shoff; + elf->e_shentsize = sizeof(*shdr4extnum); + elf->e_shnum = 1; + elf->e_shstrndx = SHN_UNDEF; + + memset(shdr4extnum, 0, sizeof(*shdr4extnum)); + + shdr4extnum->sh_type = SHT_NULL; + shdr4extnum->sh_size = elf->e_shnum; + shdr4extnum->sh_link = elf->e_shstrndx; + shdr4extnum->sh_info = segs; +} + +/* + * Actual dumper + * + * This is a two-pass process; first we find the offsets of the bits, + * and then they are actually written out. If we run out of core limit + * we just truncate. + */ +static int elf_core_dump(struct coredump_params *cprm) +{ + int has_dumped = 0; + int segs, i; + struct elfhdr elf; + loff_t offset = 0, dataoff; + struct elf_note_info info = { }; + struct elf_phdr *phdr4note = NULL; + struct elf_shdr *shdr4extnum = NULL; + Elf_Half e_phnum; + elf_addr_t e_shoff; + + /* + * The number of segs are recored into ELF header as 16bit value. + * Please check DEFAULT_MAX_MAP_COUNT definition when you modify here. + */ + segs = cprm->vma_count + elf_core_extra_phdrs(cprm); + + /* for notes section */ + segs++; + + /* If segs > PN_XNUM(0xffff), then e_phnum overflows. To avoid + * this, kernel supports extended numbering. Have a look at + * include/linux/elf.h for further information. */ + e_phnum = segs > PN_XNUM ? PN_XNUM : segs; + + /* + * Collect all the non-memory information about the process for the + * notes. This also sets up the file header. + */ + if (!fill_note_info(&elf, e_phnum, &info, cprm)) + goto end_coredump; + + has_dumped = 1; + + offset += sizeof(elf); /* ELF header */ + offset += segs * sizeof(struct elf_phdr); /* Program headers */ + + /* Write notes phdr entry */ + { + size_t sz = info.size; + + /* For cell spufs and x86 xstate */ + sz += elf_coredump_extra_notes_size(); + + phdr4note = kmalloc_obj(*phdr4note); + if (!phdr4note) + goto end_coredump; + + fill_elf_note_phdr(phdr4note, sz, offset); + offset += sz; + } + + dataoff = offset = roundup(offset, ELF_EXEC_PAGESIZE); + + offset += cprm->vma_data_size; + offset += elf_core_extra_data_size(cprm); + e_shoff = offset; + + if (e_phnum == PN_XNUM) { + shdr4extnum = kmalloc_obj(*shdr4extnum); + if (!shdr4extnum) + goto end_coredump; + fill_extnum_info(&elf, shdr4extnum, e_shoff, segs); + } + + offset = dataoff; + + if (!dump_emit(cprm, &elf, sizeof(elf))) + goto end_coredump; + + if (!dump_emit(cprm, phdr4note, sizeof(*phdr4note))) + goto end_coredump; + + /* Write program headers for segments dump */ + for (i = 0; i < cprm->vma_count; i++) { + struct core_vma_metadata *meta = cprm->vma_meta + i; + struct elf_phdr phdr; + + phdr.p_type = PT_LOAD; + phdr.p_offset = offset; + phdr.p_vaddr = meta->start; + phdr.p_paddr = 0; + phdr.p_filesz = meta->dump_size; + phdr.p_memsz = meta->end - meta->start; + offset += phdr.p_filesz; + phdr.p_flags = 0; + if (meta->flags & VM_READ) + phdr.p_flags |= PF_R; + if (meta->flags & VM_WRITE) + phdr.p_flags |= PF_W; + if (meta->flags & VM_EXEC) + phdr.p_flags |= PF_X; + phdr.p_align = ELF_EXEC_PAGESIZE; + + if (!dump_emit(cprm, &phdr, sizeof(phdr))) + goto end_coredump; + } + + if (!elf_core_write_extra_phdrs(cprm, offset)) + goto end_coredump; + + /* write out the notes section */ + if (!write_note_info(&info, cprm)) + goto end_coredump; + + /* For cell spufs and x86 xstate */ + if (elf_coredump_extra_notes_write(cprm)) + goto end_coredump; + + /* Align to page */ + dump_skip_to(cprm, dataoff); + + for (i = 0; i < cprm->vma_count; i++) { + struct core_vma_metadata *meta = cprm->vma_meta + i; + + if (!dump_user_range(cprm, meta->start, meta->dump_size)) + goto end_coredump; + } + + if (!elf_core_write_extra_data(cprm)) + goto end_coredump; + + if (e_phnum == PN_XNUM) { + if (!dump_emit(cprm, shdr4extnum, sizeof(*shdr4extnum))) + goto end_coredump; + } + +end_coredump: + free_note_info(&info); + kfree(shdr4extnum); + kfree(phdr4note); + return has_dumped; +} + +#endif /* CONFIG_ELF_CORE */ + +static int __init init_elf_binfmt(void) +{ + register_binfmt(&elf_format); + return 0; +} + +static void __exit exit_elf_binfmt(void) +{ + /* Remove the COFF and ELF loaders. */ + unregister_binfmt(&elf_format); +} + +core_initcall(init_elf_binfmt); +module_exit(exit_elf_binfmt); + +#ifdef CONFIG_BINFMT_ELF_KUNIT_TEST +#include "tests/binfmt_elf_kunit.c" +#endif diff --git a/fs/binfmt_elf_fdpic.c b/fs/binfmt_elf_fdpic.c new file mode 100644 index 000000000..068c46875 --- /dev/null +++ b/fs/binfmt_elf_fdpic.c @@ -0,0 +1,1672 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* binfmt_elf_fdpic.c: FDPIC ELF binary format + * + * Copyright (C) 2003, 2004, 2006 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + * Derived from binfmt_elf.c + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +typedef char *elf_caddr_t; + +#if 0 +#define kdebug(fmt, ...) printk("FDPIC "fmt"\n" ,##__VA_ARGS__ ) +#else +#define kdebug(fmt, ...) do {} while(0) +#endif + +#if 0 +#define kdcore(fmt, ...) printk("FDPIC "fmt"\n" ,##__VA_ARGS__ ) +#else +#define kdcore(fmt, ...) do {} while(0) +#endif + +MODULE_LICENSE("GPL"); + +static int load_elf_fdpic_binary(struct linux_binprm *); +static int elf_fdpic_fetch_phdrs(struct elf_fdpic_params *, struct file *); +static int elf_fdpic_map_file(struct elf_fdpic_params *, struct file *, + struct mm_struct *, const char *); + +static int create_elf_fdpic_tables(struct linux_binprm *, struct mm_struct *, + struct elf_fdpic_params *, + struct elf_fdpic_params *); + +#ifndef CONFIG_MMU +static int elf_fdpic_map_file_constdisp_on_uclinux(struct elf_fdpic_params *, + struct file *, + struct mm_struct *); +#endif + +static int elf_fdpic_map_file_by_direct_mmap(struct elf_fdpic_params *, + struct file *, struct mm_struct *); + +#ifdef CONFIG_ELF_CORE +static int elf_fdpic_core_dump(struct coredump_params *cprm); +#endif + +static struct linux_binfmt elf_fdpic_format = { + .module = THIS_MODULE, + .load_binary = load_elf_fdpic_binary, +#ifdef CONFIG_ELF_CORE + .core_dump = elf_fdpic_core_dump, + .min_coredump = ELF_EXEC_PAGESIZE, +#endif +}; + +static int __init init_elf_fdpic_binfmt(void) +{ + register_binfmt(&elf_fdpic_format); + return 0; +} + +static void __exit exit_elf_fdpic_binfmt(void) +{ + unregister_binfmt(&elf_fdpic_format); +} + +core_initcall(init_elf_fdpic_binfmt); +module_exit(exit_elf_fdpic_binfmt); + +static int is_elf(struct elfhdr *hdr, struct file *file) +{ + if (memcmp(hdr->e_ident, ELFMAG, SELFMAG) != 0) + return 0; + if (hdr->e_type != ET_EXEC && hdr->e_type != ET_DYN) + return 0; + if (!elf_check_arch(hdr)) + return 0; + if (!can_mmap_file(file)) + return 0; + return 1; +} + +#ifndef elf_check_fdpic +#define elf_check_fdpic(x) 0 +#endif + +#ifndef elf_check_const_displacement +#define elf_check_const_displacement(x) 0 +#endif + +static int is_constdisp(struct elfhdr *hdr) +{ + if (!elf_check_fdpic(hdr)) + return 1; + if (elf_check_const_displacement(hdr)) + return 1; + return 0; +} + +/*****************************************************************************/ +/* + * read the program headers table into memory + */ +static int elf_fdpic_fetch_phdrs(struct elf_fdpic_params *params, + struct file *file) +{ + struct elf_phdr *phdr; + unsigned long size; + int retval, loop; + loff_t pos = params->hdr.e_phoff; + + if (params->hdr.e_phentsize != sizeof(struct elf_phdr)) + return -ENOMEM; + if (params->hdr.e_phnum > 65536U / sizeof(struct elf_phdr)) + return -ENOMEM; + + size = params->hdr.e_phnum * sizeof(struct elf_phdr); + params->phdrs = kmalloc(size, GFP_KERNEL); + if (!params->phdrs) + return -ENOMEM; + + retval = kernel_read(file, params->phdrs, size, &pos); + if (unlikely(retval != size)) + return retval < 0 ? retval : -ENOEXEC; + + /* determine stack size for this binary */ + phdr = params->phdrs; + for (loop = 0; loop < params->hdr.e_phnum; loop++, phdr++) { + if (phdr->p_type != PT_GNU_STACK) + continue; + + if (phdr->p_flags & PF_X) + params->flags |= ELF_FDPIC_FLAG_EXEC_STACK; + else + params->flags |= ELF_FDPIC_FLAG_NOEXEC_STACK; + + params->stack_size = phdr->p_memsz; + break; + } + + return 0; +} + +/*****************************************************************************/ +/* + * load an fdpic binary into various bits of memory + */ +static int load_elf_fdpic_binary(struct linux_binprm *bprm) +{ + struct elf_fdpic_params exec_params, interp_params; + struct pt_regs *regs = current_pt_regs(); + struct elf_phdr *phdr; + unsigned long stack_size, entryaddr; +#ifdef ELF_FDPIC_PLAT_INIT + unsigned long dynaddr; +#endif +#ifndef CONFIG_MMU + unsigned long stack_prot; +#endif + struct file *interpreter = NULL; /* to shut gcc up */ + char *interpreter_name = NULL; + int executable_stack; + int retval, i; + loff_t pos; + + kdebug("____ LOAD %d ____", current->pid); + + memset(&exec_params, 0, sizeof(exec_params)); + memset(&interp_params, 0, sizeof(interp_params)); + + exec_params.hdr = *(struct elfhdr *) bprm->buf; + exec_params.flags = ELF_FDPIC_FLAG_PRESENT | ELF_FDPIC_FLAG_EXECUTABLE; + + /* check that this is a binary we know how to deal with */ + retval = -ENOEXEC; + if (!is_elf(&exec_params.hdr, bprm->file)) + goto error; + if (!elf_check_fdpic(&exec_params.hdr)) { +#ifdef CONFIG_MMU + /* binfmt_elf handles non-fdpic elf except on nommu */ + goto error; +#else + /* nommu can only load ET_DYN (PIE) ELF */ + if (exec_params.hdr.e_type != ET_DYN) + goto error; +#endif + } + + /* read the program header table */ + retval = elf_fdpic_fetch_phdrs(&exec_params, bprm->file); + if (retval < 0) + goto error; + + /* scan for a program header that specifies an interpreter */ + phdr = exec_params.phdrs; + + for (i = 0; i < exec_params.hdr.e_phnum; i++, phdr++) { + switch (phdr->p_type) { + case PT_INTERP: + /* elf ABI allows only one interpreter */ + if (interpreter_name) + continue; + + retval = -ENOMEM; + if (phdr->p_filesz > PATH_MAX) + goto error; + retval = -ENOENT; + if (phdr->p_filesz < 2) + goto error; + + /* read the name of the interpreter into memory */ + interpreter_name = kmalloc(phdr->p_filesz, GFP_KERNEL); + if (!interpreter_name) + goto error; + + pos = phdr->p_offset; + retval = kernel_read(bprm->file, interpreter_name, + phdr->p_filesz, &pos); + if (unlikely(retval != phdr->p_filesz)) { + if (retval >= 0) + retval = -ENOEXEC; + goto error; + } + + retval = -ENOENT; + if (interpreter_name[phdr->p_filesz - 1] != '\0') + goto error; + + kdebug("Using ELF interpreter %s", interpreter_name); + + /* replace the program with the interpreter */ + interpreter = bprm_open_interpreter(bprm, + interpreter_name); + retval = PTR_ERR(interpreter); + if (IS_ERR(interpreter)) { + interpreter = NULL; + goto error; + } + + /* + * If the binary is not readable then enforce + * mm->dumpable = 0 regardless of the interpreter's + * permissions. + */ + would_dump(bprm, interpreter); + + pos = 0; + retval = kernel_read(interpreter, bprm->buf, + BINPRM_BUF_SIZE, &pos); + if (unlikely(retval != BINPRM_BUF_SIZE)) { + if (retval >= 0) + retval = -ENOEXEC; + goto error; + } + + interp_params.hdr = *((struct elfhdr *) bprm->buf); + break; + + case PT_LOAD: +#ifdef CONFIG_MMU + if (exec_params.load_addr == 0) + exec_params.load_addr = phdr->p_vaddr; +#endif + break; + } + + } + + /* No PT_INTERP to substitute for: the override does not apply. */ + bprm_drop_loader(bprm); + + if (is_constdisp(&exec_params.hdr)) + exec_params.flags |= ELF_FDPIC_FLAG_CONSTDISP; + + /* perform insanity checks on the interpreter */ + if (interpreter_name) { + retval = -ELIBBAD; + if (!is_elf(&interp_params.hdr, interpreter)) + goto error; + + interp_params.flags = ELF_FDPIC_FLAG_PRESENT; + + /* read the interpreter's program header table */ + retval = elf_fdpic_fetch_phdrs(&interp_params, interpreter); + if (retval < 0) + goto error; + } + + stack_size = exec_params.stack_size; + if (exec_params.flags & ELF_FDPIC_FLAG_EXEC_STACK) + executable_stack = EXSTACK_ENABLE_X; + else if (exec_params.flags & ELF_FDPIC_FLAG_NOEXEC_STACK) + executable_stack = EXSTACK_DISABLE_X; + else + executable_stack = EXSTACK_DEFAULT; + + if (stack_size == 0 && interp_params.flags & ELF_FDPIC_FLAG_PRESENT) { + stack_size = interp_params.stack_size; + if (interp_params.flags & ELF_FDPIC_FLAG_EXEC_STACK) + executable_stack = EXSTACK_ENABLE_X; + else if (interp_params.flags & ELF_FDPIC_FLAG_NOEXEC_STACK) + executable_stack = EXSTACK_DISABLE_X; + else + executable_stack = EXSTACK_DEFAULT; + } + + retval = -ENOEXEC; + if (stack_size == 0) + stack_size = 131072UL; /* same as exec.c's default commit */ + + if (is_constdisp(&interp_params.hdr)) + interp_params.flags |= ELF_FDPIC_FLAG_CONSTDISP; + + /* flush all traces of the currently running executable */ + retval = begin_new_exec(bprm); + if (retval) + goto error; + + /* there's now no turning back... the old userspace image is dead, + * defunct, deceased, etc. + */ + SET_PERSONALITY(exec_params.hdr); + if (elf_check_fdpic(&exec_params.hdr)) + current->personality |= PER_LINUX_FDPIC; + if (elf_read_implies_exec(&exec_params.hdr, executable_stack)) + current->personality |= READ_IMPLIES_EXEC; + + setup_new_exec(bprm); + + set_binfmt(&elf_fdpic_format); + + current->mm->start_code = 0; + current->mm->end_code = 0; + current->mm->start_stack = 0; + current->mm->start_data = 0; + current->mm->end_data = 0; + current->mm->context.exec_fdpic_loadmap = 0; + current->mm->context.interp_fdpic_loadmap = 0; + +#ifdef CONFIG_MMU + elf_fdpic_arch_lay_out_mm(&exec_params, + &interp_params, + ¤t->mm->start_stack, + ¤t->mm->start_brk); + + retval = setup_arg_pages(bprm, current->mm->start_stack, + executable_stack); + if (retval < 0) + goto error; +#ifdef ARCH_HAS_SETUP_ADDITIONAL_PAGES + retval = arch_setup_additional_pages(bprm, !!interpreter_name); + if (retval < 0) + goto error; +#endif +#endif + + /* load the executable and interpreter into memory */ + retval = elf_fdpic_map_file(&exec_params, bprm->file, current->mm, + "executable"); + if (retval < 0) + goto error; + + if (interpreter_name) { + retval = elf_fdpic_map_file(&interp_params, interpreter, + current->mm, "interpreter"); + if (retval < 0) { + printk(KERN_ERR "Unable to load interpreter\n"); + goto error; + } + + exe_file_allow_write_access(interpreter); + fput(interpreter); + interpreter = NULL; + } + +#ifdef CONFIG_MMU + if (!current->mm->start_brk) + current->mm->start_brk = current->mm->end_data; + + current->mm->brk = current->mm->start_brk = + PAGE_ALIGN(current->mm->start_brk); + +#else + /* create a stack area and zero-size brk area */ + stack_size = (stack_size + PAGE_SIZE - 1) & PAGE_MASK; + if (stack_size < PAGE_SIZE * 2) + stack_size = PAGE_SIZE * 2; + + stack_prot = PROT_READ | PROT_WRITE; + if (executable_stack == EXSTACK_ENABLE_X || + (executable_stack == EXSTACK_DEFAULT && VM_STACK_FLAGS & VM_EXEC)) + stack_prot |= PROT_EXEC; + + current->mm->start_brk = vm_mmap(NULL, 0, stack_size, stack_prot, + MAP_PRIVATE | MAP_ANONYMOUS | + MAP_UNINITIALIZED | MAP_GROWSDOWN, + 0); + + if (IS_ERR_VALUE(current->mm->start_brk)) { + retval = current->mm->start_brk; + current->mm->start_brk = 0; + goto error; + } + + current->mm->brk = current->mm->start_brk; + current->mm->context.end_brk = current->mm->start_brk; + current->mm->start_stack = current->mm->start_brk + stack_size; +#endif + + retval = create_elf_fdpic_tables(bprm, current->mm, &exec_params, + &interp_params); + if (retval < 0) + goto error; + + kdebug("- start_code %lx", current->mm->start_code); + kdebug("- end_code %lx", current->mm->end_code); + kdebug("- start_data %lx", current->mm->start_data); + kdebug("- end_data %lx", current->mm->end_data); + kdebug("- start_brk %lx", current->mm->start_brk); + kdebug("- brk %lx", current->mm->brk); + kdebug("- start_stack %lx", current->mm->start_stack); + +#ifdef ELF_FDPIC_PLAT_INIT + /* + * The ABI may specify that certain registers be set up in special + * ways (on i386 %edx is the address of a DT_FINI function, for + * example. This macro performs whatever initialization to + * the regs structure is required. + */ + dynaddr = interp_params.dynamic_addr ?: exec_params.dynamic_addr; + ELF_FDPIC_PLAT_INIT(regs, exec_params.map_addr, interp_params.map_addr, + dynaddr); +#endif + + finalize_exec(bprm); + /* everything is now ready... get the userspace context ready to roll */ + entryaddr = interp_params.entry_addr ?: exec_params.entry_addr; + start_thread(regs, entryaddr, current->mm->start_stack); + + retval = 0; + +error: + if (interpreter) { + exe_file_allow_write_access(interpreter); + fput(interpreter); + } + kfree(interpreter_name); + kfree(exec_params.phdrs); + kfree(exec_params.loadmap); + kfree(interp_params.phdrs); + kfree(interp_params.loadmap); + return retval; +} + +/*****************************************************************************/ + +#ifndef ELF_BASE_PLATFORM +/* + * AT_BASE_PLATFORM indicates the "real" hardware/microarchitecture. + * If the arch defines ELF_BASE_PLATFORM (in asm/elf.h), the value + * will be copied to the user stack in the same manner as AT_PLATFORM. + */ +#define ELF_BASE_PLATFORM NULL +#endif + +/* + * present useful information to the program by shovelling it onto the new + * process's stack + */ +static int create_elf_fdpic_tables(struct linux_binprm *bprm, + struct mm_struct *mm, + struct elf_fdpic_params *exec_params, + struct elf_fdpic_params *interp_params) +{ + const struct cred *cred = current_cred(); + unsigned long sp, csp, nitems; + elf_caddr_t __user *argv, *envp; + size_t platform_len = 0, len; + char *k_platform, *k_base_platform; + char __user *u_platform, *u_base_platform, *p; + int loop; + int ei_index; + elf_addr_t *elf_info; + +#ifdef CONFIG_MMU + /* In some cases (e.g. Hyper-Threading), we want to avoid L1 evictions + * by the processes running on the same package. One thing we can do is + * to shuffle the initial stack for them, so we give the architecture + * an opportunity to do so here. + */ + sp = arch_align_stack(bprm->p); +#else + sp = mm->start_stack; + + /* stack the program arguments and environment */ + if (transfer_args_to_stack(bprm, &sp) < 0) + return -EFAULT; + sp &= ~15; +#endif + + /* + * If this architecture has a platform capability string, copy it + * to userspace. In some cases (Sparc), this info is impossible + * for userspace to get any other way, in others (i386) it is + * merely difficult. + */ + k_platform = ELF_PLATFORM; + u_platform = NULL; + + if (k_platform) { + platform_len = strlen(k_platform) + 1; + sp -= platform_len; + u_platform = (char __user *) sp; + if (copy_to_user(u_platform, k_platform, platform_len) != 0) + return -EFAULT; + } + + /* + * If this architecture has a "base" platform capability + * string, copy it to userspace. + */ + k_base_platform = ELF_BASE_PLATFORM; + u_base_platform = NULL; + + if (k_base_platform) { + platform_len = strlen(k_base_platform) + 1; + sp -= platform_len; + u_base_platform = (char __user *) sp; + if (copy_to_user(u_base_platform, k_base_platform, platform_len) != 0) + return -EFAULT; + } + + sp &= ~7UL; + + /* stack the load map(s) */ + len = sizeof(struct elf_fdpic_loadmap); + len += sizeof(struct elf_fdpic_loadseg) * exec_params->loadmap->nsegs; + sp = (sp - len) & ~7UL; + exec_params->map_addr = sp; + + if (copy_to_user((void __user *) sp, exec_params->loadmap, len) != 0) + return -EFAULT; + + current->mm->context.exec_fdpic_loadmap = (unsigned long) sp; + + if (interp_params->loadmap) { + len = sizeof(struct elf_fdpic_loadmap); + len += sizeof(struct elf_fdpic_loadseg) * + interp_params->loadmap->nsegs; + sp = (sp - len) & ~7UL; + interp_params->map_addr = sp; + + if (copy_to_user((void __user *) sp, interp_params->loadmap, + len) != 0) + return -EFAULT; + + current->mm->context.interp_fdpic_loadmap = (unsigned long) sp; + } + + /* force 16 byte _final_ alignment here for generality */ +#define DLINFO_ITEMS 15 + + nitems = 1 + DLINFO_ITEMS + (k_platform ? 1 : 0) + + (k_base_platform ? 1 : 0) + AT_VECTOR_SIZE_ARCH; + + if (bprm->have_execfd) + nitems++; +#ifdef ELF_HWCAP2 + nitems++; +#endif +#ifdef ELF_HWCAP3 + nitems++; +#endif +#ifdef ELF_HWCAP4 + nitems++; +#endif + + csp = sp; + sp -= nitems * 2 * sizeof(unsigned long); + sp -= (bprm->envc + 1) * sizeof(char *); /* envv[] */ + sp -= (bprm->argc + 1) * sizeof(char *); /* argv[] */ + sp -= 1 * sizeof(unsigned long); /* argc */ + + csp -= sp & 15UL; + sp -= sp & 15UL; + + /* Create the ELF interpreter info */ + elf_info = (elf_addr_t *)mm->saved_auxv; + /* update AT_VECTOR_SIZE_BASE if the number of NEW_AUX_ENT() changes */ +#define NEW_AUX_ENT(id, val) \ + do { \ + *elf_info++ = id; \ + *elf_info++ = val; \ + } while (0) + +#ifdef ARCH_DLINFO + /* + * ARCH_DLINFO must come first so PPC can do its special alignment of + * AUXV. + * update AT_VECTOR_SIZE_ARCH if the number of NEW_AUX_ENT() in + * ARCH_DLINFO changes + */ + ARCH_DLINFO; +#endif + NEW_AUX_ENT(AT_HWCAP, ELF_HWCAP); +#ifdef ELF_HWCAP2 + NEW_AUX_ENT(AT_HWCAP2, ELF_HWCAP2); +#endif +#ifdef ELF_HWCAP3 + NEW_AUX_ENT(AT_HWCAP3, ELF_HWCAP3); +#endif +#ifdef ELF_HWCAP4 + NEW_AUX_ENT(AT_HWCAP4, ELF_HWCAP4); +#endif + NEW_AUX_ENT(AT_PAGESZ, PAGE_SIZE); + NEW_AUX_ENT(AT_CLKTCK, CLOCKS_PER_SEC); + NEW_AUX_ENT(AT_PHDR, exec_params->ph_addr); + NEW_AUX_ENT(AT_PHENT, sizeof(struct elf_phdr)); + NEW_AUX_ENT(AT_PHNUM, exec_params->hdr.e_phnum); + NEW_AUX_ENT(AT_BASE, interp_params->elfhdr_addr); + NEW_AUX_ENT(AT_FLAGS, bprm_at_flags(bprm)); + NEW_AUX_ENT(AT_ENTRY, exec_params->entry_addr); + NEW_AUX_ENT(AT_UID, (elf_addr_t) from_kuid_munged(cred->user_ns, cred->uid)); + NEW_AUX_ENT(AT_EUID, (elf_addr_t) from_kuid_munged(cred->user_ns, cred->euid)); + NEW_AUX_ENT(AT_GID, (elf_addr_t) from_kgid_munged(cred->user_ns, cred->gid)); + NEW_AUX_ENT(AT_EGID, (elf_addr_t) from_kgid_munged(cred->user_ns, cred->egid)); + NEW_AUX_ENT(AT_SECURE, bprm->secureexec); + NEW_AUX_ENT(AT_EXECFN, bprm->exec); + if (k_platform) + NEW_AUX_ENT(AT_PLATFORM, + (elf_addr_t)(unsigned long)u_platform); + if (k_base_platform) + NEW_AUX_ENT(AT_BASE_PLATFORM, + (elf_addr_t)(unsigned long)u_base_platform); + if (bprm->have_execfd) + NEW_AUX_ENT(AT_EXECFD, bprm->execfd); +#undef NEW_AUX_ENT + /* AT_NULL is zero; clear the rest too */ + memset(elf_info, 0, (char *)mm->saved_auxv + + sizeof(mm->saved_auxv) - (char *)elf_info); + + /* And advance past the AT_NULL entry. */ + elf_info += 2; + + ei_index = elf_info - (elf_addr_t *)mm->saved_auxv; + csp -= ei_index * sizeof(elf_addr_t); + + /* Put the elf_info on the stack in the right place. */ + if (copy_to_user((void __user *)csp, mm->saved_auxv, + ei_index * sizeof(elf_addr_t))) + return -EFAULT; + + /* allocate room for argv[] and envv[] */ + csp -= (bprm->envc + 1) * sizeof(elf_caddr_t); + envp = (elf_caddr_t __user *) csp; + csp -= (bprm->argc + 1) * sizeof(elf_caddr_t); + argv = (elf_caddr_t __user *) csp; + + /* stack argc */ + csp -= sizeof(unsigned long); + if (put_user(bprm->argc, (unsigned long __user *) csp)) + return -EFAULT; + + BUG_ON(csp != sp); + + /* fill in the argv[] array */ +#ifdef CONFIG_MMU + current->mm->arg_start = bprm->p; +#else + current->mm->arg_start = current->mm->start_stack - + (MAX_ARG_PAGES * PAGE_SIZE - bprm->p); +#endif + + p = (char __user *) current->mm->arg_start; + for (loop = bprm->argc; loop > 0; loop--) { + if (put_user((elf_caddr_t) p, argv++)) + return -EFAULT; + len = strnlen_user(p, MAX_ARG_STRLEN); + if (!len || len > MAX_ARG_STRLEN) + return -EINVAL; + p += len; + } + if (put_user(NULL, argv)) + return -EFAULT; + current->mm->arg_end = (unsigned long) p; + + /* fill in the envv[] array */ + current->mm->env_start = (unsigned long) p; + for (loop = bprm->envc; loop > 0; loop--) { + if (put_user((elf_caddr_t)(unsigned long) p, envp++)) + return -EFAULT; + len = strnlen_user(p, MAX_ARG_STRLEN); + if (!len || len > MAX_ARG_STRLEN) + return -EINVAL; + p += len; + } + if (put_user(NULL, envp)) + return -EFAULT; + current->mm->env_end = (unsigned long) p; + + mm->start_stack = (unsigned long) sp; + return 0; +} + +/*****************************************************************************/ +/* + * load the appropriate binary image (executable or interpreter) into memory + * - we assume no MMU is available + * - if no other PIC bits are set in params->hdr->e_flags + * - we assume that the LOADable segments in the binary are independently relocatable + * - we assume R/O executable segments are shareable + * - else + * - we assume the loadable parts of the image to require fixed displacement + * - the image is not shareable + */ +static int elf_fdpic_map_file(struct elf_fdpic_params *params, + struct file *file, + struct mm_struct *mm, + const char *what) +{ + struct elf_fdpic_loadmap *loadmap; +#ifdef CONFIG_MMU + struct elf_fdpic_loadseg *mseg; + unsigned long load_addr; +#endif + struct elf_fdpic_loadseg *seg; + struct elf_phdr *phdr; + unsigned nloads, tmp; + unsigned long stop; + int loop, ret; + + /* allocate a load map table */ + nloads = 0; + for (loop = 0; loop < params->hdr.e_phnum; loop++) + if (params->phdrs[loop].p_type == PT_LOAD) + nloads++; + + if (nloads == 0) + return -ELIBBAD; + + loadmap = kzalloc_flex(*loadmap, segs, nloads); + if (!loadmap) + return -ENOMEM; + + params->loadmap = loadmap; + + loadmap->version = ELF_FDPIC_LOADMAP_VERSION; + loadmap->nsegs = nloads; + + /* map the requested LOADs into the memory space */ + switch (params->flags & ELF_FDPIC_FLAG_ARRANGEMENT) { + case ELF_FDPIC_FLAG_CONSTDISP: + case ELF_FDPIC_FLAG_CONTIGUOUS: +#ifndef CONFIG_MMU + ret = elf_fdpic_map_file_constdisp_on_uclinux(params, file, mm); + if (ret < 0) + return ret; + break; +#endif + default: + ret = elf_fdpic_map_file_by_direct_mmap(params, file, mm); + if (ret < 0) + return ret; + break; + } + + /* map the entry point */ + if (params->hdr.e_entry) { + seg = loadmap->segs; + for (loop = loadmap->nsegs; loop > 0; loop--, seg++) { + if (params->hdr.e_entry >= seg->p_vaddr && + params->hdr.e_entry < seg->p_vaddr + seg->p_memsz) { + params->entry_addr = + (params->hdr.e_entry - seg->p_vaddr) + + seg->addr; + break; + } + } + } + + /* determine where the program header table has wound up if mapped */ + stop = params->hdr.e_phoff; + stop += params->hdr.e_phnum * sizeof (struct elf_phdr); + phdr = params->phdrs; + + for (loop = 0; loop < params->hdr.e_phnum; loop++, phdr++) { + if (phdr->p_type != PT_LOAD) + continue; + + if (phdr->p_offset > params->hdr.e_phoff || + phdr->p_offset + phdr->p_filesz < stop) + continue; + + seg = loadmap->segs; + for (loop = loadmap->nsegs; loop > 0; loop--, seg++) { + if (phdr->p_vaddr >= seg->p_vaddr && + phdr->p_vaddr + phdr->p_filesz <= + seg->p_vaddr + seg->p_memsz) { + params->ph_addr = + (phdr->p_vaddr - seg->p_vaddr) + + seg->addr + + params->hdr.e_phoff - phdr->p_offset; + break; + } + } + break; + } + + /* determine where the dynamic section has wound up if there is one */ + phdr = params->phdrs; + for (loop = 0; loop < params->hdr.e_phnum; loop++, phdr++) { + if (phdr->p_type != PT_DYNAMIC) + continue; + + seg = loadmap->segs; + for (loop = loadmap->nsegs; loop > 0; loop--, seg++) { + if (phdr->p_vaddr >= seg->p_vaddr && + phdr->p_vaddr + phdr->p_memsz <= + seg->p_vaddr + seg->p_memsz) { + Elf_Dyn __user *dyn; + Elf_Sword d_tag; + + params->dynamic_addr = + (phdr->p_vaddr - seg->p_vaddr) + + seg->addr; + + /* check the dynamic section contains at least + * one item, and that the last item is a NULL + * entry */ + if (phdr->p_memsz == 0 || + phdr->p_memsz % sizeof(Elf_Dyn) != 0) + goto dynamic_error; + + tmp = phdr->p_memsz / sizeof(Elf_Dyn); + dyn = (Elf_Dyn __user *)params->dynamic_addr; + if (get_user(d_tag, &dyn[tmp - 1].d_tag) || + d_tag != 0) + goto dynamic_error; + break; + } + } + break; + } + + /* now elide adjacent segments in the load map on MMU linux + * - on uClinux the holes between may actually be filled with system + * stuff or stuff from other processes + */ +#ifdef CONFIG_MMU + nloads = loadmap->nsegs; + mseg = loadmap->segs; + seg = mseg + 1; + for (loop = 1; loop < nloads; loop++) { + /* see if we have a candidate for merging */ + if (seg->p_vaddr - mseg->p_vaddr == seg->addr - mseg->addr) { + load_addr = PAGE_ALIGN(mseg->addr + mseg->p_memsz); + if (load_addr == (seg->addr & PAGE_MASK)) { + mseg->p_memsz += + load_addr - + (mseg->addr + mseg->p_memsz); + mseg->p_memsz += seg->addr & ~PAGE_MASK; + mseg->p_memsz += seg->p_memsz; + loadmap->nsegs--; + continue; + } + } + + mseg++; + if (mseg != seg) + *mseg = *seg; + } +#endif + + kdebug("Mapped Object [%s]:", what); + kdebug("- elfhdr : %lx", params->elfhdr_addr); + kdebug("- entry : %lx", params->entry_addr); + kdebug("- PHDR[] : %lx", params->ph_addr); + kdebug("- DYNAMIC[]: %lx", params->dynamic_addr); + seg = loadmap->segs; + for (loop = 0; loop < loadmap->nsegs; loop++, seg++) + kdebug("- LOAD[%d] : %08llx-%08llx [va=%llx ms=%llx]", + loop, + (unsigned long long) seg->addr, + (unsigned long long) seg->addr + seg->p_memsz - 1, + (unsigned long long) seg->p_vaddr, + (unsigned long long) seg->p_memsz); + + return 0; + +dynamic_error: + printk("ELF FDPIC %s with invalid DYNAMIC section (inode=%llu)\n", + what, file_inode(file)->i_ino); + return -ELIBBAD; +} + +/*****************************************************************************/ +/* + * map a file with constant displacement under uClinux + */ +#ifndef CONFIG_MMU +static int elf_fdpic_map_file_constdisp_on_uclinux( + struct elf_fdpic_params *params, + struct file *file, + struct mm_struct *mm) +{ + struct elf_fdpic_loadseg *seg; + struct elf_phdr *phdr; + unsigned long load_addr, base = ULONG_MAX, top = 0, maddr = 0; + int loop, ret; + + load_addr = params->load_addr; + seg = params->loadmap->segs; + + /* determine the bounds of the contiguous overall allocation we must + * make */ + phdr = params->phdrs; + for (loop = 0; loop < params->hdr.e_phnum; loop++, phdr++) { + if (params->phdrs[loop].p_type != PT_LOAD) + continue; + + if (base > phdr->p_vaddr) + base = phdr->p_vaddr; + if (top < phdr->p_vaddr + phdr->p_memsz) + top = phdr->p_vaddr + phdr->p_memsz; + } + + /* allocate one big anon block for everything */ + maddr = vm_mmap(NULL, load_addr, top - base, + PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE, 0); + if (IS_ERR_VALUE(maddr)) + return (int) maddr; + + if (load_addr != 0) + load_addr += PAGE_ALIGN(top - base); + + /* and then load the file segments into it */ + phdr = params->phdrs; + for (loop = 0; loop < params->hdr.e_phnum; loop++, phdr++) { + if (params->phdrs[loop].p_type != PT_LOAD) + continue; + + seg->addr = maddr + (phdr->p_vaddr - base); + seg->p_vaddr = phdr->p_vaddr; + seg->p_memsz = phdr->p_memsz; + + ret = read_code(file, seg->addr, phdr->p_offset, + phdr->p_filesz); + if (ret < 0) + return ret; + + /* map the ELF header address if in this segment */ + if (phdr->p_offset == 0) + params->elfhdr_addr = seg->addr; + + /* clear any space allocated but not loaded */ + if (phdr->p_filesz < phdr->p_memsz) { + if (clear_user((void *) (seg->addr + phdr->p_filesz), + phdr->p_memsz - phdr->p_filesz)) + return -EFAULT; + } + + if (mm) { + if (phdr->p_flags & PF_X) { + if (!mm->start_code) { + mm->start_code = seg->addr; + mm->end_code = seg->addr + + phdr->p_memsz; + } + } else if (!mm->start_data) { + mm->start_data = seg->addr; + mm->end_data = seg->addr + phdr->p_memsz; + } + } + + seg++; + } + + return 0; +} +#endif + +/*****************************************************************************/ +/* + * map a binary by direct mmap() of the individual PT_LOAD segments + */ +static int elf_fdpic_map_file_by_direct_mmap(struct elf_fdpic_params *params, + struct file *file, + struct mm_struct *mm) +{ + struct elf_fdpic_loadseg *seg; + struct elf_phdr *phdr; + unsigned long load_addr, delta_vaddr; + int loop, dvset; + + load_addr = params->load_addr; + delta_vaddr = 0; + dvset = 0; + + seg = params->loadmap->segs; + + /* deal with each load segment separately */ + phdr = params->phdrs; + for (loop = 0; loop < params->hdr.e_phnum; loop++, phdr++) { + unsigned long maddr, disp, excess; + int prot = 0, flags; + + if (phdr->p_type != PT_LOAD) + continue; + + kdebug("[LOAD] va=%lx of=%lx fs=%lx ms=%lx", + (unsigned long) phdr->p_vaddr, + (unsigned long) phdr->p_offset, + (unsigned long) phdr->p_filesz, + (unsigned long) phdr->p_memsz); + + /* determine the mapping parameters */ + if (phdr->p_flags & PF_R) prot |= PROT_READ; + if (phdr->p_flags & PF_W) prot |= PROT_WRITE; + if (phdr->p_flags & PF_X) prot |= PROT_EXEC; + + flags = MAP_PRIVATE; + maddr = 0; + + switch (params->flags & ELF_FDPIC_FLAG_ARRANGEMENT) { + case ELF_FDPIC_FLAG_INDEPENDENT: + /* PT_LOADs are independently locatable */ + break; + + case ELF_FDPIC_FLAG_HONOURVADDR: + /* the specified virtual address must be honoured */ + maddr = phdr->p_vaddr; + flags |= MAP_FIXED; + break; + + case ELF_FDPIC_FLAG_CONSTDISP: + /* constant displacement + * - can be mapped anywhere, but must be mapped as a + * unit + */ + if (!dvset) { + maddr = load_addr; + delta_vaddr = phdr->p_vaddr; + dvset = 1; + } else { + maddr = load_addr + phdr->p_vaddr - delta_vaddr; + flags |= MAP_FIXED; + } + break; + + case ELF_FDPIC_FLAG_CONTIGUOUS: + /* contiguity handled later */ + break; + + default: + BUG(); + } + + maddr &= PAGE_MASK; + + /* create the mapping */ + disp = phdr->p_vaddr & ~PAGE_MASK; + maddr = vm_mmap(file, maddr, phdr->p_memsz + disp, prot, flags, + phdr->p_offset - disp); + + kdebug("mmap[%d] sz=%llx pr=%x fl=%x of=%llx --> %08lx", + loop, (unsigned long long) phdr->p_memsz + disp, + prot, flags, (unsigned long long) phdr->p_offset - disp, + maddr); + + if (IS_ERR_VALUE(maddr)) + return (int) maddr; + + if ((params->flags & ELF_FDPIC_FLAG_ARRANGEMENT) == + ELF_FDPIC_FLAG_CONTIGUOUS) + load_addr += PAGE_ALIGN(phdr->p_memsz + disp); + + seg->addr = maddr + disp; + seg->p_vaddr = phdr->p_vaddr; + seg->p_memsz = phdr->p_memsz; + + /* map the ELF header address if in this segment */ + if (phdr->p_offset == 0) + params->elfhdr_addr = seg->addr; + + /* clear the bit between beginning of mapping and beginning of + * PT_LOAD */ + if (prot & PROT_WRITE && disp > 0) { + kdebug("clear[%d] ad=%lx sz=%lx", loop, maddr, disp); + if (clear_user((void __user *) maddr, disp)) + return -EFAULT; + maddr += disp; + } + + /* clear any space allocated but not loaded + * - on uClinux we can just clear the lot + * - on MMU linux we'll get a SIGBUS beyond the last page + * extant in the file + */ + excess = phdr->p_memsz - phdr->p_filesz; + +#ifdef CONFIG_MMU + unsigned long excess1 + = PAGE_SIZE - ((maddr + phdr->p_filesz) & ~PAGE_MASK); + if (excess > excess1) { + unsigned long xaddr = maddr + phdr->p_filesz + excess1; + unsigned long xmaddr; + + flags |= MAP_FIXED | MAP_ANONYMOUS; + xmaddr = vm_mmap(NULL, xaddr, excess - excess1, + prot, flags, 0); + + kdebug("mmap[%d] " + " ad=%lx sz=%lx pr=%x fl=%x of=0 --> %08lx", + loop, xaddr, excess - excess1, prot, flags, + xmaddr); + + if (xmaddr != xaddr) + return -ENOMEM; + } + + if (prot & PROT_WRITE && excess1 > 0) { + kdebug("clear[%d] ad=%lx sz=%lx", + loop, maddr + phdr->p_filesz, excess1); + if (clear_user((void __user *) maddr + phdr->p_filesz, + excess1)) + return -EFAULT; + } + +#else + if (excess > 0) { + kdebug("clear[%d] ad=%llx sz=%lx", loop, + (unsigned long long) maddr + phdr->p_filesz, + excess); + if (clear_user((void *) maddr + phdr->p_filesz, excess)) + return -EFAULT; + } +#endif + + if (mm) { + if (phdr->p_flags & PF_X) { + if (!mm->start_code) { + mm->start_code = maddr; + mm->end_code = maddr + phdr->p_memsz; + } + } else if (!mm->start_data) { + mm->start_data = maddr; + mm->end_data = maddr + phdr->p_memsz; + } + } + + seg++; + } + + return 0; +} + +/*****************************************************************************/ +/* + * ELF-FDPIC core dumper + * + * Modelled on fs/exec.c:aout_core_dump() + * Jeremy Fitzhardinge + * + * Modelled on fs/binfmt_elf.c core dumper + */ +#ifdef CONFIG_ELF_CORE + +struct elf_prstatus_fdpic +{ + struct elf_prstatus_common common; + elf_gregset_t pr_reg; /* GP registers */ + /* When using FDPIC, the loadmap addresses need to be communicated + * to GDB in order for GDB to do the necessary relocations. The + * fields (below) used to communicate this information are placed + * immediately after ``pr_reg'', so that the loadmap addresses may + * be viewed as part of the register set if so desired. + */ + unsigned long pr_exec_fdpic_loadmap; + unsigned long pr_interp_fdpic_loadmap; + int pr_fpvalid; /* True if math co-processor being used. */ +}; + +/* An ELF note in memory */ +struct memelfnote +{ + const char *name; + int type; + unsigned int datasz; + void *data; +}; + +static int notesize(struct memelfnote *en) +{ + int sz; + + sz = sizeof(struct elf_note); + sz += roundup(strlen(en->name) + 1, 4); + sz += roundup(en->datasz, 4); + + return sz; +} + +/* #define DEBUG */ + +static int writenote(struct memelfnote *men, struct coredump_params *cprm) +{ + struct elf_note en; + en.n_namesz = strlen(men->name) + 1; + en.n_descsz = men->datasz; + en.n_type = men->type; + + return dump_emit(cprm, &en, sizeof(en)) && + dump_emit(cprm, men->name, en.n_namesz) && dump_align(cprm, 4) && + dump_emit(cprm, men->data, men->datasz) && dump_align(cprm, 4); +} + +static inline void fill_elf_fdpic_header(struct elfhdr *elf, int segs) +{ + memcpy(elf->e_ident, ELFMAG, SELFMAG); + elf->e_ident[EI_CLASS] = ELF_CLASS; + elf->e_ident[EI_DATA] = ELF_DATA; + elf->e_ident[EI_VERSION] = EV_CURRENT; + elf->e_ident[EI_OSABI] = ELF_OSABI; + memset(elf->e_ident+EI_PAD, 0, EI_NIDENT-EI_PAD); + + elf->e_type = ET_CORE; + elf->e_machine = ELF_ARCH; + elf->e_version = EV_CURRENT; + elf->e_entry = 0; + elf->e_phoff = sizeof(struct elfhdr); + elf->e_shoff = 0; + elf->e_flags = ELF_FDPIC_CORE_EFLAGS; + elf->e_ehsize = sizeof(struct elfhdr); + elf->e_phentsize = sizeof(struct elf_phdr); + elf->e_phnum = segs; + elf->e_shentsize = 0; + elf->e_shnum = 0; + elf->e_shstrndx = 0; + return; +} + +static inline void fill_elf_note_phdr(struct elf_phdr *phdr, int sz, loff_t offset) +{ + phdr->p_type = PT_NOTE; + phdr->p_offset = offset; + phdr->p_vaddr = 0; + phdr->p_paddr = 0; + phdr->p_filesz = sz; + phdr->p_memsz = 0; + phdr->p_flags = 0; + phdr->p_align = 4; + return; +} + +static inline void __fill_note(struct memelfnote *note, const char *name, int type, + unsigned int sz, void *data) +{ + note->name = name; + note->type = type; + note->datasz = sz; + note->data = data; + return; +} + +#define fill_note(note, type, sz, data) \ + __fill_note(note, NN_ ## type, NT_ ## type, sz, data) + +/* + * fill up all the fields in prstatus from the given task struct, except + * registers which need to be filled up separately. + */ +static void fill_prstatus(struct elf_prstatus_common *prstatus, + struct task_struct *p, long signr) +{ + prstatus->pr_info.si_signo = prstatus->pr_cursig = signr; + prstatus->pr_sigpend = p->pending.signal.sig[0]; + prstatus->pr_sighold = p->blocked.sig[0]; + rcu_read_lock(); + prstatus->pr_ppid = task_pid_vnr(rcu_dereference(p->real_parent)); + rcu_read_unlock(); + prstatus->pr_pid = task_pid_vnr(p); + prstatus->pr_pgrp = task_pgrp_vnr(p); + prstatus->pr_sid = task_session_vnr(p); + if (thread_group_leader(p)) { + struct task_cputime cputime; + + /* + * This is the record for the group leader. It shows the + * group-wide total, not its individual thread total. + */ + thread_group_cputime(p, &cputime); + prstatus->pr_utime = ns_to_kernel_old_timeval(cputime.utime); + prstatus->pr_stime = ns_to_kernel_old_timeval(cputime.stime); + } else { + u64 utime, stime; + + task_cputime(p, &utime, &stime); + prstatus->pr_utime = ns_to_kernel_old_timeval(utime); + prstatus->pr_stime = ns_to_kernel_old_timeval(stime); + } + prstatus->pr_cutime = ns_to_kernel_old_timeval(p->signal->cutime); + prstatus->pr_cstime = ns_to_kernel_old_timeval(p->signal->cstime); +} + +static int fill_psinfo(struct elf_prpsinfo *psinfo, struct task_struct *p, + struct mm_struct *mm) +{ + const struct cred *cred; + unsigned int i, len; + unsigned int state; + + /* first copy the parameters from user space */ + memset(psinfo, 0, sizeof(struct elf_prpsinfo)); + + len = mm->arg_end - mm->arg_start; + if (len >= ELF_PRARGSZ) + len = ELF_PRARGSZ - 1; + if (copy_from_user(&psinfo->pr_psargs, + (const char __user *) mm->arg_start, len)) + return -EFAULT; + for (i = 0; i < len; i++) + if (psinfo->pr_psargs[i] == 0) + psinfo->pr_psargs[i] = ' '; + psinfo->pr_psargs[len] = 0; + + rcu_read_lock(); + psinfo->pr_ppid = task_pid_vnr(rcu_dereference(p->real_parent)); + rcu_read_unlock(); + psinfo->pr_pid = task_pid_vnr(p); + psinfo->pr_pgrp = task_pgrp_vnr(p); + psinfo->pr_sid = task_session_vnr(p); + + state = READ_ONCE(p->__state); + i = state ? ffz(~state) + 1 : 0; + psinfo->pr_state = i; + psinfo->pr_sname = (i > 5) ? '.' : "RSDTZW"[i]; + psinfo->pr_zomb = psinfo->pr_sname == 'Z'; + psinfo->pr_nice = task_nice(p); + psinfo->pr_flag = p->flags; + rcu_read_lock(); + cred = __task_cred(p); + SET_UID(psinfo->pr_uid, from_kuid_munged(cred->user_ns, cred->uid)); + SET_GID(psinfo->pr_gid, from_kgid_munged(cred->user_ns, cred->gid)); + rcu_read_unlock(); + get_task_comm(psinfo->pr_fname, p); + + return 0; +} + +/* Here is the structure in which status of each thread is captured. */ +struct elf_thread_status +{ + struct elf_thread_status *next; + struct elf_prstatus_fdpic prstatus; /* NT_PRSTATUS */ + elf_fpregset_t fpu; /* NT_PRFPREG */ + struct memelfnote notes[2]; + int num_notes; +}; + +/* + * In order to add the specific thread information for the elf file format, + * we need to keep a linked list of every thread's pr_status and then create + * a single section for them in the final core file. + */ +static struct elf_thread_status *elf_dump_thread_status(long signr, struct task_struct *p, int *sz) +{ + const struct user_regset_view *view = task_user_regset_view(p); + struct elf_thread_status *t; + int i, ret; + + t = kzalloc_obj(struct elf_thread_status); + if (!t) + return t; + + fill_prstatus(&t->prstatus.common, p, signr); + t->prstatus.pr_exec_fdpic_loadmap = p->mm->context.exec_fdpic_loadmap; + t->prstatus.pr_interp_fdpic_loadmap = p->mm->context.interp_fdpic_loadmap; + regset_get(p, &view->regsets[0], + sizeof(t->prstatus.pr_reg), &t->prstatus.pr_reg); + + fill_note(&t->notes[0], PRSTATUS, sizeof(t->prstatus), &t->prstatus); + t->num_notes++; + *sz += notesize(&t->notes[0]); + + for (i = 1; i < view->n; ++i) { + const struct user_regset *regset = &view->regsets[i]; + if (regset->core_note_type != NT_PRFPREG) + continue; + if (regset->active && regset->active(p, regset) <= 0) + continue; + ret = regset_get(p, regset, sizeof(t->fpu), &t->fpu); + if (ret >= 0) + t->prstatus.pr_fpvalid = 1; + break; + } + + if (t->prstatus.pr_fpvalid) { + fill_note(&t->notes[1], PRFPREG, sizeof(t->fpu), &t->fpu); + t->num_notes++; + *sz += notesize(&t->notes[1]); + } + return t; +} + +static void fill_extnum_info(struct elfhdr *elf, struct elf_shdr *shdr4extnum, + elf_addr_t e_shoff, int segs) +{ + elf->e_shoff = e_shoff; + elf->e_shentsize = sizeof(*shdr4extnum); + elf->e_shnum = 1; + elf->e_shstrndx = SHN_UNDEF; + + memset(shdr4extnum, 0, sizeof(*shdr4extnum)); + + shdr4extnum->sh_type = SHT_NULL; + shdr4extnum->sh_size = elf->e_shnum; + shdr4extnum->sh_link = elf->e_shstrndx; + shdr4extnum->sh_info = segs; +} + +/* + * dump the segments for an MMU process + */ +static bool elf_fdpic_dump_segments(struct coredump_params *cprm, + struct core_vma_metadata *vma_meta, + int vma_count) +{ + int i; + + for (i = 0; i < vma_count; i++) { + struct core_vma_metadata *meta = vma_meta + i; + + if (!dump_user_range(cprm, meta->start, meta->dump_size)) + return false; + } + return true; +} + +/* + * Actual dumper + * + * This is a two-pass process; first we find the offsets of the bits, + * and then they are actually written out. If we run out of core limit + * we just truncate. + */ +static int elf_fdpic_core_dump(struct coredump_params *cprm) +{ + int has_dumped = 0; + int segs; + int i; + struct elfhdr *elf = NULL; + loff_t offset = 0, dataoff; + struct memelfnote psinfo_note, auxv_note; + struct elf_prpsinfo *psinfo = NULL; /* NT_PRPSINFO */ + struct elf_thread_status *thread_list = NULL; + int thread_status_size = 0; + elf_addr_t *auxv; + struct elf_phdr *phdr4note = NULL; + struct elf_shdr *shdr4extnum = NULL; + Elf_Half e_phnum; + elf_addr_t e_shoff; + struct core_thread *ct; + struct elf_thread_status *tmp; + + /* alloc memory for large data structures: too large to be on stack */ + elf = kmalloc_obj(*elf); + if (!elf) + goto end_coredump; + psinfo = kmalloc_obj(*psinfo); + if (!psinfo) + goto end_coredump; + + for (ct = current->signal->core_state->dumper.next; + ct; ct = ct->next) { + tmp = elf_dump_thread_status(cprm->siginfo->si_signo, + ct->task, &thread_status_size); + if (!tmp) + goto end_coredump; + + tmp->next = thread_list; + thread_list = tmp; + } + + /* now collect the dump for the current */ + tmp = elf_dump_thread_status(cprm->siginfo->si_signo, + current, &thread_status_size); + if (!tmp) + goto end_coredump; + tmp->next = thread_list; + thread_list = tmp; + + segs = cprm->vma_count + elf_core_extra_phdrs(cprm); + + /* for notes section */ + segs++; + + /* If segs > PN_XNUM(0xffff), then e_phnum overflows. To avoid + * this, kernel supports extended numbering. Have a look at + * include/linux/elf.h for further information. */ + e_phnum = segs > PN_XNUM ? PN_XNUM : segs; + + /* Set up header */ + fill_elf_fdpic_header(elf, e_phnum); + + has_dumped = 1; + /* + * Set up the notes in similar form to SVR4 core dumps made + * with info from their /proc. + */ + + fill_psinfo(psinfo, current->group_leader, current->mm); + fill_note(&psinfo_note, PRPSINFO, sizeof(*psinfo), psinfo); + thread_status_size += notesize(&psinfo_note); + + auxv = (elf_addr_t *) current->mm->saved_auxv; + i = 0; + do + i += 2; + while (auxv[i - 2] != AT_NULL); + fill_note(&auxv_note, AUXV, i * sizeof(elf_addr_t), auxv); + thread_status_size += notesize(&auxv_note); + + offset = sizeof(*elf); /* ELF header */ + offset += segs * sizeof(struct elf_phdr); /* Program headers */ + + /* Write notes phdr entry */ + phdr4note = kmalloc_obj(*phdr4note); + if (!phdr4note) + goto end_coredump; + + fill_elf_note_phdr(phdr4note, thread_status_size, offset); + offset += thread_status_size; + + /* Page-align dumped data */ + dataoff = offset = roundup(offset, ELF_EXEC_PAGESIZE); + + offset += cprm->vma_data_size; + offset += elf_core_extra_data_size(cprm); + e_shoff = offset; + + if (e_phnum == PN_XNUM) { + shdr4extnum = kmalloc_obj(*shdr4extnum); + if (!shdr4extnum) + goto end_coredump; + fill_extnum_info(elf, shdr4extnum, e_shoff, segs); + } + + offset = dataoff; + + if (!dump_emit(cprm, elf, sizeof(*elf))) + goto end_coredump; + + if (!dump_emit(cprm, phdr4note, sizeof(*phdr4note))) + goto end_coredump; + + /* write program headers for segments dump */ + for (i = 0; i < cprm->vma_count; i++) { + struct core_vma_metadata *meta = cprm->vma_meta + i; + struct elf_phdr phdr; + size_t sz; + + sz = meta->end - meta->start; + + phdr.p_type = PT_LOAD; + phdr.p_offset = offset; + phdr.p_vaddr = meta->start; + phdr.p_paddr = 0; + phdr.p_filesz = meta->dump_size; + phdr.p_memsz = sz; + offset += phdr.p_filesz; + phdr.p_flags = 0; + if (meta->flags & VM_READ) + phdr.p_flags |= PF_R; + if (meta->flags & VM_WRITE) + phdr.p_flags |= PF_W; + if (meta->flags & VM_EXEC) + phdr.p_flags |= PF_X; + phdr.p_align = ELF_EXEC_PAGESIZE; + + if (!dump_emit(cprm, &phdr, sizeof(phdr))) + goto end_coredump; + } + + if (!elf_core_write_extra_phdrs(cprm, offset)) + goto end_coredump; + + /* write out the notes section */ + if (!writenote(thread_list->notes, cprm)) + goto end_coredump; + if (!writenote(&psinfo_note, cprm)) + goto end_coredump; + if (!writenote(&auxv_note, cprm)) + goto end_coredump; + for (i = 1; i < thread_list->num_notes; i++) + if (!writenote(thread_list->notes + i, cprm)) + goto end_coredump; + + /* write out the thread status notes section */ + for (tmp = thread_list->next; tmp; tmp = tmp->next) { + for (i = 0; i < tmp->num_notes; i++) + if (!writenote(&tmp->notes[i], cprm)) + goto end_coredump; + } + + dump_skip_to(cprm, dataoff); + + if (!elf_fdpic_dump_segments(cprm, cprm->vma_meta, cprm->vma_count)) + goto end_coredump; + + if (!elf_core_write_extra_data(cprm)) + goto end_coredump; + + if (e_phnum == PN_XNUM) { + if (!dump_emit(cprm, shdr4extnum, sizeof(*shdr4extnum))) + goto end_coredump; + } + + if (cprm->file->f_pos != offset) { + /* Sanity check */ + printk(KERN_WARNING + "elf_core_dump: file->f_pos (%lld) != offset (%lld)\n", + cprm->file->f_pos, offset); + } + +end_coredump: + while (thread_list) { + tmp = thread_list; + thread_list = thread_list->next; + kfree(tmp); + } + kfree(phdr4note); + kfree(elf); + kfree(psinfo); + kfree(shdr4extnum); + return has_dumped; +} + +#endif /* CONFIG_ELF_CORE */ diff --git a/fs/binfmt_flat.c b/fs/binfmt_flat.c new file mode 100644 index 000000000..b5b5ca1a4 --- /dev/null +++ b/fs/binfmt_flat.c @@ -0,0 +1,946 @@ +// SPDX-License-Identifier: GPL-2.0 +/****************************************************************************/ +/* + * linux/fs/binfmt_flat.c + * + * Copyright (C) 2000-2003 David McCullough + * Copyright (C) 2002 Greg Ungerer + * Copyright (C) 2002 SnapGear, by Paul Dale + * Copyright (C) 2000, 2001 Lineo, by David McCullough + * based heavily on: + * + * linux/fs/binfmt_aout.c: + * Copyright (C) 1991, 1992, 1996 Linus Torvalds + * linux/fs/binfmt_flat.c for 2.0 kernel + * Copyright (C) 1998 Kenneth Albanowski + * JAN/99 -- coded full program relocation (gerg@snapgear.com) + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#ifndef flat_get_relocate_addr +#define flat_get_relocate_addr(rel) (rel) +#endif + +/****************************************************************************/ + +/* + * User data (data section and bss) needs to be aligned. + * We pick 0x20 here because it is the max value elf2flt has always + * used in producing FLAT files, and because it seems to be large + * enough to make all the gcc alignment related tests happy. + */ +#define FLAT_DATA_ALIGN (0x20) + +/* + * User data (stack) also needs to be aligned. + * Here we can be a bit looser than the data sections since this + * needs to only meet arch ABI requirements. + */ +#define FLAT_STACK_ALIGN max_t(unsigned long, sizeof(void *), ARCH_SLAB_MINALIGN) + +#define RELOC_FAILED 0xff00ff01 /* Relocation incorrect somewhere */ +#define UNLOADED_LIB 0x7ff000ff /* Placeholder for unused library */ + +#define MAX_SHARED_LIBS (1) + +#ifdef CONFIG_BINFMT_FLAT_NO_DATA_START_OFFSET +#define DATA_START_OFFSET_WORDS (0) +#define MAX_SHARED_LIBS_UPDATE (0) +#else +#define DATA_START_OFFSET_WORDS (MAX_SHARED_LIBS) +#define MAX_SHARED_LIBS_UPDATE (MAX_SHARED_LIBS) +#endif + +struct lib_info { + struct { + unsigned long start_code; /* Start of text segment */ + unsigned long start_data; /* Start of data segment */ + unsigned long start_brk; /* End of data segment */ + unsigned long text_len; /* Length of text segment */ + unsigned long entry; /* Start address for this module */ + unsigned long build_date; /* When this one was compiled */ + bool loaded; /* Has this library been loaded? */ + } lib_list[MAX_SHARED_LIBS]; +}; + +static int load_flat_binary(struct linux_binprm *); + +static struct linux_binfmt flat_format = { + .module = THIS_MODULE, + .load_binary = load_flat_binary, +}; + + +/****************************************************************************/ +/* + * create_flat_tables() parses the env- and arg-strings in new user + * memory and creates the pointer tables from them, and puts their + * addresses on the "stack", recording the new stack pointer value. + */ + +static int create_flat_tables(struct linux_binprm *bprm, unsigned long arg_start) +{ + char __user *p; + unsigned long __user *sp; + long i, len; + + p = (char __user *)arg_start; + sp = (unsigned long __user *)current->mm->start_stack; + + sp -= bprm->envc + 1; + sp -= bprm->argc + 1; + if (IS_ENABLED(CONFIG_BINFMT_FLAT_ARGVP_ENVP_ON_STACK)) + sp -= 2; /* argvp + envp */ + sp -= 1; /* &argc */ + + current->mm->start_stack = (unsigned long)sp & -FLAT_STACK_ALIGN; + sp = (unsigned long __user *)current->mm->start_stack; + + if (put_user(bprm->argc, sp++)) + return -EFAULT; + if (IS_ENABLED(CONFIG_BINFMT_FLAT_ARGVP_ENVP_ON_STACK)) { + unsigned long argv, envp; + argv = (unsigned long)(sp + 2); + envp = (unsigned long)(sp + 2 + bprm->argc + 1); + if (put_user(argv, sp++) || put_user(envp, sp++)) + return -EFAULT; + } + + current->mm->arg_start = (unsigned long)p; + for (i = bprm->argc; i > 0; i--) { + if (put_user((unsigned long)p, sp++)) + return -EFAULT; + len = strnlen_user(p, MAX_ARG_STRLEN); + if (!len || len > MAX_ARG_STRLEN) + return -EINVAL; + p += len; + } + if (put_user(0, sp++)) + return -EFAULT; + current->mm->arg_end = (unsigned long)p; + + current->mm->env_start = (unsigned long) p; + for (i = bprm->envc; i > 0; i--) { + if (put_user((unsigned long)p, sp++)) + return -EFAULT; + len = strnlen_user(p, MAX_ARG_STRLEN); + if (!len || len > MAX_ARG_STRLEN) + return -EINVAL; + p += len; + } + if (put_user(0, sp++)) + return -EFAULT; + current->mm->env_end = (unsigned long)p; + + return 0; +} + +/****************************************************************************/ + +#ifdef CONFIG_BINFMT_ZFLAT + +#include + +#define LBUFSIZE 4000 + +/* gzip flag byte */ +#define ASCII_FLAG 0x01 /* bit 0 set: file probably ASCII text */ +#define CONTINUATION 0x02 /* bit 1 set: continuation of multi-part gzip file */ +#define EXTRA_FIELD 0x04 /* bit 2 set: extra field present */ +#define ORIG_NAME 0x08 /* bit 3 set: original file name present */ +#define COMMENT 0x10 /* bit 4 set: file comment present */ +#define ENCRYPTED 0x20 /* bit 5 set: file is encrypted */ +#define RESERVED 0xC0 /* bit 6,7: reserved */ + +static int decompress_exec(struct linux_binprm *bprm, loff_t fpos, char *dst, + long len, int fd) +{ + unsigned char *buf; + z_stream strm; + int ret, retval; + + pr_debug("decompress_exec(offset=%llx,buf=%p,len=%lx)\n", fpos, dst, len); + + memset(&strm, 0, sizeof(strm)); + strm.workspace = kmalloc(zlib_inflate_workspacesize(), GFP_KERNEL); + if (!strm.workspace) + return -ENOMEM; + + buf = kmalloc(LBUFSIZE, GFP_KERNEL); + if (!buf) { + retval = -ENOMEM; + goto out_free; + } + + /* Read in first chunk of data and parse gzip header. */ + ret = kernel_read(bprm->file, buf, LBUFSIZE, &fpos); + + strm.next_in = buf; + strm.avail_in = ret; + strm.total_in = 0; + + retval = -ENOEXEC; + + /* Check minimum size -- gzip header */ + if (ret < 10) { + pr_debug("file too small?\n"); + goto out_free_buf; + } + + /* Check gzip magic number */ + if ((buf[0] != 037) || ((buf[1] != 0213) && (buf[1] != 0236))) { + pr_debug("unknown compression magic?\n"); + goto out_free_buf; + } + + /* Check gzip method */ + if (buf[2] != 8) { + pr_debug("unknown compression method?\n"); + goto out_free_buf; + } + /* Check gzip flags */ + if ((buf[3] & ENCRYPTED) || (buf[3] & CONTINUATION) || + (buf[3] & RESERVED)) { + pr_debug("unknown flags?\n"); + goto out_free_buf; + } + + ret = 10; + if (buf[3] & EXTRA_FIELD) { + ret += 2 + buf[10] + (buf[11] << 8); + if (unlikely(ret >= LBUFSIZE)) { + pr_debug("buffer overflow (EXTRA)?\n"); + goto out_free_buf; + } + } + if (buf[3] & ORIG_NAME) { + while (ret < LBUFSIZE && buf[ret++] != 0) + ; + if (unlikely(ret == LBUFSIZE)) { + pr_debug("buffer overflow (ORIG_NAME)?\n"); + goto out_free_buf; + } + } + if (buf[3] & COMMENT) { + while (ret < LBUFSIZE && buf[ret++] != 0) + ; + if (unlikely(ret == LBUFSIZE)) { + pr_debug("buffer overflow (COMMENT)?\n"); + goto out_free_buf; + } + } + + strm.next_in += ret; + strm.avail_in -= ret; + + strm.next_out = dst; + strm.avail_out = len; + strm.total_out = 0; + + if (zlib_inflateInit2(&strm, -MAX_WBITS) != Z_OK) { + pr_debug("zlib init failed?\n"); + goto out_free_buf; + } + + while ((ret = zlib_inflate(&strm, Z_NO_FLUSH)) == Z_OK) { + ret = kernel_read(bprm->file, buf, LBUFSIZE, &fpos); + if (ret <= 0) + break; + len -= ret; + + strm.next_in = buf; + strm.avail_in = ret; + strm.total_in = 0; + } + + if (ret < 0) { + pr_debug("decompression failed (%d), %s\n", + ret, strm.msg); + goto out_zlib; + } + + retval = 0; +out_zlib: + zlib_inflateEnd(&strm); +out_free_buf: + kfree(buf); +out_free: + kfree(strm.workspace); + return retval; +} + +#endif /* CONFIG_BINFMT_ZFLAT */ + +/****************************************************************************/ + +static unsigned long +calc_reloc(unsigned long r, struct lib_info *p) +{ + unsigned long addr; + unsigned long start_brk; + unsigned long start_data; + unsigned long text_len; + unsigned long start_code; + + start_brk = p->lib_list[0].start_brk; + start_data = p->lib_list[0].start_data; + start_code = p->lib_list[0].start_code; + text_len = p->lib_list[0].text_len; + + if (r > start_brk - start_data + text_len) { + pr_err("reloc outside program 0x%lx (0 - 0x%lx/0x%lx)", + r, start_brk-start_data+text_len, text_len); + goto failed; + } + + if (r < text_len) /* In text segment */ + addr = r + start_code; + else /* In data segment */ + addr = r - text_len + start_data; + + /* Range checked already above so doing the range tests is redundant...*/ + return addr; + +failed: + pr_cont(", killing %s!\n", current->comm); + send_sig(SIGSEGV, current, 0); + + return RELOC_FAILED; +} + +/****************************************************************************/ + +#ifdef CONFIG_BINFMT_FLAT_OLD +static void old_reloc(unsigned long rl) +{ + static const char *segment[] = { "TEXT", "DATA", "BSS", "*UNKNOWN*" }; + flat_v2_reloc_t r; + unsigned long __user *ptr; + unsigned long val; + + r.value = rl; +#if defined(CONFIG_COLDFIRE) + ptr = (unsigned long __user *)(current->mm->start_code + r.reloc.offset); +#else + ptr = (unsigned long __user *)(current->mm->start_data + r.reloc.offset); +#endif + get_user(val, ptr); + + pr_debug("Relocation of variable at DATASEG+%x " + "(address %p, currently %lx) into segment %s\n", + r.reloc.offset, ptr, val, segment[r.reloc.type]); + + switch (r.reloc.type) { + case OLD_FLAT_RELOC_TYPE_TEXT: + val += current->mm->start_code; + break; + case OLD_FLAT_RELOC_TYPE_DATA: + val += current->mm->start_data; + break; + case OLD_FLAT_RELOC_TYPE_BSS: + val += current->mm->end_data; + break; + default: + pr_err("Unknown relocation type=%x\n", r.reloc.type); + break; + } + put_user(val, ptr); + + pr_debug("Relocation became %lx\n", val); +} +#endif /* CONFIG_BINFMT_FLAT_OLD */ + +/****************************************************************************/ + +static inline u32 __user *skip_got_header(u32 __user *rp) +{ + if (IS_ENABLED(CONFIG_RISCV)) { + /* + * RISC-V has a 16 byte GOT PLT header for elf64-riscv + * and 8 byte GOT PLT header for elf32-riscv. + * Skip the whole GOT PLT header, since it is reserved + * for the dynamic linker (ld.so). + */ + u32 rp_val0, rp_val1; + + if (get_user(rp_val0, rp)) + return rp; + if (get_user(rp_val1, rp + 1)) + return rp; + + if (rp_val0 == 0xffffffff && rp_val1 == 0xffffffff) + rp += 4; + else if (rp_val0 == 0xffffffff) + rp += 2; + } + return rp; +} + +static int load_flat_file(struct linux_binprm *bprm, + struct lib_info *libinfo, unsigned long *extra_stack) +{ + struct flat_hdr *hdr; + unsigned long textpos, datapos, realdatastart; + u32 text_len, data_len, bss_len, stack_len, full_data, flags; + unsigned long len, memp, memp_size, extra, rlim; + __be32 __user *reloc; + u32 __user *rp; + int i, rev, relocs; + loff_t fpos; + unsigned long start_code, end_code; + ssize_t result; + int ret; + + hdr = ((struct flat_hdr *) bprm->buf); /* exec-header */ + + text_len = ntohl(hdr->data_start); + data_len = ntohl(hdr->data_end) - ntohl(hdr->data_start); + bss_len = ntohl(hdr->bss_end) - ntohl(hdr->data_end); + stack_len = ntohl(hdr->stack_size); + if (extra_stack) { + stack_len += *extra_stack; + *extra_stack = stack_len; + } + relocs = ntohl(hdr->reloc_count); + flags = ntohl(hdr->flags); + rev = ntohl(hdr->rev); + full_data = data_len + relocs * sizeof(unsigned long); + + if (strncmp(hdr->magic, "bFLT", 4)) { + /* + * Previously, here was a printk to tell people + * "BINFMT_FLAT: bad header magic". + * But for the kernel which also use ELF FD-PIC format, this + * error message is confusing. + * because a lot of people do not manage to produce good + */ + ret = -ENOEXEC; + goto err; + } + + if (flags & FLAT_FLAG_KTRACE) + pr_info("Loading file: %s\n", bprm->filename); + +#ifdef CONFIG_BINFMT_FLAT_OLD + if (rev != FLAT_VERSION && rev != OLD_FLAT_VERSION) { + pr_err("bad flat file version 0x%x (supported 0x%lx and 0x%lx)\n", + rev, FLAT_VERSION, OLD_FLAT_VERSION); + ret = -ENOEXEC; + goto err; + } + + /* + * fix up the flags for the older format, there were all kinds + * of endian hacks, this only works for the simple cases + */ + if (rev == OLD_FLAT_VERSION && + (flags || IS_ENABLED(CONFIG_BINFMT_FLAT_OLD_ALWAYS_RAM))) + flags = FLAT_FLAG_RAM; + +#else /* CONFIG_BINFMT_FLAT_OLD */ + if (rev != FLAT_VERSION) { + pr_err("bad flat file version 0x%x (supported 0x%lx)\n", + rev, FLAT_VERSION); + ret = -ENOEXEC; + goto err; + } +#endif /* !CONFIG_BINFMT_FLAT_OLD */ + + /* + * Make sure the header params are sane. + * 28 bits (256 MB) is way more than reasonable in this case. + * If some top bits are set we have probable binary corruption. + */ + if ((text_len | data_len | bss_len | stack_len | relocs | full_data) >> 28) { + pr_err("bad header\n"); + ret = -ENOEXEC; + goto err; + } + +#ifndef CONFIG_BINFMT_ZFLAT + if (flags & (FLAT_FLAG_GZIP|FLAT_FLAG_GZDATA)) { + pr_err("Support for ZFLAT executables is not enabled.\n"); + ret = -ENOEXEC; + goto err; + } +#endif + + /* + * Check initial limits. This avoids letting people circumvent + * size limits imposed on them by creating programs with large + * arrays in the data or bss. + */ + rlim = rlimit(RLIMIT_DATA); + if (rlim >= RLIM_INFINITY) + rlim = ~0; + if (data_len + bss_len > rlim) { + ret = -ENOMEM; + goto err; + } + + /* Flush all traces of the currently running executable */ + ret = begin_new_exec(bprm); + if (ret) + goto err; + + /* OK, This is the point of no return */ + set_personality(PER_LINUX_32BIT); + setup_new_exec(bprm); + + /* + * calculate the extra space we need to map in + */ + extra = max_t(unsigned long, bss_len + stack_len, + relocs * sizeof(unsigned long)); + + /* + * there are a couple of cases here, the separate code/data + * case, and then the fully copied to RAM case which lumps + * it all together. + */ + if (!IS_ENABLED(CONFIG_MMU) && !(flags & (FLAT_FLAG_RAM|FLAT_FLAG_GZIP))) { + /* + * this should give us a ROM ptr, but if it doesn't we don't + * really care + */ + pr_debug("ROM mapping of file (we hope)\n"); + + textpos = vm_mmap(bprm->file, 0, text_len, PROT_READ|PROT_EXEC, + MAP_PRIVATE, 0); + if (!textpos || IS_ERR_VALUE(textpos)) { + ret = textpos; + if (!textpos) + ret = -ENOMEM; + pr_err("Unable to mmap process text, errno %d\n", ret); + goto err; + } + + len = data_len + extra + + DATA_START_OFFSET_WORDS * sizeof(unsigned long); + len = PAGE_ALIGN(len); + realdatastart = vm_mmap(NULL, 0, len, + PROT_READ|PROT_WRITE|PROT_EXEC, MAP_PRIVATE, 0); + + if (realdatastart == 0 || IS_ERR_VALUE(realdatastart)) { + ret = realdatastart; + if (!realdatastart) + ret = -ENOMEM; + pr_err("Unable to allocate RAM for process data, " + "errno %d\n", ret); + vm_munmap(textpos, text_len); + goto err; + } + datapos = ALIGN(realdatastart + + DATA_START_OFFSET_WORDS * sizeof(unsigned long), + FLAT_DATA_ALIGN); + + pr_debug("Allocated data+bss+stack (%u bytes): %lx\n", + data_len + bss_len + stack_len, datapos); + + fpos = ntohl(hdr->data_start); +#ifdef CONFIG_BINFMT_ZFLAT + if (flags & FLAT_FLAG_GZDATA) { + result = decompress_exec(bprm, fpos, (char *)datapos, + full_data, 0); + } else +#endif + { + result = read_code(bprm->file, datapos, fpos, + full_data); + } + if (IS_ERR_VALUE(result)) { + ret = result; + pr_err("Unable to read data+bss, errno %d\n", ret); + vm_munmap(textpos, text_len); + vm_munmap(realdatastart, len); + goto err; + } + + reloc = (__be32 __user *) + (datapos + (ntohl(hdr->reloc_start) - text_len)); + memp = realdatastart; + memp_size = len; + } else { + + len = text_len + data_len + extra + + DATA_START_OFFSET_WORDS * sizeof(u32); + len = PAGE_ALIGN(len); + textpos = vm_mmap(NULL, 0, len, + PROT_READ | PROT_EXEC | PROT_WRITE, MAP_PRIVATE, 0); + + if (!textpos || IS_ERR_VALUE(textpos)) { + ret = textpos; + if (!textpos) + ret = -ENOMEM; + pr_err("Unable to allocate RAM for process text/data, " + "errno %d\n", ret); + goto err; + } + + realdatastart = textpos + ntohl(hdr->data_start); + datapos = ALIGN(realdatastart + + DATA_START_OFFSET_WORDS * sizeof(u32), + FLAT_DATA_ALIGN); + + reloc = (__be32 __user *) + (datapos + (ntohl(hdr->reloc_start) - text_len)); + memp = textpos; + memp_size = len; +#ifdef CONFIG_BINFMT_ZFLAT + /* + * load it all in and treat it like a RAM load from now on + */ + if (flags & FLAT_FLAG_GZIP) { +#ifndef CONFIG_MMU + result = decompress_exec(bprm, sizeof(struct flat_hdr), + (((char *)textpos) + sizeof(struct flat_hdr)), + (text_len + full_data + - sizeof(struct flat_hdr)), + 0); + memmove((void *) datapos, (void *) realdatastart, + full_data); +#else + /* + * This is used on MMU systems mainly for testing. + * Let's use a kernel buffer to simplify things. + */ + long unz_text_len = text_len - sizeof(struct flat_hdr); + long unz_len = unz_text_len + full_data; + char *unz_data = vmalloc(unz_len); + if (!unz_data) { + result = -ENOMEM; + } else { + result = decompress_exec(bprm, sizeof(struct flat_hdr), + unz_data, unz_len, 0); + if (result == 0 && + (copy_to_user((void __user *)textpos + sizeof(struct flat_hdr), + unz_data, unz_text_len) || + copy_to_user((void __user *)datapos, + unz_data + unz_text_len, full_data))) + result = -EFAULT; + vfree(unz_data); + } +#endif + } else if (flags & FLAT_FLAG_GZDATA) { + result = read_code(bprm->file, textpos, 0, text_len); + if (!IS_ERR_VALUE(result)) { +#ifndef CONFIG_MMU + result = decompress_exec(bprm, text_len, (char *) datapos, + full_data, 0); +#else + char *unz_data = vmalloc(full_data); + if (!unz_data) { + result = -ENOMEM; + } else { + result = decompress_exec(bprm, text_len, + unz_data, full_data, 0); + if (result == 0 && + copy_to_user((void __user *)datapos, + unz_data, full_data)) + result = -EFAULT; + vfree(unz_data); + } +#endif + } + } else +#endif /* CONFIG_BINFMT_ZFLAT */ + { + result = read_code(bprm->file, textpos, 0, text_len); + if (!IS_ERR_VALUE(result)) + result = read_code(bprm->file, datapos, + ntohl(hdr->data_start), + full_data); + } + if (IS_ERR_VALUE(result)) { + ret = result; + pr_err("Unable to read code+data+bss, errno %d\n", ret); + vm_munmap(textpos, text_len + data_len + extra + + DATA_START_OFFSET_WORDS * sizeof(u32)); + goto err; + } + } + + start_code = textpos + sizeof(struct flat_hdr); + end_code = textpos + text_len; + text_len -= sizeof(struct flat_hdr); /* the real code len */ + + /* The main program needs a little extra setup in the task structure */ + current->mm->start_code = start_code; + current->mm->end_code = end_code; + current->mm->start_data = datapos; + current->mm->end_data = datapos + data_len; + /* + * set up the brk stuff, uses any slack left in data/bss/stack + * allocation. We put the brk after the bss (between the bss + * and stack) like other platforms. + * Userspace code relies on the stack pointer starting out at + * an address right at the end of a page. + */ + current->mm->start_brk = datapos + data_len + bss_len; + current->mm->brk = (current->mm->start_brk + 3) & ~3; +#ifndef CONFIG_MMU + current->mm->context.end_brk = memp + memp_size - stack_len; +#endif + + if (flags & FLAT_FLAG_KTRACE) { + pr_info("Mapping is %lx, Entry point is %x, data_start is %x\n", + textpos, 0x00ffffff&ntohl(hdr->entry), ntohl(hdr->data_start)); + pr_info("%s %s: TEXT=%lx-%lx DATA=%lx-%lx BSS=%lx-%lx\n", + "Load", bprm->filename, + start_code, end_code, datapos, datapos + data_len, + datapos + data_len, (datapos + data_len + bss_len + 3) & ~3); + } + + /* Store the current module values into the global library structure */ + libinfo->lib_list[0].start_code = start_code; + libinfo->lib_list[0].start_data = datapos; + libinfo->lib_list[0].start_brk = datapos + data_len + bss_len; + libinfo->lib_list[0].text_len = text_len; + libinfo->lib_list[0].loaded = 1; + libinfo->lib_list[0].entry = (0x00ffffff & ntohl(hdr->entry)) + textpos; + libinfo->lib_list[0].build_date = ntohl(hdr->build_date); + + /* + * We just load the allocations into some temporary memory to + * help simplify all this mumbo jumbo + * + * We've got two different sections of relocation entries. + * The first is the GOT which resides at the beginning of the data segment + * and is terminated with a -1. This one can be relocated in place. + * The second is the extra relocation entries tacked after the image's + * data segment. These require a little more processing as the entry is + * really an offset into the image which contains an offset into the + * image. + */ + if (flags & FLAT_FLAG_GOTPIC) { + rp = skip_got_header((u32 __user *) datapos); + for (; ; rp++) { + u32 addr, rp_val; + if (get_user(rp_val, rp)) + return -EFAULT; + if (rp_val == 0xffffffff) + break; + if (rp_val) { + addr = calc_reloc(rp_val, libinfo); + if (addr == RELOC_FAILED) { + ret = -ENOEXEC; + goto err; + } + if (put_user(addr, rp)) + return -EFAULT; + } + } + } + + /* + * Now run through the relocation entries. + * We've got to be careful here as C++ produces relocatable zero + * entries in the constructor and destructor tables which are then + * tested for being not zero (which will always occur unless we're + * based from address zero). This causes an endless loop as __start + * is at zero. The solution used is to not relocate zero addresses. + * This has the negative side effect of not allowing a global data + * reference to be statically initialised to _stext (I've moved + * __start to address 4 so that is okay). + */ + if (rev > OLD_FLAT_VERSION) { + for (i = 0; i < relocs; i++) { + u32 addr, relval; + __be32 tmp; + + /* + * Get the address of the pointer to be + * relocated (of course, the address has to be + * relocated first). + */ + if (get_user(tmp, reloc + i)) + return -EFAULT; + relval = ntohl(tmp); + addr = flat_get_relocate_addr(relval); + rp = (u32 __user *)calc_reloc(addr, libinfo); + if (rp == (u32 __user *)RELOC_FAILED) { + ret = -ENOEXEC; + goto err; + } + + /* Get the pointer's value. */ + ret = flat_get_addr_from_rp(rp, relval, flags, &addr); + if (unlikely(ret)) + goto err; + + if (addr != 0) { + /* + * Do the relocation. PIC relocs in the data section are + * already in target order + */ + if ((flags & FLAT_FLAG_GOTPIC) == 0) { + /* + * Meh, the same value can have a different + * byte order based on a flag.. + */ + addr = ntohl((__force __be32)addr); + } + addr = calc_reloc(addr, libinfo); + if (addr == RELOC_FAILED) { + ret = -ENOEXEC; + goto err; + } + + /* Write back the relocated pointer. */ + ret = flat_put_addr_at_rp(rp, addr, relval); + if (unlikely(ret)) + goto err; + } + } +#ifdef CONFIG_BINFMT_FLAT_OLD + } else { + for (i = 0; i < relocs; i++) { + __be32 relval; + if (get_user(relval, reloc + i)) + return -EFAULT; + old_reloc(ntohl(relval)); + } +#endif /* CONFIG_BINFMT_FLAT_OLD */ + } + + flush_icache_user_range(start_code, end_code); + + /* zero the BSS, BRK and stack areas */ + if (clear_user((void __user *)(datapos + data_len), bss_len + + (memp + memp_size - stack_len - /* end brk */ + libinfo->lib_list[0].start_brk) + /* start brk */ + stack_len)) + return -EFAULT; + + return 0; +err: + return ret; +} + + +/****************************************************************************/ + +/* + * These are the functions used to load flat style executables and shared + * libraries. There is no binary dependent code anywhere else. + */ + +static int load_flat_binary(struct linux_binprm *bprm) +{ + struct lib_info libinfo; + struct pt_regs *regs = current_pt_regs(); + unsigned long stack_len = 0; + unsigned long start_addr; + int res; + int i, j; + + memset(&libinfo, 0, sizeof(libinfo)); + + /* + * We have to add the size of our arguments to our stack size + * otherwise it's too easy for users to create stack overflows + * by passing in a huge argument list. And yes, we have to be + * pedantic and include space for the argv/envp array as it may have + * a lot of entries. + */ +#ifndef CONFIG_MMU + stack_len += PAGE_SIZE * MAX_ARG_PAGES - bprm->p; /* the strings */ +#endif + stack_len += (bprm->argc + 1) * sizeof(char *); /* the argv array */ + stack_len += (bprm->envc + 1) * sizeof(char *); /* the envp array */ + stack_len = ALIGN(stack_len, FLAT_STACK_ALIGN); + + res = load_flat_file(bprm, &libinfo, &stack_len); + if (res < 0) + return res; + + /* Update data segment pointers for all libraries */ + for (i = 0; i < MAX_SHARED_LIBS_UPDATE; i++) { + if (!libinfo.lib_list[i].loaded) + continue; + for (j = 0; j < MAX_SHARED_LIBS; j++) { + unsigned long val = libinfo.lib_list[j].loaded ? + libinfo.lib_list[j].start_data : UNLOADED_LIB; + unsigned long __user *p = (unsigned long __user *) + libinfo.lib_list[i].start_data; + p -= j + 1; + if (put_user(val, p)) + return -EFAULT; + } + } + + set_binfmt(&flat_format); + +#ifdef CONFIG_MMU + res = setup_arg_pages(bprm, STACK_TOP, EXSTACK_DEFAULT); + if (!res) + res = create_flat_tables(bprm, bprm->p); +#else + /* Stash our initial stack pointer into the mm structure */ + current->mm->start_stack = + ((current->mm->context.end_brk + stack_len + 3) & ~3) - 4; + pr_debug("sp=%lx\n", current->mm->start_stack); + + /* copy the arg pages onto the stack */ + res = transfer_args_to_stack(bprm, ¤t->mm->start_stack); + if (!res) + res = create_flat_tables(bprm, current->mm->start_stack); +#endif + if (res) + return res; + + /* Fake some return addresses to ensure the call chain will + * initialise library in order for us. We are required to call + * lib 1 first, then 2, ... and finally the main program (id 0). + */ + start_addr = libinfo.lib_list[0].entry; + +#ifdef FLAT_PLAT_INIT + FLAT_PLAT_INIT(regs); +#endif + + finalize_exec(bprm); + pr_debug("start_thread(regs=0x%p, entry=0x%lx, start_stack=0x%lx)\n", + regs, start_addr, current->mm->start_stack); + start_thread(regs, start_addr, current->mm->start_stack); + + return 0; +} + +/****************************************************************************/ + +static int __init init_flat_binfmt(void) +{ + register_binfmt(&flat_format); + return 0; +} +core_initcall(init_flat_binfmt); + +/****************************************************************************/ diff --git a/fs/binfmt_misc.c b/fs/binfmt_misc.c new file mode 100644 index 000000000..ddfd3aa57 --- /dev/null +++ b/fs/binfmt_misc.c @@ -0,0 +1,1695 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * binfmt_misc.c + * + * Copyright (C) 1997 Richard Günther + * + * binfmt_misc detects binaries via a magic or filename extension and invokes + * a specified wrapper. See Documentation/admin-guide/binfmt-misc.rst for more details. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +/* Entry status and match type bit numbers. */ +enum binfmt_misc_entry_bits { + MISC_FMT_ENABLED_BIT = 0, + MISC_FMT_MAGIC_BIT = 1, + MISC_FMT_BPF_BIT = 2, +}; + +/* Entry behavior flags, fixed at registration time. */ +enum binfmt_misc_entry_flags { + MISC_FMT_PRESERVE_ARGV0 = (1U << 31), + MISC_FMT_OPEN_BINARY = (1U << 30), + MISC_FMT_CREDENTIALS = (1U << 29), + MISC_FMT_OPEN_FILE = (1U << 28), + MISC_FMT_TRANSPARENT = (1U << 27), + MISC_FMT_LOADER = (1U << 26), + MISC_FMT_DISABLED = (1U << 25), +}; + +/* The flags that shape the invocation; a 'B' handler picks those per exec. */ +#define MISC_FMT_INVOCATION_FLAGS (MISC_FMT_PRESERVE_ARGV0 | \ + MISC_FMT_OPEN_BINARY | \ + MISC_FMT_CREDENTIALS | \ + MISC_FMT_OPEN_FILE | \ + MISC_FMT_TRANSPARENT | \ + MISC_FMT_LOADER) + +/** + * struct binfmt_misc_flag - a flag character of the register string + * @c: the character userspace writes and reads back + * @flag: the entry flag it sets + * @implies: entry flags it turns on in addition + * @desc: what it does, for the registration debug output + */ +struct binfmt_misc_flag { + char c; + unsigned long flag; + unsigned long implies; + const char *desc; +}; + +static const struct binfmt_misc_flag misc_flags[] = { + { 'P', MISC_FMT_PRESERVE_ARGV0, 0, "preserve argv0" }, + { 'O', MISC_FMT_OPEN_BINARY, 0, "open binary" }, + { 'C', MISC_FMT_CREDENTIALS, MISC_FMT_OPEN_BINARY, "credentials from the binary" }, + { 'F', MISC_FMT_OPEN_FILE, 0, "open interpreter file now" }, + { 'T', MISC_FMT_TRANSPARENT, MISC_FMT_OPEN_BINARY, "transparent" }, + { 'L', MISC_FMT_LOADER, 0, "loader substitution" }, + { 'D', MISC_FMT_DISABLED, 0, "register disabled" }, +}; + +/* Look up a flag character, NULL if @c is not one. */ +static const struct binfmt_misc_flag *misc_flag_by_char(const char c) +{ + for (int i = 0; i < ARRAY_SIZE(misc_flags); i++) + if (misc_flags[i].c == c) + return &misc_flags[i]; + return NULL; +} + +struct binfmt_misc_entry { + struct hlist_node node; + unsigned long flags; /* type, status, etc. */ + int offset; /* offset of magic */ + int size; /* size of magic/mask */ + char *magic; /* magic or filename extension */ + char *mask; /* mask, NULL for exact match */ + const char *interpreter; /* filename of interpreter */ + char *name; + struct dentry *dentry; + const struct binfmt_misc_ops *bpf_ops; /* bpf-backed handler ('B') */ + const char *bpf_ops_name; + struct list_head interps; /* the interpreters it bound */ + refcount_t users; /* sync removal with load_misc_binary() */ + struct rcu_head rcu; + char buf[]; /* register string, fields point in here */ +}; + +/* + * Max length of the register string. Determined by: + * - 7 delimiters + * - name: ~50 bytes + * - type: 1 byte + * - offset: 3 bytes (has to be smaller than BINPRM_BUF_SIZE) + * - magic: 128 bytes (512 in escaped form) + * - mask: 128 bytes (512 in escaped form) + * - interp: ~50 bytes + * - flags: 5 bytes + * Round that up a bit, and then back off to hold the internal data + * (like struct binfmt_misc_entry). + */ +#define MAX_REGISTER_LENGTH 1920 + +/* Trailing delimiter pad so field parsing always terminates at a delimiter. */ +#define MISC_DELIM_PAD 8 + +/* Protects the entry walk in load_misc_binary(), which may sleep in it. */ +DEFINE_STATIC_SRCU_FAST(bm_entries_srcu); + +/* Check if @e's magic matches @bprm's buffer, applying the mask if set. */ +static bool entry_matches_magic(const struct binfmt_misc_entry *e, + const struct linux_binprm *bprm) +{ + const char *s = bprm->buf + e->offset; + int i; + + if (!e->mask) + return !memcmp(s, e->magic, e->size); + + for (i = 0; i < e->size; i++) + if ((s[i] ^ e->magic[i]) & e->mask[i]) + return false; + return true; +} + +/* Check if @e's registered extension matches @ext, NULL if there is none. */ +static bool entry_matches_extension(const struct binfmt_misc_entry *e, + const char *ext) +{ + return ext && !strcmp(e->magic, ext); +} + +/** + * search_binfmt_handler - search for a binary handler for @bprm + * @misc: handle to binfmt_misc instance + * @bprm: binary for which we are looking for a handler + * + * Search for a binary type handler for @bprm in the list of registered binary + * type handlers. A 'B' entry's match program decides whether the handler + * applies; it may sleep to read the binary. The matched entry is returned + * with a reference taken while the walk still held it; a dying entry - + * unlinked with its last reference gone - cannot be matched and the walk + * moves on. + * + * The caller must hold the bm_entries_srcu read lock, which allows an + * entry's evaluation to sleep. + * + * Return: referenced binary type list entry on success, NULL on failure + */ +static struct binfmt_misc_entry * +search_binfmt_handler(struct binfmt_misc *misc, struct linux_binprm *bprm) +{ + char *dot = strrchr(bprm->interp, '.'); + const char *ext = dot ? dot + 1 : NULL; + struct binfmt_misc_entry *e; + + /* Walk all the registered handlers. */ + hlist_for_each_entry_rcu(e, &misc->entries, node, + srcu_read_lock_held(&bm_entries_srcu)) { + /* + * Make sure this one is currently enabled. An entry enters + * the list at most once and only whole: its configuration is + * ordered before the rcu insertion that makes it visible + * here. + */ + if (!test_bit(MISC_FMT_ENABLED_BIT, &e->flags)) + continue; + + if (test_bit(MISC_FMT_BPF_BIT, &e->flags)) { + if (!e->bpf_ops->match(bprm)) + continue; + } else if (test_bit(MISC_FMT_MAGIC_BIT, &e->flags)) { + if (!entry_matches_magic(e, bprm)) + continue; + } else { + if (!entry_matches_extension(e, ext)) + continue; + } + + /* A dying entry cannot be matched, walk on. */ + if (refcount_inc_not_zero(&e->users)) + return e; + } + + return NULL; +} + +/** + * get_binfmt_handler - try to find a binary type handler + * @misc: handle to binfmt_misc instance + * @bprm: binary for which we are looking for a handler + * + * Try to find a binfmt handler for the binary type. If one is found it is + * returned with a reference protecting it against removal via + * bm_{entry,status}_write(). + * + * Return: binary type list entry on success, NULL on failure + */ +static struct binfmt_misc_entry *get_binfmt_handler(struct binfmt_misc *misc, + struct linux_binprm *bprm) +{ + guard(srcu_fast)(&bm_entries_srcu); + return search_binfmt_handler(misc, bprm); +} + +/** + * binfmt_misc_find_interp - find a bound interpreter by name + * @interps: the interpreters the matched entry was registered with + * @name: the name to look for + * + * Return: the interpreter on success, NULL if @interps has none by that name + */ +const struct binfmt_misc_interp * +binfmt_misc_find_interp(const struct list_head *interps, const char *name) +{ + struct binfmt_misc_interp *interp; + + list_for_each_entry(interp, interps, list) + if (!strcmp(interp->name, name)) + return interp; + return NULL; +} + +/* Undo the open_exec() a pre-opened interpreter file came from. */ +static void close_interp_file(struct file *f) +{ + if (IS_ERR_OR_NULL(f)) + return; + exe_file_allow_write_access(f); + filp_close(f, NULL); +} + +DEFINE_FREE(close_interp_file, struct file *, close_interp_file(_T)) + +/* + * Open an interpreter @path for execution: now, in the writer's context, + * and - since binfmt_misc mounts can be unprivileged - with @cred, the + * credentials the control file being written was opened with, not the + * writer's own. + */ +static struct file *open_interp_file(const struct cred *cred, const char *path) +{ + struct file *f; + + scoped_with_creds(cred) + f = open_exec(path); + if (IS_ERR(f)) + pr_notice("register: failed to install interpreter %s\n", path); + return f; +} + +/* Release the interpreters an entry was registered with. */ +static void entry_put_interpreters(struct binfmt_misc_entry *e) +{ + struct binfmt_misc_interp *interp, *tmp; + + list_for_each_entry_safe(interp, tmp, &e->interps, list) { + list_del(&interp->list); + close_interp_file(interp->file); + dec_ucount(interp->ucounts, UCOUNT_BINFMT_MISC_INTERPRETERS); + kfree(interp); + } +} + +/** + * entry_attach_interpreter - bind an opened interpreter to @e + * @e: entry being configured + * @name: name the load program will select it by; empty for the fixed + * interpreter of a static entry + * @path: the path @f was opened from + * @f: the interpreter, opened for execution + * + * Every exec runs a clone of @f, so the path decided which file is bound + * and nothing else: it is not resolved again, in any namespace. + * + * The caller has to have validated @name and @path, established that @e + * cannot be matched yet, and owns @f until this succeeds. + * + * Return: 0 on success, -ENOSPC if the entry is full or the binder is out of + * UCOUNT_BINFMT_MISC_INTERPRETERS budget, a negative errno on failure + */ +static int entry_attach_interpreter(struct binfmt_misc_entry *e, + const char *name, const char *path, + struct file *f) +{ + size_t nlen = strlen(name), plen = strlen(path); + struct binfmt_misc_interp *interp; + struct ucounts *ucounts; + + if (binfmt_misc_find_interp(&e->interps, name)) + return -EEXIST; + if (list_count_nodes(&e->interps) >= BINFMT_MISC_INTERP_MAX) + return -ENOSPC; + + /* The binding keeps a file open, so charge it to whoever binds it. */ + ucounts = inc_ucount(current_user_ns(), current_euid(), + UCOUNT_BINFMT_MISC_INTERPRETERS); + if (!ucounts) + return -ENOSPC; + + /* One allocation, both strings in it, like the entry's own buffer. */ + interp = kmalloc(struct_size(interp, name, nlen + plen + 2), + GFP_KERNEL_ACCOUNT); + if (!interp) { + dec_ucount(ucounts, UCOUNT_BINFMT_MISC_INTERPRETERS); + return -ENOMEM; + } + + interp->path = interp->name + nlen + 1; + strscpy(interp->name, name, nlen + 1); + strscpy(interp->name + nlen + 1, path, plen + 1); + interp->file = f; + interp->ucounts = ucounts; + /* Publish the node: a lockless cat may be walking the list. */ + list_add_tail_rcu(&interp->list, &e->interps); + pr_debug("register: interpreter: %s {%s}\n", name, path); + return 0; +} + +static void bm_entry_free_rcu(struct rcu_head *rcu) +{ + struct binfmt_misc_entry *e = container_of(rcu, struct binfmt_misc_entry, rcu); + + /* No walker that could sleep in the handler's programs is left. */ + if (e->bpf_ops) + binfmt_misc_put_ops(e->bpf_ops); + kfree(e); +} + +/** + * put_binfmt_handler - put binary handler entry + * @e: entry to put + * + * Free entry syncing with load_misc_binary() and defer final free to + * load_misc_binary() in case it is using the binary type handler we were + * requested to remove. Also the teardown for a registration that fails + * before add_entry() publishes the entry. + */ +static void put_binfmt_handler(struct binfmt_misc_entry *e) +{ + if (IS_ERR_OR_NULL(e)) + return; + + if (refcount_dec_and_test(&e->users)) { + entry_put_interpreters(e); + /* Walkers may still dereference this entry, even sleeping. */ + call_srcu(&bm_entries_srcu, &e->rcu, bm_entry_free_rcu); + } +} + +DEFINE_FREE(put_binfmt_handler, struct binfmt_misc_entry *, put_binfmt_handler(_T)) + +/* Drop everything a load program staged for this exec. */ +static void drop_staged_selection(struct linux_binprm *bprm) +{ + kfree(bprm->bpf_interp); + bprm->bpf_interp = NULL; + kfree(bprm->bpf_interp_arg); + bprm->bpf_interp_arg = NULL; + if (bprm->bpf_interp_file) { + fput(bprm->bpf_interp_file); + bprm->bpf_interp_file = NULL; + } + bprm->bpf_flags = 0; +} + +/** + * current_binfmt_misc - get the binfmt_misc instance of the caller's user namespace + * + * If a user namespace doesn't have its own binfmt_misc mount it uses the + * handlers of its closest ancestor with one. This mimics the behavior of + * pre-namespaced binfmt_misc where all registered handlers were available + * to all users and user namespaces on the system. The init user namespace + * instance is statically set up so the fallback is never reached in + * practice. + * + * Return: the binfmt_misc instance of the caller's user namespace + */ +static struct binfmt_misc *current_binfmt_misc(void) +{ + const struct user_namespace *user_ns; + struct binfmt_misc *misc; + + for (user_ns = current_user_ns(); user_ns; user_ns = user_ns->parent) { + /* Pairs with smp_store_release() in bm_fill_super(). */ + misc = smp_load_acquire(&user_ns->binfmt_misc); + if (misc) + return misc; + } + + return &init_binfmt_misc; +} + +/** + * entry_select_interpreter - get the interpreter for the matched @e + * @e: matched binary type handler + * @bprm: binary that is being executed + * + * A static entry carries its interpreter path, for a 'B' entry the + * handler's load program selects it, either by path or by the name of one + * of the interpreters the entry bound. The match is committed, so a failing + * program fails the exec. + * + * Return: the interpreter on success, an ERR_PTR on failure + */ +static const char *entry_select_interpreter(const struct binfmt_misc_entry *e, + struct linux_binprm *bprm) +{ + int retval; + + /* + * Drop what a previous chain level staged before anything can pick it + * up. A static entry stages nothing but consumes a staged file just + * like a 'B' entry does. + */ + drop_staged_selection(bprm); + + if (!test_bit(MISC_FMT_BPF_BIT, &e->flags)) + return e->interpreter; + + /* The interpreters this entry lets the program choose from. */ + bprm->bpf_interps = &e->interps; + retval = e->bpf_ops->load(bprm); + bprm->bpf_interps = NULL; + if (retval) { + /* Keep a program-supplied error within errno range. */ + if (retval > 0 || retval < -MAX_ERRNO) + retval = -ENOEXEC; + goto drop_staged; + } + + /* Selecting an interpreter is part of the contract. */ + if (!bprm->bpf_interp) { + retval = -ENOEXEC; + goto drop_staged; + } + + return bprm->bpf_interp; + +drop_staged: + /* A failing load leaves nothing behind for later entries. */ + drop_staged_selection(bprm); + return ERR_PTR(retval); +} + +/** + * entry_invocation_flags - the invocation flags in effect for this exec + * @e: matched binary type handler + * @bprm: binary that is being executed + * + * A static entry fixes its flags at registration, a 'B' entry's load program + * picks them per exec with bpf_binprm_set_flags(). Translate the latter into + * the former, implications included, so the dispatch has one set to act on. + * + * Return: the invocation flags for this exec + */ +static unsigned long entry_invocation_flags(const struct binfmt_misc_entry *e, + struct linux_binprm *bprm) +{ + unsigned long flags = 0; + u64 bpf_flags; + + if (!test_bit(MISC_FMT_BPF_BIT, &e->flags)) + return e->flags; + + bpf_flags = bprm->bpf_flags; + /* Clear so they can't accumulate into a nested interpreter level. */ + bprm->bpf_flags = 0; + + if (bpf_flags & BPF_BINPRM_PRESERVE_ARGV0) + flags |= MISC_FMT_PRESERVE_ARGV0; + if (bpf_flags & BPF_BINPRM_EXECFD) + flags |= MISC_FMT_OPEN_BINARY; + if (bpf_flags & BPF_BINPRM_CREDENTIALS) + flags |= MISC_FMT_CREDENTIALS | MISC_FMT_OPEN_BINARY; + if (bpf_flags & BPF_BINPRM_TRANSPARENT) + flags |= MISC_FMT_TRANSPARENT | MISC_FMT_OPEN_BINARY; + if (bpf_flags & BPF_BINPRM_LOADER) + flags |= MISC_FMT_LOADER; + + return flags; +} + +/** + * entry_open_interpreter - open the entry's interpreter for execution + * @e: matched binary type handler + * @bprm: binary that is being executed + * @interpreter: the interpreter selected for this exec + * + * An 'F' entry hands out a clone of the file it pre-opened at registration, + * and so does a 'B' entry whose load program selected one of the + * interpreters it bound. Any other entry opens the selected path. + * + * Return: the opened interpreter on success, an ERR_PTR on failure + */ +static struct file *entry_open_interpreter(const struct binfmt_misc_entry *e, + struct linux_binprm *bprm, + const char *interpreter) +{ + struct file *interp_file __free(fput) = NULL; + struct binfmt_misc_interp *interp; + struct file *bound; + int retval; + + if (bprm->bpf_interp_file) { + bound = bprm->bpf_interp_file; + } else if (e->flags & MISC_FMT_OPEN_FILE) { + /* An 'F' entry pre-opened exactly one interpreter. */ + interp = list_first_entry(&e->interps, + struct binfmt_misc_interp, list); + bound = interp->file; + } else { + return open_exec(interpreter); + } + + interp_file = file_clone_open(bound); + if (IS_ERR(interp_file)) + return interp_file; + + retval = exe_file_deny_write_access(interp_file); + if (retval) + return ERR_PTR(retval); + + return no_free_ptr(interp_file); +} + +/** + * build_interp_argv - splice the interpreter invocation into the argv + * @bprm: binary that is being executed + * @interpreter: the interpreter selected for this exec + * @flags: invocation flags in effect for this exec + * + * The interpreter becomes argv[0] and the binary its last argument, with an + * optional staged argument in between. The caller's argv[0] is dropped + * unless 'P' keeps it. + * + * Return: 0 on success, a negative error code on failure + */ +static int build_interp_argv(struct linux_binprm *bprm, const char *interpreter, + unsigned long flags) +{ + int retval; + + /* The interpreter has to be able to load the binary by path. */ + if (bprm->interp_flags & BINPRM_FLAGS_PATH_INACCESSIBLE) + return -ENOENT; + + /* The entry's own choice - not one accumulated from an earlier level. */ + if (flags & MISC_FMT_PRESERVE_ARGV0) { + bprm->interp_flags |= BINPRM_FLAGS_PRESERVE_ARGV0; + } else { + retval = remove_arg_zero(bprm); + if (retval) + return retval; + } + + /* make the binary the last argument to the interpreter */ + retval = copy_string_kernel(bprm->interp, bprm); + if (retval < 0) + return retval; + bprm->argc++; + + /* + * A single optional argument to the interpreter, inserted between it + * and the binary just like the argument of a #! interpreter line. + */ + if (bprm->bpf_interp_arg) { + retval = copy_string_kernel(bprm->bpf_interp_arg, bprm); + if (retval < 0) + return retval; + bprm->argc++; + /* Consumed - don't let it leak into a nested interpreter's argv. */ + kfree(bprm->bpf_interp_arg); + bprm->bpf_interp_arg = NULL; + } + + /* add the interp as argv[0] */ + retval = copy_string_kernel(interpreter, bprm); + if (retval < 0) + return retval; + bprm->argc++; + + return 0; +} + +/* + * the loader itself + */ +static int load_misc_binary(struct linux_binprm *bprm) +{ + struct binfmt_misc_entry *fmt __free(put_binfmt_handler) = NULL; + const char *interpreter; + struct file *interp_file; + struct binfmt_misc *misc; + unsigned long flags; + int retval; + + /* Only binfmt_misc stages one and exec_binprm() clears it per round. */ + WARN_ON_ONCE(bprm->loader); + + misc = current_binfmt_misc(); + if (!READ_ONCE(misc->enabled)) + return -ENOEXEC; + + fmt = get_binfmt_handler(misc, bprm); + if (!fmt) + return -ENOEXEC; + + interpreter = entry_select_interpreter(fmt, bprm); + if (IS_ERR(interpreter)) + return PTR_ERR(interpreter); + + flags = entry_invocation_flags(fmt, bprm); + + /* No argv is built for a staged argument to land in. */ + if ((flags & (MISC_FMT_LOADER | MISC_FMT_TRANSPARENT)) && + bprm->bpf_interp_arg) + return -EINVAL; + + /* + * Stash the interpreter for binfmt_elf to consume in place of the + * binary's PT_INTERP and decline the match, so the search continues + * to the real format in the same round. + */ + if (flags & MISC_FMT_LOADER) { + interp_file = entry_open_interpreter(fmt, bprm, interpreter); + if (IS_ERR(interp_file)) { + retval = PTR_ERR(interp_file); + /* Declining here would run the binary's own PT_INTERP. */ + return retval == -ENOEXEC ? -EACCES : retval; + } + + bprm->loader = interp_file; + return -ENOEXEC; + } + + if (!(flags & MISC_FMT_TRANSPARENT)) { + retval = build_interp_argv(bprm, interpreter, flags); + if (retval) + return retval; + } + + /* Update interp for the next round; sched_prepare_exec reports it. */ + retval = bprm_change_interp(interpreter, bprm); + if (retval < 0) + return retval; + + interp_file = entry_open_interpreter(fmt, bprm, interpreter); + if (IS_ERR(interp_file)) + return PTR_ERR(interp_file); + + /* Raise only past the last failure, or an -ENOEXEC decline leaks it. */ + if (flags & MISC_FMT_TRANSPARENT) + bprm->interp_flags |= BINPRM_FLAGS_TRANSPARENT_INTERP; + + bprm->interpreter = interp_file; + if (flags & MISC_FMT_OPEN_BINARY) + bprm->have_execfd = 1; + if (flags & MISC_FMT_CREDENTIALS) + bprm->execfd_creds = 1; + return 0; +} + +/* Command parsers */ + +/* + * Scan the argument starting at @s up to the delimiter @del, recognising + * the \x escape. Terminates the argument with a NUL and returns a pointer + * past it or NULL on a malformed escape. + */ +static char *scanarg(char *s, char del) +{ + char c; + + while ((c = *s++) != del) { + if (c == '\\' && *s == 'x') { + s++; + if (!isxdigit(*s++)) + return NULL; + if (!isxdigit(*s++)) + return NULL; + } + } + s[-1] = '\0'; + return s; +} + +/* Parse the 'flags' field, stopping at the first character that is not one. */ +static char *check_special_flags(char *p, struct binfmt_misc_entry *e) +{ + for (;; p++) { + const struct binfmt_misc_flag *f = misc_flag_by_char(*p); + + if (!f) + return p; + pr_debug("register: flag: %c (%s)\n", f->c, f->desc); + e->flags |= f->flag | f->implies; + } +} + +/* Parse the 'offset', 'magic' and 'mask' fields of an 'M' entry. */ +static char *parse_magic_fields(struct binfmt_misc_entry *e, char *p, char del) +{ + char *s; + + /* Parse the 'offset' field. */ + s = strchr(p, del); + if (!s) + return NULL; + *s = '\0'; + if (p != s) { + if (kstrtoint(p, 10, &e->offset) || e->offset < 0) + return NULL; + } + p = s + 1; + pr_debug("register: offset: %#x\n", e->offset); + + /* Parse the 'magic' field. */ + e->magic = p; + p = scanarg(p, del); + if (!p || !e->magic[0]) + return NULL; + print_hex_dump_debug( + KBUILD_MODNAME ": register: magic[raw]: ", + DUMP_PREFIX_NONE, 16, 1, e->magic, p - e->magic, true); + + /* Parse the 'mask' field. */ + e->mask = p; + p = scanarg(p, del); + if (!p) + return NULL; + if (!e->mask[0]) { + e->mask = NULL; + pr_debug("register: mask[raw]: none\n"); + } else { + print_hex_dump_debug( + KBUILD_MODNAME ": register: mask[raw]: ", + DUMP_PREFIX_NONE, 16, 1, e->mask, p - e->mask, true); + } + + /* + * Decode the magic & mask fields. Note: while we might have accepted + * embedded NUL bytes from above, the unescape helpers will stop at + * the first one they encounter. + */ + e->size = string_unescape_inplace(e->magic, UNESCAPE_HEX); + if (e->mask && string_unescape_inplace(e->mask, UNESCAPE_HEX) != e->size) + return NULL; + if (e->size > BINPRM_BUF_SIZE || BINPRM_BUF_SIZE - e->size < e->offset) + return NULL; + pr_debug("register: magic/mask length: %i\n", e->size); + print_hex_dump_debug( + KBUILD_MODNAME ": register: magic[decoded]: ", + DUMP_PREFIX_NONE, 16, 1, e->magic, e->size, true); + if (e->mask) + print_hex_dump_debug( + KBUILD_MODNAME ": register: mask[decoded]: ", + DUMP_PREFIX_NONE, 16, 1, e->mask, e->size, true); + return p; +} + +/* Parse the 'magic' field of an 'E' entry: the filename extension. */ +static char *parse_extension_fields(struct binfmt_misc_entry *e, char *p, + char del) +{ + /* Skip the 'offset' field. */ + p = strchr(p, del); + if (!p) + return NULL; + *p++ = '\0'; + + /* Parse the 'magic' field. */ + e->magic = p; + p = strchr(p, del); + if (!p) + return NULL; + *p++ = '\0'; + if (!e->magic[0] || strchr(e->magic, '/')) + return NULL; + pr_debug("register: extension: {%s}\n", e->magic); + + /* Skip the 'mask' field. */ + p = strchr(p, del); + if (!p) + return NULL; + *p++ = '\0'; + return p; +} + +/* + * Parse the fields of a 'B' entry: the 'offset', 'magic' and 'mask' fields + * must be empty. The handler name is carried in the 'interpreter' field. + */ +static char *parse_bpf_fields(struct binfmt_misc_entry *e, char *p, char del) +{ + /* The 'offset' field must be empty. */ + if (*p++ != del) + return NULL; + + /* The 'magic' field must be empty. */ + if (*p++ != del) + return NULL; + + /* The 'mask' field must be empty. */ + if (*p++ != del) + return NULL; + + return p; +} + +/* + * This registers a new binary format, it recognises the syntax + * ':name:type:offset:magic:mask:interpreter:flags' + * where the ':' is the IFS, that can be chosen with the first char + */ +static struct binfmt_misc_entry *create_entry(const char __user *buffer, + size_t count) +{ + struct binfmt_misc_entry *e __free(kfree) = NULL; + char *buf, *p; + char del; + + pr_debug("register: received %zu bytes\n", count); + + /* some sanity checks */ + if ((count < 11) || (count > MAX_REGISTER_LENGTH)) + return ERR_PTR(-EINVAL); + + e = kmalloc(struct_size(e, buf, count + MISC_DELIM_PAD), + GFP_KERNEL_ACCOUNT); + if (!e) + return ERR_PTR(-ENOMEM); + + p = buf = e->buf; + + memset(e, 0, sizeof(*e)); + INIT_LIST_HEAD(&e->interps); + if (copy_from_user(buf, buffer, count)) + return ERR_PTR(-EFAULT); + + del = *p++; /* delimiter */ + + pr_debug("register: delim: %#x {%c}\n", del, del); + + /* A flag-char delimiter runs the flag scan off the buffer. */ + if (misc_flag_by_char(del)) + return ERR_PTR(-EINVAL); + + /* Pad the buffer with the delim to simplify parsing below. */ + memset(buf + count, del, MISC_DELIM_PAD); + + /* Parse the 'name' field. */ + e->name = p; + p = strchr(p, del); + if (!p) + return ERR_PTR(-EINVAL); + *p++ = '\0'; + if (!e->name[0] || + !strcmp(e->name, ".") || + !strcmp(e->name, "..") || + strchr(e->name, '/')) + return ERR_PTR(-EINVAL); + + pr_debug("register: name: {%s}\n", e->name); + + /* Parse the 'type' field. */ + switch (*p++) { + case 'E': + pr_debug("register: type: E (extension)\n"); + e->flags = BIT(MISC_FMT_ENABLED_BIT); + break; + case 'M': + pr_debug("register: type: M (magic)\n"); + e->flags = BIT(MISC_FMT_ENABLED_BIT) | BIT(MISC_FMT_MAGIC_BIT); + break; + case 'B': + pr_debug("register: type: B (bpf)\n"); + if (!IS_ENABLED(CONFIG_BINFMT_MISC_BPF)) + return ERR_PTR(-EINVAL); + e->flags = BIT(MISC_FMT_ENABLED_BIT) | BIT(MISC_FMT_BPF_BIT); + break; + default: + return ERR_PTR(-EINVAL); + } + if (*p++ != del) + return ERR_PTR(-EINVAL); + + if (test_bit(MISC_FMT_BPF_BIT, &e->flags)) + p = parse_bpf_fields(e, p, del); + else if (test_bit(MISC_FMT_MAGIC_BIT, &e->flags)) + p = parse_magic_fields(e, p, del); + else + p = parse_extension_fields(e, p, del); + if (!p) + return ERR_PTR(-EINVAL); + + /* Parse the 'interpreter' field. */ + e->interpreter = p; + p = strchr(p, del); + if (!p) + return ERR_PTR(-EINVAL); + *p++ = '\0'; + if (test_bit(MISC_FMT_BPF_BIT, &e->flags)) { + /* The 'interpreter' field carries the handler name. */ + e->bpf_ops_name = e->interpreter; + e->interpreter = NULL; + if (!e->bpf_ops_name[0]) + return ERR_PTR(-EINVAL); + pr_debug("register: bpf handler: {%s}\n", e->bpf_ops_name); + } else if (!e->interpreter[0]) { + return ERR_PTR(-EINVAL); + } else { + pr_debug("register: interpreter: {%s}\n", e->interpreter); + } + + /* Parse the 'flags' field. */ + p = check_special_flags(p, e); + + /* + * A bpf handler decides the invocation flags per exec with + * bpf_binprm_set_flags() rather than fixing them at registration, and + * the interpreters it binds pre-open what 'F' would have, so a 'B' + * entry carries no invocation flags. + */ + if (test_bit(MISC_FMT_BPF_BIT, &e->flags) && + (e->flags & MISC_FMT_INVOCATION_FLAGS)) + return ERR_PTR(-EINVAL); + + /* + * 'D' is a directive for this registration rather than a lasting + * property, so consume it: the entry is created disabled and stays + * out of the search list until '1' is written to its entry file. + * Staying out is what leaves it open to being given interpreters; + * the first enable publishes it, for good. + */ + if (e->flags & MISC_FMT_DISABLED) { + e->flags &= ~MISC_FMT_DISABLED; + clear_bit(MISC_FMT_ENABLED_BIT, &e->flags); + } + + /* Transparency preserves the whole argv, argv[0] included. */ + if ((e->flags & MISC_FMT_TRANSPARENT) && + (e->flags & MISC_FMT_PRESERVE_ARGV0)) + return ERR_PTR(-EINVAL); + + /* A native exec splices no argv, passes no execfd and needs no creds. */ + if ((e->flags & MISC_FMT_LOADER) && + (e->flags & (MISC_FMT_TRANSPARENT | MISC_FMT_PRESERVE_ARGV0 | + MISC_FMT_CREDENTIALS | MISC_FMT_OPEN_BINARY))) + return ERR_PTR(-EINVAL); + + if (*p == '\n') + p++; + if (p != buf + count) + return ERR_PTR(-EINVAL); + + /* Non-F opens the interp at exec against the caller's cwd; require absolute. */ + if ((e->flags & (MISC_FMT_LOADER | MISC_FMT_CREDENTIALS)) && + !(e->flags & MISC_FMT_OPEN_FILE) && + e->interpreter[0] != '/') + return ERR_PTR(-EINVAL); + + /* Born holding one reference; put_binfmt_handler() is the teardown. */ + refcount_set(&e->users, 1); + return no_free_ptr(e); +} + +/* Commands accepted by the /status and / files. */ +enum bm_command { + BM_CMD_IGNORE, /* empty write */ + BM_CMD_DISABLE, /* "0" */ + BM_CMD_ENABLE, /* "1" */ + BM_CMD_REMOVE, /* "-1" */ +}; + +/* Longest of the commands above, "-1\n". */ +#define MAX_COMMAND_LENGTH 3 + +/* + * Parse what userspace wrote to /status or an entry file: '1' enables, + * '0' disables and '-1' removes the entry or all entries. + */ +static int parse_command(const char *s, size_t count) +{ + if (count > MAX_COMMAND_LENGTH) + return -EINVAL; + if (!count) + return BM_CMD_IGNORE; + if (s[count - 1] == '\n') + count--; + if (count == 1 && s[0] == '0') + return BM_CMD_DISABLE; + if (count == 1 && s[0] == '1') + return BM_CMD_ENABLE; + if (count == 2 && s[0] == '-' && s[1] == '1') + return BM_CMD_REMOVE; + return -EINVAL; +} + +/* Copy in a command from a file that takes nothing else, and parse it. */ +static int read_command(const char __user *buffer, size_t count) +{ + char s[MAX_COMMAND_LENGTH + 1]; + + if (count > sizeof(s) - 1) + return -EINVAL; + if (copy_from_user(s, buffer, count)) + return -EFAULT; + return parse_command(s, count); +} + +/* generic stuff */ + +/* The root directory's inode; its lock serializes configuring an instance. */ +static struct inode *bm_root_inode(struct super_block *sb) +{ + return d_inode(sb->s_root); +} + +static void bm_seq_hex(struct seq_file *m, const u8 *data, int size) +{ + for (int i = 0; i < size; i++) + seq_printf(m, "%02x", data[i]); +} + +static int bm_entry_show(struct seq_file *m, void *unused) +{ + struct binfmt_misc_entry *e = m->private; + + if (test_bit(MISC_FMT_ENABLED_BIT, &e->flags)) + seq_puts(m, "enabled\n"); + else + seq_puts(m, "disabled\n"); + + if (test_bit(MISC_FMT_BPF_BIT, &e->flags)) { + struct binfmt_misc_interp *interp; + + seq_printf(m, "bpf %s\n", e->bpf_ops->name); + /* + * A staged entry's set can still grow, so every binding is + * rcu-published. The open file pins the entry and with it + * every node, so rcu is for the tearing, not the lifetime. + */ + rcu_read_lock(); + list_for_each_entry_rcu(interp, &e->interps, list) + seq_printf(m, "bpf-interpreter %s %s\n", + interp->name, interp->path); + rcu_read_unlock(); + } else { + seq_printf(m, "interpreter %s\n", e->interpreter); + } + + /* print the special flags */ + seq_puts(m, "flags: "); + for (int i = 0; i < ARRAY_SIZE(misc_flags); i++) + if (e->flags & misc_flags[i].flag) + seq_putc(m, misc_flags[i].c); + seq_putc(m, '\n'); + + if (test_bit(MISC_FMT_BPF_BIT, &e->flags)) { + /* The program does the matching. */ + } else if (!test_bit(MISC_FMT_MAGIC_BIT, &e->flags)) { + seq_printf(m, "extension .%s\n", e->magic); + } else { + seq_printf(m, "offset %i\nmagic ", e->offset); + bm_seq_hex(m, e->magic, e->size); + if (e->mask) { + seq_puts(m, "\nmask "); + bm_seq_hex(m, e->mask, e->size); + } + seq_putc(m, '\n'); + } + return 0; +} + +static struct inode *bm_get_inode(struct super_block *sb, umode_t mode) +{ + struct inode *inode = new_inode(sb); + + if (inode) { + inode->i_ino = get_next_ino(); + inode->i_mode = mode; + simple_inode_init_ts(inode); + } + return inode; +} + +/** + * i_binfmt_misc - retrieve struct binfmt_misc from a binfmt_misc inode + * @inode: inode of the relevant binfmt_misc instance + * + * This helper retrieves struct binfmt_misc from a binfmt_misc inode. This can + * be done without any memory barriers because we are guaranteed that + * user_ns->binfmt_misc is fully initialized. It was fully initialized when the + * binfmt_misc mount was first created. + * + * Return: struct binfmt_misc of the relevant binfmt_misc instance + */ +static struct binfmt_misc *i_binfmt_misc(struct inode *inode) +{ + return inode->i_sb->s_user_ns->binfmt_misc; +} + +/** + * bm_evict_inode - cleanup data associated with @inode + * @inode: inode to which the data is attached + * + * Cleanup the binary type handler data associated with @inode if a binary type + * entry is removed or the filesystem is unmounted and the super block is + * shutdown. + * + * If the ->evict call was not caused by a super block shutdown but by + * removing the entry via bm_{entry,status}_write() or unlink(2) the entry + * will have already been removed from the list. We keep the hlist_unhashed() + * check to make that explicit. +*/ +static void bm_evict_inode(struct inode *inode) +{ + struct binfmt_misc_entry *e = inode->i_private; + + clear_inode(inode); + + if (e) { + struct binfmt_misc *misc; + + misc = i_binfmt_misc(inode); + spin_lock(&misc->entries_lock); + if (!hlist_unhashed(&e->node)) + hlist_del_init_rcu(&e->node); + spin_unlock(&misc->entries_lock); + put_binfmt_handler(e); + } +} + +/** + * unlink_binfmt_handler - unhash a binary type handler + * @misc: handle to binfmt_misc instance + * @e: binary type handler to unhash + * + * Adding and removing entries via bm_{entry,register,status}_write() and + * unlink(2) happens under the exclusively held inode lock of the root + * dentry keeping the list stable for writers. load_misc_binary() walks it + * concurrently under SRCU. The entries_lock is only held around the actual + * unlink to serialize against bm_evict_inode() which unlinks entries + * during umount without holding the root inode lock. + */ +static void unlink_binfmt_handler(struct binfmt_misc *misc, + struct binfmt_misc_entry *e) +{ + spin_lock(&misc->entries_lock); + hlist_del_init_rcu(&e->node); + spin_unlock(&misc->entries_lock); +} + +/** + * remove_binfmt_handler - remove a binary type handler + * @misc: handle to binfmt_misc instance + * @e: binary type handler to remove + * + * Remove a binary type handler from the list of binary type handlers and + * remove its associated dentry. + */ +static void remove_binfmt_handler(struct binfmt_misc *misc, + struct binfmt_misc_entry *e) +{ + unlink_binfmt_handler(misc, e); + locked_recursive_removal(e->dentry, NULL); +} + +/* Remove @e unless it was already removed. */ +static void bm_remove_entry(struct binfmt_misc_entry *e, struct super_block *sb) +{ + struct inode *root = bm_root_inode(sb); + + inode_lock_nested(root, I_MUTEX_PARENT); + /* A staged entry is not hashed; the dentry says if it was removed. */ + if (!d_unhashed(e->dentry)) + remove_binfmt_handler(i_binfmt_misc(root), e); + inode_unlock(root); +} + +/* Remove all entries of the binfmt_misc instance @misc belonging to @sb. */ +static void bm_remove_all_entries(struct binfmt_misc *misc, + struct super_block *sb) +{ + struct inode *root = bm_root_inode(sb); + struct dentry *child = NULL; + + inode_lock_nested(root, I_MUTEX_PARENT); + /* + * Walk the directory rather than the search list: a staged entry + * is in the former but not yet in the latter. The control files + * carry no entry and stay. + */ + while ((child = find_next_child(sb->s_root, child))) { + struct binfmt_misc_entry *e = d_inode(child)->i_private; + + if (e) + remove_binfmt_handler(misc, e); + } + inode_unlock(root); +} + +/** + * bm_unlink - remove a binary type handler via unlink(2) + * @dir: inode of the root directory + * @dentry: entry file to remove + * + * Removing the entry file removes its binary type handler, exactly like + * writing -1 to it does. The status and register control files can't be + * removed. The VFS calls this with the root inode lock held which + * serializes against the write based add and remove paths. + */ +static int bm_unlink(struct inode *dir, struct dentry *dentry) +{ + struct binfmt_misc_entry *e = d_inode(dentry)->i_private; + + if (!e) + return -EPERM; + + unlink_binfmt_handler(i_binfmt_misc(dir), e); + return simple_unlink(dir, dentry); +} + +static const struct inode_operations bm_dir_inode_operations = { + .lookup = simple_lookup, + .unlink = bm_unlink, +}; + +/* / */ + +static int bm_entry_open(struct inode *inode, struct file *file) +{ + int ret; + + ret = single_open(file, bm_entry_show, inode->i_private); + if (ret) + return ret; + + /* seq_open() clears FMODE_PWRITE, bm_entry_write() takes any offset */ + if (file->f_mode & FMODE_WRITE) + file->f_mode |= FMODE_PWRITE; + return 0; +} + +/* + * Longest '+ ' a write can spell, and with it the longest + * command an entry file takes: the two delimiters and a newline on top of + * the two names. + */ +#define MAX_BINDING_LENGTH (BINFMT_MISC_INTERP_NAME_MAX + PATH_MAX + 3) + +/** + * bm_entry_add_interp - bind another interpreter to a staged entry + * @e: the entry + * @file: the entry file being written to, for its credentials + * @buf: the '+ ' command, parsed in place and owned by the caller + * @count: its length + * + * A 'D' entry is registered outside the search list, which is what leaves + * it open to being configured: it cannot be matched, so no exec can be + * holding its interpreters and the set can still grow. Its first enable + * publishes it and ends that. One interpreter per write, up to + * BINFMT_MISC_INTERP_MAX of them, none of which has to fit in a register + * string. + * + * Return: @count on success, a negative errno on failure + */ +static ssize_t bm_entry_add_interp(struct binfmt_misc_entry *e, + struct file *file, char *buf, size_t count) +{ + struct file *f __free(close_interp_file) = NULL; + struct inode *root = bm_root_inode(file_inode(file)->i_sb); + size_t nlen, plen; + char *name, *path; + int retval; + + /* Settled before the open: type is fixed, publication is permanent. */ + if (!test_bit(MISC_FMT_BPF_BIT, &e->flags)) + return -EINVAL; + if (!hlist_unhashed_lockless(&e->node)) + return -EBUSY; + + /* '+ ': the path is everything past the first space. */ + name = buf + 1; + path = strchr(name, ' '); + if (!path) + return -EINVAL; + *path++ = '\0'; + + plen = strlen(path); + /* The command has to end at the write, like a register string. */ + if (path + plen != buf + count) + return -EINVAL; + if (plen && path[plen - 1] == '\n') + path[--plen] = '\0'; + /* Resolved now, so a relative path would name the writer's cwd. */ + if (path[0] != '/') + return -EINVAL; + + nlen = path - name - 1; + if (!nlen || nlen > BINFMT_MISC_INTERP_NAME_MAX) + return -EINVAL; + /* The name prints between delimiters, so keep it a printable word. */ + for (const char *p = name; *p; p++) + if (!isascii(*p) || !isgraph(*p)) + return -EINVAL; + + /* Opened before the lock: resolving it may walk this very filesystem. */ + f = open_interp_file(file->f_cred, path); + if (IS_ERR(f)) + return PTR_ERR(f); + + inode_lock(root); + if (d_unhashed(e->dentry)) + retval = -ENOENT; /* removed while we were opening it */ + else if (!hlist_unhashed(&e->node)) + retval = -EBUSY; /* published while we were opening it */ + else + retval = entry_attach_interpreter(e, name, path, f); + inode_unlock(root); + if (retval) + return retval; + + /* The file is owned by the entry now. */ + retain_and_null_ptr(f); + return count; +} + +static ssize_t bm_entry_write(struct file *file, const char __user *buffer, + size_t count, loff_t *ppos) +{ + struct inode *inode = file_inode(file); + struct binfmt_misc_entry *e = inode->i_private; + char *buf __free(kfree) = NULL; + int res; + + /* A binding is the longest command this file takes. */ + if (count > MAX_BINDING_LENGTH) + return -E2BIG; + + buf = memdup_user_nul(buffer, count); + if (IS_ERR(buf)) + return PTR_ERR(buf); + + /* '+ ' binds an interpreter, everything else toggles. */ + if (buf[0] == '+') + return bm_entry_add_interp(e, file, buf, count); + + res = parse_command(buf, count); + + switch (res) { + case BM_CMD_DISABLE: + clear_bit(MISC_FMT_ENABLED_BIT, &e->flags); + break; + case BM_CMD_ENABLE: { + struct inode *root = bm_root_inode(inode->i_sb); + + /* + * The first enable publishes a 'D' entry into the search + * list, whole. The lock keeps that ordered against a second + * enable, against removal - a removed entry has nothing left + * to publish - and against binding: what can be matched can + * no longer be configured. + */ + inode_lock(root); + set_bit(MISC_FMT_ENABLED_BIT, &e->flags); + if (hlist_unhashed(&e->node) && !d_unhashed(e->dentry)) { + struct binfmt_misc *misc = i_binfmt_misc(inode); + + spin_lock(&misc->entries_lock); + hlist_add_head_rcu(&e->node, &misc->entries); + spin_unlock(&misc->entries_lock); + } + inode_unlock(root); + break; + } + case BM_CMD_REMOVE: + bm_remove_entry(e, inode->i_sb); + break; + default: + return res; + } + + return count; +} + +static const struct file_operations bm_entry_operations = { + .open = bm_entry_open, + .read = seq_read, + .write = bm_entry_write, + .llseek = seq_lseek, + .release = single_release, +}; + +/* /register */ + +/* add to filesystem */ +static int add_entry(struct binfmt_misc_entry *e, struct super_block *sb) +{ + struct dentry *dentry = simple_start_creating(sb->s_root, e->name); + struct inode *inode; + struct binfmt_misc *misc; + + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + + inode = bm_get_inode(sb, S_IFREG | 0644); + if (unlikely(!inode)) { + simple_done_creating(dentry); + return -ENOMEM; + } + + e->dentry = dentry; + inode->i_private = e; + inode->i_fop = &bm_entry_operations; + + d_make_persistent(dentry, inode); + /* A 'D' entry stays out of the search list until its first enable. */ + if (test_bit(MISC_FMT_ENABLED_BIT, &e->flags)) { + misc = i_binfmt_misc(inode); + spin_lock(&misc->entries_lock); + hlist_add_head_rcu(&e->node, &misc->entries); + spin_unlock(&misc->entries_lock); + } + simple_done_creating(dentry); + return 0; +} + +static ssize_t bm_register_write(struct file *file, const char __user *buffer, + size_t count, loff_t *ppos) +{ + struct binfmt_misc_entry *e __free(put_binfmt_handler) = NULL; + struct super_block *sb = file_inode(file)->i_sb; + int err; + + e = create_entry(buffer, count); + if (IS_ERR(e)) + return PTR_ERR(e); + + if (test_bit(MISC_FMT_BPF_BIT, &e->flags)) { + e->bpf_ops = binfmt_misc_get_ops(sb->s_user_ns, e->bpf_ops_name); + if (!e->bpf_ops) { + pr_notice("register: no bpf handler named %s\n", + e->bpf_ops_name); + return -ENOENT; + } + } + + if (e->flags & MISC_FMT_OPEN_FILE) { + struct file *f = open_interp_file(file->f_cred, e->interpreter); + + if (IS_ERR(f)) + return PTR_ERR(f); + err = entry_attach_interpreter(e, "", e->interpreter, f); + if (err) { + close_interp_file(f); + return err; + } + } + + err = add_entry(e, sb); + if (err) + return err; + + /* The entry is owned by its inode now. */ + retain_and_null_ptr(e); + return count; +} + +static const struct file_operations bm_register_operations = { + .write = bm_register_write, + .llseek = noop_llseek, +}; + +/* /status */ + +static ssize_t +bm_status_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos) +{ + struct binfmt_misc *misc; + const char *s; + + misc = i_binfmt_misc(file_inode(file)); + s = READ_ONCE(misc->enabled) ? "enabled\n" : "disabled\n"; + return simple_read_from_buffer(buf, nbytes, ppos, s, strlen(s)); +} + +static ssize_t bm_status_write(struct file *file, const char __user *buffer, + size_t count, loff_t *ppos) +{ + struct binfmt_misc *misc; + int res = read_command(buffer, count); + + misc = i_binfmt_misc(file_inode(file)); + switch (res) { + case BM_CMD_DISABLE: + WRITE_ONCE(misc->enabled, false); + break; + case BM_CMD_ENABLE: + WRITE_ONCE(misc->enabled, true); + break; + case BM_CMD_REMOVE: + bm_remove_all_entries(misc, file_inode(file)->i_sb); + break; + default: + return res; + } + + return count; +} + +static const struct file_operations bm_status_operations = { + .read = bm_status_read, + .write = bm_status_write, + .llseek = default_llseek, +}; + +/* Superblock handling */ + +static const struct super_operations bm_super_ops = { + .statfs = simple_statfs, + .evict_inode = bm_evict_inode, +}; + +static int bm_fill_super(struct super_block *sb, struct fs_context *fc) +{ + int err; + struct user_namespace *user_ns = sb->s_user_ns; + struct binfmt_misc *misc; + static const struct tree_descr bm_files[] = { + [2] = {"status", &bm_status_operations, S_IWUSR|S_IRUGO}, + [3] = {"register", &bm_register_operations, S_IWUSR}, + /* last one */ {""} + }; + + /* The fscontext fd may have been passed to another user namespace. */ + if (user_ns != current_user_ns()) + return -EINVAL; + + /* Never exec off this instance and never let anything stack on it. */ + sb->s_iflags |= SB_I_NOEXEC | SB_I_NODEV; + sb->s_stack_depth = FILESYSTEM_MAX_STACK_DEPTH; + + /* + * Lazily allocate a new binfmt_misc instance for this namespace, i.e. + * do it here during the first mount of binfmt_misc. We don't need to + * waste memory for every user namespace allocation. It's likely much + * more common to not mount a separate binfmt_misc instance than it is + * to mount one. + * + * While multiple superblocks can exist they are keyed by userns in + * s_fs_info for binfmt_misc. Hence, the vfs guarantees that + * bm_fill_super() is called exactly once whenever a binfmt_misc + * superblock for a userns is created. This in turn lets us conclude + * that when a binfmt_misc superblock is created for the first time for + * a userns there's no one racing us. Therefore we don't need any + * barriers when we dereference binfmt_misc. + */ + misc = user_ns->binfmt_misc; + if (!misc) { + /* + * If it turns out that most user namespaces actually want to + * register their own binary type handler and therefore all + * create their own separate binfmt_misc mounts we should + * consider turning this into a kmem cache. + */ + misc = kzalloc_obj(struct binfmt_misc); + if (!misc) + return -ENOMEM; + + INIT_HLIST_HEAD(&misc->entries); + spin_lock_init(&misc->entries_lock); + + /* Pairs with smp_load_acquire() in current_binfmt_misc(). */ + smp_store_release(&user_ns->binfmt_misc, misc); + } + + /* + * When the binfmt_misc superblock for this userns is shutdown + * ->enabled might have been set to false and we don't reinitialize + * ->enabled again during shutdown as someone might already be mounting + * binfmt_misc again. It also would be pointless since by then we know + * that the binary type list for this binfmt_misc mount is empty making + * load_misc_binary() return -ENOEXEC independent of whether ->enabled + * is true. Instead, if someone mounts binfmt_misc for the first time or + * again we simply reset ->enabled to true. + */ + WRITE_ONCE(misc->enabled, true); + + err = simple_fill_super(sb, BINFMTFS_MAGIC, bm_files); + if (err) + return err; + + sb->s_op = &bm_super_ops; + d_inode(sb->s_root)->i_op = &bm_dir_inode_operations; + return 0; +} + +static void bm_free(struct fs_context *fc) +{ + if (fc->s_fs_info) + put_user_ns(fc->s_fs_info); +} + +static int bm_get_tree(struct fs_context *fc) +{ + return get_tree_keyed(fc, bm_fill_super, get_user_ns(fc->user_ns)); +} + +static const struct fs_context_operations bm_context_ops = { + .free = bm_free, + .get_tree = bm_get_tree, +}; + +static void bm_kill_sb(struct super_block *sb) +{ + struct user_namespace *user_ns = sb->s_fs_info; + + kill_anon_super(sb); + put_user_ns(user_ns); +} + +static int bm_init_fs_context(struct fs_context *fc) +{ + fc->ops = &bm_context_ops; + return 0; +} + +static struct linux_binfmt misc_format = { + .module = THIS_MODULE, + .load_binary = load_misc_binary, +}; + +static struct file_system_type bm_fs_type = { + .owner = THIS_MODULE, + .name = "binfmt_misc", + .init_fs_context = bm_init_fs_context, + .fs_flags = FS_USERNS_MOUNT, + .kill_sb = bm_kill_sb, +}; +MODULE_ALIAS_FS("binfmt_misc"); + +static int __init init_misc_binfmt(void) +{ + int err = register_filesystem(&bm_fs_type); + if (!err) + insert_binfmt(&misc_format); + return err; +} + +static void __exit exit_misc_binfmt(void) +{ + unregister_binfmt(&misc_format); + unregister_filesystem(&bm_fs_type); + /* Flush pending bm_entry_free_rcu() callbacks before the text goes. */ + srcu_barrier(&bm_entries_srcu); +} + +core_initcall(init_misc_binfmt); +module_exit(exit_misc_binfmt); +MODULE_DESCRIPTION("Kernel support for miscellaneous binaries"); +MODULE_LICENSE("GPL"); diff --git a/fs/binfmt_misc_bpf.c b/fs/binfmt_misc_bpf.c new file mode 100644 index 000000000..91576ff05 --- /dev/null +++ b/fs/binfmt_misc_bpf.c @@ -0,0 +1,434 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * BPF-backed binary type handlers for binfmt_misc. + * + * A handler is a struct binfmt_misc_ops struct_ops map. Loading and + * registering it makes the handler available under its name in the user + * namespace it was registered in. A binfmt_misc 'B' entry activates it: + * + * echo ':entry:B:::::' > /register + * + * The entry can bind the interpreters the handler may run its binaries + * with, each opened by the write that binds it and selected by name per + * exec. An entry registered with 'D' is not matchable yet, which is what + * leaves it open to being given them: + * + * echo ':entry:B:::::D' > /register + * echo '+ ' > /entry + * echo 1 > /entry + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct bm_bpf_ops_reg { + struct list_head list; + const struct binfmt_misc_ops *ops; + struct bpf_link *link; + struct user_namespace *user_ns; +}; + +static DEFINE_SPINLOCK(bm_bpf_ops_lock); +static LIST_HEAD(bm_bpf_ops_list); + +static struct bpf_struct_ops bpf_binfmt_misc_ops; + +static struct bm_bpf_ops_reg *bm_bpf_ops_find(const struct user_namespace *user_ns, + const char *name) +{ + struct bm_bpf_ops_reg *reg; + + lockdep_assert_held(&bm_bpf_ops_lock); + + list_for_each_entry(reg, &bm_bpf_ops_list, list) { + if (reg->user_ns == user_ns && !strcmp(reg->ops->name, name)) + return reg; + } + return NULL; +} + +/** + * binfmt_misc_get_ops - look up a bpf binary type handler by name + * @user_ns: user namespace of the binfmt_misc instance + * @name: name the handler was registered under + * + * Look for a handler named @name registered in @user_ns. A handler is not + * inherited from ancestor user namespaces: an entry can only name a handler + * registered in the same user namespace as its instance. The returned handler + * stays callable until binfmt_misc_put_ops() even if the backing struct_ops + * map is detached or deleted in the meantime. + * + * Return: the handler on success, NULL on failure + */ +const struct binfmt_misc_ops *binfmt_misc_get_ops(struct user_namespace *user_ns, + const char *name) +{ + struct bm_bpf_ops_reg *reg; + + guard(spinlock)(&bm_bpf_ops_lock); + + reg = bm_bpf_ops_find(user_ns, name); + if (!reg) + return NULL; + if (!bpf_struct_ops_get(reg->ops)) + return NULL; + return reg->ops; +} + +void binfmt_misc_put_ops(const struct binfmt_misc_ops *ops) +{ + bpf_struct_ops_put(ops); +} + +bool bpf_prog_is_binfmt_misc_ops(const struct bpf_prog *prog) +{ + return prog->type == BPF_PROG_TYPE_STRUCT_OPS && + prog->aux->st_ops == &bpf_binfmt_misc_ops; +} + +/* + * Replace the staged interpreter selection: naming a path drops a bound + * file, selecting a bound interpreter carries its file along. + */ +static void bm_bpf_stage_selection(struct linux_binprm *bprm, char *path, + struct file *f) +{ + if (bprm->bpf_interp_file) + fput(bprm->bpf_interp_file); + kfree(bprm->bpf_interp); + bprm->bpf_interp = path; + bprm->bpf_interp_file = f; +} + +__bpf_kfunc_start_defs(); + +/** + * bpf_binprm_set_interp - select the interpreter for the current exec + * @bprm: binary that is being executed + * @path: absolute path to the interpreter + * @path__sz: size of the @path buffer, including the terminating NUL + * + * To be called from the load program of a struct binfmt_misc_ops handler + * before returning zero; the verifier rejects the call from any other + * program, including the handler's own match program. The path is opened + * with the credentials of the task doing the exec after the program + * returns. Calling it again replaces the selection, as does selecting an + * interpreter the entry bound with bpf_binprm_select_interp(). + * + * Return: 0 on success, a negative errno on failure + */ +__bpf_kfunc int bpf_binprm_set_interp(struct linux_binprm *bprm, + const char *path, size_t path__sz) +{ + size_t len; + char *interp; + + if (!path__sz) + return -EINVAL; + len = strnlen(path, path__sz); + if (len == path__sz) + return -EINVAL; + if (path[0] != '/') + return -EINVAL; + if (len >= PATH_MAX) + return -ENAMETOOLONG; + + interp = kmemdup_nul(path, len, GFP_KERNEL); + if (!interp) + return -ENOMEM; + + bm_bpf_stage_selection(bprm, interp, NULL); + return 0; +} + +/** + * bpf_binprm_select_interp - run this exec under an interpreter the entry bound + * @bprm: binary that is being executed + * @name: name the interpreter was registered under + * @name__sz: size of the @name buffer, including the terminating NUL + * + * To be called from the load program of a struct binfmt_misc_ops handler + * instead of bpf_binprm_set_interp(). It selects one of the interpreters + * the matched entry was registered with, each of which was opened once when + * the entry was registered. Nothing is resolved at exec time, so no + * filesystem view can redirect the interpreter. + * + * The interpreter runs under the path the entry registered it under. + * Calling it again replaces the selection. + * + * Return: 0 on success, -ENOENT if the matched entry bound no interpreter + * of that name, a negative errno on failure + */ +__bpf_kfunc int bpf_binprm_select_interp(struct linux_binprm *bprm, + const char *name, size_t name__sz) +{ + const struct binfmt_misc_interp *interp; + size_t len; + char *path; + + if (!name__sz) + return -EINVAL; + len = strnlen(name, name__sz); + if (len == name__sz || !len) + return -EINVAL; + + interp = binfmt_misc_find_interp(bprm->bpf_interps, name); + if (!interp) + return -ENOENT; + + path = kstrdup(interp->path, GFP_KERNEL); + if (!path) + return -ENOMEM; + + bm_bpf_stage_selection(bprm, path, get_file(interp->file)); + return 0; +} + +/** + * bpf_binprm_set_interp_arg - set a single argument for the interpreter + * @bprm: binary that is being executed + * @arg: argument to pass to the interpreter + * @arg__sz: size of the @arg buffer, including the terminating NUL + * + * To be called from the load program of a struct binfmt_misc_ops handler. The + * argument is passed to the interpreter ahead of the binary, mirroring the + * single optional argument of a #! interpreter line. Calling it again + * replaces the argument. + * + * Return: 0 on success, a negative errno on failure + */ +__bpf_kfunc int bpf_binprm_set_interp_arg(struct linux_binprm *bprm, + const char *arg, size_t arg__sz) +{ + size_t len; + char *val; + + if (!arg__sz) + return -EINVAL; + len = strnlen(arg, arg__sz); + if (len == arg__sz) + return -EINVAL; + if (!len) + return -EINVAL; + + val = kmemdup_nul(arg, len, GFP_KERNEL); + if (!val) + return -ENOMEM; + + kfree(bprm->bpf_interp_arg); + bprm->bpf_interp_arg = val; + return 0; +} + +/** + * bpf_binprm_set_flags - choose the interpreter invocation flags for this exec + * @bprm: binary that is being executed + * @flags: an OR of enum bpf_binprm_flags values + * + * To be called from the load program of a struct binfmt_misc_ops handler. It + * decides per exec what a static entry fixes at registration with the P, C, + * O, T and L flags: BPF_BINPRM_PRESERVE_ARGV0 keeps the caller's argv[0], + * BPF_BINPRM_CREDENTIALS computes credentials from the binary, and + * BPF_BINPRM_EXECFD hands the binary to the interpreter through AT_EXECFD. + * BPF_BINPRM_TRANSPARENT additionally leaves the argument vector untouched, + * making the exec look like a direct execution of the binary. + * BPF_BINPRM_LOADER substitutes the interpreter for the binary's PT_INTERP + * and runs the binary as a native exec; it excludes every other flag. + * Calling it again replaces the flags, passing zero clears them again. + * + * Return: 0 on success, -EINVAL if @flags contains an unknown bit or an + * invalid combination + */ +__bpf_kfunc int bpf_binprm_set_flags(struct linux_binprm *bprm, + enum bpf_binprm_flags flags) +{ + if (flags & ~(BPF_BINPRM_PRESERVE_ARGV0 | BPF_BINPRM_CREDENTIALS | + BPF_BINPRM_EXECFD | BPF_BINPRM_TRANSPARENT | + BPF_BINPRM_LOADER)) + return -EINVAL; + + /* Loader substitution is a native exec: no splice, execfd or creds work. */ + if ((flags & BPF_BINPRM_LOADER) && (flags & ~BPF_BINPRM_LOADER)) + return -EINVAL; + + /* Transparency preserves the whole argv, argv[0] included. */ + if ((flags & BPF_BINPRM_TRANSPARENT) && (flags & BPF_BINPRM_PRESERVE_ARGV0)) + return -EINVAL; + + bprm->bpf_flags = flags; + return 0; +} + +__bpf_kfunc_end_defs(); + +BTF_KFUNCS_START(bm_bpf_kfunc_ids) +BTF_ID_FLAGS(func, bpf_binprm_set_interp, KF_SLEEPABLE) +BTF_ID_FLAGS(func, bpf_binprm_select_interp, KF_SLEEPABLE) +BTF_ID_FLAGS(func, bpf_binprm_set_interp_arg, KF_SLEEPABLE) +BTF_ID_FLAGS(func, bpf_binprm_set_flags, KF_SLEEPABLE) +BTF_KFUNCS_END(bm_bpf_kfunc_ids) + +static int bm_bpf_kfunc_filter(const struct bpf_prog *prog, u32 kfunc_id) +{ + if (!btf_id_set8_contains(&bm_bpf_kfunc_ids, kfunc_id)) + return 0; + if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) + return -EACCES; + /* ->st_ops is unset during the cfg pass; enforced once it is set. */ + if (!prog->aux->st_ops) + return 0; + /* Only the load program decides how a binary is run. */ + if (bpf_prog_is_binfmt_misc_ops(prog) && + prog->aux->attach_st_ops_member_off == offsetof(struct binfmt_misc_ops, load)) + return 0; + return -EACCES; +} + +static const struct btf_kfunc_id_set bm_bpf_kfunc_set = { + .owner = THIS_MODULE, + .set = &bm_bpf_kfunc_ids, + .filter = bm_bpf_kfunc_filter, +}; + +static bool bm_bpf_ops__match(struct linux_binprm *bprm) +{ + return false; +} + +static int bm_bpf_ops__load(struct linux_binprm *bprm) +{ + return 0; +} + +static struct binfmt_misc_ops bm_bpf_ops_stubs = { + .match = bm_bpf_ops__match, + .load = bm_bpf_ops__load, +}; + +static int bm_bpf_init(struct btf *btf) +{ + return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, + &bm_bpf_kfunc_set); +} + +static int bm_bpf_check_member(const struct btf_type *t, + const struct btf_member *member, + const struct bpf_prog *prog) +{ + u32 moff = __btf_member_bit_offset(t, member) / 8; + + switch (moff) { + case offsetof(struct binfmt_misc_ops, match): + case offsetof(struct binfmt_misc_ops, load): + /* Reliable file reads at exec time require sleeping. */ + if (!prog->sleepable) + return -EINVAL; + break; + } + return 0; +} + +static int bm_bpf_init_member(const struct btf_type *t, + const struct btf_member *member, + void *kdata, const void *udata) +{ + const struct binfmt_misc_ops *uops = udata; + struct binfmt_misc_ops *ops = kdata; + u32 moff = __btf_member_bit_offset(t, member) / 8; + + switch (moff) { + case offsetof(struct binfmt_misc_ops, name): + if (bpf_obj_name_cpy(ops->name, uops->name, + sizeof(ops->name)) <= 0) + return -EINVAL; + return 1; + } + return 0; +} + +static int bm_bpf_validate(void *kdata) +{ + struct binfmt_misc_ops *ops = kdata; + + if (!ops->match || !ops->load) + return -EINVAL; + return 0; +} + +static int bm_bpf_reg(void *kdata, struct bpf_link *link) +{ + struct binfmt_misc_ops *ops = kdata; + struct bm_bpf_ops_reg *reg; + + reg = kzalloc_obj(*reg, GFP_KERNEL_ACCOUNT); + if (!reg) + return -ENOMEM; + + reg->ops = ops; + reg->link = link; + reg->user_ns = get_user_ns(current_user_ns()); + + guard(spinlock)(&bm_bpf_ops_lock); + + if (bm_bpf_ops_find(reg->user_ns, ops->name)) { + put_user_ns(reg->user_ns); + kfree(reg); + return -EEXIST; + } + + list_add(®->list, &bm_bpf_ops_list); + return 0; +} + +static void bm_bpf_unreg(void *kdata, struct bpf_link *link) +{ + struct bm_bpf_ops_reg *reg; + + guard(spinlock)(&bm_bpf_ops_lock); + + list_for_each_entry(reg, &bm_bpf_ops_list, list) { + if (reg->ops == kdata && reg->link == link) { + list_del(®->list); + put_user_ns(reg->user_ns); + kfree(reg); + return; + } + } +} + +static const struct bpf_verifier_ops bm_bpf_verifier_ops = { + .get_func_proto = bpf_base_func_proto, + .is_valid_access = bpf_tracing_btf_ctx_access, +}; + +static struct bpf_struct_ops bpf_binfmt_misc_ops = { + .verifier_ops = &bm_bpf_verifier_ops, + .init = bm_bpf_init, + .check_member = bm_bpf_check_member, + .init_member = bm_bpf_init_member, + .validate = bm_bpf_validate, + .reg = bm_bpf_reg, + .unreg = bm_bpf_unreg, + .cfi_stubs = &bm_bpf_ops_stubs, + .name = "binfmt_misc_ops", + .owner = THIS_MODULE, +}; + +static int __init bm_bpf_struct_ops_init(void) +{ + return register_bpf_struct_ops(&bpf_binfmt_misc_ops, binfmt_misc_ops); +} +late_initcall(bm_bpf_struct_ops_init); diff --git a/fs/binfmt_script.c b/fs/binfmt_script.c new file mode 100644 index 000000000..637daf6e4 --- /dev/null +++ b/fs/binfmt_script.c @@ -0,0 +1,159 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/binfmt_script.c + * + * Copyright (C) 1996 Martin von Löwis + * original #!-checking implemented by tytso. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +static inline bool spacetab(char c) { return c == ' ' || c == '\t'; } +static inline const char *next_non_spacetab(const char *first, const char *last) +{ + for (; first <= last; first++) + if (!spacetab(*first)) + return first; + return NULL; +} +static inline const char *next_terminator(const char *first, const char *last) +{ + for (; first <= last; first++) + if (spacetab(*first) || !*first) + return first; + return NULL; +} + +static int load_script(struct linux_binprm *bprm) +{ + const char *i_name, *i_sep, *i_arg, *i_end, *buf_end; + struct file *file; + int retval; + + /* Not ours to exec if we don't start with "#!". */ + if ((bprm->buf[0] != '#') || (bprm->buf[1] != '!')) + return -ENOEXEC; + + /* + * This section handles parsing the #! line into separate + * interpreter path and argument strings. We must be careful + * because bprm->buf is not yet guaranteed to be NUL-terminated + * (though the buffer will have trailing NUL padding when the + * file size was smaller than the buffer size). + * + * We do not want to exec a truncated interpreter path, so either + * we find a newline (which indicates nothing is truncated), or + * we find a space/tab/NUL after the interpreter path (which + * itself may be preceded by spaces/tabs). Truncating the + * arguments is fine: the interpreter can re-read the script to + * parse them on its own. + */ + buf_end = bprm->buf + sizeof(bprm->buf) - 1; + i_end = strnchr(bprm->buf, sizeof(bprm->buf), '\n'); + if (!i_end) { + i_end = next_non_spacetab(bprm->buf + 2, buf_end); + if (!i_end) + return -ENOEXEC; /* Entire buf is spaces/tabs */ + /* + * If there is no later space/tab/NUL we must assume the + * interpreter path is truncated. + */ + if (!next_terminator(i_end, buf_end)) + return -ENOEXEC; + i_end = buf_end; + } + /* Trim any trailing spaces/tabs from i_end */ + while (spacetab(i_end[-1])) + i_end--; + + /* Skip over leading spaces/tabs */ + i_name = next_non_spacetab(bprm->buf+2, i_end); + if (!i_name || (i_name == i_end)) + return -ENOEXEC; /* No interpreter name found */ + + /* Is there an optional argument? */ + i_arg = NULL; + i_sep = next_terminator(i_name, i_end); + if (i_sep && (*i_sep != '\0')) + i_arg = next_non_spacetab(i_sep, i_end); + + /* + * If the script filename will be inaccessible after exec, typically + * because it is a "/dev/fd//.." path against an O_CLOEXEC fd, give + * up now (on the assumption that the interpreter will want to load + * this file). + */ + if (bprm->interp_flags & BINPRM_FLAGS_PATH_INACCESSIBLE) + return -ENOENT; + + /* + * OK, we've parsed out the interpreter name and + * (optional) argument. + * Splice in (1) the interpreter's name for argv[0] + * (2) (optional) argument to interpreter + * (3) filename of shell script (replace argv[0]) + * + * This is done in reverse order, because of how the + * user environment and arguments are stored. + */ + retval = remove_arg_zero(bprm); + if (retval) + return retval; + retval = copy_string_kernel(bprm->interp, bprm); + if (retval < 0) + return retval; + bprm->argc++; + *((char *)i_end) = '\0'; + if (i_arg) { + *((char *)i_sep) = '\0'; + retval = copy_string_kernel(i_arg, bprm); + if (retval < 0) + return retval; + bprm->argc++; + } + retval = copy_string_kernel(i_name, bprm); + if (retval) + return retval; + bprm->argc++; + retval = bprm_change_interp(i_name, bprm); + if (retval < 0) + return retval; + + /* + * OK, now restart the process with the interpreter's dentry. + */ + file = open_exec(i_name); + if (IS_ERR(file)) + return PTR_ERR(file); + + bprm->interpreter = file; + return 0; +} + +static struct linux_binfmt script_format = { + .module = THIS_MODULE, + .load_binary = load_script, +}; + +static int __init init_script_binfmt(void) +{ + register_binfmt(&script_format); + return 0; +} + +static void __exit exit_script_binfmt(void) +{ + unregister_binfmt(&script_format); +} + +core_initcall(init_script_binfmt); +module_exit(exit_script_binfmt); +MODULE_DESCRIPTION("Kernel support for scripts starting with #!"); +MODULE_LICENSE("GPL"); diff --git a/fs/bpf_fs_kfuncs.c b/fs/bpf_fs_kfuncs.c new file mode 100644 index 000000000..6cb877267 --- /dev/null +++ b/fs/bpf_fs_kfuncs.c @@ -0,0 +1,503 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2024 Google LLC. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +__bpf_kfunc_start_defs(); + +/** + * bpf_get_task_exe_file - get a reference on the exe_file struct file member of + * the mm_struct that is nested within the supplied + * task_struct + * @task: task_struct of which the nested mm_struct exe_file member to get a + * reference on + * + * Get a reference on the exe_file struct file member field of the mm_struct + * nested within the supplied *task*. The referenced file pointer acquired by + * this BPF kfunc must be released using bpf_put_file(). Failing to call + * bpf_put_file() on the returned referenced struct file pointer that has been + * acquired by this BPF kfunc will result in the BPF program being rejected by + * the BPF verifier. + * + * This BPF kfunc may only be called from BPF LSM programs. + * + * Internally, this BPF kfunc leans on get_task_exe_file(), such that calling + * bpf_get_task_exe_file() would be analogous to calling get_task_exe_file() + * directly in kernel context. + * + * Return: A referenced struct file pointer to the exe_file member of the + * mm_struct that is nested within the supplied *task*. On error, NULL is + * returned. + */ +__bpf_kfunc struct file *bpf_get_task_exe_file(struct task_struct *task) +{ + return get_task_exe_file(task); +} + +/** + * bpf_put_file - put a reference on the supplied file + * @file: file to put a reference on + * + * Put a reference on the supplied *file*. Only referenced file pointers may be + * passed to this BPF kfunc. Attempting to pass an unreferenced file pointer, or + * any other arbitrary pointer for that matter, will result in the BPF program + * being rejected by the BPF verifier. + * + * This BPF kfunc may only be called from BPF LSM programs. + */ +__bpf_kfunc void bpf_put_file(struct file *file) +{ + fput(file); +} + +/** + * bpf_path_d_path - resolve the pathname for the supplied path + * @path: path to resolve the pathname for + * @buf: buffer to return the resolved pathname in + * @buf__sz: length of the supplied buffer + * + * Resolve the pathname for the supplied *path* and store it in *buf*. This BPF + * kfunc is the safer variant of the legacy bpf_d_path() helper and should be + * used in place of bpf_d_path() whenever possible. + * + * This BPF kfunc may only be called from BPF LSM programs. + * + * Return: A positive integer corresponding to the length of the resolved + * pathname in *buf*, including the NUL termination character. On error, a + * negative integer is returned. + */ +__bpf_kfunc int bpf_path_d_path(const struct path *path, char *buf, size_t buf__sz) +{ + int len; + char *ret; + + if (!buf__sz) + return -EINVAL; + + ret = d_path(path, buf, buf__sz); + if (IS_ERR(ret)) + return PTR_ERR(ret); + + len = buf + buf__sz - ret; + memmove(buf, ret, len); + return len; +} + +static bool match_security_bpf_prefix(const char *name__str) +{ + return !strncmp(name__str, XATTR_NAME_BPF_LSM, XATTR_NAME_BPF_LSM_LEN); +} + +static int bpf_xattr_read_permission(const char *name, struct inode *inode) +{ + if (!inode) + return -EINVAL; + + /* Allow reading xattr with user. and security.bpf. prefix */ + if (strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN) && + !match_security_bpf_prefix(name)) + return -EPERM; + + return inode_permission(&nop_mnt_idmap, inode, MAY_READ); +} + +/** + * bpf_get_dentry_xattr - get xattr of a dentry + * @dentry: dentry to get xattr from + * @name__str: name of the xattr + * @value_p: output buffer of the xattr value + * + * Get xattr *name__str* of *dentry* and store the output in *value_ptr*. + * + * For security reasons, only *name__str* with prefixes "user." or + * "security.bpf." are allowed. + * + * Return: length of the xattr value on success, a negative value on error. + */ +__bpf_kfunc int bpf_get_dentry_xattr(struct dentry *dentry, const char *name__str, + struct bpf_dynptr *value_p) +{ + struct bpf_dynptr_kern *value_ptr = (struct bpf_dynptr_kern *)value_p; + struct inode *inode = d_inode(dentry); + u32 value_len; + void *value; + int ret; + + value_len = __bpf_dynptr_size(value_ptr); + value = __bpf_dynptr_data_rw(value_ptr, value_len); + if (!value) + return -EINVAL; + + ret = bpf_xattr_read_permission(name__str, inode); + if (ret) + return ret; + return __vfs_getxattr(dentry, inode, name__str, value, value_len); +} + +/** + * bpf_get_file_xattr - get xattr of a file + * @file: file to get xattr from + * @name__str: name of the xattr + * @value_p: output buffer of the xattr value + * + * Get xattr *name__str* of *file* and store the output in *value_ptr*. + * + * For security reasons, only *name__str* with prefixes "user." or + * "security.bpf." are allowed. + * + * Return: length of the xattr value on success, a negative value on error. + */ +__bpf_kfunc int bpf_get_file_xattr(struct file *file, const char *name__str, + struct bpf_dynptr *value_p) +{ + struct dentry *dentry; + + dentry = file_dentry(file); + return bpf_get_dentry_xattr(dentry, name__str, value_p); +} + +__bpf_kfunc_end_defs(); + +static int bpf_xattr_write_permission(const char *name, struct inode *inode) +{ + if (!inode) + return -EINVAL; + + /* Only allow setting and removing security.bpf. xattrs */ + if (!match_security_bpf_prefix(name)) + return -EPERM; + + return inode_permission(&nop_mnt_idmap, inode, MAY_WRITE); +} + +/** + * bpf_set_dentry_xattr_locked - set a xattr of a dentry + * @dentry: dentry to get xattr from + * @name__str: name of the xattr + * @value_p: xattr value + * @flags: flags to pass into filesystem operations + * + * Set xattr *name__str* of *dentry* to the value in *value_ptr*. + * + * For security reasons, only *name__str* with prefix "security.bpf." + * is allowed. + * + * The caller already locked dentry->d_inode. + * + * Return: 0 on success, a negative value on error. + */ +int bpf_set_dentry_xattr_locked(struct dentry *dentry, const char *name__str, + const struct bpf_dynptr *value_p, int flags) +{ + + const struct bpf_dynptr_kern *value_ptr = (struct bpf_dynptr_kern *)value_p; + struct inode *inode = d_inode(dentry); + const void *value; + u32 value_len; + int ret; + + value_len = __bpf_dynptr_size(value_ptr); + value = __bpf_dynptr_data(value_ptr, value_len); + if (!value) + return -EINVAL; + + ret = bpf_xattr_write_permission(name__str, inode); + if (ret) + return ret; + + ret = __vfs_setxattr(&nop_mnt_idmap, dentry, inode, name__str, + value, value_len, flags); + if (!ret) { + fsnotify_xattr(dentry); + + /* This xattr is set by BPF LSM, so we do not call + * security_inode_post_setxattr. Otherwise, we would + * risk deadlocks by calling back to the same kfunc. + * + * This is the same as security_inode_setsecurity(). + */ + } + return ret; +} + +/** + * bpf_remove_dentry_xattr_locked - remove a xattr of a dentry + * @dentry: dentry to get xattr from + * @name__str: name of the xattr + * + * Rmove xattr *name__str* of *dentry*. + * + * For security reasons, only *name__str* with prefix "security.bpf." + * is allowed. + * + * The caller already locked dentry->d_inode. + * + * Return: 0 on success, a negative value on error. + */ +int bpf_remove_dentry_xattr_locked(struct dentry *dentry, const char *name__str) +{ + struct inode *inode = d_inode(dentry); + int ret; + + ret = bpf_xattr_write_permission(name__str, inode); + if (ret) + return ret; + + ret = __vfs_removexattr(&nop_mnt_idmap, dentry, name__str); + if (!ret) { + fsnotify_xattr(dentry); + + /* This xattr is removed by BPF LSM, so we do not call + * security_inode_post_removexattr. Otherwise, we would + * risk deadlocks by calling back to the same kfunc. + */ + } + return ret; +} + +__bpf_kfunc_start_defs(); + +/** + * bpf_set_dentry_xattr - set a xattr of a dentry + * @dentry: dentry to get xattr from + * @name__str: name of the xattr + * @value_p: xattr value + * @flags: flags to pass into filesystem operations + * + * Set xattr *name__str* of *dentry* to the value in *value_ptr*. + * + * For security reasons, only *name__str* with prefix "security.bpf." + * is allowed. + * + * The caller has not locked dentry->d_inode. + * + * Return: 0 on success, a negative value on error. + */ +__bpf_kfunc int bpf_set_dentry_xattr(struct dentry *dentry, const char *name__str, + const struct bpf_dynptr *value_p, int flags) +{ + struct inode *inode = d_inode(dentry); + int ret; + + if (!inode) + return -EINVAL; + + inode_lock(inode); + ret = bpf_set_dentry_xattr_locked(dentry, name__str, value_p, flags); + inode_unlock(inode); + return ret; +} + +/** + * bpf_remove_dentry_xattr - remove a xattr of a dentry + * @dentry: dentry to get xattr from + * @name__str: name of the xattr + * + * Rmove xattr *name__str* of *dentry*. + * + * For security reasons, only *name__str* with prefix "security.bpf." + * is allowed. + * + * The caller has not locked dentry->d_inode. + * + * Return: 0 on success, a negative value on error. + */ +__bpf_kfunc int bpf_remove_dentry_xattr(struct dentry *dentry, const char *name__str) +{ + struct inode *inode = d_inode(dentry); + int ret; + + if (!inode) + return -EINVAL; + + inode_lock(inode); + ret = bpf_remove_dentry_xattr_locked(dentry, name__str); + inode_unlock(inode); + return ret; +} + +#ifdef CONFIG_CGROUPS +/** + * bpf_cgroup_read_xattr - read xattr of a cgroup's node in cgroupfs + * @cgroup: cgroup to get xattr from + * @name__str: name of the xattr + * @value_p: output buffer of the xattr value + * + * Get xattr *name__str* of *cgroup* and store the output in *value_ptr*. + * + * For security reasons, only *name__str* with prefix "user." is allowed. + * + * Return: length of the xattr value on success, a negative value on error. + */ +__bpf_kfunc int bpf_cgroup_read_xattr(struct cgroup *cgroup, const char *name__str, + struct bpf_dynptr *value_p) +{ + struct bpf_dynptr_kern *value_ptr = (struct bpf_dynptr_kern *)value_p; + u32 value_len; + void *value; + + /* Only allow reading "user.*" xattrs */ + if (strncmp(name__str, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN)) + return -EPERM; + + value_len = __bpf_dynptr_size(value_ptr); + value = __bpf_dynptr_data_rw(value_ptr, value_len); + if (!value) + return -EINVAL; + + return kernfs_xattr_get(cgroup->kn, name__str, value, value_len); +} +#endif /* CONFIG_CGROUPS */ + +#ifdef CONFIG_NET +/** + * bpf_sock_read_xattr - read xattr of a socket's inode in sockfs + * @sock: socket to get xattr from + * @name__str: name of the xattr + * @value_p: output buffer of the xattr value + * + * Get xattr *name__str* of *sock* and store the output in *value_p*. + * + * For security reasons, only *name__str* with prefix "user." is allowed. + * + * Return: length of the xattr value on success, a negative value on error. + */ +__bpf_kfunc int bpf_sock_read_xattr(struct socket *sock, const char *name__str, + struct bpf_dynptr *value_p) +{ + struct bpf_dynptr_kern *value_ptr = (struct bpf_dynptr_kern *)value_p; + u32 value_len; + void *value; + + /* Only allow reading "user.*" xattrs */ + if (strncmp(name__str, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN)) + return -EPERM; + + value_len = __bpf_dynptr_size(value_ptr); + value = __bpf_dynptr_data_rw(value_ptr, value_len); + if (!value) + return -EINVAL; + + return sock_read_xattr(sock, name__str, value, value_len); +} +#endif /* CONFIG_NET */ + +/** + * bpf_real_data_inode - get the real inode hosting a file's data + * @file: file to resolve + * + * Resolve @file to the inode that hosts its data. For a regular file on a + * union/overlay filesystem this is the underlying (upper or lower) inode that + * stores the data, not the overlay inode. + * + * Data resolution only applies to regular files. For a non-regular file (e.g. + * a device node, fifo or socket) on a union/overlay filesystem the overlay + * inode itself is returned; for any file on a non-union filesystem the inode + * attached to @file is returned. + * + * Return: The inode hosting @file's data, or NULL. + */ +__bpf_kfunc struct inode *bpf_real_data_inode(struct file *file) +{ + return d_real_inode(file_dentry(file)); +} + +__bpf_kfunc_end_defs(); + +BTF_KFUNCS_START(bpf_fs_kfunc_set_ids) +BTF_ID_FLAGS(func, bpf_get_task_exe_file, KF_ACQUIRE | KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_put_file, KF_RELEASE) +BTF_ID_FLAGS(func, bpf_path_d_path) +BTF_ID_FLAGS(func, bpf_get_dentry_xattr, KF_SLEEPABLE) +BTF_ID_FLAGS(func, bpf_get_file_xattr, KF_SLEEPABLE) +BTF_ID_FLAGS(func, bpf_set_dentry_xattr, KF_SLEEPABLE) +BTF_ID_FLAGS(func, bpf_remove_dentry_xattr, KF_SLEEPABLE) +BTF_ID_FLAGS(func, bpf_real_data_inode, KF_SLEEPABLE | KF_RET_NULL) +#ifdef CONFIG_NET +BTF_ID_FLAGS(func, bpf_sock_read_xattr, KF_RCU) +#endif +BTF_KFUNCS_END(bpf_fs_kfunc_set_ids) + +/* Side-effecting kfuncs that stay exclusive to LSM programs. */ +BTF_SET_START(bpf_fs_kfunc_lsm_only_ids) +BTF_ID(func, bpf_set_dentry_xattr) +BTF_ID(func, bpf_remove_dentry_xattr) +BTF_SET_END(bpf_fs_kfunc_lsm_only_ids) + +static int bpf_fs_kfuncs_filter(const struct bpf_prog *prog, u32 kfunc_id) +{ + if (!btf_id_set8_contains(&bpf_fs_kfunc_set_ids, kfunc_id)) + return 0; + if (prog->type == BPF_PROG_TYPE_LSM) + return 0; + if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) + return -EACCES; + /* ->st_ops is unset during the cfg pass; enforced once it is set. */ + if (!prog->aux->st_ops) + return 0; + if (bpf_prog_is_binfmt_misc_ops(prog) && + !btf_id_set_contains(&bpf_fs_kfunc_lsm_only_ids, kfunc_id)) + return 0; + return -EACCES; +} + +/* bpf_[set|remove]_dentry_xattr.* hooks have KF_SLEEPABLE, so they are only + * available to sleepable hooks with dentry arguments. + * + * Setting and removing xattr requires exclusive lock on dentry->d_inode. + * Some hooks already locked d_inode, while some hooks have not locked + * d_inode. Therefore, we need different kfuncs for different hooks. + * Specifically, hooks in the following list (d_inode_locked_hooks) + * should call bpf_[set|remove]_dentry_xattr_locked; while other hooks + * should call bpf_[set|remove]_dentry_xattr. + */ +BTF_SET_START(d_inode_locked_hooks) +BTF_ID(func, bpf_lsm_inode_post_removexattr) +BTF_ID(func, bpf_lsm_inode_post_setattr) +BTF_ID(func, bpf_lsm_inode_post_setxattr) +BTF_ID(func, bpf_lsm_inode_removexattr) +BTF_ID(func, bpf_lsm_inode_rmdir) +BTF_ID(func, bpf_lsm_inode_setattr) +BTF_ID(func, bpf_lsm_inode_setxattr) +BTF_ID(func, bpf_lsm_inode_unlink) +#ifdef CONFIG_SECURITY_PATH +BTF_ID(func, bpf_lsm_path_unlink) +BTF_ID(func, bpf_lsm_path_rmdir) +#endif /* CONFIG_SECURITY_PATH */ +BTF_SET_END(d_inode_locked_hooks) + +bool bpf_lsm_has_d_inode_locked(const struct bpf_prog *prog) +{ + return btf_id_set_contains(&d_inode_locked_hooks, prog->aux->attach_btf_id); +} + +static const struct btf_kfunc_id_set bpf_fs_kfunc_set = { + .owner = THIS_MODULE, + .set = &bpf_fs_kfunc_set_ids, + .filter = bpf_fs_kfuncs_filter, +}; + +static int __init bpf_fs_kfuncs_init(void) +{ + int ret; + + ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_LSM, &bpf_fs_kfunc_set); + if (ret || !IS_ENABLED(CONFIG_BINFMT_MISC_BPF)) + return ret; + return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, + &bpf_fs_kfunc_set); +} + +late_initcall(bpf_fs_kfuncs_init); diff --git a/fs/btrfs/Kconfig b/fs/btrfs/Kconfig new file mode 100644 index 000000000..4b10d78ed --- /dev/null +++ b/fs/btrfs/Kconfig @@ -0,0 +1,118 @@ +# SPDX-License-Identifier: GPL-2.0 + +config BTRFS_FS + tristate "Btrfs filesystem support" + select BLK_CGROUP_PUNT_BIO + select CRC32 + select CRYPTO_LIB_BLAKE2B + select CRYPTO_LIB_SHA256 + select ZLIB_INFLATE + select ZLIB_DEFLATE + select LZO_COMPRESS + select LZO_DECOMPRESS + select ZSTD_COMPRESS + select ZSTD_DECOMPRESS + select FS_IOMAP + select RAID6_PQ + select XOR_BLOCKS + select XXHASH + depends on PAGE_SIZE_LESS_THAN_256KB + + help + Btrfs is a general purpose copy-on-write filesystem with extents, + writable snapshotting, support for multiple devices and many more + features focused on fault tolerance, repair and easy administration. + + The filesystem disk format is no longer unstable, and it's not + expected to change unless there are strong reasons to do so. If there + is a format change, file systems with a unchanged format will + continue to be mountable and usable by newer kernels. + + For more information, please see the web pages at + https://btrfs.readthedocs.io + + To compile this file system support as a module, choose M here. The + module will be called btrfs. + + If unsure, say N. + +config BTRFS_FS_POSIX_ACL + bool "Btrfs POSIX Access Control Lists" + depends on BTRFS_FS + select FS_POSIX_ACL + help + POSIX Access Control Lists (ACLs) support permissions for users and + groups beyond the owner/group/world scheme. + + If you don't know what Access Control Lists are, say N + +config BTRFS_FS_RUN_SANITY_TESTS + bool "Btrfs will run sanity tests upon loading" + depends on BTRFS_FS + help + This will run sanity tests for core functionality like free space, + extent maps, extent io, extent buffers, inodes, qgroups and others, + at module load time. These are mostly regression tests and are only + interesting to developers. + + If unsure, say N. + +config BTRFS_DEBUG + bool "Btrfs debugging support" + depends on BTRFS_FS + select REF_TRACKER if STACKTRACE_SUPPORT + help + Enable run-time debugging support for the btrfs filesystem. + + Additional potentially expensive checks, debugging functionality or + sysfs exported information is enabled, like leak checks of internal + objects, optional forced space fragmentation and /sys/fs/btrfs/debug . + This has negative impact on performance. + + If unsure, say N. + +config BTRFS_ASSERT + bool "Btrfs assert support" + depends on BTRFS_FS + help + Enable run-time assertion checking. Additional safety checks are + done, simple enough not to affect performance but verify invariants + and assumptions of code to run properly. This may result in panics, + and is meant for developers but can be enabled in general. + + If unsure, say N. + +config BTRFS_EXPERIMENTAL + bool "Btrfs experimental features" + depends on BTRFS_FS + default n + help + Enable experimental features. These features may not be stable enough + for end users. This is meant for btrfs developers or users who wish + to test the functionality and report problems. + + Current list: + + - RAID mirror read policy - additional read policies for balancing + reading from redundant block group + profiles (currently: pid, round-robin, + fixed devid) + + - send stream protocol v3 - fs-verity support + + - raid-stripe-tree - additional mapping of extents to devices to + support RAID1* profiles on zoned devices, + RAID56 not yet supported + + - extent tree v2 - complex rework of extent tracking + + - block size > page size support - needs transparent huge page and + non-HIGHMEM system + + - huge folios for data - folios can be as large as 2MiB now + + - asynchronous checksum generation for data writes + + - remap-tree - logical address remapping tree + + If unsure, say N. diff --git a/fs/btrfs/Makefile b/fs/btrfs/Makefile new file mode 100644 index 000000000..b15122aa2 --- /dev/null +++ b/fs/btrfs/Makefile @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: GPL-2.0 + +# Subset of W=1 warnings +subdir-ccflags-y += -Wextra -Wunused -Wno-unused-parameter +subdir-ccflags-y += -Wmissing-declarations +subdir-ccflags-y += -Wmissing-format-attribute +subdir-ccflags-y += -Wmissing-prototypes +subdir-ccflags-y += -Wold-style-definition +subdir-ccflags-y += -Wmissing-include-dirs +condflags := \ + $(call cc-option, -Wunused-but-set-variable) \ + $(call cc-option, -Wunused-const-variable) \ + $(call cc-option, -Wpacked-not-aligned) \ + $(call cc-option, -Wstringop-truncation) \ + $(call cc-option, -Wmaybe-uninitialized) +subdir-ccflags-y += $(condflags) +# The following turn off the warnings enabled by -Wextra +subdir-ccflags-y += -Wno-missing-field-initializers +subdir-ccflags-y += -Wno-sign-compare +subdir-ccflags-y += -Wno-shift-negative-value + +obj-$(CONFIG_BTRFS_FS) := btrfs.o + +btrfs-y += super.o ctree.o extent-tree.o print-tree.o root-tree.o dir-item.o \ + file-item.o inode-item.o disk-io.o \ + transaction.o inode.o file.o defrag.o \ + extent_map.o sysfs.o accessors.o xattr.o ordered-data.o \ + extent_io.o volumes.o async-thread.o ioctl.o locking.o orphan.o \ + export.o tree-log.o free-space-cache.o zlib.o lzo.o zstd.o \ + compression.o delayed-ref.o relocation.o delayed-inode.o scrub.o \ + backref.o ulist.o qgroup.o send.o dev-replace.o raid56.o \ + uuid-tree.o props.o free-space-tree.o tree-checker.o space-info.o \ + block-rsv.o delalloc-space.o block-group.o discard.o reflink.o \ + subpage.o tree-mod-log.o extent-io-tree.o fs.o messages.o bio.o \ + lru_cache.o raid-stripe-tree.o fiemap.o direct-io.o + +btrfs-$(CONFIG_BTRFS_FS_POSIX_ACL) += acl.o +btrfs-$(CONFIG_BTRFS_DEBUG) += ref-verify.o +btrfs-$(CONFIG_BLK_DEV_ZONED) += zoned.o +btrfs-$(CONFIG_FS_VERITY) += verity.o + +btrfs-$(CONFIG_BTRFS_FS_RUN_SANITY_TESTS) += tests/free-space-tests.o \ + tests/extent-buffer-tests.o tests/btrfs-tests.o \ + tests/extent-io-tests.o tests/inode-tests.o tests/qgroup-tests.o \ + tests/free-space-tree-tests.o tests/extent-map-tests.o \ + tests/raid-stripe-tree-tests.o tests/delayed-refs-tests.o \ + tests/chunk-allocation-tests.o + +ifeq ($(CONFIG_BLK_DEV_ZONED),y) +btrfs-$(CONFIG_BTRFS_FS_RUN_SANITY_TESTS) += tests/zoned-tests.o +endif diff --git a/fs/btrfs/accessors.c b/fs/btrfs/accessors.c new file mode 100644 index 000000000..1248aa253 --- /dev/null +++ b/fs/btrfs/accessors.c @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#include +#include "messages.h" +#include "extent_io.h" +#include "fs.h" +#include "accessors.h" + +static void __cold report_setget_bounds(const struct extent_buffer *eb, + const void *ptr, unsigned off, int size) +{ + unsigned long member_offset = (unsigned long)ptr + off; + + btrfs_warn(eb->fs_info, + "bad eb member %s: ptr 0x%lx start %llu member offset %lu size %d", + (member_offset > eb->len ? "start" : "end"), + (unsigned long)ptr, eb->start, member_offset, size); +} + +/* Copy bytes from @src1 and @src2 to @dest. */ +static __always_inline void memcpy_split_src(char *dest, const char *src1, + const char *src2, const size_t len1, + const size_t total) +{ + memcpy(dest, src1, len1); + memcpy(dest + len1, src2, total - len1); +} + +/* + * Macro templates that define helpers to read/write extent buffer data of a + * given size, that are also used via ctree.h for access to item members by + * specialized helpers. + * + * Generic helpers: + * - btrfs_set_8 (for 8/16/32/64) + * - btrfs_get_8 (for 8/16/32/64) + * + * The set/get functions handle data spanning two pages transparently, in case + * metadata block size is larger than page. Every pointer to metadata items is + * an offset into the extent buffer page array, cast to a specific type. This + * gives us all the type checking. + * + * The extent buffer pages stored in the array folios may not form a contiguous + * physical range, but the API functions assume the linear offset to the range + * from 0 to metadata node size. + */ + +#define DEFINE_BTRFS_SETGET_BITS(bits) \ +u##bits btrfs_get_##bits(const struct extent_buffer *eb, \ + const void *ptr, unsigned long off) \ +{ \ + const unsigned long member_offset = (unsigned long)ptr + off; \ + const unsigned long idx = get_eb_folio_index(eb, member_offset);\ + const unsigned long oif = get_eb_offset_in_folio(eb, \ + member_offset);\ + char *kaddr = folio_address(eb->folios[idx]) + oif; \ + const int part = eb->folio_size - oif; \ + u8 lebytes[sizeof(u##bits)]; \ + \ + if (unlikely(member_offset + sizeof(u##bits) > eb->len)) { \ + report_setget_bounds(eb, ptr, off, sizeof(u##bits)); \ + return 0; \ + } \ + if (INLINE_EXTENT_BUFFER_PAGES == 1 || sizeof(u##bits) == 1 || \ + likely(sizeof(u##bits) <= part)) \ + return get_unaligned_le##bits(kaddr); \ + \ + if (sizeof(u##bits) == 2) { \ + lebytes[0] = *kaddr; \ + kaddr = folio_address(eb->folios[idx + 1]); \ + lebytes[1] = *kaddr; \ + } else { \ + memcpy_split_src(lebytes, kaddr, \ + folio_address(eb->folios[idx + 1]), \ + part, sizeof(u##bits)); \ + } \ + return get_unaligned_le##bits(lebytes); \ +} \ +void btrfs_set_##bits(const struct extent_buffer *eb, void *ptr, \ + unsigned long off, u##bits val) \ +{ \ + const unsigned long member_offset = (unsigned long)ptr + off; \ + const unsigned long idx = get_eb_folio_index(eb, member_offset);\ + const unsigned long oif = get_eb_offset_in_folio(eb, \ + member_offset);\ + char *kaddr = folio_address(eb->folios[idx]) + oif; \ + const int part = eb->folio_size - oif; \ + u8 lebytes[sizeof(u##bits)]; \ + \ + if (unlikely(member_offset + sizeof(u##bits) > eb->len)) { \ + report_setget_bounds(eb, ptr, off, sizeof(u##bits)); \ + return; \ + } \ + if (INLINE_EXTENT_BUFFER_PAGES == 1 || sizeof(u##bits) == 1 || \ + likely(sizeof(u##bits) <= part)) { \ + put_unaligned_le##bits(val, kaddr); \ + return; \ + } \ + put_unaligned_le##bits(val, lebytes); \ + if (sizeof(u##bits) == 2) { \ + *kaddr = lebytes[0]; \ + kaddr = folio_address(eb->folios[idx + 1]); \ + *kaddr = lebytes[1]; \ + } else { \ + memcpy(kaddr, lebytes, part); \ + kaddr = folio_address(eb->folios[idx + 1]); \ + memcpy(kaddr, lebytes + part, sizeof(u##bits) - part); \ + } \ +} + +DEFINE_BTRFS_SETGET_BITS(8) +DEFINE_BTRFS_SETGET_BITS(16) +DEFINE_BTRFS_SETGET_BITS(32) +DEFINE_BTRFS_SETGET_BITS(64) + +void btrfs_node_key(const struct extent_buffer *eb, + struct btrfs_disk_key *disk_key, int nr) +{ + unsigned long ptr = btrfs_node_key_ptr_offset(eb, nr); + read_eb_member(eb, (struct btrfs_key_ptr *)ptr, + struct btrfs_key_ptr, key, disk_key); +} diff --git a/fs/btrfs/accessors.h b/fs/btrfs/accessors.h new file mode 100644 index 000000000..8938357fc --- /dev/null +++ b/fs/btrfs/accessors.h @@ -0,0 +1,1050 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_ACCESSORS_H +#define BTRFS_ACCESSORS_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fs.h" +#include "extent_io.h" + +struct extent_buffer; + +/* + * Some macros to generate set/get functions for the struct fields. This + * assumes there is a lefoo_to_cpu for every type, so lets make a simple one + * for u8: + */ +#define le8_to_cpu(v) (v) +#define cpu_to_le8(v) (v) +#define __le8 u8 + +static inline u8 get_unaligned_le8(const void *p) +{ + return *(const u8 *)p; +} + +static inline void put_unaligned_le8(u8 val, void *p) +{ + *(u8 *)p = val; +} + +#define read_eb_member(eb, ptr, type, member, result) (\ + read_extent_buffer(eb, (char *)(result), \ + ((unsigned long)(ptr)) + \ + offsetof(type, member), \ + sizeof_field(type, member))) + +#define write_eb_member(eb, ptr, type, member, source) ( \ + write_extent_buffer(eb, (const char *)(source), \ + ((unsigned long)(ptr)) + \ + offsetof(type, member), \ + sizeof_field(type, member))) + +#define DECLARE_BTRFS_SETGET_BITS(bits) \ +u##bits btrfs_get_##bits(const struct extent_buffer *eb, \ + const void *ptr, unsigned long off); \ +void btrfs_set_##bits(const struct extent_buffer *eb, void *ptr, \ + unsigned long off, u##bits val); + +DECLARE_BTRFS_SETGET_BITS(8) +DECLARE_BTRFS_SETGET_BITS(16) +DECLARE_BTRFS_SETGET_BITS(32) +DECLARE_BTRFS_SETGET_BITS(64) + +#define BTRFS_SETGET_FUNCS(name, type, member, bits) \ +static inline u##bits btrfs_##name(const struct extent_buffer *eb, \ + const type *s) \ +{ \ + static_assert(sizeof(u##bits) == sizeof_field(type, member)); \ + return btrfs_get_##bits(eb, s, offsetof(type, member)); \ +} \ +static inline void btrfs_set_##name(const struct extent_buffer *eb, type *s, \ + u##bits val) \ +{ \ + static_assert(sizeof(u##bits) == sizeof_field(type, member)); \ + btrfs_set_##bits(eb, s, offsetof(type, member), val); \ +} + +#define BTRFS_SETGET_HEADER_FUNCS(name, type, member, bits) \ +static inline u##bits btrfs_##name(const struct extent_buffer *eb) \ +{ \ + const type *p = folio_address(eb->folios[0]) + \ + offset_in_page(eb->start); \ + return get_unaligned_le##bits(&p->member); \ +} \ +static inline void btrfs_set_##name(const struct extent_buffer *eb, \ + u##bits val) \ +{ \ + type *p = folio_address(eb->folios[0]) + offset_in_page(eb->start); \ + put_unaligned_le##bits(val, &p->member); \ +} + +#define BTRFS_SETGET_STACK_FUNCS(name, type, member, bits) \ +static inline u##bits btrfs_##name(const type *s) \ +{ \ + return get_unaligned_le##bits(&s->member); \ +} \ +static inline void btrfs_set_##name(type *s, u##bits val) \ +{ \ + put_unaligned_le##bits(val, &s->member); \ +} + +static inline u64 btrfs_device_total_bytes(const struct extent_buffer *eb, + struct btrfs_dev_item *s) +{ + static_assert(sizeof(u64) == sizeof_field(struct btrfs_dev_item, total_bytes)); + return btrfs_get_64(eb, s, offsetof(struct btrfs_dev_item, total_bytes)); +} +static inline void btrfs_set_device_total_bytes(const struct extent_buffer *eb, + struct btrfs_dev_item *s, + u64 val) +{ + static_assert(sizeof(u64) == sizeof_field(struct btrfs_dev_item, total_bytes)); + WARN_ON(!IS_ALIGNED(val, eb->fs_info->sectorsize)); + btrfs_set_64(eb, s, offsetof(struct btrfs_dev_item, total_bytes), val); +} + +BTRFS_SETGET_FUNCS(device_type, struct btrfs_dev_item, type, 64); +BTRFS_SETGET_FUNCS(device_bytes_used, struct btrfs_dev_item, bytes_used, 64); +BTRFS_SETGET_FUNCS(device_io_align, struct btrfs_dev_item, io_align, 32); +BTRFS_SETGET_FUNCS(device_io_width, struct btrfs_dev_item, io_width, 32); +BTRFS_SETGET_FUNCS(device_start_offset, struct btrfs_dev_item, start_offset, 64); +BTRFS_SETGET_FUNCS(device_sector_size, struct btrfs_dev_item, sector_size, 32); +BTRFS_SETGET_FUNCS(device_id, struct btrfs_dev_item, devid, 64); +BTRFS_SETGET_FUNCS(device_group, struct btrfs_dev_item, dev_group, 32); +BTRFS_SETGET_FUNCS(device_seek_speed, struct btrfs_dev_item, seek_speed, 8); +BTRFS_SETGET_FUNCS(device_bandwidth, struct btrfs_dev_item, bandwidth, 8); +BTRFS_SETGET_FUNCS(device_generation, struct btrfs_dev_item, generation, 64); + +BTRFS_SETGET_STACK_FUNCS(stack_device_type, struct btrfs_dev_item, type, 64); +BTRFS_SETGET_STACK_FUNCS(stack_device_total_bytes, struct btrfs_dev_item, + total_bytes, 64); +BTRFS_SETGET_STACK_FUNCS(stack_device_bytes_used, struct btrfs_dev_item, + bytes_used, 64); +BTRFS_SETGET_STACK_FUNCS(stack_device_io_align, struct btrfs_dev_item, + io_align, 32); +BTRFS_SETGET_STACK_FUNCS(stack_device_io_width, struct btrfs_dev_item, + io_width, 32); +BTRFS_SETGET_STACK_FUNCS(stack_device_sector_size, struct btrfs_dev_item, + sector_size, 32); +BTRFS_SETGET_STACK_FUNCS(stack_device_id, struct btrfs_dev_item, devid, 64); +BTRFS_SETGET_STACK_FUNCS(stack_device_group, struct btrfs_dev_item, dev_group, 32); +BTRFS_SETGET_STACK_FUNCS(stack_device_seek_speed, struct btrfs_dev_item, + seek_speed, 8); +BTRFS_SETGET_STACK_FUNCS(stack_device_bandwidth, struct btrfs_dev_item, + bandwidth, 8); +BTRFS_SETGET_STACK_FUNCS(stack_device_generation, struct btrfs_dev_item, + generation, 64); + +static inline unsigned long btrfs_device_uuid(struct btrfs_dev_item *d) +{ + return (unsigned long)d + offsetof(struct btrfs_dev_item, uuid); +} + +static inline unsigned long btrfs_device_fsid(struct btrfs_dev_item *d) +{ + return (unsigned long)d + offsetof(struct btrfs_dev_item, fsid); +} + +BTRFS_SETGET_FUNCS(chunk_length, struct btrfs_chunk, length, 64); +BTRFS_SETGET_FUNCS(chunk_owner, struct btrfs_chunk, owner, 64); +BTRFS_SETGET_FUNCS(chunk_stripe_len, struct btrfs_chunk, stripe_len, 64); +BTRFS_SETGET_FUNCS(chunk_io_align, struct btrfs_chunk, io_align, 32); +BTRFS_SETGET_FUNCS(chunk_io_width, struct btrfs_chunk, io_width, 32); +BTRFS_SETGET_FUNCS(chunk_sector_size, struct btrfs_chunk, sector_size, 32); +BTRFS_SETGET_FUNCS(chunk_type, struct btrfs_chunk, type, 64); +BTRFS_SETGET_FUNCS(chunk_num_stripes, struct btrfs_chunk, num_stripes, 16); +BTRFS_SETGET_FUNCS(chunk_sub_stripes, struct btrfs_chunk, sub_stripes, 16); +BTRFS_SETGET_FUNCS(stripe_devid, struct btrfs_stripe, devid, 64); +BTRFS_SETGET_FUNCS(stripe_offset, struct btrfs_stripe, offset, 64); + +static inline char *btrfs_stripe_dev_uuid(struct btrfs_stripe *s) +{ + return (char *)s + offsetof(struct btrfs_stripe, dev_uuid); +} + +BTRFS_SETGET_STACK_FUNCS(stack_chunk_length, struct btrfs_chunk, length, 64); +BTRFS_SETGET_STACK_FUNCS(stack_chunk_owner, struct btrfs_chunk, owner, 64); +BTRFS_SETGET_STACK_FUNCS(stack_chunk_stripe_len, struct btrfs_chunk, + stripe_len, 64); +BTRFS_SETGET_STACK_FUNCS(stack_chunk_io_align, struct btrfs_chunk, io_align, 32); +BTRFS_SETGET_STACK_FUNCS(stack_chunk_io_width, struct btrfs_chunk, io_width, 32); +BTRFS_SETGET_STACK_FUNCS(stack_chunk_sector_size, struct btrfs_chunk, + sector_size, 32); +BTRFS_SETGET_STACK_FUNCS(stack_chunk_type, struct btrfs_chunk, type, 64); +BTRFS_SETGET_STACK_FUNCS(stack_chunk_num_stripes, struct btrfs_chunk, + num_stripes, 16); +BTRFS_SETGET_STACK_FUNCS(stack_chunk_sub_stripes, struct btrfs_chunk, + sub_stripes, 16); +BTRFS_SETGET_STACK_FUNCS(stack_stripe_devid, struct btrfs_stripe, devid, 64); +BTRFS_SETGET_STACK_FUNCS(stack_stripe_offset, struct btrfs_stripe, offset, 64); + +static inline struct btrfs_stripe *btrfs_stripe_nr(struct btrfs_chunk *c, int nr) +{ + unsigned long offset = (unsigned long)c; + + offset += offsetof(struct btrfs_chunk, stripe); + offset += nr * sizeof(struct btrfs_stripe); + return (struct btrfs_stripe *)offset; +} + +static inline char *btrfs_stripe_dev_uuid_nr(struct btrfs_chunk *c, int nr) +{ + return btrfs_stripe_dev_uuid(btrfs_stripe_nr(c, nr)); +} + +static inline u64 btrfs_stripe_offset_nr(const struct extent_buffer *eb, + struct btrfs_chunk *c, int nr) +{ + return btrfs_stripe_offset(eb, btrfs_stripe_nr(c, nr)); +} + +static inline void btrfs_set_stripe_offset_nr(struct extent_buffer *eb, + struct btrfs_chunk *c, int nr, + u64 val) +{ + btrfs_set_stripe_offset(eb, btrfs_stripe_nr(c, nr), val); +} + +static inline u64 btrfs_stripe_devid_nr(const struct extent_buffer *eb, + struct btrfs_chunk *c, int nr) +{ + return btrfs_stripe_devid(eb, btrfs_stripe_nr(c, nr)); +} + +static inline void btrfs_set_stripe_devid_nr(struct extent_buffer *eb, + struct btrfs_chunk *c, int nr, + u64 val) +{ + btrfs_set_stripe_devid(eb, btrfs_stripe_nr(c, nr), val); +} + +/* struct btrfs_block_group_item */ +BTRFS_SETGET_STACK_FUNCS(stack_block_group_used, struct btrfs_block_group_item, + used, 64); +BTRFS_SETGET_FUNCS(block_group_used, struct btrfs_block_group_item, used, 64); +BTRFS_SETGET_STACK_FUNCS(stack_block_group_chunk_objectid, + struct btrfs_block_group_item, chunk_objectid, 64); + +BTRFS_SETGET_FUNCS(block_group_chunk_objectid, + struct btrfs_block_group_item, chunk_objectid, 64); +BTRFS_SETGET_FUNCS(block_group_flags, struct btrfs_block_group_item, flags, 64); +BTRFS_SETGET_STACK_FUNCS(stack_block_group_flags, + struct btrfs_block_group_item, flags, 64); + +/* struct btrfs_block_group_item_v2 */ +BTRFS_SETGET_STACK_FUNCS(stack_block_group_v2_used, struct btrfs_block_group_item_v2, + used, 64); +BTRFS_SETGET_FUNCS(block_group_v2_used, struct btrfs_block_group_item_v2, used, 64); +BTRFS_SETGET_STACK_FUNCS(stack_block_group_v2_chunk_objectid, + struct btrfs_block_group_item_v2, chunk_objectid, 64); +BTRFS_SETGET_FUNCS(block_group_v2_chunk_objectid, + struct btrfs_block_group_item_v2, chunk_objectid, 64); +BTRFS_SETGET_STACK_FUNCS(stack_block_group_v2_flags, + struct btrfs_block_group_item_v2, flags, 64); +BTRFS_SETGET_FUNCS(block_group_v2_flags, struct btrfs_block_group_item_v2, flags, 64); +BTRFS_SETGET_STACK_FUNCS(stack_block_group_v2_remap_bytes, + struct btrfs_block_group_item_v2, remap_bytes, 64); +BTRFS_SETGET_FUNCS(block_group_v2_remap_bytes, struct btrfs_block_group_item_v2, + remap_bytes, 64); +BTRFS_SETGET_STACK_FUNCS(stack_block_group_v2_identity_remap_count, + struct btrfs_block_group_item_v2, identity_remap_count, 32); +BTRFS_SETGET_FUNCS(block_group_v2_identity_remap_count, struct btrfs_block_group_item_v2, + identity_remap_count, 32); + +/* struct btrfs_free_space_info */ +BTRFS_SETGET_FUNCS(free_space_extent_count, struct btrfs_free_space_info, + extent_count, 32); +BTRFS_SETGET_FUNCS(free_space_flags, struct btrfs_free_space_info, flags, 32); + +/* struct btrfs_inode_ref */ +BTRFS_SETGET_FUNCS(inode_ref_name_len, struct btrfs_inode_ref, name_len, 16); +BTRFS_SETGET_FUNCS(inode_ref_index, struct btrfs_inode_ref, index, 64); +BTRFS_SETGET_STACK_FUNCS(stack_inode_ref_name_len, struct btrfs_inode_ref, name_len, 16); +BTRFS_SETGET_STACK_FUNCS(stack_inode_ref_index, struct btrfs_inode_ref, index, 64); + +/* struct btrfs_inode_extref */ +BTRFS_SETGET_FUNCS(inode_extref_parent, struct btrfs_inode_extref, + parent_objectid, 64); +BTRFS_SETGET_FUNCS(inode_extref_name_len, struct btrfs_inode_extref, + name_len, 16); +BTRFS_SETGET_FUNCS(inode_extref_index, struct btrfs_inode_extref, index, 64); + +/* struct btrfs_inode_item */ +BTRFS_SETGET_FUNCS(inode_generation, struct btrfs_inode_item, generation, 64); +BTRFS_SETGET_FUNCS(inode_sequence, struct btrfs_inode_item, sequence, 64); +BTRFS_SETGET_FUNCS(inode_transid, struct btrfs_inode_item, transid, 64); +BTRFS_SETGET_FUNCS(inode_size, struct btrfs_inode_item, size, 64); +BTRFS_SETGET_FUNCS(inode_nbytes, struct btrfs_inode_item, nbytes, 64); +BTRFS_SETGET_FUNCS(inode_block_group, struct btrfs_inode_item, block_group, 64); +BTRFS_SETGET_FUNCS(inode_nlink, struct btrfs_inode_item, nlink, 32); +BTRFS_SETGET_FUNCS(inode_uid, struct btrfs_inode_item, uid, 32); +BTRFS_SETGET_FUNCS(inode_gid, struct btrfs_inode_item, gid, 32); +BTRFS_SETGET_FUNCS(inode_mode, struct btrfs_inode_item, mode, 32); +BTRFS_SETGET_FUNCS(inode_rdev, struct btrfs_inode_item, rdev, 64); +BTRFS_SETGET_FUNCS(inode_flags, struct btrfs_inode_item, flags, 64); +BTRFS_SETGET_STACK_FUNCS(stack_inode_generation, struct btrfs_inode_item, + generation, 64); +BTRFS_SETGET_STACK_FUNCS(stack_inode_sequence, struct btrfs_inode_item, + sequence, 64); +BTRFS_SETGET_STACK_FUNCS(stack_inode_transid, struct btrfs_inode_item, + transid, 64); +BTRFS_SETGET_STACK_FUNCS(stack_inode_size, struct btrfs_inode_item, size, 64); +BTRFS_SETGET_STACK_FUNCS(stack_inode_nbytes, struct btrfs_inode_item, nbytes, 64); +BTRFS_SETGET_STACK_FUNCS(stack_inode_block_group, struct btrfs_inode_item, + block_group, 64); +BTRFS_SETGET_STACK_FUNCS(stack_inode_nlink, struct btrfs_inode_item, nlink, 32); +BTRFS_SETGET_STACK_FUNCS(stack_inode_uid, struct btrfs_inode_item, uid, 32); +BTRFS_SETGET_STACK_FUNCS(stack_inode_gid, struct btrfs_inode_item, gid, 32); +BTRFS_SETGET_STACK_FUNCS(stack_inode_mode, struct btrfs_inode_item, mode, 32); +BTRFS_SETGET_STACK_FUNCS(stack_inode_rdev, struct btrfs_inode_item, rdev, 64); +BTRFS_SETGET_STACK_FUNCS(stack_inode_flags, struct btrfs_inode_item, flags, 64); +BTRFS_SETGET_FUNCS(timespec_sec, struct btrfs_timespec, sec, 64); +BTRFS_SETGET_FUNCS(timespec_nsec, struct btrfs_timespec, nsec, 32); +BTRFS_SETGET_STACK_FUNCS(stack_timespec_sec, struct btrfs_timespec, sec, 64); +BTRFS_SETGET_STACK_FUNCS(stack_timespec_nsec, struct btrfs_timespec, nsec, 32); + +BTRFS_SETGET_FUNCS(raid_stride_devid, struct btrfs_raid_stride, devid, 64); +BTRFS_SETGET_FUNCS(raid_stride_physical, struct btrfs_raid_stride, physical, 64); +BTRFS_SETGET_STACK_FUNCS(stack_raid_stride_devid, struct btrfs_raid_stride, devid, 64); +BTRFS_SETGET_STACK_FUNCS(stack_raid_stride_physical, struct btrfs_raid_stride, physical, 64); + +/* struct btrfs_dev_extent */ +BTRFS_SETGET_FUNCS(dev_extent_chunk_tree, struct btrfs_dev_extent, chunk_tree, 64); +BTRFS_SETGET_FUNCS(dev_extent_chunk_objectid, struct btrfs_dev_extent, + chunk_objectid, 64); +BTRFS_SETGET_FUNCS(dev_extent_chunk_offset, struct btrfs_dev_extent, + chunk_offset, 64); +BTRFS_SETGET_FUNCS(dev_extent_length, struct btrfs_dev_extent, length, 64); +BTRFS_SETGET_STACK_FUNCS(stack_dev_extent_chunk_tree, struct btrfs_dev_extent, + chunk_tree, 64); +BTRFS_SETGET_STACK_FUNCS(stack_dev_extent_chunk_objectid, struct btrfs_dev_extent, + chunk_objectid, 64); +BTRFS_SETGET_STACK_FUNCS(stack_dev_extent_chunk_offset, struct btrfs_dev_extent, + chunk_offset, 64); +BTRFS_SETGET_STACK_FUNCS(stack_dev_extent_length, struct btrfs_dev_extent, length, 64); + +BTRFS_SETGET_FUNCS(extent_refs, struct btrfs_extent_item, refs, 64); +BTRFS_SETGET_FUNCS(extent_generation, struct btrfs_extent_item, generation, 64); +BTRFS_SETGET_FUNCS(extent_flags, struct btrfs_extent_item, flags, 64); + +BTRFS_SETGET_FUNCS(tree_block_level, struct btrfs_tree_block_info, level, 8); + +static inline void btrfs_tree_block_key(const struct extent_buffer *eb, + struct btrfs_tree_block_info *item, + struct btrfs_disk_key *key) +{ + read_eb_member(eb, item, struct btrfs_tree_block_info, key, key); +} + +static inline void btrfs_set_tree_block_key(const struct extent_buffer *eb, + struct btrfs_tree_block_info *item, + const struct btrfs_disk_key *key) +{ + write_eb_member(eb, item, struct btrfs_tree_block_info, key, key); +} + +BTRFS_SETGET_FUNCS(extent_data_ref_root, struct btrfs_extent_data_ref, root, 64); +BTRFS_SETGET_FUNCS(extent_data_ref_objectid, struct btrfs_extent_data_ref, + objectid, 64); +BTRFS_SETGET_FUNCS(extent_data_ref_offset, struct btrfs_extent_data_ref, + offset, 64); +BTRFS_SETGET_FUNCS(extent_data_ref_count, struct btrfs_extent_data_ref, count, 32); + +BTRFS_SETGET_FUNCS(shared_data_ref_count, struct btrfs_shared_data_ref, count, 32); + +BTRFS_SETGET_FUNCS(extent_owner_ref_root_id, struct btrfs_extent_owner_ref, + root_id, 64); + +BTRFS_SETGET_FUNCS(extent_inline_ref_type, struct btrfs_extent_inline_ref, + type, 8); +BTRFS_SETGET_FUNCS(extent_inline_ref_offset, struct btrfs_extent_inline_ref, + offset, 64); + +static inline u32 btrfs_extent_inline_ref_size(int type) +{ + if (type == BTRFS_TREE_BLOCK_REF_KEY || + type == BTRFS_SHARED_BLOCK_REF_KEY) + return sizeof(struct btrfs_extent_inline_ref); + if (type == BTRFS_SHARED_DATA_REF_KEY) + return sizeof(struct btrfs_shared_data_ref) + + sizeof(struct btrfs_extent_inline_ref); + if (type == BTRFS_EXTENT_DATA_REF_KEY) + return sizeof(struct btrfs_extent_data_ref) + + offsetof(struct btrfs_extent_inline_ref, offset); + if (type == BTRFS_EXTENT_OWNER_REF_KEY) + return sizeof(struct btrfs_extent_inline_ref); + return 0; +} + +/* struct btrfs_node */ +BTRFS_SETGET_FUNCS(key_blockptr, struct btrfs_key_ptr, blockptr, 64); +BTRFS_SETGET_FUNCS(key_generation, struct btrfs_key_ptr, generation, 64); +BTRFS_SETGET_STACK_FUNCS(stack_key_blockptr, struct btrfs_key_ptr, blockptr, 64); +BTRFS_SETGET_STACK_FUNCS(stack_key_generation, struct btrfs_key_ptr, + generation, 64); + +static inline u64 btrfs_node_blockptr(const struct extent_buffer *eb, int nr) +{ + unsigned long ptr; + + ptr = offsetof(struct btrfs_node, ptrs) + + sizeof(struct btrfs_key_ptr) * nr; + return btrfs_key_blockptr(eb, (struct btrfs_key_ptr *)ptr); +} + +static inline void btrfs_set_node_blockptr(const struct extent_buffer *eb, + int nr, u64 val) +{ + unsigned long ptr; + + ptr = offsetof(struct btrfs_node, ptrs) + + sizeof(struct btrfs_key_ptr) * nr; + btrfs_set_key_blockptr(eb, (struct btrfs_key_ptr *)ptr, val); +} + +static inline u64 btrfs_node_ptr_generation(const struct extent_buffer *eb, int nr) +{ + unsigned long ptr; + + ptr = offsetof(struct btrfs_node, ptrs) + + sizeof(struct btrfs_key_ptr) * nr; + return btrfs_key_generation(eb, (struct btrfs_key_ptr *)ptr); +} + +static inline void btrfs_set_node_ptr_generation(const struct extent_buffer *eb, + int nr, u64 val) +{ + unsigned long ptr; + + ptr = offsetof(struct btrfs_node, ptrs) + + sizeof(struct btrfs_key_ptr) * nr; + btrfs_set_key_generation(eb, (struct btrfs_key_ptr *)ptr, val); +} + +static inline unsigned long btrfs_node_key_ptr_offset(const struct extent_buffer *eb, int nr) +{ + return offsetof(struct btrfs_node, ptrs) + + sizeof(struct btrfs_key_ptr) * nr; +} + +void btrfs_node_key(const struct extent_buffer *eb, + struct btrfs_disk_key *disk_key, int nr); + +static inline void btrfs_set_node_key(const struct extent_buffer *eb, + const struct btrfs_disk_key *disk_key, int nr) +{ + unsigned long ptr; + + ptr = btrfs_node_key_ptr_offset(eb, nr); + write_eb_member(eb, (struct btrfs_key_ptr *)ptr, + struct btrfs_key_ptr, key, disk_key); +} + +/* struct btrfs_item */ +BTRFS_SETGET_FUNCS(raw_item_offset, struct btrfs_item, offset, 32); +BTRFS_SETGET_FUNCS(raw_item_size, struct btrfs_item, size, 32); +BTRFS_SETGET_STACK_FUNCS(stack_item_offset, struct btrfs_item, offset, 32); +BTRFS_SETGET_STACK_FUNCS(stack_item_size, struct btrfs_item, size, 32); + +static inline unsigned long btrfs_item_nr_offset(const struct extent_buffer *eb, int nr) +{ + return offsetof(struct btrfs_leaf, items) + + sizeof(struct btrfs_item) * nr; +} + +static inline struct btrfs_item *btrfs_item_nr(const struct extent_buffer *eb, int nr) +{ + return (struct btrfs_item *)btrfs_item_nr_offset(eb, nr); +} + +#define BTRFS_ITEM_SETGET_FUNCS(member) \ +static inline u32 btrfs_item_##member(const struct extent_buffer *eb, int slot) \ +{ \ + return btrfs_raw_item_##member(eb, btrfs_item_nr(eb, slot)); \ +} \ +static inline void btrfs_set_item_##member(const struct extent_buffer *eb, \ + int slot, u32 val) \ +{ \ + btrfs_set_raw_item_##member(eb, btrfs_item_nr(eb, slot), val); \ +} + +BTRFS_ITEM_SETGET_FUNCS(offset) +BTRFS_ITEM_SETGET_FUNCS(size); + +static inline u32 btrfs_item_data_end(const struct extent_buffer *eb, int nr) +{ + return btrfs_item_offset(eb, nr) + btrfs_item_size(eb, nr); +} + +static inline void btrfs_item_key(const struct extent_buffer *eb, + struct btrfs_disk_key *disk_key, int nr) +{ + struct btrfs_item *item = btrfs_item_nr(eb, nr); + + read_eb_member(eb, item, struct btrfs_item, key, disk_key); +} + +static inline void btrfs_set_item_key(struct extent_buffer *eb, + const struct btrfs_disk_key *disk_key, int nr) +{ + struct btrfs_item *item = btrfs_item_nr(eb, nr); + + write_eb_member(eb, item, struct btrfs_item, key, disk_key); +} + +BTRFS_SETGET_FUNCS(dir_log_end, struct btrfs_dir_log_item, end, 64); + +/* struct btrfs_root_ref */ +BTRFS_SETGET_FUNCS(root_ref_dirid, struct btrfs_root_ref, dirid, 64); +BTRFS_SETGET_FUNCS(root_ref_sequence, struct btrfs_root_ref, sequence, 64); +BTRFS_SETGET_FUNCS(root_ref_name_len, struct btrfs_root_ref, name_len, 16); +BTRFS_SETGET_STACK_FUNCS(stack_root_ref_dirid, struct btrfs_root_ref, dirid, 64); +BTRFS_SETGET_STACK_FUNCS(stack_root_ref_sequence, struct btrfs_root_ref, sequence, 64); +BTRFS_SETGET_STACK_FUNCS(stack_root_ref_name_len, struct btrfs_root_ref, name_len, 16); + +/* struct btrfs_dir_item */ +BTRFS_SETGET_FUNCS(dir_data_len, struct btrfs_dir_item, data_len, 16); +BTRFS_SETGET_FUNCS(dir_flags, struct btrfs_dir_item, type, 8); +BTRFS_SETGET_FUNCS(dir_name_len, struct btrfs_dir_item, name_len, 16); +BTRFS_SETGET_FUNCS(dir_transid, struct btrfs_dir_item, transid, 64); +BTRFS_SETGET_STACK_FUNCS(stack_dir_flags, struct btrfs_dir_item, type, 8); +BTRFS_SETGET_STACK_FUNCS(stack_dir_data_len, struct btrfs_dir_item, data_len, 16); +BTRFS_SETGET_STACK_FUNCS(stack_dir_name_len, struct btrfs_dir_item, name_len, 16); +BTRFS_SETGET_STACK_FUNCS(stack_dir_transid, struct btrfs_dir_item, transid, 64); + +static inline u8 btrfs_dir_ftype(const struct extent_buffer *eb, + const struct btrfs_dir_item *item) +{ + return btrfs_dir_flags_to_ftype(btrfs_dir_flags(eb, item)); +} + +static inline u8 btrfs_stack_dir_ftype(const struct btrfs_dir_item *item) +{ + return btrfs_dir_flags_to_ftype(btrfs_stack_dir_flags(item)); +} + +static inline void btrfs_dir_item_key(const struct extent_buffer *eb, + const struct btrfs_dir_item *item, + struct btrfs_disk_key *key) +{ + read_eb_member(eb, item, struct btrfs_dir_item, location, key); +} + +static inline void btrfs_set_dir_item_key(struct extent_buffer *eb, + struct btrfs_dir_item *item, + const struct btrfs_disk_key *key) +{ + write_eb_member(eb, item, struct btrfs_dir_item, location, key); +} + +BTRFS_SETGET_FUNCS(free_space_entries, struct btrfs_free_space_header, + num_entries, 64); +BTRFS_SETGET_FUNCS(free_space_bitmaps, struct btrfs_free_space_header, + num_bitmaps, 64); +BTRFS_SETGET_FUNCS(free_space_generation, struct btrfs_free_space_header, + generation, 64); + +static inline void btrfs_free_space_key(const struct extent_buffer *eb, + const struct btrfs_free_space_header *h, + struct btrfs_disk_key *key) +{ + read_eb_member(eb, h, struct btrfs_free_space_header, location, key); +} + +static inline void btrfs_set_free_space_key(struct extent_buffer *eb, + struct btrfs_free_space_header *h, + const struct btrfs_disk_key *key) +{ + write_eb_member(eb, h, struct btrfs_free_space_header, location, key); +} + +/* struct btrfs_disk_key */ +BTRFS_SETGET_STACK_FUNCS(disk_key_objectid, struct btrfs_disk_key, objectid, 64); +BTRFS_SETGET_STACK_FUNCS(disk_key_offset, struct btrfs_disk_key, offset, 64); +BTRFS_SETGET_STACK_FUNCS(disk_key_type, struct btrfs_disk_key, type, 8); + +#ifdef __LITTLE_ENDIAN + +/* + * Optimized helpers for little-endian architectures where CPU and on-disk + * structures have the same endianness and we can skip conversions. + */ + +static inline void btrfs_disk_key_to_cpu(struct btrfs_key *cpu_key, + const struct btrfs_disk_key *disk_key) +{ + memcpy(cpu_key, disk_key, sizeof(struct btrfs_key)); +} + +static inline void btrfs_cpu_key_to_disk(struct btrfs_disk_key *disk_key, + const struct btrfs_key *cpu_key) +{ + memcpy(disk_key, cpu_key, sizeof(struct btrfs_key)); +} + +static inline void btrfs_node_key_to_cpu(const struct extent_buffer *eb, + struct btrfs_key *cpu_key, int nr) +{ + struct btrfs_disk_key *disk_key = (struct btrfs_disk_key *)cpu_key; + + btrfs_node_key(eb, disk_key, nr); +} + +static inline void btrfs_item_key_to_cpu(const struct extent_buffer *eb, + struct btrfs_key *cpu_key, int nr) +{ + struct btrfs_disk_key *disk_key = (struct btrfs_disk_key *)cpu_key; + + btrfs_item_key(eb, disk_key, nr); +} + +static inline void btrfs_dir_item_key_to_cpu(const struct extent_buffer *eb, + const struct btrfs_dir_item *item, + struct btrfs_key *cpu_key) +{ + struct btrfs_disk_key *disk_key = (struct btrfs_disk_key *)cpu_key; + + btrfs_dir_item_key(eb, item, disk_key); +} + +#else + +static inline void btrfs_disk_key_to_cpu(struct btrfs_key *cpu, + const struct btrfs_disk_key *disk) +{ + cpu->offset = le64_to_cpu(disk->offset); + cpu->type = disk->type; + cpu->objectid = le64_to_cpu(disk->objectid); +} + +static inline void btrfs_cpu_key_to_disk(struct btrfs_disk_key *disk, + const struct btrfs_key *cpu) +{ + disk->offset = cpu_to_le64(cpu->offset); + disk->type = cpu->type; + disk->objectid = cpu_to_le64(cpu->objectid); +} + +static inline void btrfs_node_key_to_cpu(const struct extent_buffer *eb, + struct btrfs_key *key, int nr) +{ + struct btrfs_disk_key disk_key; + + btrfs_node_key(eb, &disk_key, nr); + btrfs_disk_key_to_cpu(key, &disk_key); +} + +static inline void btrfs_item_key_to_cpu(const struct extent_buffer *eb, + struct btrfs_key *key, int nr) +{ + struct btrfs_disk_key disk_key; + + btrfs_item_key(eb, &disk_key, nr); + btrfs_disk_key_to_cpu(key, &disk_key); +} + +static inline void btrfs_dir_item_key_to_cpu(const struct extent_buffer *eb, + const struct btrfs_dir_item *item, + struct btrfs_key *key) +{ + struct btrfs_disk_key disk_key; + + btrfs_dir_item_key(eb, item, &disk_key); + btrfs_disk_key_to_cpu(key, &disk_key); +} + +#endif + +/* struct btrfs_header */ +BTRFS_SETGET_HEADER_FUNCS(header_bytenr, struct btrfs_header, bytenr, 64); +BTRFS_SETGET_HEADER_FUNCS(header_generation, struct btrfs_header, generation, 64); +BTRFS_SETGET_HEADER_FUNCS(header_owner, struct btrfs_header, owner, 64); +BTRFS_SETGET_HEADER_FUNCS(header_nritems, struct btrfs_header, nritems, 32); +BTRFS_SETGET_HEADER_FUNCS(header_flags, struct btrfs_header, flags, 64); +BTRFS_SETGET_HEADER_FUNCS(header_level, struct btrfs_header, level, 8); +BTRFS_SETGET_STACK_FUNCS(stack_header_generation, struct btrfs_header, + generation, 64); +BTRFS_SETGET_STACK_FUNCS(stack_header_owner, struct btrfs_header, owner, 64); +BTRFS_SETGET_STACK_FUNCS(stack_header_nritems, struct btrfs_header, nritems, 32); +BTRFS_SETGET_STACK_FUNCS(stack_header_bytenr, struct btrfs_header, bytenr, 64); + +static inline int btrfs_header_flag(const struct extent_buffer *eb, u64 flag) +{ + return (btrfs_header_flags(eb) & flag) == flag; +} + +static inline void btrfs_set_header_flag(struct extent_buffer *eb, u64 flag) +{ + u64 flags = btrfs_header_flags(eb); + + btrfs_set_header_flags(eb, flags | flag); +} + +static inline void btrfs_clear_header_flag(struct extent_buffer *eb, u64 flag) +{ + u64 flags = btrfs_header_flags(eb); + + btrfs_set_header_flags(eb, flags & ~flag); +} + +static inline int btrfs_header_backref_rev(const struct extent_buffer *eb) +{ + u64 flags = btrfs_header_flags(eb); + + return flags >> BTRFS_BACKREF_REV_SHIFT; +} + +static inline void btrfs_set_header_backref_rev(struct extent_buffer *eb, int rev) +{ + u64 flags = btrfs_header_flags(eb); + + flags &= ~BTRFS_BACKREF_REV_MASK; + flags |= (u64)rev << BTRFS_BACKREF_REV_SHIFT; + btrfs_set_header_flags(eb, flags); +} + +static inline int btrfs_is_leaf(const struct extent_buffer *eb) +{ + return btrfs_header_level(eb) == 0; +} + +/* struct btrfs_root_item */ +BTRFS_SETGET_FUNCS(disk_root_generation, struct btrfs_root_item, generation, 64); +BTRFS_SETGET_FUNCS(disk_root_refs, struct btrfs_root_item, refs, 32); +BTRFS_SETGET_FUNCS(disk_root_bytenr, struct btrfs_root_item, bytenr, 64); +BTRFS_SETGET_FUNCS(disk_root_level, struct btrfs_root_item, level, 8); + +BTRFS_SETGET_STACK_FUNCS(root_generation, struct btrfs_root_item, generation, 64); +BTRFS_SETGET_STACK_FUNCS(root_bytenr, struct btrfs_root_item, bytenr, 64); +BTRFS_SETGET_STACK_FUNCS(root_drop_level, struct btrfs_root_item, drop_level, 8); +BTRFS_SETGET_STACK_FUNCS(root_level, struct btrfs_root_item, level, 8); +BTRFS_SETGET_STACK_FUNCS(root_dirid, struct btrfs_root_item, root_dirid, 64); +BTRFS_SETGET_STACK_FUNCS(root_refs, struct btrfs_root_item, refs, 32); +BTRFS_SETGET_STACK_FUNCS(root_flags, struct btrfs_root_item, flags, 64); +BTRFS_SETGET_STACK_FUNCS(root_used, struct btrfs_root_item, bytes_used, 64); +BTRFS_SETGET_STACK_FUNCS(root_limit, struct btrfs_root_item, byte_limit, 64); +BTRFS_SETGET_STACK_FUNCS(root_last_snapshot, struct btrfs_root_item, + last_snapshot, 64); +BTRFS_SETGET_STACK_FUNCS(root_generation_v2, struct btrfs_root_item, + generation_v2, 64); +BTRFS_SETGET_STACK_FUNCS(root_ctransid, struct btrfs_root_item, ctransid, 64); +BTRFS_SETGET_STACK_FUNCS(root_otransid, struct btrfs_root_item, otransid, 64); +BTRFS_SETGET_STACK_FUNCS(root_stransid, struct btrfs_root_item, stransid, 64); +BTRFS_SETGET_STACK_FUNCS(root_rtransid, struct btrfs_root_item, rtransid, 64); + +/* struct btrfs_root_backup */ +BTRFS_SETGET_STACK_FUNCS(backup_tree_root, struct btrfs_root_backup, + tree_root, 64); +BTRFS_SETGET_STACK_FUNCS(backup_tree_root_gen, struct btrfs_root_backup, + tree_root_gen, 64); +BTRFS_SETGET_STACK_FUNCS(backup_tree_root_level, struct btrfs_root_backup, + tree_root_level, 8); + +BTRFS_SETGET_STACK_FUNCS(backup_chunk_root, struct btrfs_root_backup, + chunk_root, 64); +BTRFS_SETGET_STACK_FUNCS(backup_chunk_root_gen, struct btrfs_root_backup, + chunk_root_gen, 64); +BTRFS_SETGET_STACK_FUNCS(backup_chunk_root_level, struct btrfs_root_backup, + chunk_root_level, 8); + +BTRFS_SETGET_STACK_FUNCS(backup_extent_root, struct btrfs_root_backup, + extent_root, 64); +BTRFS_SETGET_STACK_FUNCS(backup_extent_root_gen, struct btrfs_root_backup, + extent_root_gen, 64); +BTRFS_SETGET_STACK_FUNCS(backup_extent_root_level, struct btrfs_root_backup, + extent_root_level, 8); + +BTRFS_SETGET_STACK_FUNCS(backup_fs_root, struct btrfs_root_backup, + fs_root, 64); +BTRFS_SETGET_STACK_FUNCS(backup_fs_root_gen, struct btrfs_root_backup, + fs_root_gen, 64); +BTRFS_SETGET_STACK_FUNCS(backup_fs_root_level, struct btrfs_root_backup, + fs_root_level, 8); + +BTRFS_SETGET_STACK_FUNCS(backup_dev_root, struct btrfs_root_backup, + dev_root, 64); +BTRFS_SETGET_STACK_FUNCS(backup_dev_root_gen, struct btrfs_root_backup, + dev_root_gen, 64); +BTRFS_SETGET_STACK_FUNCS(backup_dev_root_level, struct btrfs_root_backup, + dev_root_level, 8); + +BTRFS_SETGET_STACK_FUNCS(backup_csum_root, struct btrfs_root_backup, + csum_root, 64); +BTRFS_SETGET_STACK_FUNCS(backup_csum_root_gen, struct btrfs_root_backup, + csum_root_gen, 64); +BTRFS_SETGET_STACK_FUNCS(backup_csum_root_level, struct btrfs_root_backup, + csum_root_level, 8); +BTRFS_SETGET_STACK_FUNCS(backup_total_bytes, struct btrfs_root_backup, + total_bytes, 64); +BTRFS_SETGET_STACK_FUNCS(backup_bytes_used, struct btrfs_root_backup, + bytes_used, 64); +BTRFS_SETGET_STACK_FUNCS(backup_num_devices, struct btrfs_root_backup, + num_devices, 64); + +/* struct btrfs_balance_item */ +BTRFS_SETGET_FUNCS(balance_flags, struct btrfs_balance_item, flags, 64); + +static inline void btrfs_balance_data(const struct extent_buffer *eb, + const struct btrfs_balance_item *bi, + struct btrfs_disk_balance_args *ba) +{ + read_eb_member(eb, bi, struct btrfs_balance_item, data, ba); +} + +static inline void btrfs_set_balance_data(struct extent_buffer *eb, + struct btrfs_balance_item *bi, + const struct btrfs_disk_balance_args *ba) +{ + write_eb_member(eb, bi, struct btrfs_balance_item, data, ba); +} + +static inline void btrfs_balance_meta(const struct extent_buffer *eb, + const struct btrfs_balance_item *bi, + struct btrfs_disk_balance_args *ba) +{ + read_eb_member(eb, bi, struct btrfs_balance_item, meta, ba); +} + +static inline void btrfs_set_balance_meta(struct extent_buffer *eb, + struct btrfs_balance_item *bi, + const struct btrfs_disk_balance_args *ba) +{ + write_eb_member(eb, bi, struct btrfs_balance_item, meta, ba); +} + +static inline void btrfs_balance_sys(const struct extent_buffer *eb, + const struct btrfs_balance_item *bi, + struct btrfs_disk_balance_args *ba) +{ + read_eb_member(eb, bi, struct btrfs_balance_item, sys, ba); +} + +static inline void btrfs_set_balance_sys(struct extent_buffer *eb, + struct btrfs_balance_item *bi, + const struct btrfs_disk_balance_args *ba) +{ + write_eb_member(eb, bi, struct btrfs_balance_item, sys, ba); +} + +/* struct btrfs_super_block */ +BTRFS_SETGET_STACK_FUNCS(super_bytenr, struct btrfs_super_block, bytenr, 64); +BTRFS_SETGET_STACK_FUNCS(super_flags, struct btrfs_super_block, flags, 64); +BTRFS_SETGET_STACK_FUNCS(super_generation, struct btrfs_super_block, + generation, 64); +BTRFS_SETGET_STACK_FUNCS(super_root, struct btrfs_super_block, root, 64); +BTRFS_SETGET_STACK_FUNCS(super_sys_array_size, + struct btrfs_super_block, sys_chunk_array_size, 32); +BTRFS_SETGET_STACK_FUNCS(super_chunk_root_generation, + struct btrfs_super_block, chunk_root_generation, 64); +BTRFS_SETGET_STACK_FUNCS(super_root_level, struct btrfs_super_block, + root_level, 8); +BTRFS_SETGET_STACK_FUNCS(super_chunk_root, struct btrfs_super_block, + chunk_root, 64); +BTRFS_SETGET_STACK_FUNCS(super_chunk_root_level, struct btrfs_super_block, + chunk_root_level, 8); +BTRFS_SETGET_STACK_FUNCS(super_log_root, struct btrfs_super_block, log_root, 64); +BTRFS_SETGET_STACK_FUNCS(super_log_root_level, struct btrfs_super_block, + log_root_level, 8); +BTRFS_SETGET_STACK_FUNCS(super_total_bytes, struct btrfs_super_block, + total_bytes, 64); +BTRFS_SETGET_STACK_FUNCS(super_bytes_used, struct btrfs_super_block, + bytes_used, 64); +BTRFS_SETGET_STACK_FUNCS(super_sectorsize, struct btrfs_super_block, + sectorsize, 32); +BTRFS_SETGET_STACK_FUNCS(super_nodesize, struct btrfs_super_block, + nodesize, 32); +BTRFS_SETGET_STACK_FUNCS(super_stripesize, struct btrfs_super_block, + stripesize, 32); +BTRFS_SETGET_STACK_FUNCS(super_root_dir, struct btrfs_super_block, + root_dir_objectid, 64); +BTRFS_SETGET_STACK_FUNCS(super_num_devices, struct btrfs_super_block, + num_devices, 64); +BTRFS_SETGET_STACK_FUNCS(super_compat_flags, struct btrfs_super_block, + compat_flags, 64); +BTRFS_SETGET_STACK_FUNCS(super_compat_ro_flags, struct btrfs_super_block, + compat_ro_flags, 64); +BTRFS_SETGET_STACK_FUNCS(super_incompat_flags, struct btrfs_super_block, + incompat_flags, 64); +BTRFS_SETGET_STACK_FUNCS(super_csum_type, struct btrfs_super_block, + csum_type, 16); +BTRFS_SETGET_STACK_FUNCS(super_cache_generation, struct btrfs_super_block, + cache_generation, 64); +BTRFS_SETGET_STACK_FUNCS(super_magic, struct btrfs_super_block, magic, 64); +BTRFS_SETGET_STACK_FUNCS(super_uuid_tree_generation, struct btrfs_super_block, + uuid_tree_generation, 64); +BTRFS_SETGET_STACK_FUNCS(super_nr_global_roots, struct btrfs_super_block, + nr_global_roots, 64); +BTRFS_SETGET_STACK_FUNCS(super_remap_root, struct btrfs_super_block, + remap_root, 64); +BTRFS_SETGET_STACK_FUNCS(super_remap_root_generation, struct btrfs_super_block, + remap_root_generation, 64); +BTRFS_SETGET_STACK_FUNCS(super_remap_root_level, struct btrfs_super_block, + remap_root_level, 8); + +/* struct btrfs_file_extent_item */ +BTRFS_SETGET_STACK_FUNCS(stack_file_extent_type, struct btrfs_file_extent_item, + type, 8); +BTRFS_SETGET_STACK_FUNCS(stack_file_extent_disk_bytenr, + struct btrfs_file_extent_item, disk_bytenr, 64); +BTRFS_SETGET_STACK_FUNCS(stack_file_extent_offset, + struct btrfs_file_extent_item, offset, 64); +BTRFS_SETGET_STACK_FUNCS(stack_file_extent_generation, + struct btrfs_file_extent_item, generation, 64); +BTRFS_SETGET_STACK_FUNCS(stack_file_extent_num_bytes, + struct btrfs_file_extent_item, num_bytes, 64); +BTRFS_SETGET_STACK_FUNCS(stack_file_extent_ram_bytes, + struct btrfs_file_extent_item, ram_bytes, 64); +BTRFS_SETGET_STACK_FUNCS(stack_file_extent_disk_num_bytes, + struct btrfs_file_extent_item, disk_num_bytes, 64); +BTRFS_SETGET_STACK_FUNCS(stack_file_extent_compression, + struct btrfs_file_extent_item, compression, 8); + + +BTRFS_SETGET_FUNCS(file_extent_type, struct btrfs_file_extent_item, type, 8); +BTRFS_SETGET_FUNCS(file_extent_disk_bytenr, struct btrfs_file_extent_item, + disk_bytenr, 64); +BTRFS_SETGET_FUNCS(file_extent_generation, struct btrfs_file_extent_item, + generation, 64); +BTRFS_SETGET_FUNCS(file_extent_disk_num_bytes, struct btrfs_file_extent_item, + disk_num_bytes, 64); +BTRFS_SETGET_FUNCS(file_extent_offset, struct btrfs_file_extent_item, + offset, 64); +BTRFS_SETGET_FUNCS(file_extent_num_bytes, struct btrfs_file_extent_item, + num_bytes, 64); +BTRFS_SETGET_FUNCS(file_extent_ram_bytes, struct btrfs_file_extent_item, + ram_bytes, 64); +BTRFS_SETGET_FUNCS(file_extent_compression, struct btrfs_file_extent_item, + compression, 8); +BTRFS_SETGET_FUNCS(file_extent_encryption, struct btrfs_file_extent_item, + encryption, 8); +BTRFS_SETGET_FUNCS(file_extent_other_encoding, struct btrfs_file_extent_item, + other_encoding, 16); + +/* btrfs_qgroup_status_item */ +BTRFS_SETGET_FUNCS(qgroup_status_generation, struct btrfs_qgroup_status_item, + generation, 64); +BTRFS_SETGET_FUNCS(qgroup_status_version, struct btrfs_qgroup_status_item, + version, 64); +BTRFS_SETGET_FUNCS(qgroup_status_flags, struct btrfs_qgroup_status_item, + flags, 64); +BTRFS_SETGET_FUNCS(qgroup_status_rescan, struct btrfs_qgroup_status_item, + rescan, 64); +BTRFS_SETGET_FUNCS(qgroup_status_enable_gen, struct btrfs_qgroup_status_item, + enable_gen, 64); + +/* btrfs_qgroup_info_item */ +BTRFS_SETGET_FUNCS(qgroup_info_generation, struct btrfs_qgroup_info_item, + generation, 64); +BTRFS_SETGET_FUNCS(qgroup_info_rfer, struct btrfs_qgroup_info_item, rfer, 64); +BTRFS_SETGET_FUNCS(qgroup_info_rfer_cmpr, struct btrfs_qgroup_info_item, + rfer_cmpr, 64); +BTRFS_SETGET_FUNCS(qgroup_info_excl, struct btrfs_qgroup_info_item, excl, 64); +BTRFS_SETGET_FUNCS(qgroup_info_excl_cmpr, struct btrfs_qgroup_info_item, + excl_cmpr, 64); + +BTRFS_SETGET_STACK_FUNCS(stack_qgroup_info_generation, + struct btrfs_qgroup_info_item, generation, 64); +BTRFS_SETGET_STACK_FUNCS(stack_qgroup_info_rfer, struct btrfs_qgroup_info_item, + rfer, 64); +BTRFS_SETGET_STACK_FUNCS(stack_qgroup_info_rfer_cmpr, + struct btrfs_qgroup_info_item, rfer_cmpr, 64); +BTRFS_SETGET_STACK_FUNCS(stack_qgroup_info_excl, struct btrfs_qgroup_info_item, + excl, 64); +BTRFS_SETGET_STACK_FUNCS(stack_qgroup_info_excl_cmpr, + struct btrfs_qgroup_info_item, excl_cmpr, 64); + +/* btrfs_qgroup_limit_item */ +BTRFS_SETGET_FUNCS(qgroup_limit_flags, struct btrfs_qgroup_limit_item, flags, 64); +BTRFS_SETGET_FUNCS(qgroup_limit_max_rfer, struct btrfs_qgroup_limit_item, + max_rfer, 64); +BTRFS_SETGET_FUNCS(qgroup_limit_max_excl, struct btrfs_qgroup_limit_item, + max_excl, 64); +BTRFS_SETGET_FUNCS(qgroup_limit_rsv_rfer, struct btrfs_qgroup_limit_item, + rsv_rfer, 64); +BTRFS_SETGET_FUNCS(qgroup_limit_rsv_excl, struct btrfs_qgroup_limit_item, + rsv_excl, 64); +BTRFS_SETGET_STACK_FUNCS(stack_qgroup_limit_flags, + struct btrfs_qgroup_limit_item, flags, 64); +BTRFS_SETGET_STACK_FUNCS(stack_qgroup_limit_max_rfer, + struct btrfs_qgroup_limit_item, max_rfer, 64); +BTRFS_SETGET_STACK_FUNCS(stack_qgroup_limit_max_excl, + struct btrfs_qgroup_limit_item, max_excl, 64); +BTRFS_SETGET_STACK_FUNCS(stack_qgroup_limit_rsv_rfer, + struct btrfs_qgroup_limit_item, rsv_rfer, 64); +BTRFS_SETGET_STACK_FUNCS(stack_qgroup_limit_rsv_excl, + struct btrfs_qgroup_limit_item, rsv_excl, 64); + +/* btrfs_dev_replace_item */ +BTRFS_SETGET_FUNCS(dev_replace_src_devid, + struct btrfs_dev_replace_item, src_devid, 64); +BTRFS_SETGET_FUNCS(dev_replace_cont_reading_from_srcdev_mode, + struct btrfs_dev_replace_item, cont_reading_from_srcdev_mode, + 64); +BTRFS_SETGET_FUNCS(dev_replace_replace_state, struct btrfs_dev_replace_item, + replace_state, 64); +BTRFS_SETGET_FUNCS(dev_replace_time_started, struct btrfs_dev_replace_item, + time_started, 64); +BTRFS_SETGET_FUNCS(dev_replace_time_stopped, struct btrfs_dev_replace_item, + time_stopped, 64); +BTRFS_SETGET_FUNCS(dev_replace_num_write_errors, struct btrfs_dev_replace_item, + num_write_errors, 64); +BTRFS_SETGET_FUNCS(dev_replace_num_uncorrectable_read_errors, + struct btrfs_dev_replace_item, num_uncorrectable_read_errors, + 64); +BTRFS_SETGET_FUNCS(dev_replace_cursor_left, struct btrfs_dev_replace_item, + cursor_left, 64); +BTRFS_SETGET_FUNCS(dev_replace_cursor_right, struct btrfs_dev_replace_item, + cursor_right, 64); + +BTRFS_SETGET_STACK_FUNCS(stack_dev_replace_src_devid, + struct btrfs_dev_replace_item, src_devid, 64); +BTRFS_SETGET_STACK_FUNCS(stack_dev_replace_cont_reading_from_srcdev_mode, + struct btrfs_dev_replace_item, + cont_reading_from_srcdev_mode, 64); +BTRFS_SETGET_STACK_FUNCS(stack_dev_replace_replace_state, + struct btrfs_dev_replace_item, replace_state, 64); +BTRFS_SETGET_STACK_FUNCS(stack_dev_replace_time_started, + struct btrfs_dev_replace_item, time_started, 64); +BTRFS_SETGET_STACK_FUNCS(stack_dev_replace_time_stopped, + struct btrfs_dev_replace_item, time_stopped, 64); +BTRFS_SETGET_STACK_FUNCS(stack_dev_replace_num_write_errors, + struct btrfs_dev_replace_item, num_write_errors, 64); +BTRFS_SETGET_STACK_FUNCS(stack_dev_replace_num_uncorrectable_read_errors, + struct btrfs_dev_replace_item, + num_uncorrectable_read_errors, 64); +BTRFS_SETGET_STACK_FUNCS(stack_dev_replace_cursor_left, + struct btrfs_dev_replace_item, cursor_left, 64); +BTRFS_SETGET_STACK_FUNCS(stack_dev_replace_cursor_right, + struct btrfs_dev_replace_item, cursor_right, 64); + +/* btrfs_verity_descriptor_item */ +BTRFS_SETGET_FUNCS(verity_descriptor_encryption, struct btrfs_verity_descriptor_item, + encryption, 8); +BTRFS_SETGET_FUNCS(verity_descriptor_size, struct btrfs_verity_descriptor_item, + size, 64); +BTRFS_SETGET_STACK_FUNCS(stack_verity_descriptor_encryption, + struct btrfs_verity_descriptor_item, encryption, 8); +BTRFS_SETGET_STACK_FUNCS(stack_verity_descriptor_size, + struct btrfs_verity_descriptor_item, size, 64); + +BTRFS_SETGET_FUNCS(remap_address, struct btrfs_remap_item, address, 64); +BTRFS_SETGET_STACK_FUNCS(stack_remap_address, struct btrfs_remap_item, + address, 64); + +/* Cast into the data area of the leaf. */ +#define btrfs_item_ptr(leaf, slot, type) \ + ((type *)(btrfs_item_nr_offset(leaf, 0) + btrfs_item_offset(leaf, slot))) + +#define btrfs_item_ptr_offset(leaf, slot) \ + ((unsigned long)(btrfs_item_nr_offset(leaf, 0) + btrfs_item_offset(leaf, slot))) + +#endif diff --git a/fs/btrfs/acl.c b/fs/btrfs/acl.c new file mode 100644 index 000000000..662cdd1cb --- /dev/null +++ b/fs/btrfs/acl.c @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2007 Red Hat. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "ctree.h" +#include "xattr.h" +#include "acl.h" +#include "misc.h" +#include "btrfs_inode.h" + +struct posix_acl *btrfs_get_acl(struct inode *inode, int type, bool rcu) +{ + int size; + const char *name; + char AUTO_KFREE(value); + struct posix_acl *acl; + + if (rcu) + return ERR_PTR(-ECHILD); + + switch (type) { + case ACL_TYPE_ACCESS: + name = XATTR_NAME_POSIX_ACL_ACCESS; + break; + case ACL_TYPE_DEFAULT: + name = XATTR_NAME_POSIX_ACL_DEFAULT; + break; + default: + return ERR_PTR(-EINVAL); + } + + size = btrfs_getxattr(inode, name, NULL, 0); + if (size > 0) { + value = kzalloc(size, GFP_KERNEL); + if (!value) + return ERR_PTR(-ENOMEM); + size = btrfs_getxattr(inode, name, value, size); + } + if (size > 0) + acl = posix_acl_from_xattr(&init_user_ns, value, size); + else if (size == -ENODATA || size == 0) + acl = NULL; + else + acl = ERR_PTR(size); + + return acl; +} + +int __btrfs_set_acl(struct btrfs_trans_handle *trans, struct inode *inode, + struct posix_acl *acl, int type) +{ + int ret; + size_t size = 0; + const char *name; + char AUTO_KFREE(value); + + switch (type) { + case ACL_TYPE_ACCESS: + name = XATTR_NAME_POSIX_ACL_ACCESS; + break; + case ACL_TYPE_DEFAULT: + if (!S_ISDIR(inode->i_mode)) + return acl ? -EINVAL : 0; + name = XATTR_NAME_POSIX_ACL_DEFAULT; + break; + default: + return -EINVAL; + } + + if (acl) { + unsigned int nofs_flag; + + /* + * We're holding a transaction handle, so use a NOFS memory + * allocation context to avoid deadlock if reclaim happens. + */ + nofs_flag = memalloc_nofs_save(); + value = posix_acl_to_xattr(&init_user_ns, acl, &size, GFP_KERNEL); + memalloc_nofs_restore(nofs_flag); + if (!value) + return -ENOMEM; + } + + if (trans) + ret = btrfs_setxattr(trans, inode, name, value, size, 0); + else + ret = btrfs_setxattr_trans(inode, name, value, size, 0); + if (ret < 0) + return ret; + + set_cached_acl(inode, type, acl); + return 0; +} + +int btrfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type) +{ + int ret; + struct inode *inode = d_inode(dentry); + umode_t old_mode = inode->i_mode; + + if (btrfs_root_readonly(BTRFS_I(inode)->root)) + return -EROFS; + + if (type == ACL_TYPE_ACCESS && acl) { + ret = posix_acl_update_mode(idmap, inode, + &inode->i_mode, &acl); + if (ret) + return ret; + } + ret = __btrfs_set_acl(NULL, inode, acl, type); + if (ret) + inode->i_mode = old_mode; + return ret; +} diff --git a/fs/btrfs/acl.h b/fs/btrfs/acl.h new file mode 100644 index 000000000..0458cd51e --- /dev/null +++ b/fs/btrfs/acl.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_ACL_H +#define BTRFS_ACL_H + +#include + +struct posix_acl; +struct inode; +struct btrfs_trans_handle; + +#ifdef CONFIG_BTRFS_FS_POSIX_ACL + +struct mnt_idmap; +struct dentry; + +struct posix_acl *btrfs_get_acl(struct inode *inode, int type, bool rcu); +int btrfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type); +int __btrfs_set_acl(struct btrfs_trans_handle *trans, struct inode *inode, + struct posix_acl *acl, int type); + +#else + +#include + +struct btrfs_trans_handle; + +#define btrfs_get_acl NULL +#define btrfs_set_acl NULL +static inline int __btrfs_set_acl(struct btrfs_trans_handle *trans, + struct inode *inode, struct posix_acl *acl, + int type) +{ + return -EOPNOTSUPP; +} + +#endif + +#endif diff --git a/fs/btrfs/async-thread.c b/fs/btrfs/async-thread.c new file mode 100644 index 000000000..e6f33d094 --- /dev/null +++ b/fs/btrfs/async-thread.c @@ -0,0 +1,372 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2007 Oracle. All rights reserved. + * Copyright (C) 2014 Fujitsu. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include "async-thread.h" + +enum { + WORK_DONE_BIT, + WORK_ORDER_DONE_BIT, +}; + +#define NO_THRESHOLD (-1) +#define DEFAULT_THRESHOLD (32) + +struct btrfs_workqueue { + struct workqueue_struct *normal_wq; + + /* File system this workqueue services */ + struct btrfs_fs_info *fs_info; + + /* List head pointing to ordered work list */ + struct list_head ordered_list; + + /* Spinlock for ordered_list */ + spinlock_t list_lock; + + /* Thresholding related variants */ + atomic_t pending; + + /* Up limit of concurrency workers */ + int limit_active; + + /* Current number of concurrency workers */ + int current_active; + + /* Threshold to change current_active */ + int thresh; + unsigned int count; + spinlock_t thres_lock; +}; + +struct btrfs_fs_info * __pure btrfs_workqueue_owner(const struct btrfs_workqueue *wq) +{ + return wq->fs_info; +} + +struct btrfs_fs_info * __pure btrfs_work_owner(const struct btrfs_work *work) +{ + return work->wq->fs_info; +} + +bool btrfs_workqueue_normal_congested(const struct btrfs_workqueue *wq) +{ + /* + * We could compare wq->pending with num_online_cpus() + * to support "thresh == NO_THRESHOLD" case, but it requires + * moving up atomic_inc/dec in thresh_queue/exec_hook. Let's + * postpone it until someone needs the support of that case. + */ + if (wq->thresh == NO_THRESHOLD) + return false; + + return atomic_read(&wq->pending) > wq->thresh * 2; +} + +static void btrfs_init_workqueue(struct btrfs_workqueue *wq, + struct btrfs_fs_info *fs_info) +{ + wq->fs_info = fs_info; + atomic_set(&wq->pending, 0); + INIT_LIST_HEAD(&wq->ordered_list); + spin_lock_init(&wq->list_lock); + spin_lock_init(&wq->thres_lock); +} + +struct btrfs_workqueue *btrfs_alloc_workqueue(struct btrfs_fs_info *fs_info, + const char *name, unsigned int flags, + int limit_active, int thresh) +{ + struct btrfs_workqueue *ret = kzalloc_obj(*ret); + + if (!ret) + return NULL; + + btrfs_init_workqueue(ret, fs_info); + + ret->limit_active = limit_active; + if (thresh == 0) + thresh = DEFAULT_THRESHOLD; + /* For low threshold, disabling threshold is a better choice */ + if (thresh < DEFAULT_THRESHOLD) { + ret->current_active = limit_active; + ret->thresh = NO_THRESHOLD; + } else { + /* + * For threshold-able wq, let its concurrency grow on demand. + * Use minimal max_active at alloc time to reduce resource + * usage. + */ + ret->current_active = 1; + ret->thresh = thresh; + } + + ret->normal_wq = alloc_workqueue("btrfs-%s", flags, ret->current_active, + name); + if (!ret->normal_wq) { + kfree(ret); + return NULL; + } + + trace_btrfs_workqueue_alloc(ret, name); + return ret; +} + +struct btrfs_workqueue *btrfs_alloc_ordered_workqueue( + struct btrfs_fs_info *fs_info, const char *name, + unsigned int flags) +{ + struct btrfs_workqueue *ret; + + ret = kzalloc_obj(*ret); + if (!ret) + return NULL; + + btrfs_init_workqueue(ret, fs_info); + + /* Ordered workqueues don't allow @max_active adjustments. */ + ret->limit_active = 1; + ret->current_active = 1; + ret->thresh = NO_THRESHOLD; + + ret->normal_wq = alloc_ordered_workqueue("btrfs-%s", flags, name); + if (!ret->normal_wq) { + kfree(ret); + return NULL; + } + + trace_btrfs_workqueue_alloc(ret, name); + return ret; +} + +/* + * Hook for threshold which will be called in btrfs_queue_work. + * This hook WILL be called in IRQ handler context, + * so workqueue_set_max_active MUST NOT be called in this hook + */ +static inline void thresh_queue_hook(struct btrfs_workqueue *wq) +{ + if (wq->thresh == NO_THRESHOLD) + return; + atomic_inc(&wq->pending); +} + +/* + * Hook for threshold which will be called before executing the work, + * This hook is called in kthread content. + * So workqueue_set_max_active is called here. + */ +static inline void thresh_exec_hook(struct btrfs_workqueue *wq) +{ + int new_current_active; + long pending; + bool need_change = false; + + if (wq->thresh == NO_THRESHOLD) + return; + + atomic_dec(&wq->pending); + spin_lock(&wq->thres_lock); + /* + * Use wq->count to limit the calling frequency of + * workqueue_set_max_active. + */ + wq->count++; + wq->count %= (wq->thresh / 4); + if (!wq->count) + goto out; + new_current_active = wq->current_active; + + /* + * pending may be changed later, but it's OK since we really + * don't need it so accurate to calculate new_max_active. + */ + pending = atomic_read(&wq->pending); + if (pending > wq->thresh) + new_current_active++; + if (pending < wq->thresh / 2) + new_current_active--; + new_current_active = clamp_val(new_current_active, 1, wq->limit_active); + if (new_current_active != wq->current_active) { + need_change = true; + wq->current_active = new_current_active; + } +out: + spin_unlock(&wq->thres_lock); + + if (need_change) + workqueue_set_max_active(wq->normal_wq, wq->current_active); +} + +static void run_ordered_work(struct btrfs_workqueue *wq, + struct btrfs_work *self) +{ + struct list_head *list = &wq->ordered_list; + struct btrfs_work *work; + spinlock_t *lock = &wq->list_lock; + unsigned long flags; + bool free_self = false; + + while (1) { + spin_lock_irqsave(lock, flags); + if (list_empty(list)) + break; + work = list_first_entry(list, struct btrfs_work, ordered_list); + if (!test_bit(WORK_DONE_BIT, &work->flags)) + break; + /* + * Orders all subsequent loads after reading WORK_DONE_BIT, + * paired with the smp_mb__before_atomic in btrfs_work_helper + * this guarantees that the ordered function will see all + * updates from ordinary work function. + */ + smp_rmb(); + + /* + * we are going to call the ordered done function, but + * we leave the work item on the list as a barrier so + * that later work items that are done don't have their + * functions called before this one returns + */ + if (test_and_set_bit(WORK_ORDER_DONE_BIT, &work->flags)) + break; + trace_btrfs_ordered_sched(work); + spin_unlock_irqrestore(lock, flags); + work->ordered_func(work, false); + + /* now take the lock again and drop our item from the list */ + spin_lock_irqsave(lock, flags); + list_del(&work->ordered_list); + spin_unlock_irqrestore(lock, flags); + + if (work == self) { + /* + * This is the work item that the worker is currently + * executing. + * + * The kernel workqueue code guarantees non-reentrancy + * of work items. I.e., if a work item with the same + * address and work function is queued twice, the second + * execution is blocked until the first one finishes. A + * work item may be freed and recycled with the same + * work function; the workqueue code assumes that the + * original work item cannot depend on the recycled work + * item in that case (see find_worker_executing_work()). + * + * Note that different types of Btrfs work can depend on + * each other, and one type of work on one Btrfs + * filesystem may even depend on the same type of work + * on another Btrfs filesystem via, e.g., a loop device. + * Therefore, we must not allow the current work item to + * be recycled until we are really done, otherwise we + * break the above assumption and can deadlock. + */ + free_self = true; + } else { + /* + * We don't want to call the ordered free functions with + * the lock held. + */ + work->ordered_func(work, true); + /* NB: work must not be dereferenced past this point. */ + trace_btrfs_all_work_done(wq->fs_info, work); + } + } + spin_unlock_irqrestore(lock, flags); + + if (free_self) { + self->ordered_func(self, true); + /* NB: self must not be dereferenced past this point. */ + trace_btrfs_all_work_done(wq->fs_info, self); + } +} + +static void btrfs_work_helper(struct work_struct *normal_work) +{ + struct btrfs_work *work = container_of(normal_work, struct btrfs_work, + normal_work); + struct btrfs_workqueue *wq = work->wq; + bool need_order = false; + + /* + * We should not touch things inside work in the following cases: + * 1) after work->func() if it has no ordered_func(..., true) to free + * Since the struct is freed in work->func(). + * 2) after setting WORK_DONE_BIT + * The work may be freed in other threads almost instantly. + * So we save the needed things here. + */ + if (work->ordered_func) + need_order = true; + + trace_btrfs_work_sched(work); + thresh_exec_hook(wq); + work->func(work); + if (need_order) { + /* + * Ensures all memory accesses done in the work function are + * ordered before setting the WORK_DONE_BIT. Ensuring the thread + * which is going to executed the ordered work sees them. + * Pairs with the smp_rmb in run_ordered_work. + */ + smp_mb__before_atomic(); + set_bit(WORK_DONE_BIT, &work->flags); + run_ordered_work(wq, work); + } else { + /* NB: work must not be dereferenced past this point. */ + trace_btrfs_all_work_done(wq->fs_info, work); + } +} + +void btrfs_init_work(struct btrfs_work *work, btrfs_func_t func, + btrfs_ordered_func_t ordered_func) +{ + work->func = func; + work->ordered_func = ordered_func; + INIT_WORK(&work->normal_work, btrfs_work_helper); + INIT_LIST_HEAD(&work->ordered_list); + work->flags = 0; +} + +void btrfs_queue_work(struct btrfs_workqueue *wq, struct btrfs_work *work) +{ + unsigned long flags; + + work->wq = wq; + thresh_queue_hook(wq); + if (work->ordered_func) { + spin_lock_irqsave(&wq->list_lock, flags); + list_add_tail(&work->ordered_list, &wq->ordered_list); + spin_unlock_irqrestore(&wq->list_lock, flags); + } + trace_btrfs_work_queued(work); + queue_work(wq->normal_wq, &work->normal_work); +} + +void btrfs_destroy_workqueue(struct btrfs_workqueue *wq) +{ + if (!wq) + return; + destroy_workqueue(wq->normal_wq); + trace_btrfs_workqueue_destroy(wq); + kfree(wq); +} + +void btrfs_workqueue_set_max(struct btrfs_workqueue *wq, int limit_active) +{ + if (wq) + wq->limit_active = limit_active; +} + +void btrfs_flush_workqueue(struct btrfs_workqueue *wq) +{ + flush_workqueue(wq->normal_wq); +} diff --git a/fs/btrfs/async-thread.h b/fs/btrfs/async-thread.h new file mode 100644 index 000000000..04c2f3175 --- /dev/null +++ b/fs/btrfs/async-thread.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2007 Oracle. All rights reserved. + * Copyright (C) 2014 Fujitsu. All rights reserved. + */ + +#ifndef BTRFS_ASYNC_THREAD_H +#define BTRFS_ASYNC_THREAD_H + +#include +#include +#include + +struct btrfs_fs_info; +struct btrfs_workqueue; +struct btrfs_work; + +typedef void (*btrfs_func_t)(struct btrfs_work *arg); +typedef void (*btrfs_ordered_func_t)(struct btrfs_work *arg, bool); + +struct btrfs_work { + btrfs_func_t func; + btrfs_ordered_func_t ordered_func; + + /* Don't touch things below */ + struct work_struct normal_work; + struct list_head ordered_list; + struct btrfs_workqueue *wq; + unsigned long flags; +}; + +struct btrfs_workqueue *btrfs_alloc_workqueue(struct btrfs_fs_info *fs_info, + const char *name, + unsigned int flags, + int limit_active, + int thresh); +struct btrfs_workqueue *btrfs_alloc_ordered_workqueue( + struct btrfs_fs_info *fs_info, const char *name, + unsigned int flags); +void btrfs_init_work(struct btrfs_work *work, btrfs_func_t func, + btrfs_ordered_func_t ordered_func); +void btrfs_queue_work(struct btrfs_workqueue *wq, + struct btrfs_work *work); +void btrfs_destroy_workqueue(struct btrfs_workqueue *wq); +void btrfs_workqueue_set_max(struct btrfs_workqueue *wq, int max); +struct btrfs_fs_info * __pure btrfs_work_owner(const struct btrfs_work *work); +struct btrfs_fs_info * __pure btrfs_workqueue_owner(const struct btrfs_workqueue *wq); +bool btrfs_workqueue_normal_congested(const struct btrfs_workqueue *wq); +void btrfs_flush_workqueue(struct btrfs_workqueue *wq); + +#endif diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c new file mode 100644 index 000000000..1be632c74 --- /dev/null +++ b/fs/btrfs/backref.c @@ -0,0 +1,3697 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2011 STRATO. All rights reserved. + */ + +#include +#include +#include +#include "ctree.h" +#include "disk-io.h" +#include "backref.h" +#include "ulist.h" +#include "transaction.h" +#include "delayed-ref.h" +#include "locking.h" +#include "misc.h" +#include "tree-mod-log.h" +#include "fs.h" +#include "accessors.h" +#include "extent-tree.h" +#include "relocation.h" +#include "tree-checker.h" + +/* Just arbitrary numbers so we can be sure one of these happened. */ +#define BACKREF_FOUND_SHARED 6 +#define BACKREF_FOUND_NOT_SHARED 7 + +struct extent_inode_elem { + u64 inum; + u64 offset; + u64 num_bytes; + struct extent_inode_elem *next; +}; + +static int check_extent_in_eb(struct btrfs_backref_walk_ctx *ctx, + const struct btrfs_key *key, + const struct extent_buffer *eb, + const struct btrfs_file_extent_item *fi, + struct extent_inode_elem **eie) +{ + const u64 data_len = btrfs_file_extent_num_bytes(eb, fi); + u64 offset = key->offset; + struct extent_inode_elem *e; + const u64 *root_ids; + int root_count; + bool cached; + + if (!ctx->ignore_extent_item_pos && + !btrfs_file_extent_compression(eb, fi) && + !btrfs_file_extent_encryption(eb, fi) && + !btrfs_file_extent_other_encoding(eb, fi)) { + u64 data_offset; + + data_offset = btrfs_file_extent_offset(eb, fi); + + if (ctx->extent_item_pos < data_offset || + ctx->extent_item_pos >= data_offset + data_len) + return 1; + offset += ctx->extent_item_pos - data_offset; + } + + if (!ctx->indirect_ref_iterator || !ctx->cache_lookup) + goto add_inode_elem; + + cached = ctx->cache_lookup(eb->start, ctx->user_ctx, &root_ids, + &root_count); + if (!cached) + goto add_inode_elem; + + for (int i = 0; i < root_count; i++) { + int ret; + + ret = ctx->indirect_ref_iterator(key->objectid, offset, + data_len, root_ids[i], + ctx->user_ctx); + if (ret) + return ret; + } + +add_inode_elem: + e = kmalloc_obj(*e, GFP_NOFS); + if (!e) + return -ENOMEM; + + e->next = *eie; + e->inum = key->objectid; + e->offset = offset; + e->num_bytes = data_len; + *eie = e; + + return 0; +} + +static void free_inode_elem_list(struct extent_inode_elem *eie) +{ + struct extent_inode_elem *eie_next; + + for (; eie; eie = eie_next) { + eie_next = eie->next; + kfree(eie); + } +} + +static int find_extent_in_eb(struct btrfs_backref_walk_ctx *ctx, + const struct extent_buffer *eb, + struct extent_inode_elem **eie) +{ + u64 disk_byte; + struct btrfs_key key; + struct btrfs_file_extent_item *fi; + int slot; + int nritems; + int extent_type; + int ret; + + /* + * from the shared data ref, we only have the leaf but we need + * the key. thus, we must look into all items and see that we + * find one (some) with a reference to our extent item. + */ + nritems = btrfs_header_nritems(eb); + for (slot = 0; slot < nritems; ++slot) { + btrfs_item_key_to_cpu(eb, &key, slot); + if (key.type != BTRFS_EXTENT_DATA_KEY) + continue; + fi = btrfs_item_ptr(eb, slot, struct btrfs_file_extent_item); + extent_type = btrfs_file_extent_type(eb, fi); + if (extent_type == BTRFS_FILE_EXTENT_INLINE) + continue; + /* don't skip BTRFS_FILE_EXTENT_PREALLOC, we can handle that */ + disk_byte = btrfs_file_extent_disk_bytenr(eb, fi); + if (disk_byte != ctx->bytenr) + continue; + + ret = check_extent_in_eb(ctx, &key, eb, fi, eie); + if (ret == BTRFS_ITERATE_EXTENT_INODES_STOP || ret < 0) + return ret; + } + + return 0; +} + +struct preftree { + struct rb_root_cached root; + unsigned int count; +}; + +#define PREFTREE_INIT { .root = RB_ROOT_CACHED, .count = 0 } + +struct preftrees { + struct preftree direct; /* BTRFS_SHARED_[DATA|BLOCK]_REF_KEY */ + struct preftree indirect; /* BTRFS_[TREE_BLOCK|EXTENT_DATA]_REF_KEY */ + struct preftree indirect_missing_keys; +}; + +/* + * Checks for a shared extent during backref search. + * + * The share_count tracks prelim_refs (direct and indirect) having a + * ref->count >0: + * - incremented when a ref->count transitions to >0 + * - decremented when a ref->count transitions to <1 + */ +struct share_check { + struct btrfs_backref_share_check_ctx *ctx; + struct btrfs_root *root; + u64 inum; + u64 data_bytenr; + u64 data_extent_gen; + /* + * Counts number of inodes that refer to an extent (different inodes in + * the same root or different roots) that we could find. The sharedness + * check typically stops once this counter gets greater than 1, so it + * may not reflect the total number of inodes. + */ + int share_count; + /* + * The number of times we found our inode refers to the data extent we + * are determining the sharedness. In other words, how many file extent + * items we could find for our inode that point to our target data + * extent. The value we get here after finishing the extent sharedness + * check may be smaller than reality, but if it ends up being greater + * than 1, then we know for sure the inode has multiple file extent + * items that point to our inode, and we can safely assume it's useful + * to cache the sharedness check result. + */ + int self_ref_count; + bool have_delayed_delete_refs; +}; + +static inline int extent_is_shared(struct share_check *sc) +{ + return (sc && sc->share_count > 1) ? BACKREF_FOUND_SHARED : 0; +} + +static struct kmem_cache *btrfs_prelim_ref_cache; + +int __init btrfs_prelim_ref_init(void) +{ + btrfs_prelim_ref_cache = kmem_cache_create("btrfs_prelim_ref", + sizeof(struct prelim_ref), 0, 0, NULL); + if (!btrfs_prelim_ref_cache) + return -ENOMEM; + return 0; +} + +void __cold btrfs_prelim_ref_exit(void) +{ + kmem_cache_destroy(btrfs_prelim_ref_cache); +} + +static void free_pref(struct prelim_ref *ref) +{ + kmem_cache_free(btrfs_prelim_ref_cache, ref); +} + +/* + * Return 0 when both refs are for the same block (and can be merged). + * A -1 return indicates ref1 is a 'lower' block than ref2, while 1 + * indicates a 'higher' block. + */ +static int prelim_ref_compare(const struct prelim_ref *ref1, + const struct prelim_ref *ref2) +{ + if (ref1->level < ref2->level) + return -1; + if (ref1->level > ref2->level) + return 1; + if (ref1->root_id < ref2->root_id) + return -1; + if (ref1->root_id > ref2->root_id) + return 1; + if (ref1->key_for_search.type < ref2->key_for_search.type) + return -1; + if (ref1->key_for_search.type > ref2->key_for_search.type) + return 1; + if (ref1->key_for_search.objectid < ref2->key_for_search.objectid) + return -1; + if (ref1->key_for_search.objectid > ref2->key_for_search.objectid) + return 1; + if (ref1->key_for_search.offset < ref2->key_for_search.offset) + return -1; + if (ref1->key_for_search.offset > ref2->key_for_search.offset) + return 1; + if (ref1->parent < ref2->parent) + return -1; + if (ref1->parent > ref2->parent) + return 1; + + return 0; +} + +static int prelim_ref_rb_add_cmp(const struct rb_node *new, + const struct rb_node *exist) +{ + const struct prelim_ref *ref_new = + rb_entry(new, struct prelim_ref, rbnode); + const struct prelim_ref *ref_exist = + rb_entry(exist, struct prelim_ref, rbnode); + + /* + * prelim_ref_compare() expects the first parameter as the existing one, + * different from the rb_find_add_cached() order. + */ + return prelim_ref_compare(ref_exist, ref_new); +} + +static void update_share_count(struct share_check *sc, int oldcount, + int newcount, const struct prelim_ref *newref) +{ + if ((!sc) || (oldcount == 0 && newcount < 1)) + return; + + if (oldcount > 0 && newcount < 1) + sc->share_count--; + else if (oldcount < 1 && newcount > 0) + sc->share_count++; + + if (newref->root_id == btrfs_root_id(sc->root) && + newref->wanted_disk_byte == sc->data_bytenr && + newref->key_for_search.objectid == sc->inum) + sc->self_ref_count += newref->count; +} + +/* + * Add @newref to the @root rbtree, merging identical refs. + * + * Callers should assume that newref has been freed after calling. + */ +static void prelim_ref_insert(const struct btrfs_fs_info *fs_info, + struct preftree *preftree, + struct prelim_ref *newref, + struct share_check *sc) +{ + struct rb_root_cached *root; + struct rb_node *exist; + + root = &preftree->root; + exist = rb_find_add_cached(&newref->rbnode, root, prelim_ref_rb_add_cmp); + if (exist) { + struct prelim_ref *ref = rb_entry(exist, struct prelim_ref, rbnode); + /* Identical refs, merge them and free @newref */ + struct extent_inode_elem *eie = ref->inode_list; + + while (eie && eie->next) + eie = eie->next; + + if (!eie) + ref->inode_list = newref->inode_list; + else + eie->next = newref->inode_list; + trace_btrfs_prelim_ref_merge(fs_info, ref, newref, + preftree->count); + /* + * A delayed ref can have newref->count < 0. + * The ref->count is updated to follow any + * BTRFS_[ADD|DROP]_DELAYED_REF actions. + */ + update_share_count(sc, ref->count, + ref->count + newref->count, newref); + ref->count += newref->count; + free_pref(newref); + return; + } + + update_share_count(sc, 0, newref->count, newref); + preftree->count++; + trace_btrfs_prelim_ref_insert(fs_info, newref, NULL, preftree->count); +} + +/* + * Release the entire tree. We don't care about internal consistency so + * just free everything and then reset the tree root. + */ +static void prelim_release(struct preftree *preftree) +{ + struct prelim_ref *ref, *next_ref; + + rbtree_postorder_for_each_entry_safe(ref, next_ref, + &preftree->root.rb_root, rbnode) { + free_inode_elem_list(ref->inode_list); + free_pref(ref); + } + + preftree->root = RB_ROOT_CACHED; + preftree->count = 0; +} + +/* + * the rules for all callers of this function are: + * - obtaining the parent is the goal + * - if you add a key, you must know that it is a correct key + * - if you cannot add the parent or a correct key, then we will look into the + * block later to set a correct key + * + * delayed refs + * ============ + * backref type | shared | indirect | shared | indirect + * information | tree | tree | data | data + * --------------------+--------+----------+--------+---------- + * parent logical | y | - | - | - + * key to resolve | - | y | y | y + * tree block logical | - | - | - | - + * root for resolving | y | y | y | y + * + * - column 1: we've the parent -> done + * - column 2, 3, 4: we use the key to find the parent + * + * on disk refs (inline or keyed) + * ============================== + * backref type | shared | indirect | shared | indirect + * information | tree | tree | data | data + * --------------------+--------+----------+--------+---------- + * parent logical | y | - | y | - + * key to resolve | - | - | - | y + * tree block logical | y | y | y | y + * root for resolving | - | y | y | y + * + * - column 1, 3: we've the parent -> done + * - column 2: we take the first key from the block to find the parent + * (see add_missing_keys) + * - column 4: we use the key to find the parent + * + * additional information that's available but not required to find the parent + * block might help in merging entries to gain some speed. + */ +static int add_prelim_ref(const struct btrfs_fs_info *fs_info, + struct preftree *preftree, u64 root_id, + const struct btrfs_key *key, int level, u64 parent, + u64 wanted_disk_byte, int count, + struct share_check *sc, gfp_t gfp_mask) +{ + struct prelim_ref *ref; + + if (root_id == BTRFS_DATA_RELOC_TREE_OBJECTID) + return 0; + + ref = kmem_cache_alloc(btrfs_prelim_ref_cache, gfp_mask); + if (!ref) + return -ENOMEM; + + ref->root_id = root_id; + if (key) + ref->key_for_search = *key; + else + memset(&ref->key_for_search, 0, sizeof(ref->key_for_search)); + + ref->inode_list = NULL; + ref->level = level; + ref->count = count; + ref->parent = parent; + ref->wanted_disk_byte = wanted_disk_byte; + prelim_ref_insert(fs_info, preftree, ref, sc); + return extent_is_shared(sc); +} + +/* direct refs use root == 0, key == NULL */ +static int add_direct_ref(const struct btrfs_fs_info *fs_info, + struct preftrees *preftrees, int level, u64 parent, + u64 wanted_disk_byte, int count, + struct share_check *sc, gfp_t gfp_mask) +{ + return add_prelim_ref(fs_info, &preftrees->direct, 0, NULL, level, + parent, wanted_disk_byte, count, sc, gfp_mask); +} + +/* indirect refs use parent == 0 */ +static int add_indirect_ref(const struct btrfs_fs_info *fs_info, + struct preftrees *preftrees, u64 root_id, + const struct btrfs_key *key, int level, + u64 wanted_disk_byte, int count, + struct share_check *sc, gfp_t gfp_mask) +{ + struct preftree *tree = &preftrees->indirect; + + if (!key) + tree = &preftrees->indirect_missing_keys; + return add_prelim_ref(fs_info, tree, root_id, key, level, 0, + wanted_disk_byte, count, sc, gfp_mask); +} + +static int is_shared_data_backref(struct preftrees *preftrees, u64 bytenr) +{ + struct rb_node **p = &preftrees->direct.root.rb_root.rb_node; + struct rb_node *parent = NULL; + struct prelim_ref *ref = NULL; + struct prelim_ref target = {}; + int result; + + target.parent = bytenr; + + while (*p) { + parent = *p; + ref = rb_entry(parent, struct prelim_ref, rbnode); + result = prelim_ref_compare(ref, &target); + + if (result < 0) + p = &(*p)->rb_left; + else if (result > 0) + p = &(*p)->rb_right; + else + return 1; + } + return 0; +} + +static int add_all_parents(struct btrfs_backref_walk_ctx *ctx, + struct btrfs_root *root, struct btrfs_path *path, + struct ulist *parents, + struct preftrees *preftrees, struct prelim_ref *ref, + int level) +{ + int ret = 0; + int slot; + struct extent_buffer *eb; + struct btrfs_key key; + struct btrfs_key *key_for_search = &ref->key_for_search; + struct btrfs_file_extent_item *fi; + struct extent_inode_elem *eie = NULL, *old = NULL; + u64 disk_byte; + u64 wanted_disk_byte = ref->wanted_disk_byte; + u64 count = 0; + u64 data_offset; + u8 type; + + if (level != 0) { + eb = path->nodes[level]; + ret = ulist_add(parents, eb->start, 0, GFP_NOFS); + if (ret < 0) + return ret; + return 0; + } + + /* + * 1. We normally enter this function with the path already pointing to + * the first item to check. But sometimes, we may enter it with + * slot == nritems. + * 2. We are searching for normal backref but bytenr of this leaf + * matches shared data backref + * 3. The leaf owner is not equal to the root we are searching + * + * For these cases, go to the next leaf before we continue. + */ + eb = path->nodes[0]; + if (path->slots[0] >= btrfs_header_nritems(eb) || + is_shared_data_backref(preftrees, eb->start) || + ref->root_id != btrfs_header_owner(eb)) { + if (ctx->time_seq == BTRFS_SEQ_LAST) + ret = btrfs_next_leaf(root, path); + else + ret = btrfs_next_old_leaf(root, path, ctx->time_seq); + } + + while (!ret && count < ref->count) { + eb = path->nodes[0]; + slot = path->slots[0]; + + btrfs_item_key_to_cpu(eb, &key, slot); + + if (key.objectid != key_for_search->objectid || + key.type != BTRFS_EXTENT_DATA_KEY) + break; + + /* + * We are searching for normal backref but bytenr of this leaf + * matches shared data backref, OR + * the leaf owner is not equal to the root we are searching for + */ + if (slot == 0 && + (is_shared_data_backref(preftrees, eb->start) || + ref->root_id != btrfs_header_owner(eb))) { + if (ctx->time_seq == BTRFS_SEQ_LAST) + ret = btrfs_next_leaf(root, path); + else + ret = btrfs_next_old_leaf(root, path, ctx->time_seq); + continue; + } + fi = btrfs_item_ptr(eb, slot, struct btrfs_file_extent_item); + type = btrfs_file_extent_type(eb, fi); + if (type == BTRFS_FILE_EXTENT_INLINE) + goto next; + disk_byte = btrfs_file_extent_disk_bytenr(eb, fi); + data_offset = btrfs_file_extent_offset(eb, fi); + + if (disk_byte == wanted_disk_byte) { + eie = NULL; + old = NULL; + if (ref->key_for_search.offset == key.offset - data_offset) + count++; + else + goto next; + if (!ctx->skip_inode_ref_list) { + ret = check_extent_in_eb(ctx, &key, eb, fi, &eie); + if (ret == BTRFS_ITERATE_EXTENT_INODES_STOP || + ret < 0) + break; + } + if (ret > 0) + goto next; + ret = ulist_add_merge_ptr(parents, eb->start, + eie, (void **)&old, GFP_NOFS); + if (ret < 0) + break; + if (!ret && !ctx->skip_inode_ref_list) { + while (old->next) + old = old->next; + old->next = eie; + } + eie = NULL; + } +next: + if (ctx->time_seq == BTRFS_SEQ_LAST) + ret = btrfs_next_item(root, path); + else + ret = btrfs_next_old_item(root, path, ctx->time_seq); + } + + if (ret == BTRFS_ITERATE_EXTENT_INODES_STOP || ret < 0) + free_inode_elem_list(eie); + else if (ret > 0) + ret = 0; + + return ret; +} + +/* + * resolve an indirect backref in the form (root_id, key, level) + * to a logical address + */ +static int resolve_indirect_ref(struct btrfs_backref_walk_ctx *ctx, + struct btrfs_path *path, + struct preftrees *preftrees, + struct prelim_ref *ref, struct ulist *parents) +{ + struct btrfs_root *root; + struct extent_buffer *eb; + int ret = 0; + int root_level; + int level = ref->level; + struct btrfs_key search_key = ref->key_for_search; + + /* + * If we're search_commit_root we could possibly be holding locks on + * other tree nodes. This happens when qgroups does backref walks when + * adding new delayed refs. To deal with this we need to look in cache + * for the root, and if we don't find it then we need to search the + * tree_root's commit root, thus the btrfs_get_fs_root_commit_root usage + * here. + */ + if (path->search_commit_root) + root = btrfs_get_fs_root_commit_root(ctx->fs_info, path, ref->root_id); + else + root = btrfs_get_fs_root(ctx->fs_info, ref->root_id, false); + if (IS_ERR(root)) { + ret = PTR_ERR(root); + goto out_free; + } + + if (!path->search_commit_root && + test_bit(BTRFS_ROOT_DELETING, &root->state)) { + ret = -ENOENT; + goto out; + } + + if (btrfs_is_testing(ctx->fs_info)) { + ret = -ENOENT; + goto out; + } + + if (path->search_commit_root) + root_level = btrfs_header_level(root->commit_root); + else if (ctx->time_seq == BTRFS_SEQ_LAST) + root_level = btrfs_header_level(root->node); + else + root_level = btrfs_old_root_level(root, ctx->time_seq); + + if (root_level + 1 == level) + goto out; + + /* + * We can often find data backrefs with an offset that is too large + * (>= LLONG_MAX, maximum allowed file offset) due to underflows when + * subtracting a file's offset with the data offset of its + * corresponding extent data item. This can happen for example in the + * clone ioctl. + * + * So if we detect such case we set the search key's offset to zero to + * make sure we will find the matching file extent item at + * add_all_parents(), otherwise we will miss it because the offset + * taken form the backref is much larger then the offset of the file + * extent item. This can make us scan a very large number of file + * extent items, but at least it will not make us miss any. + * + * This is an ugly workaround for a behaviour that should have never + * existed, but it does and a fix for the clone ioctl would touch a lot + * of places, cause backwards incompatibility and would not fix the + * problem for extents cloned with older kernels. + */ + if (search_key.type == BTRFS_EXTENT_DATA_KEY && + search_key.offset >= LLONG_MAX) + search_key.offset = 0; + path->lowest_level = level; + if (ctx->time_seq == BTRFS_SEQ_LAST) + ret = btrfs_search_slot(NULL, root, &search_key, path, 0, 0); + else + ret = btrfs_search_old_slot(root, &search_key, path, ctx->time_seq); + + btrfs_debug(ctx->fs_info, +"search slot in root %llu (level %d, ref count %d) returned %d for key " BTRFS_KEY_FMT, + ref->root_id, level, ref->count, ret, + BTRFS_KEY_FMT_VALUE(&ref->key_for_search)); + if (ret < 0) + goto out; + + eb = path->nodes[level]; + while (!eb) { + if (WARN_ON(!level)) { + ret = 1; + goto out; + } + level--; + eb = path->nodes[level]; + } + + ret = add_all_parents(ctx, root, path, parents, preftrees, ref, level); +out: + btrfs_put_root(root); +out_free: + path->lowest_level = 0; + btrfs_release_path(path); + return ret; +} + +static struct extent_inode_elem * +unode_aux_to_inode_list(struct ulist_node *node) +{ + if (!node) + return NULL; + return (struct extent_inode_elem *)(uintptr_t)node->aux; +} + +static void free_leaf_list(struct ulist *ulist) +{ + struct ulist_node *node; + struct ulist_iterator uiter; + + ULIST_ITER_INIT(&uiter); + while ((node = ulist_next(ulist, &uiter))) + free_inode_elem_list(unode_aux_to_inode_list(node)); + + ulist_free(ulist); +} + +/* + * We maintain three separate rbtrees: one for direct refs, one for + * indirect refs which have a key, and one for indirect refs which do not + * have a key. Each tree does merge on insertion. + * + * Once all of the references are located, we iterate over the tree of + * indirect refs with missing keys. An appropriate key is located and + * the ref is moved onto the tree for indirect refs. After all missing + * keys are thus located, we iterate over the indirect ref tree, resolve + * each reference, and then insert the resolved reference onto the + * direct tree (merging there too). + * + * New backrefs (i.e., for parent nodes) are added to the appropriate + * rbtree as they are encountered. The new backrefs are subsequently + * resolved as above. + */ +static int resolve_indirect_refs(struct btrfs_backref_walk_ctx *ctx, + struct btrfs_path *path, + struct preftrees *preftrees, + struct share_check *sc) +{ + int ret = 0; + struct ulist *parents; + struct ulist_node *node; + struct ulist_iterator uiter; + struct rb_node *rnode; + + parents = ulist_alloc(GFP_NOFS); + if (!parents) + return -ENOMEM; + + /* + * We could trade memory usage for performance here by iterating + * the tree, allocating new refs for each insertion, and then + * freeing the entire indirect tree when we're done. In some test + * cases, the tree can grow quite large (~200k objects). + */ + while ((rnode = rb_first_cached(&preftrees->indirect.root))) { + struct prelim_ref *ref; + int ret2; + + ref = rb_entry(rnode, struct prelim_ref, rbnode); + if (WARN(ref->parent, + "BUG: direct ref found in indirect tree")) { + ret = -EINVAL; + goto out; + } + + rb_erase_cached(&ref->rbnode, &preftrees->indirect.root); + preftrees->indirect.count--; + + if (ref->count == 0) { + free_pref(ref); + continue; + } + + if (sc && ref->root_id != btrfs_root_id(sc->root)) { + free_pref(ref); + ret = BACKREF_FOUND_SHARED; + goto out; + } + ret2 = resolve_indirect_ref(ctx, path, preftrees, ref, parents); + /* + * we can only tolerate ENOENT,otherwise,we should catch error + * and return directly. + */ + if (ret2 == -ENOENT) { + prelim_ref_insert(ctx->fs_info, &preftrees->direct, ref, + NULL); + continue; + } else if (ret2) { + free_pref(ref); + ret = ret2; + goto out; + } + + /* we put the first parent into the ref at hand */ + ULIST_ITER_INIT(&uiter); + node = ulist_next(parents, &uiter); + ref->parent = node ? node->val : 0; + ref->inode_list = unode_aux_to_inode_list(node); + + /* Add a prelim_ref(s) for any other parent(s). */ + while ((node = ulist_next(parents, &uiter))) { + struct prelim_ref *new_ref; + + new_ref = kmem_cache_alloc(btrfs_prelim_ref_cache, + GFP_NOFS); + if (!new_ref) { + free_pref(ref); + ret = -ENOMEM; + goto out; + } + memcpy(new_ref, ref, sizeof(*ref)); + new_ref->parent = node->val; + new_ref->inode_list = unode_aux_to_inode_list(node); + prelim_ref_insert(ctx->fs_info, &preftrees->direct, + new_ref, NULL); + } + + /* + * Now it's a direct ref, put it in the direct tree. We must + * do this last because the ref could be merged/freed here. + */ + prelim_ref_insert(ctx->fs_info, &preftrees->direct, ref, NULL); + + ulist_reinit(parents); + cond_resched(); + } +out: + /* + * We may have inode lists attached to refs in the parents ulist, so we + * must free them before freeing the ulist and its refs. + */ + free_leaf_list(parents); + return ret; +} + +/* + * read tree blocks and add keys where required. + */ +static int add_missing_keys(struct btrfs_fs_info *fs_info, + struct preftrees *preftrees, bool lock) +{ + struct prelim_ref *ref; + struct extent_buffer *eb; + struct preftree *tree = &preftrees->indirect_missing_keys; + struct rb_node *node; + + while ((node = rb_first_cached(&tree->root))) { + struct btrfs_tree_parent_check check = { 0 }; + + ref = rb_entry(node, struct prelim_ref, rbnode); + rb_erase_cached(node, &tree->root); + + BUG_ON(ref->parent); /* should not be a direct ref */ + BUG_ON(ref->key_for_search.type); + BUG_ON(!ref->wanted_disk_byte); + + check.level = ref->level - 1; + check.owner_root = ref->root_id; + + eb = read_tree_block(fs_info, ref->wanted_disk_byte, &check); + if (IS_ERR(eb)) { + free_pref(ref); + return PTR_ERR(eb); + } + + if (lock) + btrfs_tree_read_lock(eb); + if (btrfs_header_level(eb) == 0) + btrfs_item_key_to_cpu(eb, &ref->key_for_search, 0); + else + btrfs_node_key_to_cpu(eb, &ref->key_for_search, 0); + if (lock) + btrfs_tree_read_unlock(eb); + free_extent_buffer(eb); + prelim_ref_insert(fs_info, &preftrees->indirect, ref, NULL); + cond_resched(); + } + return 0; +} + +/* + * add all currently queued delayed refs from this head whose seq nr is + * smaller or equal that seq to the list + */ +static int add_delayed_refs(const struct btrfs_fs_info *fs_info, + struct btrfs_delayed_ref_head *head, u64 seq, + struct preftrees *preftrees, struct share_check *sc) +{ + struct btrfs_delayed_ref_node *node; + struct btrfs_key key; + struct rb_node *n; + int count; + int ret = 0; + + spin_lock(&head->lock); + for (n = rb_first_cached(&head->ref_tree); n; n = rb_next(n)) { + node = rb_entry(n, struct btrfs_delayed_ref_node, + ref_node); + if (node->seq > seq) + continue; + + switch (node->action) { + case BTRFS_ADD_DELAYED_EXTENT: + case BTRFS_UPDATE_DELAYED_HEAD: + WARN_ON(1); + continue; + case BTRFS_ADD_DELAYED_REF: + count = node->ref_mod; + break; + case BTRFS_DROP_DELAYED_REF: + count = node->ref_mod * -1; + break; + default: + BUG(); + } + switch (node->type) { + case BTRFS_TREE_BLOCK_REF_KEY: { + /* NORMAL INDIRECT METADATA backref */ + struct btrfs_key *key_ptr = NULL; + /* The owner of a tree block ref is the level. */ + int level = btrfs_delayed_ref_owner(node); + + if (head->extent_op && head->extent_op->update_key) { + btrfs_disk_key_to_cpu(&key, &head->extent_op->key); + key_ptr = &key; + } + + ret = add_indirect_ref(fs_info, preftrees, node->ref_root, + key_ptr, level + 1, node->bytenr, + count, sc, GFP_ATOMIC); + break; + } + case BTRFS_SHARED_BLOCK_REF_KEY: { + /* + * SHARED DIRECT METADATA backref + * + * The owner of a tree block ref is the level. + */ + int level = btrfs_delayed_ref_owner(node); + + ret = add_direct_ref(fs_info, preftrees, level + 1, + node->parent, node->bytenr, count, + sc, GFP_ATOMIC); + break; + } + case BTRFS_EXTENT_DATA_REF_KEY: { + /* NORMAL INDIRECT DATA backref */ + key.objectid = btrfs_delayed_ref_owner(node); + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = btrfs_delayed_ref_offset(node); + + /* + * If we have a share check context and a reference for + * another inode, we can't exit immediately. This is + * because even if this is a BTRFS_ADD_DELAYED_REF + * reference we may find next a BTRFS_DROP_DELAYED_REF + * which cancels out this ADD reference. + * + * If this is a DROP reference and there was no previous + * ADD reference, then we need to signal that when we + * process references from the extent tree (through + * add_inline_refs() and add_keyed_refs()), we should + * not exit early if we find a reference for another + * inode, because one of the delayed DROP references + * may cancel that reference in the extent tree. + */ + if (sc && count < 0) + sc->have_delayed_delete_refs = true; + + ret = add_indirect_ref(fs_info, preftrees, node->ref_root, + &key, 0, node->bytenr, count, sc, + GFP_ATOMIC); + break; + } + case BTRFS_SHARED_DATA_REF_KEY: { + /* SHARED DIRECT FULL backref */ + ret = add_direct_ref(fs_info, preftrees, 0, node->parent, + node->bytenr, count, sc, + GFP_ATOMIC); + break; + } + default: + WARN_ON(1); + } + /* + * We must ignore BACKREF_FOUND_SHARED until all delayed + * refs have been checked. + */ + if (ret && (ret != BACKREF_FOUND_SHARED)) + break; + } + if (!ret) + ret = extent_is_shared(sc); + + spin_unlock(&head->lock); + return ret; +} + +/* + * add all inline backrefs for bytenr to the list + * + * Returns 0 on success, <0 on error, or BACKREF_FOUND_SHARED. + */ +static int add_inline_refs(struct btrfs_backref_walk_ctx *ctx, + struct btrfs_path *path, + int *info_level, struct preftrees *preftrees, + struct share_check *sc) +{ + int ret = 0; + int slot; + struct extent_buffer *leaf; + struct btrfs_key key; + struct btrfs_key found_key; + unsigned long ptr; + unsigned long end; + struct btrfs_extent_item *ei; + u64 flags; + u64 item_size; + + /* + * enumerate all inline refs + */ + leaf = path->nodes[0]; + slot = path->slots[0]; + + item_size = btrfs_item_size(leaf, slot); + ei = btrfs_item_ptr(leaf, slot, struct btrfs_extent_item); + + if (ctx->check_extent_item) { + ret = ctx->check_extent_item(ctx->bytenr, ei, leaf, ctx->user_ctx); + if (ret) + return ret; + } + + flags = btrfs_extent_flags(leaf, ei); + btrfs_item_key_to_cpu(leaf, &found_key, slot); + + ptr = (unsigned long)(ei + 1); + end = (unsigned long)ei + item_size; + + if (found_key.type == BTRFS_EXTENT_ITEM_KEY && + flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) { + struct btrfs_tree_block_info *info; + + info = (struct btrfs_tree_block_info *)ptr; + *info_level = btrfs_tree_block_level(leaf, info); + ptr += sizeof(struct btrfs_tree_block_info); + BUG_ON(ptr > end); + } else if (found_key.type == BTRFS_METADATA_ITEM_KEY) { + *info_level = found_key.offset; + } else { + BUG_ON(!(flags & BTRFS_EXTENT_FLAG_DATA)); + } + + while (ptr < end) { + struct btrfs_extent_inline_ref *iref; + u64 offset; + int type; + + iref = (struct btrfs_extent_inline_ref *)ptr; + type = btrfs_get_extent_inline_ref_type(leaf, iref, + BTRFS_REF_TYPE_ANY); + if (unlikely(type == BTRFS_REF_TYPE_INVALID)) + return -EUCLEAN; + + offset = btrfs_extent_inline_ref_offset(leaf, iref); + + switch (type) { + case BTRFS_SHARED_BLOCK_REF_KEY: + ret = add_direct_ref(ctx->fs_info, preftrees, + *info_level + 1, offset, + ctx->bytenr, 1, NULL, GFP_NOFS); + break; + case BTRFS_SHARED_DATA_REF_KEY: { + struct btrfs_shared_data_ref *sdref; + int count; + + sdref = (struct btrfs_shared_data_ref *)(iref + 1); + count = btrfs_shared_data_ref_count(leaf, sdref); + + ret = add_direct_ref(ctx->fs_info, preftrees, 0, offset, + ctx->bytenr, count, sc, GFP_NOFS); + break; + } + case BTRFS_TREE_BLOCK_REF_KEY: + ret = add_indirect_ref(ctx->fs_info, preftrees, offset, + NULL, *info_level + 1, + ctx->bytenr, 1, NULL, GFP_NOFS); + break; + case BTRFS_EXTENT_DATA_REF_KEY: { + struct btrfs_extent_data_ref *dref; + int count; + u64 root; + + dref = (struct btrfs_extent_data_ref *)(&iref->offset); + count = btrfs_extent_data_ref_count(leaf, dref); + key.objectid = btrfs_extent_data_ref_objectid(leaf, + dref); + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = btrfs_extent_data_ref_offset(leaf, dref); + + if (sc && key.objectid != sc->inum && + !sc->have_delayed_delete_refs) { + ret = BACKREF_FOUND_SHARED; + break; + } + + root = btrfs_extent_data_ref_root(leaf, dref); + + if (!ctx->skip_data_ref || + !ctx->skip_data_ref(root, key.objectid, key.offset, + ctx->user_ctx)) + ret = add_indirect_ref(ctx->fs_info, preftrees, + root, &key, 0, ctx->bytenr, + count, sc, GFP_NOFS); + break; + } + case BTRFS_EXTENT_OWNER_REF_KEY: + ASSERT(btrfs_fs_incompat(ctx->fs_info, SIMPLE_QUOTA)); + break; + default: + WARN_ON(1); + } + if (ret) + return ret; + ptr += btrfs_extent_inline_ref_size(type); + } + + return 0; +} + +/* + * add all non-inline backrefs for bytenr to the list + * + * Returns 0 on success, <0 on error, or BACKREF_FOUND_SHARED. + */ +static int add_keyed_refs(struct btrfs_backref_walk_ctx *ctx, + struct btrfs_root *extent_root, + struct btrfs_path *path, + int info_level, struct preftrees *preftrees, + struct share_check *sc) +{ + struct btrfs_fs_info *fs_info = extent_root->fs_info; + int ret; + int slot; + struct extent_buffer *leaf; + struct btrfs_key key; + + while (1) { + ret = btrfs_next_item(extent_root, path); + if (ret < 0) + break; + if (ret) { + ret = 0; + break; + } + + slot = path->slots[0]; + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &key, slot); + + if (key.objectid != ctx->bytenr) + break; + if (key.type < BTRFS_TREE_BLOCK_REF_KEY) + continue; + if (key.type > BTRFS_SHARED_DATA_REF_KEY) + break; + + switch (key.type) { + case BTRFS_SHARED_BLOCK_REF_KEY: + /* SHARED DIRECT METADATA backref */ + ret = add_direct_ref(fs_info, preftrees, + info_level + 1, key.offset, + ctx->bytenr, 1, NULL, GFP_NOFS); + break; + case BTRFS_SHARED_DATA_REF_KEY: { + /* SHARED DIRECT FULL backref */ + struct btrfs_shared_data_ref *sdref; + int count; + + sdref = btrfs_item_ptr(leaf, slot, + struct btrfs_shared_data_ref); + count = btrfs_shared_data_ref_count(leaf, sdref); + ret = add_direct_ref(fs_info, preftrees, 0, + key.offset, ctx->bytenr, count, + sc, GFP_NOFS); + break; + } + case BTRFS_TREE_BLOCK_REF_KEY: + /* NORMAL INDIRECT METADATA backref */ + ret = add_indirect_ref(fs_info, preftrees, key.offset, + NULL, info_level + 1, ctx->bytenr, + 1, NULL, GFP_NOFS); + break; + case BTRFS_EXTENT_DATA_REF_KEY: { + /* NORMAL INDIRECT DATA backref */ + struct btrfs_extent_data_ref *dref; + int count; + u64 root; + + dref = btrfs_item_ptr(leaf, slot, + struct btrfs_extent_data_ref); + count = btrfs_extent_data_ref_count(leaf, dref); + key.objectid = btrfs_extent_data_ref_objectid(leaf, + dref); + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = btrfs_extent_data_ref_offset(leaf, dref); + + if (sc && key.objectid != sc->inum && + !sc->have_delayed_delete_refs) { + ret = BACKREF_FOUND_SHARED; + break; + } + + root = btrfs_extent_data_ref_root(leaf, dref); + + if (!ctx->skip_data_ref || + !ctx->skip_data_ref(root, key.objectid, key.offset, + ctx->user_ctx)) + ret = add_indirect_ref(fs_info, preftrees, root, + &key, 0, ctx->bytenr, + count, sc, GFP_NOFS); + break; + } + default: + WARN_ON(1); + } + if (ret) + return ret; + + } + + return ret; +} + +/* + * The caller has joined a transaction or is holding a read lock on the + * fs_info->commit_root_sem semaphore, so no need to worry about the root's last + * snapshot field changing while updating or checking the cache. + */ +static bool lookup_backref_shared_cache(struct btrfs_backref_share_check_ctx *ctx, + struct btrfs_root *root, + u64 bytenr, int level, bool *is_shared) +{ + const struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_backref_shared_cache_entry *entry; + + if (!current->journal_info) + lockdep_assert_held(&fs_info->commit_root_sem); + + if (!ctx->use_path_cache) + return false; + + if (WARN_ON_ONCE(level >= BTRFS_MAX_LEVEL)) + return false; + + /* + * Level -1 is used for the data extent, which is not reliable to cache + * because its reference count can increase or decrease without us + * realizing. We cache results only for extent buffers that lead from + * the root node down to the leaf with the file extent item. + */ + ASSERT(level >= 0, "level=%d", level); + + entry = &ctx->path_cache_entries[level]; + + /* Unused cache entry or being used for some other extent buffer. */ + if (entry->bytenr != bytenr) + return false; + + /* + * We cached a false result, but the last snapshot generation of the + * root changed, so we now have a snapshot. Don't trust the result. + */ + if (!entry->is_shared && + entry->gen != btrfs_root_last_snapshot(&root->root_item)) + return false; + + /* + * If we cached a true result and the last generation used for dropping + * a root changed, we can not trust the result, because the dropped root + * could be a snapshot sharing this extent buffer. + */ + if (entry->is_shared && + entry->gen != btrfs_get_last_root_drop_gen(fs_info)) + return false; + + *is_shared = entry->is_shared; + /* + * If the node at this level is shared, than all nodes below are also + * shared. Currently some of the nodes below may be marked as not shared + * because we have just switched from one leaf to another, and switched + * also other nodes above the leaf and below the current level, so mark + * them as shared. + */ + if (*is_shared) { + for (int i = 0; i < level; i++) { + ctx->path_cache_entries[i].is_shared = true; + ctx->path_cache_entries[i].gen = entry->gen; + } + } + + return true; +} + +/* + * The caller has joined a transaction or is holding a read lock on the + * fs_info->commit_root_sem semaphore, so no need to worry about the root's last + * snapshot field changing while updating or checking the cache. + */ +static void store_backref_shared_cache(struct btrfs_backref_share_check_ctx *ctx, + struct btrfs_root *root, + u64 bytenr, int level, bool is_shared) +{ + const struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_backref_shared_cache_entry *entry; + u64 gen; + + if (!current->journal_info) + lockdep_assert_held(&fs_info->commit_root_sem); + + if (!ctx->use_path_cache) + return; + + if (WARN_ON_ONCE(level >= BTRFS_MAX_LEVEL)) + return; + + /* + * Level -1 is used for the data extent, which is not reliable to cache + * because its reference count can increase or decrease without us + * realizing. We cache results only for extent buffers that lead from + * the root node down to the leaf with the file extent item. + */ + ASSERT(level >= 0, "level=%d", level); + + if (is_shared) + gen = btrfs_get_last_root_drop_gen(fs_info); + else + gen = btrfs_root_last_snapshot(&root->root_item); + + entry = &ctx->path_cache_entries[level]; + entry->bytenr = bytenr; + entry->is_shared = is_shared; + entry->gen = gen; + + /* + * If we found an extent buffer is shared, set the cache result for all + * extent buffers below it to true. As nodes in the path are COWed, + * their sharedness is moved to their children, and if a leaf is COWed, + * then the sharedness of a data extent becomes direct, the refcount of + * data extent is increased in the extent item at the extent tree. + */ + if (is_shared) { + for (int i = 0; i < level; i++) { + entry = &ctx->path_cache_entries[i]; + entry->is_shared = is_shared; + entry->gen = gen; + } + } +} + +/* + * this adds all existing backrefs (inline backrefs, backrefs and delayed + * refs) for the given bytenr to the refs list, merges duplicates and resolves + * indirect refs to their parent bytenr. + * When roots are found, they're added to the roots list + * + * @ctx: Backref walking context object, must be not NULL. + * @sc: If !NULL, then immediately return BACKREF_FOUND_SHARED when a + * shared extent is detected. + * + * Otherwise this returns 0 for success and <0 for an error. + * + * FIXME some caching might speed things up + */ +static int find_parent_nodes(struct btrfs_backref_walk_ctx *ctx, + struct share_check *sc) +{ + struct btrfs_root *root = btrfs_extent_root(ctx->fs_info, ctx->bytenr); + struct btrfs_key key; + struct btrfs_path *path; + struct btrfs_delayed_ref_root *delayed_refs = NULL; + struct btrfs_delayed_ref_head *head; + int info_level = 0; + int ret; + struct prelim_ref *ref; + struct rb_node *node; + struct extent_inode_elem *eie = NULL; + struct preftrees preftrees = { + .direct = PREFTREE_INIT, + .indirect = PREFTREE_INIT, + .indirect_missing_keys = PREFTREE_INIT + }; + + if (unlikely(!root)) { + btrfs_err(ctx->fs_info, + "missing extent root for extent at bytenr %llu", + ctx->bytenr); + return -EUCLEAN; + } + + /* Roots ulist is not needed when using a sharedness check context. */ + if (sc) + ASSERT(ctx->roots == NULL); + + key.objectid = ctx->bytenr; + if (btrfs_fs_incompat(ctx->fs_info, SKINNY_METADATA)) + key.type = BTRFS_METADATA_ITEM_KEY; + else + key.type = BTRFS_EXTENT_ITEM_KEY; + key.offset = (u64)-1; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + if (!ctx->trans) { + path->search_commit_root = true; + path->skip_locking = true; + } + + if (ctx->time_seq == BTRFS_SEQ_LAST) + path->skip_locking = true; + +again: + head = NULL; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + goto out; + if (unlikely(ret == 0)) { + /* + * Key with offset -1 found, there would have to exist an extent + * item with such offset, but this is out of the valid range. + */ + ret = -EUCLEAN; + goto out; + } + + if (ctx->trans && likely(ctx->trans->type != __TRANS_DUMMY) && + ctx->time_seq != BTRFS_SEQ_LAST) { + /* + * We have a specific time_seq we care about and trans which + * means we have the path lock, we need to grab the ref head and + * lock it so we have a consistent view of the refs at the given + * time. + */ + delayed_refs = &ctx->trans->transaction->delayed_refs; + spin_lock(&delayed_refs->lock); + head = btrfs_find_delayed_ref_head(ctx->fs_info, delayed_refs, + ctx->bytenr); + if (head) { + if (!mutex_trylock(&head->mutex)) { + refcount_inc(&head->refs); + spin_unlock(&delayed_refs->lock); + + btrfs_release_path(path); + + /* + * Mutex was contended, block until it's + * released and try again + */ + mutex_lock(&head->mutex); + mutex_unlock(&head->mutex); + btrfs_put_delayed_ref_head(head); + goto again; + } + spin_unlock(&delayed_refs->lock); + ret = add_delayed_refs(ctx->fs_info, head, ctx->time_seq, + &preftrees, sc); + mutex_unlock(&head->mutex); + if (ret) + goto out; + } else { + spin_unlock(&delayed_refs->lock); + } + } + + if (path->slots[0]) { + struct extent_buffer *leaf; + int slot; + + path->slots[0]--; + leaf = path->nodes[0]; + slot = path->slots[0]; + btrfs_item_key_to_cpu(leaf, &key, slot); + if (key.objectid == ctx->bytenr && + (key.type == BTRFS_EXTENT_ITEM_KEY || + key.type == BTRFS_METADATA_ITEM_KEY)) { + ret = add_inline_refs(ctx, path, &info_level, + &preftrees, sc); + if (ret) + goto out; + ret = add_keyed_refs(ctx, root, path, info_level, + &preftrees, sc); + if (ret) + goto out; + } + } + + /* + * If we have a share context and we reached here, it means the extent + * is not directly shared (no multiple reference items for it), + * otherwise we would have exited earlier with a return value of + * BACKREF_FOUND_SHARED after processing delayed references or while + * processing inline or keyed references from the extent tree. + * The extent may however be indirectly shared through shared subtrees + * as a result from creating snapshots, so we determine below what is + * its parent node, in case we are dealing with a metadata extent, or + * what's the leaf (or leaves), from a fs tree, that has a file extent + * item pointing to it in case we are dealing with a data extent. + */ + ASSERT(extent_is_shared(sc) == 0); + + /* + * If we are here for a data extent and we have a share_check structure + * it means the data extent is not directly shared (does not have + * multiple reference items), so we have to check if a path in the fs + * tree (going from the root node down to the leaf that has the file + * extent item pointing to the data extent) is shared, that is, if any + * of the extent buffers in the path is referenced by other trees. + */ + if (sc && ctx->bytenr == sc->data_bytenr) { + /* + * If our data extent is from a generation more recent than the + * last generation used to snapshot the root, then we know that + * it can not be shared through subtrees, so we can skip + * resolving indirect references, there's no point in + * determining the extent buffers for the path from the fs tree + * root node down to the leaf that has the file extent item that + * points to the data extent. + */ + if (sc->data_extent_gen > + btrfs_root_last_snapshot(&sc->root->root_item)) { + ret = BACKREF_FOUND_NOT_SHARED; + goto out; + } + + /* + * If we are only determining if a data extent is shared or not + * and the corresponding file extent item is located in the same + * leaf as the previous file extent item, we can skip resolving + * indirect references for a data extent, since the fs tree path + * is the same (same leaf, so same path). We skip as long as the + * cached result for the leaf is valid and only if there's only + * one file extent item pointing to the data extent, because in + * the case of multiple file extent items, they may be located + * in different leaves and therefore we have multiple paths. + */ + if (sc->ctx->curr_leaf_bytenr == sc->ctx->prev_leaf_bytenr && + sc->self_ref_count == 1) { + bool cached; + bool is_shared; + + cached = lookup_backref_shared_cache(sc->ctx, sc->root, + sc->ctx->curr_leaf_bytenr, + 0, &is_shared); + if (cached) { + if (is_shared) + ret = BACKREF_FOUND_SHARED; + else + ret = BACKREF_FOUND_NOT_SHARED; + goto out; + } + } + } + + btrfs_release_path(path); + + ret = add_missing_keys(ctx->fs_info, &preftrees, !path->skip_locking); + if (ret) + goto out; + + WARN_ON(!RB_EMPTY_ROOT(&preftrees.indirect_missing_keys.root.rb_root)); + + ret = resolve_indirect_refs(ctx, path, &preftrees, sc); + if (ret) + goto out; + + WARN_ON(!RB_EMPTY_ROOT(&preftrees.indirect.root.rb_root)); + + /* + * This walks the tree of merged and resolved refs. Tree blocks are + * read in as needed. Unique entries are added to the ulist, and + * the list of found roots is updated. + * + * We release the entire tree in one go before returning. + */ + node = rb_first_cached(&preftrees.direct.root); + while (node) { + ref = rb_entry(node, struct prelim_ref, rbnode); + node = rb_next(&ref->rbnode); + /* + * ref->count < 0 can happen here if there are delayed + * refs with a node->action of BTRFS_DROP_DELAYED_REF. + * prelim_ref_insert() relies on this when merging + * identical refs to keep the overall count correct. + * prelim_ref_insert() will merge only those refs + * which compare identically. Any refs having + * e.g. different offsets would not be merged, + * and would retain their original ref->count < 0. + */ + if (ctx->roots && ref->count && ref->root_id && ref->parent == 0) { + /* no parent == root of tree */ + ret = ulist_add(ctx->roots, ref->root_id, 0, GFP_NOFS); + if (ret < 0) + goto out; + } + if (ref->count && ref->parent) { + if (!ctx->skip_inode_ref_list && !ref->inode_list && + ref->level == 0) { + struct btrfs_tree_parent_check check = { 0 }; + struct extent_buffer *eb; + + check.level = ref->level; + + eb = read_tree_block(ctx->fs_info, ref->parent, + &check); + if (IS_ERR(eb)) { + ret = PTR_ERR(eb); + goto out; + } + + if (!path->skip_locking) + btrfs_tree_read_lock(eb); + ret = find_extent_in_eb(ctx, eb, &eie); + if (!path->skip_locking) + btrfs_tree_read_unlock(eb); + free_extent_buffer(eb); + if (ret == BTRFS_ITERATE_EXTENT_INODES_STOP || + ret < 0) + goto out; + ref->inode_list = eie; + /* + * We transferred the list ownership to the ref, + * so set to NULL to avoid a double free in case + * an error happens after this. + */ + eie = NULL; + } + ret = ulist_add_merge_ptr(ctx->refs, ref->parent, + ref->inode_list, + (void **)&eie, GFP_NOFS); + if (ret < 0) + goto out; + if (!ret && !ctx->skip_inode_ref_list) { + /* + * We've recorded that parent, so we must extend + * its inode list here. + * + * However if there was corruption we may not + * have found an eie, return an error in this + * case. + */ + ASSERT(eie); + if (unlikely(!eie)) { + ret = -EUCLEAN; + goto out; + } + while (eie->next) + eie = eie->next; + eie->next = ref->inode_list; + } + eie = NULL; + /* + * We have transferred the inode list ownership from + * this ref to the ref we added to the 'refs' ulist. + * So set this ref's inode list to NULL to avoid + * use-after-free when our caller uses it or double + * frees in case an error happens before we return. + */ + ref->inode_list = NULL; + } + cond_resched(); + } + +out: + btrfs_free_path(path); + + prelim_release(&preftrees.direct); + prelim_release(&preftrees.indirect); + prelim_release(&preftrees.indirect_missing_keys); + + if (ret == BTRFS_ITERATE_EXTENT_INODES_STOP || ret < 0) + free_inode_elem_list(eie); + return ret; +} + +/* + * Finds all leaves with a reference to the specified combination of + * @ctx->bytenr and @ctx->extent_item_pos. The bytenr of the found leaves are + * added to the ulist at @ctx->refs, and that ulist is allocated by this + * function. The caller should free the ulist with free_leaf_list() if + * @ctx->ignore_extent_item_pos is false, otherwise a simple ulist_free() is + * enough. + * + * Returns 0 on success and < 0 on error. On error @ctx->refs is not allocated. + */ +int btrfs_find_all_leafs(struct btrfs_backref_walk_ctx *ctx) +{ + int ret; + + ASSERT(ctx->refs == NULL); + + ctx->refs = ulist_alloc(GFP_NOFS); + if (!ctx->refs) + return -ENOMEM; + + ret = find_parent_nodes(ctx, NULL); + if (ret == BTRFS_ITERATE_EXTENT_INODES_STOP || + (ret < 0 && ret != -ENOENT)) { + free_leaf_list(ctx->refs); + ctx->refs = NULL; + return ret; + } + + return 0; +} + +/* + * Walk all backrefs for a given extent to find all roots that reference this + * extent. Walking a backref means finding all extents that reference this + * extent and in turn walk the backrefs of those, too. Naturally this is a + * recursive process, but here it is implemented in an iterative fashion: We + * find all referencing extents for the extent in question and put them on a + * list. In turn, we find all referencing extents for those, further appending + * to the list. The way we iterate the list allows adding more elements after + * the current while iterating. The process stops when we reach the end of the + * list. + * + * Found roots are added to @ctx->roots, which is allocated by this function if + * it points to NULL, in which case the caller is responsible for freeing it + * after it's not needed anymore. + * This function requires @ctx->refs to be NULL, as it uses it for allocating a + * ulist to do temporary work, and frees it before returning. + * + * Returns 0 on success, < 0 on error. + */ +static int btrfs_find_all_roots_safe(struct btrfs_backref_walk_ctx *ctx) +{ + const u64 orig_bytenr = ctx->bytenr; + const bool orig_skip_inode_ref_list = ctx->skip_inode_ref_list; + bool roots_ulist_allocated = false; + struct ulist_iterator uiter; + int ret = 0; + + ASSERT(ctx->refs == NULL); + + ctx->refs = ulist_alloc(GFP_NOFS); + if (!ctx->refs) + return -ENOMEM; + + if (!ctx->roots) { + ctx->roots = ulist_alloc(GFP_NOFS); + if (!ctx->roots) { + ulist_free(ctx->refs); + ctx->refs = NULL; + return -ENOMEM; + } + roots_ulist_allocated = true; + } + + ctx->skip_inode_ref_list = true; + + ULIST_ITER_INIT(&uiter); + while (1) { + struct ulist_node *node; + + ret = find_parent_nodes(ctx, NULL); + if (ret < 0 && ret != -ENOENT) { + if (roots_ulist_allocated) { + ulist_free(ctx->roots); + ctx->roots = NULL; + } + break; + } + ret = 0; + node = ulist_next(ctx->refs, &uiter); + if (!node) + break; + ctx->bytenr = node->val; + cond_resched(); + } + + ulist_free(ctx->refs); + ctx->refs = NULL; + ctx->bytenr = orig_bytenr; + ctx->skip_inode_ref_list = orig_skip_inode_ref_list; + + return ret; +} + +int btrfs_find_all_roots(struct btrfs_backref_walk_ctx *ctx, + bool skip_commit_root_sem) +{ + int ret; + + if (!ctx->trans && !skip_commit_root_sem) + down_read(&ctx->fs_info->commit_root_sem); + ret = btrfs_find_all_roots_safe(ctx); + if (!ctx->trans && !skip_commit_root_sem) + up_read(&ctx->fs_info->commit_root_sem); + return ret; +} + +struct btrfs_backref_share_check_ctx *btrfs_alloc_backref_share_check_ctx(void) +{ + struct btrfs_backref_share_check_ctx *ctx; + + ctx = kzalloc_obj(*ctx); + if (!ctx) + return NULL; + + ulist_init(&ctx->refs); + + return ctx; +} + +void btrfs_free_backref_share_ctx(struct btrfs_backref_share_check_ctx *ctx) +{ + if (!ctx) + return; + + ulist_release(&ctx->refs); + kfree(ctx); +} + +/* + * Check if a data extent is shared or not. + * + * @inode: The inode whose extent we are checking. + * @bytenr: Logical bytenr of the extent we are checking. + * @extent_gen: Generation of the extent (file extent item) or 0 if it is + * not known. + * @ctx: A backref sharedness check context. + * + * btrfs_is_data_extent_shared uses the backref walking code but will short + * circuit as soon as it finds a root or inode that doesn't match the + * one passed in. This provides a significant performance benefit for + * callers (such as fiemap) which want to know whether the extent is + * shared but do not need a ref count. + * + * This attempts to attach to the running transaction in order to account for + * delayed refs, but continues on even when no running transaction exists. + * + * Return: 0 if extent is not shared, 1 if it is shared, < 0 on error. + */ +int btrfs_is_data_extent_shared(struct btrfs_inode *inode, u64 bytenr, + u64 extent_gen, + struct btrfs_backref_share_check_ctx *ctx) +{ + struct btrfs_backref_walk_ctx walk_ctx = { 0 }; + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_trans_handle *trans; + struct ulist_iterator uiter; + struct ulist_node *node; + struct btrfs_seq_list elem = BTRFS_SEQ_LIST_INIT(elem); + int ret = 0; + struct share_check shared = { + .ctx = ctx, + .root = root, + .inum = btrfs_ino(inode), + .data_bytenr = bytenr, + .data_extent_gen = extent_gen, + .share_count = 0, + .self_ref_count = 0, + .have_delayed_delete_refs = false, + }; + int level; + bool leaf_cached; + bool leaf_is_shared; + + for (int i = 0; i < BTRFS_BACKREF_CTX_PREV_EXTENTS_SIZE; i++) { + if (ctx->prev_extents_cache[i].bytenr == bytenr) + return ctx->prev_extents_cache[i].is_shared; + } + + ulist_init(&ctx->refs); + + trans = btrfs_join_transaction_nostart(root); + if (IS_ERR(trans)) { + if (PTR_ERR(trans) != -ENOENT && PTR_ERR(trans) != -EROFS) { + ret = PTR_ERR(trans); + goto out; + } + trans = NULL; + down_read(&fs_info->commit_root_sem); + } else { + btrfs_get_tree_mod_seq(fs_info, &elem); + walk_ctx.time_seq = elem.seq; + } + + ctx->use_path_cache = true; + + /* + * We may have previously determined that the current leaf is shared. + * If it is, then we have a data extent that is shared due to a shared + * subtree (caused by snapshotting) and we don't need to check for data + * backrefs. If the leaf is not shared, then we must do backref walking + * to determine if the data extent is shared through reflinks. + */ + leaf_cached = lookup_backref_shared_cache(ctx, root, + ctx->curr_leaf_bytenr, 0, + &leaf_is_shared); + if (leaf_cached && leaf_is_shared) { + ret = 1; + goto out_trans; + } + + walk_ctx.skip_inode_ref_list = true; + walk_ctx.trans = trans; + walk_ctx.fs_info = fs_info; + walk_ctx.refs = &ctx->refs; + + /* -1 means we are in the bytenr of the data extent. */ + level = -1; + ULIST_ITER_INIT(&uiter); + while (1) { + const unsigned long prev_ref_count = ctx->refs.nnodes; + + walk_ctx.bytenr = bytenr; + ret = find_parent_nodes(&walk_ctx, &shared); + if (ret == BACKREF_FOUND_SHARED || + ret == BACKREF_FOUND_NOT_SHARED) { + /* If shared must return 1, otherwise return 0. */ + ret = (ret == BACKREF_FOUND_SHARED) ? 1 : 0; + if (level >= 0) + store_backref_shared_cache(ctx, root, bytenr, + level, ret == 1); + break; + } + if (ret < 0 && ret != -ENOENT) + break; + ret = 0; + + /* + * More than one extent buffer (bytenr) may have been added to + * the ctx->refs ulist, in which case we have to check multiple + * tree paths in case the first one is not shared, so we can not + * use the path cache which is made for a single path. Multiple + * extent buffers at the current level happen when: + * + * 1) level -1, the data extent: If our data extent was not + * directly shared (without multiple reference items), then + * it might have a single reference item with a count > 1 for + * the same offset, which means there are 2 (or more) file + * extent items that point to the data extent - this happens + * when a file extent item needs to be split and then one + * item gets moved to another leaf due to a b+tree leaf split + * when inserting some item. In this case the file extent + * items may be located in different leaves and therefore + * some of the leaves may be referenced through shared + * subtrees while others are not. Since our extent buffer + * cache only works for a single path (by far the most common + * case and simpler to deal with), we can not use it if we + * have multiple leaves (which implies multiple paths). + * + * 2) level >= 0, a tree node/leaf: We can have a mix of direct + * and indirect references on a b+tree node/leaf, so we have + * to check multiple paths, and the extent buffer (the + * current bytenr) may be shared or not. One example is + * during relocation as we may get a shared tree block ref + * (direct ref) and a non-shared tree block ref (indirect + * ref) for the same node/leaf. + */ + if ((ctx->refs.nnodes - prev_ref_count) > 1) + ctx->use_path_cache = false; + + if (level >= 0) + store_backref_shared_cache(ctx, root, bytenr, + level, false); + node = ulist_next(&ctx->refs, &uiter); + if (!node) + break; + bytenr = node->val; + if (ctx->use_path_cache) { + bool is_shared; + bool cached; + + level++; + cached = lookup_backref_shared_cache(ctx, root, bytenr, + level, &is_shared); + if (cached) { + ret = (is_shared ? 1 : 0); + break; + } + } + shared.share_count = 0; + shared.have_delayed_delete_refs = false; + cond_resched(); + } + + /* + * If the path cache is disabled, then it means at some tree level we + * got multiple parents due to a mix of direct and indirect backrefs or + * multiple leaves with file extent items pointing to the same data + * extent. We have to invalidate the cache and cache only the sharedness + * result for the levels where we got only one node/reference. + */ + if (!ctx->use_path_cache) { + int i = 0; + + level--; + if (ret >= 0 && level >= 0) { + bytenr = ctx->path_cache_entries[level].bytenr; + ctx->use_path_cache = true; + store_backref_shared_cache(ctx, root, bytenr, level, ret); + i = level + 1; + } + + for ( ; i < BTRFS_MAX_LEVEL; i++) + ctx->path_cache_entries[i].bytenr = 0; + } + + /* + * Cache the sharedness result for the data extent if we know our inode + * has more than 1 file extent item that refers to the data extent. + */ + if (ret >= 0 && shared.self_ref_count > 1) { + int slot = ctx->prev_extents_cache_slot; + + ctx->prev_extents_cache[slot].bytenr = shared.data_bytenr; + ctx->prev_extents_cache[slot].is_shared = (ret == 1); + + slot = (slot + 1) % BTRFS_BACKREF_CTX_PREV_EXTENTS_SIZE; + ctx->prev_extents_cache_slot = slot; + } + +out_trans: + if (trans) { + btrfs_put_tree_mod_seq(fs_info, &elem); + btrfs_end_transaction(trans); + } else { + up_read(&fs_info->commit_root_sem); + } +out: + ulist_release(&ctx->refs); + ctx->prev_leaf_bytenr = ctx->curr_leaf_bytenr; + + return ret; +} + +int btrfs_find_one_extref(struct btrfs_root *root, u64 inode_objectid, + u64 start_off, struct btrfs_path *path, + struct btrfs_inode_extref **ret_extref, + u64 *found_off) +{ + int ret, slot; + struct btrfs_key key; + struct btrfs_key found_key; + struct btrfs_inode_extref *extref; + const struct extent_buffer *leaf; + unsigned long ptr; + + key.objectid = inode_objectid; + key.type = BTRFS_INODE_EXTREF_KEY; + key.offset = start_off; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + return ret; + + while (1) { + leaf = path->nodes[0]; + slot = path->slots[0]; + if (slot >= btrfs_header_nritems(leaf)) { + /* + * If the item at offset is not found, + * btrfs_search_slot will point us to the slot + * where it should be inserted. In our case + * that will be the slot directly before the + * next INODE_REF_KEY_V2 item. In the case + * that we're pointing to the last slot in a + * leaf, we must move one leaf over. + */ + ret = btrfs_next_leaf(root, path); + if (ret) { + if (ret >= 1) + ret = -ENOENT; + break; + } + continue; + } + + btrfs_item_key_to_cpu(leaf, &found_key, slot); + + /* + * Check that we're still looking at an extended ref key for + * this particular objectid. If we have different + * objectid or type then there are no more to be found + * in the tree and we can exit. + */ + ret = -ENOENT; + if (found_key.objectid != inode_objectid) + break; + if (found_key.type != BTRFS_INODE_EXTREF_KEY) + break; + + ret = 0; + ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); + extref = (struct btrfs_inode_extref *)ptr; + *ret_extref = extref; + if (found_off) + *found_off = found_key.offset; + break; + } + + return ret; +} + +/* + * this iterates to turn a name (from iref/extref) into a full filesystem path. + * Elements of the path are separated by '/' and the path is guaranteed to be + * 0-terminated. the path is only given within the current file system. + * Therefore, it never starts with a '/'. the caller is responsible to provide + * "size" bytes in "dest". the dest buffer will be filled backwards. finally, + * the start point of the resulting string is returned. this pointer is within + * dest, normally. + * in case the path buffer would overflow, the pointer is decremented further + * as if output was written to the buffer, though no more output is actually + * generated. that way, the caller can determine how much space would be + * required for the path to fit into the buffer. in that case, the returned + * value will be smaller than dest. callers must check this! + */ +char *btrfs_ref_to_path(struct btrfs_root *fs_root, struct btrfs_path *path, + u32 name_len, unsigned long name_off, + struct extent_buffer *eb_in, u64 parent, + char *dest, u32 size) +{ + int slot; + u64 next_inum; + int ret; + s64 bytes_left = ((s64)size) - 1; + struct extent_buffer *eb = eb_in; + struct btrfs_key found_key; + struct btrfs_inode_ref *iref; + + if (bytes_left >= 0) + dest[bytes_left] = '\0'; + + while (1) { + bytes_left -= name_len; + if (bytes_left >= 0) + read_extent_buffer(eb, dest + bytes_left, + name_off, name_len); + if (eb != eb_in) { + if (!path->skip_locking) + btrfs_tree_read_unlock(eb); + free_extent_buffer(eb); + } + ret = btrfs_find_item(fs_root, path, parent, 0, + BTRFS_INODE_REF_KEY, &found_key); + if (ret > 0) + ret = -ENOENT; + if (ret) + break; + + next_inum = found_key.offset; + + /* regular exit ahead */ + if (parent == next_inum) + break; + + slot = path->slots[0]; + eb = path->nodes[0]; + /* make sure we can use eb after releasing the path */ + if (eb != eb_in) { + path->nodes[0] = NULL; + path->locks[0] = 0; + } + btrfs_release_path(path); + iref = btrfs_item_ptr(eb, slot, struct btrfs_inode_ref); + + name_len = btrfs_inode_ref_name_len(eb, iref); + name_off = (unsigned long)(iref + 1); + + parent = next_inum; + --bytes_left; + if (bytes_left >= 0) + dest[bytes_left] = '/'; + } + + btrfs_release_path(path); + + if (ret) + return ERR_PTR(ret); + + return dest + bytes_left; +} + +/* + * this makes the path point to (logical EXTENT_ITEM *) + * returns BTRFS_EXTENT_FLAG_DATA for data, BTRFS_EXTENT_FLAG_TREE_BLOCK for + * tree blocks and <0 on error. + */ +int extent_from_logical(struct btrfs_fs_info *fs_info, u64 logical, + struct btrfs_path *path, struct btrfs_key *found_key, + u64 *flags_ret) +{ + struct btrfs_root *extent_root = btrfs_extent_root(fs_info, logical); + int ret; + u64 flags; + u64 size = 0; + const struct extent_buffer *eb; + struct btrfs_extent_item *ei; + struct btrfs_key key; + + if (unlikely(!extent_root)) { + btrfs_err(fs_info, + "missing extent root for extent at bytenr %llu", + logical); + return -EUCLEAN; + } + + key.objectid = logical; + if (btrfs_fs_incompat(fs_info, SKINNY_METADATA)) + key.type = BTRFS_METADATA_ITEM_KEY; + else + key.type = BTRFS_EXTENT_ITEM_KEY; + key.offset = (u64)-1; + + ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0); + if (ret < 0) + return ret; + if (unlikely(ret == 0)) { + /* + * Key with offset -1 found, there would have to exist an extent + * item with such offset, but this is out of the valid range. + */ + return -EUCLEAN; + } + + ret = btrfs_previous_extent_item(extent_root, path, 0); + if (ret) { + if (ret > 0) + ret = -ENOENT; + return ret; + } + btrfs_item_key_to_cpu(path->nodes[0], found_key, path->slots[0]); + if (found_key->type == BTRFS_METADATA_ITEM_KEY) + size = fs_info->nodesize; + else if (found_key->type == BTRFS_EXTENT_ITEM_KEY) + size = found_key->offset; + + if (found_key->objectid > logical || + found_key->objectid + size <= logical) { + btrfs_debug(fs_info, + "logical %llu is not within any extent", logical); + return -ENOENT; + } + + eb = path->nodes[0]; + + ei = btrfs_item_ptr(eb, path->slots[0], struct btrfs_extent_item); + flags = btrfs_extent_flags(eb, ei); + + btrfs_debug(fs_info, + "logical %llu is at position %llu within the extent (%llu EXTENT_ITEM %llu) flags %#llx size %u", + logical, logical - found_key->objectid, found_key->objectid, + found_key->offset, flags, btrfs_item_size(eb, path->slots[0])); + + WARN_ON(!flags_ret); + if (flags_ret) { + if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) + *flags_ret = BTRFS_EXTENT_FLAG_TREE_BLOCK; + else if (flags & BTRFS_EXTENT_FLAG_DATA) + *flags_ret = BTRFS_EXTENT_FLAG_DATA; + else + BUG(); + return 0; + } + + return -EIO; +} + +/* + * helper function to iterate extent inline refs. ptr must point to a 0 value + * for the first call and may be modified. it is used to track state. + * if more refs exist, 0 is returned and the next call to + * get_extent_inline_ref must pass the modified ptr parameter to get the + * next ref. after the last ref was processed, 1 is returned. + * returns <0 on error + */ +static int get_extent_inline_ref(unsigned long *ptr, + const struct extent_buffer *eb, + const struct btrfs_key *key, + const struct btrfs_extent_item *ei, + u32 item_size, + struct btrfs_extent_inline_ref **out_eiref, + int *out_type) +{ + unsigned long end; + u64 flags; + struct btrfs_tree_block_info *info; + + if (!*ptr) { + /* first call */ + flags = btrfs_extent_flags(eb, ei); + if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) { + if (key->type == BTRFS_METADATA_ITEM_KEY) { + /* a skinny metadata extent */ + *out_eiref = + (struct btrfs_extent_inline_ref *)(ei + 1); + } else { + WARN_ON(key->type != BTRFS_EXTENT_ITEM_KEY); + info = (struct btrfs_tree_block_info *)(ei + 1); + *out_eiref = + (struct btrfs_extent_inline_ref *)(info + 1); + } + } else { + *out_eiref = (struct btrfs_extent_inline_ref *)(ei + 1); + } + *ptr = (unsigned long)*out_eiref; + if ((unsigned long)(*ptr) >= (unsigned long)ei + item_size) + return -ENOENT; + } + + end = (unsigned long)ei + item_size; + *out_eiref = (struct btrfs_extent_inline_ref *)(*ptr); + *out_type = btrfs_get_extent_inline_ref_type(eb, *out_eiref, + BTRFS_REF_TYPE_ANY); + if (unlikely(*out_type == BTRFS_REF_TYPE_INVALID)) + return -EUCLEAN; + + *ptr += btrfs_extent_inline_ref_size(*out_type); + WARN_ON(*ptr > end); + if (*ptr == end) + return 1; /* last */ + + return 0; +} + +/* + * reads the tree block backref for an extent. tree level and root are returned + * through out_level and out_root. ptr must point to a 0 value for the first + * call and may be modified (see get_extent_inline_ref comment). + * returns 0 if data was provided, 1 if there was no more data to provide or + * <0 on error. + */ +int tree_backref_for_extent(unsigned long *ptr, struct extent_buffer *eb, + struct btrfs_key *key, struct btrfs_extent_item *ei, + u32 item_size, u64 *out_root, u8 *out_level) +{ + int ret; + int type; + struct btrfs_extent_inline_ref *eiref; + + if (*ptr == (unsigned long)-1) + return 1; + + while (1) { + ret = get_extent_inline_ref(ptr, eb, key, ei, item_size, + &eiref, &type); + if (ret < 0) + return ret; + + if (type == BTRFS_TREE_BLOCK_REF_KEY || + type == BTRFS_SHARED_BLOCK_REF_KEY) + break; + + if (ret == 1) + return 1; + } + + /* we can treat both ref types equally here */ + *out_root = btrfs_extent_inline_ref_offset(eb, eiref); + + if (key->type == BTRFS_EXTENT_ITEM_KEY) { + struct btrfs_tree_block_info *info; + + info = (struct btrfs_tree_block_info *)(ei + 1); + *out_level = btrfs_tree_block_level(eb, info); + } else { + ASSERT(key->type == BTRFS_METADATA_ITEM_KEY, "key->type=%hhu", key->type); + *out_level = (u8)key->offset; + } + + if (ret == 1) + *ptr = (unsigned long)-1; + + return 0; +} + +static int iterate_leaf_refs(struct btrfs_fs_info *fs_info, + struct extent_inode_elem *inode_list, + u64 root, u64 extent_item_objectid, + iterate_extent_inodes_t *iterate, void *ctx) +{ + struct extent_inode_elem *eie; + int ret = 0; + + for (eie = inode_list; eie; eie = eie->next) { + btrfs_debug(fs_info, + "ref for %llu resolved, key (%llu EXTEND_DATA %llu), root %llu", + extent_item_objectid, eie->inum, + eie->offset, root); + ret = iterate(eie->inum, eie->offset, eie->num_bytes, root, ctx); + if (ret) { + btrfs_debug(fs_info, + "stopping iteration for %llu due to ret=%d", + extent_item_objectid, ret); + break; + } + } + + return ret; +} + +/* + * calls iterate() for every inode that references the extent identified by + * the given parameters. + * when the iterator function returns a non-zero value, iteration stops. + */ +int iterate_extent_inodes(struct btrfs_backref_walk_ctx *ctx, + bool search_commit_root, + iterate_extent_inodes_t *iterate, void *user_ctx) +{ + int ret; + struct ulist *refs; + struct ulist_node *ref_node; + struct btrfs_seq_list seq_elem = BTRFS_SEQ_LIST_INIT(seq_elem); + struct ulist_iterator ref_uiter; + + btrfs_debug(ctx->fs_info, "resolving all inodes for extent %llu", + ctx->bytenr); + + ASSERT(ctx->trans == NULL); + ASSERT(ctx->roots == NULL); + + if (!search_commit_root) { + struct btrfs_trans_handle *trans; + + trans = btrfs_attach_transaction(ctx->fs_info->tree_root); + if (IS_ERR(trans)) { + if (PTR_ERR(trans) != -ENOENT && + PTR_ERR(trans) != -EROFS) + return PTR_ERR(trans); + trans = NULL; + } + ctx->trans = trans; + } + + if (ctx->trans) { + btrfs_get_tree_mod_seq(ctx->fs_info, &seq_elem); + ctx->time_seq = seq_elem.seq; + } else { + down_read(&ctx->fs_info->commit_root_sem); + } + + ret = btrfs_find_all_leafs(ctx); + if (ret) + goto out; + refs = ctx->refs; + ctx->refs = NULL; + + ULIST_ITER_INIT(&ref_uiter); + while (!ret && (ref_node = ulist_next(refs, &ref_uiter))) { + const u64 leaf_bytenr = ref_node->val; + struct ulist_node *root_node; + struct ulist_iterator root_uiter; + struct extent_inode_elem *inode_list; + + inode_list = (struct extent_inode_elem *)(uintptr_t)ref_node->aux; + + if (ctx->cache_lookup) { + const u64 *root_ids; + int root_count; + bool cached; + + cached = ctx->cache_lookup(leaf_bytenr, ctx->user_ctx, + &root_ids, &root_count); + if (cached) { + for (int i = 0; i < root_count; i++) { + ret = iterate_leaf_refs(ctx->fs_info, + inode_list, + root_ids[i], + leaf_bytenr, + iterate, + user_ctx); + if (ret) + break; + } + continue; + } + } + + if (!ctx->roots) { + ctx->roots = ulist_alloc(GFP_NOFS); + if (!ctx->roots) { + ret = -ENOMEM; + break; + } + } + + ctx->bytenr = leaf_bytenr; + ret = btrfs_find_all_roots_safe(ctx); + if (ret) + break; + + if (ctx->cache_store) + ctx->cache_store(leaf_bytenr, ctx->roots, ctx->user_ctx); + + ULIST_ITER_INIT(&root_uiter); + while (!ret && (root_node = ulist_next(ctx->roots, &root_uiter))) { + btrfs_debug(ctx->fs_info, + "root %llu references leaf %llu, data list %#llx", + root_node->val, ref_node->val, + ref_node->aux); + ret = iterate_leaf_refs(ctx->fs_info, inode_list, + root_node->val, ctx->bytenr, + iterate, user_ctx); + } + ulist_reinit(ctx->roots); + } + + free_leaf_list(refs); +out: + if (ctx->trans) { + btrfs_put_tree_mod_seq(ctx->fs_info, &seq_elem); + btrfs_end_transaction(ctx->trans); + ctx->trans = NULL; + } else { + up_read(&ctx->fs_info->commit_root_sem); + } + + ulist_free(ctx->roots); + ctx->roots = NULL; + + if (ret == BTRFS_ITERATE_EXTENT_INODES_STOP) + ret = 0; + + return ret; +} + +static int build_ino_list(u64 inum, u64 offset, u64 num_bytes, u64 root, void *ctx) +{ + struct btrfs_data_container *inodes = ctx; + const size_t c = 3 * sizeof(u64); + + if (inodes->bytes_left >= c) { + inodes->bytes_left -= c; + inodes->val[inodes->elem_cnt] = inum; + inodes->val[inodes->elem_cnt + 1] = offset; + inodes->val[inodes->elem_cnt + 2] = root; + inodes->elem_cnt += 3; + } else { + inodes->bytes_missing += c - inodes->bytes_left; + inodes->bytes_left = 0; + inodes->elem_missed += 3; + } + + return 0; +} + +int iterate_inodes_from_logical(u64 logical, struct btrfs_fs_info *fs_info, + void *ctx, bool ignore_offset) +{ + struct btrfs_backref_walk_ctx walk_ctx = { 0 }; + int ret; + u64 flags = 0; + struct btrfs_key found_key; + struct btrfs_path *path; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + ret = extent_from_logical(fs_info, logical, path, &found_key, &flags); + btrfs_free_path(path); + if (ret < 0) + return ret; + if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) + return -EINVAL; + + walk_ctx.bytenr = found_key.objectid; + if (ignore_offset) + walk_ctx.ignore_extent_item_pos = true; + else + walk_ctx.extent_item_pos = logical - found_key.objectid; + walk_ctx.fs_info = fs_info; + + return iterate_extent_inodes(&walk_ctx, false, build_ino_list, ctx); +} + +static int inode_to_path(u64 inum, u32 name_len, unsigned long name_off, + struct extent_buffer *eb, struct inode_fs_paths *ipath); + +static int iterate_inode_refs(u64 inum, struct inode_fs_paths *ipath) +{ + int ret = 0; + int slot; + u32 cur; + u32 len; + u32 name_len; + u64 parent = 0; + int found = 0; + struct btrfs_root *fs_root = ipath->fs_root; + struct btrfs_path *path = ipath->btrfs_path; + struct extent_buffer *eb; + struct btrfs_inode_ref *iref; + struct btrfs_key found_key; + + while (!ret) { + ret = btrfs_find_item(fs_root, path, inum, + parent ? parent + 1 : 0, BTRFS_INODE_REF_KEY, + &found_key); + + if (ret < 0) + break; + if (ret) { + ret = found ? 0 : -ENOENT; + break; + } + ++found; + + parent = found_key.offset; + slot = path->slots[0]; + eb = btrfs_clone_extent_buffer(path->nodes[0]); + if (!eb) { + ret = -ENOMEM; + break; + } + btrfs_release_path(path); + + iref = btrfs_item_ptr(eb, slot, struct btrfs_inode_ref); + + for (cur = 0; cur < btrfs_item_size(eb, slot); cur += len) { + name_len = btrfs_inode_ref_name_len(eb, iref); + /* path must be released before calling iterate()! */ + btrfs_debug(fs_root->fs_info, + "following ref at offset %u for inode %llu in tree %llu", + cur, found_key.objectid, + btrfs_root_id(fs_root)); + ret = inode_to_path(parent, name_len, + (unsigned long)(iref + 1), eb, ipath); + if (ret) + break; + len = sizeof(*iref) + name_len; + iref = (struct btrfs_inode_ref *)((char *)iref + len); + } + free_extent_buffer(eb); + } + + btrfs_release_path(path); + + return ret; +} + +static int iterate_inode_extrefs(u64 inum, struct inode_fs_paths *ipath) +{ + int ret; + int slot; + u64 offset = 0; + u64 parent; + int found = 0; + struct btrfs_root *fs_root = ipath->fs_root; + struct btrfs_path *path = ipath->btrfs_path; + struct extent_buffer *eb; + struct btrfs_inode_extref *extref; + u32 item_size; + u32 cur_offset; + unsigned long ptr; + + while (1) { + ret = btrfs_find_one_extref(fs_root, inum, offset, path, &extref, + &offset); + if (ret < 0) + break; + if (ret) { + ret = found ? 0 : -ENOENT; + break; + } + ++found; + + slot = path->slots[0]; + eb = btrfs_clone_extent_buffer(path->nodes[0]); + if (!eb) { + ret = -ENOMEM; + break; + } + btrfs_release_path(path); + + item_size = btrfs_item_size(eb, slot); + ptr = btrfs_item_ptr_offset(eb, slot); + cur_offset = 0; + + while (cur_offset < item_size) { + u32 name_len; + + extref = (struct btrfs_inode_extref *)(ptr + cur_offset); + parent = btrfs_inode_extref_parent(eb, extref); + name_len = btrfs_inode_extref_name_len(eb, extref); + ret = inode_to_path(parent, name_len, + (unsigned long)&extref->name, eb, ipath); + if (ret) + break; + + cur_offset += btrfs_inode_extref_name_len(eb, extref); + cur_offset += sizeof(*extref); + } + free_extent_buffer(eb); + + offset++; + } + + btrfs_release_path(path); + + return ret; +} + +/* + * returns 0 if the path could be dumped (probably truncated) + * returns <0 in case of an error + */ +static int inode_to_path(u64 inum, u32 name_len, unsigned long name_off, + struct extent_buffer *eb, struct inode_fs_paths *ipath) +{ + char *fspath; + char *fspath_min; + int i = ipath->fspath->elem_cnt; + const int s_ptr = sizeof(char *); + u32 bytes_left; + + bytes_left = ipath->fspath->bytes_left > s_ptr ? + ipath->fspath->bytes_left - s_ptr : 0; + + fspath_min = (char *)ipath->fspath->val + (i + 1) * s_ptr; + fspath = btrfs_ref_to_path(ipath->fs_root, ipath->btrfs_path, name_len, + name_off, eb, inum, fspath_min, bytes_left); + if (IS_ERR(fspath)) + return PTR_ERR(fspath); + + if (fspath > fspath_min) { + ipath->fspath->val[i] = (u64)(unsigned long)fspath; + ++ipath->fspath->elem_cnt; + ipath->fspath->bytes_left = fspath - fspath_min; + } else { + ++ipath->fspath->elem_missed; + ipath->fspath->bytes_missing += fspath_min - fspath; + ipath->fspath->bytes_left = 0; + } + + return 0; +} + +/* + * this dumps all file system paths to the inode into the ipath struct, provided + * is has been created large enough. each path is zero-terminated and accessed + * from ipath->fspath->val[i]. + * when it returns, there are ipath->fspath->elem_cnt number of paths available + * in ipath->fspath->val[]. when the allocated space wasn't sufficient, the + * number of missed paths is recorded in ipath->fspath->elem_missed, otherwise, + * it's zero. ipath->fspath->bytes_missing holds the number of bytes that would + * have been needed to return all paths. + */ +int paths_from_inode(u64 inum, struct inode_fs_paths *ipath) +{ + int ret; + int found_refs = 0; + + ret = iterate_inode_refs(inum, ipath); + if (!ret) + ++found_refs; + else if (ret != -ENOENT) + return ret; + + ret = iterate_inode_extrefs(inum, ipath); + if (ret == -ENOENT && found_refs) + return 0; + + return ret; +} + +struct btrfs_data_container *init_data_container(u32 total_bytes) +{ + struct btrfs_data_container *data; + size_t alloc_bytes; + + alloc_bytes = max_t(size_t, total_bytes, sizeof(*data)); + data = kvzalloc(alloc_bytes, GFP_KERNEL); + if (!data) + return ERR_PTR(-ENOMEM); + + if (total_bytes >= sizeof(*data)) + data->bytes_left = total_bytes - sizeof(*data); + else + data->bytes_missing = sizeof(*data) - total_bytes; + + return data; +} + +/* + * allocates space to return multiple file system paths for an inode. + * total_bytes to allocate are passed, note that space usable for actual path + * information will be total_bytes - sizeof(struct inode_fs_paths). + * the returned pointer must be freed with __free_inode_fs_paths() in the end. + */ +struct inode_fs_paths *init_ipath(s32 total_bytes, struct btrfs_root *fs_root, + struct btrfs_path *path) +{ + struct inode_fs_paths *ifp; + struct btrfs_data_container *fspath; + + fspath = init_data_container(total_bytes); + if (IS_ERR(fspath)) + return ERR_CAST(fspath); + + ifp = kmalloc_obj(*ifp); + if (!ifp) { + kvfree(fspath); + return ERR_PTR(-ENOMEM); + } + + ifp->btrfs_path = path; + ifp->fspath = fspath; + ifp->fs_root = fs_root; + + return ifp; +} + +int btrfs_backref_iter_init(struct btrfs_backref_iter *iter) +{ + iter->path = btrfs_alloc_path(); + if (!iter->path) + return -ENOMEM; + + /* Current backref iterator only supports iteration in commit root */ + iter->path->search_commit_root = true; + iter->path->skip_locking = true; + + return 0; +} + +static void btrfs_backref_iter_release(struct btrfs_backref_iter *iter) +{ + iter->bytenr = 0; + iter->item_ptr = 0; + iter->cur_ptr = 0; + iter->end_ptr = 0; + btrfs_release_path(iter->path); + memset(&iter->cur_key, 0, sizeof(iter->cur_key)); +} + +int btrfs_backref_iter_start(struct btrfs_fs_info *fs_info, struct btrfs_backref_iter *iter, u64 bytenr) +{ + struct btrfs_root *extent_root = btrfs_extent_root(fs_info, bytenr); + struct btrfs_path *path = iter->path; + struct btrfs_extent_item *ei; + struct btrfs_key key; + int ret; + + if (unlikely(!extent_root)) { + btrfs_err(fs_info, + "missing extent root for extent at bytenr %llu", + bytenr); + return -EUCLEAN; + } + + key.objectid = bytenr; + key.type = BTRFS_METADATA_ITEM_KEY; + key.offset = (u64)-1; + iter->bytenr = bytenr; + + ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0); + if (ret < 0) + return ret; + if (unlikely(ret == 0)) { + /* + * Key with offset -1 found, there would have to exist an extent + * item with such offset, but this is out of the valid range. + */ + ret = -EUCLEAN; + goto release; + } + if (unlikely(path->slots[0] == 0)) { + DEBUG_WARN(); + ret = -EUCLEAN; + goto release; + } + path->slots[0]--; + + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + if ((key.type != BTRFS_EXTENT_ITEM_KEY && + key.type != BTRFS_METADATA_ITEM_KEY) || key.objectid != bytenr) { + ret = -ENOENT; + goto release; + } + memcpy(&iter->cur_key, &key, sizeof(key)); + iter->item_ptr = (u32)btrfs_item_ptr_offset(path->nodes[0], + path->slots[0]); + iter->end_ptr = (u32)(iter->item_ptr + + btrfs_item_size(path->nodes[0], path->slots[0])); + ei = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_extent_item); + + /* + * Only support iteration on tree backref yet. + * + * This is an extra precaution for non skinny-metadata, where + * EXTENT_ITEM is also used for tree blocks, that we can only use + * extent flags to determine if it's a tree block. + */ + if (btrfs_extent_flags(path->nodes[0], ei) & BTRFS_EXTENT_FLAG_DATA) { + ret = -ENOTSUPP; + goto release; + } + iter->cur_ptr = (u32)(iter->item_ptr + sizeof(*ei)); + + /* If there is no inline backref, go search for keyed backref */ + if (iter->cur_ptr >= iter->end_ptr) { + ret = btrfs_next_item(extent_root, path); + + /* No inline nor keyed ref */ + if (ret > 0) { + ret = -ENOENT; + goto release; + } + if (ret < 0) + goto release; + + btrfs_item_key_to_cpu(path->nodes[0], &iter->cur_key, + path->slots[0]); + if (iter->cur_key.objectid != bytenr || + (iter->cur_key.type != BTRFS_SHARED_BLOCK_REF_KEY && + iter->cur_key.type != BTRFS_TREE_BLOCK_REF_KEY)) { + ret = -ENOENT; + goto release; + } + iter->cur_ptr = (u32)btrfs_item_ptr_offset(path->nodes[0], + path->slots[0]); + iter->item_ptr = iter->cur_ptr; + iter->end_ptr = (u32)(iter->item_ptr + btrfs_item_size( + path->nodes[0], path->slots[0])); + } + + return 0; +release: + btrfs_backref_iter_release(iter); + return ret; +} + +static bool btrfs_backref_iter_is_inline_ref(struct btrfs_backref_iter *iter) +{ + if (iter->cur_key.type == BTRFS_EXTENT_ITEM_KEY || + iter->cur_key.type == BTRFS_METADATA_ITEM_KEY) + return true; + return false; +} + +/* + * Go to the next backref item of current bytenr, can be either inlined or + * keyed. + * + * Caller needs to check whether it's inline ref or not by iter->cur_key. + * + * Return 0 if we get next backref without problem. + * Return >0 if there is no extra backref for this bytenr. + * Return <0 if there is something wrong happened. + */ +int btrfs_backref_iter_next(struct btrfs_fs_info *fs_info, struct btrfs_backref_iter *iter) +{ + struct extent_buffer *eb = iter->path->nodes[0]; + struct btrfs_root *extent_root; + struct btrfs_path *path = iter->path; + struct btrfs_extent_inline_ref *iref; + int ret; + u32 size; + + if (btrfs_backref_iter_is_inline_ref(iter)) { + /* We're still inside the inline refs */ + ASSERT(iter->cur_ptr < iter->end_ptr, + "iter->cur_ptr=%u iter->end_ptr=%u", + iter->cur_ptr, iter->end_ptr); + + if (btrfs_backref_has_tree_block_info(iter)) { + /* First tree block info */ + size = sizeof(struct btrfs_tree_block_info); + } else { + /* Use inline ref type to determine the size */ + int type; + + iref = (struct btrfs_extent_inline_ref *) + ((unsigned long)iter->cur_ptr); + type = btrfs_extent_inline_ref_type(eb, iref); + + size = btrfs_extent_inline_ref_size(type); + } + iter->cur_ptr += size; + if (iter->cur_ptr < iter->end_ptr) + return 0; + + /* All inline items iterated, fall through */ + } + + /* We're at keyed items, there is no inline item, go to the next one */ + extent_root = btrfs_extent_root(fs_info, iter->bytenr); + if (unlikely(!extent_root)) { + btrfs_err(fs_info, "missing extent root for extent at bytenr %llu", + iter->bytenr); + return -EUCLEAN; + } + + ret = btrfs_next_item(extent_root, iter->path); + if (ret) + return ret; + + btrfs_item_key_to_cpu(path->nodes[0], &iter->cur_key, path->slots[0]); + if (iter->cur_key.objectid != iter->bytenr || + (iter->cur_key.type != BTRFS_TREE_BLOCK_REF_KEY && + iter->cur_key.type != BTRFS_SHARED_BLOCK_REF_KEY)) + return 1; + iter->item_ptr = (u32)btrfs_item_ptr_offset(path->nodes[0], + path->slots[0]); + iter->cur_ptr = iter->item_ptr; + iter->end_ptr = iter->item_ptr + (u32)btrfs_item_size(path->nodes[0], + path->slots[0]); + return 0; +} + +void btrfs_backref_init_cache(struct btrfs_fs_info *fs_info, + struct btrfs_backref_cache *cache, bool is_reloc) +{ + int i; + + cache->rb_root = RB_ROOT; + for (i = 0; i < BTRFS_MAX_LEVEL; i++) + INIT_LIST_HEAD(&cache->pending[i]); + INIT_LIST_HEAD(&cache->pending_edge); + INIT_LIST_HEAD(&cache->useless_node); + cache->fs_info = fs_info; + cache->is_reloc = is_reloc; +} + +struct btrfs_backref_node *btrfs_backref_alloc_node( + struct btrfs_backref_cache *cache, u64 bytenr, int level) +{ + struct btrfs_backref_node *node; + + ASSERT(level >= 0 && level < BTRFS_MAX_LEVEL, "level=%d", level); + node = kzalloc_obj(*node, GFP_NOFS); + if (!node) + return node; + + INIT_LIST_HEAD(&node->list); + INIT_LIST_HEAD(&node->upper); + INIT_LIST_HEAD(&node->lower); + RB_CLEAR_NODE(&node->rb_node); + cache->nr_nodes++; + node->level = level; + node->bytenr = bytenr; + + return node; +} + +void btrfs_backref_free_node(struct btrfs_backref_cache *cache, + struct btrfs_backref_node *node) +{ + if (node) { + ASSERT(list_empty(&node->list)); + ASSERT(list_empty(&node->lower)); + ASSERT(node->eb == NULL, "node->eb->start=%llu level=%d owner=%llu", + node->eb ? node->eb->start : 0, + node->eb ? btrfs_header_level(node->eb) : 0, + node->eb ? btrfs_header_owner(node->eb) : 0); + cache->nr_nodes--; + btrfs_put_root(node->root); + kfree(node); + } +} + +struct btrfs_backref_edge *btrfs_backref_alloc_edge( + struct btrfs_backref_cache *cache) +{ + struct btrfs_backref_edge *edge; + + edge = kzalloc_obj(*edge, GFP_NOFS); + if (edge) + cache->nr_edges++; + return edge; +} + +void btrfs_backref_free_edge(struct btrfs_backref_cache *cache, + struct btrfs_backref_edge *edge) +{ + if (edge) { + cache->nr_edges--; + kfree(edge); + } +} + +void btrfs_backref_unlock_node_buffer(struct btrfs_backref_node *node) +{ + if (node->locked) { + btrfs_tree_unlock(node->eb); + node->locked = 0; + } +} + +void btrfs_backref_drop_node_buffer(struct btrfs_backref_node *node) +{ + if (node->eb) { + btrfs_backref_unlock_node_buffer(node); + free_extent_buffer(node->eb); + node->eb = NULL; + } +} + +/* + * Drop the backref node from cache without cleaning up its children + * edges. + * + * This can only be called on node without parent edges. + * The children edges are still kept as is. + */ +void btrfs_backref_drop_node(struct btrfs_backref_cache *tree, + struct btrfs_backref_node *node) +{ + ASSERT(list_empty(&node->upper)); + + btrfs_backref_drop_node_buffer(node); + list_del_init(&node->list); + list_del_init(&node->lower); + if (!RB_EMPTY_NODE(&node->rb_node)) + rb_erase(&node->rb_node, &tree->rb_root); + btrfs_backref_free_node(tree, node); +} + +/* + * Drop the backref node from cache, also cleaning up all its + * upper edges and any uncached nodes in the path. + * + * This cleanup happens bottom up, thus the node should either + * be the lowest node in the cache or a detached node. + */ +void btrfs_backref_cleanup_node(struct btrfs_backref_cache *cache, + struct btrfs_backref_node *node) +{ + struct btrfs_backref_edge *edge; + + if (!node) + return; + + while (!list_empty(&node->upper)) { + edge = list_first_entry(&node->upper, struct btrfs_backref_edge, + list[LOWER]); + list_del(&edge->list[LOWER]); + list_del(&edge->list[UPPER]); + btrfs_backref_free_edge(cache, edge); + } + + btrfs_backref_drop_node(cache, node); +} + +/* + * Release all nodes/edges from current cache + */ +void btrfs_backref_release_cache(struct btrfs_backref_cache *cache) +{ + struct btrfs_backref_node *node; + + while ((node = rb_entry_safe(rb_first(&cache->rb_root), + struct btrfs_backref_node, rb_node))) + btrfs_backref_cleanup_node(cache, node); + + ASSERT(list_empty(&cache->pending_edge)); + ASSERT(list_empty(&cache->useless_node)); + ASSERT(!cache->nr_nodes, "cache->nr_nodes=%d", cache->nr_nodes); + ASSERT(!cache->nr_edges, "cache->nr_edges=%d", cache->nr_edges); +} + +static void btrfs_backref_link_edge(struct btrfs_backref_edge *edge, + struct btrfs_backref_node *lower, + struct btrfs_backref_node *upper) +{ + ASSERT(upper != NULL); + ASSERT(lower != NULL); + ASSERT(upper->level == lower->level + 1, "upper->level=%d lower->level=%d", + upper->level, lower->level); + edge->node[LOWER] = lower; + edge->node[UPPER] = upper; + list_add_tail(&edge->list[LOWER], &lower->upper); +} +/* + * Handle direct tree backref + * + * Direct tree backref means, the backref item shows its parent bytenr + * directly. This is for SHARED_BLOCK_REF backref (keyed or inlined). + * + * @ref_key: The converted backref key. + * For keyed backref, it's the item key. + * For inlined backref, objectid is the bytenr, + * type is btrfs_inline_ref_type, offset is + * btrfs_inline_ref_offset. + */ +static int handle_direct_tree_backref(struct btrfs_backref_cache *cache, + struct btrfs_key *ref_key, + struct btrfs_backref_node *cur) +{ + struct btrfs_backref_edge *edge; + struct btrfs_backref_node *upper; + struct rb_node *rb_node; + + ASSERT(ref_key->type == BTRFS_SHARED_BLOCK_REF_KEY, "ref_key->type=%hhu", ref_key->type); + + /* Only reloc root uses backref pointing to itself */ + if (ref_key->objectid == ref_key->offset) { + struct btrfs_root *root; + + cur->is_reloc_root = 1; + /* Only reloc backref cache cares about a specific root */ + if (cache->is_reloc) { + root = find_reloc_root(cache->fs_info, cur->bytenr); + if (!root) + return -ENOENT; + cur->root = root; + } else { + /* + * For generic purpose backref cache, reloc root node + * is useless. + */ + list_add(&cur->list, &cache->useless_node); + } + return 0; + } + + edge = btrfs_backref_alloc_edge(cache); + if (!edge) + return -ENOMEM; + + rb_node = rb_simple_search(&cache->rb_root, ref_key->offset); + if (!rb_node) { + /* Parent node not yet cached */ + upper = btrfs_backref_alloc_node(cache, ref_key->offset, + cur->level + 1); + if (!upper) { + btrfs_backref_free_edge(cache, edge); + return -ENOMEM; + } + + /* + * Backrefs for the upper level block isn't cached, add the + * block to pending list + */ + list_add_tail(&edge->list[UPPER], &cache->pending_edge); + } else { + /* Parent node already cached */ + upper = rb_entry(rb_node, struct btrfs_backref_node, rb_node); + ASSERT(upper->checked); + INIT_LIST_HEAD(&edge->list[UPPER]); + } + btrfs_backref_link_edge(edge, cur, upper); + return 0; +} + +/* + * Handle indirect tree backref + * + * Indirect tree backref means, we only know which tree the node belongs to. + * We still need to do a tree search to find out the parents. This is for + * TREE_BLOCK_REF backref (keyed or inlined). + * + * @trans: Transaction handle. + * @ref_key: The same as @ref_key in handle_direct_tree_backref() + * @tree_key: The first key of this tree block. + * @path: A clean (released) path, to avoid allocating path every time + * the function get called. + */ +static int handle_indirect_tree_backref(struct btrfs_trans_handle *trans, + struct btrfs_backref_cache *cache, + struct btrfs_path *path, + struct btrfs_key *ref_key, + struct btrfs_key *tree_key, + struct btrfs_backref_node *cur) +{ + struct btrfs_fs_info *fs_info = cache->fs_info; + struct btrfs_backref_node *upper; + struct btrfs_backref_node *lower; + struct btrfs_backref_edge *edge; + struct extent_buffer *eb; + struct btrfs_root *root; + struct rb_node *rb_node; + int level; + bool need_check = true; + int ret; + + root = btrfs_get_fs_root(fs_info, ref_key->offset, false); + if (IS_ERR(root)) + return PTR_ERR(root); + + /* We shouldn't be using backref cache for non-shareable roots. */ + if (unlikely(!test_bit(BTRFS_ROOT_SHAREABLE, &root->state))) { + btrfs_put_root(root); + return -EUCLEAN; + } + + if (btrfs_root_level(&root->root_item) == cur->level) { + /* Tree root */ + ASSERT(btrfs_root_bytenr(&root->root_item) == cur->bytenr, + "root_bytenr=%llu cur->bytenr=%llu", + btrfs_root_bytenr(&root->root_item), cur->bytenr); + /* + * For reloc backref cache, we may ignore reloc root. But for + * general purpose backref cache, we can't rely on + * btrfs_should_ignore_reloc_root() as it may conflict with + * current running relocation and lead to missing root. + * + * For general purpose backref cache, reloc root detection is + * completely relying on direct backref (key->offset is parent + * bytenr), thus only do such check for reloc cache. + */ + if (btrfs_should_ignore_reloc_root(root) && cache->is_reloc) { + btrfs_put_root(root); + list_add(&cur->list, &cache->useless_node); + } else { + cur->root = root; + } + return 0; + } + + level = cur->level + 1; + + /* Search the tree to find parent blocks referring to the block */ + path->search_commit_root = true; + path->skip_locking = true; + path->lowest_level = level; + ret = btrfs_search_slot(NULL, root, tree_key, path, 0, 0); + path->lowest_level = 0; + if (ret < 0) { + btrfs_put_root(root); + return ret; + } + if (ret > 0 && path->slots[level] > 0) + path->slots[level]--; + + eb = path->nodes[level]; + if (btrfs_node_blockptr(eb, path->slots[level]) != cur->bytenr) { + btrfs_err(fs_info, +"couldn't find block (%llu) (level %d) in tree (%llu) with key " BTRFS_KEY_FMT, + cur->bytenr, level - 1, btrfs_root_id(root), + BTRFS_KEY_FMT_VALUE(tree_key)); + btrfs_put_root(root); + ret = -ENOENT; + goto out; + } + lower = cur; + + /* Add all nodes and edges in the path */ + for (; level < BTRFS_MAX_LEVEL; level++) { + if (!path->nodes[level]) { + ASSERT(btrfs_root_bytenr(&root->root_item) == lower->bytenr, + "root_bytenr=%llu lower->bytenr=%llu", + btrfs_root_bytenr(&root->root_item), lower->bytenr); + /* Same as previous should_ignore_reloc_root() call */ + if (btrfs_should_ignore_reloc_root(root) && + cache->is_reloc) { + btrfs_put_root(root); + list_add(&lower->list, &cache->useless_node); + } else { + lower->root = root; + } + break; + } + + edge = btrfs_backref_alloc_edge(cache); + if (!edge) { + btrfs_put_root(root); + ret = -ENOMEM; + goto out; + } + + eb = path->nodes[level]; + rb_node = rb_simple_search(&cache->rb_root, eb->start); + if (!rb_node) { + upper = btrfs_backref_alloc_node(cache, eb->start, + lower->level + 1); + if (!upper) { + btrfs_put_root(root); + btrfs_backref_free_edge(cache, edge); + ret = -ENOMEM; + goto out; + } + upper->owner = btrfs_header_owner(eb); + + /* We shouldn't be using backref cache for non shareable roots. */ + if (unlikely(!test_bit(BTRFS_ROOT_SHAREABLE, &root->state))) { + btrfs_put_root(root); + btrfs_backref_free_edge(cache, edge); + btrfs_backref_free_node(cache, upper); + ret = -EUCLEAN; + goto out; + } + + /* + * If we know the block isn't shared we can avoid + * checking its backrefs. + */ + if (btrfs_block_can_be_shared(trans, root, eb)) + upper->checked = 0; + else + upper->checked = 1; + + /* + * Add the block to pending list if we need to check its + * backrefs, we only do this once while walking up a + * tree as we will catch anything else later on. + */ + if (!upper->checked && need_check) { + need_check = false; + list_add_tail(&edge->list[UPPER], + &cache->pending_edge); + } else { + if (upper->checked) + need_check = true; + INIT_LIST_HEAD(&edge->list[UPPER]); + } + } else { + upper = rb_entry(rb_node, struct btrfs_backref_node, + rb_node); + ASSERT(upper->checked); + INIT_LIST_HEAD(&edge->list[UPPER]); + if (!upper->owner) + upper->owner = btrfs_header_owner(eb); + } + btrfs_backref_link_edge(edge, lower, upper); + + if (rb_node) { + btrfs_put_root(root); + break; + } + lower = upper; + upper = NULL; + } +out: + btrfs_release_path(path); + return ret; +} + +/* + * Add backref node @cur into @cache. + * + * NOTE: Even if the function returned 0, @cur is not yet cached as its upper + * links aren't yet bi-directional. Needs to finish such links. + * Use btrfs_backref_finish_upper_links() to finish such linkage. + * + * @trans: Transaction handle. + * @path: Released path for indirect tree backref lookup + * @iter: Released backref iter for extent tree search + * @node_key: The first key of the tree block + */ +int btrfs_backref_add_tree_node(struct btrfs_trans_handle *trans, + struct btrfs_backref_cache *cache, + struct btrfs_path *path, + struct btrfs_backref_iter *iter, + struct btrfs_key *node_key, + struct btrfs_backref_node *cur) +{ + struct btrfs_backref_edge *edge; + struct btrfs_backref_node *exist; + int ret; + + ret = btrfs_backref_iter_start(trans->fs_info, iter, cur->bytenr); + if (ret < 0) + return ret; + /* + * We skip the first btrfs_tree_block_info, as we don't use the key + * stored in it, but fetch it from the tree block + */ + if (btrfs_backref_has_tree_block_info(iter)) { + ret = btrfs_backref_iter_next(trans->fs_info, iter); + if (ret < 0) + goto out; + /* No extra backref? This means the tree block is corrupted */ + if (unlikely(ret > 0)) { + ret = -EUCLEAN; + goto out; + } + } + WARN_ON(cur->checked); + if (!list_empty(&cur->upper)) { + /* + * The backref was added previously when processing backref of + * type BTRFS_TREE_BLOCK_REF_KEY + */ + ASSERT(list_is_singular(&cur->upper)); + edge = list_first_entry(&cur->upper, struct btrfs_backref_edge, + list[LOWER]); + ASSERT(list_empty(&edge->list[UPPER])); + exist = edge->node[UPPER]; + /* + * Add the upper level block to pending list if we need check + * its backrefs + */ + if (!exist->checked) + list_add_tail(&edge->list[UPPER], &cache->pending_edge); + } else { + exist = NULL; + } + + for (; ret == 0; ret = btrfs_backref_iter_next(trans->fs_info, iter)) { + struct extent_buffer *eb; + struct btrfs_key key; + int type; + + cond_resched(); + eb = iter->path->nodes[0]; + + key.objectid = iter->bytenr; + if (btrfs_backref_iter_is_inline_ref(iter)) { + struct btrfs_extent_inline_ref *iref; + + /* Update key for inline backref */ + iref = (struct btrfs_extent_inline_ref *) + ((unsigned long)iter->cur_ptr); + type = btrfs_get_extent_inline_ref_type(eb, iref, + BTRFS_REF_TYPE_BLOCK); + if (unlikely(type == BTRFS_REF_TYPE_INVALID)) { + ret = -EUCLEAN; + goto out; + } + key.type = type; + key.offset = btrfs_extent_inline_ref_offset(eb, iref); + } else { + key.type = iter->cur_key.type; + key.offset = iter->cur_key.offset; + } + + /* + * Parent node found and matches current inline ref, no need to + * rebuild this node for this inline ref + */ + if (exist && + ((key.type == BTRFS_TREE_BLOCK_REF_KEY && + exist->owner == key.offset) || + (key.type == BTRFS_SHARED_BLOCK_REF_KEY && + exist->bytenr == key.offset))) { + exist = NULL; + continue; + } + + /* SHARED_BLOCK_REF means key.offset is the parent bytenr */ + if (key.type == BTRFS_SHARED_BLOCK_REF_KEY) { + ret = handle_direct_tree_backref(cache, &key, cur); + if (ret < 0) + goto out; + } else if (key.type == BTRFS_TREE_BLOCK_REF_KEY) { + /* + * key.type == BTRFS_TREE_BLOCK_REF_KEY, inline ref + * offset means the root objectid. We need to search + * the tree to get its parent bytenr. + */ + ret = handle_indirect_tree_backref(trans, cache, path, + &key, node_key, cur); + if (ret < 0) + goto out; + } + /* + * Unrecognized tree backref items (if it can pass tree-checker) + * would be ignored. + */ + } + ret = 0; + cur->checked = 1; + WARN_ON(exist); +out: + btrfs_backref_iter_release(iter); + return ret; +} + +/* + * Finish the upwards linkage created by btrfs_backref_add_tree_node() + */ +int btrfs_backref_finish_upper_links(struct btrfs_backref_cache *cache, + struct btrfs_backref_node *start) +{ + struct list_head *useless_node = &cache->useless_node; + struct btrfs_backref_edge *edge; + struct rb_node *rb_node; + LIST_HEAD(pending_edge); + + ASSERT(start->checked); + + rb_node = rb_simple_insert(&cache->rb_root, &start->simple_node); + if (rb_node) + btrfs_backref_panic(cache->fs_info, start->bytenr, -EEXIST); + + /* + * Use breadth first search to iterate all related edges. + * + * The starting points are all the edges of this node + */ + list_for_each_entry(edge, &start->upper, list[LOWER]) + list_add_tail(&edge->list[UPPER], &pending_edge); + + while (!list_empty(&pending_edge)) { + struct btrfs_backref_node *upper; + struct btrfs_backref_node *lower; + + edge = list_first_entry(&pending_edge, + struct btrfs_backref_edge, list[UPPER]); + list_del_init(&edge->list[UPPER]); + upper = edge->node[UPPER]; + lower = edge->node[LOWER]; + + /* Parent is detached, no need to keep any edges */ + if (upper->detached) { + list_del(&edge->list[LOWER]); + btrfs_backref_free_edge(cache, edge); + + /* Lower node is orphan, queue for cleanup */ + if (list_empty(&lower->upper)) + list_add(&lower->list, useless_node); + continue; + } + + /* + * All new nodes added in current build_backref_tree() haven't + * been linked to the cache rb tree. + * So if we have upper->rb_node populated, this means a cache + * hit. We only need to link the edge, as @upper and all its + * parents have already been linked. + */ + if (!RB_EMPTY_NODE(&upper->rb_node)) { + list_add_tail(&edge->list[UPPER], &upper->lower); + continue; + } + + /* Sanity check, we shouldn't have any unchecked nodes */ + if (unlikely(!upper->checked)) { + DEBUG_WARN("we should not have any unchecked nodes"); + return -EUCLEAN; + } + + rb_node = rb_simple_insert(&cache->rb_root, &upper->simple_node); + if (unlikely(rb_node)) + btrfs_backref_panic(cache->fs_info, upper->bytenr, -EEXIST); + + list_add_tail(&edge->list[UPPER], &upper->lower); + + /* + * Also queue all the parent edges of this uncached node + * to finish the upper linkage + */ + list_for_each_entry(edge, &upper->upper, list[LOWER]) + list_add_tail(&edge->list[UPPER], &pending_edge); + } + return 0; +} + +void btrfs_backref_error_cleanup(struct btrfs_backref_cache *cache, + struct btrfs_backref_node *node) +{ + struct btrfs_backref_node *lower; + struct btrfs_backref_node *upper; + struct btrfs_backref_edge *edge; + + while (!list_empty(&cache->useless_node)) { + lower = list_first_entry(&cache->useless_node, + struct btrfs_backref_node, list); + list_del_init(&lower->list); + } + while (!list_empty(&cache->pending_edge)) { + edge = list_first_entry(&cache->pending_edge, + struct btrfs_backref_edge, list[UPPER]); + list_del(&edge->list[UPPER]); + list_del(&edge->list[LOWER]); + lower = edge->node[LOWER]; + upper = edge->node[UPPER]; + btrfs_backref_free_edge(cache, edge); + + /* + * Lower is no longer linked to any upper backref nodes and + * isn't in the cache, we can free it ourselves. + */ + if (list_empty(&lower->upper) && + RB_EMPTY_NODE(&lower->rb_node)) + list_add(&lower->list, &cache->useless_node); + + if (!RB_EMPTY_NODE(&upper->rb_node)) + continue; + + /* Add this guy's upper edges to the list to process */ + list_for_each_entry(edge, &upper->upper, list[LOWER]) + list_add_tail(&edge->list[UPPER], + &cache->pending_edge); + if (list_empty(&upper->upper)) + list_add(&upper->list, &cache->useless_node); + } + + while (!list_empty(&cache->useless_node)) { + lower = list_first_entry(&cache->useless_node, + struct btrfs_backref_node, list); + list_del_init(&lower->list); + if (lower == node) + node = NULL; + btrfs_backref_drop_node(cache, lower); + } + + btrfs_backref_cleanup_node(cache, node); + ASSERT(list_empty(&cache->useless_node) && + list_empty(&cache->pending_edge)); +} diff --git a/fs/btrfs/backref.h b/fs/btrfs/backref.h new file mode 100644 index 000000000..179791de6 --- /dev/null +++ b/fs/btrfs/backref.h @@ -0,0 +1,469 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2011 STRATO. All rights reserved. + */ + +#ifndef BTRFS_BACKREF_H +#define BTRFS_BACKREF_H + +#include +#include +#include +#include +#include +#include +#include "messages.h" +#include "locking.h" +#include "disk-io.h" +#include "extent_io.h" +#include "ctree.h" + +struct extent_inode_elem; +struct ulist; +struct btrfs_extent_item; +struct btrfs_trans_handle; +struct btrfs_fs_info; + +/* + * Used by implementations of iterate_extent_inodes_t (see definition below) to + * signal that backref iteration can stop immediately and no error happened. + * The value must be non-negative and must not be 0, 1 (which is a common return + * value from things like btrfs_search_slot() and used internally in the backref + * walking code) and different from BACKREF_FOUND_SHARED and + * BACKREF_FOUND_NOT_SHARED + */ +#define BTRFS_ITERATE_EXTENT_INODES_STOP 5 + +/* + * Should return 0 if no errors happened and iteration of backrefs should + * continue. Can return BTRFS_ITERATE_EXTENT_INODES_STOP or any other non-zero + * value to immediately stop iteration and possibly signal an error back to + * the caller. + */ +typedef int (iterate_extent_inodes_t)(u64 inum, u64 offset, u64 num_bytes, + u64 root, void *ctx); + +/* + * Context and arguments for backref walking functions. Some of the fields are + * to be filled by the caller of such functions while other are filled by the + * functions themselves, as described below. + */ +struct btrfs_backref_walk_ctx { + /* + * The address of the extent for which we are doing backref walking. + * Can be either a data extent or a metadata extent. + * + * Must always be set by the top level caller. + */ + u64 bytenr; + /* + * Offset relative to the target extent. This is only used for data + * extents, and it's meaningful because we can have file extent items + * that point only to a section of a data extent ("bookend" extents), + * and we want to filter out any that don't point to a section of the + * data extent containing the given offset. + * + * Must always be set by the top level caller. + */ + u64 extent_item_pos; + /* + * If true and bytenr corresponds to a data extent, then references from + * all file extent items that point to the data extent are considered, + * @extent_item_pos is ignored. + */ + bool ignore_extent_item_pos; + /* + * If true and bytenr corresponds to a data extent, then the inode list + * (each member describing inode number, file offset and root) is not + * added to each reference added to the @refs ulist. + */ + bool skip_inode_ref_list; + /* A valid transaction handle or NULL. */ + struct btrfs_trans_handle *trans; + /* + * The file system's info object, can not be NULL. + * + * Must always be set by the top level caller. + */ + struct btrfs_fs_info *fs_info; + /* + * Time sequence acquired from btrfs_get_tree_mod_seq(), in case the + * caller joined the tree mod log to get a consistent view of b+trees + * while we do backref walking, or BTRFS_SEQ_LAST. + * When using BTRFS_SEQ_LAST, delayed refs are not checked and it uses + * commit roots when searching b+trees - this is a special case for + * qgroups used during a transaction commit. + */ + u64 time_seq; + /* + * Used to collect the bytenr of metadata extents that point to the + * target extent. + */ + struct ulist *refs; + /* + * List used to collect the IDs of the roots from which the target + * extent is accessible. Can be NULL in case the caller does not care + * about collecting root IDs. + */ + struct ulist *roots; + /* + * Used by iterate_extent_inodes() and the main backref walk code + * (find_parent_nodes()). Lookup and store functions for an optional + * cache which maps the logical address (bytenr) of leaves to an array + * of root IDs. + */ + bool (*cache_lookup)(u64 leaf_bytenr, void *user_ctx, + const u64 **root_ids_ret, int *root_count_ret); + void (*cache_store)(u64 leaf_bytenr, const struct ulist *root_ids, + void *user_ctx); + /* + * If this is not NULL, then the backref walking code will call this + * for each indirect data extent reference as soon as it finds one, + * before collecting all the remaining backrefs and before resolving + * indirect backrefs. This allows for the caller to terminate backref + * walking as soon as it finds one backref that matches some specific + * criteria. The @cache_lookup and @cache_store callbacks should not + * be NULL in order to use this callback. + */ + iterate_extent_inodes_t *indirect_ref_iterator; + /* + * If this is not NULL, then the backref walking code will call this for + * each extent item it's meant to process before it actually starts + * processing it. If this returns anything other than 0, then it stops + * the backref walking code immediately. + */ + int (*check_extent_item)(u64 bytenr, const struct btrfs_extent_item *ei, + const struct extent_buffer *leaf, void *user_ctx); + /* + * If this is not NULL, then the backref walking code will call this for + * each extent data ref it finds (BTRFS_EXTENT_DATA_REF_KEY keys) before + * processing that data ref. If this callback return false, then it will + * ignore this data ref and it will never resolve the indirect data ref, + * saving time searching for leaves in a fs tree with file extent items + * matching the data ref. + */ + bool (*skip_data_ref)(u64 root, u64 ino, u64 offset, void *user_ctx); + /* Context object to pass to the callbacks defined above. */ + void *user_ctx; +}; + +struct inode_fs_paths { + struct btrfs_path *btrfs_path; + struct btrfs_root *fs_root; + struct btrfs_data_container *fspath; +}; + +struct btrfs_backref_shared_cache_entry { + u64 bytenr; + u64 gen; + bool is_shared; +}; + +#define BTRFS_BACKREF_CTX_PREV_EXTENTS_SIZE 8 + +struct btrfs_backref_share_check_ctx { + /* Ulists used during backref walking. */ + struct ulist refs; + /* + * The current leaf the caller of btrfs_is_data_extent_shared() is at. + * Typically the caller (at the moment only fiemap) tries to determine + * the sharedness of data extents point by file extent items from entire + * leaves. + */ + u64 curr_leaf_bytenr; + /* + * The previous leaf the caller was at in the previous call to + * btrfs_is_data_extent_shared(). This may be the same as the current + * leaf. On the first call it must be 0. + */ + u64 prev_leaf_bytenr; + /* + * A path from a root to a leaf that has a file extent item pointing to + * a given data extent should never exceed the maximum b+tree height. + */ + struct btrfs_backref_shared_cache_entry path_cache_entries[BTRFS_MAX_LEVEL]; + bool use_path_cache; + /* + * Cache the sharedness result for the last few extents we have found, + * but only for extents for which we have multiple file extent items + * that point to them. + * It's very common to have several file extent items that point to the + * same extent (bytenr) but with different offsets and lengths. This + * typically happens for COW writes, partial writes into prealloc + * extents, NOCOW writes after snapshotting a root, hole punching or + * reflinking within the same file (less common perhaps). + * So keep a small cache with the lookup results for the extent pointed + * by the last few file extent items. This cache is checked, with a + * linear scan, whenever btrfs_is_data_extent_shared() is called, so + * it must be small so that it does not negatively affect performance in + * case we don't have multiple file extent items that point to the same + * data extent. + */ + struct { + u64 bytenr; + bool is_shared; + } prev_extents_cache[BTRFS_BACKREF_CTX_PREV_EXTENTS_SIZE]; + /* + * The slot in the prev_extents_cache array that will be used for + * storing the sharedness result of a new data extent. + */ + int prev_extents_cache_slot; +}; + +struct btrfs_backref_share_check_ctx *btrfs_alloc_backref_share_check_ctx(void); +void btrfs_free_backref_share_ctx(struct btrfs_backref_share_check_ctx *ctx); + +int extent_from_logical(struct btrfs_fs_info *fs_info, u64 logical, + struct btrfs_path *path, struct btrfs_key *found_key, + u64 *flags); + +int tree_backref_for_extent(unsigned long *ptr, struct extent_buffer *eb, + struct btrfs_key *key, struct btrfs_extent_item *ei, + u32 item_size, u64 *out_root, u8 *out_level); + +int iterate_extent_inodes(struct btrfs_backref_walk_ctx *ctx, + bool search_commit_root, + iterate_extent_inodes_t *iterate, void *user_ctx); + +int iterate_inodes_from_logical(u64 logical, struct btrfs_fs_info *fs_info, + void *ctx, bool ignore_offset); + +int paths_from_inode(u64 inum, struct inode_fs_paths *ipath); + +int btrfs_find_all_leafs(struct btrfs_backref_walk_ctx *ctx); +int btrfs_find_all_roots(struct btrfs_backref_walk_ctx *ctx, + bool skip_commit_root_sem); +char *btrfs_ref_to_path(struct btrfs_root *fs_root, struct btrfs_path *path, + u32 name_len, unsigned long name_off, + struct extent_buffer *eb_in, u64 parent, + char *dest, u32 size); + +struct btrfs_data_container *init_data_container(u32 total_bytes); +struct inode_fs_paths *init_ipath(s32 total_bytes, struct btrfs_root *fs_root, + struct btrfs_path *path); + +DEFINE_FREE(inode_fs_paths, struct inode_fs_paths *, + if (_T) { + kvfree(_T->fspath); + kfree(_T); + }) + +int btrfs_find_one_extref(struct btrfs_root *root, u64 inode_objectid, + u64 start_off, struct btrfs_path *path, + struct btrfs_inode_extref **ret_extref, + u64 *found_off); +int btrfs_is_data_extent_shared(struct btrfs_inode *inode, u64 bytenr, + u64 extent_gen, + struct btrfs_backref_share_check_ctx *ctx); + +int __init btrfs_prelim_ref_init(void); +void __cold btrfs_prelim_ref_exit(void); + +struct prelim_ref { + struct rb_node rbnode; + u64 root_id; + struct btrfs_key key_for_search; + u8 level; + int count; + struct extent_inode_elem *inode_list; + u64 parent; + u64 wanted_disk_byte; +}; + +/* + * Iterate backrefs of one extent. + * + * Now it only supports iteration of tree block in commit root. + */ +struct btrfs_backref_iter { + u64 bytenr; + struct btrfs_path *path; + struct btrfs_key cur_key; + u32 item_ptr; + u32 cur_ptr; + u32 end_ptr; +}; + +/* + * For metadata with EXTENT_ITEM key (non-skinny) case, the first inline data + * is btrfs_tree_block_info, without a btrfs_extent_inline_ref header. + * + * This helper determines if that's the case. + */ +static inline bool btrfs_backref_has_tree_block_info( + struct btrfs_backref_iter *iter) +{ + if (iter->cur_key.type == BTRFS_EXTENT_ITEM_KEY && + iter->cur_ptr - iter->item_ptr == sizeof(struct btrfs_extent_item)) + return true; + return false; +} + +int btrfs_backref_iter_init(struct btrfs_backref_iter *iter); + +int btrfs_backref_iter_start(struct btrfs_fs_info *fs_info, struct btrfs_backref_iter *iter, u64 bytenr); + +int btrfs_backref_iter_next(struct btrfs_fs_info *fs_info, struct btrfs_backref_iter *iter); + +/* + * Backref cache related structures + * + * The whole objective of backref_cache is to build a bi-directional map + * of tree blocks (represented by backref_node) and all their parents. + */ + +/* + * Represent a tree block in the backref cache + */ +struct btrfs_backref_node { + union{ + /* Use rb_simple_node for search/insert */ + struct { + struct rb_node rb_node; + u64 bytenr; + }; + + struct rb_simple_node simple_node; + }; + + /* + * This is a sanity check, whenever we COW a block we will update + * new_bytenr with it's current location, and we will check this in + * various places to validate that the cache makes sense, it shouldn't + * be used for anything else. + */ + u64 new_bytenr; + /* Objectid of tree block owner, can be not uptodate */ + u64 owner; + /* Link to pending, changed or detached list */ + struct list_head list; + + /* List of upper level edges, which link this node to its parents */ + struct list_head upper; + /* List of lower level edges, which link this node to its children */ + struct list_head lower; + + /* NULL if this node is not tree root */ + struct btrfs_root *root; + /* Extent buffer got by COWing the block */ + struct extent_buffer *eb; + /* Level of the tree block */ + unsigned int level:8; + /* Is the extent buffer locked */ + unsigned int locked:1; + /* Has the block been processed */ + unsigned int processed:1; + /* Have backrefs of this block been checked */ + unsigned int checked:1; + /* + * 1 if corresponding block has been COWed but some upper level block + * pointers may not point to the new location + */ + unsigned int pending:1; + /* 1 if the backref node isn't connected to any other backref node */ + unsigned int detached:1; + + /* + * For generic purpose backref cache, where we only care if it's a reloc + * root, doesn't care the source subvolid. + */ + unsigned int is_reloc_root:1; +}; + +#define LOWER 0 +#define UPPER 1 + +/* + * Represent an edge connecting upper and lower backref nodes. + */ +struct btrfs_backref_edge { + /* + * list[LOWER] is linked to btrfs_backref_node::upper of lower level + * node, and list[UPPER] is linked to btrfs_backref_node::lower of + * upper level node. + * + * Also, build_backref_tree() uses list[UPPER] for pending edges, before + * linking list[UPPER] to its upper level nodes. + */ + struct list_head list[2]; + + /* Two related nodes */ + struct btrfs_backref_node *node[2]; +}; + +struct btrfs_backref_cache { + /* Red black tree of all backref nodes in the cache */ + struct rb_root rb_root; + /* For passing backref nodes to btrfs_reloc_cow_block */ + struct btrfs_backref_node *path[BTRFS_MAX_LEVEL]; + /* + * List of blocks that have been COWed but some block pointers in upper + * level blocks may not reflect the new location + */ + struct list_head pending[BTRFS_MAX_LEVEL]; + + u64 last_trans; + + int nr_nodes; + int nr_edges; + + /* List of unchecked backref edges during backref cache build */ + struct list_head pending_edge; + + /* List of useless backref nodes during backref cache build */ + struct list_head useless_node; + + struct btrfs_fs_info *fs_info; + + /* + * Whether this cache is for relocation + * + * Relocation backref cache require more info for reloc root compared + * to generic backref cache. + */ + bool is_reloc; +}; + +void btrfs_backref_init_cache(struct btrfs_fs_info *fs_info, + struct btrfs_backref_cache *cache, bool is_reloc); +struct btrfs_backref_node *btrfs_backref_alloc_node( + struct btrfs_backref_cache *cache, u64 bytenr, int level); +struct btrfs_backref_edge *btrfs_backref_alloc_edge( + struct btrfs_backref_cache *cache); + +void btrfs_backref_free_node(struct btrfs_backref_cache *cache, + struct btrfs_backref_node *node); +void btrfs_backref_free_edge(struct btrfs_backref_cache *cache, + struct btrfs_backref_edge *edge); +void btrfs_backref_unlock_node_buffer(struct btrfs_backref_node *node); +void btrfs_backref_drop_node_buffer(struct btrfs_backref_node *node); + +void btrfs_backref_cleanup_node(struct btrfs_backref_cache *cache, + struct btrfs_backref_node *node); +void btrfs_backref_drop_node(struct btrfs_backref_cache *tree, + struct btrfs_backref_node *node); + +void btrfs_backref_release_cache(struct btrfs_backref_cache *cache); + +static inline void btrfs_backref_panic(struct btrfs_fs_info *fs_info, + u64 bytenr, int error) +{ + btrfs_panic(fs_info, error, + "Inconsistency in backref cache found at offset %llu", + bytenr); +} + +int btrfs_backref_add_tree_node(struct btrfs_trans_handle *trans, + struct btrfs_backref_cache *cache, + struct btrfs_path *path, + struct btrfs_backref_iter *iter, + struct btrfs_key *node_key, + struct btrfs_backref_node *cur); + +int btrfs_backref_finish_upper_links(struct btrfs_backref_cache *cache, + struct btrfs_backref_node *start); + +void btrfs_backref_error_cleanup(struct btrfs_backref_cache *cache, + struct btrfs_backref_node *node); + +#endif diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c new file mode 100644 index 000000000..cc0bd0304 --- /dev/null +++ b/fs/btrfs/bio.c @@ -0,0 +1,1072 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2007 Oracle. All rights reserved. + * Copyright (C) 2022 Christoph Hellwig. + */ + +#include +#include +#include "bio.h" +#include "ctree.h" +#include "volumes.h" +#include "raid56.h" +#include "async-thread.h" +#include "dev-replace.h" +#include "zoned.h" +#include "file-item.h" +#include "raid-stripe-tree.h" + +static struct bio_set btrfs_bioset; +static struct bio_set btrfs_clone_bioset; +static struct bio_set btrfs_repair_bioset; +static mempool_t btrfs_failed_bio_pool; + +struct btrfs_failed_bio { + struct btrfs_bio *bbio; + int num_copies; + atomic_t repair_count; +}; + +/* Is this a data path I/O that needs storage layer checksum and repair? */ +static inline bool is_data_bbio(const struct btrfs_bio *bbio) +{ + return bbio->inode && is_data_inode(bbio->inode); +} + +static bool bbio_has_ordered_extent(const struct btrfs_bio *bbio) +{ + return is_data_bbio(bbio) && btrfs_op(&bbio->bio) == BTRFS_MAP_WRITE; +} + +/* + * Initialize a btrfs_bio structure. This skips the embedded bio itself as it + * is already initialized by the block layer. + */ +void btrfs_bio_init(struct btrfs_bio *bbio, struct btrfs_inode *inode, u64 file_offset, + btrfs_bio_end_io_t end_io, void *private) +{ + /* @inode parameter is mandatory. */ + ASSERT(inode); + + memset(bbio, 0, offsetof(struct btrfs_bio, bio)); + bbio->inode = inode; + bbio->end_io = end_io; + bbio->private = private; + bbio->file_offset = file_offset; + atomic_set(&bbio->pending_ios, 1); + WRITE_ONCE(bbio->status, BLK_STS_OK); +} + +/* + * Allocate a btrfs_bio structure. The btrfs_bio is the main I/O container for + * btrfs, and is used for all I/O submitted through btrfs_submit_bbio(). + * + * Just like the underlying bio_alloc_bioset it will not fail as it is backed by + * a mempool. + */ +struct btrfs_bio *btrfs_bio_alloc(unsigned int nr_vecs, blk_opf_t opf, + struct btrfs_inode *inode, u64 file_offset, + btrfs_bio_end_io_t end_io, void *private) +{ + struct btrfs_bio *bbio; + struct bio *bio; + + bio = bio_alloc_bioset(NULL, nr_vecs, opf, GFP_NOFS, &btrfs_bioset); + bbio = btrfs_bio(bio); + btrfs_bio_init(bbio, inode, file_offset, end_io, private); + return bbio; +} + +static struct btrfs_bio *btrfs_split_bio(struct btrfs_fs_info *fs_info, + struct btrfs_bio *orig_bbio, + u64 map_length) +{ + struct btrfs_bio *bbio; + struct bio *bio; + + bio = bio_split(&orig_bbio->bio, map_length >> SECTOR_SHIFT, GFP_NOFS, + &btrfs_clone_bioset); + if (IS_ERR(bio)) + return ERR_CAST(bio); + + bbio = btrfs_bio(bio); + btrfs_bio_init(bbio, orig_bbio->inode, orig_bbio->file_offset, NULL, orig_bbio); + orig_bbio->file_offset += map_length; + if (bbio_has_ordered_extent(bbio)) { + refcount_inc(&orig_bbio->ordered->refs); + bbio->ordered = orig_bbio->ordered; + bbio->orig_logical = orig_bbio->orig_logical; + orig_bbio->orig_logical += map_length; + } + + bbio->csum_search_commit_root = orig_bbio->csum_search_commit_root; + bbio->can_use_append = orig_bbio->can_use_append; + bbio->is_scrub = orig_bbio->is_scrub; + bbio->is_remap = orig_bbio->is_remap; + bbio->async_csum = orig_bbio->async_csum; + + atomic_inc(&orig_bbio->pending_ios); + return bbio; +} + +void btrfs_bio_end_io(struct btrfs_bio *bbio, blk_status_t status) +{ + /* Make sure we're already in task context. */ + ASSERT(in_task()); + + if (bbio->async_csum) + wait_for_completion(&bbio->csum_done); + + bbio->bio.bi_status = status; + if (bbio->bio.bi_pool == &btrfs_clone_bioset) { + struct btrfs_bio *orig_bbio = bbio->private; + + /* Free bio that was never submitted to the underlying device. */ + if (bbio_has_ordered_extent(bbio)) + btrfs_put_ordered_extent(bbio->ordered); + bio_put(&bbio->bio); + + bbio = orig_bbio; + } + + /* + * At this point, bbio always points to the original btrfs_bio. Save + * the first error in it. + */ + if (status != BLK_STS_OK) + cmpxchg(&bbio->status, BLK_STS_OK, status); + + if (atomic_dec_and_test(&bbio->pending_ios)) { + /* Load split bio's error which might be set above. */ + if (status == BLK_STS_OK) + bbio->bio.bi_status = READ_ONCE(bbio->status); + + if (bbio_has_ordered_extent(bbio)) { + struct btrfs_ordered_extent *ordered = bbio->ordered; + + bbio->end_io(bbio); + btrfs_put_ordered_extent(ordered); + } else { + bbio->end_io(bbio); + } + } +} + +static int next_repair_mirror(const struct btrfs_failed_bio *fbio, int cur_mirror) +{ + if (cur_mirror == fbio->num_copies) + return cur_mirror + 1 - fbio->num_copies; + return cur_mirror + 1; +} + +static int prev_repair_mirror(const struct btrfs_failed_bio *fbio, int cur_mirror) +{ + if (cur_mirror == 1) + return fbio->num_copies; + return cur_mirror - 1; +} + +static void btrfs_repair_done(struct btrfs_failed_bio *fbio) +{ + if (atomic_dec_and_test(&fbio->repair_count)) { + btrfs_bio_end_io(fbio->bbio, fbio->bbio->bio.bi_status); + mempool_free(fbio, &btrfs_failed_bio_pool); + } +} + +static void btrfs_end_repair_bio(struct btrfs_bio *repair_bbio, + struct btrfs_device *dev) +{ + struct btrfs_failed_bio *fbio = repair_bbio->private; + struct btrfs_inode *inode = repair_bbio->inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + /* + * We can not move forward the saved_iter, as it will be later + * utilized by repair_bbio again. + */ + struct bvec_iter saved_iter = repair_bbio->saved_iter; + const u32 step = min(fs_info->sectorsize, PAGE_SIZE); + const u64 logical = repair_bbio->saved_iter.bi_sector << SECTOR_SHIFT; + const u32 nr_steps = repair_bbio->saved_iter.bi_size / step; + int mirror = repair_bbio->mirror_num; + phys_addr_t paddrs[BTRFS_MAX_BLOCKSIZE / PAGE_SIZE]; + phys_addr_t paddr; + unsigned int slot = 0; + + /* Repair bbio should be eaxctly one block sized. */ + ASSERT(repair_bbio->saved_iter.bi_size == fs_info->sectorsize); + + btrfs_bio_for_each_block(paddr, &repair_bbio->bio, &saved_iter, step) { + ASSERT(slot < nr_steps); + paddrs[slot] = paddr; + slot++; + } + + if (repair_bbio->bio.bi_status || + !btrfs_data_csum_ok(repair_bbio, dev, 0, paddrs)) { + bio_reset(&repair_bbio->bio, NULL, REQ_OP_READ); + repair_bbio->bio.bi_iter = repair_bbio->saved_iter; + + mirror = next_repair_mirror(fbio, mirror); + if (mirror == fbio->bbio->mirror_num) { + btrfs_debug(fs_info, "no mirror left"); + fbio->bbio->bio.bi_status = BLK_STS_IOERR; + goto done; + } + + btrfs_submit_bbio(repair_bbio, mirror); + return; + } + + do { + mirror = prev_repair_mirror(fbio, mirror); + btrfs_repair_io_failure(fs_info, btrfs_ino(inode), + repair_bbio->file_offset, fs_info->sectorsize, + logical, paddrs, step, mirror); + } while (mirror != fbio->bbio->mirror_num); + +done: + btrfs_repair_done(fbio); + bio_put(&repair_bbio->bio); +} + +/* + * Try to kick off a repair read to the next available mirror for a bad sector. + * + * This primarily tries to recover good data to serve the actual read request, + * but also tries to write the good data back to the bad mirror(s) when a + * read succeeded to restore the redundancy. + */ +static struct btrfs_failed_bio *repair_one_sector(struct btrfs_bio *failed_bbio, + u32 bio_offset, + phys_addr_t paddrs[], + struct btrfs_failed_bio *fbio) +{ + struct btrfs_inode *inode = failed_bbio->inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + const u32 sectorsize = fs_info->sectorsize; + const u32 step = min(fs_info->sectorsize, PAGE_SIZE); + const u32 nr_steps = sectorsize / step; + /* + * For bs > ps cases, the saved_iter can be partially moved forward. + * In that case we should round it down to the block boundary. + */ + const u64 logical = round_down(failed_bbio->saved_iter.bi_sector << SECTOR_SHIFT, + sectorsize); + struct btrfs_bio *repair_bbio; + struct bio *repair_bio; + int num_copies; + int mirror; + + btrfs_debug(fs_info, "repair read error: read error at %llu", + failed_bbio->file_offset + bio_offset); + + num_copies = btrfs_num_copies(fs_info, logical, sectorsize); + if (num_copies == 1) { + btrfs_debug(fs_info, "no copy to repair from"); + failed_bbio->bio.bi_status = BLK_STS_IOERR; + return fbio; + } + + if (!fbio) { + fbio = mempool_alloc(&btrfs_failed_bio_pool, GFP_NOFS); + fbio->bbio = failed_bbio; + fbio->num_copies = num_copies; + atomic_set(&fbio->repair_count, 1); + } + + atomic_inc(&fbio->repair_count); + + repair_bio = bio_alloc_bioset(NULL, nr_steps, REQ_OP_READ, GFP_NOFS, + &btrfs_repair_bioset); + repair_bio->bi_iter.bi_sector = logical >> SECTOR_SHIFT; + for (int i = 0; i < nr_steps; i++) { + int ret; + + ASSERT(offset_in_page(paddrs[i]) + step <= PAGE_SIZE); + + ret = bio_add_page(repair_bio, phys_to_page(paddrs[i]), step, + offset_in_page(paddrs[i])); + ASSERT(ret == step); + } + + repair_bbio = btrfs_bio(repair_bio); + btrfs_bio_init(repair_bbio, failed_bbio->inode, failed_bbio->file_offset + bio_offset, + NULL, fbio); + + mirror = next_repair_mirror(fbio, failed_bbio->mirror_num); + btrfs_debug(fs_info, "submitting repair read to mirror %d", mirror); + btrfs_submit_bbio(repair_bbio, mirror); + return fbio; +} + +static void btrfs_check_read_bio(struct btrfs_bio *bbio, struct btrfs_device *dev) +{ + struct btrfs_inode *inode = bbio->inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + const u32 sectorsize = fs_info->sectorsize; + const u32 step = min(sectorsize, PAGE_SIZE); + const u32 nr_steps = sectorsize / step; + struct bvec_iter *iter = &bbio->saved_iter; + blk_status_t status = bbio->bio.bi_status; + struct btrfs_failed_bio *fbio = NULL; + phys_addr_t paddrs[BTRFS_MAX_BLOCKSIZE / PAGE_SIZE]; + phys_addr_t paddr; + u32 offset = 0; + + /* Read-repair requires the inode field to be set by the submitter. */ + ASSERT(inode); + + /* + * Hand off repair bios to the repair code as there is no upper level + * submitter for them. + */ + if (bbio->bio.bi_pool == &btrfs_repair_bioset) { + btrfs_end_repair_bio(bbio, dev); + return; + } + + /* Clear the I/O error. A failed repair will reset it. */ + bbio->bio.bi_status = BLK_STS_OK; + + btrfs_bio_for_each_block(paddr, &bbio->bio, iter, step) { + paddrs[(offset / step) % nr_steps] = paddr; + offset += step; + + if (IS_ALIGNED(offset, sectorsize)) { + if (status || + !btrfs_data_csum_ok(bbio, dev, offset - sectorsize, paddrs)) + fbio = repair_one_sector(bbio, offset - sectorsize, + paddrs, fbio); + } + } + if (bbio->csum != bbio->csum_inline) + kvfree(bbio->csum); + + if (fbio) + btrfs_repair_done(fbio); + else + btrfs_bio_end_io(bbio, bbio->bio.bi_status); +} + +static void btrfs_log_dev_io_error(const struct bio *bio, struct btrfs_device *dev) +{ + blk_status_t sts = bio->bi_status; + + if (!dev || !dev->bdev) + return; + if (unlikely(sts == BLK_STS_OK)) + return; + if (unlikely(sts != BLK_STS_IOERR && sts != BLK_STS_TARGET && + sts != BLK_STS_MEDIUM && sts != BLK_STS_PROTECTION)) { + btrfs_warn_rl(dev->fs_info, "bdev %s unexpected block io error: %d", + btrfs_dev_name(dev), sts); + return; + } + if (btrfs_op(bio) == BTRFS_MAP_WRITE) + btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS); + else if (!(bio->bi_opf & REQ_RAHEAD)) + btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS); + if (bio->bi_opf & REQ_PREFLUSH) + btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_FLUSH_ERRS); +} + +static struct workqueue_struct *btrfs_end_io_wq(const struct btrfs_fs_info *fs_info, + const struct bio *bio) +{ + if (bio->bi_opf & REQ_META) + return fs_info->endio_meta_workers; + return fs_info->endio_workers; +} + +static void simple_end_io_work(struct work_struct *work) +{ + struct btrfs_bio *bbio = container_of(work, struct btrfs_bio, end_io_work); + struct bio *bio = &bbio->bio; + + if (bio_op(bio) == REQ_OP_READ) { + /* Metadata reads are checked and repaired by the submitter. */ + if (is_data_bbio(bbio)) + return btrfs_check_read_bio(bbio, bbio->bio.bi_private); + return btrfs_bio_end_io(bbio, bbio->bio.bi_status); + } + if (bio_is_zone_append(bio) && !bio->bi_status) + btrfs_record_physical_zoned(bbio); + btrfs_bio_end_io(bbio, bbio->bio.bi_status); +} + +static void btrfs_simple_end_io(struct bio *bio) +{ + struct btrfs_bio *bbio = btrfs_bio(bio); + struct btrfs_device *dev = bio->bi_private; + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; + + btrfs_bio_counter_dec(fs_info); + + if (bio->bi_status) + btrfs_log_dev_io_error(bio, dev); + + INIT_WORK(&bbio->end_io_work, simple_end_io_work); + queue_work(btrfs_end_io_wq(fs_info, bio), &bbio->end_io_work); +} + +static void btrfs_raid56_end_io(struct bio *bio) +{ + struct btrfs_io_context *bioc = bio->bi_private; + struct btrfs_bio *bbio = btrfs_bio(bio); + + /* RAID56 endio is always handled in workqueue. */ + ASSERT(in_task()); + + btrfs_bio_counter_dec(bioc->fs_info); + bbio->mirror_num = bioc->mirror_num; + if (bio_op(bio) == REQ_OP_READ && is_data_bbio(bbio)) + btrfs_check_read_bio(bbio, NULL); + else + btrfs_bio_end_io(bbio, bbio->bio.bi_status); + + btrfs_put_bioc(bioc); +} + +static void orig_write_end_io_work(struct work_struct *work) +{ + struct btrfs_bio *bbio = container_of(work, struct btrfs_bio, end_io_work); + struct bio *bio = &bbio->bio; + struct btrfs_io_stripe *stripe = bio->bi_private; + struct btrfs_io_context *bioc = stripe->bioc; + + btrfs_bio_counter_dec(bioc->fs_info); + + if (bio->bi_status) { + atomic_inc(&bioc->error); + btrfs_log_dev_io_error(bio, stripe->dev); + } + + /* + * Only send an error to the higher layers if it is beyond the tolerance + * threshold. + */ + if (atomic_read(&bioc->error) > bioc->max_errors) + bio->bi_status = BLK_STS_IOERR; + else + bio->bi_status = BLK_STS_OK; + + if (bio_is_zone_append(bio) && !bio->bi_status) + stripe->physical = bio->bi_iter.bi_sector << SECTOR_SHIFT; + + btrfs_bio_end_io(bbio, bbio->bio.bi_status); + btrfs_put_bioc(bioc); +} + +static void btrfs_orig_write_end_io(struct bio *bio) +{ + struct btrfs_bio *bbio = btrfs_bio(bio); + + INIT_WORK(&bbio->end_io_work, orig_write_end_io_work); + queue_work(btrfs_end_io_wq(bbio->inode->root->fs_info, bio), &bbio->end_io_work); +} + +static void clone_write_end_io_work(struct work_struct *work) +{ + struct btrfs_bio *bbio = container_of(work, struct btrfs_bio, end_io_work); + struct bio *bio = &bbio->bio; + struct btrfs_io_stripe *stripe = bio->bi_private; + + if (bio->bi_status) { + atomic_inc(&stripe->bioc->error); + btrfs_log_dev_io_error(bio, stripe->dev); + } else if (bio_is_zone_append(bio)) { + stripe->physical = bio->bi_iter.bi_sector << SECTOR_SHIFT; + } + + /* Pass on control to the original bio this one was cloned from */ + bio_endio(stripe->bioc->orig_bio); + bio_put(bio); +} + +static void btrfs_clone_write_end_io(struct bio *bio) +{ + struct btrfs_bio *bbio = btrfs_bio(bio); + + INIT_WORK(&bbio->end_io_work, clone_write_end_io_work); + queue_work(btrfs_end_io_wq(bbio->inode->root->fs_info, bio), &bbio->end_io_work); +} + +static void btrfs_submit_dev_bio(struct btrfs_device *dev, struct bio *bio) +{ + u64 physical = bio->bi_iter.bi_sector << SECTOR_SHIFT; + + if (!dev || !dev->bdev || + test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) || + (btrfs_op(bio) == BTRFS_MAP_WRITE && + !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))) { + bio_io_error(bio); + return; + } + + bio_set_dev(bio, dev->bdev); + + /* + * For zone append writing, bi_sector must point the beginning of the + * zone + */ + if (btrfs_bio(bio)->can_use_append && btrfs_dev_is_sequential(dev, physical)) { + u64 zone_start = round_down(physical, dev->fs_info->zone_size); + + ASSERT(btrfs_dev_is_sequential(dev, physical)); + bio->bi_iter.bi_sector = zone_start >> SECTOR_SHIFT; + bio->bi_opf &= ~REQ_OP_WRITE; + bio->bi_opf |= REQ_OP_ZONE_APPEND; + } + btrfs_debug(dev->fs_info, + "%s: rw %d 0x%x, sector=%llu, dev=%lu (%s id %llu), size=%u", + __func__, bio_op(bio), bio->bi_opf, bio->bi_iter.bi_sector, + (unsigned long)dev->bdev->bd_dev, btrfs_dev_name(dev), + dev->devid, bio->bi_iter.bi_size); + + /* + * Track reads if tracking is enabled; ignore I/O operations before the + * filesystem is fully initialized. + */ + if (dev->fs_devices->collect_fs_stats && bio_op(bio) == REQ_OP_READ && dev->fs_info) + percpu_counter_add(&dev->fs_info->stats_read_blocks, + bio->bi_iter.bi_size >> dev->fs_info->sectorsize_bits); + + if (bio->bi_opf & REQ_BTRFS_CGROUP_PUNT) + blkcg_punt_bio_submit(bio); + else + submit_bio(bio); +} + +static void btrfs_submit_mirrored_bio(struct btrfs_io_context *bioc, int dev_nr) +{ + struct bio *orig_bio = bioc->orig_bio, *bio; + struct btrfs_bio *orig_bbio = btrfs_bio(orig_bio); + + ASSERT(bio_op(orig_bio) != REQ_OP_READ); + + /* Reuse the bio embedded into the btrfs_bio for the last mirror */ + if (dev_nr == bioc->num_stripes - 1) { + bio = orig_bio; + bio->bi_end_io = btrfs_orig_write_end_io; + } else { + /* We need to use endio_work to run end_io in task context. */ + bio = bio_alloc_clone(NULL, orig_bio, GFP_NOFS, &btrfs_bioset); + bio_inc_remaining(orig_bio); + btrfs_bio_init(btrfs_bio(bio), orig_bbio->inode, + orig_bbio->file_offset, NULL, NULL); + bio->bi_end_io = btrfs_clone_write_end_io; + } + + bio->bi_private = &bioc->stripes[dev_nr]; + bio->bi_iter.bi_sector = bioc->stripes[dev_nr].physical >> SECTOR_SHIFT; + bioc->stripes[dev_nr].bioc = bioc; + bioc->size = bio->bi_iter.bi_size; + btrfs_submit_dev_bio(bioc->stripes[dev_nr].dev, bio); +} + +static void btrfs_submit_bio(struct bio *bio, struct btrfs_io_context *bioc, + struct btrfs_io_stripe *smap, int mirror_num) +{ + if (!bioc) { + /* Single mirror read/write fast path. */ + btrfs_bio(bio)->mirror_num = mirror_num; + bio->bi_iter.bi_sector = smap->physical >> SECTOR_SHIFT; + if (bio_op(bio) != REQ_OP_READ) + btrfs_bio(bio)->orig_physical = smap->physical; + bio->bi_private = smap->dev; + bio->bi_end_io = btrfs_simple_end_io; + btrfs_submit_dev_bio(smap->dev, bio); + } else if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) { + /* Parity RAID write or read recovery. */ + bio->bi_private = bioc; + bio->bi_end_io = btrfs_raid56_end_io; + if (bio_op(bio) == REQ_OP_READ) + raid56_parity_recover(bio, bioc, mirror_num); + else + raid56_parity_write(bio, bioc); + } else { + /* Write to multiple mirrors. */ + int total_devs = bioc->num_stripes; + + bioc->orig_bio = bio; + for (int dev_nr = 0; dev_nr < total_devs; dev_nr++) + btrfs_submit_mirrored_bio(bioc, dev_nr); + } +} + +static int btrfs_bio_csum(struct btrfs_bio *bbio) +{ + if (bbio->bio.bi_opf & REQ_META) + return btree_csum_one_bio(bbio); +#ifdef CONFIG_BTRFS_EXPERIMENTAL + return btrfs_csum_one_bio(bbio, true); +#else + return btrfs_csum_one_bio(bbio, false); +#endif +} + +/* + * Async submit bios are used to offload expensive checksumming onto the worker + * threads. + */ +struct async_submit_bio { + struct btrfs_bio *bbio; + struct btrfs_io_context *bioc; + struct btrfs_io_stripe smap; + int mirror_num; + struct btrfs_work work; +}; + +/* + * In order to insert checksums into the metadata in large chunks, we wait + * until bio submission time. All the pages in the bio are checksummed and + * sums are attached onto the ordered extent record. + * + * At IO completion time the csums attached on the ordered extent record are + * inserted into the btree. + */ +static void run_one_async_start(struct btrfs_work *work) +{ + struct async_submit_bio *async = + container_of(work, struct async_submit_bio, work); + int ret; + + ret = btrfs_bio_csum(async->bbio); + if (ret) + async->bbio->bio.bi_status = errno_to_blk_status(ret); +} + +/* + * In order to insert checksums into the metadata in large chunks, we wait + * until bio submission time. All the pages in the bio are checksummed and + * sums are attached onto the ordered extent record. + * + * At IO completion time the csums attached on the ordered extent record are + * inserted into the tree. + * + * If called with @do_free == true, then it will free the work struct. + */ +static void run_one_async_done(struct btrfs_work *work, bool do_free) +{ + struct async_submit_bio *async = + container_of(work, struct async_submit_bio, work); + struct bio *bio = &async->bbio->bio; + + if (do_free) { + kfree(container_of(work, struct async_submit_bio, work)); + return; + } + + /* If an error occurred we just want to clean up the bio and move on. */ + if (bio->bi_status) { + btrfs_bio_end_io(async->bbio, bio->bi_status); + return; + } + + /* + * All of the bios that pass through here are from async helpers. + * Use REQ_BTRFS_CGROUP_PUNT to issue them from the owning cgroup's + * context. This changes nothing when cgroups aren't in use. + */ + bio->bi_opf |= REQ_BTRFS_CGROUP_PUNT; + btrfs_submit_bio(bio, async->bioc, &async->smap, async->mirror_num); +} + +static bool should_async_write(struct btrfs_bio *bbio) +{ + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; + bool auto_csum_mode = true; + +#ifdef CONFIG_BTRFS_EXPERIMENTAL + /* + * Write bios will calculate checksum and submit bio at the same time. + * Unless explicitly required don't offload serial csum calculate and bio + * submit into a workqueue. + */ + return false; +#endif + + /* Submit synchronously if the checksum implementation is fast. */ + if (auto_csum_mode && test_bit(BTRFS_FS_CSUM_IMPL_FAST, &fs_info->flags)) + return false; + + /* + * Try to defer the submission to a workqueue to parallelize the + * checksum calculation unless the I/O is issued synchronously. + */ + if (op_is_sync(bbio->bio.bi_opf)) + return false; + + /* Zoned devices require I/O to be submitted in order. */ + if ((bbio->bio.bi_opf & REQ_META) && btrfs_is_zoned(fs_info)) + return false; + + return true; +} + +/* + * Submit bio to an async queue. + * + * Return true if the work has been successfully submitted, else false. + */ +static bool btrfs_wq_submit_bio(struct btrfs_bio *bbio, + struct btrfs_io_context *bioc, + struct btrfs_io_stripe *smap, int mirror_num) +{ + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; + struct async_submit_bio *async; + + async = kmalloc_obj(*async, GFP_NOFS); + if (!async) + return false; + + async->bbio = bbio; + async->bioc = bioc; + async->smap = *smap; + async->mirror_num = mirror_num; + + btrfs_init_work(&async->work, run_one_async_start, run_one_async_done); + btrfs_queue_work(fs_info->workers, &async->work); + return true; +} + +static u64 btrfs_append_map_length(struct btrfs_bio *bbio, u64 map_length) +{ + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; + unsigned int nr_segs; + int sector_offset; + + map_length = min(map_length, fs_info->max_zone_append_size); + sector_offset = bio_split_rw_at(&bbio->bio, &fs_info->limits, + &nr_segs, map_length); + if (sector_offset) { + /* + * bio_split_rw_at() could split at a size smaller than our + * sectorsize and thus cause unaligned I/Os. Fix that by + * always rounding down to the nearest boundary. + */ + return ALIGN_DOWN(sector_offset << SECTOR_SHIFT, fs_info->sectorsize); + } + return map_length; +} + +static bool btrfs_submit_chunk(struct btrfs_bio *bbio, int mirror_num) +{ + struct btrfs_inode *inode = bbio->inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct bio *bio = &bbio->bio; + u64 logical = bio->bi_iter.bi_sector << SECTOR_SHIFT; + u64 length = bio->bi_iter.bi_size; + u64 map_length = length; + struct btrfs_io_context *bioc = NULL; + struct btrfs_io_stripe smap; + blk_status_t status; + int ret; + + if (bbio->is_scrub || btrfs_is_data_reloc_root(inode->root)) + smap.rst_search_commit_root = true; + else + smap.rst_search_commit_root = false; + + btrfs_bio_counter_inc_blocked(fs_info); + ret = btrfs_map_block(fs_info, btrfs_op(bio), logical, &map_length, + &bioc, &smap, &mirror_num); + if (ret) { + status = errno_to_blk_status(ret); + btrfs_bio_counter_dec(fs_info); + goto end_bbio; + } + + /* + * For fscrypt writes we will get the encrypted bio after we've remapped + * our bio to the physical disk location, so we need to save the + * original bytenr so we know what we're checksumming. + */ + if (bio_op(bio) == REQ_OP_WRITE && is_data_bbio(bbio)) + bbio->orig_logical = logical; + + bbio->can_use_append = btrfs_use_zone_append(bbio); + + map_length = min(map_length, length); + if (bbio->can_use_append) + map_length = btrfs_append_map_length(bbio, map_length); + + if (map_length < length) { + struct btrfs_bio *split; + + split = btrfs_split_bio(fs_info, bbio, map_length); + if (IS_ERR(split)) { + status = errno_to_blk_status(PTR_ERR(split)); + btrfs_bio_counter_dec(fs_info); + goto end_bbio; + } + bbio = split; + bio = &bbio->bio; + } + + /* + * Save the iter for the end_io handler and preload the checksums for + * data reads. + */ + if (bio_op(bio) == REQ_OP_READ && is_data_bbio(bbio)) { + bbio->saved_iter = bio->bi_iter; + ret = btrfs_lookup_bio_sums(bbio); + status = errno_to_blk_status(ret); + if (status) + goto fail; + } + + if (btrfs_op(bio) == BTRFS_MAP_WRITE) { + if (is_data_bbio(bbio) && bioc && bioc->use_rst) { + /* + * No locking for the list update, as we only add to + * the list in the I/O submission path, and list + * iteration only happens in the completion path, which + * can't happen until after the last submission. + */ + btrfs_get_bioc(bioc); + list_add_tail(&bioc->rst_ordered_entry, &bbio->ordered->bioc_list); + } + + /* + * Csum items for reloc roots have already been cloned at this + * point, so they are handled as part of the no-checksum case. + */ + if (!(inode->flags & BTRFS_INODE_NODATASUM) && + !test_bit(BTRFS_FS_STATE_NO_DATA_CSUMS, &fs_info->fs_state) && + !btrfs_is_data_reloc_root(inode->root) && !bbio->is_remap) { + if (should_async_write(bbio) && + btrfs_wq_submit_bio(bbio, bioc, &smap, mirror_num)) + goto done; + + ret = btrfs_bio_csum(bbio); + status = errno_to_blk_status(ret); + if (status) + goto fail; + } else if (bbio->can_use_append || + (btrfs_is_zoned(fs_info) && inode->flags & BTRFS_INODE_NODATASUM)) { + ret = btrfs_alloc_dummy_sum(bbio); + status = errno_to_blk_status(ret); + if (status) + goto fail; + } + } + + btrfs_submit_bio(bio, bioc, &smap, mirror_num); +done: + return map_length == length; + +fail: + btrfs_bio_counter_dec(fs_info); + /* + * We have split the original bbio, now we have to end both the current + * @bbio and remaining one, as the remaining one will never be submitted. + */ + if (map_length < length) { + struct btrfs_bio *remaining = bbio->private; + + ASSERT(bbio->bio.bi_pool == &btrfs_clone_bioset); + ASSERT(remaining); + + btrfs_bio_end_io(remaining, status); + } +end_bbio: + btrfs_bio_end_io(bbio, status); + /* Do not submit another chunk */ + return true; +} + +static void assert_bbio_alignment(struct btrfs_bio *bbio) +{ +#ifdef CONFIG_BTRFS_ASSERT + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; + struct bio_vec bvec; + struct bvec_iter iter; + const u32 blocksize = fs_info->sectorsize; + const u32 alignment = min(blocksize, PAGE_SIZE); + const u64 logical = bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT; + const u32 length = bbio->bio.bi_iter.bi_size; + + /* The logical and length should still be aligned to blocksize. */ + ASSERT(IS_ALIGNED(logical, blocksize) && IS_ALIGNED(length, blocksize) && + length != 0, "root=%llu inode=%llu logical=%llu length=%u", + btrfs_root_id(bbio->inode->root), + btrfs_ino(bbio->inode), logical, length); + + bio_for_each_bvec(bvec, &bbio->bio, iter) + ASSERT(IS_ALIGNED(bvec.bv_offset, alignment) && + IS_ALIGNED(bvec.bv_len, alignment), + "root=%llu inode=%llu logical=%llu length=%u index=%u bv_offset=%u bv_len=%u", + btrfs_root_id(bbio->inode->root), + btrfs_ino(bbio->inode), logical, length, iter.bi_idx, + bvec.bv_offset, bvec.bv_len); +#endif +} + +void btrfs_submit_bbio(struct btrfs_bio *bbio, int mirror_num) +{ + /* If bbio->inode is not populated, its file_offset must be 0. */ + ASSERT(bbio->inode || bbio->file_offset == 0); + + assert_bbio_alignment(bbio); + + while (!btrfs_submit_chunk(bbio, mirror_num)) + ; +} + +/* + * Submit a repair write. + * + * This bypasses btrfs_submit_bbio() deliberately, as that writes all copies in a + * RAID setup. Here we only want to write the one bad copy, so we do the + * mapping ourselves and submit the bio directly. + * + * The I/O is issued synchronously to block the repair read completion from + * freeing the bio. + * + * @ino: Offending inode number + * @fileoff: File offset inside the inode + * @length: Length of the repair write + * @logical: Logical address of the range + * @paddrs: Physical address array of the content + * @step: Length of for each paddrs + * @mirror_num: Mirror number to write to. Must not be zero + */ +int btrfs_repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 fileoff, + u32 length, u64 logical, const phys_addr_t paddrs[], + unsigned int step, int mirror_num) +{ + const u32 nr_steps = DIV_ROUND_UP_POW2(length, step); + struct btrfs_io_stripe smap = { 0 }; + struct bio *bio = NULL; + int ret = 0; + + BUG_ON(!mirror_num); + + /* Basic alignment checks. */ + ASSERT(IS_ALIGNED(logical, fs_info->sectorsize)); + ASSERT(IS_ALIGNED(length, fs_info->sectorsize)); + ASSERT(IS_ALIGNED(fileoff, fs_info->sectorsize)); + /* Either it's a single data or metadata block. */ + ASSERT(length <= BTRFS_MAX_BLOCKSIZE); + ASSERT(step <= length); + ASSERT(is_power_of_2(step)); + + /* + * The fs either mounted RO or hit critical errors, no need + * to continue repairing. + */ + if (unlikely(sb_rdonly(fs_info->sb))) + return 0; + + if (btrfs_repair_one_zone(fs_info, logical)) + return 0; + + /* + * Avoid races with device replace and make sure our bioc has devices + * associated to its stripes that don't go away while we are doing the + * read repair operation. + */ + btrfs_bio_counter_inc_blocked(fs_info); + ret = btrfs_map_repair_block(fs_info, &smap, logical, length, mirror_num); + if (ret < 0) + goto out_counter_dec; + + if (unlikely(!smap.dev->bdev || + !test_bit(BTRFS_DEV_STATE_WRITEABLE, &smap.dev->dev_state))) { + ret = -EIO; + goto out_counter_dec; + } + + bio = bio_alloc(smap.dev->bdev, nr_steps, REQ_OP_WRITE | REQ_SYNC, GFP_NOFS); + bio->bi_iter.bi_sector = smap.physical >> SECTOR_SHIFT; + for (int i = 0; i < nr_steps; i++) { + ret = bio_add_page(bio, phys_to_page(paddrs[i]), step, offset_in_page(paddrs[i])); + /* We should have allocated enough slots to contain all the different pages. */ + ASSERT(ret == step); + } + ret = submit_bio_wait(bio); + bio_put(bio); + if (ret) { + /* try to remap that extent elsewhere? */ + btrfs_dev_stat_inc_and_print(smap.dev, BTRFS_DEV_STAT_WRITE_ERRS); + goto out_counter_dec; + } + + btrfs_info_rl(fs_info, + "read error corrected: ino %llu off %llu (dev %s sector %llu)", + ino, fileoff, btrfs_dev_name(smap.dev), + smap.physical >> SECTOR_SHIFT); + ret = 0; + +out_counter_dec: + btrfs_bio_counter_dec(fs_info); + return ret; +} + +/* + * Submit a btrfs_bio based repair write. + * + * If @dev_replace is true, the write would be submitted to dev-replace target. + */ +void btrfs_submit_repair_write(struct btrfs_bio *bbio, int mirror_num, bool dev_replace) +{ + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; + u64 logical = bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT; + u64 length = bbio->bio.bi_iter.bi_size; + struct btrfs_io_stripe smap = { 0 }; + int ret; + + ASSERT(mirror_num > 0); + ASSERT(btrfs_op(&bbio->bio) == BTRFS_MAP_WRITE); + ASSERT(!is_data_inode(bbio->inode)); + ASSERT(bbio->is_scrub); + + btrfs_bio_counter_inc_blocked(fs_info); + ret = btrfs_map_repair_block(fs_info, &smap, logical, length, mirror_num); + if (ret < 0) + goto fail; + + if (dev_replace) { + ASSERT(smap.dev == fs_info->dev_replace.srcdev); + smap.dev = fs_info->dev_replace.tgtdev; + } + btrfs_submit_bio(&bbio->bio, NULL, &smap, mirror_num); + return; + +fail: + btrfs_bio_counter_dec(fs_info); + btrfs_bio_end_io(bbio, errno_to_blk_status(ret)); +} + +int __init btrfs_bioset_init(void) +{ + if (bioset_init(&btrfs_bioset, BIO_POOL_SIZE, + offsetof(struct btrfs_bio, bio), + BIOSET_NEED_BVECS)) + return -ENOMEM; + if (bioset_init(&btrfs_clone_bioset, BIO_POOL_SIZE, + offsetof(struct btrfs_bio, bio), 0)) + goto out; + if (bioset_init(&btrfs_repair_bioset, BIO_POOL_SIZE, + offsetof(struct btrfs_bio, bio), + BIOSET_NEED_BVECS)) + goto out; + if (mempool_init_kmalloc_pool(&btrfs_failed_bio_pool, BIO_POOL_SIZE, + sizeof(struct btrfs_failed_bio))) + goto out; + return 0; + +out: + btrfs_bioset_exit(); + return -ENOMEM; +} + +void __cold btrfs_bioset_exit(void) +{ + mempool_exit(&btrfs_failed_bio_pool); + bioset_exit(&btrfs_repair_bioset); + bioset_exit(&btrfs_clone_bioset); + bioset_exit(&btrfs_bioset); +} diff --git a/fs/btrfs/bio.h b/fs/btrfs/bio.h new file mode 100644 index 000000000..303ed6c71 --- /dev/null +++ b/fs/btrfs/bio.h @@ -0,0 +1,133 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2007 Oracle. All rights reserved. + * Copyright (C) 2022 Christoph Hellwig. + */ + +#ifndef BTRFS_BIO_H +#define BTRFS_BIO_H + +#include +#include +#include +#include "tree-checker.h" + +struct btrfs_bio; +struct btrfs_fs_info; +struct btrfs_inode; + +#define BTRFS_BIO_INLINE_CSUM_SIZE 64 + +typedef void (*btrfs_bio_end_io_t)(struct btrfs_bio *bbio); + +/* + * Highlevel btrfs I/O structure. It is allocated by btrfs_bio_alloc and + * passed to btrfs_submit_bbio() for mapping to the physical devices. + */ +struct btrfs_bio { + /* + * Inode and offset into it that this I/O operates on. + * + * If the inode is a data one, csum verification and read-repair + * will be done automatically. + * If the inode is a metadata one, everything is handled by the caller. + */ + struct btrfs_inode *inode; + u64 file_offset; + + union { + /* + * For data reads: checksumming and original I/O information. + * (for internal use in the btrfs_submit_bbio() machinery only) + */ + struct { + u8 *csum; + u8 csum_inline[BTRFS_BIO_INLINE_CSUM_SIZE]; + struct bvec_iter saved_iter; + }; + + /* + * For data writes: + * - ordered extent covering the bio + * - pointer to the checksums for this bio + * - original physical address from the allocator + * (for zone append only) + * - original logical address, used for checksumming fscrypt bios + */ + struct { + struct btrfs_ordered_extent *ordered; + struct btrfs_ordered_sum *sums; + struct work_struct csum_work; + struct completion csum_done; + struct bvec_iter csum_saved_iter; + u64 orig_physical; + u64 orig_logical; + }; + + /* For metadata reads: parentness verification. */ + struct btrfs_tree_parent_check parent_check; + }; + + /* For internal use in read end I/O handling */ + struct work_struct end_io_work; + + /* End I/O information supplied to btrfs_bio_alloc */ + btrfs_bio_end_io_t end_io; + void *private; + + atomic_t pending_ios; + u16 mirror_num; + + /* Save the first error status of split bio. */ + blk_status_t status; + + /* Use the commit root to look up csums (data read bio only). */ + bool csum_search_commit_root:1; + + /* + * Since scrub will reuse btree inode, we need this flag to distinguish + * scrub bios. + */ + bool is_scrub:1; + + /* Whether the bio is coming from copy_remapped_data_io(). */ + bool is_remap:1; + + /* Whether the csum generation for data write is async. */ + bool async_csum:1; + + /* Whether the bio is written using zone append. */ + bool can_use_append:1; + + /* + * This member must come last, bio_alloc_bioset will allocate enough + * bytes for entire btrfs_bio but relies on bio being last. + */ + struct bio bio; +}; + +static inline struct btrfs_bio *btrfs_bio(struct bio *bio) +{ + return container_of(bio, struct btrfs_bio, bio); +} + +int __init btrfs_bioset_init(void); +void __cold btrfs_bioset_exit(void); + +void btrfs_bio_init(struct btrfs_bio *bbio, struct btrfs_inode *inode, u64 file_offset, + btrfs_bio_end_io_t end_io, void *private); +struct btrfs_bio *btrfs_bio_alloc(unsigned int nr_vecs, blk_opf_t opf, + struct btrfs_inode *inode, u64 file_offset, + btrfs_bio_end_io_t end_io, void *private); +void btrfs_bio_end_io(struct btrfs_bio *bbio, blk_status_t status); + +/* Submit using blkcg_punt_bio_submit. */ +#define REQ_BTRFS_CGROUP_PUNT REQ_FS_PRIVATE + +void btrfs_submit_bbio(struct btrfs_bio *bbio, int mirror_num); +void btrfs_submit_repair_write(struct btrfs_bio *bbio, int mirror_num, bool dev_replace); +int btrfs_repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 fileoff, + u32 length, u64 logical, const phys_addr_t paddrs[], + unsigned int step, int mirror_num); + +#endif diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c new file mode 100644 index 000000000..830460a40 --- /dev/null +++ b/fs/btrfs/block-group.c @@ -0,0 +1,5027 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include "misc.h" +#include "ctree.h" +#include "block-group.h" +#include "space-info.h" +#include "disk-io.h" +#include "free-space-cache.h" +#include "free-space-tree.h" +#include "volumes.h" +#include "transaction.h" +#include "ref-verify.h" +#include "sysfs.h" +#include "tree-log.h" +#include "delalloc-space.h" +#include "discard.h" +#include "raid56.h" +#include "zoned.h" +#include "fs.h" +#include "accessors.h" +#include "extent-tree.h" + +static struct kmem_cache *block_group_cache; +static struct kmem_cache *free_space_ctl_cache; + +int __init btrfs_init_block_group(void) +{ + block_group_cache = kmem_cache_create("btrfs_block_group", + sizeof(struct btrfs_block_group), + 0, 0, NULL); + if (!block_group_cache) + return -ENOMEM; + + free_space_ctl_cache = kmem_cache_create("btrfs_free_space_ctl", + sizeof(struct btrfs_free_space_ctl), + 0, 0, NULL); + if (!free_space_ctl_cache) { + kmem_cache_destroy(block_group_cache); + return -ENOMEM; + } + + return 0; +} + +void __cold btrfs_exit_block_group(void) +{ + kmem_cache_destroy(block_group_cache); + kmem_cache_destroy(free_space_ctl_cache); +} + +#ifdef CONFIG_BTRFS_DEBUG +int btrfs_should_fragment_free_space(const struct btrfs_block_group *block_group) +{ + struct btrfs_fs_info *fs_info = block_group->fs_info; + + return (btrfs_test_opt(fs_info, FRAGMENT_METADATA) && + block_group->flags & BTRFS_BLOCK_GROUP_METADATA) || + (btrfs_test_opt(fs_info, FRAGMENT_DATA) && + block_group->flags & BTRFS_BLOCK_GROUP_DATA); +} +#endif + +static inline bool has_unwritten_metadata(struct btrfs_block_group *block_group) +{ + /* The meta_write_pointer is available only on the zoned setup. */ + if (!btrfs_is_zoned(block_group->fs_info)) + return false; + + if (block_group->flags & BTRFS_BLOCK_GROUP_DATA) + return false; + + return block_group->start + block_group->alloc_offset > + block_group->meta_write_pointer; +} + +/* + * Return target flags in extended format or 0 if restripe for this chunk_type + * is not in progress + * + * Should be called with balance_lock held + */ +static u64 get_restripe_target(const struct btrfs_fs_info *fs_info, u64 flags) +{ + const struct btrfs_balance_control *bctl = fs_info->balance_ctl; + u64 target = 0; + + if (!bctl) + return 0; + + if (flags & BTRFS_BLOCK_GROUP_DATA && + bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) { + target = BTRFS_BLOCK_GROUP_DATA | bctl->data.target; + } else if (flags & BTRFS_BLOCK_GROUP_SYSTEM && + bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) { + target = BTRFS_BLOCK_GROUP_SYSTEM | bctl->sys.target; + } else if (flags & BTRFS_BLOCK_GROUP_METADATA && + bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) { + target = BTRFS_BLOCK_GROUP_METADATA | bctl->meta.target; + } + + return target; +} + +/* + * @flags: available profiles in extended format (see ctree.h) + * + * Return reduced profile in chunk format. If profile changing is in progress + * (either running or paused) picks the target profile (if it's already + * available), otherwise falls back to plain reducing. + */ +static u64 btrfs_reduce_alloc_profile(struct btrfs_fs_info *fs_info, u64 flags) +{ + u64 num_devices = fs_info->fs_devices->rw_devices; + u64 target; + u64 raid_type; + u64 allowed = 0; + + /* + * See if restripe for this chunk_type is in progress, if so try to + * reduce to the target profile + */ + spin_lock(&fs_info->balance_lock); + target = get_restripe_target(fs_info, flags); + if (target) { + spin_unlock(&fs_info->balance_lock); + return extended_to_chunk(target); + } + spin_unlock(&fs_info->balance_lock); + + /* First, mask out the RAID levels which aren't possible */ + for (raid_type = 0; raid_type < BTRFS_NR_RAID_TYPES; raid_type++) { + if (num_devices >= btrfs_raid_array[raid_type].devs_min) + allowed |= btrfs_raid_array[raid_type].bg_flag; + } + allowed &= flags; + + /* Select the highest-redundancy RAID level. */ + if (allowed & BTRFS_BLOCK_GROUP_RAID1C4) + allowed = BTRFS_BLOCK_GROUP_RAID1C4; + else if (allowed & BTRFS_BLOCK_GROUP_RAID6) + allowed = BTRFS_BLOCK_GROUP_RAID6; + else if (allowed & BTRFS_BLOCK_GROUP_RAID1C3) + allowed = BTRFS_BLOCK_GROUP_RAID1C3; + else if (allowed & BTRFS_BLOCK_GROUP_RAID5) + allowed = BTRFS_BLOCK_GROUP_RAID5; + else if (allowed & BTRFS_BLOCK_GROUP_RAID10) + allowed = BTRFS_BLOCK_GROUP_RAID10; + else if (allowed & BTRFS_BLOCK_GROUP_RAID1) + allowed = BTRFS_BLOCK_GROUP_RAID1; + else if (allowed & BTRFS_BLOCK_GROUP_DUP) + allowed = BTRFS_BLOCK_GROUP_DUP; + else if (allowed & BTRFS_BLOCK_GROUP_RAID0) + allowed = BTRFS_BLOCK_GROUP_RAID0; + + flags &= ~BTRFS_BLOCK_GROUP_PROFILE_MASK; + + return extended_to_chunk(flags | allowed); +} + +u64 btrfs_get_alloc_profile(struct btrfs_fs_info *fs_info, u64 orig_flags) +{ + unsigned seq; + u64 flags; + + do { + flags = orig_flags; + seq = read_seqbegin(&fs_info->profiles_lock); + + if (flags & BTRFS_BLOCK_GROUP_DATA) + flags |= fs_info->avail_data_alloc_bits; + else if (flags & BTRFS_BLOCK_GROUP_SYSTEM) + flags |= fs_info->avail_system_alloc_bits; + else if (flags & BTRFS_BLOCK_GROUP_METADATA) + flags |= fs_info->avail_metadata_alloc_bits; + } while (read_seqretry(&fs_info->profiles_lock, seq)); + + return btrfs_reduce_alloc_profile(fs_info, flags); +} + +void btrfs_get_block_group(struct btrfs_block_group *cache) +{ + refcount_inc(&cache->refs); +} + +void btrfs_put_block_group(struct btrfs_block_group *cache) +{ + if (refcount_dec_and_test(&cache->refs)) { + WARN_ON(cache->pinned > 0); + /* + * If there was a failure to cleanup a log tree, very likely due + * to an IO failure on a writeback attempt of one or more of its + * extent buffers, we could not do proper (and cheap) unaccounting + * of their reserved space, so don't warn on reserved > 0 in that + * case. + */ + if (!(cache->flags & BTRFS_BLOCK_GROUP_METADATA) || + !BTRFS_FS_LOG_CLEANUP_ERROR(cache->fs_info)) + WARN_ON(cache->reserved > 0); + + /* + * A block_group shouldn't be on the discard_list anymore. + * Remove the block_group from the discard_list to prevent us + * from causing a panic due to NULL pointer dereference. + */ + if (WARN_ON(!list_empty(&cache->discard_list))) + btrfs_discard_cancel_work(&cache->fs_info->discard_ctl, + cache); + + kmem_cache_free(free_space_ctl_cache, cache->free_space_ctl); + btrfs_free_chunk_map(cache->physical_map); + kmem_cache_free(block_group_cache, cache); + } +} + +static int btrfs_bg_start_cmp(const struct rb_node *new, + const struct rb_node *exist) +{ + const struct btrfs_block_group *new_bg = + rb_entry(new, struct btrfs_block_group, cache_node); + const struct btrfs_block_group *exist_bg = + rb_entry(exist, struct btrfs_block_group, cache_node); + + if (new_bg->start < exist_bg->start) + return -1; + if (new_bg->start > exist_bg->start) + return 1; + return 0; +} + +/* + * This adds the block group to the fs_info rb tree for the block group cache + */ +static int btrfs_add_block_group_cache(struct btrfs_block_group *block_group) +{ + struct btrfs_fs_info *fs_info = block_group->fs_info; + struct rb_node *exist; + int ret = 0; + + ASSERT(block_group->length != 0); + + write_lock(&fs_info->block_group_cache_lock); + + exist = rb_find_add_cached(&block_group->cache_node, + &fs_info->block_group_cache_tree, btrfs_bg_start_cmp); + if (exist) + ret = -EEXIST; + write_unlock(&fs_info->block_group_cache_lock); + + return ret; +} + +/* + * This will return the block group at or after bytenr if contains is 0, else + * it will return the block group that contains the bytenr + */ +static struct btrfs_block_group *block_group_cache_tree_search( + struct btrfs_fs_info *info, u64 bytenr, int contains) +{ + struct btrfs_block_group *cache, *ret = NULL; + struct rb_node *n; + u64 end, start; + + read_lock(&info->block_group_cache_lock); + n = info->block_group_cache_tree.rb_root.rb_node; + + while (n) { + cache = rb_entry(n, struct btrfs_block_group, cache_node); + end = btrfs_block_group_end(cache) - 1; + start = cache->start; + + if (bytenr < start) { + if (!contains && (!ret || start < ret->start)) + ret = cache; + n = n->rb_left; + } else if (bytenr > start) { + if (contains && bytenr <= end) { + ret = cache; + break; + } + n = n->rb_right; + } else { + ret = cache; + break; + } + } + if (ret) + btrfs_get_block_group(ret); + read_unlock(&info->block_group_cache_lock); + + return ret; +} + +/* + * Return the block group that starts at or after bytenr + */ +struct btrfs_block_group *btrfs_lookup_first_block_group( + struct btrfs_fs_info *info, u64 bytenr) +{ + return block_group_cache_tree_search(info, bytenr, 0); +} + +/* + * Return the block group that contains the given bytenr + */ +struct btrfs_block_group *btrfs_lookup_block_group( + struct btrfs_fs_info *info, u64 bytenr) +{ + return block_group_cache_tree_search(info, bytenr, 1); +} + +struct btrfs_block_group *btrfs_next_block_group( + struct btrfs_block_group *cache) +{ + struct btrfs_fs_info *fs_info = cache->fs_info; + struct rb_node *node; + + read_lock(&fs_info->block_group_cache_lock); + + /* If our block group was removed, we need a full search. */ + if (RB_EMPTY_NODE(&cache->cache_node)) { + const u64 next_bytenr = btrfs_block_group_end(cache); + + read_unlock(&fs_info->block_group_cache_lock); + btrfs_put_block_group(cache); + return btrfs_lookup_first_block_group(fs_info, next_bytenr); + } + node = rb_next(&cache->cache_node); + btrfs_put_block_group(cache); + if (node) { + cache = rb_entry(node, struct btrfs_block_group, cache_node); + btrfs_get_block_group(cache); + } else + cache = NULL; + read_unlock(&fs_info->block_group_cache_lock); + return cache; +} + +/* + * Check if we can do a NOCOW write for a given extent. + * + * @fs_info: The filesystem information object. + * @bytenr: Logical start address of the extent. + * + * Check if we can do a NOCOW write for the given extent, and increments the + * number of NOCOW writers in the block group that contains the extent, as long + * as the block group exists and it's currently not in read-only mode. + * + * Returns: A non-NULL block group pointer if we can do a NOCOW write, the caller + * is responsible for calling btrfs_dec_nocow_writers() later. + * + * Or NULL if we can not do a NOCOW write + */ +struct btrfs_block_group *btrfs_inc_nocow_writers(struct btrfs_fs_info *fs_info, + u64 bytenr) +{ + struct btrfs_block_group *bg; + bool can_nocow = true; + + bg = btrfs_lookup_block_group(fs_info, bytenr); + if (!bg) + return NULL; + + spin_lock(&bg->lock); + if (bg->ro) + can_nocow = false; + else + atomic_inc(&bg->nocow_writers); + spin_unlock(&bg->lock); + + if (!can_nocow) { + btrfs_put_block_group(bg); + return NULL; + } + + /* No put on block group, done by btrfs_dec_nocow_writers(). */ + return bg; +} + +/* + * Decrement the number of NOCOW writers in a block group. + * + * This is meant to be called after a previous call to btrfs_inc_nocow_writers(), + * and on the block group returned by that call. Typically this is called after + * creating an ordered extent for a NOCOW write, to prevent races with scrub and + * relocation. + * + * After this call, the caller should not use the block group anymore. It it wants + * to use it, then it should get a reference on it before calling this function. + */ +void btrfs_dec_nocow_writers(struct btrfs_block_group *bg) +{ + if (atomic_dec_and_test(&bg->nocow_writers)) + wake_up_var(&bg->nocow_writers); + + /* For the lookup done by a previous call to btrfs_inc_nocow_writers(). */ + btrfs_put_block_group(bg); +} + +void btrfs_wait_nocow_writers(struct btrfs_block_group *bg) +{ + wait_var_event(&bg->nocow_writers, !atomic_read(&bg->nocow_writers)); +} + +void btrfs_dec_block_group_reservations(struct btrfs_fs_info *fs_info, + const u64 start) +{ + struct btrfs_block_group *bg; + + bg = btrfs_lookup_block_group(fs_info, start); + ASSERT(bg); + if (atomic_dec_and_test(&bg->reservations)) + wake_up_var(&bg->reservations); + btrfs_put_block_group(bg); +} + +void btrfs_wait_block_group_reservations(struct btrfs_block_group *bg) +{ + struct btrfs_space_info *space_info = bg->space_info; + + ASSERT(bg->ro); + + if (!(bg->flags & BTRFS_BLOCK_GROUP_DATA)) + return; + + /* + * Our block group is read only but before we set it to read only, + * some task might have had allocated an extent from it already, but it + * has not yet created a respective ordered extent (and added it to a + * root's list of ordered extents). + * Therefore wait for any task currently allocating extents, since the + * block group's reservations counter is incremented while a read lock + * on the groups' semaphore is held and decremented after releasing + * the read access on that semaphore and creating the ordered extent. + */ + down_write(&space_info->groups_sem); + up_write(&space_info->groups_sem); + + wait_var_event(&bg->reservations, !atomic_read(&bg->reservations)); +} + +struct btrfs_caching_control *btrfs_get_caching_control( + struct btrfs_block_group *cache) +{ + struct btrfs_caching_control *ctl; + + spin_lock(&cache->lock); + if (!cache->caching_ctl) { + spin_unlock(&cache->lock); + return NULL; + } + + ctl = cache->caching_ctl; + refcount_inc(&ctl->count); + spin_unlock(&cache->lock); + return ctl; +} + +static void btrfs_put_caching_control(struct btrfs_caching_control *ctl) +{ + if (refcount_dec_and_test(&ctl->count)) + kfree(ctl); +} + +/* + * When we wait for progress in the block group caching, its because our + * allocation attempt failed at least once. So, we must sleep and let some + * progress happen before we try again. + * + * This function will sleep at least once waiting for new free space to show + * up, and then it will check the block group free space numbers for our min + * num_bytes. Another option is to have it go ahead and look in the rbtree for + * a free extent of a given size, but this is a good start. + * + * Callers of this must check if cache->cached == BTRFS_CACHE_ERROR before using + * any of the information in this block group. + */ +void btrfs_wait_block_group_cache_progress(struct btrfs_block_group *cache, + u64 num_bytes) +{ + struct btrfs_caching_control *caching_ctl; + int progress; + + caching_ctl = btrfs_get_caching_control(cache); + if (!caching_ctl) + return; + + /* + * We've already failed to allocate from this block group, so even if + * there's enough space in the block group it isn't contiguous enough to + * allow for an allocation, so wait for at least the next wakeup tick, + * or for the thing to be done. + */ + progress = atomic_read(&caching_ctl->progress); + + wait_event(caching_ctl->wait, btrfs_block_group_done(cache) || + (progress != atomic_read(&caching_ctl->progress) && + (cache->free_space_ctl->free_space >= num_bytes))); + + btrfs_put_caching_control(caching_ctl); +} + +static int btrfs_caching_ctl_wait_done(struct btrfs_block_group *cache, + struct btrfs_caching_control *caching_ctl) +{ + wait_event(caching_ctl->wait, btrfs_block_group_done(cache)); + return cache->cached == BTRFS_CACHE_ERROR ? -EIO : 0; +} + +static int btrfs_wait_block_group_cache_done(struct btrfs_block_group *cache) +{ + struct btrfs_caching_control *caching_ctl; + int ret; + + caching_ctl = btrfs_get_caching_control(cache); + if (!caching_ctl) + return (cache->cached == BTRFS_CACHE_ERROR) ? -EIO : 0; + ret = btrfs_caching_ctl_wait_done(cache, caching_ctl); + btrfs_put_caching_control(caching_ctl); + return ret; +} + +#ifdef CONFIG_BTRFS_DEBUG +static void fragment_free_space(struct btrfs_block_group *block_group) +{ + struct btrfs_fs_info *fs_info = block_group->fs_info; + u64 start = block_group->start; + u64 len = block_group->length; + u64 chunk = block_group->flags & BTRFS_BLOCK_GROUP_METADATA ? + fs_info->nodesize : fs_info->sectorsize; + u64 step = chunk << 1; + + while (len > chunk) { + btrfs_remove_free_space(block_group, start, chunk); + start += step; + if (len < step) + len = 0; + else + len -= step; + } +} +#endif + +/* + * Add a free space range to the in memory free space cache of a block group. + * This checks if the range contains super block locations and any such + * locations are not added to the free space cache. + * + * @block_group: The target block group. + * @start: Start offset of the range. + * @end: End offset of the range (exclusive). + * @total_added_ret: Optional pointer to return the total amount of space + * added to the block group's free space cache. + * + * Returns 0 on success or < 0 on error. + */ +int btrfs_add_new_free_space(struct btrfs_block_group *block_group, u64 start, + u64 end, u64 *total_added_ret) +{ + struct btrfs_fs_info *info = block_group->fs_info; + u64 extent_start, extent_end, size; + int ret; + + if (total_added_ret) + *total_added_ret = 0; + + while (start < end) { + if (!btrfs_find_first_extent_bit(&info->excluded_extents, start, + &extent_start, &extent_end, + EXTENT_DIRTY, NULL)) + break; + + if (extent_start <= start) { + start = extent_end + 1; + } else if (extent_start > start && extent_start < end) { + size = extent_start - start; + ret = btrfs_add_free_space_async_trimmed(block_group, + start, size); + if (ret) + return ret; + if (total_added_ret) + *total_added_ret += size; + start = extent_end + 1; + } else { + break; + } + } + + if (start < end) { + size = end - start; + ret = btrfs_add_free_space_async_trimmed(block_group, start, + size); + if (ret) + return ret; + if (total_added_ret) + *total_added_ret += size; + } + + return 0; +} + +/* + * Get an arbitrary extent item index / max_index through the block group + * + * @caching_ctl the caching control containing the block group to sample from + * @index: the integral step through the block group to grab from + * @max_index: the granularity of the sampling + * @key: return value parameter for the item we find + * @path: path to use for searching in the extent tree + * + * Pre-conditions on indices: + * 0 <= index <= max_index + * 0 < max_index + * + * Returns: 0 on success, 1 if the search didn't yield a useful item. + */ +static int sample_block_group_extent_item(struct btrfs_caching_control *caching_ctl, + int index, int max_index, + struct btrfs_key *found_key, + struct btrfs_path *path) +{ + struct btrfs_block_group *block_group = caching_ctl->block_group; + struct btrfs_fs_info *fs_info = block_group->fs_info; + struct btrfs_root *extent_root; + u64 search_offset; + const u64 search_end = btrfs_block_group_end(block_group); + struct btrfs_key search_key; + int ret = 0; + + ASSERT(index >= 0); + ASSERT(index <= max_index); + ASSERT(max_index > 0); + lockdep_assert_held(&caching_ctl->mutex); + lockdep_assert_held_read(&fs_info->commit_root_sem); + + extent_root = btrfs_extent_root(fs_info, block_group->start); + if (unlikely(!extent_root)) { + btrfs_err(fs_info, + "missing extent root for block group at offset %llu", + block_group->start); + return -EUCLEAN; + } + + search_offset = index * div_u64(block_group->length, max_index); + search_key.objectid = block_group->start + search_offset; + search_key.type = BTRFS_EXTENT_ITEM_KEY; + search_key.offset = 0; + + btrfs_for_each_slot(extent_root, &search_key, found_key, path, ret) { + /* Success; sampled an extent item in the block group */ + if (found_key->type == BTRFS_EXTENT_ITEM_KEY && + found_key->objectid >= block_group->start && + found_key->objectid + found_key->offset <= search_end) + break; + + /* We can't possibly find a valid extent item anymore */ + if (found_key->objectid >= search_end) { + ret = 1; + break; + } + } + + lockdep_assert_held(&caching_ctl->mutex); + lockdep_assert_held_read(&fs_info->commit_root_sem); + return ret; +} + +/* + * Best effort attempt to compute a block group's size class while caching it. + * + * @block_group: the block group we are caching + * + * We cannot infer the size class while adding free space extents, because that + * logic doesn't care about contiguous file extents (it doesn't differentiate + * between a 100M extent and 100 contiguous 1M extents). So we need to read the + * file extent items. Reading all of them is quite wasteful, because usually + * only a handful are enough to give a good answer. Therefore, we just grab 5 of + * them at even steps through the block group and pick the smallest size class + * we see. Since size class is best effort, and not guaranteed in general, + * inaccuracy is acceptable. + * + * To be more explicit about why this algorithm makes sense: + * + * If we are caching in a block group from disk, then there are three major cases + * to consider: + * 1. the block group is well behaved and all extents in it are the same size + * class. + * 2. the block group is mostly one size class with rare exceptions for last + * ditch allocations + * 3. the block group was populated before size classes and can have a totally + * arbitrary mix of size classes. + * + * In case 1, looking at any extent in the block group will yield the correct + * result. For the mixed cases, taking the minimum size class seems like a good + * approximation, since gaps from frees will be usable to the size class. For + * 2., a small handful of file extents is likely to yield the right answer. For + * 3, we can either read every file extent, or admit that this is best effort + * anyway and try to stay fast. + * + * No errors are returned since failing to determine the size class is not a + * critical error, size classes are just an optimization. + */ +static void load_block_group_size_class(struct btrfs_caching_control *caching_ctl) +{ + BTRFS_PATH_AUTO_RELEASE(path); + struct btrfs_block_group *block_group = caching_ctl->block_group; + struct btrfs_fs_info *fs_info = block_group->fs_info; + struct btrfs_key key; + int i; + u64 min_size = block_group->length; + enum btrfs_block_group_size_class size_class = BTRFS_BG_SZ_NONE; + + /* + * Since we run in workqueue context, we allocate the path on stack to + * avoid memory allocation failure, as the stack in a work queue task + * is not deep. + */ + ASSERT(current_work() == &caching_ctl->work.normal_work); + + if (!btrfs_block_group_should_use_size_class(block_group)) + return; + + path.skip_locking = true; + path.search_commit_root = true; + path.reada = READA_FORWARD; + + lockdep_assert_held(&caching_ctl->mutex); + lockdep_assert_held_read(&fs_info->commit_root_sem); + for (i = 0; i < 5; ++i) { + int ret; + + ret = sample_block_group_extent_item(caching_ctl, i, 5, &key, &path); + if (ret < 0) + return; + btrfs_release_path(&path); + if (ret > 0) + continue; + min_size = min_t(u64, min_size, key.offset); + size_class = btrfs_calc_block_group_size_class(min_size); + } + if (size_class != BTRFS_BG_SZ_NONE) { + spin_lock(&block_group->lock); + block_group->size_class = size_class; + spin_unlock(&block_group->lock); + } +} + +static int load_extent_tree_free(struct btrfs_caching_control *caching_ctl) +{ + struct btrfs_block_group *block_group = caching_ctl->block_group; + const u64 block_group_end = btrfs_block_group_end(block_group); + struct btrfs_fs_info *fs_info = block_group->fs_info; + struct btrfs_root *extent_root; + BTRFS_PATH_AUTO_FREE(path); + struct extent_buffer *leaf; + struct btrfs_key key; + u64 total_found = 0; + u64 last = block_group->start; + u32 nritems; + int ret; + bool wakeup = true; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + extent_root = btrfs_extent_root(fs_info, last); + if (unlikely(!extent_root)) { + btrfs_err(fs_info, + "missing extent root for block group at offset %llu", + block_group->start); + return -EUCLEAN; + } + +#ifdef CONFIG_BTRFS_DEBUG + /* + * If we're fragmenting we don't want to make anybody think we can + * allocate from this block group until we've had a chance to fragment + * the free space. + */ + if (btrfs_should_fragment_free_space(block_group)) + wakeup = false; +#endif + /* + * We don't want to deadlock with somebody trying to allocate a new + * extent for the extent root while also trying to search the extent + * root to add free space. So we skip locking and search the commit + * root, since its read-only + */ + path->skip_locking = true; + path->search_commit_root = true; + path->reada = READA_FORWARD; + + key.objectid = last; + key.type = BTRFS_EXTENT_ITEM_KEY; + key.offset = 0; + +next: + ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0); + if (ret < 0) + return ret; + + leaf = path->nodes[0]; + nritems = btrfs_header_nritems(leaf); + + while (1) { + if (btrfs_fs_closing_done(fs_info)) { + last = (u64)-1; + break; + } + + if (path->slots[0] < nritems) { + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + } else { + ret = btrfs_find_next_key(extent_root, path, &key, 0, 0); + if (ret) + break; + + if (need_resched() || + rwsem_is_contended(&fs_info->commit_root_sem)) { + btrfs_release_path(path); + up_read(&fs_info->commit_root_sem); + mutex_unlock(&caching_ctl->mutex); + cond_resched(); + mutex_lock(&caching_ctl->mutex); + down_read(&fs_info->commit_root_sem); + goto next; + } + + ret = btrfs_next_leaf(extent_root, path); + if (ret < 0) + return ret; + if (ret) + break; + leaf = path->nodes[0]; + nritems = btrfs_header_nritems(leaf); + continue; + } + + if (key.objectid < last) { + key.objectid = last; + key.type = BTRFS_EXTENT_ITEM_KEY; + key.offset = 0; + btrfs_release_path(path); + goto next; + } + + if (key.objectid < block_group->start) { + path->slots[0]++; + continue; + } + + if (key.objectid >= block_group_end) + break; + + if (key.type == BTRFS_EXTENT_ITEM_KEY || + key.type == BTRFS_METADATA_ITEM_KEY) { + u64 space_added; + + ret = btrfs_add_new_free_space(block_group, last, + key.objectid, &space_added); + if (ret) + return ret; + total_found += space_added; + if (key.type == BTRFS_METADATA_ITEM_KEY) + last = key.objectid + + fs_info->nodesize; + else + last = key.objectid + key.offset; + + if (total_found > CACHING_CTL_WAKE_UP) { + total_found = 0; + if (wakeup) { + atomic_inc(&caching_ctl->progress); + wake_up(&caching_ctl->wait); + } + } + } + path->slots[0]++; + } + + return btrfs_add_new_free_space(block_group, last, block_group_end, NULL); +} + +static inline void btrfs_free_excluded_extents(const struct btrfs_block_group *bg) +{ + btrfs_clear_extent_bit(&bg->fs_info->excluded_extents, bg->start, + btrfs_block_group_end(bg) - 1, EXTENT_DIRTY, NULL); +} + +static noinline void caching_thread(struct btrfs_work *work) +{ + struct btrfs_block_group *block_group; + struct btrfs_fs_info *fs_info; + struct btrfs_caching_control *caching_ctl; + int ret; + + caching_ctl = container_of(work, struct btrfs_caching_control, work); + block_group = caching_ctl->block_group; + fs_info = block_group->fs_info; + + mutex_lock(&caching_ctl->mutex); + down_read(&fs_info->commit_root_sem); + + load_block_group_size_class(caching_ctl); + if (btrfs_test_opt(fs_info, SPACE_CACHE)) { + ret = load_free_space_cache(block_group); + if (ret == 1) { + ret = 0; + goto done; + } + + /* + * We failed to load the space cache, set ourselves to + * CACHE_STARTED and carry on. + */ + spin_lock(&block_group->lock); + block_group->cached = BTRFS_CACHE_STARTED; + spin_unlock(&block_group->lock); + wake_up(&caching_ctl->wait); + } + + /* + * If we are in the transaction that populated the free space tree we + * can't actually cache from the free space tree as our commit root and + * real root are the same, so we could change the contents of the blocks + * while caching. Instead do the slow caching in this case, and after + * the transaction has committed we will be safe. + */ + if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE) && + !(test_bit(BTRFS_FS_FREE_SPACE_TREE_UNTRUSTED, &fs_info->flags))) + ret = btrfs_load_free_space_tree(caching_ctl); + else + ret = load_extent_tree_free(caching_ctl); +done: + spin_lock(&block_group->lock); + block_group->caching_ctl = NULL; + block_group->cached = ret ? BTRFS_CACHE_ERROR : BTRFS_CACHE_FINISHED; + spin_unlock(&block_group->lock); + +#ifdef CONFIG_BTRFS_DEBUG + if (btrfs_should_fragment_free_space(block_group)) { + u64 bytes_used; + + spin_lock(&block_group->space_info->lock); + spin_lock(&block_group->lock); + bytes_used = block_group->length - block_group->used; + block_group->space_info->bytes_used += bytes_used >> 1; + spin_unlock(&block_group->lock); + spin_unlock(&block_group->space_info->lock); + fragment_free_space(block_group); + } +#endif + + up_read(&fs_info->commit_root_sem); + btrfs_free_excluded_extents(block_group); + mutex_unlock(&caching_ctl->mutex); + + wake_up(&caching_ctl->wait); + + btrfs_put_caching_control(caching_ctl); + btrfs_put_block_group(block_group); +} + +int btrfs_cache_block_group(struct btrfs_block_group *cache, bool wait) +{ + struct btrfs_fs_info *fs_info = cache->fs_info; + struct btrfs_caching_control *caching_ctl = NULL; + int ret = 0; + + /* Allocator for zoned filesystems does not use the cache at all */ + if (btrfs_is_zoned(fs_info)) + return 0; + + /* + * No allocations can be done from remapped block groups, so they have + * no entries in the free-space tree. + */ + if (cache->flags & BTRFS_BLOCK_GROUP_REMAPPED) + return 0; + + caching_ctl = kzalloc_obj(*caching_ctl, GFP_NOFS); + if (!caching_ctl) + return -ENOMEM; + + INIT_LIST_HEAD(&caching_ctl->list); + mutex_init(&caching_ctl->mutex); + init_waitqueue_head(&caching_ctl->wait); + caching_ctl->block_group = cache; + refcount_set(&caching_ctl->count, 2); + atomic_set(&caching_ctl->progress, 0); + btrfs_init_work(&caching_ctl->work, caching_thread, NULL); + + spin_lock(&cache->lock); + if (cache->cached != BTRFS_CACHE_NO) { + kfree(caching_ctl); + + caching_ctl = cache->caching_ctl; + if (caching_ctl) + refcount_inc(&caching_ctl->count); + spin_unlock(&cache->lock); + goto out; + } + WARN_ON(cache->caching_ctl); + cache->caching_ctl = caching_ctl; + cache->cached = BTRFS_CACHE_STARTED; + spin_unlock(&cache->lock); + + write_lock(&fs_info->block_group_cache_lock); + refcount_inc(&caching_ctl->count); + list_add_tail(&caching_ctl->list, &fs_info->caching_block_groups); + write_unlock(&fs_info->block_group_cache_lock); + + btrfs_get_block_group(cache); + + btrfs_queue_work(fs_info->caching_workers, &caching_ctl->work); +out: + if (wait && caching_ctl) + ret = btrfs_caching_ctl_wait_done(cache, caching_ctl); + if (caching_ctl) + btrfs_put_caching_control(caching_ctl); + + return ret; +} + +static void clear_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags) +{ + u64 extra_flags = chunk_to_extended(flags) & + BTRFS_EXTENDED_PROFILE_MASK; + + write_seqlock(&fs_info->profiles_lock); + if (flags & BTRFS_BLOCK_GROUP_DATA) + fs_info->avail_data_alloc_bits &= ~extra_flags; + if (flags & BTRFS_BLOCK_GROUP_METADATA) + fs_info->avail_metadata_alloc_bits &= ~extra_flags; + if (flags & BTRFS_BLOCK_GROUP_SYSTEM) + fs_info->avail_system_alloc_bits &= ~extra_flags; + write_sequnlock(&fs_info->profiles_lock); +} + +/* + * Clear incompat bits for the following feature(s): + * + * - RAID56 - in case there's neither RAID5 nor RAID6 profile block group + * in the whole filesystem + * + * - RAID1C34 - same as above for RAID1C3 and RAID1C4 block groups + */ +static void clear_incompat_bg_bits(struct btrfs_fs_info *fs_info, u64 flags) +{ + bool found_raid56 = false; + bool found_raid1c34 = false; + + if ((flags & BTRFS_BLOCK_GROUP_RAID56_MASK) || + (flags & BTRFS_BLOCK_GROUP_RAID1C3) || + (flags & BTRFS_BLOCK_GROUP_RAID1C4)) { + struct list_head *head = &fs_info->space_info; + struct btrfs_space_info *sinfo; + + list_for_each_entry_rcu(sinfo, head, list) { + down_read(&sinfo->groups_sem); + if (!list_empty(&sinfo->block_groups[BTRFS_RAID_RAID5])) + found_raid56 = true; + if (!list_empty(&sinfo->block_groups[BTRFS_RAID_RAID6])) + found_raid56 = true; + if (!list_empty(&sinfo->block_groups[BTRFS_RAID_RAID1C3])) + found_raid1c34 = true; + if (!list_empty(&sinfo->block_groups[BTRFS_RAID_RAID1C4])) + found_raid1c34 = true; + up_read(&sinfo->groups_sem); + } + if (!found_raid56) + btrfs_clear_fs_incompat(fs_info, RAID56); + if (!found_raid1c34) + btrfs_clear_fs_incompat(fs_info, RAID1C34); + } +} + +static struct btrfs_root *btrfs_block_group_root(struct btrfs_fs_info *fs_info) +{ + if (btrfs_fs_compat_ro(fs_info, BLOCK_GROUP_TREE)) + return fs_info->block_group_root; + return btrfs_extent_root(fs_info, 0); +} + +static int remove_block_group_item(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + struct btrfs_block_group *block_group) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *root; + struct btrfs_key key; + int ret; + + root = btrfs_block_group_root(fs_info); + if (unlikely(!root)) { + btrfs_err(fs_info, "missing block group root"); + return -EUCLEAN; + } + + key.objectid = block_group->start; + key.type = BTRFS_BLOCK_GROUP_ITEM_KEY; + key.offset = block_group->length; + + ret = btrfs_search_slot(trans, root, &key, path, -1, 1); + if (ret > 0) + ret = -ENOENT; + if (ret < 0) + return ret; + + return btrfs_del_item(trans, root, path); +} + +void btrfs_remove_bg_from_sinfo(struct btrfs_block_group *bg) +{ + int factor = btrfs_bg_type_to_factor(bg->flags); + + spin_lock(&bg->space_info->lock); + if (btrfs_test_opt(bg->fs_info, ENOSPC_DEBUG)) { + WARN_ON(bg->space_info->total_bytes < bg->length); + WARN_ON(bg->space_info->bytes_readonly < bg->length - bg->zone_unusable); + WARN_ON(bg->space_info->bytes_zone_unusable < bg->zone_unusable); + WARN_ON(bg->space_info->disk_total < bg->length * factor); + } + bg->space_info->total_bytes -= bg->length; + bg->space_info->bytes_readonly -= (bg->length - bg->zone_unusable); + btrfs_space_info_update_bytes_zone_unusable(bg->space_info, -bg->zone_unusable); + bg->space_info->disk_total -= bg->length * factor; + spin_unlock(&bg->space_info->lock); +} + +int btrfs_remove_block_group(struct btrfs_trans_handle *trans, + struct btrfs_chunk_map *map) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_block_group *block_group; + struct btrfs_free_cluster *cluster; + struct inode *inode; + struct kobject *kobj = NULL; + int ret; + int index; + struct btrfs_caching_control *caching_ctl = NULL; + bool remove_map; + bool remove_rsv = false; + + block_group = btrfs_lookup_block_group(fs_info, map->start); + if (unlikely(!block_group)) { + btrfs_abort_transaction(trans, -ENOENT); + return -ENOENT; + } + + if (unlikely(!block_group->ro && + !(block_group->flags & BTRFS_BLOCK_GROUP_REMAPPED))) { + ret = -EUCLEAN; + btrfs_abort_transaction(trans, ret); + goto out; + } + + trace_btrfs_remove_block_group(block_group); + /* + * Free the reserved super bytes from this block group before + * remove it. + */ + btrfs_free_excluded_extents(block_group); + btrfs_free_ref_tree_range(fs_info, block_group->start, + block_group->length); + + index = btrfs_bg_flags_to_raid_index(block_group->flags); + + /* make sure this block group isn't part of an allocation cluster */ + cluster = &fs_info->data_alloc_cluster; + spin_lock(&cluster->refill_lock); + btrfs_return_cluster_to_free_space(block_group, cluster); + spin_unlock(&cluster->refill_lock); + + /* + * make sure this block group isn't part of a metadata + * allocation cluster + */ + cluster = &fs_info->meta_alloc_cluster; + spin_lock(&cluster->refill_lock); + btrfs_return_cluster_to_free_space(block_group, cluster); + spin_unlock(&cluster->refill_lock); + + btrfs_clear_treelog_bg(block_group); + btrfs_clear_data_reloc_bg(block_group); + + path = btrfs_alloc_path(); + if (unlikely(!path)) { + ret = -ENOMEM; + btrfs_abort_transaction(trans, ret); + goto out; + } + + /* + * get the inode first so any iput calls done for the io_list + * aren't the final iput (no unlinks allowed now) + */ + inode = lookup_free_space_inode(block_group, path); + + mutex_lock(&trans->transaction->cache_write_mutex); + /* + * Make sure our free space cache IO is done before removing the + * free space inode + */ + spin_lock(&trans->transaction->dirty_bgs_lock); + if (!list_empty(&block_group->io_list)) { + list_del_init(&block_group->io_list); + + WARN_ON(!IS_ERR(inode) && inode != block_group->io_ctl.inode); + + spin_unlock(&trans->transaction->dirty_bgs_lock); + btrfs_wait_cache_io(trans, block_group, path); + btrfs_put_block_group(block_group); + spin_lock(&trans->transaction->dirty_bgs_lock); + } + + if (!list_empty(&block_group->dirty_list)) { + list_del_init(&block_group->dirty_list); + remove_rsv = true; + btrfs_put_block_group(block_group); + } + spin_unlock(&trans->transaction->dirty_bgs_lock); + mutex_unlock(&trans->transaction->cache_write_mutex); + + ret = btrfs_remove_free_space_inode(trans, inode, block_group); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + + write_lock(&fs_info->block_group_cache_lock); + rb_erase_cached(&block_group->cache_node, + &fs_info->block_group_cache_tree); + RB_CLEAR_NODE(&block_group->cache_node); + + /* Once for the block groups rbtree */ + btrfs_put_block_group(block_group); + + write_unlock(&fs_info->block_group_cache_lock); + + down_write(&block_group->space_info->groups_sem); + /* + * we must use list_del_init so people can check to see if they + * are still on the list after taking the semaphore + */ + list_del_init(&block_group->list); + if (list_empty(&block_group->space_info->block_groups[index])) { + kobj = block_group->space_info->block_group_kobjs[index]; + block_group->space_info->block_group_kobjs[index] = NULL; + clear_avail_alloc_bits(fs_info, block_group->flags); + } + up_write(&block_group->space_info->groups_sem); + clear_incompat_bg_bits(fs_info, block_group->flags); + if (kobj) { + kobject_del(kobj); + kobject_put(kobj); + } + + if (block_group->cached == BTRFS_CACHE_STARTED) + btrfs_wait_block_group_cache_done(block_group); + + write_lock(&fs_info->block_group_cache_lock); + caching_ctl = btrfs_get_caching_control(block_group); + if (!caching_ctl) { + struct btrfs_caching_control *ctl; + + list_for_each_entry(ctl, &fs_info->caching_block_groups, list) { + if (ctl->block_group == block_group) { + caching_ctl = ctl; + refcount_inc(&caching_ctl->count); + break; + } + } + } + if (caching_ctl) + list_del_init(&caching_ctl->list); + write_unlock(&fs_info->block_group_cache_lock); + + if (caching_ctl) { + /* Once for the caching bgs list and once for us. */ + btrfs_put_caching_control(caching_ctl); + btrfs_put_caching_control(caching_ctl); + } + + spin_lock(&trans->transaction->dirty_bgs_lock); + WARN_ON(!list_empty(&block_group->dirty_list)); + WARN_ON(!list_empty(&block_group->io_list)); + spin_unlock(&trans->transaction->dirty_bgs_lock); + + btrfs_remove_free_space_cache(block_group); + + spin_lock(&block_group->space_info->lock); + list_del_init(&block_group->ro_list); + spin_unlock(&block_group->space_info->lock); + + if (!(block_group->flags & BTRFS_BLOCK_GROUP_REMAPPED)) + btrfs_remove_bg_from_sinfo(block_group); + + /* + * Remove the free space for the block group from the free space tree + * and the block group's item from the extent tree before marking the + * block group as removed. This is to prevent races with tasks that + * freeze and unfreeze a block group, this task and another task + * allocating a new block group - the unfreeze task ends up removing + * the block group's extent map before the task calling this function + * deletes the block group item from the extent tree, allowing for + * another task to attempt to create another block group with the same + * item key (and failing with -EEXIST and a transaction abort). + * + * If the REMAPPED flag has been set the block group's free space + * has already been removed, so we can skip the call to + * btrfs_remove_block_group_free_space(). + */ + if (!(block_group->flags & BTRFS_BLOCK_GROUP_REMAPPED)) { + ret = btrfs_remove_block_group_free_space(trans, block_group); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + } + + ret = remove_block_group_item(trans, path, block_group); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + + spin_lock(&block_group->lock); + /* + * Hitting this WARN means we removed a block group with an unwritten + * region. It will cause "unable to find chunk map for logical" errors. + */ + if (WARN_ON(has_unwritten_metadata(block_group))) + btrfs_warn(fs_info, + "block group %llu is removed before metadata write out", + block_group->start); + + set_bit(BLOCK_GROUP_FLAG_REMOVED, &block_group->runtime_flags); + + /* + * At this point trimming or scrub can't start on this block group, + * because we removed the block group from the rbtree + * fs_info->block_group_cache_tree so no one can't find it anymore and + * even if someone already got this block group before we removed it + * from the rbtree, they have already incremented block_group->frozen - + * if they didn't, for the trimming case they won't find any free space + * entries because we already removed them all when we called + * btrfs_remove_free_space_cache(). + * + * And we must not remove the chunk map from the fs_info->mapping_tree + * to prevent the same logical address range and physical device space + * ranges from being reused for a new block group. This is needed to + * avoid races with trimming and scrub. + * + * An fs trim operation (btrfs_trim_fs() / btrfs_ioctl_fitrim()) is + * completely transactionless, so while it is trimming a range the + * currently running transaction might finish and a new one start, + * allowing for new block groups to be created that can reuse the same + * physical device locations unless we take this special care. + * + * There may also be an implicit trim operation if the file system + * is mounted with -odiscard. The same protections must remain + * in place until the extents have been discarded completely when + * the transaction commit has completed. + */ + remove_map = (atomic_read(&block_group->frozen) == 0); + spin_unlock(&block_group->lock); + + if (remove_map) + btrfs_remove_chunk_map(fs_info, map); + +out: + /* Once for the lookup reference */ + btrfs_put_block_group(block_group); + if (remove_rsv) + btrfs_dec_delayed_refs_rsv_bg_updates(fs_info); + return ret; +} + +struct btrfs_trans_handle *btrfs_start_trans_remove_block_group( + struct btrfs_fs_info *fs_info, const u64 chunk_offset) +{ + struct btrfs_root *root = btrfs_block_group_root(fs_info); + struct btrfs_chunk_map *map; + unsigned int num_items; + + if (unlikely(!root)) { + btrfs_err(fs_info, "missing block group root"); + return ERR_PTR(-EUCLEAN); + } + + map = btrfs_find_chunk_map(fs_info, chunk_offset, 1); + ASSERT(map != NULL); + ASSERT(map->start == chunk_offset); + + /* + * We need to reserve 3 + N units from the metadata space info in order + * to remove a block group (done at btrfs_remove_chunk() and at + * btrfs_remove_block_group()), which are used for: + * + * 1 unit for adding the free space inode's orphan (located in the tree + * of tree roots). + * 1 unit for deleting the block group item (located in the extent + * tree). + * 1 unit for deleting the free space item (located in tree of tree + * roots). + * N units for deleting N device extent items corresponding to each + * stripe (located in the device tree). + * + * In order to remove a block group we also need to reserve units in the + * system space info in order to update the chunk tree (update one or + * more device items and remove one chunk item), but this is done at + * btrfs_remove_chunk() through a call to check_system_chunk(). + */ + num_items = 3 + map->num_stripes; + btrfs_free_chunk_map(map); + + return btrfs_start_transaction_fallback_global_rsv(root, num_items); +} + +/* + * Mark block group @cache read-only, so later write won't happen to block + * group @cache. + * + * If @force is not set, this function will only mark the block group readonly + * if we have enough free space (1M) in other metadata/system block groups. + * If @force is not set, this function will mark the block group readonly + * without checking free space. + * + * NOTE: This function doesn't care if other block groups can contain all the + * data in this block group. That check should be done by relocation routine, + * not this function. + */ +static int inc_block_group_ro(struct btrfs_block_group *cache, bool force) +{ + struct btrfs_space_info *sinfo = cache->space_info; + u64 num_bytes; + int ret = -ENOSPC; + + spin_lock(&sinfo->lock); + spin_lock(&cache->lock); + + if (cache->swap_extents) { + ret = -ETXTBSY; + goto out; + } + + if (cache->ro) { + cache->ro++; + ret = 0; + goto out; + } + + num_bytes = btrfs_block_group_available_space(cache); + + /* + * Data never overcommits, even in mixed mode, so do just the straight + * check of left over space in how much we have allocated. + */ + if (force) { + ret = 0; + } else if (sinfo->flags & BTRFS_BLOCK_GROUP_DATA) { + u64 sinfo_used = btrfs_space_info_used(sinfo, true); + + /* + * Here we make sure if we mark this bg RO, we still have enough + * free space as buffer. + */ + if (sinfo_used + num_bytes <= sinfo->total_bytes) + ret = 0; + } else { + /* + * We overcommit metadata, so we need to do the + * btrfs_can_overcommit check here, and we need to pass in + * BTRFS_RESERVE_NO_FLUSH to give ourselves the most amount of + * leeway to allow us to mark this block group as read only. + */ + if (btrfs_can_overcommit(sinfo, num_bytes, BTRFS_RESERVE_NO_FLUSH)) + ret = 0; + } + + if (!ret) { + sinfo->bytes_readonly += num_bytes; + if (btrfs_is_zoned(cache->fs_info)) { + /* Migrate zone_unusable bytes to readonly */ + sinfo->bytes_readonly += cache->zone_unusable; + btrfs_space_info_update_bytes_zone_unusable(sinfo, -cache->zone_unusable); + cache->zone_unusable = 0; + } + cache->ro++; + list_add_tail(&cache->ro_list, &sinfo->ro_bgs); + } +out: + spin_unlock(&cache->lock); + spin_unlock(&sinfo->lock); + if (ret == -ENOSPC && btrfs_test_opt(cache->fs_info, ENOSPC_DEBUG)) { + btrfs_info(cache->fs_info, + "unable to make block group %llu ro", cache->start); + btrfs_dump_space_info(cache->space_info, 0, false); + } + return ret; +} + +static bool clean_pinned_extents(struct btrfs_trans_handle *trans, + const struct btrfs_block_group *bg) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_transaction *prev_trans = NULL; + const u64 start = bg->start; + const u64 end = start + bg->length - 1; + int ret; + + spin_lock(&fs_info->trans_lock); + if (!list_is_first(&trans->transaction->list, &fs_info->trans_list)) { + prev_trans = list_prev_entry(trans->transaction, list); + refcount_inc(&prev_trans->use_count); + } + spin_unlock(&fs_info->trans_lock); + + /* + * Hold the unused_bg_unpin_mutex lock to avoid racing with + * btrfs_finish_extent_commit(). If we are at transaction N, another + * task might be running finish_extent_commit() for the previous + * transaction N - 1, and have seen a range belonging to the block + * group in pinned_extents before we were able to clear the whole block + * group range from pinned_extents. This means that task can lookup for + * the block group after we unpinned it from pinned_extents and removed + * it, leading to an error at unpin_extent_range(). + */ + mutex_lock(&fs_info->unused_bg_unpin_mutex); + if (prev_trans) { + ret = btrfs_clear_extent_bit(&prev_trans->pinned_extents, start, end, + EXTENT_DIRTY, NULL); + if (ret) + goto out; + } + + ret = btrfs_clear_extent_bit(&trans->transaction->pinned_extents, start, end, + EXTENT_DIRTY, NULL); +out: + mutex_unlock(&fs_info->unused_bg_unpin_mutex); + if (prev_trans) + btrfs_put_transaction(prev_trans); + + return ret == 0; +} + +/* + * Link the block_group to a list via bg_list. + * + * @bg: The block_group to link to the list. + * @list: The list to link it to. + * + * Use this rather than list_add_tail() directly to ensure proper respect + * to locking and refcounting. + * + * Returns: true if the bg was linked with a refcount bump and false otherwise. + */ +static bool btrfs_link_bg_list(struct btrfs_block_group *bg, struct list_head *list) +{ + struct btrfs_fs_info *fs_info = bg->fs_info; + bool added = false; + + spin_lock(&fs_info->unused_bgs_lock); + if (list_empty(&bg->bg_list)) { + btrfs_get_block_group(bg); + list_add_tail(&bg->bg_list, list); + added = true; + } + spin_unlock(&fs_info->unused_bgs_lock); + return added; +} + +/* + * Process the unused_bgs list and remove any that don't have any allocated + * space inside of them. + */ +void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info) +{ + LIST_HEAD(retry_list); + struct btrfs_block_group *block_group; + struct btrfs_space_info *space_info; + struct btrfs_trans_handle *trans; + const bool async_trim_enabled = btrfs_test_opt(fs_info, DISCARD_ASYNC); + int ret = 0; + + if (!test_bit(BTRFS_FS_OPEN, &fs_info->flags)) + return; + + if (btrfs_fs_closing(fs_info)) + return; + + /* + * Long running balances can keep us blocked here for eternity, so + * simply skip deletion if we're unable to get the mutex. + */ + if (!mutex_trylock(&fs_info->reclaim_bgs_lock)) + return; + + spin_lock(&fs_info->unused_bgs_lock); + while (!list_empty(&fs_info->unused_bgs)) { + u64 used; + int trimming; + + block_group = list_first_entry(&fs_info->unused_bgs, + struct btrfs_block_group, + bg_list); + list_del_init(&block_group->bg_list); + + space_info = block_group->space_info; + + if (ret || btrfs_mixed_space_info(space_info)) { + btrfs_put_block_group(block_group); + continue; + } + spin_unlock(&fs_info->unused_bgs_lock); + + btrfs_discard_cancel_work(&fs_info->discard_ctl, block_group); + + /* Don't want to race with allocators so take the groups_sem */ + down_write(&space_info->groups_sem); + + /* + * Async discard moves the final block group discard to be prior + * to the unused_bgs code path. Therefore, if it's not fully + * trimmed, punt it back to the async discard lists. + */ + if (btrfs_test_opt(fs_info, DISCARD_ASYNC) && + !btrfs_is_free_space_trimmed(block_group)) { + trace_btrfs_skip_unused_block_group(block_group); + up_write(&space_info->groups_sem); + /* Requeue if we failed because of async discard */ + btrfs_discard_queue_work(&fs_info->discard_ctl, + block_group); + goto next; + } + + spin_lock(&space_info->lock); + spin_lock(&block_group->lock); + + if (btrfs_is_zoned(fs_info) && btrfs_is_block_group_used(block_group) && + block_group->zone_unusable >= div_u64(block_group->length, 2)) { + /* + * If the block group has data left, but at least half + * of the block group is zone_unusable, mark it as + * reclaimable before continuing with the next block group. + */ + + spin_unlock(&block_group->lock); + spin_unlock(&space_info->lock); + up_write(&space_info->groups_sem); + + btrfs_mark_bg_to_reclaim(block_group); + + goto next; + } + + if (btrfs_is_block_group_used(block_group) || + (block_group->ro && !(block_group->flags & BTRFS_BLOCK_GROUP_REMAPPED)) || + list_is_singular(&block_group->list) || + test_bit(BLOCK_GROUP_FLAG_FULLY_REMAPPED, &block_group->runtime_flags)) { + /* + * We want to bail if we made new allocations or have + * outstanding allocations in this block group. We do + * the ro check in case balance is currently acting on + * this block group. + * + * Also bail out if this is the only block group for its + * type, because otherwise we would lose profile + * information from fs_info->avail_*_alloc_bits and the + * next block group of this type would be created with a + * "single" profile (even if we're in a raid fs) because + * fs_info->avail_*_alloc_bits would be 0. + */ + trace_btrfs_skip_unused_block_group(block_group); + spin_unlock(&block_group->lock); + spin_unlock(&space_info->lock); + up_write(&space_info->groups_sem); + goto next; + } + + /* + * The block group may be unused but there may be space reserved + * accounting with the existence of that block group, that is, + * space_info->bytes_may_use was incremented by a task but no + * space was yet allocated from the block group by the task. + * That space may or may not be allocated, as we are generally + * pessimistic about space reservation for metadata as well as + * for data when using compression (as we reserve space based on + * the worst case, when data can't be compressed, and before + * actually attempting compression, before starting writeback). + * + * So check if the total space of the space_info minus the size + * of this block group is less than the used space of the + * space_info - if that's the case, then it means we have tasks + * that might be relying on the block group in order to allocate + * extents, and add back the block group to the unused list when + * we finish, so that we retry later in case no tasks ended up + * needing to allocate extents from the block group. + */ + used = btrfs_space_info_used(space_info, true); + if (((space_info->total_bytes - block_group->length < used && + block_group->zone_unusable < block_group->length) || + has_unwritten_metadata(block_group)) && + !(block_group->flags & BTRFS_BLOCK_GROUP_REMAPPED)) { + /* + * Add a reference for the list, compensate for the ref + * drop under the "next" label for the + * fs_info->unused_bgs list. + */ + btrfs_link_bg_list(block_group, &retry_list); + + trace_btrfs_skip_unused_block_group(block_group); + spin_unlock(&block_group->lock); + spin_unlock(&space_info->lock); + up_write(&space_info->groups_sem); + goto next; + } + + spin_unlock(&block_group->lock); + spin_unlock(&space_info->lock); + + /* We don't want to force the issue, only flip if it's ok. */ + ret = inc_block_group_ro(block_group, false); + up_write(&space_info->groups_sem); + if (ret < 0) { + ret = 0; + goto next; + } + + ret = btrfs_zone_finish(block_group); + if (ret < 0) { + btrfs_dec_block_group_ro(block_group); + if (ret == -EAGAIN) { + btrfs_link_bg_list(block_group, &retry_list); + ret = 0; + } + goto next; + } + + /* + * Want to do this before we do anything else so we can recover + * properly if we fail to join the transaction. + */ + trans = btrfs_start_trans_remove_block_group(fs_info, + block_group->start); + if (IS_ERR(trans)) { + btrfs_dec_block_group_ro(block_group); + ret = PTR_ERR(trans); + goto next; + } + + /* + * We could have pending pinned extents for this block group, + * just delete them, we don't care about them anymore. + */ + if (!clean_pinned_extents(trans, block_group)) { + btrfs_dec_block_group_ro(block_group); + goto end_trans; + } + + /* + * At this point, the block_group is read only and should fail + * new allocations. However, btrfs_finish_extent_commit() can + * cause this block_group to be placed back on the discard + * lists because now the block_group isn't fully discarded. + * Bail here and try again later after discarding everything. + */ + spin_lock(&fs_info->discard_ctl.lock); + if (!list_empty(&block_group->discard_list)) { + spin_unlock(&fs_info->discard_ctl.lock); + btrfs_dec_block_group_ro(block_group); + btrfs_discard_queue_work(&fs_info->discard_ctl, + block_group); + goto end_trans; + } + spin_unlock(&fs_info->discard_ctl.lock); + + /* Reset pinned so btrfs_put_block_group doesn't complain */ + spin_lock(&space_info->lock); + spin_lock(&block_group->lock); + + btrfs_space_info_update_bytes_pinned(space_info, -block_group->pinned); + space_info->bytes_readonly += block_group->pinned; + block_group->pinned = 0; + + spin_unlock(&block_group->lock); + spin_unlock(&space_info->lock); + + /* + * The normal path here is an unused block group is passed here, + * then trimming is handled in the transaction commit path. + * Async discard interposes before this to do the trimming + * before coming down the unused block group path as trimming + * will no longer be done later in the transaction commit path. + */ + if (!async_trim_enabled && btrfs_test_opt(fs_info, DISCARD_ASYNC)) + goto flip_async; + + /* + * DISCARD can flip during remount. On zoned filesystems, we + * need to reset sequential-required zones. + */ + trimming = btrfs_test_opt(fs_info, DISCARD_SYNC) || + btrfs_is_zoned(fs_info); + + /* Implicit trim during transaction commit. */ + if (trimming) + btrfs_freeze_block_group(block_group); + + /* + * Btrfs_remove_chunk will abort the transaction if things go + * horribly wrong. + */ + ret = btrfs_remove_chunk(trans, block_group->start); + + if (ret) { + if (trimming) + btrfs_unfreeze_block_group(block_group); + goto end_trans; + } + + /* + * If we're not mounted with -odiscard, we can just forget + * about this block group. Otherwise we'll need to wait + * until transaction commit to do the actual discard. + */ + if (trimming) { + spin_lock(&fs_info->unused_bgs_lock); + /* + * A concurrent scrub might have added us to the list + * fs_info->unused_bgs, so use a list_move operation + * to add the block group to the deleted_bgs list. + */ + list_move(&block_group->bg_list, + &trans->transaction->deleted_bgs); + spin_unlock(&fs_info->unused_bgs_lock); + btrfs_get_block_group(block_group); + } +end_trans: + btrfs_end_transaction(trans); +next: + btrfs_put_block_group(block_group); + spin_lock(&fs_info->unused_bgs_lock); + } + list_splice_tail(&retry_list, &fs_info->unused_bgs); + spin_unlock(&fs_info->unused_bgs_lock); + mutex_unlock(&fs_info->reclaim_bgs_lock); + return; + +flip_async: + btrfs_end_transaction(trans); + spin_lock(&fs_info->unused_bgs_lock); + list_splice_tail(&retry_list, &fs_info->unused_bgs); + spin_unlock(&fs_info->unused_bgs_lock); + mutex_unlock(&fs_info->reclaim_bgs_lock); + btrfs_put_block_group(block_group); + btrfs_discard_punt_unused_bgs_list(fs_info); +} + +void btrfs_mark_bg_unused(struct btrfs_block_group *bg) +{ + struct btrfs_fs_info *fs_info = bg->fs_info; + + spin_lock(&fs_info->unused_bgs_lock); + if (list_empty(&bg->bg_list)) { + btrfs_get_block_group(bg); + trace_btrfs_add_unused_block_group(bg); + list_add_tail(&bg->bg_list, &fs_info->unused_bgs); + } else if (bg->flags & BTRFS_BLOCK_GROUP_REMAPPED && + bg->identity_remap_count == 0) { + /* Leave fully remapped block groups on the fully_remapped_bgs list. */ + } else if (!test_bit(BLOCK_GROUP_FLAG_NEW, &bg->runtime_flags)) { + /* Pull out the block group from the reclaim_bgs list. */ + trace_btrfs_add_unused_block_group(bg); + list_move_tail(&bg->bg_list, &fs_info->unused_bgs); + } + spin_unlock(&fs_info->unused_bgs_lock); +} + +/* + * We want block groups with a low number of used bytes to be in the beginning + * of the list, so they will get reclaimed first. + */ +static int reclaim_bgs_cmp(void *unused, const struct list_head *a, + const struct list_head *b) +{ + const struct btrfs_block_group *bg1, *bg2; + + bg1 = list_entry(a, struct btrfs_block_group, bg_list); + bg2 = list_entry(b, struct btrfs_block_group, bg_list); + + /* + * Some other task may be updating the ->used field concurrently, but it + * is not serious if we get a stale value or load/store tearing issues, + * as sorting the list of block groups to reclaim is not critical and an + * occasional imperfect order is ok. So silence KCSAN and avoid the + * overhead of locking or any other synchronization. + */ + return data_race(bg1->used > bg2->used); +} + +static inline bool btrfs_should_reclaim(const struct btrfs_fs_info *fs_info) +{ + if (!test_bit(BTRFS_FS_OPEN, &fs_info->flags)) + return false; + + if (btrfs_fs_closing(fs_info)) + return false; + + if (btrfs_is_zoned(fs_info)) + return btrfs_zoned_should_reclaim(fs_info); + return true; +} + +static bool should_reclaim_block_group(const struct btrfs_block_group *bg, u64 bytes_freed) +{ + const int thresh_pct = btrfs_calc_reclaim_threshold(bg->space_info); + u64 thresh_bytes = mult_perc(bg->length, thresh_pct); + const u64 new_val = bg->used; + const u64 old_val = new_val + bytes_freed; + + if (thresh_bytes == 0) + return false; + + /* + * If we were below the threshold before don't reclaim, we are likely a + * brand new block group and we don't want to relocate new block groups. + */ + if (old_val < thresh_bytes) + return false; + if (new_val >= thresh_bytes) + return false; + return true; +} + +static int btrfs_reclaim_block_group(struct btrfs_block_group *bg, int *reclaimed) +{ + struct btrfs_fs_info *fs_info = bg->fs_info; + struct btrfs_space_info *space_info = bg->space_info; + u64 used; + u64 reserved; + u64 old_total; + int ret = 0; + + /* Don't race with allocators so take the groups_sem */ + down_write(&space_info->groups_sem); + + spin_lock(&space_info->lock); + spin_lock(&bg->lock); + if (bg->reserved || bg->pinned || bg->ro) { + /* + * We want to bail if we made new allocations or have + * outstanding allocations in this block group. We do + * the ro check in case balance is currently acting on + * this block group. + */ + spin_unlock(&bg->lock); + spin_unlock(&space_info->lock); + up_write(&space_info->groups_sem); + return 0; + } + + if (bg->used == 0) { + /* + * It is possible that we trigger relocation on a block + * group as its extents are deleted and it first goes + * below the threshold, then shortly after goes empty. + * + * In this case, relocating it does delete it, but has + * some overhead in relocation specific metadata, looking + * for the non-existent extents and running some extra + * transactions, which we can avoid by using one of the + * other mechanisms for dealing with empty block groups. + */ + if (!btrfs_test_opt(fs_info, DISCARD_ASYNC)) + btrfs_mark_bg_unused(bg); + spin_unlock(&bg->lock); + spin_unlock(&space_info->lock); + up_write(&space_info->groups_sem); + return 0; + } + + /* + * The block group might no longer meet the reclaim condition by + * the time we get around to reclaiming it, so to avoid + * reclaiming overly full block_groups, skip reclaiming them. + * + * Since the decision making process also depends on the amount + * being freed, pass in a fake giant value to skip that extra + * check, which is more meaningful when adding to the list in + * the first place. + */ + if (!should_reclaim_block_group(bg, bg->length)) { + spin_unlock(&bg->lock); + spin_unlock(&space_info->lock); + up_write(&space_info->groups_sem); + return 0; + } + + spin_unlock(&bg->lock); + old_total = space_info->total_bytes; + spin_unlock(&space_info->lock); + + /* + * Get out fast, in case we're read-only or unmounting the + * filesystem. It is OK to drop block groups from the list even + * for the read-only case. As we did take the super write lock, + * "mount -o remount,ro" won't happen and read-only filesystem + * means it is forced read-only due to a fatal error. So, it + * never gets back to read-write to let us reclaim again. + */ + if (btrfs_need_cleaner_sleep(fs_info)) { + up_write(&space_info->groups_sem); + return 0; + } + + ret = inc_block_group_ro(bg, false); + up_write(&space_info->groups_sem); + if (ret < 0) + return ret; + + /* + * The amount of bytes reclaimed corresponds to the sum of the + * "used" and "reserved" counters. We have set the block group + * to RO above, which prevents reservations from happening but + * we may have existing reservations for which allocation has + * not yet been done - btrfs_update_block_group() was not yet + * called, which is where we will transfer a reserved extent's + * size from the "reserved" counter to the "used" counter - this + * happens when running delayed references. When we relocate the + * chunk below, relocation first flushes delalloc, waits for + * ordered extent completion (which is where we create delayed + * references for data extents) and commits the current + * transaction (which runs delayed references), and only after + * it does the actual work to move extents out of the block + * group. So the reported amount of reclaimed bytes is + * effectively the sum of the 'used' and 'reserved' counters. + */ + spin_lock(&bg->lock); + used = bg->used; + reserved = bg->reserved; + spin_unlock(&bg->lock); + + trace_btrfs_reclaim_block_group(bg); + ret = btrfs_relocate_chunk(fs_info, bg->start, false); + if (btrfs_is_zoned(fs_info) && ret == -EAGAIN) { + btrfs_dec_block_group_ro(bg); + btrfs_debug(fs_info, "deferring reclaim of chunk %llu", bg->start); + return ret; + } + if (ret) { + btrfs_dec_block_group_ro(bg); + btrfs_err(fs_info, "error relocating chunk %llu", + bg->start); + used = 0; + reserved = 0; + spin_lock(&space_info->lock); + space_info->reclaim_errors++; + spin_unlock(&space_info->lock); + } + spin_lock(&space_info->lock); + space_info->reclaim_count++; + space_info->reclaim_bytes += used; + space_info->reclaim_bytes += reserved; + if (space_info->total_bytes < old_total) + btrfs_set_periodic_reclaim_ready(space_info, true); + spin_unlock(&space_info->lock); + if (!ret) + (*reclaimed)++; + + return ret; +} + +void btrfs_reclaim_block_groups(struct btrfs_fs_info *fs_info, unsigned int limit) +{ + struct btrfs_block_group *bg; + struct btrfs_space_info *space_info; + LIST_HEAD(retry_list); + int reclaimed = 0; + + if (!btrfs_should_reclaim(fs_info)) + return; + + guard(super_write)(fs_info->sb); + + if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_BALANCE)) + return; + + /* + * Long running balances can keep us blocked here for eternity, so + * simply skip reclaim if we're unable to get the mutex. + */ + if (!mutex_trylock(&fs_info->reclaim_bgs_lock)) { + btrfs_exclop_finish(fs_info); + return; + } + + spin_lock(&fs_info->unused_bgs_lock); + /* + * Sort happens under lock because we can't simply splice it and sort. + * The block groups might still be in use and reachable via bg_list, + * and their presence in the reclaim_bgs list must be preserved. + */ + list_sort(NULL, &fs_info->reclaim_bgs, reclaim_bgs_cmp); + while (!list_empty(&fs_info->reclaim_bgs)) { + int ret; + + bg = list_first_entry(&fs_info->reclaim_bgs, + struct btrfs_block_group, + bg_list); + list_del_init(&bg->bg_list); + + space_info = bg->space_info; + spin_unlock(&fs_info->unused_bgs_lock); + ret = btrfs_reclaim_block_group(bg, &reclaimed); + + if ((btrfs_is_zoned(fs_info) && ret == -EAGAIN) || + (ret && !READ_ONCE(space_info->periodic_reclaim))) + btrfs_link_bg_list(bg, &retry_list); + btrfs_put_block_group(bg); + + mutex_unlock(&fs_info->reclaim_bgs_lock); + /* + * Reclaiming all the block groups in the list can take really + * long. Prioritize cleaning up unused block groups. + */ + btrfs_delete_unused_bgs(fs_info); + /* + * If we are interrupted by a balance, we can just bail out. The + * cleaner thread restart again if necessary. + */ + if (!mutex_trylock(&fs_info->reclaim_bgs_lock)) + goto end; + spin_lock(&fs_info->unused_bgs_lock); + if (reclaimed >= limit) + break; + } + spin_unlock(&fs_info->unused_bgs_lock); + mutex_unlock(&fs_info->reclaim_bgs_lock); +end: + spin_lock(&fs_info->unused_bgs_lock); + list_splice_tail(&retry_list, &fs_info->reclaim_bgs); + spin_unlock(&fs_info->unused_bgs_lock); + btrfs_exclop_finish(fs_info); +} + +void btrfs_reclaim_bgs_work(struct work_struct *work) +{ + struct btrfs_fs_info *fs_info = + container_of(work, struct btrfs_fs_info, reclaim_bgs_work); + + btrfs_reclaim_block_groups(fs_info, -1); +} + +void btrfs_reclaim_bgs(struct btrfs_fs_info *fs_info) +{ + btrfs_reclaim_sweep(fs_info); + spin_lock(&fs_info->unused_bgs_lock); + if (!list_empty(&fs_info->reclaim_bgs)) + queue_work(system_dfl_wq, &fs_info->reclaim_bgs_work); + spin_unlock(&fs_info->unused_bgs_lock); +} + +void btrfs_mark_bg_to_reclaim(struct btrfs_block_group *bg) +{ + struct btrfs_fs_info *fs_info = bg->fs_info; + + if (btrfs_link_bg_list(bg, &fs_info->reclaim_bgs)) + trace_btrfs_add_reclaim_block_group(bg); +} + +static int read_bg_from_eb(struct btrfs_fs_info *fs_info, const struct btrfs_key *key, + const struct btrfs_path *path) +{ + struct btrfs_chunk_map *map; + struct btrfs_block_group_item bg; + struct extent_buffer *leaf; + int slot; + u64 flags; + int ret = 0; + + slot = path->slots[0]; + leaf = path->nodes[0]; + + map = btrfs_find_chunk_map(fs_info, key->objectid, key->offset); + if (!map) { + btrfs_err(fs_info, + "logical %llu len %llu found bg but no related chunk", + key->objectid, key->offset); + return -ENOENT; + } + + if (unlikely(map->start != key->objectid || map->chunk_len != key->offset)) { + btrfs_err(fs_info, + "block group %llu len %llu mismatch with chunk %llu len %llu", + key->objectid, key->offset, map->start, map->chunk_len); + ret = -EUCLEAN; + goto out_free_map; + } + + read_extent_buffer(leaf, &bg, btrfs_item_ptr_offset(leaf, slot), + sizeof(bg)); + flags = btrfs_stack_block_group_flags(&bg) & + BTRFS_BLOCK_GROUP_TYPE_MASK; + + if (unlikely(flags != (map->type & BTRFS_BLOCK_GROUP_TYPE_MASK))) { + btrfs_err(fs_info, +"block group %llu len %llu type flags 0x%llx mismatch with chunk type flags 0x%llx", + key->objectid, key->offset, flags, + (BTRFS_BLOCK_GROUP_TYPE_MASK & map->type)); + ret = -EUCLEAN; + } + +out_free_map: + btrfs_free_chunk_map(map); + return ret; +} + +static int find_first_block_group(struct btrfs_fs_info *fs_info, + struct btrfs_path *path, + const struct btrfs_key *key) +{ + struct btrfs_root *root = btrfs_block_group_root(fs_info); + int ret; + struct btrfs_key found_key; + + if (unlikely(!root)) { + btrfs_err(fs_info, "missing block group root"); + return -EUCLEAN; + } + + btrfs_for_each_slot(root, key, &found_key, path, ret) { + if (found_key.objectid >= key->objectid && + found_key.type == BTRFS_BLOCK_GROUP_ITEM_KEY) { + return read_bg_from_eb(fs_info, &found_key, path); + } + } + return ret; +} + +static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags) +{ + u64 extra_flags = chunk_to_extended(flags) & + BTRFS_EXTENDED_PROFILE_MASK; + + write_seqlock(&fs_info->profiles_lock); + if (flags & BTRFS_BLOCK_GROUP_DATA) + fs_info->avail_data_alloc_bits |= extra_flags; + if (flags & BTRFS_BLOCK_GROUP_METADATA) + fs_info->avail_metadata_alloc_bits |= extra_flags; + if (flags & BTRFS_BLOCK_GROUP_SYSTEM) + fs_info->avail_system_alloc_bits |= extra_flags; + write_sequnlock(&fs_info->profiles_lock); +} + +/* + * Map a physical disk address to a list of logical addresses. + * + * @fs_info: the filesystem + * @chunk_start: logical address of block group + * @physical: physical address to map to logical addresses + * @logical: return array of logical addresses which map to @physical + * @naddrs: length of @logical + * @stripe_len: size of IO stripe for the given block group + * + * Maps a particular @physical disk address to a list of @logical addresses. + * Used primarily to exclude those portions of a block group that contain super + * block copies. + */ +int btrfs_rmap_block(struct btrfs_fs_info *fs_info, u64 chunk_start, + u64 physical, u64 **logical, int *naddrs, int *stripe_len) +{ + struct btrfs_chunk_map *map; + u64 *buf; + u64 bytenr; + u64 data_stripe_length; + u64 io_stripe_size; + int i, nr = 0; + int ret = 0; + + map = btrfs_get_chunk_map(fs_info, chunk_start, 1); + if (IS_ERR(map)) + return -EIO; + + data_stripe_length = map->stripe_size; + io_stripe_size = BTRFS_STRIPE_LEN; + chunk_start = map->start; + + /* For RAID5/6 adjust to a full IO stripe length */ + if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) + io_stripe_size = btrfs_stripe_nr_to_offset(nr_data_stripes(map)); + + buf = kzalloc_objs(u64, map->num_stripes, GFP_NOFS); + if (!buf) { + ret = -ENOMEM; + goto out; + } + + for (i = 0; i < map->num_stripes; i++) { + bool already_inserted = false; + u32 stripe_nr; + u32 offset; + int j; + + if (!in_range(physical, map->stripes[i].physical, + data_stripe_length)) + continue; + + stripe_nr = (physical - map->stripes[i].physical) >> + BTRFS_STRIPE_LEN_SHIFT; + offset = (physical - map->stripes[i].physical) & + BTRFS_STRIPE_LEN_MASK; + + if (map->type & (BTRFS_BLOCK_GROUP_RAID0 | + BTRFS_BLOCK_GROUP_RAID10)) + stripe_nr = div_u64(stripe_nr * map->num_stripes + i, + map->sub_stripes); + /* + * The remaining case would be for RAID56, multiply by + * nr_data_stripes(). Alternatively, just use rmap_len below + * instead of map->stripe_len + */ + bytenr = chunk_start + stripe_nr * io_stripe_size + offset; + + /* Ensure we don't add duplicate addresses */ + for (j = 0; j < nr; j++) { + if (buf[j] == bytenr) { + already_inserted = true; + break; + } + } + + if (!already_inserted) + buf[nr++] = bytenr; + } + + *logical = buf; + *naddrs = nr; + *stripe_len = io_stripe_size; +out: + btrfs_free_chunk_map(map); + return ret; +} + +static int exclude_super_stripes(struct btrfs_block_group *cache) +{ + struct btrfs_fs_info *fs_info = cache->fs_info; + const bool zoned = btrfs_is_zoned(fs_info); + u64 bytenr; + u64 *logical; + int stripe_len; + int i, nr, ret; + + if (cache->start < BTRFS_SUPER_INFO_OFFSET) { + stripe_len = BTRFS_SUPER_INFO_OFFSET - cache->start; + cache->bytes_super += stripe_len; + ret = btrfs_set_extent_bit(&fs_info->excluded_extents, cache->start, + cache->start + stripe_len - 1, + EXTENT_DIRTY, NULL); + if (ret) + return ret; + } + + for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) { + bytenr = btrfs_sb_offset(i); + ret = btrfs_rmap_block(fs_info, cache->start, + bytenr, &logical, &nr, &stripe_len); + if (ret) + return ret; + + /* Shouldn't have super stripes in sequential zones */ + if (unlikely(zoned && nr)) { + kfree(logical); + btrfs_err(fs_info, + "zoned: block group %llu must not contain super block", + cache->start); + return -EUCLEAN; + } + + while (nr--) { + u64 len = min_t(u64, stripe_len, + btrfs_block_group_end(cache) - logical[nr]); + + cache->bytes_super += len; + ret = btrfs_set_extent_bit(&fs_info->excluded_extents, + logical[nr], logical[nr] + len - 1, + EXTENT_DIRTY, NULL); + if (ret) { + kfree(logical); + return ret; + } + } + + kfree(logical); + } + return 0; +} + +static struct btrfs_block_group *btrfs_create_block_group( + struct btrfs_fs_info *fs_info, u64 start) +{ + struct btrfs_block_group *cache; + + cache = kmem_cache_zalloc(block_group_cache, GFP_NOFS); + if (!cache) + return NULL; + + cache->free_space_ctl = kmem_cache_zalloc(free_space_ctl_cache, GFP_NOFS); + if (!cache->free_space_ctl) { + kmem_cache_free(block_group_cache, cache); + return NULL; + } + + cache->start = start; + + cache->fs_info = fs_info; + cache->full_stripe_len = btrfs_full_stripe_len(fs_info, start); + + cache->discard_index = BTRFS_DISCARD_INDEX_UNUSED; + + refcount_set(&cache->refs, 1); + spin_lock_init(&cache->lock); + init_rwsem(&cache->data_rwsem); + INIT_LIST_HEAD(&cache->list); + INIT_LIST_HEAD(&cache->cluster_list); + INIT_LIST_HEAD(&cache->bg_list); + INIT_LIST_HEAD(&cache->ro_list); + INIT_LIST_HEAD(&cache->discard_list); + INIT_LIST_HEAD(&cache->dirty_list); + INIT_LIST_HEAD(&cache->io_list); + INIT_LIST_HEAD(&cache->active_bg_list); + btrfs_init_free_space_ctl(cache, cache->free_space_ctl); + atomic_set(&cache->frozen, 0); + mutex_init(&cache->free_space_lock); + + return cache; +} + +/* + * Iterate all chunks and verify that each of them has the corresponding block + * group + */ +static int check_chunk_block_group_mappings(struct btrfs_fs_info *fs_info) +{ + struct rb_node *node; + int ret = 0; + + /* + * This is called during mount from btrfs_read_block_groups(), before + * any background threads are started, so no concurrent writers can + * modify the mapping_tree. No lock is needed here. + */ + for (node = rb_first_cached(&fs_info->mapping_tree); node; node = rb_next(node)) { + struct btrfs_chunk_map *map; + struct btrfs_block_group *bg; + + map = rb_entry(node, struct btrfs_chunk_map, rb_node); + bg = btrfs_lookup_block_group(fs_info, map->start); + if (unlikely(!bg)) { + btrfs_err(fs_info, + "chunk start=%llu len=%llu doesn't have corresponding block group", + map->start, map->chunk_len); + ret = -EUCLEAN; + break; + } + if (unlikely(bg->start != map->start || bg->length != map->chunk_len || + (bg->flags & BTRFS_BLOCK_GROUP_TYPE_MASK) != + (map->type & BTRFS_BLOCK_GROUP_TYPE_MASK))) { + btrfs_err(fs_info, +"chunk start=%llu len=%llu flags=0x%llx doesn't match block group start=%llu len=%llu flags=0x%llx", + map->start, map->chunk_len, + map->type & BTRFS_BLOCK_GROUP_TYPE_MASK, + bg->start, bg->length, + bg->flags & BTRFS_BLOCK_GROUP_TYPE_MASK); + ret = -EUCLEAN; + btrfs_put_block_group(bg); + break; + } + btrfs_put_block_group(bg); + } + return ret; +} + +static int read_one_block_group(struct btrfs_fs_info *info, + struct btrfs_block_group_item_v2 *bgi, + const struct btrfs_key *key, + bool need_clear) +{ + struct btrfs_block_group *cache; + const bool mixed = btrfs_fs_incompat(info, MIXED_GROUPS); + int ret; + + ASSERT(key->type == BTRFS_BLOCK_GROUP_ITEM_KEY); + + cache = btrfs_create_block_group(info, key->objectid); + if (!cache) + return -ENOMEM; + + cache->length = key->offset; + cache->used = btrfs_stack_block_group_v2_used(bgi); + cache->last_used = cache->used; + cache->flags = btrfs_stack_block_group_v2_flags(bgi); + cache->last_flags = cache->flags; + cache->global_root_id = btrfs_stack_block_group_v2_chunk_objectid(bgi); + cache->space_info = btrfs_find_space_info(info, cache->flags); + cache->remap_bytes = btrfs_stack_block_group_v2_remap_bytes(bgi); + cache->last_remap_bytes = cache->remap_bytes; + cache->identity_remap_count = btrfs_stack_block_group_v2_identity_remap_count(bgi); + cache->last_identity_remap_count = cache->identity_remap_count; + + btrfs_set_free_space_tree_thresholds(cache); + + if (need_clear) { + /* + * When we mount with old space cache, we need to + * set BTRFS_DC_CLEAR and set dirty flag. + * + * a) Setting 'BTRFS_DC_CLEAR' makes sure that we + * truncate the old free space cache inode and + * setup a new one. + * b) Setting 'dirty flag' makes sure that we flush + * the new space cache info onto disk. + */ + if (btrfs_test_opt(info, SPACE_CACHE)) + cache->disk_cache_state = BTRFS_DC_CLEAR; + } + if (!mixed && ((cache->flags & BTRFS_BLOCK_GROUP_METADATA) && + (cache->flags & BTRFS_BLOCK_GROUP_DATA))) { + btrfs_err(info, +"bg %llu is a mixed block group but filesystem hasn't enabled mixed block groups", + cache->start); + ret = -EINVAL; + goto error; + } + + ret = btrfs_load_block_group_zone_info(cache, false); + if (ret) { + btrfs_err(info, "zoned: failed to load zone info of bg %llu", + cache->start); + goto error; + } + + /* + * We need to exclude the super stripes now so that the space info has + * super bytes accounted for, otherwise we'll think we have more space + * than we actually do. + */ + ret = exclude_super_stripes(cache); + if (ret) { + /* We may have excluded something, so call this just in case. */ + btrfs_free_excluded_extents(cache); + goto error; + } + + /* + * For zoned filesystem, space after the allocation offset is the only + * free space for a block group. So, we don't need any caching work. + * btrfs_calc_zone_unusable() will set the amount of free space and + * zone_unusable space. + * + * For regular filesystem, check for two cases, either we are full, and + * therefore don't need to bother with the caching work since we won't + * find any space, or we are empty, and we can just add all the space + * in and be done with it. This saves us _a_lot_ of time, particularly + * in the full case. + */ + if (btrfs_is_zoned(info)) { + btrfs_calc_zone_unusable(cache); + /* Should not have any excluded extents. Just in case, though. */ + btrfs_free_excluded_extents(cache); + } else if (cache->length == cache->used) { + cache->cached = BTRFS_CACHE_FINISHED; + btrfs_free_excluded_extents(cache); + } else if (cache->used == 0 && cache->remap_bytes == 0) { + cache->cached = BTRFS_CACHE_FINISHED; + ret = btrfs_add_new_free_space(cache, cache->start, + btrfs_block_group_end(cache), NULL); + btrfs_free_excluded_extents(cache); + if (ret) + goto error; + } + + ret = btrfs_add_block_group_cache(cache); + if (ret) { + btrfs_remove_free_space_cache(cache); + goto error; + } + + trace_btrfs_add_block_group(info, cache, 0); + btrfs_add_bg_to_space_info(info, cache); + + set_avail_alloc_bits(info, cache->flags); + if (btrfs_chunk_writeable(info, cache->start)) { + if (cache->used == 0 && cache->remap_bytes == 0) { + ASSERT(list_empty(&cache->bg_list)); + if (btrfs_test_opt(info, DISCARD_ASYNC)) + btrfs_discard_queue_work(&info->discard_ctl, cache); + else + btrfs_mark_bg_unused(cache); + } + } else { + inc_block_group_ro(cache, true); + } + + return 0; +error: + btrfs_put_block_group(cache); + return ret; +} + +static int fill_dummy_bgs(struct btrfs_fs_info *fs_info) +{ + struct rb_node *node; + int ret = 0; + + for (node = rb_first_cached(&fs_info->mapping_tree); node; node = rb_next(node)) { + struct btrfs_chunk_map *map; + struct btrfs_block_group *bg; + + map = rb_entry(node, struct btrfs_chunk_map, rb_node); + bg = btrfs_create_block_group(fs_info, map->start); + if (!bg) { + ret = -ENOMEM; + break; + } + + /* Fill dummy cache as FULL */ + bg->length = map->chunk_len; + bg->flags = map->on_disk_type; + bg->cached = BTRFS_CACHE_FINISHED; + bg->used = map->chunk_len; + bg->space_info = btrfs_find_space_info(fs_info, bg->flags); + ret = btrfs_add_block_group_cache(bg); + /* + * We may have some valid block group cache added already, in + * that case we skip to the next one. + */ + if (ret == -EEXIST) { + ret = 0; + btrfs_put_block_group(bg); + continue; + } + + if (ret) { + btrfs_remove_free_space_cache(bg); + btrfs_put_block_group(bg); + break; + } + + btrfs_add_bg_to_space_info(fs_info, bg); + + set_avail_alloc_bits(fs_info, bg->flags); + } + if (!ret) + btrfs_init_global_block_rsv(fs_info); + return ret; +} + +int btrfs_read_block_groups(struct btrfs_fs_info *info) +{ + struct btrfs_root *root = btrfs_block_group_root(info); + struct btrfs_path *path; + int ret; + struct btrfs_block_group *cache; + struct btrfs_space_info *space_info; + struct btrfs_key key; + bool need_clear = false; + u64 cache_gen; + + /* + * Either no extent root (with ibadroots rescue option) or we have + * unsupported RO options. The fs can never be mounted read-write, so no + * need to waste time searching block group items. + * + * This also allows new extent tree related changes to be RO compat, + * no need for a full incompat flag. + */ + if (!root || (btrfs_super_compat_ro_flags(info->super_copy) & + ~BTRFS_FEATURE_COMPAT_RO_SUPP)) + return fill_dummy_bgs(info); + + key.objectid = 0; + key.type = BTRFS_BLOCK_GROUP_ITEM_KEY; + key.offset = 0; + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + cache_gen = btrfs_super_cache_generation(info->super_copy); + if (btrfs_test_opt(info, SPACE_CACHE) && + btrfs_super_generation(info->super_copy) != cache_gen) + need_clear = true; + if (btrfs_test_opt(info, CLEAR_CACHE)) + need_clear = true; + + while (1) { + struct btrfs_block_group_item_v2 bgi; + struct extent_buffer *leaf; + int slot; + size_t size; + + ret = find_first_block_group(info, path, &key); + if (ret > 0) + break; + if (ret != 0) + goto error; + + leaf = path->nodes[0]; + slot = path->slots[0]; + + if (btrfs_fs_incompat(info, REMAP_TREE)) { + size = sizeof(struct btrfs_block_group_item_v2); + } else { + size = sizeof(struct btrfs_block_group_item); + btrfs_set_stack_block_group_v2_remap_bytes(&bgi, 0); + btrfs_set_stack_block_group_v2_identity_remap_count(&bgi, 0); + } + + read_extent_buffer(leaf, &bgi, btrfs_item_ptr_offset(leaf, slot), + size); + + btrfs_item_key_to_cpu(leaf, &key, slot); + btrfs_release_path(path); + ret = read_one_block_group(info, &bgi, &key, need_clear); + if (ret < 0) + goto error; + key.objectid += key.offset; + key.offset = 0; + } + btrfs_release_path(path); + + list_for_each_entry(space_info, &info->space_info, list) { + int i; + + for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) { + if (list_empty(&space_info->block_groups[i])) + continue; + cache = list_first_entry(&space_info->block_groups[i], + struct btrfs_block_group, + list); + btrfs_sysfs_add_block_group_type(cache); + } + + if (!(btrfs_get_alloc_profile(info, space_info->flags) & + (BTRFS_BLOCK_GROUP_RAID10 | + BTRFS_BLOCK_GROUP_RAID1_MASK | + BTRFS_BLOCK_GROUP_RAID56_MASK | + BTRFS_BLOCK_GROUP_DUP))) + continue; + /* + * Avoid allocating from un-mirrored block group if there are + * mirrored block groups. + */ + list_for_each_entry(cache, + &space_info->block_groups[BTRFS_RAID_RAID0], + list) + inc_block_group_ro(cache, true); + list_for_each_entry(cache, + &space_info->block_groups[BTRFS_RAID_SINGLE], + list) + inc_block_group_ro(cache, true); + } + + btrfs_init_global_block_rsv(info); + ret = check_chunk_block_group_mappings(info); +error: + btrfs_free_path(path); + /* + * We've hit some error while reading the extent tree, and have + * rescue=ibadroots mount option. + * Try to fill the tree using dummy block groups so that the user can + * continue to mount and grab their data. + */ + if (ret && btrfs_test_opt(info, IGNOREBADROOTS)) + ret = fill_dummy_bgs(info); + return ret; +} + +/* + * This function, insert_block_group_item(), belongs to the phase 2 of chunk + * allocation. + * + * See the comment at btrfs_chunk_alloc() for details about the chunk allocation + * phases. + */ +static int insert_block_group_item(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_block_group_item_v2 bgi; + struct btrfs_root *root = btrfs_block_group_root(fs_info); + struct btrfs_key key; + u64 old_last_used; + size_t size; + int ret; + + if (unlikely(!root)) { + btrfs_err(fs_info, "missing block group root"); + return -EUCLEAN; + } + + spin_lock(&block_group->lock); + btrfs_set_stack_block_group_v2_used(&bgi, block_group->used); + btrfs_set_stack_block_group_v2_chunk_objectid(&bgi, block_group->global_root_id); + btrfs_set_stack_block_group_v2_flags(&bgi, block_group->flags); + btrfs_set_stack_block_group_v2_remap_bytes(&bgi, block_group->remap_bytes); + btrfs_set_stack_block_group_v2_identity_remap_count(&bgi, block_group->identity_remap_count); + old_last_used = block_group->last_used; + block_group->last_used = block_group->used; + block_group->last_remap_bytes = block_group->remap_bytes; + block_group->last_identity_remap_count = block_group->identity_remap_count; + block_group->last_flags = block_group->flags; + key.objectid = block_group->start; + key.type = BTRFS_BLOCK_GROUP_ITEM_KEY; + key.offset = block_group->length; + spin_unlock(&block_group->lock); + + if (btrfs_fs_incompat(fs_info, REMAP_TREE)) + size = sizeof(struct btrfs_block_group_item_v2); + else + size = sizeof(struct btrfs_block_group_item); + + ret = btrfs_insert_item(trans, root, &key, &bgi, size); + if (ret < 0) { + spin_lock(&block_group->lock); + block_group->last_used = old_last_used; + spin_unlock(&block_group->lock); + } + + return ret; +} + +static int insert_dev_extent(struct btrfs_trans_handle *trans, + const struct btrfs_device *device, u64 chunk_offset, + u64 start, u64 num_bytes) +{ + struct btrfs_fs_info *fs_info = device->fs_info; + struct btrfs_root *root = fs_info->dev_root; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_dev_extent *extent; + struct extent_buffer *leaf; + struct btrfs_key key; + int ret; + + WARN_ON(!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state)); + WARN_ON(test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)); + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = device->devid; + key.type = BTRFS_DEV_EXTENT_KEY; + key.offset = start; + ret = btrfs_insert_empty_item(trans, root, path, &key, sizeof(*extent)); + if (ret) + return ret; + + leaf = path->nodes[0]; + extent = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_extent); + btrfs_set_dev_extent_chunk_tree(leaf, extent, BTRFS_CHUNK_TREE_OBJECTID); + btrfs_set_dev_extent_chunk_objectid(leaf, extent, + BTRFS_FIRST_CHUNK_TREE_OBJECTID); + btrfs_set_dev_extent_chunk_offset(leaf, extent, chunk_offset); + btrfs_set_dev_extent_length(leaf, extent, num_bytes); + + return ret; +} + +/* + * This function belongs to phase 2. + * + * See the comment at btrfs_chunk_alloc() for details about the chunk allocation + * phases. + */ +static int insert_dev_extents(struct btrfs_trans_handle *trans, + u64 chunk_offset, u64 chunk_size) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_device *device; + struct btrfs_chunk_map *map; + u64 dev_offset; + int i; + int ret = 0; + + map = btrfs_get_chunk_map(fs_info, chunk_offset, chunk_size); + if (IS_ERR(map)) + return PTR_ERR(map); + + /* + * Take the device list mutex to prevent races with the final phase of + * a device replace operation that replaces the device object associated + * with the map's stripes, because the device object's id can change + * at any time during that final phase of the device replace operation + * (dev-replace.c:btrfs_dev_replace_finishing()), so we could grab the + * replaced device and then see it with an ID of BTRFS_DEV_REPLACE_DEVID, + * resulting in persisting a device extent item with such ID. + */ + mutex_lock(&fs_info->fs_devices->device_list_mutex); + for (i = 0; i < map->num_stripes; i++) { + device = map->stripes[i].dev; + dev_offset = map->stripes[i].physical; + + ret = insert_dev_extent(trans, device, chunk_offset, dev_offset, + map->stripe_size); + if (ret) + break; + } + mutex_unlock(&fs_info->fs_devices->device_list_mutex); + + btrfs_free_chunk_map(map); + return ret; +} + +/* + * This function, btrfs_create_pending_block_groups(), belongs to the phase 2 of + * chunk allocation. + * + * See the comment at btrfs_chunk_alloc() for details about the chunk allocation + * phases. + */ +void btrfs_create_pending_block_groups(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_block_group *block_group; + int ret = 0; + + while (!list_empty(&trans->new_bgs)) { + int index; + + block_group = list_first_entry(&trans->new_bgs, + struct btrfs_block_group, + bg_list); + if (ret) + goto next; + + index = btrfs_bg_flags_to_raid_index(block_group->flags); + + ret = insert_block_group_item(trans, block_group); + if (ret) + btrfs_abort_transaction(trans, ret); + if (!test_bit(BLOCK_GROUP_FLAG_CHUNK_ITEM_INSERTED, + &block_group->runtime_flags)) { + mutex_lock(&fs_info->chunk_mutex); + ret = btrfs_chunk_alloc_add_chunk_item(trans, block_group); + mutex_unlock(&fs_info->chunk_mutex); + if (ret) + btrfs_abort_transaction(trans, ret); + } + ret = insert_dev_extents(trans, block_group->start, + block_group->length); + if (ret) + btrfs_abort_transaction(trans, ret); + btrfs_add_block_group_free_space(trans, block_group); + + /* + * If we restriped during balance, we may have added a new raid + * type, so now add the sysfs entries when it is safe to do so. + * We don't have to worry about locking here as it's handled in + * btrfs_sysfs_add_block_group_type. + */ + if (block_group->space_info->block_group_kobjs[index] == NULL) + btrfs_sysfs_add_block_group_type(block_group); + + /* Already aborted the transaction if it failed. */ +next: + btrfs_dec_delayed_refs_rsv_bg_inserts(fs_info); + + spin_lock(&fs_info->unused_bgs_lock); + list_del_init(&block_group->bg_list); + clear_bit(BLOCK_GROUP_FLAG_NEW, &block_group->runtime_flags); + btrfs_put_block_group(block_group); + spin_unlock(&fs_info->unused_bgs_lock); + + /* + * If the block group is still unused, add it to the list of + * unused block groups. The block group may have been created in + * order to satisfy a space reservation, in which case the + * extent allocation only happens later. But often we don't + * actually need to allocate space that we previously reserved, + * so the block group may become unused for a long time. For + * example for metadata we generally reserve space for a worst + * possible scenario, but then don't end up allocating all that + * space or none at all (due to no need to COW, extent buffers + * were already COWed in the current transaction and still + * unwritten, tree heights lower than the maximum possible + * height, etc). For data we generally reserve the exact amount + * of space we are going to allocate later, the exception is + * when using compression, as we must reserve space based on the + * uncompressed data size, because the compression is only done + * when writeback triggered and we don't know how much space we + * are actually going to need, so we reserve the uncompressed + * size because the data may be incompressible in the worst case. + */ + if (ret == 0) { + bool used; + + spin_lock(&block_group->lock); + used = btrfs_is_block_group_used(block_group); + spin_unlock(&block_group->lock); + + if (!used) + btrfs_mark_bg_unused(block_group); + } + } + btrfs_trans_release_chunk_metadata(trans); +} + +/* + * For extent tree v2 we use the block_group_item->chunk_offset to point at our + * global root id. For v1 it's always set to BTRFS_FIRST_CHUNK_TREE_OBJECTID. + */ +static u64 calculate_global_root_id(const struct btrfs_fs_info *fs_info, u64 offset) +{ + u64 div = SZ_1G; + u64 index; + + if (!btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) + return BTRFS_FIRST_CHUNK_TREE_OBJECTID; + + /* If we have a smaller fs index based on 128MiB. */ + if (btrfs_super_total_bytes(fs_info->super_copy) <= (SZ_1G * 10ULL)) + div = SZ_128M; + + offset = div64_u64(offset, div); + div64_u64_rem(offset, fs_info->nr_global_roots, &index); + return index; +} + +struct btrfs_block_group *btrfs_make_block_group(struct btrfs_trans_handle *trans, + struct btrfs_space_info *space_info, + u64 type, u64 chunk_offset, u64 size) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_block_group *cache; + int ret; + + btrfs_set_log_full_commit(trans); + + cache = btrfs_create_block_group(fs_info, chunk_offset); + if (!cache) + return ERR_PTR(-ENOMEM); + + /* + * Mark it as new before adding it to the rbtree of block groups or any + * list, so that no other task finds it and calls btrfs_mark_bg_unused() + * before the new flag is set. + */ + set_bit(BLOCK_GROUP_FLAG_NEW, &cache->runtime_flags); + + cache->length = size; + btrfs_set_free_space_tree_thresholds(cache); + cache->flags = type; + cache->cached = BTRFS_CACHE_FINISHED; + cache->global_root_id = calculate_global_root_id(fs_info, cache->start); + + if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) + set_bit(BLOCK_GROUP_FLAG_NEEDS_FREE_SPACE, &cache->runtime_flags); + + ret = btrfs_load_block_group_zone_info(cache, true); + if (ret) { + btrfs_put_block_group(cache); + return ERR_PTR(ret); + } + + ret = exclude_super_stripes(cache); + if (ret) { + /* We may have excluded something, so call this just in case */ + btrfs_free_excluded_extents(cache); + btrfs_put_block_group(cache); + return ERR_PTR(ret); + } + + ret = btrfs_add_new_free_space(cache, chunk_offset, chunk_offset + size, NULL); + btrfs_free_excluded_extents(cache); + if (ret) { + btrfs_put_block_group(cache); + return ERR_PTR(ret); + } + + /* + * Ensure the corresponding space_info object is created and + * assigned to our block group. We want our bg to be added to the rbtree + * with its ->space_info set. + */ + cache->space_info = space_info; + ASSERT(cache->space_info); + + ret = btrfs_add_block_group_cache(cache); + if (ret) { + btrfs_remove_free_space_cache(cache); + btrfs_put_block_group(cache); + return ERR_PTR(ret); + } + + /* + * Now that our block group has its ->space_info set and is inserted in + * the rbtree, update the space info's counters. + */ + trace_btrfs_add_block_group(fs_info, cache, 1); + btrfs_add_bg_to_space_info(fs_info, cache); + btrfs_update_global_block_rsv(fs_info); + +#ifdef CONFIG_BTRFS_DEBUG + if (btrfs_should_fragment_free_space(cache)) { + cache->space_info->bytes_used += size >> 1; + fragment_free_space(cache); + } +#endif + + btrfs_link_bg_list(cache, &trans->new_bgs); + btrfs_inc_delayed_refs_rsv_bg_inserts(fs_info); + + set_avail_alloc_bits(fs_info, type); + return cache; +} + +/* + * Mark one block group RO, can be called several times for the same block + * group. + * + * @cache: the destination block group + * @do_chunk_alloc: whether need to do chunk pre-allocation, this is to + * ensure we still have some free space after marking this + * block group RO. + */ +int btrfs_inc_block_group_ro(struct btrfs_block_group *cache, + bool do_chunk_alloc) +{ + struct btrfs_fs_info *fs_info = cache->fs_info; + struct btrfs_space_info *space_info = cache->space_info; + struct btrfs_trans_handle *trans; + struct btrfs_root *root = btrfs_block_group_root(fs_info); + u64 alloc_flags; + int ret; + bool dirty_bg_running; + + if (unlikely(!root)) { + btrfs_err(fs_info, "missing block group root"); + return -EUCLEAN; + } + + /* + * This can only happen when we are doing read-only scrub on read-only + * mount. + * In that case we should not start a new transaction on read-only fs. + * Thus here we skip all chunk allocations. + */ + if (sb_rdonly(fs_info->sb)) { + mutex_lock(&fs_info->ro_block_group_mutex); + ret = inc_block_group_ro(cache, false); + mutex_unlock(&fs_info->ro_block_group_mutex); + return ret; + } + + do { + trans = btrfs_join_transaction(root); + if (IS_ERR(trans)) + return PTR_ERR(trans); + + dirty_bg_running = false; + + /* + * We're not allowed to set block groups readonly after the dirty + * block group cache has started writing. If it already started, + * back off and let this transaction commit. + */ + mutex_lock(&fs_info->ro_block_group_mutex); + if (test_bit(BTRFS_TRANS_DIRTY_BG_RUN, &trans->transaction->flags)) { + u64 transid = trans->transid; + + mutex_unlock(&fs_info->ro_block_group_mutex); + btrfs_end_transaction(trans); + + ret = btrfs_wait_for_commit(fs_info, transid); + if (ret) + return ret; + dirty_bg_running = true; + } + } while (dirty_bg_running); + + if (do_chunk_alloc) { + /* + * If we are changing raid levels, try to allocate a + * corresponding block group with the new raid level. + */ + alloc_flags = btrfs_get_alloc_profile(fs_info, cache->flags); + if (alloc_flags != cache->flags) { + ret = btrfs_chunk_alloc(trans, space_info, alloc_flags, + CHUNK_ALLOC_FORCE); + /* + * ENOSPC is allowed here, we may have enough space + * already allocated at the new raid level to carry on + */ + if (ret == -ENOSPC) + ret = 0; + if (ret < 0) + goto out; + } + } + + ret = inc_block_group_ro(cache, false); + if (!ret) + goto out; + if (ret == -ETXTBSY) + goto unlock_out; + + /* + * Skip chunk allocation if the bg is SYSTEM, this is to avoid system + * chunk allocation storm to exhaust the system chunk array. Otherwise + * we still want to try our best to mark the block group read-only. + */ + if (!do_chunk_alloc && ret == -ENOSPC && + (cache->flags & BTRFS_BLOCK_GROUP_SYSTEM)) + goto unlock_out; + + alloc_flags = btrfs_get_alloc_profile(fs_info, space_info->flags); + ret = btrfs_chunk_alloc(trans, space_info, alloc_flags, CHUNK_ALLOC_FORCE); + if (ret < 0) + goto out; + /* + * We have allocated a new chunk. We also need to activate that chunk to + * grant metadata tickets for zoned filesystem. + */ + ret = btrfs_zoned_activate_one_bg(space_info, true); + if (ret < 0) + goto out; + + ret = inc_block_group_ro(cache, false); + if (ret == -ETXTBSY) + goto unlock_out; +out: + if (cache->flags & BTRFS_BLOCK_GROUP_SYSTEM) { + alloc_flags = btrfs_get_alloc_profile(fs_info, cache->flags); + mutex_lock(&fs_info->chunk_mutex); + check_system_chunk(trans, alloc_flags); + mutex_unlock(&fs_info->chunk_mutex); + } +unlock_out: + mutex_unlock(&fs_info->ro_block_group_mutex); + + btrfs_end_transaction(trans); + return ret; +} + +void btrfs_dec_block_group_ro(struct btrfs_block_group *cache) +{ + struct btrfs_space_info *sinfo = cache->space_info; + + BUG_ON(!cache->ro); + + spin_lock(&sinfo->lock); + spin_lock(&cache->lock); + if (!--cache->ro) { + if (btrfs_is_zoned(cache->fs_info)) { + /* Migrate zone_unusable bytes back */ + cache->zone_unusable = + (cache->alloc_offset - cache->used - cache->pinned - + cache->reserved) + + (cache->length - cache->zone_capacity); + btrfs_space_info_update_bytes_zone_unusable(sinfo, cache->zone_unusable); + sinfo->bytes_readonly -= cache->zone_unusable; + } + sinfo->bytes_readonly -= btrfs_block_group_available_space(cache); + list_del_init(&cache->ro_list); + } + spin_unlock(&cache->lock); + spin_unlock(&sinfo->lock); +} + +static int update_block_group_item(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + struct btrfs_block_group *cache) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + int ret; + struct btrfs_root *root = btrfs_block_group_root(fs_info); + unsigned long bi; + struct extent_buffer *leaf; + struct btrfs_block_group_item_v2 bgi; + struct btrfs_key key; + u64 old_last_used, old_last_remap_bytes; + u32 old_last_identity_remap_count; + u64 used, remap_bytes; + u32 identity_remap_count; + + if (unlikely(!root)) { + btrfs_err(fs_info, "missing block group root"); + return -EUCLEAN; + } + + /* + * Block group items update can be triggered out of commit transaction + * critical section, thus we need a consistent view of used bytes. + * We cannot use cache->used directly outside of the spin lock, as it + * may be changed. + */ + spin_lock(&cache->lock); + old_last_used = cache->last_used; + old_last_remap_bytes = cache->last_remap_bytes; + old_last_identity_remap_count = cache->last_identity_remap_count; + used = cache->used; + remap_bytes = cache->remap_bytes; + identity_remap_count = cache->identity_remap_count; + /* No change in values, can safely skip it. */ + if (cache->last_used == used && + cache->last_remap_bytes == remap_bytes && + cache->last_identity_remap_count == identity_remap_count && + cache->last_flags == cache->flags) { + spin_unlock(&cache->lock); + return 0; + } + cache->last_used = used; + cache->last_remap_bytes = remap_bytes; + cache->last_identity_remap_count = identity_remap_count; + cache->last_flags = cache->flags; + spin_unlock(&cache->lock); + + key.objectid = cache->start; + key.type = BTRFS_BLOCK_GROUP_ITEM_KEY; + key.offset = cache->length; + + ret = btrfs_search_slot(trans, root, &key, path, 0, 1); + if (ret) { + if (ret > 0) + ret = -ENOENT; + goto fail; + } + + leaf = path->nodes[0]; + bi = btrfs_item_ptr_offset(leaf, path->slots[0]); + btrfs_set_stack_block_group_v2_used(&bgi, used); + btrfs_set_stack_block_group_v2_chunk_objectid(&bgi, cache->global_root_id); + btrfs_set_stack_block_group_v2_flags(&bgi, cache->flags); + + if (btrfs_fs_incompat(fs_info, REMAP_TREE)) { + btrfs_set_stack_block_group_v2_remap_bytes(&bgi, cache->remap_bytes); + btrfs_set_stack_block_group_v2_identity_remap_count(&bgi, + cache->identity_remap_count); + write_extent_buffer(leaf, &bgi, bi, + sizeof(struct btrfs_block_group_item_v2)); + } else { + write_extent_buffer(leaf, &bgi, bi, + sizeof(struct btrfs_block_group_item)); + } + +fail: + btrfs_release_path(path); + /* + * We didn't update the block group item, need to revert last_used + * unless the block group item didn't exist yet - this is to prevent a + * race with a concurrent insertion of the block group item, with + * insert_block_group_item(), that happened just after we attempted to + * update. In that case we would reset last_used to 0 just after the + * insertion set it to a value greater than 0 - if the block group later + * becomes with 0 used bytes, we would incorrectly skip its update. + */ + if (ret < 0 && ret != -ENOENT) { + spin_lock(&cache->lock); + cache->last_used = old_last_used; + cache->last_remap_bytes = old_last_remap_bytes; + cache->last_identity_remap_count = old_last_identity_remap_count; + spin_unlock(&cache->lock); + } + return ret; + +} + +static void cache_save_setup(struct btrfs_block_group *block_group, + struct btrfs_trans_handle *trans, + struct btrfs_path *path) +{ + struct btrfs_fs_info *fs_info = block_group->fs_info; + struct inode *inode = NULL; + struct extent_changeset *data_reserved = NULL; + u64 alloc_hint = 0; + int dcs = BTRFS_DC_ERROR; + u64 cache_size = 0; + int retries = 0; + int ret = 0; + + if (!btrfs_test_opt(fs_info, SPACE_CACHE)) + return; + + /* + * If this block group is smaller than 100 megs don't bother caching the + * block group. + */ + if (block_group->length < (100 * SZ_1M)) { + spin_lock(&block_group->lock); + block_group->disk_cache_state = BTRFS_DC_WRITTEN; + spin_unlock(&block_group->lock); + return; + } + + if (TRANS_ABORTED(trans)) + return; +again: + inode = lookup_free_space_inode(block_group, path); + if (IS_ERR(inode) && PTR_ERR(inode) != -ENOENT) { + ret = PTR_ERR(inode); + btrfs_release_path(path); + goto out; + } + + if (IS_ERR(inode)) { + if (retries) { + ret = PTR_ERR(inode); + btrfs_err(fs_info, + "failed to lookup free space inode after creation for block group %llu: %d", + block_group->start, ret); + goto out_free; + } + retries++; + + if (block_group->ro) + goto out_free; + + ret = create_free_space_inode(trans, block_group, path); + if (ret) + goto out_free; + goto again; + } + + /* + * We want to set the generation to 0, that way if anything goes wrong + * from here on out we know not to trust this cache when we load up next + * time. + */ + BTRFS_I(inode)->generation = 0; + ret = btrfs_update_inode(trans, BTRFS_I(inode)); + if (unlikely(ret)) { + /* + * So theoretically we could recover from this, simply set the + * super cache generation to 0 so we know to invalidate the + * cache, but then we'd have to keep track of the block groups + * that fail this way so we know we _have_ to reset this cache + * before the next commit or risk reading stale cache. So to + * limit our exposure to horrible edge cases lets just abort the + * transaction, this only happens in really bad situations + * anyway. + */ + btrfs_abort_transaction(trans, ret); + goto out_put; + } + + /* We've already setup this transaction, go ahead and exit */ + if (block_group->cache_generation == trans->transid && + i_size_read(inode)) { + dcs = BTRFS_DC_SETUP; + goto out_put; + } + + if (i_size_read(inode) > 0) { + ret = btrfs_check_trunc_cache_free_space(fs_info, + &fs_info->global_block_rsv); + if (ret) + goto out_put; + + ret = btrfs_truncate_free_space_cache(trans, NULL, inode); + if (ret) + goto out_put; + } + + spin_lock(&block_group->lock); + if (block_group->cached != BTRFS_CACHE_FINISHED || + !btrfs_test_opt(fs_info, SPACE_CACHE)) { + /* + * don't bother trying to write stuff out _if_ + * a) we're not cached, + * b) we're with nospace_cache mount option, + * c) we're with v2 space_cache (FREE_SPACE_TREE). + */ + dcs = BTRFS_DC_WRITTEN; + spin_unlock(&block_group->lock); + goto out_put; + } + spin_unlock(&block_group->lock); + + /* + * We hit an ENOSPC when setting up the cache in this transaction, just + * skip doing the setup, we've already cleared the cache so we're safe. + */ + if (test_bit(BTRFS_TRANS_CACHE_ENOSPC, &trans->transaction->flags)) + goto out_put; + + /* + * Try to preallocate enough space based on how big the block group is. + * Keep in mind this has to include any pinned space which could end up + * taking up quite a bit since it's not folded into the other space + * cache. + */ + cache_size = div_u64(block_group->length, SZ_256M); + if (!cache_size) + cache_size = 1; + + cache_size *= 16; + cache_size *= fs_info->sectorsize; + + ret = btrfs_check_data_free_space(BTRFS_I(inode), &data_reserved, 0, + cache_size, false); + if (ret) + goto out_put; + + ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, cache_size, + cache_size, cache_size, + &alloc_hint); + /* + * Our cache requires contiguous chunks so that we don't modify a bunch + * of metadata or split extents when writing the cache out, which means + * we can enospc if we are heavily fragmented in addition to just normal + * out of space conditions. So if we hit this just skip setting up any + * other block groups for this transaction, maybe we'll unpin enough + * space the next time around. + */ + if (!ret) + dcs = BTRFS_DC_SETUP; + else if (ret == -ENOSPC) + set_bit(BTRFS_TRANS_CACHE_ENOSPC, &trans->transaction->flags); + +out_put: + iput(inode); +out_free: + btrfs_release_path(path); +out: + spin_lock(&block_group->lock); + if (!ret && dcs == BTRFS_DC_SETUP) + block_group->cache_generation = trans->transid; + block_group->disk_cache_state = dcs; + spin_unlock(&block_group->lock); + + extent_changeset_free(data_reserved); +} + +int btrfs_setup_space_cache(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_block_group *cache, *tmp; + struct btrfs_transaction *cur_trans = trans->transaction; + BTRFS_PATH_AUTO_FREE(path); + + if (list_empty(&cur_trans->dirty_bgs) || + !btrfs_test_opt(fs_info, SPACE_CACHE)) + return 0; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + /* Could add new block groups, use _safe just in case */ + list_for_each_entry_safe(cache, tmp, &cur_trans->dirty_bgs, + dirty_list) { + if (cache->disk_cache_state == BTRFS_DC_CLEAR) + cache_save_setup(cache, trans, path); + } + + return 0; +} + +/* + * Transaction commit does final block group cache writeback during a critical + * section where nothing is allowed to change the FS. This is required in + * order for the cache to actually match the block group, but can introduce a + * lot of latency into the commit. + * + * So, btrfs_start_dirty_block_groups is here to kick off block group cache IO. + * There's a chance we'll have to redo some of it if the block group changes + * again during the commit, but it greatly reduces the commit latency by + * getting rid of the easy block groups while we're still allowing others to + * join the commit. + */ +int btrfs_start_dirty_block_groups(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_block_group *cache; + struct btrfs_transaction *cur_trans = trans->transaction; + int ret = 0; + int should_put; + BTRFS_PATH_AUTO_FREE(path); + LIST_HEAD(dirty); + struct list_head *io = &cur_trans->io_bgs; + int loops = 0; + + spin_lock(&cur_trans->dirty_bgs_lock); + if (list_empty(&cur_trans->dirty_bgs)) { + spin_unlock(&cur_trans->dirty_bgs_lock); + return 0; + } + list_splice_init(&cur_trans->dirty_bgs, &dirty); + spin_unlock(&cur_trans->dirty_bgs_lock); + +again: + /* Make sure all the block groups on our dirty list actually exist */ + btrfs_create_pending_block_groups(trans); + + if (!path) { + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto out; + } + } + + /* + * cache_write_mutex is here only to save us from balance or automatic + * removal of empty block groups deleting this block group while we are + * writing out the cache + */ + mutex_lock(&trans->transaction->cache_write_mutex); + while (!list_empty(&dirty)) { + bool drop_reserve = true; + + cache = list_first_entry(&dirty, struct btrfs_block_group, + dirty_list); + /* + * This can happen if something re-dirties a block group that + * is already under IO. Just wait for it to finish and then do + * it all again + */ + if (!list_empty(&cache->io_list)) { + list_del_init(&cache->io_list); + btrfs_wait_cache_io(trans, cache, path); + btrfs_put_block_group(cache); + } + + + /* + * btrfs_wait_cache_io uses the cache->dirty_list to decide if + * it should update the cache_state. Don't delete until after + * we wait. + * + * Since we're not running in the commit critical section + * we need the dirty_bgs_lock to protect from update_block_group + */ + spin_lock(&cur_trans->dirty_bgs_lock); + list_del_init(&cache->dirty_list); + spin_unlock(&cur_trans->dirty_bgs_lock); + + should_put = 1; + + cache_save_setup(cache, trans, path); + + if (cache->disk_cache_state == BTRFS_DC_SETUP) { + cache->io_ctl.inode = NULL; + ret = btrfs_write_out_cache(trans, cache, path); + if (ret == 0 && cache->io_ctl.inode) { + should_put = 0; + + /* + * The cache_write_mutex is protecting the + * io_list, also refer to the definition of + * btrfs_transaction::io_bgs for more details + */ + list_add_tail(&cache->io_list, io); + } else { + /* + * If we failed to write the cache, the + * generation will be bad and life goes on + */ + ret = 0; + } + } + if (!ret) { + ret = update_block_group_item(trans, path, cache); + /* + * Our block group might still be attached to the list + * of new block groups in the transaction handle of some + * other task (struct btrfs_trans_handle->new_bgs). This + * means its block group item isn't yet in the extent + * tree. If this happens ignore the error, as we will + * try again later in the critical section of the + * transaction commit. + */ + if (ret == -ENOENT) { + ret = 0; + spin_lock(&cur_trans->dirty_bgs_lock); + if (list_empty(&cache->dirty_list)) { + list_add_tail(&cache->dirty_list, + &cur_trans->dirty_bgs); + btrfs_get_block_group(cache); + drop_reserve = false; + } + spin_unlock(&cur_trans->dirty_bgs_lock); + } else if (ret) { + btrfs_abort_transaction(trans, ret); + } + } + + /* If it's not on the io list, we need to put the block group */ + if (should_put) + btrfs_put_block_group(cache); + if (drop_reserve) + btrfs_dec_delayed_refs_rsv_bg_updates(fs_info); + /* + * Avoid blocking other tasks for too long. It might even save + * us from writing caches for block groups that are going to be + * removed. + */ + mutex_unlock(&trans->transaction->cache_write_mutex); + if (ret) + goto out; + mutex_lock(&trans->transaction->cache_write_mutex); + } + mutex_unlock(&trans->transaction->cache_write_mutex); + + /* + * Go through delayed refs for all the stuff we've just kicked off + * and then loop back (just once) + */ + if (!ret) + ret = btrfs_run_delayed_refs(trans, 0); + if (!ret && loops == 0) { + loops++; + spin_lock(&cur_trans->dirty_bgs_lock); + list_splice_init(&cur_trans->dirty_bgs, &dirty); + /* + * dirty_bgs_lock protects us from concurrent block group + * deletes too (not just cache_write_mutex). + */ + if (!list_empty(&dirty)) { + spin_unlock(&cur_trans->dirty_bgs_lock); + goto again; + } + spin_unlock(&cur_trans->dirty_bgs_lock); + } +out: + if (ret < 0) { + spin_lock(&cur_trans->dirty_bgs_lock); + list_splice_init(&dirty, &cur_trans->dirty_bgs); + spin_unlock(&cur_trans->dirty_bgs_lock); + btrfs_cleanup_dirty_bgs(cur_trans, fs_info); + } + + return ret; +} + +int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_block_group *cache; + struct btrfs_transaction *cur_trans = trans->transaction; + int ret = 0; + int should_put; + BTRFS_PATH_AUTO_FREE(path); + struct list_head *io = &cur_trans->io_bgs; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + /* + * Even though we are in the critical section of the transaction commit, + * we can still have concurrent tasks adding elements to this + * transaction's list of dirty block groups. These tasks correspond to + * endio free space workers started when writeback finishes for a + * space cache, which run inode.c:btrfs_finish_ordered_io(), and can + * allocate new block groups as a result of COWing nodes of the root + * tree when updating the free space inode. The writeback for the space + * caches is triggered by an earlier call to + * btrfs_start_dirty_block_groups() and iterations of the following + * loop. + * Also we want to do the cache_save_setup first and then run the + * delayed refs to make sure we have the best chance at doing this all + * in one shot. + */ + spin_lock(&cur_trans->dirty_bgs_lock); + while (!list_empty(&cur_trans->dirty_bgs)) { + cache = list_first_entry(&cur_trans->dirty_bgs, + struct btrfs_block_group, + dirty_list); + + /* + * This can happen if cache_save_setup re-dirties a block group + * that is already under IO. Just wait for it to finish and + * then do it all again + */ + if (!list_empty(&cache->io_list)) { + spin_unlock(&cur_trans->dirty_bgs_lock); + list_del_init(&cache->io_list); + btrfs_wait_cache_io(trans, cache, path); + btrfs_put_block_group(cache); + spin_lock(&cur_trans->dirty_bgs_lock); + } + + /* + * Don't remove from the dirty list until after we've waited on + * any pending IO + */ + list_del_init(&cache->dirty_list); + spin_unlock(&cur_trans->dirty_bgs_lock); + should_put = 1; + + cache_save_setup(cache, trans, path); + + if (!ret) + ret = btrfs_run_delayed_refs(trans, U64_MAX); + + if (!ret && cache->disk_cache_state == BTRFS_DC_SETUP) { + cache->io_ctl.inode = NULL; + ret = btrfs_write_out_cache(trans, cache, path); + if (ret == 0 && cache->io_ctl.inode) { + should_put = 0; + list_add_tail(&cache->io_list, io); + } else { + /* + * If we failed to write the cache, the + * generation will be bad and life goes on + */ + ret = 0; + } + } + if (!ret) { + ret = update_block_group_item(trans, path, cache); + /* + * One of the free space endio workers might have + * created a new block group while updating a free space + * cache's inode (at inode.c:btrfs_finish_ordered_io()) + * and hasn't released its transaction handle yet, in + * which case the new block group is still attached to + * its transaction handle and its creation has not + * finished yet (no block group item in the extent tree + * yet, etc). If this is the case, wait for all free + * space endio workers to finish and retry. This is a + * very rare case so no need for a more efficient and + * complex approach. + */ + if (ret == -ENOENT) { + wait_event(cur_trans->writer_wait, + atomic_read(&cur_trans->num_writers) == 1); + ret = update_block_group_item(trans, path, cache); + if (ret) + btrfs_abort_transaction(trans, ret); + } else if (ret) { + btrfs_abort_transaction(trans, ret); + } + } + + /* If its not on the io list, we need to put the block group */ + if (should_put) + btrfs_put_block_group(cache); + btrfs_dec_delayed_refs_rsv_bg_updates(fs_info); + spin_lock(&cur_trans->dirty_bgs_lock); + } + spin_unlock(&cur_trans->dirty_bgs_lock); + + /* + * Refer to the definition of io_bgs member for details why it's safe + * to use it without any locking + */ + while (!list_empty(io)) { + cache = list_first_entry(io, struct btrfs_block_group, + io_list); + list_del_init(&cache->io_list); + btrfs_wait_cache_io(trans, cache, path); + btrfs_put_block_group(cache); + } + + return ret; +} + +static void btrfs_maybe_reset_size_class(struct btrfs_block_group *bg) +{ + lockdep_assert_held(&bg->lock); + if (btrfs_block_group_should_use_size_class(bg) && + bg->used == 0 && bg->reserved == 0) + bg->size_class = BTRFS_BG_SZ_NONE; +} + +int btrfs_update_block_group(struct btrfs_trans_handle *trans, + u64 bytenr, u64 num_bytes, bool alloc) +{ + struct btrfs_fs_info *info = trans->fs_info; + struct btrfs_space_info *space_info; + struct btrfs_block_group *cache; + u64 old_val; + bool reclaim = false; + bool bg_already_dirty = true; + int factor; + + /* Block accounting for super block */ + spin_lock(&info->delalloc_root_lock); + old_val = btrfs_super_bytes_used(info->super_copy); + if (alloc) + old_val += num_bytes; + else + old_val -= num_bytes; + btrfs_set_super_bytes_used(info->super_copy, old_val); + spin_unlock(&info->delalloc_root_lock); + + cache = btrfs_lookup_block_group(info, bytenr); + if (!cache) + return -ENOENT; + + /* An extent can not span multiple block groups. */ + ASSERT(bytenr + num_bytes <= btrfs_block_group_end(cache)); + + space_info = cache->space_info; + factor = btrfs_bg_type_to_factor(cache->flags); + + /* + * If this block group has free space cache written out, we need to make + * sure to load it if we are removing space. This is because we need + * the unpinning stage to actually add the space back to the block group, + * otherwise we will leak space. + */ + if (!alloc && !btrfs_block_group_done(cache)) + btrfs_cache_block_group(cache, true); + + spin_lock(&space_info->lock); + spin_lock(&cache->lock); + + if (btrfs_test_opt(info, SPACE_CACHE) && + cache->disk_cache_state < BTRFS_DC_CLEAR) + cache->disk_cache_state = BTRFS_DC_CLEAR; + + old_val = cache->used; + if (alloc) { + old_val += num_bytes; + cache->used = old_val; + cache->reserved -= num_bytes; + cache->reclaim_mark = false; + space_info->bytes_reserved -= num_bytes; + space_info->bytes_used += num_bytes; + space_info->disk_used += num_bytes * factor; + if (READ_ONCE(space_info->periodic_reclaim)) + btrfs_space_info_update_reclaimable(space_info, -num_bytes); + spin_unlock(&cache->lock); + spin_unlock(&space_info->lock); + } else { + old_val -= num_bytes; + cache->used = old_val; + cache->pinned += num_bytes; + btrfs_maybe_reset_size_class(cache); + btrfs_space_info_update_bytes_pinned(space_info, num_bytes); + space_info->bytes_used -= num_bytes; + space_info->disk_used -= num_bytes * factor; + if (READ_ONCE(space_info->periodic_reclaim)) + btrfs_space_info_update_reclaimable(space_info, num_bytes); + else + reclaim = should_reclaim_block_group(cache, num_bytes); + + spin_unlock(&cache->lock); + spin_unlock(&space_info->lock); + + btrfs_set_extent_bit(&trans->transaction->pinned_extents, bytenr, + bytenr + num_bytes - 1, EXTENT_DIRTY, NULL); + } + + spin_lock(&trans->transaction->dirty_bgs_lock); + if (list_empty(&cache->dirty_list)) { + list_add_tail(&cache->dirty_list, &trans->transaction->dirty_bgs); + bg_already_dirty = false; + btrfs_get_block_group(cache); + } + spin_unlock(&trans->transaction->dirty_bgs_lock); + + /* + * No longer have used bytes in this block group, queue it for deletion. + * We do this after adding the block group to the dirty list to avoid + * races between cleaner kthread and space cache writeout. + */ + if (!alloc && old_val == 0) { + if (!btrfs_test_opt(info, DISCARD_ASYNC)) + btrfs_mark_bg_unused(cache); + } else if (!alloc && reclaim) { + btrfs_mark_bg_to_reclaim(cache); + } + + btrfs_put_block_group(cache); + + /* Modified block groups are accounted for in the delayed_refs_rsv. */ + if (!bg_already_dirty) + btrfs_inc_delayed_refs_rsv_bg_updates(info); + + return 0; +} + +/* + * Update the block_group and space info counters. + * + * @cache: The cache we are manipulating + * @ram_bytes: The number of bytes of file content, and will be same to + * @num_bytes except for the compress path. + * @num_bytes: The number of bytes in question + * @delalloc: The blocks are allocated for the delalloc write + * + * This is called by the allocator when it reserves space. If this is a + * reservation and the block group has become read only we cannot make the + * reservation and return -EAGAIN, otherwise this function always succeeds. + */ +int btrfs_add_reserved_bytes(struct btrfs_block_group *cache, + u64 ram_bytes, u64 num_bytes, bool delalloc, + bool force_wrong_size_class) +{ + struct btrfs_space_info *space_info = cache->space_info; + enum btrfs_block_group_size_class size_class; + int ret = 0; + + spin_lock(&space_info->lock); + spin_lock(&cache->lock); + if (cache->ro) { + ret = -EAGAIN; + goto out_error; + } + + if (btrfs_block_group_should_use_size_class(cache)) { + size_class = btrfs_calc_block_group_size_class(num_bytes); + ret = btrfs_use_block_group_size_class(cache, size_class, force_wrong_size_class); + if (ret) + goto out_error; + } + + cache->reserved += num_bytes; + if (delalloc) + cache->delalloc_bytes += num_bytes; + + trace_btrfs_space_reservation(cache->fs_info, "space_info", + space_info->flags, num_bytes, 1); + spin_unlock(&cache->lock); + + space_info->bytes_reserved += num_bytes; + btrfs_space_info_update_bytes_may_use(space_info, -ram_bytes); + + /* + * Compression can use less space than we reserved, so wake tickets if + * that happens. + */ + if (num_bytes < ram_bytes) + btrfs_try_granting_tickets(space_info); + spin_unlock(&space_info->lock); + + return 0; + +out_error: + spin_unlock(&cache->lock); + spin_unlock(&space_info->lock); + return ret; +} + +/* + * Update the block_group and space info counters. + * + * @cache: The cache we are manipulating. + * @num_bytes: The number of bytes in question. + * @is_delalloc: Whether the blocks are allocated for a delalloc write. + * + * This is called by somebody who is freeing space that was never actually used + * on disk. For example if you reserve some space for a new leaf in transaction + * A and before transaction A commits you free that leaf, you call this with + * reserve set to 0 in order to clear the reservation. + */ +void btrfs_free_reserved_bytes(struct btrfs_block_group *cache, u64 num_bytes, + bool is_delalloc) +{ + struct btrfs_space_info *space_info = cache->space_info; + bool bg_ro; + + spin_lock(&space_info->lock); + spin_lock(&cache->lock); + bg_ro = cache->ro; + cache->reserved -= num_bytes; + btrfs_maybe_reset_size_class(cache); + if (is_delalloc) + cache->delalloc_bytes -= num_bytes; + spin_unlock(&cache->lock); + + if (bg_ro) + space_info->bytes_readonly += num_bytes; + else if (btrfs_is_zoned(cache->fs_info)) + space_info->bytes_zone_unusable += num_bytes; + + space_info->bytes_reserved -= num_bytes; + space_info->max_extent_size = 0; + + btrfs_try_granting_tickets(space_info); + spin_unlock(&space_info->lock); +} + +static void force_metadata_allocation(struct btrfs_fs_info *info) +{ + struct list_head *head = &info->space_info; + struct btrfs_space_info *found; + + list_for_each_entry(found, head, list) { + if (found->flags & BTRFS_BLOCK_GROUP_METADATA) + found->force_alloc = CHUNK_ALLOC_FORCE; + } +} + +static bool should_alloc_chunk(const struct btrfs_fs_info *fs_info, + const struct btrfs_space_info *sinfo, int force) +{ + u64 bytes_used = btrfs_space_info_used(sinfo, false); + u64 thresh; + + if (force == CHUNK_ALLOC_FORCE) + return true; + + /* + * in limited mode, we want to have some free space up to + * about 1% of the FS size. + */ + if (force == CHUNK_ALLOC_LIMITED) { + thresh = btrfs_super_total_bytes(fs_info->super_copy); + thresh = max_t(u64, SZ_64M, mult_perc(thresh, 1)); + + if (sinfo->total_bytes - bytes_used < thresh) + return true; + } + + if (bytes_used + SZ_2M < mult_perc(sinfo->total_bytes, 80)) + return false; + return true; +} + +int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans, u64 type) +{ + u64 alloc_flags = btrfs_get_alloc_profile(trans->fs_info, type); + struct btrfs_space_info *space_info; + + space_info = btrfs_find_space_info(trans->fs_info, type); + if (unlikely(!space_info)) { + DEBUG_WARN(); + return -EINVAL; + } + + return btrfs_chunk_alloc(trans, space_info, alloc_flags, CHUNK_ALLOC_FORCE); +} + +static struct btrfs_block_group *do_chunk_alloc(struct btrfs_trans_handle *trans, + struct btrfs_space_info *space_info, + u64 flags) +{ + struct btrfs_block_group *bg; + int ret; + + /* + * Check if we have enough space in the system space info because we + * will need to update device items in the chunk btree and insert a new + * chunk item in the chunk btree as well. This will allocate a new + * system block group if needed. + */ + check_system_chunk(trans, flags); + + bg = btrfs_create_chunk(trans, space_info, flags); + if (IS_ERR(bg)) { + ret = PTR_ERR(bg); + goto out; + } + + ret = btrfs_chunk_alloc_add_chunk_item(trans, bg); + /* + * Normally we are not expected to fail with -ENOSPC here, since we have + * previously reserved space in the system space_info and allocated one + * new system chunk if necessary. However there are three exceptions: + * + * 1) We may have enough free space in the system space_info but all the + * existing system block groups have a profile which can not be used + * for extent allocation. + * + * This happens when mounting in degraded mode. For example we have a + * RAID1 filesystem with 2 devices, lose one device and mount the fs + * using the other device in degraded mode. If we then allocate a chunk, + * we may have enough free space in the existing system space_info, but + * none of the block groups can be used for extent allocation since they + * have a RAID1 profile, and because we are in degraded mode with a + * single device, we are forced to allocate a new system chunk with a + * SINGLE profile. Making check_system_chunk() iterate over all system + * block groups and check if they have a usable profile and enough space + * can be slow on very large filesystems, so we tolerate the -ENOSPC and + * try again after forcing allocation of a new system chunk. Like this + * we avoid paying the cost of that search in normal circumstances, when + * we were not mounted in degraded mode; + * + * 2) We had enough free space info the system space_info, and one suitable + * block group to allocate from when we called check_system_chunk() + * above. However right after we called it, the only system block group + * with enough free space got turned into RO mode by a running scrub, + * and in this case we have to allocate a new one and retry. We only + * need do this allocate and retry once, since we have a transaction + * handle and scrub uses the commit root to search for block groups; + * + * 3) We had one system block group with enough free space when we called + * check_system_chunk(), but after that, right before we tried to + * allocate the last extent buffer we needed, a discard operation came + * in and it temporarily removed the last free space entry from the + * block group (discard removes a free space entry, discards it, and + * then adds back the entry to the block group cache). + */ + if (ret == -ENOSPC) { + const u64 sys_flags = btrfs_system_alloc_profile(trans->fs_info); + struct btrfs_block_group *sys_bg; + struct btrfs_space_info *sys_space_info; + + sys_space_info = btrfs_find_space_info(trans->fs_info, sys_flags); + if (unlikely(!sys_space_info)) { + ret = -EINVAL; + btrfs_abort_transaction(trans, ret); + goto out; + } + + sys_bg = btrfs_create_chunk(trans, sys_space_info, sys_flags); + if (IS_ERR(sys_bg)) { + ret = PTR_ERR(sys_bg); + btrfs_abort_transaction(trans, ret); + goto out; + } + + ret = btrfs_chunk_alloc_add_chunk_item(trans, sys_bg); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + + ret = btrfs_chunk_alloc_add_chunk_item(trans, bg); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + } else if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } +out: + btrfs_trans_release_chunk_metadata(trans); + + if (ret) + return ERR_PTR(ret); + + btrfs_get_block_group(bg); + return bg; +} + +/* + * Chunk allocation is done in 2 phases: + * + * 1) Phase 1 - through btrfs_chunk_alloc() we allocate device extents for + * the chunk, the chunk mapping, create its block group and add the items + * that belong in the chunk btree to it - more specifically, we need to + * update device items in the chunk btree and add a new chunk item to it. + * + * 2) Phase 2 - through btrfs_create_pending_block_groups(), we add the block + * group item to the extent btree and the device extent items to the devices + * btree. + * + * This is done to prevent deadlocks. For example when COWing a node from the + * extent btree we are holding a write lock on the node's parent and if we + * trigger chunk allocation and attempted to insert the new block group item + * in the extent btree right way, we could deadlock because the path for the + * insertion can include that parent node. At first glance it seems impossible + * to trigger chunk allocation after starting a transaction since tasks should + * reserve enough transaction units (metadata space), however while that is true + * most of the time, chunk allocation may still be triggered for several reasons: + * + * 1) When reserving metadata, we check if there is enough free space in the + * metadata space_info and therefore don't trigger allocation of a new chunk. + * However later when the task actually tries to COW an extent buffer from + * the extent btree or from the device btree for example, it is forced to + * allocate a new block group (chunk) because the only one that had enough + * free space was just turned to RO mode by a running scrub for example (or + * device replace, block group reclaim thread, etc), so we can not use it + * for allocating an extent and end up being forced to allocate a new one; + * + * 2) Because we only check that the metadata space_info has enough free bytes, + * we end up not allocating a new metadata chunk in that case. However if + * the filesystem was mounted in degraded mode, none of the existing block + * groups might be suitable for extent allocation due to their incompatible + * profile (for e.g. mounting a 2 devices filesystem, where all block groups + * use a RAID1 profile, in degraded mode using a single device). In this case + * when the task attempts to COW some extent buffer of the extent btree for + * example, it will trigger allocation of a new metadata block group with a + * suitable profile (SINGLE profile in the example of the degraded mount of + * the RAID1 filesystem); + * + * 3) The task has reserved enough transaction units / metadata space, but when + * it attempts to COW an extent buffer from the extent or device btree for + * example, it does not find any free extent in any metadata block group, + * therefore forced to try to allocate a new metadata block group. + * This is because some other task allocated all available extents in the + * meanwhile - this typically happens with tasks that don't reserve space + * properly, either intentionally or as a bug. One example where this is + * done intentionally is fsync, as it does not reserve any transaction units + * and ends up allocating a variable number of metadata extents for log + * tree extent buffers; + * + * 4) The task has reserved enough transaction units / metadata space, but right + * before it tries to allocate the last extent buffer it needs, a discard + * operation comes in and, temporarily, removes the last free space entry from + * the only metadata block group that had free space (discard starts by + * removing a free space entry from a block group, then does the discard + * operation and, once it's done, it adds back the free space entry to the + * block group). + * + * We also need this 2 phases setup when adding a device to a filesystem with + * a seed device - we must create new metadata and system chunks without adding + * any of the block group items to the chunk, extent and device btrees. If we + * did not do it this way, we would get ENOSPC when attempting to update those + * btrees, since all the chunks from the seed device are read-only. + * + * Phase 1 does the updates and insertions to the chunk btree because if we had + * it done in phase 2 and have a thundering herd of tasks allocating chunks in + * parallel, we risk having too many system chunks allocated by many tasks if + * many tasks reach phase 1 without the previous ones completing phase 2. In the + * extreme case this leads to exhaustion of the system chunk array in the + * superblock. This is easier to trigger if using a btree node/leaf size of 64K + * and with RAID filesystems (so we have more device items in the chunk btree). + * This has happened before and commit eafa4fd0ad0607 ("btrfs: fix exhaustion of + * the system chunk array due to concurrent allocations") provides more details. + * + * Allocation of system chunks does not happen through this function. A task that + * needs to update the chunk btree (the only btree that uses system chunks), must + * preallocate chunk space by calling either check_system_chunk() or + * btrfs_reserve_chunk_metadata() - the former is used when allocating a data or + * metadata chunk or when removing a chunk, while the later is used before doing + * a modification to the chunk btree - use cases for the later are adding, + * removing and resizing a device as well as relocation of a system chunk. + * See the comment below for more details. + * + * The reservation of system space, done through check_system_chunk(), as well + * as all the updates and insertions into the chunk btree must be done while + * holding fs_info->chunk_mutex. This is important to guarantee that while COWing + * an extent buffer from the chunks btree we never trigger allocation of a new + * system chunk, which would result in a deadlock (trying to lock twice an + * extent buffer of the chunk btree, first time before triggering the chunk + * allocation and the second time during chunk allocation while attempting to + * update the chunks btree). The system chunk array is also updated while holding + * that mutex. The same logic applies to removing chunks - we must reserve system + * space, update the chunk btree and the system chunk array in the superblock + * while holding fs_info->chunk_mutex. + * + * This function, btrfs_chunk_alloc(), belongs to phase 1. + * + * @space_info: specify which space_info the new chunk should belong to. + * + * If @force is CHUNK_ALLOC_FORCE: + * - return 1 if it successfully allocates a chunk, + * - return errors including -ENOSPC otherwise. + * If @force is NOT CHUNK_ALLOC_FORCE: + * - return 0 if it doesn't need to allocate a new chunk, + * - return 1 if it successfully allocates a chunk, + * - return errors including -ENOSPC otherwise. + */ +int btrfs_chunk_alloc(struct btrfs_trans_handle *trans, + struct btrfs_space_info *space_info, u64 flags, + enum btrfs_chunk_alloc_enum force) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_block_group *ret_bg; + bool wait_for_alloc = false; + bool should_alloc = false; + bool from_extent_allocation = false; + int ret = 0; + + if (force == CHUNK_ALLOC_FORCE_FOR_EXTENT) { + from_extent_allocation = true; + force = CHUNK_ALLOC_FORCE; + } + + /* Don't re-enter if we're already allocating a chunk */ + if (trans->allocating_chunk) + return -ENOSPC; + /* + * Allocation of system chunks can not happen through this path, as we + * could end up in a deadlock if we are allocating a data or metadata + * chunk and there is another task modifying the chunk btree. + * + * This is because while we are holding the chunk mutex, we will attempt + * to add the new chunk item to the chunk btree or update an existing + * device item in the chunk btree, while the other task that is modifying + * the chunk btree is attempting to COW an extent buffer while holding a + * lock on it and on its parent - if the COW operation triggers a system + * chunk allocation, then we can deadlock because we are holding the + * chunk mutex and we may need to access that extent buffer or its parent + * in order to add the chunk item or update a device item. + * + * Tasks that want to modify the chunk tree should reserve system space + * before updating the chunk btree, by calling either + * btrfs_reserve_chunk_metadata() or check_system_chunk(). + * It's possible that after a task reserves the space, it still ends up + * here - this happens in the cases described above at do_chunk_alloc(). + * The task will have to either retry or fail. + */ + if (flags & BTRFS_BLOCK_GROUP_SYSTEM) + return -ENOSPC; + + do { + spin_lock(&space_info->lock); + if (force < space_info->force_alloc) + force = space_info->force_alloc; + should_alloc = should_alloc_chunk(fs_info, space_info, force); + if (space_info->full) { + /* No more free physical space */ + spin_unlock(&space_info->lock); + if (should_alloc) + ret = -ENOSPC; + else + ret = 0; + return ret; + } else if (!should_alloc) { + spin_unlock(&space_info->lock); + return 0; + } else if (space_info->chunk_alloc) { + /* + * Someone is already allocating, so we need to block + * until this someone is finished and then loop to + * recheck if we should continue with our allocation + * attempt. + */ + spin_unlock(&space_info->lock); + wait_for_alloc = true; + force = CHUNK_ALLOC_NO_FORCE; + mutex_lock(&fs_info->chunk_mutex); + mutex_unlock(&fs_info->chunk_mutex); + } else { + /* Proceed with allocation */ + space_info->chunk_alloc = true; + spin_unlock(&space_info->lock); + wait_for_alloc = false; + } + + cond_resched(); + } while (wait_for_alloc); + + mutex_lock(&fs_info->chunk_mutex); + trans->allocating_chunk = true; + + /* + * If we have mixed data/metadata chunks we want to make sure we keep + * allocating mixed chunks instead of individual chunks. + */ + if (btrfs_mixed_space_info(space_info)) + flags |= (BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_METADATA); + + /* + * if we're doing a data chunk, go ahead and make sure that + * we keep a reasonable number of metadata chunks allocated in the + * FS as well. + */ + if (flags & BTRFS_BLOCK_GROUP_DATA && fs_info->metadata_ratio) { + fs_info->data_chunk_allocations++; + if (!(fs_info->data_chunk_allocations % + fs_info->metadata_ratio)) + force_metadata_allocation(fs_info); + } + + ret_bg = do_chunk_alloc(trans, space_info, flags); + trans->allocating_chunk = false; + + if (IS_ERR(ret_bg)) { + ret = PTR_ERR(ret_bg); + } else if (from_extent_allocation && (flags & BTRFS_BLOCK_GROUP_DATA)) { + /* + * New block group is likely to be used soon. Try to activate + * it now. Failure is OK for now. + */ + btrfs_zone_activate(ret_bg); + } + + if (!ret) + btrfs_put_block_group(ret_bg); + + spin_lock(&space_info->lock); + if (ret < 0) { + if (ret == -ENOSPC) + space_info->full = true; + else + goto out; + } else { + ret = 1; + space_info->max_extent_size = 0; + } + + space_info->force_alloc = CHUNK_ALLOC_NO_FORCE; +out: + space_info->chunk_alloc = false; + spin_unlock(&space_info->lock); + mutex_unlock(&fs_info->chunk_mutex); + + return ret; +} + +static u64 get_profile_num_devs(const struct btrfs_fs_info *fs_info, u64 type) +{ + u64 num_dev; + + num_dev = btrfs_raid_array[btrfs_bg_flags_to_raid_index(type)].devs_max; + if (!num_dev) + num_dev = fs_info->fs_devices->rw_devices; + + return num_dev; +} + +static void reserve_chunk_space(struct btrfs_trans_handle *trans, + u64 bytes, + u64 type) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_space_info *info; + u64 left; + int ret = 0; + + /* + * Needed because we can end up allocating a system chunk and for an + * atomic and race free space reservation in the chunk block reserve. + */ + lockdep_assert_held(&fs_info->chunk_mutex); + + info = btrfs_find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM); + spin_lock(&info->lock); + left = info->total_bytes - btrfs_space_info_used(info, true); + spin_unlock(&info->lock); + + if (left < bytes && btrfs_test_opt(fs_info, ENOSPC_DEBUG)) { + btrfs_info(fs_info, "left=%llu, need=%llu, flags=%llu", + left, bytes, type); + btrfs_dump_space_info(info, 0, false); + } + + if (left < bytes) { + u64 flags = btrfs_system_alloc_profile(fs_info); + struct btrfs_block_group *bg; + struct btrfs_space_info *space_info; + + space_info = btrfs_find_space_info(fs_info, flags); + ASSERT(space_info); + + /* + * Ignore failure to create system chunk. We might end up not + * needing it, as we might not need to COW all nodes/leafs from + * the paths we visit in the chunk tree (they were already COWed + * or created in the current transaction for example). + */ + bg = btrfs_create_chunk(trans, space_info, flags); + if (IS_ERR(bg)) { + ret = PTR_ERR(bg); + } else { + int activate_ret; + + /* + * We have a new chunk. We also need to activate it for + * zoned filesystem. + */ + activate_ret = btrfs_zoned_activate_one_bg(info, true); + if (activate_ret < 0) { + ret = activate_ret; + } else { + /* + * If we fail to add the chunk item here, we end + * up trying again at phase 2 of chunk allocation, + * at btrfs_create_pending_block_groups(). So + * ignore any error here. An ENOSPC here could + * happen, due to the cases described at + * do_chunk_alloc() - the system block group we + * just created was just turned into RO mode by a + * scrub for example, or a running discard + * temporarily removed its free space entries, etc. + */ + btrfs_chunk_alloc_add_chunk_item(trans, bg); + } + } + } + + if (!ret) { + ret = btrfs_block_rsv_add(fs_info, + &fs_info->chunk_block_rsv, + bytes, BTRFS_RESERVE_NO_FLUSH); + if (!ret) + trans->chunk_bytes_reserved += bytes; + } +} + +/* + * Reserve space in the system space for allocating or removing a chunk. + * The caller must be holding fs_info->chunk_mutex. + */ +void check_system_chunk(struct btrfs_trans_handle *trans, u64 type) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + const u64 num_devs = get_profile_num_devs(fs_info, type); + u64 bytes; + + /* num_devs device items to update and 1 chunk item to add or remove. */ + bytes = btrfs_calc_metadata_size(fs_info, num_devs) + + btrfs_calc_insert_metadata_size(fs_info, 1); + + reserve_chunk_space(trans, bytes, type); +} + +/* + * Reserve space in the system space, if needed, for doing a modification to the + * chunk btree. + * + * @trans: A transaction handle. + * @is_item_insertion: Indicate if the modification is for inserting a new item + * in the chunk btree or if it's for the deletion or update + * of an existing item. + * + * This is used in a context where we need to update the chunk btree outside + * block group allocation and removal, to avoid a deadlock with a concurrent + * task that is allocating a metadata or data block group and therefore needs to + * update the chunk btree while holding the chunk mutex. After the update to the + * chunk btree is done, btrfs_trans_release_chunk_metadata() should be called. + * + */ +void btrfs_reserve_chunk_metadata(struct btrfs_trans_handle *trans, + bool is_item_insertion) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + u64 bytes; + + if (is_item_insertion) + bytes = btrfs_calc_insert_metadata_size(fs_info, 1); + else + bytes = btrfs_calc_metadata_size(fs_info, 1); + + mutex_lock(&fs_info->chunk_mutex); + reserve_chunk_space(trans, bytes, BTRFS_BLOCK_GROUP_SYSTEM); + mutex_unlock(&fs_info->chunk_mutex); +} + +void btrfs_put_block_group_cache(struct btrfs_fs_info *info) +{ + struct btrfs_block_group *block_group; + + block_group = btrfs_lookup_first_block_group(info, 0); + while (block_group) { + btrfs_wait_block_group_cache_done(block_group); + spin_lock(&block_group->lock); + if (test_and_clear_bit(BLOCK_GROUP_FLAG_IREF, + &block_group->runtime_flags)) { + struct btrfs_inode *inode = block_group->inode; + + block_group->inode = NULL; + spin_unlock(&block_group->lock); + + ASSERT(block_group->io_ctl.inode == NULL); + iput(&inode->vfs_inode); + } else { + spin_unlock(&block_group->lock); + } + block_group = btrfs_next_block_group(block_group); + } +} + +static void check_removing_space_info(struct btrfs_space_info *space_info) +{ + struct btrfs_fs_info *info = space_info->fs_info; + + if (space_info->subgroup_id == BTRFS_SUB_GROUP_PRIMARY) { + /* This is a top space_info, proceed with its children first. */ + for (int i = 0; i < BTRFS_SPACE_INFO_SUB_GROUP_MAX; i++) { + if (space_info->sub_group[i]) { + check_removing_space_info(space_info->sub_group[i]); + btrfs_sysfs_remove_space_info(space_info->sub_group[i]); + space_info->sub_group[i] = NULL; + } + } + } + + /* + * Do not hide this behind enospc_debug, this is actually important and + * indicates a real bug if this happens. + */ + if (WARN_ON(space_info->bytes_pinned > 0 || space_info->bytes_may_use > 0)) + btrfs_dump_space_info(space_info, 0, false); + + /* + * If there was a failure to cleanup a log tree, very likely due to an + * IO failure on a writeback attempt of one or more of its extent + * buffers, we could not do proper (and cheap) unaccounting of their + * reserved space, so don't warn on bytes_reserved > 0 in that case. + */ + if (!(space_info->flags & BTRFS_BLOCK_GROUP_METADATA) || + !BTRFS_FS_LOG_CLEANUP_ERROR(info)) { + if (WARN_ON(space_info->bytes_reserved > 0)) + btrfs_dump_space_info(space_info, 0, false); + } + + WARN_ON(space_info->reclaim_size > 0); +} + +/* + * Must be called only after stopping all workers, since we could have block + * group caching kthreads running, and therefore they could race with us if we + * freed the block groups before stopping them. + */ +int btrfs_free_block_groups(struct btrfs_fs_info *info) +{ + struct btrfs_block_group *block_group; + struct btrfs_space_info *space_info; + struct btrfs_caching_control *caching_ctl; + struct rb_node *n; + + if (btrfs_is_zoned(info)) { + if (info->active_meta_bg) { + btrfs_put_block_group(info->active_meta_bg); + info->active_meta_bg = NULL; + } + if (info->active_system_bg) { + btrfs_put_block_group(info->active_system_bg); + info->active_system_bg = NULL; + } + } + + write_lock(&info->block_group_cache_lock); + while (!list_empty(&info->caching_block_groups)) { + caching_ctl = list_first_entry(&info->caching_block_groups, + struct btrfs_caching_control, list); + list_del(&caching_ctl->list); + btrfs_put_caching_control(caching_ctl); + } + write_unlock(&info->block_group_cache_lock); + + spin_lock(&info->unused_bgs_lock); + while (!list_empty(&info->unused_bgs)) { + block_group = list_first_entry(&info->unused_bgs, + struct btrfs_block_group, + bg_list); + list_del_init(&block_group->bg_list); + btrfs_put_block_group(block_group); + } + + while (!list_empty(&info->reclaim_bgs)) { + block_group = list_first_entry(&info->reclaim_bgs, + struct btrfs_block_group, + bg_list); + list_del_init(&block_group->bg_list); + btrfs_put_block_group(block_group); + } + + while (!list_empty(&info->fully_remapped_bgs)) { + block_group = list_first_entry(&info->fully_remapped_bgs, + struct btrfs_block_group, bg_list); + list_del_init(&block_group->bg_list); + btrfs_put_block_group(block_group); + } + spin_unlock(&info->unused_bgs_lock); + + spin_lock(&info->zone_active_bgs_lock); + while (!list_empty(&info->zone_active_bgs)) { + block_group = list_first_entry(&info->zone_active_bgs, + struct btrfs_block_group, + active_bg_list); + list_del_init(&block_group->active_bg_list); + btrfs_put_block_group(block_group); + } + spin_unlock(&info->zone_active_bgs_lock); + + write_lock(&info->block_group_cache_lock); + while ((n = rb_last(&info->block_group_cache_tree.rb_root)) != NULL) { + block_group = rb_entry(n, struct btrfs_block_group, + cache_node); + rb_erase_cached(&block_group->cache_node, + &info->block_group_cache_tree); + RB_CLEAR_NODE(&block_group->cache_node); + write_unlock(&info->block_group_cache_lock); + + down_write(&block_group->space_info->groups_sem); + list_del(&block_group->list); + up_write(&block_group->space_info->groups_sem); + + /* + * We haven't cached this block group, which means we could + * possibly have excluded extents on this block group. + */ + if (block_group->cached == BTRFS_CACHE_NO || + block_group->cached == BTRFS_CACHE_ERROR) + btrfs_free_excluded_extents(block_group); + + btrfs_remove_free_space_cache(block_group); + ASSERT(block_group->cached != BTRFS_CACHE_STARTED); + ASSERT(list_empty(&block_group->dirty_list)); + ASSERT(list_empty(&block_group->io_list)); + ASSERT(list_empty(&block_group->bg_list)); + ASSERT(refcount_read(&block_group->refs) == 1); + ASSERT(block_group->swap_extents == 0); + btrfs_put_block_group(block_group); + + write_lock(&info->block_group_cache_lock); + } + write_unlock(&info->block_group_cache_lock); + + btrfs_release_global_block_rsv(info); + + while (!list_empty(&info->space_info)) { + space_info = list_first_entry(&info->space_info, + struct btrfs_space_info, list); + + check_removing_space_info(space_info); + list_del(&space_info->list); + btrfs_sysfs_remove_space_info(space_info); + } + return 0; +} + +void btrfs_freeze_block_group(struct btrfs_block_group *cache) +{ + atomic_inc(&cache->frozen); +} + +void btrfs_unfreeze_block_group(struct btrfs_block_group *block_group) +{ + struct btrfs_fs_info *fs_info = block_group->fs_info; + bool cleanup; + + spin_lock(&block_group->lock); + cleanup = (atomic_dec_and_test(&block_group->frozen) && + test_bit(BLOCK_GROUP_FLAG_REMOVED, &block_group->runtime_flags)); + spin_unlock(&block_group->lock); + + if (cleanup) { + struct btrfs_chunk_map *map; + + map = btrfs_find_chunk_map(fs_info, block_group->start, 1); + /* Logic error, can't happen. */ + ASSERT(map); + + btrfs_remove_chunk_map(fs_info, map); + + /* Once for our lookup reference. */ + btrfs_free_chunk_map(map); + + /* + * We may have left one free space entry and other possible + * tasks trimming this block group have left 1 entry each one. + * Free them if any. + */ + btrfs_remove_free_space_cache(block_group); + } +} + +bool btrfs_inc_block_group_swap_extents(struct btrfs_block_group *bg) +{ + bool ret = true; + + spin_lock(&bg->lock); + if (bg->ro) + ret = false; + else + bg->swap_extents++; + spin_unlock(&bg->lock); + + return ret; +} + +void btrfs_dec_block_group_swap_extents(struct btrfs_block_group *bg, int amount) +{ + spin_lock(&bg->lock); + ASSERT(!bg->ro); + ASSERT(bg->swap_extents >= amount); + bg->swap_extents -= amount; + spin_unlock(&bg->lock); +} + +enum btrfs_block_group_size_class btrfs_calc_block_group_size_class(u64 size) +{ + if (size <= SZ_128K) + return BTRFS_BG_SZ_SMALL; + if (size <= SZ_8M) + return BTRFS_BG_SZ_MEDIUM; + return BTRFS_BG_SZ_LARGE; +} + +/* + * Handle a block group allocating an extent in a size class + * + * @bg: The block group we allocated in. + * @size_class: The size class of the allocation. + * @force_wrong_size_class: Whether we are desperate enough to allow + * mismatched size classes. + * + * Returns: 0 if the size class was valid for this block_group, -EAGAIN in the + * case of a race that leads to the wrong size class without + * force_wrong_size_class set. + * + * find_free_extent will skip block groups with a mismatched size class until + * it really needs to avoid ENOSPC. In that case it will set + * force_wrong_size_class. However, if a block group is newly allocated and + * doesn't yet have a size class, then it is possible for two allocations of + * different sizes to race and both try to use it. The loser is caught here and + * has to retry. + */ +int btrfs_use_block_group_size_class(struct btrfs_block_group *bg, + enum btrfs_block_group_size_class size_class, + bool force_wrong_size_class) +{ + lockdep_assert_held(&bg->lock); + ASSERT(size_class != BTRFS_BG_SZ_NONE); + + /* The new allocation is in the right size class, do nothing */ + if (bg->size_class == size_class) + return 0; + /* + * The new allocation is in a mismatched size class. + * This means one of two things: + * + * 1. Two tasks in find_free_extent for different size_classes raced + * and hit the same empty block_group. Make the loser try again. + * 2. A call to find_free_extent got desperate enough to set + * 'force_wrong_slab'. Don't change the size_class, but allow the + * allocation. + */ + if (bg->size_class != BTRFS_BG_SZ_NONE) { + if (force_wrong_size_class) + return 0; + return -EAGAIN; + } + /* + * The happy new block group case: the new allocation is the first + * one in the block_group so we set size_class. + */ + bg->size_class = size_class; + + return 0; +} + +bool btrfs_block_group_should_use_size_class(const struct btrfs_block_group *bg) +{ + if (btrfs_is_zoned(bg->fs_info)) + return false; + if (!btrfs_is_block_group_data_only(bg)) + return false; + return true; +} + +void btrfs_mark_bg_fully_remapped(struct btrfs_block_group *bg, + struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + + + if (btrfs_test_opt(fs_info, DISCARD_ASYNC)) { + spin_lock(&bg->lock); + set_bit(BLOCK_GROUP_FLAG_STRIPE_REMOVAL_PENDING, &bg->runtime_flags); + spin_unlock(&bg->lock); + + btrfs_discard_queue_work(&fs_info->discard_ctl, bg); + } else { + spin_lock(&fs_info->unused_bgs_lock); + /* + * The block group might already be on the unused_bgs list, + * remove it if it is. It'll get readded after + * btrfs_handle_fully_remapped_bgs() finishes. + */ + if (!list_empty(&bg->bg_list)) + list_del(&bg->bg_list); + else + btrfs_get_block_group(bg); + + list_add_tail(&bg->bg_list, &fs_info->fully_remapped_bgs); + spin_unlock(&fs_info->unused_bgs_lock); + } +} + +/* + * Compare the block group and chunk trees, and find any fully-remapped block + * groups which haven't yet had their chunk stripes and device extents removed, + * and put them on the fully_remapped_bgs list so this gets done. + * + * This happens when a block group becomes fully remapped, i.e. its last + * identity mapping is removed, and the volume is unmounted before async + * discard has finished. It's important this gets done as until it is the + * chunk's stripes are dead space. + */ +int btrfs_populate_fully_remapped_bgs_list(struct btrfs_fs_info *fs_info) +{ + struct rb_node *node_bg, *node_chunk; + + node_bg = rb_first_cached(&fs_info->block_group_cache_tree); + node_chunk = rb_first_cached(&fs_info->mapping_tree); + + while (node_bg && node_chunk) { + struct btrfs_block_group *bg; + struct btrfs_chunk_map *map; + + bg = rb_entry(node_bg, struct btrfs_block_group, cache_node); + map = rb_entry(node_chunk, struct btrfs_chunk_map, rb_node); + + ASSERT(bg->start == map->start); + + if (!(bg->flags & BTRFS_BLOCK_GROUP_REMAPPED)) + goto next; + + if (bg->identity_remap_count != 0) + goto next; + + if (map->num_stripes == 0) + goto next; + + spin_lock(&fs_info->unused_bgs_lock); + + if (list_empty(&bg->bg_list)) { + btrfs_get_block_group(bg); + list_add_tail(&bg->bg_list, &fs_info->fully_remapped_bgs); + } else { + list_move_tail(&bg->bg_list, &fs_info->fully_remapped_bgs); + } + + spin_unlock(&fs_info->unused_bgs_lock); + + /* + * Ideally we'd want to call btrfs_discard_queue_work() here, + * but it'd do nothing as the discard worker hasn't been + * started yet. + * + * The block group will get added to the discard list when + * btrfs_handle_fully_remapped_bgs() gets called, when we + * commit the first transaction. + */ + if (btrfs_test_opt(fs_info, DISCARD_ASYNC)) { + spin_lock(&bg->lock); + set_bit(BLOCK_GROUP_FLAG_STRIPE_REMOVAL_PENDING, &bg->runtime_flags); + spin_unlock(&bg->lock); + } + +next: + node_bg = rb_next(node_bg); + node_chunk = rb_next(node_chunk); + } + + ASSERT(!node_bg && !node_chunk); + + return 0; +} diff --git a/fs/btrfs/block-group.h b/fs/btrfs/block-group.h new file mode 100644 index 000000000..69d56864d --- /dev/null +++ b/fs/btrfs/block-group.h @@ -0,0 +1,429 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_BLOCK_GROUP_H +#define BTRFS_BLOCK_GROUP_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "free-space-cache.h" + +struct btrfs_chunk_map; +struct btrfs_fs_info; +struct btrfs_inode; +struct btrfs_trans_handle; + +enum btrfs_disk_cache_state { + BTRFS_DC_WRITTEN, + BTRFS_DC_ERROR, + BTRFS_DC_CLEAR, + BTRFS_DC_SETUP, +}; + +enum btrfs_block_group_size_class { + /* Unset */ + BTRFS_BG_SZ_NONE, + /* 0 < size <= 128K */ + BTRFS_BG_SZ_SMALL, + /* 128K < size <= 8M */ + BTRFS_BG_SZ_MEDIUM, + /* 8M < size < BG_LENGTH */ + BTRFS_BG_SZ_LARGE, +}; + +/* + * This describes the state of the block_group for async discard. This is due + * to the two pass nature of it where extent discarding is prioritized over + * bitmap discarding. BTRFS_DISCARD_RESET_CURSOR is set when we are resetting + * between lists to prevent contention for discard state variables + * (eg. discard_cursor). + */ +enum btrfs_discard_state { + BTRFS_DISCARD_EXTENTS, + BTRFS_DISCARD_BITMAPS, + BTRFS_DISCARD_RESET_CURSOR, + BTRFS_DISCARD_FULLY_REMAPPED, +}; + +/* + * Control flags for do_chunk_alloc's force field CHUNK_ALLOC_NO_FORCE means to + * only allocate a chunk if we really need one. + * + * CHUNK_ALLOC_LIMITED means to only try and allocate one if we have very few + * chunks already allocated. This is used as part of the clustering code to + * help make sure we have a good pool of storage to cluster in, without filling + * the FS with empty chunks + * + * CHUNK_ALLOC_FORCE means it must try to allocate one + * + * CHUNK_ALLOC_FORCE_FOR_EXTENT like CHUNK_ALLOC_FORCE but called from + * find_free_extent() that also activates the zone + */ +enum btrfs_chunk_alloc_enum { + CHUNK_ALLOC_NO_FORCE, + CHUNK_ALLOC_LIMITED, + CHUNK_ALLOC_FORCE, + CHUNK_ALLOC_FORCE_FOR_EXTENT, +}; + +/* Block group flags set at runtime */ +enum btrfs_block_group_flags { + BLOCK_GROUP_FLAG_IREF, + BLOCK_GROUP_FLAG_REMOVED, + BLOCK_GROUP_FLAG_TO_COPY, + BLOCK_GROUP_FLAG_RELOCATING_REPAIR, + BLOCK_GROUP_FLAG_CHUNK_ITEM_INSERTED, + BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, + BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, + /* Does the block group need to be added to the free space tree? */ + BLOCK_GROUP_FLAG_NEEDS_FREE_SPACE, + /* Set after we add a new block group to the free space tree. */ + BLOCK_GROUP_FLAG_FREE_SPACE_ADDED, + /* Indicate that the block group is placed on a sequential zone */ + BLOCK_GROUP_FLAG_SEQUENTIAL_ZONE, + /* + * Indicate that block group is in the list of new block groups of a + * transaction. + */ + BLOCK_GROUP_FLAG_NEW, + BLOCK_GROUP_FLAG_FULLY_REMAPPED, + BLOCK_GROUP_FLAG_STRIPE_REMOVAL_PENDING, +}; + +enum btrfs_caching_type { + BTRFS_CACHE_NO, + BTRFS_CACHE_STARTED, + BTRFS_CACHE_FINISHED, + BTRFS_CACHE_ERROR, +}; + +struct btrfs_caching_control { + struct list_head list; + struct mutex mutex; + wait_queue_head_t wait; + struct btrfs_work work; + struct btrfs_block_group *block_group; + /* Track progress of caching during allocation. */ + atomic_t progress; + refcount_t count; +}; + +/* Once caching_thread() finds this much free space, it will wake up waiters. */ +#define CACHING_CTL_WAKE_UP SZ_2M + +struct btrfs_block_group { + struct btrfs_fs_info *fs_info; + struct btrfs_inode *inode; + spinlock_t lock; + unsigned int ro; + u64 start; + u64 length; + u64 pinned; + u64 reserved; + u64 used; + u64 delalloc_bytes; + u64 bytes_super; + u64 flags; + u64 cache_generation; + u64 global_root_id; + u64 remap_bytes; + u32 identity_remap_count; + /* The last commited identity_remap_count value of this block group. */ + u32 last_identity_remap_count; + /* + * The last committed used bytes of this block group, if the above @used + * is still the same as @last_used, we don't need to update block + * group item of this block group. + */ + u64 last_used; + /* The last committed remap_bytes value of this block group. */ + u64 last_remap_bytes; + /* The last committed flags value for this block group. */ + u64 last_flags; + + /* + * If the free space extent count exceeds this number, convert the block + * group to bitmaps. + */ + u32 bitmap_high_thresh; + + /* + * If the free space extent count drops below this number, convert the + * block group back to extents. + */ + u32 bitmap_low_thresh; + + /* + * It is just used for the delayed data space allocation because + * only the data space allocation and the relative metadata update + * can be done cross the transaction. + */ + struct rw_semaphore data_rwsem; + + /* For raid56, this is a full stripe, without parity */ + unsigned long full_stripe_len; + unsigned long runtime_flags; + + enum btrfs_disk_cache_state disk_cache_state; + + /* Cache tracking stuff */ + enum btrfs_caching_type cached; + struct btrfs_caching_control *caching_ctl; + + struct btrfs_space_info *space_info; + + /* Free space cache stuff */ + struct btrfs_free_space_ctl *free_space_ctl; + + /* Block group cache stuff */ + struct rb_node cache_node; + + /* For block groups in the same raid type */ + struct list_head list; + + refcount_t refs; + + /* + * When non-zero it means the block group's logical address and its + * device extents can not be reused for future block group allocations + * until the counter goes down to 0. This is to prevent them from being + * reused while some task is still using the block group after it was + * deleted - we want to make sure they can only be reused for new block + * groups after that task is done with the deleted block group. + */ + atomic_t frozen; + + /* + * List of struct btrfs_free_clusters for this block group. + * Today it will only have one thing on it, but that may change + */ + struct list_head cluster_list; + + /* + * Used for several lists: + * + * 1) struct btrfs_fs_info::unused_bgs + * 2) struct btrfs_fs_info::reclaim_bgs + * 3) struct btrfs_transaction::deleted_bgs + * 4) struct btrfs_trans_handle::new_bgs + */ + struct list_head bg_list; + + /* For read-only block groups */ + struct list_head ro_list; + + /* For discard operations */ + struct list_head discard_list; + int discard_index; + enum btrfs_discard_state discard_state; + u64 discard_eligible_time; + u64 discard_cursor; + + /* For dirty block groups */ + struct list_head dirty_list; + struct list_head io_list; + + struct btrfs_io_ctl io_ctl; + + /* + * Incremented when doing extent allocations and holding a read lock + * on the space_info's groups_sem semaphore. + * Decremented when an ordered extent that represents an IO against this + * block group's range is created (after it's added to its inode's + * root's list of ordered extents) or immediately after the allocation + * if it's a metadata extent or fallocate extent (for these cases we + * don't create ordered extents). + */ + atomic_t reservations; + + /* + * Incremented while holding the spinlock *lock* by a task checking if + * it can perform a nocow write (incremented if the value for the *ro* + * field is 0). Decremented by such tasks once they create an ordered + * extent or before that if some error happens before reaching that step. + * This is to prevent races between block group relocation and nocow + * writes through direct IO. + */ + atomic_t nocow_writers; + + /* Lock for free space tree operations. */ + struct mutex free_space_lock; + + /* Protected by @free_space_lock. */ + bool using_free_space_bitmaps; + /* Protected by @free_space_lock. */ + bool using_free_space_bitmaps_cached; + + enum btrfs_block_group_size_class size_class:8; + + /* If set, this blockgroup is not used for allocation between two reclaim sweeps. */ + bool reclaim_mark; + + /* + * Number of extents in this block group used for swap files. + * All accesses protected by the spinlock 'lock'. + */ + int swap_extents; + + /* + * Allocation offset for the block group to implement sequential + * allocation. This is used only on a zoned filesystem. + */ + u64 alloc_offset; + u64 zone_unusable; + u64 zone_capacity; + u64 meta_write_pointer; + struct btrfs_chunk_map *physical_map; + struct list_head active_bg_list; + struct work_struct zone_finish_work; + struct extent_buffer *last_eb; +}; + +static inline u64 btrfs_block_group_end(const struct btrfs_block_group *block_group) +{ + return (block_group->start + block_group->length); +} + +static inline bool btrfs_is_block_group_used(const struct btrfs_block_group *bg) +{ + lockdep_assert_held(&bg->lock); + + return (bg->used > 0 || bg->reserved > 0 || bg->pinned > 0 || + bg->remap_bytes > 0); +} + +static inline bool btrfs_is_block_group_data_only(const struct btrfs_block_group *block_group) +{ + /* + * In mixed mode the fragmentation is expected to be high, lowering the + * efficiency, so only proper data block groups are considered. + */ + return (block_group->flags & BTRFS_BLOCK_GROUP_DATA) && + !(block_group->flags & BTRFS_BLOCK_GROUP_METADATA); +} + +static inline u64 btrfs_block_group_available_space(const struct btrfs_block_group *bg) +{ + lockdep_assert_held(&bg->lock); + + return (bg->length - bg->used - bg->pinned - bg->reserved - + bg->bytes_super - bg->zone_unusable); +} + +#ifdef CONFIG_BTRFS_DEBUG +int btrfs_should_fragment_free_space(const struct btrfs_block_group *block_group); +#endif + +int __init btrfs_init_block_group(void); +void __cold btrfs_exit_block_group(void); + +struct btrfs_block_group *btrfs_lookup_first_block_group( + struct btrfs_fs_info *info, u64 bytenr); +struct btrfs_block_group *btrfs_lookup_block_group( + struct btrfs_fs_info *info, u64 bytenr); +struct btrfs_block_group *btrfs_next_block_group( + struct btrfs_block_group *cache); +void btrfs_get_block_group(struct btrfs_block_group *cache); +void btrfs_put_block_group(struct btrfs_block_group *cache); +void btrfs_dec_block_group_reservations(struct btrfs_fs_info *fs_info, + const u64 start); +void btrfs_wait_block_group_reservations(struct btrfs_block_group *bg); +struct btrfs_block_group *btrfs_inc_nocow_writers(struct btrfs_fs_info *fs_info, + u64 bytenr); +void btrfs_dec_nocow_writers(struct btrfs_block_group *bg); +void btrfs_wait_nocow_writers(struct btrfs_block_group *bg); +void btrfs_wait_block_group_cache_progress(struct btrfs_block_group *cache, + u64 num_bytes); +int btrfs_cache_block_group(struct btrfs_block_group *cache, bool wait); +struct btrfs_caching_control *btrfs_get_caching_control( + struct btrfs_block_group *cache); +int btrfs_add_new_free_space(struct btrfs_block_group *block_group, + u64 start, u64 end, u64 *total_added_ret); +struct btrfs_trans_handle *btrfs_start_trans_remove_block_group( + struct btrfs_fs_info *fs_info, + const u64 chunk_offset); +void btrfs_remove_bg_from_sinfo(struct btrfs_block_group *bg); +int btrfs_remove_block_group(struct btrfs_trans_handle *trans, + struct btrfs_chunk_map *map); +void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info); +void btrfs_mark_bg_unused(struct btrfs_block_group *bg); +void btrfs_reclaim_block_groups(struct btrfs_fs_info *fs_info, unsigned int limit); +void btrfs_reclaim_bgs_work(struct work_struct *work); +void btrfs_reclaim_bgs(struct btrfs_fs_info *fs_info); +void btrfs_mark_bg_to_reclaim(struct btrfs_block_group *bg); +int btrfs_read_block_groups(struct btrfs_fs_info *info); +struct btrfs_block_group *btrfs_make_block_group(struct btrfs_trans_handle *trans, + struct btrfs_space_info *space_info, + u64 type, u64 chunk_offset, u64 size); +void btrfs_create_pending_block_groups(struct btrfs_trans_handle *trans); +int btrfs_inc_block_group_ro(struct btrfs_block_group *cache, + bool do_chunk_alloc); +void btrfs_dec_block_group_ro(struct btrfs_block_group *cache); +int btrfs_start_dirty_block_groups(struct btrfs_trans_handle *trans); +int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans); +int btrfs_setup_space_cache(struct btrfs_trans_handle *trans); +int btrfs_update_block_group(struct btrfs_trans_handle *trans, + u64 bytenr, u64 num_bytes, bool alloc); +int btrfs_add_reserved_bytes(struct btrfs_block_group *cache, + u64 ram_bytes, u64 num_bytes, bool delalloc, + bool force_wrong_size_class); +void btrfs_free_reserved_bytes(struct btrfs_block_group *cache, u64 num_bytes, + bool is_delalloc); +int btrfs_chunk_alloc(struct btrfs_trans_handle *trans, + struct btrfs_space_info *space_info, u64 flags, + enum btrfs_chunk_alloc_enum force); +int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans, u64 type); +void check_system_chunk(struct btrfs_trans_handle *trans, const u64 type); +void btrfs_reserve_chunk_metadata(struct btrfs_trans_handle *trans, + bool is_item_insertion); +u64 btrfs_get_alloc_profile(struct btrfs_fs_info *fs_info, u64 orig_flags); +void btrfs_put_block_group_cache(struct btrfs_fs_info *info); +int btrfs_free_block_groups(struct btrfs_fs_info *info); +int btrfs_rmap_block(struct btrfs_fs_info *fs_info, u64 chunk_start, + u64 physical, u64 **logical, int *naddrs, int *stripe_len); + +static inline u64 btrfs_data_alloc_profile(struct btrfs_fs_info *fs_info) +{ + return btrfs_get_alloc_profile(fs_info, BTRFS_BLOCK_GROUP_DATA); +} + +static inline u64 btrfs_metadata_alloc_profile(struct btrfs_fs_info *fs_info) +{ + return btrfs_get_alloc_profile(fs_info, BTRFS_BLOCK_GROUP_METADATA); +} + +static inline u64 btrfs_system_alloc_profile(struct btrfs_fs_info *fs_info) +{ + return btrfs_get_alloc_profile(fs_info, BTRFS_BLOCK_GROUP_SYSTEM); +} + +static inline int btrfs_block_group_done(const struct btrfs_block_group *cache) +{ + smp_mb(); + return cache->cached == BTRFS_CACHE_FINISHED || + cache->cached == BTRFS_CACHE_ERROR; +} + +void btrfs_freeze_block_group(struct btrfs_block_group *cache); +void btrfs_unfreeze_block_group(struct btrfs_block_group *cache); + +bool btrfs_inc_block_group_swap_extents(struct btrfs_block_group *bg); +void btrfs_dec_block_group_swap_extents(struct btrfs_block_group *bg, int amount); + +enum btrfs_block_group_size_class btrfs_calc_block_group_size_class(u64 size); +int btrfs_use_block_group_size_class(struct btrfs_block_group *bg, + enum btrfs_block_group_size_class size_class, + bool force_wrong_size_class); +bool btrfs_block_group_should_use_size_class(const struct btrfs_block_group *bg); +void btrfs_mark_bg_fully_remapped(struct btrfs_block_group *bg, + struct btrfs_trans_handle *trans); +int btrfs_populate_fully_remapped_bgs_list(struct btrfs_fs_info *fs_info); + +#endif /* BTRFS_BLOCK_GROUP_H */ diff --git a/fs/btrfs/block-rsv.c b/fs/btrfs/block-rsv.c new file mode 100644 index 000000000..c68a8f4b7 --- /dev/null +++ b/fs/btrfs/block-rsv.c @@ -0,0 +1,627 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "misc.h" +#include "ctree.h" +#include "block-rsv.h" +#include "space-info.h" +#include "transaction.h" +#include "block-group.h" +#include "fs.h" +#include "accessors.h" + +/* + * HOW DO BLOCK RESERVES WORK + * + * Think of block_rsv's as buckets for logically grouped metadata + * reservations. Each block_rsv has a ->size and a ->reserved. ->size is + * how large we want our block rsv to be, ->reserved is how much space is + * currently reserved for this block reserve. + * + * ->failfast exists for the truncate case, and is described below. + * + * NORMAL OPERATION + * + * -> Reserve + * Entrance: btrfs_block_rsv_add, btrfs_block_rsv_refill + * + * We call into btrfs_reserve_metadata_bytes() with our bytes, which is + * accounted for in space_info->bytes_may_use, and then add the bytes to + * ->reserved, and ->size in the case of btrfs_block_rsv_add. + * + * ->size is an over-estimation of how much we may use for a particular + * operation. + * + * -> Use + * Entrance: btrfs_use_block_rsv + * + * When we do a btrfs_alloc_tree_block() we call into btrfs_use_block_rsv() + * to determine the appropriate block_rsv to use, and then verify that + * ->reserved has enough space for our tree block allocation. Once + * successful we subtract fs_info->nodesize from ->reserved. + * + * -> Finish + * Entrance: btrfs_block_rsv_release + * + * We are finished with our operation, subtract our individual reservation + * from ->size, and then subtract ->size from ->reserved and free up the + * excess if there is any. + * + * There is some logic here to refill the delayed refs rsv or the global rsv + * as needed, otherwise the excess is subtracted from + * space_info->bytes_may_use. + * + * TYPES OF BLOCK RESERVES + * + * BLOCK_RSV_TRANS, BLOCK_RSV_DELOPS, BLOCK_RSV_CHUNK + * These behave normally, as described above, just within the confines of the + * lifetime of their particular operation (transaction for the whole trans + * handle lifetime, for example). + * + * BLOCK_RSV_GLOBAL + * It is impossible to properly account for all the space that may be required + * to make our extent tree updates. This block reserve acts as an overflow + * buffer in case our delayed refs reserve does not reserve enough space to + * update the extent tree. + * + * We can steal from this in some cases as well, notably on evict() or + * truncate() in order to help users recover from ENOSPC conditions. + * + * BLOCK_RSV_DELALLOC + * The individual item sizes are determined by the per-inode size + * calculations, which are described with the delalloc code. This is pretty + * straightforward, it's just the calculation of ->size encodes a lot of + * different items, and thus it gets used when updating inodes, inserting file + * extents, and inserting checksums. + * + * BLOCK_RSV_DELREFS + * We keep a running tally of how many delayed refs we have on the system. + * We assume each one of these delayed refs are going to use a full + * reservation. We use the transaction items and pre-reserve space for every + * operation, and use this reservation to refill any gap between ->size and + * ->reserved that may exist. + * + * From there it's straightforward, removing a delayed ref means we remove its + * count from ->size and free up reservations as necessary. Since this is + * the most dynamic block reserve in the system, we will try to refill this + * block reserve first with any excess returned by any other block reserve. + * + * BLOCK_RSV_EMPTY + * This is the fallback block reserve to make us try to reserve space if we + * don't have a specific bucket for this allocation. It is mostly used for + * updating the device tree and such, since that is a separate pool we're + * content to just reserve space from the space_info on demand. + * + * BLOCK_RSV_TEMP + * This is used by things like truncate and iput. We will temporarily + * allocate a block reserve, set it to some size, and then truncate bytes + * until we have no space left. With ->failfast set we'll simply return + * ENOSPC from btrfs_use_block_rsv() to signal that we need to unwind and try + * to make a new reservation. This is because these operations are + * unbounded, so we want to do as much work as we can, and then back off and + * re-reserve. + */ + +static u64 block_rsv_release_bytes(struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *block_rsv, + struct btrfs_block_rsv *dest, u64 num_bytes, + u64 *qgroup_to_release_ret) +{ + struct btrfs_space_info *space_info = block_rsv->space_info; + u64 qgroup_to_release = 0; + u64 ret; + + spin_lock(&block_rsv->lock); + if (num_bytes == (u64)-1) { + num_bytes = block_rsv->size; + qgroup_to_release = block_rsv->qgroup_rsv_size; + } + block_rsv->size -= num_bytes; + if (block_rsv->reserved >= block_rsv->size) { + num_bytes = block_rsv->reserved - block_rsv->size; + block_rsv->reserved = block_rsv->size; + block_rsv->full = true; + } else { + num_bytes = 0; + } + if (qgroup_to_release_ret && + block_rsv->qgroup_rsv_reserved >= block_rsv->qgroup_rsv_size) { + qgroup_to_release = block_rsv->qgroup_rsv_reserved - + block_rsv->qgroup_rsv_size; + block_rsv->qgroup_rsv_reserved = block_rsv->qgroup_rsv_size; + } else { + qgroup_to_release = 0; + } + spin_unlock(&block_rsv->lock); + + ret = num_bytes; + if (num_bytes > 0) { + if (dest) { + spin_lock(&dest->lock); + if (!dest->full) { + u64 bytes_to_add; + + bytes_to_add = dest->size - dest->reserved; + bytes_to_add = min(num_bytes, bytes_to_add); + dest->reserved += bytes_to_add; + if (dest->reserved >= dest->size) + dest->full = true; + num_bytes -= bytes_to_add; + } + spin_unlock(&dest->lock); + } + if (num_bytes) + btrfs_space_info_free_bytes_may_use(space_info, num_bytes); + } + if (qgroup_to_release_ret) + *qgroup_to_release_ret = qgroup_to_release; + return ret; +} + +int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src, + struct btrfs_block_rsv *dst, u64 num_bytes, + bool update_size) +{ + int ret; + + ret = btrfs_block_rsv_use_bytes(src, num_bytes); + if (ret) + return ret; + + btrfs_block_rsv_add_bytes(dst, num_bytes, update_size); + return 0; +} + +void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv, enum btrfs_rsv_type type) +{ + memset(rsv, 0, sizeof(*rsv)); + spin_lock_init(&rsv->lock); + rsv->type = type; +} + +void btrfs_init_metadata_block_rsv(struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *rsv, + enum btrfs_rsv_type type) +{ + btrfs_init_block_rsv(rsv, type); + rsv->space_info = btrfs_find_space_info(fs_info, + BTRFS_BLOCK_GROUP_METADATA); +} + +struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_fs_info *fs_info, + enum btrfs_rsv_type type) +{ + struct btrfs_block_rsv *block_rsv; + + block_rsv = kmalloc_obj(*block_rsv, GFP_NOFS); + if (!block_rsv) + return NULL; + + btrfs_init_metadata_block_rsv(fs_info, block_rsv, type); + return block_rsv; +} + +void btrfs_free_block_rsv(struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *rsv) +{ + if (!rsv) + return; + btrfs_block_rsv_release(fs_info, rsv, (u64)-1, NULL); + kfree(rsv); +} + +int btrfs_block_rsv_add(struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *block_rsv, u64 num_bytes, + enum btrfs_reserve_flush_enum flush) +{ + int ret; + + if (num_bytes == 0) + return 0; + + ret = btrfs_reserve_metadata_bytes(block_rsv->space_info, num_bytes, flush); + if (!ret) + btrfs_block_rsv_add_bytes(block_rsv, num_bytes, true); + + return ret; +} + +int btrfs_block_rsv_check(struct btrfs_block_rsv *block_rsv, int min_percent) +{ + u64 num_bytes = 0; + int ret = -ENOSPC; + + spin_lock(&block_rsv->lock); + num_bytes = mult_perc(block_rsv->size, min_percent); + if (block_rsv->reserved >= num_bytes) + ret = 0; + spin_unlock(&block_rsv->lock); + + return ret; +} + +int btrfs_block_rsv_refill(struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *block_rsv, u64 num_bytes, + enum btrfs_reserve_flush_enum flush) +{ + int ret = -ENOSPC; + + if (!block_rsv) + return 0; + + spin_lock(&block_rsv->lock); + if (block_rsv->reserved >= num_bytes) + ret = 0; + else + num_bytes -= block_rsv->reserved; + spin_unlock(&block_rsv->lock); + + if (!ret) + return 0; + + ret = btrfs_reserve_metadata_bytes(block_rsv->space_info, num_bytes, flush); + if (!ret) { + btrfs_block_rsv_add_bytes(block_rsv, num_bytes, false); + return 0; + } + + return ret; +} + +u64 btrfs_block_rsv_release(struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *block_rsv, u64 num_bytes, + u64 *qgroup_to_release) +{ + struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; + struct btrfs_block_rsv *delayed_rsv = &fs_info->delayed_refs_rsv; + struct btrfs_block_rsv *target = NULL; + + /* + * If we are a delayed refs block reserve then push to the global + * reserve, otherwise dump into the global delayed refs reserve if it is + * not full. + */ + if (block_rsv->type == BTRFS_BLOCK_RSV_DELREFS) + target = global_rsv; + else if (block_rsv != global_rsv && !btrfs_block_rsv_full(delayed_rsv)) + target = delayed_rsv; + + if (target && block_rsv->space_info != target->space_info) + target = NULL; + + return block_rsv_release_bytes(fs_info, block_rsv, target, num_bytes, + qgroup_to_release); +} + +int btrfs_block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv, u64 num_bytes) +{ + int ret = -ENOSPC; + + spin_lock(&block_rsv->lock); + if (block_rsv->reserved >= num_bytes) { + block_rsv->reserved -= num_bytes; + if (block_rsv->reserved < block_rsv->size) + block_rsv->full = false; + ret = 0; + } + spin_unlock(&block_rsv->lock); + return ret; +} + +void btrfs_block_rsv_add_bytes(struct btrfs_block_rsv *block_rsv, + u64 num_bytes, bool update_size) +{ + spin_lock(&block_rsv->lock); + block_rsv->reserved += num_bytes; + if (update_size) + block_rsv->size += num_bytes; + else if (block_rsv->reserved >= block_rsv->size) + block_rsv->full = true; + spin_unlock(&block_rsv->lock); +} + +void btrfs_update_global_block_rsv(struct btrfs_fs_info *fs_info) +{ + struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv; + struct btrfs_space_info *sinfo; + struct btrfs_root *root, *tmp; + unsigned int min_items = 1; + u64 num_bytes; + + /* + * A full read-only mount (rescue options) cannot start transactions, + * so the global reserve is never consumed. Mark it as full and skip + * the accounting. + */ + if (btrfs_is_full_ro(fs_info)) { + spin_lock(&block_rsv->lock); + block_rsv->full = true; + spin_unlock(&block_rsv->lock); + return; + } + + sinfo = block_rsv->space_info; + num_bytes = btrfs_root_used(&fs_info->tree_root->root_item); + + /* + * The global block rsv is based on the size of the extent tree, the + * checksum tree and the root tree. If the fs is empty we want to set + * it to a minimal amount for safety. + * + * We also are going to need to modify the minimum of the tree root and + * any global roots we could touch. + */ + read_lock(&fs_info->global_root_lock); + rbtree_postorder_for_each_entry_safe(root, tmp, &fs_info->global_root_tree, + rb_node) { + if (btrfs_root_id(root) == BTRFS_EXTENT_TREE_OBJECTID || + btrfs_root_id(root) == BTRFS_CSUM_TREE_OBJECTID || + btrfs_root_id(root) == BTRFS_FREE_SPACE_TREE_OBJECTID) { + num_bytes += btrfs_root_used(&root->root_item); + min_items++; + } + } + read_unlock(&fs_info->global_root_lock); + + if (btrfs_fs_compat_ro(fs_info, BLOCK_GROUP_TREE)) { + num_bytes += btrfs_root_used(&fs_info->block_group_root->root_item); + min_items++; + } + + if (btrfs_fs_incompat(fs_info, RAID_STRIPE_TREE)) { + num_bytes += btrfs_root_used(&fs_info->stripe_root->root_item); + min_items++; + } + + /* + * But we also want to reserve enough space so we can do the fallback + * global reserve for an unlink, which is an additional + * BTRFS_UNLINK_METADATA_UNITS items. + * + * But we also need space for the delayed ref updates from the unlink, + * so add BTRFS_UNLINK_METADATA_UNITS units for delayed refs, one for + * each unlink metadata item. + */ + min_items += BTRFS_UNLINK_METADATA_UNITS; + + num_bytes = max_t(u64, num_bytes, + btrfs_calc_insert_metadata_size(fs_info, min_items) + + btrfs_calc_delayed_ref_bytes(fs_info, + BTRFS_UNLINK_METADATA_UNITS)); + + spin_lock(&sinfo->lock); + spin_lock(&block_rsv->lock); + + block_rsv->size = min_t(u64, num_bytes, SZ_512M); + + if (block_rsv->reserved < block_rsv->size) { + num_bytes = block_rsv->size - block_rsv->reserved; + btrfs_space_info_update_bytes_may_use(sinfo, num_bytes); + block_rsv->reserved = block_rsv->size; + } else if (block_rsv->reserved > block_rsv->size) { + num_bytes = block_rsv->reserved - block_rsv->size; + btrfs_space_info_update_bytes_may_use(sinfo, -num_bytes); + block_rsv->reserved = block_rsv->size; + btrfs_try_granting_tickets(sinfo); + } + + block_rsv->full = (block_rsv->reserved == block_rsv->size); + + if (block_rsv->size >= sinfo->total_bytes) + sinfo->force_alloc = CHUNK_ALLOC_FORCE; + spin_unlock(&block_rsv->lock); + spin_unlock(&sinfo->lock); +} + +void btrfs_init_root_block_rsv(struct btrfs_root *root) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + + switch (btrfs_root_id(root)) { + case BTRFS_CSUM_TREE_OBJECTID: + case BTRFS_EXTENT_TREE_OBJECTID: + case BTRFS_FREE_SPACE_TREE_OBJECTID: + case BTRFS_BLOCK_GROUP_TREE_OBJECTID: + case BTRFS_RAID_STRIPE_TREE_OBJECTID: + root->block_rsv = &fs_info->delayed_refs_rsv; + break; + case BTRFS_ROOT_TREE_OBJECTID: + case BTRFS_DEV_TREE_OBJECTID: + case BTRFS_QUOTA_TREE_OBJECTID: + root->block_rsv = &fs_info->global_block_rsv; + break; + case BTRFS_CHUNK_TREE_OBJECTID: + root->block_rsv = &fs_info->chunk_block_rsv; + break; + case BTRFS_TREE_LOG_OBJECTID: + root->block_rsv = &fs_info->treelog_rsv; + break; + case BTRFS_REMAP_TREE_OBJECTID: + root->block_rsv = &fs_info->remap_block_rsv; + break; + default: + root->block_rsv = NULL; + break; + } +} + +void btrfs_init_global_block_rsv(struct btrfs_fs_info *fs_info) +{ + struct btrfs_space_info *space_info; + + space_info = btrfs_find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM); + fs_info->chunk_block_rsv.space_info = space_info; + + space_info = btrfs_find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA_REMAP); + fs_info->remap_block_rsv.space_info = space_info; + + space_info = btrfs_find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA); + fs_info->global_block_rsv.space_info = space_info; + fs_info->trans_block_rsv.space_info = space_info; + fs_info->empty_block_rsv.space_info = space_info; + fs_info->delayed_block_rsv.space_info = space_info; + fs_info->delayed_refs_rsv.space_info = space_info; + + /* The treelog_rsv uses a dedicated space_info on the zoned mode. */ + if (!btrfs_is_zoned(fs_info)) { + fs_info->treelog_rsv.space_info = space_info; + } else { + ASSERT(space_info->sub_group[0]->subgroup_id == BTRFS_SUB_GROUP_TREELOG); + fs_info->treelog_rsv.space_info = space_info->sub_group[0]; + } + + btrfs_update_global_block_rsv(fs_info); +} + +void btrfs_release_global_block_rsv(struct btrfs_fs_info *fs_info) +{ + btrfs_block_rsv_release(fs_info, &fs_info->global_block_rsv, (u64)-1, + NULL); + WARN_ON(fs_info->trans_block_rsv.size > 0); + WARN_ON(fs_info->trans_block_rsv.reserved > 0); + WARN_ON(fs_info->chunk_block_rsv.size > 0); + WARN_ON(fs_info->chunk_block_rsv.reserved > 0); + WARN_ON(fs_info->remap_block_rsv.size > 0); + WARN_ON(fs_info->remap_block_rsv.reserved > 0); + WARN_ON(fs_info->delayed_block_rsv.size > 0); + WARN_ON(fs_info->delayed_block_rsv.reserved > 0); + WARN_ON(fs_info->delayed_refs_rsv.reserved > 0); + WARN_ON(fs_info->delayed_refs_rsv.size > 0); +} + +static struct btrfs_block_rsv *get_block_rsv( + const struct btrfs_trans_handle *trans, + const struct btrfs_root *root) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_block_rsv *block_rsv = NULL; + + if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state) || + (root == fs_info->uuid_root) || + (trans->adding_csums && btrfs_root_id(root) == BTRFS_CSUM_TREE_OBJECTID)) + block_rsv = trans->block_rsv; + + if (!block_rsv) + block_rsv = root->block_rsv; + + if (!block_rsv) + block_rsv = &fs_info->empty_block_rsv; + + return block_rsv; +} + +struct btrfs_block_rsv *btrfs_use_block_rsv(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + u32 blocksize) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_block_rsv *block_rsv; + struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; + int ret; + bool global_updated = false; + + block_rsv = get_block_rsv(trans, root); + + if (unlikely(btrfs_block_rsv_size(block_rsv) == 0)) + goto try_reserve; +again: + ret = btrfs_block_rsv_use_bytes(block_rsv, blocksize); + if (!ret) + return block_rsv; + + if (block_rsv->failfast) + return ERR_PTR(ret); + + if (block_rsv->type == BTRFS_BLOCK_RSV_GLOBAL && !global_updated) { + global_updated = true; + btrfs_update_global_block_rsv(fs_info); + goto again; + } + + /* + * The global reserve still exists to save us from ourselves, so don't + * warn_on if we are short on our delayed refs reserve. + */ + if (block_rsv->type != BTRFS_BLOCK_RSV_DELREFS && + btrfs_test_opt(fs_info, ENOSPC_DEBUG)) { + static DEFINE_RATELIMIT_STATE(_rs, + DEFAULT_RATELIMIT_INTERVAL * 10, + /*DEFAULT_RATELIMIT_BURST*/ 1); + if (__ratelimit(&_rs)) + WARN(1, KERN_DEBUG + "BTRFS: block rsv %d returned %d\n", + block_rsv->type, ret); + } +try_reserve: + ret = btrfs_reserve_metadata_bytes(block_rsv->space_info, blocksize, + BTRFS_RESERVE_NO_FLUSH); + if (!ret) + return block_rsv; + + /* + * If we are being used for updating a log tree, fail immediately, which + * makes the fsync fallback to a transaction commit. + * + * We don't want to consume from the global block reserve, as that is + * precious space that may be needed to do updates to some trees for + * which we don't reserve space during a transaction commit (update root + * items in the root tree, device stat items in the device tree and + * quota tree updates, see btrfs_init_root_block_rsv()), or to fallback + * to in case we did not reserve enough space to run delayed items, + * delayed references, or anything else we need in order to avoid a + * transaction abort. + * + * We also don't want to do a reservation in flush emergency mode, as + * we end up using metadata that could be critical to allow a + * transaction to complete successfully and therefore increase the + * chances for a transaction abort. + * + * Log trees are an optimization and should never consume from the + * global reserve or be allowed overcommitting metadata. + */ + if (btrfs_root_id(root) == BTRFS_TREE_LOG_OBJECTID) + return ERR_PTR(ret); + + /* + * If we couldn't reserve metadata bytes try and use some from + * the global reserve if its space type is the same as the global + * reservation. + */ + if (block_rsv->type != BTRFS_BLOCK_RSV_GLOBAL && + block_rsv->space_info == global_rsv->space_info) { + ret = btrfs_block_rsv_use_bytes(global_rsv, blocksize); + if (!ret) + return global_rsv; + } + + /* + * All hope is lost, but of course our reservations are overly + * pessimistic, so instead of possibly having an ENOSPC abort here, try + * one last time to force a reservation if there's enough actual space + * on disk to make the reservation. + */ + ret = btrfs_reserve_metadata_bytes(block_rsv->space_info, blocksize, + BTRFS_RESERVE_FLUSH_EMERGENCY); + if (!ret) + return block_rsv; + + return ERR_PTR(ret); +} + +int btrfs_check_trunc_cache_free_space(const struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *rsv) +{ + u64 needed_bytes; + int ret; + + /* 1 for slack space, 1 for updating the inode */ + needed_bytes = btrfs_calc_insert_metadata_size(fs_info, 1) + + btrfs_calc_metadata_size(fs_info, 1); + + spin_lock(&rsv->lock); + if (rsv->reserved < needed_bytes) + ret = -ENOSPC; + else + ret = 0; + spin_unlock(&rsv->lock); + return ret; +} diff --git a/fs/btrfs/block-rsv.h b/fs/btrfs/block-rsv.h new file mode 100644 index 000000000..8359fb96b --- /dev/null +++ b/fs/btrfs/block-rsv.h @@ -0,0 +1,145 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_BLOCK_RSV_H +#define BTRFS_BLOCK_RSV_H + +#include +#include +#include + +struct btrfs_trans_handle; +struct btrfs_root; +struct btrfs_space_info; +struct btrfs_block_rsv; +struct btrfs_fs_info; +enum btrfs_reserve_flush_enum; + +/* + * Types of block reserves + */ +enum btrfs_rsv_type { + BTRFS_BLOCK_RSV_GLOBAL, + BTRFS_BLOCK_RSV_DELALLOC, + BTRFS_BLOCK_RSV_TRANS, + BTRFS_BLOCK_RSV_CHUNK, + BTRFS_BLOCK_RSV_REMAP, + BTRFS_BLOCK_RSV_DELOPS, + BTRFS_BLOCK_RSV_DELREFS, + BTRFS_BLOCK_RSV_TREELOG, + BTRFS_BLOCK_RSV_EMPTY, + BTRFS_BLOCK_RSV_TEMP, +}; + +struct btrfs_block_rsv { + u64 size; + u64 reserved; + struct btrfs_space_info *space_info; + spinlock_t lock; + bool full; + bool failfast; + /* Block reserve type, one of BTRFS_BLOCK_RSV_* */ + enum btrfs_rsv_type type:8; + + /* + * Qgroup equivalent for @size @reserved + * + * Unlike normal @size/@reserved for inode rsv, qgroup doesn't care + * about things like csum size nor how many tree blocks it will need to + * reserve. + * + * Qgroup cares more about net change of the extent usage. + * + * So for one newly inserted file extent, in worst case it will cause + * leaf split and level increase, nodesize for each file extent is + * already too much. + * + * In short, qgroup_size/reserved is the upper limit of possible needed + * qgroup metadata reservation. + */ + u64 qgroup_rsv_size; + u64 qgroup_rsv_reserved; +}; + +void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv, enum btrfs_rsv_type type); +void btrfs_init_root_block_rsv(struct btrfs_root *root); +struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_fs_info *fs_info, + enum btrfs_rsv_type type); +void btrfs_init_metadata_block_rsv(struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *rsv, + enum btrfs_rsv_type type); +void btrfs_free_block_rsv(struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *rsv); +int btrfs_block_rsv_add(struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *block_rsv, u64 num_bytes, + enum btrfs_reserve_flush_enum flush); +int btrfs_block_rsv_check(struct btrfs_block_rsv *block_rsv, int min_percent); +int btrfs_block_rsv_refill(struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *block_rsv, u64 num_bytes, + enum btrfs_reserve_flush_enum flush); +int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src_rsv, + struct btrfs_block_rsv *dst_rsv, u64 num_bytes, + bool update_size); +int btrfs_block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv, u64 num_bytes); +void btrfs_block_rsv_add_bytes(struct btrfs_block_rsv *block_rsv, + u64 num_bytes, bool update_size); +u64 btrfs_block_rsv_release(struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *block_rsv, + u64 num_bytes, u64 *qgroup_to_release); +void btrfs_update_global_block_rsv(struct btrfs_fs_info *fs_info); +void btrfs_init_global_block_rsv(struct btrfs_fs_info *fs_info); +void btrfs_release_global_block_rsv(struct btrfs_fs_info *fs_info); +struct btrfs_block_rsv *btrfs_use_block_rsv(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + u32 blocksize); +int btrfs_check_trunc_cache_free_space(const struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *rsv); +static inline void btrfs_unuse_block_rsv(struct btrfs_fs_info *fs_info, + struct btrfs_block_rsv *block_rsv, + u32 blocksize) +{ + btrfs_block_rsv_add_bytes(block_rsv, blocksize, false); + btrfs_block_rsv_release(fs_info, block_rsv, 0, NULL); +} + +/* + * Fast path to check if the reserve is full, may be carefully used outside of + * locks. + */ +static inline bool btrfs_block_rsv_full(const struct btrfs_block_rsv *rsv) +{ + return data_race(rsv->full); +} + +/* + * Get the reserved mount of a block reserve in a context where getting a stale + * value is acceptable, instead of accessing it directly and trigger data race + * warning from KCSAN. + */ +static inline u64 btrfs_block_rsv_reserved(struct btrfs_block_rsv *rsv) +{ + u64 ret; + + spin_lock(&rsv->lock); + ret = rsv->reserved; + spin_unlock(&rsv->lock); + + return ret; +} + +/* + * Get the size of a block reserve in a context where getting a stale value is + * acceptable, instead of accessing it directly and trigger data race warning + * from KCSAN. + */ +static inline u64 btrfs_block_rsv_size(struct btrfs_block_rsv *rsv) +{ + u64 ret; + + spin_lock(&rsv->lock); + ret = rsv->size; + spin_unlock(&rsv->lock); + + return ret; +} + +#endif /* BTRFS_BLOCK_RSV_H */ diff --git a/fs/btrfs/btrfs_inode.h b/fs/btrfs/btrfs_inode.h new file mode 100644 index 000000000..1082fa92c --- /dev/null +++ b/fs/btrfs/btrfs_inode.h @@ -0,0 +1,643 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#ifndef BTRFS_INODE_H +#define BTRFS_INODE_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ctree.h" +#include "block-rsv.h" +#include "extent_map.h" +#include "extent-io-tree.h" + +struct posix_acl; +struct iov_iter; +struct writeback_control; +struct btrfs_root; +struct btrfs_fs_info; +struct btrfs_trans_handle; +struct btrfs_bio; +struct btrfs_file_extent; +struct btrfs_delayed_node; + +/* + * Since we search a directory based on f_pos (struct dir_context::pos) we have + * to start at 2 since '.' and '..' have f_pos of 0 and 1 respectively, so + * everybody else has to start at 2 (see btrfs_real_readdir() and dir_emit_dots()). + */ +#define BTRFS_DIR_START_INDEX 2 + +/* + * ordered_data_close is set by truncate when a file that used + * to have good data has been truncated to zero. When it is set + * the btrfs file release call will add this inode to the + * ordered operations list so that we make sure to flush out any + * new data the application may have written before commit. + */ +enum { + BTRFS_INODE_FLUSH_ON_CLOSE, + BTRFS_INODE_DUMMY, + BTRFS_INODE_IN_DEFRAG, + BTRFS_INODE_HAS_ASYNC_EXTENT, + /* + * Always set under the VFS' inode lock, otherwise it can cause races + * during fsync (we start as a fast fsync and then end up in a full + * fsync racing with ordered extent completion). + */ + BTRFS_INODE_NEEDS_FULL_SYNC, + BTRFS_INODE_COPY_EVERYTHING, + BTRFS_INODE_HAS_PROPS, + BTRFS_INODE_SNAPSHOT_FLUSH, + /* + * Set and used when logging an inode and it serves to signal that an + * inode does not have xattrs, so subsequent fsyncs can avoid searching + * for xattrs to log. This bit must be cleared whenever a xattr is added + * to an inode. + */ + BTRFS_INODE_NO_XATTRS, + /* + * Set when we are in a context where we need to start a transaction and + * have dirty pages with the respective file range locked. This is to + * ensure that when reserving space for the transaction, if we are low + * on available space and need to flush delalloc, we will not flush + * delalloc for this inode, because that could result in a deadlock (on + * the file range, inode's io_tree). + */ + BTRFS_INODE_NO_DELALLOC_FLUSH, + /* + * Set when we are working on enabling verity for a file. Computing and + * writing the whole Merkle tree can take a while so we want to prevent + * races where two separate tasks attempt to simultaneously start verity + * on the same file. + */ + BTRFS_INODE_VERITY_IN_PROGRESS, + /* Set when this inode is a free space inode. */ + BTRFS_INODE_FREE_SPACE_INODE, + /* Set when there are no capabilities in XATTs for the inode. */ + BTRFS_INODE_NO_CAP_XATTR, + /* + * Set if an error happened when doing a COW write before submitting a + * bio or during writeback. Used for both buffered writes and direct IO + * writes. This is to signal a fast fsync that it has to wait for + * ordered extents to complete and therefore not log extent maps that + * point to unwritten extents (when an ordered extent completes and it + * has the BTRFS_ORDERED_IOERR flag set, it drops extent maps in its + * range). + */ + BTRFS_INODE_COW_WRITE_ERROR, + /* + * Indicate this is a directory that points to a subvolume for which + * there is no root reference item. That's a case like the following: + * + * $ btrfs subvolume create /mnt/parent + * $ btrfs subvolume create /mnt/parent/child + * $ btrfs subvolume snapshot /mnt/parent /mnt/snap + * + * If subvolume "parent" is root 256, subvolume "child" is root 257 and + * snapshot "snap" is root 258, then there's no root reference item (key + * BTRFS_ROOT_REF_KEY in the root tree) for the subvolume "child" + * associated to root 258 (the snapshot) - there's only for the root + * of the "parent" subvolume (root 256). In the chunk root we have a + * (256 BTRFS_ROOT_REF_KEY 257) key but we don't have a + * (258 BTRFS_ROOT_REF_KEY 257) key - the sames goes for backrefs, we + * have a (257 BTRFS_ROOT_BACKREF_KEY 256) but we don't have a + * (257 BTRFS_ROOT_BACKREF_KEY 258) key. + * + * So when opening the "child" dentry from the snapshot's directory, + * we don't find a root ref item and we create a stub inode. This is + * done at new_simple_dir(), called from btrfs_lookup_dentry(). + */ + BTRFS_INODE_ROOT_STUB, +}; + +/* in memory btrfs inode */ +struct btrfs_inode { + /* which subvolume this inode belongs to */ + struct btrfs_root *root; + + /* Cached value of inode property 'compression'. */ + u8 prop_compress; + + /* + * Force compression on the file using the defrag ioctl, could be + * different from prop_compress and takes precedence if set. + */ + u8 defrag_compress; + s8 defrag_compress_level; + + /* + * Lock for counters and all fields used to determine if the inode is in + * the log or not (last_trans, last_sub_trans, last_log_commit, + * logged_trans), to access/update delalloc_bytes, new_delalloc_bytes, + * defrag_bytes, disk_i_size, outstanding_extents, csum_bytes and to + * update the VFS' inode number of bytes used. + * Also protects setting struct file::private_data. + */ + spinlock_t lock; + + /* the extent_tree has caches of all the extent mappings to disk */ + struct extent_map_tree extent_tree; + + /* the io_tree does range state (DIRTY, LOCKED etc) */ + struct extent_io_tree io_tree; + + /* + * Keep track of where the inode has extent items mapped in order to + * make sure the i_size adjustments are accurate. Not required when the + * filesystem is NO_HOLES, the status can't be set while mounted as + * it's a mkfs-time feature. + */ + struct extent_io_tree *file_extent_tree; + + /* held while logging the inode in tree-log.c */ + struct mutex log_mutex; + + /* + * Counters to keep track of the number of extent item's we may use due + * to delalloc and such. outstanding_extents is the number of extent + * items we think we'll end up using, and reserved_extents is the number + * of extent items we've reserved metadata for. Protected by 'lock'. + */ + unsigned outstanding_extents; + + /* used to order data wrt metadata */ + spinlock_t ordered_tree_lock; + struct rb_root ordered_tree; + struct rb_node *ordered_tree_last; + + /* list of all the delalloc inodes in the FS. There are times we need + * to write all the delalloc pages to disk, and this list is used + * to walk them all. + */ + struct list_head delalloc_inodes; + + unsigned long runtime_flags; + + /* full 64 bit generation number, struct vfs_inode doesn't have a big + * enough field for this. + */ + u64 generation; + + /* + * ID of the transaction handle that last modified this inode. + * Protected by 'lock'. + */ + u64 last_trans; + + /* + * ID of the transaction that last logged this inode. + * Protected by 'lock'. + */ + u64 logged_trans; + + /* + * Log transaction ID when this inode was last modified. + * Protected by 'lock'. + */ + int last_sub_trans; + + /* A local copy of root's last_log_commit. Protected by 'lock'. */ + int last_log_commit; + + union { + /* + * Total number of bytes pending delalloc, used by stat to + * calculate the real block usage of the file. This is used + * only for files. Protected by 'lock'. + */ + u64 delalloc_bytes; + /* + * The lowest possible index of the next dir index key which + * points to an inode that needs to be logged. + * This is used only for directories. + * Use the helpers btrfs_get_first_dir_index_to_log() and + * btrfs_set_first_dir_index_to_log() to access this field. + */ + u64 first_dir_index_to_log; + }; + + union { + /* + * Total number of bytes pending delalloc that fall within a file + * range that is either a hole or beyond EOF (and no prealloc extent + * exists in the range). This is always <= delalloc_bytes and this + * is used only for files. Protected by 'lock'. + */ + u64 new_delalloc_bytes; + /* + * The offset of the last dir index key that was logged. + * This is used only for directories. Protected by 'log_mutex'. + */ + u64 last_dir_index_offset; + }; + + union { + /* + * Total number of bytes pending defrag, used by stat to check whether + * it needs COW. Protected by 'lock'. + * Used by inodes other than the data relocation inode. + */ + u64 defrag_bytes; + + /* + * Logical address of the block group being relocated. + * Used only by the data relocation inode. + */ + u64 reloc_block_group_start; + }; + + /* + * The size of the file stored in the metadata on disk. data=ordered + * means the in-memory i_size might be larger than the size on disk + * because not all the blocks are written yet. Protected by 'lock'. + */ + u64 disk_i_size; + + union { + /* + * If this is a directory then index_cnt is the counter for the + * index number for new files that are created. For an empty + * directory, this must be initialized to BTRFS_DIR_START_INDEX. + */ + u64 index_cnt; + + /* + * If this is not a directory, this is the number of bytes + * outstanding that are going to need csums. This is used in + * ENOSPC accounting. Protected by 'lock'. + */ + u64 csum_bytes; + }; + + /* Cache the directory index number to speed the dir/file remove */ + u64 dir_index; + + /* the fsync log has some corner cases that mean we have to check + * directories to see if any unlinks have been done before + * the directory was logged. See tree-log.c for all the + * details + */ + u64 last_unlink_trans; + + union { + /* + * The id/generation of the last transaction where this inode + * was either the source or the destination of a clone/dedupe + * operation. Used when logging an inode to know if there are + * shared extents that need special care when logging checksum + * items, to avoid duplicate checksum items in a log (which can + * lead to a corruption where we end up with missing checksum + * ranges after log replay). Protected by the VFS inode lock. + * Used for regular files only. + */ + u64 last_reflink_trans; + + /* + * In case this a root stub inode (BTRFS_INODE_ROOT_STUB flag set), + * the ID of that root. + */ + u64 ref_root_id; + }; + + /* Backwards incompatible flags, lower half of inode_item::flags */ + u32 flags; + /* Read-only compatibility flags, upper half of inode_item::flags */ + u32 ro_flags; + + struct btrfs_block_rsv block_rsv; + + struct btrfs_delayed_node *delayed_node; + + /* File creation time. */ + u64 i_otime_sec; + u32 i_otime_nsec; + + /* Hook into fs_info->delayed_iputs */ + struct list_head delayed_iput; + + struct rw_semaphore i_mmap_lock; + + struct inode vfs_inode; +}; + +static inline u64 btrfs_get_first_dir_index_to_log(const struct btrfs_inode *inode) +{ + return READ_ONCE(inode->first_dir_index_to_log); +} + +static inline void btrfs_set_first_dir_index_to_log(struct btrfs_inode *inode, + u64 index) +{ + WRITE_ONCE(inode->first_dir_index_to_log, index); +} + +/* Type checked and const-preserving VFS inode -> btrfs inode. */ +#define BTRFS_I(_inode) \ + _Generic(_inode, \ + struct inode *: container_of(_inode, struct btrfs_inode, vfs_inode), \ + const struct inode *: (const struct btrfs_inode *)container_of( \ + _inode, const struct btrfs_inode, vfs_inode)) + +static inline unsigned long btrfs_inode_hash(u64 objectid, + const struct btrfs_root *root) +{ + u64 h = objectid ^ (root->root_key.objectid * GOLDEN_RATIO_PRIME); + +#if BITS_PER_LONG == 32 + h = (h >> 32) ^ (h & 0xffffffff); +#endif + + return (unsigned long)h; +} + +static inline u64 btrfs_ino(const struct btrfs_inode *inode) +{ + return inode->vfs_inode.i_ino; +} + +static inline void btrfs_get_inode_key(const struct btrfs_inode *inode, + struct btrfs_key *key) +{ + key->objectid = btrfs_ino(inode); + key->type = BTRFS_INODE_ITEM_KEY; + key->offset = 0; +} + +static inline void btrfs_set_inode_number(struct btrfs_inode *inode, u64 ino) +{ + inode->vfs_inode.i_ino = ino; +} + +static inline void btrfs_i_size_write(struct btrfs_inode *inode, u64 size) +{ + i_size_write(&inode->vfs_inode, size); + inode->disk_i_size = size; +} + +static inline bool btrfs_is_free_space_inode(const struct btrfs_inode *inode) +{ + return test_bit(BTRFS_INODE_FREE_SPACE_INODE, &inode->runtime_flags); +} + +static inline bool is_data_inode(const struct btrfs_inode *inode) +{ + return btrfs_ino(inode) != BTRFS_BTREE_INODE_OBJECTID; +} + +static inline void btrfs_mod_outstanding_extents(struct btrfs_inode *inode, + int mod) +{ + lockdep_assert_held(&inode->lock); + inode->outstanding_extents += mod; + if (btrfs_is_free_space_inode(inode)) + return; + trace_btrfs_inode_mod_outstanding_extents(inode->root, btrfs_ino(inode), + mod, inode->outstanding_extents); +} + +/* + * Called every time after doing a buffered, direct IO or memory mapped write. + * + * This is to ensure that if we write to a file that was previously fsynced in + * the current transaction, then try to fsync it again in the same transaction, + * we will know that there were changes in the file and that it needs to be + * logged. + */ +static inline void btrfs_set_inode_last_sub_trans(struct btrfs_inode *inode) +{ + spin_lock(&inode->lock); + inode->last_sub_trans = inode->root->log_transid; + spin_unlock(&inode->lock); +} + +/* + * Should be called while holding the inode's VFS lock in exclusive mode, or + * while holding the inode's mmap lock (struct btrfs_inode::i_mmap_lock) in + * either shared or exclusive mode, or in a context where no one else can access + * the inode concurrently (during inode creation or when loading an inode from + * disk). + */ +static inline void btrfs_set_inode_full_sync(struct btrfs_inode *inode) +{ + set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags); + /* + * The inode may have been part of a reflink operation in the last + * transaction that modified it, and then a fsync has reset the + * last_reflink_trans to avoid subsequent fsyncs in the same + * transaction to do unnecessary work. So update last_reflink_trans + * to the last_trans value (we have to be pessimistic and assume a + * reflink happened). + * + * The ->last_trans is protected by the inode's spinlock and we can + * have a concurrent ordered extent completion update it. Also set + * last_reflink_trans to ->last_trans only if the former is less than + * the later, because we can be called in a context where + * last_reflink_trans was set to the current transaction generation + * while ->last_trans was not yet updated in the current transaction, + * and therefore has a lower value. + */ + spin_lock(&inode->lock); + if (inode->last_reflink_trans < inode->last_trans) + inode->last_reflink_trans = inode->last_trans; + spin_unlock(&inode->lock); +} + +static inline bool btrfs_inode_in_log(struct btrfs_inode *inode, u64 generation) +{ + bool ret = false; + + spin_lock(&inode->lock); + if (inode->logged_trans == generation && + inode->last_sub_trans <= inode->last_log_commit && + inode->last_sub_trans <= btrfs_get_root_last_log_commit(inode->root)) + ret = true; + spin_unlock(&inode->lock); + return ret; +} + +/* + * Check if the inode has flags compatible with compression + */ +static inline bool btrfs_inode_can_compress(const struct btrfs_inode *inode) +{ + if (inode->flags & BTRFS_INODE_NODATACOW || + inode->flags & BTRFS_INODE_NODATASUM) + return false; + if (btrfs_is_data_reloc_root(inode->root)) + return false; + return true; +} + +static inline void btrfs_assert_inode_locked(struct btrfs_inode *inode) +{ + /* Immediately trigger a crash if the inode is not locked. */ + ASSERT(inode_is_locked(&inode->vfs_inode)); + /* Trigger a splat in dmesg if this task is not holding the lock. */ + lockdep_assert_held(&inode->vfs_inode.i_rwsem); +} + +static inline void btrfs_update_inode_mapping_flags(struct btrfs_inode *inode) +{ + if (inode->flags & BTRFS_INODE_NODATASUM) + mapping_clear_stable_writes(inode->vfs_inode.i_mapping); + else + mapping_set_stable_writes(inode->vfs_inode.i_mapping); +} + +static inline void btrfs_set_inode_mapping_order(struct btrfs_inode *inode) +{ + /* Metadata inode should not reach here. */ + ASSERT(is_data_inode(inode)); + + mapping_set_folio_order_range(inode->vfs_inode.i_mapping, + inode->root->fs_info->block_min_order, + inode->root->fs_info->block_max_order); +} + +void btrfs_calculate_block_csum_folio(struct btrfs_fs_info *fs_info, + const phys_addr_t paddr, u8 *dest); +void btrfs_calculate_block_csum_pages(struct btrfs_fs_info *fs_info, + const phys_addr_t paddrs[], u8 *dest); +int btrfs_check_block_csum(struct btrfs_fs_info *fs_info, phys_addr_t paddr, u8 *csum, + const u8 * const csum_expected); +bool btrfs_data_csum_ok(struct btrfs_bio *bbio, struct btrfs_device *dev, + u32 bio_offset, const phys_addr_t paddrs[]); +noinline int can_nocow_extent(struct btrfs_inode *inode, u64 offset, u64 *len, + struct btrfs_file_extent *file_extent, + bool nowait); + +void btrfs_del_delalloc_inode(struct btrfs_inode *inode); +struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry); +int btrfs_set_inode_index(struct btrfs_inode *dir, u64 *index); +int btrfs_unlink_inode(struct btrfs_trans_handle *trans, + struct btrfs_inode *dir, struct btrfs_inode *inode, + const struct fscrypt_str *name); +int btrfs_add_link(struct btrfs_trans_handle *trans, + struct btrfs_inode *parent_inode, struct btrfs_inode *inode, + const struct fscrypt_str *name, bool add_backref, u64 index); +int btrfs_delete_subvolume(struct btrfs_inode *dir, struct dentry *dentry); +int btrfs_truncate_block(struct btrfs_inode *inode, u64 offset, u64 start, u64 end); + +int btrfs_start_delalloc_snapshot(struct btrfs_root *root, bool in_reclaim_context); +int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, long nr, + bool in_reclaim_context); +int btrfs_set_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, + unsigned int extra_bits, + struct extent_state **cached_state); +int btrfs_reset_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, + unsigned int extra_bits, struct extent_state **cached_state); + +struct btrfs_new_inode_args { + /* Input */ + struct inode *dir; + struct dentry *dentry; + struct inode *inode; + bool orphan; + bool subvol; + + /* Output from btrfs_new_inode_prepare(), input to btrfs_create_new_inode(). */ + struct posix_acl *default_acl; + struct posix_acl *acl; + struct fscrypt_name fname; +}; + +int btrfs_new_inode_prepare(struct btrfs_new_inode_args *args, + unsigned int *trans_num_items); +int btrfs_create_new_inode(struct btrfs_trans_handle *trans, + struct btrfs_new_inode_args *args); +void btrfs_new_inode_args_destroy(struct btrfs_new_inode_args *args); +struct inode *btrfs_new_subvol_inode(struct mnt_idmap *idmap, + struct inode *dir); + void btrfs_set_delalloc_extent(struct btrfs_inode *inode, struct extent_state *state, + u32 bits); +void btrfs_clear_delalloc_extent(struct btrfs_inode *inode, + struct extent_state *state, u32 bits); +void btrfs_merge_delalloc_extent(struct btrfs_inode *inode, struct extent_state *new, + struct extent_state *other); +void btrfs_split_delalloc_extent(struct btrfs_inode *inode, + struct extent_state *orig, u64 split); +void btrfs_evict_inode(struct inode *inode); +struct inode *btrfs_alloc_inode(struct super_block *sb); +void btrfs_destroy_inode(struct inode *inode); +void btrfs_free_inode(struct inode *inode); +int btrfs_drop_inode(struct inode *inode); +int __init btrfs_init_cachep(void); +void __cold btrfs_destroy_cachep(void); +struct btrfs_inode *btrfs_iget_path(u64 ino, struct btrfs_root *root, + struct btrfs_path *path); +struct btrfs_inode *btrfs_iget(u64 ino, struct btrfs_root *root); +struct extent_map *btrfs_get_extent(struct btrfs_inode *inode, + struct folio *folio, u64 start, u64 len); +int btrfs_update_inode(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode); +int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode); +int btrfs_orphan_add(struct btrfs_trans_handle *trans, struct btrfs_inode *inode); +int btrfs_orphan_cleanup(struct btrfs_root *root); +int btrfs_cont_expand(struct btrfs_inode *inode, loff_t oldsize, loff_t size); +void btrfs_add_delayed_iput(struct btrfs_inode *inode); +void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info); +int btrfs_wait_on_delayed_iputs(struct btrfs_fs_info *fs_info); +int btrfs_prealloc_file_range(struct inode *inode, int mode, + u64 start, u64 num_bytes, u64 min_size, + loff_t actual_len, u64 *alloc_hint); +int btrfs_prealloc_file_range_trans(struct inode *inode, + struct btrfs_trans_handle *trans, int mode, + u64 start, u64 num_bytes, u64 min_size, + loff_t actual_len, u64 *alloc_hint); +int btrfs_run_delalloc_range(struct btrfs_inode *inode, struct folio *locked_folio, + u64 start, u64 end, struct writeback_control *wbc); +void btrfs_queue_writepage_fixup(struct btrfs_inode *inode, struct folio *folio); +int btrfs_encoded_io_compression_from_extent(struct btrfs_fs_info *fs_info, + int compress_type); +int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode, + u64 disk_bytenr, u64 disk_io_size, + struct page **pages, void *uring_ctx); +ssize_t btrfs_encoded_read(struct kiocb *iocb, struct iov_iter *iter, + struct btrfs_ioctl_encoded_io_args *encoded, + struct extent_state **cached_state, + u64 *disk_bytenr, u64 *disk_io_size); +ssize_t btrfs_encoded_read_regular(struct kiocb *iocb, struct iov_iter *iter, + u64 start, u64 lockend, + struct extent_state **cached_state, + u64 disk_bytenr, u64 disk_io_size, + size_t count, bool compressed, bool *unlocked); +ssize_t btrfs_do_encoded_write(struct kiocb *iocb, struct iov_iter *from, + const struct btrfs_ioctl_encoded_io_args *encoded); + +struct btrfs_inode *btrfs_find_first_inode(struct btrfs_root *root, u64 min_ino); + +extern const struct dentry_operations btrfs_dentry_operations; + +/* Inode locking type flags, by default the exclusive lock is taken. */ +enum btrfs_ilock_type { + ENUM_BIT(BTRFS_ILOCK_SHARED), + ENUM_BIT(BTRFS_ILOCK_TRY), + ENUM_BIT(BTRFS_ILOCK_MMAP), +}; + +int btrfs_inode_lock(struct btrfs_inode *inode, unsigned int ilock_flags); +void btrfs_inode_unlock(struct btrfs_inode *inode, unsigned int ilock_flags); +void btrfs_update_inode_bytes(struct btrfs_inode *inode, const u64 add_bytes, + const u64 del_bytes); +void btrfs_assert_inode_range_clean(struct btrfs_inode *inode, u64 start, u64 end); +u64 btrfs_get_extent_allocation_hint(struct btrfs_inode *inode, u64 start, + u64 num_bytes); +struct extent_map *btrfs_create_io_em(struct btrfs_inode *inode, u64 start, + const struct btrfs_file_extent *file_extent, + int type); + +#endif diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c new file mode 100644 index 000000000..c62b5148d --- /dev/null +++ b/fs/btrfs/compression.c @@ -0,0 +1,1652 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2008 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "misc.h" +#include "ctree.h" +#include "fs.h" +#include "btrfs_inode.h" +#include "bio.h" +#include "ordered-data.h" +#include "compression.h" +#include "extent_io.h" +#include "extent_map.h" +#include "subpage.h" +#include "messages.h" +#include "super.h" + +static struct bio_set btrfs_compressed_bioset; + +static const char* const btrfs_compress_types[] = { "", "zlib", "lzo", "zstd" }; + +const char* btrfs_compress_type2str(enum btrfs_compression_type type) +{ + switch (type) { + case BTRFS_COMPRESS_ZLIB: + case BTRFS_COMPRESS_LZO: + case BTRFS_COMPRESS_ZSTD: + case BTRFS_COMPRESS_NONE: + return btrfs_compress_types[type]; + default: + break; + } + + return NULL; +} + +static inline struct compressed_bio *to_compressed_bio(struct btrfs_bio *bbio) +{ + return container_of(bbio, struct compressed_bio, bbio); +} + +static struct compressed_bio *alloc_compressed_bio(struct btrfs_inode *inode, + u64 start, blk_opf_t op, + btrfs_bio_end_io_t end_io) +{ + struct btrfs_bio *bbio; + + bbio = btrfs_bio(bio_alloc_bioset(NULL, BTRFS_MAX_COMPRESSED_PAGES, op, + GFP_NOFS, &btrfs_compressed_bioset)); + btrfs_bio_init(bbio, inode, start, end_io, NULL); + return to_compressed_bio(bbio); +} + +bool btrfs_compress_is_valid_type(const char *str, size_t len) +{ + int i; + + for (i = 1; i < ARRAY_SIZE(btrfs_compress_types); i++) { + size_t comp_len = strlen(btrfs_compress_types[i]); + + if (len < comp_len) + continue; + + if (!strncmp(btrfs_compress_types[i], str, comp_len)) + return true; + } + return false; +} + +static int compression_decompress_bio(struct list_head *ws, + struct compressed_bio *cb) +{ + switch (cb->compress_type) { + case BTRFS_COMPRESS_ZLIB: return zlib_decompress_bio(ws, cb); + case BTRFS_COMPRESS_LZO: return lzo_decompress_bio(ws, cb); + case BTRFS_COMPRESS_ZSTD: return zstd_decompress_bio(ws, cb); + case BTRFS_COMPRESS_NONE: + default: + /* + * This can't happen, the type is validated several times + * before we get here. + */ + BUG(); + } +} + +static int compression_decompress(int type, struct list_head *ws, + const u8 *data_in, struct folio *dest_folio, + unsigned long dest_pgoff, size_t srclen, size_t destlen) +{ + switch (type) { + case BTRFS_COMPRESS_ZLIB: return zlib_decompress(ws, data_in, dest_folio, + dest_pgoff, srclen, destlen); + case BTRFS_COMPRESS_LZO: return lzo_decompress(ws, data_in, dest_folio, + dest_pgoff, srclen, destlen); + case BTRFS_COMPRESS_ZSTD: return zstd_decompress(ws, data_in, dest_folio, + dest_pgoff, srclen, destlen); + case BTRFS_COMPRESS_NONE: + default: + /* + * This can't happen, the type is validated several times + * before we get here. + */ + BUG(); + } +} + +static int btrfs_decompress_bio(struct compressed_bio *cb); + +/* + * Global cache of last unused pages for compression/decompression. + */ +static struct btrfs_compr_pool { + struct shrinker *shrinker; + spinlock_t lock; + struct list_head list; + int count; + int thresh; +} compr_pool; + +static unsigned long btrfs_compr_pool_count(struct shrinker *sh, struct shrink_control *sc) +{ + int ret; + + /* + * We must not read the values more than once if 'ret' gets expanded in + * the return statement so we don't accidentally return a negative + * number, even if the first condition finds it positive. + */ + ret = READ_ONCE(compr_pool.count) - READ_ONCE(compr_pool.thresh); + + return ret > 0 ? ret : 0; +} + +static unsigned long btrfs_compr_pool_scan(struct shrinker *sh, struct shrink_control *sc) +{ + LIST_HEAD(remove); + struct list_head *tmp, *next; + int freed; + + if (compr_pool.count == 0) + return SHRINK_STOP; + + /* For now, just simply drain the whole list. */ + spin_lock(&compr_pool.lock); + list_splice_init(&compr_pool.list, &remove); + freed = compr_pool.count; + compr_pool.count = 0; + spin_unlock(&compr_pool.lock); + + list_for_each_safe(tmp, next, &remove) { + struct page *page = list_entry(tmp, struct page, lru); + + ASSERT(page_ref_count(page) == 1); + put_page(page); + } + + return freed; +} + +/* + * Common wrappers for page allocation from compression wrappers + */ +struct folio *btrfs_alloc_compr_folio(struct btrfs_fs_info *fs_info, gfp_t gfp) +{ + struct folio *folio = NULL; + + /* For bs > ps cases, no cached folio pool for now. */ + if (fs_info->block_min_order) + goto alloc; + + spin_lock(&compr_pool.lock); + if (compr_pool.count > 0) { + folio = list_first_entry(&compr_pool.list, struct folio, lru); + list_del_init(&folio->lru); + compr_pool.count--; + } + spin_unlock(&compr_pool.lock); + + if (folio) + return folio; + +alloc: + return folio_alloc(gfp, fs_info->block_min_order); +} + +void btrfs_free_compr_folio(struct folio *folio) +{ + bool do_free = false; + + /* The folio is from bs > ps fs, no cached pool for now. */ + if (folio_order(folio)) + goto free; + + spin_lock(&compr_pool.lock); + if (compr_pool.count > compr_pool.thresh) { + do_free = true; + } else { + list_add(&folio->lru, &compr_pool.list); + compr_pool.count++; + } + spin_unlock(&compr_pool.lock); + + if (!do_free) + return; + +free: + ASSERT(folio_ref_count(folio) == 1); + folio_put(folio); +} + +static void end_bbio_compressed_read(struct btrfs_bio *bbio) +{ + struct compressed_bio *cb = to_compressed_bio(bbio); + blk_status_t status = bbio->bio.bi_status; + struct folio_iter fi; + + if (!status) + status = errno_to_blk_status(btrfs_decompress_bio(cb)); + + btrfs_bio_end_io(cb->orig_bbio, status); + bio_for_each_folio_all(fi, &bbio->bio) + btrfs_free_compr_folio(fi.folio); + bio_put(&bbio->bio); +} + +/* + * Clear the writeback bits on all of the file + * pages for a compressed write + */ +static noinline void end_compressed_writeback(const struct compressed_bio *cb) +{ + struct inode *inode = &cb->bbio.inode->vfs_inode; + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + pgoff_t index = cb->start >> PAGE_SHIFT; + const pgoff_t end_index = (cb->start + cb->len - 1) >> PAGE_SHIFT; + struct folio_batch fbatch; + int i; + int ret; + + ret = blk_status_to_errno(cb->bbio.bio.bi_status); + if (ret) + mapping_set_error(inode->i_mapping, ret); + + folio_batch_init(&fbatch); + while (index <= end_index) { + ret = filemap_get_folios(inode->i_mapping, &index, end_index, + &fbatch); + + if (ret == 0) + return; + + for (i = 0; i < ret; i++) { + struct folio *folio = fbatch.folios[i]; + + btrfs_folio_clamp_clear_writeback(fs_info, folio, + cb->start, cb->len); + } + folio_batch_release(&fbatch); + } + /* the inode may be gone now */ +} + +/* + * Do the cleanup once all the compressed pages hit the disk. This will clear + * writeback on the file pages and free the compressed pages. + * + * This also calls the writeback end hooks for the file pages so that metadata + * and checksums can be updated in the file. + */ +static void end_bbio_compressed_write(struct btrfs_bio *bbio) +{ + struct compressed_bio *cb = to_compressed_bio(bbio); + struct folio_iter fi; + + btrfs_finish_ordered_extent(cb->bbio.ordered, cb->start, cb->len, + cb->bbio.bio.bi_status == BLK_STS_OK); + + if (cb->writeback) + end_compressed_writeback(cb); + /* Note, our inode could be gone now. */ + bio_for_each_folio_all(fi, &bbio->bio) + btrfs_free_compr_folio(fi.folio); + bio_put(&cb->bbio.bio); +} + +/* + * worker function to build and submit bios for previously compressed pages. + * The corresponding pages in the inode should be marked for writeback + * and the compressed pages should have a reference on them for dropping + * when the IO is complete. + * + * This also checksums the file bytes and gets things ready for + * the end io hooks. + */ +void btrfs_submit_compressed_write(struct btrfs_ordered_extent *ordered, + struct compressed_bio *cb) +{ + struct btrfs_inode *inode = ordered->inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + + ASSERT(IS_ALIGNED(ordered->file_offset, fs_info->sectorsize)); + ASSERT(IS_ALIGNED(ordered->num_bytes, fs_info->sectorsize)); + /* + * This flag determines if we should clear the writeback flag from the + * page cache. But this function is only utilized by encoded writes, it + * never goes through the page cache. + */ + ASSERT(!cb->writeback); + + cb->start = ordered->file_offset; + cb->len = ordered->num_bytes; + ASSERT(cb->bbio.bio.bi_iter.bi_size == ordered->disk_num_bytes); + cb->bbio.bio.bi_iter.bi_sector = ordered->disk_bytenr >> SECTOR_SHIFT; + cb->bbio.ordered = ordered; + + btrfs_submit_bbio(&cb->bbio, 0); +} + +/* + * Allocate a compressed write bio for @inode file offset @start length @len. + * + * The caller still needs to properly queue all folios and populate involved + * members. + */ +struct compressed_bio *btrfs_alloc_compressed_write(struct btrfs_inode *inode, + u64 start, u64 len) +{ + struct compressed_bio *cb; + + cb = alloc_compressed_bio(inode, start, REQ_OP_WRITE, end_bbio_compressed_write); + cb->start = start; + cb->len = len; + cb->writeback = false; + return cb; +} + +/* + * Add extra folios in the same compressed file extent so that we don't need to + * re-read the same extent again and again. + * + * If in the same folio, we have several non-contiguous blocks which are pointing + * to the same on-disk compressed data, we will re-read the same extent many + * times, as this function can only help cross folio situations. + */ +static noinline int add_ra_bio_folios(struct inode *inode, u64 compressed_end, + struct compressed_bio *cb, int *memstall, + unsigned long *pflags, bool direct_reclaim) +{ + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + pgoff_t end_index; + struct bio *orig_bio = &cb->orig_bbio->bio; + u64 cur = cb->orig_bbio->file_offset + orig_bio->bi_iter.bi_size; + u64 isize = i_size_read(inode); + int ret; + gfp_t constraint_gfp, cache_gfp; + struct folio *folio; + struct extent_map *em; + struct address_space *mapping = inode->i_mapping; + struct extent_map_tree *em_tree; + struct extent_io_tree *tree; + int sectors_missed = 0; + + em_tree = &BTRFS_I(inode)->extent_tree; + tree = &BTRFS_I(inode)->io_tree; + + if (isize == 0) + return 0; + + /* For bs > ps cases, we don't support readahead for compressed folios for now. */ + if (fs_info->block_min_order) + return 0; + + end_index = (i_size_read(inode) - 1) >> PAGE_SHIFT; + + /* Avoid direct reclaim when the caller does not allow it. */ + constraint_gfp = ~__GFP_FS; + cache_gfp = GFP_NOFS | __GFP_NOWARN; + if (!direct_reclaim) { + constraint_gfp &= ~__GFP_DIRECT_RECLAIM; + cache_gfp &= ~__GFP_DIRECT_RECLAIM; + } + + while (cur < compressed_end) { + u64 folio_end; + pgoff_t pg_index = cur >> PAGE_SHIFT; + gfp_t masked_constraint_gfp; + u32 add_size; + + if (pg_index > end_index) + break; + + folio = filemap_get_folio(mapping, pg_index); + if (!IS_ERR(folio)) { + u64 folio_sz = folio_size(folio); + u64 offset = offset_in_folio(folio, cur); + + folio_put(folio); + sectors_missed += (folio_sz - offset) >> + fs_info->sectorsize_bits; + + /* Beyond threshold, no need to continue */ + if (sectors_missed > 4) + break; + + /* + * Jump to the next folio as we already have a folio for + * the current offset. + */ + cur += (folio_sz - offset); + continue; + } + + /* + * Since add_ra_bio_pages() is always speculative, suppress + * allocation warnings. + */ + masked_constraint_gfp = mapping_gfp_constraint(mapping, constraint_gfp); + masked_constraint_gfp |= __GFP_NOWARN; + + folio = filemap_alloc_folio(masked_constraint_gfp, 0, NULL); + if (!folio) + break; + + if (filemap_add_folio(mapping, folio, pg_index, cache_gfp)) { + /* There is already a folio, skip to the folio end. */ + cur += folio_size(folio) - offset_in_folio(folio, cur); + folio_put(folio); + continue; + } + + if (!*memstall && folio_test_workingset(folio)) { + psi_memstall_enter(pflags); + *memstall = 1; + } + + ret = set_folio_extent_mapped(folio); + if (ret < 0) { + folio_unlock(folio); + folio_put(folio); + break; + } + + folio_end = folio_next_pos(folio) - 1; + btrfs_lock_extent(tree, cur, folio_end, NULL); + read_lock(&em_tree->lock); + em = btrfs_lookup_extent_mapping(em_tree, cur, folio_end + 1 - cur); + read_unlock(&em_tree->lock); + + /* + * At this point, we have a locked folio in the page cache for + * these bytes in the file. But, we have to make sure they map + * to this compressed extent on disk. + */ + if (!em || cur < em->start || + (cur + fs_info->sectorsize > btrfs_extent_map_end(em)) || + (btrfs_extent_map_block_start(em) >> SECTOR_SHIFT) != + orig_bio->bi_iter.bi_sector) { + btrfs_free_extent_map(em); + btrfs_unlock_extent(tree, cur, folio_end, NULL); + folio_unlock(folio); + folio_put(folio); + break; + } + add_size = min(btrfs_extent_map_end(em), folio_end + 1) - cur; + btrfs_free_extent_map(em); + btrfs_unlock_extent(tree, cur, folio_end, NULL); + + if (folio_contains(folio, end_index)) { + size_t zero_offset = offset_in_folio(folio, isize); + + if (zero_offset) { + int zeros; + zeros = folio_size(folio) - zero_offset; + folio_zero_range(folio, zero_offset, zeros); + } + } + + if (!bio_add_folio(orig_bio, folio, add_size, + offset_in_folio(folio, cur))) { + folio_unlock(folio); + folio_put(folio); + break; + } + btrfs_folio_set_lock(fs_info, folio, cur, add_size); + folio_put(folio); + cur += add_size; + } + return 0; +} + +/* + * for a compressed read, the bio we get passed has all the inode pages + * in it. We don't actually do IO on those pages but allocate new ones + * to hold the compressed pages on disk. + * + * bio->bi_iter.bi_sector points to the compressed extent on disk + * bio->bi_io_vec points to all of the inode pages + * + * After the compressed pages are read, we copy the bytes into the + * bio we were passed and then call the bio end_io calls + */ +void btrfs_submit_compressed_read(struct btrfs_bio *bbio) +{ + struct btrfs_inode *inode = bbio->inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct extent_map_tree *em_tree = &inode->extent_tree; + struct compressed_bio *cb; + unsigned int compressed_len; + const u32 min_folio_size = btrfs_min_folio_size(fs_info); + u64 file_offset = bbio->file_offset; + gfp_t gfp; + u64 em_len; + u64 em_start; + struct extent_map *em; + unsigned long pflags; + int memstall = 0; + int ret; + + /* + * If this is a readahead bio, prevent direct reclaim. This is done to + * avoid stalling on speculative allocations when memory pressure is + * high. The demand fault will retry with GFP_NOFS and enter direct + * reclaim if needed. + */ + if (bbio->bio.bi_opf & REQ_RAHEAD) + gfp = (GFP_NOFS & ~__GFP_DIRECT_RECLAIM) | __GFP_NOWARN; + else + gfp = GFP_NOFS; + + /* we need the actual starting offset of this extent in the file */ + read_lock(&em_tree->lock); + em = btrfs_lookup_extent_mapping(em_tree, file_offset, fs_info->sectorsize); + read_unlock(&em_tree->lock); + if (!em) { + ret = -EIO; + goto out; + } + + ASSERT(btrfs_extent_map_is_compressed(em)); + compressed_len = em->disk_num_bytes; + + cb = alloc_compressed_bio(inode, file_offset, REQ_OP_READ, + end_bbio_compressed_read); + + cb->start = em->start - em->offset; + em_len = em->len; + em_start = em->start; + + cb->len = bbio->bio.bi_iter.bi_size; + cb->compress_type = btrfs_extent_map_compression(em); + cb->orig_bbio = bbio; + cb->bbio.csum_search_commit_root = bbio->csum_search_commit_root; + + btrfs_free_extent_map(em); + + for (int i = 0; i * min_folio_size < compressed_len; i++) { + struct folio *folio; + u32 cur_len = min(compressed_len - i * min_folio_size, min_folio_size); + + folio = btrfs_alloc_compr_folio(fs_info, gfp); + if (!folio) { + ret = -ENOMEM; + goto out_free_bio; + } + + ret = bio_add_folio(&cb->bbio.bio, folio, cur_len, 0); + if (unlikely(!ret)) { + folio_put(folio); + ret = -EINVAL; + goto out_free_bio; + } + } + ASSERT(cb->bbio.bio.bi_iter.bi_size == compressed_len); + + add_ra_bio_folios(&inode->vfs_inode, em_start + em_len, cb, &memstall, + &pflags, !(bbio->bio.bi_opf & REQ_RAHEAD)); + + cb->len = bbio->bio.bi_iter.bi_size; + cb->bbio.bio.bi_iter.bi_sector = bbio->bio.bi_iter.bi_sector; + + if (memstall) + psi_memstall_leave(&pflags); + + btrfs_submit_bbio(&cb->bbio, 0); + return; + +out_free_bio: + cleanup_compressed_bio(cb); +out: + btrfs_bio_end_io(bbio, errno_to_blk_status(ret)); +} + +/* + * Heuristic uses systematic sampling to collect data from the input data + * range, the logic can be tuned by the following constants: + * + * @SAMPLING_READ_SIZE - how many bytes will be copied from for each sample + * @SAMPLING_INTERVAL - range from which the sampled data can be collected + */ +#define SAMPLING_READ_SIZE (16) +#define SAMPLING_INTERVAL (256) + +/* + * For statistical analysis of the input data we consider bytes that form a + * Galois Field of 256 objects. Each object has an attribute count, ie. how + * many times the object appeared in the sample. + */ +#define BUCKET_SIZE (256) + +/* + * The size of the sample is based on a statistical sampling rule of thumb. + * The common way is to perform sampling tests as long as the number of + * elements in each cell is at least 5. + * + * Instead of 5, we choose 32 to obtain more accurate results. + * If the data contain the maximum number of symbols, which is 256, we obtain a + * sample size bound by 8192. + * + * For a sample of at most 8KB of data per data range: 16 consecutive bytes + * from up to 512 locations. + */ +#define MAX_SAMPLE_SIZE (BTRFS_MAX_UNCOMPRESSED * \ + SAMPLING_READ_SIZE / SAMPLING_INTERVAL) + +struct bucket_item { + u32 count; +}; + +struct heuristic_ws { + /* Partial copy of input data */ + u8 *sample; + u32 sample_size; + /* Buckets store counters for each byte value */ + struct bucket_item bucket[BUCKET_SIZE]; + /* Sorting buffer */ + struct bucket_item bucket_b[BUCKET_SIZE]; + struct list_head list; +}; + +static void free_heuristic_ws(struct list_head *ws) +{ + struct heuristic_ws *workspace; + + workspace = list_entry(ws, struct heuristic_ws, list); + + kvfree(workspace->sample); + kfree(workspace); +} + +static struct list_head *alloc_heuristic_ws(struct btrfs_fs_info *fs_info) +{ + struct heuristic_ws *ws; + + ws = kzalloc_obj(*ws); + if (!ws) + return ERR_PTR(-ENOMEM); + + ws->sample = kvmalloc(MAX_SAMPLE_SIZE, GFP_KERNEL); + if (!ws->sample) + goto fail; + + INIT_LIST_HEAD(&ws->list); + return &ws->list; +fail: + free_heuristic_ws(&ws->list); + return ERR_PTR(-ENOMEM); +} + +const struct btrfs_compress_levels btrfs_heuristic_compress = { 0 }; + +static const struct btrfs_compress_levels * const btrfs_compress_levels[] = { + /* The heuristic is represented as compression type 0 */ + &btrfs_heuristic_compress, + &btrfs_zlib_compress, + &btrfs_lzo_compress, + &btrfs_zstd_compress, +}; + +static struct list_head *alloc_workspace(struct btrfs_fs_info *fs_info, int type, int level) +{ + switch (type) { + case BTRFS_COMPRESS_NONE: return alloc_heuristic_ws(fs_info); + case BTRFS_COMPRESS_ZLIB: return zlib_alloc_workspace(fs_info, level); + case BTRFS_COMPRESS_LZO: return lzo_alloc_workspace(fs_info); + case BTRFS_COMPRESS_ZSTD: return zstd_alloc_workspace(fs_info, level); + default: + /* + * This can't happen, the type is validated several times + * before we get here. + */ + BUG(); + } +} + +static void free_workspace(int type, struct list_head *ws) +{ + switch (type) { + case BTRFS_COMPRESS_NONE: return free_heuristic_ws(ws); + case BTRFS_COMPRESS_ZLIB: return zlib_free_workspace(ws); + case BTRFS_COMPRESS_LZO: return lzo_free_workspace(ws); + case BTRFS_COMPRESS_ZSTD: return zstd_free_workspace(ws); + default: + /* + * This can't happen, the type is validated several times + * before we get here. + */ + BUG(); + } +} + +static int alloc_workspace_manager(struct btrfs_fs_info *fs_info, + enum btrfs_compression_type type) +{ + struct workspace_manager *gwsm; + struct list_head *workspace; + + ASSERT(fs_info->compr_wsm[type] == NULL); + gwsm = kzalloc_obj(*gwsm); + if (!gwsm) + return -ENOMEM; + + INIT_LIST_HEAD(&gwsm->idle_ws); + spin_lock_init(&gwsm->ws_lock); + atomic_set(&gwsm->total_ws, 0); + init_waitqueue_head(&gwsm->ws_wait); + fs_info->compr_wsm[type] = gwsm; + + /* + * Preallocate one workspace for each compression type so we can + * guarantee forward progress in the worst case + */ + workspace = alloc_workspace(fs_info, type, 0); + if (IS_ERR(workspace)) { + btrfs_warn(fs_info, + "cannot preallocate compression workspace for %s, will try later", + btrfs_compress_type2str(type)); + } else { + atomic_set(&gwsm->total_ws, 1); + gwsm->free_ws = 1; + list_add(workspace, &gwsm->idle_ws); + } + return 0; +} + +static void free_workspace_manager(struct btrfs_fs_info *fs_info, + enum btrfs_compression_type type) +{ + struct list_head *ws; + struct workspace_manager *gwsm = fs_info->compr_wsm[type]; + + /* ZSTD uses its own workspace manager, should enter here. */ + ASSERT(type != BTRFS_COMPRESS_ZSTD && type < BTRFS_NR_COMPRESS_TYPES); + if (!gwsm) + return; + fs_info->compr_wsm[type] = NULL; + while (!list_empty(&gwsm->idle_ws)) { + ws = gwsm->idle_ws.next; + list_del(ws); + free_workspace(type, ws); + atomic_dec(&gwsm->total_ws); + } + kfree(gwsm); +} + +/* + * This finds an available workspace or allocates a new one. + * If it's not possible to allocate a new one, waits until there's one. + * Preallocation makes a forward progress guarantees and we do not return + * errors. + */ +struct list_head *btrfs_get_workspace(struct btrfs_fs_info *fs_info, int type, int level) +{ + struct workspace_manager *wsm = fs_info->compr_wsm[type]; + struct list_head *workspace; + int cpus = num_online_cpus(); + unsigned nofs_flag; + struct list_head *idle_ws; + spinlock_t *ws_lock; + atomic_t *total_ws; + wait_queue_head_t *ws_wait; + int *free_ws; + + ASSERT(wsm); + idle_ws = &wsm->idle_ws; + ws_lock = &wsm->ws_lock; + total_ws = &wsm->total_ws; + ws_wait = &wsm->ws_wait; + free_ws = &wsm->free_ws; + +again: + spin_lock(ws_lock); + if (!list_empty(idle_ws)) { + workspace = idle_ws->next; + list_del(workspace); + (*free_ws)--; + spin_unlock(ws_lock); + return workspace; + + } + if (atomic_read(total_ws) > cpus) { + DEFINE_WAIT(wait); + + spin_unlock(ws_lock); + prepare_to_wait(ws_wait, &wait, TASK_UNINTERRUPTIBLE); + if (atomic_read(total_ws) > cpus && !*free_ws) + schedule(); + finish_wait(ws_wait, &wait); + goto again; + } + atomic_inc(total_ws); + spin_unlock(ws_lock); + + /* + * Allocation helpers call vmalloc that can't use GFP_NOFS, so we have + * to turn it off here because we might get called from the restricted + * context of btrfs_compress_bio/btrfs_compress_pages + */ + nofs_flag = memalloc_nofs_save(); + workspace = alloc_workspace(fs_info, type, level); + memalloc_nofs_restore(nofs_flag); + + if (IS_ERR(workspace)) { + atomic_dec(total_ws); + wake_up(ws_wait); + + /* + * Do not return the error but go back to waiting. There's a + * workspace preallocated for each type and the compression + * time is bounded so we get to a workspace eventually. This + * makes our caller's life easier. + * + * To prevent silent and low-probability deadlocks (when the + * initial preallocation fails), check if there are any + * workspaces at all. + */ + if (atomic_read(total_ws) == 0) { + static DEFINE_RATELIMIT_STATE(_rs, + /* once per minute */ 60 * HZ, + /* no burst */ 1); + + if (__ratelimit(&_rs)) + btrfs_warn(fs_info, + "no compression workspaces, low memory, retrying"); + } + goto again; + } + return workspace; +} + +static struct list_head *get_workspace(struct btrfs_fs_info *fs_info, int type, int level) +{ + switch (type) { + case BTRFS_COMPRESS_NONE: return btrfs_get_workspace(fs_info, type, level); + case BTRFS_COMPRESS_ZLIB: return zlib_get_workspace(fs_info, level); + case BTRFS_COMPRESS_LZO: return btrfs_get_workspace(fs_info, type, level); + case BTRFS_COMPRESS_ZSTD: return zstd_get_workspace(fs_info, level); + default: + /* + * This can't happen, the type is validated several times + * before we get here. + */ + BUG(); + } +} + +/* + * put a workspace struct back on the list or free it if we have enough + * idle ones sitting around + */ +void btrfs_put_workspace(struct btrfs_fs_info *fs_info, int type, struct list_head *ws) +{ + struct workspace_manager *gwsm = fs_info->compr_wsm[type]; + struct list_head *idle_ws; + spinlock_t *ws_lock; + atomic_t *total_ws; + wait_queue_head_t *ws_wait; + int *free_ws; + + ASSERT(gwsm); + idle_ws = &gwsm->idle_ws; + ws_lock = &gwsm->ws_lock; + total_ws = &gwsm->total_ws; + ws_wait = &gwsm->ws_wait; + free_ws = &gwsm->free_ws; + + spin_lock(ws_lock); + if (*free_ws <= num_online_cpus()) { + list_add(ws, idle_ws); + (*free_ws)++; + spin_unlock(ws_lock); + goto wake; + } + spin_unlock(ws_lock); + + free_workspace(type, ws); + atomic_dec(total_ws); +wake: + cond_wake_up(ws_wait); +} + +static void put_workspace(struct btrfs_fs_info *fs_info, int type, struct list_head *ws) +{ + switch (type) { + case BTRFS_COMPRESS_NONE: return btrfs_put_workspace(fs_info, type, ws); + case BTRFS_COMPRESS_ZLIB: return btrfs_put_workspace(fs_info, type, ws); + case BTRFS_COMPRESS_LZO: return btrfs_put_workspace(fs_info, type, ws); + case BTRFS_COMPRESS_ZSTD: return zstd_put_workspace(fs_info, ws); + default: + /* + * This can't happen, the type is validated several times + * before we get here. + */ + BUG(); + } +} + +/* + * Adjust @level according to the limits of the compression algorithm or + * fallback to default + */ +static int btrfs_compress_set_level(unsigned int type, int level) +{ + const struct btrfs_compress_levels *levels = btrfs_compress_levels[type]; + + if (level == 0) + level = levels->default_level; + else + level = clamp(level, levels->min_level, levels->max_level); + + return level; +} + +/* + * Check whether the @level is within the valid range for the given type. + */ +bool btrfs_compress_level_valid(unsigned int type, int level) +{ + const struct btrfs_compress_levels *levels = btrfs_compress_levels[type]; + + return levels->min_level <= level && level <= levels->max_level; +} + +/* Wrapper around find_get_page(), with extra error message. */ +int btrfs_compress_filemap_get_folio(struct address_space *mapping, u64 start, + struct folio **in_folio_ret) +{ + struct folio *in_folio; + + /* + * The compressed write path should have the folio locked already, thus + * we only need to grab one reference. + */ + in_folio = filemap_get_folio(mapping, start >> PAGE_SHIFT); + if (IS_ERR(in_folio)) { + struct btrfs_inode *inode = BTRFS_I(mapping->host); + + btrfs_crit(inode->root->fs_info, + "failed to get page cache, root %lld ino %llu file offset %llu", + btrfs_root_id(inode->root), btrfs_ino(inode), start); + return -ENOENT; + } + *in_folio_ret = in_folio; + return 0; +} + +/* + * Given an address space and start and length, compress the page cache + * contents into @cb. + * + * @type_level: is encoded algorithm and level, where level 0 means whatever + * default the algorithm chooses and is opaque here; + * - compression algo are 0-3 + * - the level are bits 4-7 + * + * @cb->bbio.bio.bi_iter.bi_size will indicate the compressed data size. + * The bi_size may not be sectorsize aligned, thus the caller still need + * to do the round up before submission. + * + * This function will allocate compressed folios with btrfs_alloc_compr_folio(), + * thus callers must make sure the endio function and error handling are using + * btrfs_free_compr_folio() to release those folios. + * This is already done in end_bbio_compressed_write() and cleanup_compressed_bio(). + */ +struct compressed_bio *btrfs_compress_bio(struct btrfs_inode *inode, + u64 start, u32 len, unsigned int type, + int level, blk_opf_t write_flags) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct list_head *workspace; + struct compressed_bio *cb; + int ret; + + cb = alloc_compressed_bio(inode, start, REQ_OP_WRITE | write_flags, + end_bbio_compressed_write); + cb->start = start; + cb->len = len; + cb->writeback = true; + cb->compress_type = type; + + level = btrfs_compress_set_level(type, level); + workspace = get_workspace(fs_info, type, level); + switch (type) { + case BTRFS_COMPRESS_ZLIB: + ret = zlib_compress_bio(workspace, cb); + break; + case BTRFS_COMPRESS_LZO: + ret = lzo_compress_bio(workspace, cb); + break; + case BTRFS_COMPRESS_ZSTD: + ret = zstd_compress_bio(workspace, cb); + break; + case BTRFS_COMPRESS_NONE: + default: + /* + * This can happen when compression races with remount setting + * it to 'no compress', while caller doesn't call + * inode_need_compress() to check if we really need to + * compress. + * + * Not a big deal, just need to inform caller that we + * haven't allocated any pages yet. + */ + ret = -E2BIG; + } + + put_workspace(fs_info, type, workspace); + if (ret < 0) { + cleanup_compressed_bio(cb); + return ERR_PTR(ret); + } + return cb; +} + +static int btrfs_decompress_bio(struct compressed_bio *cb) +{ + struct btrfs_fs_info *fs_info = cb_to_fs_info(cb); + struct list_head *workspace; + int ret; + int type = cb->compress_type; + + workspace = get_workspace(fs_info, type, 0); + ret = compression_decompress_bio(workspace, cb); + put_workspace(fs_info, type, workspace); + + if (!ret) + zero_fill_bio(&cb->orig_bbio->bio); + return ret; +} + +/* + * a less complex decompression routine. Our compressed data fits in a + * single page, and we want to read a single page out of it. + * dest_pgoff tells us the offset into the destination folio where we write the + * decompressed data. + */ +int btrfs_decompress(int type, const u8 *data_in, struct folio *dest_folio, + unsigned long dest_pgoff, size_t srclen, size_t destlen) +{ + struct btrfs_fs_info *fs_info = folio_to_fs_info(dest_folio); + struct list_head *workspace; + const u32 sectorsize = fs_info->sectorsize; + int ret; + + /* + * The full destination folio range should not exceed the folio size. + * And the @destlen should not exceed sectorsize, as this is only called for + * inline file extents, which should not exceed sectorsize. + */ + ASSERT(dest_pgoff + destlen <= folio_size(dest_folio) && destlen <= sectorsize); + + workspace = get_workspace(fs_info, type, 0); + ret = compression_decompress(type, workspace, data_in, dest_folio, + dest_pgoff, srclen, destlen); + put_workspace(fs_info, type, workspace); + + return ret; +} + +int btrfs_alloc_compress_wsm(struct btrfs_fs_info *fs_info) +{ + int ret; + + ret = alloc_workspace_manager(fs_info, BTRFS_COMPRESS_NONE); + if (ret < 0) + goto error; + ret = alloc_workspace_manager(fs_info, BTRFS_COMPRESS_ZLIB); + if (ret < 0) + goto error; + ret = alloc_workspace_manager(fs_info, BTRFS_COMPRESS_LZO); + if (ret < 0) + goto error; + ret = zstd_alloc_workspace_manager(fs_info); + if (ret < 0) + goto error; + return 0; +error: + btrfs_free_compress_wsm(fs_info); + return ret; +} + +void btrfs_free_compress_wsm(struct btrfs_fs_info *fs_info) +{ + free_workspace_manager(fs_info, BTRFS_COMPRESS_NONE); + free_workspace_manager(fs_info, BTRFS_COMPRESS_ZLIB); + free_workspace_manager(fs_info, BTRFS_COMPRESS_LZO); + zstd_free_workspace_manager(fs_info); +} + +int __init btrfs_init_compress(void) +{ + if (bioset_init(&btrfs_compressed_bioset, BIO_POOL_SIZE, + offsetof(struct compressed_bio, bbio.bio), + BIOSET_NEED_BVECS)) + return -ENOMEM; + + compr_pool.shrinker = shrinker_alloc(SHRINKER_NONSLAB, "btrfs-compr-pages"); + if (!compr_pool.shrinker) + return -ENOMEM; + + spin_lock_init(&compr_pool.lock); + INIT_LIST_HEAD(&compr_pool.list); + compr_pool.count = 0; + /* 128K / 4K = 32, for 8 threads is 256 pages. */ + compr_pool.thresh = BTRFS_MAX_COMPRESSED / PAGE_SIZE * 8; + compr_pool.shrinker->count_objects = btrfs_compr_pool_count; + compr_pool.shrinker->scan_objects = btrfs_compr_pool_scan; + compr_pool.shrinker->batch = 32; + compr_pool.shrinker->seeks = DEFAULT_SEEKS; + shrinker_register(compr_pool.shrinker); + + return 0; +} + +void __cold btrfs_exit_compress(void) +{ + /* For now scan drains all pages and does not touch the parameters. */ + btrfs_compr_pool_scan(NULL, NULL); + shrinker_free(compr_pool.shrinker); + + bioset_exit(&btrfs_compressed_bioset); +} + +/* + * Copy decompressed data from working buffer to pages. + * + * @buf: The decompressed data buffer + * @buf_len: The decompressed data length + * @decompressed: Number of bytes that are already decompressed inside the + * compressed extent + * @cb: The compressed extent descriptor + * @orig_bio: The original bio that the caller wants to read for + * + * An easier to understand graph is like below: + * + * |<- orig_bio ->| |<- orig_bio->| + * |<------- full decompressed extent ----->| + * |<----------- @cb range ---->| + * | |<-- @buf_len -->| + * |<--- @decompressed --->| + * + * Note that, @cb can be a subpage of the full decompressed extent, but + * @cb->start always has the same as the orig_file_offset value of the full + * decompressed extent. + * + * When reading compressed extent, we have to read the full compressed extent, + * while @orig_bio may only want part of the range. + * Thus this function will ensure only data covered by @orig_bio will be copied + * to. + * + * Return 0 if we have copied all needed contents for @orig_bio. + * Return >0 if we need continue decompress. + */ +int btrfs_decompress_buf2page(const char *buf, u32 buf_len, + struct compressed_bio *cb, u32 decompressed) +{ + struct bio *orig_bio = &cb->orig_bbio->bio; + /* Offset inside the full decompressed extent */ + u32 cur_offset; + + cur_offset = decompressed; + /* The main loop to do the copy */ + while (cur_offset < decompressed + buf_len) { + struct bio_vec bvec; + size_t copy_len; + u32 copy_start; + /* Offset inside the full decompressed extent */ + u32 bvec_offset; + void *kaddr; + + bvec = bio_iter_iovec(orig_bio, orig_bio->bi_iter); + /* + * cb->start may underflow, but subtracting that value can still + * give us correct offset inside the full decompressed extent. + */ + bvec_offset = page_offset(bvec.bv_page) + bvec.bv_offset - cb->start; + + /* Haven't reached the bvec range, exit */ + if (decompressed + buf_len <= bvec_offset) + return 1; + + copy_start = max(cur_offset, bvec_offset); + copy_len = min(bvec_offset + bvec.bv_len, + decompressed + buf_len) - copy_start; + ASSERT(copy_len); + + /* + * Extra range check to ensure we didn't go beyond + * @buf + @buf_len. + */ + ASSERT(copy_start - decompressed < buf_len); + + kaddr = bvec_kmap_local(&bvec); + memcpy(kaddr, buf + copy_start - decompressed, copy_len); + kunmap_local(kaddr); + + cur_offset += copy_len; + bio_advance(orig_bio, copy_len); + /* Finished the bio */ + if (!orig_bio->bi_iter.bi_size) + return 0; + } + return 1; +} + +/* + * Shannon Entropy calculation + * + * Pure byte distribution analysis fails to determine compressibility of data. + * Try calculating entropy to estimate the average minimum number of bits + * needed to encode the sampled data. + * + * For convenience, return the percentage of needed bits, instead of amount of + * bits directly. + * + * @ENTROPY_LVL_ACEPTABLE - below that threshold, sample has low byte entropy + * and can be compressible with high probability + * + * @ENTROPY_LVL_HIGH - data are not compressible with high probability + * + * Use of ilog2() decreases precision, we lower the LVL to 5 to compensate. + */ +#define ENTROPY_LVL_ACEPTABLE (65) +#define ENTROPY_LVL_HIGH (80) + +/* + * For increased precision in shannon_entropy calculation, + * let's do pow(n, M) to save more digits after comma: + * + * - maximum int bit length is 64 + * - ilog2(MAX_SAMPLE_SIZE) -> 13 + * - 13 * 4 = 52 < 64 -> M = 4 + * + * So use pow(n, 4). + */ +static inline u32 ilog2_w(u64 n) +{ + return ilog2(n * n * n * n); +} + +static u32 shannon_entropy(struct heuristic_ws *ws) +{ + const u32 entropy_max = 8 * ilog2_w(2); + u32 entropy_sum = 0; + u32 p, p_base, sz_base; + u32 i; + + sz_base = ilog2_w(ws->sample_size); + for (i = 0; i < BUCKET_SIZE && ws->bucket[i].count > 0; i++) { + p = ws->bucket[i].count; + p_base = ilog2_w(p); + entropy_sum += p * (sz_base - p_base); + } + + entropy_sum /= ws->sample_size; + return entropy_sum * 100 / entropy_max; +} + +#define RADIX_BASE 4U +#define COUNTERS_SIZE (1U << RADIX_BASE) + +static u8 get4bits(u64 num, int shift) { + u8 low4bits; + + num >>= shift; + /* Reverse order */ + low4bits = (COUNTERS_SIZE - 1) - (num % COUNTERS_SIZE); + return low4bits; +} + +/* + * Use 4 bits as radix base + * Use 16 u32 counters for calculating new position in buf array + * + * @array - array that will be sorted + * @array_buf - buffer array to store sorting results + * must be equal in size to @array + * @num - array size + */ +static void radix_sort(struct bucket_item *array, struct bucket_item *array_buf, + int num) +{ + u64 max_num; + u64 buf_num; + u32 counters[COUNTERS_SIZE]; + u32 new_addr; + u32 addr; + int bitlen; + int shift; + int i; + + /* + * Try avoid useless loop iterations for small numbers stored in big + * counters. Example: 48 33 4 ... in 64bit array + */ + max_num = array[0].count; + for (i = 1; i < num; i++) { + buf_num = array[i].count; + if (buf_num > max_num) + max_num = buf_num; + } + + buf_num = ilog2(max_num); + bitlen = ALIGN(buf_num, RADIX_BASE * 2); + + shift = 0; + while (shift < bitlen) { + memset(counters, 0, sizeof(counters)); + + for (i = 0; i < num; i++) { + buf_num = array[i].count; + addr = get4bits(buf_num, shift); + counters[addr]++; + } + + for (i = 1; i < COUNTERS_SIZE; i++) + counters[i] += counters[i - 1]; + + for (i = num - 1; i >= 0; i--) { + buf_num = array[i].count; + addr = get4bits(buf_num, shift); + counters[addr]--; + new_addr = counters[addr]; + array_buf[new_addr] = array[i]; + } + + shift += RADIX_BASE; + + /* + * Normal radix expects to move data from a temporary array, to + * the main one. But that requires some CPU time. Avoid that + * by doing another sort iteration to original array instead of + * memcpy() + */ + memset(counters, 0, sizeof(counters)); + + for (i = 0; i < num; i ++) { + buf_num = array_buf[i].count; + addr = get4bits(buf_num, shift); + counters[addr]++; + } + + for (i = 1; i < COUNTERS_SIZE; i++) + counters[i] += counters[i - 1]; + + for (i = num - 1; i >= 0; i--) { + buf_num = array_buf[i].count; + addr = get4bits(buf_num, shift); + counters[addr]--; + new_addr = counters[addr]; + array[new_addr] = array_buf[i]; + } + + shift += RADIX_BASE; + } +} + +/* + * Size of the core byte set - how many bytes cover 90% of the sample + * + * There are several types of structured binary data that use nearly all byte + * values. The distribution can be uniform and counts in all buckets will be + * nearly the same (eg. encrypted data). Unlikely to be compressible. + * + * Other possibility is normal (Gaussian) distribution, where the data could + * be potentially compressible, but we have to take a few more steps to decide + * how much. + * + * @BYTE_CORE_SET_LOW - main part of byte values repeated frequently, + * compression algo can easy fix that + * @BYTE_CORE_SET_HIGH - data have uniform distribution and with high + * probability is not compressible + */ +#define BYTE_CORE_SET_LOW (64) +#define BYTE_CORE_SET_HIGH (200) + +static int byte_core_set_size(struct heuristic_ws *ws) +{ + u32 i; + u32 coreset_sum = 0; + const u32 core_set_threshold = ws->sample_size * 90 / 100; + struct bucket_item *bucket = ws->bucket; + + /* Sort in reverse order */ + radix_sort(ws->bucket, ws->bucket_b, BUCKET_SIZE); + + for (i = 0; i < BYTE_CORE_SET_LOW; i++) + coreset_sum += bucket[i].count; + + if (coreset_sum > core_set_threshold) + return i; + + for (; i < BYTE_CORE_SET_HIGH && bucket[i].count > 0; i++) { + coreset_sum += bucket[i].count; + if (coreset_sum > core_set_threshold) + break; + } + + return i; +} + +/* + * Count byte values in buckets. + * This heuristic can detect textual data (configs, xml, json, html, etc). + * Because in most text-like data byte set is restricted to limited number of + * possible characters, and that restriction in most cases makes data easy to + * compress. + * + * @BYTE_SET_THRESHOLD - consider all data within this byte set size: + * less - compressible + * more - need additional analysis + */ +#define BYTE_SET_THRESHOLD (64) + +static u32 byte_set_size(const struct heuristic_ws *ws) +{ + u32 i; + u32 byte_set_size = 0; + + for (i = 0; i < BYTE_SET_THRESHOLD; i++) { + if (ws->bucket[i].count > 0) + byte_set_size++; + } + + /* + * Continue collecting count of byte values in buckets. If the byte + * set size is bigger then the threshold, it's pointless to continue, + * the detection technique would fail for this type of data. + */ + for (; i < BUCKET_SIZE; i++) { + if (ws->bucket[i].count > 0) { + byte_set_size++; + if (byte_set_size > BYTE_SET_THRESHOLD) + return byte_set_size; + } + } + + return byte_set_size; +} + +static bool sample_repeated_patterns(struct heuristic_ws *ws) +{ + const u32 half_of_sample = ws->sample_size / 2; + const u8 *data = ws->sample; + + return memcmp(&data[0], &data[half_of_sample], half_of_sample) == 0; +} + +static void heuristic_collect_sample(struct inode *inode, u64 start, u64 end, + struct heuristic_ws *ws) +{ + struct page *page; + pgoff_t index, index_end; + u32 i, curr_sample_pos; + u8 *in_data; + + /* + * Compression handles the input data by chunks of 128KiB + * (defined by BTRFS_MAX_UNCOMPRESSED) + * + * We do the same for the heuristic and loop over the whole range. + * + * MAX_SAMPLE_SIZE - calculated under assumption that heuristic will + * process no more than BTRFS_MAX_UNCOMPRESSED at a time. + */ + if (end - start > BTRFS_MAX_UNCOMPRESSED) + end = start + BTRFS_MAX_UNCOMPRESSED; + + index = start >> PAGE_SHIFT; + index_end = end >> PAGE_SHIFT; + + /* Don't miss unaligned end */ + if (!PAGE_ALIGNED(end)) + index_end++; + + curr_sample_pos = 0; + while (index < index_end) { + page = find_get_page(inode->i_mapping, index); + in_data = kmap_local_page(page); + /* Handle case where the start is not aligned to PAGE_SIZE */ + i = start % PAGE_SIZE; + while (i < PAGE_SIZE - SAMPLING_READ_SIZE) { + /* Don't sample any garbage from the last page */ + if (start > end - SAMPLING_READ_SIZE) + break; + memcpy(&ws->sample[curr_sample_pos], &in_data[i], + SAMPLING_READ_SIZE); + i += SAMPLING_INTERVAL; + start += SAMPLING_INTERVAL; + curr_sample_pos += SAMPLING_READ_SIZE; + } + kunmap_local(in_data); + put_page(page); + + index++; + } + + ws->sample_size = curr_sample_pos; +} + +/* + * Compression heuristic. + * + * The following types of analysis can be performed: + * - detect mostly zero data + * - detect data with low "byte set" size (text, etc) + * - detect data with low/high "core byte" set + * + * Return non-zero if the compression should be done, 0 otherwise. + */ +int btrfs_compress_heuristic(struct btrfs_inode *inode, u64 start, u64 end) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct list_head *ws_list = get_workspace(fs_info, 0, 0); + struct heuristic_ws *ws; + u32 i; + u8 byte; + int ret = 0; + + ws = list_entry(ws_list, struct heuristic_ws, list); + + heuristic_collect_sample(&inode->vfs_inode, start, end, ws); + + if (sample_repeated_patterns(ws)) { + ret = 1; + goto out; + } + + memset(ws->bucket, 0, sizeof(*ws->bucket)*BUCKET_SIZE); + + for (i = 0; i < ws->sample_size; i++) { + byte = ws->sample[i]; + ws->bucket[byte].count++; + } + + i = byte_set_size(ws); + if (i < BYTE_SET_THRESHOLD) { + ret = 2; + goto out; + } + + i = byte_core_set_size(ws); + if (i <= BYTE_CORE_SET_LOW) { + ret = 3; + goto out; + } + + if (i >= BYTE_CORE_SET_HIGH) { + ret = 0; + goto out; + } + + i = shannon_entropy(ws); + if (i <= ENTROPY_LVL_ACEPTABLE) { + ret = 4; + goto out; + } + + /* + * For the levels below ENTROPY_LVL_HIGH, additional analysis would be + * needed to give green light to compression. + * + * For now just assume that compression at that level is not worth the + * resources because: + * + * 1. it is possible to defrag the data later + * + * 2. the data would turn out to be hardly compressible, eg. 150 byte + * values, every bucket has counter at level ~54. The heuristic would + * be confused. This can happen when data have some internal repeated + * patterns like "abbacbbc...". This can be detected by analyzing + * pairs of bytes, which is too costly. + */ + if (i < ENTROPY_LVL_HIGH) { + ret = 5; + goto out; + } else { + ret = 0; + goto out; + } + +out: + put_workspace(fs_info, 0, ws_list); + return ret; +} + +/* + * Convert the compression suffix (eg. after "zlib" starting with ":") to level. + * + * If the resulting level exceeds the algo's supported levels, it will be clamped. + * + * Return <0 if no valid string can be found. + * Return 0 if everything is fine. + */ +int btrfs_compress_str2level(unsigned int type, const char *str, int *level_ret) +{ + int level = 0; + int ret; + + if (!type) { + *level_ret = btrfs_compress_set_level(type, level); + return 0; + } + + if (str[0] == ':') { + ret = kstrtoint(str + 1, 10, &level); + if (ret) + return ret; + } + + *level_ret = btrfs_compress_set_level(type, level); + return 0; +} diff --git a/fs/btrfs/compression.h b/fs/btrfs/compression.h new file mode 100644 index 000000000..1022dc53e --- /dev/null +++ b/fs/btrfs/compression.h @@ -0,0 +1,183 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2008 Oracle. All rights reserved. + */ + +#ifndef BTRFS_COMPRESSION_H +#define BTRFS_COMPRESSION_H + +#include +#include +#include +#include +#include +#include +#include "bio.h" +#include "fs.h" +#include "btrfs_inode.h" + +struct address_space; +struct inode; +struct btrfs_inode; +struct btrfs_ordered_extent; + +/* + * We want to make sure that amount of RAM required to uncompress an extent is + * reasonable, so we limit the total size in ram of a compressed extent to + * 128k. This is a crucial number because it also controls how easily we can + * spread reads across cpus for decompression. + * + * We also want to make sure the amount of IO required to do a random read is + * reasonably small, so we limit the size of a compressed extent to 128k. + */ + +/* Maximum length of compressed data stored on disk */ +#define BTRFS_MAX_COMPRESSED (SZ_128K) +#define BTRFS_MAX_COMPRESSED_PAGES (BTRFS_MAX_COMPRESSED / PAGE_SIZE) +static_assert((BTRFS_MAX_COMPRESSED % PAGE_SIZE) == 0); + +/* The max size for a single worker to compress. */ +#define BTRFS_COMPRESSION_CHUNK_SIZE (SZ_512K) + +/* Maximum size of data before compression */ +#define BTRFS_MAX_UNCOMPRESSED (SZ_128K) + +#define BTRFS_ZLIB_DEFAULT_LEVEL 3 + +struct compressed_bio { + /* starting offset in the inode for our pages */ + u64 start; + + /* Number of bytes in the inode we're working on */ + unsigned int len; + + /* The compression algorithm for this bio */ + u8 compress_type; + + /* Whether this is a write for writeback. */ + bool writeback; + + /* For reads, this is the bio we are copying the data into. */ + struct btrfs_bio *orig_bbio; + + /* Must be last. */ + struct btrfs_bio bbio; +}; + +static inline struct btrfs_fs_info *cb_to_fs_info(const struct compressed_bio *cb) +{ + return cb->bbio.inode->root->fs_info; +} + +/* @range_end must be exclusive. */ +static inline u32 btrfs_calc_input_length(struct folio *folio, u64 range_end, u64 cur) +{ + /* @cur must be inside the folio. */ + ASSERT(folio_pos(folio) <= cur); + ASSERT(cur < folio_next_pos(folio)); + return umin(range_end, folio_next_pos(folio)) - cur; +} + +int btrfs_alloc_compress_wsm(struct btrfs_fs_info *fs_info); +void btrfs_free_compress_wsm(struct btrfs_fs_info *fs_info); + +int __init btrfs_init_compress(void); +void __cold btrfs_exit_compress(void); + +bool btrfs_compress_level_valid(unsigned int type, int level); +int btrfs_decompress(int type, const u8 *data_in, struct folio *dest_folio, + unsigned long dest_pgoff, size_t srclen, size_t destlen); +int btrfs_decompress_buf2page(const char *buf, u32 buf_len, + struct compressed_bio *cb, u32 decompressed); + +struct compressed_bio *btrfs_alloc_compressed_write(struct btrfs_inode *inode, + u64 start, u64 len); +void btrfs_submit_compressed_write(struct btrfs_ordered_extent *ordered, + struct compressed_bio *cb); +void btrfs_submit_compressed_read(struct btrfs_bio *bbio); + +int btrfs_compress_str2level(unsigned int type, const char *str, int *level_ret); + +struct folio *btrfs_alloc_compr_folio(struct btrfs_fs_info *fs_info, gfp_t gfp); +void btrfs_free_compr_folio(struct folio *folio); + +struct workspace_manager { + struct list_head idle_ws; + spinlock_t ws_lock; + /* Number of free workspaces */ + int free_ws; + /* Total number of allocated workspaces */ + atomic_t total_ws; + /* Waiters for a free workspace */ + wait_queue_head_t ws_wait; +}; + +struct list_head *btrfs_get_workspace(struct btrfs_fs_info *fs_info, int type, int level); +void btrfs_put_workspace(struct btrfs_fs_info *fs_info, int type, struct list_head *ws); + +struct btrfs_compress_levels { + /* Maximum level supported by the compression algorithm */ + int min_level; + int max_level; + int default_level; +}; + +/* The heuristic workspaces are managed via the 0th workspace manager */ +#define BTRFS_NR_WORKSPACE_MANAGERS BTRFS_NR_COMPRESS_TYPES + +extern const struct btrfs_compress_levels btrfs_heuristic_compress; +extern const struct btrfs_compress_levels btrfs_zlib_compress; +extern const struct btrfs_compress_levels btrfs_lzo_compress; +extern const struct btrfs_compress_levels btrfs_zstd_compress; + +const char* btrfs_compress_type2str(enum btrfs_compression_type type); +bool btrfs_compress_is_valid_type(const char *str, size_t len); + +int btrfs_compress_heuristic(struct btrfs_inode *inode, u64 start, u64 end); + +int btrfs_compress_filemap_get_folio(struct address_space *mapping, u64 start, + struct folio **in_folio_ret); +struct compressed_bio *btrfs_compress_bio(struct btrfs_inode *inode, + u64 start, u32 len, unsigned int type, + int level, blk_opf_t write_flags); + +static inline void cleanup_compressed_bio(struct compressed_bio *cb) +{ + struct bio *bio = &cb->bbio.bio; + struct folio_iter fi; + + bio_for_each_folio_all(fi, bio) + btrfs_free_compr_folio(fi.folio); + bio_put(bio); +} + +int zlib_compress_bio(struct list_head *ws, struct compressed_bio *cb); +int zlib_decompress_bio(struct list_head *ws, struct compressed_bio *cb); +int zlib_decompress(struct list_head *ws, const u8 *data_in, + struct folio *dest_folio, unsigned long dest_pgoff, size_t srclen, + size_t destlen); +struct list_head *zlib_alloc_workspace(struct btrfs_fs_info *fs_info, unsigned int level); +void zlib_free_workspace(struct list_head *ws); +struct list_head *zlib_get_workspace(struct btrfs_fs_info *fs_info, unsigned int level); + +int lzo_compress_bio(struct list_head *ws, struct compressed_bio *cb); +int lzo_decompress_bio(struct list_head *ws, struct compressed_bio *cb); +int lzo_decompress(struct list_head *ws, const u8 *data_in, + struct folio *dest_folio, unsigned long dest_pgoff, size_t srclen, + size_t destlen); +struct list_head *lzo_alloc_workspace(struct btrfs_fs_info *fs_info); +void lzo_free_workspace(struct list_head *ws); + +int zstd_compress_bio(struct list_head *ws, struct compressed_bio *cb); +int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb); +int zstd_decompress(struct list_head *ws, const u8 *data_in, + struct folio *dest_folio, unsigned long dest_pgoff, size_t srclen, + size_t destlen); +int zstd_alloc_workspace_manager(struct btrfs_fs_info *fs_info); +void zstd_free_workspace_manager(struct btrfs_fs_info *fs_info); +struct list_head *zstd_alloc_workspace(struct btrfs_fs_info *fs_info, int level); +void zstd_free_workspace(struct list_head *ws); +struct list_head *zstd_get_workspace(struct btrfs_fs_info *fs_info, int level); +void zstd_put_workspace(struct btrfs_fs_info *fs_info, struct list_head *ws); + +#endif diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c new file mode 100644 index 000000000..8fe330d81 --- /dev/null +++ b/fs/btrfs/ctree.c @@ -0,0 +1,5091 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2007,2008 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include "messages.h" +#include "ctree.h" +#include "disk-io.h" +#include "transaction.h" +#include "print-tree.h" +#include "locking.h" +#include "volumes.h" +#include "qgroup.h" +#include "tree-mod-log.h" +#include "tree-checker.h" +#include "fs.h" +#include "accessors.h" +#include "extent-tree.h" +#include "extent_io.h" +#include "relocation.h" +#include "file-item.h" + +static struct kmem_cache *btrfs_path_cachep; + +static int split_node(struct btrfs_trans_handle *trans, struct btrfs_root + *root, struct btrfs_path *path, int level); +static int split_leaf(struct btrfs_trans_handle *trans, struct btrfs_root *root, + const struct btrfs_key *ins_key, struct btrfs_path *path, + int data_size, bool extend); +static int push_node_left(struct btrfs_trans_handle *trans, + struct extent_buffer *dst, + struct extent_buffer *src, bool empty); +static int balance_node_right(struct btrfs_trans_handle *trans, + struct extent_buffer *dst_buf, + struct extent_buffer *src_buf); +/* + * The leaf data grows from end-to-front in the node. this returns the address + * of the start of the last item, which is the stop of the leaf data stack. + */ +static unsigned int leaf_data_end(const struct extent_buffer *leaf) +{ + u32 nr = btrfs_header_nritems(leaf); + + if (nr == 0) + return BTRFS_LEAF_DATA_SIZE(leaf->fs_info); + return btrfs_item_offset(leaf, nr - 1); +} + +/* + * Move data in a @leaf (using memmove, safe for overlapping ranges). + * + * @leaf: leaf that we're doing a memmove on + * @dst_offset: item data offset we're moving to + * @src_offset: item data offset were' moving from + * @len: length of the data we're moving + * + * Wrapper around memmove_extent_buffer() that takes into account the header on + * the leaf. The btrfs_item offset's start directly after the header, so we + * have to adjust any offsets to account for the header in the leaf. This + * handles that math to simplify the callers. + */ +static inline void memmove_leaf_data(const struct extent_buffer *leaf, + unsigned long dst_offset, + unsigned long src_offset, + unsigned long len) +{ + memmove_extent_buffer(leaf, btrfs_item_nr_offset(leaf, 0) + dst_offset, + btrfs_item_nr_offset(leaf, 0) + src_offset, len); +} + +/* + * Copy item data from @src into @dst at the given @offset. + * + * @dst: destination leaf that we're copying into + * @src: source leaf that we're copying from + * @dst_offset: item data offset we're copying to + * @src_offset: item data offset were' copying from + * @len: length of the data we're copying + * + * Wrapper around copy_extent_buffer() that takes into account the header on + * the leaf. The btrfs_item offset's start directly after the header, so we + * have to adjust any offsets to account for the header in the leaf. This + * handles that math to simplify the callers. + */ +static inline void copy_leaf_data(const struct extent_buffer *dst, + const struct extent_buffer *src, + unsigned long dst_offset, + unsigned long src_offset, unsigned long len) +{ + copy_extent_buffer(dst, src, btrfs_item_nr_offset(dst, 0) + dst_offset, + btrfs_item_nr_offset(src, 0) + src_offset, len); +} + +/* + * Move items in a @leaf (using memmove). + * + * @dst: destination leaf for the items + * @dst_item: the item nr we're copying into + * @src_item: the item nr we're copying from + * @nr_items: the number of items to copy + * + * Wrapper around memmove_extent_buffer() that does the math to get the + * appropriate offsets into the leaf from the item numbers. + */ +static inline void memmove_leaf_items(const struct extent_buffer *leaf, + int dst_item, int src_item, int nr_items) +{ + memmove_extent_buffer(leaf, btrfs_item_nr_offset(leaf, dst_item), + btrfs_item_nr_offset(leaf, src_item), + nr_items * sizeof(struct btrfs_item)); +} + +/* + * Copy items from @src into @dst at the given @offset. + * + * @dst: destination leaf for the items + * @src: source leaf for the items + * @dst_item: the item nr we're copying into + * @src_item: the item nr we're copying from + * @nr_items: the number of items to copy + * + * Wrapper around copy_extent_buffer() that does the math to get the + * appropriate offsets into the leaf from the item numbers. + */ +static inline void copy_leaf_items(const struct extent_buffer *dst, + const struct extent_buffer *src, + int dst_item, int src_item, int nr_items) +{ + copy_extent_buffer(dst, src, btrfs_item_nr_offset(dst, dst_item), + btrfs_item_nr_offset(src, src_item), + nr_items * sizeof(struct btrfs_item)); +} + +struct btrfs_path *btrfs_alloc_path(void) +{ + might_sleep(); + + return kmem_cache_zalloc(btrfs_path_cachep, GFP_NOFS); +} + +/* this also releases the path */ +void btrfs_free_path(struct btrfs_path *p) +{ + if (!p) + return; + btrfs_release_path(p); + kmem_cache_free(btrfs_path_cachep, p); +} + +/* + * path release drops references on the extent buffers in the path + * and it drops any locks held by this path + * + * It is safe to call this on paths that no locks or extent buffers held. + */ +noinline void btrfs_release_path(struct btrfs_path *p) +{ + int i; + + for (i = 0; i < BTRFS_MAX_LEVEL; i++) { + p->slots[i] = 0; + if (!p->nodes[i]) + continue; + if (p->locks[i]) { + btrfs_tree_unlock_rw(p->nodes[i], p->locks[i]); + p->locks[i] = 0; + } + free_extent_buffer(p->nodes[i]); + p->nodes[i] = NULL; + } +} + +/* + * safely gets a reference on the root node of a tree. A lock + * is not taken, so a concurrent writer may put a different node + * at the root of the tree. See btrfs_lock_root_node for the + * looping required. + * + * The extent buffer returned by this has a reference taken, so + * it won't disappear. It may stop being the root of the tree + * at any time because there are no locks held. + */ +struct extent_buffer *btrfs_root_node(struct btrfs_root *root) +{ + struct extent_buffer *eb; + + while (1) { + rcu_read_lock(); + eb = rcu_dereference(root->node); + + /* + * RCU really hurts here, we could free up the root node because + * it was COWed but we may not get the new root node yet so do + * the inc_not_zero dance and if it doesn't work then + * synchronize_rcu and try again. + */ + if (refcount_inc_not_zero(&eb->refs)) { + rcu_read_unlock(); + break; + } + rcu_read_unlock(); + synchronize_rcu(); + } + return eb; +} + +/* + * Cowonly root (not-shareable trees, everything not subvolume or reloc roots), + * just get put onto a simple dirty list. Transaction walks this list to make + * sure they get properly updated on disk. + */ +static void add_root_to_dirty_list(struct btrfs_root *root) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + + if (test_bit(BTRFS_ROOT_DIRTY, &root->state) || + !test_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state)) + return; + + spin_lock(&fs_info->trans_lock); + if (!test_and_set_bit(BTRFS_ROOT_DIRTY, &root->state)) { + /* Want the extent tree to be the last on the list */ + if (btrfs_root_id(root) == BTRFS_EXTENT_TREE_OBJECTID) + list_move_tail(&root->dirty_list, + &fs_info->dirty_cowonly_roots); + else + list_move(&root->dirty_list, + &fs_info->dirty_cowonly_roots); + } + spin_unlock(&fs_info->trans_lock); +} + +/* + * used by snapshot creation to make a copy of a root for a tree with + * a given objectid. The buffer with the new root node is returned in + * cow_ret, and this func returns zero on success or a negative error code. + */ +int btrfs_copy_root(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct extent_buffer *buf, + struct extent_buffer **cow_ret, u64 new_root_objectid) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct extent_buffer *cow; + int ret = 0; + int level; + struct btrfs_disk_key disk_key; + const bool is_reloc_root = (new_root_objectid == BTRFS_TREE_RELOC_OBJECTID); + u64 reloc_src_root = 0; + + WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && + trans->transid != fs_info->running_transaction->transid); + WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && + trans->transid != btrfs_get_root_last_trans(root)); + + level = btrfs_header_level(buf); + if (level == 0) + btrfs_item_key(buf, &disk_key, 0); + else + btrfs_node_key(buf, &disk_key, 0); + + if (is_reloc_root) + reloc_src_root = btrfs_header_owner(buf); + cow = btrfs_alloc_tree_block(trans, root, 0, new_root_objectid, + &disk_key, level, buf->start, 0, + reloc_src_root, BTRFS_NESTING_NEW_ROOT); + if (IS_ERR(cow)) + return PTR_ERR(cow); + + copy_extent_buffer_full(cow, buf); + btrfs_set_header_bytenr(cow, cow->start); + btrfs_set_header_generation(cow, trans->transid); + btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); + btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | + BTRFS_HEADER_FLAG_RELOC); + if (is_reloc_root) + btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); + else + btrfs_set_header_owner(cow, new_root_objectid); + + write_extent_buffer_fsid(cow, fs_info->fs_devices->metadata_uuid); + + if (unlikely(btrfs_header_generation(buf) > trans->transid)) { + btrfs_tree_unlock(cow); + free_extent_buffer(cow); + ret = -EUCLEAN; + btrfs_abort_transaction(trans, ret); + return ret; + } + + ret = btrfs_inc_ref(trans, root, cow, is_reloc_root); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + btrfs_tree_unlock(cow); + free_extent_buffer(cow); + return ret; + } + + btrfs_mark_buffer_dirty(trans, cow); + *cow_ret = cow; + return 0; +} + +/* + * check if the tree block can be shared by multiple trees + */ +bool btrfs_block_can_be_shared(const struct btrfs_trans_handle *trans, + const struct btrfs_root *root, + const struct extent_buffer *buf) +{ + const u64 buf_gen = btrfs_header_generation(buf); + + /* + * Tree blocks not in shareable trees and tree roots are never shared. + * If a block was allocated after the last snapshot and the block was + * not allocated by tree relocation, we know the block is not shared. + */ + + if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) + return false; + + if (buf == root->node) + return false; + + if (buf_gen > btrfs_root_last_snapshot(&root->root_item) && + !btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC)) + return false; + + if (buf != root->commit_root) + return true; + + /* + * An extent buffer that used to be the commit root may still be shared + * because the tree height may have increased and it became a child of a + * higher level root. This can happen when snapshotting a subvolume + * created in the current transaction. + */ + if (buf_gen == trans->transid) + return true; + + return false; +} + +static noinline int update_ref_for_cow(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct extent_buffer *buf, + struct extent_buffer *cow, + int *last_ref) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + u64 refs; + u64 owner; + u64 flags; + int ret; + const bool is_reloc_root = (btrfs_root_id(root) == BTRFS_TREE_RELOC_OBJECTID); + + /* + * Backrefs update rules: + * + * Always use full backrefs for extent pointers in tree block + * allocated by tree relocation. + * + * If a shared tree block is no longer referenced by its owner + * tree (btrfs_header_owner(buf) == root->root_key.objectid), + * use full backrefs for extent pointers in tree block. + * + * If a tree block is been relocating + * (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID), + * use full backrefs for extent pointers in tree block. + * The reason for this is some operations (such as drop tree) + * are only allowed for blocks use full backrefs. + */ + + if (btrfs_block_can_be_shared(trans, root, buf)) { + ret = btrfs_lookup_extent_info(trans, fs_info, buf->start, + btrfs_header_level(buf), 1, + &refs, &flags, NULL); + if (ret) + return ret; + if (unlikely(refs == 0)) { + btrfs_crit(fs_info, + "found 0 references for tree block at bytenr %llu level %d root %llu", + buf->start, btrfs_header_level(buf), + btrfs_root_id(root)); + ret = -EUCLEAN; + btrfs_abort_transaction(trans, ret); + return ret; + } + } else { + refs = 1; + if (is_reloc_root || btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) + flags = BTRFS_BLOCK_FLAG_FULL_BACKREF; + else + flags = 0; + } + + owner = btrfs_header_owner(buf); + if (unlikely(owner == BTRFS_TREE_RELOC_OBJECTID && + !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))) { + btrfs_crit(fs_info, +"found tree block at bytenr %llu level %d root %llu refs %llu flags %llx without full backref flag set", + buf->start, btrfs_header_level(buf), + btrfs_root_id(root), refs, flags); + ret = -EUCLEAN; + btrfs_abort_transaction(trans, ret); + return ret; + } + + if (refs > 1) { + if ((owner == btrfs_root_id(root) || is_reloc_root) && + !(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) { + ret = btrfs_inc_ref(trans, root, buf, true); + if (ret) + return ret; + + if (is_reloc_root) { + ret = btrfs_dec_ref(trans, root, buf, false); + if (ret) + return ret; + ret = btrfs_inc_ref(trans, root, cow, true); + if (ret) + return ret; + } + ret = btrfs_set_disk_extent_flags(trans, buf, + BTRFS_BLOCK_FLAG_FULL_BACKREF); + if (ret) + return ret; + } else { + ret = btrfs_inc_ref(trans, root, cow, is_reloc_root); + if (ret) + return ret; + } + } else { + if (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF) { + ret = btrfs_inc_ref(trans, root, cow, is_reloc_root); + if (ret) + return ret; + ret = btrfs_dec_ref(trans, root, buf, true); + if (ret) + return ret; + } + btrfs_clear_buffer_dirty(trans, buf); + *last_ref = 1; + } + return 0; +} + +/* + * does the dirty work in cow of a single block. The parent block (if + * supplied) is updated to point to the new cow copy. The new buffer is marked + * dirty and returned locked. If you modify the block it needs to be marked + * dirty again. + * + * search_start -- an allocation hint for the new block + * + * empty_size -- a hint that you plan on doing more cow. This is the size in + * bytes the allocator should try to find free next to the block it returns. + * This is just a hint and may be ignored by the allocator. + */ +int btrfs_force_cow_block(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct extent_buffer *buf, + struct extent_buffer *parent, int parent_slot, + struct extent_buffer **cow_ret, + u64 search_start, u64 empty_size, + enum btrfs_lock_nesting nest) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_disk_key disk_key; + struct extent_buffer *cow; + int level, ret; + int last_ref = 0; + const bool unlock_orig = (*cow_ret == buf); + u64 parent_start = 0; + u64 reloc_src_root = 0; + + btrfs_assert_tree_write_locked(buf); + + WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && + trans->transid != fs_info->running_transaction->transid); + WARN_ON(test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && + trans->transid != btrfs_get_root_last_trans(root)); + + level = btrfs_header_level(buf); + + if (level == 0) + btrfs_item_key(buf, &disk_key, 0); + else + btrfs_node_key(buf, &disk_key, 0); + + if (btrfs_root_id(root) == BTRFS_TREE_RELOC_OBJECTID) { + if (parent) + parent_start = parent->start; + reloc_src_root = btrfs_header_owner(buf); + } + cow = btrfs_alloc_tree_block(trans, root, parent_start, + btrfs_root_id(root), &disk_key, level, + search_start, empty_size, reloc_src_root, nest); + if (IS_ERR(cow)) + return PTR_ERR(cow); + + /* cow is set to blocking by btrfs_init_new_buffer */ + + copy_extent_buffer_full(cow, buf); + btrfs_set_header_bytenr(cow, cow->start); + btrfs_set_header_generation(cow, trans->transid); + btrfs_set_header_backref_rev(cow, BTRFS_MIXED_BACKREF_REV); + btrfs_clear_header_flag(cow, BTRFS_HEADER_FLAG_WRITTEN | + BTRFS_HEADER_FLAG_RELOC); + if (btrfs_root_id(root) == BTRFS_TREE_RELOC_OBJECTID) + btrfs_set_header_flag(cow, BTRFS_HEADER_FLAG_RELOC); + else + btrfs_set_header_owner(cow, btrfs_root_id(root)); + + write_extent_buffer_fsid(cow, fs_info->fs_devices->metadata_uuid); + + ret = update_ref_for_cow(trans, root, buf, cow, &last_ref); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto error_unlock_cow; + } + + if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) { + ret = btrfs_reloc_cow_block(trans, root, buf, cow); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto error_unlock_cow; + } + } + + if (buf == root->node) { + WARN_ON(parent && parent != buf); + if (btrfs_root_id(root) == BTRFS_TREE_RELOC_OBJECTID || + btrfs_header_backref_rev(buf) < BTRFS_MIXED_BACKREF_REV) + parent_start = buf->start; + + ret = btrfs_tree_mod_log_insert_root(root->node, cow, true); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + goto error_unlock_cow; + } + refcount_inc(&cow->refs); + rcu_assign_pointer(root->node, cow); + + ret = btrfs_free_tree_block(trans, btrfs_root_id(root), buf, + parent_start, last_ref); + free_extent_buffer(buf); + add_root_to_dirty_list(root); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + goto error_unlock_cow; + } + } else { + WARN_ON(trans->transid != btrfs_header_generation(parent)); + ret = btrfs_tree_mod_log_insert_key(parent, parent_slot, + BTRFS_MOD_LOG_KEY_REPLACE); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto error_unlock_cow; + } + btrfs_set_node_blockptr(parent, parent_slot, + cow->start); + btrfs_set_node_ptr_generation(parent, parent_slot, + trans->transid); + btrfs_mark_buffer_dirty(trans, parent); + if (last_ref) { + ret = btrfs_tree_mod_log_free_eb(buf); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto error_unlock_cow; + } + } + ret = btrfs_free_tree_block(trans, btrfs_root_id(root), buf, + parent_start, last_ref); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + goto error_unlock_cow; + } + } + + trace_btrfs_cow_block(root, buf, cow); + if (unlock_orig) + btrfs_tree_unlock(buf); + free_extent_buffer_stale(buf); + btrfs_mark_buffer_dirty(trans, cow); + + btrfs_inhibit_eb_writeback(trans, cow); + + *cow_ret = cow; + return 0; + +error_unlock_cow: + btrfs_tree_unlock(cow); + free_extent_buffer(cow); + return ret; +} + +static inline bool should_cow_block(struct btrfs_trans_handle *trans, + const struct btrfs_root *root, + struct extent_buffer *buf) +{ + if (btrfs_is_testing(root->fs_info)) + return false; + + /* + * We do not need to cow a block if + * 1) this block is not created or changed in this transaction; + * 2) this block does not belong to TREE_RELOC tree; + * 3) the root is not forced COW. + * + * What is forced COW: + * when we create snapshot during committing the transaction, + * after we've finished copying src root, we must COW the shared + * block to ensure the metadata consistency. + */ + + if (btrfs_header_generation(buf) != trans->transid) + return true; + + if (btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) + return true; + + /* Ensure we can see the FORCE_COW bit. */ + smp_mb__before_atomic(); + if (test_bit(BTRFS_ROOT_FORCE_COW, &root->state)) + return true; + + if (btrfs_root_id(root) == BTRFS_TREE_RELOC_OBJECTID) + return false; + + if (btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC)) + return true; + + btrfs_inhibit_eb_writeback(trans, buf); + return false; +} + +/* + * COWs a single block, see btrfs_force_cow_block() for the real work. + * This version of it has extra checks so that a block isn't COWed more than + * once per transaction, as long as it hasn't been written yet + */ +int btrfs_cow_block(struct btrfs_trans_handle *trans, + struct btrfs_root *root, struct extent_buffer *buf, + struct extent_buffer *parent, int parent_slot, + struct extent_buffer **cow_ret, + enum btrfs_lock_nesting nest) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + u64 search_start; + + if (unlikely(test_bit(BTRFS_ROOT_DELETING, &root->state))) { + btrfs_abort_transaction(trans, -EUCLEAN); + btrfs_crit(fs_info, + "attempt to COW block %llu on root %llu that is being deleted", + buf->start, btrfs_root_id(root)); + return -EUCLEAN; + } + + /* + * COWing must happen through a running transaction, which always + * matches the current fs generation (it's a transaction with a state + * less than TRANS_STATE_UNBLOCKED). If it doesn't, then turn the fs + * into error state to prevent the commit of any transaction. + */ + if (unlikely(trans->transaction != fs_info->running_transaction || + trans->transid != fs_info->generation)) { + btrfs_abort_transaction(trans, -EUCLEAN); + btrfs_crit(fs_info, +"unexpected transaction when attempting to COW block %llu on root %llu, transaction %llu running transaction %llu fs generation %llu", + buf->start, btrfs_root_id(root), trans->transid, + fs_info->running_transaction->transid, + fs_info->generation); + return -EUCLEAN; + } + + if (!should_cow_block(trans, root, buf)) { + *cow_ret = buf; + return 0; + } + + search_start = round_down(buf->start, SZ_1G); + + /* + * Before CoWing this block for later modification, check if it's + * the subtree root and do the delayed subtree trace if needed. + * + * Also We don't care about the error, as it's handled internally. + */ + btrfs_qgroup_trace_subtree_after_cow(trans, root, buf); + return btrfs_force_cow_block(trans, root, buf, parent, parent_slot, + cow_ret, search_start, 0, nest); +} +ALLOW_ERROR_INJECTION(btrfs_cow_block, ERRNO); + +/* + * same as comp_keys only with two btrfs_key's + */ +int __pure btrfs_comp_cpu_keys(const struct btrfs_key *k1, const struct btrfs_key *k2) +{ + if (k1->objectid > k2->objectid) + return 1; + if (k1->objectid < k2->objectid) + return -1; + if (k1->type > k2->type) + return 1; + if (k1->type < k2->type) + return -1; + if (k1->offset > k2->offset) + return 1; + if (k1->offset < k2->offset) + return -1; + return 0; +} + +/* + * Search for a key in the given extent_buffer. + * + * The lower boundary for the search is specified by the slot number @first_slot. + * Use a value of 0 to search over the whole extent buffer. Works for both + * leaves and nodes. + * + * The slot in the extent buffer is returned via @slot. If the key exists in the + * extent buffer, then @slot will point to the slot where the key is, otherwise + * it points to the slot where you would insert the key. + * + * Slot may point to the total number of items (i.e. one position beyond the last + * key) if the key is bigger than the last key in the extent buffer. + */ +int btrfs_bin_search(const struct extent_buffer *eb, int first_slot, + const struct btrfs_key *key, int *slot) +{ + unsigned long p; + int item_size; + /* + * Use unsigned types for the low and high slots, so that we get a more + * efficient division in the search loop below. + */ + u32 low = first_slot; + u32 high = btrfs_header_nritems(eb); + int ret; + const int key_size = sizeof(struct btrfs_disk_key); + + if (unlikely(low > high)) { + btrfs_err(eb->fs_info, + "%s: low (%u) > high (%u) eb %llu owner %llu level %d", + __func__, low, high, eb->start, + btrfs_header_owner(eb), btrfs_header_level(eb)); + return -EINVAL; + } + + if (btrfs_header_level(eb) == 0) { + p = offsetof(struct btrfs_leaf, items); + item_size = sizeof(struct btrfs_item); + } else { + p = offsetof(struct btrfs_node, ptrs); + item_size = sizeof(struct btrfs_key_ptr); + } + + while (low < high) { + const int unit_size = eb->folio_size; + unsigned long oif; + unsigned long offset; + struct btrfs_disk_key *tmp; + struct btrfs_disk_key unaligned; + u32 mid; + + mid = (low + high) / 2; + offset = p + mid * item_size; + oif = get_eb_offset_in_folio(eb, offset); + + if (oif + key_size <= unit_size) { + const unsigned long idx = get_eb_folio_index(eb, offset); + char *kaddr = folio_address(eb->folios[idx]); + + tmp = (struct btrfs_disk_key *)(kaddr + oif); + } else { + read_extent_buffer(eb, &unaligned, offset, key_size); + tmp = &unaligned; + } + + ret = btrfs_comp_keys(tmp, key); + + if (ret < 0) + low = mid + 1; + else if (ret > 0) + high = mid; + else { + *slot = mid; + return 0; + } + } + *slot = low; + return 1; +} + +static void root_add_used_bytes(struct btrfs_root *root) +{ + spin_lock(&root->accounting_lock); + btrfs_set_root_used(&root->root_item, + btrfs_root_used(&root->root_item) + root->fs_info->nodesize); + spin_unlock(&root->accounting_lock); +} + +static void root_sub_used_bytes(struct btrfs_root *root) +{ + spin_lock(&root->accounting_lock); + btrfs_set_root_used(&root->root_item, + btrfs_root_used(&root->root_item) - root->fs_info->nodesize); + spin_unlock(&root->accounting_lock); +} + +/* given a node and slot number, this reads the blocks it points to. The + * extent buffer is returned with a reference taken (but unlocked). + */ +struct extent_buffer *btrfs_read_node_slot(struct extent_buffer *parent, + int slot) +{ + int level = btrfs_header_level(parent); + struct btrfs_tree_parent_check check = { 0 }; + + if (slot < 0 || slot >= btrfs_header_nritems(parent)) + return ERR_PTR(-ENOENT); + + ASSERT(level); + + check.level = level - 1; + check.transid = btrfs_node_ptr_generation(parent, slot); + check.owner_root = btrfs_header_owner(parent); + check.has_first_key = true; + btrfs_node_key_to_cpu(parent, &check.first_key, slot); + + return read_tree_block(parent->fs_info, btrfs_node_blockptr(parent, slot), + &check); +} + +/* + * Promote a child node to become the new tree root. + * + * @trans: Transaction handle + * @root: Tree root structure to update + * @path: Path holding nodes and locks + * @level: Level of the parent (old root) + * @parent: The parent (old root) with exactly one item + * + * This helper is called during rebalancing when the root node contains only + * a single item (nritems == 1). We can reduce the tree height by promoting + * that child to become the new root and freeing the old root node. The path + * locks and references are updated accordingly. + * + * Return: 0 on success, negative errno on failure. The transaction is aborted + * on critical errors. + */ +static int promote_child_to_root(struct btrfs_trans_handle *trans, + struct btrfs_root *root, struct btrfs_path *path, + int level, struct extent_buffer *parent) +{ + struct extent_buffer *child; + int ret; + + ASSERT(btrfs_header_nritems(parent) == 1); + + child = btrfs_read_node_slot(parent, 0); + if (IS_ERR(child)) + return PTR_ERR(child); + + btrfs_tree_lock(child); + ret = btrfs_cow_block(trans, root, child, parent, 0, &child, BTRFS_NESTING_COW); + if (ret) { + btrfs_tree_unlock(child); + free_extent_buffer(child); + return ret; + } + + ret = btrfs_tree_mod_log_insert_root(root->node, child, true); + if (unlikely(ret < 0)) { + btrfs_tree_unlock(child); + free_extent_buffer(child); + btrfs_abort_transaction(trans, ret); + return ret; + } + rcu_assign_pointer(root->node, child); + + add_root_to_dirty_list(root); + btrfs_tree_unlock(child); + + path->locks[level] = 0; + path->nodes[level] = NULL; + btrfs_clear_buffer_dirty(trans, parent); + btrfs_tree_unlock(parent); + /* Once for the path. */ + free_extent_buffer(parent); + + root_sub_used_bytes(root); + ret = btrfs_free_tree_block(trans, btrfs_root_id(root), parent, 0, 1); + /* Once for the root ptr. */ + free_extent_buffer_stale(parent); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + + return 0; +} + +/* + * node level balancing, used to make sure nodes are in proper order for + * item deletion. We balance from the top down, so we have to make sure + * that a deletion won't leave an node completely empty later on. + */ +static noinline int balance_level(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, int level) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct extent_buffer *right = NULL; + struct extent_buffer *mid; + struct extent_buffer *left = NULL; + struct extent_buffer *parent = NULL; + int ret = 0; + int wret; + int pslot; + int orig_slot = path->slots[level]; + u64 orig_ptr; + + ASSERT(level > 0); + + mid = path->nodes[level]; + + WARN_ON(path->locks[level] != BTRFS_WRITE_LOCK); + WARN_ON(btrfs_header_generation(mid) != trans->transid); + + orig_ptr = btrfs_node_blockptr(mid, orig_slot); + + if (level < BTRFS_MAX_LEVEL - 1) { + parent = path->nodes[level + 1]; + pslot = path->slots[level + 1]; + } + + /* + * deal with the case where there is only one pointer in the root + * by promoting the node below to a root + */ + if (!parent) { + if (btrfs_header_nritems(mid) != 1) + return 0; + + return promote_child_to_root(trans, root, path, level, mid); + } + if (btrfs_header_nritems(mid) > + BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 4) + return 0; + + if (pslot) { + left = btrfs_read_node_slot(parent, pslot - 1); + if (IS_ERR(left)) { + ret = PTR_ERR(left); + left = NULL; + goto out; + } + + btrfs_tree_lock_nested(left, BTRFS_NESTING_LEFT); + wret = btrfs_cow_block(trans, root, left, + parent, pslot - 1, &left, + BTRFS_NESTING_LEFT_COW); + if (wret) { + ret = wret; + goto out; + } + } + + if (pslot + 1 < btrfs_header_nritems(parent)) { + right = btrfs_read_node_slot(parent, pslot + 1); + if (IS_ERR(right)) { + ret = PTR_ERR(right); + right = NULL; + goto out; + } + + btrfs_tree_lock_nested(right, BTRFS_NESTING_RIGHT); + wret = btrfs_cow_block(trans, root, right, + parent, pslot + 1, &right, + BTRFS_NESTING_RIGHT_COW); + if (wret) { + ret = wret; + goto out; + } + } + + /* first, try to make some room in the middle buffer */ + if (left) { + orig_slot += btrfs_header_nritems(left); + wret = push_node_left(trans, left, mid, 1); + if (wret < 0) + ret = wret; + } + + /* + * then try to empty the right most buffer into the middle + */ + if (right) { + wret = push_node_left(trans, mid, right, 1); + if (wret < 0 && wret != -ENOSPC) + ret = wret; + if (btrfs_header_nritems(right) == 0) { + btrfs_clear_buffer_dirty(trans, right); + btrfs_tree_unlock(right); + ret = btrfs_del_ptr(trans, root, path, level + 1, pslot + 1); + if (ret < 0) { + free_extent_buffer_stale(right); + right = NULL; + goto out; + } + root_sub_used_bytes(root); + ret = btrfs_free_tree_block(trans, btrfs_root_id(root), + right, 0, 1); + free_extent_buffer_stale(right); + right = NULL; + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + } else { + struct btrfs_disk_key right_key; + btrfs_node_key(right, &right_key, 0); + ret = btrfs_tree_mod_log_insert_key(parent, pslot + 1, + BTRFS_MOD_LOG_KEY_REPLACE); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + btrfs_set_node_key(parent, &right_key, pslot + 1); + btrfs_mark_buffer_dirty(trans, parent); + } + } + if (btrfs_header_nritems(mid) == 1) { + /* + * we're not allowed to leave a node with one item in the + * tree during a delete. A deletion from lower in the tree + * could try to delete the only pointer in this node. + * So, pull some keys from the left. + * There has to be a left pointer at this point because + * otherwise we would have pulled some pointers from the + * right + */ + if (unlikely(!left)) { + btrfs_crit(fs_info, +"missing left child when middle child only has 1 item, parent bytenr %llu level %d mid bytenr %llu root %llu", + parent->start, btrfs_header_level(parent), + mid->start, btrfs_root_id(root)); + ret = -EUCLEAN; + btrfs_abort_transaction(trans, ret); + goto out; + } + wret = balance_node_right(trans, mid, left); + if (wret < 0) { + ret = wret; + goto out; + } + if (wret == 1) { + wret = push_node_left(trans, left, mid, 1); + if (wret < 0) + ret = wret; + } + BUG_ON(wret == 1); + } + if (btrfs_header_nritems(mid) == 0) { + btrfs_clear_buffer_dirty(trans, mid); + btrfs_tree_unlock(mid); + ret = btrfs_del_ptr(trans, root, path, level + 1, pslot); + if (ret < 0) { + free_extent_buffer_stale(mid); + mid = NULL; + goto out; + } + root_sub_used_bytes(root); + ret = btrfs_free_tree_block(trans, btrfs_root_id(root), mid, 0, 1); + free_extent_buffer_stale(mid); + mid = NULL; + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + } else { + /* update the parent key to reflect our changes */ + struct btrfs_disk_key mid_key; + btrfs_node_key(mid, &mid_key, 0); + ret = btrfs_tree_mod_log_insert_key(parent, pslot, + BTRFS_MOD_LOG_KEY_REPLACE); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + btrfs_set_node_key(parent, &mid_key, pslot); + btrfs_mark_buffer_dirty(trans, parent); + } + + /* update the path */ + if (left) { + if (btrfs_header_nritems(left) > orig_slot) { + /* left was locked after cow */ + path->nodes[level] = left; + path->slots[level + 1] -= 1; + path->slots[level] = orig_slot; + /* Left is now owned by path. */ + left = NULL; + if (mid) { + btrfs_tree_unlock(mid); + free_extent_buffer(mid); + } + } else { + orig_slot -= btrfs_header_nritems(left); + path->slots[level] = orig_slot; + } + } + /* double check we haven't messed things up */ + if (orig_ptr != + btrfs_node_blockptr(path->nodes[level], path->slots[level])) + BUG(); +out: + if (right) { + btrfs_tree_unlock(right); + free_extent_buffer(right); + } + if (left) { + btrfs_tree_unlock(left); + free_extent_buffer(left); + } + return ret; +} + +/* Node balancing for insertion. Here we only split or push nodes around + * when they are completely full. This is also done top down, so we + * have to be pessimistic. + */ +static noinline int push_nodes_for_insert(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, int level) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct extent_buffer *right = NULL; + struct extent_buffer *mid; + struct extent_buffer *left = NULL; + struct extent_buffer *parent = NULL; + int ret = 0; + int wret; + int pslot; + int orig_slot = path->slots[level]; + + if (level == 0) + return 1; + + mid = path->nodes[level]; + WARN_ON(btrfs_header_generation(mid) != trans->transid); + + if (level < BTRFS_MAX_LEVEL - 1) { + parent = path->nodes[level + 1]; + pslot = path->slots[level + 1]; + } + + if (!parent) + return 1; + + /* first, try to make some room in the middle buffer */ + if (pslot) { + u32 left_nr; + + left = btrfs_read_node_slot(parent, pslot - 1); + if (IS_ERR(left)) + return PTR_ERR(left); + + btrfs_tree_lock_nested(left, BTRFS_NESTING_LEFT); + + left_nr = btrfs_header_nritems(left); + if (left_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) { + wret = 1; + } else { + ret = btrfs_cow_block(trans, root, left, parent, + pslot - 1, &left, + BTRFS_NESTING_LEFT_COW); + if (ret) + wret = 1; + else { + wret = push_node_left(trans, left, mid, 0); + } + } + if (wret < 0) + ret = wret; + if (wret == 0) { + struct btrfs_disk_key disk_key; + orig_slot += left_nr; + btrfs_node_key(mid, &disk_key, 0); + ret = btrfs_tree_mod_log_insert_key(parent, pslot, + BTRFS_MOD_LOG_KEY_REPLACE); + if (unlikely(ret < 0)) { + btrfs_tree_unlock(left); + free_extent_buffer(left); + btrfs_abort_transaction(trans, ret); + return ret; + } + btrfs_set_node_key(parent, &disk_key, pslot); + btrfs_mark_buffer_dirty(trans, parent); + if (btrfs_header_nritems(left) > orig_slot) { + path->nodes[level] = left; + path->slots[level + 1] -= 1; + path->slots[level] = orig_slot; + btrfs_tree_unlock(mid); + free_extent_buffer(mid); + } else { + orig_slot -= + btrfs_header_nritems(left); + path->slots[level] = orig_slot; + btrfs_tree_unlock(left); + free_extent_buffer(left); + } + return 0; + } + btrfs_tree_unlock(left); + free_extent_buffer(left); + } + + /* + * then try to empty the right most buffer into the middle + */ + if (pslot + 1 < btrfs_header_nritems(parent)) { + u32 right_nr; + + right = btrfs_read_node_slot(parent, pslot + 1); + if (IS_ERR(right)) + return PTR_ERR(right); + + btrfs_tree_lock_nested(right, BTRFS_NESTING_RIGHT); + + right_nr = btrfs_header_nritems(right); + if (right_nr >= BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 1) { + wret = 1; + } else { + ret = btrfs_cow_block(trans, root, right, + parent, pslot + 1, + &right, BTRFS_NESTING_RIGHT_COW); + if (ret) + wret = 1; + else { + wret = balance_node_right(trans, right, mid); + } + } + if (wret < 0) + ret = wret; + if (wret == 0) { + struct btrfs_disk_key disk_key; + + btrfs_node_key(right, &disk_key, 0); + ret = btrfs_tree_mod_log_insert_key(parent, pslot + 1, + BTRFS_MOD_LOG_KEY_REPLACE); + if (unlikely(ret < 0)) { + btrfs_tree_unlock(right); + free_extent_buffer(right); + btrfs_abort_transaction(trans, ret); + return ret; + } + btrfs_set_node_key(parent, &disk_key, pslot + 1); + btrfs_mark_buffer_dirty(trans, parent); + + if (btrfs_header_nritems(mid) <= orig_slot) { + path->nodes[level] = right; + path->slots[level + 1] += 1; + path->slots[level] = orig_slot - + btrfs_header_nritems(mid); + btrfs_tree_unlock(mid); + free_extent_buffer(mid); + } else { + btrfs_tree_unlock(right); + free_extent_buffer(right); + } + return 0; + } + btrfs_tree_unlock(right); + free_extent_buffer(right); + } + return 1; +} + +/* + * readahead one full node of leaves, finding things that are close + * to the block in 'slot', and triggering ra on them. + */ +static void reada_for_search(struct btrfs_fs_info *fs_info, + const struct btrfs_path *path, + int level, int slot, u64 objectid) +{ + struct extent_buffer *node; + struct btrfs_disk_key disk_key; + u32 nritems; + u64 search; + u64 target; + u64 nread = 0; + u64 nread_max; + u32 nr; + u32 blocksize; + u32 nscan = 0; + + if (level != 1 && path->reada != READA_FORWARD_ALWAYS) + return; + + if (!path->nodes[level]) + return; + + node = path->nodes[level]; + + /* + * Since the time between visiting leaves is much shorter than the time + * between visiting nodes, limit read ahead of nodes to 1, to avoid too + * much IO at once (possibly random). + */ + if (path->reada == READA_FORWARD_ALWAYS) { + if (level > 1) + nread_max = node->fs_info->nodesize; + else + nread_max = SZ_128K; + } else { + nread_max = SZ_64K; + } + + search = btrfs_node_blockptr(node, slot); + blocksize = fs_info->nodesize; + if (path->reada != READA_FORWARD_ALWAYS) { + struct extent_buffer *eb; + + eb = find_extent_buffer(fs_info, search); + if (eb) { + free_extent_buffer(eb); + return; + } + } + + target = search; + + nritems = btrfs_header_nritems(node); + nr = slot; + + while (1) { + if (path->reada == READA_BACK) { + if (nr == 0) + break; + nr--; + } else if (path->reada == READA_FORWARD || + path->reada == READA_FORWARD_ALWAYS) { + nr++; + if (nr >= nritems) + break; + } + if (path->reada == READA_BACK && objectid) { + btrfs_node_key(node, &disk_key, nr); + if (btrfs_disk_key_objectid(&disk_key) != objectid) + break; + } + search = btrfs_node_blockptr(node, nr); + if (path->reada == READA_FORWARD_ALWAYS || + (search <= target && target - search <= 65536) || + (search > target && search - target <= 65536)) { + btrfs_readahead_node_child(node, nr); + nread += blocksize; + } + nscan++; + if (nread > nread_max || nscan > 32) + break; + } +} + +static noinline void reada_for_balance(const struct btrfs_path *path, int level) +{ + struct extent_buffer *parent; + int slot; + int nritems; + + parent = path->nodes[level + 1]; + if (!parent) + return; + + nritems = btrfs_header_nritems(parent); + slot = path->slots[level + 1]; + + if (slot > 0) + btrfs_readahead_node_child(parent, slot - 1); + if (slot + 1 < nritems) + btrfs_readahead_node_child(parent, slot + 1); +} + + +/* + * when we walk down the tree, it is usually safe to unlock the higher layers + * in the tree. The exceptions are when our path goes through slot 0, because + * operations on the tree might require changing key pointers higher up in the + * tree. + * + * callers might also have set path->keep_locks, which tells this code to keep + * the lock if the path points to the last slot in the block. This is part of + * walking through the tree, and selecting the next slot in the higher block. + * + * lowest_unlock sets the lowest level in the tree we're allowed to unlock. so + * if lowest_unlock is 1, level 0 won't be unlocked + */ +static noinline void unlock_up(struct btrfs_path *path, int level, + int lowest_unlock, int min_write_lock_level, + int *write_lock_level) +{ + int i; + int skip_level = level; + bool check_skip = true; + + for (i = level; i < BTRFS_MAX_LEVEL; i++) { + if (!path->nodes[i]) + break; + if (!path->locks[i]) + break; + + if (check_skip) { + if (path->slots[i] == 0) { + skip_level = i + 1; + continue; + } + + if (path->keep_locks) { + u32 nritems; + + nritems = btrfs_header_nritems(path->nodes[i]); + if (nritems < 1 || path->slots[i] >= nritems - 1) { + skip_level = i + 1; + continue; + } + } + } + + if (i >= lowest_unlock && i > skip_level) { + btrfs_tree_unlock_rw(path->nodes[i], path->locks[i]); + check_skip = false; + path->locks[i] = 0; + if (write_lock_level && + i > min_write_lock_level && + i <= *write_lock_level) { + *write_lock_level = i - 1; + } + } + } +} + +/* + * Helper function for btrfs_search_slot() and other functions that do a search + * on a btree. The goal is to find a tree block in the cache (the radix tree at + * fs_info->buffer_radix), but if we can't find it, or it's not up to date, read + * its pages from disk. + * + * Returns -EAGAIN, with the path unlocked, if the caller needs to repeat the + * whole btree search, starting again from the current root node. + */ +static int +read_block_for_search(struct btrfs_root *root, struct btrfs_path *p, + struct btrfs_eb_prealloc *pa, + struct extent_buffer **eb_ret, int slot, + const struct btrfs_key *key) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_tree_parent_check check = { 0 }; + u64 blocknr; + struct extent_buffer *tmp = NULL; + int ret = 0; + int ret2; + int parent_level; + bool read_tmp = false; + bool tmp_locked = false; + bool path_released = false; + + blocknr = btrfs_node_blockptr(*eb_ret, slot); + parent_level = btrfs_header_level(*eb_ret); + btrfs_node_key_to_cpu(*eb_ret, &check.first_key, slot); + check.has_first_key = true; + check.level = parent_level - 1; + check.transid = btrfs_node_ptr_generation(*eb_ret, slot); + check.owner_root = btrfs_root_id(root); + + /* + * If we need to read an extent buffer from disk and we are holding locks + * on upper level nodes, we unlock all the upper nodes before reading the + * extent buffer, and then return -EAGAIN to the caller as it needs to + * restart the search. We don't release the lock on the current level + * because we need to walk this node to figure out which blocks to read. + */ + tmp = find_extent_buffer(fs_info, blocknr); + if (tmp) { + if (p->reada == READA_FORWARD_ALWAYS) + reada_for_search(fs_info, p, parent_level, slot, key->objectid); + + /* Check if the cached eb is uptodate. */ + ret = btrfs_buffer_uptodate(tmp, check.transid, &check); + if (unlikely(ret < 0)) + goto out; + if (ret > 0) { + *eb_ret = tmp; + tmp = NULL; + ret = 0; + goto out; + } + + if (p->nowait) { + ret = -EAGAIN; + goto out; + } + + if (!p->skip_locking) { + btrfs_unlock_up_safe(p, parent_level + 1); + btrfs_maybe_reset_lockdep_class(root, tmp); + tmp_locked = true; + btrfs_tree_read_lock(tmp); + btrfs_release_path(p); + ret = -EAGAIN; + path_released = true; + } + + /* Now we're allowed to do a blocking uptodate check. */ + ret2 = btrfs_read_extent_buffer(tmp, &check); + if (ret2) { + ret = ret2; + goto out; + } + + if (ret == 0) { + ASSERT(!tmp_locked); + *eb_ret = tmp; + tmp = NULL; + } + goto out; + } else if (p->nowait) { + ret = -EAGAIN; + goto out; + } + + if (!p->skip_locking) { + btrfs_unlock_up_safe(p, parent_level + 1); + ret = -EAGAIN; + } + + if (p->reada != READA_NONE) + reada_for_search(fs_info, p, parent_level, slot, key->objectid); + + tmp = btrfs_find_create_tree_block(fs_info, pa, blocknr, + check.owner_root, check.level); + if (IS_ERR(tmp)) { + ret = PTR_ERR(tmp); + tmp = NULL; + goto out; + } + read_tmp = true; + + if (!p->skip_locking) { + ASSERT(ret == -EAGAIN); + btrfs_maybe_reset_lockdep_class(root, tmp); + tmp_locked = true; + btrfs_tree_read_lock(tmp); + btrfs_release_path(p); + path_released = true; + } + + /* Now we're allowed to do a blocking uptodate check. */ + ret2 = btrfs_read_extent_buffer(tmp, &check); + if (ret2) { + ret = ret2; + goto out; + } + + /* + * If the read above didn't mark this buffer up to date, + * it will never end up being up to date. Set ret to EIO now + * and give up so that our caller doesn't loop forever + * on our EAGAINs. + */ + if (unlikely(!extent_buffer_uptodate(tmp))) { + ret = -EIO; + goto out; + } + + if (ret == 0) { + ASSERT(!tmp_locked); + *eb_ret = tmp; + tmp = NULL; + } +out: + if (tmp) { + if (tmp_locked) + btrfs_tree_read_unlock(tmp); + if (read_tmp && ret && ret != -EAGAIN) + free_extent_buffer_stale(tmp); + else + free_extent_buffer(tmp); + } + if (ret && !path_released) + btrfs_release_path(p); + + return ret; +} + +/* + * helper function for btrfs_search_slot. This does all of the checks + * for node-level blocks and does any balancing required based on + * the ins_len. + * + * If no extra work was required, zero is returned. If we had to + * drop the path, -EAGAIN is returned and btrfs_search_slot must + * start over + */ +static int +setup_nodes_for_search(struct btrfs_trans_handle *trans, + struct btrfs_root *root, struct btrfs_path *p, + struct extent_buffer *b, int level, int ins_len, + int *write_lock_level) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + int ret = 0; + + if ((p->search_for_split || ins_len > 0) && btrfs_header_nritems(b) >= + BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3) { + + if (*write_lock_level < level + 1) { + *write_lock_level = level + 1; + btrfs_release_path(p); + return -EAGAIN; + } + + reada_for_balance(p, level); + ret = split_node(trans, root, p, level); + + b = p->nodes[level]; + } else if (ins_len < 0 && btrfs_header_nritems(b) < + BTRFS_NODEPTRS_PER_BLOCK(fs_info) / 2) { + + if (*write_lock_level < level + 1) { + *write_lock_level = level + 1; + btrfs_release_path(p); + return -EAGAIN; + } + + reada_for_balance(p, level); + ret = balance_level(trans, root, p, level); + if (ret) + return ret; + + b = p->nodes[level]; + if (!b) { + btrfs_release_path(p); + return -EAGAIN; + } + BUG_ON(btrfs_header_nritems(b) == 1); + } + return ret; +} + +int btrfs_find_item(struct btrfs_root *fs_root, struct btrfs_path *path, + u64 iobjectid, u64 ioff, u8 key_type, + struct btrfs_key *found_key) +{ + int ret; + struct btrfs_key key; + struct extent_buffer *eb; + + ASSERT(path); + ASSERT(found_key); + + key.type = key_type; + key.objectid = iobjectid; + key.offset = ioff; + + ret = btrfs_search_slot(NULL, fs_root, &key, path, 0, 0); + if (ret < 0) + return ret; + + eb = path->nodes[0]; + if (ret && path->slots[0] >= btrfs_header_nritems(eb)) { + ret = btrfs_next_leaf(fs_root, path); + if (ret) + return ret; + eb = path->nodes[0]; + } + + btrfs_item_key_to_cpu(eb, found_key, path->slots[0]); + if (found_key->type != key.type || + found_key->objectid != key.objectid) + return 1; + + return 0; +} + +static struct extent_buffer *btrfs_search_slot_get_root(struct btrfs_root *root, + struct btrfs_path *p, + int write_lock_level) +{ + struct extent_buffer *b; + int root_lock = 0; + int level = 0; + + if (p->search_commit_root) { + b = root->commit_root; + refcount_inc(&b->refs); + level = btrfs_header_level(b); + /* + * Ensure that all callers have set skip_locking when + * p->search_commit_root is true. + */ + ASSERT(p->skip_locking); + + goto out; + } + + if (p->skip_locking) { + b = btrfs_root_node(root); + level = btrfs_header_level(b); + goto out; + } + + /* We try very hard to do read locks on the root */ + root_lock = BTRFS_READ_LOCK; + + /* + * If the level is set to maximum, we can skip trying to get the read + * lock. + */ + if (write_lock_level < BTRFS_MAX_LEVEL) { + /* + * We don't know the level of the root node until we actually + * have it read locked + */ + if (p->nowait) { + b = btrfs_try_read_lock_root_node(root); + if (IS_ERR(b)) + return b; + } else { + b = btrfs_read_lock_root_node(root); + } + level = btrfs_header_level(b); + if (level > write_lock_level) + goto out; + + /* Whoops, must trade for write lock */ + btrfs_tree_read_unlock(b); + free_extent_buffer(b); + } + + b = btrfs_lock_root_node(root); + root_lock = BTRFS_WRITE_LOCK; + + /* The level might have changed, check again */ + level = btrfs_header_level(b); + +out: + /* + * The root may have failed to write out at some point, and thus is no + * longer valid, return an error in this case. + */ + if (unlikely(!extent_buffer_uptodate(b))) { + if (root_lock) + btrfs_tree_unlock_rw(b, root_lock); + free_extent_buffer(b); + return ERR_PTR(-EIO); + } + + p->nodes[level] = b; + if (!p->skip_locking) + p->locks[level] = root_lock; + /* + * Callers are responsible for dropping b's references. + */ + return b; +} + +/* + * Replace the extent buffer at the lowest level of the path with a cloned + * version. The purpose is to be able to use it safely, after releasing the + * commit root semaphore, even if relocation is happening in parallel, the + * transaction used for relocation is committed and the extent buffer is + * reallocated in the next transaction. + * + * This is used in a context where the caller does not prevent transaction + * commits from happening, either by holding a transaction handle or holding + * some lock, while it's doing searches through a commit root. + * At the moment it's only used for send operations. + */ +static int finish_need_commit_sem_search(struct btrfs_path *path) +{ + const int i = path->lowest_level; + const int slot = path->slots[i]; + struct extent_buffer *lowest = path->nodes[i]; + struct extent_buffer *clone; + + ASSERT(path->need_commit_sem); + + if (!lowest) + return 0; + + lockdep_assert_held_read(&lowest->fs_info->commit_root_sem); + + clone = btrfs_clone_extent_buffer(lowest); + if (!clone) + return -ENOMEM; + + btrfs_release_path(path); + path->nodes[i] = clone; + path->slots[i] = slot; + + return 0; +} + +static inline int search_for_key_slot(const struct extent_buffer *eb, + int search_low_slot, + const struct btrfs_key *key, + int prev_cmp, + int *slot) +{ + /* + * If a previous call to btrfs_bin_search() on a parent node returned an + * exact match (prev_cmp == 0), we can safely assume the target key will + * always be at slot 0 on lower levels, since each key pointer + * (struct btrfs_key_ptr) refers to the lowest key accessible from the + * subtree it points to. Thus we can skip searching lower levels. + */ + if (prev_cmp == 0) { + *slot = 0; + return 0; + } + + return btrfs_bin_search(eb, search_low_slot, key, slot); +} + +static int search_leaf(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + const struct btrfs_key *key, + struct btrfs_path *path, + int ins_len, + int prev_cmp) +{ + struct extent_buffer *leaf = path->nodes[0]; + int leaf_free_space = -1; + int search_low_slot = 0; + int ret; + bool do_bin_search = true; + + /* + * If we are doing an insertion, the leaf has enough free space and the + * destination slot for the key is not slot 0, then we can unlock our + * write lock on the parent, and any other upper nodes, before doing the + * binary search on the leaf (with search_for_key_slot()), allowing other + * tasks to lock the parent and any other upper nodes. + */ + if (ins_len > 0) { + /* + * Cache the leaf free space, since we will need it later and it + * will not change until then. + */ + leaf_free_space = btrfs_leaf_free_space(leaf); + + /* + * !path->locks[1] means we have a single node tree, the leaf is + * the root of the tree. + */ + if (path->locks[1] && leaf_free_space >= ins_len) { + struct btrfs_disk_key first_key; + + ASSERT(btrfs_header_nritems(leaf) > 0); + btrfs_item_key(leaf, &first_key, 0); + + /* + * Doing the extra comparison with the first key is cheap, + * taking into account that the first key is very likely + * already in a cache line because it immediately follows + * the extent buffer's header and we have recently accessed + * the header's level field. + */ + ret = btrfs_comp_keys(&first_key, key); + if (ret < 0) { + /* + * The first key is smaller than the key we want + * to insert, so we are safe to unlock all upper + * nodes and we have to do the binary search. + * + * We do use btrfs_unlock_up_safe() and not + * unlock_up() because the later does not unlock + * nodes with a slot of 0 - we can safely unlock + * any node even if its slot is 0 since in this + * case the key does not end up at slot 0 of the + * leaf and there's no need to split the leaf. + */ + btrfs_unlock_up_safe(path, 1); + search_low_slot = 1; + } else { + /* + * The first key is >= then the key we want to + * insert, so we can skip the binary search as + * the target key will be at slot 0. + * + * We can not unlock upper nodes when the key is + * less than the first key, because we will need + * to update the key at slot 0 of the parent node + * and possibly of other upper nodes too. + * If the key matches the first key, then we can + * unlock all the upper nodes, using + * btrfs_unlock_up_safe() instead of unlock_up() + * as stated above. + */ + if (ret == 0) + btrfs_unlock_up_safe(path, 1); + /* + * ret is already 0 or 1, matching the result of + * a btrfs_bin_search() call, so there is no need + * to adjust it. + */ + do_bin_search = false; + path->slots[0] = 0; + } + } + } + + if (do_bin_search) { + ret = search_for_key_slot(leaf, search_low_slot, key, + prev_cmp, &path->slots[0]); + if (ret < 0) + return ret; + } + + if (ins_len > 0) { + /* + * Item key already exists. In this case, if we are allowed to + * insert the item (for example, in dir_item case, item key + * collision is allowed), it will be merged with the original + * item. Only the item size grows, no new btrfs item will be + * added. If search_for_extension is not set, ins_len already + * accounts the size btrfs_item, deduct it here so leaf space + * check will be correct. + */ + if (ret == 0 && !path->search_for_extension) { + ASSERT(ins_len >= sizeof(struct btrfs_item)); + ins_len -= sizeof(struct btrfs_item); + } + + ASSERT(leaf_free_space >= 0); + + if (leaf_free_space < ins_len) { + int ret2; + + ret2 = split_leaf(trans, root, key, path, ins_len, (ret == 0)); + ASSERT(ret2 <= 0); + if (WARN_ON(ret2 > 0)) + ret2 = -EUCLEAN; + if (ret2) + ret = ret2; + } + } + + return ret; +} + +/* + * Look for a key in a tree and perform necessary modifications to preserve + * tree invariants. + * + * @trans: Handle of transaction, used when modifying the tree + * @p: Holds all btree nodes along the search path + * @root: The root node of the tree + * @key: The key we are looking for + * @ins_len: Indicates purpose of search: + * >0 for inserts it's size of item inserted (*) + * <0 for deletions + * 0 for plain searches, not modifying the tree + * + * (*) If size of item inserted doesn't include + * sizeof(struct btrfs_item), then p->search_for_extension must + * be set. + * @cow: boolean should CoW operations be performed. Must always be 1 + * when modifying the tree. + * + * If @ins_len > 0, nodes and leaves will be split as we walk down the tree. + * If @ins_len < 0, nodes will be merged as we walk down the tree (if possible) + * + * If @key is found, 0 is returned and you can find the item in the leaf level + * of the path (level 0) + * + * If @key isn't found, 1 is returned and the leaf level of the path (level 0) + * points to the slot where it should be inserted + * + * If an error is encountered while searching the tree a negative error number + * is returned + */ +int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root, + const struct btrfs_key *key, struct btrfs_path *p, + int ins_len, int cow) +{ + struct btrfs_fs_info *fs_info; + struct extent_buffer *b; + int slot; + int ret; + int level; + int lowest_unlock = 1; + /* everything at write_lock_level or lower must be write locked */ + int write_lock_level = 0; + u8 lowest_level = 0; + int min_write_lock_level; + int prev_cmp; + struct btrfs_eb_prealloc pa = { .supports_nowait = true }; + + if (!root) + return -EINVAL; + + fs_info = root->fs_info; + might_sleep(); + + lowest_level = p->lowest_level; + WARN_ON(lowest_level && ins_len > 0); + WARN_ON(p->nodes[0] != NULL); + BUG_ON(!cow && ins_len); + + /* + * For now only allow nowait for read only operations. There's no + * strict reason why we can't, we just only need it for reads so it's + * only implemented for reads. + */ + ASSERT(!p->nowait || !cow); + + if (ins_len < 0) { + lowest_unlock = 2; + + /* when we are removing items, we might have to go up to level + * two as we update tree pointers Make sure we keep write + * for those levels as well + */ + write_lock_level = 2; + } else if (ins_len > 0) { + /* + * for inserting items, make sure we have a write lock on + * level 1 so we can update keys + */ + write_lock_level = 1; + } + + if (!cow) + write_lock_level = -1; + + if (cow && (p->keep_locks || p->lowest_level)) + write_lock_level = BTRFS_MAX_LEVEL; + + min_write_lock_level = write_lock_level; + + if (p->need_commit_sem) { + ASSERT(p->search_commit_root); + if (p->nowait) { + if (!down_read_trylock(&fs_info->commit_root_sem)) + return -EAGAIN; + } else { + down_read(&fs_info->commit_root_sem); + } + } + +again: + if (pa.needs_prealloc) { + ret = btrfs_init_eb_prealloc(fs_info, &pa, false); + if (ret) + goto done; + } + prev_cmp = -1; + b = btrfs_search_slot_get_root(root, p, write_lock_level); + if (IS_ERR(b)) { + ret = PTR_ERR(b); + goto done; + } + + while (b) { + bool dec = false; + int ret2; + + level = btrfs_header_level(b); + + if (cow) { + bool last_level = (level == (BTRFS_MAX_LEVEL - 1)); + + /* + * if we don't really need to cow this block + * then we don't want to set the path blocking, + * so we test it here + */ + if (!should_cow_block(trans, root, b)) + goto cow_done; + + /* + * must have write locks on this node and the + * parent + */ + if (level > write_lock_level || + (level + 1 > write_lock_level && + level + 1 < BTRFS_MAX_LEVEL && + p->nodes[level + 1])) { + write_lock_level = level + 1; + btrfs_release_path(p); + trace_btrfs_search_slot_restart(root, level, "write_lock"); + goto again; + } + + if (last_level) + ret2 = btrfs_cow_block(trans, root, b, NULL, 0, + &b, BTRFS_NESTING_COW); + else + ret2 = btrfs_cow_block(trans, root, b, + p->nodes[level + 1], + p->slots[level + 1], &b, + BTRFS_NESTING_COW); + if (ret2) { + ret = ret2; + goto done; + } + } +cow_done: + p->nodes[level] = b; + + /* + * we have a lock on b and as long as we aren't changing + * the tree, there is no way to for the items in b to change. + * It is safe to drop the lock on our parent before we + * go through the expensive btree search on b. + * + * If we're inserting or deleting (ins_len != 0), then we might + * be changing slot zero, which may require changing the parent. + * So, we can't drop the lock until after we know which slot + * we're operating on. + */ + if (!ins_len && !p->keep_locks) { + int u = level + 1; + + if (u < BTRFS_MAX_LEVEL && p->locks[u]) { + btrfs_tree_unlock_rw(p->nodes[u], p->locks[u]); + p->locks[u] = 0; + } + } + + if (level == 0) { + if (ins_len > 0) + ASSERT(write_lock_level >= 1); + + ret = search_leaf(trans, root, key, p, ins_len, prev_cmp); + if (!p->search_for_split) + unlock_up(p, level, lowest_unlock, + min_write_lock_level, NULL); + goto done; + } + + ret = search_for_key_slot(b, 0, key, prev_cmp, &slot); + if (ret < 0) + goto done; + prev_cmp = ret; + + if (ret && slot > 0) { + dec = true; + slot--; + } + p->slots[level] = slot; + ret2 = setup_nodes_for_search(trans, root, p, b, level, ins_len, + &write_lock_level); + if (ret2 == -EAGAIN) { + trace_btrfs_search_slot_restart(root, level, "setup_nodes"); + goto again; + } + if (ret2) { + ret = ret2; + goto done; + } + b = p->nodes[level]; + slot = p->slots[level]; + + /* + * Slot 0 is special, if we change the key we have to update + * the parent pointer which means we must have a write lock on + * the parent + */ + if (slot == 0 && ins_len && write_lock_level < level + 1) { + write_lock_level = level + 1; + btrfs_release_path(p); + trace_btrfs_search_slot_restart(root, level, "slot_zero"); + goto again; + } + + unlock_up(p, level, lowest_unlock, min_write_lock_level, + &write_lock_level); + + if (level == lowest_level) { + if (dec) + p->slots[level]++; + goto done; + } + + ret2 = read_block_for_search(root, p, &pa, &b, slot, key); + if (ret2 == -EAGAIN && !p->nowait) { + trace_btrfs_search_slot_restart(root, level, "read_block"); + goto again; + } + if (ret2) { + ret = ret2; + goto done; + } + + if (!p->skip_locking) { + level = btrfs_header_level(b); + + btrfs_maybe_reset_lockdep_class(root, b); + + if (level <= write_lock_level) { + btrfs_tree_lock(b); + p->locks[level] = BTRFS_WRITE_LOCK; + } else { + if (p->nowait) { + if (!btrfs_try_tree_read_lock(b)) { + free_extent_buffer(b); + ret = -EAGAIN; + goto done; + } + } else { + btrfs_tree_read_lock(b); + } + p->locks[level] = BTRFS_READ_LOCK; + } + p->nodes[level] = b; + } + } + ret = 1; +done: + if (ret < 0 && !p->skip_release_on_error) + btrfs_release_path(p); + + if (p->need_commit_sem) { + int ret2; + + ret2 = finish_need_commit_sem_search(p); + up_read(&fs_info->commit_root_sem); + if (ret2) + ret = ret2; + } + + btrfs_free_eb_prealloc(&pa); + + return ret; +} +ALLOW_ERROR_INJECTION(btrfs_search_slot, ERRNO); + +/* + * Like btrfs_search_slot, this looks for a key in the given tree. It uses the + * current state of the tree together with the operations recorded in the tree + * modification log to search for the key in a previous version of this tree, as + * denoted by the time_seq parameter. + * + * Naturally, there is no support for insert, delete or cow operations. + * + * The resulting path and return value will be set up as if we called + * btrfs_search_slot at that point in time with ins_len and cow both set to 0. + */ +int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key, + struct btrfs_path *p, u64 time_seq) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct extent_buffer *b; + int slot; + int ret; + int level; + int lowest_unlock = 1; + u8 lowest_level = 0; + struct btrfs_eb_prealloc pa = { .supports_nowait = true }; + + lowest_level = p->lowest_level; + WARN_ON(p->nodes[0] != NULL); + ASSERT(!p->nowait); + + if (p->search_commit_root) { + BUG_ON(time_seq); + return btrfs_search_slot(NULL, root, key, p, 0, 0); + } + +again: + if (pa.needs_prealloc) { + ret = btrfs_init_eb_prealloc(fs_info, &pa, false); + if (ret) + goto done; + } + b = btrfs_get_old_root(root, time_seq); + if (unlikely(!b)) { + ret = -EIO; + goto done; + } + level = btrfs_header_level(b); + p->locks[level] = BTRFS_READ_LOCK; + + while (b) { + bool dec = false; + int ret2; + + level = btrfs_header_level(b); + p->nodes[level] = b; + + /* + * we have a lock on b and as long as we aren't changing + * the tree, there is no way to for the items in b to change. + * It is safe to drop the lock on our parent before we + * go through the expensive btree search on b. + */ + btrfs_unlock_up_safe(p, level + 1); + + ret = btrfs_bin_search(b, 0, key, &slot); + if (ret < 0) + goto done; + + if (level == 0) { + p->slots[level] = slot; + unlock_up(p, level, lowest_unlock, 0, NULL); + goto done; + } + + if (ret && slot > 0) { + dec = true; + slot--; + } + p->slots[level] = slot; + unlock_up(p, level, lowest_unlock, 0, NULL); + + if (level == lowest_level) { + if (dec) + p->slots[level]++; + goto done; + } + + ret2 = read_block_for_search(root, p, &pa, &b, slot, key); + if (ret2 == -EAGAIN && !p->nowait) + goto again; + if (ret2) { + ret = ret2; + goto done; + } + + level = btrfs_header_level(b); + btrfs_tree_read_lock(b); + b = btrfs_tree_mod_log_rewind(fs_info, b, time_seq); + if (!b) { + ret = -ENOMEM; + goto done; + } + p->locks[level] = BTRFS_READ_LOCK; + p->nodes[level] = b; + } + ret = 1; +done: + if (ret < 0) + btrfs_release_path(p); + + btrfs_free_eb_prealloc(&pa); + + return ret; +} + +/* + * Search the tree again to find a leaf with smaller keys. + * Returns 0 if it found something. + * Returns 1 if there are no smaller keys. + * Returns < 0 on error. + * + * This may release the path, and so you may lose any locks held at the + * time you call it. + */ +static int btrfs_prev_leaf(struct btrfs_root *root, struct btrfs_path *path) +{ + struct btrfs_key key; + struct btrfs_key orig_key; + struct btrfs_disk_key found_key; + int ret; + + btrfs_item_key_to_cpu(path->nodes[0], &key, 0); + orig_key = key; + + if (key.offset > 0) { + key.offset--; + } else if (key.type > 0) { + key.type--; + key.offset = (u64)-1; + } else if (key.objectid > 0) { + key.objectid--; + key.type = (u8)-1; + key.offset = (u64)-1; + } else { + return 1; + } + + btrfs_release_path(path); + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret <= 0) + return ret; + + /* + * Previous key not found. Even if we were at slot 0 of the leaf we had + * before releasing the path and calling btrfs_search_slot(), we now may + * be in a slot pointing to the same original key - this can happen if + * after we released the path, one of more items were moved from a + * sibling leaf into the front of the leaf we had due to an insertion + * (see push_leaf_right()). + * If we hit this case and our slot is > 0 and just decrement the slot + * so that the caller does not process the same key again, which may or + * may not break the caller, depending on its logic. + */ + if (path->slots[0] < btrfs_header_nritems(path->nodes[0])) { + btrfs_item_key(path->nodes[0], &found_key, path->slots[0]); + ret = btrfs_comp_keys(&found_key, &orig_key); + if (ret == 0) { + if (path->slots[0] > 0) { + path->slots[0]--; + return 0; + } + /* + * At slot 0, same key as before, it means orig_key is + * the lowest, leftmost, key in the tree. We're done. + */ + return 1; + } + } + + btrfs_item_key(path->nodes[0], &found_key, 0); + ret = btrfs_comp_keys(&found_key, &key); + /* + * We might have had an item with the previous key in the tree right + * before we released our path. And after we released our path, that + * item might have been pushed to the first slot (0) of the leaf we + * were holding due to a tree balance. Alternatively, an item with the + * previous key can exist as the only element of a leaf (big fat item). + * Therefore account for these 2 cases, so that our callers (like + * btrfs_previous_item) don't miss an existing item with a key matching + * the previous key we computed above. + */ + if (ret <= 0) + return 0; + return 1; +} + +/* + * helper to use instead of search slot if no exact match is needed but + * instead the next or previous item should be returned. + * When find_higher is true, the next higher item is returned, the next lower + * otherwise. + * When return_any and find_higher are both true, and no higher item is found, + * return the next lower instead. + * When return_any is true and find_higher is false, and no lower item is found, + * return the next higher instead. + * It returns 0 if any item is found, 1 if none is found (tree empty), and + * < 0 on error + */ +int btrfs_search_slot_for_read(struct btrfs_root *root, + const struct btrfs_key *key, + struct btrfs_path *p, int find_higher, + int return_any) +{ + int ret; + struct extent_buffer *leaf; + +again: + ret = btrfs_search_slot(NULL, root, key, p, 0, 0); + if (ret <= 0) + return ret; + /* + * a return value of 1 means the path is at the position where the + * item should be inserted. Normally this is the next bigger item, + * but in case the previous item is the last in a leaf, path points + * to the first free slot in the previous leaf, i.e. at an invalid + * item. + */ + leaf = p->nodes[0]; + + if (find_higher) { + if (p->slots[0] >= btrfs_header_nritems(leaf)) { + ret = btrfs_next_leaf(root, p); + if (ret <= 0) + return ret; + if (!return_any) + return 1; + /* + * no higher item found, return the next + * lower instead + */ + return_any = 0; + find_higher = 0; + btrfs_release_path(p); + goto again; + } + } else { + if (p->slots[0] == 0) { + ret = btrfs_prev_leaf(root, p); + if (ret < 0) + return ret; + if (!ret) { + leaf = p->nodes[0]; + if (p->slots[0] == btrfs_header_nritems(leaf)) + p->slots[0]--; + return 0; + } + if (!return_any) + return 1; + /* + * no lower item found, return the next + * higher instead + */ + return_any = 0; + find_higher = 1; + btrfs_release_path(p); + goto again; + } else { + --p->slots[0]; + } + } + return 0; +} + +/* + * Execute search and call btrfs_previous_item to traverse backwards if the item + * was not found. + * + * Return 0 if found, 1 if not found and < 0 if error. + */ +int btrfs_search_backwards(struct btrfs_root *root, struct btrfs_key *key, + struct btrfs_path *path) +{ + int ret; + + ret = btrfs_search_slot(NULL, root, key, path, 0, 0); + if (ret > 0) + ret = btrfs_previous_item(root, path, key->objectid, key->type); + + if (ret == 0) + btrfs_item_key_to_cpu(path->nodes[0], key, path->slots[0]); + + return ret; +} + +/* + * Search for a valid slot for the given path. + * + * @root: The root node of the tree. + * @key: Will contain a valid item if found. + * @path: The starting point to validate the slot. + * + * Return: 0 if the item is valid + * 1 if not found + * <0 if error. + */ +int btrfs_get_next_valid_item(struct btrfs_root *root, struct btrfs_key *key, + struct btrfs_path *path) +{ + if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) { + int ret; + + ret = btrfs_next_leaf(root, path); + if (ret) + return ret; + } + + btrfs_item_key_to_cpu(path->nodes[0], key, path->slots[0]); + return 0; +} + +/* + * adjust the pointers going up the tree, starting at level + * making sure the right key of each node is points to 'key'. + * This is used after shifting pointers to the left, so it stops + * fixing up pointers when a given leaf/node is not in slot 0 of the + * higher levels + * + */ +static void fixup_low_keys(struct btrfs_trans_handle *trans, + const struct btrfs_path *path, + const struct btrfs_disk_key *key, int level) +{ + int i; + struct extent_buffer *t; + int ret; + + for (i = level; i < BTRFS_MAX_LEVEL; i++) { + int tslot = path->slots[i]; + + if (!path->nodes[i]) + break; + t = path->nodes[i]; + ret = btrfs_tree_mod_log_insert_key(t, tslot, + BTRFS_MOD_LOG_KEY_REPLACE); + BUG_ON(ret < 0); + btrfs_set_node_key(t, key, tslot); + btrfs_mark_buffer_dirty(trans, path->nodes[i]); + if (tslot != 0) + break; + } +} + +/* + * update item key. + * + * This function isn't completely safe. It's the caller's responsibility + * that the new key won't break the order + */ +void btrfs_set_item_key_safe(struct btrfs_trans_handle *trans, + const struct btrfs_path *path, + const struct btrfs_key *new_key) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_disk_key disk_key; + struct extent_buffer *eb; + int slot; + + eb = path->nodes[0]; + slot = path->slots[0]; + if (slot > 0) { + btrfs_item_key(eb, &disk_key, slot - 1); + if (unlikely(btrfs_comp_keys(&disk_key, new_key) >= 0)) { + btrfs_print_leaf(eb); + btrfs_crit(fs_info, + "slot %u key " BTRFS_KEY_FMT " new key " BTRFS_KEY_FMT, + slot, btrfs_disk_key_objectid(&disk_key), + btrfs_disk_key_type(&disk_key), + btrfs_disk_key_offset(&disk_key), + BTRFS_KEY_FMT_VALUE(new_key)); + BUG(); + } + } + if (slot < btrfs_header_nritems(eb) - 1) { + btrfs_item_key(eb, &disk_key, slot + 1); + if (unlikely(btrfs_comp_keys(&disk_key, new_key) <= 0)) { + btrfs_print_leaf(eb); + btrfs_crit(fs_info, + "slot %u key " BTRFS_KEY_FMT " new key " BTRFS_KEY_FMT, + slot, btrfs_disk_key_objectid(&disk_key), + btrfs_disk_key_type(&disk_key), + btrfs_disk_key_offset(&disk_key), + BTRFS_KEY_FMT_VALUE(new_key)); + BUG(); + } + } + + btrfs_cpu_key_to_disk(&disk_key, new_key); + btrfs_set_item_key(eb, &disk_key, slot); + btrfs_mark_buffer_dirty(trans, eb); + if (slot == 0) + fixup_low_keys(trans, path, &disk_key, 1); +} + +/* + * Check key order of two sibling extent buffers. + * + * Return true if something is wrong. + * Return false if everything is fine. + * + * Tree-checker only works inside one tree block, thus the following + * corruption can not be detected by tree-checker: + * + * Leaf @left | Leaf @right + * -------------------------------------------------------------- + * | 1 | 2 | 3 | 4 | 5 | f6 | | 7 | 8 | + * + * Key f6 in leaf @left itself is valid, but not valid when the next + * key in leaf @right is 7. + * This can only be checked at tree block merge time. + * And since tree checker has ensured all key order in each tree block + * is correct, we only need to bother the last key of @left and the first + * key of @right. + */ +static bool check_sibling_keys(const struct extent_buffer *left, + const struct extent_buffer *right) +{ + struct btrfs_key left_last; + struct btrfs_key right_first; + int level = btrfs_header_level(left); + int nr_left = btrfs_header_nritems(left); + int nr_right = btrfs_header_nritems(right); + + /* No key to check in one of the tree blocks */ + if (!nr_left || !nr_right) + return false; + + if (level) { + btrfs_node_key_to_cpu(left, &left_last, nr_left - 1); + btrfs_node_key_to_cpu(right, &right_first, 0); + } else { + btrfs_item_key_to_cpu(left, &left_last, nr_left - 1); + btrfs_item_key_to_cpu(right, &right_first, 0); + } + + if (unlikely(btrfs_comp_cpu_keys(&left_last, &right_first) >= 0)) { + btrfs_crit(left->fs_info, "left extent buffer:"); + btrfs_print_tree(left, false); + btrfs_crit(left->fs_info, "right extent buffer:"); + btrfs_print_tree(right, false); + btrfs_crit(left->fs_info, +"bad key order, sibling blocks, left last " BTRFS_KEY_FMT " right first " BTRFS_KEY_FMT, + BTRFS_KEY_FMT_VALUE(&left_last), + BTRFS_KEY_FMT_VALUE(&right_first)); + return true; + } + return false; +} + +/* + * try to push data from one node into the next node left in the + * tree. + * + * returns 0 if some ptrs were pushed left, < 0 if there was some horrible + * error, and > 0 if there was no room in the left hand block. + */ +static int push_node_left(struct btrfs_trans_handle *trans, + struct extent_buffer *dst, + struct extent_buffer *src, bool empty) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + int push_items = 0; + int src_nritems; + int dst_nritems; + int ret = 0; + + src_nritems = btrfs_header_nritems(src); + dst_nritems = btrfs_header_nritems(dst); + push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems; + WARN_ON(btrfs_header_generation(src) != trans->transid); + WARN_ON(btrfs_header_generation(dst) != trans->transid); + + if (!empty && src_nritems <= 8) + return 1; + + if (push_items <= 0) + return 1; + + if (empty) { + push_items = min(src_nritems, push_items); + if (push_items < src_nritems) { + /* leave at least 8 pointers in the node if + * we aren't going to empty it + */ + if (src_nritems - push_items < 8) { + if (push_items <= 8) + return 1; + push_items -= 8; + } + } + } else + push_items = min(src_nritems - 8, push_items); + + /* dst is the left eb, src is the middle eb */ + if (unlikely(check_sibling_keys(dst, src))) { + ret = -EUCLEAN; + btrfs_abort_transaction(trans, ret); + return ret; + } + ret = btrfs_tree_mod_log_eb_copy(dst, src, dst_nritems, 0, push_items); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + copy_extent_buffer(dst, src, + btrfs_node_key_ptr_offset(dst, dst_nritems), + btrfs_node_key_ptr_offset(src, 0), + push_items * sizeof(struct btrfs_key_ptr)); + + if (push_items < src_nritems) { + /* + * btrfs_tree_mod_log_eb_copy handles logging the move, so we + * don't need to do an explicit tree mod log operation for it. + */ + memmove_extent_buffer(src, btrfs_node_key_ptr_offset(src, 0), + btrfs_node_key_ptr_offset(src, push_items), + (src_nritems - push_items) * + sizeof(struct btrfs_key_ptr)); + } + btrfs_set_header_nritems(src, src_nritems - push_items); + btrfs_set_header_nritems(dst, dst_nritems + push_items); + btrfs_mark_buffer_dirty(trans, src); + btrfs_mark_buffer_dirty(trans, dst); + + return ret; +} + +/* + * try to push data from one node into the next node right in the + * tree. + * + * returns 0 if some ptrs were pushed, < 0 if there was some horrible + * error, and > 0 if there was no room in the right hand block. + * + * this will only push up to 1/2 the contents of the left node over + */ +static int balance_node_right(struct btrfs_trans_handle *trans, + struct extent_buffer *dst, + struct extent_buffer *src) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + int push_items = 0; + int max_push; + int src_nritems; + int dst_nritems; + int ret = 0; + + WARN_ON(btrfs_header_generation(src) != trans->transid); + WARN_ON(btrfs_header_generation(dst) != trans->transid); + + src_nritems = btrfs_header_nritems(src); + dst_nritems = btrfs_header_nritems(dst); + push_items = BTRFS_NODEPTRS_PER_BLOCK(fs_info) - dst_nritems; + if (push_items <= 0) + return 1; + + if (src_nritems < 4) + return 1; + + max_push = src_nritems / 2 + 1; + /* don't try to empty the node */ + if (max_push >= src_nritems) + return 1; + + if (max_push < push_items) + push_items = max_push; + + /* dst is the right eb, src is the middle eb */ + if (unlikely(check_sibling_keys(src, dst))) { + ret = -EUCLEAN; + btrfs_abort_transaction(trans, ret); + return ret; + } + + /* + * btrfs_tree_mod_log_eb_copy handles logging the move, so we don't + * need to do an explicit tree mod log operation for it. + */ + memmove_extent_buffer(dst, btrfs_node_key_ptr_offset(dst, push_items), + btrfs_node_key_ptr_offset(dst, 0), + (dst_nritems) * + sizeof(struct btrfs_key_ptr)); + + ret = btrfs_tree_mod_log_eb_copy(dst, src, 0, src_nritems - push_items, + push_items); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + copy_extent_buffer(dst, src, + btrfs_node_key_ptr_offset(dst, 0), + btrfs_node_key_ptr_offset(src, src_nritems - push_items), + push_items * sizeof(struct btrfs_key_ptr)); + + btrfs_set_header_nritems(src, src_nritems - push_items); + btrfs_set_header_nritems(dst, dst_nritems + push_items); + + btrfs_mark_buffer_dirty(trans, src); + btrfs_mark_buffer_dirty(trans, dst); + + return ret; +} + +/* + * helper function to insert a new root level in the tree. + * A new node is allocated, and a single item is inserted to + * point to the existing root + * + * returns zero on success or < 0 on failure. + */ +static noinline int insert_new_root(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, int level) +{ + u64 lower_gen; + struct extent_buffer *lower; + struct extent_buffer *c; + struct extent_buffer *old; + struct btrfs_disk_key lower_key; + int ret; + + BUG_ON(path->nodes[level]); + BUG_ON(path->nodes[level-1] != root->node); + + lower = path->nodes[level-1]; + if (level == 1) + btrfs_item_key(lower, &lower_key, 0); + else + btrfs_node_key(lower, &lower_key, 0); + + c = btrfs_alloc_tree_block(trans, root, 0, btrfs_root_id(root), + &lower_key, level, root->node->start, 0, + 0, BTRFS_NESTING_NEW_ROOT); + if (IS_ERR(c)) + return PTR_ERR(c); + + root_add_used_bytes(root); + + btrfs_set_header_nritems(c, 1); + btrfs_set_node_key(c, &lower_key, 0); + btrfs_set_node_blockptr(c, 0, lower->start); + lower_gen = btrfs_header_generation(lower); + WARN_ON(lower_gen != trans->transid); + + btrfs_set_node_ptr_generation(c, 0, lower_gen); + + btrfs_mark_buffer_dirty(trans, c); + + old = root->node; + ret = btrfs_tree_mod_log_insert_root(root->node, c, false); + if (ret < 0) { + int ret2; + + btrfs_clear_buffer_dirty(trans, c); + ret2 = btrfs_free_tree_block(trans, btrfs_root_id(root), c, 0, 1); + if (unlikely(ret2 < 0)) + btrfs_abort_transaction(trans, ret2); + btrfs_tree_unlock(c); + free_extent_buffer(c); + return ret; + } + rcu_assign_pointer(root->node, c); + + /* the super has an extra ref to root->node */ + free_extent_buffer(old); + + add_root_to_dirty_list(root); + refcount_inc(&c->refs); + path->nodes[level] = c; + path->locks[level] = BTRFS_WRITE_LOCK; + path->slots[level] = 0; + return 0; +} + +/* + * worker function to insert a single pointer in a node. + * the node should have enough room for the pointer already + * + * slot and level indicate where you want the key to go, and + * blocknr is the block the key points to. + */ +static int insert_ptr(struct btrfs_trans_handle *trans, + const struct btrfs_path *path, + const struct btrfs_disk_key *key, u64 bytenr, + int slot, int level) +{ + struct extent_buffer *lower; + int nritems; + int ret; + + BUG_ON(!path->nodes[level]); + btrfs_assert_tree_write_locked(path->nodes[level]); + lower = path->nodes[level]; + nritems = btrfs_header_nritems(lower); + BUG_ON(slot > nritems); + BUG_ON(nritems == BTRFS_NODEPTRS_PER_BLOCK(trans->fs_info)); + if (slot != nritems) { + if (level) { + ret = btrfs_tree_mod_log_insert_move(lower, slot + 1, + slot, nritems - slot); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + } + memmove_extent_buffer(lower, + btrfs_node_key_ptr_offset(lower, slot + 1), + btrfs_node_key_ptr_offset(lower, slot), + (nritems - slot) * sizeof(struct btrfs_key_ptr)); + } + if (level) { + ret = btrfs_tree_mod_log_insert_key(lower, slot, + BTRFS_MOD_LOG_KEY_ADD); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + } + btrfs_set_node_key(lower, key, slot); + btrfs_set_node_blockptr(lower, slot, bytenr); + WARN_ON(trans->transid == 0); + btrfs_set_node_ptr_generation(lower, slot, trans->transid); + btrfs_set_header_nritems(lower, nritems + 1); + btrfs_mark_buffer_dirty(trans, lower); + + return 0; +} + +/* + * split the node at the specified level in path in two. + * The path is corrected to point to the appropriate node after the split + * + * Before splitting this tries to make some room in the node by pushing + * left and right, if either one works, it returns right away. + * + * returns 0 on success and < 0 on failure + */ +static noinline int split_node(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, int level) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct extent_buffer *c; + struct extent_buffer *split; + struct btrfs_disk_key disk_key; + int mid; + int ret; + u32 c_nritems; + + c = path->nodes[level]; + WARN_ON(btrfs_header_generation(c) != trans->transid); + if (c == root->node) { + /* + * trying to split the root, lets make a new one + * + * tree mod log: We don't log_removal old root in + * insert_new_root, because that root buffer will be kept as a + * normal node. We are going to log removal of half of the + * elements below with btrfs_tree_mod_log_eb_copy(). We're + * holding a tree lock on the buffer, which is why we cannot + * race with other tree_mod_log users. + */ + ret = insert_new_root(trans, root, path, level + 1); + if (ret) + return ret; + } else { + ret = push_nodes_for_insert(trans, root, path, level); + c = path->nodes[level]; + if (!ret && btrfs_header_nritems(c) < + BTRFS_NODEPTRS_PER_BLOCK(fs_info) - 3) + return 0; + if (ret < 0) + return ret; + } + + c_nritems = btrfs_header_nritems(c); + mid = (c_nritems + 1) / 2; + btrfs_node_key(c, &disk_key, mid); + + split = btrfs_alloc_tree_block(trans, root, 0, btrfs_root_id(root), + &disk_key, level, c->start, 0, + 0, BTRFS_NESTING_SPLIT); + if (IS_ERR(split)) + return PTR_ERR(split); + + root_add_used_bytes(root); + ASSERT(btrfs_header_level(c) == level); + + ret = btrfs_tree_mod_log_eb_copy(split, c, 0, mid, c_nritems - mid); + if (unlikely(ret)) { + btrfs_tree_unlock(split); + free_extent_buffer(split); + btrfs_abort_transaction(trans, ret); + return ret; + } + copy_extent_buffer(split, c, + btrfs_node_key_ptr_offset(split, 0), + btrfs_node_key_ptr_offset(c, mid), + (c_nritems - mid) * sizeof(struct btrfs_key_ptr)); + btrfs_set_header_nritems(split, c_nritems - mid); + btrfs_set_header_nritems(c, mid); + + btrfs_mark_buffer_dirty(trans, c); + btrfs_mark_buffer_dirty(trans, split); + + ret = insert_ptr(trans, path, &disk_key, split->start, + path->slots[level + 1] + 1, level + 1); + if (ret < 0) { + btrfs_tree_unlock(split); + free_extent_buffer(split); + return ret; + } + + if (path->slots[level] >= mid) { + path->slots[level] -= mid; + btrfs_tree_unlock(c); + free_extent_buffer(c); + path->nodes[level] = split; + path->slots[level + 1] += 1; + } else { + btrfs_tree_unlock(split); + free_extent_buffer(split); + } + return 0; +} + +/* + * how many bytes are required to store the items in a leaf. start + * and nr indicate which items in the leaf to check. This totals up the + * space used both by the item structs and the item data + */ +static int leaf_space_used(const struct extent_buffer *l, int start, int nr) +{ + int data_len; + int nritems = btrfs_header_nritems(l); + int end = min(nritems, start + nr) - 1; + + if (!nr) + return 0; + data_len = btrfs_item_offset(l, start) + btrfs_item_size(l, start); + data_len = data_len - btrfs_item_offset(l, end); + data_len += sizeof(struct btrfs_item) * nr; + WARN_ON(data_len < 0); + return data_len; +} + +/* + * The space between the end of the leaf items and + * the start of the leaf data. IOW, how much room + * the leaf has left for both items and data + */ +int btrfs_leaf_free_space(const struct extent_buffer *leaf) +{ + struct btrfs_fs_info *fs_info = leaf->fs_info; + int nritems = btrfs_header_nritems(leaf); + int ret; + + ret = BTRFS_LEAF_DATA_SIZE(fs_info) - leaf_space_used(leaf, 0, nritems); + if (unlikely(ret < 0)) { + btrfs_crit(fs_info, + "leaf free space ret %d, leaf data size %lu, used %d nritems %d", + ret, + (unsigned long) BTRFS_LEAF_DATA_SIZE(fs_info), + leaf_space_used(leaf, 0, nritems), nritems); + } + return ret; +} + +/* + * min slot controls the lowest index we're willing to push to the + * right. We'll push up to and including min_slot, but no lower + */ +static noinline int __push_leaf_right(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + int data_size, bool empty, + struct extent_buffer *right, + int free_space, u32 left_nritems, + u32 min_slot) +{ + struct btrfs_fs_info *fs_info = right->fs_info; + struct extent_buffer *left = path->nodes[0]; + struct extent_buffer *upper = path->nodes[1]; + struct btrfs_disk_key disk_key; + int slot; + u32 i; + int push_space = 0; + int push_items = 0; + u32 nr; + u32 right_nritems; + u32 data_end; + u32 this_item_size; + + if (empty) + nr = 0; + else + nr = max_t(u32, 1, min_slot); + + if (path->slots[0] >= left_nritems) + push_space += data_size; + + slot = path->slots[1]; + i = left_nritems - 1; + while (i >= nr) { + if (!empty && push_items > 0) { + if (path->slots[0] > i) + break; + if (path->slots[0] == i) { + int space = btrfs_leaf_free_space(left); + + if (space + push_space * 2 > free_space) + break; + } + } + + if (path->slots[0] == i) + push_space += data_size; + + this_item_size = btrfs_item_size(left, i); + if (this_item_size + sizeof(struct btrfs_item) + + push_space > free_space) + break; + + push_items++; + push_space += this_item_size + sizeof(struct btrfs_item); + if (i == 0) + break; + i--; + } + + if (push_items == 0) + goto out_unlock; + + WARN_ON(!empty && push_items == left_nritems); + + /* push left to right */ + right_nritems = btrfs_header_nritems(right); + + push_space = btrfs_item_data_end(left, left_nritems - push_items); + push_space -= leaf_data_end(left); + + /* make room in the right data area */ + data_end = leaf_data_end(right); + memmove_leaf_data(right, data_end - push_space, data_end, + BTRFS_LEAF_DATA_SIZE(fs_info) - data_end); + + /* copy from the left data area */ + copy_leaf_data(right, left, BTRFS_LEAF_DATA_SIZE(fs_info) - push_space, + leaf_data_end(left), push_space); + + memmove_leaf_items(right, push_items, 0, right_nritems); + + /* copy the items from left to right */ + copy_leaf_items(right, left, 0, left_nritems - push_items, push_items); + + /* update the item pointers */ + right_nritems += push_items; + btrfs_set_header_nritems(right, right_nritems); + push_space = BTRFS_LEAF_DATA_SIZE(fs_info); + for (i = 0; i < right_nritems; i++) { + push_space -= btrfs_item_size(right, i); + btrfs_set_item_offset(right, i, push_space); + } + + left_nritems -= push_items; + btrfs_set_header_nritems(left, left_nritems); + + if (left_nritems) + btrfs_mark_buffer_dirty(trans, left); + else + btrfs_clear_buffer_dirty(trans, left); + + btrfs_mark_buffer_dirty(trans, right); + + btrfs_item_key(right, &disk_key, 0); + btrfs_set_node_key(upper, &disk_key, slot + 1); + btrfs_mark_buffer_dirty(trans, upper); + + /* then fixup the leaf pointer in the path */ + if (path->slots[0] >= left_nritems) { + path->slots[0] -= left_nritems; + btrfs_tree_unlock(left); + free_extent_buffer(left); + path->nodes[0] = right; + path->slots[1] += 1; + } else { + btrfs_tree_unlock(right); + free_extent_buffer(right); + } + return 0; + +out_unlock: + btrfs_tree_unlock(right); + free_extent_buffer(right); + return 1; +} + +/* + * push some data in the path leaf to the right, trying to free up at + * least data_size bytes. returns zero if the push worked, nonzero otherwise + * + * returns 1 if the push failed because the other node didn't have enough + * room, 0 if everything worked out and < 0 if there were major errors. + * + * this will push starting from min_slot to the end of the leaf. It won't + * push any slot lower than min_slot + */ +static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root + *root, struct btrfs_path *path, + int min_data_size, int data_size, + bool empty, u32 min_slot) +{ + struct extent_buffer *left = path->nodes[0]; + struct extent_buffer *right; + struct extent_buffer *upper; + int slot; + int free_space; + u32 left_nritems; + int ret; + + if (!path->nodes[1]) + return 1; + + slot = path->slots[1]; + upper = path->nodes[1]; + if (slot >= btrfs_header_nritems(upper) - 1) + return 1; + + btrfs_assert_tree_write_locked(path->nodes[1]); + + right = btrfs_read_node_slot(upper, slot + 1); + if (IS_ERR(right)) + return PTR_ERR(right); + + btrfs_tree_lock_nested(right, BTRFS_NESTING_RIGHT); + + free_space = btrfs_leaf_free_space(right); + if (free_space < data_size) + goto out_unlock; + + ret = btrfs_cow_block(trans, root, right, upper, + slot + 1, &right, BTRFS_NESTING_RIGHT_COW); + if (ret) + goto out_unlock; + + left_nritems = btrfs_header_nritems(left); + if (left_nritems == 0) + goto out_unlock; + + if (unlikely(check_sibling_keys(left, right))) { + ret = -EUCLEAN; + btrfs_abort_transaction(trans, ret); + btrfs_tree_unlock(right); + free_extent_buffer(right); + return ret; + } + if (path->slots[0] == left_nritems && !empty) { + /* Key greater than all keys in the leaf, right neighbor has + * enough room for it and we're not emptying our leaf to delete + * it, therefore use right neighbor to insert the new item and + * no need to touch/dirty our left leaf. */ + btrfs_tree_unlock(left); + free_extent_buffer(left); + path->nodes[0] = right; + path->slots[0] = 0; + path->slots[1]++; + return 0; + } + + return __push_leaf_right(trans, path, min_data_size, empty, right, + free_space, left_nritems, min_slot); +out_unlock: + btrfs_tree_unlock(right); + free_extent_buffer(right); + return 1; +} + +/* + * push some data in the path leaf to the left, trying to free up at + * least data_size bytes. returns zero if the push worked, nonzero otherwise + * + * max_slot can put a limit on how far into the leaf we'll push items. The + * item at 'max_slot' won't be touched. Use (u32)-1 to make us do all the + * items + */ +static noinline int __push_leaf_left(struct btrfs_trans_handle *trans, + struct btrfs_path *path, int data_size, + bool empty, struct extent_buffer *left, + int free_space, u32 right_nritems, + u32 max_slot) +{ + struct btrfs_fs_info *fs_info = left->fs_info; + struct btrfs_disk_key disk_key; + struct extent_buffer *right = path->nodes[0]; + int i; + int push_space = 0; + int push_items = 0; + u32 old_left_nritems; + u32 nr; + int ret = 0; + u32 this_item_size; + u32 old_left_item_size; + + if (empty) + nr = min(right_nritems, max_slot); + else + nr = min(right_nritems - 1, max_slot); + + for (i = 0; i < nr; i++) { + if (!empty && push_items > 0) { + if (path->slots[0] < i) + break; + if (path->slots[0] == i) { + int space = btrfs_leaf_free_space(right); + + if (space + push_space * 2 > free_space) + break; + } + } + + if (path->slots[0] == i) + push_space += data_size; + + this_item_size = btrfs_item_size(right, i); + if (this_item_size + sizeof(struct btrfs_item) + push_space > + free_space) + break; + + push_items++; + push_space += this_item_size + sizeof(struct btrfs_item); + } + + if (push_items == 0) { + ret = 1; + goto out; + } + WARN_ON(!empty && push_items == btrfs_header_nritems(right)); + + /* push data from right to left */ + copy_leaf_items(left, right, btrfs_header_nritems(left), 0, push_items); + + push_space = BTRFS_LEAF_DATA_SIZE(fs_info) - + btrfs_item_offset(right, push_items - 1); + + copy_leaf_data(left, right, leaf_data_end(left) - push_space, + btrfs_item_offset(right, push_items - 1), push_space); + old_left_nritems = btrfs_header_nritems(left); + BUG_ON(old_left_nritems <= 0); + + old_left_item_size = btrfs_item_offset(left, old_left_nritems - 1); + for (i = old_left_nritems; i < old_left_nritems + push_items; i++) { + u32 ioff; + + ioff = btrfs_item_offset(left, i); + btrfs_set_item_offset(left, i, + ioff - (BTRFS_LEAF_DATA_SIZE(fs_info) - old_left_item_size)); + } + btrfs_set_header_nritems(left, old_left_nritems + push_items); + + /* fixup right node */ + if (unlikely(push_items > right_nritems)) { + ret = -EUCLEAN; + btrfs_abort_transaction(trans, ret); + btrfs_crit(fs_info, "push items (%d) > right leaf items (%u)", + push_items, right_nritems); + goto out; + } + + if (push_items < right_nritems) { + push_space = btrfs_item_offset(right, push_items - 1) - + leaf_data_end(right); + memmove_leaf_data(right, + BTRFS_LEAF_DATA_SIZE(fs_info) - push_space, + leaf_data_end(right), push_space); + + memmove_leaf_items(right, 0, push_items, + btrfs_header_nritems(right) - push_items); + } + + right_nritems -= push_items; + btrfs_set_header_nritems(right, right_nritems); + push_space = BTRFS_LEAF_DATA_SIZE(fs_info); + for (i = 0; i < right_nritems; i++) { + push_space = push_space - btrfs_item_size(right, i); + btrfs_set_item_offset(right, i, push_space); + } + + btrfs_mark_buffer_dirty(trans, left); + if (right_nritems) + btrfs_mark_buffer_dirty(trans, right); + else + btrfs_clear_buffer_dirty(trans, right); + + btrfs_item_key(right, &disk_key, 0); + fixup_low_keys(trans, path, &disk_key, 1); + + /* then fixup the leaf pointer in the path */ + if (path->slots[0] < push_items) { + path->slots[0] += old_left_nritems; + btrfs_tree_unlock(right); + free_extent_buffer(right); + path->nodes[0] = left; + path->slots[1] -= 1; + } else { + btrfs_tree_unlock(left); + free_extent_buffer(left); + path->slots[0] -= push_items; + } + BUG_ON(path->slots[0] < 0); + return ret; +out: + btrfs_tree_unlock(left); + free_extent_buffer(left); + return ret; +} + +/* + * push some data in the path leaf to the left, trying to free up at + * least data_size bytes. returns zero if the push worked, nonzero otherwise + * + * max_slot can put a limit on how far into the leaf we'll push items. The + * item at 'max_slot' won't be touched. Use (u32)-1 to make us push all the + * items + */ +static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root + *root, struct btrfs_path *path, int min_data_size, + int data_size, int empty, u32 max_slot) +{ + struct extent_buffer *right = path->nodes[0]; + struct extent_buffer *left; + int slot; + int free_space; + u32 right_nritems; + int ret = 0; + + slot = path->slots[1]; + if (slot == 0) + return 1; + if (!path->nodes[1]) + return 1; + + right_nritems = btrfs_header_nritems(right); + if (right_nritems == 0) + return 1; + + btrfs_assert_tree_write_locked(path->nodes[1]); + + left = btrfs_read_node_slot(path->nodes[1], slot - 1); + if (IS_ERR(left)) + return PTR_ERR(left); + + btrfs_tree_lock_nested(left, BTRFS_NESTING_LEFT); + + free_space = btrfs_leaf_free_space(left); + if (free_space < data_size) { + ret = 1; + goto out; + } + + ret = btrfs_cow_block(trans, root, left, + path->nodes[1], slot - 1, &left, + BTRFS_NESTING_LEFT_COW); + if (ret) { + /* we hit -ENOSPC, but it isn't fatal here */ + if (ret == -ENOSPC) + ret = 1; + goto out; + } + + if (unlikely(check_sibling_keys(left, right))) { + ret = -EUCLEAN; + btrfs_abort_transaction(trans, ret); + goto out; + } + return __push_leaf_left(trans, path, min_data_size, empty, left, + free_space, right_nritems, max_slot); +out: + btrfs_tree_unlock(left); + free_extent_buffer(left); + return ret; +} + +/* + * split the path's leaf in two, making sure there is at least data_size + * available for the resulting leaf level of the path. + */ +static noinline int copy_for_split(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + struct extent_buffer *l, + struct extent_buffer *right, + int slot, int mid, int nritems) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + int data_copy_size; + int rt_data_off; + int i; + int ret; + struct btrfs_disk_key disk_key; + + nritems = nritems - mid; + btrfs_set_header_nritems(right, nritems); + data_copy_size = btrfs_item_data_end(l, mid) - leaf_data_end(l); + + copy_leaf_items(right, l, 0, mid, nritems); + + copy_leaf_data(right, l, BTRFS_LEAF_DATA_SIZE(fs_info) - data_copy_size, + leaf_data_end(l), data_copy_size); + + rt_data_off = BTRFS_LEAF_DATA_SIZE(fs_info) - btrfs_item_data_end(l, mid); + + for (i = 0; i < nritems; i++) { + u32 ioff; + + ioff = btrfs_item_offset(right, i); + btrfs_set_item_offset(right, i, ioff + rt_data_off); + } + + btrfs_set_header_nritems(l, mid); + btrfs_item_key(right, &disk_key, 0); + ret = insert_ptr(trans, path, &disk_key, right->start, path->slots[1] + 1, 1); + if (ret < 0) + return ret; + + btrfs_mark_buffer_dirty(trans, right); + btrfs_mark_buffer_dirty(trans, l); + BUG_ON(path->slots[0] != slot); + + if (mid <= slot) { + btrfs_tree_unlock(path->nodes[0]); + free_extent_buffer(path->nodes[0]); + path->nodes[0] = right; + path->slots[0] -= mid; + path->slots[1] += 1; + } else { + btrfs_tree_unlock(right); + free_extent_buffer(right); + } + + BUG_ON(path->slots[0] < 0); + + return 0; +} + +/* + * double splits happen when we need to insert a big item in the middle + * of a leaf. A double split can leave us with 3 mostly empty leaves: + * leaf: [ slots 0 - N] [ our target ] [ N + 1 - total in leaf ] + * A B C + * + * We avoid this by trying to push the items on either side of our target + * into the adjacent leaves. If all goes well we can avoid the double split + * completely. + */ +static noinline int push_for_double_split(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + int data_size) +{ + int ret; + int progress = 0; + int slot; + u32 nritems; + int space_needed = data_size; + + slot = path->slots[0]; + if (slot < btrfs_header_nritems(path->nodes[0])) + space_needed -= btrfs_leaf_free_space(path->nodes[0]); + + /* + * try to push all the items after our slot into the + * right leaf + */ + ret = push_leaf_right(trans, root, path, 1, space_needed, 0, slot); + if (ret < 0) + return ret; + + if (ret == 0) + progress++; + + nritems = btrfs_header_nritems(path->nodes[0]); + /* + * our goal is to get our slot at the start or end of a leaf. If + * we've done so we're done + */ + if (path->slots[0] == 0 || path->slots[0] == nritems) + return 0; + + if (btrfs_leaf_free_space(path->nodes[0]) >= data_size) + return 0; + + /* try to push all the items before our slot into the next leaf */ + slot = path->slots[0]; + space_needed = data_size; + if (slot > 0) + space_needed -= btrfs_leaf_free_space(path->nodes[0]); + ret = push_leaf_left(trans, root, path, 1, space_needed, 0, slot); + if (ret < 0) + return ret; + + if (ret == 0) + progress++; + + if (progress) + return 0; + return 1; +} + +/* + * split the path's leaf in two, making sure there is at least data_size + * available for the resulting leaf level of the path. + * + * returns 0 if all went well and < 0 on failure. + */ +static noinline int split_leaf(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + const struct btrfs_key *ins_key, + struct btrfs_path *path, int data_size, + bool extend) +{ + struct btrfs_disk_key disk_key; + struct extent_buffer *l; + u32 nritems; + int mid; + int slot; + struct extent_buffer *right; + struct btrfs_fs_info *fs_info = root->fs_info; + int ret = 0; + int wret; + int split; + int num_doubles = 0; + bool tried_avoid_double = false; + + l = path->nodes[0]; + slot = path->slots[0]; + if (extend && data_size + btrfs_item_size(l, slot) + + sizeof(struct btrfs_item) > BTRFS_LEAF_DATA_SIZE(fs_info)) + return -EOVERFLOW; + + /* first try to make some room by pushing left and right */ + if (data_size && path->nodes[1]) { + int space_needed = data_size; + + if (slot < btrfs_header_nritems(l)) + space_needed -= btrfs_leaf_free_space(l); + + wret = push_leaf_right(trans, root, path, space_needed, + space_needed, 0, 0); + if (wret < 0) + return wret; + if (wret) { + space_needed = data_size; + if (slot > 0) + space_needed -= btrfs_leaf_free_space(l); + wret = push_leaf_left(trans, root, path, space_needed, + space_needed, 0, (u32)-1); + if (wret < 0) + return wret; + } + l = path->nodes[0]; + + /* did the pushes work? */ + if (btrfs_leaf_free_space(l) >= data_size) + return 0; + } + + if (!path->nodes[1]) { + ret = insert_new_root(trans, root, path, 1); + if (ret) + return ret; + } +again: + split = 1; + l = path->nodes[0]; + slot = path->slots[0]; + nritems = btrfs_header_nritems(l); + mid = (nritems + 1) / 2; + + if (mid <= slot) { + if (nritems == 1 || + leaf_space_used(l, mid, nritems - mid) + data_size > + BTRFS_LEAF_DATA_SIZE(fs_info)) { + if (slot >= nritems) { + split = 0; + } else { + mid = slot; + if (mid != nritems && + leaf_space_used(l, mid, nritems - mid) + + data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) { + if (data_size && !tried_avoid_double) + goto push_for_double; + split = 2; + } + } + } + } else { + if (leaf_space_used(l, 0, mid) + data_size > + BTRFS_LEAF_DATA_SIZE(fs_info)) { + if (!extend && data_size && slot == 0) { + split = 0; + } else if ((extend || !data_size) && slot == 0) { + mid = 1; + } else { + mid = slot; + if (mid != nritems && + leaf_space_used(l, mid, nritems - mid) + + data_size > BTRFS_LEAF_DATA_SIZE(fs_info)) { + if (data_size && !tried_avoid_double) + goto push_for_double; + split = 2; + } + } + } + } + + if (split == 0) + btrfs_cpu_key_to_disk(&disk_key, ins_key); + else + btrfs_item_key(l, &disk_key, mid); + + /* + * We have to about BTRFS_NESTING_NEW_ROOT here if we've done a double + * split, because we're only allowed to have MAX_LOCKDEP_SUBCLASSES + * subclasses, which is 8 at the time of this patch, and we've maxed it + * out. In the future we could add a + * BTRFS_NESTING_SPLIT_THE_SPLITTENING if we need to, but for now just + * use BTRFS_NESTING_NEW_ROOT. + */ + right = btrfs_alloc_tree_block(trans, root, 0, btrfs_root_id(root), + &disk_key, 0, l->start, 0, 0, + num_doubles ? BTRFS_NESTING_NEW_ROOT : + BTRFS_NESTING_SPLIT); + if (IS_ERR(right)) + return PTR_ERR(right); + + root_add_used_bytes(root); + + if (split == 0) { + if (mid <= slot) { + btrfs_set_header_nritems(right, 0); + ret = insert_ptr(trans, path, &disk_key, + right->start, path->slots[1] + 1, 1); + if (ret < 0) { + btrfs_tree_unlock(right); + free_extent_buffer(right); + return ret; + } + btrfs_tree_unlock(path->nodes[0]); + free_extent_buffer(path->nodes[0]); + path->nodes[0] = right; + path->slots[0] = 0; + path->slots[1] += 1; + } else { + btrfs_set_header_nritems(right, 0); + ret = insert_ptr(trans, path, &disk_key, + right->start, path->slots[1], 1); + if (ret < 0) { + btrfs_tree_unlock(right); + free_extent_buffer(right); + return ret; + } + btrfs_tree_unlock(path->nodes[0]); + free_extent_buffer(path->nodes[0]); + path->nodes[0] = right; + path->slots[0] = 0; + if (path->slots[1] == 0) + fixup_low_keys(trans, path, &disk_key, 1); + } + /* + * We create a new leaf 'right' for the required ins_len and + * we'll do btrfs_mark_buffer_dirty() on this leaf after copying + * the content of ins_len to 'right'. + */ + return ret; + } + + ret = copy_for_split(trans, path, l, right, slot, mid, nritems); + if (ret < 0) { + btrfs_tree_unlock(right); + free_extent_buffer(right); + return ret; + } + + if (split == 2) { + BUG_ON(num_doubles != 0); + num_doubles++; + goto again; + } + + return 0; + +push_for_double: + push_for_double_split(trans, root, path, data_size); + tried_avoid_double = true; + if (btrfs_leaf_free_space(path->nodes[0]) >= data_size) + return 0; + goto again; +} + +static noinline int setup_leaf_for_split(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, int ins_len) +{ + struct btrfs_key key; + struct extent_buffer *leaf; + struct btrfs_file_extent_item *fi; + u64 extent_len = 0; + u32 item_size; + int ret; + + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + + BUG_ON(key.type != BTRFS_EXTENT_DATA_KEY && + key.type != BTRFS_RAID_STRIPE_KEY && + key.type != BTRFS_EXTENT_CSUM_KEY); + + if (btrfs_leaf_free_space(leaf) >= ins_len) + return 0; + + item_size = btrfs_item_size(leaf, path->slots[0]); + if (key.type == BTRFS_EXTENT_DATA_KEY) { + fi = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_file_extent_item); + extent_len = btrfs_file_extent_num_bytes(leaf, fi); + } + btrfs_release_path(path); + + path->keep_locks = true; + path->search_for_split = true; + ret = btrfs_search_slot(trans, root, &key, path, 0, 1); + path->search_for_split = false; + if (ret > 0) + ret = -EAGAIN; + if (ret < 0) + goto err; + + ret = -EAGAIN; + leaf = path->nodes[0]; + /* if our item isn't there, return now */ + if (item_size != btrfs_item_size(leaf, path->slots[0])) + goto err; + + /* the leaf has changed, it now has room. return now */ + if (btrfs_leaf_free_space(path->nodes[0]) >= ins_len) + goto err; + + if (key.type == BTRFS_EXTENT_DATA_KEY) { + fi = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_file_extent_item); + if (extent_len != btrfs_file_extent_num_bytes(leaf, fi)) + goto err; + } + + ret = split_leaf(trans, root, &key, path, ins_len, true); + if (ret) + goto err; + + path->keep_locks = false; + btrfs_unlock_up_safe(path, 1); + return 0; +err: + path->keep_locks = false; + return ret; +} + +static noinline int split_item(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + const struct btrfs_key *new_key, + unsigned long split_offset) +{ + struct extent_buffer *leaf; + int orig_slot, slot; + char *buf; + u32 nritems; + u32 item_size; + u32 orig_offset; + struct btrfs_disk_key disk_key; + + leaf = path->nodes[0]; + /* + * Shouldn't happen because the caller must have previously called + * setup_leaf_for_split() to make room for the new item in the leaf. + */ + if (WARN_ON(btrfs_leaf_free_space(leaf) < sizeof(struct btrfs_item))) + return -ENOSPC; + + orig_slot = path->slots[0]; + orig_offset = btrfs_item_offset(leaf, path->slots[0]); + item_size = btrfs_item_size(leaf, path->slots[0]); + + buf = kmalloc(item_size, GFP_NOFS); + if (!buf) + return -ENOMEM; + + read_extent_buffer(leaf, buf, btrfs_item_ptr_offset(leaf, + path->slots[0]), item_size); + + slot = path->slots[0] + 1; + nritems = btrfs_header_nritems(leaf); + if (slot != nritems) { + /* shift the items */ + memmove_leaf_items(leaf, slot + 1, slot, nritems - slot); + } + + btrfs_cpu_key_to_disk(&disk_key, new_key); + btrfs_set_item_key(leaf, &disk_key, slot); + + btrfs_set_item_offset(leaf, slot, orig_offset); + btrfs_set_item_size(leaf, slot, item_size - split_offset); + + btrfs_set_item_offset(leaf, orig_slot, + orig_offset + item_size - split_offset); + btrfs_set_item_size(leaf, orig_slot, split_offset); + + btrfs_set_header_nritems(leaf, nritems + 1); + + /* write the data for the start of the original item */ + write_extent_buffer(leaf, buf, + btrfs_item_ptr_offset(leaf, path->slots[0]), + split_offset); + + /* write the data for the new item */ + write_extent_buffer(leaf, buf + split_offset, + btrfs_item_ptr_offset(leaf, slot), + item_size - split_offset); + btrfs_mark_buffer_dirty(trans, leaf); + + BUG_ON(btrfs_leaf_free_space(leaf) < 0); + kfree(buf); + return 0; +} + +/* + * This function splits a single item into two items, + * giving 'new_key' to the new item and splitting the + * old one at split_offset (from the start of the item). + * + * The path may be released by this operation. After + * the split, the path is pointing to the old item. The + * new item is going to be in the same node as the old one. + * + * Note, the item being split must be smaller enough to live alone on + * a tree block with room for one extra struct btrfs_item + * + * This allows us to split the item in place, keeping a lock on the + * leaf the entire time. + */ +int btrfs_split_item(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + const struct btrfs_key *new_key, + unsigned long split_offset) +{ + int ret; + ret = setup_leaf_for_split(trans, root, path, + sizeof(struct btrfs_item)); + if (ret) + return ret; + + return split_item(trans, path, new_key, split_offset); +} + +/* + * make the item pointed to by the path smaller. new_size indicates + * how small to make it, and from_end tells us if we just chop bytes + * off the end of the item or if we shift the item to chop bytes off + * the front. + */ +void btrfs_truncate_item(struct btrfs_trans_handle *trans, + const struct btrfs_path *path, u32 new_size, int from_end) +{ + int slot; + struct extent_buffer *leaf; + u32 nritems; + unsigned int data_end; + unsigned int old_data_start; + unsigned int old_size; + unsigned int size_diff; + int i; + + leaf = path->nodes[0]; + slot = path->slots[0]; + + old_size = btrfs_item_size(leaf, slot); + if (old_size == new_size) + return; + + nritems = btrfs_header_nritems(leaf); + data_end = leaf_data_end(leaf); + + old_data_start = btrfs_item_offset(leaf, slot); + + size_diff = old_size - new_size; + + BUG_ON(slot < 0); + BUG_ON(slot >= nritems); + + /* + * item0..itemN ... dataN.offset..dataN.size .. data0.size + */ + /* first correct the data pointers */ + for (i = slot; i < nritems; i++) { + u32 ioff; + + ioff = btrfs_item_offset(leaf, i); + btrfs_set_item_offset(leaf, i, ioff + size_diff); + } + + /* shift the data */ + if (from_end) { + memmove_leaf_data(leaf, data_end + size_diff, data_end, + old_data_start + new_size - data_end); + } else { + struct btrfs_disk_key disk_key; + u64 offset; + + btrfs_item_key(leaf, &disk_key, slot); + + if (btrfs_disk_key_type(&disk_key) == BTRFS_EXTENT_DATA_KEY) { + unsigned long ptr; + struct btrfs_file_extent_item *fi; + + fi = btrfs_item_ptr(leaf, slot, + struct btrfs_file_extent_item); + fi = (struct btrfs_file_extent_item *)( + (unsigned long)fi - size_diff); + + if (btrfs_file_extent_type(leaf, fi) == + BTRFS_FILE_EXTENT_INLINE) { + ptr = btrfs_item_ptr_offset(leaf, slot); + memmove_extent_buffer(leaf, ptr, + (unsigned long)fi, + BTRFS_FILE_EXTENT_INLINE_DATA_START); + } + } + + memmove_leaf_data(leaf, data_end + size_diff, data_end, + old_data_start - data_end); + + offset = btrfs_disk_key_offset(&disk_key); + btrfs_set_disk_key_offset(&disk_key, offset + size_diff); + btrfs_set_item_key(leaf, &disk_key, slot); + if (slot == 0) + fixup_low_keys(trans, path, &disk_key, 1); + } + + btrfs_set_item_size(leaf, slot, new_size); + btrfs_mark_buffer_dirty(trans, leaf); + + if (unlikely(btrfs_leaf_free_space(leaf) < 0)) { + btrfs_print_leaf(leaf); + BUG(); + } +} + +/* + * make the item pointed to by the path bigger, data_size is the added size. + */ +void btrfs_extend_item(struct btrfs_trans_handle *trans, + const struct btrfs_path *path, u32 data_size) +{ + int slot; + struct extent_buffer *leaf; + u32 nritems; + unsigned int data_end; + unsigned int old_data; + unsigned int old_size; + int i; + + leaf = path->nodes[0]; + + nritems = btrfs_header_nritems(leaf); + data_end = leaf_data_end(leaf); + + if (unlikely(btrfs_leaf_free_space(leaf) < data_size)) { + btrfs_print_leaf(leaf); + BUG(); + } + slot = path->slots[0]; + old_data = btrfs_item_data_end(leaf, slot); + + BUG_ON(slot < 0); + if (unlikely(slot >= nritems)) { + btrfs_print_leaf(leaf); + btrfs_crit(leaf->fs_info, "slot %d too large, nritems %d", + slot, nritems); + BUG(); + } + + /* + * item0..itemN ... dataN.offset..dataN.size .. data0.size + */ + /* first correct the data pointers */ + for (i = slot; i < nritems; i++) { + u32 ioff; + + ioff = btrfs_item_offset(leaf, i); + btrfs_set_item_offset(leaf, i, ioff - data_size); + } + + /* shift the data */ + memmove_leaf_data(leaf, data_end - data_size, data_end, + old_data - data_end); + + old_size = btrfs_item_size(leaf, slot); + btrfs_set_item_size(leaf, slot, old_size + data_size); + btrfs_mark_buffer_dirty(trans, leaf); + + if (unlikely(btrfs_leaf_free_space(leaf) < 0)) { + btrfs_print_leaf(leaf); + BUG(); + } +} + +/* + * Make space in the node before inserting one or more items. + * + * @trans: transaction handle + * @root: root we are inserting items to + * @path: points to the leaf/slot where we are going to insert new items + * @batch: information about the batch of items to insert + * + * Main purpose is to save stack depth by doing the bulk of the work in a + * function that doesn't call btrfs_search_slot + */ +static void setup_items_for_insert(struct btrfs_trans_handle *trans, + struct btrfs_root *root, struct btrfs_path *path, + const struct btrfs_item_batch *batch) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + int i; + u32 nritems; + unsigned int data_end; + struct btrfs_disk_key disk_key; + struct extent_buffer *leaf; + int slot; + u32 total_size; + + /* + * Before anything else, update keys in the parent and other ancestors + * if needed, then release the write locks on them, so that other tasks + * can use them while we modify the leaf. + */ + if (path->slots[0] == 0) { + btrfs_cpu_key_to_disk(&disk_key, &batch->keys[0]); + fixup_low_keys(trans, path, &disk_key, 1); + } + btrfs_unlock_up_safe(path, 1); + + leaf = path->nodes[0]; + slot = path->slots[0]; + + nritems = btrfs_header_nritems(leaf); + data_end = leaf_data_end(leaf); + total_size = batch->total_data_size + (batch->nr * sizeof(struct btrfs_item)); + + if (unlikely(btrfs_leaf_free_space(leaf) < total_size)) { + btrfs_print_leaf(leaf); + btrfs_crit(fs_info, "not enough freespace need %u have %d", + total_size, btrfs_leaf_free_space(leaf)); + BUG(); + } + + if (slot != nritems) { + unsigned int old_data = btrfs_item_data_end(leaf, slot); + + if (unlikely(old_data < data_end)) { + btrfs_print_leaf(leaf); + btrfs_crit(fs_info, + "item at slot %d with data offset %u beyond data end of leaf %u", + slot, old_data, data_end); + BUG(); + } + /* + * item0..itemN ... dataN.offset..dataN.size .. data0.size + */ + /* first correct the data pointers */ + for (i = slot; i < nritems; i++) { + u32 ioff; + + ioff = btrfs_item_offset(leaf, i); + btrfs_set_item_offset(leaf, i, + ioff - batch->total_data_size); + } + /* shift the items */ + memmove_leaf_items(leaf, slot + batch->nr, slot, nritems - slot); + + /* shift the data */ + memmove_leaf_data(leaf, data_end - batch->total_data_size, + data_end, old_data - data_end); + data_end = old_data; + } + + /* setup the item for the new data */ + for (i = 0; i < batch->nr; i++) { + btrfs_cpu_key_to_disk(&disk_key, &batch->keys[i]); + btrfs_set_item_key(leaf, &disk_key, slot + i); + data_end -= batch->data_sizes[i]; + btrfs_set_item_offset(leaf, slot + i, data_end); + btrfs_set_item_size(leaf, slot + i, batch->data_sizes[i]); + } + + btrfs_set_header_nritems(leaf, nritems + batch->nr); + btrfs_mark_buffer_dirty(trans, leaf); + + if (unlikely(btrfs_leaf_free_space(leaf) < 0)) { + btrfs_print_leaf(leaf); + BUG(); + } +} + +/* + * Insert a new item into a leaf. + * + * @trans: Transaction handle. + * @root: The root of the btree. + * @path: A path pointing to the target leaf and slot. + * @key: The key of the new item. + * @data_size: The size of the data associated with the new key. + */ +void btrfs_setup_item_for_insert(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + const struct btrfs_key *key, + u32 data_size) +{ + struct btrfs_item_batch batch; + + batch.keys = key; + batch.data_sizes = &data_size; + batch.total_data_size = data_size; + batch.nr = 1; + + setup_items_for_insert(trans, root, path, &batch); +} + +/* + * Given a key and some data, insert items into the tree. + * This does all the path init required, making room in the tree if needed. + * + * Returns: 0 on success + * -EEXIST if the first key already exists + * < 0 on other errors + */ +int btrfs_insert_empty_items(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + const struct btrfs_item_batch *batch) +{ + int ret = 0; + int slot; + u32 total_size; + + total_size = batch->total_data_size + (batch->nr * sizeof(struct btrfs_item)); + ret = btrfs_search_slot(trans, root, &batch->keys[0], path, total_size, 1); + if (ret == 0) + return -EEXIST; + if (ret < 0) + return ret; + + slot = path->slots[0]; + BUG_ON(slot < 0); + + setup_items_for_insert(trans, root, path, batch); + return 0; +} + +/* + * Given a key and some data, insert an item into the tree. + * This does all the path init required, making room in the tree if needed. + */ +int btrfs_insert_item(struct btrfs_trans_handle *trans, struct btrfs_root *root, + const struct btrfs_key *cpu_key, void *data, + u32 data_size) +{ + int ret = 0; + BTRFS_PATH_AUTO_FREE(path); + struct extent_buffer *leaf; + unsigned long ptr; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + ret = btrfs_insert_empty_item(trans, root, path, cpu_key, data_size); + if (!ret) { + leaf = path->nodes[0]; + ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); + write_extent_buffer(leaf, data, ptr, data_size); + btrfs_mark_buffer_dirty(trans, leaf); + } + return ret; +} + +/* + * This function duplicates an item, giving 'new_key' to the new item. + * It guarantees both items live in the same tree leaf and the new item is + * contiguous with the original item. + * + * This allows us to split a file extent in place, keeping a lock on the leaf + * the entire time. + */ +int btrfs_duplicate_item(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + const struct btrfs_key *new_key) +{ + struct extent_buffer *leaf; + int ret; + u32 item_size; + + leaf = path->nodes[0]; + item_size = btrfs_item_size(leaf, path->slots[0]); + ret = setup_leaf_for_split(trans, root, path, + item_size + sizeof(struct btrfs_item)); + if (ret) + return ret; + + path->slots[0]++; + btrfs_setup_item_for_insert(trans, root, path, new_key, item_size); + leaf = path->nodes[0]; + memcpy_extent_buffer(leaf, + btrfs_item_ptr_offset(leaf, path->slots[0]), + btrfs_item_ptr_offset(leaf, path->slots[0] - 1), + item_size); + return 0; +} + +/* + * delete the pointer from a given node. + * + * the tree should have been previously balanced so the deletion does not + * empty a node. + * + * This is exported for use inside btrfs-progs, don't un-export it. + */ +int btrfs_del_ptr(struct btrfs_trans_handle *trans, struct btrfs_root *root, + struct btrfs_path *path, int level, int slot) +{ + struct extent_buffer *parent = path->nodes[level]; + u32 nritems; + int ret; + + nritems = btrfs_header_nritems(parent); + if (slot != nritems - 1) { + if (level) { + ret = btrfs_tree_mod_log_insert_move(parent, slot, + slot + 1, nritems - slot - 1); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + } + memmove_extent_buffer(parent, + btrfs_node_key_ptr_offset(parent, slot), + btrfs_node_key_ptr_offset(parent, slot + 1), + sizeof(struct btrfs_key_ptr) * + (nritems - slot - 1)); + } else if (level) { + ret = btrfs_tree_mod_log_insert_key(parent, slot, + BTRFS_MOD_LOG_KEY_REMOVE); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + } + + nritems--; + btrfs_set_header_nritems(parent, nritems); + if (nritems == 0 && parent == root->node) { + BUG_ON(btrfs_header_level(root->node) != 1); + /* just turn the root into a leaf and break */ + btrfs_set_header_level(root->node, 0); + } else if (slot == 0) { + struct btrfs_disk_key disk_key; + + btrfs_node_key(parent, &disk_key, 0); + fixup_low_keys(trans, path, &disk_key, level + 1); + } + btrfs_mark_buffer_dirty(trans, parent); + return 0; +} + +/* + * a helper function to delete the leaf pointed to by path->slots[1] and + * path->nodes[1]. + * + * This deletes the pointer in path->nodes[1] and frees the leaf + * block extent. zero is returned if it all worked out, < 0 otherwise. + * + * The path must have already been setup for deleting the leaf, including + * all the proper balancing. path->nodes[1] must be locked. + */ +static noinline int btrfs_del_leaf(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + struct extent_buffer *leaf) +{ + int ret; + + WARN_ON(btrfs_header_generation(leaf) != trans->transid); + ret = btrfs_del_ptr(trans, root, path, 1, path->slots[1]); + if (ret < 0) + return ret; + + /* + * btrfs_free_extent is expensive, we want to make sure we + * aren't holding any locks when we call it + */ + btrfs_unlock_up_safe(path, 0); + + root_sub_used_bytes(root); + + refcount_inc(&leaf->refs); + ret = btrfs_free_tree_block(trans, btrfs_root_id(root), leaf, 0, 1); + free_extent_buffer_stale(leaf); + if (ret < 0) + btrfs_abort_transaction(trans, ret); + + return ret; +} +/* + * delete the item at the leaf level in path. If that empties + * the leaf, remove it from the tree + */ +int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root, + struct btrfs_path *path, int slot, int nr) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct extent_buffer *leaf; + int ret = 0; + int wret; + u32 nritems; + + leaf = path->nodes[0]; + nritems = btrfs_header_nritems(leaf); + + if (slot + nr != nritems) { + const u32 last_off = btrfs_item_offset(leaf, slot + nr - 1); + const int data_end = leaf_data_end(leaf); + u32 dsize = 0; + int i; + + for (i = 0; i < nr; i++) + dsize += btrfs_item_size(leaf, slot + i); + + memmove_leaf_data(leaf, data_end + dsize, data_end, + last_off - data_end); + + for (i = slot + nr; i < nritems; i++) { + u32 ioff; + + ioff = btrfs_item_offset(leaf, i); + btrfs_set_item_offset(leaf, i, ioff + dsize); + } + + memmove_leaf_items(leaf, slot, slot + nr, nritems - slot - nr); + } + btrfs_set_header_nritems(leaf, nritems - nr); + nritems -= nr; + + /* delete the leaf if we've emptied it */ + if (nritems == 0) { + if (leaf != root->node) { + btrfs_clear_buffer_dirty(trans, leaf); + ret = btrfs_del_leaf(trans, root, path, leaf); + if (ret < 0) + return ret; + } + } else { + int used = leaf_space_used(leaf, 0, nritems); + if (slot == 0) { + struct btrfs_disk_key disk_key; + + btrfs_item_key(leaf, &disk_key, 0); + fixup_low_keys(trans, path, &disk_key, 1); + } + + /* + * Try to delete the leaf if it is mostly empty. We do this by + * trying to move all its items into its left and right neighbours. + * If we can't move all the items, then we don't delete it - it's + * not ideal, but future insertions might fill the leaf with more + * items, or items from other leaves might be moved later into our + * leaf due to deletions on those leaves. + */ + if (used < BTRFS_LEAF_DATA_SIZE(fs_info) / 3) { + u32 min_push_space; + + /* push_leaf_left fixes the path. + * make sure the path still points to our leaf + * for possible call to btrfs_del_ptr below + */ + slot = path->slots[1]; + refcount_inc(&leaf->refs); + /* + * We want to be able to at least push one item to the + * left neighbour leaf, and that's the first item. + */ + min_push_space = sizeof(struct btrfs_item) + + btrfs_item_size(leaf, 0); + wret = push_leaf_left(trans, root, path, 0, + min_push_space, 1, (u32)-1); + if (wret < 0 && wret != -ENOSPC) + ret = wret; + + if (path->nodes[0] == leaf && + btrfs_header_nritems(leaf)) { + /* + * If we were not able to push all items from our + * leaf to its left neighbour, then attempt to + * either push all the remaining items to the + * right neighbour or none. There's no advantage + * in pushing only some items, instead of all, as + * it's pointless to end up with a leaf having + * too few items while the neighbours can be full + * or nearly full. + */ + nritems = btrfs_header_nritems(leaf); + min_push_space = leaf_space_used(leaf, 0, nritems); + wret = push_leaf_right(trans, root, path, 0, + min_push_space, 1, 0); + if (wret < 0 && wret != -ENOSPC) + ret = wret; + } + + if (btrfs_header_nritems(leaf) == 0) { + path->slots[1] = slot; + ret = btrfs_del_leaf(trans, root, path, leaf); + free_extent_buffer(leaf); + if (ret < 0) + return ret; + } else { + /* if we're still in the path, make sure + * we're dirty. Otherwise, one of the + * push_leaf functions must have already + * dirtied this buffer + */ + if (path->nodes[0] == leaf) + btrfs_mark_buffer_dirty(trans, leaf); + free_extent_buffer(leaf); + } + } else { + btrfs_mark_buffer_dirty(trans, leaf); + } + } + return ret; +} + +/* + * A helper function to walk down the tree starting at min_key, and looking + * for leaves that have a minimum transaction id. + * This is used by the btree defrag code, and tree logging + * + * This does not cow, but it does stuff the starting key it finds back + * into min_key, so you can call btrfs_search_slot with cow=1 on the + * key and get a writable path. + * + * min_trans indicates the oldest transaction that you are interested + * in walking through. Any nodes or leaves older than min_trans are + * skipped over (without reading them). + * + * returns zero if something useful was found, < 0 on error and 1 if there + * was nothing in the tree that matched the search criteria. + */ +int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key, + struct btrfs_path *path, + u64 min_trans) +{ + struct extent_buffer *cur; + int slot; + int sret; + u32 nritems; + int level; + int ret = 1; + const bool keep_locks = path->keep_locks; + + ASSERT(!path->nowait); + ASSERT(path->lowest_level == 0); + path->keep_locks = true; +again: + cur = btrfs_read_lock_root_node(root); + level = btrfs_header_level(cur); + WARN_ON(path->nodes[level]); + path->nodes[level] = cur; + path->locks[level] = BTRFS_READ_LOCK; + + if (btrfs_header_generation(cur) < min_trans) { + ret = 1; + goto out; + } + while (1) { + nritems = btrfs_header_nritems(cur); + level = btrfs_header_level(cur); + sret = btrfs_bin_search(cur, 0, min_key, &slot); + if (sret < 0) { + ret = sret; + goto out; + } + + /* At level 0 we're done, setup the path and exit. */ + if (level == 0) { + if (slot >= nritems) + goto find_next_key; + ret = 0; + path->slots[level] = slot; + /* Save our key for returning back. */ + btrfs_item_key_to_cpu(cur, min_key, slot); + goto out; + } + if (sret && slot > 0) + slot--; + /* + * check this node pointer against the min_trans parameters. + * If it is too old, skip to the next one. + */ + while (slot < nritems) { + u64 gen; + + gen = btrfs_node_ptr_generation(cur, slot); + if (gen < min_trans) { + slot++; + continue; + } + break; + } +find_next_key: + /* + * we didn't find a candidate key in this node, walk forward + * and find another one + */ + path->slots[level] = slot; + if (slot >= nritems) { + sret = btrfs_find_next_key(root, path, min_key, level, + min_trans); + if (sret == 0) { + btrfs_release_path(path); + goto again; + } else { + goto out; + } + } + cur = btrfs_read_node_slot(cur, slot); + if (IS_ERR(cur)) { + ret = PTR_ERR(cur); + goto out; + } + + btrfs_tree_read_lock(cur); + + path->locks[level - 1] = BTRFS_READ_LOCK; + path->nodes[level - 1] = cur; + unlock_up(path, level, 1, 0, NULL); + } +out: + path->keep_locks = keep_locks; + if (ret == 0) + btrfs_unlock_up_safe(path, 1); + return ret; +} + +/* + * this is similar to btrfs_next_leaf, but does not try to preserve + * and fixup the path. It looks for and returns the next key in the + * tree based on the current path and the min_trans parameters. + * + * 0 is returned if another key is found, < 0 if there are any errors + * and 1 is returned if there are no higher keys in the tree + * + * path->keep_locks should be set to true on the search made before + * calling this function. + */ +int btrfs_find_next_key(struct btrfs_root *root, struct btrfs_path *path, + struct btrfs_key *key, int level, u64 min_trans) +{ + int slot; + struct extent_buffer *c; + + WARN_ON(!path->keep_locks && !path->skip_locking); + while (level < BTRFS_MAX_LEVEL) { + if (!path->nodes[level]) + return 1; + + slot = path->slots[level] + 1; + c = path->nodes[level]; +next: + if (slot >= btrfs_header_nritems(c)) { + int ret; + int orig_lowest; + struct btrfs_key cur_key; + if (level + 1 >= BTRFS_MAX_LEVEL || + !path->nodes[level + 1]) + return 1; + + if (path->locks[level + 1] || path->skip_locking) { + level++; + continue; + } + + slot = btrfs_header_nritems(c) - 1; + if (level == 0) + btrfs_item_key_to_cpu(c, &cur_key, slot); + else + btrfs_node_key_to_cpu(c, &cur_key, slot); + + orig_lowest = path->lowest_level; + btrfs_release_path(path); + path->lowest_level = level; + ret = btrfs_search_slot(NULL, root, &cur_key, path, + 0, 0); + path->lowest_level = orig_lowest; + if (ret < 0) + return ret; + + c = path->nodes[level]; + slot = path->slots[level]; + if (ret == 0) + slot++; + goto next; + } + + if (level == 0) + btrfs_item_key_to_cpu(c, key, slot); + else { + u64 gen = btrfs_node_ptr_generation(c, slot); + + if (gen < min_trans) { + slot++; + goto next; + } + btrfs_node_key_to_cpu(c, key, slot); + } + return 0; + } + return 1; +} + +int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path, + u64 time_seq) +{ + int slot; + int level; + struct extent_buffer *c; + struct extent_buffer *next; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_key key; + struct btrfs_eb_prealloc pa = { .supports_nowait = true }; + bool need_commit_sem = false; + u32 nritems; + int ret; + int i; + + /* + * The nowait semantics are used only for write paths, where we don't + * use the tree mod log and sequence numbers. + */ + if (time_seq) + ASSERT(!path->nowait); + + nritems = btrfs_header_nritems(path->nodes[0]); + if (nritems == 0) + return 1; + + btrfs_item_key_to_cpu(path->nodes[0], &key, nritems - 1); +again: + if (pa.needs_prealloc) { + ret = btrfs_init_eb_prealloc(fs_info, &pa, false); + if (ret) + goto done; + } + level = 1; + next = NULL; + btrfs_release_path(path); + + path->keep_locks = true; + + if (time_seq) { + ret = btrfs_search_old_slot(root, &key, path, time_seq); + } else { + if (path->need_commit_sem) { + path->need_commit_sem = false; + need_commit_sem = true; + if (path->nowait) { + if (!down_read_trylock(&fs_info->commit_root_sem)) { + ret = -EAGAIN; + goto done; + } + } else { + down_read(&fs_info->commit_root_sem); + } + } + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + } + path->keep_locks = false; + + if (ret < 0) + goto done; + + nritems = btrfs_header_nritems(path->nodes[0]); + /* + * By releasing the path above we dropped all our locks. A balance + * could have happened and + * + * 1. added more items after the previous last item + * 2. deleted the previous last item + * + * So, check again here and advance the path if there are now more + * items available. + */ + if (nritems > 0 && path->slots[0] <= nritems - 1) { + if (ret == 0 && path->slots[0] != nritems - 1) { + path->slots[0]++; + goto done; + } else if (ret > 0) { + ret = 0; + goto done; + } + } + + while (level < BTRFS_MAX_LEVEL) { + if (!path->nodes[level]) { + ret = 1; + goto done; + } + + slot = path->slots[level] + 1; + c = path->nodes[level]; + if (slot >= btrfs_header_nritems(c)) { + level++; + if (level == BTRFS_MAX_LEVEL) { + ret = 1; + goto done; + } + continue; + } + + + /* + * Our current level is where we're going to start from, and to + * make sure lockdep doesn't complain we need to drop our locks + * and nodes from 0 to our current level. + */ + for (i = 0; i < level; i++) { + if (path->locks[level]) { + btrfs_tree_read_unlock(path->nodes[i]); + path->locks[i] = 0; + } + free_extent_buffer(path->nodes[i]); + path->nodes[i] = NULL; + } + + next = c; + ret = read_block_for_search(root, path, &pa, &next, slot, &key); + if (ret == -EAGAIN && !path->nowait) + goto again; + + if (ret < 0) { + btrfs_release_path(path); + goto done; + } + + if (!path->skip_locking) { + ret = btrfs_try_tree_read_lock(next); + if (!ret && path->nowait) { + ret = -EAGAIN; + goto done; + } + if (!ret && time_seq) { + /* + * If we don't get the lock, we may be racing + * with push_leaf_left, holding that lock while + * itself waiting for the leaf we've currently + * locked. To solve this situation, we give up + * on our lock and cycle. + */ + free_extent_buffer(next); + btrfs_release_path(path); + cond_resched(); + goto again; + } + if (!ret) + btrfs_tree_read_lock(next); + } + break; + } + path->slots[level] = slot; + while (1) { + level--; + path->nodes[level] = next; + path->slots[level] = 0; + if (!path->skip_locking) + path->locks[level] = BTRFS_READ_LOCK; + if (!level) + break; + + ret = read_block_for_search(root, path, &pa, &next, 0, &key); + if (ret == -EAGAIN && !path->nowait) + goto again; + + if (ret < 0) { + btrfs_release_path(path); + goto done; + } + + if (!path->skip_locking) { + if (path->nowait) { + if (!btrfs_try_tree_read_lock(next)) { + ret = -EAGAIN; + goto done; + } + } else { + btrfs_tree_read_lock(next); + } + } + } + ret = 0; +done: + unlock_up(path, 0, 1, 0, NULL); + if (need_commit_sem) { + int ret2; + + path->need_commit_sem = true; + ret2 = finish_need_commit_sem_search(path); + up_read(&fs_info->commit_root_sem); + if (ret2) + ret = ret2; + } + + btrfs_free_eb_prealloc(&pa); + + return ret; +} + +int btrfs_next_old_item(struct btrfs_root *root, struct btrfs_path *path, u64 time_seq) +{ + path->slots[0]++; + if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) + return btrfs_next_old_leaf(root, path, time_seq); + return 0; +} + +/* + * this uses btrfs_prev_leaf to walk backwards in the tree, and keeps + * searching until it gets past min_objectid or finds an item of 'type' + * + * returns 0 if something is found, 1 if nothing was found and < 0 on error + */ +int btrfs_previous_item(struct btrfs_root *root, + struct btrfs_path *path, u64 min_objectid, + int type) +{ + struct btrfs_key found_key; + struct extent_buffer *leaf; + u32 nritems; + int ret; + + while (1) { + if (path->slots[0] == 0) { + ret = btrfs_prev_leaf(root, path); + if (ret != 0) + return ret; + } else { + path->slots[0]--; + } + leaf = path->nodes[0]; + nritems = btrfs_header_nritems(leaf); + if (nritems == 0) + return 1; + if (path->slots[0] == nritems) + path->slots[0]--; + + btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); + if (found_key.objectid < min_objectid) + break; + if (found_key.type == type) + return 0; + if (found_key.objectid == min_objectid && + found_key.type < type) + break; + } + return 1; +} + +/* + * search in extent tree to find a previous Metadata/Data extent item with + * min objecitd. + * + * returns 0 if something is found, 1 if nothing was found and < 0 on error + */ +int btrfs_previous_extent_item(struct btrfs_root *root, + struct btrfs_path *path, u64 min_objectid) +{ + struct btrfs_key found_key; + struct extent_buffer *leaf; + u32 nritems; + int ret; + + while (1) { + if (path->slots[0] == 0) { + ret = btrfs_prev_leaf(root, path); + if (ret != 0) + return ret; + } else { + path->slots[0]--; + } + leaf = path->nodes[0]; + nritems = btrfs_header_nritems(leaf); + if (nritems == 0) + return 1; + if (path->slots[0] == nritems) + path->slots[0]--; + + btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); + if (found_key.objectid < min_objectid) + break; + if (found_key.type == BTRFS_EXTENT_ITEM_KEY || + found_key.type == BTRFS_METADATA_ITEM_KEY) + return 0; + if (found_key.objectid == min_objectid && + found_key.type < BTRFS_EXTENT_ITEM_KEY) + break; + } + return 1; +} + +int __init btrfs_ctree_init(void) +{ + btrfs_path_cachep = KMEM_CACHE(btrfs_path, 0); + if (!btrfs_path_cachep) + return -ENOMEM; + return 0; +} + +void __cold btrfs_ctree_exit(void) +{ + kmem_cache_destroy(btrfs_path_cachep); +} diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h new file mode 100644 index 000000000..22ba2b450 --- /dev/null +++ b/fs/btrfs/ctree.h @@ -0,0 +1,746 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#ifndef BTRFS_CTREE_H +#define BTRFS_CTREE_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "locking.h" +#include "accessors.h" + +struct extent_buffer; +struct btrfs_block_rsv; +struct btrfs_trans_handle; +struct btrfs_block_group; + +/* Read ahead values for struct btrfs_path.reada */ +enum { + READA_NONE, + READA_BACK, + READA_FORWARD, + /* + * Similar to READA_FORWARD but unlike it: + * + * 1) It will trigger readahead even for leaves that are not close to + * each other on disk; + * 2) It also triggers readahead for nodes; + * 3) During a search, even when a node or leaf is already in memory, it + * will still trigger readahead for other nodes and leaves that follow + * it. + * + * This is meant to be used only when we know we are iterating over the + * entire tree or a very large part of it. + */ + READA_FORWARD_ALWAYS, +}; + +/* + * btrfs_paths remember the path taken from the root down to the leaf. + * level 0 is always the leaf, and nodes[1...BTRFS_MAX_LEVEL] will point + * to any other levels that are present. + * + * The slots array records the index of the item or block pointer + * used while walking the tree. + */ +struct btrfs_path { + struct extent_buffer *nodes[BTRFS_MAX_LEVEL]; + int slots[BTRFS_MAX_LEVEL]; + /* if there is real range locking, this locks field will change */ + u8 locks[BTRFS_MAX_LEVEL]; + u8 reada; + u8 lowest_level; + + /* + * set by btrfs_split_item, tells search_slot to keep all locks + * and to force calls to keep space in the nodes + */ + bool search_for_split:1; + /* Keep some upper locks as we walk down. */ + bool keep_locks:1; + bool skip_locking:1; + bool search_commit_root:1; + bool need_commit_sem:1; + bool skip_release_on_error:1; + /* + * Indicate that new item (btrfs_search_slot) is extending already + * existing item and ins_len contains only the data size and not item + * header (ie. sizeof(struct btrfs_item) is not included). + */ + bool search_for_extension:1; + /* Stop search if any locks need to be taken (for read) */ + bool nowait:1; +}; + +#define BTRFS_PATH_AUTO_FREE(path_name) \ + struct btrfs_path *path_name __free(btrfs_free_path) = NULL + +/* + * This defines an on-stack path that will be auto released when exiting the scope. + * + * It is compatible with any existing manual btrfs_release_path() calls. + */ +#define BTRFS_PATH_AUTO_RELEASE(path_name) \ + struct btrfs_path path_name __free(btrfs_release_path) = { 0 } + +/* + * The state of btrfs root + */ +enum { + /* + * btrfs_record_root_in_trans is a multi-step process, and it can race + * with the balancing code. But the race is very small, and only the + * first time the root is added to each transaction. So IN_TRANS_SETUP + * is used to tell us when more checks are required + */ + BTRFS_ROOT_IN_TRANS_SETUP, + + /* + * Set if tree blocks of this root can be shared by other roots. + * Only subvolume trees and their reloc trees have this bit set. + * Conflicts with TRACK_DIRTY bit. + * + * This affects two things: + * + * - How balance works + * For shareable roots, we need to use reloc tree and do path + * replacement for balance, and need various pre/post hooks for + * snapshot creation to handle them. + * + * While for non-shareable trees, we just simply do a tree search + * with COW. + * + * - How dirty roots are tracked + * For shareable roots, btrfs_record_root_in_trans() is needed to + * track them, while non-subvolume roots have TRACK_DIRTY bit, they + * don't need to set this manually. + */ + BTRFS_ROOT_SHAREABLE, + BTRFS_ROOT_TRACK_DIRTY, + BTRFS_ROOT_IN_RADIX, + BTRFS_ROOT_ORPHAN_ITEM_INSERTED, + BTRFS_ROOT_DEFRAG_RUNNING, + BTRFS_ROOT_FORCE_COW, + BTRFS_ROOT_DIRTY, + BTRFS_ROOT_DELETING, + + /* + * Reloc tree is orphan, only kept here for qgroup delayed subtree scan + * + * Set for the subvolume tree owning the reloc tree. + */ + BTRFS_ROOT_DEAD_RELOC_TREE, + /* Mark dead root stored on device whose cleanup needs to be resumed */ + BTRFS_ROOT_DEAD_TREE, + /* The root has a log tree. Used for subvolume roots and the tree root. */ + BTRFS_ROOT_HAS_LOG_TREE, + /* Qgroup flushing is in progress */ + BTRFS_ROOT_QGROUP_FLUSHING, + /* We started the orphan cleanup for this root. */ + BTRFS_ROOT_ORPHAN_CLEANUP, + /* This root has a drop operation that was started previously. */ + BTRFS_ROOT_UNFINISHED_DROP, + /* This reloc root needs to have its buffers lockdep class reset. */ + BTRFS_ROOT_RESET_LOCKDEP_CLASS, +}; + +/* + * Record swapped tree blocks of a subvolume tree for delayed subtree trace + * code. For detail check comment in fs/btrfs/qgroup.c. + */ +struct btrfs_qgroup_swapped_blocks { + spinlock_t lock; + /* RM_EMPTY_ROOT() of above blocks[] */ + bool swapped; + struct rb_root blocks[BTRFS_MAX_LEVEL]; +}; + +/* + * in ram representation of the tree. extent_root is used for all allocations + * and for the extent tree extent_root root. + */ +struct btrfs_root { + struct rb_node rb_node; + + struct extent_buffer *node; + + struct extent_buffer *commit_root; + struct btrfs_root *log_root; + struct btrfs_root *reloc_root; + + unsigned long state; + struct btrfs_root_item root_item; + struct btrfs_key root_key; + struct btrfs_fs_info *fs_info; + struct extent_io_tree dirty_log_pages; + + struct mutex objectid_mutex; + + spinlock_t accounting_lock; + struct btrfs_block_rsv *block_rsv; + + struct mutex log_mutex; + wait_queue_head_t log_writer_wait; + wait_queue_head_t log_commit_wait[2]; + struct list_head log_ctxs[2]; + /* Used only for log trees of subvolumes, not for the log root tree */ + atomic_t log_writers; + bool log_commit[2]; + /* + * Protected by the 'log_mutex' lock but can be read without holding + * that lock to avoid unnecessary lock contention, in which case it + * should be read using btrfs_get_root_log_transid() except if it's a + * log tree in which case it can be directly accessed. Updates to this + * field should always use btrfs_set_root_log_transid(), except for log + * trees where the field can be updated directly. + */ + int log_transid; + /* No matter the commit succeeds or not*/ + int log_transid_committed; + /* + * Just be updated when the commit succeeds. Use + * btrfs_get_root_last_log_commit() and btrfs_set_root_last_log_commit() + * to access this field. + */ + int last_log_commit; + + u64 last_trans; + + u64 free_objectid; + + struct btrfs_key defrag_progress; + struct btrfs_key defrag_max; + + /* The dirty list is only used by non-shareable roots */ + struct list_head dirty_list; + + struct list_head root_list; + + /* Xarray that keeps track of in-memory inodes. */ + struct xarray inodes; + + /* Xarray that keeps track of delayed nodes of every inode. */ + struct xarray delayed_nodes; + /* + * right now this just gets used so that a root has its own devid + * for stat. It may be used for more later + */ + dev_t anon_dev; + + spinlock_t root_item_lock; + refcount_t refs; + + struct mutex delalloc_mutex; + spinlock_t delalloc_lock; + /* + * all of the inodes that have delalloc bytes. It is possible for + * this list to be empty even when there is still dirty data=ordered + * extents waiting to finish IO. + */ + struct list_head delalloc_inodes; + struct list_head delalloc_root; + u64 nr_delalloc_inodes; + + struct mutex ordered_extent_mutex; + /* + * this is used by the balancing code to wait for all the pending + * ordered extents + */ + spinlock_t ordered_extent_lock; + + /* + * all of the data=ordered extents pending writeback + * these can span multiple transactions and basically include + * every dirty data page that isn't from nodatacow + */ + struct list_head ordered_extents; + struct list_head ordered_root; + u64 nr_ordered_extents; + + /* + * Not empty if this subvolume root has gone through tree block swap + * (relocation) + * + * Will be used by reloc_control::dirty_subvol_roots. + */ + struct list_head reloc_dirty_list; + + /* + * Number of currently running SEND ioctls to prevent + * manipulation with the read-only status via SUBVOL_SETFLAGS + */ + int send_in_progress; + /* + * Number of currently running deduplication operations that have a + * destination inode belonging to this root. Protected by the lock + * root_item_lock. + */ + int dedupe_in_progress; + /* For exclusion of snapshot creation and nocow writes */ + struct btrfs_drew_lock snapshot_lock; + + atomic_t snapshot_force_cow; + + /* For qgroup metadata reserved space */ + spinlock_t qgroup_meta_rsv_lock; + u64 qgroup_meta_rsv_pertrans; + u64 qgroup_meta_rsv_prealloc; + wait_queue_head_t qgroup_flush_wait; + + /* Number of active swapfiles */ + atomic_t nr_swapfiles; + + /* Record pairs of swapped blocks for qgroup */ + struct btrfs_qgroup_swapped_blocks swapped_blocks; + + /* Used only by log trees, when logging csum items */ + struct extent_io_tree log_csum_range; + + /* Used in simple quotas, track root during relocation. */ + u64 relocation_src_root; + +#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS + u64 alloc_bytenr; +#endif + +#ifdef CONFIG_BTRFS_DEBUG + struct list_head leak_list; +#endif +}; + +static inline bool btrfs_root_readonly(const struct btrfs_root *root) +{ + /* Byte-swap the constant at compile time, root_item::flags is LE */ + return (root->root_item.flags & cpu_to_le64(BTRFS_ROOT_SUBVOL_RDONLY)) != 0; +} + +static inline bool btrfs_root_dead(const struct btrfs_root *root) +{ + /* Byte-swap the constant at compile time, root_item::flags is LE */ + return (root->root_item.flags & cpu_to_le64(BTRFS_ROOT_SUBVOL_DEAD)) != 0; +} + +static inline u64 btrfs_root_id(const struct btrfs_root *root) +{ + return root->root_key.objectid; +} + +static inline int btrfs_get_root_log_transid(const struct btrfs_root *root) +{ + return READ_ONCE(root->log_transid); +} + +static inline void btrfs_set_root_log_transid(struct btrfs_root *root, int log_transid) +{ + WRITE_ONCE(root->log_transid, log_transid); +} + +static inline int btrfs_get_root_last_log_commit(const struct btrfs_root *root) +{ + return READ_ONCE(root->last_log_commit); +} + +static inline void btrfs_set_root_last_log_commit(struct btrfs_root *root, int commit_id) +{ + WRITE_ONCE(root->last_log_commit, commit_id); +} + +static inline u64 btrfs_get_root_last_trans(const struct btrfs_root *root) +{ + return READ_ONCE(root->last_trans); +} + +static inline void btrfs_set_root_last_trans(struct btrfs_root *root, u64 transid) +{ + WRITE_ONCE(root->last_trans, transid); +} + +/* + * Return the generation this root started with. + * + * Every normal root that is created with root->root_key.offset set to it's + * originating generation. If it is a snapshot it is the generation when the + * snapshot was created. + * + * However for TREE_RELOC roots root_key.offset is the objectid of the owning + * tree root. Thankfully we copy the root item of the owning tree root, which + * has it's last_snapshot set to what we would have root_key.offset set to, so + * return that if this is a TREE_RELOC root. + */ +static inline u64 btrfs_root_origin_generation(const struct btrfs_root *root) +{ + if (btrfs_root_id(root) == BTRFS_TREE_RELOC_OBJECTID) + return btrfs_root_last_snapshot(&root->root_item); + return root->root_key.offset; +} + +/* + * Structure that conveys information about an extent that is going to replace + * all the extents in a file range. + */ +struct btrfs_replace_extent_info { + u64 disk_offset; + u64 disk_len; + u64 data_offset; + u64 data_len; + u64 file_offset; + /* Pointer to a file extent item of type regular or prealloc. */ + char *extent_buf; + /* + * Set to true when attempting to replace a file range with a new extent + * described by this structure, set to false when attempting to clone an + * existing extent into a file range. + */ + bool is_new_extent; + /* Indicate if we should update the inode's mtime and ctime. */ + bool update_times; + /* Meaningful only if is_new_extent is true. */ + int qgroup_reserved; + /* + * Meaningful only if is_new_extent is true. + * Used to track how many extent items we have already inserted in a + * subvolume tree that refer to the extent described by this structure, + * so that we know when to create a new delayed ref or update an existing + * one. + */ + int insertions; +}; + +/* Arguments for btrfs_drop_extents() */ +struct btrfs_drop_extents_args { + /* Input parameters */ + + /* + * If NULL, btrfs_drop_extents() will allocate and free its own path. + * If 'replace_extent' is true, this must not be NULL. Also the path + * is always released except if 'replace_extent' is true and + * btrfs_drop_extents() sets 'extent_inserted' to true, in which case + * the path is kept locked. + */ + struct btrfs_path *path; + /* Start offset of the range to drop extents from */ + u64 start; + /* End (exclusive, last byte + 1) of the range to drop extents from */ + u64 end; + /* If true drop all the extent maps in the range */ + bool drop_cache; + /* + * If true it means we want to insert a new extent after dropping all + * the extents in the range. If this is true, the 'extent_item_size' + * parameter must be set as well and the 'extent_inserted' field will + * be set to true by btrfs_drop_extents() if it could insert the new + * extent. + * Note: when this is set to true the path must not be NULL. + */ + bool replace_extent; + /* + * Used if 'replace_extent' is true. Size of the file extent item to + * insert after dropping all existing extents in the range + */ + u32 extent_item_size; + + /* Output parameters */ + + /* + * Set to the minimum between the input parameter 'end' and the end + * (exclusive, last byte + 1) of the last dropped extent. This is always + * set even if btrfs_drop_extents() returns an error. + */ + u64 drop_end; + /* + * The number of allocated bytes found in the range. This can be smaller + * than the range's length when there are holes in the range. + */ + u64 bytes_found; + /* + * Only set if 'replace_extent' is true. Set to true if we were able + * to insert a replacement extent after dropping all extents in the + * range, otherwise set to false by btrfs_drop_extents(). + * Also, if btrfs_drop_extents() has set this to true it means it + * returned with the path locked, otherwise if it has set this to + * false it has returned with the path released. + */ + bool extent_inserted; +}; + +struct btrfs_file_private { + void *filldir_buf; + u64 last_index; + struct extent_state *llseek_cached_state; + /* Task that allocated this structure. */ + struct task_struct *owner_task; +}; + +static inline u32 BTRFS_LEAF_DATA_SIZE(const struct btrfs_fs_info *info) +{ + return info->nodesize - sizeof(struct btrfs_header); +} + +static inline u32 BTRFS_MAX_ITEM_SIZE(const struct btrfs_fs_info *info) +{ + return BTRFS_LEAF_DATA_SIZE(info) - sizeof(struct btrfs_item); +} + +static inline u32 BTRFS_NODEPTRS_PER_BLOCK(const struct btrfs_fs_info *info) +{ + return BTRFS_LEAF_DATA_SIZE(info) / sizeof(struct btrfs_key_ptr); +} + +static inline u32 BTRFS_MAX_XATTR_SIZE(const struct btrfs_fs_info *info) +{ + return BTRFS_MAX_ITEM_SIZE(info) - sizeof(struct btrfs_dir_item); +} + +int __init btrfs_ctree_init(void); +void __cold btrfs_ctree_exit(void); + +int btrfs_bin_search(const struct extent_buffer *eb, int first_slot, + const struct btrfs_key *key, int *slot); + +int __pure btrfs_comp_cpu_keys(const struct btrfs_key *k1, const struct btrfs_key *k2); + +#ifdef __LITTLE_ENDIAN + +/* + * Compare two keys, on little-endian the disk order is same as CPU order and + * we can avoid the conversion. + */ +static inline int btrfs_comp_keys(const struct btrfs_disk_key *disk_key, + const struct btrfs_key *k2) +{ + const struct btrfs_key *k1 = (const struct btrfs_key *)disk_key; + + return btrfs_comp_cpu_keys(k1, k2); +} + +#else + +/* Compare two keys in a memcmp fashion. */ +static inline int btrfs_comp_keys(const struct btrfs_disk_key *disk, + const struct btrfs_key *k2) +{ + struct btrfs_key k1; + + btrfs_disk_key_to_cpu(&k1, disk); + + return btrfs_comp_cpu_keys(&k1, k2); +} + +#endif + +int btrfs_previous_item(struct btrfs_root *root, + struct btrfs_path *path, u64 min_objectid, + int type); +int btrfs_previous_extent_item(struct btrfs_root *root, + struct btrfs_path *path, u64 min_objectid); +void btrfs_set_item_key_safe(struct btrfs_trans_handle *trans, + const struct btrfs_path *path, + const struct btrfs_key *new_key); +struct extent_buffer *btrfs_root_node(struct btrfs_root *root); +int btrfs_find_next_key(struct btrfs_root *root, struct btrfs_path *path, + struct btrfs_key *key, int lowest_level, + u64 min_trans); +int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key, + struct btrfs_path *path, + u64 min_trans); +struct extent_buffer *btrfs_read_node_slot(struct extent_buffer *parent, + int slot); + +int btrfs_cow_block(struct btrfs_trans_handle *trans, + struct btrfs_root *root, struct extent_buffer *buf, + struct extent_buffer *parent, int parent_slot, + struct extent_buffer **cow_ret, + enum btrfs_lock_nesting nest); +int btrfs_force_cow_block(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct extent_buffer *buf, + struct extent_buffer *parent, int parent_slot, + struct extent_buffer **cow_ret, + u64 search_start, u64 empty_size, + enum btrfs_lock_nesting nest); +int btrfs_copy_root(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct extent_buffer *buf, + struct extent_buffer **cow_ret, u64 new_root_objectid); +bool btrfs_block_can_be_shared(const struct btrfs_trans_handle *trans, + const struct btrfs_root *root, + const struct extent_buffer *buf); +int btrfs_del_ptr(struct btrfs_trans_handle *trans, struct btrfs_root *root, + struct btrfs_path *path, int level, int slot); +void btrfs_extend_item(struct btrfs_trans_handle *trans, + const struct btrfs_path *path, u32 data_size); +void btrfs_truncate_item(struct btrfs_trans_handle *trans, + const struct btrfs_path *path, u32 new_size, int from_end); +int btrfs_split_item(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + const struct btrfs_key *new_key, + unsigned long split_offset); +int btrfs_duplicate_item(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + const struct btrfs_key *new_key); +int btrfs_find_item(struct btrfs_root *fs_root, struct btrfs_path *path, + u64 inum, u64 ioff, u8 key_type, struct btrfs_key *found_key); +int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root, + const struct btrfs_key *key, struct btrfs_path *p, + int ins_len, int cow); +int btrfs_search_old_slot(struct btrfs_root *root, const struct btrfs_key *key, + struct btrfs_path *p, u64 time_seq); +int btrfs_search_slot_for_read(struct btrfs_root *root, + const struct btrfs_key *key, + struct btrfs_path *p, int find_higher, + int return_any); +void btrfs_release_path(struct btrfs_path *p); +struct btrfs_path *btrfs_alloc_path(void); +void btrfs_free_path(struct btrfs_path *p); +DEFINE_FREE(btrfs_free_path, struct btrfs_path *, btrfs_free_path(_T)) +DEFINE_FREE(btrfs_release_path, struct btrfs_path, btrfs_release_path(&_T)) + +int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root, + struct btrfs_path *path, int slot, int nr); +static inline int btrfs_del_item(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path) +{ + return btrfs_del_items(trans, root, path, path->slots[0], 1); +} + +/* + * Describes a batch of items to insert in a btree. This is used by + * btrfs_insert_empty_items(). + */ +struct btrfs_item_batch { + /* + * Pointer to an array containing the keys of the items to insert (in + * sorted order). + */ + const struct btrfs_key *keys; + /* Pointer to an array containing the data size for each item to insert. */ + const u32 *data_sizes; + /* + * The sum of data sizes for all items. The caller can compute this while + * setting up the data_sizes array, so it ends up being more efficient + * than having btrfs_insert_empty_items() or setup_item_for_insert() + * doing it, as it would avoid an extra loop over a potentially large + * array, and in the case of setup_item_for_insert(), we would be doing + * it while holding a write lock on a leaf and often on upper level nodes + * too, unnecessarily increasing the size of a critical section. + */ + u32 total_data_size; + /* Size of the keys and data_sizes arrays (number of items in the batch). */ + int nr; +}; + +void btrfs_setup_item_for_insert(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + const struct btrfs_key *key, + u32 data_size); +int btrfs_insert_item(struct btrfs_trans_handle *trans, struct btrfs_root *root, + const struct btrfs_key *key, void *data, u32 data_size); +int btrfs_insert_empty_items(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + const struct btrfs_item_batch *batch); + +static inline int btrfs_insert_empty_item(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + const struct btrfs_key *key, + u32 data_size) +{ + struct btrfs_item_batch batch; + + batch.keys = key; + batch.data_sizes = &data_size; + batch.total_data_size = data_size; + batch.nr = 1; + + return btrfs_insert_empty_items(trans, root, path, &batch); +} + +int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path, + u64 time_seq); + +int btrfs_search_backwards(struct btrfs_root *root, struct btrfs_key *key, + struct btrfs_path *path); + +int btrfs_get_next_valid_item(struct btrfs_root *root, struct btrfs_key *key, + struct btrfs_path *path); + +/* + * Search in @root for a given @key, and store the slot found in @found_key. + * + * @root: The root node of the tree. + * @key: The key we are looking for. + * @found_key: Will hold the found item. + * @path: Holds the current slot/leaf. + * @iter_ret: Contains the value returned from btrfs_search_slot or + * btrfs_get_next_valid_item, whichever was executed last. + * + * The @iter_ret is an output variable that will contain the return value of + * btrfs_search_slot, if it encountered an error, or the value returned from + * btrfs_get_next_valid_item otherwise. That return value can be 0, if a valid + * slot was found, 1 if there were no more leaves, and <0 if there was an error. + * + * It's recommended to use a separate variable for iter_ret and then use it to + * set the function return value so there's no confusion of the 0/1/errno + * values stemming from btrfs_search_slot. + */ +#define btrfs_for_each_slot(root, key, found_key, path, iter_ret) \ + for (iter_ret = btrfs_search_slot(NULL, (root), (key), (path), 0, 0); \ + (iter_ret) >= 0 && \ + (iter_ret = btrfs_get_next_valid_item((root), (found_key), (path))) == 0; \ + (path)->slots[0]++ \ + ) + +int btrfs_next_old_item(struct btrfs_root *root, struct btrfs_path *path, u64 time_seq); + +/* + * Search the tree again to find a leaf with greater keys. + * + * Returns 0 if it found something or 1 if there are no greater leaves. + * Returns < 0 on error. + */ +static inline int btrfs_next_leaf(struct btrfs_root *root, struct btrfs_path *path) +{ + return btrfs_next_old_leaf(root, path, 0); +} + +static inline int btrfs_next_item(struct btrfs_root *root, struct btrfs_path *p) +{ + return btrfs_next_old_item(root, p, 0); +} +int btrfs_leaf_free_space(const struct extent_buffer *leaf); + +static inline bool btrfs_is_fstree(u64 rootid) +{ + if (rootid == BTRFS_FS_TREE_OBJECTID) + return true; + + if ((s64)rootid < (s64)BTRFS_FIRST_FREE_OBJECTID) + return false; + + if (btrfs_qgroup_level(rootid) != 0) + return false; + + return true; +} + +static inline bool btrfs_is_data_reloc_root(const struct btrfs_root *root) +{ + return root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID; +} + +#endif diff --git a/fs/btrfs/defrag.c b/fs/btrfs/defrag.c new file mode 100644 index 000000000..6ec5dd760 --- /dev/null +++ b/fs/btrfs/defrag.c @@ -0,0 +1,1506 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#include +#include "ctree.h" +#include "disk-io.h" +#include "transaction.h" +#include "locking.h" +#include "accessors.h" +#include "messages.h" +#include "delalloc-space.h" +#include "subpage.h" +#include "defrag.h" +#include "file-item.h" +#include "super.h" +#include "compression.h" + +static struct kmem_cache *btrfs_inode_defrag_cachep; + +/* + * When auto defrag is enabled we queue up these defrag structs to remember + * which inodes need defragging passes. + */ +struct inode_defrag { + struct rb_node rb_node; + /* Inode number */ + u64 ino; + /* + * Transid where the defrag was added, we search for extents newer than + * this. + */ + u64 transid; + + /* Root objectid */ + u64 root; + + /* + * The extent size threshold for autodefrag. + * + * This value is different for compressed/non-compressed extents, thus + * needs to be passed from higher layer. + * (aka, inode_should_defrag()) + */ + u32 extent_thresh; +}; + +static int compare_inode_defrag(const struct inode_defrag *defrag1, + const struct inode_defrag *defrag2) +{ + if (defrag1->root > defrag2->root) + return 1; + else if (defrag1->root < defrag2->root) + return -1; + else if (defrag1->ino > defrag2->ino) + return 1; + else if (defrag1->ino < defrag2->ino) + return -1; + else + return 0; +} + +static int inode_defrag_cmp(struct rb_node *new, const struct rb_node *existing) +{ + const struct inode_defrag *new_defrag = rb_entry(new, struct inode_defrag, rb_node); + const struct inode_defrag *existing_defrag = rb_entry(existing, struct inode_defrag, rb_node); + + return compare_inode_defrag(new_defrag, existing_defrag); +} + +/* + * Insert a record for an inode into the defrag tree. The lock must be held + * already. + * + * If you're inserting a record for an older transid than an existing record, + * the transid already in the tree is lowered. + */ +static int btrfs_insert_inode_defrag(struct btrfs_inode *inode, + struct inode_defrag *defrag) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct rb_node *node; + + node = rb_find_add(&defrag->rb_node, &fs_info->defrag_inodes, inode_defrag_cmp); + if (node) { + struct inode_defrag *entry; + + entry = rb_entry(node, struct inode_defrag, rb_node); + /* + * If we're reinserting an entry for an old defrag run, make + * sure to lower the transid of our existing record. + */ + if (defrag->transid < entry->transid) + entry->transid = defrag->transid; + entry->extent_thresh = min(defrag->extent_thresh, entry->extent_thresh); + return -EEXIST; + } + set_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags); + return 0; +} + +static inline bool need_auto_defrag(struct btrfs_fs_info *fs_info) +{ + if (!btrfs_test_opt(fs_info, AUTO_DEFRAG)) + return false; + + if (btrfs_fs_closing(fs_info)) + return false; + + return true; +} + +/* + * Insert a defrag record for this inode if auto defrag is enabled. No errors + * returned as they're not considered fatal. + */ +void btrfs_add_inode_defrag(struct btrfs_inode *inode, u32 extent_thresh) +{ + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct inode_defrag *defrag; + int ret; + + if (!need_auto_defrag(fs_info)) + return; + + if (test_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags)) + return; + + defrag = kmem_cache_zalloc(btrfs_inode_defrag_cachep, GFP_NOFS); + if (!defrag) + return; + + defrag->ino = btrfs_ino(inode); + defrag->transid = btrfs_get_root_last_trans(root); + defrag->root = btrfs_root_id(root); + defrag->extent_thresh = extent_thresh; + + spin_lock(&fs_info->defrag_inodes_lock); + if (!test_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags)) { + /* + * If we set IN_DEFRAG flag and evict the inode from memory, + * and then re-read this inode, this new inode doesn't have + * IN_DEFRAG flag. At the case, we may find the existed defrag. + */ + ret = btrfs_insert_inode_defrag(inode, defrag); + if (ret) + kmem_cache_free(btrfs_inode_defrag_cachep, defrag); + } else { + kmem_cache_free(btrfs_inode_defrag_cachep, defrag); + } + spin_unlock(&fs_info->defrag_inodes_lock); +} + +/* + * Pick the defraggable inode that we want, if it doesn't exist, we will get the + * next one. + */ +static struct inode_defrag *btrfs_pick_defrag_inode( + struct btrfs_fs_info *fs_info, u64 root, u64 ino) +{ + struct inode_defrag *entry = NULL; + struct inode_defrag tmp; + struct rb_node *p; + struct rb_node *parent = NULL; + int ret; + + tmp.ino = ino; + tmp.root = root; + + spin_lock(&fs_info->defrag_inodes_lock); + p = fs_info->defrag_inodes.rb_node; + while (p) { + parent = p; + entry = rb_entry(parent, struct inode_defrag, rb_node); + + ret = compare_inode_defrag(&tmp, entry); + if (ret < 0) + p = parent->rb_left; + else if (ret > 0) + p = parent->rb_right; + else + goto out; + } + + if (parent && compare_inode_defrag(&tmp, entry) > 0) { + parent = rb_next(parent); + entry = rb_entry_safe(parent, struct inode_defrag, rb_node); + } +out: + if (entry) + rb_erase(parent, &fs_info->defrag_inodes); + spin_unlock(&fs_info->defrag_inodes_lock); + return entry; +} + +void btrfs_cleanup_defrag_inodes(struct btrfs_fs_info *fs_info) +{ + struct inode_defrag *defrag, *next; + + spin_lock(&fs_info->defrag_inodes_lock); + + rbtree_postorder_for_each_entry_safe(defrag, next, + &fs_info->defrag_inodes, rb_node) + kmem_cache_free(btrfs_inode_defrag_cachep, defrag); + + fs_info->defrag_inodes = RB_ROOT; + + spin_unlock(&fs_info->defrag_inodes_lock); +} + +#define BTRFS_DEFRAG_BATCH 1024 + +static int btrfs_run_defrag_inode(struct btrfs_fs_info *fs_info, + struct inode_defrag *defrag, + struct file_ra_state *ra) +{ + struct btrfs_root *inode_root; + struct btrfs_inode *inode; + struct btrfs_ioctl_defrag_range_args range; + int ret = 0; + u64 cur = 0; + +again: + if (test_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state)) + goto cleanup; + if (!need_auto_defrag(fs_info)) + goto cleanup; + + /* Get the inode */ + inode_root = btrfs_get_fs_root(fs_info, defrag->root, true); + if (IS_ERR(inode_root)) { + ret = PTR_ERR(inode_root); + goto cleanup; + } + + inode = btrfs_iget(defrag->ino, inode_root); + btrfs_put_root(inode_root); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + goto cleanup; + } + + if (cur >= i_size_read(&inode->vfs_inode)) { + iput(&inode->vfs_inode); + goto cleanup; + } + + /* Do a chunk of defrag */ + clear_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags); + memset(&range, 0, sizeof(range)); + range.len = (u64)-1; + range.start = cur; + range.extent_thresh = defrag->extent_thresh; + file_ra_state_init(ra, inode->vfs_inode.i_mapping); + + scoped_guard(super_write, fs_info->sb) + ret = btrfs_defrag_file(inode, ra, &range, + defrag->transid, BTRFS_DEFRAG_BATCH); + iput(&inode->vfs_inode); + + if (ret < 0) + goto cleanup; + + cur = max(cur + fs_info->sectorsize, range.start); + goto again; + +cleanup: + kmem_cache_free(btrfs_inode_defrag_cachep, defrag); + return ret; +} + +/* + * Run through the list of inodes in the FS that need defragging. + */ +int btrfs_run_defrag_inodes(struct btrfs_fs_info *fs_info) +{ + struct inode_defrag *defrag; + u64 first_ino = 0; + u64 root_objectid = 0; + + atomic_inc(&fs_info->defrag_running); + while (1) { + struct file_ra_state ra = { 0 }; + + /* Pause the auto defragger. */ + if (test_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state)) + break; + + if (!need_auto_defrag(fs_info)) + break; + + /* find an inode to defrag */ + defrag = btrfs_pick_defrag_inode(fs_info, root_objectid, first_ino); + if (!defrag) { + if (root_objectid || first_ino) { + root_objectid = 0; + first_ino = 0; + continue; + } else { + break; + } + } + + first_ino = defrag->ino + 1; + root_objectid = defrag->root; + + btrfs_run_defrag_inode(fs_info, defrag, &ra); + } + atomic_dec(&fs_info->defrag_running); + + /* + * During unmount, we use the transaction_wait queue to wait for the + * defragger to stop. + */ + wake_up(&fs_info->transaction_wait); + return 0; +} + +/* + * Check if two blocks addresses are close, used by defrag. + */ +static bool close_blocks(u64 blocknr, u64 other, u32 blocksize) +{ + if (blocknr < other && other - (blocknr + blocksize) < SZ_32K) + return true; + if (blocknr > other && blocknr - (other + blocksize) < SZ_32K) + return true; + return false; +} + +/* + * Go through all the leaves pointed to by a node and reallocate them so that + * disk order is close to key order. + */ +static int btrfs_realloc_node(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct extent_buffer *parent, + int start_slot, u64 *last_ret, + struct btrfs_key *progress) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + const u32 blocksize = fs_info->nodesize; + const int end_slot = btrfs_header_nritems(parent) - 1; + u64 search_start = *last_ret; + u64 last_block = 0; + int ret = 0; + bool progress_passed = false; + + /* + * COWing must happen through a running transaction, which always + * matches the current fs generation (it's a transaction with a state + * less than TRANS_STATE_UNBLOCKED). If it doesn't, then turn the fs + * into error state to prevent the commit of any transaction. + */ + if (unlikely(trans->transaction != fs_info->running_transaction || + trans->transid != fs_info->generation)) { + btrfs_abort_transaction(trans, -EUCLEAN); + btrfs_crit(fs_info, +"unexpected transaction when attempting to reallocate parent %llu for root %llu, transaction %llu running transaction %llu fs generation %llu", + parent->start, btrfs_root_id(root), trans->transid, + fs_info->running_transaction->transid, + fs_info->generation); + return -EUCLEAN; + } + + if (btrfs_header_nritems(parent) <= 1) + return 0; + + for (int i = start_slot; i <= end_slot; i++) { + struct extent_buffer *cur; + struct btrfs_disk_key disk_key; + u64 blocknr; + u64 other; + bool close = true; + + btrfs_node_key(parent, &disk_key, i); + if (!progress_passed && btrfs_comp_keys(&disk_key, progress) < 0) + continue; + + progress_passed = true; + blocknr = btrfs_node_blockptr(parent, i); + if (last_block == 0) + last_block = blocknr; + + if (i > 0) { + other = btrfs_node_blockptr(parent, i - 1); + close = close_blocks(blocknr, other, blocksize); + } + if (!close && i < end_slot) { + other = btrfs_node_blockptr(parent, i + 1); + close = close_blocks(blocknr, other, blocksize); + } + if (close) { + last_block = blocknr; + continue; + } + + cur = btrfs_read_node_slot(parent, i); + if (IS_ERR(cur)) + return PTR_ERR(cur); + if (search_start == 0) + search_start = last_block; + + btrfs_tree_lock(cur); + ret = btrfs_force_cow_block(trans, root, cur, parent, i, + &cur, search_start, + min(16 * blocksize, + (end_slot - i) * blocksize), + BTRFS_NESTING_COW); + if (ret) { + btrfs_tree_unlock(cur); + free_extent_buffer(cur); + break; + } + search_start = cur->start; + last_block = cur->start; + *last_ret = search_start; + btrfs_tree_unlock(cur); + free_extent_buffer(cur); + } + return ret; +} + +/* + * Defrag all the leaves in a given btree. + * Read all the leaves and try to get key order to + * better reflect disk order + */ + +static int btrfs_defrag_leaves(struct btrfs_trans_handle *trans, + struct btrfs_root *root) +{ + struct btrfs_path *path = NULL; + struct btrfs_key key; + int ret = 0; + int wret; + int level; + int next_key_ret = 0; + u64 last_ret = 0; + + if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) + goto out; + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto out; + } + + level = btrfs_header_level(root->node); + + if (level == 0) + goto out; + + if (root->defrag_progress.objectid == 0) { + struct extent_buffer *root_node; + u32 nritems; + + root_node = btrfs_lock_root_node(root); + nritems = btrfs_header_nritems(root_node); + root->defrag_max.objectid = 0; + /* from above we know this is not a leaf */ + btrfs_node_key_to_cpu(root_node, &root->defrag_max, + nritems - 1); + btrfs_tree_unlock(root_node); + free_extent_buffer(root_node); + memset(&key, 0, sizeof(key)); + } else { + memcpy(&key, &root->defrag_progress, sizeof(key)); + } + + path->keep_locks = true; + + ret = btrfs_search_forward(root, &key, path, BTRFS_OLDEST_GENERATION); + if (ret < 0) + goto out; + if (ret > 0) { + ret = 0; + goto out; + } + btrfs_release_path(path); + /* + * We don't need a lock on a leaf. btrfs_realloc_node() will lock all + * leafs from path->nodes[1], so set lowest_level to 1 to avoid later + * a deadlock (attempting to write lock an already write locked leaf). + */ + path->lowest_level = 1; + wret = btrfs_search_slot(trans, root, &key, path, 0, 1); + + if (wret < 0) { + ret = wret; + goto out; + } + if (!path->nodes[1]) { + ret = 0; + goto out; + } + /* + * The node at level 1 must always be locked when our path has + * keep_locks set and lowest_level is 1, regardless of the value of + * path->slots[1]. + */ + ASSERT(path->locks[1] != 0); + ret = btrfs_realloc_node(trans, root, + path->nodes[1], 0, + &last_ret, + &root->defrag_progress); + if (ret) { + WARN_ON(ret == -EAGAIN); + goto out; + } + /* + * Now that we reallocated the node we can find the next key. Note that + * btrfs_find_next_key() can release our path and do another search + * without COWing, this is because even with path->keep_locks == true, + * btrfs_search_slot() / ctree.c:unlock_up() does not keeps a lock on a + * node when path->slots[node_level - 1] does not point to the last + * item or a slot beyond the last item (ctree.c:unlock_up()). Therefore + * we search for the next key after reallocating our node. + */ + path->slots[1] = btrfs_header_nritems(path->nodes[1]); + next_key_ret = btrfs_find_next_key(root, path, &key, 1, + BTRFS_OLDEST_GENERATION); + if (next_key_ret == 0) { + memcpy(&root->defrag_progress, &key, sizeof(key)); + ret = -EAGAIN; + } +out: + btrfs_free_path(path); + if (ret == -EAGAIN) { + if (root->defrag_max.objectid > root->defrag_progress.objectid) + goto done; + if (root->defrag_max.type > root->defrag_progress.type) + goto done; + if (root->defrag_max.offset > root->defrag_progress.offset) + goto done; + ret = 0; + } +done: + if (ret != -EAGAIN) + memset(&root->defrag_progress, 0, + sizeof(root->defrag_progress)); + + return ret; +} + +/* + * Defrag a given btree. Every leaf in the btree is read and defragmented. + */ +int btrfs_defrag_root(struct btrfs_root *root) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + int ret; + + if (test_and_set_bit(BTRFS_ROOT_DEFRAG_RUNNING, &root->state)) + return 0; + + while (1) { + struct btrfs_trans_handle *trans; + + trans = btrfs_start_transaction(root, 0); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + break; + } + + ret = btrfs_defrag_leaves(trans, root); + + btrfs_end_transaction(trans); + btrfs_btree_balance_dirty(fs_info); + cond_resched(); + + if (btrfs_fs_closing(fs_info) || ret != -EAGAIN) + break; + + if (btrfs_defrag_cancelled(fs_info)) { + btrfs_debug(fs_info, "defrag_root cancelled"); + ret = -EAGAIN; + break; + } + } + clear_bit(BTRFS_ROOT_DEFRAG_RUNNING, &root->state); + return ret; +} + +/* + * Defrag specific helper to get an extent map. + * + * Differences between this and btrfs_get_extent() are: + * + * - No extent_map will be added to inode->extent_tree + * To reduce memory usage in the long run. + * + * - Extra optimization to skip file extents older than @newer_than + * By using btrfs_search_forward() we can skip entire file ranges that + * have extents created in past transactions, because btrfs_search_forward() + * will not visit leaves and nodes with a generation smaller than given + * minimal generation threshold (@newer_than). + * + * Return valid em if we find a file extent matching the requirement. + * Return NULL if we can not find a file extent matching the requirement. + * + * Return ERR_PTR() for error. + */ +static struct extent_map *defrag_get_extent(struct btrfs_inode *inode, + u64 start, u64 newer_than) +{ + struct btrfs_root *root = inode->root; + struct btrfs_file_extent_item *fi; + BTRFS_PATH_AUTO_RELEASE(path); + struct extent_map *em; + struct btrfs_key key; + u64 ino = btrfs_ino(inode); + int ret; + + em = btrfs_alloc_extent_map(); + if (!em) { + ret = -ENOMEM; + goto err; + } + + key.objectid = ino; + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = start; + + if (newer_than) { + ret = btrfs_search_forward(root, &key, &path, newer_than); + if (ret < 0) + goto err; + /* Can't find anything newer */ + if (ret > 0) + goto not_found; + } else { + ret = btrfs_search_slot(NULL, root, &key, &path, 0, 0); + if (ret < 0) + goto err; + } + if (path.slots[0] >= btrfs_header_nritems(path.nodes[0])) { + /* + * If btrfs_search_slot() makes path to point beyond nritems, + * we should not have an empty leaf, as this inode must at + * least have its INODE_ITEM. + */ + ASSERT(btrfs_header_nritems(path.nodes[0])); + path.slots[0] = btrfs_header_nritems(path.nodes[0]) - 1; + } + btrfs_item_key_to_cpu(path.nodes[0], &key, path.slots[0]); + /* Perfect match, no need to go one slot back */ + if (key.objectid == ino && key.type == BTRFS_EXTENT_DATA_KEY && + key.offset == start) + goto iterate; + + /* We didn't find a perfect match, needs to go one slot back */ + if (path.slots[0] > 0) { + btrfs_item_key_to_cpu(path.nodes[0], &key, path.slots[0]); + if (key.objectid == ino && key.type == BTRFS_EXTENT_DATA_KEY) + path.slots[0]--; + } + +iterate: + /* Iterate through the path to find a file extent covering @start */ + while (true) { + u64 extent_end; + + if (path.slots[0] >= btrfs_header_nritems(path.nodes[0])) + goto next; + + btrfs_item_key_to_cpu(path.nodes[0], &key, path.slots[0]); + + /* + * We may go one slot back to INODE_REF/XATTR item, then + * need to go forward until we reach an EXTENT_DATA. + * But we should still has the correct ino as key.objectid. + */ + if (WARN_ON(key.objectid < ino) || key.type < BTRFS_EXTENT_DATA_KEY) + goto next; + + /* It's beyond our target range, definitely not extent found */ + if (key.objectid > ino || key.type > BTRFS_EXTENT_DATA_KEY) + goto not_found; + + /* + * | |<- File extent ->| + * \- start + * + * This means there is a hole between start and key.offset. + */ + if (key.offset > start) { + em->start = start; + em->disk_bytenr = EXTENT_MAP_HOLE; + em->disk_num_bytes = 0; + em->ram_bytes = 0; + em->offset = 0; + em->len = key.offset - start; + break; + } + + fi = btrfs_item_ptr(path.nodes[0], path.slots[0], + struct btrfs_file_extent_item); + extent_end = btrfs_file_extent_end(&path); + + /* + * |<- file extent ->| | + * \- start + * + * We haven't reached start, search next slot. + */ + if (extent_end <= start) + goto next; + + /* Now this extent covers @start, convert it to em */ + btrfs_extent_item_to_extent_map(inode, &path, fi, em); + break; +next: + ret = btrfs_next_item(root, &path); + if (ret < 0) + goto err; + if (ret > 0) + goto not_found; + } + return em; + +not_found: + btrfs_free_extent_map(em); + return NULL; + +err: + btrfs_free_extent_map(em); + return ERR_PTR(ret); +} + +static struct extent_map *defrag_lookup_extent(struct inode *inode, u64 start, + u64 newer_than, bool locked) +{ + struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; + struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; + struct extent_map *em; + const u32 sectorsize = BTRFS_I(inode)->root->fs_info->sectorsize; + + /* + * Hopefully we have this extent in the tree already, try without the + * full extent lock. + */ + read_lock(&em_tree->lock); + em = btrfs_lookup_extent_mapping(em_tree, start, sectorsize); + read_unlock(&em_tree->lock); + + /* + * We can get a merged extent, in that case, we need to re-search + * tree to get the original em for defrag. + * + * This is because even if we have adjacent extents that are contiguous + * and compatible (same type and flags), we still want to defrag them + * so that we use less metadata (extent items in the extent tree and + * file extent items in the inode's subvolume tree). + */ + if (em && (em->flags & EXTENT_FLAG_MERGED)) { + btrfs_free_extent_map(em); + em = NULL; + } + + if (!em) { + struct extent_state *cached = NULL; + u64 end = start + sectorsize - 1; + + /* Get the big lock and read metadata off disk. */ + if (!locked) + btrfs_lock_extent(io_tree, start, end, &cached); + em = defrag_get_extent(BTRFS_I(inode), start, newer_than); + if (!locked) + btrfs_unlock_extent(io_tree, start, end, &cached); + + if (IS_ERR(em)) + return NULL; + } + + return em; +} + +static u32 get_extent_max_capacity(const struct btrfs_fs_info *fs_info, + const struct extent_map *em) +{ + if (btrfs_extent_map_is_compressed(em)) + return BTRFS_MAX_COMPRESSED; + return fs_info->max_extent_size; +} + +static bool defrag_check_next_extent(struct inode *inode, struct extent_map *em, + u32 extent_thresh, u64 newer_than, bool locked) +{ + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + struct extent_map *next; + const u64 em_end = btrfs_extent_map_end(em); + bool ret = false; + + /* This is the last extent */ + if (em_end >= i_size_read(inode)) + return false; + + /* + * Here we need to pass @newer_then when checking the next extent, or + * we will hit a case we mark current extent for defrag, but the next + * one will not be a target. + * This will just cause extra IO without really reducing the fragments. + */ + next = defrag_lookup_extent(inode, em_end, newer_than, locked); + /* No more em or hole */ + if (!next || next->disk_bytenr >= EXTENT_MAP_LAST_BYTE) + goto out; + if (next->flags & EXTENT_FLAG_PREALLOC) + goto out; + /* + * If the next extent is at its max capacity, defragging current extent + * makes no sense, as the total number of extents won't change. + */ + if (next->len >= get_extent_max_capacity(fs_info, em)) + goto out; + /* Skip older extent */ + if (next->generation < newer_than) + goto out; + /* Also check extent size */ + if (next->len >= extent_thresh) + goto out; + + ret = true; +out: + btrfs_free_extent_map(next); + return ret; +} + +/* + * Prepare one page to be defragged. + * + * This will ensure: + * + * - Returned page is locked and has been set up properly. + * - No ordered extent exists in the page. + * - The page is uptodate. + * + * NOTE: Caller should also wait for page writeback after the cluster is + * prepared, here we don't do writeback wait for each page. + */ +static struct folio *defrag_prepare_one_folio(struct btrfs_inode *inode, pgoff_t index) +{ + struct address_space *mapping = inode->vfs_inode.i_mapping; + gfp_t mask = btrfs_alloc_write_mask(mapping); + u64 lock_start; + u64 lock_end; + struct extent_state *cached_state = NULL; + struct folio *folio; + int ret; + +again: + /* TODO: Add order fgp order flags when large folios are fully enabled. */ + folio = __filemap_get_folio(mapping, index, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, mask); + if (IS_ERR(folio)) + return folio; + + ret = set_folio_extent_mapped(folio); + if (ret < 0) { + folio_unlock(folio); + folio_put(folio); + return ERR_PTR(ret); + } + + lock_start = folio_pos(folio); + lock_end = folio_next_pos(folio) - 1; + /* Wait for any existing ordered extent in the range */ + while (1) { + struct btrfs_ordered_extent *ordered; + + btrfs_lock_extent(&inode->io_tree, lock_start, lock_end, &cached_state); + ordered = btrfs_lookup_ordered_range(inode, lock_start, folio_size(folio)); + btrfs_unlock_extent(&inode->io_tree, lock_start, lock_end, &cached_state); + if (!ordered) + break; + + folio_unlock(folio); + btrfs_start_ordered_extent(ordered); + btrfs_put_ordered_extent(ordered); + folio_lock(folio); + /* + * We unlocked the folio above, so we need check if it was + * released or not. + */ + if (folio->mapping != mapping || !folio->private) { + folio_unlock(folio); + folio_put(folio); + goto again; + } + } + + /* + * Now the page range has no ordered extent any more. Read the page to + * make it uptodate. + */ + if (!folio_test_uptodate(folio)) { + btrfs_read_folio(NULL, folio); + folio_lock(folio); + if (folio->mapping != mapping || !folio->private) { + folio_unlock(folio); + folio_put(folio); + goto again; + } + if (unlikely(!folio_test_uptodate(folio))) { + folio_unlock(folio); + folio_put(folio); + return ERR_PTR(-EIO); + } + } + return folio; +} + +struct defrag_target_range { + struct list_head list; + u64 start; + u64 len; +}; + +/* + * Collect all valid target extents. + * + * @start: file offset to lookup + * @len: length to lookup + * @extent_thresh: file extent size threshold, any extent size >= this value + * will be ignored + * @newer_than: only defrag extents newer than this value + * @do_compress: whether the defrag is doing compression or no-compression + * if true, @extent_thresh will be ignored and all regular + * file extents meeting @newer_than will be targets. + * @locked: if the range has already held extent lock + * @target_list: list of targets file extents + */ +static int defrag_collect_targets(struct btrfs_inode *inode, + u64 start, u64 len, u32 extent_thresh, + u64 newer_than, bool do_compress, + bool locked, struct list_head *target_list, + u64 *last_scanned_ret) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + bool last_is_target = false; + u64 cur = start; + int ret = 0; + + while (cur < start + len) { + struct extent_map *em; + struct defrag_target_range *new; + bool next_mergeable = true; + u64 range_len; + + last_is_target = false; + em = defrag_lookup_extent(&inode->vfs_inode, cur, newer_than, locked); + if (!em) + break; + + /* + * If the file extent is an inlined one, we may still want to + * defrag it (fallthrough) if it will cause a regular extent. + * This is for users who want to convert inline extents to + * regular ones through max_inline= mount option. + */ + if (em->disk_bytenr == EXTENT_MAP_INLINE && + em->len <= inode->root->fs_info->max_inline) + goto next; + + /* Skip holes and preallocated extents. */ + if (em->disk_bytenr == EXTENT_MAP_HOLE || + (em->flags & EXTENT_FLAG_PREALLOC)) + goto next; + + /* Skip older extent */ + if (em->generation < newer_than) + goto next; + + /* This em is under writeback, no need to defrag */ + if (em->generation == (u64)-1) + goto next; + + /* + * Our start offset might be in the middle of an existing extent + * map, so take that into account. + */ + range_len = em->len - (cur - em->start); + /* + * If this range of the extent map is already flagged for delalloc, + * skip it, because: + * + * 1) We could deadlock later, when trying to reserve space for + * delalloc, because in case we can't immediately reserve space + * the flusher can start delalloc and wait for the respective + * ordered extents to complete. The deadlock would happen + * because we do the space reservation while holding the range + * locked, and starting writeback, or finishing an ordered + * extent, requires locking the range; + * + * 2) If there's delalloc there, it means there's dirty pages for + * which writeback has not started yet (we clean the delalloc + * flag when starting writeback and after creating an ordered + * extent). If we mark pages in an adjacent range for defrag, + * then we will have a larger contiguous range for delalloc, + * very likely resulting in a larger extent after writeback is + * triggered (except in a case of free space fragmentation). + */ + if (btrfs_test_range_bit_exists(&inode->io_tree, cur, cur + range_len - 1, + EXTENT_DELALLOC)) + goto next; + + /* + * For do_compress case, we want to compress all valid file + * extents, thus no @extent_thresh or mergeable check. + */ + if (do_compress) + goto add; + + /* Skip too large extent */ + if (em->len >= extent_thresh) + goto next; + + /* + * Skip extents already at its max capacity, this is mostly for + * compressed extents, which max cap is only 128K. + */ + if (em->len >= get_extent_max_capacity(fs_info, em)) + goto next; + + /* + * Normally there are no more extents after an inline one, thus + * @next_mergeable will normally be false and not defragged. + * So if an inline extent passed all above checks, just add it + * for defrag, and be converted to regular extents. + */ + if (em->disk_bytenr == EXTENT_MAP_INLINE) + goto add; + + next_mergeable = defrag_check_next_extent(&inode->vfs_inode, em, + extent_thresh, newer_than, locked); + if (!next_mergeable) { + struct defrag_target_range *last; + + /* Empty target list, no way to merge with last entry */ + if (list_empty(target_list)) + goto next; + last = list_last_entry(target_list, + struct defrag_target_range, list); + /* Not mergeable with last entry */ + if (last->start + last->len != cur) + goto next; + + /* Mergeable, fall through to add it to @target_list. */ + } + +add: + last_is_target = true; + range_len = min(btrfs_extent_map_end(em), start + len) - cur; + /* + * This one is a good target, check if it can be merged into + * last range of the target list. + */ + if (!list_empty(target_list)) { + struct defrag_target_range *last; + + last = list_last_entry(target_list, + struct defrag_target_range, list); + ASSERT(last->start + last->len <= cur); + if (last->start + last->len == cur) { + /* Mergeable, enlarge the last entry */ + last->len += range_len; + goto next; + } + /* Fall through to allocate a new entry */ + } + + /* Allocate new defrag_target_range */ + new = kmalloc_obj(*new, GFP_NOFS); + if (!new) { + btrfs_free_extent_map(em); + ret = -ENOMEM; + break; + } + new->start = cur; + new->len = range_len; + list_add_tail(&new->list, target_list); + +next: + cur = btrfs_extent_map_end(em); + btrfs_free_extent_map(em); + } + if (ret < 0) { + struct defrag_target_range *entry; + struct defrag_target_range *tmp; + + list_for_each_entry_safe(entry, tmp, target_list, list) { + list_del(&entry->list); + kfree(entry); + } + } + if (!ret && last_scanned_ret) { + /* + * If the last extent is not a target, the caller can skip to + * the end of that extent. + * Otherwise, we can only go the end of the specified range. + */ + if (!last_is_target) + *last_scanned_ret = max(cur, *last_scanned_ret); + else + *last_scanned_ret = max(start + len, *last_scanned_ret); + } + return ret; +} + +#define CLUSTER_SIZE (SZ_256K) +static_assert(PAGE_ALIGNED(CLUSTER_SIZE)); + +/* + * Defrag one contiguous target range. + * + * @inode: target inode + * @target: target range to defrag + * @pages: locked pages covering the defrag range + * @nr_pages: number of locked pages + * + * Caller should ensure: + * + * - Pages are prepared + * Pages should be locked, no ordered extent in the pages range, + * no writeback. + * + * - Extent bits are locked + */ +static void defrag_one_locked_target(struct btrfs_inode *inode, + struct defrag_target_range *target, + struct folio **folios, int nr_pages, + struct extent_state **cached_state) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + const u64 start = target->start; + const u64 len = target->len; + + btrfs_clear_extent_bit(&inode->io_tree, start, start + len - 1, + EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | + EXTENT_DEFRAG, cached_state); + btrfs_set_extent_bit(&inode->io_tree, start, start + len - 1, + EXTENT_DELALLOC | EXTENT_DEFRAG, cached_state); + + /* + * Update the page status. + * Due to possible large folios, we have to check all folios one by one. + */ + for (int i = 0; i < nr_pages && folios[i]; i++) { + struct folio *folio = folios[i]; + + if (!folio) + break; + if (start >= folio_next_pos(folio) || + start + len <= folio_pos(folio)) + continue; + btrfs_folio_clamp_set_dirty(fs_info, folio, start, len); + } +} + +static int defrag_one_range(struct btrfs_inode *inode, u64 start, u32 len, + u32 extent_thresh, u64 newer_than, bool do_compress, + u64 *last_scanned_ret) +{ + struct extent_state *cached_state = NULL; + struct defrag_target_range *entry; + struct defrag_target_range *tmp; + LIST_HEAD(target_list); + struct folio AUTO_KFREE(*folios); + const u32 sectorsize = inode->root->fs_info->sectorsize; + u64 cur = start; + const unsigned int nr_pages = ((start + len - 1) >> PAGE_SHIFT) - + (start >> PAGE_SHIFT) + 1; + struct extent_changeset *data_reserved = NULL; + u64 last_defrag_end = start; + int ret = 0; + + ASSERT(nr_pages <= CLUSTER_SIZE / PAGE_SIZE); + ASSERT(IS_ALIGNED(start, sectorsize) && IS_ALIGNED(len, sectorsize)); + + folios = kzalloc_objs(struct folio *, nr_pages, GFP_NOFS); + if (!folios) + return -ENOMEM; + + /* + * Reserve delalloc space before locking the range and before locking + * and dirtying any folios - otherwise we could deadlock, for example + * after defrag of one range we dirty folios and keep them locked when + * we move to the next range, so reserving delalloc space right before + * each range could trigger flushing of delalloc and deadlock on the + * extent lock or trigger a transaction commit with flushoncommit, which + * can either deadlock on the lock of a folio made dirty in the previous + * range or the extent lock. + */ + ret = btrfs_delalloc_reserve_space(inode, &data_reserved, start, len); + if (ret < 0) + return ret; + + /* Prepare all pages */ + for (int i = 0; cur < start + len && i < nr_pages; i++) { + folios[i] = defrag_prepare_one_folio(inode, cur >> PAGE_SHIFT); + if (IS_ERR(folios[i])) { + ret = PTR_ERR(folios[i]); + folios[i] = NULL; + goto free_folios; + } + cur = folio_next_pos(folios[i]); + } + for (int i = 0; i < nr_pages; i++) { + if (!folios[i]) + break; + folio_wait_writeback(folios[i]); + } + + /* We should get at least one folio. */ + ASSERT(folios[0]); + /* Lock the pages range */ + btrfs_lock_extent(&inode->io_tree, folio_pos(folios[0]), cur - 1, &cached_state); + /* + * Now we have a consistent view about the extent map, re-check + * which range really needs to be defragged. + * + * And this time we have extent locked already, pass @locked = true + * so that we won't relock the extent range and cause deadlock. + */ + ret = defrag_collect_targets(inode, start, len, extent_thresh, + newer_than, do_compress, true, + &target_list, last_scanned_ret); + if (ret < 0) + goto unlock_extent; + + list_for_each_entry_safe(entry, tmp, &target_list, list) { + defrag_one_locked_target(inode, entry, folios, nr_pages, &cached_state); + if (entry->start > last_defrag_end) + btrfs_delalloc_release_space(inode, data_reserved, last_defrag_end, + entry->start - last_defrag_end, true); + last_defrag_end = entry->start + entry->len; + kfree(entry); + } +unlock_extent: + btrfs_unlock_extent(&inode->io_tree, folio_pos(folios[0]), cur - 1, &cached_state); +free_folios: + for (int i = 0; i < nr_pages; i++) { + if (!folios[i]) + break; + folio_unlock(folios[i]); + folio_put(folios[i]); + } + btrfs_delalloc_release_extents(inode, len); + if (last_defrag_end < start + len) + btrfs_delalloc_release_space(inode, data_reserved, last_defrag_end, + start + len - last_defrag_end, true); + extent_changeset_free(data_reserved); + + return ret; +} + +static int defrag_one_cluster(struct btrfs_inode *inode, + struct file_ra_state *ra, + u64 start, u32 len, u32 extent_thresh, + u64 newer_than, bool do_compress, + unsigned long *sectors_defragged, + unsigned long max_sectors, + u64 *last_scanned_ret) +{ + const u32 sectorsize = inode->root->fs_info->sectorsize; + struct defrag_target_range *entry; + struct defrag_target_range *tmp; + LIST_HEAD(target_list); + int ret; + + ret = defrag_collect_targets(inode, start, len, extent_thresh, + newer_than, do_compress, false, + &target_list, NULL); + if (ret < 0) + goto out; + + list_for_each_entry(entry, &target_list, list) { + u32 range_len = entry->len; + + /* Reached or beyond the limit */ + if (max_sectors && *sectors_defragged >= max_sectors) { + ret = 1; + break; + } + + if (max_sectors) + range_len = min_t(u32, range_len, + (max_sectors - *sectors_defragged) * sectorsize); + + /* + * If defrag_one_range() has updated last_scanned_ret, + * our range may already be invalid (e.g. hole punched). + * Skip if our range is before last_scanned_ret, as there is + * no need to defrag the range anymore. + */ + if (entry->start + range_len <= *last_scanned_ret) + continue; + + page_cache_sync_readahead(inode->vfs_inode.i_mapping, + ra, NULL, entry->start >> PAGE_SHIFT, + ((entry->start + range_len - 1) >> PAGE_SHIFT) - + (entry->start >> PAGE_SHIFT) + 1); + /* + * Here we may not defrag any range if holes are punched before + * we locked the pages. + * But that's fine, it only affects the @sectors_defragged + * accounting. + */ + ret = defrag_one_range(inode, entry->start, range_len, + extent_thresh, newer_than, do_compress, + last_scanned_ret); + if (ret < 0) + break; + *sectors_defragged += range_len >> + inode->root->fs_info->sectorsize_bits; + } +out: + list_for_each_entry_safe(entry, tmp, &target_list, list) + kfree(entry); + if (ret >= 0) + *last_scanned_ret = max(*last_scanned_ret, start + len); + return ret; +} + +/* + * Entry point to file defragmentation. + * + * @inode: inode to be defragged + * @ra: readahead state + * @range: defrag options including range and flags + * @newer_than: minimum transid to defrag + * @max_to_defrag: max number of sectors to be defragged, if 0, the whole inode + * will be defragged. + * + * Return <0 for error. + * Return >=0 for the number of sectors defragged, and range->start will be updated + * to indicate the file offset where next defrag should be started at. + * (Mostly for autodefrag, which sets @max_to_defrag thus we may exit early without + * defragging all the range). + */ +int btrfs_defrag_file(struct btrfs_inode *inode, struct file_ra_state *ra, + struct btrfs_ioctl_defrag_range_args *range, + u64 newer_than, unsigned long max_to_defrag) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + unsigned long sectors_defragged = 0; + u64 isize = i_size_read(&inode->vfs_inode); + u64 cur; + u64 last_byte; + bool do_compress = (range->flags & BTRFS_DEFRAG_RANGE_COMPRESS); + bool no_compress = (range->flags & BTRFS_DEFRAG_RANGE_NOCOMPRESS); + int compress_type = BTRFS_COMPRESS_ZLIB; + int compress_level = 0; + int ret = 0; + u32 extent_thresh = range->extent_thresh; + pgoff_t start_index; + + ASSERT(ra); + + if (isize == 0) + return 0; + + if (range->start >= isize) + return -EINVAL; + + if (do_compress) { + if (range->flags & BTRFS_DEFRAG_RANGE_COMPRESS_LEVEL) { + if (range->compress.type >= BTRFS_NR_COMPRESS_TYPES) + return -EINVAL; + if (range->compress.type) { + compress_type = range->compress.type; + compress_level = range->compress.level; + if (!btrfs_compress_level_valid(compress_type, compress_level)) + return -EINVAL; + } + } else { + if (range->compress_type >= BTRFS_NR_COMPRESS_TYPES) + return -EINVAL; + if (range->compress_type) + compress_type = range->compress_type; + } + } else if (range->flags & BTRFS_DEFRAG_RANGE_NOCOMPRESS) { + compress_type = BTRFS_DEFRAG_DONT_COMPRESS; + compress_level = 1; + } + + if (extent_thresh == 0) + extent_thresh = SZ_256K; + + if (range->start + range->len > range->start) { + /* Got a specific range */ + last_byte = min(isize, range->start + range->len); + } else { + /* Defrag until file end */ + last_byte = isize; + } + + /* Align the range */ + cur = round_down(range->start, fs_info->sectorsize); + last_byte = round_up(last_byte, fs_info->sectorsize) - 1; + + /* + * Make writeback start from the beginning of the range, so that the + * defrag range can be written sequentially. + */ + start_index = cur >> PAGE_SHIFT; + if (start_index < inode->vfs_inode.i_mapping->writeback_index) + inode->vfs_inode.i_mapping->writeback_index = start_index; + + while (cur < last_byte) { + const unsigned long prev_sectors_defragged = sectors_defragged; + u64 last_scanned = cur; + u64 cluster_end; + + if (btrfs_defrag_cancelled(fs_info)) { + ret = -EAGAIN; + break; + } + + /* We want the cluster end at page boundary when possible */ + cluster_end = (((cur >> PAGE_SHIFT) + + (SZ_256K >> PAGE_SHIFT)) << PAGE_SHIFT) - 1; + cluster_end = min(cluster_end, last_byte); + + btrfs_inode_lock(inode, 0); + if (IS_SWAPFILE(&inode->vfs_inode)) { + ret = -ETXTBSY; + btrfs_inode_unlock(inode, 0); + break; + } + if (!(inode->vfs_inode.i_sb->s_flags & SB_ACTIVE)) { + btrfs_inode_unlock(inode, 0); + break; + } + if (do_compress || no_compress) { + inode->defrag_compress = compress_type; + inode->defrag_compress_level = compress_level; + } + ret = defrag_one_cluster(inode, ra, cur, + cluster_end + 1 - cur, extent_thresh, + newer_than, do_compress || no_compress, + §ors_defragged, + max_to_defrag, &last_scanned); + + if (sectors_defragged > prev_sectors_defragged) + balance_dirty_pages_ratelimited(inode->vfs_inode.i_mapping); + + btrfs_inode_unlock(inode, 0); + if (ret < 0) + break; + cur = max(cluster_end + 1, last_scanned); + if (ret > 0) { + ret = 0; + break; + } + cond_resched(); + } + + /* + * Update range.start for autodefrag, this will indicate where to start + * in next run. + */ + range->start = cur; + if (sectors_defragged) { + /* + * We have defragged some sectors, for compression case they + * need to be written back immediately. + */ + if (range->flags & BTRFS_DEFRAG_RANGE_START_IO) { + filemap_flush(inode->vfs_inode.i_mapping); + if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, + &inode->runtime_flags)) + filemap_flush(inode->vfs_inode.i_mapping); + } + if (range->compress_type == BTRFS_COMPRESS_LZO) + btrfs_set_fs_incompat(fs_info, COMPRESS_LZO); + else if (range->compress_type == BTRFS_COMPRESS_ZSTD) + btrfs_set_fs_incompat(fs_info, COMPRESS_ZSTD); + ret = sectors_defragged; + } + if (do_compress || no_compress) { + btrfs_inode_lock(inode, 0); + inode->defrag_compress = BTRFS_COMPRESS_NONE; + btrfs_inode_unlock(inode, 0); + } + return ret; +} + +void __cold btrfs_auto_defrag_exit(void) +{ + kmem_cache_destroy(btrfs_inode_defrag_cachep); +} + +int __init btrfs_auto_defrag_init(void) +{ + btrfs_inode_defrag_cachep = kmem_cache_create("btrfs_inode_defrag", + sizeof(struct inode_defrag), 0, 0, NULL); + if (!btrfs_inode_defrag_cachep) + return -ENOMEM; + + return 0; +} diff --git a/fs/btrfs/defrag.h b/fs/btrfs/defrag.h new file mode 100644 index 000000000..a7f917a38 --- /dev/null +++ b/fs/btrfs/defrag.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_DEFRAG_H +#define BTRFS_DEFRAG_H + +#include +#include + +struct file_ra_state; +struct btrfs_inode; +struct btrfs_fs_info; +struct btrfs_root; +struct btrfs_trans_handle; +struct btrfs_ioctl_defrag_range_args; + +int btrfs_defrag_file(struct btrfs_inode *inode, struct file_ra_state *ra, + struct btrfs_ioctl_defrag_range_args *range, + u64 newer_than, unsigned long max_to_defrag); +int __init btrfs_auto_defrag_init(void); +void __cold btrfs_auto_defrag_exit(void); +void btrfs_add_inode_defrag(struct btrfs_inode *inode, u32 extent_thresh); +int btrfs_run_defrag_inodes(struct btrfs_fs_info *fs_info); +void btrfs_cleanup_defrag_inodes(struct btrfs_fs_info *fs_info); +int btrfs_defrag_root(struct btrfs_root *root); + +static inline int btrfs_defrag_cancelled(struct btrfs_fs_info *fs_info) +{ + return signal_pending(current); +} + +#endif diff --git a/fs/btrfs/delalloc-space.c b/fs/btrfs/delalloc-space.c new file mode 100644 index 000000000..d357ed7ef --- /dev/null +++ b/fs/btrfs/delalloc-space.c @@ -0,0 +1,539 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "messages.h" +#include "ctree.h" +#include "delalloc-space.h" +#include "block-rsv.h" +#include "btrfs_inode.h" +#include "space-info.h" +#include "qgroup.h" +#include "fs.h" + +/* + * HOW DOES THIS WORK + * + * There are two stages to data reservations, one for data and one for metadata + * to handle the new extents and checksums generated by writing data. + * + * + * DATA RESERVATION + * The general flow of the data reservation is as follows + * + * -> Reserve + * We call into btrfs_reserve_data_bytes() for the user request bytes that + * they wish to write. We make this reservation and add it to + * space_info->bytes_may_use. We set EXTENT_DELALLOC on the inode io_tree + * for the range and carry on if this is buffered, or follow up trying to + * make a real allocation if we are pre-allocating or doing O_DIRECT. + * + * -> Use + * At writepages()/prealloc/O_DIRECT time we will call into + * btrfs_reserve_extent() for some part or all of this range of bytes. We + * will make the allocation and subtract space_info->bytes_may_use by the + * original requested length and increase the space_info->bytes_reserved by + * the allocated length. This distinction is important because compression + * may allocate a smaller on disk extent than we previously reserved. + * + * -> Allocation + * finish_ordered_io() will insert the new file extent item for this range, + * and then add a delayed ref update for the extent tree. Once that delayed + * ref is written the extent size is subtracted from + * space_info->bytes_reserved and added to space_info->bytes_used. + * + * Error handling + * + * -> By the reservation maker + * This is the simplest case, we haven't completed our operation and we know + * how much we reserved, we can simply call + * btrfs_free_reserved_data_space*() and it will be removed from + * space_info->bytes_may_use. + * + * -> After the reservation has been made, but before cow_file_range() + * This is specifically for the delalloc case. You must clear + * EXTENT_DELALLOC with the EXTENT_CLEAR_DATA_RESV bit, and the range will + * be subtracted from space_info->bytes_may_use. + * + * METADATA RESERVATION + * The general metadata reservation lifetimes are discussed elsewhere, this + * will just focus on how it is used for delalloc space. + * + * We keep track of two things on a per inode bases + * + * ->outstanding_extents + * This is the number of file extent items we'll need to handle all of the + * outstanding DELALLOC space we have in this inode. We limit the maximum + * size of an extent, so a large contiguous dirty area may require more than + * one outstanding_extent, which is why count_max_extents() is used to + * determine how many outstanding_extents get added. + * + * ->csum_bytes + * This is essentially how many dirty bytes we have for this inode, so we + * can calculate the number of checksum items we would have to add in order + * to checksum our outstanding data. + * + * We keep a per-inode block_rsv in order to make it easier to keep track of + * our reservation. We use btrfs_calculate_inode_block_rsv_size() to + * calculate the current theoretical maximum reservation we would need for the + * metadata for this inode. We call this and then adjust our reservation as + * necessary, either by attempting to reserve more space, or freeing up excess + * space. + * + * OUTSTANDING_EXTENTS HANDLING + * + * ->outstanding_extents is used for keeping track of how many extents we will + * need to use for this inode, and it will fluctuate depending on where you are + * in the life cycle of the dirty data. Consider the following normal case for + * a completely clean inode, with a num_bytes < our maximum allowed extent size + * + * -> reserve + * ->outstanding_extents += 1 (current value is 1) + * + * -> set_delalloc + * ->outstanding_extents += 1 (current value is 2) + * + * -> btrfs_delalloc_release_extents() + * ->outstanding_extents -= 1 (current value is 1) + * + * We must call this once we are done, as we hold our reservation for the + * duration of our operation, and then assume set_delalloc will update the + * counter appropriately. + * + * -> add ordered extent + * ->outstanding_extents += 1 (current value is 2) + * + * -> btrfs_clear_delalloc_extent + * ->outstanding_extents -= 1 (current value is 1) + * + * -> finish_ordered_io/btrfs_remove_ordered_extent + * ->outstanding_extents -= 1 (current value is 0) + * + * Each stage is responsible for their own accounting of the extent, thus + * making error handling and cleanup easier. + */ + +static inline struct btrfs_space_info *data_sinfo_for_inode(const struct btrfs_inode *inode) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + + if (btrfs_is_zoned(fs_info) && btrfs_is_data_reloc_root(inode->root)) { + ASSERT(fs_info->data_sinfo->sub_group[0]->subgroup_id == + BTRFS_SUB_GROUP_DATA_RELOC); + return fs_info->data_sinfo->sub_group[0]; + } + return fs_info->data_sinfo; +} + +int btrfs_alloc_data_chunk_ondemand(const struct btrfs_inode *inode, u64 bytes) +{ + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_FLUSH_DATA; + + /* Make sure bytes are sectorsize aligned */ + bytes = ALIGN(bytes, fs_info->sectorsize); + + if (btrfs_is_free_space_inode(inode)) + flush = BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE; + else if (btrfs_is_zoned(fs_info) && btrfs_is_data_reloc_root(root)) + flush = BTRFS_RESERVE_FLUSH_ZONED_RELOCATION; + + return btrfs_reserve_data_bytes(data_sinfo_for_inode(inode), bytes, flush); +} + +int btrfs_check_data_free_space(struct btrfs_inode *inode, + struct extent_changeset **reserved, u64 start, + u64 len, bool noflush) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_FLUSH_DATA; + int ret; + + /* align the range */ + len = round_up(start + len, fs_info->sectorsize) - + round_down(start, fs_info->sectorsize); + start = round_down(start, fs_info->sectorsize); + + if (noflush) + flush = BTRFS_RESERVE_NO_FLUSH; + else if (btrfs_is_free_space_inode(inode)) + flush = BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE; + + ret = btrfs_reserve_data_bytes(data_sinfo_for_inode(inode), len, flush); + if (ret < 0) + return ret; + + /* Use new btrfs_qgroup_reserve_data to reserve precious data space. */ + ret = btrfs_qgroup_reserve_data(inode, reserved, start, len); + if (ret < 0) { + btrfs_free_reserved_data_space_noquota(inode, len); + extent_changeset_free(*reserved); + *reserved = NULL; + } else { + ret = 0; + } + return ret; +} + +/* + * Called if we need to clear a data reservation for this inode + * Normally in a error case. + * + * This one will *NOT* use accurate qgroup reserved space API, just for case + * which we can't sleep and is sure it won't affect qgroup reserved space. + * Like clear_bit_hook(). + */ +void btrfs_free_reserved_data_space_noquota(struct btrfs_inode *inode, u64 len) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + + ASSERT(IS_ALIGNED(len, fs_info->sectorsize)); + + btrfs_space_info_free_bytes_may_use(data_sinfo_for_inode(inode), len); +} + +/* + * Called if we need to clear a data reservation for this inode + * Normally in a error case. + * + * This one will handle the per-inode data rsv map for accurate reserved + * space framework. + */ +void btrfs_free_reserved_data_space(struct btrfs_inode *inode, + struct extent_changeset *reserved, u64 start, u64 len) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + + /* Make sure the range is aligned to sectorsize */ + len = round_up(start + len, fs_info->sectorsize) - + round_down(start, fs_info->sectorsize); + start = round_down(start, fs_info->sectorsize); + + btrfs_free_reserved_data_space_noquota(inode, len); + btrfs_qgroup_free_data(inode, reserved, start, len, NULL); +} + +/* + * Release any excessive reservations for an inode. + * + * @inode: the inode we need to release from + * @qgroup_free: free or convert qgroup meta. Unlike normal operation, qgroup + * meta reservation needs to know if we are freeing qgroup + * reservation or just converting it into per-trans. Normally + * @qgroup_free is true for error handling, and false for normal + * release. + * + * This is the same as btrfs_block_rsv_release, except that it handles the + * tracepoint for the reservation. + */ +static void btrfs_inode_rsv_release(struct btrfs_inode *inode, bool qgroup_free) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct btrfs_block_rsv *block_rsv = &inode->block_rsv; + u64 released = 0; + u64 qgroup_to_release = 0; + + /* + * Since we statically set the block_rsv->size we just want to say we + * are releasing 0 bytes, and then we'll just get the reservation over + * the size free'd. + */ + released = btrfs_block_rsv_release(fs_info, block_rsv, 0, + &qgroup_to_release); + if (released > 0) + trace_btrfs_space_reservation(fs_info, "delalloc", + btrfs_ino(inode), released, 0); + if (qgroup_free) + btrfs_qgroup_free_meta_prealloc(inode->root, qgroup_to_release); + else + btrfs_qgroup_convert_reserved_meta(inode->root, + qgroup_to_release); +} + +static void btrfs_calculate_inode_block_rsv_size(struct btrfs_fs_info *fs_info, + struct btrfs_inode *inode) +{ + struct btrfs_block_rsv *block_rsv = &inode->block_rsv; + u64 reserve_size = 0; + u64 qgroup_rsv_size = 0; + unsigned outstanding_extents; + + lockdep_assert_held(&inode->lock); + outstanding_extents = inode->outstanding_extents; + + /* + * Insert size for the number of outstanding extents, 1 normal size for + * updating the inode. + */ + if (outstanding_extents) { + reserve_size = btrfs_calc_insert_metadata_size(fs_info, + outstanding_extents); + reserve_size += btrfs_calc_metadata_size(fs_info, 1); + } + if (!(inode->flags & BTRFS_INODE_NODATASUM)) { + u64 csum_leaves; + + csum_leaves = btrfs_csum_bytes_to_leaves(fs_info, inode->csum_bytes); + reserve_size += btrfs_calc_insert_metadata_size(fs_info, csum_leaves); + } + /* + * For qgroup rsv, the calculation is very simple: + * account one nodesize for each outstanding extent + * + * This is overestimating in most cases. + */ + qgroup_rsv_size = ((u64)outstanding_extents << fs_info->nodesize_bits); + + spin_lock(&block_rsv->lock); + block_rsv->size = reserve_size; + block_rsv->qgroup_rsv_size = qgroup_rsv_size; + spin_unlock(&block_rsv->lock); +} + +static void calc_inode_reservations(struct btrfs_inode *inode, + u64 num_bytes, u64 disk_num_bytes, + u64 *meta_reserve, u64 *qgroup_reserve) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + u64 nr_extents = count_max_extents(fs_info, num_bytes); + u64 csum_leaves; + u64 inode_update = btrfs_calc_metadata_size(fs_info, 1); + + if (inode->flags & BTRFS_INODE_NODATASUM) + csum_leaves = 0; + else + csum_leaves = btrfs_csum_bytes_to_leaves(fs_info, disk_num_bytes); + + *meta_reserve = btrfs_calc_insert_metadata_size(fs_info, + nr_extents + csum_leaves); + + /* + * finish_ordered_io has to update the inode, so add the space required + * for an inode update. + */ + *meta_reserve += inode_update; + *qgroup_reserve = (nr_extents << fs_info->nodesize_bits); +} + +int btrfs_delalloc_reserve_metadata(struct btrfs_inode *inode, u64 num_bytes, + u64 disk_num_bytes, bool noflush) +{ + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_block_rsv *block_rsv = &inode->block_rsv; + u64 meta_reserve, qgroup_reserve; + unsigned nr_extents; + enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_FLUSH_ALL; + int ret = 0; + + /* + * If we are a free space inode we need to not flush since we will be in + * the middle of a transaction commit. We also don't need the delalloc + * mutex since we won't race with anybody. We need this mostly to make + * lockdep shut its filthy mouth. + * + * If we have a transaction open (can happen if we call truncate_block + * from truncate), then we need FLUSH_LIMIT so we don't deadlock. + */ + if (noflush || btrfs_is_free_space_inode(inode)) { + flush = BTRFS_RESERVE_NO_FLUSH; + } else { + if (current->journal_info) + flush = BTRFS_RESERVE_FLUSH_LIMIT; + } + + num_bytes = ALIGN(num_bytes, fs_info->sectorsize); + disk_num_bytes = ALIGN(disk_num_bytes, fs_info->sectorsize); + + /* + * We always want to do it this way, every other way is wrong and ends + * in tears. Pre-reserving the amount we are going to add will always + * be the right way, because otherwise if we have enough parallelism we + * could end up with thousands of inodes all holding little bits of + * reservations they were able to make previously and the only way to + * reclaim that space is to ENOSPC out the operations and clear + * everything out and try again, which is bad. This way we just + * over-reserve slightly, and clean up the mess when we are done. + */ + calc_inode_reservations(inode, num_bytes, disk_num_bytes, + &meta_reserve, &qgroup_reserve); + ret = btrfs_qgroup_reserve_meta_prealloc(root, qgroup_reserve, true, + noflush); + if (ret) + return ret; + ret = btrfs_reserve_metadata_bytes(block_rsv->space_info, meta_reserve, + flush); + if (ret) { + btrfs_qgroup_free_meta_prealloc(root, qgroup_reserve); + return ret; + } + + /* + * Now we need to update our outstanding extents and csum bytes _first_ + * and then add the reservation to the block_rsv. This keeps us from + * racing with an ordered completion or some such that would think it + * needs to free the reservation we just made. + */ + nr_extents = count_max_extents(fs_info, num_bytes); + spin_lock(&inode->lock); + btrfs_mod_outstanding_extents(inode, nr_extents); + if (!(inode->flags & BTRFS_INODE_NODATASUM)) + inode->csum_bytes += disk_num_bytes; + btrfs_calculate_inode_block_rsv_size(fs_info, inode); + spin_unlock(&inode->lock); + + /* Now we can safely add our space to our block rsv */ + btrfs_block_rsv_add_bytes(block_rsv, meta_reserve, false); + trace_btrfs_space_reservation(root->fs_info, "delalloc", + btrfs_ino(inode), meta_reserve, 1); + + spin_lock(&block_rsv->lock); + block_rsv->qgroup_rsv_reserved += qgroup_reserve; + spin_unlock(&block_rsv->lock); + + return 0; +} + +/* + * Release a metadata reservation for an inode. + * + * @inode: the inode to release the reservation for. + * @num_bytes: the number of bytes we are releasing. + * @qgroup_free: free qgroup reservation or convert it to per-trans reservation + * + * This will release the metadata reservation for an inode. This can be called + * once we complete IO for a given set of bytes to release their metadata + * reservations, or on error for the same reason. + */ +void btrfs_delalloc_release_metadata(struct btrfs_inode *inode, u64 num_bytes, + bool qgroup_free) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + + num_bytes = ALIGN(num_bytes, fs_info->sectorsize); + spin_lock(&inode->lock); + if (!(inode->flags & BTRFS_INODE_NODATASUM)) + inode->csum_bytes -= num_bytes; + btrfs_calculate_inode_block_rsv_size(fs_info, inode); + spin_unlock(&inode->lock); + + if (btrfs_is_testing(fs_info)) + return; + + btrfs_inode_rsv_release(inode, qgroup_free); +} + +/* + * Release our outstanding_extents for an inode. + * + * @inode: the inode to balance the reservation for. + * @num_bytes: the number of bytes we originally reserved with + * + * When we reserve space we increase outstanding_extents for the extents we may + * add. Once we've set the range as delalloc or created our ordered extents we + * have outstanding_extents to track the real usage, so we use this to free our + * temporarily tracked outstanding_extents. This _must_ be used in conjunction + * with btrfs_delalloc_reserve_metadata. + */ +void btrfs_delalloc_release_extents(struct btrfs_inode *inode, u64 num_bytes) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + unsigned num_extents; + + spin_lock(&inode->lock); + num_extents = count_max_extents(fs_info, num_bytes); + btrfs_mod_outstanding_extents(inode, -num_extents); + btrfs_calculate_inode_block_rsv_size(fs_info, inode); + spin_unlock(&inode->lock); + + if (btrfs_is_testing(fs_info)) + return; + + btrfs_inode_rsv_release(inode, true); +} + +/* Shrink a previously reserved extent to a new length. */ +void btrfs_delalloc_shrink_extents(struct btrfs_inode *inode, u64 reserved_len, u64 new_len) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + const u32 reserved_num_extents = count_max_extents(fs_info, reserved_len); + const u32 new_num_extents = count_max_extents(fs_info, new_len); + const int diff_num_extents = new_num_extents - reserved_num_extents; + + ASSERT(new_len <= reserved_len); + if (new_num_extents == reserved_num_extents) + return; + + spin_lock(&inode->lock); + btrfs_mod_outstanding_extents(inode, diff_num_extents); + btrfs_calculate_inode_block_rsv_size(fs_info, inode); + spin_unlock(&inode->lock); + + if (btrfs_is_testing(fs_info)) + return; + + btrfs_inode_rsv_release(inode, true); +} + +/* + * Reserve data and metadata space for delalloc + * + * @inode: inode we're writing to + * @start: start range we are writing to + * @len: how long the range we are writing to + * @reserved: mandatory parameter, record actually reserved qgroup ranges of + * current reservation. + * + * This will do the following things + * + * - reserve space in data space info for num bytes and reserve precious + * corresponding qgroup space + * (Done in check_data_free_space) + * + * - reserve space for metadata space, based on the number of outstanding + * extents and how much csums will be needed also reserve metadata space in a + * per root over-reserve method. + * - add to the inodes->delalloc_bytes + * - add it to the fs_info's delalloc inodes list. + * (Above 3 all done in delalloc_reserve_metadata) + * + * Return 0 for success + * Return <0 for error(-ENOSPC or -EDQUOT) + */ +int btrfs_delalloc_reserve_space(struct btrfs_inode *inode, + struct extent_changeset **reserved, u64 start, u64 len) +{ + int ret; + + ret = btrfs_check_data_free_space(inode, reserved, start, len, false); + if (ret < 0) + return ret; + ret = btrfs_delalloc_reserve_metadata(inode, len, len, false); + if (ret < 0) { + btrfs_free_reserved_data_space(inode, *reserved, start, len); + extent_changeset_free(*reserved); + *reserved = NULL; + } + return ret; +} + +/* + * Release data and metadata space for delalloc + * + * @inode: inode we're releasing space for + * @reserved: list of changed/reserved ranges + * @start: start position of the space already reserved + * @len: length of the space already reserved + * @qgroup_free: should qgroup reserved-space also be freed + * + * Release the metadata space that was not used and will decrement + * ->delalloc_bytes and remove it from the fs_info->delalloc_inodes list if + * there are no delalloc bytes left. Also it will handle the qgroup reserved + * space. + */ +void btrfs_delalloc_release_space(struct btrfs_inode *inode, + struct extent_changeset *reserved, + u64 start, u64 len, bool qgroup_free) +{ + btrfs_delalloc_release_metadata(inode, len, qgroup_free); + btrfs_free_reserved_data_space(inode, reserved, start, len); +} diff --git a/fs/btrfs/delalloc-space.h b/fs/btrfs/delalloc-space.h new file mode 100644 index 000000000..6119c0d3f --- /dev/null +++ b/fs/btrfs/delalloc-space.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_DELALLOC_SPACE_H +#define BTRFS_DELALLOC_SPACE_H + +#include + +struct extent_changeset; +struct btrfs_inode; +struct btrfs_fs_info; + +int btrfs_alloc_data_chunk_ondemand(const struct btrfs_inode *inode, u64 bytes); +int btrfs_check_data_free_space(struct btrfs_inode *inode, + struct extent_changeset **reserved, u64 start, u64 len, + bool noflush); +void btrfs_free_reserved_data_space(struct btrfs_inode *inode, + struct extent_changeset *reserved, u64 start, u64 len); +void btrfs_delalloc_release_space(struct btrfs_inode *inode, + struct extent_changeset *reserved, + u64 start, u64 len, bool qgroup_free); +void btrfs_free_reserved_data_space_noquota(struct btrfs_inode *inode, u64 len); +void btrfs_delalloc_release_metadata(struct btrfs_inode *inode, u64 num_bytes, + bool qgroup_free); +int btrfs_delalloc_reserve_space(struct btrfs_inode *inode, + struct extent_changeset **reserved, u64 start, u64 len); +int btrfs_delalloc_reserve_metadata(struct btrfs_inode *inode, u64 num_bytes, + u64 disk_num_bytes, bool noflush); +void btrfs_delalloc_release_extents(struct btrfs_inode *inode, u64 num_bytes); +void btrfs_delalloc_shrink_extents(struct btrfs_inode *inode, u64 reserved_len, u64 new_len); + +#endif /* BTRFS_DELALLOC_SPACE_H */ diff --git a/fs/btrfs/delayed-inode.c b/fs/btrfs/delayed-inode.c new file mode 100644 index 000000000..db2ffab09 --- /dev/null +++ b/fs/btrfs/delayed-inode.c @@ -0,0 +1,2228 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2011 Fujitsu. All rights reserved. + * Written by Miao Xie + */ + +#include +#include +#include "ctree.h" +#include "fs.h" +#include "messages.h" +#include "misc.h" +#include "delayed-inode.h" +#include "disk-io.h" +#include "transaction.h" +#include "qgroup.h" +#include "locking.h" +#include "inode-item.h" +#include "space-info.h" +#include "accessors.h" +#include "file-item.h" + +#define BTRFS_DELAYED_WRITEBACK 512 +#define BTRFS_DELAYED_BACKGROUND 128 +#define BTRFS_DELAYED_BATCH 16 + +static struct kmem_cache *delayed_node_cache; + +int __init btrfs_delayed_inode_init(void) +{ + delayed_node_cache = KMEM_CACHE(btrfs_delayed_node, 0); + if (!delayed_node_cache) + return -ENOMEM; + return 0; +} + +void __cold btrfs_delayed_inode_exit(void) +{ + kmem_cache_destroy(delayed_node_cache); +} + +void btrfs_init_delayed_root(struct btrfs_delayed_root *delayed_root) +{ + atomic_set(&delayed_root->items, 0); + atomic_set(&delayed_root->items_seq, 0); + delayed_root->nodes = 0; + spin_lock_init(&delayed_root->lock); + init_waitqueue_head(&delayed_root->wait); + INIT_LIST_HEAD(&delayed_root->node_list); + INIT_LIST_HEAD(&delayed_root->prepare_list); +} + +static inline void btrfs_init_delayed_node( + struct btrfs_delayed_node *delayed_node, + struct btrfs_root *root, u64 inode_id) +{ + delayed_node->root = root; + delayed_node->inode_id = inode_id; + refcount_set(&delayed_node->refs, 0); + btrfs_delayed_node_ref_tracker_dir_init(delayed_node); + delayed_node->ins_root = RB_ROOT_CACHED; + delayed_node->del_root = RB_ROOT_CACHED; + mutex_init(&delayed_node->mutex); + INIT_LIST_HEAD(&delayed_node->n_list); + INIT_LIST_HEAD(&delayed_node->p_list); +} + +static struct btrfs_delayed_node *btrfs_get_delayed_node( + struct btrfs_inode *btrfs_inode, + struct btrfs_ref_tracker *tracker) +{ + struct btrfs_root *root = btrfs_inode->root; + u64 ino = btrfs_ino(btrfs_inode); + struct btrfs_delayed_node *node; + + node = READ_ONCE(btrfs_inode->delayed_node); + if (node) { + refcount_inc(&node->refs); + btrfs_delayed_node_ref_tracker_alloc(node, tracker, GFP_NOFS); + return node; + } + + xa_lock(&root->delayed_nodes); + node = xa_load(&root->delayed_nodes, ino); + + if (node) { + if (btrfs_inode->delayed_node) { + refcount_inc(&node->refs); /* can be accessed */ + btrfs_delayed_node_ref_tracker_alloc(node, tracker, GFP_ATOMIC); + BUG_ON(btrfs_inode->delayed_node != node); + xa_unlock(&root->delayed_nodes); + return node; + } + + /* + * It's possible that we're racing into the middle of removing + * this node from the xarray. In this case, the refcount + * was zero and it should never go back to one. Just return + * NULL like it was never in the xarray at all; our release + * function is in the process of removing it. + * + * Some implementations of refcount_inc refuse to bump the + * refcount once it has hit zero. If we don't do this dance + * here, refcount_inc() may decide to just WARN_ONCE() instead + * of actually bumping the refcount. + * + * If this node is properly in the xarray, we want to bump the + * refcount twice, once for the inode and once for this get + * operation. + */ + if (refcount_inc_not_zero(&node->refs)) { + refcount_inc(&node->refs); + btrfs_delayed_node_ref_tracker_alloc(node, tracker, GFP_ATOMIC); + btrfs_delayed_node_ref_tracker_alloc(node, &node->inode_cache_tracker, + GFP_ATOMIC); + btrfs_inode->delayed_node = node; + } else { + node = NULL; + } + + xa_unlock(&root->delayed_nodes); + return node; + } + xa_unlock(&root->delayed_nodes); + + return NULL; +} + +/* + * Look up an existing delayed node associated with @btrfs_inode or create a new + * one and insert it to the delayed nodes of the root. + * + * Return the delayed node, or error pointer on failure. + */ +static struct btrfs_delayed_node *btrfs_get_or_create_delayed_node( + struct btrfs_inode *btrfs_inode, + struct btrfs_ref_tracker *tracker) +{ + struct btrfs_delayed_node *node; + struct btrfs_root *root = btrfs_inode->root; + u64 ino = btrfs_ino(btrfs_inode); + int ret; + void *ptr; + +again: + node = btrfs_get_delayed_node(btrfs_inode, tracker); + if (node) + return node; + + node = kmem_cache_zalloc(delayed_node_cache, GFP_NOFS); + if (!node) + return ERR_PTR(-ENOMEM); + btrfs_init_delayed_node(node, root, ino); + + /* Cached in the inode and can be accessed. */ + refcount_set(&node->refs, 2); + btrfs_delayed_node_ref_tracker_alloc(node, tracker, GFP_NOFS); + btrfs_delayed_node_ref_tracker_alloc(node, &node->inode_cache_tracker, GFP_NOFS); + + /* Allocate and reserve the slot, from now it can return a NULL from xa_load(). */ + ret = xa_reserve(&root->delayed_nodes, ino, GFP_NOFS); + if (ret == -ENOMEM) + goto cleanup; + + xa_lock(&root->delayed_nodes); + ptr = xa_load(&root->delayed_nodes, ino); + if (ptr) { + /* Somebody inserted it, go back and read it. */ + xa_unlock(&root->delayed_nodes); + goto cleanup; + } + ptr = __xa_store(&root->delayed_nodes, ino, node, GFP_ATOMIC); + ASSERT(xa_err(ptr) != -EINVAL); + ASSERT(xa_err(ptr) != -ENOMEM); + ASSERT(ptr == NULL); + btrfs_inode->delayed_node = node; + xa_unlock(&root->delayed_nodes); + + return node; +cleanup: + btrfs_delayed_node_ref_tracker_free(node, tracker); + btrfs_delayed_node_ref_tracker_free(node, &node->inode_cache_tracker); + btrfs_delayed_node_ref_tracker_dir_exit(node); + kmem_cache_free(delayed_node_cache, node); + if (ret) + return ERR_PTR(ret); + goto again; +} + +/* + * Call it when holding delayed_node->mutex + * + * If mod = 1, add this node into the prepared list. + */ +static void btrfs_queue_delayed_node(struct btrfs_delayed_root *root, + struct btrfs_delayed_node *node, + int mod) +{ + spin_lock(&root->lock); + if (test_bit(BTRFS_DELAYED_NODE_IN_LIST, &node->flags)) { + if (!list_empty(&node->p_list)) + list_move_tail(&node->p_list, &root->prepare_list); + else if (mod) + list_add_tail(&node->p_list, &root->prepare_list); + } else { + list_add_tail(&node->n_list, &root->node_list); + list_add_tail(&node->p_list, &root->prepare_list); + refcount_inc(&node->refs); /* inserted into list */ + btrfs_delayed_node_ref_tracker_alloc(node, &node->node_list_tracker, + GFP_ATOMIC); + root->nodes++; + set_bit(BTRFS_DELAYED_NODE_IN_LIST, &node->flags); + } + spin_unlock(&root->lock); +} + +/* Call it when holding delayed_node->mutex */ +static void btrfs_dequeue_delayed_node(struct btrfs_delayed_root *root, + struct btrfs_delayed_node *node) +{ + spin_lock(&root->lock); + if (test_bit(BTRFS_DELAYED_NODE_IN_LIST, &node->flags)) { + root->nodes--; + btrfs_delayed_node_ref_tracker_free(node, &node->node_list_tracker); + refcount_dec(&node->refs); /* not in the list */ + list_del_init(&node->n_list); + if (!list_empty(&node->p_list)) + list_del_init(&node->p_list); + clear_bit(BTRFS_DELAYED_NODE_IN_LIST, &node->flags); + } + spin_unlock(&root->lock); +} + +static struct btrfs_delayed_node *btrfs_first_delayed_node( + struct btrfs_fs_info *fs_info, + struct btrfs_ref_tracker *tracker) +{ + struct btrfs_delayed_node *node; + + spin_lock(&fs_info->delayed_root.lock); + node = list_first_entry_or_null(&fs_info->delayed_root.node_list, + struct btrfs_delayed_node, n_list); + if (node) { + refcount_inc(&node->refs); + btrfs_delayed_node_ref_tracker_alloc(node, tracker, GFP_ATOMIC); + } + spin_unlock(&fs_info->delayed_root.lock); + + return node; +} + +static struct btrfs_delayed_node *btrfs_next_delayed_node( + struct btrfs_delayed_node *node, + struct btrfs_ref_tracker *tracker) +{ + struct btrfs_delayed_root *delayed_root; + struct list_head *p; + struct btrfs_delayed_node *next = NULL; + + delayed_root = &node->root->fs_info->delayed_root; + spin_lock(&delayed_root->lock); + if (!test_bit(BTRFS_DELAYED_NODE_IN_LIST, &node->flags)) { + /* not in the list */ + if (list_empty(&delayed_root->node_list)) + goto out; + p = delayed_root->node_list.next; + } else if (list_is_last(&node->n_list, &delayed_root->node_list)) + goto out; + else + p = node->n_list.next; + + next = list_entry(p, struct btrfs_delayed_node, n_list); + refcount_inc(&next->refs); + btrfs_delayed_node_ref_tracker_alloc(next, tracker, GFP_ATOMIC); +out: + spin_unlock(&delayed_root->lock); + + return next; +} + +static void __btrfs_release_delayed_node( + struct btrfs_delayed_node *delayed_node, + int mod, struct btrfs_ref_tracker *tracker) +{ + struct btrfs_delayed_root *delayed_root; + + if (!delayed_node) + return; + + delayed_root = &delayed_node->root->fs_info->delayed_root; + + mutex_lock(&delayed_node->mutex); + if (delayed_node->count) + btrfs_queue_delayed_node(delayed_root, delayed_node, mod); + else + btrfs_dequeue_delayed_node(delayed_root, delayed_node); + mutex_unlock(&delayed_node->mutex); + + btrfs_delayed_node_ref_tracker_free(delayed_node, tracker); + if (refcount_dec_and_test(&delayed_node->refs)) { + struct btrfs_root *root = delayed_node->root; + + xa_erase(&root->delayed_nodes, delayed_node->inode_id); + /* + * Once our refcount goes to zero, nobody is allowed to bump it + * back up. We can delete it now. + */ + ASSERT(refcount_read(&delayed_node->refs) == 0); + btrfs_delayed_node_ref_tracker_dir_exit(delayed_node); + kmem_cache_free(delayed_node_cache, delayed_node); + } +} + +static inline void btrfs_release_delayed_node(struct btrfs_delayed_node *node, + struct btrfs_ref_tracker *tracker) +{ + __btrfs_release_delayed_node(node, 0, tracker); +} + +static struct btrfs_delayed_node *btrfs_first_prepared_delayed_node( + struct btrfs_delayed_root *delayed_root, + struct btrfs_ref_tracker *tracker) +{ + struct btrfs_delayed_node *node; + + spin_lock(&delayed_root->lock); + node = list_first_entry_or_null(&delayed_root->prepare_list, + struct btrfs_delayed_node, p_list); + if (node) { + list_del_init(&node->p_list); + refcount_inc(&node->refs); + btrfs_delayed_node_ref_tracker_alloc(node, tracker, GFP_ATOMIC); + } + spin_unlock(&delayed_root->lock); + + return node; +} + +static inline void btrfs_release_prepared_delayed_node( + struct btrfs_delayed_node *node, + struct btrfs_ref_tracker *tracker) +{ + __btrfs_release_delayed_node(node, 1, tracker); +} + +static struct btrfs_delayed_item *btrfs_alloc_delayed_item(u16 data_len, + struct btrfs_delayed_node *node, + enum btrfs_delayed_item_type type) +{ + struct btrfs_delayed_item *item; + + item = kmalloc_flex(*item, data, data_len, GFP_NOFS); + if (item) { + item->data_len = data_len; + item->type = type; + item->bytes_reserved = 0; + item->delayed_node = node; + RB_CLEAR_NODE(&item->rb_node); + INIT_LIST_HEAD(&item->log_list); + item->logged = false; + refcount_set(&item->refs, 1); + } + return item; +} + +static int delayed_item_index_cmp(const void *key, const struct rb_node *node) +{ + const u64 *index = key; + const struct btrfs_delayed_item *delayed_item = rb_entry(node, + struct btrfs_delayed_item, rb_node); + + if (delayed_item->index < *index) + return 1; + else if (delayed_item->index > *index) + return -1; + + return 0; +} + +/* + * Look up the delayed item by key. + * + * @delayed_node: pointer to the delayed node + * @index: the dir index value to lookup (offset of a dir index key) + * + * Note: if we don't find the right item, we will return the prev item and + * the next item. + */ +static struct btrfs_delayed_item *__btrfs_lookup_delayed_item( + struct rb_root *root, + u64 index) +{ + struct rb_node *node; + + node = rb_find(&index, root, delayed_item_index_cmp); + return rb_entry_safe(node, struct btrfs_delayed_item, rb_node); +} + +static int btrfs_delayed_item_cmp(const struct rb_node *new, + const struct rb_node *exist) +{ + const struct btrfs_delayed_item *new_item = + rb_entry(new, struct btrfs_delayed_item, rb_node); + + return delayed_item_index_cmp(&new_item->index, exist); +} + +static int __btrfs_add_delayed_item(struct btrfs_delayed_node *delayed_node, + struct btrfs_delayed_item *ins) +{ + struct rb_root_cached *root; + struct rb_node *exist; + + if (ins->type == BTRFS_DELAYED_INSERTION_ITEM) + root = &delayed_node->ins_root; + else + root = &delayed_node->del_root; + + exist = rb_find_add_cached(&ins->rb_node, root, btrfs_delayed_item_cmp); + if (exist) + return -EEXIST; + + if (ins->type == BTRFS_DELAYED_INSERTION_ITEM && + ins->index >= delayed_node->index_cnt) + delayed_node->index_cnt = ins->index + 1; + + delayed_node->count++; + atomic_inc(&delayed_node->root->fs_info->delayed_root.items); + return 0; +} + +static void finish_one_item(struct btrfs_delayed_root *delayed_root) +{ + int seq = atomic_inc_return(&delayed_root->items_seq); + + /* atomic_dec_return implies a barrier */ + if ((atomic_dec_return(&delayed_root->items) < + BTRFS_DELAYED_BACKGROUND || seq % BTRFS_DELAYED_BATCH == 0)) + cond_wake_up_nomb(&delayed_root->wait); +} + +static void __btrfs_remove_delayed_item(struct btrfs_delayed_item *delayed_item) +{ + struct btrfs_delayed_node *delayed_node = delayed_item->delayed_node; + struct rb_root_cached *root; + + /* Not inserted, ignore it. */ + if (RB_EMPTY_NODE(&delayed_item->rb_node)) + return; + + /* If it's in a rbtree, then we need to have delayed node locked. */ + lockdep_assert_held(&delayed_node->mutex); + + if (delayed_item->type == BTRFS_DELAYED_INSERTION_ITEM) + root = &delayed_node->ins_root; + else + root = &delayed_node->del_root; + + rb_erase_cached(&delayed_item->rb_node, root); + RB_CLEAR_NODE(&delayed_item->rb_node); + delayed_node->count--; + finish_one_item(&delayed_node->root->fs_info->delayed_root); +} + +static void btrfs_release_delayed_item(struct btrfs_delayed_item *item) +{ + if (item) { + __btrfs_remove_delayed_item(item); + if (refcount_dec_and_test(&item->refs)) + kfree(item); + } +} + +static struct btrfs_delayed_item *__btrfs_first_delayed_insertion_item( + struct btrfs_delayed_node *delayed_node) +{ + struct rb_node *p = rb_first_cached(&delayed_node->ins_root); + + return rb_entry_safe(p, struct btrfs_delayed_item, rb_node); +} + +static struct btrfs_delayed_item *__btrfs_first_delayed_deletion_item( + struct btrfs_delayed_node *delayed_node) +{ + struct rb_node *p = rb_first_cached(&delayed_node->del_root); + + return rb_entry_safe(p, struct btrfs_delayed_item, rb_node); +} + +static struct btrfs_delayed_item *__btrfs_next_delayed_item( + struct btrfs_delayed_item *item) +{ + struct rb_node *p = rb_next(&item->rb_node); + + return rb_entry_safe(p, struct btrfs_delayed_item, rb_node); +} + +static int btrfs_delayed_item_reserve_metadata(struct btrfs_trans_handle *trans, + struct btrfs_delayed_item *item) +{ + struct btrfs_block_rsv *src_rsv; + struct btrfs_block_rsv *dst_rsv; + struct btrfs_fs_info *fs_info = trans->fs_info; + u64 num_bytes; + int ret; + + if (!trans->bytes_reserved) + return 0; + + src_rsv = trans->block_rsv; + dst_rsv = &fs_info->delayed_block_rsv; + + num_bytes = btrfs_calc_insert_metadata_size(fs_info, 1); + + /* + * Here we migrate space rsv from transaction rsv, since have already + * reserved space when starting a transaction. So no need to reserve + * qgroup space here. + */ + ret = btrfs_block_rsv_migrate(src_rsv, dst_rsv, num_bytes, true); + if (!ret) { + trace_btrfs_space_reservation(fs_info, "delayed_item", + item->delayed_node->inode_id, + num_bytes, 1); + /* + * For insertions we track reserved metadata space by accounting + * for the number of leaves that will be used, based on the delayed + * node's curr_index_batch_size and index_item_leaves fields. + */ + if (item->type == BTRFS_DELAYED_DELETION_ITEM) + item->bytes_reserved = num_bytes; + } + + return ret; +} + +static void btrfs_delayed_item_release_metadata(struct btrfs_root *root, + struct btrfs_delayed_item *item) +{ + struct btrfs_block_rsv *rsv; + struct btrfs_fs_info *fs_info = root->fs_info; + + if (!item->bytes_reserved) + return; + + rsv = &fs_info->delayed_block_rsv; + /* + * Check btrfs_delayed_item_reserve_metadata() to see why we don't need + * to release/reserve qgroup space. + */ + trace_btrfs_space_reservation(fs_info, "delayed_item", + item->delayed_node->inode_id, + item->bytes_reserved, 0); + btrfs_block_rsv_release(fs_info, rsv, item->bytes_reserved, NULL); +} + +static void btrfs_delayed_item_release_leaves(struct btrfs_delayed_node *node, + unsigned int num_leaves) +{ + struct btrfs_fs_info *fs_info = node->root->fs_info; + const u64 bytes = btrfs_calc_insert_metadata_size(fs_info, num_leaves); + + /* There are no space reservations during log replay, bail out. */ + if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) + return; + + trace_btrfs_space_reservation(fs_info, "delayed_item", node->inode_id, + bytes, 0); + btrfs_block_rsv_release(fs_info, &fs_info->delayed_block_rsv, bytes, NULL); +} + +static int btrfs_delayed_inode_reserve_metadata( + struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_delayed_node *node) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_block_rsv *src_rsv; + struct btrfs_block_rsv *dst_rsv; + u64 num_bytes; + int ret; + + src_rsv = trans->block_rsv; + dst_rsv = &fs_info->delayed_block_rsv; + + num_bytes = btrfs_calc_metadata_size(fs_info, 1); + + /* + * btrfs_dirty_inode will update the inode under btrfs_join_transaction + * which doesn't reserve space for speed. This is a problem since we + * still need to reserve space for this update, so try to reserve the + * space. + * + * Now if src_rsv == delalloc_block_rsv we'll let it just steal since + * we always reserve enough to update the inode item. + */ + if (!src_rsv || (!trans->bytes_reserved && + src_rsv->type != BTRFS_BLOCK_RSV_DELALLOC)) { + ret = btrfs_qgroup_reserve_meta_prealloc(root, num_bytes, true, true); + if (ret < 0) + return ret; + ret = btrfs_block_rsv_add(fs_info, dst_rsv, num_bytes, + BTRFS_RESERVE_NO_FLUSH); + /* NO_FLUSH could only fail with -ENOSPC */ + ASSERT(ret == 0 || ret == -ENOSPC); + if (ret) + btrfs_qgroup_free_meta_prealloc(root, num_bytes); + } else { + ret = btrfs_block_rsv_migrate(src_rsv, dst_rsv, num_bytes, true); + } + + if (!ret) { + trace_btrfs_space_reservation(fs_info, "delayed_inode", + node->inode_id, num_bytes, 1); + node->bytes_reserved = num_bytes; + } + + return ret; +} + +static void btrfs_delayed_inode_release_metadata(struct btrfs_fs_info *fs_info, + struct btrfs_delayed_node *node, + bool qgroup_free) +{ + struct btrfs_block_rsv *rsv; + + if (!node->bytes_reserved) + return; + + rsv = &fs_info->delayed_block_rsv; + trace_btrfs_space_reservation(fs_info, "delayed_inode", + node->inode_id, node->bytes_reserved, 0); + btrfs_block_rsv_release(fs_info, rsv, node->bytes_reserved, NULL); + if (qgroup_free) + btrfs_qgroup_free_meta_prealloc(node->root, + node->bytes_reserved); + else + btrfs_qgroup_convert_reserved_meta(node->root, + node->bytes_reserved); + node->bytes_reserved = 0; +} + +/* + * Insert a single delayed item or a batch of delayed items, as many as possible + * that fit in a leaf. The delayed items (dir index keys) are sorted by their key + * in the rbtree, and if there's a gap between two consecutive dir index items, + * then it means at some point we had delayed dir indexes to add but they got + * removed (by btrfs_delete_delayed_dir_index()) before we attempted to flush them + * into the subvolume tree. Dir index keys also have their offsets coming from a + * monotonically increasing counter, so we can't get new keys with an offset that + * fits within a gap between delayed dir index items. + */ +static int btrfs_insert_delayed_item(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + struct btrfs_delayed_item *first_item) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_delayed_node *node = first_item->delayed_node; + LIST_HEAD(item_list); + struct btrfs_delayed_item *curr; + struct btrfs_delayed_item *next; + const int max_size = BTRFS_LEAF_DATA_SIZE(fs_info); + struct btrfs_item_batch batch; + struct btrfs_key first_key; + const u32 first_data_size = first_item->data_len; + int total_size; + char AUTO_KFREE(ins_data); + int ret; + bool continuous_keys_only = false; + + lockdep_assert_held(&node->mutex); + + /* + * During normal operation the delayed index offset is continuously + * increasing, so we can batch insert all items as there will not be any + * overlapping keys in the tree. + * + * The exception to this is log replay, where we may have interleaved + * offsets in the tree, so our batch needs to be continuous keys only in + * order to ensure we do not end up with out of order items in our leaf. + */ + if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) + continuous_keys_only = true; + + /* + * For delayed items to insert, we track reserved metadata bytes based + * on the number of leaves that we will use. + * See btrfs_insert_delayed_dir_index() and + * btrfs_delayed_item_reserve_metadata()). + */ + ASSERT(first_item->bytes_reserved == 0); + + list_add_tail(&first_item->tree_list, &item_list); + batch.total_data_size = first_data_size; + batch.nr = 1; + total_size = first_data_size + sizeof(struct btrfs_item); + curr = first_item; + + while (true) { + int next_size; + + next = __btrfs_next_delayed_item(curr); + if (!next) + break; + + /* + * We cannot allow gaps in the key space if we're doing log + * replay. + */ + if (continuous_keys_only && (next->index != curr->index + 1)) + break; + + ASSERT(next->bytes_reserved == 0); + + next_size = next->data_len + sizeof(struct btrfs_item); + if (total_size + next_size > max_size) + break; + + list_add_tail(&next->tree_list, &item_list); + batch.nr++; + total_size += next_size; + batch.total_data_size += next->data_len; + curr = next; + } + + if (batch.nr == 1) { + first_key.objectid = node->inode_id; + first_key.type = BTRFS_DIR_INDEX_KEY; + first_key.offset = first_item->index; + batch.keys = &first_key; + batch.data_sizes = &first_data_size; + } else { + struct btrfs_key *ins_keys; + u32 *ins_sizes; + int i = 0; + + ins_data = kmalloc_array(batch.nr, + sizeof(u32) + sizeof(struct btrfs_key), GFP_NOFS); + if (!ins_data) + return -ENOMEM; + ins_sizes = (u32 *)ins_data; + ins_keys = (struct btrfs_key *)(ins_data + batch.nr * sizeof(u32)); + batch.keys = ins_keys; + batch.data_sizes = ins_sizes; + list_for_each_entry(curr, &item_list, tree_list) { + ins_keys[i].objectid = node->inode_id; + ins_keys[i].type = BTRFS_DIR_INDEX_KEY; + ins_keys[i].offset = curr->index; + ins_sizes[i] = curr->data_len; + i++; + } + } + + ret = btrfs_insert_empty_items(trans, root, path, &batch); + if (ret) + return ret; + + list_for_each_entry(curr, &item_list, tree_list) { + char *data_ptr; + + data_ptr = btrfs_item_ptr(path->nodes[0], path->slots[0], char); + write_extent_buffer(path->nodes[0], &curr->data, + (unsigned long)data_ptr, curr->data_len); + path->slots[0]++; + } + + /* + * Now release our path before releasing the delayed items and their + * metadata reservations, so that we don't block other tasks for more + * time than needed. + */ + btrfs_release_path(path); + + ASSERT(node->index_item_leaves > 0); + + /* + * For normal operations we will batch an entire leaf's worth of delayed + * items, so if there are more items to process we can decrement + * index_item_leaves by 1 as we inserted 1 leaf's worth of items. + * + * However for log replay we may not have inserted an entire leaf's + * worth of items, we may have not had continuous items, so decrementing + * here would mess up the index_item_leaves accounting. For this case + * only clean up the accounting when there are no items left. + */ + if (next && !continuous_keys_only) { + /* + * We inserted one batch of items into a leaf a there are more + * items to flush in a future batch, now release one unit of + * metadata space from the delayed block reserve, corresponding + * the leaf we just flushed to. + */ + btrfs_delayed_item_release_leaves(node, 1); + node->index_item_leaves--; + } else if (!next) { + /* + * There are no more items to insert. We can have a number of + * reserved leaves > 1 here - this happens when many dir index + * items are added and then removed before they are flushed (file + * names with a very short life, never span a transaction). So + * release all remaining leaves. + */ + btrfs_delayed_item_release_leaves(node, node->index_item_leaves); + node->index_item_leaves = 0; + } + + list_for_each_entry_safe(curr, next, &item_list, tree_list) { + list_del(&curr->tree_list); + btrfs_release_delayed_item(curr); + } + + return 0; +} + +static int btrfs_insert_delayed_items(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + struct btrfs_root *root, + struct btrfs_delayed_node *node) +{ + int ret = 0; + + while (ret == 0) { + struct btrfs_delayed_item *curr; + + mutex_lock(&node->mutex); + curr = __btrfs_first_delayed_insertion_item(node); + if (!curr) { + mutex_unlock(&node->mutex); + break; + } + ret = btrfs_insert_delayed_item(trans, root, path, curr); + mutex_unlock(&node->mutex); + } + + return ret; +} + +static int btrfs_batch_delete_items(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + struct btrfs_delayed_item *item) +{ + const u64 ino = item->delayed_node->inode_id; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_delayed_item *curr, *next; + struct extent_buffer *leaf = path->nodes[0]; + LIST_HEAD(batch_list); + int nitems, slot, last_slot; + int ret; + u64 total_reserved_size = item->bytes_reserved; + + ASSERT(leaf != NULL); + + slot = path->slots[0]; + last_slot = btrfs_header_nritems(leaf) - 1; + /* + * Our caller always gives us a path pointing to an existing item, so + * this can not happen. + */ + ASSERT(slot <= last_slot); + if (WARN_ON(slot > last_slot)) + return -ENOENT; + + nitems = 1; + curr = item; + list_add_tail(&curr->tree_list, &batch_list); + + /* + * Keep checking if the next delayed item matches the next item in the + * leaf - if so, we can add it to the batch of items to delete from the + * leaf. + */ + while (slot < last_slot) { + struct btrfs_key key; + + next = __btrfs_next_delayed_item(curr); + if (!next) + break; + + slot++; + btrfs_item_key_to_cpu(leaf, &key, slot); + if (key.objectid != ino || + key.type != BTRFS_DIR_INDEX_KEY || + key.offset != next->index) + break; + nitems++; + curr = next; + list_add_tail(&curr->tree_list, &batch_list); + total_reserved_size += curr->bytes_reserved; + } + + ret = btrfs_del_items(trans, root, path, path->slots[0], nitems); + if (ret) + return ret; + + /* In case of BTRFS_FS_LOG_RECOVERING items won't have reserved space */ + if (total_reserved_size > 0) { + /* + * Check btrfs_delayed_item_reserve_metadata() to see why we + * don't need to release/reserve qgroup space. + */ + trace_btrfs_space_reservation(fs_info, "delayed_item", ino, + total_reserved_size, 0); + btrfs_block_rsv_release(fs_info, &fs_info->delayed_block_rsv, + total_reserved_size, NULL); + } + + list_for_each_entry_safe(curr, next, &batch_list, tree_list) { + list_del(&curr->tree_list); + btrfs_release_delayed_item(curr); + } + + return 0; +} + +static int btrfs_delete_delayed_items(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + struct btrfs_root *root, + struct btrfs_delayed_node *node) +{ + struct btrfs_key key; + int ret = 0; + + key.objectid = node->inode_id; + key.type = BTRFS_DIR_INDEX_KEY; + + while (ret == 0) { + struct btrfs_delayed_item *item; + + mutex_lock(&node->mutex); + item = __btrfs_first_delayed_deletion_item(node); + if (!item) { + mutex_unlock(&node->mutex); + break; + } + + key.offset = item->index; + ret = btrfs_search_slot(trans, root, &key, path, -1, 1); + if (ret > 0) { + /* + * There's no matching item in the leaf. This means we + * have already deleted this item in a past run of the + * delayed items. We ignore errors when running delayed + * items from an async context, through a work queue job + * running btrfs_async_run_delayed_root(), and don't + * release delayed items that failed to complete. This + * is because we will retry later, and at transaction + * commit time we always run delayed items and will + * then deal with errors if they fail to run again. + * + * So just release delayed items for which we can't find + * an item in the tree, and move to the next item. + */ + btrfs_release_path(path); + btrfs_release_delayed_item(item); + ret = 0; + } else if (ret == 0) { + ret = btrfs_batch_delete_items(trans, root, path, item); + btrfs_release_path(path); + } + + /* + * We unlock and relock on each iteration, this is to prevent + * blocking other tasks for too long while we are being run from + * the async context (work queue job). Those tasks are typically + * running system calls like creat/mkdir/rename/unlink/etc which + * need to add delayed items to this delayed node. + */ + mutex_unlock(&node->mutex); + } + + return ret; +} + +static void btrfs_release_delayed_inode(struct btrfs_delayed_node *delayed_node) +{ + if (delayed_node && + test_bit(BTRFS_DELAYED_NODE_INODE_DIRTY, &delayed_node->flags)) { + ASSERT(delayed_node->root); + clear_bit(BTRFS_DELAYED_NODE_INODE_DIRTY, &delayed_node->flags); + delayed_node->count--; + finish_one_item(&delayed_node->root->fs_info->delayed_root); + } +} + +static void btrfs_release_delayed_iref(struct btrfs_delayed_node *delayed_node) +{ + if (test_and_clear_bit(BTRFS_DELAYED_NODE_DEL_IREF, &delayed_node->flags)) { + ASSERT(delayed_node->root); + delayed_node->count--; + finish_one_item(&delayed_node->root->fs_info->delayed_root); + } +} + +static int __btrfs_update_delayed_inode(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + struct btrfs_delayed_node *node) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_key key; + struct btrfs_inode_item *inode_item; + struct extent_buffer *leaf; + int mod; + int ret; + + key.objectid = node->inode_id; + key.type = BTRFS_INODE_ITEM_KEY; + key.offset = 0; + + if (test_bit(BTRFS_DELAYED_NODE_DEL_IREF, &node->flags)) + mod = -1; + else + mod = 1; + + ret = btrfs_lookup_inode(trans, root, path, &key, mod); + if (ret > 0) + ret = -ENOENT; + if (ret < 0) { + /* + * If we fail to update the delayed inode we need to abort the + * transaction, because we could leave the inode with the + * improper counts behind. + */ + if (unlikely(ret != -ENOENT)) + btrfs_abort_transaction(trans, ret); + goto out; + } + + leaf = path->nodes[0]; + inode_item = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_inode_item); + write_extent_buffer(leaf, &node->inode_item, (unsigned long)inode_item, + sizeof(struct btrfs_inode_item)); + + if (!test_bit(BTRFS_DELAYED_NODE_DEL_IREF, &node->flags)) + goto out; + + /* + * Now we're going to delete the INODE_REF/EXTREF, which should be the + * only one ref left. Check if the next item is an INODE_REF/EXTREF. + * + * But if we're the last item already, release and search for the last + * INODE_REF/EXTREF. + */ + if (path->slots[0] + 1 >= btrfs_header_nritems(leaf)) { + key.objectid = node->inode_id; + key.type = BTRFS_INODE_EXTREF_KEY; + key.offset = (u64)-1; + + btrfs_release_path(path); + ret = btrfs_search_slot(trans, root, &key, path, -1, 1); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + goto err_out; + } + ASSERT(ret > 0); + ASSERT(path->slots[0] > 0); + ret = 0; + path->slots[0]--; + leaf = path->nodes[0]; + } else { + path->slots[0]++; + } + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + if (key.objectid != node->inode_id) + goto out; + if (key.type != BTRFS_INODE_REF_KEY && + key.type != BTRFS_INODE_EXTREF_KEY) + goto out; + + /* + * Delayed iref deletion is for the inode who has only one link, + * so there is only one iref. The case that several irefs are + * in the same item doesn't exist. + */ + ret = btrfs_del_item(trans, root, path); + if (ret < 0) + btrfs_abort_transaction(trans, ret); +out: + btrfs_release_delayed_iref(node); + btrfs_release_path(path); +err_out: + btrfs_delayed_inode_release_metadata(fs_info, node, (ret < 0)); + btrfs_release_delayed_inode(node); + return ret; +} + +static inline int btrfs_update_delayed_inode(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + struct btrfs_delayed_node *node) +{ + int ret; + + mutex_lock(&node->mutex); + if (!test_bit(BTRFS_DELAYED_NODE_INODE_DIRTY, &node->flags)) { + mutex_unlock(&node->mutex); + return 0; + } + + ret = __btrfs_update_delayed_inode(trans, root, path, node); + mutex_unlock(&node->mutex); + return ret; +} + +static inline int +__btrfs_commit_inode_delayed_items(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + struct btrfs_delayed_node *node) +{ + int ret; + + ret = btrfs_insert_delayed_items(trans, path, node->root, node); + if (ret) + return ret; + + ret = btrfs_delete_delayed_items(trans, path, node->root, node); + if (ret) + return ret; + + ret = btrfs_record_root_in_trans(trans, node->root); + if (ret) + return ret; + + return btrfs_update_delayed_inode(trans, node->root, path, node); +} + +/* + * Called when committing the transaction. + * Returns 0 on success. + * Returns < 0 on error and returns with an aborted transaction with any + * outstanding delayed items cleaned up. + */ +static int __btrfs_run_delayed_items(struct btrfs_trans_handle *trans, int nr) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_delayed_node *curr_node, *prev_node; + struct btrfs_ref_tracker curr_delayed_node_tracker, prev_delayed_node_tracker; + struct btrfs_path *path; + struct btrfs_block_rsv *block_rsv; + int ret = 0; + bool count = (nr > 0); + + if (TRANS_ABORTED(trans)) + return -EIO; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + block_rsv = trans->block_rsv; + trans->block_rsv = &fs_info->delayed_block_rsv; + + curr_node = btrfs_first_delayed_node(fs_info, &curr_delayed_node_tracker); + while (curr_node && (!count || nr--)) { + ret = __btrfs_commit_inode_delayed_items(trans, path, + curr_node); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + break; + } + + prev_node = curr_node; + prev_delayed_node_tracker = curr_delayed_node_tracker; + curr_node = btrfs_next_delayed_node(curr_node, &curr_delayed_node_tracker); + /* + * See the comment below about releasing path before releasing + * node. If the commit of delayed items was successful the path + * should always be released, but in case of an error, it may + * point to locked extent buffers (a leaf at the very least). + */ + ASSERT(path->nodes[0] == NULL); + btrfs_release_delayed_node(prev_node, &prev_delayed_node_tracker); + } + + /* + * Release the path to avoid a potential deadlock and lockdep splat when + * releasing the delayed node, as that requires taking the delayed node's + * mutex. If another task starts running delayed items before we take + * the mutex, it will first lock the mutex and then it may try to lock + * the same btree path (leaf). + */ + btrfs_free_path(path); + + if (curr_node) + btrfs_release_delayed_node(curr_node, &curr_delayed_node_tracker); + trans->block_rsv = block_rsv; + + return ret; +} + +int btrfs_run_delayed_items(struct btrfs_trans_handle *trans) +{ + return __btrfs_run_delayed_items(trans, -1); +} + +int btrfs_run_delayed_items_nr(struct btrfs_trans_handle *trans, int nr) +{ + return __btrfs_run_delayed_items(trans, nr); +} + +int btrfs_commit_inode_delayed_items(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode) +{ + struct btrfs_ref_tracker delayed_node_tracker; + struct btrfs_delayed_node *delayed_node = + btrfs_get_delayed_node(inode, &delayed_node_tracker); + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_block_rsv *block_rsv; + int ret; + + if (!delayed_node) + return 0; + + mutex_lock(&delayed_node->mutex); + if (!delayed_node->count) { + mutex_unlock(&delayed_node->mutex); + btrfs_release_delayed_node(delayed_node, &delayed_node_tracker); + return 0; + } + mutex_unlock(&delayed_node->mutex); + + path = btrfs_alloc_path(); + if (!path) { + btrfs_release_delayed_node(delayed_node, &delayed_node_tracker); + return -ENOMEM; + } + + block_rsv = trans->block_rsv; + trans->block_rsv = &delayed_node->root->fs_info->delayed_block_rsv; + + ret = __btrfs_commit_inode_delayed_items(trans, path, delayed_node); + + btrfs_release_delayed_node(delayed_node, &delayed_node_tracker); + trans->block_rsv = block_rsv; + + return ret; +} + +int btrfs_commit_inode_delayed_inode(struct btrfs_inode *inode) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct btrfs_trans_handle *trans; + struct btrfs_ref_tracker delayed_node_tracker; + struct btrfs_delayed_node *delayed_node; + struct btrfs_path *path; + struct btrfs_block_rsv *block_rsv; + int ret; + + delayed_node = btrfs_get_delayed_node(inode, &delayed_node_tracker); + if (!delayed_node) + return 0; + + mutex_lock(&delayed_node->mutex); + if (!test_bit(BTRFS_DELAYED_NODE_INODE_DIRTY, &delayed_node->flags)) { + mutex_unlock(&delayed_node->mutex); + btrfs_release_delayed_node(delayed_node, &delayed_node_tracker); + return 0; + } + mutex_unlock(&delayed_node->mutex); + + trans = btrfs_join_transaction(delayed_node->root); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto out; + } + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto trans_out; + } + + block_rsv = trans->block_rsv; + trans->block_rsv = &fs_info->delayed_block_rsv; + + mutex_lock(&delayed_node->mutex); + if (test_bit(BTRFS_DELAYED_NODE_INODE_DIRTY, &delayed_node->flags)) + ret = __btrfs_update_delayed_inode(trans, delayed_node->root, + path, delayed_node); + else + ret = 0; + mutex_unlock(&delayed_node->mutex); + + btrfs_free_path(path); + trans->block_rsv = block_rsv; +trans_out: + btrfs_end_transaction(trans); + btrfs_btree_balance_dirty(fs_info); +out: + btrfs_release_delayed_node(delayed_node, &delayed_node_tracker); + + return ret; +} + +void btrfs_remove_delayed_node(struct btrfs_inode *inode) +{ + struct btrfs_delayed_node *delayed_node; + + delayed_node = READ_ONCE(inode->delayed_node); + if (!delayed_node) + return; + + inode->delayed_node = NULL; + + btrfs_release_delayed_node(delayed_node, &delayed_node->inode_cache_tracker); +} + +struct btrfs_async_delayed_work { + struct btrfs_delayed_root *delayed_root; + int nr; + struct btrfs_work work; +}; + +static void btrfs_async_run_delayed_root(struct btrfs_work *work) +{ + struct btrfs_async_delayed_work *async_work; + struct btrfs_delayed_root *delayed_root; + struct btrfs_trans_handle *trans; + struct btrfs_path *path; + struct btrfs_delayed_node *delayed_node = NULL; + struct btrfs_ref_tracker delayed_node_tracker; + struct btrfs_root *root; + struct btrfs_block_rsv *block_rsv; + int total_done = 0; + + async_work = container_of(work, struct btrfs_async_delayed_work, work); + delayed_root = async_work->delayed_root; + + path = btrfs_alloc_path(); + if (!path) + goto out; + + do { + if (atomic_read(&delayed_root->items) < + BTRFS_DELAYED_BACKGROUND / 2) + break; + + delayed_node = btrfs_first_prepared_delayed_node(delayed_root, + &delayed_node_tracker); + if (!delayed_node) + break; + + root = delayed_node->root; + + trans = btrfs_join_transaction(root); + if (IS_ERR(trans)) { + btrfs_release_path(path); + btrfs_release_prepared_delayed_node(delayed_node, + &delayed_node_tracker); + total_done++; + continue; + } + + block_rsv = trans->block_rsv; + trans->block_rsv = &root->fs_info->delayed_block_rsv; + + __btrfs_commit_inode_delayed_items(trans, path, delayed_node); + + trans->block_rsv = block_rsv; + btrfs_end_transaction(trans); + btrfs_btree_balance_dirty_nodelay(root->fs_info); + + btrfs_release_path(path); + btrfs_release_prepared_delayed_node(delayed_node, + &delayed_node_tracker); + total_done++; + + } while ((async_work->nr == 0 && total_done < BTRFS_DELAYED_WRITEBACK) + || total_done < async_work->nr); + + btrfs_free_path(path); +out: + wake_up(&delayed_root->wait); + kfree(async_work); +} + + +static int btrfs_wq_run_delayed_node(struct btrfs_delayed_root *delayed_root, + struct btrfs_fs_info *fs_info, int nr) +{ + struct btrfs_async_delayed_work *async_work; + + async_work = kmalloc_obj(*async_work, GFP_NOFS); + if (!async_work) + return -ENOMEM; + + async_work->delayed_root = delayed_root; + btrfs_init_work(&async_work->work, btrfs_async_run_delayed_root, NULL); + async_work->nr = nr; + + btrfs_queue_work(fs_info->delayed_workers, &async_work->work); + return 0; +} + +void btrfs_assert_delayed_root_empty(struct btrfs_fs_info *fs_info) +{ + struct btrfs_ref_tracker delayed_node_tracker; + struct btrfs_delayed_node *node; + + node = btrfs_first_delayed_node(fs_info, &delayed_node_tracker); + if (WARN_ON(node)) { + btrfs_delayed_node_ref_tracker_free(node, + &delayed_node_tracker); + refcount_dec(&node->refs); + } +} + +static bool could_end_wait(struct btrfs_delayed_root *delayed_root, int seq) +{ + int val = atomic_read(&delayed_root->items_seq); + + if (val < seq || val >= seq + BTRFS_DELAYED_BATCH) + return true; + + if (atomic_read(&delayed_root->items) < BTRFS_DELAYED_BACKGROUND) + return true; + + return false; +} + +void btrfs_balance_delayed_items(struct btrfs_fs_info *fs_info) +{ + struct btrfs_delayed_root *delayed_root = &fs_info->delayed_root; + + if ((atomic_read(&delayed_root->items) < BTRFS_DELAYED_BACKGROUND) || + btrfs_workqueue_normal_congested(fs_info->delayed_workers)) + return; + + if (atomic_read(&delayed_root->items) >= BTRFS_DELAYED_WRITEBACK) { + int seq; + int ret; + + seq = atomic_read(&delayed_root->items_seq); + + ret = btrfs_wq_run_delayed_node(delayed_root, fs_info, 0); + if (ret) + return; + + wait_event_interruptible(delayed_root->wait, + could_end_wait(delayed_root, seq)); + return; + } + + btrfs_wq_run_delayed_node(delayed_root, fs_info, BTRFS_DELAYED_BATCH); +} + +static void btrfs_release_dir_index_item_space(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + const u64 bytes = btrfs_calc_insert_metadata_size(fs_info, 1); + + if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) + return; + + /* + * Adding the new dir index item does not require touching another + * leaf, so we can release 1 unit of metadata that was previously + * reserved when starting the transaction. This applies only to + * the case where we had a transaction start and excludes the + * transaction join case (when replaying log trees). + */ + trace_btrfs_space_reservation(fs_info, "transaction", + trans->transid, bytes, 0); + btrfs_block_rsv_release(fs_info, trans->block_rsv, bytes, NULL); + ASSERT(trans->bytes_reserved >= bytes); + trans->bytes_reserved -= bytes; +} + +/* Will return 0, -ENOMEM or -EEXIST (index number collision, unexpected). */ +int btrfs_insert_delayed_dir_index(struct btrfs_trans_handle *trans, + const char *name, int name_len, + struct btrfs_inode *dir, + const struct btrfs_disk_key *disk_key, u8 flags, + u64 index) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + const unsigned int leaf_data_size = BTRFS_LEAF_DATA_SIZE(fs_info); + struct btrfs_delayed_node *delayed_node; + struct btrfs_ref_tracker delayed_node_tracker; + struct btrfs_delayed_item *delayed_item; + struct btrfs_dir_item *dir_item; + bool reserve_leaf_space; + u32 data_len; + int ret; + + delayed_node = btrfs_get_or_create_delayed_node(dir, &delayed_node_tracker); + if (IS_ERR(delayed_node)) + return PTR_ERR(delayed_node); + + delayed_item = btrfs_alloc_delayed_item(sizeof(*dir_item) + name_len, + delayed_node, + BTRFS_DELAYED_INSERTION_ITEM); + if (!delayed_item) { + ret = -ENOMEM; + goto release_node; + } + + delayed_item->index = index; + + dir_item = (struct btrfs_dir_item *)delayed_item->data; + dir_item->location = *disk_key; + btrfs_set_stack_dir_transid(dir_item, trans->transid); + btrfs_set_stack_dir_data_len(dir_item, 0); + btrfs_set_stack_dir_name_len(dir_item, name_len); + btrfs_set_stack_dir_flags(dir_item, flags); + memcpy((char *)(dir_item + 1), name, name_len); + + data_len = delayed_item->data_len + sizeof(struct btrfs_item); + + mutex_lock(&delayed_node->mutex); + + /* + * First attempt to insert the delayed item. This is to make the error + * handling path simpler in case we fail (-EEXIST). There's no risk of + * any other task coming in and running the delayed item before we do + * the metadata space reservation below, because we are holding the + * delayed node's mutex and that mutex must also be locked before the + * node's delayed items can be run. + */ + ret = __btrfs_add_delayed_item(delayed_node, delayed_item); + if (unlikely(ret)) { + btrfs_err(trans->fs_info, +"error adding delayed dir index item, name: %.*s, index: %llu, root: %llu, dir: %llu, dir->index_cnt: %llu, delayed_node->index_cnt: %llu, error: %pe", + name_len, name, index, btrfs_root_id(delayed_node->root), + delayed_node->inode_id, dir->index_cnt, + delayed_node->index_cnt, ERR_PTR(ret)); + btrfs_release_delayed_item(delayed_item); + btrfs_release_dir_index_item_space(trans); + mutex_unlock(&delayed_node->mutex); + goto release_node; + } + + if (delayed_node->index_item_leaves == 0 || + delayed_node->curr_index_batch_size + data_len > leaf_data_size) { + delayed_node->curr_index_batch_size = data_len; + reserve_leaf_space = true; + } else { + delayed_node->curr_index_batch_size += data_len; + reserve_leaf_space = false; + } + + if (reserve_leaf_space) { + ret = btrfs_delayed_item_reserve_metadata(trans, delayed_item); + /* + * Space was reserved for a dir index item insertion when we + * started the transaction, so getting a failure here should be + * impossible. + */ + if (WARN_ON(ret)) { + btrfs_release_delayed_item(delayed_item); + mutex_unlock(&delayed_node->mutex); + goto release_node; + } + + delayed_node->index_item_leaves++; + } else { + btrfs_release_dir_index_item_space(trans); + } + mutex_unlock(&delayed_node->mutex); + +release_node: + btrfs_release_delayed_node(delayed_node, &delayed_node_tracker); + return ret; +} + +static bool btrfs_delete_delayed_insertion_item(struct btrfs_delayed_node *node, + u64 index) +{ + struct btrfs_delayed_item *item; + + mutex_lock(&node->mutex); + item = __btrfs_lookup_delayed_item(&node->ins_root.rb_root, index); + if (!item) { + mutex_unlock(&node->mutex); + return false; + } + + /* + * For delayed items to insert, we track reserved metadata bytes based + * on the number of leaves that we will use. + * See btrfs_insert_delayed_dir_index() and + * btrfs_delayed_item_reserve_metadata()). + */ + ASSERT(item->bytes_reserved == 0); + ASSERT(node->index_item_leaves > 0); + + /* + * If there's only one leaf reserved, we can decrement this item from the + * current batch, otherwise we can not because we don't know which leaf + * it belongs to. With the current limit on delayed items, we rarely + * accumulate enough dir index items to fill more than one leaf (even + * when using a leaf size of 4K). + */ + if (node->index_item_leaves == 1) { + const u32 data_len = item->data_len + sizeof(struct btrfs_item); + + ASSERT(node->curr_index_batch_size >= data_len); + node->curr_index_batch_size -= data_len; + } + + btrfs_release_delayed_item(item); + + /* If we now have no more dir index items, we can release all leaves. */ + if (RB_EMPTY_ROOT(&node->ins_root.rb_root)) { + btrfs_delayed_item_release_leaves(node, node->index_item_leaves); + node->index_item_leaves = 0; + } + + mutex_unlock(&node->mutex); + return true; +} + +int btrfs_delete_delayed_dir_index(struct btrfs_trans_handle *trans, + struct btrfs_inode *dir, u64 index) +{ + struct btrfs_delayed_node *node; + struct btrfs_ref_tracker delayed_node_tracker; + struct btrfs_delayed_item *item; + int ret; + + node = btrfs_get_or_create_delayed_node(dir, &delayed_node_tracker); + if (IS_ERR(node)) + return PTR_ERR(node); + + if (btrfs_delete_delayed_insertion_item(node, index)) { + ret = 0; + goto end; + } + + item = btrfs_alloc_delayed_item(0, node, BTRFS_DELAYED_DELETION_ITEM); + if (!item) { + ret = -ENOMEM; + goto end; + } + + item->index = index; + + ret = btrfs_delayed_item_reserve_metadata(trans, item); + /* + * we have reserved enough space when we start a new transaction, + * so reserving metadata failure is impossible. + */ + if (ret < 0) { + btrfs_err(trans->fs_info, +"metadata reservation failed for delayed dir item deletion, index: %llu, root: %llu, inode: %llu, error: %pe", + index, btrfs_root_id(node->root), node->inode_id, ERR_PTR(ret)); + btrfs_release_delayed_item(item); + goto end; + } + + mutex_lock(&node->mutex); + ret = __btrfs_add_delayed_item(node, item); + if (unlikely(ret)) { + btrfs_err(trans->fs_info, +"failed to add delayed dir index item, root: %llu, inode: %llu, index: %llu, error: %pe", + btrfs_root_id(node->root), node->inode_id, index, ERR_PTR(ret)); + btrfs_delayed_item_release_metadata(dir->root, item); + btrfs_release_delayed_item(item); + } + mutex_unlock(&node->mutex); +end: + btrfs_release_delayed_node(node, &delayed_node_tracker); + return ret; +} + +int btrfs_inode_delayed_dir_index_count(struct btrfs_inode *inode) +{ + struct btrfs_ref_tracker delayed_node_tracker; + struct btrfs_delayed_node *delayed_node; + + delayed_node = btrfs_get_delayed_node(inode, &delayed_node_tracker); + if (!delayed_node) + return -ENOENT; + + /* + * Since we have held i_mutex of this directory, it is impossible that + * a new directory index is added into the delayed node and index_cnt + * is updated now. So we needn't lock the delayed node. + */ + if (!delayed_node->index_cnt) { + btrfs_release_delayed_node(delayed_node, &delayed_node_tracker); + return -EINVAL; + } + + inode->index_cnt = delayed_node->index_cnt; + btrfs_release_delayed_node(delayed_node, &delayed_node_tracker); + return 0; +} + +bool btrfs_readdir_get_delayed_items(struct btrfs_inode *inode, + u64 last_index, + struct list_head *ins_list, + struct list_head *del_list) +{ + struct btrfs_delayed_node *delayed_node; + struct btrfs_delayed_item *item; + struct btrfs_ref_tracker delayed_node_tracker; + + delayed_node = btrfs_get_delayed_node(inode, &delayed_node_tracker); + if (!delayed_node) + return false; + + /* + * We can only do one readdir with delayed items at a time because of + * item->readdir_list. + */ + btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); + btrfs_inode_lock(inode, 0); + + mutex_lock(&delayed_node->mutex); + item = __btrfs_first_delayed_insertion_item(delayed_node); + while (item && item->index <= last_index) { + refcount_inc(&item->refs); + list_add_tail(&item->readdir_list, ins_list); + item = __btrfs_next_delayed_item(item); + } + + item = __btrfs_first_delayed_deletion_item(delayed_node); + while (item && item->index <= last_index) { + refcount_inc(&item->refs); + list_add_tail(&item->readdir_list, del_list); + item = __btrfs_next_delayed_item(item); + } + mutex_unlock(&delayed_node->mutex); + /* + * This delayed node is still cached in the btrfs inode, so refs + * must be > 1 now, and we needn't check it is going to be freed + * or not. + * + * Besides that, this function is used to read dir, we do not + * insert/delete delayed items in this period. So we also needn't + * requeue or dequeue this delayed node. + */ + btrfs_delayed_node_ref_tracker_free(delayed_node, &delayed_node_tracker); + refcount_dec(&delayed_node->refs); + + return true; +} + +void btrfs_readdir_put_delayed_items(struct btrfs_inode *inode, + struct list_head *ins_list, + struct list_head *del_list) +{ + struct btrfs_delayed_item *curr, *next; + + list_for_each_entry_safe(curr, next, ins_list, readdir_list) { + list_del(&curr->readdir_list); + if (refcount_dec_and_test(&curr->refs)) + kfree(curr); + } + + list_for_each_entry_safe(curr, next, del_list, readdir_list) { + list_del(&curr->readdir_list); + if (refcount_dec_and_test(&curr->refs)) + kfree(curr); + } + + /* + * The VFS is going to do up_read(), so we need to downgrade back to a + * read lock. + */ + downgrade_write(&inode->vfs_inode.i_rwsem); +} + +bool btrfs_should_delete_dir_index(const struct list_head *del_list, u64 index) +{ + struct btrfs_delayed_item *curr; + bool ret = false; + + list_for_each_entry(curr, del_list, readdir_list) { + if (curr->index > index) + break; + if (curr->index == index) { + ret = true; + break; + } + } + return ret; +} + +/* + * Read dir info stored in the delayed tree. + */ +bool btrfs_readdir_delayed_dir_index(struct dir_context *ctx, + const struct list_head *ins_list) +{ + struct btrfs_dir_item *di; + struct btrfs_delayed_item *curr, *next; + struct btrfs_key location; + char *name; + int name_len; + unsigned char d_type; + + /* + * Changing the data of the delayed item is impossible. So + * we needn't lock them. And we have held i_mutex of the + * directory, nobody can delete any directory indexes now. + */ + list_for_each_entry_safe(curr, next, ins_list, readdir_list) { + bool over; + + list_del(&curr->readdir_list); + + if (curr->index < ctx->pos) { + if (refcount_dec_and_test(&curr->refs)) + kfree(curr); + continue; + } + + ctx->pos = curr->index; + + di = (struct btrfs_dir_item *)curr->data; + name = (char *)(di + 1); + name_len = btrfs_stack_dir_name_len(di); + + d_type = fs_ftype_to_dtype(btrfs_dir_flags_to_ftype(di->type)); + btrfs_disk_key_to_cpu(&location, &di->location); + + over = !dir_emit(ctx, name, name_len, location.objectid, d_type); + + if (refcount_dec_and_test(&curr->refs)) + kfree(curr); + + if (over) + return true; + ctx->pos++; + } + return false; +} + +static void fill_stack_inode_item(struct btrfs_trans_handle *trans, + struct btrfs_inode_item *inode_item, + struct btrfs_inode *inode) +{ + struct inode *vfs_inode = &inode->vfs_inode; + u64 flags; + + btrfs_set_stack_inode_uid(inode_item, i_uid_read(vfs_inode)); + btrfs_set_stack_inode_gid(inode_item, i_gid_read(vfs_inode)); + btrfs_set_stack_inode_size(inode_item, inode->disk_i_size); + btrfs_set_stack_inode_mode(inode_item, vfs_inode->i_mode); + btrfs_set_stack_inode_nlink(inode_item, vfs_inode->i_nlink); + btrfs_set_stack_inode_nbytes(inode_item, inode_get_bytes(vfs_inode)); + btrfs_set_stack_inode_generation(inode_item, inode->generation); + btrfs_set_stack_inode_sequence(inode_item, + inode_peek_iversion(vfs_inode)); + btrfs_set_stack_inode_transid(inode_item, trans->transid); + btrfs_set_stack_inode_rdev(inode_item, vfs_inode->i_rdev); + flags = btrfs_inode_combine_flags(inode->flags, inode->ro_flags); + btrfs_set_stack_inode_flags(inode_item, flags); + btrfs_set_stack_inode_block_group(inode_item, 0); + + btrfs_set_stack_timespec_sec(&inode_item->atime, + inode_get_atime_sec(vfs_inode)); + btrfs_set_stack_timespec_nsec(&inode_item->atime, + inode_get_atime_nsec(vfs_inode)); + + btrfs_set_stack_timespec_sec(&inode_item->mtime, + inode_get_mtime_sec(vfs_inode)); + btrfs_set_stack_timespec_nsec(&inode_item->mtime, + inode_get_mtime_nsec(vfs_inode)); + + btrfs_set_stack_timespec_sec(&inode_item->ctime, + inode_get_ctime_sec(vfs_inode)); + btrfs_set_stack_timespec_nsec(&inode_item->ctime, + inode_get_ctime_nsec(vfs_inode)); + + btrfs_set_stack_timespec_sec(&inode_item->otime, inode->i_otime_sec); + btrfs_set_stack_timespec_nsec(&inode_item->otime, inode->i_otime_nsec); +} + +int btrfs_fill_inode(struct btrfs_inode *inode, u32 *rdev) +{ + struct btrfs_delayed_node *delayed_node; + struct btrfs_ref_tracker delayed_node_tracker; + struct btrfs_inode_item *inode_item; + struct inode *vfs_inode = &inode->vfs_inode; + + delayed_node = btrfs_get_delayed_node(inode, &delayed_node_tracker); + if (!delayed_node) + return -ENOENT; + + mutex_lock(&delayed_node->mutex); + if (!test_bit(BTRFS_DELAYED_NODE_INODE_DIRTY, &delayed_node->flags)) { + mutex_unlock(&delayed_node->mutex); + btrfs_release_delayed_node(delayed_node, &delayed_node_tracker); + return -ENOENT; + } + + inode_item = &delayed_node->inode_item; + + i_uid_write(vfs_inode, btrfs_stack_inode_uid(inode_item)); + i_gid_write(vfs_inode, btrfs_stack_inode_gid(inode_item)); + btrfs_i_size_write(inode, btrfs_stack_inode_size(inode_item)); + vfs_inode->i_mode = btrfs_stack_inode_mode(inode_item); + set_nlink(vfs_inode, btrfs_stack_inode_nlink(inode_item)); + inode_set_bytes(vfs_inode, btrfs_stack_inode_nbytes(inode_item)); + inode->generation = btrfs_stack_inode_generation(inode_item); + inode->last_trans = btrfs_stack_inode_transid(inode_item); + + inode_set_iversion_queried(vfs_inode, btrfs_stack_inode_sequence(inode_item)); + vfs_inode->i_rdev = 0; + *rdev = btrfs_stack_inode_rdev(inode_item); + btrfs_inode_split_flags(btrfs_stack_inode_flags(inode_item), + &inode->flags, &inode->ro_flags); + + inode_set_atime(vfs_inode, btrfs_stack_timespec_sec(&inode_item->atime), + btrfs_stack_timespec_nsec(&inode_item->atime)); + + inode_set_mtime(vfs_inode, btrfs_stack_timespec_sec(&inode_item->mtime), + btrfs_stack_timespec_nsec(&inode_item->mtime)); + + inode_set_ctime(vfs_inode, btrfs_stack_timespec_sec(&inode_item->ctime), + btrfs_stack_timespec_nsec(&inode_item->ctime)); + + inode->i_otime_sec = btrfs_stack_timespec_sec(&inode_item->otime); + inode->i_otime_nsec = btrfs_stack_timespec_nsec(&inode_item->otime); + + vfs_inode->i_generation = inode->generation; + if (S_ISDIR(vfs_inode->i_mode)) + inode->index_cnt = (u64)-1; + + mutex_unlock(&delayed_node->mutex); + btrfs_release_delayed_node(delayed_node, &delayed_node_tracker); + return 0; +} + +int btrfs_delayed_update_inode(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode) +{ + struct btrfs_root *root = inode->root; + struct btrfs_delayed_node *delayed_node; + struct btrfs_ref_tracker delayed_node_tracker; + int ret = 0; + + delayed_node = btrfs_get_or_create_delayed_node(inode, &delayed_node_tracker); + if (IS_ERR(delayed_node)) + return PTR_ERR(delayed_node); + + mutex_lock(&delayed_node->mutex); + if (test_bit(BTRFS_DELAYED_NODE_INODE_DIRTY, &delayed_node->flags)) { + fill_stack_inode_item(trans, &delayed_node->inode_item, inode); + goto release_node; + } + + ret = btrfs_delayed_inode_reserve_metadata(trans, root, delayed_node); + if (ret) + goto release_node; + + fill_stack_inode_item(trans, &delayed_node->inode_item, inode); + set_bit(BTRFS_DELAYED_NODE_INODE_DIRTY, &delayed_node->flags); + delayed_node->count++; + atomic_inc(&root->fs_info->delayed_root.items); +release_node: + mutex_unlock(&delayed_node->mutex); + btrfs_release_delayed_node(delayed_node, &delayed_node_tracker); + return ret; +} + +int btrfs_delayed_delete_inode_ref(struct btrfs_inode *inode) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct btrfs_delayed_node *delayed_node; + struct btrfs_ref_tracker delayed_node_tracker; + + /* + * we don't do delayed inode updates during log recovery because it + * leads to enospc problems. This means we also can't do + * delayed inode refs + */ + if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) + return -EAGAIN; + + delayed_node = btrfs_get_or_create_delayed_node(inode, &delayed_node_tracker); + if (IS_ERR(delayed_node)) + return PTR_ERR(delayed_node); + + /* + * We don't reserve space for inode ref deletion is because: + * - We ONLY do async inode ref deletion for the inode who has only + * one link(i_nlink == 1), it means there is only one inode ref. + * And in most case, the inode ref and the inode item are in the + * same leaf, and we will deal with them at the same time. + * Since we are sure we will reserve the space for the inode item, + * it is unnecessary to reserve space for inode ref deletion. + * - If the inode ref and the inode item are not in the same leaf, + * We also needn't worry about enospc problem, because we reserve + * much more space for the inode update than it needs. + * - At the worst, we can steal some space from the global reservation. + * It is very rare. + */ + mutex_lock(&delayed_node->mutex); + if (!test_and_set_bit(BTRFS_DELAYED_NODE_DEL_IREF, &delayed_node->flags)) { + delayed_node->count++; + atomic_inc(&fs_info->delayed_root.items); + } + mutex_unlock(&delayed_node->mutex); + btrfs_release_delayed_node(delayed_node, &delayed_node_tracker); + return 0; +} + +static void __btrfs_kill_delayed_node(struct btrfs_delayed_node *delayed_node) +{ + struct btrfs_root *root = delayed_node->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_delayed_item *curr_item, *prev_item; + + mutex_lock(&delayed_node->mutex); + curr_item = __btrfs_first_delayed_insertion_item(delayed_node); + while (curr_item) { + prev_item = curr_item; + curr_item = __btrfs_next_delayed_item(prev_item); + btrfs_release_delayed_item(prev_item); + } + + if (delayed_node->index_item_leaves > 0) { + btrfs_delayed_item_release_leaves(delayed_node, + delayed_node->index_item_leaves); + delayed_node->index_item_leaves = 0; + } + + curr_item = __btrfs_first_delayed_deletion_item(delayed_node); + while (curr_item) { + btrfs_delayed_item_release_metadata(root, curr_item); + prev_item = curr_item; + curr_item = __btrfs_next_delayed_item(prev_item); + btrfs_release_delayed_item(prev_item); + } + + btrfs_release_delayed_iref(delayed_node); + + if (test_bit(BTRFS_DELAYED_NODE_INODE_DIRTY, &delayed_node->flags)) { + btrfs_delayed_inode_release_metadata(fs_info, delayed_node, false); + btrfs_release_delayed_inode(delayed_node); + } + mutex_unlock(&delayed_node->mutex); +} + +void btrfs_kill_delayed_inode_items(struct btrfs_inode *inode) +{ + struct btrfs_delayed_node *delayed_node; + struct btrfs_ref_tracker delayed_node_tracker; + + delayed_node = btrfs_get_delayed_node(inode, &delayed_node_tracker); + if (!delayed_node) + return; + + __btrfs_kill_delayed_node(delayed_node); + btrfs_release_delayed_node(delayed_node, &delayed_node_tracker); +} + +void btrfs_kill_all_delayed_nodes(struct btrfs_root *root) +{ + unsigned long index = 0; + struct btrfs_delayed_node *delayed_nodes[8]; + struct btrfs_ref_tracker delayed_node_trackers[8]; + + while (1) { + struct btrfs_delayed_node *node; + int count; + + xa_lock(&root->delayed_nodes); + if (xa_empty(&root->delayed_nodes)) { + xa_unlock(&root->delayed_nodes); + return; + } + + count = 0; + xa_for_each_start(&root->delayed_nodes, index, node, index) { + /* + * Don't increase refs in case the node is dead and + * about to be removed from the tree in the loop below + */ + if (refcount_inc_not_zero(&node->refs)) { + btrfs_delayed_node_ref_tracker_alloc(node, + &delayed_node_trackers[count], + GFP_ATOMIC); + delayed_nodes[count] = node; + count++; + } + if (count >= ARRAY_SIZE(delayed_nodes)) + break; + } + xa_unlock(&root->delayed_nodes); + index++; + + for (int i = 0; i < count; i++) { + __btrfs_kill_delayed_node(delayed_nodes[i]); + btrfs_delayed_node_ref_tracker_dir_print(delayed_nodes[i]); + btrfs_release_delayed_node(delayed_nodes[i], + &delayed_node_trackers[i]); + } + } +} + +void btrfs_destroy_delayed_inodes(struct btrfs_fs_info *fs_info) +{ + struct btrfs_delayed_node *curr_node, *prev_node; + struct btrfs_ref_tracker curr_delayed_node_tracker, prev_delayed_node_tracker; + + curr_node = btrfs_first_delayed_node(fs_info, &curr_delayed_node_tracker); + while (curr_node) { + __btrfs_kill_delayed_node(curr_node); + + prev_node = curr_node; + prev_delayed_node_tracker = curr_delayed_node_tracker; + curr_node = btrfs_next_delayed_node(curr_node, &curr_delayed_node_tracker); + btrfs_release_delayed_node(prev_node, &prev_delayed_node_tracker); + } +} + +void btrfs_log_get_delayed_items(struct btrfs_inode *inode, + struct list_head *ins_list, + struct list_head *del_list) +{ + struct btrfs_delayed_node *node; + struct btrfs_delayed_item *item; + struct btrfs_ref_tracker delayed_node_tracker; + + node = btrfs_get_delayed_node(inode, &delayed_node_tracker); + if (!node) + return; + + mutex_lock(&node->mutex); + item = __btrfs_first_delayed_insertion_item(node); + while (item) { + /* + * It's possible that the item is already in a log list. This + * can happen in case two tasks are trying to log the same + * directory. For example if we have tasks A and task B: + * + * Task A collected the delayed items into a log list while + * under the inode's log_mutex (at btrfs_log_inode()), but it + * only releases the items after logging the inodes they point + * to (if they are new inodes), which happens after unlocking + * the log mutex; + * + * Task B enters btrfs_log_inode() and acquires the log_mutex + * of the same directory inode, before task B releases the + * delayed items. This can happen for example when logging some + * inode we need to trigger logging of its parent directory, so + * logging two files that have the same parent directory can + * lead to this. + * + * If this happens, just ignore delayed items already in a log + * list. All the tasks logging the directory are under a log + * transaction and whichever finishes first can not sync the log + * before the other completes and leaves the log transaction. + */ + if (!item->logged && list_empty(&item->log_list)) { + refcount_inc(&item->refs); + list_add_tail(&item->log_list, ins_list); + } + item = __btrfs_next_delayed_item(item); + } + + item = __btrfs_first_delayed_deletion_item(node); + while (item) { + /* It may be non-empty, for the same reason mentioned above. */ + if (!item->logged && list_empty(&item->log_list)) { + refcount_inc(&item->refs); + list_add_tail(&item->log_list, del_list); + } + item = __btrfs_next_delayed_item(item); + } + mutex_unlock(&node->mutex); + + /* + * We are called during inode logging, which means the inode is in use + * and can not be evicted before we finish logging the inode. So we never + * have the last reference on the delayed inode. + * Also, we don't use btrfs_release_delayed_node() because that would + * requeue the delayed inode (change its order in the list of prepared + * nodes) and we don't want to do such change because we don't create or + * delete delayed items. + */ + ASSERT(refcount_read(&node->refs) > 1); + btrfs_delayed_node_ref_tracker_free(node, &delayed_node_tracker); + refcount_dec(&node->refs); +} + +void btrfs_log_put_delayed_items(struct btrfs_inode *inode, + struct list_head *ins_list, + struct list_head *del_list) +{ + struct btrfs_delayed_node *node; + struct btrfs_delayed_item *item; + struct btrfs_delayed_item *next; + struct btrfs_ref_tracker delayed_node_tracker; + + node = btrfs_get_delayed_node(inode, &delayed_node_tracker); + if (!node) + return; + + mutex_lock(&node->mutex); + + list_for_each_entry_safe(item, next, ins_list, log_list) { + item->logged = true; + list_del_init(&item->log_list); + if (refcount_dec_and_test(&item->refs)) + kfree(item); + } + + list_for_each_entry_safe(item, next, del_list, log_list) { + item->logged = true; + list_del_init(&item->log_list); + if (refcount_dec_and_test(&item->refs)) + kfree(item); + } + + mutex_unlock(&node->mutex); + + /* + * We are called during inode logging, which means the inode is in use + * and can not be evicted before we finish logging the inode. So we never + * have the last reference on the delayed inode. + * Also, we don't use btrfs_release_delayed_node() because that would + * requeue the delayed inode (change its order in the list of prepared + * nodes) and we don't want to do such change because we don't create or + * delete delayed items. + */ + ASSERT(refcount_read(&node->refs) > 1); + btrfs_delayed_node_ref_tracker_free(node, &delayed_node_tracker); + refcount_dec(&node->refs); +} diff --git a/fs/btrfs/delayed-inode.h b/fs/btrfs/delayed-inode.h new file mode 100644 index 000000000..fc752863f --- /dev/null +++ b/fs/btrfs/delayed-inode.h @@ -0,0 +1,257 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2011 Fujitsu. All rights reserved. + * Written by Miao Xie + */ + +#ifndef BTRFS_DELAYED_INODE_H +#define BTRFS_DELAYED_INODE_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ctree.h" + +struct btrfs_disk_key; +struct btrfs_fs_info; +struct btrfs_inode; +struct btrfs_root; +struct btrfs_trans_handle; + +enum btrfs_delayed_item_type { + BTRFS_DELAYED_INSERTION_ITEM, + BTRFS_DELAYED_DELETION_ITEM +}; + +struct btrfs_ref_tracker_dir { +#ifdef CONFIG_BTRFS_DEBUG + struct ref_tracker_dir dir; +#else + struct {} tracker; +#endif +}; + +struct btrfs_ref_tracker { +#ifdef CONFIG_BTRFS_DEBUG + struct ref_tracker *tracker; +#else + struct {} tracker; +#endif +}; + +#define BTRFS_DELAYED_NODE_IN_LIST 0 +#define BTRFS_DELAYED_NODE_INODE_DIRTY 1 +#define BTRFS_DELAYED_NODE_DEL_IREF 2 + +struct btrfs_delayed_node { + u64 inode_id; + u64 bytes_reserved; + struct btrfs_root *root; + /* Used to add the node into the delayed root's node list. */ + struct list_head n_list; + /* + * Used to add the node into the prepare list, the nodes in this list + * is waiting to be dealt with by the async worker. + */ + struct list_head p_list; + struct rb_root_cached ins_root; + struct rb_root_cached del_root; + struct mutex mutex; + struct btrfs_inode_item inode_item; + refcount_t refs; + int count; + u64 index_cnt; + unsigned long flags; + /* + * The size of the next batch of dir index items to insert (if this + * node is from a directory inode). Protected by @mutex. + */ + u32 curr_index_batch_size; + /* + * Number of leaves reserved for inserting dir index items (if this + * node belongs to a directory inode). This may be larger then the + * actual number of leaves we end up using. Protected by @mutex. + */ + u32 index_item_leaves; + /* Track all references to this delayed node. */ + struct btrfs_ref_tracker_dir ref_dir; + /* Track delayed node reference stored in node list. */ + struct btrfs_ref_tracker node_list_tracker; + /* Track delayed node reference stored in inode cache. */ + struct btrfs_ref_tracker inode_cache_tracker; +}; + +struct btrfs_delayed_item { + struct rb_node rb_node; + /* Offset value of the corresponding dir index key. */ + u64 index; + struct list_head tree_list; /* used for batch insert/delete items */ + struct list_head readdir_list; /* used for readdir items */ + /* + * Used when logging a directory. + * Insertions and deletions to this list are protected by the parent + * delayed node's mutex. + */ + struct list_head log_list; + u64 bytes_reserved; + struct btrfs_delayed_node *delayed_node; + refcount_t refs; + enum btrfs_delayed_item_type type:8; + /* + * Track if this delayed item was already logged. + * Protected by the mutex of the parent delayed inode. + */ + bool logged; + /* The maximum leaf size is 64K, so u16 is more than enough. */ + u16 data_len; + char data[] __counted_by(data_len); +}; + +void btrfs_init_delayed_root(struct btrfs_delayed_root *delayed_root); +int btrfs_insert_delayed_dir_index(struct btrfs_trans_handle *trans, + const char *name, int name_len, + struct btrfs_inode *dir, + const struct btrfs_disk_key *disk_key, u8 flags, + u64 index); + +int btrfs_delete_delayed_dir_index(struct btrfs_trans_handle *trans, + struct btrfs_inode *dir, u64 index); + +int btrfs_inode_delayed_dir_index_count(struct btrfs_inode *inode); + +int btrfs_run_delayed_items(struct btrfs_trans_handle *trans); +int btrfs_run_delayed_items_nr(struct btrfs_trans_handle *trans, int nr); + +void btrfs_balance_delayed_items(struct btrfs_fs_info *fs_info); + +int btrfs_commit_inode_delayed_items(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode); +/* Used for evicting the inode. */ +void btrfs_remove_delayed_node(struct btrfs_inode *inode); +void btrfs_kill_delayed_inode_items(struct btrfs_inode *inode); +int btrfs_commit_inode_delayed_inode(struct btrfs_inode *inode); + + +int btrfs_delayed_update_inode(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode); +int btrfs_fill_inode(struct btrfs_inode *inode, u32 *rdev); +int btrfs_delayed_delete_inode_ref(struct btrfs_inode *inode); + +/* Used for drop dead root */ +void btrfs_kill_all_delayed_nodes(struct btrfs_root *root); + +/* Used for clean the transaction */ +void btrfs_destroy_delayed_inodes(struct btrfs_fs_info *fs_info); + +/* Used for readdir() */ +bool btrfs_readdir_get_delayed_items(struct btrfs_inode *inode, + u64 last_index, + struct list_head *ins_list, + struct list_head *del_list); +void btrfs_readdir_put_delayed_items(struct btrfs_inode *inode, + struct list_head *ins_list, + struct list_head *del_list); +bool btrfs_should_delete_dir_index(const struct list_head *del_list, u64 index); +bool btrfs_readdir_delayed_dir_index(struct dir_context *ctx, + const struct list_head *ins_list); + +/* Used during directory logging. */ +void btrfs_log_get_delayed_items(struct btrfs_inode *inode, + struct list_head *ins_list, + struct list_head *del_list); +void btrfs_log_put_delayed_items(struct btrfs_inode *inode, + struct list_head *ins_list, + struct list_head *del_list); + +/* for init */ +int __init btrfs_delayed_inode_init(void); +void __cold btrfs_delayed_inode_exit(void); + +/* for debugging */ +void btrfs_assert_delayed_root_empty(struct btrfs_fs_info *fs_info); + +#define BTRFS_DELAYED_NODE_REF_TRACKER_QUARANTINE_COUNT 16 +#define BTRFS_DELAYED_NODE_REF_TRACKER_DISPLAY_LIMIT 16 + +#ifdef CONFIG_BTRFS_DEBUG +static inline void btrfs_delayed_node_ref_tracker_dir_init(struct btrfs_delayed_node *node) +{ + if (!btrfs_test_opt(node->root->fs_info, REF_TRACKER)) + return; + + ref_tracker_dir_init(&node->ref_dir.dir, + BTRFS_DELAYED_NODE_REF_TRACKER_QUARANTINE_COUNT, + "delayed_node"); +} + +static inline void btrfs_delayed_node_ref_tracker_dir_exit(struct btrfs_delayed_node *node) +{ + if (!btrfs_test_opt(node->root->fs_info, REF_TRACKER)) + return; + + ref_tracker_dir_exit(&node->ref_dir.dir); +} + +static inline void btrfs_delayed_node_ref_tracker_dir_print(struct btrfs_delayed_node *node) +{ + if (!btrfs_test_opt(node->root->fs_info, REF_TRACKER)) + return; + + /* + * Only print if there are leaked references. The caller is + * holding one reference, so if refs == 1 there is no leak. + */ + if (refcount_read(&node->refs) == 1) + return; + + ref_tracker_dir_print(&node->ref_dir.dir, + BTRFS_DELAYED_NODE_REF_TRACKER_DISPLAY_LIMIT); +} + +static inline int btrfs_delayed_node_ref_tracker_alloc(struct btrfs_delayed_node *node, + struct btrfs_ref_tracker *tracker, + gfp_t gfp) +{ + if (!btrfs_test_opt(node->root->fs_info, REF_TRACKER)) + return 0; + + return ref_tracker_alloc(&node->ref_dir.dir, &tracker->tracker, gfp); +} + +static inline int btrfs_delayed_node_ref_tracker_free(struct btrfs_delayed_node *node, + struct btrfs_ref_tracker *tracker) +{ + if (!btrfs_test_opt(node->root->fs_info, REF_TRACKER)) + return 0; + + return ref_tracker_free(&node->ref_dir.dir, &tracker->tracker); +} +#else +static inline void btrfs_delayed_node_ref_tracker_dir_init(struct btrfs_delayed_node *node) { } + +static inline void btrfs_delayed_node_ref_tracker_dir_exit(struct btrfs_delayed_node *node) { } + +static inline void btrfs_delayed_node_ref_tracker_dir_print(struct btrfs_delayed_node *node) { } + +static inline int btrfs_delayed_node_ref_tracker_alloc(struct btrfs_delayed_node *node, + struct btrfs_ref_tracker *tracker, + gfp_t gfp) +{ + return 0; +} + +static inline int btrfs_delayed_node_ref_tracker_free(struct btrfs_delayed_node *node, + struct btrfs_ref_tracker *tracker) +{ + return 0; +} +#endif + +#endif diff --git a/fs/btrfs/delayed-ref.c b/fs/btrfs/delayed-ref.c new file mode 100644 index 000000000..8bc192923 --- /dev/null +++ b/fs/btrfs/delayed-ref.c @@ -0,0 +1,1406 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2009 Oracle. All rights reserved. + */ + +#include +#include +#include +#include "messages.h" +#include "ctree.h" +#include "delayed-ref.h" +#include "extent-tree.h" +#include "transaction.h" +#include "qgroup.h" +#include "space-info.h" +#include "tree-mod-log.h" +#include "fs.h" + +struct kmem_cache *btrfs_delayed_ref_head_cachep; +struct kmem_cache *btrfs_delayed_ref_node_cachep; +struct kmem_cache *btrfs_delayed_extent_op_cachep; +/* + * delayed back reference update tracking. For subvolume trees + * we queue up extent allocations and backref maintenance for + * delayed processing. This avoids deep call chains where we + * add extents in the middle of btrfs_search_slot, and it allows + * us to buffer up frequently modified backrefs in an rb tree instead + * of hammering updates on the extent allocation tree. + */ + +bool btrfs_check_space_for_delayed_refs(struct btrfs_fs_info *fs_info) +{ + struct btrfs_block_rsv *delayed_refs_rsv = &fs_info->delayed_refs_rsv; + struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; + bool ret = false; + u64 reserved; + + spin_lock(&global_rsv->lock); + reserved = global_rsv->reserved; + spin_unlock(&global_rsv->lock); + + /* + * Since the global reserve is just kind of magic we don't really want + * to rely on it to save our bacon, so if our size is more than the + * delayed_refs_rsv and the global rsv then it's time to think about + * bailing. + */ + spin_lock(&delayed_refs_rsv->lock); + reserved += delayed_refs_rsv->reserved; + if (delayed_refs_rsv->size >= reserved) + ret = true; + spin_unlock(&delayed_refs_rsv->lock); + return ret; +} + +/* + * Release a ref head's reservation. + * + * @fs_info: the filesystem + * @nr_refs: number of delayed refs to drop + * @nr_csums: number of csum items to drop + * + * Drops the delayed ref head's count from the delayed refs rsv and free any + * excess reservation we had. + */ +void btrfs_delayed_refs_rsv_release(struct btrfs_fs_info *fs_info, int nr_refs, int nr_csums) +{ + struct btrfs_block_rsv *block_rsv = &fs_info->delayed_refs_rsv; + u64 num_bytes; + u64 released; + + num_bytes = btrfs_calc_delayed_ref_bytes(fs_info, nr_refs); + num_bytes += btrfs_calc_delayed_ref_csum_bytes(fs_info, nr_csums); + + released = btrfs_block_rsv_release(fs_info, block_rsv, num_bytes, NULL); + if (released) + trace_btrfs_space_reservation(fs_info, "delayed_refs_rsv", + 0, released, 0); +} + +/* + * Adjust the size of the delayed refs rsv. + * + * This is to be called anytime we may have adjusted trans->delayed_ref_updates + * or trans->delayed_ref_csum_deletions, it'll calculate the additional size and + * add it to the delayed_refs_rsv. + */ +void btrfs_update_delayed_refs_rsv(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_block_rsv *delayed_rsv = &fs_info->delayed_refs_rsv; + struct btrfs_block_rsv *local_rsv = &trans->delayed_rsv; + u64 num_bytes; + u64 reserved_bytes; + + if (btrfs_is_testing(fs_info)) + return; + + num_bytes = btrfs_calc_delayed_ref_bytes(fs_info, trans->delayed_ref_updates); + num_bytes += btrfs_calc_delayed_ref_csum_bytes(fs_info, + trans->delayed_ref_csum_deletions); + + if (num_bytes == 0) + return; + + /* + * Try to take num_bytes from the transaction's local delayed reserve. + * If not possible, try to take as much as it's available. If the local + * reserve doesn't have enough reserved space, the delayed refs reserve + * will be refilled next time btrfs_delayed_refs_rsv_refill() is called + * by someone or if a transaction commit is triggered before that, the + * global block reserve will be used. We want to minimize using the + * global block reserve for cases we can account for in advance, to + * avoid exhausting it and reach -ENOSPC during a transaction commit. + */ + spin_lock(&local_rsv->lock); + reserved_bytes = min(num_bytes, local_rsv->reserved); + local_rsv->reserved -= reserved_bytes; + local_rsv->full = (local_rsv->reserved >= local_rsv->size); + spin_unlock(&local_rsv->lock); + + spin_lock(&delayed_rsv->lock); + delayed_rsv->size += num_bytes; + delayed_rsv->reserved += reserved_bytes; + delayed_rsv->full = (delayed_rsv->reserved >= delayed_rsv->size); + spin_unlock(&delayed_rsv->lock); + trans->delayed_ref_updates = 0; + trans->delayed_ref_csum_deletions = 0; +} + +/* + * Adjust the size of the delayed refs block reserve for 1 block group item + * insertion, used after allocating a block group. + */ +void btrfs_inc_delayed_refs_rsv_bg_inserts(struct btrfs_fs_info *fs_info) +{ + struct btrfs_block_rsv *delayed_rsv = &fs_info->delayed_refs_rsv; + + spin_lock(&delayed_rsv->lock); + /* + * Inserting a block group item does not require changing the free space + * tree, only the extent tree or the block group tree, so this is all we + * need. + */ + delayed_rsv->size += btrfs_calc_insert_metadata_size(fs_info, 1); + delayed_rsv->full = false; + spin_unlock(&delayed_rsv->lock); +} + +/* + * Adjust the size of the delayed refs block reserve to release space for 1 + * block group item insertion. + */ +void btrfs_dec_delayed_refs_rsv_bg_inserts(struct btrfs_fs_info *fs_info) +{ + struct btrfs_block_rsv *delayed_rsv = &fs_info->delayed_refs_rsv; + const u64 num_bytes = btrfs_calc_insert_metadata_size(fs_info, 1); + u64 released; + + released = btrfs_block_rsv_release(fs_info, delayed_rsv, num_bytes, NULL); + if (released > 0) + trace_btrfs_space_reservation(fs_info, "delayed_refs_rsv", + 0, released, 0); +} + +/* + * Adjust the size of the delayed refs block reserve for 1 block group item + * update. + */ +void btrfs_inc_delayed_refs_rsv_bg_updates(struct btrfs_fs_info *fs_info) +{ + struct btrfs_block_rsv *delayed_rsv = &fs_info->delayed_refs_rsv; + + spin_lock(&delayed_rsv->lock); + /* + * Updating a block group item does not result in new nodes/leaves and + * does not require changing the free space tree, only the extent tree + * or the block group tree, so this is all we need. + */ + delayed_rsv->size += btrfs_calc_metadata_size(fs_info, 1); + delayed_rsv->full = false; + spin_unlock(&delayed_rsv->lock); +} + +/* + * Adjust the size of the delayed refs block reserve to release space for 1 + * block group item update. + */ +void btrfs_dec_delayed_refs_rsv_bg_updates(struct btrfs_fs_info *fs_info) +{ + struct btrfs_block_rsv *delayed_rsv = &fs_info->delayed_refs_rsv; + const u64 num_bytes = btrfs_calc_metadata_size(fs_info, 1); + u64 released; + + released = btrfs_block_rsv_release(fs_info, delayed_rsv, num_bytes, NULL); + if (released > 0) + trace_btrfs_space_reservation(fs_info, "delayed_refs_rsv", + 0, released, 0); +} + +/* + * Refill based on our delayed refs usage. + * + * @fs_info: the filesystem + * @flush: control how we can flush for this reservation. + * + * This will refill the delayed block_rsv up to 1 items size worth of space and + * will return -ENOSPC if we can't make the reservation. + */ +static int btrfs_zoned_cap_metadata_reservation(struct btrfs_space_info *space_info) +{ + struct btrfs_fs_info *fs_info = space_info->fs_info; + struct btrfs_block_rsv *block_rsv = &fs_info->delayed_refs_rsv; + u64 usable; + u64 cap; + int ret = 0; + + if (!btrfs_is_zoned(fs_info)) + return 0; + + spin_lock(&space_info->lock); + usable = space_info->total_bytes - space_info->bytes_zone_unusable; + spin_unlock(&space_info->lock); + cap = usable >> 1; + + spin_lock(&block_rsv->lock); + if (block_rsv->size > cap) + ret = -EAGAIN; + spin_unlock(&block_rsv->lock); + + return ret; +} + +int btrfs_delayed_refs_rsv_refill(struct btrfs_fs_info *fs_info, + enum btrfs_reserve_flush_enum flush) +{ + struct btrfs_block_rsv *block_rsv = &fs_info->delayed_refs_rsv; + struct btrfs_space_info *space_info = block_rsv->space_info; + u64 limit = btrfs_calc_delayed_ref_bytes(fs_info, 1); + u64 num_bytes = 0; + u64 refilled_bytes; + u64 to_free; + int ret = -ENOSPC; + + spin_lock(&block_rsv->lock); + if (block_rsv->reserved < block_rsv->size) { + num_bytes = block_rsv->size - block_rsv->reserved; + num_bytes = min(num_bytes, limit); + } + spin_unlock(&block_rsv->lock); + + if (!num_bytes) + return 0; + + ret = btrfs_zoned_cap_metadata_reservation(space_info); + if (ret) + return ret; + + ret = btrfs_reserve_metadata_bytes(space_info, num_bytes, flush); + if (ret) + return ret; + + /* + * We may have raced with someone else, so check again if we the block + * reserve is still not full and release any excess space. + */ + spin_lock(&block_rsv->lock); + if (block_rsv->reserved < block_rsv->size) { + u64 needed = block_rsv->size - block_rsv->reserved; + + if (num_bytes >= needed) { + block_rsv->reserved += needed; + block_rsv->full = true; + to_free = num_bytes - needed; + refilled_bytes = needed; + } else { + block_rsv->reserved += num_bytes; + to_free = 0; + refilled_bytes = num_bytes; + } + } else { + to_free = num_bytes; + refilled_bytes = 0; + } + spin_unlock(&block_rsv->lock); + + if (to_free > 0) + btrfs_space_info_free_bytes_may_use(space_info, to_free); + + if (refilled_bytes > 0) + trace_btrfs_space_reservation(fs_info, "delayed_refs_rsv", 0, + refilled_bytes, 1); + return 0; +} + +/* + * compare two delayed data backrefs with same bytenr and type + */ +static int comp_data_refs(const struct btrfs_delayed_ref_node *ref1, + const struct btrfs_delayed_ref_node *ref2) +{ + if (ref1->data_ref.objectid < ref2->data_ref.objectid) + return -1; + if (ref1->data_ref.objectid > ref2->data_ref.objectid) + return 1; + if (ref1->data_ref.offset < ref2->data_ref.offset) + return -1; + if (ref1->data_ref.offset > ref2->data_ref.offset) + return 1; + return 0; +} + +static int comp_refs(const struct btrfs_delayed_ref_node *ref1, + const struct btrfs_delayed_ref_node *ref2, + bool check_seq) +{ + int ret = 0; + + if (ref1->type < ref2->type) + return -1; + if (ref1->type > ref2->type) + return 1; + if (ref1->type == BTRFS_SHARED_BLOCK_REF_KEY || + ref1->type == BTRFS_SHARED_DATA_REF_KEY) { + if (ref1->parent < ref2->parent) + return -1; + if (ref1->parent > ref2->parent) + return 1; + } else { + if (ref1->ref_root < ref2->ref_root) + return -1; + if (ref1->ref_root > ref2->ref_root) + return 1; + if (ref1->type == BTRFS_EXTENT_DATA_REF_KEY) + ret = comp_data_refs(ref1, ref2); + } + if (ret) + return ret; + if (check_seq) { + if (ref1->seq < ref2->seq) + return -1; + if (ref1->seq > ref2->seq) + return 1; + } + return 0; +} + +static int cmp_refs_node(const struct rb_node *new, const struct rb_node *exist) +{ + const struct btrfs_delayed_ref_node *new_node = + rb_entry(new, struct btrfs_delayed_ref_node, ref_node); + const struct btrfs_delayed_ref_node *exist_node = + rb_entry(exist, struct btrfs_delayed_ref_node, ref_node); + + return comp_refs(new_node, exist_node, true); +} + +static struct btrfs_delayed_ref_node* tree_insert(struct rb_root_cached *root, + struct btrfs_delayed_ref_node *ins) +{ + struct rb_node *node = &ins->ref_node; + struct rb_node *exist = rb_find_add_cached(node, root, cmp_refs_node); + + return rb_entry_safe(exist, struct btrfs_delayed_ref_node, ref_node); +} + +static struct btrfs_delayed_ref_head *find_first_ref_head( + struct btrfs_delayed_ref_root *dr) +{ + unsigned long from = 0; + + lockdep_assert_held(&dr->lock); + + return xa_find(&dr->head_refs, &from, ULONG_MAX, XA_PRESENT); +} + +static bool btrfs_delayed_ref_lock(struct btrfs_delayed_ref_root *delayed_refs, + struct btrfs_delayed_ref_head *head) +{ + lockdep_assert_held(&delayed_refs->lock); + if (mutex_trylock(&head->mutex)) + return true; + + refcount_inc(&head->refs); + spin_unlock(&delayed_refs->lock); + + mutex_lock(&head->mutex); + spin_lock(&delayed_refs->lock); + if (!head->tracked) { + mutex_unlock(&head->mutex); + btrfs_put_delayed_ref_head(head); + return false; + } + btrfs_put_delayed_ref_head(head); + return true; +} + +static inline void drop_delayed_ref(struct btrfs_fs_info *fs_info, + struct btrfs_delayed_ref_root *delayed_refs, + struct btrfs_delayed_ref_head *head, + struct btrfs_delayed_ref_node *ref) +{ + lockdep_assert_held(&head->lock); + rb_erase_cached(&ref->ref_node, &head->ref_tree); + RB_CLEAR_NODE(&ref->ref_node); + if (!list_empty(&ref->add_list)) + list_del(&ref->add_list); + btrfs_put_delayed_ref(ref); + btrfs_delayed_refs_rsv_release(fs_info, 1, 0); +} + +static bool merge_ref(struct btrfs_fs_info *fs_info, + struct btrfs_delayed_ref_root *delayed_refs, + struct btrfs_delayed_ref_head *head, + struct btrfs_delayed_ref_node *ref, + u64 seq) +{ + struct btrfs_delayed_ref_node *next; + struct rb_node *node = rb_next(&ref->ref_node); + bool done = false; + + while (!done && node) { + int mod; + + next = rb_entry(node, struct btrfs_delayed_ref_node, ref_node); + node = rb_next(node); + if (seq && next->seq >= seq) + break; + if (comp_refs(ref, next, false)) + break; + + if (ref->action == next->action) { + mod = next->ref_mod; + } else { + if (ref->ref_mod < next->ref_mod) { + swap(ref, next); + done = true; + } + mod = -next->ref_mod; + } + + drop_delayed_ref(fs_info, delayed_refs, head, next); + ref->ref_mod += mod; + if (ref->ref_mod == 0) { + drop_delayed_ref(fs_info, delayed_refs, head, ref); + done = true; + } else { + /* + * Can't have multiples of the same ref on a tree block. + */ + WARN_ON(ref->type == BTRFS_TREE_BLOCK_REF_KEY || + ref->type == BTRFS_SHARED_BLOCK_REF_KEY); + } + } + + return done; +} + +void btrfs_merge_delayed_refs(struct btrfs_fs_info *fs_info, + struct btrfs_delayed_ref_root *delayed_refs, + struct btrfs_delayed_ref_head *head) +{ + struct btrfs_delayed_ref_node *ref; + struct rb_node *node; + u64 seq = 0; + + lockdep_assert_held(&head->lock); + + if (RB_EMPTY_ROOT(&head->ref_tree.rb_root)) + return; + + /* We don't have too many refs to merge for data. */ + if (head->is_data) + return; + + seq = btrfs_tree_mod_log_lowest_seq(fs_info); +again: + for (node = rb_first_cached(&head->ref_tree); node; + node = rb_next(node)) { + ref = rb_entry(node, struct btrfs_delayed_ref_node, ref_node); + if (seq && ref->seq >= seq) + continue; + if (merge_ref(fs_info, delayed_refs, head, ref, seq)) + goto again; + } +} + +int btrfs_check_delayed_seq(struct btrfs_fs_info *fs_info, u64 seq) +{ + int ret = 0; + u64 min_seq = btrfs_tree_mod_log_lowest_seq(fs_info); + + if (min_seq != 0 && seq >= min_seq) { + btrfs_debug(fs_info, + "holding back delayed_ref %llu, lowest is %llu", + seq, min_seq); + ret = 1; + } + + return ret; +} + +struct btrfs_delayed_ref_head *btrfs_select_ref_head( + const struct btrfs_fs_info *fs_info, + struct btrfs_delayed_ref_root *delayed_refs) +{ + struct btrfs_delayed_ref_head *head; + unsigned long start_index; + unsigned long found_index; + bool found_head = false; + bool locked; + + spin_lock(&delayed_refs->lock); +again: + start_index = (delayed_refs->run_delayed_start >> fs_info->sectorsize_bits); + xa_for_each_start(&delayed_refs->head_refs, found_index, head, start_index) { + if (!head->processing) { + found_head = true; + break; + } + } + if (!found_head) { + if (delayed_refs->run_delayed_start == 0) { + spin_unlock(&delayed_refs->lock); + return NULL; + } + delayed_refs->run_delayed_start = 0; + goto again; + } + + head->processing = true; + WARN_ON(delayed_refs->num_heads_ready == 0); + delayed_refs->num_heads_ready--; + delayed_refs->run_delayed_start = head->bytenr + + head->num_bytes; + + locked = btrfs_delayed_ref_lock(delayed_refs, head); + spin_unlock(&delayed_refs->lock); + + /* + * We may have dropped the spin lock to get the head mutex lock, and + * that might have given someone else time to free the head. If that's + * true, it has been removed from our list and we can move on. + */ + if (!locked) + return ERR_PTR(-EAGAIN); + + return head; +} + +void btrfs_unselect_ref_head(struct btrfs_delayed_ref_root *delayed_refs, + struct btrfs_delayed_ref_head *head) +{ + spin_lock(&delayed_refs->lock); + head->processing = false; + delayed_refs->num_heads_ready++; + spin_unlock(&delayed_refs->lock); + btrfs_delayed_ref_unlock(head); +} + +void btrfs_delete_ref_head(const struct btrfs_fs_info *fs_info, + struct btrfs_delayed_ref_root *delayed_refs, + struct btrfs_delayed_ref_head *head) +{ + const unsigned long index = (head->bytenr >> fs_info->sectorsize_bits); + + lockdep_assert_held(&delayed_refs->lock); + lockdep_assert_held(&head->lock); + + xa_erase(&delayed_refs->head_refs, index); + head->tracked = false; + delayed_refs->num_heads--; + if (!head->processing) + delayed_refs->num_heads_ready--; +} + +struct btrfs_delayed_ref_node *btrfs_select_delayed_ref(struct btrfs_delayed_ref_head *head) +{ + struct btrfs_delayed_ref_node *ref; + + lockdep_assert_held(&head->mutex); + lockdep_assert_held(&head->lock); + + if (RB_EMPTY_ROOT(&head->ref_tree.rb_root)) + return NULL; + + /* + * Select a delayed ref of type BTRFS_ADD_DELAYED_REF first. + * This is to prevent a ref count from going down to zero, which deletes + * the extent item from the extent tree, when there still are references + * to add, which would fail because they would not find the extent item. + */ + if (!list_empty(&head->ref_add_list)) + return list_first_entry(&head->ref_add_list, + struct btrfs_delayed_ref_node, add_list); + + ref = rb_entry(rb_first_cached(&head->ref_tree), + struct btrfs_delayed_ref_node, ref_node); + ASSERT(list_empty(&ref->add_list)); + return ref; +} + +/* + * Helper to insert the ref_node to the tail or merge with tail. + * + * Return false if the ref was inserted. + * Return true if the ref was merged into an existing one (and therefore can be + * freed by the caller). + */ +static bool insert_delayed_ref(struct btrfs_trans_handle *trans, + struct btrfs_delayed_ref_head *href, + struct btrfs_delayed_ref_node *ref) +{ + struct btrfs_delayed_ref_root *root = &trans->transaction->delayed_refs; + struct btrfs_delayed_ref_node *exist; + int mod; + + ASSERT(ref->action == BTRFS_ADD_DELAYED_REF || + ref->action == BTRFS_DROP_DELAYED_REF); + + spin_lock(&href->lock); + exist = tree_insert(&href->ref_tree, ref); + if (!exist) { + if (ref->action == BTRFS_ADD_DELAYED_REF) + list_add_tail(&ref->add_list, &href->ref_add_list); + spin_unlock(&href->lock); + trans->delayed_ref_updates++; + return false; + } + + /* Now we are sure we can merge */ + if (exist->action == ref->action) { + mod = ref->ref_mod; + } else { + /* Need to change action */ + if (exist->ref_mod < ref->ref_mod) { + exist->action = ref->action; + mod = -exist->ref_mod; + exist->ref_mod = ref->ref_mod; + if (ref->action == BTRFS_ADD_DELAYED_REF) + list_add_tail(&exist->add_list, + &href->ref_add_list); + else if (ref->action == BTRFS_DROP_DELAYED_REF) { + ASSERT(!list_empty(&exist->add_list)); + list_del_init(&exist->add_list); + } else { + DEBUG_WARN(); + } + } else + mod = -ref->ref_mod; + } + exist->ref_mod += mod; + + /* remove existing tail if its ref_mod is zero */ + if (exist->ref_mod == 0) + drop_delayed_ref(trans->fs_info, root, href, exist); + spin_unlock(&href->lock); + return true; +} + +/* + * helper function to update the accounting in the head ref + * existing and update must have the same bytenr + */ +static noinline void update_existing_head_ref(struct btrfs_trans_handle *trans, + struct btrfs_delayed_ref_head *existing, + struct btrfs_delayed_ref_head *update) +{ + struct btrfs_delayed_ref_root *delayed_refs = + &trans->transaction->delayed_refs; + struct btrfs_fs_info *fs_info = trans->fs_info; + int old_ref_mod; + + BUG_ON(existing->is_data != update->is_data); + + spin_lock(&existing->lock); + + /* + * When freeing an extent, we may not know the owning root when we + * first create the head_ref. However, some deref before the last deref + * will know it, so we just need to update the head_ref accordingly. + */ + if (!existing->owning_root) + existing->owning_root = update->owning_root; + + if (update->must_insert_reserved) { + /* if the extent was freed and then + * reallocated before the delayed ref + * entries were processed, we can end up + * with an existing head ref without + * the must_insert_reserved flag set. + * Set it again here + */ + existing->must_insert_reserved = update->must_insert_reserved; + existing->owning_root = update->owning_root; + + /* + * update the num_bytes so we make sure the accounting + * is done correctly + */ + existing->num_bytes = update->num_bytes; + + } + + if (update->extent_op) { + if (!existing->extent_op) { + existing->extent_op = update->extent_op; + } else { + if (update->extent_op->update_key) { + memcpy(&existing->extent_op->key, + &update->extent_op->key, + sizeof(update->extent_op->key)); + existing->extent_op->update_key = true; + } + if (update->extent_op->update_flags) { + existing->extent_op->flags_to_set |= + update->extent_op->flags_to_set; + existing->extent_op->update_flags = true; + } + btrfs_free_delayed_extent_op(update->extent_op); + } + } + /* + * update the reference mod on the head to reflect this new operation, + * only need the lock for this case cause we could be processing it + * currently, for refs we just added we know we're a-ok. + */ + old_ref_mod = existing->total_ref_mod; + existing->ref_mod += update->ref_mod; + existing->total_ref_mod += update->ref_mod; + + /* + * If we are going to from a positive ref mod to a negative or vice + * versa we need to make sure to adjust pending_csums accordingly. + * We reserve bytes for csum deletion when adding or updating a ref head + * see add_delayed_ref_head() for more details. + */ + if (existing->is_data) { + u64 csum_leaves = + btrfs_csum_bytes_to_leaves(fs_info, + existing->num_bytes); + + if (existing->total_ref_mod >= 0 && old_ref_mod < 0) { + delayed_refs->pending_csums -= existing->num_bytes; + btrfs_delayed_refs_rsv_release(fs_info, 0, csum_leaves); + } + if (existing->total_ref_mod < 0 && old_ref_mod >= 0) { + delayed_refs->pending_csums += existing->num_bytes; + trans->delayed_ref_csum_deletions += csum_leaves; + } + } + + spin_unlock(&existing->lock); +} + +static void init_delayed_ref_head(struct btrfs_delayed_ref_head *head_ref, + struct btrfs_ref *generic_ref, + struct btrfs_qgroup_extent_record *qrecord, + u64 reserved) +{ + int count_mod = 1; + bool must_insert_reserved = false; + + /* If reserved is provided, it must be a data extent. */ + BUG_ON(generic_ref->type != BTRFS_REF_DATA && reserved); + + switch (generic_ref->action) { + case BTRFS_ADD_DELAYED_REF: + /* count_mod is already set to 1. */ + break; + case BTRFS_UPDATE_DELAYED_HEAD: + count_mod = 0; + break; + case BTRFS_DROP_DELAYED_REF: + /* + * The head node stores the sum of all the mods, so dropping a ref + * should drop the sum in the head node by one. + */ + count_mod = -1; + break; + case BTRFS_ADD_DELAYED_EXTENT: + /* + * BTRFS_ADD_DELAYED_EXTENT means that we need to update the + * reserved accounting when the extent is finally added, or if a + * later modification deletes the delayed ref without ever + * inserting the extent into the extent allocation tree. + * ref->must_insert_reserved is the flag used to record that + * accounting mods are required. + * + * Once we record must_insert_reserved, switch the action to + * BTRFS_ADD_DELAYED_REF because other special casing is not + * required. + */ + must_insert_reserved = true; + break; + } + + refcount_set(&head_ref->refs, 1); + head_ref->bytenr = generic_ref->bytenr; + head_ref->num_bytes = generic_ref->num_bytes; + head_ref->ref_mod = count_mod; + head_ref->reserved_bytes = reserved; + head_ref->must_insert_reserved = must_insert_reserved; + head_ref->owning_root = generic_ref->owning_root; + head_ref->is_data = (generic_ref->type == BTRFS_REF_DATA); + head_ref->is_system = (generic_ref->ref_root == BTRFS_CHUNK_TREE_OBJECTID); + head_ref->ref_tree = RB_ROOT_CACHED; + INIT_LIST_HEAD(&head_ref->ref_add_list); + head_ref->tracked = false; + head_ref->processing = false; + head_ref->total_ref_mod = count_mod; + spin_lock_init(&head_ref->lock); + mutex_init(&head_ref->mutex); + + /* If not metadata set an impossible level to help debugging. */ + if (generic_ref->type == BTRFS_REF_METADATA) + head_ref->level = generic_ref->tree_ref.level; + else + head_ref->level = U8_MAX; + + if (qrecord) { + if (generic_ref->ref_root && reserved) { + qrecord->data_rsv = reserved; + qrecord->data_rsv_refroot = generic_ref->ref_root; + } + qrecord->num_bytes = generic_ref->num_bytes; + qrecord->old_roots = NULL; + } +} + +/* + * Helper function to actually insert a head node into the xarray. This does all + * the dirty work in terms of maintaining the correct overall modification + * count. + * + * The caller is responsible for calling kfree() on @qrecord. More specifically, + * if this function reports that it did not insert it as noted in + * @qrecord_inserted_ret, then it's safe to call kfree() on it. + * + * Returns an error pointer in case of an error. + */ +static noinline struct btrfs_delayed_ref_head * +add_delayed_ref_head(struct btrfs_trans_handle *trans, + struct btrfs_delayed_ref_head *head_ref, + struct btrfs_qgroup_extent_record *qrecord, + int action, bool *qrecord_inserted_ret) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_delayed_ref_head *existing; + struct btrfs_delayed_ref_root *delayed_refs; + const unsigned long index = (head_ref->bytenr >> fs_info->sectorsize_bits); + + /* + * If 'qrecord_inserted_ret' is provided, then the first thing we need + * to do is to initialize it to false just in case we have an exit + * before trying to insert the record. + */ + if (qrecord_inserted_ret) + *qrecord_inserted_ret = false; + + delayed_refs = &trans->transaction->delayed_refs; + lockdep_assert_held(&delayed_refs->lock); + +#if BITS_PER_LONG == 32 + if (head_ref->bytenr >= MAX_LFS_FILESIZE) { + if (qrecord) + xa_release(&delayed_refs->dirty_extents, index); + btrfs_err_rl(fs_info, +"delayed ref head %llu is beyond 32bit page cache and xarray index limit", + head_ref->bytenr); + btrfs_err_32bit_limit(fs_info); + return ERR_PTR(-EOVERFLOW); + } +#endif + + /* Record qgroup extent info if provided */ + if (qrecord) { + /* + * Setting 'qrecord' but not 'qrecord_inserted_ret' will likely + * result in a memory leakage. + */ + ASSERT(qrecord_inserted_ret != NULL); + + int ret; + + ret = btrfs_qgroup_trace_extent_nolock(fs_info, delayed_refs, qrecord, + head_ref->bytenr); + if (ret) { + /* Clean up if insertion fails or item exists. */ + xa_release(&delayed_refs->dirty_extents, index); + if (ret < 0) + return ERR_PTR(ret); + } else if (qrecord_inserted_ret) { + *qrecord_inserted_ret = true; + } + } + + trace_add_delayed_ref_head(fs_info, head_ref, action); + + existing = xa_load(&delayed_refs->head_refs, index); + if (existing) { + update_existing_head_ref(trans, existing, head_ref); + /* + * we've updated the existing ref, free the newly + * allocated ref + */ + kmem_cache_free(btrfs_delayed_ref_head_cachep, head_ref); + head_ref = existing; + } else { + existing = xa_store(&delayed_refs->head_refs, index, head_ref, GFP_ATOMIC); + if (xa_is_err(existing)) { + /* Memory was preallocated by the caller. */ + ASSERT(xa_err(existing) != -ENOMEM); + return ERR_PTR(xa_err(existing)); + } else if (WARN_ON(existing)) { + /* + * Shouldn't happen we just did a lookup before under + * delayed_refs->lock. + */ + return ERR_PTR(-EEXIST); + } + head_ref->tracked = true; + /* + * We reserve the amount of bytes needed to delete csums when + * adding the ref head and not when adding individual drop refs + * since the csum items are deleted only after running the last + * delayed drop ref (the data extent's ref count drops to 0). + */ + if (head_ref->is_data && head_ref->ref_mod < 0) { + delayed_refs->pending_csums += head_ref->num_bytes; + trans->delayed_ref_csum_deletions += + btrfs_csum_bytes_to_leaves(fs_info, head_ref->num_bytes); + } + delayed_refs->num_heads++; + delayed_refs->num_heads_ready++; + } + + return head_ref; +} + +/* + * Initialize the structure which represents a modification to an extent. + * + * @fs_info: Internal to the mounted filesystem mount structure. + * + * @ref: The structure which is going to be initialized. + * + * @bytenr: The logical address of the extent for which a modification is + * going to be recorded. + * + * @num_bytes: Size of the extent whose modification is being recorded. + * + * @ref_root: The id of the root where this modification has originated, this + * can be either one of the well-known metadata trees or the + * subvolume id which references this extent. + * + * @action: Can be one of BTRFS_ADD_DELAYED_REF/BTRFS_DROP_DELAYED_REF or + * BTRFS_ADD_DELAYED_EXTENT + * + * @ref_type: Holds the type of the extent which is being recorded, can be + * one of BTRFS_SHARED_BLOCK_REF_KEY/BTRFS_TREE_BLOCK_REF_KEY + * when recording a metadata extent or BTRFS_SHARED_DATA_REF_KEY/ + * BTRFS_EXTENT_DATA_REF_KEY when recording data extent + */ +static void init_delayed_ref_common(struct btrfs_fs_info *fs_info, + struct btrfs_delayed_ref_node *ref, + struct btrfs_ref *generic_ref) +{ + int action = generic_ref->action; + u64 seq = 0; + + if (action == BTRFS_ADD_DELAYED_EXTENT) + action = BTRFS_ADD_DELAYED_REF; + + if (btrfs_is_fstree(generic_ref->ref_root)) + seq = atomic64_read(&fs_info->tree_mod_seq); + + refcount_set(&ref->refs, 1); + ref->bytenr = generic_ref->bytenr; + ref->num_bytes = generic_ref->num_bytes; + ref->ref_mod = 1; + ref->action = action; + ref->seq = seq; + ref->type = btrfs_ref_type(generic_ref); + ref->ref_root = generic_ref->ref_root; + ref->parent = generic_ref->parent; + RB_CLEAR_NODE(&ref->ref_node); + INIT_LIST_HEAD(&ref->add_list); + + if (generic_ref->type == BTRFS_REF_DATA) + ref->data_ref = generic_ref->data_ref; + else + ref->tree_ref = generic_ref->tree_ref; +} + +void btrfs_init_tree_ref(struct btrfs_ref *generic_ref, int level, u64 mod_root, + bool skip_qgroup) +{ +#ifdef CONFIG_BTRFS_DEBUG + /* If @real_root not set, use @root as fallback */ + generic_ref->real_root = mod_root ?: generic_ref->ref_root; +#endif + generic_ref->tree_ref.level = level; + generic_ref->type = BTRFS_REF_METADATA; + if (skip_qgroup || !(btrfs_is_fstree(generic_ref->ref_root) && + (!mod_root || btrfs_is_fstree(mod_root)))) + generic_ref->skip_qgroup = true; + else + generic_ref->skip_qgroup = false; + +} + +void btrfs_init_data_ref(struct btrfs_ref *generic_ref, u64 ino, u64 offset, + u64 mod_root, bool skip_qgroup) +{ +#ifdef CONFIG_BTRFS_DEBUG + /* If @real_root not set, use @root as fallback */ + generic_ref->real_root = mod_root ?: generic_ref->ref_root; +#endif + generic_ref->data_ref.objectid = ino; + generic_ref->data_ref.offset = offset; + generic_ref->type = BTRFS_REF_DATA; + if (skip_qgroup || !(btrfs_is_fstree(generic_ref->ref_root) && + (!mod_root || btrfs_is_fstree(mod_root)))) + generic_ref->skip_qgroup = true; + else + generic_ref->skip_qgroup = false; +} + +static int add_delayed_ref(struct btrfs_trans_handle *trans, + struct btrfs_ref *generic_ref, + struct btrfs_delayed_extent_op *extent_op, + u64 reserved) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_delayed_ref_node *node; + struct btrfs_delayed_ref_head *head_ref; + struct btrfs_delayed_ref_head *new_head_ref; + struct btrfs_delayed_ref_root *delayed_refs; + struct btrfs_qgroup_extent_record *record = NULL; + const unsigned long index = (generic_ref->bytenr >> fs_info->sectorsize_bits); + bool qrecord_reserved = false; + bool qrecord_inserted; + int action = generic_ref->action; + bool merged; + int ret; + + node = kmem_cache_alloc(btrfs_delayed_ref_node_cachep, GFP_NOFS); + if (!node) + return -ENOMEM; + + head_ref = kmem_cache_alloc(btrfs_delayed_ref_head_cachep, GFP_NOFS); + if (!head_ref) { + ret = -ENOMEM; + goto free_node; + } + + delayed_refs = &trans->transaction->delayed_refs; + + if (btrfs_qgroup_full_accounting(fs_info) && !generic_ref->skip_qgroup) { + record = kzalloc_obj(*record, GFP_NOFS); + if (!record) { + ret = -ENOMEM; + goto free_head_ref; + } + if (xa_reserve(&delayed_refs->dirty_extents, index, GFP_NOFS)) { + ret = -ENOMEM; + goto free_record; + } + qrecord_reserved = true; + } + + ret = xa_reserve(&delayed_refs->head_refs, index, GFP_NOFS); + if (ret) { + if (qrecord_reserved) + xa_release(&delayed_refs->dirty_extents, index); + goto free_record; + } + + init_delayed_ref_common(fs_info, node, generic_ref); + init_delayed_ref_head(head_ref, generic_ref, record, reserved); + head_ref->extent_op = extent_op; + + spin_lock(&delayed_refs->lock); + + /* + * insert both the head node and the new ref without dropping + * the spin lock + */ + new_head_ref = add_delayed_ref_head(trans, head_ref, record, + action, &qrecord_inserted); + if (IS_ERR(new_head_ref)) { + xa_release(&delayed_refs->head_refs, index); + spin_unlock(&delayed_refs->lock); + ret = PTR_ERR(new_head_ref); + + /* + * It's only safe to call kfree() on 'qrecord' if + * add_delayed_ref_head() has _not_ inserted it for + * tracing. Otherwise we need to handle this here. + */ + if (!qrecord_reserved || qrecord_inserted) + goto free_head_ref; + goto free_record; + } + head_ref = new_head_ref; + + merged = insert_delayed_ref(trans, head_ref, node); + spin_unlock(&delayed_refs->lock); + + /* + * Need to update the delayed_refs_rsv with any changes we may have + * made. + */ + btrfs_update_delayed_refs_rsv(trans); + + if (generic_ref->type == BTRFS_REF_DATA) + trace_add_delayed_data_ref(trans->fs_info, node); + else + trace_add_delayed_tree_ref(trans->fs_info, node); + if (merged) + kmem_cache_free(btrfs_delayed_ref_node_cachep, node); + + if (qrecord_inserted) + return btrfs_qgroup_trace_extent_post(trans, record, generic_ref->bytenr); + + kfree(record); + return 0; + +free_record: + kfree(record); +free_head_ref: + kmem_cache_free(btrfs_delayed_ref_head_cachep, head_ref); +free_node: + kmem_cache_free(btrfs_delayed_ref_node_cachep, node); + return ret; +} + +/* + * Add a delayed tree ref. This does all of the accounting required to make sure + * the delayed ref is eventually processed before this transaction commits. + */ +int btrfs_add_delayed_tree_ref(struct btrfs_trans_handle *trans, + struct btrfs_ref *generic_ref, + struct btrfs_delayed_extent_op *extent_op) +{ + ASSERT(generic_ref->type == BTRFS_REF_METADATA && generic_ref->action); + return add_delayed_ref(trans, generic_ref, extent_op, 0); +} + +/* + * add a delayed data ref. it's similar to btrfs_add_delayed_tree_ref. + */ +int btrfs_add_delayed_data_ref(struct btrfs_trans_handle *trans, + struct btrfs_ref *generic_ref, + u64 reserved) +{ + ASSERT(generic_ref->type == BTRFS_REF_DATA && generic_ref->action); + return add_delayed_ref(trans, generic_ref, NULL, reserved); +} + +int btrfs_add_delayed_extent_op(struct btrfs_trans_handle *trans, + u64 bytenr, u64 num_bytes, u8 level, + struct btrfs_delayed_extent_op *extent_op) +{ + const unsigned long index = (bytenr >> trans->fs_info->sectorsize_bits); + struct btrfs_delayed_ref_head *head_ref; + struct btrfs_delayed_ref_head *head_ref_ret; + struct btrfs_delayed_ref_root *delayed_refs; + struct btrfs_ref generic_ref = { + .type = BTRFS_REF_METADATA, + .action = BTRFS_UPDATE_DELAYED_HEAD, + .bytenr = bytenr, + .num_bytes = num_bytes, + .tree_ref.level = level, + }; + int ret; + + head_ref = kmem_cache_alloc(btrfs_delayed_ref_head_cachep, GFP_NOFS); + if (!head_ref) + return -ENOMEM; + + init_delayed_ref_head(head_ref, &generic_ref, NULL, 0); + head_ref->extent_op = extent_op; + + delayed_refs = &trans->transaction->delayed_refs; + + ret = xa_reserve(&delayed_refs->head_refs, index, GFP_NOFS); + if (ret) { + kmem_cache_free(btrfs_delayed_ref_head_cachep, head_ref); + return ret; + } + + spin_lock(&delayed_refs->lock); + head_ref_ret = add_delayed_ref_head(trans, head_ref, NULL, + BTRFS_UPDATE_DELAYED_HEAD, NULL); + if (IS_ERR(head_ref_ret)) { + xa_release(&delayed_refs->head_refs, index); + spin_unlock(&delayed_refs->lock); + kmem_cache_free(btrfs_delayed_ref_head_cachep, head_ref); + return PTR_ERR(head_ref_ret); + } + spin_unlock(&delayed_refs->lock); + + /* + * Need to update the delayed_refs_rsv with any changes we may have + * made. + */ + btrfs_update_delayed_refs_rsv(trans); + return 0; +} + +void btrfs_put_delayed_ref(struct btrfs_delayed_ref_node *ref) +{ + if (refcount_dec_and_test(&ref->refs)) { + WARN_ON(!RB_EMPTY_NODE(&ref->ref_node)); + kmem_cache_free(btrfs_delayed_ref_node_cachep, ref); + } +} + +/* + * This does a simple search for the head node for a given extent. Returns the + * head node if found, or NULL if not. + */ +struct btrfs_delayed_ref_head * +btrfs_find_delayed_ref_head(const struct btrfs_fs_info *fs_info, + struct btrfs_delayed_ref_root *delayed_refs, + u64 bytenr) +{ + const unsigned long index = (bytenr >> fs_info->sectorsize_bits); + + lockdep_assert_held(&delayed_refs->lock); + + return xa_load(&delayed_refs->head_refs, index); +} + +static int find_comp(struct btrfs_delayed_ref_node *entry, u64 root, u64 parent) +{ + int type = parent ? BTRFS_SHARED_BLOCK_REF_KEY : BTRFS_TREE_BLOCK_REF_KEY; + + if (type < entry->type) + return -1; + if (type > entry->type) + return 1; + + if (type == BTRFS_TREE_BLOCK_REF_KEY) { + if (root < entry->ref_root) + return -1; + if (root > entry->ref_root) + return 1; + } else { + if (parent < entry->parent) + return -1; + if (parent > entry->parent) + return 1; + } + return 0; +} + +/* + * Check to see if a given root/parent reference is attached to the head. This + * only checks for BTRFS_ADD_DELAYED_REF references that match, as that + * indicates the reference exists for the given root or parent. This is for + * tree blocks only. + * + * @head: the head of the bytenr we're searching. + * @root: the root objectid of the reference if it is a normal reference. + * @parent: the parent if this is a shared backref. + */ +bool btrfs_find_delayed_tree_ref(struct btrfs_delayed_ref_head *head, + u64 root, u64 parent) +{ + struct rb_node *node; + bool found = false; + + lockdep_assert_held(&head->mutex); + + spin_lock(&head->lock); + node = head->ref_tree.rb_root.rb_node; + while (node) { + struct btrfs_delayed_ref_node *entry; + int ret; + + entry = rb_entry(node, struct btrfs_delayed_ref_node, ref_node); + ret = find_comp(entry, root, parent); + if (ret < 0) { + node = node->rb_left; + } else if (ret > 0) { + node = node->rb_right; + } else { + /* + * We only want to count ADD actions, as drops mean the + * ref doesn't exist. + */ + if (entry->action == BTRFS_ADD_DELAYED_REF) + found = true; + break; + } + } + spin_unlock(&head->lock); + return found; +} + +void btrfs_destroy_delayed_refs(struct btrfs_transaction *trans) +{ + struct btrfs_delayed_ref_root *delayed_refs = &trans->delayed_refs; + struct btrfs_fs_info *fs_info = trans->fs_info; + + spin_lock(&delayed_refs->lock); + while (true) { + struct btrfs_delayed_ref_head *head; + struct rb_node *n; + bool pin_bytes = false; + + head = find_first_ref_head(delayed_refs); + if (!head) + break; + + if (!btrfs_delayed_ref_lock(delayed_refs, head)) + continue; + + spin_lock(&head->lock); + while ((n = rb_first_cached(&head->ref_tree)) != NULL) { + struct btrfs_delayed_ref_node *ref; + + ref = rb_entry(n, struct btrfs_delayed_ref_node, ref_node); + drop_delayed_ref(fs_info, delayed_refs, head, ref); + } + if (head->must_insert_reserved) + pin_bytes = true; + btrfs_free_delayed_extent_op(head->extent_op); + btrfs_delete_ref_head(fs_info, delayed_refs, head); + spin_unlock(&head->lock); + spin_unlock(&delayed_refs->lock); + mutex_unlock(&head->mutex); + + if (!btrfs_is_testing(fs_info) && pin_bytes) { + struct btrfs_block_group *bg; + + bg = btrfs_lookup_block_group(fs_info, head->bytenr); + if (WARN_ON_ONCE(bg == NULL)) { + /* + * Unexpected and there's nothing we can do here + * because we are in a transaction abort path, + * so any errors can only be ignored or reported + * while attempting to cleanup all resources. + */ + btrfs_err(fs_info, +"block group for delayed ref at %llu was not found while destroying ref head", + head->bytenr); + } else { + spin_lock(&bg->space_info->lock); + spin_lock(&bg->lock); + bg->pinned += head->num_bytes; + btrfs_space_info_update_bytes_pinned(bg->space_info, + head->num_bytes); + bg->reserved -= head->num_bytes; + bg->space_info->bytes_reserved -= head->num_bytes; + spin_unlock(&bg->lock); + spin_unlock(&bg->space_info->lock); + + btrfs_put_block_group(bg); + } + + btrfs_error_unpin_extent_range(fs_info, head->bytenr, + head->bytenr + head->num_bytes - 1); + } + if (!btrfs_is_testing(fs_info)) + btrfs_cleanup_ref_head_accounting(fs_info, delayed_refs, head); + btrfs_put_delayed_ref_head(head); + cond_resched(); + spin_lock(&delayed_refs->lock); + } + + if (!btrfs_is_testing(fs_info)) + btrfs_qgroup_destroy_extent_records(trans); + + spin_unlock(&delayed_refs->lock); +} + +void __cold btrfs_delayed_ref_exit(void) +{ + kmem_cache_destroy(btrfs_delayed_ref_head_cachep); + kmem_cache_destroy(btrfs_delayed_ref_node_cachep); + kmem_cache_destroy(btrfs_delayed_extent_op_cachep); +} + +int __init btrfs_delayed_ref_init(void) +{ + btrfs_delayed_ref_head_cachep = KMEM_CACHE(btrfs_delayed_ref_head, 0); + if (!btrfs_delayed_ref_head_cachep) + return -ENOMEM; + + btrfs_delayed_ref_node_cachep = KMEM_CACHE(btrfs_delayed_ref_node, 0); + if (!btrfs_delayed_ref_node_cachep) + goto fail; + + btrfs_delayed_extent_op_cachep = KMEM_CACHE(btrfs_delayed_extent_op, 0); + if (!btrfs_delayed_extent_op_cachep) + goto fail; + + return 0; +fail: + btrfs_delayed_ref_exit(); + return -ENOMEM; +} diff --git a/fs/btrfs/delayed-ref.h b/fs/btrfs/delayed-ref.h new file mode 100644 index 000000000..5ce940532 --- /dev/null +++ b/fs/btrfs/delayed-ref.h @@ -0,0 +1,459 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2008 Oracle. All rights reserved. + */ + +#ifndef BTRFS_DELAYED_REF_H +#define BTRFS_DELAYED_REF_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "fs.h" +#include "messages.h" + +struct btrfs_trans_handle; +struct btrfs_fs_info; + +/* these are the possible values of struct btrfs_delayed_ref_node->action */ +enum btrfs_delayed_ref_action { + /* Add one backref to the tree */ + BTRFS_ADD_DELAYED_REF = 1, + /* Delete one backref from the tree */ + BTRFS_DROP_DELAYED_REF, + /* Record a full extent allocation */ + BTRFS_ADD_DELAYED_EXTENT, + /* Not changing ref count on head ref */ + BTRFS_UPDATE_DELAYED_HEAD, +} __packed; + +struct btrfs_data_ref { + /* For EXTENT_DATA_REF */ + + /* Inode which refers to this data extent */ + u64 objectid; + + /* + * file_offset - extent_offset + * + * file_offset is the key.offset of the EXTENT_DATA key. + * extent_offset is btrfs_file_extent_offset() of the EXTENT_DATA data. + */ + u64 offset; +}; + +struct btrfs_tree_ref { + /* + * Level of this tree block. + * + * Shared for skinny (TREE_BLOCK_REF) and normal tree ref. + */ + int level; + + /* For non-skinny metadata, no special member needed */ +}; + +struct btrfs_delayed_ref_node { + struct rb_node ref_node; + /* + * If action is BTRFS_ADD_DELAYED_REF, also link this node to + * ref_head->ref_add_list, then we do not need to iterate the + * refs rbtree in the corresponding delayed ref head + * (struct btrfs_delayed_ref_head::ref_tree). + */ + struct list_head add_list; + + /* the starting bytenr of the extent */ + u64 bytenr; + + /* the size of the extent */ + u64 num_bytes; + + /* seq number to keep track of insertion order */ + u64 seq; + + /* The ref_root for this ref */ + u64 ref_root; + + /* + * The parent for this ref, if this isn't set the ref_root is the + * reference owner. + */ + u64 parent; + + /* ref count on this data structure */ + refcount_t refs; + + /* + * how many refs is this entry adding or deleting. For + * head refs, this may be a negative number because it is keeping + * track of the total mods done to the reference count. + * For individual refs, this will always be a positive number + * + * It may be more than one, since it is possible for a single + * parent to have more than one ref on an extent + */ + int ref_mod; + + unsigned int action:8; + unsigned int type:8; + + union { + struct btrfs_tree_ref tree_ref; + struct btrfs_data_ref data_ref; + }; +}; + +struct btrfs_delayed_extent_op { + struct btrfs_disk_key key; + bool update_key; + bool update_flags; + u64 flags_to_set; +}; + +/* + * the head refs are used to hold a lock on a given extent, which allows us + * to make sure that only one process is running the delayed refs + * at a time for a single extent. They also store the sum of all the + * reference count modifications we've queued up. + */ +struct btrfs_delayed_ref_head { + u64 bytenr; + u64 num_bytes; + /* + * the mutex is held while running the refs, and it is also + * held when checking the sum of reference modifications. + */ + struct mutex mutex; + + refcount_t refs; + + /* Protects 'ref_tree' and 'ref_add_list'. */ + spinlock_t lock; + struct rb_root_cached ref_tree; + /* accumulate add BTRFS_ADD_DELAYED_REF nodes to this ref_add_list. */ + struct list_head ref_add_list; + + struct btrfs_delayed_extent_op *extent_op; + + /* + * This is used to track the final ref_mod from all the refs associated + * with this head ref, this is not adjusted as delayed refs are run, + * this is meant to track if we need to do the csum accounting or not. + */ + int total_ref_mod; + + /* + * This is the current outstanding mod references for this bytenr. This + * is used with lookup_extent_info to get an accurate reference count + * for a bytenr, so it is adjusted as delayed refs are run so that any + * on disk reference count + ref_mod is accurate. + */ + int ref_mod; + + /* + * The root that triggered the allocation when must_insert_reserved is + * set to true. + */ + u64 owning_root; + + /* + * Track reserved bytes when setting must_insert_reserved. On success + * or cleanup, we will need to free the reservation. + */ + u64 reserved_bytes; + + /* Tree block level, for metadata only. */ + u8 level; + + /* + * when a new extent is allocated, it is just reserved in memory + * The actual extent isn't inserted into the extent allocation tree + * until the delayed ref is processed. must_insert_reserved is + * used to flag a delayed ref so the accounting can be updated + * when a full insert is done. + * + * It is possible the extent will be freed before it is ever + * inserted into the extent allocation tree. In this case + * we need to update the in ram accounting to properly reflect + * the free has happened. + */ + bool must_insert_reserved; + + bool is_data; + bool is_system; + bool processing; + /* + * Indicate if it's currently in the data structure that tracks head + * refs (struct btrfs_delayed_ref_root::head_refs). + */ + bool tracked; +}; + +enum btrfs_delayed_ref_flags { + /* Indicate that we are flushing delayed refs for the commit */ + BTRFS_DELAYED_REFS_FLUSHING, +}; + +struct btrfs_delayed_ref_root { + /* + * Track head references. + * The keys correspond to the logical address of the extent ("bytenr") + * right shifted by fs_info->sectorsize_bits. This is both to get a more + * dense index space (optimizes xarray structure) and because indexes in + * xarrays are of "unsigned long" type, meaning they are 32 bits wide on + * 32 bits platforms, limiting the extent range to 4G which is too low + * and makes it unusable (truncated index values) on 32 bits platforms. + * Protected by the spinlock 'lock' defined below. + */ + struct xarray head_refs; + + /* + * Track dirty extent records. + * The keys correspond to the logical address of the extent ("bytenr") + * right shifted by fs_info->sectorsize_bits, for same reasons as above. + */ + struct xarray dirty_extents; + + /* + * Protects the xarray head_refs, its entries and the following fields: + * num_heads, num_heads_ready, pending_csums and run_delayed_start. + */ + spinlock_t lock; + + /* Total number of head refs, protected by the spinlock 'lock'. */ + unsigned long num_heads; + + /* + * Total number of head refs ready for processing, protected by the + * spinlock 'lock'. + */ + unsigned long num_heads_ready; + + /* + * Track space reserved for deleting csums of data extents. + * Protected by the spinlock 'lock'. + */ + u64 pending_csums; + + unsigned long flags; + + /* + * Track from which bytenr to start searching ref heads. + * Protected by the spinlock 'lock'. + */ + u64 run_delayed_start; + + /* + * To make qgroup to skip given root. + * This is for snapshot, as btrfs_qgroup_inherit() will manually + * modify counters for snapshot and its source, so we should skip + * the snapshot in new_root/old_roots or it will get calculated twice + */ + u64 qgroup_to_skip; +}; + +enum btrfs_ref_type { + BTRFS_REF_NOT_SET, + BTRFS_REF_DATA, + BTRFS_REF_METADATA, +} __packed; + +struct btrfs_ref { + enum btrfs_ref_type type; + enum btrfs_delayed_ref_action action; + + /* + * Whether this extent should go through qgroup record. + * + * Normally false, but for certain cases like delayed subtree scan, + * setting this flag can hugely reduce qgroup overhead. + */ + bool skip_qgroup; + + u64 bytenr; + u64 num_bytes; + u64 owning_root; + + /* + * The root that owns the reference for this reference, this will be set + * or ->parent will be set, depending on what type of reference this is. + */ + u64 ref_root; + + /* Bytenr of the parent tree block */ + u64 parent; + union { + struct btrfs_data_ref data_ref; + struct btrfs_tree_ref tree_ref; + }; + +#ifdef CONFIG_BTRFS_DEBUG + /* Through which root is this modification. */ + u64 real_root; +#endif +}; + +extern struct kmem_cache *btrfs_delayed_ref_head_cachep; +extern struct kmem_cache *btrfs_delayed_ref_node_cachep; +extern struct kmem_cache *btrfs_delayed_extent_op_cachep; + +int __init btrfs_delayed_ref_init(void); +void __cold btrfs_delayed_ref_exit(void); + +static inline u64 btrfs_calc_delayed_ref_bytes(const struct btrfs_fs_info *fs_info, + int num_delayed_refs) +{ + u64 num_bytes; + + num_bytes = btrfs_calc_insert_metadata_size(fs_info, num_delayed_refs); + + /* + * We have to check the mount option here because we could be enabling + * the free space tree for the first time and don't have the compat_ro + * option set yet. + * + * We need extra reservations if we have the free space tree because + * we'll have to modify that tree as well. + */ + if (btrfs_test_opt(fs_info, FREE_SPACE_TREE)) + num_bytes *= 2; + + return num_bytes; +} + +static inline u64 btrfs_calc_delayed_ref_csum_bytes(const struct btrfs_fs_info *fs_info, + int num_csum_items) +{ + /* + * Deleting csum items does not result in new nodes/leaves and does not + * require changing the free space tree, only the csum tree, so this is + * all we need. + */ + return btrfs_calc_metadata_size(fs_info, num_csum_items); +} + +void btrfs_init_tree_ref(struct btrfs_ref *generic_ref, int level, u64 mod_root, + bool skip_qgroup); +void btrfs_init_data_ref(struct btrfs_ref *generic_ref, u64 ino, u64 offset, + u64 mod_root, bool skip_qgroup); + +static inline struct btrfs_delayed_extent_op * +btrfs_alloc_delayed_extent_op(void) +{ + return kmem_cache_alloc(btrfs_delayed_extent_op_cachep, GFP_NOFS); +} + +static inline void +btrfs_free_delayed_extent_op(struct btrfs_delayed_extent_op *op) +{ + if (op) + kmem_cache_free(btrfs_delayed_extent_op_cachep, op); +} + +void btrfs_put_delayed_ref(struct btrfs_delayed_ref_node *ref); + +static inline u64 btrfs_ref_head_to_space_flags( + struct btrfs_delayed_ref_head *head_ref) +{ + if (head_ref->is_data) + return BTRFS_BLOCK_GROUP_DATA; + else if (head_ref->is_system) + return BTRFS_BLOCK_GROUP_SYSTEM; + return BTRFS_BLOCK_GROUP_METADATA; +} + +static inline void btrfs_put_delayed_ref_head(struct btrfs_delayed_ref_head *head) +{ + if (refcount_dec_and_test(&head->refs)) + kmem_cache_free(btrfs_delayed_ref_head_cachep, head); +} + +int btrfs_add_delayed_tree_ref(struct btrfs_trans_handle *trans, + struct btrfs_ref *generic_ref, + struct btrfs_delayed_extent_op *extent_op); +int btrfs_add_delayed_data_ref(struct btrfs_trans_handle *trans, + struct btrfs_ref *generic_ref, + u64 reserved); +int btrfs_add_delayed_extent_op(struct btrfs_trans_handle *trans, + u64 bytenr, u64 num_bytes, u8 level, + struct btrfs_delayed_extent_op *extent_op); +void btrfs_merge_delayed_refs(struct btrfs_fs_info *fs_info, + struct btrfs_delayed_ref_root *delayed_refs, + struct btrfs_delayed_ref_head *head); + +struct btrfs_delayed_ref_head * +btrfs_find_delayed_ref_head(const struct btrfs_fs_info *fs_info, + struct btrfs_delayed_ref_root *delayed_refs, + u64 bytenr); +static inline void btrfs_delayed_ref_unlock(struct btrfs_delayed_ref_head *head) +{ + mutex_unlock(&head->mutex); +} +void btrfs_delete_ref_head(const struct btrfs_fs_info *fs_info, + struct btrfs_delayed_ref_root *delayed_refs, + struct btrfs_delayed_ref_head *head); + +struct btrfs_delayed_ref_head *btrfs_select_ref_head( + const struct btrfs_fs_info *fs_info, + struct btrfs_delayed_ref_root *delayed_refs); +void btrfs_unselect_ref_head(struct btrfs_delayed_ref_root *delayed_refs, + struct btrfs_delayed_ref_head *head); +struct btrfs_delayed_ref_node *btrfs_select_delayed_ref(struct btrfs_delayed_ref_head *head); + +int btrfs_check_delayed_seq(struct btrfs_fs_info *fs_info, u64 seq); + +void btrfs_delayed_refs_rsv_release(struct btrfs_fs_info *fs_info, int nr_refs, int nr_csums); +void btrfs_update_delayed_refs_rsv(struct btrfs_trans_handle *trans); +void btrfs_inc_delayed_refs_rsv_bg_inserts(struct btrfs_fs_info *fs_info); +void btrfs_dec_delayed_refs_rsv_bg_inserts(struct btrfs_fs_info *fs_info); +void btrfs_inc_delayed_refs_rsv_bg_updates(struct btrfs_fs_info *fs_info); +void btrfs_dec_delayed_refs_rsv_bg_updates(struct btrfs_fs_info *fs_info); +int btrfs_delayed_refs_rsv_refill(struct btrfs_fs_info *fs_info, + enum btrfs_reserve_flush_enum flush); +bool btrfs_check_space_for_delayed_refs(struct btrfs_fs_info *fs_info); +bool btrfs_find_delayed_tree_ref(struct btrfs_delayed_ref_head *head, + u64 root, u64 parent); +void btrfs_destroy_delayed_refs(struct btrfs_transaction *trans); + +static inline u64 btrfs_delayed_ref_owner(const struct btrfs_delayed_ref_node *node) +{ + if (node->type == BTRFS_EXTENT_DATA_REF_KEY || + node->type == BTRFS_SHARED_DATA_REF_KEY) + return node->data_ref.objectid; + return node->tree_ref.level; +} + +static inline u64 btrfs_delayed_ref_offset(const struct btrfs_delayed_ref_node *node) +{ + if (node->type == BTRFS_EXTENT_DATA_REF_KEY || + node->type == BTRFS_SHARED_DATA_REF_KEY) + return node->data_ref.offset; + return 0; +} + +static inline u8 btrfs_ref_type(const struct btrfs_ref *ref) +{ + ASSERT(ref->type == BTRFS_REF_DATA || ref->type == BTRFS_REF_METADATA); + + if (ref->type == BTRFS_REF_DATA) { + if (ref->parent) + return BTRFS_SHARED_DATA_REF_KEY; + else + return BTRFS_EXTENT_DATA_REF_KEY; + } else { + if (ref->parent) + return BTRFS_SHARED_BLOCK_REF_KEY; + else + return BTRFS_TREE_BLOCK_REF_KEY; + } + + return 0; +} + +#endif diff --git a/fs/btrfs/dev-replace.c b/fs/btrfs/dev-replace.c new file mode 100644 index 000000000..dc0834f92 --- /dev/null +++ b/fs/btrfs/dev-replace.c @@ -0,0 +1,1372 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) STRATO AG 2012. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include "misc.h" +#include "ctree.h" +#include "disk-io.h" +#include "transaction.h" +#include "volumes.h" +#include "async-thread.h" +#include "dev-replace.h" +#include "sysfs.h" +#include "zoned.h" +#include "block-group.h" +#include "fs.h" +#include "accessors.h" +#include "scrub.h" + +/* + * Device replace overview + * + * [Objective] + * To copy all extents (both new and on-disk) from source device to target + * device, while still keeping the filesystem read-write. + * + * [Method] + * There are two main methods involved: + * + * - Write duplication + * + * All new writes will be written to both target and source devices, so even + * if replace gets canceled, sources device still contains up-to-date data. + * + * Location: handle_ops_on_dev_replace() from btrfs_map_block() + * Start: btrfs_dev_replace_start() + * End: btrfs_dev_replace_finishing() + * Content: Latest data/metadata + * + * - Copy existing extents + * + * This happens by reusing scrub facility, as scrub also iterates through + * existing extents from commit root. + * + * Location: scrub_write_block_to_dev_replace() from + * scrub_block_complete() + * Content: Data/meta from commit root. + * + * Due to the content difference, we need to avoid nocow write when dev-replace + * is happening. This is done by marking the block group read-only and waiting + * for NOCOW writes. + * + * After replace is done, the finishing part is done by swapping the target and + * source devices. + * + * Location: btrfs_dev_replace_update_device_in_mapping_tree() from + * btrfs_dev_replace_finishing() + */ + +static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info, + int scrub_ret); +static int btrfs_dev_replace_kthread(void *data); + +int btrfs_init_dev_replace(struct btrfs_fs_info *fs_info) +{ + struct btrfs_dev_lookup_args args = { .devid = BTRFS_DEV_REPLACE_DEVID }; + struct btrfs_key key; + struct btrfs_root *dev_root = fs_info->dev_root; + struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; + struct extent_buffer *eb; + int slot; + int ret = 0; + BTRFS_PATH_AUTO_FREE(path); + int item_size; + struct btrfs_dev_replace_item *ptr; + u64 src_devid; + + if (!dev_root) + return 0; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = 0; + key.type = BTRFS_DEV_REPLACE_KEY; + key.offset = 0; + ret = btrfs_search_slot(NULL, dev_root, &key, path, 0, 0); + if (ret) { +no_valid_dev_replace_entry_found: + /* + * We don't have a replace item or it's corrupted. If there is + * a replace target, fail the mount. + */ + if (unlikely(btrfs_find_device(fs_info->fs_devices, &args))) { + btrfs_err(fs_info, + "found replace target device without a valid replace item"); + return -EUCLEAN; + } + dev_replace->replace_state = + BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED; + dev_replace->cont_reading_from_srcdev_mode = + BTRFS_DEV_REPLACE_ITEM_CONT_READING_FROM_SRCDEV_MODE_ALWAYS; + dev_replace->time_started = 0; + dev_replace->time_stopped = 0; + atomic64_set(&dev_replace->num_write_errors, 0); + atomic64_set(&dev_replace->num_uncorrectable_read_errors, 0); + dev_replace->cursor_left = 0; + dev_replace->committed_cursor_left = 0; + dev_replace->cursor_left_last_write_of_item = 0; + dev_replace->cursor_right = 0; + dev_replace->srcdev = NULL; + dev_replace->tgtdev = NULL; + dev_replace->is_valid = 0; + dev_replace->item_needs_writeback = 0; + return 0; + } + slot = path->slots[0]; + eb = path->nodes[0]; + item_size = btrfs_item_size(eb, slot); + ptr = btrfs_item_ptr(eb, slot, struct btrfs_dev_replace_item); + + if (item_size != sizeof(struct btrfs_dev_replace_item)) { + btrfs_warn(fs_info, + "dev_replace entry found has unexpected size, ignore entry"); + goto no_valid_dev_replace_entry_found; + } + + src_devid = btrfs_dev_replace_src_devid(eb, ptr); + dev_replace->cont_reading_from_srcdev_mode = + btrfs_dev_replace_cont_reading_from_srcdev_mode(eb, ptr); + dev_replace->replace_state = btrfs_dev_replace_replace_state(eb, ptr); + dev_replace->time_started = btrfs_dev_replace_time_started(eb, ptr); + dev_replace->time_stopped = + btrfs_dev_replace_time_stopped(eb, ptr); + atomic64_set(&dev_replace->num_write_errors, + btrfs_dev_replace_num_write_errors(eb, ptr)); + atomic64_set(&dev_replace->num_uncorrectable_read_errors, + btrfs_dev_replace_num_uncorrectable_read_errors(eb, ptr)); + dev_replace->cursor_left = btrfs_dev_replace_cursor_left(eb, ptr); + dev_replace->committed_cursor_left = dev_replace->cursor_left; + dev_replace->cursor_left_last_write_of_item = dev_replace->cursor_left; + dev_replace->cursor_right = btrfs_dev_replace_cursor_right(eb, ptr); + dev_replace->is_valid = 1; + + dev_replace->item_needs_writeback = 0; + switch (dev_replace->replace_state) { + case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED: + case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED: + case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED: + /* + * We don't have an active replace item but if there is a + * replace target, fail the mount. + */ + if (unlikely(btrfs_find_device(fs_info->fs_devices, &args))) { + btrfs_err(fs_info, +"replace without active item, run 'device scan --forget' on the target device"); + ret = -EUCLEAN; + } else { + dev_replace->srcdev = NULL; + dev_replace->tgtdev = NULL; + } + break; + case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED: + case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED: + dev_replace->tgtdev = btrfs_find_device(fs_info->fs_devices, &args); + args.devid = src_devid; + dev_replace->srcdev = btrfs_find_device(fs_info->fs_devices, &args); + + /* + * allow 'btrfs dev replace_cancel' if src/tgt device is + * missing + */ + if (unlikely(!dev_replace->srcdev && !btrfs_test_opt(fs_info, DEGRADED))) { + ret = -EIO; + btrfs_warn(fs_info, + "cannot mount because device replace operation is ongoing and"); + btrfs_warn(fs_info, + "srcdev (devid %llu) is missing, need to run 'btrfs dev scan'?", + src_devid); + } + if (unlikely(!dev_replace->tgtdev && !btrfs_test_opt(fs_info, DEGRADED))) { + ret = -EIO; + btrfs_warn(fs_info, + "cannot mount because device replace operation is ongoing and"); + btrfs_warn(fs_info, + "tgtdev (devid %llu) is missing, need to run 'btrfs dev scan'?", + BTRFS_DEV_REPLACE_DEVID); + } + if (dev_replace->tgtdev) { + if (dev_replace->srcdev) { + dev_replace->tgtdev->total_bytes = + dev_replace->srcdev->total_bytes; + dev_replace->tgtdev->disk_total_bytes = + dev_replace->srcdev->disk_total_bytes; + dev_replace->tgtdev->commit_total_bytes = + dev_replace->srcdev->commit_total_bytes; + dev_replace->tgtdev->bytes_used = + dev_replace->srcdev->bytes_used; + dev_replace->tgtdev->commit_bytes_used = + dev_replace->srcdev->commit_bytes_used; + } + set_bit(BTRFS_DEV_STATE_REPLACE_TGT, + &dev_replace->tgtdev->dev_state); + + WARN_ON(fs_info->fs_devices->rw_devices == 0); + dev_replace->tgtdev->io_width = fs_info->sectorsize; + dev_replace->tgtdev->io_align = fs_info->sectorsize; + dev_replace->tgtdev->sector_size = fs_info->sectorsize; + dev_replace->tgtdev->fs_info = fs_info; + set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, + &dev_replace->tgtdev->dev_state); + } + break; + } + + return ret; +} + +/* + * Initialize a new device for device replace target from a given source dev + * and path. + * + * Return 0 and new device in @device_out, otherwise return < 0 + */ +static int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info, + const char *device_path, + struct btrfs_device *srcdev, + struct btrfs_device **device_out) +{ + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; + struct btrfs_device *device; + struct file *bdev_file; + struct block_device *bdev; + u64 devid = BTRFS_DEV_REPLACE_DEVID; + int ret = 0; + + *device_out = NULL; + if (srcdev->fs_devices->seeding) { + btrfs_err(fs_info, "the filesystem is a seed filesystem!"); + return -EINVAL; + } + + /* Unfreezable for the whole replace; see btrfs_dev_replace_start(). */ + bdev_file = btrfs_open_device_deny_freeze(device_path, fs_info->sb); + if (IS_ERR(bdev_file)) { + btrfs_err(fs_info, "target device %s is invalid!", device_path); + return PTR_ERR(bdev_file); + } + bdev = file_bdev(bdev_file); + + if (!btrfs_check_device_zone_type(fs_info, bdev)) { + btrfs_err(fs_info, + "dev-replace: zoned type of target device mismatch with filesystem"); + ret = -EINVAL; + goto error; + } + + sync_blockdev(bdev); + + list_for_each_entry(device, &fs_devices->devices, dev_list) { + if (device->bdev == bdev) { + btrfs_err(fs_info, + "target device is in the filesystem!"); + ret = -EEXIST; + goto error; + } + } + + + if (bdev_nr_bytes(bdev) < btrfs_device_get_total_bytes(srcdev)) { + btrfs_err(fs_info, + "target device is smaller than source device!"); + ret = -EINVAL; + goto error; + } + + + device = btrfs_alloc_device(NULL, &devid, NULL, device_path); + if (IS_ERR(device)) { + ret = PTR_ERR(device); + goto error; + } + + ret = lookup_bdev(device_path, &device->devt); + if (ret) + goto error; + + set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state); + device->generation = 0; + device->io_width = fs_info->sectorsize; + device->io_align = fs_info->sectorsize; + device->sector_size = fs_info->sectorsize; + device->total_bytes = btrfs_device_get_total_bytes(srcdev); + device->disk_total_bytes = btrfs_device_get_disk_total_bytes(srcdev); + device->bytes_used = btrfs_device_get_bytes_used(srcdev); + device->commit_total_bytes = srcdev->commit_total_bytes; + device->commit_bytes_used = device->bytes_used; + device->fs_info = fs_info; + device->bdev = bdev; + device->bdev_file = bdev_file; + set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state); + set_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state); + /* Check the comment in btrfs_init_new_device() for the reason. */ + atomic_inc(&device->dev_stats_ccnt); + device->dev_stats_valid = 1; + set_blocksize(bdev_file, BTRFS_BDEV_BLOCKSIZE); + device->fs_devices = fs_devices; + + ret = btrfs_get_dev_zone_info(device, false); + if (ret) + goto error; + + mutex_lock(&fs_devices->device_list_mutex); + list_add(&device->dev_list, &fs_devices->devices); + fs_devices->num_devices++; + fs_devices->open_devices++; + mutex_unlock(&fs_devices->device_list_mutex); + + *device_out = device; + return 0; + +error: + /* Undo the open-time freeze deny. */ + btrfs_release_device_allow_freeze(bdev_file); + return ret; +} + +/* + * called from commit_transaction. Writes changed device replace state to + * disk. + */ +int btrfs_run_dev_replace(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + int ret; + struct btrfs_root *dev_root = fs_info->dev_root; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + struct extent_buffer *eb; + struct btrfs_dev_replace_item *ptr; + struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; + + down_read(&dev_replace->rwsem); + if (!dev_replace->is_valid || + !dev_replace->item_needs_writeback) { + up_read(&dev_replace->rwsem); + return 0; + } + up_read(&dev_replace->rwsem); + + key.objectid = 0; + key.type = BTRFS_DEV_REPLACE_KEY; + key.offset = 0; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + ret = btrfs_search_slot(trans, dev_root, &key, path, -1, 1); + if (ret < 0) { + btrfs_warn(fs_info, + "error %d while searching for dev_replace item!", + ret); + return ret; + } + + if (ret == 0 && + btrfs_item_size(path->nodes[0], path->slots[0]) < sizeof(*ptr)) { + /* + * need to delete old one and insert a new one. + * Since no attempt is made to recover any old state, if the + * dev_replace state is 'running', the data on the target + * drive is lost. + * It would be possible to recover the state: just make sure + * that the beginning of the item is never changed and always + * contains all the essential information. Then read this + * minimal set of information and use it as a base for the + * new state. + */ + ret = btrfs_del_item(trans, dev_root, path); + if (ret != 0) { + btrfs_warn(fs_info, + "delete too small dev_replace item failed %d!", + ret); + return ret; + } + ret = 1; + } + + if (ret == 1) { + /* need to insert a new item */ + btrfs_release_path(path); + ret = btrfs_insert_empty_item(trans, dev_root, path, + &key, sizeof(*ptr)); + if (ret < 0) { + btrfs_warn(fs_info, + "insert dev_replace item failed %d!", ret); + return ret; + } + } + + eb = path->nodes[0]; + ptr = btrfs_item_ptr(eb, path->slots[0], + struct btrfs_dev_replace_item); + + down_write(&dev_replace->rwsem); + if (dev_replace->srcdev) + btrfs_set_dev_replace_src_devid(eb, ptr, + dev_replace->srcdev->devid); + else + btrfs_set_dev_replace_src_devid(eb, ptr, (u64)-1); + btrfs_set_dev_replace_cont_reading_from_srcdev_mode(eb, ptr, + dev_replace->cont_reading_from_srcdev_mode); + btrfs_set_dev_replace_replace_state(eb, ptr, + dev_replace->replace_state); + btrfs_set_dev_replace_time_started(eb, ptr, dev_replace->time_started); + btrfs_set_dev_replace_time_stopped(eb, ptr, dev_replace->time_stopped); + btrfs_set_dev_replace_num_write_errors(eb, ptr, + atomic64_read(&dev_replace->num_write_errors)); + btrfs_set_dev_replace_num_uncorrectable_read_errors(eb, ptr, + atomic64_read(&dev_replace->num_uncorrectable_read_errors)); + dev_replace->cursor_left_last_write_of_item = + dev_replace->cursor_left; + btrfs_set_dev_replace_cursor_left(eb, ptr, + dev_replace->cursor_left_last_write_of_item); + btrfs_set_dev_replace_cursor_right(eb, ptr, + dev_replace->cursor_right); + dev_replace->item_needs_writeback = 0; + up_write(&dev_replace->rwsem); + + return ret; +} + +static int mark_block_group_to_copy(struct btrfs_fs_info *fs_info, + struct btrfs_device *src_dev) +{ + struct btrfs_path *path; + struct btrfs_key key; + struct btrfs_key found_key; + struct btrfs_root *root = fs_info->dev_root; + struct btrfs_dev_extent *dev_extent = NULL; + struct btrfs_block_group *cache; + struct btrfs_trans_handle *trans; + int iter_ret = 0; + int ret = 0; + u64 chunk_offset; + + /* Do not use "to_copy" on non zoned filesystem for now */ + if (!btrfs_is_zoned(fs_info)) + return 0; + + mutex_lock(&fs_info->chunk_mutex); + + /* Ensure we don't have pending new block group */ + spin_lock(&fs_info->trans_lock); + while (fs_info->running_transaction && + !list_empty(&fs_info->running_transaction->dev_update_list)) { + spin_unlock(&fs_info->trans_lock); + mutex_unlock(&fs_info->chunk_mutex); + trans = btrfs_attach_transaction(root); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + mutex_lock(&fs_info->chunk_mutex); + if (ret == -ENOENT) { + spin_lock(&fs_info->trans_lock); + continue; + } else { + goto unlock; + } + } + + ret = btrfs_commit_transaction(trans); + mutex_lock(&fs_info->chunk_mutex); + if (ret) + goto unlock; + + spin_lock(&fs_info->trans_lock); + } + spin_unlock(&fs_info->trans_lock); + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto unlock; + } + + path->reada = READA_FORWARD; + path->search_commit_root = true; + path->skip_locking = true; + + key.objectid = src_dev->devid; + key.type = BTRFS_DEV_EXTENT_KEY; + key.offset = 0; + + btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) { + struct extent_buffer *leaf = path->nodes[0]; + + if (found_key.objectid != src_dev->devid) + break; + + if (found_key.type != BTRFS_DEV_EXTENT_KEY) + break; + + if (found_key.offset < key.offset) + break; + + dev_extent = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_extent); + + chunk_offset = btrfs_dev_extent_chunk_offset(leaf, dev_extent); + + cache = btrfs_lookup_block_group(fs_info, chunk_offset); + if (!cache) + continue; + + set_bit(BLOCK_GROUP_FLAG_TO_COPY, &cache->runtime_flags); + btrfs_put_block_group(cache); + } + if (iter_ret < 0) + ret = iter_ret; + + btrfs_free_path(path); +unlock: + mutex_unlock(&fs_info->chunk_mutex); + + return ret; +} + +bool btrfs_finish_block_group_to_copy(struct btrfs_device *srcdev, + struct btrfs_block_group *cache, + u64 physical) +{ + struct btrfs_fs_info *fs_info = cache->fs_info; + struct btrfs_chunk_map *map; + u64 chunk_offset = cache->start; + int num_extents, cur_extent; + int i; + + /* Do not use "to_copy" on non zoned filesystem for now */ + if (!btrfs_is_zoned(fs_info)) + return true; + + spin_lock(&cache->lock); + if (test_bit(BLOCK_GROUP_FLAG_REMOVED, &cache->runtime_flags)) { + spin_unlock(&cache->lock); + return true; + } + spin_unlock(&cache->lock); + + map = btrfs_get_chunk_map(fs_info, chunk_offset, 1); + ASSERT(!IS_ERR(map)); + + num_extents = 0; + cur_extent = 0; + for (i = 0; i < map->num_stripes; i++) { + /* We have more device extent to copy */ + if (srcdev != map->stripes[i].dev) + continue; + + num_extents++; + if (physical == map->stripes[i].physical) + cur_extent = i; + } + + btrfs_free_chunk_map(map); + + if (num_extents > 1 && cur_extent < num_extents - 1) { + /* + * Has more stripes on this device. Keep this block group + * readonly until we finish all the stripes. + */ + return false; + } + + /* Last stripe on this device */ + clear_bit(BLOCK_GROUP_FLAG_TO_COPY, &cache->runtime_flags); + + return true; +} + +static int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info, + const char *tgtdev_name, u64 srcdevid, const char *srcdev_name, + int read_src) +{ + struct btrfs_root *root = fs_info->dev_root; + struct btrfs_trans_handle *trans; + struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; + int ret; + struct btrfs_device *tgt_device = NULL; + struct btrfs_device *src_device = NULL; + + src_device = btrfs_find_device_by_devspec(fs_info, srcdevid, + srcdev_name); + if (IS_ERR(src_device)) + return PTR_ERR(src_device); + + if (btrfs_pinned_by_swapfile(fs_info, src_device)) { + btrfs_warn(fs_info, + "cannot replace device %s (devid %llu) due to active swapfile", + btrfs_dev_name(src_device), src_device->devid); + return -ETXTBSY; + } + + /* + * Here we commit the transaction to make sure commit_total_bytes + * of all the devices are updated. + */ + trans = btrfs_attach_transaction(root); + if (!IS_ERR(trans)) { + ret = btrfs_commit_transaction(trans); + if (ret) + return ret; + } else if (PTR_ERR(trans) != -ENOENT) { + return PTR_ERR(trans); + } + + ret = btrfs_init_dev_replace_tgtdev(fs_info, tgtdev_name, + src_device, &tgt_device); + if (ret) + return ret; + + /* Deny the source before mark, so every 'leave' unwinds both denied. */ + if (src_device->bdev) { + ret = bdev_deny_freeze(src_device->bdev); + if (ret) { + btrfs_destroy_dev_replace_tgtdev(tgt_device, true); + return ret; + } + } + + ret = mark_block_group_to_copy(fs_info, src_device); + if (ret) + return ret; + + down_write(&dev_replace->rwsem); + dev_replace->replace_task = current; + switch (dev_replace->replace_state) { + case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED: + case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED: + case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED: + break; + case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED: + case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED: + DEBUG_WARN("unexpected STARTED or SUSPENDED dev-replace state"); + ret = BTRFS_IOCTL_DEV_REPLACE_RESULT_ALREADY_STARTED; + up_write(&dev_replace->rwsem); + goto leave; + } + + dev_replace->cont_reading_from_srcdev_mode = read_src; + dev_replace->srcdev = src_device; + dev_replace->tgtdev = tgt_device; + + btrfs_info(fs_info, + "dev_replace from %s (devid %llu) to %s started", + btrfs_dev_name(src_device), + src_device->devid, + btrfs_dev_name(tgt_device)); + + /* + * from now on, the writes to the srcdev are all duplicated to + * go to the tgtdev as well (refer to btrfs_map_block()). + */ + dev_replace->replace_state = BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED; + dev_replace->time_started = ktime_get_real_seconds(); + dev_replace->cursor_left = 0; + dev_replace->committed_cursor_left = 0; + dev_replace->cursor_left_last_write_of_item = 0; + dev_replace->cursor_right = 0; + dev_replace->is_valid = 1; + dev_replace->item_needs_writeback = 1; + atomic64_set(&dev_replace->num_write_errors, 0); + atomic64_set(&dev_replace->num_uncorrectable_read_errors, 0); + up_write(&dev_replace->rwsem); + + ret = btrfs_sysfs_add_device(tgt_device); + if (ret) + btrfs_err(fs_info, "kobj add dev failed %d", ret); + + btrfs_wait_ordered_roots(fs_info, U64_MAX, NULL); + + /* + * Commit dev_replace state and reserve 1 item for it. + * This is crucial to ensure we won't miss copying extents for new block + * groups that are allocated after we started the device replace, and + * must be done after setting up the device replace state. + */ + trans = btrfs_start_transaction(root, 1); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + down_write(&dev_replace->rwsem); + dev_replace->replace_state = + BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED; + dev_replace->srcdev = NULL; + dev_replace->tgtdev = NULL; + up_write(&dev_replace->rwsem); + goto leave; + } + + ret = btrfs_commit_transaction(trans); + WARN_ON(ret); + + /* the disk copy procedure reuses the scrub code */ + ret = btrfs_scrub_dev(fs_info, src_device->devid, 0, + btrfs_device_get_total_bytes(src_device), + &dev_replace->scrub_progress, false, true); + + ret = btrfs_dev_replace_finishing(fs_info, ret); + if (ret == -EINPROGRESS) + ret = BTRFS_IOCTL_DEV_REPLACE_RESULT_SCRUB_INPROGRESS; + + return ret; + +leave: + if (src_device->bdev) + bdev_allow_freeze(src_device->bdev); + btrfs_destroy_dev_replace_tgtdev(tgt_device, true); + return ret; +} + +static int btrfs_check_replace_dev_names(struct btrfs_ioctl_dev_replace_args *args) +{ + if (args->start.srcdevid == 0) { + if (memchr(args->start.srcdev_name, 0, + sizeof(args->start.srcdev_name)) == NULL) + return -ENAMETOOLONG; + } else { + args->start.srcdev_name[0] = 0; + } + + if (memchr(args->start.tgtdev_name, 0, + sizeof(args->start.tgtdev_name)) == NULL) + return -ENAMETOOLONG; + + return 0; +} + +int btrfs_dev_replace_by_ioctl(struct btrfs_fs_info *fs_info, + struct btrfs_ioctl_dev_replace_args *args) +{ + int ret; + + switch (args->start.cont_reading_from_srcdev_mode) { + case BTRFS_IOCTL_DEV_REPLACE_CONT_READING_FROM_SRCDEV_MODE_ALWAYS: + case BTRFS_IOCTL_DEV_REPLACE_CONT_READING_FROM_SRCDEV_MODE_AVOID: + break; + default: + return -EINVAL; + } + ret = btrfs_check_replace_dev_names(args); + if (ret < 0) + return ret; + + ret = btrfs_dev_replace_start(fs_info, args->start.tgtdev_name, + args->start.srcdevid, + args->start.srcdev_name, + args->start.cont_reading_from_srcdev_mode); + args->result = ret; + /* don't warn if EINPROGRESS, someone else might be running scrub */ + if (ret == BTRFS_IOCTL_DEV_REPLACE_RESULT_SCRUB_INPROGRESS || + ret == BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR) + return 0; + + return ret; +} + +/* + * blocked until all in-flight bios operations are finished. + */ +static void btrfs_rm_dev_replace_blocked(struct btrfs_fs_info *fs_info) +{ + set_bit(BTRFS_FS_STATE_DEV_REPLACING, &fs_info->fs_state); + wait_event(fs_info->dev_replace.replace_wait, !percpu_counter_sum( + &fs_info->dev_replace.bio_counter)); +} + +/* + * we have removed target device, it is safe to allow new bios request. + */ +static void btrfs_rm_dev_replace_unblocked(struct btrfs_fs_info *fs_info) +{ + clear_bit(BTRFS_FS_STATE_DEV_REPLACING, &fs_info->fs_state); + wake_up(&fs_info->dev_replace.replace_wait); +} + +/* + * When finishing the device replace, before swapping the source device with the + * target device we must update the chunk allocation state in the target device, + * as it is empty because replace works by directly copying the chunks and not + * through the normal chunk allocation path. + */ +static int btrfs_set_target_alloc_state(struct btrfs_device *srcdev, + struct btrfs_device *tgtdev) +{ + struct extent_state *cached_state = NULL; + u64 start = 0; + u64 found_start; + u64 found_end; + int ret = 0; + + lockdep_assert_held(&srcdev->fs_info->chunk_mutex); + + while (btrfs_find_first_extent_bit(&srcdev->alloc_state, start, + &found_start, &found_end, + CHUNK_ALLOCATED, &cached_state)) { + ret = btrfs_set_extent_bit(&tgtdev->alloc_state, found_start, + found_end, CHUNK_ALLOCATED, NULL); + if (ret) + break; + start = found_end + 1; + } + + btrfs_free_extent_state(cached_state); + return ret; +} + +static void btrfs_dev_replace_update_device_in_mapping_tree( + struct btrfs_fs_info *fs_info, + struct btrfs_device *srcdev, + struct btrfs_device *tgtdev) +{ + struct rb_node *node; + + /* + * The chunk mutex must be held so that no new chunks can be created + * while we are updating existing chunks. This guarantees we don't miss + * any new chunk that gets created for a range that falls before the + * range of the last chunk we processed. + */ + lockdep_assert_held(&fs_info->chunk_mutex); + + write_lock(&fs_info->mapping_tree_lock); + node = rb_first_cached(&fs_info->mapping_tree); + while (node) { + struct rb_node *next = rb_next(node); + struct btrfs_chunk_map *map; + u64 next_start; + + map = rb_entry(node, struct btrfs_chunk_map, rb_node); + next_start = map->start + map->chunk_len; + + for (int i = 0; i < map->num_stripes; i++) + if (srcdev == map->stripes[i].dev) + map->stripes[i].dev = tgtdev; + + if (cond_resched_rwlock_write(&fs_info->mapping_tree_lock)) { + map = btrfs_find_chunk_map_nolock(fs_info, next_start, U64_MAX); + if (!map) + break; + node = &map->rb_node; + /* + * Drop the lookup reference since we are holding the + * lock in write mode and no one can remove the chunk + * map from the tree and drop its tree reference. + */ + btrfs_free_chunk_map(map); + } else { + node = next; + } + } + write_unlock(&fs_info->mapping_tree_lock); +} + +static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info, + int scrub_ret) +{ + struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; + struct btrfs_device *tgt_device; + struct btrfs_device *src_device; + struct btrfs_root *root = fs_info->tree_root; + u8 uuid_tmp[BTRFS_UUID_SIZE]; + struct btrfs_trans_handle *trans; + int ret = 0; + + /* don't allow cancel or unmount to disturb the finishing procedure */ + mutex_lock(&dev_replace->lock_finishing_cancel_unmount); + + down_read(&dev_replace->rwsem); + /* was the operation canceled, or is it finished? */ + if (dev_replace->replace_state != + BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED) { + up_read(&dev_replace->rwsem); + mutex_unlock(&dev_replace->lock_finishing_cancel_unmount); + return 0; + } + + tgt_device = dev_replace->tgtdev; + src_device = dev_replace->srcdev; + up_read(&dev_replace->rwsem); + + /* + * flush all outstanding I/O and inode extent mappings before the + * copy operation is declared as being finished + */ + ret = btrfs_start_delalloc_roots(fs_info, LONG_MAX, false); + if (ret) { + /* Stays started/resumable; keep both denied. */ + mutex_unlock(&dev_replace->lock_finishing_cancel_unmount); + return ret; + } + btrfs_wait_ordered_roots(fs_info, U64_MAX, NULL); + + /* + * We have to use this loop approach because at this point src_device + * has to be available for transaction commit to complete, yet new + * chunks shouldn't be allocated on the device. + */ + while (1) { + trans = btrfs_start_transaction(root, 0); + if (IS_ERR(trans)) { + /* Stays started/resumable; keep both denied. */ + mutex_unlock(&dev_replace->lock_finishing_cancel_unmount); + return PTR_ERR(trans); + } + ret = btrfs_commit_transaction(trans); + WARN_ON(ret); + + /* Prevent write_all_supers() during the finishing procedure */ + mutex_lock(&fs_devices->device_list_mutex); + /* Prevent new chunks being allocated on the source device */ + mutex_lock(&fs_info->chunk_mutex); + + if (!list_empty(&src_device->post_commit_list)) { + mutex_unlock(&fs_devices->device_list_mutex); + mutex_unlock(&fs_info->chunk_mutex); + } else { + break; + } + } + + down_write(&dev_replace->rwsem); + dev_replace->replace_state = + scrub_ret ? BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED + : BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED; + dev_replace->tgtdev = NULL; + dev_replace->srcdev = NULL; + dev_replace->time_stopped = ktime_get_real_seconds(); + dev_replace->item_needs_writeback = 1; + + /* + * Update allocation state in the new device and replace the old device + * with the new one in the mapping tree. + */ + if (!scrub_ret) { + scrub_ret = btrfs_set_target_alloc_state(src_device, tgt_device); + if (scrub_ret) + goto error; + btrfs_dev_replace_update_device_in_mapping_tree(fs_info, + src_device, + tgt_device); + } else { + if (scrub_ret != -ECANCELED) + btrfs_err(fs_info, + "btrfs_scrub_dev(%s, %llu, %s) failed %d", + btrfs_dev_name(src_device), + src_device->devid, + btrfs_dev_name(tgt_device), scrub_ret); +error: + up_write(&dev_replace->rwsem); + mutex_unlock(&fs_info->chunk_mutex); + mutex_unlock(&fs_devices->device_list_mutex); + btrfs_rm_dev_replace_blocked(fs_info); + if (tgt_device) + btrfs_destroy_dev_replace_tgtdev(tgt_device, true); + /* The source stays a member; re-allow freezing it. */ + if (src_device->bdev) + bdev_allow_freeze(src_device->bdev); + btrfs_rm_dev_replace_unblocked(fs_info); + mutex_unlock(&dev_replace->lock_finishing_cancel_unmount); + + return scrub_ret; + } + + btrfs_info(fs_info, + "dev_replace from %s (devid %llu) to %s finished", + btrfs_dev_name(src_device), + src_device->devid, + btrfs_dev_name(tgt_device)); + clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &tgt_device->dev_state); + tgt_device->devid = src_device->devid; + src_device->devid = BTRFS_DEV_REPLACE_DEVID; + memcpy(uuid_tmp, tgt_device->uuid, sizeof(uuid_tmp)); + memcpy(tgt_device->uuid, src_device->uuid, sizeof(tgt_device->uuid)); + memcpy(src_device->uuid, uuid_tmp, sizeof(src_device->uuid)); + btrfs_device_set_total_bytes(tgt_device, src_device->total_bytes); + btrfs_device_set_disk_total_bytes(tgt_device, + src_device->disk_total_bytes); + btrfs_device_set_bytes_used(tgt_device, src_device->bytes_used); + tgt_device->commit_bytes_used = src_device->bytes_used; + + btrfs_assign_next_active_device(src_device, tgt_device); + + list_add(&tgt_device->dev_alloc_list, &fs_devices->alloc_list); + fs_devices->rw_devices++; + + dev_replace->replace_task = NULL; + up_write(&dev_replace->rwsem); + btrfs_rm_dev_replace_blocked(fs_info); + + btrfs_rm_dev_replace_remove_srcdev(src_device); + + btrfs_rm_dev_replace_unblocked(fs_info); + + /* + * Increment dev_stats_ccnt so that btrfs_run_dev_stats() will + * update on-disk dev stats value during commit transaction + */ + atomic_inc(&tgt_device->dev_stats_ccnt); + + /* + * this is again a consistent state where no dev_replace procedure + * is running, the target device is part of the filesystem, the + * source device is not part of the filesystem anymore and its 1st + * superblock is scratched out so that it is no longer marked to + * belong to this filesystem. + */ + mutex_unlock(&fs_info->chunk_mutex); + mutex_unlock(&fs_devices->device_list_mutex); + + /* replace the sysfs entry */ + btrfs_sysfs_remove_device(src_device); + btrfs_sysfs_update_devid(tgt_device); + if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &src_device->dev_state)) + btrfs_scratch_superblocks(fs_info, src_device); + + /* write back the superblocks */ + trans = btrfs_start_transaction(root, 0); + if (!IS_ERR(trans)) { + /* + * Ignore any error here, if we failed to remove the DEV_STATS + * item for devid 0, it's not a big deal. We have other ways + * to address it. + */ + btrfs_remove_dev_stat_item(trans, BTRFS_DEV_REPLACE_DEVID); + btrfs_commit_transaction(trans); + } + + mutex_unlock(&dev_replace->lock_finishing_cancel_unmount); + + /* The target is now a member; the source is freed (allow + release). */ + bdev_allow_freeze(tgt_device->bdev); + btrfs_rm_dev_replace_free_srcdev(src_device); + + return 0; +} + +/* + * Read progress of device replace status according to the state and last + * stored position. The value format is the same as for + * btrfs_dev_replace::progress_1000 + */ +static u64 btrfs_dev_replace_progress(struct btrfs_fs_info *fs_info) +{ + struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; + u64 ret = 0; + + switch (dev_replace->replace_state) { + case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED: + case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED: + ret = 0; + break; + case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED: + ret = 1000; + break; + case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED: + case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED: + ret = div64_u64(dev_replace->cursor_left, + div_u64(btrfs_device_get_total_bytes( + dev_replace->srcdev), 1000)); + break; + } + + return ret; +} + +void btrfs_dev_replace_status(struct btrfs_fs_info *fs_info, + struct btrfs_ioctl_dev_replace_args *args) +{ + struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; + + down_read(&dev_replace->rwsem); + /* even if !dev_replace_is_valid, the values are good enough for + * the replace_status ioctl */ + args->result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR; + args->status.replace_state = dev_replace->replace_state; + args->status.time_started = dev_replace->time_started; + args->status.time_stopped = dev_replace->time_stopped; + args->status.num_write_errors = + atomic64_read(&dev_replace->num_write_errors); + args->status.num_uncorrectable_read_errors = + atomic64_read(&dev_replace->num_uncorrectable_read_errors); + args->status.progress_1000 = btrfs_dev_replace_progress(fs_info); + up_read(&dev_replace->rwsem); +} + +int btrfs_dev_replace_cancel(struct btrfs_fs_info *fs_info) +{ + struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; + struct btrfs_device *tgt_device = NULL; + struct btrfs_device *src_device = NULL; + struct btrfs_trans_handle *trans; + struct btrfs_root *root = fs_info->tree_root; + int result; + int ret; + + if (sb_rdonly(fs_info->sb)) + return -EROFS; + + mutex_lock(&dev_replace->lock_finishing_cancel_unmount); + down_write(&dev_replace->rwsem); + switch (dev_replace->replace_state) { + case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED: + case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED: + case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED: + result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NOT_STARTED; + up_write(&dev_replace->rwsem); + break; + case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED: + tgt_device = dev_replace->tgtdev; + src_device = dev_replace->srcdev; + up_write(&dev_replace->rwsem); + ret = btrfs_scrub_cancel(fs_info); + if (ret < 0) { + result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NOT_STARTED; + } else { + result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR; + /* + * btrfs_dev_replace_finishing() will handle the + * cleanup part + */ + btrfs_info(fs_info, + "dev_replace from %s (devid %llu) to %s canceled", + btrfs_dev_name(src_device), src_device->devid, + btrfs_dev_name(tgt_device)); + } + break; + case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED: + /* + * Scrub doing the replace isn't running so we need to do the + * cleanup step of btrfs_dev_replace_finishing() here + */ + result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR; + tgt_device = dev_replace->tgtdev; + src_device = dev_replace->srcdev; + dev_replace->tgtdev = NULL; + dev_replace->srcdev = NULL; + dev_replace->replace_state = + BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED; + dev_replace->time_stopped = ktime_get_real_seconds(); + dev_replace->item_needs_writeback = 1; + + up_write(&dev_replace->rwsem); + + /* Scrub for replace must not be running in suspended state */ + btrfs_scrub_cancel(fs_info); + + trans = btrfs_start_transaction(root, 0); + if (IS_ERR(trans)) { + mutex_unlock(&dev_replace->lock_finishing_cancel_unmount); + return PTR_ERR(trans); + } + ret = btrfs_commit_transaction(trans); + WARN_ON(ret); + + btrfs_info(fs_info, + "suspended dev_replace from %s (devid %llu) to %s canceled", + btrfs_dev_name(src_device), src_device->devid, + btrfs_dev_name(tgt_device)); + + /* A suspended replace never re-denied freezing; do not allow. */ + if (tgt_device) + btrfs_destroy_dev_replace_tgtdev(tgt_device, false); + break; + default: + up_write(&dev_replace->rwsem); + result = -EINVAL; + } + + mutex_unlock(&dev_replace->lock_finishing_cancel_unmount); + return result; +} + +void btrfs_dev_replace_suspend_for_unmount(struct btrfs_fs_info *fs_info) +{ + struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; + + mutex_lock(&dev_replace->lock_finishing_cancel_unmount); + down_write(&dev_replace->rwsem); + + switch (dev_replace->replace_state) { + case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED: + case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED: + case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED: + case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED: + break; + case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED: + dev_replace->replace_state = + BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED; + dev_replace->time_stopped = ktime_get_real_seconds(); + dev_replace->item_needs_writeback = 1; + btrfs_info(fs_info, "suspending dev_replace for unmount"); + /* Reopened freezable next mount; resume re-denies. */ + if (dev_replace->srcdev && dev_replace->srcdev->bdev) + bdev_allow_freeze(dev_replace->srcdev->bdev); + if (dev_replace->tgtdev && dev_replace->tgtdev->bdev) + bdev_allow_freeze(dev_replace->tgtdev->bdev); + break; + } + + up_write(&dev_replace->rwsem); + mutex_unlock(&dev_replace->lock_finishing_cancel_unmount); +} + +/* resume dev_replace procedure that was interrupted by unmount */ +int btrfs_resume_dev_replace_async(struct btrfs_fs_info *fs_info) +{ + struct task_struct *task; + struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; + int ret = 0; + + down_write(&dev_replace->rwsem); + + switch (dev_replace->replace_state) { + case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED: + case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED: + case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED: + up_write(&dev_replace->rwsem); + return 0; + case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED: + break; + case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED: + dev_replace->replace_state = + BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED; + break; + } + if (!dev_replace->tgtdev || !dev_replace->tgtdev->bdev) { + btrfs_info(fs_info, + "cannot continue dev_replace, tgtdev is missing"); + btrfs_info(fs_info, + "you may cancel the operation after 'mount -o degraded'"); + dev_replace->replace_state = + BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED; + up_write(&dev_replace->rwsem); + return 0; + } + up_write(&dev_replace->rwsem); + + /* + * This could collide with a paused balance, but the exclusive op logic + * should never allow both to start and pause. We don't want to allow + * dev-replace to start anyway. + */ + if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_DEV_REPLACE)) { + down_write(&dev_replace->rwsem); + dev_replace->replace_state = + BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED; + up_write(&dev_replace->rwsem); + btrfs_info(fs_info, + "cannot resume dev-replace, other exclusive operation running"); + return 0; + } + + /* Re-deny for the resumed replace; stay suspended if frozen now. */ + if (dev_replace->srcdev->bdev && + bdev_deny_freeze(dev_replace->srcdev->bdev)) + goto suspend; + if (bdev_deny_freeze(dev_replace->tgtdev->bdev)) { + if (dev_replace->srcdev->bdev) + bdev_allow_freeze(dev_replace->srcdev->bdev); + goto suspend; + } + + task = kthread_run(btrfs_dev_replace_kthread, fs_info, "btrfs-devrepl"); + if (IS_ERR(task)) { + bdev_allow_freeze(dev_replace->tgtdev->bdev); + if (dev_replace->srcdev->bdev) + bdev_allow_freeze(dev_replace->srcdev->bdev); + /* Undo the deny and suspend, but still fail the mount. */ + ret = PTR_ERR(task); + goto suspend; + } + return 0; + +suspend: + btrfs_exclop_finish(fs_info); + down_write(&dev_replace->rwsem); + dev_replace->replace_state = BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED; + up_write(&dev_replace->rwsem); + return ret; +} + +static int btrfs_dev_replace_kthread(void *data) +{ + struct btrfs_fs_info *fs_info = data; + struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; + u64 progress; + int ret; + + progress = btrfs_dev_replace_progress(fs_info); + progress = div_u64(progress, 10); + btrfs_info(fs_info, + "continuing dev_replace from %s (devid %llu) to target %s @%u%%", + btrfs_dev_name(dev_replace->srcdev), + dev_replace->srcdev->devid, + btrfs_dev_name(dev_replace->tgtdev), + (unsigned int)progress); + + ret = btrfs_scrub_dev(fs_info, dev_replace->srcdev->devid, + dev_replace->committed_cursor_left, + btrfs_device_get_total_bytes(dev_replace->srcdev), + &dev_replace->scrub_progress, false, true); + ret = btrfs_dev_replace_finishing(fs_info, ret); + WARN_ON(ret && ret != -ECANCELED); + + btrfs_exclop_finish(fs_info); + return 0; +} + +bool __pure btrfs_dev_replace_is_ongoing(struct btrfs_dev_replace *dev_replace) +{ + if (!dev_replace->is_valid) + return false; + + switch (dev_replace->replace_state) { + case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED: + case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED: + case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED: + return false; + case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED: + case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED: + /* + * return true even if tgtdev is missing (this is + * something that can happen if the dev_replace + * procedure is suspended by an umount and then + * the tgtdev is missing (or "btrfs dev scan") was + * not called and the filesystem is remounted + * in degraded state. This does not stop the + * dev_replace procedure. It needs to be canceled + * manually if the cancellation is wanted. + */ + break; + } + return true; +} + +void btrfs_bio_counter_sub(struct btrfs_fs_info *fs_info, s64 amount) +{ + percpu_counter_sub(&fs_info->dev_replace.bio_counter, amount); + cond_wake_up_nomb(&fs_info->dev_replace.replace_wait); +} + +void btrfs_bio_counter_inc_blocked(struct btrfs_fs_info *fs_info) +{ + while (1) { + percpu_counter_inc(&fs_info->dev_replace.bio_counter); + if (likely(!test_bit(BTRFS_FS_STATE_DEV_REPLACING, + &fs_info->fs_state))) + break; + + btrfs_bio_counter_dec(fs_info); + wait_event(fs_info->dev_replace.replace_wait, + !test_bit(BTRFS_FS_STATE_DEV_REPLACING, + &fs_info->fs_state)); + } +} diff --git a/fs/btrfs/dev-replace.h b/fs/btrfs/dev-replace.h new file mode 100644 index 000000000..b35cecf38 --- /dev/null +++ b/fs/btrfs/dev-replace.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) STRATO AG 2012. All rights reserved. + */ + +#ifndef BTRFS_DEV_REPLACE_H +#define BTRFS_DEV_REPLACE_H + +#include +#include + +struct btrfs_ioctl_dev_replace_args; +struct btrfs_fs_info; +struct btrfs_trans_handle; +struct btrfs_dev_replace; +struct btrfs_block_group; +struct btrfs_device; + +int btrfs_init_dev_replace(struct btrfs_fs_info *fs_info); +int btrfs_run_dev_replace(struct btrfs_trans_handle *trans); +int btrfs_dev_replace_by_ioctl(struct btrfs_fs_info *fs_info, + struct btrfs_ioctl_dev_replace_args *args); +void btrfs_dev_replace_status(struct btrfs_fs_info *fs_info, + struct btrfs_ioctl_dev_replace_args *args); +int btrfs_dev_replace_cancel(struct btrfs_fs_info *fs_info); +void btrfs_dev_replace_suspend_for_unmount(struct btrfs_fs_info *fs_info); +int btrfs_resume_dev_replace_async(struct btrfs_fs_info *fs_info); +bool __pure btrfs_dev_replace_is_ongoing(struct btrfs_dev_replace *dev_replace); +bool btrfs_finish_block_group_to_copy(struct btrfs_device *srcdev, + struct btrfs_block_group *cache, + u64 physical); +void btrfs_bio_counter_inc_blocked(struct btrfs_fs_info *fs_info); +void btrfs_bio_counter_sub(struct btrfs_fs_info *fs_info, s64 amount); + +static inline void btrfs_bio_counter_dec(struct btrfs_fs_info *fs_info) +{ + btrfs_bio_counter_sub(fs_info, 1); +} + + +#endif diff --git a/fs/btrfs/dir-item.c b/fs/btrfs/dir-item.c new file mode 100644 index 000000000..84f1c6442 --- /dev/null +++ b/fs/btrfs/dir-item.c @@ -0,0 +1,432 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#include "messages.h" +#include "ctree.h" +#include "disk-io.h" +#include "transaction.h" +#include "accessors.h" +#include "dir-item.h" +#include "delayed-inode.h" + +/* + * insert a name into a directory, doing overflow properly if there is a hash + * collision. data_size indicates how big the item inserted should be. On + * success a struct btrfs_dir_item pointer is returned, otherwise it is + * an ERR_PTR. + * + * The name is not copied into the dir item, you have to do that yourself. + */ +static struct btrfs_dir_item *insert_with_overflow(struct btrfs_trans_handle + *trans, + struct btrfs_root *root, + struct btrfs_path *path, + const struct btrfs_key *cpu_key, + u32 data_size, + const char *name, + int name_len) +{ + int ret; + char *ptr; + struct extent_buffer *leaf; + + ret = btrfs_insert_empty_item(trans, root, path, cpu_key, data_size); + if (ret == -EEXIST) { + struct btrfs_dir_item *di; + di = btrfs_match_dir_item_name(path, name, name_len); + if (di) + return ERR_PTR(-EEXIST); + btrfs_extend_item(trans, path, data_size); + } else if (ret < 0) + return ERR_PTR(ret); + WARN_ON(ret > 0); + leaf = path->nodes[0]; + ptr = btrfs_item_ptr(leaf, path->slots[0], char); + ASSERT(data_size <= btrfs_item_size(leaf, path->slots[0])); + ptr += btrfs_item_size(leaf, path->slots[0]) - data_size; + return (struct btrfs_dir_item *)ptr; +} + +/* + * xattrs work a lot like directories, this inserts an xattr item + * into the tree + */ +int btrfs_insert_xattr_item(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, u64 objectid, + const char *name, u16 name_len, + const void *data, u16 data_len) +{ + int ret = 0; + struct btrfs_dir_item *dir_item; + unsigned long name_ptr, data_ptr; + struct btrfs_key key, location; + struct btrfs_disk_key disk_key; + struct extent_buffer *leaf; + u32 data_size; + + if (name_len + data_len > BTRFS_MAX_XATTR_SIZE(root->fs_info)) + return -ENOSPC; + + key.objectid = objectid; + key.type = BTRFS_XATTR_ITEM_KEY; + key.offset = btrfs_name_hash(name, name_len); + + data_size = sizeof(*dir_item) + name_len + data_len; + dir_item = insert_with_overflow(trans, root, path, &key, data_size, + name, name_len); + if (IS_ERR(dir_item)) + return PTR_ERR(dir_item); + memset(&location, 0, sizeof(location)); + + leaf = path->nodes[0]; + btrfs_cpu_key_to_disk(&disk_key, &location); + btrfs_set_dir_item_key(leaf, dir_item, &disk_key); + btrfs_set_dir_flags(leaf, dir_item, BTRFS_FT_XATTR); + btrfs_set_dir_name_len(leaf, dir_item, name_len); + btrfs_set_dir_transid(leaf, dir_item, trans->transid); + btrfs_set_dir_data_len(leaf, dir_item, data_len); + name_ptr = (unsigned long)(dir_item + 1); + data_ptr = (unsigned long)((char *)name_ptr + name_len); + + write_extent_buffer(leaf, name, name_ptr, name_len); + write_extent_buffer(leaf, data, data_ptr, data_len); + + return ret; +} + +/* + * insert a directory item in the tree, doing all the magic for + * both indexes. 'dir' indicates which objectid to insert it into, + * 'location' is the key to stuff into the directory item, 'type' is the + * type of the inode we're pointing to, and 'index' is the sequence number + * to use for the second index (if one is created). + * Will return 0 or -ENOMEM + */ +int btrfs_insert_dir_item(struct btrfs_trans_handle *trans, + const struct fscrypt_str *name, struct btrfs_inode *dir, + const struct btrfs_key *location, u8 type, u64 index) +{ + int ret = 0; + int ret2 = 0; + struct btrfs_root *root = dir->root; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_dir_item *dir_item; + struct extent_buffer *leaf; + unsigned long name_ptr; + struct btrfs_key key; + struct btrfs_disk_key disk_key; + u32 data_size; + + key.objectid = btrfs_ino(dir); + key.type = BTRFS_DIR_ITEM_KEY; + key.offset = btrfs_name_hash(name->name, name->len); + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + btrfs_cpu_key_to_disk(&disk_key, location); + + data_size = sizeof(*dir_item) + name->len; + dir_item = insert_with_overflow(trans, root, path, &key, data_size, + name->name, name->len); + if (IS_ERR(dir_item)) { + ret = PTR_ERR(dir_item); + if (ret == -EEXIST) + goto second_insert; + goto out_free; + } + + if (IS_ENCRYPTED(&dir->vfs_inode)) + type |= BTRFS_FT_ENCRYPTED; + + leaf = path->nodes[0]; + btrfs_set_dir_item_key(leaf, dir_item, &disk_key); + btrfs_set_dir_flags(leaf, dir_item, type); + btrfs_set_dir_data_len(leaf, dir_item, 0); + btrfs_set_dir_name_len(leaf, dir_item, name->len); + btrfs_set_dir_transid(leaf, dir_item, trans->transid); + name_ptr = (unsigned long)(dir_item + 1); + + write_extent_buffer(leaf, name->name, name_ptr, name->len); + +second_insert: + /* FIXME, use some real flag for selecting the extra index */ + if (root == root->fs_info->tree_root) { + ret = 0; + goto out_free; + } + btrfs_release_path(path); + + ret2 = btrfs_insert_delayed_dir_index(trans, name->name, name->len, dir, + &disk_key, type, index); +out_free: + if (ret) + return ret; + if (ret2) + return ret2; + return 0; +} + +static struct btrfs_dir_item *btrfs_lookup_match_dir( + struct btrfs_trans_handle *trans, + struct btrfs_root *root, struct btrfs_path *path, + struct btrfs_key *key, const char *name, + int name_len, int mod) +{ + const int ins_len = (mod < 0 ? -1 : 0); + const int cow = (mod != 0); + int ret; + + ret = btrfs_search_slot(trans, root, key, path, ins_len, cow); + if (ret < 0) + return ERR_PTR(ret); + if (ret > 0) + return ERR_PTR(-ENOENT); + + return btrfs_match_dir_item_name(path, name, name_len); +} + +/* + * Lookup for a directory item by name. + * + * @trans: The transaction handle to use. Can be NULL if @mod is 0. + * @root: The root of the target tree. + * @path: Path to use for the search. + * @dir: The inode number (objectid) of the directory. + * @name: The name associated to the directory entry we are looking for. + * @name_len: The length of the name. + * @mod: Used to indicate if the tree search is meant for a read only + * lookup, for a modification lookup or for a deletion lookup, so + * its value should be 0, 1 or -1, respectively. + * + * Returns: NULL if the dir item does not exists, an error pointer if an error + * happened, or a pointer to a dir item if a dir item exists for the given name. + */ +struct btrfs_dir_item *btrfs_lookup_dir_item(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, u64 dir, + const struct fscrypt_str *name, + int mod) +{ + struct btrfs_key key; + struct btrfs_dir_item *di; + + key.objectid = dir; + key.type = BTRFS_DIR_ITEM_KEY; + key.offset = btrfs_name_hash(name->name, name->len); + + di = btrfs_lookup_match_dir(trans, root, path, &key, name->name, + name->len, mod); + if (IS_ERR(di) && PTR_ERR(di) == -ENOENT) + return NULL; + + return di; +} + +int btrfs_check_dir_item_collision(struct btrfs_root *root, u64 dir_ino, + const struct fscrypt_str *name) +{ + int ret; + struct btrfs_key key; + struct btrfs_dir_item *di; + int data_size; + struct extent_buffer *leaf; + int slot; + BTRFS_PATH_AUTO_FREE(path); + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = dir_ino; + key.type = BTRFS_DIR_ITEM_KEY; + key.offset = btrfs_name_hash(name->name, name->len); + + di = btrfs_lookup_match_dir(NULL, root, path, &key, name->name, + name->len, 0); + if (IS_ERR(di)) { + ret = PTR_ERR(di); + /* Nothing found, we're safe */ + if (ret == -ENOENT) + return 0; + return ret; + } + + /* we found an item, look for our name in the item */ + if (di) { + /* our exact name was found */ + return -EEXIST; + } + + /* See if there is room in the item to insert this name. */ + data_size = sizeof(*di) + name->len; + leaf = path->nodes[0]; + slot = path->slots[0]; + if (data_size + btrfs_item_size(leaf, slot) + + sizeof(struct btrfs_item) > BTRFS_LEAF_DATA_SIZE(root->fs_info)) { + return -EOVERFLOW; + } + + /* Plenty of insertion room. */ + return 0; +} + +/* + * Lookup for a directory index item by name and index number. + * + * @trans: The transaction handle to use. Can be NULL if @mod is 0. + * @root: The root of the target tree. + * @path: Path to use for the search. + * @dir: The inode number (objectid) of the directory. + * @index: The index number. + * @name: The name associated to the directory entry we are looking for. + * @name_len: The length of the name. + * @mod: Used to indicate if the tree search is meant for a read only + * lookup, for a modification lookup or for a deletion lookup, so + * its value should be 0, 1 or -1, respectively. + * + * Returns: NULL if the dir index item does not exists, an error pointer if an + * error happened, or a pointer to a dir item if the dir index item exists and + * matches the criteria (name and index number). + */ +struct btrfs_dir_item * +btrfs_lookup_dir_index_item(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, u64 dir, + u64 index, const struct fscrypt_str *name, int mod) +{ + struct btrfs_dir_item *di; + struct btrfs_key key; + + key.objectid = dir; + key.type = BTRFS_DIR_INDEX_KEY; + key.offset = index; + + di = btrfs_lookup_match_dir(trans, root, path, &key, name->name, + name->len, mod); + if (di == ERR_PTR(-ENOENT)) + return NULL; + + return di; +} + +struct btrfs_dir_item * +btrfs_search_dir_index_item(struct btrfs_root *root, struct btrfs_path *path, + u64 dirid, const struct fscrypt_str *name) +{ + struct btrfs_dir_item *di; + struct btrfs_key key; + int ret; + + key.objectid = dirid; + key.type = BTRFS_DIR_INDEX_KEY; + key.offset = 0; + + btrfs_for_each_slot(root, &key, &key, path, ret) { + if (key.objectid != dirid || key.type != BTRFS_DIR_INDEX_KEY) + break; + + di = btrfs_match_dir_item_name(path, name->name, name->len); + if (di) + return di; + } + /* Adjust return code if the key was not found in the next leaf. */ + if (ret >= 0) + ret = -ENOENT; + + return ERR_PTR(ret); +} + +struct btrfs_dir_item *btrfs_lookup_xattr(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, u64 dir, + const char *name, u16 name_len, + int mod) +{ + struct btrfs_key key; + struct btrfs_dir_item *di; + + key.objectid = dir; + key.type = BTRFS_XATTR_ITEM_KEY; + key.offset = btrfs_name_hash(name, name_len); + + di = btrfs_lookup_match_dir(trans, root, path, &key, name, name_len, mod); + if (IS_ERR(di) && PTR_ERR(di) == -ENOENT) + return NULL; + + return di; +} + +/* + * helper function to look at the directory item pointed to by 'path' + * this walks through all the entries in a dir item and finds one + * for a specific name. + */ +struct btrfs_dir_item *btrfs_match_dir_item_name(const struct btrfs_path *path, + const char *name, int name_len) +{ + struct btrfs_dir_item *dir_item; + unsigned long name_ptr; + u32 total_len; + u32 cur = 0; + u32 this_len; + struct extent_buffer *leaf; + + leaf = path->nodes[0]; + dir_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dir_item); + + total_len = btrfs_item_size(leaf, path->slots[0]); + while (cur < total_len) { + this_len = sizeof(*dir_item) + + btrfs_dir_name_len(leaf, dir_item) + + btrfs_dir_data_len(leaf, dir_item); + name_ptr = (unsigned long)(dir_item + 1); + + if (btrfs_dir_name_len(leaf, dir_item) == name_len && + memcmp_extent_buffer(leaf, name, name_ptr, name_len) == 0) + return dir_item; + + cur += this_len; + dir_item = (struct btrfs_dir_item *)((char *)dir_item + + this_len); + } + return NULL; +} + +/* + * given a pointer into a directory item, delete it. This + * handles items that have more than one entry in them. + */ +int btrfs_delete_one_dir_name(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + const struct btrfs_dir_item *di) +{ + + struct extent_buffer *leaf; + u32 sub_item_len; + u32 item_len; + int ret = 0; + + leaf = path->nodes[0]; + sub_item_len = sizeof(*di) + btrfs_dir_name_len(leaf, di) + + btrfs_dir_data_len(leaf, di); + item_len = btrfs_item_size(leaf, path->slots[0]); + if (sub_item_len == item_len) { + ret = btrfs_del_item(trans, root, path); + } else { + /* MARKER */ + unsigned long ptr = (unsigned long)di; + unsigned long start; + + start = btrfs_item_ptr_offset(leaf, path->slots[0]); + memmove_extent_buffer(leaf, ptr, ptr + sub_item_len, + item_len - (ptr + sub_item_len - start)); + btrfs_truncate_item(trans, path, item_len - sub_item_len, 1); + } + return ret; +} diff --git a/fs/btrfs/dir-item.h b/fs/btrfs/dir-item.h new file mode 100644 index 000000000..e52174a8b --- /dev/null +++ b/fs/btrfs/dir-item.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_DIR_ITEM_H +#define BTRFS_DIR_ITEM_H + +#include +#include + +struct fscrypt_str; +struct btrfs_fs_info; +struct btrfs_key; +struct btrfs_path; +struct btrfs_inode; +struct btrfs_root; +struct btrfs_trans_handle; + +int btrfs_check_dir_item_collision(struct btrfs_root *root, u64 dir_ino, + const struct fscrypt_str *name); +int btrfs_insert_dir_item(struct btrfs_trans_handle *trans, + const struct fscrypt_str *name, struct btrfs_inode *dir, + const struct btrfs_key *location, u8 type, u64 index); +struct btrfs_dir_item *btrfs_lookup_dir_item(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, u64 dir, + const struct fscrypt_str *name, int mod); +struct btrfs_dir_item *btrfs_lookup_dir_index_item( + struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, u64 dir, + u64 index, const struct fscrypt_str *name, int mod); +struct btrfs_dir_item *btrfs_search_dir_index_item(struct btrfs_root *root, + struct btrfs_path *path, u64 dirid, + const struct fscrypt_str *name); +int btrfs_delete_one_dir_name(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + const struct btrfs_dir_item *di); +int btrfs_insert_xattr_item(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, u64 objectid, + const char *name, u16 name_len, + const void *data, u16 data_len); +struct btrfs_dir_item *btrfs_lookup_xattr(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, u64 dir, + const char *name, u16 name_len, + int mod); +struct btrfs_dir_item *btrfs_match_dir_item_name(const struct btrfs_path *path, + const char *name, + int name_len); + +static inline u64 btrfs_name_hash(const char *name, int len) +{ + return crc32c((u32)~1, name, len); +} + +#endif diff --git a/fs/btrfs/direct-io.c b/fs/btrfs/direct-io.c new file mode 100644 index 000000000..3075d7992 --- /dev/null +++ b/fs/btrfs/direct-io.c @@ -0,0 +1,1164 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include "ctree.h" +#include "delalloc-space.h" +#include "direct-io.h" +#include "extent-tree.h" +#include "file.h" +#include "fs.h" +#include "transaction.h" +#include "volumes.h" +#include "bio.h" +#include "ordered-data.h" + +struct btrfs_dio_data { + loff_t old_isize; + struct extent_changeset *data_reserved; + struct btrfs_ordered_extent *ordered; + bool data_space_reserved; + bool nocow_done; + bool updated_isize; +}; + +struct btrfs_dio_private { + /* Range of I/O */ + u64 file_offset; + u32 bytes; + + /* This must be last */ + struct btrfs_bio bbio; +}; + +static struct bio_set btrfs_dio_bioset; + +static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend, + struct extent_state **cached_state, + unsigned int iomap_flags) +{ + const bool writing = (iomap_flags & IOMAP_WRITE); + const bool nowait = (iomap_flags & IOMAP_NOWAIT); + struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; + struct btrfs_ordered_extent *ordered; + int ret = 0; + + /* Direct lock must be taken before the extent lock. */ + if (nowait) { + if (!btrfs_try_lock_dio_extent(io_tree, lockstart, lockend, cached_state)) + return -EAGAIN; + } else { + btrfs_lock_dio_extent(io_tree, lockstart, lockend, cached_state); + } + + while (1) { + if (nowait) { + if (!btrfs_try_lock_extent(io_tree, lockstart, lockend, + cached_state)) { + ret = -EAGAIN; + break; + } + } else { + btrfs_lock_extent(io_tree, lockstart, lockend, cached_state); + } + /* + * We're concerned with the entire range that we're going to be + * doing DIO to, so we need to make sure there's no ordered + * extents in this range. + */ + ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), lockstart, + lockend - lockstart + 1); + + /* + * We need to make sure there are no buffered pages in this + * range either, we could have raced between the invalidate in + * generic_file_direct_write and locking the extent. The + * invalidate needs to happen so that reads after a write do not + * get stale data. + */ + if (!ordered && + (!writing || !filemap_range_has_page(inode->i_mapping, + lockstart, lockend))) + break; + + btrfs_unlock_extent(io_tree, lockstart, lockend, cached_state); + + if (ordered) { + if (nowait) { + btrfs_put_ordered_extent(ordered); + ret = -EAGAIN; + break; + } + /* + * If we are doing a DIO read and the ordered extent we + * found is for a buffered write, we can not wait for it + * to complete and retry, because if we do so we can + * deadlock with concurrent buffered writes on page + * locks. This happens only if our DIO read covers more + * than one extent map, if at this point has already + * created an ordered extent for a previous extent map + * and locked its range in the inode's io tree, and a + * concurrent write against that previous extent map's + * range and this range started (we unlock the ranges + * in the io tree only when the bios complete and + * buffered writes always lock pages before attempting + * to lock range in the io tree). + */ + if (writing || + test_bit(BTRFS_ORDERED_DIRECT, &ordered->flags)) + btrfs_start_ordered_extent(ordered); + else + ret = -ENOTBLK; + btrfs_put_ordered_extent(ordered); + } else { + /* + * We could trigger writeback for this range (and wait + * for it to complete) and then invalidate the pages for + * this range (through invalidate_inode_pages2_range()), + * but that can lead us to a deadlock with a concurrent + * call to readahead (a buffered read or a defrag call + * triggered a readahead) on a page lock due to an + * ordered dio extent we created before but did not have + * yet a corresponding bio submitted (whence it can not + * complete), which makes readahead wait for that + * ordered extent to complete while holding a lock on + * that page. + */ + ret = nowait ? -EAGAIN : -ENOTBLK; + } + + if (ret) + break; + + cond_resched(); + } + + if (ret) + btrfs_unlock_dio_extent(io_tree, lockstart, lockend, cached_state); + return ret; +} + +static struct extent_map *btrfs_create_dio_extent(struct btrfs_inode *inode, + struct btrfs_dio_data *dio_data, + const u64 start, + const struct btrfs_file_extent *file_extent, + const int type) +{ + struct extent_map *em = NULL; + struct btrfs_ordered_extent *ordered; + + if (type != BTRFS_ORDERED_NOCOW) { + em = btrfs_create_io_em(inode, start, file_extent, type); + if (IS_ERR(em)) + return em; + } + + ordered = btrfs_alloc_ordered_extent(inode, start, file_extent, + (1U << type) | + (1U << BTRFS_ORDERED_DIRECT)); + if (IS_ERR(ordered)) { + if (em) { + btrfs_free_extent_map(em); + btrfs_drop_extent_map_range(inode, start, + start + file_extent->num_bytes - 1, false); + } + em = ERR_CAST(ordered); + } else { + ASSERT(!dio_data->ordered); + dio_data->ordered = ordered; + } + + return em; +} + +static struct extent_map *btrfs_new_extent_direct(struct btrfs_inode *inode, + struct btrfs_dio_data *dio_data, + u64 start, u64 len) +{ + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_file_extent file_extent; + struct extent_map *em; + struct btrfs_key ins; + u64 alloc_hint; + int ret; + + alloc_hint = btrfs_get_extent_allocation_hint(inode, start, len); +again: + ret = btrfs_reserve_extent(root, len, len, fs_info->sectorsize, + 0, alloc_hint, &ins, true, true); + if (ret == -EAGAIN) { + ASSERT(btrfs_is_zoned(fs_info)); + wait_on_bit_io(&inode->root->fs_info->flags, BTRFS_FS_NEED_ZONE_FINISH, + TASK_UNINTERRUPTIBLE); + goto again; + } + if (ret) + return ERR_PTR(ret); + + file_extent.disk_bytenr = ins.objectid; + file_extent.disk_num_bytes = ins.offset; + file_extent.num_bytes = ins.offset; + file_extent.ram_bytes = ins.offset; + file_extent.offset = 0; + file_extent.compression = BTRFS_COMPRESS_NONE; + em = btrfs_create_dio_extent(inode, dio_data, start, &file_extent, + BTRFS_ORDERED_REGULAR); + btrfs_dec_block_group_reservations(fs_info, ins.objectid); + if (IS_ERR(em)) + btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, true); + + return em; +} + +static int btrfs_get_blocks_direct_write(struct extent_map **map, + struct inode *inode, + struct btrfs_dio_data *dio_data, + u64 start, u64 *lenp, + unsigned int iomap_flags) +{ + const bool nowait = (iomap_flags & IOMAP_NOWAIT); + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + struct btrfs_file_extent file_extent; + struct extent_map *em = *map; + int type; + u64 block_start; + struct btrfs_block_group *bg; + bool can_nocow = false; + bool space_reserved = false; + u64 len = *lenp; + u64 prev_len; + loff_t old_isize; + int ret = 0; + + /* + * We don't allocate a new extent in the following cases + * + * 1) The inode is marked as NODATACOW. In this case we'll just use the + * existing extent. + * 2) The extent is marked as PREALLOC. We're good to go here and can + * just use the extent. + * + */ + if ((em->flags & EXTENT_FLAG_PREALLOC) || + ((BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) && + em->disk_bytenr != EXTENT_MAP_HOLE)) { + if (em->flags & EXTENT_FLAG_PREALLOC) + type = BTRFS_ORDERED_PREALLOC; + else + type = BTRFS_ORDERED_NOCOW; + len = min(len, em->len - (start - em->start)); + block_start = btrfs_extent_map_block_start(em) + (start - em->start); + + if (can_nocow_extent(BTRFS_I(inode), start, &len, &file_extent, + false) == 1) { + bg = btrfs_inc_nocow_writers(fs_info, block_start); + if (bg) + can_nocow = true; + } + } + + prev_len = len; + if (can_nocow) { + struct extent_map *em2; + + /* We can NOCOW, so only need to reserve metadata space. */ + ret = btrfs_delalloc_reserve_metadata(BTRFS_I(inode), len, len, + nowait); + if (ret < 0) { + /* Our caller expects us to free the input extent map. */ + btrfs_free_extent_map(em); + *map = NULL; + btrfs_dec_nocow_writers(bg); + if (nowait && (ret == -ENOSPC || ret == -EDQUOT)) + ret = -EAGAIN; + goto out; + } + space_reserved = true; + + em2 = btrfs_create_dio_extent(BTRFS_I(inode), dio_data, start, + &file_extent, type); + btrfs_dec_nocow_writers(bg); + if (IS_ERR(em2)) { + ret = PTR_ERR(em2); + btrfs_free_extent_map(em); + *map = NULL; + goto out; + } + + /* + * True NOCOW writes don't need to create a new extent map, + * while PREALLOC writes must replace the existing one. + */ + if (em2) { + ASSERT(type == BTRFS_ORDERED_PREALLOC); + btrfs_free_extent_map(em); + *map = em2; + em = em2; + } + + dio_data->nocow_done = true; + } else { + /* Our caller expects us to free the input extent map. */ + btrfs_free_extent_map(em); + *map = NULL; + + if (nowait) { + ret = -EAGAIN; + goto out; + } + + /* + * If we could not allocate data space before locking the file + * range and we can't do a NOCOW write, then we have to fail. + */ + if (!dio_data->data_space_reserved) { + ret = -ENOSPC; + goto out; + } + + /* + * We have to COW and we have already reserved data space before, + * so now we reserve only metadata. + */ + ret = btrfs_delalloc_reserve_metadata(BTRFS_I(inode), len, len, + false); + if (ret < 0) + goto out; + space_reserved = true; + + em = btrfs_new_extent_direct(BTRFS_I(inode), dio_data, start, len); + if (IS_ERR(em)) { + ret = PTR_ERR(em); + goto out; + } + *map = em; + len = min(len, em->len - (start - em->start)); + if (len < prev_len) + btrfs_delalloc_release_metadata(BTRFS_I(inode), + prev_len - len, true); + } + + /* + * We have created our ordered extent, so we can now release our reservation + * for an outstanding extent. + */ + btrfs_delalloc_release_extents(BTRFS_I(inode), prev_len); + + /* + * Need to update the i_size under the extent lock so buffered + * readers will get the updated i_size when we unlock. + */ + old_isize = i_size_read(inode); + if (start + len > old_isize) { + if (!dio_data->updated_isize) { + dio_data->old_isize = old_isize; + dio_data->updated_isize = true; + } + i_size_write(inode, start + len); + } +out: + if (ret && space_reserved) { + btrfs_delalloc_release_extents(BTRFS_I(inode), len); + btrfs_delalloc_release_metadata(BTRFS_I(inode), len, true); + } + *lenp = len; + return ret; +} + +static int btrfs_dio_iomap_begin(struct inode *inode, loff_t start, + loff_t length, unsigned int flags, struct iomap *iomap, + struct iomap *srcmap) +{ + struct iomap_iter *iter = container_of(iomap, struct iomap_iter, iomap); + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + struct extent_map *em; + struct extent_state *cached_state = NULL; + struct btrfs_dio_data *dio_data = iter->private; + u64 lockstart, lockend; + const bool write = !!(flags & IOMAP_WRITE); + int ret = 0; + u64 len = length; + const u64 data_alloc_len = length; + u32 unlock_bits = EXTENT_LOCKED; + + /* + * We could potentially fault if we have a buffer > PAGE_SIZE, and if + * we're NOWAIT we may submit a bio for a partial range and return + * EIOCBQUEUED, which would result in an errant short read. + * + * The best way to handle this would be to allow for partial completions + * of iocb's, so we could submit the partial bio, return and fault in + * the rest of the pages, and then submit the io for the rest of the + * range. However we don't have that currently, so simply return + * -EAGAIN at this point so that the normal path is used. + */ + if (!write && (flags & IOMAP_NOWAIT) && length > PAGE_SIZE) + return -EAGAIN; + + /* + * Cap the size of reads to that usually seen in buffered I/O as we need + * to allocate a contiguous array for the checksums. + */ + if (!write) + len = min_t(u64, len, fs_info->sectorsize * BIO_MAX_VECS); + + lockstart = start; + lockend = start + len - 1; + + /* + * iomap_dio_rw() only does filemap_write_and_wait_range(), which isn't + * enough if we've written compressed pages to this area, so we need to + * flush the dirty pages again to make absolutely sure that any + * outstanding dirty pages are on disk - the first flush only starts + * compression on the data, while keeping the pages locked, so by the + * time the second flush returns we know bios for the compressed pages + * were submitted and finished, and the pages no longer under writeback. + * + * If we have a NOWAIT request and we have any pages in the range that + * are locked, likely due to compression still in progress, we don't want + * to block on page locks. We also don't want to block on pages marked as + * dirty or under writeback (same as for the non-compression case). + * iomap_dio_rw() did the same check, but after that and before we got + * here, mmap'ed writes may have happened or buffered reads started + * (readpage() and readahead(), which lock pages), as we haven't locked + * the file range yet. + */ + if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, + &BTRFS_I(inode)->runtime_flags)) { + if (flags & IOMAP_NOWAIT) { + if (filemap_range_needs_writeback(inode->i_mapping, + lockstart, lockend)) + return -EAGAIN; + } else { + ret = filemap_fdatawrite_range(inode->i_mapping, start, + start + length - 1); + if (ret) + return ret; + } + } + + memset(dio_data, 0, sizeof(*dio_data)); + + /* + * We always try to allocate data space and must do it before locking + * the file range, to avoid deadlocks with concurrent writes to the same + * range if the range has several extents and the writes don't expand the + * current i_size (the inode lock is taken in shared mode). If we fail to + * allocate data space here we continue and later, after locking the + * file range, we fail with ENOSPC only if we figure out we can not do a + * NOCOW write. + */ + if (write && !(flags & IOMAP_NOWAIT)) { + ret = btrfs_check_data_free_space(BTRFS_I(inode), + &dio_data->data_reserved, + start, data_alloc_len, false); + if (!ret) + dio_data->data_space_reserved = true; + else if (!(BTRFS_I(inode)->flags & + (BTRFS_INODE_NODATACOW | BTRFS_INODE_PREALLOC))) + goto err; + } + + /* + * If this errors out it's because we couldn't invalidate pagecache for + * this range and we need to fallback to buffered IO, or we are doing a + * NOWAIT read/write and we need to block. + */ + ret = lock_extent_direct(inode, lockstart, lockend, &cached_state, flags); + if (ret < 0) + goto err; + + em = btrfs_get_extent(BTRFS_I(inode), NULL, start, len); + if (IS_ERR(em)) { + ret = PTR_ERR(em); + goto unlock_err; + } + + /* + * Ok for INLINE and COMPRESSED extents we need to fallback on buffered + * io. INLINE is special, and we could probably kludge it in here, but + * it's still buffered so for safety lets just fall back to the generic + * buffered path. + * + * For COMPRESSED we _have_ to read the entire extent in so we can + * decompress it, so there will be buffering required no matter what we + * do, so go ahead and fallback to buffered. + * + * We return -ENOTBLK because that's what makes DIO go ahead and go back + * to buffered IO. Don't blame me, this is the price we pay for using + * the generic code. + */ + if (btrfs_extent_map_is_compressed(em) || em->disk_bytenr == EXTENT_MAP_INLINE) { + btrfs_free_extent_map(em); + /* + * If we are in a NOWAIT context, return -EAGAIN in order to + * fallback to buffered IO. This is not only because we can + * block with buffered IO (no support for NOWAIT semantics at + * the moment) but also to avoid returning short reads to user + * space - this happens if we were able to read some data from + * previous non-compressed extents and then when we fallback to + * buffered IO, at btrfs_file_read_iter() by calling + * filemap_read(), we fail to fault in pages for the read buffer, + * in which case filemap_read() returns a short read (the number + * of bytes previously read is > 0, so it does not return -EFAULT). + */ + ret = (flags & IOMAP_NOWAIT) ? -EAGAIN : -ENOTBLK; + goto unlock_err; + } + + len = min(len, em->len - (start - em->start)); + + /* + * If we have a NOWAIT request and the range contains multiple extents + * (or a mix of extents and holes), then we return -EAGAIN to make the + * caller fallback to a context where it can do a blocking (without + * NOWAIT) request. This way we avoid doing partial IO and returning + * success to the caller, which is not optimal for writes and for reads + * it can result in unexpected behaviour for an application. + * + * When doing a read, because we use IOMAP_DIO_PARTIAL when calling + * iomap_dio_rw(), we can end up returning less data then what the caller + * asked for, resulting in an unexpected, and incorrect, short read. + * That is, the caller asked to read N bytes and we return less than that, + * which is wrong unless we are crossing EOF. This happens if we get a + * page fault error when trying to fault in pages for the buffer that is + * associated to the struct iov_iter passed to iomap_dio_rw(), and we + * have previously submitted bios for other extents in the range, in + * which case iomap_dio_rw() may return us EIOCBQUEUED if not all of + * those bios have completed by the time we get the page fault error, + * which we return back to our caller - we should only return EIOCBQUEUED + * after we have submitted bios for all the extents in the range. + */ + if ((flags & IOMAP_NOWAIT) && len < length) { + btrfs_free_extent_map(em); + ret = -EAGAIN; + goto unlock_err; + } + + if (write) { + ret = btrfs_get_blocks_direct_write(&em, inode, dio_data, + start, &len, flags); + if (ret < 0) + goto unlock_err; + /* Recalc len in case the new em is smaller than requested */ + len = min(len, em->len - (start - em->start)); + if (dio_data->data_space_reserved) { + u64 release_offset; + u64 release_len = 0; + + if (dio_data->nocow_done) { + release_offset = start; + release_len = data_alloc_len; + } else if (len < data_alloc_len) { + release_offset = start + len; + release_len = data_alloc_len - len; + } + + if (release_len > 0) + btrfs_free_reserved_data_space(BTRFS_I(inode), + dio_data->data_reserved, + release_offset, + release_len); + } + } + + /* + * Translate extent map information to iomap. + * We trim the extents (and move the addr) even though iomap code does + * that, since we have locked only the parts we are performing I/O in. + */ + if ((em->disk_bytenr == EXTENT_MAP_HOLE) || + ((em->flags & EXTENT_FLAG_PREALLOC) && !write)) { + iomap->addr = IOMAP_NULL_ADDR; + iomap->type = IOMAP_HOLE; + } else { + iomap->addr = btrfs_extent_map_block_start(em) + (start - em->start); + iomap->type = IOMAP_MAPPED; + } + iomap->offset = start; + iomap->bdev = fs_info->fs_devices->latest_dev->bdev; + iomap->length = len; + btrfs_free_extent_map(em); + + /* + * Reads will hold the EXTENT_DIO_LOCKED bit until the io is completed, + * writes only hold it for this part. We hold the extent lock until + * we're completely done with the extent map to make sure it remains + * valid. + */ + if (write) + unlock_bits |= EXTENT_DIO_LOCKED; + + btrfs_clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart, lockend, + unlock_bits, &cached_state); + + /* We didn't use everything, unlock the dio extent for the remainder. */ + if (!write && (start + len) < lockend) + btrfs_unlock_dio_extent(&BTRFS_I(inode)->io_tree, start + len, + lockend, NULL); + + return 0; + +unlock_err: + /* + * Don't use EXTENT_LOCK_BITS here in case we extend it later and forget + * to update this, be explicit that we expect EXTENT_LOCKED and + * EXTENT_DIO_LOCKED to be set here, and so that's what we're clearing. + */ + btrfs_clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart, lockend, + EXTENT_LOCKED | EXTENT_DIO_LOCKED, &cached_state); +err: + if (dio_data->data_space_reserved) { + btrfs_free_reserved_data_space(BTRFS_I(inode), + dio_data->data_reserved, + start, data_alloc_len); + extent_changeset_free(dio_data->data_reserved); + } + + return ret; +} + +static int btrfs_dio_iomap_end(struct inode *inode, loff_t pos, loff_t length, + ssize_t written, unsigned int flags, struct iomap *iomap) +{ + struct iomap_iter *iter = container_of(iomap, struct iomap_iter, iomap); + struct btrfs_dio_data *dio_data = iter->private; + const bool write = !!(flags & IOMAP_WRITE); + int ret = 0; + + if (!write) { + /* + * Hole read, nothing is submitted, thus we have to unlock + * the whole range. + */ + if (iomap->type == IOMAP_HOLE) { + btrfs_unlock_dio_extent(&BTRFS_I(inode)->io_tree, pos, + pos + length - 1, NULL); + return 0; + } + /* + * Short read, needs to unlock the remaining range, and + * return -ENOTBLK so we can later fault in the pages and retry. + */ + if (written < length) { + btrfs_unlock_dio_extent(&BTRFS_I(inode)->io_tree, pos + written, + pos + length - 1, NULL); + return -ENOTBLK; + } + /* The full range is submitted, endio will do the unlock. */ + return 0; + } + + if (written < length) { + /* + * Got a short write and have updated the i_size, need to revert + * the i_size change. + * + * Normally we need to update i_size with extent lock held, but + * we're safe due to the following factors: + * + * - Only a single writer can be enlarging i_size + * Enlarging i_size will take the exclusive inode lock. + * + * - Buffered readers need to wait for the OE we're holding + * Buffered readers will lock extent and wait for OE + * of the folio range, and since page cache is invalidated + * the OE wait cannot be skipped. + * + * So here we are safe to revert the isize before finishing the + * OE, and no reader of the remaining range can see the enlarged + * size. + * + * TODO: Extend the DIO_LOCKED lifespan for direct writes, + * and only enlarge isize after a successful write. + */ + if (dio_data->updated_isize) { + u64 new_isize; + + if (written == 0) + new_isize = dio_data->old_isize; + else + new_isize = max(dio_data->old_isize, pos + written); + i_size_write(inode, new_isize); + dio_data->updated_isize = false; + } + /* + * We have a short write, if there is any range that is submitted + * properly, that part will have its own OE split from the + * original one. + * + * So for the OE at dio_data->ordered, it's the part that is not + * submitted, and should be marked as fully truncated. + */ + btrfs_mark_ordered_extent_truncated(dio_data->ordered, 0); + btrfs_finish_ordered_extent(dio_data->ordered, + pos + written, length - written, true); + ret = -ENOTBLK; + } + btrfs_put_ordered_extent(dio_data->ordered); + dio_data->ordered = NULL; + extent_changeset_free(dio_data->data_reserved); + return ret; +} + +static void btrfs_dio_end_io(struct btrfs_bio *bbio) +{ + struct btrfs_dio_private *dip = + container_of(bbio, struct btrfs_dio_private, bbio); + struct btrfs_inode *inode = bbio->inode; + struct bio *bio = &bbio->bio; + + if (bio->bi_status) { + btrfs_warn(inode->root->fs_info, + "direct IO failed ino %llu op 0x%0x offset %#llx len %u err no %d", + btrfs_ino(inode), bio->bi_opf, + dip->file_offset, dip->bytes, bio->bi_status); + } + + if (btrfs_op(bio) == BTRFS_MAP_WRITE) { + btrfs_finish_ordered_extent(bbio->ordered, dip->file_offset, + dip->bytes, !bio->bi_status); + } else { + btrfs_unlock_dio_extent(&inode->io_tree, dip->file_offset, + dip->file_offset + dip->bytes - 1, NULL); + } + + bbio->bio.bi_private = bbio->private; + iomap_dio_bio_end_io(bio); +} + +static int btrfs_extract_ordered_extent(struct btrfs_bio *bbio, + struct btrfs_ordered_extent *ordered) +{ + u64 start = (u64)bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT; + u64 len = bbio->bio.bi_iter.bi_size; + struct btrfs_ordered_extent *new; + int ret; + + /* Must always be called for the beginning of an ordered extent. */ + if (WARN_ON_ONCE(start != ordered->disk_bytenr)) + return -EINVAL; + + /* No need to split if the ordered extent covers the entire bio. */ + if (ordered->disk_num_bytes == len) { + refcount_inc(&ordered->refs); + bbio->ordered = ordered; + return 0; + } + + /* + * Don't split the extent_map for NOCOW extents, as we're writing into + * a pre-existing one. + */ + if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered->flags)) { + ret = btrfs_split_extent_map(bbio->inode, bbio->file_offset, + ordered->num_bytes, len, + ordered->disk_bytenr); + if (ret) + return ret; + } + + new = btrfs_split_ordered_extent(ordered, len); + if (IS_ERR(new)) + return PTR_ERR(new); + bbio->ordered = new; + return 0; +} + +static void btrfs_dio_submit_io(const struct iomap_iter *iter, struct bio *bio, + loff_t file_offset) +{ + struct btrfs_bio *bbio = btrfs_bio(bio); + struct btrfs_dio_private *dip = + container_of(bbio, struct btrfs_dio_private, bbio); + struct btrfs_dio_data *dio_data = iter->private; + + btrfs_bio_init(bbio, BTRFS_I(iter->inode), file_offset, + btrfs_dio_end_io, bio->bi_private); + + dip->file_offset = file_offset; + dip->bytes = bio->bi_iter.bi_size; + + /* + * Check if we are doing a partial write. If we are, we need to split + * the ordered extent to match the submitted bio. Hang on to the + * remaining unfinishable ordered_extent in dio_data so that it can be + * cancelled in iomap_end to avoid a deadlock wherein faulting the + * remaining pages is blocked on the outstanding ordered extent. + */ + if (iter->flags & IOMAP_WRITE) { + int ret; + + ret = btrfs_extract_ordered_extent(bbio, dio_data->ordered); + if (ret) { + btrfs_finish_ordered_extent(dio_data->ordered, + file_offset, dip->bytes, + !ret); + bio->bi_status = errno_to_blk_status(ret); + iomap_dio_bio_end_io(bio); + return; + } + } + + btrfs_submit_bbio(bbio, 0); +} + +static DEFINE_IOMAP_ITER_NEXT_END(btrfs_dio_iomap_next, btrfs_dio_iomap_begin, + btrfs_dio_iomap_end); + +static const struct iomap_ops btrfs_dio_iomap_ops = { + .iomap_next = btrfs_dio_iomap_next, +}; + +static const struct iomap_dio_ops btrfs_dio_ops = { + .submit_io = btrfs_dio_submit_io, + .bio_set = &btrfs_dio_bioset, +}; + +static ssize_t btrfs_dio_read(struct kiocb *iocb, struct iov_iter *iter, + size_t done_before) +{ + struct btrfs_dio_data data = { 0 }; + + return iomap_dio_rw(iocb, iter, &btrfs_dio_iomap_ops, &btrfs_dio_ops, + IOMAP_DIO_PARTIAL | IOMAP_DIO_FSBLOCK_ALIGNED, &data, done_before); +} + +static bool need_stable_write(struct btrfs_inode *inode) +{ + const u64 data_profile = btrfs_data_alloc_profile(inode->root->fs_info) & + BTRFS_BLOCK_GROUP_PROFILE_MASK; + + /* Data checksum requires stable buffer. */ + if (!(inode->flags & BTRFS_INODE_NODATASUM)) + return true; + /* + * Any profile with mirror/parity will require stable buffer. + * Otherwise the mirror may differ from each other. + * + * Thus only SINGLE and RAID0 doesn't require stable buffer. + */ + if (data_profile != 0 && data_profile != BTRFS_BLOCK_GROUP_RAID0) + return true; + return false; +} + +static struct iomap_dio *btrfs_dio_write(struct kiocb *iocb, struct iov_iter *iter, + size_t done_before) +{ + struct btrfs_dio_data data = { 0 }; + unsigned int dio_flags = IOMAP_DIO_PARTIAL | IOMAP_DIO_FSBLOCK_ALIGNED; + + if (need_stable_write(BTRFS_I(file_inode(iocb->ki_filp)))) { + /* For now no support for BOUNCE and NOWAIT direct write. */ + if (iocb->ki_flags & IOCB_NOWAIT) + return ERR_PTR(-EAGAIN); + + dio_flags |= IOMAP_DIO_BOUNCE; + } + + return __iomap_dio_rw(iocb, iter, &btrfs_dio_iomap_ops, &btrfs_dio_ops, + dio_flags, &data, done_before); +} + +static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info, + const struct iov_iter *iter, loff_t offset) +{ + const u32 blocksize_mask = fs_info->sectorsize - 1; + + if (offset & blocksize_mask) + return -EINVAL; + + if (iov_iter_alignment(iter) & blocksize_mask) + return -EINVAL; + return 0; +} + +ssize_t btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + loff_t pos; + ssize_t written = 0; + ssize_t written_buffered; + size_t prev_left = 0; + loff_t endbyte; + ssize_t ret; + unsigned int ilock_flags = 0; + struct iomap_dio *dio; + + if (iocb->ki_flags & IOCB_NOWAIT) + ilock_flags |= BTRFS_ILOCK_TRY; + + /* + * If the write DIO is within EOF, use a shared lock and also only if + * security bits will likely not be dropped by file_remove_privs() called + * from btrfs_write_check(). Either will need to be rechecked after the + * lock was acquired. + */ + if (iocb->ki_pos + iov_iter_count(from) <= i_size_read(inode) && IS_NOSEC(inode)) + ilock_flags |= BTRFS_ILOCK_SHARED; + +relock: + ret = btrfs_inode_lock(BTRFS_I(inode), ilock_flags); + if (ret < 0) + return ret; + + /* Shared lock cannot be used with security bits set. */ + if ((ilock_flags & BTRFS_ILOCK_SHARED) && !IS_NOSEC(inode)) { + btrfs_inode_unlock(BTRFS_I(inode), ilock_flags); + ilock_flags &= ~BTRFS_ILOCK_SHARED; + goto relock; + } + + ret = generic_write_checks(iocb, from); + if (ret <= 0) { + btrfs_inode_unlock(BTRFS_I(inode), ilock_flags); + return ret; + } + + ret = btrfs_write_check(iocb, ret); + if (ret < 0) { + btrfs_inode_unlock(BTRFS_I(inode), ilock_flags); + goto out; + } + + pos = iocb->ki_pos; + /* + * Re-check since file size may have changed just before taking the + * lock or pos may have changed because of O_APPEND in generic_write_check() + */ + if ((ilock_flags & BTRFS_ILOCK_SHARED) && + pos + iov_iter_count(from) > i_size_read(inode)) { + btrfs_inode_unlock(BTRFS_I(inode), ilock_flags); + ilock_flags &= ~BTRFS_ILOCK_SHARED; + goto relock; + } + + if (check_direct_IO(fs_info, from, pos)) { + btrfs_inode_unlock(BTRFS_I(inode), ilock_flags); + goto buffered; + } + + /* + * The iov_iter can be mapped to the same file range we are writing to. + * If that's the case, then we will deadlock in the iomap code, because + * it first calls our callback btrfs_dio_iomap_begin(), which will create + * an ordered extent, and after that it will fault in the pages that the + * iov_iter refers to. During the fault in we end up in the readahead + * pages code (starting at btrfs_readahead()), which will lock the range, + * find that ordered extent and then wait for it to complete (at + * btrfs_lock_and_flush_ordered_range()), resulting in a deadlock since + * obviously the ordered extent can never complete as we didn't submit + * yet the respective bio(s). This always happens when the buffer is + * memory mapped to the same file range, since the iomap DIO code always + * invalidates pages in the target file range (after starting and waiting + * for any writeback). + * + * So here we disable page faults in the iov_iter and then retry if we + * got -EFAULT, faulting in the pages before the retry. + */ +again: + from->nofault = true; + dio = btrfs_dio_write(iocb, from, written); + from->nofault = false; + + if (IS_ERR_OR_NULL(dio)) { + ret = PTR_ERR_OR_ZERO(dio); + } else { + /* + * If we have a synchronous write, we must make sure the fsync + * triggered by the iomap_dio_complete() call below doesn't + * deadlock on the inode lock - we are already holding it and we + * can't call it after unlocking because we may need to complete + * partial writes due to the input buffer (or parts of it) not + * being already faulted in. + */ + ASSERT(current->journal_info == NULL); + current->journal_info = BTRFS_TRANS_DIO_WRITE_STUB; + ret = iomap_dio_complete(dio); + current->journal_info = NULL; + } + + /* No increment (+=) because iomap returns a cumulative value. */ + if (ret > 0) + written = ret; + + if (iov_iter_count(from) > 0 && (ret == -EFAULT || ret >= 0)) { + const size_t left = iov_iter_count(from); + /* + * We have more data left to write. Try to fault in as many as + * possible of the remainder pages and retry. We do this without + * releasing and locking again the inode, to prevent races with + * truncate. + * + * Also, in case the iov refers to pages in the file range of the + * file we want to write to (due to a mmap), we could enter an + * infinite loop if we retry after faulting the pages in, since + * iomap will invalidate any pages in the range early on, before + * it tries to fault in the pages of the iov. So we keep track of + * how much was left of iov in the previous EFAULT and fallback + * to buffered IO in case we haven't made any progress. + */ + if (left == prev_left) { + ret = -ENOTBLK; + } else { + fault_in_iov_iter_readable(from, left); + prev_left = left; + goto again; + } + } + + btrfs_inode_unlock(BTRFS_I(inode), ilock_flags); + + /* + * If 'ret' is -ENOTBLK or we have not written all data, then it means + * we must fallback to buffered IO. + */ + if ((ret < 0 && ret != -ENOTBLK) || !iov_iter_count(from)) + goto out; + +buffered: + /* + * If we are in a NOWAIT context, then return -EAGAIN to signal the caller + * it must retry the operation in a context where blocking is acceptable, + * because even if we end up not blocking during the buffered IO attempt + * below, we will block when flushing and waiting for the IO. + */ + if (iocb->ki_flags & IOCB_NOWAIT) { + ret = -EAGAIN; + goto out; + } + + pos = iocb->ki_pos; + written_buffered = btrfs_buffered_write(iocb, from); + if (written_buffered < 0) { + ret = written_buffered; + goto out; + } + /* + * Ensure all data is persisted. We want the next direct IO read to be + * able to read what was just written. + */ + endbyte = pos + written_buffered - 1; + ret = btrfs_fdatawrite_range(BTRFS_I(inode), pos, endbyte); + if (ret) + goto out; + ret = filemap_fdatawait_range(inode->i_mapping, pos, endbyte); + if (ret) + goto out; + written += written_buffered; + iocb->ki_pos = pos + written_buffered; + invalidate_mapping_pages(file->f_mapping, pos >> PAGE_SHIFT, + endbyte >> PAGE_SHIFT); +out: + return ret < 0 ? ret : written; +} + +static int check_direct_read(struct btrfs_fs_info *fs_info, + const struct iov_iter *iter, loff_t offset) +{ + int ret; + int i, seg; + + ret = check_direct_IO(fs_info, iter, offset); + if (ret < 0) + return ret; + + if (!iter_is_iovec(iter)) + return 0; + + for (seg = 0; seg < iter->nr_segs; seg++) { + for (i = seg + 1; i < iter->nr_segs; i++) { + const struct iovec *iov1 = iter_iov(iter) + seg; + const struct iovec *iov2 = iter_iov(iter) + i; + + if (iov1->iov_base == iov2->iov_base) + return -EINVAL; + } + } + return 0; +} + +ssize_t btrfs_direct_read(struct kiocb *iocb, struct iov_iter *to) +{ + struct inode *inode = file_inode(iocb->ki_filp); + size_t prev_left = 0; + ssize_t read = 0; + ssize_t ret; + + if (fsverity_active(inode)) + return 0; + + if (check_direct_read(inode_to_fs_info(inode), to, iocb->ki_pos)) + return 0; + + btrfs_inode_lock(BTRFS_I(inode), BTRFS_ILOCK_SHARED); +again: + /* + * This is similar to what we do for direct IO writes, see the comment + * at btrfs_direct_write(), but we also disable page faults in addition + * to disabling them only at the iov_iter level. This is because when + * reading from a hole or prealloc extent, iomap calls iov_iter_zero(), + * which can still trigger page fault ins despite having set ->nofault + * to true of our 'to' iov_iter. + * + * The difference to direct IO writes is that we deadlock when trying + * to lock the extent range in the inode's tree during he page reads + * triggered by the fault in (while for writes it is due to waiting for + * our own ordered extent). This is because for direct IO reads, + * btrfs_dio_iomap_begin() returns with the extent range locked, which + * is only unlocked in the endio callback (end_bio_extent_readpage()). + */ + pagefault_disable(); + to->nofault = true; + ret = btrfs_dio_read(iocb, to, read); + to->nofault = false; + pagefault_enable(); + + /* No increment (+=) because iomap returns a cumulative value. */ + if (ret > 0) + read = ret; + + if (iov_iter_count(to) > 0 && (ret == -EFAULT || ret > 0)) { + const size_t left = iov_iter_count(to); + + if (left == prev_left) { + /* + * We didn't make any progress since the last attempt, + * fallback to a buffered read for the remainder of the + * range. This is just to avoid any possibility of looping + * for too long. + */ + ret = read; + } else { + /* + * We made some progress since the last retry or this is + * the first time we are retrying. Fault in as many pages + * as possible and retry. + */ + fault_in_iov_iter_writeable(to, left); + prev_left = left; + goto again; + } + } + btrfs_inode_unlock(BTRFS_I(inode), BTRFS_ILOCK_SHARED); + return ret < 0 ? ret : read; +} + +int __init btrfs_init_dio(void) +{ + if (bioset_init(&btrfs_dio_bioset, BIO_POOL_SIZE, + offsetof(struct btrfs_dio_private, bbio.bio), + BIOSET_NEED_BVECS)) + return -ENOMEM; + + return 0; +} + +void __cold btrfs_destroy_dio(void) +{ + bioset_exit(&btrfs_dio_bioset); +} diff --git a/fs/btrfs/direct-io.h b/fs/btrfs/direct-io.h new file mode 100644 index 000000000..df5d45ee6 --- /dev/null +++ b/fs/btrfs/direct-io.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_DIRECT_IO_H +#define BTRFS_DIRECT_IO_H + +#include + +struct kiocb; + +int __init btrfs_init_dio(void); +void __cold btrfs_destroy_dio(void); + +ssize_t btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from); +ssize_t btrfs_direct_read(struct kiocb *iocb, struct iov_iter *to); + +#endif /* BTRFS_DIRECT_IO_H */ diff --git a/fs/btrfs/discard.c b/fs/btrfs/discard.c new file mode 100644 index 000000000..1c304bf47 --- /dev/null +++ b/fs/btrfs/discard.c @@ -0,0 +1,828 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include +#include +#include +#include +#include "ctree.h" +#include "block-group.h" +#include "discard.h" +#include "free-space-cache.h" +#include "fs.h" + +/* + * This contains the logic to handle async discard. + * + * Async discard manages trimming of free space outside of transaction commit. + * Discarding is done by managing the block_groups on a LRU list based on free + * space recency. Two passes are used to first prioritize discarding extents + * and then allow for trimming in the bitmap the best opportunity to coalesce. + * The block_groups are maintained on multiple lists to allow for multiple + * passes with different discard filter requirements. A delayed work item is + * used to manage discarding with timeout determined by a max of the delay + * incurred by the iops rate limit, the byte rate limit, and the max delay of + * BTRFS_DISCARD_MAX_DELAY. + * + * Note, this only keeps track of block_groups that are explicitly for data. + * Mixed block_groups are not supported. + * + * The first list is special to manage discarding of fully free block groups. + * This is necessary because we issue a final trim for a full free block group + * after forgetting it. When a block group becomes unused, instead of directly + * being added to the unused_bgs list, we add it to this first list. Then + * from there, if it becomes fully discarded, we place it onto the unused_bgs + * list. + * + * The in-memory free space cache serves as the backing state for discard. + * Consequently this means there is no persistence. We opt to load all the + * block groups in as not discarded, so the mount case degenerates to the + * crashing case. + * + * As the free space cache uses bitmaps, there exists a tradeoff between + * ease/efficiency for find_free_extent() and the accuracy of discard state. + * Here we opt to let untrimmed regions merge with everything while only letting + * trimmed regions merge with other trimmed regions. This can cause + * overtrimming, but the coalescing benefit seems to be worth it. Additionally, + * bitmap state is tracked as a whole. If we're able to fully trim a bitmap, + * the trimmed flag is set on the bitmap. Otherwise, if an allocation comes in, + * this resets the state and we will retry trimming the whole bitmap. This is a + * tradeoff between discard state accuracy and the cost of accounting. + */ + +/* This is an initial delay to give some chance for block reuse */ +#define BTRFS_DISCARD_DELAY (120ULL * NSEC_PER_SEC) +#define BTRFS_DISCARD_UNUSED_DELAY (10ULL * NSEC_PER_SEC) + +#define BTRFS_DISCARD_MIN_DELAY_MSEC (1UL) +#define BTRFS_DISCARD_MAX_DELAY_MSEC (1000UL) +#define BTRFS_DISCARD_MAX_IOPS (1000U) + +/* Monotonically decreasing minimum length filters after index 0 */ +static int discard_minlen[BTRFS_NR_DISCARD_LISTS] = { + 0, + BTRFS_ASYNC_DISCARD_MAX_FILTER, + BTRFS_ASYNC_DISCARD_MIN_FILTER +}; + +static struct list_head *get_discard_list(struct btrfs_discard_ctl *discard_ctl, + const struct btrfs_block_group *block_group) +{ + return &discard_ctl->discard_list[block_group->discard_index]; +} + +/* + * Determine if async discard should be running. + * + * @discard_ctl: discard control + * + * Check if the file system is writeable and BTRFS_FS_DISCARD_RUNNING is set. + */ +static bool btrfs_run_discard_work(const struct btrfs_discard_ctl *discard_ctl) +{ + struct btrfs_fs_info *fs_info = container_of(discard_ctl, + struct btrfs_fs_info, + discard_ctl); + + return (!(fs_info->sb->s_flags & SB_RDONLY) && + test_bit(BTRFS_FS_DISCARD_RUNNING, &fs_info->flags)); +} + +static void __add_to_discard_list(struct btrfs_discard_ctl *discard_ctl, + struct btrfs_block_group *block_group) +{ + lockdep_assert_held(&discard_ctl->lock); + + if (list_empty(&block_group->discard_list) || + block_group->discard_index == BTRFS_DISCARD_INDEX_UNUSED) { + if (block_group->discard_index == BTRFS_DISCARD_INDEX_UNUSED) + block_group->discard_index = BTRFS_DISCARD_INDEX_START; + block_group->discard_eligible_time = (ktime_get_ns() + + BTRFS_DISCARD_DELAY); + block_group->discard_state = BTRFS_DISCARD_RESET_CURSOR; + } + if (list_empty(&block_group->discard_list)) + btrfs_get_block_group(block_group); + + list_move_tail(&block_group->discard_list, + get_discard_list(discard_ctl, block_group)); +} + +static void add_to_discard_list(struct btrfs_discard_ctl *discard_ctl, + struct btrfs_block_group *block_group) +{ + if (!btrfs_is_block_group_data_only(block_group)) + return; + + if (!btrfs_run_discard_work(discard_ctl)) + return; + + spin_lock(&discard_ctl->lock); + __add_to_discard_list(discard_ctl, block_group); + spin_unlock(&discard_ctl->lock); +} + +static void add_to_discard_unused_list(struct btrfs_discard_ctl *discard_ctl, + struct btrfs_block_group *block_group) +{ + bool queued; + + spin_lock(&discard_ctl->lock); + + queued = !list_empty(&block_group->discard_list); + + if (!btrfs_run_discard_work(discard_ctl)) { + spin_unlock(&discard_ctl->lock); + return; + } + + list_del_init(&block_group->discard_list); + + block_group->discard_index = BTRFS_DISCARD_INDEX_UNUSED; + block_group->discard_eligible_time = (ktime_get_ns() + + BTRFS_DISCARD_UNUSED_DELAY); + block_group->discard_state = BTRFS_DISCARD_RESET_CURSOR; + if (!queued) + btrfs_get_block_group(block_group); + list_add_tail(&block_group->discard_list, + &discard_ctl->discard_list[BTRFS_DISCARD_INDEX_UNUSED]); + + spin_unlock(&discard_ctl->lock); +} + +static bool remove_from_discard_list(struct btrfs_discard_ctl *discard_ctl, + struct btrfs_block_group *block_group) +{ + bool running = false; + bool queued = false; + + spin_lock(&discard_ctl->lock); + + if (block_group == discard_ctl->block_group) { + running = true; + discard_ctl->block_group = NULL; + } + + block_group->discard_eligible_time = 0; + queued = !list_empty(&block_group->discard_list); + list_del_init(&block_group->discard_list); + if (queued) + btrfs_put_block_group(block_group); + + spin_unlock(&discard_ctl->lock); + + return running; +} + +/* + * Find block_group that's up next for discarding. + * + * @discard_ctl: discard control + * @now: current time + * + * Iterate over the discard lists to find the next block_group up for + * discarding checking the discard_eligible_time of block_group. + */ +static struct btrfs_block_group *find_next_block_group( + struct btrfs_discard_ctl *discard_ctl, + u64 now) +{ + struct btrfs_block_group *ret_block_group = NULL, *block_group; + int i; + + for (i = 0; i < BTRFS_NR_DISCARD_LISTS; i++) { + struct list_head *discard_list = &discard_ctl->discard_list[i]; + + if (!list_empty(discard_list)) { + block_group = list_first_entry(discard_list, + struct btrfs_block_group, + discard_list); + + if (!ret_block_group) + ret_block_group = block_group; + + if (ret_block_group->discard_eligible_time < now) + break; + + if (ret_block_group->discard_eligible_time > + block_group->discard_eligible_time) + ret_block_group = block_group; + } + } + + return ret_block_group; +} + +/* + * Check whether a block group is empty. + * + * "Empty" here means that there are no extents physically located within the + * device extents corresponding to this block group. + * + * For a remapped block group, this means that all of its identity remaps have + * been removed. For a non-remapped block group, this means that no extents + * have an address within its range, and that nothing has been remapped to be + * within it. + */ +static bool block_group_is_empty(const struct btrfs_block_group *bg) +{ + if (bg->flags & BTRFS_BLOCK_GROUP_REMAPPED) + return bg->identity_remap_count == 0; + + return bg->used == 0 && bg->remap_bytes == 0; +} + +/* + * Look up next block group and set it for use. + * + * @discard_ctl: discard control + * @discard_state: the discard_state of the block_group after state management + * @discard_index: the discard_index of the block_group after state management + * @now: time when discard was invoked, in ns + * + * Wrap find_next_block_group() and set the block_group to be in use. + * @discard_state's control flow is managed here. Variables related to + * @discard_state are reset here as needed (eg. @discard_cursor). @discard_state + * and @discard_index are remembered as it may change while we're discarding, + * but we want the discard to execute in the context determined here. + */ +static struct btrfs_block_group *peek_discard_list( + struct btrfs_discard_ctl *discard_ctl, + enum btrfs_discard_state *discard_state, + int *discard_index, u64 now) +{ + struct btrfs_block_group *block_group; + + spin_lock(&discard_ctl->lock); +again: + block_group = find_next_block_group(discard_ctl, now); + + if (block_group && now >= block_group->discard_eligible_time) { + const bool empty = block_group_is_empty(block_group); + + if (block_group->discard_index == BTRFS_DISCARD_INDEX_UNUSED && + !empty) { + if (btrfs_is_block_group_data_only(block_group)) { + __add_to_discard_list(discard_ctl, block_group); + /* + * The block group must have been moved to other + * discard list even if discard was disabled in + * the meantime or a transaction abort happened, + * otherwise we can end up in an infinite loop, + * always jumping into the 'again' label and + * keep getting this block group over and over + * in case there are no other block groups in + * the discard lists. + */ + ASSERT(block_group->discard_index != + BTRFS_DISCARD_INDEX_UNUSED, + "discard_index=%d", + block_group->discard_index); + } else { + list_del_init(&block_group->discard_list); + btrfs_put_block_group(block_group); + } + goto again; + } + if (block_group->discard_state == BTRFS_DISCARD_RESET_CURSOR) { + block_group->discard_cursor = block_group->start; + + if (block_group->flags & BTRFS_BLOCK_GROUP_REMAPPED && empty) { + block_group->discard_state = BTRFS_DISCARD_FULLY_REMAPPED; + } else { + block_group->discard_state = BTRFS_DISCARD_EXTENTS; + } + } + } + if (block_group) { + btrfs_get_block_group(block_group); + discard_ctl->block_group = block_group; + *discard_state = block_group->discard_state; + *discard_index = block_group->discard_index; + } + spin_unlock(&discard_ctl->lock); + + return block_group; +} + +/* + * Update a block group's filters. + * + * @block_group: block group of interest + * @bytes: recently freed region size after coalescing + * + * Async discard maintains multiple lists with progressively smaller filters + * to prioritize discarding based on size. Should a free space that matches + * a larger filter be returned to the free_space_cache, prioritize that discard + * by moving @block_group to the proper filter. + */ +void btrfs_discard_check_filter(struct btrfs_block_group *block_group, + u64 bytes) +{ + struct btrfs_discard_ctl *discard_ctl; + + if (!block_group || + !btrfs_test_opt(block_group->fs_info, DISCARD_ASYNC)) + return; + + discard_ctl = &block_group->fs_info->discard_ctl; + + if (block_group->discard_index > BTRFS_DISCARD_INDEX_START && + bytes >= discard_minlen[block_group->discard_index - 1]) { + int i; + + remove_from_discard_list(discard_ctl, block_group); + + for (i = BTRFS_DISCARD_INDEX_START; i < BTRFS_NR_DISCARD_LISTS; + i++) { + if (bytes >= discard_minlen[i]) { + block_group->discard_index = i; + add_to_discard_list(discard_ctl, block_group); + break; + } + } + } +} + +/* + * Move a block group along the discard lists. + * + * @discard_ctl: discard control + * @block_group: block_group of interest + * + * Increment @block_group's discard_index. If it falls of the list, let it be. + * Otherwise add it back to the appropriate list. + */ +static void btrfs_update_discard_index(struct btrfs_discard_ctl *discard_ctl, + struct btrfs_block_group *block_group) +{ + block_group->discard_index++; + if (block_group->discard_index == BTRFS_NR_DISCARD_LISTS) { + block_group->discard_index = 1; + return; + } + + add_to_discard_list(discard_ctl, block_group); +} + +/* + * Remove a block_group from the discard lists. + * + * @discard_ctl: discard control + * @block_group: block_group of interest + * + * Remove @block_group from the discard lists. If necessary, wait on the + * current work and then reschedule the delayed work. + */ +void btrfs_discard_cancel_work(struct btrfs_discard_ctl *discard_ctl, + struct btrfs_block_group *block_group) +{ + if (remove_from_discard_list(discard_ctl, block_group)) { + cancel_delayed_work_sync(&discard_ctl->work); + btrfs_discard_schedule_work(discard_ctl, true); + } +} + +/* + * Handles queuing the block_groups. + * + * @discard_ctl: discard control + * @block_group: block_group of interest + * + * Maintain the LRU order of the discard lists. + */ +void btrfs_discard_queue_work(struct btrfs_discard_ctl *discard_ctl, + struct btrfs_block_group *block_group) +{ + if (!block_group || !btrfs_test_opt(block_group->fs_info, DISCARD_ASYNC)) + return; + + if (block_group_is_empty(block_group)) + add_to_discard_unused_list(discard_ctl, block_group); + else + add_to_discard_list(discard_ctl, block_group); + + if (!delayed_work_pending(&discard_ctl->work)) + btrfs_discard_schedule_work(discard_ctl, false); +} + +static void __btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl, + u64 now, bool override) +{ + struct btrfs_block_group *block_group; + + if (!btrfs_run_discard_work(discard_ctl)) + return; + if (!override && delayed_work_pending(&discard_ctl->work)) + return; + + block_group = find_next_block_group(discard_ctl, now); + if (block_group) { + u64 delay = discard_ctl->delay_ms * NSEC_PER_MSEC; + u32 kbps_limit = READ_ONCE(discard_ctl->kbps_limit); + + /* + * A single delayed workqueue item is responsible for + * discarding, so we can manage the bytes rate limit by keeping + * track of the previous discard. + */ + if (kbps_limit && discard_ctl->prev_discard) { + u64 bps_limit = ((u64)kbps_limit) * SZ_1K; + u64 bps_delay = div64_u64(discard_ctl->prev_discard * + NSEC_PER_SEC, bps_limit); + + delay = max(delay, bps_delay); + } + + /* + * This timeout is to hopefully prevent immediate discarding + * in a recently allocated block group. + */ + if (now < block_group->discard_eligible_time) { + u64 bg_timeout = block_group->discard_eligible_time - now; + + delay = max(delay, bg_timeout); + } + + if (override && discard_ctl->prev_discard) { + u64 elapsed = now - discard_ctl->prev_discard_time; + + if (delay > elapsed) + delay -= elapsed; + else + delay = 0; + } + + mod_delayed_work(discard_ctl->discard_workers, + &discard_ctl->work, nsecs_to_jiffies(delay)); + } +} + +/* + * Responsible for scheduling the discard work. + * + * @discard_ctl: discard control + * @override: override the current timer + * + * Discards are issued by a delayed workqueue item. @override is used to + * update the current delay as the baseline delay interval is reevaluated on + * transaction commit. This is also maxed with any other rate limit. + */ +void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl, + bool override) +{ + const u64 now = ktime_get_ns(); + + spin_lock(&discard_ctl->lock); + __btrfs_discard_schedule_work(discard_ctl, now, override); + spin_unlock(&discard_ctl->lock); +} + +/* + * Determine next step of a block_group. + * + * @discard_ctl: discard control + * @block_group: block_group of interest + * + * Determine the next step for a block group after it's finished going through + * a pass on a discard list. If it is unused and fully trimmed, we can mark it + * unused and send it to the unused_bgs path. Otherwise, pass it onto the + * appropriate filter list or let it fall off. + */ +static void btrfs_finish_discard_pass(struct btrfs_discard_ctl *discard_ctl, + struct btrfs_block_group *block_group) +{ + remove_from_discard_list(discard_ctl, block_group); + + if (block_group_is_empty(block_group)) { + if (btrfs_is_free_space_trimmed(block_group)) + btrfs_mark_bg_unused(block_group); + else + add_to_discard_unused_list(discard_ctl, block_group); + } else { + btrfs_update_discard_index(discard_ctl, block_group); + } +} + +/* + * Discard work queue callback + * + * @work: work + * + * Find the next block_group to start discarding and then discard a single + * region. It does this in a two-pass fashion: first extents and second + * bitmaps. Completely discarded block groups are sent to the unused_bgs path. + */ +static void btrfs_discard_workfn(struct work_struct *work) +{ + struct btrfs_discard_ctl *discard_ctl; + struct btrfs_block_group *block_group; + enum btrfs_discard_state discard_state; + int discard_index = 0; + u64 trimmed = 0; + u64 minlen = 0; + u64 now = ktime_get_ns(); + + discard_ctl = container_of(work, struct btrfs_discard_ctl, work.work); + + block_group = peek_discard_list(discard_ctl, &discard_state, + &discard_index, now); + if (!block_group) + return; + if (!btrfs_run_discard_work(discard_ctl)) { + spin_lock(&discard_ctl->lock); + btrfs_put_block_group(block_group); + discard_ctl->block_group = NULL; + spin_unlock(&discard_ctl->lock); + return; + } + if (now < block_group->discard_eligible_time) { + spin_lock(&discard_ctl->lock); + btrfs_put_block_group(block_group); + discard_ctl->block_group = NULL; + spin_unlock(&discard_ctl->lock); + btrfs_discard_schedule_work(discard_ctl, false); + return; + } + + /* Perform discarding */ + minlen = discard_minlen[discard_index]; + + switch (discard_state) { + case BTRFS_DISCARD_BITMAPS: { + u64 maxlen = 0; + + /* + * Use the previous levels minimum discard length as the max + * length filter. In the case something is added to make a + * region go beyond the max filter, the entire bitmap is set + * back to BTRFS_TRIM_STATE_UNTRIMMED. + */ + if (discard_index != BTRFS_DISCARD_INDEX_UNUSED) + maxlen = discard_minlen[discard_index - 1]; + + btrfs_trim_block_group_bitmaps(block_group, &trimmed, + block_group->discard_cursor, + btrfs_block_group_end(block_group), + minlen, maxlen, true); + discard_ctl->discard_bitmap_bytes += trimmed; + + break; + } + + case BTRFS_DISCARD_FULLY_REMAPPED: + btrfs_trim_fully_remapped_block_group(block_group); + break; + + default: + btrfs_trim_block_group_extents(block_group, &trimmed, + block_group->discard_cursor, + btrfs_block_group_end(block_group), + minlen, true); + discard_ctl->discard_extent_bytes += trimmed; + + break; + } + + /* Determine next steps for a block_group */ + if (block_group->discard_cursor >= btrfs_block_group_end(block_group)) { + if (discard_state == BTRFS_DISCARD_BITMAPS || + discard_state == BTRFS_DISCARD_FULLY_REMAPPED) { + btrfs_finish_discard_pass(discard_ctl, block_group); + } else { + block_group->discard_cursor = block_group->start; + spin_lock(&discard_ctl->lock); + if (block_group->discard_state != + BTRFS_DISCARD_RESET_CURSOR) + block_group->discard_state = + BTRFS_DISCARD_BITMAPS; + spin_unlock(&discard_ctl->lock); + } + } + + now = ktime_get_ns(); + spin_lock(&discard_ctl->lock); + discard_ctl->prev_discard = trimmed; + discard_ctl->prev_discard_time = now; + btrfs_put_block_group(block_group); + discard_ctl->block_group = NULL; + __btrfs_discard_schedule_work(discard_ctl, now, false); + spin_unlock(&discard_ctl->lock); +} + +/* + * Recalculate the base delay. + * + * @discard_ctl: discard control + * + * Recalculate the base delay which is based off the total number of + * discardable_extents. Clamp this between the lower_limit (iops_limit or 1ms) + * and the upper_limit (BTRFS_DISCARD_MAX_DELAY_MSEC). + */ +void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl) +{ + s32 discardable_extents; + s64 discardable_bytes; + u32 iops_limit; + unsigned long min_delay = BTRFS_DISCARD_MIN_DELAY_MSEC; + unsigned long delay; + + discardable_extents = atomic_read(&discard_ctl->discardable_extents); + if (!discardable_extents) + return; + + spin_lock(&discard_ctl->lock); + + /* + * The following is to fix a potential -1 discrepancy that we're not + * sure how to reproduce. But given that this is the only place that + * utilizes these numbers and this is only called by from + * btrfs_finish_extent_commit() which is synchronized, we can correct + * here. + */ + if (discardable_extents < 0) + atomic_add(-discardable_extents, + &discard_ctl->discardable_extents); + + discardable_bytes = atomic64_read(&discard_ctl->discardable_bytes); + if (discardable_bytes < 0) + atomic64_add(-discardable_bytes, + &discard_ctl->discardable_bytes); + + if (discardable_extents <= 0) { + spin_unlock(&discard_ctl->lock); + return; + } + + iops_limit = READ_ONCE(discard_ctl->iops_limit); + + if (iops_limit) { + delay = MSEC_PER_SEC / iops_limit; + } else { + /* + * Unset iops_limit means go as fast as possible, so allow a + * delay of 0. + */ + delay = 0; + min_delay = 0; + } + + delay = clamp(delay, min_delay, BTRFS_DISCARD_MAX_DELAY_MSEC); + discard_ctl->delay_ms = delay; + + spin_unlock(&discard_ctl->lock); +} + +/* + * Propagate discard counters. + * + * @block_group: block_group of interest + * + * Propagate deltas of counters up to the discard_ctl. It maintains a current + * counter and a previous counter passing the delta up to the global stat. + * Then the current counter value becomes the previous counter value. + */ +void btrfs_discard_update_discardable(struct btrfs_block_group *block_group) +{ + struct btrfs_free_space_ctl *ctl; + struct btrfs_discard_ctl *discard_ctl; + s32 extents_delta; + s64 bytes_delta; + + if (!block_group || + !btrfs_test_opt(block_group->fs_info, DISCARD_ASYNC) || + !btrfs_is_block_group_data_only(block_group)) + return; + + ctl = block_group->free_space_ctl; + discard_ctl = &block_group->fs_info->discard_ctl; + + lockdep_assert_held(&ctl->tree_lock); + extents_delta = ctl->discardable_extents[BTRFS_STAT_CURR] - + ctl->discardable_extents[BTRFS_STAT_PREV]; + if (extents_delta) { + atomic_add(extents_delta, &discard_ctl->discardable_extents); + ctl->discardable_extents[BTRFS_STAT_PREV] = + ctl->discardable_extents[BTRFS_STAT_CURR]; + } + + bytes_delta = ctl->discardable_bytes[BTRFS_STAT_CURR] - + ctl->discardable_bytes[BTRFS_STAT_PREV]; + if (bytes_delta) { + atomic64_add(bytes_delta, &discard_ctl->discardable_bytes); + ctl->discardable_bytes[BTRFS_STAT_PREV] = + ctl->discardable_bytes[BTRFS_STAT_CURR]; + } +} + +/* + * Punt unused_bgs list to discard lists. + * + * @fs_info: fs_info of interest + * + * The unused_bgs list needs to be punted to the discard lists because the + * order of operations is changed. In the normal synchronous discard path, the + * block groups are trimmed via a single large trim in transaction commit. This + * is ultimately what we are trying to avoid with asynchronous discard. Thus, + * it must be done before going down the unused_bgs path. + */ +void btrfs_discard_punt_unused_bgs_list(struct btrfs_fs_info *fs_info) +{ + struct btrfs_block_group *block_group, *next; + + spin_lock(&fs_info->unused_bgs_lock); + /* We enabled async discard, so punt all to the queue */ + list_for_each_entry_safe(block_group, next, &fs_info->unused_bgs, + bg_list) { + list_del_init(&block_group->bg_list); + btrfs_discard_queue_work(&fs_info->discard_ctl, block_group); + /* + * This put is for the get done by btrfs_mark_bg_unused. + * Queueing discard incremented it for discard's reference. + */ + btrfs_put_block_group(block_group); + } + spin_unlock(&fs_info->unused_bgs_lock); +} + +/* + * Purge discard lists. + * + * @discard_ctl: discard control + * + * If we are disabling async discard, we may have intercepted block groups that + * are completely free and ready for the unused_bgs path. As discarding will + * now happen in transaction commit or not at all, we can safely mark the + * corresponding block groups as unused and they will be sent on their merry + * way to the unused_bgs list. + */ +static void btrfs_discard_purge_list(struct btrfs_discard_ctl *discard_ctl) +{ + struct btrfs_block_group *block_group, *next; + int i; + + spin_lock(&discard_ctl->lock); + for (i = 0; i < BTRFS_NR_DISCARD_LISTS; i++) { + list_for_each_entry_safe(block_group, next, + &discard_ctl->discard_list[i], + discard_list) { + list_del_init(&block_group->discard_list); + spin_unlock(&discard_ctl->lock); + if (block_group->used == 0) + btrfs_mark_bg_unused(block_group); + spin_lock(&discard_ctl->lock); + btrfs_put_block_group(block_group); + } + } + spin_unlock(&discard_ctl->lock); +} + +void btrfs_discard_resume(struct btrfs_fs_info *fs_info) +{ + if (!btrfs_test_opt(fs_info, DISCARD_ASYNC)) { + btrfs_discard_cleanup(fs_info); + return; + } + + btrfs_discard_punt_unused_bgs_list(fs_info); + + set_bit(BTRFS_FS_DISCARD_RUNNING, &fs_info->flags); +} + +void btrfs_discard_stop(struct btrfs_fs_info *fs_info) +{ + clear_bit(BTRFS_FS_DISCARD_RUNNING, &fs_info->flags); +} + +void btrfs_discard_init(struct btrfs_fs_info *fs_info) +{ + struct btrfs_discard_ctl *discard_ctl = &fs_info->discard_ctl; + int i; + + spin_lock_init(&discard_ctl->lock); + INIT_DELAYED_WORK(&discard_ctl->work, btrfs_discard_workfn); + + for (i = 0; i < BTRFS_NR_DISCARD_LISTS; i++) + INIT_LIST_HEAD(&discard_ctl->discard_list[i]); + + discard_ctl->prev_discard = 0; + discard_ctl->prev_discard_time = 0; + atomic_set(&discard_ctl->discardable_extents, 0); + atomic64_set(&discard_ctl->discardable_bytes, 0); + discard_ctl->max_discard_size = BTRFS_ASYNC_DISCARD_DEFAULT_MAX_SIZE; + discard_ctl->delay_ms = BTRFS_DISCARD_MAX_DELAY_MSEC; + discard_ctl->iops_limit = BTRFS_DISCARD_MAX_IOPS; + discard_ctl->kbps_limit = 0; + discard_ctl->discard_extent_bytes = 0; + discard_ctl->discard_bitmap_bytes = 0; + atomic64_set(&discard_ctl->discard_bytes_saved, 0); +} + +void btrfs_discard_cleanup(struct btrfs_fs_info *fs_info) +{ + btrfs_discard_stop(fs_info); + cancel_delayed_work_sync(&fs_info->discard_ctl.work); + btrfs_discard_purge_list(&fs_info->discard_ctl); +} diff --git a/fs/btrfs/discard.h b/fs/btrfs/discard.h new file mode 100644 index 000000000..2c5e85394 --- /dev/null +++ b/fs/btrfs/discard.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_DISCARD_H +#define BTRFS_DISCARD_H + +#include +#include + +struct btrfs_fs_info; +struct btrfs_discard_ctl; +struct btrfs_block_group; + +/* Discard size limits */ +#define BTRFS_ASYNC_DISCARD_DEFAULT_MAX_SIZE (SZ_64M) +#define BTRFS_ASYNC_DISCARD_MAX_FILTER (SZ_1M) +#define BTRFS_ASYNC_DISCARD_MIN_FILTER (SZ_32K) + +/* List operations */ +void btrfs_discard_check_filter(struct btrfs_block_group *block_group, u64 bytes); + +/* Work operations */ +void btrfs_discard_cancel_work(struct btrfs_discard_ctl *discard_ctl, + struct btrfs_block_group *block_group); +void btrfs_discard_queue_work(struct btrfs_discard_ctl *discard_ctl, + struct btrfs_block_group *block_group); +void btrfs_discard_schedule_work(struct btrfs_discard_ctl *discard_ctl, + bool override); + +/* Update operations */ +void btrfs_discard_calc_delay(struct btrfs_discard_ctl *discard_ctl); +void btrfs_discard_update_discardable(struct btrfs_block_group *block_group); + +/* Setup/cleanup operations */ +void btrfs_discard_punt_unused_bgs_list(struct btrfs_fs_info *fs_info); +void btrfs_discard_resume(struct btrfs_fs_info *fs_info); +void btrfs_discard_stop(struct btrfs_fs_info *fs_info); +void btrfs_discard_init(struct btrfs_fs_info *fs_info); +void btrfs_discard_cleanup(struct btrfs_fs_info *fs_info); + +#endif diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c new file mode 100644 index 000000000..819727460 --- /dev/null +++ b/fs/btrfs/disk-io.c @@ -0,0 +1,5097 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ctree.h" +#include "disk-io.h" +#include "transaction.h" +#include "btrfs_inode.h" +#include "delayed-inode.h" +#include "bio.h" +#include "print-tree.h" +#include "locking.h" +#include "tree-log.h" +#include "free-space-cache.h" +#include "free-space-tree.h" +#include "dev-replace.h" +#include "raid56.h" +#include "sysfs.h" +#include "qgroup.h" +#include "compression.h" +#include "tree-checker.h" +#include "ref-verify.h" +#include "block-group.h" +#include "discard.h" +#include "space-info.h" +#include "zoned.h" +#include "subpage.h" +#include "fs.h" +#include "accessors.h" +#include "extent-tree.h" +#include "root-tree.h" +#include "defrag.h" +#include "uuid-tree.h" +#include "relocation.h" +#include "scrub.h" +#include "super.h" + +#define BTRFS_SUPER_FLAG_SUPP (BTRFS_HEADER_FLAG_WRITTEN |\ + BTRFS_HEADER_FLAG_RELOC |\ + BTRFS_SUPER_FLAG_ERROR |\ + BTRFS_SUPER_FLAG_SEEDING |\ + BTRFS_SUPER_FLAG_METADUMP |\ + BTRFS_SUPER_FLAG_METADUMP_V2) + +static int btrfs_cleanup_transaction(struct btrfs_fs_info *fs_info); +static void btrfs_error_commit_super(struct btrfs_fs_info *fs_info); + +/* + * Compute the csum of a btree block and store the result to provided buffer. + */ +static void csum_tree_block(struct extent_buffer *buf, u8 *result) +{ + struct btrfs_fs_info *fs_info = buf->fs_info; + int num_pages; + u32 first_page_part; + struct btrfs_csum_ctx csum; + char *kaddr; + int i; + + btrfs_csum_init(&csum, fs_info->csum_type); + + if (buf->addr) { + /* Pages are contiguous, handle them as a big one. */ + kaddr = buf->addr; + first_page_part = fs_info->nodesize; + num_pages = 1; + } else { + kaddr = folio_address(buf->folios[0]); + first_page_part = min_t(u32, PAGE_SIZE, fs_info->nodesize); + num_pages = num_extent_pages(buf); + } + + btrfs_csum_update(&csum, kaddr + BTRFS_CSUM_SIZE, + first_page_part - BTRFS_CSUM_SIZE); + + /* + * Multiple single-page folios case would reach here. + * + * nodesize <= PAGE_SIZE and large folio all handled by above + * btrfs_csum_update() already. + */ + for (i = 1; i < num_pages && INLINE_EXTENT_BUFFER_PAGES > 1; i++) { + kaddr = folio_address(buf->folios[i]); + btrfs_csum_update(&csum, kaddr, PAGE_SIZE); + } + memset(result, 0, BTRFS_CSUM_SIZE); + btrfs_csum_final(&csum, result); +} + +/* + * we can't consider a given block up to date unless the transid of the + * block matches the transid in the parent node's pointer. This is how we + * detect blocks that either didn't get written at all or got written + * in the wrong place. + */ +int btrfs_buffer_uptodate(struct extent_buffer *eb, u64 parent_transid, + const struct btrfs_tree_parent_check *check) +{ + if (!extent_buffer_uptodate(eb)) + return 0; + + if (!parent_transid || btrfs_header_generation(eb) == parent_transid) { + /* + * On a cache hit, the caller may still need tree parent + * verification before reusing the buffer. + */ + if (unlikely(check && btrfs_verify_level_key(eb, check))) + return -EUCLEAN; + return 1; + } + + if (btrfs_header_generation(eb) != parent_transid) { + btrfs_err_rl(eb->fs_info, +"parent transid verify failed on logical %llu mirror %u wanted %llu found %llu", + eb->start, eb->read_mirror, + parent_transid, btrfs_header_generation(eb)); + clear_extent_buffer_uptodate(eb); + return 0; + } + return 1; +} + +static bool btrfs_supported_super_csum(u16 csum_type) +{ + switch (csum_type) { + case BTRFS_CSUM_TYPE_CRC32: + case BTRFS_CSUM_TYPE_XXHASH: + case BTRFS_CSUM_TYPE_SHA256: + case BTRFS_CSUM_TYPE_BLAKE2: + return true; + default: + return false; + } +} + +/* + * Return 0 if the superblock checksum type matches the checksum value of that + * algorithm. Pass the raw disk superblock data. + */ +int btrfs_check_super_csum(struct btrfs_fs_info *fs_info, + const struct btrfs_super_block *disk_sb) +{ + u8 result[BTRFS_CSUM_SIZE]; + + /* + * The super_block structure does not span the whole + * BTRFS_SUPER_INFO_SIZE range, we expect that the unused space is + * filled with zeros and is included in the checksum. + */ + btrfs_csum(fs_info->csum_type, (const u8 *)disk_sb + BTRFS_CSUM_SIZE, + BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE, result); + + if (memcmp(disk_sb->csum, result, fs_info->csum_size)) + return 1; + + return 0; +} + +static int btrfs_repair_eb_io_failure(const struct extent_buffer *eb, + int mirror_num) +{ + struct btrfs_fs_info *fs_info = eb->fs_info; + const u32 step = min(fs_info->nodesize, PAGE_SIZE); + const u32 nr_steps = eb->len / step; + phys_addr_t paddrs[BTRFS_MAX_BLOCKSIZE / PAGE_SIZE]; + + if (sb_rdonly(fs_info->sb)) + return -EROFS; + + for (int i = 0; i < num_extent_pages(eb); i++) { + struct folio *folio = eb->folios[i]; + + /* No large folio support yet. */ + ASSERT(folio_order(folio) == 0); + ASSERT(i < nr_steps); + + /* + * For nodesize < page size, there is just one paddr, with some + * offset inside the page. + * + * For nodesize >= page size, it's one or more paddrs, and eb->start + * must be aligned to page boundary. + */ + paddrs[i] = page_to_phys(&folio->page) + offset_in_page(eb->start); + } + + return btrfs_repair_io_failure(fs_info, 0, eb->start, eb->len, + eb->start, paddrs, step, mirror_num); +} + +/* + * helper to read a given tree block, doing retries as required when + * the checksums don't match and we have alternate mirrors to try. + * + * @check: expected tree parentness check, see the comments of the + * structure for details. + */ +int btrfs_read_extent_buffer(struct extent_buffer *eb, + const struct btrfs_tree_parent_check *check) +{ + struct btrfs_fs_info *fs_info = eb->fs_info; + bool failed = false; + int ret; + int num_copies = 0; + int mirror_num = 0; + int failed_mirror = 0; + + ASSERT(check); + + while (1) { + ret = read_extent_buffer_pages(eb, mirror_num, check); + if (!ret) + break; + + num_copies = btrfs_num_copies(fs_info, + eb->start, eb->len); + if (num_copies == 1) + break; + + if (!failed_mirror) { + failed = true; + failed_mirror = eb->read_mirror; + } + + mirror_num++; + if (mirror_num == failed_mirror) + mirror_num++; + + if (mirror_num > num_copies) + break; + } + + if (failed && !ret && failed_mirror) + btrfs_repair_eb_io_failure(eb, failed_mirror); + + return ret; +} + +/* + * Checksum a dirty tree block before IO. + */ +int btree_csum_one_bio(struct btrfs_bio *bbio) +{ + struct extent_buffer *eb = bbio->private; + struct btrfs_fs_info *fs_info = eb->fs_info; + u64 found_start = btrfs_header_bytenr(eb); + u64 last_trans; + u8 result[BTRFS_CSUM_SIZE]; + int ret; + + /* Btree blocks are always contiguous on disk. */ + if (WARN_ON_ONCE(bbio->file_offset != eb->start)) + return -EIO; + if (WARN_ON_ONCE(bbio->bio.bi_iter.bi_size != eb->len)) + return -EIO; + + /* + * An extent_buffer marked EXTENT_BUFFER_ZONED_ZEROOUT is written out as + * zeros to preserve ordering of I/O without persisting the now + * unnecessary block. The bio is fed from the shared zero page (see + * write_one_eb()), so there is nothing to checksum here. Crucially, the + * buffer's own content is left intact: it may still be referenced, e.g. + * btrfs_free_tree_block() reads its header to add a delayed reference. + */ + if (test_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags)) + return 0; + + if (WARN_ON_ONCE(found_start != eb->start)) + return -EIO; + if (WARN_ON(!btrfs_meta_folio_test_uptodate(eb->folios[0], eb))) + return -EIO; + + ASSERT(memcmp_extent_buffer(eb, fs_info->fs_devices->metadata_uuid, + offsetof(struct btrfs_header, fsid), + BTRFS_FSID_SIZE) == 0); + csum_tree_block(eb, result); + + if (btrfs_header_level(eb)) + ret = btrfs_check_node(eb); + else + ret = btrfs_check_leaf(eb); + + if (ret < 0) + goto error; + + /* + * Also check the generation, the eb reached here must be newer than + * last committed. Or something seriously wrong happened. + */ + last_trans = btrfs_get_last_trans_committed(fs_info); + if (unlikely(btrfs_header_generation(eb) <= last_trans)) { + ret = -EUCLEAN; + btrfs_err(fs_info, + "block=%llu bad generation, have %llu expect > %llu", + eb->start, btrfs_header_generation(eb), last_trans); + goto error; + } + write_extent_buffer(eb, result, 0, fs_info->csum_size); + return 0; + +error: + btrfs_print_tree(eb, 0); + btrfs_err(fs_info, "block=%llu write time tree block corruption detected", + eb->start); + /* + * Be noisy if this is an extent buffer from a log tree. We don't abort + * a transaction in case there's a bad log tree extent buffer, we just + * fallback to a transaction commit. Still we want to know when there is + * a bad log tree extent buffer, as that may signal a bug somewhere. + */ + WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG) || + btrfs_header_owner(eb) == BTRFS_TREE_LOG_OBJECTID); + return ret; +} + +static bool check_tree_block_fsid(struct extent_buffer *eb) +{ + struct btrfs_fs_info *fs_info = eb->fs_info; + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs; + u8 fsid[BTRFS_FSID_SIZE]; + + read_extent_buffer(eb, fsid, offsetof(struct btrfs_header, fsid), + BTRFS_FSID_SIZE); + + /* + * alloc_fsid_devices() copies the fsid into fs_devices::metadata_uuid. + * This is then overwritten by metadata_uuid if it is present in the + * device_list_add(). The same true for a seed device as well. So use of + * fs_devices::metadata_uuid is appropriate here. + */ + if (memcmp(fsid, fs_info->fs_devices->metadata_uuid, BTRFS_FSID_SIZE) == 0) + return false; + + list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) + if (!memcmp(fsid, seed_devs->fsid, BTRFS_FSID_SIZE)) + return false; + + return true; +} + +/* Do basic extent buffer checks at read time */ +int btrfs_validate_extent_buffer(struct extent_buffer *eb, + const struct btrfs_tree_parent_check *check) +{ + struct btrfs_fs_info *fs_info = eb->fs_info; + u64 found_start; + const u32 csum_size = fs_info->csum_size; + u8 found_level; + u8 result[BTRFS_CSUM_SIZE]; + const u8 *header_csum; + int ret = 0; + const bool ignore_csum = btrfs_test_opt(fs_info, IGNOREMETACSUMS); + + ASSERT(check); + + found_start = btrfs_header_bytenr(eb); + if (unlikely(found_start != eb->start)) { + btrfs_err_rl(fs_info, + "bad tree block start, mirror %u want %llu have %llu", + eb->read_mirror, eb->start, found_start); + return -EIO; + } + if (unlikely(check_tree_block_fsid(eb))) { + btrfs_err_rl(fs_info, "bad fsid on logical %llu mirror %u", + eb->start, eb->read_mirror); + return -EIO; + } + found_level = btrfs_header_level(eb); + if (unlikely(found_level >= BTRFS_MAX_LEVEL)) { + btrfs_err(fs_info, + "bad tree block level, mirror %u level %d on logical %llu", + eb->read_mirror, btrfs_header_level(eb), eb->start); + return -EIO; + } + + csum_tree_block(eb, result); + header_csum = folio_address(eb->folios[0]) + + get_eb_offset_in_folio(eb, offsetof(struct btrfs_header, csum)); + + if (memcmp(result, header_csum, csum_size) != 0) { + btrfs_warn_rl(fs_info, +"checksum verify failed on logical %llu mirror %u wanted " BTRFS_CSUM_FMT " found " BTRFS_CSUM_FMT " level %d%s", + eb->start, eb->read_mirror, + BTRFS_CSUM_FMT_VALUE(csum_size, header_csum), + BTRFS_CSUM_FMT_VALUE(csum_size, result), + btrfs_header_level(eb), + ignore_csum ? ", ignored" : ""); + if (unlikely(!ignore_csum)) + return -EUCLEAN; + } + + if (unlikely(found_level != check->level)) { + btrfs_err(fs_info, + "level verify failed on logical %llu mirror %u wanted %u found %u", + eb->start, eb->read_mirror, check->level, found_level); + return -EIO; + } + if (unlikely(check->transid && + btrfs_header_generation(eb) != check->transid)) { + btrfs_err_rl(eb->fs_info, +"parent transid verify failed on logical %llu mirror %u wanted %llu found %llu", + eb->start, eb->read_mirror, check->transid, + btrfs_header_generation(eb)); + return -EIO; + } + if (check->has_first_key) { + const struct btrfs_key *expect_key = &check->first_key; + struct btrfs_key found_key; + + if (found_level) + btrfs_node_key_to_cpu(eb, &found_key, 0); + else + btrfs_item_key_to_cpu(eb, &found_key, 0); + if (unlikely(btrfs_comp_cpu_keys(expect_key, &found_key))) { + btrfs_err(fs_info, +"tree first key mismatch detected, bytenr=%llu parent_transid=%llu key expected=(%llu,%u,%llu) has=(%llu,%u,%llu)", + eb->start, check->transid, + expect_key->objectid, + expect_key->type, expect_key->offset, + found_key.objectid, found_key.type, + found_key.offset); + return -EUCLEAN; + } + } + if (check->owner_root) { + ret = btrfs_check_eb_owner(eb, check->owner_root); + if (ret < 0) + return ret; + } + + /* If this is a leaf block and it is corrupt, just return -EIO. */ + if (found_level == 0 && btrfs_check_leaf(eb)) + ret = -EIO; + + if (found_level > 0 && btrfs_check_node(eb)) + ret = -EIO; + + if (ret) + btrfs_err(fs_info, + "read time tree block corruption detected on logical %llu mirror %u", + eb->start, eb->read_mirror); + return ret; +} + +#ifdef CONFIG_MIGRATION +static int btree_migrate_folio(struct address_space *mapping, + struct folio *dst, struct folio *src, enum migrate_mode mode) +{ + /* + * we can't safely write a btree page from here, + * we haven't done the locking hook + */ + if (folio_test_dirty(src)) + return -EAGAIN; + /* + * Buffers may be managed in a filesystem specific way. + * We must have no buffers or drop them. + */ + if (folio_get_private(src) && + !filemap_release_folio(src, GFP_KERNEL)) + return -EAGAIN; + return migrate_folio(mapping, dst, src, mode); +} +#else +#define btree_migrate_folio NULL +#endif + +static bool btree_release_folio(struct folio *folio, gfp_t gfp_flags) +{ + if (folio_test_writeback(folio) || folio_test_dirty(folio)) + return false; + + return try_release_extent_buffer(folio); +} + +static void btree_invalidate_folio(struct folio *folio, size_t offset, + size_t length) +{ + struct extent_io_tree *tree = &folio_to_inode(folio)->io_tree; + struct extent_state *cached_state = NULL; + const u64 start = folio_pos(folio); + const u64 end = folio_next_pos(folio) - 1; + + /* + * The range must cover the full @folio. + * Btree inode is never exposed to regular file operations, thus there + * is no partial truncation. + * The folio is only invalidated when the btree inode is evicted. + */ + ASSERT(offset == 0, "folio=%llu offset=%zu", folio_pos(folio), offset); + ASSERT(length == folio_size(folio), "folio=%llu folio_size=%zu length=%zu", + folio_pos(folio), folio_size(folio), length); + + /* This function is only called for the btree inode */ + ASSERT(tree->owner == IO_TREE_BTREE_INODE_IO); + + btrfs_lock_extent(tree, start, end, &cached_state); + folio_wait_writeback(folio); + + /* + * Currently for btree io tree, only EXTENT_LOCKED is utilized, + * so here we only need to unlock the extent range to free any + * existing extent state. + */ + btrfs_unlock_extent(tree, start, end, &cached_state); + + btree_release_folio(folio, GFP_NOFS); + if (folio_get_private(folio)) { + btrfs_warn(folio_to_fs_info(folio), + "folio private not zero on folio %llu", + (unsigned long long)folio_pos(folio)); + folio_detach_private(folio); + } +} + +#ifdef DEBUG +static bool btree_dirty_folio(struct address_space *mapping, + struct folio *folio) +{ + struct btrfs_fs_info *fs_info = inode_to_fs_info(mapping->host); + struct btrfs_subpage_info *spi = fs_info->subpage_info; + struct btrfs_subpage *subpage; + struct extent_buffer *eb; + int cur_bit = 0; + u64 page_start = folio_pos(folio); + + if (fs_info->sectorsize == PAGE_SIZE) { + eb = folio_get_private(folio); + BUG_ON(!eb); + BUG_ON(!test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)); + BUG_ON(!atomic_read(&eb->refs)); + btrfs_assert_tree_write_locked(eb); + return filemap_dirty_folio(mapping, folio); + } + + ASSERT(spi); + subpage = folio_get_private(folio); + + for (cur_bit = spi->dirty_offset; + cur_bit < spi->dirty_offset + spi->bitmap_nr_bits; + cur_bit++) { + unsigned long flags; + u64 cur; + + spin_lock_irqsave(&subpage->lock, flags); + if (!test_bit(cur_bit, subpage->bitmaps)) { + spin_unlock_irqrestore(&subpage->lock, flags); + continue; + } + spin_unlock_irqrestore(&subpage->lock, flags); + cur = page_start + (cur_bit << fs_info->sectorsize_bits); + + eb = find_extent_buffer(fs_info, cur); + ASSERT(eb); + ASSERT(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)); + ASSERT(atomic_read(&eb->refs)); + btrfs_assert_tree_write_locked(eb); + free_extent_buffer(eb); + + cur_bit += (fs_info->nodesize >> fs_info->sectorsize_bits) - 1; + } + return filemap_dirty_folio(mapping, folio); +} +#else +#define btree_dirty_folio filemap_dirty_folio +#endif + +static const struct address_space_operations btree_aops = { + .writepages = btree_writepages, + .release_folio = btree_release_folio, + .invalidate_folio = btree_invalidate_folio, + .migrate_folio = btree_migrate_folio, + .dirty_folio = btree_dirty_folio, +}; + +struct extent_buffer *btrfs_find_create_tree_block( + struct btrfs_fs_info *fs_info, + struct btrfs_eb_prealloc *pa, + u64 bytenr, u64 owner_root, + int level) +{ + if (btrfs_is_testing(fs_info)) + return alloc_test_extent_buffer(fs_info, bytenr); + return alloc_extent_buffer(fs_info, pa, bytenr, owner_root, level); +} + +/* + * Read tree block at logical address @bytenr and do variant basic but critical + * verification. + * + * @check: expected tree parentness check, see comments of the + * structure for details. + */ +struct extent_buffer *read_tree_block(struct btrfs_fs_info *fs_info, u64 bytenr, + struct btrfs_tree_parent_check *check) +{ + struct btrfs_eb_prealloc pa = { 0 }; + struct extent_buffer *buf = NULL; + int ret; + + ASSERT(check); + + buf = btrfs_find_create_tree_block(fs_info, &pa, bytenr, check->owner_root, + check->level); + if (IS_ERR(buf)) + return buf; + + ret = btrfs_read_extent_buffer(buf, check); + if (ret) { + free_extent_buffer_stale(buf); + return ERR_PTR(ret); + } + return buf; + +} + +static struct btrfs_root *btrfs_alloc_root(struct btrfs_fs_info *fs_info, + u64 objectid, gfp_t flags) +{ + struct btrfs_root *root; + + root = kzalloc_obj(*root, flags); + if (!root) + return NULL; + + root->fs_info = fs_info; + root->root_key.objectid = objectid; + RB_CLEAR_NODE(&root->rb_node); + + xa_init(&root->inodes); + xa_init(&root->delayed_nodes); + + btrfs_init_root_block_rsv(root); + + INIT_LIST_HEAD(&root->dirty_list); + INIT_LIST_HEAD(&root->root_list); + INIT_LIST_HEAD(&root->delalloc_inodes); + INIT_LIST_HEAD(&root->delalloc_root); + INIT_LIST_HEAD(&root->ordered_extents); + INIT_LIST_HEAD(&root->ordered_root); + INIT_LIST_HEAD(&root->reloc_dirty_list); + spin_lock_init(&root->delalloc_lock); + spin_lock_init(&root->ordered_extent_lock); + spin_lock_init(&root->accounting_lock); + spin_lock_init(&root->qgroup_meta_rsv_lock); + mutex_init(&root->objectid_mutex); + mutex_init(&root->log_mutex); + mutex_init(&root->ordered_extent_mutex); + mutex_init(&root->delalloc_mutex); + init_waitqueue_head(&root->qgroup_flush_wait); + init_waitqueue_head(&root->log_writer_wait); + init_waitqueue_head(&root->log_commit_wait[0]); + init_waitqueue_head(&root->log_commit_wait[1]); + INIT_LIST_HEAD(&root->log_ctxs[0]); + INIT_LIST_HEAD(&root->log_ctxs[1]); + atomic_set(&root->log_writers, 0); + refcount_set(&root->refs, 1); + atomic_set(&root->snapshot_force_cow, 0); + atomic_set(&root->nr_swapfiles, 0); + root->log_transid_committed = -1; + if (!btrfs_is_testing(fs_info)) { + btrfs_extent_io_tree_init(fs_info, &root->dirty_log_pages, + IO_TREE_ROOT_DIRTY_LOG_PAGES); + btrfs_extent_io_tree_init(fs_info, &root->log_csum_range, + IO_TREE_LOG_CSUM_RANGE); + } + + spin_lock_init(&root->root_item_lock); + btrfs_qgroup_init_swapped_blocks(&root->swapped_blocks); +#ifdef CONFIG_BTRFS_DEBUG + INIT_LIST_HEAD(&root->leak_list); + spin_lock(&fs_info->fs_roots_radix_lock); + list_add_tail(&root->leak_list, &fs_info->allocated_roots); + spin_unlock(&fs_info->fs_roots_radix_lock); +#endif + + return root; +} + +#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS +/* Should only be used by the testing infrastructure */ +struct btrfs_root *btrfs_alloc_dummy_root(struct btrfs_fs_info *fs_info) +{ + struct btrfs_root *root; + + if (!fs_info) + return ERR_PTR(-EINVAL); + + root = btrfs_alloc_root(fs_info, BTRFS_ROOT_TREE_OBJECTID, GFP_KERNEL); + if (!root) + return ERR_PTR(-ENOMEM); + + /* We don't use the stripesize in selftest, set it as sectorsize */ + root->alloc_bytenr = 0; + + return root; +} +#endif + +static int global_root_cmp(struct rb_node *a_node, const struct rb_node *b_node) +{ + const struct btrfs_root *a = rb_entry(a_node, struct btrfs_root, rb_node); + const struct btrfs_root *b = rb_entry(b_node, struct btrfs_root, rb_node); + + return btrfs_comp_cpu_keys(&a->root_key, &b->root_key); +} + +static int global_root_key_cmp(const void *k, const struct rb_node *node) +{ + const struct btrfs_key *key = k; + const struct btrfs_root *root = rb_entry(node, struct btrfs_root, rb_node); + + return btrfs_comp_cpu_keys(key, &root->root_key); +} + +int btrfs_global_root_insert(struct btrfs_root *root) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct rb_node *tmp; + int ret = 0; + + write_lock(&fs_info->global_root_lock); + tmp = rb_find_add(&root->rb_node, &fs_info->global_root_tree, global_root_cmp); + write_unlock(&fs_info->global_root_lock); + + if (tmp) { + ret = -EEXIST; + btrfs_warn(fs_info, "global root %llu %llu already exists", + btrfs_root_id(root), root->root_key.offset); + } + return ret; +} + +void btrfs_global_root_delete(struct btrfs_root *root) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + + write_lock(&fs_info->global_root_lock); + rb_erase(&root->rb_node, &fs_info->global_root_tree); + write_unlock(&fs_info->global_root_lock); +} + +struct btrfs_root *btrfs_global_root(struct btrfs_fs_info *fs_info, + const struct btrfs_key *key) +{ + struct rb_node *node; + struct btrfs_root *root = NULL; + + read_lock(&fs_info->global_root_lock); + node = rb_find(key, &fs_info->global_root_tree, global_root_key_cmp); + if (node) + root = container_of(node, struct btrfs_root, rb_node); + read_unlock(&fs_info->global_root_lock); + + return root; +} + +static u64 btrfs_global_root_id(struct btrfs_fs_info *fs_info, u64 bytenr) +{ + struct btrfs_block_group *block_group; + u64 ret; + + if (!btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) + return 0; + + if (bytenr) + block_group = btrfs_lookup_block_group(fs_info, bytenr); + else + block_group = btrfs_lookup_first_block_group(fs_info, bytenr); + ASSERT(block_group); + if (!block_group) + return 0; + ret = block_group->global_root_id; + btrfs_put_block_group(block_group); + + return ret; +} + +struct btrfs_root *btrfs_csum_root(struct btrfs_fs_info *fs_info, u64 bytenr) +{ + const struct btrfs_key key = { + .objectid = BTRFS_CSUM_TREE_OBJECTID, + .type = BTRFS_ROOT_ITEM_KEY, + .offset = btrfs_global_root_id(fs_info, bytenr), + }; + + return btrfs_global_root(fs_info, &key); +} + +struct btrfs_root *btrfs_extent_root(struct btrfs_fs_info *fs_info, u64 bytenr) +{ + const struct btrfs_key key = { + .objectid = BTRFS_EXTENT_TREE_OBJECTID, + .type = BTRFS_ROOT_ITEM_KEY, + .offset = btrfs_global_root_id(fs_info, bytenr), + }; + + return btrfs_global_root(fs_info, &key); +} + +struct btrfs_root *btrfs_create_tree(struct btrfs_trans_handle *trans, + u64 objectid) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct extent_buffer *leaf; + struct btrfs_root *tree_root = fs_info->tree_root; + struct btrfs_root *root; + unsigned int nofs_flag; + int ret = 0; + + /* + * We're holding a transaction handle, so use a NOFS memory allocation + * context to avoid deadlock if reclaim happens. + */ + nofs_flag = memalloc_nofs_save(); + root = btrfs_alloc_root(fs_info, objectid, GFP_KERNEL); + memalloc_nofs_restore(nofs_flag); + if (!root) + return ERR_PTR(-ENOMEM); + + root->root_key.objectid = objectid; + root->root_key.type = BTRFS_ROOT_ITEM_KEY; + root->root_key.offset = 0; + + leaf = btrfs_alloc_tree_block(trans, root, 0, objectid, NULL, 0, 0, 0, + 0, BTRFS_NESTING_NORMAL); + if (IS_ERR(leaf)) { + ret = PTR_ERR(leaf); + leaf = NULL; + goto fail; + } + + root->node = leaf; + btrfs_mark_buffer_dirty(trans, leaf); + + root->commit_root = btrfs_root_node(root); + set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state); + + btrfs_set_root_flags(&root->root_item, 0); + btrfs_set_root_limit(&root->root_item, 0); + btrfs_set_root_bytenr(&root->root_item, leaf->start); + btrfs_set_root_generation(&root->root_item, trans->transid); + btrfs_set_root_level(&root->root_item, 0); + btrfs_set_root_refs(&root->root_item, 1); + btrfs_set_root_used(&root->root_item, leaf->len); + btrfs_set_root_last_snapshot(&root->root_item, 0); + btrfs_set_root_dirid(&root->root_item, 0); + if (btrfs_is_fstree(objectid)) + generate_random_guid(root->root_item.uuid); + else + export_guid(root->root_item.uuid, &guid_null); + btrfs_set_root_drop_level(&root->root_item, 0); + + btrfs_tree_unlock(leaf); + + ret = btrfs_insert_root(trans, tree_root, &root->root_key, &root->root_item); + if (ret) + goto fail; + + return root; + +fail: + btrfs_put_root(root); + + return ERR_PTR(ret); +} + +static struct btrfs_root *alloc_log_tree(struct btrfs_fs_info *fs_info) +{ + struct btrfs_root *root; + + root = btrfs_alloc_root(fs_info, BTRFS_TREE_LOG_OBJECTID, GFP_NOFS); + if (!root) + return ERR_PTR(-ENOMEM); + + root->root_key.objectid = BTRFS_TREE_LOG_OBJECTID; + root->root_key.type = BTRFS_ROOT_ITEM_KEY; + root->root_key.offset = BTRFS_TREE_LOG_OBJECTID; + + return root; +} + +int btrfs_alloc_log_tree_node(struct btrfs_trans_handle *trans, + struct btrfs_root *root) +{ + struct extent_buffer *leaf; + + /* + * DON'T set SHAREABLE bit for log trees. + * + * Log trees are not exposed to user space thus can't be snapshotted, + * and they go away before a real commit is actually done. + * + * They do store pointers to file data extents, and those reference + * counts still get updated (along with back refs to the log tree). + */ + + leaf = btrfs_alloc_tree_block(trans, root, 0, BTRFS_TREE_LOG_OBJECTID, + NULL, 0, 0, 0, 0, BTRFS_NESTING_NORMAL); + if (IS_ERR(leaf)) + return PTR_ERR(leaf); + + root->node = leaf; + + btrfs_mark_buffer_dirty(trans, root->node); + btrfs_tree_unlock(root->node); + + return 0; +} + +int btrfs_init_log_root_tree(struct btrfs_trans_handle *trans, + struct btrfs_fs_info *fs_info) +{ + struct btrfs_root *log_root; + + log_root = alloc_log_tree(fs_info); + if (IS_ERR(log_root)) + return PTR_ERR(log_root); + + if (!btrfs_is_zoned(fs_info)) { + int ret = btrfs_alloc_log_tree_node(trans, log_root); + + if (ret) { + btrfs_put_root(log_root); + return ret; + } + } + + WARN_ON(fs_info->log_root_tree); + fs_info->log_root_tree = log_root; + return 0; +} + +int btrfs_add_log_tree(struct btrfs_trans_handle *trans, + struct btrfs_root *root) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_root *log_root; + struct btrfs_inode_item *inode_item; + int ret; + + log_root = alloc_log_tree(fs_info); + if (IS_ERR(log_root)) + return PTR_ERR(log_root); + + ret = btrfs_alloc_log_tree_node(trans, log_root); + if (ret) { + btrfs_put_root(log_root); + return ret; + } + + btrfs_set_root_last_trans(log_root, trans->transid); + log_root->root_key.offset = btrfs_root_id(root); + + inode_item = &log_root->root_item.inode; + btrfs_set_stack_inode_generation(inode_item, 1); + btrfs_set_stack_inode_size(inode_item, 3); + btrfs_set_stack_inode_nlink(inode_item, 1); + btrfs_set_stack_inode_nbytes(inode_item, + fs_info->nodesize); + btrfs_set_stack_inode_mode(inode_item, S_IFDIR | 0755); + + btrfs_set_root_node(&log_root->root_item, log_root->node); + + WARN_ON(root->log_root); + root->log_root = log_root; + btrfs_set_root_log_transid(root, 0); + root->log_transid_committed = -1; + btrfs_set_root_last_log_commit(root, 0); + return 0; +} + +static struct btrfs_root *read_tree_root_path(struct btrfs_root *tree_root, + struct btrfs_path *path, + const struct btrfs_key *key) +{ + struct btrfs_root *root; + struct btrfs_tree_parent_check check = { 0 }; + struct btrfs_fs_info *fs_info = tree_root->fs_info; + u64 generation; + int ret; + int level; + + root = btrfs_alloc_root(fs_info, key->objectid, GFP_NOFS); + if (!root) + return ERR_PTR(-ENOMEM); + + ret = btrfs_find_root(tree_root, key, path, + &root->root_item, &root->root_key); + if (ret) { + if (ret > 0) + ret = -ENOENT; + goto fail; + } + + generation = btrfs_root_generation(&root->root_item); + level = btrfs_root_level(&root->root_item); + check.level = level; + check.transid = generation; + check.owner_root = key->objectid; + root->node = read_tree_block(fs_info, btrfs_root_bytenr(&root->root_item), + &check); + if (IS_ERR(root->node)) { + ret = PTR_ERR(root->node); + root->node = NULL; + goto fail; + } + + ret = btrfs_buffer_uptodate(root->node, generation, &check); + if (unlikely(ret <= 0)) { + if (ret == 0) + ret = -EIO; + goto fail; + } + + /* + * For real fs, and not log/reloc trees, root owner must + * match its root node owner + */ + if (unlikely(!btrfs_is_testing(fs_info) && + btrfs_root_id(root) != BTRFS_TREE_LOG_OBJECTID && + btrfs_root_id(root) != BTRFS_TREE_RELOC_OBJECTID && + btrfs_root_id(root) != btrfs_header_owner(root->node))) { + btrfs_crit(fs_info, +"root=%llu block=%llu, tree root owner mismatch, have %llu expect %llu", + btrfs_root_id(root), root->node->start, + btrfs_header_owner(root->node), + btrfs_root_id(root)); + ret = -EUCLEAN; + goto fail; + } + root->commit_root = btrfs_root_node(root); + return root; +fail: + btrfs_put_root(root); + return ERR_PTR(ret); +} + +struct btrfs_root *btrfs_read_tree_root(struct btrfs_root *tree_root, + const struct btrfs_key *key) +{ + struct btrfs_root *root; + BTRFS_PATH_AUTO_FREE(path); + + path = btrfs_alloc_path(); + if (!path) + return ERR_PTR(-ENOMEM); + root = read_tree_root_path(tree_root, path, key); + + return root; +} + +/* + * Initialize subvolume root in-memory structure. + * + * @anon_dev: anonymous device to attach to the root, if zero, allocate new + * + * In case of failure the caller is responsible to call btrfs_free_fs_root() + */ +static int btrfs_init_fs_root(struct btrfs_root *root, dev_t anon_dev) +{ + int ret; + + btrfs_drew_lock_init(&root->snapshot_lock); + + if (btrfs_root_id(root) != BTRFS_TREE_LOG_OBJECTID && + !btrfs_is_data_reloc_root(root) && + btrfs_is_fstree(btrfs_root_id(root))) { + set_bit(BTRFS_ROOT_SHAREABLE, &root->state); + btrfs_check_and_init_root_item(&root->root_item); + } + + /* + * Don't assign anonymous block device to roots that are not exposed to + * userspace, the id pool is limited to 1M + */ + if (btrfs_is_fstree(btrfs_root_id(root)) && + btrfs_root_refs(&root->root_item) > 0) { + if (!anon_dev) { + ret = get_anon_bdev(&root->anon_dev); + if (ret) + return ret; + } else { + root->anon_dev = anon_dev; + } + } + + mutex_lock(&root->objectid_mutex); + ret = btrfs_init_root_free_objectid(root); + if (ret) { + mutex_unlock(&root->objectid_mutex); + return ret; + } + + ASSERT(root->free_objectid <= BTRFS_LAST_FREE_OBJECTID); + + mutex_unlock(&root->objectid_mutex); + + return 0; +} + +static struct btrfs_root *btrfs_lookup_fs_root(struct btrfs_fs_info *fs_info, + u64 root_id) +{ + struct btrfs_root *root; + + spin_lock(&fs_info->fs_roots_radix_lock); + root = radix_tree_lookup(&fs_info->fs_roots_radix, + (unsigned long)root_id); + root = btrfs_grab_root(root); + spin_unlock(&fs_info->fs_roots_radix_lock); + return root; +} + +static struct btrfs_root *btrfs_get_global_root(struct btrfs_fs_info *fs_info, + u64 objectid) +{ + struct btrfs_key key = { + .objectid = objectid, + .type = BTRFS_ROOT_ITEM_KEY, + .offset = 0, + }; + + switch (objectid) { + case BTRFS_ROOT_TREE_OBJECTID: + return btrfs_grab_root(fs_info->tree_root); + case BTRFS_EXTENT_TREE_OBJECTID: + return btrfs_grab_root(btrfs_global_root(fs_info, &key)); + case BTRFS_CHUNK_TREE_OBJECTID: + return btrfs_grab_root(fs_info->chunk_root); + case BTRFS_DEV_TREE_OBJECTID: + return btrfs_grab_root(fs_info->dev_root); + case BTRFS_CSUM_TREE_OBJECTID: + return btrfs_grab_root(btrfs_global_root(fs_info, &key)); + case BTRFS_QUOTA_TREE_OBJECTID: + return btrfs_grab_root(fs_info->quota_root); + case BTRFS_UUID_TREE_OBJECTID: + return btrfs_grab_root(fs_info->uuid_root); + case BTRFS_BLOCK_GROUP_TREE_OBJECTID: + return btrfs_grab_root(fs_info->block_group_root); + case BTRFS_FREE_SPACE_TREE_OBJECTID: + return btrfs_grab_root(btrfs_global_root(fs_info, &key)); + case BTRFS_RAID_STRIPE_TREE_OBJECTID: + return btrfs_grab_root(fs_info->stripe_root); + case BTRFS_REMAP_TREE_OBJECTID: + return btrfs_grab_root(fs_info->remap_root); + default: + return NULL; + } +} + +int btrfs_insert_fs_root(struct btrfs_fs_info *fs_info, + struct btrfs_root *root) +{ + int ret; + + ret = radix_tree_preload(GFP_NOFS); + if (ret) + return ret; + + spin_lock(&fs_info->fs_roots_radix_lock); + ret = radix_tree_insert(&fs_info->fs_roots_radix, + (unsigned long)btrfs_root_id(root), + root); + if (ret == 0) { + btrfs_grab_root(root); + set_bit(BTRFS_ROOT_IN_RADIX, &root->state); + } + spin_unlock(&fs_info->fs_roots_radix_lock); + radix_tree_preload_end(); + + return ret; +} + +void btrfs_check_leaked_roots(const struct btrfs_fs_info *fs_info) +{ +#ifdef CONFIG_BTRFS_DEBUG + struct btrfs_root *root; + + while (!list_empty(&fs_info->allocated_roots)) { + char buf[BTRFS_ROOT_NAME_BUF_LEN]; + + root = list_first_entry(&fs_info->allocated_roots, + struct btrfs_root, leak_list); + btrfs_err(fs_info, "leaked root %s refcount %d", + btrfs_root_name(&root->root_key, buf), + refcount_read(&root->refs)); + WARN_ON_ONCE(1); + while (refcount_read(&root->refs) > 1) + btrfs_put_root(root); + btrfs_put_root(root); + } +#endif +} + +static void free_global_roots(struct btrfs_fs_info *fs_info) +{ + struct btrfs_root *root; + struct rb_node *node; + + while ((node = rb_first_postorder(&fs_info->global_root_tree)) != NULL) { + root = rb_entry(node, struct btrfs_root, rb_node); + rb_erase(&root->rb_node, &fs_info->global_root_tree); + btrfs_put_root(root); + } +} + +void btrfs_free_fs_info(struct btrfs_fs_info *fs_info) +{ + struct percpu_counter *em_counter = &fs_info->evictable_extent_maps; + + if (fs_info->fs_devices) + btrfs_close_devices(fs_info->fs_devices); + btrfs_free_compress_wsm(fs_info); + percpu_counter_destroy(&fs_info->stats_read_blocks); + percpu_counter_destroy(&fs_info->dirty_metadata_bytes); + percpu_counter_destroy(&fs_info->delalloc_bytes); + percpu_counter_destroy(&fs_info->ordered_bytes); + if (percpu_counter_initialized(em_counter)) + ASSERT(percpu_counter_sum_positive(em_counter) == 0); + percpu_counter_destroy(em_counter); + percpu_counter_destroy(&fs_info->dev_replace.bio_counter); + btrfs_free_stripe_hash_table(fs_info); + btrfs_free_ref_cache(fs_info); + kfree(fs_info->balance_ctl); + free_global_roots(fs_info); + btrfs_put_root(fs_info->tree_root); + btrfs_put_root(fs_info->chunk_root); + btrfs_put_root(fs_info->dev_root); + btrfs_put_root(fs_info->quota_root); + btrfs_put_root(fs_info->uuid_root); + btrfs_put_root(fs_info->fs_root); + btrfs_put_root(fs_info->data_reloc_root); + btrfs_put_root(fs_info->block_group_root); + btrfs_put_root(fs_info->stripe_root); + btrfs_put_root(fs_info->remap_root); + btrfs_check_leaked_roots(fs_info); + btrfs_extent_buffer_leak_debug_check(fs_info); + kfree(fs_info->super_copy); + kfree(fs_info->super_for_commit); + kvfree(fs_info); +} + + +/* + * Get an in-memory reference of a root structure. + * + * For essential trees like root/extent tree, we grab it from fs_info directly. + * For subvolume trees, we check the cached filesystem roots first. If not + * found, then read it from disk and add it to cached fs roots. + * + * Caller should release the root by calling btrfs_put_root() after the usage. + * + * NOTE: Reloc and log trees can't be read by this function as they share the + * same root objectid. + * + * @objectid: root id + * @anon_dev: preallocated anonymous block device number for new roots, + * pass NULL for a new allocation. + * @check_ref: whether to check root item references, If true, return -ENOENT + * for orphan roots + */ +static struct btrfs_root *btrfs_get_root_ref(struct btrfs_fs_info *fs_info, + u64 objectid, dev_t *anon_dev, + bool check_ref) +{ + struct btrfs_root *root; + struct btrfs_path *path; + struct btrfs_key key; + int ret; + + root = btrfs_get_global_root(fs_info, objectid); + if (root) + return root; + + /* + * If we're called for non-subvolume trees, and above function didn't + * find one, do not try to read it from disk. + * + * This is namely for free-space-tree and quota tree, which can change + * at runtime and should only be grabbed from fs_info. + */ + if (!btrfs_is_fstree(objectid) && objectid != BTRFS_DATA_RELOC_TREE_OBJECTID) + return ERR_PTR(-ENOENT); +again: + root = btrfs_lookup_fs_root(fs_info, objectid); + if (root) { + /* + * Some other caller may have read out the newly inserted + * subvolume already (for things like backref walk etc). Not + * that common but still possible. In that case, we just need + * to free the anon_dev. + */ + if (unlikely(anon_dev && *anon_dev)) { + free_anon_bdev(*anon_dev); + *anon_dev = 0; + } + + if (check_ref && btrfs_root_refs(&root->root_item) == 0) { + btrfs_put_root(root); + return ERR_PTR(-ENOENT); + } + return root; + } + + key.objectid = objectid; + key.type = BTRFS_ROOT_ITEM_KEY; + key.offset = (u64)-1; + root = btrfs_read_tree_root(fs_info->tree_root, &key); + if (IS_ERR(root)) + return root; + + if (check_ref && btrfs_root_refs(&root->root_item) == 0) { + ret = -ENOENT; + goto fail; + } + + ret = btrfs_init_fs_root(root, anon_dev ? *anon_dev : 0); + if (ret) + goto fail; + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto fail; + } + key.objectid = BTRFS_ORPHAN_OBJECTID; + key.type = BTRFS_ORPHAN_ITEM_KEY; + key.offset = objectid; + + ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0); + btrfs_free_path(path); + if (ret < 0) + goto fail; + if (ret == 0) + set_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state); + + ret = btrfs_insert_fs_root(fs_info, root); + if (ret) { + if (ret == -EEXIST) { + btrfs_put_root(root); + goto again; + } + goto fail; + } + return root; +fail: + /* + * If our caller provided us an anonymous device, then it's his + * responsibility to free it in case we fail. So we have to set our + * root's anon_dev to 0 to avoid a double free, once by btrfs_put_root() + * and once again by our caller. + */ + if (anon_dev && *anon_dev) + root->anon_dev = 0; + btrfs_put_root(root); + return ERR_PTR(ret); +} + +/* + * Get in-memory reference of a root structure + * + * @objectid: tree objectid + * @check_ref: if set, verify that the tree exists and the item has at least + * one reference + */ +struct btrfs_root *btrfs_get_fs_root(struct btrfs_fs_info *fs_info, + u64 objectid, bool check_ref) +{ + return btrfs_get_root_ref(fs_info, objectid, NULL, check_ref); +} + +/* + * Get in-memory reference of a root structure, created as new, optionally pass + * the anonymous block device id + * + * @objectid: tree objectid + * @anon_dev: if NULL, allocate a new anonymous block device or use the + * parameter value if not NULL + */ +struct btrfs_root *btrfs_get_new_fs_root(struct btrfs_fs_info *fs_info, + u64 objectid, dev_t *anon_dev) +{ + return btrfs_get_root_ref(fs_info, objectid, anon_dev, true); +} + +/* + * Return a root for the given objectid. + * + * @fs_info: the fs_info + * @objectid: the objectid we need to lookup + * + * This is exclusively used for backref walking, and exists specifically because + * of how qgroups does lookups. Qgroups will do a backref lookup at delayed ref + * creation time, which means we may have to read the tree_root in order to look + * up a fs root that is not in memory. If the root is not in memory we will + * read the tree root commit root and look up the fs root from there. This is a + * temporary root, it will not be inserted into the radix tree as it doesn't + * have the most uptodate information, it'll simply be discarded once the + * backref code is finished using the root. + */ +struct btrfs_root *btrfs_get_fs_root_commit_root(struct btrfs_fs_info *fs_info, + struct btrfs_path *path, + u64 objectid) +{ + struct btrfs_root *root; + struct btrfs_key key; + + ASSERT(path->search_commit_root && path->skip_locking); + + /* + * This can return -ENOENT if we ask for a root that doesn't exist, but + * since this is called via the backref walking code we won't be looking + * up a root that doesn't exist, unless there's corruption. So if root + * != NULL just return it. + */ + root = btrfs_get_global_root(fs_info, objectid); + if (root) + return root; + + root = btrfs_lookup_fs_root(fs_info, objectid); + if (root) + return root; + + key.objectid = objectid; + key.type = BTRFS_ROOT_ITEM_KEY; + key.offset = (u64)-1; + root = read_tree_root_path(fs_info->tree_root, path, &key); + btrfs_release_path(path); + + return root; +} + +static int cleaner_kthread(void *arg) +{ + struct btrfs_fs_info *fs_info = arg; + int again; + + while (1) { + again = 0; + + set_bit(BTRFS_FS_CLEANER_RUNNING, &fs_info->flags); + + /* Make the cleaner go to sleep early. */ + if (btrfs_need_cleaner_sleep(fs_info)) + goto sleep; + + /* + * Do not do anything if we might cause open_ctree() to block + * before we have finished mounting the filesystem. + */ + if (!test_bit(BTRFS_FS_OPEN, &fs_info->flags)) + goto sleep; + + if (!mutex_trylock(&fs_info->cleaner_mutex)) + goto sleep; + + /* + * Avoid the problem that we change the status of the fs + * during the above check and trylock. + */ + if (btrfs_need_cleaner_sleep(fs_info)) { + mutex_unlock(&fs_info->cleaner_mutex); + goto sleep; + } + + if (test_and_clear_bit(BTRFS_FS_FEATURE_CHANGED, &fs_info->flags)) + btrfs_sysfs_feature_update(fs_info); + + btrfs_run_delayed_iputs(fs_info); + + again = btrfs_clean_one_deleted_snapshot(fs_info); + mutex_unlock(&fs_info->cleaner_mutex); + + /* + * The defragger has dealt with the R/O remount and umount, + * needn't do anything special here. + */ + btrfs_run_defrag_inodes(fs_info); + + if (btrfs_fs_incompat(fs_info, REMAP_TREE) && + !btrfs_test_opt(fs_info, DISCARD_ASYNC)) + btrfs_handle_fully_remapped_bgs(fs_info); + + /* + * Acquires fs_info->reclaim_bgs_lock to avoid racing + * with relocation (btrfs_relocate_chunk) and relocation + * acquires fs_info->cleaner_mutex (btrfs_relocate_block_group) + * after acquiring fs_info->reclaim_bgs_lock. So we + * can't hold, nor need to, fs_info->cleaner_mutex when deleting + * unused block groups. + */ + btrfs_delete_unused_bgs(fs_info); + + /* + * Reclaim block groups in the reclaim_bgs list after we deleted + * all unused block_groups. This possibly gives us some more free + * space. + */ + btrfs_reclaim_bgs(fs_info); +sleep: + clear_and_wake_up_bit(BTRFS_FS_CLEANER_RUNNING, &fs_info->flags); + if (kthread_should_park()) + kthread_parkme(); + if (kthread_should_stop()) + return 0; + if (!again) { + set_current_state(TASK_INTERRUPTIBLE); + schedule(); + __set_current_state(TASK_RUNNING); + } + } +} + +static int transaction_kthread(void *arg) +{ + struct btrfs_root *root = arg; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_trans_handle *trans; + struct btrfs_transaction *cur; + u64 transid; + time64_t delta; + unsigned long delay; + bool cannot_commit; + + do { + cannot_commit = false; + delay = secs_to_jiffies(fs_info->commit_interval); + mutex_lock(&fs_info->transaction_kthread_mutex); + + spin_lock(&fs_info->trans_lock); + cur = fs_info->running_transaction; + if (!cur) { + spin_unlock(&fs_info->trans_lock); + goto sleep; + } + + delta = ktime_get_seconds() - cur->start_time; + if (!test_and_clear_bit(BTRFS_FS_COMMIT_TRANS, &fs_info->flags) && + cur->state < TRANS_STATE_COMMIT_PREP && + delta < fs_info->commit_interval) { + spin_unlock(&fs_info->trans_lock); + delay -= secs_to_jiffies(delta - 1); + delay = min(delay, + secs_to_jiffies(fs_info->commit_interval)); + goto sleep; + } + transid = cur->transid; + spin_unlock(&fs_info->trans_lock); + + /* If the file system is aborted, this will always fail. */ + trans = btrfs_attach_transaction(root); + if (IS_ERR(trans)) { + if (PTR_ERR(trans) != -ENOENT) + cannot_commit = true; + goto sleep; + } + if (transid == trans->transid) { + btrfs_commit_transaction(trans); + } else { + btrfs_end_transaction(trans); + } +sleep: + wake_up_process(fs_info->cleaner_kthread); + mutex_unlock(&fs_info->transaction_kthread_mutex); + + if (unlikely(BTRFS_FS_ERROR(fs_info))) + btrfs_cleanup_transaction(fs_info); + if (!kthread_should_stop() && + (!btrfs_transaction_blocked(fs_info) || + cannot_commit)) + schedule_timeout_interruptible(delay); + } while (!kthread_should_stop()); + return 0; +} + +/* + * This will find the highest generation in the array of root backups. The + * index of the highest array is returned, or -EINVAL if we can't find + * anything. + * + * We check to make sure the array is valid by comparing the + * generation of the latest root in the array with the generation + * in the super block. If they don't match we pitch it. + */ +static int find_newest_super_backup(struct btrfs_fs_info *info) +{ + const u64 newest_gen = btrfs_super_generation(info->super_copy); + u64 cur; + struct btrfs_root_backup *root_backup; + int i; + + for (i = 0; i < BTRFS_NUM_BACKUP_ROOTS; i++) { + root_backup = info->super_copy->super_roots + i; + cur = btrfs_backup_tree_root_gen(root_backup); + if (cur == newest_gen) + return i; + } + + return -EINVAL; +} + +/* + * copy all the root pointers into the super backup array. + * this will bump the backup pointer by one when it is + * done + */ +static int backup_super_roots(struct btrfs_fs_info *info) +{ + const int next_backup = info->backup_root_index; + struct btrfs_root_backup *root_backup; + + root_backup = info->super_for_commit->super_roots + next_backup; + + /* + * make sure all of our padding and empty slots get zero filled + * regardless of which ones we use today + */ + memset(root_backup, 0, sizeof(*root_backup)); + + info->backup_root_index = (next_backup + 1) % BTRFS_NUM_BACKUP_ROOTS; + + btrfs_set_backup_tree_root(root_backup, info->tree_root->node->start); + btrfs_set_backup_tree_root_gen(root_backup, + btrfs_header_generation(info->tree_root->node)); + + btrfs_set_backup_tree_root_level(root_backup, + btrfs_header_level(info->tree_root->node)); + + btrfs_set_backup_chunk_root(root_backup, info->chunk_root->node->start); + btrfs_set_backup_chunk_root_gen(root_backup, + btrfs_header_generation(info->chunk_root->node)); + btrfs_set_backup_chunk_root_level(root_backup, + btrfs_header_level(info->chunk_root->node)); + + if (!btrfs_fs_incompat(info, EXTENT_TREE_V2)) { + struct btrfs_root *extent_root = btrfs_extent_root(info, 0); + struct btrfs_root *csum_root = btrfs_csum_root(info, 0); + + if (unlikely(!extent_root)) { + btrfs_err(info, "missing extent root for extent at bytenr 0"); + return -EUCLEAN; + } + if (unlikely(!csum_root)) { + btrfs_err(info, "missing csum root for extent at bytenr 0"); + return -EUCLEAN; + } + + btrfs_set_backup_extent_root(root_backup, + extent_root->node->start); + btrfs_set_backup_extent_root_gen(root_backup, + btrfs_header_generation(extent_root->node)); + btrfs_set_backup_extent_root_level(root_backup, + btrfs_header_level(extent_root->node)); + + btrfs_set_backup_csum_root(root_backup, csum_root->node->start); + btrfs_set_backup_csum_root_gen(root_backup, + btrfs_header_generation(csum_root->node)); + btrfs_set_backup_csum_root_level(root_backup, + btrfs_header_level(csum_root->node)); + } + + /* + * we might commit during log recovery, which happens before we set + * the fs_root. Make sure it is valid before we fill it in. + */ + if (info->fs_root && info->fs_root->node) { + btrfs_set_backup_fs_root(root_backup, + info->fs_root->node->start); + btrfs_set_backup_fs_root_gen(root_backup, + btrfs_header_generation(info->fs_root->node)); + btrfs_set_backup_fs_root_level(root_backup, + btrfs_header_level(info->fs_root->node)); + } + + btrfs_set_backup_dev_root(root_backup, info->dev_root->node->start); + btrfs_set_backup_dev_root_gen(root_backup, + btrfs_header_generation(info->dev_root->node)); + btrfs_set_backup_dev_root_level(root_backup, + btrfs_header_level(info->dev_root->node)); + + btrfs_set_backup_total_bytes(root_backup, + btrfs_super_total_bytes(info->super_copy)); + btrfs_set_backup_bytes_used(root_backup, + btrfs_super_bytes_used(info->super_copy)); + btrfs_set_backup_num_devices(root_backup, + btrfs_super_num_devices(info->super_copy)); + + /* + * if we don't copy this out to the super_copy, it won't get remembered + * for the next commit + */ + memcpy(&info->super_copy->super_roots, + &info->super_for_commit->super_roots, + sizeof(*root_backup) * BTRFS_NUM_BACKUP_ROOTS); + + return 0; +} + +/* + * Reads a backup root based on the passed priority. Prio 0 is the newest, prio + * 1/2/3 are 2nd newest/3rd newest/4th (oldest) backup roots + * + * @fs_info: filesystem whose backup roots need to be read + * @priority: priority of backup root required + * + * Returns backup root index on success and -EINVAL otherwise. + */ +static int read_backup_root(struct btrfs_fs_info *fs_info, u8 priority) +{ + int backup_index = find_newest_super_backup(fs_info); + struct btrfs_super_block *super = fs_info->super_copy; + struct btrfs_root_backup *root_backup; + + if (priority < BTRFS_NUM_BACKUP_ROOTS && backup_index >= 0) { + if (priority == 0) + return backup_index; + + backup_index = backup_index + BTRFS_NUM_BACKUP_ROOTS - priority; + backup_index %= BTRFS_NUM_BACKUP_ROOTS; + } else { + return -EINVAL; + } + + root_backup = super->super_roots + backup_index; + + btrfs_set_super_generation(super, + btrfs_backup_tree_root_gen(root_backup)); + btrfs_set_super_root(super, btrfs_backup_tree_root(root_backup)); + btrfs_set_super_root_level(super, + btrfs_backup_tree_root_level(root_backup)); + btrfs_set_super_bytes_used(super, btrfs_backup_bytes_used(root_backup)); + + /* + * Fixme: the total bytes and num_devices need to match or we should + * need a fsck + */ + btrfs_set_super_total_bytes(super, btrfs_backup_total_bytes(root_backup)); + btrfs_set_super_num_devices(super, btrfs_backup_num_devices(root_backup)); + + return backup_index; +} + +/* helper to cleanup workers */ +static void btrfs_stop_all_workers(struct btrfs_fs_info *fs_info) +{ + if (fs_info->fixup_workers) + destroy_workqueue(fs_info->fixup_workers); + btrfs_destroy_workqueue(fs_info->delalloc_workers); + btrfs_destroy_workqueue(fs_info->workers); + if (fs_info->endio_workers) + destroy_workqueue(fs_info->endio_workers); + if (fs_info->rmw_workers) + destroy_workqueue(fs_info->rmw_workers); + btrfs_destroy_workqueue(fs_info->endio_write_workers); + btrfs_destroy_workqueue(fs_info->endio_freespace_worker); + btrfs_destroy_workqueue(fs_info->delayed_workers); + btrfs_destroy_workqueue(fs_info->caching_workers); + btrfs_destroy_workqueue(fs_info->flush_workers); + btrfs_destroy_workqueue(fs_info->qgroup_rescan_workers); + if (fs_info->discard_ctl.discard_workers) + destroy_workqueue(fs_info->discard_ctl.discard_workers); + /* + * Now that all other work queues are destroyed, we can safely destroy + * the queues used for metadata I/O, since tasks from those other work + * queues can do metadata I/O operations. + */ + if (fs_info->endio_meta_workers) + destroy_workqueue(fs_info->endio_meta_workers); +} + +static void free_root_extent_buffers(struct btrfs_root *root) +{ + if (root) { + free_extent_buffer(root->node); + free_extent_buffer(root->commit_root); + root->node = NULL; + root->commit_root = NULL; + } +} + +static void free_global_root_pointers(struct btrfs_fs_info *fs_info) +{ + struct btrfs_root *root, *tmp; + + rbtree_postorder_for_each_entry_safe(root, tmp, + &fs_info->global_root_tree, + rb_node) + free_root_extent_buffers(root); +} + +/* helper to cleanup tree roots */ +static void free_root_pointers(struct btrfs_fs_info *info, bool free_chunk_root) +{ + free_root_extent_buffers(info->tree_root); + + free_global_root_pointers(info); + free_root_extent_buffers(info->dev_root); + free_root_extent_buffers(info->quota_root); + free_root_extent_buffers(info->uuid_root); + free_root_extent_buffers(info->fs_root); + free_root_extent_buffers(info->data_reloc_root); + free_root_extent_buffers(info->block_group_root); + free_root_extent_buffers(info->stripe_root); + free_root_extent_buffers(info->remap_root); + if (free_chunk_root) + free_root_extent_buffers(info->chunk_root); +} + +void btrfs_put_root(struct btrfs_root *root) +{ + if (!root) + return; + + if (refcount_dec_and_test(&root->refs)) { + if (WARN_ON(!xa_empty(&root->inodes))) + xa_destroy(&root->inodes); + if (WARN_ON(!xa_empty(&root->delayed_nodes))) + xa_destroy(&root->delayed_nodes); + WARN_ON(test_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state)); + if (root->anon_dev) + free_anon_bdev(root->anon_dev); + free_root_extent_buffers(root); +#ifdef CONFIG_BTRFS_DEBUG + spin_lock(&root->fs_info->fs_roots_radix_lock); + list_del_init(&root->leak_list); + spin_unlock(&root->fs_info->fs_roots_radix_lock); +#endif + kfree(root); + } +} + +void btrfs_free_fs_roots(struct btrfs_fs_info *fs_info) +{ + int ret; + struct btrfs_root *gang[8]; + int i; + + while (!list_empty(&fs_info->dead_roots)) { + gang[0] = list_first_entry(&fs_info->dead_roots, + struct btrfs_root, root_list); + list_del(&gang[0]->root_list); + + if (test_bit(BTRFS_ROOT_IN_RADIX, &gang[0]->state)) + btrfs_drop_and_free_fs_root(fs_info, gang[0]); + btrfs_put_root(gang[0]); + } + + while (1) { + ret = radix_tree_gang_lookup(&fs_info->fs_roots_radix, + (void **)gang, 0, + ARRAY_SIZE(gang)); + if (!ret) + break; + for (i = 0; i < ret; i++) + btrfs_drop_and_free_fs_root(fs_info, gang[i]); + } +} + +static void btrfs_init_scrub(struct btrfs_fs_info *fs_info) +{ + mutex_init(&fs_info->scrub_lock); + atomic_set(&fs_info->scrubs_running, 0); + atomic_set(&fs_info->scrub_pause_req, 0); + atomic_set(&fs_info->scrubs_paused, 0); + atomic_set(&fs_info->scrub_cancel_req, 0); + init_waitqueue_head(&fs_info->scrub_pause_wait); + refcount_set(&fs_info->scrub_workers_refcnt, 0); +} + +static void btrfs_init_balance(struct btrfs_fs_info *fs_info) +{ + spin_lock_init(&fs_info->balance_lock); + mutex_init(&fs_info->balance_mutex); + atomic_set(&fs_info->balance_pause_req, 0); + atomic_set(&fs_info->balance_cancel_req, 0); + fs_info->balance_ctl = NULL; + init_waitqueue_head(&fs_info->balance_wait_q); + atomic_set(&fs_info->reloc_cancel_req, 0); +} + +static int btrfs_init_btree_inode(struct super_block *sb) +{ + struct btrfs_fs_info *fs_info = btrfs_sb(sb); + unsigned long hash = btrfs_inode_hash(BTRFS_BTREE_INODE_OBJECTID, + fs_info->tree_root); + struct inode *inode; + + inode = new_inode(sb); + if (!inode) + return -ENOMEM; + + btrfs_set_inode_number(BTRFS_I(inode), BTRFS_BTREE_INODE_OBJECTID); + set_nlink(inode, 1); + /* + * we set the i_size on the btree inode to the max possible int. + * the real end of the address space is determined by all of + * the devices in the system + */ + inode->i_size = OFFSET_MAX; + inode->i_mapping->a_ops = &btree_aops; + mapping_set_gfp_mask(inode->i_mapping, GFP_NOFS); + + btrfs_extent_io_tree_init(fs_info, &BTRFS_I(inode)->io_tree, + IO_TREE_BTREE_INODE_IO); + btrfs_extent_map_tree_init(&BTRFS_I(inode)->extent_tree); + + BTRFS_I(inode)->root = btrfs_grab_root(fs_info->tree_root); + set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags); + __insert_inode_hash(inode, hash); + set_bit(AS_KERNEL_FILE, &inode->i_mapping->flags); + fs_info->btree_inode = inode; + + return 0; +} + +static void btrfs_init_dev_replace_locks(struct btrfs_fs_info *fs_info) +{ + mutex_init(&fs_info->dev_replace.lock_finishing_cancel_unmount); + init_rwsem(&fs_info->dev_replace.rwsem); + init_waitqueue_head(&fs_info->dev_replace.replace_wait); +} + +static void btrfs_init_qgroup(struct btrfs_fs_info *fs_info) +{ + spin_lock_init(&fs_info->qgroup_lock); + mutex_init(&fs_info->qgroup_ioctl_lock); + fs_info->qgroup_tree = RB_ROOT; + INIT_LIST_HEAD(&fs_info->dirty_qgroups); + fs_info->qgroup_seq = 1; + fs_info->qgroup_rescan_running = false; + fs_info->qgroup_drop_subtree_thres = BTRFS_QGROUP_DROP_SUBTREE_THRES_DEFAULT; + mutex_init(&fs_info->qgroup_rescan_lock); +} + +static int btrfs_init_workqueues(struct btrfs_fs_info *fs_info) +{ + u32 max_active = fs_info->thread_pool_size; + unsigned int flags = WQ_MEM_RECLAIM | WQ_FREEZABLE | WQ_UNBOUND; + unsigned int ordered_flags = WQ_MEM_RECLAIM | WQ_FREEZABLE; + + fs_info->workers = + btrfs_alloc_workqueue(fs_info, "worker", flags, max_active, 16); + + fs_info->delalloc_workers = + btrfs_alloc_workqueue(fs_info, "delalloc", + flags, max_active, 2); + + fs_info->flush_workers = + btrfs_alloc_workqueue(fs_info, "flush_delalloc", + flags, max_active, 0); + + fs_info->caching_workers = + btrfs_alloc_workqueue(fs_info, "cache", flags, max_active, 0); + + fs_info->fixup_workers = + alloc_ordered_workqueue("btrfs-fixup", ordered_flags); + + fs_info->endio_workers = + alloc_workqueue("btrfs-endio", flags, max_active); + fs_info->endio_meta_workers = + alloc_workqueue("btrfs-endio-meta", flags, max_active); + fs_info->rmw_workers = alloc_workqueue("btrfs-rmw", flags, max_active); + fs_info->endio_write_workers = + btrfs_alloc_workqueue(fs_info, "endio-write", flags, + max_active, 2); + fs_info->endio_freespace_worker = + btrfs_alloc_workqueue(fs_info, "freespace-write", flags, + max_active, 0); + fs_info->delayed_workers = + btrfs_alloc_workqueue(fs_info, "delayed-meta", flags, + max_active, 0); + fs_info->qgroup_rescan_workers = + btrfs_alloc_ordered_workqueue(fs_info, "qgroup-rescan", + ordered_flags); + fs_info->discard_ctl.discard_workers = + alloc_ordered_workqueue("btrfs-discard", WQ_FREEZABLE); + + if (!(fs_info->workers && + fs_info->delalloc_workers && fs_info->flush_workers && + fs_info->endio_workers && fs_info->endio_meta_workers && + fs_info->endio_write_workers && + fs_info->endio_freespace_worker && fs_info->rmw_workers && + fs_info->caching_workers && fs_info->fixup_workers && + fs_info->delayed_workers && fs_info->qgroup_rescan_workers && + fs_info->discard_ctl.discard_workers)) { + return -ENOMEM; + } + + return 0; +} + +static void btrfs_init_csum_hash(struct btrfs_fs_info *fs_info, u16 csum_type) +{ + /* Check if the checksum implementation is a fast accelerated one. */ + switch (csum_type) { + case BTRFS_CSUM_TYPE_CRC32: + if (crc32_optimizations() & CRC32C_OPTIMIZATION) + set_bit(BTRFS_FS_CSUM_IMPL_FAST, &fs_info->flags); + break; + case BTRFS_CSUM_TYPE_XXHASH: + set_bit(BTRFS_FS_CSUM_IMPL_FAST, &fs_info->flags); + break; + default: + break; + } + + btrfs_info(fs_info, "using %s checksum algorithm", + btrfs_super_csum_name(csum_type)); +} + +static int btrfs_replay_log(struct btrfs_fs_info *fs_info, + struct btrfs_fs_devices *fs_devices) +{ + int ret; + struct btrfs_tree_parent_check check = { 0 }; + struct btrfs_root *log_tree_root; + struct btrfs_super_block *disk_super = fs_info->super_copy; + u64 bytenr = btrfs_super_log_root(disk_super); + int level = btrfs_super_log_root_level(disk_super); + + if (unlikely(fs_devices->rw_devices == 0)) { + btrfs_err(fs_info, "log replay required on RO media"); + return -EIO; + } + + log_tree_root = btrfs_alloc_root(fs_info, BTRFS_TREE_LOG_OBJECTID, + GFP_KERNEL); + if (!log_tree_root) + return -ENOMEM; + + check.level = level; + check.transid = fs_info->generation + 1; + check.owner_root = BTRFS_TREE_LOG_OBJECTID; + log_tree_root->node = read_tree_block(fs_info, bytenr, &check); + if (IS_ERR(log_tree_root->node)) { + ret = PTR_ERR(log_tree_root->node); + log_tree_root->node = NULL; + btrfs_err(fs_info, "failed to read log tree with error: %pe", ERR_PTR(ret)); + btrfs_put_root(log_tree_root); + return ret; + } + + /* returns with log_tree_root freed on success */ + ret = btrfs_recover_log_trees(log_tree_root); + btrfs_put_root(log_tree_root); + if (unlikely(ret)) { + ASSERT(BTRFS_FS_ERROR(fs_info) != 0); + btrfs_err(fs_info, "failed to recover log trees with error: %pe", ERR_PTR(ret)); + return ret; + } + + if (sb_rdonly(fs_info->sb)) { + ret = btrfs_commit_super(fs_info); + if (ret) + return ret; + } + + return 0; +} + +static int load_global_roots_objectid(struct btrfs_root *tree_root, + struct btrfs_path *path, u64 objectid, + const char *name) +{ + struct btrfs_fs_info *fs_info = tree_root->fs_info; + struct btrfs_root *root; + u64 max_global_id = 0; + int ret; + struct btrfs_key key = { + .objectid = objectid, + .type = BTRFS_ROOT_ITEM_KEY, + .offset = 0, + }; + bool found = false; + + /* If we have IGNOREDATACSUMS skip loading these roots. */ + if (objectid == BTRFS_CSUM_TREE_OBJECTID && + btrfs_test_opt(fs_info, IGNOREDATACSUMS)) { + set_bit(BTRFS_FS_STATE_NO_DATA_CSUMS, &fs_info->fs_state); + return 0; + } + + while (1) { + ret = btrfs_search_slot(NULL, tree_root, &key, path, 0, 0); + if (ret < 0) + break; + + if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) { + ret = btrfs_next_leaf(tree_root, path); + if (ret) { + if (ret > 0) + ret = 0; + break; + } + } + ret = 0; + + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + if (key.objectid != objectid) + break; + btrfs_release_path(path); + + /* + * Just worry about this for extent tree, it'll be the same for + * everybody. + */ + if (objectid == BTRFS_EXTENT_TREE_OBJECTID) + max_global_id = max(max_global_id, key.offset); + + found = true; + root = read_tree_root_path(tree_root, path, &key); + if (IS_ERR(root)) { + ret = PTR_ERR(root); + break; + } + set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state); + ret = btrfs_global_root_insert(root); + if (ret) { + btrfs_put_root(root); + break; + } + key.offset++; + } + btrfs_release_path(path); + + if (objectid == BTRFS_EXTENT_TREE_OBJECTID) + fs_info->nr_global_roots = max_global_id + 1; + + if (!found || ret) { + if (objectid == BTRFS_CSUM_TREE_OBJECTID) + set_bit(BTRFS_FS_STATE_NO_DATA_CSUMS, &fs_info->fs_state); + + if (!btrfs_test_opt(fs_info, IGNOREBADROOTS)) + ret = ret ? ret : -ENOENT; + else + ret = 0; + btrfs_err(fs_info, "failed to load root %s", name); + } + return ret; +} + +static int load_global_roots(struct btrfs_root *tree_root) +{ + BTRFS_PATH_AUTO_FREE(path); + int ret; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + ret = load_global_roots_objectid(tree_root, path, + BTRFS_EXTENT_TREE_OBJECTID, "extent"); + if (ret) + return ret; + ret = load_global_roots_objectid(tree_root, path, + BTRFS_CSUM_TREE_OBJECTID, "csum"); + if (ret) + return ret; + if (!btrfs_fs_compat_ro(tree_root->fs_info, FREE_SPACE_TREE)) + return ret; + + return load_global_roots_objectid(tree_root, path, + BTRFS_FREE_SPACE_TREE_OBJECTID, + "free space"); +} + +static int btrfs_read_roots(struct btrfs_fs_info *fs_info) +{ + struct btrfs_root *tree_root = fs_info->tree_root; + struct btrfs_root *root; + struct btrfs_key location; + int ret; + + ASSERT(fs_info->tree_root); + + ret = load_global_roots(tree_root); + if (ret) + return ret; + + location.type = BTRFS_ROOT_ITEM_KEY; + location.offset = 0; + + if (btrfs_fs_compat_ro(fs_info, BLOCK_GROUP_TREE)) { + location.objectid = BTRFS_BLOCK_GROUP_TREE_OBJECTID; + root = btrfs_read_tree_root(tree_root, &location); + if (IS_ERR(root)) { + if (!btrfs_test_opt(fs_info, IGNOREBADROOTS)) { + ret = PTR_ERR(root); + goto out; + } + } else { + set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state); + fs_info->block_group_root = root; + } + } + + location.objectid = BTRFS_DEV_TREE_OBJECTID; + root = btrfs_read_tree_root(tree_root, &location); + if (IS_ERR(root)) { + if (!btrfs_test_opt(fs_info, IGNOREBADROOTS)) { + ret = PTR_ERR(root); + goto out; + } + } else { + set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state); + fs_info->dev_root = root; + } + /* Initialize fs_info for all devices in any case */ + ret = btrfs_init_devices_late(fs_info); + if (ret) + goto out; + + if (btrfs_fs_incompat(fs_info, REMAP_TREE)) { + /* The remap_root has already been loaded in load_important_roots(). */ + root = fs_info->remap_root; + + set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state); + + root->root_key.objectid = BTRFS_REMAP_TREE_OBJECTID; + root->root_key.type = BTRFS_ROOT_ITEM_KEY; + root->root_key.offset = 0; + + /* Check that data reloc tree doesn't also exist. */ + location.objectid = BTRFS_DATA_RELOC_TREE_OBJECTID; + root = btrfs_read_tree_root(fs_info->tree_root, &location); + if (!IS_ERR(root)) { + btrfs_err(fs_info, "data reloc tree exists when remap-tree enabled"); + btrfs_put_root(root); + return -EIO; + } else if (PTR_ERR(root) != -ENOENT) { + btrfs_warn(fs_info, "error %ld when checking for data reloc tree", + PTR_ERR(root)); + } + } else { + /* + * This tree can share blocks with some other fs tree during + * relocation and we need a proper setup by btrfs_get_fs_root(). + */ + root = btrfs_get_fs_root(tree_root->fs_info, + BTRFS_DATA_RELOC_TREE_OBJECTID, true); + if (IS_ERR(root)) { + if (!btrfs_test_opt(fs_info, IGNOREBADROOTS)) { + location.objectid = BTRFS_DATA_RELOC_TREE_OBJECTID; + ret = PTR_ERR(root); + goto out; + } + } else { + set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state); + fs_info->data_reloc_root = root; + } + } + + location.objectid = BTRFS_QUOTA_TREE_OBJECTID; + root = btrfs_read_tree_root(tree_root, &location); + if (!IS_ERR(root)) { + set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state); + fs_info->quota_root = root; + } + + location.objectid = BTRFS_UUID_TREE_OBJECTID; + root = btrfs_read_tree_root(tree_root, &location); + if (IS_ERR(root)) { + if (!btrfs_test_opt(fs_info, IGNOREBADROOTS)) { + ret = PTR_ERR(root); + if (ret != -ENOENT) + goto out; + } + } else { + set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state); + fs_info->uuid_root = root; + } + + if (btrfs_fs_incompat(fs_info, RAID_STRIPE_TREE)) { + location.objectid = BTRFS_RAID_STRIPE_TREE_OBJECTID; + root = btrfs_read_tree_root(tree_root, &location); + if (IS_ERR(root)) { + if (!btrfs_test_opt(fs_info, IGNOREBADROOTS)) { + ret = PTR_ERR(root); + goto out; + } + } else { + set_bit(BTRFS_ROOT_TRACK_DIRTY, &root->state); + fs_info->stripe_root = root; + } + } + + return 0; +out: + btrfs_warn(fs_info, "failed to read root (objectid=%llu): %pe", + location.objectid, ERR_PTR(ret)); + return ret; +} + +static int validate_sys_chunk_array(const struct btrfs_fs_info *fs_info, + const struct btrfs_super_block *sb) +{ + unsigned int cur = 0; /* Offset inside the sys chunk array */ + /* + * At sb read time, fs_info is not fully initialized. Thus we have + * to use super block sectorsize, which should have been validated. + */ + const u32 sectorsize = btrfs_super_sectorsize(sb); + u32 sys_array_size = btrfs_super_sys_array_size(sb); + + if (unlikely(sys_array_size > BTRFS_SYSTEM_CHUNK_ARRAY_SIZE)) { + btrfs_err(fs_info, "system chunk array too big %u > %u", + sys_array_size, BTRFS_SYSTEM_CHUNK_ARRAY_SIZE); + return -EUCLEAN; + } + + /* It must hold at least one key and one chunk. */ + if (unlikely(sys_array_size < sizeof(struct btrfs_disk_key) + + sizeof(struct btrfs_chunk))) { + btrfs_err(fs_info, "system chunk array too small %u < %zu", + sys_array_size, + sizeof(struct btrfs_disk_key) + sizeof(struct btrfs_chunk)); + return -EUCLEAN; + } + + while (cur < sys_array_size) { + struct btrfs_disk_key *disk_key; + struct btrfs_chunk *chunk; + struct btrfs_key key; + u64 type; + u16 num_stripes; + u32 len; + int ret; + + disk_key = (struct btrfs_disk_key *)(sb->sys_chunk_array + cur); + len = sizeof(*disk_key); + + if (unlikely(cur + len > sys_array_size)) + goto short_read; + cur += len; + + btrfs_disk_key_to_cpu(&key, disk_key); + if (unlikely(key.type != BTRFS_CHUNK_ITEM_KEY)) { + btrfs_err(fs_info, + "unexpected item type %u in sys_array at offset %u", + key.type, cur); + return -EUCLEAN; + } + chunk = (struct btrfs_chunk *)(sb->sys_chunk_array + cur); + num_stripes = btrfs_stack_chunk_num_stripes(chunk); + if (unlikely(cur + btrfs_chunk_item_size(num_stripes) > sys_array_size)) + goto short_read; + type = btrfs_stack_chunk_type(chunk); + if (unlikely(!(type & BTRFS_BLOCK_GROUP_SYSTEM))) { + btrfs_err(fs_info, + "invalid chunk type %llu in sys_array at offset %u", + type, cur); + return -EUCLEAN; + } + ret = btrfs_check_chunk_valid(fs_info, NULL, chunk, key.offset, + sectorsize); + if (ret < 0) + return ret; + cur += btrfs_chunk_item_size(num_stripes); + } + return 0; +short_read: + btrfs_err(fs_info, + "super block sys chunk array short read, cur=%u sys_array_size=%u", + cur, sys_array_size); + return -EUCLEAN; +} + +/* + * Real super block validation + * NOTE: super csum type and incompat features will not be checked here. + * + * @sb: super block to check + * @mirror_num: the super block number to check its bytenr: + * 0 the primary (1st) sb + * 1, 2 2nd and 3rd backup copy + * -1 skip bytenr check + */ +int btrfs_validate_super(const struct btrfs_fs_info *fs_info, + const struct btrfs_super_block *sb, int mirror_num) +{ + const u32 nodesize = btrfs_super_nodesize(sb); + const u32 sectorsize = btrfs_super_sectorsize(sb); + int ret = 0; + const bool ignore_flags = btrfs_test_opt(fs_info, IGNORESUPERFLAGS); + + if (unlikely(btrfs_super_magic(sb) != BTRFS_MAGIC)) { + btrfs_err(fs_info, "no valid FS found"); + ret = -EINVAL; + } + if (unlikely(btrfs_super_flags(sb) & ~BTRFS_SUPER_FLAG_SUPP)) { + if (!ignore_flags) { + btrfs_err(fs_info, + "unrecognized or unsupported super flag 0x%llx", + btrfs_super_flags(sb) & ~BTRFS_SUPER_FLAG_SUPP); + ret = -EINVAL; + } else { + btrfs_info(fs_info, + "unrecognized or unsupported super flags: 0x%llx, ignored", + btrfs_super_flags(sb) & ~BTRFS_SUPER_FLAG_SUPP); + } + } + if (unlikely(btrfs_super_root_level(sb) >= BTRFS_MAX_LEVEL)) { + btrfs_err(fs_info, "tree_root level too big: %d >= %d", + btrfs_super_root_level(sb), BTRFS_MAX_LEVEL); + ret = -EINVAL; + } + if (unlikely(btrfs_super_chunk_root_level(sb) >= BTRFS_MAX_LEVEL)) { + btrfs_err(fs_info, "chunk_root level too big: %d >= %d", + btrfs_super_chunk_root_level(sb), BTRFS_MAX_LEVEL); + ret = -EINVAL; + } + if (unlikely(btrfs_super_log_root_level(sb) >= BTRFS_MAX_LEVEL)) { + btrfs_err(fs_info, "log_root level too big: %d >= %d", + btrfs_super_log_root_level(sb), BTRFS_MAX_LEVEL); + ret = -EINVAL; + } + + /* + * Check sectorsize and nodesize first, other check will need it. + * Check all possible sectorsize(4K, 8K, 16K, 32K, 64K) here. + */ + if (unlikely(!is_power_of_2(sectorsize) || sectorsize < BTRFS_MIN_BLOCKSIZE || + sectorsize > BTRFS_MAX_METADATA_BLOCKSIZE)) { + btrfs_err(fs_info, "invalid sectorsize %u", sectorsize); + ret = -EINVAL; + } + + if (unlikely(!btrfs_supported_blocksize(sectorsize))) { + btrfs_err(fs_info, + "sectorsize %u not yet supported for page size %lu", + sectorsize, PAGE_SIZE); + ret = -EINVAL; + } + + if (unlikely(!is_power_of_2(nodesize) || nodesize < sectorsize || + nodesize > BTRFS_MAX_METADATA_BLOCKSIZE)) { + btrfs_err(fs_info, "invalid nodesize %u", nodesize); + ret = -EINVAL; + } + if (unlikely(nodesize != le32_to_cpu(sb->__unused_leafsize))) { + btrfs_err(fs_info, "invalid leafsize %u, should be %u", + le32_to_cpu(sb->__unused_leafsize), nodesize); + ret = -EINVAL; + } + + /* Root alignment check */ + if (unlikely(!IS_ALIGNED(btrfs_super_root(sb), sectorsize))) { + btrfs_err(fs_info, "tree_root block unaligned: %llu", + btrfs_super_root(sb)); + ret = -EINVAL; + } + if (unlikely(!IS_ALIGNED(btrfs_super_chunk_root(sb), sectorsize))) { + btrfs_err(fs_info, "chunk_root block unaligned: %llu", + btrfs_super_chunk_root(sb)); + ret = -EINVAL; + } + if (unlikely(!IS_ALIGNED(btrfs_super_log_root(sb), sectorsize))) { + btrfs_err(fs_info, "log_root block unaligned: %llu", + btrfs_super_log_root(sb)); + ret = -EINVAL; + } + + if (unlikely(!fs_info->fs_devices->temp_fsid && + memcmp(fs_info->fs_devices->fsid, sb->fsid, BTRFS_FSID_SIZE) != 0)) { + btrfs_err(fs_info, + "superblock fsid doesn't match fsid of fs_devices: %pU != %pU", + sb->fsid, fs_info->fs_devices->fsid); + ret = -EINVAL; + } + + if (unlikely(memcmp(fs_info->fs_devices->metadata_uuid, btrfs_sb_fsid_ptr(sb), + BTRFS_FSID_SIZE) != 0)) { + btrfs_err(fs_info, +"superblock metadata_uuid doesn't match metadata uuid of fs_devices: %pU != %pU", + btrfs_sb_fsid_ptr(sb), fs_info->fs_devices->metadata_uuid); + ret = -EINVAL; + } + + if (unlikely(memcmp(fs_info->fs_devices->metadata_uuid, sb->dev_item.fsid, + BTRFS_FSID_SIZE) != 0)) { + btrfs_err(fs_info, + "dev_item UUID does not match metadata fsid: %pU != %pU", + fs_info->fs_devices->metadata_uuid, sb->dev_item.fsid); + ret = -EINVAL; + } + + /* + * Artificial requirement for block-group-tree to force newer features + * (free-space-tree, no-holes) so the test matrix is smaller. + */ + if (unlikely(btrfs_fs_compat_ro(fs_info, BLOCK_GROUP_TREE) && + (!btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE_VALID) || + !btrfs_fs_incompat(fs_info, NO_HOLES)))) { + btrfs_err(fs_info, + "block-group-tree feature requires free-space-tree and no-holes"); + ret = -EINVAL; + } + + if (btrfs_fs_incompat(fs_info, REMAP_TREE)) { + /* + * Reduce test matrix for remap tree by requiring block-group-tree + * and no-holes. Free-space-tree is a hard requirement. + */ + if (unlikely(!btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE_VALID) || + !btrfs_fs_incompat(fs_info, NO_HOLES) || + !btrfs_fs_compat_ro(fs_info, BLOCK_GROUP_TREE))) { + btrfs_err(fs_info, +"remap-tree feature requires free-space-tree, no-holes, and block-group-tree"); + ret = -EINVAL; + } + + if (unlikely(btrfs_fs_incompat(fs_info, MIXED_GROUPS))) { + btrfs_err(fs_info, "remap-tree not supported with mixed-bg"); + ret = -EINVAL; + } + + if (unlikely(btrfs_fs_incompat(fs_info, ZONED))) { + btrfs_err(fs_info, "remap-tree not supported with zoned devices"); + ret = -EINVAL; + } + + if (unlikely(sectorsize > PAGE_SIZE)) { + btrfs_err(fs_info, "remap-tree not supported when block size > page size"); + ret = -EINVAL; + } + } + + /* + * Hint to catch really bogus numbers, bitflips or so, more exact checks are + * done later + */ + if (unlikely(btrfs_super_bytes_used(sb) < 6 * btrfs_super_nodesize(sb))) { + btrfs_err(fs_info, "bytes_used is too small %llu", + btrfs_super_bytes_used(sb)); + ret = -EINVAL; + } + if (unlikely(!is_power_of_2(btrfs_super_stripesize(sb)))) { + btrfs_err(fs_info, "invalid stripesize %u", + btrfs_super_stripesize(sb)); + ret = -EINVAL; + } + if (unlikely(btrfs_super_num_devices(sb) > (1UL << 31))) + btrfs_warn(fs_info, "suspicious number of devices: %llu", + btrfs_super_num_devices(sb)); + if (unlikely(btrfs_super_num_devices(sb) == 0)) { + btrfs_err(fs_info, "number of devices is 0"); + ret = -EINVAL; + } + + if (unlikely(mirror_num >= 0 && + btrfs_super_bytenr(sb) != btrfs_sb_offset(mirror_num))) { + btrfs_err(fs_info, "super offset mismatch %llu != %llu", + btrfs_super_bytenr(sb), btrfs_sb_offset(mirror_num)); + ret = -EINVAL; + } + + if (unlikely(ret)) + return ret; + + ret = validate_sys_chunk_array(fs_info, sb); + + /* + * The generation is a global counter, we'll trust it more than the others + * but it's still possible that it's the one that's wrong. + */ + if (unlikely(btrfs_super_generation(sb) < btrfs_super_chunk_root_generation(sb))) + btrfs_warn(fs_info, + "suspicious: generation < chunk_root_generation: %llu < %llu", + btrfs_super_generation(sb), + btrfs_super_chunk_root_generation(sb)); + if (unlikely(btrfs_super_generation(sb) < btrfs_super_cache_generation(sb) && + btrfs_super_cache_generation(sb) != (u64)-1)) + btrfs_warn(fs_info, + "suspicious: generation < cache_generation: %llu < %llu", + btrfs_super_generation(sb), + btrfs_super_cache_generation(sb)); + + return ret; +} + +/* + * Validation of super block at mount time. + * Some checks already done early at mount time, like csum type and incompat + * flags will be skipped. + */ +static int btrfs_validate_mount_super(struct btrfs_fs_info *fs_info) +{ + return btrfs_validate_super(fs_info, fs_info->super_copy, 0); +} + +/* + * Validation of super block at write time. + * Some checks like bytenr check will be skipped as their values will be + * overwritten soon. + * Extra checks like csum type and incompat flags will be done here. + */ +static int btrfs_validate_write_super(struct btrfs_fs_info *fs_info, + struct btrfs_super_block *sb) +{ + int ret; + + ret = btrfs_validate_super(fs_info, sb, -1); + if (unlikely(ret < 0)) + goto out; + if (unlikely(!btrfs_supported_super_csum(btrfs_super_csum_type(sb)))) { + ret = -EUCLEAN; + btrfs_err(fs_info, "invalid csum type, has %u want %u", + btrfs_super_csum_type(sb), BTRFS_CSUM_TYPE_CRC32); + goto out; + } + if (unlikely(btrfs_super_incompat_flags(sb) & ~BTRFS_FEATURE_INCOMPAT_SUPP)) { + ret = -EUCLEAN; + btrfs_err(fs_info, + "invalid incompat flags, has 0x%llx valid mask 0x%llx", + btrfs_super_incompat_flags(sb), + (unsigned long long)BTRFS_FEATURE_INCOMPAT_SUPP); + goto out; + } +out: + if (unlikely(ret < 0)) + btrfs_err(fs_info, + "super block corruption detected before writing it to disk"); + return ret; +} + +static int load_super_root(struct btrfs_root *root, u64 bytenr, u64 gen, int level) +{ + struct btrfs_tree_parent_check check = { + .level = level, + .transid = gen, + .owner_root = btrfs_root_id(root) + }; + int ret = 0; + + root->node = read_tree_block(root->fs_info, bytenr, &check); + if (IS_ERR(root->node)) { + ret = PTR_ERR(root->node); + root->node = NULL; + return ret; + } + + btrfs_set_root_node(&root->root_item, root->node); + root->commit_root = btrfs_root_node(root); + btrfs_set_root_refs(&root->root_item, 1); + return ret; +} + +static int load_important_roots(struct btrfs_fs_info *fs_info) +{ + struct btrfs_super_block *sb = fs_info->super_copy; + u64 gen, bytenr; + int level, ret; + + bytenr = btrfs_super_root(sb); + gen = btrfs_super_generation(sb); + level = btrfs_super_root_level(sb); + ret = load_super_root(fs_info->tree_root, bytenr, gen, level); + if (ret) { + btrfs_warn(fs_info, "couldn't read tree root"); + return ret; + } + + if (btrfs_fs_incompat(fs_info, REMAP_TREE)) { + bytenr = btrfs_super_remap_root(sb); + gen = btrfs_super_remap_root_generation(sb); + level = btrfs_super_remap_root_level(sb); + ret = load_super_root(fs_info->remap_root, bytenr, gen, level); + if (ret) { + btrfs_warn(fs_info, "couldn't read remap root"); + return ret; + } + } + + return 0; +} + +static int __cold init_tree_roots(struct btrfs_fs_info *fs_info) +{ + int backup_index = find_newest_super_backup(fs_info); + struct btrfs_super_block *sb = fs_info->super_copy; + struct btrfs_root *tree_root = fs_info->tree_root; + bool handle_error = false; + int ret = 0; + int i; + + for (i = 0; i < BTRFS_NUM_BACKUP_ROOTS; i++) { + if (handle_error) { + if (!IS_ERR(tree_root->node)) + free_extent_buffer(tree_root->node); + tree_root->node = NULL; + + if (!btrfs_test_opt(fs_info, USEBACKUPROOT)) + break; + + free_root_pointers(fs_info, 0); + + /* + * Don't use the log in recovery mode, it won't be + * valid + */ + btrfs_set_super_log_root(sb, 0); + + btrfs_warn(fs_info, "try to load backup roots slot %d", i); + ret = read_backup_root(fs_info, i); + backup_index = ret; + if (ret < 0) + return ret; + } + + ret = load_important_roots(fs_info); + if (ret) { + handle_error = true; + continue; + } + + /* + * No need to hold btrfs_root::objectid_mutex since the fs + * hasn't been fully initialised and we are the only user + */ + ret = btrfs_init_root_free_objectid(tree_root); + if (ret < 0) { + handle_error = true; + continue; + } + + ASSERT(tree_root->free_objectid <= BTRFS_LAST_FREE_OBJECTID); + + ret = btrfs_read_roots(fs_info); + if (ret < 0) { + handle_error = true; + continue; + } + + /* All successful */ + fs_info->generation = btrfs_header_generation(tree_root->node); + btrfs_set_last_trans_committed(fs_info, fs_info->generation); + fs_info->last_reloc_trans = 0; + + /* Always begin writing backup roots after the one being used */ + if (backup_index < 0) { + fs_info->backup_root_index = 0; + } else { + fs_info->backup_root_index = backup_index + 1; + fs_info->backup_root_index %= BTRFS_NUM_BACKUP_ROOTS; + } + break; + } + + return ret; +} + +/* + * Lockdep gets confused between our buffer_tree which requires IRQ locking because + * we modify marks in the IRQ context, and our delayed inode xarray which doesn't + * have these requirements. Use a class key so lockdep doesn't get them mixed up. + */ +static struct lock_class_key buffer_xa_class; + +void btrfs_init_fs_info(struct btrfs_fs_info *fs_info) +{ + INIT_RADIX_TREE(&fs_info->fs_roots_radix, GFP_ATOMIC); + + /* Use the same flags as mapping->i_pages. */ + xa_init_flags(&fs_info->buffer_tree, XA_FLAGS_LOCK_IRQ | XA_FLAGS_ACCOUNT); + lockdep_set_class(&fs_info->buffer_tree.xa_lock, &buffer_xa_class); + + INIT_LIST_HEAD(&fs_info->trans_list); + INIT_LIST_HEAD(&fs_info->dead_roots); + INIT_LIST_HEAD(&fs_info->delayed_iputs); + INIT_LIST_HEAD(&fs_info->delalloc_roots); + INIT_LIST_HEAD(&fs_info->caching_block_groups); + spin_lock_init(&fs_info->delalloc_root_lock); + spin_lock_init(&fs_info->trans_lock); + spin_lock_init(&fs_info->fs_roots_radix_lock); + spin_lock_init(&fs_info->delayed_iput_lock); + spin_lock_init(&fs_info->defrag_inodes_lock); + spin_lock_init(&fs_info->super_lock); + spin_lock_init(&fs_info->unused_bgs_lock); + spin_lock_init(&fs_info->treelog_bg_lock); + spin_lock_init(&fs_info->zone_active_bgs_lock); + spin_lock_init(&fs_info->relocation_bg_lock); + rwlock_init(&fs_info->tree_mod_log_lock); + rwlock_init(&fs_info->global_root_lock); + mutex_init(&fs_info->unused_bg_unpin_mutex); + mutex_init(&fs_info->reclaim_bgs_lock); + mutex_init(&fs_info->reloc_mutex); + spin_lock_init(&fs_info->reloc_ctl_lock); + mutex_init(&fs_info->delalloc_root_mutex); + mutex_init(&fs_info->zoned_meta_io_lock); + mutex_init(&fs_info->zoned_data_reloc_io_lock); + seqlock_init(&fs_info->profiles_lock); + + btrfs_lockdep_init_map(fs_info, btrfs_trans_num_writers); + btrfs_lockdep_init_map(fs_info, btrfs_trans_num_extwriters); + btrfs_lockdep_init_map(fs_info, btrfs_trans_pending_ordered); + btrfs_lockdep_init_map(fs_info, btrfs_ordered_extent); + btrfs_state_lockdep_init_map(fs_info, btrfs_trans_commit_prep, + BTRFS_LOCKDEP_TRANS_COMMIT_PREP); + btrfs_state_lockdep_init_map(fs_info, btrfs_trans_unblocked, + BTRFS_LOCKDEP_TRANS_UNBLOCKED); + btrfs_state_lockdep_init_map(fs_info, btrfs_trans_super_committed, + BTRFS_LOCKDEP_TRANS_SUPER_COMMITTED); + btrfs_state_lockdep_init_map(fs_info, btrfs_trans_completed, + BTRFS_LOCKDEP_TRANS_COMPLETED); + + INIT_LIST_HEAD(&fs_info->dirty_cowonly_roots); + INIT_LIST_HEAD(&fs_info->space_info); + INIT_LIST_HEAD(&fs_info->tree_mod_seq_list); + INIT_LIST_HEAD(&fs_info->unused_bgs); + INIT_LIST_HEAD(&fs_info->reclaim_bgs); + INIT_LIST_HEAD(&fs_info->fully_remapped_bgs); + INIT_LIST_HEAD(&fs_info->zone_active_bgs); +#ifdef CONFIG_BTRFS_DEBUG + INIT_LIST_HEAD(&fs_info->allocated_roots); + INIT_LIST_HEAD(&fs_info->allocated_ebs); + spin_lock_init(&fs_info->eb_leak_lock); +#endif + fs_info->mapping_tree = RB_ROOT_CACHED; + rwlock_init(&fs_info->mapping_tree_lock); + btrfs_init_block_rsv(&fs_info->global_block_rsv, + BTRFS_BLOCK_RSV_GLOBAL); + btrfs_init_block_rsv(&fs_info->trans_block_rsv, BTRFS_BLOCK_RSV_TRANS); + btrfs_init_block_rsv(&fs_info->chunk_block_rsv, BTRFS_BLOCK_RSV_CHUNK); + btrfs_init_block_rsv(&fs_info->remap_block_rsv, BTRFS_BLOCK_RSV_REMAP); + btrfs_init_block_rsv(&fs_info->treelog_rsv, BTRFS_BLOCK_RSV_TREELOG); + btrfs_init_block_rsv(&fs_info->empty_block_rsv, BTRFS_BLOCK_RSV_EMPTY); + btrfs_init_block_rsv(&fs_info->delayed_block_rsv, + BTRFS_BLOCK_RSV_DELOPS); + btrfs_init_block_rsv(&fs_info->delayed_refs_rsv, + BTRFS_BLOCK_RSV_DELREFS); + + atomic_set(&fs_info->async_delalloc_pages, 0); + atomic_set(&fs_info->defrag_running, 0); + atomic_set(&fs_info->nr_delayed_iputs, 0); + atomic64_set(&fs_info->tree_mod_seq, 0); + fs_info->global_root_tree = RB_ROOT; + fs_info->max_inline = BTRFS_DEFAULT_MAX_INLINE; + fs_info->metadata_ratio = 0; + fs_info->defrag_inodes = RB_ROOT; + atomic64_set(&fs_info->free_chunk_space, 0); + fs_info->tree_mod_log = RB_ROOT; + fs_info->commit_interval = BTRFS_DEFAULT_COMMIT_INTERVAL; + btrfs_init_ref_verify(fs_info); + + fs_info->thread_pool_size = min_t(unsigned long, + num_online_cpus() + 2, 8); + + INIT_LIST_HEAD(&fs_info->ordered_roots); + spin_lock_init(&fs_info->ordered_root_lock); + + btrfs_init_scrub(fs_info); + btrfs_init_balance(fs_info); + btrfs_init_async_reclaim_work(fs_info); + btrfs_init_extent_map_shrinker_work(fs_info); + + rwlock_init(&fs_info->block_group_cache_lock); + fs_info->block_group_cache_tree = RB_ROOT_CACHED; + + btrfs_extent_io_tree_init(fs_info, &fs_info->excluded_extents, + IO_TREE_FS_EXCLUDED_EXTENTS); + + mutex_init(&fs_info->ordered_operations_mutex); + mutex_init(&fs_info->tree_log_mutex); + mutex_init(&fs_info->chunk_mutex); + mutex_init(&fs_info->transaction_kthread_mutex); + mutex_init(&fs_info->cleaner_mutex); + mutex_init(&fs_info->remap_mutex); + mutex_init(&fs_info->ro_block_group_mutex); + init_rwsem(&fs_info->commit_root_sem); + init_rwsem(&fs_info->cleanup_work_sem); + init_rwsem(&fs_info->subvol_sem); + sema_init(&fs_info->uuid_tree_rescan_sem, 1); + + btrfs_init_dev_replace_locks(fs_info); + btrfs_init_qgroup(fs_info); + btrfs_discard_init(fs_info); + + btrfs_init_free_cluster(&fs_info->meta_alloc_cluster); + btrfs_init_free_cluster(&fs_info->data_alloc_cluster); + + init_waitqueue_head(&fs_info->transaction_throttle); + init_waitqueue_head(&fs_info->transaction_wait); + init_waitqueue_head(&fs_info->transaction_blocked_wait); + init_waitqueue_head(&fs_info->async_submit_wait); + init_waitqueue_head(&fs_info->delayed_iputs_wait); + + /* Usable values until the real ones are cached from the superblock */ + fs_info->nodesize = 4096; + fs_info->sectorsize = 4096; + fs_info->sectorsize_bits = ilog2(4096); + + /* Default compress algorithm when user does -o compress */ + fs_info->compress_type = BTRFS_COMPRESS_ZLIB; + + fs_info->max_extent_size = BTRFS_MAX_EXTENT_SIZE; + + spin_lock_init(&fs_info->swapfile_pins_lock); + fs_info->swapfile_pins = RB_ROOT; + + fs_info->bg_reclaim_threshold = BTRFS_DEFAULT_RECLAIM_THRESH; + INIT_WORK(&fs_info->reclaim_bgs_work, btrfs_reclaim_bgs_work); +} + +static int init_mount_fs_info(struct btrfs_fs_info *fs_info, struct super_block *sb) +{ + int ret; + + fs_info->sb = sb; + /* Temporary fixed values for block size until we read the superblock. */ + sb->s_blocksize = BTRFS_BDEV_BLOCKSIZE; + sb->s_blocksize_bits = blksize_bits(BTRFS_BDEV_BLOCKSIZE); + + ret = percpu_counter_init(&fs_info->ordered_bytes, 0, GFP_KERNEL); + if (ret) + return ret; + + ret = percpu_counter_init(&fs_info->evictable_extent_maps, 0, GFP_KERNEL); + if (ret) + return ret; + + ret = percpu_counter_init(&fs_info->dirty_metadata_bytes, 0, GFP_KERNEL); + if (ret) + return ret; + + ret = percpu_counter_init(&fs_info->stats_read_blocks, 0, GFP_KERNEL); + if (ret) + return ret; + + fs_info->dirty_metadata_batch = PAGE_SIZE * + (1 + ilog2(nr_cpu_ids)); + + ret = percpu_counter_init(&fs_info->delalloc_bytes, 0, GFP_KERNEL); + if (ret) + return ret; + + ret = percpu_counter_init(&fs_info->dev_replace.bio_counter, 0, + GFP_KERNEL); + if (ret) + return ret; + + btrfs_init_delayed_root(&fs_info->delayed_root); + + if (sb_rdonly(sb)) + set_bit(BTRFS_FS_STATE_RO, &fs_info->fs_state); + if (btrfs_test_opt(fs_info, IGNOREMETACSUMS)) + set_bit(BTRFS_FS_STATE_SKIP_META_CSUMS, &fs_info->fs_state); + + return btrfs_alloc_stripe_hash_table(fs_info); +} + +static int btrfs_uuid_rescan_kthread(void *data) +{ + struct btrfs_fs_info *fs_info = data; + int ret; + + /* + * 1st step is to iterate through the existing UUID tree and + * to delete all entries that contain outdated data. + * 2nd step is to add all missing entries to the UUID tree. + */ + ret = btrfs_uuid_tree_iterate(fs_info); + if (ret < 0) { + if (ret != -EINTR) + btrfs_warn(fs_info, "iterating uuid_tree failed %pe", + ERR_PTR(ret)); + up(&fs_info->uuid_tree_rescan_sem); + return ret; + } + return btrfs_uuid_scan_kthread(data); +} + +static int btrfs_check_uuid_tree(struct btrfs_fs_info *fs_info) +{ + struct task_struct *task; + + down(&fs_info->uuid_tree_rescan_sem); + task = kthread_run(btrfs_uuid_rescan_kthread, fs_info, "btrfs-uuid"); + if (IS_ERR(task)) { + /* fs_info->update_uuid_tree_gen remains 0 in all error case */ + up(&fs_info->uuid_tree_rescan_sem); + return PTR_ERR(task); + } + + return 0; +} + +static int btrfs_cleanup_fs_roots(struct btrfs_fs_info *fs_info) +{ + u64 root_objectid = 0; + struct btrfs_root *gang[8]; + int ret = 0; + + while (1) { + unsigned int found; + + spin_lock(&fs_info->fs_roots_radix_lock); + found = radix_tree_gang_lookup(&fs_info->fs_roots_radix, + (void **)gang, root_objectid, + ARRAY_SIZE(gang)); + if (!found) { + spin_unlock(&fs_info->fs_roots_radix_lock); + break; + } + root_objectid = btrfs_root_id(gang[found - 1]) + 1; + + for (int i = 0; i < found; i++) { + /* Avoid to grab roots in dead_roots. */ + if (btrfs_root_refs(&gang[i]->root_item) == 0) { + gang[i] = NULL; + continue; + } + /* Grab all the search result for later use. */ + gang[i] = btrfs_grab_root(gang[i]); + } + spin_unlock(&fs_info->fs_roots_radix_lock); + + for (int i = 0; i < found; i++) { + if (!gang[i]) + continue; + root_objectid = btrfs_root_id(gang[i]); + /* + * Continue to release the remaining roots after the first + * error without cleanup and preserve the first error + * for the return. + */ + if (!ret) + ret = btrfs_orphan_cleanup(gang[i]); + btrfs_put_root(gang[i]); + } + if (ret) + break; + + root_objectid++; + } + return ret; +} + +/* + * Mounting logic specific to read-write file systems. Shared by open_ctree + * and btrfs_remount when remounting from read-only to read-write. + */ +int btrfs_start_pre_rw_mount(struct btrfs_fs_info *fs_info) +{ + int ret; + const bool cache_opt = btrfs_test_opt(fs_info, SPACE_CACHE); + bool rebuild_free_space_tree = false; + + if (btrfs_test_opt(fs_info, CLEAR_CACHE) && + btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) { + if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) + btrfs_warn(fs_info, + "'clear_cache' option is ignored with extent tree v2"); + else if (btrfs_fs_incompat(fs_info, REMAP_TREE)) + btrfs_warn(fs_info, "'clear_cache' option is ignored with remap tree"); + else + rebuild_free_space_tree = true; + } else if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE) && + !btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE_VALID)) { + btrfs_warn(fs_info, "free space tree is invalid"); + rebuild_free_space_tree = true; + } + + if (rebuild_free_space_tree) { + btrfs_info(fs_info, "rebuilding free space tree"); + ret = btrfs_rebuild_free_space_tree(fs_info); + if (ret) { + btrfs_warn(fs_info, + "failed to rebuild free space tree: %pe", ERR_PTR(ret)); + return ret; + } + } + + if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE) && + !btrfs_test_opt(fs_info, FREE_SPACE_TREE)) { + btrfs_info(fs_info, "disabling free space tree"); + ret = btrfs_delete_free_space_tree(fs_info); + if (ret) { + btrfs_warn(fs_info, + "failed to disable free space tree: %pe", ERR_PTR(ret)); + return ret; + } + } + + /* + * Before btrfs-progs v6.16.1 mkfs.btrfs can leave free space entries + * for deleted temporary chunks. Delete them if they exist. + */ + ret = btrfs_delete_orphan_free_space_entries(fs_info); + if (ret < 0) { + btrfs_err(fs_info, "failed to delete orphan free space tree entries: %pe", + ERR_PTR(ret)); + return ret; + } + /* + * btrfs_find_orphan_roots() is responsible for finding all the dead + * roots (with 0 refs), flag them with BTRFS_ROOT_DEAD_TREE and load + * them into the fs_info->fs_roots_radix tree. This must be done before + * calling btrfs_orphan_cleanup() on the tree root. If we don't do it + * first, then btrfs_orphan_cleanup() will delete a dead root's orphan + * item before the root's tree is deleted - this means that if we unmount + * or crash before the deletion completes, on the next mount we will not + * delete what remains of the tree because the orphan item does not + * exists anymore, which is what tells us we have a pending deletion. + */ + ret = btrfs_find_orphan_roots(fs_info); + if (ret) + return ret; + + ret = btrfs_cleanup_fs_roots(fs_info); + if (ret) + return ret; + + down_read(&fs_info->cleanup_work_sem); + if ((ret = btrfs_orphan_cleanup(fs_info->fs_root)) || + (ret = btrfs_orphan_cleanup(fs_info->tree_root))) { + up_read(&fs_info->cleanup_work_sem); + return ret; + } + up_read(&fs_info->cleanup_work_sem); + + mutex_lock(&fs_info->cleaner_mutex); + ret = btrfs_recover_relocation(fs_info); + mutex_unlock(&fs_info->cleaner_mutex); + if (ret < 0) { + btrfs_warn(fs_info, "failed to recover relocation: %pe", ERR_PTR(ret)); + return ret; + } + + if (btrfs_test_opt(fs_info, FREE_SPACE_TREE) && + !btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) { + btrfs_info(fs_info, "creating free space tree"); + ret = btrfs_create_free_space_tree(fs_info); + if (ret) { + btrfs_warn(fs_info, + "failed to create free space tree: %pe", ERR_PTR(ret)); + return ret; + } + } + + if (cache_opt != btrfs_free_space_cache_v1_active(fs_info)) { + ret = btrfs_set_free_space_cache_v1_active(fs_info, cache_opt); + if (ret) + return ret; + } + + ret = btrfs_resume_balance_async(fs_info); + if (ret) + return ret; + + ret = btrfs_resume_dev_replace_async(fs_info); + if (ret) { + btrfs_warn(fs_info, "failed to resume dev_replace"); + return ret; + } + + btrfs_qgroup_rescan_resume(fs_info); + + if (!fs_info->uuid_root) { + btrfs_info(fs_info, "creating UUID tree"); + ret = btrfs_create_uuid_tree(fs_info); + if (ret) { + btrfs_warn(fs_info, + "failed to create the UUID tree %pe", ERR_PTR(ret)); + return ret; + } + } + + return 0; +} + +/* + * Do various sanity and dependency checks of different features. + * + * @is_rw_mount: If the mount is read-write. + * + * This is the place for less strict checks (like for subpage or artificial + * feature dependencies). + * + * For strict checks or possible corruption detection, see + * btrfs_validate_super(). + * + * This should be called after btrfs_parse_options(), as some mount options + * (space cache related) can modify on-disk format like free space tree and + * screw up certain feature dependencies. + */ +int btrfs_check_features(struct btrfs_fs_info *fs_info, bool is_rw_mount) +{ + struct btrfs_super_block *disk_super = fs_info->super_copy; + u64 incompat = btrfs_super_incompat_flags(disk_super); + const u64 compat_ro = btrfs_super_compat_ro_flags(disk_super); + const u64 compat_ro_unsupp = (compat_ro & ~BTRFS_FEATURE_COMPAT_RO_SUPP); + + if (incompat & ~BTRFS_FEATURE_INCOMPAT_SUPP) { + btrfs_err(fs_info, + "cannot mount because of unknown incompat features (0x%llx)", + incompat & ~BTRFS_FEATURE_INCOMPAT_SUPP); + return -EINVAL; + } + + /* Runtime limitation for mixed block groups. */ + if ((incompat & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS) && + (fs_info->sectorsize != fs_info->nodesize)) { + btrfs_err(fs_info, +"unequal nodesize/sectorsize (%u != %u) are not allowed for mixed block groups", + fs_info->nodesize, fs_info->sectorsize); + return -EINVAL; + } + + /* Mixed backref is an always-enabled feature. */ + incompat |= BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF; + + /* Set compression related flags just in case. */ + if (fs_info->compress_type == BTRFS_COMPRESS_LZO) + incompat |= BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO; + else if (fs_info->compress_type == BTRFS_COMPRESS_ZSTD) + incompat |= BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD; + + /* + * An ancient flag, which should really be marked deprecated. + * Such runtime limitation doesn't really need a incompat flag. + */ + if (btrfs_super_nodesize(disk_super) > PAGE_SIZE) + incompat |= BTRFS_FEATURE_INCOMPAT_BIG_METADATA; + + if (compat_ro_unsupp && is_rw_mount) { + btrfs_err(fs_info, + "cannot mount read-write because of unknown compat_ro features (0x%llx)", + compat_ro_unsupp); + return -EINVAL; + } + + /* + * We have unsupported RO compat features, although RO mounted, we + * should not cause any metadata writes, including log replay. + * Or we could screw up whatever the new feature requires. + */ + if (compat_ro_unsupp && btrfs_super_log_root(disk_super) && + !btrfs_test_opt(fs_info, NOLOGREPLAY)) { + btrfs_err(fs_info, +"cannot replay dirty log with unsupported compat_ro features (0x%llx), try rescue=nologreplay", + compat_ro_unsupp); + return -EINVAL; + } + + /* + * Artificial limitations for block group tree, to force + * block-group-tree to rely on no-holes and free-space-tree. + */ + if (btrfs_fs_compat_ro(fs_info, BLOCK_GROUP_TREE) && + (!btrfs_fs_incompat(fs_info, NO_HOLES) || + !btrfs_test_opt(fs_info, FREE_SPACE_TREE))) { + btrfs_err(fs_info, +"block-group-tree feature requires no-holes and free-space-tree features"); + return -EINVAL; + } + + /* + * Subpage/bs > ps runtime limitation on v1 cache. + * + * V1 space cache still has some hard coded PAGE_SIZE usage, while + * we're already defaulting to v2 cache, no need to bother v1 as it's + * going to be deprecated anyway. + */ + if (fs_info->sectorsize != PAGE_SIZE && btrfs_test_opt(fs_info, SPACE_CACHE)) { + btrfs_warn(fs_info, + "v1 space cache is not supported for page size %lu with sectorsize %u", + PAGE_SIZE, fs_info->sectorsize); + return -EINVAL; + } + + /* This can be called by remount, we need to protect the super block. */ + spin_lock(&fs_info->super_lock); + btrfs_set_super_incompat_flags(disk_super, incompat); + spin_unlock(&fs_info->super_lock); + + return 0; +} + +/* + * Try to wait for any metadata readahead, and invalidate all btree folios. + * + * If the invalidation failed, report any dirty/held extent buffers. + */ +static void invalidate_and_check_btree_folios(struct btrfs_fs_info *fs_info) +{ + unsigned long index = 0; + struct extent_buffer *eb; + int ret; + + ret = invalidate_inode_pages2(fs_info->btree_inode->i_mapping); + if (likely(ret == 0)) + return; + + /* + * Some btree pages can not be invalidated, this happens when some tree + * blocks are still held (either by readahead or some task is holding a ref). + */ + rcu_read_lock(); + xa_for_each(&fs_info->buffer_tree, index, eb) { + unsigned int refs; + + /* Increase the ref so that the eb won't disappear. */ + if (!refcount_inc_not_zero(&eb->refs)) + continue; + rcu_read_unlock(); + + /* Wait for any readahead first. */ + if (test_bit(EXTENT_BUFFER_READING, &eb->bflags)) + wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_READING, + TASK_UNINTERRUPTIBLE); + /* + * We hold the spinlock to make sure above + * EXTENT_BUFFER_READING flag is cleared with the held + * ref dropped. + * Or we can hit a race window and lead to false alerts. + */ + spin_lock(&eb->refs_lock); + refs = refcount_read(&eb->refs); + spin_unlock(&eb->refs_lock); + + /* + * The refs threshold is 2, one held by us at the beginning + * of the loop, one for the ownership in the buffer tree. + */ + if (unlikely(refs > 2 || extent_buffer_under_io(eb))) { + WARN_ON_ONCE(IS_ENABLED(CONFIG_BTRFS_DEBUG)); + btrfs_warn(fs_info, + "unable to release extent buffer %llu owner %llu gen %llu refs %u flags 0x%lx", + eb->start, btrfs_header_owner(eb), + btrfs_header_generation(eb), + refs, eb->bflags); + } + free_extent_buffer(eb); + rcu_read_lock(); + } + rcu_read_unlock(); + invalidate_inode_pages2(fs_info->btree_inode->i_mapping); +} + +static u32 calc_block_max_order(u32 sectorsize_bits) +{ + u32 max_size; + + max_size = min(BTRFS_MAX_BLOCKS_PER_FOLIO << sectorsize_bits, + BTRFS_MAX_FOLIO_SIZE); + return ilog2(round_up(max_size, PAGE_SIZE) >> PAGE_SHIFT); +} + +int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_devices) +{ + u32 sectorsize; + u32 nodesize; + u64 generation; + u16 csum_type; + struct btrfs_super_block *disk_super; + struct btrfs_fs_info *fs_info = btrfs_sb(sb); + struct btrfs_root *tree_root; + struct btrfs_root *chunk_root; + struct btrfs_root *remap_root; + int ret; + int level; + + ret = init_mount_fs_info(fs_info, sb); + if (ret) + goto fail; + + /* These need to be init'ed before we start creating inodes and such. */ + tree_root = btrfs_alloc_root(fs_info, BTRFS_ROOT_TREE_OBJECTID, + GFP_KERNEL); + fs_info->tree_root = tree_root; + chunk_root = btrfs_alloc_root(fs_info, BTRFS_CHUNK_TREE_OBJECTID, + GFP_KERNEL); + fs_info->chunk_root = chunk_root; + if (!tree_root || !chunk_root) { + ret = -ENOMEM; + goto fail; + } + + ret = btrfs_init_btree_inode(sb); + if (ret) + goto fail; + + invalidate_bdev(fs_devices->latest_dev->bdev); + + /* + * Read super block and check the signature bytes only + */ + disk_super = btrfs_read_disk_super(fs_devices->latest_dev->bdev, 0, false); + if (IS_ERR(disk_super)) { + ret = PTR_ERR(disk_super); + goto fail_alloc; + } + + btrfs_info(fs_info, "first mount of filesystem %pU", disk_super->fsid); + /* + * Verify the type first, if that or the checksum value are + * corrupted, we'll find out + */ + csum_type = btrfs_super_csum_type(disk_super); + if (!btrfs_supported_super_csum(csum_type)) { + btrfs_err(fs_info, "unsupported checksum algorithm: %u", + csum_type); + ret = -EINVAL; + btrfs_release_disk_super(disk_super); + goto fail_alloc; + } + + fs_info->csum_size = btrfs_super_csum_size(disk_super); + fs_info->csum_type = csum_type; + + btrfs_init_csum_hash(fs_info, csum_type); + + /* + * We want to check superblock checksum, the type is stored inside. + * Pass the whole disk block of size BTRFS_SUPER_INFO_SIZE (4k). + */ + if (btrfs_check_super_csum(fs_info, disk_super)) { + btrfs_err(fs_info, "superblock checksum mismatch"); + ret = -EINVAL; + btrfs_release_disk_super(disk_super); + goto fail_alloc; + } + + /* + * super_copy is zeroed at allocation time and we never touch the + * following bytes up to INFO_SIZE, the checksum is calculated from + * the whole block of INFO_SIZE + */ + memcpy(fs_info->super_copy, disk_super, sizeof(*fs_info->super_copy)); + btrfs_release_disk_super(disk_super); + + disk_super = fs_info->super_copy; + + memcpy(fs_info->super_for_commit, fs_info->super_copy, + sizeof(*fs_info->super_for_commit)); + + ret = btrfs_validate_mount_super(fs_info); + if (ret) { + btrfs_err(fs_info, "superblock contains fatal errors"); + ret = -EINVAL; + goto fail_alloc; + } + + if (!btrfs_super_root(disk_super)) { + btrfs_err(fs_info, "invalid superblock tree root bytenr"); + ret = -EINVAL; + goto fail_alloc; + } + + /* check FS state, whether FS is broken. */ + if (btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_ERROR) + WRITE_ONCE(fs_info->fs_error, -EUCLEAN); + + /* If the fs has any rescue options, no transaction is allowed. */ + if (btrfs_is_full_ro(fs_info)) + WRITE_ONCE(fs_info->fs_error, -EROFS); + + /* Set up fs_info before parsing mount options */ + nodesize = btrfs_super_nodesize(disk_super); + sectorsize = btrfs_super_sectorsize(disk_super); + fs_info->dirty_metadata_batch = nodesize * (1 + ilog2(nr_cpu_ids)); + fs_info->delalloc_batch = sectorsize * 512 * (1 + ilog2(nr_cpu_ids)); + + fs_info->nodesize = nodesize; + fs_info->nodesize_bits = ilog2(nodesize); + fs_info->sectorsize = sectorsize; + fs_info->sectorsize_bits = ilog2(sectorsize); + fs_info->block_min_order = ilog2(round_up(sectorsize, PAGE_SIZE) >> PAGE_SHIFT); + /* + * For HIGHMEM, a large folio cannot be mapped in one go, breaking a lot + * of basic assumptions for btrfs IOs. + * Disable large folios for such 32-bit systems. + */ + if (IS_ENABLED(CONFIG_HIGHMEM)) + fs_info->block_max_order = fs_info->block_min_order; + else + fs_info->block_max_order = calc_block_max_order(fs_info->sectorsize_bits); + fs_info->csums_per_leaf = BTRFS_MAX_ITEM_SIZE(fs_info) / fs_info->csum_size; + fs_info->fs_devices->fs_info = fs_info; + + if (fs_info->sectorsize > PAGE_SIZE) + btrfs_warn(fs_info, + "support for block size %u with page size %lu is experimental, some features may be missing", + fs_info->sectorsize, PAGE_SIZE); + /* + * Handle the space caching options appropriately now that we have the + * super block loaded and validated. + */ + btrfs_set_free_space_cache_settings(fs_info); + + if (!btrfs_check_options(fs_info, &fs_info->mount_opt, sb->s_flags)) { + ret = -EINVAL; + goto fail_alloc; + } + + ret = btrfs_check_features(fs_info, !sb_rdonly(sb)); + if (ret < 0) + goto fail_alloc; + + if (btrfs_super_incompat_flags(disk_super) & BTRFS_FEATURE_INCOMPAT_REMAP_TREE) { + remap_root = btrfs_alloc_root(fs_info, BTRFS_REMAP_TREE_OBJECTID, + GFP_KERNEL); + fs_info->remap_root = remap_root; + if (!remap_root) { + ret = -ENOMEM; + goto fail_alloc; + } + } + + /* + * At this point our mount options are validated, if we set ->max_inline + * to something non-standard make sure we truncate it to sectorsize. + */ + fs_info->max_inline = min_t(u64, fs_info->max_inline, fs_info->sectorsize); + + ret = btrfs_alloc_compress_wsm(fs_info); + if (ret) + goto fail_sb_buffer; + ret = btrfs_init_workqueues(fs_info); + if (ret) + goto fail_sb_buffer; + + sb->s_bdi->ra_pages *= btrfs_super_num_devices(disk_super); + sb->s_bdi->ra_pages = max(sb->s_bdi->ra_pages, SZ_4M / PAGE_SIZE); + + /* Update the values for the current filesystem. */ + sb->s_blocksize = sectorsize; + sb->s_blocksize_bits = blksize_bits(sectorsize); + /* + * When temp_fsid is active, fs_devices->fsid is assigned a random UUID + * at mount. This inconsistent UUID causes issues for layered filesystems + * like OverlayFS. Since metadata_uuid may or may not be set, provide the + * on-disk UUID directly from the super_copy. + */ + if (fs_info->fs_devices->temp_fsid) + memcpy(&sb->s_uuid, fs_info->super_copy->fsid, BTRFS_FSID_SIZE); + else + memcpy(&sb->s_uuid, fs_info->fs_devices->fsid, BTRFS_FSID_SIZE); + + mutex_lock(&fs_info->chunk_mutex); + ret = btrfs_read_sys_array(fs_info); + mutex_unlock(&fs_info->chunk_mutex); + if (ret) { + btrfs_err(fs_info, "failed to read the system array: %pe", ERR_PTR(ret)); + goto fail_sb_buffer; + } + + generation = btrfs_super_chunk_root_generation(disk_super); + level = btrfs_super_chunk_root_level(disk_super); + ret = load_super_root(chunk_root, btrfs_super_chunk_root(disk_super), + generation, level); + if (ret) { + btrfs_err(fs_info, "failed to read chunk root"); + goto fail_tree_roots; + } + + read_extent_buffer(chunk_root->node, fs_info->chunk_tree_uuid, + offsetof(struct btrfs_header, chunk_tree_uuid), + BTRFS_UUID_SIZE); + + ret = btrfs_read_chunk_tree(fs_info); + if (ret) { + btrfs_err(fs_info, "failed to read chunk tree: %pe", ERR_PTR(ret)); + goto fail_tree_roots; + } + + /* + * At this point we know all the devices that make this filesystem, + * including the seed devices but we don't know yet if the replace + * target is required. So free devices that are not part of this + * filesystem but skip the replace target device which is checked + * below in btrfs_init_dev_replace(). + */ + btrfs_free_extra_devids(fs_devices); + if (unlikely(!fs_devices->latest_dev->bdev)) { + btrfs_err(fs_info, "failed to read devices"); + ret = -EIO; + goto fail_tree_roots; + } + + ret = init_tree_roots(fs_info); + if (ret) + goto fail_tree_roots; + + /* + * Get zone type information of zoned block devices. This will also + * handle emulation of a zoned filesystem if a regular device has the + * zoned incompat feature flag set. + */ + ret = btrfs_get_dev_zone_info_all_devices(fs_info); + if (ret) { + btrfs_err(fs_info, + "zoned: failed to read device zone info: %pe", ERR_PTR(ret)); + goto fail_block_groups; + } + + /* + * If we have a uuid root and we're not being told to rescan we need to + * check the generation here so we can set the + * BTRFS_FS_UPDATE_UUID_TREE_GEN bit. Otherwise we could commit the + * transaction during a balance or the log replay without updating the + * uuid generation, and then if we crash we would rescan the uuid tree, + * even though it was perfectly fine. + */ + if (fs_info->uuid_root && !btrfs_test_opt(fs_info, RESCAN_UUID_TREE) && + fs_info->generation == btrfs_super_uuid_tree_generation(disk_super)) + set_bit(BTRFS_FS_UPDATE_UUID_TREE_GEN, &fs_info->flags); + + if (unlikely(btrfs_verify_dev_items(fs_info))) { + ret = -EUCLEAN; + goto fail_block_groups; + } + ret = btrfs_verify_dev_extents(fs_info); + if (ret) { + btrfs_err(fs_info, + "failed to verify dev extents against chunks: %pe", + ERR_PTR(ret)); + goto fail_block_groups; + } + ret = btrfs_recover_balance(fs_info); + if (ret) { + btrfs_err(fs_info, "failed to recover balance: %pe", ERR_PTR(ret)); + goto fail_block_groups; + } + + ret = btrfs_init_dev_stats(fs_info); + if (ret) { + btrfs_err(fs_info, "failed to init dev_stats: %pe", ERR_PTR(ret)); + goto fail_block_groups; + } + + ret = btrfs_init_dev_replace(fs_info); + if (ret) { + btrfs_err(fs_info, "failed to init dev_replace: %pe", ERR_PTR(ret)); + goto fail_block_groups; + } + + ret = btrfs_check_zoned_mode(fs_info); + if (ret) { + btrfs_err(fs_info, "failed to initialize zoned mode: %pe", + ERR_PTR(ret)); + goto fail_block_groups; + } + + ret = btrfs_sysfs_add_fsid(fs_devices); + if (ret) { + btrfs_err(fs_info, "failed to init sysfs fsid interface: %pe", + ERR_PTR(ret)); + goto fail_block_groups; + } + + ret = btrfs_sysfs_add_mounted(fs_info); + if (ret) { + btrfs_err(fs_info, "failed to init sysfs interface: %pe", ERR_PTR(ret)); + goto fail_fsdev_sysfs; + } + + ret = btrfs_init_space_info(fs_info); + if (ret) { + btrfs_err(fs_info, "failed to initialize space info: %pe", ERR_PTR(ret)); + goto fail_sysfs; + } + + ret = btrfs_read_block_groups(fs_info); + if (ret) { + btrfs_err(fs_info, "failed to read block groups: %pe", ERR_PTR(ret)); + goto fail_sysfs; + } + + if (btrfs_fs_incompat(fs_info, REMAP_TREE)) { + ret = btrfs_populate_fully_remapped_bgs_list(fs_info); + if (ret) { + btrfs_err(fs_info, "failed to populate fully_remapped_bgs list: %pe", + ERR_PTR(ret)); + goto fail_sysfs; + } + } + + ret = btrfs_init_writeback_bio_size(fs_info); + if (ret) { + btrfs_err(fs_info, "failed to get optimum writeback size: %pe", + ERR_PTR(ret)); + goto fail_sysfs; + } + + btrfs_free_zone_cache(fs_info); + + btrfs_check_active_zone_reservation(fs_info); + + if (!sb_rdonly(sb) && fs_info->fs_devices->missing_devices && + !btrfs_check_rw_degradable(fs_info, NULL)) { + btrfs_warn(fs_info, + "writable mount is not allowed due to too many missing devices"); + ret = -EINVAL; + goto fail_sysfs; + } + + fs_info->cleaner_kthread = kthread_run(cleaner_kthread, fs_info, + "btrfs-cleaner"); + if (IS_ERR(fs_info->cleaner_kthread)) { + ret = PTR_ERR(fs_info->cleaner_kthread); + goto fail_sysfs; + } + + fs_info->transaction_kthread = kthread_run(transaction_kthread, + tree_root, + "btrfs-transaction"); + if (IS_ERR(fs_info->transaction_kthread)) { + ret = PTR_ERR(fs_info->transaction_kthread); + goto fail_cleaner; + } + + /* + * Starts a transaction, must be called after the transaction kthread + * is initialized. + */ + btrfs_zoned_reserve_data_reloc_bg(fs_info); + + ret = btrfs_read_qgroup_config(fs_info); + if (ret) + goto fail_trans_kthread; + + if (btrfs_build_ref_tree(fs_info)) + btrfs_err(fs_info, "couldn't build ref tree"); + + /* do not make disk changes in broken FS or nologreplay is given */ + if (btrfs_super_log_root(disk_super) != 0 && + !btrfs_test_opt(fs_info, NOLOGREPLAY)) { + btrfs_info(fs_info, "start tree-log replay"); + ret = btrfs_replay_log(fs_info, fs_devices); + if (ret) + goto fail_qgroup; + } + + fs_info->fs_root = btrfs_get_fs_root(fs_info, BTRFS_FS_TREE_OBJECTID, true); + if (IS_ERR(fs_info->fs_root)) { + ret = PTR_ERR(fs_info->fs_root); + btrfs_err(fs_info, "failed to read fs tree: %pe", ERR_PTR(ret)); + fs_info->fs_root = NULL; + goto fail_qgroup; + } + + if (sb_rdonly(sb)) + return 0; + + ret = btrfs_start_pre_rw_mount(fs_info); + if (ret) { + close_ctree(fs_info); + return ret; + } + btrfs_discard_resume(fs_info); + + if (fs_info->uuid_root && + (btrfs_test_opt(fs_info, RESCAN_UUID_TREE) || + !test_bit(BTRFS_FS_UPDATE_UUID_TREE_GEN, &fs_info->flags))) { + btrfs_info(fs_info, "checking UUID tree"); + ret = btrfs_check_uuid_tree(fs_info); + if (ret) { + btrfs_err(fs_info, "failed to check the UUID tree: %pe", ERR_PTR(ret)); + close_ctree(fs_info); + return ret; + } + } + + set_bit(BTRFS_FS_OPEN, &fs_info->flags); + + /* Kick the cleaner thread so it'll start deleting snapshots. */ + if (test_bit(BTRFS_FS_UNFINISHED_DROPS, &fs_info->flags)) + wake_up_process(fs_info->cleaner_kthread); + + return 0; + +fail_qgroup: + btrfs_free_qgroup_config(fs_info); +fail_trans_kthread: + kthread_stop(fs_info->transaction_kthread); + btrfs_cleanup_transaction(fs_info); + btrfs_free_fs_roots(fs_info); +fail_cleaner: + kthread_stop(fs_info->cleaner_kthread); + + /* + * make sure we're done with the btree inode before we stop our + * kthreads + */ + filemap_write_and_wait(fs_info->btree_inode->i_mapping); + +fail_sysfs: + btrfs_sysfs_remove_mounted(fs_info); + +fail_fsdev_sysfs: + btrfs_sysfs_remove_fsid(fs_info->fs_devices); + +fail_block_groups: + btrfs_put_block_group_cache(fs_info); + +fail_tree_roots: + if (fs_info->data_reloc_root) + btrfs_drop_and_free_fs_root(fs_info, fs_info->data_reloc_root); + free_root_pointers(fs_info, true); + invalidate_and_check_btree_folios(fs_info); + +fail_sb_buffer: + btrfs_stop_all_workers(fs_info); + btrfs_free_block_groups(fs_info); +fail_alloc: + btrfs_mapping_tree_free(fs_info); + + iput(fs_info->btree_inode); +fail: + ASSERT(ret < 0); + return ret; +} +ALLOW_ERROR_INJECTION(open_ctree, ERRNO); + +static void btrfs_end_super_write(struct bio *bio) +{ + struct btrfs_device *device = bio->bi_private; + struct folio_iter fi; + + bio_for_each_folio_all(fi, bio) { + if (bio->bi_status) { + btrfs_warn_rl(device->fs_info, + "lost super block write due to IO error on %s (%d)", + btrfs_dev_name(device), + blk_status_to_errno(bio->bi_status)); + btrfs_dev_stat_inc_and_print(device, + BTRFS_DEV_STAT_WRITE_ERRS); + /* Ensure failure if the primary sb fails. */ + if (bio->bi_opf & REQ_FUA) + atomic_add(BTRFS_SUPER_PRIMARY_WRITE_ERROR, + &device->sb_write_errors); + else + atomic_inc(&device->sb_write_errors); + } + folio_unlock(fi.folio); + folio_put(fi.folio); + } + + bio_put(bio); +} + +/* + * Write superblock @sb to the @device. Do not wait for completion, all the + * folios we use for writing are locked. + * + * Write @max_mirrors copies of the superblock. Note that max_mirrors must be + * same for write and wait phases. + * + * Return number of errors when folio is not found or submission fails. + */ +static int write_dev_supers(struct btrfs_device *device, + struct btrfs_super_block *sb, int max_mirrors) +{ + struct btrfs_fs_info *fs_info = device->fs_info; + struct address_space *mapping = device->bdev->bd_mapping; + int i; + int ret; + u64 bytenr, bytenr_orig; + + atomic_set(&device->sb_write_errors, 0); + + for (i = 0; i < max_mirrors; i++) { + struct folio *folio; + struct bio *bio; + struct btrfs_super_block *disk_super; + size_t offset; + + bytenr_orig = btrfs_sb_offset(i); + ret = btrfs_sb_log_location(device, i, WRITE, &bytenr); + if (ret == -ENOENT) { + continue; + } else if (ret < 0) { + btrfs_err(device->fs_info, + "couldn't get super block location for mirror %d error %pe", + i, ERR_PTR(ret)); + atomic_inc(&device->sb_write_errors); + continue; + } + if (bytenr + BTRFS_SUPER_INFO_SIZE >= + device->commit_total_bytes) + break; + + btrfs_set_super_bytenr(sb, bytenr_orig); + + btrfs_csum(fs_info->csum_type, (const u8 *)sb + BTRFS_CSUM_SIZE, + BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE, sb->csum); + + folio = __filemap_get_folio(mapping, bytenr >> PAGE_SHIFT, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, + GFP_NOFS); + if (IS_ERR(folio)) { + btrfs_err(device->fs_info, + "couldn't get super block page for bytenr %llu error %pe", + bytenr, folio); + atomic_inc(&device->sb_write_errors); + continue; + } + + offset = offset_in_folio(folio, bytenr); + disk_super = folio_address(folio) + offset; + memcpy(disk_super, sb, BTRFS_SUPER_INFO_SIZE); + + /* + * Directly use bios here instead of relying on the page cache + * to do I/O, so we don't lose the ability to do integrity + * checking. + */ + bio = bio_alloc(device->bdev, 1, + REQ_OP_WRITE | REQ_SYNC | REQ_META | REQ_PRIO, + GFP_NOFS); + bio->bi_iter.bi_sector = bytenr >> SECTOR_SHIFT; + bio->bi_private = device; + bio->bi_end_io = btrfs_end_super_write; + bio_add_folio_nofail(bio, folio, BTRFS_SUPER_INFO_SIZE, offset); + + /* + * We FUA only the first super block. The others we allow to + * go down lazy and there's a short window where the on-disk + * copies might still contain the older version. + */ + if (i == 0 && !btrfs_test_opt(device->fs_info, NOBARRIER)) + bio->bi_opf |= REQ_FUA; + submit_bio(bio); + + if (btrfs_advance_sb_log(device, i)) + atomic_inc(&device->sb_write_errors); + } + return atomic_read(&device->sb_write_errors) < i ? 0 : -1; +} + +/* + * Wait for write completion of superblocks done by write_dev_supers, + * @max_mirrors same for write and wait phases. + * + * Return -1 if primary super block write failed or when there were no super block + * copies written. Otherwise 0. + */ +static int wait_dev_supers(struct btrfs_device *device, int max_mirrors) +{ + int i; + int errors = 0; + bool primary_failed = false; + int ret; + u64 bytenr; + + for (i = 0; i < max_mirrors; i++) { + struct folio *folio; + + ret = btrfs_sb_log_location(device, i, READ, &bytenr); + if (ret == -ENOENT) { + break; + } else if (unlikely(ret < 0)) { + errors++; + if (i == 0) + primary_failed = true; + continue; + } + if (bytenr + BTRFS_SUPER_INFO_SIZE >= + device->commit_total_bytes) + break; + + folio = filemap_get_folio(device->bdev->bd_mapping, + bytenr >> PAGE_SHIFT); + /* If the folio has been removed, then we know it completed. */ + if (IS_ERR(folio)) + continue; + + /* Folio will be unlocked once the write completes. */ + folio_wait_locked(folio); + folio_put(folio); + } + + errors += atomic_read(&device->sb_write_errors); + + if (unlikely(primary_failed || errors >= BTRFS_SUPER_PRIMARY_WRITE_ERROR)) { + btrfs_err(device->fs_info, "error writing primary super block to device %llu", + device->devid); + return -1; + } + + return errors < i ? 0 : -1; +} + +/* + * endio for the write_dev_flush, this will wake anyone waiting + * for the barrier when it is done + */ +static void btrfs_end_empty_barrier(struct bio *bio) +{ + bio_uninit(bio); + complete(bio->bi_private); +} + +/* + * Submit a flush request to the device if it supports it. Error handling is + * done in the waiting counterpart. + */ +static void write_dev_flush(struct btrfs_device *device) +{ + struct bio *bio = &device->flush_bio; + + clear_bit(BTRFS_DEV_STATE_FLUSH_FAILED, &device->dev_state); + + bio_init(bio, device->bdev, NULL, 0, + REQ_OP_WRITE | REQ_SYNC | REQ_PREFLUSH); + bio->bi_end_io = btrfs_end_empty_barrier; + init_completion(&device->flush_wait); + bio->bi_private = &device->flush_wait; + submit_bio(bio); + set_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state); +} + +/* + * If the flush bio has been submitted by write_dev_flush, wait for it. + * Return true for any error, and false otherwise. + */ +static bool wait_dev_flush(struct btrfs_device *device) +{ + struct bio *bio = &device->flush_bio; + + if (!test_and_clear_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state)) + return false; + + wait_for_completion_io(&device->flush_wait); + + if (unlikely(bio->bi_status)) { + set_bit(BTRFS_DEV_STATE_FLUSH_FAILED, &device->dev_state); + btrfs_dev_stat_inc_and_print(device, BTRFS_DEV_STAT_FLUSH_ERRS); + return true; + } + + return false; +} + +/* + * send an empty flush down to each device in parallel, + * then wait for them + */ +static int barrier_all_devices(struct btrfs_fs_info *info) +{ + struct list_head *head; + struct btrfs_device *dev; + int errors_wait = 0; + + lockdep_assert_held(&info->fs_devices->device_list_mutex); + /* send down all the barriers */ + head = &info->fs_devices->devices; + list_for_each_entry(dev, head, dev_list) { + if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state)) + continue; + if (!dev->bdev) + continue; + if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) || + !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) + continue; + + write_dev_flush(dev); + } + + /* wait for all the barriers */ + list_for_each_entry(dev, head, dev_list) { + if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state)) + continue; + if (unlikely(!dev->bdev)) { + errors_wait++; + continue; + } + if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) || + !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) + continue; + + if (unlikely(wait_dev_flush(dev))) + errors_wait++; + } + + /* + * Checks flush failure of disks in order to determine the device + * state. + */ + if (unlikely(errors_wait && !btrfs_check_rw_degradable(info, NULL))) + return -EIO; + + return 0; +} + +int btrfs_get_num_tolerated_disk_barrier_failures(u64 flags) +{ + int raid_type; + int min_tolerated = INT_MAX; + + if ((flags & BTRFS_BLOCK_GROUP_PROFILE_MASK) == 0 || + (flags & BTRFS_AVAIL_ALLOC_BIT_SINGLE)) + min_tolerated = min_t(int, min_tolerated, + btrfs_raid_array[BTRFS_RAID_SINGLE]. + tolerated_failures); + + for (raid_type = 0; raid_type < BTRFS_NR_RAID_TYPES; raid_type++) { + if (raid_type == BTRFS_RAID_SINGLE) + continue; + if (!(flags & btrfs_raid_array[raid_type].bg_flag)) + continue; + min_tolerated = min_t(int, min_tolerated, + btrfs_raid_array[raid_type]. + tolerated_failures); + } + + if (min_tolerated == INT_MAX) { + btrfs_warn(NULL, "unknown raid flag: %llu", flags); + min_tolerated = 0; + } + + return min_tolerated; +} + +int write_all_supers(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct list_head *head; + struct btrfs_device *dev; + struct btrfs_super_block *sb; + struct btrfs_dev_item *dev_item; + int max_mirrors; + int ret; + int do_barriers; + int max_errors; + int total_errors = 0; + + do_barriers = !btrfs_test_opt(fs_info, NOBARRIER); + + if (trans->transaction->state < TRANS_STATE_UNBLOCKED) { + /* We are called from fsync. */ + max_mirrors = 1; + } else { + /* We are called from transaction commit. */ + max_mirrors = BTRFS_SUPER_MIRROR_MAX; + ret = backup_super_roots(fs_info); + if (ret < 0) + return ret; + } + + sb = fs_info->super_for_commit; + dev_item = &sb->dev_item; + + mutex_lock(&fs_info->fs_devices->device_list_mutex); + head = &fs_info->fs_devices->devices; + max_errors = btrfs_super_num_devices(fs_info->super_copy) - 1; + + if (do_barriers) { + ret = barrier_all_devices(fs_info); + if (unlikely(ret)) { + mutex_unlock( + &fs_info->fs_devices->device_list_mutex); + btrfs_abort_transaction(trans, ret); + btrfs_err(fs_info, "error while submitting device barriers"); + return ret; + } + } + + btrfs_set_super_flags(sb, btrfs_super_flags(sb) | BTRFS_HEADER_FLAG_WRITTEN); + + list_for_each_entry(dev, head, dev_list) { + if (unlikely(!dev->bdev)) { + total_errors++; + continue; + } + if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) || + !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) + continue; + + btrfs_set_stack_device_generation(dev_item, 0); + btrfs_set_stack_device_type(dev_item, dev->type); + btrfs_set_stack_device_id(dev_item, dev->devid); + btrfs_set_stack_device_total_bytes(dev_item, + dev->commit_total_bytes); + btrfs_set_stack_device_bytes_used(dev_item, + dev->commit_bytes_used); + btrfs_set_stack_device_io_align(dev_item, dev->io_align); + btrfs_set_stack_device_io_width(dev_item, dev->io_width); + btrfs_set_stack_device_sector_size(dev_item, dev->sector_size); + memcpy(dev_item->uuid, dev->uuid, BTRFS_UUID_SIZE); + memcpy(dev_item->fsid, dev->fs_devices->metadata_uuid, + BTRFS_FSID_SIZE); + + ret = btrfs_validate_write_super(fs_info, sb); + if (unlikely(ret < 0)) { + mutex_unlock(&fs_info->fs_devices->device_list_mutex); + btrfs_abort_transaction(trans, ret); + btrfs_err(fs_info, + "unexpected superblock corruption before writing it"); + return ret; + } + + ret = write_dev_supers(dev, sb, max_mirrors); + if (unlikely(ret)) + total_errors++; + } + if (unlikely(total_errors > max_errors)) { + btrfs_err(fs_info, "%d errors while writing supers", + total_errors); + mutex_unlock(&fs_info->fs_devices->device_list_mutex); + + /* FUA is masked off if unsupported and can't be the reason */ + btrfs_abort_transaction(trans, -EIO); + btrfs_err(fs_info, "%d errors while writing supers", total_errors); + return -EIO; + } + + total_errors = 0; + list_for_each_entry(dev, head, dev_list) { + if (unlikely(!dev->bdev)) + continue; + if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) || + !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) + continue; + + ret = wait_dev_supers(dev, max_mirrors); + if (unlikely(ret)) + total_errors++; + } + mutex_unlock(&fs_info->fs_devices->device_list_mutex); + if (unlikely(total_errors > max_errors)) { + btrfs_abort_transaction(trans, -EIO); + btrfs_err(fs_info, "%d errors while writing supers", total_errors); + return -EIO; + } + return 0; +} + +/* Drop a fs root from the radix tree and free it. */ +void btrfs_drop_and_free_fs_root(struct btrfs_fs_info *fs_info, + struct btrfs_root *root) +{ + bool drop_ref = false; + + spin_lock(&fs_info->fs_roots_radix_lock); + radix_tree_delete(&fs_info->fs_roots_radix, + (unsigned long)btrfs_root_id(root)); + if (test_and_clear_bit(BTRFS_ROOT_IN_RADIX, &root->state)) + drop_ref = true; + spin_unlock(&fs_info->fs_roots_radix_lock); + + if (unlikely(BTRFS_FS_ERROR(fs_info))) { + ASSERT(root->log_root == NULL); + if (root->reloc_root) { + btrfs_put_root(root->reloc_root); + root->reloc_root = NULL; + } + } + + if (drop_ref) + btrfs_put_root(root); +} + +int btrfs_commit_super(struct btrfs_fs_info *fs_info) +{ + mutex_lock(&fs_info->cleaner_mutex); + btrfs_run_delayed_iputs(fs_info); + mutex_unlock(&fs_info->cleaner_mutex); + wake_up_process(fs_info->cleaner_kthread); + + /* wait until ongoing cleanup work done */ + down_write(&fs_info->cleanup_work_sem); + up_write(&fs_info->cleanup_work_sem); + + return btrfs_commit_current_transaction(fs_info->tree_root); +} + +static void warn_about_uncommitted_trans(struct btrfs_fs_info *fs_info) +{ + struct btrfs_transaction *trans; + struct btrfs_transaction *tmp; + bool found = false; + + /* + * This function is only called at the very end of close_ctree(), + * thus no other running transaction, no need to take trans_lock. + */ + ASSERT(test_bit(BTRFS_FS_CLOSING_DONE, &fs_info->flags)); + list_for_each_entry_safe(trans, tmp, &fs_info->trans_list, list) { + struct extent_state *cached = NULL; + u64 dirty_bytes = 0; + u64 cur = 0; + u64 found_start; + u64 found_end; + + found = true; + while (btrfs_find_first_extent_bit(&trans->dirty_pages, cur, + &found_start, &found_end, + EXTENT_DIRTY, &cached)) { + dirty_bytes += found_end + 1 - found_start; + cur = found_end + 1; + } + btrfs_warn(fs_info, + "transaction %llu (with %llu dirty metadata bytes) is not committed", + trans->transid, dirty_bytes); + btrfs_cleanup_one_transaction(trans); + + if (trans == fs_info->running_transaction) + fs_info->running_transaction = NULL; + list_del_init(&trans->list); + + btrfs_put_transaction(trans); + } + ASSERT(!found); +} + +void __cold close_ctree(struct btrfs_fs_info *fs_info) +{ + int ret; + + set_bit(BTRFS_FS_CLOSING_START, &fs_info->flags); + + /* + * If we had UNFINISHED_DROPS we could still be processing them, so + * clear that bit and wake up relocation so it can stop. + * We must do this before stopping the block group reclaim task, because + * at btrfs_relocate_block_group() we wait for this bit, and after the + * wait we stop with -EINTR if btrfs_fs_closing() returns non-zero - we + * have just set BTRFS_FS_CLOSING_START, so btrfs_fs_closing() will + * return 1. + */ + btrfs_wake_unfinished_drop(fs_info); + + /* + * We may have the reclaim task running and relocating a data block group, + * in which case it may create delayed iputs. So stop it before we park + * the cleaner kthread otherwise we can get new delayed iputs after + * parking the cleaner, and that can make the async reclaim task to hang + * if it's waiting for delayed iputs to complete, since the cleaner is + * parked and can not run delayed iputs - this will make us hang when + * trying to stop the async reclaim task. + */ + cancel_work_sync(&fs_info->reclaim_bgs_work); + /* + * We don't want the cleaner to start new transactions, add more delayed + * iputs, etc. while we're closing. We can't use kthread_stop() yet + * because that frees the task_struct, and the transaction kthread might + * still try to wake up the cleaner. + */ + kthread_park(fs_info->cleaner_kthread); + + /* wait for the qgroup rescan worker to stop */ + btrfs_qgroup_wait_for_completion(fs_info, false); + + /* wait for the uuid_scan task to finish */ + down(&fs_info->uuid_tree_rescan_sem); + /* avoid complains from lockdep et al., set sem back to initial state */ + up(&fs_info->uuid_tree_rescan_sem); + + /* pause restriper - we want to resume on mount */ + btrfs_pause_balance(fs_info); + + btrfs_dev_replace_suspend_for_unmount(fs_info); + + btrfs_scrub_cancel(fs_info); + + /* wait for any defraggers to finish */ + wait_event(fs_info->transaction_wait, + (atomic_read(&fs_info->defrag_running) == 0)); + + /* clear out the rbtree of defraggable inodes */ + btrfs_cleanup_defrag_inodes(fs_info); + + /* + * Before the unmount, we sync down all the writeback which can + * generate fixup work. We are about to run delalloc for autodefrag so + * piggy back on that by also flushing the fixup work which can also + * generate delalloc we would like to get run. + * + * After this, it is still possible that some thread doing writeback is + * in btrfs_queue_writepage_fixup() and might finish queueing some final + * work, racing the btrfs_fs_closing() check there. + */ + flush_workqueue(fs_info->fixup_workers); + + /* + * After we entered close_ctree() autodefrag could be running and before + * we parked the cleaner kthread, it dirtied folios of some inode. + * We don't want to leave any delalloc here, it may be flushed any time + * after this point and result in ordered extents that create delayed + * iputs after flushed the ordered extent queues further below, run + * delayed iputs and set BTRFS_FS_STATE_NO_DELAYED_IPUT. If we are + * mounted with flushoncommit, then btrfs_commit_super() called below + * will flush delalloc and wait for ordered extents but we end up + * getting delayed iputs than are never run. So flush delalloc and wait + * for ordered extents. + */ + btrfs_start_delalloc_roots(fs_info, LONG_MAX, false); + btrfs_wait_ordered_roots(fs_info, U64_MAX, NULL); + + /* + * Handle the error fs first, as it will flush and wait for all ordered + * extents. This will generate delayed iputs, thus we want to handle + * it first. + */ + if (unlikely(BTRFS_FS_ERROR(fs_info))) + btrfs_error_commit_super(fs_info); + + /* + * Similar case here, we have to wait for delalloc workers before we + * proceed below and stop the cleaner kthread, otherwise we trigger a + * use-after-tree on the cleaner kthread task_struct when a delalloc + * worker running submit_compressed_extents() adds a delayed iput, which + * does a wake up on the cleaner kthread, which was already freed below + * when we call kthread_stop(). + */ + btrfs_flush_workqueue(fs_info->delalloc_workers); + + /* + * We can have ordered extents getting their last reference dropped from + * the fs_info->workers queue because for async writes for data bios we + * queue a work for that queue, at btrfs_wq_submit_bio(), that runs + * run_one_async_done() which calls btrfs_bio_end_io() in case the bio + * has an error, and that later function can do the final + * btrfs_put_ordered_extent() on the ordered extent attached to the bio, + * which adds a delayed iput for the inode. So we must flush the queue + * so that we don't have delayed iputs after committing the current + * transaction below and stopping the cleaner and transaction kthreads. + */ + btrfs_flush_workqueue(fs_info->workers); + + /* + * When finishing a compressed write bio we schedule a work queue item + * to finish an ordered extent - end_bbio_compressed_write() + * calls btrfs_finish_ordered_extent() which in turns does a call to + * btrfs_queue_ordered_fn(), and that queues the ordered extent + * completion either in the endio_write_workers work queue or in the + * fs_info->endio_freespace_worker work queue. We flush those queues + * below, so before we flush them we must flush this queue for the + * workers of compressed writes. + */ + flush_workqueue(fs_info->endio_workers); + + /* + * After we parked the cleaner kthread, ordered extents may have + * completed and created new delayed iputs. If one of the async reclaim + * tasks is running and in the RUN_DELAYED_IPUTS flush state, then we + * can hang forever trying to stop it, because if a delayed iput is + * added after it ran btrfs_run_delayed_iputs() and before it called + * btrfs_wait_on_delayed_iputs(), it will hang forever since there is + * no one else to run iputs. + * + * So wait for all ongoing ordered extents to complete and then run + * delayed iputs. This works because once we reach this point no one + * can create new ordered extents, but delayed iputs can still be added + * by a reclaim worker (see comments further below). + * + * Also note that btrfs_wait_ordered_roots() is not safe here, because + * it waits for BTRFS_ORDERED_COMPLETE to be set on an ordered extent, + * but the delayed iput for the respective inode is made only when doing + * the final btrfs_put_ordered_extent() (which must happen at + * btrfs_finish_ordered_io() when we are unmounting). + */ + btrfs_flush_workqueue(fs_info->endio_write_workers); + /* Ordered extents for free space inodes. */ + btrfs_flush_workqueue(fs_info->endio_freespace_worker); + /* + * Run delayed iputs in case an async reclaim worker is waiting for them + * to be run as mentioned above. + */ + btrfs_run_delayed_iputs(fs_info); + + cancel_work_sync(&fs_info->async_reclaim_work); + cancel_work_sync(&fs_info->async_data_reclaim_work); + cancel_work_sync(&fs_info->preempt_reclaim_work); + cancel_work_sync(&fs_info->em_shrinker_work); + + /* + * Reclaim workers can run writeback which can queue fixup. + * After the above cancel_work_sync() calls, any such queueing attempts are + * guaranteed to see btrfs_fs_closing(), so at this point we can genuinely fully + * flush the fixup workqueue. This relies on the belief that *now* no thread can + * still be sitting in btrfs_queue_writepage_fixup(). + */ + flush_workqueue(fs_info->fixup_workers); + + /* + * Run delayed iputs again because an async reclaim worker may have + * added new ones if it was flushing delalloc: + * + * shrink_delalloc() -> btrfs_start_delalloc_roots() -> + * start_delalloc_inodes() -> btrfs_add_delayed_iput() + */ + btrfs_run_delayed_iputs(fs_info); + + /* There should be no more workload to generate new delayed iputs. */ + set_bit(BTRFS_FS_STATE_NO_DELAYED_IPUT, &fs_info->fs_state); + + /* Cancel or finish ongoing discard work */ + btrfs_discard_cleanup(fs_info); + + if (!sb_rdonly(fs_info->sb)) { + /* + * The cleaner kthread is stopped, so do one final pass over + * unused block groups. + */ + btrfs_delete_unused_bgs(fs_info); + + /* + * There might be existing delayed inode workers still running + * and holding an empty delayed inode item. We must wait for + * them to complete first because they can create a transaction. + * This happens when someone calls btrfs_balance_delayed_items() + * and then a transaction commit runs the same delayed nodes + * before any delayed worker has done something with the nodes. + * We must wait for any worker here and not at transaction + * commit time since that could cause a deadlock. + * This is a very rare case. + */ + btrfs_flush_workqueue(fs_info->delayed_workers); + + /* + * If the filesystem is shutdown, then an attempt to commit the + * super block (or any write) will just fail. Since we freeze + * the filesystem before shutting it down, the filesystem is in + * a consistent state and we don't need to commit super blocks. + */ + if (!btrfs_is_shutdown(fs_info)) { + ret = btrfs_commit_super(fs_info); + if (ret) + btrfs_err(fs_info, "commit super block returned %pe", ERR_PTR(ret)); + } + } + + kthread_stop(fs_info->transaction_kthread); + kthread_stop(fs_info->cleaner_kthread); + + ASSERT(list_empty(&fs_info->delayed_iputs)); + set_bit(BTRFS_FS_CLOSING_DONE, &fs_info->flags); + + if (unlikely(btrfs_check_quota_leak(fs_info))) { + DEBUG_WARN("qgroup reserved space leaked"); + btrfs_err(fs_info, "qgroup reserved space leaked"); + } + + btrfs_free_qgroup_config(fs_info); + ASSERT(list_empty(&fs_info->delalloc_roots)); + + if (percpu_counter_sum(&fs_info->delalloc_bytes)) { + btrfs_info(fs_info, "at unmount delalloc count %lld", + percpu_counter_sum(&fs_info->delalloc_bytes)); + } + + if (percpu_counter_sum(&fs_info->ordered_bytes)) + btrfs_info(fs_info, "at unmount dio bytes count %lld", + percpu_counter_sum(&fs_info->ordered_bytes)); + + btrfs_sysfs_remove_mounted(fs_info); + btrfs_sysfs_remove_fsid(fs_info->fs_devices); + + btrfs_put_block_group_cache(fs_info); + + /* We shouldn't have any transaction open at this point */ + warn_about_uncommitted_trans(fs_info); + + clear_bit(BTRFS_FS_OPEN, &fs_info->flags); + free_root_pointers(fs_info, true); + btrfs_free_fs_roots(fs_info); + + /* + * Drop metadata left stranded ahead of a zone write pointer while the + * endio workqueues are still up, so the final iput() of the btree inode + * below does not hang submitting a write that can no longer complete. + */ + btrfs_zoned_release_dirty_metadata(fs_info); + + /* + * We must make sure there is not any read request to + * submit after we stop all workers. + */ + invalidate_and_check_btree_folios(fs_info); + btrfs_stop_all_workers(fs_info); + + /* + * We must free the block groups after dropping the fs_roots as we could + * have had an IO error and have left over tree log blocks that aren't + * cleaned up until the fs roots are freed. This makes the block group + * accounting appear to be wrong because there's pending reserved bytes, + * so make sure we do the block group cleanup afterwards. + */ + btrfs_free_block_groups(fs_info); + + iput(fs_info->btree_inode); + + btrfs_mapping_tree_free(fs_info); +} + +void btrfs_mark_buffer_dirty(struct btrfs_trans_handle *trans, + struct extent_buffer *buf) +{ + struct btrfs_fs_info *fs_info = buf->fs_info; + u64 transid = btrfs_header_generation(buf); + +#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS + /* + * This is a fast path so only do this check if we have sanity tests + * enabled. Normal people shouldn't be using unmapped buffers as dirty + * outside of the sanity tests. + */ + if (unlikely(test_bit(EXTENT_BUFFER_UNMAPPED, &buf->bflags))) + return; +#endif + /* This is an active transaction (its state < TRANS_STATE_UNBLOCKED). */ + ASSERT(trans->transid == fs_info->generation); + btrfs_assert_tree_write_locked(buf); + if (unlikely(transid != fs_info->generation)) { + btrfs_abort_transaction(trans, -EUCLEAN); + btrfs_crit(fs_info, +"dirty buffer transid mismatch, logical %llu found transid %llu running transid %llu", + buf->start, transid, fs_info->generation); + } + set_extent_buffer_dirty(buf); +} + +static void __btrfs_btree_balance_dirty(struct btrfs_fs_info *fs_info, + int flush_delayed) +{ + /* + * looks as though older kernels can get into trouble with + * this code, they end up stuck in balance_dirty_pages forever + */ + int ret; + + if (current->flags & PF_MEMALLOC) + return; + + if (flush_delayed) + btrfs_balance_delayed_items(fs_info); + + ret = __percpu_counter_compare(&fs_info->dirty_metadata_bytes, + BTRFS_DIRTY_METADATA_THRESH, + fs_info->dirty_metadata_batch); + if (ret > 0) { + balance_dirty_pages_ratelimited(fs_info->btree_inode->i_mapping); + } +} + +void btrfs_btree_balance_dirty(struct btrfs_fs_info *fs_info) +{ + __btrfs_btree_balance_dirty(fs_info, 1); +} + +void btrfs_btree_balance_dirty_nodelay(struct btrfs_fs_info *fs_info) +{ + __btrfs_btree_balance_dirty(fs_info, 0); +} + +static void btrfs_error_commit_super(struct btrfs_fs_info *fs_info) +{ + /* cleanup FS via transaction */ + btrfs_cleanup_transaction(fs_info); + + down_write(&fs_info->cleanup_work_sem); + up_write(&fs_info->cleanup_work_sem); +} + +static void btrfs_drop_all_logs(struct btrfs_fs_info *fs_info) +{ + struct btrfs_root *gang[8]; + u64 root_objectid = 0; + int ret; + + spin_lock(&fs_info->fs_roots_radix_lock); + while ((ret = radix_tree_gang_lookup(&fs_info->fs_roots_radix, + (void **)gang, root_objectid, + ARRAY_SIZE(gang))) != 0) { + int i; + + for (i = 0; i < ret; i++) + gang[i] = btrfs_grab_root(gang[i]); + spin_unlock(&fs_info->fs_roots_radix_lock); + + for (i = 0; i < ret; i++) { + if (!gang[i]) + continue; + root_objectid = btrfs_root_id(gang[i]); + btrfs_free_log(NULL, gang[i]); + btrfs_put_root(gang[i]); + } + root_objectid++; + spin_lock(&fs_info->fs_roots_radix_lock); + } + spin_unlock(&fs_info->fs_roots_radix_lock); + btrfs_free_log_root_tree(NULL, fs_info); +} + +static void btrfs_destroy_ordered_extents(struct btrfs_root *root) +{ + struct btrfs_ordered_extent *ordered; + + spin_lock(&root->ordered_extent_lock); + /* + * This will just short circuit the ordered completion stuff which will + * make sure the ordered extent gets properly cleaned up. + */ + list_for_each_entry(ordered, &root->ordered_extents, + root_extent_list) + set_bit(BTRFS_ORDERED_IOERR, &ordered->flags); + spin_unlock(&root->ordered_extent_lock); +} + +static void btrfs_destroy_all_ordered_extents(struct btrfs_fs_info *fs_info) +{ + struct btrfs_root *root; + LIST_HEAD(splice); + + spin_lock(&fs_info->ordered_root_lock); + list_splice_init(&fs_info->ordered_roots, &splice); + while (!list_empty(&splice)) { + root = list_first_entry(&splice, struct btrfs_root, + ordered_root); + list_move_tail(&root->ordered_root, + &fs_info->ordered_roots); + + spin_unlock(&fs_info->ordered_root_lock); + btrfs_destroy_ordered_extents(root); + + cond_resched(); + spin_lock(&fs_info->ordered_root_lock); + } + spin_unlock(&fs_info->ordered_root_lock); + + /* + * We need this here because if we've been flipped read-only we won't + * get sync() from the umount, so we need to make sure any ordered + * extents that haven't had their dirty pages IO start writeout yet + * actually get run and error out properly. + */ + btrfs_wait_ordered_roots(fs_info, U64_MAX, NULL); +} + +static void btrfs_destroy_delalloc_inodes(struct btrfs_root *root) +{ + struct btrfs_inode *btrfs_inode; + LIST_HEAD(splice); + + spin_lock(&root->delalloc_lock); + list_splice_init(&root->delalloc_inodes, &splice); + + while (!list_empty(&splice)) { + struct inode *inode = NULL; + btrfs_inode = list_first_entry(&splice, struct btrfs_inode, + delalloc_inodes); + btrfs_del_delalloc_inode(btrfs_inode); + spin_unlock(&root->delalloc_lock); + + /* + * Make sure we get a live inode and that it'll not disappear + * meanwhile. + */ + inode = igrab(&btrfs_inode->vfs_inode); + if (inode) { + unsigned int nofs_flag; + + nofs_flag = memalloc_nofs_save(); + invalidate_inode_pages2(inode->i_mapping); + memalloc_nofs_restore(nofs_flag); + iput(inode); + } + spin_lock(&root->delalloc_lock); + } + spin_unlock(&root->delalloc_lock); +} + +static void btrfs_destroy_all_delalloc_inodes(struct btrfs_fs_info *fs_info) +{ + struct btrfs_root *root; + LIST_HEAD(splice); + + spin_lock(&fs_info->delalloc_root_lock); + list_splice_init(&fs_info->delalloc_roots, &splice); + while (!list_empty(&splice)) { + root = list_first_entry(&splice, struct btrfs_root, + delalloc_root); + root = btrfs_grab_root(root); + BUG_ON(!root); + spin_unlock(&fs_info->delalloc_root_lock); + + btrfs_destroy_delalloc_inodes(root); + btrfs_put_root(root); + + spin_lock(&fs_info->delalloc_root_lock); + } + spin_unlock(&fs_info->delalloc_root_lock); +} + +static void btrfs_destroy_marked_extents(struct btrfs_fs_info *fs_info, + struct extent_io_tree *dirty_pages, + int mark) +{ + struct extent_buffer *eb; + u64 start = 0; + u64 end; + + while (btrfs_find_first_extent_bit(dirty_pages, start, &start, &end, + mark, NULL)) { + btrfs_clear_extent_bit(dirty_pages, start, end, mark, NULL); + while (start <= end) { + eb = find_extent_buffer(fs_info, start); + start += fs_info->nodesize; + if (!eb) + continue; + + btrfs_tree_lock(eb); + wait_on_extent_buffer_writeback(eb); + btrfs_clear_buffer_dirty(NULL, eb); + btrfs_tree_unlock(eb); + + free_extent_buffer_stale(eb); + } + } + btrfs_extent_io_tree_release(dirty_pages); +} + +static void btrfs_destroy_pinned_extent(struct btrfs_fs_info *fs_info, + struct extent_io_tree *unpin) +{ + u64 start; + u64 end; + + while (1) { + struct extent_state *cached_state = NULL; + + /* + * The btrfs_finish_extent_commit() may get the same range as + * ours between find_first_extent_bit and clear_extent_dirty. + * Hence, hold the unused_bg_unpin_mutex to avoid double unpin + * the same extent range. + */ + mutex_lock(&fs_info->unused_bg_unpin_mutex); + if (!btrfs_find_first_extent_bit(unpin, 0, &start, &end, + EXTENT_DIRTY, &cached_state)) { + mutex_unlock(&fs_info->unused_bg_unpin_mutex); + break; + } + + btrfs_clear_extent_dirty(unpin, start, end, &cached_state); + btrfs_free_extent_state(cached_state); + btrfs_error_unpin_extent_range(fs_info, start, end); + mutex_unlock(&fs_info->unused_bg_unpin_mutex); + cond_resched(); + } +} + +static void btrfs_cleanup_bg_io(struct btrfs_block_group *cache) +{ + struct inode *inode; + + inode = cache->io_ctl.inode; + if (inode) { + unsigned int nofs_flag; + + nofs_flag = memalloc_nofs_save(); + invalidate_inode_pages2(inode->i_mapping); + memalloc_nofs_restore(nofs_flag); + + BTRFS_I(inode)->generation = 0; + cache->io_ctl.inode = NULL; + iput(inode); + } + ASSERT(cache->io_ctl.pages == NULL); + btrfs_put_block_group(cache); +} + +void btrfs_cleanup_dirty_bgs(struct btrfs_transaction *cur_trans, + struct btrfs_fs_info *fs_info) +{ + struct btrfs_block_group *cache; + + spin_lock(&cur_trans->dirty_bgs_lock); + while (!list_empty(&cur_trans->dirty_bgs)) { + cache = list_first_entry(&cur_trans->dirty_bgs, + struct btrfs_block_group, + dirty_list); + + if (!list_empty(&cache->io_list)) { + spin_unlock(&cur_trans->dirty_bgs_lock); + list_del_init(&cache->io_list); + btrfs_cleanup_bg_io(cache); + spin_lock(&cur_trans->dirty_bgs_lock); + } + + list_del_init(&cache->dirty_list); + spin_lock(&cache->lock); + cache->disk_cache_state = BTRFS_DC_ERROR; + spin_unlock(&cache->lock); + + spin_unlock(&cur_trans->dirty_bgs_lock); + btrfs_put_block_group(cache); + btrfs_dec_delayed_refs_rsv_bg_updates(fs_info); + spin_lock(&cur_trans->dirty_bgs_lock); + } + spin_unlock(&cur_trans->dirty_bgs_lock); + + /* + * Refer to the definition of io_bgs member for details why it's safe + * to use it without any locking + */ + while (!list_empty(&cur_trans->io_bgs)) { + cache = list_first_entry(&cur_trans->io_bgs, + struct btrfs_block_group, + io_list); + + list_del_init(&cache->io_list); + spin_lock(&cache->lock); + cache->disk_cache_state = BTRFS_DC_ERROR; + spin_unlock(&cache->lock); + btrfs_cleanup_bg_io(cache); + } +} + +static void btrfs_free_all_qgroup_pertrans(struct btrfs_fs_info *fs_info) +{ + struct btrfs_root *gang[8]; + int i; + int ret; + + spin_lock(&fs_info->fs_roots_radix_lock); + while (1) { + ret = radix_tree_gang_lookup_tag(&fs_info->fs_roots_radix, + (void **)gang, 0, + ARRAY_SIZE(gang), + BTRFS_ROOT_TRANS_TAG); + if (ret == 0) + break; + for (i = 0; i < ret; i++) { + struct btrfs_root *root = gang[i]; + + btrfs_qgroup_free_meta_all_pertrans(root); + radix_tree_tag_clear(&fs_info->fs_roots_radix, + (unsigned long)btrfs_root_id(root), + BTRFS_ROOT_TRANS_TAG); + } + } + spin_unlock(&fs_info->fs_roots_radix_lock); +} + +void btrfs_cleanup_one_transaction(struct btrfs_transaction *cur_trans) +{ + struct btrfs_fs_info *fs_info = cur_trans->fs_info; + struct btrfs_device *dev, *tmp; + + btrfs_cleanup_dirty_bgs(cur_trans, fs_info); + ASSERT(list_empty(&cur_trans->dirty_bgs)); + ASSERT(list_empty(&cur_trans->io_bgs)); + + list_for_each_entry_safe(dev, tmp, &cur_trans->dev_update_list, + post_commit_list) { + list_del_init(&dev->post_commit_list); + } + + btrfs_destroy_delayed_refs(cur_trans); + + cur_trans->state = TRANS_STATE_COMMIT_START; + wake_up(&fs_info->transaction_blocked_wait); + + cur_trans->state = TRANS_STATE_UNBLOCKED; + wake_up(&fs_info->transaction_wait); + + btrfs_destroy_marked_extents(fs_info, &cur_trans->dirty_pages, + EXTENT_DIRTY); + btrfs_destroy_pinned_extent(fs_info, &cur_trans->pinned_extents); + + cur_trans->state =TRANS_STATE_COMPLETED; + wake_up(&cur_trans->commit_wait); +} + +static int btrfs_cleanup_transaction(struct btrfs_fs_info *fs_info) +{ + struct btrfs_transaction *t; + + mutex_lock(&fs_info->transaction_kthread_mutex); + + spin_lock(&fs_info->trans_lock); + while (!list_empty(&fs_info->trans_list)) { + t = list_first_entry(&fs_info->trans_list, + struct btrfs_transaction, list); + if (t->state >= TRANS_STATE_COMMIT_PREP) { + refcount_inc(&t->use_count); + spin_unlock(&fs_info->trans_lock); + btrfs_wait_for_commit(fs_info, t->transid); + btrfs_put_transaction(t); + spin_lock(&fs_info->trans_lock); + continue; + } + if (t == fs_info->running_transaction) { + t->state = TRANS_STATE_COMMIT_DOING; + spin_unlock(&fs_info->trans_lock); + /* + * We wait for 0 num_writers since we don't hold a trans + * handle open currently for this transaction. + */ + wait_event(t->writer_wait, + atomic_read(&t->num_writers) == 0); + } else { + spin_unlock(&fs_info->trans_lock); + } + btrfs_cleanup_one_transaction(t); + + spin_lock(&fs_info->trans_lock); + if (t == fs_info->running_transaction) + fs_info->running_transaction = NULL; + list_del_init(&t->list); + spin_unlock(&fs_info->trans_lock); + + btrfs_put_transaction(t); + spin_lock(&fs_info->trans_lock); + } + spin_unlock(&fs_info->trans_lock); + btrfs_destroy_all_ordered_extents(fs_info); + btrfs_destroy_delayed_inodes(fs_info); + btrfs_assert_delayed_root_empty(fs_info); + btrfs_destroy_all_delalloc_inodes(fs_info); + btrfs_drop_all_logs(fs_info); + btrfs_zoned_release_dirty_metadata(fs_info); + btrfs_free_all_qgroup_pertrans(fs_info); + mutex_unlock(&fs_info->transaction_kthread_mutex); + + return 0; +} + +int btrfs_init_root_free_objectid(struct btrfs_root *root) +{ + BTRFS_PATH_AUTO_FREE(path); + int ret; + struct extent_buffer *l; + struct btrfs_key search_key; + struct btrfs_key found_key; + int slot; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + search_key.objectid = BTRFS_LAST_FREE_OBJECTID; + search_key.type = -1; + search_key.offset = (u64)-1; + ret = btrfs_search_slot(NULL, root, &search_key, path, 0, 0); + if (ret < 0) + return ret; + if (unlikely(ret == 0)) { + /* + * Key with offset -1 found, there would have to exist a root + * with such id, but this is out of valid range. + */ + return -EUCLEAN; + } + if (path->slots[0] > 0) { + slot = path->slots[0] - 1; + l = path->nodes[0]; + btrfs_item_key_to_cpu(l, &found_key, slot); + root->free_objectid = max_t(u64, found_key.objectid + 1, + BTRFS_FIRST_FREE_OBJECTID); + } else { + root->free_objectid = BTRFS_FIRST_FREE_OBJECTID; + } + + return 0; +} + +int btrfs_get_free_objectid(struct btrfs_root *root, u64 *objectid) +{ + int ret; + mutex_lock(&root->objectid_mutex); + + if (unlikely(root->free_objectid >= BTRFS_LAST_FREE_OBJECTID)) { + btrfs_warn(root->fs_info, + "the objectid of root %llu reaches its highest value", + btrfs_root_id(root)); + ret = -ENOSPC; + goto out; + } + + *objectid = root->free_objectid++; + ret = 0; +out: + mutex_unlock(&root->objectid_mutex); + return ret; +} diff --git a/fs/btrfs/disk-io.h b/fs/btrfs/disk-io.h new file mode 100644 index 000000000..290508894 --- /dev/null +++ b/fs/btrfs/disk-io.h @@ -0,0 +1,133 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#ifndef BTRFS_DISK_IO_H +#define BTRFS_DISK_IO_H + +#include +#include +#include "ctree.h" +#include "bio.h" +#include "ordered-data.h" + +struct block_device; +struct super_block; +struct extent_buffer; +struct btrfs_eb_prealloc; +struct btrfs_device; +struct btrfs_fs_devices; +struct btrfs_fs_info; +struct btrfs_super_block; +struct btrfs_trans_handle; +struct btrfs_tree_parent_check; +struct btrfs_transaction; + +#define BTRFS_SUPER_MIRROR_MAX 3 +#define BTRFS_SUPER_MIRROR_SHIFT 12 + +/* + * Fixed blocksize for all devices, applies to specific ways of reading + * metadata like superblock. Must meet the set_blocksize requirements. + * + * Do not change. + */ +#define BTRFS_BDEV_BLOCKSIZE (4096) + +static inline u64 btrfs_sb_offset(int mirror) +{ + u64 start = SZ_16K; + if (mirror) + return start << (BTRFS_SUPER_MIRROR_SHIFT * mirror); + return BTRFS_SUPER_INFO_OFFSET; +} + +void btrfs_check_leaked_roots(const struct btrfs_fs_info *fs_info); +void btrfs_init_fs_info(struct btrfs_fs_info *fs_info); +struct extent_buffer *read_tree_block(struct btrfs_fs_info *fs_info, u64 bytenr, + struct btrfs_tree_parent_check *check); +struct extent_buffer *btrfs_find_create_tree_block( + struct btrfs_fs_info *fs_info, + struct btrfs_eb_prealloc *pa, + u64 bytenr, u64 owner_root, + int level); +int btrfs_start_pre_rw_mount(struct btrfs_fs_info *fs_info); +int btrfs_check_super_csum(struct btrfs_fs_info *fs_info, + const struct btrfs_super_block *disk_sb); +int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_devices); +void __cold close_ctree(struct btrfs_fs_info *fs_info); +int btrfs_validate_super(const struct btrfs_fs_info *fs_info, + const struct btrfs_super_block *sb, int mirror_num); +int btrfs_check_features(struct btrfs_fs_info *fs_info, bool is_rw_mount); +int write_all_supers(struct btrfs_trans_handle *trans); +int btrfs_commit_super(struct btrfs_fs_info *fs_info); +struct btrfs_root *btrfs_read_tree_root(struct btrfs_root *tree_root, + const struct btrfs_key *key); +int btrfs_insert_fs_root(struct btrfs_fs_info *fs_info, + struct btrfs_root *root); +void btrfs_free_fs_roots(struct btrfs_fs_info *fs_info); + +struct btrfs_root *btrfs_get_fs_root(struct btrfs_fs_info *fs_info, + u64 objectid, bool check_ref); +struct btrfs_root *btrfs_get_new_fs_root(struct btrfs_fs_info *fs_info, + u64 objectid, dev_t *anon_dev); +struct btrfs_root *btrfs_get_fs_root_commit_root(struct btrfs_fs_info *fs_info, + struct btrfs_path *path, + u64 objectid); +int btrfs_global_root_insert(struct btrfs_root *root); +void btrfs_global_root_delete(struct btrfs_root *root); +struct btrfs_root *btrfs_global_root(struct btrfs_fs_info *fs_info, + const struct btrfs_key *key); +struct btrfs_root *btrfs_csum_root(struct btrfs_fs_info *fs_info, u64 bytenr); +struct btrfs_root *btrfs_extent_root(struct btrfs_fs_info *fs_info, u64 bytenr); + +void btrfs_free_fs_info(struct btrfs_fs_info *fs_info); +void btrfs_btree_balance_dirty(struct btrfs_fs_info *fs_info); +void btrfs_btree_balance_dirty_nodelay(struct btrfs_fs_info *fs_info); +void btrfs_drop_and_free_fs_root(struct btrfs_fs_info *fs_info, + struct btrfs_root *root); +int btrfs_validate_extent_buffer(struct extent_buffer *eb, + const struct btrfs_tree_parent_check *check); +#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS +struct btrfs_root *btrfs_alloc_dummy_root(struct btrfs_fs_info *fs_info); +#endif + +/* + * This function is used to grab the root, and avoid it is freed when we + * access it. But it doesn't ensure that the tree is not dropped. + */ +static inline struct btrfs_root *btrfs_grab_root(struct btrfs_root *root) +{ + if (!root) + return NULL; + if (refcount_inc_not_zero(&root->refs)) + return root; + return NULL; +} + +void btrfs_put_root(struct btrfs_root *root); +void btrfs_mark_buffer_dirty(struct btrfs_trans_handle *trans, + struct extent_buffer *buf); +int btrfs_buffer_uptodate(struct extent_buffer *buf, u64 parent_transid, + const struct btrfs_tree_parent_check *check); +int btrfs_read_extent_buffer(struct extent_buffer *buf, + const struct btrfs_tree_parent_check *check); + +int btree_csum_one_bio(struct btrfs_bio *bbio); +int btrfs_alloc_log_tree_node(struct btrfs_trans_handle *trans, + struct btrfs_root *root); +int btrfs_init_log_root_tree(struct btrfs_trans_handle *trans, + struct btrfs_fs_info *fs_info); +int btrfs_add_log_tree(struct btrfs_trans_handle *trans, + struct btrfs_root *root); +void btrfs_cleanup_dirty_bgs(struct btrfs_transaction *trans, + struct btrfs_fs_info *fs_info); +void btrfs_cleanup_one_transaction(struct btrfs_transaction *trans); +struct btrfs_root *btrfs_create_tree(struct btrfs_trans_handle *trans, + u64 objectid); +int btrfs_get_num_tolerated_disk_barrier_failures(u64 flags); +int btrfs_get_free_objectid(struct btrfs_root *root, u64 *objectid); +int btrfs_init_root_free_objectid(struct btrfs_root *root); + +#endif diff --git a/fs/btrfs/export.c b/fs/btrfs/export.c new file mode 100644 index 000000000..230d9326b --- /dev/null +++ b/fs/btrfs/export.c @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include "ctree.h" +#include "disk-io.h" +#include "btrfs_inode.h" +#include "export.h" +#include "accessors.h" +#include "super.h" + +#define BTRFS_FID_SIZE_NON_CONNECTABLE (offsetof(struct btrfs_fid, \ + parent_objectid) / 4) +#define BTRFS_FID_SIZE_CONNECTABLE (offsetof(struct btrfs_fid, \ + parent_root_objectid) / 4) +#define BTRFS_FID_SIZE_CONNECTABLE_ROOT (sizeof(struct btrfs_fid) / 4) + +static int btrfs_encode_fh(struct inode *inode, u32 *fh, int *max_len, + struct inode *parent) +{ + struct btrfs_fid *fid = (struct btrfs_fid *)fh; + int len = *max_len; + int type; + + if (parent && (len < BTRFS_FID_SIZE_CONNECTABLE)) { + if (btrfs_root_id(BTRFS_I(inode)->root) != + btrfs_root_id(BTRFS_I(parent)->root)) + *max_len = BTRFS_FID_SIZE_CONNECTABLE_ROOT; + else + *max_len = BTRFS_FID_SIZE_CONNECTABLE; + return FILEID_INVALID; + } else if (len < BTRFS_FID_SIZE_NON_CONNECTABLE) { + *max_len = BTRFS_FID_SIZE_NON_CONNECTABLE; + return FILEID_INVALID; + } + + len = BTRFS_FID_SIZE_NON_CONNECTABLE; + type = FILEID_BTRFS_WITHOUT_PARENT; + + fid->objectid = btrfs_ino(BTRFS_I(inode)); + fid->root_objectid = btrfs_root_id(BTRFS_I(inode)->root); + fid->gen = inode->i_generation; + + if (parent) { + u64 parent_root_id; + + fid->parent_objectid = btrfs_ino(BTRFS_I(parent)); + fid->parent_gen = parent->i_generation; + parent_root_id = btrfs_root_id(BTRFS_I(parent)->root); + + if (parent_root_id != fid->root_objectid) { + if (*max_len < BTRFS_FID_SIZE_CONNECTABLE_ROOT) + return FILEID_INVALID; + fid->parent_root_objectid = parent_root_id; + len = BTRFS_FID_SIZE_CONNECTABLE_ROOT; + type = FILEID_BTRFS_WITH_PARENT_ROOT; + } else { + len = BTRFS_FID_SIZE_CONNECTABLE; + type = FILEID_BTRFS_WITH_PARENT; + } + } + + *max_len = len; + return type; +} + +/* + * Read dentry of inode with @objectid from filesystem root @root_objectid. + * + * @sb: the filesystem super block + * @objectid: inode objectid + * @root_objectid: object id of the subvolume root where to look up the inode + * @generation: optional, if not zero, verify that the found inode + * generation matches + * + * Return dentry alias for the inode, otherwise an error. In case the + * generation does not match return ESTALE. + */ +struct dentry *btrfs_get_dentry(struct super_block *sb, u64 objectid, + u64 root_objectid, u64 generation) +{ + struct btrfs_fs_info *fs_info = btrfs_sb(sb); + struct btrfs_root *root; + struct btrfs_inode *inode; + + if (objectid < BTRFS_FIRST_FREE_OBJECTID) + return ERR_PTR(-ESTALE); + + root = btrfs_get_fs_root(fs_info, root_objectid, true); + if (IS_ERR(root)) + return ERR_CAST(root); + + inode = btrfs_iget(objectid, root); + btrfs_put_root(root); + if (IS_ERR(inode)) + return ERR_CAST(inode); + + if (generation != 0 && generation != inode->vfs_inode.i_generation) { + iput(&inode->vfs_inode); + return ERR_PTR(-ESTALE); + } + + return d_obtain_alias(&inode->vfs_inode); +} + +static struct dentry *btrfs_fh_to_parent(struct super_block *sb, struct fid *fh, + int fh_len, int fh_type) +{ + struct btrfs_fid *fid = (struct btrfs_fid *) fh; + u64 objectid, root_objectid; + u32 generation; + + if (fh_type == FILEID_BTRFS_WITH_PARENT) { + if (fh_len < BTRFS_FID_SIZE_CONNECTABLE) + return NULL; + root_objectid = fid->root_objectid; + } else if (fh_type == FILEID_BTRFS_WITH_PARENT_ROOT) { + if (fh_len < BTRFS_FID_SIZE_CONNECTABLE_ROOT) + return NULL; + root_objectid = fid->parent_root_objectid; + } else + return NULL; + + objectid = fid->parent_objectid; + generation = fid->parent_gen; + + return btrfs_get_dentry(sb, objectid, root_objectid, generation); +} + +static struct dentry *btrfs_fh_to_dentry(struct super_block *sb, struct fid *fh, + int fh_len, int fh_type) +{ + struct btrfs_fid *fid = (struct btrfs_fid *) fh; + u64 objectid, root_objectid; + u32 generation; + + if ((fh_type != FILEID_BTRFS_WITH_PARENT || + fh_len < BTRFS_FID_SIZE_CONNECTABLE) && + (fh_type != FILEID_BTRFS_WITH_PARENT_ROOT || + fh_len < BTRFS_FID_SIZE_CONNECTABLE_ROOT) && + (fh_type != FILEID_BTRFS_WITHOUT_PARENT || + fh_len < BTRFS_FID_SIZE_NON_CONNECTABLE)) + return NULL; + + objectid = fid->objectid; + root_objectid = fid->root_objectid; + generation = fid->gen; + + return btrfs_get_dentry(sb, objectid, root_objectid, generation); +} + +struct dentry *btrfs_get_parent(struct dentry *child) +{ + struct btrfs_inode *dir = BTRFS_I(d_inode(child)); + struct btrfs_inode *inode; + struct btrfs_root *root = dir->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_path *path; + struct extent_buffer *leaf; + struct btrfs_root_ref *ref; + struct btrfs_key key; + struct btrfs_key found_key; + int ret; + + path = btrfs_alloc_path(); + if (!path) + return ERR_PTR(-ENOMEM); + + if (btrfs_ino(dir) == BTRFS_FIRST_FREE_OBJECTID) { + key.objectid = btrfs_root_id(root); + key.type = BTRFS_ROOT_BACKREF_KEY; + key.offset = (u64)-1; + root = fs_info->tree_root; + } else { + key.objectid = btrfs_ino(dir); + key.type = BTRFS_INODE_REF_KEY; + key.offset = (u64)-1; + } + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + goto fail; + if (unlikely(ret == 0)) { + /* + * Key with offset of -1 found, there would have to exist an + * inode with such number or a root with such id. + */ + ret = -EUCLEAN; + goto fail; + } + + if (path->slots[0] == 0) { + ret = -ENOENT; + goto fail; + } + + path->slots[0]--; + leaf = path->nodes[0]; + + btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); + if (found_key.objectid != key.objectid || found_key.type != key.type) { + ret = -ENOENT; + goto fail; + } + + if (found_key.type == BTRFS_ROOT_BACKREF_KEY) { + ref = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_root_ref); + key.objectid = btrfs_root_ref_dirid(leaf, ref); + } else { + key.objectid = found_key.offset; + } + btrfs_free_path(path); + + if (found_key.type == BTRFS_ROOT_BACKREF_KEY) { + return btrfs_get_dentry(fs_info->sb, key.objectid, + found_key.offset, 0); + } + + inode = btrfs_iget(key.objectid, root); + if (IS_ERR(inode)) + return ERR_CAST(inode); + + return d_obtain_alias(&inode->vfs_inode); +fail: + btrfs_free_path(path); + return ERR_PTR(ret); +} + +static int btrfs_get_name(struct dentry *parent, char *name, + struct dentry *child) +{ + struct btrfs_inode *inode = BTRFS_I(d_inode(child)); + struct btrfs_inode *dir = BTRFS_I(d_inode(parent)); + struct btrfs_root *root = dir->root; + struct btrfs_fs_info *fs_info = root->fs_info; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_inode_ref *iref; + struct btrfs_root_ref *rref; + struct extent_buffer *leaf; + unsigned long name_ptr; + struct btrfs_key key; + int name_len; + int ret; + u64 ino; + + if (!S_ISDIR(dir->vfs_inode.i_mode)) + return -EINVAL; + + ino = btrfs_ino(inode); + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + if (ino == BTRFS_FIRST_FREE_OBJECTID) { + key.objectid = btrfs_root_id(inode->root); + key.type = BTRFS_ROOT_BACKREF_KEY; + key.offset = (u64)-1; + root = fs_info->tree_root; + } else { + key.objectid = ino; + key.type = BTRFS_INODE_REF_KEY; + key.offset = btrfs_ino(dir); + } + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) { + return ret; + } else if (ret > 0) { + if (ino == BTRFS_FIRST_FREE_OBJECTID) + path->slots[0]--; + else + return -ENOENT; + } + leaf = path->nodes[0]; + + if (ino == BTRFS_FIRST_FREE_OBJECTID) { + rref = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_root_ref); + name_ptr = (unsigned long)(rref + 1); + name_len = btrfs_root_ref_name_len(leaf, rref); + } else { + iref = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_inode_ref); + name_ptr = (unsigned long)(iref + 1); + name_len = btrfs_inode_ref_name_len(leaf, iref); + } + + read_extent_buffer(leaf, name, name_ptr, name_len); + + /* + * have to add the null termination to make sure that reconnect_path + * gets the right len for strlen + */ + name[name_len] = '\0'; + + return 0; +} + +const struct export_operations btrfs_export_ops = { + .encode_fh = btrfs_encode_fh, + .fh_to_dentry = btrfs_fh_to_dentry, + .fh_to_parent = btrfs_fh_to_parent, + .get_parent = btrfs_get_parent, + .get_name = btrfs_get_name, +}; diff --git a/fs/btrfs/export.h b/fs/btrfs/export.h new file mode 100644 index 000000000..464582273 --- /dev/null +++ b/fs/btrfs/export.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_EXPORT_H +#define BTRFS_EXPORT_H + +#include +#include + +struct dentry; +struct super_block; + +extern const struct export_operations btrfs_export_ops; + +struct btrfs_fid { + u64 objectid; + u64 root_objectid; + u32 gen; + + u64 parent_objectid; + u32 parent_gen; + + u64 parent_root_objectid; +} __attribute__ ((packed)); + +struct dentry *btrfs_get_dentry(struct super_block *sb, u64 objectid, + u64 root_objectid, u64 generation); +struct dentry *btrfs_get_parent(struct dentry *child); + +#endif diff --git a/fs/btrfs/extent-io-tree.c b/fs/btrfs/extent-io-tree.c new file mode 100644 index 000000000..d6df11f60 --- /dev/null +++ b/fs/btrfs/extent-io-tree.c @@ -0,0 +1,2071 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include "messages.h" +#include "ctree.h" +#include "extent_io.h" +#include "extent-io-tree.h" +#include "btrfs_inode.h" + +static struct kmem_cache *extent_state_cache; + +static inline bool extent_state_in_tree(const struct extent_state *state) +{ + return !RB_EMPTY_NODE(&state->rb_node); +} + +#ifdef CONFIG_BTRFS_DEBUG +static LIST_HEAD(states); +static DEFINE_SPINLOCK(leak_lock); + +static inline void btrfs_leak_debug_add_state(struct extent_state *state) +{ + unsigned long flags; + + spin_lock_irqsave(&leak_lock, flags); + list_add(&state->leak_list, &states); + spin_unlock_irqrestore(&leak_lock, flags); +} + +static inline void btrfs_leak_debug_del_state(struct extent_state *state) +{ + unsigned long flags; + + spin_lock_irqsave(&leak_lock, flags); + list_del(&state->leak_list); + spin_unlock_irqrestore(&leak_lock, flags); +} + +static inline void btrfs_extent_state_leak_debug_check(void) +{ + struct extent_state *state; + + while (!list_empty(&states)) { + state = list_first_entry(&states, struct extent_state, leak_list); + btrfs_err(NULL, + "state leak: start %llu end %llu state %u in tree %d refs %d", + state->start, state->end, state->state, + extent_state_in_tree(state), + refcount_read(&state->refs)); + list_del(&state->leak_list); + WARN_ON_ONCE(1); + kmem_cache_free(extent_state_cache, state); + } +} + +#define btrfs_debug_check_extent_io_range(tree, start, end) \ + __btrfs_debug_check_extent_io_range(__func__, (tree), (start), (end)) +static inline void __btrfs_debug_check_extent_io_range(const char *caller, + struct extent_io_tree *tree, + u64 start, u64 end) +{ + const struct btrfs_inode *inode = tree->inode; + u64 isize; + + if (tree->owner != IO_TREE_INODE_IO) + return; + + isize = i_size_read(&inode->vfs_inode); + if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) { + btrfs_debug_rl(inode->root->fs_info, + "%s: ino %llu isize %llu odd range [%llu,%llu]", + caller, btrfs_ino(inode), isize, start, end); + } +} +#else +#define btrfs_leak_debug_add_state(state) do {} while (0) +#define btrfs_leak_debug_del_state(state) do {} while (0) +#define btrfs_extent_state_leak_debug_check() do {} while (0) +#define btrfs_debug_check_extent_io_range(c, s, e) do {} while (0) +#endif + +/* Read-only access to the inode. */ +const struct btrfs_inode *btrfs_extent_io_tree_to_inode(const struct extent_io_tree *tree) +{ + if (tree->owner == IO_TREE_INODE_IO) + return tree->inode; + return NULL; +} + +/* For read-only access to fs_info. */ +const struct btrfs_fs_info *btrfs_extent_io_tree_to_fs_info(const struct extent_io_tree *tree) +{ + if (tree->owner == IO_TREE_INODE_IO) + return tree->inode->root->fs_info; + return tree->fs_info; +} + +void btrfs_extent_io_tree_init(struct btrfs_fs_info *fs_info, + struct extent_io_tree *tree, unsigned int owner) +{ + tree->state = RB_ROOT; + spin_lock_init(&tree->lock); + tree->fs_info = fs_info; + tree->owner = owner; +} + +/* + * Empty an io tree, removing and freeing every extent state record from the + * tree. This should be called once we are sure no other task can access the + * tree anymore, so no tree updates happen after we empty the tree and there + * aren't any waiters on any extent state record (EXTENT_LOCK_BITS are never + * set on any extent state when calling this function). + */ +void btrfs_extent_io_tree_release(struct extent_io_tree *tree) +{ + struct rb_root root; + struct extent_state *state; + struct extent_state *tmp; + + spin_lock(&tree->lock); + root = tree->state; + tree->state = RB_ROOT; + rbtree_postorder_for_each_entry_safe(state, tmp, &root, rb_node) { + /* Clear node to keep free_extent_state() happy. */ + RB_CLEAR_NODE(&state->rb_node); + ASSERT(!(state->state & EXTENT_LOCK_BITS)); + /* + * No need for a memory barrier here, as we are holding the tree + * lock and we only change the waitqueue while holding that lock + * (see wait_extent_bit()). + */ + ASSERT(!waitqueue_active(&state->wq)); + btrfs_free_extent_state(state); + cond_resched_lock(&tree->lock); + } + /* + * Should still be empty even after a reschedule, no other task should + * be accessing the tree anymore. + */ + ASSERT(RB_EMPTY_ROOT(&tree->state)); + spin_unlock(&tree->lock); +} + +static struct extent_state *alloc_extent_state(gfp_t mask) +{ + struct extent_state *state; + + /* + * The given mask might be not appropriate for the slab allocator, + * drop the unsupported bits + */ + mask &= ~(__GFP_DMA32|__GFP_HIGHMEM); + state = kmem_cache_alloc(extent_state_cache, mask); + if (!state) + return state; + state->state = 0; + RB_CLEAR_NODE(&state->rb_node); + btrfs_leak_debug_add_state(state); + refcount_set(&state->refs, 1); + init_waitqueue_head(&state->wq); + trace_btrfs_alloc_extent_state(state, mask, _RET_IP_); + return state; +} + +static struct extent_state *alloc_extent_state_atomic(struct extent_state *prealloc) +{ + if (!prealloc) + prealloc = alloc_extent_state(GFP_ATOMIC); + + return prealloc; +} + +void btrfs_free_extent_state(struct extent_state *state) +{ + if (!state) + return; + if (refcount_dec_and_test(&state->refs)) { + WARN_ON(extent_state_in_tree(state)); + btrfs_leak_debug_del_state(state); + trace_btrfs_free_extent_state(state, _RET_IP_); + kmem_cache_free(extent_state_cache, state); + } +} + +static int add_extent_changeset(struct extent_state *state, u32 bits, + struct extent_changeset *changeset, + bool set) +{ + int ret; + + if (!changeset) + return 0; + if (set && (state->state & bits) == bits) + return 0; + if (!set && (state->state & bits) == 0) + return 0; + + changeset->bytes_changed += state->end - state->start + 1; + if (!extent_changeset_tracks_ranges(changeset)) + return 0; + + ret = ulist_add(&changeset->range_changed, state->start, state->end, GFP_ATOMIC); + if (ret < 0) + return ret; + return 0; +} + +static inline struct extent_state *next_state(struct extent_state *state) +{ + struct rb_node *next = rb_next(&state->rb_node); + + return rb_entry_safe(next, struct extent_state, rb_node); +} + +static inline struct extent_state *prev_state(struct extent_state *state) +{ + struct rb_node *next = rb_prev(&state->rb_node); + + return rb_entry_safe(next, struct extent_state, rb_node); +} + +/* + * Search @tree for an entry that contains @offset or if none exists for the + * first entry that starts and ends after that offset. + * + * @tree: the tree to search + * @offset: search offset + * @node_ret: pointer where new node should be anchored (used when inserting an + * entry in the tree) + * @parent_ret: points to entry which would have been the parent of the entry, + * containing @offset + * + * Return a pointer to the entry that contains @offset byte address. + * + * If no such entry exists, return the first entry that starts and ends after + * @offset if one exists, otherwise NULL. + * + * If the returned entry starts at @offset, then @node_ret and @parent_ret + * aren't changed. + */ +static inline struct extent_state *tree_search_for_insert(struct extent_io_tree *tree, + u64 offset, + struct rb_node ***node_ret, + struct rb_node **parent_ret) +{ + struct rb_root *root = &tree->state; + struct rb_node **node = &root->rb_node; + struct rb_node *prev = NULL; + struct extent_state *entry = NULL; + + while (*node) { + prev = *node; + entry = rb_entry(prev, struct extent_state, rb_node); + + if (offset < entry->start) + node = &(*node)->rb_left; + else if (offset > entry->end) + node = &(*node)->rb_right; + else + return entry; + } + + if (node_ret) + *node_ret = node; + if (parent_ret) + *parent_ret = prev; + + /* + * Return either the current entry if it contains offset (it ends after + * or at offset) or the first entry that starts and ends after offset if + * one exists, or NULL. + */ + while (entry && offset > entry->end) + entry = next_state(entry); + + return entry; +} + +/* + * Search offset in the tree or fill neighbor rbtree node pointers. + * + * @tree: the tree to search + * @offset: offset that should fall within an entry in @tree + * @next_ret: pointer to the first entry whose range ends after @offset + * @prev_ret: pointer to the first entry whose range begins before @offset + * + * Return a pointer to the entry that contains @offset byte address. If no + * such entry exists, then return NULL and fill @prev_ret and @next_ret. + * Otherwise return the found entry and other pointers are left untouched. + */ +static struct extent_state *tree_search_prev_next(struct extent_io_tree *tree, + u64 offset, + struct extent_state **prev_ret, + struct extent_state **next_ret) +{ + struct rb_root *root = &tree->state; + struct rb_node **node = &root->rb_node; + struct extent_state *orig_prev; + struct extent_state *entry = NULL; + + ASSERT(prev_ret); + ASSERT(next_ret); + + while (*node) { + entry = rb_entry(*node, struct extent_state, rb_node); + + if (offset < entry->start) + node = &(*node)->rb_left; + else if (offset > entry->end) + node = &(*node)->rb_right; + else + return entry; + } + + orig_prev = entry; + while (entry && offset > entry->end) + entry = next_state(entry); + *next_ret = entry; + entry = orig_prev; + + while (entry && offset < entry->start) + entry = prev_state(entry); + *prev_ret = entry; + + return NULL; +} + +/* + * Inexact rb-tree search, return the next entry if @offset is not found + */ +static inline struct extent_state *tree_search(struct extent_io_tree *tree, u64 offset) +{ + return tree_search_for_insert(tree, offset, NULL, NULL); +} + +static void validate_extent_state(const struct extent_io_tree *tree, + const struct extent_state *state) +{ + u32 blocksize; + + if (tree->owner != IO_TREE_INODE_IO) + return; + + blocksize = btrfs_extent_io_tree_to_fs_info(tree)->sectorsize; + ASSERT(IS_ALIGNED(state->start, blocksize) && + IS_ALIGNED(state->end + 1, blocksize), + "unaligned extent state, blocksize=%u start=%llu end=%llu state=0x%x", + blocksize, state->start, state->end, state->state); +} + +#define extent_io_tree_panic(tree, state, opname, err) \ + btrfs_panic(btrfs_extent_io_tree_to_fs_info((tree)), (err), \ + "extent io tree error on %s state start %llu end %llu", \ + (opname), (state)->start, (state)->end) + +static void merge_prev_state(struct extent_io_tree *tree, struct extent_state *state) +{ + struct extent_state *prev; + + prev = prev_state(state); + if (prev && prev->end == state->start - 1 && prev->state == state->state) { + if (tree->owner == IO_TREE_INODE_IO) + btrfs_merge_delalloc_extent(tree->inode, state, prev); + state->start = prev->start; + rb_erase(&prev->rb_node, &tree->state); + RB_CLEAR_NODE(&prev->rb_node); + btrfs_free_extent_state(prev); + } +} + +static void merge_next_state(struct extent_io_tree *tree, struct extent_state *state) +{ + struct extent_state *next; + + next = next_state(state); + if (next && next->start == state->end + 1 && next->state == state->state) { + if (tree->owner == IO_TREE_INODE_IO) + btrfs_merge_delalloc_extent(tree->inode, state, next); + state->end = next->end; + rb_erase(&next->rb_node, &tree->state); + RB_CLEAR_NODE(&next->rb_node); + btrfs_free_extent_state(next); + } +} + +/* + * Utility function to look for merge candidates inside a given range. Any + * extents with matching state are merged together into a single extent in the + * tree. Extents with EXTENT_IO in their state field are not merged because + * the end_io handlers need to be able to do operations on them without + * sleeping (or doing allocations/splits). + * + * This should be called with the tree lock held. + */ +static void merge_state(struct extent_io_tree *tree, struct extent_state *state) +{ + if (state->state & (EXTENT_LOCK_BITS | EXTENT_BOUNDARY)) + return; + + merge_prev_state(tree, state); + merge_next_state(tree, state); +} + +static void set_state_bits(struct extent_io_tree *tree, + struct extent_state *state, + u32 bits, struct extent_changeset *changeset) +{ + u32 bits_to_set = bits & ~EXTENT_CTLBITS; + int ret; + + if (tree->owner == IO_TREE_INODE_IO) + btrfs_set_delalloc_extent(tree->inode, state, bits); + + ret = add_extent_changeset(state, bits_to_set, changeset, true); + if (unlikely(ret)) + extent_io_tree_panic(tree, state, "add_extent_changeset", ret); + state->state |= bits_to_set; +} + +/* + * Insert an extent_state struct into the tree. 'bits' are set on the + * struct before it is inserted. + * + * Returns a pointer to the struct extent_state record containing the range + * requested for insertion, which may be the same as the given struct or it + * may be an existing record in the tree that was expanded to accommodate the + * requested range. In case of an extent_state different from the one that was + * given, the later can be freed or reused by the caller. + * + * On error it returns an error pointer. + * + * The tree lock is not taken internally. This is a utility function and + * probably isn't what you want to call (see set/clear_extent_bit). + */ +static struct extent_state *insert_state(struct extent_io_tree *tree, + struct extent_state *state, + u32 bits, + struct extent_changeset *changeset) +{ + struct rb_node **node; + struct rb_node *parent = NULL; + const u64 start = state->start - 1; + const u64 end = state->end + 1; + const bool try_merge = !(bits & (EXTENT_LOCK_BITS | EXTENT_BOUNDARY)); + + validate_extent_state(tree, state); + + set_state_bits(tree, state, bits, changeset); + + node = &tree->state.rb_node; + while (*node) { + struct extent_state *entry; + + parent = *node; + entry = rb_entry(parent, struct extent_state, rb_node); + + if (state->end < entry->start) { + if (try_merge && end == entry->start && + state->state == entry->state) { + if (tree->owner == IO_TREE_INODE_IO) + btrfs_merge_delalloc_extent(tree->inode, + state, entry); + entry->start = state->start; + merge_prev_state(tree, entry); + state->state = 0; + return entry; + } + node = &(*node)->rb_left; + } else if (state->end > entry->end) { + if (try_merge && entry->end == start && + state->state == entry->state) { + if (tree->owner == IO_TREE_INODE_IO) + btrfs_merge_delalloc_extent(tree->inode, + state, entry); + entry->end = state->end; + merge_next_state(tree, entry); + state->state = 0; + return entry; + } + node = &(*node)->rb_right; + } else { + return ERR_PTR(-EEXIST); + } + } + + rb_link_node(&state->rb_node, parent, node); + rb_insert_color(&state->rb_node, &tree->state); + + return state; +} + +/* + * Insert state to @tree to the location given by @node and @parent. + */ +static void insert_state_fast(struct extent_io_tree *tree, + struct extent_state *state, struct rb_node **node, + struct rb_node *parent, unsigned bits, + struct extent_changeset *changeset) +{ + validate_extent_state(tree, state); + + set_state_bits(tree, state, bits, changeset); + rb_link_node(&state->rb_node, parent, node); + rb_insert_color(&state->rb_node, &tree->state); + merge_state(tree, state); +} + +/* + * Split a given extent state struct in two, inserting the preallocated + * struct 'prealloc' as the newly created second half. 'split' indicates an + * offset inside 'orig' where it should be split. + * + * Before calling, + * the tree has 'orig' at [orig->start, orig->end]. After calling, there + * are two extent state structs in the tree: + * prealloc: [orig->start, split - 1] + * orig: [ split, orig->end ] + * + * The tree locks are not taken by this function. They need to be held + * by the caller. + */ +static int split_state(struct extent_io_tree *tree, struct extent_state *orig, + struct extent_state *prealloc, u64 split) +{ + struct rb_node *parent = NULL; + struct rb_node **node; + + if (tree->owner == IO_TREE_INODE_IO) + btrfs_split_delalloc_extent(tree->inode, orig, split); + + prealloc->start = orig->start; + prealloc->end = split - 1; + prealloc->state = orig->state; + orig->start = split; + + parent = &orig->rb_node; + node = &parent; + while (*node) { + struct extent_state *entry; + + parent = *node; + entry = rb_entry(parent, struct extent_state, rb_node); + + if (prealloc->end < entry->start) { + node = &(*node)->rb_left; + } else if (prealloc->end > entry->end) { + node = &(*node)->rb_right; + } else { + btrfs_free_extent_state(prealloc); + return -EEXIST; + } + } + + validate_extent_state(tree, orig); + validate_extent_state(tree, prealloc); + rb_link_node(&prealloc->rb_node, parent, node); + rb_insert_color(&prealloc->rb_node, &tree->state); + + return 0; +} + +static inline void state_wake_up(struct extent_io_tree *tree, + struct extent_state *state, u32 bits) +{ + lockdep_assert_held(&tree->lock); + + if (!(bits & EXTENT_LOCK_BITS)) + return; + + /* + * No memory barriers because the tree's lock is held while: + * + * 1) Adding waiters to the queue. + * 2) Waking up waiters. + * 3) Removing waiters from queue. + */ + cond_wake_up_nomb(&state->wq); +} + +/* + * Use this during tree iteration to avoid doing next node searches when it's + * not needed (the current record ends at or after the target range's end). + */ +static inline struct extent_state *next_search_state(struct extent_state *state, u64 end) +{ + if (state->end < end) + return next_state(state); + + return NULL; +} + +/* + * Utility function to clear some bits in an extent state struct. It will + * optionally wake up anyone waiting on this state. + * + * If no bits are set on the state struct after clearing things, the + * struct is freed and removed from the tree + */ +static struct extent_state *clear_state_bit(struct extent_io_tree *tree, + struct extent_state *state, + u32 bits, u64 end, + struct extent_changeset *changeset) +{ + struct extent_state *next; + u32 bits_to_clear = bits & ~EXTENT_CTLBITS; + int ret; + + if (tree->owner == IO_TREE_INODE_IO) + btrfs_clear_delalloc_extent(tree->inode, state, bits); + + ret = add_extent_changeset(state, bits_to_clear, changeset, false); + if (unlikely(ret)) + extent_io_tree_panic(tree, state, "add_extent_changeset", ret); + state->state &= ~bits_to_clear; + state_wake_up(tree, state, bits); + if (state->state == 0) { + if (unlikely(!extent_state_in_tree(state))) + extent_io_tree_panic(tree, state, "extent_state_in_tree", -EUCLEAN); + + next = next_search_state(state, end); + rb_erase(&state->rb_node, &tree->state); + RB_CLEAR_NODE(&state->rb_node); + btrfs_free_extent_state(state); + } else { + merge_state(tree, state); + next = next_search_state(state, end); + } + return next; +} + +/* + * Detect if extent bits request NOWAIT semantics and set the gfp mask accordingly, + * unset the EXTENT_NOWAIT bit. + */ +static void set_gfp_mask_from_bits(u32 *bits, gfp_t *mask) +{ + *mask = (*bits & EXTENT_NOWAIT ? GFP_NOWAIT : GFP_NOFS); + *bits &= EXTENT_NOWAIT - 1; +} + +/* + * Clear some bits on a range in the tree. This may require splitting or + * inserting elements in the tree, so the gfp mask is used to indicate which + * allocations or sleeping are allowed. + * + * The range [start, end] is inclusive. + * + * This takes the tree lock, and returns 0 on success and < 0 on error. + */ +int btrfs_clear_extent_bit_changeset(struct extent_io_tree *tree, u64 start, u64 end, + u32 bits, struct extent_state **cached_state, + struct extent_changeset *changeset) +{ + struct extent_state *state; + struct extent_state *cached; + struct extent_state *prealloc = NULL; + u64 last_end; + int ret = 0; + bool clear; + const bool delete = (bits & EXTENT_CLEAR_ALL_BITS); + const u32 bits_to_clear = (bits & ~EXTENT_CTLBITS); + gfp_t mask; + + set_gfp_mask_from_bits(&bits, &mask); + btrfs_debug_check_extent_io_range(tree, start, end); + trace_btrfs_clear_extent_bit(tree, start, end - start + 1, bits); + + if (delete) + bits |= ~EXTENT_CTLBITS; + + if (bits & EXTENT_DELALLOC) + bits |= EXTENT_NORESERVE; + + clear = (bits & (EXTENT_LOCK_BITS | EXTENT_BOUNDARY)); +again: + if (!prealloc) { + /* + * Don't care for allocation failure here because we might end + * up not needing the pre-allocated extent state at all, which + * is the case if we only have in the tree extent states that + * cover our input range and don't cover too any other range. + * If we end up needing a new extent state we allocate it later. + */ + prealloc = alloc_extent_state(mask); + } + + spin_lock(&tree->lock); + if (cached_state) { + cached = *cached_state; + + if (clear) { + *cached_state = NULL; + cached_state = NULL; + } + + if (cached && extent_state_in_tree(cached) && + cached->start <= start && cached->end > start) { + if (clear) + refcount_dec(&cached->refs); + state = cached; + goto hit_next; + } + if (clear) + btrfs_free_extent_state(cached); + } + + /* This search will find the extents that end after our range starts. */ + state = tree_search(tree, start); + if (!state) + goto out; +hit_next: + if (state->start > end) + goto out; + WARN_ON(state->end < start); + last_end = state->end; + + /* The state doesn't have the wanted bits, go ahead. */ + if (!(state->state & bits)) { + state = next_search_state(state, end); + goto next; + } + + /* + * | ---- desired range ---- | + * | state | or + * | ------------- state -------------- | + * + * We need to split the extent we found, and may flip bits on second + * half. + * + * If the extent we found extends past our range, we just split and + * search again. It'll get split again the next time though. + * + * If the extent we found is inside our range, we clear the desired bit + * on it. + */ + + if (state->start < start) { + /* + * If all bits are cleared, there's no point in allocating or + * using the prealloc extent, split the state record, insert the + * prealloc record and then remove this record. We can just + * adjust this record and move on to the next without adding or + * removing anything to the tree. + */ + if (state->end <= end && (state->state & ~bits_to_clear) == 0) { + const u64 orig_start = state->start; + + if (tree->owner == IO_TREE_INODE_IO) + btrfs_split_delalloc_extent(tree->inode, state, start); + + /* + * Temporarilly ajdust this state's range to match the + * range for which we are clearing bits. + */ + state->start = start; + + ret = add_extent_changeset(state, bits_to_clear, changeset, false); + if (unlikely(ret < 0)) { + extent_io_tree_panic(tree, state, + "add_extent_changeset", ret); + goto out; + } + + if (tree->owner == IO_TREE_INODE_IO) + btrfs_clear_delalloc_extent(tree->inode, state, bits); + + /* + * Now adjust the range to the section for which no bits + * are cleared. + */ + state->start = orig_start; + state->end = start - 1; + + state_wake_up(tree, state, bits); + state = next_search_state(state, end); + goto next; + } + + prealloc = alloc_extent_state_atomic(prealloc); + if (!prealloc) + goto search_again; + ret = split_state(tree, state, prealloc, start); + prealloc = NULL; + if (unlikely(ret)) { + extent_io_tree_panic(tree, state, "split", ret); + goto out; + } + if (state->end <= end) { + state = clear_state_bit(tree, state, bits, end, changeset); + goto next; + } + if (need_resched()) + goto search_again; + /* + * Fallthrough and try atomic extent state allocation if needed. + * If it fails we'll jump to 'search_again' retry the allocation + * in non-atomic mode and start the search again. + */ + } + /* + * | ---- desired range ---- | + * | state | + * We need to split the extent, and clear the bit on the first half. + */ + if (state->start <= end && state->end > end) { + /* + * If all bits are cleared, there's no point in allocating or + * using the prealloc extent, split the state record, insert the + * prealloc record and then remove it. We can just adjust the + * start offset of the current state and avoid all that. + */ + if ((state->state & ~bits_to_clear) == 0) { + const u64 orig_end = state->end; + + if (tree->owner == IO_TREE_INODE_IO) + btrfs_split_delalloc_extent(tree->inode, state, end + 1); + + /* + * Temporarily adjust the end offset to match the + * removed subrange to update the changeset. + */ + state->end = end; + + ret = add_extent_changeset(state, bits_to_clear, changeset, false); + if (unlikely(ret < 0)) { + extent_io_tree_panic(tree, state, + "add_extent_changeset", ret); + goto out; + } + + if (tree->owner == IO_TREE_INODE_IO) + btrfs_clear_delalloc_extent(tree->inode, state, bits); + + state->start = end + 1; + state->end = orig_end; + + state_wake_up(tree, state, bits); + goto out; + } + + prealloc = alloc_extent_state_atomic(prealloc); + if (!prealloc) + goto search_again; + ret = split_state(tree, state, prealloc, end + 1); + if (unlikely(ret)) { + extent_io_tree_panic(tree, state, "split", ret); + prealloc = NULL; + goto out; + } + + state_wake_up(tree, state, bits); + + clear_state_bit(tree, prealloc, bits, end, changeset); + + prealloc = NULL; + goto out; + } + + state = clear_state_bit(tree, state, bits, end, changeset); +next: + if (last_end >= end) + goto out; + start = last_end + 1; + if (state && !need_resched()) + goto hit_next; + +search_again: + spin_unlock(&tree->lock); + if (gfpflags_allow_blocking(mask)) + cond_resched(); + goto again; + +out: + spin_unlock(&tree->lock); + btrfs_free_extent_state(prealloc); + + return ret; + +} + +/* + * Wait for one or more bits to clear on a range in the state tree. + * The range [start, end] is inclusive. + * The tree lock is taken by this function + */ +static void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, + u32 bits, struct extent_state **cached_state) +{ + struct extent_state *state; + + btrfs_debug_check_extent_io_range(tree, start, end); + + spin_lock(&tree->lock); +again: + /* + * Maintain cached_state, as we may not remove it from the tree if there + * are more bits than the bits we're waiting on set on this state. + */ + if (cached_state && *cached_state) { + state = *cached_state; + if (extent_state_in_tree(state) && + state->start <= start && start < state->end) + goto process_node; + } + while (1) { + /* + * This search will find all the extents that end after our + * range starts. + */ + state = tree_search(tree, start); +process_node: + if (!state) + break; + if (state->start > end) + goto out; + + if (state->state & bits) { + DEFINE_WAIT(wait); + + start = state->start; + refcount_inc(&state->refs); + prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE); + spin_unlock(&tree->lock); + schedule(); + spin_lock(&tree->lock); + finish_wait(&state->wq, &wait); + btrfs_free_extent_state(state); + goto again; + } + start = state->end + 1; + + if (start > end) + break; + + if (!cond_resched_lock(&tree->lock)) { + state = next_state(state); + goto process_node; + } + } +out: + spin_unlock(&tree->lock); + /* This state is no longer useful, clear it and free it up. */ + if (cached_state && *cached_state) { + state = *cached_state; + *cached_state = NULL; + btrfs_free_extent_state(state); + } +} + +static void cache_state_if_flags(struct extent_state *state, + struct extent_state **cached_ptr, + unsigned flags) +{ + if (cached_ptr && !(*cached_ptr)) { + if (!flags || (state->state & flags)) { + *cached_ptr = state; + refcount_inc(&state->refs); + } + } +} + +static void cache_state(struct extent_state *state, + struct extent_state **cached_ptr) +{ + return cache_state_if_flags(state, cached_ptr, EXTENT_LOCK_BITS | EXTENT_BOUNDARY); +} + +/* + * Find the first state struct with 'bits' set after 'start', and return it. + * tree->lock must be held. NULL will returned if nothing was found after + * 'start'. + */ +static struct extent_state *find_first_extent_bit_state(struct extent_io_tree *tree, + u64 start, u32 bits) +{ + struct extent_state *state; + + /* + * This search will find all the extents that end after our range + * starts. + */ + state = tree_search(tree, start); + while (state) { + if (state->state & bits) + return state; + state = next_state(state); + } + return NULL; +} + +/* + * Find the first offset in the io tree with one or more @bits set. + * + * Note: If there are multiple bits set in @bits, any of them will match. + * + * Return true if we find something, and update @start_ret and @end_ret. + * Return false if we found nothing. + */ +bool btrfs_find_first_extent_bit(struct extent_io_tree *tree, u64 start, + u64 *start_ret, u64 *end_ret, u32 bits, + struct extent_state **cached_state) +{ + struct extent_state *state; + bool ret = false; + + spin_lock(&tree->lock); + if (cached_state && *cached_state) { + state = *cached_state; + if (state->end == start - 1 && extent_state_in_tree(state)) { + while ((state = next_state(state)) != NULL) { + if (state->state & bits) + break; + } + /* + * If we found the next extent state, clear cached_state + * so that we can cache the next extent state below and + * avoid future calls going over the same extent state + * again. If we haven't found any, clear as well since + * it's now useless. + */ + btrfs_free_extent_state(*cached_state); + *cached_state = NULL; + if (state) + goto got_it; + goto out; + } + btrfs_free_extent_state(*cached_state); + *cached_state = NULL; + } + + state = find_first_extent_bit_state(tree, start, bits); +got_it: + if (state) { + cache_state_if_flags(state, cached_state, 0); + *start_ret = state->start; + *end_ret = state->end; + ret = true; + } +out: + spin_unlock(&tree->lock); + return ret; +} + +/* + * Find a contiguous area of bits + * + * @tree: io tree to check + * @start: offset to start the search from + * @start_ret: the first offset we found with the bits set + * @end_ret: the final contiguous range of the bits that were set + * @bits: bits to look for + * + * set_extent_bit and clear_extent_bit can temporarily split contiguous ranges + * to set bits appropriately, and then merge them again. During this time it + * will drop the tree->lock, so use this helper if you want to find the actual + * contiguous area for given bits. We will search to the first bit we find, and + * then walk down the tree until we find a non-contiguous area. The area + * returned will be the full contiguous area with the bits set. + * + * Returns true if we found a range with the given bits set, in which case + * @start_ret and @end_ret are updated, or false if no range was found. + */ +bool btrfs_find_contiguous_extent_bit(struct extent_io_tree *tree, u64 start, + u64 *start_ret, u64 *end_ret, u32 bits) +{ + struct extent_state *state; + bool ret = false; + + ASSERT(!btrfs_fs_incompat(btrfs_extent_io_tree_to_fs_info(tree), NO_HOLES)); + + spin_lock(&tree->lock); + state = find_first_extent_bit_state(tree, start, bits); + if (state) { + *start_ret = state->start; + *end_ret = state->end; + while ((state = next_state(state)) != NULL) { + if (state->start > (*end_ret + 1)) + break; + *end_ret = state->end; + } + ret = true; + } + spin_unlock(&tree->lock); + return ret; +} + +/* + * Find a contiguous range of bytes in the file marked as delalloc, not more + * than 'max_bytes'. start and end are used to return the range, + * + * True is returned if we find something, false if nothing was in the tree. + */ +bool btrfs_find_delalloc_range(struct extent_io_tree *tree, u64 *start, + u64 *end, u64 max_bytes, + struct extent_state **cached_state) +{ + struct extent_state *state; + u64 cur_start = *start; + bool found = false; + u64 total_bytes = 0; + + spin_lock(&tree->lock); + + /* + * This search will find all the extents that end after our range + * starts. + */ + state = tree_search(tree, cur_start); + if (!state) { + *end = (u64)-1; + goto out; + } + + while (state) { + if (found && (state->start != cur_start || + (state->state & EXTENT_BOUNDARY))) { + goto out; + } + if (!(state->state & EXTENT_DELALLOC)) { + if (!found) + *end = state->end; + goto out; + } + if (!found) { + *start = state->start; + *cached_state = state; + refcount_inc(&state->refs); + } + found = true; + *end = state->end; + cur_start = state->end + 1; + total_bytes += state->end - state->start + 1; + if (total_bytes >= max_bytes) + break; + state = next_state(state); + } +out: + spin_unlock(&tree->lock); + return found; +} + +/* + * Set some bits on a range in the tree. This may require allocations or + * sleeping. By default all allocations use GFP_NOFS, use EXTENT_NOWAIT for + * GFP_NOWAIT. + * + * If any of the exclusive bits are set, this will fail with -EEXIST if some + * part of the range already has the desired bits set. The extent_state of the + * existing range is returned in failed_state in this case, and the start of the + * existing range is returned in failed_start. failed_state is used as an + * optimization for wait_extent_bit, failed_start must be used as the source of + * truth as failed_state may have changed since we returned. + * + * [start, end] is inclusive This takes the tree lock. + */ +static int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, + u32 bits, u64 *failed_start, + struct extent_state **failed_state, + struct extent_state **cached_state, + struct extent_changeset *changeset) +{ + struct extent_state *state; + struct extent_state *prealloc = NULL; + struct rb_node **p = NULL; + struct rb_node *parent = NULL; + int ret = 0; + u64 last_start; + u64 last_end; + u32 exclusive_bits = (bits & EXTENT_LOCK_BITS); + gfp_t mask; + + set_gfp_mask_from_bits(&bits, &mask); + btrfs_debug_check_extent_io_range(tree, start, end); + trace_btrfs_set_extent_bit(tree, start, end - start + 1, bits); + + if (exclusive_bits) + ASSERT(failed_start); + else + ASSERT(failed_start == NULL && failed_state == NULL); +again: + if (!prealloc) { + /* + * Don't care for allocation failure here because we might end + * up not needing the pre-allocated extent state at all, which + * is the case if we only have in the tree extent states that + * cover our input range and don't cover too any other range. + * If we end up needing a new extent state we allocate it later. + */ + prealloc = alloc_extent_state(mask); + } + /* Optimistically preallocate the extent changeset ulist node. */ + if (changeset) + extent_changeset_prealloc(changeset, mask); + + spin_lock(&tree->lock); + if (cached_state && *cached_state) { + state = *cached_state; + if (state->start <= start && state->end > start && + extent_state_in_tree(state)) + goto hit_next; + } + /* + * This search will find all the extents that end after our range + * starts. + */ + state = tree_search_for_insert(tree, start, &p, &parent); + if (!state) { + prealloc = alloc_extent_state_atomic(prealloc); + if (!prealloc) + goto search_again; + prealloc->start = start; + prealloc->end = end; + insert_state_fast(tree, prealloc, p, parent, bits, changeset); + cache_state(prealloc, cached_state); + prealloc = NULL; + goto out; + } +hit_next: + last_start = state->start; + last_end = state->end; + + /* + * | ---- desired range ---- | + * | state | + * + * Just lock what we found and keep going + */ + if (state->start == start && state->end <= end) { + if (state->state & exclusive_bits) { + *failed_start = state->start; + cache_state(state, failed_state); + ret = -EEXIST; + goto out; + } + + set_state_bits(tree, state, bits, changeset); + cache_state(state, cached_state); + merge_state(tree, state); + if (last_end >= end) + goto out; + start = last_end + 1; + state = next_state(state); + if (state && state->start == start && !need_resched()) + goto hit_next; + goto search_again; + } + + /* + * | ---- desired range ---- | + * | state | + * or + * | ------------- state -------------- | + * + * We need to split the extent we found, and may flip bits on second + * half. + * + * If the extent we found extends past our range, we just split and + * search again. It'll get split again the next time though. + * + * If the extent we found is inside our range, we set the desired bit + * on it. + */ + if (state->start < start) { + if (state->state & exclusive_bits) { + *failed_start = start; + cache_state(state, failed_state); + ret = -EEXIST; + goto out; + } + + /* + * If this extent already has all the bits we want set, then + * skip it, not necessary to split it or do anything with it. + */ + if ((state->state & bits) == bits) { + start = state->end + 1; + cache_state(state, cached_state); + goto search_again; + } + + prealloc = alloc_extent_state_atomic(prealloc); + if (!prealloc) + goto search_again; + ret = split_state(tree, state, prealloc, start); + if (unlikely(ret)) + extent_io_tree_panic(tree, state, "split", ret); + + prealloc = NULL; + if (ret) + goto out; + if (state->end <= end) { + set_state_bits(tree, state, bits, changeset); + cache_state(state, cached_state); + merge_state(tree, state); + if (last_end >= end) + goto out; + start = last_end + 1; + state = next_state(state); + if (state && state->start == start && !need_resched()) + goto hit_next; + } + goto search_again; + } + /* + * | ---- desired range ---- | + * | state | or | state | + * + * There's a hole, we need to insert something in it and ignore the + * extent we found. + */ + if (state->start > start) { + struct extent_state *inserted_state; + + prealloc = alloc_extent_state_atomic(prealloc); + if (!prealloc) + goto search_again; + + /* + * Avoid to free 'prealloc' if it can be merged with the later + * extent. + */ + prealloc->start = start; + if (end < last_start) + prealloc->end = end; + else + prealloc->end = last_start - 1; + + inserted_state = insert_state(tree, prealloc, bits, changeset); + if (IS_ERR(inserted_state)) { + ret = PTR_ERR(inserted_state); + extent_io_tree_panic(tree, prealloc, "insert", ret); + goto out; + } + + cache_state(inserted_state, cached_state); + if (inserted_state == prealloc) + prealloc = NULL; + start = inserted_state->end + 1; + + /* Beyond target range, stop. */ + if (start > end) + goto out; + + if (need_resched()) + goto search_again; + + state = next_search_state(inserted_state, end); + /* + * If there's a next state, whether contiguous or not, we don't + * need to unlock and start search again. If it's not contiguous + * we will end up here and try to allocate a prealloc state and insert. + */ + if (state) + goto hit_next; + goto search_again; + } + /* + * | ---- desired range ---- | + * | state | + * + * We need to split the extent, and set the bit on the first half + */ + if (state->start <= end && state->end > end) { + if (state->state & exclusive_bits) { + *failed_start = start; + cache_state(state, failed_state); + ret = -EEXIST; + goto out; + } + + prealloc = alloc_extent_state_atomic(prealloc); + if (!prealloc) + goto search_again; + ret = split_state(tree, state, prealloc, end + 1); + if (unlikely(ret)) { + extent_io_tree_panic(tree, state, "split", ret); + prealloc = NULL; + goto out; + } + + set_state_bits(tree, prealloc, bits, changeset); + cache_state(prealloc, cached_state); + merge_state(tree, prealloc); + prealloc = NULL; + goto out; + } + +search_again: + if (start > end) + goto out; + spin_unlock(&tree->lock); + if (gfpflags_allow_blocking(mask)) + cond_resched(); + goto again; + +out: + spin_unlock(&tree->lock); + btrfs_free_extent_state(prealloc); + + return ret; + +} + +int btrfs_set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, + u32 bits, struct extent_state **cached_state) +{ + return set_extent_bit(tree, start, end, bits, NULL, NULL, cached_state, NULL); +} + +/* + * Convert all bits in a given range from one bit to another + * + * @tree: the io tree to search + * @start: the start offset in bytes + * @end: the end offset in bytes (inclusive) + * @bits: the bits to set in this range + * @clear_bits: the bits to clear in this range + * @cached_state: state that we're going to cache + * + * This will go through and set bits for the given range. If any states exist + * already in this range they are set with the given bit and cleared of the + * clear_bits. This is only meant to be used by things that are mergeable, ie. + * converting from say DELALLOC to DIRTY. This is not meant to be used with + * boundary bits like LOCK. + * + * All allocations are done with GFP_NOFS. + */ +int btrfs_convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, + u32 bits, u32 clear_bits, + struct extent_state **cached_state) +{ + struct extent_state *state; + struct extent_state *prealloc = NULL; + struct rb_node **p = NULL; + struct rb_node *parent = NULL; + int ret = 0; + u64 last_start; + u64 last_end; + bool first_iteration = true; + + btrfs_debug_check_extent_io_range(tree, start, end); + trace_btrfs_convert_extent_bit(tree, start, end - start + 1, bits, + clear_bits); + +again: + if (!prealloc) { + /* + * Best effort, don't worry if extent state allocation fails + * here for the first iteration. We might have a cached state + * that matches exactly the target range, in which case no + * extent state allocations are needed. We'll only know this + * after locking the tree. + */ + prealloc = alloc_extent_state(GFP_NOFS); + if (!prealloc && !first_iteration) + return -ENOMEM; + } + + spin_lock(&tree->lock); + if (cached_state && *cached_state) { + state = *cached_state; + if (state->start <= start && state->end > start && + extent_state_in_tree(state)) + goto hit_next; + } + + /* + * This search will find all the extents that end after our range + * starts. + */ + state = tree_search_for_insert(tree, start, &p, &parent); + if (!state) { + prealloc = alloc_extent_state_atomic(prealloc); + if (!prealloc) { + ret = -ENOMEM; + goto out; + } + prealloc->start = start; + prealloc->end = end; + insert_state_fast(tree, prealloc, p, parent, bits, NULL); + cache_state(prealloc, cached_state); + prealloc = NULL; + goto out; + } +hit_next: + last_start = state->start; + last_end = state->end; + + /* + * | ---- desired range ---- | + * | state | + * + * Just lock what we found and keep going. + */ + if (state->start == start && state->end <= end) { + set_state_bits(tree, state, bits, NULL); + cache_state(state, cached_state); + state = clear_state_bit(tree, state, clear_bits, end, NULL); + if (last_end >= end) + goto out; + start = last_end + 1; + if (state && state->start == start && !need_resched()) + goto hit_next; + goto search_again; + } + + /* + * | ---- desired range ---- | + * | state | + * or + * | ------------- state -------------- | + * + * We need to split the extent we found, and may flip bits on second + * half. + * + * If the extent we found extends past our range, we just split and + * search again. It'll get split again the next time though. + * + * If the extent we found is inside our range, we set the desired bit + * on it. + */ + if (state->start < start) { + prealloc = alloc_extent_state_atomic(prealloc); + if (!prealloc) { + ret = -ENOMEM; + goto out; + } + ret = split_state(tree, state, prealloc, start); + prealloc = NULL; + if (unlikely(ret)) { + extent_io_tree_panic(tree, state, "split", ret); + goto out; + } + if (state->end <= end) { + set_state_bits(tree, state, bits, NULL); + cache_state(state, cached_state); + state = clear_state_bit(tree, state, clear_bits, end, NULL); + if (last_end >= end) + goto out; + start = last_end + 1; + if (state && state->start == start && !need_resched()) + goto hit_next; + } + goto search_again; + } + /* + * | ---- desired range ---- | + * | state | or | state | + * + * There's a hole, we need to insert something in it and ignore the + * extent we found. + */ + if (state->start > start) { + struct extent_state *inserted_state; + + prealloc = alloc_extent_state_atomic(prealloc); + if (!prealloc) { + ret = -ENOMEM; + goto out; + } + + /* + * Avoid to free 'prealloc' if it can be merged with the later + * extent. + */ + prealloc->start = start; + if (end < last_start) + prealloc->end = end; + else + prealloc->end = last_start - 1; + + inserted_state = insert_state(tree, prealloc, bits, NULL); + if (IS_ERR(inserted_state)) { + ret = PTR_ERR(inserted_state); + extent_io_tree_panic(tree, prealloc, "insert", ret); + goto out; + } + cache_state(inserted_state, cached_state); + if (inserted_state == prealloc) + prealloc = NULL; + start = inserted_state->end + 1; + + /* Beyond target range, stop. */ + if (start > end) + goto out; + + if (need_resched()) + goto search_again; + + state = next_search_state(inserted_state, end); + /* + * If there's a next state, whether contiguous or not, we don't + * need to unlock and start search again. If it's not contiguous + * we will end up here and try to allocate a prealloc state and insert. + */ + if (state) + goto hit_next; + goto search_again; + } + /* + * | ---- desired range ---- | + * | state | + * + * We need to split the extent, and set the bit on the first half. + */ + if (state->start <= end && state->end > end) { + prealloc = alloc_extent_state_atomic(prealloc); + if (!prealloc) { + ret = -ENOMEM; + goto out; + } + + ret = split_state(tree, state, prealloc, end + 1); + if (unlikely(ret)) { + extent_io_tree_panic(tree, state, "split", ret); + prealloc = NULL; + goto out; + } + + set_state_bits(tree, prealloc, bits, NULL); + cache_state(prealloc, cached_state); + clear_state_bit(tree, prealloc, clear_bits, end, NULL); + prealloc = NULL; + goto out; + } + +search_again: + if (start > end) + goto out; + spin_unlock(&tree->lock); + cond_resched(); + first_iteration = false; + goto again; + +out: + spin_unlock(&tree->lock); + btrfs_free_extent_state(prealloc); + + return ret; +} + +/* + * Find the first range that has @bits not set. This range could start before + * @start. + * + * @tree: the tree to search + * @start: offset at/after which the found extent should start + * @start_ret: records the beginning of the range + * @end_ret: records the end of the range (inclusive) + * @bits: the set of bits which must be unset + * + * Since unallocated range is also considered one which doesn't have the bits + * set it's possible that @end_ret contains -1, this happens in case the range + * spans (last_range_end, end of device]. In this case it's up to the caller to + * trim @end_ret to the appropriate size. + */ +void btrfs_find_first_clear_extent_bit(struct extent_io_tree *tree, u64 start, + u64 *start_ret, u64 *end_ret, u32 bits) +{ + struct extent_state *state; + struct extent_state *prev = NULL, *next = NULL; + + spin_lock(&tree->lock); + + /* Find first extent with bits cleared */ + while (1) { + state = tree_search_prev_next(tree, start, &prev, &next); + if (!state && !next && !prev) { + /* + * Tree is completely empty, send full range and let + * caller deal with it + */ + *start_ret = 0; + *end_ret = -1; + goto out; + } else if (!state && !next) { + /* + * We are past the last allocated chunk, set start at + * the end of the last extent. + */ + *start_ret = prev->end + 1; + *end_ret = -1; + goto out; + } else if (!state) { + state = next; + } + + /* + * At this point 'state' either contains 'start' or start is + * before 'state' + */ + if (in_range(start, state->start, state->end - state->start + 1)) { + if (state->state & bits) { + /* + * |--range with bits sets--| + * | + * start + */ + start = state->end + 1; + } else { + /* + * 'start' falls within a range that doesn't + * have the bits set, so take its start as the + * beginning of the desired range + * + * |--range with bits cleared----| + * | + * start + */ + *start_ret = state->start; + break; + } + } else { + /* + * |---prev range---|---hole/unset---|---node range---| + * | + * start + * + * or + * + * |---hole/unset--||--first node--| + * 0 | + * start + */ + if (prev) + *start_ret = prev->end + 1; + else + *start_ret = 0; + break; + } + } + + /* + * Find the longest stretch from start until an entry which has the + * bits set + */ + while (state) { + if (state->end >= start && !(state->state & bits)) { + *end_ret = state->end; + } else { + *end_ret = state->start - 1; + break; + } + state = next_state(state); + } +out: + spin_unlock(&tree->lock); +} + +/* + * Count the number of bytes in the tree that have a given bit(s) set for a + * given range. + * + * @tree: The io tree to search. + * @start: The start offset of the range. This value is updated to the + * offset of the first byte found with the given bit(s), so it + * can end up being bigger than the initial value. + * @search_end: The end offset (inclusive value) of the search range. + * @max_bytes: The maximum byte count we are interested. The search stops + * once it reaches this count. + * @bits: The bits the range must have in order to be accounted for. + * If multiple bits are set, then only subranges that have all + * the bits set are accounted for. + * @contig: Indicate if we should ignore holes in the range or not. If + * this is true, then stop once we find a hole. + * @cached_state: A cached state to be used across multiple calls to this + * function in order to speedup searches. Use NULL if this is + * called only once or if each call does not start where the + * previous one ended. + * + * Returns the total number of bytes found within the given range that have + * all given bits set. If the returned number of bytes is greater than zero + * then @start is updated with the offset of the first byte with the bits set. + */ +u64 btrfs_count_range_bits(struct extent_io_tree *tree, + u64 *start, u64 search_end, u64 max_bytes, + u32 bits, bool contig, + struct extent_state **cached_state) +{ + struct extent_state *state = NULL; + struct extent_state *cached; + u64 cur_start = *start; + u64 total_bytes = 0; + u64 last = 0; + bool found = false; + + if (WARN_ON(search_end < cur_start)) + return 0; + + spin_lock(&tree->lock); + + if (!cached_state || !*cached_state) + goto search; + + cached = *cached_state; + + if (!extent_state_in_tree(cached)) + goto search; + + if (cached->start <= cur_start && cur_start <= cached->end) { + state = cached; + } else if (cached->start > cur_start) { + struct extent_state *prev; + + /* + * The cached state starts after our search range's start. Check + * if the previous state record starts at or before the range we + * are looking for, and if so, use it - this is a common case + * when there are holes between records in the tree. If there is + * no previous state record, we can start from our cached state. + */ + prev = prev_state(cached); + if (!prev) + state = cached; + else if (prev->start <= cur_start && cur_start <= prev->end) + state = prev; + } + + /* + * This search will find all the extents that end after our range + * starts. + */ +search: + if (!state) + state = tree_search(tree, cur_start); + + while (state) { + if (state->start > search_end) + break; + if (contig && found && state->start > last + 1) + break; + if (state->end >= cur_start && (state->state & bits) == bits) { + total_bytes += min(search_end, state->end) + 1 - + max(cur_start, state->start); + if (total_bytes >= max_bytes) + break; + if (!found) { + *start = max(cur_start, state->start); + found = true; + } + last = state->end; + } else if (contig && found) { + break; + } + state = next_state(state); + } + + if (cached_state) { + btrfs_free_extent_state(*cached_state); + *cached_state = state; + if (state) + refcount_inc(&state->refs); + } + + spin_unlock(&tree->lock); + + return total_bytes; +} + +/* + * Check if the single @bit exists in the given range. + */ +bool btrfs_test_range_bit_exists(struct extent_io_tree *tree, u64 start, u64 end, u32 bit) +{ + struct extent_state *state; + bool bitset = false; + + ASSERT(is_power_of_2(bit)); + + spin_lock(&tree->lock); + state = tree_search(tree, start); + while (state) { + if (state->start > end) + break; + + if (state->state & bit) { + bitset = true; + break; + } + + if (state->end >= end) + break; + state = next_state(state); + } + spin_unlock(&tree->lock); + return bitset; +} + +void btrfs_get_range_bits(struct extent_io_tree *tree, u64 start, u64 end, u32 *bits, + struct extent_state **cached_state) +{ + struct extent_state *state; + + /* + * The cached state is currently mandatory and not used to start the + * search, only to cache the first state record found in the range. + */ + ASSERT(cached_state != NULL); + ASSERT(*cached_state == NULL); + + *bits = 0; + + spin_lock(&tree->lock); + state = tree_search(tree, start); + if (state && state->start < end) { + *cached_state = state; + refcount_inc(&state->refs); + } + while (state) { + if (state->start > end) + break; + + *bits |= state->state; + + if (state->end >= end) + break; + + state = next_state(state); + } + spin_unlock(&tree->lock); +} + +/* + * Check if the whole range [@start,@end) contains the single @bit set. + */ +bool btrfs_test_range_bit(struct extent_io_tree *tree, u64 start, u64 end, u32 bit, + struct extent_state *cached) +{ + struct extent_state *state; + bool bitset = true; + + ASSERT(is_power_of_2(bit)); + ASSERT(start < end); + + spin_lock(&tree->lock); + if (cached && extent_state_in_tree(cached) && cached->start <= start && + cached->end > start) + state = cached; + else + state = tree_search(tree, start); + while (state) { + if (state->start > start) { + bitset = false; + break; + } + + if ((state->state & bit) == 0) { + bitset = false; + break; + } + + if (state->end >= end) + break; + + /* Next state must start where this one ends. */ + start = state->end + 1; + state = next_state(state); + } + + /* We ran out of states and were still inside of our range. */ + if (!state) + bitset = false; + spin_unlock(&tree->lock); + return bitset; +} + +/* Wrappers around set/clear extent bit */ +int btrfs_set_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, + u32 bits, struct extent_changeset *changeset) +{ + /* + * We don't support EXTENT_LOCK_BITS yet, as current changeset will + * record any bits changed, so for EXTENT_LOCK_BITS case, it will either + * fail with -EEXIST or changeset will record the whole range. + */ + ASSERT(!(bits & EXTENT_LOCK_BITS)); + + return set_extent_bit(tree, start, end, bits, NULL, NULL, NULL, changeset); +} + +int btrfs_clear_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, + u32 bits, struct extent_changeset *changeset) +{ + /* + * Don't support EXTENT_LOCK_BITS case, same reason as + * set_record_extent_bits(). + */ + ASSERT(!(bits & EXTENT_LOCK_BITS)); + + return btrfs_clear_extent_bit_changeset(tree, start, end, bits, NULL, changeset); +} + +bool btrfs_try_lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, + u32 bits, struct extent_state **cached) +{ + int ret; + u64 failed_start; + + ret = set_extent_bit(tree, start, end, bits, &failed_start, NULL, cached, NULL); + if (ret == -EEXIST) { + if (failed_start > start) + btrfs_clear_extent_bit(tree, start, failed_start - 1, + bits, cached); + return 0; + } + return 1; +} + +/* + * Either insert or lock state struct between start and end use mask to tell + * us if waiting is desired. + */ +int btrfs_lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, u32 bits, + struct extent_state **cached_state) +{ + struct extent_state *failed_state = NULL; + int ret; + u64 failed_start; + + ret = set_extent_bit(tree, start, end, bits, &failed_start, + &failed_state, cached_state, NULL); + while (ret == -EEXIST) { + if (failed_start != start) + btrfs_clear_extent_bit(tree, start, failed_start - 1, + bits, cached_state); + + wait_extent_bit(tree, failed_start, end, bits, &failed_state); + ret = set_extent_bit(tree, start, end, bits, &failed_start, + &failed_state, cached_state, NULL); + } + return ret; +} + +/* + * Get the extent state that follows the given extent state. + * This is meant to be used in a context where we know no other tasks can + * concurrently modify the tree. + */ +struct extent_state *btrfs_next_extent_state(struct extent_io_tree *tree, + struct extent_state *state) +{ + struct extent_state *next; + + spin_lock(&tree->lock); + ASSERT(extent_state_in_tree(state)); + next = next_state(state); + if (next) + refcount_inc(&next->refs); + spin_unlock(&tree->lock); + + return next; +} + +void __cold btrfs_extent_state_free_cachep(void) +{ + btrfs_extent_state_leak_debug_check(); + kmem_cache_destroy(extent_state_cache); +} + +int __init btrfs_extent_state_init_cachep(void) +{ + extent_state_cache = kmem_cache_create("btrfs_extent_state", + sizeof(struct extent_state), 0, 0, + NULL); + if (!extent_state_cache) + return -ENOMEM; + + return 0; +} diff --git a/fs/btrfs/extent-io-tree.h b/fs/btrfs/extent-io-tree.h new file mode 100644 index 000000000..6f07b965e --- /dev/null +++ b/fs/btrfs/extent-io-tree.h @@ -0,0 +1,244 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_EXTENT_IO_TREE_H +#define BTRFS_EXTENT_IO_TREE_H + +#include +#include +#include +#include +#include +#include "misc.h" + +struct extent_changeset; +struct btrfs_fs_info; +struct btrfs_inode; + +/* Bits for the extent state */ +enum { + ENUM_BIT(EXTENT_DIRTY), + ENUM_BIT(EXTENT_LOCKED), + ENUM_BIT(EXTENT_DIO_LOCKED), + ENUM_BIT(EXTENT_DIRTY_LOG1), + ENUM_BIT(EXTENT_DIRTY_LOG2), + ENUM_BIT(EXTENT_DELALLOC), + ENUM_BIT(EXTENT_DEFRAG), + ENUM_BIT(EXTENT_BOUNDARY), + ENUM_BIT(EXTENT_NODATASUM), + ENUM_BIT(EXTENT_CLEAR_META_RESV), + ENUM_BIT(EXTENT_NEED_WAIT), + ENUM_BIT(EXTENT_NORESERVE), + ENUM_BIT(EXTENT_QGROUP_RESERVED), + ENUM_BIT(EXTENT_CLEAR_DATA_RESV), + /* + * Must be cleared only during ordered extent completion or on error + * paths if we did not manage to submit bios and create the ordered + * extents for the range. Should not be cleared during page release + * and page invalidation (if there is an ordered extent in flight), + * that is left for the ordered extent completion. + */ + ENUM_BIT(EXTENT_DELALLOC_NEW), + /* + * Mark that a range is being locked for finishing an ordered extent. + * Used together with EXTENT_LOCKED. + */ + ENUM_BIT(EXTENT_FINISHING_ORDERED), + /* + * When an ordered extent successfully completes for a region marked as + * a new delalloc range, use this flag when clearing a new delalloc + * range to indicate that the VFS' inode number of bytes should be + * incremented and the inode's new delalloc bytes decremented, in an + * atomic way to prevent races with stat(2). + */ + ENUM_BIT(EXTENT_ADD_INODE_BYTES), + /* + * Set during truncate when we're clearing an entire range and we just + * want the extent states to go away. + */ + ENUM_BIT(EXTENT_CLEAR_ALL_BITS), + + /* + * This must be last. + * + * Bit not representing a state but a request for NOWAIT semantics, + * e.g. when allocating memory, and must be masked out from the other + * bits. + */ + ENUM_BIT(EXTENT_NOWAIT) +}; + +#define EXTENT_DO_ACCOUNTING (EXTENT_CLEAR_META_RESV | \ + EXTENT_CLEAR_DATA_RESV) +#define EXTENT_CTLBITS (EXTENT_DO_ACCOUNTING | \ + EXTENT_ADD_INODE_BYTES | \ + EXTENT_CLEAR_ALL_BITS) + +#define EXTENT_LOCK_BITS (EXTENT_LOCKED | EXTENT_DIO_LOCKED) + +/* + * Redefined bits above which are used only in the device allocation tree, + * shouldn't be using EXTENT_LOCKED / EXTENT_BOUNDARY / EXTENT_CLEAR_META_RESV + * / EXTENT_CLEAR_DATA_RESV because they have special meaning to the bit + * manipulation functions + */ +#define CHUNK_ALLOCATED EXTENT_DIRTY +#define CHUNK_TRIMMED EXTENT_DEFRAG +#define CHUNK_STATE_MASK (CHUNK_ALLOCATED | \ + CHUNK_TRIMMED) + +enum { + IO_TREE_FS_PINNED_EXTENTS, + IO_TREE_FS_EXCLUDED_EXTENTS, + IO_TREE_BTREE_INODE_IO, + IO_TREE_INODE_IO, + IO_TREE_RELOC_BLOCKS, + IO_TREE_TRANS_DIRTY_PAGES, + IO_TREE_ROOT_DIRTY_LOG_PAGES, + IO_TREE_INODE_FILE_EXTENT, + IO_TREE_LOG_CSUM_RANGE, + IO_TREE_SELFTEST, + IO_TREE_DEVICE_ALLOC_STATE, +}; + +struct extent_io_tree { + struct rb_root state; + /* + * The fs_info is needed for trace points, a tree attached to an inode + * needs the inode. + * + * owner == IO_TREE_INODE_IO - then inode is valid and fs_info can be + * accessed as inode->root->fs_info + */ + union { + struct btrfs_fs_info *fs_info; + struct btrfs_inode *inode; + }; + + /* Who owns this io tree, should be one of IO_TREE_* */ + u8 owner; + + spinlock_t lock; +}; + +struct extent_state { + u64 start; + u64 end; /* inclusive */ + struct rb_node rb_node; + + /* ADD NEW ELEMENTS AFTER THIS */ + wait_queue_head_t wq; + refcount_t refs; + u32 state; + +#ifdef CONFIG_BTRFS_DEBUG + struct list_head leak_list; +#endif +}; + +const struct btrfs_inode *btrfs_extent_io_tree_to_inode(const struct extent_io_tree *tree); +const struct btrfs_fs_info *btrfs_extent_io_tree_to_fs_info(const struct extent_io_tree *tree); + +void btrfs_extent_io_tree_init(struct btrfs_fs_info *fs_info, + struct extent_io_tree *tree, unsigned int owner); +void btrfs_extent_io_tree_release(struct extent_io_tree *tree); +int btrfs_lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, u32 bits, + struct extent_state **cached); +bool btrfs_try_lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, + u32 bits, struct extent_state **cached); + +static inline int btrfs_lock_extent(struct extent_io_tree *tree, u64 start, u64 end, + struct extent_state **cached) +{ + return btrfs_lock_extent_bits(tree, start, end, EXTENT_LOCKED, cached); +} + +static inline bool btrfs_try_lock_extent(struct extent_io_tree *tree, u64 start, + u64 end, struct extent_state **cached) +{ + return btrfs_try_lock_extent_bits(tree, start, end, EXTENT_LOCKED, cached); +} + +int __init btrfs_extent_state_init_cachep(void); +void __cold btrfs_extent_state_free_cachep(void); + +u64 btrfs_count_range_bits(struct extent_io_tree *tree, + u64 *start, u64 search_end, + u64 max_bytes, u32 bits, bool contig, + struct extent_state **cached_state); + +void btrfs_free_extent_state(struct extent_state *state); +bool btrfs_test_range_bit(struct extent_io_tree *tree, u64 start, u64 end, u32 bit, + struct extent_state *cached_state); +bool btrfs_test_range_bit_exists(struct extent_io_tree *tree, u64 start, u64 end, u32 bit); +void btrfs_get_range_bits(struct extent_io_tree *tree, u64 start, u64 end, u32 *bits, + struct extent_state **cached_state); +int btrfs_clear_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, + u32 bits, struct extent_changeset *changeset); +int btrfs_clear_extent_bit_changeset(struct extent_io_tree *tree, u64 start, u64 end, + u32 bits, struct extent_state **cached, + struct extent_changeset *changeset); + +static inline int btrfs_clear_extent_bit(struct extent_io_tree *tree, u64 start, + u64 end, u32 bits, + struct extent_state **cached) +{ + return btrfs_clear_extent_bit_changeset(tree, start, end, bits, cached, NULL); +} + +static inline int btrfs_unlock_extent(struct extent_io_tree *tree, u64 start, u64 end, + struct extent_state **cached) +{ + return btrfs_clear_extent_bit_changeset(tree, start, end, EXTENT_LOCKED, + cached, NULL); +} + +int btrfs_set_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, + u32 bits, struct extent_changeset *changeset); +int btrfs_set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, + u32 bits, struct extent_state **cached_state); + +static inline int btrfs_clear_extent_dirty(struct extent_io_tree *tree, u64 start, + u64 end, struct extent_state **cached) +{ + return btrfs_clear_extent_bit(tree, start, end, + EXTENT_DIRTY | EXTENT_DELALLOC | + EXTENT_DO_ACCOUNTING, cached); +} + +int btrfs_convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, + u32 bits, u32 clear_bits, + struct extent_state **cached_state); + +bool btrfs_find_first_extent_bit(struct extent_io_tree *tree, u64 start, + u64 *start_ret, u64 *end_ret, u32 bits, + struct extent_state **cached_state); +void btrfs_find_first_clear_extent_bit(struct extent_io_tree *tree, u64 start, + u64 *start_ret, u64 *end_ret, u32 bits); +bool btrfs_find_contiguous_extent_bit(struct extent_io_tree *tree, u64 start, + u64 *start_ret, u64 *end_ret, u32 bits); +bool btrfs_find_delalloc_range(struct extent_io_tree *tree, u64 *start, + u64 *end, u64 max_bytes, + struct extent_state **cached_state); +static inline int btrfs_lock_dio_extent(struct extent_io_tree *tree, u64 start, + u64 end, struct extent_state **cached) +{ + return btrfs_lock_extent_bits(tree, start, end, EXTENT_DIO_LOCKED, cached); +} + +static inline bool btrfs_try_lock_dio_extent(struct extent_io_tree *tree, u64 start, + u64 end, struct extent_state **cached) +{ + return btrfs_try_lock_extent_bits(tree, start, end, EXTENT_DIO_LOCKED, cached); +} + +static inline int btrfs_unlock_dio_extent(struct extent_io_tree *tree, u64 start, + u64 end, struct extent_state **cached) +{ + return btrfs_clear_extent_bit_changeset(tree, start, end, EXTENT_DIO_LOCKED, + cached, NULL); +} + +struct extent_state *btrfs_next_extent_state(struct extent_io_tree *tree, + struct extent_state *state); + +#endif /* BTRFS_EXTENT_IO_TREE_H */ diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c new file mode 100644 index 000000000..d6a4390ee --- /dev/null +++ b/fs/btrfs/extent-tree.c @@ -0,0 +1,6938 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ctree.h" +#include "extent-tree.h" +#include "transaction.h" +#include "disk-io.h" +#include "print-tree.h" +#include "volumes.h" +#include "raid56.h" +#include "locking.h" +#include "free-space-cache.h" +#include "free-space-tree.h" +#include "qgroup.h" +#include "ref-verify.h" +#include "space-info.h" +#include "block-rsv.h" +#include "discard.h" +#include "zoned.h" +#include "dev-replace.h" +#include "fs.h" +#include "accessors.h" +#include "root-tree.h" +#include "file-item.h" +#include "orphan.h" +#include "tree-checker.h" +#include "raid-stripe-tree.h" +#include "delayed-inode.h" +#include "relocation.h" + +#undef SCRAMBLE_DELAYED_REFS + + +static int __btrfs_free_extent(struct btrfs_trans_handle *trans, + struct btrfs_delayed_ref_head *href, + const struct btrfs_delayed_ref_node *node, + struct btrfs_delayed_extent_op *extra_op); +static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op, + struct extent_buffer *leaf, + struct btrfs_extent_item *ei); +static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans, + u64 parent, u64 root_objectid, + u64 flags, u64 owner, u64 offset, + struct btrfs_key *ins, int ref_mod, u64 oref_root); +static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans, + const struct btrfs_delayed_ref_node *node, + struct btrfs_delayed_extent_op *extent_op); +static int find_next_key(const struct btrfs_path *path, int level, + struct btrfs_key *key); + +static int block_group_bits(const struct btrfs_block_group *cache, u64 bits) +{ + return (cache->flags & bits) == bits; +} + +/* simple helper to search for an existing data extent at a given offset */ +int btrfs_lookup_data_extent(struct btrfs_fs_info *fs_info, u64 start, u64 len) +{ + struct btrfs_root *root = btrfs_extent_root(fs_info, start); + struct btrfs_key key; + BTRFS_PATH_AUTO_FREE(path); + + if (unlikely(!root)) { + btrfs_err(fs_info, + "missing extent root for extent at bytenr %llu", start); + return -EUCLEAN; + } + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = start; + key.type = BTRFS_EXTENT_ITEM_KEY; + key.offset = len; + return btrfs_search_slot(NULL, root, &key, path, 0, 0); +} + +/* + * helper function to lookup reference count and flags of a tree block. + * + * the head node for delayed ref is used to store the sum of all the + * reference count modifications queued up in the rbtree. the head + * node may also store the extent flags to set. This way you can check + * to see what the reference count and extent flags would be if all of + * the delayed refs are not processed. + */ +int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans, + struct btrfs_fs_info *fs_info, u64 bytenr, + u64 offset, int metadata, u64 *refs, u64 *flags, + u64 *owning_root) +{ + struct btrfs_root *extent_root; + struct btrfs_delayed_ref_head *head; + struct btrfs_delayed_ref_root *delayed_refs; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + u64 num_refs; + u64 extent_flags; + u64 owner = 0; + int ret; + + /* + * If we don't have skinny metadata, don't bother doing anything + * different + */ + if (metadata && !btrfs_fs_incompat(fs_info, SKINNY_METADATA)) { + offset = fs_info->nodesize; + metadata = 0; + } + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + +search_again: + key.objectid = bytenr; + if (metadata) + key.type = BTRFS_METADATA_ITEM_KEY; + else + key.type = BTRFS_EXTENT_ITEM_KEY; + key.offset = offset; + + extent_root = btrfs_extent_root(fs_info, bytenr); + if (unlikely(!extent_root)) { + btrfs_err(fs_info, + "missing extent root for extent at bytenr %llu", bytenr); + return -EUCLEAN; + } + + ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0); + if (ret < 0) + return ret; + + if (ret > 0 && key.type == BTRFS_METADATA_ITEM_KEY) { + if (path->slots[0]) { + path->slots[0]--; + btrfs_item_key_to_cpu(path->nodes[0], &key, + path->slots[0]); + if (key.objectid == bytenr && + key.type == BTRFS_EXTENT_ITEM_KEY && + key.offset == fs_info->nodesize) + ret = 0; + } + } + + if (ret == 0) { + struct extent_buffer *leaf = path->nodes[0]; + struct btrfs_extent_item *ei; + const u32 item_size = btrfs_item_size(leaf, path->slots[0]); + + if (unlikely(item_size < sizeof(*ei))) { + ret = -EUCLEAN; + btrfs_err(fs_info, + "unexpected extent item size, has %u expect >= %zu", + item_size, sizeof(*ei)); + btrfs_abort_transaction(trans, ret); + return ret; + } + + ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); + num_refs = btrfs_extent_refs(leaf, ei); + if (unlikely(num_refs == 0)) { + ret = -EUCLEAN; + btrfs_err(fs_info, + "unexpected zero reference count for extent item " BTRFS_KEY_FMT, + BTRFS_KEY_FMT_VALUE(&key)); + btrfs_abort_transaction(trans, ret); + return ret; + } + extent_flags = btrfs_extent_flags(leaf, ei); + owner = btrfs_get_extent_owner_root(fs_info, leaf, path->slots[0]); + } else { + num_refs = 0; + extent_flags = 0; + ret = 0; + } + + delayed_refs = &trans->transaction->delayed_refs; + spin_lock(&delayed_refs->lock); + head = btrfs_find_delayed_ref_head(fs_info, delayed_refs, bytenr); + if (head) { + if (!mutex_trylock(&head->mutex)) { + refcount_inc(&head->refs); + spin_unlock(&delayed_refs->lock); + + btrfs_release_path(path); + + /* + * Mutex was contended, block until it's released and try + * again + */ + mutex_lock(&head->mutex); + mutex_unlock(&head->mutex); + btrfs_put_delayed_ref_head(head); + goto search_again; + } + spin_lock(&head->lock); + if (head->extent_op && head->extent_op->update_flags) + extent_flags |= head->extent_op->flags_to_set; + + num_refs += head->ref_mod; + spin_unlock(&head->lock); + mutex_unlock(&head->mutex); + } + spin_unlock(&delayed_refs->lock); + + WARN_ON(num_refs == 0); + if (refs) + *refs = num_refs; + if (flags) + *flags = extent_flags; + if (owning_root) + *owning_root = owner; + + return ret; +} + +/* + * Back reference rules. Back refs have three main goals: + * + * 1) differentiate between all holders of references to an extent so that + * when a reference is dropped we can make sure it was a valid reference + * before freeing the extent. + * + * 2) Provide enough information to quickly find the holders of an extent + * if we notice a given block is corrupted or bad. + * + * 3) Make it easy to migrate blocks for FS shrinking or storage pool + * maintenance. This is actually the same as #2, but with a slightly + * different use case. + * + * There are two kinds of back refs. The implicit back refs is optimized + * for pointers in non-shared tree blocks. For a given pointer in a block, + * back refs of this kind provide information about the block's owner tree + * and the pointer's key. These information allow us to find the block by + * b-tree searching. The full back refs is for pointers in tree blocks not + * referenced by their owner trees. The location of tree block is recorded + * in the back refs. Actually the full back refs is generic, and can be + * used in all cases the implicit back refs is used. The major shortcoming + * of the full back refs is its overhead. Every time a tree block gets + * COWed, we have to update back refs entry for all pointers in it. + * + * For a newly allocated tree block, we use implicit back refs for + * pointers in it. This means most tree related operations only involve + * implicit back refs. For a tree block created in old transaction, the + * only way to drop a reference to it is COW it. So we can detect the + * event that tree block loses its owner tree's reference and do the + * back refs conversion. + * + * When a tree block is COWed through a tree, there are four cases: + * + * The reference count of the block is one and the tree is the block's + * owner tree. Nothing to do in this case. + * + * The reference count of the block is one and the tree is not the + * block's owner tree. In this case, full back refs is used for pointers + * in the block. Remove these full back refs, add implicit back refs for + * every pointers in the new block. + * + * The reference count of the block is greater than one and the tree is + * the block's owner tree. In this case, implicit back refs is used for + * pointers in the block. Add full back refs for every pointers in the + * block, increase lower level extents' reference counts. The original + * implicit back refs are entailed to the new block. + * + * The reference count of the block is greater than one and the tree is + * not the block's owner tree. Add implicit back refs for every pointer in + * the new block, increase lower level extents' reference count. + * + * Back Reference Key composing: + * + * The key objectid corresponds to the first byte in the extent, + * The key type is used to differentiate between types of back refs. + * There are different meanings of the key offset for different types + * of back refs. + * + * File extents can be referenced by: + * + * - multiple snapshots, subvolumes, or different generations in one subvol + * - different files inside a single subvolume + * - different offsets inside a file (bookend extents in file.c) + * + * The extent ref structure for the implicit back refs has fields for: + * + * - Objectid of the subvolume root + * - objectid of the file holding the reference + * - original offset in the file + * - how many bookend extents + * + * The key offset for the implicit back refs is hash of the first + * three fields. + * + * The extent ref structure for the full back refs has field for: + * + * - number of pointers in the tree leaf + * + * The key offset for the implicit back refs is the first byte of + * the tree leaf + * + * When a file extent is allocated, The implicit back refs is used. + * the fields are filled in: + * + * (root_key.objectid, inode objectid, offset in file, 1) + * + * When a file extent is removed file truncation, we find the + * corresponding implicit back refs and check the following fields: + * + * (btrfs_header_owner(leaf), inode objectid, offset in file) + * + * Btree extents can be referenced by: + * + * - Different subvolumes + * + * Both the implicit back refs and the full back refs for tree blocks + * only consist of key. The key offset for the implicit back refs is + * objectid of block's owner tree. The key offset for the full back refs + * is the first byte of parent block. + * + * When implicit back refs is used, information about the lowest key and + * level of the tree block are required. These information are stored in + * tree block info structure. + */ + +/* + * is_data == BTRFS_REF_TYPE_BLOCK, tree block type is required, + * is_data == BTRFS_REF_TYPE_DATA, data type is required, + * is_data == BTRFS_REF_TYPE_ANY, either type is OK. + */ +int btrfs_get_extent_inline_ref_type(const struct extent_buffer *eb, + const struct btrfs_extent_inline_ref *iref, + enum btrfs_inline_ref_type is_data) +{ + struct btrfs_fs_info *fs_info = eb->fs_info; + int type = btrfs_extent_inline_ref_type(eb, iref); + u64 offset = btrfs_extent_inline_ref_offset(eb, iref); + + if (type == BTRFS_EXTENT_OWNER_REF_KEY) { + ASSERT(btrfs_fs_incompat(fs_info, SIMPLE_QUOTA)); + return type; + } + + if (type == BTRFS_TREE_BLOCK_REF_KEY || + type == BTRFS_SHARED_BLOCK_REF_KEY || + type == BTRFS_SHARED_DATA_REF_KEY || + type == BTRFS_EXTENT_DATA_REF_KEY) { + if (is_data == BTRFS_REF_TYPE_BLOCK) { + if (type == BTRFS_TREE_BLOCK_REF_KEY) + return type; + if (type == BTRFS_SHARED_BLOCK_REF_KEY) { + ASSERT(fs_info); + /* + * Every shared one has parent tree block, + * which must be aligned to sector size. + */ + if (offset && IS_ALIGNED(offset, fs_info->sectorsize)) + return type; + } + } else if (is_data == BTRFS_REF_TYPE_DATA) { + if (type == BTRFS_EXTENT_DATA_REF_KEY) + return type; + if (type == BTRFS_SHARED_DATA_REF_KEY) { + ASSERT(fs_info); + /* + * Every shared one has parent tree block, + * which must be aligned to sector size. + */ + if (offset && + IS_ALIGNED(offset, fs_info->sectorsize)) + return type; + } + } else { + ASSERT(is_data == BTRFS_REF_TYPE_ANY, "is_data=%d", is_data); + return type; + } + } + + WARN_ON(1); + btrfs_print_leaf(eb); + btrfs_err(fs_info, + "eb %llu iref 0x%lx invalid extent inline ref type %d", + eb->start, (unsigned long)iref, type); + + return BTRFS_REF_TYPE_INVALID; +} + +u64 hash_extent_data_ref(u64 root_objectid, u64 owner, u64 offset) +{ + u32 high_crc = ~(u32)0; + u32 low_crc = ~(u32)0; + __le64 lenum; + + lenum = cpu_to_le64(root_objectid); + high_crc = crc32c(high_crc, &lenum, sizeof(lenum)); + lenum = cpu_to_le64(owner); + low_crc = crc32c(low_crc, &lenum, sizeof(lenum)); + lenum = cpu_to_le64(offset); + low_crc = crc32c(low_crc, &lenum, sizeof(lenum)); + + return ((u64)high_crc << 31) ^ (u64)low_crc; +} + +static u64 hash_extent_data_ref_item(const struct extent_buffer *leaf, + const struct btrfs_extent_data_ref *ref) +{ + return hash_extent_data_ref(btrfs_extent_data_ref_root(leaf, ref), + btrfs_extent_data_ref_objectid(leaf, ref), + btrfs_extent_data_ref_offset(leaf, ref)); +} + +static bool match_extent_data_ref(const struct extent_buffer *leaf, + const struct btrfs_extent_data_ref *ref, + u64 root_objectid, u64 owner, u64 offset) +{ + if (btrfs_extent_data_ref_root(leaf, ref) != root_objectid || + btrfs_extent_data_ref_objectid(leaf, ref) != owner || + btrfs_extent_data_ref_offset(leaf, ref) != offset) + return false; + return true; +} + +static noinline int lookup_extent_data_ref(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + u64 bytenr, u64 parent, + u64 root_objectid, + u64 owner, u64 offset) +{ + struct btrfs_root *root = btrfs_extent_root(trans->fs_info, bytenr); + struct btrfs_key key; + struct btrfs_extent_data_ref *ref; + struct extent_buffer *leaf; + u32 nritems; + int recow; + int ret; + + if (unlikely(!root)) { + btrfs_err(trans->fs_info, + "missing extent root for extent at bytenr %llu", bytenr); + return -EUCLEAN; + } + + key.objectid = bytenr; + if (parent) { + key.type = BTRFS_SHARED_DATA_REF_KEY; + key.offset = parent; + } else { + key.type = BTRFS_EXTENT_DATA_REF_KEY; + key.offset = hash_extent_data_ref(root_objectid, + owner, offset); + } +again: + recow = 0; + ret = btrfs_search_slot(trans, root, &key, path, -1, 1); + if (ret < 0) + return ret; + + if (parent) { + if (ret) + return -ENOENT; + return 0; + } + + ret = -ENOENT; + leaf = path->nodes[0]; + nritems = btrfs_header_nritems(leaf); + while (1) { + if (path->slots[0] >= nritems) { + ret = btrfs_next_leaf(root, path); + if (ret) { + if (ret > 0) + return -ENOENT; + return ret; + } + + leaf = path->nodes[0]; + nritems = btrfs_header_nritems(leaf); + recow = 1; + } + + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + if (key.objectid != bytenr || + key.type != BTRFS_EXTENT_DATA_REF_KEY) + return -ENOENT; + + ref = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_extent_data_ref); + + if (match_extent_data_ref(leaf, ref, root_objectid, + owner, offset)) { + if (recow) { + btrfs_release_path(path); + goto again; + } + return 0; + } + path->slots[0]++; + } + + return ret; +} + +static noinline int insert_extent_data_ref(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + const struct btrfs_delayed_ref_node *node, + u64 bytenr) +{ + struct btrfs_root *root = btrfs_extent_root(trans->fs_info, bytenr); + struct btrfs_key key; + struct extent_buffer *leaf; + u64 owner = btrfs_delayed_ref_owner(node); + u64 offset = btrfs_delayed_ref_offset(node); + u32 size; + u32 num_refs; + int ret; + + if (unlikely(!root)) { + btrfs_err(trans->fs_info, + "missing extent root for extent at bytenr %llu", bytenr); + return -EUCLEAN; + } + + key.objectid = bytenr; + if (node->parent) { + key.type = BTRFS_SHARED_DATA_REF_KEY; + key.offset = node->parent; + size = sizeof(struct btrfs_shared_data_ref); + } else { + key.type = BTRFS_EXTENT_DATA_REF_KEY; + key.offset = hash_extent_data_ref(node->ref_root, owner, offset); + size = sizeof(struct btrfs_extent_data_ref); + } + + ret = btrfs_insert_empty_item(trans, root, path, &key, size); + if (ret && ret != -EEXIST) + goto fail; + + leaf = path->nodes[0]; + if (node->parent) { + struct btrfs_shared_data_ref *ref; + ref = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_shared_data_ref); + if (ret == 0) { + btrfs_set_shared_data_ref_count(leaf, ref, node->ref_mod); + } else { + num_refs = btrfs_shared_data_ref_count(leaf, ref); + num_refs += node->ref_mod; + btrfs_set_shared_data_ref_count(leaf, ref, num_refs); + } + } else { + struct btrfs_extent_data_ref *ref; + while (ret == -EEXIST) { + ref = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_extent_data_ref); + if (match_extent_data_ref(leaf, ref, node->ref_root, + owner, offset)) + break; + btrfs_release_path(path); + key.offset++; + ret = btrfs_insert_empty_item(trans, root, path, &key, + size); + if (ret && ret != -EEXIST) + goto fail; + + leaf = path->nodes[0]; + } + ref = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_extent_data_ref); + if (ret == 0) { + btrfs_set_extent_data_ref_root(leaf, ref, node->ref_root); + btrfs_set_extent_data_ref_objectid(leaf, ref, owner); + btrfs_set_extent_data_ref_offset(leaf, ref, offset); + btrfs_set_extent_data_ref_count(leaf, ref, node->ref_mod); + } else { + num_refs = btrfs_extent_data_ref_count(leaf, ref); + num_refs += node->ref_mod; + btrfs_set_extent_data_ref_count(leaf, ref, num_refs); + } + } + ret = 0; +fail: + btrfs_release_path(path); + return ret; +} + +static noinline int remove_extent_data_ref(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + int refs_to_drop) +{ + struct btrfs_key key; + struct btrfs_extent_data_ref *ref1 = NULL; + struct btrfs_shared_data_ref *ref2 = NULL; + struct extent_buffer *leaf; + u32 num_refs = 0; + int ret = 0; + + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + + if (key.type == BTRFS_EXTENT_DATA_REF_KEY) { + ref1 = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_extent_data_ref); + num_refs = btrfs_extent_data_ref_count(leaf, ref1); + } else if (key.type == BTRFS_SHARED_DATA_REF_KEY) { + ref2 = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_shared_data_ref); + num_refs = btrfs_shared_data_ref_count(leaf, ref2); + } else { + btrfs_err(trans->fs_info, + "unrecognized backref key " BTRFS_KEY_FMT, + BTRFS_KEY_FMT_VALUE(&key)); + btrfs_abort_transaction(trans, -EUCLEAN); + return -EUCLEAN; + } + + BUG_ON(num_refs < refs_to_drop); + num_refs -= refs_to_drop; + + if (num_refs == 0) { + ret = btrfs_del_item(trans, root, path); + } else { + if (key.type == BTRFS_EXTENT_DATA_REF_KEY) + btrfs_set_extent_data_ref_count(leaf, ref1, num_refs); + else if (key.type == BTRFS_SHARED_DATA_REF_KEY) + btrfs_set_shared_data_ref_count(leaf, ref2, num_refs); + } + return ret; +} + +static noinline u32 extent_data_ref_count(const struct btrfs_path *path, + const struct btrfs_extent_inline_ref *iref) +{ + struct btrfs_key key; + struct extent_buffer *leaf; + const struct btrfs_extent_data_ref *ref1; + const struct btrfs_shared_data_ref *ref2; + u32 num_refs = 0; + int type; + + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + + if (iref) { + /* + * If type is invalid, we should have bailed out earlier than + * this call. + */ + type = btrfs_get_extent_inline_ref_type(leaf, iref, BTRFS_REF_TYPE_DATA); + ASSERT(type != BTRFS_REF_TYPE_INVALID); + if (type == BTRFS_EXTENT_DATA_REF_KEY) { + ref1 = (const struct btrfs_extent_data_ref *)(&iref->offset); + num_refs = btrfs_extent_data_ref_count(leaf, ref1); + } else { + ref2 = (const struct btrfs_shared_data_ref *)(iref + 1); + num_refs = btrfs_shared_data_ref_count(leaf, ref2); + } + } else if (key.type == BTRFS_EXTENT_DATA_REF_KEY) { + ref1 = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_extent_data_ref); + num_refs = btrfs_extent_data_ref_count(leaf, ref1); + } else if (key.type == BTRFS_SHARED_DATA_REF_KEY) { + ref2 = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_shared_data_ref); + num_refs = btrfs_shared_data_ref_count(leaf, ref2); + } else { + WARN_ON(1); + } + return num_refs; +} + +static noinline int lookup_tree_block_ref(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + u64 bytenr, u64 parent, + u64 root_objectid) +{ + struct btrfs_root *root = btrfs_extent_root(trans->fs_info, bytenr); + struct btrfs_key key; + int ret; + + if (unlikely(!root)) { + btrfs_err(trans->fs_info, + "missing extent root for extent at bytenr %llu", bytenr); + return -EUCLEAN; + } + + key.objectid = bytenr; + if (parent) { + key.type = BTRFS_SHARED_BLOCK_REF_KEY; + key.offset = parent; + } else { + key.type = BTRFS_TREE_BLOCK_REF_KEY; + key.offset = root_objectid; + } + + ret = btrfs_search_slot(trans, root, &key, path, -1, 1); + if (ret > 0) + ret = -ENOENT; + return ret; +} + +static noinline int insert_tree_block_ref(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + const struct btrfs_delayed_ref_node *node, + u64 bytenr) +{ + struct btrfs_root *root = btrfs_extent_root(trans->fs_info, bytenr); + struct btrfs_key key; + int ret; + + if (unlikely(!root)) { + btrfs_err(trans->fs_info, + "missing extent root for extent at bytenr %llu", bytenr); + return -EUCLEAN; + } + + key.objectid = bytenr; + if (node->parent) { + key.type = BTRFS_SHARED_BLOCK_REF_KEY; + key.offset = node->parent; + } else { + key.type = BTRFS_TREE_BLOCK_REF_KEY; + key.offset = node->ref_root; + } + + ret = btrfs_insert_empty_item(trans, root, path, &key, 0); + btrfs_release_path(path); + return ret; +} + +static inline int extent_ref_type(u64 parent, u64 owner) +{ + int type; + if (owner < BTRFS_FIRST_FREE_OBJECTID) { + if (parent > 0) + type = BTRFS_SHARED_BLOCK_REF_KEY; + else + type = BTRFS_TREE_BLOCK_REF_KEY; + } else { + if (parent > 0) + type = BTRFS_SHARED_DATA_REF_KEY; + else + type = BTRFS_EXTENT_DATA_REF_KEY; + } + return type; +} + +static int find_next_key(const struct btrfs_path *path, int level, + struct btrfs_key *key) + +{ + for (; level < BTRFS_MAX_LEVEL; level++) { + if (!path->nodes[level]) + break; + if (path->slots[level] + 1 >= + btrfs_header_nritems(path->nodes[level])) + continue; + if (level == 0) + btrfs_item_key_to_cpu(path->nodes[level], key, + path->slots[level] + 1); + else + btrfs_node_key_to_cpu(path->nodes[level], key, + path->slots[level] + 1); + return 0; + } + return 1; +} + +/* + * look for inline back ref. if back ref is found, *ref_ret is set + * to the address of inline back ref, and 0 is returned. + * + * if back ref isn't found, *ref_ret is set to the address where it + * should be inserted, and -ENOENT is returned. + * + * if insert is true and there are too many inline back refs, the path + * points to the extent item, and -EAGAIN is returned. + * + * NOTE: inline back refs are ordered in the same way that back ref + * items in the tree are ordered. + */ +static noinline_for_stack +int lookup_inline_extent_backref(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + struct btrfs_extent_inline_ref **ref_ret, + u64 bytenr, u64 num_bytes, + u64 parent, u64 root_objectid, + u64 owner, u64 offset, int insert) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *root = btrfs_extent_root(fs_info, bytenr); + struct btrfs_key key; + struct extent_buffer *leaf; + struct btrfs_extent_item *ei; + struct btrfs_extent_inline_ref *iref; + u64 flags; + u64 item_size; + unsigned long ptr; + unsigned long end; + int extra_size; + int type; + int want; + int ret; + bool skinny_metadata = btrfs_fs_incompat(fs_info, SKINNY_METADATA); + int needed; + + if (unlikely(!root)) { + btrfs_err(fs_info, + "missing extent root for extent at bytenr %llu", bytenr); + return -EUCLEAN; + } + + key.objectid = bytenr; + key.type = BTRFS_EXTENT_ITEM_KEY; + key.offset = num_bytes; + + want = extent_ref_type(parent, owner); + if (insert) { + extra_size = btrfs_extent_inline_ref_size(want); + path->search_for_extension = true; + } else + extra_size = -1; + + /* + * Owner is our level, so we can just add one to get the level for the + * block we are interested in. + */ + if (skinny_metadata && owner < BTRFS_FIRST_FREE_OBJECTID) { + key.type = BTRFS_METADATA_ITEM_KEY; + key.offset = owner; + } + +again: + ret = btrfs_search_slot(trans, root, &key, path, extra_size, 1); + if (ret < 0) + goto out; + + /* + * We may be a newly converted file system which still has the old fat + * extent entries for metadata, so try and see if we have one of those. + */ + if (ret > 0 && skinny_metadata) { + skinny_metadata = false; + if (path->slots[0]) { + path->slots[0]--; + btrfs_item_key_to_cpu(path->nodes[0], &key, + path->slots[0]); + if (key.objectid == bytenr && + key.type == BTRFS_EXTENT_ITEM_KEY && + key.offset == num_bytes) + ret = 0; + } + if (ret) { + key.objectid = bytenr; + key.type = BTRFS_EXTENT_ITEM_KEY; + key.offset = num_bytes; + btrfs_release_path(path); + goto again; + } + } + + if (ret && !insert) { + ret = -ENOENT; + goto out; + } else if (WARN_ON(ret)) { + btrfs_print_leaf(path->nodes[0]); + btrfs_err(fs_info, +"extent item not found for insert, bytenr %llu num_bytes %llu parent %llu root_objectid %llu owner %llu offset %llu", + bytenr, num_bytes, parent, root_objectid, owner, + offset); + ret = -EUCLEAN; + goto out; + } + + leaf = path->nodes[0]; + item_size = btrfs_item_size(leaf, path->slots[0]); + if (unlikely(item_size < sizeof(*ei))) { + ret = -EUCLEAN; + btrfs_err(fs_info, + "unexpected extent item size, has %llu expect >= %zu", + item_size, sizeof(*ei)); + btrfs_abort_transaction(trans, ret); + goto out; + } + + ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); + flags = btrfs_extent_flags(leaf, ei); + + ptr = (unsigned long)(ei + 1); + end = (unsigned long)ei + item_size; + + if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK && !skinny_metadata) { + ptr += sizeof(struct btrfs_tree_block_info); + BUG_ON(ptr > end); + } + + if (owner >= BTRFS_FIRST_FREE_OBJECTID) + needed = BTRFS_REF_TYPE_DATA; + else + needed = BTRFS_REF_TYPE_BLOCK; + + ret = -ENOENT; + while (ptr < end) { + iref = (struct btrfs_extent_inline_ref *)ptr; + type = btrfs_get_extent_inline_ref_type(leaf, iref, needed); + if (type == BTRFS_EXTENT_OWNER_REF_KEY) { + ASSERT(btrfs_fs_incompat(fs_info, SIMPLE_QUOTA)); + ptr += btrfs_extent_inline_ref_size(type); + continue; + } + if (unlikely(type == BTRFS_REF_TYPE_INVALID)) { + ret = -EUCLEAN; + goto out; + } + + if (want < type) + break; + if (want > type) { + ptr += btrfs_extent_inline_ref_size(type); + continue; + } + + if (type == BTRFS_EXTENT_DATA_REF_KEY) { + struct btrfs_extent_data_ref *dref; + dref = (struct btrfs_extent_data_ref *)(&iref->offset); + if (match_extent_data_ref(leaf, dref, root_objectid, + owner, offset)) { + ret = 0; + break; + } + if (hash_extent_data_ref_item(leaf, dref) < + hash_extent_data_ref(root_objectid, owner, offset)) + break; + } else { + u64 ref_offset; + ref_offset = btrfs_extent_inline_ref_offset(leaf, iref); + if (parent > 0) { + if (parent == ref_offset) { + ret = 0; + break; + } + if (ref_offset < parent) + break; + } else { + if (root_objectid == ref_offset) { + ret = 0; + break; + } + if (ref_offset < root_objectid) + break; + } + } + ptr += btrfs_extent_inline_ref_size(type); + } + + if (unlikely(ptr > end)) { + ret = -EUCLEAN; + btrfs_print_leaf(path->nodes[0]); + btrfs_crit(fs_info, +"overrun extent record at slot %d while looking for inline extent for root %llu owner %llu offset %llu parent %llu", + path->slots[0], root_objectid, owner, offset, parent); + goto out; + } + + if (ret == -ENOENT && insert) { + if (item_size + extra_size >= + BTRFS_MAX_EXTENT_ITEM_SIZE(root)) { + ret = -EAGAIN; + goto out; + } + + if (path->slots[0] + 1 < btrfs_header_nritems(path->nodes[0])) { + struct btrfs_key tmp_key; + + btrfs_item_key_to_cpu(path->nodes[0], &tmp_key, path->slots[0] + 1); + if (tmp_key.objectid == bytenr && + tmp_key.type < BTRFS_BLOCK_GROUP_ITEM_KEY) { + ret = -EAGAIN; + goto out; + } + goto out_no_entry; + } + + if (!path->keep_locks) { + btrfs_release_path(path); + path->keep_locks = true; + goto again; + } + + /* + * To add new inline back ref, we have to make sure + * there is no corresponding back ref item. + * For simplicity, we just do not add new inline back + * ref if there is any kind of item for this block + */ + if (find_next_key(path, 0, &key) == 0 && + key.objectid == bytenr && + key.type < BTRFS_BLOCK_GROUP_ITEM_KEY) { + ret = -EAGAIN; + goto out; + } + } +out_no_entry: + *ref_ret = (struct btrfs_extent_inline_ref *)ptr; +out: + if (path->keep_locks) { + path->keep_locks = false; + btrfs_unlock_up_safe(path, 1); + } + if (insert) + path->search_for_extension = false; + return ret; +} + +/* + * helper to add new inline back ref + */ +static noinline_for_stack +void setup_inline_extent_backref(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + struct btrfs_extent_inline_ref *iref, + u64 parent, u64 root_objectid, + u64 owner, u64 offset, int refs_to_add, + struct btrfs_delayed_extent_op *extent_op) +{ + struct extent_buffer *leaf; + struct btrfs_extent_item *ei; + unsigned long ptr; + unsigned long end; + unsigned long item_offset; + u64 refs; + int size; + int type; + + leaf = path->nodes[0]; + ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); + item_offset = (unsigned long)iref - (unsigned long)ei; + + type = extent_ref_type(parent, owner); + size = btrfs_extent_inline_ref_size(type); + + btrfs_extend_item(trans, path, size); + + ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); + refs = btrfs_extent_refs(leaf, ei); + refs += refs_to_add; + btrfs_set_extent_refs(leaf, ei, refs); + if (extent_op) + __run_delayed_extent_op(extent_op, leaf, ei); + + ptr = (unsigned long)ei + item_offset; + end = (unsigned long)ei + btrfs_item_size(leaf, path->slots[0]); + if (ptr < end - size) + memmove_extent_buffer(leaf, ptr + size, ptr, + end - size - ptr); + + iref = (struct btrfs_extent_inline_ref *)ptr; + btrfs_set_extent_inline_ref_type(leaf, iref, type); + if (type == BTRFS_EXTENT_DATA_REF_KEY) { + struct btrfs_extent_data_ref *dref; + dref = (struct btrfs_extent_data_ref *)(&iref->offset); + btrfs_set_extent_data_ref_root(leaf, dref, root_objectid); + btrfs_set_extent_data_ref_objectid(leaf, dref, owner); + btrfs_set_extent_data_ref_offset(leaf, dref, offset); + btrfs_set_extent_data_ref_count(leaf, dref, refs_to_add); + } else if (type == BTRFS_SHARED_DATA_REF_KEY) { + struct btrfs_shared_data_ref *sref; + sref = (struct btrfs_shared_data_ref *)(iref + 1); + btrfs_set_shared_data_ref_count(leaf, sref, refs_to_add); + btrfs_set_extent_inline_ref_offset(leaf, iref, parent); + } else if (type == BTRFS_SHARED_BLOCK_REF_KEY) { + btrfs_set_extent_inline_ref_offset(leaf, iref, parent); + } else { + btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid); + } +} + +static int lookup_extent_backref(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + struct btrfs_extent_inline_ref **ref_ret, + u64 bytenr, u64 num_bytes, u64 parent, + u64 root_objectid, u64 owner, u64 offset) +{ + int ret; + + ret = lookup_inline_extent_backref(trans, path, ref_ret, bytenr, + num_bytes, parent, root_objectid, + owner, offset, 0); + if (ret != -ENOENT) + return ret; + + btrfs_release_path(path); + *ref_ret = NULL; + + if (owner < BTRFS_FIRST_FREE_OBJECTID) { + ret = lookup_tree_block_ref(trans, path, bytenr, parent, + root_objectid); + } else { + ret = lookup_extent_data_ref(trans, path, bytenr, parent, + root_objectid, owner, offset); + } + return ret; +} + +/* + * helper to update/remove inline back ref + */ +static noinline_for_stack int update_inline_extent_backref( + struct btrfs_trans_handle *trans, + struct btrfs_path *path, + struct btrfs_extent_inline_ref *iref, + int refs_to_mod, + struct btrfs_delayed_extent_op *extent_op) +{ + struct extent_buffer *leaf = path->nodes[0]; + struct btrfs_fs_info *fs_info = leaf->fs_info; + struct btrfs_extent_item *ei; + struct btrfs_extent_data_ref *dref = NULL; + struct btrfs_shared_data_ref *sref = NULL; + unsigned long ptr; + unsigned long end; + u32 item_size; + int size; + int type; + u64 refs; + + ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); + refs = btrfs_extent_refs(leaf, ei); + if (unlikely(refs_to_mod < 0 && refs + refs_to_mod <= 0)) { + struct btrfs_key key; + u32 extent_size; + + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + if (key.type == BTRFS_METADATA_ITEM_KEY) + extent_size = fs_info->nodesize; + else + extent_size = key.offset; + btrfs_print_leaf(leaf); + btrfs_err(fs_info, + "invalid refs_to_mod for extent %llu num_bytes %u, has %d expect >= -%llu", + key.objectid, extent_size, refs_to_mod, refs); + return -EUCLEAN; + } + refs += refs_to_mod; + btrfs_set_extent_refs(leaf, ei, refs); + if (extent_op) + __run_delayed_extent_op(extent_op, leaf, ei); + + type = btrfs_get_extent_inline_ref_type(leaf, iref, BTRFS_REF_TYPE_ANY); + /* + * Function btrfs_get_extent_inline_ref_type() has already printed + * error messages. + */ + if (unlikely(type == BTRFS_REF_TYPE_INVALID)) + return -EUCLEAN; + + if (type == BTRFS_EXTENT_DATA_REF_KEY) { + dref = (struct btrfs_extent_data_ref *)(&iref->offset); + refs = btrfs_extent_data_ref_count(leaf, dref); + } else if (type == BTRFS_SHARED_DATA_REF_KEY) { + sref = (struct btrfs_shared_data_ref *)(iref + 1); + refs = btrfs_shared_data_ref_count(leaf, sref); + } else { + refs = 1; + /* + * For tree blocks we can only drop one ref for it, and tree + * blocks should not have refs > 1. + * + * Furthermore if we're inserting a new inline backref, we + * won't reach this path either. That would be + * setup_inline_extent_backref(). + */ + if (unlikely(refs_to_mod != -1)) { + struct btrfs_key key; + + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + + btrfs_print_leaf(leaf); + btrfs_err(fs_info, + "invalid refs_to_mod for tree block %llu, has %d expect -1", + key.objectid, refs_to_mod); + return -EUCLEAN; + } + } + + if (unlikely(refs_to_mod < 0 && refs < -refs_to_mod)) { + struct btrfs_key key; + u32 extent_size; + + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + if (key.type == BTRFS_METADATA_ITEM_KEY) + extent_size = fs_info->nodesize; + else + extent_size = key.offset; + btrfs_print_leaf(leaf); + btrfs_err(fs_info, +"invalid refs_to_mod for backref entry, iref %lu extent %llu num_bytes %u, has %d expect >= -%llu", + (unsigned long)iref, key.objectid, extent_size, + refs_to_mod, refs); + return -EUCLEAN; + } + refs += refs_to_mod; + + if (refs > 0) { + if (type == BTRFS_EXTENT_DATA_REF_KEY) + btrfs_set_extent_data_ref_count(leaf, dref, refs); + else + btrfs_set_shared_data_ref_count(leaf, sref, refs); + } else { + size = btrfs_extent_inline_ref_size(type); + item_size = btrfs_item_size(leaf, path->slots[0]); + ptr = (unsigned long)iref; + end = (unsigned long)ei + item_size; + if (ptr + size < end) + memmove_extent_buffer(leaf, ptr, ptr + size, + end - ptr - size); + item_size -= size; + btrfs_truncate_item(trans, path, item_size, 1); + } + return 0; +} + +static noinline_for_stack +int insert_inline_extent_backref(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + u64 bytenr, u64 num_bytes, u64 parent, + u64 root_objectid, u64 owner, + u64 offset, int refs_to_add, + struct btrfs_delayed_extent_op *extent_op) +{ + struct btrfs_extent_inline_ref *iref; + int ret; + + ret = lookup_inline_extent_backref(trans, path, &iref, bytenr, + num_bytes, parent, root_objectid, + owner, offset, 1); + if (ret == 0) { + /* + * We're adding refs to a tree block we already own, this + * should not happen at all. + */ + if (unlikely(owner < BTRFS_FIRST_FREE_OBJECTID)) { + btrfs_print_leaf(path->nodes[0]); + btrfs_crit(trans->fs_info, +"adding refs to an existing tree ref, bytenr %llu num_bytes %llu root_objectid %llu slot %u", + bytenr, num_bytes, root_objectid, path->slots[0]); + return -EUCLEAN; + } + ret = update_inline_extent_backref(trans, path, iref, + refs_to_add, extent_op); + } else if (ret == -ENOENT) { + setup_inline_extent_backref(trans, path, iref, parent, + root_objectid, owner, offset, + refs_to_add, extent_op); + ret = 0; + } + return ret; +} + +static int remove_extent_backref(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + struct btrfs_extent_inline_ref *iref, + int refs_to_drop, int is_data) +{ + int ret = 0; + + BUG_ON(!is_data && refs_to_drop != 1); + if (iref) + ret = update_inline_extent_backref(trans, path, iref, + -refs_to_drop, NULL); + else if (is_data) + ret = remove_extent_data_ref(trans, root, path, refs_to_drop); + else + ret = btrfs_del_item(trans, root, path); + return ret; +} + +static int btrfs_issue_discard(struct block_device *bdev, u64 start, u64 len, + u64 *discarded_bytes) +{ + int j, ret = 0; + u64 bytes_left, end; + u64 aligned_start = ALIGN(start, SECTOR_SIZE); + + /* Adjust the range to be aligned to 512B sectors if necessary. */ + if (start != aligned_start) { + len -= aligned_start - start; + len = round_down(len, SECTOR_SIZE); + start = aligned_start; + } + + *discarded_bytes = 0; + + if (!len) + return 0; + + end = start + len; + bytes_left = len; + + /* Skip any superblocks on this device. */ + for (j = 0; j < BTRFS_SUPER_MIRROR_MAX; j++) { + u64 sb_start = btrfs_sb_offset(j); + u64 sb_end = sb_start + BTRFS_SUPER_INFO_SIZE; + u64 size = sb_start - start; + + if (!in_range(sb_start, start, bytes_left) && + !in_range(sb_end, start, bytes_left) && + !in_range(start, sb_start, BTRFS_SUPER_INFO_SIZE)) + continue; + + /* + * Superblock spans beginning of range. Adjust start and + * try again. + */ + if (sb_start <= start) { + start += sb_end - start; + if (start > end) { + bytes_left = 0; + break; + } + bytes_left = end - start; + continue; + } + + if (size) { + ret = blkdev_issue_discard(bdev, start >> SECTOR_SHIFT, + size >> SECTOR_SHIFT, + GFP_NOFS); + if (!ret) + *discarded_bytes += size; + else if (ret != -EOPNOTSUPP) + return ret; + } + + start = sb_end; + if (start > end) { + bytes_left = 0; + break; + } + bytes_left = end - start; + } + + while (bytes_left) { + u64 bytes_to_discard = min(BTRFS_MAX_DISCARD_CHUNK_SIZE, bytes_left); + + ret = blkdev_issue_discard(bdev, start >> SECTOR_SHIFT, + bytes_to_discard >> SECTOR_SHIFT, + GFP_NOFS); + + if (ret) { + if (ret != -EOPNOTSUPP) + break; + continue; + } + + start += bytes_to_discard; + bytes_left -= bytes_to_discard; + *discarded_bytes += bytes_to_discard; + + if (btrfs_trim_interrupted()) { + ret = -ERESTARTSYS; + break; + } + } + + return ret; +} + +static int do_discard_extent(struct btrfs_discard_stripe *stripe, u64 *bytes) +{ + struct btrfs_device *dev = stripe->dev; + struct btrfs_fs_info *fs_info = dev->fs_info; + struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; + u64 phys = stripe->physical; + u64 len = stripe->length; + u64 discarded = 0; + int ret = 0; + + /* Zone reset on a zoned filesystem */ + if (btrfs_can_zone_reset(dev, phys, len)) { + u64 src_disc; + + ret = btrfs_reset_device_zone(dev, phys, len, &discarded); + if (ret) + goto out; + + if (!btrfs_dev_replace_is_ongoing(dev_replace) || + dev != dev_replace->srcdev) + goto out; + + src_disc = discarded; + + /* Send to replace target as well */ + ret = btrfs_reset_device_zone(dev_replace->tgtdev, phys, len, + &discarded); + discarded += src_disc; + } else if (bdev_max_discard_sectors(stripe->dev->bdev)) { + ret = btrfs_issue_discard(dev->bdev, phys, len, &discarded); + } else { + ret = 0; + *bytes = 0; + } + +out: + *bytes = discarded; + return ret; +} + +int btrfs_discard_extent(struct btrfs_fs_info *fs_info, u64 bytenr, + u64 num_bytes, u64 *actual_bytes, bool do_remap) +{ + int ret = 0; + u64 discarded_bytes = 0; + u64 end = bytenr + num_bytes; + u64 cur = bytenr; + + /* + * Avoid races with device replace and make sure the devices in the + * stripes don't go away while we are discarding. + */ + btrfs_bio_counter_inc_blocked(fs_info); + while (cur < end) { + struct btrfs_discard_stripe *stripes; + unsigned int num_stripes; + int i; + + num_bytes = end - cur; + stripes = btrfs_map_discard(fs_info, cur, &num_bytes, &num_stripes, + do_remap); + if (IS_ERR(stripes)) { + ret = PTR_ERR(stripes); + if (ret == -EOPNOTSUPP) + ret = 0; + break; + } + + for (i = 0; i < num_stripes; i++) { + struct btrfs_discard_stripe *stripe = stripes + i; + u64 bytes; + + if (!stripe->dev->bdev) { + ASSERT(btrfs_test_opt(fs_info, DEGRADED)); + continue; + } + + if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, + &stripe->dev->dev_state)) + continue; + + ret = do_discard_extent(stripe, &bytes); + if (ret) { + /* + * Keep going if discard is not supported by the + * device. + */ + if (ret != -EOPNOTSUPP) + break; + ret = 0; + } else { + discarded_bytes += bytes; + } + } + kfree(stripes); + if (ret) + break; + cur += num_bytes; + } + btrfs_bio_counter_dec(fs_info); + if (actual_bytes) + *actual_bytes = discarded_bytes; + return ret; +} + +/* Can return -ENOMEM */ +int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans, + struct btrfs_ref *generic_ref) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + int ret; + + ASSERT(generic_ref->type != BTRFS_REF_NOT_SET && + generic_ref->action); + BUG_ON(generic_ref->type == BTRFS_REF_METADATA && + generic_ref->ref_root == BTRFS_TREE_LOG_OBJECTID); + + if (generic_ref->type == BTRFS_REF_METADATA) + ret = btrfs_add_delayed_tree_ref(trans, generic_ref, NULL); + else + ret = btrfs_add_delayed_data_ref(trans, generic_ref, 0); + + btrfs_ref_tree_mod(fs_info, generic_ref); + + return ret; +} + +/* + * Insert backreference for a given extent. + * + * The counterpart is in __btrfs_free_extent(), with examples and more details + * how it works. + * + * @trans: Handle of transaction + * + * @node: The delayed ref node used to get the bytenr/length for + * extent whose references are incremented. + * + * @extent_op Pointer to a structure, holding information necessary when + * updating a tree block's flags + * + */ +static int __btrfs_inc_extent_ref(struct btrfs_trans_handle *trans, + const struct btrfs_delayed_ref_node *node, + struct btrfs_delayed_extent_op *extent_op) +{ + BTRFS_PATH_AUTO_FREE(path); + struct extent_buffer *leaf; + struct btrfs_extent_item *item; + struct btrfs_key key; + u64 bytenr = node->bytenr; + u64 num_bytes = node->num_bytes; + u64 owner = btrfs_delayed_ref_owner(node); + u64 offset = btrfs_delayed_ref_offset(node); + u64 refs; + int refs_to_add = node->ref_mod; + int ret; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + /* this will setup the path even if it fails to insert the back ref */ + ret = insert_inline_extent_backref(trans, path, bytenr, num_bytes, + node->parent, node->ref_root, owner, + offset, refs_to_add, extent_op); + if ((ret < 0 && ret != -EAGAIN) || !ret) + return ret; + + /* + * Ok we had -EAGAIN which means we didn't have space to insert and + * inline extent ref, so just update the reference count and add a + * normal backref. + */ + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); + refs = btrfs_extent_refs(leaf, item); + btrfs_set_extent_refs(leaf, item, refs + refs_to_add); + if (extent_op) + __run_delayed_extent_op(extent_op, leaf, item); + + btrfs_release_path(path); + + /* now insert the actual backref */ + if (owner < BTRFS_FIRST_FREE_OBJECTID) { + ret = insert_tree_block_ref(trans, path, node, bytenr); + if (ret) + btrfs_abort_transaction(trans, ret); + } else { + ret = insert_extent_data_ref(trans, path, node, bytenr); + if (ret) + btrfs_abort_transaction(trans, ret); + } + + return ret; +} + +static void free_head_ref_squota_rsv(struct btrfs_fs_info *fs_info, + const struct btrfs_delayed_ref_head *href) +{ + u64 root = href->owning_root; + + /* + * Don't check must_insert_reserved, as this is called from contexts + * where it has already been unset. + */ + if (btrfs_qgroup_mode(fs_info) != BTRFS_QGROUP_MODE_SIMPLE || + !href->is_data || !btrfs_is_fstree(root)) + return; + + btrfs_qgroup_free_refroot(fs_info, root, href->reserved_bytes, + BTRFS_QGROUP_RSV_DATA); +} + +static int drop_remap_tree_ref(struct btrfs_trans_handle *trans, + const struct btrfs_delayed_ref_node *node) +{ + u64 bytenr = node->bytenr; + u64 num_bytes = node->num_bytes; + int ret; + + ret = btrfs_add_to_free_space_tree(trans, bytenr, num_bytes); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + + ret = btrfs_update_block_group(trans, bytenr, num_bytes, false); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + + return 0; +} + +static int run_delayed_data_ref(struct btrfs_trans_handle *trans, + struct btrfs_delayed_ref_head *href, + const struct btrfs_delayed_ref_node *node, + struct btrfs_delayed_extent_op *extent_op, + bool insert_reserved) +{ + int ret = 0; + u64 parent = 0; + u64 flags = 0; + + trace_run_delayed_data_ref(trans->fs_info, node); + + if (node->type == BTRFS_SHARED_DATA_REF_KEY) + parent = node->parent; + + if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) { + struct btrfs_key key; + struct btrfs_squota_delta delta = { + .root = href->owning_root, + .num_bytes = node->num_bytes, + .is_data = true, + .is_inc = true, + .generation = trans->transid, + }; + u64 owner = btrfs_delayed_ref_owner(node); + u64 offset = btrfs_delayed_ref_offset(node); + + if (extent_op) + flags |= extent_op->flags_to_set; + + key.objectid = node->bytenr; + key.type = BTRFS_EXTENT_ITEM_KEY; + key.offset = node->num_bytes; + + ret = alloc_reserved_file_extent(trans, parent, node->ref_root, + flags, owner, offset, &key, + node->ref_mod, + href->owning_root); + free_head_ref_squota_rsv(trans->fs_info, href); + if (!ret) + ret = btrfs_record_squota_delta(trans->fs_info, &delta); + } else if (node->action == BTRFS_ADD_DELAYED_REF) { + ret = __btrfs_inc_extent_ref(trans, node, extent_op); + } else if (node->action == BTRFS_DROP_DELAYED_REF) { + ret = __btrfs_free_extent(trans, href, node, extent_op); + } else { + BUG(); + } + return ret; +} + +static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op, + struct extent_buffer *leaf, + struct btrfs_extent_item *ei) +{ + u64 flags = btrfs_extent_flags(leaf, ei); + if (extent_op->update_flags) { + flags |= extent_op->flags_to_set; + btrfs_set_extent_flags(leaf, ei, flags); + } + + if (extent_op->update_key) { + struct btrfs_tree_block_info *bi; + BUG_ON(!(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)); + bi = (struct btrfs_tree_block_info *)(ei + 1); + btrfs_set_tree_block_key(leaf, bi, &extent_op->key); + } +} + +static int run_delayed_extent_op(struct btrfs_trans_handle *trans, + const struct btrfs_delayed_ref_head *head, + struct btrfs_delayed_extent_op *extent_op) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *root; + struct btrfs_key key; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_extent_item *ei; + struct extent_buffer *leaf; + u32 item_size; + int ret; + bool metadata = true; + + if (TRANS_ABORTED(trans)) + return 0; + + if (!btrfs_fs_incompat(fs_info, SKINNY_METADATA)) + metadata = false; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = head->bytenr; + + if (metadata) { + key.type = BTRFS_METADATA_ITEM_KEY; + key.offset = head->level; + } else { + key.type = BTRFS_EXTENT_ITEM_KEY; + key.offset = head->num_bytes; + } + + root = btrfs_extent_root(fs_info, key.objectid); + if (unlikely(!root)) { + btrfs_err(fs_info, + "missing extent root for extent at bytenr %llu", + key.objectid); + return -EUCLEAN; + } +again: + ret = btrfs_search_slot(trans, root, &key, path, 0, 1); + if (ret < 0) { + return ret; + } else if (ret > 0) { + if (metadata) { + if (path->slots[0] > 0) { + path->slots[0]--; + btrfs_item_key_to_cpu(path->nodes[0], &key, + path->slots[0]); + if (key.objectid == head->bytenr && + key.type == BTRFS_EXTENT_ITEM_KEY && + key.offset == head->num_bytes) + ret = 0; + } + if (ret > 0) { + btrfs_release_path(path); + metadata = false; + + key.objectid = head->bytenr; + key.type = BTRFS_EXTENT_ITEM_KEY; + key.offset = head->num_bytes; + goto again; + } + } else { + ret = -EUCLEAN; + btrfs_err(fs_info, + "missing extent item for extent %llu num_bytes %llu level %d", + head->bytenr, head->num_bytes, head->level); + return ret; + } + } + + leaf = path->nodes[0]; + item_size = btrfs_item_size(leaf, path->slots[0]); + + if (unlikely(item_size < sizeof(*ei))) { + ret = -EUCLEAN; + btrfs_err(fs_info, + "unexpected extent item size, has %u expect >= %zu", + item_size, sizeof(*ei)); + btrfs_abort_transaction(trans, ret); + return ret; + } + + ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); + __run_delayed_extent_op(extent_op, leaf, ei); + + return ret; +} + +static int run_delayed_tree_ref(struct btrfs_trans_handle *trans, + struct btrfs_delayed_ref_head *href, + const struct btrfs_delayed_ref_node *node, + struct btrfs_delayed_extent_op *extent_op, + bool insert_reserved) +{ + int ret = 0; + struct btrfs_fs_info *fs_info = trans->fs_info; + u64 parent = 0; + u64 ref_root = 0; + + trace_run_delayed_tree_ref(trans->fs_info, node); + + if (node->type == BTRFS_SHARED_BLOCK_REF_KEY) + parent = node->parent; + ref_root = node->ref_root; + + if (unlikely(node->ref_mod != 1)) { + btrfs_err(trans->fs_info, + "btree block %llu has %d references rather than 1: action %d ref_root %llu parent %llu", + node->bytenr, node->ref_mod, node->action, ref_root, + parent); + return -EUCLEAN; + } + if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) { + struct btrfs_squota_delta delta = { + .root = href->owning_root, + .num_bytes = fs_info->nodesize, + .is_data = false, + .is_inc = true, + .generation = trans->transid, + }; + + ret = alloc_reserved_tree_block(trans, node, extent_op); + if (!ret) + btrfs_record_squota_delta(fs_info, &delta); + } else if (node->action == BTRFS_ADD_DELAYED_REF) { + ret = __btrfs_inc_extent_ref(trans, node, extent_op); + } else if (node->action == BTRFS_DROP_DELAYED_REF) { + if (node->ref_root == BTRFS_REMAP_TREE_OBJECTID) + ret = drop_remap_tree_ref(trans, node); + else + ret = __btrfs_free_extent(trans, href, node, extent_op); + } else { + BUG(); + } + return ret; +} + +/* helper function to actually process a single delayed ref entry */ +static int run_one_delayed_ref(struct btrfs_trans_handle *trans, + struct btrfs_delayed_ref_head *href, + const struct btrfs_delayed_ref_node *node, + struct btrfs_delayed_extent_op *extent_op, + bool insert_reserved) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + int ret = 0; + + if (TRANS_ABORTED(trans)) { + if (insert_reserved) { + btrfs_pin_extent(trans, node->bytenr, node->num_bytes); + free_head_ref_squota_rsv(fs_info, href); + } + return 0; + } + + if (node->type == BTRFS_TREE_BLOCK_REF_KEY || + node->type == BTRFS_SHARED_BLOCK_REF_KEY) { + ret = run_delayed_tree_ref(trans, href, node, extent_op, + insert_reserved); + } else if (node->type == BTRFS_EXTENT_DATA_REF_KEY || + node->type == BTRFS_SHARED_DATA_REF_KEY) { + ret = run_delayed_data_ref(trans, href, node, extent_op, + insert_reserved); + } else if (unlikely(node->type != BTRFS_EXTENT_OWNER_REF_KEY)) { + ret = -EUCLEAN; + btrfs_err(fs_info, "unexpected delayed ref node type: %u", node->type); + } + + if (unlikely(ret)) { + if (insert_reserved) + btrfs_pin_extent(trans, node->bytenr, node->num_bytes); + btrfs_err(fs_info, +"failed to run delayed ref for logical %llu num_bytes %llu type %u action %u ref_mod %d: %d", + node->bytenr, node->num_bytes, node->type, + node->action, node->ref_mod, ret); + } + + return ret; +} + +static struct btrfs_delayed_extent_op *cleanup_extent_op( + struct btrfs_delayed_ref_head *head) +{ + struct btrfs_delayed_extent_op *extent_op = head->extent_op; + + if (!extent_op) + return NULL; + + if (head->must_insert_reserved) { + head->extent_op = NULL; + btrfs_free_delayed_extent_op(extent_op); + return NULL; + } + return extent_op; +} + +static int run_and_cleanup_extent_op(struct btrfs_trans_handle *trans, + struct btrfs_delayed_ref_head *head) +{ + struct btrfs_delayed_extent_op *extent_op; + int ret; + + extent_op = cleanup_extent_op(head); + if (!extent_op) + return 0; + head->extent_op = NULL; + spin_unlock(&head->lock); + ret = run_delayed_extent_op(trans, head, extent_op); + btrfs_free_delayed_extent_op(extent_op); + return ret ? ret : 1; +} + +u64 btrfs_cleanup_ref_head_accounting(struct btrfs_fs_info *fs_info, + struct btrfs_delayed_ref_root *delayed_refs, + struct btrfs_delayed_ref_head *head) +{ + u64 ret = 0; + + /* + * We had csum deletions accounted for in our delayed refs rsv, we need + * to drop the csum leaves for this update from our delayed_refs_rsv. + */ + if (head->total_ref_mod < 0 && head->is_data) { + int nr_csums; + + spin_lock(&delayed_refs->lock); + delayed_refs->pending_csums -= head->num_bytes; + spin_unlock(&delayed_refs->lock); + nr_csums = btrfs_csum_bytes_to_leaves(fs_info, head->num_bytes); + + btrfs_delayed_refs_rsv_release(fs_info, 0, nr_csums); + + ret = btrfs_calc_delayed_ref_csum_bytes(fs_info, nr_csums); + } + /* must_insert_reserved can be set only if we didn't run the head ref. */ + if (head->must_insert_reserved) + free_head_ref_squota_rsv(fs_info, head); + + return ret; +} + +static int cleanup_ref_head(struct btrfs_trans_handle *trans, + struct btrfs_delayed_ref_head *head, + u64 *bytes_released) +{ + + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_delayed_ref_root *delayed_refs; + int ret; + + delayed_refs = &trans->transaction->delayed_refs; + + ret = run_and_cleanup_extent_op(trans, head); + if (ret < 0) { + btrfs_unselect_ref_head(delayed_refs, head); + btrfs_debug(fs_info, "run_delayed_extent_op returned %d", ret); + return ret; + } else if (ret) { + return ret; + } + + /* + * Need to drop our head ref lock and re-acquire the delayed ref lock + * and then re-check to make sure nobody got added. + */ + spin_unlock(&head->lock); + spin_lock(&delayed_refs->lock); + spin_lock(&head->lock); + if (!RB_EMPTY_ROOT(&head->ref_tree.rb_root) || head->extent_op) { + spin_unlock(&head->lock); + spin_unlock(&delayed_refs->lock); + return 1; + } + btrfs_delete_ref_head(fs_info, delayed_refs, head); + spin_unlock(&head->lock); + spin_unlock(&delayed_refs->lock); + + if (head->must_insert_reserved) { + btrfs_pin_extent(trans, head->bytenr, head->num_bytes); + if (head->is_data) { + struct btrfs_root *csum_root; + + csum_root = btrfs_csum_root(fs_info, head->bytenr); + if (unlikely(!csum_root)) { + btrfs_err(fs_info, + "missing csum root for extent at bytenr %llu", + head->bytenr); + ret = -EUCLEAN; + } else { + ret = btrfs_del_csums(trans, csum_root, head->bytenr, + head->num_bytes); + } + } + } + + *bytes_released += btrfs_cleanup_ref_head_accounting(fs_info, delayed_refs, head); + + trace_run_delayed_ref_head(fs_info, head, 0); + btrfs_delayed_ref_unlock(head); + btrfs_put_delayed_ref_head(head); + return ret; +} + +static int btrfs_run_delayed_refs_for_head(struct btrfs_trans_handle *trans, + struct btrfs_delayed_ref_head *locked_ref, + u64 *bytes_released) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_delayed_ref_root *delayed_refs; + struct btrfs_delayed_extent_op *extent_op; + struct btrfs_delayed_ref_node *ref; + bool must_insert_reserved; + int ret; + + delayed_refs = &trans->transaction->delayed_refs; + + lockdep_assert_held(&locked_ref->mutex); + lockdep_assert_held(&locked_ref->lock); + + while ((ref = btrfs_select_delayed_ref(locked_ref))) { + if (ref->seq && + btrfs_check_delayed_seq(fs_info, ref->seq)) { + spin_unlock(&locked_ref->lock); + btrfs_unselect_ref_head(delayed_refs, locked_ref); + return -EAGAIN; + } + + rb_erase_cached(&ref->ref_node, &locked_ref->ref_tree); + RB_CLEAR_NODE(&ref->ref_node); + if (!list_empty(&ref->add_list)) + list_del(&ref->add_list); + /* + * When we play the delayed ref, also correct the ref_mod on + * head + */ + switch (ref->action) { + case BTRFS_ADD_DELAYED_REF: + case BTRFS_ADD_DELAYED_EXTENT: + locked_ref->ref_mod -= ref->ref_mod; + break; + case BTRFS_DROP_DELAYED_REF: + locked_ref->ref_mod += ref->ref_mod; + break; + default: + WARN_ON(1); + } + + /* + * Record the must_insert_reserved flag before we drop the + * spin lock. + */ + must_insert_reserved = locked_ref->must_insert_reserved; + /* + * Unsetting this on the head ref relinquishes ownership of + * the rsv_bytes, so it is critical that every possible code + * path from here forward frees all reserves including qgroup + * reserve. + */ + locked_ref->must_insert_reserved = false; + + extent_op = locked_ref->extent_op; + locked_ref->extent_op = NULL; + spin_unlock(&locked_ref->lock); + + ret = run_one_delayed_ref(trans, locked_ref, ref, extent_op, + must_insert_reserved); + btrfs_delayed_refs_rsv_release(fs_info, 1, 0); + *bytes_released += btrfs_calc_delayed_ref_bytes(fs_info, 1); + + btrfs_free_delayed_extent_op(extent_op); + if (ret) { + btrfs_unselect_ref_head(delayed_refs, locked_ref); + btrfs_put_delayed_ref(ref); + return ret; + } + + btrfs_put_delayed_ref(ref); + cond_resched(); + + spin_lock(&locked_ref->lock); + btrfs_merge_delayed_refs(fs_info, delayed_refs, locked_ref); + } + + return 0; +} + +/* + * Returns 0 on success or if called with an already aborted transaction. + * Returns -ENOMEM or -EIO on failure and will abort the transaction. + */ +static noinline int __btrfs_run_delayed_refs(struct btrfs_trans_handle *trans, + u64 min_bytes) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_delayed_ref_root *delayed_refs; + struct btrfs_delayed_ref_head *locked_ref = NULL; + int ret; + unsigned long count = 0; + unsigned long max_count = 0; + u64 bytes_processed = 0; + + delayed_refs = &trans->transaction->delayed_refs; + if (min_bytes == 0) { + /* + * We may be subject to a harmless race if some task is + * concurrently adding or removing a delayed ref, so silence + * KCSAN and similar tools. + */ + max_count = data_race(delayed_refs->num_heads_ready); + min_bytes = U64_MAX; + } + + do { + if (!locked_ref) { + locked_ref = btrfs_select_ref_head(fs_info, delayed_refs); + if (IS_ERR_OR_NULL(locked_ref)) { + if (PTR_ERR(locked_ref) == -EAGAIN) { + count++; + goto again; + } else { + break; + } + } + count++; + } + /* + * We need to try and merge add/drops of the same ref since we + * can run into issues with relocate dropping the implicit ref + * and then it being added back again before the drop can + * finish. If we merged anything we need to re-loop so we can + * get a good ref. + * Or we can get node references of the same type that weren't + * merged when created due to bumps in the tree mod seq, and + * we need to merge them to prevent adding an inline extent + * backref before dropping it (triggering a BUG_ON at + * insert_inline_extent_backref()). + */ + spin_lock(&locked_ref->lock); + btrfs_merge_delayed_refs(fs_info, delayed_refs, locked_ref); + + ret = btrfs_run_delayed_refs_for_head(trans, locked_ref, &bytes_processed); + if (ret < 0 && ret != -EAGAIN) { + /* + * Error, btrfs_run_delayed_refs_for_head already + * unlocked everything so just bail out + */ + return ret; + } else if (!ret) { + /* + * Success, perform the usual cleanup of a processed + * head + */ + ret = cleanup_ref_head(trans, locked_ref, &bytes_processed); + if (ret > 0 ) { + /* We dropped our lock, we need to loop. */ + ret = 0; + continue; + } else if (ret) { + return ret; + } + } + + /* + * Either success case or btrfs_run_delayed_refs_for_head + * returned -EAGAIN, meaning we need to select another head + */ +again: + locked_ref = NULL; + cond_resched(); + } while ((min_bytes != U64_MAX && bytes_processed < min_bytes) || + (max_count > 0 && count < max_count) || + locked_ref); + + return 0; +} + +#ifdef SCRAMBLE_DELAYED_REFS +/* + * Normally delayed refs get processed in ascending bytenr order. This + * correlates in most cases to the order added. To expose dependencies on this + * order, we start to process the tree in the middle instead of the beginning + */ +static u64 find_middle(struct rb_root *root) +{ + struct rb_node *n = root->rb_node; + struct btrfs_delayed_ref_node *entry; + int alt = 1; + u64 middle; + u64 first = 0, last = 0; + + n = rb_first(root); + if (n) { + entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node); + first = entry->bytenr; + } + n = rb_last(root); + if (n) { + entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node); + last = entry->bytenr; + } + n = root->rb_node; + + while (n) { + entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node); + WARN_ON(!entry->in_tree); + + middle = entry->bytenr; + + if (alt) + n = n->rb_left; + else + n = n->rb_right; + + alt = 1 - alt; + } + return middle; +} +#endif + +/* + * Start processing the delayed reference count updates and extent insertions + * we have queued up so far. + * + * @trans: Transaction handle. + * @min_bytes: How many bytes of delayed references to process. After this + * many bytes we stop processing delayed references if there are + * any more. If 0 it means to run all existing delayed references, + * but not new ones added after running all existing ones. + * Use (u64)-1 (U64_MAX) to run all existing delayed references + * plus any new ones that are added. + * + * Returns 0 on success or if called with an aborted transaction + * Returns <0 on error and aborts the transaction + */ +int btrfs_run_delayed_refs(struct btrfs_trans_handle *trans, u64 min_bytes) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_delayed_ref_root *delayed_refs; + int ret; + + /* We'll clean this up in btrfs_cleanup_transaction */ + if (TRANS_ABORTED(trans)) + return 0; + + if (test_bit(BTRFS_FS_CREATING_FREE_SPACE_TREE, &fs_info->flags)) + return 0; + + delayed_refs = &trans->transaction->delayed_refs; +again: +#ifdef SCRAMBLE_DELAYED_REFS + delayed_refs->run_delayed_start = find_middle(&delayed_refs->root); +#endif + ret = __btrfs_run_delayed_refs(trans, min_bytes); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + + if (min_bytes == U64_MAX) { + btrfs_create_pending_block_groups(trans); + + spin_lock(&delayed_refs->lock); + if (xa_empty(&delayed_refs->head_refs)) { + spin_unlock(&delayed_refs->lock); + return 0; + } + spin_unlock(&delayed_refs->lock); + + cond_resched(); + goto again; + } + + return 0; +} + +int btrfs_set_disk_extent_flags(struct btrfs_trans_handle *trans, + struct extent_buffer *eb, u64 flags) +{ + struct btrfs_delayed_extent_op *extent_op; + int ret; + + extent_op = btrfs_alloc_delayed_extent_op(); + if (!extent_op) + return -ENOMEM; + + extent_op->flags_to_set = flags; + extent_op->update_flags = true; + extent_op->update_key = false; + + ret = btrfs_add_delayed_extent_op(trans, eb->start, eb->len, + btrfs_header_level(eb), extent_op); + if (ret) + btrfs_free_delayed_extent_op(extent_op); + return ret; +} + +static noinline int check_delayed_ref(struct btrfs_inode *inode, + struct btrfs_path *path, + u64 offset, u64 bytenr) +{ + struct btrfs_root *root = inode->root; + struct btrfs_delayed_ref_head *head; + struct btrfs_delayed_ref_node *ref; + struct btrfs_delayed_ref_root *delayed_refs; + struct btrfs_transaction *cur_trans; + struct rb_node *node; + int ret = 0; + + spin_lock(&root->fs_info->trans_lock); + cur_trans = root->fs_info->running_transaction; + if (cur_trans) + refcount_inc(&cur_trans->use_count); + spin_unlock(&root->fs_info->trans_lock); + if (!cur_trans) + return 0; + + delayed_refs = &cur_trans->delayed_refs; + spin_lock(&delayed_refs->lock); + head = btrfs_find_delayed_ref_head(root->fs_info, delayed_refs, bytenr); + if (!head) { + spin_unlock(&delayed_refs->lock); + btrfs_put_transaction(cur_trans); + return 0; + } + + if (!mutex_trylock(&head->mutex)) { + if (path->nowait) { + spin_unlock(&delayed_refs->lock); + btrfs_put_transaction(cur_trans); + return -EAGAIN; + } + + refcount_inc(&head->refs); + spin_unlock(&delayed_refs->lock); + + btrfs_release_path(path); + + /* + * Mutex was contended, block until it's released and let + * caller try again + */ + mutex_lock(&head->mutex); + mutex_unlock(&head->mutex); + btrfs_put_delayed_ref_head(head); + btrfs_put_transaction(cur_trans); + return -EAGAIN; + } + spin_unlock(&delayed_refs->lock); + + spin_lock(&head->lock); + /* + * XXX: We should replace this with a proper search function in the + * future. + */ + for (node = rb_first_cached(&head->ref_tree); node; + node = rb_next(node)) { + u64 ref_owner; + u64 ref_offset; + + ref = rb_entry(node, struct btrfs_delayed_ref_node, ref_node); + /* If it's a shared ref we know a cross reference exists */ + if (ref->type != BTRFS_EXTENT_DATA_REF_KEY) { + ret = 1; + break; + } + + ref_owner = btrfs_delayed_ref_owner(ref); + ref_offset = btrfs_delayed_ref_offset(ref); + + /* + * If our ref doesn't match the one we're currently looking at + * then we have a cross reference. + */ + if (ref->ref_root != btrfs_root_id(root) || + ref_owner != btrfs_ino(inode) || ref_offset != offset) { + ret = 1; + break; + } + } + spin_unlock(&head->lock); + mutex_unlock(&head->mutex); + btrfs_put_transaction(cur_trans); + return ret; +} + +/* + * Check if there are references for a data extent other than the one belonging + * to the given inode and offset. + * + * @inode: The only inode we expect to find associated with the data extent. + * @path: A path to use for searching the extent tree. + * @offset: The only offset we expect to find associated with the data extent. + * @bytenr: The logical address of the data extent. + * + * When the extent does not have any other references other than the one we + * expect to find, we always return a value of 0 with the path having a locked + * leaf that contains the extent's extent item - this is necessary to ensure + * we don't race with a task running delayed references, and our caller must + * have such a path when calling check_delayed_ref() - it must lock a delayed + * ref head while holding the leaf locked. In case the extent item is not found + * in the extent tree, we return -ENOENT with the path having the leaf (locked) + * where the extent item should be, in order to prevent races with another task + * running delayed references, so that we don't miss any reference when calling + * check_delayed_ref(). + * + * Note: this may return false positives, and this is because we want to be + * quick here as we're called in write paths (when flushing delalloc and + * in the direct IO write path). For example we can have an extent with + * a single reference but that reference is not inlined, or we may have + * many references in the extent tree but we also have delayed references + * that cancel all the reference except the one for our inode and offset, + * but it would be expensive to do such checks and complex due to all + * locking to avoid races between the checks and flushing delayed refs, + * plus non-inline references may be located on leaves other than the one + * that contains the extent item in the extent tree. The important thing + * here is to not return false negatives and that the false positives are + * not very common. + * + * Returns: 0 if there are no cross references and with the path having a locked + * leaf from the extent tree that contains the extent's extent item. + * + * 1 if there are cross references (false positives can happen). + * + * < 0 in case of an error. In case of -ENOENT the leaf in the extent + * tree where the extent item should be located at is read locked and + * accessible in the given path. + */ +static noinline int check_committed_ref(struct btrfs_inode *inode, + struct btrfs_path *path, + u64 offset, u64 bytenr) +{ + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_root *extent_root = btrfs_extent_root(fs_info, bytenr); + struct extent_buffer *leaf; + struct btrfs_extent_data_ref *ref; + struct btrfs_extent_inline_ref *iref; + struct btrfs_extent_item *ei; + struct btrfs_key key; + u32 item_size; + u32 expected_size; + int type; + int ret; + + if (unlikely(!extent_root)) { + btrfs_err(fs_info, + "missing extent root for extent at bytenr %llu", bytenr); + return -EUCLEAN; + } + + key.objectid = bytenr; + key.type = BTRFS_EXTENT_ITEM_KEY; + key.offset = (u64)-1; + + ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0); + if (ret < 0) + return ret; + if (unlikely(ret == 0)) { + /* + * Key with offset -1 found, there would have to exist an extent + * item with such offset, but this is out of the valid range. + */ + return -EUCLEAN; + } + + if (path->slots[0] == 0) + return -ENOENT; + + path->slots[0]--; + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + + if (key.objectid != bytenr || key.type != BTRFS_EXTENT_ITEM_KEY) + return -ENOENT; + + item_size = btrfs_item_size(leaf, path->slots[0]); + ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); + expected_size = sizeof(*ei) + btrfs_extent_inline_ref_size(BTRFS_EXTENT_DATA_REF_KEY); + + /* No inline refs; we need to bail before checking for owner ref. */ + if (item_size == sizeof(*ei)) + return 1; + + /* Check for an owner ref; skip over it to the real inline refs. */ + iref = (struct btrfs_extent_inline_ref *)(ei + 1); + type = btrfs_get_extent_inline_ref_type(leaf, iref, BTRFS_REF_TYPE_DATA); + if (btrfs_fs_incompat(fs_info, SIMPLE_QUOTA) && type == BTRFS_EXTENT_OWNER_REF_KEY) { + expected_size += btrfs_extent_inline_ref_size(BTRFS_EXTENT_OWNER_REF_KEY); + iref = (struct btrfs_extent_inline_ref *)(iref + 1); + type = btrfs_get_extent_inline_ref_type(leaf, iref, BTRFS_REF_TYPE_DATA); + } + + /* If extent item has more than 1 inline ref then it's shared */ + if (item_size != expected_size) + return 1; + + /* If this extent has SHARED_DATA_REF then it's shared */ + if (type != BTRFS_EXTENT_DATA_REF_KEY) + return 1; + + ref = (struct btrfs_extent_data_ref *)(&iref->offset); + if (btrfs_extent_refs(leaf, ei) != + btrfs_extent_data_ref_count(leaf, ref) || + btrfs_extent_data_ref_root(leaf, ref) != btrfs_root_id(root) || + btrfs_extent_data_ref_objectid(leaf, ref) != btrfs_ino(inode) || + btrfs_extent_data_ref_offset(leaf, ref) != offset) + return 1; + + return 0; +} + +int btrfs_cross_ref_exist(struct btrfs_inode *inode, u64 offset, + u64 bytenr, struct btrfs_path *path) +{ + int ret; + + do { + ret = check_committed_ref(inode, path, offset, bytenr); + if (ret && ret != -ENOENT) + goto out; + + /* + * The path must have a locked leaf from the extent tree where + * the extent item for our extent is located, in case it exists, + * or where it should be located in case it doesn't exist yet + * because it's new and its delayed ref was not yet flushed. + * We need to lock the delayed ref head at check_delayed_ref(), + * if one exists, while holding the leaf locked in order to not + * race with delayed ref flushing, missing references and + * incorrectly reporting that the extent is not shared. + */ + if (IS_ENABLED(CONFIG_BTRFS_ASSERT)) { + struct extent_buffer *leaf = path->nodes[0]; + + ASSERT(leaf != NULL); + btrfs_assert_tree_read_locked(leaf); + + if (ret != -ENOENT) { + struct btrfs_key key; + + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + ASSERT(key.objectid == bytenr, + "key.objectid=%llu bytenr=%llu", + key.objectid, bytenr); + ASSERT(key.type == BTRFS_EXTENT_ITEM_KEY, "key.type=%u", + key.type); + } + } + + ret = check_delayed_ref(inode, path, offset, bytenr); + } while (ret == -EAGAIN && !path->nowait); + +out: + btrfs_release_path(path); + if (btrfs_is_data_reloc_root(inode->root)) + WARN_ON(ret > 0); + return ret; +} + +static int __btrfs_mod_ref(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct extent_buffer *buf, + bool full_backref, bool inc) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + u64 parent; + u64 ref_root; + u32 nritems; + struct btrfs_key key; + struct btrfs_file_extent_item *fi; + bool for_reloc = btrfs_header_flag(buf, BTRFS_HEADER_FLAG_RELOC); + int i; + int action; + int level; + int ret; + + if (btrfs_is_testing(fs_info)) + return 0; + + ref_root = btrfs_header_owner(buf); + nritems = btrfs_header_nritems(buf); + level = btrfs_header_level(buf); + + if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && level == 0) + return 0; + + if (full_backref) + parent = buf->start; + else + parent = 0; + if (inc) + action = BTRFS_ADD_DELAYED_REF; + else + action = BTRFS_DROP_DELAYED_REF; + + for (i = 0; i < nritems; i++) { + struct btrfs_ref ref = { + .action = action, + .parent = parent, + .ref_root = ref_root, + }; + + if (level == 0) { + btrfs_item_key_to_cpu(buf, &key, i); + if (key.type != BTRFS_EXTENT_DATA_KEY) + continue; + fi = btrfs_item_ptr(buf, i, + struct btrfs_file_extent_item); + if (btrfs_file_extent_type(buf, fi) == + BTRFS_FILE_EXTENT_INLINE) + continue; + ref.bytenr = btrfs_file_extent_disk_bytenr(buf, fi); + if (ref.bytenr == 0) + continue; + + ref.num_bytes = btrfs_file_extent_disk_num_bytes(buf, fi); + ref.owning_root = ref_root; + + key.offset -= btrfs_file_extent_offset(buf, fi); + btrfs_init_data_ref(&ref, key.objectid, key.offset, + btrfs_root_id(root), for_reloc); + if (inc) + ret = btrfs_inc_extent_ref(trans, &ref); + else + ret = btrfs_free_extent(trans, &ref); + if (ret) + return ret; + } else { + /* We don't know the owning_root, leave as 0. */ + ref.bytenr = btrfs_node_blockptr(buf, i); + ref.num_bytes = fs_info->nodesize; + + btrfs_init_tree_ref(&ref, level - 1, + btrfs_root_id(root), for_reloc); + if (inc) + ret = btrfs_inc_extent_ref(trans, &ref); + else + ret = btrfs_free_extent(trans, &ref); + if (ret) + return ret; + } + } + return 0; +} + +int btrfs_inc_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root, + struct extent_buffer *buf, bool full_backref) +{ + return __btrfs_mod_ref(trans, root, buf, full_backref, true); +} + +int btrfs_dec_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root, + struct extent_buffer *buf, bool full_backref) +{ + return __btrfs_mod_ref(trans, root, buf, full_backref, false); +} + +static u64 get_alloc_profile_by_root(struct btrfs_root *root, int data) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + u64 flags; + + if (data) + flags = BTRFS_BLOCK_GROUP_DATA; + else if (root == fs_info->chunk_root) + flags = BTRFS_BLOCK_GROUP_SYSTEM; + else if (root == fs_info->remap_root) + flags = BTRFS_BLOCK_GROUP_METADATA_REMAP; + else + flags = BTRFS_BLOCK_GROUP_METADATA; + + return btrfs_get_alloc_profile(fs_info, flags); +} + +static u64 first_logical_byte(struct btrfs_fs_info *fs_info) +{ + struct rb_node *leftmost; + u64 bytenr = 0; + + read_lock(&fs_info->block_group_cache_lock); + /* Get the block group with the lowest logical start address. */ + leftmost = rb_first_cached(&fs_info->block_group_cache_tree); + if (leftmost) { + struct btrfs_block_group *bg; + + bg = rb_entry(leftmost, struct btrfs_block_group, cache_node); + bytenr = bg->start; + } + read_unlock(&fs_info->block_group_cache_lock); + + return bytenr; +} + +static int pin_down_extent(struct btrfs_trans_handle *trans, + struct btrfs_block_group *bg, + u64 bytenr, u64 num_bytes, bool reserved) +{ + struct btrfs_space_info *space_info = bg->space_info; + const u64 reserved_bytes = (reserved ? num_bytes : 0); + + spin_lock(&space_info->lock); + spin_lock(&bg->lock); + bg->pinned += num_bytes; + bg->reserved -= reserved_bytes; + spin_unlock(&bg->lock); + space_info->bytes_reserved -= reserved_bytes; + btrfs_space_info_update_bytes_pinned(space_info, num_bytes); + spin_unlock(&space_info->lock); + + btrfs_set_extent_bit(&trans->transaction->pinned_extents, bytenr, + bytenr + num_bytes - 1, EXTENT_DIRTY, NULL); + return 0; +} + +int btrfs_pin_extent(struct btrfs_trans_handle *trans, u64 bytenr, u64 num_bytes) +{ + struct btrfs_block_group *cache; + + cache = btrfs_lookup_block_group(trans->fs_info, bytenr); + BUG_ON(!cache); /* Logic error */ + + pin_down_extent(trans, cache, bytenr, num_bytes, true); + + btrfs_put_block_group(cache); + return 0; +} + +int btrfs_pin_extent_for_log_replay(struct btrfs_trans_handle *trans, + const struct extent_buffer *eb) +{ + struct btrfs_block_group *cache; + int ret; + + cache = btrfs_lookup_block_group(trans->fs_info, eb->start); + if (!cache) + return -EINVAL; + + /* + * Fully cache the free space first so that our pin removes the free space + * from the cache. + */ + ret = btrfs_cache_block_group(cache, true); + if (ret) + goto out; + + pin_down_extent(trans, cache, eb->start, eb->len, false); + + /* remove us from the free space cache (if we're there at all) */ + ret = btrfs_remove_free_space(cache, eb->start, eb->len); +out: + btrfs_put_block_group(cache); + return ret; +} + +static int __exclude_logged_extent(struct btrfs_fs_info *fs_info, + u64 start, u64 num_bytes) +{ + int ret; + struct btrfs_block_group *block_group; + + block_group = btrfs_lookup_block_group(fs_info, start); + if (!block_group) + return -EINVAL; + + ret = btrfs_cache_block_group(block_group, true); + if (ret) + goto out; + + ret = btrfs_remove_free_space(block_group, start, num_bytes); +out: + btrfs_put_block_group(block_group); + return ret; +} + +int btrfs_exclude_logged_extents(struct extent_buffer *eb) +{ + struct btrfs_fs_info *fs_info = eb->fs_info; + struct btrfs_file_extent_item *item; + struct btrfs_key key; + int found_type; + int i; + int ret = 0; + + if (!btrfs_fs_incompat(fs_info, MIXED_GROUPS)) + return 0; + + for (i = 0; i < btrfs_header_nritems(eb); i++) { + btrfs_item_key_to_cpu(eb, &key, i); + if (key.type != BTRFS_EXTENT_DATA_KEY) + continue; + item = btrfs_item_ptr(eb, i, struct btrfs_file_extent_item); + found_type = btrfs_file_extent_type(eb, item); + if (found_type == BTRFS_FILE_EXTENT_INLINE) + continue; + if (btrfs_file_extent_disk_bytenr(eb, item) == 0) + continue; + key.objectid = btrfs_file_extent_disk_bytenr(eb, item); + key.offset = btrfs_file_extent_disk_num_bytes(eb, item); + ret = __exclude_logged_extent(fs_info, key.objectid, key.offset); + if (ret) + break; + } + + return ret; +} + +static void +btrfs_inc_block_group_reservations(struct btrfs_block_group *bg) +{ + atomic_inc(&bg->reservations); +} + +/* + * Returns the free cluster for the given space info and sets empty_cluster to + * what it should be based on the mount options. + */ +static struct btrfs_free_cluster * +fetch_cluster_info(struct btrfs_fs_info *fs_info, + struct btrfs_space_info *space_info, u64 *empty_cluster) +{ + struct btrfs_free_cluster *ret = NULL; + + *empty_cluster = 0; + if (btrfs_mixed_space_info(space_info)) + return ret; + + if (space_info->flags & BTRFS_BLOCK_GROUP_METADATA) { + ret = &fs_info->meta_alloc_cluster; + if (btrfs_test_opt(fs_info, SSD)) + *empty_cluster = SZ_2M; + else + *empty_cluster = SZ_64K; + } else if ((space_info->flags & BTRFS_BLOCK_GROUP_DATA) && + btrfs_test_opt(fs_info, SSD_SPREAD)) { + *empty_cluster = SZ_2M; + ret = &fs_info->data_alloc_cluster; + } + + return ret; +} + +static int unpin_extent_range(struct btrfs_fs_info *fs_info, + u64 start, u64 end, + const bool return_free_space) +{ + struct btrfs_block_group *cache = NULL; + struct btrfs_space_info *space_info; + struct btrfs_free_cluster *cluster = NULL; + u64 total_unpinned = 0; + u64 empty_cluster = 0; + + while (start <= end) { + u64 len; + bool readonly; + + if (!cache || start >= btrfs_block_group_end(cache)) { + if (cache) + btrfs_put_block_group(cache); + total_unpinned = 0; + cache = btrfs_lookup_block_group(fs_info, start); + if (unlikely(cache == NULL)) { + /* Logic error, something removed the block group. */ + return -EUCLEAN; + } + + cluster = fetch_cluster_info(fs_info, + cache->space_info, + &empty_cluster); + empty_cluster <<= 1; + } + + len = btrfs_block_group_end(cache) - start; + len = min(len, end + 1 - start); + + if (return_free_space) + btrfs_add_free_space(cache, start, len); + + start += len; + total_unpinned += len; + space_info = cache->space_info; + + /* + * If this space cluster has been marked as fragmented and we've + * unpinned enough in this block group to potentially allow a + * cluster to be created inside of it go ahead and clear the + * fragmented check. + */ + if (cluster && cluster->fragmented && + total_unpinned > empty_cluster) { + spin_lock(&cluster->lock); + cluster->fragmented = 0; + spin_unlock(&cluster->lock); + } + + spin_lock(&space_info->lock); + spin_lock(&cache->lock); + readonly = cache->ro; + cache->pinned -= len; + spin_unlock(&cache->lock); + + btrfs_space_info_update_bytes_pinned(space_info, -len); + space_info->max_extent_size = 0; + + if (readonly) { + space_info->bytes_readonly += len; + } else if (btrfs_is_zoned(fs_info)) { + /* Need reset before reusing in a zoned block group */ + btrfs_space_info_update_bytes_zone_unusable(space_info, len); + } else if (return_free_space) { + btrfs_return_free_space(space_info, len); + } + spin_unlock(&space_info->lock); + } + + if (cache) + btrfs_put_block_group(cache); + + return 0; +} + +/* + * Complete the remapping of a block group by removing its chunk stripes and + * device extents, and adding it to the unused list if there's no longer any + * extents nominally within it. + */ +int btrfs_complete_bg_remapping(struct btrfs_block_group *bg) +{ + struct btrfs_fs_info *fs_info = bg->fs_info; + struct btrfs_chunk_map *map; + int ret; + + map = btrfs_get_chunk_map(fs_info, bg->start, 1); + if (IS_ERR(map)) + return PTR_ERR(map); + + ret = btrfs_last_identity_remap_gone(map, bg); + if (ret) { + btrfs_free_chunk_map(map); + return ret; + } + + /* + * Set num_stripes to 0, so that btrfs_remove_dev_extents() won't run a + * second time. + */ + map->num_stripes = 0; + + btrfs_free_chunk_map(map); + + if (bg->used == 0) { + spin_lock(&fs_info->unused_bgs_lock); + if (!list_empty(&bg->bg_list)) { + list_del_init(&bg->bg_list); + btrfs_put_block_group(bg); + } + spin_unlock(&fs_info->unused_bgs_lock); + + btrfs_mark_bg_unused(bg); + } + + return 0; +} + +void btrfs_handle_fully_remapped_bgs(struct btrfs_fs_info *fs_info) +{ + struct btrfs_block_group *bg; + int ret; + + spin_lock(&fs_info->unused_bgs_lock); + while (!list_empty(&fs_info->fully_remapped_bgs)) { + bg = list_first_entry(&fs_info->fully_remapped_bgs, + struct btrfs_block_group, bg_list); + list_del_init(&bg->bg_list); + spin_unlock(&fs_info->unused_bgs_lock); + + btrfs_discard_extent(fs_info, bg->start, bg->length, NULL, false); + + ret = btrfs_complete_bg_remapping(bg); + if (ret) { + btrfs_put_block_group(bg); + return; + } + + btrfs_put_block_group(bg); + spin_lock(&fs_info->unused_bgs_lock); + } + spin_unlock(&fs_info->unused_bgs_lock); +} + +int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_block_group *block_group, *tmp; + struct list_head *deleted_bgs; + struct extent_io_tree *unpin = &trans->transaction->pinned_extents; + struct extent_state *cached_state = NULL; + u64 start; + u64 end; + int unpin_error = 0; + int ret; + + mutex_lock(&fs_info->unused_bg_unpin_mutex); + btrfs_find_first_extent_bit(unpin, 0, &start, &end, EXTENT_DIRTY, &cached_state); + + while (!TRANS_ABORTED(trans) && cached_state) { + struct extent_state *next_state; + + if (btrfs_test_opt(fs_info, DISCARD_SYNC)) { + ret = btrfs_discard_extent(fs_info, start, + end + 1 - start, NULL, true); + if (ret) { + btrfs_warn(fs_info, + "discard failed for extent [%llu, %llu]: errno=%d %s", + start, end, ret, btrfs_decode_error(ret)); + } + } + + next_state = btrfs_next_extent_state(unpin, cached_state); + btrfs_clear_extent_dirty(unpin, start, end, &cached_state); + ret = unpin_extent_range(fs_info, start, end, true); + /* + * If we get an error unpinning an extent range, store the first + * error to return later after trying to unpin all ranges and do + * the sync discards. Our caller will abort the transaction + * (which already wrote new superblocks) and on the next mount + * the space will be available as it was pinned by in-memory + * only structures in this phase. + */ + if (ret) { + btrfs_err_rl(fs_info, +"failed to unpin extent range [%llu, %llu] when committing transaction %llu: %s (%d)", + start, end, trans->transid, + btrfs_decode_error(ret), ret); + if (!unpin_error) + unpin_error = ret; + } + + btrfs_free_extent_state(cached_state); + + if (need_resched()) { + btrfs_free_extent_state(next_state); + mutex_unlock(&fs_info->unused_bg_unpin_mutex); + cond_resched(); + cached_state = NULL; + mutex_lock(&fs_info->unused_bg_unpin_mutex); + btrfs_find_first_extent_bit(unpin, 0, &start, &end, + EXTENT_DIRTY, &cached_state); + } else { + cached_state = next_state; + if (cached_state) { + start = cached_state->start; + end = cached_state->end; + } + } + } + mutex_unlock(&fs_info->unused_bg_unpin_mutex); + btrfs_free_extent_state(cached_state); + + if (btrfs_test_opt(fs_info, DISCARD_ASYNC)) { + btrfs_discard_calc_delay(&fs_info->discard_ctl); + btrfs_discard_schedule_work(&fs_info->discard_ctl, true); + } + + /* + * Transaction is finished. We don't need the lock anymore. We + * do need to clean up the block groups in case of a transaction + * abort. + */ + deleted_bgs = &trans->transaction->deleted_bgs; + list_for_each_entry_safe(block_group, tmp, deleted_bgs, bg_list) { + ret = -EROFS; + if (!TRANS_ABORTED(trans)) + ret = btrfs_discard_extent(fs_info, block_group->start, + block_group->length, NULL, true); + + /* + * Not strictly necessary to lock, as the block_group should be + * read-only from btrfs_delete_unused_bgs(). + */ + ASSERT(block_group->ro); + spin_lock(&fs_info->unused_bgs_lock); + list_del_init(&block_group->bg_list); + spin_unlock(&fs_info->unused_bgs_lock); + + btrfs_unfreeze_block_group(block_group); + btrfs_put_block_group(block_group); + + if (ret) { + const char *errstr = btrfs_decode_error(ret); + btrfs_warn(fs_info, + "discard failed while removing blockgroup: errno=%d %s", + ret, errstr); + } + } + + return unpin_error; +} + +/* + * Parse an extent item's inline extents looking for a simple quotas owner ref. + * + * @fs_info: the btrfs_fs_info for this mount + * @leaf: a leaf in the extent tree containing the extent item + * @slot: the slot in the leaf where the extent item is found + * + * Returns the objectid of the root that originally allocated the extent item + * if the inline owner ref is expected and present, otherwise 0. + * + * If an extent item has an owner ref item, it will be the first inline ref + * item. Therefore the logic is to check whether there are any inline ref + * items, then check the type of the first one. + */ +u64 btrfs_get_extent_owner_root(struct btrfs_fs_info *fs_info, + struct extent_buffer *leaf, int slot) +{ + struct btrfs_extent_item *ei; + struct btrfs_extent_inline_ref *iref; + struct btrfs_extent_owner_ref *oref; + unsigned long ptr; + unsigned long end; + int type; + + if (!btrfs_fs_incompat(fs_info, SIMPLE_QUOTA)) + return 0; + + ei = btrfs_item_ptr(leaf, slot, struct btrfs_extent_item); + ptr = (unsigned long)(ei + 1); + end = (unsigned long)ei + btrfs_item_size(leaf, slot); + + /* No inline ref items of any kind, can't check type. */ + if (ptr == end) + return 0; + + iref = (struct btrfs_extent_inline_ref *)ptr; + type = btrfs_get_extent_inline_ref_type(leaf, iref, BTRFS_REF_TYPE_ANY); + + /* We found an owner ref, get the root out of it. */ + if (type == BTRFS_EXTENT_OWNER_REF_KEY) { + oref = (struct btrfs_extent_owner_ref *)(&iref->offset); + return btrfs_extent_owner_ref_root_id(leaf, oref); + } + + /* We have inline refs, but not an owner ref. */ + return 0; +} + +static int do_free_extent_accounting(struct btrfs_trans_handle *trans, + u64 bytenr, struct btrfs_squota_delta *delta, + struct btrfs_path *path) +{ + int ret; + bool remapped = false; + u64 num_bytes = delta->num_bytes; + + /* Returns 1 on success and 0 on no-op. */ + ret = btrfs_remove_extent_from_remap_tree(trans, path, bytenr, num_bytes); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + return ret; + } else if (ret == 1) { + remapped = true; + } + + if (delta->is_data) { + struct btrfs_root *csum_root; + + csum_root = btrfs_csum_root(trans->fs_info, bytenr); + if (unlikely(!csum_root)) { + ret = -EUCLEAN; + btrfs_abort_transaction(trans, ret); + btrfs_err(trans->fs_info, + "missing csum root for extent at bytenr %llu", + bytenr); + return ret; + } + + ret = btrfs_del_csums(trans, csum_root, bytenr, num_bytes); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + + ret = btrfs_delete_raid_extent(trans, bytenr, num_bytes); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + } + + ret = btrfs_record_squota_delta(trans->fs_info, delta); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + + /* If remapped, FST has already been taken care of in remove_range_from_remap_tree(). */ + if (!remapped) { + ret = btrfs_add_to_free_space_tree(trans, bytenr, num_bytes); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + } + + ret = btrfs_update_block_group(trans, bytenr, num_bytes, false); + if (ret) + btrfs_abort_transaction(trans, ret); + + return ret; +} + +#define abort_and_dump(trans, path, fmt, args...) \ +({ \ + btrfs_abort_transaction(trans, -EUCLEAN); \ + btrfs_print_leaf(path->nodes[0]); \ + btrfs_crit(trans->fs_info, fmt, ##args); \ +}) + +/* + * Drop one or more refs of @node. + * + * 1. Locate the extent refs. + * It's either inline in EXTENT/METADATA_ITEM or in keyed SHARED_* item. + * Locate it, then reduce the refs number or remove the ref line completely. + * + * 2. Update the refs count in EXTENT/METADATA_ITEM + * + * Inline backref case: + * + * in extent tree we have: + * + * item 0 key (13631488 EXTENT_ITEM 1048576) itemoff 16201 itemsize 82 + * refs 2 gen 6 flags DATA + * extent data backref root FS_TREE objectid 258 offset 0 count 1 + * extent data backref root FS_TREE objectid 257 offset 0 count 1 + * + * This function gets called with: + * + * node->bytenr = 13631488 + * node->num_bytes = 1048576 + * root_objectid = FS_TREE + * owner_objectid = 257 + * owner_offset = 0 + * refs_to_drop = 1 + * + * Then we should get some like: + * + * item 0 key (13631488 EXTENT_ITEM 1048576) itemoff 16201 itemsize 82 + * refs 1 gen 6 flags DATA + * extent data backref root FS_TREE objectid 258 offset 0 count 1 + * + * Keyed backref case: + * + * in extent tree we have: + * + * item 0 key (13631488 EXTENT_ITEM 1048576) itemoff 3971 itemsize 24 + * refs 754 gen 6 flags DATA + * [...] + * item 2 key (13631488 EXTENT_DATA_REF ) itemoff 3915 itemsize 28 + * extent data backref root FS_TREE objectid 866 offset 0 count 1 + * + * This function get called with: + * + * node->bytenr = 13631488 + * node->num_bytes = 1048576 + * root_objectid = FS_TREE + * owner_objectid = 866 + * owner_offset = 0 + * refs_to_drop = 1 + * + * Then we should get some like: + * + * item 0 key (13631488 EXTENT_ITEM 1048576) itemoff 3971 itemsize 24 + * refs 753 gen 6 flags DATA + * + * And that (13631488 EXTENT_DATA_REF ) gets removed. + */ +static int __btrfs_free_extent(struct btrfs_trans_handle *trans, + struct btrfs_delayed_ref_head *href, + const struct btrfs_delayed_ref_node *node, + struct btrfs_delayed_extent_op *extent_op) +{ + struct btrfs_fs_info *info = trans->fs_info; + struct btrfs_key key; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_root *extent_root; + struct extent_buffer *leaf; + struct btrfs_extent_item *ei; + struct btrfs_extent_inline_ref *iref; + int ret; + int is_data; + int extent_slot = 0; + bool found_extent = false; + int num_to_del = 1; + int refs_to_drop = node->ref_mod; + u32 item_size; + u64 refs; + u64 bytenr = node->bytenr; + u64 num_bytes = node->num_bytes; + u64 owner_objectid = btrfs_delayed_ref_owner(node); + u64 owner_offset = btrfs_delayed_ref_offset(node); + bool skinny_metadata = btrfs_fs_incompat(info, SKINNY_METADATA); + u64 delayed_ref_root = href->owning_root; + + extent_root = btrfs_extent_root(info, bytenr); + if (unlikely(!extent_root)) { + btrfs_err(info, + "missing extent root for extent at bytenr %llu", bytenr); + return -EUCLEAN; + } + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + is_data = owner_objectid >= BTRFS_FIRST_FREE_OBJECTID; + + if (unlikely(!is_data && refs_to_drop != 1)) { + btrfs_crit(info, +"invalid refs_to_drop, dropping more than 1 refs for tree block %llu refs_to_drop %u", + node->bytenr, refs_to_drop); + ret = -EINVAL; + btrfs_abort_transaction(trans, ret); + return ret; + } + + if (is_data) + skinny_metadata = false; + + ret = lookup_extent_backref(trans, path, &iref, bytenr, num_bytes, + node->parent, node->ref_root, owner_objectid, + owner_offset); + if (ret == 0) { + /* + * Either the inline backref or the SHARED_DATA_REF/ + * SHARED_BLOCK_REF is found + * + * Here is a quick path to locate EXTENT/METADATA_ITEM. + * It's possible the EXTENT/METADATA_ITEM is near current slot. + */ + extent_slot = path->slots[0]; + while (extent_slot >= 0) { + btrfs_item_key_to_cpu(path->nodes[0], &key, + extent_slot); + if (key.objectid != bytenr) + break; + if (key.type == BTRFS_EXTENT_ITEM_KEY && + key.offset == num_bytes) { + found_extent = true; + break; + } + if (key.type == BTRFS_METADATA_ITEM_KEY && + key.offset == owner_objectid) { + found_extent = true; + break; + } + + /* Quick path didn't find the EXTENT/METADATA_ITEM */ + if (path->slots[0] - extent_slot > 5) + break; + extent_slot--; + } + + if (!found_extent) { + if (unlikely(iref)) { + abort_and_dump(trans, path, +"invalid iref slot %u, no EXTENT/METADATA_ITEM found but has inline extent ref", + path->slots[0]); + return -EUCLEAN; + } + /* Must be SHARED_* item, remove the backref first */ + ret = remove_extent_backref(trans, extent_root, path, + NULL, refs_to_drop, is_data); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + btrfs_release_path(path); + + /* Slow path to locate EXTENT/METADATA_ITEM */ + key.objectid = bytenr; + key.type = BTRFS_EXTENT_ITEM_KEY; + key.offset = num_bytes; + + if (!is_data && skinny_metadata) { + key.type = BTRFS_METADATA_ITEM_KEY; + key.offset = owner_objectid; + } + + ret = btrfs_search_slot(trans, extent_root, + &key, path, -1, 1); + if (ret > 0 && skinny_metadata && path->slots[0]) { + /* + * Couldn't find our skinny metadata item, + * see if we have ye olde extent item. + */ + path->slots[0]--; + btrfs_item_key_to_cpu(path->nodes[0], &key, + path->slots[0]); + if (key.objectid == bytenr && + key.type == BTRFS_EXTENT_ITEM_KEY && + key.offset == num_bytes) + ret = 0; + } + + if (ret > 0 && skinny_metadata) { + skinny_metadata = false; + key.objectid = bytenr; + key.type = BTRFS_EXTENT_ITEM_KEY; + key.offset = num_bytes; + btrfs_release_path(path); + ret = btrfs_search_slot(trans, extent_root, + &key, path, -1, 1); + } + + if (ret) { + if (ret > 0) + btrfs_print_leaf(path->nodes[0]); + btrfs_err(info, + "umm, got %d back from search, was looking for %llu, slot %d", + ret, bytenr, path->slots[0]); + } + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + extent_slot = path->slots[0]; + } + } else if (WARN_ON(ret == -ENOENT)) { + abort_and_dump(trans, path, +"unable to find ref byte nr %llu parent %llu root %llu owner %llu offset %llu slot %d", + bytenr, node->parent, node->ref_root, owner_objectid, + owner_offset, path->slots[0]); + return ret; + } else { + btrfs_abort_transaction(trans, ret); + return ret; + } + + leaf = path->nodes[0]; + item_size = btrfs_item_size(leaf, extent_slot); + if (unlikely(item_size < sizeof(*ei))) { + ret = -EUCLEAN; + btrfs_err(trans->fs_info, + "unexpected extent item size, has %u expect >= %zu", + item_size, sizeof(*ei)); + btrfs_abort_transaction(trans, ret); + return ret; + } + ei = btrfs_item_ptr(leaf, extent_slot, + struct btrfs_extent_item); + if (owner_objectid < BTRFS_FIRST_FREE_OBJECTID && + key.type == BTRFS_EXTENT_ITEM_KEY) { + struct btrfs_tree_block_info *bi; + + if (unlikely(item_size < sizeof(*ei) + sizeof(*bi))) { + abort_and_dump(trans, path, +"invalid extent item size for key (%llu, %u, %llu) slot %u owner %llu, has %u expect >= %zu", + key.objectid, key.type, key.offset, + path->slots[0], owner_objectid, item_size, + sizeof(*ei) + sizeof(*bi)); + return -EUCLEAN; + } + bi = (struct btrfs_tree_block_info *)(ei + 1); + WARN_ON(owner_objectid != btrfs_tree_block_level(leaf, bi)); + } + + refs = btrfs_extent_refs(leaf, ei); + if (unlikely(refs < refs_to_drop)) { + abort_and_dump(trans, path, + "trying to drop %d refs but we only have %llu for bytenr %llu slot %u", + refs_to_drop, refs, bytenr, path->slots[0]); + return -EUCLEAN; + } + refs -= refs_to_drop; + + if (refs > 0) { + if (extent_op) + __run_delayed_extent_op(extent_op, leaf, ei); + /* + * In the case of inline back ref, reference count will + * be updated by remove_extent_backref + */ + if (iref) { + if (unlikely(!found_extent)) { + abort_and_dump(trans, path, +"invalid iref, got inlined extent ref but no EXTENT/METADATA_ITEM found, slot %u", + path->slots[0]); + return -EUCLEAN; + } + } else { + btrfs_set_extent_refs(leaf, ei, refs); + } + if (found_extent) { + ret = remove_extent_backref(trans, extent_root, path, + iref, refs_to_drop, is_data); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + } + } else { + struct btrfs_squota_delta delta = { + .root = delayed_ref_root, + .num_bytes = num_bytes, + .is_data = is_data, + .is_inc = false, + .generation = btrfs_extent_generation(leaf, ei), + }; + + /* In this branch refs == 1 */ + if (found_extent) { + if (unlikely(is_data && refs_to_drop != + extent_data_ref_count(path, iref))) { + abort_and_dump(trans, path, + "invalid refs_to_drop, current refs %u refs_to_drop %u slot %u", + extent_data_ref_count(path, iref), + refs_to_drop, path->slots[0]); + return -EUCLEAN; + } + if (iref) { + if (unlikely(path->slots[0] != extent_slot)) { + abort_and_dump(trans, path, +"invalid iref, extent item key " BTRFS_KEY_FMT " slot %u doesn't have wanted iref", + BTRFS_KEY_FMT_VALUE(&key), + path->slots[0]); + return -EUCLEAN; + } + } else { + /* + * No inline ref, we must be at SHARED_* item, + * And it's single ref, it must be: + * | extent_slot ||extent_slot + 1| + * [ EXTENT/METADATA_ITEM ][ SHARED_* ITEM ] + */ + if (unlikely(path->slots[0] != extent_slot + 1)) { + abort_and_dump(trans, path, + "invalid SHARED_* item slot %u, previous item is not EXTENT/METADATA_ITEM", + path->slots[0]); + return -EUCLEAN; + } + path->slots[0] = extent_slot; + num_to_del = 2; + } + } + /* + * We can't infer the data owner from the delayed ref, so we need + * to try to get it from the owning ref item. + * + * If it is not present, then that extent was not written under + * simple quotas mode, so we don't need to account for its deletion. + */ + if (is_data) + delta.root = btrfs_get_extent_owner_root(trans->fs_info, + leaf, extent_slot); + + ret = btrfs_del_items(trans, extent_root, path, path->slots[0], + num_to_del); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + btrfs_release_path(path); + + ret = do_free_extent_accounting(trans, bytenr, &delta, path); + } + btrfs_release_path(path); + + return ret; +} + +/* + * when we free an block, it is possible (and likely) that we free the last + * delayed ref for that extent as well. This searches the delayed ref tree for + * a given extent, and if there are no other delayed refs to be processed, it + * removes it from the tree. + */ +static noinline int check_ref_cleanup(struct btrfs_trans_handle *trans, + u64 bytenr) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_delayed_ref_head *head; + struct btrfs_delayed_ref_root *delayed_refs; + int ret = 0; + + delayed_refs = &trans->transaction->delayed_refs; + spin_lock(&delayed_refs->lock); + head = btrfs_find_delayed_ref_head(fs_info, delayed_refs, bytenr); + if (!head) + goto out_delayed_unlock; + + spin_lock(&head->lock); + if (!RB_EMPTY_ROOT(&head->ref_tree.rb_root)) + goto out; + + if (cleanup_extent_op(head) != NULL) + goto out; + + /* + * waiting for the lock here would deadlock. If someone else has it + * locked they are already in the process of dropping it anyway + */ + if (!mutex_trylock(&head->mutex)) + goto out; + + btrfs_delete_ref_head(fs_info, delayed_refs, head); + head->processing = false; + + spin_unlock(&head->lock); + spin_unlock(&delayed_refs->lock); + + BUG_ON(head->extent_op); + if (head->must_insert_reserved) + ret = 1; + + btrfs_cleanup_ref_head_accounting(fs_info, delayed_refs, head); + mutex_unlock(&head->mutex); + btrfs_put_delayed_ref_head(head); + return ret; +out: + spin_unlock(&head->lock); + +out_delayed_unlock: + spin_unlock(&delayed_refs->lock); + return 0; +} + +int btrfs_free_tree_block(struct btrfs_trans_handle *trans, + u64 root_id, + struct extent_buffer *buf, + u64 parent, int last_ref) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_block_group *bg; + int ret; + + if (root_id != BTRFS_TREE_LOG_OBJECTID) { + struct btrfs_ref generic_ref = { + .action = BTRFS_DROP_DELAYED_REF, + .bytenr = buf->start, + .num_bytes = buf->len, + .parent = parent, + .owning_root = btrfs_header_owner(buf), + .ref_root = root_id, + }; + + /* + * Assert that the extent buffer is not cleared due to + * EXTENT_BUFFER_ZONED_ZEROOUT. Please refer + * btrfs_clear_buffer_dirty() and btree_csum_one_bio() for + * detail. + */ + ASSERT(btrfs_header_bytenr(buf) != 0); + + btrfs_init_tree_ref(&generic_ref, btrfs_header_level(buf), 0, false); + btrfs_ref_tree_mod(fs_info, &generic_ref); + ret = btrfs_add_delayed_tree_ref(trans, &generic_ref, NULL); + if (ret < 0) + return ret; + } + + if (!last_ref) + return 0; + + if (btrfs_header_generation(buf) != trans->transid) + return 0; + + if (root_id != BTRFS_TREE_LOG_OBJECTID) { + ret = check_ref_cleanup(trans, buf->start); + if (!ret) + return 0; + } + + bg = btrfs_lookup_block_group(fs_info, buf->start); + + if (btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) { + pin_down_extent(trans, bg, buf->start, buf->len, true); + btrfs_put_block_group(bg); + return 0; + } + + /* + * If there are tree mod log users we may have recorded mod log + * operations for this node. If we re-allocate this node we + * could replay operations on this node that happened when it + * existed in a completely different root. For example if it + * was part of root A, then was reallocated to root B, and we + * are doing a btrfs_old_search_slot(root b), we could replay + * operations that happened when the block was part of root A, + * giving us an inconsistent view of the btree. + * + * We are safe from races here because at this point no other + * node or root points to this extent buffer, so if after this + * check a new tree mod log user joins we will not have an + * existing log of operations on this node that we have to + * contend with. + */ + + if (test_bit(BTRFS_FS_TREE_MOD_LOG_USERS, &fs_info->flags) + || btrfs_is_zoned(fs_info)) { + pin_down_extent(trans, bg, buf->start, buf->len, true); + btrfs_put_block_group(bg); + return 0; + } + + WARN_ON(test_bit(EXTENT_BUFFER_DIRTY, &buf->bflags)); + + btrfs_add_free_space(bg, buf->start, buf->len); + btrfs_free_reserved_bytes(bg, buf->len, false); + btrfs_put_block_group(bg); + trace_btrfs_reserved_extent_free(fs_info, buf->start, buf->len); + + return 0; +} + +/* Can return -ENOMEM */ +int btrfs_free_extent(struct btrfs_trans_handle *trans, struct btrfs_ref *ref) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + int ret; + + if (btrfs_is_testing(fs_info)) + return 0; + + /* + * tree log blocks never actually go into the extent allocation + * tree, just update pinning info and exit early. + */ + if (ref->ref_root == BTRFS_TREE_LOG_OBJECTID) { + btrfs_pin_extent(trans, ref->bytenr, ref->num_bytes); + ret = 0; + } else if (ref->type == BTRFS_REF_METADATA) { + ret = btrfs_add_delayed_tree_ref(trans, ref, NULL); + } else { + ret = btrfs_add_delayed_data_ref(trans, ref, 0); + } + + if (ref->ref_root != BTRFS_TREE_LOG_OBJECTID) + btrfs_ref_tree_mod(fs_info, ref); + + return ret; +} + +enum btrfs_loop_type { + /* + * Start caching block groups but do not wait for progress or for them + * to be done. + */ + LOOP_CACHING_NOWAIT, + + /* + * Wait for the block group free_space >= the space we're waiting for if + * the block group isn't cached. + */ + LOOP_CACHING_WAIT, + + /* + * Allow allocations to happen from block groups that do not yet have a + * size classification. + */ + LOOP_UNSET_SIZE_CLASS, + + /* + * Allocate a chunk and then retry the allocation. + */ + LOOP_ALLOC_CHUNK, + + /* + * Ignore the size class restrictions for this allocation. + */ + LOOP_WRONG_SIZE_CLASS, + + /* + * Ignore the empty size, only try to allocate the number of bytes + * needed for this allocation. + */ + LOOP_NO_EMPTY_SIZE, +}; + +static inline void +btrfs_lock_block_group(struct btrfs_block_group *cache, bool delalloc) +{ + if (delalloc) + down_read(&cache->data_rwsem); +} + +static inline void btrfs_grab_block_group(struct btrfs_block_group *cache, + bool delalloc) +{ + btrfs_get_block_group(cache); + if (delalloc) + down_read(&cache->data_rwsem); +} + +static struct btrfs_block_group *btrfs_lock_cluster( + struct btrfs_block_group *block_group, + struct btrfs_free_cluster *cluster, + bool delalloc) + __acquires(&cluster->refill_lock) +{ + struct btrfs_block_group *used_bg = NULL; + + spin_lock(&cluster->refill_lock); + while (1) { + used_bg = cluster->block_group; + if (!used_bg) + return NULL; + + if (used_bg == block_group) + return used_bg; + + btrfs_get_block_group(used_bg); + + if (!delalloc) + return used_bg; + + if (down_read_trylock(&used_bg->data_rwsem)) + return used_bg; + + spin_unlock(&cluster->refill_lock); + + /* We should only have one-level nested. */ + down_read_nested(&used_bg->data_rwsem, SINGLE_DEPTH_NESTING); + + spin_lock(&cluster->refill_lock); + if (used_bg == cluster->block_group) + return used_bg; + + up_read(&used_bg->data_rwsem); + btrfs_put_block_group(used_bg); + } +} + +static inline void +btrfs_release_block_group(struct btrfs_block_group *cache, bool delalloc) +{ + if (delalloc) + up_read(&cache->data_rwsem); + btrfs_put_block_group(cache); +} + +static bool find_free_extent_check_size_class(const struct find_free_extent_ctl *ffe_ctl, + const struct btrfs_block_group *bg) +{ + if (ffe_ctl->policy == BTRFS_EXTENT_ALLOC_ZONED) + return true; + if (!btrfs_block_group_should_use_size_class(bg)) + return true; + if (ffe_ctl->loop >= LOOP_WRONG_SIZE_CLASS) + return true; + if (ffe_ctl->loop >= LOOP_UNSET_SIZE_CLASS && + bg->size_class == BTRFS_BG_SZ_NONE) + return true; + return ffe_ctl->size_class == bg->size_class; +} + +/* + * Helper function for find_free_extent(). + * + * Return -ENOENT to inform caller that we need fallback to unclustered mode. + * Return >0 to inform caller that we find nothing + * Return 0 means we have found a location and set ffe_ctl->found_offset. + */ +static int find_free_extent_clustered(struct btrfs_block_group *bg, + struct find_free_extent_ctl *ffe_ctl, + struct btrfs_block_group **cluster_bg_ret) +{ + struct btrfs_block_group *cluster_bg; + struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr; + u64 aligned_cluster; + u64 offset; + int ret; + + cluster_bg = btrfs_lock_cluster(bg, last_ptr, ffe_ctl->delalloc); + if (!cluster_bg) + goto refill_cluster; + if (cluster_bg != bg && (cluster_bg->ro || + !block_group_bits(cluster_bg, ffe_ctl->flags) || + !find_free_extent_check_size_class(ffe_ctl, cluster_bg))) + goto release_cluster; + + offset = btrfs_alloc_from_cluster(cluster_bg, last_ptr, + ffe_ctl->num_bytes, cluster_bg->start, + &ffe_ctl->max_extent_size); + if (offset) { + /* We have a block, we're done */ + spin_unlock(&last_ptr->refill_lock); + trace_btrfs_reserve_extent_cluster(cluster_bg, ffe_ctl); + *cluster_bg_ret = cluster_bg; + ffe_ctl->found_offset = offset; + return 0; + } + WARN_ON(last_ptr->block_group != cluster_bg); + +release_cluster: + /* + * If we are on LOOP_NO_EMPTY_SIZE, we can't set up a new clusters, so + * lets just skip it and let the allocator find whatever block it can + * find. If we reach this point, we will have tried the cluster + * allocator plenty of times and not have found anything, so we are + * likely way too fragmented for the clustering stuff to find anything. + * + * However, if the cluster is taken from the current block group, + * release the cluster first, so that we stand a better chance of + * succeeding in the unclustered allocation. + */ + if (ffe_ctl->loop >= LOOP_NO_EMPTY_SIZE && cluster_bg != bg) { + spin_unlock(&last_ptr->refill_lock); + btrfs_release_block_group(cluster_bg, ffe_ctl->delalloc); + return -ENOENT; + } + + /* This cluster didn't work out, free it and start over */ + btrfs_return_cluster_to_free_space(NULL, last_ptr); + + if (cluster_bg != bg) + btrfs_release_block_group(cluster_bg, ffe_ctl->delalloc); + +refill_cluster: + if (ffe_ctl->loop >= LOOP_NO_EMPTY_SIZE) { + spin_unlock(&last_ptr->refill_lock); + return -ENOENT; + } + + aligned_cluster = max_t(u64, + ffe_ctl->empty_cluster + ffe_ctl->empty_size, + bg->full_stripe_len); + ret = btrfs_find_space_cluster(bg, last_ptr, ffe_ctl->search_start, + ffe_ctl->num_bytes, aligned_cluster); + if (ret == 0) { + /* Now pull our allocation out of this cluster */ + offset = btrfs_alloc_from_cluster(bg, last_ptr, + ffe_ctl->num_bytes, ffe_ctl->search_start, + &ffe_ctl->max_extent_size); + if (offset) { + /* We found one, proceed */ + spin_unlock(&last_ptr->refill_lock); + ffe_ctl->found_offset = offset; + trace_btrfs_reserve_extent_cluster(bg, ffe_ctl); + return 0; + } + } + /* + * At this point we either didn't find a cluster or we weren't able to + * allocate a block from our cluster. Free the cluster we've been + * trying to use, and go to the next block group. + */ + btrfs_return_cluster_to_free_space(NULL, last_ptr); + spin_unlock(&last_ptr->refill_lock); + return 1; +} + +/* + * Return >0 to inform caller that we find nothing + * Return 0 when we found an free extent and set ffe_ctrl->found_offset + */ +static int find_free_extent_unclustered(struct btrfs_block_group *bg, + struct find_free_extent_ctl *ffe_ctl) +{ + struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr; + u64 offset; + + /* + * We are doing an unclustered allocation, set the fragmented flag so + * we don't bother trying to setup a cluster again until we get more + * space. + */ + if (unlikely(last_ptr)) { + spin_lock(&last_ptr->lock); + last_ptr->fragmented = 1; + spin_unlock(&last_ptr->lock); + } + if (ffe_ctl->cached) { + struct btrfs_free_space_ctl *free_space_ctl; + + free_space_ctl = bg->free_space_ctl; + spin_lock(&free_space_ctl->tree_lock); + if (free_space_ctl->free_space < + ffe_ctl->num_bytes + ffe_ctl->empty_cluster + + ffe_ctl->empty_size) { + ffe_ctl->total_free_space = max_t(u64, + ffe_ctl->total_free_space, + free_space_ctl->free_space); + spin_unlock(&free_space_ctl->tree_lock); + return 1; + } + spin_unlock(&free_space_ctl->tree_lock); + } + + offset = btrfs_find_space_for_alloc(bg, ffe_ctl->search_start, + ffe_ctl->num_bytes, ffe_ctl->empty_size, + &ffe_ctl->max_extent_size); + if (!offset) + return 1; + ffe_ctl->found_offset = offset; + return 0; +} + +static int do_allocation_clustered(struct btrfs_block_group *block_group, + struct find_free_extent_ctl *ffe_ctl, + struct btrfs_block_group **bg_ret) +{ + int ret; + + /* We want to try and use the cluster allocator, so lets look there */ + if (ffe_ctl->last_ptr && ffe_ctl->use_cluster) { + ret = find_free_extent_clustered(block_group, ffe_ctl, bg_ret); + if (ret >= 0) + return ret; + /* ret == -ENOENT case falls through */ + } + + return find_free_extent_unclustered(block_group, ffe_ctl); +} + +/* + * Tree-log block group locking + * ============================ + * + * fs_info::treelog_bg_lock protects the fs_info::treelog_bg which + * indicates the starting address of a block group, which is reserved only + * for tree-log metadata. + * + * Lock nesting + * ============ + * + * block_group::lock + * fs_info::treelog_bg_lock + */ + +/* + * Simple allocator for sequential-only block group. It only allows sequential + * allocation. No need to play with trees. This function also reserves the + * bytes as in btrfs_add_reserved_bytes. + */ +static int do_allocation_zoned(struct btrfs_block_group *block_group, + struct find_free_extent_ctl *ffe_ctl, + struct btrfs_block_group **bg_ret) +{ + struct btrfs_fs_info *fs_info = block_group->fs_info; + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + u64 start = block_group->start; + u64 num_bytes = ffe_ctl->num_bytes; + u64 avail; + u64 bytenr = block_group->start; + u64 log_bytenr; + u64 data_reloc_bytenr; + int ret = 0; + bool skip = false; + + ASSERT(btrfs_is_zoned(block_group->fs_info)); + + /* + * Do not allow non-tree-log blocks in the dedicated tree-log block + * group, and vice versa. + */ + spin_lock(&fs_info->treelog_bg_lock); + log_bytenr = fs_info->treelog_bg; + if (log_bytenr && ((ffe_ctl->for_treelog && bytenr != log_bytenr) || + (!ffe_ctl->for_treelog && bytenr == log_bytenr))) + skip = true; + spin_unlock(&fs_info->treelog_bg_lock); + if (skip) + return 1; + + /* + * Do not allow non-relocation blocks in the dedicated relocation block + * group, and vice versa. + */ + spin_lock(&fs_info->relocation_bg_lock); + data_reloc_bytenr = fs_info->data_reloc_bg; + if (data_reloc_bytenr && + ((ffe_ctl->for_data_reloc && bytenr != data_reloc_bytenr) || + (!ffe_ctl->for_data_reloc && bytenr == data_reloc_bytenr))) + skip = true; + spin_unlock(&fs_info->relocation_bg_lock); + if (skip) + return 1; + + /* Check RO and no space case before trying to activate it */ + spin_lock(&block_group->lock); + if (block_group->ro || btrfs_zoned_bg_is_full(block_group)) { + ret = 1; + /* + * May need to clear fs_info->{treelog,data_reloc}_bg. + * Return the error after taking the locks. + */ + } + spin_unlock(&block_group->lock); + + /* Metadata block group is activated at write time. */ + if (!ret && (block_group->flags & BTRFS_BLOCK_GROUP_DATA) && + !btrfs_zone_activate(block_group)) { + ret = 1; + /* + * May need to clear fs_info->{treelog,data_reloc}_bg. + * Return the error after taking the locks. + */ + } + + spin_lock(&block_group->lock); + spin_lock(&fs_info->treelog_bg_lock); + spin_lock(&fs_info->relocation_bg_lock); + + if (ret) + goto out; + + ASSERT(!ffe_ctl->for_treelog || + block_group->start == fs_info->treelog_bg || + fs_info->treelog_bg == 0); + ASSERT(!ffe_ctl->for_data_reloc || + block_group->start == fs_info->data_reloc_bg || + fs_info->data_reloc_bg == 0); + + if (block_group->ro || + (!ffe_ctl->for_data_reloc && + test_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, &block_group->runtime_flags))) { + ret = 1; + goto out; + } + + /* + * Do not allow currently using block group to be tree-log dedicated + * block group. + */ + if (ffe_ctl->for_treelog && !fs_info->treelog_bg && + (block_group->used || block_group->reserved)) { + ret = 1; + goto out; + } + + /* + * Do not allow currently used block group to be the data relocation + * dedicated block group. + */ + if (ffe_ctl->for_data_reloc && !fs_info->data_reloc_bg && + (block_group->used || block_group->reserved)) { + ret = 1; + goto out; + } + + WARN_ON_ONCE(block_group->alloc_offset > block_group->zone_capacity); + avail = block_group->zone_capacity - block_group->alloc_offset; + if (avail < num_bytes) { + if (ffe_ctl->max_extent_size < avail) { + /* + * With sequential allocator, free space is always + * contiguous + */ + ffe_ctl->max_extent_size = avail; + ffe_ctl->total_free_space = avail; + } + ret = 1; + goto out; + } + + if (ffe_ctl->for_treelog && !fs_info->treelog_bg) + fs_info->treelog_bg = block_group->start; + + if (ffe_ctl->for_data_reloc) { + if (!fs_info->data_reloc_bg) + fs_info->data_reloc_bg = block_group->start; + /* + * Do not allow allocations from this block group, unless it is + * for data relocation. Compared to increasing the ->ro, setting + * the ->zoned_data_reloc_ongoing flag still allows nocow + * writers to come in. See btrfs_inc_nocow_writers(). + * + * We need to disable an allocation to avoid an allocation of + * regular (non-relocation data) extent. With mix of relocation + * extents and regular extents, we can dispatch WRITE commands + * (for relocation extents) and ZONE APPEND commands (for + * regular extents) at the same time to the same zone, which + * easily break the write pointer. + * + * Also, this flag avoids this block group to be zone finished. + */ + set_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, &block_group->runtime_flags); + } + + ffe_ctl->found_offset = start + block_group->alloc_offset; + block_group->alloc_offset += num_bytes; + spin_lock(&ctl->tree_lock); + ctl->free_space -= num_bytes; + spin_unlock(&ctl->tree_lock); + + /* + * We do not check if found_offset is aligned to stripesize. The + * address is anyway rewritten when using zone append writing. + */ + + ffe_ctl->search_start = ffe_ctl->found_offset; + +out: + if (ret && ffe_ctl->for_treelog) + fs_info->treelog_bg = 0; + if (ret && ffe_ctl->for_data_reloc) + fs_info->data_reloc_bg = 0; + spin_unlock(&fs_info->relocation_bg_lock); + spin_unlock(&fs_info->treelog_bg_lock); + spin_unlock(&block_group->lock); + return ret; +} + +static int do_allocation(struct btrfs_block_group *block_group, + struct find_free_extent_ctl *ffe_ctl, + struct btrfs_block_group **bg_ret) +{ + switch (ffe_ctl->policy) { + case BTRFS_EXTENT_ALLOC_CLUSTERED: + return do_allocation_clustered(block_group, ffe_ctl, bg_ret); + case BTRFS_EXTENT_ALLOC_ZONED: + return do_allocation_zoned(block_group, ffe_ctl, bg_ret); + default: + BUG(); + } +} + +static void release_block_group(struct btrfs_block_group *block_group, + struct find_free_extent_ctl *ffe_ctl, + bool delalloc) +{ + switch (ffe_ctl->policy) { + case BTRFS_EXTENT_ALLOC_CLUSTERED: + ffe_ctl->retry_uncached = false; + break; + case BTRFS_EXTENT_ALLOC_ZONED: + /* Nothing to do */ + break; + default: + BUG(); + } + + BUG_ON(btrfs_bg_flags_to_raid_index(block_group->flags) != + ffe_ctl->index); + btrfs_release_block_group(block_group, delalloc); +} + +static void found_extent_clustered(struct find_free_extent_ctl *ffe_ctl, + struct btrfs_key *ins) +{ + struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr; + + if (!ffe_ctl->use_cluster && last_ptr) { + spin_lock(&last_ptr->lock); + last_ptr->window_start = ins->objectid; + spin_unlock(&last_ptr->lock); + } +} + +static void found_extent(struct find_free_extent_ctl *ffe_ctl, + struct btrfs_key *ins) +{ + switch (ffe_ctl->policy) { + case BTRFS_EXTENT_ALLOC_CLUSTERED: + found_extent_clustered(ffe_ctl, ins); + break; + case BTRFS_EXTENT_ALLOC_ZONED: + /* Nothing to do */ + break; + default: + BUG(); + } +} + +static int can_allocate_chunk_zoned(struct btrfs_fs_info *fs_info, + struct find_free_extent_ctl *ffe_ctl) +{ + /* Block group's activeness is not a requirement for METADATA block groups. */ + if (!(ffe_ctl->flags & BTRFS_BLOCK_GROUP_DATA)) + return 0; + + /* If we can activate new zone, just allocate a chunk and use it */ + if (btrfs_can_activate_zone(fs_info->fs_devices, ffe_ctl->flags)) + return 0; + + /* + * We already reached the max active zones. Try to finish one block + * group to make a room for a new block group. This is only possible + * for a data block group because btrfs_zone_finish() may need to wait + * for a running transaction which can cause a deadlock for metadata + * allocation. + */ + if (ffe_ctl->flags & BTRFS_BLOCK_GROUP_DATA) { + int ret = btrfs_zone_finish_one_bg(fs_info); + + if (ret == 1) + return 0; + else if (ret < 0) + return ret; + } + + /* + * If we have enough free space left in an already active block group + * and we can't activate any other zone now, do not allow allocating a + * new chunk and let find_free_extent() retry with a smaller size. + */ + if (ffe_ctl->max_extent_size >= ffe_ctl->min_alloc_size) + return -ENOSPC; + + /* + * Even min_alloc_size is not left in any block groups. Since we cannot + * activate a new block group, allocating it may not help. Let's tell a + * caller to try again and hope it progress something by writing some + * parts of the region. That is only possible for data block groups, + * where a part of the region can be written. + */ + if (ffe_ctl->flags & BTRFS_BLOCK_GROUP_DATA) + return -EAGAIN; + + /* + * We cannot activate a new block group and no enough space left in any + * block groups. So, allocating a new block group may not help. But, + * there is nothing to do anyway, so let's go with it. + */ + return 0; +} + +static int can_allocate_chunk(struct btrfs_fs_info *fs_info, + struct find_free_extent_ctl *ffe_ctl) +{ + switch (ffe_ctl->policy) { + case BTRFS_EXTENT_ALLOC_CLUSTERED: + return 0; + case BTRFS_EXTENT_ALLOC_ZONED: + return can_allocate_chunk_zoned(fs_info, ffe_ctl); + default: + BUG(); + } +} + +/* + * Return >0 means caller needs to re-search for free extent + * Return 0 means we have the needed free extent. + * Return <0 means we failed to locate any free extent. + */ +static int find_free_extent_update_loop(struct btrfs_fs_info *fs_info, + struct btrfs_key *ins, + struct find_free_extent_ctl *ffe_ctl, + struct btrfs_space_info *space_info, + bool full_search) +{ + struct btrfs_root *root = fs_info->chunk_root; + int ret; + + if ((ffe_ctl->loop == LOOP_CACHING_NOWAIT) && + ffe_ctl->have_caching_bg && !ffe_ctl->orig_have_caching_bg) + ffe_ctl->orig_have_caching_bg = true; + + if (ins->objectid) { + found_extent(ffe_ctl, ins); + return 0; + } + + if (ffe_ctl->loop >= LOOP_CACHING_WAIT && ffe_ctl->have_caching_bg) + return 1; + + ffe_ctl->index++; + if (ffe_ctl->index < BTRFS_NR_RAID_TYPES) + return 1; + + /* See the comments for btrfs_loop_type for an explanation of the phases. */ + if (ffe_ctl->loop == LOOP_NO_EMPTY_SIZE) + return -ENOSPC; + + ffe_ctl->index = 0; + /* + * We want to skip the LOOP_CACHING_WAIT step if we don't have any + * uncached bgs and we've already done a full search through. + */ + if (ffe_ctl->loop == LOOP_CACHING_NOWAIT && + (!ffe_ctl->orig_have_caching_bg && full_search)) + ffe_ctl->loop++; + ffe_ctl->loop++; + + if (ffe_ctl->loop == LOOP_ALLOC_CHUNK) { + struct btrfs_trans_handle *trans; + bool have_trans = false; + + /* Check if allocation policy allows to create a new chunk. */ + ret = can_allocate_chunk(fs_info, ffe_ctl); + if (ret) + return ret; + + trans = current->journal_info; + if (trans) + have_trans = true; + else + trans = btrfs_join_transaction(root); + + if (IS_ERR(trans)) + return PTR_ERR(trans); + + ret = btrfs_chunk_alloc(trans, space_info, ffe_ctl->flags, + CHUNK_ALLOC_FORCE_FOR_EXTENT); + + /* Do not bail out on ENOSPC since we can do more. */ + if (ret == -ENOSPC) { + ret = 0; + ffe_ctl->loop++; + } else if (ret < 0) { + btrfs_abort_transaction(trans, ret); + } else { + ret = 0; + } + + if (!have_trans) + btrfs_end_transaction(trans); + + if (ret) + return ret; + } + + if (ffe_ctl->loop == LOOP_NO_EMPTY_SIZE) { + if (ffe_ctl->policy != BTRFS_EXTENT_ALLOC_CLUSTERED) + return -ENOSPC; + + /* + * Don't loop again if we already have no empty_size and + * no empty_cluster. + */ + if (ffe_ctl->empty_size == 0 && ffe_ctl->empty_cluster == 0) + return -ENOSPC; + ffe_ctl->empty_size = 0; + ffe_ctl->empty_cluster = 0; + } + + return 1; +} + +static int prepare_allocation_clustered(struct btrfs_fs_info *fs_info, + struct find_free_extent_ctl *ffe_ctl, + struct btrfs_space_info *space_info, + struct btrfs_key *ins) +{ + /* + * If our free space is heavily fragmented we may not be able to make + * big contiguous allocations, so instead of doing the expensive search + * for free space, simply return ENOSPC with our max_extent_size so we + * can go ahead and search for a more manageable chunk. + * + * If our max_extent_size is large enough for our allocation simply + * disable clustering since we will likely not be able to find enough + * space to create a cluster and induce latency trying. + */ + if (space_info->max_extent_size) { + spin_lock(&space_info->lock); + if (space_info->max_extent_size && + ffe_ctl->num_bytes > space_info->max_extent_size) { + ins->offset = space_info->max_extent_size; + spin_unlock(&space_info->lock); + return -ENOSPC; + } else if (space_info->max_extent_size) { + ffe_ctl->use_cluster = false; + } + spin_unlock(&space_info->lock); + } + + ffe_ctl->last_ptr = fetch_cluster_info(fs_info, space_info, + &ffe_ctl->empty_cluster); + if (ffe_ctl->last_ptr) { + struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr; + + spin_lock(&last_ptr->lock); + if (last_ptr->block_group) + ffe_ctl->hint_byte = last_ptr->window_start; + if (last_ptr->fragmented) { + /* + * We still set window_start so we can keep track of the + * last place we found an allocation to try and save + * some time. + */ + ffe_ctl->hint_byte = last_ptr->window_start; + ffe_ctl->use_cluster = false; + } + spin_unlock(&last_ptr->lock); + } + + return 0; +} + +static int prepare_allocation_zoned(struct btrfs_fs_info *fs_info, + struct find_free_extent_ctl *ffe_ctl, + struct btrfs_space_info *space_info) +{ + struct btrfs_block_group *block_group; + + if (ffe_ctl->for_treelog) { + spin_lock(&fs_info->treelog_bg_lock); + if (fs_info->treelog_bg) + ffe_ctl->hint_byte = fs_info->treelog_bg; + spin_unlock(&fs_info->treelog_bg_lock); + return 0; + } + + if (ffe_ctl->for_data_reloc) { + spin_lock(&fs_info->relocation_bg_lock); + if (fs_info->data_reloc_bg) + ffe_ctl->hint_byte = fs_info->data_reloc_bg; + spin_unlock(&fs_info->relocation_bg_lock); + return 0; + } + + if (!(ffe_ctl->flags & BTRFS_BLOCK_GROUP_DATA)) + return 0; + + spin_lock(&fs_info->zone_active_bgs_lock); + list_for_each_entry(block_group, &fs_info->zone_active_bgs, active_bg_list) { + /* + * No lock is OK here because avail is monotonically + * decreasing, and this is just a hint. + */ + u64 avail = block_group->zone_capacity - block_group->alloc_offset; + + if (block_group_bits(block_group, ffe_ctl->flags) && + block_group->space_info == space_info && + avail >= ffe_ctl->num_bytes) { + ffe_ctl->hint_byte = block_group->start; + break; + } + } + spin_unlock(&fs_info->zone_active_bgs_lock); + + return 0; +} + +static int prepare_allocation(struct btrfs_fs_info *fs_info, + struct find_free_extent_ctl *ffe_ctl, + struct btrfs_space_info *space_info, + struct btrfs_key *ins) +{ + switch (ffe_ctl->policy) { + case BTRFS_EXTENT_ALLOC_CLUSTERED: + return prepare_allocation_clustered(fs_info, ffe_ctl, + space_info, ins); + case BTRFS_EXTENT_ALLOC_ZONED: + return prepare_allocation_zoned(fs_info, ffe_ctl, space_info); + default: + BUG(); + } +} + +/* + * walks the btree of allocated extents and find a hole of a given size. + * The key ins is changed to record the hole: + * ins->objectid == start position + * ins->flags = BTRFS_EXTENT_ITEM_KEY + * ins->offset == the size of the hole. + * Any available blocks before search_start are skipped. + * + * If there is no suitable free space, we will record the max size of + * the free space extent currently. + * + * The overall logic and call chain: + * + * find_free_extent() + * |- Iterate through all block groups + * | |- Get a valid block group + * | |- Try to do clustered allocation in that block group + * | |- Try to do unclustered allocation in that block group + * | |- Check if the result is valid + * | | |- If valid, then exit + * | |- Jump to next block group + * | + * |- Push harder to find free extents + * |- If not found, re-iterate all block groups + */ +static noinline int find_free_extent(struct btrfs_root *root, + struct btrfs_key *ins, + struct find_free_extent_ctl *ffe_ctl) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + int ret = 0; + int cache_block_group_error = 0; + struct btrfs_block_group *block_group = NULL; + struct btrfs_space_info *space_info; + bool full_search = false; + + WARN_ON(ffe_ctl->num_bytes < fs_info->sectorsize); + + ffe_ctl->search_start = 0; + /* For clustered allocation */ + ffe_ctl->empty_cluster = 0; + ffe_ctl->last_ptr = NULL; + ffe_ctl->use_cluster = true; + ffe_ctl->have_caching_bg = false; + ffe_ctl->orig_have_caching_bg = false; + ffe_ctl->index = btrfs_bg_flags_to_raid_index(ffe_ctl->flags); + ffe_ctl->loop = 0; + ffe_ctl->retry_uncached = false; + ffe_ctl->cached = 0; + ffe_ctl->max_extent_size = 0; + ffe_ctl->total_free_space = 0; + ffe_ctl->found_offset = 0; + ffe_ctl->policy = BTRFS_EXTENT_ALLOC_CLUSTERED; + ffe_ctl->size_class = btrfs_calc_block_group_size_class(ffe_ctl->num_bytes); + + if (btrfs_is_zoned(fs_info)) + ffe_ctl->policy = BTRFS_EXTENT_ALLOC_ZONED; + + ins->type = BTRFS_EXTENT_ITEM_KEY; + ins->objectid = 0; + ins->offset = 0; + + trace_btrfs_find_free_extent(root, ffe_ctl); + + space_info = btrfs_find_space_info(fs_info, ffe_ctl->flags); + if (btrfs_is_zoned(fs_info) && space_info) { + /* Use dedicated sub-space_info for dedicated block group users. */ + if (ffe_ctl->for_data_reloc) { + space_info = space_info->sub_group[0]; + ASSERT(space_info->subgroup_id == BTRFS_SUB_GROUP_DATA_RELOC, + "space_info->subgroup_id=%d", space_info->subgroup_id); + } else if (ffe_ctl->for_treelog) { + space_info = space_info->sub_group[0]; + ASSERT(space_info->subgroup_id == BTRFS_SUB_GROUP_TREELOG, + "space_info->subgroup_id=%d", space_info->subgroup_id); + } + } + if (!space_info) { + btrfs_err(fs_info, "no space info for %llu, tree-log %d, relocation %d", + ffe_ctl->flags, ffe_ctl->for_treelog, ffe_ctl->for_data_reloc); + return -ENOSPC; + } + + ret = prepare_allocation(fs_info, ffe_ctl, space_info, ins); + if (ret < 0) + return ret; + + ffe_ctl->search_start = max(ffe_ctl->search_start, + first_logical_byte(fs_info)); + ffe_ctl->search_start = max(ffe_ctl->search_start, ffe_ctl->hint_byte); + if (ffe_ctl->search_start == ffe_ctl->hint_byte) { + block_group = btrfs_lookup_block_group(fs_info, + ffe_ctl->search_start); + /* + * we don't want to use the block group if it doesn't match our + * allocation bits, or if its not cached. + * + * However if we are re-searching with an ideal block group + * picked out then we don't care that the block group is cached. + */ + if (block_group && block_group_bits(block_group, ffe_ctl->flags) && + block_group->space_info == space_info && + block_group->cached != BTRFS_CACHE_NO) { + down_read(&space_info->groups_sem); + if (list_empty(&block_group->list) || + block_group->ro || + (block_group->flags & BTRFS_BLOCK_GROUP_REMAPPED)) { + /* + * someone is removing this block group, + * we can't jump into the have_block_group + * target because our list pointers are not + * valid + */ + btrfs_put_block_group(block_group); + up_read(&space_info->groups_sem); + } else { + ffe_ctl->index = btrfs_bg_flags_to_raid_index( + block_group->flags); + btrfs_lock_block_group(block_group, + ffe_ctl->delalloc); + ffe_ctl->hinted = true; + goto have_block_group; + } + } else if (block_group) { + btrfs_put_block_group(block_group); + } + } +search: + trace_btrfs_find_free_extent_search_loop(root, ffe_ctl); + ffe_ctl->have_caching_bg = false; + if (ffe_ctl->index == btrfs_bg_flags_to_raid_index(ffe_ctl->flags) || + ffe_ctl->index == 0) + full_search = true; + down_read(&space_info->groups_sem); + list_for_each_entry(block_group, + &space_info->block_groups[ffe_ctl->index], list) { + struct btrfs_block_group *bg_ret; + + ffe_ctl->hinted = false; + /* If the block group is read-only, we can skip it entirely. */ + if (unlikely(block_group->ro || + (block_group->flags & BTRFS_BLOCK_GROUP_REMAPPED))) { + if (ffe_ctl->for_treelog) + btrfs_clear_treelog_bg(block_group); + if (ffe_ctl->for_data_reloc) + btrfs_clear_data_reloc_bg(block_group); + continue; + } + + btrfs_grab_block_group(block_group, ffe_ctl->delalloc); + ffe_ctl->search_start = block_group->start; + + /* + * this can happen if we end up cycling through all the + * raid types, but we want to make sure we only allocate + * for the proper type. + */ + if (!block_group_bits(block_group, ffe_ctl->flags)) { + u64 extra = BTRFS_BLOCK_GROUP_DUP | + BTRFS_BLOCK_GROUP_RAID1_MASK | + BTRFS_BLOCK_GROUP_RAID56_MASK | + BTRFS_BLOCK_GROUP_RAID10; + + /* + * if they asked for extra copies and this block group + * doesn't provide them, bail. This does allow us to + * fill raid0 from raid1. + */ + if ((ffe_ctl->flags & extra) && !(block_group->flags & extra)) + goto loop; + + /* + * This block group has different flags than we want. + * It's possible that we have MIXED_GROUP flag but no + * block group is mixed. Just skip such block group. + */ + btrfs_release_block_group(block_group, ffe_ctl->delalloc); + continue; + } + +have_block_group: + trace_btrfs_find_free_extent_have_block_group(root, ffe_ctl, block_group); + ffe_ctl->cached = btrfs_block_group_done(block_group); + if (unlikely(!ffe_ctl->cached)) { + ffe_ctl->have_caching_bg = true; + ret = btrfs_cache_block_group(block_group, false); + + /* + * If we get ENOMEM here or something else we want to + * try other block groups, because it may not be fatal. + * However if we can't find anything else we need to + * save our return here so that we return the actual + * error that caused problems, not ENOSPC. + */ + if (ret < 0) { + if (!cache_block_group_error) + cache_block_group_error = ret; + ret = 0; + goto loop; + } + ret = 0; + } + + if (unlikely(block_group->cached == BTRFS_CACHE_ERROR)) { + if (!cache_block_group_error) + cache_block_group_error = -EIO; + goto loop; + } + + if (!find_free_extent_check_size_class(ffe_ctl, block_group)) + goto loop; + + bg_ret = NULL; + ret = do_allocation(block_group, ffe_ctl, &bg_ret); + if (ret > 0) + goto loop; + + if (bg_ret && bg_ret != block_group) { + btrfs_release_block_group(block_group, ffe_ctl->delalloc); + block_group = bg_ret; + } + + /* Checks */ + ffe_ctl->search_start = round_up(ffe_ctl->found_offset, + fs_info->sectorsize); + + /* move on to the next group */ + if (ffe_ctl->search_start + ffe_ctl->num_bytes > + btrfs_block_group_end(block_group)) { + btrfs_add_free_space_unused(block_group, + ffe_ctl->found_offset, + ffe_ctl->num_bytes); + goto loop; + } + + if (ffe_ctl->found_offset < ffe_ctl->search_start) + btrfs_add_free_space_unused(block_group, + ffe_ctl->found_offset, + ffe_ctl->search_start - ffe_ctl->found_offset); + + ret = btrfs_add_reserved_bytes(block_group, ffe_ctl->ram_bytes, + ffe_ctl->num_bytes, + ffe_ctl->delalloc, + ffe_ctl->loop >= LOOP_WRONG_SIZE_CLASS); + if (ret == -EAGAIN) { + btrfs_add_free_space_unused(block_group, + ffe_ctl->found_offset, + ffe_ctl->num_bytes); + goto loop; + } + btrfs_inc_block_group_reservations(block_group); + + /* we are all good, lets return */ + ins->objectid = ffe_ctl->search_start; + ins->offset = ffe_ctl->num_bytes; + + trace_btrfs_reserve_extent(block_group, ffe_ctl); + btrfs_release_block_group(block_group, ffe_ctl->delalloc); + break; +loop: + if (!ffe_ctl->cached && ffe_ctl->loop > LOOP_CACHING_NOWAIT && + !ffe_ctl->retry_uncached) { + ffe_ctl->retry_uncached = true; + btrfs_wait_block_group_cache_progress(block_group, + ffe_ctl->num_bytes + + ffe_ctl->empty_cluster + + ffe_ctl->empty_size); + goto have_block_group; + } + release_block_group(block_group, ffe_ctl, ffe_ctl->delalloc); + cond_resched(); + } + up_read(&space_info->groups_sem); + + ret = find_free_extent_update_loop(fs_info, ins, ffe_ctl, space_info, + full_search); + if (ret > 0) + goto search; + + if (ret == -ENOSPC && !cache_block_group_error) { + /* + * Use ffe_ctl->total_free_space as fallback if we can't find + * any contiguous hole. + */ + if (!ffe_ctl->max_extent_size) + ffe_ctl->max_extent_size = ffe_ctl->total_free_space; + spin_lock(&space_info->lock); + space_info->max_extent_size = ffe_ctl->max_extent_size; + spin_unlock(&space_info->lock); + ins->offset = ffe_ctl->max_extent_size; + } else if (ret == -ENOSPC) { + ret = cache_block_group_error; + } + return ret; +} + +/* + * Entry point to the extent allocator. Tries to find a hole that is at least + * as big as @num_bytes. + * + * @root - The root that will contain this extent + * + * @ram_bytes - The amount of space in ram that @num_bytes take. This + * is used for accounting purposes. This value differs + * from @num_bytes only in the case of compressed extents. + * + * @num_bytes - Number of bytes to allocate on-disk. + * + * @min_alloc_size - Indicates the minimum amount of space that the + * allocator should try to satisfy. In some cases + * @num_bytes may be larger than what is required and if + * the filesystem is fragmented then allocation fails. + * However, the presence of @min_alloc_size gives a + * chance to try and satisfy the smaller allocation. + * + * @empty_size - A hint that you plan on doing more COW. This is the + * size in bytes the allocator should try to find free + * next to the block it returns. This is just a hint and + * may be ignored by the allocator. + * + * @hint_byte - Hint to the allocator to start searching above the byte + * address passed. It might be ignored. + * + * @ins - This key is modified to record the found hole. It will + * have the following values: + * ins->objectid == start position + * ins->flags = BTRFS_EXTENT_ITEM_KEY + * ins->offset == the size of the hole. + * + * @is_data - Boolean flag indicating whether an extent is + * allocated for data (true) or metadata (false) + * + * @delalloc - Boolean flag indicating whether this allocation is for + * delalloc or not. If 'true' data_rwsem of block groups + * is going to be acquired. + * + * + * Returns 0 when an allocation succeeded or < 0 when an error occurred. In + * case -ENOSPC is returned then @ins->offset will contain the size of the + * largest available hole the allocator managed to find. + */ +int btrfs_reserve_extent(struct btrfs_root *root, u64 ram_bytes, + u64 num_bytes, u64 min_alloc_size, + u64 empty_size, u64 hint_byte, + struct btrfs_key *ins, bool is_data, bool delalloc) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct find_free_extent_ctl ffe_ctl = {}; + bool final_tried = num_bytes == min_alloc_size; + u64 flags; + int ret; + bool for_treelog = (btrfs_root_id(root) == BTRFS_TREE_LOG_OBJECTID); + bool for_data_reloc = (btrfs_is_data_reloc_root(root) && is_data); + + flags = get_alloc_profile_by_root(root, is_data); +again: + WARN_ON(num_bytes < fs_info->sectorsize); + + ffe_ctl.ram_bytes = ram_bytes; + ffe_ctl.num_bytes = num_bytes; + ffe_ctl.min_alloc_size = min_alloc_size; + ffe_ctl.empty_size = empty_size; + ffe_ctl.flags = flags; + ffe_ctl.delalloc = delalloc; + ffe_ctl.hint_byte = hint_byte; + ffe_ctl.for_treelog = for_treelog; + ffe_ctl.for_data_reloc = for_data_reloc; + + ret = find_free_extent(root, ins, &ffe_ctl); + if (!ret && !is_data) { + btrfs_dec_block_group_reservations(fs_info, ins->objectid); + } else if (ret == -ENOSPC) { + if (!final_tried && ins->offset) { + num_bytes = min(num_bytes >> 1, ins->offset); + num_bytes = round_down(num_bytes, + fs_info->sectorsize); + num_bytes = max(num_bytes, min_alloc_size); + ram_bytes = num_bytes; + if (num_bytes == min_alloc_size) + final_tried = true; + goto again; + } else if (btrfs_test_opt(fs_info, ENOSPC_DEBUG)) { + struct btrfs_space_info *sinfo; + + sinfo = btrfs_find_space_info(fs_info, flags); + btrfs_err(fs_info, + "allocation failed flags %llu, wanted %llu tree-log %d, relocation: %d", + flags, num_bytes, for_treelog, for_data_reloc); + if (sinfo) + btrfs_dump_space_info(sinfo, num_bytes, 1); + } + } + + return ret; +} + +int btrfs_free_reserved_extent(struct btrfs_fs_info *fs_info, u64 start, u64 len, + bool is_delalloc) +{ + struct btrfs_block_group *cache; + + cache = btrfs_lookup_block_group(fs_info, start); + if (!cache) { + btrfs_err(fs_info, "Unable to find block group for %llu", + start); + return -ENOSPC; + } + + btrfs_add_free_space(cache, start, len); + btrfs_free_reserved_bytes(cache, len, is_delalloc); + trace_btrfs_reserved_extent_free(fs_info, start, len); + + btrfs_put_block_group(cache); + return 0; +} + +int btrfs_pin_reserved_extent(struct btrfs_trans_handle *trans, + const struct extent_buffer *eb) +{ + struct btrfs_block_group *cache; + int ret = 0; + + cache = btrfs_lookup_block_group(trans->fs_info, eb->start); + if (!cache) { + btrfs_err(trans->fs_info, "unable to find block group for %llu", + eb->start); + return -ENOSPC; + } + + ret = pin_down_extent(trans, cache, eb->start, eb->len, true); + btrfs_put_block_group(cache); + return ret; +} + +static int alloc_reserved_extent(struct btrfs_trans_handle *trans, u64 bytenr, + u64 num_bytes) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + int ret; + + ret = btrfs_remove_from_free_space_tree(trans, bytenr, num_bytes); + if (ret) + return ret; + + ret = btrfs_update_block_group(trans, bytenr, num_bytes, true); + if (ret) { + ASSERT(!ret); + btrfs_err(fs_info, "update block group failed for %llu %llu", + bytenr, num_bytes); + return ret; + } + + trace_btrfs_reserved_extent_alloc(fs_info, bytenr, num_bytes); + return 0; +} + +static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans, + u64 parent, u64 root_objectid, + u64 flags, u64 owner, u64 offset, + struct btrfs_key *ins, int ref_mod, u64 oref_root) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *extent_root; + int ret; + struct btrfs_extent_item *extent_item; + struct btrfs_extent_owner_ref *oref; + struct btrfs_extent_inline_ref *iref; + struct btrfs_path *path; + struct extent_buffer *leaf; + int type; + u32 size; + const bool simple_quota = (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE); + + if (parent > 0) + type = BTRFS_SHARED_DATA_REF_KEY; + else + type = BTRFS_EXTENT_DATA_REF_KEY; + + size = sizeof(*extent_item); + if (simple_quota) + size += btrfs_extent_inline_ref_size(BTRFS_EXTENT_OWNER_REF_KEY); + size += btrfs_extent_inline_ref_size(type); + + extent_root = btrfs_extent_root(fs_info, ins->objectid); + if (unlikely(!extent_root)) { + btrfs_err(fs_info, + "missing extent root for extent at bytenr %llu", + ins->objectid); + return -EUCLEAN; + } + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + ret = btrfs_insert_empty_item(trans, extent_root, path, ins, size); + if (ret) { + btrfs_free_path(path); + return ret; + } + + leaf = path->nodes[0]; + extent_item = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_extent_item); + btrfs_set_extent_refs(leaf, extent_item, ref_mod); + btrfs_set_extent_generation(leaf, extent_item, trans->transid); + btrfs_set_extent_flags(leaf, extent_item, + flags | BTRFS_EXTENT_FLAG_DATA); + + iref = (struct btrfs_extent_inline_ref *)(extent_item + 1); + if (simple_quota) { + btrfs_set_extent_inline_ref_type(leaf, iref, BTRFS_EXTENT_OWNER_REF_KEY); + oref = (struct btrfs_extent_owner_ref *)(&iref->offset); + btrfs_set_extent_owner_ref_root_id(leaf, oref, oref_root); + iref = (struct btrfs_extent_inline_ref *)(oref + 1); + } + btrfs_set_extent_inline_ref_type(leaf, iref, type); + + if (parent > 0) { + struct btrfs_shared_data_ref *ref; + ref = (struct btrfs_shared_data_ref *)(iref + 1); + btrfs_set_extent_inline_ref_offset(leaf, iref, parent); + btrfs_set_shared_data_ref_count(leaf, ref, ref_mod); + } else { + struct btrfs_extent_data_ref *ref; + ref = (struct btrfs_extent_data_ref *)(&iref->offset); + btrfs_set_extent_data_ref_root(leaf, ref, root_objectid); + btrfs_set_extent_data_ref_objectid(leaf, ref, owner); + btrfs_set_extent_data_ref_offset(leaf, ref, offset); + btrfs_set_extent_data_ref_count(leaf, ref, ref_mod); + } + + btrfs_free_path(path); + + return alloc_reserved_extent(trans, ins->objectid, ins->offset); +} + +static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans, + const struct btrfs_delayed_ref_node *node, + struct btrfs_delayed_extent_op *extent_op) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *extent_root; + int ret; + struct btrfs_extent_item *extent_item; + struct btrfs_key extent_key; + struct btrfs_tree_block_info *block_info; + struct btrfs_extent_inline_ref *iref; + struct btrfs_path *path; + struct extent_buffer *leaf; + u32 size = sizeof(*extent_item) + sizeof(*iref); + const u64 flags = (extent_op ? extent_op->flags_to_set : 0); + /* The owner of a tree block is the level. */ + int level = btrfs_delayed_ref_owner(node); + bool skinny_metadata = btrfs_fs_incompat(fs_info, SKINNY_METADATA); + + if (unlikely(node->ref_root == BTRFS_REMAP_TREE_OBJECTID)) + goto skip; + + extent_key.objectid = node->bytenr; + if (skinny_metadata) { + /* The owner of a tree block is the level. */ + extent_key.offset = level; + extent_key.type = BTRFS_METADATA_ITEM_KEY; + } else { + extent_key.offset = node->num_bytes; + extent_key.type = BTRFS_EXTENT_ITEM_KEY; + size += sizeof(*block_info); + } + + extent_root = btrfs_extent_root(fs_info, extent_key.objectid); + if (unlikely(!extent_root)) { + btrfs_err(fs_info, + "missing extent root for extent at bytenr %llu", + extent_key.objectid); + return -EUCLEAN; + } + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + ret = btrfs_insert_empty_item(trans, extent_root, path, &extent_key, + size); + if (ret) { + btrfs_free_path(path); + return ret; + } + + leaf = path->nodes[0]; + extent_item = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_extent_item); + btrfs_set_extent_refs(leaf, extent_item, 1); + btrfs_set_extent_generation(leaf, extent_item, trans->transid); + btrfs_set_extent_flags(leaf, extent_item, + flags | BTRFS_EXTENT_FLAG_TREE_BLOCK); + + if (skinny_metadata) { + iref = (struct btrfs_extent_inline_ref *)(extent_item + 1); + } else { + block_info = (struct btrfs_tree_block_info *)(extent_item + 1); + btrfs_set_tree_block_key(leaf, block_info, &extent_op->key); + btrfs_set_tree_block_level(leaf, block_info, level); + iref = (struct btrfs_extent_inline_ref *)(block_info + 1); + } + + if (node->type == BTRFS_SHARED_BLOCK_REF_KEY) { + btrfs_set_extent_inline_ref_type(leaf, iref, + BTRFS_SHARED_BLOCK_REF_KEY); + btrfs_set_extent_inline_ref_offset(leaf, iref, node->parent); + } else { + btrfs_set_extent_inline_ref_type(leaf, iref, + BTRFS_TREE_BLOCK_REF_KEY); + btrfs_set_extent_inline_ref_offset(leaf, iref, node->ref_root); + } + + btrfs_free_path(path); + +skip: + return alloc_reserved_extent(trans, node->bytenr, fs_info->nodesize); +} + +int btrfs_alloc_reserved_file_extent(struct btrfs_trans_handle *trans, + struct btrfs_root *root, u64 owner, + u64 offset, u64 ram_bytes, + struct btrfs_key *ins) +{ + struct btrfs_ref generic_ref = { + .action = BTRFS_ADD_DELAYED_EXTENT, + .bytenr = ins->objectid, + .num_bytes = ins->offset, + .owning_root = btrfs_root_id(root), + .ref_root = btrfs_root_id(root), + }; + + ASSERT(generic_ref.ref_root != BTRFS_TREE_LOG_OBJECTID); + + if (btrfs_is_data_reloc_root(root) && btrfs_is_fstree(root->relocation_src_root)) + generic_ref.owning_root = root->relocation_src_root; + + btrfs_init_data_ref(&generic_ref, owner, offset, 0, false); + btrfs_ref_tree_mod(root->fs_info, &generic_ref); + + return btrfs_add_delayed_data_ref(trans, &generic_ref, ram_bytes); +} + +/* + * this is used by the tree logging recovery code. It records that + * an extent has been allocated and makes sure to clear the free + * space cache bits as well + */ +int btrfs_alloc_logged_file_extent(struct btrfs_trans_handle *trans, + u64 root_objectid, u64 owner, u64 offset, + struct btrfs_key *ins) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + int ret; + struct btrfs_block_group *block_group; + struct btrfs_space_info *space_info; + const struct btrfs_squota_delta delta = { + .root = root_objectid, + .num_bytes = ins->offset, + .generation = trans->transid, + .is_data = true, + .is_inc = true, + }; + + /* + * Mixed block groups will exclude before processing the log so we only + * need to do the exclude dance if this fs isn't mixed. + */ + if (!btrfs_fs_incompat(fs_info, MIXED_GROUPS)) { + ret = __exclude_logged_extent(fs_info, ins->objectid, + ins->offset); + if (ret) + return ret; + } + + block_group = btrfs_lookup_block_group(fs_info, ins->objectid); + if (!block_group) + return -EINVAL; + + space_info = block_group->space_info; + spin_lock(&space_info->lock); + spin_lock(&block_group->lock); + space_info->bytes_reserved += ins->offset; + block_group->reserved += ins->offset; + spin_unlock(&block_group->lock); + spin_unlock(&space_info->lock); + + ret = alloc_reserved_file_extent(trans, 0, root_objectid, 0, owner, + offset, ins, 1, root_objectid); + if (ret) + btrfs_pin_extent(trans, ins->objectid, ins->offset); + ret = btrfs_record_squota_delta(fs_info, &delta); + btrfs_put_block_group(block_group); + return ret; +} + +#ifdef CONFIG_BTRFS_DEBUG +/* + * Extra safety check in case the extent tree is corrupted and extent allocator + * chooses to use a tree block which is already used and locked. + */ +static bool check_eb_lock_owner(const struct extent_buffer *eb) +{ + if (eb->lock_owner == current->pid) { + btrfs_err_rl(eb->fs_info, +"tree block %llu owner %llu already locked by pid=%d, extent tree corruption detected", + eb->start, btrfs_header_owner(eb), current->pid); + return true; + } + return false; +} +#else +static bool check_eb_lock_owner(struct extent_buffer *eb) +{ + return false; +} +#endif + +static struct extent_buffer * +btrfs_init_new_buffer(struct btrfs_trans_handle *trans, struct btrfs_root *root, + u64 bytenr, int level, u64 owner, + enum btrfs_lock_nesting nest) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_eb_prealloc pa = { 0 }; + struct extent_buffer *buf; + u64 lockdep_owner = owner; + + buf = btrfs_find_create_tree_block(fs_info, &pa, bytenr, owner, level); + if (IS_ERR(buf)) + return buf; + + if (unlikely(check_eb_lock_owner(buf))) { + free_extent_buffer(buf); + return ERR_PTR(-EUCLEAN); + } + + /* + * The reloc trees are just snapshots, so we need them to appear to be + * just like any other fs tree WRT lockdep. + * + * The exception however is in replace_path() in relocation, where we + * hold the lock on the original fs root and then search for the reloc + * root. At that point we need to make sure any reloc root buffers are + * set to the BTRFS_TREE_RELOC_OBJECTID lockdep class in order to make + * lockdep happy. + */ + if (lockdep_owner == BTRFS_TREE_RELOC_OBJECTID && + !test_bit(BTRFS_ROOT_RESET_LOCKDEP_CLASS, &root->state)) + lockdep_owner = BTRFS_FS_TREE_OBJECTID; + + /* btrfs_clear_buffer_dirty() accesses generation field. */ + btrfs_set_header_generation(buf, trans->transid); + + /* + * This needs to stay, because we could allocate a freed block from an + * old tree into a new tree, so we need to make sure this new block is + * set to the appropriate level and owner. + */ + btrfs_set_buffer_lockdep_class(lockdep_owner, buf, level); + + btrfs_tree_lock_nested(buf, nest); + btrfs_clear_buffer_dirty(trans, buf); + clear_bit(EXTENT_BUFFER_STALE, &buf->bflags); + clear_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &buf->bflags); + + set_extent_buffer_uptodate(buf); + + memzero_extent_buffer(buf, 0, sizeof(struct btrfs_header)); + btrfs_set_header_level(buf, level); + btrfs_set_header_bytenr(buf, buf->start); + btrfs_set_header_generation(buf, trans->transid); + btrfs_set_header_backref_rev(buf, BTRFS_MIXED_BACKREF_REV); + btrfs_set_header_owner(buf, owner); + write_extent_buffer_fsid(buf, fs_info->fs_devices->metadata_uuid); + write_extent_buffer_chunk_tree_uuid(buf, fs_info->chunk_tree_uuid); + if (btrfs_root_id(root) == BTRFS_TREE_LOG_OBJECTID) { + buf->log_index = root->log_transid % 2; + /* + * we allow two log transactions at a time, use different + * EXTENT bit to differentiate dirty pages. + */ + if (buf->log_index == 0) + btrfs_set_extent_bit(&root->dirty_log_pages, buf->start, + buf->start + buf->len - 1, + EXTENT_DIRTY_LOG1, NULL); + else + btrfs_set_extent_bit(&root->dirty_log_pages, buf->start, + buf->start + buf->len - 1, + EXTENT_DIRTY_LOG2, NULL); + } else { + buf->log_index = -1; + btrfs_set_extent_bit(&trans->transaction->dirty_pages, buf->start, + buf->start + buf->len - 1, EXTENT_DIRTY, NULL); + } + /* this returns a buffer locked for blocking */ + return buf; +} + +/* + * finds a free extent and does all the dirty work required for allocation + * returns the tree buffer or an ERR_PTR on error. + */ +struct extent_buffer *btrfs_alloc_tree_block(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + u64 parent, u64 root_objectid, + const struct btrfs_disk_key *key, + int level, u64 hint, + u64 empty_size, + u64 reloc_src_root, + enum btrfs_lock_nesting nest) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_key ins; + struct btrfs_block_rsv *block_rsv; + struct extent_buffer *buf; + u64 flags = 0; + int ret; + u32 blocksize = fs_info->nodesize; + bool skinny_metadata = btrfs_fs_incompat(fs_info, SKINNY_METADATA); + u64 owning_root; + +#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS + if (btrfs_is_testing(fs_info)) { + buf = btrfs_init_new_buffer(trans, root, root->alloc_bytenr, + level, root_objectid, nest); + if (!IS_ERR(buf)) + root->alloc_bytenr += blocksize; + return buf; + } +#endif + + block_rsv = btrfs_use_block_rsv(trans, root, blocksize); + if (IS_ERR(block_rsv)) + return ERR_CAST(block_rsv); + + ret = btrfs_reserve_extent(root, blocksize, blocksize, blocksize, + empty_size, hint, &ins, false, false); + if (ret) + goto out_unuse; + + buf = btrfs_init_new_buffer(trans, root, ins.objectid, level, + root_objectid, nest); + if (IS_ERR(buf)) { + ret = PTR_ERR(buf); + goto out_free_reserved; + } + owning_root = btrfs_header_owner(buf); + + if (root_objectid == BTRFS_TREE_RELOC_OBJECTID) { + if (parent == 0) + parent = ins.objectid; + flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF; + owning_root = reloc_src_root; + } else + BUG_ON(parent > 0); + + if (root_objectid != BTRFS_TREE_LOG_OBJECTID) { + struct btrfs_delayed_extent_op *extent_op; + struct btrfs_ref generic_ref = { + .action = BTRFS_ADD_DELAYED_EXTENT, + .bytenr = ins.objectid, + .num_bytes = ins.offset, + .parent = parent, + .owning_root = owning_root, + .ref_root = root_objectid, + }; + + if (!skinny_metadata || flags != 0) { + extent_op = btrfs_alloc_delayed_extent_op(); + if (!extent_op) { + ret = -ENOMEM; + goto out_free_buf; + } + if (key) + memcpy(&extent_op->key, key, sizeof(extent_op->key)); + else + memset(&extent_op->key, 0, sizeof(extent_op->key)); + extent_op->flags_to_set = flags; + extent_op->update_key = (skinny_metadata ? false : true); + extent_op->update_flags = (flags != 0); + } else { + extent_op = NULL; + } + + btrfs_init_tree_ref(&generic_ref, level, btrfs_root_id(root), false); + btrfs_ref_tree_mod(fs_info, &generic_ref); + ret = btrfs_add_delayed_tree_ref(trans, &generic_ref, extent_op); + if (ret) { + btrfs_free_delayed_extent_op(extent_op); + goto out_free_buf; + } + } + return buf; + +out_free_buf: + btrfs_tree_unlock(buf); + free_extent_buffer(buf); +out_free_reserved: + btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, false); +out_unuse: + btrfs_unuse_block_rsv(fs_info, block_rsv, blocksize); + return ERR_PTR(ret); +} + +struct walk_control { + u64 refs[BTRFS_MAX_LEVEL]; + u64 flags[BTRFS_MAX_LEVEL]; + struct btrfs_key update_progress; + struct btrfs_key drop_progress; + int drop_level; + int stage; + int level; + int shared_level; + int update_ref; + int keep_locks; + int reada_slot; + int reada_count; + int restarted; + /* Indicate that extent info needs to be looked up when walking the tree. */ + int lookup_info; +}; + +/* + * This is our normal stage. We are traversing blocks the current snapshot owns + * and we are dropping any of our references to any children we are able to, and + * then freeing the block once we've processed all of the children. + */ +#define DROP_REFERENCE 1 + +/* + * We enter this stage when we have to walk into a child block (meaning we can't + * simply drop our reference to it from our current parent node) and there are + * more than one reference on it. If we are the owner of any of the children + * blocks from the current parent node then we have to do the FULL_BACKREF dance + * on them in order to drop our normal ref and add the shared ref. + */ +#define UPDATE_BACKREF 2 + +/* + * Decide if we need to walk down into this node to adjust the references. + * + * @root: the root we are currently deleting + * @wc: the walk control for this deletion + * @eb: the parent eb that we're currently visiting + * @flags: the flags for wc->level - 1 + * @slot: the slot in the eb that we're currently checking + * + * This is meant to be called when we're evaluating if a node we point to at + * wc->level should be read and walked into, or if we can simply delete our + * reference to it. We return true if we should walk into the node, false if we + * can skip it. + * + * We have assertions in here to make sure this is called correctly. We assume + * that sanity checking on the blocks read to this point has been done, so any + * corrupted file systems must have been caught before calling this function. + */ +static bool visit_node_for_delete(struct btrfs_root *root, struct walk_control *wc, + struct extent_buffer *eb, u64 flags, int slot) +{ + struct btrfs_key key; + u64 generation; + int level = wc->level; + + ASSERT(level > 0); + ASSERT(wc->refs[level - 1] > 0); + + /* + * The update backref stage we only want to skip if we already have + * FULL_BACKREF set, otherwise we need to read. + */ + if (wc->stage == UPDATE_BACKREF) { + if (level == 1 && flags & BTRFS_BLOCK_FLAG_FULL_BACKREF) + return false; + return true; + } + + /* + * We're the last ref on this block, we must walk into it and process + * any refs it's pointing at. + */ + if (wc->refs[level - 1] == 1) + return true; + + /* + * If we're already FULL_BACKREF then we know we can just drop our + * current reference. + */ + if (level == 1 && flags & BTRFS_BLOCK_FLAG_FULL_BACKREF) + return false; + + /* + * This block is older than our creation generation, we can drop our + * reference to it. + */ + generation = btrfs_node_ptr_generation(eb, slot); + if (!wc->update_ref || generation <= btrfs_root_origin_generation(root)) + return false; + + /* + * This block was processed from a previous snapshot deletion run, we + * can skip it. + */ + btrfs_node_key_to_cpu(eb, &key, slot); + if (btrfs_comp_cpu_keys(&key, &wc->update_progress) < 0) + return false; + + /* All other cases we need to wander into the node. */ + return true; +} + +static noinline void reada_walk_down(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct walk_control *wc, + struct btrfs_path *path) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + u64 bytenr; + u64 generation; + u64 refs; + u64 flags; + u32 nritems; + struct extent_buffer *eb; + int ret; + int slot; + int nread = 0; + + if (path->slots[wc->level] < wc->reada_slot) { + wc->reada_count = wc->reada_count * 2 / 3; + wc->reada_count = max(wc->reada_count, 2); + } else { + wc->reada_count = wc->reada_count * 3 / 2; + wc->reada_count = min_t(int, wc->reada_count, + BTRFS_NODEPTRS_PER_BLOCK(fs_info)); + } + + eb = path->nodes[wc->level]; + nritems = btrfs_header_nritems(eb); + + for (slot = path->slots[wc->level]; slot < nritems; slot++) { + if (nread >= wc->reada_count) + break; + + cond_resched(); + bytenr = btrfs_node_blockptr(eb, slot); + generation = btrfs_node_ptr_generation(eb, slot); + + if (slot == path->slots[wc->level]) + goto reada; + + if (wc->stage == UPDATE_BACKREF && + generation <= btrfs_root_origin_generation(root)) + continue; + + /* We don't lock the tree block, it's OK to be racy here */ + ret = btrfs_lookup_extent_info(trans, fs_info, bytenr, + wc->level - 1, 1, &refs, + &flags, NULL); + /* We don't care about errors in readahead. */ + if (ret < 0) + continue; + + /* + * This could be racey, it's conceivable that we raced and end + * up with a bogus refs count, if that's the case just skip, if + * we are actually corrupt we will notice when we look up + * everything again with our locks. + */ + if (refs == 0) + continue; + + /* If we don't need to visit this node don't reada. */ + if (!visit_node_for_delete(root, wc, eb, flags, slot)) + continue; +reada: + btrfs_readahead_node_child(eb, slot); + nread++; + } + wc->reada_slot = slot; +} + +/* + * helper to process tree block while walking down the tree. + * + * when wc->stage == UPDATE_BACKREF, this function updates + * back refs for pointers in the block. + * + * NOTE: return value 1 means we should stop walking down. + */ +static noinline int walk_down_proc(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + struct walk_control *wc) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + int level = wc->level; + struct extent_buffer *eb = path->nodes[level]; + u64 flag = BTRFS_BLOCK_FLAG_FULL_BACKREF; + int ret; + + if (wc->stage == UPDATE_BACKREF && btrfs_header_owner(eb) != btrfs_root_id(root)) + return 1; + + /* + * when reference count of tree block is 1, it won't increase + * again. once full backref flag is set, we never clear it. + */ + if (wc->lookup_info && + ((wc->stage == DROP_REFERENCE && wc->refs[level] != 1) || + (wc->stage == UPDATE_BACKREF && !(wc->flags[level] & flag)))) { + ASSERT(path->locks[level]); + ret = btrfs_lookup_extent_info(trans, fs_info, + eb->start, level, 1, + &wc->refs[level], + &wc->flags[level], + NULL); + if (ret) + return ret; + if (unlikely(wc->refs[level] == 0)) { + btrfs_err(fs_info, "bytenr %llu has 0 references, expect > 0", + eb->start); + return -EUCLEAN; + } + } + + if (wc->stage == DROP_REFERENCE) { + if (wc->refs[level] > 1) + return 1; + + if (path->locks[level] && !wc->keep_locks) { + btrfs_tree_unlock_rw(eb, path->locks[level]); + path->locks[level] = 0; + } + return 0; + } + + /* wc->stage == UPDATE_BACKREF */ + if (!(wc->flags[level] & flag)) { + ASSERT(path->locks[level]); + ret = btrfs_inc_ref(trans, root, eb, true); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + ret = btrfs_dec_ref(trans, root, eb, false); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + ret = btrfs_set_disk_extent_flags(trans, eb, flag); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + wc->flags[level] |= flag; + } + + /* + * the block is shared by multiple trees, so it's not good to + * keep the tree lock + */ + if (path->locks[level] && level > 0) { + btrfs_tree_unlock_rw(eb, path->locks[level]); + path->locks[level] = 0; + } + return 0; +} + +/* + * This is used to verify a ref exists for this root to deal with a bug where we + * would have a drop_progress key that hadn't been updated properly. + */ +static int check_ref_exists(struct btrfs_trans_handle *trans, + struct btrfs_root *root, u64 bytenr, u64 parent, + int level) +{ + struct btrfs_delayed_ref_root *delayed_refs; + struct btrfs_delayed_ref_head *head; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_extent_inline_ref *iref; + int ret; + bool exists = false; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; +again: + ret = lookup_extent_backref(trans, path, &iref, bytenr, + root->fs_info->nodesize, parent, + btrfs_root_id(root), level, 0); + if (ret != -ENOENT) { + /* + * If we get 0 then we found our reference, return 1, else + * return the error if it's not -ENOENT; + */ + return (ret < 0 ) ? ret : 1; + } + + /* + * We could have a delayed ref with this reference, so look it up while + * we're holding the path open to make sure we don't race with the + * delayed ref running. + */ + delayed_refs = &trans->transaction->delayed_refs; + spin_lock(&delayed_refs->lock); + head = btrfs_find_delayed_ref_head(root->fs_info, delayed_refs, bytenr); + if (!head) + goto out; + if (!mutex_trylock(&head->mutex)) { + /* + * We're contended, means that the delayed ref is running, get a + * reference and wait for the ref head to be complete and then + * try again. + */ + refcount_inc(&head->refs); + spin_unlock(&delayed_refs->lock); + + btrfs_release_path(path); + + mutex_lock(&head->mutex); + mutex_unlock(&head->mutex); + btrfs_put_delayed_ref_head(head); + goto again; + } + + exists = btrfs_find_delayed_tree_ref(head, btrfs_root_id(root), parent); + mutex_unlock(&head->mutex); +out: + spin_unlock(&delayed_refs->lock); + return exists ? 1 : 0; +} + +/* + * We may not have an uptodate block, so if we are going to walk down into this + * block we need to drop the lock, read it off of the disk, re-lock it and + * return to continue dropping the snapshot. + */ +static int check_next_block_uptodate(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + struct walk_control *wc, + struct extent_buffer *next) +{ + struct btrfs_tree_parent_check check = { 0 }; + u64 generation; + int level = wc->level; + int ret; + + btrfs_assert_tree_write_locked(next); + + generation = btrfs_node_ptr_generation(path->nodes[level], path->slots[level]); + + check.level = level - 1; + check.transid = generation; + check.owner_root = btrfs_root_id(root); + check.has_first_key = true; + btrfs_node_key_to_cpu(path->nodes[level], &check.first_key, path->slots[level]); + + ret = btrfs_buffer_uptodate(next, generation, &check); + if (ret > 0) + return 0; + btrfs_tree_unlock(next); + if (ret < 0) { + free_extent_buffer(next); + return ret; + } + + if (level == 1) + reada_walk_down(trans, root, wc, path); + ret = btrfs_read_extent_buffer(next, &check); + if (ret) { + free_extent_buffer(next); + return ret; + } + btrfs_tree_lock(next); + wc->lookup_info = 1; + return 0; +} + +/* + * If we determine that we don't have to visit wc->level - 1 then we need to + * determine if we can drop our reference. + * + * If we are UPDATE_BACKREF then we will not, we need to update our backrefs. + * + * If we are DROP_REFERENCE this will figure out if we need to drop our current + * reference, skipping it if we dropped it from a previous uncompleted drop, or + * dropping it if we still have a reference to it. + */ +static int maybe_drop_reference(struct btrfs_trans_handle *trans, struct btrfs_root *root, + struct btrfs_path *path, struct walk_control *wc, + struct extent_buffer *next, u64 owner_root) +{ + struct btrfs_ref ref = { + .action = BTRFS_DROP_DELAYED_REF, + .bytenr = next->start, + .num_bytes = root->fs_info->nodesize, + .owning_root = owner_root, + .ref_root = btrfs_root_id(root), + }; + int level = wc->level; + int ret; + + /* We are UPDATE_BACKREF, we're not dropping anything. */ + if (wc->stage == UPDATE_BACKREF) + return 0; + + if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) { + ref.parent = path->nodes[level]->start; + } else { + ASSERT(btrfs_root_id(root) == btrfs_header_owner(path->nodes[level])); + if (unlikely(btrfs_root_id(root) != btrfs_header_owner(path->nodes[level]))) { + btrfs_err(root->fs_info, "mismatched block owner"); + return -EIO; + } + } + + /* + * If we had a drop_progress we need to verify the refs are set as + * expected. If we find our ref then we know that from here on out + * everything should be correct, and we can clear the + * ->restarted flag. + */ + if (wc->restarted) { + ret = check_ref_exists(trans, root, next->start, ref.parent, + level - 1); + if (ret <= 0) + return ret; + ret = 0; + wc->restarted = 0; + } + + /* + * Reloc tree doesn't contribute to qgroup numbers, and we have already + * accounted them at merge time (replace_path), thus we could skip + * expensive subtree trace here. + */ + if (btrfs_root_id(root) != BTRFS_TREE_RELOC_OBJECTID && + wc->refs[level - 1] > 1) { + u64 generation = btrfs_node_ptr_generation(path->nodes[level], + path->slots[level]); + + ret = btrfs_qgroup_trace_subtree(trans, next, generation, level - 1); + if (ret) { + btrfs_err_rl(root->fs_info, +"error %pe accounting shared subtree, quota is out of sync, rescan required", + ERR_PTR(ret)); + } + } + + /* + * We need to update the next key in our walk control so we can update + * the drop_progress key accordingly. We don't care if find_next_key + * doesn't find a key because that means we're at the end and are going + * to clean up now. + */ + wc->drop_level = level; + find_next_key(path, level, &wc->drop_progress); + + btrfs_init_tree_ref(&ref, level - 1, 0, false); + return btrfs_free_extent(trans, &ref); +} + +/* + * helper to process tree block pointer. + * + * when wc->stage == DROP_REFERENCE, this function checks + * reference count of the block pointed to. if the block + * is shared and we need update back refs for the subtree + * rooted at the block, this function changes wc->stage to + * UPDATE_BACKREF. if the block is shared and there is no + * need to update back, this function drops the reference + * to the block. + * + * NOTE: return value 1 means we should stop walking down. + */ +static noinline int do_walk_down(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + struct walk_control *wc) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_eb_prealloc pa = { 0 }; + u64 bytenr; + u64 generation; + u64 owner_root = 0; + struct extent_buffer *next; + int level = wc->level; + int ret = 0; + + generation = btrfs_node_ptr_generation(path->nodes[level], + path->slots[level]); + /* + * if the lower level block was created before the snapshot + * was created, we know there is no need to update back refs + * for the subtree + */ + if (wc->stage == UPDATE_BACKREF && + generation <= btrfs_root_origin_generation(root)) { + wc->lookup_info = 1; + return 1; + } + + bytenr = btrfs_node_blockptr(path->nodes[level], path->slots[level]); + + next = btrfs_find_create_tree_block(fs_info, &pa, bytenr, btrfs_root_id(root), + level - 1); + if (IS_ERR(next)) + return PTR_ERR(next); + + btrfs_tree_lock(next); + + ret = btrfs_lookup_extent_info(trans, fs_info, bytenr, level - 1, 1, + &wc->refs[level - 1], + &wc->flags[level - 1], + &owner_root); + if (ret < 0) + goto out_unlock; + + if (unlikely(wc->refs[level - 1] == 0)) { + btrfs_err(fs_info, "bytenr %llu has 0 references, expect > 0", + bytenr); + ret = -EUCLEAN; + goto out_unlock; + } + wc->lookup_info = 0; + + /* If we don't have to walk into this node skip it. */ + if (!visit_node_for_delete(root, wc, path->nodes[level], + wc->flags[level - 1], path->slots[level])) + goto skip; + + /* + * We have to walk down into this node, and if we're currently at the + * DROP_REFERENCE stage and this block is shared then we need to switch + * to the UPDATE_BACKREF stage in order to convert to FULL_BACKREF. + */ + if (wc->stage == DROP_REFERENCE && wc->refs[level - 1] > 1) { + wc->stage = UPDATE_BACKREF; + wc->shared_level = level - 1; + } + + ret = check_next_block_uptodate(trans, root, path, wc, next); + if (ret) + return ret; + + level--; + ASSERT(level == btrfs_header_level(next)); + if (unlikely(level != btrfs_header_level(next))) { + btrfs_err(root->fs_info, "mismatched level"); + ret = -EIO; + goto out_unlock; + } + path->nodes[level] = next; + path->slots[level] = 0; + path->locks[level] = BTRFS_WRITE_LOCK; + wc->level = level; + if (wc->level == 1) + wc->reada_slot = 0; + return 0; +skip: + ret = maybe_drop_reference(trans, root, path, wc, next, owner_root); + if (ret) + goto out_unlock; + wc->refs[level - 1] = 0; + wc->flags[level - 1] = 0; + wc->lookup_info = 1; + ret = 1; + +out_unlock: + btrfs_tree_unlock(next); + free_extent_buffer(next); + + return ret; +} + +/* + * helper to process tree block while walking up the tree. + * + * when wc->stage == DROP_REFERENCE, this function drops + * reference count on the block. + * + * when wc->stage == UPDATE_BACKREF, this function changes + * wc->stage back to DROP_REFERENCE if we changed wc->stage + * to UPDATE_BACKREF previously while processing the block. + * + * NOTE: return value 1 means we should stop walking up. + */ +static noinline int walk_up_proc(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + struct walk_control *wc) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + int ret = 0; + int level = wc->level; + struct extent_buffer *eb = path->nodes[level]; + u64 parent = 0; + + if (wc->stage == UPDATE_BACKREF) { + ASSERT(wc->shared_level >= level); + if (level < wc->shared_level) + goto out; + + ret = find_next_key(path, level + 1, &wc->update_progress); + if (ret > 0) + wc->update_ref = 0; + + wc->stage = DROP_REFERENCE; + wc->shared_level = -1; + path->slots[level] = 0; + + /* + * check reference count again if the block isn't locked. + * we should start walking down the tree again if reference + * count is one. + */ + if (!path->locks[level]) { + ASSERT(level > 0); + btrfs_tree_lock(eb); + path->locks[level] = BTRFS_WRITE_LOCK; + + ret = btrfs_lookup_extent_info(trans, fs_info, + eb->start, level, 1, + &wc->refs[level], + &wc->flags[level], + NULL); + if (ret < 0) { + btrfs_tree_unlock_rw(eb, path->locks[level]); + path->locks[level] = 0; + return ret; + } + if (unlikely(wc->refs[level] == 0)) { + btrfs_tree_unlock_rw(eb, path->locks[level]); + btrfs_err(fs_info, "bytenr %llu has 0 references, expect > 0", + eb->start); + return -EUCLEAN; + } + if (wc->refs[level] == 1) { + btrfs_tree_unlock_rw(eb, path->locks[level]); + path->locks[level] = 0; + return 1; + } + } + } + + /* wc->stage == DROP_REFERENCE */ + ASSERT(path->locks[level] || wc->refs[level] == 1); + + if (wc->refs[level] == 1) { + if (level == 0) { + const bool full_backref = (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF); + + ret = btrfs_dec_ref(trans, root, eb, full_backref); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + if (btrfs_is_fstree(btrfs_root_id(root))) { + ret = btrfs_qgroup_trace_leaf_items(trans, eb); + if (ret) { + btrfs_err_rl(fs_info, + "error %pe accounting leaf items, quota is out of sync, rescan required", + ERR_PTR(ret)); + } + } + } + /* Make block locked assertion in btrfs_clear_buffer_dirty happy. */ + if (!path->locks[level]) { + btrfs_tree_lock(eb); + path->locks[level] = BTRFS_WRITE_LOCK; + } + btrfs_clear_buffer_dirty(trans, eb); + } + + if (eb == root->node) { + if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) + parent = eb->start; + else if (unlikely(btrfs_root_id(root) != btrfs_header_owner(eb))) + goto owner_mismatch; + } else { + if (wc->flags[level + 1] & BTRFS_BLOCK_FLAG_FULL_BACKREF) + parent = path->nodes[level + 1]->start; + else if (unlikely(btrfs_root_id(root) != + btrfs_header_owner(path->nodes[level + 1]))) + goto owner_mismatch; + } + + ret = btrfs_free_tree_block(trans, btrfs_root_id(root), eb, parent, + wc->refs[level] == 1); + if (ret < 0) + btrfs_abort_transaction(trans, ret); +out: + wc->refs[level] = 0; + wc->flags[level] = 0; + return ret; + +owner_mismatch: + btrfs_err_rl(fs_info, "unexpected tree owner, have %llu expect %llu", + btrfs_header_owner(eb), btrfs_root_id(root)); + return -EUCLEAN; +} + +/* + * walk_down_tree consists of two steps. + * + * walk_down_proc(). Look up the reference count and reference of our current + * wc->level. At this point path->nodes[wc->level] should be populated and + * uptodate, and in most cases should already be locked. If we are in + * DROP_REFERENCE and our refcount is > 1 then we've entered a shared node and + * we can walk back up the tree. If we are UPDATE_BACKREF we have to set + * FULL_BACKREF on this node if it's not already set, and then do the + * FULL_BACKREF conversion dance, which is to drop the root reference and add + * the shared reference to all of this nodes children. + * + * do_walk_down(). This is where we actually start iterating on the children of + * our current path->nodes[wc->level]. For DROP_REFERENCE that means dropping + * our reference to the children that return false from visit_node_for_delete(), + * which has various conditions where we know we can just drop our reference + * without visiting the node. For UPDATE_BACKREF we will skip any children that + * visit_node_for_delete() returns false for, only walking down when necessary. + * The bulk of the work for UPDATE_BACKREF occurs in the walk_up_tree() part of + * snapshot deletion. + */ +static noinline int walk_down_tree(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + struct walk_control *wc) +{ + int level = wc->level; + int ret = 0; + + wc->lookup_info = 1; + while (level >= 0) { + ret = walk_down_proc(trans, root, path, wc); + if (ret) + break; + + if (level == 0) + break; + + if (path->slots[level] >= + btrfs_header_nritems(path->nodes[level])) + break; + + ret = do_walk_down(trans, root, path, wc); + if (ret > 0) { + path->slots[level]++; + continue; + } else if (ret < 0) + break; + level = wc->level; + } + return (ret == 1) ? 0 : ret; +} + +/* + * walk_up_tree() is responsible for making sure we visit every slot on our + * current node, and if we're at the end of that node then we call + * walk_up_proc() on our current node which will do one of a few things based on + * our stage. + * + * UPDATE_BACKREF. If we wc->level is currently less than our wc->shared_level + * then we need to walk back up the tree, and then going back down into the + * other slots via walk_down_tree to update any other children from our original + * wc->shared_level. Once we're at or above our wc->shared_level we can switch + * back to DROP_REFERENCE, lookup the current nodes refs and flags, and carry on. + * + * DROP_REFERENCE. If our refs == 1 then we're going to free this tree block. + * If we're level 0 then we need to btrfs_dec_ref() on all of the data extents + * in our current leaf. After that we call btrfs_free_tree_block() on the + * current node and walk up to the next node to walk down the next slot. + */ +static noinline int walk_up_tree(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + struct walk_control *wc, int max_level) +{ + int level = wc->level; + int ret; + + path->slots[level] = btrfs_header_nritems(path->nodes[level]); + while (level < max_level && path->nodes[level]) { + wc->level = level; + if (path->slots[level] + 1 < + btrfs_header_nritems(path->nodes[level])) { + path->slots[level]++; + return 0; + } else { + ret = walk_up_proc(trans, root, path, wc); + if (ret > 0) + return 0; + if (ret < 0) + return ret; + + if (path->locks[level]) { + btrfs_tree_unlock_rw(path->nodes[level], + path->locks[level]); + path->locks[level] = 0; + } + free_extent_buffer(path->nodes[level]); + path->nodes[level] = NULL; + level++; + } + } + return 1; +} + +/* + * drop a subvolume tree. + * + * this function traverses the tree freeing any blocks that only + * referenced by the tree. + * + * when a shared tree block is found. this function decreases its + * reference count by one. if update_ref is true, this function + * also make sure backrefs for the shared block and all lower level + * blocks are properly updated. + * + * If called with for_reloc set, may exit early with -EAGAIN + */ +int btrfs_drop_snapshot(struct btrfs_root *root, bool update_ref, bool for_reloc) +{ + const bool is_reloc_root = (btrfs_root_id(root) == BTRFS_TREE_RELOC_OBJECTID); + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_path *path; + struct btrfs_trans_handle *trans; + struct btrfs_root *tree_root = fs_info->tree_root; + struct btrfs_root_item *root_item = &root->root_item; + struct walk_control AUTO_KFREE(wc); + struct btrfs_key key; + const u64 rootid = btrfs_root_id(root); + int ret = 0; + int level; + bool root_dropped = false; + bool unfinished_drop = false; + + btrfs_debug(fs_info, "Drop subvolume %llu", btrfs_root_id(root)); + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto out; + } + + wc = kzalloc(sizeof(*wc), GFP_NOFS); + if (!wc) { + ret = -ENOMEM; + goto out_free; + } + + /* + * Use join to avoid potential EINTR from transaction start. See + * wait_reserve_ticket and the whole reservation callchain. + */ + if (for_reloc) + trans = btrfs_join_transaction(tree_root); + else + trans = btrfs_start_transaction(tree_root, 0); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto out_free; + } + + ret = btrfs_run_delayed_items(trans); + if (ret) + goto out_end_trans; + + /* + * This will help us catch people modifying the fs tree while we're + * dropping it. It is unsafe to mess with the fs tree while it's being + * dropped as we unlock the root node and parent nodes as we walk down + * the tree, assuming nothing will change. If something does change + * then we'll have stale information and drop references to blocks we've + * already dropped. + */ + set_bit(BTRFS_ROOT_DELETING, &root->state); + unfinished_drop = test_bit(BTRFS_ROOT_UNFINISHED_DROP, &root->state); + + if (btrfs_disk_key_objectid(&root_item->drop_progress) == 0) { + level = btrfs_header_level(root->node); + path->nodes[level] = btrfs_lock_root_node(root); + path->slots[level] = 0; + path->locks[level] = BTRFS_WRITE_LOCK; + memset(&wc->update_progress, 0, + sizeof(wc->update_progress)); + } else { + btrfs_disk_key_to_cpu(&key, &root_item->drop_progress); + memcpy(&wc->update_progress, &key, + sizeof(wc->update_progress)); + + level = btrfs_root_drop_level(root_item); + BUG_ON(level == 0); + path->lowest_level = level; + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + path->lowest_level = 0; + if (ret < 0) + goto out_end_trans; + + WARN_ON(ret > 0); + ret = 0; + + /* + * unlock our path, this is safe because only this + * function is allowed to delete this snapshot + */ + btrfs_unlock_up_safe(path, 0); + + level = btrfs_header_level(root->node); + while (1) { + btrfs_tree_lock(path->nodes[level]); + path->locks[level] = BTRFS_WRITE_LOCK; + + /* + * btrfs_lookup_extent_info() returns 0 for success, + * or < 0 for error. + */ + ret = btrfs_lookup_extent_info(trans, fs_info, + path->nodes[level]->start, + level, 1, &wc->refs[level], + &wc->flags[level], NULL); + if (ret < 0) + goto out_end_trans; + + BUG_ON(wc->refs[level] == 0); + + if (level == btrfs_root_drop_level(root_item)) + break; + + btrfs_tree_unlock(path->nodes[level]); + path->locks[level] = 0; + WARN_ON(wc->refs[level] != 1); + level--; + } + } + + wc->restarted = test_bit(BTRFS_ROOT_DEAD_TREE, &root->state); + wc->level = level; + wc->shared_level = -1; + wc->stage = DROP_REFERENCE; + wc->update_ref = update_ref; + wc->keep_locks = 0; + wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(fs_info); + + while (1) { + + ret = walk_down_tree(trans, root, path, wc); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + break; + } + + ret = walk_up_tree(trans, root, path, wc, BTRFS_MAX_LEVEL); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + break; + } + + if (ret > 0) { + BUG_ON(wc->stage != DROP_REFERENCE); + ret = 0; + break; + } + + if (wc->stage == DROP_REFERENCE) { + wc->drop_level = wc->level; + btrfs_node_key_to_cpu(path->nodes[wc->drop_level], + &wc->drop_progress, + path->slots[wc->drop_level]); + } + btrfs_cpu_key_to_disk(&root_item->drop_progress, + &wc->drop_progress); + btrfs_set_root_drop_level(root_item, wc->drop_level); + + BUG_ON(wc->level == 0); + if (btrfs_should_end_transaction(trans) || + (!for_reloc && btrfs_need_cleaner_sleep(fs_info))) { + ret = btrfs_update_root(trans, tree_root, + &root->root_key, + root_item); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out_end_trans; + } + + if (!is_reloc_root) + btrfs_set_last_root_drop_gen(fs_info, trans->transid); + + btrfs_end_transaction_throttle(trans); + if (!for_reloc && btrfs_need_cleaner_sleep(fs_info)) { + btrfs_debug(fs_info, + "drop snapshot early exit"); + ret = -EAGAIN; + goto out_free; + } + + /* + * Use join to avoid potential EINTR from transaction + * start. See wait_reserve_ticket and the whole + * reservation callchain. + */ + if (for_reloc) + trans = btrfs_join_transaction(tree_root); + else + trans = btrfs_start_transaction(tree_root, 0); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto out_free; + } + } + } + btrfs_release_path(path); + if (ret) + goto out_end_trans; + + ret = btrfs_del_root(trans, &root->root_key); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out_end_trans; + } + + if (!is_reloc_root) { + ret = btrfs_find_root(tree_root, &root->root_key, path, + NULL, NULL); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + goto out_end_trans; + } else if (ret > 0) { + ret = 0; + /* + * If we fail to delete the orphan item this time + * around, it'll get picked up the next time. + * + * The most common failure here is just -ENOENT. + */ + btrfs_del_orphan_item(trans, tree_root, btrfs_root_id(root)); + } + } + + /* + * This subvolume is going to be completely dropped, and won't be + * recorded as dirty roots, thus pertrans meta rsv will not be freed at + * commit transaction time. So free it here manually. + */ + btrfs_qgroup_convert_reserved_meta(root, INT_MAX); + btrfs_qgroup_free_meta_all_pertrans(root); + + if (test_bit(BTRFS_ROOT_IN_RADIX, &root->state)) + btrfs_add_dropped_root(trans, root); + else + btrfs_put_root(root); + root_dropped = true; +out_end_trans: + if (!is_reloc_root) + btrfs_set_last_root_drop_gen(fs_info, trans->transid); + + btrfs_end_transaction_throttle(trans); +out_free: + btrfs_free_path(path); +out: + if (!ret && root_dropped) { + ret = btrfs_qgroup_cleanup_dropped_subvolume(fs_info, rootid); + if (ret < 0) + btrfs_warn_rl(fs_info, + "failed to cleanup qgroup 0/%llu: %pe", + rootid, ERR_PTR(ret)); + ret = 0; + } + /* + * We were an unfinished drop root, check to see if there are any + * pending, and if not clear and wake up any waiters. + */ + if (!ret && unfinished_drop) + btrfs_maybe_wake_unfinished_drop(fs_info); + + /* + * So if we need to stop dropping the snapshot for whatever reason we + * need to make sure to add it back to the dead root list so that we + * keep trying to do the work later. This also cleans up roots if we + * don't have it in the radix (like when we recover after a power fail + * or unmount) so we don't leak memory. + */ + if (!for_reloc && !root_dropped) + btrfs_add_dead_root(root); + return ret; +} + +/* + * drop subtree rooted at tree block 'node'. + * + * NOTE: this function will unlock and release tree block 'node' + * only used by relocation code + */ +int btrfs_drop_subtree(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct extent_buffer *node, + struct extent_buffer *parent) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + BTRFS_PATH_AUTO_FREE(path); + struct walk_control AUTO_KFREE(wc); + int level; + int parent_level; + int ret = 0; + + BUG_ON(btrfs_root_id(root) != BTRFS_TREE_RELOC_OBJECTID); + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + wc = kzalloc(sizeof(*wc), GFP_NOFS); + if (!wc) + return -ENOMEM; + + btrfs_assert_tree_write_locked(parent); + parent_level = btrfs_header_level(parent); + refcount_inc(&parent->refs); + path->nodes[parent_level] = parent; + path->slots[parent_level] = btrfs_header_nritems(parent); + + btrfs_assert_tree_write_locked(node); + level = btrfs_header_level(node); + path->nodes[level] = node; + path->slots[level] = 0; + path->locks[level] = BTRFS_WRITE_LOCK; + + wc->refs[parent_level] = 1; + wc->flags[parent_level] = BTRFS_BLOCK_FLAG_FULL_BACKREF; + wc->level = level; + wc->shared_level = -1; + wc->stage = DROP_REFERENCE; + wc->update_ref = 0; + wc->keep_locks = 1; + wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(fs_info); + + while (1) { + ret = walk_down_tree(trans, root, path, wc); + if (ret < 0) + return ret; + + ret = walk_up_tree(trans, root, path, wc, parent_level); + if (ret) { + if (ret < 0) + return ret; + break; + } + } + + return 0; +} + +/* + * Unpin the extent range in an error context and don't add the space back. + * Errors are not propagated further. + */ +void btrfs_error_unpin_extent_range(struct btrfs_fs_info *fs_info, u64 start, u64 end) +{ + unpin_extent_range(fs_info, start, end, false); +} + +/* + * It used to be that old block groups would be left around forever. + * Iterating over them would be enough to trim unused space. Since we + * now automatically remove them, we also need to iterate over unallocated + * space. + * + * We don't want a transaction for this since the discard may take a + * substantial amount of time. We don't require that a transaction be + * running, but we do need to take a running transaction into account + * to ensure that we're not discarding chunks that were released or + * allocated in the current transaction. + * + * Holding the chunks lock will prevent other threads from allocating + * or releasing chunks, but it won't prevent a running transaction + * from committing and releasing the memory that the pending chunks + * list head uses. For that, we need to take a reference to the + * transaction and hold the commit root sem. We only need to hold + * it while performing the free space search since we have already + * held back allocations. + */ +static int btrfs_trim_free_extents_throttle(struct btrfs_device *device, + u64 *trimmed, u64 pos, u64 *ret_next_pos) +{ + int ret; + u64 start = pos; + u64 trim_len = 0; + + *trimmed = 0; + + /* + * The caller only filters out MISSING devices, but a device that was + * missing at mount and later rescanned has MISSING cleared while bdev + * is still NULL and WRITEABLE is still unset. Skip those here. + */ + if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) || !device->bdev) + return 0; + + /* Discard not supported = nothing to do. */ + if (!bdev_max_discard_sectors(device->bdev)) + return 0; + + /* No free space = nothing to do. */ + if (device->total_bytes <= device->bytes_used) + return 0; + + ret = 0; + + while (1) { + struct btrfs_fs_info *fs_info = device->fs_info; + u64 cur_start; + u64 end; + u64 len; + u64 bytes; + + ret = mutex_lock_interruptible(&fs_info->chunk_mutex); + if (ret) + break; + + cur_start = start; + btrfs_find_first_clear_extent_bit(&device->alloc_state, start, + &start, &end, + CHUNK_TRIMMED | CHUNK_ALLOCATED); + start = max(start, cur_start); + + /* Check if there are any CHUNK_* bits left */ + if (unlikely(start > device->total_bytes)) { + DEBUG_WARN(); + btrfs_warn(fs_info, +"ignoring attempt to trim beyond device size: offset %llu length %llu device %s device size %llu", + start, end - start + 1, + btrfs_dev_name(device), + device->total_bytes); + mutex_unlock(&fs_info->chunk_mutex); + ret = 0; + break; + } + + /* Ensure we skip the reserved space on each device. */ + start = max_t(u64, start, BTRFS_DEVICE_RANGE_RESERVED); + + /* + * If find_first_clear_extent_bit find a range that spans the + * end of the device it will set end to -1, in this case it's up + * to the caller to trim the value to the size of the device. + */ + end = min(end, device->total_bytes - 1); + + len = end - start + 1; + len = min(len, BTRFS_MAX_TRIM_LENGTH); + + /* We didn't find any extents */ + if (!len) { + mutex_unlock(&fs_info->chunk_mutex); + ret = 0; + break; + } + + ret = btrfs_issue_discard(device->bdev, start, len, + &bytes); + if (!ret) + btrfs_set_extent_bit(&device->alloc_state, start, + start + bytes - 1, CHUNK_TRIMMED, NULL); + mutex_unlock(&fs_info->chunk_mutex); + + if (ret) + break; + + start += len; + *trimmed += bytes; + trim_len += len; + if (trim_len >= BTRFS_MAX_TRIM_LENGTH) { + *ret_next_pos = start; + ret = -EAGAIN; + break; + } + + if (btrfs_trim_interrupted()) { + ret = -ERESTARTSYS; + break; + } + + cond_resched(); + } + + return ret; +} + +static int btrfs_trim_free_extents(struct btrfs_fs_info *fs_info, u64 *trimmed, + u64 *dev_failed, int *dev_ret) +{ + struct btrfs_device *dev; + struct btrfs_device *working_dev = NULL; + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; + u8 uuid[BTRFS_UUID_SIZE]; + u64 start = BTRFS_DEVICE_RANGE_RESERVED; + + *trimmed = 0; + *dev_failed = 0; + *dev_ret = 0; + + /* Find the device with the smallest UUID to start. */ + mutex_lock(&fs_devices->device_list_mutex); + list_for_each_entry(dev, &fs_devices->devices, dev_list) { + if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state)) + continue; + if (!working_dev || + memcmp(dev->uuid, working_dev->uuid, BTRFS_UUID_SIZE) < 0) + working_dev = dev; + } + if (working_dev) + memcpy(uuid, working_dev->uuid, BTRFS_UUID_SIZE); + mutex_unlock(&fs_devices->device_list_mutex); + + if (!working_dev) + return 0; + + while (1) { + u64 group_trimmed = 0; + u64 next_pos = 0; + int ret = 0; + + mutex_lock(&fs_devices->device_list_mutex); + + /* Find and trim the current device. */ + list_for_each_entry(dev, &fs_devices->devices, dev_list) { + if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state)) + continue; + if (dev == working_dev) { + ret = btrfs_trim_free_extents_throttle(working_dev, + &group_trimmed, start, &next_pos); + break; + } + } + + /* Throttle: continue the same device from the new position. */ + if (ret == -EAGAIN && next_pos > start) { + mutex_unlock(&fs_devices->device_list_mutex); + *trimmed += group_trimmed; + start = next_pos; + cond_resched(); + continue; + } + + /* User interrupted. */ + if (ret == -ERESTARTSYS || ret == -EINTR) { + mutex_unlock(&fs_devices->device_list_mutex); + *trimmed += group_trimmed; + return ret; + } + + /* + * Device completed (ret == 0), failed, or EAGAIN with no progress. + * Record error if any, then move to next device. + */ + if (ret == -EAGAIN) { + /* No progress - log and skip device. */ + btrfs_warn(fs_info, + "trim throttle: no progress, offset=%llu device %s, skipping", + start, btrfs_dev_name(working_dev)); + (*dev_failed)++; + if (!*dev_ret) + *dev_ret = ret; + } else if (ret) { + /* Device failed with error. */ + (*dev_failed)++; + if (!*dev_ret) + *dev_ret = ret; + } + + /* + * Find next device: smallest UUID larger than current. + * Devices added during trim with smaller UUID will be skipped. + */ + working_dev = NULL; + list_for_each_entry(dev, &fs_devices->devices, dev_list) { + if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state)) + continue; + /* Must larger than current UUID. */ + if (memcmp(dev->uuid, uuid, BTRFS_UUID_SIZE) <= 0) + continue; + /* Find the smallest. */ + if (!working_dev || + memcmp(dev->uuid, working_dev->uuid, BTRFS_UUID_SIZE) < 0) + working_dev = dev; + } + if (working_dev) + memcpy(uuid, working_dev->uuid, BTRFS_UUID_SIZE); + + mutex_unlock(&fs_devices->device_list_mutex); + + *trimmed += group_trimmed; + start = BTRFS_DEVICE_RANGE_RESERVED; + + /* No more devices. */ + if (!working_dev) + break; + + cond_resched(); + } + + return 0; +} + +/* + * Trim the whole filesystem by: + * 1) trimming the free space in each block group + * 2) trimming the unallocated space on each device + * + * This will also continue trimming even if a block group or device encounters + * an error. The return value will be the first error, or 0 if nothing bad + * happens. + */ +int btrfs_trim_fs(struct btrfs_fs_info *fs_info, struct fstrim_range *range) +{ + struct btrfs_block_group *cache = NULL; + u64 group_trimmed; + u64 range_end = U64_MAX; + u64 start; + u64 end; + u64 trimmed = 0; + u64 bg_failed = 0; + u64 dev_failed = 0; + int bg_ret = 0; + int dev_ret = 0; + int ret = 0; + + if (range->start == U64_MAX) + return -EINVAL; + + /* + * Check range overflow if range->len is set. + * The default range->len is U64_MAX. + */ + if (range->len != U64_MAX && + check_add_overflow(range->start, range->len, &range_end)) + return -EINVAL; + + cache = btrfs_lookup_first_block_group(fs_info, range->start); + for (; cache; cache = btrfs_next_block_group(cache)) { + if (cache->start >= range_end) { + btrfs_put_block_group(cache); + break; + } + + start = max(range->start, cache->start); + end = min(range_end, btrfs_block_group_end(cache)); + + if (end - start >= range->minlen) { + if (!btrfs_block_group_done(cache)) { + ret = btrfs_cache_block_group(cache, true); + if (ret) { + bg_failed++; + if (!bg_ret) + bg_ret = ret; + continue; + } + } + ret = btrfs_trim_block_group(cache, + &group_trimmed, + start, + end, + range->minlen); + + trimmed += group_trimmed; + if (ret == -ERESTARTSYS || ret == -EINTR) { + btrfs_put_block_group(cache); + break; + } + if (ret) { + bg_failed++; + if (!bg_ret) + bg_ret = ret; + continue; + } + } + } + + if (bg_failed) + btrfs_warn(fs_info, + "failed to trim %llu block group(s), first error %pe", + bg_failed, ERR_PTR(bg_ret)); + + if (ret == -ERESTARTSYS || ret == -EINTR) + return ret; + + ret = btrfs_trim_free_extents(fs_info, &group_trimmed, &dev_failed, &dev_ret); + trimmed += group_trimmed; + + if (dev_failed) + btrfs_warn(fs_info, + "failed to trim %llu device(s), first error %pe", + dev_failed, ERR_PTR(dev_ret)); + range->len = trimmed; + if (ret == -ERESTARTSYS || ret == -EINTR) + return ret; + if (bg_ret) + return bg_ret; + return dev_ret; +} diff --git a/fs/btrfs/extent-tree.h b/fs/btrfs/extent-tree.h new file mode 100644 index 000000000..ff330d489 --- /dev/null +++ b/fs/btrfs/extent-tree.h @@ -0,0 +1,169 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_EXTENT_TREE_H +#define BTRFS_EXTENT_TREE_H + +#include +#include "block-group.h" +#include "locking.h" + +struct extent_buffer; +struct btrfs_free_cluster; +struct btrfs_fs_info; +struct btrfs_root; +struct btrfs_path; +struct btrfs_ref; +struct btrfs_disk_key; +struct btrfs_delayed_ref_head; +struct btrfs_delayed_ref_root; +struct btrfs_extent_inline_ref; + +enum btrfs_extent_allocation_policy { + BTRFS_EXTENT_ALLOC_CLUSTERED, + BTRFS_EXTENT_ALLOC_ZONED, +}; + +struct find_free_extent_ctl { + /* Basic allocation info */ + u64 ram_bytes; + u64 num_bytes; + u64 min_alloc_size; + u64 empty_size; + u64 flags; + + /* Where to start the search inside the bg */ + u64 search_start; + + /* For clustered allocation */ + u64 empty_cluster; + struct btrfs_free_cluster *last_ptr; + bool use_cluster; + + bool delalloc; + bool have_caching_bg; + bool orig_have_caching_bg; + + /* Allocation is called for tree-log */ + bool for_treelog; + + /* Allocation is called for data relocation */ + bool for_data_reloc; + + /* + * Set to true if we're retrying the allocation on this block group + * after waiting for caching progress, this is so that we retry only + * once before moving on to another block group. + */ + bool retry_uncached; + + /* Whether or not the allocator is currently following a hint. */ + bool hinted; + + /* RAID index, converted from flags */ + int index; + + /* + * Current loop number, check find_free_extent_update_loop() for details + */ + int loop; + + /* If current block group is cached */ + int cached; + + /* Max contiguous hole found */ + u64 max_extent_size; + + /* Total free space from free space cache, not always contiguous */ + u64 total_free_space; + + /* Found result */ + u64 found_offset; + + /* Hint where to start looking for an empty space */ + u64 hint_byte; + + /* Allocation policy */ + enum btrfs_extent_allocation_policy policy; + + /* Size class of block groups to prefer in early loops */ + enum btrfs_block_group_size_class size_class; +}; + +enum btrfs_inline_ref_type { + BTRFS_REF_TYPE_INVALID, + BTRFS_REF_TYPE_BLOCK, + BTRFS_REF_TYPE_DATA, + BTRFS_REF_TYPE_ANY, +}; + +int btrfs_get_extent_inline_ref_type(const struct extent_buffer *eb, + const struct btrfs_extent_inline_ref *iref, + enum btrfs_inline_ref_type is_data); +u64 hash_extent_data_ref(u64 root_objectid, u64 owner, u64 offset); + +int btrfs_run_delayed_refs(struct btrfs_trans_handle *trans, u64 min_bytes); +u64 btrfs_cleanup_ref_head_accounting(struct btrfs_fs_info *fs_info, + struct btrfs_delayed_ref_root *delayed_refs, + struct btrfs_delayed_ref_head *head); +int btrfs_lookup_data_extent(struct btrfs_fs_info *fs_info, u64 start, u64 len); +int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans, + struct btrfs_fs_info *fs_info, u64 bytenr, + u64 offset, int metadata, u64 *refs, u64 *flags, + u64 *owner_root); +int btrfs_pin_extent(struct btrfs_trans_handle *trans, u64 bytenr, u64 num); +int btrfs_pin_extent_for_log_replay(struct btrfs_trans_handle *trans, + const struct extent_buffer *eb); +int btrfs_exclude_logged_extents(struct extent_buffer *eb); +int btrfs_cross_ref_exist(struct btrfs_inode *inode, u64 offset, u64 bytenr, + struct btrfs_path *path); +struct extent_buffer *btrfs_alloc_tree_block(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + u64 parent, u64 root_objectid, + const struct btrfs_disk_key *key, + int level, u64 hint, + u64 empty_size, + u64 reloc_src_root, + enum btrfs_lock_nesting nest); +int btrfs_free_tree_block(struct btrfs_trans_handle *trans, + u64 root_id, + struct extent_buffer *buf, + u64 parent, int last_ref); +int btrfs_alloc_reserved_file_extent(struct btrfs_trans_handle *trans, + struct btrfs_root *root, u64 owner, + u64 offset, u64 ram_bytes, + struct btrfs_key *ins); +int btrfs_alloc_logged_file_extent(struct btrfs_trans_handle *trans, + u64 root_objectid, u64 owner, u64 offset, + struct btrfs_key *ins); +int btrfs_reserve_extent(struct btrfs_root *root, u64 ram_bytes, u64 num_bytes, + u64 min_alloc_size, u64 empty_size, u64 hint_byte, + struct btrfs_key *ins, bool is_data, bool delalloc); +int btrfs_inc_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root, + struct extent_buffer *buf, bool full_backref); +int btrfs_dec_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root, + struct extent_buffer *buf, bool full_backref); +int btrfs_set_disk_extent_flags(struct btrfs_trans_handle *trans, + struct extent_buffer *eb, u64 flags); +int btrfs_free_extent(struct btrfs_trans_handle *trans, struct btrfs_ref *ref); + +u64 btrfs_get_extent_owner_root(struct btrfs_fs_info *fs_info, + struct extent_buffer *leaf, int slot); +int btrfs_free_reserved_extent(struct btrfs_fs_info *fs_info, u64 start, u64 len, + bool is_delalloc); +int btrfs_pin_reserved_extent(struct btrfs_trans_handle *trans, + const struct extent_buffer *eb); +int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans); +int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans, struct btrfs_ref *generic_ref); +int btrfs_drop_snapshot(struct btrfs_root *root, bool update_ref, bool for_reloc); +int btrfs_drop_subtree(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct extent_buffer *node, + struct extent_buffer *parent); +void btrfs_error_unpin_extent_range(struct btrfs_fs_info *fs_info, u64 start, u64 end); +int btrfs_discard_extent(struct btrfs_fs_info *fs_info, u64 bytenr, + u64 num_bytes, u64 *actual_bytes, bool do_remap); +int btrfs_trim_fs(struct btrfs_fs_info *fs_info, struct fstrim_range *range); +void btrfs_handle_fully_remapped_bgs(struct btrfs_fs_info *fs_info); +int btrfs_complete_bg_remapping(struct btrfs_block_group *bg); + +#endif diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c new file mode 100644 index 000000000..d7600e5fa --- /dev/null +++ b/fs/btrfs/extent_io.c @@ -0,0 +1,5124 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "extent_io.h" +#include "extent-io-tree.h" +#include "extent_map.h" +#include "ctree.h" +#include "btrfs_inode.h" +#include "bio.h" +#include "locking.h" +#include "backref.h" +#include "disk-io.h" +#include "subpage.h" +#include "zoned.h" +#include "block-group.h" +#include "compression.h" +#include "fs.h" +#include "accessors.h" +#include "file-item.h" +#include "file.h" +#include "dev-replace.h" +#include "super.h" +#include "transaction.h" + +static struct kmem_cache *extent_buffer_cache; + +#ifdef CONFIG_BTRFS_DEBUG +static inline void btrfs_leak_debug_add_eb(struct extent_buffer *eb) +{ + struct btrfs_fs_info *fs_info = eb->fs_info; + unsigned long flags; + + spin_lock_irqsave(&fs_info->eb_leak_lock, flags); + list_add(&eb->leak_list, &fs_info->allocated_ebs); + spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags); +} + +static inline void btrfs_leak_debug_del_eb(struct extent_buffer *eb) +{ + struct btrfs_fs_info *fs_info = eb->fs_info; + unsigned long flags; + + spin_lock_irqsave(&fs_info->eb_leak_lock, flags); + list_del(&eb->leak_list); + spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags); +} + +void btrfs_extent_buffer_leak_debug_check(struct btrfs_fs_info *fs_info) +{ + struct extent_buffer *eb; + unsigned long flags; + + /* + * If we didn't get into open_ctree our allocated_ebs will not be + * initialized, so just skip this. + */ + if (!fs_info->allocated_ebs.next) + return; + + WARN_ON(!list_empty(&fs_info->allocated_ebs)); + spin_lock_irqsave(&fs_info->eb_leak_lock, flags); + while (!list_empty(&fs_info->allocated_ebs)) { + eb = list_first_entry(&fs_info->allocated_ebs, + struct extent_buffer, leak_list); + btrfs_err(fs_info, + "buffer leak start %llu len %u refs %d bflags %lu owner %llu", + eb->start, eb->len, refcount_read(&eb->refs), eb->bflags, + btrfs_header_owner(eb)); + list_del(&eb->leak_list); + WARN_ON_ONCE(1); + kmem_cache_free(extent_buffer_cache, eb); + } + spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags); +} +#else +#define btrfs_leak_debug_add_eb(eb) do {} while (0) +#define btrfs_leak_debug_del_eb(eb) do {} while (0) +#endif + +/* + * Structure to record info about the bio being assembled, and other info like + * how many bytes are there before stripe/ordered extent boundary. + */ +struct btrfs_bio_ctrl { + struct btrfs_bio *bbio; + /* Last byte contained in bbio + 1 . */ + loff_t next_file_offset; + enum btrfs_compression_type compress_type; + u32 len_to_oe_boundary; + blk_opf_t opf; + /* + * For data read bios, we attempt to optimize csum lookups if the extent + * generation is older than the current one. To make this possible, we + * need to track the maximum generation of an extent in a bio_ctrl to + * make the decision when submitting the bio. + * + * The pattern between do_readpage(), submit_one_bio() and + * submit_extent_folio() is quite subtle, so tracking this is tricky. + * + * As we process extent E, we might submit a bio with existing built up + * extents before adding E to a new bio, or we might just add E to the + * bio. As a result, E's generation could apply to the current bio or + * to the next one, so we need to be careful to update the bio_ctrl's + * generation with E's only when we are sure E is added to bio_ctrl->bbio + * in submit_extent_folio(). + * + * See the comment in btrfs_lookup_bio_sums() for more detail on the + * need for this optimization. + */ + u64 generation; + btrfs_bio_end_io_t end_io_func; + struct writeback_control *wbc; + + /* + * The sectors of the page which are going to be submitted by + * extent_writepage_io(). + * This is to avoid touching ranges covered by compression/inline. + */ + unsigned long submit_bitmap[BITS_TO_LONGS(BTRFS_MAX_BLOCKS_PER_FOLIO)]; + + struct readahead_control *ractl; + + /* + * The start offset of the last used extent map by a read operation. + * + * This is for proper compressed read merge. + * U64_MAX means we are starting the read and have made no progress yet. + * + * The current btrfs_bio_is_contig() only uses disk_bytenr as + * the condition to check if the read can be merged with previous + * bio, which is not correct. E.g. two file extents pointing to the + * same extent but with different offset. + * + * So here we need to do extra checks to only merge reads that are + * covered by the same extent map. + * Just extent_map::start will be enough, as they are unique + * inside the same inode. + */ + u64 last_em_start; +}; + +/* + * Helper to set the csum search commit root option for a bio_ctrl's bbio + * before submitting the bio. + * + * Only for use by submit_one_bio(). + */ +static void bio_set_csum_search_commit_root(struct btrfs_bio_ctrl *bio_ctrl) +{ + struct btrfs_bio *bbio = bio_ctrl->bbio; + + ASSERT(bbio); + + if (!(btrfs_op(&bbio->bio) == BTRFS_MAP_READ && is_data_inode(bbio->inode))) + return; + + bio_ctrl->bbio->csum_search_commit_root = + (bio_ctrl->generation && + bio_ctrl->generation < btrfs_get_fs_generation(bbio->inode->root->fs_info)); +} + +static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl) +{ + struct btrfs_bio *bbio = bio_ctrl->bbio; + + if (!bbio) + return; + + /* Caller should ensure the bio has at least some range added */ + ASSERT(bbio->bio.bi_iter.bi_size); + + bio_set_csum_search_commit_root(bio_ctrl); + + if (btrfs_op(&bbio->bio) == BTRFS_MAP_READ && + bio_ctrl->compress_type != BTRFS_COMPRESS_NONE) + btrfs_submit_compressed_read(bbio); + else + btrfs_submit_bbio(bbio, 0); + + /* The bbio is owned by the end_io handler now */ + bio_ctrl->bbio = NULL; + /* + * We used the generation to decide whether to lookup csums in the + * commit_root or not when we called bio_set_csum_search_commit_root() + * above. Now, reset the generation for the next bio. + */ + bio_ctrl->generation = 0; +} + +/* + * Submit or fail the current bio in the bio_ctrl structure. + */ +static void submit_write_bio(struct btrfs_bio_ctrl *bio_ctrl, int ret) +{ + struct btrfs_bio *bbio = bio_ctrl->bbio; + + if (!bbio) + return; + + if (ret) { + ASSERT(ret < 0); + btrfs_bio_end_io(bbio, errno_to_blk_status(ret)); + /* The bio is owned by the end_io handler now */ + bio_ctrl->bbio = NULL; + } else { + submit_one_bio(bio_ctrl); + } +} + +int __init extent_buffer_init_cachep(void) +{ + extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer", + sizeof(struct extent_buffer), 0, 0, + NULL); + if (!extent_buffer_cache) + return -ENOMEM; + + return 0; +} + +void __cold extent_buffer_free_cachep(void) +{ + /* + * Make sure all delayed rcu free are flushed before we + * destroy caches. + */ + rcu_barrier(); + kmem_cache_destroy(extent_buffer_cache); +} + +static void process_one_folio(struct btrfs_fs_info *fs_info, + struct folio *folio, const struct folio *locked_folio, + unsigned long page_ops, u64 start, u64 end) +{ + u32 len; + + ASSERT(end + 1 - start != 0 && end + 1 - start < U32_MAX); + len = end + 1 - start; + + if (page_ops & PAGE_START_WRITEBACK) { + btrfs_folio_clamp_clear_dirty(fs_info, folio, start, len); + btrfs_folio_clamp_set_writeback(fs_info, folio, start, len); + } + if (page_ops & PAGE_END_WRITEBACK) + btrfs_folio_clamp_clear_writeback(fs_info, folio, start, len); + + if (folio != locked_folio && (page_ops & PAGE_UNLOCK)) + btrfs_folio_end_lock(fs_info, folio, start, len); +} + +static void __process_folios_contig(struct address_space *mapping, + const struct folio *locked_folio, u64 start, + u64 end, unsigned long page_ops) +{ + struct btrfs_fs_info *fs_info = inode_to_fs_info(mapping->host); + pgoff_t index = start >> PAGE_SHIFT; + pgoff_t end_index = end >> PAGE_SHIFT; + struct folio_batch fbatch; + int i; + + folio_batch_init(&fbatch); + while (index <= end_index) { + int found_folios; + + found_folios = filemap_get_folios_contig(mapping, &index, + end_index, &fbatch); + for (i = 0; i < found_folios; i++) { + struct folio *folio = fbatch.folios[i]; + + process_one_folio(fs_info, folio, locked_folio, + page_ops, start, end); + } + folio_batch_release(&fbatch); + cond_resched(); + } +} + +static noinline void unlock_delalloc_folio(const struct inode *inode, + struct folio *locked_folio, + u64 start, u64 end) +{ + ASSERT(locked_folio); + + __process_folios_contig(inode->i_mapping, locked_folio, start, end, + PAGE_UNLOCK); +} + +#ifdef CONFIG_BTRFS_DEBUG +/* + * Writeback must write-protect a folio when locking it for IO, before + * anything consumes its data (zeroing, inline copy, compression, + * checksumming). If this fails, then an mmap writer would be able to + * modify the data concurrently while we need it to be stable. + */ +void btrfs_check_folio_write_protected(struct folio *folio) +{ + if (folio_mkclean(folio)) { + const struct btrfs_inode *inode = BTRFS_I(folio->mapping->host); + + DEBUG_WARN("writable mmap PTEs, root %llu ino %llu pos %llu order %u", + btrfs_root_id(inode->root), btrfs_ino(inode), folio_pos(folio), + folio_order(folio)); + } +} +#endif + +static noinline int lock_delalloc_folios(struct inode *inode, + struct folio *locked_folio, + u64 start, u64 end) +{ + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + struct address_space *mapping = inode->i_mapping; + pgoff_t index = start >> PAGE_SHIFT; + pgoff_t end_index = end >> PAGE_SHIFT; + u64 processed_end = start; + struct folio_batch fbatch; + + folio_batch_init(&fbatch); + while (index <= end_index) { + unsigned int found_folios, i; + + found_folios = filemap_get_folios_contig(mapping, &index, + end_index, &fbatch); + if (found_folios == 0) + goto out; + + for (i = 0; i < found_folios; i++) { + struct folio *folio = fbatch.folios[i]; + u64 range_start; + u32 range_len; + + if (folio == locked_folio) + continue; + + folio_lock(folio); + if (!folio_test_dirty(folio) || folio->mapping != mapping) { + folio_unlock(folio); + goto out; + } + /* Locked for writeback; revoke writable mmap PTEs before using the data. */ + folio_mkclean(folio); + range_start = max_t(u64, folio_pos(folio), start); + range_len = min_t(u64, folio_next_pos(folio), end + 1) - range_start; + btrfs_folio_set_lock(fs_info, folio, range_start, range_len); + + processed_end = range_start + range_len - 1; + } + folio_batch_release(&fbatch); + cond_resched(); + } + + return 0; +out: + folio_batch_release(&fbatch); + if (processed_end > start) + unlock_delalloc_folio(inode, locked_folio, start, processed_end); + return -EAGAIN; +} + +/* + * Find and lock a contiguous range of bytes in the file marked as delalloc, no + * more than @max_bytes. + * + * @start: The original start bytenr to search. + * Will store the extent range start bytenr. + * @end: The original end bytenr of the search range + * Will store the extent range end bytenr. + * + * Return true if we find a delalloc range which starts inside the original + * range, and @start/@end will store the delalloc range start/end. + * + * Return false if we can't find any delalloc range which starts inside the + * original range, and @start/@end will be the non-delalloc range start/end. + */ +EXPORT_FOR_TESTS +noinline_for_stack bool find_lock_delalloc_range(struct inode *inode, + struct folio *locked_folio, + u64 *start, u64 *end) +{ + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree; + const u64 orig_start = *start; + const u64 orig_end = *end; + u64 max_bytes = fs_info->max_extent_size; + u64 delalloc_start; + u64 delalloc_end; + bool found; + struct extent_state *cached_state = NULL; + int ret; + bool loops = false; + + /* Caller should pass a valid @end to indicate the search range end */ + ASSERT(orig_end > orig_start); + + /* The range should at least cover part of the folio */ + ASSERT(!(orig_start >= folio_next_pos(locked_folio) || + orig_end <= folio_pos(locked_folio))); +again: + /* step one, find a bunch of delalloc bytes starting at start */ + delalloc_start = *start; + delalloc_end = 0; + + /* + * If @max_bytes is smaller than a block, btrfs_find_delalloc_range() can + * return early without handling any dirty ranges. + */ + ASSERT(max_bytes >= fs_info->sectorsize); + + found = btrfs_find_delalloc_range(tree, &delalloc_start, &delalloc_end, + max_bytes, &cached_state); + if (!found || delalloc_end <= *start || delalloc_start > orig_end) { + *start = delalloc_start; + + /* @delalloc_end can be -1, never go beyond @orig_end */ + *end = min(delalloc_end, orig_end); + btrfs_free_extent_state(cached_state); + return false; + } + + /* + * start comes from the offset of locked_folio. We have to lock + * folios in order, so we can't process delalloc bytes before + * locked_folio + */ + if (delalloc_start < *start) + delalloc_start = *start; + + /* + * make sure to limit the number of folios we try to lock down + */ + if (delalloc_end + 1 - delalloc_start > max_bytes) + delalloc_end = delalloc_start + max_bytes - 1; + + /* step two, lock all the folios after the folios that has start */ + ret = lock_delalloc_folios(inode, locked_folio, delalloc_start, + delalloc_end); + ASSERT(!ret || ret == -EAGAIN); + if (ret == -EAGAIN) { + /* + * Some of the folios are gone, lets avoid looping by + * shortening the size of the delalloc range we're searching. + */ + btrfs_free_extent_state(cached_state); + cached_state = NULL; + if (!loops) { + max_bytes = fs_info->sectorsize; + loops = true; + goto again; + } else { + return false; + } + } + + /* step three, lock the state bits for the whole range */ + btrfs_lock_extent(tree, delalloc_start, delalloc_end, &cached_state); + + /* then test to make sure it is all still delalloc */ + ret = btrfs_test_range_bit(tree, delalloc_start, delalloc_end, + EXTENT_DELALLOC, cached_state); + + btrfs_unlock_extent(tree, delalloc_start, delalloc_end, &cached_state); + if (!ret) { + unlock_delalloc_folio(inode, locked_folio, delalloc_start, + delalloc_end); + cond_resched(); + goto again; + } + *start = delalloc_start; + *end = delalloc_end; + + return found; +} + +void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end, + const struct folio *locked_folio, + struct extent_state **cached, + u32 clear_bits, unsigned long page_ops) +{ + btrfs_clear_extent_bit(&inode->io_tree, start, end, clear_bits, cached); + + __process_folios_contig(inode->vfs_inode.i_mapping, locked_folio, start, + end, page_ops); +} + +static bool btrfs_verify_folio(struct fsverity_info *vi, struct folio *folio, + u64 start, u32 len) +{ + struct btrfs_fs_info *fs_info = folio_to_fs_info(folio); + + if (!vi || btrfs_folio_test_uptodate(fs_info, folio, start, len)) + return true; + return fsverity_verify_folio(vi, folio); +} + +static void end_folio_read(struct fsverity_info *vi, struct folio *folio, + bool uptodate, u64 start, u32 len) +{ + struct btrfs_fs_info *fs_info = folio_to_fs_info(folio); + + ASSERT(folio_pos(folio) <= start && + start + len <= folio_next_pos(folio)); + + if (uptodate && btrfs_verify_folio(vi, folio, start, len)) + btrfs_folio_set_uptodate(fs_info, folio, start, len); + else + btrfs_folio_clear_uptodate(fs_info, folio, start, len); + + if (!btrfs_is_subpage(fs_info, folio)) + folio_unlock(folio); + else + btrfs_folio_end_lock(fs_info, folio, start, len); +} + +/* + * After a write IO is done, we need to: + * + * - clear the uptodate bits on error + * - clear the writeback bits in the extent tree for the range + * - filio_end_writeback() if there is no more pending io for the folio + * + * Scheduling is not allowed, so the extent state tree is expected + * to have one and only one object corresponding to this IO. + */ +static void end_bbio_data_write(struct btrfs_bio *bbio) +{ + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; + struct bio *bio = &bbio->bio; + int error = blk_status_to_errno(bio->bi_status); + struct folio_iter fi; + u32 bio_size = 0; + + ASSERT(!bio_flagged(bio, BIO_CLONED)); + bio_for_each_folio_all(fi, bio) { + struct folio *folio = fi.folio; + u64 start = folio_pos(folio) + fi.offset; + u32 len = fi.length; + + bio_size += len; + btrfs_folio_clear_writeback(fs_info, folio, start, len); + } + + if (error) + mapping_set_error(bbio->inode->vfs_inode.i_mapping, error); + + btrfs_finish_ordered_extent(bbio->ordered, bbio->file_offset, bio_size, !error); + bio_put(bio); +} + +static void begin_folio_read(struct btrfs_fs_info *fs_info, struct folio *folio) +{ + ASSERT(folio_test_locked(folio)); + if (!btrfs_is_subpage(fs_info, folio)) + return; + + ASSERT(folio_test_private(folio)); + btrfs_folio_set_lock(fs_info, folio, folio_pos(folio), folio_size(folio)); +} + +/* + * After a data read IO is done, we need to: + * + * - clear the uptodate bits on error + * - set the uptodate bits if things worked + * - set the folio up to date if all extents in the tree are uptodate + * - clear the lock bit in the extent tree + * - unlock the folio if there are no other extents locked for it + * + * Scheduling is not allowed, so the extent state tree is expected + * to have one and only one object corresponding to this IO. + */ +static void end_bbio_data_read(struct btrfs_bio *bbio) +{ + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; + struct inode *inode = &bbio->inode->vfs_inode; + struct bio *bio = &bbio->bio; + struct fsverity_info *vi = NULL; + struct folio_iter fi; + + ASSERT(!bio_flagged(bio, BIO_CLONED)); + + if (bbio->file_offset < i_size_read(inode)) + vi = fsverity_get_info(inode); + + bio_for_each_folio_all(fi, &bbio->bio) { + bool uptodate = !bio->bi_status; + struct folio *folio = fi.folio; + u64 start = folio_pos(folio) + fi.offset; + + btrfs_debug(fs_info, + "%s: bi_sector=%llu, err=%d, mirror=%u", + __func__, bio->bi_iter.bi_sector, bio->bi_status, + bbio->mirror_num); + + + if (likely(uptodate)) { + u64 end = start + fi.length - 1; + loff_t i_size = i_size_read(inode); + + /* + * Zero out the remaining part if this range straddles + * i_size. + * + * Here we should only zero the range inside the folio, + * not touch anything else. + * + * NOTE: i_size is exclusive while end is inclusive and + * folio_contains() takes PAGE_SIZE units. + */ + if (folio_contains(folio, i_size >> PAGE_SHIFT) && + i_size <= end) { + u32 zero_start = max(offset_in_folio(folio, i_size), + offset_in_folio(folio, start)); + u32 zero_len = offset_in_folio(folio, end) + 1 - + zero_start; + + folio_zero_range(folio, zero_start, zero_len); + } + } + + /* Update page status and unlock. */ + end_folio_read(vi, folio, uptodate, start, fi.length); + } + bio_put(bio); +} + +/* + * Populate every free slot in a provided array with folios. + * + * @nr_folios: number of folios to allocate + * @order: folio order + * @folio_array: array to fill with folios; non-NULL entries are skipped + * @gfp: GFP flags for the allocation + * + * Return: 0 if all folios were able to be allocated; + * -ENOMEM otherwise, the partially allocated folios would be freed and + * the array slots zeroed + */ +int btrfs_alloc_folio_array(unsigned int nr_folios, unsigned int order, + struct folio **folio_array, gfp_t gfp) +{ + for (int i = 0; i < nr_folios; i++) { + if (folio_array[i]) + continue; + folio_array[i] = folio_alloc(gfp, order); + if (!folio_array[i]) + goto error; + } + return 0; +error: + for (int i = 0; i < nr_folios; i++) { + if (folio_array[i]) + folio_put(folio_array[i]); + folio_array[i] = NULL; + } + return -ENOMEM; +} + +/* + * Populate every free slot in a provided array with pages. + * + * @nr_pages: number of pages to allocate + * @page_array: array to fill; non-NULL entries are skipped + * @gfp: GFP flags for the allocation + * + * Return: 0 if all pages were able to be allocated; + * -ENOMEM otherwise, the partially allocated pages would be freed and + * the array slots zeroed + */ +int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array, gfp_t gfp) +{ + unsigned int allocated; + + for (allocated = 0; allocated < nr_pages;) { + unsigned int last = allocated; + + allocated = alloc_pages_bulk(gfp, nr_pages, page_array); + if (unlikely(allocated == last)) { + /* No progress, fail and do cleanup. */ + for (int i = 0; i < allocated; i++) { + __free_page(page_array[i]); + page_array[i] = NULL; + } + return -ENOMEM; + } + } + return 0; +} + +/* + * Populate needed folios for the extent buffer. + * + * For now, the folios populated are always in order 0 (aka, single page). + */ +static int alloc_eb_folio_array(struct extent_buffer *eb, gfp_t gfp) +{ + struct page *page_array[INLINE_EXTENT_BUFFER_PAGES] = { 0 }; + int num_pages = num_extent_pages(eb); + int ret; + + ret = btrfs_alloc_page_array(num_pages, page_array, gfp); + if (ret < 0) + return ret; + + for (int i = 0; i < num_pages; i++) + eb->folios[i] = page_folio(page_array[i]); + eb->folio_size = PAGE_SIZE; + eb->folio_shift = PAGE_SHIFT; + return 0; +} + +static bool btrfs_bio_is_contig(struct btrfs_bio_ctrl *bio_ctrl, + u64 disk_bytenr, loff_t file_offset) +{ + struct bio *bio = &bio_ctrl->bbio->bio; + const sector_t sector = disk_bytenr >> SECTOR_SHIFT; + + if (bio_ctrl->compress_type != BTRFS_COMPRESS_NONE) { + /* + * For compression, all IO should have its logical bytenr set + * to the starting bytenr of the compressed extent. + */ + return bio->bi_iter.bi_sector == sector; + } + + /* + * To merge into a bio both the disk sector and the logical offset in + * the file need to be contiguous. + */ + return bio_ctrl->next_file_offset == file_offset && + bio_end_sector(bio) == sector; +} + +static int alloc_new_bio(struct btrfs_inode *inode, + struct btrfs_bio_ctrl *bio_ctrl, + u64 disk_bytenr, u64 file_offset) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct btrfs_bio *bbio; + + bbio = btrfs_bio_alloc(BIO_MAX_VECS, bio_ctrl->opf, inode, + file_offset, bio_ctrl->end_io_func, NULL); + bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT; + bbio->bio.bi_write_hint = inode->vfs_inode.i_write_hint; + bio_ctrl->bbio = bbio; + bio_ctrl->len_to_oe_boundary = U32_MAX; + bio_ctrl->next_file_offset = file_offset; + + /* Limit data write bios to the ordered boundary. */ + if (bio_ctrl->wbc) { + struct btrfs_ordered_extent *ordered; + + /* This must be a write for data inodes. */ + ASSERT(btrfs_op(&bio_ctrl->bbio->bio) == BTRFS_MAP_WRITE); + ASSERT(is_data_inode(inode)); + + ordered = btrfs_lookup_ordered_extent(inode, file_offset); + if (unlikely(!ordered)) { + bio_ctrl->bbio = NULL; + bio_ctrl->next_file_offset = 0; + bio_put(&bbio->bio); + btrfs_err_rl(fs_info, + "root %lld ino %llu file offset %llu is marked dirty without notifying the fs", + btrfs_root_id(inode->root), btrfs_ino(inode), + file_offset); + return -EUCLEAN; + } + bio_ctrl->len_to_oe_boundary = min_t(u32, U32_MAX, + ordered->file_offset + + ordered->disk_num_bytes - file_offset); + bbio->ordered = ordered; + + /* + * Pick the last added device to support cgroup writeback. For + * multi-device file systems this means blk-cgroup policies have + * to always be set on the last added/replaced device. + * This is a bit odd but has been like that for a long time. + */ + bio_set_dev(&bbio->bio, fs_info->fs_devices->latest_dev->bdev); + wbc_init_bio(bio_ctrl->wbc, &bbio->bio); + } + return 0; +} + +/* + * @disk_bytenr: logical bytenr where the write will be + * @page: page to add to the bio + * @size: portion of page that we want to write to + * @pg_offset: offset of the new bio or to check whether we are adding + * a contiguous page to the previous one + * @read_em_generation: generation of the extent_map we are submitting + * (only used for read) + * + * The will either add the page into the existing @bio_ctrl->bbio, or allocate a + * new one in @bio_ctrl->bbio. + * The mirror number for this IO should already be initialized in + * @bio_ctrl->mirror_num. + * + * Return the number of bytes that are queued into a bio. + * If the returned bytes is smaller than @size, it means we hit a critical error + * for data write, where there is no ordered extent for the range. + */ +static unsigned int submit_extent_folio(struct btrfs_bio_ctrl *bio_ctrl, + u64 disk_bytenr, struct folio *folio, + size_t size, unsigned long pg_offset, + u64 read_em_generation) +{ + struct btrfs_inode *inode = folio_to_inode(folio); + loff_t file_offset = folio_pos(folio) + pg_offset; + unsigned int queued = 0; + + ASSERT(pg_offset + size <= folio_size(folio)); + ASSERT(bio_ctrl->end_io_func); + + if (bio_ctrl->bbio && + !btrfs_bio_is_contig(bio_ctrl, disk_bytenr, file_offset)) + submit_one_bio(bio_ctrl); + + do { + u32 len = size; + + /* Allocate new bio if needed */ + if (!bio_ctrl->bbio) { + int ret; + + ret = alloc_new_bio(inode, bio_ctrl, disk_bytenr, file_offset); + if (ret < 0) + break; + } + + /* Cap to the current ordered extent boundary if there is one. */ + if (len > bio_ctrl->len_to_oe_boundary) { + ASSERT(bio_ctrl->compress_type == BTRFS_COMPRESS_NONE); + ASSERT(is_data_inode(inode)); + len = bio_ctrl->len_to_oe_boundary; + } + + if (!bio_add_folio(&bio_ctrl->bbio->bio, folio, len, pg_offset)) { + /* bio full: move on to a new one */ + submit_one_bio(bio_ctrl); + continue; + } + /* + * Now that the folio is definitely added to the bio, include its + * generation in the max generation calculation. + */ + bio_ctrl->generation = max(bio_ctrl->generation, read_em_generation); + bio_ctrl->next_file_offset += len; + + if (bio_ctrl->wbc) + wbc_account_cgroup_owner(bio_ctrl->wbc, folio, len); + + size -= len; + pg_offset += len; + disk_bytenr += len; + file_offset += len; + queued += len; + + /* + * len_to_oe_boundary defaults to U32_MAX, which isn't folio or + * sector aligned. alloc_new_bio() then sets it to the end of + * our ordered extent for writes into zoned devices. + * + * When len_to_oe_boundary is tracking an ordered extent, we + * trust the ordered extent code to align things properly, and + * the check above to cap our write to the ordered extent + * boundary is correct. + * + * When len_to_oe_boundary is U32_MAX, the cap above would + * result in a 4095 byte IO for the last folio right before + * we hit the bio limit of UINT_MAX. bio_add_folio() has all + * the checks required to make sure we don't overflow the bio, + * and we should just ignore len_to_oe_boundary completely + * unless we're using it to track an ordered extent. + * + * It's pretty hard to make a bio sized U32_MAX, but it can + * happen when the page cache is able to feed us contiguous + * folios for large extents. + */ + if (bio_ctrl->len_to_oe_boundary != U32_MAX) + bio_ctrl->len_to_oe_boundary -= len; + + /* Ordered extent boundary: move on to a new bio. */ + if (bio_ctrl->len_to_oe_boundary == 0) + submit_one_bio(bio_ctrl); + /* + * If we have accumulated decent amount of IO, send it to the + * block layer so that IO can run while we are accumulating + * more folios to write. + */ + else if (bio_ctrl->wbc && + bio_ctrl->bbio->bio.bi_iter.bi_size >= + inode->root->fs_info->writeback_bio_size) + submit_one_bio(bio_ctrl); + + } while (size); + return queued; +} + +static int attach_extent_buffer_folio(struct extent_buffer *eb, + struct folio *folio, + struct btrfs_folio_state *prealloc) +{ + struct btrfs_fs_info *fs_info = eb->fs_info; + int ret = 0; + + /* + * If the page is mapped to btree inode, we should hold the private + * lock to prevent race. + * For cloned or dummy extent buffers, their pages are not mapped and + * will not race with any other ebs. + */ + if (folio->mapping) + lockdep_assert_held(&folio->mapping->i_private_lock); + + if (!btrfs_meta_is_subpage(fs_info)) { + if (!folio_test_private(folio)) + folio_attach_private(folio, eb); + else + WARN_ON(folio_get_private(folio) != eb); + return 0; + } + + /* Already mapped, just free prealloc */ + if (folio_test_private(folio)) { + btrfs_free_folio_state(prealloc); + return 0; + } + + if (prealloc) + /* Has preallocated memory for subpage */ + folio_attach_private(folio, prealloc); + else + /* Do new allocation to attach subpage */ + ret = btrfs_attach_folio_state(fs_info, folio, BTRFS_SUBPAGE_METADATA); + return ret; +} + +int set_folio_extent_mapped(struct folio *folio) +{ + struct btrfs_fs_info *fs_info; + + ASSERT(folio->mapping); + + if (folio_test_private(folio)) + return 0; + + fs_info = folio_to_fs_info(folio); + + if (btrfs_is_subpage(fs_info, folio)) + return btrfs_attach_folio_state(fs_info, folio, BTRFS_SUBPAGE_DATA); + + folio_attach_private(folio, (void *)EXTENT_FOLIO_PRIVATE); + return 0; +} + +void clear_folio_extent_mapped(struct folio *folio) +{ + struct btrfs_fs_info *fs_info; + + ASSERT(folio->mapping); + /* + * The folio should not have writeback nor dirty flag set. + * + * If dirty flag is set, the folio can be written back again and we + * expect the private flag set for the folio. + * + * If writeback flag is set, the endio may need to utilize the + * private for btrfs_folio_state. + */ + ASSERT(!folio_test_dirty(folio)); + ASSERT(!folio_test_writeback(folio)); + + if (!folio_test_private(folio)) + return; + + fs_info = folio_to_fs_info(folio); + if (btrfs_is_subpage(fs_info, folio)) + return btrfs_detach_folio_state(fs_info, folio, BTRFS_SUBPAGE_DATA); + + folio_detach_private(folio); +} + +static struct extent_map *get_extent_map(struct btrfs_inode *inode, + struct folio *folio, u64 start, + u64 len, struct extent_map **em_cached) +{ + struct extent_map *em; + + ASSERT(em_cached); + + if (*em_cached) { + em = *em_cached; + if (btrfs_extent_map_in_tree(em) && start >= em->start && + start < btrfs_extent_map_end(em)) { + refcount_inc(&em->refs); + return em; + } + + btrfs_free_extent_map(em); + *em_cached = NULL; + } + + em = btrfs_get_extent(inode, folio, start, len); + if (!IS_ERR(em)) { + BUG_ON(*em_cached); + refcount_inc(&em->refs); + *em_cached = em; + } + + return em; +} + +static void btrfs_readahead_expand(struct readahead_control *ractl, + const struct extent_map *em) +{ + const u64 ra_pos = readahead_pos(ractl); + const u64 ra_end = ra_pos + readahead_length(ractl); + const u64 em_end = btrfs_extent_map_end(em); + + /* No expansion for holes and inline extents. */ + if (em->disk_bytenr > EXTENT_MAP_LAST_BYTE) + return; + + ASSERT(em_end >= ra_pos, + "extent_map %llu %llu ends before current readahead position %llu", + em->start, em->len, ra_pos); + if (em_end > ra_end) + readahead_expand(ractl, ra_pos, em_end - ra_pos); +} + +/* + * basic readpage implementation. Locked extent state structs are inserted + * into the tree that are removed when the IO is done (by the end_io + * handlers) + * XXX JDM: This needs looking at to ensure proper page locking + * return 0 on success, otherwise return error + */ +static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached, + struct btrfs_bio_ctrl *bio_ctrl, + struct fsverity_info *vi) +{ + struct inode *inode = folio->mapping->host; + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + u64 start = folio_pos(folio); + const u64 end = start + folio_size(folio) - 1; + u64 extent_offset; + u64 locked_end; + u64 last_byte = i_size_read(inode); + struct extent_map *em; + int ret = 0; + const size_t blocksize = fs_info->sectorsize; + + if (bio_ctrl->ractl) + locked_end = readahead_pos(bio_ctrl->ractl) + readahead_length(bio_ctrl->ractl) - 1; + else + locked_end = end; + + ret = set_folio_extent_mapped(folio); + if (ret < 0) { + folio_unlock(folio); + return ret; + } + + if (folio_contains(folio, last_byte >> PAGE_SHIFT)) { + size_t zero_offset = offset_in_folio(folio, last_byte); + + if (zero_offset) + folio_zero_range(folio, zero_offset, + folio_size(folio) - zero_offset); + } + bio_ctrl->end_io_func = end_bbio_data_read; + begin_folio_read(fs_info, folio); + for (u64 cur = start; cur <= end; cur += blocksize) { + enum btrfs_compression_type compress_type = BTRFS_COMPRESS_NONE; + unsigned long pg_offset = offset_in_folio(folio, cur); + bool force_bio_submit = false; + u64 disk_bytenr; + u64 block_start; + u64 em_gen; + unsigned int queued; + + ASSERT(IS_ALIGNED(cur, fs_info->sectorsize)); + if (cur >= last_byte) { + folio_zero_range(folio, pg_offset, end - cur + 1); + end_folio_read(vi, folio, true, cur, end - cur + 1); + break; + } + if (btrfs_folio_test_uptodate(fs_info, folio, cur, blocksize)) { + end_folio_read(vi, folio, true, cur, blocksize); + continue; + } + /* + * Search extent map for the whole locked range. + * This will allow btrfs_get_extent() to return a larger hole + * when possible. + * This can reduce duplicated btrfs_get_extent() calls for large + * holes. + */ + em = get_extent_map(BTRFS_I(inode), folio, cur, locked_end - cur + 1, em_cached); + if (IS_ERR(em)) { + end_folio_read(vi, folio, false, cur, end + 1 - cur); + return PTR_ERR(em); + } + extent_offset = cur - em->start; + BUG_ON(btrfs_extent_map_end(em) <= cur); + BUG_ON(end < cur); + + compress_type = btrfs_extent_map_compression(em); + + /* + * Only expand readahead for extents which are already creating + * the pages anyway in add_ra_bio_pages, which is compressed + * extents in the non subpage case. + */ + if (bio_ctrl->ractl && + !btrfs_is_subpage(fs_info, folio) && + compress_type != BTRFS_COMPRESS_NONE) + btrfs_readahead_expand(bio_ctrl->ractl, em); + + if (compress_type != BTRFS_COMPRESS_NONE) + disk_bytenr = em->disk_bytenr; + else + disk_bytenr = btrfs_extent_map_block_start(em) + extent_offset; + + if (em->flags & EXTENT_FLAG_PREALLOC) + block_start = EXTENT_MAP_HOLE; + else + block_start = btrfs_extent_map_block_start(em); + + /* + * If we have a file range that points to a compressed extent + * and it's followed by a consecutive file range that points + * to the same compressed extent (possibly with a different + * offset and/or length, so it either points to the whole extent + * or only part of it), we must make sure we do not submit a + * single bio to populate the folios for the 2 ranges because + * this makes the compressed extent read zero out the folios + * belonging to the 2nd range. Imagine the following scenario: + * + * File layout + * [0 - 8K] [8K - 24K] + * | | + * | | + * points to extent X, points to extent X, + * offset 4K, length of 8K offset 0, length 16K + * + * [extent X, compressed length = 4K uncompressed length = 16K] + * + * If the bio to read the compressed extent covers both ranges, + * it will decompress extent X into the folios belonging to the + * first range and then it will stop, zeroing out the remaining + * folios that belong to the other range that points to extent X. + * So here we make sure we submit 2 bios, one for the first + * range and another one for the third range. Both will target + * the same physical extent from disk, but we can't currently + * make the compressed bio endio callback populate the folios + * for both ranges because each compressed bio is tightly + * coupled with a single extent map, and each range can have + * an extent map with a different offset value relative to the + * uncompressed data of our extent and different lengths. This + * is a corner case so we prioritize correctness over + * non-optimal behavior (submitting 2 bios for the same extent). + */ + if (compress_type != BTRFS_COMPRESS_NONE && + bio_ctrl->last_em_start != U64_MAX && + bio_ctrl->last_em_start != em->start) + force_bio_submit = true; + + bio_ctrl->last_em_start = em->start; + + em_gen = em->generation; + btrfs_free_extent_map(em); + em = NULL; + + /* we've found a hole, just zero and go on */ + if (block_start == EXTENT_MAP_HOLE) { + folio_zero_range(folio, pg_offset, blocksize); + end_folio_read(vi, folio, true, cur, blocksize); + continue; + } + /* the get_extent function already copied into the folio */ + if (block_start == EXTENT_MAP_INLINE) { + end_folio_read(vi, folio, true, cur, blocksize); + continue; + } + + if (bio_ctrl->compress_type != compress_type) { + submit_one_bio(bio_ctrl); + bio_ctrl->compress_type = compress_type; + } + + if (force_bio_submit) + submit_one_bio(bio_ctrl); + queued = submit_extent_folio(bio_ctrl, disk_bytenr, folio, blocksize, + pg_offset, em_gen); + /* Read submission should not fail. */ + ASSERT(queued == blocksize); + } + return 0; +} + +/* + * Check if we can skip waiting the @ordered extent covering the block at @fileoff. + * + * @fileoff: Both input and output. + * Input as the file offset where the check should start at. + * Output as where the next check should start at, + * if the function returns true. + * + * Return true if we can skip to @fileoff. The caller needs to check the new + * @fileoff value to make sure it covers the full range, before skipping the + * full OE. + * + * Return false if we must wait for the ordered extent. + */ +static bool can_skip_one_ordered_range(struct btrfs_inode *inode, + struct btrfs_ordered_extent *ordered, + u64 *fileoff) +{ + const struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct folio *folio; + const u32 blocksize = fs_info->sectorsize; + u64 cur = *fileoff; + bool ret; + + folio = filemap_get_folio(inode->vfs_inode.i_mapping, cur >> PAGE_SHIFT); + + /* + * We should have locked the folio(s) for range [start, end], thus + * there must be a folio and it must be locked. + */ + ASSERT(!IS_ERR(folio)); + ASSERT(folio_test_locked(folio)); + + /* + * There are several cases for the folio and OE combination: + * + * 1) Folio has no private flag + * The OE has all its IO done but not yet finished, and folio got + * invalidated. + * + * Have we have to wait for the OE to finish, as it may contain the + * to-be-inserted data checksum. + * Without the data checksum inserted into the csum tree, read will + * just fail with missing csum. + */ + if (!folio_test_private(folio)) { + ret = false; + goto out; + } + + /* + * 2) The first block is DIRTY. + * + * This means the OE is created by some other folios whose file pos is + * before this one. And since we are holding the folio lock, the writeback + * of this folio cannot start. + * + * We must skip the whole OE, because it will never start until we + * finished our folio read and unlocked the folio. + */ + if (btrfs_folio_test_dirty(fs_info, folio, cur, blocksize)) { + u64 range_len = umin(folio_next_pos(folio), + ordered->file_offset + ordered->num_bytes) - cur; + + ret = true; + /* + * At least inside the folio, all the remaining blocks should + * also be dirty. + */ + ASSERT(btrfs_folio_test_dirty(fs_info, folio, cur, range_len)); + *fileoff = ordered->file_offset + ordered->num_bytes; + goto out; + } + + /* + * 3) The first block is uptodate. + * + * At least the first block can be skipped, but we are still not fully + * sure. E.g. if the OE has some other folios in the range that cannot + * be skipped. + * So we return true and update @next_ret to the OE/folio boundary. + */ + if (btrfs_folio_test_uptodate(fs_info, folio, cur, blocksize)) { + u64 range_len = umin(folio_next_pos(folio), + ordered->file_offset + ordered->num_bytes) - cur; + + /* + * The whole range to the OE end or folio boundary should also + * be uptodate. + */ + ASSERT(btrfs_folio_test_uptodate(fs_info, folio, cur, range_len)); + ret = true; + *fileoff = cur + range_len; + goto out; + } + + /* + * 4) The first block is not uptodate. + * + * This means the folio is invalidated after the writeback was finished, + * but by some other operations (e.g. block aligned buffered write) the + * folio is inserted into filemap. + * Very much the same as case 1). + */ + ret = false; +out: + folio_put(folio); + return ret; +} + +static bool can_skip_ordered_extent(struct btrfs_inode *inode, + struct btrfs_ordered_extent *ordered, + u64 start, u64 end) +{ + const u64 range_end = min(end, ordered->file_offset + ordered->num_bytes - 1); + u64 cur = max(start, ordered->file_offset); + + while (cur < range_end) { + bool can_skip; + + can_skip = can_skip_one_ordered_range(inode, ordered, &cur); + if (!can_skip) + return false; + } + return true; +} + +/* + * Locking helper to make sure we get a stable view of extent maps for the + * involved range. + * + * This is for folio read paths (read and readahead), thus the involved range + * should have all the folios locked. + */ +static void lock_extents_for_read(struct btrfs_inode *inode, u64 start, u64 end, + struct extent_state **cached_state) +{ + u64 cur_pos; + + /* Caller must provide a valid @cached_state. */ + ASSERT(cached_state); + + /* The range must at least be page aligned, as all read paths are folio based. */ + ASSERT(IS_ALIGNED(start, PAGE_SIZE)); + ASSERT(IS_ALIGNED(end + 1, PAGE_SIZE)); + +again: + btrfs_lock_extent(&inode->io_tree, start, end, cached_state); + cur_pos = start; + while (cur_pos < end) { + struct btrfs_ordered_extent *ordered; + + ordered = btrfs_lookup_ordered_range(inode, cur_pos, + end - cur_pos + 1); + /* + * No ordered extents in the range, and we hold the extent lock, + * no one can modify the extent maps in the range, we're safe to return. + */ + if (!ordered) + break; + + /* Check if we can skip waiting for the whole OE. */ + if (can_skip_ordered_extent(inode, ordered, start, end)) { + cur_pos = min(ordered->file_offset + ordered->num_bytes, + end + 1); + btrfs_put_ordered_extent(ordered); + continue; + } + + /* Now wait for the OE to finish. */ + btrfs_unlock_extent(&inode->io_tree, start, end, cached_state); + btrfs_start_ordered_extent_nowriteback(ordered, start, end + 1 - start); + btrfs_put_ordered_extent(ordered); + /* We have unlocked the whole range, restart from the beginning. */ + goto again; + } +} + +static void assert_folio_range(const struct btrfs_inode *inode, + u64 start, u64 end) +{ + const u32 blocksize = inode->root->fs_info->sectorsize; + + /* + * For btrfs page cache, a folio always contains at least one block, + * so the range should always be block size aligned. + */ + ASSERT(IS_ALIGNED(start, blocksize) && IS_ALIGNED(end + 1, blocksize), + "blocksize=%u root=%lld ino=%llu start=%llu end=%llu mapping min order=%u", + blocksize, btrfs_root_id(inode->root), btrfs_ino(inode), + start, end, + mapping_min_folio_order(inode->vfs_inode.i_mapping)); +} + +int btrfs_read_folio(struct file *file, struct folio *folio) +{ + struct inode *vfs_inode = folio->mapping->host; + struct btrfs_inode *inode = BTRFS_I(vfs_inode); + const u64 start = folio_pos(folio); + const u64 end = start + folio_size(folio) - 1; + struct extent_state *cached_state = NULL; + struct btrfs_bio_ctrl bio_ctrl = { + .opf = REQ_OP_READ, + .last_em_start = U64_MAX, + }; + struct extent_map *em_cached = NULL; + struct fsverity_info *vi = NULL; + int ret; + + assert_folio_range(inode, start, end); + lock_extents_for_read(inode, start, end, &cached_state); + if (folio_pos(folio) < i_size_read(vfs_inode)) + vi = fsverity_get_info(vfs_inode); + ret = btrfs_do_readpage(folio, &em_cached, &bio_ctrl, vi); + btrfs_unlock_extent(&inode->io_tree, start, end, &cached_state); + + btrfs_free_extent_map(em_cached); + + /* + * If btrfs_do_readpage() failed we will want to submit the assembled + * bio to do the cleanup. + */ + submit_one_bio(&bio_ctrl); + return ret; +} + +static void set_delalloc_bitmap(struct folio *folio, unsigned long *delalloc_bitmap, + u64 start, u32 len) +{ + struct btrfs_fs_info *fs_info = folio_to_fs_info(folio); + const u64 folio_start = folio_pos(folio); + unsigned int start_bit; + unsigned int nbits; + + ASSERT(start >= folio_start && start + len <= folio_start + folio_size(folio)); + start_bit = (start - folio_start) >> fs_info->sectorsize_bits; + nbits = len >> fs_info->sectorsize_bits; + ASSERT(bitmap_test_range_all_zero(delalloc_bitmap, start_bit, nbits)); + bitmap_set(delalloc_bitmap, start_bit, nbits); +} + +static bool find_next_delalloc_bitmap(struct folio *folio, + unsigned long *delalloc_bitmap, u64 start, + u64 *found_start, u32 *found_len) +{ + struct btrfs_fs_info *fs_info = folio_to_fs_info(folio); + const u64 folio_start = folio_pos(folio); + const unsigned int bitmap_size = btrfs_blocks_per_folio(fs_info, folio); + unsigned int start_bit; + unsigned int first_zero; + unsigned int first_set; + + ASSERT(start >= folio_start && start < folio_start + folio_size(folio)); + + start_bit = (start - folio_start) >> fs_info->sectorsize_bits; + first_set = find_next_bit(delalloc_bitmap, bitmap_size, start_bit); + if (first_set >= bitmap_size) + return false; + + *found_start = folio_start + (first_set << fs_info->sectorsize_bits); + first_zero = find_next_zero_bit(delalloc_bitmap, bitmap_size, first_set); + *found_len = (first_zero - first_set) << fs_info->sectorsize_bits; + return true; +} + +/* + * Debug checks for fixup selection logic to help ensure the invariants + * we expect for fixup marking hold in practice. + * + * - A dirty block without a fixup bit is covered by delalloc or a running + * ordered extent (it was dirtied by a reserving write path). + * - A block with a fixup bit is never covered by delalloc: every delalloc + * setter holds the folio lock and cancels the fixup state of the blocks + * it covers (btrfs_folio_set_dirty()) before releasing it. + */ +static void debug_check_writepage_fixup(struct btrfs_inode *inode, u64 start, + u32 len, bool needs_fixup) +{ + struct btrfs_ordered_extent *ordered; + bool delalloc; + + if (!IS_ENABLED(CONFIG_BTRFS_DEBUG)) + return; + + delalloc = btrfs_test_range_bit_exists(&inode->io_tree, start, + start + len - 1, EXTENT_DELALLOC); + if (needs_fixup) { + if (unlikely(delalloc)) + DEBUG_WARN("writeback: delalloc and fixup conflict. ino %llu start %llu", + btrfs_ino(inode), start); + } else { + if (delalloc) + return; + + ordered = btrfs_lookup_ordered_range(inode, start, len); + if (unlikely(!ordered)) + DEBUG_WARN("dirty block, no delalloc, fixup, ordered. ino %llu start %llu", + btrfs_ino(inode), start); + else + btrfs_put_ordered_extent(ordered); + } +} + +/* + * Handle folios dirtied without a delalloc reservation, e.g. + * O_DIRECT read into a MAP_SHARED mapping dirtying via set_page_dirty_lock(). + * + * btrfs_data_dirty_folio() records the affected blocks in the fixup bitmap + * and the folio fixup flag and we check them here in writeback. + * + * Don't submit such blocks and queue work for the fixup worker to reserve + * space for them so that they can be submitted properly by writeback. + * + * Return 1 if the folio needed fixup, 0 if not, and a negative error code + * on error. + */ +static noinline_for_stack int writepage_fixup(struct btrfs_inode *inode, + struct folio *folio, + struct btrfs_bio_ctrl *bio_ctrl) +{ + struct btrfs_fs_info *fs_info = inode_to_fs_info(&inode->vfs_inode); + const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio); + const u32 sectorsize = fs_info->sectorsize; + const u64 page_start = folio_pos(folio); + bool found_fixup = false; + unsigned int bit; + + /* + * A folio was dirtied without calling aops->dirty_folio() which we + * explicitly assert is not allowed. + */ + if (unlikely(bitmap_empty(bio_ctrl->submit_bitmap, blocks_per_folio))) { + DEBUG_WARN(); + btrfs_err_rl(fs_info, + "root %lld ino %llu folio %llu is dirty with an empty dirty bitmap", + btrfs_root_id(inode->root), btrfs_ino(inode), + folio_pos(folio)); + return -EUCLEAN; + } + + /* Cheap check on the folio flag. Set iff the fixup bitmap is non-empty. */ + if (likely(!folio_test_fixup_pending(folio))) + return 0; + + for_each_set_bit(bit, bio_ctrl->submit_bitmap, blocks_per_folio) { + const u64 start = page_start + (bit << fs_info->sectorsize_bits); + const bool needs_fixup = btrfs_folio_test_fixup(fs_info, folio, + start, sectorsize); + + debug_check_writepage_fixup(inode, start, sectorsize, needs_fixup); + if (needs_fixup) { + bitmap_clear(bio_ctrl->submit_bitmap, bit, 1); + found_fixup = true; + } + } + if (likely(found_fixup)) { + btrfs_queue_writepage_fixup(inode, folio); + folio_redirty_for_writepage(bio_ctrl->wbc, folio); + if (bitmap_empty(bio_ctrl->submit_bitmap, blocks_per_folio)) { + folio_unlock(folio); + return 1; + } + return 0; + } + /* We should always find fixup if the folio fixup flag was set. */ + DEBUG_WARN(); + btrfs_err_rl(fs_info, + "root %lld ino %llu folio %llu is fixup with an empty fixup bitmap", + btrfs_root_id(inode->root), btrfs_ino(inode), + folio_pos(folio)); + + return -EUCLEAN; +} + +/* + * Do all of the delayed allocation setup. + * + * Return >0 if all the dirty blocks are submitted async (compression) or inlined. + * The @folio should no longer be touched (treat it as already unlocked). + * + * Return 0 if there is still dirty block that needs to be submitted through + * extent_writepage_io(). + * bio_ctrl->submit_bitmap will indicate which blocks of the folio should be + * submitted, and @folio is still kept locked. + * + * Return <0 if there is any error hit. + * Any allocated ordered extent range covering this folio will be marked + * finished (IOERR), and @folio is still kept locked. + */ +static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode, + struct folio *folio, + struct btrfs_bio_ctrl *bio_ctrl) +{ + struct btrfs_fs_info *fs_info = inode_to_fs_info(&inode->vfs_inode); + struct writeback_control *wbc = bio_ctrl->wbc; + const bool is_subpage = btrfs_is_subpage(fs_info, folio); + const u64 page_start = folio_pos(folio); + const u64 page_end = page_start + folio_size(folio) - 1; + const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio); + unsigned long delalloc_bitmap[BITS_TO_LONGS(BTRFS_MAX_BLOCKS_PER_FOLIO)] = { 0 }; + /* + * Save the last found delalloc end. As the delalloc end can go beyond + * page boundary, thus we cannot rely on subpage bitmap to locate the + * last delalloc end. + */ + u64 last_delalloc_end = 0; + /* + * The range end (exclusive) of the last successfully finished delalloc + * range. + * Any range covered by ordered extent must either be manually marked + * finished (error handling), or has IO submitted (and finish the + * ordered extent normally). + * + * This records the end of ordered extent cleanup if we hit an error. + */ + u64 last_finished_delalloc_end = page_start; + u64 delalloc_start = page_start; + u64 delalloc_end = page_end; + u64 delalloc_to_write = 0; + unsigned int start_bit; + unsigned int end_bit; + int ret = 0; + + /* Save the dirty bitmap as our submission bitmap will be a subset of it. */ + btrfs_copy_subpage_dirty_bitmap(fs_info, folio, bio_ctrl->submit_bitmap); + + ret = writepage_fixup(inode, folio, bio_ctrl); + if (ret) + return ret; + + for_each_set_bitrange(start_bit, end_bit, bio_ctrl->submit_bitmap, + blocks_per_folio) { + u64 start = page_start + (start_bit << fs_info->sectorsize_bits); + u32 len = (end_bit - start_bit) << fs_info->sectorsize_bits; + + btrfs_folio_set_lock(fs_info, folio, start, len); + } + + /* Lock all (subpage) delalloc ranges inside the folio first. */ + while (delalloc_start < page_end) { + delalloc_end = page_end; + if (!find_lock_delalloc_range(&inode->vfs_inode, folio, + &delalloc_start, &delalloc_end)) { + delalloc_start = delalloc_end + 1; + continue; + } + set_delalloc_bitmap(folio, delalloc_bitmap, delalloc_start, + min(delalloc_end, page_end) + 1 - delalloc_start); + last_delalloc_end = delalloc_end; + delalloc_start = delalloc_end + 1; + } + delalloc_start = page_start; + + if (!last_delalloc_end) + goto out; + + /* Run the delalloc ranges for the above locked ranges. */ + while (delalloc_start < page_end) { + u64 found_start; + u32 found_len; + bool found; + + if (!is_subpage) { + /* + * For non-subpage case, the found delalloc range must + * cover this folio and there must be only one locked + * delalloc range. + */ + found_start = page_start; + found_len = last_delalloc_end + 1 - found_start; + found = true; + } else { + found = find_next_delalloc_bitmap(folio, delalloc_bitmap, + delalloc_start, &found_start, &found_len); + } + if (!found) + break; + /* + * The subpage range covers the last sector, the delalloc range may + * end beyond the folio boundary, use the saved delalloc_end + * instead. + */ + if (found_start + found_len >= page_end) + found_len = last_delalloc_end + 1 - found_start; + + if (ret >= 0) { + /* + * Some delalloc range may be created by previous folios. + * Thus we still need to clean up this range during error + * handling. + */ + last_finished_delalloc_end = found_start; + /* No errors hit so far, run the current delalloc range. */ + ret = btrfs_run_delalloc_range(inode, folio, + found_start, + found_start + found_len - 1, + wbc); + if (ret >= 0) + last_finished_delalloc_end = found_start + found_len; + if (unlikely(ret < 0)) + btrfs_err_rl(fs_info, +"failed to run delalloc range, root=%lld ino=%llu folio=%llu submit_bitmap=%*pbl start=%llu len=%u: %pe", + btrfs_root_id(inode->root), + btrfs_ino(inode), + folio_pos(folio), + blocks_per_folio, + bio_ctrl->submit_bitmap, + found_start, found_len, ERR_PTR(ret)); + } else { + /* + * We've hit an error during previous delalloc range, + * have to cleanup the remaining locked ranges. + */ + btrfs_unlock_extent(&inode->io_tree, found_start, + found_start + found_len - 1, NULL); + unlock_delalloc_folio(&inode->vfs_inode, folio, + found_start, + found_start + found_len - 1); + } + + /* + * We have some ranges that's going to be submitted asynchronously + * (compression or inline). These range have their own control + * on when to unlock the pages. We should not touch them + * anymore, so clear the range from the submission bitmap. + */ + if (ret > 0) { + unsigned int start_bit = (found_start - page_start) >> + fs_info->sectorsize_bits; + unsigned int end_bit = (min(page_end + 1, found_start + found_len) - + page_start) >> fs_info->sectorsize_bits; + bitmap_clear(bio_ctrl->submit_bitmap, start_bit, end_bit - start_bit); + } + /* + * Above btrfs_run_delalloc_range() may have unlocked the folio, + * thus for the last range, we cannot touch the folio anymore. + */ + if (found_start + found_len >= last_delalloc_end + 1) + break; + + delalloc_start = found_start + found_len; + } + /* + * It's possible we had some ordered extents created before we hit + * an error, cleanup non-async successfully created delalloc ranges. + */ + if (unlikely(ret < 0)) { + unsigned int bitmap_size = min( + (last_finished_delalloc_end - page_start) >> + fs_info->sectorsize_bits, + blocks_per_folio); + + for_each_set_bitrange(start_bit, end_bit, bio_ctrl->submit_bitmap, + bitmap_size) { + u64 start = page_start + (start_bit << fs_info->sectorsize_bits); + u32 len = (end_bit - start_bit) << fs_info->sectorsize_bits; + + btrfs_mark_ordered_io_finished(inode, start, len, false); + } + return ret; + } +out: + if (last_delalloc_end) + delalloc_end = last_delalloc_end; + else + delalloc_end = page_end; + /* + * delalloc_end is already one less than the total length, so + * we don't subtract one from PAGE_SIZE. + */ + delalloc_to_write += + DIV_ROUND_UP(delalloc_end + 1 - page_start, PAGE_SIZE); + + /* + * If all ranges are submitted asynchronously, we just need to account + * for them here. + */ + if (bitmap_empty(bio_ctrl->submit_bitmap, blocks_per_folio)) { + wbc->nr_to_write -= delalloc_to_write; + return 1; + } + + if (wbc->nr_to_write < delalloc_to_write) { + int thresh = 8192; + + if (delalloc_to_write < thresh * 2) + thresh = delalloc_to_write; + wbc->nr_to_write = min_t(u64, delalloc_to_write, + thresh); + } + + return 0; +} + +/* + * Return 0 if we have submitted or queued the sector for submission. + * Return <0 for critical errors, and the involved sector will be cleaned up. + * + * Caller should make sure filepos < i_size and handle filepos >= i_size case. + */ +static int submit_one_sector(struct btrfs_inode *inode, + struct folio *folio, + u64 filepos, struct btrfs_bio_ctrl *bio_ctrl, + loff_t i_size) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct extent_map *em; + u64 block_start; + u64 disk_bytenr; + u64 extent_offset; + u64 em_end; + const u32 sectorsize = fs_info->sectorsize; + unsigned int queued; + + ASSERT(IS_ALIGNED(filepos, sectorsize)); + + /* @filepos >= i_size case should be handled by the caller. */ + ASSERT(filepos < i_size); + + em = btrfs_get_extent(inode, NULL, filepos, sectorsize); + if (IS_ERR(em)) { + /* + * bio_ctrl may contain a bio crossing several folios. + * Submit it immediately so that the bio has a chance + * to finish normally, other than marked as error. + */ + submit_one_bio(bio_ctrl); + + /* + * When submission failed, we should still clear the folio dirty. + * Or the folio will be written back again but without any + * ordered extent. + */ + btrfs_folio_clear_dirty(fs_info, folio, filepos, sectorsize); + btrfs_folio_set_writeback(fs_info, folio, filepos, sectorsize); + btrfs_folio_clear_writeback(fs_info, folio, filepos, sectorsize); + + /* + * Since there is no bio submitted to finish the ordered + * extent, we have to manually finish this sector. + */ + btrfs_mark_ordered_io_finished(inode, filepos, fs_info->sectorsize, + false); + return PTR_ERR(em); + } + + extent_offset = filepos - em->start; + em_end = btrfs_extent_map_end(em); + ASSERT(filepos <= em_end); + ASSERT(IS_ALIGNED(em->start, sectorsize)); + ASSERT(IS_ALIGNED(em->len, sectorsize)); + + block_start = btrfs_extent_map_block_start(em); + disk_bytenr = btrfs_extent_map_block_start(em) + extent_offset; + + ASSERT(!btrfs_extent_map_is_compressed(em)); + ASSERT(block_start != EXTENT_MAP_HOLE); + ASSERT(block_start != EXTENT_MAP_INLINE); + + btrfs_free_extent_map(em); + em = NULL; + + /* + * Although the PageDirty bit is cleared before entering this + * function, subpage dirty bit is not cleared. + * So clear subpage dirty bit here so next time we won't submit + * a folio for a range already written to disk. + */ + btrfs_folio_clear_dirty(fs_info, folio, filepos, sectorsize); + btrfs_folio_set_writeback(fs_info, folio, filepos, sectorsize); + /* + * Above call should set the whole folio with writeback flag, even + * just for a single subpage sector. + * As long as the folio is properly locked and the range is correct, + * we should always get the folio with writeback flag. + */ + ASSERT(folio_test_writeback(folio)); + + queued = submit_extent_folio(bio_ctrl, disk_bytenr, folio, + sectorsize, filepos - folio_pos(folio), 0); + if (unlikely(queued < sectorsize)) { + btrfs_folio_clear_writeback(fs_info, folio, filepos, sectorsize); + btrfs_mark_ordered_io_finished(inode, filepos, fs_info->sectorsize, + false); + return -EUCLEAN; + } + return 0; +} + +/* + * Helper for extent_writepage(). This calls the writepage start hooks, + * and does the loop to map the page into extents and bios. + * + * We return 1 if the IO is started and the page is unlocked, + * 0 if all went well (page still locked) + * < 0 if there were errors (page still locked) + */ +static noinline_for_stack int extent_writepage_io(struct btrfs_inode *inode, + struct folio *folio, + u64 start, u32 len, + struct btrfs_bio_ctrl *bio_ctrl, + loff_t i_size) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + bool submitted_io = false; + int found_error = 0; + const u64 end = start + len; + const u64 folio_start = folio_pos(folio); + const u64 folio_end = folio_start + folio_size(folio); + const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio); + u64 cur; + int bit; + int ret = 0; + + ASSERT(start >= folio_start, "start=%llu folio_start=%llu", start, folio_start); + ASSERT(end <= folio_end, "start=%llu len=%u folio_start=%llu folio_size=%zu", + start, len, folio_start, folio_size(folio)); + assert_folio_range(inode, folio_start, folio_end - 1); + + /* + * We are about to checksum and write out the data, so it must not be + * mmap writeable, or we could corrupt the data and end up with invalid + * checksums. + */ + btrfs_check_folio_write_protected(folio); + + /* Truncate the submit bitmap to the current range. */ + if (start > folio_start) + bitmap_clear(bio_ctrl->submit_bitmap, 0, + (start - folio_start) >> fs_info->sectorsize_bits); + if (start + len < folio_end) + bitmap_clear(bio_ctrl->submit_bitmap, + (end - folio_start) >> fs_info->sectorsize_bits, + (folio_end - end) >> fs_info->sectorsize_bits); + + bio_ctrl->end_io_func = end_bbio_data_write; + + for_each_set_bit(bit, bio_ctrl->submit_bitmap, blocks_per_folio) { + cur = folio_pos(folio) + (bit << fs_info->sectorsize_bits); + + if (cur >= i_size) { + struct btrfs_ordered_extent *ordered; + + ordered = btrfs_lookup_first_ordered_range(inode, cur, + fs_info->sectorsize); + /* + * We have just run delalloc before getting here, so + * there must be an ordered extent. + */ + ASSERT(ordered != NULL); + spin_lock(&inode->ordered_tree_lock); + set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags); + ordered->truncated_len = min(ordered->truncated_len, + cur - ordered->file_offset); + spin_unlock(&inode->ordered_tree_lock); + btrfs_put_ordered_extent(ordered); + + btrfs_mark_ordered_io_finished(inode, cur, fs_info->sectorsize, true); + /* + * This range is beyond i_size, thus we don't need to + * bother writing back. + * But we still need to clear the dirty subpage bit, or + * the next time the folio gets dirtied, we will try to + * writeback the sectors with subpage dirty bits, + * causing writeback without ordered extent. + */ + btrfs_folio_clear_dirty(fs_info, folio, cur, fs_info->sectorsize); + continue; + } + ret = submit_one_sector(inode, folio, cur, bio_ctrl, i_size); + if (unlikely(ret < 0)) { + if (!found_error) + found_error = ret; + continue; + } + submitted_io = true; + } + + /* + * If we didn't submitted any sector (>= i_size), folio dirty get + * cleared but PAGECACHE_TAG_DIRTY is not cleared (only cleared + * by folio_start_writeback() if the folio is not dirty). + * + * Here we set writeback and clear for the range. If the full folio + * is no longer dirty then we clear the PAGECACHE_TAG_DIRTY tag. + * + * If we hit any error, the corresponding sector will have its dirty + * flag cleared and writeback finished, thus no need to handle the error case. + */ + if (!submitted_io && !found_error) { + btrfs_folio_set_writeback(fs_info, folio, start, len); + btrfs_folio_clear_writeback(fs_info, folio, start, len); + } + return found_error; +} + +static void bio_ctrl_init_submit_bitmap(struct btrfs_fs_info *fs_info, + struct folio *folio, + struct btrfs_bio_ctrl *bio_ctrl) +{ + const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio); + + ASSERT(blocks_per_folio <= BTRFS_MAX_BLOCKS_PER_FOLIO); + + /* + * Default to unlock the whole folio. + * The proper bitmap is not initialized until writepage_delalloc(). + * + * We're safe just to set the bitmap range [0, blocks_per_folio), as + * all later usage of the bitmap will follow the same range limit. + * Any bits beyond blocks_per_folio will be ignored. + */ + bitmap_set(bio_ctrl->submit_bitmap, 0, blocks_per_folio); +} + +/* + * the writepage semantics are similar to regular writepage. extent + * records are inserted to lock ranges in the tree, and as dirty areas + * are found, they are marked writeback. Then the lock bits are removed + * and the end_io handler clears the writeback ranges + * + * Return 0 if everything goes well. + * Return <0 for error. + */ +static int extent_writepage(struct folio *folio, struct btrfs_bio_ctrl *bio_ctrl) +{ + struct btrfs_inode *inode = BTRFS_I(folio->mapping->host); + struct btrfs_fs_info *fs_info = inode->root->fs_info; + int ret; + size_t pg_offset; + loff_t i_size = i_size_read(&inode->vfs_inode); + const pgoff_t end_index = i_size >> PAGE_SHIFT; + const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio); + + trace_extent_writepage(folio, &inode->vfs_inode, bio_ctrl->wbc); + + WARN_ON(!folio_test_locked(folio)); + + pg_offset = offset_in_folio(folio, i_size); + if (folio->index > end_index || + (folio->index == end_index && !pg_offset)) { + folio_invalidate(folio, 0, folio_size(folio)); + folio_unlock(folio); + return 0; + } + + if (folio_contains(folio, end_index)) + folio_zero_range(folio, pg_offset, folio_size(folio) - pg_offset); + + bio_ctrl_init_submit_bitmap(fs_info, folio, bio_ctrl); + /* + * If the page is dirty but without private set, it's marked dirty + * without informing the fs. + * Nowadays that is a bug, since the introduction of + * pin_user_pages*(). + * + * So here we check if the page has private set to rule out such + * case. + */ + if (unlikely(!folio_test_private(folio))) { + DEBUG_WARN(); + btrfs_err_rl(fs_info, + "root %lld ino %llu folio %llu is marked dirty without notifying the fs", + btrfs_root_id(inode->root), + btrfs_ino(inode), folio_pos(folio)); + ret = -EUCLEAN; + goto done; + } + + ret = set_folio_extent_mapped(folio); + if (ret < 0) + goto done; + + ret = writepage_delalloc(inode, folio, bio_ctrl); + if (ret == 1) + return 0; + if (ret) + goto done; + + ret = extent_writepage_io(inode, folio, folio_pos(folio), + folio_size(folio), bio_ctrl, i_size); + if (ret == 1) + return 0; + if (unlikely(ret < 0)) + btrfs_err_rl(fs_info, +"failed to submit blocks, root=%lld inode=%llu folio=%llu submit_bitmap=%*pbl: %pe", + btrfs_root_id(inode->root), btrfs_ino(inode), + folio_pos(folio), blocks_per_folio, + bio_ctrl->submit_bitmap, ERR_PTR(ret)); + + bio_ctrl->wbc->nr_to_write--; + +done: + if (ret < 0) + mapping_set_error(folio->mapping, ret); + /* + * Only unlock ranges that are submitted. As there can be some async + * submitted ranges inside the folio. + */ + btrfs_folio_end_lock_bitmap(fs_info, folio, bio_ctrl->submit_bitmap); + ASSERT(ret <= 0); + return ret; +} + +/* + * Lock extent buffer status and pages for writeback. + * + * Return %false if the extent buffer doesn't need to be submitted (e.g. the + * extent buffer is not dirty) + * Return %true is the extent buffer is submitted to bio. + */ +static noinline_for_stack bool lock_extent_buffer_for_io(struct extent_buffer *eb, + struct writeback_control *wbc) +{ + struct btrfs_fs_info *fs_info = eb->fs_info; + bool ret = false; + + btrfs_tree_lock(eb); + while (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) { + btrfs_tree_unlock(eb); + if (wbc->sync_mode != WB_SYNC_ALL) + return false; + wait_on_extent_buffer_writeback(eb); + btrfs_tree_lock(eb); + } + + /* + * We need to do this to prevent races in people who check if the eb is + * under IO since we can end up having no IO bits set for a short period + * of time. + */ + spin_lock(&eb->refs_lock); + if ((wbc->sync_mode == WB_SYNC_ALL || + atomic_read(&eb->writeback_inhibitors) == 0) && + test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) { + XA_STATE(xas, &fs_info->buffer_tree, eb->start >> fs_info->nodesize_bits); + unsigned long flags; + + set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags); + spin_unlock(&eb->refs_lock); + + xas_lock_irqsave(&xas, flags); + xas_load(&xas); + xas_set_mark(&xas, PAGECACHE_TAG_WRITEBACK); + xas_clear_mark(&xas, PAGECACHE_TAG_DIRTY); + xas_clear_mark(&xas, PAGECACHE_TAG_TOWRITE); + xas_unlock_irqrestore(&xas, flags); + + btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN); + percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, + -(s64)eb->len, + fs_info->dirty_metadata_batch); + ret = true; + } else { + spin_unlock(&eb->refs_lock); + } + btrfs_tree_unlock(eb); + return ret; +} + +static void set_btree_ioerr(struct extent_buffer *eb) +{ + struct btrfs_fs_info *fs_info = eb->fs_info; + + set_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags); + + /* + * A read may stumble upon this buffer later, make sure that it gets an + * error and knows there was an error. + */ + clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); + + /* + * We need to set the mapping with the io error as well because a write + * error will flip the file system readonly, and then syncfs() will + * return a 0 because we are readonly if we don't modify the err seq for + * the superblock. + */ + mapping_set_error(eb->fs_info->btree_inode->i_mapping, -EIO); + + /* + * If writeback for a btree extent that doesn't belong to a log tree + * failed, increment the counter transaction->eb_write_errors. + * We do this because while the transaction is running and before it's + * committing (when we call filemap_fdata[write|wait]_range against + * the btree inode), we might have + * btree_inode->i_mapping->a_ops->writepages() called by the VM - if it + * returns an error or an error happens during writeback, when we're + * committing the transaction we wouldn't know about it, since the pages + * can be no longer dirty nor marked anymore for writeback (if a + * subsequent modification to the extent buffer didn't happen before the + * transaction commit), which makes filemap_fdata[write|wait]_range not + * able to find the pages which contain errors at transaction + * commit time. So if this happens we must abort the transaction, + * otherwise we commit a super block with btree roots that point to + * btree nodes/leafs whose content on disk is invalid - either garbage + * or the content of some node/leaf from a past generation that got + * cowed or deleted and is no longer valid. + * + * Note: setting AS_EIO/AS_ENOSPC in the btree inode's i_mapping would + * not be enough - we need to distinguish between log tree extents vs + * non-log tree extents, and the next filemap_fdatawait_range() call + * will catch and clear such errors in the mapping - and that call might + * be from a log sync and not from a transaction commit. Also, checking + * for the eb flag EXTENT_BUFFER_WRITE_ERR at transaction commit time is + * not done and would not be reliable - the eb might have been released + * from memory and reading it back again means that flag would not be + * set (since it's a runtime flag, not persisted on disk). + * + * Using the flags below in the btree inode also makes us achieve the + * goal of AS_EIO/AS_ENOSPC when writepages() returns success, started + * writeback for all dirty pages and before filemap_fdatawait_range() + * is called, the writeback for all dirty pages had already finished + * with errors - because we were not using AS_EIO/AS_ENOSPC, + * filemap_fdatawait_range() would return success, as it could not know + * that writeback errors happened (the pages were no longer tagged for + * writeback). + */ + switch (eb->log_index) { + case -1: + set_bit(BTRFS_FS_BTREE_ERR, &fs_info->flags); + break; + case 0: + set_bit(BTRFS_FS_LOG1_ERR, &fs_info->flags); + break; + case 1: + set_bit(BTRFS_FS_LOG2_ERR, &fs_info->flags); + break; + default: + BUG(); /* unexpected, logic error */ + } +} + +static void buffer_tree_set_mark(const struct extent_buffer *eb, xa_mark_t mark) +{ + struct btrfs_fs_info *fs_info = eb->fs_info; + XA_STATE(xas, &fs_info->buffer_tree, eb->start >> fs_info->nodesize_bits); + unsigned long flags; + + xas_lock_irqsave(&xas, flags); + xas_load(&xas); + xas_set_mark(&xas, mark); + xas_unlock_irqrestore(&xas, flags); +} + +static void buffer_tree_clear_mark(const struct extent_buffer *eb, xa_mark_t mark) +{ + struct btrfs_fs_info *fs_info = eb->fs_info; + XA_STATE(xas, &fs_info->buffer_tree, eb->start >> fs_info->nodesize_bits); + unsigned long flags; + + xas_lock_irqsave(&xas, flags); + xas_load(&xas); + xas_clear_mark(&xas, mark); + xas_unlock_irqrestore(&xas, flags); +} + +static void buffer_tree_tag_for_writeback(struct btrfs_fs_info *fs_info, + unsigned long start, unsigned long end) +{ + XA_STATE(xas, &fs_info->buffer_tree, start); + unsigned int tagged = 0; + void *eb; + + xas_lock_irq(&xas); + xas_for_each_marked(&xas, eb, end, PAGECACHE_TAG_DIRTY) { + xas_set_mark(&xas, PAGECACHE_TAG_TOWRITE); + if (++tagged % XA_CHECK_SCHED) + continue; + xas_pause(&xas); + xas_unlock_irq(&xas); + cond_resched(); + xas_lock_irq(&xas); + } + xas_unlock_irq(&xas); +} + +struct eb_batch { + unsigned int nr; + unsigned int cur; + struct extent_buffer *ebs[FOLIO_BATCH_SIZE]; +}; + +static inline bool eb_batch_add(struct eb_batch *batch, struct extent_buffer *eb) +{ + batch->ebs[batch->nr++] = eb; + return (batch->nr < FOLIO_BATCH_SIZE); +} + +static inline void eb_batch_init(struct eb_batch *batch) +{ + batch->nr = 0; + batch->cur = 0; +} + +static inline struct extent_buffer *eb_batch_next(struct eb_batch *batch) +{ + if (batch->cur >= batch->nr) + return NULL; + return batch->ebs[batch->cur++]; +} + +static inline void eb_batch_release(struct eb_batch *batch) +{ + for (unsigned int i = 0; i < batch->nr; i++) + free_extent_buffer(batch->ebs[i]); + eb_batch_init(batch); +} + +static inline struct extent_buffer *find_get_eb(struct xa_state *xas, unsigned long max, + xa_mark_t mark) +{ + struct extent_buffer *eb; + +retry: + eb = xas_find_marked(xas, max, mark); + + if (xas_retry(xas, eb)) + goto retry; + + if (!eb) + return NULL; + + if (!refcount_inc_not_zero(&eb->refs)) { + xas_reset(xas); + goto retry; + } + + if (unlikely(eb != xas_reload(xas))) { + free_extent_buffer(eb); + xas_reset(xas); + goto retry; + } + + return eb; +} + +static unsigned int buffer_tree_get_ebs_tag(struct btrfs_fs_info *fs_info, + unsigned long *start, + unsigned long end, xa_mark_t tag, + struct eb_batch *batch) +{ + XA_STATE(xas, &fs_info->buffer_tree, *start); + struct extent_buffer *eb; + + rcu_read_lock(); + while ((eb = find_get_eb(&xas, end, tag)) != NULL) { + if (!eb_batch_add(batch, eb)) { + *start = ((eb->start + eb->len) >> fs_info->nodesize_bits); + goto out; + } + } + if (end == ULONG_MAX) + *start = ULONG_MAX; + else + *start = end + 1; +out: + rcu_read_unlock(); + + return batch->nr; +} + +/* + * The endio specific version which won't touch any unsafe spinlock in endio + * context. + */ +static struct extent_buffer *find_extent_buffer_nolock( + struct btrfs_fs_info *fs_info, u64 start) +{ + struct extent_buffer *eb; + unsigned long index = (start >> fs_info->nodesize_bits); + + rcu_read_lock(); + eb = xa_load(&fs_info->buffer_tree, index); + if (eb && !refcount_inc_not_zero(&eb->refs)) + eb = NULL; + rcu_read_unlock(); + return eb; +} + +static void end_bbio_meta_write(struct btrfs_bio *bbio) +{ + struct extent_buffer *eb = bbio->private; + + if (bbio->bio.bi_status != BLK_STS_OK) + set_btree_ioerr(eb); + + /* + * Clear writeback on the buffer's own folios. The bio may carry the + * shared zero page instead (EXTENT_BUFFER_ZONED_ZEROOUT), so iterate + * the extent buffer folios rather than the bio folios. + */ + for (int i = 0; i < num_extent_folios(eb); i++) + btrfs_meta_folio_clear_writeback(eb->folios[i], eb); + + buffer_tree_clear_mark(eb, PAGECACHE_TAG_WRITEBACK); + clear_and_wake_up_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags); + bio_put(&bbio->bio); +} + +static void prepare_eb_write(struct extent_buffer *eb) +{ + u32 nritems; + unsigned long start; + unsigned long end; + + clear_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags); + + /* Set btree blocks beyond nritems with 0 to avoid stale content */ + nritems = btrfs_header_nritems(eb); + if (btrfs_header_level(eb) > 0) { + end = btrfs_node_key_ptr_offset(eb, nritems); + memzero_extent_buffer(eb, end, eb->len - end); + } else { + /* + * Leaf: + * header 0 1 2 .. N ... data_N .. data_2 data_1 data_0 + */ + start = btrfs_item_nr_offset(eb, nritems); + end = btrfs_item_nr_offset(eb, 0); + if (nritems == 0) + end += BTRFS_LEAF_DATA_SIZE(eb->fs_info); + else + end += btrfs_item_offset(eb, nritems - 1); + memzero_extent_buffer(eb, start, end - start); + } +} + +static noinline_for_stack void write_one_eb(struct extent_buffer *eb, + struct writeback_control *wbc) +{ + struct btrfs_fs_info *fs_info = eb->fs_info; + struct btrfs_bio *bbio; + + if (!test_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags)) + prepare_eb_write(eb); + + bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES, + REQ_OP_WRITE | REQ_META | wbc_to_write_flags(wbc), + BTRFS_I(fs_info->btree_inode), eb->start, + end_bbio_meta_write, eb); + bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT; + bio_set_dev(&bbio->bio, fs_info->fs_devices->latest_dev->bdev); + wbc_init_bio(wbc, &bbio->bio); + for (int i = 0; i < num_extent_folios(eb); i++) { + struct folio *folio = eb->folios[i]; + u64 range_start = max_t(u64, eb->start, folio_pos(folio)); + u32 range_len = min_t(u64, folio_next_pos(folio), + eb->start + eb->len) - range_start; + + folio_lock(folio); + btrfs_meta_folio_clear_dirty(folio, eb); + btrfs_meta_folio_set_writeback(folio, eb); + if (!folio_test_dirty(folio)) + wbc->nr_to_write -= folio_nr_pages(folio); + if (test_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags)) { + u32 off = 0; + + while (off < range_len) { + u32 add = min_t(u32, PAGE_SIZE, range_len - off); + + bio_add_folio_nofail(&bbio->bio, + page_folio(ZERO_PAGE(0)), + add, 0); + off += add; + } + } else { + bio_add_folio_nofail(&bbio->bio, folio, range_len, + offset_in_folio(folio, range_start)); + } + wbc_account_cgroup_owner(wbc, folio, range_len); + folio_unlock(folio); + } + /* + * If the fs is already in error status, do not submit any writeback + * but immediately finish it. + */ + if (unlikely(BTRFS_FS_ERROR(fs_info))) { + btrfs_bio_end_io(bbio, errno_to_blk_status(BTRFS_FS_ERROR(fs_info))); + return; + } + btrfs_submit_bbio(bbio, 0); +} + +/* + * Wait for all eb writeback in the given range to finish. + * + * @fs_info: The fs_info for this file system. + * @start: The offset of the range to start waiting on writeback. + * @end: The end of the range, inclusive. This is meant to be used in + * conjunction with wait_marked_extents, so this will usually be + * the_next_eb->start - 1. + */ +void btrfs_btree_wait_writeback_range(struct btrfs_fs_info *fs_info, u64 start, + u64 end) +{ + struct eb_batch batch; + unsigned long start_index = (start >> fs_info->nodesize_bits); + unsigned long end_index = (end >> fs_info->nodesize_bits); + + eb_batch_init(&batch); + while (start_index <= end_index) { + struct extent_buffer *eb; + unsigned int nr_ebs; + + nr_ebs = buffer_tree_get_ebs_tag(fs_info, &start_index, end_index, + PAGECACHE_TAG_WRITEBACK, &batch); + if (!nr_ebs) + break; + + while ((eb = eb_batch_next(&batch)) != NULL) + wait_on_extent_buffer_writeback(eb); + eb_batch_release(&batch); + cond_resched(); + } +} + +static int write_meta_extent_buffer(struct btrfs_eb_write_context *ctx, + struct writeback_control *wbc) +{ + struct extent_buffer *eb = ctx->eb; + int ret; + + ret = btrfs_check_meta_write_pointer(eb->fs_info, ctx); + if (ret) + return ret; + + if (!lock_extent_buffer_for_io(eb, wbc)) + return 0; + + /* Implies write in zoned mode. */ + if (ctx->zoned_bg) { + /* Mark the last eb in the block group. */ + btrfs_schedule_zone_finish_bg(ctx->zoned_bg, eb); + ctx->zoned_bg->meta_write_pointer += eb->len; + } + write_one_eb(eb, wbc); + return 0; +} + +/* + * On a zoned filesystem, write out the currently dirty metadata extent buffers + * of @bg. Used to flush the active metadata/system block group before the + * ascending-address walk in btree_writepages(), so that walk can pivot the + * active block group away (finishing it) instead of aborting the commit; see + * the caller for details. + */ +static void flush_active_meta_bg(struct address_space *mapping, + struct writeback_control *wbc, + struct btrfs_eb_write_context *ctx, + struct btrfs_block_group *bg) +{ + struct btrfs_fs_info *fs_info = inode_to_fs_info(mapping->host); + unsigned long index = bg->start >> fs_info->nodesize_bits; + unsigned long end = (btrfs_block_group_end(bg) - 1) >> fs_info->nodesize_bits; + struct eb_batch batch; + unsigned int nr_ebs; + + ASSERT(btrfs_is_zoned(fs_info)); + lockdep_assert_held(&fs_info->zoned_meta_io_lock); + + eb_batch_init(&batch); + while (index <= end && + (nr_ebs = buffer_tree_get_ebs_tag(fs_info, &index, end, + PAGECACHE_TAG_DIRTY, &batch))) { + struct extent_buffer *eb; + + while ((eb = eb_batch_next(&batch)) != NULL) { + ctx->eb = eb; + + /* + * If the eb is behind the write pointer (-EBUSY, e.g. + * already being written by someone else) skip it and + * carry on. Only a hole at the write pointer (-EAGAIN) + * stops the flush. The main walk in btree_writepages() + * then deals with it. + */ + if (write_meta_extent_buffer(ctx, wbc) == -EAGAIN) { + eb_batch_release(&batch); + return; + } + } + eb_batch_release(&batch); + cond_resched(); + } +} + +int btree_writepages(struct address_space *mapping, struct writeback_control *wbc) +{ + struct btrfs_eb_write_context ctx = { .wbc = wbc }; + struct btrfs_fs_info *fs_info = inode_to_fs_info(mapping->host); + int ret = 0; + bool done = false; + int nr_to_write_done = 0; + struct eb_batch batch; + unsigned int nr_ebs; + unsigned long index; + unsigned long end; + bool scanned = false; + xa_mark_t tag; + + eb_batch_init(&batch); + if (wbc->range_cyclic) { + index = ((mapping->writeback_index << PAGE_SHIFT) >> fs_info->nodesize_bits); + end = -1; + + /* + * Start from the beginning does not need to cycle over the + * range, mark it as scanned. + */ + scanned = (index == 0); + } else { + index = (wbc->range_start >> fs_info->nodesize_bits); + end = (wbc->range_end >> fs_info->nodesize_bits); + + scanned = true; + } + if (wbc->sync_mode == WB_SYNC_ALL) + tag = PAGECACHE_TAG_TOWRITE; + else + tag = PAGECACHE_TAG_DIRTY; + btrfs_zoned_meta_io_lock(fs_info); + + /* + * On a zoned filesystem, flush the currently active metadata/system + * block group(s) first, under this same lock, so the ascending-address + * walk below can pivot the active block group instead of aborting the + * transaction commit with -EAGAIN. + */ + if (btrfs_is_zoned(fs_info) && wbc->sync_mode == WB_SYNC_ALL && + !wbc->for_sync) { + if (fs_info->active_meta_bg) + flush_active_meta_bg(mapping, wbc, &ctx, + fs_info->active_meta_bg); + if (fs_info->active_system_bg) + flush_active_meta_bg(mapping, wbc, &ctx, + fs_info->active_system_bg); + } +retry: + if (wbc->sync_mode == WB_SYNC_ALL) + buffer_tree_tag_for_writeback(fs_info, index, end); + while (!done && !nr_to_write_done && (index <= end) && + (nr_ebs = buffer_tree_get_ebs_tag(fs_info, &index, end, tag, &batch))) { + struct extent_buffer *eb; + + while ((eb = eb_batch_next(&batch)) != NULL) { + ctx.eb = eb; + + ret = write_meta_extent_buffer(&ctx, wbc); + if (ret == -EBUSY) { + ret = 0; + } else if (ret) { + done = true; + break; + } + } + nr_to_write_done = (wbc->nr_to_write <= 0); + eb_batch_release(&batch); + cond_resched(); + } + if (!scanned && !done) { + /* + * We hit the last page and there is more work to be done: wrap + * back to the start of the file + */ + scanned = true; + index = 0; + goto retry; + } + + /* + * Only btrfs_check_meta_write_pointer() can update @ret, + * and it only returns 0 or errors. + */ + ASSERT(ret <= 0); + if (unlikely(!ret && BTRFS_FS_ERROR(fs_info))) + ret = -EROFS; + + if (ctx.zoned_bg) + btrfs_put_block_group(ctx.zoned_bg); + btrfs_zoned_meta_io_unlock(fs_info); + return ret; +} + +/* + * Walk the list of dirty pages of the given address space and write all of them. + * + * @mapping: address space structure to write + * @wbc: subtract the number of written pages from *@wbc->nr_to_write + * @bio_ctrl: holds context for the write, namely the bio + * + * If a page is already under I/O, write_cache_pages() skips it, even + * if it's dirty. This is desirable behaviour for memory-cleaning writeback, + * but it is INCORRECT for data-integrity system calls such as fsync(). fsync() + * and msync() need to guarantee that all the data which was dirty at the time + * the call was made get new I/O started against them. If wbc->sync_mode is + * WB_SYNC_ALL then we were called for data integrity and we must wait for + * existing IO to complete. + */ +static int extent_write_cache_pages(struct address_space *mapping, + struct btrfs_bio_ctrl *bio_ctrl) +{ + struct writeback_control *wbc = bio_ctrl->wbc; + struct inode *inode = mapping->host; + int ret = 0; + bool done = false; + int nr_to_write_done = 0; + struct folio_batch fbatch; + unsigned int nr_folios; + pgoff_t index; + pgoff_t end; /* Inclusive */ + pgoff_t done_index; + bool range_whole = false; + bool scanned = false; + xa_mark_t tag; + + /* + * We have to hold onto the inode so that ordered extents can do their + * work when the IO finishes. The alternative to this is failing to add + * an ordered extent if the igrab() fails there and that is a huge pain + * to deal with, so instead just hold onto the inode throughout the + * writepages operation. If it fails here we are freeing up the inode + * anyway and we'd rather not waste our time writing out stuff that is + * going to be truncated anyway. + */ + if (!igrab(inode)) + return 0; + + folio_batch_init(&fbatch); + if (wbc->range_cyclic) { + index = mapping->writeback_index; /* Start from prev offset */ + end = -1; + /* + * Start from the beginning does not need to cycle over the + * range, mark it as scanned. + */ + scanned = (index == 0); + } else { + index = wbc->range_start >> PAGE_SHIFT; + end = wbc->range_end >> PAGE_SHIFT; + if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) + range_whole = true; + scanned = true; + } + + /* + * We do the tagged writepage as long as the snapshot flush bit is set + * and we are the first one who do the filemap_flush() on this inode. + * + * The nr_to_write == LONG_MAX is needed to make sure other flushers do + * not race in and drop the bit. + */ + if (range_whole && wbc->nr_to_write == LONG_MAX && + test_and_clear_bit(BTRFS_INODE_SNAPSHOT_FLUSH, + &BTRFS_I(inode)->runtime_flags)) + wbc->tagged_writepages = 1; + + tag = wbc_to_tag(wbc); +retry: + if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) + tag_pages_for_writeback(mapping, index, end); + done_index = index; + while (!done && !nr_to_write_done && (index <= end) && + (nr_folios = filemap_get_folios_tag(mapping, &index, + end, tag, &fbatch))) { + unsigned i; + + for (i = 0; i < nr_folios; i++) { + struct folio *folio = fbatch.folios[i]; + + done_index = folio_next_index(folio); + /* + * At this point we hold neither the i_pages lock nor + * the folio lock: the folio may be truncated or + * invalidated (changing folio->mapping to NULL). + */ + if (!folio_trylock(folio)) { + submit_write_bio(bio_ctrl, 0); + folio_lock(folio); + } + + if (unlikely(folio->mapping != mapping)) { + folio_unlock(folio); + continue; + } + + if (!folio_test_dirty(folio)) { + /* Someone wrote it for us. */ + folio_unlock(folio); + continue; + } + + /* + * For subpage case, compression can lead to mixed + * writeback and dirty flags, e.g: + * 0 32K 64K 96K 128K + * | |//////||/////| |//| + * + * In above case, [32K, 96K) is asynchronously submitted + * for compression, and [124K, 128K) needs to be written back. + * + * If we didn't wait writeback for page 64K, [128K, 128K) + * won't be submitted as the page still has writeback flag + * and will be skipped in the next check. + * + * This mixed writeback and dirty case is only possible for + * subpage case. + * + * TODO: Remove this check after migrating compression to + * regular submission. + */ + if (wbc->sync_mode != WB_SYNC_NONE || + btrfs_is_subpage(inode_to_fs_info(inode), folio)) { + if (folio_test_writeback(folio)) + submit_write_bio(bio_ctrl, 0); + folio_wait_writeback(folio); + } + + if (folio_test_writeback(folio) || + !folio_test_dirty(folio)) { + folio_unlock(folio); + continue; + } + + /* Locked for writeback; revoke writable mmap PTEs before using the data. */ + folio_mkclean(folio); + ret = extent_writepage(folio, bio_ctrl); + if (ret < 0) { + done = true; + break; + } + + /* + * The filesystem may choose to bump up nr_to_write. + * We have to make sure to honor the new nr_to_write + * at any time. + */ + nr_to_write_done = (wbc->sync_mode == WB_SYNC_NONE && + wbc->nr_to_write <= 0); + } + folio_batch_release(&fbatch); + cond_resched(); + } + if (!scanned && !done) { + /* + * We hit the last page and there is more work to be done: wrap + * back to the start of the file + */ + scanned = true; + index = 0; + + /* + * If we're looping we could run into a page that is locked by a + * writer and that writer could be waiting on writeback for a + * page in our current bio, and thus deadlock, so flush the + * write bio here. + */ + submit_write_bio(bio_ctrl, 0); + goto retry; + } + + if (wbc->range_cyclic || (wbc->nr_to_write > 0 && range_whole)) + mapping->writeback_index = done_index; + + btrfs_add_delayed_iput(BTRFS_I(inode)); + return ret; +} + +/* + * Submit the pages in the range to bio for call sites which delalloc range has + * already been ran (aka, ordered extent inserted) and all pages are still + * locked. + */ +void extent_write_locked_range(struct inode *inode, const struct folio *locked_folio, + u64 start, u64 end, struct writeback_control *wbc, + bool pages_dirty) +{ + bool found_error = false; + int ret = 0; + struct address_space *mapping = inode->i_mapping; + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + const u32 sectorsize = fs_info->sectorsize; + loff_t i_size = i_size_read(inode); + u64 cur = start; + struct btrfs_bio_ctrl bio_ctrl = { + .wbc = wbc, + .opf = REQ_OP_WRITE | wbc_to_write_flags(wbc), + }; + + if (wbc->no_cgroup_owner) + bio_ctrl.opf |= REQ_BTRFS_CGROUP_PUNT; + + ASSERT(IS_ALIGNED(start, sectorsize) && IS_ALIGNED(end + 1, sectorsize)); + + while (cur <= end) { + u64 cur_end; + u32 cur_len; + struct folio *folio; + + folio = filemap_get_folio(mapping, cur >> PAGE_SHIFT); + + /* + * This shouldn't happen, the pages are pinned and locked, this + * code is just in case, but shouldn't actually be run. + */ + if (IS_ERR(folio)) { + cur_end = min(round_down(cur, PAGE_SIZE) + PAGE_SIZE - 1, end); + cur_len = cur_end + 1 - cur; + btrfs_mark_ordered_io_finished(BTRFS_I(inode), cur, cur_len, false); + mapping_set_error(mapping, PTR_ERR(folio)); + cur = cur_end; + continue; + } + + cur_end = min_t(u64, folio_next_pos(folio) - 1, end); + cur_len = cur_end + 1 - cur; + + ASSERT(folio_test_locked(folio)); + if (pages_dirty && folio != locked_folio) + ASSERT(folio_test_dirty(folio)); + + /* + * Set the submission bitmap to submit all sectors. + * extent_writepage_io() will do the truncation correctly. + */ + bio_ctrl_init_submit_bitmap(fs_info, folio, &bio_ctrl); + ret = extent_writepage_io(BTRFS_I(inode), folio, cur, cur_len, + &bio_ctrl, i_size); + if (ret == 1) + goto next_page; + + if (ret) + mapping_set_error(mapping, ret); + btrfs_folio_end_lock(fs_info, folio, cur, cur_len); + if (ret < 0) + found_error = true; +next_page: + folio_put(folio); + cur = cur_end + 1; + } + + submit_write_bio(&bio_ctrl, found_error ? ret : 0); +} + +int btrfs_writepages(struct address_space *mapping, struct writeback_control *wbc) +{ + struct inode *inode = mapping->host; + int ret = 0; + struct btrfs_bio_ctrl bio_ctrl = { + .wbc = wbc, + .opf = REQ_OP_WRITE | wbc_to_write_flags(wbc), + }; + + /* + * Allow only a single thread to do the reloc work in zoned mode to + * protect the write pointer updates. + */ + btrfs_zoned_data_reloc_lock(BTRFS_I(inode)); + ret = extent_write_cache_pages(mapping, &bio_ctrl); + submit_write_bio(&bio_ctrl, ret); + btrfs_zoned_data_reloc_unlock(BTRFS_I(inode)); + return ret; +} + +void btrfs_readahead(struct readahead_control *rac) +{ + struct btrfs_bio_ctrl bio_ctrl = { + .opf = REQ_OP_READ | REQ_RAHEAD, + .ractl = rac, + .last_em_start = U64_MAX, + }; + struct folio *folio; + struct inode *vfs_inode = rac->mapping->host; + struct btrfs_inode *inode = BTRFS_I(vfs_inode); + const u64 start = readahead_pos(rac); + const u64 end = start + readahead_length(rac) - 1; + struct extent_state *cached_state = NULL; + struct extent_map *em_cached = NULL; + struct fsverity_info *vi = NULL; + + assert_folio_range(inode, start, end); + lock_extents_for_read(inode, start, end, &cached_state); + /* We don't use cached state for a bulk unlock, just free it. */ + btrfs_free_extent_state(cached_state); + if (start < i_size_read(vfs_inode)) + vi = fsverity_get_info(vfs_inode); + while ((folio = readahead_folio(rac)) != NULL) { + /* + * Read start and end before btrfs_do_readpage(). It unlocks the + * folio, so our reference might not be valid after. + */ + const u64 folio_start = folio_pos(folio); + const u64 folio_end = folio_start + folio_size(folio) - 1; + + btrfs_do_readpage(folio, &em_cached, &bio_ctrl, vi); + /* Only unlock the range we locked, even if readahead expands. */ + if (folio_start >= start && folio_end <= end) + btrfs_unlock_extent(&inode->io_tree, folio_start, folio_end, NULL); + } + + if (em_cached) + btrfs_free_extent_map(em_cached); + submit_one_bio(&bio_ctrl); +} + +/* + * A helper for struct address_space_operations::release_folio, this tests for + * areas of the folio that are locked or under IO and drops the related state + * bits if it is safe to drop the folio. + */ +static bool try_release_extent_state(struct extent_io_tree *tree, + struct folio *folio) +{ + struct extent_state *cached_state = NULL; + u64 start = folio_pos(folio); + u64 end = start + folio_size(folio) - 1; + u32 range_bits; + u32 clear_bits; + bool ret = false; + int ret2; + + btrfs_get_range_bits(tree, start, end, &range_bits, &cached_state); + + /* + * We can release the folio if it's locked only for ordered extent + * completion, since that doesn't require using the folio. + */ + if ((range_bits & EXTENT_LOCKED) && + !(range_bits & EXTENT_FINISHING_ORDERED)) + goto out; + + clear_bits = ~(EXTENT_LOCKED | EXTENT_NODATASUM | EXTENT_DELALLOC_NEW | + EXTENT_CTLBITS | EXTENT_QGROUP_RESERVED | + EXTENT_FINISHING_ORDERED); + /* + * At this point we can safely clear everything except the locked, + * nodatasum, delalloc new and finishing ordered bits. The delalloc new + * bit will be cleared by ordered extent completion. + */ + ret2 = btrfs_clear_extent_bit(tree, start, end, clear_bits, &cached_state); + /* + * If clear_extent_bit failed for enomem reasons, we can't allow the + * release to continue. + */ + if (ret2 == 0) + ret = true; +out: + btrfs_free_extent_state(cached_state); + + return ret; +} + +/* + * a helper for release_folio. As long as there are no locked extents + * in the range corresponding to the page, both state records and extent + * map records are removed + */ +bool try_release_extent_mapping(struct folio *folio, gfp_t mask) +{ + u64 start = folio_pos(folio); + u64 end = start + folio_size(folio) - 1; + struct btrfs_inode *inode = folio_to_inode(folio); + struct extent_io_tree *io_tree = &inode->io_tree; + + while (start <= end) { + const u64 cur_gen = btrfs_get_fs_generation(inode->root->fs_info); + const u64 len = end - start + 1; + struct extent_map_tree *extent_tree = &inode->extent_tree; + struct extent_map *em; + + write_lock(&extent_tree->lock); + em = btrfs_lookup_extent_mapping(extent_tree, start, len); + if (!em) { + write_unlock(&extent_tree->lock); + break; + } + if ((em->flags & EXTENT_FLAG_PINNED) || em->start != start) { + write_unlock(&extent_tree->lock); + btrfs_free_extent_map(em); + break; + } + if (btrfs_test_range_bit_exists(io_tree, em->start, + btrfs_extent_map_end(em) - 1, + EXTENT_LOCKED)) + goto next; + /* + * If it's not in the list of modified extents, used by a fast + * fsync, we can remove it. If it's being logged we can safely + * remove it since fsync took an extra reference on the em. + */ + if (list_empty(&em->list) || (em->flags & EXTENT_FLAG_LOGGING)) + goto remove_em; + /* + * If it's in the list of modified extents, remove it only if + * its generation is older then the current one, in which case + * we don't need it for a fast fsync. Otherwise don't remove it, + * we could be racing with an ongoing fast fsync that could miss + * the new extent. + */ + if (em->generation >= cur_gen) + goto next; +remove_em: + /* + * We only remove extent maps that are not in the list of + * modified extents or that are in the list but with a + * generation lower then the current generation, so there is no + * need to set the full fsync flag on the inode (it hurts the + * fsync performance for workloads with a data size that exceeds + * or is close to the system's memory). + */ + btrfs_remove_extent_mapping(inode, em); + /* Once for the inode's extent map tree. */ + btrfs_free_extent_map(em); +next: + start = btrfs_extent_map_end(em); + write_unlock(&extent_tree->lock); + + /* Once for us, for the lookup_extent_mapping() reference. */ + btrfs_free_extent_map(em); + + if (need_resched()) { + /* + * If we need to resched but we can't block just exit + * and leave any remaining extent maps. + */ + if (!gfpflags_allow_blocking(mask)) + break; + + cond_resched(); + } + } + return try_release_extent_state(io_tree, folio); +} + +static bool folio_range_has_eb(struct folio *folio) +{ + struct btrfs_folio_state *bfs; + + lockdep_assert_held(&folio->mapping->i_private_lock); + + if (folio_test_private(folio)) { + bfs = folio_get_private(folio); + if (atomic_read(&bfs->eb_refs)) + return true; + } + return false; +} + +static void detach_extent_buffer_folio(const struct extent_buffer *eb, struct folio *folio) +{ + struct btrfs_fs_info *fs_info = eb->fs_info; + struct address_space *mapping = folio->mapping; + const bool mapped = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags); + + /* + * For mapped eb, we're going to change the folio private, which should + * be done under the i_private_lock. + */ + if (mapped) + spin_lock(&mapping->i_private_lock); + + if (!folio_test_private(folio)) { + if (mapped) + spin_unlock(&mapping->i_private_lock); + return; + } + + if (!btrfs_meta_is_subpage(fs_info)) { + /* + * We do this since we'll remove the pages after we've removed + * the eb from the xarray, so we could race and have this page + * now attached to the new eb. So only clear folio if it's + * still connected to this eb. + */ + if (folio_test_private(folio) && folio_get_private(folio) == eb) { + BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)); + BUG_ON(folio_test_dirty(folio)); + BUG_ON(folio_test_writeback(folio)); + /* We need to make sure we haven't be attached to a new eb. */ + folio_detach_private(folio); + } + if (mapped) + spin_unlock(&mapping->i_private_lock); + return; + } + + /* + * For subpage, we can have dummy eb with folio private attached. In + * this case, we can directly detach the private as such folio is only + * attached to one dummy eb, no sharing. + */ + if (!mapped) { + btrfs_detach_folio_state(fs_info, folio, BTRFS_SUBPAGE_METADATA); + return; + } + + btrfs_folio_dec_eb_refs(fs_info, folio); + + /* + * We can only detach the folio private if there are no other ebs in the + * page range and no unfinished IO. + */ + if (!folio_range_has_eb(folio)) + btrfs_detach_folio_state(fs_info, folio, BTRFS_SUBPAGE_METADATA); + + spin_unlock(&mapping->i_private_lock); +} + +/* Release all folios attached to the extent buffer */ +static void btrfs_release_extent_buffer_folios(const struct extent_buffer *eb) +{ + ASSERT(!extent_buffer_under_io(eb)); + + for (int i = 0; i < INLINE_EXTENT_BUFFER_PAGES; i++) { + struct folio *folio = eb->folios[i]; + + if (!folio) + continue; + + detach_extent_buffer_folio(eb, folio); + } +} + +/* + * Helper for releasing the extent buffer. + */ +static inline void btrfs_release_extent_buffer(struct extent_buffer *eb) +{ + btrfs_release_extent_buffer_folios(eb); + btrfs_leak_debug_del_eb(eb); + kmem_cache_free(extent_buffer_cache, eb); +} + +/* + * Claim a slot to track an extent buffer in, evicting the coldest tracked buffer + * when the array is full. + * + * Slots fill in order until the array is full. After that a CLOCK (second + * chance) scan advances the hand, clearing one reference bit per step, until + * it lands on an unreferenced slot whose buffer is evicted. Clearing a bit per + * step bounds the scan to BTRFS_INHIBITED_EBS_SLOTS iterations. + */ +static int btrfs_inhibit_claim_slot(struct btrfs_trans_handle *trans) +{ + int slot; + + if (trans->nr_inhibited_ebs < BTRFS_INHIBITED_EBS_SLOTS) + return trans->nr_inhibited_ebs++; + + while (trans->inhibited_ebs_referenced & (1U << trans->inhibited_ebs_hand)) { + trans->inhibited_ebs_referenced &= ~(1U << trans->inhibited_ebs_hand); + trans->inhibited_ebs_hand = + (trans->inhibited_ebs_hand + 1) % BTRFS_INHIBITED_EBS_SLOTS; + } + slot = trans->inhibited_ebs_hand; + trans->inhibited_ebs_hand = (trans->inhibited_ebs_hand + 1) % BTRFS_INHIBITED_EBS_SLOTS; + + atomic_dec(&trans->inhibited_ebs[slot]->writeback_inhibitors); + free_extent_buffer(trans->inhibited_ebs[slot]); + + return slot; +} + +/* + * Inhibit writeback on buffer during transaction. + * + * @trans: transaction handle that will own the inhibitor + * @eb: extent buffer to inhibit writeback on + * + * Attempt to track this extent buffer in the transaction's inhibited set. When + * the set is full the coldest tracked buffer is evicted instead. An untracked + * buffer may be written back and need re-COW, which is the original behavior. + * This is acceptable since inhibiting writeback is an optimization. + */ +void btrfs_inhibit_eb_writeback(struct btrfs_trans_handle *trans, struct extent_buffer *eb) +{ + int slot; + + lockdep_assert_held(&eb->lock); + + /* Already tracked: set its reference bit (second chance) and return. */ + for (int i = 0; i < trans->nr_inhibited_ebs; i++) { + if (trans->inhibited_ebs[i] == eb) { + trans->inhibited_ebs_referenced |= 1U << i; + return; + } + } + + slot = btrfs_inhibit_claim_slot(trans); + + /* + * Pin the eb while the array holds a raw pointer to it; the counter is + * what lock_extent_buffer_for_io() checks. + */ + refcount_inc(&eb->refs); + atomic_inc(&eb->writeback_inhibitors); + trans->inhibited_ebs[slot] = eb; + trans->inhibited_ebs_referenced |= 1U << slot; +} + +/* + * Uninhibit writeback on all extent buffers. + */ +void btrfs_uninhibit_all_eb_writeback(struct btrfs_trans_handle *trans) +{ + for (int i = 0; i < trans->nr_inhibited_ebs; i++) { + atomic_dec(&trans->inhibited_ebs[i]->writeback_inhibitors); + free_extent_buffer(trans->inhibited_ebs[i]); + } + trans->nr_inhibited_ebs = 0; + trans->inhibited_ebs_referenced = 0; + trans->inhibited_ebs_hand = 0; +} + +static void init_extent_buffer(struct btrfs_fs_info *fs_info, + struct extent_buffer *eb, u64 start) +{ + eb->start = start; + eb->len = fs_info->nodesize; + eb->fs_info = fs_info; + init_rwsem(&eb->lock); + atomic_set(&eb->writeback_inhibitors, 0); + + btrfs_leak_debug_add_eb(eb); + + spin_lock_init(&eb->refs_lock); + refcount_set(&eb->refs, 1); + + ASSERT(eb->len <= BTRFS_MAX_METADATA_BLOCKSIZE); +} + +static struct extent_buffer *__alloc_extent_buffer(struct btrfs_fs_info *fs_info, + u64 start) +{ + struct extent_buffer *eb; + + eb = kmem_cache_zalloc(extent_buffer_cache, GFP_NOFS | __GFP_NOFAIL); + init_extent_buffer(fs_info, eb, start); + return eb; +} + +/* + * For use in eb allocation error cleanup paths, as btrfs_release_extent_buffer() + * does not call folio_put(), and we need to set the folios to NULL so that + * btrfs_release_extent_buffer() will not detach them a second time. + */ +static void cleanup_extent_buffer_folios(struct extent_buffer *eb) +{ + const int num_folios = num_extent_folios(eb); + + /* We cannot use num_extent_folios() as loop bound as eb->folios changes. */ + for (int i = 0; i < num_folios; i++) { + ASSERT(eb->folios[i]); + detach_extent_buffer_folio(eb, eb->folios[i]); + folio_put(eb->folios[i]); + eb->folios[i] = NULL; + } +} + +struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src) +{ + struct extent_buffer *new; + int num_folios; + int ret; + + new = __alloc_extent_buffer(src->fs_info, src->start); + if (new == NULL) + return NULL; + + /* + * Set UNMAPPED before calling btrfs_release_extent_buffer(), as + * btrfs_release_extent_buffer() have different behavior for + * UNMAPPED subpage extent buffer. + */ + set_bit(EXTENT_BUFFER_UNMAPPED, &new->bflags); + + ret = alloc_eb_folio_array(new, GFP_NOFS); + if (ret) + goto release_eb; + + ASSERT(num_extent_folios(src) == num_extent_folios(new), + "%d != %d", num_extent_folios(src), num_extent_folios(new)); + /* Explicitly use the cached num_extent value from now on. */ + num_folios = num_extent_folios(src); + for (int i = 0; i < num_folios; i++) { + struct folio *folio = new->folios[i]; + + ret = attach_extent_buffer_folio(new, folio, NULL); + if (ret < 0) + goto cleanup_folios; + WARN_ON(folio_test_dirty(folio)); + } + for (int i = 0; i < num_folios; i++) + folio_put(new->folios[i]); + + copy_extent_buffer_full(new, src); + set_extent_buffer_uptodate(new); + + return new; + +cleanup_folios: + cleanup_extent_buffer_folios(new); +release_eb: + btrfs_release_extent_buffer(new); + return NULL; +} + +struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info, + u64 start) +{ + struct extent_buffer *eb; + int ret; + + eb = __alloc_extent_buffer(fs_info, start); + if (!eb) + return NULL; + + ret = alloc_eb_folio_array(eb, GFP_NOFS); + if (ret) + goto release_eb; + + for (int i = 0; i < num_extent_folios(eb); i++) { + ret = attach_extent_buffer_folio(eb, eb->folios[i], NULL); + if (ret < 0) + goto cleanup_folios; + } + for (int i = 0; i < num_extent_folios(eb); i++) + folio_put(eb->folios[i]); + + set_extent_buffer_uptodate(eb); + btrfs_set_header_nritems(eb, 0); + set_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags); + + return eb; + +cleanup_folios: + cleanup_extent_buffer_folios(eb); +release_eb: + btrfs_release_extent_buffer(eb); + return NULL; +} + +static void check_buffer_tree_ref(struct extent_buffer *eb) +{ + int refs; + /* + * The TREE_REF bit is first set when the extent_buffer is added to the + * xarray. It is also reset, if unset, when a new reference is created + * by find_extent_buffer. + * + * It is only cleared in two cases: freeing the last non-tree + * reference to the extent_buffer when its STALE bit is set or + * calling release_folio when the tree reference is the only reference. + * + * In both cases, care is taken to ensure that the extent_buffer's + * pages are not under io. However, release_folio can be concurrently + * called with creating new references, which is prone to race + * conditions between the calls to check_buffer_tree_ref in those + * codepaths and clearing TREE_REF in try_release_extent_buffer. + * + * The actual lifetime of the extent_buffer in the xarray is adequately + * protected by the refcount, but the TREE_REF bit and its corresponding + * reference are not. To protect against this class of races, we call + * check_buffer_tree_ref() from the code paths which trigger io. Note that + * once io is initiated, TREE_REF can no longer be cleared, so that is + * the moment at which any such race is best fixed. + */ + refs = refcount_read(&eb->refs); + if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) + return; + + spin_lock(&eb->refs_lock); + if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) + refcount_inc(&eb->refs); + spin_unlock(&eb->refs_lock); +} + +static void mark_extent_buffer_accessed(struct extent_buffer *eb) +{ + check_buffer_tree_ref(eb); + + for (int i = 0; i < num_extent_folios(eb); i++) + folio_mark_accessed(eb->folios[i]); +} + +struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info, + u64 start) +{ + struct extent_buffer *eb; + + eb = find_extent_buffer_nolock(fs_info, start); + if (!eb) + return NULL; + /* + * Lock our eb's refs_lock to avoid races with free_extent_buffer(). + * When we get our eb it might be flagged with EXTENT_BUFFER_STALE and + * another task running free_extent_buffer() might have seen that flag + * set, eb->refs == 2, that the buffer isn't under IO (dirty and + * writeback flags not set) and it's still in the tree (flag + * EXTENT_BUFFER_TREE_REF set), therefore being in the process of + * decrementing the extent buffer's reference count twice. So here we + * could race and increment the eb's reference count, clear its stale + * flag, mark it as dirty and drop our reference before the other task + * finishes executing free_extent_buffer, which would later result in + * an attempt to free an extent buffer that is dirty. + */ + if (test_bit(EXTENT_BUFFER_STALE, &eb->bflags)) { + spin_lock(&eb->refs_lock); + spin_unlock(&eb->refs_lock); + } + mark_extent_buffer_accessed(eb); + return eb; +} + +struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info, + u64 start) +{ +#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS + struct extent_buffer *eb, *exists = NULL; + int ret; + + eb = find_extent_buffer(fs_info, start); + if (eb) + return eb; + eb = alloc_dummy_extent_buffer(fs_info, start); + if (!eb) + return ERR_PTR(-ENOMEM); + eb->fs_info = fs_info; +again: + xa_lock_irq(&fs_info->buffer_tree); + exists = __xa_cmpxchg(&fs_info->buffer_tree, start >> fs_info->nodesize_bits, + NULL, eb, GFP_NOFS); + if (xa_is_err(exists)) { + ret = xa_err(exists); + xa_unlock_irq(&fs_info->buffer_tree); + btrfs_release_extent_buffer(eb); + return ERR_PTR(ret); + } + if (exists) { + if (!refcount_inc_not_zero(&exists->refs)) { + /* The extent buffer is being freed, retry. */ + xa_unlock_irq(&fs_info->buffer_tree); + goto again; + } + xa_unlock_irq(&fs_info->buffer_tree); + btrfs_release_extent_buffer(eb); + return exists; + } + xa_unlock_irq(&fs_info->buffer_tree); + check_buffer_tree_ref(eb); + + return eb; +#else + /* Stub to avoid linker error when compiled with optimizations turned off. */ + return NULL; +#endif +} + +static struct extent_buffer *grab_extent_buffer(struct btrfs_fs_info *fs_info, + struct folio *folio) +{ + struct extent_buffer *exists; + + lockdep_assert_held(&folio->mapping->i_private_lock); + + /* + * For subpage case, we completely rely on xarray to ensure we don't try + * to insert two ebs for the same bytenr. So here we always return NULL + * and just continue. + */ + if (btrfs_meta_is_subpage(fs_info)) + return NULL; + + /* Page not yet attached to an extent buffer */ + if (!folio_test_private(folio)) + return NULL; + + /* + * We could have already allocated an eb for this folio and attached one + * so lets see if we can get a ref on the existing eb, and if we can we + * know it's good and we can just return that one, else we know we can + * just overwrite folio private. + */ + exists = folio_get_private(folio); + if (refcount_inc_not_zero(&exists->refs)) + return exists; + + WARN_ON(folio_test_dirty(folio)); + folio_detach_private(folio); + return NULL; +} + +/* + * Validate alignment constraints of eb at logical address @start. + */ +static bool check_eb_alignment(struct btrfs_fs_info *fs_info, u64 start) +{ + const u32 nodesize = fs_info->nodesize; + + if (unlikely(!IS_ALIGNED(start, fs_info->sectorsize))) { + btrfs_err(fs_info, "bad tree block start %llu", start); + return true; + } + + if (unlikely(nodesize < PAGE_SIZE && !IS_ALIGNED(start, nodesize))) { + btrfs_err(fs_info, + "tree block is not nodesize aligned, start %llu nodesize %u", + start, nodesize); + return true; + } + if (unlikely(nodesize >= PAGE_SIZE && !PAGE_ALIGNED(start))) { + btrfs_err(fs_info, + "tree block is not page aligned, start %llu nodesize %u", + start, nodesize); + return true; + } + if (unlikely(!IS_ALIGNED(start, nodesize) && + !test_and_set_bit(BTRFS_FS_UNALIGNED_TREE_BLOCK, &fs_info->flags))) { + btrfs_warn(fs_info, +"tree block not nodesize aligned, start %llu nodesize %u, can be resolved by a full metadata balance", + start, nodesize); + } + return false; +} + +/* + * Return 0 if eb->folios[i] is attached to btree inode successfully. + * Return >0 if there is already another extent buffer for the range, + * and @found_eb_ret would be updated. + * Return -EAGAIN if the filemap has an existing folio but with different size + * than @eb. + * The caller needs to free the existing folios and retry using the same order. + */ +static int attach_eb_folio_to_filemap(struct extent_buffer *eb, int i, + struct btrfs_eb_prealloc *pa, + struct extent_buffer **found_eb_ret) +{ + + struct btrfs_fs_info *fs_info = eb->fs_info; + struct address_space *mapping = fs_info->btree_inode->i_mapping; + const pgoff_t index = eb->start >> PAGE_SHIFT; + struct folio *existing_folio; + int ret; + + ASSERT(found_eb_ret); + + /* Caller should ensure the folio exists. */ + ASSERT(eb->folios[i]); + +retry: + existing_folio = NULL; + ret = filemap_add_folio(mapping, eb->folios[i], index + i, + GFP_NOFS | __GFP_NOFAIL); + if (!ret) + goto finish; + + /* ret == -EEXIST: a folio already lives at this index. */ + existing_folio = filemap_lock_folio(mapping, index + i); + /* The page cache only exists for a very short time, just retry. */ + if (IS_ERR(existing_folio)) + goto retry; + + /* For now, we should only have single-page folios for btree inode. */ + ASSERT(folio_nr_pages(existing_folio) == 1); + + /* + * TODO: Special handling for a corner case where the order of + * folios mismatch between the new eb and filemap. + * + * This happens when: + * + * - the new eb is using higher order folio + * + * - the filemap is still using 0-order folios for the range + * This can happen at the previous eb allocation, and we don't + * have higher order folio for the call. + * + * - the existing eb has already been freed + * + * In this case, we have to free the existing folios first, and + * re-allocate using the same order. + * Thankfully this is not going to happen yet, as we're still + * using 0-order folios. + */ + if (folio_size(existing_folio) != eb->folio_size) { + DEBUG_WARN("folio order mismatch between new eb and filemap"); + folio_unlock(existing_folio); + folio_put(existing_folio); + return -EAGAIN; + } + +finish: + spin_lock(&mapping->i_private_lock); + if (existing_folio && btrfs_meta_is_subpage(fs_info)) { + /* We're going to reuse the existing folio, can drop our folio now. */ + folio_put(eb->folios[i]); + eb->folios[i] = existing_folio; + } else if (existing_folio) { + struct extent_buffer *existing_eb; + + existing_eb = grab_extent_buffer(fs_info, existing_folio); + if (existing_eb) { + /* The extent buffer still exists, we can use it directly. */ + *found_eb_ret = existing_eb; + spin_unlock(&mapping->i_private_lock); + folio_unlock(existing_folio); + folio_put(existing_folio); + return 1; + } + /* The extent buffer no longer exists, we can reuse the folio. */ + folio_put(eb->folios[i]); + eb->folios[i] = existing_folio; + } + eb->folio_size = folio_size(eb->folios[i]); + eb->folio_shift = folio_shift(eb->folios[i]); + /* Should not fail, as we have preallocated the memory. */ + ret = attach_extent_buffer_folio(eb, eb->folios[i], pa->bfs); + ASSERT(!ret); + /* The subpage state, if any, is now attached to the folio or freed. */ + pa->bfs = NULL; + /* + * To inform we have an extra eb under allocation, so that + * detach_extent_buffer_page() won't release the folio private when the + * eb hasn't been inserted into the xarray yet. + * + * The ref will be decreased when the eb releases the page, in + * detach_extent_buffer_page(). Thus needs no special handling in the + * error path. + */ + btrfs_folio_inc_eb_refs(fs_info, eb->folios[i]); + spin_unlock(&mapping->i_private_lock); + return 0; +} + +/* + * Allocate the extent_buffer, its folios, and btrfs_folio_state, if needed. + * + * @pa: The holder struct to do the allocation in. + * @nowait: Whether to do a speculative GFP_NOWAIT allocation while holding locks. + * + * Return 0 on success and a negative errno otherwise. On failure, pa->eb/bfs + * will be NULL. If @nowait=true, then on ENOMEM, mark @pa->needs_prealloc and + * return -EAGAIN to signal the caller to unlock and retry. + */ +int btrfs_init_eb_prealloc(struct btrfs_fs_info *fs_info, + struct btrfs_eb_prealloc *pa, bool nowait) +{ + gfp_t gfp = nowait ? GFP_NOWAIT : GFP_NOFS | __GFP_NOFAIL; + int ret; + + ASSERT(!pa->eb, "unexpected non-null eb: %p", pa->eb); + ASSERT(!pa->bfs, "unexpected non-null bfs: %p", pa->bfs); + pa->needs_prealloc = false; + + pa->eb = kmem_cache_zalloc(extent_buffer_cache, gfp); + if (!pa->eb) { + ret = -ENOMEM; + goto out; + } + /* alloc_eb_folio_array() needs len; init_extent_buffer() sets it again later. */ + pa->eb->len = fs_info->nodesize; + + /* + * Preallocate folio private for subpage case, so that we won't + * allocate memory with i_private_lock nor page lock hold. + * + * The memory will be freed by attach_extent_buffer_page() or freed + * manually if we exit earlier. + */ + if (btrfs_meta_is_subpage(fs_info)) { + pa->bfs = btrfs_alloc_folio_state(fs_info, PAGE_SIZE, + BTRFS_SUBPAGE_METADATA, gfp); + if (IS_ERR(pa->bfs)) { + ret = PTR_ERR(pa->bfs); + pa->bfs = NULL; + goto free_eb; + } + } + + /* + * Allocate pages without attaching them. Caller is ultimately responsible + * for attaching the folios to the mapping with attach_eb_folio_to_filemap(). + */ + ret = alloc_eb_folio_array(pa->eb, gfp | __GFP_MOVABLE); + if (ret < 0) + goto free_bfs; + + return 0; + +free_bfs: + btrfs_free_folio_state(pa->bfs); + pa->bfs = NULL; +free_eb: + kmem_cache_free(extent_buffer_cache, pa->eb); + pa->eb = NULL; +out: + if (nowait && ret == -ENOMEM) { + pa->needs_prealloc = true; + ret = -EAGAIN; + } + return ret; +} + +/* + * Used to cleanup a btrfs_eb_prealloc which had its contents allocated but + * folios not yet attached and eb/bfs consumed, and refs still 0. + * + * Safe to call on a fully used btrfs_eb_prealloc as the internal structs will + * be null once they are owned by the context using them. + */ +void btrfs_free_eb_prealloc(struct btrfs_eb_prealloc *pa) +{ + if (!pa->eb) + return; + + for (int i = 0; i < num_extent_pages(pa->eb); i++) { + if (pa->eb->folios[i]) + folio_put(pa->eb->folios[i]); + } + btrfs_free_folio_state(pa->bfs); + kmem_cache_free(extent_buffer_cache, pa->eb); + pa->eb = NULL; + pa->bfs = NULL; +} + +struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info, + struct btrfs_eb_prealloc *pa, + u64 start, u64 owner_root, int level) +{ + int attached = 0; + struct extent_buffer *eb; + struct extent_buffer *existing_eb = NULL; + u64 lockdep_owner = owner_root; + bool page_contig = true; + bool uptodate = true; + int ret; + + if (check_eb_alignment(fs_info, start)) + return ERR_PTR(-EINVAL); + +#if BITS_PER_LONG == 32 + if (start >= MAX_LFS_FILESIZE) { + btrfs_err_rl(fs_info, + "extent buffer %llu is beyond 32bit page cache limit", start); + btrfs_err_32bit_limit(fs_info); + return ERR_PTR(-EOVERFLOW); + } + if (start >= BTRFS_32BIT_EARLY_WARN_THRESHOLD) + btrfs_warn_32bit_limit(fs_info); +#endif + + eb = find_extent_buffer(fs_info, start); + if (eb) + return eb; + + if (!pa->eb) { + ret = btrfs_init_eb_prealloc(fs_info, pa, pa->supports_nowait); + if (ret) + return ERR_PTR(ret); + } + eb = pa->eb; + pa->eb = NULL; + init_extent_buffer(fs_info, eb, start); + + /* + * The reloc trees are just snapshots, so we need them to appear to be + * just like any other fs tree WRT lockdep. + */ + if (lockdep_owner == BTRFS_TREE_RELOC_OBJECTID) + lockdep_owner = BTRFS_FS_TREE_OBJECTID; + + btrfs_set_buffer_lockdep_class(lockdep_owner, eb, level); + + /* Attach all pages to the filemap. */ + for (int i = 0; i < num_extent_folios(eb); i++) { + struct folio *folio; + + ret = attach_eb_folio_to_filemap(eb, i, pa, &existing_eb); + if (ret > 0) { + ASSERT(existing_eb); + goto out; + } + /* -EAGAIN: folio order mismatch, unreachable with 0-order folios. */ + if (ret < 0) + goto out; + attached++; + + /* + * Only after attach_eb_folio_to_filemap(), eb->folios[] is + * reliable, as we may choose to reuse the existing page cache + * and free the allocated page. + */ + folio = eb->folios[i]; + WARN_ON(btrfs_meta_folio_test_dirty(folio, eb)); + + /* + * Check if the current page is physically contiguous with previous eb + * page. + * At this stage, either we allocated a large folio, thus @i + * would only be 0, or we fall back to per-page allocation. + */ + if (i && folio_page(eb->folios[i - 1], 0) + 1 != folio_page(folio, 0)) + page_contig = false; + + if (!btrfs_meta_folio_test_uptodate(folio, eb)) + uptodate = false; + + /* + * We can't unlock the pages just yet since the extent buffer + * hasn't been properly inserted into the xarray, this opens a + * race with btree_release_folio() which can free a page while we + * are still filling in all pages for the buffer and we could crash. + */ + } + if (uptodate) + set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); + /* All pages are physically contiguous, can skip cross page handling. */ + if (page_contig) + eb->addr = folio_address(eb->folios[0]) + offset_in_page(eb->start); +again: + xa_lock_irq(&fs_info->buffer_tree); + existing_eb = __xa_cmpxchg(&fs_info->buffer_tree, + start >> fs_info->nodesize_bits, NULL, eb, + GFP_NOFS); + if (xa_is_err(existing_eb)) { + ret = xa_err(existing_eb); + xa_unlock_irq(&fs_info->buffer_tree); + goto out; + } + if (existing_eb) { + if (!refcount_inc_not_zero(&existing_eb->refs)) { + xa_unlock_irq(&fs_info->buffer_tree); + goto again; + } + xa_unlock_irq(&fs_info->buffer_tree); + goto out; + } + xa_unlock_irq(&fs_info->buffer_tree); + + /* add one reference for the tree */ + check_buffer_tree_ref(eb); + + /* + * Now it's safe to unlock the pages because any calls to + * btree_release_folio will correctly detect that a page belongs to a + * live buffer and won't free them prematurely. + */ + for (int i = 0; i < num_extent_folios(eb); i++) { + folio_unlock(eb->folios[i]); + /* + * A folio that has been added to an address_space mapping + * should not continue holding the refcount from its original + * allocation indefinitely. + */ + folio_put(eb->folios[i]); + } + return eb; + +out: + WARN_ON(!refcount_dec_and_test(&eb->refs)); + + /* Attach hands off pa->bfs; free it if we bailed first. */ + btrfs_free_folio_state(pa->bfs); + pa->bfs = NULL; + + /* + * Any attached folios need to be detached before we unlock them. This + * is because when we're inserting our new folios into the mapping, and + * then attaching our eb to that folio. If we fail to insert our folio + * we'll lookup the folio for that index, and grab that EB. We do not + * want that to grab this eb, as we're getting ready to free it. So we + * have to detach it first and then unlock it. + * + * Note: the bounds is num_extent_pages() as we need to go through all slots. + */ + for (int i = 0; i < num_extent_pages(eb); i++) { + struct folio *folio = eb->folios[i]; + + if (i < attached) { + ASSERT(folio); + detach_extent_buffer_folio(eb, folio); + folio_unlock(folio); + } else if (!folio) { + continue; + } + + folio_put(folio); + eb->folios[i] = NULL; + } + btrfs_release_extent_buffer(eb); + if (ret < 0) + return ERR_PTR(ret); + ASSERT(existing_eb); + return existing_eb; +} + +static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head) +{ + struct extent_buffer *eb = + container_of(head, struct extent_buffer, rcu_head); + + kmem_cache_free(extent_buffer_cache, eb); +} + +static int release_extent_buffer(struct extent_buffer *eb) + __releases(&eb->refs_lock) +{ + lockdep_assert_held(&eb->refs_lock); + + if (refcount_dec_and_test(&eb->refs)) { + struct btrfs_fs_info *fs_info = eb->fs_info; + + spin_unlock(&eb->refs_lock); + + /* + * We're erasing, theoretically there will be no allocations, so + * just use GFP_ATOMIC. + * + * We use cmpxchg instead of erase because we do not know if + * this eb is actually in the tree or not, we could be cleaning + * up an eb that we allocated but never inserted into the tree. + * Thus use cmpxchg to remove it from the tree if it is there, + * or leave the other entry if this isn't in the tree. + * + * The documentation says that putting a NULL value is the same + * as erase as long as XA_FLAGS_ALLOC is not set, which it isn't + * in this case. + */ + xa_cmpxchg_irq(&fs_info->buffer_tree, + eb->start >> fs_info->nodesize_bits, eb, NULL, + GFP_ATOMIC); + + btrfs_leak_debug_del_eb(eb); + /* Should be safe to release folios at this point. */ + btrfs_release_extent_buffer_folios(eb); +#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS + if (unlikely(test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags))) { + kmem_cache_free(extent_buffer_cache, eb); + return 1; + } +#endif + call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu); + return 1; + } + spin_unlock(&eb->refs_lock); + + return 0; +} + +static void clear_extent_buffer_reading(struct extent_buffer *eb) +{ + clear_and_wake_up_bit(EXTENT_BUFFER_READING, &eb->bflags); +} + +static void free_extent_buffer_clear_reading(struct extent_buffer *eb, + bool clear_reading) +{ + int refs; + + if (!eb) + return; + + /* + * We want to clear EXTENT_BUFFER_READING flag and decrease refs + * in the same critical section. + * This will make sure invalidate_and_check_btree_folios() won't + * see an eb with EXTENT_BUFFER_READING cleared but refs not yet + * decreased. + */ + if (clear_reading) { + spin_lock(&eb->refs_lock); + clear_extent_buffer_reading(eb); + } + + refs = refcount_read(&eb->refs); + while (1) { + if (test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags)) { + if (refs == 1) + break; + } else if (refs <= 3) { + break; + } + + /* Optimization to avoid locking eb->refs_lock. */ + if (atomic_try_cmpxchg(&eb->refs.refs, &refs, refs - 1)) { + if (clear_reading) + spin_unlock(&eb->refs_lock); + return; + } + } + + if (!clear_reading) + spin_lock(&eb->refs_lock); + + if (refcount_read(&eb->refs) == 2 && + test_bit(EXTENT_BUFFER_STALE, &eb->bflags) && + !extent_buffer_under_io(eb) && + test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) + refcount_dec(&eb->refs); + + /* + * I know this is terrible, but it's temporary until we stop tracking + * the uptodate bits and such for the extent buffers. + */ + release_extent_buffer(eb); +} + +void free_extent_buffer(struct extent_buffer *eb) +{ + return free_extent_buffer_clear_reading(eb, false); +} + +void free_extent_buffer_stale(struct extent_buffer *eb) +{ + if (!eb) + return; + + spin_lock(&eb->refs_lock); + set_bit(EXTENT_BUFFER_STALE, &eb->bflags); + + if (refcount_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) && + test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) + refcount_dec(&eb->refs); + release_extent_buffer(eb); +} + +static void clear_extent_buffer_dirty(struct extent_buffer *eb) +{ + struct btrfs_fs_info *fs_info = eb->fs_info; + + if (!test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) + return; + + buffer_tree_clear_mark(eb, PAGECACHE_TAG_DIRTY); + percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, -(s64)eb->len, + fs_info->dirty_metadata_batch); + + for (int i = 0; i < num_extent_folios(eb); i++) { + struct folio *folio = eb->folios[i]; + bool last; + + if (!folio_test_dirty(folio)) + continue; + folio_lock(folio); + last = btrfs_meta_folio_clear_and_test_dirty(folio, eb); + if (last) + btrfs_clear_folio_dirty_tag(folio); + folio_unlock(folio); + } + WARN_ON(refcount_read(&eb->refs) == 0); +} + +void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans, + struct extent_buffer *eb) +{ + struct btrfs_fs_info *fs_info = eb->fs_info; + + btrfs_assert_tree_write_locked(eb); + + if (trans && btrfs_header_generation(eb) != trans->transid) + return; + + /* + * Instead of clearing the dirty flag off of the buffer, mark it as + * EXTENT_BUFFER_ZONED_ZEROOUT. This allows us to preserve + * write-ordering in zoned mode, without the need to later re-dirty + * the extent_buffer. + * + * The actual zeroout of the buffer will happen later in + * btree_csum_one_bio. + */ + if (btrfs_is_zoned(fs_info) && test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) { + set_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags); + return; + } + + clear_extent_buffer_dirty(eb); +} + +/* + * On a zoned filesystem a freed tree block is kept dirty and flagged as + * EXTENT_BUFFER_ZONED_ZEROOUT so a later writeback zeroes it out and advances + * the zone write pointer. Such buffers still dirty when the filesystem is torn + * down can no longer be written back and are stale; if left dirty they hang the + * final iput() of the btree inode. Drop their dirty state, and the deferred + * zero-out along with it. + */ +void btrfs_zoned_release_dirty_metadata(struct btrfs_fs_info *fs_info) +{ + struct eb_batch batch; + unsigned long index = 0; + + if (!btrfs_is_zoned(fs_info)) + return; + + btrfs_zoned_meta_io_lock(fs_info); + eb_batch_init(&batch); + while (buffer_tree_get_ebs_tag(fs_info, &index, ULONG_MAX, + PAGECACHE_TAG_DIRTY, &batch)) { + struct extent_buffer *eb; + + while ((eb = eb_batch_next(&batch)) != NULL) { + btrfs_tree_lock(eb); + if (test_and_clear_bit(EXTENT_BUFFER_ZONED_ZEROOUT, + &eb->bflags)) + clear_extent_buffer_dirty(eb); + btrfs_tree_unlock(eb); + } + eb_batch_release(&batch); + cond_resched(); + } + btrfs_zoned_meta_io_unlock(fs_info); +} + +void set_extent_buffer_dirty(struct extent_buffer *eb) +{ + bool was_dirty; + + check_buffer_tree_ref(eb); + + was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags); + + WARN_ON(refcount_read(&eb->refs) == 0); + WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)); + WARN_ON(test_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags)); + + if (!was_dirty) { + bool subpage = btrfs_meta_is_subpage(eb->fs_info); + + /* + * For subpage case, we can have other extent buffers in the + * same page, and in clear_extent_buffer_dirty() we + * have to clear page dirty without subpage lock held. + * This can cause race where our page gets dirty cleared after + * we just set it. + * + * Thankfully, clear_extent_buffer_dirty() has locked + * its page for other reasons, we can use page lock to prevent + * the above race. + */ + if (subpage) + folio_lock(eb->folios[0]); + for (int i = 0; i < num_extent_folios(eb); i++) + btrfs_meta_folio_set_dirty(eb->folios[i], eb); + buffer_tree_set_mark(eb, PAGECACHE_TAG_DIRTY); + if (subpage) + folio_unlock(eb->folios[0]); + percpu_counter_add_batch(&eb->fs_info->dirty_metadata_bytes, + eb->len, + eb->fs_info->dirty_metadata_batch); + } +#ifdef CONFIG_BTRFS_DEBUG + for (int i = 0; i < num_extent_folios(eb); i++) + ASSERT(folio_test_dirty(eb->folios[i])); +#endif +} + +void clear_extent_buffer_uptodate(struct extent_buffer *eb) +{ + + clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); + for (int i = 0; i < num_extent_folios(eb); i++) { + struct folio *folio = eb->folios[i]; + + if (!folio) + continue; + + btrfs_meta_folio_clear_uptodate(folio, eb); + } +} + +void set_extent_buffer_uptodate(struct extent_buffer *eb) +{ + + set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); + for (int i = 0; i < num_extent_folios(eb); i++) + btrfs_meta_folio_set_uptodate(eb->folios[i], eb); +} + +static void end_bbio_meta_read(struct btrfs_bio *bbio) +{ + struct extent_buffer *eb = bbio->private; + bool uptodate = !bbio->bio.bi_status; + + /* + * If the extent buffer is marked UPTODATE before the read operation + * completes, other calls to read_extent_buffer_pages() will return + * early without waiting for the read to finish, causing data races. + */ + WARN_ON(test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags)); + + eb->read_mirror = bbio->mirror_num; + + if (uptodate && + btrfs_validate_extent_buffer(eb, &bbio->parent_check) < 0) + uptodate = false; + + if (uptodate) + set_extent_buffer_uptodate(eb); + else + clear_extent_buffer_uptodate(eb); + + free_extent_buffer_clear_reading(eb, true); + + bio_put(&bbio->bio); +} + +int read_extent_buffer_pages_nowait(struct extent_buffer *eb, int mirror_num, + const struct btrfs_tree_parent_check *check) +{ + struct btrfs_fs_info *fs_info = eb->fs_info; + struct btrfs_bio *bbio; + + if (extent_buffer_uptodate(eb)) { + int ret; + + ret = btrfs_buffer_uptodate(eb, 0, check); + if (unlikely(ret <= 0)) { + if (ret == 0) + ret = -EIO; + return ret; + } + return 0; + } + + /* + * We could have had EXTENT_BUFFER_UPTODATE cleared by the write + * operation, which could potentially still be in flight. In this case + * we simply want to return an error. + */ + if (unlikely(test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags))) + return -EIO; + + /* Someone else is already reading the buffer, just wait for it. */ + if (test_and_set_bit(EXTENT_BUFFER_READING, &eb->bflags)) + return 0; + + /* + * Between the initial test_bit(EXTENT_BUFFER_UPTODATE) and the above + * test_and_set_bit(EXTENT_BUFFER_READING), someone else could have + * started and finished reading the same eb. In this case, UPTODATE + * will now be set, and we shouldn't read it in again. + */ + if (unlikely(extent_buffer_uptodate(eb))) { + int ret; + + clear_extent_buffer_reading(eb); + ret = btrfs_buffer_uptodate(eb, 0, check); + if (unlikely(ret <= 0)) { + if (ret == 0) + ret = -EIO; + return ret; + } + return 0; + } + + eb->read_mirror = 0; + check_buffer_tree_ref(eb); + refcount_inc(&eb->refs); + + bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES, + REQ_OP_READ | REQ_META, BTRFS_I(fs_info->btree_inode), + eb->start, end_bbio_meta_read, eb); + bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT; + memcpy(&bbio->parent_check, check, sizeof(*check)); + for (int i = 0; i < num_extent_folios(eb); i++) { + struct folio *folio = eb->folios[i]; + u64 range_start = max_t(u64, eb->start, folio_pos(folio)); + u32 range_len = min_t(u64, folio_next_pos(folio), + eb->start + eb->len) - range_start; + + bio_add_folio_nofail(&bbio->bio, folio, range_len, + offset_in_folio(folio, range_start)); + } + btrfs_submit_bbio(bbio, mirror_num); + return 0; +} + +int read_extent_buffer_pages(struct extent_buffer *eb, int mirror_num, + const struct btrfs_tree_parent_check *check) +{ + int ret; + + ret = read_extent_buffer_pages_nowait(eb, mirror_num, check); + if (ret < 0) + return ret; + + wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_READING, TASK_UNINTERRUPTIBLE); + if (unlikely(!extent_buffer_uptodate(eb))) + return -EIO; + return 0; +} + +/* Never inlined to decrease code size, as this is called in a cold path. */ +static noinline void report_eb_range(const struct extent_buffer *eb, + unsigned long start, unsigned long len) +{ + btrfs_warn(eb->fs_info, + "access to eb bytenr %llu len %u out of range start %lu len %lu", + eb->start, eb->len, start, len); + DEBUG_WARN(); +} + +/* + * Check if the [start, start + len) range is valid before reading/writing + * the eb. + * NOTE: @start and @len are offset inside the eb, not logical address. + * + * Caller should not touch the dst/src memory if this function returns error. + */ +static inline bool check_eb_range(const struct extent_buffer *eb, + unsigned long start, unsigned long len) +{ + unsigned long offset; + + /* start, start + len should not go beyond eb->len nor overflow */ + if (unlikely(check_add_overflow(start, len, &offset) || offset > eb->len)) { + report_eb_range(eb, start, len); + return true; + } + + return false; +} + +void read_extent_buffer(const struct extent_buffer *eb, void *dstv, + unsigned long start, unsigned long len) +{ + const int unit_size = eb->folio_size; + size_t cur; + size_t offset; + char *dst = (char *)dstv; + unsigned long i; + + if (check_eb_range(eb, start, len)) { + /* + * Invalid range hit, reset the memory, so callers won't get + * some random garbage for their uninitialized memory. + */ + memset(dstv, 0, len); + return; + } + + if (eb->addr) { + memcpy(dstv, eb->addr + start, len); + return; + } + + offset = get_eb_offset_in_folio(eb, start); + i = get_eb_folio_index(eb, start); + while (len > 0) { + char *kaddr; + + cur = min(len, unit_size - offset); + kaddr = folio_address(eb->folios[i]); + memcpy(dst, kaddr + offset, cur); + + dst += cur; + len -= cur; + offset = 0; + i++; + } +} + +int read_extent_buffer_to_user_nofault(const struct extent_buffer *eb, + void __user *dstv, + unsigned long start, unsigned long len) +{ + const int unit_size = eb->folio_size; + size_t cur; + size_t offset; + char __user *dst = (char __user *)dstv; + unsigned long i; + int ret = 0; + + if (check_eb_range(eb, start, len)) + return -EINVAL; + + if (eb->addr) { + if (copy_to_user_nofault(dstv, eb->addr + start, len)) + ret = -EFAULT; + return ret; + } + + offset = get_eb_offset_in_folio(eb, start); + i = get_eb_folio_index(eb, start); + while (len > 0) { + char *kaddr; + + cur = min(len, unit_size - offset); + kaddr = folio_address(eb->folios[i]); + if (copy_to_user_nofault(dst, kaddr + offset, cur)) { + ret = -EFAULT; + break; + } + + dst += cur; + len -= cur; + offset = 0; + i++; + } + + return ret; +} + +int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv, + unsigned long start, unsigned long len) +{ + const int unit_size = eb->folio_size; + size_t cur; + size_t offset; + char *kaddr; + char *ptr = (char *)ptrv; + unsigned long i; + int ret = 0; + + if (check_eb_range(eb, start, len)) + return -EINVAL; + + if (eb->addr) + return memcmp(ptrv, eb->addr + start, len); + + offset = get_eb_offset_in_folio(eb, start); + i = get_eb_folio_index(eb, start); + while (len > 0) { + cur = min(len, unit_size - offset); + kaddr = folio_address(eb->folios[i]); + ret = memcmp(ptr, kaddr + offset, cur); + if (ret) + break; + + ptr += cur; + len -= cur; + offset = 0; + i++; + } + return ret; +} + +/* + * Check that the extent buffer is uptodate. + * + * For regular sector size == PAGE_SIZE case, check if @page is uptodate. + * For subpage case, check if the range covered by the eb has EXTENT_UPTODATE. + */ +static void assert_eb_folio_uptodate(const struct extent_buffer *eb, int i) +{ + struct btrfs_fs_info *fs_info = eb->fs_info; + struct folio *folio = eb->folios[i]; + + ASSERT(folio); + + /* + * If we are using the commit root we could potentially clear a page + * Uptodate while we're using the extent buffer that we've previously + * looked up. We don't want to complain in this case, as the page was + * valid before, we just didn't write it out. Instead we want to catch + * the case where we didn't actually read the block properly, which + * would have !PageUptodate and !EXTENT_BUFFER_WRITE_ERR. + */ + if (test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)) + return; + + if (btrfs_meta_is_subpage(fs_info)) { + folio = eb->folios[0]; + ASSERT(i == 0); + if (WARN_ON(!btrfs_subpage_test_uptodate(fs_info, folio, + eb->start, eb->len))) + btrfs_subpage_dump_bitmap(fs_info, folio, eb->start, eb->len); + } else { + WARN_ON(!folio_test_uptodate(folio)); + } +} + +static void __write_extent_buffer(const struct extent_buffer *eb, + const void *srcv, unsigned long start, + unsigned long len, bool use_memmove) +{ + const int unit_size = eb->folio_size; + size_t cur; + size_t offset; + char *kaddr; + const char *src = (const char *)srcv; + unsigned long i; + /* For unmapped (dummy) ebs, no need to check their uptodate status. */ + const bool check_uptodate = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags); + + if (check_eb_range(eb, start, len)) + return; + + if (eb->addr) { + if (use_memmove) + memmove(eb->addr + start, srcv, len); + else + memcpy(eb->addr + start, srcv, len); + return; + } + + offset = get_eb_offset_in_folio(eb, start); + i = get_eb_folio_index(eb, start); + while (len > 0) { + if (check_uptodate) + assert_eb_folio_uptodate(eb, i); + + cur = min(len, unit_size - offset); + kaddr = folio_address(eb->folios[i]); + if (use_memmove) + memmove(kaddr + offset, src, cur); + else + memcpy(kaddr + offset, src, cur); + + src += cur; + len -= cur; + offset = 0; + i++; + } +} + +void write_extent_buffer(const struct extent_buffer *eb, const void *srcv, + unsigned long start, unsigned long len) +{ + return __write_extent_buffer(eb, srcv, start, len, false); +} + +static void memset_extent_buffer(const struct extent_buffer *eb, int c, + unsigned long start, unsigned long len) +{ + const int unit_size = eb->folio_size; + unsigned long cur = start; + + if (eb->addr) { + memset(eb->addr + start, c, len); + return; + } + + while (cur < start + len) { + unsigned long index = get_eb_folio_index(eb, cur); + unsigned int offset = get_eb_offset_in_folio(eb, cur); + unsigned int cur_len = min(start + len - cur, unit_size - offset); + + assert_eb_folio_uptodate(eb, index); + memset(folio_address(eb->folios[index]) + offset, c, cur_len); + + cur += cur_len; + } +} + +void memzero_extent_buffer(const struct extent_buffer *eb, unsigned long start, + unsigned long len) +{ + if (check_eb_range(eb, start, len)) + return; + return memset_extent_buffer(eb, 0, start, len); +} + +void copy_extent_buffer_full(const struct extent_buffer *dst, + const struct extent_buffer *src) +{ + const int unit_size = src->folio_size; + unsigned long cur = 0; + + ASSERT(dst->len == src->len); + + while (cur < src->len) { + unsigned long index = get_eb_folio_index(src, cur); + unsigned long offset = get_eb_offset_in_folio(src, cur); + unsigned long cur_len = min(src->len, unit_size - offset); + void *addr = folio_address(src->folios[index]) + offset; + + write_extent_buffer(dst, addr, cur, cur_len); + + cur += cur_len; + } +} + +void copy_extent_buffer(const struct extent_buffer *dst, + const struct extent_buffer *src, + unsigned long dst_offset, unsigned long src_offset, + unsigned long len) +{ + const int unit_size = dst->folio_size; + u64 dst_len = dst->len; + size_t cur; + size_t offset; + char *kaddr; + unsigned long i; + + if (check_eb_range(dst, dst_offset, len) || + check_eb_range(src, src_offset, len)) + return; + + WARN_ON(src->len != dst_len); + + offset = get_eb_offset_in_folio(dst, dst_offset); + + i = get_eb_folio_index(dst, dst_offset); + while (len > 0) { + assert_eb_folio_uptodate(dst, i); + + cur = min(len, (unsigned long)(unit_size - offset)); + + kaddr = folio_address(dst->folios[i]); + read_extent_buffer(src, kaddr + offset, src_offset, cur); + + src_offset += cur; + len -= cur; + offset = 0; + i++; + } +} + +/* + * Calculate the folio and offset of the byte containing the given bit number. + * + * @eb: the extent buffer + * @start: offset of the bitmap item in the extent buffer + * @nr: bit number + * @folio_index: return index of the folio in the extent buffer that contains + * the given bit number + * @folio_offset: return offset into the folio given by folio_index + * + * This helper hides the ugliness of finding the byte in an extent buffer which + * contains a given bit. + */ +static inline void eb_bitmap_offset(const struct extent_buffer *eb, + unsigned long start, unsigned long nr, + unsigned long *folio_index, + size_t *folio_offset) +{ + size_t byte_offset = BIT_BYTE(nr); + size_t offset; + + /* + * The byte we want is the offset of the extent buffer + the offset of + * the bitmap item in the extent buffer + the offset of the byte in the + * bitmap item. + */ + offset = start + offset_in_eb_folio(eb, eb->start) + byte_offset; + + *folio_index = offset >> eb->folio_shift; + *folio_offset = offset_in_eb_folio(eb, offset); +} + +/* + * Determine whether a bit in a bitmap item is set. + * + * @eb: the extent buffer + * @start: offset of the bitmap item in the extent buffer + * @nr: bit number to test + */ +bool extent_buffer_test_bit(const struct extent_buffer *eb, unsigned long start, + unsigned long nr) +{ + unsigned long i; + size_t offset; + u8 *kaddr; + + eb_bitmap_offset(eb, start, nr, &i, &offset); + assert_eb_folio_uptodate(eb, i); + kaddr = folio_address(eb->folios[i]); + return 1U & (kaddr[offset] >> (nr & (BITS_PER_BYTE - 1))); +} + +static u8 *extent_buffer_get_byte(const struct extent_buffer *eb, unsigned long bytenr) +{ + unsigned long index = get_eb_folio_index(eb, bytenr); + + if (check_eb_range(eb, bytenr, 1)) + return NULL; + return folio_address(eb->folios[index]) + get_eb_offset_in_folio(eb, bytenr); +} + +/* + * Set an area of a bitmap to 1. + * + * @eb: the extent buffer + * @start: offset of the bitmap item in the extent buffer + * @pos: bit number of the first bit + * @len: number of bits to set + */ +void extent_buffer_bitmap_set(const struct extent_buffer *eb, unsigned long start, + unsigned long pos, unsigned long len) +{ + unsigned int first_byte = start + BIT_BYTE(pos); + unsigned int last_byte = start + BIT_BYTE(pos + len - 1); + const bool same_byte = (first_byte == last_byte); + u8 mask = BITMAP_FIRST_BYTE_MASK(pos); + u8 *kaddr; + + if (same_byte) + mask &= BITMAP_LAST_BYTE_MASK(pos + len); + + /* Handle the first byte. */ + kaddr = extent_buffer_get_byte(eb, first_byte); + *kaddr |= mask; + if (same_byte) + return; + + /* Handle the byte aligned part. */ + ASSERT(first_byte + 1 <= last_byte); + memset_extent_buffer(eb, 0xff, first_byte + 1, last_byte - first_byte - 1); + + /* Handle the last byte. */ + kaddr = extent_buffer_get_byte(eb, last_byte); + *kaddr |= BITMAP_LAST_BYTE_MASK(pos + len); +} + + +/* + * Clear an area of a bitmap. + * + * @eb: the extent buffer + * @start: offset of the bitmap item in the extent buffer + * @pos: bit number of the first bit + * @len: number of bits to clear + */ +void extent_buffer_bitmap_clear(const struct extent_buffer *eb, + unsigned long start, unsigned long pos, + unsigned long len) +{ + unsigned int first_byte = start + BIT_BYTE(pos); + unsigned int last_byte = start + BIT_BYTE(pos + len - 1); + const bool same_byte = (first_byte == last_byte); + u8 mask = BITMAP_FIRST_BYTE_MASK(pos); + u8 *kaddr; + + if (same_byte) + mask &= BITMAP_LAST_BYTE_MASK(pos + len); + + /* Handle the first byte. */ + kaddr = extent_buffer_get_byte(eb, first_byte); + *kaddr &= ~mask; + if (same_byte) + return; + + /* Handle the byte aligned part. */ + ASSERT(first_byte + 1 <= last_byte); + memset_extent_buffer(eb, 0, first_byte + 1, last_byte - first_byte - 1); + + /* Handle the last byte. */ + kaddr = extent_buffer_get_byte(eb, last_byte); + *kaddr &= ~BITMAP_LAST_BYTE_MASK(pos + len); +} + +static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len) +{ + unsigned long distance = (src > dst) ? src - dst : dst - src; + return distance < len; +} + +void memcpy_extent_buffer(const struct extent_buffer *dst, + unsigned long dst_offset, unsigned long src_offset, + unsigned long len) +{ + const int unit_size = dst->folio_size; + unsigned long cur_off = 0; + + if (check_eb_range(dst, dst_offset, len) || + check_eb_range(dst, src_offset, len)) + return; + + if (dst->addr) { + const bool use_memmove = areas_overlap(src_offset, dst_offset, len); + + if (use_memmove) + memmove(dst->addr + dst_offset, dst->addr + src_offset, len); + else + memcpy(dst->addr + dst_offset, dst->addr + src_offset, len); + return; + } + + while (cur_off < len) { + unsigned long cur_src = cur_off + src_offset; + unsigned long folio_index = get_eb_folio_index(dst, cur_src); + unsigned long folio_off = get_eb_offset_in_folio(dst, cur_src); + unsigned long cur_len = min(src_offset + len - cur_src, + unit_size - folio_off); + void *src_addr = folio_address(dst->folios[folio_index]) + folio_off; + const bool use_memmove = areas_overlap(src_offset + cur_off, + dst_offset + cur_off, cur_len); + + __write_extent_buffer(dst, src_addr, dst_offset + cur_off, cur_len, + use_memmove); + cur_off += cur_len; + } +} + +void memmove_extent_buffer(const struct extent_buffer *dst, + unsigned long dst_offset, unsigned long src_offset, + unsigned long len) +{ + unsigned long dst_end = dst_offset + len - 1; + unsigned long src_end = src_offset + len - 1; + + if (check_eb_range(dst, dst_offset, len) || + check_eb_range(dst, src_offset, len)) + return; + + if (dst_offset < src_offset) { + memcpy_extent_buffer(dst, dst_offset, src_offset, len); + return; + } + + if (dst->addr) { + memmove(dst->addr + dst_offset, dst->addr + src_offset, len); + return; + } + + while (len > 0) { + unsigned long src_i; + size_t cur; + size_t dst_off_in_folio; + size_t src_off_in_folio; + void *src_addr; + bool use_memmove; + + src_i = get_eb_folio_index(dst, src_end); + + dst_off_in_folio = get_eb_offset_in_folio(dst, dst_end); + src_off_in_folio = get_eb_offset_in_folio(dst, src_end); + + cur = min_t(unsigned long, len, src_off_in_folio + 1); + cur = min(cur, dst_off_in_folio + 1); + + src_addr = folio_address(dst->folios[src_i]) + src_off_in_folio - + cur + 1; + use_memmove = areas_overlap(src_end - cur + 1, dst_end - cur + 1, + cur); + + __write_extent_buffer(dst, src_addr, dst_end - cur + 1, cur, + use_memmove); + + dst_end -= cur; + src_end -= cur; + len -= cur; + } +} + +static int try_release_subpage_extent_buffer(struct folio *folio) +{ + struct btrfs_fs_info *fs_info = folio_to_fs_info(folio); + struct extent_buffer *eb; + unsigned long start = (folio_pos(folio) >> fs_info->nodesize_bits); + unsigned long index = start; + unsigned long end = index + (PAGE_SIZE >> fs_info->nodesize_bits) - 1; + int ret; + + rcu_read_lock(); + xa_for_each_range(&fs_info->buffer_tree, index, eb, start, end) { + /* + * The same as try_release_extent_buffer(), to ensure the eb + * won't disappear out from under us. + */ + spin_lock(&eb->refs_lock); + rcu_read_unlock(); + + if (refcount_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) { + spin_unlock(&eb->refs_lock); + rcu_read_lock(); + continue; + } + + /* + * If tree ref isn't set then we know the ref on this eb is a + * real ref, so just return, this eb will likely be freed soon + * anyway. + */ + if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) { + spin_unlock(&eb->refs_lock); + rcu_read_lock(); + break; + } + + /* + * Here we don't care about the return value, we will always + * check the folio private at the end. And + * release_extent_buffer() will release the refs_lock. + */ + release_extent_buffer(eb); + rcu_read_lock(); + } + rcu_read_unlock(); + + /* + * Finally to check if we have cleared folio private, as if we have + * released all ebs in the page, the folio private should be cleared now. + */ + spin_lock(&folio->mapping->i_private_lock); + if (!folio_test_private(folio)) + ret = 1; + else + ret = 0; + spin_unlock(&folio->mapping->i_private_lock); + return ret; +} + +int try_release_extent_buffer(struct folio *folio) +{ + struct extent_buffer *eb; + + if (btrfs_meta_is_subpage(folio_to_fs_info(folio))) + return try_release_subpage_extent_buffer(folio); + + /* + * We need to make sure nobody is changing folio private, as we rely on + * folio private as the pointer to extent buffer. + */ + spin_lock(&folio->mapping->i_private_lock); + if (!folio_test_private(folio)) { + spin_unlock(&folio->mapping->i_private_lock); + return 1; + } + + eb = folio_get_private(folio); + BUG_ON(!eb); + + /* + * This is a little awful but should be ok, we need to make sure that + * the eb doesn't disappear out from under us while we're looking at + * this page. + */ + spin_lock(&eb->refs_lock); + if (refcount_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) { + spin_unlock(&eb->refs_lock); + spin_unlock(&folio->mapping->i_private_lock); + return 0; + } + spin_unlock(&folio->mapping->i_private_lock); + + /* + * If tree ref isn't set then we know the ref on this eb is a real ref, + * so just return, this page will likely be freed soon anyway. + */ + if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) { + spin_unlock(&eb->refs_lock); + return 0; + } + + return release_extent_buffer(eb); +} + +/* + * Attempt to readahead a child block. + * + * @fs_info: the fs_info + * @bytenr: bytenr to read + * @owner_root: objectid of the root that owns this eb + * @gen: generation for the uptodate check, can be 0 + * @level: level for the eb + * + * Attempt to readahead a tree block at @bytenr. If @gen is 0 then we do a + * normal uptodate check of the eb, without checking the generation. If we have + * to read the block we will not block on anything. + */ +void btrfs_readahead_tree_block(struct btrfs_fs_info *fs_info, + u64 bytenr, u64 owner_root, u64 gen, int level, + const struct btrfs_key *first_key) +{ + struct btrfs_tree_parent_check check = { + .level = level, + .transid = gen + }; + /* Readahead is best effort so prefer to fail rather than block in reclaim. */ + struct btrfs_eb_prealloc pa = { .supports_nowait = true }; + struct extent_buffer *eb; + int ret; + + if (first_key) { + memcpy(&check.first_key, first_key, sizeof(struct btrfs_key)); + check.has_first_key = true; + } + + eb = btrfs_find_create_tree_block(fs_info, &pa, bytenr, owner_root, level); + if (IS_ERR(eb)) + return; + + if (btrfs_buffer_uptodate(eb, gen, &check)) { + free_extent_buffer(eb); + return; + } + + ret = read_extent_buffer_pages_nowait(eb, 0, &check); + if (ret < 0) + free_extent_buffer_stale(eb); + else + free_extent_buffer(eb); +} + +/* + * Readahead a node's child block. + * + * @node: parent node we're reading from + * @slot: slot in the parent node for the child we want to read + * + * A helper for btrfs_readahead_tree_block, we simply read the bytenr pointed at + * the slot in the node provided. + */ +void btrfs_readahead_node_child(struct extent_buffer *node, int slot) +{ + struct btrfs_key node_key; + + btrfs_node_key_to_cpu(node, &node_key, slot); + btrfs_readahead_tree_block(node->fs_info, + btrfs_node_blockptr(node, slot), + btrfs_header_owner(node), + btrfs_node_ptr_generation(node, slot), + btrfs_header_level(node) - 1, + &node_key); +} diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h new file mode 100644 index 000000000..d8dd2ae9f --- /dev/null +++ b/fs/btrfs/extent_io.h @@ -0,0 +1,454 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_EXTENT_IO_H +#define BTRFS_EXTENT_IO_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "messages.h" +#include "ulist.h" +#include "misc.h" + +struct page; +struct file; +struct folio; +struct inode; +struct fiemap_extent_info; +struct readahead_control; +struct address_space; +struct writeback_control; +struct extent_io_tree; +struct extent_map_tree; +struct extent_state; +struct btrfs_block_group; +struct btrfs_fs_info; +struct btrfs_inode; +struct btrfs_root; +struct btrfs_trans_handle; +struct btrfs_tree_parent_check; + +enum { + EXTENT_BUFFER_UPTODATE, + EXTENT_BUFFER_DIRTY, + EXTENT_BUFFER_TREE_REF, + EXTENT_BUFFER_STALE, + EXTENT_BUFFER_WRITEBACK, + EXTENT_BUFFER_UNMAPPED, + /* write IO error */ + EXTENT_BUFFER_WRITE_ERR, + /* Indicate the extent buffer is written zeroed out (for zoned) */ + EXTENT_BUFFER_ZONED_ZEROOUT, + /* Indicate that extent buffer pages a being read */ + EXTENT_BUFFER_READING, +}; + +/* these are flags for __process_pages_contig */ +enum { + ENUM_BIT(PAGE_UNLOCK), + /* Page starts writeback, clear dirty bit and set writeback bit */ + ENUM_BIT(PAGE_START_WRITEBACK), + ENUM_BIT(PAGE_END_WRITEBACK), +}; + +/* + * Folio private values. Every page that is controlled by the extent map has + * folio private set to this value. + */ +#define EXTENT_FOLIO_PRIVATE 1 + +/* + * The extent buffer bitmap operations are done with byte granularity instead of + * word granularity for two reasons: + * 1. The bitmaps must be little-endian on disk. + * 2. Bitmap items are not guaranteed to be aligned to a word and therefore a + * single word in a bitmap may straddle two pages in the extent buffer. + */ +#define BIT_BYTE(nr) ((nr) / BITS_PER_BYTE) +#define BYTE_MASK ((1U << BITS_PER_BYTE) - 1) +#define BITMAP_FIRST_BYTE_MASK(start) \ + ((BYTE_MASK << ((start) & (BITS_PER_BYTE - 1))) & BYTE_MASK) +#define BITMAP_LAST_BYTE_MASK(nbits) \ + (BYTE_MASK >> (-(nbits) & (BITS_PER_BYTE - 1))) + + +int __init extent_buffer_init_cachep(void); +void __cold extent_buffer_free_cachep(void); + +#define INLINE_EXTENT_BUFFER_PAGES (BTRFS_MAX_METADATA_BLOCKSIZE / PAGE_SIZE) +struct extent_buffer { + u64 start; + u32 len; + u32 folio_size; + unsigned long bflags; + struct btrfs_fs_info *fs_info; + + /* + * The address where the eb can be accessed without any cross-page handling. + * This can be NULL if not possible. + */ + void *addr; + + spinlock_t refs_lock; + refcount_t refs; + int read_mirror; + /* Inhibit WB_SYNC_NONE writeback when > 0. */ + atomic_t writeback_inhibitors; + /* >= 0 if eb belongs to a log tree, -1 otherwise */ + s8 log_index; + u8 folio_shift; + struct rcu_head rcu_head; + + struct rw_semaphore lock; + + /* + * Pointers to all the folios of the extent buffer. + * + * For now the folio is always order 0 (aka, a single page). + */ + struct folio *folios[INLINE_EXTENT_BUFFER_PAGES]; +#ifdef CONFIG_BTRFS_DEBUG + struct list_head leak_list; + pid_t lock_owner; +#endif +}; + +/* + * Wrapper struct for managing preallocating an extent_buffer, its folios and a + * btrfs_folio_state if needed. + * + * Only used to mediate allocation, do not refer to the eb directly if not + * returned from a successful eb allocating API. + * + * The eb folios and bfs should generally not be fully attached, except briefly + * before they are NULLed in the struct after successful attachment. + */ +struct btrfs_eb_prealloc { + struct extent_buffer *eb; + struct btrfs_folio_state *bfs; + /* eb alloc may use GFP_NOWAIT; caller can drop locks and retry. */ + bool supports_nowait; + /* GFP_NOWAIT eb alloc failed; preallocate again and retry. */ + bool needs_prealloc; +}; + +struct btrfs_eb_write_context { + struct writeback_control *wbc; + struct extent_buffer *eb; + /* Block group @eb resides in. Only used for zoned mode. */ + struct btrfs_block_group *zoned_bg; +}; + +static inline unsigned long offset_in_eb_folio(const struct extent_buffer *eb, + u64 start) +{ + ASSERT(eb->folio_size); + return start & (eb->folio_size - 1); +} + +/* + * Get the correct offset inside the page of extent buffer. + * + * @eb: target extent buffer + * @start: offset inside the extent buffer + * + * Will handle both sectorsize == PAGE_SIZE and sectorsize < PAGE_SIZE cases. + */ +static inline size_t get_eb_offset_in_folio(const struct extent_buffer *eb, + unsigned long offset) +{ + /* + * 1) sectorsize == PAGE_SIZE and nodesize >= PAGE_SIZE case + * 1.1) One large folio covering the whole eb + * The eb->start is aligned to folio size, thus adding it + * won't cause any difference. + * 1.2) Several page sized folios + * The eb->start is aligned to folio (page) size, thus + * adding it won't cause any difference. + * + * 2) sectorsize < PAGE_SIZE and nodesize < PAGE_SIZE case + * In this case there would only be one page sized folio, and there + * may be several different extent buffers in the page/folio. + * We need to add eb->start to properly access the offset inside + * that eb. + */ + return offset_in_folio(eb->folios[0], offset + eb->start); +} + +static inline unsigned long get_eb_folio_index(const struct extent_buffer *eb, + unsigned long offset) +{ + /* + * 1) sectorsize == PAGE_SIZE and nodesize >= PAGE_SIZE case + * 1.1) One large folio covering the whole eb. + * the folio_shift would be large enough to always make us + * return 0 as index. + * 1.2) Several page sized folios + * The folio_shift would be PAGE_SHIFT, giving us the correct + * index. + * + * 2) sectorsize < PAGE_SIZE and nodesize < PAGE_SIZE case + * The folio would only be page sized, and always give us 0 as index. + */ + return offset >> eb->folio_shift; +} + +/* + * Structure to record how many bytes and which ranges are set/cleared + */ +struct extent_changeset { + /* How many bytes are set/cleared in this operation */ + u64 bytes_changed; + + /* Changed ranges */ + struct ulist range_changed; +}; + +static inline void extent_changeset_init(struct extent_changeset *changeset) +{ + changeset->bytes_changed = 0; + ulist_init(&changeset->range_changed); +} + +/* + * Sentinel value for range_changed.prealloc indicating that the changeset + * only tracks bytes_changed and does not record individual ranges. This + * avoids GFP_ATOMIC allocations inside add_extent_changeset() when the + * caller doesn't need to iterate the changed ranges afterwards. + */ +#define EXTENT_CHANGESET_BYTES_ONLY ((struct ulist_node *)1) + +static inline void extent_changeset_init_bytes_only(struct extent_changeset *changeset) +{ + changeset->bytes_changed = 0; + changeset->range_changed.prealloc = EXTENT_CHANGESET_BYTES_ONLY; +} + +static inline bool extent_changeset_tracks_ranges(const struct extent_changeset *changeset) +{ + return changeset->range_changed.prealloc != EXTENT_CHANGESET_BYTES_ONLY; +} + +static inline struct extent_changeset *extent_changeset_alloc(void) +{ + struct extent_changeset *ret; + + ret = kmalloc_obj(*ret); + if (!ret) + return NULL; + + extent_changeset_init(ret); + return ret; +} + +static inline void extent_changeset_prealloc(struct extent_changeset *changeset, gfp_t gfp_mask) +{ + ASSERT(extent_changeset_tracks_ranges(changeset)); + ulist_prealloc(&changeset->range_changed, gfp_mask); +} + +static inline void extent_changeset_release(struct extent_changeset *changeset) +{ + if (!changeset) + return; + changeset->bytes_changed = 0; + if (extent_changeset_tracks_ranges(changeset)) + ulist_release(&changeset->range_changed); +} + +static inline void extent_changeset_free(struct extent_changeset *changeset) +{ + if (!changeset) + return; + extent_changeset_release(changeset); + kfree(changeset); +} + +bool try_release_extent_mapping(struct folio *folio, gfp_t mask); +int try_release_extent_buffer(struct folio *folio); + +int btrfs_read_folio(struct file *file, struct folio *folio); +#ifdef CONFIG_BTRFS_DEBUG +void btrfs_check_folio_write_protected(struct folio *folio); +#else +static inline void btrfs_check_folio_write_protected(struct folio *folio) { } +#endif +void extent_write_locked_range(struct inode *inode, const struct folio *locked_folio, + u64 start, u64 end, struct writeback_control *wbc, + bool pages_dirty); +int btrfs_writepages(struct address_space *mapping, struct writeback_control *wbc); +int btree_writepages(struct address_space *mapping, struct writeback_control *wbc); +void btrfs_btree_wait_writeback_range(struct btrfs_fs_info *fs_info, u64 start, u64 end); +void btrfs_readahead(struct readahead_control *rac); +int set_folio_extent_mapped(struct folio *folio); +void clear_folio_extent_mapped(struct folio *folio); + +struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info, + struct btrfs_eb_prealloc *pa, + u64 start, u64 owner_root, int level); +int btrfs_init_eb_prealloc(struct btrfs_fs_info *fs_info, + struct btrfs_eb_prealloc *pa, bool nowait); +void btrfs_free_eb_prealloc(struct btrfs_eb_prealloc *pa); +struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info, + u64 start); +struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src); +struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info, + u64 start); +void free_extent_buffer(struct extent_buffer *eb); +void free_extent_buffer_stale(struct extent_buffer *eb); +int read_extent_buffer_pages(struct extent_buffer *eb, int mirror_num, + const struct btrfs_tree_parent_check *parent_check); +int read_extent_buffer_pages_nowait(struct extent_buffer *eb, int mirror_num, + const struct btrfs_tree_parent_check *parent_check); + +static inline void wait_on_extent_buffer_writeback(struct extent_buffer *eb) +{ + wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_WRITEBACK, + TASK_UNINTERRUPTIBLE); +} + +void btrfs_readahead_tree_block(struct btrfs_fs_info *fs_info, + u64 bytenr, u64 owner_root, u64 gen, int level, + const struct btrfs_key *first_key); +void btrfs_readahead_node_child(struct extent_buffer *node, int slot); + +/* Note: this can be used in for loops without caching the value in a variable. */ +static inline int __pure num_extent_pages(const struct extent_buffer *eb) +{ + /* + * For sectorsize == PAGE_SIZE case, since nodesize is always aligned to + * sectorsize, it's just eb->len >> PAGE_SHIFT. + * + * For sectorsize < PAGE_SIZE case, we could have nodesize < PAGE_SIZE, + * thus have to ensure we get at least one page. + */ + return (eb->len >> PAGE_SHIFT) ?: 1; +} + +/* + * This can only be determined at runtime by checking eb::folios[0]. + * + * As we can have either one large folio covering the whole eb + * (either nodesize <= PAGE_SIZE, or high order folio), or multiple + * single-paged folios. + * + * Note: this can be used in for loops without caching the value in a variable. + */ +static inline int __pure num_extent_folios(const struct extent_buffer *eb) +{ + if (!eb->folios[0]) + return 0; + if (folio_order(eb->folios[0])) + return 1; + return num_extent_pages(eb); +} + +static inline bool extent_buffer_uptodate(const struct extent_buffer *eb) +{ + return test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); +} + +static inline bool extent_buffer_under_io(const struct extent_buffer *eb) +{ + return (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) || + test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)); +} + +int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv, + unsigned long start, unsigned long len); +void read_extent_buffer(const struct extent_buffer *eb, void *dst, + unsigned long start, + unsigned long len); +int read_extent_buffer_to_user_nofault(const struct extent_buffer *eb, + void __user *dst, unsigned long start, + unsigned long len); +void write_extent_buffer(const struct extent_buffer *eb, const void *src, + unsigned long start, unsigned long len); + +static inline void write_extent_buffer_chunk_tree_uuid( + const struct extent_buffer *eb, const void *chunk_tree_uuid) +{ + write_extent_buffer(eb, chunk_tree_uuid, + offsetof(struct btrfs_header, chunk_tree_uuid), + BTRFS_FSID_SIZE); +} + +static inline void write_extent_buffer_fsid(const struct extent_buffer *eb, + const void *fsid) +{ + write_extent_buffer(eb, fsid, offsetof(struct btrfs_header, fsid), + BTRFS_FSID_SIZE); +} + +void copy_extent_buffer_full(const struct extent_buffer *dst, + const struct extent_buffer *src); +void copy_extent_buffer(const struct extent_buffer *dst, + const struct extent_buffer *src, + unsigned long dst_offset, unsigned long src_offset, + unsigned long len); +void memcpy_extent_buffer(const struct extent_buffer *dst, + unsigned long dst_offset, unsigned long src_offset, + unsigned long len); +void memmove_extent_buffer(const struct extent_buffer *dst, + unsigned long dst_offset, unsigned long src_offset, + unsigned long len); +void memzero_extent_buffer(const struct extent_buffer *eb, unsigned long start, + unsigned long len); +bool extent_buffer_test_bit(const struct extent_buffer *eb, unsigned long start, + unsigned long pos); +void extent_buffer_bitmap_set(const struct extent_buffer *eb, unsigned long start, + unsigned long pos, unsigned long len); +void extent_buffer_bitmap_clear(const struct extent_buffer *eb, + unsigned long start, unsigned long pos, + unsigned long len); +void set_extent_buffer_dirty(struct extent_buffer *eb); +void set_extent_buffer_uptodate(struct extent_buffer *eb); +void clear_extent_buffer_uptodate(struct extent_buffer *eb); +void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end, + const struct folio *locked_folio, + struct extent_state **cached, + u32 bits_to_clear, unsigned long page_ops); +void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans, + struct extent_buffer *buf); +void btrfs_zoned_release_dirty_metadata(struct btrfs_fs_info *fs_info); + +static inline void btrfs_clear_folio_dirty_tag(struct folio *folio) +{ + ASSERT(!folio_test_dirty(folio)); + ASSERT(folio_test_locked(folio)); + ASSERT(folio->mapping); + xa_lock_irq(&folio->mapping->i_pages); + __xa_clear_mark(&folio->mapping->i_pages, folio->index, + PAGECACHE_TAG_DIRTY); + xa_unlock_irq(&folio->mapping->i_pages); +} + +int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array, gfp_t gfp); +int btrfs_alloc_folio_array(unsigned int nr_folios, unsigned int order, + struct folio **folio_array, gfp_t gfp); + +#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS +bool find_lock_delalloc_range(struct inode *inode, + struct folio *locked_folio, u64 *start, + u64 *end); +#endif +struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info, + u64 start); + +#ifdef CONFIG_BTRFS_DEBUG +void btrfs_extent_buffer_leak_debug_check(struct btrfs_fs_info *fs_info); +#else +#define btrfs_extent_buffer_leak_debug_check(fs_info) do {} while (0) +#endif + +void btrfs_inhibit_eb_writeback(struct btrfs_trans_handle *trans, + struct extent_buffer *eb); +void btrfs_uninhibit_all_eb_writeback(struct btrfs_trans_handle *trans); + +#endif diff --git a/fs/btrfs/extent_map.c b/fs/btrfs/extent_map.c new file mode 100644 index 000000000..6ad7b39ae --- /dev/null +++ b/fs/btrfs/extent_map.c @@ -0,0 +1,1396 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include "messages.h" +#include "ctree.h" +#include "extent_map.h" +#include "compression.h" +#include "btrfs_inode.h" +#include "disk-io.h" + + +static struct kmem_cache *extent_map_cache; + +int __init btrfs_extent_map_init(void) +{ + extent_map_cache = kmem_cache_create("btrfs_extent_map", + sizeof(struct extent_map), 0, 0, NULL); + if (!extent_map_cache) + return -ENOMEM; + return 0; +} + +void __cold btrfs_extent_map_exit(void) +{ + kmem_cache_destroy(extent_map_cache); +} + +/* + * Initialize the extent tree @tree. Should be called for each new inode or + * other user of the extent_map interface. + */ +void btrfs_extent_map_tree_init(struct extent_map_tree *tree) +{ + tree->root = RB_ROOT; + INIT_LIST_HEAD(&tree->modified_extents); + rwlock_init(&tree->lock); +} + +/* + * Allocate a new extent_map structure. The new structure is returned with a + * reference count of one and needs to be freed using free_extent_map() + */ +struct extent_map *btrfs_alloc_extent_map(void) +{ + struct extent_map *em; + em = kmem_cache_zalloc(extent_map_cache, GFP_NOFS); + if (!em) + return NULL; + RB_CLEAR_NODE(&em->rb_node); + refcount_set(&em->refs, 1); + INIT_LIST_HEAD(&em->list); + return em; +} + +/* + * Drop the reference out on @em by one and free the structure if the reference + * count hits zero. + */ +void btrfs_free_extent_map(struct extent_map *em) +{ + if (!em) + return; + if (refcount_dec_and_test(&em->refs)) { + WARN_ON(btrfs_extent_map_in_tree(em)); + WARN_ON(!list_empty(&em->list)); + kmem_cache_free(extent_map_cache, em); + } +} + +/* Do the math around the end of an extent, handling wrapping. */ +static u64 range_end(u64 start, u64 len) +{ + if (start + len < start) + return (u64)-1; + return start + len; +} + +static void remove_em(struct btrfs_inode *inode, struct extent_map *em) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + + rb_erase(&em->rb_node, &inode->extent_tree.root); + RB_CLEAR_NODE(&em->rb_node); + + if (!btrfs_is_testing(fs_info) && btrfs_is_fstree(btrfs_root_id(inode->root))) + percpu_counter_dec(&fs_info->evictable_extent_maps); +} + +static int tree_insert(struct rb_root *root, struct extent_map *em) +{ + struct rb_node **p = &root->rb_node; + struct rb_node *parent = NULL; + struct extent_map *entry = NULL; + struct rb_node *orig_parent = NULL; + u64 end = range_end(em->start, em->len); + + while (*p) { + parent = *p; + entry = rb_entry(parent, struct extent_map, rb_node); + + if (em->start < entry->start) + p = &(*p)->rb_left; + else if (em->start >= btrfs_extent_map_end(entry)) + p = &(*p)->rb_right; + else + return -EEXIST; + } + + orig_parent = parent; + while (parent && em->start >= btrfs_extent_map_end(entry)) { + parent = rb_next(parent); + entry = rb_entry(parent, struct extent_map, rb_node); + } + if (parent) + if (end > entry->start && em->start < btrfs_extent_map_end(entry)) + return -EEXIST; + + parent = orig_parent; + entry = rb_entry(parent, struct extent_map, rb_node); + while (parent && em->start < entry->start) { + parent = rb_prev(parent); + entry = rb_entry(parent, struct extent_map, rb_node); + } + if (parent) + if (end > entry->start && em->start < btrfs_extent_map_end(entry)) + return -EEXIST; + + rb_link_node(&em->rb_node, orig_parent, p); + rb_insert_color(&em->rb_node, root); + return 0; +} + +/* + * Search through the tree for an extent_map with a given offset. If it can't + * be found, try to find some neighboring extents + */ +static struct rb_node *tree_search(struct rb_root *root, u64 offset, + struct rb_node **prev_or_next_ret) +{ + struct rb_node *n = root->rb_node; + struct rb_node *prev = NULL; + struct rb_node *orig_prev = NULL; + struct extent_map *entry; + struct extent_map *prev_entry = NULL; + + ASSERT(prev_or_next_ret); + + while (n) { + entry = rb_entry(n, struct extent_map, rb_node); + prev = n; + prev_entry = entry; + + if (offset < entry->start) + n = n->rb_left; + else if (offset >= btrfs_extent_map_end(entry)) + n = n->rb_right; + else + return n; + } + + orig_prev = prev; + while (prev && offset >= btrfs_extent_map_end(prev_entry)) { + prev = rb_next(prev); + prev_entry = rb_entry(prev, struct extent_map, rb_node); + } + + /* + * Previous extent map found, return as in this case the caller does not + * care about the next one. + */ + if (prev) { + *prev_or_next_ret = prev; + return NULL; + } + + prev = orig_prev; + prev_entry = rb_entry(prev, struct extent_map, rb_node); + while (prev && offset < prev_entry->start) { + prev = rb_prev(prev); + prev_entry = rb_entry(prev, struct extent_map, rb_node); + } + *prev_or_next_ret = prev; + + return NULL; +} + +static inline u64 extent_map_block_len(const struct extent_map *em) +{ + if (btrfs_extent_map_is_compressed(em)) + return em->disk_num_bytes; + return em->len; +} + +static inline u64 extent_map_block_end(const struct extent_map *em) +{ + const u64 block_start = btrfs_extent_map_block_start(em); + const u64 block_end = block_start + extent_map_block_len(em); + + if (block_end < block_start) + return (u64)-1; + + return block_end; +} + +static bool can_merge_extent_map(const struct extent_map *em) +{ + if (em->flags & EXTENT_FLAG_PINNED) + return false; + + /* Don't merge compressed extents, we need to know their actual size. */ + if (btrfs_extent_map_is_compressed(em)) + return false; + + if (em->flags & EXTENT_FLAG_LOGGING) + return false; + + /* + * We don't want to merge stuff that hasn't been written to the log yet + * since it may not reflect exactly what is on disk, and that would be + * bad. + */ + if (!list_empty(&em->list)) + return false; + + return true; +} + +/* Check to see if two extent_map structs are adjacent and safe to merge. */ +static bool mergeable_maps(const struct extent_map *prev, const struct extent_map *next) +{ + if (btrfs_extent_map_end(prev) != next->start) + return false; + + /* + * The merged flag is not an on-disk flag, it just indicates we had the + * extent maps of 2 (or more) adjacent extents merged, so factor it out. + */ + if ((prev->flags & ~EXTENT_FLAG_MERGED) != + (next->flags & ~EXTENT_FLAG_MERGED)) + return false; + + if (next->disk_bytenr < EXTENT_MAP_LAST_BYTE - 1) + return btrfs_extent_map_block_start(next) == extent_map_block_end(prev); + + /* HOLES and INLINE extents. */ + return next->disk_bytenr == prev->disk_bytenr; +} + +/* + * Handle the on-disk data extents merge for @prev and @next. + * + * @prev: left extent to merge + * @next: right extent to merge + * @merged: the extent we will not discard after the merge; updated with new values + * + * After this, one of the two extents is the new merged extent and the other is + * removed from the tree and likely freed. Note that @merged is one of @prev/@next + * so there is const/non-const aliasing occurring here. + * + * Only touches disk_bytenr/disk_num_bytes/offset/ram_bytes. + * For now only uncompressed regular extent can be merged. + */ +static void merge_ondisk_extents(const struct extent_map *prev, const struct extent_map *next, + struct extent_map *merged) +{ + u64 new_disk_bytenr; + u64 new_disk_num_bytes; + u64 new_offset; + + /* @prev and @next should not be compressed. */ + ASSERT(!btrfs_extent_map_is_compressed(prev)); + ASSERT(!btrfs_extent_map_is_compressed(next)); + + /* + * There are two different cases where @prev and @next can be merged. + * + * 1) They are referring to the same data extent: + * + * |<----- data extent A ----->| + * |<- prev ->|<- next ->| + * + * 2) They are referring to different data extents but still adjacent: + * + * |<-- data extent A -->|<-- data extent B -->| + * |<- prev ->|<- next ->| + * + * The calculation here always merges the data extents first, then updates + * @offset using the new data extents. + * + * For case 1), the merged data extent would be the same. + * For case 2), we just merge the two data extents into one. + */ + new_disk_bytenr = min(prev->disk_bytenr, next->disk_bytenr); + new_disk_num_bytes = max(prev->disk_bytenr + prev->disk_num_bytes, + next->disk_bytenr + next->disk_num_bytes) - + new_disk_bytenr; + new_offset = prev->disk_bytenr + prev->offset - new_disk_bytenr; + + merged->disk_bytenr = new_disk_bytenr; + merged->disk_num_bytes = new_disk_num_bytes; + merged->ram_bytes = new_disk_num_bytes; + merged->offset = new_offset; +} + +static void dump_extent_map(struct btrfs_fs_info *fs_info, const char *prefix, + struct extent_map *em) +{ + if (!IS_ENABLED(CONFIG_BTRFS_DEBUG)) + return; + btrfs_crit(fs_info, +"%s, start=%llu len=%llu disk_bytenr=%llu disk_num_bytes=%llu ram_bytes=%llu offset=%llu flags=0x%x", + prefix, em->start, em->len, em->disk_bytenr, em->disk_num_bytes, + em->ram_bytes, em->offset, em->flags); + ASSERT(0); +} + +/* Internal sanity checks for btrfs debug builds. */ +static void validate_extent_map(struct btrfs_fs_info *fs_info, struct extent_map *em) +{ + const u32 blocksize = fs_info->sectorsize; + + if (!IS_ENABLED(CONFIG_BTRFS_DEBUG)) + return; + + if (!IS_ALIGNED(em->start, blocksize) || + !IS_ALIGNED(em->len, blocksize)) + dump_extent_map(fs_info, "unaligned start offset or length members", em); + + if (em->disk_bytenr < EXTENT_MAP_LAST_BYTE) { + if (em->disk_num_bytes == 0) + dump_extent_map(fs_info, "zero disk_num_bytes", em); + if (em->offset + em->len > em->ram_bytes) + dump_extent_map(fs_info, "ram_bytes too small", em); + if (em->offset + em->len > em->disk_num_bytes && + !btrfs_extent_map_is_compressed(em)) + dump_extent_map(fs_info, "disk_num_bytes too small", em); + if (!btrfs_extent_map_is_compressed(em) && + em->ram_bytes != em->disk_num_bytes) + dump_extent_map(fs_info, + "ram_bytes mismatch with disk_num_bytes for non-compressed em", + em); + if (!IS_ALIGNED(em->disk_bytenr, blocksize) || + !IS_ALIGNED(em->disk_num_bytes, blocksize) || + !IS_ALIGNED(em->offset, blocksize) || + !IS_ALIGNED(em->ram_bytes, blocksize)) + dump_extent_map(fs_info, "unaligned members", em); + } else if (em->offset) { + dump_extent_map(fs_info, "non-zero offset for hole/inline", em); + } +} + +static void try_merge_map(struct btrfs_inode *inode, struct extent_map *em) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct extent_map *merge = NULL; + struct rb_node *rb; + + /* + * We can't modify an extent map that is in the tree and that is being + * used by another task, as it can cause that other task to see it in + * inconsistent state during the merging. We always have 1 reference for + * the tree and 1 for this task (which is unpinning the extent map or + * clearing the logging flag), so anything > 2 means it's being used by + * other tasks too. + */ + if (refcount_read(&em->refs) > 2) + return; + + if (!can_merge_extent_map(em)) + return; + + if (em->start != 0) { + rb = rb_prev(&em->rb_node); + merge = rb_entry_safe(rb, struct extent_map, rb_node); + + if (rb && can_merge_extent_map(merge) && mergeable_maps(merge, em)) { + em->start = merge->start; + em->len += merge->len; + em->generation = max(em->generation, merge->generation); + + if (em->disk_bytenr < EXTENT_MAP_LAST_BYTE) + merge_ondisk_extents(merge, em, em); + em->flags |= EXTENT_FLAG_MERGED; + + validate_extent_map(fs_info, em); + remove_em(inode, merge); + btrfs_free_extent_map(merge); + } + } + + rb = rb_next(&em->rb_node); + merge = rb_entry_safe(rb, struct extent_map, rb_node); + + if (rb && can_merge_extent_map(merge) && mergeable_maps(em, merge)) { + em->len += merge->len; + if (em->disk_bytenr < EXTENT_MAP_LAST_BYTE) + merge_ondisk_extents(em, merge, em); + validate_extent_map(fs_info, em); + em->generation = max(em->generation, merge->generation); + em->flags |= EXTENT_FLAG_MERGED; + remove_em(inode, merge); + btrfs_free_extent_map(merge); + } +} + +/* + * Unpin an extent from the cache. + * + * @inode: the inode from which we are unpinning an extent range + * @start: logical offset in the file + * @len: length of the extent + * @gen: generation that this extent has been modified in + * + * Called after an extent has been written to disk properly. Set the generation + * to the generation that actually added the file item to the inode so we know + * we need to sync this extent when we call fsync(). + * + * Returns: 0 on success + * -ENOENT when the extent is not found in the tree + * -EUCLEAN if the found extent does not match the expected start + */ +int btrfs_unpin_extent_cache(struct btrfs_inode *inode, u64 start, u64 len, u64 gen) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct extent_map_tree *tree = &inode->extent_tree; + int ret = 0; + struct extent_map *em; + + write_lock(&tree->lock); + em = btrfs_lookup_extent_mapping(tree, start, len); + + if (WARN_ON(!em)) { + btrfs_warn(fs_info, +"no extent map found for inode %llu (root %lld) when unpinning extent range [%llu, %llu), generation %llu", + btrfs_ino(inode), btrfs_root_id(inode->root), + start, start + len, gen); + ret = -ENOENT; + goto out; + } + + if (WARN_ON(em->start != start)) { + btrfs_warn(fs_info, +"found extent map for inode %llu (root %lld) with unexpected start offset %llu when unpinning extent range [%llu, %llu), generation %llu", + btrfs_ino(inode), btrfs_root_id(inode->root), + em->start, start, start + len, gen); + ret = -EUCLEAN; + goto out; + } + + em->generation = gen; + em->flags &= ~EXTENT_FLAG_PINNED; + + try_merge_map(inode, em); + +out: + write_unlock(&tree->lock); + btrfs_free_extent_map(em); + return ret; + +} + +void btrfs_clear_em_logging(struct btrfs_inode *inode, struct extent_map *em) +{ + lockdep_assert_held_write(&inode->extent_tree.lock); + + em->flags &= ~EXTENT_FLAG_LOGGING; + if (btrfs_extent_map_in_tree(em)) + try_merge_map(inode, em); +} + +static inline void setup_extent_mapping(struct btrfs_inode *inode, + struct extent_map *em, + bool modified) +{ + refcount_inc(&em->refs); + + ASSERT(list_empty(&em->list)); + + if (modified) + list_add(&em->list, &inode->extent_tree.modified_extents); + else + try_merge_map(inode, em); +} + +/* + * Add a new extent map to an inode's extent map tree. + * + * @inode: the target inode + * @em: map to insert + * @modified: indicate whether the given @em should be added to the + * modified list, which indicates the extent needs to be logged + * + * Insert @em into the @inode's extent map tree or perform a simple + * forward/backward merge with existing mappings. The extent_map struct passed + * in will be inserted into the tree directly, with an additional reference + * taken, or a reference dropped if the merge attempt was successful. + */ +static int add_extent_mapping(struct btrfs_inode *inode, + struct extent_map *em, bool modified) +{ + struct extent_map_tree *tree = &inode->extent_tree; + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + int ret; + + lockdep_assert_held_write(&tree->lock); + + validate_extent_map(fs_info, em); + ret = tree_insert(&tree->root, em); + if (ret) + return ret; + + setup_extent_mapping(inode, em, modified); + + if (!btrfs_is_testing(fs_info) && btrfs_is_fstree(btrfs_root_id(root))) + percpu_counter_inc(&fs_info->evictable_extent_maps); + + return 0; +} + +static struct extent_map *lookup_extent_mapping(struct extent_map_tree *tree, + u64 start, u64 len, bool strict) +{ + struct extent_map *em; + struct rb_node *rb_node; + struct rb_node *prev_or_next = NULL; + u64 end = range_end(start, len); + + rb_node = tree_search(&tree->root, start, &prev_or_next); + if (!rb_node) { + if (prev_or_next) + rb_node = prev_or_next; + else + return NULL; + } + + em = rb_entry(rb_node, struct extent_map, rb_node); + + if (strict && !(end > em->start && start < btrfs_extent_map_end(em))) + return NULL; + + refcount_inc(&em->refs); + return em; +} + +/* + * Lookup extent_map that intersects @start + @len range. + * + * @tree: tree to lookup in + * @start: byte offset to start the search + * @len: length of the lookup range + * + * Find and return the first extent_map struct in @tree that intersects the + * [start, len] range. There may be additional objects in the tree that + * intersect, so check the object returned carefully to make sure that no + * additional lookups are needed. + */ +struct extent_map *btrfs_lookup_extent_mapping(struct extent_map_tree *tree, + u64 start, u64 len) +{ + return lookup_extent_mapping(tree, start, len, true); +} + +/* + * Find a nearby extent map intersecting @start + @len (not an exact search). + * + * @tree: tree to lookup in + * @start: byte offset to start the search + * @len: length of the lookup range + * + * Find and return the first extent_map struct in @tree that intersects the + * [start, len] range. + * + * If one can't be found, any nearby extent may be returned + */ +struct extent_map *btrfs_search_extent_mapping(struct extent_map_tree *tree, + u64 start, u64 len) +{ + return lookup_extent_mapping(tree, start, len, false); +} + +/* + * Remove an extent_map from its inode's extent tree. + * + * @inode: the inode the extent map belongs to + * @em: extent map being removed + * + * Remove @em from the extent tree of @inode. No reference counts are dropped, + * and no checks are done to see if the range is in use. + */ +void btrfs_remove_extent_mapping(struct btrfs_inode *inode, struct extent_map *em) +{ + struct extent_map_tree *tree = &inode->extent_tree; + + lockdep_assert_held_write(&tree->lock); + + WARN_ON(em->flags & EXTENT_FLAG_PINNED); + if (!(em->flags & EXTENT_FLAG_LOGGING)) + list_del_init(&em->list); + + remove_em(inode, em); +} + +static void replace_extent_mapping(struct btrfs_inode *inode, + struct extent_map *cur, + struct extent_map *new, + bool modified) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct extent_map_tree *tree = &inode->extent_tree; + + lockdep_assert_held_write(&tree->lock); + + validate_extent_map(fs_info, new); + + WARN_ON(cur->flags & EXTENT_FLAG_PINNED); + ASSERT(btrfs_extent_map_in_tree(cur)); + if (!(cur->flags & EXTENT_FLAG_LOGGING)) + list_del_init(&cur->list); + rb_replace_node(&cur->rb_node, &new->rb_node, &tree->root); + RB_CLEAR_NODE(&cur->rb_node); + + setup_extent_mapping(inode, new, modified); +} + +static struct extent_map *next_extent_map(const struct extent_map *em) +{ + struct rb_node *next; + + next = rb_next(&em->rb_node); + if (!next) + return NULL; + return container_of(next, struct extent_map, rb_node); +} + +static struct extent_map *prev_extent_map(struct extent_map *em) +{ + struct rb_node *prev; + + prev = rb_prev(&em->rb_node); + if (!prev) + return NULL; + return container_of(prev, struct extent_map, rb_node); +} + +/* + * Helper for btrfs_get_extent. Given an existing extent in the tree, + * the existing extent is the nearest extent to map_start, + * and an extent that you want to insert, deal with overlap and insert + * the best fitted new extent into the tree. + */ +static noinline int merge_extent_mapping(struct btrfs_inode *inode, + struct extent_map *existing, + struct extent_map *em, + u64 map_start) +{ + struct extent_map *prev; + struct extent_map *next; + u64 start; + u64 end; + u64 start_diff; + + if (map_start < em->start || map_start >= btrfs_extent_map_end(em)) + return -EINVAL; + + if (existing->start > map_start) { + next = existing; + prev = prev_extent_map(next); + } else { + prev = existing; + next = next_extent_map(prev); + } + + start = prev ? btrfs_extent_map_end(prev) : em->start; + start = max_t(u64, start, em->start); + end = next ? next->start : btrfs_extent_map_end(em); + end = min_t(u64, end, btrfs_extent_map_end(em)); + start_diff = start - em->start; + em->start = start; + em->len = end - start; + if (em->disk_bytenr < EXTENT_MAP_LAST_BYTE) + em->offset += start_diff; + return add_extent_mapping(inode, em, false); +} + +/* + * Add extent mapping into an inode's extent map tree. + * + * @inode: target inode + * @em_in: extent we are inserting + * @start: start of the logical range btrfs_get_extent() is requesting + * @len: length of the logical range btrfs_get_extent() is requesting + * + * Note that @em_in's range may be different from [start, start+len), + * but they must be overlapped. + * + * Insert @em_in into the inode's extent map tree. In case there is an + * overlapping range, handle the -EEXIST by either: + * a) Returning the existing extent in @em_in if @start is within the + * existing em. + * b) Merge the existing extent with @em_in passed in. + * + * Return 0 on success, otherwise -EEXIST. + * + */ +int btrfs_add_extent_mapping(struct btrfs_inode *inode, + struct extent_map **em_in, u64 start, u64 len) +{ + int ret; + struct extent_map *em = *em_in; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + + /* + * Tree-checker should have rejected any inline extent with non-zero + * file offset. Here just do a sanity check. + */ + if (em->disk_bytenr == EXTENT_MAP_INLINE) + ASSERT(em->start == 0, "em->start=%llu", em->start); + + ret = add_extent_mapping(inode, em, false); + /* it is possible that someone inserted the extent into the tree + * while we had the lock dropped. It is also possible that + * an overlapping map exists in the tree + */ + if (ret == -EEXIST) { + struct extent_map *existing; + + existing = btrfs_search_extent_mapping(&inode->extent_tree, start, len); + + trace_btrfs_handle_em_exist(fs_info, existing, em, start, len); + + /* + * existing will always be non-NULL, since there must be + * extent causing the -EEXIST. + */ + if (start >= existing->start && + start < btrfs_extent_map_end(existing)) { + btrfs_free_extent_map(em); + *em_in = existing; + ret = 0; + } else { + u64 orig_start = em->start; + u64 orig_len = em->len; + + /* + * The existing extent map is the one nearest to + * the [start, start + len) range which overlaps + */ + ret = merge_extent_mapping(inode, existing, em, start); + if (WARN_ON(ret)) { + btrfs_free_extent_map(em); + *em_in = NULL; + btrfs_warn(fs_info, +"extent map merge error existing [%llu, %llu) with em [%llu, %llu) start %llu", + existing->start, btrfs_extent_map_end(existing), + orig_start, orig_start + orig_len, start); + } + btrfs_free_extent_map(existing); + } + } + + ASSERT(ret == 0 || ret == -EEXIST, "ret=%d", ret); + return ret; +} + +/* + * Drop all extent maps from a tree in the fastest possible way, rescheduling + * if needed. This avoids searching the tree, from the root down to the first + * extent map, before each deletion. + */ +static void drop_all_extent_maps_fast(struct btrfs_inode *inode) +{ + struct extent_map_tree *tree = &inode->extent_tree; + struct rb_node *node; + + write_lock(&tree->lock); + node = rb_first(&tree->root); + while (node) { + struct extent_map *em; + struct rb_node *next = rb_next(node); + + em = rb_entry(node, struct extent_map, rb_node); + em->flags &= ~(EXTENT_FLAG_PINNED | EXTENT_FLAG_LOGGING); + btrfs_remove_extent_mapping(inode, em); + btrfs_free_extent_map(em); + + if (cond_resched_rwlock_write(&tree->lock)) + node = rb_first(&tree->root); + else + node = next; + } + write_unlock(&tree->lock); +} + +/* + * Drop all extent maps in a given range. + * + * @inode: The target inode. + * @start: Start offset of the range. + * @end: End offset of the range (inclusive value). + * @skip_pinned: Indicate if pinned extent maps should be ignored or not. + * + * This drops all the extent maps that intersect the given range [@start, @end]. + * Extent maps that partially overlap the range and extend behind or beyond it, + * are split. + * The caller should have locked an appropriate file range in the inode's io + * tree before calling this function. + */ +void btrfs_drop_extent_map_range(struct btrfs_inode *inode, u64 start, u64 end, + bool skip_pinned) +{ + struct extent_map *split; + struct extent_map *split2; + struct extent_map *em; + struct extent_map_tree *em_tree = &inode->extent_tree; + u64 len = end - start + 1; + + WARN_ON(end < start); + if (end == (u64)-1) { + if (start == 0 && !skip_pinned) { + drop_all_extent_maps_fast(inode); + return; + } + len = (u64)-1; + } else { + /* Make end offset exclusive for use in the loop below. */ + end++; + } + + /* + * It's ok if we fail to allocate the extent maps, see the comment near + * the bottom of the loop below. We only need two spare extent maps in + * the worst case, where the first extent map that intersects our range + * starts before the range and the last extent map that intersects our + * range ends after our range (and they might be the same extent map), + * because we need to split those two extent maps at the boundaries. + */ + split = btrfs_alloc_extent_map(); + split2 = btrfs_alloc_extent_map(); + + write_lock(&em_tree->lock); + em = btrfs_lookup_extent_mapping(em_tree, start, len); + + while (em) { + /* extent_map_end() returns exclusive value (last byte + 1). */ + const u64 em_end = btrfs_extent_map_end(em); + struct extent_map *next_em = NULL; + u64 gen; + unsigned long flags; + bool modified; + + if (em_end < end) { + next_em = next_extent_map(em); + if (next_em) { + if (next_em->start < end) + refcount_inc(&next_em->refs); + else + next_em = NULL; + } + } + + if (skip_pinned && (em->flags & EXTENT_FLAG_PINNED)) { + start = em_end; + goto next; + } + + /* + * In case we split the extent map, we want to preserve the + * EXTENT_FLAG_LOGGING flag on our extent map, but we don't want + * it on the new extent maps. + */ + flags = em->flags & ~EXTENT_FLAG_LOGGING; + em->flags &= ~EXTENT_FLAG_PINNED; + modified = !list_empty(&em->list); + + /* + * The extent map does not cross our target range, so no need to + * split it, we can remove it directly. + */ + if (em->start >= start && em_end <= end) + goto remove_em; + + gen = em->generation; + + if (em->start < start) { + if (!split) { + split = split2; + split2 = NULL; + if (!split) + goto remove_em; + } + split->start = em->start; + split->len = start - em->start; + + if (em->disk_bytenr < EXTENT_MAP_LAST_BYTE) { + split->disk_bytenr = em->disk_bytenr; + split->disk_num_bytes = em->disk_num_bytes; + split->offset = em->offset; + split->ram_bytes = em->ram_bytes; + } else { + split->disk_bytenr = em->disk_bytenr; + split->disk_num_bytes = 0; + split->offset = 0; + split->ram_bytes = split->len; + } + + split->generation = gen; + split->flags = flags; + replace_extent_mapping(inode, em, split, modified); + btrfs_free_extent_map(split); + split = split2; + split2 = NULL; + } + if (em_end > end) { + if (!split) { + split = split2; + split2 = NULL; + if (!split) + goto remove_em; + } + split->start = end; + split->len = em_end - end; + split->disk_bytenr = em->disk_bytenr; + split->flags = flags; + split->generation = gen; + + if (em->disk_bytenr < EXTENT_MAP_LAST_BYTE) { + split->disk_num_bytes = em->disk_num_bytes; + split->offset = em->offset + end - em->start; + split->ram_bytes = em->ram_bytes; + } else { + split->disk_num_bytes = 0; + split->offset = 0; + split->ram_bytes = split->len; + } + + if (btrfs_extent_map_in_tree(em)) { + replace_extent_mapping(inode, em, split, modified); + } else { + int ret; + + ret = add_extent_mapping(inode, split, modified); + /* Logic error, shouldn't happen. */ + ASSERT(ret == 0, "ret=%d", ret); + if (WARN_ON(ret != 0) && modified) + btrfs_set_inode_full_sync(inode); + } + btrfs_free_extent_map(split); + split = NULL; + } +remove_em: + if (btrfs_extent_map_in_tree(em)) { + /* + * If the extent map is still in the tree it means that + * either of the following is true: + * + * 1) It fits entirely in our range (doesn't end beyond + * it or starts before it); + * + * 2) It starts before our range and/or ends after our + * range, and we were not able to allocate the extent + * maps for split operations, @split and @split2. + * + * If we are at case 2) then we just remove the entire + * extent map - this is fine since if anyone needs it to + * access the subranges outside our range, will just + * load it again from the subvolume tree's file extent + * item. However if the extent map was in the list of + * modified extents, then we must mark the inode for a + * full fsync, otherwise a fast fsync will miss this + * extent if it's new and needs to be logged. + */ + if ((em->start < start || em_end > end) && modified) { + ASSERT(!split); + btrfs_set_inode_full_sync(inode); + } + btrfs_remove_extent_mapping(inode, em); + } + + /* + * Once for the tree reference (we replaced or removed the + * extent map from the tree). + */ + btrfs_free_extent_map(em); +next: + /* Once for us (for our lookup reference). */ + btrfs_free_extent_map(em); + + em = next_em; + } + + write_unlock(&em_tree->lock); + + btrfs_free_extent_map(split); + btrfs_free_extent_map(split2); +} + +/* + * Replace a range in the inode's extent map tree with a new extent map. + * + * @inode: The target inode. + * @new_em: The new extent map to add to the inode's extent map tree. + * @modified: Indicate if the new extent map should be added to the list of + * modified extents (for fast fsync tracking). + * + * Drops all the extent maps in the inode's extent map tree that intersect the + * range of the new extent map and adds the new extent map to the tree. + * The caller should have locked an appropriate file range in the inode's io + * tree before calling this function. + */ +int btrfs_replace_extent_map_range(struct btrfs_inode *inode, + struct extent_map *new_em, + bool modified) +{ + const u64 end = new_em->start + new_em->len - 1; + struct extent_map_tree *tree = &inode->extent_tree; + int ret; + + ASSERT(!btrfs_extent_map_in_tree(new_em)); + + /* + * The caller has locked an appropriate file range in the inode's io + * tree, but getting -EEXIST when adding the new extent map can still + * happen in case there are extents that partially cover the range, and + * this is due to two tasks operating on different parts of the extent. + * See commit 18e83ac75bfe67 ("Btrfs: fix unexpected EEXIST from + * btrfs_get_extent") for an example and details. + */ + do { + btrfs_drop_extent_map_range(inode, new_em->start, end, false); + write_lock(&tree->lock); + ret = add_extent_mapping(inode, new_em, modified); + write_unlock(&tree->lock); + } while (ret == -EEXIST); + + return ret; +} + +/* + * Split off the first pre bytes from the extent_map at [start, start + len], + * and set the block_start for it to new_logical. + * + * This function is used when an ordered_extent needs to be split. + */ +int btrfs_split_extent_map(struct btrfs_inode *inode, u64 start, u64 len, u64 pre, + u64 new_logical) +{ + struct extent_map_tree *em_tree = &inode->extent_tree; + struct extent_map *em; + struct extent_map *split_pre = NULL; + struct extent_map *split_mid = NULL; + int ret = 0; + unsigned long flags; + + ASSERT(pre != 0); + ASSERT(pre < len); + + split_pre = btrfs_alloc_extent_map(); + if (!split_pre) + return -ENOMEM; + split_mid = btrfs_alloc_extent_map(); + if (!split_mid) { + ret = -ENOMEM; + goto out_free_pre; + } + + btrfs_lock_extent(&inode->io_tree, start, start + len - 1, NULL); + write_lock(&em_tree->lock); + em = btrfs_lookup_extent_mapping(em_tree, start, len); + if (unlikely(!em)) { + ret = -EIO; + goto out_unlock; + } + + ASSERT(em->len == len); + ASSERT(!btrfs_extent_map_is_compressed(em)); + ASSERT(em->disk_bytenr < EXTENT_MAP_LAST_BYTE); + ASSERT(em->flags & EXTENT_FLAG_PINNED); + ASSERT(!(em->flags & EXTENT_FLAG_LOGGING)); + ASSERT(!list_empty(&em->list)); + + flags = em->flags; + em->flags &= ~EXTENT_FLAG_PINNED; + + /* First, replace the em with a new extent_map starting from * em->start */ + split_pre->start = em->start; + split_pre->len = pre; + split_pre->disk_bytenr = new_logical; + split_pre->disk_num_bytes = split_pre->len; + split_pre->offset = 0; + split_pre->ram_bytes = split_pre->len; + split_pre->flags = flags; + split_pre->generation = em->generation; + + replace_extent_mapping(inode, em, split_pre, true); + + /* + * Now we only have an extent_map at: + * [em->start, em->start + pre] + */ + + /* Insert the middle extent_map. */ + split_mid->start = em->start + pre; + split_mid->len = em->len - pre; + split_mid->disk_bytenr = btrfs_extent_map_block_start(em) + pre; + split_mid->disk_num_bytes = split_mid->len; + split_mid->offset = 0; + split_mid->ram_bytes = split_mid->len; + split_mid->flags = flags; + split_mid->generation = em->generation; + add_extent_mapping(inode, split_mid, true); + + /* Once for us */ + btrfs_free_extent_map(em); + /* Once for the tree */ + btrfs_free_extent_map(em); + +out_unlock: + write_unlock(&em_tree->lock); + btrfs_unlock_extent(&inode->io_tree, start, start + len - 1, NULL); + btrfs_free_extent_map(split_mid); +out_free_pre: + btrfs_free_extent_map(split_pre); + return ret; +} + +struct btrfs_em_shrink_ctx { + long nr_to_scan; + long scanned; +}; + +static long btrfs_scan_inode(struct btrfs_inode *inode, struct btrfs_em_shrink_ctx *ctx) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + const u64 cur_fs_gen = btrfs_get_fs_generation(fs_info); + struct extent_map_tree *tree = &inode->extent_tree; + long nr_dropped = 0; + struct rb_node *node; + + lockdep_assert_held_write(&tree->lock); + + /* + * Take the mmap lock so that we serialize with the inode logging phase + * of fsync because we may need to set the full sync flag on the inode, + * in case we have to remove extent maps in the tree's list of modified + * extents. If we set the full sync flag in the inode while an fsync is + * in progress, we may risk missing new extents because before the flag + * is set, fsync decides to only wait for writeback to complete and then + * during inode logging it sees the flag set and uses the subvolume tree + * to find new extents, which may not be there yet because ordered + * extents haven't completed yet. + * + * We also do a try lock because we don't want to block for too long and + * we are holding the extent map tree's lock in write mode. + */ + if (!down_read_trylock(&inode->i_mmap_lock)) + return 0; + + node = rb_first(&tree->root); + while (node) { + struct rb_node *next = rb_next(node); + struct extent_map *em; + + em = rb_entry(node, struct extent_map, rb_node); + ctx->scanned++; + + if (em->flags & EXTENT_FLAG_PINNED) + goto next; + + /* + * If the inode is in the list of modified extents (new) and its + * generation is the same (or is greater than) the current fs + * generation, it means it was not yet persisted so we have to + * set the full sync flag so that the next fsync will not miss + * it. + */ + if (!list_empty(&em->list) && em->generation >= cur_fs_gen) + btrfs_set_inode_full_sync(inode); + + btrfs_remove_extent_mapping(inode, em); + trace_btrfs_extent_map_shrinker_remove_em(inode, em); + /* Drop the reference for the tree. */ + btrfs_free_extent_map(em); + nr_dropped++; +next: + if (ctx->scanned >= ctx->nr_to_scan) + break; + + /* + * Stop if we need to reschedule or there's contention on the + * lock. This is to avoid slowing other tasks trying to take the + * lock. + */ + if (need_resched() || rwlock_needbreak(&tree->lock) || + btrfs_fs_closing(fs_info)) + break; + node = next; + } + up_read(&inode->i_mmap_lock); + + return nr_dropped; +} + +static struct btrfs_inode *find_first_inode_to_shrink(struct btrfs_root *root, + u64 min_ino) +{ + struct btrfs_inode *inode; + unsigned long from = min_ino; + + xa_lock(&root->inodes); + while (true) { + struct extent_map_tree *tree; + + inode = xa_find(&root->inodes, &from, ULONG_MAX, XA_PRESENT); + if (!inode) + break; + + tree = &inode->extent_tree; + + /* + * We want to be fast so if the lock is busy we don't want to + * spend time waiting for it (some task is about to do IO for + * the inode). + */ + if (!write_trylock(&tree->lock)) + goto next; + + /* + * Skip inode if it doesn't have loaded extent maps, so we avoid + * getting a reference and doing an iput later. This includes + * cases like files that were opened for things like stat(2), or + * files with all extent maps previously released through the + * release folio callback (btrfs_release_folio()) or released in + * a previous run, or directories which never have extent maps. + */ + if (RB_EMPTY_ROOT(&tree->root)) { + write_unlock(&tree->lock); + goto next; + } + + if (igrab(&inode->vfs_inode)) + break; + + write_unlock(&tree->lock); +next: + from = btrfs_ino(inode) + 1; + xa_unlock(&root->inodes); + cond_resched(); + xa_lock(&root->inodes); + } + xa_unlock(&root->inodes); + + return inode; +} + +static long btrfs_scan_root(struct btrfs_root *root, struct btrfs_em_shrink_ctx *ctx) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_inode *inode; + long nr_dropped = 0; + u64 min_ino = fs_info->em_shrinker_last_ino + 1; + + inode = find_first_inode_to_shrink(root, min_ino); + while (inode) { + nr_dropped += btrfs_scan_inode(inode, ctx); + write_unlock(&inode->extent_tree.lock); + + min_ino = btrfs_ino(inode) + 1; + fs_info->em_shrinker_last_ino = btrfs_ino(inode); + iput(&inode->vfs_inode); + + if (ctx->scanned >= ctx->nr_to_scan || btrfs_fs_closing(fs_info)) + break; + + cond_resched(); + + inode = find_first_inode_to_shrink(root, min_ino); + } + + if (inode) { + /* + * There are still inodes in this root or we happened to process + * the last one and reached the scan limit. In either case set + * the current root to this one, so we'll resume from the next + * inode if there is one or we will find out this was the last + * one and move to the next root. + */ + fs_info->em_shrinker_last_root = btrfs_root_id(root); + } else { + /* + * No more inodes in this root, set extent_map_shrinker_last_ino to 0 so + * that when processing the next root we start from its first inode. + */ + fs_info->em_shrinker_last_ino = 0; + fs_info->em_shrinker_last_root = btrfs_root_id(root) + 1; + } + + return nr_dropped; +} + +static void btrfs_extent_map_shrinker_worker(struct work_struct *work) +{ + struct btrfs_fs_info *fs_info; + struct btrfs_em_shrink_ctx ctx; + u64 start_root_id; + u64 next_root_id; + bool cycled = false; + long nr_dropped = 0; + + fs_info = container_of(work, struct btrfs_fs_info, em_shrinker_work); + + ctx.scanned = 0; + ctx.nr_to_scan = atomic64_read(&fs_info->em_shrinker_nr_to_scan); + + start_root_id = fs_info->em_shrinker_last_root; + next_root_id = fs_info->em_shrinker_last_root; + + if (trace_btrfs_extent_map_shrinker_scan_enter_enabled()) { + s64 nr = percpu_counter_sum_positive(&fs_info->evictable_extent_maps); + + trace_call__btrfs_extent_map_shrinker_scan_enter(fs_info, nr); + } + + while (ctx.scanned < ctx.nr_to_scan && !btrfs_fs_closing(fs_info)) { + struct btrfs_root *root; + unsigned long count; + + cond_resched(); + + spin_lock(&fs_info->fs_roots_radix_lock); + count = radix_tree_gang_lookup(&fs_info->fs_roots_radix, + (void **)&root, + (unsigned long)next_root_id, 1); + if (count == 0) { + spin_unlock(&fs_info->fs_roots_radix_lock); + if (start_root_id > 0 && !cycled) { + next_root_id = 0; + fs_info->em_shrinker_last_root = 0; + fs_info->em_shrinker_last_ino = 0; + cycled = true; + continue; + } + break; + } + next_root_id = btrfs_root_id(root) + 1; + root = btrfs_grab_root(root); + spin_unlock(&fs_info->fs_roots_radix_lock); + + if (!root) + continue; + + if (btrfs_is_fstree(btrfs_root_id(root))) + nr_dropped += btrfs_scan_root(root, &ctx); + + btrfs_put_root(root); + } + + if (trace_btrfs_extent_map_shrinker_scan_exit_enabled()) { + s64 nr = percpu_counter_sum_positive(&fs_info->evictable_extent_maps); + + trace_call__btrfs_extent_map_shrinker_scan_exit(fs_info, nr_dropped, nr); + } + + atomic64_set(&fs_info->em_shrinker_nr_to_scan, 0); +} + +void btrfs_free_extent_maps(struct btrfs_fs_info *fs_info, long nr_to_scan) +{ + /* + * Do nothing if the shrinker is already running. In case of high memory + * pressure we can have a lot of tasks calling us and all passing the + * same nr_to_scan value, but in reality we may need only to free + * nr_to_scan extent maps (or less). In case we need to free more than + * that, we will be called again by the fs shrinker, so no worries about + * not doing enough work to reclaim memory from extent maps. + * We can also be repeatedly called with the same nr_to_scan value + * simply because the shrinker runs asynchronously and multiple calls + * to this function are made before the shrinker does enough progress. + * + * That's why we set the atomic counter to nr_to_scan only if its + * current value is zero, instead of incrementing the counter by + * nr_to_scan. + */ + if (atomic64_cmpxchg(&fs_info->em_shrinker_nr_to_scan, 0, nr_to_scan) != 0) + return; + + queue_work(system_dfl_wq, &fs_info->em_shrinker_work); +} + +void btrfs_init_extent_map_shrinker_work(struct btrfs_fs_info *fs_info) +{ + atomic64_set(&fs_info->em_shrinker_nr_to_scan, 0); + INIT_WORK(&fs_info->em_shrinker_work, btrfs_extent_map_shrinker_worker); +} diff --git a/fs/btrfs/extent_map.h b/fs/btrfs/extent_map.h new file mode 100644 index 000000000..6f685f3c9 --- /dev/null +++ b/fs/btrfs/extent_map.h @@ -0,0 +1,195 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_EXTENT_MAP_H +#define BTRFS_EXTENT_MAP_H + +#include +#include +#include +#include +#include +#include "fs.h" + +struct btrfs_inode; +struct btrfs_fs_info; + +#define EXTENT_MAP_LAST_BYTE ((u64)-4) +#define EXTENT_MAP_HOLE ((u64)-3) +#define EXTENT_MAP_INLINE ((u64)-2) + +/* bits for the extent_map::flags field */ +enum { + /* this entry not yet on disk, don't free it */ + ENUM_BIT(EXTENT_FLAG_PINNED), + ENUM_BIT(EXTENT_FLAG_COMPRESS_ZLIB), + ENUM_BIT(EXTENT_FLAG_COMPRESS_LZO), + ENUM_BIT(EXTENT_FLAG_COMPRESS_ZSTD), + /* pre-allocated extent */ + ENUM_BIT(EXTENT_FLAG_PREALLOC), + /* Logging this extent */ + ENUM_BIT(EXTENT_FLAG_LOGGING), + /* This em is merged from two or more physically adjacent ems */ + ENUM_BIT(EXTENT_FLAG_MERGED), +}; + +/* + * This structure represents file extents and holes. + * + * Unlike on-disk file extent items, extent maps can be merged to save memory. + * This means members only match file extent items before any merging. + * + * Keep this structure as compact as possible, as we can have really large + * amounts of allocated extent maps at any time. + */ +struct extent_map { + struct rb_node rb_node; + + /* All of these are in bytes. */ + + /* File offset matching the offset of a BTRFS_EXTENT_ITEM_KEY key. */ + u64 start; + + /* + * Length of the file extent. + * + * For non-inlined file extents it's btrfs_file_extent_item::num_bytes. + * For inline extents it's sectorsize, since inline data starts at + * offsetof(struct btrfs_file_extent_item, disk_bytenr) thus + * btrfs_file_extent_item::num_bytes is not valid. + */ + u64 len; + + /* + * The bytenr of the full on-disk extent. + * + * For regular extents it's btrfs_file_extent_item::disk_bytenr. + * For holes it's EXTENT_MAP_HOLE and for inline extents it's + * EXTENT_MAP_INLINE. + */ + u64 disk_bytenr; + + /* + * The full on-disk extent length, matching + * btrfs_file_extent_item::disk_num_bytes. + */ + u64 disk_num_bytes; + + /* + * Offset inside the decompressed extent. + * + * For regular extents it's btrfs_file_extent_item::offset. + * For holes and inline extents it's 0. + */ + u64 offset; + + /* + * The decompressed size of the whole on-disk extent, matching + * btrfs_file_extent_item::ram_bytes. + */ + u64 ram_bytes; + + /* + * Generation of the extent map, for merged em it's the highest + * generation of all merged ems. + * For non-merged extents, it's from btrfs_file_extent_item::generation. + */ + u64 generation; + u32 flags; + refcount_t refs; + struct list_head list; +}; + +struct extent_map_tree { + struct rb_root root; + struct list_head modified_extents; + rwlock_t lock; +}; + +struct btrfs_inode; + +static inline void btrfs_extent_map_set_compression(struct extent_map *em, + enum btrfs_compression_type type) +{ + if (type == BTRFS_COMPRESS_ZLIB) + em->flags |= EXTENT_FLAG_COMPRESS_ZLIB; + else if (type == BTRFS_COMPRESS_LZO) + em->flags |= EXTENT_FLAG_COMPRESS_LZO; + else if (type == BTRFS_COMPRESS_ZSTD) + em->flags |= EXTENT_FLAG_COMPRESS_ZSTD; +} + +static inline enum btrfs_compression_type btrfs_extent_map_compression( + const struct extent_map *em) +{ + if (em->flags & EXTENT_FLAG_COMPRESS_ZLIB) + return BTRFS_COMPRESS_ZLIB; + + if (em->flags & EXTENT_FLAG_COMPRESS_LZO) + return BTRFS_COMPRESS_LZO; + + if (em->flags & EXTENT_FLAG_COMPRESS_ZSTD) + return BTRFS_COMPRESS_ZSTD; + + return BTRFS_COMPRESS_NONE; +} + +/* + * More efficient way to determine if extent is compressed, instead of using + * 'extent_map_compression() != BTRFS_COMPRESS_NONE'. + */ +static inline bool btrfs_extent_map_is_compressed(const struct extent_map *em) +{ + return (em->flags & (EXTENT_FLAG_COMPRESS_ZLIB | + EXTENT_FLAG_COMPRESS_LZO | + EXTENT_FLAG_COMPRESS_ZSTD)) != 0; +} + +static inline int btrfs_extent_map_in_tree(const struct extent_map *em) +{ + return !RB_EMPTY_NODE(&em->rb_node); +} + +static inline u64 btrfs_extent_map_block_start(const struct extent_map *em) +{ + if (em->disk_bytenr < EXTENT_MAP_LAST_BYTE) { + if (btrfs_extent_map_is_compressed(em)) + return em->disk_bytenr; + return em->disk_bytenr + em->offset; + } + return em->disk_bytenr; +} + +static inline u64 btrfs_extent_map_end(const struct extent_map *em) +{ + if (em->start + em->len < em->start) + return (u64)-1; + return em->start + em->len; +} + +void btrfs_extent_map_tree_init(struct extent_map_tree *tree); +struct extent_map *btrfs_lookup_extent_mapping(struct extent_map_tree *tree, + u64 start, u64 len); +void btrfs_remove_extent_mapping(struct btrfs_inode *inode, struct extent_map *em); +int btrfs_split_extent_map(struct btrfs_inode *inode, u64 start, u64 len, u64 pre, + u64 new_logical); + +struct extent_map *btrfs_alloc_extent_map(void); +void btrfs_free_extent_map(struct extent_map *em); +int __init btrfs_extent_map_init(void); +void __cold btrfs_extent_map_exit(void); +int btrfs_unpin_extent_cache(struct btrfs_inode *inode, u64 start, u64 len, u64 gen); +void btrfs_clear_em_logging(struct btrfs_inode *inode, struct extent_map *em); +struct extent_map *btrfs_search_extent_mapping(struct extent_map_tree *tree, + u64 start, u64 len); +int btrfs_add_extent_mapping(struct btrfs_inode *inode, + struct extent_map **em_in, u64 start, u64 len); +void btrfs_drop_extent_map_range(struct btrfs_inode *inode, + u64 start, u64 end, + bool skip_pinned); +int btrfs_replace_extent_map_range(struct btrfs_inode *inode, + struct extent_map *new_em, + bool modified); +void btrfs_free_extent_maps(struct btrfs_fs_info *fs_info, long nr_to_scan); +void btrfs_init_extent_map_shrinker_work(struct btrfs_fs_info *fs_info); + +#endif diff --git a/fs/btrfs/fiemap.c b/fs/btrfs/fiemap.c new file mode 100644 index 000000000..7a2a97180 --- /dev/null +++ b/fs/btrfs/fiemap.c @@ -0,0 +1,928 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "backref.h" +#include "btrfs_inode.h" +#include "fiemap.h" +#include "file.h" +#include "file-item.h" + +struct btrfs_fiemap_entry { + u64 offset; + u64 phys; + u64 len; + u32 flags; +}; + +/* + * Indicate the caller of emit_fiemap_extent() that it needs to unlock the file + * range from the inode's io tree, unlock the subvolume tree search path, flush + * the fiemap cache and relock the file range and research the subvolume tree. + * The value here is something negative that can't be confused with a valid + * errno value and different from 1 because that's also a return value from + * fiemap_fill_next_extent() and also it's often used to mean some btree search + * did not find a key, so make it some distinct negative value. + */ +#define BTRFS_FIEMAP_FLUSH_CACHE (-(MAX_ERRNO + 1)) + +/* + * Used to: + * + * - Cache the next entry to be emitted to the fiemap buffer, so that we can + * merge extents that are contiguous and can be grouped as a single one; + * + * - Store extents ready to be written to the fiemap buffer in an intermediary + * buffer. This intermediary buffer is to ensure that in case the fiemap + * buffer is memory mapped to the fiemap target file, we don't deadlock + * during btrfs_page_mkwrite(). This is because during fiemap we are locking + * an extent range in order to prevent races with delalloc flushing and + * ordered extent completion, which is needed in order to reliably detect + * delalloc in holes and prealloc extents. And this can lead to a deadlock + * if the fiemap buffer is memory mapped to the file we are running fiemap + * against (a silly, useless in practice scenario, but possible) because + * btrfs_page_mkwrite() will try to lock the same extent range. + */ +struct fiemap_cache { + /* An array of ready fiemap entries. */ + struct btrfs_fiemap_entry *entries; + /* Number of entries in the entries array. */ + int entries_size; + /* Index of the next entry in the entries array to write to. */ + int entries_pos; + /* + * Once the entries array is full, this indicates what's the offset for + * the next file extent item we must search for in the inode's subvolume + * tree after unlocking the extent range in the inode's io tree and + * releasing the search path. + */ + u64 next_search_offset; + /* + * This matches struct fiemap_extent_info::fi_mapped_extents, we use it + * to count ourselves emitted extents and stop instead of relying on + * fiemap_fill_next_extent() because we buffer ready fiemap entries at + * the @entries array, and we want to stop as soon as we hit the max + * amount of extents to map, not just to save time but also to make the + * logic at extent_fiemap() simpler. + */ + unsigned int extents_mapped; + /* Fields for the cached extent (unsubmitted, not ready, extent). */ + u64 offset; + u64 phys; + u64 len; + u32 flags; + bool cached; +}; + +static int flush_fiemap_cache(struct fiemap_extent_info *fieinfo, + struct fiemap_cache *cache) +{ + for (int i = 0; i < cache->entries_pos; i++) { + struct btrfs_fiemap_entry *entry = &cache->entries[i]; + int ret; + + ret = fiemap_fill_next_extent(fieinfo, entry->offset, + entry->phys, entry->len, + entry->flags); + /* + * Ignore 1 (reached max entries) because we keep track of that + * ourselves in emit_fiemap_extent(). + */ + if (ret < 0) + return ret; + } + cache->entries_pos = 0; + + return 0; +} + +/* + * Helper to submit fiemap extent. + * + * Will try to merge current fiemap extent specified by @offset, @phys, + * @len and @flags with cached one. + * And only when we fails to merge, cached one will be submitted as + * fiemap extent. + * + * Return value is the same as fiemap_fill_next_extent(). + */ +static int emit_fiemap_extent(struct fiemap_extent_info *fieinfo, + struct fiemap_cache *cache, + u64 offset, u64 phys, u64 len, u32 flags) +{ + struct btrfs_fiemap_entry *entry; + u64 cache_end; + + /* Set at the end of extent_fiemap(). */ + ASSERT((flags & FIEMAP_EXTENT_LAST) == 0, "flags=0x%u", flags); + + if (!cache->cached) + goto assign; + + /* + * When iterating the extents of the inode, at extent_fiemap(), we may + * find an extent that starts at an offset behind the end offset of the + * previous extent we processed. This happens if fiemap is called + * without FIEMAP_FLAG_SYNC and there are ordered extents completing + * after we had to unlock the file range, release the search path, emit + * the fiemap extents stored in the buffer (cache->entries array) and + * the lock the remainder of the range and re-search the btree. + * + * For example we are in leaf X processing its last item, which is the + * file extent item for file range [512K, 1M[, and after + * btrfs_next_leaf() releases the path, there's an ordered extent that + * completes for the file range [768K, 2M[, and that results in trimming + * the file extent item so that it now corresponds to the file range + * [512K, 768K[ and a new file extent item is inserted for the file + * range [768K, 2M[, which may end up as the last item of leaf X or as + * the first item of the next leaf - in either case btrfs_next_leaf() + * will leave us with a path pointing to the new extent item, for the + * file range [768K, 2M[, since that's the first key that follows the + * last one we processed. So in order not to report overlapping extents + * to user space, we trim the length of the previously cached extent and + * emit it. + * + * Upon calling btrfs_next_leaf() we may also find an extent with an + * offset smaller than or equals to cache->offset, and this happens + * when we had a hole or prealloc extent with several delalloc ranges in + * it, but after btrfs_next_leaf() released the path, delalloc was + * flushed and the resulting ordered extents were completed, so we can + * now have found a file extent item for an offset that is smaller than + * or equals to what we have in cache->offset. We deal with this as + * described below. + */ + cache_end = cache->offset + cache->len; + if (cache_end > offset) { + if (offset == cache->offset) { + /* + * We cached a delalloc range (found in the io tree) for + * a hole or prealloc extent and we have now found a + * file extent item for the same offset. What we have + * now is more recent and up to date, so discard what + * we had in the cache and use what we have just found. + */ + goto assign; + } else if (offset > cache->offset) { + /* + * The extent range we previously found ends after the + * offset of the file extent item we found and that + * offset falls somewhere in the middle of that previous + * extent range. So adjust the range we previously found + * to end at the offset of the file extent item we have + * just found, since this extent is more up to date. + * Emit that adjusted range and cache the file extent + * item we have just found. This corresponds to the case + * where a previously found file extent item was split + * due to an ordered extent completing. + */ + cache->len = offset - cache->offset; + goto emit; + } else { + const u64 range_end = offset + len; + + /* + * The offset of the file extent item we have just found + * is behind the cached offset. This means we were + * processing a hole or prealloc extent for which we + * have found delalloc ranges (in the io tree), so what + * we have in the cache is the last delalloc range we + * found while the file extent item we found can be + * either for a whole delalloc range we previously + * emitted or only a part of that range. + * + * We have two cases here: + * + * 1) The file extent item's range ends at or behind the + * cached extent's end. In this case just ignore the + * current file extent item because we don't want to + * overlap with previous ranges that may have been + * emitted already; + * + * 2) The file extent item starts behind the currently + * cached extent but its end offset goes beyond the + * end offset of the cached extent. We don't want to + * overlap with a previous range that may have been + * emitted already, so we emit the currently cached + * extent and then partially store the current file + * extent item's range in the cache, for the subrange + * going the cached extent's end to the end of the + * file extent item. + */ + if (range_end <= cache_end) + return 0; + + if (!(flags & (FIEMAP_EXTENT_ENCODED | FIEMAP_EXTENT_DELALLOC))) + phys += cache_end - offset; + + offset = cache_end; + len = range_end - cache_end; + goto emit; + } + } + + /* + * Only merges fiemap extents if + * 1) Their logical addresses are continuous + * + * 2) Their physical addresses are continuous + * So truly compressed (physical size smaller than logical size) + * extents won't get merged with each other + * + * 3) Share same flags + */ + if (cache->offset + cache->len == offset && + cache->phys + cache->len == phys && + cache->flags == flags) { + cache->len += len; + return 0; + } + +emit: + /* Not mergeable, need to submit cached one */ + + if (cache->entries_pos == cache->entries_size) { + /* + * We will need to research for the end offset of the last + * stored extent and not from the current offset, because after + * unlocking the range and releasing the path, if there's a hole + * between that end offset and this current offset, a new extent + * may have been inserted due to a new write, so we don't want + * to miss it. + */ + entry = &cache->entries[cache->entries_size - 1]; + cache->next_search_offset = entry->offset + entry->len; + cache->cached = false; + + return BTRFS_FIEMAP_FLUSH_CACHE; + } + + entry = &cache->entries[cache->entries_pos]; + entry->offset = cache->offset; + entry->phys = cache->phys; + entry->len = cache->len; + entry->flags = cache->flags; + cache->entries_pos++; + cache->extents_mapped++; + + if (cache->extents_mapped == fieinfo->fi_extents_max) { + cache->cached = false; + return 1; + } +assign: + cache->cached = true; + cache->offset = offset; + cache->phys = phys; + cache->len = len; + cache->flags = flags; + + return 0; +} + +/* + * Emit last fiemap cache + * + * The last fiemap cache may still be cached in the following case: + * 0 4k 8k + * |<- Fiemap range ->| + * |<------------ First extent ----------->| + * + * In this case, the first extent range will be cached but not emitted. + * So we must emit it before ending extent_fiemap(). + */ +static int emit_last_fiemap_cache(struct fiemap_extent_info *fieinfo, + struct fiemap_cache *cache) +{ + int ret; + + if (!cache->cached) + return 0; + + ret = fiemap_fill_next_extent(fieinfo, cache->offset, cache->phys, + cache->len, cache->flags); + cache->cached = false; + if (ret > 0) + ret = 0; + return ret; +} + +static int fiemap_next_leaf_item(struct btrfs_inode *inode, struct btrfs_path *path) +{ + struct extent_buffer *clone = path->nodes[0]; + struct btrfs_key key; + int slot; + int ret; + + path->slots[0]++; + if (path->slots[0] < btrfs_header_nritems(path->nodes[0])) + return 0; + + /* + * Add a temporary extra ref to an already cloned extent buffer to + * prevent btrfs_next_leaf() freeing it, we want to reuse it to avoid + * the cost of allocating a new one. + */ + ASSERT(test_bit(EXTENT_BUFFER_UNMAPPED, &clone->bflags)); + refcount_inc(&clone->refs); + + ret = btrfs_next_leaf(inode->root, path); + if (ret != 0) + goto out; + + /* + * Don't bother with cloning if there are no more file extent items for + * our inode. + */ + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + if (key.objectid != btrfs_ino(inode) || key.type != BTRFS_EXTENT_DATA_KEY) { + ret = 1; + goto out; + } + + /* + * Important to preserve the start field, for the optimizations when + * checking if extents are shared (see extent_fiemap()). + * + * We must set ->start before calling copy_extent_buffer_full(). If we + * are on sub-pagesize blocksize, we use ->start to determine the offset + * into the folio where our eb exists, and if we update ->start after + * the fact then any subsequent reads of the eb may read from a + * different offset in the folio than where we originally copied into. + */ + clone->start = path->nodes[0]->start; + /* See the comment at fiemap_search_slot() about why we clone. */ + copy_extent_buffer_full(clone, path->nodes[0]); + + slot = path->slots[0]; + btrfs_release_path(path); + path->nodes[0] = clone; + path->slots[0] = slot; +out: + if (ret) + free_extent_buffer(clone); + + return ret; +} + +/* + * Search for the first file extent item that starts at a given file offset or + * the one that starts immediately before that offset. + * Returns: 0 on success, < 0 on error, 1 if not found. + */ +static int fiemap_search_slot(struct btrfs_inode *inode, struct btrfs_path *path, + u64 file_offset) +{ + const u64 ino = btrfs_ino(inode); + struct btrfs_root *root = inode->root; + struct extent_buffer *clone; + struct btrfs_key key; + int slot; + int ret; + + key.objectid = ino; + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = file_offset; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + return ret; + + if (ret > 0 && path->slots[0] > 0) { + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0] - 1); + if (key.objectid == ino && key.type == BTRFS_EXTENT_DATA_KEY) + path->slots[0]--; + } + + if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) { + ret = btrfs_next_leaf(root, path); + if (ret != 0) + return ret; + + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY) + return 1; + } + + /* + * We clone the leaf and use it during fiemap. This is because while + * using the leaf we do expensive things like checking if an extent is + * shared, which can take a long time. In order to prevent blocking + * other tasks for too long, we use a clone of the leaf. We have locked + * the file range in the inode's io tree, so we know none of our file + * extent items can change. This way we avoid blocking other tasks that + * want to insert items for other inodes in the same leaf or b+tree + * rebalance operations (triggered for example when someone is trying + * to push items into this leaf when trying to insert an item in a + * neighbour leaf). + * We also need the private clone because holding a read lock on an + * extent buffer of the subvolume's b+tree will make lockdep unhappy + * when we check if extents are shared, as backref walking may need to + * lock the same leaf we are processing. + */ + clone = btrfs_clone_extent_buffer(path->nodes[0]); + if (!clone) + return -ENOMEM; + + slot = path->slots[0]; + btrfs_release_path(path); + path->nodes[0] = clone; + path->slots[0] = slot; + + return 0; +} + +/* + * Process a range which is a hole or a prealloc extent in the inode's subvolume + * btree. If @disk_bytenr is 0, we are dealing with a hole, otherwise a prealloc + * extent. The end offset (@end) is inclusive. + */ +static int fiemap_process_hole(struct btrfs_inode *inode, + struct fiemap_extent_info *fieinfo, + struct fiemap_cache *cache, + struct extent_state **delalloc_cached_state, + struct btrfs_backref_share_check_ctx *backref_ctx, + u64 disk_bytenr, u64 extent_offset, + u64 extent_gen, + u64 start, u64 end) +{ + const u64 i_size = i_size_read(&inode->vfs_inode); + u64 cur_offset = start; + u64 last_delalloc_end = 0; + u32 prealloc_flags = FIEMAP_EXTENT_UNWRITTEN; + bool checked_extent_shared = false; + int ret; + + /* + * There can be no delalloc past i_size, so don't waste time looking for + * it beyond i_size. + */ + while (cur_offset < end && cur_offset < i_size) { + u64 delalloc_start; + u64 delalloc_end; + u64 prealloc_start; + u64 prealloc_len = 0; + bool delalloc; + + delalloc = btrfs_find_delalloc_in_range(inode, cur_offset, end, + delalloc_cached_state, + &delalloc_start, + &delalloc_end); + if (!delalloc) + break; + + /* + * If this is a prealloc extent we have to report every section + * of it that has no delalloc. + */ + if (disk_bytenr != 0) { + if (last_delalloc_end == 0) { + prealloc_start = start; + prealloc_len = delalloc_start - start; + } else { + prealloc_start = last_delalloc_end + 1; + prealloc_len = delalloc_start - prealloc_start; + } + } + + if (prealloc_len > 0) { + if (!checked_extent_shared && fieinfo->fi_extents_max) { + ret = btrfs_is_data_extent_shared(inode, + disk_bytenr, + extent_gen, + backref_ctx); + if (ret < 0) + return ret; + else if (ret > 0) + prealloc_flags |= FIEMAP_EXTENT_SHARED; + + checked_extent_shared = true; + } + ret = emit_fiemap_extent(fieinfo, cache, prealloc_start, + disk_bytenr + extent_offset, + prealloc_len, prealloc_flags); + if (ret) + return ret; + extent_offset += prealloc_len; + } + + ret = emit_fiemap_extent(fieinfo, cache, delalloc_start, 0, + delalloc_end + 1 - delalloc_start, + FIEMAP_EXTENT_DELALLOC | + FIEMAP_EXTENT_UNKNOWN); + if (ret) + return ret; + + last_delalloc_end = delalloc_end; + cur_offset = delalloc_end + 1; + extent_offset += cur_offset - delalloc_start; + cond_resched(); + } + + /* + * Either we found no delalloc for the whole prealloc extent or we have + * a prealloc extent that spans i_size or starts at or after i_size. + */ + if (disk_bytenr != 0 && last_delalloc_end < end) { + u64 prealloc_start; + u64 prealloc_len; + + if (last_delalloc_end == 0) { + prealloc_start = start; + prealloc_len = end + 1 - start; + } else { + prealloc_start = last_delalloc_end + 1; + prealloc_len = end + 1 - prealloc_start; + } + + if (!checked_extent_shared && fieinfo->fi_extents_max) { + ret = btrfs_is_data_extent_shared(inode, + disk_bytenr, + extent_gen, + backref_ctx); + if (ret < 0) + return ret; + else if (ret > 0) + prealloc_flags |= FIEMAP_EXTENT_SHARED; + } + ret = emit_fiemap_extent(fieinfo, cache, prealloc_start, + disk_bytenr + extent_offset, + prealloc_len, prealloc_flags); + if (ret) + return ret; + } + + return 0; +} + +static int fiemap_find_last_extent_offset(struct btrfs_inode *inode, + struct btrfs_path *path, + u64 *last_extent_end_ret) +{ + const u64 ino = btrfs_ino(inode); + struct btrfs_root *root = inode->root; + struct extent_buffer *leaf; + struct btrfs_file_extent_item *ei; + struct btrfs_key key; + u64 disk_bytenr; + int ret; + + /* + * Lookup the last file extent. We're not using i_size here because + * there might be preallocation past i_size. + */ + ret = btrfs_lookup_file_extent(NULL, root, path, ino, (u64)-1, 0); + /* There can't be a file extent item at offset (u64)-1 */ + ASSERT(ret != 0); + if (ret < 0) + return ret; + + /* + * For a non-existing key, btrfs_search_slot() always leaves us at a + * slot > 0, except if the btree is empty, which is impossible because + * at least it has the inode item for this inode and all the items for + * the root inode 256. + */ + ASSERT(path->slots[0] > 0); + path->slots[0]--; + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY) { + /* No file extent items in the subvolume tree. */ + *last_extent_end_ret = 0; + return 0; + } + + /* + * For an inline extent, the disk_bytenr is where inline data starts at, + * so first check if we have an inline extent item before checking if we + * have an implicit hole (disk_bytenr == 0). + */ + ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); + if (btrfs_file_extent_type(leaf, ei) == BTRFS_FILE_EXTENT_INLINE) { + *last_extent_end_ret = btrfs_file_extent_end(path); + return 0; + } + + /* + * Find the last file extent item that is not a hole (when NO_HOLES is + * not enabled). This should take at most 2 iterations in the worst + * case: we have one hole file extent item at slot 0 of a leaf and + * another hole file extent item as the last item in the previous leaf. + * This is because we merge file extent items that represent holes. + */ + disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei); + while (disk_bytenr == 0) { + ret = btrfs_previous_item(root, path, ino, BTRFS_EXTENT_DATA_KEY); + if (ret < 0) { + return ret; + } else if (ret > 0) { + /* No file extent items that are not holes. */ + *last_extent_end_ret = 0; + return 0; + } + leaf = path->nodes[0]; + ei = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_file_extent_item); + disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei); + } + + *last_extent_end_ret = btrfs_file_extent_end(path); + return 0; +} + +static int extent_fiemap(struct btrfs_inode *inode, + struct fiemap_extent_info *fieinfo, + u64 start, u64 len) +{ + const u64 ino = btrfs_ino(inode); + struct extent_state *cached_state = NULL; + struct extent_state *delalloc_cached_state = NULL; + BTRFS_PATH_AUTO_FREE(path); + struct fiemap_cache cache = { 0 }; + struct btrfs_backref_share_check_ctx *backref_ctx; + u64 last_extent_end = 0; + u64 prev_extent_end; + u64 range_start; + u64 range_end; + const u32 sectorsize = inode->root->fs_info->sectorsize; + bool stopped = false; + int ret; + + cache.entries_size = PAGE_SIZE / sizeof(struct btrfs_fiemap_entry); + cache.entries = kmalloc_objs(struct btrfs_fiemap_entry, + cache.entries_size); + backref_ctx = btrfs_alloc_backref_share_check_ctx(); + path = btrfs_alloc_path(); + if (!cache.entries || !backref_ctx || !path) { + ret = -ENOMEM; + goto out; + } + +restart: + range_start = round_down(start, sectorsize); + range_end = round_up(start + len, sectorsize); + prev_extent_end = range_start; + + btrfs_lock_extent(&inode->io_tree, range_start, range_end - 1, &cached_state); + + ret = fiemap_find_last_extent_offset(inode, path, &last_extent_end); + if (ret < 0) + goto out_unlock; + btrfs_release_path(path); + + path->reada = READA_FORWARD; + ret = fiemap_search_slot(inode, path, range_start); + if (ret < 0) { + goto out_unlock; + } else if (ret > 0) { + /* + * No file extent item found, but we may have delalloc between + * the current offset and i_size. So check for that. + */ + ret = 0; + goto check_eof_delalloc; + } + + while (prev_extent_end < range_end) { + struct extent_buffer *leaf = path->nodes[0]; + struct btrfs_file_extent_item *ei; + struct btrfs_key key; + u64 extent_end; + u64 extent_len; + u64 extent_offset = 0; + u64 extent_gen; + u64 disk_bytenr = 0; + u64 flags = 0; + int extent_type; + u8 compression; + + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY) + break; + + extent_end = btrfs_file_extent_end(path); + + /* + * The first iteration can leave us at an extent item that ends + * before our range's start. Move to the next item. + */ + if (extent_end <= range_start) + goto next_item; + + backref_ctx->curr_leaf_bytenr = leaf->start; + + /* We have in implicit hole (NO_HOLES feature enabled). */ + if (prev_extent_end < key.offset) { + const u64 hole_end = min(key.offset, range_end) - 1; + + ret = fiemap_process_hole(inode, fieinfo, &cache, + &delalloc_cached_state, + backref_ctx, 0, 0, 0, + prev_extent_end, hole_end); + if (ret < 0) { + goto out_unlock; + } else if (ret > 0) { + /* fiemap_fill_next_extent() told us to stop. */ + stopped = true; + break; + } + + /* We've reached the end of the fiemap range, stop. */ + if (key.offset >= range_end) { + stopped = true; + break; + } + } + + extent_len = extent_end - key.offset; + ei = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_file_extent_item); + compression = btrfs_file_extent_compression(leaf, ei); + extent_type = btrfs_file_extent_type(leaf, ei); + extent_gen = btrfs_file_extent_generation(leaf, ei); + + if (extent_type != BTRFS_FILE_EXTENT_INLINE) { + disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei); + if (compression == BTRFS_COMPRESS_NONE) + extent_offset = btrfs_file_extent_offset(leaf, ei); + } + + if (compression != BTRFS_COMPRESS_NONE) + flags |= FIEMAP_EXTENT_ENCODED; + + if (extent_type == BTRFS_FILE_EXTENT_INLINE) { + flags |= FIEMAP_EXTENT_DATA_INLINE; + flags |= FIEMAP_EXTENT_NOT_ALIGNED; + ret = emit_fiemap_extent(fieinfo, &cache, key.offset, 0, + extent_len, flags); + } else if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) { + ret = fiemap_process_hole(inode, fieinfo, &cache, + &delalloc_cached_state, + backref_ctx, + disk_bytenr, extent_offset, + extent_gen, key.offset, + extent_end - 1); + } else if (disk_bytenr == 0) { + /* We have an explicit hole. */ + ret = fiemap_process_hole(inode, fieinfo, &cache, + &delalloc_cached_state, + backref_ctx, 0, 0, 0, + key.offset, extent_end - 1); + } else { + /* We have a regular extent. */ + if (fieinfo->fi_extents_max) { + ret = btrfs_is_data_extent_shared(inode, + disk_bytenr, + extent_gen, + backref_ctx); + if (ret < 0) + goto out_unlock; + else if (ret > 0) + flags |= FIEMAP_EXTENT_SHARED; + } + + ret = emit_fiemap_extent(fieinfo, &cache, key.offset, + disk_bytenr + extent_offset, + extent_len, flags); + } + + if (ret < 0) { + goto out_unlock; + } else if (ret > 0) { + /* emit_fiemap_extent() told us to stop. */ + stopped = true; + break; + } + + prev_extent_end = extent_end; +next_item: + if (fatal_signal_pending(current)) { + ret = -EINTR; + goto out_unlock; + } + + ret = fiemap_next_leaf_item(inode, path); + if (ret < 0) { + goto out_unlock; + } else if (ret > 0) { + /* No more file extent items for this inode. */ + break; + } + cond_resched(); + } + +check_eof_delalloc: + if (!stopped && prev_extent_end < range_end) { + ret = fiemap_process_hole(inode, fieinfo, &cache, + &delalloc_cached_state, backref_ctx, + 0, 0, 0, prev_extent_end, range_end - 1); + if (ret < 0) + goto out_unlock; + prev_extent_end = range_end; + } + + if (cache.cached && cache.offset + cache.len >= last_extent_end) { + const u64 i_size = i_size_read(&inode->vfs_inode); + + if (prev_extent_end < i_size) { + u64 delalloc_start; + u64 delalloc_end; + bool delalloc; + + delalloc = btrfs_find_delalloc_in_range(inode, + prev_extent_end, + i_size - 1, + &delalloc_cached_state, + &delalloc_start, + &delalloc_end); + if (!delalloc) + cache.flags |= FIEMAP_EXTENT_LAST; + } else { + cache.flags |= FIEMAP_EXTENT_LAST; + } + } + +out_unlock: + btrfs_unlock_extent(&inode->io_tree, range_start, range_end - 1, &cached_state); + + if (ret == BTRFS_FIEMAP_FLUSH_CACHE) { + btrfs_release_path(path); + ret = flush_fiemap_cache(fieinfo, &cache); + if (ret) + goto out; + len -= cache.next_search_offset - start; + start = cache.next_search_offset; + goto restart; + } else if (ret < 0) { + goto out; + } + + /* + * Must free the path before emitting to the fiemap buffer because we + * may have a non-cloned leaf and if the fiemap buffer is memory mapped + * to a file, a write into it (through btrfs_page_mkwrite()) may trigger + * waiting for an ordered extent that in order to complete needs to + * modify that leaf, therefore leading to a deadlock. + */ + btrfs_free_path(path); + path = NULL; + + ret = flush_fiemap_cache(fieinfo, &cache); + if (ret) + goto out; + + ret = emit_last_fiemap_cache(fieinfo, &cache); +out: + btrfs_free_extent_state(delalloc_cached_state); + kfree(cache.entries); + btrfs_free_backref_share_ctx(backref_ctx); + return ret; +} + +int btrfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + u64 start, u64 len) +{ + struct btrfs_inode *btrfs_inode = BTRFS_I(inode); + int ret; + + ret = fiemap_prep(inode, fieinfo, start, &len, 0); + if (ret) + return ret; + + /* + * fiemap_prep() called filemap_write_and_wait() for the whole possible + * file range (0 to LLONG_MAX), but that is not enough if we have + * compression enabled. The first filemap_fdatawrite_range() only kicks + * in the compression of data (in an async thread) and will return + * before the compression is done and writeback is started. A second + * filemap_fdatawrite_range() is needed to wait for the compression to + * complete and writeback to start. We also need to wait for ordered + * extents to complete, because our fiemap implementation uses mainly + * file extent items to list the extents, searching for extent maps + * only for file ranges with holes or prealloc extents to figure out + * if we have delalloc in those ranges. + */ + if (fieinfo->fi_flags & FIEMAP_FLAG_SYNC) { + ret = btrfs_wait_ordered_range(btrfs_inode, 0, LLONG_MAX); + if (ret) + return ret; + } + + btrfs_inode_lock(btrfs_inode, BTRFS_ILOCK_SHARED); + + /* + * We did an initial flush to avoid holding the inode's lock while + * triggering writeback and waiting for the completion of IO and ordered + * extents. Now after we locked the inode we do it again, because it's + * possible a new write may have happened in between those two steps. + */ + if (fieinfo->fi_flags & FIEMAP_FLAG_SYNC) { + ret = btrfs_wait_ordered_range(btrfs_inode, 0, LLONG_MAX); + if (ret) { + btrfs_inode_unlock(btrfs_inode, BTRFS_ILOCK_SHARED); + return ret; + } + } + + ret = extent_fiemap(btrfs_inode, fieinfo, start, len); + btrfs_inode_unlock(btrfs_inode, BTRFS_ILOCK_SHARED); + + return ret; +} diff --git a/fs/btrfs/fiemap.h b/fs/btrfs/fiemap.h new file mode 100644 index 000000000..cfd74b359 --- /dev/null +++ b/fs/btrfs/fiemap.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_FIEMAP_H +#define BTRFS_FIEMAP_H + +#include + +int btrfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + u64 start, u64 len); + +#endif /* BTRFS_FIEMAP_H */ diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c new file mode 100644 index 000000000..cf50fd623 --- /dev/null +++ b/fs/btrfs/file-item.c @@ -0,0 +1,1433 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include "messages.h" +#include "ctree.h" +#include "disk-io.h" +#include "transaction.h" +#include "bio.h" +#include "compression.h" +#include "fs.h" +#include "accessors.h" +#include "file-item.h" +#include "volumes.h" + +#define __MAX_CSUM_ITEMS(r, size) ((unsigned long)(((BTRFS_LEAF_DATA_SIZE(r) - \ + sizeof(struct btrfs_item) * 2) / \ + size) - 1)) + +#define MAX_CSUM_ITEMS(r, size) (min_t(u32, __MAX_CSUM_ITEMS(r, size), \ + PAGE_SIZE)) + +/* + * Set inode's size according to filesystem options. + * + * @inode: inode we want to update the disk_i_size for + * @new_i_size: i_size we want to set to, 0 if we use i_size + * + * With NO_HOLES set this simply sets the disk_is_size to whatever i_size_read() + * returns as it is perfectly fine with a file that has holes without hole file + * extent items. + * + * However without NO_HOLES we need to only return the area that is contiguous + * from the 0 offset of the file. Otherwise we could end up adjust i_size up + * to an extent that has a gap in between. + * + * Finally new_i_size should only be set in the case of truncate where we're not + * ready to use i_size_read() as the limiter yet. + */ +void btrfs_inode_safe_disk_i_size_write(struct btrfs_inode *inode, u64 new_i_size) +{ + u64 start, end, i_size; + bool found; + + spin_lock(&inode->lock); + i_size = new_i_size ?: i_size_read(&inode->vfs_inode); + if (!inode->file_extent_tree) { + inode->disk_i_size = i_size; + goto out_unlock; + } + + found = btrfs_find_contiguous_extent_bit(inode->file_extent_tree, 0, &start, + &end, EXTENT_DIRTY); + if (found && start == 0) + i_size = min(i_size, end + 1); + else + i_size = 0; + inode->disk_i_size = i_size; +out_unlock: + spin_unlock(&inode->lock); +} + +/* + * Mark range within a file as having a new extent inserted. + * + * @inode: inode being modified + * @start: start file offset of the file extent we've inserted + * @len: logical length of the file extent item + * + * Call when we are inserting a new file extent where there was none before. + * Does not need to call this in the case where we're replacing an existing file + * extent, however if not sure it's fine to call this multiple times. + * + * The start and len must match the file extent item, so thus must be sectorsize + * aligned. + */ +int btrfs_inode_set_file_extent_range(struct btrfs_inode *inode, u64 start, + u64 len) +{ + if (!inode->file_extent_tree) + return 0; + + if (len == 0) + return 0; + + ASSERT(IS_ALIGNED(start + len, inode->root->fs_info->sectorsize)); + + return btrfs_set_extent_bit(inode->file_extent_tree, start, start + len - 1, + EXTENT_DIRTY, NULL); +} + +/* + * Mark an inode range as not having a backing extent. + * + * @inode: inode being modified + * @start: start file offset of the file extent we've inserted + * @len: logical length of the file extent item + * + * Called when we drop a file extent, for example when we truncate. Doesn't + * need to be called for cases where we're replacing a file extent, like when + * we've COWed a file extent. + * + * The start and len must match the file extent item, so thus must be sectorsize + * aligned. + */ +int btrfs_inode_clear_file_extent_range(struct btrfs_inode *inode, u64 start, + u64 len) +{ + if (!inode->file_extent_tree) + return 0; + + if (len == 0) + return 0; + + ASSERT(IS_ALIGNED(start + len, inode->root->fs_info->sectorsize) || + len == (u64)-1); + + return btrfs_clear_extent_bit(inode->file_extent_tree, start, + start + len - 1, EXTENT_DIRTY, NULL); +} + +static size_t bytes_to_csum_size(const struct btrfs_fs_info *fs_info, u32 bytes) +{ + ASSERT(IS_ALIGNED(bytes, fs_info->sectorsize)); + + return (bytes >> fs_info->sectorsize_bits) * fs_info->csum_size; +} + +static size_t csum_size_to_bytes(const struct btrfs_fs_info *fs_info, u32 csum_size) +{ + ASSERT(IS_ALIGNED(csum_size, fs_info->csum_size)); + + return (csum_size / fs_info->csum_size) << fs_info->sectorsize_bits; +} + +static inline u32 max_ordered_sum_bytes(const struct btrfs_fs_info *fs_info) +{ + u32 max_csum_size = round_down(PAGE_SIZE - sizeof(struct btrfs_ordered_sum), + fs_info->csum_size); + + return csum_size_to_bytes(fs_info, max_csum_size); +} + +/* + * Calculate the total size needed to allocate for an ordered sum structure + * spanning @bytes in the file. + */ +static int btrfs_ordered_sum_size(const struct btrfs_fs_info *fs_info, unsigned long bytes) +{ + return sizeof(struct btrfs_ordered_sum) + bytes_to_csum_size(fs_info, bytes); +} + +int btrfs_insert_hole_extent(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + u64 objectid, u64 pos, u64 num_bytes) +{ + int ret = 0; + struct btrfs_file_extent_item *item; + struct btrfs_key file_key; + BTRFS_PATH_AUTO_FREE(path); + struct extent_buffer *leaf; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + file_key.objectid = objectid; + file_key.type = BTRFS_EXTENT_DATA_KEY; + file_key.offset = pos; + + ret = btrfs_insert_empty_item(trans, root, path, &file_key, + sizeof(*item)); + if (ret < 0) + return ret; + leaf = path->nodes[0]; + item = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_file_extent_item); + btrfs_set_file_extent_disk_bytenr(leaf, item, 0); + btrfs_set_file_extent_disk_num_bytes(leaf, item, 0); + btrfs_set_file_extent_offset(leaf, item, 0); + btrfs_set_file_extent_num_bytes(leaf, item, num_bytes); + btrfs_set_file_extent_ram_bytes(leaf, item, num_bytes); + btrfs_set_file_extent_generation(leaf, item, trans->transid); + btrfs_set_file_extent_type(leaf, item, BTRFS_FILE_EXTENT_REG); + btrfs_set_file_extent_compression(leaf, item, 0); + btrfs_set_file_extent_encryption(leaf, item, 0); + btrfs_set_file_extent_other_encoding(leaf, item, 0); + + return ret; +} + +static struct btrfs_csum_item * +btrfs_lookup_csum(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + u64 bytenr, int cow) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + int ret; + struct btrfs_key file_key; + struct btrfs_key found_key; + struct btrfs_csum_item *item; + struct extent_buffer *leaf; + u64 csum_offset = 0; + const u32 csum_size = fs_info->csum_size; + int csums_in_item; + + file_key.objectid = BTRFS_EXTENT_CSUM_OBJECTID; + file_key.type = BTRFS_EXTENT_CSUM_KEY; + file_key.offset = bytenr; + ret = btrfs_search_slot(trans, root, &file_key, path, 0, cow); + if (ret < 0) + goto fail; + leaf = path->nodes[0]; + if (ret > 0) { + ret = 1; + if (path->slots[0] == 0) + goto fail; + path->slots[0]--; + btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); + if (found_key.type != BTRFS_EXTENT_CSUM_KEY) + goto fail; + + csum_offset = (bytenr - found_key.offset) >> + fs_info->sectorsize_bits; + csums_in_item = btrfs_item_size(leaf, path->slots[0]); + csums_in_item /= csum_size; + + if (csum_offset == csums_in_item) { + ret = -EFBIG; + goto fail; + } else if (csum_offset > csums_in_item) { + goto fail; + } + } + item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_csum_item); + item = (struct btrfs_csum_item *)((unsigned char *)item + + csum_offset * csum_size); + return item; +fail: + if (ret > 0) + ret = -ENOENT; + return ERR_PTR(ret); +} + +int btrfs_lookup_file_extent(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, u64 objectid, + u64 offset, int mod) +{ + struct btrfs_key file_key; + int ins_len = mod < 0 ? -1 : 0; + int cow = mod != 0; + + file_key.objectid = objectid; + file_key.type = BTRFS_EXTENT_DATA_KEY; + file_key.offset = offset; + + return btrfs_search_slot(trans, root, &file_key, path, ins_len, cow); +} + +/* + * Find checksums for logical bytenr range [disk_bytenr, disk_bytenr + len) and + * store the result to @dst. + * + * Return >0 for the number of sectors we found. + * Return 0 for the range [disk_bytenr, disk_bytenr + sectorsize) has no csum + * for it. Caller may want to try next sector until one range is hit. + * Return <0 for fatal error. + */ +static int search_csum_tree(struct btrfs_fs_info *fs_info, + struct btrfs_path *path, u64 disk_bytenr, + u64 len, u8 *dst) +{ + struct btrfs_root *csum_root; + struct btrfs_csum_item *item = NULL; + struct btrfs_key key; + const u32 sectorsize = fs_info->sectorsize; + const u32 csum_size = fs_info->csum_size; + u32 itemsize; + int ret; + u64 csum_start; + u64 csum_len; + + ASSERT(IS_ALIGNED(disk_bytenr, sectorsize) && + IS_ALIGNED(len, sectorsize)); + + /* Check if the current csum item covers disk_bytenr */ + if (path->nodes[0]) { + item = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_csum_item); + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + itemsize = btrfs_item_size(path->nodes[0], path->slots[0]); + + csum_start = key.offset; + csum_len = (itemsize / csum_size) * sectorsize; + + if (in_range(disk_bytenr, csum_start, csum_len)) + goto found; + } + + /* Current item doesn't contain the desired range, search again */ + btrfs_release_path(path); + csum_root = btrfs_csum_root(fs_info, disk_bytenr); + if (unlikely(!csum_root)) { + btrfs_err(fs_info, + "missing csum root for extent at bytenr %llu", + disk_bytenr); + return -EUCLEAN; + } + + item = btrfs_lookup_csum(NULL, csum_root, path, disk_bytenr, 0); + if (IS_ERR(item)) { + ret = PTR_ERR(item); + goto out; + } + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + itemsize = btrfs_item_size(path->nodes[0], path->slots[0]); + + csum_start = key.offset; + csum_len = (itemsize / csum_size) * sectorsize; + ASSERT(in_range(disk_bytenr, csum_start, csum_len), + "disk_bytenr=%llu csum_start=%llu csum_len=%llu", + disk_bytenr, csum_start, csum_len); + +found: + ret = (min(csum_start + csum_len, disk_bytenr + len) - + disk_bytenr) >> fs_info->sectorsize_bits; + read_extent_buffer(path->nodes[0], dst, (unsigned long)item, + ret * csum_size); +out: + if (ret == -ENOENT || ret == -EFBIG) + ret = 0; + return ret; +} + +/* + * Lookup the checksum for the read bio in csum tree. + * + * Return: BLK_STS_RESOURCE if allocating memory fails, BLK_STS_OK otherwise. + */ +int btrfs_lookup_bio_sums(struct btrfs_bio *bbio) +{ + struct btrfs_inode *inode = bbio->inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct bio *bio = &bbio->bio; + BTRFS_PATH_AUTO_FREE(path); + const u32 sectorsize = fs_info->sectorsize; + const u32 csum_size = fs_info->csum_size; + u32 orig_len = bio->bi_iter.bi_size; + u64 orig_disk_bytenr = bio->bi_iter.bi_sector << SECTOR_SHIFT; + const unsigned int nblocks = orig_len >> fs_info->sectorsize_bits; + int ret = 0; + u32 bio_offset = 0; + bool using_commit_root = false; + + if ((inode->flags & BTRFS_INODE_NODATASUM) || + test_bit(BTRFS_FS_STATE_NO_DATA_CSUMS, &fs_info->fs_state)) + return 0; + + /* + * This function is only called for read bio. + * + * This means two things: + * - All our csums should only be in csum tree + * No ordered extents csums, as ordered extents are only for write + * path. + * - No need to bother any other info from bvec + * Since we're looking up csums, the only important info is the + * disk_bytenr and the length, which can be extracted from bi_iter + * directly. + */ + ASSERT(bio_op(bio) == REQ_OP_READ); + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + if (nblocks * csum_size > BTRFS_BIO_INLINE_CSUM_SIZE) { + bbio->csum = kvcalloc(nblocks, csum_size, GFP_NOFS); + if (!bbio->csum) + return -ENOMEM; + } else { + bbio->csum = bbio->csum_inline; + } + + /* + * If requested number of sectors is larger than one leaf can contain, + * kick the readahead for csum tree. + */ + if (nblocks > fs_info->csums_per_leaf) + path->reada = READA_FORWARD; + + /* + * the free space stuff is only read when it hasn't been + * updated in the current transaction. So, we can safely + * read from the commit root and sidestep a nasty deadlock + * between reading the free space cache and updating the csum tree. + */ + if (btrfs_is_free_space_inode(inode)) { + path->search_commit_root = true; + path->skip_locking = true; + } + + /* + * If we are searching for a csum of an extent from a past + * transaction, we can search in the commit root and reduce + * lock contention on the csum tree extent buffers. + * + * This is important because that lock is an rwsem which gets + * pretty heavy write load under memory pressure and sustained + * csum overwrites, unlike the commit_root_sem. (Memory pressure + * makes us writeback the nodes multiple times per transaction, + * which makes us cow them each time, taking the write lock.) + * + * Due to how rwsem is implemented, there is a possible + * priority inversion where the readers holding the lock don't + * get scheduled (say they're in a cgroup stuck in heavy reclaim) + * which then blocks writers, including transaction commit. By + * using a semaphore with fewer writers (only a commit switching + * the roots), we make this issue less likely. + * + * Note that we don't rely on btrfs_search_slot to lock the + * commit root csum. We call search_slot multiple times, which would + * create a potential race where a commit comes in between searches + * while we are not holding the commit_root_sem, and we get csums + * from across transactions. + */ + if (bbio->csum_search_commit_root) { + using_commit_root = true; + path->search_commit_root = true; + path->skip_locking = true; + down_read(&fs_info->commit_root_sem); + } + + while (bio_offset < orig_len) { + int count; + u64 cur_disk_bytenr = orig_disk_bytenr + bio_offset; + u8 *csum_dst = bbio->csum + + (bio_offset >> fs_info->sectorsize_bits) * csum_size; + + count = search_csum_tree(fs_info, path, cur_disk_bytenr, + orig_len - bio_offset, csum_dst); + if (count < 0) { + ret = count; + if (bbio->csum != bbio->csum_inline) + kvfree(bbio->csum); + bbio->csum = NULL; + break; + } + + /* + * We didn't find a csum for this range. We need to make sure + * we complain loudly about this, because we are not NODATASUM. + * + * However for the DATA_RELOC inode we could potentially be + * relocating data extents for a NODATASUM inode, so the inode + * itself won't be marked with NODATASUM, but the extent we're + * copying is in fact NODATASUM. If we don't find a csum we + * assume this is the case. + */ + if (count == 0) { + /* + * If an extent is relocated in the current transaction + * then relocation writes a new csum without updating + * the extent map generation. Until the next commit, we + * will see a hole in that case, so we need to fallback + * to searching the transaction csum root. + * + * Note that a commit root lookup of a referenced extent can + * only miss, not return a stale csum. A freed extent's csum + * is deleted in the same transaction and its bytenr is not + * reusable until that transaction has committed and the + * extent is unpinned. + */ + if (using_commit_root) { + up_read(&fs_info->commit_root_sem); + using_commit_root = false; + path->search_commit_root = false; + path->skip_locking = false; + btrfs_release_path(path); + continue; + } + + memset(csum_dst, 0, csum_size); + count = 1; + + if (btrfs_is_data_reloc_root(inode->root)) { + u64 file_offset = bbio->file_offset + bio_offset; + + btrfs_set_extent_bit(&inode->io_tree, file_offset, + file_offset + sectorsize - 1, + EXTENT_NODATASUM, NULL); + } else { + btrfs_warn_rl(fs_info, + "csum hole found for disk bytenr range [%llu, %llu)", + cur_disk_bytenr, cur_disk_bytenr + sectorsize); + } + } + bio_offset += count * sectorsize; + } + + if (using_commit_root) + up_read(&fs_info->commit_root_sem); + return ret; +} + +/* + * Search for checksums for a given logical range. + * + * @root: The root where to look for checksums. + * @start: Logical address of target checksum range. + * @end: End offset (inclusive) of the target checksum range. + * @list: List for adding each checksum that was found. + * Can be NULL in case the caller only wants to check if + * there any checksums for the range. + * @nowait: Indicate if the search must be non-blocking or not. + * + * Return < 0 on error, 0 if no checksums were found, or 1 if checksums were + * found. + */ +int btrfs_lookup_csums_list(struct btrfs_root *root, u64 start, u64 end, + struct list_head *list, bool nowait) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_key key; + struct btrfs_path *path; + struct extent_buffer *leaf; + struct btrfs_ordered_sum *sums; + struct btrfs_csum_item *item; + int ret; + bool found_csums = false; + + ASSERT(IS_ALIGNED(start, fs_info->sectorsize) && + IS_ALIGNED(end + 1, fs_info->sectorsize)); + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + path->nowait = nowait; + + key.objectid = BTRFS_EXTENT_CSUM_OBJECTID; + key.type = BTRFS_EXTENT_CSUM_KEY; + key.offset = start; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + goto out; + if (ret > 0 && path->slots[0] > 0) { + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1); + + /* + * There are two cases we can hit here for the previous csum + * item: + * + * |<- search range ->| + * |<- csum item ->| + * + * Or + * |<- search range ->| + * |<- csum item ->| + * + * Check if the previous csum item covers the leading part of + * the search range. If so we have to start from previous csum + * item. + */ + if (key.objectid == BTRFS_EXTENT_CSUM_OBJECTID && + key.type == BTRFS_EXTENT_CSUM_KEY) { + if (bytes_to_csum_size(fs_info, start - key.offset) < + btrfs_item_size(leaf, path->slots[0] - 1)) + path->slots[0]--; + } + } + + while (start <= end) { + u64 csum_end; + + leaf = path->nodes[0]; + if (path->slots[0] >= btrfs_header_nritems(leaf)) { + ret = btrfs_next_leaf(root, path); + if (ret < 0) + goto out; + if (ret > 0) + break; + leaf = path->nodes[0]; + } + + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + if (key.objectid != BTRFS_EXTENT_CSUM_OBJECTID || + key.type != BTRFS_EXTENT_CSUM_KEY || + key.offset > end) + break; + + if (key.offset > start) + start = key.offset; + + csum_end = key.offset + csum_size_to_bytes(fs_info, + btrfs_item_size(leaf, path->slots[0])); + if (csum_end <= start) { + path->slots[0]++; + continue; + } + + found_csums = true; + if (!list) + goto out; + + csum_end = min(csum_end, end + 1); + item = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_csum_item); + while (start < csum_end) { + unsigned long offset; + size_t size; + + size = min_t(size_t, csum_end - start, + max_ordered_sum_bytes(fs_info)); + sums = kzalloc(btrfs_ordered_sum_size(fs_info, size), + GFP_NOFS); + if (!sums) { + ret = -ENOMEM; + goto out; + } + + sums->logical = start; + sums->len = size; + + offset = bytes_to_csum_size(fs_info, start - key.offset); + + read_extent_buffer(path->nodes[0], + sums->sums, + ((unsigned long)item) + offset, + bytes_to_csum_size(fs_info, size)); + + start += size; + list_add_tail(&sums->list, list); + } + path->slots[0]++; + } +out: + btrfs_free_path(path); + if (ret < 0) { + if (list) { + struct btrfs_ordered_sum *tmp_sums; + + list_for_each_entry_safe(sums, tmp_sums, list, list) + kfree(sums); + } + + return ret; + } + + return found_csums ? 1 : 0; +} + +/* + * Do the same work as btrfs_lookup_csums_list(), the difference is in how + * we return the result. + * + * This version will set the corresponding bits in @csum_bitmap to represent + * that there is a csum found. + * Each bit represents a sector. Thus caller should ensure @csum_buf passed + * in is large enough to contain all csums. + */ +int btrfs_lookup_csums_bitmap(struct btrfs_root *root, struct btrfs_path *path, + u64 start, u64 end, u8 *csum_buf, + unsigned long *csum_bitmap) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_key key; + struct extent_buffer *leaf; + struct btrfs_csum_item *item; + const u64 orig_start = start; + bool free_path = false; + int ret; + + ASSERT(IS_ALIGNED(start, fs_info->sectorsize) && + IS_ALIGNED(end + 1, fs_info->sectorsize)); + + if (!path) { + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + free_path = true; + } + + /* Check if we can reuse the previous path. */ + if (path->nodes[0]) { + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + + if (key.objectid == BTRFS_EXTENT_CSUM_OBJECTID && + key.type == BTRFS_EXTENT_CSUM_KEY && + key.offset <= start) + goto search_forward; + btrfs_release_path(path); + } + + key.objectid = BTRFS_EXTENT_CSUM_OBJECTID; + key.type = BTRFS_EXTENT_CSUM_KEY; + key.offset = start; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + goto fail; + if (ret > 0 && path->slots[0] > 0) { + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1); + + /* + * There are two cases we can hit here for the previous csum + * item: + * + * |<- search range ->| + * |<- csum item ->| + * + * Or + * |<- search range ->| + * |<- csum item ->| + * + * Check if the previous csum item covers the leading part of + * the search range. If so we have to start from previous csum + * item. + */ + if (key.objectid == BTRFS_EXTENT_CSUM_OBJECTID && + key.type == BTRFS_EXTENT_CSUM_KEY) { + if (bytes_to_csum_size(fs_info, start - key.offset) < + btrfs_item_size(leaf, path->slots[0] - 1)) + path->slots[0]--; + } + } + +search_forward: + while (start <= end) { + u64 csum_end; + + leaf = path->nodes[0]; + if (path->slots[0] >= btrfs_header_nritems(leaf)) { + ret = btrfs_next_leaf(root, path); + if (ret < 0) + goto fail; + if (ret > 0) + break; + leaf = path->nodes[0]; + } + + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + if (key.objectid != BTRFS_EXTENT_CSUM_OBJECTID || + key.type != BTRFS_EXTENT_CSUM_KEY || + key.offset > end) + break; + + if (key.offset > start) + start = key.offset; + + csum_end = key.offset + csum_size_to_bytes(fs_info, + btrfs_item_size(leaf, path->slots[0])); + if (csum_end <= start) { + path->slots[0]++; + continue; + } + + csum_end = min(csum_end, end + 1); + item = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_csum_item); + while (start < csum_end) { + unsigned long offset; + size_t size; + u8 *csum_dest = csum_buf + bytes_to_csum_size(fs_info, + start - orig_start); + + size = min_t(size_t, csum_end - start, end + 1 - start); + + offset = bytes_to_csum_size(fs_info, start - key.offset); + + read_extent_buffer(path->nodes[0], csum_dest, + ((unsigned long)item) + offset, + bytes_to_csum_size(fs_info, size)); + + bitmap_set(csum_bitmap, + (start - orig_start) >> fs_info->sectorsize_bits, + size >> fs_info->sectorsize_bits); + + start += size; + } + path->slots[0]++; + } + ret = 0; +fail: + if (free_path) + btrfs_free_path(path); + return ret; +} + +static void csum_one_bio(struct btrfs_bio *bbio, struct bvec_iter *src) +{ + struct btrfs_inode *inode = bbio->inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct bio *bio = &bbio->bio; + struct btrfs_ordered_sum *sums = bbio->sums; + struct bvec_iter iter = *src; + phys_addr_t paddr; + const u32 blocksize = fs_info->sectorsize; + const u32 step = min(blocksize, PAGE_SIZE); + const u32 nr_steps = blocksize / step; + phys_addr_t paddrs[BTRFS_MAX_BLOCKSIZE / PAGE_SIZE]; + u32 offset = 0; + int index = 0; + + btrfs_bio_for_each_block(paddr, bio, &iter, step) { + paddrs[(offset / step) % nr_steps] = paddr; + offset += step; + + if (IS_ALIGNED(offset, blocksize)) { + btrfs_calculate_block_csum_pages(fs_info, paddrs, sums->sums + index); + index += fs_info->csum_size; + } + } +} + +static void csum_one_bio_work(struct work_struct *work) +{ + struct btrfs_bio *bbio = container_of(work, struct btrfs_bio, csum_work); + + ASSERT(btrfs_op(&bbio->bio) == BTRFS_MAP_WRITE); + ASSERT(bbio->async_csum == true); + csum_one_bio(bbio, &bbio->csum_saved_iter); + complete(&bbio->csum_done); +} + +/* + * Calculate checksums of the data contained inside a bio. + */ +int btrfs_csum_one_bio(struct btrfs_bio *bbio, bool async) +{ + struct btrfs_ordered_extent *ordered = bbio->ordered; + struct btrfs_inode *inode = bbio->inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct bio *bio = &bbio->bio; + struct btrfs_ordered_sum *sums; + unsigned nofs_flag; + + nofs_flag = memalloc_nofs_save(); + sums = kvzalloc(btrfs_ordered_sum_size(fs_info, bio->bi_iter.bi_size), + GFP_KERNEL); + memalloc_nofs_restore(nofs_flag); + + if (!sums) + return -ENOMEM; + + sums->logical = bbio->orig_logical; + sums->len = bio->bi_iter.bi_size; + INIT_LIST_HEAD(&sums->list); + bbio->sums = sums; + btrfs_add_ordered_sum(ordered, sums); + + if (!async) { + csum_one_bio(bbio, &bbio->bio.bi_iter); + return 0; + } + init_completion(&bbio->csum_done); + bbio->async_csum = true; + bbio->csum_saved_iter = bbio->bio.bi_iter; + INIT_WORK(&bbio->csum_work, csum_one_bio_work); + schedule_work(&bbio->csum_work); + return 0; +} + +/* + * Nodatasum I/O on zoned file systems still requires an btrfs_ordered_sum to + * record the updated logical address on Zone Append completion. + * Allocate just the structure with an empty sums array here for that case. + */ +int btrfs_alloc_dummy_sum(struct btrfs_bio *bbio) +{ + bbio->sums = kmalloc_obj(*bbio->sums, GFP_NOFS); + if (!bbio->sums) + return -ENOMEM; + bbio->sums->len = bbio->bio.bi_iter.bi_size; + bbio->sums->logical = bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT; + btrfs_add_ordered_sum(bbio->ordered, bbio->sums); + return 0; +} + +/* + * Remove one checksum overlapping a range. + * + * This expects the key to describe the csum pointed to by the path, and it + * expects the csum to overlap the range [bytenr, len] + * + * The csum should not be entirely contained in the range and the range should + * not be entirely contained in the csum. + * + * This calls btrfs_truncate_item with the correct args based on the overlap, + * and fixes up the key as required. + */ +static noinline void truncate_one_csum(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + struct btrfs_key *key, + u64 bytenr, u64 len) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct extent_buffer *leaf; + const u32 csum_size = fs_info->csum_size; + u64 csum_end; + u64 end_byte = bytenr + len; + u32 blocksize_bits = fs_info->sectorsize_bits; + + leaf = path->nodes[0]; + csum_end = btrfs_item_size(leaf, path->slots[0]) / csum_size; + csum_end <<= blocksize_bits; + csum_end += key->offset; + + if (key->offset < bytenr && csum_end <= end_byte) { + /* + * [ bytenr - len ] + * [ ] + * [csum ] + * A simple truncate off the end of the item + */ + u32 new_size = (bytenr - key->offset) >> blocksize_bits; + new_size *= csum_size; + btrfs_truncate_item(trans, path, new_size, 1); + } else if (key->offset >= bytenr && csum_end > end_byte && + end_byte > key->offset) { + /* + * [ bytenr - len ] + * [ ] + * [csum ] + * we need to truncate from the beginning of the csum + */ + u32 new_size = (csum_end - end_byte) >> blocksize_bits; + new_size *= csum_size; + + btrfs_truncate_item(trans, path, new_size, 0); + + key->offset = end_byte; + btrfs_set_item_key_safe(trans, path, key); + } else { + BUG(); + } +} + +/* + * Delete the csum items from the csum tree for a given range of bytes. + */ +int btrfs_del_csums(struct btrfs_trans_handle *trans, + struct btrfs_root *root, u64 bytenr, u64 len) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + u64 end_byte = bytenr + len; + u64 csum_end; + struct extent_buffer *leaf; + int ret = 0; + const u32 csum_size = fs_info->csum_size; + u32 blocksize_bits = fs_info->sectorsize_bits; + + ASSERT(btrfs_root_id(root) == BTRFS_CSUM_TREE_OBJECTID || + btrfs_root_id(root) == BTRFS_TREE_LOG_OBJECTID); + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + while (1) { + key.objectid = BTRFS_EXTENT_CSUM_OBJECTID; + key.type = BTRFS_EXTENT_CSUM_KEY; + key.offset = end_byte - 1; + + ret = btrfs_search_slot(trans, root, &key, path, -1, 1); + if (ret > 0) { + ret = 0; + if (path->slots[0] == 0) + break; + path->slots[0]--; + } else if (ret < 0) { + break; + } + + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + + if (key.objectid != BTRFS_EXTENT_CSUM_OBJECTID || + key.type != BTRFS_EXTENT_CSUM_KEY) { + break; + } + + if (key.offset >= end_byte) + break; + + csum_end = btrfs_item_size(leaf, path->slots[0]) / csum_size; + csum_end <<= blocksize_bits; + csum_end += key.offset; + + /* this csum ends before we start, we're done */ + if (csum_end <= bytenr) + break; + + /* delete the entire item, it is inside our range */ + if (key.offset >= bytenr && csum_end <= end_byte) { + int del_nr = 1; + + /* + * Check how many csum items preceding this one in this + * leaf correspond to our range and then delete them all + * at once. + */ + if (key.offset > bytenr && path->slots[0] > 0) { + int slot = path->slots[0] - 1; + + while (slot >= 0) { + struct btrfs_key pk; + + btrfs_item_key_to_cpu(leaf, &pk, slot); + if (pk.offset < bytenr || + pk.type != BTRFS_EXTENT_CSUM_KEY || + pk.objectid != + BTRFS_EXTENT_CSUM_OBJECTID) + break; + path->slots[0] = slot; + del_nr++; + key.offset = pk.offset; + slot--; + } + } + ret = btrfs_del_items(trans, root, path, + path->slots[0], del_nr); + if (ret) + break; + if (key.offset == bytenr) + break; + } else if (key.offset < bytenr && csum_end > end_byte) { + unsigned long offset; + unsigned long shift_len; + unsigned long item_offset; + /* + * [ bytenr - len ] + * [csum ] + * + * Our bytes are in the middle of the csum, + * we need to split this item and insert a new one. + * + * But we can't drop the path because the + * csum could change, get removed, extended etc. + * + * The trick here is the max size of a csum item leaves + * enough room in the tree block for a single + * item header. So, we split the item in place, + * adding a new header pointing to the existing + * bytes. Then we loop around again and we have + * a nicely formed csum item that we can neatly + * truncate. + */ + offset = (bytenr - key.offset) >> blocksize_bits; + offset *= csum_size; + + shift_len = (len >> blocksize_bits) * csum_size; + + item_offset = btrfs_item_ptr_offset(leaf, + path->slots[0]); + + memzero_extent_buffer(leaf, item_offset + offset, + shift_len); + key.offset = bytenr; + + /* + * btrfs_split_item returns -EAGAIN when the + * item changed size or key + */ + ret = btrfs_split_item(trans, root, path, &key, offset); + if (unlikely(ret && ret != -EAGAIN)) { + btrfs_abort_transaction(trans, ret); + break; + } + ret = 0; + + key.offset = end_byte - 1; + } else { + truncate_one_csum(trans, path, &key, bytenr, len); + if (key.offset < bytenr) + break; + } + btrfs_release_path(path); + } + return ret; +} + +static int find_next_csum_offset(struct btrfs_root *root, + struct btrfs_path *path, + u64 *next_offset) +{ + const u32 nritems = btrfs_header_nritems(path->nodes[0]); + struct btrfs_key found_key; + int slot = path->slots[0] + 1; + int ret; + + if (nritems == 0 || slot >= nritems) { + ret = btrfs_next_leaf(root, path); + if (ret < 0) { + return ret; + } else if (ret > 0) { + *next_offset = (u64)-1; + return 0; + } + slot = path->slots[0]; + } + + btrfs_item_key_to_cpu(path->nodes[0], &found_key, slot); + + if (found_key.objectid != BTRFS_EXTENT_CSUM_OBJECTID || + found_key.type != BTRFS_EXTENT_CSUM_KEY) + *next_offset = (u64)-1; + else + *next_offset = found_key.offset; + + return 0; +} + +int btrfs_insert_data_csums(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_ordered_sum *sums) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_key file_key; + struct btrfs_key found_key; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_csum_item *item; + struct btrfs_csum_item *item_end; + struct extent_buffer *leaf = NULL; + u64 next_offset; + u64 total_bytes = 0; + u64 csum_offset; + u64 bytenr; + u32 ins_size; + int index = 0; + int found_next; + int ret; + const u32 csum_size = fs_info->csum_size; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; +again: + next_offset = (u64)-1; + found_next = 0; + bytenr = sums->logical + total_bytes; + file_key.objectid = BTRFS_EXTENT_CSUM_OBJECTID; + file_key.type = BTRFS_EXTENT_CSUM_KEY; + file_key.offset = bytenr; + + item = btrfs_lookup_csum(trans, root, path, bytenr, 1); + if (!IS_ERR(item)) { + ret = 0; + leaf = path->nodes[0]; + item_end = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_csum_item); + item_end = (struct btrfs_csum_item *)((char *)item_end + + btrfs_item_size(leaf, path->slots[0])); + goto found; + } + ret = PTR_ERR(item); + if (ret != -EFBIG && ret != -ENOENT) + return ret; + + if (ret == -EFBIG) { + u32 item_size; + /* we found one, but it isn't big enough yet */ + leaf = path->nodes[0]; + item_size = btrfs_item_size(leaf, path->slots[0]); + if ((item_size / csum_size) >= + MAX_CSUM_ITEMS(fs_info, csum_size)) { + /* already at max size, make a new one */ + goto insert; + } + } else { + /* We didn't find a csum item, insert one. */ + ret = find_next_csum_offset(root, path, &next_offset); + if (ret < 0) + return ret; + found_next = 1; + goto insert; + } + + /* + * At this point, we know the tree has a checksum item that ends at an + * offset matching the start of the checksum range we want to insert. + * We try to extend that item as much as possible and then add as many + * checksums to it as they fit. + * + * First check if the leaf has enough free space for at least one + * checksum. If it has go directly to the item extension code, otherwise + * release the path and do a search for insertion before the extension. + */ + if (btrfs_leaf_free_space(leaf) >= csum_size) { + btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); + csum_offset = (bytenr - found_key.offset) >> + fs_info->sectorsize_bits; + goto extend_csum; + } + + btrfs_release_path(path); + path->search_for_extension = true; + ret = btrfs_search_slot(trans, root, &file_key, path, + csum_size, 1); + path->search_for_extension = false; + if (ret < 0) + return ret; + + if (ret > 0) { + if (path->slots[0] == 0) + goto insert; + path->slots[0]--; + } + + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); + csum_offset = (bytenr - found_key.offset) >> fs_info->sectorsize_bits; + + if (found_key.type != BTRFS_EXTENT_CSUM_KEY || + found_key.objectid != BTRFS_EXTENT_CSUM_OBJECTID || + csum_offset >= MAX_CSUM_ITEMS(fs_info, csum_size)) { + goto insert; + } + +extend_csum: + if (csum_offset == btrfs_item_size(leaf, path->slots[0]) / + csum_size) { + int extend_nr; + u64 tmp; + u32 diff; + + tmp = sums->len - total_bytes; + tmp >>= fs_info->sectorsize_bits; + WARN_ON(tmp < 1); + extend_nr = max_t(int, 1, tmp); + + /* + * A log tree can already have checksum items with a subset of + * the checksums we are trying to log. This can happen after + * doing a sequence of partial writes into prealloc extents and + * fsyncs in between, with a full fsync logging a larger subrange + * of an extent for which a previous fast fsync logged a smaller + * subrange. And this happens in particular due to merging file + * extent items when we complete an ordered extent for a range + * covered by a prealloc extent - this is done at + * btrfs_mark_extent_written(). + * + * So if we try to extend the previous checksum item, which has + * a range that ends at the start of the range we want to insert, + * make sure we don't extend beyond the start offset of the next + * checksum item. If we are at the last item in the leaf, then + * forget the optimization of extending and add a new checksum + * item - it is not worth the complexity of releasing the path, + * getting the first key for the next leaf, repeat the btree + * search, etc, because log trees are temporary anyway and it + * would only save a few bytes of leaf space. + */ + if (btrfs_root_id(root) == BTRFS_TREE_LOG_OBJECTID) { + if (path->slots[0] + 1 >= + btrfs_header_nritems(path->nodes[0])) { + ret = find_next_csum_offset(root, path, &next_offset); + if (ret < 0) + return ret; + found_next = 1; + goto insert; + } + + ret = find_next_csum_offset(root, path, &next_offset); + if (ret < 0) + return ret; + + tmp = (next_offset - bytenr) >> fs_info->sectorsize_bits; + if (tmp <= INT_MAX) + extend_nr = min_t(int, extend_nr, tmp); + } + + diff = (csum_offset + extend_nr) * csum_size; + diff = min(diff, + MAX_CSUM_ITEMS(fs_info, csum_size) * csum_size); + + diff = diff - btrfs_item_size(leaf, path->slots[0]); + diff = min_t(u32, btrfs_leaf_free_space(leaf), diff); + diff /= csum_size; + diff *= csum_size; + + btrfs_extend_item(trans, path, diff); + ret = 0; + goto csum; + } + +insert: + btrfs_release_path(path); + csum_offset = 0; + if (found_next) { + u64 tmp; + + tmp = sums->len - total_bytes; + tmp >>= fs_info->sectorsize_bits; + tmp = min(tmp, (next_offset - file_key.offset) >> + fs_info->sectorsize_bits); + + tmp = max_t(u64, 1, tmp); + tmp = min_t(u64, tmp, MAX_CSUM_ITEMS(fs_info, csum_size)); + ins_size = csum_size * tmp; + } else { + ins_size = csum_size; + } + ret = btrfs_insert_empty_item(trans, root, path, &file_key, + ins_size); + if (ret < 0) + return ret; + leaf = path->nodes[0]; +csum: + item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_csum_item); + item_end = (struct btrfs_csum_item *)((unsigned char *)item + + btrfs_item_size(leaf, path->slots[0])); + item = (struct btrfs_csum_item *)((unsigned char *)item + + csum_offset * csum_size); +found: + ins_size = (u32)(sums->len - total_bytes) >> fs_info->sectorsize_bits; + ins_size *= csum_size; + ins_size = min_t(u32, (unsigned long)item_end - (unsigned long)item, + ins_size); + write_extent_buffer(leaf, sums->sums + index, (unsigned long)item, + ins_size); + + index += ins_size; + ins_size /= csum_size; + total_bytes += (ins_size << fs_info->sectorsize_bits); + + if (total_bytes < sums->len) { + btrfs_release_path(path); + cond_resched(); + goto again; + } + + return 0; +} + +void btrfs_extent_item_to_extent_map(struct btrfs_inode *inode, + const struct btrfs_path *path, + const struct btrfs_file_extent_item *fi, + struct extent_map *em) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct btrfs_root *root = inode->root; + struct extent_buffer *leaf = path->nodes[0]; + const int slot = path->slots[0]; + struct btrfs_key key; + u64 extent_start; + u8 type = btrfs_file_extent_type(leaf, fi); + int compress_type = btrfs_file_extent_compression(leaf, fi); + + btrfs_item_key_to_cpu(leaf, &key, slot); + extent_start = key.offset; + em->ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi); + em->generation = btrfs_file_extent_generation(leaf, fi); + if (type == BTRFS_FILE_EXTENT_REG || + type == BTRFS_FILE_EXTENT_PREALLOC) { + const u64 disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi); + + em->start = extent_start; + em->len = btrfs_file_extent_end(path) - extent_start; + if (disk_bytenr == 0) { + em->disk_bytenr = EXTENT_MAP_HOLE; + em->disk_num_bytes = 0; + em->offset = 0; + return; + } + em->disk_bytenr = disk_bytenr; + em->disk_num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi); + em->offset = btrfs_file_extent_offset(leaf, fi); + if (compress_type != BTRFS_COMPRESS_NONE) { + btrfs_extent_map_set_compression(em, compress_type); + } else { + /* + * Older kernels can create regular non-hole data + * extents with ram_bytes smaller than disk_num_bytes. + * Not a big deal, just always use disk_num_bytes + * for ram_bytes. + */ + em->ram_bytes = em->disk_num_bytes; + if (type == BTRFS_FILE_EXTENT_PREALLOC) + em->flags |= EXTENT_FLAG_PREALLOC; + } + } else if (type == BTRFS_FILE_EXTENT_INLINE) { + /* Tree-checker has ensured this. */ + ASSERT(extent_start == 0); + + em->disk_bytenr = EXTENT_MAP_INLINE; + em->start = 0; + em->len = fs_info->sectorsize; + em->offset = 0; + btrfs_extent_map_set_compression(em, compress_type); + } else { + btrfs_err(fs_info, + "unknown file extent item type %d, inode %llu, offset %llu, " + "root %llu", type, btrfs_ino(inode), extent_start, + btrfs_root_id(root)); + } +} + +/* + * Returns the end offset (non inclusive) of the file extent item the given path + * points to. If it points to an inline extent, the returned offset is rounded + * up to the sector size. + */ +u64 btrfs_file_extent_end(const struct btrfs_path *path) +{ + const struct extent_buffer *leaf = path->nodes[0]; + const int slot = path->slots[0]; + struct btrfs_file_extent_item *fi; + struct btrfs_key key; + u64 end; + + btrfs_item_key_to_cpu(leaf, &key, slot); + ASSERT(key.type == BTRFS_EXTENT_DATA_KEY); + fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item); + + if (btrfs_file_extent_type(leaf, fi) == BTRFS_FILE_EXTENT_INLINE) + end = leaf->fs_info->sectorsize; + else + end = key.offset + btrfs_file_extent_num_bytes(leaf, fi); + + return end; +} diff --git a/fs/btrfs/file-item.h b/fs/btrfs/file-item.h new file mode 100644 index 000000000..6c678787c --- /dev/null +++ b/fs/btrfs/file-item.h @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_FILE_ITEM_H +#define BTRFS_FILE_ITEM_H + +#include +#include +#include +#include "ctree.h" +#include "ordered-data.h" + +struct extent_map; +struct btrfs_file_extent_item; +struct btrfs_fs_info; +struct btrfs_path; +struct btrfs_bio; +struct btrfs_trans_handle; +struct btrfs_root; +struct btrfs_ordered_sum; +struct btrfs_path; +struct btrfs_inode; + +#define BTRFS_FILE_EXTENT_INLINE_DATA_START \ + (offsetof(struct btrfs_file_extent_item, disk_bytenr)) + +static inline u32 BTRFS_MAX_INLINE_DATA_SIZE(const struct btrfs_fs_info *info) +{ + return BTRFS_MAX_ITEM_SIZE(info) - BTRFS_FILE_EXTENT_INLINE_DATA_START; +} + +/* + * Return the number of bytes used by the item on disk, minus the size of any + * extent headers. If a file is compressed on disk, this is the compressed + * size. + */ +static inline u32 btrfs_file_extent_inline_item_len( + const struct extent_buffer *eb, + int nr) +{ + return btrfs_item_size(eb, nr) - BTRFS_FILE_EXTENT_INLINE_DATA_START; +} + +static inline unsigned long btrfs_file_extent_inline_start( + const struct btrfs_file_extent_item *e) +{ + return (unsigned long)e + BTRFS_FILE_EXTENT_INLINE_DATA_START; +} + +static inline u32 btrfs_file_extent_calc_inline_size(u32 datasize) +{ + return BTRFS_FILE_EXTENT_INLINE_DATA_START + datasize; +} + +int btrfs_del_csums(struct btrfs_trans_handle *trans, + struct btrfs_root *root, u64 bytenr, u64 len); +int btrfs_lookup_bio_sums(struct btrfs_bio *bbio); +int btrfs_insert_hole_extent(struct btrfs_trans_handle *trans, + struct btrfs_root *root, u64 objectid, u64 pos, + u64 num_bytes); +int btrfs_lookup_file_extent(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, u64 objectid, + u64 bytenr, int mod); +int btrfs_insert_data_csums(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_ordered_sum *sums); +int btrfs_csum_one_bio(struct btrfs_bio *bbio, bool async); +int btrfs_alloc_dummy_sum(struct btrfs_bio *bbio); +int btrfs_lookup_csums_range(struct btrfs_root *root, u64 start, u64 end, + struct list_head *list, int search_commit, + bool nowait); +int btrfs_lookup_csums_list(struct btrfs_root *root, u64 start, u64 end, + struct list_head *list, bool nowait); +int btrfs_lookup_csums_bitmap(struct btrfs_root *root, struct btrfs_path *path, + u64 start, u64 end, u8 *csum_buf, + unsigned long *csum_bitmap); +void btrfs_extent_item_to_extent_map(struct btrfs_inode *inode, + const struct btrfs_path *path, + const struct btrfs_file_extent_item *fi, + struct extent_map *em); +int btrfs_inode_clear_file_extent_range(struct btrfs_inode *inode, u64 start, + u64 len); +int btrfs_inode_set_file_extent_range(struct btrfs_inode *inode, u64 start, u64 len); +void btrfs_inode_safe_disk_i_size_write(struct btrfs_inode *inode, u64 new_i_size); +u64 btrfs_file_extent_end(const struct btrfs_path *path); + +#endif diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c new file mode 100644 index 000000000..20e15dc30 --- /dev/null +++ b/fs/btrfs/file.c @@ -0,0 +1,3863 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ctree.h" +#include "direct-io.h" +#include "disk-io.h" +#include "transaction.h" +#include "btrfs_inode.h" +#include "tree-log.h" +#include "locking.h" +#include "qgroup.h" +#include "compression.h" +#include "delalloc-space.h" +#include "reflink.h" +#include "subpage.h" +#include "fs.h" +#include "accessors.h" +#include "extent-tree.h" +#include "file-item.h" +#include "ioctl.h" +#include "file.h" +#include "super.h" +#include "print-tree.h" + +/* + * Unlock folio after btrfs_file_write() is done with it. + */ +static void btrfs_drop_folio(struct btrfs_fs_info *fs_info, struct folio *folio, + u64 pos, u64 copied) +{ + u64 block_start = round_down(pos, fs_info->sectorsize); + u64 block_len = round_up(pos + copied, fs_info->sectorsize) - block_start; + + ASSERT(block_len <= U32_MAX); + folio_unlock(folio); + folio_put(folio); +} + +/* + * After copy_folio_from_iter_atomic(), update the following things for delalloc: + * - Mark newly dirtied folio as DELALLOC in the io tree. + * Used to advise which range is to be written back. + * - Mark modified folio as Uptodate/Dirty + * - Update inode size for past EOF write + */ +int btrfs_dirty_folio(struct btrfs_inode *inode, struct folio *folio, loff_t pos, + size_t write_bytes, struct extent_state **cached, bool noreserve) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + int ret = 0; + u64 num_bytes; + u64 start_pos; + u64 end_of_last_block; + const u64 end_pos = pos + write_bytes; + loff_t isize = i_size_read(&inode->vfs_inode); + unsigned int extra_bits = 0; + + if (write_bytes == 0) + return 0; + + if (noreserve) + extra_bits |= EXTENT_NORESERVE; + + start_pos = round_down(pos, fs_info->sectorsize); + num_bytes = round_up(end_pos - start_pos, fs_info->sectorsize); + ASSERT(num_bytes <= U32_MAX); + ASSERT(folio_pos(folio) <= pos && folio_next_pos(folio) >= end_pos); + + end_of_last_block = start_pos + num_bytes - 1; + + ret = btrfs_reset_extent_delalloc(inode, start_pos, end_of_last_block, + extra_bits, cached); + if (ret) + return ret; + + btrfs_folio_clamp_set_uptodate(fs_info, folio, start_pos, num_bytes); + btrfs_folio_clamp_set_dirty(fs_info, folio, start_pos, num_bytes); + + /* + * we've only changed i_size in ram, and we haven't updated + * the disk i_size. There is no need to log the inode + * at this time. + */ + if (end_pos > isize) + i_size_write(&inode->vfs_inode, end_pos); + return 0; +} + +/* + * this is very complex, but the basic idea is to drop all extents + * in the range start - end. hint_block is filled in with a block number + * that would be a good hint to the block allocator for this file. + * + * If an extent intersects the range but is not entirely inside the range + * it is either truncated or split. Anything entirely inside the range + * is deleted from the tree. + * + * Note: the VFS' inode number of bytes is not updated, it's up to the caller + * to deal with that. We set the field 'bytes_found' of the arguments structure + * with the number of allocated bytes found in the target range, so that the + * caller can update the inode's number of bytes in an atomic way when + * replacing extents in a range to avoid races with stat(2). + */ +int btrfs_drop_extents(struct btrfs_trans_handle *trans, + struct btrfs_root *root, struct btrfs_inode *inode, + struct btrfs_drop_extents_args *args) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct extent_buffer *leaf; + struct btrfs_file_extent_item *fi; + struct btrfs_key key; + struct btrfs_key new_key; + u64 ino = btrfs_ino(inode); + u64 search_start = args->start; + u64 disk_bytenr = 0; + u64 num_bytes = 0; + u64 extent_offset = 0; + u64 extent_end = 0; + u64 last_end = args->start; + int del_nr = 0; + int del_slot = 0; + int extent_type; + int recow; + int ret; + int modify_tree = -1; + int update_refs; + bool found = false; + struct btrfs_path *path = args->path; + + args->bytes_found = 0; + args->extent_inserted = false; + + /* Must always have a path if ->replace_extent is true */ + ASSERT(!(args->replace_extent && !args->path)); + + if (!path) { + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto out; + } + } + + if (args->drop_cache) + btrfs_drop_extent_map_range(inode, args->start, args->end - 1, false); + + if (data_race(args->start >= inode->disk_i_size) && !args->replace_extent) + modify_tree = 0; + + update_refs = (btrfs_root_id(root) != BTRFS_TREE_LOG_OBJECTID); + while (1) { + recow = 0; + ret = btrfs_lookup_file_extent(trans, root, path, ino, + search_start, modify_tree); + if (ret < 0) + break; + if (ret > 0 && path->slots[0] > 0 && search_start == args->start) { + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1); + if (key.objectid == ino && + key.type == BTRFS_EXTENT_DATA_KEY) + path->slots[0]--; + } + ret = 0; +next_slot: + leaf = path->nodes[0]; + if (path->slots[0] >= btrfs_header_nritems(leaf)) { + if (WARN_ON(del_nr > 0)) { + btrfs_print_leaf(leaf); + ret = -EINVAL; + break; + } + ret = btrfs_next_leaf(root, path); + if (ret < 0) + break; + if (ret > 0) { + ret = 0; + break; + } + leaf = path->nodes[0]; + recow = 1; + } + + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + + if (key.objectid > ino) + break; + if (WARN_ON_ONCE(key.objectid < ino) || + key.type < BTRFS_EXTENT_DATA_KEY) { + ASSERT(del_nr == 0); + path->slots[0]++; + goto next_slot; + } + if (key.type > BTRFS_EXTENT_DATA_KEY || key.offset >= args->end) + break; + + fi = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_file_extent_item); + extent_type = btrfs_file_extent_type(leaf, fi); + + if (extent_type == BTRFS_FILE_EXTENT_REG || + extent_type == BTRFS_FILE_EXTENT_PREALLOC) { + disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi); + num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi); + extent_offset = btrfs_file_extent_offset(leaf, fi); + extent_end = key.offset + + btrfs_file_extent_num_bytes(leaf, fi); + } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { + extent_end = key.offset + + btrfs_file_extent_ram_bytes(leaf, fi); + } else { + /* can't happen */ + BUG(); + } + + /* + * Don't skip extent items representing 0 byte lengths. They + * used to be created (bug) if while punching holes we hit + * -ENOSPC condition. So if we find one here, just ensure we + * delete it, otherwise we would insert a new file extent item + * with the same key (offset) as that 0 bytes length file + * extent item in the call to setup_items_for_insert() later + * in this function. + */ + if (extent_end == key.offset && extent_end >= search_start) { + last_end = extent_end; + goto delete_extent_item; + } + + if (extent_end <= search_start) { + path->slots[0]++; + goto next_slot; + } + + found = true; + search_start = max(key.offset, args->start); + if (recow || !modify_tree) { + modify_tree = -1; + btrfs_release_path(path); + continue; + } + + /* + * | - range to drop - | + * | -------- extent -------- | + */ + if (args->start > key.offset && args->end < extent_end) { + if (WARN_ON(del_nr > 0)) { + btrfs_print_leaf(leaf); + ret = -EINVAL; + break; + } + if (extent_type == BTRFS_FILE_EXTENT_INLINE) { + ret = -EOPNOTSUPP; + break; + } + + memcpy(&new_key, &key, sizeof(new_key)); + new_key.offset = args->start; + ret = btrfs_duplicate_item(trans, root, path, + &new_key); + if (ret == -EAGAIN) { + btrfs_release_path(path); + continue; + } + if (ret < 0) + break; + + leaf = path->nodes[0]; + fi = btrfs_item_ptr(leaf, path->slots[0] - 1, + struct btrfs_file_extent_item); + btrfs_set_file_extent_num_bytes(leaf, fi, + args->start - key.offset); + + fi = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_file_extent_item); + + extent_offset += args->start - key.offset; + btrfs_set_file_extent_offset(leaf, fi, extent_offset); + btrfs_set_file_extent_num_bytes(leaf, fi, + extent_end - args->start); + + if (update_refs && disk_bytenr > 0) { + struct btrfs_ref ref = { + .action = BTRFS_ADD_DELAYED_REF, + .bytenr = disk_bytenr, + .num_bytes = num_bytes, + .parent = 0, + .owning_root = btrfs_root_id(root), + .ref_root = btrfs_root_id(root), + }; + btrfs_init_data_ref(&ref, new_key.objectid, + args->start - extent_offset, + 0, false); + ret = btrfs_inc_extent_ref(trans, &ref); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + break; + } + } + key.offset = args->start; + } + /* + * From here on out we will have actually dropped something, so + * last_end can be updated. + */ + last_end = extent_end; + + /* + * | ---- range to drop ----- | + * | -------- extent -------- | + */ + if (args->start <= key.offset && args->end < extent_end) { + if (extent_type == BTRFS_FILE_EXTENT_INLINE) { + ret = -EOPNOTSUPP; + break; + } + + memcpy(&new_key, &key, sizeof(new_key)); + new_key.offset = args->end; + btrfs_set_item_key_safe(trans, path, &new_key); + + extent_offset += args->end - key.offset; + btrfs_set_file_extent_offset(leaf, fi, extent_offset); + btrfs_set_file_extent_num_bytes(leaf, fi, + extent_end - args->end); + if (update_refs && disk_bytenr > 0) + args->bytes_found += args->end - key.offset; + break; + } + + search_start = extent_end; + /* + * | ---- range to drop ----- | + * | -------- extent -------- | + */ + if (args->start > key.offset && args->end >= extent_end) { + if (WARN_ON(del_nr > 0)) { + btrfs_print_leaf(leaf); + ret = -EINVAL; + break; + } + if (extent_type == BTRFS_FILE_EXTENT_INLINE) { + ret = -EOPNOTSUPP; + break; + } + + btrfs_set_file_extent_num_bytes(leaf, fi, + args->start - key.offset); + if (update_refs && disk_bytenr > 0) + args->bytes_found += extent_end - args->start; + if (args->end == extent_end) + break; + + path->slots[0]++; + goto next_slot; + } + + /* + * | ---- range to drop ----- | + * | ------ extent ------ | + */ + if (args->start <= key.offset && args->end >= extent_end) { +delete_extent_item: + if (del_nr == 0) { + del_slot = path->slots[0]; + del_nr = 1; + } else { + if (WARN_ON(del_slot + del_nr != path->slots[0])) { + btrfs_print_leaf(leaf); + ret = -EINVAL; + break; + } + del_nr++; + } + + if (update_refs && + extent_type == BTRFS_FILE_EXTENT_INLINE) { + args->bytes_found += extent_end - key.offset; + extent_end = ALIGN(extent_end, + fs_info->sectorsize); + } else if (update_refs && disk_bytenr > 0) { + struct btrfs_ref ref = { + .action = BTRFS_DROP_DELAYED_REF, + .bytenr = disk_bytenr, + .num_bytes = num_bytes, + .parent = 0, + .owning_root = btrfs_root_id(root), + .ref_root = btrfs_root_id(root), + }; + btrfs_init_data_ref(&ref, key.objectid, + key.offset - extent_offset, + 0, false); + ret = btrfs_free_extent(trans, &ref); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + break; + } + args->bytes_found += extent_end - key.offset; + } + + if (args->end == extent_end) + break; + + if (path->slots[0] + 1 < btrfs_header_nritems(leaf)) { + path->slots[0]++; + goto next_slot; + } + + ret = btrfs_del_items(trans, root, path, del_slot, + del_nr); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + break; + } + + del_nr = 0; + del_slot = 0; + + btrfs_release_path(path); + continue; + } + + BUG(); + } + + if (!ret && del_nr > 0) { + /* + * Set path->slots[0] to first slot, so that after the delete + * if items are move off from our leaf to its immediate left or + * right neighbor leafs, we end up with a correct and adjusted + * path->slots[0] for our insertion (if args->replace_extent). + */ + path->slots[0] = del_slot; + ret = btrfs_del_items(trans, root, path, del_slot, del_nr); + if (ret) + btrfs_abort_transaction(trans, ret); + } + + leaf = path->nodes[0]; + /* + * If btrfs_del_items() was called, it might have deleted a leaf, in + * which case it unlocked our path, so check path->locks[0] matches a + * write lock. + */ + if (!ret && args->replace_extent && + path->locks[0] == BTRFS_WRITE_LOCK && + btrfs_leaf_free_space(leaf) >= + sizeof(struct btrfs_item) + args->extent_item_size) { + + key.objectid = ino; + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = args->start; + if (!del_nr && path->slots[0] < btrfs_header_nritems(leaf)) { + struct btrfs_key slot_key; + + btrfs_item_key_to_cpu(leaf, &slot_key, path->slots[0]); + if (btrfs_comp_cpu_keys(&key, &slot_key) > 0) + path->slots[0]++; + } + btrfs_setup_item_for_insert(trans, root, path, &key, + args->extent_item_size); + args->extent_inserted = true; + } + + if (!args->path) + btrfs_free_path(path); + else if (!args->extent_inserted) + btrfs_release_path(path); +out: + args->drop_end = found ? min(args->end, last_end) : args->end; + + return ret; +} + +static bool extent_mergeable(struct extent_buffer *leaf, int slot, u64 objectid, + u64 bytenr, u64 orig_offset, u64 *start, u64 *end) +{ + struct btrfs_file_extent_item *fi; + struct btrfs_key key; + u64 extent_end; + + if (slot < 0 || slot >= btrfs_header_nritems(leaf)) + return false; + + btrfs_item_key_to_cpu(leaf, &key, slot); + if (key.objectid != objectid || key.type != BTRFS_EXTENT_DATA_KEY) + return false; + + fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item); + if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG || + btrfs_file_extent_disk_bytenr(leaf, fi) != bytenr || + btrfs_file_extent_offset(leaf, fi) != key.offset - orig_offset || + btrfs_file_extent_compression(leaf, fi) || + btrfs_file_extent_encryption(leaf, fi) || + btrfs_file_extent_other_encoding(leaf, fi)) + return false; + + extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi); + if ((*start && *start != key.offset) || (*end && *end != extent_end)) + return false; + + *start = key.offset; + *end = extent_end; + return true; +} + +/* + * Mark extent in the range start - end as written. + * + * This changes extent type from 'pre-allocated' to 'regular'. If only + * part of extent is marked as written, the extent will be split into + * two or three. + */ +int btrfs_mark_extent_written(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, u64 start, u64 end) +{ + struct btrfs_root *root = inode->root; + struct extent_buffer *leaf; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_file_extent_item *fi; + struct btrfs_ref ref = { 0 }; + struct btrfs_key key; + struct btrfs_key new_key; + u64 bytenr; + u64 num_bytes; + u64 extent_end; + u64 orig_offset; + u64 other_start; + u64 other_end; + u64 split; + int del_nr = 0; + int del_slot = 0; + int recow; + int ret; + u64 ino = btrfs_ino(inode); + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; +again: + recow = 0; + split = start; + key.objectid = ino; + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = split; + + ret = btrfs_search_slot(trans, root, &key, path, -1, 1); + if (ret < 0) + return ret; + if (ret > 0 && path->slots[0] > 0) + path->slots[0]--; + + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + if (unlikely(key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY)) { + ret = -EINVAL; + btrfs_abort_transaction(trans, ret); + return ret; + } + fi = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_file_extent_item); + if (unlikely(btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_PREALLOC)) { + ret = -EINVAL; + btrfs_abort_transaction(trans, ret); + return ret; + } + extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi); + if (unlikely(key.offset > start || extent_end < end)) { + ret = -EINVAL; + btrfs_abort_transaction(trans, ret); + return ret; + } + + bytenr = btrfs_file_extent_disk_bytenr(leaf, fi); + num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi); + orig_offset = key.offset - btrfs_file_extent_offset(leaf, fi); + memcpy(&new_key, &key, sizeof(new_key)); + + if (start == key.offset && end < extent_end) { + other_start = 0; + other_end = start; + if (extent_mergeable(leaf, path->slots[0] - 1, + ino, bytenr, orig_offset, + &other_start, &other_end)) { + new_key.offset = end; + btrfs_set_item_key_safe(trans, path, &new_key); + fi = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_file_extent_item); + btrfs_set_file_extent_generation(leaf, fi, + trans->transid); + btrfs_set_file_extent_num_bytes(leaf, fi, + extent_end - end); + btrfs_set_file_extent_offset(leaf, fi, + end - orig_offset); + fi = btrfs_item_ptr(leaf, path->slots[0] - 1, + struct btrfs_file_extent_item); + btrfs_set_file_extent_generation(leaf, fi, + trans->transid); + btrfs_set_file_extent_num_bytes(leaf, fi, + end - other_start); + goto mark_dirty; + } + } + + if (start > key.offset && end == extent_end) { + other_start = end; + other_end = 0; + if (extent_mergeable(leaf, path->slots[0] + 1, + ino, bytenr, orig_offset, + &other_start, &other_end)) { + fi = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_file_extent_item); + btrfs_set_file_extent_num_bytes(leaf, fi, + start - key.offset); + btrfs_set_file_extent_generation(leaf, fi, + trans->transid); + path->slots[0]++; + new_key.offset = start; + btrfs_set_item_key_safe(trans, path, &new_key); + + fi = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_file_extent_item); + btrfs_set_file_extent_generation(leaf, fi, + trans->transid); + btrfs_set_file_extent_num_bytes(leaf, fi, + other_end - start); + btrfs_set_file_extent_offset(leaf, fi, + start - orig_offset); + goto mark_dirty; + } + } + + while (start > key.offset || end < extent_end) { + if (key.offset == start) + split = end; + + new_key.offset = split; + ret = btrfs_duplicate_item(trans, root, path, &new_key); + if (ret == -EAGAIN) { + btrfs_release_path(path); + goto again; + } + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + + leaf = path->nodes[0]; + fi = btrfs_item_ptr(leaf, path->slots[0] - 1, + struct btrfs_file_extent_item); + btrfs_set_file_extent_generation(leaf, fi, trans->transid); + btrfs_set_file_extent_num_bytes(leaf, fi, + split - key.offset); + + fi = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_file_extent_item); + + btrfs_set_file_extent_generation(leaf, fi, trans->transid); + btrfs_set_file_extent_offset(leaf, fi, split - orig_offset); + btrfs_set_file_extent_num_bytes(leaf, fi, + extent_end - split); + + ref.action = BTRFS_ADD_DELAYED_REF; + ref.bytenr = bytenr; + ref.num_bytes = num_bytes; + ref.parent = 0; + ref.owning_root = btrfs_root_id(root); + ref.ref_root = btrfs_root_id(root); + btrfs_init_data_ref(&ref, ino, orig_offset, 0, false); + ret = btrfs_inc_extent_ref(trans, &ref); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + + if (split == start) { + key.offset = start; + } else { + if (unlikely(start != key.offset)) { + ret = -EINVAL; + btrfs_abort_transaction(trans, ret); + return ret; + } + path->slots[0]--; + extent_end = end; + } + recow = 1; + } + + other_start = end; + other_end = 0; + + ref.action = BTRFS_DROP_DELAYED_REF; + ref.bytenr = bytenr; + ref.num_bytes = num_bytes; + ref.parent = 0; + ref.owning_root = btrfs_root_id(root); + ref.ref_root = btrfs_root_id(root); + btrfs_init_data_ref(&ref, ino, orig_offset, 0, false); + if (extent_mergeable(leaf, path->slots[0] + 1, + ino, bytenr, orig_offset, + &other_start, &other_end)) { + if (recow) { + btrfs_release_path(path); + goto again; + } + extent_end = other_end; + del_slot = path->slots[0] + 1; + del_nr++; + ret = btrfs_free_extent(trans, &ref); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + } + other_start = 0; + other_end = start; + if (extent_mergeable(leaf, path->slots[0] - 1, + ino, bytenr, orig_offset, + &other_start, &other_end)) { + if (recow) { + btrfs_release_path(path); + goto again; + } + key.offset = other_start; + del_slot = path->slots[0]; + del_nr++; + ret = btrfs_free_extent(trans, &ref); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + } + if (del_nr == 0) { + fi = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_file_extent_item); + btrfs_set_file_extent_type(leaf, fi, + BTRFS_FILE_EXTENT_REG); + btrfs_set_file_extent_generation(leaf, fi, trans->transid); + } else { + fi = btrfs_item_ptr(leaf, del_slot - 1, + struct btrfs_file_extent_item); + btrfs_set_file_extent_type(leaf, fi, + BTRFS_FILE_EXTENT_REG); + btrfs_set_file_extent_generation(leaf, fi, trans->transid); + btrfs_set_file_extent_num_bytes(leaf, fi, + extent_end - key.offset); + + ret = btrfs_del_items(trans, root, path, del_slot, del_nr); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + } + +mark_dirty: + ret = btrfs_inode_set_file_extent_range(inode, start, end - start); + if (ret) + btrfs_abort_transaction(trans, ret); + + return ret; +} + +/* + * On error return an unlocked folio and the error value + * On success return a locked folio and 0 + */ +static int prepare_uptodate_folio(struct inode *inode, struct folio *folio, u64 pos, + u64 len) +{ + u64 clamp_start = max_t(u64, pos, folio_pos(folio)); + u64 clamp_end = min_t(u64, pos + len, folio_next_pos(folio)); + const u32 blocksize = inode_to_fs_info(inode)->sectorsize; + int ret = 0; + + if (folio_test_uptodate(folio)) + return 0; + + if (IS_ALIGNED(clamp_start, blocksize) && + IS_ALIGNED(clamp_end, blocksize)) + return 0; + + ret = btrfs_read_folio(NULL, folio); + if (ret) + return ret; + folio_lock(folio); + if (unlikely(!folio_test_uptodate(folio))) { + folio_unlock(folio); + return -EIO; + } + + /* + * Since btrfs_read_folio() will unlock the folio before it returns, + * there is a window where btrfs_release_folio() can be called to + * release the page. Here we check both inode mapping and page + * private to make sure the page was not released. + * + * The private flag check is essential for subpage as we need to store + * extra bitmap using folio private. + */ + if (folio->mapping != inode->i_mapping || !folio_test_private(folio)) { + folio_unlock(folio); + return -EAGAIN; + } + return 0; +} + +static gfp_t get_prepare_gfp_flags(struct inode *inode, bool nowait) +{ + gfp_t gfp; + + gfp = btrfs_alloc_write_mask(inode->i_mapping); + if (nowait) { + gfp &= ~__GFP_DIRECT_RECLAIM; + gfp |= GFP_NOWAIT; + } + + return gfp; +} + +/* + * Get folio into the page cache and lock it. + */ +static noinline int prepare_one_folio(struct inode *inode, struct folio **folio_ret, + loff_t pos, size_t write_bytes, + bool nowait) +{ + const pgoff_t index = pos >> PAGE_SHIFT; + gfp_t mask = get_prepare_gfp_flags(inode, nowait); + fgf_t fgp_flags = (nowait ? FGP_WRITEBEGIN | FGP_NOWAIT : FGP_WRITEBEGIN) | + fgf_set_order(write_bytes); + struct folio *folio; + int ret; + +again: + folio = __filemap_get_folio(inode->i_mapping, index, fgp_flags, mask); + if (IS_ERR(folio)) + return PTR_ERR(folio); + + ret = set_folio_extent_mapped(folio); + if (ret < 0) { + folio_unlock(folio); + folio_put(folio); + return ret; + } + ret = prepare_uptodate_folio(inode, folio, pos, write_bytes); + if (ret) { + /* The folio is already unlocked. */ + folio_put(folio); + if (!nowait && ret == -EAGAIN) + goto again; + return ret; + } + *folio_ret = folio; + return 0; +} + +/* + * Locks the extent and properly waits for data=ordered extents to finish + * before allowing the folios to be modified. + * + * Return: + * 0 - the extent is locked + * -EAGAIN - need to prepare the folios again + */ +static noinline int +lock_and_cleanup_extent(struct btrfs_inode *inode, struct folio *folio, + loff_t pos, size_t write_bytes, + u64 *lockstart, u64 *lockend, bool nowait, + struct extent_state **cached_state) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct btrfs_ordered_extent *ordered; + u64 start_pos; + u64 last_pos; + + start_pos = round_down(pos, fs_info->sectorsize); + last_pos = round_up(pos + write_bytes, fs_info->sectorsize) - 1; + + if (nowait) { + if (!btrfs_try_lock_extent(&inode->io_tree, start_pos, + last_pos, cached_state)) { + folio_unlock(folio); + folio_put(folio); + return -EAGAIN; + } + } else { + btrfs_lock_extent(&inode->io_tree, start_pos, last_pos, + cached_state); + } + + ordered = btrfs_lookup_ordered_range(inode, start_pos, + last_pos - start_pos + 1); + if (ordered && + ordered->file_offset + ordered->num_bytes > start_pos && + ordered->file_offset <= last_pos) { + btrfs_unlock_extent(&inode->io_tree, start_pos, last_pos, + cached_state); + folio_unlock(folio); + folio_put(folio); + btrfs_start_ordered_extent(ordered); + btrfs_put_ordered_extent(ordered); + return -EAGAIN; + } + if (ordered) + btrfs_put_ordered_extent(ordered); + + *lockstart = start_pos; + *lockend = last_pos; + + /* + * We should be called after prepare_one_folio() which should have locked + * all pages in the range. + */ + WARN_ON(!folio_test_locked(folio)); + + return 0; +} + +/* + * Check if we can do nocow write into the range [@pos, @pos + @write_bytes) + * + * @pos: File offset. + * @write_bytes: The length to write, will be updated to the nocow writeable + * range. + * @nowait: Indicate if we can block or not (non-blocking IO context). + * + * This function will flush ordered extents in the range to ensure proper + * nocow checks. + * + * Return: + * > 0 If we can nocow, and updates @write_bytes. + * 0 If we can't do a nocow write. + * -EAGAIN If we can't do a nocow write because snapshotting of the inode's + * root is in progress or because we are in a non-blocking IO + * context and need to block (@nowait is true). + * < 0 If an error happened. + * + * NOTE: Callers need to call btrfs_check_nocow_unlock() if we return > 0. + */ +int btrfs_check_nocow_lock(struct btrfs_inode *inode, loff_t pos, + size_t *write_bytes, bool nowait) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct btrfs_root *root = inode->root; + struct extent_state *cached_state = NULL; + u64 lockstart, lockend; + u64 cur_offset; + int ret = 0; + + if (!(inode->flags & (BTRFS_INODE_NODATACOW | BTRFS_INODE_PREALLOC))) + return 0; + + if (!btrfs_drew_try_write_lock(&root->snapshot_lock)) + return -EAGAIN; + + lockstart = round_down(pos, fs_info->sectorsize); + lockend = round_up(pos + *write_bytes, + fs_info->sectorsize) - 1; + + if (nowait) { + if (!btrfs_try_lock_ordered_range(inode, lockstart, lockend, + &cached_state)) { + btrfs_drew_write_unlock(&root->snapshot_lock); + return -EAGAIN; + } + } else { + btrfs_lock_and_flush_ordered_range(inode, lockstart, lockend, + &cached_state); + } + + cur_offset = lockstart; + while (cur_offset < lockend) { + u64 num_bytes = lockend - cur_offset + 1; + + ret = can_nocow_extent(inode, cur_offset, &num_bytes, NULL, nowait); + if (ret <= 0) { + /* + * If cur_offset == lockstart it means we haven't found + * any extent against which we can NOCOW, so unlock the + * snapshot lock. + */ + if (cur_offset == lockstart) + btrfs_drew_write_unlock(&root->snapshot_lock); + break; + } + cur_offset += num_bytes; + } + + btrfs_unlock_extent(&inode->io_tree, lockstart, lockend, &cached_state); + + /* + * cur_offset > lockstart means there's at least a partial range we can + * NOCOW, and that range can cover one or more extents. + */ + if (cur_offset > lockstart) { + *write_bytes = min_t(size_t, *write_bytes, cur_offset - pos); + return 1; + } + + return ret; +} + +void btrfs_check_nocow_unlock(struct btrfs_inode *inode) +{ + btrfs_drew_write_unlock(&inode->root->snapshot_lock); +} + +int btrfs_write_check(struct kiocb *iocb, size_t count) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + loff_t pos = iocb->ki_pos; + int ret; + loff_t oldsize; + + /* + * Quickly bail out on NOWAIT writes if we don't have the nodatacow or + * prealloc flags, as without those flags we always have to COW. We will + * later check if we can really COW into the target range (using + * can_nocow_extent() at btrfs_get_blocks_direct_write()). + */ + if ((iocb->ki_flags & IOCB_NOWAIT) && + !(BTRFS_I(inode)->flags & (BTRFS_INODE_NODATACOW | BTRFS_INODE_PREALLOC))) + return -EAGAIN; + + ret = file_remove_privs(file); + if (ret) + return ret; + + /* + * We reserve space for updating the inode when we reserve space for the + * extent we are going to write, so we will enospc out there. We don't + * need to start yet another transaction to update the inode as we will + * update the inode when we finish writing whatever data we write. + */ + if (!IS_NOCMTIME(inode)) { + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + inode_inc_iversion(inode); + } + + oldsize = i_size_read(inode); + if (pos > oldsize) { + /* Expand hole size to cover write data, preventing empty gap */ + loff_t end_pos = round_up(pos + count, fs_info->sectorsize); + + ret = btrfs_cont_expand(BTRFS_I(inode), oldsize, end_pos); + if (ret) + return ret; + } + + return 0; +} + +static void release_space(struct btrfs_inode *inode, struct extent_changeset *data_reserved, + u64 start, u64 len, bool only_release_metadata) +{ + if (len == 0) + return; + + if (only_release_metadata) { + btrfs_check_nocow_unlock(inode); + btrfs_delalloc_release_metadata(inode, len, true); + } else { + const struct btrfs_fs_info *fs_info = inode->root->fs_info; + + btrfs_delalloc_release_space(inode, data_reserved, + round_down(start, fs_info->sectorsize), + len, true); + } +} + +/* + * Reserve data and metadata space for this buffered write range. + * + * Return >0 for the number of bytes reserved, which is always block aligned. + * Return <0 for error. + */ +static ssize_t reserve_space(struct btrfs_inode *inode, + struct extent_changeset **data_reserved, + u64 start, size_t *len, bool nowait, + bool *only_release_metadata) +{ + const struct btrfs_fs_info *fs_info = inode->root->fs_info; + const unsigned int block_offset = (start & (fs_info->sectorsize - 1)); + size_t reserve_bytes; + int ret; + + ret = btrfs_check_data_free_space(inode, data_reserved, start, *len, nowait); + if (ret < 0) { + int can_nocow; + + if (nowait && (ret == -ENOSPC || ret == -EAGAIN)) + return -EAGAIN; + + /* + * If we don't have to COW at the offset, reserve metadata only. + * write_bytes may get smaller than requested here. + */ + can_nocow = btrfs_check_nocow_lock(inode, start, len, nowait); + if (can_nocow < 0) + ret = can_nocow; + if (can_nocow > 0) + ret = 0; + if (ret) + return ret; + *only_release_metadata = true; + } + + reserve_bytes = round_up(*len + block_offset, fs_info->sectorsize); + WARN_ON(reserve_bytes == 0); + ret = btrfs_delalloc_reserve_metadata(inode, reserve_bytes, + reserve_bytes, nowait); + if (ret) { + if (!*only_release_metadata) + btrfs_free_reserved_data_space(inode, *data_reserved, + start, *len); + else + btrfs_check_nocow_unlock(inode); + + if (nowait && ret == -ENOSPC) + ret = -EAGAIN; + return ret; + } + return reserve_bytes; +} + +/* Shrink the reserved data and metadata space from @reserved_len to @new_len. */ +static void shrink_reserved_space(struct btrfs_inode *inode, + struct extent_changeset *data_reserved, + u64 reserved_start, u64 reserved_len, + u64 new_len, bool only_release_metadata) +{ + const u64 diff = reserved_len - new_len; + + ASSERT(new_len <= reserved_len); + btrfs_delalloc_shrink_extents(inode, reserved_len, new_len); + if (only_release_metadata) + btrfs_delalloc_release_metadata(inode, diff, true); + else + btrfs_delalloc_release_space(inode, data_reserved, + reserved_start + new_len, diff, true); +} + +/* Calculate the maximum amount of bytes we can write into one folio. */ +static size_t calc_write_bytes(const struct btrfs_inode *inode, + const struct iov_iter *iter, u64 start) +{ + const size_t max_folio_size = mapping_max_folio_size(inode->vfs_inode.i_mapping); + + return min(max_folio_size - (start & (max_folio_size - 1)), + iov_iter_count(iter)); +} + +/* + * Do the heavy-lifting work to copy one range into one folio of the page cache. + * + * Return > 0 in case we copied all bytes or just some of them. + * Return 0 if no bytes were copied, in which case the caller should retry. + * Return <0 on error. + */ +static int copy_one_range(struct btrfs_inode *inode, struct iov_iter *iter, + struct extent_changeset **data_reserved, u64 start, + bool nowait) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct extent_state *cached_state = NULL; + size_t write_bytes = calc_write_bytes(inode, iter, start); + size_t copied; + const u64 reserved_start = round_down(start, fs_info->sectorsize); + u64 reserved_len; + struct folio *folio = NULL; + u64 lockstart; + u64 lockend; + bool only_release_metadata = false; + const unsigned int bdp_flags = (nowait ? BDP_ASYNC : 0); + int ret; + + /* + * Fault all pages before locking them in prepare_one_folio() to avoid + * recursive lock. + */ + if (unlikely(fault_in_iov_iter_readable(iter, write_bytes))) + return -EFAULT; + extent_changeset_release(*data_reserved); + ret = reserve_space(inode, data_reserved, start, &write_bytes, nowait, + &only_release_metadata); + if (ret < 0) + return ret; + reserved_len = ret; + /* Write range must be inside the reserved range. */ + ASSERT(reserved_start <= start, "reserved_start=%llu start=%llu", + reserved_start, start); + ASSERT(start + write_bytes <= reserved_start + reserved_len, + "start=%llu write_bytes=%zu reserved_start=%llu reserved_len=%llu", + start, write_bytes, reserved_start, reserved_len); + +again: + ret = balance_dirty_pages_ratelimited_flags(inode->vfs_inode.i_mapping, + bdp_flags); + if (ret) { + btrfs_delalloc_release_extents(inode, reserved_len); + release_space(inode, *data_reserved, reserved_start, reserved_len, + only_release_metadata); + return ret; + } + + ret = prepare_one_folio(&inode->vfs_inode, &folio, start, write_bytes, false); + if (ret) { + btrfs_delalloc_release_extents(inode, reserved_len); + release_space(inode, *data_reserved, reserved_start, reserved_len, + only_release_metadata); + return ret; + } + + /* + * The reserved range goes beyond the current folio, shrink the reserved + * space to the folio boundary. + */ + if (reserved_start + reserved_len > folio_next_pos(folio)) { + const u64 last_block = folio_next_pos(folio); + + shrink_reserved_space(inode, *data_reserved, reserved_start, + reserved_len, last_block - reserved_start, + only_release_metadata); + write_bytes = last_block - start; + reserved_len = last_block - reserved_start; + } + + ret = lock_and_cleanup_extent(inode, folio, start, write_bytes, + &lockstart, &lockend, nowait, &cached_state); + if (ret < 0) { + if (!nowait) + goto again; + + btrfs_delalloc_release_extents(inode, reserved_len); + release_space(inode, *data_reserved, reserved_start, reserved_len, + only_release_metadata); + return ret; + } + + copied = copy_folio_from_iter_atomic(folio, offset_in_folio(folio, start), + write_bytes, iter); + flush_dcache_folio(folio); + + if (unlikely(copied < write_bytes)) { + u64 last_block; + + /* + * The original write range doesn't need an uptodate folio as + * the range is block aligned. But now a short copy happened. + * We cannot handle it without an uptodate folio. + * + * So just revert the range and we will retry. + */ + if (!folio_test_uptodate(folio)) { + iov_iter_revert(iter, copied); + copied = 0; + } + + /* No copied bytes, unlock, release reserved space and exit. */ + if (copied == 0) { + btrfs_unlock_extent(&inode->io_tree, lockstart, lockend, + &cached_state); + btrfs_delalloc_release_extents(inode, reserved_len); + release_space(inode, *data_reserved, reserved_start, reserved_len, + only_release_metadata); + btrfs_drop_folio(fs_info, folio, start, copied); + return 0; + } + + /* Release the reserved space beyond the last block. */ + last_block = round_up(start + copied, fs_info->sectorsize); + + shrink_reserved_space(inode, *data_reserved, reserved_start, + reserved_len, last_block - reserved_start, + only_release_metadata); + reserved_len = last_block - reserved_start; + } + + ret = btrfs_dirty_folio(inode, folio, start, copied, &cached_state, + only_release_metadata); + btrfs_unlock_extent(&inode->io_tree, lockstart, lockend, &cached_state); + + btrfs_delalloc_release_extents(inode, reserved_len); + if (ret) { + btrfs_drop_folio(fs_info, folio, start, copied); + release_space(inode, *data_reserved, reserved_start, reserved_len, + only_release_metadata); + return ret; + } + if (only_release_metadata) + btrfs_check_nocow_unlock(inode); + + btrfs_drop_folio(fs_info, folio, start, copied); + return copied; +} + +ssize_t btrfs_buffered_write(struct kiocb *iocb, struct iov_iter *iter) +{ + struct file *file = iocb->ki_filp; + loff_t pos; + struct inode *inode = file_inode(file); + struct extent_changeset *data_reserved = NULL; + size_t num_written = 0; + ssize_t ret; + loff_t old_isize; + unsigned int ilock_flags = 0; + const bool nowait = (iocb->ki_flags & IOCB_NOWAIT); + + if (nowait) + ilock_flags |= BTRFS_ILOCK_TRY; + + ret = btrfs_inode_lock(BTRFS_I(inode), ilock_flags); + if (ret < 0) + return ret; + + /* + * We can only trust the isize with inode lock held, or it can race with + * other buffered writes and cause incorrect call of + * pagecache_isize_extended() to overwrite existing data. + */ + old_isize = i_size_read(inode); + + ret = generic_write_checks(iocb, iter); + if (ret <= 0) + goto out; + + ret = btrfs_write_check(iocb, ret); + if (ret < 0) + goto out; + + pos = iocb->ki_pos; + while (iov_iter_count(iter) > 0) { + ret = copy_one_range(BTRFS_I(inode), iter, &data_reserved, pos, nowait); + if (ret < 0) + break; + pos += ret; + num_written += ret; + cond_resched(); + } + + extent_changeset_free(data_reserved); + if (num_written > 0) { + pagecache_isize_extended(inode, old_isize, iocb->ki_pos); + iocb->ki_pos += num_written; + } +out: + btrfs_inode_unlock(BTRFS_I(inode), ilock_flags); + return num_written ? num_written : ret; +} + +static ssize_t btrfs_encoded_write(struct kiocb *iocb, struct iov_iter *from, + const struct btrfs_ioctl_encoded_io_args *encoded) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + loff_t count; + ssize_t ret; + + btrfs_inode_lock(BTRFS_I(inode), 0); + count = encoded->len; + ret = generic_write_checks_count(iocb, &count); + if (ret == 0 && count != encoded->len) { + /* + * The write got truncated by generic_write_checks_count(). We + * can't do a partial encoded write. + */ + ret = -EFBIG; + } + if (ret || encoded->len == 0) + goto out; + + ret = btrfs_write_check(iocb, encoded->len); + if (ret < 0) + goto out; + + ret = btrfs_do_encoded_write(iocb, from, encoded); +out: + btrfs_inode_unlock(BTRFS_I(inode), 0); + return ret; +} + +ssize_t btrfs_do_write_iter(struct kiocb *iocb, struct iov_iter *from, + const struct btrfs_ioctl_encoded_io_args *encoded) +{ + struct file *file = iocb->ki_filp; + struct btrfs_inode *inode = BTRFS_I(file_inode(file)); + ssize_t num_written, num_sync; + + if (btrfs_is_shutdown(inode->root->fs_info)) + return -EIO; + /* + * If the fs flips readonly due to some impossible error, although we + * have opened a file as writable, we have to stop this write operation + * to ensure consistency. + */ + if (unlikely(BTRFS_FS_ERROR(inode->root->fs_info))) + return -EROFS; + + if (encoded && (iocb->ki_flags & IOCB_NOWAIT)) + return -EOPNOTSUPP; + + if (encoded) { + num_written = btrfs_encoded_write(iocb, from, encoded); + num_sync = encoded->len; + } else if (iocb->ki_flags & IOCB_DIRECT) { + num_written = btrfs_direct_write(iocb, from); + num_sync = num_written; + } else { + num_written = btrfs_buffered_write(iocb, from); + num_sync = num_written; + } + + btrfs_set_inode_last_sub_trans(inode); + + if (num_sync > 0) { + num_sync = generic_write_sync(iocb, num_sync); + if (num_sync < 0) + num_written = num_sync; + } + + return num_written; +} + +static ssize_t btrfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + return btrfs_do_write_iter(iocb, from, NULL); +} + +int btrfs_release_file(struct inode *inode, struct file *filp) +{ + struct btrfs_file_private *private = filp->private_data; + + if (private) { + kfree(private->filldir_buf); + btrfs_free_extent_state(private->llseek_cached_state); + kfree(private); + filp->private_data = NULL; + } + + /* + * Set by setattr when we are about to truncate a file from a non-zero + * size to a zero size. This tries to flush down new bytes that may + * have been written if the application were using truncate to replace + * a file in place. + */ + if (test_and_clear_bit(BTRFS_INODE_FLUSH_ON_CLOSE, + &BTRFS_I(inode)->runtime_flags)) + filemap_flush(inode->i_mapping); + return 0; +} + +static int start_ordered_ops(struct btrfs_inode *inode, loff_t start, loff_t end) +{ + int ret; + struct blk_plug plug; + + /* + * This is only called in fsync, which would do synchronous writes, so + * a plug can merge adjacent IOs as much as possible. Esp. in case of + * multiple disks using raid profile, a large IO can be split to + * several segments of stripe length (currently 64K). + */ + blk_start_plug(&plug); + ret = btrfs_fdatawrite_range(inode, start, end); + blk_finish_plug(&plug); + + return ret; +} + +static inline bool skip_inode_logging(const struct btrfs_log_ctx *ctx) +{ + struct btrfs_inode *inode = ctx->inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + + if (btrfs_inode_in_log(inode, btrfs_get_fs_generation(fs_info)) && + list_empty(&ctx->ordered_extents)) + return true; + + /* + * If we are doing a fast fsync we can not bail out if the inode's + * last_trans is <= then the last committed transaction, because we only + * update the last_trans of the inode during ordered extent completion, + * and for a fast fsync we don't wait for that, we only wait for the + * writeback to complete. + */ + if (inode->last_trans <= btrfs_get_last_trans_committed(fs_info) && + (test_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags) || + list_empty(&ctx->ordered_extents))) + return true; + + return false; +} + +/* + * fsync call for both files and directories. This logs the inode into + * the tree log instead of forcing full commits whenever possible. + * + * It needs to call filemap_fdatawait so that all ordered extent updates are + * in the metadata btree are up to date for copying to the log. + * + * It drops the inode mutex before doing the tree log commit. This is an + * important optimization for directories because holding the mutex prevents + * new operations on the dir while we write to disk. + */ +int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync) +{ + struct dentry *dentry = file_dentry(file); + struct btrfs_inode *inode = BTRFS_I(d_inode(dentry)); + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_trans_handle *trans; + struct btrfs_log_ctx ctx; + int ret = 0, err; + u64 len; + bool full_sync; + bool skip_ilock = false; + + if (current->journal_info == BTRFS_TRANS_DIO_WRITE_STUB) { + skip_ilock = true; + current->journal_info = NULL; + btrfs_assert_inode_locked(inode); + } + + trace_btrfs_sync_file_enter(file, datasync); + + btrfs_init_log_ctx(&ctx, inode); + + /* + * Always set the range to a full range, otherwise we can get into + * several problems, from missing file extent items to represent holes + * when not using the NO_HOLES feature, to log tree corruption due to + * races between hole detection during logging and completion of ordered + * extents outside the range, to missing checksums due to ordered extents + * for which we flushed only a subset of their pages. + */ + start = 0; + end = LLONG_MAX; + len = (u64)LLONG_MAX + 1; + + /* + * We write the dirty pages in the range and wait until they complete + * out of the ->i_mutex. If so, we can flush the dirty pages by + * multi-task, and make the performance up. See + * btrfs_wait_ordered_range for an explanation of the ASYNC check. + */ + ret = start_ordered_ops(inode, start, end); + if (ret) + goto out; + + if (skip_ilock) + down_write(&inode->i_mmap_lock); + else + btrfs_inode_lock(inode, BTRFS_ILOCK_MMAP); + + /* + * Before we acquired the inode's lock and the mmap lock, someone may + * have dirtied more pages in the target range. We need to make sure + * that writeback for any such pages does not start while we are logging + * the inode, because if it does, any of the following might happen when + * we are not doing a full inode sync: + * + * 1) We log an extent after its writeback finishes but before its + * checksums are added to the csum tree, leading to -EIO errors + * when attempting to read the extent after a log replay. + * + * 2) We can end up logging an extent before its writeback finishes. + * Therefore after the log replay we will have a file extent item + * pointing to an unwritten extent (and no data checksums as well). + * + * So trigger writeback for any eventual new dirty pages and then we + * wait for all ordered extents to complete below. + */ + ret = start_ordered_ops(inode, start, end); + if (ret) { + if (skip_ilock) + up_write(&inode->i_mmap_lock); + else + btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP); + goto out; + } + + /* + * Always check for the full sync flag while holding the inode's lock, + * to avoid races with other tasks. The flag must be either set all the + * time during logging or always off all the time while logging. + * We check the flag here after starting delalloc above, because when + * running delalloc the full sync flag may be set if we need to drop + * extra extent map ranges due to temporary memory allocation failures. + */ + full_sync = test_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags); + + /* + * We have to do this here to avoid the priority inversion of waiting on + * IO of a lower priority task while holding a transaction open. + * + * For a full fsync we wait for the ordered extents to complete while + * for a fast fsync we wait just for writeback to complete, and then + * attach the ordered extents to the transaction so that a transaction + * commit waits for their completion, to avoid data loss if we fsync, + * the current transaction commits before the ordered extents complete + * and a power failure happens right after that. + * + * For zoned filesystem, if a write IO uses a ZONE_APPEND command, the + * logical address recorded in the ordered extent may change. We need + * to wait for the IO to stabilize the logical address. + */ + if (full_sync || btrfs_is_zoned(fs_info)) { + ret = btrfs_wait_ordered_range(inode, start, len); + clear_bit(BTRFS_INODE_COW_WRITE_ERROR, &inode->runtime_flags); + } else { + /* + * Get our ordered extents as soon as possible to avoid doing + * checksum lookups in the csum tree, and use instead the + * checksums attached to the ordered extents. + */ + btrfs_get_ordered_extents_for_logging(inode, &ctx.ordered_extents); + ret = filemap_fdatawait_range(inode->vfs_inode.i_mapping, start, end); + if (ret) + goto out_release_extents; + + /* + * Check and clear the BTRFS_INODE_COW_WRITE_ERROR now after + * starting and waiting for writeback, because for buffered IO + * it may have been set during the end IO callback + * (end_bbio_data_write() -> btrfs_finish_ordered_extent()) in + * case an error happened and we need to wait for ordered + * extents to complete so that any extent maps that point to + * unwritten locations are dropped and we don't log them. + */ + if (test_and_clear_bit(BTRFS_INODE_COW_WRITE_ERROR, &inode->runtime_flags)) + ret = btrfs_wait_ordered_range(inode, start, len); + } + + if (ret) + goto out_release_extents; + + if (skip_inode_logging(&ctx)) { + /* + * We've had everything committed since the last time we were + * modified so clear this flag in case it was set for whatever + * reason, it's no longer relevant. + */ + clear_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags); + goto out_release_extents; + } + + btrfs_init_log_ctx_scratch_eb(&ctx); + + /* + * We use start here because we will need to wait on the IO to complete + * in btrfs_sync_log, which could require joining a transaction (for + * example checking cross references in the nocow path). If we use join + * here we could get into a situation where we're waiting on IO to + * happen that is blocked on a transaction trying to commit. With start + * we inc the extwriter counter, so we wait for all extwriters to exit + * before we start blocking joiners. This comment is to keep somebody + * from thinking they are super smart and changing this to + * btrfs_join_transaction *cough*Josef*cough*. + */ + trans = btrfs_start_transaction(root, 0); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto out_release_extents; + } + trans->in_fsync = true; + + ret = btrfs_log_dentry_safe(trans, dentry, &ctx); + /* + * Scratch eb no longer needed, release before syncing log or commit + * transaction, to avoid holding unnecessary memory during such long + * operations. + */ + if (ctx.scratch_eb) { + free_extent_buffer(ctx.scratch_eb); + ctx.scratch_eb = NULL; + } + btrfs_release_log_ctx_extents(&ctx); + if (ret < 0) { + /* Fallthrough and commit/free transaction. */ + ret = BTRFS_LOG_FORCE_COMMIT; + } + + /* we've logged all the items and now have a consistent + * version of the file in the log. It is possible that + * someone will come in and modify the file, but that's + * fine because the log is consistent on disk, and we + * have references to all of the file's extents + * + * It is possible that someone will come in and log the + * file again, but that will end up using the synchronization + * inside btrfs_sync_log to keep things safe. + */ + if (skip_ilock) + up_write(&inode->i_mmap_lock); + else + btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP); + + if (ret == BTRFS_NO_LOG_SYNC) { + ret = btrfs_end_transaction(trans); + goto out; + } + + /* We successfully logged the inode, attempt to sync the log. */ + if (!ret) { + ret = btrfs_sync_log(trans, root, &ctx); + if (!ret) { + ret = btrfs_end_transaction(trans); + goto out; + } + } + + /* + * At this point we need to commit the transaction because we had + * btrfs_need_log_full_commit() or some other error. + * + * If we didn't do a full sync we have to stop the trans handle, wait on + * the ordered extents, start it again and commit the transaction. If + * we attempt to wait on the ordered extents here we could deadlock with + * something like fallocate() that is holding the extent lock trying to + * start a transaction while some other thread is trying to commit the + * transaction while we (fsync) are currently holding the transaction + * open. + */ + if (!full_sync) { + ret = btrfs_end_transaction(trans); + if (ret) + goto out; + ret = btrfs_wait_ordered_range(inode, start, len); + if (ret) + goto out; + + /* + * This is safe to use here because we're only interested in + * making sure the transaction that had the ordered extents is + * committed. We aren't waiting on anything past this point, + * we're purely getting the transaction and committing it. + */ + trans = btrfs_attach_transaction_barrier(root); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + + /* + * We committed the transaction and there's no currently + * running transaction, this means everything we care + * about made it to disk and we are done. + */ + if (ret == -ENOENT) + ret = 0; + goto out; + } + } + + ret = btrfs_commit_transaction(trans); +out: + free_extent_buffer(ctx.scratch_eb); + ASSERT(list_empty(&ctx.list)); + ASSERT(list_empty(&ctx.conflict_inodes)); + ASSERT(ret <= 0, "ret=%d", ret); + /* + * Ordered extents might have started and completed before this fsync, + * so check for any io errors and advance the writeback error sequence. + */ + err = file_check_and_advance_wb_err(file); + if (!ret) + ret = err; + trace_btrfs_sync_file_exit(file, ret); + + return ret; + +out_release_extents: + btrfs_release_log_ctx_extents(&ctx); + if (skip_ilock) + up_write(&inode->i_mmap_lock); + else + btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP); + goto out; +} + +/* + * btrfs_page_mkwrite() is not allowed to change the file size as it gets + * called from a page fault handler when a page is first dirtied. Hence we must + * be careful to check for EOF conditions here. We set the page up correctly + * for a written page which means we get ENOSPC checking when writing into + * holes and correct delalloc and unwritten extent mapping on filesystems that + * support these features. + * + * We are not allowed to take the i_mutex here so we have to play games to + * protect against truncate races as the page could now be beyond EOF. Because + * truncate_setsize() writes the inode size before removing pages, once we have + * the page lock we can determine safely if the page is beyond EOF. If it is not + * beyond EOF, then the page is guaranteed safe against truncation until we + * unlock the page. + */ +static vm_fault_t btrfs_page_mkwrite(struct vm_fault *vmf) +{ + struct page *page = vmf->page; + struct folio *folio = page_folio(page); + struct btrfs_inode *inode = BTRFS_I(file_inode(vmf->vma->vm_file)); + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct extent_io_tree *io_tree = &inode->io_tree; + struct btrfs_ordered_extent *ordered; + struct extent_state *cached_state = NULL; + struct extent_changeset *data_reserved = NULL; + unsigned long zero_start; + loff_t size; + size_t fsize = folio_size(folio); + int ret; + bool only_release_metadata = false; + u64 reserved_space; + u64 page_start; + u64 page_end; + u64 end; + + reserved_space = fsize; + + sb_start_pagefault(inode->vfs_inode.i_sb); + page_start = folio_pos(folio); + page_end = page_start + folio_size(folio) - 1; + end = page_end; + + /* + * Reserving delalloc space after obtaining the page lock can lead to + * deadlock. For example, if a dirty page is locked by this function + * and the call to btrfs_delalloc_reserve_space() ends up triggering + * dirty page write out, then the btrfs_writepages() function could + * end up waiting indefinitely to get a lock on the page currently + * being processed by btrfs_page_mkwrite() function. + */ + ret = btrfs_check_data_free_space(inode, &data_reserved, page_start, + reserved_space, false); + if (ret < 0) { + size_t write_bytes = reserved_space; + + if (btrfs_check_nocow_lock(inode, page_start, &write_bytes, false) <= 0) + goto out_noreserve; + + only_release_metadata = true; + + /* + * Can't write the whole range, there may be shared extents or + * holes in the range, bail out with @only_release_metadata set + * to true so that we unlock the nocow lock before returning the + * error. + */ + if (write_bytes < reserved_space) + goto out_noreserve; + } + ret = btrfs_delalloc_reserve_metadata(inode, reserved_space, + reserved_space, false); + if (ret < 0) { + if (!only_release_metadata) + btrfs_free_reserved_data_space(inode, data_reserved, + page_start, reserved_space); + goto out_noreserve; + } + + ret = file_update_time(vmf->vma->vm_file); + if (ret < 0) + goto out; +again: + down_read(&inode->i_mmap_lock); + folio_lock(folio); + size = i_size_read(&inode->vfs_inode); + + if ((folio->mapping != inode->vfs_inode.i_mapping) || + (page_start >= size)) { + /* Page got truncated out from underneath us. */ + goto out_unlock; + } + folio_wait_writeback(folio); + + btrfs_lock_extent(io_tree, page_start, page_end, &cached_state); + ret = set_folio_extent_mapped(folio); + if (ret < 0) { + btrfs_unlock_extent(io_tree, page_start, page_end, &cached_state); + goto out_unlock; + } + + /* + * We can't set the delalloc bits if there are pending ordered + * extents. Drop our locks and wait for them to finish. + */ + ordered = btrfs_lookup_ordered_range(inode, page_start, fsize); + if (ordered) { + btrfs_unlock_extent(io_tree, page_start, page_end, &cached_state); + folio_unlock(folio); + up_read(&inode->i_mmap_lock); + btrfs_start_ordered_extent(ordered); + btrfs_put_ordered_extent(ordered); + goto again; + } + + if (folio_contains(folio, (size - 1) >> PAGE_SHIFT)) { + reserved_space = round_up(size - page_start, fs_info->sectorsize); + if (reserved_space < fsize) { + const u64 to_free = fsize - reserved_space; + + end = page_start + reserved_space - 1; + if (only_release_metadata) + btrfs_delalloc_release_metadata(inode, to_free, true); + else + btrfs_delalloc_release_space(inode, data_reserved, + end + 1, to_free, true); + } + } + + ret = btrfs_reset_extent_delalloc(inode, page_start, end, 0, &cached_state); + if (ret < 0) { + btrfs_unlock_extent(io_tree, page_start, page_end, &cached_state); + goto out_unlock; + } + + /* Page is wholly or partially inside EOF. */ + if (page_start + folio_size(folio) > size) + zero_start = offset_in_folio(folio, size); + else + zero_start = fsize; + + if (zero_start != fsize) + folio_zero_range(folio, zero_start, folio_size(folio) - zero_start); + + btrfs_folio_set_dirty(fs_info, folio, page_start, end + 1 - page_start); + btrfs_folio_set_uptodate(fs_info, folio, page_start, end + 1 - page_start); + + btrfs_set_inode_last_sub_trans(inode); + + if (only_release_metadata) + btrfs_set_extent_bit(io_tree, page_start, end, EXTENT_NORESERVE, + &cached_state); + + btrfs_unlock_extent(io_tree, page_start, page_end, &cached_state); + up_read(&inode->i_mmap_lock); + + btrfs_delalloc_release_extents(inode, fsize); + if (only_release_metadata) + btrfs_check_nocow_unlock(inode); + sb_end_pagefault(inode->vfs_inode.i_sb); + extent_changeset_free(data_reserved); + return VM_FAULT_LOCKED; + +out_unlock: + folio_unlock(folio); + up_read(&inode->i_mmap_lock); +out: + btrfs_delalloc_release_extents(inode, fsize); + if (only_release_metadata) + btrfs_delalloc_release_metadata(inode, reserved_space, true); + else + btrfs_delalloc_release_space(inode, data_reserved, page_start, + reserved_space, true); +out_noreserve: + if (only_release_metadata) + btrfs_check_nocow_unlock(inode); + + sb_end_pagefault(inode->vfs_inode.i_sb); + + extent_changeset_free(data_reserved); + + if (ret < 0) + return vmf_error(ret); + + /* Make the VM retry the fault. */ + return VM_FAULT_NOPAGE; +} + +static const struct vm_operations_struct btrfs_file_vm_ops = { + .fault = filemap_fault, + .map_pages = filemap_map_pages, + .page_mkwrite = btrfs_page_mkwrite, +}; + +static int btrfs_file_mmap_prepare(struct vm_area_desc *desc) +{ + struct file *filp = desc->file; + struct address_space *mapping = filp->f_mapping; + + if (btrfs_is_shutdown(inode_to_fs_info(file_inode(filp)))) + return -EIO; + if (!mapping->a_ops->read_folio) + return -ENOEXEC; + + file_accessed(filp); + desc->vm_ops = &btrfs_file_vm_ops; + + return 0; +} + +static bool hole_mergeable(struct btrfs_inode *inode, struct extent_buffer *leaf, + int slot, u64 start, u64 end) +{ + struct btrfs_file_extent_item *fi; + struct btrfs_key key; + + if (slot < 0 || slot >= btrfs_header_nritems(leaf)) + return false; + + btrfs_item_key_to_cpu(leaf, &key, slot); + if (key.objectid != btrfs_ino(inode) || + key.type != BTRFS_EXTENT_DATA_KEY) + return false; + + fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item); + + if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG) + return false; + + if (btrfs_file_extent_disk_bytenr(leaf, fi)) + return false; + + if (key.offset == end) + return true; + if (key.offset + btrfs_file_extent_num_bytes(leaf, fi) == start) + return true; + return false; +} + +static int fill_holes(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_path *path, u64 offset, u64 end) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *root = inode->root; + struct extent_buffer *leaf; + struct btrfs_file_extent_item *fi; + struct extent_map *hole_em; + struct btrfs_key key; + int modify_slot = -1; + int del_slot = -1; + bool update_offset = false; + u64 num_bytes = 0; + int ret; + + if (btrfs_fs_incompat(fs_info, NO_HOLES)) + goto out; + + key.objectid = btrfs_ino(inode); + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = offset; + + ret = btrfs_search_slot(trans, root, &key, path, -1, 1); + if (ret <= 0) { + /* + * We should have dropped this offset, so if we find it then + * something has gone horribly wrong. + */ + if (ret == 0) + ret = -EINVAL; + return ret; + } + + leaf = path->nodes[0]; + if (hole_mergeable(inode, leaf, path->slots[0] - 1, offset, end)) { + fi = btrfs_item_ptr(leaf, path->slots[0] - 1, + struct btrfs_file_extent_item); + num_bytes = btrfs_file_extent_num_bytes(leaf, fi) + + end - offset; + modify_slot = path->slots[0] - 1; + } + if (hole_mergeable(inode, leaf, path->slots[0], offset, end)) { + fi = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_file_extent_item); + if (modify_slot != -1) { + num_bytes += btrfs_file_extent_num_bytes(leaf, fi); + del_slot = path->slots[0]; + } else { + num_bytes = btrfs_file_extent_num_bytes(leaf, fi) + + end - offset; + modify_slot = path->slots[0]; + update_offset = true; + } + } + if (modify_slot >= 0) { + fi = btrfs_item_ptr(leaf, modify_slot, + struct btrfs_file_extent_item); + btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes); + btrfs_set_file_extent_ram_bytes(leaf, fi, num_bytes); + if (update_offset) { + key.offset = offset; + btrfs_set_item_key_safe(trans, path, &key); + } + btrfs_set_file_extent_offset(leaf, fi, 0); + btrfs_set_file_extent_generation(leaf, fi, trans->transid); + if (del_slot >= 0) { + ret = btrfs_del_items(trans, root, path, del_slot, 1); + if (ret) { + btrfs_abort_transaction(trans, ret); + btrfs_release_path(path); + return ret; + } + } + goto out; + } + btrfs_release_path(path); + + ret = btrfs_insert_hole_extent(trans, root, btrfs_ino(inode), offset, + end - offset); + if (ret) + return ret; + +out: + btrfs_release_path(path); + + hole_em = btrfs_alloc_extent_map(); + if (!hole_em) { + btrfs_drop_extent_map_range(inode, offset, end - 1, false); + btrfs_set_inode_full_sync(inode); + } else { + hole_em->start = offset; + hole_em->len = end - offset; + hole_em->ram_bytes = hole_em->len; + + hole_em->disk_bytenr = EXTENT_MAP_HOLE; + hole_em->disk_num_bytes = 0; + hole_em->generation = trans->transid; + + ret = btrfs_replace_extent_map_range(inode, hole_em, true); + btrfs_free_extent_map(hole_em); + if (ret) + btrfs_set_inode_full_sync(inode); + } + + return 0; +} + +/* + * Find a hole extent on given inode and change start/len to the end of hole + * extent.(hole/vacuum extent whose em->start <= start && + * em->start + em->len > start) + * When a hole extent is found, return 1 and modify start/len. + */ +static int find_first_non_hole(struct btrfs_inode *inode, u64 *start, u64 *len) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct extent_map *em; + int ret = 0; + + em = btrfs_get_extent(inode, NULL, + round_down(*start, fs_info->sectorsize), + round_up(*len, fs_info->sectorsize)); + if (IS_ERR(em)) + return PTR_ERR(em); + + /* Hole or vacuum extent(only exists in no-hole mode) */ + if (em->disk_bytenr == EXTENT_MAP_HOLE) { + const u64 em_end = btrfs_extent_map_end(em); + + ret = 1; + *len = (em_end > *start + *len) ? 0 : (*start + *len - em_end); + *start = em_end; + } + btrfs_free_extent_map(em); + return ret; +} + +/* + * Check if there is no folio in the range. + * + * We cannot utilize filemap_range_has_page() in a filemap with large folios + * as we can hit the following false positive: + * + * start end + * | | + * |//|//|//|//| | | | | | | | |//|//| + * \ / \ / + * Folio A Folio B + * + * That large folio A and B cover the start and end indexes. + * In that case filemap_range_has_page() will always return true, but the above + * case is fine for btrfs_punch_hole_lock_range() usage. + * + * So here we only ensure that no other folios is in the range, excluding the + * head/tail large folio. + */ +static bool check_range_has_page(struct inode *inode, u64 start, u64 end) +{ + struct folio_batch fbatch; + bool ret = false; + /* + * For subpage case, if the range is not at page boundary, we could + * have pages at the leading/tailing part of the range. + * This could lead to dead loop since filemap_range_has_page() + * will always return true. + * So here we need to do extra page alignment for + * filemap_range_has_page(). + * + * And do not decrease page_lockend right now, as it can be 0. + */ + const u64 page_lockstart = round_up(start, PAGE_SIZE); + const u64 page_lockend = round_down(end + 1, PAGE_SIZE); + const pgoff_t start_index = page_lockstart >> PAGE_SHIFT; + const pgoff_t end_index = (page_lockend - 1) >> PAGE_SHIFT; + pgoff_t tmp = start_index; + int found_folios; + + /* The same page or adjacent pages. */ + if (page_lockend <= page_lockstart) + return false; + + folio_batch_init(&fbatch); + found_folios = filemap_get_folios(inode->i_mapping, &tmp, end_index, &fbatch); + for (int i = 0; i < found_folios; i++) { + struct folio *folio = fbatch.folios[i]; + + /* A large folio begins before the start. Not a target. */ + if (folio->index < start_index) + continue; + /* A large folio extends beyond the end. Not a target. */ + if (folio_next_index(folio) > end_index) + continue; + /* A folio doesn't cover the head/tail index. Found a target. */ + ret = true; + break; + } + folio_batch_release(&fbatch); + return ret; +} + +static void btrfs_punch_hole_lock_range(struct inode *inode, + const u64 lockstart, const u64 lockend, + struct extent_state **cached_state) +{ + while (1) { + truncate_pagecache_range(inode, lockstart, lockend); + + btrfs_lock_extent(&BTRFS_I(inode)->io_tree, lockstart, lockend, + cached_state); + /* + * We can't have ordered extents in the range, nor dirty/writeback + * pages, because we have locked the inode's VFS lock in exclusive + * mode, we have locked the inode's i_mmap_lock in exclusive mode, + * we have flushed all delalloc in the range and we have waited + * for any ordered extents in the range to complete. + * We can race with anyone reading pages from this range, so after + * locking the range check if we have pages in the range, and if + * we do, unlock the range and retry. + */ + if (!check_range_has_page(inode, lockstart, lockend)) + break; + + btrfs_unlock_extent(&BTRFS_I(inode)->io_tree, lockstart, lockend, + cached_state); + } + + btrfs_assert_inode_range_clean(BTRFS_I(inode), lockstart, lockend); +} + +static int btrfs_insert_replace_extent(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_path *path, + struct btrfs_replace_extent_info *extent_info, + const u64 replace_len, + const u64 bytes_to_drop) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *root = inode->root; + struct btrfs_file_extent_item *extent; + struct extent_buffer *leaf; + struct btrfs_key key; + int slot; + int ret; + + if (replace_len == 0) + return 0; + + if (extent_info->disk_offset == 0 && + btrfs_fs_incompat(fs_info, NO_HOLES)) { + btrfs_update_inode_bytes(inode, 0, bytes_to_drop); + return 0; + } + + key.objectid = btrfs_ino(inode); + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = extent_info->file_offset; + ret = btrfs_insert_empty_item(trans, root, path, &key, + sizeof(struct btrfs_file_extent_item)); + if (ret) + return ret; + leaf = path->nodes[0]; + slot = path->slots[0]; + write_extent_buffer(leaf, extent_info->extent_buf, + btrfs_item_ptr_offset(leaf, slot), + sizeof(struct btrfs_file_extent_item)); + extent = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item); + ASSERT(btrfs_file_extent_type(leaf, extent) != BTRFS_FILE_EXTENT_INLINE); + btrfs_set_file_extent_offset(leaf, extent, extent_info->data_offset); + btrfs_set_file_extent_num_bytes(leaf, extent, replace_len); + if (extent_info->is_new_extent) + btrfs_set_file_extent_generation(leaf, extent, trans->transid); + btrfs_release_path(path); + + ret = btrfs_inode_set_file_extent_range(inode, extent_info->file_offset, + replace_len); + if (ret) + return ret; + + /* If it's a hole, nothing more needs to be done. */ + if (extent_info->disk_offset == 0) { + btrfs_update_inode_bytes(inode, 0, bytes_to_drop); + return 0; + } + + btrfs_update_inode_bytes(inode, replace_len, bytes_to_drop); + + if (extent_info->is_new_extent && extent_info->insertions == 0) { + key.objectid = extent_info->disk_offset; + key.type = BTRFS_EXTENT_ITEM_KEY; + key.offset = extent_info->disk_len; + ret = btrfs_alloc_reserved_file_extent(trans, root, + btrfs_ino(inode), + extent_info->file_offset, + extent_info->qgroup_reserved, + &key); + } else { + struct btrfs_ref ref = { + .action = BTRFS_ADD_DELAYED_REF, + .bytenr = extent_info->disk_offset, + .num_bytes = extent_info->disk_len, + .owning_root = btrfs_root_id(root), + .ref_root = btrfs_root_id(root), + }; + u64 ref_offset; + + ref_offset = extent_info->file_offset - extent_info->data_offset; + btrfs_init_data_ref(&ref, btrfs_ino(inode), ref_offset, 0, false); + ret = btrfs_inc_extent_ref(trans, &ref); + } + + extent_info->insertions++; + + return ret; +} + +/* + * The respective range must have been previously locked, as well as the inode. + * The end offset is inclusive (last byte of the range). + * @extent_info is NULL for fallocate's hole punching and non-NULL when replacing + * the file range with an extent. + * When not punching a hole, we don't want to end up in a state where we dropped + * extents without inserting a new one, so we must abort the transaction to avoid + * a corruption. + */ +int btrfs_replace_file_extents(struct btrfs_inode *inode, + struct btrfs_path *path, const u64 start, + const u64 end, + struct btrfs_replace_extent_info *extent_info, + struct btrfs_trans_handle **trans_out) +{ + struct btrfs_drop_extents_args drop_args = { 0 }; + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + const u64 min_size = btrfs_calc_insert_metadata_size(fs_info, 1); + u64 ino_size = round_up(inode->vfs_inode.i_size, fs_info->sectorsize); + struct btrfs_trans_handle *trans = NULL; + struct btrfs_block_rsv rsv; + unsigned int rsv_count; + u64 cur_offset; + u64 len = end - start; + int ret = 0; + + if (end <= start) + return -EINVAL; + + btrfs_init_metadata_block_rsv(fs_info, &rsv, BTRFS_BLOCK_RSV_TEMP); + rsv.size = min_size; + rsv.failfast = true; + + /* + * 1 - update the inode + * 1 - removing the extents in the range + * 1 - adding the hole extent if no_holes isn't set or if we are + * replacing the range with a new extent + */ + if (!btrfs_fs_incompat(fs_info, NO_HOLES) || extent_info) + rsv_count = 3; + else + rsv_count = 2; + + trans = btrfs_start_transaction(root, rsv_count); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + trans = NULL; + goto out_release; + } + + ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, &rsv, + min_size, false); + if (WARN_ON(ret)) + goto out_trans; + trans->block_rsv = &rsv; + + cur_offset = start; + drop_args.path = path; + drop_args.end = end + 1; + drop_args.drop_cache = true; + while (cur_offset < end) { + drop_args.start = cur_offset; + ret = btrfs_drop_extents(trans, root, inode, &drop_args); + /* If we are punching a hole decrement the inode's byte count */ + if (!extent_info) + btrfs_update_inode_bytes(inode, 0, + drop_args.bytes_found); + if (ret != -ENOSPC) { + /* + * The only time we don't want to abort is if we are + * attempting to clone a partial inline extent, in which + * case we'll get EOPNOTSUPP. However if we aren't + * clone we need to abort no matter what, because if we + * got EOPNOTSUPP via prealloc then we messed up and + * need to abort. + */ + if (unlikely(ret && + (ret != -EOPNOTSUPP || + (extent_info && extent_info->is_new_extent)))) + btrfs_abort_transaction(trans, ret); + break; + } + + trans->block_rsv = &fs_info->trans_block_rsv; + + if (!extent_info && cur_offset < drop_args.drop_end && + cur_offset < ino_size) { + ret = fill_holes(trans, inode, path, cur_offset, + drop_args.drop_end); + if (unlikely(ret)) { + /* + * If we failed then we didn't insert our hole + * entries for the area we dropped, so now the + * fs is corrupted, so we must abort the + * transaction. + */ + btrfs_abort_transaction(trans, ret); + break; + } + } else if (!extent_info && cur_offset < drop_args.drop_end) { + /* + * We are past the i_size here, but since we didn't + * insert holes we need to clear the mapped area so we + * know to not set disk_i_size in this area until a new + * file extent is inserted here. + */ + ret = btrfs_inode_clear_file_extent_range(inode, + cur_offset, + drop_args.drop_end - cur_offset); + if (unlikely(ret)) { + /* + * We couldn't clear our area, so we could + * presumably adjust up and corrupt the fs, so + * we need to abort. + */ + btrfs_abort_transaction(trans, ret); + break; + } + } + + if (extent_info && + drop_args.drop_end > extent_info->file_offset) { + u64 replace_len = drop_args.drop_end - + extent_info->file_offset; + + ret = btrfs_insert_replace_extent(trans, inode, path, + extent_info, replace_len, + drop_args.bytes_found); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + break; + } + extent_info->data_len -= replace_len; + extent_info->data_offset += replace_len; + extent_info->file_offset += replace_len; + } + + /* + * We are releasing our handle on the transaction, balance the + * dirty pages of the btree inode and flush delayed items, and + * then get a new transaction handle, which may now point to a + * new transaction in case someone else may have committed the + * transaction we used to replace/drop file extent items. So + * bump the inode's iversion and update mtime and ctime except + * if we are called from a dedupe context. This is because a + * power failure/crash may happen after the transaction is + * committed and before we finish replacing/dropping all the + * file extent items we need. + */ + inode_inc_iversion(&inode->vfs_inode); + + if (!extent_info || extent_info->update_times) + inode_set_mtime_to_ts(&inode->vfs_inode, + inode_set_ctime_current(&inode->vfs_inode)); + + ret = btrfs_update_inode(trans, inode); + if (ret) + break; + + btrfs_end_transaction(trans); + btrfs_btree_balance_dirty(fs_info); + + trans = btrfs_start_transaction(root, rsv_count); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + trans = NULL; + break; + } + + ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, + &rsv, min_size, false); + if (WARN_ON(ret)) + break; + trans->block_rsv = &rsv; + + cur_offset = drop_args.drop_end; + len = end - cur_offset; + if (!extent_info && len) { + ret = find_first_non_hole(inode, &cur_offset, &len); + if (unlikely(ret < 0)) + break; + if (ret && !len) { + ret = 0; + break; + } + } + } + + /* + * If we were cloning, force the next fsync to be a full one since we + * we replaced (or just dropped in the case of cloning holes when + * NO_HOLES is enabled) file extent items and did not setup new extent + * maps for the replacement extents (or holes). + */ + if (extent_info && !extent_info->is_new_extent) + btrfs_set_inode_full_sync(inode); + + if (ret) + goto out_trans; + + trans->block_rsv = &fs_info->trans_block_rsv; + /* + * If we are using the NO_HOLES feature we might have had already an + * hole that overlaps a part of the region [lockstart, lockend] and + * ends at (or beyond) lockend. Since we have no file extent items to + * represent holes, drop_end can be less than lockend and so we must + * make sure we have an extent map representing the existing hole (the + * call to __btrfs_drop_extents() might have dropped the existing extent + * map representing the existing hole), otherwise the fast fsync path + * will not record the existence of the hole region + * [existing_hole_start, lockend]. + */ + if (drop_args.drop_end <= end) + drop_args.drop_end = end + 1; + /* + * Don't insert file hole extent item if it's for a range beyond eof + * (because it's useless) or if it represents a 0 bytes range (when + * cur_offset == drop_end). + */ + if (!extent_info && cur_offset < ino_size && + cur_offset < drop_args.drop_end) { + ret = fill_holes(trans, inode, path, cur_offset, + drop_args.drop_end); + if (unlikely(ret)) { + /* Same comment as above. */ + btrfs_abort_transaction(trans, ret); + goto out_trans; + } + } else if (!extent_info && cur_offset < drop_args.drop_end) { + /* See the comment in the loop above for the reasoning here. */ + ret = btrfs_inode_clear_file_extent_range(inode, cur_offset, + drop_args.drop_end - cur_offset); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out_trans; + } + + } + if (extent_info) { + ret = btrfs_insert_replace_extent(trans, inode, path, + extent_info, extent_info->data_len, + drop_args.bytes_found); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out_trans; + } + } + +out_trans: + if (!trans) + goto out_release; + + trans->block_rsv = &fs_info->trans_block_rsv; + if (ret) + btrfs_end_transaction(trans); + else + *trans_out = trans; +out_release: + btrfs_block_rsv_release(fs_info, &rsv, (u64)-1, NULL); + return ret; +} + +static int btrfs_punch_hole(struct file *file, loff_t offset, loff_t len) +{ + struct inode *inode = file_inode(file); + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + struct btrfs_root *root = BTRFS_I(inode)->root; + struct extent_state *cached_state = NULL; + struct btrfs_path *path; + struct btrfs_trans_handle *trans = NULL; + u64 lockstart; + u64 lockend; + u64 tail_start; + u64 tail_len; + const u64 orig_start = offset; + const u64 orig_end = offset + len - 1; + int ret = 0; + bool same_block; + u64 ino_size; + bool truncated_block = false; + bool updated_inode = false; + + btrfs_inode_lock(BTRFS_I(inode), BTRFS_ILOCK_MMAP); + + ret = btrfs_wait_ordered_range(BTRFS_I(inode), offset, len); + if (ret) + goto out_only_mutex; + + ino_size = round_up(inode->i_size, fs_info->sectorsize); + ret = find_first_non_hole(BTRFS_I(inode), &offset, &len); + if (ret < 0) + goto out_only_mutex; + if (ret && !len) { + /* Already in a large hole */ + ret = 0; + goto out_only_mutex; + } + + ret = file_modified(file); + if (ret) + goto out_only_mutex; + + lockstart = round_up(offset, fs_info->sectorsize); + lockend = round_down(offset + len, fs_info->sectorsize) - 1; + same_block = (offset >> fs_info->sectorsize_bits) == + ((offset + len - 1) >> fs_info->sectorsize_bits); + /* + * Only do this if we are in the same block and we aren't doing the + * entire block. + */ + if (same_block && len < fs_info->sectorsize) { + if (offset < ino_size) { + truncated_block = true; + ret = btrfs_truncate_block(BTRFS_I(inode), offset + len - 1, + orig_start, orig_end); + } else { + ret = 0; + } + goto out_only_mutex; + } + + /* zero back part of the first block */ + if (offset < ino_size) { + truncated_block = true; + ret = btrfs_truncate_block(BTRFS_I(inode), offset, orig_start, orig_end); + if (ret) { + btrfs_inode_unlock(BTRFS_I(inode), BTRFS_ILOCK_MMAP); + return ret; + } + } + + /* Check the aligned pages after the first unaligned page, + * if offset != orig_start, which means the first unaligned page + * including several following pages are already in holes, + * the extra check can be skipped */ + if (offset == orig_start) { + /* after truncate page, check hole again */ + len = offset + len - lockstart; + offset = lockstart; + ret = find_first_non_hole(BTRFS_I(inode), &offset, &len); + if (ret < 0) + goto out_only_mutex; + if (ret && !len) { + ret = 0; + goto out_only_mutex; + } + lockstart = offset; + } + + /* Check the tail unaligned part is in a hole */ + tail_start = lockend + 1; + tail_len = offset + len - tail_start; + if (tail_len) { + ret = find_first_non_hole(BTRFS_I(inode), &tail_start, &tail_len); + if (unlikely(ret < 0)) + goto out_only_mutex; + if (!ret) { + /* zero the front end of the last page */ + if (tail_start + tail_len < ino_size) { + truncated_block = true; + ret = btrfs_truncate_block(BTRFS_I(inode), + tail_start + tail_len - 1, + orig_start, orig_end); + if (ret) + goto out_only_mutex; + } + } + } + + if (lockend < lockstart) { + ret = 0; + goto out_only_mutex; + } + + btrfs_punch_hole_lock_range(inode, lockstart, lockend, &cached_state); + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto out; + } + + ret = btrfs_replace_file_extents(BTRFS_I(inode), path, lockstart, + lockend, NULL, &trans); + btrfs_free_path(path); + if (ret) + goto out; + + ASSERT(trans != NULL); + inode_inc_iversion(inode); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + ret = btrfs_update_inode(trans, BTRFS_I(inode)); + updated_inode = true; + btrfs_end_transaction(trans); + btrfs_btree_balance_dirty(fs_info); +out: + btrfs_unlock_extent(&BTRFS_I(inode)->io_tree, lockstart, lockend, + &cached_state); +out_only_mutex: + if (!updated_inode && truncated_block && !ret) { + /* + * If we only end up zeroing part of a page, we still need to + * update the inode item, so that all the time fields are + * updated as well as the necessary btrfs inode in memory fields + * for detecting, at fsync time, if the inode isn't yet in the + * log tree or it's there but not up to date. + */ + struct timespec64 now = inode_set_ctime_current(inode); + + inode_inc_iversion(inode); + inode_set_mtime_to_ts(inode, now); + trans = btrfs_start_transaction(root, 1); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + } else { + int ret2; + + ret = btrfs_update_inode(trans, BTRFS_I(inode)); + ret2 = btrfs_end_transaction(trans); + if (!ret) + ret = ret2; + } + } + btrfs_inode_unlock(BTRFS_I(inode), BTRFS_ILOCK_MMAP); + return ret; +} + +/* Helper structure to record which range is already reserved */ +struct falloc_range { + struct list_head list; + u64 start; + u64 len; +}; + +/* + * Helper function to add falloc range + * + * Caller should have locked the larger range of extent containing + * [start, len) + */ +static int add_falloc_range(struct list_head *head, u64 start, u64 len) +{ + struct falloc_range *range = NULL; + + if (!list_empty(head)) { + /* + * As fallocate iterates by bytenr order, we only need to check + * the last range. + */ + range = list_last_entry(head, struct falloc_range, list); + if (range->start + range->len == start) { + range->len += len; + return 0; + } + } + + range = kmalloc_obj(*range); + if (!range) + return -ENOMEM; + range->start = start; + range->len = len; + list_add_tail(&range->list, head); + return 0; +} + +static int btrfs_fallocate_update_isize(struct inode *inode, + const u64 end, + const int mode) +{ + struct btrfs_trans_handle *trans; + struct btrfs_root *root = BTRFS_I(inode)->root; + u64 range_start; + u64 range_end; + int ret; + int ret2; + + if (mode & FALLOC_FL_KEEP_SIZE || end <= i_size_read(inode)) + return 0; + + range_start = round_down(i_size_read(inode), root->fs_info->sectorsize); + range_end = round_up(end, root->fs_info->sectorsize); + + ret = btrfs_inode_set_file_extent_range(BTRFS_I(inode), range_start, + range_end - range_start); + if (ret) + return ret; + + trans = btrfs_start_transaction(root, 1); + if (IS_ERR(trans)) + return PTR_ERR(trans); + + inode_set_ctime_current(inode); + i_size_write(inode, end); + btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), 0); + ret = btrfs_update_inode(trans, BTRFS_I(inode)); + ret2 = btrfs_end_transaction(trans); + + return ret ? ret : ret2; +} + +enum { + RANGE_BOUNDARY_WRITTEN_EXTENT, + RANGE_BOUNDARY_PREALLOC_EXTENT, + RANGE_BOUNDARY_HOLE, +}; + +static int btrfs_zero_range_check_range_boundary(struct btrfs_inode *inode, + u64 offset) +{ + const u32 sectorsize = inode->root->fs_info->sectorsize; + struct extent_map *em; + int ret; + + offset = round_down(offset, sectorsize); + em = btrfs_get_extent(inode, NULL, offset, sectorsize); + if (IS_ERR(em)) + return PTR_ERR(em); + + if (em->disk_bytenr == EXTENT_MAP_HOLE) + ret = RANGE_BOUNDARY_HOLE; + else if (em->flags & EXTENT_FLAG_PREALLOC) + ret = RANGE_BOUNDARY_PREALLOC_EXTENT; + else + ret = RANGE_BOUNDARY_WRITTEN_EXTENT; + + btrfs_free_extent_map(em); + return ret; +} + +static int btrfs_zero_range(struct inode *inode, + loff_t offset, + loff_t len, + const int mode) +{ + struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; + struct extent_map *em; + struct extent_changeset *data_reserved = NULL; + int ret; + u64 alloc_hint = 0; + const u32 sectorsize = fs_info->sectorsize; + const u64 orig_start = offset; + const u64 orig_end = offset + len - 1; + u64 alloc_start = round_down(offset, sectorsize); + u64 alloc_end = round_up(offset + len, sectorsize); + u64 bytes_to_reserve = 0; + bool space_reserved = false; + + em = btrfs_get_extent(BTRFS_I(inode), NULL, alloc_start, + alloc_end - alloc_start); + if (IS_ERR(em)) { + ret = PTR_ERR(em); + goto out; + } + + /* + * Avoid hole punching and extent allocation for some cases. More cases + * could be considered, but these are unlikely common and we keep things + * as simple as possible for now. Also, intentionally, if the target + * range contains one or more prealloc extents together with regular + * extents and holes, we drop all the existing extents and allocate a + * new prealloc extent, so that we get a larger contiguous disk extent. + */ + if (em->start <= alloc_start && (em->flags & EXTENT_FLAG_PREALLOC)) { + const u64 em_end = btrfs_extent_map_end(em); + + if (em_end >= offset + len) { + /* + * The whole range is already a prealloc extent, + * do nothing except updating the inode's i_size if + * needed. + */ + btrfs_free_extent_map(em); + ret = btrfs_fallocate_update_isize(inode, offset + len, + mode); + goto out; + } + /* + * Part of the range is already a prealloc extent, so operate + * only on the remaining part of the range. + */ + alloc_start = em_end; + ASSERT(IS_ALIGNED(alloc_start, sectorsize)); + len = offset + len - alloc_start; + offset = alloc_start; + alloc_hint = btrfs_extent_map_block_start(em) + em->len; + } + btrfs_free_extent_map(em); + + if ((offset >> fs_info->sectorsize_bits) == + ((offset + len - 1) >> fs_info->sectorsize_bits)) { + em = btrfs_get_extent(BTRFS_I(inode), NULL, alloc_start, sectorsize); + if (IS_ERR(em)) { + ret = PTR_ERR(em); + goto out; + } + + if (em->flags & EXTENT_FLAG_PREALLOC) { + btrfs_free_extent_map(em); + ret = btrfs_fallocate_update_isize(inode, offset + len, + mode); + goto out; + } + if (len < sectorsize && em->disk_bytenr != EXTENT_MAP_HOLE) { + btrfs_free_extent_map(em); + ret = btrfs_truncate_block(BTRFS_I(inode), offset + len - 1, + orig_start, orig_end); + if (!ret) + ret = btrfs_fallocate_update_isize(inode, + offset + len, + mode); + return ret; + } + btrfs_free_extent_map(em); + alloc_start = round_down(offset, sectorsize); + alloc_end = alloc_start + sectorsize; + goto reserve_space; + } + + alloc_start = round_up(offset, sectorsize); + alloc_end = round_down(offset + len, sectorsize); + + /* + * For unaligned ranges, check the pages at the boundaries, they might + * map to an extent, in which case we need to partially zero them, or + * they might map to a hole, in which case we need our allocation range + * to cover them. + */ + if (!IS_ALIGNED(offset, sectorsize)) { + ret = btrfs_zero_range_check_range_boundary(BTRFS_I(inode), + offset); + if (ret < 0) + goto out; + if (ret == RANGE_BOUNDARY_HOLE) { + alloc_start = round_down(offset, sectorsize); + ret = 0; + } else if (ret == RANGE_BOUNDARY_WRITTEN_EXTENT) { + ret = btrfs_truncate_block(BTRFS_I(inode), offset, + orig_start, orig_end); + if (ret) + goto out; + } else { + ret = 0; + } + } + + if (!IS_ALIGNED(offset + len, sectorsize)) { + ret = btrfs_zero_range_check_range_boundary(BTRFS_I(inode), + offset + len); + if (ret < 0) + goto out; + if (ret == RANGE_BOUNDARY_HOLE) { + alloc_end = round_up(offset + len, sectorsize); + ret = 0; + } else if (ret == RANGE_BOUNDARY_WRITTEN_EXTENT) { + ret = btrfs_truncate_block(BTRFS_I(inode), offset + len - 1, + orig_start, orig_end); + if (ret) + goto out; + } else { + ret = 0; + } + } + +reserve_space: + if (alloc_start < alloc_end) { + struct extent_state *cached_state = NULL; + const u64 lockstart = alloc_start; + const u64 lockend = alloc_end - 1; + + bytes_to_reserve = alloc_end - alloc_start; + ret = btrfs_alloc_data_chunk_ondemand(BTRFS_I(inode), + bytes_to_reserve); + if (ret < 0) + goto out; + space_reserved = true; + btrfs_punch_hole_lock_range(inode, lockstart, lockend, + &cached_state); + ret = btrfs_qgroup_reserve_data(BTRFS_I(inode), &data_reserved, + alloc_start, bytes_to_reserve); + if (ret) { + btrfs_unlock_extent(&BTRFS_I(inode)->io_tree, lockstart, + lockend, &cached_state); + goto out; + } + ret = btrfs_prealloc_file_range(inode, mode, alloc_start, + alloc_end - alloc_start, + fs_info->sectorsize, + offset + len, &alloc_hint); + btrfs_unlock_extent(&BTRFS_I(inode)->io_tree, lockstart, lockend, + &cached_state); + /* btrfs_prealloc_file_range releases reserved space on error */ + if (ret) { + space_reserved = false; + goto out; + } + } + ret = btrfs_fallocate_update_isize(inode, offset + len, mode); + out: + if (ret && space_reserved) + btrfs_free_reserved_data_space(BTRFS_I(inode), data_reserved, + alloc_start, bytes_to_reserve); + extent_changeset_free(data_reserved); + + return ret; +} + +static long btrfs_fallocate(struct file *file, int mode, + loff_t offset, loff_t len) +{ + struct inode *inode = file_inode(file); + struct extent_state *cached_state = NULL; + struct extent_changeset *data_reserved = NULL; + struct falloc_range *range; + struct falloc_range *tmp; + LIST_HEAD(reserve_list); + u64 cur_offset; + u64 last_byte; + u64 alloc_start; + u64 alloc_end; + u64 alloc_hint = 0; + u64 locked_end; + u64 actual_end = 0; + u64 data_space_needed = 0; + u64 data_space_reserved = 0; + u64 qgroup_reserved = 0; + struct extent_map *em; + int blocksize = BTRFS_I(inode)->root->fs_info->sectorsize; + int ret; + + if (btrfs_is_shutdown(inode_to_fs_info(inode))) + return -EIO; + + /* Do not allow fallocate in ZONED mode */ + if (btrfs_is_zoned(inode_to_fs_info(inode))) + return -EOPNOTSUPP; + + alloc_start = round_down(offset, blocksize); + alloc_end = round_up(offset + len, blocksize); + cur_offset = alloc_start; + + /* Make sure we aren't being give some crap mode */ + if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE | + FALLOC_FL_ZERO_RANGE)) + return -EOPNOTSUPP; + + if (mode & FALLOC_FL_PUNCH_HOLE) + return btrfs_punch_hole(file, offset, len); + + btrfs_inode_lock(BTRFS_I(inode), BTRFS_ILOCK_MMAP); + + if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size) { + ret = inode_newsize_ok(inode, offset + len); + if (ret) + goto out; + } + + ret = file_modified(file); + if (ret) + goto out; + + /* + * TODO: Move these two operations after we have checked + * accurate reserved space, or fallocate can still fail but + * with page truncated or size expanded. + * + * But that's a minor problem and won't do much harm BTW. + */ + if (alloc_start > inode->i_size) { + ret = btrfs_cont_expand(BTRFS_I(inode), i_size_read(inode), + alloc_start); + if (ret) + goto out; + } else if (offset + len > inode->i_size) { + /* + * If we are fallocating from the end of the file onward we + * need to zero out the end of the block if i_size lands in the + * middle of a block. + */ + ret = btrfs_truncate_block(BTRFS_I(inode), inode->i_size, + inode->i_size, (u64)-1); + if (ret) + goto out; + } + + /* + * We have locked the inode at the VFS level (in exclusive mode) and we + * have locked the i_mmap_lock lock (in exclusive mode). Now before + * locking the file range, flush all dealloc in the range and wait for + * all ordered extents in the range to complete. After this we can lock + * the file range and, due to the previous locking we did, we know there + * can't be more delalloc or ordered extents in the range. + */ + ret = btrfs_wait_ordered_range(BTRFS_I(inode), alloc_start, + alloc_end - alloc_start); + if (ret) + goto out; + + if (mode & FALLOC_FL_ZERO_RANGE) { + ret = btrfs_zero_range(inode, offset, len, mode); + btrfs_inode_unlock(BTRFS_I(inode), BTRFS_ILOCK_MMAP); + return ret; + } + + locked_end = alloc_end - 1; + btrfs_lock_extent(&BTRFS_I(inode)->io_tree, alloc_start, locked_end, + &cached_state); + + btrfs_assert_inode_range_clean(BTRFS_I(inode), alloc_start, locked_end); + + /* First, check if we exceed the qgroup limit */ + while (cur_offset < alloc_end) { + em = btrfs_get_extent(BTRFS_I(inode), NULL, cur_offset, + alloc_end - cur_offset); + if (IS_ERR(em)) { + ret = PTR_ERR(em); + break; + } + last_byte = min(btrfs_extent_map_end(em), alloc_end); + actual_end = min_t(u64, btrfs_extent_map_end(em), offset + len); + last_byte = ALIGN(last_byte, blocksize); + if (em->disk_bytenr == EXTENT_MAP_HOLE || + (cur_offset >= inode->i_size && + !(em->flags & EXTENT_FLAG_PREALLOC))) { + const u64 range_len = last_byte - cur_offset; + + ret = add_falloc_range(&reserve_list, cur_offset, range_len); + if (ret < 0) { + btrfs_free_extent_map(em); + break; + } + ret = btrfs_qgroup_reserve_data(BTRFS_I(inode), + &data_reserved, cur_offset, range_len); + if (ret < 0) { + btrfs_free_extent_map(em); + break; + } + qgroup_reserved += range_len; + data_space_needed += range_len; + } + btrfs_free_extent_map(em); + cur_offset = last_byte; + } + + if (!ret && data_space_needed > 0) { + /* + * We are safe to reserve space here as we can't have delalloc + * in the range, see above. + */ + ret = btrfs_alloc_data_chunk_ondemand(BTRFS_I(inode), + data_space_needed); + if (!ret) + data_space_reserved = data_space_needed; + } + + /* + * If ret is still 0, means we're OK to fallocate. + * Or just cleanup the list and exit. + */ + list_for_each_entry_safe(range, tmp, &reserve_list, list) { + if (!ret) { + ret = btrfs_prealloc_file_range(inode, mode, + range->start, + range->len, blocksize, + offset + len, &alloc_hint); + /* + * btrfs_prealloc_file_range() releases space even + * if it returns an error. + */ + data_space_reserved -= range->len; + qgroup_reserved -= range->len; + } else if (data_space_reserved > 0) { + btrfs_free_reserved_data_space(BTRFS_I(inode), + data_reserved, range->start, + range->len); + data_space_reserved -= range->len; + qgroup_reserved -= range->len; + } else if (qgroup_reserved > 0) { + btrfs_qgroup_free_data(BTRFS_I(inode), data_reserved, + range->start, range->len, NULL); + qgroup_reserved -= range->len; + } + list_del(&range->list); + kfree(range); + } + if (ret < 0) + goto out_unlock; + + /* + * We didn't need to allocate any more space, but we still extended the + * size of the file so we need to update i_size and the inode item. + */ + ret = btrfs_fallocate_update_isize(inode, actual_end, mode); +out_unlock: + btrfs_unlock_extent(&BTRFS_I(inode)->io_tree, alloc_start, locked_end, + &cached_state); +out: + btrfs_inode_unlock(BTRFS_I(inode), BTRFS_ILOCK_MMAP); + extent_changeset_free(data_reserved); + return ret; +} + +/* + * Helper for btrfs_find_delalloc_in_range(). Find a subrange in a given range + * that has unflushed and/or flushing delalloc. There might be other adjacent + * subranges after the one it found, so btrfs_find_delalloc_in_range() keeps + * looping while it gets adjacent subranges, and merging them together. + */ +static bool find_delalloc_subrange(struct btrfs_inode *inode, u64 start, u64 end, + struct extent_state **cached_state, + bool *search_io_tree, + u64 *delalloc_start_ret, u64 *delalloc_end_ret) +{ + u64 len = end + 1 - start; + u64 delalloc_len = 0; + struct btrfs_ordered_extent *oe; + u64 oe_start; + u64 oe_end; + + /* + * Search the io tree first for EXTENT_DELALLOC. If we find any, it + * means we have delalloc (dirty pages) for which writeback has not + * started yet. + */ + if (*search_io_tree) { + spin_lock(&inode->lock); + if (inode->delalloc_bytes > 0) { + spin_unlock(&inode->lock); + *delalloc_start_ret = start; + delalloc_len = btrfs_count_range_bits(&inode->io_tree, + delalloc_start_ret, end, + len, EXTENT_DELALLOC, + true, cached_state); + } else { + spin_unlock(&inode->lock); + } + } + + if (delalloc_len > 0) { + /* + * If delalloc was found then *delalloc_start_ret has a sector size + * aligned value (rounded down). + */ + *delalloc_end_ret = *delalloc_start_ret + delalloc_len - 1; + + if (*delalloc_start_ret == start) { + /* Delalloc for the whole range, nothing more to do. */ + if (*delalloc_end_ret == end) + return true; + /* Else trim our search range for ordered extents. */ + start = *delalloc_end_ret + 1; + len = end + 1 - start; + } + } else { + /* No delalloc, future calls don't need to search again. */ + *search_io_tree = false; + } + + /* + * Now also check if there's any ordered extent in the range. + * We do this because: + * + * 1) When delalloc is flushed, the file range is locked, we clear the + * EXTENT_DELALLOC bit from the io tree and create an extent map and + * an ordered extent for the write. So we might just have been called + * after delalloc is flushed and before the ordered extent completes + * and inserts the new file extent item in the subvolume's btree; + * + * 2) We may have an ordered extent created by flushing delalloc for a + * subrange that starts before the subrange we found marked with + * EXTENT_DELALLOC in the io tree. + * + * We could also use the extent map tree to find such delalloc that is + * being flushed, but using the ordered extents tree is more efficient + * because it's usually much smaller as ordered extents are removed from + * the tree once they complete. With the extent maps, we may have them + * in the extent map tree for a very long time, and they were either + * created by previous writes or loaded by read operations. + */ + oe = btrfs_lookup_first_ordered_range(inode, start, len); + if (!oe) + return (delalloc_len > 0); + + /* The ordered extent may span beyond our search range. */ + oe_start = max(oe->file_offset, start); + oe_end = min(oe->file_offset + oe->num_bytes - 1, end); + + btrfs_put_ordered_extent(oe); + + /* Don't have unflushed delalloc, return the ordered extent range. */ + if (delalloc_len == 0) { + *delalloc_start_ret = oe_start; + *delalloc_end_ret = oe_end; + return true; + } + + /* + * We have both unflushed delalloc (io_tree) and an ordered extent. + * If the ranges are adjacent returned a combined range, otherwise + * return the leftmost range. + */ + if (oe_start < *delalloc_start_ret) { + if (oe_end < *delalloc_start_ret) + *delalloc_end_ret = oe_end; + *delalloc_start_ret = oe_start; + } else if (*delalloc_end_ret + 1 == oe_start) { + *delalloc_end_ret = oe_end; + } + + return true; +} + +/* + * Check if there's delalloc in a given range. + * + * @inode: The inode. + * @start: The start offset of the range. It does not need to be + * sector size aligned. + * @end: The end offset (inclusive value) of the search range. + * It does not need to be sector size aligned. + * @cached_state: Extent state record used for speeding up delalloc + * searches in the inode's io_tree. Can be NULL. + * @delalloc_start_ret: Output argument, set to the start offset of the + * subrange found with delalloc (may not be sector size + * aligned). + * @delalloc_end_ret: Output argument, set to he end offset (inclusive value) + * of the subrange found with delalloc. + * + * Returns true if a subrange with delalloc is found within the given range, and + * if so it sets @delalloc_start_ret and @delalloc_end_ret with the start and + * end offsets of the subrange. + */ +bool btrfs_find_delalloc_in_range(struct btrfs_inode *inode, u64 start, u64 end, + struct extent_state **cached_state, + u64 *delalloc_start_ret, u64 *delalloc_end_ret) +{ + u64 cur_offset = round_down(start, inode->root->fs_info->sectorsize); + u64 prev_delalloc_end = 0; + bool search_io_tree = true; + bool ret = false; + + while (cur_offset <= end) { + u64 delalloc_start; + u64 delalloc_end; + bool delalloc; + + delalloc = find_delalloc_subrange(inode, cur_offset, end, + cached_state, &search_io_tree, + &delalloc_start, + &delalloc_end); + if (!delalloc) + break; + + if (prev_delalloc_end == 0) { + /* First subrange found. */ + *delalloc_start_ret = max(delalloc_start, start); + *delalloc_end_ret = delalloc_end; + ret = true; + } else if (delalloc_start == prev_delalloc_end + 1) { + /* Subrange adjacent to the previous one, merge them. */ + *delalloc_end_ret = delalloc_end; + } else { + /* Subrange not adjacent to the previous one, exit. */ + break; + } + + prev_delalloc_end = delalloc_end; + cur_offset = delalloc_end + 1; + cond_resched(); + } + + return ret; +} + +/* + * Check if there's a hole or delalloc range in a range representing a hole (or + * prealloc extent) found in the inode's subvolume btree. + * + * @inode: The inode. + * @whence: Seek mode (SEEK_DATA or SEEK_HOLE). + * @start: Start offset of the hole region. It does not need to be sector + * size aligned. + * @end: End offset (inclusive value) of the hole region. It does not + * need to be sector size aligned. + * @start_ret: Return parameter, used to set the start of the subrange in the + * hole that matches the search criteria (seek mode), if such + * subrange is found (return value of the function is true). + * The value returned here may not be sector size aligned. + * + * Returns true if a subrange matching the given seek mode is found, and if one + * is found, it updates @start_ret with the start of the subrange. + */ +static bool find_desired_extent_in_hole(struct btrfs_inode *inode, int whence, + struct extent_state **cached_state, + u64 start, u64 end, u64 *start_ret) +{ + u64 delalloc_start; + u64 delalloc_end; + bool delalloc; + + delalloc = btrfs_find_delalloc_in_range(inode, start, end, cached_state, + &delalloc_start, &delalloc_end); + if (delalloc && whence == SEEK_DATA) { + *start_ret = delalloc_start; + return true; + } + + if (delalloc && whence == SEEK_HOLE) { + /* + * We found delalloc but it starts after out start offset. So we + * have a hole between our start offset and the delalloc start. + */ + if (start < delalloc_start) { + *start_ret = start; + return true; + } + /* + * Delalloc range starts at our start offset. + * If the delalloc range's length is smaller than our range, + * then it means we have a hole that starts where the delalloc + * subrange ends. + */ + if (delalloc_end < end) { + *start_ret = delalloc_end + 1; + return true; + } + + /* There's delalloc for the whole range. */ + return false; + } + + if (!delalloc && whence == SEEK_HOLE) { + *start_ret = start; + return true; + } + + /* + * No delalloc in the range and we are seeking for data. The caller has + * to iterate to the next extent item in the subvolume btree. + */ + return false; +} + +static loff_t find_desired_extent(struct file *file, loff_t offset, int whence) +{ + struct btrfs_inode *inode = BTRFS_I(file->f_mapping->host); + struct btrfs_file_private *private; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct extent_state *cached_state = NULL; + struct extent_state **delalloc_cached_state; + const loff_t i_size = i_size_read(&inode->vfs_inode); + const u64 ino = btrfs_ino(inode); + struct btrfs_root *root = inode->root; + struct btrfs_path *path; + struct btrfs_key key; + u64 last_extent_end; + u64 lockstart; + u64 lockend; + u64 start; + int ret; + bool found = false; + + if (i_size == 0 || offset >= i_size) + return -ENXIO; + + /* + * Quick path. If the inode has no prealloc extents and its number of + * bytes used matches its i_size, then it can not have holes. + */ + if (whence == SEEK_HOLE && + !(inode->flags & BTRFS_INODE_PREALLOC) && + inode_get_bytes(&inode->vfs_inode) == i_size) + return i_size; + + spin_lock(&inode->lock); + private = file->private_data; + spin_unlock(&inode->lock); + + if (private && private->owner_task != current) { + /* + * Not allocated by us, don't use it as its cached state is used + * by the task that allocated it and we don't want neither to + * mess with it nor get incorrect results because it reflects an + * invalid state for the current task. + */ + private = NULL; + } else if (!private) { + private = kzalloc_obj(*private); + /* + * No worries if memory allocation failed. + * The private structure is used only for speeding up multiple + * lseek SEEK_HOLE/DATA calls to a file when there's delalloc, + * so everything will still be correct. + */ + if (private) { + bool free = false; + + private->owner_task = current; + + spin_lock(&inode->lock); + if (file->private_data) + free = true; + else + file->private_data = private; + spin_unlock(&inode->lock); + + if (free) { + kfree(private); + private = NULL; + } + } + } + + if (private) + delalloc_cached_state = &private->llseek_cached_state; + else + delalloc_cached_state = NULL; + + /* + * offset can be negative, in this case we start finding DATA/HOLE from + * the very start of the file. + */ + start = max_t(loff_t, 0, offset); + + lockstart = round_down(start, fs_info->sectorsize); + lockend = round_up(i_size, fs_info->sectorsize); + if (lockend <= lockstart) + lockend = lockstart + fs_info->sectorsize; + lockend--; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + path->reada = READA_FORWARD; + + key.objectid = ino; + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = start; + + last_extent_end = lockstart; + + btrfs_lock_extent(&inode->io_tree, lockstart, lockend, &cached_state); + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) { + goto out; + } else if (ret > 0 && path->slots[0] > 0) { + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0] - 1); + if (key.objectid == ino && key.type == BTRFS_EXTENT_DATA_KEY) + path->slots[0]--; + } + + while (start < i_size) { + struct extent_buffer *leaf = path->nodes[0]; + struct btrfs_file_extent_item *extent; + u64 extent_end; + u8 type; + + if (path->slots[0] >= btrfs_header_nritems(leaf)) { + ret = btrfs_next_leaf(root, path); + if (ret < 0) + goto out; + else if (ret > 0) + break; + + leaf = path->nodes[0]; + } + + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY) + break; + + extent_end = btrfs_file_extent_end(path); + + /* + * In the first iteration we may have a slot that points to an + * extent that ends before our start offset, so skip it. + */ + if (extent_end <= start) { + path->slots[0]++; + continue; + } + + /* We have an implicit hole, NO_HOLES feature is likely set. */ + if (last_extent_end < key.offset) { + u64 search_start = last_extent_end; + u64 found_start; + + /* + * First iteration, @start matches @offset and it's + * within the hole. + */ + if (start == offset) + search_start = offset; + + found = find_desired_extent_in_hole(inode, whence, + delalloc_cached_state, + search_start, + key.offset - 1, + &found_start); + if (found) { + start = found_start; + break; + } + /* + * Didn't find data or a hole (due to delalloc) in the + * implicit hole range, so need to analyze the extent. + */ + } + + extent = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_file_extent_item); + type = btrfs_file_extent_type(leaf, extent); + + /* + * Can't access the extent's disk_bytenr field if this is an + * inline extent, since at that offset, it's where the extent + * data starts. + */ + if (type == BTRFS_FILE_EXTENT_PREALLOC || + (type == BTRFS_FILE_EXTENT_REG && + btrfs_file_extent_disk_bytenr(leaf, extent) == 0)) { + /* + * Explicit hole or prealloc extent, search for delalloc. + * A prealloc extent is treated like a hole. + */ + u64 search_start = key.offset; + u64 found_start; + + /* + * First iteration, @start matches @offset and it's + * within the hole. + */ + if (start == offset) + search_start = offset; + + found = find_desired_extent_in_hole(inode, whence, + delalloc_cached_state, + search_start, + extent_end - 1, + &found_start); + if (found) { + start = found_start; + break; + } + /* + * Didn't find data or a hole (due to delalloc) in the + * implicit hole range, so need to analyze the next + * extent item. + */ + } else { + /* + * Found a regular or inline extent. + * If we are seeking for data, adjust the start offset + * and stop, we're done. + */ + if (whence == SEEK_DATA) { + start = max_t(u64, key.offset, offset); + found = true; + break; + } + /* + * Else, we are seeking for a hole, check the next file + * extent item. + */ + } + + start = extent_end; + last_extent_end = extent_end; + path->slots[0]++; + if (fatal_signal_pending(current)) { + ret = -EINTR; + goto out; + } + cond_resched(); + } + + /* We have an implicit hole from the last extent found up to i_size. */ + if (!found && start < i_size) { + found = find_desired_extent_in_hole(inode, whence, + delalloc_cached_state, start, + i_size - 1, &start); + if (!found) + start = i_size; + } + +out: + btrfs_unlock_extent(&inode->io_tree, lockstart, lockend, &cached_state); + btrfs_free_path(path); + + if (ret < 0) + return ret; + + if (whence == SEEK_DATA && start >= i_size) + return -ENXIO; + + return min_t(loff_t, start, i_size); +} + +static loff_t btrfs_file_llseek(struct file *file, loff_t offset, int whence) +{ + struct inode *inode = file->f_mapping->host; + + switch (whence) { + default: + return generic_file_llseek(file, offset, whence); + case SEEK_DATA: + case SEEK_HOLE: + btrfs_inode_lock(BTRFS_I(inode), BTRFS_ILOCK_SHARED); + offset = find_desired_extent(file, offset, whence); + btrfs_inode_unlock(BTRFS_I(inode), BTRFS_ILOCK_SHARED); + break; + } + + if (offset < 0) + return offset; + + return vfs_setpos(file, offset, inode->i_sb->s_maxbytes); +} + +static int btrfs_file_open(struct inode *inode, struct file *filp) +{ + int ret; + + if (btrfs_is_shutdown(inode_to_fs_info(inode))) + return -EIO; + + filp->f_mode |= FMODE_NOWAIT | FMODE_CAN_ODIRECT; + + ret = fsverity_file_open(inode, filp); + if (ret) + return ret; + return generic_file_open(inode, filp); +} + +static ssize_t btrfs_file_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + ssize_t ret = 0; + + if (btrfs_is_shutdown(inode_to_fs_info(file_inode(iocb->ki_filp)))) + return -EIO; + + if (iocb->ki_flags & IOCB_DIRECT) { + ret = btrfs_direct_read(iocb, to); + if (ret < 0 || !iov_iter_count(to) || + iocb->ki_pos >= i_size_read(file_inode(iocb->ki_filp))) + return ret; + } + + return filemap_read(iocb, to, ret); +} + +static ssize_t btrfs_file_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, + size_t len, unsigned int flags) +{ + if (btrfs_is_shutdown(inode_to_fs_info(file_inode(in)))) + return -EIO; + + return filemap_splice_read(in, ppos, pipe, len, flags); +} + +const struct file_operations btrfs_file_operations = { + .llseek = btrfs_file_llseek, + .read_iter = btrfs_file_read_iter, + .splice_read = btrfs_file_splice_read, + .write_iter = btrfs_file_write_iter, + .splice_write = iter_file_splice_write, + .mmap_prepare = btrfs_file_mmap_prepare, + .open = btrfs_file_open, + .release = btrfs_release_file, + .get_unmapped_area = thp_get_unmapped_area, + .fsync = btrfs_sync_file, + .fallocate = btrfs_fallocate, + .unlocked_ioctl = btrfs_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = btrfs_compat_ioctl, +#endif + .remap_file_range = btrfs_remap_file_range, + .uring_cmd = btrfs_uring_cmd, + .fop_flags = FOP_BUFFER_RASYNC | FOP_BUFFER_WASYNC, + .setlease = generic_setlease, +}; + +int btrfs_fdatawrite_range(struct btrfs_inode *inode, loff_t start, loff_t end) +{ + struct address_space *mapping = inode->vfs_inode.i_mapping; + int ret; + + /* + * So with compression we will find and lock a dirty page and clear the + * first one as dirty, setup an async extent, and immediately return + * with the entire range locked but with nobody actually marked with + * writeback. So we can't just filemap_write_and_wait_range() and + * expect it to work since it will just kick off a thread to do the + * actual work. So we need to call filemap_fdatawrite_range _again_ + * since it will wait on the page lock, which won't be unlocked until + * after the pages have been marked as writeback and so we're good to go + * from there. We have to do this otherwise we'll miss the ordered + * extents and that results in badness. Please Josef, do not think you + * know better and pull this out at some point in the future, it is + * right and you are wrong. + */ + ret = filemap_fdatawrite_range(mapping, start, end); + if (!ret && test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, &inode->runtime_flags)) + ret = filemap_fdatawrite_range(mapping, start, end); + + return ret; +} diff --git a/fs/btrfs/file.h b/fs/btrfs/file.h new file mode 100644 index 000000000..d7df81388 --- /dev/null +++ b/fs/btrfs/file.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_FILE_H +#define BTRFS_FILE_H + +#include + +struct file; +struct extent_state; +struct kiocb; +struct iov_iter; +struct inode; +struct folio; +struct page; +struct btrfs_ioctl_encoded_io_args; +struct btrfs_drop_extents_args; +struct btrfs_inode; +struct btrfs_root; +struct btrfs_path; +struct btrfs_replace_extent_info; +struct btrfs_trans_handle; + +extern const struct file_operations btrfs_file_operations; + +int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync); +int btrfs_drop_extents(struct btrfs_trans_handle *trans, + struct btrfs_root *root, struct btrfs_inode *inode, + struct btrfs_drop_extents_args *args); +int btrfs_replace_file_extents(struct btrfs_inode *inode, + struct btrfs_path *path, const u64 start, + const u64 end, + struct btrfs_replace_extent_info *extent_info, + struct btrfs_trans_handle **trans_out); +int btrfs_mark_extent_written(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, u64 start, u64 end); +ssize_t btrfs_do_write_iter(struct kiocb *iocb, struct iov_iter *from, + const struct btrfs_ioctl_encoded_io_args *encoded); +int btrfs_release_file(struct inode *inode, struct file *file); +int btrfs_dirty_folio(struct btrfs_inode *inode, struct folio *folio, loff_t pos, + size_t write_bytes, struct extent_state **cached, bool noreserve); +int btrfs_fdatawrite_range(struct btrfs_inode *inode, loff_t start, loff_t end); +int btrfs_check_nocow_lock(struct btrfs_inode *inode, loff_t pos, + size_t *write_bytes, bool nowait); +void btrfs_check_nocow_unlock(struct btrfs_inode *inode); +bool btrfs_find_delalloc_in_range(struct btrfs_inode *inode, u64 start, u64 end, + struct extent_state **cached_state, + u64 *delalloc_start_ret, u64 *delalloc_end_ret); +int btrfs_write_check(struct kiocb *iocb, size_t count); +ssize_t btrfs_buffered_write(struct kiocb *iocb, struct iov_iter *i); + +#endif diff --git a/fs/btrfs/free-space-cache.c b/fs/btrfs/free-space-cache.c new file mode 100644 index 000000000..e2af75a20 --- /dev/null +++ b/fs/btrfs/free-space-cache.c @@ -0,0 +1,4390 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2008 Red Hat. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "extent-tree.h" +#include "fs.h" +#include "messages.h" +#include "misc.h" +#include "free-space-cache.h" +#include "transaction.h" +#include "disk-io.h" +#include "extent_io.h" +#include "space-info.h" +#include "block-group.h" +#include "discard.h" +#include "subpage.h" +#include "inode-item.h" +#include "accessors.h" +#include "file-item.h" +#include "file.h" +#include "super.h" +#include "relocation.h" + +#define BITS_PER_BITMAP (PAGE_SIZE * 8UL) +#define MAX_CACHE_BYTES_PER_GIG SZ_64K +#define FORCE_EXTENT_THRESHOLD SZ_1M + +static struct kmem_cache *btrfs_free_space_cachep; +static struct kmem_cache *btrfs_free_space_bitmap_cachep; + +struct btrfs_trim_range { + u64 start; + u64 bytes; + struct list_head list; +}; + +static int link_free_space(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info); +static void unlink_free_space(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info, bool update_stat); +static int search_bitmap(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *bitmap_info, u64 *offset, + u64 *bytes, bool for_alloc); +static void free_bitmap(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *bitmap_info); +static void bitmap_clear_bits(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info, u64 offset, + u64 bytes, bool update_stats); + +static void btrfs_crc32c_final(u32 crc, u8 *result) +{ + put_unaligned_le32(~crc, result); +} + +static void __btrfs_remove_free_space_cache(struct btrfs_free_space_ctl *ctl) +{ + struct btrfs_free_space *info; + struct rb_node *node; + + while ((node = rb_last(&ctl->free_space_offset)) != NULL) { + info = rb_entry(node, struct btrfs_free_space, offset_index); + if (!info->bitmap) { + unlink_free_space(ctl, info, true); + kmem_cache_free(btrfs_free_space_cachep, info); + } else { + free_bitmap(ctl, info); + } + + cond_resched_lock(&ctl->tree_lock); + } +} + +static struct inode *__lookup_free_space_inode(struct btrfs_root *root, + struct btrfs_path *path, + u64 offset) +{ + struct btrfs_key key; + struct btrfs_key location; + struct btrfs_disk_key disk_key; + struct btrfs_free_space_header *header; + struct extent_buffer *leaf; + struct btrfs_inode *inode; + unsigned nofs_flag; + int ret; + + key.objectid = BTRFS_FREE_SPACE_OBJECTID; + key.type = 0; + key.offset = offset; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + return ERR_PTR(ret); + if (ret > 0) { + btrfs_release_path(path); + return ERR_PTR(-ENOENT); + } + + leaf = path->nodes[0]; + header = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_free_space_header); + btrfs_free_space_key(leaf, header, &disk_key); + btrfs_disk_key_to_cpu(&location, &disk_key); + btrfs_release_path(path); + + /* + * We are often under a trans handle at this point, so we need to make + * sure NOFS is set to keep us from deadlocking. + */ + nofs_flag = memalloc_nofs_save(); + inode = btrfs_iget_path(location.objectid, root, path); + btrfs_release_path(path); + memalloc_nofs_restore(nofs_flag); + if (IS_ERR(inode)) + return ERR_CAST(inode); + + mapping_set_gfp_mask(inode->vfs_inode.i_mapping, + mapping_gfp_constraint(inode->vfs_inode.i_mapping, + ~(__GFP_FS | __GFP_HIGHMEM))); + + return &inode->vfs_inode; +} + +struct inode *lookup_free_space_inode(struct btrfs_block_group *block_group, + struct btrfs_path *path) +{ + struct btrfs_fs_info *fs_info = block_group->fs_info; + struct inode *inode = NULL; + u32 flags = BTRFS_INODE_NODATASUM | BTRFS_INODE_NODATACOW; + + spin_lock(&block_group->lock); + if (block_group->inode) + inode = igrab(&block_group->inode->vfs_inode); + spin_unlock(&block_group->lock); + if (inode) + return inode; + + inode = __lookup_free_space_inode(fs_info->tree_root, path, + block_group->start); + if (IS_ERR(inode)) + return inode; + + spin_lock(&block_group->lock); + if (!((BTRFS_I(inode)->flags & flags) == flags)) { + btrfs_info(fs_info, "Old style space inode found, converting."); + BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM | + BTRFS_INODE_NODATACOW; + block_group->disk_cache_state = BTRFS_DC_CLEAR; + } + + if (!test_and_set_bit(BLOCK_GROUP_FLAG_IREF, &block_group->runtime_flags)) + block_group->inode = BTRFS_I(igrab(inode)); + spin_unlock(&block_group->lock); + + return inode; +} + +static int __create_free_space_inode(struct btrfs_root *root, + struct btrfs_trans_handle *trans, + struct btrfs_path *path, + u64 ino, u64 offset) +{ + struct btrfs_key key; + struct btrfs_disk_key disk_key; + struct btrfs_free_space_header *header; + struct btrfs_inode_item *inode_item; + struct extent_buffer *leaf; + /* We inline CRCs for the free disk space cache */ + const u64 flags = BTRFS_INODE_NOCOMPRESS | BTRFS_INODE_PREALLOC | + BTRFS_INODE_NODATASUM | BTRFS_INODE_NODATACOW; + int ret; + + ret = btrfs_insert_empty_inode(trans, root, path, ino); + if (ret) + return ret; + + leaf = path->nodes[0]; + inode_item = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_inode_item); + btrfs_item_key(leaf, &disk_key, path->slots[0]); + memzero_extent_buffer(leaf, (unsigned long)inode_item, + sizeof(*inode_item)); + btrfs_set_inode_generation(leaf, inode_item, trans->transid); + btrfs_set_inode_size(leaf, inode_item, 0); + btrfs_set_inode_nbytes(leaf, inode_item, 0); + btrfs_set_inode_uid(leaf, inode_item, 0); + btrfs_set_inode_gid(leaf, inode_item, 0); + btrfs_set_inode_mode(leaf, inode_item, S_IFREG | 0600); + btrfs_set_inode_flags(leaf, inode_item, flags); + btrfs_set_inode_nlink(leaf, inode_item, 1); + btrfs_set_inode_transid(leaf, inode_item, trans->transid); + btrfs_set_inode_block_group(leaf, inode_item, offset); + btrfs_release_path(path); + + key.objectid = BTRFS_FREE_SPACE_OBJECTID; + key.type = 0; + key.offset = offset; + ret = btrfs_insert_empty_item(trans, root, path, &key, + sizeof(struct btrfs_free_space_header)); + if (ret < 0) { + btrfs_release_path(path); + return ret; + } + + leaf = path->nodes[0]; + header = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_free_space_header); + memzero_extent_buffer(leaf, (unsigned long)header, sizeof(*header)); + btrfs_set_free_space_key(leaf, header, &disk_key); + btrfs_release_path(path); + + return 0; +} + +int create_free_space_inode(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct btrfs_path *path) +{ + int ret; + u64 ino; + + ret = btrfs_get_free_objectid(trans->fs_info->tree_root, &ino); + if (ret < 0) + return ret; + + return __create_free_space_inode(trans->fs_info->tree_root, trans, path, + ino, block_group->start); +} + +/* + * inode is an optional sink: if it is NULL, btrfs_remove_free_space_inode + * handles lookup, otherwise it takes ownership and iputs the inode. + * Don't reuse an inode pointer after passing it into this function. + */ +int btrfs_remove_free_space_inode(struct btrfs_trans_handle *trans, + struct inode *inode, + struct btrfs_block_group *block_group) +{ + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + int ret = 0; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + if (!inode) + inode = lookup_free_space_inode(block_group, path); + if (IS_ERR(inode)) { + if (PTR_ERR(inode) != -ENOENT) + ret = PTR_ERR(inode); + return ret; + } + ret = btrfs_orphan_add(trans, BTRFS_I(inode)); + if (ret) { + btrfs_add_delayed_iput(BTRFS_I(inode)); + return ret; + } + clear_nlink(inode); + /* One for the block groups ref */ + spin_lock(&block_group->lock); + if (test_and_clear_bit(BLOCK_GROUP_FLAG_IREF, &block_group->runtime_flags)) { + block_group->inode = NULL; + spin_unlock(&block_group->lock); + iput(inode); + } else { + spin_unlock(&block_group->lock); + } + /* One for the lookup ref */ + btrfs_add_delayed_iput(BTRFS_I(inode)); + + key.objectid = BTRFS_FREE_SPACE_OBJECTID; + key.type = 0; + key.offset = block_group->start; + ret = btrfs_search_slot(trans, trans->fs_info->tree_root, &key, path, + -1, 1); + if (ret) { + if (ret > 0) + ret = 0; + return ret; + } + return btrfs_del_item(trans, trans->fs_info->tree_root, path); +} + +int btrfs_truncate_free_space_cache(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct inode *vfs_inode) +{ + struct btrfs_truncate_control control = { + .inode = BTRFS_I(vfs_inode), + .new_size = 0, + .ino = btrfs_ino(BTRFS_I(vfs_inode)), + .min_type = BTRFS_EXTENT_DATA_KEY, + .clear_extent_range = true, + }; + struct btrfs_inode *inode = BTRFS_I(vfs_inode); + struct btrfs_root *root = inode->root; + struct extent_state *cached_state = NULL; + int ret = 0; + bool locked = false; + + if (block_group) { + BTRFS_PATH_AUTO_FREE(path); + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto fail; + } + locked = true; + mutex_lock(&trans->transaction->cache_write_mutex); + if (!list_empty(&block_group->io_list)) { + list_del_init(&block_group->io_list); + + btrfs_wait_cache_io(trans, block_group, path); + btrfs_put_block_group(block_group); + } + + /* + * now that we've truncated the cache away, its no longer + * setup or written + */ + spin_lock(&block_group->lock); + block_group->disk_cache_state = BTRFS_DC_CLEAR; + spin_unlock(&block_group->lock); + } + + btrfs_i_size_write(inode, 0); + truncate_pagecache(vfs_inode, 0); + + btrfs_lock_extent(&inode->io_tree, 0, (u64)-1, &cached_state); + btrfs_drop_extent_map_range(inode, 0, (u64)-1, false); + + /* + * We skip the throttling logic for free space cache inodes, so we don't + * need to check for -EAGAIN. + */ + ret = btrfs_truncate_inode_items(trans, root, &control); + + inode_sub_bytes(&inode->vfs_inode, control.sub_bytes); + btrfs_inode_safe_disk_i_size_write(inode, control.last_size); + + btrfs_unlock_extent(&inode->io_tree, 0, (u64)-1, &cached_state); + if (ret) + goto fail; + + ret = btrfs_update_inode(trans, inode); + +fail: + if (locked) + mutex_unlock(&trans->transaction->cache_write_mutex); + if (ret) + btrfs_abort_transaction(trans, ret); + + return ret; +} + +static void readahead_cache(struct inode *inode) +{ + struct file_ra_state ra; + pgoff_t last_index; + + file_ra_state_init(&ra, inode->i_mapping); + last_index = (i_size_read(inode) - 1) >> PAGE_SHIFT; + + page_cache_sync_readahead(inode->i_mapping, &ra, NULL, 0, last_index); +} + +static int io_ctl_init(struct btrfs_io_ctl *io_ctl, struct inode *inode, + int write) +{ + int num_pages; + + num_pages = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); + + /* Make sure we can fit our crcs and generation into the first page */ + if (write && (num_pages * sizeof(u32) + sizeof(u64)) > PAGE_SIZE) + return -ENOSPC; + + memset(io_ctl, 0, sizeof(struct btrfs_io_ctl)); + + io_ctl->pages = kzalloc_objs(struct page *, num_pages, GFP_NOFS); + if (!io_ctl->pages) + return -ENOMEM; + + io_ctl->num_pages = num_pages; + io_ctl->fs_info = inode_to_fs_info(inode); + io_ctl->inode = inode; + + return 0; +} +ALLOW_ERROR_INJECTION(io_ctl_init, ERRNO); + +static void io_ctl_free(struct btrfs_io_ctl *io_ctl) +{ + kfree(io_ctl->pages); + io_ctl->pages = NULL; +} + +static void io_ctl_unmap_page(struct btrfs_io_ctl *io_ctl) +{ + if (io_ctl->cur) { + io_ctl->cur = NULL; + io_ctl->orig = NULL; + } +} + +static void io_ctl_map_page(struct btrfs_io_ctl *io_ctl, int clear) +{ + ASSERT(io_ctl->index < io_ctl->num_pages); + io_ctl->page = io_ctl->pages[io_ctl->index++]; + io_ctl->cur = page_address(io_ctl->page); + io_ctl->orig = io_ctl->cur; + io_ctl->size = PAGE_SIZE; + if (clear) + clear_page(io_ctl->cur); +} + +static void io_ctl_drop_pages(struct btrfs_io_ctl *io_ctl) +{ + int i; + + io_ctl_unmap_page(io_ctl); + + for (i = 0; i < io_ctl->num_pages; i++) { + if (io_ctl->pages[i]) { + unlock_page(io_ctl->pages[i]); + put_page(io_ctl->pages[i]); + } + } +} + +static int io_ctl_prepare_pages(struct btrfs_io_ctl *io_ctl, bool uptodate) +{ + struct folio *folio; + struct inode *inode = io_ctl->inode; + gfp_t mask = btrfs_alloc_write_mask(inode->i_mapping); + int i; + + for (i = 0; i < io_ctl->num_pages; i++) { + int ret; + + folio = __filemap_get_folio(inode->i_mapping, i, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, + mask); + if (IS_ERR(folio)) { + io_ctl_drop_pages(io_ctl); + return PTR_ERR(folio); + } + + ret = set_folio_extent_mapped(folio); + if (ret < 0) { + folio_unlock(folio); + folio_put(folio); + io_ctl_drop_pages(io_ctl); + return ret; + } + + io_ctl->pages[i] = &folio->page; + if (uptodate && !folio_test_uptodate(folio)) { + btrfs_read_folio(NULL, folio); + folio_lock(folio); + if (folio->mapping != inode->i_mapping) { + btrfs_err(BTRFS_I(inode)->root->fs_info, + "free space cache page truncated"); + io_ctl_drop_pages(io_ctl); + return -EIO; + } + if (!folio_test_uptodate(folio)) { + btrfs_err(BTRFS_I(inode)->root->fs_info, + "error reading free space cache"); + io_ctl_drop_pages(io_ctl); + return -EIO; + } + } + } + + for (i = 0; i < io_ctl->num_pages; i++) + clear_page_dirty_for_io(io_ctl->pages[i]); + + return 0; +} + +static void io_ctl_set_generation(struct btrfs_io_ctl *io_ctl, u64 generation) +{ + io_ctl_map_page(io_ctl, 1); + + /* + * Skip the csum areas. If we don't check crcs then we just have a + * 64bit chunk at the front of the first page. + */ + io_ctl->cur += (sizeof(u32) * io_ctl->num_pages); + io_ctl->size -= sizeof(u64) + (sizeof(u32) * io_ctl->num_pages); + + put_unaligned_le64(generation, io_ctl->cur); + io_ctl->cur += sizeof(u64); +} + +static int io_ctl_check_generation(struct btrfs_io_ctl *io_ctl, u64 generation) +{ + u64 cache_gen; + + /* + * Skip the crc area. If we don't check crcs then we just have a 64bit + * chunk at the front of the first page. + */ + io_ctl->cur += sizeof(u32) * io_ctl->num_pages; + io_ctl->size -= sizeof(u64) + (sizeof(u32) * io_ctl->num_pages); + + cache_gen = get_unaligned_le64(io_ctl->cur); + if (cache_gen != generation) { + btrfs_err_rl(io_ctl->fs_info, + "space cache generation (%llu) does not match inode (%llu)", + cache_gen, generation); + io_ctl_unmap_page(io_ctl); + return -EIO; + } + io_ctl->cur += sizeof(u64); + return 0; +} + +static void io_ctl_set_crc(struct btrfs_io_ctl *io_ctl, int index) +{ + u32 *tmp; + u32 crc = ~(u32)0; + unsigned offset = 0; + + if (index == 0) + offset = sizeof(u32) * io_ctl->num_pages; + + crc = crc32c(crc, io_ctl->orig + offset, PAGE_SIZE - offset); + btrfs_crc32c_final(crc, (u8 *)&crc); + io_ctl_unmap_page(io_ctl); + tmp = page_address(io_ctl->pages[0]); + tmp += index; + *tmp = crc; +} + +static int io_ctl_check_crc(struct btrfs_io_ctl *io_ctl, int index) +{ + u32 *tmp, val; + u32 crc = ~(u32)0; + unsigned offset = 0; + + if (index >= io_ctl->num_pages) + return -EIO; + + if (index == 0) + offset = sizeof(u32) * io_ctl->num_pages; + + tmp = page_address(io_ctl->pages[0]); + tmp += index; + val = *tmp; + + io_ctl_map_page(io_ctl, 0); + crc = crc32c(crc, io_ctl->orig + offset, PAGE_SIZE - offset); + btrfs_crc32c_final(crc, (u8 *)&crc); + if (val != crc) { + btrfs_err_rl(io_ctl->fs_info, + "csum mismatch on free space cache"); + io_ctl_unmap_page(io_ctl); + return -EIO; + } + + return 0; +} + +static int io_ctl_add_entry(struct btrfs_io_ctl *io_ctl, u64 offset, u64 bytes, + void *bitmap) +{ + struct btrfs_free_space_entry *entry; + + if (!io_ctl->cur) + return -ENOSPC; + + entry = io_ctl->cur; + put_unaligned_le64(offset, &entry->offset); + put_unaligned_le64(bytes, &entry->bytes); + entry->type = (bitmap) ? BTRFS_FREE_SPACE_BITMAP : + BTRFS_FREE_SPACE_EXTENT; + io_ctl->cur += sizeof(struct btrfs_free_space_entry); + io_ctl->size -= sizeof(struct btrfs_free_space_entry); + + if (io_ctl->size >= sizeof(struct btrfs_free_space_entry)) + return 0; + + io_ctl_set_crc(io_ctl, io_ctl->index - 1); + + /* No more pages to map */ + if (io_ctl->index >= io_ctl->num_pages) + return 0; + + /* map the next page */ + io_ctl_map_page(io_ctl, 1); + return 0; +} + +static int io_ctl_add_bitmap(struct btrfs_io_ctl *io_ctl, void *bitmap) +{ + if (!io_ctl->cur) + return -ENOSPC; + + /* + * If we aren't at the start of the current page, unmap this one and + * map the next one if there is any left. + */ + if (io_ctl->cur != io_ctl->orig) { + io_ctl_set_crc(io_ctl, io_ctl->index - 1); + if (io_ctl->index >= io_ctl->num_pages) + return -ENOSPC; + io_ctl_map_page(io_ctl, 0); + } + + copy_page(io_ctl->cur, bitmap); + io_ctl_set_crc(io_ctl, io_ctl->index - 1); + if (io_ctl->index < io_ctl->num_pages) + io_ctl_map_page(io_ctl, 0); + return 0; +} + +static void io_ctl_zero_remaining_pages(struct btrfs_io_ctl *io_ctl) +{ + /* + * If we're not on the boundary we know we've modified the page and we + * need to crc the page. + */ + if (io_ctl->cur != io_ctl->orig) + io_ctl_set_crc(io_ctl, io_ctl->index - 1); + else + io_ctl_unmap_page(io_ctl); + + while (io_ctl->index < io_ctl->num_pages) { + io_ctl_map_page(io_ctl, 1); + io_ctl_set_crc(io_ctl, io_ctl->index - 1); + } +} + +static int io_ctl_read_entry(struct btrfs_io_ctl *io_ctl, + struct btrfs_free_space *entry, u8 *type) +{ + struct btrfs_free_space_entry *e; + int ret; + + if (!io_ctl->cur) { + ret = io_ctl_check_crc(io_ctl, io_ctl->index); + if (ret) + return ret; + } + + e = io_ctl->cur; + entry->offset = get_unaligned_le64(&e->offset); + entry->bytes = get_unaligned_le64(&e->bytes); + *type = e->type; + io_ctl->cur += sizeof(struct btrfs_free_space_entry); + io_ctl->size -= sizeof(struct btrfs_free_space_entry); + + if (io_ctl->size >= sizeof(struct btrfs_free_space_entry)) + return 0; + + io_ctl_unmap_page(io_ctl); + + return 0; +} + +static int io_ctl_read_bitmap(struct btrfs_io_ctl *io_ctl, + struct btrfs_free_space *entry) +{ + int ret; + + ret = io_ctl_check_crc(io_ctl, io_ctl->index); + if (ret) + return ret; + + copy_page(entry->bitmap, io_ctl->cur); + io_ctl_unmap_page(io_ctl); + + return 0; +} + +static void recalculate_thresholds(struct btrfs_free_space_ctl *ctl) +{ + struct btrfs_block_group *block_group = ctl->block_group; + const int unit = block_group->fs_info->sectorsize; + u64 max_bytes; + u64 bitmap_bytes; + u64 extent_bytes; + u64 size = block_group->length; + u64 bytes_per_bg = BITS_PER_BITMAP * unit; + u64 max_bitmaps = div64_u64(size + bytes_per_bg - 1, bytes_per_bg); + + max_bitmaps = max_t(u64, max_bitmaps, 1); + + if (ctl->total_bitmaps > max_bitmaps) + btrfs_err(block_group->fs_info, +"invalid free space control: bg start=%llu len=%llu total_bitmaps=%u unit=%u max_bitmaps=%llu bytes_per_bg=%llu", + block_group->start, block_group->length, + ctl->total_bitmaps, unit, max_bitmaps, + bytes_per_bg); + ASSERT(ctl->total_bitmaps <= max_bitmaps); + + /* + * We are trying to keep the total amount of memory used per 1GiB of + * space to be MAX_CACHE_BYTES_PER_GIG. However, with a reclamation + * mechanism of pulling extents >= FORCE_EXTENT_THRESHOLD out of + * bitmaps, we may end up using more memory than this. + */ + if (size < SZ_1G) + max_bytes = MAX_CACHE_BYTES_PER_GIG; + else + max_bytes = MAX_CACHE_BYTES_PER_GIG * div_u64(size, SZ_1G); + + bitmap_bytes = ctl->total_bitmaps * unit; + + /* + * we want the extent entry threshold to always be at most 1/2 the max + * bytes we can have, or whatever is less than that. + */ + extent_bytes = max_bytes - bitmap_bytes; + extent_bytes = min_t(u64, extent_bytes, max_bytes >> 1); + + ctl->extents_thresh = + div_u64(extent_bytes, sizeof(struct btrfs_free_space)); +} + +static int __load_free_space_cache(struct btrfs_root *root, struct inode *inode, + struct btrfs_free_space_ctl *ctl, + struct btrfs_path *path, u64 offset) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_free_space_header *header; + struct extent_buffer *leaf; + struct btrfs_io_ctl io_ctl; + struct btrfs_key key; + struct btrfs_free_space *e, *n; + LIST_HEAD(bitmaps); + u64 num_entries; + u64 num_bitmaps; + u64 generation; + u8 type; + int ret = 0; + + /* Nothing in the space cache, goodbye */ + if (!i_size_read(inode)) + return 0; + + key.objectid = BTRFS_FREE_SPACE_OBJECTID; + key.type = 0; + key.offset = offset; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + return 0; + else if (ret > 0) { + btrfs_release_path(path); + return 0; + } + + ret = -1; + + leaf = path->nodes[0]; + header = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_free_space_header); + num_entries = btrfs_free_space_entries(leaf, header); + num_bitmaps = btrfs_free_space_bitmaps(leaf, header); + generation = btrfs_free_space_generation(leaf, header); + btrfs_release_path(path); + + if (!BTRFS_I(inode)->generation) { + btrfs_info(fs_info, + "the free space cache file (%llu) is invalid, skip it", + offset); + return 0; + } + + if (BTRFS_I(inode)->generation != generation) { + btrfs_err(fs_info, + "free space inode generation (%llu) did not match free space cache generation (%llu)", + BTRFS_I(inode)->generation, generation); + return 0; + } + + if (!num_entries) + return 0; + + ret = io_ctl_init(&io_ctl, inode, 0); + if (ret) + return ret; + + readahead_cache(inode); + + ret = io_ctl_prepare_pages(&io_ctl, true); + if (ret) + goto out; + + ret = io_ctl_check_crc(&io_ctl, 0); + if (ret) + goto free_cache; + + ret = io_ctl_check_generation(&io_ctl, generation); + if (ret) + goto free_cache; + + while (num_entries) { + e = kmem_cache_zalloc(btrfs_free_space_cachep, + GFP_NOFS); + if (!e) { + ret = -ENOMEM; + goto free_cache; + } + + ret = io_ctl_read_entry(&io_ctl, e, &type); + if (ret) { + kmem_cache_free(btrfs_free_space_cachep, e); + goto free_cache; + } + + if (!e->bytes) { + ret = -1; + kmem_cache_free(btrfs_free_space_cachep, e); + goto free_cache; + } + + if (type == BTRFS_FREE_SPACE_EXTENT) { + spin_lock(&ctl->tree_lock); + ret = link_free_space(ctl, e); + spin_unlock(&ctl->tree_lock); + if (ret) { + btrfs_err(fs_info, + "Duplicate entries in free space cache, dumping"); + kmem_cache_free(btrfs_free_space_cachep, e); + goto free_cache; + } + } else { + ASSERT(num_bitmaps); + num_bitmaps--; + e->bitmap = kmem_cache_zalloc( + btrfs_free_space_bitmap_cachep, GFP_NOFS); + if (!e->bitmap) { + ret = -ENOMEM; + kmem_cache_free( + btrfs_free_space_cachep, e); + goto free_cache; + } + spin_lock(&ctl->tree_lock); + ret = link_free_space(ctl, e); + if (ret) { + spin_unlock(&ctl->tree_lock); + btrfs_err(fs_info, + "Duplicate entries in free space cache, dumping"); + kmem_cache_free(btrfs_free_space_bitmap_cachep, e->bitmap); + kmem_cache_free(btrfs_free_space_cachep, e); + goto free_cache; + } + ctl->total_bitmaps++; + recalculate_thresholds(ctl); + spin_unlock(&ctl->tree_lock); + list_add_tail(&e->list, &bitmaps); + } + + num_entries--; + } + + io_ctl_unmap_page(&io_ctl); + + /* + * We add the bitmaps at the end of the entries in order that + * the bitmap entries are added to the cache. + */ + list_for_each_entry_safe(e, n, &bitmaps, list) { + list_del_init(&e->list); + ret = io_ctl_read_bitmap(&io_ctl, e); + if (ret) + goto free_cache; + } + + io_ctl_drop_pages(&io_ctl); + ret = 1; +out: + io_ctl_free(&io_ctl); + return ret; +free_cache: + io_ctl_drop_pages(&io_ctl); + + spin_lock(&ctl->tree_lock); + __btrfs_remove_free_space_cache(ctl); + spin_unlock(&ctl->tree_lock); + goto out; +} + +static int copy_free_space_cache(struct btrfs_free_space_ctl *ctl) +{ + struct btrfs_free_space *info; + struct rb_node *n; + int ret = 0; + + while (!ret && (n = rb_first(&ctl->free_space_offset)) != NULL) { + info = rb_entry(n, struct btrfs_free_space, offset_index); + if (!info->bitmap) { + const u64 offset = info->offset; + const u64 bytes = info->bytes; + + unlink_free_space(ctl, info, true); + spin_unlock(&ctl->tree_lock); + kmem_cache_free(btrfs_free_space_cachep, info); + ret = btrfs_add_free_space(ctl->block_group, offset, bytes); + spin_lock(&ctl->tree_lock); + } else { + u64 offset = info->offset; + u64 bytes = ctl->block_group->fs_info->sectorsize; + + ret = search_bitmap(ctl, info, &offset, &bytes, false); + if (ret == 0) { + bitmap_clear_bits(ctl, info, offset, bytes, true); + spin_unlock(&ctl->tree_lock); + ret = btrfs_add_free_space(ctl->block_group, offset, + bytes); + spin_lock(&ctl->tree_lock); + } else { + free_bitmap(ctl, info); + ret = 0; + } + } + cond_resched_lock(&ctl->tree_lock); + } + return ret; +} + +static struct lock_class_key btrfs_free_space_inode_key; + +int load_free_space_cache(struct btrfs_block_group *block_group) +{ + struct btrfs_fs_info *fs_info = block_group->fs_info; + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + struct btrfs_free_space_ctl tmp_ctl = {}; + struct inode *inode; + struct btrfs_path *path; + int ret = 0; + bool matched; + u64 used = block_group->used; + + /* + * Because we could potentially discard our loaded free space, we want + * to load everything into a temporary structure first, and then if it's + * valid copy it all into the actual free space ctl. + */ + btrfs_init_free_space_ctl(block_group, &tmp_ctl); + + /* + * If this block group has been marked to be cleared for one reason or + * another then we can't trust the on disk cache, so just return. + */ + spin_lock(&block_group->lock); + if (block_group->disk_cache_state != BTRFS_DC_WRITTEN) { + spin_unlock(&block_group->lock); + return 0; + } + spin_unlock(&block_group->lock); + + path = btrfs_alloc_path(); + if (!path) + return 0; + path->search_commit_root = true; + path->skip_locking = true; + + /* + * We must pass a path with search_commit_root set to btrfs_iget in + * order to avoid a deadlock when allocating extents for the tree root. + * + * When we are COWing an extent buffer from the tree root, when looking + * for a free extent, at extent-tree.c:find_free_extent(), we can find + * block group without its free space cache loaded. When we find one + * we must load its space cache which requires reading its free space + * cache's inode item from the root tree. If this inode item is located + * in the same leaf that we started COWing before, then we end up in + * deadlock on the extent buffer (trying to read lock it when we + * previously write locked it). + * + * It's safe to read the inode item using the commit root because + * block groups, once loaded, stay in memory forever (until they are + * removed) as well as their space caches once loaded. New block groups + * once created get their ->cached field set to BTRFS_CACHE_FINISHED so + * we will never try to read their inode item while the fs is mounted. + */ + inode = lookup_free_space_inode(block_group, path); + if (IS_ERR(inode)) { + btrfs_free_path(path); + return 0; + } + + /* We may have converted the inode and made the cache invalid. */ + spin_lock(&block_group->lock); + if (block_group->disk_cache_state != BTRFS_DC_WRITTEN) { + spin_unlock(&block_group->lock); + btrfs_free_path(path); + goto out; + } + spin_unlock(&block_group->lock); + + /* + * Reinitialize the class of struct inode's mapping->invalidate_lock for + * free space inodes to prevent false positives related to locks for normal + * inodes. + */ + lockdep_set_class(&(&inode->i_data)->invalidate_lock, + &btrfs_free_space_inode_key); + + ret = __load_free_space_cache(fs_info->tree_root, inode, &tmp_ctl, + path, block_group->start); + btrfs_free_path(path); + if (ret <= 0) + goto out; + + matched = (tmp_ctl.free_space == (block_group->length - used - + block_group->bytes_super)); + + if (matched) { + spin_lock(&tmp_ctl.tree_lock); + ret = copy_free_space_cache(&tmp_ctl); + spin_unlock(&tmp_ctl.tree_lock); + /* + * ret == 1 means we successfully loaded the free space cache, + * so we need to re-set it here. + */ + if (ret == 0) + ret = 1; + } else { + /* + * We need to call the _locked variant so we don't try to update + * the discard counters. + */ + spin_lock(&tmp_ctl.tree_lock); + __btrfs_remove_free_space_cache(&tmp_ctl); + spin_unlock(&tmp_ctl.tree_lock); + btrfs_warn(fs_info, + "block group %llu has wrong amount of free space", + block_group->start); + ret = -1; + } +out: + if (ret < 0) { + /* This cache is bogus, make sure it gets cleared */ + spin_lock(&block_group->lock); + block_group->disk_cache_state = BTRFS_DC_CLEAR; + spin_unlock(&block_group->lock); + ret = 0; + + btrfs_warn(fs_info, + "failed to load free space cache for block group %llu, rebuilding it now", + block_group->start); + } + + spin_lock(&ctl->tree_lock); + btrfs_discard_update_discardable(block_group); + spin_unlock(&ctl->tree_lock); + iput(inode); + return ret; +} + +static noinline_for_stack +int write_cache_extent_entries(struct btrfs_io_ctl *io_ctl, + struct btrfs_block_group *block_group, + int *entries, int *bitmaps, + struct list_head *bitmap_list) +{ + int ret; + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + struct btrfs_free_cluster *cluster = NULL; + struct btrfs_free_cluster *cluster_locked = NULL; + struct rb_node *node = rb_first(&ctl->free_space_offset); + struct btrfs_trim_range *trim_entry; + + /* Get the cluster for this block_group if it exists */ + if (!list_empty(&block_group->cluster_list)) { + cluster = list_first_entry(&block_group->cluster_list, + struct btrfs_free_cluster, block_group_list); + } + + if (!node && cluster) { + cluster_locked = cluster; + spin_lock(&cluster_locked->lock); + node = rb_first(&cluster->root); + cluster = NULL; + } + + /* Write out the extent entries */ + while (node) { + struct btrfs_free_space *e; + + e = rb_entry(node, struct btrfs_free_space, offset_index); + *entries += 1; + + ret = io_ctl_add_entry(io_ctl, e->offset, e->bytes, + e->bitmap); + if (ret) + goto fail; + + if (e->bitmap) { + list_add_tail(&e->list, bitmap_list); + *bitmaps += 1; + } + node = rb_next(node); + if (!node && cluster) { + node = rb_first(&cluster->root); + cluster_locked = cluster; + spin_lock(&cluster_locked->lock); + cluster = NULL; + } + } + if (cluster_locked) { + spin_unlock(&cluster_locked->lock); + cluster_locked = NULL; + } + + /* + * Make sure we don't miss any range that was removed from our rbtree + * because trimming is running. Otherwise after a umount+mount (or crash + * after committing the transaction) we would leak free space and get + * an inconsistent free space cache report from fsck. + */ + list_for_each_entry(trim_entry, &ctl->trimming_ranges, list) { + ret = io_ctl_add_entry(io_ctl, trim_entry->start, + trim_entry->bytes, NULL); + if (ret) + goto fail; + *entries += 1; + } + + return 0; +fail: + if (cluster_locked) + spin_unlock(&cluster_locked->lock); + return -ENOSPC; +} + +static noinline_for_stack int +update_cache_item(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct inode *inode, + struct btrfs_path *path, u64 offset, + int entries, int bitmaps) +{ + struct btrfs_key key; + struct btrfs_free_space_header *header; + struct extent_buffer *leaf; + int ret; + + key.objectid = BTRFS_FREE_SPACE_OBJECTID; + key.type = 0; + key.offset = offset; + + ret = btrfs_search_slot(trans, root, &key, path, 0, 1); + if (ret < 0) { + btrfs_clear_extent_bit(&BTRFS_I(inode)->io_tree, 0, inode->i_size - 1, + EXTENT_DELALLOC, NULL); + return ret; + } + leaf = path->nodes[0]; + if (ret > 0) { + struct btrfs_key found_key; + ASSERT(path->slots[0]); + path->slots[0]--; + btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); + if (found_key.objectid != BTRFS_FREE_SPACE_OBJECTID || + found_key.offset != offset) { + btrfs_clear_extent_bit(&BTRFS_I(inode)->io_tree, 0, + inode->i_size - 1, EXTENT_DELALLOC, + NULL); + btrfs_release_path(path); + return -ENOENT; + } + } + + BTRFS_I(inode)->generation = trans->transid; + header = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_free_space_header); + btrfs_set_free_space_entries(leaf, header, entries); + btrfs_set_free_space_bitmaps(leaf, header, bitmaps); + btrfs_set_free_space_generation(leaf, header, trans->transid); + btrfs_release_path(path); + + return 0; +} + +static noinline_for_stack int write_pinned_extent_entries( + struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct btrfs_io_ctl *io_ctl, + int *entries) +{ + u64 start, extent_start, extent_end, len; + const u64 block_group_end = btrfs_block_group_end(block_group); + struct extent_io_tree *unpin = NULL; + int ret; + + /* + * We want to add any pinned extents to our free space cache + * so we don't leak the space + * + * We shouldn't have switched the pinned extents yet so this is the + * right one + */ + unpin = &trans->transaction->pinned_extents; + + start = block_group->start; + + while (start < block_group_end) { + if (!btrfs_find_first_extent_bit(unpin, start, + &extent_start, &extent_end, + EXTENT_DIRTY, NULL)) + return 0; + + /* This pinned extent is out of our range */ + if (extent_start >= block_group_end) + return 0; + + extent_start = max(extent_start, start); + extent_end = min(block_group_end, extent_end + 1); + len = extent_end - extent_start; + + *entries += 1; + ret = io_ctl_add_entry(io_ctl, extent_start, len, NULL); + if (ret) + return -ENOSPC; + + start = extent_end; + } + + return 0; +} + +static noinline_for_stack int +write_bitmap_entries(struct btrfs_io_ctl *io_ctl, struct list_head *bitmap_list) +{ + struct btrfs_free_space *entry, *next; + int ret; + + /* Write out the bitmaps */ + list_for_each_entry_safe(entry, next, bitmap_list, list) { + ret = io_ctl_add_bitmap(io_ctl, entry->bitmap); + if (ret) + return -ENOSPC; + list_del_init(&entry->list); + } + + return 0; +} + +static int flush_dirty_cache(struct inode *inode) +{ + int ret; + + ret = btrfs_wait_ordered_range(BTRFS_I(inode), 0, (u64)-1); + if (ret) + btrfs_clear_extent_bit(&BTRFS_I(inode)->io_tree, 0, inode->i_size - 1, + EXTENT_DELALLOC, NULL); + + return ret; +} + +static void noinline_for_stack +cleanup_bitmap_list(struct list_head *bitmap_list) +{ + struct btrfs_free_space *entry, *next; + + list_for_each_entry_safe(entry, next, bitmap_list, list) + list_del_init(&entry->list); +} + +static void noinline_for_stack +cleanup_write_cache_enospc(struct inode *inode, + struct btrfs_io_ctl *io_ctl, + struct extent_state **cached_state) +{ + io_ctl_drop_pages(io_ctl); + btrfs_unlock_extent(&BTRFS_I(inode)->io_tree, 0, i_size_read(inode) - 1, + cached_state); +} + +static int __btrfs_wait_cache_io(struct btrfs_root *root, + struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct btrfs_io_ctl *io_ctl, + struct btrfs_path *path, u64 offset) +{ + int ret; + struct inode *inode = io_ctl->inode; + + if (!inode) + return 0; + + /* Flush the dirty pages in the cache file. */ + ret = flush_dirty_cache(inode); + if (ret) + goto out; + + /* Update the cache item to tell everyone this cache file is valid. */ + ret = update_cache_item(trans, root, inode, path, offset, + io_ctl->entries, io_ctl->bitmaps); +out: + if (ret) { + invalidate_inode_pages2(inode->i_mapping); + BTRFS_I(inode)->generation = 0; + if (block_group) + btrfs_debug(root->fs_info, + "failed to write free space cache for block group %llu error %d", + block_group->start, ret); + } + btrfs_update_inode(trans, BTRFS_I(inode)); + + if (block_group) { + /* the dirty list is protected by the dirty_bgs_lock */ + spin_lock(&trans->transaction->dirty_bgs_lock); + + /* the disk_cache_state is protected by the block group lock */ + spin_lock(&block_group->lock); + + /* + * only mark this as written if we didn't get put back on + * the dirty list while waiting for IO. Otherwise our + * cache state won't be right, and we won't get written again + */ + if (!ret && list_empty(&block_group->dirty_list)) + block_group->disk_cache_state = BTRFS_DC_WRITTEN; + else if (ret) + block_group->disk_cache_state = BTRFS_DC_ERROR; + + spin_unlock(&block_group->lock); + spin_unlock(&trans->transaction->dirty_bgs_lock); + io_ctl->inode = NULL; + iput(inode); + } + + return ret; + +} + +int btrfs_wait_cache_io(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct btrfs_path *path) +{ + return __btrfs_wait_cache_io(block_group->fs_info->tree_root, trans, + block_group, &block_group->io_ctl, + path, block_group->start); +} + +/* + * Write out cached info to an inode. + * + * @inode: freespace inode we are writing out + * @ctl: free space cache we are going to write out + * @block_group: block_group for this cache if it belongs to a block_group + * @io_ctl: holds context for the io + * @trans: the trans handle + * + * This function writes out a free space cache struct to disk for quick recovery + * on mount. This will return 0 if it was successful in writing the cache out, + * or an errno if it was not. + */ +static int __btrfs_write_out_cache(struct inode *inode, + struct btrfs_block_group *block_group, + struct btrfs_trans_handle *trans) +{ + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + struct btrfs_io_ctl *io_ctl = &block_group->io_ctl; + struct extent_state *cached_state = NULL; + LIST_HEAD(bitmap_list); + int entries = 0; + int bitmaps = 0; + int ret; + bool must_iput = false; + int i_size; + + if (!i_size_read(inode)) + return -EIO; + + WARN_ON(io_ctl->pages); + ret = io_ctl_init(io_ctl, inode, 1); + if (ret) + return ret; + + if (block_group->flags & BTRFS_BLOCK_GROUP_DATA) { + down_write(&block_group->data_rwsem); + spin_lock(&block_group->lock); + if (block_group->delalloc_bytes) { + block_group->disk_cache_state = BTRFS_DC_WRITTEN; + spin_unlock(&block_group->lock); + up_write(&block_group->data_rwsem); + BTRFS_I(inode)->generation = 0; + ret = 0; + must_iput = true; + goto out; + } + spin_unlock(&block_group->lock); + } + + /* Lock all pages first so we can lock the extent safely. */ + ret = io_ctl_prepare_pages(io_ctl, false); + if (ret) + goto out_unlock; + + btrfs_lock_extent(&BTRFS_I(inode)->io_tree, 0, i_size_read(inode) - 1, + &cached_state); + + io_ctl_set_generation(io_ctl, trans->transid); + + mutex_lock(&ctl->cache_writeout_mutex); + /* Write out the extent entries in the free space cache */ + spin_lock(&ctl->tree_lock); + ret = write_cache_extent_entries(io_ctl, block_group, &entries, &bitmaps, + &bitmap_list); + if (ret) + goto out_nospc_locked; + + /* + * Some spaces that are freed in the current transaction are pinned, + * they will be added into free space cache after the transaction is + * committed, we shouldn't lose them. + * + * If this changes while we are working we'll get added back to + * the dirty list and redo it. No locking needed + */ + ret = write_pinned_extent_entries(trans, block_group, io_ctl, &entries); + if (ret) + goto out_nospc_locked; + + /* + * At last, we write out all the bitmaps and keep cache_writeout_mutex + * locked while doing it because a concurrent trim can be manipulating + * or freeing the bitmap. + */ + ret = write_bitmap_entries(io_ctl, &bitmap_list); + spin_unlock(&ctl->tree_lock); + mutex_unlock(&ctl->cache_writeout_mutex); + if (ret) + goto out_nospc; + + /* Zero out the rest of the pages just to make sure */ + io_ctl_zero_remaining_pages(io_ctl); + + /* Everything is written out, now we dirty the pages in the file. */ + i_size = i_size_read(inode); + for (int i = 0; i < round_up(i_size, PAGE_SIZE) / PAGE_SIZE; i++) { + u64 dirty_start = i * PAGE_SIZE; + u64 dirty_len = min_t(u64, dirty_start + PAGE_SIZE, i_size) - dirty_start; + + ret = btrfs_dirty_folio(BTRFS_I(inode), page_folio(io_ctl->pages[i]), + dirty_start, dirty_len, &cached_state, false); + if (ret < 0) + goto out_nospc; + } + + if (block_group->flags & BTRFS_BLOCK_GROUP_DATA) + up_write(&block_group->data_rwsem); + /* + * Release the pages and unlock the extent, we will flush + * them out later + */ + io_ctl_drop_pages(io_ctl); + io_ctl_free(io_ctl); + + btrfs_unlock_extent(&BTRFS_I(inode)->io_tree, 0, i_size_read(inode) - 1, + &cached_state); + + /* + * at this point the pages are under IO and we're happy, + * The caller is responsible for waiting on them and updating + * the cache and the inode + */ + io_ctl->entries = entries; + io_ctl->bitmaps = bitmaps; + + ret = btrfs_fdatawrite_range(BTRFS_I(inode), 0, (u64)-1); + if (ret) + goto out; + + return 0; + +out_nospc_locked: + cleanup_bitmap_list(&bitmap_list); + spin_unlock(&ctl->tree_lock); + mutex_unlock(&ctl->cache_writeout_mutex); + +out_nospc: + cleanup_write_cache_enospc(inode, io_ctl, &cached_state); + +out_unlock: + if (block_group->flags & BTRFS_BLOCK_GROUP_DATA) + up_write(&block_group->data_rwsem); + +out: + io_ctl->inode = NULL; + io_ctl_free(io_ctl); + if (ret) { + invalidate_inode_pages2(inode->i_mapping); + BTRFS_I(inode)->generation = 0; + } + btrfs_update_inode(trans, BTRFS_I(inode)); + if (must_iput) + iput(inode); + return ret; +} + +int btrfs_write_out_cache(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct btrfs_path *path) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct inode *inode; + int ret = 0; + + spin_lock(&block_group->lock); + if (block_group->disk_cache_state < BTRFS_DC_SETUP) { + spin_unlock(&block_group->lock); + return 0; + } + spin_unlock(&block_group->lock); + + inode = lookup_free_space_inode(block_group, path); + if (IS_ERR(inode)) + return 0; + + ret = __btrfs_write_out_cache(inode, block_group, trans); + if (ret) { + btrfs_debug(fs_info, + "failed to write free space cache for block group %llu error %d", + block_group->start, ret); + spin_lock(&block_group->lock); + block_group->disk_cache_state = BTRFS_DC_ERROR; + spin_unlock(&block_group->lock); + + block_group->io_ctl.inode = NULL; + iput(inode); + } + + /* + * if ret == 0 the caller is expected to call btrfs_wait_cache_io + * to wait for IO and put the inode + */ + + return ret; +} + +static inline unsigned long offset_to_bit(u64 bitmap_start, u32 unit, + u64 offset) +{ + ASSERT(offset >= bitmap_start); + offset -= bitmap_start; + return (unsigned long)(div_u64(offset, unit)); +} + +static inline unsigned long bytes_to_bits(u64 bytes, u32 unit) +{ + return (unsigned long)(div_u64(bytes, unit)); +} + +static inline u64 offset_to_bitmap(struct btrfs_free_space_ctl *ctl, + u64 offset) +{ + u64 bitmap_start; + u64 bytes_per_bitmap; + + bytes_per_bitmap = BITS_PER_BITMAP * ctl->block_group->fs_info->sectorsize; + bitmap_start = offset - ctl->block_group->start; + bitmap_start = div64_u64(bitmap_start, bytes_per_bitmap); + bitmap_start *= bytes_per_bitmap; + bitmap_start += ctl->block_group->start; + + return bitmap_start; +} + +static int tree_insert_offset(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_cluster *cluster, + struct btrfs_free_space *new_entry) +{ + struct rb_root *root; + struct rb_node **p; + struct rb_node *parent = NULL; + + lockdep_assert_held(&ctl->tree_lock); + + if (cluster) { + lockdep_assert_held(&cluster->lock); + root = &cluster->root; + } else { + root = &ctl->free_space_offset; + } + + p = &root->rb_node; + + while (*p) { + struct btrfs_free_space *info; + + parent = *p; + info = rb_entry(parent, struct btrfs_free_space, offset_index); + + if (new_entry->offset < info->offset) { + p = &(*p)->rb_left; + } else if (new_entry->offset > info->offset) { + p = &(*p)->rb_right; + } else { + /* + * we could have a bitmap entry and an extent entry + * share the same offset. If this is the case, we want + * the extent entry to always be found first if we do a + * linear search through the tree, since we want to have + * the quickest allocation time, and allocating from an + * extent is faster than allocating from a bitmap. So + * if we're inserting a bitmap and we find an entry at + * this offset, we want to go right, or after this entry + * logically. If we are inserting an extent and we've + * found a bitmap, we want to go left, or before + * logically. + */ + if (new_entry->bitmap) { + if (info->bitmap) { + WARN_ON_ONCE(1); + return -EEXIST; + } + p = &(*p)->rb_right; + } else { + if (!info->bitmap) { + WARN_ON_ONCE(1); + return -EEXIST; + } + p = &(*p)->rb_left; + } + } + } + + rb_link_node(&new_entry->offset_index, parent, p); + rb_insert_color(&new_entry->offset_index, root); + + return 0; +} + +/* + * This is a little subtle. We *only* have ->max_extent_size set if we actually + * searched through the bitmap and figured out the largest ->max_extent_size, + * otherwise it's 0. In the case that it's 0 we don't want to tell the + * allocator the wrong thing, we want to use the actual real max_extent_size + * we've found already if it's larger, or we want to use ->bytes. + * + * This matters because find_free_space() will skip entries who's ->bytes is + * less than the required bytes. So if we didn't search down this bitmap, we + * may pick some previous entry that has a smaller ->max_extent_size than we + * have. For example, assume we have two entries, one that has + * ->max_extent_size set to 4K and ->bytes set to 1M. A second entry hasn't set + * ->max_extent_size yet, has ->bytes set to 8K and it's contiguous. We will + * call into find_free_space(), and return with max_extent_size == 4K, because + * that first bitmap entry had ->max_extent_size set, but the second one did + * not. If instead we returned 8K we'd come in searching for 8K, and find the + * 8K contiguous range. + * + * Consider the other case, we have 2 8K chunks in that second entry and still + * don't have ->max_extent_size set. We'll return 16K, and the next time the + * allocator comes in it'll fully search our second bitmap, and this time it'll + * get an uptodate value of 8K as the maximum chunk size. Then we'll get the + * right allocation the next loop through. + */ +static inline u64 get_max_extent_size(const struct btrfs_free_space *entry) +{ + if (entry->bitmap && entry->max_extent_size) + return entry->max_extent_size; + return entry->bytes; +} + +/* + * We want the largest entry to be leftmost, so this is inverted from what you'd + * normally expect. + */ +static bool entry_less(struct rb_node *node, const struct rb_node *parent) +{ + const struct btrfs_free_space *entry, *exist; + + entry = rb_entry(node, struct btrfs_free_space, bytes_index); + exist = rb_entry(parent, struct btrfs_free_space, bytes_index); + return get_max_extent_size(exist) < get_max_extent_size(entry); +} + +/* + * searches the tree for the given offset. + * + * fuzzy - If this is set, then we are trying to make an allocation, and we just + * want a section that has at least bytes size and comes at or after the given + * offset. + */ +static struct btrfs_free_space * +tree_search_offset(struct btrfs_free_space_ctl *ctl, + u64 offset, int bitmap_only, int fuzzy) +{ + struct rb_node *n = ctl->free_space_offset.rb_node; + struct btrfs_free_space *entry = NULL, *prev = NULL; + const int unit = ctl->block_group->fs_info->sectorsize; + + lockdep_assert_held(&ctl->tree_lock); + + /* find entry that is closest to the 'offset' */ + while (n) { + entry = rb_entry(n, struct btrfs_free_space, offset_index); + prev = entry; + + if (offset < entry->offset) + n = n->rb_left; + else if (offset > entry->offset) + n = n->rb_right; + else + break; + + entry = NULL; + } + + if (bitmap_only) { + if (!entry) + return NULL; + if (entry->bitmap) + return entry; + + /* + * bitmap entry and extent entry may share same offset, + * in that case, bitmap entry comes after extent entry. + */ + n = rb_next(n); + if (!n) + return NULL; + entry = rb_entry(n, struct btrfs_free_space, offset_index); + if (entry->offset != offset) + return NULL; + + WARN_ON(!entry->bitmap); + return entry; + } else if (entry) { + if (entry->bitmap) { + /* + * if previous extent entry covers the offset, + * we should return it instead of the bitmap entry + */ + n = rb_prev(&entry->offset_index); + if (n) { + prev = rb_entry(n, struct btrfs_free_space, + offset_index); + if (!prev->bitmap && + prev->offset + prev->bytes > offset) + entry = prev; + } + } + return entry; + } + + if (!prev) + return NULL; + + /* find last entry before the 'offset' */ + entry = prev; + if (entry->offset > offset) { + n = rb_prev(&entry->offset_index); + if (n) { + entry = rb_entry(n, struct btrfs_free_space, + offset_index); + ASSERT(entry->offset <= offset); + } else { + if (fuzzy) + return entry; + else + return NULL; + } + } + + if (entry->bitmap) { + n = rb_prev(&entry->offset_index); + if (n) { + prev = rb_entry(n, struct btrfs_free_space, + offset_index); + if (!prev->bitmap && + prev->offset + prev->bytes > offset) + return prev; + } + if (entry->offset + BITS_PER_BITMAP * unit > offset) + return entry; + } else if (entry->offset + entry->bytes > offset) + return entry; + + if (!fuzzy) + return NULL; + + while (1) { + n = rb_next(&entry->offset_index); + if (!n) + return NULL; + entry = rb_entry(n, struct btrfs_free_space, offset_index); + if (entry->bitmap) { + if (entry->offset + BITS_PER_BITMAP * unit > offset) + break; + } else { + if (entry->offset + entry->bytes > offset) + break; + } + } + return entry; +} + +static inline void unlink_free_space(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info, + bool update_stat) +{ + lockdep_assert_held(&ctl->tree_lock); + + rb_erase(&info->offset_index, &ctl->free_space_offset); + rb_erase_cached(&info->bytes_index, &ctl->free_space_bytes); + ctl->free_extents--; + + if (!info->bitmap && !btrfs_free_space_trimmed(info)) { + ctl->discardable_extents[BTRFS_STAT_CURR]--; + ctl->discardable_bytes[BTRFS_STAT_CURR] -= info->bytes; + } + + if (update_stat) + ctl->free_space -= info->bytes; +} + +static int link_free_space(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info) +{ + int ret = 0; + + lockdep_assert_held(&ctl->tree_lock); + + ASSERT(info->bytes || info->bitmap); + ret = tree_insert_offset(ctl, NULL, info); + if (ret) + return ret; + + rb_add_cached(&info->bytes_index, &ctl->free_space_bytes, entry_less); + + if (!info->bitmap && !btrfs_free_space_trimmed(info)) { + ctl->discardable_extents[BTRFS_STAT_CURR]++; + ctl->discardable_bytes[BTRFS_STAT_CURR] += info->bytes; + } + + ctl->free_space += info->bytes; + ctl->free_extents++; + return ret; +} + +static void relink_bitmap_entry(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info) +{ + ASSERT(info->bitmap); + + /* + * If our entry is empty it's because we're on a cluster and we don't + * want to re-link it into our ctl bytes index. + */ + if (RB_EMPTY_NODE(&info->bytes_index)) + return; + + lockdep_assert_held(&ctl->tree_lock); + + rb_erase_cached(&info->bytes_index, &ctl->free_space_bytes); + rb_add_cached(&info->bytes_index, &ctl->free_space_bytes, entry_less); +} + +static inline void bitmap_clear_bits(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info, + u64 offset, u64 bytes, bool update_stat) +{ + const int unit = ctl->block_group->fs_info->sectorsize; + unsigned long start, count, end; + int extent_delta = -1; + + start = offset_to_bit(info->offset, unit, offset); + count = bytes_to_bits(bytes, unit); + end = start + count; + ASSERT(end <= BITS_PER_BITMAP); + + bitmap_clear(info->bitmap, start, count); + + info->bytes -= bytes; + if (info->max_extent_size > unit) + info->max_extent_size = 0; + + relink_bitmap_entry(ctl, info); + + if (start && test_bit(start - 1, info->bitmap)) + extent_delta++; + + if (end < BITS_PER_BITMAP && test_bit(end, info->bitmap)) + extent_delta++; + + info->bitmap_extents += extent_delta; + if (!btrfs_free_space_trimmed(info)) { + ctl->discardable_extents[BTRFS_STAT_CURR] += extent_delta; + ctl->discardable_bytes[BTRFS_STAT_CURR] -= bytes; + } + + if (update_stat) + ctl->free_space -= bytes; +} + +static void btrfs_bitmap_set_bits(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info, u64 offset, + u64 bytes) +{ + const int unit = ctl->block_group->fs_info->sectorsize; + unsigned long start, count, end; + int extent_delta = 1; + + start = offset_to_bit(info->offset, unit, offset); + count = bytes_to_bits(bytes, unit); + end = start + count; + ASSERT(end <= BITS_PER_BITMAP); + + bitmap_set(info->bitmap, start, count); + + /* + * We set some bytes, we have no idea what the max extent size is + * anymore. + */ + info->max_extent_size = 0; + info->bytes += bytes; + ctl->free_space += bytes; + + relink_bitmap_entry(ctl, info); + + if (start && test_bit(start - 1, info->bitmap)) + extent_delta--; + + if (end < BITS_PER_BITMAP && test_bit(end, info->bitmap)) + extent_delta--; + + info->bitmap_extents += extent_delta; + if (!btrfs_free_space_trimmed(info)) { + ctl->discardable_extents[BTRFS_STAT_CURR] += extent_delta; + ctl->discardable_bytes[BTRFS_STAT_CURR] += bytes; + } +} + +/* + * If we can not find suitable extent, we will use bytes to record + * the size of the max extent. + */ +static int search_bitmap(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *bitmap_info, u64 *offset, + u64 *bytes, bool for_alloc) +{ + const int unit = ctl->block_group->fs_info->sectorsize; + unsigned long found_bits = 0; + unsigned long max_bits = 0; + unsigned long bits, i; + unsigned long next_zero; + unsigned long extent_bits; + + /* + * Skip searching the bitmap if we don't have a contiguous section that + * is large enough for this allocation. + */ + if (for_alloc && + bitmap_info->max_extent_size && + bitmap_info->max_extent_size < *bytes) { + *bytes = bitmap_info->max_extent_size; + return -1; + } + + i = offset_to_bit(bitmap_info->offset, unit, + max_t(u64, *offset, bitmap_info->offset)); + bits = bytes_to_bits(*bytes, unit); + + for_each_set_bit_from(i, bitmap_info->bitmap, BITS_PER_BITMAP) { + if (for_alloc && bits == 1) { + found_bits = 1; + break; + } + next_zero = find_next_zero_bit(bitmap_info->bitmap, + BITS_PER_BITMAP, i); + extent_bits = next_zero - i; + if (extent_bits >= bits) { + found_bits = extent_bits; + break; + } else if (extent_bits > max_bits) { + max_bits = extent_bits; + } + i = next_zero; + } + + if (found_bits) { + *offset = (u64)(i * unit) + bitmap_info->offset; + *bytes = (u64)(found_bits) * unit; + return 0; + } + + *bytes = (u64)(max_bits) * unit; + bitmap_info->max_extent_size = *bytes; + relink_bitmap_entry(ctl, bitmap_info); + return -1; +} + +/* Cache the size of the max extent in bytes */ +static struct btrfs_free_space * +find_free_space(struct btrfs_free_space_ctl *ctl, u64 *offset, u64 *bytes, + unsigned long align, u64 *max_extent_size, bool use_bytes_index) +{ + struct btrfs_free_space *entry; + struct rb_node *node; + u64 tmp; + u64 align_off; + int ret; + + if (!ctl->free_space_offset.rb_node) + return NULL; +again: + if (use_bytes_index) { + node = rb_first_cached(&ctl->free_space_bytes); + } else { + entry = tree_search_offset(ctl, offset_to_bitmap(ctl, *offset), + 0, 1); + if (!entry) + return NULL; + node = &entry->offset_index; + } + + for (; node; node = rb_next(node)) { + if (use_bytes_index) + entry = rb_entry(node, struct btrfs_free_space, + bytes_index); + else + entry = rb_entry(node, struct btrfs_free_space, + offset_index); + + /* + * If we are using the bytes index then all subsequent entries + * in this tree are going to be < bytes, so simply set the max + * extent size and exit the loop. + * + * If we're using the offset index then we need to keep going + * through the rest of the tree. + */ + if (entry->bytes < *bytes) { + *max_extent_size = max(get_max_extent_size(entry), + *max_extent_size); + if (use_bytes_index) + break; + continue; + } + + /* make sure the space returned is big enough + * to match our requested alignment + */ + if (*bytes >= align) { + tmp = entry->offset - ctl->block_group->start + align - 1; + tmp = div64_u64(tmp, align); + tmp = tmp * align + ctl->block_group->start; + align_off = tmp - entry->offset; + } else { + align_off = 0; + tmp = entry->offset; + } + + /* + * We don't break here if we're using the bytes index because we + * may have another entry that has the correct alignment that is + * the right size, so we don't want to miss that possibility. + * At worst this adds another loop through the logic, but if we + * broke here we could prematurely ENOSPC. + */ + if (entry->bytes < *bytes + align_off) { + *max_extent_size = max(get_max_extent_size(entry), + *max_extent_size); + continue; + } + + if (entry->bitmap) { + struct rb_node *old_next = rb_next(node); + u64 size = *bytes; + + ret = search_bitmap(ctl, entry, &tmp, &size, true); + if (!ret) { + *offset = tmp; + *bytes = size; + return entry; + } else { + *max_extent_size = + max(get_max_extent_size(entry), + *max_extent_size); + } + + /* + * The bitmap may have gotten re-arranged in the space + * index here because the max_extent_size may have been + * updated. Start from the beginning again if this + * happened. + */ + if (use_bytes_index && old_next != rb_next(node)) + goto again; + continue; + } + + *offset = tmp; + *bytes = entry->bytes - align_off; + return entry; + } + + return NULL; +} + +static void add_new_bitmap(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info, u64 offset) +{ + info->offset = offset_to_bitmap(ctl, offset); + info->bytes = 0; + info->bitmap_extents = 0; + INIT_LIST_HEAD(&info->list); + link_free_space(ctl, info); + ctl->total_bitmaps++; + recalculate_thresholds(ctl); +} + +static void free_bitmap(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *bitmap_info) +{ + /* + * Normally when this is called, the bitmap is completely empty. However, + * if we are blowing up the free space cache for one reason or another + * via __btrfs_remove_free_space_cache(), then it may not be freed and + * we may leave stats on the table. + */ + if (bitmap_info->bytes && !btrfs_free_space_trimmed(bitmap_info)) { + ctl->discardable_extents[BTRFS_STAT_CURR] -= + bitmap_info->bitmap_extents; + ctl->discardable_bytes[BTRFS_STAT_CURR] -= bitmap_info->bytes; + + } + unlink_free_space(ctl, bitmap_info, true); + kmem_cache_free(btrfs_free_space_bitmap_cachep, bitmap_info->bitmap); + kmem_cache_free(btrfs_free_space_cachep, bitmap_info); + ctl->total_bitmaps--; + recalculate_thresholds(ctl); +} + +static noinline int remove_from_bitmap(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *bitmap_info, + u64 *offset, u64 *bytes) +{ + const int unit = ctl->block_group->fs_info->sectorsize; + u64 end; + u64 search_start, search_bytes; + int ret; + +again: + end = bitmap_info->offset + (u64)(BITS_PER_BITMAP * unit) - 1; + + /* + * We need to search for bits in this bitmap. We could only cover some + * of the extent in this bitmap thanks to how we add space, so we need + * to search for as much as it as we can and clear that amount, and then + * go searching for the next bit. + */ + search_start = *offset; + search_bytes = unit; + search_bytes = min(search_bytes, end - search_start + 1); + ret = search_bitmap(ctl, bitmap_info, &search_start, &search_bytes, + false); + if (ret < 0 || search_start != *offset) + return -EINVAL; + + /* We may have found more bits than what we need */ + search_bytes = min(search_bytes, *bytes); + + /* Cannot clear past the end of the bitmap */ + search_bytes = min(search_bytes, end - search_start + 1); + + bitmap_clear_bits(ctl, bitmap_info, search_start, search_bytes, true); + *offset += search_bytes; + *bytes -= search_bytes; + + if (*bytes) { + struct rb_node *next = rb_next(&bitmap_info->offset_index); + if (!bitmap_info->bytes) + free_bitmap(ctl, bitmap_info); + + /* + * no entry after this bitmap, but we still have bytes to + * remove, so something has gone wrong. + */ + if (!next) + return -EINVAL; + + bitmap_info = rb_entry(next, struct btrfs_free_space, + offset_index); + + /* + * if the next entry isn't a bitmap we need to return to let the + * extent stuff do its work. + */ + if (!bitmap_info->bitmap) + return -EAGAIN; + + /* + * Ok the next item is a bitmap, but it may not actually hold + * the information for the rest of this free space stuff, so + * look for it, and if we don't find it return so we can try + * everything over again. + */ + search_start = *offset; + search_bytes = unit; + ret = search_bitmap(ctl, bitmap_info, &search_start, + &search_bytes, false); + if (ret < 0 || search_start != *offset) + return -EAGAIN; + + goto again; + } else if (!bitmap_info->bytes) + free_bitmap(ctl, bitmap_info); + + return 0; +} + +static u64 add_bytes_to_bitmap(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info, u64 offset, + u64 bytes, enum btrfs_trim_state trim_state) +{ + const int unit = ctl->block_group->fs_info->sectorsize; + u64 bytes_to_set = 0; + u64 end; + + /* + * This is a tradeoff to make bitmap trim state minimal. We mark the + * whole bitmap untrimmed if at any point we add untrimmed regions. + */ + if (trim_state == BTRFS_TRIM_STATE_UNTRIMMED) { + if (btrfs_free_space_trimmed(info)) { + ctl->discardable_extents[BTRFS_STAT_CURR] += + info->bitmap_extents; + ctl->discardable_bytes[BTRFS_STAT_CURR] += info->bytes; + } + info->trim_state = BTRFS_TRIM_STATE_UNTRIMMED; + } + + end = info->offset + (u64)(BITS_PER_BITMAP * unit); + + bytes_to_set = min(end - offset, bytes); + + btrfs_bitmap_set_bits(ctl, info, offset, bytes_to_set); + + return bytes_to_set; + +} + +EXPORT_FOR_TESTS +bool btrfs_use_bitmap(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info) +{ + struct btrfs_block_group *block_group = ctl->block_group; + struct btrfs_fs_info *fs_info = block_group->fs_info; + bool forced = false; + +#ifdef CONFIG_BTRFS_DEBUG + if (btrfs_should_fragment_free_space(block_group)) + forced = true; +#endif + + /* This is a way to reclaim large regions from the bitmaps. */ + if (!forced && info->bytes >= FORCE_EXTENT_THRESHOLD) + return false; + + /* + * If we are below the extents threshold then we can add this as an + * extent, and don't have to deal with the bitmap + */ + if (!forced && ctl->free_extents < ctl->extents_thresh) { + /* + * If this block group has some small extents we don't want to + * use up all of our free slots in the cache with them, we want + * to reserve them to larger extents, however if we have plenty + * of cache left then go ahead and add them, no sense in adding + * the overhead of a bitmap if we don't have to. + */ + if (info->bytes <= fs_info->sectorsize * 8) { + if (ctl->free_extents * 3 <= ctl->extents_thresh) + return false; + } else { + return false; + } + } + + /* + * The original block groups from mkfs can be really small, like 8 + * megabytes, so don't bother with a bitmap for those entries. However + * some block groups can be smaller than what a bitmap would cover but + * are still large enough that they could overflow the 32k memory limit, + * so allow those block groups to still be allowed to have a bitmap + * entry. + */ + if (((BITS_PER_BITMAP * fs_info->sectorsize) >> 1) > block_group->length) + return false; + + return true; +} + +static int insert_into_bitmap(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info) +{ + struct btrfs_free_space *bitmap_info; + struct btrfs_block_group *block_group = ctl->block_group; + bool added = false; + u64 bytes, offset, bytes_added; + enum btrfs_trim_state trim_state; + int ret; + + bytes = info->bytes; + offset = info->offset; + trim_state = info->trim_state; + + if (btrfs_is_testing(block_group->fs_info)) { + if (!block_group->fs_info->use_bitmap(ctl, info)) + return 0; + } else { + if (!btrfs_use_bitmap(ctl, info)) + return 0; + } +again: + /* + * Since we link bitmaps right into the cluster we need to see if we + * have a cluster here, and if so and it has our bitmap we need to add + * the free space to that bitmap. + */ + if (!list_empty(&block_group->cluster_list)) { + struct btrfs_free_cluster *cluster; + struct rb_node *node; + struct btrfs_free_space *entry; + + cluster = list_first_entry(&block_group->cluster_list, + struct btrfs_free_cluster, block_group_list); + spin_lock(&cluster->lock); + node = rb_first(&cluster->root); + if (!node) { + spin_unlock(&cluster->lock); + goto no_cluster_bitmap; + } + + entry = rb_entry(node, struct btrfs_free_space, offset_index); + if (!entry->bitmap) { + spin_unlock(&cluster->lock); + goto no_cluster_bitmap; + } + + if (entry->offset == offset_to_bitmap(ctl, offset)) { + bytes_added = add_bytes_to_bitmap(ctl, entry, offset, + bytes, trim_state); + bytes -= bytes_added; + offset += bytes_added; + } + spin_unlock(&cluster->lock); + if (!bytes) { + ret = 1; + goto out; + } + } + +no_cluster_bitmap: + bitmap_info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), + 1, 0); + if (!bitmap_info) { + ASSERT(!added); + goto new_bitmap; + } + + bytes_added = add_bytes_to_bitmap(ctl, bitmap_info, offset, bytes, + trim_state); + bytes -= bytes_added; + offset += bytes_added; + added = false; + + if (!bytes) { + ret = 1; + goto out; + } else + goto again; + +new_bitmap: + if (info && info->bitmap) { + add_new_bitmap(ctl, info, offset); + added = true; + info = NULL; + goto again; + } else { + spin_unlock(&ctl->tree_lock); + + /* no pre-allocated info, allocate a new one */ + if (!info) { + info = kmem_cache_zalloc(btrfs_free_space_cachep, + GFP_NOFS); + if (!info) { + spin_lock(&ctl->tree_lock); + ret = -ENOMEM; + goto out; + } + } + + /* allocate the bitmap */ + info->bitmap = kmem_cache_zalloc(btrfs_free_space_bitmap_cachep, + GFP_NOFS); + info->trim_state = BTRFS_TRIM_STATE_TRIMMED; + spin_lock(&ctl->tree_lock); + if (!info->bitmap) { + ret = -ENOMEM; + goto out; + } + goto again; + } + +out: + if (info) { + if (info->bitmap) + kmem_cache_free(btrfs_free_space_bitmap_cachep, + info->bitmap); + kmem_cache_free(btrfs_free_space_cachep, info); + } + + return ret; +} + +/* + * Free space merging rules: + * 1) Merge trimmed areas together + * 2) Let untrimmed areas coalesce with trimmed areas + * 3) Always pull neighboring regions from bitmaps + * + * The above rules are for when we merge free space based on btrfs_trim_state. + * Rules 2 and 3 are subtle because they are suboptimal, but are done for the + * same reason: to promote larger extent regions which makes life easier for + * find_free_extent(). Rule 2 enables coalescing based on the common path + * being returning free space from btrfs_finish_extent_commit(). So when free + * space is trimmed, it will prevent aggregating trimmed new region and + * untrimmed regions in the rb_tree. Rule 3 is purely to obtain larger extents + * and provide find_free_extent() with the largest extents possible hoping for + * the reuse path. + */ +static bool try_merge_free_space(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info, bool update_stat) +{ + struct btrfs_free_space *left_info = NULL; + struct btrfs_free_space *right_info; + bool merged = false; + u64 offset = info->offset; + u64 bytes = info->bytes; + const bool is_trimmed = btrfs_free_space_trimmed(info); + struct rb_node *right_prev = NULL; + + /* + * first we want to see if there is free space adjacent to the range we + * are adding, if there is remove that struct and add a new one to + * cover the entire range + */ + right_info = tree_search_offset(ctl, offset + bytes, 0, 0); + if (right_info) + right_prev = rb_prev(&right_info->offset_index); + + if (right_prev) + left_info = rb_entry(right_prev, struct btrfs_free_space, offset_index); + else if (!right_info) + left_info = tree_search_offset(ctl, offset - 1, 0, 0); + + /* See try_merge_free_space() comment. */ + if (right_info && !right_info->bitmap && + (!is_trimmed || btrfs_free_space_trimmed(right_info))) { + unlink_free_space(ctl, right_info, update_stat); + info->bytes += right_info->bytes; + kmem_cache_free(btrfs_free_space_cachep, right_info); + merged = true; + } + + /* See try_merge_free_space() comment. */ + if (left_info && !left_info->bitmap && + left_info->offset + left_info->bytes == offset && + (!is_trimmed || btrfs_free_space_trimmed(left_info))) { + unlink_free_space(ctl, left_info, update_stat); + info->offset = left_info->offset; + info->bytes += left_info->bytes; + kmem_cache_free(btrfs_free_space_cachep, left_info); + merged = true; + } + + return merged; +} + +static bool steal_from_bitmap_to_end(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info, + bool update_stat) +{ + const int unit = ctl->block_group->fs_info->sectorsize; + struct btrfs_free_space *bitmap; + unsigned long i; + unsigned long j; + const u64 end = info->offset + info->bytes; + const u64 bitmap_offset = offset_to_bitmap(ctl, end); + u64 bytes; + + bitmap = tree_search_offset(ctl, bitmap_offset, 1, 0); + if (!bitmap) + return false; + + i = offset_to_bit(bitmap->offset, unit, end); + j = find_next_zero_bit(bitmap->bitmap, BITS_PER_BITMAP, i); + if (j == i) + return false; + bytes = (j - i) * unit; + info->bytes += bytes; + + /* See try_merge_free_space() comment. */ + if (!btrfs_free_space_trimmed(bitmap)) + info->trim_state = BTRFS_TRIM_STATE_UNTRIMMED; + + bitmap_clear_bits(ctl, bitmap, end, bytes, update_stat); + + if (!bitmap->bytes) + free_bitmap(ctl, bitmap); + + return true; +} + +static bool steal_from_bitmap_to_front(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info, + bool update_stat) +{ + const int unit = ctl->block_group->fs_info->sectorsize; + struct btrfs_free_space *bitmap; + u64 bitmap_offset; + unsigned long i; + unsigned long j; + unsigned long prev_j; + u64 bytes; + + bitmap_offset = offset_to_bitmap(ctl, info->offset); + /* If we're on a boundary, try the previous logical bitmap. */ + if (bitmap_offset == info->offset) { + if (info->offset == 0) + return false; + bitmap_offset = offset_to_bitmap(ctl, info->offset - 1); + } + + bitmap = tree_search_offset(ctl, bitmap_offset, 1, 0); + if (!bitmap) + return false; + + i = offset_to_bit(bitmap->offset, unit, info->offset) - 1; + j = 0; + prev_j = (unsigned long)-1; + for_each_clear_bit_from(j, bitmap->bitmap, BITS_PER_BITMAP) { + if (j > i) + break; + prev_j = j; + } + if (prev_j == i) + return false; + + if (prev_j == (unsigned long)-1) + bytes = (i + 1) * unit; + else + bytes = (i - prev_j) * unit; + + info->offset -= bytes; + info->bytes += bytes; + + /* See try_merge_free_space() comment. */ + if (!btrfs_free_space_trimmed(bitmap)) + info->trim_state = BTRFS_TRIM_STATE_UNTRIMMED; + + bitmap_clear_bits(ctl, bitmap, info->offset, bytes, update_stat); + + if (!bitmap->bytes) + free_bitmap(ctl, bitmap); + + return true; +} + +/* + * We prefer always to allocate from extent entries, both for clustered and + * non-clustered allocation requests. So when attempting to add a new extent + * entry, try to see if there's adjacent free space in bitmap entries, and if + * there is, migrate that space from the bitmaps to the extent. + * Like this we get better chances of satisfying space allocation requests + * because we attempt to satisfy them based on a single cache entry, and never + * on 2 or more entries - even if the entries represent a contiguous free space + * region (e.g. 1 extent entry + 1 bitmap entry starting where the extent entry + * ends). + */ +static void steal_from_bitmap(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info, + bool update_stat) +{ + /* + * Only work with disconnected entries, as we can change their offset, + * and must be extent entries. + */ + ASSERT(!info->bitmap); + ASSERT(RB_EMPTY_NODE(&info->offset_index)); + + if (ctl->total_bitmaps > 0) { + bool stole_end; + bool stole_front = false; + + stole_end = steal_from_bitmap_to_end(ctl, info, update_stat); + if (ctl->total_bitmaps > 0) + stole_front = steal_from_bitmap_to_front(ctl, info, + update_stat); + + if (stole_end || stole_front) + try_merge_free_space(ctl, info, update_stat); + } +} + +static int __btrfs_add_free_space(struct btrfs_block_group *block_group, + u64 offset, u64 bytes, + enum btrfs_trim_state trim_state) +{ + struct btrfs_fs_info *fs_info = block_group->fs_info; + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + struct btrfs_free_space *info; + int ret = 0; + u64 filter_bytes = bytes; + + ASSERT(!btrfs_is_zoned(fs_info)); + + info = kmem_cache_zalloc(btrfs_free_space_cachep, GFP_NOFS); + if (!info) + return -ENOMEM; + + info->offset = offset; + info->bytes = bytes; + info->trim_state = trim_state; + RB_CLEAR_NODE(&info->offset_index); + RB_CLEAR_NODE(&info->bytes_index); + + spin_lock(&ctl->tree_lock); + + if (try_merge_free_space(ctl, info, true)) + goto link; + + /* + * There was no extent directly to the left or right of this new + * extent then we know we're going to have to allocate a new extent, so + * before we do that see if we need to drop this into a bitmap + */ + ret = insert_into_bitmap(ctl, info); + if (ret < 0) { + goto out; + } else if (ret) { + ret = 0; + goto out; + } +link: + /* + * Only steal free space from adjacent bitmaps if we're sure we're not + * going to add the new free space to existing bitmap entries - because + * that would mean unnecessary work that would be reverted. Therefore + * attempt to steal space from bitmaps if we're adding an extent entry. + */ + steal_from_bitmap(ctl, info, true); + + filter_bytes = max(filter_bytes, info->bytes); + + ret = link_free_space(ctl, info); + if (ret) + kmem_cache_free(btrfs_free_space_cachep, info); +out: + btrfs_discard_update_discardable(block_group); + spin_unlock(&ctl->tree_lock); + + if (ret) { + btrfs_crit(fs_info, "unable to add free space :%d", ret); + ASSERT(ret != -EEXIST); + } + + if (trim_state != BTRFS_TRIM_STATE_TRIMMED) { + btrfs_discard_check_filter(block_group, filter_bytes); + btrfs_discard_queue_work(&fs_info->discard_ctl, block_group); + } + + return ret; +} + +static int __btrfs_add_free_space_zoned(struct btrfs_block_group *block_group, + u64 bytenr, u64 size, bool used) +{ + struct btrfs_space_info *sinfo = block_group->space_info; + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + u64 offset = bytenr - block_group->start; + u64 to_free, to_unusable; + int bg_reclaim_threshold = 0; + bool initial; + u64 reclaimable_unusable; + + spin_lock(&block_group->lock); + + initial = ((size == block_group->length) && (block_group->alloc_offset == 0)); + WARN_ON(!initial && offset + size > block_group->zone_capacity); + if (!initial) + bg_reclaim_threshold = READ_ONCE(sinfo->bg_reclaim_threshold); + + if (!used) + to_free = size; + else if (initial) + to_free = block_group->zone_capacity; + else if (offset >= block_group->alloc_offset) + to_free = size; + else if (offset + size <= block_group->alloc_offset) + to_free = 0; + else + to_free = offset + size - block_group->alloc_offset; + to_unusable = size - to_free; + + spin_lock(&ctl->tree_lock); + ctl->free_space += to_free; + spin_unlock(&ctl->tree_lock); + /* + * If the block group is read-only, we should account freed space into + * bytes_readonly. + */ + if (!block_group->ro) { + block_group->zone_unusable += to_unusable; + WARN_ON(block_group->zone_unusable > block_group->length); + } + if (!used) { + block_group->alloc_offset -= size; + } + + reclaimable_unusable = block_group->zone_unusable - + (block_group->length - block_group->zone_capacity); + /* All the region is now unusable. Mark it as unused and reclaim */ + if (block_group->zone_unusable == block_group->length) { + btrfs_mark_bg_unused(block_group); + } else if (bg_reclaim_threshold && + reclaimable_unusable >= + mult_perc(block_group->zone_capacity, bg_reclaim_threshold)) { + btrfs_mark_bg_to_reclaim(block_group); + } + + spin_unlock(&block_group->lock); + + return 0; +} + +int btrfs_add_free_space(struct btrfs_block_group *block_group, + u64 bytenr, u64 size) +{ + enum btrfs_trim_state trim_state = BTRFS_TRIM_STATE_UNTRIMMED; + + if (block_group->flags & BTRFS_BLOCK_GROUP_REMAPPED) + return 0; + + if (btrfs_is_zoned(block_group->fs_info)) + return __btrfs_add_free_space_zoned(block_group, bytenr, size, + true); + + if (btrfs_test_opt(block_group->fs_info, DISCARD_SYNC)) + trim_state = BTRFS_TRIM_STATE_TRIMMED; + + return __btrfs_add_free_space(block_group, bytenr, size, trim_state); +} + +int btrfs_add_free_space_unused(struct btrfs_block_group *block_group, + u64 bytenr, u64 size) +{ + if (btrfs_is_zoned(block_group->fs_info)) + return __btrfs_add_free_space_zoned(block_group, bytenr, size, + false); + + return btrfs_add_free_space(block_group, bytenr, size); +} + +/* + * This is a subtle distinction because when adding free space back in general, + * we want it to be added as untrimmed for async. But in the case where we add + * it on loading of a block group, we want to consider it trimmed. + */ +int btrfs_add_free_space_async_trimmed(struct btrfs_block_group *block_group, + u64 bytenr, u64 size) +{ + enum btrfs_trim_state trim_state = BTRFS_TRIM_STATE_UNTRIMMED; + + if (btrfs_is_zoned(block_group->fs_info)) + return __btrfs_add_free_space_zoned(block_group, bytenr, size, + true); + + if (btrfs_test_opt(block_group->fs_info, DISCARD_SYNC) || + btrfs_test_opt(block_group->fs_info, DISCARD_ASYNC)) + trim_state = BTRFS_TRIM_STATE_TRIMMED; + + return __btrfs_add_free_space(block_group, bytenr, size, trim_state); +} + +int btrfs_remove_free_space(struct btrfs_block_group *block_group, + u64 offset, u64 bytes) +{ + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + struct btrfs_free_space *info; + int ret; + bool re_search = false; + + if (btrfs_is_zoned(block_group->fs_info)) { + /* + * This can happen with conventional zones when replaying log. + * Since the allocation info of tree-log nodes are not recorded + * to the extent-tree, calculate_alloc_pointer() failed to + * advance the allocation pointer after last allocated tree log + * node blocks. + * + * This function is called from + * btrfs_pin_extent_for_log_replay() when replaying the log. + * Advance the pointer not to overwrite the tree-log nodes. + */ + if (block_group->start + block_group->alloc_offset < + offset + bytes) { + block_group->alloc_offset = + offset + bytes - block_group->start; + } + return 0; + } + + spin_lock(&ctl->tree_lock); + +again: + ret = 0; + if (!bytes) + goto out_lock; + + info = tree_search_offset(ctl, offset, 0, 0); + if (!info) { + /* + * oops didn't find an extent that matched the space we wanted + * to remove, look for a bitmap instead + */ + info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), + 1, 0); + if (!info) { + /* + * If we found a partial bit of our free space in a + * bitmap but then couldn't find the other part this may + * be a problem, so WARN about it. + */ + WARN_ON(re_search); + goto out_lock; + } + } + + re_search = false; + if (!info->bitmap) { + unlink_free_space(ctl, info, true); + if (offset == info->offset) { + u64 to_free = min(bytes, info->bytes); + + info->bytes -= to_free; + info->offset += to_free; + if (info->bytes) { + ret = link_free_space(ctl, info); + WARN_ON(ret); + } else { + kmem_cache_free(btrfs_free_space_cachep, info); + } + + offset += to_free; + bytes -= to_free; + goto again; + } else { + u64 old_end = info->bytes + info->offset; + + info->bytes = offset - info->offset; + ret = link_free_space(ctl, info); + WARN_ON(ret); + if (ret) + goto out_lock; + + /* Not enough bytes in this entry to satisfy us */ + if (old_end < offset + bytes) { + bytes -= old_end - offset; + offset = old_end; + goto again; + } else if (old_end == offset + bytes) { + /* all done */ + goto out_lock; + } + spin_unlock(&ctl->tree_lock); + + ret = __btrfs_add_free_space(block_group, + offset + bytes, + old_end - (offset + bytes), + info->trim_state); + WARN_ON(ret); + return ret; + } + } + + ret = remove_from_bitmap(ctl, info, &offset, &bytes); + if (ret == -EAGAIN) { + re_search = true; + goto again; + } +out_lock: + btrfs_discard_update_discardable(block_group); + spin_unlock(&ctl->tree_lock); + + return ret; +} + +void btrfs_dump_free_space(struct btrfs_block_group *block_group, + u64 bytes) +{ + struct btrfs_fs_info *fs_info = block_group->fs_info; + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + struct btrfs_free_space *info; + struct rb_node *n; + int count = 0; + + /* + * Zoned btrfs does not use free space tree and cluster. Just print + * out the free space after the allocation offset. + */ + if (btrfs_is_zoned(fs_info)) { + btrfs_info(fs_info, "free space %llu active %d", + block_group->zone_capacity - block_group->alloc_offset, + test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, + &block_group->runtime_flags)); + return; + } + + spin_lock(&ctl->tree_lock); + for (n = rb_first(&ctl->free_space_offset); n; n = rb_next(n)) { + info = rb_entry(n, struct btrfs_free_space, offset_index); + if (info->bytes >= bytes && !block_group->ro) + count++; + btrfs_crit(fs_info, "entry offset %llu, bytes %llu, bitmap %s", + info->offset, info->bytes, str_yes_no(info->bitmap)); + } + spin_unlock(&ctl->tree_lock); + btrfs_info(fs_info, "block group has cluster?: %s", + str_no_yes(list_empty(&block_group->cluster_list))); + btrfs_info(fs_info, + "%d free space entries at or bigger than %llu bytes", + count, bytes); +} + +void btrfs_init_free_space_ctl(struct btrfs_block_group *block_group, + struct btrfs_free_space_ctl *ctl) +{ + spin_lock_init(&ctl->tree_lock); + ctl->block_group = block_group; + ctl->free_space_bytes = RB_ROOT_CACHED; + INIT_LIST_HEAD(&ctl->trimming_ranges); + mutex_init(&ctl->cache_writeout_mutex); + + /* + * we only want to have 32k of ram per block group for keeping + * track of free space, and if we pass 1/2 of that we want to + * start converting things over to using bitmaps + */ + ctl->extents_thresh = (SZ_32K / 2) / sizeof(struct btrfs_free_space); +} + +/* + * for a given cluster, put all of its extents back into the free + * space cache. If the block group passed doesn't match the block group + * pointed to by the cluster, someone else raced in and freed the + * cluster already. In that case, we just return without changing anything + */ +static void __btrfs_return_cluster_to_free_space( + struct btrfs_block_group *block_group, + struct btrfs_free_cluster *cluster) +{ + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + struct rb_node *node; + + lockdep_assert_held(&ctl->tree_lock); + + spin_lock(&cluster->lock); + if (cluster->block_group != block_group) { + spin_unlock(&cluster->lock); + return; + } + + cluster->block_group = NULL; + cluster->window_start = 0; + list_del_init(&cluster->block_group_list); + + node = rb_first(&cluster->root); + while (node) { + struct btrfs_free_space *entry; + + entry = rb_entry(node, struct btrfs_free_space, offset_index); + node = rb_next(&entry->offset_index); + rb_erase(&entry->offset_index, &cluster->root); + RB_CLEAR_NODE(&entry->offset_index); + + if (!entry->bitmap) { + /* Merging treats extents as if they were new */ + if (!btrfs_free_space_trimmed(entry)) { + ctl->discardable_extents[BTRFS_STAT_CURR]--; + ctl->discardable_bytes[BTRFS_STAT_CURR] -= + entry->bytes; + } + + try_merge_free_space(ctl, entry, false); + steal_from_bitmap(ctl, entry, false); + + /* As we insert directly, update these statistics */ + if (!btrfs_free_space_trimmed(entry)) { + ctl->discardable_extents[BTRFS_STAT_CURR]++; + ctl->discardable_bytes[BTRFS_STAT_CURR] += + entry->bytes; + } + } + tree_insert_offset(ctl, NULL, entry); + rb_add_cached(&entry->bytes_index, &ctl->free_space_bytes, + entry_less); + } + cluster->root = RB_ROOT; + spin_unlock(&cluster->lock); + btrfs_put_block_group(block_group); +} + +void btrfs_remove_free_space_cache(struct btrfs_block_group *block_group) +{ + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + struct btrfs_free_cluster *cluster; + struct list_head *head; + + spin_lock(&ctl->tree_lock); + while ((head = block_group->cluster_list.next) != + &block_group->cluster_list) { + cluster = list_entry(head, struct btrfs_free_cluster, + block_group_list); + + WARN_ON(cluster->block_group != block_group); + __btrfs_return_cluster_to_free_space(block_group, cluster); + + cond_resched_lock(&ctl->tree_lock); + } + __btrfs_remove_free_space_cache(ctl); + btrfs_discard_update_discardable(block_group); + spin_unlock(&ctl->tree_lock); + +} + +/* + * Walk @block_group's free space rb_tree to determine if everything is trimmed. + */ +bool btrfs_is_free_space_trimmed(struct btrfs_block_group *block_group) +{ + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + struct btrfs_free_space *info; + struct rb_node *node; + bool ret = true; + + if (block_group->flags & BTRFS_BLOCK_GROUP_REMAPPED && + !test_bit(BLOCK_GROUP_FLAG_STRIPE_REMOVAL_PENDING, &block_group->runtime_flags) && + block_group->identity_remap_count == 0) { + return true; + } + + spin_lock(&ctl->tree_lock); + node = rb_first(&ctl->free_space_offset); + + while (node) { + info = rb_entry(node, struct btrfs_free_space, offset_index); + + if (!btrfs_free_space_trimmed(info)) { + ret = false; + break; + } + + node = rb_next(node); + } + + spin_unlock(&ctl->tree_lock); + return ret; +} + +u64 btrfs_find_space_for_alloc(struct btrfs_block_group *block_group, + u64 offset, u64 bytes, u64 empty_size, + u64 *max_extent_size) +{ + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + struct btrfs_discard_ctl *discard_ctl = + &block_group->fs_info->discard_ctl; + struct btrfs_free_space *entry = NULL; + u64 bytes_search = bytes + empty_size; + u64 ret = 0; + u64 align_gap = 0; + u64 align_gap_len = 0; + enum btrfs_trim_state align_gap_trim_state = BTRFS_TRIM_STATE_UNTRIMMED; + bool use_bytes_index = (offset == block_group->start); + + ASSERT(!btrfs_is_zoned(block_group->fs_info)); + + spin_lock(&ctl->tree_lock); + entry = find_free_space(ctl, &offset, &bytes_search, + block_group->full_stripe_len, max_extent_size, + use_bytes_index); + if (!entry) + goto out; + + ret = offset; + if (entry->bitmap) { + bitmap_clear_bits(ctl, entry, offset, bytes, true); + + if (!btrfs_free_space_trimmed(entry)) + atomic64_add(bytes, &discard_ctl->discard_bytes_saved); + + if (!entry->bytes) + free_bitmap(ctl, entry); + } else { + unlink_free_space(ctl, entry, true); + align_gap_len = offset - entry->offset; + align_gap = entry->offset; + align_gap_trim_state = entry->trim_state; + + if (!btrfs_free_space_trimmed(entry)) + atomic64_add(bytes, &discard_ctl->discard_bytes_saved); + + entry->offset = offset + bytes; + WARN_ON(entry->bytes < bytes + align_gap_len); + + entry->bytes -= bytes + align_gap_len; + if (!entry->bytes) + kmem_cache_free(btrfs_free_space_cachep, entry); + else + link_free_space(ctl, entry); + } +out: + btrfs_discard_update_discardable(block_group); + spin_unlock(&ctl->tree_lock); + + if (align_gap_len) + __btrfs_add_free_space(block_group, align_gap, align_gap_len, + align_gap_trim_state); + return ret; +} + +/* + * given a cluster, put all of its extents back into the free space + * cache. If a block group is passed, this function will only free + * a cluster that belongs to the passed block group. + * + * Otherwise, it'll get a reference on the block group pointed to by the + * cluster and remove the cluster from it. + */ +void btrfs_return_cluster_to_free_space( + struct btrfs_block_group *block_group, + struct btrfs_free_cluster *cluster) +{ + struct btrfs_free_space_ctl *ctl; + + /* first, get a safe pointer to the block group */ + spin_lock(&cluster->lock); + if (!block_group) { + block_group = cluster->block_group; + if (!block_group) { + spin_unlock(&cluster->lock); + return; + } + } else if (cluster->block_group != block_group) { + /* someone else has already freed it don't redo their work */ + spin_unlock(&cluster->lock); + return; + } + btrfs_get_block_group(block_group); + spin_unlock(&cluster->lock); + + ctl = block_group->free_space_ctl; + + /* now return any extents the cluster had on it */ + spin_lock(&ctl->tree_lock); + __btrfs_return_cluster_to_free_space(block_group, cluster); + spin_unlock(&ctl->tree_lock); + + btrfs_discard_queue_work(&block_group->fs_info->discard_ctl, block_group); + + /* finally drop our ref */ + btrfs_put_block_group(block_group); +} + +static u64 btrfs_alloc_from_bitmap(struct btrfs_block_group *block_group, + struct btrfs_free_cluster *cluster, + struct btrfs_free_space *entry, + u64 bytes, u64 min_start, + u64 *max_extent_size) +{ + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + int ret2; + u64 search_start = cluster->window_start; + u64 search_bytes = bytes; + u64 ret = 0; + + search_start = min_start; + search_bytes = bytes; + + ret2 = search_bitmap(ctl, entry, &search_start, &search_bytes, true); + if (ret2) { + *max_extent_size = max(get_max_extent_size(entry), + *max_extent_size); + return 0; + } + + ret = search_start; + bitmap_clear_bits(ctl, entry, ret, bytes, false); + + return ret; +} + +/* + * given a cluster, try to allocate 'bytes' from it, returns 0 + * if it couldn't find anything suitably large, or a logical disk offset + * if things worked out + */ +u64 btrfs_alloc_from_cluster(struct btrfs_block_group *block_group, + struct btrfs_free_cluster *cluster, u64 bytes, + u64 min_start, u64 *max_extent_size) +{ + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + struct btrfs_discard_ctl *discard_ctl = + &block_group->fs_info->discard_ctl; + struct btrfs_free_space *entry = NULL; + struct rb_node *node; + u64 ret = 0; + + ASSERT(!btrfs_is_zoned(block_group->fs_info)); + + spin_lock(&cluster->lock); + if (bytes > cluster->max_size) + goto out; + + if (cluster->block_group != block_group) + goto out; + + node = rb_first(&cluster->root); + if (!node) + goto out; + + entry = rb_entry(node, struct btrfs_free_space, offset_index); + while (1) { + if (entry->bytes < bytes) + *max_extent_size = max(get_max_extent_size(entry), + *max_extent_size); + + if (entry->bytes < bytes || + (!entry->bitmap && entry->offset < min_start)) { + node = rb_next(&entry->offset_index); + if (!node) + break; + entry = rb_entry(node, struct btrfs_free_space, + offset_index); + continue; + } + + if (entry->bitmap) { + ret = btrfs_alloc_from_bitmap(block_group, + cluster, entry, bytes, + cluster->window_start, + max_extent_size); + if (ret == 0) { + node = rb_next(&entry->offset_index); + if (!node) + break; + entry = rb_entry(node, struct btrfs_free_space, + offset_index); + continue; + } + cluster->window_start += bytes; + } else { + ret = entry->offset; + + entry->offset += bytes; + entry->bytes -= bytes; + } + + break; + } +out: + spin_unlock(&cluster->lock); + + if (!ret) + return 0; + + spin_lock(&ctl->tree_lock); + + if (!btrfs_free_space_trimmed(entry)) + atomic64_add(bytes, &discard_ctl->discard_bytes_saved); + + ctl->free_space -= bytes; + if (!entry->bitmap && !btrfs_free_space_trimmed(entry)) + ctl->discardable_bytes[BTRFS_STAT_CURR] -= bytes; + + spin_lock(&cluster->lock); + if (entry->bytes == 0) { + rb_erase(&entry->offset_index, &cluster->root); + ctl->free_extents--; + if (entry->bitmap) { + kmem_cache_free(btrfs_free_space_bitmap_cachep, + entry->bitmap); + ctl->total_bitmaps--; + recalculate_thresholds(ctl); + } else if (!btrfs_free_space_trimmed(entry)) { + ctl->discardable_extents[BTRFS_STAT_CURR]--; + } + kmem_cache_free(btrfs_free_space_cachep, entry); + } + + spin_unlock(&cluster->lock); + spin_unlock(&ctl->tree_lock); + + return ret; +} + +static int btrfs_bitmap_cluster(struct btrfs_block_group *block_group, + struct btrfs_free_space *entry, + struct btrfs_free_cluster *cluster, + u64 offset, u64 bytes, + u64 cont1_bytes, u64 min_bytes) +{ + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + const int unit = block_group->fs_info->sectorsize; + unsigned long next_zero; + unsigned long i; + unsigned long want_bits; + unsigned long min_bits; + unsigned long found_bits; + unsigned long max_bits = 0; + unsigned long start = 0; + unsigned long total_found = 0; + int ret; + + lockdep_assert_held(&ctl->tree_lock); + + i = offset_to_bit(entry->offset, unit, + max_t(u64, offset, entry->offset)); + want_bits = bytes_to_bits(bytes, unit); + min_bits = bytes_to_bits(min_bytes, unit); + + /* + * Don't bother looking for a cluster in this bitmap if it's heavily + * fragmented. + */ + if (entry->max_extent_size && + entry->max_extent_size < cont1_bytes) + return -ENOSPC; +again: + found_bits = 0; + for_each_set_bit_from(i, entry->bitmap, BITS_PER_BITMAP) { + next_zero = find_next_zero_bit(entry->bitmap, + BITS_PER_BITMAP, i); + if (next_zero - i >= min_bits) { + found_bits = next_zero - i; + if (found_bits > max_bits) + max_bits = found_bits; + break; + } + if (next_zero - i > max_bits) + max_bits = next_zero - i; + i = next_zero; + } + + if (!found_bits) { + entry->max_extent_size = (u64)max_bits * unit; + return -ENOSPC; + } + + if (!total_found) { + start = i; + cluster->max_size = 0; + } + + total_found += found_bits; + + if (cluster->max_size < found_bits * unit) + cluster->max_size = found_bits * unit; + + if (total_found < want_bits || cluster->max_size < cont1_bytes) { + i = next_zero + 1; + goto again; + } + + cluster->window_start = start * unit + entry->offset; + rb_erase(&entry->offset_index, &ctl->free_space_offset); + rb_erase_cached(&entry->bytes_index, &ctl->free_space_bytes); + + /* + * We need to know if we're currently on the normal space index when we + * manipulate the bitmap so that we know we need to remove and re-insert + * it into the space_index tree. Clear the bytes_index node here so the + * bitmap manipulation helpers know not to mess with the space_index + * until this bitmap entry is added back into the normal cache. + */ + RB_CLEAR_NODE(&entry->bytes_index); + + ret = tree_insert_offset(ctl, cluster, entry); + ASSERT(!ret); /* -EEXIST; Logic error */ + + trace_btrfs_setup_cluster(block_group, cluster, total_found * unit, 1); + return 0; +} + +/* + * This searches the block group for just extents to fill the cluster with. + * Try to find a cluster with at least bytes total bytes, at least one + * extent of cont1_bytes, and other clusters of at least min_bytes. + */ +static noinline int +setup_cluster_no_bitmap(struct btrfs_block_group *block_group, + struct btrfs_free_cluster *cluster, + struct list_head *bitmaps, u64 offset, u64 bytes, + u64 cont1_bytes, u64 min_bytes) +{ + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + struct btrfs_free_space *first = NULL; + struct btrfs_free_space *entry = NULL; + struct btrfs_free_space *last; + struct rb_node *node; + u64 window_free; + u64 max_extent; + u64 total_size = 0; + + lockdep_assert_held(&ctl->tree_lock); + + entry = tree_search_offset(ctl, offset, 0, 1); + if (!entry) + return -ENOSPC; + + /* + * We don't want bitmaps, so just move along until we find a normal + * extent entry. + */ + while (entry->bitmap || entry->bytes < min_bytes) { + if (entry->bitmap && list_empty(&entry->list)) + list_add_tail(&entry->list, bitmaps); + node = rb_next(&entry->offset_index); + if (!node) + return -ENOSPC; + entry = rb_entry(node, struct btrfs_free_space, offset_index); + } + + window_free = entry->bytes; + max_extent = entry->bytes; + first = entry; + last = entry; + + for (node = rb_next(&entry->offset_index); node; + node = rb_next(&entry->offset_index)) { + entry = rb_entry(node, struct btrfs_free_space, offset_index); + + if (entry->bitmap) { + if (list_empty(&entry->list)) + list_add_tail(&entry->list, bitmaps); + continue; + } + + if (entry->bytes < min_bytes) + continue; + + last = entry; + window_free += entry->bytes; + if (entry->bytes > max_extent) + max_extent = entry->bytes; + } + + if (window_free < bytes || max_extent < cont1_bytes) + return -ENOSPC; + + cluster->window_start = first->offset; + + node = &first->offset_index; + + /* + * now we've found our entries, pull them out of the free space + * cache and put them into the cluster rbtree + */ + do { + int ret; + + entry = rb_entry(node, struct btrfs_free_space, offset_index); + node = rb_next(&entry->offset_index); + if (entry->bitmap || entry->bytes < min_bytes) + continue; + + rb_erase(&entry->offset_index, &ctl->free_space_offset); + rb_erase_cached(&entry->bytes_index, &ctl->free_space_bytes); + ret = tree_insert_offset(ctl, cluster, entry); + total_size += entry->bytes; + ASSERT(!ret); /* -EEXIST; Logic error */ + } while (node && entry != last); + + cluster->max_size = max_extent; + trace_btrfs_setup_cluster(block_group, cluster, total_size, 0); + return 0; +} + +/* + * This specifically looks for bitmaps that may work in the cluster, we assume + * that we have already failed to find extents that will work. + */ +static noinline int +setup_cluster_bitmap(struct btrfs_block_group *block_group, + struct btrfs_free_cluster *cluster, + struct list_head *bitmaps, u64 offset, u64 bytes, + u64 cont1_bytes, u64 min_bytes) +{ + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + struct btrfs_free_space *entry = NULL; + int ret = -ENOSPC; + u64 bitmap_offset = offset_to_bitmap(ctl, offset); + + if (ctl->total_bitmaps == 0) + return -ENOSPC; + + /* + * The bitmap that covers offset won't be in the list unless offset + * is just its start offset. + */ + if (!list_empty(bitmaps)) + entry = list_first_entry(bitmaps, struct btrfs_free_space, list); + + if (!entry || entry->offset != bitmap_offset) { + entry = tree_search_offset(ctl, bitmap_offset, 1, 0); + if (entry && list_empty(&entry->list)) + list_add(&entry->list, bitmaps); + } + + list_for_each_entry(entry, bitmaps, list) { + if (entry->bytes < bytes) + continue; + ret = btrfs_bitmap_cluster(block_group, entry, cluster, offset, + bytes, cont1_bytes, min_bytes); + if (!ret) + return 0; + } + + /* + * The bitmaps list has all the bitmaps that record free space + * starting after offset, so no more search is required. + */ + return -ENOSPC; +} + +/* + * here we try to find a cluster of blocks in a block group. The goal + * is to find at least bytes+empty_size. + * We might not find them all in one contiguous area. + * + * returns zero and sets up cluster if things worked out, otherwise + * it returns -enospc + */ +int btrfs_find_space_cluster(struct btrfs_block_group *block_group, + struct btrfs_free_cluster *cluster, + u64 offset, u64 bytes, u64 empty_size) +{ + struct btrfs_fs_info *fs_info = block_group->fs_info; + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + struct btrfs_free_space *entry, *tmp; + LIST_HEAD(bitmaps); + u64 min_bytes; + u64 cont1_bytes; + int ret; + + /* + * Choose the minimum extent size we'll require for this + * cluster. For SSD_SPREAD, don't allow any fragmentation. + * For metadata, allow allocates with smaller extents. For + * data, keep it dense. + */ + if (btrfs_test_opt(fs_info, SSD_SPREAD)) { + cont1_bytes = bytes + empty_size; + min_bytes = cont1_bytes; + } else if (block_group->flags & BTRFS_BLOCK_GROUP_METADATA) { + cont1_bytes = bytes; + min_bytes = fs_info->sectorsize; + } else { + cont1_bytes = max(bytes, (bytes + empty_size) >> 2); + min_bytes = fs_info->sectorsize; + } + + spin_lock(&ctl->tree_lock); + + /* + * If we know we don't have enough space to make a cluster don't even + * bother doing all the work to try and find one. + */ + if (ctl->free_space < bytes) { + spin_unlock(&ctl->tree_lock); + return -ENOSPC; + } + + spin_lock(&cluster->lock); + + /* someone already found a cluster, hooray */ + if (cluster->block_group) { + ret = 0; + goto out; + } + + trace_btrfs_find_cluster(block_group, offset, bytes, empty_size, + min_bytes); + + ret = setup_cluster_no_bitmap(block_group, cluster, &bitmaps, offset, + bytes + empty_size, + cont1_bytes, min_bytes); + if (ret) + ret = setup_cluster_bitmap(block_group, cluster, &bitmaps, + offset, bytes + empty_size, + cont1_bytes, min_bytes); + + /* Clear our temporary list */ + list_for_each_entry_safe(entry, tmp, &bitmaps, list) + list_del_init(&entry->list); + + if (!ret) { + btrfs_get_block_group(block_group); + list_add_tail(&cluster->block_group_list, + &block_group->cluster_list); + cluster->block_group = block_group; + } else { + trace_btrfs_failed_cluster_setup(block_group); + } +out: + spin_unlock(&cluster->lock); + spin_unlock(&ctl->tree_lock); + + return ret; +} + +/* + * simple code to zero out a cluster + */ +void btrfs_init_free_cluster(struct btrfs_free_cluster *cluster) +{ + spin_lock_init(&cluster->lock); + spin_lock_init(&cluster->refill_lock); + cluster->root = RB_ROOT; + cluster->max_size = 0; + cluster->fragmented = false; + INIT_LIST_HEAD(&cluster->block_group_list); + cluster->block_group = NULL; +} + +static int do_trimming(struct btrfs_block_group *block_group, + u64 *total_trimmed, u64 start, u64 bytes, + u64 reserved_start, u64 reserved_bytes, + enum btrfs_trim_state reserved_trim_state, + struct btrfs_trim_range *trim_entry) +{ + struct btrfs_space_info *space_info = block_group->space_info; + struct btrfs_fs_info *fs_info = block_group->fs_info; + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + int ret; + bool bg_ro; + const u64 end = start + bytes; + const u64 reserved_end = reserved_start + reserved_bytes; + enum btrfs_trim_state trim_state = BTRFS_TRIM_STATE_UNTRIMMED; + u64 trimmed = 0; + + spin_lock(&space_info->lock); + spin_lock(&block_group->lock); + bg_ro = block_group->ro; + if (!bg_ro) { + block_group->reserved += reserved_bytes; + spin_unlock(&block_group->lock); + space_info->bytes_reserved += reserved_bytes; + } else { + spin_unlock(&block_group->lock); + } + spin_unlock(&space_info->lock); + + ret = btrfs_discard_extent(fs_info, start, bytes, &trimmed, false); + if (!ret) { + *total_trimmed += trimmed; + trim_state = BTRFS_TRIM_STATE_TRIMMED; + } + + mutex_lock(&ctl->cache_writeout_mutex); + if (reserved_start < start) + __btrfs_add_free_space(block_group, reserved_start, + start - reserved_start, + reserved_trim_state); + if (end < reserved_end) + __btrfs_add_free_space(block_group, end, reserved_end - end, + reserved_trim_state); + __btrfs_add_free_space(block_group, start, bytes, trim_state); + list_del(&trim_entry->list); + mutex_unlock(&ctl->cache_writeout_mutex); + + if (!bg_ro) { + spin_lock(&space_info->lock); + spin_lock(&block_group->lock); + bg_ro = block_group->ro; + block_group->reserved -= reserved_bytes; + spin_unlock(&block_group->lock); + + space_info->bytes_reserved -= reserved_bytes; + if (bg_ro) + space_info->bytes_readonly += reserved_bytes; + spin_unlock(&space_info->lock); + } + + return ret; +} + +/* + * If @async is set, then we will trim 1 region and return. + */ +static int trim_no_bitmap(struct btrfs_block_group *block_group, + u64 *total_trimmed, u64 start, u64 end, u64 minlen, + bool async) +{ + struct btrfs_discard_ctl *discard_ctl = + &block_group->fs_info->discard_ctl; + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + struct btrfs_free_space *entry; + struct rb_node *node; + int ret = 0; + u64 extent_start; + u64 extent_bytes; + enum btrfs_trim_state extent_trim_state; + u64 bytes; + const u64 max_discard_size = READ_ONCE(discard_ctl->max_discard_size); + + while (start < end) { + struct btrfs_trim_range trim_entry; + + mutex_lock(&ctl->cache_writeout_mutex); + spin_lock(&ctl->tree_lock); + + if (ctl->free_space < minlen) + goto out_unlock; + + entry = tree_search_offset(ctl, start, 0, 1); + if (!entry) + goto out_unlock; + + /* Skip bitmaps and if async, already trimmed entries */ + while (entry->bitmap || + (async && btrfs_free_space_trimmed(entry))) { + node = rb_next(&entry->offset_index); + if (!node) + goto out_unlock; + entry = rb_entry(node, struct btrfs_free_space, + offset_index); + } + + if (entry->offset >= end) + goto out_unlock; + + extent_start = entry->offset; + extent_bytes = entry->bytes; + extent_trim_state = entry->trim_state; + if (async) { + start = entry->offset; + bytes = entry->bytes; + if (bytes < minlen) { + spin_unlock(&ctl->tree_lock); + mutex_unlock(&ctl->cache_writeout_mutex); + goto next; + } + unlink_free_space(ctl, entry, true); + /* + * Let bytes = BTRFS_MAX_DISCARD_SIZE + X. + * If X < BTRFS_ASYNC_DISCARD_MIN_FILTER, we won't trim + * X when we come back around. So trim it now. + */ + if (max_discard_size && + bytes >= (max_discard_size + + BTRFS_ASYNC_DISCARD_MIN_FILTER)) { + bytes = max_discard_size; + extent_bytes = max_discard_size; + entry->offset += max_discard_size; + entry->bytes -= max_discard_size; + link_free_space(ctl, entry); + } else { + kmem_cache_free(btrfs_free_space_cachep, entry); + } + } else { + start = max(start, extent_start); + bytes = min(extent_start + extent_bytes, end) - start; + if (bytes < minlen) { + spin_unlock(&ctl->tree_lock); + mutex_unlock(&ctl->cache_writeout_mutex); + goto next; + } + + unlink_free_space(ctl, entry, true); + kmem_cache_free(btrfs_free_space_cachep, entry); + } + + spin_unlock(&ctl->tree_lock); + trim_entry.start = extent_start; + trim_entry.bytes = extent_bytes; + list_add_tail(&trim_entry.list, &ctl->trimming_ranges); + mutex_unlock(&ctl->cache_writeout_mutex); + + ret = do_trimming(block_group, total_trimmed, start, bytes, + extent_start, extent_bytes, extent_trim_state, + &trim_entry); + if (ret) { + block_group->discard_cursor = start + bytes; + break; + } +next: + start += bytes; + block_group->discard_cursor = start; + if (async && *total_trimmed) + break; + + if (btrfs_trim_interrupted()) { + ret = -ERESTARTSYS; + break; + } + + cond_resched(); + } + + return ret; + +out_unlock: + block_group->discard_cursor = btrfs_block_group_end(block_group); + spin_unlock(&ctl->tree_lock); + mutex_unlock(&ctl->cache_writeout_mutex); + + return ret; +} + +void btrfs_trim_fully_remapped_block_group(struct btrfs_block_group *bg) +{ + struct btrfs_fs_info *fs_info = bg->fs_info; + struct btrfs_discard_ctl *discard_ctl = &fs_info->discard_ctl; + int ret = 0; + u64 bytes, trimmed; + const u64 max_discard_size = READ_ONCE(discard_ctl->max_discard_size); + u64 end = btrfs_block_group_end(bg); + + if (!test_bit(BLOCK_GROUP_FLAG_STRIPE_REMOVAL_PENDING, &bg->runtime_flags)) { + bg->discard_cursor = end; + + if (bg->used == 0) { + spin_lock(&fs_info->unused_bgs_lock); + if (!list_empty(&bg->bg_list)) { + list_del_init(&bg->bg_list); + btrfs_put_block_group(bg); + } + spin_unlock(&fs_info->unused_bgs_lock); + + btrfs_mark_bg_unused(bg); + } + + return; + } + + bytes = end - bg->discard_cursor; + + if (max_discard_size && + bytes >= (max_discard_size + BTRFS_ASYNC_DISCARD_MIN_FILTER)) + bytes = max_discard_size; + + ret = btrfs_discard_extent(fs_info, bg->discard_cursor, bytes, &trimmed, false); + if (ret) + return; + + bg->discard_cursor += trimmed; + + if (bg->discard_cursor < end) + return; + + btrfs_complete_bg_remapping(bg); +} + +/* + * If we break out of trimming a bitmap prematurely, we should reset the + * trimming bit. In a rather contrived case, it's possible to race here so + * reset the state to BTRFS_TRIM_STATE_UNTRIMMED. + * + * start = start of bitmap + * end = near end of bitmap + * + * Thread 1: Thread 2: + * trim_bitmaps(start) + * trim_bitmaps(end) + * end_trimming_bitmap() + * reset_trimming_bitmap() + */ +static void reset_trimming_bitmap(struct btrfs_free_space_ctl *ctl, u64 offset) +{ + struct btrfs_free_space *entry; + + spin_lock(&ctl->tree_lock); + entry = tree_search_offset(ctl, offset, 1, 0); + if (entry) { + if (btrfs_free_space_trimmed(entry)) { + ctl->discardable_extents[BTRFS_STAT_CURR] += + entry->bitmap_extents; + ctl->discardable_bytes[BTRFS_STAT_CURR] += entry->bytes; + } + entry->trim_state = BTRFS_TRIM_STATE_UNTRIMMED; + } + + spin_unlock(&ctl->tree_lock); +} + +static void end_trimming_bitmap(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *entry) +{ + if (btrfs_free_space_trimming_bitmap(entry)) { + entry->trim_state = BTRFS_TRIM_STATE_TRIMMED; + ctl->discardable_extents[BTRFS_STAT_CURR] -= + entry->bitmap_extents; + ctl->discardable_bytes[BTRFS_STAT_CURR] -= entry->bytes; + } +} + +/* + * If @async is set, then we will trim 1 region and return. + */ +static int trim_bitmaps(struct btrfs_block_group *block_group, + u64 *total_trimmed, u64 start, u64 end, u64 minlen, + u64 maxlen, bool async) +{ + struct btrfs_discard_ctl *discard_ctl = + &block_group->fs_info->discard_ctl; + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + struct btrfs_free_space *entry; + int ret = 0; + int ret2; + u64 bytes; + u64 offset = offset_to_bitmap(ctl, start); + const u64 max_discard_size = READ_ONCE(discard_ctl->max_discard_size); + + while (offset < end) { + bool next_bitmap = false; + struct btrfs_trim_range trim_entry; + + mutex_lock(&ctl->cache_writeout_mutex); + spin_lock(&ctl->tree_lock); + + if (ctl->free_space < minlen) { + block_group->discard_cursor = + btrfs_block_group_end(block_group); + spin_unlock(&ctl->tree_lock); + mutex_unlock(&ctl->cache_writeout_mutex); + break; + } + + entry = tree_search_offset(ctl, offset, 1, 0); + /* + * Bitmaps are marked trimmed lossily now to prevent constant + * discarding of the same bitmap (the reason why we are bound + * by the filters). So, retrim the block group bitmaps when we + * are preparing to punt to the unused_bgs list. This uses + * @minlen to determine if we are in BTRFS_DISCARD_INDEX_UNUSED + * which is the only discard index which sets minlen to 0. + */ + if (!entry || (async && minlen && start == offset && + btrfs_free_space_trimmed(entry))) { + spin_unlock(&ctl->tree_lock); + mutex_unlock(&ctl->cache_writeout_mutex); + next_bitmap = true; + goto next; + } + + /* + * Async discard bitmap trimming begins at by setting the start + * to be key.objectid and the offset_to_bitmap() aligns to the + * start of the bitmap. This lets us know we are fully + * scanning the bitmap rather than only some portion of it. + */ + if (start == offset) + entry->trim_state = BTRFS_TRIM_STATE_TRIMMING; + + bytes = minlen; + ret2 = search_bitmap(ctl, entry, &start, &bytes, false); + if (ret2 || start >= end) { + /* + * We lossily consider a bitmap trimmed if we only skip + * over regions <= BTRFS_ASYNC_DISCARD_MIN_FILTER. + */ + if (ret2 && minlen <= BTRFS_ASYNC_DISCARD_MIN_FILTER) + end_trimming_bitmap(ctl, entry); + else + entry->trim_state = BTRFS_TRIM_STATE_UNTRIMMED; + spin_unlock(&ctl->tree_lock); + mutex_unlock(&ctl->cache_writeout_mutex); + next_bitmap = true; + goto next; + } + + /* + * We already trimmed a region, but are using the locking above + * to reset the trim_state. + */ + if (async && *total_trimmed) { + spin_unlock(&ctl->tree_lock); + mutex_unlock(&ctl->cache_writeout_mutex); + return ret; + } + + bytes = min(bytes, end - start); + if (bytes < minlen || (async && maxlen && bytes > maxlen)) { + spin_unlock(&ctl->tree_lock); + mutex_unlock(&ctl->cache_writeout_mutex); + goto next; + } + + /* + * Let bytes = BTRFS_MAX_DISCARD_SIZE + X. + * If X < @minlen, we won't trim X when we come back around. + * So trim it now. We differ here from trimming extents as we + * don't keep individual state per bit. + */ + if (async && + max_discard_size && + bytes > (max_discard_size + minlen)) + bytes = max_discard_size; + + bitmap_clear_bits(ctl, entry, start, bytes, true); + if (entry->bytes == 0) + free_bitmap(ctl, entry); + + spin_unlock(&ctl->tree_lock); + trim_entry.start = start; + trim_entry.bytes = bytes; + list_add_tail(&trim_entry.list, &ctl->trimming_ranges); + mutex_unlock(&ctl->cache_writeout_mutex); + + ret = do_trimming(block_group, total_trimmed, start, bytes, + start, bytes, 0, &trim_entry); + if (ret) { + reset_trimming_bitmap(ctl, offset); + block_group->discard_cursor = + btrfs_block_group_end(block_group); + break; + } +next: + if (next_bitmap) { + const int unit = block_group->fs_info->sectorsize; + + offset += BITS_PER_BITMAP * unit; + start = offset; + } else { + start += bytes; + } + block_group->discard_cursor = start; + + if (btrfs_trim_interrupted()) { + if (start != offset) + reset_trimming_bitmap(ctl, offset); + ret = -ERESTARTSYS; + break; + } + + cond_resched(); + } + + if (offset >= end) + block_group->discard_cursor = end; + + return ret; +} + +int btrfs_trim_block_group(struct btrfs_block_group *block_group, + u64 *trimmed, u64 start, u64 end, u64 minlen) +{ + struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; + const int unit = block_group->fs_info->sectorsize; + int ret; + u64 rem = 0; + + ASSERT(!btrfs_is_zoned(block_group->fs_info)); + + *trimmed = 0; + + spin_lock(&block_group->lock); + if (test_bit(BLOCK_GROUP_FLAG_REMOVED, &block_group->runtime_flags)) { + spin_unlock(&block_group->lock); + return 0; + } + btrfs_freeze_block_group(block_group); + spin_unlock(&block_group->lock); + + ret = trim_no_bitmap(block_group, trimmed, start, end, minlen, false); + if (ret) + goto out; + + ret = trim_bitmaps(block_group, trimmed, start, end, minlen, 0, false); + div64_u64_rem(end, BITS_PER_BITMAP * unit, &rem); + /* If we ended in the middle of a bitmap, reset the trimming flag */ + if (rem) + reset_trimming_bitmap(ctl, offset_to_bitmap(ctl, end)); +out: + btrfs_unfreeze_block_group(block_group); + return ret; +} + +int btrfs_trim_block_group_extents(struct btrfs_block_group *block_group, + u64 *trimmed, u64 start, u64 end, u64 minlen, + bool async) +{ + int ret; + + *trimmed = 0; + + spin_lock(&block_group->lock); + if (test_bit(BLOCK_GROUP_FLAG_REMOVED, &block_group->runtime_flags)) { + spin_unlock(&block_group->lock); + return 0; + } + btrfs_freeze_block_group(block_group); + spin_unlock(&block_group->lock); + + ret = trim_no_bitmap(block_group, trimmed, start, end, minlen, async); + btrfs_unfreeze_block_group(block_group); + + return ret; +} + +int btrfs_trim_block_group_bitmaps(struct btrfs_block_group *block_group, + u64 *trimmed, u64 start, u64 end, u64 minlen, + u64 maxlen, bool async) +{ + int ret; + + *trimmed = 0; + + spin_lock(&block_group->lock); + if (test_bit(BLOCK_GROUP_FLAG_REMOVED, &block_group->runtime_flags)) { + spin_unlock(&block_group->lock); + return 0; + } + btrfs_freeze_block_group(block_group); + spin_unlock(&block_group->lock); + + ret = trim_bitmaps(block_group, trimmed, start, end, minlen, maxlen, + async); + + btrfs_unfreeze_block_group(block_group); + + return ret; +} + +bool btrfs_free_space_cache_v1_active(struct btrfs_fs_info *fs_info) +{ + return btrfs_super_cache_generation(fs_info->super_copy); +} + +static int cleanup_free_space_cache_v1(struct btrfs_fs_info *fs_info, + struct btrfs_trans_handle *trans) +{ + struct btrfs_block_group *block_group; + struct rb_node *node; + + btrfs_info(fs_info, "cleaning free space cache v1"); + + node = rb_first_cached(&fs_info->block_group_cache_tree); + while (node) { + int ret; + + block_group = rb_entry(node, struct btrfs_block_group, cache_node); + ret = btrfs_remove_free_space_inode(trans, NULL, block_group); + if (ret) + return ret; + node = rb_next(node); + } + return 0; +} + +int btrfs_set_free_space_cache_v1_active(struct btrfs_fs_info *fs_info, bool active) +{ + struct btrfs_trans_handle *trans; + int ret; + + /* + * update_super_roots will appropriately set or unset + * super_copy->cache_generation based on SPACE_CACHE and + * BTRFS_FS_CLEANUP_SPACE_CACHE_V1. For this reason, we need a + * transaction commit whether we are enabling space cache v1 and don't + * have any other work to do, or are disabling it and removing free + * space inodes. + */ + trans = btrfs_start_transaction(fs_info->tree_root, 0); + if (IS_ERR(trans)) + return PTR_ERR(trans); + + if (!active) { + set_bit(BTRFS_FS_CLEANUP_SPACE_CACHE_V1, &fs_info->flags); + ret = cleanup_free_space_cache_v1(fs_info, trans); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + btrfs_end_transaction(trans); + goto out; + } + } + + ret = btrfs_commit_transaction(trans); +out: + clear_bit(BTRFS_FS_CLEANUP_SPACE_CACHE_V1, &fs_info->flags); + + return ret; +} + +int __init btrfs_free_space_init(void) +{ + btrfs_free_space_cachep = KMEM_CACHE(btrfs_free_space, 0); + if (!btrfs_free_space_cachep) + return -ENOMEM; + + btrfs_free_space_bitmap_cachep = kmem_cache_create("btrfs_free_space_bitmap", + PAGE_SIZE, PAGE_SIZE, + 0, NULL); + if (!btrfs_free_space_bitmap_cachep) { + kmem_cache_destroy(btrfs_free_space_cachep); + return -ENOMEM; + } + + return 0; +} + +void __cold btrfs_free_space_exit(void) +{ + kmem_cache_destroy(btrfs_free_space_cachep); + kmem_cache_destroy(btrfs_free_space_bitmap_cachep); +} + +#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS +/* + * Use this if you need to make a bitmap or extent entry specifically, it + * doesn't do any of the merging that add_free_space does, this acts a lot like + * how the free space cache loading stuff works, so you can get really weird + * configurations. + */ +int test_add_free_space_entry(struct btrfs_block_group *cache, + u64 offset, u64 bytes, bool bitmap) +{ + struct btrfs_free_space_ctl *ctl = cache->free_space_ctl; + struct btrfs_free_space *info = NULL, *bitmap_info; + void *map = NULL; + enum btrfs_trim_state trim_state = BTRFS_TRIM_STATE_TRIMMED; + u64 bytes_added; + int ret; + +again: + if (!info) { + info = kmem_cache_zalloc(btrfs_free_space_cachep, GFP_NOFS); + if (!info) + return -ENOMEM; + } + + if (!bitmap) { + spin_lock(&ctl->tree_lock); + info->offset = offset; + info->bytes = bytes; + info->max_extent_size = 0; + ret = link_free_space(ctl, info); + spin_unlock(&ctl->tree_lock); + if (ret) + kmem_cache_free(btrfs_free_space_cachep, info); + return ret; + } + + if (!map) { + map = kmem_cache_zalloc(btrfs_free_space_bitmap_cachep, GFP_NOFS); + if (!map) { + kmem_cache_free(btrfs_free_space_cachep, info); + return -ENOMEM; + } + } + + spin_lock(&ctl->tree_lock); + bitmap_info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), + 1, 0); + if (!bitmap_info) { + info->bitmap = map; + map = NULL; + add_new_bitmap(ctl, info, offset); + bitmap_info = info; + info = NULL; + } + + bytes_added = add_bytes_to_bitmap(ctl, bitmap_info, offset, bytes, + trim_state); + + bytes -= bytes_added; + offset += bytes_added; + spin_unlock(&ctl->tree_lock); + + if (bytes) + goto again; + + if (info) + kmem_cache_free(btrfs_free_space_cachep, info); + if (map) + kmem_cache_free(btrfs_free_space_bitmap_cachep, map); + return 0; +} + +/* + * Checks to see if the given range is in the free space cache. This is really + * just used to check the absence of space, so if there is free space in the + * range at all we will return 1. + */ +int test_check_exists(struct btrfs_block_group *cache, + u64 offset, u64 bytes) +{ + struct btrfs_free_space_ctl *ctl = cache->free_space_ctl; + const int unit = cache->fs_info->sectorsize; + struct btrfs_free_space *info; + int ret = 0; + + spin_lock(&ctl->tree_lock); + info = tree_search_offset(ctl, offset, 0, 0); + if (!info) { + info = tree_search_offset(ctl, offset_to_bitmap(ctl, offset), + 1, 0); + if (!info) + goto out; + } + +have_info: + if (info->bitmap) { + u64 bit_off, bit_bytes; + struct rb_node *n; + struct btrfs_free_space *tmp; + + bit_off = offset; + bit_bytes = unit; + ret = search_bitmap(ctl, info, &bit_off, &bit_bytes, false); + if (!ret) { + if (bit_off == offset) { + ret = 1; + goto out; + } else if (bit_off > offset && + offset + bytes > bit_off) { + ret = 1; + goto out; + } + } + + n = rb_prev(&info->offset_index); + while (n) { + tmp = rb_entry(n, struct btrfs_free_space, + offset_index); + if (tmp->offset + tmp->bytes < offset) + break; + if (offset + bytes < tmp->offset) { + n = rb_prev(&tmp->offset_index); + continue; + } + info = tmp; + goto have_info; + } + + n = rb_next(&info->offset_index); + while (n) { + tmp = rb_entry(n, struct btrfs_free_space, + offset_index); + if (offset + bytes < tmp->offset) + break; + if (tmp->offset + tmp->bytes < offset) { + n = rb_next(&tmp->offset_index); + continue; + } + info = tmp; + goto have_info; + } + + ret = 0; + goto out; + } + + if (info->offset == offset) { + ret = 1; + goto out; + } + + if (offset > info->offset && offset < info->offset + info->bytes) + ret = 1; +out: + spin_unlock(&ctl->tree_lock); + return ret; +} +#endif /* CONFIG_BTRFS_FS_RUN_SANITY_TESTS */ diff --git a/fs/btrfs/free-space-cache.h b/fs/btrfs/free-space-cache.h new file mode 100644 index 000000000..53fe8e293 --- /dev/null +++ b/fs/btrfs/free-space-cache.h @@ -0,0 +1,174 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2009 Oracle. All rights reserved. + */ + +#ifndef BTRFS_FREE_SPACE_CACHE_H +#define BTRFS_FREE_SPACE_CACHE_H + +#include +#include +#include +#include +#include +#include "fs.h" + +struct inode; +struct page; +struct btrfs_fs_info; +struct btrfs_path; +struct btrfs_trans_handle; +struct btrfs_trim_block_group; + +/* + * This is the trim state of an extent or bitmap. + * + * BTRFS_TRIM_STATE_TRIMMING is special and used to maintain the state of a + * bitmap as we may need several trims to fully trim a single bitmap entry. + * This is reset should any free space other than trimmed space be added to the + * bitmap. + */ +enum btrfs_trim_state { + BTRFS_TRIM_STATE_UNTRIMMED, + BTRFS_TRIM_STATE_TRIMMED, + BTRFS_TRIM_STATE_TRIMMING, +}; + +struct btrfs_free_space { + struct rb_node offset_index; + struct rb_node bytes_index; + u64 offset; + u64 bytes; + u64 max_extent_size; + unsigned long *bitmap; + struct list_head list; + enum btrfs_trim_state trim_state; + s32 bitmap_extents; +}; + +static inline bool btrfs_free_space_trimmed(struct btrfs_free_space *info) +{ + return (info->trim_state == BTRFS_TRIM_STATE_TRIMMED); +} + +static inline bool btrfs_free_space_trimming_bitmap( + struct btrfs_free_space *info) +{ + return (info->trim_state == BTRFS_TRIM_STATE_TRIMMING); +} + +static inline bool btrfs_trim_interrupted(void) +{ + return fatal_signal_pending(current) || freezing(current); +} + +/* + * Deltas are an effective way to populate global statistics. Give macro names + * to make it clear what we're doing. An example is discard_extents in + * btrfs_free_space_ctl. + */ +enum { + BTRFS_STAT_CURR, + BTRFS_STAT_PREV, + BTRFS_STAT_NR_ENTRIES, +}; + +struct btrfs_free_space_ctl { + struct rb_root free_space_offset; + struct rb_root_cached free_space_bytes; + spinlock_t tree_lock; + int extents_thresh; + int free_extents; + int total_bitmaps; + u64 free_space; + s32 discardable_extents[BTRFS_STAT_NR_ENTRIES]; + s64 discardable_bytes[BTRFS_STAT_NR_ENTRIES]; + struct btrfs_block_group *block_group; + struct mutex cache_writeout_mutex; + struct list_head trimming_ranges; +}; + +struct btrfs_io_ctl { + void *cur, *orig; + struct page *page; + struct page **pages; + struct btrfs_fs_info *fs_info; + struct inode *inode; + unsigned long size; + int index; + int num_pages; + int entries; + int bitmaps; +}; + +int __init btrfs_free_space_init(void); +void __cold btrfs_free_space_exit(void); +struct inode *lookup_free_space_inode(struct btrfs_block_group *block_group, + struct btrfs_path *path); +int create_free_space_inode(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct btrfs_path *path); +int btrfs_remove_free_space_inode(struct btrfs_trans_handle *trans, + struct inode *inode, + struct btrfs_block_group *block_group); + +int btrfs_truncate_free_space_cache(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct inode *inode); +int load_free_space_cache(struct btrfs_block_group *block_group); +int btrfs_wait_cache_io(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct btrfs_path *path); +int btrfs_write_out_cache(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct btrfs_path *path); + +void btrfs_init_free_space_ctl(struct btrfs_block_group *block_group, + struct btrfs_free_space_ctl *ctl); +int btrfs_add_free_space(struct btrfs_block_group *block_group, + u64 bytenr, u64 size); +int btrfs_add_free_space_unused(struct btrfs_block_group *block_group, + u64 bytenr, u64 size); +int btrfs_add_free_space_async_trimmed(struct btrfs_block_group *block_group, + u64 bytenr, u64 size); +int btrfs_remove_free_space(struct btrfs_block_group *block_group, + u64 bytenr, u64 size); +void btrfs_remove_free_space_cache(struct btrfs_block_group *block_group); +bool btrfs_is_free_space_trimmed(struct btrfs_block_group *block_group); +u64 btrfs_find_space_for_alloc(struct btrfs_block_group *block_group, + u64 offset, u64 bytes, u64 empty_size, + u64 *max_extent_size); +void btrfs_dump_free_space(struct btrfs_block_group *block_group, + u64 bytes); +int btrfs_find_space_cluster(struct btrfs_block_group *block_group, + struct btrfs_free_cluster *cluster, + u64 offset, u64 bytes, u64 empty_size); +void btrfs_init_free_cluster(struct btrfs_free_cluster *cluster); +u64 btrfs_alloc_from_cluster(struct btrfs_block_group *block_group, + struct btrfs_free_cluster *cluster, u64 bytes, + u64 min_start, u64 *max_extent_size); +void btrfs_return_cluster_to_free_space( + struct btrfs_block_group *block_group, + struct btrfs_free_cluster *cluster); +int btrfs_trim_block_group(struct btrfs_block_group *block_group, + u64 *trimmed, u64 start, u64 end, u64 minlen); +int btrfs_trim_block_group_extents(struct btrfs_block_group *block_group, + u64 *trimmed, u64 start, u64 end, u64 minlen, + bool async); +int btrfs_trim_block_group_bitmaps(struct btrfs_block_group *block_group, + u64 *trimmed, u64 start, u64 end, u64 minlen, + u64 maxlen, bool async); +void btrfs_trim_fully_remapped_block_group(struct btrfs_block_group *bg); + +bool btrfs_free_space_cache_v1_active(struct btrfs_fs_info *fs_info); +int btrfs_set_free_space_cache_v1_active(struct btrfs_fs_info *fs_info, bool active); +/* Support functions for running our sanity tests */ +#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS +bool btrfs_use_bitmap(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info); +int test_add_free_space_entry(struct btrfs_block_group *cache, + u64 offset, u64 bytes, bool bitmap); +int test_check_exists(struct btrfs_block_group *cache, u64 offset, u64 bytes); +#endif + +#endif diff --git a/fs/btrfs/free-space-tree.c b/fs/btrfs/free-space-tree.c new file mode 100644 index 000000000..1b3d82ae3 --- /dev/null +++ b/fs/btrfs/free-space-tree.c @@ -0,0 +1,1852 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015 Facebook. All rights reserved. + */ + +#include +#include +#include "messages.h" +#include "ctree.h" +#include "disk-io.h" +#include "locking.h" +#include "free-space-tree.h" +#include "transaction.h" +#include "block-group.h" +#include "fs.h" +#include "accessors.h" +#include "extent-tree.h" +#include "root-tree.h" + +static int __add_block_group_free_space(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct btrfs_path *path); + +struct btrfs_root *btrfs_free_space_root(struct btrfs_block_group *block_group) +{ + struct btrfs_key key = { + .objectid = BTRFS_FREE_SPACE_TREE_OBJECTID, + .type = BTRFS_ROOT_ITEM_KEY, + .offset = 0, + }; + + if (btrfs_fs_incompat(block_group->fs_info, EXTENT_TREE_V2)) + key.offset = block_group->global_root_id; + return btrfs_global_root(block_group->fs_info, &key); +} + +void btrfs_set_free_space_tree_thresholds(struct btrfs_block_group *cache) +{ + u32 bitmap_range; + size_t bitmap_size; + u64 num_bitmaps, total_bitmap_size; + + if (WARN_ON(cache->length == 0)) + btrfs_warn(cache->fs_info, "block group %llu length is zero", + cache->start); + + /* + * We convert to bitmaps when the disk space required for using extents + * exceeds that required for using bitmaps. + */ + bitmap_range = cache->fs_info->sectorsize * BTRFS_FREE_SPACE_BITMAP_BITS; + num_bitmaps = div_u64(cache->length + bitmap_range - 1, bitmap_range); + bitmap_size = sizeof(struct btrfs_item) + BTRFS_FREE_SPACE_BITMAP_SIZE; + total_bitmap_size = num_bitmaps * bitmap_size; + cache->bitmap_high_thresh = div_u64(total_bitmap_size, + sizeof(struct btrfs_item)); + + /* + * We allow for a small buffer between the high threshold and low + * threshold to avoid thrashing back and forth between the two formats. + */ + if (cache->bitmap_high_thresh > 100) + cache->bitmap_low_thresh = cache->bitmap_high_thresh - 100; + else + cache->bitmap_low_thresh = 0; +} + +static int add_new_free_space_info(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct btrfs_path *path) +{ + struct btrfs_root *root = btrfs_free_space_root(block_group); + struct btrfs_free_space_info *info; + struct btrfs_key key; + struct extent_buffer *leaf; + int ret; + + key.objectid = block_group->start; + key.type = BTRFS_FREE_SPACE_INFO_KEY; + key.offset = block_group->length; + + ret = btrfs_insert_empty_item(trans, root, path, &key, sizeof(*info)); + if (ret) + return ret; + + leaf = path->nodes[0]; + info = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_free_space_info); + btrfs_set_free_space_extent_count(leaf, info, 0); + btrfs_set_free_space_flags(leaf, info, 0); + btrfs_release_path(path); + return 0; +} + +struct btrfs_free_space_info *btrfs_search_free_space_info( + struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct btrfs_path *path, int cow) +{ + struct btrfs_fs_info *fs_info = block_group->fs_info; + struct btrfs_root *root = btrfs_free_space_root(block_group); + struct btrfs_key key; + int ret; + + key.objectid = block_group->start; + key.type = BTRFS_FREE_SPACE_INFO_KEY; + key.offset = block_group->length; + + ret = btrfs_search_slot(trans, root, &key, path, 0, cow); + if (ret < 0) + return ERR_PTR(ret); + if (unlikely(ret != 0)) { + btrfs_warn(fs_info, "missing free space info for %llu", + block_group->start); + DEBUG_WARN(); + return ERR_PTR(-ENOENT); + } + + return btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_free_space_info); +} + +/* + * btrfs_search_slot() but we're looking for the greatest key less than the + * passed key. + */ +static int btrfs_search_prev_slot(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_key *key, struct btrfs_path *p, + int ins_len, int cow) +{ + int ret; + + ret = btrfs_search_slot(trans, root, key, p, ins_len, cow); + if (ret < 0) + return ret; + + if (unlikely(ret == 0)) { + DEBUG_WARN(); + return -EIO; + } + + if (unlikely(p->slots[0] == 0)) { + DEBUG_WARN("no previous slot found"); + return -EIO; + } + p->slots[0]--; + + return 0; +} + +static inline u32 free_space_bitmap_size(const struct btrfs_fs_info *fs_info, + u64 size) +{ + return DIV_ROUND_UP(size >> fs_info->sectorsize_bits, BITS_PER_BYTE); +} + +static unsigned long *alloc_bitmap(u32 bitmap_size) +{ + unsigned long *ret; + unsigned int nofs_flag; + u32 bitmap_rounded_size = round_up(bitmap_size, sizeof(unsigned long)); + + /* + * GFP_NOFS doesn't work with kvmalloc(), but we really can't recurse + * into the filesystem here. All callers hold a transaction handle + * open, so if a GFP_KERNEL allocation recurses into the filesystem + * and triggers a transaction commit, we would deadlock. + */ + nofs_flag = memalloc_nofs_save(); + ret = kvzalloc(bitmap_rounded_size, GFP_KERNEL); + memalloc_nofs_restore(nofs_flag); + return ret; +} + +static void le_bitmap_set(unsigned long *map, unsigned int start, int len) +{ + u8 *p = ((u8 *)map) + BIT_BYTE(start); + const unsigned int size = start + len; + int bits_to_set = BITS_PER_BYTE - (start % BITS_PER_BYTE); + u8 mask_to_set = BITMAP_FIRST_BYTE_MASK(start); + + while (len - bits_to_set >= 0) { + *p |= mask_to_set; + len -= bits_to_set; + bits_to_set = BITS_PER_BYTE; + mask_to_set = ~0; + p++; + } + if (len) { + mask_to_set &= BITMAP_LAST_BYTE_MASK(size); + *p |= mask_to_set; + } +} + +EXPORT_FOR_TESTS +int btrfs_convert_free_space_to_bitmaps(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct btrfs_path *path) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *root = btrfs_free_space_root(block_group); + struct btrfs_free_space_info *info; + struct btrfs_key key, found_key; + struct extent_buffer *leaf; + unsigned long *bitmap; + char *bitmap_cursor; + u64 start, end; + u64 bitmap_range, i; + u32 bitmap_size, flags, expected_extent_count; + u32 extent_count = 0; + bool done = false; + int nr; + int ret; + + bitmap_size = free_space_bitmap_size(fs_info, block_group->length); + bitmap = alloc_bitmap(bitmap_size); + if (unlikely(!bitmap)) + return 0; + + start = block_group->start; + end = btrfs_block_group_end(block_group); + + key.objectid = end - 1; + key.type = (u8)-1; + key.offset = (u64)-1; + + while (!done) { + ret = btrfs_search_prev_slot(trans, root, &key, path, -1, 1); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + + leaf = path->nodes[0]; + nr = 0; + path->slots[0]++; + while (path->slots[0] > 0) { + btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0] - 1); + + if (found_key.type == BTRFS_FREE_SPACE_INFO_KEY) { + ASSERT(found_key.objectid == block_group->start); + ASSERT(found_key.offset == block_group->length); + done = true; + break; + } else if (found_key.type == BTRFS_FREE_SPACE_EXTENT_KEY) { + u64 first, last; + + ASSERT(found_key.objectid >= start); + ASSERT(found_key.objectid < end); + ASSERT(found_key.objectid + found_key.offset <= end); + + first = div_u64(found_key.objectid - start, + fs_info->sectorsize); + last = div_u64(found_key.objectid + found_key.offset - start, + fs_info->sectorsize); + le_bitmap_set(bitmap, first, last - first); + + extent_count++; + nr++; + path->slots[0]--; + } else { + btrfs_err(fs_info, "unexpected free space tree key type %u", + found_key.type); + ret = -EUCLEAN; + btrfs_abort_transaction(trans, ret); + goto out; + } + } + + ret = btrfs_del_items(trans, root, path, path->slots[0], nr); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + btrfs_release_path(path); + } + + info = btrfs_search_free_space_info(trans, block_group, path, 1); + if (IS_ERR(info)) { + ret = PTR_ERR(info); + btrfs_abort_transaction(trans, ret); + goto out; + } + leaf = path->nodes[0]; + flags = btrfs_free_space_flags(leaf, info); + flags |= BTRFS_FREE_SPACE_USING_BITMAPS; + block_group->using_free_space_bitmaps = true; + block_group->using_free_space_bitmaps_cached = true; + btrfs_set_free_space_flags(leaf, info, flags); + expected_extent_count = btrfs_free_space_extent_count(leaf, info); + btrfs_release_path(path); + + if (unlikely(extent_count != expected_extent_count)) { + btrfs_err(fs_info, + "incorrect extent count for %llu; counted %u, expected %u", + block_group->start, extent_count, + expected_extent_count); + ret = -EIO; + btrfs_abort_transaction(trans, ret); + goto out; + } + + bitmap_cursor = (char *)bitmap; + bitmap_range = fs_info->sectorsize * BTRFS_FREE_SPACE_BITMAP_BITS; + i = start; + while (i < end) { + unsigned long ptr; + u64 extent_size; + u32 data_size; + + extent_size = min(end - i, bitmap_range); + data_size = free_space_bitmap_size(fs_info, extent_size); + + key.objectid = i; + key.type = BTRFS_FREE_SPACE_BITMAP_KEY; + key.offset = extent_size; + + ret = btrfs_insert_empty_item(trans, root, path, &key, + data_size); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + + leaf = path->nodes[0]; + ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); + write_extent_buffer(leaf, bitmap_cursor, ptr, + data_size); + btrfs_release_path(path); + + i += extent_size; + bitmap_cursor += data_size; + } + + ret = 0; +out: + kvfree(bitmap); + return ret; +} + +EXPORT_FOR_TESTS +int btrfs_convert_free_space_to_extents(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct btrfs_path *path) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *root = btrfs_free_space_root(block_group); + struct btrfs_free_space_info *info; + struct btrfs_key key, found_key; + struct extent_buffer *leaf; + unsigned long *bitmap; + u64 start, end; + u32 bitmap_size, flags, expected_extent_count; + unsigned long nrbits, start_bit, end_bit; + u32 extent_count = 0; + bool done = false; + int nr; + int ret; + + bitmap_size = free_space_bitmap_size(fs_info, block_group->length); + bitmap = alloc_bitmap(bitmap_size); + if (unlikely(!bitmap)) + return 0; + + start = block_group->start; + end = btrfs_block_group_end(block_group); + + key.objectid = end - 1; + key.type = (u8)-1; + key.offset = (u64)-1; + + while (!done) { + ret = btrfs_search_prev_slot(trans, root, &key, path, -1, 1); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + + leaf = path->nodes[0]; + nr = 0; + path->slots[0]++; + while (path->slots[0] > 0) { + btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0] - 1); + + if (found_key.type == BTRFS_FREE_SPACE_INFO_KEY) { + ASSERT(found_key.objectid == block_group->start); + ASSERT(found_key.offset == block_group->length); + done = true; + break; + } else if (found_key.type == BTRFS_FREE_SPACE_BITMAP_KEY) { + unsigned long ptr; + char *bitmap_cursor; + u32 bitmap_pos, data_size; + + ASSERT(found_key.objectid >= start); + ASSERT(found_key.objectid < end); + ASSERT(found_key.objectid + found_key.offset <= end); + + bitmap_pos = div_u64(found_key.objectid - start, + fs_info->sectorsize * + BITS_PER_BYTE); + bitmap_cursor = ((char *)bitmap) + bitmap_pos; + data_size = free_space_bitmap_size(fs_info, + found_key.offset); + + path->slots[0]--; + ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); + read_extent_buffer(leaf, bitmap_cursor, ptr, + data_size); + + nr++; + } else { + btrfs_err(fs_info, "unexpected free space tree key type %u", + found_key.type); + ret = -EUCLEAN; + btrfs_abort_transaction(trans, ret); + goto out; + } + } + + ret = btrfs_del_items(trans, root, path, path->slots[0], nr); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + btrfs_release_path(path); + } + + info = btrfs_search_free_space_info(trans, block_group, path, 1); + if (IS_ERR(info)) { + ret = PTR_ERR(info); + btrfs_abort_transaction(trans, ret); + goto out; + } + leaf = path->nodes[0]; + flags = btrfs_free_space_flags(leaf, info); + flags &= ~BTRFS_FREE_SPACE_USING_BITMAPS; + block_group->using_free_space_bitmaps = false; + block_group->using_free_space_bitmaps_cached = true; + btrfs_set_free_space_flags(leaf, info, flags); + expected_extent_count = btrfs_free_space_extent_count(leaf, info); + btrfs_release_path(path); + + nrbits = block_group->length >> fs_info->sectorsize_bits; + start_bit = find_next_bit_le(bitmap, nrbits, 0); + + while (start_bit < nrbits) { + end_bit = find_next_zero_bit_le(bitmap, nrbits, start_bit); + ASSERT(start_bit < end_bit); + + key.objectid = start + start_bit * fs_info->sectorsize; + key.type = BTRFS_FREE_SPACE_EXTENT_KEY; + key.offset = (end_bit - start_bit) * fs_info->sectorsize; + + ret = btrfs_insert_empty_item(trans, root, path, &key, 0); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + btrfs_release_path(path); + + extent_count++; + + start_bit = find_next_bit_le(bitmap, nrbits, end_bit); + } + + if (unlikely(extent_count != expected_extent_count)) { + btrfs_err(fs_info, + "incorrect extent count for %llu; counted %u, expected %u", + block_group->start, extent_count, + expected_extent_count); + ret = -EIO; + btrfs_abort_transaction(trans, ret); + goto out; + } + + ret = 0; +out: + kvfree(bitmap); + return ret; +} + +static int update_free_space_extent_count(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct btrfs_path *path, + int new_extents) +{ + struct btrfs_free_space_info *info; + u32 flags; + u32 extent_count; + int ret = 0; + + if (new_extents == 0) + return 0; + + info = btrfs_search_free_space_info(trans, block_group, path, 1); + if (IS_ERR(info)) + return PTR_ERR(info); + + flags = btrfs_free_space_flags(path->nodes[0], info); + extent_count = btrfs_free_space_extent_count(path->nodes[0], info); + + extent_count += new_extents; + btrfs_set_free_space_extent_count(path->nodes[0], info, extent_count); + btrfs_release_path(path); + + if (!(flags & BTRFS_FREE_SPACE_USING_BITMAPS) && + extent_count > block_group->bitmap_high_thresh) { + ret = btrfs_convert_free_space_to_bitmaps(trans, block_group, path); + } else if ((flags & BTRFS_FREE_SPACE_USING_BITMAPS) && + extent_count < block_group->bitmap_low_thresh) { + ret = btrfs_convert_free_space_to_extents(trans, block_group, path); + } + + return ret; +} + +EXPORT_FOR_TESTS +bool btrfs_free_space_test_bit(struct btrfs_block_group *block_group, + struct btrfs_path *path, u64 offset) +{ + struct extent_buffer *leaf; + struct btrfs_key key; + u64 found_start, found_end; + unsigned long ptr, i; + + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + ASSERT(key.type == BTRFS_FREE_SPACE_BITMAP_KEY); + + found_start = key.objectid; + found_end = key.objectid + key.offset; + ASSERT(offset >= found_start && offset < found_end); + + ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); + i = div_u64(offset - found_start, + block_group->fs_info->sectorsize); + return extent_buffer_test_bit(leaf, ptr, i); +} + +static void free_space_modify_bits(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct btrfs_path *path, u64 *start, u64 *size, + bool set_bits) +{ + struct btrfs_fs_info *fs_info = block_group->fs_info; + struct extent_buffer *leaf; + struct btrfs_key key; + u64 end = *start + *size; + u64 found_start, found_end; + unsigned long ptr, first, last; + + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + ASSERT(key.type == BTRFS_FREE_SPACE_BITMAP_KEY); + + found_start = key.objectid; + found_end = key.objectid + key.offset; + ASSERT(*start >= found_start && *start < found_end); + ASSERT(end > found_start); + + if (end > found_end) + end = found_end; + + ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); + first = (*start - found_start) >> fs_info->sectorsize_bits; + last = (end - found_start) >> fs_info->sectorsize_bits; + if (set_bits) + extent_buffer_bitmap_set(leaf, ptr, first, last - first); + else + extent_buffer_bitmap_clear(leaf, ptr, first, last - first); + btrfs_mark_buffer_dirty(trans, leaf); + + *size -= end - *start; + *start = end; +} + +/* + * We can't use btrfs_next_item() in modify_free_space_bitmap() because + * btrfs_next_leaf() doesn't get the path for writing. We can forgo the fancy + * tree walking in btrfs_next_leaf() anyways because we know exactly what we're + * looking for. + */ +static int free_space_next_bitmap(struct btrfs_trans_handle *trans, + struct btrfs_root *root, struct btrfs_path *p) +{ + struct btrfs_key key; + + if (p->slots[0] + 1 < btrfs_header_nritems(p->nodes[0])) { + p->slots[0]++; + return 0; + } + + btrfs_item_key_to_cpu(p->nodes[0], &key, p->slots[0]); + btrfs_release_path(p); + + key.objectid += key.offset; + key.type = (u8)-1; + key.offset = (u64)-1; + + return btrfs_search_prev_slot(trans, root, &key, p, 0, 1); +} + +/* + * If remove is 1, then we are removing free space, thus clearing bits in the + * bitmap. If remove is 0, then we are adding free space, thus setting bits in + * the bitmap. + */ +static int modify_free_space_bitmap(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct btrfs_path *path, + u64 start, u64 size, bool remove) +{ + struct btrfs_root *root = btrfs_free_space_root(block_group); + struct btrfs_key key; + u64 end = start + size; + u64 cur_start, cur_size; + bool prev_bit_set = false; + bool next_bit_set = false; + int new_extents; + int ret; + + /* + * Read the bit for the block immediately before the extent of space if + * that block is within the block group. + */ + if (start > block_group->start) { + u64 prev_block = start - block_group->fs_info->sectorsize; + + key.objectid = prev_block; + key.type = (u8)-1; + key.offset = (u64)-1; + + ret = btrfs_search_prev_slot(trans, root, &key, path, 0, 1); + if (ret) + return ret; + + prev_bit_set = btrfs_free_space_test_bit(block_group, path, prev_block); + + /* The previous block may have been in the previous bitmap. */ + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + if (start >= key.objectid + key.offset) { + ret = free_space_next_bitmap(trans, root, path); + if (ret) + return ret; + } + } else { + key.objectid = start; + key.type = (u8)-1; + key.offset = (u64)-1; + + ret = btrfs_search_prev_slot(trans, root, &key, path, 0, 1); + if (ret) + return ret; + } + + /* + * Iterate over all of the bitmaps overlapped by the extent of space, + * clearing/setting bits as required. + */ + cur_start = start; + cur_size = size; + while (1) { + free_space_modify_bits(trans, block_group, path, &cur_start, + &cur_size, !remove); + if (cur_size == 0) + break; + ret = free_space_next_bitmap(trans, root, path); + if (ret) + return ret; + } + + /* + * Read the bit for the block immediately after the extent of space if + * that block is within the block group. + */ + if (end < btrfs_block_group_end(block_group)) { + /* The next block may be in the next bitmap. */ + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + if (end >= key.objectid + key.offset) { + ret = free_space_next_bitmap(trans, root, path); + if (ret) + return ret; + } + + next_bit_set = btrfs_free_space_test_bit(block_group, path, end); + } + + if (remove) { + new_extents = -1; + if (prev_bit_set) { + /* Leftover on the left. */ + new_extents++; + } + if (next_bit_set) { + /* Leftover on the right. */ + new_extents++; + } + } else { + new_extents = 1; + if (prev_bit_set) { + /* Merging with neighbor on the left. */ + new_extents--; + } + if (next_bit_set) { + /* Merging with neighbor on the right. */ + new_extents--; + } + } + + btrfs_release_path(path); + return update_free_space_extent_count(trans, block_group, path, new_extents); +} + +static int remove_free_space_extent(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct btrfs_path *path, + u64 start, u64 size) +{ + struct btrfs_root *root = btrfs_free_space_root(block_group); + struct btrfs_key key; + u64 found_start, found_end; + u64 end = start + size; + int new_extents = -1; + int ret; + + key.objectid = start; + key.type = (u8)-1; + key.offset = (u64)-1; + + ret = btrfs_search_prev_slot(trans, root, &key, path, -1, 1); + if (ret) + return ret; + + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + + ASSERT(key.type == BTRFS_FREE_SPACE_EXTENT_KEY); + + found_start = key.objectid; + found_end = key.objectid + key.offset; + ASSERT(start >= found_start && end <= found_end); + + /* + * Okay, now that we've found the free space extent which contains the + * free space that we are removing, there are four cases: + * + * 1. We're using the whole extent: delete the key we found and + * decrement the free space extent count. + * 2. We are using part of the extent starting at the beginning: delete + * the key we found and insert a new key representing the leftover at + * the end. There is no net change in the number of extents. + * 3. We are using part of the extent ending at the end: delete the key + * we found and insert a new key representing the leftover at the + * beginning. There is no net change in the number of extents. + * 4. We are using part of the extent in the middle: delete the key we + * found and insert two new keys representing the leftovers on each + * side. Where we used to have one extent, we now have two, so increment + * the extent count. We may need to convert the block group to bitmaps + * as a result. + */ + + /* Delete the existing key (cases 1-4). */ + ret = btrfs_del_item(trans, root, path); + if (ret) + return ret; + + /* Add a key for leftovers at the beginning (cases 3 and 4). */ + if (start > found_start) { + key.objectid = found_start; + key.type = BTRFS_FREE_SPACE_EXTENT_KEY; + key.offset = start - found_start; + + btrfs_release_path(path); + ret = btrfs_insert_empty_item(trans, root, path, &key, 0); + if (ret) + return ret; + new_extents++; + } + + /* Add a key for leftovers at the end (cases 2 and 4). */ + if (end < found_end) { + key.objectid = end; + key.type = BTRFS_FREE_SPACE_EXTENT_KEY; + key.offset = found_end - end; + + btrfs_release_path(path); + ret = btrfs_insert_empty_item(trans, root, path, &key, 0); + if (ret) + return ret; + new_extents++; + } + + btrfs_release_path(path); + return update_free_space_extent_count(trans, block_group, path, new_extents); +} + +static int using_bitmaps(struct btrfs_block_group *bg, struct btrfs_path *path) +{ + struct btrfs_free_space_info *info; + u32 flags; + + if (bg->using_free_space_bitmaps_cached) + return bg->using_free_space_bitmaps; + + info = btrfs_search_free_space_info(NULL, bg, path, 0); + if (IS_ERR(info)) + return PTR_ERR(info); + flags = btrfs_free_space_flags(path->nodes[0], info); + btrfs_release_path(path); + + bg->using_free_space_bitmaps = (flags & BTRFS_FREE_SPACE_USING_BITMAPS); + bg->using_free_space_bitmaps_cached = true; + + return bg->using_free_space_bitmaps; +} + +EXPORT_FOR_TESTS +int __btrfs_remove_from_free_space_tree(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct btrfs_path *path, u64 start, u64 size) +{ + int ret; + + ret = __add_block_group_free_space(trans, block_group, path); + if (ret) + return ret; + + ret = using_bitmaps(block_group, path); + if (ret < 0) + return ret; + + if (ret) + return modify_free_space_bitmap(trans, block_group, path, + start, size, true); + + return remove_free_space_extent(trans, block_group, path, start, size); +} + +int btrfs_remove_from_free_space_tree(struct btrfs_trans_handle *trans, + u64 start, u64 size) +{ + struct btrfs_block_group *block_group; + BTRFS_PATH_AUTO_FREE(path); + int ret; + + if (!btrfs_fs_compat_ro(trans->fs_info, FREE_SPACE_TREE)) + return 0; + + path = btrfs_alloc_path(); + if (unlikely(!path)) { + ret = -ENOMEM; + btrfs_abort_transaction(trans, ret); + return ret; + } + + block_group = btrfs_lookup_block_group(trans->fs_info, start); + if (unlikely(!block_group)) { + DEBUG_WARN("no block group found for start=%llu", start); + ret = -ENOENT; + btrfs_abort_transaction(trans, ret); + return ret; + } + + mutex_lock(&block_group->free_space_lock); + ret = __btrfs_remove_from_free_space_tree(trans, block_group, path, start, size); + mutex_unlock(&block_group->free_space_lock); + if (ret) + btrfs_abort_transaction(trans, ret); + + btrfs_put_block_group(block_group); + + return ret; +} + +static int add_free_space_extent(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct btrfs_path *path, + u64 start, u64 size) +{ + struct btrfs_root *root = btrfs_free_space_root(block_group); + struct btrfs_key key, new_key; + u64 found_start, found_end; + u64 end = start + size; + int new_extents = 1; + int ret; + + /* + * We are adding a new extent of free space, but we need to merge + * extents. There are four cases here: + * + * 1. The new extent does not have any immediate neighbors to merge + * with: add the new key and increment the free space extent count. We + * may need to convert the block group to bitmaps as a result. + * 2. The new extent has an immediate neighbor before it: remove the + * previous key and insert a new key combining both of them. There is no + * net change in the number of extents. + * 3. The new extent has an immediate neighbor after it: remove the next + * key and insert a new key combining both of them. There is no net + * change in the number of extents. + * 4. The new extent has immediate neighbors on both sides: remove both + * of the keys and insert a new key combining all of them. Where we used + * to have two extents, we now have one, so decrement the extent count. + */ + + new_key.objectid = start; + new_key.type = BTRFS_FREE_SPACE_EXTENT_KEY; + new_key.offset = size; + + /* Search for a neighbor on the left. */ + if (start == block_group->start) + goto right; + key.objectid = start - 1; + key.type = (u8)-1; + key.offset = (u64)-1; + + ret = btrfs_search_prev_slot(trans, root, &key, path, -1, 1); + if (ret) + return ret; + + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + + if (key.type != BTRFS_FREE_SPACE_EXTENT_KEY) { + ASSERT(key.type == BTRFS_FREE_SPACE_INFO_KEY); + btrfs_release_path(path); + goto right; + } + + found_start = key.objectid; + found_end = key.objectid + key.offset; + ASSERT(found_start >= block_group->start && + found_end > block_group->start); + ASSERT(found_start < start && found_end <= start); + + /* + * Delete the neighbor on the left and absorb it into the new key (cases + * 2 and 4). + */ + if (found_end == start) { + ret = btrfs_del_item(trans, root, path); + if (ret) + return ret; + new_key.objectid = found_start; + new_key.offset += key.offset; + new_extents--; + } + btrfs_release_path(path); + +right: + /* Search for a neighbor on the right. */ + if (end == btrfs_block_group_end(block_group)) + goto insert; + key.objectid = end; + key.type = (u8)-1; + key.offset = (u64)-1; + + ret = btrfs_search_prev_slot(trans, root, &key, path, -1, 1); + if (ret) + return ret; + + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + + if (key.type != BTRFS_FREE_SPACE_EXTENT_KEY) { + ASSERT(key.type == BTRFS_FREE_SPACE_INFO_KEY); + btrfs_release_path(path); + goto insert; + } + + found_start = key.objectid; + found_end = key.objectid + key.offset; + ASSERT(found_start >= block_group->start && + found_end > block_group->start); + ASSERT((found_start < start && found_end <= start) || + (found_start >= end && found_end > end)); + + /* + * Delete the neighbor on the right and absorb it into the new key + * (cases 3 and 4). + */ + if (found_start == end) { + ret = btrfs_del_item(trans, root, path); + if (ret) + return ret; + new_key.offset += key.offset; + new_extents--; + } + btrfs_release_path(path); + +insert: + /* Insert the new key (cases 1-4). */ + ret = btrfs_insert_empty_item(trans, root, path, &new_key, 0); + if (ret) + return ret; + + btrfs_release_path(path); + return update_free_space_extent_count(trans, block_group, path, new_extents); +} + +EXPORT_FOR_TESTS +int __btrfs_add_to_free_space_tree(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct btrfs_path *path, u64 start, u64 size) +{ + int ret; + + ret = __add_block_group_free_space(trans, block_group, path); + if (ret) + return ret; + + ret = using_bitmaps(block_group, path); + if (ret < 0) + return ret; + + if (ret) + return modify_free_space_bitmap(trans, block_group, path, + start, size, false); + + return add_free_space_extent(trans, block_group, path, start, size); +} + +int btrfs_add_to_free_space_tree(struct btrfs_trans_handle *trans, + u64 start, u64 size) +{ + struct btrfs_block_group *block_group; + BTRFS_PATH_AUTO_FREE(path); + int ret; + + if (!btrfs_fs_compat_ro(trans->fs_info, FREE_SPACE_TREE)) + return 0; + + path = btrfs_alloc_path(); + if (unlikely(!path)) { + ret = -ENOMEM; + btrfs_abort_transaction(trans, ret); + return ret; + } + + block_group = btrfs_lookup_block_group(trans->fs_info, start); + if (unlikely(!block_group)) { + DEBUG_WARN("no block group found for start=%llu", start); + ret = -ENOENT; + btrfs_abort_transaction(trans, ret); + return ret; + } + + mutex_lock(&block_group->free_space_lock); + ret = __btrfs_add_to_free_space_tree(trans, block_group, path, start, size); + mutex_unlock(&block_group->free_space_lock); + if (ret) + btrfs_abort_transaction(trans, ret); + + btrfs_put_block_group(block_group); + + return ret; +} + +/* + * Populate the free space tree by walking the extent tree. Operations on the + * extent tree that happen as a result of writes to the free space tree will go + * through the normal add/remove hooks. + */ +static int populate_free_space_tree(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group) +{ + struct btrfs_root *extent_root; + BTRFS_PATH_AUTO_FREE(path); + BTRFS_PATH_AUTO_FREE(path2); + struct btrfs_key key; + u64 start, end; + int ret; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + path2 = btrfs_alloc_path(); + if (!path2) + return -ENOMEM; + + path->reada = READA_FORWARD; + + ret = add_new_free_space_info(trans, block_group, path2); + if (ret) + return ret; + + extent_root = btrfs_extent_root(trans->fs_info, block_group->start); + if (unlikely(!extent_root)) { + btrfs_err(trans->fs_info, + "missing extent root for block group at offset %llu", + block_group->start); + return -EUCLEAN; + } + + mutex_lock(&block_group->free_space_lock); + + /* + * Iterate through all of the extent and metadata items in this block + * group, adding the free space between them and the free space at the + * end. Note that EXTENT_ITEM and METADATA_ITEM are less than + * BLOCK_GROUP_ITEM, so an extent may precede the block group that it's + * contained in. + */ + key.objectid = block_group->start; + key.type = BTRFS_EXTENT_ITEM_KEY; + key.offset = 0; + + ret = btrfs_search_slot_for_read(extent_root, &key, path, 1, 0); + if (ret < 0) + goto out_locked; + /* + * If ret is 1 (no key found), it means this is an empty block group, + * without any extents allocated from it and there's no block group + * item (key BTRFS_BLOCK_GROUP_ITEM_KEY) located in the extent tree + * because we are using the block group tree feature (so block group + * items are stored in the block group tree) or this is a new block + * group created in the current transaction and its block group item + * was not yet inserted in the extent tree (that happens in + * btrfs_create_pending_block_groups() -> insert_block_group_item()). + * It also means there are no extents allocated for block groups with a + * start offset beyond this block group's end offset (this is the last, + * highest, block group). + */ + start = block_group->start; + end = btrfs_block_group_end(block_group); + while (ret == 0) { + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + + if (key.type == BTRFS_EXTENT_ITEM_KEY || + key.type == BTRFS_METADATA_ITEM_KEY) { + if (key.objectid >= end) + break; + + if (start < key.objectid) { + ret = __btrfs_add_to_free_space_tree(trans, + block_group, + path2, start, + key.objectid - + start); + if (ret) + goto out_locked; + } + start = key.objectid; + if (key.type == BTRFS_METADATA_ITEM_KEY) + start += trans->fs_info->nodesize; + else + start += key.offset; + } else if (key.type == BTRFS_BLOCK_GROUP_ITEM_KEY) { + if (key.objectid != block_group->start) + break; + } + + ret = btrfs_next_item(extent_root, path); + if (ret < 0) + goto out_locked; + } + if (start < end) { + ret = __btrfs_add_to_free_space_tree(trans, block_group, path2, + start, end - start); + if (ret) + goto out_locked; + } + + ret = 0; +out_locked: + mutex_unlock(&block_group->free_space_lock); + + return ret; +} + +int btrfs_create_free_space_tree(struct btrfs_fs_info *fs_info) +{ + struct btrfs_trans_handle *trans; + struct btrfs_root *tree_root = fs_info->tree_root; + struct btrfs_root *free_space_root; + struct btrfs_block_group *block_group; + struct rb_node *node; + int ret; + + trans = btrfs_start_transaction(tree_root, 0); + if (IS_ERR(trans)) + return PTR_ERR(trans); + + set_bit(BTRFS_FS_CREATING_FREE_SPACE_TREE, &fs_info->flags); + set_bit(BTRFS_FS_FREE_SPACE_TREE_UNTRUSTED, &fs_info->flags); + free_space_root = btrfs_create_tree(trans, + BTRFS_FREE_SPACE_TREE_OBJECTID); + if (IS_ERR(free_space_root)) { + ret = PTR_ERR(free_space_root); + btrfs_abort_transaction(trans, ret); + btrfs_end_transaction(trans); + goto out_clear; + } + ret = btrfs_global_root_insert(free_space_root); + if (unlikely(ret)) { + btrfs_put_root(free_space_root); + btrfs_abort_transaction(trans, ret); + btrfs_end_transaction(trans); + goto out_clear; + } + + node = rb_first_cached(&fs_info->block_group_cache_tree); + while (node) { + block_group = rb_entry(node, struct btrfs_block_group, + cache_node); + ret = populate_free_space_tree(trans, block_group); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + btrfs_end_transaction(trans); + goto out_clear; + } + node = rb_next(node); + } + + btrfs_set_fs_compat_ro(fs_info, FREE_SPACE_TREE); + btrfs_set_fs_compat_ro(fs_info, FREE_SPACE_TREE_VALID); + clear_bit(BTRFS_FS_CREATING_FREE_SPACE_TREE, &fs_info->flags); + ret = btrfs_commit_transaction(trans); + + /* + * Now that we've committed the transaction any reading of our commit + * root will be safe, so we can cache from the free space tree now. + */ + clear_bit(BTRFS_FS_FREE_SPACE_TREE_UNTRUSTED, &fs_info->flags); + return ret; + +out_clear: + clear_bit(BTRFS_FS_CREATING_FREE_SPACE_TREE, &fs_info->flags); + clear_bit(BTRFS_FS_FREE_SPACE_TREE_UNTRUSTED, &fs_info->flags); + return ret; +} + +static int clear_free_space_tree(struct btrfs_trans_handle *trans, + struct btrfs_root *root) +{ + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + struct rb_node *node; + int nr; + int ret; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = 0; + key.type = 0; + key.offset = 0; + + while (1) { + ret = btrfs_search_slot(trans, root, &key, path, -1, 1); + if (ret < 0) + return ret; + + nr = btrfs_header_nritems(path->nodes[0]); + if (!nr) + break; + + path->slots[0] = 0; + ret = btrfs_del_items(trans, root, path, 0, nr); + if (ret) + return ret; + + btrfs_release_path(path); + } + + node = rb_first_cached(&trans->fs_info->block_group_cache_tree); + while (node) { + struct btrfs_block_group *bg; + + bg = rb_entry(node, struct btrfs_block_group, cache_node); + clear_bit(BLOCK_GROUP_FLAG_FREE_SPACE_ADDED, &bg->runtime_flags); + node = rb_next(node); + cond_resched(); + } + + return 0; +} + +int btrfs_delete_free_space_tree(struct btrfs_fs_info *fs_info) +{ + struct btrfs_trans_handle *trans; + struct btrfs_root *tree_root = fs_info->tree_root; + struct btrfs_key key = { + .objectid = BTRFS_FREE_SPACE_TREE_OBJECTID, + .type = BTRFS_ROOT_ITEM_KEY, + .offset = 0, + }; + struct btrfs_root *free_space_root = btrfs_global_root(fs_info, &key); + int ret; + + trans = btrfs_start_transaction(tree_root, 0); + if (IS_ERR(trans)) + return PTR_ERR(trans); + + btrfs_clear_fs_compat_ro(fs_info, FREE_SPACE_TREE); + btrfs_clear_fs_compat_ro(fs_info, FREE_SPACE_TREE_VALID); + + ret = clear_free_space_tree(trans, free_space_root); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + btrfs_end_transaction(trans); + return ret; + } + + ret = btrfs_del_root(trans, &free_space_root->root_key); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + btrfs_end_transaction(trans); + return ret; + } + + btrfs_global_root_delete(free_space_root); + + spin_lock(&fs_info->trans_lock); + list_del(&free_space_root->dirty_list); + spin_unlock(&fs_info->trans_lock); + + btrfs_tree_lock(free_space_root->node); + btrfs_clear_buffer_dirty(trans, free_space_root->node); + btrfs_tree_unlock(free_space_root->node); + ret = btrfs_free_tree_block(trans, btrfs_root_id(free_space_root), + free_space_root->node, 0, 1); + btrfs_put_root(free_space_root); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + btrfs_end_transaction(trans); + return ret; + } + + return btrfs_commit_transaction(trans); +} + +int btrfs_rebuild_free_space_tree(struct btrfs_fs_info *fs_info) +{ + struct btrfs_trans_handle *trans; + struct btrfs_key key = { + .objectid = BTRFS_FREE_SPACE_TREE_OBJECTID, + .type = BTRFS_ROOT_ITEM_KEY, + .offset = 0, + }; + struct btrfs_root *free_space_root = btrfs_global_root(fs_info, &key); + struct rb_node *node; + int ret; + + trans = btrfs_start_transaction(free_space_root, 1); + if (IS_ERR(trans)) + return PTR_ERR(trans); + + set_bit(BTRFS_FS_CREATING_FREE_SPACE_TREE, &fs_info->flags); + set_bit(BTRFS_FS_FREE_SPACE_TREE_UNTRUSTED, &fs_info->flags); + + ret = clear_free_space_tree(trans, free_space_root); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + btrfs_end_transaction(trans); + return ret; + } + + node = rb_first_cached(&fs_info->block_group_cache_tree); + while (node) { + struct btrfs_block_group *block_group; + + block_group = rb_entry(node, struct btrfs_block_group, + cache_node); + + if (test_bit(BLOCK_GROUP_FLAG_FREE_SPACE_ADDED, + &block_group->runtime_flags)) + goto next; + + ret = populate_free_space_tree(trans, block_group); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + btrfs_end_transaction(trans); + return ret; + } +next: + if (btrfs_should_end_transaction(trans)) { + btrfs_end_transaction(trans); + trans = btrfs_start_transaction(free_space_root, 1); + if (IS_ERR(trans)) + return PTR_ERR(trans); + } + node = rb_next(node); + } + + btrfs_set_fs_compat_ro(fs_info, FREE_SPACE_TREE); + btrfs_set_fs_compat_ro(fs_info, FREE_SPACE_TREE_VALID); + clear_bit(BTRFS_FS_CREATING_FREE_SPACE_TREE, &fs_info->flags); + + ret = btrfs_commit_transaction(trans); + clear_bit(BTRFS_FS_FREE_SPACE_TREE_UNTRUSTED, &fs_info->flags); + return ret; +} + +static int __add_block_group_free_space(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct btrfs_path *path) +{ + bool own_path = false; + int ret; + + if (!test_and_clear_bit(BLOCK_GROUP_FLAG_NEEDS_FREE_SPACE, + &block_group->runtime_flags)) + return 0; + + /* + * While rebuilding the free space tree we may allocate new metadata + * block groups while modifying the free space tree. + * + * Because during the rebuild (at btrfs_rebuild_free_space_tree()) we + * can use multiple transactions, every time btrfs_end_transaction() is + * called at btrfs_rebuild_free_space_tree() we finish the creation of + * new block groups by calling btrfs_create_pending_block_groups(), and + * that in turn calls us, through btrfs_add_block_group_free_space(), + * to add a free space info item and a free space extent item for the + * block group. + * + * Then later btrfs_rebuild_free_space_tree() may find such new block + * groups and processes them with populate_free_space_tree(), which can + * fail with EEXIST since there are already items for the block group in + * the free space tree. Notice that we say "may find" because a new + * block group may be added to the block groups rbtree in a node before + * or after the block group currently being processed by the rebuild + * process. So signal the rebuild process to skip such new block groups + * if it finds them. + */ + set_bit(BLOCK_GROUP_FLAG_FREE_SPACE_ADDED, &block_group->runtime_flags); + + if (!path) { + path = btrfs_alloc_path(); + if (unlikely(!path)) { + btrfs_abort_transaction(trans, -ENOMEM); + return -ENOMEM; + } + own_path = true; + } + + ret = add_new_free_space_info(trans, block_group, path); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + + ret = __btrfs_add_to_free_space_tree(trans, block_group, path, + block_group->start, block_group->length); + if (ret) + btrfs_abort_transaction(trans, ret); + +out: + if (own_path) + btrfs_free_path(path); + + return ret; +} + +int btrfs_add_block_group_free_space(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group) +{ + int ret; + + if (!btrfs_fs_compat_ro(trans->fs_info, FREE_SPACE_TREE)) + return 0; + + mutex_lock(&block_group->free_space_lock); + ret = __add_block_group_free_space(trans, block_group, NULL); + mutex_unlock(&block_group->free_space_lock); + return ret; +} + +int btrfs_remove_block_group_free_space(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group) +{ + struct btrfs_root *root = btrfs_free_space_root(block_group); + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key, found_key; + struct extent_buffer *leaf; + u64 start, end; + bool done = false; + int nr; + int ret; + + if (!btrfs_fs_compat_ro(trans->fs_info, FREE_SPACE_TREE)) + return 0; + + if (test_bit(BLOCK_GROUP_FLAG_NEEDS_FREE_SPACE, &block_group->runtime_flags)) { + /* We never added this block group to the free space tree. */ + return 0; + } + + path = btrfs_alloc_path(); + if (unlikely(!path)) { + ret = -ENOMEM; + btrfs_abort_transaction(trans, ret); + return ret; + } + + start = block_group->start; + end = btrfs_block_group_end(block_group); + + key.objectid = end - 1; + key.type = (u8)-1; + key.offset = (u64)-1; + + while (!done) { + ret = btrfs_search_prev_slot(trans, root, &key, path, -1, 1); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + + leaf = path->nodes[0]; + nr = 0; + path->slots[0]++; + while (path->slots[0] > 0) { + btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0] - 1); + + if (found_key.type == BTRFS_FREE_SPACE_INFO_KEY) { + ASSERT(found_key.objectid == block_group->start); + ASSERT(found_key.offset == block_group->length); + done = true; + nr++; + path->slots[0]--; + break; + } else if (found_key.type == BTRFS_FREE_SPACE_EXTENT_KEY || + found_key.type == BTRFS_FREE_SPACE_BITMAP_KEY) { + ASSERT(found_key.objectid >= start); + ASSERT(found_key.objectid < end); + ASSERT(found_key.objectid + found_key.offset <= end); + nr++; + path->slots[0]--; + } else { + btrfs_err(trans->fs_info, "unexpected free space tree key type %u", + found_key.type); + ret = -EUCLEAN; + btrfs_abort_transaction(trans, ret); + return ret; + } + } + + ret = btrfs_del_items(trans, root, path, path->slots[0], nr); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + btrfs_release_path(path); + } + + return 0; +} + +static int validate_free_space_key(struct btrfs_block_group *block_group, + const struct btrfs_key *key, u8 expected_type) +{ + const u64 end = btrfs_block_group_end(block_group); + + if (unlikely(key->type != expected_type)) { + btrfs_err(block_group->fs_info, + "block group %llu has unexpected free space key type %u, expected %u", + block_group->start, key->type, expected_type); + return -EUCLEAN; + } + + if (unlikely(key->objectid + key->offset > end)) { + btrfs_err(block_group->fs_info, + "block group %llu has invalid free space key (%llu %u %llu)", + block_group->start, key->objectid, key->type, + key->offset); + return -EUCLEAN; + } + + return 0; +} + +static int load_free_space_bitmaps(struct btrfs_caching_control *caching_ctl, + struct btrfs_path *path, + u32 expected_extent_count) +{ + struct btrfs_block_group *block_group = caching_ctl->block_group; + struct btrfs_fs_info *fs_info = block_group->fs_info; + struct btrfs_root *root; + struct btrfs_key key; + bool prev_bit_set = false; + /* Initialize to silence GCC. */ + u64 extent_start = 0; + const u64 end = btrfs_block_group_end(block_group); + u64 offset; + u64 total_found = 0; + u32 extent_count = 0; + int ret; + + root = btrfs_free_space_root(block_group); + + while (1) { + ret = btrfs_next_item(root, path); + if (ret < 0) + return ret; + if (ret) + break; + + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + + if (key.type == BTRFS_FREE_SPACE_INFO_KEY) + break; + + ret = validate_free_space_key(block_group, &key, BTRFS_FREE_SPACE_BITMAP_KEY); + if (unlikely(ret)) + return ret; + + offset = key.objectid; + while (offset < key.objectid + key.offset) { + bool bit_set; + + bit_set = btrfs_free_space_test_bit(block_group, path, offset); + if (!prev_bit_set && bit_set) { + extent_start = offset; + } else if (prev_bit_set && !bit_set) { + u64 space_added; + + ret = btrfs_add_new_free_space(block_group, + extent_start, + offset, + &space_added); + if (ret) + return ret; + total_found += space_added; + if (total_found > CACHING_CTL_WAKE_UP) { + total_found = 0; + wake_up(&caching_ctl->wait); + } + extent_count++; + } + prev_bit_set = bit_set; + offset += fs_info->sectorsize; + } + } + if (prev_bit_set) { + ret = btrfs_add_new_free_space(block_group, extent_start, end, NULL); + if (ret) + return ret; + extent_count++; + } + + if (unlikely(extent_count != expected_extent_count)) { + btrfs_err(fs_info, + "incorrect extent count for %llu; counted %u, expected %u", + block_group->start, extent_count, + expected_extent_count); + DEBUG_WARN(); + return -EIO; + } + + return 0; +} + +static int load_free_space_extents(struct btrfs_caching_control *caching_ctl, + struct btrfs_path *path, + u32 expected_extent_count) +{ + struct btrfs_block_group *block_group = caching_ctl->block_group; + struct btrfs_fs_info *fs_info = block_group->fs_info; + struct btrfs_root *root; + struct btrfs_key key; + u64 total_found = 0; + u32 extent_count = 0; + int ret; + + root = btrfs_free_space_root(block_group); + + while (1) { + u64 space_added; + + ret = btrfs_next_item(root, path); + if (ret < 0) + return ret; + if (ret) + break; + + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + + if (key.type == BTRFS_FREE_SPACE_INFO_KEY) + break; + + ret = validate_free_space_key(block_group, &key, BTRFS_FREE_SPACE_EXTENT_KEY); + if (unlikely(ret)) + return ret; + + ret = btrfs_add_new_free_space(block_group, key.objectid, + key.objectid + key.offset, + &space_added); + if (ret) + return ret; + total_found += space_added; + if (total_found > CACHING_CTL_WAKE_UP) { + total_found = 0; + wake_up(&caching_ctl->wait); + } + extent_count++; + } + + if (unlikely(extent_count != expected_extent_count)) { + btrfs_err(fs_info, + "incorrect extent count for %llu; counted %u, expected %u", + block_group->start, extent_count, + expected_extent_count); + DEBUG_WARN(); + return -EIO; + } + + return 0; +} + +int btrfs_load_free_space_tree(struct btrfs_caching_control *caching_ctl) +{ + struct btrfs_block_group *block_group; + struct btrfs_free_space_info *info; + BTRFS_PATH_AUTO_FREE(path); + u32 extent_count, flags; + + block_group = caching_ctl->block_group; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + /* + * Just like caching_thread() doesn't want to deadlock on the extent + * tree, we don't want to deadlock on the free space tree. + */ + path->skip_locking = true; + path->search_commit_root = true; + path->reada = READA_FORWARD; + + info = btrfs_search_free_space_info(NULL, block_group, path, 0); + if (IS_ERR(info)) + return PTR_ERR(info); + + extent_count = btrfs_free_space_extent_count(path->nodes[0], info); + flags = btrfs_free_space_flags(path->nodes[0], info); + + /* + * We left path pointing to the free space info item, so now + * load_free_space_foo can just iterate through the free space tree from + * there. + */ + if (flags & BTRFS_FREE_SPACE_USING_BITMAPS) + return load_free_space_bitmaps(caching_ctl, path, extent_count); + else + return load_free_space_extents(caching_ctl, path, extent_count); +} + +static int delete_orphan_free_space_entries(struct btrfs_root *fst_root, + struct btrfs_path *path, + u64 first_bg_bytenr) +{ + struct btrfs_trans_handle *trans; + int ret; + + trans = btrfs_start_transaction(fst_root, 1); + if (IS_ERR(trans)) + return PTR_ERR(trans); + + while (true) { + struct btrfs_key key = { 0 }; + int i; + + ret = btrfs_search_slot(trans, fst_root, &key, path, -1, 1); + if (ret < 0) + break; + ASSERT(ret > 0); + ret = 0; + for (i = 0; i < btrfs_header_nritems(path->nodes[0]); i++) { + btrfs_item_key_to_cpu(path->nodes[0], &key, i); + if (key.objectid >= first_bg_bytenr) { + /* + * Only break the for() loop and continue to + * delete items. + */ + break; + } + } + /* No items to delete, finished. */ + if (i == 0) + break; + + ret = btrfs_del_items(trans, fst_root, path, 0, i); + if (ret < 0) + break; + btrfs_release_path(path); + } + btrfs_release_path(path); + btrfs_end_transaction(trans); + if (ret == 0) + btrfs_info(fst_root->fs_info, "deleted orphan free space tree entries"); + return ret; +} + +/* Remove any free space entry before the first block group. */ +int btrfs_delete_orphan_free_space_entries(struct btrfs_fs_info *fs_info) +{ + BTRFS_PATH_AUTO_RELEASE(path); + struct btrfs_key key = { + .objectid = BTRFS_FREE_SPACE_TREE_OBJECTID, + .type = BTRFS_ROOT_ITEM_KEY, + .offset = 0, + }; + struct btrfs_root *root; + struct btrfs_block_group *bg; + u64 first_bg_bytenr; + int ret; + + /* + * Extent tree v2 has multiple global roots based on the block group. + * This means we cannot easily grab the global free space tree and locate + * orphan items. Furthermore this is still experimental, all users + * should use the latest btrfs-progs anyway. + */ + if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) + return 0; + if (!btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) + return 0; + root = btrfs_global_root(fs_info, &key); + if (!root) + return 0; + + key.objectid = 0; + key.type = 0; + key.offset = 0; + + bg = btrfs_lookup_first_block_group(fs_info, 0); + if (unlikely(!bg)) { + btrfs_err(fs_info, "no block group found"); + return -EUCLEAN; + } + first_bg_bytenr = bg->start; + btrfs_put_block_group(bg); + + ret = btrfs_search_slot(NULL, root, &key, &path, 0, 0); + if (ret < 0) + return ret; + /* There should not be an all-zero key in fst. */ + ASSERT(ret > 0); + + /* Empty free space tree. */ + if (path.slots[0] >= btrfs_header_nritems(path.nodes[0])) + return 0; + + btrfs_item_key_to_cpu(path.nodes[0], &key, path.slots[0]); + if (key.objectid >= first_bg_bytenr) + return 0; + btrfs_release_path(&path); + return delete_orphan_free_space_entries(root, &path, first_bg_bytenr); +} diff --git a/fs/btrfs/free-space-tree.h b/fs/btrfs/free-space-tree.h new file mode 100644 index 000000000..709730e36 --- /dev/null +++ b/fs/btrfs/free-space-tree.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2015 Facebook. All rights reserved. + */ + +#ifndef BTRFS_FREE_SPACE_TREE_H +#define BTRFS_FREE_SPACE_TREE_H + +#include + +struct btrfs_caching_control; +struct btrfs_fs_info; +struct btrfs_path; +struct btrfs_block_group; +struct btrfs_trans_handle; + +/* + * The default size for new free space bitmap items. The last bitmap in a block + * group may be truncated, and none of the free space tree code assumes that + * existing bitmaps are this size. + */ +#define BTRFS_FREE_SPACE_BITMAP_SIZE 256 +#define BTRFS_FREE_SPACE_BITMAP_BITS (BTRFS_FREE_SPACE_BITMAP_SIZE * BITS_PER_BYTE) + +void btrfs_set_free_space_tree_thresholds(struct btrfs_block_group *block_group); +int btrfs_create_free_space_tree(struct btrfs_fs_info *fs_info); +int btrfs_delete_free_space_tree(struct btrfs_fs_info *fs_info); +int btrfs_rebuild_free_space_tree(struct btrfs_fs_info *fs_info); +int btrfs_load_free_space_tree(struct btrfs_caching_control *caching_ctl); +int btrfs_add_block_group_free_space(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group); +int btrfs_remove_block_group_free_space(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group); +int btrfs_add_to_free_space_tree(struct btrfs_trans_handle *trans, + u64 start, u64 size); +int btrfs_remove_from_free_space_tree(struct btrfs_trans_handle *trans, + u64 start, u64 size); +int btrfs_delete_orphan_free_space_entries(struct btrfs_fs_info *fs_info); +struct btrfs_free_space_info * +btrfs_search_free_space_info(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct btrfs_path *path, int cow); +struct btrfs_root *btrfs_free_space_root(struct btrfs_block_group *block_group); + +#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS +int __btrfs_add_to_free_space_tree(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct btrfs_path *path, u64 start, u64 size); +int __btrfs_remove_from_free_space_tree(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct btrfs_path *path, u64 start, u64 size); +int btrfs_convert_free_space_to_bitmaps(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct btrfs_path *path); +int btrfs_convert_free_space_to_extents(struct btrfs_trans_handle *trans, + struct btrfs_block_group *block_group, + struct btrfs_path *path); +bool btrfs_free_space_test_bit(struct btrfs_block_group *block_group, + struct btrfs_path *path, u64 offset); +#endif + +#endif diff --git a/fs/btrfs/fs.c b/fs/btrfs/fs.c new file mode 100644 index 000000000..de160d29d --- /dev/null +++ b/fs/btrfs/fs.c @@ -0,0 +1,344 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include "messages.h" +#include "fs.h" +#include "accessors.h" +#include "volumes.h" + +static const struct btrfs_csums { + u16 size; + const char name[10]; +} btrfs_csums[] = { + [BTRFS_CSUM_TYPE_CRC32] = { .size = 4, .name = "crc32c" }, + [BTRFS_CSUM_TYPE_XXHASH] = { .size = 8, .name = "xxhash64" }, + [BTRFS_CSUM_TYPE_SHA256] = { .size = 32, .name = "sha256" }, + [BTRFS_CSUM_TYPE_BLAKE2] = { .size = 32, .name = "blake2b" }, +}; + +/* This exists for btrfs-progs usages. */ +u16 btrfs_csum_type_size(u16 type) +{ + return btrfs_csums[type].size; +} + +int btrfs_super_csum_size(const struct btrfs_super_block *s) +{ + u16 t = btrfs_super_csum_type(s); + + /* csum type is validated at mount time. */ + return btrfs_csum_type_size(t); +} + +const char *btrfs_super_csum_name(u16 csum_type) +{ + /* csum type is validated at mount time. */ + return btrfs_csums[csum_type].name; +} + +size_t __attribute_const__ btrfs_get_num_csums(void) +{ + return ARRAY_SIZE(btrfs_csums); +} + +void btrfs_csum(u16 csum_type, const u8 *data, size_t len, u8 *out) +{ + switch (csum_type) { + case BTRFS_CSUM_TYPE_CRC32: + put_unaligned_le32(~crc32c(~0, data, len), out); + break; + case BTRFS_CSUM_TYPE_XXHASH: + put_unaligned_le64(xxh64(data, len, 0), out); + break; + case BTRFS_CSUM_TYPE_SHA256: + sha256(data, len, out); + break; + case BTRFS_CSUM_TYPE_BLAKE2: + blake2b(NULL, 0, data, len, out, 32); + break; + default: + /* Checksum type is validated at mount time. */ + BUG(); + } +} + +void btrfs_csum_init(struct btrfs_csum_ctx *ctx, u16 csum_type) +{ + ctx->csum_type = csum_type; + switch (ctx->csum_type) { + case BTRFS_CSUM_TYPE_CRC32: + ctx->crc32 = ~0; + break; + case BTRFS_CSUM_TYPE_XXHASH: + xxh64_reset(&ctx->xxh64, 0); + break; + case BTRFS_CSUM_TYPE_SHA256: + sha256_init(&ctx->sha256); + break; + case BTRFS_CSUM_TYPE_BLAKE2: + blake2b_init(&ctx->blake2b, 32); + break; + default: + /* Checksume type is validated at mount time. */ + BUG(); + } +} + +void btrfs_csum_update(struct btrfs_csum_ctx *ctx, const u8 *data, size_t len) +{ + switch (ctx->csum_type) { + case BTRFS_CSUM_TYPE_CRC32: + ctx->crc32 = crc32c(ctx->crc32, data, len); + break; + case BTRFS_CSUM_TYPE_XXHASH: + xxh64_update(&ctx->xxh64, data, len); + break; + case BTRFS_CSUM_TYPE_SHA256: + sha256_update(&ctx->sha256, data, len); + break; + case BTRFS_CSUM_TYPE_BLAKE2: + blake2b_update(&ctx->blake2b, data, len); + break; + default: + /* Checksum type is validated at mount time. */ + BUG(); + } +} + +void btrfs_csum_final(struct btrfs_csum_ctx *ctx, u8 *out) +{ + switch (ctx->csum_type) { + case BTRFS_CSUM_TYPE_CRC32: + put_unaligned_le32(~ctx->crc32, out); + break; + case BTRFS_CSUM_TYPE_XXHASH: + put_unaligned_le64(xxh64_digest(&ctx->xxh64), out); + break; + case BTRFS_CSUM_TYPE_SHA256: + sha256_final(&ctx->sha256, out); + break; + case BTRFS_CSUM_TYPE_BLAKE2: + blake2b_final(&ctx->blake2b, out); + break; + default: + /* Checksum type is validated at mount time. */ + BUG(); + } +} + +/* + * For regular builds, any block size <= page size is supported. + * For experimental builds, any block size between BTRFS_MIN_BLOCKSIZE + * and BTRFS_MAX_BLOCKSIZE (inclusive) is supported. + */ +bool __attribute_const__ btrfs_supported_blocksize(u32 blocksize) +{ + /* @blocksize should be validated first. */ + ASSERT(is_power_of_2(blocksize) && blocksize >= BTRFS_MIN_BLOCKSIZE && + blocksize <= BTRFS_MAX_BLOCKSIZE); + + if (blocksize <= PAGE_SIZE) + return true; +#ifdef CONFIG_BTRFS_EXPERIMENTAL + /* + * For bs > ps support it's done by specifying a minimal folio order + * for filemap, thus implying large data folios. + * For HIGHMEM systems, we can not always access the content of a (large) + * folio in one go, but go through them page by page. + * + * A lot of features don't implement a proper PAGE sized loop for large + * folios, this includes: + * + * - compression + * - verity + * - encoded write + * + * Considering HIGHMEM is such a pain to deal with and it's going + * to be deprecated eventually, just reject HIGHMEM && bs > ps cases. + * + * Finally, for bs > ps cases, we need to set the minimal folio order, + * which requires transparent hugepage. + */ + if (blocksize > PAGE_SIZE) { + if (IS_ENABLED(CONFIG_HIGHMEM)) + return false; + + if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) + return false; + } + return true; +#endif + return false; +} + +/* + * Start exclusive operation @type, return true on success. + */ +bool btrfs_exclop_start(struct btrfs_fs_info *fs_info, + enum btrfs_exclusive_operation type) +{ + bool ret = false; + + spin_lock(&fs_info->super_lock); + if (fs_info->exclusive_operation == BTRFS_EXCLOP_NONE) { + fs_info->exclusive_operation = type; + ret = true; + } + spin_unlock(&fs_info->super_lock); + + return ret; +} + +/* + * Conditionally allow to enter the exclusive operation in case it's compatible + * with the running one. This must be paired with btrfs_exclop_start_unlock() + * and btrfs_exclop_finish(). + * + * Compatibility: + * - the same type is already running + * - when trying to add a device and balance has been paused + * - not BTRFS_EXCLOP_NONE - this is intentionally incompatible and the caller + * must check the condition first that would allow none -> @type + */ +bool btrfs_exclop_start_try_lock(struct btrfs_fs_info *fs_info, + enum btrfs_exclusive_operation type) +{ + spin_lock(&fs_info->super_lock); + if (fs_info->exclusive_operation == type || + (fs_info->exclusive_operation == BTRFS_EXCLOP_BALANCE_PAUSED && + type == BTRFS_EXCLOP_DEV_ADD)) + return true; + + spin_unlock(&fs_info->super_lock); + return false; +} + +void btrfs_exclop_start_unlock(struct btrfs_fs_info *fs_info) +{ + spin_unlock(&fs_info->super_lock); +} + +void btrfs_exclop_finish(struct btrfs_fs_info *fs_info) +{ + spin_lock(&fs_info->super_lock); + WRITE_ONCE(fs_info->exclusive_operation, BTRFS_EXCLOP_NONE); + spin_unlock(&fs_info->super_lock); + sysfs_notify(&fs_info->fs_devices->fsid_kobj, NULL, "exclusive_operation"); +} + +void btrfs_exclop_balance(struct btrfs_fs_info *fs_info, + enum btrfs_exclusive_operation op) +{ + switch (op) { + case BTRFS_EXCLOP_BALANCE_PAUSED: + spin_lock(&fs_info->super_lock); + ASSERT(fs_info->exclusive_operation == BTRFS_EXCLOP_BALANCE || + fs_info->exclusive_operation == BTRFS_EXCLOP_DEV_ADD || + fs_info->exclusive_operation == BTRFS_EXCLOP_NONE || + fs_info->exclusive_operation == BTRFS_EXCLOP_BALANCE_PAUSED); + fs_info->exclusive_operation = BTRFS_EXCLOP_BALANCE_PAUSED; + spin_unlock(&fs_info->super_lock); + break; + case BTRFS_EXCLOP_BALANCE: + spin_lock(&fs_info->super_lock); + ASSERT(fs_info->exclusive_operation == BTRFS_EXCLOP_BALANCE_PAUSED); + fs_info->exclusive_operation = BTRFS_EXCLOP_BALANCE; + spin_unlock(&fs_info->super_lock); + break; + default: + btrfs_warn(fs_info, + "invalid exclop balance operation %d requested", op); + } +} + +void __btrfs_set_fs_incompat(struct btrfs_fs_info *fs_info, u64 flag, + const char *name) +{ + struct btrfs_super_block *disk_super; + u64 features; + + disk_super = fs_info->super_copy; + features = btrfs_super_incompat_flags(disk_super); + if (!(features & flag)) { + spin_lock(&fs_info->super_lock); + features = btrfs_super_incompat_flags(disk_super); + if (!(features & flag)) { + features |= flag; + btrfs_set_super_incompat_flags(disk_super, features); + btrfs_info(fs_info, + "setting incompat feature flag for %s (0x%llx)", + name, flag); + } + spin_unlock(&fs_info->super_lock); + set_bit(BTRFS_FS_FEATURE_CHANGED, &fs_info->flags); + } +} + +void __btrfs_clear_fs_incompat(struct btrfs_fs_info *fs_info, u64 flag, + const char *name) +{ + struct btrfs_super_block *disk_super; + u64 features; + + disk_super = fs_info->super_copy; + features = btrfs_super_incompat_flags(disk_super); + if (features & flag) { + spin_lock(&fs_info->super_lock); + features = btrfs_super_incompat_flags(disk_super); + if (features & flag) { + features &= ~flag; + btrfs_set_super_incompat_flags(disk_super, features); + btrfs_info(fs_info, + "clearing incompat feature flag for %s (0x%llx)", + name, flag); + } + spin_unlock(&fs_info->super_lock); + set_bit(BTRFS_FS_FEATURE_CHANGED, &fs_info->flags); + } +} + +void __btrfs_set_fs_compat_ro(struct btrfs_fs_info *fs_info, u64 flag, + const char *name) +{ + struct btrfs_super_block *disk_super; + u64 features; + + disk_super = fs_info->super_copy; + features = btrfs_super_compat_ro_flags(disk_super); + if (!(features & flag)) { + spin_lock(&fs_info->super_lock); + features = btrfs_super_compat_ro_flags(disk_super); + if (!(features & flag)) { + features |= flag; + btrfs_set_super_compat_ro_flags(disk_super, features); + btrfs_info(fs_info, + "setting compat-ro feature flag for %s (0x%llx)", + name, flag); + } + spin_unlock(&fs_info->super_lock); + set_bit(BTRFS_FS_FEATURE_CHANGED, &fs_info->flags); + } +} + +void __btrfs_clear_fs_compat_ro(struct btrfs_fs_info *fs_info, u64 flag, + const char *name) +{ + struct btrfs_super_block *disk_super; + u64 features; + + disk_super = fs_info->super_copy; + features = btrfs_super_compat_ro_flags(disk_super); + if (features & flag) { + spin_lock(&fs_info->super_lock); + features = btrfs_super_compat_ro_flags(disk_super); + if (features & flag) { + features &= ~flag; + btrfs_set_super_compat_ro_flags(disk_super, features); + btrfs_info(fs_info, + "clearing compat-ro feature flag for %s (0x%llx)", + name, flag); + } + spin_unlock(&fs_info->super_lock); + set_bit(BTRFS_FS_FEATURE_CHANGED, &fs_info->flags); + } +} diff --git a/fs/btrfs/fs.h b/fs/btrfs/fs.h new file mode 100644 index 000000000..10e15a319 --- /dev/null +++ b/fs/btrfs/fs.h @@ -0,0 +1,1263 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_FS_H +#define BTRFS_FS_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "extent-io-tree.h" +#include "async-thread.h" +#include "block-rsv.h" +#include "messages.h" + +struct inode; +struct super_block; +struct kobject; +struct reloc_control; +struct ulist; +struct btrfs_device; +struct btrfs_block_group; +struct btrfs_root; +struct btrfs_fs_devices; +struct btrfs_transaction; +struct btrfs_balance_control; +struct btrfs_subpage_info; +struct btrfs_stripe_hash_table; +struct btrfs_space_info; + +/* Minimum data and metadata block size. */ +#define BTRFS_MIN_BLOCKSIZE (SZ_4K) +#define BTRFS_MAX_BLOCKSIZE (SZ_64K) + +/* The maximum folio size btrfs supports. */ +#define BTRFS_MAX_FOLIO_SIZE (SZ_2M) +static_assert(BTRFS_MAX_FOLIO_SIZE > PAGE_SIZE); + +/* + * The maximum number of blocks a huge folio can support. + * + * Depending on the filesystem block size, the real maximum blocks per folio + * may also be limited by the above BTRFS_MAX_FOLIO_SIZE. + */ +#ifdef CONFIG_BTRFS_EXPERIMENTAL +#define BTRFS_MAX_BLOCKS_PER_FOLIO (512) +#else +#define BTRFS_MAX_BLOCKS_PER_FOLIO (BITS_PER_LONG) +#endif + +#define BTRFS_MAX_EXTENT_SIZE SZ_128M + +/* + * Maximum length to trim in a single iteration to avoid holding device list + * mutex for too long. + */ +#define BTRFS_MAX_TRIM_LENGTH SZ_2G + +#define BTRFS_OLDEST_GENERATION 0ULL + +#define BTRFS_EMPTY_DIR_SIZE 0 + +#define BTRFS_DIRTY_METADATA_THRESH SZ_32M + +#define BTRFS_SUPER_INFO_OFFSET SZ_64K +#define BTRFS_SUPER_INFO_SIZE 4096 +static_assert(sizeof(struct btrfs_super_block) == BTRFS_SUPER_INFO_SIZE); + +/* Array of bytes with variable length, hexadecimal format 0x1234 */ +#define BTRFS_CSUM_FMT "0x%*phN" +#define BTRFS_CSUM_FMT_VALUE(size, bytes) size, bytes + +#define BTRFS_KEY_FMT "(%llu %u %llu)" +#define BTRFS_KEY_FMT_VALUE(key) (key)->objectid, (key)->type, (key)->offset + +#define BTRFS_QGROUP_FMT "%hu/%llu" +#define BTRFS_QGROUP_FMT_VALUE(qgroup) btrfs_qgroup_level((qgroup)->qgroupid), \ + btrfs_qgroup_subvolid((qgroup)->qgroupid) + +/* + * Number of metadata items necessary for an unlink operation: + * + * 1 for the possible orphan item + * 1 for the dir item + * 1 for the dir index + * 1 for the inode ref + * 1 for the inode + * 1 for the parent inode + */ +#define BTRFS_UNLINK_METADATA_UNITS 6 + +/* + * The reserved space at the beginning of each device. It covers the primary + * super block and leaves space for potential use by other tools like + * bootloaders or to lower potential damage of accidental overwrite. + */ +#define BTRFS_DEVICE_RANGE_RESERVED (SZ_1M) +/* + * Runtime (in-memory) states of filesystem + */ +enum { + /* + * Filesystem is being remounted, allow to skip some operations, like + * defrag + */ + BTRFS_FS_STATE_REMOUNTING, + /* Filesystem in RO mode */ + BTRFS_FS_STATE_RO, + /* Track if a transaction abort has been reported on this filesystem */ + BTRFS_FS_STATE_TRANS_ABORTED, + /* Track if log replay has failed. */ + BTRFS_FS_STATE_LOG_REPLAY_ABORTED, + /* + * Bio operations should be blocked on this filesystem because a source + * or target device is being destroyed as part of a device replace + */ + BTRFS_FS_STATE_DEV_REPLACING, + /* The btrfs_fs_info created for self-tests */ + BTRFS_FS_STATE_DUMMY_FS_INFO, + + /* Checksum errors are ignored. */ + BTRFS_FS_STATE_NO_DATA_CSUMS, + BTRFS_FS_STATE_SKIP_META_CSUMS, + + /* Indicates there was an error cleaning up a log tree. */ + BTRFS_FS_STATE_LOG_CLEANUP_ERROR, + + /* No more delayed iput can be queued. */ + BTRFS_FS_STATE_NO_DELAYED_IPUT, + + /* + * Emergency shutdown, a step further than transaction aborted by + * rejecting all operations. + */ + BTRFS_FS_STATE_EMERGENCY_SHUTDOWN, + + BTRFS_FS_STATE_COUNT +}; + +enum { + BTRFS_FS_CLOSING_START, + BTRFS_FS_CLOSING_DONE, + BTRFS_FS_LOG_RECOVERING, + BTRFS_FS_OPEN, + BTRFS_FS_QUOTA_ENABLED, + BTRFS_FS_SQUOTA_ENABLING, + BTRFS_FS_UPDATE_UUID_TREE_GEN, + BTRFS_FS_CREATING_FREE_SPACE_TREE, + BTRFS_FS_BTREE_ERR, + BTRFS_FS_LOG1_ERR, + BTRFS_FS_LOG2_ERR, + BTRFS_FS_QUOTA_OVERRIDE, + /* Used to record internally whether fs has been frozen */ + BTRFS_FS_FROZEN, + /* + * Indicate that balance has been set up from the ioctl and is in the + * main phase. The fs_info::balance_ctl is initialized. + */ + BTRFS_FS_BALANCE_RUNNING, + + /* + * Indicate that relocation of a chunk has started, it's set per chunk + * and is toggled between chunks. + */ + BTRFS_FS_RELOC_RUNNING, + + /* Indicate that the cleaner thread is awake and doing something. */ + BTRFS_FS_CLEANER_RUNNING, + + /* + * The checksumming has an optimized version and is considered fast, + * so we don't need to offload checksums to workqueues. + */ + BTRFS_FS_CSUM_IMPL_FAST, + + /* Indicate that the discard workqueue can service discards. */ + BTRFS_FS_DISCARD_RUNNING, + + /* Indicate that we need to cleanup space cache v1 */ + BTRFS_FS_CLEANUP_SPACE_CACHE_V1, + + /* Indicate that we can't trust the free space tree for caching yet */ + BTRFS_FS_FREE_SPACE_TREE_UNTRUSTED, + + /* Indicate whether there are any tree modification log users */ + BTRFS_FS_TREE_MOD_LOG_USERS, + + /* Indicate that we want the transaction kthread to commit right now. */ + BTRFS_FS_COMMIT_TRANS, + + /* Indicate we have half completed snapshot deletions pending. */ + BTRFS_FS_UNFINISHED_DROPS, + + /* Indicate we have to finish a zone to do next allocation. */ + BTRFS_FS_NEED_ZONE_FINISH, + + /* Indicate that we want to commit the transaction. */ + BTRFS_FS_NEED_TRANS_COMMIT, + + /* This is set when active zone tracking is needed. */ + BTRFS_FS_ACTIVE_ZONE_TRACKING, + + /* + * Indicate if we have some features changed, this is mostly for + * cleaner thread to update the sysfs interface. + */ + BTRFS_FS_FEATURE_CHANGED, + + /* + * Indicate that we have found a tree block which is only aligned to + * sectorsize, but not to nodesize. This should be rare nowadays. + */ + BTRFS_FS_UNALIGNED_TREE_BLOCK, + +#if BITS_PER_LONG == 32 + /* Indicate if we have error/warn message printed on 32bit systems */ + BTRFS_FS_32BIT_ERROR, + BTRFS_FS_32BIT_WARN, +#endif +}; + +/* + * Flags for mount options. + * + * Note: don't forget to add new options to btrfs_show_options() + */ +enum { + BTRFS_MOUNT_NODATASUM = (1ULL << 0), + BTRFS_MOUNT_NODATACOW = (1ULL << 1), + BTRFS_MOUNT_NOBARRIER = (1ULL << 2), + BTRFS_MOUNT_SSD = (1ULL << 3), + BTRFS_MOUNT_DEGRADED = (1ULL << 4), + BTRFS_MOUNT_COMPRESS = (1ULL << 5), + BTRFS_MOUNT_NOTREELOG = (1ULL << 6), + BTRFS_MOUNT_FLUSHONCOMMIT = (1ULL << 7), + BTRFS_MOUNT_SSD_SPREAD = (1ULL << 8), + BTRFS_MOUNT_NOSSD = (1ULL << 9), + BTRFS_MOUNT_DISCARD_SYNC = (1ULL << 10), + BTRFS_MOUNT_FORCE_COMPRESS = (1ULL << 11), + BTRFS_MOUNT_SPACE_CACHE = (1ULL << 12), + BTRFS_MOUNT_CLEAR_CACHE = (1ULL << 13), + BTRFS_MOUNT_USER_SUBVOL_RM_ALLOWED = (1ULL << 14), + BTRFS_MOUNT_ENOSPC_DEBUG = (1ULL << 15), + BTRFS_MOUNT_AUTO_DEFRAG = (1ULL << 16), + BTRFS_MOUNT_USEBACKUPROOT = (1ULL << 17), + BTRFS_MOUNT_SKIP_BALANCE = (1ULL << 18), + BTRFS_MOUNT_PANIC_ON_FATAL_ERROR = (1ULL << 19), + BTRFS_MOUNT_RESCAN_UUID_TREE = (1ULL << 20), + BTRFS_MOUNT_FRAGMENT_DATA = (1ULL << 21), + BTRFS_MOUNT_FRAGMENT_METADATA = (1ULL << 22), + BTRFS_MOUNT_FREE_SPACE_TREE = (1ULL << 23), + BTRFS_MOUNT_NOLOGREPLAY = (1ULL << 24), + BTRFS_MOUNT_REF_VERIFY = (1ULL << 25), + BTRFS_MOUNT_DISCARD_ASYNC = (1ULL << 26), + BTRFS_MOUNT_IGNOREBADROOTS = (1ULL << 27), + BTRFS_MOUNT_IGNOREDATACSUMS = (1ULL << 28), + BTRFS_MOUNT_NODISCARD = (1ULL << 29), + BTRFS_MOUNT_NOSPACECACHE = (1ULL << 30), + BTRFS_MOUNT_IGNOREMETACSUMS = (1ULL << 31), + BTRFS_MOUNT_IGNORESUPERFLAGS = (1ULL << 32), + BTRFS_MOUNT_REF_TRACKER = (1ULL << 33), +}; + +/* These mount options require a full read-only fs, no new transaction is allowed. */ +#define BTRFS_MOUNT_FULL_RO_MASK \ + (BTRFS_MOUNT_NOLOGREPLAY | \ + BTRFS_MOUNT_IGNOREBADROOTS | \ + BTRFS_MOUNT_IGNOREDATACSUMS | \ + BTRFS_MOUNT_IGNOREMETACSUMS | \ + BTRFS_MOUNT_IGNORESUPERFLAGS | \ + BTRFS_MOUNT_USEBACKUPROOT) + +/* + * Compat flags that we support. If any incompat flags are set other than the + * ones specified below then we will fail to mount + */ +#define BTRFS_FEATURE_COMPAT_SUPP 0ULL +#define BTRFS_FEATURE_COMPAT_SAFE_SET 0ULL +#define BTRFS_FEATURE_COMPAT_SAFE_CLEAR 0ULL + +#define BTRFS_FEATURE_COMPAT_RO_SUPP \ + (BTRFS_FEATURE_COMPAT_RO_FREE_SPACE_TREE | \ + BTRFS_FEATURE_COMPAT_RO_FREE_SPACE_TREE_VALID | \ + BTRFS_FEATURE_COMPAT_RO_VERITY | \ + BTRFS_FEATURE_COMPAT_RO_BLOCK_GROUP_TREE) + +#define BTRFS_FEATURE_COMPAT_RO_SAFE_SET 0ULL +#define BTRFS_FEATURE_COMPAT_RO_SAFE_CLEAR 0ULL + +#define BTRFS_FEATURE_INCOMPAT_SUPP_STABLE \ + (BTRFS_FEATURE_INCOMPAT_MIXED_BACKREF | \ + BTRFS_FEATURE_INCOMPAT_DEFAULT_SUBVOL | \ + BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS | \ + BTRFS_FEATURE_INCOMPAT_BIG_METADATA | \ + BTRFS_FEATURE_INCOMPAT_COMPRESS_LZO | \ + BTRFS_FEATURE_INCOMPAT_COMPRESS_ZSTD | \ + BTRFS_FEATURE_INCOMPAT_RAID56 | \ + BTRFS_FEATURE_INCOMPAT_EXTENDED_IREF | \ + BTRFS_FEATURE_INCOMPAT_SKINNY_METADATA | \ + BTRFS_FEATURE_INCOMPAT_NO_HOLES | \ + BTRFS_FEATURE_INCOMPAT_METADATA_UUID | \ + BTRFS_FEATURE_INCOMPAT_RAID1C34 | \ + BTRFS_FEATURE_INCOMPAT_ZONED | \ + BTRFS_FEATURE_INCOMPAT_SIMPLE_QUOTA) + +#ifdef CONFIG_BTRFS_EXPERIMENTAL + /* + * Features under development like Extent tree v2 support is enabled + * only under CONFIG_BTRFS_EXPERIMENTAL + */ +#define BTRFS_FEATURE_INCOMPAT_SUPP \ + (BTRFS_FEATURE_INCOMPAT_SUPP_STABLE | \ + BTRFS_FEATURE_INCOMPAT_RAID_STRIPE_TREE | \ + BTRFS_FEATURE_INCOMPAT_EXTENT_TREE_V2 | \ + BTRFS_FEATURE_INCOMPAT_REMAP_TREE) + +#else + +#define BTRFS_FEATURE_INCOMPAT_SUPP \ + (BTRFS_FEATURE_INCOMPAT_SUPP_STABLE) + +#endif + +#define BTRFS_FEATURE_INCOMPAT_SAFE_SET \ + (BTRFS_FEATURE_INCOMPAT_EXTENDED_IREF) +#define BTRFS_FEATURE_INCOMPAT_SAFE_CLEAR 0ULL + +#define BTRFS_DEFAULT_COMMIT_INTERVAL (30) +#define BTRFS_WARNING_COMMIT_INTERVAL (300) +#define BTRFS_DEFAULT_MAX_INLINE (2048) + +enum btrfs_compression_type { + BTRFS_COMPRESS_NONE = 0, + BTRFS_COMPRESS_ZLIB = 1, + BTRFS_COMPRESS_LZO = 2, + BTRFS_COMPRESS_ZSTD = 3, + BTRFS_NR_COMPRESS_TYPES = 4, + + BTRFS_DEFRAG_DONT_COMPRESS, +}; + +struct btrfs_dev_replace { + /* See #define above */ + u64 replace_state; + /* Seconds since 1-Jan-1970 */ + time64_t time_started; + /* Seconds since 1-Jan-1970 */ + time64_t time_stopped; + atomic64_t num_write_errors; + atomic64_t num_uncorrectable_read_errors; + + u64 cursor_left; + u64 committed_cursor_left; + u64 cursor_left_last_write_of_item; + u64 cursor_right; + + /* See #define above */ + u64 cont_reading_from_srcdev_mode; + + int is_valid; + int item_needs_writeback; + struct btrfs_device *srcdev; + struct btrfs_device *tgtdev; + + struct mutex lock_finishing_cancel_unmount; + struct rw_semaphore rwsem; + + struct btrfs_scrub_progress scrub_progress; + + struct percpu_counter bio_counter; + wait_queue_head_t replace_wait; + + struct task_struct *replace_task; +}; + +/* + * Free clusters are used to claim free space in relatively large chunks, + * allowing us to do less seeky writes. They are used for all metadata + * allocations. In ssd_spread mode they are also used for data allocations. + */ +struct btrfs_free_cluster { + spinlock_t lock; + spinlock_t refill_lock; + struct rb_root root; + + /* Largest extent in this cluster */ + u64 max_size; + + /* First extent starting offset */ + u64 window_start; + + /* We did a full search and couldn't create a cluster */ + bool fragmented; + + struct btrfs_block_group *block_group; + /* + * When a cluster is allocated from a block group, we put the cluster + * onto a list in the block group so that it can be freed before the + * block group is freed. + */ + struct list_head block_group_list; +}; + +/* Discard control. */ +/* + * Async discard uses multiple lists to differentiate the discard filter + * parameters. Index 0 is for completely free block groups where we need to + * ensure the entire block group is trimmed without being lossy. Indices + * afterwards represent monotonically decreasing discard filter sizes to + * prioritize what should be discarded next. + */ +#define BTRFS_NR_DISCARD_LISTS 3 +#define BTRFS_DISCARD_INDEX_UNUSED 0 +#define BTRFS_DISCARD_INDEX_START 1 + +struct btrfs_discard_ctl { + struct workqueue_struct *discard_workers; + struct delayed_work work; + spinlock_t lock; + struct btrfs_block_group *block_group; + struct list_head discard_list[BTRFS_NR_DISCARD_LISTS]; + u64 prev_discard; + u64 prev_discard_time; + atomic_t discardable_extents; + atomic64_t discardable_bytes; + u64 max_discard_size; + u64 delay_ms; + u32 iops_limit; + u32 kbps_limit; + u64 discard_extent_bytes; + u64 discard_bitmap_bytes; + atomic64_t discard_bytes_saved; +}; + +/* + * Exclusive operations (device replace, resize, device add/remove, balance) + */ +enum btrfs_exclusive_operation { + BTRFS_EXCLOP_NONE, + BTRFS_EXCLOP_BALANCE_PAUSED, + BTRFS_EXCLOP_BALANCE, + BTRFS_EXCLOP_DEV_ADD, + BTRFS_EXCLOP_DEV_REMOVE, + BTRFS_EXCLOP_DEV_REPLACE, + BTRFS_EXCLOP_RESIZE, + BTRFS_EXCLOP_SWAP_ACTIVATE, +}; + +/* Store data about transaction commits, exported via sysfs. */ +struct btrfs_commit_stats { + /* Total number of commits */ + u64 commit_count; + /* The maximum commit duration so far in ns */ + u64 max_commit_dur; + /* The last commit duration in ns */ + u64 last_commit_dur; + /* The total commit duration in ns */ + u64 total_commit_dur; + /* Start of the last critical section in ns. */ + u64 critical_section_start_time; +}; + +struct btrfs_delayed_root { + spinlock_t lock; + int nodes; /* for delayed nodes */ + struct list_head node_list; + /* + * Used for delayed nodes which is waiting to be dealt with by the + * worker. If the delayed node is inserted into the work queue, we + * drop it from this list. + */ + struct list_head prepare_list; + atomic_t items; /* for delayed items */ + atomic_t items_seq; /* for delayed items */ + wait_queue_head_t wait; +}; + +struct btrfs_free_space_ctl; +struct btrfs_free_space; + +struct btrfs_fs_info { + u8 chunk_tree_uuid[BTRFS_UUID_SIZE]; + unsigned long flags; + struct btrfs_root *tree_root; + struct btrfs_root *chunk_root; + struct btrfs_root *dev_root; + struct btrfs_root *fs_root; + struct btrfs_root *quota_root; + struct btrfs_root *uuid_root; + struct btrfs_root *data_reloc_root; + struct btrfs_root *block_group_root; + struct btrfs_root *stripe_root; + struct btrfs_root *remap_root; + + /* The log root tree is a directory of all the other log roots */ + struct btrfs_root *log_root_tree; + + /* The tree that holds the global roots (csum, extent, etc) */ + rwlock_t global_root_lock; + struct rb_root global_root_tree; + + spinlock_t fs_roots_radix_lock; + struct radix_tree_root fs_roots_radix; + + /* Block group cache stuff */ + rwlock_t block_group_cache_lock; + struct rb_root_cached block_group_cache_tree; + + /* Keep track of unallocated space */ + atomic64_t free_chunk_space; + + /* Track ranges which are used by log trees blocks/logged data extents */ + struct extent_io_tree excluded_extents; + + /* logical->physical extent mapping */ + struct rb_root_cached mapping_tree; + rwlock_t mapping_tree_lock; + + /* + * Block reservation for extent, checksum, root tree and delayed dir + * index item. + */ + struct btrfs_block_rsv global_block_rsv; + /* Block reservation for metadata operations */ + struct btrfs_block_rsv trans_block_rsv; + /* Block reservation for chunk tree */ + struct btrfs_block_rsv chunk_block_rsv; + /* Block reservation for remap tree. */ + struct btrfs_block_rsv remap_block_rsv; + /* Block reservation for delayed operations */ + struct btrfs_block_rsv delayed_block_rsv; + /* Block reservation for delayed refs */ + struct btrfs_block_rsv delayed_refs_rsv; + /* Block reservation for treelog tree */ + struct btrfs_block_rsv treelog_rsv; + + struct btrfs_block_rsv empty_block_rsv; + + /* + * Updated while holding the lock 'trans_lock'. Due to the life cycle of + * a transaction, it can be directly read while holding a transaction + * handle, everywhere else must be read with btrfs_get_fs_generation(). + * Should always be updated using btrfs_set_fs_generation(). + */ + u64 generation; + /* + * Always use btrfs_get_last_trans_committed() and + * btrfs_set_last_trans_committed() to read and update this field. + */ + u64 last_trans_committed; + /* + * Generation of the last transaction used for block group relocation + * since the filesystem was last mounted (or 0 if none happened yet). + * Must be written and read while holding btrfs_fs_info::commit_root_sem. + */ + u64 last_reloc_trans; + + /* + * This is updated to the current trans every time a full commit is + * required instead of the faster short fsync log commits + */ + u64 last_trans_log_full_commit; + unsigned long long mount_opt; + + /* Compress related structures. */ + void *compr_wsm[BTRFS_NR_COMPRESS_TYPES]; + + int compress_type; + int compress_level; + u32 commit_interval; + /* + * It is a suggestive number, the read side is safe even it gets a + * wrong number because we will write out the data into a regular + * extent. The write side(mount/remount) is under ->s_umount lock, + * so it is also safe. + */ + u64 max_inline; + + struct btrfs_transaction *running_transaction; + wait_queue_head_t transaction_throttle; + wait_queue_head_t transaction_wait; + wait_queue_head_t transaction_blocked_wait; + wait_queue_head_t async_submit_wait; + + /* + * Used to protect the incompat_flags, compat_flags, compat_ro_flags + * when they are updated. + * + * Because we do not clear the flags for ever, so we needn't use + * the lock on the read side. + * + * We also needn't use the lock when we mount the fs, because + * there is no other task which will update the flag. + */ + spinlock_t super_lock; + struct btrfs_super_block *super_copy; + struct btrfs_super_block *super_for_commit; + struct super_block *sb; + struct inode *btree_inode; + struct mutex tree_log_mutex; + struct mutex transaction_kthread_mutex; + struct mutex cleaner_mutex; + struct mutex chunk_mutex; + struct mutex remap_mutex; + + /* + * This is taken to make sure we don't set block groups ro after the + * free space cache has been allocated on them. + */ + struct mutex ro_block_group_mutex; + + /* + * This is used during read/modify/write to make sure no two ios are + * trying to mod the same stripe at the same time. + */ + struct btrfs_stripe_hash_table *stripe_hash_table; + + /* + * This protects the ordered operations list only while we are + * processing all of the entries on it. This way we make sure the + * commit code doesn't find the list temporarily empty because another + * function happens to be doing non-waiting preflush before jumping + * into the main commit. + */ + struct mutex ordered_operations_mutex; + + struct rw_semaphore commit_root_sem; + + struct rw_semaphore cleanup_work_sem; + + struct rw_semaphore subvol_sem; + + spinlock_t trans_lock; + /* + * The reloc mutex goes with the trans lock, it is taken during commit + * to protect us from the relocation code. + */ + struct mutex reloc_mutex; + /* Protects setting, clearing and getting fs_info->reloc_ctl. */ + spinlock_t reloc_ctl_lock; + + struct list_head trans_list; + struct list_head dead_roots; + struct list_head caching_block_groups; + + spinlock_t delayed_iput_lock; + struct list_head delayed_iputs; + atomic_t nr_delayed_iputs; + wait_queue_head_t delayed_iputs_wait; + + atomic64_t tree_mod_seq; + + /* This protects tree_mod_log and tree_mod_seq_list */ + rwlock_t tree_mod_log_lock; + struct rb_root tree_mod_log; + struct list_head tree_mod_seq_list; + + atomic_t async_delalloc_pages; + + /* This is used to protect the following list -- ordered_roots. */ + spinlock_t ordered_root_lock; + + /* + * All fs/file tree roots in which there are data=ordered extents + * pending writeback are added into this list. + * + * These can span multiple transactions and basically include every + * dirty data page that isn't from nodatacow. + */ + struct list_head ordered_roots; + + struct mutex delalloc_root_mutex; + spinlock_t delalloc_root_lock; + /* All fs/file tree roots that have delalloc inodes. */ + struct list_head delalloc_roots; + + /* + * There is a pool of worker threads for checksumming during writes and + * a pool for checksumming after reads. This is because readers can + * run with FS locks held, and the writers may be waiting for those + * locks. We don't want ordering in the pending list to cause + * deadlocks, and so the two are serviced separately. + * + * A third pool does submit_bio to avoid deadlocking with the other two. + */ + struct btrfs_workqueue *workers; + struct btrfs_workqueue *delalloc_workers; + struct btrfs_workqueue *flush_workers; + struct workqueue_struct *endio_workers; + struct workqueue_struct *endio_meta_workers; + struct workqueue_struct *rmw_workers; + struct btrfs_workqueue *endio_write_workers; + struct btrfs_workqueue *endio_freespace_worker; + struct btrfs_workqueue *caching_workers; + + struct workqueue_struct *fixup_workers; + struct btrfs_workqueue *delayed_workers; + + struct task_struct *transaction_kthread; + struct task_struct *cleaner_kthread; + u32 thread_pool_size; + + struct kobject *space_info_kobj; + struct kobject *qgroups_kobj; + struct kobject *discard_kobj; + + /* Track the number of blocks (sectors) read by the filesystem. */ + struct percpu_counter stats_read_blocks; + + /* Used to keep from writing metadata until there is a nice batch */ + struct percpu_counter dirty_metadata_bytes; + struct percpu_counter delalloc_bytes; + struct percpu_counter ordered_bytes; + s32 dirty_metadata_batch; + s32 delalloc_batch; + + struct percpu_counter evictable_extent_maps; + u64 em_shrinker_last_root; + u64 em_shrinker_last_ino; + atomic64_t em_shrinker_nr_to_scan; + struct work_struct em_shrinker_work; + + /* Protected by 'trans_lock'. */ + struct list_head dirty_cowonly_roots; + + struct btrfs_fs_devices *fs_devices; + + /* + * The space_info list is effectively read only after initial setup. + * It is populated at mount time and cleaned up after all block groups + * are removed. RCU is used to protect it. + */ + struct list_head space_info; + + struct btrfs_space_info *data_sinfo; + + struct reloc_control *reloc_ctl; + + /* data_alloc_cluster is only used in ssd_spread mode */ + struct btrfs_free_cluster data_alloc_cluster; + + /* All metadata allocations go through this cluster. */ + struct btrfs_free_cluster meta_alloc_cluster; + + /* Auto defrag inodes go here. */ + spinlock_t defrag_inodes_lock; + struct rb_root defrag_inodes; + atomic_t defrag_running; + + /* Used to protect avail_{data, metadata, system}_alloc_bits */ + seqlock_t profiles_lock; + /* + * These three are in extended format (availability of single chunks is + * denoted by BTRFS_AVAIL_ALLOC_BIT_SINGLE bit, other types are denoted + * by corresponding BTRFS_BLOCK_GROUP_* bits) + */ + u64 avail_data_alloc_bits; + u64 avail_metadata_alloc_bits; + u64 avail_system_alloc_bits; + + /* Balance state */ + spinlock_t balance_lock; + struct mutex balance_mutex; + atomic_t balance_pause_req; + atomic_t balance_cancel_req; + struct btrfs_balance_control *balance_ctl; + wait_queue_head_t balance_wait_q; + + /* Cancellation requests for chunk relocation */ + atomic_t reloc_cancel_req; + + u32 data_chunk_allocations; + u32 metadata_ratio; + + /* Private scrub information */ + struct mutex scrub_lock; + atomic_t scrubs_running; + atomic_t scrub_pause_req; + atomic_t scrubs_paused; + atomic_t scrub_cancel_req; + wait_queue_head_t scrub_pause_wait; + /* + * The worker pointers are NULL iff the refcount is 0, ie. scrub is not + * running. + */ + refcount_t scrub_workers_refcnt; + struct workqueue_struct *scrub_workers; + + struct btrfs_discard_ctl discard_ctl; + + /* Is qgroup tracking in a consistent state? */ + u64 qgroup_flags; + + /* Holds configuration and tracking. Protected by qgroup_lock. */ + struct rb_root qgroup_tree; + spinlock_t qgroup_lock; + + /* + * Protect user change for quota operations. If a transaction is needed, + * it must be started before locking this lock. + */ + struct mutex qgroup_ioctl_lock; + + /* List of dirty qgroups to be written at next commit. */ + struct list_head dirty_qgroups; + + /* Used by qgroup for an efficient tree traversal. */ + u64 qgroup_seq; + + /* Qgroup rescan items. */ + /* Protects the progress item */ + struct mutex qgroup_rescan_lock; + struct btrfs_key qgroup_rescan_progress; + struct btrfs_workqueue *qgroup_rescan_workers; + struct completion qgroup_rescan_completion; + struct btrfs_work qgroup_rescan_work; + /* Protected by qgroup_rescan_lock */ + bool qgroup_rescan_running; + u8 qgroup_drop_subtree_thres; + u64 qgroup_enable_gen; + + /* + * If this is not 0, then it indicates a serious filesystem error has + * happened and it contains that error (negative errno value). + */ + int fs_error; + + /* Filesystem state */ + unsigned long fs_state; + + struct btrfs_delayed_root delayed_root; + + /* Entries are eb->start >> nodesize_bits */ + struct xarray buffer_tree; + + /* Next backup root to be overwritten */ + int backup_root_index; + + /* Device replace state */ + struct btrfs_dev_replace dev_replace; + + struct semaphore uuid_tree_rescan_sem; + + /* Used to reclaim the metadata space in the background. */ + struct work_struct async_reclaim_work; + struct work_struct async_data_reclaim_work; + struct work_struct preempt_reclaim_work; + + /* Reclaim partially filled block groups in the background */ + struct work_struct reclaim_bgs_work; + /* Protected by unused_bgs_lock. */ + struct list_head reclaim_bgs; + int bg_reclaim_threshold; + + /* Protects the lists unused_bgs, reclaim_bgs, and fully_remapped_bgs. */ + spinlock_t unused_bgs_lock; + /* Protected by unused_bgs_lock. */ + struct list_head unused_bgs; + struct list_head fully_remapped_bgs; + struct mutex unused_bg_unpin_mutex; + /* Protect block groups that are going to be deleted */ + struct mutex reclaim_bgs_lock; + + /* Cached block sizes */ + u32 nodesize; + u32 nodesize_bits; + u32 sectorsize; + /* ilog2 of sectorsize, use to avoid 64bit division */ + u32 sectorsize_bits; + u32 block_min_order; + u32 block_max_order; + u32 writeback_bio_size; + u32 csum_size; + u32 csums_per_leaf; + u32 csum_type; + + /* + * Maximum size of an extent. BTRFS_MAX_EXTENT_SIZE on regular + * filesystem, on zoned it depends on the device constraints. + */ + u64 max_extent_size; + + /* Block groups and devices containing active swapfiles. */ + spinlock_t swapfile_pins_lock; + struct rb_root swapfile_pins; + + /* Type of exclusive operation running, protected by super_lock */ + enum btrfs_exclusive_operation exclusive_operation; + + /* + * Zone size > 0 when in ZONED mode, otherwise it's used for a check + * if the mode is enabled + */ + u64 zone_size; + + /* Constraints for ZONE_APPEND commands: */ + struct queue_limits limits; + u64 max_zone_append_size; + + struct mutex zoned_meta_io_lock; + spinlock_t treelog_bg_lock; + u64 treelog_bg; + + /* + * Start of the dedicated data relocation block group, protected by + * relocation_bg_lock. + */ + spinlock_t relocation_bg_lock; + u64 data_reloc_bg; + struct mutex zoned_data_reloc_io_lock; + + struct btrfs_block_group *active_meta_bg; + struct btrfs_block_group *active_system_bg; + + u64 nr_global_roots; + + spinlock_t zone_active_bgs_lock; + struct list_head zone_active_bgs; + + /* Updates are not protected by any lock */ + struct btrfs_commit_stats commit_stats; + + /* + * Last generation where we dropped a non-relocation root. + * Use btrfs_set_last_root_drop_gen() and btrfs_get_last_root_drop_gen() + * to change it and to read it, respectively. + */ + u64 last_root_drop_gen; + + /* + * Annotations for transaction events (structures are empty when + * compiled without lockdep). + */ + struct lockdep_map btrfs_trans_num_writers_map; + struct lockdep_map btrfs_trans_num_extwriters_map; + struct lockdep_map btrfs_state_change_map[4]; + struct lockdep_map btrfs_trans_pending_ordered_map; + struct lockdep_map btrfs_ordered_extent_map; + +#ifdef CONFIG_BTRFS_DEBUG + spinlock_t ref_verify_lock; + struct rb_root block_tree; + + struct kobject *debug_kobj; + struct list_head allocated_roots; + + spinlock_t eb_leak_lock; + struct list_head allocated_ebs; +#endif + + /* Used by self tests only. */ + bool (*use_bitmap)(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info); +}; + +#define folio_to_inode(_folio) (BTRFS_I(_Generic((_folio), \ + struct folio *: (_folio))->mapping->host)) + +#define folio_to_fs_info(_folio) (folio_to_inode(_folio)->root->fs_info) + +#define inode_to_fs_info(_inode) (BTRFS_I(_Generic((_inode), \ + struct inode *: (_inode)))->root->fs_info) + +static inline gfp_t btrfs_alloc_write_mask(const struct address_space *mapping) +{ + return mapping_gfp_constraint(mapping, ~__GFP_FS); +} + +/* Return the minimal folio size of the fs. */ +static inline unsigned int btrfs_min_folio_size(const struct btrfs_fs_info *fs_info) +{ + return 1U << (PAGE_SHIFT + fs_info->block_min_order); +} + +static inline u64 btrfs_get_fs_generation(const struct btrfs_fs_info *fs_info) +{ + return READ_ONCE(fs_info->generation); +} + +static inline void btrfs_set_fs_generation(struct btrfs_fs_info *fs_info, u64 gen) +{ + WRITE_ONCE(fs_info->generation, gen); +} + +static inline u64 btrfs_get_last_trans_committed(const struct btrfs_fs_info *fs_info) +{ + return READ_ONCE(fs_info->last_trans_committed); +} + +static inline void btrfs_set_last_trans_committed(struct btrfs_fs_info *fs_info, u64 gen) +{ + WRITE_ONCE(fs_info->last_trans_committed, gen); +} + +static inline void btrfs_set_last_root_drop_gen(struct btrfs_fs_info *fs_info, + u64 gen) +{ + WRITE_ONCE(fs_info->last_root_drop_gen, gen); +} + +static inline u64 btrfs_get_last_root_drop_gen(const struct btrfs_fs_info *fs_info) +{ + return READ_ONCE(fs_info->last_root_drop_gen); +} + +/* + * Take the number of bytes to be checksummed and figure out how many leaves + * it would require to store the csums for that many bytes. + */ +static inline u64 btrfs_csum_bytes_to_leaves( + const struct btrfs_fs_info *fs_info, u64 csum_bytes) +{ + const u64 num_csums = csum_bytes >> fs_info->sectorsize_bits; + + return DIV_ROUND_UP_ULL(num_csums, fs_info->csums_per_leaf); +} + +/* + * Use this if we would be adding new items, as we could split nodes as we cow + * down the tree. + */ +static inline u64 btrfs_calc_insert_metadata_size(const struct btrfs_fs_info *fs_info, + unsigned num_items) +{ + return (u64)fs_info->nodesize * BTRFS_MAX_LEVEL * 2 * num_items; +} + +/* + * Doing a truncate or a modification won't result in new nodes or leaves, just + * what we need for COW. + */ +static inline u64 btrfs_calc_metadata_size(const struct btrfs_fs_info *fs_info, + unsigned num_items) +{ + return (u64)fs_info->nodesize * BTRFS_MAX_LEVEL * num_items; +} + +#define BTRFS_MAX_EXTENT_ITEM_SIZE(r) ((BTRFS_LEAF_DATA_SIZE(r->fs_info) >> 4) - \ + sizeof(struct btrfs_item)) + +static inline bool btrfs_is_zoned(const struct btrfs_fs_info *fs_info) +{ + return IS_ENABLED(CONFIG_BLK_DEV_ZONED) && fs_info->zone_size > 0; +} + +/* + * Count how many fs_info->max_extent_size cover the @size + */ +static inline u32 count_max_extents(const struct btrfs_fs_info *fs_info, u64 size) +{ +#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS + if (!fs_info) + return div_u64(size + BTRFS_MAX_EXTENT_SIZE - 1, BTRFS_MAX_EXTENT_SIZE); +#endif + + return div_u64(size + fs_info->max_extent_size - 1, fs_info->max_extent_size); +} + +static inline unsigned int btrfs_blocks_per_folio(const struct btrfs_fs_info *fs_info, + const struct folio *folio) +{ + return folio_size(folio) >> fs_info->sectorsize_bits; +} + +bool __attribute_const__ btrfs_supported_blocksize(u32 blocksize); +bool btrfs_exclop_start(struct btrfs_fs_info *fs_info, + enum btrfs_exclusive_operation type); +bool btrfs_exclop_start_try_lock(struct btrfs_fs_info *fs_info, + enum btrfs_exclusive_operation type); +void btrfs_exclop_start_unlock(struct btrfs_fs_info *fs_info); +void btrfs_exclop_finish(struct btrfs_fs_info *fs_info); +void btrfs_exclop_balance(struct btrfs_fs_info *fs_info, + enum btrfs_exclusive_operation op); + +int btrfs_check_ioctl_vol_args_path(const struct btrfs_ioctl_vol_args *vol_args); + +u16 btrfs_csum_type_size(u16 type); +int btrfs_super_csum_size(const struct btrfs_super_block *s); +const char *btrfs_super_csum_name(u16 csum_type); +size_t __attribute_const__ btrfs_get_num_csums(void); +struct btrfs_csum_ctx { + u16 csum_type; + union { + u32 crc32; + struct xxh64_state xxh64; + struct sha256_ctx sha256; + struct blake2b_ctx blake2b; + }; +}; +void btrfs_csum(u16 csum_type, const u8 *data, size_t len, u8 *out); +void btrfs_csum_init(struct btrfs_csum_ctx *ctx, u16 csum_type); +void btrfs_csum_update(struct btrfs_csum_ctx *ctx, const u8 *data, size_t len); +void btrfs_csum_final(struct btrfs_csum_ctx *ctx, u8 *out); + +static inline bool btrfs_is_empty_uuid(const u8 *uuid) +{ + return uuid_is_null((const uuid_t *)uuid); +} + +/* Compatibility and incompatibility defines */ +void __btrfs_set_fs_incompat(struct btrfs_fs_info *fs_info, u64 flag, + const char *name); +void __btrfs_clear_fs_incompat(struct btrfs_fs_info *fs_info, u64 flag, + const char *name); +void __btrfs_set_fs_compat_ro(struct btrfs_fs_info *fs_info, u64 flag, + const char *name); +void __btrfs_clear_fs_compat_ro(struct btrfs_fs_info *fs_info, u64 flag, + const char *name); + +#define __btrfs_fs_incompat(fs_info, flags) \ + (!!(btrfs_super_incompat_flags((fs_info)->super_copy) & (flags))) + +#define __btrfs_fs_compat_ro(fs_info, flags) \ + (!!(btrfs_super_compat_ro_flags((fs_info)->super_copy) & (flags))) + +#define btrfs_set_fs_incompat(__fs_info, opt) \ + __btrfs_set_fs_incompat((__fs_info), BTRFS_FEATURE_INCOMPAT_##opt, #opt) + +#define btrfs_clear_fs_incompat(__fs_info, opt) \ + __btrfs_clear_fs_incompat((__fs_info), BTRFS_FEATURE_INCOMPAT_##opt, #opt) + +#define btrfs_fs_incompat(fs_info, opt) \ + __btrfs_fs_incompat((fs_info), BTRFS_FEATURE_INCOMPAT_##opt) + +#define btrfs_set_fs_compat_ro(__fs_info, opt) \ + __btrfs_set_fs_compat_ro((__fs_info), BTRFS_FEATURE_COMPAT_RO_##opt, #opt) + +#define btrfs_clear_fs_compat_ro(__fs_info, opt) \ + __btrfs_clear_fs_compat_ro((__fs_info), BTRFS_FEATURE_COMPAT_RO_##opt, #opt) + +#define btrfs_fs_compat_ro(fs_info, opt) \ + __btrfs_fs_compat_ro((fs_info), BTRFS_FEATURE_COMPAT_RO_##opt) + +#define btrfs_clear_opt(o, opt) ((o) &= ~BTRFS_MOUNT_##opt) +#define btrfs_set_opt(o, opt) ((o) |= BTRFS_MOUNT_##opt) +#define btrfs_raw_test_opt(o, opt) ((o) & BTRFS_MOUNT_##opt) +#define btrfs_test_opt(fs_info, opt) ((fs_info)->mount_opt & \ + BTRFS_MOUNT_##opt) + +static inline bool btrfs_is_full_ro(const struct btrfs_fs_info *fs_info) +{ + if (!sb_rdonly(fs_info->sb)) + return false; + if (unlikely(fs_info->mount_opt & BTRFS_MOUNT_FULL_RO_MASK)) + return true; + return false; +} + +static inline bool btrfs_fs_closing(const struct btrfs_fs_info *fs_info) +{ + return unlikely(test_bit(BTRFS_FS_CLOSING_START, &fs_info->flags)); +} + +static inline bool btrfs_fs_closing_done(const struct btrfs_fs_info *fs_info) +{ + if (btrfs_fs_closing(fs_info) && test_bit(BTRFS_FS_CLOSING_DONE, &fs_info->flags)) + return true; + + return false; +} + +/* + * If we remount the fs to be R/O or umount the fs, the cleaner needn't do + * anything except sleeping. This function is used to check the status of + * the fs. + * We check for BTRFS_FS_STATE_RO to avoid races with a concurrent remount, + * since setting and checking for SB_RDONLY in the superblock's flags is not + * atomic. + */ +static inline int btrfs_need_cleaner_sleep(const struct btrfs_fs_info *fs_info) +{ + return test_bit(BTRFS_FS_STATE_RO, &fs_info->fs_state) || + btrfs_fs_closing(fs_info); +} + +static inline void btrfs_wake_unfinished_drop(struct btrfs_fs_info *fs_info) +{ + clear_and_wake_up_bit(BTRFS_FS_UNFINISHED_DROPS, &fs_info->flags); +} + +/* + * We use the folio owner_2 flag to indicate the folio has blocks that were + * dirtied without a space reservation and need the writepage fixup before + * writeback. For bs < folio_size the fixup bitmap tracks the affected + * blocks. + */ +#define folio_test_fixup_pending(folio) folio_test_owner_2(folio) +#define folio_set_fixup_pending(folio) folio_set_owner_2(folio) +#define folio_clear_fixup_pending(folio) folio_clear_owner_2(folio) + +#define BTRFS_FS_ERROR(fs_info) (READ_ONCE((fs_info)->fs_error)) + +#define BTRFS_FS_LOG_CLEANUP_ERROR(fs_info) \ + (unlikely(test_bit(BTRFS_FS_STATE_LOG_CLEANUP_ERROR, \ + &(fs_info)->fs_state))) + +static inline bool btrfs_is_shutdown(const struct btrfs_fs_info *fs_info) +{ + return unlikely(test_bit(BTRFS_FS_STATE_EMERGENCY_SHUTDOWN, &fs_info->fs_state)); +} + +static inline void btrfs_force_shutdown(struct btrfs_fs_info *fs_info) +{ + /* + * Here we do not want to use handle_fs_error(), which will mark the fs + * read-only. + * Some call sites like shutdown ioctl will mark the fs shutdown when + * the fs is frozen. But thaw path will handle RO and RW fs + * differently. + * + * So here we only mark the fs error without flipping it RO. + */ + WRITE_ONCE(fs_info->fs_error, -EIO); + if (!test_and_set_bit(BTRFS_FS_STATE_EMERGENCY_SHUTDOWN, &fs_info->fs_state)) { + btrfs_crit(fs_info, "emergency shutdown"); + fserror_report_shutdown(fs_info->sb, GFP_KERNEL); + } +} + +#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS + +#define EXPORT_FOR_TESTS + +static inline bool btrfs_is_testing(const struct btrfs_fs_info *fs_info) +{ + return unlikely(test_bit(BTRFS_FS_STATE_DUMMY_FS_INFO, &fs_info->fs_state)); +} + +void btrfs_test_destroy_inode(struct inode *inode); + +#else + +#define EXPORT_FOR_TESTS static + +static inline bool btrfs_is_testing(const struct btrfs_fs_info *fs_info) +{ + return false; +} +#endif + +#endif diff --git a/fs/btrfs/inode-item.c b/fs/btrfs/inode-item.c new file mode 100644 index 000000000..a864f8c99 --- /dev/null +++ b/fs/btrfs/inode-item.c @@ -0,0 +1,733 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#include "ctree.h" +#include "fs.h" +#include "messages.h" +#include "inode-item.h" +#include "disk-io.h" +#include "transaction.h" +#include "space-info.h" +#include "accessors.h" +#include "extent-tree.h" +#include "file-item.h" + +struct btrfs_inode_ref *btrfs_find_name_in_backref(const struct extent_buffer *leaf, + int slot, + const struct fscrypt_str *name) +{ + struct btrfs_inode_ref *ref; + unsigned long ptr; + unsigned long name_ptr; + u32 item_size; + u32 cur_offset = 0; + int len; + + item_size = btrfs_item_size(leaf, slot); + ptr = btrfs_item_ptr_offset(leaf, slot); + while (cur_offset < item_size) { + ref = (struct btrfs_inode_ref *)(ptr + cur_offset); + len = btrfs_inode_ref_name_len(leaf, ref); + name_ptr = (unsigned long)(ref + 1); + cur_offset += len + sizeof(*ref); + if (len != name->len) + continue; + if (memcmp_extent_buffer(leaf, name->name, name_ptr, + name->len) == 0) + return ref; + } + return NULL; +} + +struct btrfs_inode_extref *btrfs_find_name_in_ext_backref( + const struct extent_buffer *leaf, int slot, u64 ref_objectid, + const struct fscrypt_str *name) +{ + struct btrfs_inode_extref *extref; + unsigned long ptr; + unsigned long name_ptr; + u32 item_size; + u32 cur_offset = 0; + int ref_name_len; + + item_size = btrfs_item_size(leaf, slot); + ptr = btrfs_item_ptr_offset(leaf, slot); + + /* + * Search all extended backrefs in this item. We're only + * looking through any collisions so most of the time this is + * just going to compare against one buffer. If all is well, + * we'll return success and the inode ref object. + */ + while (cur_offset < item_size) { + extref = (struct btrfs_inode_extref *) (ptr + cur_offset); + name_ptr = (unsigned long)(&extref->name); + ref_name_len = btrfs_inode_extref_name_len(leaf, extref); + + if (ref_name_len == name->len && + btrfs_inode_extref_parent(leaf, extref) == ref_objectid && + (memcmp_extent_buffer(leaf, name->name, name_ptr, + name->len) == 0)) + return extref; + + cur_offset += ref_name_len + sizeof(*extref); + } + return NULL; +} + +/* Returns NULL if no extref found */ +struct btrfs_inode_extref *btrfs_lookup_inode_extref(struct btrfs_root *root, + struct btrfs_path *path, + const struct fscrypt_str *name, + u64 inode_objectid, u64 ref_objectid) +{ + int ret; + struct btrfs_key key; + + key.objectid = inode_objectid; + key.type = BTRFS_INODE_EXTREF_KEY; + key.offset = btrfs_extref_hash(ref_objectid, name->name, name->len); + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + return ERR_PTR(ret); + if (ret > 0) + return NULL; + return btrfs_find_name_in_ext_backref(path->nodes[0], path->slots[0], + ref_objectid, name); + +} + +static int btrfs_del_inode_extref(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + const struct fscrypt_str *name, + u64 inode_objectid, u64 ref_objectid, + u64 *index) +{ + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + struct btrfs_inode_extref *extref; + struct extent_buffer *leaf; + int ret; + int del_len = name->len + sizeof(*extref); + unsigned long ptr; + unsigned long item_start; + u32 item_size; + + key.objectid = inode_objectid; + key.type = BTRFS_INODE_EXTREF_KEY; + key.offset = btrfs_extref_hash(ref_objectid, name->name, name->len); + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + ret = btrfs_search_slot(trans, root, &key, path, -1, 1); + if (ret > 0) + return -ENOENT; + if (ret < 0) + return ret; + + /* + * Sanity check - did we find the right item for this name? + * This should always succeed so error here will make the FS + * readonly. + */ + extref = btrfs_find_name_in_ext_backref(path->nodes[0], path->slots[0], + ref_objectid, name); + if (unlikely(!extref)) { + btrfs_abort_transaction(trans, -ENOENT); + return -ENOENT; + } + + leaf = path->nodes[0]; + item_size = btrfs_item_size(leaf, path->slots[0]); + if (index) + *index = btrfs_inode_extref_index(leaf, extref); + + if (del_len == item_size) { + /* Common case only one ref in the item, remove the whole item. */ + return btrfs_del_item(trans, root, path); + } + + ptr = (unsigned long)extref; + item_start = btrfs_item_ptr_offset(leaf, path->slots[0]); + + memmove_extent_buffer(leaf, ptr, ptr + del_len, + item_size - (ptr + del_len - item_start)); + + btrfs_truncate_item(trans, path, item_size - del_len, 1); + + return ret; +} + +int btrfs_del_inode_ref(struct btrfs_trans_handle *trans, + struct btrfs_root *root, const struct fscrypt_str *name, + u64 inode_objectid, u64 ref_objectid, u64 *index) +{ + struct btrfs_path *path; + struct btrfs_key key; + struct btrfs_inode_ref *ref; + struct extent_buffer *leaf; + unsigned long ptr; + unsigned long item_start; + u32 item_size; + u32 sub_item_len; + int ret; + int search_ext_refs = 0; + int del_len = name->len + sizeof(*ref); + + key.objectid = inode_objectid; + key.type = BTRFS_INODE_REF_KEY; + key.offset = ref_objectid; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + ret = btrfs_search_slot(trans, root, &key, path, -1, 1); + if (ret > 0) { + ret = -ENOENT; + search_ext_refs = 1; + goto out; + } else if (ret < 0) { + goto out; + } + + ref = btrfs_find_name_in_backref(path->nodes[0], path->slots[0], name); + if (!ref) { + ret = -ENOENT; + search_ext_refs = 1; + goto out; + } + leaf = path->nodes[0]; + item_size = btrfs_item_size(leaf, path->slots[0]); + + if (index) + *index = btrfs_inode_ref_index(leaf, ref); + + if (del_len == item_size) { + ret = btrfs_del_item(trans, root, path); + goto out; + } + ptr = (unsigned long)ref; + sub_item_len = name->len + sizeof(*ref); + item_start = btrfs_item_ptr_offset(leaf, path->slots[0]); + memmove_extent_buffer(leaf, ptr, ptr + sub_item_len, + item_size - (ptr + sub_item_len - item_start)); + btrfs_truncate_item(trans, path, item_size - sub_item_len, 1); +out: + btrfs_free_path(path); + + if (search_ext_refs) { + /* + * No refs were found, or we could not find the + * name in our ref array. Find and remove the extended + * inode ref then. + */ + return btrfs_del_inode_extref(trans, root, name, + inode_objectid, ref_objectid, index); + } + + return ret; +} + +/* + * Insert an extended inode ref into a tree. + * + * The caller must have checked against BTRFS_LINK_MAX already. + */ +static int btrfs_insert_inode_extref(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + const struct fscrypt_str *name, + u64 inode_objectid, u64 ref_objectid, + u64 index) +{ + struct btrfs_inode_extref *extref; + int ret; + int ins_len = name->len + sizeof(*extref); + unsigned long ptr; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + struct extent_buffer *leaf; + + key.objectid = inode_objectid; + key.type = BTRFS_INODE_EXTREF_KEY; + key.offset = btrfs_extref_hash(ref_objectid, name->name, name->len); + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + ret = btrfs_insert_empty_item(trans, root, path, &key, + ins_len); + if (ret == -EEXIST) { + if (btrfs_find_name_in_ext_backref(path->nodes[0], + path->slots[0], + ref_objectid, + name)) + return ret; + + btrfs_extend_item(trans, path, ins_len); + ret = 0; + } + if (ret < 0) + return ret; + + leaf = path->nodes[0]; + ptr = (unsigned long)btrfs_item_ptr(leaf, path->slots[0], char); + ptr += btrfs_item_size(leaf, path->slots[0]) - ins_len; + extref = (struct btrfs_inode_extref *)ptr; + + btrfs_set_inode_extref_name_len(path->nodes[0], extref, name->len); + btrfs_set_inode_extref_index(path->nodes[0], extref, index); + btrfs_set_inode_extref_parent(path->nodes[0], extref, ref_objectid); + + ptr = (unsigned long)&extref->name; + write_extent_buffer(path->nodes[0], name->name, ptr, name->len); + + return 0; +} + +/* Will return 0, -ENOMEM, -EMLINK, or -EEXIST or anything from the CoW path */ +int btrfs_insert_inode_ref(struct btrfs_trans_handle *trans, + struct btrfs_root *root, const struct fscrypt_str *name, + u64 inode_objectid, u64 ref_objectid, u64 index) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_path *path; + struct btrfs_key key; + struct btrfs_inode_ref *ref; + unsigned long ptr; + int ret; + int ins_len = name->len + sizeof(*ref); + + key.objectid = inode_objectid; + key.type = BTRFS_INODE_REF_KEY; + key.offset = ref_objectid; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + path->skip_release_on_error = true; + ret = btrfs_insert_empty_item(trans, root, path, &key, + ins_len); + if (ret == -EEXIST) { + u32 old_size; + ref = btrfs_find_name_in_backref(path->nodes[0], path->slots[0], + name); + if (ref) + goto out; + + old_size = btrfs_item_size(path->nodes[0], path->slots[0]); + btrfs_extend_item(trans, path, ins_len); + ref = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_inode_ref); + ref = (struct btrfs_inode_ref *)((unsigned long)ref + old_size); + btrfs_set_inode_ref_name_len(path->nodes[0], ref, name->len); + btrfs_set_inode_ref_index(path->nodes[0], ref, index); + ptr = (unsigned long)(ref + 1); + ret = 0; + } else if (ret < 0) { + if (ret == -EOVERFLOW) { + if (btrfs_find_name_in_backref(path->nodes[0], + path->slots[0], + name)) + ret = -EEXIST; + else + ret = -EMLINK; + } + goto out; + } else { + ref = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_inode_ref); + btrfs_set_inode_ref_name_len(path->nodes[0], ref, name->len); + btrfs_set_inode_ref_index(path->nodes[0], ref, index); + ptr = (unsigned long)(ref + 1); + } + write_extent_buffer(path->nodes[0], name->name, ptr, name->len); +out: + btrfs_free_path(path); + + if (ret == -EMLINK) { + struct btrfs_super_block *disk_super = fs_info->super_copy; + /* We ran out of space in the ref array. Need to + * add an extended ref. */ + if (btrfs_super_incompat_flags(disk_super) + & BTRFS_FEATURE_INCOMPAT_EXTENDED_IREF) + ret = btrfs_insert_inode_extref(trans, root, name, + inode_objectid, + ref_objectid, index); + } + + return ret; +} + +int btrfs_insert_empty_inode(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, u64 objectid) +{ + struct btrfs_key key; + + key.objectid = objectid; + key.type = BTRFS_INODE_ITEM_KEY; + key.offset = 0; + + return btrfs_insert_empty_item(trans, root, path, &key, + sizeof(struct btrfs_inode_item)); +} + +int btrfs_lookup_inode(struct btrfs_trans_handle *trans, struct btrfs_root + *root, struct btrfs_path *path, + struct btrfs_key *location, int mod) +{ + int ins_len = mod < 0 ? -1 : 0; + int cow = mod != 0; + int ret; + int slot; + struct extent_buffer *leaf; + struct btrfs_key found_key; + + ret = btrfs_search_slot(trans, root, location, path, ins_len, cow); + if (ret > 0 && location->type == BTRFS_ROOT_ITEM_KEY && + location->offset == (u64)-1 && path->slots[0] != 0) { + slot = path->slots[0] - 1; + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &found_key, slot); + if (found_key.objectid == location->objectid && + found_key.type == location->type) { + path->slots[0]--; + return 0; + } + } + return ret; +} + +static inline void btrfs_trace_truncate(const struct btrfs_inode *inode, + const struct extent_buffer *leaf, + const struct btrfs_file_extent_item *fi, + u64 offset, int extent_type, int slot) +{ + if (!inode) + return; + if (extent_type == BTRFS_FILE_EXTENT_INLINE) + trace_btrfs_truncate_show_fi_inline(inode, leaf, fi, slot, + offset); + else + trace_btrfs_truncate_show_fi_regular(inode, leaf, fi, offset); +} + +/* + * Remove inode items from a given root. + * + * @trans: A transaction handle. + * @root: The root from which to remove items. + * @inode: The inode whose items we want to remove. + * @control: The btrfs_truncate_control to control how and what we + * are truncating. + * + * Remove all keys associated with the inode from the given root that have a key + * with a type greater than or equals to @min_type. When @min_type has a value of + * BTRFS_EXTENT_DATA_KEY, only remove file extent items that have an offset value + * greater than or equals to @new_size. If a file extent item that starts before + * @new_size and ends after it is found, its length is adjusted. + * + * Returns: 0 on success, < 0 on error and NEED_TRUNCATE_BLOCK when @min_type is + * BTRFS_EXTENT_DATA_KEY and the caller must truncate the last block. + */ +int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_truncate_control *control) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + BTRFS_PATH_AUTO_FREE(path); + struct extent_buffer *leaf; + struct btrfs_file_extent_item *fi; + struct btrfs_key key; + struct btrfs_key found_key; + u64 new_size = control->new_size; + u64 extent_num_bytes = 0; + u64 extent_offset = 0; + u64 item_end = 0; + u32 found_type = (u8)-1; + int del_item; + int pending_del_nr = 0; + int pending_del_slot = 0; + int extent_type = -1; + int ret; + u64 bytes_deleted = 0; + bool be_nice = false; + + ASSERT(control->inode || !control->clear_extent_range); + ASSERT(new_size == 0 || control->min_type == BTRFS_EXTENT_DATA_KEY); + + control->last_size = new_size; + control->sub_bytes = 0; + + /* + * For shareable roots we want to back off from time to time, this turns + * out to be subvolume roots, reloc roots, and data reloc roots. + */ + if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) + be_nice = true; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + path->reada = READA_BACK; + + key.objectid = control->ino; + key.type = (u8)-1; + key.offset = (u64)-1; + +search_again: + /* + * With a 16K leaf size and 128MiB extents, you can actually queue up a + * huge file in a single leaf. Most of the time that bytes_deleted is + * > 0, it will be huge by the time we get here + */ + if (be_nice && bytes_deleted > SZ_32M && + btrfs_should_end_transaction(trans)) { + ret = -EAGAIN; + goto out; + } + + ret = btrfs_search_slot(trans, root, &key, path, -1, 1); + if (ret < 0) + goto out; + + if (ret > 0) { + ret = 0; + /* There are no items in the tree for us to truncate, we're done */ + if (path->slots[0] == 0) + goto out; + path->slots[0]--; + } + + while (1) { + u64 clear_start = 0, clear_len = 0, extent_start = 0; + bool refill_delayed_refs_rsv = false; + + fi = NULL; + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); + found_type = found_key.type; + + if (found_key.objectid != control->ino) + break; + + if (found_type < control->min_type) + break; + + item_end = found_key.offset; + if (found_type == BTRFS_EXTENT_DATA_KEY) { + fi = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_file_extent_item); + extent_type = btrfs_file_extent_type(leaf, fi); + if (extent_type != BTRFS_FILE_EXTENT_INLINE) + item_end += + btrfs_file_extent_num_bytes(leaf, fi); + else if (extent_type == BTRFS_FILE_EXTENT_INLINE) + item_end += btrfs_file_extent_ram_bytes(leaf, fi); + + btrfs_trace_truncate(control->inode, leaf, fi, + found_key.offset, extent_type, + path->slots[0]); + item_end--; + } + if (found_type > control->min_type) { + del_item = 1; + } else { + if (item_end < new_size) + break; + if (found_key.offset >= new_size) + del_item = 1; + else + del_item = 0; + } + + /* FIXME, shrink the extent if the ref count is only 1 */ + if (found_type != BTRFS_EXTENT_DATA_KEY) + goto delete; + + control->extents_found++; + + if (extent_type != BTRFS_FILE_EXTENT_INLINE) { + u64 num_dec; + + clear_start = found_key.offset; + extent_start = btrfs_file_extent_disk_bytenr(leaf, fi); + if (!del_item) { + u64 orig_num_bytes = + btrfs_file_extent_num_bytes(leaf, fi); + extent_num_bytes = ALIGN(new_size - + found_key.offset, + fs_info->sectorsize); + clear_start = ALIGN(new_size, fs_info->sectorsize); + + btrfs_set_file_extent_num_bytes(leaf, fi, + extent_num_bytes); + num_dec = (orig_num_bytes - extent_num_bytes); + if (extent_start != 0) + control->sub_bytes += num_dec; + } else { + extent_num_bytes = + btrfs_file_extent_disk_num_bytes(leaf, fi); + extent_offset = found_key.offset - + btrfs_file_extent_offset(leaf, fi); + + /* FIXME blocksize != 4096 */ + num_dec = btrfs_file_extent_num_bytes(leaf, fi); + if (extent_start != 0) + control->sub_bytes += num_dec; + } + clear_len = num_dec; + } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { + /* + * We can't truncate inline items that have had + * special encodings + */ + if (!del_item && + btrfs_file_extent_encryption(leaf, fi) == 0 && + btrfs_file_extent_other_encoding(leaf, fi) == 0 && + btrfs_file_extent_compression(leaf, fi) == 0) { + u32 size = (u32)(new_size - found_key.offset); + + btrfs_set_file_extent_ram_bytes(leaf, fi, size); + size = btrfs_file_extent_calc_inline_size(size); + btrfs_truncate_item(trans, path, size, 1); + } else if (!del_item) { + /* + * We have to bail so the last_size is set to + * just before this extent. + */ + ret = BTRFS_NEED_TRUNCATE_BLOCK; + break; + } else { + /* + * Inline extents are special, we just treat + * them as a full sector worth in the file + * extent tree just for simplicity sake. + */ + clear_len = fs_info->sectorsize; + } + + control->sub_bytes += item_end + 1 - new_size; + } +delete: + /* + * We only want to clear the file extent range if we're + * modifying the actual inode's mapping, which is just the + * normal truncate path. + */ + if (control->clear_extent_range) { + ret = btrfs_inode_clear_file_extent_range(control->inode, + clear_start, clear_len); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + break; + } + } + + if (del_item) { + ASSERT(!pending_del_nr || + ((path->slots[0] + 1) == pending_del_slot)); + + control->last_size = found_key.offset; + if (!pending_del_nr) { + /* No pending yet, add ourselves */ + pending_del_slot = path->slots[0]; + pending_del_nr = 1; + } else if (path->slots[0] + 1 == pending_del_slot) { + /* Hop on the pending chunk */ + pending_del_nr++; + pending_del_slot = path->slots[0]; + } + } else { + control->last_size = new_size; + break; + } + + if (del_item && extent_start != 0 && !control->skip_ref_updates) { + struct btrfs_ref ref = { + .action = BTRFS_DROP_DELAYED_REF, + .bytenr = extent_start, + .num_bytes = extent_num_bytes, + .owning_root = btrfs_root_id(root), + .ref_root = btrfs_header_owner(leaf), + }; + + bytes_deleted += extent_num_bytes; + + btrfs_init_data_ref(&ref, control->ino, extent_offset, + btrfs_root_id(root), false); + ret = btrfs_free_extent(trans, &ref); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + break; + } + if (be_nice && btrfs_check_space_for_delayed_refs(fs_info)) + refill_delayed_refs_rsv = true; + } + + if (found_type == BTRFS_INODE_ITEM_KEY) + break; + + if (path->slots[0] == 0 || + path->slots[0] != pending_del_slot || + refill_delayed_refs_rsv) { + if (pending_del_nr) { + ret = btrfs_del_items(trans, root, path, + pending_del_slot, + pending_del_nr); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + break; + } + pending_del_nr = 0; + } + btrfs_release_path(path); + + /* + * We can generate a lot of delayed refs, so we need to + * throttle every once and a while and make sure we're + * adding enough space to keep up with the work we are + * generating. Since we hold a transaction here we + * can't flush, and we don't want to FLUSH_LIMIT because + * we could have generated too many delayed refs to + * actually allocate, so just bail if we're short and + * let the normal reservation dance happen higher up. + */ + if (refill_delayed_refs_rsv) { + ret = btrfs_delayed_refs_rsv_refill(fs_info, + BTRFS_RESERVE_NO_FLUSH); + if (ret) { + ret = -EAGAIN; + break; + } + } + goto search_again; + } else { + path->slots[0]--; + } + } +out: + if (ret >= 0 && pending_del_nr) { + int ret2; + + ret2 = btrfs_del_items(trans, root, path, pending_del_slot, pending_del_nr); + if (unlikely(ret2)) { + btrfs_abort_transaction(trans, ret2); + ret = ret2; + } + } + + ASSERT(control->last_size >= new_size); + if (!ret && control->last_size > new_size) + control->last_size = new_size; + + return ret; +} diff --git a/fs/btrfs/inode-item.h b/fs/btrfs/inode-item.h new file mode 100644 index 000000000..6d9f5ad20 --- /dev/null +++ b/fs/btrfs/inode-item.h @@ -0,0 +1,116 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_INODE_ITEM_H +#define BTRFS_INODE_ITEM_H + +#include +#include + +struct fscrypt_str; +struct extent_buffer; +struct btrfs_trans_handle; +struct btrfs_root; +struct btrfs_path; +struct btrfs_key; +struct btrfs_inode_extref; +struct btrfs_inode; +struct btrfs_truncate_control; + +/* + * Return this if we need to call truncate_block for the last bit of the + * truncate. + */ +#define BTRFS_NEED_TRUNCATE_BLOCK 1 + +struct btrfs_truncate_control { + /* + * IN: the inode we're operating on, this can be NULL if + * ->clear_extent_range is false. + */ + struct btrfs_inode *inode; + + /* IN: the size we're truncating to. */ + u64 new_size; + + /* OUT: the number of extents truncated. */ + u64 extents_found; + + /* OUT: the last size we truncated this inode to. */ + u64 last_size; + + /* OUT: the number of bytes to sub from this inode. */ + u64 sub_bytes; + + /* IN: the ino we are truncating. */ + u64 ino; + + /* + * IN: minimum key type to remove. All key types with this type are + * removed only if their offset >= new_size. + */ + u32 min_type; + + /* + * IN: true if we don't want to do extent reference updates for any file + * extents we drop. + */ + bool skip_ref_updates; + + /* + * IN: true if we need to clear the file extent range for the inode as + * we drop the file extent items. + */ + bool clear_extent_range; +}; + +/* + * btrfs_inode_item stores flags in a u64, btrfs_inode stores them in two + * separate u32s. These two functions convert between the two representations. + */ +static inline u64 btrfs_inode_combine_flags(u32 flags, u32 ro_flags) +{ + return (flags | ((u64)ro_flags << 32)); +} + +static inline void btrfs_inode_split_flags(u64 inode_item_flags, + u32 *flags, u32 *ro_flags) +{ + *flags = (u32)inode_item_flags; + *ro_flags = (u32)(inode_item_flags >> 32); +} + +/* Figure the key offset of an extended inode ref. */ +static inline u64 btrfs_extref_hash(u64 parent_objectid, const char *name, int len) +{ + return (u64)crc32c(parent_objectid, name, len); +} + +int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_truncate_control *control); +int btrfs_insert_inode_ref(struct btrfs_trans_handle *trans, + struct btrfs_root *root, const struct fscrypt_str *name, + u64 inode_objectid, u64 ref_objectid, u64 index); +int btrfs_del_inode_ref(struct btrfs_trans_handle *trans, + struct btrfs_root *root, const struct fscrypt_str *name, + u64 inode_objectid, u64 ref_objectid, u64 *index); +int btrfs_insert_empty_inode(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, u64 objectid); +int btrfs_lookup_inode(struct btrfs_trans_handle *trans, + struct btrfs_root *root, struct btrfs_path *path, + struct btrfs_key *location, int mod); + +struct btrfs_inode_extref *btrfs_lookup_inode_extref(struct btrfs_root *root, + struct btrfs_path *path, + const struct fscrypt_str *name, + u64 inode_objectid, u64 ref_objectid); + +struct btrfs_inode_ref *btrfs_find_name_in_backref(const struct extent_buffer *leaf, + int slot, + const struct fscrypt_str *name); +struct btrfs_inode_extref *btrfs_find_name_in_ext_backref( + const struct extent_buffer *leaf, int slot, u64 ref_objectid, + const struct fscrypt_str *name); + +#endif diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c new file mode 100644 index 000000000..3c10a0ef0 --- /dev/null +++ b/fs/btrfs/inode.c @@ -0,0 +1,10862 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "misc.h" +#include "ctree.h" +#include "disk-io.h" +#include "transaction.h" +#include "btrfs_inode.h" +#include "ordered-data.h" +#include "xattr.h" +#include "tree-log.h" +#include "bio.h" +#include "compression.h" +#include "locking.h" +#include "props.h" +#include "qgroup.h" +#include "delalloc-space.h" +#include "block-group.h" +#include "space-info.h" +#include "zoned.h" +#include "subpage.h" +#include "inode-item.h" +#include "fs.h" +#include "accessors.h" +#include "extent-tree.h" +#include "root-tree.h" +#include "defrag.h" +#include "dir-item.h" +#include "file-item.h" +#include "uuid-tree.h" +#include "ioctl.h" +#include "file.h" +#include "acl.h" +#include "relocation.h" +#include "verity.h" +#include "super.h" +#include "orphan.h" +#include "backref.h" +#include "raid-stripe-tree.h" +#include "fiemap.h" +#include "delayed-inode.h" + +#define COW_FILE_RANGE_KEEP_LOCKED (1UL << 0) + +struct btrfs_iget_args { + u64 ino; + struct btrfs_root *root; +}; + +struct btrfs_rename_ctx { + /* Output field. Stores the index number of the old directory entry. */ + u64 index; +}; + +/* + * Used by data_reloc_print_warning_inode() to pass needed info for filename + * resolution and output of error message. + */ +struct data_reloc_warn { + struct btrfs_path path; + struct btrfs_fs_info *fs_info; + u64 extent_item_size; + u64 logical; + int mirror_num; +}; + +/* + * For the file_extent_tree, we want to hold the inode lock when we lookup and + * update the disk_i_size, but lockdep will complain because our io_tree we hold + * the tree lock and get the inode lock when setting delalloc. These two things + * are unrelated, so make a class for the file_extent_tree so we don't get the + * two locking patterns mixed up. + */ +static struct lock_class_key file_extent_tree_class; + +static const struct inode_operations btrfs_dir_inode_operations; +static const struct inode_operations btrfs_symlink_inode_operations; +static const struct inode_operations btrfs_special_inode_operations; +static const struct inode_operations btrfs_file_inode_operations; +static const struct address_space_operations btrfs_aops; +static const struct file_operations btrfs_dir_file_operations; + +static struct kmem_cache *btrfs_inode_cachep; + +static int btrfs_setsize(struct inode *inode, struct iattr *attr); +static int btrfs_truncate(struct btrfs_inode *inode, bool skip_writeback); + +static noinline int run_delalloc_cow(struct btrfs_inode *inode, + struct folio *locked_folio, u64 start, + u64 end, struct writeback_control *wbc, + bool pages_dirty); + +static int data_reloc_print_warning_inode(u64 inum, u64 offset, u64 num_bytes, + u64 root, void *warn_ctx) +{ + struct data_reloc_warn *warn = warn_ctx; + struct btrfs_fs_info *fs_info = warn->fs_info; + struct extent_buffer *eb; + struct btrfs_inode_item *inode_item; + struct inode_fs_paths *ipath __free(inode_fs_paths) = NULL; + struct btrfs_root *local_root; + struct btrfs_key key; + unsigned int nofs_flag; + u32 nlink; + int ret; + + local_root = btrfs_get_fs_root(fs_info, root, true); + if (IS_ERR(local_root)) { + ret = PTR_ERR(local_root); + goto err; + } + + /* This makes the path point to (inum INODE_ITEM ioff). */ + key.objectid = inum; + key.type = BTRFS_INODE_ITEM_KEY; + key.offset = 0; + + ret = btrfs_search_slot(NULL, local_root, &key, &warn->path, 0, 0); + if (ret) { + btrfs_put_root(local_root); + btrfs_release_path(&warn->path); + goto err; + } + + eb = warn->path.nodes[0]; + inode_item = btrfs_item_ptr(eb, warn->path.slots[0], struct btrfs_inode_item); + nlink = btrfs_inode_nlink(eb, inode_item); + btrfs_release_path(&warn->path); + + nofs_flag = memalloc_nofs_save(); + ipath = init_ipath(4096, local_root, &warn->path); + memalloc_nofs_restore(nofs_flag); + if (IS_ERR(ipath)) { + btrfs_put_root(local_root); + ret = PTR_ERR(ipath); + ipath = NULL; + /* + * -ENOMEM, not a critical error, just output an generic error + * without filename. + */ + btrfs_warn(fs_info, +"checksum error at logical %llu mirror %u root %llu, inode %llu offset %llu", + warn->logical, warn->mirror_num, root, inum, offset); + return ret; + } + ret = paths_from_inode(inum, ipath); + if (ret < 0) { + btrfs_put_root(local_root); + goto err; + } + + /* + * We deliberately ignore the bit ipath might have been too small to + * hold all of the paths here + */ + for (int i = 0; i < ipath->fspath->elem_cnt; i++) { + btrfs_warn(fs_info, +"checksum error at logical %llu mirror %u root %llu inode %llu offset %llu length %u links %u (path: %s)", + warn->logical, warn->mirror_num, root, inum, offset, + fs_info->sectorsize, nlink, + (char *)(unsigned long)ipath->fspath->val[i]); + } + + btrfs_put_root(local_root); + return 0; + +err: + btrfs_warn(fs_info, +"checksum error at logical %llu mirror %u root %llu inode %llu offset %llu, path resolving failed with ret=%d", + warn->logical, warn->mirror_num, root, inum, offset, ret); + + return ret; +} + +/* + * Do extra user-friendly error output (e.g. lookup all the affected files). + * + * Return true if we succeeded doing the backref lookup. + * Return false if such lookup failed, and has to fallback to the old error message. + */ +static void print_data_reloc_error(const struct btrfs_inode *inode, u64 file_off, + const u8 *csum, const u8 *csum_expected, + int mirror_num) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + BTRFS_PATH_AUTO_RELEASE(path); + struct btrfs_key found_key = { 0 }; + struct extent_buffer *eb; + struct btrfs_extent_item *ei; + const u32 csum_size = fs_info->csum_size; + u64 logical; + u64 flags; + u32 item_size; + int ret; + + logical = btrfs_get_reloc_bg_bytenr(fs_info); + + if (logical == U64_MAX) { + btrfs_warn_rl(fs_info, "has data reloc tree but no running relocation"); + btrfs_warn_rl(fs_info, +"csum failed root %lld ino %llu off %llu csum " BTRFS_CSUM_FMT " expected csum " BTRFS_CSUM_FMT " mirror %d", + btrfs_root_id(inode->root), btrfs_ino(inode), file_off, + BTRFS_CSUM_FMT_VALUE(csum_size, csum), + BTRFS_CSUM_FMT_VALUE(csum_size, csum_expected), + mirror_num); + return; + } + + logical += file_off; + btrfs_warn_rl(fs_info, +"csum failed root %lld ino %llu off %llu logical %llu csum " BTRFS_CSUM_FMT " expected csum " BTRFS_CSUM_FMT " mirror %d", + btrfs_root_id(inode->root), + btrfs_ino(inode), file_off, logical, + BTRFS_CSUM_FMT_VALUE(csum_size, csum), + BTRFS_CSUM_FMT_VALUE(csum_size, csum_expected), + mirror_num); + + ret = extent_from_logical(fs_info, logical, &path, &found_key, &flags); + if (ret < 0) { + btrfs_err_rl(fs_info, "failed to lookup extent item for logical %llu: %pe", + logical, ERR_PTR(ret)); + return; + } + eb = path.nodes[0]; + ei = btrfs_item_ptr(eb, path.slots[0], struct btrfs_extent_item); + item_size = btrfs_item_size(eb, path.slots[0]); + if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) { + unsigned long ptr = 0; + u64 ref_root; + u8 ref_level; + + while (true) { + ret = tree_backref_for_extent(&ptr, eb, &found_key, ei, + item_size, &ref_root, + &ref_level); + if (ret < 0) { + btrfs_warn_rl(fs_info, + "failed to resolve tree backref for logical %llu: %d", + logical, ret); + break; + } + if (ret > 0) + break; + + btrfs_warn_rl(fs_info, +"csum error at logical %llu mirror %u: metadata %s (level %d) in tree %llu", + logical, mirror_num, + (ref_level ? "node" : "leaf"), + ref_level, ref_root); + } + } else { + struct btrfs_backref_walk_ctx ctx = { 0 }; + struct data_reloc_warn reloc_warn = { 0 }; + + /* + * Do not hold the path as later iterate_extent_inodes() call + * can be time consuming. + */ + btrfs_release_path(&path); + + ctx.bytenr = found_key.objectid; + ctx.extent_item_pos = logical - found_key.objectid; + ctx.fs_info = fs_info; + + reloc_warn.logical = logical; + reloc_warn.extent_item_size = found_key.offset; + reloc_warn.mirror_num = mirror_num; + reloc_warn.fs_info = fs_info; + + iterate_extent_inodes(&ctx, true, + data_reloc_print_warning_inode, &reloc_warn); + } +} + +static void __cold btrfs_print_data_csum_error(struct btrfs_inode *inode, + u64 logical_start, u8 *csum, u8 *csum_expected, int mirror_num) +{ + struct btrfs_root *root = inode->root; + const u32 csum_size = root->fs_info->csum_size; + + /* For data reloc tree, it's better to do a backref lookup instead. */ + if (btrfs_is_data_reloc_root(root)) + return print_data_reloc_error(inode, logical_start, csum, + csum_expected, mirror_num); + + /* Output without objectid, which is more meaningful */ + if (btrfs_root_id(root) >= BTRFS_LAST_FREE_OBJECTID) { + btrfs_warn_rl(root->fs_info, +"csum failed root %lld ino %lld off %llu csum " BTRFS_CSUM_FMT " expected csum " BTRFS_CSUM_FMT " mirror %d", + btrfs_root_id(root), btrfs_ino(inode), + logical_start, + BTRFS_CSUM_FMT_VALUE(csum_size, csum), + BTRFS_CSUM_FMT_VALUE(csum_size, csum_expected), + mirror_num); + } else { + btrfs_warn_rl(root->fs_info, +"csum failed root %llu ino %llu off %llu csum " BTRFS_CSUM_FMT " expected csum " BTRFS_CSUM_FMT " mirror %d", + btrfs_root_id(root), btrfs_ino(inode), + logical_start, + BTRFS_CSUM_FMT_VALUE(csum_size, csum), + BTRFS_CSUM_FMT_VALUE(csum_size, csum_expected), + mirror_num); + } +} + +/* + * Lock inode i_rwsem based on arguments passed. + * + * ilock_flags can have the following bit set: + * + * BTRFS_ILOCK_SHARED - acquire a shared lock on the inode + * BTRFS_ILOCK_TRY - try to acquire the lock, if fails on first attempt + * return -EAGAIN + * BTRFS_ILOCK_MMAP - acquire a write lock on the i_mmap_lock + */ +int btrfs_inode_lock(struct btrfs_inode *inode, unsigned int ilock_flags) +{ + if (ilock_flags & BTRFS_ILOCK_SHARED) { + if (ilock_flags & BTRFS_ILOCK_TRY) { + if (!inode_trylock_shared(&inode->vfs_inode)) + return -EAGAIN; + else + return 0; + } + inode_lock_shared(&inode->vfs_inode); + } else { + if (ilock_flags & BTRFS_ILOCK_TRY) { + if (!inode_trylock(&inode->vfs_inode)) + return -EAGAIN; + else + return 0; + } + inode_lock(&inode->vfs_inode); + } + if (ilock_flags & BTRFS_ILOCK_MMAP) + down_write(&inode->i_mmap_lock); + return 0; +} + +/* + * Unlock inode i_rwsem. + * + * ilock_flags should contain the same bits set as passed to btrfs_inode_lock() + * to decide whether the lock acquired is shared or exclusive. + */ +void btrfs_inode_unlock(struct btrfs_inode *inode, unsigned int ilock_flags) +{ + if (ilock_flags & BTRFS_ILOCK_MMAP) + up_write(&inode->i_mmap_lock); + if (ilock_flags & BTRFS_ILOCK_SHARED) + inode_unlock_shared(&inode->vfs_inode); + else + inode_unlock(&inode->vfs_inode); +} + +/* + * Cleanup all submitted ordered extents in specified range to handle errors + * from the btrfs_run_delalloc_range() callback. + * + * NOTE: caller must ensure that when an error happens, it can not call + * extent_clear_unlock_delalloc() to clear both the bits EXTENT_DO_ACCOUNTING + * and EXTENT_DELALLOC simultaneously, because that causes the reserved metadata + * to be released, which we want to happen only when finishing the ordered + * extent (btrfs_finish_ordered_io()). + */ +static inline void btrfs_cleanup_ordered_extents(struct btrfs_inode *inode, + u64 offset, u64 bytes) +{ + return btrfs_mark_ordered_io_finished(inode, offset, bytes, false); +} + +static int btrfs_dirty_inode(struct btrfs_inode *inode); + +static int btrfs_init_inode_security(struct btrfs_trans_handle *trans, + struct btrfs_new_inode_args *args) +{ + int ret; + + if (args->default_acl) { + ret = __btrfs_set_acl(trans, args->inode, args->default_acl, + ACL_TYPE_DEFAULT); + if (ret) + return ret; + } + if (args->acl) { + ret = __btrfs_set_acl(trans, args->inode, args->acl, ACL_TYPE_ACCESS); + if (ret) + return ret; + } + if (!args->default_acl && !args->acl) + cache_no_acl(args->inode); + return btrfs_xattr_security_init(trans, args->inode, args->dir, + &args->dentry->d_name); +} + +/* + * this does all the hard work for inserting an inline extent into + * the btree. The caller should have done a btrfs_drop_extents so that + * no overlapping inline items exist in the btree + */ +static int insert_inline_extent(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + struct btrfs_inode *inode, bool extent_inserted, + size_t size, size_t compressed_size, + int compress_type, + struct folio *compressed_folio, + bool update_i_size) +{ + struct btrfs_root *root = inode->root; + struct extent_buffer *leaf; + const u32 sectorsize = trans->fs_info->sectorsize; + char *kaddr; + unsigned long ptr; + struct btrfs_file_extent_item *ei; + int ret; + size_t cur_size = size; + u64 i_size; + + /* + * The decompressed size must still be no larger than a sector. Under + * heavy race, we can have size == 0 passed in, but that shouldn't be a + * big deal and we can continue the insertion. + */ + ASSERT(size <= sectorsize); + + /* + * The compressed size also needs to be no larger than a page. + * That's also why we only need one folio as the parameter. + */ + if (compressed_folio) { + ASSERT(compressed_size <= sectorsize); + ASSERT(compressed_size <= PAGE_SIZE); + } else { + ASSERT(compressed_size == 0); + } + + if (compressed_size && compressed_folio) + cur_size = compressed_size; + + if (!extent_inserted) { + struct btrfs_key key; + size_t datasize; + + key.objectid = btrfs_ino(inode); + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = 0; + + datasize = btrfs_file_extent_calc_inline_size(cur_size); + ret = btrfs_insert_empty_item(trans, root, path, &key, + datasize); + if (ret) + return ret; + } + leaf = path->nodes[0]; + ei = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_file_extent_item); + btrfs_set_file_extent_generation(leaf, ei, trans->transid); + btrfs_set_file_extent_type(leaf, ei, BTRFS_FILE_EXTENT_INLINE); + btrfs_set_file_extent_encryption(leaf, ei, 0); + btrfs_set_file_extent_other_encoding(leaf, ei, 0); + btrfs_set_file_extent_ram_bytes(leaf, ei, size); + ptr = btrfs_file_extent_inline_start(ei); + + if (compress_type != BTRFS_COMPRESS_NONE) { + kaddr = kmap_local_folio(compressed_folio, 0); + write_extent_buffer(leaf, kaddr, ptr, compressed_size); + kunmap_local(kaddr); + + btrfs_set_file_extent_compression(leaf, ei, + compress_type); + } else { + struct folio *folio; + + folio = filemap_get_folio(inode->vfs_inode.i_mapping, 0); + ASSERT(!IS_ERR(folio)); + btrfs_set_file_extent_compression(leaf, ei, 0); + kaddr = kmap_local_folio(folio, 0); + write_extent_buffer(leaf, kaddr, ptr, size); + kunmap_local(kaddr); + folio_put(folio); + } + btrfs_release_path(path); + + /* + * We align size to sectorsize for inline extents just for simplicity + * sake. + */ + ret = btrfs_inode_set_file_extent_range(inode, 0, + ALIGN(size, root->fs_info->sectorsize)); + if (ret) + return ret; + + /* + * We're an inline extent, so nobody can extend the file past i_size + * without locking a page we already have locked. + * + * We must do any i_size and inode updates before we unlock the pages. + * Otherwise we could end up racing with unlink. + */ + i_size = i_size_read(&inode->vfs_inode); + if (update_i_size && size > i_size) { + i_size_write(&inode->vfs_inode, size); + i_size = size; + } + inode->disk_i_size = i_size; + + return 0; +} + +static bool can_cow_file_range_inline(struct btrfs_inode *inode, + u64 offset, u64 size, + size_t compressed_size) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + u64 data_len = (compressed_size ?: size); + + /* Inline extents must start at offset 0. */ + if (offset != 0) + return false; + + /* + * Even for bs > ps cases, cow_file_range_inline() can only accept a + * single folio. + * + * This can be problematic and cause access beyond page boundary if a + * page sized folio is passed into that function. + * And encoded write is doing exactly that. + * So here limits the inlined extent size to PAGE_SIZE. + */ + if (size > PAGE_SIZE || compressed_size > PAGE_SIZE) + return false; + + /* Inline extents are limited to sectorsize. */ + if (size > fs_info->sectorsize) + return false; + + /* We do not allow a non-compressed extent to be as large as block size. */ + if (data_len >= fs_info->sectorsize) + return false; + + /* We cannot exceed the maximum inline data size. */ + if (data_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info)) + return false; + + /* We cannot exceed the user specified max_inline size. */ + if (data_len > fs_info->max_inline) + return false; + + /* Inline extents must be the entirety of the file. */ + if (size < i_size_read(&inode->vfs_inode)) + return false; + + /* Encrypted file cannot be inlined. */ + if (IS_ENCRYPTED(&inode->vfs_inode)) + return false; + + return true; +} + +/* + * conditionally insert an inline extent into the file. This + * does the checks required to make sure the data is small enough + * to fit as an inline extent. + * + * If being used directly, you must have already checked we're allowed to cow + * the range by getting true from can_cow_file_range_inline(). + * + * Return 0 if the inlined extent is created successfully. + * Return <0 for critical error, and should be considered as an writeback error. + * Return >0 if can not create an inlined extent (mostly due to lack of meta space). + */ +static noinline int __cow_file_range_inline(struct btrfs_inode *inode, + u64 size, size_t compressed_size, + int compress_type, + struct folio *compressed_folio, + bool update_i_size) +{ + struct btrfs_drop_extents_args drop_args = { 0 }; + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_trans_handle *trans = NULL; + u64 data_len = (compressed_size ?: size); + int ret; + struct btrfs_path *path; + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto out; + } + + trans = btrfs_join_transaction(root); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + trans = NULL; + goto out; + } + trans->block_rsv = &inode->block_rsv; + + drop_args.path = path; + drop_args.start = 0; + drop_args.end = fs_info->sectorsize; + drop_args.drop_cache = true; + drop_args.replace_extent = true; + drop_args.extent_item_size = btrfs_file_extent_calc_inline_size(data_len); + ret = btrfs_drop_extents(trans, root, inode, &drop_args); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + + ret = insert_inline_extent(trans, path, inode, drop_args.extent_inserted, + size, compressed_size, compress_type, + compressed_folio, update_i_size); + if (unlikely(ret && ret != -ENOSPC)) { + btrfs_abort_transaction(trans, ret); + goto out; + } else if (ret == -ENOSPC) { + ret = 1; + goto out; + } + + btrfs_update_inode_bytes(inode, size, drop_args.bytes_found); + ret = btrfs_update_inode(trans, inode); + if (unlikely(ret && ret != -ENOSPC)) { + btrfs_abort_transaction(trans, ret); + goto out; + } else if (ret == -ENOSPC) { + ret = 1; + goto out; + } + + btrfs_set_inode_full_sync(inode); +out: + /* + * Don't forget to free the reserved space, as for inlined extent + * it won't count as data extent, free them directly here. + * And at reserve time, it's always aligned to sector size, so + * just free one sector here. + * + * If we fallback to non-inline (ret == 1) due to -ENOSPC, then we need + * to keep the data reservation. + */ + if (ret <= 0) + btrfs_qgroup_free_data(inode, NULL, 0, fs_info->sectorsize, NULL); + btrfs_free_path(path); + if (trans) + btrfs_end_transaction(trans); + return ret; +} + +struct async_extent { + u64 start; + u64 ram_size; + struct compressed_bio *cb; + struct list_head list; +}; + +struct async_chunk { + struct btrfs_inode *inode; + struct folio *locked_folio; + u64 start; + u64 end; + blk_opf_t write_flags; + struct list_head extents; + struct cgroup_subsys_state *blkcg_css; + struct btrfs_work work; + struct async_cow *async_cow; +}; + +struct async_cow { + atomic_t num_chunks; + struct async_chunk chunks[]; +}; + +static int add_async_extent(struct async_chunk *cow, u64 start, u64 ram_size, + struct compressed_bio *cb) +{ + struct async_extent *async_extent; + + async_extent = kmalloc_obj(*async_extent, GFP_NOFS); + if (!async_extent) + return -ENOMEM; + ASSERT(ram_size < U32_MAX); + async_extent->start = start; + async_extent->ram_size = ram_size; + async_extent->cb = cb; + list_add_tail(&async_extent->list, &cow->extents); + return 0; +} + +/* + * Check if the inode needs to be submitted to compression, based on mount + * options, defragmentation, properties or heuristics. + */ +static inline int inode_need_compress(struct btrfs_inode *inode, u64 start, + u64 end, bool check_inline) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + + if (unlikely(!btrfs_inode_can_compress(inode))) { + DEBUG_WARN("BTRFS: unexpected compression for ino %llu", btrfs_ino(inode)); + return 0; + } + + /* + * If the delalloc range is only one fs block and can not be inlined, + * do not even bother try compression, as there will be no space saving + * and will always fallback to regular write later. + */ + if (end + 1 - start <= fs_info->sectorsize && + (!check_inline || (start > 0 || end + 1 < inode->disk_i_size))) + return 0; + + /* Defrag ioctl takes precedence over mount options and properties. */ + if (inode->defrag_compress == BTRFS_DEFRAG_DONT_COMPRESS) + return 0; + if (BTRFS_COMPRESS_NONE < inode->defrag_compress && + inode->defrag_compress < BTRFS_NR_COMPRESS_TYPES) + return 1; + /* force compress */ + if (btrfs_test_opt(fs_info, FORCE_COMPRESS)) + return 1; + /* bad compression ratios */ + if (inode->flags & BTRFS_INODE_NOCOMPRESS) + return 0; + if (btrfs_test_opt(fs_info, COMPRESS) || + inode->flags & BTRFS_INODE_COMPRESS || + inode->prop_compress) + return btrfs_compress_heuristic(inode, start, end); + return 0; +} + +static inline void inode_should_defrag(struct btrfs_inode *inode, + u64 start, u64 end, u64 num_bytes, u32 small_write) +{ + /* If this is a small write inside eof, kick off a defrag */ + if (num_bytes < small_write && + (start > 0 || end + 1 < inode->disk_i_size)) + btrfs_add_inode_defrag(inode, small_write); +} + +static int extent_range_clear_dirty_for_io(struct btrfs_inode *inode, u64 start, u64 end) +{ + pgoff_t index = start >> PAGE_SHIFT; + const pgoff_t end_index = end >> PAGE_SHIFT; + struct folio *folio; + int ret = 0; + + while (index <= end_index) { + folio = filemap_get_folio(inode->vfs_inode.i_mapping, index); + if (IS_ERR(folio)) { + if (!ret) + ret = PTR_ERR(folio); + index++; + continue; + } + /* + * We are about to compress the folio, so it must not be mmap + * writeable or we could corrupt the data as we attempt to + * compress it. + */ + btrfs_check_folio_write_protected(folio); + btrfs_folio_clamp_clear_dirty(inode->root->fs_info, folio, start, + end + 1 - start); + index = folio_next_index(folio); + folio_put(folio); + } + return ret; +} + +static struct folio *compressed_bio_last_folio(struct compressed_bio *cb) +{ + struct bio *bio = &cb->bbio.bio; + struct bio_vec *bvec; + phys_addr_t paddr; + + /* + * Make sure all folios have the same min_folio_size. + * + * Otherwise we cannot simply use offset_in_offset(folio, bi_size) to + * calculate the end of the last folio. + */ + if (IS_ENABLED(CONFIG_BTRFS_ASSERT)) { + struct btrfs_fs_info *fs_info = cb_to_fs_info(cb); + const u32 min_folio_size = btrfs_min_folio_size(fs_info); + struct folio_iter fi; + + bio_for_each_folio_all(fi, bio) + ASSERT(folio_size(fi.folio) == min_folio_size); + } + + /* The bio must not be empty. */ + ASSERT(bio->bi_vcnt); + + bvec = &bio->bi_io_vec[bio->bi_vcnt - 1]; + paddr = bvec_phys(bvec) + bvec->bv_len - 1; + return page_folio(phys_to_page(paddr)); +} + +static void round_up_last_block(struct compressed_bio *cb, u32 blocksize) +{ + struct bio *bio = &cb->bbio.bio; + struct folio *last_folio = compressed_bio_last_folio(cb); + const u32 bio_size = bio->bi_iter.bi_size; + const u32 foffset = offset_in_folio(last_folio, bio_size); + const u32 padding_len = round_up(foffset, blocksize) - foffset; + bool ret; + + if (IS_ALIGNED(bio_size, blocksize)) + return; + + folio_zero_range(last_folio, foffset, padding_len); + ret = bio_add_folio(bio, last_folio, padding_len, foffset); + /* The remaining part should be merged thus never fail. */ + ASSERT(ret); +} + +/* + * Work queue call back to started compression on a file and pages. + * + * This is done inside an ordered work queue, and the compression is spread + * across many cpus. The actual IO submission is step two, and the ordered work + * queue takes care of making sure that happens in the same order things were + * put onto the queue by writepages and friends. + * + * If this code finds it can't get good compression, it puts an entry onto the + * work queue to write the uncompressed bytes. This makes sure that both + * compressed inodes and uncompressed inodes are written in the same order that + * the flusher thread sent them down. + */ +static void compress_file_range(struct btrfs_work *work) +{ + struct async_chunk *async_chunk = + container_of(work, struct async_chunk, work); + struct btrfs_inode *inode = async_chunk->inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct compressed_bio *cb = NULL; + const u32 blocksize = fs_info->sectorsize; + u64 start = async_chunk->start; + u64 end = async_chunk->end; + u64 actual_end; + u64 i_size; + u32 cur_len; + int ret = 0; + unsigned long total_compressed = 0; + unsigned long total_in = 0; + int compress_type = fs_info->compress_type; + int compress_level = fs_info->compress_level; + + if (btrfs_is_shutdown(fs_info)) + goto cleanup_and_bail_uncompressed; + + inode_should_defrag(inode, start, end, end - start + 1, SZ_16K); + + ret = extent_range_clear_dirty_for_io(inode, start, end); + + /* + * All the folios should have been locked thus no failure. + * + * And even if some folios are missing, btrfs_compress_bio() + * would handle them correctly, so here just do an ASSERT() check for + * early logic errors. + */ + ASSERT(ret == 0); + + /* + * We need to save i_size before now because it could change in between + * us evaluating the size and assigning it. This is because we lock and + * unlock the page in truncate and fallocate, and then modify the i_size + * later on. + * + * The barriers are to emulate READ_ONCE, remove that once i_size_read + * does that for us. + */ + barrier(); + i_size = i_size_read(&inode->vfs_inode); + barrier(); + actual_end = min_t(u64, i_size, end + 1); +again: + total_in = 0; + cur_len = min(end + 1 - start, BTRFS_MAX_UNCOMPRESSED); + ret = 0; + cb = NULL; + + /* + * we don't want to send crud past the end of i_size through + * compression, that's just a waste of CPU time. So, if the + * end of the file is before the start of our current + * requested range of bytes, we bail out to the uncompressed + * cleanup code that can deal with all of this. + * + * It isn't really the fastest way to fix things, but this is a + * very uncommon corner. + */ + if (actual_end <= start) + goto cleanup_and_bail_uncompressed; + + /* + * We do compression for mount -o compress and when the inode has not + * been flagged as NOCOMPRESS. This flag can change at any time if we + * discover bad compression ratios. + */ + if (!inode_need_compress(inode, start, end, false)) + goto cleanup_and_bail_uncompressed; + + if (0 < inode->defrag_compress && inode->defrag_compress < BTRFS_NR_COMPRESS_TYPES) { + compress_type = inode->defrag_compress; + compress_level = inode->defrag_compress_level; + } else if (inode->prop_compress) { + compress_type = inode->prop_compress; + } + + /* Compression level is applied here. */ + cb = btrfs_compress_bio(inode, start, cur_len, compress_type, + compress_level, async_chunk->write_flags); + if (IS_ERR(cb)) { + cb = NULL; + goto mark_incompressible; + } + + total_compressed = cb->bbio.bio.bi_iter.bi_size; + total_in = cur_len; + + /* + * We aren't doing an inline extent. Round the compressed size up to a + * block size boundary so the allocator does sane things. + */ + round_up_last_block(cb, blocksize); + total_compressed = cb->bbio.bio.bi_iter.bi_size; + ASSERT(IS_ALIGNED(total_compressed, blocksize)); + + /* + * One last check to make sure the compression is really a win, compare + * the page count read with the blocks on disk, compression must free at + * least one sector. + */ + total_in = round_up(total_in, fs_info->sectorsize); + if (total_compressed + blocksize > total_in) + goto mark_incompressible; + + + /* + * The async work queues will take care of doing actual allocation on + * disk for these compressed pages, and will submit the bios. + */ + ret = add_async_extent(async_chunk, start, total_in, cb); + BUG_ON(ret); + if (start + total_in < end) { + start += total_in; + cond_resched(); + goto again; + } + return; + +mark_incompressible: + if (!btrfs_test_opt(fs_info, FORCE_COMPRESS) && !inode->prop_compress) + inode->flags |= BTRFS_INODE_NOCOMPRESS; +cleanup_and_bail_uncompressed: + ret = add_async_extent(async_chunk, start, end - start + 1, NULL); + BUG_ON(ret); + if (cb) + cleanup_compressed_bio(cb); +} + +static void submit_uncompressed_range(struct btrfs_inode *inode, + struct async_extent *async_extent, + struct folio *locked_folio) +{ + u64 start = async_extent->start; + u64 end = async_extent->start + async_extent->ram_size - 1; + int ret; + struct writeback_control wbc = { + .sync_mode = WB_SYNC_ALL, + .range_start = start, + .range_end = end, + .no_cgroup_owner = 1, + }; + + wbc_attach_fdatawrite_inode(&wbc, &inode->vfs_inode); + ret = run_delalloc_cow(inode, locked_folio, start, end, + &wbc, false); + wbc_detach_inode(&wbc); + if (ret < 0) { + if (locked_folio) + btrfs_folio_end_lock(inode->root->fs_info, locked_folio, + start, async_extent->ram_size); + btrfs_err_rl(inode->root->fs_info, + "%s failed, root=%llu inode=%llu start=%llu len=%llu: %pe", + __func__, btrfs_root_id(inode->root), + btrfs_ino(inode), start, async_extent->ram_size, + ERR_PTR(ret)); + } +} + +static void submit_one_async_extent(struct async_chunk *async_chunk, + struct async_extent *async_extent, + u64 *alloc_hint) +{ + struct btrfs_inode *inode = async_chunk->inode; + struct extent_io_tree *io_tree = &inode->io_tree; + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_ordered_extent *ordered; + struct btrfs_file_extent file_extent; + struct btrfs_key ins; + struct folio *locked_folio = NULL; + struct extent_state *cached = NULL; + struct extent_map *em; + int ret = 0; + u32 compressed_size; + u64 start = async_extent->start; + u64 end = async_extent->start + async_extent->ram_size - 1; + + if (async_chunk->blkcg_css) + kthread_associate_blkcg(async_chunk->blkcg_css); + + /* + * If async_chunk->locked_folio is in the async_extent range, we need to + * handle it. + */ + if (async_chunk->locked_folio) { + u64 locked_folio_start = folio_pos(async_chunk->locked_folio); + u64 locked_folio_end = locked_folio_start + + folio_size(async_chunk->locked_folio) - 1; + + if (!(start >= locked_folio_end || end <= locked_folio_start)) + locked_folio = async_chunk->locked_folio; + } + + if (!async_extent->cb) { + submit_uncompressed_range(inode, async_extent, locked_folio); + goto done; + } + + compressed_size = async_extent->cb->bbio.bio.bi_iter.bi_size; + ret = btrfs_reserve_extent(root, async_extent->ram_size, + compressed_size, compressed_size, + 0, *alloc_hint, &ins, true, true); + if (ret) { + /* + * We can't reserve contiguous space for the compressed size. + * Unlikely, but it's possible that we could have enough + * non-contiguous space for the uncompressed size instead. So + * fall back to uncompressed. + */ + submit_uncompressed_range(inode, async_extent, locked_folio); + cleanup_compressed_bio(async_extent->cb); + async_extent->cb = NULL; + goto done; + } + + btrfs_lock_extent(io_tree, start, end, &cached); + + /* Here we're doing allocation and writeback of the compressed pages */ + file_extent.disk_bytenr = ins.objectid; + file_extent.disk_num_bytes = ins.offset; + file_extent.ram_bytes = async_extent->ram_size; + file_extent.num_bytes = async_extent->ram_size; + file_extent.offset = 0; + file_extent.compression = async_extent->cb->compress_type; + + async_extent->cb->bbio.bio.bi_iter.bi_sector = ins.objectid >> SECTOR_SHIFT; + + em = btrfs_create_io_em(inode, start, &file_extent, BTRFS_ORDERED_COMPRESSED); + if (IS_ERR(em)) { + ret = PTR_ERR(em); + goto out_free_reserve; + } + btrfs_free_extent_map(em); + + ordered = btrfs_alloc_ordered_extent(inode, start, &file_extent, + 1U << BTRFS_ORDERED_COMPRESSED); + if (IS_ERR(ordered)) { + btrfs_drop_extent_map_range(inode, start, end, false); + ret = PTR_ERR(ordered); + goto out_free_reserve; + } + async_extent->cb->bbio.ordered = ordered; + btrfs_dec_block_group_reservations(fs_info, ins.objectid); + + /* Clear dirty, set writeback and unlock the pages. */ + extent_clear_unlock_delalloc(inode, start, end, + NULL, &cached, EXTENT_LOCKED | EXTENT_DELALLOC, + PAGE_UNLOCK | PAGE_START_WRITEBACK); + btrfs_submit_bbio(&async_extent->cb->bbio, 0); + async_extent->cb = NULL; + + *alloc_hint = ins.objectid + ins.offset; +done: + if (async_chunk->blkcg_css) + kthread_associate_blkcg(NULL); + kfree(async_extent); + return; + +out_free_reserve: + btrfs_dec_block_group_reservations(fs_info, ins.objectid); + btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, true); + mapping_set_error(inode->vfs_inode.i_mapping, -EIO); + extent_clear_unlock_delalloc(inode, start, end, + NULL, &cached, + EXTENT_LOCKED | EXTENT_DELALLOC | + EXTENT_DELALLOC_NEW | + EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV, + PAGE_UNLOCK | PAGE_START_WRITEBACK | + PAGE_END_WRITEBACK); + if (async_extent->cb) + cleanup_compressed_bio(async_extent->cb); + if (async_chunk->blkcg_css) + kthread_associate_blkcg(NULL); + btrfs_debug(fs_info, +"async extent submission failed root=%lld inode=%llu start=%llu len=%llu ret=%d", + btrfs_root_id(root), btrfs_ino(inode), start, + async_extent->ram_size, ret); + kfree(async_extent); +} + +u64 btrfs_get_extent_allocation_hint(struct btrfs_inode *inode, u64 start, + u64 num_bytes) +{ + struct extent_map_tree *em_tree = &inode->extent_tree; + struct extent_map *em; + u64 alloc_hint = 0; + + read_lock(&em_tree->lock); + em = btrfs_search_extent_mapping(em_tree, start, num_bytes); + if (em) { + /* + * if block start isn't an actual block number then find the + * first block in this inode and use that as a hint. If that + * block is also bogus then just don't worry about it. + */ + if (em->disk_bytenr >= EXTENT_MAP_LAST_BYTE) { + btrfs_free_extent_map(em); + em = btrfs_search_extent_mapping(em_tree, 0, 0); + if (em && em->disk_bytenr < EXTENT_MAP_LAST_BYTE) + alloc_hint = btrfs_extent_map_block_start(em); + if (em) + btrfs_free_extent_map(em); + } else { + alloc_hint = btrfs_extent_map_block_start(em); + btrfs_free_extent_map(em); + } + } + read_unlock(&em_tree->lock); + + return alloc_hint; +} + +/* + * Handle COW for one range. + * + * @ins: The key representing the allocated range. + * @file_offset: The file offset of the COW range + * @num_bytes: The expected length of the COW range + * The actually allocated length can be smaller than it. + * @min_alloc_size: The minimal extent size. + * @alloc_hint: The hint for the extent allocator. + * @ret_alloc_size: The COW range handles by this function. + * + * Return 0 if everything is fine and update @ret_alloc_size updated. The + * range is still locked, and caller should unlock the range after everything + * is done or for error handling. + * + * Return <0 for error and @is updated for where the extra cleanup should + * happen. The range [file_offset, file_offset + ret_alloc_size) will be + * cleaned up by this function. + */ +static int cow_one_range(struct btrfs_inode *inode, struct folio *locked_folio, + struct btrfs_key *ins, struct extent_state **cached, + u64 file_offset, u32 num_bytes, u32 min_alloc_size, + u64 alloc_hint, u32 *ret_alloc_size) +{ + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_ordered_extent *ordered; + struct btrfs_file_extent file_extent; + struct extent_map *em; + u32 cur_len = 0; + u64 cur_end; + int ret; + + ret = btrfs_reserve_extent(root, num_bytes, num_bytes, min_alloc_size, + 0, alloc_hint, ins, true, true); + if (ret < 0) { + *ret_alloc_size = cur_len; + return ret; + } + + cur_len = ins->offset; + cur_end = file_offset + cur_len - 1; + + file_extent.disk_bytenr = ins->objectid; + file_extent.disk_num_bytes = ins->offset; + file_extent.num_bytes = ins->offset; + file_extent.ram_bytes = ins->offset; + file_extent.offset = 0; + file_extent.compression = BTRFS_COMPRESS_NONE; + + /* + * Locked range will be released either during error clean up (inside + * this function or by the caller for previously successful ranges) or + * after the whole range is finished. + */ + btrfs_lock_extent(&inode->io_tree, file_offset, cur_end, cached); + em = btrfs_create_io_em(inode, file_offset, &file_extent, BTRFS_ORDERED_REGULAR); + if (IS_ERR(em)) { + ret = PTR_ERR(em); + goto free_reserved; + } + btrfs_free_extent_map(em); + + ordered = btrfs_alloc_ordered_extent(inode, file_offset, &file_extent, + 1U << BTRFS_ORDERED_REGULAR); + if (IS_ERR(ordered)) { + btrfs_drop_extent_map_range(inode, file_offset, cur_end, false); + ret = PTR_ERR(ordered); + goto free_reserved; + } + + if (btrfs_is_data_reloc_root(root)) { + ret = btrfs_reloc_clone_csums(ordered); + + /* + * Only drop cache here, and process as normal. + * + * We must not allow extent_clear_unlock_delalloc() at + * free_reserved label to free meta of this ordered extent, as + * its meta should be freed by btrfs_finish_ordered_io(). + * + * So we must continue until @start is increased to + * skip current ordered extent. + */ + if (ret) + btrfs_drop_extent_map_range(inode, file_offset, + cur_end, false); + } + btrfs_put_ordered_extent(ordered); + btrfs_dec_block_group_reservations(fs_info, ins->objectid); + /* + * Error handling for btrfs_reloc_clone_csums(). + * + * Treat the range as finished, thus only clear EXTENT_LOCKED | EXTENT_DELALLOC. + * The accounting will be done by ordered extents. + */ + if (unlikely(ret < 0)) { + btrfs_cleanup_ordered_extents(inode, file_offset, cur_len); + extent_clear_unlock_delalloc(inode, file_offset, cur_end, locked_folio, cached, + EXTENT_LOCKED | EXTENT_DELALLOC, + PAGE_UNLOCK | PAGE_START_WRITEBACK | + PAGE_END_WRITEBACK); + mapping_set_error(inode->vfs_inode.i_mapping, -EIO); + } + *ret_alloc_size = cur_len; + return ret; + +free_reserved: + /* + * If we have reserved an extent for the current range and failed to + * create the respective extent map or ordered extent, it means that + * when we reserved the extent we decremented the extent's size from + * the data space_info's bytes_may_use counter and + * incremented the space_info's bytes_reserved counter by the same + * amount. + * + * We must make sure extent_clear_unlock_delalloc() does not try + * to decrement again the data space_info's bytes_may_use counter, which + * will be handled by btrfs_free_reserved_extent(). + * + * Therefore we do not pass it the flag EXTENT_CLEAR_DATA_RESV, but only + * EXTENT_CLEAR_META_RESV. + */ + extent_clear_unlock_delalloc(inode, file_offset, cur_end, locked_folio, cached, + EXTENT_LOCKED | EXTENT_DELALLOC | + EXTENT_DELALLOC_NEW | + EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV, + PAGE_UNLOCK | PAGE_START_WRITEBACK | + PAGE_END_WRITEBACK); + btrfs_qgroup_free_data(inode, NULL, file_offset, cur_len, NULL); + btrfs_dec_block_group_reservations(fs_info, ins->objectid); + btrfs_free_reserved_extent(fs_info, ins->objectid, ins->offset, true); + mapping_set_error(inode->vfs_inode.i_mapping, -EIO); + *ret_alloc_size = cur_len; + /* + * We should not return -EAGAIN where it's a special return code for + * zoned to catch btrfs_reserved_extent(). + */ + ASSERT(ret != -EAGAIN); + return ret; +} + +/* + * when extent_io.c finds a delayed allocation range in the file, + * the call backs end up in this code. The basic idea is to + * allocate extents on disk for the range, and create ordered data structs + * in ram to track those extents. + * + * locked_folio is the folio that writepage had locked already. We use + * it to make sure we don't do extra locks or unlocks. + * + * When this function fails, it unlocks all folios except @locked_folio. + * + * When this function succeed and creates a normal extent, the folio locking + * status depends on the passed in flags: + * + * - If COW_FILE_RANGE_KEEP_LOCKED flag is set, all folios are kept locked. + * - Else all folios except for @locked_folio are unlocked. + * + * When a failure happens in the second or later iteration of the + * while-loop, the ordered extents created in previous iterations are cleaned up. + */ +static noinline int cow_file_range(struct btrfs_inode *inode, + struct folio *locked_folio, u64 start, + u64 end, u64 *done_offset, + unsigned long flags) +{ + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct extent_state *cached = NULL; + u64 alloc_hint = 0; + u64 orig_start = start; + u64 num_bytes; + u32 min_alloc_size; + u32 blocksize = fs_info->sectorsize; + u32 cur_alloc_size = 0; + struct btrfs_key ins; + unsigned clear_bits; + unsigned long page_ops; + int ret = 0; + + if (btrfs_is_shutdown(fs_info)) { + ret = -EIO; + goto out_unlock; + } + + if (btrfs_is_free_space_inode(inode)) { + ret = -EINVAL; + goto out_unlock; + } + + num_bytes = ALIGN(end - start + 1, blocksize); + num_bytes = max(blocksize, num_bytes); + ASSERT(num_bytes <= btrfs_super_total_bytes(fs_info->super_copy)); + + inode_should_defrag(inode, start, end, num_bytes, SZ_64K); + alloc_hint = btrfs_get_extent_allocation_hint(inode, start, num_bytes); + + /* + * We're not doing compressed IO, don't unlock the first page (which + * the caller expects to stay locked), don't clear any dirty bits and + * don't set any writeback bits. + * + * Do set the Ordered (Private2) bit so we know this page was properly + * setup for writepage. + */ + page_ops = ((flags & COW_FILE_RANGE_KEEP_LOCKED) ? 0 : PAGE_UNLOCK); + + /* + * Relocation relies on the relocated extents to have exactly the same + * size as the original extents. Normally writeback for relocation data + * extents follows a NOCOW path because relocation preallocates the + * extents. However, due to an operation such as scrub turning a block + * group to RO mode, it may fallback to COW mode, so we must make sure + * an extent allocated during COW has exactly the requested size and can + * not be split into smaller extents, otherwise relocation breaks and + * fails during the stage where it updates the bytenr of file extent + * items. + */ + if (btrfs_is_data_reloc_root(root)) + min_alloc_size = num_bytes; + else + min_alloc_size = fs_info->sectorsize; + + while (num_bytes > 0) { + ret = cow_one_range(inode, locked_folio, &ins, &cached, start, + num_bytes, min_alloc_size, alloc_hint, &cur_alloc_size); + + if (ret == -EAGAIN) { + /* + * cow_one_range() only returns -EAGAIN for zoned + * file systems (from btrfs_reserve_extent()), which + * is an indication that there are + * no active zones to allocate from at the moment. + * + * If this is the first loop iteration, wait for at + * least one zone to finish before retrying the + * allocation. Otherwise ask the caller to write out + * the already allocated blocks before coming back to + * us, or return -ENOSPC if it can't handle retries. + */ + ASSERT(btrfs_is_zoned(fs_info)); + if (start == orig_start) { + wait_on_bit_io(&inode->root->fs_info->flags, + BTRFS_FS_NEED_ZONE_FINISH, + TASK_UNINTERRUPTIBLE); + continue; + } + if (done_offset) { + /* + * Move @end to the end of the processed range, + * and exit the loop to unlock the processed extents. + */ + end = start - 1; + ret = 0; + break; + } + ret = -ENOSPC; + } + if (ret < 0) + goto out_unlock; + + /* We should not allocate an extent larger than requested.*/ + ASSERT(cur_alloc_size <= num_bytes); + + num_bytes -= cur_alloc_size; + alloc_hint = ins.objectid + ins.offset; + start += cur_alloc_size; + cur_alloc_size = 0; + } + extent_clear_unlock_delalloc(inode, orig_start, end, locked_folio, &cached, + EXTENT_LOCKED | EXTENT_DELALLOC, page_ops); + if (done_offset) + *done_offset = end; + return ret; + +out_unlock: + /* + * Now, we have three regions to clean up: + * + * |-------(1)----|---(2)---|-------------(3)----------| + * `- orig_start `- start `- start + cur_alloc_size `- end + * + * We process each region below. + */ + + /* + * For the range (1). We have already instantiated the ordered extents + * for this region, thus we need to cleanup those ordered extents. + * EXTENT_DELALLOC_NEW | EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV + * are also handled by the ordered extents cleanup. + * + * So here we only clear EXTENT_LOCKED and EXTENT_DELALLOC flag, and + * finish the writeback of the involved folios, which will be never submitted. + */ + if (orig_start < start) { + clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC; + page_ops = PAGE_UNLOCK | PAGE_START_WRITEBACK | PAGE_END_WRITEBACK; + + if (!locked_folio) + mapping_set_error(inode->vfs_inode.i_mapping, ret); + + btrfs_cleanup_ordered_extents(inode, orig_start, start - orig_start); + extent_clear_unlock_delalloc(inode, orig_start, start - 1, + locked_folio, NULL, clear_bits, page_ops); + } + + clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_DELALLOC_NEW | + EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV; + page_ops = PAGE_UNLOCK | PAGE_START_WRITEBACK | PAGE_END_WRITEBACK; + + /* + * For the range (2) the error handling is done by cow_one_range() itself. + * Nothing needs to be done. + * + * For the range (3). We never touched the region. In addition to the + * clear_bits above, we add EXTENT_CLEAR_DATA_RESV to release the data + * space_info's bytes_may_use counter, reserved in + * btrfs_check_data_free_space(). + */ + if (start + cur_alloc_size < end) { + clear_bits |= EXTENT_CLEAR_DATA_RESV; + extent_clear_unlock_delalloc(inode, start + cur_alloc_size, + end, locked_folio, + &cached, clear_bits, page_ops); + btrfs_qgroup_free_data(inode, NULL, start + cur_alloc_size, + end - start - cur_alloc_size + 1, NULL); + } + btrfs_err(fs_info, +"%s failed, root=%llu inode=%llu start=%llu len=%llu cur_offset=%llu cur_alloc_size=%u: %pe", + __func__, btrfs_root_id(inode->root), + btrfs_ino(inode), orig_start, end + 1 - orig_start, + start, cur_alloc_size, ERR_PTR(ret)); + return ret; +} + +/* + * Phase two of compressed writeback. This is the ordered portion of the code, + * which only gets called in the order the work was queued. We walk all the + * async extents created by compress_file_range and send them down to the disk. + * + * If called with @do_free == true then it'll try to finish the work and free + * the work struct eventually. + */ +static noinline void submit_compressed_extents(struct btrfs_work *work, bool do_free) +{ + struct async_chunk *async_chunk = container_of(work, struct async_chunk, + work); + struct btrfs_fs_info *fs_info = btrfs_work_owner(work); + struct async_extent *async_extent; + unsigned long nr_pages; + u64 alloc_hint = 0; + + if (do_free) { + struct async_cow *async_cow; + + btrfs_add_delayed_iput(async_chunk->inode); + if (async_chunk->blkcg_css) + css_put(async_chunk->blkcg_css); + + async_cow = async_chunk->async_cow; + if (atomic_dec_and_test(&async_cow->num_chunks)) + kvfree(async_cow); + return; + } + + nr_pages = (async_chunk->end - async_chunk->start + PAGE_SIZE) >> + PAGE_SHIFT; + + while (!list_empty(&async_chunk->extents)) { + async_extent = list_first_entry(&async_chunk->extents, + struct async_extent, list); + list_del(&async_extent->list); + submit_one_async_extent(async_chunk, async_extent, &alloc_hint); + } + + /* atomic_sub_return implies a barrier */ + if (atomic_sub_return(nr_pages, &fs_info->async_delalloc_pages) < + 5 * SZ_1M) + cond_wake_up_nomb(&fs_info->async_submit_wait); +} + +static bool run_delalloc_compressed(struct btrfs_inode *inode, + struct folio *locked_folio, u64 start, + u64 end, struct writeback_control *wbc) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct cgroup_subsys_state *blkcg_css = wbc_blkcg_css(wbc); + struct async_cow *ctx; + struct async_chunk *async_chunk; + unsigned long nr_pages; + u64 num_chunks = DIV_ROUND_UP(end - start, BTRFS_COMPRESSION_CHUNK_SIZE); + int i; + unsigned nofs_flag; + const blk_opf_t write_flags = wbc_to_write_flags(wbc); + + nofs_flag = memalloc_nofs_save(); + ctx = kvmalloc_flex(*ctx, chunks, num_chunks); + memalloc_nofs_restore(nofs_flag); + if (!ctx) + return false; + + set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, &inode->runtime_flags); + + async_chunk = ctx->chunks; + atomic_set(&ctx->num_chunks, num_chunks); + + for (i = 0; i < num_chunks; i++) { + u64 cur_end = min(end, start + BTRFS_COMPRESSION_CHUNK_SIZE - 1); + + /* + * igrab is called higher up in the call chain, take only the + * lightweight reference for the callback lifetime + */ + ihold(&inode->vfs_inode); + async_chunk[i].async_cow = ctx; + async_chunk[i].inode = inode; + async_chunk[i].start = start; + async_chunk[i].end = cur_end; + async_chunk[i].write_flags = write_flags; + INIT_LIST_HEAD(&async_chunk[i].extents); + + /* + * The locked_folio comes all the way from writepage and its + * the original folio we were actually given. As we spread + * this large delalloc region across multiple async_chunk + * structs, only the first struct needs a pointer to + * locked_folio. + * + * This way we don't need racey decisions about who is supposed + * to unlock it. + */ + if (locked_folio) { + /* + * Depending on the compressibility, the pages might or + * might not go through async. We want all of them to + * be accounted against wbc once. Let's do it here + * before the paths diverge. wbc accounting is used + * only for foreign writeback detection and doesn't + * need full accuracy. Just account the whole thing + * against the first page. + */ + wbc_account_cgroup_owner(wbc, locked_folio, + cur_end - start); + async_chunk[i].locked_folio = locked_folio; + locked_folio = NULL; + } else { + async_chunk[i].locked_folio = NULL; + } + + if (blkcg_css != blkcg_root_css) { + css_get(blkcg_css); + async_chunk[i].blkcg_css = blkcg_css; + async_chunk[i].write_flags |= REQ_BTRFS_CGROUP_PUNT; + } else { + async_chunk[i].blkcg_css = NULL; + } + + btrfs_init_work(&async_chunk[i].work, compress_file_range, + submit_compressed_extents); + + nr_pages = DIV_ROUND_UP(cur_end - start, PAGE_SIZE); + atomic_add(nr_pages, &fs_info->async_delalloc_pages); + + btrfs_queue_work(fs_info->delalloc_workers, &async_chunk[i].work); + + start = cur_end + 1; + } + return true; +} + +/* + * Run the delalloc range from start to end, and write back any dirty pages + * covered by the range. + */ +static noinline int run_delalloc_cow(struct btrfs_inode *inode, + struct folio *locked_folio, u64 start, + u64 end, struct writeback_control *wbc, + bool pages_dirty) +{ + u64 done_offset = end; + int ret; + + while (start <= end) { + ret = cow_file_range(inode, locked_folio, start, end, + &done_offset, COW_FILE_RANGE_KEEP_LOCKED); + if (ret) + return ret; + extent_write_locked_range(&inode->vfs_inode, locked_folio, + start, done_offset, wbc, pages_dirty); + start = done_offset + 1; + } + + return 1; +} + +static int fallback_to_cow(struct btrfs_inode *inode, + struct folio *locked_folio, const u64 start, + const u64 end) +{ + const bool is_space_ino = btrfs_is_free_space_inode(inode); + const bool is_reloc_ino = btrfs_is_data_reloc_root(inode->root); + const u64 range_bytes = end + 1 - start; + struct extent_io_tree *io_tree = &inode->io_tree; + struct extent_state *cached_state = NULL; + u64 range_start = start; + u64 count; + int ret; + + /* + * If EXTENT_NORESERVE is set it means that when the buffered write was + * made we had not enough available data space and therefore we did not + * reserve data space for it, since we though we could do NOCOW for the + * respective file range (either there is prealloc extent or the inode + * has the NOCOW bit set). + * + * However when we need to fallback to COW mode (because for example the + * block group for the corresponding extent was turned to RO mode by a + * scrub or relocation) we need to do the following: + * + * 1) We increment the bytes_may_use counter of the data space info. + * If COW succeeds, it allocates a new data extent and after doing + * that it decrements the space info's bytes_may_use counter and + * increments its bytes_reserved counter by the same amount (we do + * this at btrfs_add_reserved_bytes()). So we need to increment the + * bytes_may_use counter to compensate (when space is reserved at + * buffered write time, the bytes_may_use counter is incremented); + * + * 2) We clear the EXTENT_NORESERVE bit from the range. We do this so + * that if the COW path fails for any reason, it decrements (through + * extent_clear_unlock_delalloc()) the bytes_may_use counter of the + * data space info, which we incremented in the step above. + * + * If we need to fallback to cow and the inode corresponds to a free + * space cache inode or an inode of the data relocation tree, we must + * also increment bytes_may_use of the data space_info for the same + * reason. Space caches and relocated data extents always get a prealloc + * extent for them, however scrub or balance may have set the block + * group that contains that extent to RO mode and therefore force COW + * when starting writeback. + */ + btrfs_lock_extent(io_tree, start, end, &cached_state); + count = btrfs_count_range_bits(io_tree, &range_start, end, range_bytes, + EXTENT_NORESERVE, false, NULL); + if (count > 0 || is_space_ino || is_reloc_ino) { + u64 bytes = count; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct btrfs_space_info *sinfo = fs_info->data_sinfo; + + if (is_space_ino || is_reloc_ino) + bytes = range_bytes; + + spin_lock(&sinfo->lock); + btrfs_space_info_update_bytes_may_use(sinfo, bytes); + spin_unlock(&sinfo->lock); + + if (count > 0) + btrfs_clear_extent_bit(io_tree, start, end, EXTENT_NORESERVE, + &cached_state); + } + btrfs_unlock_extent(io_tree, start, end, &cached_state); + + /* + * Don't try to create inline extents, as a mix of inline extent that + * is written out and unlocked directly and a normal NOCOW extent + * doesn't work. + * + * And here we do not unlock the folio after a successful run. + * The folios will be unlocked after everything is finished, or by error handling. + * + * This is to ensure error handling won't need to clear dirty/ordered flags without + * a locked folio, which can race with writeback. + */ + ret = cow_file_range(inode, locked_folio, start, end, NULL, + COW_FILE_RANGE_KEEP_LOCKED); + ASSERT(ret != 1); + return ret; +} + +struct can_nocow_file_extent_args { + /* Input fields. */ + + /* Start file offset of the range we want to NOCOW. */ + u64 start; + /* End file offset (inclusive) of the range we want to NOCOW. */ + u64 end; + bool writeback_path; + /* + * Free the path passed to can_nocow_file_extent() once it's not needed + * anymore. + */ + bool free_path; + + /* + * Output fields. Only set when can_nocow_file_extent() returns 1. + * The expected file extent for the NOCOW write. + */ + struct btrfs_file_extent file_extent; +}; + +/* + * Check if we can NOCOW the file extent that the path points to. + * This function may return with the path released, so the caller should check + * if path->nodes[0] is NULL or not if it needs to use the path afterwards. + * + * Returns: < 0 on error + * 0 if we can not NOCOW + * 1 if we can NOCOW + */ +static int can_nocow_file_extent(struct btrfs_path *path, + struct btrfs_key *key, + struct btrfs_inode *inode, + struct can_nocow_file_extent_args *args) +{ + const bool is_freespace_inode = btrfs_is_free_space_inode(inode); + struct extent_buffer *leaf = path->nodes[0]; + struct btrfs_root *root = inode->root; + struct btrfs_file_extent_item *fi; + struct btrfs_root *csum_root; + u64 io_start; + u64 extent_end; + u8 extent_type; + int can_nocow = 0; + int ret = 0; + bool nowait = path->nowait; + + /* If there are pending snapshots for this root, we must do COW. */ + if (args->writeback_path && !is_freespace_inode && + atomic_read(&root->snapshot_force_cow)) + goto out; + + fi = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); + extent_type = btrfs_file_extent_type(leaf, fi); + + if (extent_type == BTRFS_FILE_EXTENT_INLINE) + goto out; + + if (!(inode->flags & BTRFS_INODE_NODATACOW) && + extent_type == BTRFS_FILE_EXTENT_REG) + goto out; + + /* + * If the extent was created before the generation where the last snapshot + * for its subvolume was created, then this implies the extent is shared, + * hence we must COW. + */ + if (btrfs_file_extent_generation(leaf, fi) <= + btrfs_root_last_snapshot(&root->root_item)) + goto out; + + /* An explicit hole, must COW. */ + if (btrfs_file_extent_disk_bytenr(leaf, fi) == 0) + goto out; + + /* Compressed/encrypted/encoded extents must be COWed. */ + if (btrfs_file_extent_compression(leaf, fi) || + btrfs_file_extent_encryption(leaf, fi) || + btrfs_file_extent_other_encoding(leaf, fi)) + goto out; + + extent_end = btrfs_file_extent_end(path); + + args->file_extent.disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi); + args->file_extent.disk_num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi); + args->file_extent.ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi); + args->file_extent.offset = btrfs_file_extent_offset(leaf, fi); + args->file_extent.compression = btrfs_file_extent_compression(leaf, fi); + + /* + * The following checks can be expensive, as they need to take other + * locks and do btree or rbtree searches, so release the path to avoid + * blocking other tasks for too long. + */ + btrfs_release_path(path); + + ret = btrfs_cross_ref_exist(inode, key->offset - args->file_extent.offset, + args->file_extent.disk_bytenr, path); + WARN_ON_ONCE(ret > 0 && is_freespace_inode); + if (ret != 0) + goto out; + + if (args->free_path) { + /* + * We don't need the path anymore, plus through the + * btrfs_lookup_csums_list() call below we will end up allocating + * another path. So free the path to avoid unnecessary extra + * memory usage. + */ + btrfs_free_path(path); + path = NULL; + } + + args->file_extent.num_bytes = min(args->end + 1, extent_end) - args->start; + args->file_extent.offset += args->start - key->offset; + io_start = args->file_extent.disk_bytenr + args->file_extent.offset; + + /* + * Force COW if csums exist in the range. This ensures that csums for a + * given extent are either valid or do not exist. + */ + + csum_root = btrfs_csum_root(root->fs_info, io_start); + if (unlikely(!csum_root)) { + btrfs_err(root->fs_info, + "missing csum root for extent at bytenr %llu", io_start); + ret = -EUCLEAN; + goto out; + } + + ret = btrfs_lookup_csums_list(csum_root, io_start, + io_start + args->file_extent.num_bytes - 1, + NULL, nowait); + WARN_ON_ONCE(ret > 0 && is_freespace_inode); + if (ret != 0) + goto out; + + can_nocow = 1; + out: + if (args->free_path && path) + btrfs_free_path(path); + + return ret < 0 ? ret : can_nocow; +} + +static int nocow_one_range(struct btrfs_inode *inode, struct folio *locked_folio, + struct extent_state **cached, + struct can_nocow_file_extent_args *nocow_args, + u64 file_pos, bool is_prealloc) +{ + struct btrfs_ordered_extent *ordered; + const u64 len = nocow_args->file_extent.num_bytes; + const u64 end = file_pos + len - 1; + int ret = 0; + + btrfs_lock_extent(&inode->io_tree, file_pos, end, cached); + + if (is_prealloc) { + struct extent_map *em; + + em = btrfs_create_io_em(inode, file_pos, &nocow_args->file_extent, + BTRFS_ORDERED_PREALLOC); + if (IS_ERR(em)) { + ret = PTR_ERR(em); + goto error; + } + btrfs_free_extent_map(em); + } + + ordered = btrfs_alloc_ordered_extent(inode, file_pos, &nocow_args->file_extent, + is_prealloc + ? (1U << BTRFS_ORDERED_PREALLOC) + : (1U << BTRFS_ORDERED_NOCOW)); + if (IS_ERR(ordered)) { + if (is_prealloc) + btrfs_drop_extent_map_range(inode, file_pos, end, false); + ret = PTR_ERR(ordered); + goto error; + } + + if (btrfs_is_data_reloc_root(inode->root)) + /* + * Errors are handled later, as we must prevent + * extent_clear_unlock_delalloc() in error handler from freeing + * metadata of the created ordered extent. + */ + ret = btrfs_reloc_clone_csums(ordered); + btrfs_put_ordered_extent(ordered); + + if (ret < 0) + goto error; + extent_clear_unlock_delalloc(inode, file_pos, end, locked_folio, cached, + EXTENT_LOCKED | EXTENT_DELALLOC | + EXTENT_CLEAR_DATA_RESV, 0); + return ret; + +error: + btrfs_cleanup_ordered_extents(inode, file_pos, len); + extent_clear_unlock_delalloc(inode, file_pos, end, locked_folio, cached, + EXTENT_LOCKED | EXTENT_DELALLOC | + EXTENT_CLEAR_DATA_RESV, + PAGE_UNLOCK | PAGE_START_WRITEBACK | + PAGE_END_WRITEBACK); + btrfs_err(inode->root->fs_info, + "%s failed, root=%lld inode=%llu start=%llu len=%llu: %pe", + __func__, btrfs_root_id(inode->root), btrfs_ino(inode), + file_pos, len, ERR_PTR(ret)); + return ret; +} + +/* + * When nocow writeback calls back. This checks for snapshots or COW copies + * of the extents that exist in the file, and COWs the file as required. + * + * If no cow copies or snapshots exist, we write directly to the existing + * blocks on disk + */ +static noinline int run_delalloc_nocow(struct btrfs_inode *inode, + struct folio *locked_folio, + const u64 start, const u64 end) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct btrfs_root *root = inode->root; + struct btrfs_path *path = NULL; + u64 cow_start = (u64)-1; + /* + * If not 0, represents the inclusive end of the last fallback_to_cow() + * range. Only for error handling. + * + * The same for nocow_end, it's to avoid double cleaning up the range + * already cleaned by nocow_one_range(). + */ + u64 cow_end = 0; + u64 nocow_end = 0; + u64 cur_offset = start; + int ret; + bool check_prev = true; + u64 ino = btrfs_ino(inode); + struct can_nocow_file_extent_args nocow_args = { 0 }; + /* The range that has ordered extent(s). */ + u64 oe_cleanup_start; + u64 oe_cleanup_len = 0; + /* The range that is untouched. */ + u64 untouched_start; + u64 untouched_len = 0; + + /* + * Normally on a zoned device we're only doing COW writes, but in case + * of relocation on a zoned filesystem serializes I/O so that we're only + * writing sequentially and can end up here as well. + */ + ASSERT(!btrfs_is_zoned(fs_info) || btrfs_is_data_reloc_root(root)); + + if (btrfs_is_shutdown(fs_info)) { + ret = -EIO; + goto error; + } + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto error; + } + + nocow_args.end = end; + nocow_args.writeback_path = true; + + while (cur_offset <= end) { + struct btrfs_block_group *nocow_bg = NULL; + struct btrfs_key found_key; + struct btrfs_file_extent_item *fi; + struct extent_buffer *leaf; + struct extent_state *cached_state = NULL; + u64 extent_end; + int extent_type; + + ret = btrfs_lookup_file_extent(NULL, root, path, ino, + cur_offset, 0); + if (ret < 0) + goto error; + + /* + * If there is no extent for our range when doing the initial + * search, then go back to the previous slot as it will be the + * one containing the search offset + */ + if (ret > 0 && path->slots[0] > 0 && check_prev) { + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &found_key, + path->slots[0] - 1); + if (found_key.objectid == ino && + found_key.type == BTRFS_EXTENT_DATA_KEY) + path->slots[0]--; + } + check_prev = false; +next_slot: + /* Go to next leaf if we have exhausted the current one */ + leaf = path->nodes[0]; + if (path->slots[0] >= btrfs_header_nritems(leaf)) { + ret = btrfs_next_leaf(root, path); + if (ret < 0) + goto error; + if (ret > 0) + break; + leaf = path->nodes[0]; + } + + btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); + + /* Didn't find anything for our INO */ + if (found_key.objectid > ino) + break; + /* + * Keep searching until we find an EXTENT_ITEM or there are no + * more extents for this inode + */ + if (WARN_ON_ONCE(found_key.objectid < ino) || + found_key.type < BTRFS_EXTENT_DATA_KEY) { + path->slots[0]++; + goto next_slot; + } + + /* Found key is not EXTENT_DATA_KEY or starts after req range */ + if (found_key.type > BTRFS_EXTENT_DATA_KEY || + found_key.offset > end) + break; + + /* + * If the found extent starts after requested offset, then + * adjust cur_offset to be right before this extent begins. + */ + if (found_key.offset > cur_offset) { + if (cow_start == (u64)-1) + cow_start = cur_offset; + cur_offset = found_key.offset; + goto next_slot; + } + + /* + * Found extent which begins before our range and potentially + * intersect it + */ + fi = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_file_extent_item); + extent_type = btrfs_file_extent_type(leaf, fi); + /* If this is triggered then we have a memory corruption. */ + ASSERT(extent_type < BTRFS_NR_FILE_EXTENT_TYPES); + if (WARN_ON(extent_type >= BTRFS_NR_FILE_EXTENT_TYPES)) { + ret = -EUCLEAN; + goto error; + } + extent_end = btrfs_file_extent_end(path); + + /* + * If the extent we got ends before our current offset, skip to + * the next extent. + */ + if (extent_end <= cur_offset) { + path->slots[0]++; + goto next_slot; + } + + nocow_args.start = cur_offset; + ret = can_nocow_file_extent(path, &found_key, inode, &nocow_args); + if (ret < 0) + goto error; + if (ret == 0) + goto must_cow; + + ret = 0; + nocow_bg = btrfs_inc_nocow_writers(fs_info, + nocow_args.file_extent.disk_bytenr + + nocow_args.file_extent.offset); + if (!nocow_bg) { +must_cow: + /* + * If we can't perform NOCOW writeback for the range, + * then record the beginning of the range that needs to + * be COWed. It will be written out before the next + * NOCOW range if we find one, or when exiting this + * loop. + */ + if (cow_start == (u64)-1) + cow_start = cur_offset; + cur_offset = extent_end; + if (cur_offset > end) + break; + if (!path->nodes[0]) + continue; + path->slots[0]++; + goto next_slot; + } + + /* + * COW range from cow_start to found_key.offset - 1. As the key + * will contain the beginning of the first extent that can be + * NOCOW, following one which needs to be COW'ed + */ + if (cow_start != (u64)-1) { + ret = fallback_to_cow(inode, locked_folio, cow_start, + found_key.offset - 1); + if (ret) { + cow_end = found_key.offset - 1; + btrfs_dec_nocow_writers(nocow_bg); + goto error; + } + cow_start = (u64)-1; + } + + ret = nocow_one_range(inode, locked_folio, &cached_state, + &nocow_args, cur_offset, + extent_type == BTRFS_FILE_EXTENT_PREALLOC); + btrfs_dec_nocow_writers(nocow_bg); + if (ret < 0) { + nocow_end = cur_offset + nocow_args.file_extent.num_bytes - 1; + goto error; + } + cur_offset = extent_end; + } + btrfs_release_path(path); + + if (cur_offset <= end && cow_start == (u64)-1) + cow_start = cur_offset; + + if (cow_start != (u64)-1) { + ret = fallback_to_cow(inode, locked_folio, cow_start, end); + if (ret) { + cow_end = end; + goto error; + } + cow_start = (u64)-1; + } + + /* + * Everything is finished without an error, can unlock the folios now. + * + * No need to touch the io tree range nor set folio ordered flag, as + * fallback_to_cow() and nocow_one_range() have already handled them. + */ + extent_clear_unlock_delalloc(inode, start, end, locked_folio, NULL, 0, PAGE_UNLOCK); + + btrfs_free_path(path); + return 0; + +error: + if (cow_start == (u64)-1) { + /* + * case a) + * start cur_offset end + * | OE cleanup | Untouched | + * + * We finished a fallback_to_cow() or nocow_one_range() call, + * but failed to check the next range. + * + * or + * start cur_offset nocow_end end + * | OE cleanup | Skip | Untouched | + * + * nocow_one_range() failed, the range [cur_offset, nocow_end] is + * already cleaned up. + */ + oe_cleanup_start = start; + oe_cleanup_len = cur_offset - start; + if (nocow_end) + untouched_start = nocow_end + 1; + else + untouched_start = cur_offset; + untouched_len = end + 1 - untouched_start; + } else if (cow_start != (u64)-1 && cow_end == 0) { + /* + * case b) + * start cow_start cur_offset end + * | OE cleanup | Untouched | + * + * We got a range that needs COW, but before we hit the next NOCOW range, + * thus [cow_start, cur_offset) doesn't yet have any OE. + */ + oe_cleanup_start = start; + oe_cleanup_len = cow_start - start; + untouched_start = cow_start; + untouched_len = end + 1 - untouched_start; + } else { + /* + * case c) + * start cow_start cow_end end + * | OE cleanup | Skip | Untouched | + * + * fallback_to_cow() failed, and fallback_to_cow() will do the + * cleanup for its range, we shouldn't touch the range + * [cow_start, cow_end]. + */ + ASSERT(cow_start != (u64)-1 && cow_end != 0); + oe_cleanup_start = start; + oe_cleanup_len = cow_start - start; + untouched_start = cow_end + 1; + untouched_len = end + 1 - untouched_start; + } + + if (oe_cleanup_len) { + const u64 oe_cleanup_end = oe_cleanup_start + oe_cleanup_len - 1; + btrfs_cleanup_ordered_extents(inode, oe_cleanup_start, oe_cleanup_len); + extent_clear_unlock_delalloc(inode, oe_cleanup_start, oe_cleanup_end, + locked_folio, NULL, + EXTENT_LOCKED | EXTENT_DELALLOC, + PAGE_UNLOCK | PAGE_START_WRITEBACK | + PAGE_END_WRITEBACK); + } + + if (untouched_len) { + struct extent_state *cached = NULL; + const u64 untouched_end = untouched_start + untouched_len - 1; + + /* + * We need to lock the extent here because we're clearing DELALLOC and + * we're not locked at this point. + */ + btrfs_lock_extent(&inode->io_tree, untouched_start, untouched_end, &cached); + extent_clear_unlock_delalloc(inode, untouched_start, untouched_end, + locked_folio, &cached, + EXTENT_LOCKED | EXTENT_DELALLOC | + EXTENT_DEFRAG | + EXTENT_DO_ACCOUNTING, PAGE_UNLOCK | + PAGE_START_WRITEBACK | + PAGE_END_WRITEBACK); + btrfs_qgroup_free_data(inode, NULL, untouched_start, untouched_len, NULL); + } + btrfs_free_path(path); + btrfs_err(fs_info, +"%s failed, root=%llu inode=%llu start=%llu len=%llu cur_offset=%llu oe_cleanup=%llu oe_cleanup_len=%llu untouched_start=%llu untouched_len=%llu: %pe", + __func__, btrfs_root_id(inode->root), btrfs_ino(inode), + start, end + 1 - start, cur_offset, oe_cleanup_start, oe_cleanup_len, + untouched_start, untouched_len, ERR_PTR(ret)); + return ret; +} + +static bool should_nocow(struct btrfs_inode *inode, u64 start, u64 end) +{ + if (inode->flags & (BTRFS_INODE_NODATACOW | BTRFS_INODE_PREALLOC)) { + if (data_race(inode->defrag_bytes) && + btrfs_test_range_bit_exists(&inode->io_tree, start, end, EXTENT_DEFRAG)) + return false; + return true; + } + return false; +} + +/* + * Return 0 if an inlined extent is created successfully. + * Return <0 if critical error happened. + * Return >0 if an inline extent can not be created. + */ +static int run_delalloc_inline(struct btrfs_inode *inode, struct folio *locked_folio) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct compressed_bio *cb = NULL; + struct extent_state *cached = NULL; + const u64 i_size = i_size_read(&inode->vfs_inode); + const u32 blocksize = fs_info->sectorsize; + int compress_type = fs_info->compress_type; + int compress_level = fs_info->compress_level; + u32 compressed_size = 0; + int ret; + + ASSERT(folio_pos(locked_folio) == 0); + /* + * If an mmap writer could modify the folio while we copy it into an + * inline extent we might see only part of their modification then + * wrongly mark it clean again after copying, losing that write. So the + * folio must be write protected here. + */ + btrfs_check_folio_write_protected(locked_folio); + + if (btrfs_inode_can_compress(inode) && + inode_need_compress(inode, 0, blocksize, true)) { + if (inode->defrag_compress > 0 && + inode->defrag_compress < BTRFS_NR_COMPRESS_TYPES) { + compress_type = inode->defrag_compress; + compress_level = inode->defrag_compress_level; + } else if (inode->prop_compress) { + compress_type = inode->prop_compress; + } + cb = btrfs_compress_bio(inode, 0, blocksize, compress_type, compress_level, 0); + if (IS_ERR(cb)) { + cb = NULL; + /* Just fall back to non-compressed case. */ + } else { + compressed_size = cb->bbio.bio.bi_iter.bi_size; + } + } + if (!can_cow_file_range_inline(inode, 0, i_size, compressed_size)) { + if (cb) + cleanup_compressed_bio(cb); + return 1; + } + + btrfs_lock_extent(&inode->io_tree, 0, blocksize - 1, &cached); + if (cb) { + ret = __cow_file_range_inline(inode, i_size, compressed_size, compress_type, + bio_first_folio_all(&cb->bbio.bio), false); + cleanup_compressed_bio(cb); + cb = NULL; + } else { + ret = __cow_file_range_inline(inode, i_size, 0, BTRFS_COMPRESS_NONE, + NULL, false); + } + /* + * We failed to insert inline extent due to lack of meta space. + * Just unlock the extent io range and fallback to regular COW/NOCOW path. + */ + if (ret > 0) { + btrfs_unlock_extent(&inode->io_tree, 0, blocksize - 1, &cached); + return ret; + } + + /* + * In the successful case (ret == 0 here), btrfs_run_delalloc_range() + * will return 1. + * + * Quite a bit further up the callstack in extent_writepage(), ret == 1 + * is treated as a short circuited success and does not unlock the folio, + * so we must do it here. + * + * For failure case, the @locked_folio does get unlocked by + * btrfs_folio_end_lock_bitmap(), so we must *not* unlock it here. + * + * So if ret == 0, we let extent_clear_unlock_delalloc() to unlock the + * folio by passing NULL as @locked_folio. + * Otherwise pass @locked_folio as usual. + */ + if (ret == 0) + locked_folio = NULL; + extent_clear_unlock_delalloc(inode, 0, blocksize - 1, locked_folio, &cached, + EXTENT_DELALLOC | EXTENT_DELALLOC_NEW | EXTENT_DEFRAG | + EXTENT_DO_ACCOUNTING | EXTENT_LOCKED, + PAGE_UNLOCK | PAGE_START_WRITEBACK | PAGE_END_WRITEBACK); + return ret; +} + +/* + * Function to process delayed allocation (create CoW) for ranges which are + * being touched for the first time. + */ +int btrfs_run_delalloc_range(struct btrfs_inode *inode, struct folio *locked_folio, + u64 start, u64 end, struct writeback_control *wbc) +{ + const bool zoned = btrfs_is_zoned(inode->root->fs_info); + + /* + * The range must cover part of the @locked_folio, or a return of 1 + * can confuse the caller. + */ + ASSERT(!(end <= folio_pos(locked_folio) || + start >= folio_next_pos(locked_folio))); + + if (start == 0 && end + 1 <= inode->root->fs_info->sectorsize && + end + 1 >= inode->disk_i_size) { + int ret; + + ret = run_delalloc_inline(inode, locked_folio); + if (ret < 0) + return ret; + if (ret == 0) + return 1; + /* + * Continue regular handling if we can not create an + * inlined extent. + */ + } + + if (should_nocow(inode, start, end)) + return run_delalloc_nocow(inode, locked_folio, start, end); + + if (btrfs_inode_can_compress(inode) && + inode_need_compress(inode, start, end, false) && + run_delalloc_compressed(inode, locked_folio, start, end, wbc)) + return 1; + + if (zoned) + return run_delalloc_cow(inode, locked_folio, start, end, wbc, true); + else + return cow_file_range(inode, locked_folio, start, end, NULL, 0); +} + +void btrfs_split_delalloc_extent(struct btrfs_inode *inode, + struct extent_state *orig, u64 split) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + u64 size; + + lockdep_assert_held(&inode->io_tree.lock); + + /* not delalloc, ignore it */ + if (!(orig->state & EXTENT_DELALLOC)) + return; + + size = orig->end - orig->start + 1; + if (size > fs_info->max_extent_size) { + u32 num_extents; + u64 new_size; + + /* + * See the explanation in btrfs_merge_delalloc_extent, the same + * applies here, just in reverse. + */ + new_size = orig->end - split + 1; + num_extents = count_max_extents(fs_info, new_size); + new_size = split - orig->start; + num_extents += count_max_extents(fs_info, new_size); + if (count_max_extents(fs_info, size) >= num_extents) + return; + } + + spin_lock(&inode->lock); + btrfs_mod_outstanding_extents(inode, 1); + spin_unlock(&inode->lock); +} + +/* + * Handle merged delayed allocation extents so we can keep track of new extents + * that are just merged onto old extents, such as when we are doing sequential + * writes, so we can properly account for the metadata space we'll need. + */ +void btrfs_merge_delalloc_extent(struct btrfs_inode *inode, struct extent_state *new, + struct extent_state *other) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + u64 new_size, old_size; + u32 num_extents; + + lockdep_assert_held(&inode->io_tree.lock); + + /* not delalloc, ignore it */ + if (!(other->state & EXTENT_DELALLOC)) + return; + + if (new->start > other->start) + new_size = new->end - other->start + 1; + else + new_size = other->end - new->start + 1; + + /* we're not bigger than the max, unreserve the space and go */ + if (new_size <= fs_info->max_extent_size) { + spin_lock(&inode->lock); + btrfs_mod_outstanding_extents(inode, -1); + spin_unlock(&inode->lock); + return; + } + + /* + * We have to add up either side to figure out how many extents were + * accounted for before we merged into one big extent. If the number of + * extents we accounted for is <= the amount we need for the new range + * then we can return, otherwise drop. Think of it like this + * + * [ 4k][MAX_SIZE] + * + * So we've grown the extent by a MAX_SIZE extent, this would mean we + * need 2 outstanding extents, on one side we have 1 and the other side + * we have 1 so they are == and we can return. But in this case + * + * [MAX_SIZE+4k][MAX_SIZE+4k] + * + * Each range on their own accounts for 2 extents, but merged together + * they are only 3 extents worth of accounting, so we need to drop in + * this case. + */ + old_size = other->end - other->start + 1; + num_extents = count_max_extents(fs_info, old_size); + old_size = new->end - new->start + 1; + num_extents += count_max_extents(fs_info, old_size); + if (count_max_extents(fs_info, new_size) >= num_extents) + return; + + spin_lock(&inode->lock); + btrfs_mod_outstanding_extents(inode, -1); + spin_unlock(&inode->lock); +} + +static void btrfs_add_delalloc_inode(struct btrfs_inode *inode) +{ + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + + spin_lock(&root->delalloc_lock); + ASSERT(list_empty(&inode->delalloc_inodes)); + list_add_tail(&inode->delalloc_inodes, &root->delalloc_inodes); + root->nr_delalloc_inodes++; + if (root->nr_delalloc_inodes == 1) { + spin_lock(&fs_info->delalloc_root_lock); + ASSERT(list_empty(&root->delalloc_root)); + list_add_tail(&root->delalloc_root, &fs_info->delalloc_roots); + spin_unlock(&fs_info->delalloc_root_lock); + } + spin_unlock(&root->delalloc_lock); +} + +void btrfs_del_delalloc_inode(struct btrfs_inode *inode) +{ + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + + lockdep_assert_held(&root->delalloc_lock); + + /* + * We may be called after the inode was already deleted from the list, + * namely in the transaction abort path btrfs_destroy_delalloc_inodes(), + * and then later through btrfs_clear_delalloc_extent() while the inode + * still has ->delalloc_bytes > 0. + */ + if (!list_empty(&inode->delalloc_inodes)) { + list_del_init(&inode->delalloc_inodes); + root->nr_delalloc_inodes--; + if (!root->nr_delalloc_inodes) { + ASSERT(list_empty(&root->delalloc_inodes)); + spin_lock(&fs_info->delalloc_root_lock); + ASSERT(!list_empty(&root->delalloc_root)); + list_del_init(&root->delalloc_root); + spin_unlock(&fs_info->delalloc_root_lock); + } + } +} + +/* + * Properly track delayed allocation bytes in the inode and to maintain the + * list of inodes that have pending delalloc work to be done. + */ +void btrfs_set_delalloc_extent(struct btrfs_inode *inode, struct extent_state *state, + u32 bits) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + + lockdep_assert_held(&inode->io_tree.lock); + + WARN_ON((bits & EXTENT_DEFRAG) && !(bits & EXTENT_DELALLOC)); + /* + * set_bit and clear bit hooks normally require _irqsave/restore + * but in this case, we are only testing for the DELALLOC + * bit, which is only set or cleared with irqs on + */ + if (!(state->state & EXTENT_DELALLOC) && (bits & EXTENT_DELALLOC)) { + u64 len = state->end + 1 - state->start; + u64 prev_delalloc_bytes; + u32 num_extents = count_max_extents(fs_info, len); + + spin_lock(&inode->lock); + btrfs_mod_outstanding_extents(inode, num_extents); + spin_unlock(&inode->lock); + + /* For sanity tests */ + if (btrfs_is_testing(fs_info)) + return; + + percpu_counter_add_batch(&fs_info->delalloc_bytes, len, + fs_info->delalloc_batch); + spin_lock(&inode->lock); + prev_delalloc_bytes = inode->delalloc_bytes; + inode->delalloc_bytes += len; + if (bits & EXTENT_DEFRAG) + inode->defrag_bytes += len; + spin_unlock(&inode->lock); + + /* + * We don't need to be under the protection of the inode's lock, + * because we are called while holding the inode's io_tree lock + * and are therefore protected against concurrent calls of this + * function and btrfs_clear_delalloc_extent(). + */ + if (!btrfs_is_free_space_inode(inode) && prev_delalloc_bytes == 0) + btrfs_add_delalloc_inode(inode); + } + + if (!(state->state & EXTENT_DELALLOC_NEW) && + (bits & EXTENT_DELALLOC_NEW)) { + spin_lock(&inode->lock); + inode->new_delalloc_bytes += state->end + 1 - state->start; + spin_unlock(&inode->lock); + } +} + +/* + * Once a range is no longer delalloc this function ensures that proper + * accounting happens. + */ +void btrfs_clear_delalloc_extent(struct btrfs_inode *inode, + struct extent_state *state, u32 bits) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + u64 len = state->end + 1 - state->start; + u32 num_extents = count_max_extents(fs_info, len); + + lockdep_assert_held(&inode->io_tree.lock); + + if ((state->state & EXTENT_DEFRAG) && (bits & EXTENT_DEFRAG)) { + spin_lock(&inode->lock); + inode->defrag_bytes -= len; + spin_unlock(&inode->lock); + } + + /* + * set_bit and clear bit hooks normally require _irqsave/restore + * but in this case, we are only testing for the DELALLOC + * bit, which is only set or cleared with irqs on + */ + if ((state->state & EXTENT_DELALLOC) && (bits & EXTENT_DELALLOC)) { + struct btrfs_root *root = inode->root; + u64 new_delalloc_bytes; + + spin_lock(&inode->lock); + btrfs_mod_outstanding_extents(inode, -num_extents); + spin_unlock(&inode->lock); + + /* + * We don't reserve metadata space for space cache inodes so we + * don't need to call delalloc_release_metadata if there is an + * error. + */ + if (bits & EXTENT_CLEAR_META_RESV && + root != fs_info->tree_root) + btrfs_delalloc_release_metadata(inode, len, true); + + /* For sanity tests. */ + if (btrfs_is_testing(fs_info)) + return; + + if (!btrfs_is_data_reloc_root(root) && + !btrfs_is_free_space_inode(inode) && + !(state->state & EXTENT_NORESERVE) && + (bits & EXTENT_CLEAR_DATA_RESV)) + btrfs_free_reserved_data_space_noquota(inode, len); + + percpu_counter_add_batch(&fs_info->delalloc_bytes, -len, + fs_info->delalloc_batch); + spin_lock(&inode->lock); + inode->delalloc_bytes -= len; + new_delalloc_bytes = inode->delalloc_bytes; + spin_unlock(&inode->lock); + + /* + * We don't need to be under the protection of the inode's lock, + * because we are called while holding the inode's io_tree lock + * and are therefore protected against concurrent calls of this + * function and btrfs_set_delalloc_extent(). + */ + if (!btrfs_is_free_space_inode(inode) && new_delalloc_bytes == 0) { + spin_lock(&root->delalloc_lock); + btrfs_del_delalloc_inode(inode); + spin_unlock(&root->delalloc_lock); + } + } + + if ((state->state & EXTENT_DELALLOC_NEW) && + (bits & EXTENT_DELALLOC_NEW)) { + spin_lock(&inode->lock); + ASSERT(inode->new_delalloc_bytes >= len); + inode->new_delalloc_bytes -= len; + if (bits & EXTENT_ADD_INODE_BYTES) + inode_add_bytes(&inode->vfs_inode, len); + spin_unlock(&inode->lock); + } +} + +/* + * Given an ordered extent and insert all its checksums into the csum tree. + * + * This happens at IO completion time based on sums calculated at bio + * submission time. + */ +static int add_pending_csums(struct btrfs_trans_handle *trans, + struct btrfs_ordered_extent *oe) +{ + struct btrfs_ordered_sum *sum; + struct btrfs_root *csum_root = NULL; + int ret; + + list_for_each_entry(sum, &oe->csum_list, list) { + if (!csum_root) { + csum_root = btrfs_csum_root(trans->fs_info, + sum->logical); + if (unlikely(!csum_root)) { + btrfs_err(trans->fs_info, + "missing csum root for extent at bytenr %llu", + sum->logical); + return -EUCLEAN; + } + } + trans->adding_csums = true; + ret = btrfs_insert_data_csums(trans, csum_root, sum); + trans->adding_csums = false; + if (ret) + return ret; + } + return 0; +} + +static int btrfs_find_new_delalloc_bytes(struct btrfs_inode *inode, + const u64 start, + const u64 len, + struct extent_state **cached_state) +{ + u64 search_start = start; + const u64 end = start + len - 1; + + while (search_start < end) { + const u64 search_len = end - search_start + 1; + struct extent_map *em; + u64 em_len; + int ret = 0; + + em = btrfs_get_extent(inode, NULL, search_start, search_len); + if (IS_ERR(em)) + return PTR_ERR(em); + + if (em->disk_bytenr != EXTENT_MAP_HOLE) + goto next; + + em_len = em->len; + if (em->start < search_start) + em_len -= search_start - em->start; + if (em_len > search_len) + em_len = search_len; + + ret = btrfs_set_extent_bit(&inode->io_tree, search_start, + search_start + em_len - 1, + EXTENT_DELALLOC_NEW, cached_state); +next: + search_start = btrfs_extent_map_end(em); + btrfs_free_extent_map(em); + if (ret) + return ret; + } + return 0; +} + +int btrfs_set_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, + unsigned int extra_bits, + struct extent_state **cached_state) +{ + const u32 blocksize = inode->root->fs_info->sectorsize; + + /* Basic alignment check. */ + ASSERT(IS_ALIGNED(start, blocksize), "start=%llu blocksize=%u", + start, blocksize); + ASSERT(IS_ALIGNED(end + 1, blocksize), "inclusive end=%llu blocksize=%u", + end, blocksize); + + if (start >= i_size_read(&inode->vfs_inode) && + !(inode->flags & BTRFS_INODE_PREALLOC)) { + /* + * There can't be any extents following eof in this case so just + * set the delalloc new bit for the range directly. + */ + extra_bits |= EXTENT_DELALLOC_NEW; + } else { + int ret; + + ret = btrfs_find_new_delalloc_bytes(inode, start, + end + 1 - start, + cached_state); + if (ret) + return ret; + } + + return btrfs_set_extent_bit(&inode->io_tree, start, end, + EXTENT_DELALLOC | extra_bits, cached_state); +} + +struct btrfs_writepage_fixup { + struct folio *folio; + struct btrfs_inode *inode; + struct work_struct work; +}; + +/* + * Do the real fixup work of reserving space for the blocks a folio's fixup + * state records. Queued by writepage_fixup() when writeback found the bits set. + * + * Since the fixup can be cancelled by a task dirtying with a reservation, we must + * re-check the state of fixup under the folio lock. + */ +static void btrfs_writepage_fixup_worker(struct work_struct *work) +{ + struct btrfs_writepage_fixup *fixup = + container_of(work, struct btrfs_writepage_fixup, work); + struct extent_state *cached_state = NULL; + struct extent_changeset *data_reserved = NULL; + unsigned long delalloc_bitmap[BITS_TO_LONGS(BTRFS_MAX_BLOCKS_PER_FOLIO)] = { 0 }; + struct folio *folio = fixup->folio; + struct btrfs_inode *inode = fixup->inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio); + const u32 sectorsize = fs_info->sectorsize; + const u64 page_start = folio_pos(folio); + const u64 page_end = folio_next_pos(folio) - 1; + unsigned int start_bit; + unsigned int end_bit; + unsigned int bit; + bool reserved; + int ret; + + /* + * We would prefer to reserve under the folio lock when we know exactly + * which blocks need a reservation. Unfortunately, since the reservation + * can go into flushers which can go into writeback, which takes folio + * locks, that is not possible. Therefore, we have to reserve for the + * whole folio here, then release what we didn't end up needing once we + * figure it out. + * + * Also note the slightly strange error checking. If fixup is actually + * not set, we don't need to mark an error on the mapping. So hang on to + * ret until after we lock and find out if we actually care. + */ + ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start, + folio_size(folio)); + reserved = (ret == 0); +again: + folio_lock(folio); + + if (!folio->mapping || !folio_test_fixup_pending(folio)) { + ret = 0; + goto out; + } + if (ret) + goto out; + + btrfs_lock_extent(&inode->io_tree, page_start, page_end, &cached_state); + + for (bit = 0; bit < blocks_per_folio; bit++) { + struct btrfs_ordered_extent *ordered; + const u64 start = page_start + (bit << fs_info->sectorsize_bits); + + if (test_bit(bit, delalloc_bitmap)) + continue; + if (!btrfs_folio_test_fixup(fs_info, folio, start, sectorsize)) + continue; + /* + * Any task that sets EXTENT_DELALLOC clears the fixup bits + * under the folio lock, so it should be impossible to observe + * both under the lock. Setting delalloc twice would wrongly + * double account the space. + */ + if (IS_ENABLED(CONFIG_BTRFS_DEBUG) && + unlikely(btrfs_test_range_bit_exists(&inode->io_tree, start, + start + sectorsize - 1, + EXTENT_DELALLOC))) { + DEBUG_WARN("fixup worker: delalloc and fixup conflict. ino %llu start %llu", + btrfs_ino(inode), start); + btrfs_folio_clear_fixup(fs_info, folio, start, sectorsize); + continue; + } + ordered = btrfs_lookup_ordered_range(inode, start, sectorsize); + if (ordered) { + trace_btrfs_writepage_fixup_defer(inode, ordered); + btrfs_unlock_extent(&inode->io_tree, page_start, + page_end, &cached_state); + folio_unlock(folio); + btrfs_start_ordered_extent(ordered); + btrfs_put_ordered_extent(ordered); + goto again; + } + ret = btrfs_set_extent_delalloc(inode, start, + start + sectorsize - 1, 0, + &cached_state); + if (ret) + break; + trace_btrfs_writepage_fixup_reserve(inode, start, sectorsize); + btrfs_folio_clear_fixup(fs_info, folio, start, sectorsize); + set_bit(bit, delalloc_bitmap); + } + + btrfs_unlock_extent(&inode->io_tree, page_start, page_end, &cached_state); +out: + if (ret < 0) { + /* Failure here is analogous to failure in writeback. */ + mapping_set_error(folio->mapping, ret); + btrfs_folio_clear_fixup_dirty(fs_info, folio, page_start, + folio_size(folio)); + } + if (reserved) { + btrfs_delalloc_release_extents(inode, folio_size(folio)); + for_each_clear_bitrange(start_bit, end_bit, delalloc_bitmap, + blocks_per_folio) + btrfs_delalloc_release_space(inode, data_reserved, + page_start + (start_bit << fs_info->sectorsize_bits), + (end_bit - start_bit) << fs_info->sectorsize_bits, + true); + } + folio_unlock(folio); + folio_put(folio); + kfree(fixup); + extent_changeset_free(data_reserved); + btrfs_add_delayed_iput(inode); +} + +/* + * Queue space reservation fixup work for blocks dirtied without a space reservation. + * + * Should be used by writeback while holding the folio locked. + * + * If we fail to queue fixup, then the folio state is unchanged and a future + * writeback pass will still see it. + */ +void btrfs_queue_writepage_fixup(struct btrfs_inode *inode, struct folio *folio) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct btrfs_writepage_fixup *fixup; + + /* + * Disallow queueing more fixup during unmount to break the cycle + * of writeback queuing fixup queuing writeback etc. + * + * If it actually hit, then something which was fixup wasn't written + * which we should warn about. + */ + if (btrfs_fs_closing(fs_info)) { + btrfs_warn_rl(fs_info, + "dropping unqueued fixup blocks at unmount. root %lld ino %llu folio %llu", + btrfs_root_id(inode->root), btrfs_ino(inode), + folio_pos(folio)); + btrfs_folio_clear_fixup_dirty(fs_info, folio, + folio_pos(folio), folio_size(folio)); + return; + } + + fixup = kzalloc_obj(*fixup, GFP_NOFS); + if (!fixup) + return; + + /* + * This is called from within extent_write_cache_pages() which + * has successfully done an igrab(). But that will be released at the + * end of the writeback pass. We need to extend it for the worker as well. + */ + ihold(&inode->vfs_inode); + folio_get(folio); + INIT_WORK(&fixup->work, btrfs_writepage_fixup_worker); + fixup->folio = folio; + fixup->inode = inode; + queue_work(fs_info->fixup_workers, &fixup->work); +} + +/* + * Clear the old accounting flags and set EXTENT_DELALLOC for the range. + * + * Return <0 for error, in that case no range has EXTENT_DELALLOC bit cleared or set. + */ +int btrfs_reset_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end, + unsigned int extra_bits, struct extent_state **cached_state) +{ + const u32 blocksize = inode->root->fs_info->sectorsize; + + /* The @extra_bits can only be EXTENT_NORESERVE for now. */ + ASSERT(!(extra_bits & ~EXTENT_NORESERVE), "extra_bits=0x%x", extra_bits); + + /* Basic alignment check. */ + ASSERT(IS_ALIGNED(start, blocksize), "start=%llu blocksize=%u", + start, blocksize); + ASSERT(IS_ALIGNED(end + 1, blocksize), "inclusive end=%llu blocksize=%u", + end, blocksize); + + /* + * Check and set DELALLOC_NEW flag, this needs to search tree thus can + * fail early. Thus we want to do this before clearing EXTENT_DELALLOC. + */ + if (start >= i_size_read(&inode->vfs_inode) && + !(inode->flags & BTRFS_INODE_PREALLOC)) { + /* + * There can't be any extents following EOF in this case so just + * set the delalloc new bit for the range directly. + */ + extra_bits |= EXTENT_DELALLOC_NEW; + } else { + int ret; + + ret = btrfs_find_new_delalloc_bytes(inode, start, end + 1 - start, + NULL); + if (unlikely(ret)) + return ret; + } + /* Clear the old accounting as the range may already be dirty. */ + btrfs_clear_extent_bit(&inode->io_tree, start, end, + EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | + EXTENT_DEFRAG, cached_state); + return btrfs_set_extent_bit(&inode->io_tree, start, end, + EXTENT_DELALLOC | extra_bits, cached_state); +} + +static int insert_reserved_file_extent(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, u64 file_pos, + struct btrfs_file_extent_item *stack_fi, + const bool update_inode_bytes, + u64 qgroup_reserved) +{ + struct btrfs_root *root = inode->root; + const u32 sectorsize = root->fs_info->sectorsize; + BTRFS_PATH_AUTO_FREE(path); + struct extent_buffer *leaf; + struct btrfs_key ins; + u64 disk_num_bytes = btrfs_stack_file_extent_disk_num_bytes(stack_fi); + u64 disk_bytenr = btrfs_stack_file_extent_disk_bytenr(stack_fi); + u64 offset = btrfs_stack_file_extent_offset(stack_fi); + u64 num_bytes = btrfs_stack_file_extent_num_bytes(stack_fi); + u64 ram_bytes = btrfs_stack_file_extent_ram_bytes(stack_fi); + struct btrfs_drop_extents_args drop_args = { 0 }; + int ret; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + /* + * we may be replacing one extent in the tree with another. + * The new extent is pinned in the extent map, and we don't want + * to drop it from the cache until it is completely in the btree. + * + * So, tell btrfs_drop_extents to leave this extent in the cache. + * the caller is expected to unpin it and allow it to be merged + * with the others. + */ + drop_args.path = path; + drop_args.start = file_pos; + drop_args.end = file_pos + num_bytes; + drop_args.replace_extent = true; + drop_args.extent_item_size = sizeof(*stack_fi); + ret = btrfs_drop_extents(trans, root, inode, &drop_args); + if (ret) + return ret; + + if (!drop_args.extent_inserted) { + ins.objectid = btrfs_ino(inode); + ins.type = BTRFS_EXTENT_DATA_KEY; + ins.offset = file_pos; + + ret = btrfs_insert_empty_item(trans, root, path, &ins, + sizeof(*stack_fi)); + if (ret) + return ret; + } + leaf = path->nodes[0]; + btrfs_set_stack_file_extent_generation(stack_fi, trans->transid); + write_extent_buffer(leaf, stack_fi, + btrfs_item_ptr_offset(leaf, path->slots[0]), + sizeof(struct btrfs_file_extent_item)); + + btrfs_release_path(path); + + /* + * If we dropped an inline extent here, we know the range where it is + * was not marked with the EXTENT_DELALLOC_NEW bit, so we update the + * number of bytes only for that range containing the inline extent. + * The remaining of the range will be processed when clearing the + * EXTENT_DELALLOC_BIT bit through the ordered extent completion. + */ + if (file_pos == 0 && !IS_ALIGNED(drop_args.bytes_found, sectorsize)) { + u64 inline_size = round_down(drop_args.bytes_found, sectorsize); + + inline_size = drop_args.bytes_found - inline_size; + btrfs_update_inode_bytes(inode, sectorsize, inline_size); + drop_args.bytes_found -= inline_size; + num_bytes -= sectorsize; + } + + if (update_inode_bytes) + btrfs_update_inode_bytes(inode, num_bytes, drop_args.bytes_found); + + ins.objectid = disk_bytenr; + ins.type = BTRFS_EXTENT_ITEM_KEY; + ins.offset = disk_num_bytes; + + ret = btrfs_inode_set_file_extent_range(inode, file_pos, ram_bytes); + if (ret) + return ret; + + return btrfs_alloc_reserved_file_extent(trans, root, btrfs_ino(inode), + file_pos - offset, + qgroup_reserved, &ins); +} + +static void btrfs_release_delalloc_bytes(struct btrfs_fs_info *fs_info, + u64 start, u64 len) +{ + struct btrfs_block_group *cache; + + cache = btrfs_lookup_block_group(fs_info, start); + ASSERT(cache); + + spin_lock(&cache->lock); + cache->delalloc_bytes -= len; + spin_unlock(&cache->lock); + + btrfs_put_block_group(cache); +} + +static int insert_ordered_extent_file_extent(struct btrfs_trans_handle *trans, + struct btrfs_ordered_extent *oe) +{ + struct btrfs_file_extent_item stack_fi; + bool update_inode_bytes; + u64 num_bytes = oe->num_bytes; + u64 ram_bytes = oe->ram_bytes; + + memset(&stack_fi, 0, sizeof(stack_fi)); + btrfs_set_stack_file_extent_type(&stack_fi, BTRFS_FILE_EXTENT_REG); + btrfs_set_stack_file_extent_disk_bytenr(&stack_fi, oe->disk_bytenr); + btrfs_set_stack_file_extent_disk_num_bytes(&stack_fi, + oe->disk_num_bytes); + btrfs_set_stack_file_extent_offset(&stack_fi, oe->offset); + if (test_bit(BTRFS_ORDERED_TRUNCATED, &oe->flags)) + num_bytes = oe->truncated_len; + btrfs_set_stack_file_extent_num_bytes(&stack_fi, num_bytes); + btrfs_set_stack_file_extent_ram_bytes(&stack_fi, ram_bytes); + btrfs_set_stack_file_extent_compression(&stack_fi, oe->compress_type); + /* Encryption and other encoding is reserved and all 0 */ + + /* + * For delalloc, when completing an ordered extent we update the inode's + * bytes when clearing the range in the inode's io tree, so pass false + * as the argument 'update_inode_bytes' to insert_reserved_file_extent(), + * except if the ordered extent was truncated. + */ + update_inode_bytes = test_bit(BTRFS_ORDERED_DIRECT, &oe->flags) || + test_bit(BTRFS_ORDERED_ENCODED, &oe->flags) || + test_bit(BTRFS_ORDERED_TRUNCATED, &oe->flags); + + return insert_reserved_file_extent(trans, oe->inode, + oe->file_offset, &stack_fi, + update_inode_bytes, oe->qgroup_rsv); +} + +/* + * As ordered data IO finishes, this gets called so we can finish + * an ordered extent if the range of bytes in the file it covers are + * fully written. + */ +int btrfs_finish_one_ordered(struct btrfs_ordered_extent *ordered_extent) +{ + struct btrfs_inode *inode = ordered_extent->inode; + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_trans_handle *trans = NULL; + struct extent_io_tree *io_tree = &inode->io_tree; + struct extent_state *cached_state = NULL; + u64 start, end; + int compress_type = 0; + int ret = 0; + u64 logical_len = ordered_extent->num_bytes; + bool freespace_inode; + bool truncated = false; + bool clear_reserved_extent = true; + unsigned int clear_bits = 0; + + start = ordered_extent->file_offset; + end = start + ordered_extent->num_bytes - 1; + + if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) && + !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags) && + !test_bit(BTRFS_ORDERED_DIRECT, &ordered_extent->flags) && + !test_bit(BTRFS_ORDERED_ENCODED, &ordered_extent->flags)) + clear_bits |= EXTENT_DELALLOC_NEW; + + if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) + clear_bits |= EXTENT_DEFRAG; + + freespace_inode = btrfs_is_free_space_inode(inode); + if (!freespace_inode) + btrfs_lockdep_acquire(fs_info, btrfs_ordered_extent); + + if (unlikely(test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags))) { + ret = -EIO; + goto out; + } + + ret = btrfs_zone_finish_endio(fs_info, ordered_extent->disk_bytenr, + ordered_extent->disk_num_bytes); + if (ret) + goto out; + + if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) { + truncated = true; + logical_len = ordered_extent->truncated_len; + /* Truncated the entire extent, don't bother adding */ + if (!logical_len) + goto out; + } + + /* + * If it's a COW write we need to lock the extent range as we will be + * inserting/replacing file extent items and unpinning an extent map. + * This must be taken before joining a transaction, as it's a higher + * level lock (like the inode's VFS lock), otherwise we can run into an + * ABBA deadlock with other tasks (transactions work like a lock, + * depending on their current state). + */ + if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) { + clear_bits |= EXTENT_LOCKED | EXTENT_FINISHING_ORDERED; + btrfs_lock_extent_bits(io_tree, start, end, + EXTENT_LOCKED | EXTENT_FINISHING_ORDERED, + &cached_state); + } + + if (freespace_inode) + trans = btrfs_join_transaction_spacecache(root); + else + trans = btrfs_join_transaction(root); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + trans = NULL; + goto out; + } + + trans->block_rsv = &inode->block_rsv; + + ret = btrfs_insert_raid_extent(trans, ordered_extent); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + + if (test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) { + /* Logic error */ + ASSERT(list_empty(&ordered_extent->csum_list)); + if (unlikely(!list_empty(&ordered_extent->csum_list))) { + ret = -EINVAL; + btrfs_abort_transaction(trans, ret); + goto out; + } + + btrfs_inode_safe_disk_i_size_write(inode, 0); + ret = btrfs_update_inode_fallback(trans, inode); + if (unlikely(ret)) { + /* -ENOMEM or corruption */ + btrfs_abort_transaction(trans, ret); + } + goto out; + } + + if (test_bit(BTRFS_ORDERED_COMPRESSED, &ordered_extent->flags)) + compress_type = ordered_extent->compress_type; + if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) { + BUG_ON(compress_type); + ret = btrfs_mark_extent_written(trans, inode, + ordered_extent->file_offset, + ordered_extent->file_offset + + logical_len); + btrfs_zoned_release_data_reloc_bg(fs_info, ordered_extent->disk_bytenr, + ordered_extent->disk_num_bytes); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + } else { + BUG_ON(root == fs_info->tree_root); + ret = insert_ordered_extent_file_extent(trans, ordered_extent); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + clear_reserved_extent = false; + btrfs_release_delalloc_bytes(fs_info, + ordered_extent->disk_bytenr, + ordered_extent->disk_num_bytes); + } + + ret = btrfs_unpin_extent_cache(inode, ordered_extent->file_offset, + ordered_extent->num_bytes, trans->transid); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + + ret = add_pending_csums(trans, ordered_extent); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + + /* + * If this is a new delalloc range, clear its new delalloc flag to + * update the inode's number of bytes. This needs to be done first + * before updating the inode item. + */ + if ((clear_bits & EXTENT_DELALLOC_NEW) && + !test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) + btrfs_clear_extent_bit(&inode->io_tree, start, end, + EXTENT_DELALLOC_NEW | EXTENT_ADD_INODE_BYTES, + &cached_state); + + btrfs_inode_safe_disk_i_size_write(inode, 0); + ret = btrfs_update_inode_fallback(trans, inode); + if (unlikely(ret)) { /* -ENOMEM or corruption */ + btrfs_abort_transaction(trans, ret); + goto out; + } +out: + if (clear_bits) + btrfs_clear_extent_bit(&inode->io_tree, start, end, clear_bits, + &cached_state); + + if (trans) + btrfs_end_transaction(trans); + + if (ret || truncated) { + /* + * If we failed to finish this ordered extent for any reason we + * need to make sure BTRFS_ORDERED_IOERR is set on the ordered + * extent, and mark the inode with the error if it wasn't + * already set. Any error during writeback would have already + * set the mapping error, so we need to set it if we're the ones + * marking this ordered extent as failed. + */ + if (ret) + btrfs_mark_ordered_extent_error(ordered_extent); + + /* + * Drop extent maps for the part of the extent we didn't write. + * + * We have an exception here for the free_space_inode, this is + * because when we do btrfs_get_extent() on the free space inode + * we will search the commit root. If this is a new block group + * we won't find anything, and we will trip over the assert in + * writepage where we do ASSERT(em->block_start != + * EXTENT_MAP_HOLE). + * + * Theoretically we could also skip this for any NOCOW extent as + * we don't mess with the extent map tree in the NOCOW case, but + * for now simply skip this if we are the free space inode. + */ + if (!btrfs_is_free_space_inode(inode)) { + u64 unwritten_start = start; + + if (truncated) + unwritten_start += logical_len; + + btrfs_drop_extent_map_range(inode, unwritten_start, + end, false); + } + + /* + * If the ordered extent had an IOERR or something else went + * wrong we need to return the space for this ordered extent + * back to the allocator. We only free the extent in the + * truncated case if we didn't write out the extent at all. + * + * If we made it past insert_reserved_file_extent before we + * errored out then we don't need to do this as the accounting + * has already been done. + */ + if ((ret || !logical_len) && + clear_reserved_extent && + !test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) && + !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) { + /* + * Discard the range before returning it back to the + * free space pool + */ + if (ret && btrfs_test_opt(fs_info, DISCARD_SYNC)) + btrfs_discard_extent(fs_info, + ordered_extent->disk_bytenr, + ordered_extent->disk_num_bytes, + NULL, true); + btrfs_free_reserved_extent(fs_info, + ordered_extent->disk_bytenr, + ordered_extent->disk_num_bytes, true); + /* + * Actually free the qgroup rsv which was released when + * the ordered extent was created. + */ + btrfs_qgroup_free_refroot(fs_info, btrfs_root_id(inode->root), + ordered_extent->qgroup_rsv, + BTRFS_QGROUP_RSV_DATA); + } + } + + /* + * This needs to be done to make sure anybody waiting knows we are done + * updating everything for this ordered extent. + */ + btrfs_remove_ordered_extent(ordered_extent); + + /* once for us */ + btrfs_put_ordered_extent(ordered_extent); + /* once for the tree */ + btrfs_put_ordered_extent(ordered_extent); + + return ret; +} + +int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered) +{ + if (btrfs_is_zoned(ordered->inode->root->fs_info) && + !test_bit(BTRFS_ORDERED_IOERR, &ordered->flags) && + list_empty(&ordered->bioc_list)) + btrfs_finish_ordered_zoned(ordered); + return btrfs_finish_one_ordered(ordered); +} + +/* + * Calculate the checksum of an fs block at physical memory address @paddr, + * and save the result to @dest. + * + * The folio containing @paddr must be large enough to contain a full fs block. + */ +void btrfs_calculate_block_csum_folio(struct btrfs_fs_info *fs_info, + const phys_addr_t paddr, u8 *dest) +{ + struct folio *folio = page_folio(phys_to_page(paddr)); + const u32 blocksize = fs_info->sectorsize; + const u32 step = min(blocksize, PAGE_SIZE); + const u32 nr_steps = blocksize / step; + phys_addr_t paddrs[BTRFS_MAX_BLOCKSIZE / PAGE_SIZE]; + + /* The full block must be inside the folio. */ + ASSERT(offset_in_folio(folio, paddr) + blocksize <= folio_size(folio)); + + for (int i = 0; i < nr_steps; i++) { + u32 pindex = offset_in_folio(folio, paddr + i * step) >> PAGE_SHIFT; + + /* + * For bs <= ps cases, we will only run the loop once, so the offset + * inside the page will only added to paddrs[0]. + * + * For bs > ps cases, the block must be page aligned, thus offset + * inside the page will always be 0. + */ + paddrs[i] = page_to_phys(folio_page(folio, pindex)) + offset_in_page(paddr); + } + return btrfs_calculate_block_csum_pages(fs_info, paddrs, dest); +} + +/* + * Calculate the checksum of a fs block backed by multiple noncontiguous pages + * at @paddrs[] and save the result to @dest. + * + * The folio containing @paddr must be large enough to contain a full fs block. + */ +void btrfs_calculate_block_csum_pages(struct btrfs_fs_info *fs_info, + const phys_addr_t paddrs[], u8 *dest) +{ + const u32 blocksize = fs_info->sectorsize; + const u32 step = min(blocksize, PAGE_SIZE); + const u32 nr_steps = blocksize / step; + struct btrfs_csum_ctx csum; + + btrfs_csum_init(&csum, fs_info->csum_type); + for (int i = 0; i < nr_steps; i++) { + const phys_addr_t paddr = paddrs[i]; + void *kaddr; + + ASSERT(offset_in_page(paddr) + step <= PAGE_SIZE); + kaddr = kmap_local_page(phys_to_page(paddr)) + offset_in_page(paddr); + btrfs_csum_update(&csum, kaddr, step); + kunmap_local(kaddr); + } + btrfs_csum_final(&csum, dest); +} + +/* + * Verify the checksum for a single sector without any extra action that depend + * on the type of I/O. + * + * @kaddr must be a properly kmapped address. + */ +int btrfs_check_block_csum(struct btrfs_fs_info *fs_info, phys_addr_t paddr, u8 *csum, + const u8 * const csum_expected) +{ + btrfs_calculate_block_csum_folio(fs_info, paddr, csum); + if (unlikely(memcmp(csum, csum_expected, fs_info->csum_size) != 0)) + return -EIO; + return 0; +} + +/* + * Verify the checksum of a single data sector, which can be scattered at + * different noncontiguous pages. + * + * @bbio: btrfs_io_bio which contains the csum + * @dev: device the sector is on + * @bio_offset: offset to the beginning of the bio (in bytes) + * @paddrs: physical addresses which back the fs block + * + * Check if the checksum on a data block is valid. When a checksum mismatch is + * detected, report the error and fill the corrupted range with zero. + * + * Return %true if the sector is ok or had no checksum to start with, else %false. + */ +bool btrfs_data_csum_ok(struct btrfs_bio *bbio, struct btrfs_device *dev, + u32 bio_offset, const phys_addr_t paddrs[]) +{ + struct btrfs_inode *inode = bbio->inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + const u32 blocksize = fs_info->sectorsize; + const u32 step = min(blocksize, PAGE_SIZE); + const u32 nr_steps = blocksize / step; + u64 file_offset = bbio->file_offset + bio_offset; + u64 end = file_offset + blocksize - 1; + u8 *csum_expected; + u8 csum[BTRFS_CSUM_SIZE]; + + if (!bbio->csum) + return true; + + if (btrfs_is_data_reloc_root(inode->root) && + btrfs_test_range_bit(&inode->io_tree, file_offset, end, EXTENT_NODATASUM, + NULL)) { + /* Skip the range without csum for data reloc inode */ + btrfs_clear_extent_bit(&inode->io_tree, file_offset, end, + EXTENT_NODATASUM, NULL); + return true; + } + + csum_expected = bbio->csum + (bio_offset >> fs_info->sectorsize_bits) * + fs_info->csum_size; + btrfs_calculate_block_csum_pages(fs_info, paddrs, csum); + if (unlikely(memcmp(csum, csum_expected, fs_info->csum_size) != 0)) + goto zeroit; + return true; + +zeroit: + btrfs_print_data_csum_error(inode, file_offset, csum, csum_expected, + bbio->mirror_num); + if (dev) + btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS); + for (int i = 0; i < nr_steps; i++) + memzero_page(phys_to_page(paddrs[i]), offset_in_page(paddrs[i]), step); + return false; +} + +/* + * Perform a delayed iput on @inode. + * + * @inode: The inode we want to perform iput on + * + * This function uses the generic vfs_inode::i_count to track whether we should + * just decrement it (in case it's > 1) or if this is the last iput then link + * the inode to the delayed iput machinery. Delayed iputs are processed at + * transaction commit time/superblock commit/cleaner kthread. + */ +void btrfs_add_delayed_iput(struct btrfs_inode *inode) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + unsigned long flags; + + if (atomic_add_unless(&inode->vfs_inode.i_count, -1, 1)) + return; + + WARN_ON_ONCE(test_bit(BTRFS_FS_STATE_NO_DELAYED_IPUT, &fs_info->fs_state)); + atomic_inc(&fs_info->nr_delayed_iputs); + /* + * Need to be irq safe here because we can be called from either an irq + * context (see bio.c and btrfs_put_ordered_extent()) or a non-irq + * context. + */ + spin_lock_irqsave(&fs_info->delayed_iput_lock, flags); + ASSERT(list_empty(&inode->delayed_iput)); + list_add_tail(&inode->delayed_iput, &fs_info->delayed_iputs); + spin_unlock_irqrestore(&fs_info->delayed_iput_lock, flags); + if (!test_bit(BTRFS_FS_CLEANER_RUNNING, &fs_info->flags)) + wake_up_process(fs_info->cleaner_kthread); +} + +static void run_delayed_iput_locked(struct btrfs_fs_info *fs_info, + struct btrfs_inode *inode) +{ + list_del_init(&inode->delayed_iput); + spin_unlock_irq(&fs_info->delayed_iput_lock); + iput(&inode->vfs_inode); + if (atomic_dec_and_test(&fs_info->nr_delayed_iputs)) + wake_up(&fs_info->delayed_iputs_wait); + spin_lock_irq(&fs_info->delayed_iput_lock); +} + +static void btrfs_run_delayed_iput(struct btrfs_fs_info *fs_info, + struct btrfs_inode *inode) +{ + if (!list_empty(&inode->delayed_iput)) { + spin_lock_irq(&fs_info->delayed_iput_lock); + if (!list_empty(&inode->delayed_iput)) + run_delayed_iput_locked(fs_info, inode); + spin_unlock_irq(&fs_info->delayed_iput_lock); + } +} + +void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info) +{ + /* + * btrfs_put_ordered_extent() can run in irq context (see bio.c), which + * calls btrfs_add_delayed_iput() and that needs to lock + * fs_info->delayed_iput_lock. So we need to disable irqs here to + * prevent a deadlock. + */ + spin_lock_irq(&fs_info->delayed_iput_lock); + while (!list_empty(&fs_info->delayed_iputs)) { + struct btrfs_inode *inode; + + inode = list_first_entry(&fs_info->delayed_iputs, + struct btrfs_inode, delayed_iput); + run_delayed_iput_locked(fs_info, inode); + if (need_resched()) { + spin_unlock_irq(&fs_info->delayed_iput_lock); + cond_resched(); + spin_lock_irq(&fs_info->delayed_iput_lock); + } + } + spin_unlock_irq(&fs_info->delayed_iput_lock); +} + +/* + * Wait for flushing all delayed iputs + * + * @fs_info: the filesystem + * + * This will wait on any delayed iputs that are currently running with KILLABLE + * set. Once they are all done running we will return, unless we are killed in + * which case we return EINTR. This helps in user operations like fallocate etc + * that might get blocked on the iputs. + * + * Return EINTR if we were killed, 0 if nothing's pending + */ +int btrfs_wait_on_delayed_iputs(struct btrfs_fs_info *fs_info) +{ + int ret = wait_event_killable(fs_info->delayed_iputs_wait, + atomic_read(&fs_info->nr_delayed_iputs) == 0); + if (ret) + return -EINTR; + return 0; +} + +/* + * This creates an orphan entry for the given inode in case something goes wrong + * in the middle of an unlink. + */ +int btrfs_orphan_add(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode) +{ + int ret; + + ret = btrfs_insert_orphan_item(trans, inode->root, btrfs_ino(inode)); + if (unlikely(ret && ret != -EEXIST)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + + return 0; +} + +/* + * We have done the delete so we can go ahead and remove the orphan item for + * this particular inode. + */ +static int btrfs_orphan_del(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode) +{ + return btrfs_del_orphan_item(trans, inode->root, btrfs_ino(inode)); +} + +/* + * this cleans up any orphans that may be left on the list from the last use + * of this root. + */ +int btrfs_orphan_cleanup(struct btrfs_root *root) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + BTRFS_PATH_AUTO_FREE(path); + struct extent_buffer *leaf; + struct btrfs_key key, found_key; + struct btrfs_trans_handle *trans; + u64 last_objectid = 0; + int ret = 0, nr_unlink = 0; + + if (test_and_set_bit(BTRFS_ROOT_ORPHAN_CLEANUP, &root->state)) + return 0; + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto out; + } + path->reada = READA_BACK; + + key.objectid = BTRFS_ORPHAN_OBJECTID; + key.type = BTRFS_ORPHAN_ITEM_KEY; + key.offset = (u64)-1; + + while (1) { + struct btrfs_inode *inode; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + goto out; + + /* + * if ret == 0 means we found what we were searching for, which + * is weird, but possible, so only screw with path if we didn't + * find the key and see if we have stuff that matches + */ + if (ret > 0) { + ret = 0; + if (path->slots[0] == 0) + break; + path->slots[0]--; + } + + /* pull out the item */ + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); + + /* make sure the item matches what we want */ + if (found_key.objectid != BTRFS_ORPHAN_OBJECTID) + break; + if (found_key.type != BTRFS_ORPHAN_ITEM_KEY) + break; + + /* release the path since we're done with it */ + btrfs_release_path(path); + + /* + * this is where we are basically btrfs_lookup, without the + * crossing root thing. we store the inode number in the + * offset of the orphan item. + */ + + if (found_key.offset == last_objectid) { + /* + * We found the same inode as before. This means we were + * not able to remove its items via eviction triggered + * by an iput(). A transaction abort may have happened, + * due to -ENOSPC for example, so try to grab the error + * that lead to a transaction abort, if any. + */ + btrfs_err(fs_info, + "Error removing orphan entry, stopping orphan cleanup"); + ret = BTRFS_FS_ERROR(fs_info) ?: -EINVAL; + goto out; + } + + last_objectid = found_key.offset; + + found_key.objectid = found_key.offset; + found_key.type = BTRFS_INODE_ITEM_KEY; + found_key.offset = 0; + inode = btrfs_iget(last_objectid, root); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + inode = NULL; + if (ret != -ENOENT) + goto out; + } + + if (!inode && root == fs_info->tree_root) { + struct btrfs_root *dead_root; + bool is_dead_root = false; + + /* + * This is an orphan in the tree root. Currently these + * could come from 2 sources: + * a) a root (snapshot/subvolume) deletion in progress + * b) a free space cache inode + * We need to distinguish those two, as the orphan item + * for a root must not get deleted before the deletion + * of the snapshot/subvolume's tree completes. + * + * btrfs_find_orphan_roots() ran before us, which has + * found all deleted roots and loaded them into + * fs_info->fs_roots_radix. So here we can find if an + * orphan item corresponds to a deleted root by looking + * up the root from that radix tree. + */ + + spin_lock(&fs_info->fs_roots_radix_lock); + dead_root = radix_tree_lookup(&fs_info->fs_roots_radix, + (unsigned long)found_key.objectid); + if (dead_root && btrfs_root_refs(&dead_root->root_item) == 0) + is_dead_root = true; + spin_unlock(&fs_info->fs_roots_radix_lock); + + if (is_dead_root) { + /* prevent this orphan from being found again */ + key.offset = found_key.objectid - 1; + continue; + } + + } + + /* + * If we have an inode with links, there are a couple of + * possibilities: + * + * 1. We were halfway through creating fsverity metadata for the + * file. In that case, the orphan item represents incomplete + * fsverity metadata which must be cleaned up with + * btrfs_drop_verity_items and deleting the orphan item. + + * 2. Old kernels (before v3.12) used to create an + * orphan item for truncate indicating that there were possibly + * extent items past i_size that needed to be deleted. In v3.12, + * truncate was changed to update i_size in sync with the extent + * items, but the (useless) orphan item was still created. Since + * v4.18, we don't create the orphan item for truncate at all. + * + * So, this item could mean that we need to do a truncate, but + * only if this filesystem was last used on a pre-v3.12 kernel + * and was not cleanly unmounted. The odds of that are quite + * slim, and it's a pain to do the truncate now, so just delete + * the orphan item. + * + * It's also possible that this orphan item was supposed to be + * deleted but wasn't. The inode number may have been reused, + * but either way, we can delete the orphan item. + */ + if (!inode || inode->vfs_inode.i_nlink) { + if (inode) { + ret = btrfs_drop_verity_items(inode); + iput(&inode->vfs_inode); + inode = NULL; + if (ret) + goto out; + } + trans = btrfs_start_transaction(root, 1); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto out; + } + btrfs_debug(fs_info, "auto deleting %Lu", + found_key.objectid); + ret = btrfs_del_orphan_item(trans, root, + found_key.objectid); + btrfs_end_transaction(trans); + if (ret) + goto out; + continue; + } + + nr_unlink++; + + /* this will do delete_inode and everything for us */ + iput(&inode->vfs_inode); + } + /* release the path since we're done with it */ + btrfs_release_path(path); + + if (test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state)) { + trans = btrfs_join_transaction(root); + if (!IS_ERR(trans)) + btrfs_end_transaction(trans); + } + + if (nr_unlink) + btrfs_debug(fs_info, "unlinked %d orphans", nr_unlink); + +out: + if (ret) + btrfs_err(fs_info, "could not do orphan cleanup %pe", ERR_PTR(ret)); + return ret; +} + +/* + * Look ahead in the leaf for xattrs. If we don't find any then we know there + * can't be any ACLs. + * + * @leaf: the eb leaf where to search + * @slot: the slot the inode is in + * @objectid: the objectid of the inode + * + * Return true if there is xattr/ACL, false otherwise. + */ +static noinline bool acls_after_inode_item(struct extent_buffer *leaf, + int slot, u64 objectid, + int *first_xattr_slot) +{ + u32 nritems = btrfs_header_nritems(leaf); + struct btrfs_key found_key; + static u64 xattr_access = 0; + static u64 xattr_default = 0; + int scanned = 0; + + if (!xattr_access) { + xattr_access = btrfs_name_hash(XATTR_NAME_POSIX_ACL_ACCESS, + strlen(XATTR_NAME_POSIX_ACL_ACCESS)); + xattr_default = btrfs_name_hash(XATTR_NAME_POSIX_ACL_DEFAULT, + strlen(XATTR_NAME_POSIX_ACL_DEFAULT)); + } + + slot++; + *first_xattr_slot = -1; + while (slot < nritems) { + btrfs_item_key_to_cpu(leaf, &found_key, slot); + + /* We found a different objectid, there must be no ACLs. */ + if (found_key.objectid != objectid) + return false; + + /* We found an xattr, assume we've got an ACL. */ + if (found_key.type == BTRFS_XATTR_ITEM_KEY) { + if (*first_xattr_slot == -1) + *first_xattr_slot = slot; + if (found_key.offset == xattr_access || + found_key.offset == xattr_default) + return true; + } + + /* + * We found a key greater than an xattr key, there can't be any + * ACLs later on. + */ + if (found_key.type > BTRFS_XATTR_ITEM_KEY) + return false; + + slot++; + scanned++; + + /* + * The item order goes like: + * - inode + * - inode backrefs + * - xattrs + * - extents, + * + * so if there are lots of hard links to an inode there can be + * a lot of backrefs. Don't waste time searching too hard, + * this is just an optimization. + */ + if (scanned >= 8) + break; + } + /* + * We hit the end of the leaf before we found an xattr or something + * larger than an xattr. We have to assume the inode has ACLs. + */ + if (*first_xattr_slot == -1) + *first_xattr_slot = slot; + return true; +} + +static int btrfs_init_file_extent_tree(struct btrfs_inode *inode) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + + if (WARN_ON_ONCE(inode->file_extent_tree)) + return 0; + if (btrfs_fs_incompat(fs_info, NO_HOLES)) + return 0; + if (!S_ISREG(inode->vfs_inode.i_mode)) + return 0; + if (btrfs_is_free_space_inode(inode)) + return 0; + + inode->file_extent_tree = kmalloc_obj(struct extent_io_tree); + if (!inode->file_extent_tree) + return -ENOMEM; + + btrfs_extent_io_tree_init(fs_info, inode->file_extent_tree, + IO_TREE_INODE_FILE_EXTENT); + /* Lockdep class is set only for the file extent tree. */ + lockdep_set_class(&inode->file_extent_tree->lock, &file_extent_tree_class); + + return 0; +} + +static int btrfs_add_inode_to_root(struct btrfs_inode *inode, bool prealloc) +{ + struct btrfs_root *root = inode->root; + struct btrfs_inode *existing; + const u64 ino = btrfs_ino(inode); + int ret; + + if (inode_unhashed(&inode->vfs_inode)) + return 0; + + if (prealloc) { + ret = xa_reserve(&root->inodes, ino, GFP_NOFS); + if (ret) + return ret; + } + + existing = xa_store(&root->inodes, ino, inode, GFP_ATOMIC); + + if (xa_is_err(existing)) { + ret = xa_err(existing); + ASSERT(ret != -EINVAL); + ASSERT(ret != -ENOMEM); + return ret; + } else if (existing) { + WARN_ON(!(inode_state_read_once(&existing->vfs_inode) & (I_WILL_FREE | I_FREEING))); + } + + return 0; +} + +/* + * Read a locked inode from the btree into the in-memory inode and add it to + * its root list/tree. + * + * On failure clean up the inode. + */ +static int btrfs_read_locked_inode(struct btrfs_inode *inode, struct btrfs_path *path) +{ + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct extent_buffer *leaf; + struct btrfs_inode_item *inode_item; + struct inode *vfs_inode = &inode->vfs_inode; + struct btrfs_key location; + unsigned long ptr; + int maybe_acls; + u32 rdev; + int ret; + bool filled = false; + int first_xattr_slot; + + ret = btrfs_fill_inode(inode, &rdev); + if (!ret) + filled = true; + + ASSERT(path); + + btrfs_get_inode_key(inode, &location); + + ret = btrfs_lookup_inode(NULL, root, path, &location, 0); + if (ret) { + /* + * ret > 0 can come from btrfs_search_slot called by + * btrfs_lookup_inode(), this means the inode was not found. + */ + if (ret > 0) + ret = -ENOENT; + goto out; + } + + leaf = path->nodes[0]; + + if (filled) + goto cache_index; + + inode_item = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_inode_item); + vfs_inode->i_mode = btrfs_inode_mode(leaf, inode_item); + set_nlink(vfs_inode, btrfs_inode_nlink(leaf, inode_item)); + i_uid_write(vfs_inode, btrfs_inode_uid(leaf, inode_item)); + i_gid_write(vfs_inode, btrfs_inode_gid(leaf, inode_item)); + btrfs_i_size_write(inode, btrfs_inode_size(leaf, inode_item)); + + inode_set_atime(vfs_inode, btrfs_timespec_sec(leaf, &inode_item->atime), + btrfs_timespec_nsec(leaf, &inode_item->atime)); + + inode_set_mtime(vfs_inode, btrfs_timespec_sec(leaf, &inode_item->mtime), + btrfs_timespec_nsec(leaf, &inode_item->mtime)); + + inode_set_ctime(vfs_inode, btrfs_timespec_sec(leaf, &inode_item->ctime), + btrfs_timespec_nsec(leaf, &inode_item->ctime)); + + inode->i_otime_sec = btrfs_timespec_sec(leaf, &inode_item->otime); + inode->i_otime_nsec = btrfs_timespec_nsec(leaf, &inode_item->otime); + + inode_set_bytes(vfs_inode, btrfs_inode_nbytes(leaf, inode_item)); + inode->generation = btrfs_inode_generation(leaf, inode_item); + inode->last_trans = btrfs_inode_transid(leaf, inode_item); + + inode_set_iversion_queried(vfs_inode, btrfs_inode_sequence(leaf, inode_item)); + vfs_inode->i_generation = inode->generation; + vfs_inode->i_rdev = 0; + rdev = btrfs_inode_rdev(leaf, inode_item); + + if (S_ISDIR(vfs_inode->i_mode)) + inode->index_cnt = (u64)-1; + + btrfs_inode_split_flags(btrfs_inode_flags(leaf, inode_item), + &inode->flags, &inode->ro_flags); + +cache_index: + btrfs_update_inode_mapping_flags(inode); + btrfs_set_inode_mapping_order(inode); + + /* + * If we were modified in the current generation and evicted from memory + * and then re-read we need to do a full sync since we don't have any + * idea about which extents were modified before we were evicted from + * cache. + * + * This is required for both inode re-read from disk and delayed inode + * in the delayed_nodes xarray. + */ + if (inode->last_trans == btrfs_get_fs_generation(fs_info)) + set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags); + + /* + * We don't persist the id of the transaction where an unlink operation + * against the inode was last made. So here we assume the inode might + * have been evicted, and therefore the exact value of last_unlink_trans + * lost, and set it to last_trans to avoid metadata inconsistencies + * between the inode and its parent if the inode is fsync'ed and the log + * replayed. For example, in the scenario: + * + * touch mydir/foo + * ln mydir/foo mydir/bar + * sync + * unlink mydir/bar + * echo 2 > /proc/sys/vm/drop_caches # evicts inode + * xfs_io -c fsync mydir/foo + * + * mount fs, triggers fsync log replay + * + * We must make sure that when we fsync our inode foo we also log its + * parent inode, otherwise after log replay the parent still has the + * dentry with the "bar" name but our inode foo has a link count of 1 + * and doesn't have an inode ref with the name "bar" anymore. + * + * Setting last_unlink_trans to last_trans is a pessimistic approach, + * but it guarantees correctness at the expense of occasional full + * transaction commits on fsync if our inode is a directory, or if our + * inode is not a directory, logging its parent unnecessarily. + */ + inode->last_unlink_trans = inode->last_trans; + + /* + * Same logic as for last_unlink_trans. We don't persist the generation + * of the last transaction where this inode was used for a reflink + * operation, so after eviction and reloading the inode we must be + * pessimistic and assume the last transaction that modified the inode. + */ + inode->last_reflink_trans = inode->last_trans; + + path->slots[0]++; + if (vfs_inode->i_nlink != 1 || + path->slots[0] >= btrfs_header_nritems(leaf)) + goto cache_acl; + + btrfs_item_key_to_cpu(leaf, &location, path->slots[0]); + if (location.objectid != btrfs_ino(inode)) + goto cache_acl; + + ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); + if (location.type == BTRFS_INODE_REF_KEY) { + struct btrfs_inode_ref *ref; + + ref = (struct btrfs_inode_ref *)ptr; + inode->dir_index = btrfs_inode_ref_index(leaf, ref); + } else if (location.type == BTRFS_INODE_EXTREF_KEY) { + struct btrfs_inode_extref *extref; + + extref = (struct btrfs_inode_extref *)ptr; + inode->dir_index = btrfs_inode_extref_index(leaf, extref); + } +cache_acl: + /* + * try to precache a NULL acl entry for files that don't have + * any xattrs or acls + */ + maybe_acls = acls_after_inode_item(leaf, path->slots[0], + btrfs_ino(inode), &first_xattr_slot); + if (first_xattr_slot != -1) { + path->slots[0] = first_xattr_slot; + ret = btrfs_load_inode_props(inode, path); + if (ret) + btrfs_err(fs_info, + "error loading props for ino %llu (root %llu): %pe", + btrfs_ino(inode), btrfs_root_id(root), ERR_PTR(ret)); + } + + /* + * We don't need the path anymore, so release it to avoid holding a read + * lock on a leaf while calling btrfs_init_file_extent_tree(), which can + * allocate memory that triggers reclaim (GFP_KERNEL) and cause a locking + * dependency. + */ + btrfs_release_path(path); + + ret = btrfs_init_file_extent_tree(inode); + if (ret) + goto out; + btrfs_inode_set_file_extent_range(inode, 0, + round_up(i_size_read(vfs_inode), fs_info->sectorsize)); + + if (!maybe_acls) + cache_no_acl(vfs_inode); + + switch (vfs_inode->i_mode & S_IFMT) { + case S_IFREG: + vfs_inode->i_mapping->a_ops = &btrfs_aops; + vfs_inode->i_fop = &btrfs_file_operations; + vfs_inode->i_op = &btrfs_file_inode_operations; + break; + case S_IFDIR: + vfs_inode->i_fop = &btrfs_dir_file_operations; + vfs_inode->i_op = &btrfs_dir_inode_operations; + break; + case S_IFLNK: + vfs_inode->i_op = &btrfs_symlink_inode_operations; + inode_nohighmem(vfs_inode); + vfs_inode->i_mapping->a_ops = &btrfs_aops; + break; + default: + vfs_inode->i_op = &btrfs_special_inode_operations; + init_special_inode(vfs_inode, vfs_inode->i_mode, rdev); + break; + } + + btrfs_sync_inode_flags_to_i_flags(inode); + + ret = btrfs_add_inode_to_root(inode, true); + if (ret) + goto out; + + return 0; +out: + /* + * We may have a read locked leaf and iget_failed() triggers inode + * eviction which needs to release the delayed inode and that needs + * to lock the delayed inode's mutex. This can cause a ABBA deadlock + * with a task running delayed items, as that require first locking + * the delayed inode's mutex and then modifying its subvolume btree. + * So release the path before iget_failed(). + */ + btrfs_release_path(path); + iget_failed(vfs_inode); + return ret; +} + +/* + * given a leaf and an inode, copy the inode fields into the leaf + */ +static void fill_inode_item(struct btrfs_trans_handle *trans, + struct extent_buffer *leaf, + struct btrfs_inode_item *item, + struct inode *inode) +{ + u64 flags; + + btrfs_set_inode_uid(leaf, item, i_uid_read(inode)); + btrfs_set_inode_gid(leaf, item, i_gid_read(inode)); + btrfs_set_inode_size(leaf, item, BTRFS_I(inode)->disk_i_size); + btrfs_set_inode_mode(leaf, item, inode->i_mode); + btrfs_set_inode_nlink(leaf, item, inode->i_nlink); + + btrfs_set_timespec_sec(leaf, &item->atime, inode_get_atime_sec(inode)); + btrfs_set_timespec_nsec(leaf, &item->atime, inode_get_atime_nsec(inode)); + + btrfs_set_timespec_sec(leaf, &item->mtime, inode_get_mtime_sec(inode)); + btrfs_set_timespec_nsec(leaf, &item->mtime, inode_get_mtime_nsec(inode)); + + btrfs_set_timespec_sec(leaf, &item->ctime, inode_get_ctime_sec(inode)); + btrfs_set_timespec_nsec(leaf, &item->ctime, inode_get_ctime_nsec(inode)); + + btrfs_set_timespec_sec(leaf, &item->otime, BTRFS_I(inode)->i_otime_sec); + btrfs_set_timespec_nsec(leaf, &item->otime, BTRFS_I(inode)->i_otime_nsec); + + btrfs_set_inode_nbytes(leaf, item, inode_get_bytes(inode)); + btrfs_set_inode_generation(leaf, item, BTRFS_I(inode)->generation); + btrfs_set_inode_sequence(leaf, item, inode_peek_iversion(inode)); + btrfs_set_inode_transid(leaf, item, trans->transid); + btrfs_set_inode_rdev(leaf, item, inode->i_rdev); + flags = btrfs_inode_combine_flags(BTRFS_I(inode)->flags, + BTRFS_I(inode)->ro_flags); + btrfs_set_inode_flags(leaf, item, flags); + btrfs_set_inode_block_group(leaf, item, 0); +} + +/* + * copy everything in the in-memory inode into the btree. + */ +static noinline int btrfs_update_inode_item(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode) +{ + struct btrfs_inode_item *inode_item; + BTRFS_PATH_AUTO_FREE(path); + struct extent_buffer *leaf; + struct btrfs_key key; + int ret; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + btrfs_get_inode_key(inode, &key); + ret = btrfs_lookup_inode(trans, inode->root, path, &key, 1); + if (ret) { + if (ret > 0) + ret = -ENOENT; + return ret; + } + + leaf = path->nodes[0]; + inode_item = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_inode_item); + + fill_inode_item(trans, leaf, inode_item, &inode->vfs_inode); + btrfs_set_inode_last_trans(trans, inode); + return 0; +} + +/* + * copy everything in the in-memory inode into the btree. + */ +int btrfs_update_inode(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode) +{ + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + int ret; + + /* + * If the inode is a free space inode, we can deadlock during commit + * if we put it into the delayed code. + * + * The data relocation inode should also be directly updated + * without delay + */ + if (!btrfs_is_free_space_inode(inode) + && !btrfs_is_data_reloc_root(root) + && !test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) { + btrfs_update_root_times(trans, root); + + ret = btrfs_delayed_update_inode(trans, inode); + if (!ret) + btrfs_set_inode_last_trans(trans, inode); + return ret; + } + + return btrfs_update_inode_item(trans, inode); +} + +int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode) +{ + int ret; + + ret = btrfs_update_inode(trans, inode); + if (ret == -ENOSPC) + return btrfs_update_inode_item(trans, inode); + return ret; +} + +static void update_time_after_link_or_unlink(struct btrfs_inode *dir) +{ + struct timespec64 now; + + /* + * If we are replaying a log tree, we do not want to update the mtime + * and ctime of the parent directory with the current time, since the + * log replay procedure is responsible for setting them to their correct + * values (the ones it had when the fsync was done). + */ + if (test_bit(BTRFS_FS_LOG_RECOVERING, &dir->root->fs_info->flags)) + return; + + now = inode_set_ctime_current(&dir->vfs_inode); + inode_set_mtime_to_ts(&dir->vfs_inode, now); +} + +/* + * unlink helper that gets used here in inode.c and in the tree logging + * recovery code. It remove a link in a directory with a given name, and + * also drops the back refs in the inode to the directory + */ +static int __btrfs_unlink_inode(struct btrfs_trans_handle *trans, + struct btrfs_inode *dir, + struct btrfs_inode *inode, + const struct fscrypt_str *name, + struct btrfs_rename_ctx *rename_ctx) +{ + struct btrfs_root *root = dir->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_path *path; + int ret = 0; + struct btrfs_dir_item *di; + u64 index; + u64 ino = btrfs_ino(inode); + u64 dir_ino = btrfs_ino(dir); + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + di = btrfs_lookup_dir_item(trans, root, path, dir_ino, name, -1); + if (IS_ERR_OR_NULL(di)) { + btrfs_free_path(path); + return di ? PTR_ERR(di) : -ENOENT; + } + ret = btrfs_delete_one_dir_name(trans, root, path, di); + /* + * Down the call chains below we'll also need to allocate a path, so no + * need to hold on to this one for longer than necessary. + */ + btrfs_free_path(path); + if (ret) + return ret; + + /* + * If we don't have dir index, we have to get it by looking up + * the inode ref, since we get the inode ref, remove it directly, + * it is unnecessary to do delayed deletion. + * + * But if we have dir index, needn't search inode ref to get it. + * Since the inode ref is close to the inode item, it is better + * that we delay to delete it, and just do this deletion when + * we update the inode item. + */ + if (inode->dir_index) { + ret = btrfs_delayed_delete_inode_ref(inode); + if (!ret) { + index = inode->dir_index; + goto skip_backref; + } + } + + ret = btrfs_del_inode_ref(trans, root, name, ino, dir_ino, &index); + if (unlikely(ret)) { + btrfs_crit(fs_info, + "failed to delete reference to %.*s, root %llu inode %llu parent %llu", + name->len, name->name, btrfs_root_id(root), ino, dir_ino); + btrfs_abort_transaction(trans, ret); + return ret; + } +skip_backref: + if (rename_ctx) + rename_ctx->index = index; + + ret = btrfs_delete_delayed_dir_index(trans, dir, index); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + + /* + * If we are in a rename context, we don't need to update anything in the + * log. That will be done later during the rename by btrfs_log_new_name(). + * Besides that, doing it here would only cause extra unnecessary btree + * operations on the log tree, increasing latency for applications. + */ + if (!rename_ctx) { + btrfs_del_inode_ref_in_log(trans, name, inode, dir); + btrfs_del_dir_entries_in_log(trans, name, dir, index); + } + + /* + * If we have a pending delayed iput we could end up with the final iput + * being run in btrfs-cleaner context. If we have enough of these built + * up we can end up burning a lot of time in btrfs-cleaner without any + * way to throttle the unlinks. Since we're currently holding a ref on + * the inode we can run the delayed iput here without any issues as the + * final iput won't be done until after we drop the ref we're currently + * holding. + */ + btrfs_run_delayed_iput(fs_info, inode); + + btrfs_i_size_write(dir, dir->vfs_inode.i_size - name->len * 2); + inode_inc_iversion(&inode->vfs_inode); + inode_set_ctime_current(&inode->vfs_inode); + inode_inc_iversion(&dir->vfs_inode); + update_time_after_link_or_unlink(dir); + + return btrfs_update_inode(trans, dir); +} + +int btrfs_unlink_inode(struct btrfs_trans_handle *trans, + struct btrfs_inode *dir, struct btrfs_inode *inode, + const struct fscrypt_str *name) +{ + int ret; + + ret = __btrfs_unlink_inode(trans, dir, inode, name, NULL); + if (!ret) { + drop_nlink(&inode->vfs_inode); + ret = btrfs_update_inode(trans, inode); + } + return ret; +} + +/* + * helper to start transaction for unlink and rmdir. + * + * unlink and rmdir are special in btrfs, they do not always free space, so + * if we cannot make our reservations the normal way try and see if there is + * plenty of slack room in the global reserve to migrate, otherwise we cannot + * allow the unlink to occur. + */ +static struct btrfs_trans_handle *__unlink_start_trans(struct btrfs_inode *dir) +{ + struct btrfs_root *root = dir->root; + + return btrfs_start_transaction_fallback_global_rsv(root, + BTRFS_UNLINK_METADATA_UNITS); +} + +static int btrfs_unlink(struct inode *dir, struct dentry *dentry) +{ + struct btrfs_trans_handle *trans; + struct inode *inode = d_inode(dentry); + int ret; + struct fscrypt_name fname; + + ret = fscrypt_setup_filename(dir, &dentry->d_name, 1, &fname); + if (ret) + return ret; + + /* This needs to handle no-key deletions later on */ + + trans = __unlink_start_trans(BTRFS_I(dir)); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto fscrypt_free; + } + + btrfs_record_unlink_dir(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)), + false); + + ret = btrfs_unlink_inode(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)), + &fname.disk_name); + if (ret) + goto end_trans; + + if (inode->i_nlink == 0) { + ret = btrfs_orphan_add(trans, BTRFS_I(inode)); + if (ret) + goto end_trans; + } + +end_trans: + btrfs_end_transaction(trans); + btrfs_btree_balance_dirty(BTRFS_I(dir)->root->fs_info); +fscrypt_free: + fscrypt_free_filename(&fname); + return ret; +} + +static int btrfs_unlink_subvol(struct btrfs_trans_handle *trans, + struct btrfs_inode *dir, struct dentry *dentry) +{ + struct btrfs_root *root = dir->root; + struct btrfs_inode *inode = BTRFS_I(d_inode(dentry)); + BTRFS_PATH_AUTO_FREE(path); + struct extent_buffer *leaf; + struct btrfs_dir_item *di; + struct btrfs_key key; + u64 index; + int ret; + u64 objectid; + u64 dir_ino = btrfs_ino(dir); + struct fscrypt_name fname; + + ret = fscrypt_setup_filename(&dir->vfs_inode, &dentry->d_name, 1, &fname); + if (ret) + return ret; + + /* This needs to handle no-key deletions later on */ + + if (btrfs_ino(inode) == BTRFS_FIRST_FREE_OBJECTID) { + objectid = btrfs_root_id(inode->root); + } else if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) { + objectid = inode->ref_root_id; + } else { + WARN_ON(1); + fscrypt_free_filename(&fname); + return -EINVAL; + } + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto out; + } + + di = btrfs_lookup_dir_item(trans, root, path, dir_ino, + &fname.disk_name, -1); + if (IS_ERR_OR_NULL(di)) { + ret = di ? PTR_ERR(di) : -ENOENT; + goto out; + } + + leaf = path->nodes[0]; + btrfs_dir_item_key_to_cpu(leaf, di, &key); + WARN_ON(key.type != BTRFS_ROOT_ITEM_KEY || key.objectid != objectid); + ret = btrfs_delete_one_dir_name(trans, root, path, di); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + btrfs_release_path(path); + + /* + * This is a placeholder inode for a subvolume we didn't have a + * reference to at the time of the snapshot creation. In the meantime + * we could have renamed the real subvol link into our snapshot, so + * depending on btrfs_del_root_ref to return -ENOENT here is incorrect. + * Instead simply lookup the dir_index_item for this entry so we can + * remove it. Otherwise we know we have a ref to the root and we can + * call btrfs_del_root_ref, and it _shouldn't_ fail. + */ + if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) { + di = btrfs_search_dir_index_item(root, path, dir_ino, &fname.disk_name); + if (IS_ERR(di)) { + ret = PTR_ERR(di); + btrfs_abort_transaction(trans, ret); + goto out; + } + + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + index = key.offset; + btrfs_release_path(path); + } else { + ret = btrfs_del_root_ref(trans, objectid, + btrfs_root_id(root), dir_ino, + &index, &fname.disk_name); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + } + + ret = btrfs_delete_delayed_dir_index(trans, dir, index); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + + btrfs_i_size_write(dir, dir->vfs_inode.i_size - fname.disk_name.len * 2); + inode_inc_iversion(&dir->vfs_inode); + inode_set_mtime_to_ts(&dir->vfs_inode, inode_set_ctime_current(&dir->vfs_inode)); + ret = btrfs_update_inode_fallback(trans, dir); + if (ret) + btrfs_abort_transaction(trans, ret); +out: + fscrypt_free_filename(&fname); + return ret; +} + +/* + * Helper to check if the subvolume references other subvolumes or if it's + * default. + */ +static noinline int may_destroy_subvol(struct btrfs_root *root) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_dir_item *di; + struct btrfs_key key; + struct fscrypt_str name = FSTR_INIT("default", 7); + u64 dir_id; + int ret; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + /* Make sure this root isn't set as the default subvol */ + dir_id = btrfs_super_root_dir(fs_info->super_copy); + di = btrfs_lookup_dir_item(NULL, fs_info->tree_root, path, + dir_id, &name, 0); + if (!IS_ERR_OR_NULL(di)) { + btrfs_dir_item_key_to_cpu(path->nodes[0], di, &key); + if (key.objectid == btrfs_root_id(root)) { + ret = -EPERM; + btrfs_err(fs_info, + "deleting default subvolume %llu is not allowed", + key.objectid); + return ret; + } + btrfs_release_path(path); + } + + key.objectid = btrfs_root_id(root); + key.type = BTRFS_ROOT_REF_KEY; + key.offset = (u64)-1; + + ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0); + if (ret < 0) + return ret; + if (unlikely(ret == 0)) { + /* + * Key with offset -1 found, there would have to exist a root + * with such id, but this is out of valid range. + */ + return -EUCLEAN; + } + + ret = 0; + if (path->slots[0] > 0) { + path->slots[0]--; + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + if (key.objectid == btrfs_root_id(root) && key.type == BTRFS_ROOT_REF_KEY) + ret = -ENOTEMPTY; + } + + return ret; +} + +/* Delete all dentries for inodes belonging to the root */ +static void btrfs_prune_dentries(struct btrfs_root *root) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_inode *inode; + u64 min_ino = 0; + + if (!BTRFS_FS_ERROR(fs_info)) + WARN_ON(btrfs_root_refs(&root->root_item) != 0); + + inode = btrfs_find_first_inode(root, min_ino); + while (inode) { + if (icount_read_once(&inode->vfs_inode) > 1) + d_prune_aliases(&inode->vfs_inode); + + min_ino = btrfs_ino(inode) + 1; + /* + * btrfs_drop_inode() will have it removed from the inode + * cache when its usage count hits zero. + */ + iput(&inode->vfs_inode); + cond_resched(); + inode = btrfs_find_first_inode(root, min_ino); + } +} + +int btrfs_delete_subvolume(struct btrfs_inode *dir, struct dentry *dentry) +{ + struct btrfs_root *root = dir->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct inode *inode = d_inode(dentry); + struct btrfs_root *dest = BTRFS_I(inode)->root; + struct btrfs_trans_handle *trans; + struct btrfs_block_rsv block_rsv; + u64 root_flags; + u64 qgroup_reserved = 0; + int ret; + + down_write(&fs_info->subvol_sem); + + /* + * Don't allow to delete a subvolume with send in progress. This is + * inside the inode lock so the error handling that has to drop the bit + * again is not run concurrently. + */ + spin_lock(&dest->root_item_lock); + if (dest->send_in_progress) { + spin_unlock(&dest->root_item_lock); + btrfs_warn(fs_info, + "attempt to delete subvolume %llu during send", + btrfs_root_id(dest)); + ret = -EPERM; + goto out_up_write; + } + if (atomic_read(&dest->nr_swapfiles)) { + spin_unlock(&dest->root_item_lock); + btrfs_warn(fs_info, + "attempt to delete subvolume %llu with active swapfile", + btrfs_root_id(dest)); + ret = -EPERM; + goto out_up_write; + } + root_flags = btrfs_root_flags(&dest->root_item); + btrfs_set_root_flags(&dest->root_item, + root_flags | BTRFS_ROOT_SUBVOL_DEAD); + spin_unlock(&dest->root_item_lock); + + ret = may_destroy_subvol(dest); + if (ret) + goto out_undead; + + btrfs_init_block_rsv(&block_rsv, BTRFS_BLOCK_RSV_TEMP); + /* + * One for dir inode, + * two for dir entries, + * two for root ref/backref. + */ + ret = btrfs_subvolume_reserve_metadata(root, &block_rsv, 5, true); + if (ret) + goto out_undead; + qgroup_reserved = block_rsv.qgroup_rsv_reserved; + + trans = btrfs_start_transaction(root, 0); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto out_release; + } + btrfs_qgroup_convert_reserved_meta(root, qgroup_reserved); + qgroup_reserved = 0; + trans->block_rsv = &block_rsv; + trans->bytes_reserved = block_rsv.size; + + btrfs_record_snapshot_destroy(trans, dir); + + ret = btrfs_unlink_subvol(trans, dir, dentry); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out_end_trans; + } + + ret = btrfs_record_root_in_trans(trans, dest); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out_end_trans; + } + + memset(&dest->root_item.drop_progress, 0, + sizeof(dest->root_item.drop_progress)); + btrfs_set_root_drop_level(&dest->root_item, 0); + btrfs_set_root_refs(&dest->root_item, 0); + + if (!test_and_set_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &dest->state)) { + ret = btrfs_insert_orphan_item(trans, + fs_info->tree_root, + btrfs_root_id(dest)); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out_end_trans; + } + } + + ret = btrfs_uuid_tree_remove(trans, dest->root_item.uuid, + BTRFS_UUID_KEY_SUBVOL, btrfs_root_id(dest)); + if (unlikely(ret && ret != -ENOENT)) { + btrfs_abort_transaction(trans, ret); + goto out_end_trans; + } + if (!btrfs_is_empty_uuid(dest->root_item.received_uuid)) { + ret = btrfs_uuid_tree_remove(trans, + dest->root_item.received_uuid, + BTRFS_UUID_KEY_RECEIVED_SUBVOL, + btrfs_root_id(dest)); + if (unlikely(ret && ret != -ENOENT)) { + btrfs_abort_transaction(trans, ret); + goto out_end_trans; + } + } + + free_anon_bdev(dest->anon_dev); + dest->anon_dev = 0; +out_end_trans: + trans->block_rsv = NULL; + trans->bytes_reserved = 0; + ret = btrfs_end_transaction(trans); + inode->i_flags |= S_DEAD; +out_release: + btrfs_block_rsv_release(fs_info, &block_rsv, (u64)-1, NULL); + if (qgroup_reserved) + btrfs_qgroup_free_meta_prealloc(root, qgroup_reserved); +out_undead: + if (ret) { + spin_lock(&dest->root_item_lock); + root_flags = btrfs_root_flags(&dest->root_item); + btrfs_set_root_flags(&dest->root_item, + root_flags & ~BTRFS_ROOT_SUBVOL_DEAD); + spin_unlock(&dest->root_item_lock); + } +out_up_write: + up_write(&fs_info->subvol_sem); + if (!ret) { + d_invalidate(dentry); + btrfs_prune_dentries(dest); + ASSERT(dest->send_in_progress == 0); + } + + return ret; +} + +static int btrfs_rmdir(struct inode *vfs_dir, struct dentry *dentry) +{ + struct btrfs_inode *dir = BTRFS_I(vfs_dir); + struct btrfs_inode *inode = BTRFS_I(d_inode(dentry)); + struct btrfs_fs_info *fs_info = inode->root->fs_info; + int ret = 0; + struct btrfs_trans_handle *trans; + struct fscrypt_name fname; + + if (inode->vfs_inode.i_size > BTRFS_EMPTY_DIR_SIZE) + return -ENOTEMPTY; + if (btrfs_ino(inode) == BTRFS_FIRST_FREE_OBJECTID) { + if (unlikely(btrfs_fs_incompat(fs_info, EXTENT_TREE_V2))) { + btrfs_err(fs_info, + "extent tree v2 doesn't support snapshot deletion yet"); + return -EOPNOTSUPP; + } + return btrfs_delete_subvolume(dir, dentry); + } + + ret = fscrypt_setup_filename(vfs_dir, &dentry->d_name, 1, &fname); + if (ret) + return ret; + + /* This needs to handle no-key deletions later on */ + + trans = __unlink_start_trans(dir); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto out_notrans; + } + + /* + * Propagate the last_unlink_trans value of the deleted dir to its + * parent directory. This is to prevent an unrecoverable log tree in the + * case we do something like this: + * 1) create dir foo + * 2) create snapshot under dir foo + * 3) delete the snapshot + * 4) rmdir foo + * 5) mkdir foo + * 6) fsync foo or some file inside foo + * + * This is because we can't unlink other roots when replaying the dir + * deletes for directory foo. + */ + if (inode->last_unlink_trans >= trans->transid) + btrfs_record_snapshot_destroy(trans, dir); + + if (unlikely(btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) { + ret = btrfs_unlink_subvol(trans, dir, dentry); + goto out; + } + + ret = btrfs_orphan_add(trans, inode); + if (ret) + goto out; + + btrfs_record_unlink_dir(trans, dir, inode, false); + + /* now the directory is empty */ + ret = btrfs_unlink_inode(trans, dir, inode, &fname.disk_name); + if (!ret) + btrfs_i_size_write(inode, 0); +out: + btrfs_end_transaction(trans); +out_notrans: + btrfs_btree_balance_dirty(fs_info); + fscrypt_free_filename(&fname); + + return ret; +} + +static bool is_inside_block(u64 bytenr, u64 blockstart, u32 blocksize) +{ + ASSERT(IS_ALIGNED(blockstart, blocksize), "blockstart=%llu blocksize=%u", + blockstart, blocksize); + + if (blockstart <= bytenr && bytenr <= blockstart + blocksize - 1) + return true; + return false; +} + +static int truncate_block_zero_beyond_eof(struct btrfs_inode *inode, u64 start) +{ + const pgoff_t index = (start >> PAGE_SHIFT); + struct address_space *mapping = inode->vfs_inode.i_mapping; + struct folio *folio; + u64 zero_start; + u64 zero_end; + int ret = 0; + +again: + folio = filemap_lock_folio(mapping, index); + /* No folio present. */ + if (IS_ERR(folio)) + return 0; + + if (!folio_test_uptodate(folio)) { + ret = btrfs_read_folio(NULL, folio); + folio_lock(folio); + if (folio->mapping != mapping) { + folio_unlock(folio); + folio_put(folio); + goto again; + } + if (unlikely(!folio_test_uptodate(folio))) { + ret = -EIO; + goto out_unlock; + } + } + folio_wait_writeback(folio); + + /* + * We do not need to lock extents nor wait for OE, as it's already + * beyond EOF. + */ + + zero_start = max_t(u64, folio_pos(folio), start); + zero_end = folio_next_pos(folio); + folio_zero_range(folio, zero_start - folio_pos(folio), + zero_end - zero_start); + +out_unlock: + folio_unlock(folio); + folio_put(folio); + return ret; +} + +/* + * Handle the truncation of a fs block. + * + * @inode - inode that we're zeroing + * @offset - the file offset of the block to truncate + * The value must be inside [@start, @end], and the function will do + * extra checks if the block that covers @offset needs to be zeroed. + * @start - the start file offset of the range we want to zero + * @end - the end (inclusive) file offset of the range we want to zero. + * + * If the range is not block aligned, read out the folio that covers @offset, + * and if needed zero blocks that are inside the folio and covered by [@start, @end). + * If @start or @end + 1 lands inside a block, that block will be marked dirty + * for writeback. + * + * This is utilized by hole punch, zero range, file expansion. + */ +int btrfs_truncate_block(struct btrfs_inode *inode, u64 offset, u64 start, u64 end) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct address_space *mapping = inode->vfs_inode.i_mapping; + struct extent_io_tree *io_tree = &inode->io_tree; + struct btrfs_ordered_extent *ordered; + struct extent_state *cached_state = NULL; + struct extent_changeset *data_reserved = NULL; + bool only_release_metadata = false; + u32 blocksize = fs_info->sectorsize; + pgoff_t index = (offset >> PAGE_SHIFT); + struct folio *folio; + gfp_t mask = btrfs_alloc_write_mask(mapping); + int ret = 0; + const bool in_head_block = is_inside_block(offset, round_down(start, blocksize), + blocksize); + const bool in_tail_block = is_inside_block(offset, round_down(end, blocksize), + blocksize); + bool need_truncate_head = false; + bool need_truncate_tail = false; + u64 zero_start; + u64 zero_end; + u64 block_start; + u64 block_end; + + /* @offset should be inside the range. */ + ASSERT(start <= offset && offset <= end, "offset=%llu start=%llu end=%llu", + offset, start, end); + + /* The range is aligned at both ends. */ + if (IS_ALIGNED(start, blocksize) && IS_ALIGNED(end + 1, blocksize)) { + /* + * For block size < page size case, we may have polluted blocks + * beyond EOF. So we also need to zero them out. + */ + if (end == (u64)-1 && blocksize < PAGE_SIZE) + ret = truncate_block_zero_beyond_eof(inode, start); + goto out; + } + + /* + * @offset may not be inside the head nor tail block. In that case we + * don't need to do anything. + */ + if (!in_head_block && !in_tail_block) + goto out; + + /* + * Skip the truncation if the range in the target block is already aligned. + * The seemingly complex check will also handle the same block case. + */ + if (in_head_block && !IS_ALIGNED(start, blocksize)) + need_truncate_head = true; + if (in_tail_block && !IS_ALIGNED(end + 1, blocksize)) + need_truncate_tail = true; + if (!need_truncate_head && !need_truncate_tail) + goto out; + + block_start = round_down(offset, blocksize); + block_end = block_start + blocksize - 1; + + ret = btrfs_check_data_free_space(inode, &data_reserved, block_start, + blocksize, false); + if (ret < 0) { + size_t write_bytes = blocksize; + + if (btrfs_check_nocow_lock(inode, block_start, &write_bytes, false) > 0) { + /* For nocow case, no need to reserve data space. */ + ASSERT(write_bytes == blocksize, "write_bytes=%zu blocksize=%u", + write_bytes, blocksize); + only_release_metadata = true; + } else { + goto out; + } + } + ret = btrfs_delalloc_reserve_metadata(inode, blocksize, blocksize, false); + if (ret < 0) { + if (!only_release_metadata) + btrfs_free_reserved_data_space(inode, data_reserved, + block_start, blocksize); + goto out; + } +again: + folio = __filemap_get_folio(mapping, index, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, mask); + if (IS_ERR(folio)) { + if (only_release_metadata) + btrfs_delalloc_release_metadata(inode, blocksize, true); + else + btrfs_delalloc_release_space(inode, data_reserved, + block_start, blocksize, true); + btrfs_delalloc_release_extents(inode, blocksize); + ret = PTR_ERR(folio); + goto out; + } + + if (!folio_test_uptodate(folio)) { + ret = btrfs_read_folio(NULL, folio); + folio_lock(folio); + if (folio->mapping != mapping) { + folio_unlock(folio); + folio_put(folio); + goto again; + } + if (unlikely(!folio_test_uptodate(folio))) { + ret = -EIO; + goto out_unlock; + } + } + + /* + * We unlock the page after the io is completed and then re-lock it + * above. release_folio() could have come in between that and cleared + * folio private, but left the page in the mapping. Set the page mapped + * here to make sure it's properly set for the subpage stuff. + */ + ret = set_folio_extent_mapped(folio); + if (ret < 0) + goto out_unlock; + + folio_wait_writeback(folio); + + btrfs_lock_extent(io_tree, block_start, block_end, &cached_state); + + ordered = btrfs_lookup_ordered_extent(inode, block_start); + if (ordered) { + btrfs_unlock_extent(io_tree, block_start, block_end, &cached_state); + folio_unlock(folio); + folio_put(folio); + btrfs_start_ordered_extent(ordered); + btrfs_put_ordered_extent(ordered); + goto again; + } + + ret = btrfs_reset_extent_delalloc(inode, block_start, block_end, 0, &cached_state); + if (ret) { + btrfs_unlock_extent(io_tree, block_start, block_end, &cached_state); + goto out_unlock; + } + + if (end == (u64)-1) { + /* + * We're truncating beyond EOF, the remaining blocks normally are + * already holes thus no need to zero again, but it's possible for + * fs block size < page size cases to have memory mapped writes + * to pollute ranges beyond EOF. + * + * In that case although such polluted blocks beyond EOF will + * not reach disk, it still affects our page caches. + */ + zero_start = max_t(u64, folio_pos(folio), start); + zero_end = min_t(u64, folio_next_pos(folio) - 1, end); + } else { + zero_start = max_t(u64, block_start, start); + zero_end = min_t(u64, block_end, end); + } + folio_zero_range(folio, zero_start - folio_pos(folio), + zero_end - zero_start + 1); + + btrfs_folio_set_dirty(fs_info, folio, block_start, + block_end + 1 - block_start); + + if (only_release_metadata) + btrfs_set_extent_bit(&inode->io_tree, block_start, block_end, + EXTENT_NORESERVE, &cached_state); + + btrfs_unlock_extent(io_tree, block_start, block_end, &cached_state); + +out_unlock: + if (ret) { + if (only_release_metadata) + btrfs_delalloc_release_metadata(inode, blocksize, true); + else + btrfs_delalloc_release_space(inode, data_reserved, + block_start, blocksize, true); + } + btrfs_delalloc_release_extents(inode, blocksize); + folio_unlock(folio); + folio_put(folio); +out: + if (only_release_metadata) + btrfs_check_nocow_unlock(inode); + extent_changeset_free(data_reserved); + return ret; +} + +static int maybe_insert_hole(struct btrfs_inode *inode, u64 offset, u64 len) +{ + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_trans_handle *trans; + struct btrfs_drop_extents_args drop_args = { 0 }; + int ret; + + /* + * If NO_HOLES is enabled, we don't need to do anything. + * Later, up in the call chain, either btrfs_set_inode_last_sub_trans() + * or btrfs_update_inode() will be called, which guarantee that the next + * fsync will know this inode was changed and needs to be logged. + */ + if (btrfs_fs_incompat(fs_info, NO_HOLES)) + return 0; + + /* + * 1 - for the one we're dropping + * 1 - for the one we're adding + * 1 - for updating the inode. + */ + trans = btrfs_start_transaction(root, 3); + if (IS_ERR(trans)) + return PTR_ERR(trans); + + drop_args.start = offset; + drop_args.end = offset + len; + drop_args.drop_cache = true; + + ret = btrfs_drop_extents(trans, root, inode, &drop_args); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + btrfs_end_transaction(trans); + return ret; + } + + ret = btrfs_insert_hole_extent(trans, root, btrfs_ino(inode), offset, len); + if (ret) { + btrfs_abort_transaction(trans, ret); + } else { + btrfs_update_inode_bytes(inode, 0, drop_args.bytes_found); + btrfs_update_inode(trans, inode); + } + btrfs_end_transaction(trans); + return ret; +} + +/* + * This function puts in dummy file extents for the area we're creating a hole + * for. So if we are truncating this file to a larger size we need to insert + * these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for + * the range between oldsize and size + */ +int btrfs_cont_expand(struct btrfs_inode *inode, loff_t oldsize, loff_t size) +{ + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct extent_io_tree *io_tree = &inode->io_tree; + struct extent_map *em = NULL; + struct extent_state *cached_state = NULL; + u64 hole_start = ALIGN(oldsize, fs_info->sectorsize); + u64 block_end = ALIGN(size, fs_info->sectorsize); + u64 last_byte; + u64 cur_offset; + u64 hole_size; + int ret = 0; + + /* + * If our size started in the middle of a block we need to zero out the + * rest of the block before we expand the i_size, otherwise we could + * expose stale data. + */ + ret = btrfs_truncate_block(inode, oldsize, oldsize, -1); + if (ret) + return ret; + + if (size <= hole_start) + return 0; + + btrfs_lock_and_flush_ordered_range(inode, hole_start, block_end - 1, + &cached_state); + cur_offset = hole_start; + while (1) { + em = btrfs_get_extent(inode, NULL, cur_offset, block_end - cur_offset); + if (IS_ERR(em)) { + ret = PTR_ERR(em); + em = NULL; + break; + } + last_byte = min(btrfs_extent_map_end(em), block_end); + last_byte = ALIGN(last_byte, fs_info->sectorsize); + hole_size = last_byte - cur_offset; + + if (!(em->flags & EXTENT_FLAG_PREALLOC)) { + struct extent_map *hole_em; + + ret = maybe_insert_hole(inode, cur_offset, hole_size); + if (ret) + break; + + ret = btrfs_inode_set_file_extent_range(inode, + cur_offset, hole_size); + if (ret) + break; + + hole_em = btrfs_alloc_extent_map(); + if (!hole_em) { + btrfs_drop_extent_map_range(inode, cur_offset, + cur_offset + hole_size - 1, + false); + btrfs_set_inode_full_sync(inode); + goto next; + } + hole_em->start = cur_offset; + hole_em->len = hole_size; + + hole_em->disk_bytenr = EXTENT_MAP_HOLE; + hole_em->disk_num_bytes = 0; + hole_em->ram_bytes = hole_size; + hole_em->generation = btrfs_get_fs_generation(fs_info); + + ret = btrfs_replace_extent_map_range(inode, hole_em, true); + btrfs_free_extent_map(hole_em); + } else { + ret = btrfs_inode_set_file_extent_range(inode, + cur_offset, hole_size); + if (ret) + break; + } +next: + btrfs_free_extent_map(em); + em = NULL; + cur_offset = last_byte; + if (cur_offset >= block_end) + break; + } + btrfs_free_extent_map(em); + btrfs_unlock_extent(io_tree, hole_start, block_end - 1, &cached_state); + return ret; +} + +static int btrfs_setsize(struct inode *inode, struct iattr *attr) +{ + struct btrfs_root *root = BTRFS_I(inode)->root; + struct btrfs_trans_handle *trans; + loff_t oldsize = i_size_read(inode); + loff_t newsize = attr->ia_size; + int mask = attr->ia_valid; + int ret; + + /* + * The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a + * special case where we need to update the times despite not having + * these flags set. For all other operations the VFS set these flags + * explicitly if it wants a timestamp update. + */ + if (newsize != oldsize) { + inode_inc_iversion(inode); + if (!(mask & (ATTR_CTIME | ATTR_MTIME))) { + inode_set_mtime_to_ts(inode, + inode_set_ctime_current(inode)); + } + } + + if (newsize > oldsize) { + /* + * Don't do an expanding truncate while snapshotting is ongoing. + * This is to ensure the snapshot captures a fully consistent + * state of this file - if the snapshot captures this expanding + * truncation, it must capture all writes that happened before + * this truncation. + */ + btrfs_drew_write_lock(&root->snapshot_lock); + ret = btrfs_cont_expand(BTRFS_I(inode), oldsize, newsize); + if (ret) { + btrfs_drew_write_unlock(&root->snapshot_lock); + return ret; + } + + trans = btrfs_start_transaction(root, 1); + if (IS_ERR(trans)) { + btrfs_drew_write_unlock(&root->snapshot_lock); + return PTR_ERR(trans); + } + + i_size_write(inode, newsize); + btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), 0); + pagecache_isize_extended(inode, oldsize, newsize); + ret = btrfs_update_inode(trans, BTRFS_I(inode)); + btrfs_drew_write_unlock(&root->snapshot_lock); + btrfs_end_transaction(trans); + } else { + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + + if (btrfs_is_zoned(fs_info)) { + ret = btrfs_wait_ordered_range(BTRFS_I(inode), + ALIGN(newsize, fs_info->sectorsize), + (u64)-1); + if (ret) + return ret; + } + + /* + * We're truncating a file that used to have good data down to + * zero. Make sure any new writes to the file get on disk + * on close. + */ + if (newsize == 0 && oldsize != 0) + set_bit(BTRFS_INODE_FLUSH_ON_CLOSE, + &BTRFS_I(inode)->runtime_flags); + + truncate_setsize(inode, newsize); + + inode_dio_wait(inode); + + ret = btrfs_truncate(BTRFS_I(inode), newsize == oldsize); + if (ret && inode->i_nlink) { + int ret2; + + /* + * Truncate failed, so fix up the in-memory size. We + * adjusted disk_i_size down as we removed extents, so + * wait for disk_i_size to be stable and then update the + * in-memory size to match. + */ + ret2 = btrfs_wait_ordered_range(BTRFS_I(inode), 0, (u64)-1); + if (ret2) + return ret2; + i_size_write(inode, BTRFS_I(inode)->disk_i_size); + } + } + + return ret; +} + +static int btrfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +{ + struct inode *inode = d_inode(dentry); + struct btrfs_root *root = BTRFS_I(inode)->root; + int ret; + + if (btrfs_root_readonly(root)) + return -EROFS; + + ret = setattr_prepare(idmap, dentry, attr); + if (ret) + return ret; + + if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) { + ret = btrfs_setsize(inode, attr); + if (ret) + return ret; + } + + if (attr->ia_valid) { + setattr_copy(idmap, inode, attr); + inode_inc_iversion(inode); + ret = btrfs_dirty_inode(BTRFS_I(inode)); + + if (!ret && attr->ia_valid & ATTR_MODE) + ret = posix_acl_chmod(idmap, dentry, inode->i_mode); + } + + return ret; +} + +/* + * While truncating the inode pages during eviction, we get the VFS + * calling btrfs_invalidate_folio() against each folio of the inode. This + * is slow because the calls to btrfs_invalidate_folio() result in a + * huge amount of calls to lock_extent() and clear_extent_bit(), + * which keep merging and splitting extent_state structures over and over, + * wasting lots of time. + * + * Therefore if the inode is being evicted, let btrfs_invalidate_folio() + * skip all those expensive operations on a per folio basis and do only + * the ordered io finishing, while we release here the extent_map and + * extent_state structures, without the excessive merging and splitting. + */ +static void evict_inode_truncate_pages(struct inode *inode) +{ + struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; + struct rb_node *node; + + ASSERT(inode_state_read_once(inode) & I_FREEING); + truncate_inode_pages_final(&inode->i_data); + + btrfs_drop_extent_map_range(BTRFS_I(inode), 0, (u64)-1, false); + + /* + * Keep looping until we have no more ranges in the io tree. + * We can have ongoing bios started by readahead that have + * their endio callback (extent_io.c:end_bio_extent_readpage) + * still in progress (unlocked the pages in the bio but did not yet + * unlocked the ranges in the io tree). Therefore this means some + * ranges can still be locked and eviction started because before + * submitting those bios, which are executed by a separate task (work + * queue kthread), inode references (inode->i_count) were not taken + * (which would be dropped in the end io callback of each bio). + * Therefore here we effectively end up waiting for those bios and + * anyone else holding locked ranges without having bumped the inode's + * reference count - if we don't do it, when they access the inode's + * io_tree to unlock a range it may be too late, leading to an + * use-after-free issue. + */ + spin_lock(&io_tree->lock); + while (!RB_EMPTY_ROOT(&io_tree->state)) { + struct extent_state *state; + struct extent_state *cached_state = NULL; + u64 start; + u64 end; + unsigned state_flags; + + node = rb_first(&io_tree->state); + state = rb_entry(node, struct extent_state, rb_node); + start = state->start; + end = state->end; + state_flags = state->state; + spin_unlock(&io_tree->lock); + + btrfs_lock_extent(io_tree, start, end, &cached_state); + + /* + * If still has DELALLOC flag, the extent didn't reach disk, + * and its reserved space won't be freed by delayed_ref. + * So we need to free its reserved space here. + * (Refer to comment in btrfs_invalidate_folio, case 2) + * + * Note, end is the bytenr of last byte, so we need + 1 here. + */ + if (state_flags & EXTENT_DELALLOC) + btrfs_qgroup_free_data(BTRFS_I(inode), NULL, start, + end - start + 1, NULL); + + btrfs_clear_extent_bit(io_tree, start, end, + EXTENT_CLEAR_ALL_BITS | EXTENT_DO_ACCOUNTING, + &cached_state); + + cond_resched(); + spin_lock(&io_tree->lock); + } + spin_unlock(&io_tree->lock); +} + +static struct btrfs_trans_handle *evict_refill_and_join(struct btrfs_root *root, + struct btrfs_block_rsv *rsv) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_trans_handle *trans; + u64 delayed_refs_extra = btrfs_calc_delayed_ref_bytes(fs_info, 1); + int ret; + + /* + * Eviction should be taking place at some place safe because of our + * delayed iputs. However the normal flushing code will run delayed + * iputs, so we cannot use FLUSH_ALL otherwise we'll deadlock. + * + * We reserve the delayed_refs_extra here again because we can't use + * btrfs_start_transaction(root, 0) for the same deadlocky reason as + * above. We reserve our extra bit here because we generate a ton of + * delayed refs activity by truncating. + * + * BTRFS_RESERVE_FLUSH_EVICT will steal from the global_rsv if it can, + * if we fail to make this reservation we can re-try without the + * delayed_refs_extra so we can make some forward progress. + */ + ret = btrfs_block_rsv_refill(fs_info, rsv, rsv->size + delayed_refs_extra, + BTRFS_RESERVE_FLUSH_EVICT); + if (ret) { + ret = btrfs_block_rsv_refill(fs_info, rsv, rsv->size, + BTRFS_RESERVE_FLUSH_EVICT); + if (ret) { + btrfs_warn(fs_info, + "could not allocate space for delete; will truncate on mount"); + return ERR_PTR(-ENOSPC); + } + delayed_refs_extra = 0; + } + + trans = btrfs_join_transaction(root); + if (IS_ERR(trans)) + return trans; + + if (delayed_refs_extra) { + trans->block_rsv = &fs_info->trans_block_rsv; + trans->bytes_reserved = delayed_refs_extra; + btrfs_block_rsv_migrate(rsv, trans->block_rsv, + delayed_refs_extra, true); + } + return trans; +} + +void btrfs_evict_inode(struct inode *inode) +{ + struct btrfs_fs_info *fs_info; + struct btrfs_trans_handle *trans; + struct btrfs_root *root = BTRFS_I(inode)->root; + struct btrfs_block_rsv rsv; + int ret; + + trace_btrfs_inode_evict(inode); + + if (!root) + goto clear_inode; + + fs_info = inode_to_fs_info(inode); + evict_inode_truncate_pages(inode); + + if (inode->i_nlink && + ((btrfs_root_refs(&root->root_item) != 0 && + btrfs_root_id(root) != BTRFS_ROOT_TREE_OBJECTID) || + btrfs_is_free_space_inode(BTRFS_I(inode)))) + goto out; + + if (is_bad_inode(inode)) + goto out; + + if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) + goto out; + + if (inode->i_nlink > 0) { + BUG_ON(btrfs_root_refs(&root->root_item) != 0 && + btrfs_root_id(root) != BTRFS_ROOT_TREE_OBJECTID); + goto out; + } + + /* + * This makes sure the inode item in tree is uptodate and the space for + * the inode update is released. + */ + ret = btrfs_commit_inode_delayed_inode(BTRFS_I(inode)); + if (ret) + goto out; + + /* + * This drops any pending insert or delete operations we have for this + * inode. We could have a delayed dir index deletion queued up, but + * we're removing the inode completely so that'll be taken care of in + * the truncate. + */ + btrfs_kill_delayed_inode_items(BTRFS_I(inode)); + + btrfs_init_metadata_block_rsv(fs_info, &rsv, BTRFS_BLOCK_RSV_TEMP); + rsv.size = btrfs_calc_metadata_size(fs_info, 1); + rsv.failfast = true; + + btrfs_i_size_write(BTRFS_I(inode), 0); + + while (1) { + struct btrfs_truncate_control control = { + .inode = BTRFS_I(inode), + .ino = btrfs_ino(BTRFS_I(inode)), + .new_size = 0, + .min_type = 0, + }; + + trans = evict_refill_and_join(root, &rsv); + if (IS_ERR(trans)) + goto out_release; + + trans->block_rsv = &rsv; + + ret = btrfs_truncate_inode_items(trans, root, &control); + trans->block_rsv = &fs_info->trans_block_rsv; + btrfs_end_transaction(trans); + /* + * We have not added new delayed items for our inode after we + * have flushed its delayed items, so no need to throttle on + * delayed items. However we have modified extent buffers. + */ + btrfs_btree_balance_dirty_nodelay(fs_info); + if (ret && ret != -ENOSPC && ret != -EAGAIN) + goto out_release; + else if (!ret) + break; + } + + /* + * Errors here aren't a big deal, it just means we leave orphan items in + * the tree. They will be cleaned up on the next mount. If the inode + * number gets reused, cleanup deletes the orphan item without doing + * anything, and unlink reuses the existing orphan item. + * + * If it turns out that we are dropping too many of these, we might want + * to add a mechanism for retrying these after a commit. + */ + trans = evict_refill_and_join(root, &rsv); + if (!IS_ERR(trans)) { + trans->block_rsv = &rsv; + btrfs_orphan_del(trans, BTRFS_I(inode)); + trans->block_rsv = &fs_info->trans_block_rsv; + btrfs_end_transaction(trans); + } + +out_release: + btrfs_block_rsv_release(fs_info, &rsv, (u64)-1, NULL); +out: + /* + * If we didn't successfully delete, the orphan item will still be in + * the tree and we'll retry on the next mount. Again, we might also want + * to retry these periodically in the future. + */ + btrfs_remove_delayed_node(BTRFS_I(inode)); +clear_inode: + clear_inode(inode); +} + +/* + * Return the key found in the dir entry in the location pointer, fill @type + * with BTRFS_FT_*, and return 0. + * + * If no dir entries were found, returns -ENOENT. + * If found a corrupted location in dir entry, returns -EUCLEAN. + */ +static int btrfs_inode_by_name(struct btrfs_inode *dir, struct dentry *dentry, + struct btrfs_key *location, u8 *type) +{ + struct btrfs_dir_item *di; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_root *root = dir->root; + int ret = 0; + struct fscrypt_name fname; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + ret = fscrypt_setup_filename(&dir->vfs_inode, &dentry->d_name, 1, &fname); + if (ret < 0) + return ret; + /* + * fscrypt_setup_filename() should never return a positive value, but + * gcc on sparc/parisc thinks it can, so assert that doesn't happen. + */ + ASSERT(ret == 0); + + /* This needs to handle no-key deletions later on */ + + di = btrfs_lookup_dir_item(NULL, root, path, btrfs_ino(dir), + &fname.disk_name, 0); + if (IS_ERR_OR_NULL(di)) { + ret = di ? PTR_ERR(di) : -ENOENT; + goto out; + } + + btrfs_dir_item_key_to_cpu(path->nodes[0], di, location); + if (unlikely(location->type != BTRFS_INODE_ITEM_KEY && + location->type != BTRFS_ROOT_ITEM_KEY)) { + ret = -EUCLEAN; + btrfs_warn(root->fs_info, +"%s gets something invalid in DIR_ITEM (name %s, directory ino %llu, location " BTRFS_KEY_FMT ")", + __func__, fname.disk_name.name, btrfs_ino(dir), + BTRFS_KEY_FMT_VALUE(location)); + } + if (!ret) + *type = btrfs_dir_ftype(path->nodes[0], di); +out: + fscrypt_free_filename(&fname); + return ret; +} + +/* + * when we hit a tree root in a directory, the btrfs part of the inode + * needs to be changed to reflect the root directory of the tree root. This + * is kind of like crossing a mount point. + */ +static int fixup_tree_root_location(struct btrfs_fs_info *fs_info, + struct btrfs_inode *dir, + struct dentry *dentry, + struct btrfs_key *location, + struct btrfs_root **sub_root) +{ + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_root *new_root; + struct btrfs_root_ref *ref; + struct extent_buffer *leaf; + struct btrfs_key key; + int ret; + int err = 0; + struct fscrypt_name fname; + + ret = fscrypt_setup_filename(&dir->vfs_inode, &dentry->d_name, 0, &fname); + if (ret) + return ret; + + path = btrfs_alloc_path(); + if (!path) { + err = -ENOMEM; + goto out; + } + + err = -ENOENT; + key.objectid = btrfs_root_id(dir->root); + key.type = BTRFS_ROOT_REF_KEY; + key.offset = location->objectid; + + ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0); + if (ret) { + if (ret < 0) + err = ret; + goto out; + } + + leaf = path->nodes[0]; + ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref); + if (btrfs_root_ref_dirid(leaf, ref) != btrfs_ino(dir) || + btrfs_root_ref_name_len(leaf, ref) != fname.disk_name.len) + goto out; + + ret = memcmp_extent_buffer(leaf, fname.disk_name.name, + (unsigned long)(ref + 1), fname.disk_name.len); + if (ret) + goto out; + + btrfs_release_path(path); + + new_root = btrfs_get_fs_root(fs_info, location->objectid, true); + if (IS_ERR(new_root)) { + err = PTR_ERR(new_root); + goto out; + } + + *sub_root = new_root; + location->objectid = btrfs_root_dirid(&new_root->root_item); + location->type = BTRFS_INODE_ITEM_KEY; + location->offset = 0; + err = 0; +out: + fscrypt_free_filename(&fname); + return err; +} + + + +static void btrfs_del_inode_from_root(struct btrfs_inode *inode) +{ + struct btrfs_root *root = inode->root; + struct btrfs_inode *entry; + bool empty = false; + + xa_lock(&root->inodes); + /* + * This btrfs_inode is being freed and has already been unhashed at this + * point. It's possible that another btrfs_inode has already been + * allocated for the same inode and inserted itself into the root, so + * don't delete it in that case. + * + * Note that this shouldn't need to allocate memory, so the gfp flags + * don't really matter. + */ + entry = __xa_cmpxchg(&root->inodes, btrfs_ino(inode), inode, NULL, + GFP_ATOMIC); + if (entry == inode) + empty = xa_empty(&root->inodes); + xa_unlock(&root->inodes); + + if (empty && btrfs_root_refs(&root->root_item) == 0) { + xa_lock(&root->inodes); + empty = xa_empty(&root->inodes); + xa_unlock(&root->inodes); + if (empty) + btrfs_add_dead_root(root); + } +} + + +static int btrfs_init_locked_inode(struct inode *inode, void *p) +{ + struct btrfs_iget_args *args = p; + + btrfs_set_inode_number(BTRFS_I(inode), args->ino); + BTRFS_I(inode)->root = btrfs_grab_root(args->root); + + if (args->root && args->root == args->root->fs_info->tree_root && + args->ino != BTRFS_BTREE_INODE_OBJECTID) + set_bit(BTRFS_INODE_FREE_SPACE_INODE, + &BTRFS_I(inode)->runtime_flags); + return 0; +} + +static int btrfs_find_actor(struct inode *inode, void *opaque) +{ + struct btrfs_iget_args *args = opaque; + + return args->ino == btrfs_ino(BTRFS_I(inode)) && + args->root == BTRFS_I(inode)->root; +} + +static struct btrfs_inode *btrfs_iget_locked(u64 ino, struct btrfs_root *root) +{ + struct inode *inode; + struct btrfs_iget_args args; + unsigned long hashval = btrfs_inode_hash(ino, root); + + args.ino = ino; + args.root = root; + + inode = iget5_locked_rcu(root->fs_info->sb, hashval, btrfs_find_actor, + btrfs_init_locked_inode, + (void *)&args); + if (!inode) + return NULL; + return BTRFS_I(inode); +} + +/* + * Get an inode object given its inode number and corresponding root. Path is + * preallocated to prevent recursing back to iget through allocator. + */ +struct btrfs_inode *btrfs_iget_path(u64 ino, struct btrfs_root *root, + struct btrfs_path *path) +{ + struct btrfs_inode *inode; + int ret; + + inode = btrfs_iget_locked(ino, root); + if (!inode) + return ERR_PTR(-ENOMEM); + + if (!(inode_state_read_once(&inode->vfs_inode) & I_NEW)) + return inode; + + ret = btrfs_read_locked_inode(inode, path); + if (ret) + return ERR_PTR(ret); + + unlock_new_inode(&inode->vfs_inode); + return inode; +} + +/* + * Get an inode object given its inode number and corresponding root. + */ +struct btrfs_inode *btrfs_iget(u64 ino, struct btrfs_root *root) +{ + struct btrfs_inode *inode; + struct btrfs_path *path; + int ret; + + inode = btrfs_iget_locked(ino, root); + if (!inode) + return ERR_PTR(-ENOMEM); + + if (!(inode_state_read_once(&inode->vfs_inode) & I_NEW)) + return inode; + + path = btrfs_alloc_path(); + if (!path) { + iget_failed(&inode->vfs_inode); + return ERR_PTR(-ENOMEM); + } + + ret = btrfs_read_locked_inode(inode, path); + btrfs_free_path(path); + if (ret) + return ERR_PTR(ret); + + if (S_ISDIR(inode->vfs_inode.i_mode)) + inode->vfs_inode.i_opflags |= IOP_FASTPERM_MAY_EXEC; + unlock_new_inode(&inode->vfs_inode); + return inode; +} + +static struct btrfs_inode *new_simple_dir(struct inode *dir, + struct btrfs_key *key, + struct btrfs_root *root) +{ + struct timespec64 ts; + struct inode *vfs_inode; + struct btrfs_inode *inode; + + vfs_inode = new_inode(dir->i_sb); + if (!vfs_inode) + return ERR_PTR(-ENOMEM); + + inode = BTRFS_I(vfs_inode); + inode->root = btrfs_grab_root(root); + inode->ref_root_id = key->objectid; + set_bit(BTRFS_INODE_ROOT_STUB, &inode->runtime_flags); + set_bit(BTRFS_INODE_DUMMY, &inode->runtime_flags); + + btrfs_set_inode_number(inode, BTRFS_EMPTY_SUBVOL_DIR_OBJECTID); + /* + * We only need lookup, the rest is read-only and there's no inode + * associated with the dentry + */ + vfs_inode->i_op = &simple_dir_inode_operations; + vfs_inode->i_opflags &= ~IOP_XATTR; + vfs_inode->i_fop = &simple_dir_operations; + vfs_inode->i_mode = S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO; + + ts = inode_set_ctime_current(vfs_inode); + inode_set_mtime_to_ts(vfs_inode, ts); + inode_set_atime_to_ts(vfs_inode, inode_get_atime(dir)); + inode->i_otime_sec = ts.tv_sec; + inode->i_otime_nsec = ts.tv_nsec; + + vfs_inode->i_uid = dir->i_uid; + vfs_inode->i_gid = dir->i_gid; + + return inode; +} + +static_assert(BTRFS_FT_UNKNOWN == FT_UNKNOWN); +static_assert(BTRFS_FT_REG_FILE == FT_REG_FILE); +static_assert(BTRFS_FT_DIR == FT_DIR); +static_assert(BTRFS_FT_CHRDEV == FT_CHRDEV); +static_assert(BTRFS_FT_BLKDEV == FT_BLKDEV); +static_assert(BTRFS_FT_FIFO == FT_FIFO); +static_assert(BTRFS_FT_SOCK == FT_SOCK); +static_assert(BTRFS_FT_SYMLINK == FT_SYMLINK); + +static inline u8 btrfs_inode_type(const struct btrfs_inode *inode) +{ + return fs_umode_to_ftype(inode->vfs_inode.i_mode); +} + +struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry) +{ + struct btrfs_fs_info *fs_info = inode_to_fs_info(dir); + struct btrfs_inode *inode; + struct btrfs_root *root = BTRFS_I(dir)->root; + struct btrfs_root *sub_root = root; + struct btrfs_key location = { 0 }; + u8 di_type = 0; + int ret = 0; + + if (dentry->d_name.len > BTRFS_NAME_LEN) + return ERR_PTR(-ENAMETOOLONG); + + ret = btrfs_inode_by_name(BTRFS_I(dir), dentry, &location, &di_type); + if (ret < 0) + return ERR_PTR(ret); + + if (location.type == BTRFS_INODE_ITEM_KEY) { + inode = btrfs_iget(location.objectid, root); + if (IS_ERR(inode)) + return ERR_CAST(inode); + + /* Do extra check against inode mode with di_type */ + if (unlikely(btrfs_inode_type(inode) != di_type)) { + btrfs_crit(fs_info, +"inode mode mismatch with dir: inode mode=0%o btrfs type=%u dir type=%u", + inode->vfs_inode.i_mode, btrfs_inode_type(inode), + di_type); + iput(&inode->vfs_inode); + return ERR_PTR(-EUCLEAN); + } + return &inode->vfs_inode; + } + + ret = fixup_tree_root_location(fs_info, BTRFS_I(dir), dentry, + &location, &sub_root); + if (ret < 0) { + if (ret != -ENOENT) + inode = ERR_PTR(ret); + else + inode = new_simple_dir(dir, &location, root); + } else { + inode = btrfs_iget(location.objectid, sub_root); + btrfs_put_root(sub_root); + + if (IS_ERR(inode)) + return ERR_CAST(inode); + + down_read(&fs_info->cleanup_work_sem); + if (!sb_rdonly(inode->vfs_inode.i_sb)) + ret = btrfs_orphan_cleanup(sub_root); + up_read(&fs_info->cleanup_work_sem); + if (ret) { + iput(&inode->vfs_inode); + inode = ERR_PTR(ret); + } + } + + if (IS_ERR(inode)) + return ERR_CAST(inode); + + return &inode->vfs_inode; +} + +static int btrfs_dentry_delete(const struct dentry *dentry) +{ + struct btrfs_root *root; + struct inode *inode = d_inode(dentry); + + if (!inode && !IS_ROOT(dentry)) + inode = d_inode(dentry->d_parent); + + if (inode) { + root = BTRFS_I(inode)->root; + if (btrfs_root_refs(&root->root_item) == 0) + return 1; + + if (btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) + return 1; + } + return 0; +} + +static struct dentry *btrfs_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + struct inode *inode = btrfs_lookup_dentry(dir, dentry); + + if (inode == ERR_PTR(-ENOENT)) + inode = NULL; + return d_splice_alias(inode, dentry); +} + +/* + * Find the highest existing sequence number in a directory and then set the + * in-memory index_cnt variable to the first free sequence number. + */ +static int btrfs_set_inode_index_count(struct btrfs_inode *inode) +{ + struct btrfs_root *root = inode->root; + struct btrfs_key key, found_key; + BTRFS_PATH_AUTO_FREE(path); + struct extent_buffer *leaf; + int ret; + + key.objectid = btrfs_ino(inode); + key.type = BTRFS_DIR_INDEX_KEY; + key.offset = (u64)-1; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + return ret; + + if (unlikely(ret == 0)) { + /* + * Key with offset -1 found, there would have to exist a dir + * index item with such offset, but this is out of the valid + * range. + */ + btrfs_err(root->fs_info, + "unexpected exact match for DIR_INDEX key, inode %llu", + btrfs_ino(inode)); + return -EUCLEAN; + } + + if (path->slots[0] == 0) { + inode->index_cnt = BTRFS_DIR_START_INDEX; + return 0; + } + + path->slots[0]--; + + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); + + if (found_key.objectid != btrfs_ino(inode) || + found_key.type != BTRFS_DIR_INDEX_KEY) { + inode->index_cnt = BTRFS_DIR_START_INDEX; + return 0; + } + + inode->index_cnt = found_key.offset + 1; + + return 0; +} + +static int btrfs_get_dir_last_index(struct btrfs_inode *dir, u64 *index) +{ + int ret = 0; + + btrfs_inode_lock(dir, 0); + if (dir->index_cnt == (u64)-1) { + ret = btrfs_inode_delayed_dir_index_count(dir); + if (ret) { + ret = btrfs_set_inode_index_count(dir); + if (ret) + goto out; + } + } + + /* index_cnt is the index number of next new entry, so decrement it. */ + *index = dir->index_cnt - 1; +out: + btrfs_inode_unlock(dir, 0); + + return ret; +} + +/* + * All this infrastructure exists because dir_emit can fault, and we are holding + * the tree lock when doing readdir. For now just allocate a buffer and copy + * our information into that, and then dir_emit from the buffer. This is + * similar to what NFS does, only we don't keep the buffer around in pagecache + * because I'm afraid I'll mess that up. Long term we need to make filldir do + * copy_to_user_inatomic so we don't have to worry about page faulting under the + * tree lock. + */ +static int btrfs_opendir(struct inode *inode, struct file *file) +{ + struct btrfs_file_private *private; + u64 last_index; + int ret; + + ret = btrfs_get_dir_last_index(BTRFS_I(inode), &last_index); + if (ret) + return ret; + + private = kzalloc_obj(struct btrfs_file_private); + if (!private) + return -ENOMEM; + private->last_index = last_index; + private->filldir_buf = kzalloc(PAGE_SIZE, GFP_KERNEL); + if (!private->filldir_buf) { + kfree(private); + return -ENOMEM; + } + file->private_data = private; + return 0; +} + +static loff_t btrfs_dir_llseek(struct file *file, loff_t offset, int whence) +{ + struct btrfs_file_private *private = file->private_data; + int ret; + + ret = btrfs_get_dir_last_index(BTRFS_I(file_inode(file)), + &private->last_index); + if (ret) + return ret; + + return generic_file_llseek(file, offset, whence); +} + +struct dir_entry { + u64 ino; + u64 offset; + unsigned type; + int name_len; +}; + +static int btrfs_filldir(void *addr, int entries, struct dir_context *ctx) +{ + while (entries--) { + struct dir_entry *entry = addr; + char *name = (char *)(entry + 1); + + ctx->pos = get_unaligned(&entry->offset); + if (!dir_emit(ctx, name, get_unaligned(&entry->name_len), + get_unaligned(&entry->ino), + get_unaligned(&entry->type))) + return 1; + addr += sizeof(struct dir_entry) + + get_unaligned(&entry->name_len); + ctx->pos++; + } + return 0; +} + +static int btrfs_real_readdir(struct file *file, struct dir_context *ctx) +{ + struct inode *inode = file_inode(file); + struct btrfs_root *root = BTRFS_I(inode)->root; + struct btrfs_file_private *private = file->private_data; + struct btrfs_dir_item *di; + struct btrfs_key key; + struct btrfs_key found_key; + BTRFS_PATH_AUTO_FREE(path); + void *addr; + LIST_HEAD(ins_list); + LIST_HEAD(del_list); + int ret; + char *name_ptr; + int name_len; + int entries = 0; + int total_len = 0; + bool put = false; + struct btrfs_key location; + + if (!dir_emit_dots(file, ctx)) + return 0; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + addr = private->filldir_buf; + path->reada = READA_FORWARD; + + put = btrfs_readdir_get_delayed_items(BTRFS_I(inode), private->last_index, + &ins_list, &del_list); + +again: + key.type = BTRFS_DIR_INDEX_KEY; + key.offset = ctx->pos; + key.objectid = btrfs_ino(BTRFS_I(inode)); + + btrfs_for_each_slot(root, &key, &found_key, path, ret) { + struct dir_entry *entry; + struct extent_buffer *leaf = path->nodes[0]; + u8 ftype; + + if (found_key.objectid != key.objectid) + break; + if (found_key.type != BTRFS_DIR_INDEX_KEY) + break; + if (found_key.offset < ctx->pos) + continue; + if (found_key.offset > private->last_index) + break; + if (btrfs_should_delete_dir_index(&del_list, found_key.offset)) + continue; + di = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dir_item); + name_len = btrfs_dir_name_len(leaf, di); + if ((total_len + sizeof(struct dir_entry) + name_len) >= + PAGE_SIZE) { + btrfs_release_path(path); + ret = btrfs_filldir(private->filldir_buf, entries, ctx); + if (ret) + goto nopos; + addr = private->filldir_buf; + entries = 0; + total_len = 0; + goto again; + } + + ftype = btrfs_dir_flags_to_ftype(btrfs_dir_flags(leaf, di)); + entry = addr; + name_ptr = (char *)(entry + 1); + read_extent_buffer(leaf, name_ptr, + (unsigned long)(di + 1), name_len); + put_unaligned(name_len, &entry->name_len); + put_unaligned(fs_ftype_to_dtype(ftype), &entry->type); + btrfs_dir_item_key_to_cpu(leaf, di, &location); + put_unaligned(location.objectid, &entry->ino); + put_unaligned(found_key.offset, &entry->offset); + entries++; + addr += sizeof(struct dir_entry) + name_len; + total_len += sizeof(struct dir_entry) + name_len; + } + /* Catch error encountered during iteration */ + if (ret < 0) + goto err; + + btrfs_release_path(path); + + ret = btrfs_filldir(private->filldir_buf, entries, ctx); + if (ret) + goto nopos; + + if (btrfs_readdir_delayed_dir_index(ctx, &ins_list)) + goto nopos; + + /* + * Stop new entries from being returned after we return the last + * entry. + * + * New directory entries are assigned a strictly increasing + * offset. This means that new entries created during readdir + * are *guaranteed* to be seen in the future by that readdir. + * This has broken buggy programs which operate on names as + * they're returned by readdir. Until we reuse freed offsets + * we have this hack to stop new entries from being returned + * under the assumption that they'll never reach this huge + * offset. + * + * This is being careful not to overflow 32bit loff_t unless the + * last entry requires it because doing so has broken 32bit apps + * in the past. + */ + if (ctx->pos >= INT_MAX) + ctx->pos = LLONG_MAX; + else + ctx->pos = INT_MAX; +nopos: + ret = 0; +err: + if (put) + btrfs_readdir_put_delayed_items(BTRFS_I(inode), &ins_list, &del_list); + return ret; +} + +/* + * This is somewhat expensive, updating the tree every time the + * inode changes. But, it is most likely to find the inode in cache. + * FIXME, needs more benchmarking...there are no reasons other than performance + * to keep or drop this code. + */ +static int btrfs_dirty_inode(struct btrfs_inode *inode) +{ + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_trans_handle *trans; + int ret; + + if (test_bit(BTRFS_INODE_DUMMY, &inode->runtime_flags)) + return 0; + + trans = btrfs_join_transaction(root); + if (IS_ERR(trans)) + return PTR_ERR(trans); + + ret = btrfs_update_inode(trans, inode); + if (ret == -ENOSPC || ret == -EDQUOT) { + /* whoops, lets try again with the full transaction */ + btrfs_end_transaction(trans); + trans = btrfs_start_transaction(root, 1); + if (IS_ERR(trans)) + return PTR_ERR(trans); + + ret = btrfs_update_inode(trans, inode); + } + btrfs_end_transaction(trans); + if (inode->delayed_node) + btrfs_balance_delayed_items(fs_info); + + return ret; +} + +/* + * We need our own ->update_time so that we can return error on ENOSPC for + * updating the inode in the case of file write and mmap writes. + */ +static int btrfs_update_time(struct inode *inode, enum fs_update_time type, + unsigned int flags) +{ + struct btrfs_root *root = BTRFS_I(inode)->root; + int dirty; + + if (btrfs_root_readonly(root)) + return -EROFS; + if (flags & IOCB_NOWAIT) + return -EAGAIN; + + dirty = inode_update_time(inode, type, flags); + if (dirty <= 0) + return dirty; + return btrfs_dirty_inode(BTRFS_I(inode)); +} + +/* + * helper to find a free sequence number in a given directory. This current + * code is very simple, later versions will do smarter things in the btree + */ +int btrfs_set_inode_index(struct btrfs_inode *dir, u64 *index) +{ + int ret = 0; + + if (dir->index_cnt == (u64)-1) { + ret = btrfs_inode_delayed_dir_index_count(dir); + if (ret) { + ret = btrfs_set_inode_index_count(dir); + if (ret) + return ret; + } + } + + *index = dir->index_cnt; + dir->index_cnt++; + + return ret; +} + +static int btrfs_insert_inode_locked(struct inode *inode) +{ + struct btrfs_iget_args args; + + args.ino = btrfs_ino(BTRFS_I(inode)); + args.root = BTRFS_I(inode)->root; + + return insert_inode_locked4(inode, + btrfs_inode_hash(inode->i_ino, BTRFS_I(inode)->root), + btrfs_find_actor, &args); +} + +int btrfs_new_inode_prepare(struct btrfs_new_inode_args *args, + unsigned int *trans_num_items) +{ + struct inode *dir = args->dir; + struct inode *inode = args->inode; + int ret; + + if (!args->orphan) { + ret = fscrypt_setup_filename(dir, &args->dentry->d_name, 0, + &args->fname); + if (ret) + return ret; + } + + ret = posix_acl_create(dir, &inode->i_mode, &args->default_acl, &args->acl); + if (ret) { + fscrypt_free_filename(&args->fname); + return ret; + } + + /* 1 to add inode item */ + *trans_num_items = 1; + /* 1 to add compression property */ + if (BTRFS_I(dir)->prop_compress) + (*trans_num_items)++; + /* 1 to add default ACL xattr */ + if (args->default_acl) + (*trans_num_items)++; + /* 1 to add access ACL xattr */ + if (args->acl) + (*trans_num_items)++; +#ifdef CONFIG_SECURITY + /* 1 to add LSM xattr */ + if (dir->i_security) + (*trans_num_items)++; +#endif + if (args->orphan) { + /* 1 to add orphan item */ + (*trans_num_items)++; + } else { + /* + * 1 to add dir item + * 1 to add dir index + * 1 to update parent inode item + * + * No need for 1 unit for the inode ref item because it is + * inserted in a batch together with the inode item at + * btrfs_create_new_inode(). + */ + *trans_num_items += 3; + } + return 0; +} + +void btrfs_new_inode_args_destroy(struct btrfs_new_inode_args *args) +{ + posix_acl_release(args->acl); + posix_acl_release(args->default_acl); + fscrypt_free_filename(&args->fname); +} + +/* + * Inherit flags from the parent inode. + * + * Currently only the compression flags and the cow flags are inherited. + */ +static void btrfs_inherit_iflags(struct btrfs_inode *inode, struct btrfs_inode *dir) +{ + unsigned int flags; + + flags = dir->flags; + + if (flags & BTRFS_INODE_NOCOMPRESS) { + inode->flags &= ~BTRFS_INODE_COMPRESS; + inode->flags |= BTRFS_INODE_NOCOMPRESS; + } else if (flags & BTRFS_INODE_COMPRESS) { + inode->flags &= ~BTRFS_INODE_NOCOMPRESS; + inode->flags |= BTRFS_INODE_COMPRESS; + } + + if (flags & BTRFS_INODE_NODATACOW) { + inode->flags |= BTRFS_INODE_NODATACOW; + if (S_ISREG(inode->vfs_inode.i_mode)) + inode->flags |= BTRFS_INODE_NODATASUM; + } + + btrfs_sync_inode_flags_to_i_flags(inode); +} + +int btrfs_create_new_inode(struct btrfs_trans_handle *trans, + struct btrfs_new_inode_args *args) +{ + struct timespec64 ts; + struct inode *dir = args->dir; + struct inode *inode = args->inode; + const struct fscrypt_str *name = args->orphan ? NULL : &args->fname.disk_name; + struct btrfs_fs_info *fs_info = inode_to_fs_info(dir); + struct btrfs_root *root; + struct btrfs_inode_item *inode_item; + struct btrfs_path *path; + u64 objectid; + struct btrfs_inode_ref *ref; + struct btrfs_key key[2]; + u32 sizes[2]; + struct btrfs_item_batch batch; + unsigned long ptr; + int ret; + bool xa_reserved = false; + + if (!args->orphan && !args->subvol) { + /* + * Before anything else, check if we can add the name to the + * parent directory. We want to avoid a dir item overflow in + * case we have an existing dir item due to existing name + * hash collisions. We do this check here before we call + * btrfs_add_link() down below so that we can avoid a + * transaction abort (which could be exploited by malicious + * users). + * + * For subvolumes we already do this in btrfs_mksubvol(). + */ + ret = btrfs_check_dir_item_collision(BTRFS_I(dir)->root, + btrfs_ino(BTRFS_I(dir)), + name); + if (ret < 0) + return ret; + } + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + if (!args->subvol) + BTRFS_I(inode)->root = btrfs_grab_root(BTRFS_I(dir)->root); + root = BTRFS_I(inode)->root; + + ret = btrfs_init_file_extent_tree(BTRFS_I(inode)); + if (ret) + goto out; + + ret = btrfs_get_free_objectid(root, &objectid); + if (ret) + goto out; + btrfs_set_inode_number(BTRFS_I(inode), objectid); + + ret = xa_reserve(&root->inodes, objectid, GFP_NOFS); + if (ret) + goto out; + xa_reserved = true; + + if (args->orphan) { + /* + * O_TMPFILE, set link count to 0, so that after this point, we + * fill in an inode item with the correct link count. + */ + set_nlink(inode, 0); + } else { + trace_btrfs_inode_request(dir); + + ret = btrfs_set_inode_index(BTRFS_I(dir), &BTRFS_I(inode)->dir_index); + if (ret) + goto out; + } + + if (S_ISDIR(inode->i_mode)) + BTRFS_I(inode)->index_cnt = BTRFS_DIR_START_INDEX; + + BTRFS_I(inode)->generation = trans->transid; + inode->i_generation = BTRFS_I(inode)->generation; + + /* + * We don't have any capability xattrs set here yet, shortcut any + * queries for the xattrs here. If we add them later via the inode + * security init path or any other path this flag will be cleared. + */ + set_bit(BTRFS_INODE_NO_CAP_XATTR, &BTRFS_I(inode)->runtime_flags); + + /* + * Subvolumes don't inherit flags from their parent directory. + * Originally this was probably by accident, but we probably can't + * change it now without compatibility issues. + */ + if (!args->subvol) + btrfs_inherit_iflags(BTRFS_I(inode), BTRFS_I(dir)); + + btrfs_set_inode_mapping_order(BTRFS_I(inode)); + if (S_ISREG(inode->i_mode)) { + if (btrfs_test_opt(fs_info, NODATASUM)) + BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM; + if (btrfs_test_opt(fs_info, NODATACOW)) + BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW | + BTRFS_INODE_NODATASUM; + btrfs_update_inode_mapping_flags(BTRFS_I(inode)); + } + + ret = btrfs_insert_inode_locked(inode); + if (ret < 0) { + if (!args->orphan) + BTRFS_I(dir)->index_cnt--; + goto out; + } + + /* + * We could have gotten an inode number from somebody who was fsynced + * and then removed in this same transaction, so let's just set full + * sync since it will be a full sync anyway and this will blow away the + * old info in the log. + */ + btrfs_set_inode_full_sync(BTRFS_I(inode)); + + key[0].objectid = objectid; + key[0].type = BTRFS_INODE_ITEM_KEY; + key[0].offset = 0; + + sizes[0] = sizeof(struct btrfs_inode_item); + + if (!args->orphan) { + /* + * Start new inodes with an inode_ref. This is slightly more + * efficient for small numbers of hard links since they will + * be packed into one item. Extended refs will kick in if we + * add more hard links than can fit in the ref item. + */ + key[1].objectid = objectid; + key[1].type = BTRFS_INODE_REF_KEY; + if (args->subvol) { + key[1].offset = objectid; + sizes[1] = 2 + sizeof(*ref); + } else { + key[1].offset = btrfs_ino(BTRFS_I(dir)); + sizes[1] = name->len + sizeof(*ref); + } + } + + batch.keys = &key[0]; + batch.data_sizes = &sizes[0]; + batch.total_data_size = sizes[0] + (args->orphan ? 0 : sizes[1]); + batch.nr = args->orphan ? 1 : 2; + ret = btrfs_insert_empty_items(trans, root, path, &batch); + if (unlikely(ret != 0)) { + btrfs_abort_transaction(trans, ret); + goto discard; + } + + ts = simple_inode_init_ts(inode); + BTRFS_I(inode)->i_otime_sec = ts.tv_sec; + BTRFS_I(inode)->i_otime_nsec = ts.tv_nsec; + + /* + * We're going to fill the inode item now, so at this point the inode + * must be fully initialized. + */ + + inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_inode_item); + memzero_extent_buffer(path->nodes[0], (unsigned long)inode_item, + sizeof(*inode_item)); + fill_inode_item(trans, path->nodes[0], inode_item, inode); + + if (!args->orphan) { + ref = btrfs_item_ptr(path->nodes[0], path->slots[0] + 1, + struct btrfs_inode_ref); + ptr = (unsigned long)(ref + 1); + if (args->subvol) { + btrfs_set_inode_ref_name_len(path->nodes[0], ref, 2); + btrfs_set_inode_ref_index(path->nodes[0], ref, 0); + write_extent_buffer(path->nodes[0], "..", ptr, 2); + } else { + btrfs_set_inode_ref_name_len(path->nodes[0], ref, + name->len); + btrfs_set_inode_ref_index(path->nodes[0], ref, + BTRFS_I(inode)->dir_index); + write_extent_buffer(path->nodes[0], name->name, ptr, + name->len); + } + } + + /* + * We don't need the path anymore, plus inheriting properties, adding + * ACLs, security xattrs, orphan item or adding the link, will result in + * allocating yet another path. So just free our path. + */ + btrfs_free_path(path); + path = NULL; + + if (args->subvol) { + struct btrfs_inode *parent; + + /* + * Subvolumes inherit properties from their parent subvolume, + * not the directory they were created in. + */ + parent = btrfs_iget(BTRFS_FIRST_FREE_OBJECTID, BTRFS_I(dir)->root); + if (IS_ERR(parent)) { + ret = PTR_ERR(parent); + } else { + ret = btrfs_inode_inherit_props(trans, BTRFS_I(inode), + parent); + iput(&parent->vfs_inode); + } + } else { + ret = btrfs_inode_inherit_props(trans, BTRFS_I(inode), + BTRFS_I(dir)); + } + if (ret) { + btrfs_err(fs_info, + "error inheriting props for ino %llu (root %llu): %pe", + btrfs_ino(BTRFS_I(inode)), btrfs_root_id(root), ERR_PTR(ret)); + } + + /* + * Subvolumes don't inherit ACLs or get passed to the LSM. This is + * probably a bug. + */ + if (!args->subvol) { + ret = btrfs_init_inode_security(trans, args); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto discard; + } + } + + ret = btrfs_add_inode_to_root(BTRFS_I(inode), false); + if (WARN_ON(ret)) { + /* Shouldn't happen, we used xa_reserve() before. */ + btrfs_abort_transaction(trans, ret); + goto discard; + } + + trace_btrfs_inode_new(inode); + btrfs_set_inode_last_trans(trans, BTRFS_I(inode)); + + btrfs_update_root_times(trans, root); + + if (args->orphan) { + ret = btrfs_orphan_add(trans, BTRFS_I(inode)); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto discard; + } + } else { + ret = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode), name, + false, BTRFS_I(inode)->dir_index); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto discard; + } + } + + return 0; + +discard: + /* + * discard_new_inode() calls iput(), but the caller owns the reference + * to the inode. + */ + ihold(inode); + discard_new_inode(inode); +out: + if (xa_reserved) + xa_release(&root->inodes, objectid); + + btrfs_free_path(path); + return ret; +} + +/* + * utility function to add 'inode' into 'parent_inode' with + * a give name and a given sequence number. + * if 'add_backref' is true, also insert a backref from the + * inode to the parent directory. + */ +int btrfs_add_link(struct btrfs_trans_handle *trans, + struct btrfs_inode *parent_inode, struct btrfs_inode *inode, + const struct fscrypt_str *name, bool add_backref, u64 index) +{ + int ret = 0; + struct btrfs_key key; + struct btrfs_root *root = parent_inode->root; + u64 ino = btrfs_ino(inode); + u64 parent_ino = btrfs_ino(parent_inode); + + if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { + memcpy(&key, &inode->root->root_key, sizeof(key)); + } else { + key.objectid = ino; + key.type = BTRFS_INODE_ITEM_KEY; + key.offset = 0; + } + + if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { + ret = btrfs_add_root_ref(trans, key.objectid, + btrfs_root_id(root), parent_ino, + index, name); + } else if (add_backref) { + ret = btrfs_insert_inode_ref(trans, root, name, + ino, parent_ino, index); + } + + /* Nothing to clean up yet */ + if (ret) + return ret; + + ret = btrfs_insert_dir_item(trans, name, parent_inode, &key, + btrfs_inode_type(inode), index); + if (ret == -EEXIST || ret == -EOVERFLOW) + goto fail_dir_item; + else if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + + btrfs_i_size_write(parent_inode, parent_inode->vfs_inode.i_size + + name->len * 2); + inode_inc_iversion(&parent_inode->vfs_inode); + update_time_after_link_or_unlink(parent_inode); + + ret = btrfs_update_inode(trans, parent_inode); + if (ret) + btrfs_abort_transaction(trans, ret); + return ret; + +fail_dir_item: + if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) { + u64 local_index; + int ret2; + + ret2 = btrfs_del_root_ref(trans, key.objectid, btrfs_root_id(root), + parent_ino, &local_index, name); + if (ret2) + btrfs_abort_transaction(trans, ret2); + } else if (add_backref) { + int ret2; + + ret2 = btrfs_del_inode_ref(trans, root, name, ino, parent_ino, NULL); + if (ret2) + btrfs_abort_transaction(trans, ret2); + } + + /* Return the original error code */ + return ret; +} + +static int btrfs_create_common(struct inode *dir, struct dentry *dentry, + struct inode *inode) +{ + struct btrfs_fs_info *fs_info = inode_to_fs_info(dir); + struct btrfs_root *root = BTRFS_I(dir)->root; + struct btrfs_new_inode_args new_inode_args = { + .dir = dir, + .dentry = dentry, + .inode = inode, + }; + unsigned int trans_num_items; + struct btrfs_trans_handle *trans; + int ret; + + ret = btrfs_new_inode_prepare(&new_inode_args, &trans_num_items); + if (ret) + goto out_inode; + + trans = btrfs_start_transaction(root, trans_num_items); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto out_new_inode_args; + } + + ret = btrfs_create_new_inode(trans, &new_inode_args); + if (!ret) { + if (S_ISDIR(inode->i_mode)) + inode->i_opflags |= IOP_FASTPERM_MAY_EXEC; + d_instantiate_new(dentry, inode); + } + + btrfs_end_transaction(trans); + btrfs_btree_balance_dirty(fs_info); +out_new_inode_args: + btrfs_new_inode_args_destroy(&new_inode_args); +out_inode: + if (ret) + iput(inode); + return ret; +} + +static int btrfs_mknod(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) +{ + struct inode *inode; + + inode = new_inode(dir->i_sb); + if (!inode) + return -ENOMEM; + inode_init_owner(idmap, inode, dir, mode); + inode->i_op = &btrfs_special_inode_operations; + init_special_inode(inode, inode->i_mode, rdev); + return btrfs_create_common(dir, dentry, inode); +} + +static int btrfs_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct inode *inode; + + inode = new_inode(dir->i_sb); + if (!inode) + return -ENOMEM; + inode_init_owner(idmap, inode, dir, mode); + inode->i_fop = &btrfs_file_operations; + inode->i_op = &btrfs_file_inode_operations; + inode->i_mapping->a_ops = &btrfs_aops; + return btrfs_create_common(dir, dentry, inode); +} + +static int btrfs_link(struct dentry *old_dentry, struct inode *dir, + struct dentry *dentry) +{ + struct btrfs_trans_handle *trans = NULL; + struct btrfs_root *root = BTRFS_I(dir)->root; + struct inode *inode = d_inode(old_dentry); + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + struct fscrypt_name fname; + u64 index; + int ret; + + /* do not allow sys_link's with other subvols of the same device */ + if (btrfs_root_id(root) != btrfs_root_id(BTRFS_I(inode)->root)) + return -EXDEV; + + if (inode->i_nlink >= BTRFS_LINK_MAX) + return -EMLINK; + + ret = fscrypt_setup_filename(dir, &dentry->d_name, 0, &fname); + if (ret) + goto fail; + + ret = btrfs_set_inode_index(BTRFS_I(dir), &index); + if (ret) + goto fail; + + /* + * 2 items for inode and inode ref + * 2 items for dir items + * 1 item for parent inode + * 1 item for orphan item deletion if O_TMPFILE + */ + trans = btrfs_start_transaction(root, inode->i_nlink ? 5 : 6); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + trans = NULL; + goto fail; + } + + /* There are several dir indexes for this inode, clear the cache. */ + BTRFS_I(inode)->dir_index = 0ULL; + inode_inc_iversion(inode); + inode_set_ctime_current(inode); + + ret = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode), + &fname.disk_name, true, index); + if (ret) + goto fail; + + /* Link added now we update the inode item with the new link count. */ + inc_nlink(inode); + ret = btrfs_update_inode(trans, BTRFS_I(inode)); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto fail; + } + + if (inode->i_nlink == 1) { + /* + * If the new hard link count is 1, it's a file created with the + * open(2) O_TMPFILE flag. + */ + ret = btrfs_orphan_del(trans, BTRFS_I(inode)); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto fail; + } + } + + /* Grab reference for the new dentry passed to d_instantiate(). */ + ihold(inode); + d_instantiate(dentry, inode); + btrfs_log_new_name(trans, old_dentry, NULL, 0, dentry->d_parent); + +fail: + fscrypt_free_filename(&fname); + if (trans) + btrfs_end_transaction(trans); + btrfs_btree_balance_dirty(fs_info); + return ret; +} + +static struct dentry *btrfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct inode *inode; + + inode = new_inode(dir->i_sb); + if (!inode) + return ERR_PTR(-ENOMEM); + inode_init_owner(idmap, inode, dir, mode); + inode->i_op = &btrfs_dir_inode_operations; + inode->i_fop = &btrfs_dir_file_operations; + return ERR_PTR(btrfs_create_common(dir, dentry, inode)); +} + +static noinline int uncompress_inline(struct btrfs_path *path, + struct folio *folio, + struct btrfs_file_extent_item *item) +{ + int ret; + struct extent_buffer *leaf = path->nodes[0]; + const u32 blocksize = leaf->fs_info->sectorsize; + char *tmp; + size_t max_size; + unsigned long inline_size; + unsigned long ptr; + int compress_type; + + compress_type = btrfs_file_extent_compression(leaf, item); + max_size = btrfs_file_extent_ram_bytes(leaf, item); + inline_size = btrfs_file_extent_inline_item_len(leaf, path->slots[0]); + tmp = kmalloc(inline_size, GFP_NOFS); + if (!tmp) + return -ENOMEM; + ptr = btrfs_file_extent_inline_start(item); + + read_extent_buffer(leaf, tmp, ptr, inline_size); + + max_size = min_t(unsigned long, blocksize, max_size); + ret = btrfs_decompress(compress_type, tmp, folio, 0, inline_size, + max_size); + + /* + * decompression code contains a memset to fill in any space between the end + * of the uncompressed data and the end of max_size in case the decompressed + * data ends up shorter than ram_bytes. That doesn't cover the hole between + * the end of an inline extent and the beginning of the next block, so we + * cover that region here. + */ + + if (max_size < blocksize) + folio_zero_range(folio, max_size, blocksize - max_size); + kfree(tmp); + return ret; +} + +static int read_inline_extent(struct btrfs_path *path, struct folio *folio) +{ + const u32 blocksize = path->nodes[0]->fs_info->sectorsize; + struct btrfs_file_extent_item *fi; + void *kaddr; + size_t copy_size; + + if (!folio || folio_test_uptodate(folio)) + return 0; + + ASSERT(folio_pos(folio) == 0); + + fi = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_file_extent_item); + if (btrfs_file_extent_compression(path->nodes[0], fi) != BTRFS_COMPRESS_NONE) + return uncompress_inline(path, folio, fi); + + copy_size = min_t(u64, blocksize, + btrfs_file_extent_ram_bytes(path->nodes[0], fi)); + kaddr = kmap_local_folio(folio, 0); + read_extent_buffer(path->nodes[0], kaddr, + btrfs_file_extent_inline_start(fi), copy_size); + kunmap_local(kaddr); + if (copy_size < blocksize) + folio_zero_range(folio, copy_size, blocksize - copy_size); + return 0; +} + +/* + * Lookup the first extent overlapping a range in a file. + * + * @inode: file to search in + * @page: page to read extent data into if the extent is inline + * @start: file offset + * @len: length of range starting at @start + * + * Return the first &struct extent_map which overlaps the given range, reading + * it from the B-tree and caching it if necessary. Note that there may be more + * extents which overlap the given range after the returned extent_map. + * + * If @page is not NULL and the extent is inline, this also reads the extent + * data directly into the page and marks the extent up to date in the io_tree. + * + * Return: ERR_PTR on error, non-NULL extent_map on success. + */ +struct extent_map *btrfs_get_extent(struct btrfs_inode *inode, + struct folio *folio, u64 start, u64 len) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + int ret = 0; + u64 extent_start = 0; + u64 extent_end = 0; + u64 objectid = btrfs_ino(inode); + int extent_type = -1; + struct btrfs_path *path = NULL; + struct btrfs_root *root = inode->root; + struct btrfs_file_extent_item *item; + struct extent_buffer *leaf; + struct btrfs_key found_key; + struct extent_map *em = NULL; + struct extent_map_tree *em_tree = &inode->extent_tree; + + read_lock(&em_tree->lock); + em = btrfs_lookup_extent_mapping(em_tree, start, len); + read_unlock(&em_tree->lock); + + if (em) { + if (em->start > start || btrfs_extent_map_end(em) <= start) + btrfs_free_extent_map(em); + else if (em->disk_bytenr == EXTENT_MAP_INLINE && folio) + btrfs_free_extent_map(em); + else + goto out; + } + em = btrfs_alloc_extent_map(); + if (!em) { + ret = -ENOMEM; + goto out; + } + em->start = EXTENT_MAP_HOLE; + em->disk_bytenr = EXTENT_MAP_HOLE; + em->len = (u64)-1; + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto out; + } + + /* Chances are we'll be called again, so go ahead and do readahead */ + path->reada = READA_FORWARD; + + /* + * The same explanation in load_free_space_cache applies here as well, + * we only read when we're loading the free space cache, and at that + * point the commit_root has everything we need. + */ + if (btrfs_is_free_space_inode(inode)) { + path->search_commit_root = true; + path->skip_locking = true; + } + + ret = btrfs_lookup_file_extent(NULL, root, path, objectid, start, 0); + if (ret < 0) { + goto out; + } else if (ret > 0) { + if (path->slots[0] == 0) + goto not_found; + path->slots[0]--; + ret = 0; + } + + leaf = path->nodes[0]; + item = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_file_extent_item); + btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); + if (found_key.objectid != objectid || + found_key.type != BTRFS_EXTENT_DATA_KEY) { + /* + * If we backup past the first extent we want to move forward + * and see if there is an extent in front of us, otherwise we'll + * say there is a hole for our whole search range which can + * cause problems. + */ + extent_end = start; + goto next; + } + + extent_type = btrfs_file_extent_type(leaf, item); + extent_start = found_key.offset; + extent_end = btrfs_file_extent_end(path); + if (extent_type == BTRFS_FILE_EXTENT_REG || + extent_type == BTRFS_FILE_EXTENT_PREALLOC) { + /* Only regular file could have regular/prealloc extent */ + if (unlikely(!S_ISREG(inode->vfs_inode.i_mode))) { + ret = -EUCLEAN; + btrfs_crit(fs_info, + "regular/prealloc extent found for non-regular inode %llu", + btrfs_ino(inode)); + goto out; + } + trace_btrfs_get_extent_show_fi_regular(inode, leaf, item, + extent_start); + } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { + trace_btrfs_get_extent_show_fi_inline(inode, leaf, item, + path->slots[0], + extent_start); + } +next: + if (start >= extent_end) { + path->slots[0]++; + if (path->slots[0] >= btrfs_header_nritems(leaf)) { + ret = btrfs_next_leaf(root, path); + if (ret < 0) + goto out; + else if (ret > 0) + goto not_found; + + leaf = path->nodes[0]; + } + btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); + if (found_key.objectid != objectid || + found_key.type != BTRFS_EXTENT_DATA_KEY) + goto not_found; + if (start + len <= found_key.offset) + goto not_found; + if (start > found_key.offset) + goto next; + + /* New extent overlaps with existing one */ + em->start = start; + em->len = found_key.offset - start; + em->disk_bytenr = EXTENT_MAP_HOLE; + goto insert; + } + + btrfs_extent_item_to_extent_map(inode, path, item, em); + + if (extent_type == BTRFS_FILE_EXTENT_REG || + extent_type == BTRFS_FILE_EXTENT_PREALLOC) { + goto insert; + } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) { + /* + * Inline extent can only exist at file offset 0. This is + * ensured by tree-checker and inline extent creation path. + * Thus all members representing file offsets should be zero. + */ + ASSERT(extent_start == 0); + ASSERT(em->start == 0); + + /* + * btrfs_extent_item_to_extent_map() should have properly + * initialized em members already. + * + * Other members are not utilized for inline extents. + */ + ASSERT(em->disk_bytenr == EXTENT_MAP_INLINE); + ASSERT(em->len == fs_info->sectorsize); + + ret = read_inline_extent(path, folio); + if (ret < 0) + goto out; + goto insert; + } +not_found: + em->start = start; + em->len = len; + em->disk_bytenr = EXTENT_MAP_HOLE; +insert: + ret = 0; + btrfs_release_path(path); + if (unlikely(em->start > start || btrfs_extent_map_end(em) <= start)) { + btrfs_err(fs_info, + "bad extent! em: [%llu %llu] passed [%llu %llu]", + em->start, em->len, start, len); + ret = -EIO; + goto out; + } + + write_lock(&em_tree->lock); + ret = btrfs_add_extent_mapping(inode, &em, start, len); + write_unlock(&em_tree->lock); +out: + btrfs_free_path(path); + + trace_btrfs_get_extent(root, inode, em); + + if (ret) { + btrfs_free_extent_map(em); + return ERR_PTR(ret); + } + return em; +} + +static bool btrfs_extent_readonly(struct btrfs_fs_info *fs_info, u64 bytenr) +{ + struct btrfs_block_group *block_group; + bool readonly = false; + + block_group = btrfs_lookup_block_group(fs_info, bytenr); + if (!block_group || block_group->ro) + readonly = true; + if (block_group) + btrfs_put_block_group(block_group); + return readonly; +} + +/* + * Check if we can do nocow write into the range [@offset, @offset + @len) + * + * @offset: File offset + * @len: The length to write, will be updated to the nocow writeable + * range + * @orig_start: (optional) Return the original file offset of the file extent + * @orig_len: (optional) Return the original on-disk length of the file extent + * @ram_bytes: (optional) Return the ram_bytes of the file extent + * + * Return: + * >0 and update @len if we can do nocow write + * 0 if we can't do nocow write + * <0 if error happened + * + * NOTE: This only checks the file extents, caller is responsible to wait for + * any ordered extents. + */ +noinline int can_nocow_extent(struct btrfs_inode *inode, u64 offset, u64 *len, + struct btrfs_file_extent *file_extent, + bool nowait) +{ + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct can_nocow_file_extent_args nocow_args = { 0 }; + BTRFS_PATH_AUTO_FREE(path); + int ret; + struct extent_buffer *leaf; + struct extent_io_tree *io_tree = &inode->io_tree; + struct btrfs_file_extent_item *fi; + struct btrfs_key key; + int found_type; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + path->nowait = nowait; + + ret = btrfs_lookup_file_extent(NULL, root, path, btrfs_ino(inode), + offset, 0); + if (ret < 0) + return ret; + + if (ret == 1) { + if (path->slots[0] == 0) { + /* Can't find the item, must COW. */ + return 0; + } + path->slots[0]--; + } + ret = 0; + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + if (key.objectid != btrfs_ino(inode) || + key.type != BTRFS_EXTENT_DATA_KEY) { + /* Not our file or wrong item type, must COW. */ + return 0; + } + + if (key.offset > offset) { + /* Wrong offset, must COW. */ + return 0; + } + + if (btrfs_file_extent_end(path) <= offset) + return 0; + + fi = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); + found_type = btrfs_file_extent_type(leaf, fi); + + nocow_args.start = offset; + nocow_args.end = offset + *len - 1; + nocow_args.free_path = true; + + ret = can_nocow_file_extent(path, &key, inode, &nocow_args); + /* can_nocow_file_extent() has freed the path. */ + path = NULL; + + if (ret != 1) { + /* Treat errors as not being able to NOCOW. */ + return 0; + } + + if (btrfs_extent_readonly(fs_info, + nocow_args.file_extent.disk_bytenr + + nocow_args.file_extent.offset)) + return 0; + + if (!(inode->flags & BTRFS_INODE_NODATACOW) && + found_type == BTRFS_FILE_EXTENT_PREALLOC) { + u64 range_end; + + range_end = round_up(offset + nocow_args.file_extent.num_bytes, + root->fs_info->sectorsize) - 1; + ret = btrfs_test_range_bit_exists(io_tree, offset, range_end, + EXTENT_DELALLOC); + if (ret) + return -EAGAIN; + } + + if (file_extent) + memcpy(file_extent, &nocow_args.file_extent, sizeof(*file_extent)); + + *len = nocow_args.file_extent.num_bytes; + + return 1; +} + +/* The callers of this must take lock_extent() */ +struct extent_map *btrfs_create_io_em(struct btrfs_inode *inode, u64 start, + const struct btrfs_file_extent *file_extent, + int type) +{ + struct extent_map *em; + int ret; + + /* + * Note the missing NOCOW type. + * + * For pure NOCOW writes, we should not create an io extent map, but + * just reusing the existing one. + * Only PREALLOC writes (NOCOW write into preallocated range) can + * create an io extent map. + */ + ASSERT(type == BTRFS_ORDERED_PREALLOC || + type == BTRFS_ORDERED_COMPRESSED || + type == BTRFS_ORDERED_REGULAR); + + switch (type) { + case BTRFS_ORDERED_PREALLOC: + /* We're only referring part of a larger preallocated extent. */ + ASSERT(file_extent->num_bytes <= file_extent->ram_bytes); + break; + case BTRFS_ORDERED_REGULAR: + /* COW results a new extent matching our file extent size. */ + ASSERT(file_extent->disk_num_bytes == file_extent->num_bytes); + ASSERT(file_extent->ram_bytes == file_extent->num_bytes); + + /* Since it's a new extent, we should not have any offset. */ + ASSERT(file_extent->offset == 0); + break; + case BTRFS_ORDERED_COMPRESSED: + /* Must be compressed. */ + ASSERT(file_extent->compression != BTRFS_COMPRESS_NONE); + + /* + * Encoded write can make us to refer to part of the + * uncompressed extent. + */ + ASSERT(file_extent->num_bytes <= file_extent->ram_bytes); + break; + } + + em = btrfs_alloc_extent_map(); + if (!em) + return ERR_PTR(-ENOMEM); + + em->start = start; + em->len = file_extent->num_bytes; + em->disk_bytenr = file_extent->disk_bytenr; + em->disk_num_bytes = file_extent->disk_num_bytes; + em->ram_bytes = file_extent->ram_bytes; + em->generation = -1; + em->offset = file_extent->offset; + em->flags |= EXTENT_FLAG_PINNED; + if (type == BTRFS_ORDERED_COMPRESSED) + btrfs_extent_map_set_compression(em, file_extent->compression); + + ret = btrfs_replace_extent_map_range(inode, em, true); + if (ret) { + btrfs_free_extent_map(em); + return ERR_PTR(ret); + } + + /* em got 2 refs now, callers needs to do btrfs_free_extent_map once. */ + return em; +} + +/* + * For release_folio() and invalidate_folio() we have a race window where + * folio_end_writeback() is called but the subpage spinlock is not yet released. + * If we continue to release/invalidate the page, we could cause use-after-free + * for subpage spinlock. So this function is to spin and wait for subpage + * spinlock. + */ +static void wait_subpage_spinlock(struct folio *folio) +{ + struct btrfs_fs_info *fs_info = folio_to_fs_info(folio); + struct btrfs_folio_state *bfs; + + if (!btrfs_is_subpage(fs_info, folio)) + return; + + ASSERT(folio_test_private(folio) && folio_get_private(folio)); + bfs = folio_get_private(folio); + + /* + * This may look insane as we just acquire the spinlock and release it, + * without doing anything. But we just want to make sure no one is + * still holding the subpage spinlock. + * And since the page is not dirty nor writeback, and we have page + * locked, the only possible way to hold a spinlock is from the endio + * function to clear page writeback. + * + * Here we just acquire the spinlock so that all existing callers + * should exit and we're safe to release/invalidate the page. + */ + spin_lock_irq(&bfs->lock); + spin_unlock_irq(&bfs->lock); +} + +static int btrfs_launder_folio(struct folio *folio) +{ + return btrfs_qgroup_free_data(folio_to_inode(folio), NULL, folio_pos(folio), + folio_size(folio), NULL); +} + +static bool __btrfs_release_folio(struct folio *folio, gfp_t gfp_flags) +{ + if (try_release_extent_mapping(folio, gfp_flags)) { + wait_subpage_spinlock(folio); + clear_folio_extent_mapped(folio); + return true; + } + return false; +} + +static bool btrfs_release_folio(struct folio *folio, gfp_t gfp_flags) +{ + if (folio_test_writeback(folio) || folio_test_dirty(folio)) + return false; + return __btrfs_release_folio(folio, gfp_flags); +} + +#ifdef CONFIG_MIGRATION +static int btrfs_migrate_folio(struct address_space *mapping, + struct folio *dst, struct folio *src, + enum migrate_mode mode) +{ + int ret = filemap_migrate_folio(mapping, dst, src, mode); + + if (ret) + return ret; + return 0; +} +#else +#define btrfs_migrate_folio NULL +#endif + +static void btrfs_invalidate_folio(struct folio *folio, size_t offset, + size_t length) +{ + struct btrfs_inode *inode = folio_to_inode(folio); + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct extent_io_tree *tree = &inode->io_tree; + struct extent_state *cached_state = NULL; + u64 page_start = folio_pos(folio); + u64 page_end = page_start + folio_size(folio) - 1; + u64 cur; + int inode_evicting = inode_state_read_once(&inode->vfs_inode) & I_FREEING; + + /* + * We have folio locked so no new ordered extent can be created on this + * page, nor bio can be submitted for this folio. + * + * But already submitted bio can still be finished on this folio. + * Furthermore, endio function won't skip folio which has Ordered + * already cleared, so it's possible for endio and + * invalidate_folio to do the same ordered extent accounting twice + * on one folio. + * + * So here we wait for any submitted bios to finish, so that we won't + * do double ordered extent accounting on the same folio. + */ + folio_wait_writeback(folio); + wait_subpage_spinlock(folio); + + /* + * The invalidated blocks are going away; drop any fixup blocks among + * them, data included, as they have no space reservation. + */ + btrfs_folio_clear_fixup_dirty(fs_info, folio, page_start + offset, length); + + /* + * For subpage case, we have call sites like + * btrfs_punch_hole_lock_range() which passes range not aligned to + * sectorsize. + * If the range doesn't cover the full folio, we don't need to and + * shouldn't clear page extent mapped, as folio->private can still + * record subpage dirty bits for other part of the range. + * + * For cases that invalidate the full folio even the range doesn't + * cover the full folio, like invalidating the last folio, we're + * still safe to wait for ordered extent to finish. + */ + if (!(offset == 0 && length == folio_size(folio))) { + btrfs_release_folio(folio, GFP_NOFS); + return; + } + + if (!inode_evicting) + btrfs_lock_extent(tree, page_start, page_end, &cached_state); + + cur = page_start; + while (cur < page_end) { + struct btrfs_ordered_extent *ordered; + u64 range_end; + u32 range_len; + u32 extra_flags = 0; + + ordered = btrfs_lookup_first_ordered_range(inode, cur, + page_end + 1 - cur); + if (!ordered) { + range_end = page_end; + /* + * No ordered extent covering this range, we are safe + * to delete all extent states in the range. + */ + extra_flags = EXTENT_CLEAR_ALL_BITS; + goto next; + } + if (ordered->file_offset > cur) { + /* + * There is a range between [cur, oe->file_offset) not + * covered by any ordered extent. + * We are safe to delete all extent states, and handle + * the ordered extent in the next iteration. + */ + range_end = ordered->file_offset - 1; + extra_flags = EXTENT_CLEAR_ALL_BITS; + goto next; + } + + range_end = min(ordered->file_offset + ordered->num_bytes - 1, + page_end); + ASSERT(range_end + 1 - cur < U32_MAX); + range_len = range_end + 1 - cur; + /* + * If the range is not dirty, the range has been submitted and + * since we have waited for the writeback, endio has been + * executed, thus we must skip the range to avoid double + * accounting for the ordered extent. + */ + if (!btrfs_folio_test_dirty(fs_info, folio, cur, range_len)) + goto next; + + /* + * The range is dirty meaning it has not been submitted. + * Here we need to truncate the OE range as the range will never + * be submitted. + * + * IO on this page will never be started, so we need to account + * for any ordered extents now. Don't clear EXTENT_DELALLOC_NEW + * here, must leave that up for the ordered extent completion. + * + * This will also unlock the range for incoming + * btrfs_finish_ordered_io(). + */ + if (!inode_evicting) + btrfs_clear_extent_bit(tree, cur, range_end, + EXTENT_DELALLOC | + EXTENT_LOCKED | EXTENT_DO_ACCOUNTING | + EXTENT_DEFRAG, &cached_state); + + btrfs_mark_ordered_extent_truncated(ordered, cur - ordered->file_offset); + + /* + * If the ordered extent has finished, we're safe to delete all + * the extent states of the range, otherwise + * btrfs_finish_ordered_io() will get executed by endio for + * other pages, so we can't delete extent states. + */ + if (btrfs_dec_test_ordered_pending(inode, &ordered, + cur, range_end + 1 - cur)) { + btrfs_finish_ordered_io(ordered); + /* + * The ordered extent has finished, now we're again + * safe to delete all extent states of the range. + */ + extra_flags = EXTENT_CLEAR_ALL_BITS; + } +next: + if (ordered) + btrfs_put_ordered_extent(ordered); + /* + * Qgroup reserved space handler + * Sector(s) here will be either: + * + * 1) Already written to disk or bio already finished + * Then its QGROUP_RESERVED bit in io_tree is already cleared. + * Qgroup will be handled by its qgroup_record then. + * btrfs_qgroup_free_data() call will do nothing here. + * + * 2) Not written to disk yet + * Then btrfs_qgroup_free_data() call will clear the + * QGROUP_RESERVED bit of its io_tree, and free the qgroup + * reserved data space. + * Since the IO will never happen for this page. + */ + btrfs_qgroup_free_data(inode, NULL, cur, range_end + 1 - cur, NULL); + if (!inode_evicting) + btrfs_clear_extent_bit(tree, cur, range_end, EXTENT_LOCKED | + EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | + EXTENT_DEFRAG | extra_flags, + &cached_state); + cur = range_end + 1; + } + btrfs_folio_clear_dirty(fs_info, folio, page_start, folio_size(folio)); + btrfs_clear_folio_dirty_tag(folio); + if (!inode_evicting) + __btrfs_release_folio(folio, GFP_NOFS); + clear_folio_extent_mapped(folio); +} + +static int btrfs_truncate(struct btrfs_inode *inode, bool skip_writeback) +{ + struct btrfs_truncate_control control = { + .inode = inode, + .ino = btrfs_ino(inode), + .min_type = BTRFS_EXTENT_DATA_KEY, + .clear_extent_range = true, + .new_size = inode->vfs_inode.i_size, + }; + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_block_rsv rsv; + int ret; + struct btrfs_trans_handle *trans; + const u64 min_size = btrfs_calc_metadata_size(fs_info, 1); + const u64 lock_start = round_down(inode->vfs_inode.i_size, fs_info->sectorsize); + const u64 i_size_up = round_up(inode->vfs_inode.i_size, fs_info->sectorsize); + + /* Our inode is locked and the i_size can't be changed concurrently. */ + btrfs_assert_inode_locked(inode); + + if (!skip_writeback) { + ret = btrfs_wait_ordered_range(inode, lock_start, (u64)-1); + if (ret) + return ret; + } + + /* + * Yes ladies and gentlemen, this is indeed ugly. We have a couple of + * things going on here: + * + * 1) We need to reserve space to update our inode. + * + * 2) We need to have something to cache all the space that is going to + * be free'd up by the truncate operation, but also have some slack + * space reserved in case it uses space during the truncate (thank you + * very much snapshotting). + * + * And we need these to be separate. The fact is we can use a lot of + * space doing the truncate, and we have no earthly idea how much space + * we will use, so we need the truncate reservation to be separate so it + * doesn't end up using space reserved for updating the inode. We also + * need to be able to stop the transaction and start a new one, which + * means we need to be able to update the inode several times, and we + * have no idea of knowing how many times that will be, so we can't just + * reserve 1 item for the entirety of the operation, so that has to be + * done separately as well. + * + * So that leaves us with + * + * 1) rsv - for the truncate reservation, which we will steal from the + * transaction reservation. + * 2) fs_info->trans_block_rsv - this will have 1 items worth left for + * updating the inode. + */ + btrfs_init_metadata_block_rsv(fs_info, &rsv, BTRFS_BLOCK_RSV_TEMP); + rsv.size = min_size; + rsv.failfast = true; + + /* + * 1 for the truncate slack space + * 1 for updating the inode. + */ + trans = btrfs_start_transaction(root, 2); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto out; + } + + /* Migrate the slack space for the truncate to our reserve */ + ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, &rsv, + min_size, false); + /* + * We have reserved 2 metadata units when we started the transaction and + * min_size matches 1 unit, so this should never fail, but if it does, + * it's not critical we just fail truncation. + */ + if (WARN_ON(ret)) { + btrfs_end_transaction(trans); + goto out; + } + + trans->block_rsv = &rsv; + + while (1) { + struct extent_state *cached_state = NULL; + + btrfs_lock_extent(&inode->io_tree, lock_start, (u64)-1, &cached_state); + /* + * We want to drop from the next block forward in case this new + * size is not block aligned since we will be keeping the last + * block of the extent just the way it is. + */ + btrfs_drop_extent_map_range(inode, i_size_up, (u64)-1, false); + + ret = btrfs_truncate_inode_items(trans, root, &control); + + inode_sub_bytes(&inode->vfs_inode, control.sub_bytes); + btrfs_inode_safe_disk_i_size_write(inode, control.last_size); + + btrfs_unlock_extent(&inode->io_tree, lock_start, (u64)-1, &cached_state); + + trans->block_rsv = &fs_info->trans_block_rsv; + if (ret != -ENOSPC && ret != -EAGAIN) + break; + + ret = btrfs_update_inode(trans, inode); + if (ret) + break; + + btrfs_end_transaction(trans); + btrfs_btree_balance_dirty(fs_info); + + trans = btrfs_start_transaction(root, 2); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + trans = NULL; + break; + } + + btrfs_block_rsv_release(fs_info, &rsv, -1, NULL); + ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, + &rsv, min_size, false); + /* + * We have reserved 2 metadata units when we started the + * transaction and min_size matches 1 unit, so this should never + * fail, but if it does, it's not critical we just fail truncation. + */ + if (WARN_ON(ret)) + break; + + trans->block_rsv = &rsv; + } + + /* + * We can't call btrfs_truncate_block inside a trans handle as we could + * deadlock with freeze, if we got BTRFS_NEED_TRUNCATE_BLOCK then we + * know we've truncated everything except the last little bit, and can + * do btrfs_truncate_block and then update the disk_i_size. + */ + if (ret == BTRFS_NEED_TRUNCATE_BLOCK) { + btrfs_end_transaction(trans); + btrfs_btree_balance_dirty(fs_info); + + ret = btrfs_truncate_block(inode, inode->vfs_inode.i_size, + inode->vfs_inode.i_size, (u64)-1); + if (ret) + goto out; + trans = btrfs_start_transaction(root, 1); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto out; + } + btrfs_inode_safe_disk_i_size_write(inode, 0); + } + + if (trans) { + int ret2; + + trans->block_rsv = &fs_info->trans_block_rsv; + ret2 = btrfs_update_inode(trans, inode); + if (ret2 && !ret) + ret = ret2; + + ret2 = btrfs_end_transaction(trans); + if (ret2 && !ret) + ret = ret2; + btrfs_btree_balance_dirty(fs_info); + } +out: + btrfs_block_rsv_release(fs_info, &rsv, (u64)-1, NULL); + /* + * So if we truncate and then write and fsync we normally would just + * write the extents that changed, which is a problem if we need to + * first truncate that entire inode. So set this flag so we write out + * all of the extents in the inode to the sync log so we're completely + * safe. + * + * If no extents were dropped or trimmed we don't need to force the next + * fsync to truncate all the inode's items from the log and re-log them + * all. This means the truncate operation did not change the file size, + * or changed it to a smaller size but there was only an implicit hole + * between the old i_size and the new i_size, and there were no prealloc + * extents beyond i_size to drop. + */ + if (control.extents_found > 0) + btrfs_set_inode_full_sync(inode); + + return ret; +} + +struct inode *btrfs_new_subvol_inode(struct mnt_idmap *idmap, + struct inode *dir) +{ + struct inode *inode; + + inode = new_inode(dir->i_sb); + if (inode) { + /* + * Subvolumes don't inherit the sgid bit or the parent's gid if + * the parent's sgid bit is set. This is probably a bug. + */ + inode_init_owner(idmap, inode, NULL, + S_IFDIR | (~current_umask() & S_IRWXUGO)); + inode->i_op = &btrfs_dir_inode_operations; + inode->i_fop = &btrfs_dir_file_operations; + } + return inode; +} + +struct inode *btrfs_alloc_inode(struct super_block *sb) +{ + struct btrfs_fs_info *fs_info = btrfs_sb(sb); + struct btrfs_inode *ei; + struct inode *inode; + + ei = alloc_inode_sb(sb, btrfs_inode_cachep, GFP_KERNEL); + if (!ei) + return NULL; + + ei->root = NULL; + ei->generation = 0; + ei->last_trans = 0; + ei->last_sub_trans = 0; + ei->logged_trans = 0; + ei->delalloc_bytes = 0; + /* new_delalloc_bytes and last_dir_index_offset are in a union. */ + ei->new_delalloc_bytes = 0; + ei->defrag_bytes = 0; + ei->disk_i_size = 0; + ei->flags = 0; + ei->ro_flags = 0; + /* + * ->index_cnt will be properly initialized later when creating a new + * inode (btrfs_create_new_inode()) or when reading an existing inode + * from disk (btrfs_read_locked_inode()). + */ + ei->csum_bytes = 0; + ei->dir_index = 0; + ei->last_unlink_trans = 0; + ei->last_reflink_trans = 0; + ei->last_log_commit = 0; + + spin_lock_init(&ei->lock); + ei->outstanding_extents = 0; + if (sb->s_magic != BTRFS_TEST_MAGIC) + btrfs_init_metadata_block_rsv(fs_info, &ei->block_rsv, + BTRFS_BLOCK_RSV_DELALLOC); + ei->runtime_flags = 0; + ei->prop_compress = BTRFS_COMPRESS_NONE; + ei->defrag_compress = BTRFS_COMPRESS_NONE; + + ei->delayed_node = NULL; + + ei->i_otime_sec = 0; + ei->i_otime_nsec = 0; + + inode = &ei->vfs_inode; + btrfs_extent_map_tree_init(&ei->extent_tree); + + /* This io tree sets the valid inode. */ + btrfs_extent_io_tree_init(fs_info, &ei->io_tree, IO_TREE_INODE_IO); + ei->io_tree.inode = ei; + + ei->file_extent_tree = NULL; + + mutex_init(&ei->log_mutex); + spin_lock_init(&ei->ordered_tree_lock); + ei->ordered_tree = RB_ROOT; + ei->ordered_tree_last = NULL; + INIT_LIST_HEAD(&ei->delalloc_inodes); + INIT_LIST_HEAD(&ei->delayed_iput); + init_rwsem(&ei->i_mmap_lock); + + return inode; +} + +#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS +void btrfs_test_destroy_inode(struct inode *inode) +{ + btrfs_drop_extent_map_range(BTRFS_I(inode), 0, (u64)-1, false); + kfree(BTRFS_I(inode)->file_extent_tree); + kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode)); +} +#endif + +void btrfs_free_inode(struct inode *inode) +{ + kfree(BTRFS_I(inode)->file_extent_tree); + kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode)); +} + +void btrfs_destroy_inode(struct inode *vfs_inode) +{ + struct btrfs_ordered_extent *ordered; + struct btrfs_inode *inode = BTRFS_I(vfs_inode); + struct btrfs_root *root = inode->root; + bool freespace_inode; + + WARN_ON(!hlist_empty(&vfs_inode->i_dentry)); + WARN_ON(vfs_inode->i_data.nrpages); + WARN_ON(inode->block_rsv.reserved); + WARN_ON(inode->block_rsv.size); + WARN_ON(inode->outstanding_extents); + if (!S_ISDIR(vfs_inode->i_mode)) { + WARN_ON(inode->delalloc_bytes); + WARN_ON(inode->new_delalloc_bytes); + WARN_ON(inode->csum_bytes); + } + if (!root || !btrfs_is_data_reloc_root(root)) + WARN_ON(inode->defrag_bytes); + + /* + * This can happen where we create an inode, but somebody else also + * created the same inode and we need to destroy the one we already + * created. + */ + if (!root) + return; + + /* + * If this is a free space inode do not take the ordered extents lockdep + * map. + */ + freespace_inode = btrfs_is_free_space_inode(inode); + + while (1) { + ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1); + if (!ordered) + break; + else { + btrfs_err(root->fs_info, + "found ordered extent %llu %llu on inode cleanup", + ordered->file_offset, ordered->num_bytes); + + if (!freespace_inode) + btrfs_lockdep_acquire(root->fs_info, btrfs_ordered_extent); + + btrfs_remove_ordered_extent(ordered); + btrfs_put_ordered_extent(ordered); + btrfs_put_ordered_extent(ordered); + } + } + btrfs_qgroup_check_reserved_leak(inode); + btrfs_del_inode_from_root(inode); + btrfs_drop_extent_map_range(inode, 0, (u64)-1, false); + btrfs_inode_clear_file_extent_range(inode, 0, (u64)-1); + btrfs_put_root(inode->root); +} + +int btrfs_drop_inode(struct inode *inode) +{ + struct btrfs_root *root = BTRFS_I(inode)->root; + + if (root == NULL) + return 1; + + /* the snap/subvol tree is on deleting */ + if (btrfs_root_refs(&root->root_item) == 0) + return 1; + else + return inode_generic_drop(inode); +} + +static void init_once(void *foo) +{ + struct btrfs_inode *ei = foo; + + inode_init_once(&ei->vfs_inode); +} + +void __cold btrfs_destroy_cachep(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(btrfs_inode_cachep); +} + +int __init btrfs_init_cachep(void) +{ + btrfs_inode_cachep = kmem_cache_create("btrfs_inode", + sizeof(struct btrfs_inode), 0, + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, + init_once); + if (!btrfs_inode_cachep) + return -ENOMEM; + + return 0; +} + +static int btrfs_getattr(struct mnt_idmap *idmap, + const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int flags) +{ + u64 delalloc_bytes; + u64 inode_bytes; + struct inode *inode = d_inode(path->dentry); + u32 blocksize = btrfs_sb(inode->i_sb)->sectorsize; + u32 bi_flags = BTRFS_I(inode)->flags; + u32 bi_ro_flags = BTRFS_I(inode)->ro_flags; + + stat->result_mask |= STATX_BTIME; + stat->btime.tv_sec = BTRFS_I(inode)->i_otime_sec; + stat->btime.tv_nsec = BTRFS_I(inode)->i_otime_nsec; + if (bi_flags & BTRFS_INODE_APPEND) + stat->attributes |= STATX_ATTR_APPEND; + if (bi_flags & BTRFS_INODE_COMPRESS) + stat->attributes |= STATX_ATTR_COMPRESSED; + if (bi_flags & BTRFS_INODE_IMMUTABLE) + stat->attributes |= STATX_ATTR_IMMUTABLE; + if (bi_flags & BTRFS_INODE_NODUMP) + stat->attributes |= STATX_ATTR_NODUMP; + if (bi_ro_flags & BTRFS_INODE_RO_VERITY) + stat->attributes |= STATX_ATTR_VERITY; + + stat->attributes_mask |= (STATX_ATTR_APPEND | + STATX_ATTR_COMPRESSED | + STATX_ATTR_IMMUTABLE | + STATX_ATTR_NODUMP); + + generic_fillattr(idmap, request_mask, inode, stat); + stat->dev = BTRFS_I(inode)->root->anon_dev; + + stat->subvol = btrfs_root_id(BTRFS_I(inode)->root); + stat->result_mask |= STATX_SUBVOL; + + spin_lock(&BTRFS_I(inode)->lock); + delalloc_bytes = S_ISREG(inode->i_mode) ? + BTRFS_I(inode)->new_delalloc_bytes : 0; + inode_bytes = inode_get_bytes(inode); + spin_unlock(&BTRFS_I(inode)->lock); + stat->blocks = (ALIGN(inode_bytes, blocksize) + + ALIGN(delalloc_bytes, blocksize)) >> SECTOR_SHIFT; + return 0; +} + +static int btrfs_rename_exchange(struct inode *old_dir, + struct dentry *old_dentry, + struct inode *new_dir, + struct dentry *new_dentry) +{ + struct btrfs_fs_info *fs_info = inode_to_fs_info(old_dir); + struct btrfs_trans_handle *trans; + unsigned int trans_num_items; + struct btrfs_root *root = BTRFS_I(old_dir)->root; + struct btrfs_root *dest = BTRFS_I(new_dir)->root; + struct inode *new_inode = new_dentry->d_inode; + struct inode *old_inode = old_dentry->d_inode; + struct btrfs_rename_ctx old_rename_ctx; + struct btrfs_rename_ctx new_rename_ctx; + u64 old_ino = btrfs_ino(BTRFS_I(old_inode)); + u64 new_ino = btrfs_ino(BTRFS_I(new_inode)); + u64 old_idx = 0; + u64 new_idx = 0; + int ret; + int ret2; + bool need_abort = false; + bool logs_pinned = false; + struct fscrypt_name old_fname, new_fname; + struct fscrypt_str *old_name, *new_name; + + /* + * For non-subvolumes allow exchange only within one subvolume, in the + * same inode namespace. Two subvolumes (represented as directory) can + * be exchanged as they're a logical link and have a fixed inode number. + */ + if (root != dest && + (old_ino != BTRFS_FIRST_FREE_OBJECTID || + new_ino != BTRFS_FIRST_FREE_OBJECTID)) + return -EXDEV; + + ret = fscrypt_setup_filename(old_dir, &old_dentry->d_name, 0, &old_fname); + if (ret) + return ret; + + ret = fscrypt_setup_filename(new_dir, &new_dentry->d_name, 0, &new_fname); + if (ret) { + fscrypt_free_filename(&old_fname); + return ret; + } + + old_name = &old_fname.disk_name; + new_name = &new_fname.disk_name; + + /* close the race window with snapshot create/destroy ioctl */ + if (old_ino == BTRFS_FIRST_FREE_OBJECTID || + new_ino == BTRFS_FIRST_FREE_OBJECTID) + down_read(&fs_info->subvol_sem); + + /* + * For each inode: + * 1 to remove old dir item + * 1 to remove old dir index + * 1 to add new dir item + * 1 to add new dir index + * 1 to update parent inode + * + * If the parents are the same, we only need to account for one + */ + trans_num_items = (old_dir == new_dir ? 9 : 10); + if (old_ino == BTRFS_FIRST_FREE_OBJECTID) { + /* + * 1 to remove old root ref + * 1 to remove old root backref + * 1 to add new root ref + * 1 to add new root backref + */ + trans_num_items += 4; + } else { + /* + * 1 to update inode item + * 1 to remove old inode ref + * 1 to add new inode ref + */ + trans_num_items += 3; + } + if (new_ino == BTRFS_FIRST_FREE_OBJECTID) + trans_num_items += 4; + else + trans_num_items += 3; + trans = btrfs_start_transaction(root, trans_num_items); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto out_notrans; + } + + if (dest != root) { + ret = btrfs_record_root_in_trans(trans, dest); + if (ret) + goto out_fail; + } + + /* + * We need to find a free sequence number both in the source and + * in the destination directory for the exchange. + */ + ret = btrfs_set_inode_index(BTRFS_I(new_dir), &old_idx); + if (ret) + goto out_fail; + ret = btrfs_set_inode_index(BTRFS_I(old_dir), &new_idx); + if (ret) + goto out_fail; + + BTRFS_I(old_inode)->dir_index = 0ULL; + BTRFS_I(new_inode)->dir_index = 0ULL; + + /* Reference for the source. */ + if (old_ino == BTRFS_FIRST_FREE_OBJECTID) { + /* force full log commit if subvolume involved. */ + btrfs_set_log_full_commit(trans); + } else { + ret = btrfs_insert_inode_ref(trans, dest, new_name, old_ino, + btrfs_ino(BTRFS_I(new_dir)), + old_idx); + if (ret) + goto out_fail; + need_abort = true; + } + + /* And now for the dest. */ + if (new_ino == BTRFS_FIRST_FREE_OBJECTID) { + /* force full log commit if subvolume involved. */ + btrfs_set_log_full_commit(trans); + } else { + ret = btrfs_insert_inode_ref(trans, root, old_name, new_ino, + btrfs_ino(BTRFS_I(old_dir)), + new_idx); + if (ret) { + if (unlikely(need_abort)) + btrfs_abort_transaction(trans, ret); + goto out_fail; + } + } + + /* Update inode version and ctime/mtime. */ + inode_inc_iversion(old_dir); + inode_inc_iversion(new_dir); + inode_inc_iversion(old_inode); + inode_inc_iversion(new_inode); + simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry); + + if (old_ino != BTRFS_FIRST_FREE_OBJECTID && + new_ino != BTRFS_FIRST_FREE_OBJECTID) { + /* + * If we are renaming in the same directory (and it's not for + * root entries) pin the log early to prevent any concurrent + * task from logging the directory after we removed the old + * entries and before we add the new entries, otherwise that + * task can sync a log without any entry for the inodes we are + * renaming and therefore replaying that log, if a power failure + * happens after syncing the log, would result in deleting the + * inodes. + * + * If the rename affects two different directories, we want to + * make sure the that there's no log commit that contains + * updates for only one of the directories but not for the + * other. + * + * If we are renaming an entry for a root, we don't care about + * log updates since we called btrfs_set_log_full_commit(). + */ + btrfs_pin_log_trans(root); + btrfs_pin_log_trans(dest); + logs_pinned = true; + } + + if (old_dentry->d_parent != new_dentry->d_parent) { + btrfs_record_unlink_dir(trans, BTRFS_I(old_dir), + BTRFS_I(old_inode), true); + btrfs_record_unlink_dir(trans, BTRFS_I(new_dir), + BTRFS_I(new_inode), true); + } + + /* src is a subvolume */ + if (old_ino == BTRFS_FIRST_FREE_OBJECTID) { + ret = btrfs_unlink_subvol(trans, BTRFS_I(old_dir), old_dentry); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out_fail; + } + } else { /* src is an inode */ + ret = __btrfs_unlink_inode(trans, BTRFS_I(old_dir), + BTRFS_I(old_dentry->d_inode), + old_name, &old_rename_ctx); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out_fail; + } + ret = btrfs_update_inode(trans, BTRFS_I(old_inode)); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out_fail; + } + } + + /* dest is a subvolume */ + if (new_ino == BTRFS_FIRST_FREE_OBJECTID) { + ret = btrfs_unlink_subvol(trans, BTRFS_I(new_dir), new_dentry); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out_fail; + } + } else { /* dest is an inode */ + ret = __btrfs_unlink_inode(trans, BTRFS_I(new_dir), + BTRFS_I(new_dentry->d_inode), + new_name, &new_rename_ctx); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out_fail; + } + ret = btrfs_update_inode(trans, BTRFS_I(new_inode)); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out_fail; + } + } + + ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode), + new_name, false, old_idx); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out_fail; + } + + ret = btrfs_add_link(trans, BTRFS_I(old_dir), BTRFS_I(new_inode), + old_name, false, new_idx); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out_fail; + } + + if (old_inode->i_nlink == 1) + BTRFS_I(old_inode)->dir_index = old_idx; + if (new_inode->i_nlink == 1) + BTRFS_I(new_inode)->dir_index = new_idx; + + /* + * Do the log updates for all inodes. + * + * If either entry is for a root we don't need to update the logs since + * we've called btrfs_set_log_full_commit() before. + */ + if (logs_pinned) { + btrfs_log_new_name(trans, old_dentry, BTRFS_I(old_dir), + old_rename_ctx.index, new_dentry->d_parent); + btrfs_log_new_name(trans, new_dentry, BTRFS_I(new_dir), + new_rename_ctx.index, old_dentry->d_parent); + } + +out_fail: + if (logs_pinned) { + btrfs_end_log_trans(root); + btrfs_end_log_trans(dest); + } + ret2 = btrfs_end_transaction(trans); + ret = ret ? ret : ret2; +out_notrans: + if (new_ino == BTRFS_FIRST_FREE_OBJECTID || + old_ino == BTRFS_FIRST_FREE_OBJECTID) + up_read(&fs_info->subvol_sem); + + fscrypt_free_filename(&new_fname); + fscrypt_free_filename(&old_fname); + return ret; +} + +static struct inode *new_whiteout_inode(struct mnt_idmap *idmap, + struct inode *dir) +{ + struct inode *inode; + + inode = new_inode(dir->i_sb); + if (inode) { + inode_init_owner(idmap, inode, dir, + S_IFCHR | WHITEOUT_MODE); + inode->i_op = &btrfs_special_inode_operations; + init_special_inode(inode, inode->i_mode, WHITEOUT_DEV); + } + return inode; +} + +static int btrfs_rename(struct mnt_idmap *idmap, + struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry, + unsigned int flags) +{ + struct btrfs_fs_info *fs_info = inode_to_fs_info(old_dir); + struct btrfs_new_inode_args whiteout_args = { + .dir = old_dir, + .dentry = old_dentry, + }; + struct btrfs_trans_handle *trans; + unsigned int trans_num_items; + struct btrfs_root *root = BTRFS_I(old_dir)->root; + struct btrfs_root *dest = BTRFS_I(new_dir)->root; + struct inode *new_inode = d_inode(new_dentry); + struct inode *old_inode = d_inode(old_dentry); + struct btrfs_rename_ctx rename_ctx; + u64 index = 0; + int ret; + int ret2; + u64 old_ino = btrfs_ino(BTRFS_I(old_inode)); + struct fscrypt_name old_fname, new_fname; + bool logs_pinned = false; + + if (btrfs_ino(BTRFS_I(new_dir)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) + return -EPERM; + + /* we only allow rename subvolume link between subvolumes */ + if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest) + return -EXDEV; + + if (old_ino == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID || + (new_inode && btrfs_ino(BTRFS_I(new_inode)) == BTRFS_FIRST_FREE_OBJECTID)) + return -ENOTEMPTY; + + if (S_ISDIR(old_inode->i_mode) && new_inode && + new_inode->i_size > BTRFS_EMPTY_DIR_SIZE) + return -ENOTEMPTY; + + ret = fscrypt_setup_filename(old_dir, &old_dentry->d_name, 0, &old_fname); + if (ret) + return ret; + + ret = fscrypt_setup_filename(new_dir, &new_dentry->d_name, 0, &new_fname); + if (ret) { + fscrypt_free_filename(&old_fname); + return ret; + } + + /* check for collisions, even if the name isn't there */ + ret = btrfs_check_dir_item_collision(dest, new_dir->i_ino, &new_fname.disk_name); + if (ret) { + if (ret == -EEXIST) { + /* we shouldn't get + * eexist without a new_inode */ + if (WARN_ON(!new_inode)) { + goto out_fscrypt_names; + } + } else { + /* maybe -EOVERFLOW */ + goto out_fscrypt_names; + } + } + ret = 0; + + /* + * we're using rename to replace one file with another. Start IO on it + * now so we don't add too much work to the end of the transaction + */ + if (new_inode && S_ISREG(old_inode->i_mode) && new_inode->i_size) + filemap_flush(old_inode->i_mapping); + + if (flags & RENAME_WHITEOUT) { + whiteout_args.inode = new_whiteout_inode(idmap, old_dir); + if (!whiteout_args.inode) { + ret = -ENOMEM; + goto out_fscrypt_names; + } + ret = btrfs_new_inode_prepare(&whiteout_args, &trans_num_items); + if (ret) + goto out_whiteout_inode; + } else { + /* 1 to update the old parent inode. */ + trans_num_items = 1; + } + + if (old_ino == BTRFS_FIRST_FREE_OBJECTID) { + /* Close the race window with snapshot create/destroy ioctl */ + down_read(&fs_info->subvol_sem); + /* + * 1 to remove old root ref + * 1 to remove old root backref + * 1 to add new root ref + * 1 to add new root backref + */ + trans_num_items += 4; + } else { + /* + * 1 to update inode + * 1 to remove old inode ref + * 1 to add new inode ref + */ + trans_num_items += 3; + } + /* + * 1 to remove old dir item + * 1 to remove old dir index + * 1 to add new dir item + * 1 to add new dir index + */ + trans_num_items += 4; + /* 1 to update new parent inode if it's not the same as the old parent */ + if (new_dir != old_dir) + trans_num_items++; + if (new_inode) { + /* + * 1 to update inode + * 1 to remove inode ref + * 1 to remove dir item + * 1 to remove dir index + * 1 to possibly add orphan item + */ + trans_num_items += 5; + } + trans = btrfs_start_transaction(root, trans_num_items); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto out_notrans; + } + + if (dest != root) { + ret = btrfs_record_root_in_trans(trans, dest); + if (ret) + goto out_fail; + } + + ret = btrfs_set_inode_index(BTRFS_I(new_dir), &index); + if (ret) + goto out_fail; + + BTRFS_I(old_inode)->dir_index = 0ULL; + if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) { + /* force full log commit if subvolume involved. */ + btrfs_set_log_full_commit(trans); + } else { + ret = btrfs_insert_inode_ref(trans, dest, &new_fname.disk_name, + old_ino, btrfs_ino(BTRFS_I(new_dir)), + index); + if (ret) + goto out_fail; + } + + inode_inc_iversion(old_dir); + inode_inc_iversion(new_dir); + inode_inc_iversion(old_inode); + simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry); + + if (old_ino != BTRFS_FIRST_FREE_OBJECTID) { + /* + * If we are renaming in the same directory (and it's not a + * root entry) pin the log to prevent any concurrent task from + * logging the directory after we removed the old entry and + * before we add the new entry, otherwise that task can sync + * a log without any entry for the inode we are renaming and + * therefore replaying that log, if a power failure happens + * after syncing the log, would result in deleting the inode. + * + * If the rename affects two different directories, we want to + * make sure the that there's no log commit that contains + * updates for only one of the directories but not for the + * other. + * + * If we are renaming an entry for a root, we don't care about + * log updates since we called btrfs_set_log_full_commit(). + */ + btrfs_pin_log_trans(root); + btrfs_pin_log_trans(dest); + logs_pinned = true; + } + + if (old_dentry->d_parent != new_dentry->d_parent) + btrfs_record_unlink_dir(trans, BTRFS_I(old_dir), + BTRFS_I(old_inode), true); + + if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) { + ret = btrfs_unlink_subvol(trans, BTRFS_I(old_dir), old_dentry); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out_fail; + } + } else { + ret = __btrfs_unlink_inode(trans, BTRFS_I(old_dir), + BTRFS_I(d_inode(old_dentry)), + &old_fname.disk_name, &rename_ctx); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out_fail; + } + ret = btrfs_update_inode(trans, BTRFS_I(old_inode)); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out_fail; + } + } + + if (new_inode) { + inode_inc_iversion(new_inode); + if (unlikely(btrfs_ino(BTRFS_I(new_inode)) == + BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) { + ret = btrfs_unlink_subvol(trans, BTRFS_I(new_dir), new_dentry); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out_fail; + } + BUG_ON(new_inode->i_nlink == 0); + } else { + ret = btrfs_unlink_inode(trans, BTRFS_I(new_dir), + BTRFS_I(d_inode(new_dentry)), + &new_fname.disk_name); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out_fail; + } + } + if (new_inode->i_nlink == 0) { + ret = btrfs_orphan_add(trans, + BTRFS_I(d_inode(new_dentry))); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out_fail; + } + } + } + + ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode), + &new_fname.disk_name, false, index); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out_fail; + } + + if (old_inode->i_nlink == 1) + BTRFS_I(old_inode)->dir_index = index; + + if (logs_pinned) + btrfs_log_new_name(trans, old_dentry, BTRFS_I(old_dir), + rename_ctx.index, new_dentry->d_parent); + + if (flags & RENAME_WHITEOUT) { + ret = btrfs_create_new_inode(trans, &whiteout_args); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out_fail; + } else { + unlock_new_inode(whiteout_args.inode); + iput(whiteout_args.inode); + whiteout_args.inode = NULL; + } + } +out_fail: + if (logs_pinned) { + btrfs_end_log_trans(root); + btrfs_end_log_trans(dest); + } + ret2 = btrfs_end_transaction(trans); + ret = ret ? ret : ret2; +out_notrans: + if (old_ino == BTRFS_FIRST_FREE_OBJECTID) + up_read(&fs_info->subvol_sem); + if (flags & RENAME_WHITEOUT) + btrfs_new_inode_args_destroy(&whiteout_args); +out_whiteout_inode: + if (flags & RENAME_WHITEOUT) + iput(whiteout_args.inode); +out_fscrypt_names: + fscrypt_free_filename(&old_fname); + fscrypt_free_filename(&new_fname); + return ret; +} + +static int btrfs_rename2(struct mnt_idmap *idmap, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) +{ + int ret; + + if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) + return -EINVAL; + + if (flags & RENAME_EXCHANGE) + ret = btrfs_rename_exchange(old_dir, old_dentry, new_dir, + new_dentry); + else + ret = btrfs_rename(idmap, old_dir, old_dentry, new_dir, + new_dentry, flags); + + btrfs_btree_balance_dirty(BTRFS_I(new_dir)->root->fs_info); + + return ret; +} + +struct btrfs_delalloc_work { + struct inode *inode; + struct completion completion; + struct list_head list; + struct btrfs_work work; +}; + +static void btrfs_run_delalloc_work(struct btrfs_work *work) +{ + struct btrfs_delalloc_work *delalloc_work; + struct inode *inode; + + delalloc_work = container_of(work, struct btrfs_delalloc_work, + work); + inode = delalloc_work->inode; + filemap_flush(inode->i_mapping); + if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, + &BTRFS_I(inode)->runtime_flags)) + filemap_flush(inode->i_mapping); + + iput(inode); + complete(&delalloc_work->completion); +} + +static struct btrfs_delalloc_work *btrfs_alloc_delalloc_work(struct inode *inode) +{ + struct btrfs_delalloc_work *work; + + work = kmalloc_obj(*work, GFP_NOFS); + if (!work) + return NULL; + + init_completion(&work->completion); + INIT_LIST_HEAD(&work->list); + work->inode = inode; + btrfs_init_work(&work->work, btrfs_run_delalloc_work, NULL); + + return work; +} + +/* + * some fairly slow code that needs optimization. This walks the list + * of all the inodes with pending delalloc and forces them to disk. + */ +static int start_delalloc_inodes(struct btrfs_root *root, long *nr_to_write, + bool snapshot, bool in_reclaim_context) +{ + struct btrfs_delalloc_work *work, *next; + LIST_HEAD(works); + LIST_HEAD(splice); + int ret = 0; + + mutex_lock(&root->delalloc_mutex); + spin_lock(&root->delalloc_lock); + list_splice_init(&root->delalloc_inodes, &splice); + while (!list_empty(&splice)) { + struct btrfs_inode *inode; + struct inode *tmp_inode; + + inode = list_first_entry(&splice, struct btrfs_inode, delalloc_inodes); + + list_move_tail(&inode->delalloc_inodes, &root->delalloc_inodes); + + if (in_reclaim_context && + test_bit(BTRFS_INODE_NO_DELALLOC_FLUSH, &inode->runtime_flags)) + continue; + + tmp_inode = igrab(&inode->vfs_inode); + if (!tmp_inode) { + cond_resched_lock(&root->delalloc_lock); + continue; + } + spin_unlock(&root->delalloc_lock); + + if (snapshot) + set_bit(BTRFS_INODE_SNAPSHOT_FLUSH, &inode->runtime_flags); + if (nr_to_write == NULL) { + work = btrfs_alloc_delalloc_work(tmp_inode); + if (!work) { + iput(tmp_inode); + ret = -ENOMEM; + goto out; + } + list_add_tail(&work->list, &works); + btrfs_queue_work(root->fs_info->flush_workers, + &work->work); + } else { + ret = filemap_flush_nr(tmp_inode->i_mapping, + nr_to_write); + btrfs_add_delayed_iput(inode); + + if (ret || *nr_to_write <= 0) + goto out; + } + cond_resched(); + spin_lock(&root->delalloc_lock); + } + spin_unlock(&root->delalloc_lock); + +out: + list_for_each_entry_safe(work, next, &works, list) { + list_del_init(&work->list); + wait_for_completion(&work->completion); + kfree(work); + } + + if (!list_empty(&splice)) { + spin_lock(&root->delalloc_lock); + list_splice_tail(&splice, &root->delalloc_inodes); + spin_unlock(&root->delalloc_lock); + } + mutex_unlock(&root->delalloc_mutex); + return ret; +} + +int btrfs_start_delalloc_snapshot(struct btrfs_root *root, bool in_reclaim_context) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + + if (unlikely(BTRFS_FS_ERROR(fs_info))) + return -EROFS; + return start_delalloc_inodes(root, NULL, true, in_reclaim_context); +} + +int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, long nr, + bool in_reclaim_context) +{ + long *nr_to_write = nr == LONG_MAX ? NULL : &nr; + struct btrfs_root *root; + LIST_HEAD(splice); + int ret; + + if (unlikely(BTRFS_FS_ERROR(fs_info))) + return -EROFS; + + mutex_lock(&fs_info->delalloc_root_mutex); + spin_lock(&fs_info->delalloc_root_lock); + list_splice_init(&fs_info->delalloc_roots, &splice); + while (!list_empty(&splice)) { + root = list_first_entry(&splice, struct btrfs_root, + delalloc_root); + root = btrfs_grab_root(root); + BUG_ON(!root); + list_move_tail(&root->delalloc_root, + &fs_info->delalloc_roots); + spin_unlock(&fs_info->delalloc_root_lock); + + ret = start_delalloc_inodes(root, nr_to_write, false, + in_reclaim_context); + btrfs_put_root(root); + if (ret < 0 || nr <= 0) + goto out; + spin_lock(&fs_info->delalloc_root_lock); + } + spin_unlock(&fs_info->delalloc_root_lock); + + ret = 0; +out: + if (!list_empty(&splice)) { + spin_lock(&fs_info->delalloc_root_lock); + list_splice_tail(&splice, &fs_info->delalloc_roots); + spin_unlock(&fs_info->delalloc_root_lock); + } + mutex_unlock(&fs_info->delalloc_root_mutex); + return ret; +} + +static int btrfs_symlink(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const char *symname) +{ + struct btrfs_fs_info *fs_info = inode_to_fs_info(dir); + struct btrfs_trans_handle *trans; + struct btrfs_root *root = BTRFS_I(dir)->root; + struct btrfs_path *path; + struct btrfs_key key; + struct inode *inode; + struct btrfs_new_inode_args new_inode_args = { + .dir = dir, + .dentry = dentry, + }; + unsigned int trans_num_items; + int ret; + int name_len; + int datasize; + unsigned long ptr; + struct btrfs_file_extent_item *ei; + struct extent_buffer *leaf; + + name_len = strlen(symname); + /* + * Symlinks utilize uncompressed inline extent data, which should not + * reach block size. + */ + if (name_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info) || + name_len >= fs_info->sectorsize) + return -ENAMETOOLONG; + + inode = new_inode(dir->i_sb); + if (!inode) + return -ENOMEM; + inode_init_owner(idmap, inode, dir, S_IFLNK | S_IRWXUGO); + inode->i_op = &btrfs_symlink_inode_operations; + inode_nohighmem(inode); + inode->i_mapping->a_ops = &btrfs_aops; + btrfs_i_size_write(BTRFS_I(inode), name_len); + inode_set_bytes(inode, name_len); + + new_inode_args.inode = inode; + ret = btrfs_new_inode_prepare(&new_inode_args, &trans_num_items); + if (ret) + goto out_inode; + /* 1 additional item for the inline extent */ + trans_num_items++; + + trans = btrfs_start_transaction(root, trans_num_items); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto out_new_inode_args; + } + + ret = btrfs_create_new_inode(trans, &new_inode_args); + if (ret) + goto out; + + path = btrfs_alloc_path(); + if (unlikely(!path)) { + ret = -ENOMEM; + btrfs_abort_transaction(trans, ret); + discard_new_inode(inode); + inode = NULL; + goto out; + } + key.objectid = btrfs_ino(BTRFS_I(inode)); + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = 0; + datasize = btrfs_file_extent_calc_inline_size(name_len); + ret = btrfs_insert_empty_item(trans, root, path, &key, datasize); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + btrfs_free_path(path); + discard_new_inode(inode); + inode = NULL; + goto out; + } + leaf = path->nodes[0]; + ei = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_file_extent_item); + btrfs_set_file_extent_generation(leaf, ei, trans->transid); + btrfs_set_file_extent_type(leaf, ei, + BTRFS_FILE_EXTENT_INLINE); + btrfs_set_file_extent_encryption(leaf, ei, 0); + btrfs_set_file_extent_compression(leaf, ei, 0); + btrfs_set_file_extent_other_encoding(leaf, ei, 0); + btrfs_set_file_extent_ram_bytes(leaf, ei, name_len); + + ptr = btrfs_file_extent_inline_start(ei); + write_extent_buffer(leaf, symname, ptr, name_len); + btrfs_free_path(path); + + d_instantiate_new(dentry, inode); + ret = 0; +out: + btrfs_end_transaction(trans); + btrfs_btree_balance_dirty(fs_info); +out_new_inode_args: + btrfs_new_inode_args_destroy(&new_inode_args); +out_inode: + if (ret) + iput(inode); + return ret; +} + +static struct btrfs_trans_handle *insert_prealloc_file_extent( + struct btrfs_trans_handle *trans_in, + struct btrfs_inode *inode, + struct btrfs_key *ins, + u64 file_offset) +{ + struct btrfs_file_extent_item stack_fi; + struct btrfs_replace_extent_info extent_info; + struct btrfs_trans_handle *trans = trans_in; + struct btrfs_path *path; + u64 start = ins->objectid; + u64 len = ins->offset; + u64 qgroup_released = 0; + int ret; + + memset(&stack_fi, 0, sizeof(stack_fi)); + + btrfs_set_stack_file_extent_type(&stack_fi, BTRFS_FILE_EXTENT_PREALLOC); + btrfs_set_stack_file_extent_disk_bytenr(&stack_fi, start); + btrfs_set_stack_file_extent_disk_num_bytes(&stack_fi, len); + btrfs_set_stack_file_extent_num_bytes(&stack_fi, len); + btrfs_set_stack_file_extent_ram_bytes(&stack_fi, len); + btrfs_set_stack_file_extent_compression(&stack_fi, BTRFS_COMPRESS_NONE); + /* Encryption and other encoding is reserved and all 0 */ + + ret = btrfs_qgroup_release_data(inode, file_offset, len, &qgroup_released); + if (ret < 0) + return ERR_PTR(ret); + + if (trans) { + ret = insert_reserved_file_extent(trans, inode, + file_offset, &stack_fi, + true, qgroup_released); + if (ret) + goto free_qgroup; + return trans; + } + + extent_info.disk_offset = start; + extent_info.disk_len = len; + extent_info.data_offset = 0; + extent_info.data_len = len; + extent_info.file_offset = file_offset; + extent_info.extent_buf = (char *)&stack_fi; + extent_info.is_new_extent = true; + extent_info.update_times = true; + extent_info.qgroup_reserved = qgroup_released; + extent_info.insertions = 0; + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto free_qgroup; + } + + ret = btrfs_replace_file_extents(inode, path, file_offset, + file_offset + len - 1, &extent_info, + &trans); + btrfs_free_path(path); + if (ret) + goto free_qgroup; + return trans; + +free_qgroup: + /* + * We have released qgroup data range at the beginning of the function, + * and normally qgroup_released bytes will be freed when committing + * transaction. + * But if we error out early, we have to free what we have released + * or we leak qgroup data reservation. + */ + btrfs_qgroup_free_refroot(inode->root->fs_info, + btrfs_root_id(inode->root), qgroup_released, + BTRFS_QGROUP_RSV_DATA); + return ERR_PTR(ret); +} + +static int __btrfs_prealloc_file_range(struct inode *inode, int mode, + u64 start, u64 num_bytes, u64 min_size, + loff_t actual_len, u64 *alloc_hint, + struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + struct extent_map *em; + struct btrfs_root *root = BTRFS_I(inode)->root; + struct btrfs_key ins; + u64 cur_offset = start; + u64 clear_offset = start; + u64 i_size; + u64 cur_bytes; + u64 last_alloc = (u64)-1; + int ret = 0; + bool own_trans = true; + u64 end = start + num_bytes - 1; + + if (trans) + own_trans = false; + while (num_bytes > 0) { + cur_bytes = min_t(u64, num_bytes, SZ_256M); + cur_bytes = max(cur_bytes, min_size); + /* + * If we are severely fragmented we could end up with really + * small allocations, so if the allocator is returning small + * chunks lets make its job easier by only searching for those + * sized chunks. + */ + cur_bytes = min(cur_bytes, last_alloc); + ret = btrfs_reserve_extent(root, cur_bytes, cur_bytes, + min_size, 0, *alloc_hint, &ins, true, false); + if (ret) + break; + + /* + * We've reserved this space, and thus converted it from + * ->bytes_may_use to ->bytes_reserved. Any error that happens + * from here on out we will only need to clear our reservation + * for the remaining unreserved area, so advance our + * clear_offset by our extent size. + */ + clear_offset += ins.offset; + + last_alloc = ins.offset; + trans = insert_prealloc_file_extent(trans, BTRFS_I(inode), + &ins, cur_offset); + /* + * Now that we inserted the prealloc extent we can finally + * decrement the number of reservations in the block group. + * If we did it before, we could race with relocation and have + * relocation miss the reserved extent, making it fail later. + */ + btrfs_dec_block_group_reservations(fs_info, ins.objectid); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + btrfs_free_reserved_extent(fs_info, ins.objectid, + ins.offset, false); + break; + } + + em = btrfs_alloc_extent_map(); + if (!em) { + btrfs_drop_extent_map_range(BTRFS_I(inode), cur_offset, + cur_offset + ins.offset - 1, false); + btrfs_set_inode_full_sync(BTRFS_I(inode)); + goto next; + } + + em->start = cur_offset; + em->len = ins.offset; + em->disk_bytenr = ins.objectid; + em->offset = 0; + em->disk_num_bytes = ins.offset; + em->ram_bytes = ins.offset; + em->flags |= EXTENT_FLAG_PREALLOC; + em->generation = trans->transid; + + ret = btrfs_replace_extent_map_range(BTRFS_I(inode), em, true); + btrfs_free_extent_map(em); +next: + num_bytes -= ins.offset; + cur_offset += ins.offset; + *alloc_hint = ins.objectid + ins.offset; + + inode_inc_iversion(inode); + inode_set_ctime_current(inode); + BTRFS_I(inode)->flags |= BTRFS_INODE_PREALLOC; + if (!(mode & FALLOC_FL_KEEP_SIZE) && + (actual_len > inode->i_size) && + (cur_offset > inode->i_size)) { + u64 range_start; + u64 range_end; + + if (cur_offset > actual_len) + i_size = actual_len; + else + i_size = cur_offset; + + /* + * Make sure the file_extent_tree covers the entire + * range [old_i_size, new_i_size) before we update + * disk_i_size. Without this, a previous KEEP_SIZE + * prealloc that extended past i_size (and was lost + * across umount/mount because file_extent_tree is + * only populated up to round_up(i_size) on inode + * load) can leave a gap inside this range. That gap + * would cause btrfs_inode_safe_disk_i_size_write() + * (via find_contiguous_extent_bit() starting at 0) + * to truncate disk_i_size to the start of the gap, + * making the persisted size smaller than i_size. + */ + range_start = round_down(inode->i_size, fs_info->sectorsize); + range_end = round_up(i_size, fs_info->sectorsize); + ret = btrfs_inode_set_file_extent_range(BTRFS_I(inode), + range_start, range_end - range_start); + if (ret) { + btrfs_abort_transaction(trans, ret); + if (own_trans) + btrfs_end_transaction(trans); + break; + } + + i_size_write(inode, i_size); + btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), 0); + } + + ret = btrfs_update_inode(trans, BTRFS_I(inode)); + + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + if (own_trans) + btrfs_end_transaction(trans); + break; + } + + if (own_trans) { + btrfs_end_transaction(trans); + trans = NULL; + } + } + if (clear_offset < end) + btrfs_free_reserved_data_space(BTRFS_I(inode), NULL, clear_offset, + end - clear_offset + 1); + return ret; +} + +int btrfs_prealloc_file_range(struct inode *inode, int mode, + u64 start, u64 num_bytes, u64 min_size, + loff_t actual_len, u64 *alloc_hint) +{ + return __btrfs_prealloc_file_range(inode, mode, start, num_bytes, + min_size, actual_len, alloc_hint, + NULL); +} + +int btrfs_prealloc_file_range_trans(struct inode *inode, + struct btrfs_trans_handle *trans, int mode, + u64 start, u64 num_bytes, u64 min_size, + loff_t actual_len, u64 *alloc_hint) +{ + return __btrfs_prealloc_file_range(inode, mode, start, num_bytes, + min_size, actual_len, alloc_hint, trans); +} + +/* + * NOTE: in case you are adding MAY_EXEC check for directories: + * we are marking them with IOP_FASTPERM_MAY_EXEC, allowing path lookup to + * elide calls here. + */ +static int btrfs_permission(struct mnt_idmap *idmap, + struct inode *inode, int mask) +{ + struct btrfs_root *root = BTRFS_I(inode)->root; + umode_t mode = inode->i_mode; + + if (mask & MAY_WRITE && + (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode))) { + if (btrfs_root_readonly(root)) + return -EROFS; + if (BTRFS_I(inode)->flags & BTRFS_INODE_READONLY) + return -EACCES; + } + return generic_permission(idmap, inode, mask); +} + +static int btrfs_tmpfile(struct mnt_idmap *idmap, struct inode *dir, + struct file *file, umode_t mode) +{ + struct btrfs_fs_info *fs_info = inode_to_fs_info(dir); + struct btrfs_trans_handle *trans; + struct btrfs_root *root = BTRFS_I(dir)->root; + struct inode *inode; + struct btrfs_new_inode_args new_inode_args = { + .dir = dir, + .dentry = file->f_path.dentry, + .orphan = true, + }; + unsigned int trans_num_items; + int ret; + + inode = new_inode(dir->i_sb); + if (!inode) + return -ENOMEM; + inode_init_owner(idmap, inode, dir, mode); + inode->i_fop = &btrfs_file_operations; + inode->i_op = &btrfs_file_inode_operations; + inode->i_mapping->a_ops = &btrfs_aops; + + new_inode_args.inode = inode; + ret = btrfs_new_inode_prepare(&new_inode_args, &trans_num_items); + if (ret) + goto out_inode; + + trans = btrfs_start_transaction(root, trans_num_items); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto out_new_inode_args; + } + + ret = btrfs_create_new_inode(trans, &new_inode_args); + + /* + * We set number of links to 0 in btrfs_create_new_inode(), and here we + * set it to 1 because d_tmpfile() will issue a warning if the count is + * 0, through: + * + * d_tmpfile() -> inode_dec_link_count() -> drop_nlink() + */ + set_nlink(inode, 1); + + if (!ret) { + d_tmpfile(file, inode); + unlock_new_inode(inode); + mark_inode_dirty(inode); + } + + btrfs_end_transaction(trans); + btrfs_btree_balance_dirty(fs_info); +out_new_inode_args: + btrfs_new_inode_args_destroy(&new_inode_args); +out_inode: + if (ret) + iput(inode); + return finish_open_simple(file, ret); +} + +int btrfs_encoded_io_compression_from_extent(struct btrfs_fs_info *fs_info, + int compress_type) +{ + switch (compress_type) { + case BTRFS_COMPRESS_NONE: + return BTRFS_ENCODED_IO_COMPRESSION_NONE; + case BTRFS_COMPRESS_ZLIB: + return BTRFS_ENCODED_IO_COMPRESSION_ZLIB; + case BTRFS_COMPRESS_LZO: + /* + * The LZO format depends on the sector size. 64K is the maximum + * sector size that we support. + */ + if (fs_info->sectorsize < SZ_4K || fs_info->sectorsize > SZ_64K) + return -EINVAL; + return BTRFS_ENCODED_IO_COMPRESSION_LZO_4K + + (fs_info->sectorsize_bits - 12); + case BTRFS_COMPRESS_ZSTD: + return BTRFS_ENCODED_IO_COMPRESSION_ZSTD; + default: + return -EUCLEAN; + } +} + +static ssize_t btrfs_encoded_read_inline( + struct kiocb *iocb, + struct iov_iter *iter, u64 start, + u64 lockend, + struct extent_state **cached_state, + u64 extent_start, size_t count, + struct btrfs_ioctl_encoded_io_args *encoded, + bool *unlocked) +{ + struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp)); + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct extent_io_tree *io_tree = &inode->io_tree; + BTRFS_PATH_AUTO_FREE(path); + struct extent_buffer *leaf; + struct btrfs_file_extent_item *item; + u64 ram_bytes; + unsigned long ptr; + void *tmp; + ssize_t ret; + const bool nowait = (iocb->ki_flags & IOCB_NOWAIT); + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + path->nowait = nowait; + + ret = btrfs_lookup_file_extent(NULL, root, path, btrfs_ino(inode), + extent_start, 0); + if (ret) { + if (unlikely(ret > 0)) { + /* The extent item disappeared? */ + return -EIO; + } + return ret; + } + leaf = path->nodes[0]; + item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); + + ram_bytes = btrfs_file_extent_ram_bytes(leaf, item); + ptr = btrfs_file_extent_inline_start(item); + + encoded->len = min_t(u64, extent_start + ram_bytes, + inode->vfs_inode.i_size) - iocb->ki_pos; + ret = btrfs_encoded_io_compression_from_extent(fs_info, + btrfs_file_extent_compression(leaf, item)); + if (ret < 0) + return ret; + encoded->compression = ret; + if (encoded->compression) { + size_t inline_size; + + inline_size = btrfs_file_extent_inline_item_len(leaf, + path->slots[0]); + if (inline_size > count) + return -ENOBUFS; + + count = inline_size; + encoded->unencoded_len = ram_bytes; + encoded->unencoded_offset = iocb->ki_pos - extent_start; + } else { + count = min_t(u64, count, encoded->len); + encoded->len = count; + encoded->unencoded_len = count; + ptr += iocb->ki_pos - extent_start; + } + + tmp = kmalloc(count, GFP_NOFS); + if (!tmp) + return -ENOMEM; + + read_extent_buffer(leaf, tmp, ptr, count); + btrfs_release_path(path); + btrfs_unlock_extent(io_tree, start, lockend, cached_state); + btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); + *unlocked = true; + + ret = copy_to_iter(tmp, count, iter); + if (ret != count) + ret = -EFAULT; + kfree(tmp); + + return ret; +} + +struct btrfs_encoded_read_private { + struct completion *sync_reads; + void *uring_ctx; + refcount_t pending_refs; + blk_status_t status; +}; + +static void btrfs_encoded_read_endio(struct btrfs_bio *bbio) +{ + struct btrfs_encoded_read_private *priv = bbio->private; + + if (bbio->bio.bi_status) { + /* + * The memory barrier implied by the refcount_dec_and_test() here + * pairs with the memory barrier implied by the refcount_dec_and_test() + * in btrfs_encoded_read_regular_fill_pages() to ensure that + * this write is observed before the load of status in + * btrfs_encoded_read_regular_fill_pages(). + */ + WRITE_ONCE(priv->status, bbio->bio.bi_status); + } + if (refcount_dec_and_test(&priv->pending_refs)) { + int err = blk_status_to_errno(READ_ONCE(priv->status)); + + if (priv->uring_ctx) { + btrfs_uring_read_extent_endio(priv->uring_ctx, err); + kfree(priv); + } else { + complete(priv->sync_reads); + } + } + bio_put(&bbio->bio); +} + +int btrfs_encoded_read_regular_fill_pages(struct btrfs_inode *inode, + u64 disk_bytenr, u64 disk_io_size, + struct page **pages, void *uring_ctx) +{ + struct btrfs_encoded_read_private *priv, sync_priv; + struct completion sync_reads; + unsigned long i = 0; + struct btrfs_bio *bbio; + int ret; + + /* + * Fast path for synchronous reads which completes in this call, io_uring + * needs longer time span. + */ + if (uring_ctx) { + priv = kmalloc_obj(struct btrfs_encoded_read_private, GFP_NOFS); + if (!priv) + return -ENOMEM; + } else { + priv = &sync_priv; + init_completion(&sync_reads); + priv->sync_reads = &sync_reads; + } + + refcount_set(&priv->pending_refs, 1); + priv->status = 0; + priv->uring_ctx = uring_ctx; + + bbio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ, inode, 0, + btrfs_encoded_read_endio, priv); + bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT; + + do { + size_t bytes = min_t(u64, disk_io_size, PAGE_SIZE); + + if (bio_add_page(&bbio->bio, pages[i], bytes, 0) < bytes) { + refcount_inc(&priv->pending_refs); + btrfs_submit_bbio(bbio, 0); + + bbio = btrfs_bio_alloc(BIO_MAX_VECS, REQ_OP_READ, inode, 0, + btrfs_encoded_read_endio, priv); + bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT; + continue; + } + + i++; + disk_bytenr += bytes; + disk_io_size -= bytes; + } while (disk_io_size); + + refcount_inc(&priv->pending_refs); + btrfs_submit_bbio(bbio, 0); + + if (uring_ctx) { + if (refcount_dec_and_test(&priv->pending_refs)) { + ret = blk_status_to_errno(READ_ONCE(priv->status)); + btrfs_uring_read_extent_endio(uring_ctx, ret); + kfree(priv); + return ret; + } + + return -EIOCBQUEUED; + } else { + if (!refcount_dec_and_test(&priv->pending_refs)) + wait_for_completion_io(&sync_reads); + /* See btrfs_encoded_read_endio() for ordering. */ + return blk_status_to_errno(READ_ONCE(priv->status)); + } +} + +ssize_t btrfs_encoded_read_regular(struct kiocb *iocb, struct iov_iter *iter, + u64 start, u64 lockend, + struct extent_state **cached_state, + u64 disk_bytenr, u64 disk_io_size, + size_t count, bool compressed, bool *unlocked) +{ + struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp)); + struct extent_io_tree *io_tree = &inode->io_tree; + struct page **pages; + unsigned long nr_pages, i; + u64 cur; + size_t page_offset; + ssize_t ret; + + nr_pages = DIV_ROUND_UP(disk_io_size, PAGE_SIZE); + pages = kzalloc_objs(struct page *, nr_pages, GFP_NOFS); + if (!pages) + return -ENOMEM; + ret = btrfs_alloc_page_array(nr_pages, pages, GFP_NOFS); + if (ret) { + ret = -ENOMEM; + goto out; + } + + ret = btrfs_encoded_read_regular_fill_pages(inode, disk_bytenr, + disk_io_size, pages, NULL); + if (ret) + goto out; + + btrfs_unlock_extent(io_tree, start, lockend, cached_state); + btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); + *unlocked = true; + + if (compressed) { + i = 0; + page_offset = 0; + } else { + i = (iocb->ki_pos - start) >> PAGE_SHIFT; + page_offset = (iocb->ki_pos - start) & (PAGE_SIZE - 1); + } + cur = 0; + while (cur < count) { + size_t bytes = min_t(size_t, count - cur, + PAGE_SIZE - page_offset); + + if (copy_page_to_iter(pages[i], page_offset, bytes, + iter) != bytes) { + ret = -EFAULT; + goto out; + } + i++; + cur += bytes; + page_offset = 0; + } + ret = count; +out: + for (i = 0; i < nr_pages; i++) { + if (pages[i]) + __free_page(pages[i]); + } + kfree(pages); + return ret; +} + +ssize_t btrfs_encoded_read(struct kiocb *iocb, struct iov_iter *iter, + struct btrfs_ioctl_encoded_io_args *encoded, + struct extent_state **cached_state, + u64 *disk_bytenr, u64 *disk_io_size) +{ + struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp)); + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct extent_io_tree *io_tree = &inode->io_tree; + ssize_t ret; + size_t count = iov_iter_count(iter); + u64 start, lockend; + struct extent_map *em; + const bool nowait = (iocb->ki_flags & IOCB_NOWAIT); + bool unlocked = false; + + file_accessed(iocb->ki_filp); + + ret = btrfs_inode_lock(inode, + BTRFS_ILOCK_SHARED | (nowait ? BTRFS_ILOCK_TRY : 0)); + if (ret) + return ret; + + if (iocb->ki_pos >= inode->vfs_inode.i_size) { + btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); + return 0; + } + start = ALIGN_DOWN(iocb->ki_pos, fs_info->sectorsize); + /* + * We don't know how long the extent containing iocb->ki_pos is, but if + * it's compressed we know that it won't be longer than this. + */ + lockend = start + BTRFS_MAX_UNCOMPRESSED - 1; + + if (nowait) { + struct btrfs_ordered_extent *ordered; + + if (filemap_range_needs_writeback(inode->vfs_inode.i_mapping, + start, lockend)) { + ret = -EAGAIN; + goto out_unlock_inode; + } + + if (!btrfs_try_lock_extent(io_tree, start, lockend, cached_state)) { + ret = -EAGAIN; + goto out_unlock_inode; + } + + ordered = btrfs_lookup_ordered_range(inode, start, + lockend - start + 1); + if (ordered) { + btrfs_put_ordered_extent(ordered); + btrfs_unlock_extent(io_tree, start, lockend, cached_state); + ret = -EAGAIN; + goto out_unlock_inode; + } + } else { + for (;;) { + struct btrfs_ordered_extent *ordered; + + ret = btrfs_wait_ordered_range(inode, start, + lockend - start + 1); + if (ret) + goto out_unlock_inode; + + btrfs_lock_extent(io_tree, start, lockend, cached_state); + ordered = btrfs_lookup_ordered_range(inode, start, + lockend - start + 1); + if (!ordered) + break; + btrfs_put_ordered_extent(ordered); + btrfs_unlock_extent(io_tree, start, lockend, cached_state); + cond_resched(); + } + } + + em = btrfs_get_extent(inode, NULL, start, lockend - start + 1); + if (IS_ERR(em)) { + ret = PTR_ERR(em); + goto out_unlock_extent; + } + + if (em->disk_bytenr == EXTENT_MAP_INLINE) { + u64 extent_start = em->start; + + /* + * For inline extents we get everything we need out of the + * extent item. + */ + btrfs_free_extent_map(em); + em = NULL; + ret = btrfs_encoded_read_inline(iocb, iter, start, lockend, + cached_state, extent_start, + count, encoded, &unlocked); + goto out_unlock_extent; + } + + /* + * We only want to return up to EOF even if the extent extends beyond + * that. + */ + encoded->len = min_t(u64, btrfs_extent_map_end(em), + inode->vfs_inode.i_size) - iocb->ki_pos; + if (em->disk_bytenr == EXTENT_MAP_HOLE || + (em->flags & EXTENT_FLAG_PREALLOC)) { + *disk_bytenr = EXTENT_MAP_HOLE; + count = min_t(u64, count, encoded->len); + encoded->len = count; + encoded->unencoded_len = count; + } else if (btrfs_extent_map_is_compressed(em)) { + *disk_bytenr = em->disk_bytenr; + /* + * Bail if the buffer isn't large enough to return the whole + * compressed extent. + */ + if (em->disk_num_bytes > count) { + ret = -ENOBUFS; + goto out_em; + } + *disk_io_size = em->disk_num_bytes; + count = em->disk_num_bytes; + encoded->unencoded_len = em->ram_bytes; + encoded->unencoded_offset = iocb->ki_pos - (em->start - em->offset); + ret = btrfs_encoded_io_compression_from_extent(fs_info, + btrfs_extent_map_compression(em)); + if (ret < 0) + goto out_em; + encoded->compression = ret; + } else { + *disk_bytenr = btrfs_extent_map_block_start(em) + (start - em->start); + if (encoded->len > count) + encoded->len = count; + /* + * Don't read beyond what we locked. This also limits the page + * allocations that we'll do. + */ + *disk_io_size = min(lockend + 1, iocb->ki_pos + encoded->len) - start; + count = start + *disk_io_size - iocb->ki_pos; + encoded->len = count; + encoded->unencoded_len = count; + *disk_io_size = ALIGN(*disk_io_size, fs_info->sectorsize); + } + btrfs_free_extent_map(em); + em = NULL; + + if (*disk_bytenr == EXTENT_MAP_HOLE) { + btrfs_unlock_extent(io_tree, start, lockend, cached_state); + btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); + unlocked = true; + ret = iov_iter_zero(count, iter); + if (ret != count) + ret = -EFAULT; + } else { + ret = -EIOCBQUEUED; + goto out_unlock_extent; + } + +out_em: + btrfs_free_extent_map(em); +out_unlock_extent: + /* Leave inode and extent locked if we need to do a read. */ + if (!unlocked && ret != -EIOCBQUEUED) + btrfs_unlock_extent(io_tree, start, lockend, cached_state); +out_unlock_inode: + if (!unlocked && ret != -EIOCBQUEUED) + btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); + return ret; +} + +ssize_t btrfs_do_encoded_write(struct kiocb *iocb, struct iov_iter *from, + const struct btrfs_ioctl_encoded_io_args *encoded) +{ + struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp)); + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct extent_io_tree *io_tree = &inode->io_tree; + struct extent_changeset *data_reserved = NULL; + struct extent_state *cached_state = NULL; + struct btrfs_ordered_extent *ordered; + struct btrfs_file_extent file_extent; + struct compressed_bio *cb = NULL; + int compression; + size_t orig_count; + const u32 min_folio_size = btrfs_min_folio_size(fs_info); + const u32 blocksize = fs_info->sectorsize; + u64 start, end; + u64 num_bytes, ram_bytes, disk_num_bytes; + struct btrfs_key ins; + bool extent_reserved = false; + struct extent_map *em; + ssize_t ret; + + switch (encoded->compression) { + case BTRFS_ENCODED_IO_COMPRESSION_ZLIB: + compression = BTRFS_COMPRESS_ZLIB; + break; + case BTRFS_ENCODED_IO_COMPRESSION_ZSTD: + compression = BTRFS_COMPRESS_ZSTD; + break; + case BTRFS_ENCODED_IO_COMPRESSION_LZO_4K: + case BTRFS_ENCODED_IO_COMPRESSION_LZO_8K: + case BTRFS_ENCODED_IO_COMPRESSION_LZO_16K: + case BTRFS_ENCODED_IO_COMPRESSION_LZO_32K: + case BTRFS_ENCODED_IO_COMPRESSION_LZO_64K: + /* The sector size must match for LZO. */ + if (encoded->compression - + BTRFS_ENCODED_IO_COMPRESSION_LZO_4K + 12 != + fs_info->sectorsize_bits) + return -EINVAL; + compression = BTRFS_COMPRESS_LZO; + break; + default: + return -EINVAL; + } + if (encoded->encryption != BTRFS_ENCODED_IO_ENCRYPTION_NONE) + return -EINVAL; + + /* + * Compressed extents should always have checksums, so error out if we + * have a NOCOW file or inode was created while mounted with NODATASUM. + */ + if (inode->flags & BTRFS_INODE_NODATASUM) + return -EINVAL; + + orig_count = iov_iter_count(from); + + /* The extent size must be sane. */ + if (encoded->unencoded_len > BTRFS_MAX_UNCOMPRESSED || + orig_count > BTRFS_MAX_COMPRESSED || orig_count == 0) + return -EINVAL; + + /* + * The compressed data must be smaller than the decompressed data. + * + * It's of course possible for data to compress to larger or the same + * size, but the buffered I/O path falls back to no compression for such + * data, and we don't want to break any assumptions by creating these + * extents. + * + * Note that this is less strict than the current check we have that the + * compressed data must be at least one sector smaller than the + * decompressed data. We only want to enforce the weaker requirement + * from old kernels that it is at least one byte smaller. + */ + if (orig_count >= encoded->unencoded_len) + return -EINVAL; + + /* The extent must start on a sector boundary. */ + start = iocb->ki_pos; + if (!IS_ALIGNED(start, fs_info->sectorsize)) + return -EINVAL; + + /* + * The extent must end on a sector boundary. However, we allow a write + * which ends at or extends i_size to have an unaligned length; we round + * up the extent size and set i_size to the unaligned end. + */ + if (start + encoded->len < inode->vfs_inode.i_size && + !IS_ALIGNED(start + encoded->len, fs_info->sectorsize)) + return -EINVAL; + + /* Finally, the offset in the unencoded data must be sector-aligned. */ + if (!IS_ALIGNED(encoded->unencoded_offset, fs_info->sectorsize)) + return -EINVAL; + + num_bytes = ALIGN(encoded->len, fs_info->sectorsize); + ram_bytes = ALIGN(encoded->unencoded_len, fs_info->sectorsize); + end = start + num_bytes - 1; + + /* + * If the extent cannot be inline, the compressed data on disk must be + * sector-aligned. For convenience, we extend it with zeroes if it + * isn't. + */ + disk_num_bytes = ALIGN(orig_count, fs_info->sectorsize); + + cb = btrfs_alloc_compressed_write(inode, start, num_bytes); + for (int i = 0; i * min_folio_size < disk_num_bytes; i++) { + struct folio *folio; + size_t bytes = min(min_folio_size, iov_iter_count(from)); + char *kaddr; + + folio = btrfs_alloc_compr_folio(fs_info, GFP_NOFS); + if (!folio) { + ret = -ENOMEM; + goto out_cb; + } + kaddr = kmap_local_folio(folio, 0); + ret = copy_from_iter(kaddr, bytes, from); + kunmap_local(kaddr); + if (ret != bytes) { + folio_put(folio); + ret = -EFAULT; + goto out_cb; + } + if (!IS_ALIGNED(bytes, blocksize)) + folio_zero_range(folio, bytes, round_up(bytes, blocksize) - bytes); + ret = bio_add_folio(&cb->bbio.bio, folio, round_up(bytes, blocksize), 0); + if (unlikely(!ret)) { + folio_put(folio); + ret = -EINVAL; + goto out_cb; + } + } + ASSERT(cb->bbio.bio.bi_iter.bi_size == disk_num_bytes); + + for (;;) { + ret = btrfs_wait_ordered_range(inode, start, num_bytes); + if (ret) + goto out_cb; + ret = invalidate_inode_pages2_range(inode->vfs_inode.i_mapping, + start >> PAGE_SHIFT, + end >> PAGE_SHIFT); + if (ret) + goto out_cb; + btrfs_lock_extent(io_tree, start, end, &cached_state); + ordered = btrfs_lookup_ordered_range(inode, start, num_bytes); + if (!ordered && + !filemap_range_has_page(inode->vfs_inode.i_mapping, start, end)) + break; + if (ordered) + btrfs_put_ordered_extent(ordered); + btrfs_unlock_extent(io_tree, start, end, &cached_state); + cond_resched(); + } + + /* + * We don't use the higher-level delalloc space functions because our + * num_bytes and disk_num_bytes are different. + */ + ret = btrfs_alloc_data_chunk_ondemand(inode, disk_num_bytes); + if (ret) + goto out_unlock; + ret = btrfs_qgroup_reserve_data(inode, &data_reserved, start, num_bytes); + if (ret) + goto out_free_data_space; + ret = btrfs_delalloc_reserve_metadata(inode, num_bytes, disk_num_bytes, + false); + if (ret) + goto out_qgroup_free_data; + + /* Try an inline extent first. */ + if (encoded->unencoded_len == encoded->len && + encoded->unencoded_offset == 0 && + can_cow_file_range_inline(inode, start, encoded->len, orig_count)) { + ret = __cow_file_range_inline(inode, encoded->len, + orig_count, compression, + bio_first_folio_all(&cb->bbio.bio), + true); + if (ret <= 0) { + if (ret == 0) + ret = orig_count; + goto out_delalloc_release; + } + } + + ret = btrfs_reserve_extent(root, disk_num_bytes, disk_num_bytes, + disk_num_bytes, 0, 0, &ins, true, true); + if (ret) + goto out_delalloc_release; + extent_reserved = true; + + file_extent.disk_bytenr = ins.objectid; + file_extent.disk_num_bytes = ins.offset; + file_extent.num_bytes = num_bytes; + file_extent.ram_bytes = ram_bytes; + file_extent.offset = encoded->unencoded_offset; + file_extent.compression = compression; + em = btrfs_create_io_em(inode, start, &file_extent, BTRFS_ORDERED_COMPRESSED); + if (IS_ERR(em)) { + ret = PTR_ERR(em); + goto out_free_reserved; + } + btrfs_free_extent_map(em); + + ordered = btrfs_alloc_ordered_extent(inode, start, &file_extent, + (1U << BTRFS_ORDERED_ENCODED) | + (1U << BTRFS_ORDERED_COMPRESSED)); + if (IS_ERR(ordered)) { + btrfs_drop_extent_map_range(inode, start, end, false); + ret = PTR_ERR(ordered); + goto out_free_reserved; + } + btrfs_dec_block_group_reservations(fs_info, ins.objectid); + + if (start + encoded->len > inode->vfs_inode.i_size) + i_size_write(&inode->vfs_inode, start + encoded->len); + + btrfs_unlock_extent(io_tree, start, end, &cached_state); + + btrfs_delalloc_release_extents(inode, num_bytes); + + btrfs_submit_compressed_write(ordered, cb); + ret = orig_count; + goto out; + +out_free_reserved: + btrfs_dec_block_group_reservations(fs_info, ins.objectid); + btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, true); +out_delalloc_release: + btrfs_delalloc_release_extents(inode, num_bytes); + btrfs_delalloc_release_metadata(inode, disk_num_bytes, ret < 0); +out_qgroup_free_data: + if (ret < 0) + btrfs_qgroup_free_data(inode, data_reserved, start, num_bytes, NULL); +out_free_data_space: + /* + * If btrfs_reserve_extent() succeeded, then we already decremented + * bytes_may_use. + */ + if (!extent_reserved) + btrfs_free_reserved_data_space_noquota(inode, disk_num_bytes); +out_unlock: + btrfs_unlock_extent(io_tree, start, end, &cached_state); +out_cb: + if (cb) + cleanup_compressed_bio(cb); +out: + extent_changeset_free(data_reserved); + if (ret >= 0) + iocb->ki_pos += encoded->len; + return ret; +} + +#ifdef CONFIG_SWAP +/* + * Add an entry indicating a block group or device which is pinned by a + * swapfile. Returns 0 on success, 1 if there is already an entry for it, or a + * negative errno on failure. + */ +static int btrfs_add_swapfile_pin(struct inode *inode, void *ptr, + bool is_block_group) +{ + struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; + struct btrfs_swapfile_pin *sp, *entry; + struct rb_node **p; + struct rb_node *parent = NULL; + + sp = kmalloc_obj(*sp, GFP_NOFS); + if (!sp) + return -ENOMEM; + sp->ptr = ptr; + sp->inode = inode; + sp->is_block_group = is_block_group; + sp->bg_extent_count = 1; + + spin_lock(&fs_info->swapfile_pins_lock); + p = &fs_info->swapfile_pins.rb_node; + while (*p) { + parent = *p; + entry = rb_entry(parent, struct btrfs_swapfile_pin, node); + if (sp->ptr < entry->ptr || + (sp->ptr == entry->ptr && sp->inode < entry->inode)) { + p = &(*p)->rb_left; + } else if (sp->ptr > entry->ptr || + (sp->ptr == entry->ptr && sp->inode > entry->inode)) { + p = &(*p)->rb_right; + } else { + if (is_block_group) + entry->bg_extent_count++; + spin_unlock(&fs_info->swapfile_pins_lock); + kfree(sp); + return 1; + } + } + rb_link_node(&sp->node, parent, p); + rb_insert_color(&sp->node, &fs_info->swapfile_pins); + spin_unlock(&fs_info->swapfile_pins_lock); + return 0; +} + +/* Free all of the entries pinned by this swapfile. */ +static void btrfs_free_swapfile_pins(struct inode *inode) +{ + struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; + struct btrfs_swapfile_pin *sp; + struct rb_node *node, *next; + u64 bg_bytes_released = 0; + u32 bg_nr_released = 0; + + spin_lock(&fs_info->swapfile_pins_lock); + node = rb_first(&fs_info->swapfile_pins); + while (node) { + next = rb_next(node); + sp = rb_entry(node, struct btrfs_swapfile_pin, node); + if (sp->inode == inode) { + rb_erase(&sp->node, &fs_info->swapfile_pins); + if (sp->is_block_group) { + struct btrfs_block_group *bg = sp->ptr; + + bg_bytes_released += bg->length; + bg_nr_released++; + btrfs_dec_block_group_swap_extents(bg, + sp->bg_extent_count); + btrfs_put_block_group(bg); + } + kfree(sp); + } + node = next; + } + spin_unlock(&fs_info->swapfile_pins_lock); + btrfs_info(fs_info, +"swapfile deactivated on root %llu ino %llu, released %llu bytes from %u block group(s)", + btrfs_root_id(BTRFS_I(inode)->root), + btrfs_ino(BTRFS_I(inode)), bg_bytes_released, + bg_nr_released); +} + +struct btrfs_swap_info { + u64 start; + u64 block_start; + u64 block_len; + u64 lowest_ppage; + u64 highest_ppage; + unsigned long nr_pages; + int nr_extents; +}; + +static int btrfs_add_swap_extent(struct swap_info_struct *sis, + struct btrfs_swap_info *bsi) +{ + unsigned long nr_pages; + unsigned long max_pages; + u64 first_ppage, first_ppage_reported, next_ppage; + int ret; + + /* + * Our swapfile may have had its size extended after the swap header was + * written. In that case activating the swapfile should not go beyond + * the max size set in the swap header. + */ + if (bsi->nr_pages >= sis->max) + return 0; + + max_pages = sis->max - bsi->nr_pages; + first_ppage = PAGE_ALIGN(bsi->block_start) >> PAGE_SHIFT; + next_ppage = PAGE_ALIGN_DOWN(bsi->block_start + bsi->block_len) >> PAGE_SHIFT; + + if (first_ppage >= next_ppage) + return 0; + nr_pages = next_ppage - first_ppage; + nr_pages = min(nr_pages, max_pages); + + first_ppage_reported = first_ppage; + if (bsi->start == 0) + first_ppage_reported++; + if (bsi->lowest_ppage > first_ppage_reported) + bsi->lowest_ppage = first_ppage_reported; + if (bsi->highest_ppage < (next_ppage - 1)) + bsi->highest_ppage = next_ppage - 1; + + ret = add_swap_extent(sis, bsi->nr_pages, nr_pages, first_ppage); + if (ret < 0) + return ret; + bsi->nr_extents += ret; + bsi->nr_pages += nr_pages; + return 0; +} + +static void btrfs_swap_deactivate(struct file *file) +{ + struct inode *inode = file_inode(file); + + btrfs_free_swapfile_pins(inode); + atomic_dec(&BTRFS_I(inode)->root->nr_swapfiles); +} + +static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file, + sector_t *span) +{ + struct inode *inode = file_inode(file); + struct btrfs_root *root = BTRFS_I(inode)->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; + struct extent_state *cached_state = NULL; + struct btrfs_chunk_map *map = NULL; + struct btrfs_device *device = NULL; + struct btrfs_swap_info bsi = { + .lowest_ppage = (sector_t)-1ULL, + }; + struct btrfs_backref_share_check_ctx *backref_ctx = NULL; + struct btrfs_path *path = NULL; + int ret = 0; + u32 pinned_bg_nr = 0; + u64 isize; + u64 prev_extent_end = 0; + u64 pinned_bg_size = 0; + + /* + * Acquire the inode's mmap lock to prevent races with memory mapped + * writes, as they could happen after we flush delalloc below and before + * we lock the extent range further below. The inode was already locked + * up in the call chain. + */ + btrfs_assert_inode_locked(BTRFS_I(inode)); + down_write(&BTRFS_I(inode)->i_mmap_lock); + + /* + * If the swap file was just created, make sure delalloc is done. If the + * file changes again after this, the user is doing something stupid and + * we don't really care. + */ + ret = btrfs_wait_ordered_range(BTRFS_I(inode), 0, (u64)-1); + if (ret) + goto out_unlock_mmap; + + /* + * The inode is locked, so these flags won't change after we check them. + */ + if (BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS) { + btrfs_warn(fs_info, "swapfile must not be compressed"); + ret = -EINVAL; + goto out_unlock_mmap; + } + if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW)) { + btrfs_warn(fs_info, "swapfile must not be copy-on-write"); + ret = -EINVAL; + goto out_unlock_mmap; + } + if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)) { + btrfs_warn(fs_info, "swapfile must not be checksummed"); + ret = -EINVAL; + goto out_unlock_mmap; + } + + path = btrfs_alloc_path(); + backref_ctx = btrfs_alloc_backref_share_check_ctx(); + if (!path || !backref_ctx) { + ret = -ENOMEM; + goto out_unlock_mmap; + } + + /* + * Balance or device remove/replace/resize can move stuff around from + * under us. The exclop protection makes sure they aren't running/won't + * run concurrently while we are mapping the swap extents, and + * fs_info->swapfile_pins prevents them from running while the swap + * file is active and moving the extents. Note that this also prevents + * a concurrent device add which isn't actually necessary, but it's not + * really worth the trouble to allow it. + */ + if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_SWAP_ACTIVATE)) { + btrfs_warn(fs_info, + "cannot activate swapfile while exclusive operation is running"); + ret = -EBUSY; + goto out_unlock_mmap; + } + + /* + * Prevent snapshot creation while we are activating the swap file. + * We do not want to race with snapshot creation. If snapshot creation + * already started before we bumped nr_swapfiles from 0 to 1 and + * completes before the first write into the swap file after it is + * activated, than that write would fallback to COW. + */ + if (!btrfs_drew_try_write_lock(&root->snapshot_lock)) { + btrfs_exclop_finish(fs_info); + btrfs_warn(fs_info, + "cannot activate swapfile because snapshot creation is in progress"); + ret = -EINVAL; + goto out_unlock_mmap; + } + /* + * Snapshots can create extents which require COW even if NODATACOW is + * set. We use this counter to prevent snapshots. We must increment it + * before walking the extents because we don't want a concurrent + * snapshot to run after we've already checked the extents. + * + * It is possible that subvolume is marked for deletion but still not + * removed yet. To prevent this race, we check the root status before + * activating the swapfile. + */ + spin_lock(&root->root_item_lock); + if (btrfs_root_dead(root)) { + spin_unlock(&root->root_item_lock); + + btrfs_drew_write_unlock(&root->snapshot_lock); + btrfs_exclop_finish(fs_info); + btrfs_warn(fs_info, + "cannot activate swapfile because subvolume %llu is being deleted", + btrfs_root_id(root)); + ret = -EPERM; + goto out_unlock_mmap; + } + atomic_inc(&root->nr_swapfiles); + spin_unlock(&root->root_item_lock); + + isize = ALIGN_DOWN(inode->i_size, fs_info->sectorsize); + + btrfs_lock_extent(io_tree, 0, isize - 1, &cached_state); + while (prev_extent_end < isize) { + struct btrfs_key key; + struct extent_buffer *leaf; + struct btrfs_file_extent_item *ei; + struct btrfs_block_group *bg; + u64 logical_block_start; + u64 physical_block_start; + u64 extent_gen; + u64 disk_bytenr; + u64 len; + + key.objectid = btrfs_ino(BTRFS_I(inode)); + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = prev_extent_end; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + goto out; + + /* + * If key not found it means we have an implicit hole (NO_HOLES + * is enabled). + */ + if (ret > 0) { + btrfs_warn(fs_info, "swapfile must not have holes"); + ret = -EINVAL; + goto out; + } + + leaf = path->nodes[0]; + ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); + + if (btrfs_file_extent_type(leaf, ei) == BTRFS_FILE_EXTENT_INLINE) { + /* + * It's unlikely we'll ever actually find ourselves + * here, as a file small enough to fit inline won't be + * big enough to store more than the swap header, but in + * case something changes in the future, let's catch it + * here rather than later. + */ + btrfs_warn(fs_info, "swapfile must not be inline"); + ret = -EINVAL; + goto out; + } + + if (btrfs_file_extent_compression(leaf, ei) != BTRFS_COMPRESS_NONE) { + btrfs_warn(fs_info, "swapfile must not be compressed"); + ret = -EINVAL; + goto out; + } + + disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei); + if (disk_bytenr == 0) { + btrfs_warn(fs_info, "swapfile must not have holes"); + ret = -EINVAL; + goto out; + } + + logical_block_start = disk_bytenr + btrfs_file_extent_offset(leaf, ei); + extent_gen = btrfs_file_extent_generation(leaf, ei); + prev_extent_end = btrfs_file_extent_end(path); + + if (prev_extent_end > isize) + len = isize - key.offset; + else + len = btrfs_file_extent_num_bytes(leaf, ei); + + backref_ctx->curr_leaf_bytenr = leaf->start; + + /* + * Don't need the path anymore, release to avoid deadlocks when + * calling btrfs_is_data_extent_shared() because when joining a + * transaction it can block waiting for the current one's commit + * which in turn may be trying to lock the same leaf to flush + * delayed items for example. + */ + btrfs_release_path(path); + + ret = btrfs_is_data_extent_shared(BTRFS_I(inode), disk_bytenr, + extent_gen, backref_ctx); + if (ret < 0) { + goto out; + } else if (ret > 0) { + btrfs_warn(fs_info, + "swapfile must not be copy-on-write"); + ret = -EINVAL; + goto out; + } + + map = btrfs_get_chunk_map(fs_info, logical_block_start, len); + if (IS_ERR(map)) { + ret = PTR_ERR(map); + goto out; + } + + if (map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) { + btrfs_warn(fs_info, + "swapfile must have single data profile"); + ret = -EINVAL; + goto out; + } + + if (device == NULL) { + device = map->stripes[0].dev; + ret = btrfs_add_swapfile_pin(inode, device, false); + if (ret == 1) + ret = 0; + else if (ret) + goto out; + } else if (device != map->stripes[0].dev) { + btrfs_warn(fs_info, "swapfile must be on one device"); + ret = -EINVAL; + goto out; + } + + physical_block_start = (map->stripes[0].physical + + (logical_block_start - map->start)); + btrfs_free_chunk_map(map); + map = NULL; + + bg = btrfs_lookup_block_group(fs_info, logical_block_start); + if (!bg) { + btrfs_warn(fs_info, + "could not find block group containing swapfile"); + ret = -EINVAL; + goto out; + } + + if (!btrfs_inc_block_group_swap_extents(bg)) { + btrfs_warn(fs_info, + "block group for swapfile at %llu is read-only%s", + bg->start, + atomic_read(&fs_info->scrubs_running) ? + " (scrub running)" : ""); + btrfs_put_block_group(bg); + ret = -EINVAL; + goto out; + } + + ret = btrfs_add_swapfile_pin(inode, bg, true); + if (ret) { + btrfs_put_block_group(bg); + if (ret == 1) + ret = 0; + else + goto out; + } else { + pinned_bg_size += bg->length; + pinned_bg_nr++; + } + + if (bsi.block_len && + bsi.block_start + bsi.block_len == physical_block_start) { + bsi.block_len += len; + } else { + if (bsi.block_len) { + ret = btrfs_add_swap_extent(sis, &bsi); + if (ret) + goto out; + } + bsi.start = key.offset; + bsi.block_start = physical_block_start; + bsi.block_len = len; + } + + if (fatal_signal_pending(current)) { + ret = -EINTR; + goto out; + } + + cond_resched(); + } + + if (bsi.block_len) + ret = btrfs_add_swap_extent(sis, &bsi); + +out: + if (!IS_ERR_OR_NULL(map)) + btrfs_free_chunk_map(map); + + btrfs_unlock_extent(io_tree, 0, isize - 1, &cached_state); + + if (ret) + btrfs_swap_deactivate(file); + + btrfs_drew_write_unlock(&root->snapshot_lock); + + btrfs_exclop_finish(fs_info); + +out_unlock_mmap: + up_write(&BTRFS_I(inode)->i_mmap_lock); + btrfs_free_backref_share_ctx(backref_ctx); + btrfs_free_path(path); + if (ret) + return ret; + + btrfs_info(fs_info, +"swapfile activated on root %llu ino %llu, pinned down %llu bytes from %u block group(s)", + btrfs_root_id(BTRFS_I(inode)->root), + btrfs_ino(BTRFS_I(inode)), + pinned_bg_size, pinned_bg_nr); + btrfs_warn(fs_info, +"block groups with swapfile extents will not be scrubbed or balanced"); + + if (device) + sis->bdev = device->bdev; + *span = bsi.highest_ppage - bsi.lowest_ppage + 1; + sis->max = bsi.nr_pages; + sis->pages = bsi.nr_pages - 1; + return bsi.nr_extents; +} +#else +static void btrfs_swap_deactivate(struct file *file) +{ +} + +static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file, + sector_t *span) +{ + return -EOPNOTSUPP; +} +#endif + +/* + * Update the number of bytes used in the VFS' inode. When we replace extents in + * a range (clone, dedupe, fallocate's zero range), we must update the number of + * bytes used by the inode in an atomic manner, so that concurrent stat(2) calls + * always get a correct value. + */ +void btrfs_update_inode_bytes(struct btrfs_inode *inode, + const u64 add_bytes, + const u64 del_bytes) +{ + if (add_bytes == del_bytes) + return; + + spin_lock(&inode->lock); + if (del_bytes > 0) + inode_sub_bytes(&inode->vfs_inode, del_bytes); + if (add_bytes > 0) + inode_add_bytes(&inode->vfs_inode, add_bytes); + spin_unlock(&inode->lock); +} + +/* + * Verify that there are no ordered extents for a given file range. + * + * @inode: The target inode. + * @start: Start offset of the file range, should be sector size aligned. + * @end: End offset (inclusive) of the file range, its value +1 should be + * sector size aligned. + * + * This should typically be used for cases where we locked an inode's VFS lock in + * exclusive mode, we have also locked the inode's i_mmap_lock in exclusive mode, + * we have flushed all delalloc in the range, we have waited for all ordered + * extents in the range to complete and finally we have locked the file range in + * the inode's io_tree. + */ +void btrfs_assert_inode_range_clean(struct btrfs_inode *inode, u64 start, u64 end) +{ + struct btrfs_root *root = inode->root; + struct btrfs_ordered_extent *ordered; + + if (!IS_ENABLED(CONFIG_BTRFS_ASSERT)) + return; + + ordered = btrfs_lookup_first_ordered_range(inode, start, end + 1 - start); + if (ordered) { + btrfs_err(root->fs_info, +"found unexpected ordered extent in file range [%llu, %llu] for inode %llu root %llu (ordered range [%llu, %llu])", + start, end, btrfs_ino(inode), btrfs_root_id(root), + ordered->file_offset, + ordered->file_offset + ordered->num_bytes - 1); + btrfs_put_ordered_extent(ordered); + } + + ASSERT(ordered == NULL); +} + +/* + * Find the first inode with a minimum number. + * + * @root: The root to search for. + * @min_ino: The minimum inode number. + * + * Find the first inode in the @root with a number >= @min_ino and return it. + * Returns NULL if no such inode found. + */ +struct btrfs_inode *btrfs_find_first_inode(struct btrfs_root *root, u64 min_ino) +{ + struct btrfs_inode *inode; + unsigned long from = min_ino; + + xa_lock(&root->inodes); + while (true) { + inode = xa_find(&root->inodes, &from, ULONG_MAX, XA_PRESENT); + if (!inode) + break; + if (igrab(&inode->vfs_inode)) + break; + + from = btrfs_ino(inode) + 1; + xa_unlock(&root->inodes); + cond_resched(); + xa_lock(&root->inodes); + } + xa_unlock(&root->inodes); + + return inode; +} + +static const struct inode_operations btrfs_dir_inode_operations = { + .getattr = btrfs_getattr, + .lookup = btrfs_lookup, + .create = btrfs_create, + .unlink = btrfs_unlink, + .link = btrfs_link, + .mkdir = btrfs_mkdir, + .rmdir = btrfs_rmdir, + .rename = btrfs_rename2, + .symlink = btrfs_symlink, + .setattr = btrfs_setattr, + .mknod = btrfs_mknod, + .listxattr = btrfs_listxattr, + .permission = btrfs_permission, + .get_inode_acl = btrfs_get_acl, + .set_acl = btrfs_set_acl, + .update_time = btrfs_update_time, + .tmpfile = btrfs_tmpfile, + .fileattr_get = btrfs_fileattr_get, + .fileattr_set = btrfs_fileattr_set, +}; + +static const struct file_operations btrfs_dir_file_operations = { + .llseek = btrfs_dir_llseek, + .read = generic_read_dir, + .iterate_shared = btrfs_real_readdir, + .open = btrfs_opendir, + .unlocked_ioctl = btrfs_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = btrfs_compat_ioctl, +#endif + .release = btrfs_release_file, + .fsync = btrfs_sync_file, + .setlease = generic_setlease, +}; + +/* + * The folio is going dirty without a btrfs delalloc space reservation. + * This requires a fixup before writeback which we might sleep so cannot + * run in this context, so we merely set state on the folio indicating it + * needs fixup before writeback. + * + * Note that there is no range in the input, so the whole folio is marked + * dirty and fixup. + * + * We believe that all callers of dirty_folio either: + * - take the folio lock (e.g. pinned folio release notification). + * - take the pte lock but must be running on a dirty pte which means + * page_mkwrite() ran on it and reserved the space. zap_pte_range() cannot + * race with writeback cleaning the folio because writeback runs + * folio_mkclean() which also uses the pte lock and revokes outstanding + * writable mappings. + * Therefore, an additional folio private lock (a la bfs->lock for all cases, + * not just subpage) is not necessary. + */ +static bool btrfs_data_dirty_folio(struct address_space *mapping, + struct folio *folio) +{ + struct btrfs_inode *inode = BTRFS_I(mapping->host); + struct btrfs_fs_info *fs_info = inode->root->fs_info; + const u64 page_start = folio_pos(folio); + const u64 range_end = min_t(u64, folio_next_pos(folio), + round_up(i_size_read(&inode->vfs_inode), + fs_info->sectorsize)); + + if (range_end > page_start) + btrfs_folio_set_fixup_dirty(fs_info, folio, page_start, + range_end - page_start); + return filemap_dirty_folio(mapping, folio); +} + +/* + * btrfs doesn't support the bmap operation because swapfiles + * use bmap to make a mapping of extents in the file. They assume + * these extents won't change over the life of the file and they + * use the bmap result to do IO directly to the drive. + * + * the btrfs bmap call would return logical addresses that aren't + * suitable for IO and they also will change frequently as COW + * operations happen. So, swapfile + btrfs == corruption. + * + * For now we're avoiding this by dropping bmap. + */ +static const struct address_space_operations btrfs_aops = { + .read_folio = btrfs_read_folio, + .writepages = btrfs_writepages, + .readahead = btrfs_readahead, + .invalidate_folio = btrfs_invalidate_folio, + .launder_folio = btrfs_launder_folio, + .release_folio = btrfs_release_folio, + .migrate_folio = btrfs_migrate_folio, + .dirty_folio = btrfs_data_dirty_folio, + .error_remove_folio = generic_error_remove_folio, + .swap_activate = btrfs_swap_activate, + .swap_deactivate = btrfs_swap_deactivate, +}; + +static const struct inode_operations btrfs_file_inode_operations = { + .getattr = btrfs_getattr, + .setattr = btrfs_setattr, + .listxattr = btrfs_listxattr, + .permission = btrfs_permission, + .fiemap = btrfs_fiemap, + .get_inode_acl = btrfs_get_acl, + .set_acl = btrfs_set_acl, + .update_time = btrfs_update_time, + .fileattr_get = btrfs_fileattr_get, + .fileattr_set = btrfs_fileattr_set, +}; +static const struct inode_operations btrfs_special_inode_operations = { + .getattr = btrfs_getattr, + .setattr = btrfs_setattr, + .permission = btrfs_permission, + .listxattr = btrfs_listxattr, + .get_inode_acl = btrfs_get_acl, + .set_acl = btrfs_set_acl, + .update_time = btrfs_update_time, +}; +static const struct inode_operations btrfs_symlink_inode_operations = { + .get_link = page_get_link, + .getattr = btrfs_getattr, + .setattr = btrfs_setattr, + .permission = btrfs_permission, + .listxattr = btrfs_listxattr, + .update_time = btrfs_update_time, +}; + +const struct dentry_operations btrfs_dentry_operations = { + .d_delete = btrfs_dentry_delete, +}; diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c new file mode 100644 index 000000000..72bc9d4f7 --- /dev/null +++ b/fs/btrfs/ioctl.c @@ -0,0 +1,5704 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ctree.h" +#include "disk-io.h" +#include "export.h" +#include "transaction.h" +#include "btrfs_inode.h" +#include "volumes.h" +#include "locking.h" +#include "backref.h" +#include "send.h" +#include "dev-replace.h" +#include "props.h" +#include "sysfs.h" +#include "qgroup.h" +#include "tree-log.h" +#include "compression.h" +#include "space-info.h" +#include "block-group.h" +#include "fs.h" +#include "accessors.h" +#include "extent-tree.h" +#include "root-tree.h" +#include "defrag.h" +#include "dir-item.h" +#include "uuid-tree.h" +#include "ioctl.h" +#include "file.h" +#include "file-item.h" +#include "scrub.h" +#include "super.h" + +#ifdef CONFIG_64BIT +/* If we have a 32-bit userspace and 64-bit kernel, then the UAPI + * structures are incorrect, as the timespec structure from userspace + * is 4 bytes too small. We define these alternatives here to teach + * the kernel about the 32-bit struct packing. + */ +struct btrfs_ioctl_timespec_32 { + __u64 sec; + __u32 nsec; +} __attribute__ ((__packed__)); + +struct btrfs_ioctl_received_subvol_args_32 { + char uuid[BTRFS_UUID_SIZE]; /* in */ + __u64 stransid; /* in */ + __u64 rtransid; /* out */ + struct btrfs_ioctl_timespec_32 stime; /* in */ + struct btrfs_ioctl_timespec_32 rtime; /* out */ + __u64 flags; /* in */ + __u64 reserved[16]; /* in */ +} __attribute__ ((__packed__)); + +#define BTRFS_IOC_SET_RECEIVED_SUBVOL_32 _IOWR(BTRFS_IOCTL_MAGIC, 37, \ + struct btrfs_ioctl_received_subvol_args_32) + +struct btrfs_ioctl_get_subvol_info_args_32 { + __u64 treeid; + char name[BTRFS_VOL_NAME_MAX + 1]; + __u64 parent_id; + __u64 dirid; + __u64 generation; + __u64 flags; + __u8 uuid[BTRFS_UUID_SIZE]; + __u8 parent_uuid[BTRFS_UUID_SIZE]; + __u8 received_uuid[BTRFS_UUID_SIZE]; + __u64 ctransid; + __u64 otransid; + __u64 stransid; + __u64 rtransid; + struct btrfs_ioctl_timespec_32 ctime; + struct btrfs_ioctl_timespec_32 otime; + struct btrfs_ioctl_timespec_32 stime; + struct btrfs_ioctl_timespec_32 rtime; + __u64 reserved[8]; +} __attribute__ ((__packed__)); + +#define BTRFS_IOC_GET_SUBVOL_INFO_32 _IOR(BTRFS_IOCTL_MAGIC, 60, \ + struct btrfs_ioctl_get_subvol_info_args_32) +#endif + +#if defined(CONFIG_64BIT) && defined(CONFIG_COMPAT) +struct btrfs_ioctl_send_args_32 { + __s64 send_fd; /* in */ + __u64 clone_sources_count; /* in */ + compat_uptr_t clone_sources; /* in */ + __u64 parent_root; /* in */ + __u64 flags; /* in */ + __u32 version; /* in */ + __u8 reserved[28]; /* in */ +} __attribute__ ((__packed__)); + +#define BTRFS_IOC_SEND_32 _IOW(BTRFS_IOCTL_MAGIC, 38, \ + struct btrfs_ioctl_send_args_32) + +struct btrfs_ioctl_encoded_io_args_32 { + compat_uptr_t iov; + compat_ulong_t iovcnt; + __s64 offset; + __u64 flags; + __u64 len; + __u64 unencoded_len; + __u64 unencoded_offset; + __u32 compression; + __u32 encryption; + __u8 reserved[64]; +}; + +#define BTRFS_IOC_ENCODED_READ_32 _IOR(BTRFS_IOCTL_MAGIC, 64, \ + struct btrfs_ioctl_encoded_io_args_32) +#define BTRFS_IOC_ENCODED_WRITE_32 _IOW(BTRFS_IOCTL_MAGIC, 64, \ + struct btrfs_ioctl_encoded_io_args_32) +#endif + +/* Mask out flags that are inappropriate for the given type of inode. */ +static unsigned int btrfs_mask_fsflags_for_type(const struct inode *inode, + unsigned int flags) +{ + if (S_ISDIR(inode->i_mode)) + return flags; + else if (S_ISREG(inode->i_mode)) + return flags & ~FS_DIRSYNC_FL; + else + return flags & (FS_NODUMP_FL | FS_NOATIME_FL); +} + +/* + * Export internal inode flags to the format expected by the FS_IOC_GETFLAGS + * ioctl. + */ +static unsigned int btrfs_inode_flags_to_fsflags(const struct btrfs_inode *inode) +{ + unsigned int iflags = 0; + u32 flags = inode->flags; + u32 ro_flags = inode->ro_flags; + + if (flags & BTRFS_INODE_SYNC) + iflags |= FS_SYNC_FL; + if (flags & BTRFS_INODE_IMMUTABLE) + iflags |= FS_IMMUTABLE_FL; + if (flags & BTRFS_INODE_APPEND) + iflags |= FS_APPEND_FL; + if (flags & BTRFS_INODE_NODUMP) + iflags |= FS_NODUMP_FL; + if (flags & BTRFS_INODE_NOATIME) + iflags |= FS_NOATIME_FL; + if (flags & BTRFS_INODE_DIRSYNC) + iflags |= FS_DIRSYNC_FL; + if (flags & BTRFS_INODE_NODATACOW) + iflags |= FS_NOCOW_FL; + if (ro_flags & BTRFS_INODE_RO_VERITY) + iflags |= FS_VERITY_FL; + + if (flags & BTRFS_INODE_NOCOMPRESS) + iflags |= FS_NOCOMP_FL; + else if (flags & BTRFS_INODE_COMPRESS) + iflags |= FS_COMPR_FL; + + return iflags; +} + +/* + * Update inode->i_flags based on the btrfs internal flags. + */ +void btrfs_sync_inode_flags_to_i_flags(struct btrfs_inode *inode) +{ + unsigned int new_fl = 0; + + if (inode->flags & BTRFS_INODE_SYNC) + new_fl |= S_SYNC; + if (inode->flags & BTRFS_INODE_IMMUTABLE) + new_fl |= S_IMMUTABLE; + if (inode->flags & BTRFS_INODE_APPEND) + new_fl |= S_APPEND; + if (inode->flags & BTRFS_INODE_NOATIME) + new_fl |= S_NOATIME; + if (inode->flags & BTRFS_INODE_DIRSYNC) + new_fl |= S_DIRSYNC; + if (inode->ro_flags & BTRFS_INODE_RO_VERITY) + new_fl |= S_VERITY; + + set_mask_bits(&inode->vfs_inode.i_flags, + S_SYNC | S_APPEND | S_IMMUTABLE | S_NOATIME | S_DIRSYNC | + S_VERITY, new_fl); +} + +/* + * Check if @flags are a supported and valid set of FS_*_FL flags and that + * the old and new flags are not conflicting + */ +static int check_fsflags(unsigned int old_flags, unsigned int flags) +{ + if (flags & ~(FS_IMMUTABLE_FL | FS_APPEND_FL | \ + FS_NOATIME_FL | FS_NODUMP_FL | \ + FS_SYNC_FL | FS_DIRSYNC_FL | \ + FS_NOCOMP_FL | FS_COMPR_FL | + FS_NOCOW_FL)) + return -EOPNOTSUPP; + + /* COMPR and NOCOMP on new/old are valid */ + if ((flags & FS_NOCOMP_FL) && (flags & FS_COMPR_FL)) + return -EINVAL; + + if ((flags & FS_COMPR_FL) && (flags & FS_NOCOW_FL)) + return -EINVAL; + + /* NOCOW and compression options are mutually exclusive */ + if ((old_flags & FS_NOCOW_FL) && (flags & (FS_COMPR_FL | FS_NOCOMP_FL))) + return -EINVAL; + if ((flags & FS_NOCOW_FL) && (old_flags & (FS_COMPR_FL | FS_NOCOMP_FL))) + return -EINVAL; + + return 0; +} + +static int check_fsflags_compatible(const struct btrfs_fs_info *fs_info, + unsigned int flags) +{ + if (btrfs_is_zoned(fs_info) && (flags & FS_NOCOW_FL)) + return -EPERM; + + return 0; +} + +int btrfs_check_ioctl_vol_args_path(const struct btrfs_ioctl_vol_args *vol_args) +{ + if (memchr(vol_args->name, 0, sizeof(vol_args->name)) == NULL) + return -ENAMETOOLONG; + return 0; +} + +static int btrfs_check_ioctl_vol_args2_subvol_name(const struct btrfs_ioctl_vol_args_v2 *vol_args2) +{ + if (memchr(vol_args2->name, 0, sizeof(vol_args2->name)) == NULL) + return -ENAMETOOLONG; + return 0; +} + +/* + * Set flags/xflags from the internal inode flags. The remaining items of + * fsxattr are zeroed. + */ +int btrfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + const struct btrfs_inode *inode = BTRFS_I(d_inode(dentry)); + + fileattr_fill_flags(fa, btrfs_inode_flags_to_fsflags(inode)); + return 0; +} + +int btrfs_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa) +{ + struct btrfs_inode *inode = BTRFS_I(d_inode(dentry)); + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_trans_handle *trans; + unsigned int fsflags, old_fsflags; + int ret; + const char *comp = NULL; + u32 inode_flags; + bool prop_set = false; + + if (btrfs_root_readonly(root)) + return -EROFS; + + if (fileattr_has_fsx(fa)) + return -EOPNOTSUPP; + + fsflags = btrfs_mask_fsflags_for_type(&inode->vfs_inode, fa->flags); + old_fsflags = btrfs_inode_flags_to_fsflags(inode); + ret = check_fsflags(old_fsflags, fsflags); + if (ret) + return ret; + + ret = check_fsflags_compatible(fs_info, fsflags); + if (ret) + return ret; + + inode_flags = inode->flags; + if (fsflags & FS_SYNC_FL) + inode_flags |= BTRFS_INODE_SYNC; + else + inode_flags &= ~BTRFS_INODE_SYNC; + if (fsflags & FS_IMMUTABLE_FL) + inode_flags |= BTRFS_INODE_IMMUTABLE; + else + inode_flags &= ~BTRFS_INODE_IMMUTABLE; + if (fsflags & FS_APPEND_FL) + inode_flags |= BTRFS_INODE_APPEND; + else + inode_flags &= ~BTRFS_INODE_APPEND; + if (fsflags & FS_NODUMP_FL) + inode_flags |= BTRFS_INODE_NODUMP; + else + inode_flags &= ~BTRFS_INODE_NODUMP; + if (fsflags & FS_NOATIME_FL) + inode_flags |= BTRFS_INODE_NOATIME; + else + inode_flags &= ~BTRFS_INODE_NOATIME; + + /* If coming from FS_IOC_FSSETXATTR then skip unconverted flags */ + if (!fa->flags_valid) { + /* 1 item for the inode */ + trans = btrfs_start_transaction(root, 1); + if (IS_ERR(trans)) + return PTR_ERR(trans); + goto update_flags; + } + + if (fsflags & FS_DIRSYNC_FL) + inode_flags |= BTRFS_INODE_DIRSYNC; + else + inode_flags &= ~BTRFS_INODE_DIRSYNC; + if (fsflags & FS_NOCOW_FL) { + if (S_ISREG(inode->vfs_inode.i_mode)) { + /* + * It's safe to turn csums off here, no extents exist. + * Otherwise we want the flag to reflect the real COW + * status of the file and will not set it. + */ + if (inode->vfs_inode.i_size == 0) + inode_flags |= BTRFS_INODE_NODATACOW | + BTRFS_INODE_NODATASUM; + } else { + inode_flags |= BTRFS_INODE_NODATACOW; + } + } else { + /* We can only change NODATACOW for zero-sized regular file. */ + if (S_ISREG(inode->vfs_inode.i_mode) && (inode->vfs_inode.i_size == 0)) { + inode_flags &= ~BTRFS_INODE_NODATACOW; + /* + * There is currently no way to change NODATASUM flag + * through fileattr API. If we unconditionally keep the + * current NODATASUM flag, chattr +C then chattr -C will + * keep the NODATASUM flag, and no way to remove that + * flag. + * + * So respect the current mount option for NODATASUM flag. + */ + if (!btrfs_test_opt(fs_info, NODATASUM)) + inode_flags &= ~BTRFS_INODE_NODATASUM; + } else if (!S_ISREG(inode->vfs_inode.i_mode)) { + inode_flags &= ~BTRFS_INODE_NODATACOW; + } + } + + /* + * The COMPRESS flag can only be changed by users, while the NOCOMPRESS + * flag may be changed automatically if compression code won't make + * things smaller. + */ + if (fsflags & FS_NOCOMP_FL) { + inode_flags &= ~BTRFS_INODE_COMPRESS; + inode_flags |= BTRFS_INODE_NOCOMPRESS; + } else if (fsflags & FS_COMPR_FL) { + + if (IS_SWAPFILE(&inode->vfs_inode)) + return -ETXTBSY; + + inode_flags |= BTRFS_INODE_COMPRESS; + inode_flags &= ~BTRFS_INODE_NOCOMPRESS; + + comp = btrfs_compress_type2str(fs_info->compress_type); + if (!comp || comp[0] == 0) + comp = btrfs_compress_type2str(BTRFS_COMPRESS_ZLIB); + } else { + inode_flags &= ~(BTRFS_INODE_COMPRESS | BTRFS_INODE_NOCOMPRESS); + } + + /* + * 1 for inode item + * 1 for property + */ + trans = btrfs_start_transaction(root, 2); + if (IS_ERR(trans)) + return PTR_ERR(trans); + + if (comp) { + ret = btrfs_set_prop(trans, inode, "btrfs.compression", + comp, strlen(comp), 0); + if (ret) + goto out_end_trans; + prop_set = true; + } else { + ret = btrfs_set_prop(trans, inode, "btrfs.compression", NULL, 0, 0); + prop_set = (ret == 0); + /* If ret == -ENODATA ignore and proceed to update inode item. */ + if (ret && ret != -ENODATA) + goto out_end_trans; + } + +update_flags: + inode->flags = inode_flags; + btrfs_update_inode_mapping_flags(inode); + btrfs_sync_inode_flags_to_i_flags(inode); + inode_inc_iversion(&inode->vfs_inode); + inode_set_ctime_current(&inode->vfs_inode); + ret = btrfs_update_inode(trans, inode); + /* + * If we set a property or deleted one, we must abort if we fail to + * update the inode, to avoid persisting an inconsistent state. + */ + if (unlikely(ret && prop_set)) + btrfs_abort_transaction(trans, ret); + + out_end_trans: + btrfs_end_transaction(trans); + return ret; +} + +static int btrfs_ioctl_getversion(const struct inode *inode, int __user *arg) +{ + return put_user(inode->i_generation, arg); +} + +static noinline int btrfs_ioctl_fitrim(struct btrfs_fs_info *fs_info, + void __user *arg) +{ + struct btrfs_device *device; + struct fstrim_range range; + u64 minlen = ULLONG_MAX; + u64 num_devices = 0; + int ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + /* + * btrfs_trim_block_group() depends on space cache, which is not + * available in zoned filesystem. So, disallow fitrim on a zoned + * filesystem for now. + */ + if (btrfs_is_zoned(fs_info)) + return -EOPNOTSUPP; + + /* + * If the fs is mounted with nologreplay, which requires it to be + * mounted in RO mode as well, we can not allow discard on free space + * inside block groups, because log trees refer to extents that are not + * pinned in a block group's free space cache (pinning the extents is + * precisely the first phase of replaying a log tree). + */ + if (btrfs_test_opt(fs_info, NOLOGREPLAY)) + return -EROFS; + + rcu_read_lock(); + list_for_each_entry_rcu(device, &fs_info->fs_devices->devices, + dev_list) { + if (!device->bdev || !bdev_max_discard_sectors(device->bdev)) + continue; + num_devices++; + minlen = min_t(u64, bdev_discard_granularity(device->bdev), + minlen); + } + rcu_read_unlock(); + + if (!num_devices) + return -EOPNOTSUPP; + if (copy_from_user(&range, arg, sizeof(range))) + return -EFAULT; + + /* + * NOTE: Don't truncate the range using super->total_bytes. Bytenr of + * block group is in the logical address space, which can be any + * sectorsize aligned bytenr in the range [0, U64_MAX]. + */ + if (range.len < fs_info->sectorsize) + return -EINVAL; + + range.minlen = max(range.minlen, minlen); + ret = btrfs_trim_fs(fs_info, &range); + + if (copy_to_user(arg, &range, sizeof(range))) + return -EFAULT; + + return ret; +} + +/* + * Calculate the number of transaction items to reserve for creating a subvolume + * or snapshot, not including the inode, directory entries, or parent directory. + */ +static unsigned int create_subvol_num_items(const struct btrfs_qgroup_inherit *inherit) +{ + /* + * 1 to add root block + * 1 to add root item + * 1 to add root ref + * 1 to add root backref + * 1 to add UUID item + * 1 to add qgroup info + * 1 to add qgroup limit + * + * Ideally the last two would only be accounted if qgroups are enabled, + * but that can change between now and the time we would insert them. + */ + unsigned int num_items = 7; + + if (inherit) { + /* 2 to add qgroup relations for each inherited qgroup */ + num_items += 2 * inherit->num_qgroups; + } + return num_items; +} + +static noinline int create_subvol(struct mnt_idmap *idmap, + struct inode *dir, struct dentry *dentry, + struct btrfs_qgroup_inherit *inherit) +{ + struct btrfs_fs_info *fs_info = inode_to_fs_info(dir); + struct btrfs_trans_handle *trans; + struct btrfs_key key; + struct btrfs_root_item AUTO_KFREE(root_item); + struct btrfs_inode_item *inode_item; + struct extent_buffer *leaf; + struct btrfs_root *root = BTRFS_I(dir)->root; + struct btrfs_root *new_root; + struct btrfs_block_rsv block_rsv; + struct timespec64 cur_time = current_time(dir); + struct btrfs_new_inode_args new_inode_args = { + .dir = dir, + .dentry = dentry, + .subvol = true, + }; + unsigned int trans_num_items; + int ret; + dev_t anon_dev; + u64 objectid; + u64 qgroup_reserved = 0; + + root_item = kzalloc(sizeof(*root_item), GFP_KERNEL); + if (!root_item) + return -ENOMEM; + + ret = btrfs_get_free_objectid(fs_info->tree_root, &objectid); + if (ret) + return ret; + + /* + * Don't create subvolume whose level is not zero. Or qgroup will be + * screwed up since it assumes subvolume qgroup's level to be 0. + */ + if (btrfs_qgroup_level(objectid)) + return -ENOSPC; + + ret = get_anon_bdev(&anon_dev); + if (ret < 0) + return ret; + + new_inode_args.inode = btrfs_new_subvol_inode(idmap, dir); + if (!new_inode_args.inode) { + ret = -ENOMEM; + goto out_anon_dev; + } + ret = btrfs_new_inode_prepare(&new_inode_args, &trans_num_items); + if (ret) + goto out_inode; + trans_num_items += create_subvol_num_items(inherit); + + btrfs_init_block_rsv(&block_rsv, BTRFS_BLOCK_RSV_TEMP); + ret = btrfs_subvolume_reserve_metadata(root, &block_rsv, + trans_num_items, false); + if (ret) + goto out_new_inode_args; + qgroup_reserved = block_rsv.qgroup_rsv_reserved; + + trans = btrfs_start_transaction(root, 0); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto out_release_rsv; + } + btrfs_qgroup_convert_reserved_meta(root, qgroup_reserved); + qgroup_reserved = 0; + trans->block_rsv = &block_rsv; + trans->bytes_reserved = block_rsv.size; + + ret = btrfs_qgroup_inherit(trans, 0, objectid, btrfs_root_id(root), inherit); + if (ret) + goto out; + + leaf = btrfs_alloc_tree_block(trans, root, 0, objectid, NULL, 0, 0, 0, + 0, BTRFS_NESTING_NORMAL); + if (IS_ERR(leaf)) { + ret = PTR_ERR(leaf); + goto out; + } + + btrfs_mark_buffer_dirty(trans, leaf); + + inode_item = &root_item->inode; + btrfs_set_stack_inode_generation(inode_item, 1); + btrfs_set_stack_inode_size(inode_item, 3); + btrfs_set_stack_inode_nlink(inode_item, 1); + btrfs_set_stack_inode_nbytes(inode_item, + fs_info->nodesize); + btrfs_set_stack_inode_mode(inode_item, S_IFDIR | 0755); + + btrfs_set_root_flags(root_item, 0); + btrfs_set_root_limit(root_item, 0); + btrfs_set_stack_inode_flags(inode_item, BTRFS_INODE_ROOT_ITEM_INIT); + + btrfs_set_root_bytenr(root_item, leaf->start); + btrfs_set_root_generation(root_item, trans->transid); + btrfs_set_root_level(root_item, 0); + btrfs_set_root_refs(root_item, 1); + btrfs_set_root_used(root_item, leaf->len); + btrfs_set_root_last_snapshot(root_item, 0); + + btrfs_set_root_generation_v2(root_item, + btrfs_root_generation(root_item)); + generate_random_guid(root_item->uuid); + btrfs_set_stack_timespec_sec(&root_item->otime, cur_time.tv_sec); + btrfs_set_stack_timespec_nsec(&root_item->otime, cur_time.tv_nsec); + root_item->ctime = root_item->otime; + btrfs_set_root_ctransid(root_item, trans->transid); + btrfs_set_root_otransid(root_item, trans->transid); + + btrfs_tree_unlock(leaf); + + btrfs_set_root_dirid(root_item, BTRFS_FIRST_FREE_OBJECTID); + + key.objectid = objectid; + key.type = BTRFS_ROOT_ITEM_KEY; + key.offset = 0; + ret = btrfs_insert_root(trans, fs_info->tree_root, &key, + root_item); + if (ret) { + int ret2; + + /* + * Since we don't abort the transaction in this case, free the + * tree block so that we don't leak space and leave the + * filesystem in an inconsistent state (an extent item in the + * extent tree with a backreference for a root that does not + * exists). + */ + btrfs_tree_lock(leaf); + btrfs_clear_buffer_dirty(trans, leaf); + btrfs_tree_unlock(leaf); + ret2 = btrfs_free_tree_block(trans, objectid, leaf, 0, 1); + if (unlikely(ret2 < 0)) + btrfs_abort_transaction(trans, ret2); + free_extent_buffer(leaf); + goto out; + } + + free_extent_buffer(leaf); + leaf = NULL; + + new_root = btrfs_get_new_fs_root(fs_info, objectid, &anon_dev); + if (IS_ERR(new_root)) { + ret = PTR_ERR(new_root); + btrfs_abort_transaction(trans, ret); + goto out; + } + /* anon_dev is owned by new_root now. */ + anon_dev = 0; + BTRFS_I(new_inode_args.inode)->root = new_root; + /* ... and new_root is owned by new_inode_args.inode now. */ + + ret = btrfs_record_root_in_trans(trans, new_root); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + + ret = btrfs_uuid_tree_add(trans, root_item->uuid, + BTRFS_UUID_KEY_SUBVOL, objectid); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + + btrfs_record_new_subvolume(trans, BTRFS_I(dir)); + + ret = btrfs_create_new_inode(trans, &new_inode_args); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + + /* + * Subvolumes have orphans cleaned on first dentry lookup. A new + * subvolume cannot have any orphans, so we should set the bit before we + * add the subvolume dentry to the dentry cache, so that it is in the + * same state as a subvolume after first lookup. + */ + set_bit(BTRFS_ROOT_ORPHAN_CLEANUP, &new_root->state); + d_instantiate_new(dentry, new_inode_args.inode); + new_inode_args.inode = NULL; + +out: + trans->block_rsv = NULL; + trans->bytes_reserved = 0; + btrfs_end_transaction(trans); +out_release_rsv: + btrfs_block_rsv_release(fs_info, &block_rsv, (u64)-1, NULL); + if (qgroup_reserved) + btrfs_qgroup_free_meta_prealloc(root, qgroup_reserved); +out_new_inode_args: + btrfs_new_inode_args_destroy(&new_inode_args); +out_inode: + iput(new_inode_args.inode); +out_anon_dev: + if (anon_dev) + free_anon_bdev(anon_dev); + + return ret; +} + +static int create_snapshot(struct btrfs_root *root, struct inode *dir, + struct dentry *dentry, bool readonly, + struct btrfs_qgroup_inherit *inherit) +{ + struct btrfs_fs_info *fs_info = inode_to_fs_info(dir); + struct inode *inode; + struct btrfs_pending_snapshot AUTO_KFREE(pending_snapshot); + unsigned int trans_num_items; + struct btrfs_trans_handle *trans; + struct btrfs_block_rsv *block_rsv; + u64 qgroup_reserved = 0; + int ret; + + /* We do not support snapshotting right now. */ + if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) { + btrfs_warn(fs_info, + "extent tree v2 doesn't support snapshotting yet"); + return -EOPNOTSUPP; + } + + if (btrfs_root_refs(&root->root_item) == 0) + return -ENOENT; + + if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) + return -EINVAL; + + if (atomic_read(&root->nr_swapfiles)) { + btrfs_warn(fs_info, + "cannot snapshot subvolume with active swapfile"); + return -ETXTBSY; + } + + pending_snapshot = kzalloc_obj(*pending_snapshot); + if (!pending_snapshot) + return -ENOMEM; + + ret = get_anon_bdev(&pending_snapshot->anon_dev); + if (ret < 0) + goto free_pending; + pending_snapshot->root_item = kzalloc_obj(struct btrfs_root_item); + pending_snapshot->path = btrfs_alloc_path(); + if (!pending_snapshot->root_item || !pending_snapshot->path) { + ret = -ENOMEM; + goto free_pending; + } + + block_rsv = &pending_snapshot->block_rsv; + btrfs_init_block_rsv(block_rsv, BTRFS_BLOCK_RSV_TEMP); + /* + * 1 to add dir item + * 1 to add dir index + * 1 to update parent inode item + */ + trans_num_items = create_subvol_num_items(inherit) + 3; + ret = btrfs_subvolume_reserve_metadata(BTRFS_I(dir)->root, block_rsv, + trans_num_items, false); + if (ret) + goto free_pending; + qgroup_reserved = block_rsv->qgroup_rsv_reserved; + + pending_snapshot->dentry = dentry; + pending_snapshot->root = root; + pending_snapshot->readonly = readonly; + pending_snapshot->dir = BTRFS_I(dir); + pending_snapshot->inherit = inherit; + + trans = btrfs_start_transaction(root, 0); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto fail; + } + ret = btrfs_record_root_in_trans(trans, BTRFS_I(dir)->root); + if (ret) { + btrfs_end_transaction(trans); + goto fail; + } + btrfs_qgroup_convert_reserved_meta(root, qgroup_reserved); + qgroup_reserved = 0; + + trans->pending_snapshot = pending_snapshot; + + ret = btrfs_commit_transaction(trans); + if (ret) + goto fail; + + ret = pending_snapshot->error; + if (ret) + goto fail; + + ret = btrfs_orphan_cleanup(pending_snapshot->snap); + if (ret) + goto fail; + + inode = btrfs_lookup_dentry(d_inode(dentry->d_parent), dentry); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + goto fail; + } + + d_instantiate(dentry, inode); + ret = 0; + pending_snapshot->anon_dev = 0; +fail: + /* Prevent double freeing of anon_dev */ + if (ret && pending_snapshot->snap) + pending_snapshot->snap->anon_dev = 0; + btrfs_put_root(pending_snapshot->snap); + btrfs_block_rsv_release(fs_info, block_rsv, (u64)-1, NULL); + if (qgroup_reserved) + btrfs_qgroup_free_meta_prealloc(root, qgroup_reserved); +free_pending: + if (pending_snapshot->anon_dev) + free_anon_bdev(pending_snapshot->anon_dev); + kfree(pending_snapshot->root_item); + btrfs_free_path(pending_snapshot->path); + + return ret; +} + +/* + * Create a new subvolume below @parent. This is largely modeled after + * sys_mkdirat and vfs_mkdir, but we only do a single component lookup + * inside this filesystem so it's quite a bit simpler. + */ +static noinline int btrfs_mksubvol(struct dentry *parent, + struct mnt_idmap *idmap, + struct qstr *qname, struct btrfs_root *snap_src, + bool readonly, + struct btrfs_qgroup_inherit *inherit) +{ + struct inode *dir = d_inode(parent); + struct btrfs_fs_info *fs_info = inode_to_fs_info(dir); + struct dentry *dentry; + struct fscrypt_str name_str = FSTR_INIT((char *)qname->name, qname->len); + int ret; + + dentry = start_creating_killable(idmap, parent, qname); + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + + ret = may_create_dentry(idmap, dir, dentry); + if (ret) + goto out_dput; + + /* + * even if this name doesn't exist, we may get hash collisions. + * check for them now when we can safely fail + */ + ret = btrfs_check_dir_item_collision(BTRFS_I(dir)->root, dir->i_ino, &name_str); + if (ret) + goto out_dput; + + down_read(&fs_info->subvol_sem); + + if (btrfs_root_refs(&BTRFS_I(dir)->root->root_item) == 0) + goto out_up_read; + + if (snap_src) + ret = create_snapshot(snap_src, dir, dentry, readonly, inherit); + else + ret = create_subvol(idmap, dir, dentry, inherit); + + if (!ret) + fsnotify_mkdir(dir, dentry); +out_up_read: + up_read(&fs_info->subvol_sem); +out_dput: + end_creating(dentry); + return ret; +} + +static noinline int btrfs_mksnapshot(struct dentry *parent, + struct mnt_idmap *idmap, + struct qstr *qname, + struct btrfs_root *root, + bool readonly, + struct btrfs_qgroup_inherit *inherit) +{ + int ret; + + /* + * Force new buffered writes to reserve space even when NOCOW is + * possible. This is to avoid later writeback (running delalloc) to + * fallback to COW mode and unexpectedly fail with ENOSPC. + */ + btrfs_drew_read_lock(&root->snapshot_lock); + + ret = btrfs_start_delalloc_snapshot(root, false); + if (ret) + goto out; + + /* + * All previous writes have started writeback in NOCOW mode, so now + * we force future writes to fallback to COW mode during snapshot + * creation. + */ + atomic_inc(&root->snapshot_force_cow); + + btrfs_wait_ordered_extents(root, U64_MAX, NULL); + + ret = btrfs_mksubvol(parent, idmap, qname, root, readonly, inherit); + + atomic_dec(&root->snapshot_force_cow); +out: + btrfs_drew_read_unlock(&root->snapshot_lock); + return ret; +} + +/* + * Try to start exclusive operation @type or cancel it if it's running. + * + * Return: + * 0 - normal mode, newly claimed op started + * >0 - normal mode, something else is running, + * return BTRFS_ERROR_DEV_EXCL_RUN_IN_PROGRESS to user space + * ECANCELED - cancel mode, successful cancel + * ENOTCONN - cancel mode, operation not running anymore + */ +static int exclop_start_or_cancel_reloc(struct btrfs_fs_info *fs_info, + enum btrfs_exclusive_operation type, bool cancel) +{ + if (!cancel) { + /* Start normal op */ + if (!btrfs_exclop_start(fs_info, type)) + return BTRFS_ERROR_DEV_EXCL_RUN_IN_PROGRESS; + /* Exclusive operation is now claimed */ + return 0; + } + + /* Cancel running op */ + if (btrfs_exclop_start_try_lock(fs_info, type)) { + /* + * This blocks any exclop finish from setting it to NONE, so we + * request cancellation. Either it runs and we will wait for it, + * or it has finished and no waiting will happen. + */ + atomic_inc(&fs_info->reloc_cancel_req); + btrfs_exclop_start_unlock(fs_info); + + if (test_bit(BTRFS_FS_RELOC_RUNNING, &fs_info->flags)) + wait_on_bit(&fs_info->flags, BTRFS_FS_RELOC_RUNNING, + TASK_INTERRUPTIBLE); + + return -ECANCELED; + } + + /* Something else is running or none */ + return -ENOTCONN; +} + +static noinline int btrfs_ioctl_resize(struct file *file, + void __user *arg) +{ + BTRFS_DEV_LOOKUP_ARGS(args); + struct btrfs_root *root = BTRFS_I(file_inode(file))->root; + struct btrfs_fs_info *fs_info = root->fs_info; + u64 new_size; + u64 old_size; + u64 devid = 1; + struct btrfs_ioctl_vol_args AUTO_KFREE(vol_args); + struct btrfs_device *device = NULL; + char *sizestr; + char *devstr = NULL; + int ret = 0; + int mod = 0; + bool cancel; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + ret = mnt_want_write_file(file); + if (ret) + return ret; + + /* + * Read the arguments before checking exclusivity to be able to + * distinguish regular resize and cancel + */ + vol_args = memdup_user(arg, sizeof(*vol_args)); + if (IS_ERR(vol_args)) { + ret = PTR_ERR(vol_args); + goto out_drop; + } + ret = btrfs_check_ioctl_vol_args_path(vol_args); + if (ret < 0) + goto out_drop; + + sizestr = vol_args->name; + cancel = (strcmp("cancel", sizestr) == 0); + ret = exclop_start_or_cancel_reloc(fs_info, BTRFS_EXCLOP_RESIZE, cancel); + if (ret) + goto out_drop; + /* Exclusive operation is now claimed */ + + devstr = strchr(sizestr, ':'); + if (devstr) { + sizestr = devstr + 1; + *devstr = '\0'; + devstr = vol_args->name; + ret = kstrtoull(devstr, 10, &devid); + if (ret) + goto out_finish; + if (!devid) { + ret = -EINVAL; + goto out_finish; + } + btrfs_info(fs_info, "resizing devid %llu", devid); + } + + args.devid = devid; + device = btrfs_find_device(fs_info->fs_devices, &args); + if (!device) { + btrfs_info(fs_info, "resizer unable to find device %llu", + devid); + ret = -ENODEV; + goto out_finish; + } + + if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { + btrfs_info(fs_info, + "resizer unable to apply on readonly device %llu", + devid); + ret = -EPERM; + goto out_finish; + } + + if (!strcmp(sizestr, "max")) + new_size = bdev_nr_bytes(device->bdev); + else { + char *retptr; + + if (sizestr[0] == '-') { + mod = -1; + sizestr++; + } else if (sizestr[0] == '+') { + mod = 1; + sizestr++; + } + new_size = memparse(sizestr, &retptr); + if (*retptr != '\0' || new_size == 0) { + ret = -EINVAL; + goto out_finish; + } + } + + if (test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) { + ret = -EPERM; + goto out_finish; + } + + old_size = btrfs_device_get_total_bytes(device); + + if (mod < 0) { + if (new_size > old_size) { + ret = -EINVAL; + goto out_finish; + } + new_size = old_size - new_size; + } else if (mod > 0) { + if (new_size > ULLONG_MAX - old_size) { + ret = -ERANGE; + goto out_finish; + } + new_size = old_size + new_size; + } + + if (new_size < SZ_256M) { + ret = -EINVAL; + goto out_finish; + } + if (new_size > bdev_nr_bytes(device->bdev)) { + ret = -EFBIG; + goto out_finish; + } + + new_size = round_down(new_size, fs_info->sectorsize); + + if (new_size > old_size) { + struct btrfs_trans_handle *trans; + + trans = btrfs_start_transaction(root, 0); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto out_finish; + } + ret = btrfs_grow_device(trans, device, new_size); + btrfs_commit_transaction(trans); + } else if (new_size < old_size) { + ret = btrfs_shrink_device(device, new_size); + } /* equal, nothing need to do */ + + if (ret == 0 && new_size != old_size) + btrfs_info(fs_info, + "resize device %s (devid %llu) from %llu to %llu", + btrfs_dev_name(device), device->devid, + old_size, new_size); +out_finish: + btrfs_exclop_finish(fs_info); +out_drop: + mnt_drop_write_file(file); + return ret; +} + +static noinline int __btrfs_ioctl_snap_create(struct file *file, + const char *name, unsigned long fd, bool subvol, + bool readonly, + struct btrfs_qgroup_inherit *inherit) +{ + int ret; + struct qstr qname = QSTR(name); + struct mnt_idmap *idmap = file_mnt_idmap(file); + + if (!S_ISDIR(file_inode(file)->i_mode)) + return -ENOTDIR; + + ret = mnt_want_write_file(file); + if (ret) + return ret; + + if (strchr(name, '/')) { + ret = -EINVAL; + goto out_drop_write; + } + + if (qname.name[0] == '.' && + (qname.len == 1 || (qname.name[1] == '.' && qname.len == 2))) { + ret = -EEXIST; + goto out_drop_write; + } + + if (subvol) { + ret = btrfs_mksubvol(file_dentry(file), idmap, &qname, NULL, + readonly, inherit); + } else { + CLASS(fd, src)(fd); + struct inode *src_inode; + if (fd_empty(src)) { + ret = -EINVAL; + goto out_drop_write; + } + + src_inode = file_inode(fd_file(src)); + if (src_inode->i_sb != file_inode(file)->i_sb) { + btrfs_info(BTRFS_I(file_inode(file))->root->fs_info, + "Snapshot src from another FS"); + ret = -EXDEV; + } else if (!inode_owner_or_capable(idmap, src_inode)) { + /* + * Subvolume creation is not restricted, but snapshots + * are limited to own subvolumes only + */ + ret = -EPERM; + } else if (btrfs_ino(BTRFS_I(src_inode)) != BTRFS_FIRST_FREE_OBJECTID) { + /* + * Snapshots must be made with the src_inode referring + * to the subvolume inode, otherwise the permission + * checking above is useless because we may have + * permission on a lower directory but not the subvol + * itself. + */ + ret = -EINVAL; + } else { + ret = btrfs_mksnapshot(file_dentry(file), idmap, &qname, + BTRFS_I(src_inode)->root, + readonly, inherit); + } + } +out_drop_write: + mnt_drop_write_file(file); + return ret; +} + +static noinline int btrfs_ioctl_snap_create(struct file *file, + void __user *arg, bool subvol) +{ + struct btrfs_ioctl_vol_args AUTO_KFREE(vol_args); + int ret; + + if (!S_ISDIR(file_inode(file)->i_mode)) + return -ENOTDIR; + + vol_args = memdup_user(arg, sizeof(*vol_args)); + if (IS_ERR(vol_args)) + return PTR_ERR(vol_args); + ret = btrfs_check_ioctl_vol_args_path(vol_args); + if (ret < 0) + return ret; + + return __btrfs_ioctl_snap_create(file, vol_args->name, vol_args->fd, subvol, + false, NULL); +} + +static noinline int btrfs_ioctl_snap_create_v2(struct file *file, + void __user *arg, bool subvol) +{ + struct btrfs_ioctl_vol_args_v2 AUTO_KFREE(vol_args); + struct btrfs_qgroup_inherit AUTO_KFREE(inherit); + int ret; + bool readonly = false; + + if (!S_ISDIR(file_inode(file)->i_mode)) + return -ENOTDIR; + + vol_args = memdup_user(arg, sizeof(*vol_args)); + if (IS_ERR(vol_args)) + return PTR_ERR(vol_args); + ret = btrfs_check_ioctl_vol_args2_subvol_name(vol_args); + if (ret < 0) + return ret; + + if (vol_args->flags & ~BTRFS_SUBVOL_CREATE_ARGS_MASK) + return -EOPNOTSUPP; + + if (vol_args->flags & BTRFS_SUBVOL_RDONLY) + readonly = true; + if (vol_args->flags & BTRFS_SUBVOL_QGROUP_INHERIT) { + struct btrfs_fs_info *fs_info = inode_to_fs_info(file_inode(file)); + + if (vol_args->size < sizeof(*inherit) || vol_args->size > PAGE_SIZE) + return -EINVAL; + + inherit = memdup_user(vol_args->qgroup_inherit, vol_args->size); + if (IS_ERR(inherit)) { + return PTR_ERR(inherit); + } + + ret = btrfs_qgroup_check_inherit(fs_info, inherit, vol_args->size); + if (ret < 0) + return ret; + } + + return __btrfs_ioctl_snap_create(file, vol_args->name, vol_args->fd, subvol, + readonly, inherit); +} + +static noinline int btrfs_ioctl_subvol_getflags(struct btrfs_inode *inode, + void __user *arg) +{ + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + int ret = 0; + u64 flags = 0; + + if (btrfs_ino(inode) != BTRFS_FIRST_FREE_OBJECTID) + return -EINVAL; + + down_read(&fs_info->subvol_sem); + if (btrfs_root_readonly(root)) + flags |= BTRFS_SUBVOL_RDONLY; + up_read(&fs_info->subvol_sem); + + if (copy_to_user(arg, &flags, sizeof(flags))) + ret = -EFAULT; + + return ret; +} + +static noinline int btrfs_ioctl_subvol_setflags(struct file *file, + void __user *arg) +{ + struct inode *inode = file_inode(file); + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + struct btrfs_root *root = BTRFS_I(inode)->root; + struct btrfs_trans_handle *trans; + u64 root_flags; + u64 flags; + int ret; + + if (!inode_owner_or_capable(file_mnt_idmap(file), inode)) + return -EPERM; + + ret = mnt_want_write_file(file); + if (ret) + return ret; + + if (btrfs_ino(BTRFS_I(inode)) != BTRFS_FIRST_FREE_OBJECTID) { + ret = -EINVAL; + goto out_drop_write; + } + + if (copy_from_user(&flags, arg, sizeof(flags))) { + ret = -EFAULT; + goto out_drop_write; + } + + if (flags & ~BTRFS_SUBVOL_RDONLY) { + ret = -EOPNOTSUPP; + goto out_drop_write; + } + + down_write(&fs_info->subvol_sem); + + /* nothing to do */ + if (!!(flags & BTRFS_SUBVOL_RDONLY) == btrfs_root_readonly(root)) + goto out_drop_sem; + + root_flags = btrfs_root_flags(&root->root_item); + if (flags & BTRFS_SUBVOL_RDONLY) { + btrfs_set_root_flags(&root->root_item, + root_flags | BTRFS_ROOT_SUBVOL_RDONLY); + } else { + /* + * Block RO -> RW transition if this subvolume is involved in + * send + */ + spin_lock(&root->root_item_lock); + if (root->send_in_progress == 0) { + btrfs_set_root_flags(&root->root_item, + root_flags & ~BTRFS_ROOT_SUBVOL_RDONLY); + spin_unlock(&root->root_item_lock); + } else { + spin_unlock(&root->root_item_lock); + btrfs_warn(fs_info, + "Attempt to set subvolume %llu read-write during send", + btrfs_root_id(root)); + ret = -EPERM; + goto out_drop_sem; + } + } + + trans = btrfs_start_transaction(root, 1); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto out_reset; + } + + ret = btrfs_update_root(trans, fs_info->tree_root, + &root->root_key, &root->root_item); + if (ret < 0) { + btrfs_end_transaction(trans); + goto out_reset; + } + + ret = btrfs_commit_transaction(trans); + +out_reset: + if (ret) + btrfs_set_root_flags(&root->root_item, root_flags); +out_drop_sem: + up_write(&fs_info->subvol_sem); +out_drop_write: + mnt_drop_write_file(file); + return ret; +} + +static noinline bool key_in_sk(const struct btrfs_key *key, + const struct btrfs_ioctl_search_key *sk) +{ + struct btrfs_key test; + int ret; + + test.objectid = sk->min_objectid; + test.type = sk->min_type; + test.offset = sk->min_offset; + + ret = btrfs_comp_cpu_keys(key, &test); + if (ret < 0) + return false; + + test.objectid = sk->max_objectid; + test.type = sk->max_type; + test.offset = sk->max_offset; + + ret = btrfs_comp_cpu_keys(key, &test); + if (ret > 0) + return false; + return true; +} + +static noinline int copy_to_sk(struct btrfs_path *path, + struct btrfs_key *key, + const struct btrfs_ioctl_search_key *sk, + u64 *buf_size, + char __user *ubuf, + unsigned long *sk_offset, + int *num_found) +{ + u64 found_transid; + struct extent_buffer *leaf; + struct btrfs_ioctl_search_header sh; + struct btrfs_key test; + unsigned long item_off; + unsigned long item_len; + int nritems; + int i; + int slot; + int ret = 0; + + leaf = path->nodes[0]; + slot = path->slots[0]; + nritems = btrfs_header_nritems(leaf); + + if (btrfs_header_generation(leaf) > sk->max_transid) { + i = nritems; + goto advance_key; + } + found_transid = btrfs_header_generation(leaf); + + for (i = slot; i < nritems; i++) { + item_off = btrfs_item_ptr_offset(leaf, i); + item_len = btrfs_item_size(leaf, i); + + btrfs_item_key_to_cpu(leaf, key, i); + if (!key_in_sk(key, sk)) + continue; + + if (sizeof(sh) + item_len > *buf_size) { + if (*num_found) + return 1; + + /* + * return one empty item back for v1, which does not + * handle -EOVERFLOW + */ + + *buf_size = sizeof(sh) + item_len; + item_len = 0; + ret = -EOVERFLOW; + } + + if (sizeof(sh) + item_len + *sk_offset > *buf_size) + return 1; + + sh.objectid = key->objectid; + sh.type = key->type; + sh.offset = key->offset; + sh.len = item_len; + sh.transid = found_transid; + + /* + * Copy search result header. If we fault then loop again so we + * can fault in the pages and -EFAULT there if there's a + * problem. Otherwise we'll fault and then copy the buffer in + * properly this next time through + */ + if (copy_to_user_nofault(ubuf + *sk_offset, &sh, sizeof(sh))) + return 0; + + *sk_offset += sizeof(sh); + + if (item_len) { + char __user *up = ubuf + *sk_offset; + /* + * Copy the item, same behavior as above, but reset the + * * sk_offset so we copy the full thing again. + */ + if (read_extent_buffer_to_user_nofault(leaf, up, + item_off, item_len)) { + *sk_offset -= sizeof(sh); + return 0; + } + + *sk_offset += item_len; + } + (*num_found)++; + + /* -EOVERFLOW from above. */ + if (ret) + return ret; + + if (*num_found >= sk->nr_items) + return 1; + } +advance_key: + ret = 0; + test.objectid = sk->max_objectid; + test.type = sk->max_type; + test.offset = sk->max_offset; + if (btrfs_comp_cpu_keys(key, &test) >= 0) + ret = 1; + else if (key->offset < (u64)-1) + key->offset++; + else if (key->type < (u8)-1) { + key->offset = 0; + key->type++; + } else if (key->objectid < (u64)-1) { + key->offset = 0; + key->type = 0; + key->objectid++; + } else + ret = 1; + + /* + * 0: all items from this leaf copied, continue with next + * 1: * more items can be copied, but unused buffer is too small + * * all items were found + * Either way, it will stops the loop which iterates to the next + * leaf + * -EOVERFLOW: item was to large for buffer + * -EFAULT: could not copy extent buffer back to userspace + */ + return ret; +} + +static noinline int search_ioctl(struct btrfs_root *root, + struct btrfs_ioctl_search_key *sk, + u64 *buf_size, + char __user *ubuf) +{ + struct btrfs_fs_info *info = root->fs_info; + struct btrfs_key key; + BTRFS_PATH_AUTO_FREE(path); + int ret; + int num_found = 0; + unsigned long sk_offset = 0; + + if (*buf_size < sizeof(struct btrfs_ioctl_search_header)) { + *buf_size = sizeof(struct btrfs_ioctl_search_header); + return -EOVERFLOW; + } + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + if (sk->tree_id == 0) { + /* Search the root that we got passed. */ + root = btrfs_grab_root(root); + } else { + /* Look up the root from the arguments. */ + root = btrfs_get_fs_root(info, sk->tree_id, true); + if (IS_ERR(root)) + return PTR_ERR(root); + } + + key.objectid = sk->min_objectid; + key.type = sk->min_type; + key.offset = sk->min_offset; + + while (1) { + /* + * Ensure that the whole user buffer is faulted in at sub-page + * granularity, otherwise the loop may live-lock. + */ + if (fault_in_subpage_writeable(ubuf + sk_offset, *buf_size - sk_offset)) { + ret = -EFAULT; + break; + } + + ret = btrfs_search_forward(root, &key, path, sk->min_transid); + if (ret) + break; + + ret = copy_to_sk(path, &key, sk, buf_size, ubuf, + &sk_offset, &num_found); + btrfs_release_path(path); + if (ret) + break; + + } + /* Normalize return values from btrfs_search_forward() and copy_to_sk(). */ + if (ret > 0) + ret = 0; + + sk->nr_items = num_found; + btrfs_put_root(root); + return ret; +} + +static noinline int btrfs_ioctl_tree_search(struct btrfs_root *root, + void __user *argp) +{ + struct btrfs_ioctl_search_args __user *uargs = argp; + struct btrfs_ioctl_search_key sk; + int ret; + u64 buf_size; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (copy_from_user(&sk, &uargs->key, sizeof(sk))) + return -EFAULT; + + buf_size = sizeof(uargs->buf); + + ret = search_ioctl(root, &sk, &buf_size, uargs->buf); + + /* + * In the origin implementation an overflow is handled by returning a + * search header with a len of zero, so reset ret. + */ + if (ret == -EOVERFLOW) + ret = 0; + + if (ret == 0 && copy_to_user(&uargs->key, &sk, sizeof(sk))) + ret = -EFAULT; + return ret; +} + +static noinline int btrfs_ioctl_tree_search_v2(struct btrfs_root *root, + void __user *argp) +{ + struct btrfs_ioctl_search_args_v2 __user *uarg = argp; + struct btrfs_ioctl_search_args_v2 args; + int ret; + u64 buf_size; + const u64 buf_limit = SZ_16M; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + /* copy search header and buffer size */ + if (copy_from_user(&args, uarg, sizeof(args))) + return -EFAULT; + + buf_size = args.buf_size; + + /* limit result size to 16MB */ + if (buf_size > buf_limit) + buf_size = buf_limit; + + ret = search_ioctl(root, &args.key, &buf_size, + (char __user *)(&uarg->buf[0])); + if (ret == 0 && copy_to_user(&uarg->key, &args.key, sizeof(args.key))) + ret = -EFAULT; + else if (ret == -EOVERFLOW && + copy_to_user(&uarg->buf_size, &buf_size, sizeof(buf_size))) + ret = -EFAULT; + + return ret; +} + +/* + * Search for an INODE_REF in a 'root' tree which identifies the path name of + * 'dirid'. When found, it sets 'name' with the path name. + */ +static noinline int btrfs_search_path_in_tree(struct btrfs_root *root, u64 dirid, char *name) +{ + struct btrfs_key key; + char *ptr; + int ret = -1; + int slot; + int len; + int total_len = 0; + struct btrfs_inode_ref *iref; + struct extent_buffer *l; + BTRFS_PATH_AUTO_FREE(path); + + if (dirid == BTRFS_FIRST_FREE_OBJECTID) { + name[0]='\0'; + return 0; + } + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + ptr = &name[BTRFS_INO_LOOKUP_PATH_MAX - 1]; + + key.objectid = dirid; + key.type = BTRFS_INODE_REF_KEY; + key.offset = (u64)-1; + + while (1) { + ret = btrfs_search_backwards(root, &key, path); + if (ret < 0) + return ret; + else if (ret > 0) + return -ENOENT; + + l = path->nodes[0]; + slot = path->slots[0]; + + iref = btrfs_item_ptr(l, slot, struct btrfs_inode_ref); + len = btrfs_inode_ref_name_len(l, iref); + ptr -= len + 1; + total_len += len + 1; + if (ptr < name) + return -ENAMETOOLONG; + + *(ptr + len) = '/'; + read_extent_buffer(l, ptr, (unsigned long)(iref + 1), len); + + if (key.offset == BTRFS_FIRST_FREE_OBJECTID) + break; + + btrfs_release_path(path); + key.objectid = key.offset; + key.offset = (u64)-1; + dirid = key.objectid; + } + memmove(name, ptr, total_len); + name[total_len] = '\0'; + + return 0; +} + +static int btrfs_search_path_in_tree_user(struct mnt_idmap *idmap, + struct inode *inode, + struct btrfs_ioctl_ino_lookup_user_args *args) +{ + struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info; + u64 upper_limit = btrfs_ino(BTRFS_I(inode)); + u64 treeid = btrfs_root_id(BTRFS_I(inode)->root); + u64 dirid = args->dirid; + unsigned long item_off; + unsigned long item_len; + struct btrfs_inode_ref *iref; + struct btrfs_root_ref *rref; + struct btrfs_root *root = NULL; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + struct extent_buffer *leaf; + char *ptr; + int slot; + int len; + int total_len = 0; + int ret; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + /* + * If the bottom subvolume does not exist directly under upper_limit, + * construct the path in from the bottom up. + */ + if (dirid != upper_limit) { + ptr = &args->path[BTRFS_INO_LOOKUP_USER_PATH_MAX - 1]; + + root = btrfs_get_fs_root(fs_info, treeid, true); + if (IS_ERR(root)) + return PTR_ERR(root); + + key.objectid = dirid; + key.type = BTRFS_INODE_REF_KEY; + key.offset = (u64)-1; + while (1) { + struct btrfs_inode *temp_inode; + + ret = btrfs_search_backwards(root, &key, path); + if (ret < 0) + goto out_put; + else if (ret > 0) { + ret = -ENOENT; + goto out_put; + } + + leaf = path->nodes[0]; + slot = path->slots[0]; + + iref = btrfs_item_ptr(leaf, slot, struct btrfs_inode_ref); + len = btrfs_inode_ref_name_len(leaf, iref); + ptr -= len + 1; + total_len += len + 1; + if (ptr < args->path) { + ret = -ENAMETOOLONG; + goto out_put; + } + + *(ptr + len) = '/'; + read_extent_buffer(leaf, ptr, + (unsigned long)(iref + 1), len); + + /* + * We don't need the path anymore, so release it and + * avoid deadlocks and lockdep warnings in case + * btrfs_iget() needs to lookup the inode from its root + * btree and lock the same leaf. + */ + btrfs_release_path(path); + temp_inode = btrfs_iget(key.offset, root); + if (IS_ERR(temp_inode)) { + ret = PTR_ERR(temp_inode); + goto out_put; + } + /* Check the read+exec permission of this directory. */ + ret = inode_permission(idmap, &temp_inode->vfs_inode, + MAY_READ | MAY_EXEC); + iput(&temp_inode->vfs_inode); + if (ret) + goto out_put; + + if (key.offset == upper_limit) + break; + if (key.objectid == BTRFS_FIRST_FREE_OBJECTID) { + ret = -EACCES; + goto out_put; + } + + key.objectid = key.offset; + key.offset = (u64)-1; + dirid = key.objectid; + } + + memmove(args->path, ptr, total_len); + args->path[total_len] = '\0'; + btrfs_put_root(root); + root = NULL; + btrfs_release_path(path); + } + + /* Get the bottom subvolume's name from ROOT_REF */ + key.objectid = treeid; + key.type = BTRFS_ROOT_REF_KEY; + key.offset = args->treeid; + ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0); + if (ret < 0) + return ret; + else if (ret > 0) + return -ENOENT; + + leaf = path->nodes[0]; + slot = path->slots[0]; + btrfs_item_key_to_cpu(leaf, &key, slot); + + item_off = btrfs_item_ptr_offset(leaf, slot); + item_len = btrfs_item_size(leaf, slot); + /* Check if dirid in ROOT_REF corresponds to passed dirid */ + rref = btrfs_item_ptr(leaf, slot, struct btrfs_root_ref); + if (args->dirid != btrfs_root_ref_dirid(leaf, rref)) + return -EINVAL; + + /* Copy subvolume's name */ + item_off += sizeof(struct btrfs_root_ref); + item_len -= sizeof(struct btrfs_root_ref); + read_extent_buffer(leaf, args->name, item_off, item_len); + args->name[item_len] = 0; + +out_put: + btrfs_put_root(root); + + return ret; +} + +static noinline int btrfs_ioctl_ino_lookup(struct btrfs_root *root, + void __user *argp) +{ + bool new_root = false; + struct btrfs_ioctl_ino_lookup_args AUTO_KFREE(args); + int ret = 0; + + args = memdup_user(argp, sizeof(*args)); + if (IS_ERR(args)) + return PTR_ERR(args); + + /* + * Unprivileged query to obtain the containing subvolume root id. The + * path is reset so it's consistent with btrfs_search_path_in_tree. + */ + if (args->treeid == 0) + args->treeid = btrfs_root_id(root); + else + new_root = true; + + if (args->objectid == BTRFS_FIRST_FREE_OBJECTID) { + args->name[0] = 0; + goto out; + } + + if (!capable(CAP_SYS_ADMIN)) { + ret = -EPERM; + goto out; + } + + if (new_root) { + root = btrfs_get_fs_root(root->fs_info, args->treeid, true); + if (IS_ERR(root)) + return PTR_ERR(root); + } + ret = btrfs_search_path_in_tree(root, args->objectid, args->name); + if (new_root) + btrfs_put_root(root); + +out: + if (ret == 0 && copy_to_user(argp, args, sizeof(*args))) + return -EFAULT; + + return ret; +} + +/* + * Version of ino_lookup ioctl (unprivileged) + * + * The main differences from ino_lookup ioctl are: + * + * 1. Read + Exec permission will be checked using inode_permission() during + * path construction. -EACCES will be returned in case of failure. + * 2. Path construction will be stopped at the inode number which corresponds + * to the fd with which this ioctl is called. If constructed path does not + * exist under fd's inode, -EACCES will be returned. + * 3. The name of bottom subvolume is also searched and filled. + */ +static int btrfs_ioctl_ino_lookup_user(struct file *file, void __user *argp) +{ + struct btrfs_ioctl_ino_lookup_user_args AUTO_KFREE(args); + struct inode *inode; + int ret; + + args = memdup_user(argp, sizeof(*args)); + if (IS_ERR(args)) + return PTR_ERR(args); + + inode = file_inode(file); + + if (args->dirid == BTRFS_FIRST_FREE_OBJECTID && + btrfs_ino(BTRFS_I(inode)) != BTRFS_FIRST_FREE_OBJECTID) { + /* + * The subvolume does not exist under fd with which this is + * called + */ + return -EACCES; + } + + ret = btrfs_search_path_in_tree_user(file_mnt_idmap(file), inode, args); + + if (ret == 0 && copy_to_user(argp, args, sizeof(*args))) + ret = -EFAULT; + + return ret; +} + +/* Get the subvolume information in BTRFS_ROOT_ITEM and BTRFS_ROOT_BACKREF */ +static int _btrfs_ioctl_get_subvol_info(struct inode *inode, + struct btrfs_ioctl_get_subvol_info_args *subvol_info) +{ + struct btrfs_fs_info *fs_info; + struct btrfs_root *root; + struct btrfs_path *path; + struct btrfs_key key; + struct btrfs_root_item *root_item; + struct btrfs_root_ref *rref; + struct extent_buffer *leaf; + unsigned long item_off; + int slot; + int ret = 0; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + fs_info = BTRFS_I(inode)->root->fs_info; + + /* Get root_item of inode's subvolume */ + key.objectid = btrfs_root_id(BTRFS_I(inode)->root); + root = btrfs_get_fs_root(fs_info, key.objectid, true); + if (IS_ERR(root)) { + ret = PTR_ERR(root); + goto out_free; + } + root_item = &root->root_item; + + subvol_info->treeid = key.objectid; + + subvol_info->generation = btrfs_root_generation(root_item); + subvol_info->flags = btrfs_root_flags(root_item); + + memcpy(subvol_info->uuid, root_item->uuid, BTRFS_UUID_SIZE); + memcpy(subvol_info->parent_uuid, root_item->parent_uuid, + BTRFS_UUID_SIZE); + memcpy(subvol_info->received_uuid, root_item->received_uuid, + BTRFS_UUID_SIZE); + + subvol_info->ctransid = btrfs_root_ctransid(root_item); + subvol_info->ctime.sec = btrfs_stack_timespec_sec(&root_item->ctime); + subvol_info->ctime.nsec = btrfs_stack_timespec_nsec(&root_item->ctime); + + subvol_info->otransid = btrfs_root_otransid(root_item); + subvol_info->otime.sec = btrfs_stack_timespec_sec(&root_item->otime); + subvol_info->otime.nsec = btrfs_stack_timespec_nsec(&root_item->otime); + + subvol_info->stransid = btrfs_root_stransid(root_item); + subvol_info->stime.sec = btrfs_stack_timespec_sec(&root_item->stime); + subvol_info->stime.nsec = btrfs_stack_timespec_nsec(&root_item->stime); + + subvol_info->rtransid = btrfs_root_rtransid(root_item); + subvol_info->rtime.sec = btrfs_stack_timespec_sec(&root_item->rtime); + subvol_info->rtime.nsec = btrfs_stack_timespec_nsec(&root_item->rtime); + + if (key.objectid != BTRFS_FS_TREE_OBJECTID) { + /* Search root tree for ROOT_BACKREF of this subvolume */ + key.type = BTRFS_ROOT_BACKREF_KEY; + key.offset = 0; + ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0); + if (ret < 0) { + goto out; + } else if (path->slots[0] >= + btrfs_header_nritems(path->nodes[0])) { + ret = btrfs_next_leaf(fs_info->tree_root, path); + if (ret < 0) { + goto out; + } else if (unlikely(ret > 0)) { + ret = -EUCLEAN; + goto out; + } + } + + leaf = path->nodes[0]; + slot = path->slots[0]; + btrfs_item_key_to_cpu(leaf, &key, slot); + if (key.objectid == subvol_info->treeid && + key.type == BTRFS_ROOT_BACKREF_KEY) { + u16 name_len; + + subvol_info->parent_id = key.offset; + + rref = btrfs_item_ptr(leaf, slot, struct btrfs_root_ref); + name_len = btrfs_root_ref_name_len(leaf, rref); + subvol_info->dirid = btrfs_root_ref_dirid(leaf, rref); + + item_off = btrfs_item_ptr_offset(leaf, slot) + sizeof(*rref); + read_extent_buffer(leaf, subvol_info->name, + item_off, name_len); + } else { + ret = -ENOENT; + goto out; + } + ret = 0; + } + +out: + btrfs_put_root(root); +out_free: + btrfs_free_path(path); + return ret; +} + +#ifdef CONFIG_64BIT +static int btrfs_ioctl_get_subvol_info_32(struct inode *inode, void __user *argp) +{ + struct btrfs_ioctl_get_subvol_info_args AUTO_KFREE(subvol_info); + struct btrfs_ioctl_get_subvol_info_args_32 AUTO_KFREE(subvol_info_32); + int ret; + + subvol_info = kzalloc_obj(*subvol_info); + if (!subvol_info) + return -ENOMEM; + + subvol_info_32 = kzalloc_obj(*subvol_info_32); + if (!subvol_info_32) + return -ENOMEM; + + ret = _btrfs_ioctl_get_subvol_info(inode, subvol_info); + if (ret) + return ret; + + subvol_info_32->treeid = subvol_info->treeid; + memcpy(subvol_info_32->name, subvol_info->name, sizeof(subvol_info_32->name)); + subvol_info_32->parent_id = subvol_info->parent_id; + subvol_info_32->dirid = subvol_info->dirid; + subvol_info_32->generation = subvol_info->generation; + subvol_info_32->flags = subvol_info->flags; + memcpy(subvol_info_32->uuid, subvol_info->uuid, BTRFS_UUID_SIZE); + memcpy(subvol_info_32->parent_uuid, subvol_info->parent_uuid, BTRFS_UUID_SIZE); + memcpy(subvol_info_32->received_uuid, subvol_info->received_uuid, BTRFS_UUID_SIZE); + subvol_info_32->ctransid = subvol_info->ctransid; + subvol_info_32->otransid = subvol_info->otransid; + subvol_info_32->stransid = subvol_info->stransid; + subvol_info_32->rtransid = subvol_info->rtransid; + subvol_info_32->ctime.sec = subvol_info->ctime.sec; + subvol_info_32->ctime.nsec = subvol_info->ctime.nsec; + subvol_info_32->otime.sec = subvol_info->otime.sec; + subvol_info_32->otime.nsec = subvol_info->otime.nsec; + subvol_info_32->stime.sec = subvol_info->stime.sec; + subvol_info_32->stime.nsec = subvol_info->stime.nsec; + subvol_info_32->rtime.sec = subvol_info->rtime.sec; + subvol_info_32->rtime.nsec = subvol_info->rtime.nsec; + + if (copy_to_user(argp, subvol_info_32, sizeof(*subvol_info_32))) + ret = -EFAULT; + + return ret; +} +#endif + +static int btrfs_ioctl_get_subvol_info(struct inode *inode, void __user *argp) +{ + struct btrfs_ioctl_get_subvol_info_args AUTO_KFREE(subvol_info); + int ret; + + subvol_info = kzalloc_obj(*subvol_info); + if (!subvol_info) + return -ENOMEM; + + ret = _btrfs_ioctl_get_subvol_info(inode, subvol_info); + if (!ret && copy_to_user(argp, subvol_info, sizeof(*subvol_info))) + ret = -EFAULT; + + return ret; +} + +/* + * Return ROOT_REF information of the subvolume containing this inode + * except the subvolume name. + */ +static int btrfs_ioctl_get_subvol_rootref(struct btrfs_root *root, + void __user *argp) +{ + struct btrfs_ioctl_get_subvol_rootref_args AUTO_KFREE(rootrefs); + struct btrfs_root_ref *rref; + struct btrfs_path *path; + struct btrfs_key key; + struct extent_buffer *leaf; + u64 objectid; + int slot; + int ret; + u8 found; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + rootrefs = memdup_user(argp, sizeof(*rootrefs)); + if (IS_ERR(rootrefs)) { + btrfs_free_path(path); + return PTR_ERR(rootrefs); + } + + objectid = btrfs_root_id(root); + key.objectid = objectid; + key.type = BTRFS_ROOT_REF_KEY; + key.offset = rootrefs->min_treeid; + found = 0; + + root = root->fs_info->tree_root; + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) { + goto out; + } else if (path->slots[0] >= + btrfs_header_nritems(path->nodes[0])) { + ret = btrfs_next_leaf(root, path); + if (ret < 0) { + goto out; + } else if (unlikely(ret > 0)) { + ret = -EUCLEAN; + goto out; + } + } + while (1) { + leaf = path->nodes[0]; + slot = path->slots[0]; + + btrfs_item_key_to_cpu(leaf, &key, slot); + if (key.objectid != objectid || key.type != BTRFS_ROOT_REF_KEY) { + ret = 0; + goto out; + } + + if (found == BTRFS_MAX_ROOTREF_BUFFER_NUM) { + ret = -EOVERFLOW; + goto out; + } + + rref = btrfs_item_ptr(leaf, slot, struct btrfs_root_ref); + rootrefs->rootref[found].treeid = key.offset; + rootrefs->rootref[found].dirid = + btrfs_root_ref_dirid(leaf, rref); + found++; + + ret = btrfs_next_item(root, path); + if (ret < 0) { + goto out; + } else if (unlikely(ret > 0)) { + ret = -EUCLEAN; + goto out; + } + } + +out: + btrfs_free_path(path); + + if (!ret || ret == -EOVERFLOW) { + rootrefs->num_items = found; + /* update min_treeid for next search */ + if (found) + rootrefs->min_treeid = + rootrefs->rootref[found - 1].treeid + 1; + if (copy_to_user(argp, rootrefs, sizeof(*rootrefs))) + ret = -EFAULT; + } + + return ret; +} + +static noinline int btrfs_ioctl_snap_destroy(struct file *file, + void __user *arg, + bool destroy_v2) +{ + struct dentry *parent = file->f_path.dentry; + struct dentry *dentry; + struct inode *dir = d_inode(parent); + struct btrfs_fs_info *fs_info = inode_to_fs_info(dir); + struct inode *inode; + struct btrfs_root *root = BTRFS_I(dir)->root; + struct btrfs_root *dest = NULL; + struct btrfs_ioctl_vol_args AUTO_KFREE(vol_args); + struct btrfs_ioctl_vol_args_v2 AUTO_KFREE(vol_args2); + struct mnt_idmap *idmap = file_mnt_idmap(file); + char *subvol_name, *subvol_name_ptr = NULL; + int ret = 0; + bool destroy_parent = false; + + /* We don't support snapshots with extent tree v2 yet. */ + if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) { + btrfs_err(fs_info, + "extent tree v2 doesn't support snapshot deletion yet"); + return -EOPNOTSUPP; + } + + if (destroy_v2) { + vol_args2 = memdup_user(arg, sizeof(*vol_args2)); + if (IS_ERR(vol_args2)) + return PTR_ERR(vol_args2); + + if (vol_args2->flags & ~BTRFS_SUBVOL_DELETE_ARGS_MASK) + return -EOPNOTSUPP; + + /* + * If SPEC_BY_ID is not set, we are looking for the subvolume by + * name, same as v1 currently does. + */ + if (!(vol_args2->flags & BTRFS_SUBVOL_SPEC_BY_ID)) { + ret = btrfs_check_ioctl_vol_args2_subvol_name(vol_args2); + if (ret < 0) + return ret; + subvol_name = vol_args2->name; + + ret = mnt_want_write_file(file); + if (ret) + return ret; + } else { + struct inode *old_dir; + + if (vol_args2->subvolid < BTRFS_FIRST_FREE_OBJECTID) + return -EINVAL; + + ret = mnt_want_write_file(file); + if (ret) + return ret; + + dentry = btrfs_get_dentry(fs_info->sb, + BTRFS_FIRST_FREE_OBJECTID, + vol_args2->subvolid, 0); + if (IS_ERR(dentry)) { + ret = PTR_ERR(dentry); + goto out_drop_write; + } + + /* + * Change the default parent since the subvolume being + * deleted can be outside of the current mount point. + */ + parent = btrfs_get_parent(dentry); + + /* + * At this point dentry->d_name can point to '/' if the + * subvolume we want to destroy is outsite of the + * current mount point, so we need to release the + * current dentry and execute the lookup to return a new + * one with ->d_name pointing to the + * /subvol_name. + */ + dput(dentry); + if (IS_ERR(parent)) { + ret = PTR_ERR(parent); + goto out_drop_write; + } + old_dir = dir; + dir = d_inode(parent); + + /* + * If v2 was used with SPEC_BY_ID, a new parent was + * allocated since the subvolume can be outside of the + * current mount point. Later on we need to release this + * new parent dentry. + */ + destroy_parent = true; + + /* + * On idmapped mounts, deletion via subvolid is + * restricted to subvolumes that are immediate + * ancestors of the inode referenced by the file + * descriptor in the ioctl. Otherwise the idmapping + * could potentially be abused to delete subvolumes + * anywhere in the filesystem the user wouldn't be able + * to delete without an idmapped mount. + */ + if (old_dir != dir && idmap != &nop_mnt_idmap) { + ret = -EOPNOTSUPP; + goto free_parent; + } + + subvol_name_ptr = btrfs_get_subvol_name_from_objectid( + fs_info, vol_args2->subvolid); + if (IS_ERR(subvol_name_ptr)) { + ret = PTR_ERR(subvol_name_ptr); + goto free_parent; + } + /* subvol_name_ptr is already nul terminated */ + subvol_name = (char *)kbasename(subvol_name_ptr); + } + } else { + vol_args = memdup_user(arg, sizeof(*vol_args)); + if (IS_ERR(vol_args)) + return PTR_ERR(vol_args); + + ret = btrfs_check_ioctl_vol_args_path(vol_args); + if (ret < 0) + return ret; + + subvol_name = vol_args->name; + + ret = mnt_want_write_file(file); + if (ret) + return ret; + } + + if (strchr(subvol_name, '/') || + strcmp(subvol_name, "..") == 0) { + ret = -EINVAL; + goto free_subvol_name; + } + + if (!S_ISDIR(dir->i_mode)) { + ret = -ENOTDIR; + goto free_subvol_name; + } + + dentry = start_removing_killable(idmap, parent, &QSTR(subvol_name)); + if (IS_ERR(dentry)) { + ret = PTR_ERR(dentry); + goto out_end_removing; + } + + inode = d_inode(dentry); + dest = BTRFS_I(inode)->root; + if (!capable(CAP_SYS_ADMIN)) { + /* + * Regular user. Only allow this with a special mount + * option, when the user has write+exec access to the + * subvol root, and when rmdir(2) would have been + * allowed. + * + * Note that this is _not_ check that the subvol is + * empty or doesn't contain data that we wouldn't + * otherwise be able to delete. + * + * Users who want to delete empty subvols should try + * rmdir(2). + */ + ret = -EPERM; + if (!btrfs_test_opt(fs_info, USER_SUBVOL_RM_ALLOWED)) + goto out_end_removing; + + /* + * Do not allow deletion if the parent dir is the same + * as the dir to be deleted. That means the ioctl + * must be called on the dentry referencing the root + * of the subvol, not a random directory contained + * within it. + */ + ret = -EINVAL; + if (root == dest) + goto out_end_removing; + + ret = inode_permission(idmap, inode, MAY_WRITE | MAY_EXEC); + if (ret) + goto out_end_removing; + } + + /* check if subvolume may be deleted by a user */ + ret = may_delete_dentry(idmap, dir, dentry, true); + if (ret) + goto out_end_removing; + + if (btrfs_ino(BTRFS_I(inode)) != BTRFS_FIRST_FREE_OBJECTID) { + ret = -EINVAL; + goto out_end_removing; + } + + btrfs_inode_lock(BTRFS_I(inode), 0); + ret = btrfs_delete_subvolume(BTRFS_I(dir), dentry); + btrfs_inode_unlock(BTRFS_I(inode), 0); + if (!ret) + d_delete_notify(dir, dentry); + +out_end_removing: + end_removing(dentry); +free_subvol_name: + kfree(subvol_name_ptr); +free_parent: + if (destroy_parent) + dput(parent); +out_drop_write: + mnt_drop_write_file(file); + return ret; +} + +static int btrfs_ioctl_defrag(struct file *file, void __user *argp) +{ + struct inode *inode = file_inode(file); + struct btrfs_root *root = BTRFS_I(inode)->root; + struct btrfs_ioctl_defrag_range_args range = {0}; + int ret; + + ret = mnt_want_write_file(file); + if (ret) + return ret; + + if (btrfs_root_readonly(root)) { + ret = -EROFS; + goto out; + } + + switch (inode->i_mode & S_IFMT) { + case S_IFDIR: + if (!capable(CAP_SYS_ADMIN)) { + ret = -EPERM; + goto out; + } + ret = btrfs_defrag_root(root); + break; + case S_IFREG: + /* + * Note that this does not check the file descriptor for write + * access. This prevents defragmenting executables that are + * running and allows defrag on files open in read-only mode. + */ + if (!capable(CAP_SYS_ADMIN) && + inode_permission(&nop_mnt_idmap, inode, MAY_WRITE)) { + ret = -EPERM; + goto out; + } + + /* + * Don't allow defrag on pre-content watched files, as it could + * populate the page cache with 0's via readahead. + */ + if (unlikely(FMODE_FSNOTIFY_HSM(file->f_mode))) { + ret = -EINVAL; + goto out; + } + + if (argp) { + if (copy_from_user(&range, argp, sizeof(range))) { + ret = -EFAULT; + goto out; + } + if (range.flags & ~BTRFS_DEFRAG_RANGE_FLAGS_SUPP) { + ret = -EOPNOTSUPP; + goto out; + } + if ((range.flags & BTRFS_DEFRAG_RANGE_COMPRESS) && + (range.flags & BTRFS_DEFRAG_RANGE_NOCOMPRESS)) { + ret = -EINVAL; + goto out; + } + /* Compression or no-compression require to start the IO. */ + if ((range.flags & BTRFS_DEFRAG_RANGE_COMPRESS) || + (range.flags & BTRFS_DEFRAG_RANGE_NOCOMPRESS)) { + range.flags |= BTRFS_DEFRAG_RANGE_START_IO; + range.extent_thresh = (u32)-1; + } + } else { + /* the rest are all set to zero by kzalloc */ + range.len = (u64)-1; + } + ret = btrfs_defrag_file(BTRFS_I(file_inode(file)), &file->f_ra, + &range, BTRFS_OLDEST_GENERATION, 0); + if (ret > 0) + ret = 0; + break; + default: + ret = -EINVAL; + } +out: + mnt_drop_write_file(file); + return ret; +} + +static long btrfs_ioctl_add_dev(struct btrfs_fs_info *fs_info, void __user *arg) +{ + struct btrfs_ioctl_vol_args AUTO_KFREE(vol_args); + bool restore_op = false; + int ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) { + btrfs_err(fs_info, "device add not supported on extent tree v2 yet"); + return -EINVAL; + } + + if (fs_info->fs_devices->temp_fsid) { + btrfs_err(fs_info, + "device add not supported on cloned temp-fsid mount"); + return -EINVAL; + } + + if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_DEV_ADD)) { + if (!btrfs_exclop_start_try_lock(fs_info, BTRFS_EXCLOP_DEV_ADD)) + return BTRFS_ERROR_DEV_EXCL_RUN_IN_PROGRESS; + + /* + * We can do the device add because we have a paused balanced, + * change the exclusive op type and remember we should bring + * back the paused balance + */ + fs_info->exclusive_operation = BTRFS_EXCLOP_DEV_ADD; + btrfs_exclop_start_unlock(fs_info); + restore_op = true; + } + + vol_args = memdup_user(arg, sizeof(*vol_args)); + if (IS_ERR(vol_args)) { + ret = PTR_ERR(vol_args); + goto out; + } + + ret = btrfs_check_ioctl_vol_args_path(vol_args); + if (ret < 0) + goto out; + + ret = btrfs_init_new_device(fs_info, vol_args->name); + + if (!ret) + btrfs_info(fs_info, "disk added %s", vol_args->name); + +out: + if (restore_op) + btrfs_exclop_balance(fs_info, BTRFS_EXCLOP_BALANCE_PAUSED); + else + btrfs_exclop_finish(fs_info); + return ret; +} + +static long btrfs_ioctl_rm_dev_v2(struct file *file, void __user *arg) +{ + BTRFS_DEV_LOOKUP_ARGS(args); + struct inode *inode = file_inode(file); + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + struct btrfs_ioctl_vol_args_v2 AUTO_KFREE(vol_args); + struct file *bdev_file = NULL; + int ret; + bool cancel = false; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + vol_args = memdup_user(arg, sizeof(*vol_args)); + if (IS_ERR(vol_args)) + return PTR_ERR(vol_args); + + if (vol_args->flags & ~BTRFS_DEVICE_REMOVE_ARGS_MASK) { + ret = -EOPNOTSUPP; + goto out; + } + + ret = btrfs_check_ioctl_vol_args2_subvol_name(vol_args); + if (ret < 0) + goto out; + + if (vol_args->flags & BTRFS_DEVICE_SPEC_BY_ID) { + args.devid = vol_args->devid; + } else if (!strcmp("cancel", vol_args->name)) { + cancel = true; + } else { + ret = btrfs_get_dev_args_from_path(fs_info, &args, vol_args->name); + if (ret) + goto out; + } + + ret = mnt_want_write_file(file); + if (ret) + goto out; + + ret = exclop_start_or_cancel_reloc(fs_info, BTRFS_EXCLOP_DEV_REMOVE, + cancel); + if (ret) + goto err_drop; + + /* Exclusive operation is now claimed */ + ret = btrfs_rm_device(fs_info, &args, &bdev_file); + + btrfs_exclop_finish(fs_info); + + if (!ret) { + if (vol_args->flags & BTRFS_DEVICE_SPEC_BY_ID) + btrfs_info(fs_info, "device deleted: id %llu", + vol_args->devid); + else + btrfs_info(fs_info, "device deleted: %s", + vol_args->name); + } +err_drop: + mnt_drop_write_file(file); + if (bdev_file) + btrfs_release_device_allow_freeze(bdev_file); +out: + btrfs_put_dev_args_from_path(&args); + return ret; +} + +static long btrfs_ioctl_rm_dev(struct file *file, void __user *arg) +{ + BTRFS_DEV_LOOKUP_ARGS(args); + struct inode *inode = file_inode(file); + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + struct btrfs_ioctl_vol_args AUTO_KFREE(vol_args); + struct file *bdev_file = NULL; + int ret; + bool cancel = false; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + vol_args = memdup_user(arg, sizeof(*vol_args)); + if (IS_ERR(vol_args)) + return PTR_ERR(vol_args); + + ret = btrfs_check_ioctl_vol_args_path(vol_args); + if (ret < 0) + return ret; + + if (!strcmp("cancel", vol_args->name)) { + cancel = true; + } else { + ret = btrfs_get_dev_args_from_path(fs_info, &args, vol_args->name); + if (ret) + goto out; + } + + ret = mnt_want_write_file(file); + if (ret) + goto out; + + ret = exclop_start_or_cancel_reloc(fs_info, BTRFS_EXCLOP_DEV_REMOVE, + cancel); + if (ret == 0) { + ret = btrfs_rm_device(fs_info, &args, &bdev_file); + if (!ret) + btrfs_info(fs_info, "disk deleted %s", vol_args->name); + btrfs_exclop_finish(fs_info); + } + + mnt_drop_write_file(file); + if (bdev_file) + btrfs_release_device_allow_freeze(bdev_file); +out: + btrfs_put_dev_args_from_path(&args); + return ret; +} + +static long btrfs_ioctl_fs_info(const struct btrfs_fs_info *fs_info, + void __user *arg) +{ + struct btrfs_ioctl_fs_info_args AUTO_KFREE(fi_args); + struct btrfs_device *device; + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; + u64 flags_in; + + fi_args = memdup_user(arg, sizeof(*fi_args)); + if (IS_ERR(fi_args)) + return PTR_ERR(fi_args); + + flags_in = fi_args->flags; + memset(fi_args, 0, sizeof(*fi_args)); + + rcu_read_lock(); + fi_args->num_devices = fs_devices->num_devices; + + list_for_each_entry_rcu(device, &fs_devices->devices, dev_list) { + if (device->devid > fi_args->max_id) + fi_args->max_id = device->devid; + } + rcu_read_unlock(); + + memcpy(&fi_args->fsid, fs_devices->fsid, sizeof(fi_args->fsid)); + fi_args->nodesize = fs_info->nodesize; + fi_args->sectorsize = fs_info->sectorsize; + fi_args->clone_alignment = fs_info->sectorsize; + + if (flags_in & BTRFS_FS_INFO_FLAG_CSUM_INFO) { + fi_args->csum_type = btrfs_super_csum_type(fs_info->super_copy); + fi_args->csum_size = btrfs_super_csum_size(fs_info->super_copy); + fi_args->flags |= BTRFS_FS_INFO_FLAG_CSUM_INFO; + } + + if (flags_in & BTRFS_FS_INFO_FLAG_GENERATION) { + fi_args->generation = btrfs_get_fs_generation(fs_info); + fi_args->flags |= BTRFS_FS_INFO_FLAG_GENERATION; + } + + if (flags_in & BTRFS_FS_INFO_FLAG_METADATA_UUID) { + memcpy(&fi_args->metadata_uuid, fs_devices->metadata_uuid, + sizeof(fi_args->metadata_uuid)); + fi_args->flags |= BTRFS_FS_INFO_FLAG_METADATA_UUID; + } + + if (copy_to_user(arg, fi_args, sizeof(*fi_args))) + return -EFAULT; + + return 0; +} + +static long btrfs_ioctl_dev_info(const struct btrfs_fs_info *fs_info, + void __user *arg) +{ + BTRFS_DEV_LOOKUP_ARGS(args); + struct btrfs_ioctl_dev_info_args AUTO_KFREE(di_args); + struct btrfs_device *dev; + int ret = 0; + + di_args = memdup_user(arg, sizeof(*di_args)); + if (IS_ERR(di_args)) + return PTR_ERR(di_args); + + args.devid = di_args->devid; + if (!btrfs_is_empty_uuid(di_args->uuid)) + args.uuid = di_args->uuid; + + rcu_read_lock(); + dev = btrfs_find_device(fs_info->fs_devices, &args); + if (!dev) { + ret = -ENODEV; + goto out; + } + + di_args->devid = dev->devid; + di_args->bytes_used = btrfs_device_get_bytes_used(dev); + di_args->total_bytes = btrfs_device_get_total_bytes(dev); + memcpy(di_args->uuid, dev->uuid, sizeof(di_args->uuid)); + memcpy(di_args->fsid, dev->fs_devices->fsid, BTRFS_UUID_SIZE); + if (dev->name) + strscpy(di_args->path, btrfs_dev_name(dev), sizeof(di_args->path)); + else + di_args->path[0] = '\0'; + +out: + rcu_read_unlock(); + if (ret == 0 && copy_to_user(arg, di_args, sizeof(*di_args))) + ret = -EFAULT; + + return ret; +} + +static long btrfs_ioctl_default_subvol(struct file *file, void __user *argp) +{ + struct inode *inode = file_inode(file); + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + struct btrfs_root *root = BTRFS_I(inode)->root; + struct btrfs_root *new_root; + struct btrfs_dir_item *di; + struct btrfs_trans_handle *trans; + struct btrfs_path *path = NULL; + struct btrfs_disk_key disk_key; + struct fscrypt_str name = FSTR_INIT("default", 7); + u64 objectid = 0; + u64 dir_id; + int ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + ret = mnt_want_write_file(file); + if (ret) + return ret; + + if (copy_from_user(&objectid, argp, sizeof(objectid))) { + ret = -EFAULT; + goto out; + } + + if (!objectid) + objectid = BTRFS_FS_TREE_OBJECTID; + + new_root = btrfs_get_fs_root(fs_info, objectid, true); + if (IS_ERR(new_root)) { + ret = PTR_ERR(new_root); + goto out; + } + if (!btrfs_is_fstree(btrfs_root_id(new_root))) { + ret = -ENOENT; + goto out_free; + } + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto out_free; + } + + trans = btrfs_start_transaction(root, 1); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto out_free; + } + + dir_id = btrfs_super_root_dir(fs_info->super_copy); + di = btrfs_lookup_dir_item(trans, fs_info->tree_root, path, + dir_id, &name, 1); + if (IS_ERR_OR_NULL(di)) { + btrfs_release_path(path); + btrfs_end_transaction(trans); + if (di) + ret = PTR_ERR(di); + else + ret = -ENOENT; + btrfs_err(fs_info, + "could not find default diritem for dir %llu: %pe", + dir_id, ERR_PTR(ret)); + goto out_free; + } + + btrfs_cpu_key_to_disk(&disk_key, &new_root->root_key); + btrfs_set_dir_item_key(path->nodes[0], di, &disk_key); + btrfs_release_path(path); + + btrfs_set_fs_incompat(fs_info, DEFAULT_SUBVOL); + btrfs_end_transaction(trans); +out_free: + btrfs_put_root(new_root); + btrfs_free_path(path); +out: + mnt_drop_write_file(file); + return ret; +} + +static void get_block_group_info(struct list_head *groups_list, + struct btrfs_ioctl_space_info *space) +{ + struct btrfs_block_group *block_group; + + space->total_bytes = 0; + space->used_bytes = 0; + space->flags = 0; + list_for_each_entry(block_group, groups_list, list) { + space->flags = block_group->flags; + space->total_bytes += block_group->length; + space->used_bytes += block_group->used; + } +} + +static long btrfs_ioctl_space_info(struct btrfs_fs_info *fs_info, + void __user *arg) +{ + struct btrfs_ioctl_space_args space_args = { 0 }; + struct btrfs_ioctl_space_info space; + struct btrfs_ioctl_space_info *dest; + struct btrfs_ioctl_space_info AUTO_KFREE(dest_orig); + struct btrfs_ioctl_space_info __user *user_dest; + struct btrfs_space_info *info; + static const u64 types[] = { + BTRFS_BLOCK_GROUP_DATA, + BTRFS_BLOCK_GROUP_SYSTEM, + BTRFS_BLOCK_GROUP_METADATA, + BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_METADATA + }; + int num_types = 4; + int alloc_size; + int ret = 0; + u64 slot_count = 0; + int i, c; + + if (copy_from_user(&space_args, + (struct btrfs_ioctl_space_args __user *)arg, + sizeof(space_args))) + return -EFAULT; + + for (i = 0; i < num_types; i++) { + struct btrfs_space_info *tmp; + + info = NULL; + list_for_each_entry(tmp, &fs_info->space_info, list) { + if (tmp->flags == types[i]) { + info = tmp; + break; + } + } + + if (!info) + continue; + + down_read(&info->groups_sem); + for (c = 0; c < BTRFS_NR_RAID_TYPES; c++) { + if (!list_empty(&info->block_groups[c])) + slot_count++; + } + up_read(&info->groups_sem); + } + + /* + * Global block reserve, exported as a space_info + */ + slot_count++; + + /* space_slots == 0 means they are asking for a count */ + if (space_args.space_slots == 0) { + space_args.total_spaces = slot_count; + goto out; + } + + slot_count = min_t(u64, space_args.space_slots, slot_count); + + alloc_size = sizeof(*dest) * slot_count; + + /* we generally have at most 6 or so space infos, one for each raid + * level. So, a whole page should be more than enough for everyone + */ + if (alloc_size > PAGE_SIZE) + return -ENOMEM; + + space_args.total_spaces = 0; + dest = kzalloc(alloc_size, GFP_KERNEL); + if (!dest) + return -ENOMEM; + dest_orig = dest; + + /* now we have a buffer to copy into */ + for (i = 0; i < num_types; i++) { + struct btrfs_space_info *tmp; + + if (!slot_count) + break; + + info = NULL; + list_for_each_entry(tmp, &fs_info->space_info, list) { + if (tmp->flags == types[i]) { + info = tmp; + break; + } + } + + if (!info) + continue; + down_read(&info->groups_sem); + for (c = 0; c < BTRFS_NR_RAID_TYPES; c++) { + if (!list_empty(&info->block_groups[c])) { + get_block_group_info(&info->block_groups[c], + &space); + memcpy(dest, &space, sizeof(space)); + dest++; + space_args.total_spaces++; + slot_count--; + } + if (!slot_count) + break; + } + up_read(&info->groups_sem); + } + + /* + * Add global block reserve + */ + if (slot_count) { + struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv; + + spin_lock(&block_rsv->lock); + space.total_bytes = block_rsv->size; + space.used_bytes = block_rsv->size - block_rsv->reserved; + spin_unlock(&block_rsv->lock); + space.flags = BTRFS_SPACE_INFO_GLOBAL_RSV; + memcpy(dest, &space, sizeof(space)); + space_args.total_spaces++; + } + + user_dest = (struct btrfs_ioctl_space_info __user *) + (arg + sizeof(struct btrfs_ioctl_space_args)); + + if (copy_to_user(user_dest, dest_orig, + space_args.total_spaces * sizeof(*dest_orig))) + return -EFAULT; + +out: + if (ret == 0 && copy_to_user(arg, &space_args, sizeof(space_args))) + ret = -EFAULT; + + return ret; +} + +static noinline long btrfs_ioctl_start_sync(struct btrfs_root *root, + void __user *argp) +{ + struct btrfs_trans_handle *trans; + u64 transid; + + /* + * Start orphan cleanup here for the given root in case it hasn't been + * started already by other means. Errors are handled in the other + * functions during transaction commit. + */ + btrfs_orphan_cleanup(root); + + trans = btrfs_attach_transaction_barrier(root); + if (IS_ERR(trans)) { + if (PTR_ERR(trans) != -ENOENT) + return PTR_ERR(trans); + + /* No running transaction, don't bother */ + transid = btrfs_get_last_trans_committed(root->fs_info); + goto out; + } + transid = trans->transid; + btrfs_commit_transaction_async(trans); +out: + if (argp) + if (copy_to_user(argp, &transid, sizeof(transid))) + return -EFAULT; + return 0; +} + +static noinline long btrfs_ioctl_wait_sync(struct btrfs_fs_info *fs_info, + void __user *argp) +{ + /* By default wait for the current transaction. */ + u64 transid = 0; + + if (argp) + if (copy_from_user(&transid, argp, sizeof(transid))) + return -EFAULT; + + return btrfs_wait_for_commit(fs_info, transid); +} + +static long btrfs_ioctl_scrub(struct file *file, void __user *arg) +{ + struct btrfs_fs_info *fs_info = inode_to_fs_info(file_inode(file)); + struct btrfs_ioctl_scrub_args AUTO_KFREE(sa); + int ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) { + btrfs_err(fs_info, "scrub: extent tree v2 not yet supported"); + return -EINVAL; + } + + sa = memdup_user(arg, sizeof(*sa)); + if (IS_ERR(sa)) + return PTR_ERR(sa); + + if (sa->flags & ~BTRFS_SCRUB_SUPPORTED_FLAGS) + return -EOPNOTSUPP; + + if (!(sa->flags & BTRFS_SCRUB_READONLY)) { + ret = mnt_want_write_file(file); + if (ret) + return ret; + } + + ret = btrfs_scrub_dev(fs_info, sa->devid, sa->start, sa->end, + &sa->progress, sa->flags & BTRFS_SCRUB_READONLY, + false); + + /* + * Copy scrub args to user space even if btrfs_scrub_dev() returned an + * error. This is important as it allows user space to know how much + * progress scrub has done. For example, if scrub is canceled we get + * -ECANCELED from btrfs_scrub_dev() and return that error back to user + * space. Later user space can inspect the progress from the structure + * btrfs_ioctl_scrub_args and resume scrub from where it left off + * previously (btrfs-progs does this). + * If we fail to copy the btrfs_ioctl_scrub_args structure to user space + * then return -EFAULT to signal the structure was not copied or it may + * be corrupt and unreliable due to a partial copy. + */ + if (copy_to_user(arg, sa, sizeof(*sa))) + ret = -EFAULT; + + if (!(sa->flags & BTRFS_SCRUB_READONLY)) + mnt_drop_write_file(file); + + return ret; +} + +static long btrfs_ioctl_scrub_cancel(struct btrfs_fs_info *fs_info) +{ + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + return btrfs_scrub_cancel(fs_info); +} + +static long btrfs_ioctl_scrub_progress(struct btrfs_fs_info *fs_info, + void __user *arg) +{ + struct btrfs_ioctl_scrub_args AUTO_KFREE(sa); + int ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + sa = memdup_user(arg, sizeof(*sa)); + if (IS_ERR(sa)) + return PTR_ERR(sa); + + ret = btrfs_scrub_progress(fs_info, sa->devid, &sa->progress); + + if (ret == 0 && copy_to_user(arg, sa, sizeof(*sa))) + return -EFAULT; + + return ret; +} + +static long btrfs_ioctl_get_dev_stats(struct btrfs_fs_info *fs_info, + void __user *arg) +{ + struct btrfs_ioctl_get_dev_stats AUTO_KFREE(sa); + int ret; + + sa = memdup_user(arg, sizeof(*sa)); + if (IS_ERR(sa)) + return PTR_ERR(sa); + + if ((sa->flags & BTRFS_DEV_STATS_RESET) && !capable(CAP_SYS_ADMIN)) + return -EPERM; + + ret = btrfs_get_dev_stats(fs_info, sa); + + if (ret == 0 && copy_to_user(arg, sa, sizeof(*sa))) + return -EFAULT; + + return ret; +} + +static long btrfs_ioctl_dev_replace(struct btrfs_fs_info *fs_info, + void __user *arg) +{ + struct btrfs_ioctl_dev_replace_args AUTO_KFREE(p); + int ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) { + btrfs_err(fs_info, "device replace not supported on extent tree v2 yet"); + return -EINVAL; + } + + p = memdup_user(arg, sizeof(*p)); + if (IS_ERR(p)) + return PTR_ERR(p); + + switch (p->cmd) { + case BTRFS_IOCTL_DEV_REPLACE_CMD_START: + if (sb_rdonly(fs_info->sb)) + return -EROFS; + if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_DEV_REPLACE)) { + ret = BTRFS_ERROR_DEV_EXCL_RUN_IN_PROGRESS; + } else { + ret = btrfs_dev_replace_by_ioctl(fs_info, p); + btrfs_exclop_finish(fs_info); + } + break; + case BTRFS_IOCTL_DEV_REPLACE_CMD_STATUS: + btrfs_dev_replace_status(fs_info, p); + ret = 0; + break; + case BTRFS_IOCTL_DEV_REPLACE_CMD_CANCEL: + p->result = btrfs_dev_replace_cancel(fs_info); + ret = 0; + break; + default: + ret = -EINVAL; + break; + } + + if ((ret == 0 || ret == -ECANCELED) && copy_to_user(arg, p, sizeof(*p))) + return -EFAULT; + + return ret; +} + +static long btrfs_ioctl_ino_to_path(struct btrfs_root *root, void __user *arg) +{ + int ret = 0; + int i; + u64 rel_ptr; + int size; + struct btrfs_ioctl_ino_path_args AUTO_KFREE(ipa); + struct inode_fs_paths *ipath __free(inode_fs_paths) = NULL; + struct btrfs_path *path; + + if (!capable(CAP_DAC_READ_SEARCH)) + return -EPERM; + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto out; + } + + ipa = memdup_user(arg, sizeof(*ipa)); + if (IS_ERR(ipa)) { + ret = PTR_ERR(ipa); + ipa = NULL; + goto out; + } + + size = min_t(u32, ipa->size, 4096); + ipath = init_ipath(size, root, path); + if (IS_ERR(ipath)) { + ret = PTR_ERR(ipath); + ipath = NULL; + goto out; + } + + ret = paths_from_inode(ipa->inum, ipath); + if (ret < 0) + goto out; + + for (i = 0; i < ipath->fspath->elem_cnt; ++i) { + rel_ptr = ipath->fspath->val[i] - + (u64)(unsigned long)ipath->fspath->val; + ipath->fspath->val[i] = rel_ptr; + } + + btrfs_free_path(path); + path = NULL; + ret = copy_to_user((void __user *)(unsigned long)ipa->fspath, + ipath->fspath, size); + if (ret) { + ret = -EFAULT; + goto out; + } + +out: + btrfs_free_path(path); + + return ret; +} + +static long btrfs_ioctl_logical_to_ino(struct btrfs_fs_info *fs_info, + void __user *arg, int version) +{ + int ret = 0; + int size; + struct btrfs_ioctl_logical_ino_args AUTO_KFREE(loi); + struct btrfs_data_container AUTO_KVFREE(inodes); + bool ignore_offset; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + loi = memdup_user(arg, sizeof(*loi)); + if (IS_ERR(loi)) + return PTR_ERR(loi); + + if (version == 1) { + ignore_offset = false; + size = min_t(u32, loi->size, SZ_64K); + } else { + /* All reserved bits must be 0 for now */ + if (memchr_inv(loi->reserved, 0, sizeof(loi->reserved))) + return -EINVAL; + + /* Only accept flags we have defined so far */ + if (loi->flags & ~(BTRFS_LOGICAL_INO_ARGS_IGNORE_OFFSET)) + return -EINVAL; + + ignore_offset = loi->flags & BTRFS_LOGICAL_INO_ARGS_IGNORE_OFFSET; + size = min_t(u32, loi->size, SZ_16M); + } + + inodes = init_data_container(size); + if (IS_ERR(inodes)) + return PTR_ERR(inodes); + + ret = iterate_inodes_from_logical(loi->logical, fs_info, inodes, ignore_offset); + if (ret == -EINVAL) + return -ENOENT; + if (ret < 0) + return ret; + + ret = copy_to_user((void __user *)(unsigned long)loi->inodes, inodes, + size); + if (ret) + ret = -EFAULT; + + return ret; +} + +void btrfs_update_ioctl_balance_args(struct btrfs_fs_info *fs_info, + struct btrfs_ioctl_balance_args *bargs) +{ + struct btrfs_balance_control *bctl = fs_info->balance_ctl; + + bargs->flags = bctl->flags; + + if (test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) + bargs->state |= BTRFS_BALANCE_STATE_RUNNING; + if (atomic_read(&fs_info->balance_pause_req)) + bargs->state |= BTRFS_BALANCE_STATE_PAUSE_REQ; + if (atomic_read(&fs_info->balance_cancel_req)) + bargs->state |= BTRFS_BALANCE_STATE_CANCEL_REQ; + + memcpy(&bargs->data, &bctl->data, sizeof(bargs->data)); + memcpy(&bargs->meta, &bctl->meta, sizeof(bargs->meta)); + memcpy(&bargs->sys, &bctl->sys, sizeof(bargs->sys)); + + spin_lock(&fs_info->balance_lock); + memcpy(&bargs->stat, &bctl->stat, sizeof(bargs->stat)); + spin_unlock(&fs_info->balance_lock); +} + +/* + * Try to acquire fs_info::balance_mutex as well as set BTRFS_EXLCOP_BALANCE as + * required. + * + * @fs_info: the filesystem + * @excl_acquired: ptr to boolean value which is set to false in case balance + * is being resumed + * + * Return 0 on success in which case both fs_info::balance is acquired as well + * as exclusive ops are blocked. In case of failure return an error code. + */ +static int btrfs_try_lock_balance(struct btrfs_fs_info *fs_info, bool *excl_acquired) +{ + int ret; + + /* + * Exclusive operation is locked. Three possibilities: + * (1) some other op is running + * (2) balance is running + * (3) balance is paused -- special case (think resume) + */ + while (1) { + if (btrfs_exclop_start(fs_info, BTRFS_EXCLOP_BALANCE)) { + *excl_acquired = true; + mutex_lock(&fs_info->balance_mutex); + return 0; + } + + mutex_lock(&fs_info->balance_mutex); + if (fs_info->balance_ctl) { + /* This is either (2) or (3) */ + if (test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) { + /* This is (2) */ + ret = -EINPROGRESS; + goto out_failure; + + } else { + mutex_unlock(&fs_info->balance_mutex); + /* + * Lock released to allow other waiters to + * continue, we'll reexamine the status again. + */ + mutex_lock(&fs_info->balance_mutex); + + if (fs_info->balance_ctl && + !test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) { + /* This is (3) */ + *excl_acquired = false; + return 0; + } + } + } else { + /* This is (1) */ + ret = BTRFS_ERROR_DEV_EXCL_RUN_IN_PROGRESS; + goto out_failure; + } + + mutex_unlock(&fs_info->balance_mutex); + } + +out_failure: + mutex_unlock(&fs_info->balance_mutex); + *excl_acquired = false; + return ret; +} + +static long btrfs_ioctl_balance(struct file *file, void __user *arg) +{ + struct btrfs_root *root = BTRFS_I(file_inode(file))->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_ioctl_balance_args AUTO_KFREE(bargs); + struct btrfs_balance_control *bctl; + bool need_unlock = true; + int ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + ret = mnt_want_write_file(file); + if (ret) + return ret; + + bargs = memdup_user(arg, sizeof(*bargs)); + if (IS_ERR(bargs)) { + ret = PTR_ERR(bargs); + bargs = NULL; + goto out; + } + + ret = btrfs_try_lock_balance(fs_info, &need_unlock); + if (ret) + goto out; + + lockdep_assert_held(&fs_info->balance_mutex); + + if (bargs->flags & BTRFS_BALANCE_RESUME) { + if (!fs_info->balance_ctl) { + ret = -ENOTCONN; + goto out_unlock; + } + + bctl = fs_info->balance_ctl; + spin_lock(&fs_info->balance_lock); + bctl->flags |= BTRFS_BALANCE_RESUME; + spin_unlock(&fs_info->balance_lock); + btrfs_exclop_balance(fs_info, BTRFS_EXCLOP_BALANCE); + + goto do_balance; + } + + if (bargs->flags & ~(BTRFS_BALANCE_ARGS_MASK | BTRFS_BALANCE_TYPE_MASK)) { + ret = -EINVAL; + goto out_unlock; + } + + if (fs_info->balance_ctl) { + ret = -EINPROGRESS; + goto out_unlock; + } + + bctl = kzalloc_obj(*bctl); + if (!bctl) { + ret = -ENOMEM; + goto out_unlock; + } + + memcpy(&bctl->data, &bargs->data, sizeof(bctl->data)); + memcpy(&bctl->meta, &bargs->meta, sizeof(bctl->meta)); + memcpy(&bctl->sys, &bargs->sys, sizeof(bctl->sys)); + + bctl->flags = bargs->flags; +do_balance: + /* + * Ownership of bctl and exclusive operation goes to btrfs_balance. + * bctl is freed in reset_balance_state, or, if restriper was paused + * all the way until unmount, in free_fs_info. The flag should be + * cleared after reset_balance_state. + */ + need_unlock = false; + + ret = btrfs_balance(fs_info, bctl, bargs); + bctl = NULL; + + if (ret == 0 || ret == -ECANCELED) { + if (copy_to_user(arg, bargs, sizeof(*bargs))) + ret = -EFAULT; + } + + kfree(bctl); +out_unlock: + mutex_unlock(&fs_info->balance_mutex); + if (need_unlock) + btrfs_exclop_finish(fs_info); +out: + mnt_drop_write_file(file); + return ret; +} + +static long btrfs_ioctl_balance_ctl(struct btrfs_fs_info *fs_info, int cmd) +{ + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + switch (cmd) { + case BTRFS_BALANCE_CTL_PAUSE: + return btrfs_pause_balance(fs_info); + case BTRFS_BALANCE_CTL_CANCEL: + return btrfs_cancel_balance(fs_info); + } + + return -EINVAL; +} + +static long btrfs_ioctl_balance_progress(struct btrfs_fs_info *fs_info, + void __user *arg) +{ + struct btrfs_ioctl_balance_args AUTO_KFREE(bargs); + int ret = 0; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + mutex_lock(&fs_info->balance_mutex); + if (!fs_info->balance_ctl) { + ret = -ENOTCONN; + goto out; + } + + bargs = kzalloc(sizeof(*bargs), GFP_KERNEL); + if (!bargs) { + ret = -ENOMEM; + goto out; + } + + btrfs_update_ioctl_balance_args(fs_info, bargs); + + if (copy_to_user(arg, bargs, sizeof(*bargs))) + ret = -EFAULT; +out: + mutex_unlock(&fs_info->balance_mutex); + return ret; +} + +static long btrfs_ioctl_quota_ctl(struct file *file, void __user *arg) +{ + struct inode *inode = file_inode(file); + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + struct btrfs_ioctl_quota_ctl_args AUTO_KFREE(sa); + int ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + ret = mnt_want_write_file(file); + if (ret) + return ret; + + sa = memdup_user(arg, sizeof(*sa)); + if (IS_ERR(sa)) { + ret = PTR_ERR(sa); + goto drop_write; + } + + switch (sa->cmd) { + case BTRFS_QUOTA_CTL_ENABLE: + case BTRFS_QUOTA_CTL_ENABLE_SIMPLE_QUOTA: + down_write(&fs_info->subvol_sem); + ret = btrfs_quota_enable(fs_info, sa); + up_write(&fs_info->subvol_sem); + break; + case BTRFS_QUOTA_CTL_DISABLE: + /* + * Lock the cleaner mutex to prevent races with concurrent + * relocation, because relocation may be building backrefs for + * blocks of the quota root while we are deleting the root. This + * is like dropping fs roots of deleted snapshots/subvolumes, we + * need the same protection. + * + * This also prevents races between concurrent tasks trying to + * disable quotas, because we will unlock and relock + * qgroup_ioctl_lock across BTRFS_FS_QUOTA_ENABLED changes. + * + * We take this here because we have the dependency of + * + * inode_lock -> subvol_sem + * + * because of rename. With relocation we can prealloc extents, + * so that makes the dependency chain + * + * cleaner_mutex -> inode_lock -> subvol_sem + * + * so we must take the cleaner_mutex here before we take the + * subvol_sem. The deadlock can't actually happen, but this + * quiets lockdep. + */ + mutex_lock(&fs_info->cleaner_mutex); + down_write(&fs_info->subvol_sem); + ret = btrfs_quota_disable(fs_info); + up_write(&fs_info->subvol_sem); + mutex_unlock(&fs_info->cleaner_mutex); + break; + default: + ret = -EINVAL; + break; + } + +drop_write: + mnt_drop_write_file(file); + return ret; +} + +static long btrfs_ioctl_qgroup_assign(struct file *file, void __user *arg) +{ + struct inode *inode = file_inode(file); + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + struct btrfs_root *quota_root; + struct btrfs_ioctl_qgroup_assign_args AUTO_KFREE(sa); + struct btrfs_qgroup_list AUTO_KFREE(prealloc); + struct btrfs_trans_handle *trans; + int ret; + int err; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (!btrfs_qgroup_enabled(fs_info)) + return -ENOTCONN; + + ret = mnt_want_write_file(file); + if (ret) + return ret; + + sa = memdup_user(arg, sizeof(*sa)); + if (IS_ERR(sa)) { + ret = PTR_ERR(sa); + goto drop_write; + } + + if (sa->assign) { + prealloc = kzalloc_obj(*prealloc); + if (!prealloc) { + ret = -ENOMEM; + goto drop_write; + } + } + + mutex_lock(&fs_info->qgroup_ioctl_lock); + quota_root = btrfs_grab_root(fs_info->quota_root); + mutex_unlock(&fs_info->qgroup_ioctl_lock); + + if (!quota_root) { + ret = -ENOTCONN; + goto drop_write; + } + + /* 2 BTRFS_QGROUP_RELATION_KEY items. */ + trans = btrfs_start_transaction(quota_root, 2); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + btrfs_put_root(quota_root); + goto drop_write; + } + + /* + * Prealloc ownership is moved to the relation handler, there it's used + * or freed on error. + */ + if (sa->assign) { + ret = btrfs_add_qgroup_relation(trans, sa->src, sa->dst, prealloc); + prealloc = NULL; + } else { + ret = btrfs_del_qgroup_relation(trans, sa->src, sa->dst); + } + + /* update qgroup status and info */ + mutex_lock(&fs_info->qgroup_ioctl_lock); + err = btrfs_run_qgroups(trans); + mutex_unlock(&fs_info->qgroup_ioctl_lock); + if (err < 0) + btrfs_warn(fs_info, + "qgroup status update failed after %s relation, marked as inconsistent", + sa->assign ? "adding" : "deleting"); + err = btrfs_end_transaction(trans); + btrfs_put_root(quota_root); + if (err && !ret) + ret = err; + +drop_write: + mnt_drop_write_file(file); + return ret; +} + +static long btrfs_ioctl_qgroup_create(struct file *file, void __user *arg) +{ + struct inode *inode = file_inode(file); + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + struct btrfs_root *quota_root; + struct btrfs_ioctl_qgroup_create_args AUTO_KFREE(sa); + struct btrfs_trans_handle *trans; + int ret; + int err; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (!btrfs_qgroup_enabled(fs_info)) + return -ENOTCONN; + + ret = mnt_want_write_file(file); + if (ret) + return ret; + + sa = memdup_user(arg, sizeof(*sa)); + if (IS_ERR(sa)) { + ret = PTR_ERR(sa); + goto drop_write; + } + + if (!sa->qgroupid) { + ret = -EINVAL; + goto drop_write; + } + + if (sa->create && btrfs_is_fstree(sa->qgroupid)) { + ret = -EINVAL; + goto drop_write; + } + + mutex_lock(&fs_info->qgroup_ioctl_lock); + quota_root = btrfs_grab_root(fs_info->quota_root); + mutex_unlock(&fs_info->qgroup_ioctl_lock); + + if (!quota_root) { + ret = -ENOTCONN; + goto drop_write; + } + + /* + * 1 BTRFS_QGROUP_INFO_KEY item. + * 1 BTRFS_QGROUP_LIMIT_KEY item. + */ + trans = btrfs_start_transaction(quota_root, 2); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + btrfs_put_root(quota_root); + goto drop_write; + } + + if (sa->create) { + ret = btrfs_create_qgroup(trans, sa->qgroupid); + } else { + ret = btrfs_remove_qgroup(trans, sa->qgroupid); + } + + err = btrfs_end_transaction(trans); + btrfs_put_root(quota_root); + if (err && !ret) + ret = err; + +drop_write: + mnt_drop_write_file(file); + return ret; +} + +static long btrfs_ioctl_qgroup_limit(struct file *file, void __user *arg) +{ + struct inode *inode = file_inode(file); + struct btrfs_root *root = BTRFS_I(inode)->root; + struct btrfs_root *quota_root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_ioctl_qgroup_limit_args AUTO_KFREE(sa); + struct btrfs_trans_handle *trans; + int ret; + int err; + u64 qgroupid; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (!btrfs_qgroup_enabled(fs_info)) + return -ENOTCONN; + + ret = mnt_want_write_file(file); + if (ret) + return ret; + + sa = memdup_user(arg, sizeof(*sa)); + if (IS_ERR(sa)) { + ret = PTR_ERR(sa); + goto drop_write; + } + + mutex_lock(&fs_info->qgroup_ioctl_lock); + quota_root = btrfs_grab_root(fs_info->quota_root); + mutex_unlock(&fs_info->qgroup_ioctl_lock); + + if (!quota_root) { + ret = -ENOTCONN; + goto drop_write; + } + + /* 1 BTRFS_QGROUP_LIMIT_KEY item. */ + trans = btrfs_start_transaction(quota_root, 1); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + btrfs_put_root(quota_root); + goto drop_write; + } + + qgroupid = sa->qgroupid; + if (!qgroupid) { + /* take the current subvol as qgroup */ + qgroupid = btrfs_root_id(root); + } + + ret = btrfs_limit_qgroup(trans, qgroupid, &sa->lim); + + err = btrfs_end_transaction(trans); + btrfs_put_root(quota_root); + if (err && !ret) + ret = err; + +drop_write: + mnt_drop_write_file(file); + return ret; +} + +static long btrfs_ioctl_quota_rescan(struct file *file, void __user *arg) +{ + struct inode *inode = file_inode(file); + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + struct btrfs_ioctl_quota_rescan_args AUTO_KFREE(qsa); + int ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (!btrfs_qgroup_enabled(fs_info)) + return -ENOTCONN; + + ret = mnt_want_write_file(file); + if (ret) + return ret; + + qsa = memdup_user(arg, sizeof(*qsa)); + if (IS_ERR(qsa)) { + ret = PTR_ERR(qsa); + goto drop_write; + } + + if (qsa->flags) { + ret = -EINVAL; + goto drop_write; + } + + ret = btrfs_qgroup_rescan(fs_info); + +drop_write: + mnt_drop_write_file(file); + return ret; +} + +static long btrfs_ioctl_quota_rescan_status(struct btrfs_fs_info *fs_info, + void __user *arg) +{ + struct btrfs_ioctl_quota_rescan_args qsa = {0}; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_RESCAN) { + qsa.flags = 1; + qsa.progress = fs_info->qgroup_rescan_progress.objectid; + } + + if (copy_to_user(arg, &qsa, sizeof(qsa))) + return -EFAULT; + + return 0; +} + +static long btrfs_ioctl_quota_rescan_wait(struct btrfs_fs_info *fs_info) +{ + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + return btrfs_qgroup_wait_for_completion(fs_info, true); +} + +static long _btrfs_ioctl_set_received_subvol(struct file *file, + struct mnt_idmap *idmap, + struct btrfs_ioctl_received_subvol_args *sa) +{ + struct inode *inode = file_inode(file); + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + struct btrfs_root *root = BTRFS_I(inode)->root; + struct btrfs_root_item *root_item = &root->root_item; + struct btrfs_trans_handle *trans; + struct timespec64 ct = current_time(inode); + int ret = 0; + int received_uuid_changed; + + if (!inode_owner_or_capable(idmap, inode)) + return -EPERM; + + ret = mnt_want_write_file(file); + if (ret < 0) + return ret; + + down_write(&fs_info->subvol_sem); + + if (btrfs_ino(BTRFS_I(inode)) != BTRFS_FIRST_FREE_OBJECTID) { + ret = -EINVAL; + goto out; + } + + if (btrfs_root_readonly(root)) { + ret = -EROFS; + goto out; + } + + received_uuid_changed = memcmp(root_item->received_uuid, sa->uuid, + BTRFS_UUID_SIZE); + + /* + * Before we attempt to add the new received uuid, check if we have room + * for it in case there's already an item. If the size of the existing + * item plus this root's ID (u64) exceeds the maximum item size, we can + * return here without the need to abort a transaction. If we don't do + * this check, the btrfs_uuid_tree_add() call below would fail with + * -EOVERFLOW and result in a transaction abort. Malicious users could + * exploit this to turn the fs into RO mode. + */ + if (received_uuid_changed && !btrfs_is_empty_uuid(sa->uuid)) { + ret = btrfs_uuid_tree_check_overflow(fs_info, sa->uuid, + BTRFS_UUID_KEY_RECEIVED_SUBVOL); + if (ret < 0) + goto out; + } + + /* + * 1 - root item + * 2 - uuid items (received uuid + subvol uuid) + */ + trans = btrfs_start_transaction(root, 3); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + trans = NULL; + goto out; + } + + sa->rtransid = trans->transid; + sa->rtime.sec = ct.tv_sec; + sa->rtime.nsec = ct.tv_nsec; + + if (received_uuid_changed && + !btrfs_is_empty_uuid(root_item->received_uuid)) { + ret = btrfs_uuid_tree_remove(trans, root_item->received_uuid, + BTRFS_UUID_KEY_RECEIVED_SUBVOL, + btrfs_root_id(root)); + if (unlikely(ret && ret != -ENOENT)) { + btrfs_end_transaction(trans); + goto out; + } + } + memcpy(root_item->received_uuid, sa->uuid, BTRFS_UUID_SIZE); + btrfs_set_root_stransid(root_item, sa->stransid); + btrfs_set_root_rtransid(root_item, sa->rtransid); + btrfs_set_stack_timespec_sec(&root_item->stime, sa->stime.sec); + btrfs_set_stack_timespec_nsec(&root_item->stime, sa->stime.nsec); + btrfs_set_stack_timespec_sec(&root_item->rtime, sa->rtime.sec); + btrfs_set_stack_timespec_nsec(&root_item->rtime, sa->rtime.nsec); + + ret = btrfs_update_root(trans, fs_info->tree_root, + &root->root_key, &root->root_item); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + btrfs_end_transaction(trans); + goto out; + } + if (received_uuid_changed && !btrfs_is_empty_uuid(sa->uuid)) { + ret = btrfs_uuid_tree_add(trans, sa->uuid, + BTRFS_UUID_KEY_RECEIVED_SUBVOL, + btrfs_root_id(root)); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + btrfs_end_transaction(trans); + goto out; + } + } + ret = btrfs_commit_transaction(trans); +out: + up_write(&fs_info->subvol_sem); + mnt_drop_write_file(file); + return ret; +} + +#ifdef CONFIG_64BIT +static long btrfs_ioctl_set_received_subvol_32(struct file *file, + void __user *arg) +{ + struct btrfs_ioctl_received_subvol_args_32 AUTO_KFREE(args32); + struct btrfs_ioctl_received_subvol_args AUTO_KFREE(args64); + int ret = 0; + + args32 = memdup_user(arg, sizeof(*args32)); + if (IS_ERR(args32)) + return PTR_ERR(args32); + + args64 = kmalloc_obj(*args64); + if (!args64) + return -ENOMEM; + + memcpy(args64->uuid, args32->uuid, BTRFS_UUID_SIZE); + args64->stransid = args32->stransid; + args64->rtransid = args32->rtransid; + args64->stime.sec = args32->stime.sec; + args64->stime.nsec = args32->stime.nsec; + args64->rtime.sec = args32->rtime.sec; + args64->rtime.nsec = args32->rtime.nsec; + args64->flags = args32->flags; + + ret = _btrfs_ioctl_set_received_subvol(file, file_mnt_idmap(file), args64); + if (ret) + return ret; + + memcpy(args32->uuid, args64->uuid, BTRFS_UUID_SIZE); + args32->stransid = args64->stransid; + args32->rtransid = args64->rtransid; + args32->stime.sec = args64->stime.sec; + args32->stime.nsec = args64->stime.nsec; + args32->rtime.sec = args64->rtime.sec; + args32->rtime.nsec = args64->rtime.nsec; + args32->flags = args64->flags; + + ret = copy_to_user(arg, args32, sizeof(*args32)); + if (ret) + return -EFAULT; + + return 0; +} +#endif + +static long btrfs_ioctl_set_received_subvol(struct file *file, + void __user *arg) +{ + struct btrfs_ioctl_received_subvol_args AUTO_KFREE(sa); + int ret = 0; + + sa = memdup_user(arg, sizeof(*sa)); + if (IS_ERR(sa)) + return PTR_ERR(sa); + + ret = _btrfs_ioctl_set_received_subvol(file, file_mnt_idmap(file), sa); + if (ret) + return ret; + + ret = copy_to_user(arg, sa, sizeof(*sa)); + if (ret) + return -EFAULT; + + return 0; +} + +static int btrfs_ioctl_get_fslabel(struct btrfs_fs_info *fs_info, + void __user *arg) +{ + size_t len; + int ret; + char label[BTRFS_LABEL_SIZE]; + + spin_lock(&fs_info->super_lock); + memcpy(label, fs_info->super_copy->label, BTRFS_LABEL_SIZE); + spin_unlock(&fs_info->super_lock); + + len = strnlen(label, BTRFS_LABEL_SIZE); + + if (len == BTRFS_LABEL_SIZE) { + btrfs_warn(fs_info, + "label is too long, return the first %zu bytes", + --len); + } + + ret = copy_to_user(arg, label, len); + + return ret ? -EFAULT : 0; +} + +static int btrfs_ioctl_set_fslabel(struct file *file, void __user *arg) +{ + struct inode *inode = file_inode(file); + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + struct btrfs_root *root = BTRFS_I(inode)->root; + struct btrfs_super_block *super_block = fs_info->super_copy; + struct btrfs_trans_handle *trans; + char label[BTRFS_LABEL_SIZE]; + int ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (copy_from_user(label, arg, sizeof(label))) + return -EFAULT; + + if (strnlen(label, BTRFS_LABEL_SIZE) == BTRFS_LABEL_SIZE) { + btrfs_err(fs_info, + "unable to set label with more than %d bytes", + BTRFS_LABEL_SIZE - 1); + return -EINVAL; + } + + ret = mnt_want_write_file(file); + if (ret) + return ret; + + trans = btrfs_start_transaction(root, 0); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto out_unlock; + } + + spin_lock(&fs_info->super_lock); + strscpy(super_block->label, label); + spin_unlock(&fs_info->super_lock); + ret = btrfs_commit_transaction(trans); + +out_unlock: + mnt_drop_write_file(file); + return ret; +} + +#define INIT_FEATURE_FLAGS(suffix) \ + { .compat_flags = BTRFS_FEATURE_COMPAT_##suffix, \ + .compat_ro_flags = BTRFS_FEATURE_COMPAT_RO_##suffix, \ + .incompat_flags = BTRFS_FEATURE_INCOMPAT_##suffix } + +int btrfs_ioctl_get_supported_features(void __user *arg) +{ + static const struct btrfs_ioctl_feature_flags features[3] = { + INIT_FEATURE_FLAGS(SUPP), + INIT_FEATURE_FLAGS(SAFE_SET), + INIT_FEATURE_FLAGS(SAFE_CLEAR) + }; + + if (copy_to_user(arg, &features, sizeof(features))) + return -EFAULT; + + return 0; +} + +static int btrfs_ioctl_get_features(struct btrfs_fs_info *fs_info, + void __user *arg) +{ + struct btrfs_super_block *super_block = fs_info->super_copy; + struct btrfs_ioctl_feature_flags features; + + features.compat_flags = btrfs_super_compat_flags(super_block); + features.compat_ro_flags = btrfs_super_compat_ro_flags(super_block); + features.incompat_flags = btrfs_super_incompat_flags(super_block); + + if (copy_to_user(arg, &features, sizeof(features))) + return -EFAULT; + + return 0; +} + +static int check_feature_bits(const struct btrfs_fs_info *fs_info, + enum btrfs_feature_set set, + u64 change_mask, u64 flags, u64 supported_flags, + u64 safe_set, u64 safe_clear) +{ + const char *type = btrfs_feature_set_name(set); + const char AUTO_KFREE(names); + u64 disallowed, unsupported; + u64 set_mask = flags & change_mask; + u64 clear_mask = ~flags & change_mask; + + unsupported = set_mask & ~supported_flags; + if (unsupported) { + names = btrfs_printable_features(set, unsupported); + if (names) + btrfs_warn(fs_info, + "this kernel does not support the %s feature bit%s", + names, strchr(names, ',') ? "s" : ""); + else + btrfs_warn(fs_info, + "this kernel does not support %s bits 0x%llx", + type, unsupported); + return -EOPNOTSUPP; + } + + disallowed = set_mask & ~safe_set; + if (disallowed) { + names = btrfs_printable_features(set, disallowed); + if (names) + btrfs_warn(fs_info, + "can't set the %s feature bit%s while mounted", + names, strchr(names, ',') ? "s" : ""); + else + btrfs_warn(fs_info, + "can't set %s bits 0x%llx while mounted", + type, disallowed); + return -EPERM; + } + + disallowed = clear_mask & ~safe_clear; + if (disallowed) { + names = btrfs_printable_features(set, disallowed); + if (names) + btrfs_warn(fs_info, + "can't clear the %s feature bit%s while mounted", + names, strchr(names, ',') ? "s" : ""); + else + btrfs_warn(fs_info, + "can't clear %s bits 0x%llx while mounted", + type, disallowed); + return -EPERM; + } + + return 0; +} + +#define check_feature(fs_info, change_mask, flags, mask_base) \ +check_feature_bits(fs_info, FEAT_##mask_base, change_mask, flags, \ + BTRFS_FEATURE_ ## mask_base ## _SUPP, \ + BTRFS_FEATURE_ ## mask_base ## _SAFE_SET, \ + BTRFS_FEATURE_ ## mask_base ## _SAFE_CLEAR) + +static int btrfs_ioctl_set_features(struct file *file, void __user *arg) +{ + struct inode *inode = file_inode(file); + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + struct btrfs_root *root = BTRFS_I(inode)->root; + struct btrfs_super_block *super_block = fs_info->super_copy; + struct btrfs_ioctl_feature_flags flags[2]; + struct btrfs_trans_handle *trans; + u64 newflags; + int ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (copy_from_user(flags, arg, sizeof(flags))) + return -EFAULT; + + /* Nothing to do */ + if (!flags[0].compat_flags && !flags[0].compat_ro_flags && + !flags[0].incompat_flags) + return 0; + + ret = check_feature(fs_info, flags[0].compat_flags, + flags[1].compat_flags, COMPAT); + if (ret) + return ret; + + ret = check_feature(fs_info, flags[0].compat_ro_flags, + flags[1].compat_ro_flags, COMPAT_RO); + if (ret) + return ret; + + ret = check_feature(fs_info, flags[0].incompat_flags, + flags[1].incompat_flags, INCOMPAT); + if (ret) + return ret; + + ret = mnt_want_write_file(file); + if (ret) + return ret; + + trans = btrfs_start_transaction(root, 0); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto out_drop_write; + } + + spin_lock(&fs_info->super_lock); + newflags = btrfs_super_compat_flags(super_block); + newflags |= flags[0].compat_flags & flags[1].compat_flags; + newflags &= ~(flags[0].compat_flags & ~flags[1].compat_flags); + btrfs_set_super_compat_flags(super_block, newflags); + + newflags = btrfs_super_compat_ro_flags(super_block); + newflags |= flags[0].compat_ro_flags & flags[1].compat_ro_flags; + newflags &= ~(flags[0].compat_ro_flags & ~flags[1].compat_ro_flags); + btrfs_set_super_compat_ro_flags(super_block, newflags); + + newflags = btrfs_super_incompat_flags(super_block); + newflags |= flags[0].incompat_flags & flags[1].incompat_flags; + newflags &= ~(flags[0].incompat_flags & ~flags[1].incompat_flags); + btrfs_set_super_incompat_flags(super_block, newflags); + spin_unlock(&fs_info->super_lock); + + ret = btrfs_commit_transaction(trans); +out_drop_write: + mnt_drop_write_file(file); + + return ret; +} + +static int _btrfs_ioctl_send(struct btrfs_root *root, void __user *argp, bool compat) +{ + struct btrfs_ioctl_send_args AUTO_KFREE(arg); + + if (compat) { +#if defined(CONFIG_64BIT) && defined(CONFIG_COMPAT) + int ret; + struct btrfs_ioctl_send_args_32 args32 = { 0 }; + + ret = copy_from_user(&args32, argp, sizeof(args32)); + if (ret) + return -EFAULT; + arg = kzalloc_obj(*arg); + if (!arg) + return -ENOMEM; + arg->send_fd = args32.send_fd; + arg->clone_sources_count = args32.clone_sources_count; + arg->clone_sources = compat_ptr(args32.clone_sources); + arg->parent_root = args32.parent_root; + arg->flags = args32.flags; + arg->version = args32.version; + memcpy(arg->reserved, args32.reserved, + sizeof(args32.reserved)); +#else + return -ENOTTY; +#endif + } else { + arg = memdup_user(argp, sizeof(*arg)); + if (IS_ERR(arg)) + return PTR_ERR(arg); + } + return btrfs_ioctl_send(root, arg); +} + +static int btrfs_ioctl_encoded_read(struct file *file, void __user *argp, + bool compat) +{ + struct btrfs_ioctl_encoded_io_args args = { 0 }; + size_t copy_end_kernel = offsetofend(struct btrfs_ioctl_encoded_io_args, + flags); + size_t copy_end; + struct btrfs_inode *inode = BTRFS_I(file_inode(file)); + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct extent_io_tree *io_tree = &inode->io_tree; + struct iovec iovstack[UIO_FASTIOV]; + struct iovec *iov = iovstack; + struct iov_iter iter; + loff_t pos; + struct kiocb kiocb; + ssize_t ret; + u64 disk_bytenr, disk_io_size; + struct extent_state *cached_state = NULL; + + if (!capable(CAP_SYS_ADMIN)) { + ret = -EPERM; + goto out_acct; + } + + if (compat) { +#if defined(CONFIG_64BIT) && defined(CONFIG_COMPAT) + struct btrfs_ioctl_encoded_io_args_32 args32; + + copy_end = offsetofend(struct btrfs_ioctl_encoded_io_args_32, + flags); + if (copy_from_user(&args32, argp, copy_end)) { + ret = -EFAULT; + goto out_acct; + } + args.iov = compat_ptr(args32.iov); + args.iovcnt = args32.iovcnt; + args.offset = args32.offset; + args.flags = args32.flags; +#else + return -ENOTTY; +#endif + } else { + copy_end = copy_end_kernel; + if (copy_from_user(&args, argp, copy_end)) { + ret = -EFAULT; + goto out_acct; + } + } + if (args.flags != 0) { + ret = -EINVAL; + goto out_acct; + } + + ret = import_iovec(ITER_DEST, args.iov, args.iovcnt, ARRAY_SIZE(iovstack), + &iov, &iter); + if (ret < 0) + goto out_acct; + + if (iov_iter_count(&iter) == 0) { + ret = 0; + goto out_iov; + } + pos = args.offset; + ret = rw_verify_area(READ, file, &pos, args.len); + if (ret < 0) + goto out_iov; + + init_sync_kiocb(&kiocb, file); + kiocb.ki_pos = pos; + + ret = btrfs_encoded_read(&kiocb, &iter, &args, &cached_state, + &disk_bytenr, &disk_io_size); + + if (ret == -EIOCBQUEUED) { + bool unlocked = false; + u64 start, lockend, count; + + start = ALIGN_DOWN(kiocb.ki_pos, fs_info->sectorsize); + lockend = start + BTRFS_MAX_UNCOMPRESSED - 1; + + if (args.compression) + count = disk_io_size; + else + count = args.len; + + ret = btrfs_encoded_read_regular(&kiocb, &iter, start, lockend, + &cached_state, disk_bytenr, + disk_io_size, count, + args.compression, &unlocked); + + if (!unlocked) { + btrfs_unlock_extent(io_tree, start, lockend, &cached_state); + btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); + } + } + + if (ret >= 0) { + fsnotify_access(file); + if (copy_to_user(argp + copy_end, + (char *)&args + copy_end_kernel, + sizeof(args) - copy_end_kernel)) + ret = -EFAULT; + } + +out_iov: + kfree(iov); +out_acct: + if (ret > 0) + add_rchar(current, ret); + inc_syscr(current); + return ret; +} + +static int btrfs_ioctl_encoded_write(struct file *file, void __user *argp, bool compat) +{ + struct btrfs_ioctl_encoded_io_args args; + struct iovec iovstack[UIO_FASTIOV]; + struct iovec *iov = iovstack; + struct iov_iter iter; + loff_t pos; + struct kiocb kiocb; + ssize_t ret; + + if (!capable(CAP_SYS_ADMIN)) { + ret = -EPERM; + goto out_acct; + } + + if (!(file->f_mode & FMODE_WRITE)) { + ret = -EBADF; + goto out_acct; + } + + if (compat) { +#if defined(CONFIG_64BIT) && defined(CONFIG_COMPAT) + struct btrfs_ioctl_encoded_io_args_32 args32; + + if (copy_from_user(&args32, argp, sizeof(args32))) { + ret = -EFAULT; + goto out_acct; + } + args.iov = compat_ptr(args32.iov); + args.iovcnt = args32.iovcnt; + args.offset = args32.offset; + args.flags = args32.flags; + args.len = args32.len; + args.unencoded_len = args32.unencoded_len; + args.unencoded_offset = args32.unencoded_offset; + args.compression = args32.compression; + args.encryption = args32.encryption; + memcpy(args.reserved, args32.reserved, sizeof(args.reserved)); +#else + return -ENOTTY; +#endif + } else { + if (copy_from_user(&args, argp, sizeof(args))) { + ret = -EFAULT; + goto out_acct; + } + } + + ret = -EINVAL; + if (args.flags != 0) + goto out_acct; + if (memchr_inv(args.reserved, 0, sizeof(args.reserved))) + goto out_acct; + if (args.compression == BTRFS_ENCODED_IO_COMPRESSION_NONE && + args.encryption == BTRFS_ENCODED_IO_ENCRYPTION_NONE) + goto out_acct; + if (args.compression >= BTRFS_ENCODED_IO_COMPRESSION_TYPES || + args.encryption >= BTRFS_ENCODED_IO_ENCRYPTION_TYPES) + goto out_acct; + if (args.unencoded_offset > args.unencoded_len) + goto out_acct; + if (args.len > args.unencoded_len - args.unencoded_offset) + goto out_acct; + + ret = import_iovec(ITER_SOURCE, args.iov, args.iovcnt, ARRAY_SIZE(iovstack), + &iov, &iter); + if (ret < 0) + goto out_acct; + + if (iov_iter_count(&iter) == 0) { + ret = 0; + goto out_iov; + } + pos = args.offset; + ret = rw_verify_area(WRITE, file, &pos, args.len); + if (ret < 0) + goto out_iov; + + init_sync_kiocb(&kiocb, file); + ret = kiocb_set_rw_flags(&kiocb, 0, WRITE); + if (ret) + goto out_iov; + kiocb.ki_pos = pos; + + file_start_write(file); + + ret = btrfs_do_write_iter(&kiocb, &iter, &args); + if (ret > 0) + fsnotify_modify(file); + + file_end_write(file); +out_iov: + kfree(iov); +out_acct: + if (ret > 0) + add_wchar(current, ret); + inc_syscw(current); + return ret; +} + +struct btrfs_uring_encoded_data { + struct btrfs_ioctl_encoded_io_args args; + struct iovec iovstack[UIO_FASTIOV]; + struct iovec *iov; + struct iov_iter iter; +}; + +/* + * Context that's attached to an encoded read io_uring command, in cmd->pdu. It + * contains the fields in btrfs_uring_read_extent that are necessary to finish + * off and cleanup the I/O in btrfs_uring_read_finished. + */ +struct btrfs_uring_priv { + struct io_uring_cmd *cmd; + struct page **pages; + unsigned long nr_pages; + struct kiocb iocb; + struct iovec *iov; + struct iov_iter iter; + struct extent_state *cached_state; + u64 count; + u64 start; + u64 lockend; + int err; + bool compressed; +}; + +struct io_btrfs_cmd { + struct btrfs_uring_encoded_data *data; + struct btrfs_uring_priv *priv; +}; + +static void btrfs_uring_read_finished(struct io_tw_req tw_req, io_tw_token_t tw) +{ + struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req); + struct io_btrfs_cmd *bc = io_uring_cmd_to_pdu(cmd, struct io_btrfs_cmd); + struct btrfs_uring_priv *priv = bc->priv; + struct btrfs_inode *inode = BTRFS_I(file_inode(priv->iocb.ki_filp)); + struct extent_io_tree *io_tree = &inode->io_tree; + pgoff_t index; + u64 cur; + size_t page_offset; + ssize_t ret; + + /* The inode lock has already been acquired in btrfs_uring_read_extent. */ + btrfs_lockdep_inode_acquire(inode, i_rwsem); + + if (priv->err) { + ret = priv->err; + goto out; + } + + if (priv->compressed) { + index = 0; + page_offset = 0; + } else { + index = (priv->iocb.ki_pos - priv->start) >> PAGE_SHIFT; + page_offset = offset_in_page(priv->iocb.ki_pos - priv->start); + } + cur = 0; + while (cur < priv->count) { + size_t bytes = min_t(size_t, priv->count - cur, PAGE_SIZE - page_offset); + + if (copy_page_to_iter(priv->pages[index], page_offset, bytes, + &priv->iter) != bytes) { + ret = -EFAULT; + goto out; + } + + index++; + cur += bytes; + page_offset = 0; + } + ret = priv->count; + +out: + btrfs_unlock_extent(io_tree, priv->start, priv->lockend, &priv->cached_state); + btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); + + io_uring_cmd_done(cmd, ret, IO_URING_CMD_TASK_WORK_ISSUE_FLAGS); + add_rchar(current, ret); + + for (index = 0; index < priv->nr_pages; index++) + __free_page(priv->pages[index]); + + kfree(priv->pages); + kfree(priv->iov); + kfree(priv); + kfree(bc->data); +} + +void btrfs_uring_read_extent_endio(void *ctx, int err) +{ + struct btrfs_uring_priv *priv = ctx; + struct io_btrfs_cmd *bc = io_uring_cmd_to_pdu(priv->cmd, struct io_btrfs_cmd); + + priv->err = err; + bc->priv = priv; + + io_uring_cmd_complete_in_task(priv->cmd, btrfs_uring_read_finished); +} + +static int btrfs_uring_read_extent(struct kiocb *iocb, struct iov_iter *iter, + u64 start, u64 lockend, + struct extent_state *cached_state, + u64 disk_bytenr, u64 disk_io_size, + size_t count, bool compressed, + struct iovec *iov, struct io_uring_cmd *cmd) +{ + struct btrfs_inode *inode = BTRFS_I(file_inode(iocb->ki_filp)); + struct extent_io_tree *io_tree = &inode->io_tree; + struct page **pages = NULL; + struct btrfs_uring_priv *priv = NULL; + unsigned long nr_pages; + int ret; + + nr_pages = DIV_ROUND_UP(disk_io_size, PAGE_SIZE); + pages = kzalloc_objs(struct page *, nr_pages, GFP_NOFS); + if (!pages) + return -ENOMEM; + ret = btrfs_alloc_page_array(nr_pages, pages, GFP_NOFS); + if (ret) { + ret = -ENOMEM; + goto out_fail; + } + + priv = kmalloc_obj(*priv, GFP_NOFS); + if (!priv) { + ret = -ENOMEM; + goto out_fail; + } + + priv->iocb = *iocb; + priv->iov = iov; + priv->iter = *iter; + priv->count = count; + priv->cmd = cmd; + priv->cached_state = cached_state; + priv->compressed = compressed; + priv->nr_pages = nr_pages; + priv->pages = pages; + priv->start = start; + priv->lockend = lockend; + priv->err = 0; + + ret = btrfs_encoded_read_regular_fill_pages(inode, disk_bytenr, + disk_io_size, pages, priv); + if (ret && ret != -EIOCBQUEUED) + goto out_fail; + + /* + * If we return -EIOCBQUEUED, we're deferring the cleanup to + * btrfs_uring_read_finished(), which will handle unlocking the extent + * and inode and freeing the allocations. + */ + + /* + * We're returning to userspace with the inode lock held, and that's + * okay - it'll get unlocked in a worker thread. Call + * btrfs_lockdep_inode_release() to avoid confusing lockdep. + */ + btrfs_lockdep_inode_release(inode, i_rwsem); + + return -EIOCBQUEUED; + +out_fail: + btrfs_unlock_extent(io_tree, start, lockend, &cached_state); + btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); + kfree(priv); + for (int i = 0; i < nr_pages; i++) { + if (pages[i]) + __free_page(pages[i]); + } + kfree(pages); + return ret; +} + +static int btrfs_uring_encoded_read(struct io_uring_cmd *cmd, unsigned int issue_flags) +{ + struct file *file = cmd->file; + struct btrfs_inode *inode = BTRFS_I(file->f_inode); + struct extent_io_tree *io_tree = &inode->io_tree; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + size_t copy_end_kernel = offsetofend(struct btrfs_ioctl_encoded_io_args, flags); + size_t copy_end; + int ret; + u64 disk_bytenr, disk_io_size; + loff_t pos; + struct kiocb kiocb; + struct extent_state *cached_state = NULL; + u64 start, lockend; + void __user *sqe_addr; + struct io_btrfs_cmd *bc = io_uring_cmd_to_pdu(cmd, struct io_btrfs_cmd); + struct btrfs_uring_encoded_data *data = NULL; + + if (cmd->flags & IORING_URING_CMD_REISSUE) + data = bc->data; + + if (!capable(CAP_SYS_ADMIN)) { + ret = -EPERM; + goto out_acct; + } + sqe_addr = u64_to_user_ptr(READ_ONCE(cmd->sqe->addr)); + + if (issue_flags & IO_URING_F_COMPAT) { +#if defined(CONFIG_64BIT) && defined(CONFIG_COMPAT) + copy_end = offsetofend(struct btrfs_ioctl_encoded_io_args_32, flags); +#else + ret = -ENOTTY; + goto out_acct; +#endif + } else { + copy_end = copy_end_kernel; + } + + if (!data) { + data = kzalloc_obj(*data, GFP_NOFS); + if (!data) { + ret = -ENOMEM; + goto out_acct; + } + + bc->data = data; + + if (issue_flags & IO_URING_F_COMPAT) { +#if defined(CONFIG_64BIT) && defined(CONFIG_COMPAT) + struct btrfs_ioctl_encoded_io_args_32 args32; + + if (copy_from_user(&args32, sqe_addr, copy_end)) { + ret = -EFAULT; + goto out_acct; + } + + data->args.iov = compat_ptr(args32.iov); + data->args.iovcnt = args32.iovcnt; + data->args.offset = args32.offset; + data->args.flags = args32.flags; +#endif + } else { + if (copy_from_user(&data->args, sqe_addr, copy_end)) { + ret = -EFAULT; + goto out_acct; + } + } + + if (data->args.flags != 0) { + ret = -EINVAL; + goto out_acct; + } + + data->iov = data->iovstack; + ret = import_iovec(ITER_DEST, data->args.iov, data->args.iovcnt, + ARRAY_SIZE(data->iovstack), &data->iov, + &data->iter); + if (ret < 0) + goto out_acct; + + if (iov_iter_count(&data->iter) == 0) { + ret = 0; + goto out_free; + } + } + + pos = data->args.offset; + ret = rw_verify_area(READ, file, &pos, data->args.len); + if (ret < 0) + goto out_free; + + init_sync_kiocb(&kiocb, file); + kiocb.ki_pos = pos; + + if (issue_flags & IO_URING_F_NONBLOCK) + kiocb.ki_flags |= IOCB_NOWAIT; + + start = ALIGN_DOWN(pos, fs_info->sectorsize); + lockend = start + BTRFS_MAX_UNCOMPRESSED - 1; + + ret = btrfs_encoded_read(&kiocb, &data->iter, &data->args, &cached_state, + &disk_bytenr, &disk_io_size); + if (ret == -EAGAIN) + goto out_acct; + if (ret < 0 && ret != -EIOCBQUEUED) + goto out_free; + + file_accessed(file); + + if (copy_to_user(sqe_addr + copy_end, + (const char *)&data->args + copy_end_kernel, + sizeof(data->args) - copy_end_kernel)) { + if (ret == -EIOCBQUEUED) { + btrfs_unlock_extent(io_tree, start, lockend, &cached_state); + btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED); + } + ret = -EFAULT; + goto out_free; + } + + if (ret == -EIOCBQUEUED) { + u64 count = min_t(u64, iov_iter_count(&data->iter), disk_io_size); + + /* Match ioctl by not returning past EOF if uncompressed. */ + if (!data->args.compression) + count = min_t(u64, count, data->args.len); + + ret = btrfs_uring_read_extent(&kiocb, &data->iter, start, lockend, + cached_state, disk_bytenr, disk_io_size, + count, data->args.compression, + data->iov, cmd); + + goto out_acct; + } + +out_free: + kfree(data->iov); + +out_acct: + if (ret > 0) + add_rchar(current, ret); + inc_syscr(current); + + if (ret != -EIOCBQUEUED && ret != -EAGAIN) + kfree(data); + + return ret; +} + +static int btrfs_uring_encoded_write(struct io_uring_cmd *cmd, unsigned int issue_flags) +{ + struct file *file = cmd->file; + loff_t pos; + struct kiocb kiocb; + ssize_t ret; + void __user *sqe_addr; + struct io_btrfs_cmd *bc = io_uring_cmd_to_pdu(cmd, struct io_btrfs_cmd); + struct btrfs_uring_encoded_data *data = NULL; + + if (cmd->flags & IORING_URING_CMD_REISSUE) + data = bc->data; + + if (!capable(CAP_SYS_ADMIN)) { + ret = -EPERM; + goto out_acct; + } + sqe_addr = u64_to_user_ptr(READ_ONCE(cmd->sqe->addr)); + + if (!(file->f_mode & FMODE_WRITE)) { + ret = -EBADF; + goto out_acct; + } + + if (!data) { + data = kzalloc_obj(*data, GFP_NOFS); + if (!data) { + ret = -ENOMEM; + goto out_acct; + } + + bc->data = data; + + if (issue_flags & IO_URING_F_COMPAT) { +#if defined(CONFIG_64BIT) && defined(CONFIG_COMPAT) + struct btrfs_ioctl_encoded_io_args_32 args32; + + if (copy_from_user(&args32, sqe_addr, sizeof(args32))) { + ret = -EFAULT; + goto out_acct; + } + data->args.iov = compat_ptr(args32.iov); + data->args.iovcnt = args32.iovcnt; + data->args.offset = args32.offset; + data->args.flags = args32.flags; + data->args.len = args32.len; + data->args.unencoded_len = args32.unencoded_len; + data->args.unencoded_offset = args32.unencoded_offset; + data->args.compression = args32.compression; + data->args.encryption = args32.encryption; + memcpy(data->args.reserved, args32.reserved, + sizeof(data->args.reserved)); +#else + ret = -ENOTTY; + goto out_acct; +#endif + } else { + if (copy_from_user(&data->args, sqe_addr, sizeof(data->args))) { + ret = -EFAULT; + goto out_acct; + } + } + + ret = -EINVAL; + if (data->args.flags != 0) + goto out_acct; + if (memchr_inv(data->args.reserved, 0, sizeof(data->args.reserved))) + goto out_acct; + if (data->args.compression == BTRFS_ENCODED_IO_COMPRESSION_NONE && + data->args.encryption == BTRFS_ENCODED_IO_ENCRYPTION_NONE) + goto out_acct; + if (data->args.compression >= BTRFS_ENCODED_IO_COMPRESSION_TYPES || + data->args.encryption >= BTRFS_ENCODED_IO_ENCRYPTION_TYPES) + goto out_acct; + if (data->args.unencoded_offset > data->args.unencoded_len) + goto out_acct; + if (data->args.len > data->args.unencoded_len - data->args.unencoded_offset) + goto out_acct; + + data->iov = data->iovstack; + ret = import_iovec(ITER_SOURCE, data->args.iov, data->args.iovcnt, + ARRAY_SIZE(data->iovstack), &data->iov, + &data->iter); + if (ret < 0) + goto out_acct; + + if (iov_iter_count(&data->iter) == 0) { + ret = 0; + goto out_iov; + } + } + + if (issue_flags & IO_URING_F_NONBLOCK) { + ret = -EAGAIN; + goto out_acct; + } + + pos = data->args.offset; + ret = rw_verify_area(WRITE, file, &pos, data->args.len); + if (ret < 0) + goto out_iov; + + init_sync_kiocb(&kiocb, file); + ret = kiocb_set_rw_flags(&kiocb, 0, WRITE); + if (ret) + goto out_iov; + kiocb.ki_pos = pos; + + file_start_write(file); + + ret = btrfs_do_write_iter(&kiocb, &data->iter, &data->args); + if (ret > 0) + fsnotify_modify(file); + + file_end_write(file); +out_iov: + kfree(data->iov); +out_acct: + if (ret > 0) + add_wchar(current, ret); + inc_syscw(current); + + if (ret != -EAGAIN) + kfree(data); + return ret; +} + +int btrfs_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) +{ + if (btrfs_is_shutdown(inode_to_fs_info(file_inode(cmd->file)))) + return -EIO; + + switch (cmd->cmd_op) { + case BTRFS_IOC_ENCODED_READ: +#if defined(CONFIG_64BIT) && defined(CONFIG_COMPAT) + case BTRFS_IOC_ENCODED_READ_32: +#endif + return btrfs_uring_encoded_read(cmd, issue_flags); + + case BTRFS_IOC_ENCODED_WRITE: +#if defined(CONFIG_64BIT) && defined(CONFIG_COMPAT) + case BTRFS_IOC_ENCODED_WRITE_32: +#endif + return btrfs_uring_encoded_write(cmd, issue_flags); + } + + return -EINVAL; +} + +static int btrfs_ioctl_subvol_sync(struct btrfs_fs_info *fs_info, void __user *argp) +{ + struct btrfs_root *root; + struct btrfs_ioctl_subvol_wait args = { 0 }; + signed long sched_ret; + int refs; + u64 root_flags; + bool wait_for_deletion = false; + bool found = false; + + if (copy_from_user(&args, argp, sizeof(args))) + return -EFAULT; + + switch (args.mode) { + case BTRFS_SUBVOL_SYNC_WAIT_FOR_QUEUED: + /* + * Wait for the first one deleted that waits until all previous + * are cleaned. + */ + spin_lock(&fs_info->trans_lock); + if (!list_empty(&fs_info->dead_roots)) { + root = list_last_entry(&fs_info->dead_roots, + struct btrfs_root, root_list); + args.subvolid = btrfs_root_id(root); + found = true; + } + spin_unlock(&fs_info->trans_lock); + if (!found) + return -ENOENT; + + fallthrough; + case BTRFS_SUBVOL_SYNC_WAIT_FOR_ONE: + if ((0 < args.subvolid && args.subvolid < BTRFS_FIRST_FREE_OBJECTID) || + BTRFS_LAST_FREE_OBJECTID < args.subvolid) + return -EINVAL; + break; + case BTRFS_SUBVOL_SYNC_COUNT: + spin_lock(&fs_info->trans_lock); + args.count = list_count_nodes(&fs_info->dead_roots); + spin_unlock(&fs_info->trans_lock); + if (copy_to_user(argp, &args, sizeof(args))) + return -EFAULT; + return 0; + case BTRFS_SUBVOL_SYNC_PEEK_FIRST: + spin_lock(&fs_info->trans_lock); + /* Last in the list was deleted first. */ + if (!list_empty(&fs_info->dead_roots)) { + root = list_last_entry(&fs_info->dead_roots, + struct btrfs_root, root_list); + args.subvolid = btrfs_root_id(root); + } else { + args.subvolid = 0; + } + spin_unlock(&fs_info->trans_lock); + if (copy_to_user(argp, &args, sizeof(args))) + return -EFAULT; + return 0; + case BTRFS_SUBVOL_SYNC_PEEK_LAST: + spin_lock(&fs_info->trans_lock); + /* First in the list was deleted last. */ + if (!list_empty(&fs_info->dead_roots)) { + root = list_first_entry(&fs_info->dead_roots, + struct btrfs_root, root_list); + args.subvolid = btrfs_root_id(root); + } else { + args.subvolid = 0; + } + spin_unlock(&fs_info->trans_lock); + if (copy_to_user(argp, &args, sizeof(args))) + return -EFAULT; + return 0; + default: + return -EINVAL; + } + + /* 32bit limitation: fs_roots_radix key is not wide enough. */ + if (sizeof(unsigned long) != sizeof(u64) && args.subvolid > U32_MAX) + return -EOVERFLOW; + + while (1) { + /* Wait for the specific one. */ + if (down_read_interruptible(&fs_info->subvol_sem) == -EINTR) + return -EINTR; + refs = -1; + spin_lock(&fs_info->fs_roots_radix_lock); + root = radix_tree_lookup(&fs_info->fs_roots_radix, + (unsigned long)args.subvolid); + if (root) { + spin_lock(&root->root_item_lock); + refs = btrfs_root_refs(&root->root_item); + root_flags = btrfs_root_flags(&root->root_item); + spin_unlock(&root->root_item_lock); + } + spin_unlock(&fs_info->fs_roots_radix_lock); + up_read(&fs_info->subvol_sem); + + /* Subvolume does not exist. */ + if (!root) + return -ENOENT; + + /* Subvolume not deleted at all. */ + if (refs > 0) + return -EEXIST; + /* We've waited and now the subvolume is gone. */ + if (wait_for_deletion && refs == -1) { + /* Return the one we waited for as the last one. */ + if (copy_to_user(argp, &args, sizeof(args))) + return -EFAULT; + return 0; + } + + /* Subvolume not found on the first try (deleted or never existed). */ + if (refs == -1) + return -ENOENT; + + wait_for_deletion = true; + ASSERT(root_flags & BTRFS_ROOT_SUBVOL_DEAD, "root_flags=0x%llx", + root_flags); + sched_ret = schedule_timeout_interruptible(HZ); + /* Early wake up or error. */ + if (sched_ret != 0) + return -EINTR; + } + + return 0; +} + +static int btrfs_ioctl_shutdown(struct btrfs_fs_info *fs_info, unsigned long arg) +{ + int ret = 0; + u32 flags; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (get_user(flags, (u32 __user *)arg)) + return -EFAULT; + + if (flags >= BTRFS_SHUTDOWN_FLAGS_LAST) + return -EINVAL; + + if (btrfs_is_shutdown(fs_info)) + return 0; + + switch (flags) { + case BTRFS_SHUTDOWN_FLAGS_LOGFLUSH: + case BTRFS_SHUTDOWN_FLAGS_DEFAULT: + ret = freeze_super(fs_info->sb, FREEZE_HOLDER_KERNEL, NULL); + if (ret) + return ret; + btrfs_force_shutdown(fs_info); + ret = thaw_super(fs_info->sb, FREEZE_HOLDER_KERNEL, NULL); + if (ret) + return ret; + break; + case BTRFS_SHUTDOWN_FLAGS_NOLOGFLUSH: + btrfs_force_shutdown(fs_info); + break; + default: + ret = -EINVAL; + break; + } + return ret; +} + +#define GET_CSUMS_BUF_MAX SZ_16M + +static int copy_csums_to_user(struct btrfs_fs_info *fs_info, u64 disk_bytenr, + u64 len, u8 __user *buf) +{ + struct btrfs_root *csum_root; + struct btrfs_ordered_sum *sums; + LIST_HEAD(list); + const u32 csum_size = fs_info->csum_size; + int ret; + + csum_root = btrfs_csum_root(fs_info, disk_bytenr); + if (unlikely(!csum_root)) { + btrfs_err(fs_info, "missing csum root for extent at bytenr %llu", disk_bytenr); + return -EUCLEAN; + } + + ret = btrfs_lookup_csums_list(csum_root, disk_bytenr, + disk_bytenr + len - 1, &list, false); + if (ret < 0) + return ret; + + ret = 0; + while (!list_empty(&list)) { + u64 offset; + size_t copy_size; + + sums = list_first_entry(&list, struct btrfs_ordered_sum, list); + list_del(&sums->list); + + offset = ((sums->logical - disk_bytenr) >> fs_info->sectorsize_bits) * csum_size; + copy_size = (sums->len >> fs_info->sectorsize_bits) * csum_size; + + if (copy_to_user(buf + offset, sums->sums, copy_size)) { + kfree(sums); + ret = -EFAULT; + goto out; + } + + kfree(sums); + } + +out: + while (!list_empty(&list)) { + sums = list_first_entry(&list, struct btrfs_ordered_sum, list); + list_del(&sums->list); + kfree(sums); + } + return ret; +} + +static int btrfs_ioctl_get_csums(struct file *file, void __user *argp) +{ + struct inode *vfs_inode = file_inode(file); + struct btrfs_inode *inode = BTRFS_I(vfs_inode); + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct btrfs_root *root = inode->root; + struct btrfs_ioctl_get_csums_args args = { 0 }; + BTRFS_PATH_AUTO_FREE(path); + const u64 ino = btrfs_ino(inode); + const u32 csum_size = fs_info->csum_size; + u8 __user *ubuf; + u64 buf_limit; + u64 buf_used = 0; + u64 cur_offset; + u64 end_offset; + u64 prev_extent_end; + struct btrfs_key key; + int ret; + + if (!(file->f_mode & FMODE_READ)) + return -EBADF; + + if (!S_ISREG(vfs_inode->i_mode)) + return -EINVAL; + + if (copy_from_user(&args, argp, sizeof(args))) + return -EFAULT; + + if (!IS_ALIGNED(args.offset, fs_info->sectorsize) || + !IS_ALIGNED(args.length, fs_info->sectorsize)) + return -EINVAL; + if (args.length == 0) + return -EINVAL; + if (args.offset + args.length < args.offset) + return -EOVERFLOW; + if (args.flags != 0) + return -EINVAL; + if (args.buf_size < sizeof(struct btrfs_ioctl_get_csums_entry)) + return -EINVAL; + + buf_limit = min_t(u64, args.buf_size, GET_CSUMS_BUF_MAX); + ubuf = (u8 __user *)(argp + offsetof(struct btrfs_ioctl_get_csums_args, buf)); + + if (clear_user(ubuf, buf_limit)) + return -EFAULT; + + cur_offset = args.offset; + end_offset = args.offset + args.length; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + ret = btrfs_wait_ordered_range(inode, cur_offset, args.length); + if (ret) + return ret; + + ret = down_read_interruptible(&vfs_inode->i_rwsem); + if (ret) + return ret; + + ret = btrfs_wait_ordered_range(inode, cur_offset, args.length); + if (ret) + goto out_unlock; + + /* NODATASUM early exit. */ + if (inode->flags & BTRFS_INODE_NODATASUM) { + struct btrfs_ioctl_get_csums_entry entry = { + .offset = cur_offset, + .length = end_offset - cur_offset, + .type = BTRFS_GET_CSUMS_NODATASUM, + }; + + if (copy_to_user(ubuf, &entry, sizeof(entry))) { + ret = -EFAULT; + goto out_unlock; + } + + buf_used = sizeof(entry); + cur_offset = end_offset; + goto done; + } + + prev_extent_end = cur_offset; + + while (cur_offset < end_offset) { + struct btrfs_file_extent_item *ei; + struct extent_buffer *leaf; + struct btrfs_ioctl_get_csums_entry entry = { 0 }; + u64 extent_end; + u64 disk_bytenr = 0; + u64 extent_offset = 0; + u64 range_start, range_len; + u64 entry_csum_size; + u64 key_offset; + int extent_type; + u8 compression; + u8 encryption; + + /* Search for the extent at or before cur_offset. */ + key.objectid = ino; + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = cur_offset; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + goto out_unlock; + + if (ret > 0 && path->slots[0] > 0) { + btrfs_item_key_to_cpu(path->nodes[0], &key, + path->slots[0] - 1); + if (key.objectid == ino && key.type == BTRFS_EXTENT_DATA_KEY) { + path->slots[0]--; + if (btrfs_file_extent_end(path) <= cur_offset) + path->slots[0]++; + } + } + + if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) { + ret = btrfs_next_leaf(root, path); + if (ret < 0) + goto out_unlock; + if (ret > 0) { + ret = 0; + btrfs_release_path(path); + break; + } + } + + leaf = path->nodes[0]; + + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY) { + btrfs_release_path(path); + break; + } + + extent_end = btrfs_file_extent_end(path); + key_offset = key.offset; + + /* Read extent fields before releasing the path. */ + ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); + extent_type = btrfs_file_extent_type(leaf, ei); + compression = btrfs_file_extent_compression(leaf, ei); + encryption = btrfs_file_extent_encryption(leaf, ei); + + if (extent_type != BTRFS_FILE_EXTENT_INLINE) { + disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei); + if (disk_bytenr && compression == BTRFS_COMPRESS_NONE) + extent_offset = btrfs_file_extent_offset(leaf, ei); + } + + btrfs_release_path(path); + + /* Implicit hole (NO_HOLES feature). */ + if (prev_extent_end < key_offset) { + u64 hole_end = min(key_offset, end_offset); + u64 hole_len = hole_end - prev_extent_end; + + if (prev_extent_end >= cur_offset) { + entry.offset = prev_extent_end; + entry.length = hole_len; + entry.type = BTRFS_GET_CSUMS_ZEROED; + + if (buf_used + sizeof(entry) > buf_limit) + goto done; + if (copy_to_user(ubuf + buf_used, &entry, sizeof(entry))) { + ret = -EFAULT; + goto out_unlock; + } + buf_used += sizeof(entry); + cur_offset = hole_end; + } + + if (key_offset >= end_offset) { + cur_offset = end_offset; + break; + } + } + + /* Clamp to our query range. */ + range_start = max(cur_offset, key_offset); + range_len = min(extent_end, end_offset) - range_start; + + entry.offset = range_start; + entry.length = range_len; + + if (extent_type == BTRFS_FILE_EXTENT_INLINE) { + entry.type = BTRFS_GET_CSUMS_INLINE; + if (compression != BTRFS_COMPRESS_NONE) + entry.type |= BTRFS_GET_CSUMS_COMPRESSED; + if (encryption != 0) + entry.type |= BTRFS_GET_CSUMS_ENCRYPTED; + entry_csum_size = 0; + } else if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) { + entry.type = BTRFS_GET_CSUMS_ZEROED; + entry_csum_size = 0; + } else { + /* BTRFS_FILE_EXTENT_REG */ + if (disk_bytenr == 0) { + /* Explicit hole. */ + entry.type = BTRFS_GET_CSUMS_ZEROED; + entry_csum_size = 0; + } else if (encryption != 0 || compression != BTRFS_COMPRESS_NONE) { + entry.type = 0; + if (encryption != 0) + entry.type |= BTRFS_GET_CSUMS_ENCRYPTED; + if (compression != BTRFS_COMPRESS_NONE) + entry.type |= BTRFS_GET_CSUMS_COMPRESSED; + entry_csum_size = 0; + } else { + entry.type = BTRFS_GET_CSUMS_HAS_CSUMS; + entry_csum_size = (range_len >> fs_info->sectorsize_bits) * csum_size; + } + } + + /* Check if this entry (+ csum data) fits in the buffer. */ + if (buf_used + sizeof(entry) + entry_csum_size > buf_limit) { + if (buf_used == 0) { + ret = -EOVERFLOW; + goto out_unlock; + } + goto done; + } + + if (copy_to_user(ubuf + buf_used, &entry, sizeof(entry))) { + ret = -EFAULT; + goto out_unlock; + } + buf_used += sizeof(entry); + + if (entry.type == BTRFS_GET_CSUMS_HAS_CSUMS) { + ret = copy_csums_to_user(fs_info, + disk_bytenr + extent_offset + (range_start - key_offset), + range_len, ubuf + buf_used); + if (ret) + goto out_unlock; + buf_used += entry_csum_size; + } + + cur_offset = range_start + range_len; + prev_extent_end = extent_end; + + if (fatal_signal_pending(current)) { + if (buf_used == 0) { + ret = -EINTR; + goto out_unlock; + } + goto done; + } + + cond_resched(); + } + + /* Handle trailing implicit hole. */ + if (cur_offset < end_offset) { + struct btrfs_ioctl_get_csums_entry entry = { + .offset = prev_extent_end, + .length = end_offset - prev_extent_end, + .type = BTRFS_GET_CSUMS_ZEROED, + }; + + if (buf_used + sizeof(entry) <= buf_limit) { + if (copy_to_user(ubuf + buf_used, &entry, sizeof(entry))) { + ret = -EFAULT; + goto out_unlock; + } + buf_used += sizeof(entry); + cur_offset = end_offset; + } + } + +done: + args.offset = cur_offset; + args.length = (cur_offset < end_offset) ? end_offset - cur_offset : 0; + args.buf_size = buf_used; + + if (copy_to_user(argp, &args, sizeof(args))) + ret = -EFAULT; + +out_unlock: + up_read(&vfs_inode->i_rwsem); + return ret; +} + +long btrfs_ioctl(struct file *file, unsigned int + cmd, unsigned long arg) +{ + struct inode *inode = file_inode(file); + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + struct btrfs_root *root = BTRFS_I(inode)->root; + void __user *argp = (void __user *)arg; + + switch (cmd) { + case FS_IOC_GETVERSION: + return btrfs_ioctl_getversion(inode, argp); + case FS_IOC_GETFSLABEL: + return btrfs_ioctl_get_fslabel(fs_info, argp); + case FS_IOC_SETFSLABEL: + return btrfs_ioctl_set_fslabel(file, argp); + case FITRIM: + return btrfs_ioctl_fitrim(fs_info, argp); + case BTRFS_IOC_SNAP_CREATE: + return btrfs_ioctl_snap_create(file, argp, false); + case BTRFS_IOC_SNAP_CREATE_V2: + return btrfs_ioctl_snap_create_v2(file, argp, false); + case BTRFS_IOC_SUBVOL_CREATE: + return btrfs_ioctl_snap_create(file, argp, true); + case BTRFS_IOC_SUBVOL_CREATE_V2: + return btrfs_ioctl_snap_create_v2(file, argp, true); + case BTRFS_IOC_SNAP_DESTROY: + return btrfs_ioctl_snap_destroy(file, argp, false); + case BTRFS_IOC_SNAP_DESTROY_V2: + return btrfs_ioctl_snap_destroy(file, argp, true); + case BTRFS_IOC_SUBVOL_GETFLAGS: + return btrfs_ioctl_subvol_getflags(BTRFS_I(inode), argp); + case BTRFS_IOC_SUBVOL_SETFLAGS: + return btrfs_ioctl_subvol_setflags(file, argp); + case BTRFS_IOC_DEFAULT_SUBVOL: + return btrfs_ioctl_default_subvol(file, argp); + case BTRFS_IOC_DEFRAG: + return btrfs_ioctl_defrag(file, NULL); + case BTRFS_IOC_DEFRAG_RANGE: + return btrfs_ioctl_defrag(file, argp); + case BTRFS_IOC_RESIZE: + return btrfs_ioctl_resize(file, argp); + case BTRFS_IOC_ADD_DEV: + return btrfs_ioctl_add_dev(fs_info, argp); + case BTRFS_IOC_RM_DEV: + return btrfs_ioctl_rm_dev(file, argp); + case BTRFS_IOC_RM_DEV_V2: + return btrfs_ioctl_rm_dev_v2(file, argp); + case BTRFS_IOC_FS_INFO: + return btrfs_ioctl_fs_info(fs_info, argp); + case BTRFS_IOC_DEV_INFO: + return btrfs_ioctl_dev_info(fs_info, argp); + case BTRFS_IOC_TREE_SEARCH: + return btrfs_ioctl_tree_search(root, argp); + case BTRFS_IOC_TREE_SEARCH_V2: + return btrfs_ioctl_tree_search_v2(root, argp); + case BTRFS_IOC_INO_LOOKUP: + return btrfs_ioctl_ino_lookup(root, argp); + case BTRFS_IOC_INO_PATHS: + return btrfs_ioctl_ino_to_path(root, argp); + case BTRFS_IOC_LOGICAL_INO: + return btrfs_ioctl_logical_to_ino(fs_info, argp, 1); + case BTRFS_IOC_LOGICAL_INO_V2: + return btrfs_ioctl_logical_to_ino(fs_info, argp, 2); + case BTRFS_IOC_SPACE_INFO: + return btrfs_ioctl_space_info(fs_info, argp); + case BTRFS_IOC_SYNC: { + int ret; + + ret = btrfs_start_delalloc_roots(fs_info, LONG_MAX, false); + if (ret) + return ret; + ret = btrfs_sync_fs(inode->i_sb, 1); + /* + * There may be work for the cleaner kthread to do (subvolume + * deletion, delayed iputs, defrag inodes, etc), so wake it up. + */ + wake_up_process(fs_info->cleaner_kthread); + return ret; + } + case BTRFS_IOC_START_SYNC: + return btrfs_ioctl_start_sync(root, argp); + case BTRFS_IOC_WAIT_SYNC: + return btrfs_ioctl_wait_sync(fs_info, argp); + case BTRFS_IOC_SCRUB: + return btrfs_ioctl_scrub(file, argp); + case BTRFS_IOC_SCRUB_CANCEL: + return btrfs_ioctl_scrub_cancel(fs_info); + case BTRFS_IOC_SCRUB_PROGRESS: + return btrfs_ioctl_scrub_progress(fs_info, argp); + case BTRFS_IOC_BALANCE_V2: + return btrfs_ioctl_balance(file, argp); + case BTRFS_IOC_BALANCE_CTL: + return btrfs_ioctl_balance_ctl(fs_info, arg); + case BTRFS_IOC_BALANCE_PROGRESS: + return btrfs_ioctl_balance_progress(fs_info, argp); + case BTRFS_IOC_SET_RECEIVED_SUBVOL: + return btrfs_ioctl_set_received_subvol(file, argp); +#ifdef CONFIG_64BIT + case BTRFS_IOC_SET_RECEIVED_SUBVOL_32: + return btrfs_ioctl_set_received_subvol_32(file, argp); +#endif + case BTRFS_IOC_SEND: + return _btrfs_ioctl_send(root, argp, false); +#if defined(CONFIG_64BIT) && defined(CONFIG_COMPAT) + case BTRFS_IOC_SEND_32: + return _btrfs_ioctl_send(root, argp, true); +#endif + case BTRFS_IOC_GET_DEV_STATS: + return btrfs_ioctl_get_dev_stats(fs_info, argp); + case BTRFS_IOC_QUOTA_CTL: + return btrfs_ioctl_quota_ctl(file, argp); + case BTRFS_IOC_QGROUP_ASSIGN: + return btrfs_ioctl_qgroup_assign(file, argp); + case BTRFS_IOC_QGROUP_CREATE: + return btrfs_ioctl_qgroup_create(file, argp); + case BTRFS_IOC_QGROUP_LIMIT: + return btrfs_ioctl_qgroup_limit(file, argp); + case BTRFS_IOC_QUOTA_RESCAN: + return btrfs_ioctl_quota_rescan(file, argp); + case BTRFS_IOC_QUOTA_RESCAN_STATUS: + return btrfs_ioctl_quota_rescan_status(fs_info, argp); + case BTRFS_IOC_QUOTA_RESCAN_WAIT: + return btrfs_ioctl_quota_rescan_wait(fs_info); + case BTRFS_IOC_DEV_REPLACE: + return btrfs_ioctl_dev_replace(fs_info, argp); + case BTRFS_IOC_GET_SUPPORTED_FEATURES: + return btrfs_ioctl_get_supported_features(argp); + case BTRFS_IOC_GET_FEATURES: + return btrfs_ioctl_get_features(fs_info, argp); + case BTRFS_IOC_SET_FEATURES: + return btrfs_ioctl_set_features(file, argp); + case BTRFS_IOC_GET_SUBVOL_INFO: + return btrfs_ioctl_get_subvol_info(inode, argp); +#ifdef CONFIG_64BIT + case BTRFS_IOC_GET_SUBVOL_INFO_32: + return btrfs_ioctl_get_subvol_info_32(inode, argp); +#endif + case BTRFS_IOC_GET_SUBVOL_ROOTREF: + return btrfs_ioctl_get_subvol_rootref(root, argp); + case BTRFS_IOC_INO_LOOKUP_USER: + return btrfs_ioctl_ino_lookup_user(file, argp); + case FS_IOC_ENABLE_VERITY: + return fsverity_ioctl_enable(file, (const void __user *)argp); + case FS_IOC_MEASURE_VERITY: + return fsverity_ioctl_measure(file, argp); + case FS_IOC_READ_VERITY_METADATA: + return fsverity_ioctl_read_metadata(file, argp); + case BTRFS_IOC_ENCODED_READ: + return btrfs_ioctl_encoded_read(file, argp, false); + case BTRFS_IOC_ENCODED_WRITE: + return btrfs_ioctl_encoded_write(file, argp, false); +#if defined(CONFIG_64BIT) && defined(CONFIG_COMPAT) + case BTRFS_IOC_ENCODED_READ_32: + return btrfs_ioctl_encoded_read(file, argp, true); + case BTRFS_IOC_ENCODED_WRITE_32: + return btrfs_ioctl_encoded_write(file, argp, true); +#endif + case BTRFS_IOC_SUBVOL_SYNC_WAIT: + return btrfs_ioctl_subvol_sync(fs_info, argp); + case BTRFS_IOC_SHUTDOWN: + return btrfs_ioctl_shutdown(fs_info, arg); + case BTRFS_IOC_GET_CSUMS: + return btrfs_ioctl_get_csums(file, argp); + } + + return -ENOTTY; +} + +#ifdef CONFIG_COMPAT +long btrfs_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + /* + * These all access 32-bit values anyway so no further + * handling is necessary. + */ + switch (cmd) { + case FS_IOC32_GETVERSION: + cmd = FS_IOC_GETVERSION; + break; + } + + return btrfs_ioctl(file, cmd, (unsigned long) compat_ptr(arg)); +} +#endif diff --git a/fs/btrfs/ioctl.h b/fs/btrfs/ioctl.h new file mode 100644 index 000000000..ccf6bed9c --- /dev/null +++ b/fs/btrfs/ioctl.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_IOCTL_H +#define BTRFS_IOCTL_H + +#include + +struct file; +struct dentry; +struct mnt_idmap; +struct file_kattr; +struct io_uring_cmd; +struct btrfs_inode; +struct btrfs_fs_info; +struct btrfs_ioctl_balance_args; + +long btrfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg); +long btrfs_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg); +int btrfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa); +int btrfs_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa); +int btrfs_ioctl_get_supported_features(void __user *arg); +void btrfs_sync_inode_flags_to_i_flags(struct btrfs_inode *inode); +void btrfs_update_ioctl_balance_args(struct btrfs_fs_info *fs_info, + struct btrfs_ioctl_balance_args *bargs); +int btrfs_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags); +void btrfs_uring_read_extent_endio(void *ctx, int err); + +#endif diff --git a/fs/btrfs/locking.c b/fs/btrfs/locking.c new file mode 100644 index 000000000..e3df5ca0b --- /dev/null +++ b/fs/btrfs/locking.c @@ -0,0 +1,383 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2008 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include "ctree.h" +#include "extent_io.h" +#include "locking.h" + +/* + * Lockdep class keys for extent_buffer->lock's in this root. For a given + * eb, the lockdep key is determined by the btrfs_root it belongs to and + * the level the eb occupies in the tree. + * + * Different roots are used for different purposes and may nest inside each + * other and they require separate keysets. As lockdep keys should be + * static, assign keysets according to the purpose of the root as indicated + * by btrfs_root->root_key.objectid. This ensures that all special purpose + * roots have separate keysets. + * + * Lock-nesting across peer nodes is always done with the immediate parent + * node locked thus preventing deadlock. As lockdep doesn't know this, use + * subclass to avoid triggering lockdep warning in such cases. + * + * The key is set by the readpage_end_io_hook after the buffer has passed + * csum validation but before the pages are unlocked. It is also set by + * btrfs_init_new_buffer on freshly allocated blocks. + * + * We also add a check to make sure the highest level of the tree is the + * same as our lockdep setup here. If BTRFS_MAX_LEVEL changes, this code + * needs update as well. + */ +#ifdef CONFIG_DEBUG_LOCK_ALLOC +#if BTRFS_MAX_LEVEL != 8 +#error +#endif + +#define DEFINE_LEVEL(stem, level) \ + .names[level] = "btrfs-" stem "-0" #level, + +#define DEFINE_NAME(stem) \ + DEFINE_LEVEL(stem, 0) \ + DEFINE_LEVEL(stem, 1) \ + DEFINE_LEVEL(stem, 2) \ + DEFINE_LEVEL(stem, 3) \ + DEFINE_LEVEL(stem, 4) \ + DEFINE_LEVEL(stem, 5) \ + DEFINE_LEVEL(stem, 6) \ + DEFINE_LEVEL(stem, 7) + +static struct btrfs_lockdep_keyset { + u64 id; /* root objectid */ + /* Longest entry: btrfs-block-group-00 */ + char names[BTRFS_MAX_LEVEL][24]; + struct lock_class_key keys[BTRFS_MAX_LEVEL]; +} btrfs_lockdep_keysets[] = { + { .id = BTRFS_ROOT_TREE_OBJECTID, DEFINE_NAME("root") }, + { .id = BTRFS_EXTENT_TREE_OBJECTID, DEFINE_NAME("extent") }, + { .id = BTRFS_CHUNK_TREE_OBJECTID, DEFINE_NAME("chunk") }, + { .id = BTRFS_DEV_TREE_OBJECTID, DEFINE_NAME("dev") }, + { .id = BTRFS_CSUM_TREE_OBJECTID, DEFINE_NAME("csum") }, + { .id = BTRFS_QUOTA_TREE_OBJECTID, DEFINE_NAME("quota") }, + { .id = BTRFS_TREE_LOG_OBJECTID, DEFINE_NAME("log") }, + { .id = BTRFS_TREE_RELOC_OBJECTID, DEFINE_NAME("treloc") }, + { .id = BTRFS_DATA_RELOC_TREE_OBJECTID, DEFINE_NAME("dreloc") }, + { .id = BTRFS_UUID_TREE_OBJECTID, DEFINE_NAME("uuid") }, + { .id = BTRFS_FREE_SPACE_TREE_OBJECTID, DEFINE_NAME("free-space") }, + { .id = BTRFS_BLOCK_GROUP_TREE_OBJECTID, DEFINE_NAME("block-group") }, + { .id = BTRFS_RAID_STRIPE_TREE_OBJECTID, DEFINE_NAME("raid-stripe") }, + { .id = BTRFS_REMAP_TREE_OBJECTID, DEFINE_NAME("remap") }, + { .id = 0, DEFINE_NAME("tree") }, +}; + +#undef DEFINE_LEVEL +#undef DEFINE_NAME + +void btrfs_set_buffer_lockdep_class(u64 objectid, struct extent_buffer *eb, int level) +{ + struct btrfs_lockdep_keyset *ks; + + ASSERT(level < ARRAY_SIZE(ks->keys)); + + /* Find the matching keyset, id 0 is the default entry */ + for (ks = btrfs_lockdep_keysets; ks->id; ks++) + if (ks->id == objectid) + break; + + lockdep_set_class_and_name(&eb->lock, &ks->keys[level], ks->names[level]); +} + +void btrfs_maybe_reset_lockdep_class(struct btrfs_root *root, struct extent_buffer *eb) +{ + if (test_bit(BTRFS_ROOT_RESET_LOCKDEP_CLASS, &root->state)) + btrfs_set_buffer_lockdep_class(btrfs_root_id(root), + eb, btrfs_header_level(eb)); +} + +#endif + +#ifdef CONFIG_BTRFS_DEBUG +static void btrfs_set_eb_lock_owner(struct extent_buffer *eb, pid_t owner) +{ + eb->lock_owner = owner; +} +#else +static void btrfs_set_eb_lock_owner(struct extent_buffer *eb, pid_t owner) { } +#endif + +/* + * Extent buffer locking + * ===================== + * + * We use a rw_semaphore for tree locking, and the semantics are exactly the + * same: + * + * - reader/writer exclusion + * - writer/writer exclusion + * - reader/reader sharing + * - try-lock semantics for readers and writers + * + * The rwsem implementation does opportunistic spinning which reduces number of + * times the locking task needs to sleep. + */ + +/* + * btrfs_tree_read_lock_nested - lock extent buffer for read + * @eb: the eb to be locked + * @nest: the nesting level to be used for lockdep + * + * This takes the read lock on the extent buffer, using the specified nesting + * level for lockdep purposes. + */ +void btrfs_tree_read_lock_nested(struct extent_buffer *eb, enum btrfs_lock_nesting nest) +{ + u64 start_ns = 0; + + if (trace_btrfs_tree_read_lock_enabled()) + start_ns = ktime_get_ns(); + + down_read_nested(&eb->lock, nest); + trace_btrfs_tree_read_lock(eb, start_ns); +} + +/* + * Try-lock for read. + * + * Return true if the rwlock has been taken, false otherwise + */ +bool btrfs_try_tree_read_lock(struct extent_buffer *eb) +{ + if (down_read_trylock(&eb->lock)) { + trace_btrfs_try_tree_read_lock(eb); + return true; + } + return false; +} + +/* + * Release read lock. + */ +void btrfs_tree_read_unlock(struct extent_buffer *eb) +{ + trace_btrfs_tree_read_unlock(eb); + up_read(&eb->lock); +} + +/* + * Lock eb for write. + * + * @eb: the eb to lock + * @nest: the nesting to use for the lock + * + * Returns with the eb->lock write locked. + */ +void btrfs_tree_lock_nested(struct extent_buffer *eb, enum btrfs_lock_nesting nest) + __acquires(&eb->lock) +{ + u64 start_ns = 0; + + if (trace_btrfs_tree_lock_enabled()) + start_ns = ktime_get_ns(); + + down_write_nested(&eb->lock, nest); + btrfs_set_eb_lock_owner(eb, current->pid); + trace_btrfs_tree_lock(eb, start_ns); +} + +/* + * Release the write lock. + */ +void btrfs_tree_unlock(struct extent_buffer *eb) +{ + trace_btrfs_tree_unlock(eb); + btrfs_set_eb_lock_owner(eb, 0); + up_write(&eb->lock); +} + +/* + * This releases any locks held in the path starting at level and going all the + * way up to the root. + * + * btrfs_search_slot will keep the lock held on higher nodes in a few corner + * cases, such as COW of the block at slot zero in the node. This ignores + * those rules, and it should only be called when there are no more updates to + * be done higher up in the tree. + */ +void btrfs_unlock_up_safe(struct btrfs_path *path, int level) +{ + int i; + + if (path->keep_locks) + return; + + for (i = level; i < BTRFS_MAX_LEVEL; i++) { + if (!path->nodes[i]) + continue; + if (!path->locks[i]) + continue; + btrfs_tree_unlock_rw(path->nodes[i], path->locks[i]); + path->locks[i] = 0; + } +} + +/* + * Loop around taking references on and locking the root node of the tree until + * we end up with a lock on the root node. + * + * Return: root extent buffer with write lock held + */ +struct extent_buffer *btrfs_lock_root_node(struct btrfs_root *root) +{ + struct extent_buffer *eb; + + while (1) { + eb = btrfs_root_node(root); + + btrfs_maybe_reset_lockdep_class(root, eb); + btrfs_tree_lock(eb); + if (eb == root->node) + break; + btrfs_tree_unlock(eb); + free_extent_buffer(eb); + } + return eb; +} + +/* + * Loop around taking references on and locking the root node of the tree until + * we end up with a lock on the root node. + * + * Return: root extent buffer with read lock held + */ +struct extent_buffer *btrfs_read_lock_root_node(struct btrfs_root *root) +{ + struct extent_buffer *eb; + + while (1) { + eb = btrfs_root_node(root); + + btrfs_maybe_reset_lockdep_class(root, eb); + btrfs_tree_read_lock(eb); + if (eb == root->node) + break; + btrfs_tree_read_unlock(eb); + free_extent_buffer(eb); + } + return eb; +} + +/* + * Loop around taking references on and locking the root node of the tree in + * nowait mode until we end up with a lock on the root node or returning to + * avoid blocking. + * + * Return: root extent buffer with read lock held or -EAGAIN. + */ +struct extent_buffer *btrfs_try_read_lock_root_node(struct btrfs_root *root) +{ + struct extent_buffer *eb; + + while (1) { + eb = btrfs_root_node(root); + if (!btrfs_try_tree_read_lock(eb)) { + free_extent_buffer(eb); + return ERR_PTR(-EAGAIN); + } + if (eb == root->node) + break; + btrfs_tree_read_unlock(eb); + free_extent_buffer(eb); + } + return eb; +} + +/* + * DREW locks + * ========== + * + * DREW stands for double-reader-writer-exclusion lock. It's used in situation + * where you want to provide A-B exclusion but not AA or BB. + * + * Currently implementation gives more priority to reader. If a reader and a + * writer both race to acquire their respective sides of the lock the writer + * would yield its lock as soon as it detects a concurrent reader. Additionally + * if there are pending readers no new writers would be allowed to come in and + * acquire the lock. + */ + +void btrfs_drew_lock_init(struct btrfs_drew_lock *lock) +{ + atomic_set(&lock->readers, 0); + atomic_set(&lock->writers, 0); + init_waitqueue_head(&lock->pending_readers); + init_waitqueue_head(&lock->pending_writers); +} + +/* Return true if acquisition is successful, false otherwise */ +bool btrfs_drew_try_write_lock(struct btrfs_drew_lock *lock) +{ + if (atomic_read(&lock->readers)) + return false; + + atomic_inc(&lock->writers); + + /* Ensure writers count is updated before we check for pending readers */ + smp_mb__after_atomic(); + if (atomic_read(&lock->readers)) { + btrfs_drew_write_unlock(lock); + return false; + } + + return true; +} + +void btrfs_drew_write_lock(struct btrfs_drew_lock *lock) +{ + while (true) { + if (btrfs_drew_try_write_lock(lock)) + return; + wait_event(lock->pending_writers, !atomic_read(&lock->readers)); + } +} + +void btrfs_drew_write_unlock(struct btrfs_drew_lock *lock) +{ + /* + * atomic_dec_and_test() implies a full barrier, so woken up readers are + * guaranteed to see the decrement. + */ + if (atomic_dec_and_test(&lock->writers)) + wake_up(&lock->pending_readers); +} + +void btrfs_drew_read_lock(struct btrfs_drew_lock *lock) +{ + atomic_inc(&lock->readers); + + /* + * Ensure the pending reader count is perceived BEFORE this reader + * goes to sleep in case of active writers. This guarantees new writers + * won't be allowed and that the current reader will be woken up when + * the last active writer finishes its jobs. + */ + smp_mb__after_atomic(); + + wait_event(lock->pending_readers, atomic_read(&lock->writers) == 0); +} + +void btrfs_drew_read_unlock(struct btrfs_drew_lock *lock) +{ + /* + * atomic_dec_and_test implies a full barrier, so woken up writers + * are guaranteed to see the decrement + */ + if (atomic_dec_and_test(&lock->readers)) + wake_up(&lock->pending_writers); +} diff --git a/fs/btrfs/locking.h b/fs/btrfs/locking.h new file mode 100644 index 000000000..a4673e7d9 --- /dev/null +++ b/fs/btrfs/locking.h @@ -0,0 +1,251 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2008 Oracle. All rights reserved. + */ + +#ifndef BTRFS_LOCKING_H +#define BTRFS_LOCKING_H + +#include +#include +#include +#include +#include "extent_io.h" + +struct extent_buffer; +struct btrfs_path; +struct btrfs_root; + +#define BTRFS_WRITE_LOCK 1 +#define BTRFS_READ_LOCK 2 + +/* + * We are limited in number of subclasses by MAX_LOCKDEP_SUBCLASSES, which at + * the time of this patch is 8, which is how many we use. Keep this in mind if + * you decide you want to add another subclass. + */ +enum btrfs_lock_nesting { + BTRFS_NESTING_NORMAL, + + /* + * When we COW a block we are holding the lock on the original block, + * and since our lockdep maps are rootid+level, this confuses lockdep + * when we lock the newly allocated COW'd block. Handle this by having + * a subclass for COW'ed blocks so that lockdep doesn't complain. + */ + BTRFS_NESTING_COW, + + /* + * Oftentimes we need to lock adjacent nodes on the same level while + * still holding the lock on the original node we searched to, such as + * for searching forward or for split/balance. + * + * Because of this we need to indicate to lockdep that this is + * acceptable by having a different subclass for each of these + * operations. + */ + BTRFS_NESTING_LEFT, + BTRFS_NESTING_RIGHT, + + /* + * When splitting we will be holding a lock on the left/right node when + * we need to cow that node, thus we need a new set of subclasses for + * these two operations. + */ + BTRFS_NESTING_LEFT_COW, + BTRFS_NESTING_RIGHT_COW, + + /* + * When splitting we may push nodes to the left or right, but still use + * the subsequent nodes in our path, keeping our locks on those adjacent + * blocks. Thus when we go to allocate a new split block we've already + * used up all of our available subclasses, so this subclass exists to + * handle this case where we need to allocate a new split block. + */ + BTRFS_NESTING_SPLIT, + + /* + * When promoting a new block to a root we need to have a special + * subclass so we don't confuse lockdep, as it will appear that we are + * locking a higher level node before a lower level one. Copying also + * has this problem as it appears we're locking the same block again + * when we make a snapshot of an existing root. + */ + BTRFS_NESTING_NEW_ROOT, + + /* + * We are limited to MAX_LOCKDEP_SUBCLASSES number of subclasses, so + * add this in here and add a static_assert to keep us from going over + * the limit. As of this writing we're limited to 8, and we're + * definitely using 8, hence this check to keep us from messing up in + * the future. + */ + BTRFS_NESTING_MAX, +}; + +enum btrfs_lockdep_trans_states { + BTRFS_LOCKDEP_TRANS_COMMIT_PREP, + BTRFS_LOCKDEP_TRANS_UNBLOCKED, + BTRFS_LOCKDEP_TRANS_SUPER_COMMITTED, + BTRFS_LOCKDEP_TRANS_COMPLETED, +}; + +/* + * Lockdep annotation for wait events. + * + * @owner: The struct where the lockdep map is defined + * @lock: The lockdep map corresponding to a wait event + * + * This macro is used to annotate a wait event. In this case a thread acquires + * the lockdep map as writer (exclusive lock) because it has to block until all + * the threads that hold the lock as readers signal the condition for the wait + * event and release their locks. + */ +#define btrfs_might_wait_for_event(owner, lock) \ + do { \ + rwsem_acquire(&owner->lock##_map, 0, 0, _THIS_IP_); \ + rwsem_release(&owner->lock##_map, _THIS_IP_); \ + } while (0) + +/* + * Protection for the resource/condition of a wait event. + * + * @owner: The struct where the lockdep map is defined + * @lock: The lockdep map corresponding to a wait event + * + * Many threads can modify the condition for the wait event at the same time + * and signal the threads that block on the wait event. The threads that modify + * the condition and do the signaling acquire the lock as readers (shared + * lock). + */ +#define btrfs_lockdep_acquire(owner, lock) \ + rwsem_acquire_read(&owner->lock##_map, 0, 0, _THIS_IP_) + +/* + * Used after signaling the condition for a wait event to release the lockdep + * map held by a reader thread. + */ +#define btrfs_lockdep_release(owner, lock) \ + rwsem_release(&owner->lock##_map, _THIS_IP_) + +/* + * Used to account for the fact that when doing io_uring encoded I/O, we can + * return to userspace with the inode lock still held. + */ +#define btrfs_lockdep_inode_acquire(owner, lock) \ + rwsem_acquire_read(&owner->vfs_inode.lock.dep_map, 0, 0, _THIS_IP_) + +#define btrfs_lockdep_inode_release(owner, lock) \ + rwsem_release(&owner->vfs_inode.lock.dep_map, _THIS_IP_) + +/* + * Macros for the transaction states wait events, similar to the generic wait + * event macros. + */ +#define btrfs_might_wait_for_state(owner, i) \ + do { \ + rwsem_acquire(&owner->btrfs_state_change_map[i], 0, 0, _THIS_IP_); \ + rwsem_release(&owner->btrfs_state_change_map[i], _THIS_IP_); \ + } while (0) + +#define btrfs_trans_state_lockdep_acquire(owner, i) \ + rwsem_acquire_read(&owner->btrfs_state_change_map[i], 0, 0, _THIS_IP_) + +#define btrfs_trans_state_lockdep_release(owner, i) \ + rwsem_release(&owner->btrfs_state_change_map[i], _THIS_IP_) + +/* Initialization of the lockdep map */ +#define btrfs_lockdep_init_map(owner, lock) \ + do { \ + static struct lock_class_key lock##_key; \ + lockdep_init_map(&owner->lock##_map, #lock, &lock##_key, 0); \ + } while (0) + +/* Initialization of the transaction states lockdep maps. */ +#define btrfs_state_lockdep_init_map(owner, lock, state) \ + do { \ + static struct lock_class_key lock##_key; \ + lockdep_init_map(&owner->btrfs_state_change_map[state], #lock, \ + &lock##_key, 0); \ + } while (0) + +static_assert(BTRFS_NESTING_MAX <= MAX_LOCKDEP_SUBCLASSES, + "too many lock subclasses defined"); + +void btrfs_tree_lock_nested(struct extent_buffer *eb, enum btrfs_lock_nesting nest); + +static inline void btrfs_tree_lock(struct extent_buffer *eb) +{ + btrfs_tree_lock_nested(eb, BTRFS_NESTING_NORMAL); +} + +void btrfs_tree_unlock(struct extent_buffer *eb); + +void btrfs_tree_read_lock_nested(struct extent_buffer *eb, enum btrfs_lock_nesting nest); + +static inline void btrfs_tree_read_lock(struct extent_buffer *eb) +{ + btrfs_tree_read_lock_nested(eb, BTRFS_NESTING_NORMAL); +} + +void btrfs_tree_read_unlock(struct extent_buffer *eb); +bool btrfs_try_tree_read_lock(struct extent_buffer *eb); +struct extent_buffer *btrfs_lock_root_node(struct btrfs_root *root); +struct extent_buffer *btrfs_read_lock_root_node(struct btrfs_root *root); +struct extent_buffer *btrfs_try_read_lock_root_node(struct btrfs_root *root); + +#ifdef CONFIG_BTRFS_DEBUG +static inline void btrfs_assert_tree_write_locked(struct extent_buffer *eb) +{ + lockdep_assert_held_write(&eb->lock); +} +static inline void btrfs_assert_tree_read_locked(struct extent_buffer *eb) +{ + lockdep_assert_held_read(&eb->lock); +} +#else +static inline void btrfs_assert_tree_write_locked(struct extent_buffer *eb) { } +static inline void btrfs_assert_tree_read_locked(struct extent_buffer *eb) { } +#endif + +void btrfs_unlock_up_safe(struct btrfs_path *path, int level); + +static inline void btrfs_tree_unlock_rw(struct extent_buffer *eb, int rw) +{ + if (rw == BTRFS_WRITE_LOCK) + btrfs_tree_unlock(eb); + else if (rw == BTRFS_READ_LOCK) + btrfs_tree_read_unlock(eb); + else + BUG(); +} + +struct btrfs_drew_lock { + atomic_t readers; + atomic_t writers; + wait_queue_head_t pending_writers; + wait_queue_head_t pending_readers; +}; + +void btrfs_drew_lock_init(struct btrfs_drew_lock *lock); +void btrfs_drew_write_lock(struct btrfs_drew_lock *lock); +bool btrfs_drew_try_write_lock(struct btrfs_drew_lock *lock); +void btrfs_drew_write_unlock(struct btrfs_drew_lock *lock); +void btrfs_drew_read_lock(struct btrfs_drew_lock *lock); +void btrfs_drew_read_unlock(struct btrfs_drew_lock *lock); + +#ifdef CONFIG_DEBUG_LOCK_ALLOC +void btrfs_set_buffer_lockdep_class(u64 objectid, struct extent_buffer *eb, int level); +void btrfs_maybe_reset_lockdep_class(struct btrfs_root *root, struct extent_buffer *eb); +#else +static inline void btrfs_set_buffer_lockdep_class(u64 objectid, + struct extent_buffer *eb, int level) +{ +} +static inline void btrfs_maybe_reset_lockdep_class(struct btrfs_root *root, + struct extent_buffer *eb) +{ +} +#endif + +#endif diff --git a/fs/btrfs/lru_cache.c b/fs/btrfs/lru_cache.c new file mode 100644 index 000000000..f5a053b90 --- /dev/null +++ b/fs/btrfs/lru_cache.c @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include "lru_cache.h" +#include "messages.h" + +/* + * Initialize a cache object. + * + * @cache: The cache. + * @max_size: Maximum size (number of entries) for the cache. + * Use 0 for unlimited size, it's the user's responsibility to + * trim the cache in that case. + */ +void btrfs_lru_cache_init(struct btrfs_lru_cache *cache, unsigned int max_size) +{ + INIT_LIST_HEAD(&cache->lru_list); + mt_init(&cache->entries); + cache->size = 0; + cache->max_size = max_size; +} + +static struct btrfs_lru_cache_entry *match_entry(struct list_head *head, u64 key, + u64 gen) +{ + struct btrfs_lru_cache_entry *entry; + + list_for_each_entry(entry, head, list) { + if (entry->key == key && entry->gen == gen) + return entry; + } + + return NULL; +} + +/* + * Lookup for an entry in the cache. + * + * @cache: The cache. + * @key: The key of the entry we are looking for. + * @gen: Generation associated to the key. + * + * Returns the entry associated with the key or NULL if none found. + */ +struct btrfs_lru_cache_entry *btrfs_lru_cache_lookup(struct btrfs_lru_cache *cache, + u64 key, u64 gen) +{ + struct list_head *head; + struct btrfs_lru_cache_entry *entry; + + head = mtree_load(&cache->entries, key); + if (!head) + return NULL; + + entry = match_entry(head, key, gen); + if (entry) + list_move_tail(&entry->lru_list, &cache->lru_list); + + return entry; +} + +/* + * Remove an entry from the cache. + * + * @cache: The cache to remove from. + * @entry: The entry to remove from the cache. + * + * Note: this also frees the memory used by the entry. + */ +void btrfs_lru_cache_remove(struct btrfs_lru_cache *cache, + struct btrfs_lru_cache_entry *entry) +{ + struct list_head *prev = entry->list.prev; + + ASSERT(cache->size > 0); + ASSERT(!mtree_empty(&cache->entries)); + + list_del(&entry->list); + list_del(&entry->lru_list); + + if (list_empty(prev)) { + struct list_head *head; + + /* + * If previous element in the list entry->list is now empty, it + * means it's a head entry not pointing to any cached entries, + * so remove it from the maple tree and free it. + */ + head = mtree_erase(&cache->entries, entry->key); + ASSERT(head == prev); + kfree(head); + } + + kfree(entry); + cache->size--; +} + +/* + * Store an entry in the cache. + * + * @cache: The cache. + * @entry: The entry to store. + * + * Returns 0 on success and < 0 on error. + */ +int btrfs_lru_cache_store(struct btrfs_lru_cache *cache, + struct btrfs_lru_cache_entry *new_entry, + gfp_t gfp) +{ + const u64 key = new_entry->key; + struct list_head *head; + int ret; + + head = kmalloc_obj(*head, gfp); + if (!head) + return -ENOMEM; + + ret = mtree_insert(&cache->entries, key, head, gfp); + if (ret == 0) { + INIT_LIST_HEAD(head); + list_add_tail(&new_entry->list, head); + } else if (ret == -EEXIST) { + kfree(head); + head = mtree_load(&cache->entries, key); + ASSERT(head != NULL); + if (match_entry(head, key, new_entry->gen) != NULL) + return -EEXIST; + list_add_tail(&new_entry->list, head); + } else if (ret < 0) { + kfree(head); + return ret; + } + + if (cache->max_size > 0 && cache->size == cache->max_size) { + struct btrfs_lru_cache_entry *lru_entry; + + lru_entry = list_first_entry(&cache->lru_list, + struct btrfs_lru_cache_entry, + lru_list); + btrfs_lru_cache_remove(cache, lru_entry); + } + + list_add_tail(&new_entry->lru_list, &cache->lru_list); + cache->size++; + + return 0; +} + +/* + * Empty a cache. + * + * @cache: The cache to empty. + * + * Removes all entries from the cache. + */ +void btrfs_lru_cache_clear(struct btrfs_lru_cache *cache) +{ + struct btrfs_lru_cache_entry *entry; + struct btrfs_lru_cache_entry *tmp; + + list_for_each_entry_safe(entry, tmp, &cache->lru_list, lru_list) + btrfs_lru_cache_remove(cache, entry); + + ASSERT(cache->size == 0); + ASSERT(mtree_empty(&cache->entries)); +} diff --git a/fs/btrfs/lru_cache.h b/fs/btrfs/lru_cache.h new file mode 100644 index 000000000..07f1bb1c6 --- /dev/null +++ b/fs/btrfs/lru_cache.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_LRU_CACHE_H +#define BTRFS_LRU_CACHE_H + +#include +#include +#include + +/* + * A cache entry. This is meant to be embedded in a structure of a user of + * this module. Similar to how struct list_head and struct rb_node are used. + * + * Note: it should be embedded as the first element in a struct (offset 0), and + * this module assumes it was allocated with kmalloc(), so it calls kfree() when + * it needs to free an entry. + */ +struct btrfs_lru_cache_entry { + struct list_head lru_list; + u64 key; + /* + * Optional generation associated to a key. Use 0 if not needed/used. + * Entries with the same key and different generations are stored in a + * linked list, so use this only for cases where there's a small number + * of different generations. + */ + u64 gen; + /* + * The maple tree uses unsigned long type for the keys, which is 32 bits + * on 32 bits systems, and 64 bits on 64 bits systems. So if we want to + * use something like inode numbers as keys, which are always a u64, we + * have to deal with this in a special way - we store the key in the + * entry itself, as a u64, and the values inserted into the maple tree + * are linked lists of entries - so in case we are on a 64 bits system, + * that list always has a single entry, while on 32 bits systems it + * may have more than one, with each entry having the same value for + * their lower 32 bits of the u64 key. + */ + struct list_head list; +}; + +struct btrfs_lru_cache { + struct list_head lru_list; + struct maple_tree entries; + /* Number of entries stored in the cache. */ + unsigned int size; + /* Maximum number of entries the cache can have. */ + unsigned int max_size; +}; + +#define btrfs_lru_cache_for_each_entry_safe(cache, entry, tmp) \ + list_for_each_entry_safe_reverse((entry), (tmp), &(cache)->lru_list, lru_list) + +static inline struct btrfs_lru_cache_entry *btrfs_lru_cache_lru_entry( + struct btrfs_lru_cache *cache) +{ + return list_first_entry_or_null(&cache->lru_list, + struct btrfs_lru_cache_entry, lru_list); +} + +void btrfs_lru_cache_init(struct btrfs_lru_cache *cache, unsigned int max_size); +struct btrfs_lru_cache_entry *btrfs_lru_cache_lookup(struct btrfs_lru_cache *cache, + u64 key, u64 gen); +int btrfs_lru_cache_store(struct btrfs_lru_cache *cache, + struct btrfs_lru_cache_entry *new_entry, + gfp_t gfp); +void btrfs_lru_cache_remove(struct btrfs_lru_cache *cache, + struct btrfs_lru_cache_entry *entry); +void btrfs_lru_cache_clear(struct btrfs_lru_cache *cache); + +#endif diff --git a/fs/btrfs/lzo.c b/fs/btrfs/lzo.c new file mode 100644 index 000000000..2f0996692 --- /dev/null +++ b/fs/btrfs/lzo.c @@ -0,0 +1,604 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2008 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "messages.h" +#include "compression.h" +#include "ctree.h" +#include "super.h" +#include "btrfs_inode.h" + +#define LZO_LEN 4 + +/* + * Btrfs LZO compression format + * + * Regular and inlined LZO compressed data extents consist of: + * + * 1. Header + * Fixed size. LZO_LEN (4) bytes long, LE32. + * Records the total size (including the header) of compressed data. + * + * 2. Segment(s) + * Variable size. Each segment includes one segment header, followed by data + * payload. + * One regular LZO compressed extent can have one or more segments. + * For inlined LZO compressed extent, only one segment is allowed. + * One segment represents at most one sector of uncompressed data. + * + * 2.1 Segment header + * Fixed size. LZO_LEN (4) bytes long, LE32. + * Records the total size of the segment (not including the header). + * Segment header never crosses sector boundary, thus it's possible to + * have at most 3 padding zeros at the end of the sector. + * + * 2.2 Data Payload + * Variable size. Size up limit should be lzo1x_worst_compress(sectorsize) + * which is 4419 for a 4KiB sectorsize. + * + * Example with 4K sectorsize: + * Page 1: + * 0 0x2 0x4 0x6 0x8 0xa 0xc 0xe 0x10 + * 0x0000 | Header | SegHdr 01 | Data payload 01 ... | + * ... + * 0x0ff0 | SegHdr N | Data payload N ... |00| + * ^^ padding zeros + * Page 2: + * 0x1000 | SegHdr N+1| Data payload N+1 ... | + */ + +struct workspace { + void *mem; + void *buf; /* where decompressed data goes */ + void *cbuf; /* where compressed data goes */ + struct list_head list; +}; + +static u32 workspace_buf_length(const struct btrfs_fs_info *fs_info) +{ + return lzo1x_worst_compress(fs_info->sectorsize); +} +static u32 workspace_cbuf_length(const struct btrfs_fs_info *fs_info) +{ + return lzo1x_worst_compress(fs_info->sectorsize); +} + +void lzo_free_workspace(struct list_head *ws) +{ + struct workspace *workspace = list_entry(ws, struct workspace, list); + + kvfree(workspace->buf); + kvfree(workspace->cbuf); + kvfree(workspace->mem); + kfree(workspace); +} + +struct list_head *lzo_alloc_workspace(struct btrfs_fs_info *fs_info) +{ + struct workspace *workspace; + + workspace = kzalloc_obj(*workspace); + if (!workspace) + return ERR_PTR(-ENOMEM); + + workspace->mem = kvmalloc(LZO1X_MEM_COMPRESS, GFP_KERNEL | __GFP_NOWARN); + workspace->buf = kvmalloc(workspace_buf_length(fs_info), GFP_KERNEL | __GFP_NOWARN); + workspace->cbuf = kvmalloc(workspace_cbuf_length(fs_info), GFP_KERNEL | __GFP_NOWARN); + if (!workspace->mem || !workspace->buf || !workspace->cbuf) + goto fail; + + INIT_LIST_HEAD(&workspace->list); + + return &workspace->list; +fail: + lzo_free_workspace(&workspace->list); + return ERR_PTR(-ENOMEM); +} + +/* + * Write data into @out_folio and queue it into @out_bio. + * + * Return 0 if everything is fine and @total_out will be increased. + * Return <0 for error. + * + * The @out_folio can be NULL after a full folio is queued. + * Thus the caller should check and allocate a new folio when needed. + */ +static int write_and_queue_folio(struct bio *out_bio, struct folio **out_folio, + u32 *total_out, u32 write_len) +{ + const u32 fsize = folio_size(*out_folio); + const u32 foffset = offset_in_folio(*out_folio, *total_out); + + ASSERT(out_folio && *out_folio); + /* Should not cross folio boundary. */ + ASSERT(foffset + write_len <= fsize); + + /* We can not use bio_add_folio_nofail() which doesn't do any merge. */ + if (!bio_add_folio(out_bio, *out_folio, write_len, foffset)) { + /* + * We have allocated a bio that havs BTRFS_MAX_COMPRESSED_PAGES + * vecs, and all ranges inside the same folio should have been + * merged. If bio_add_folio() still failed, that means we have + * reached the bvec limits. + * + * This should only happen at the beginning of a folio, and + * caller is responsible for releasing the folio, since it's + * not yet queued into the bio. + */ + ASSERT(IS_ALIGNED(*total_out, fsize)); + return -E2BIG; + } + + *total_out += write_len; + /* + * The full folio has been filled and queued, reset @out_folio to NULL, + * so that error handling is fully handled by the bio. + */ + if (IS_ALIGNED(*total_out, fsize)) + *out_folio = NULL; + return 0; +} + +/* + * Copy compressed data to bio. + * + * @out_bio: The bio that will contain all the compressed data. + * @compressed_data: The compressed data of this segment. + * @compressed_size: The size of the compressed data. + * @out_folio: The current output folio, will be updated if a new + * folio is allocated. + * @total_out: The total bytes of current output. + * @max_out: The maximum size of the compressed data. + * + * Will do: + * + * - Write a segment header into the destination + * - Copy the compressed buffer into the destination + * - Make sure we have enough space in the last sector to fit a segment header + * If not, we will pad at most (LZO_LEN (4)) - 1 bytes of zeros. + * - If a full folio is filled, it will be queued into @out_bio, and @out_folio + * will be updated. + * + * Will allocate new pages when needed. + */ +static int copy_compressed_data_to_bio(struct btrfs_fs_info *fs_info, + struct bio *out_bio, + const char *compressed_data, + size_t compressed_size, + struct folio **out_folio, + u32 *total_out, u32 max_out) +{ + const u32 sectorsize = fs_info->sectorsize; + const u32 sectorsize_bits = fs_info->sectorsize_bits; + const u32 fsize = btrfs_min_folio_size(fs_info); + const u32 old_size = out_bio->bi_iter.bi_size; + u32 copy_start; + u32 sector_bytes_left; + char *kaddr; + int ret; + + ASSERT(out_folio); + + /* There should be at least a lzo header queued. */ + ASSERT(old_size); + ASSERT(old_size == *total_out); + + /* + * We never allow a segment header crossing sector boundary, previous + * run should ensure we have enough space left inside the sector. + */ + ASSERT((old_size >> sectorsize_bits) == (old_size + LZO_LEN - 1) >> sectorsize_bits); + + if (!*out_folio) { + *out_folio = btrfs_alloc_compr_folio(fs_info, GFP_NOFS); + if (!*out_folio) + return -ENOMEM; + } + + /* Write the segment header first. */ + kaddr = kmap_local_folio(*out_folio, offset_in_folio(*out_folio, *total_out)); + put_unaligned_le32(compressed_size, kaddr); + kunmap_local(kaddr); + ret = write_and_queue_folio(out_bio, out_folio, total_out, LZO_LEN); + if (ret < 0) + return ret; + + copy_start = *total_out; + + /* Copy compressed data. */ + while (*total_out - copy_start < compressed_size) { + u32 copy_len = min_t(u32, sectorsize - *total_out % sectorsize, + copy_start + compressed_size - *total_out); + u32 foffset = *total_out & (fsize - 1); + + /* With the range copied, we're larger than the original range. */ + if (((*total_out + copy_len) >> sectorsize_bits) >= + max_out >> sectorsize_bits) + return -E2BIG; + + if (!*out_folio) { + *out_folio = btrfs_alloc_compr_folio(fs_info, GFP_NOFS); + if (!*out_folio) + return -ENOMEM; + } + + kaddr = kmap_local_folio(*out_folio, foffset); + memcpy(kaddr, compressed_data + *total_out - copy_start, copy_len); + kunmap_local(kaddr); + ret = write_and_queue_folio(out_bio, out_folio, total_out, copy_len); + if (ret < 0) + return ret; + } + + /* + * Check if we can fit the next segment header into the remaining space + * of the sector. + */ + sector_bytes_left = round_up(*total_out, sectorsize) - *total_out; + if (sector_bytes_left >= LZO_LEN || sector_bytes_left == 0) + return 0; + + ASSERT(*out_folio); + + /* The remaining size is not enough, pad it with zeros */ + folio_zero_range(*out_folio, offset_in_folio(*out_folio, *total_out), sector_bytes_left); + return write_and_queue_folio(out_bio, out_folio, total_out, sector_bytes_left); +} + +int lzo_compress_bio(struct list_head *ws, struct compressed_bio *cb) +{ + struct btrfs_inode *inode = cb->bbio.inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct workspace *workspace = list_entry(ws, struct workspace, list); + struct bio *bio = &cb->bbio.bio; + const u64 start = cb->start; + const u32 len = cb->len; + const u32 sectorsize = fs_info->sectorsize; + const u32 min_folio_size = btrfs_min_folio_size(fs_info); + struct address_space *mapping = inode->vfs_inode.i_mapping; + struct folio *folio_in = NULL; + struct folio *folio_out = NULL; + char *sizes_ptr; + int ret = 0; + /* Points to the file offset of input data. */ + u64 cur_in = start; + /* Points to the current output byte. */ + u32 total_out = 0; + + ASSERT(bio->bi_iter.bi_size == 0); + ASSERT(len); + + folio_out = btrfs_alloc_compr_folio(fs_info, GFP_NOFS); + if (!folio_out) + return -ENOMEM; + + /* Queue a segment header first. */ + ret = write_and_queue_folio(bio, &folio_out, &total_out, LZO_LEN); + /* The first header should not fail. */ + ASSERT(ret == 0); + + while (cur_in < start + len) { + char *data_in; + const u32 sectorsize_mask = sectorsize - 1; + u32 sector_off = (cur_in - start) & sectorsize_mask; + u32 in_len; + size_t out_len; + + /* Get the input page first. */ + if (!folio_in) { + ret = btrfs_compress_filemap_get_folio(mapping, cur_in, &folio_in); + if (ret < 0) + goto out; + } + + /* Compress at most one sector of data each time. */ + in_len = min_t(u32, start + len - cur_in, sectorsize - sector_off); + ASSERT(in_len); + data_in = kmap_local_folio(folio_in, offset_in_folio(folio_in, cur_in)); + ret = lzo1x_1_compress(data_in, in_len, workspace->cbuf, &out_len, + workspace->mem); + kunmap_local(data_in); + if (unlikely(ret < 0)) { + /* lzo1x_1_compress never fails. */ + ret = -EIO; + goto out; + } + + ret = copy_compressed_data_to_bio(fs_info, bio, workspace->cbuf, out_len, + &folio_out, &total_out, len); + if (ret < 0) + goto out; + + cur_in += in_len; + + /* + * Check if we're making it bigger after two sectors. And if + * it is so, give up. + */ + if (cur_in - start > sectorsize * 2 && cur_in - start < total_out) { + ret = -E2BIG; + goto out; + } + + /* Check if we have reached input folio boundary. */ + if (IS_ALIGNED(cur_in, min_folio_size)) { + folio_put(folio_in); + folio_in = NULL; + } + } + /* + * The last folio is already queued. Bio is responsible for freeing + * those folios now. + */ + folio_out = NULL; + + /* Store the size of all chunks of compressed data */ + sizes_ptr = kmap_local_folio(bio_first_folio_all(bio), 0); + put_unaligned_le32(total_out, sizes_ptr); + kunmap_local(sizes_ptr); +out: + /* + * We can only free the folio that has no part queued into the bio. + * + * As any folio that is already queued into bio will be released by + * the endio function of bio. + */ + if (folio_out && IS_ALIGNED(total_out, min_folio_size)) { + btrfs_free_compr_folio(folio_out); + folio_out = NULL; + } + if (folio_in) + folio_put(folio_in); + return ret; +} + +static struct folio *get_current_folio(struct compressed_bio *cb, struct folio_iter *fi, + u32 *cur_folio_index, u32 cur_in) +{ + struct btrfs_fs_info *fs_info = cb_to_fs_info(cb); + const u32 min_folio_shift = PAGE_SHIFT + fs_info->block_min_order; + + ASSERT(cur_folio_index); + + /* Need to switch to the next folio. */ + if (cur_in >> min_folio_shift != *cur_folio_index) { + /* We can only do the switch one folio a time. */ + ASSERT(cur_in >> min_folio_shift == *cur_folio_index + 1); + + bio_next_folio(fi, &cb->bbio.bio); + (*cur_folio_index)++; + } + return fi->folio; +} + +/* + * Copy the compressed segment payload into @dest. + * + * For the payload there will be no padding, just need to do page switching. + */ +static void copy_compressed_segment(struct compressed_bio *cb, + struct folio_iter *fi, u32 *cur_folio_index, + char *dest, u32 len, u32 *cur_in) +{ + u32 orig_in = *cur_in; + + while (*cur_in < orig_in + len) { + struct folio *cur_folio = get_current_folio(cb, fi, cur_folio_index, *cur_in); + u32 copy_len; + + ASSERT(cur_folio); + copy_len = min_t(u32, orig_in + len - *cur_in, + folio_size(cur_folio) - offset_in_folio(cur_folio, *cur_in)); + ASSERT(copy_len); + + memcpy_from_folio(dest + *cur_in - orig_in, cur_folio, + offset_in_folio(cur_folio, *cur_in), copy_len); + + *cur_in += copy_len; + } +} + +int lzo_decompress_bio(struct list_head *ws, struct compressed_bio *cb) +{ + struct workspace *workspace = list_entry(ws, struct workspace, list); + struct btrfs_fs_info *fs_info = cb->bbio.inode->root->fs_info; + const u32 sectorsize = fs_info->sectorsize; + const u32 compressed_len = bio_get_size(&cb->bbio.bio); + struct folio_iter fi; + char *kaddr; + int ret; + /* Compressed data length, can be unaligned */ + u32 len_in; + /* Offset inside the compressed data */ + u32 cur_in = 0; + /* Bytes decompressed so far */ + u32 cur_out = 0; + /* The current folio index number inside the bio. */ + u32 cur_folio_index = 0; + + bio_first_folio(&fi, &cb->bbio.bio, 0); + /* There must be a compressed folio and matches the sectorsize. */ + if (unlikely(!fi.folio)) + return -EINVAL; + ASSERT(folio_size(fi.folio) == btrfs_min_folio_size(fs_info)); + kaddr = kmap_local_folio(fi.folio, 0); + len_in = get_unaligned_le32(kaddr); + kunmap_local(kaddr); + cur_in += LZO_LEN; + + /* + * LZO header length check + * + * The total length should not exceed the maximum extent length, + * and all sectors should be used. + * If this happens, it means the compressed extent is corrupted. + */ + if (unlikely(len_in > min_t(size_t, BTRFS_MAX_COMPRESSED, compressed_len) || + round_up(len_in, sectorsize) < compressed_len)) { + struct btrfs_inode *inode = cb->bbio.inode; + + btrfs_err(fs_info, +"lzo header invalid, root %llu inode %llu offset %llu lzo len %u compressed len %u", + btrfs_root_id(inode->root), btrfs_ino(inode), + cb->start, len_in, compressed_len); + return -EUCLEAN; + } + + /* Go through each lzo segment */ + while (cur_in < len_in) { + struct folio *cur_folio; + /* Length of the compressed segment */ + u32 seg_len; + u32 sector_bytes_left; + size_t out_len = lzo1x_worst_compress(sectorsize); + + /* + * We should always have enough space for one segment header + * inside current sector. + */ + ASSERT(cur_in / sectorsize == + (cur_in + LZO_LEN - 1) / sectorsize); + cur_folio = get_current_folio(cb, &fi, &cur_folio_index, cur_in); + ASSERT(cur_folio); + kaddr = kmap_local_folio(cur_folio, 0); + seg_len = get_unaligned_le32(kaddr + offset_in_folio(cur_folio, cur_in)); + kunmap_local(kaddr); + cur_in += LZO_LEN; + + if (unlikely(seg_len > workspace_cbuf_length(fs_info))) { + struct btrfs_inode *inode = cb->bbio.inode; + + /* + * seg_len shouldn't be larger than we have allocated + * for workspace->cbuf + */ + btrfs_err(fs_info, + "lzo segment too big, root %llu inode %llu offset %llu len %u", + btrfs_root_id(inode->root), btrfs_ino(inode), + cb->start, seg_len); + return -EIO; + } + + /* The segment must not extend beyond the compressed input. */ + if (unlikely(cur_in + seg_len > compressed_len)) { + struct btrfs_inode *inode = cb->bbio.inode; + + btrfs_err(fs_info, + "lzo segment overflows compressed input, root %llu inode %llu offset %llu cur_in %u len %u compressed len %u", + btrfs_root_id(inode->root), btrfs_ino(inode), + cb->start, cur_in, seg_len, compressed_len); + return -EUCLEAN; + } + + /* Copy the compressed segment payload into workspace */ + copy_compressed_segment(cb, &fi, &cur_folio_index, workspace->cbuf, + seg_len, &cur_in); + + /* Decompress the data */ + ret = lzo1x_decompress_safe(workspace->cbuf, seg_len, + workspace->buf, &out_len); + if (unlikely(ret != LZO_E_OK)) { + struct btrfs_inode *inode = cb->bbio.inode; + + btrfs_err(fs_info, + "lzo decompression failed, error %d root %llu inode %llu offset %llu", + ret, btrfs_root_id(inode->root), btrfs_ino(inode), + cb->start); + return -EIO; + } + + /* Copy the data into inode pages */ + ret = btrfs_decompress_buf2page(workspace->buf, out_len, cb, cur_out); + cur_out += out_len; + + /* All data read, exit */ + if (ret == 0) + return 0; + ret = 0; + + /* Check if the sector has enough space for a segment header */ + sector_bytes_left = sectorsize - (cur_in % sectorsize); + if (sector_bytes_left >= LZO_LEN) + continue; + + /* Skip the padding zeros */ + cur_in += sector_bytes_left; + } + + return 0; +} + +int lzo_decompress(struct list_head *ws, const u8 *data_in, + struct folio *dest_folio, unsigned long dest_pgoff, size_t srclen, + size_t destlen) +{ + struct workspace *workspace = list_entry(ws, struct workspace, list); + struct btrfs_fs_info *fs_info = folio_to_fs_info(dest_folio); + const u32 sectorsize = fs_info->sectorsize; + size_t in_len; + size_t out_len; + size_t max_segment_len = workspace_buf_length(fs_info); + int ret; + + if (unlikely(srclen <= LZO_LEN * 2 || + srclen > max_segment_len + LZO_LEN * 2)) { + btrfs_err(fs_info, "invalid lzo header length, has %zu expect (%u, %zu)", + srclen, LZO_LEN * 2, max_segment_len + LZO_LEN * 2); + return -EUCLEAN; + } + + in_len = get_unaligned_le32(data_in); + if (unlikely(in_len != srclen)) { + btrfs_err(fs_info, "invalid lzo header length, has %zu expect %zu", + in_len, srclen); + return -EUCLEAN; + } + data_in += LZO_LEN; + + in_len = get_unaligned_le32(data_in); + if (unlikely(in_len != srclen - LZO_LEN * 2)) { + btrfs_err(fs_info, "invalid lzo segment length, has %zu expect %zu", + in_len, srclen - LZO_LEN * 2); + return -EUCLEAN; + } + data_in += LZO_LEN; + + out_len = sectorsize; + ret = lzo1x_decompress_safe(data_in, in_len, workspace->buf, &out_len); + if (unlikely(ret != LZO_E_OK)) { + struct btrfs_inode *inode = folio_to_inode(dest_folio); + + btrfs_err(fs_info, + "lzo decompression failed, error %d root %llu inode %llu offset %llu", + ret, btrfs_root_id(inode->root), btrfs_ino(inode), + folio_pos(dest_folio)); + return -EIO; + } + + ASSERT(out_len <= sectorsize); + memcpy_to_folio(dest_folio, dest_pgoff, workspace->buf, out_len); + /* Early end, considered as an error. */ + if (unlikely(out_len < destlen)) { + folio_zero_range(dest_folio, dest_pgoff + out_len, destlen - out_len); + return -EIO; + } + + return 0; +} + +const struct btrfs_compress_levels btrfs_lzo_compress = { + .max_level = 1, + .default_level = 1, +}; diff --git a/fs/btrfs/messages.c b/fs/btrfs/messages.c new file mode 100644 index 000000000..198d1747c --- /dev/null +++ b/fs/btrfs/messages.c @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "fs.h" +#include "messages.h" +#include "discard.h" +#include "super.h" + +#ifdef CONFIG_PRINTK + +#define STATE_STRING_PREFACE " state " +#define STATE_STRING_BUF_LEN (sizeof(STATE_STRING_PREFACE) + BTRFS_FS_STATE_COUNT + 1) + +/* + * Characters to print to indicate error conditions or uncommon filesystem state. + * RO is not an error. + */ +static const char fs_state_chars[] = { + [BTRFS_FS_STATE_REMOUNTING] = 'M', + [BTRFS_FS_STATE_RO] = 0, + [BTRFS_FS_STATE_TRANS_ABORTED] = 'A', + [BTRFS_FS_STATE_LOG_REPLAY_ABORTED] = 'O', + [BTRFS_FS_STATE_DEV_REPLACING] = 'R', + [BTRFS_FS_STATE_DUMMY_FS_INFO] = 0, + [BTRFS_FS_STATE_NO_DATA_CSUMS] = 'C', + [BTRFS_FS_STATE_SKIP_META_CSUMS] = 'S', + [BTRFS_FS_STATE_LOG_CLEANUP_ERROR] = 'L', + [BTRFS_FS_STATE_EMERGENCY_SHUTDOWN] = 'E', +}; + +static void btrfs_state_to_string(const struct btrfs_fs_info *info, char *buf) +{ + unsigned int bit; + bool states_printed = false; + unsigned long fs_state = READ_ONCE(info->fs_state); + char *curr = buf; + + memcpy(curr, STATE_STRING_PREFACE, sizeof(STATE_STRING_PREFACE)); + curr += sizeof(STATE_STRING_PREFACE) - 1; + + if (unlikely(BTRFS_FS_ERROR(info))) { + *curr++ = 'E'; + states_printed = true; + } + + for_each_set_bit(bit, &fs_state, sizeof(fs_state)) { + WARN_ON_ONCE(bit >= BTRFS_FS_STATE_COUNT); + if ((bit < BTRFS_FS_STATE_COUNT) && fs_state_chars[bit]) { + *curr++ = fs_state_chars[bit]; + states_printed = true; + } + } + + /* If no states were printed, reset the buffer */ + if (!states_printed) + curr = buf; + + *curr++ = 0; +} +#endif + +/* + * Generally the error codes correspond to their respective errors, but there + * are a few special cases. + * + * EUCLEAN: Any sort of corruption that we encounter. The tree-checker for + * instance will return EUCLEAN if any of the blocks are corrupted in + * a way that is problematic. We want to reserve EUCLEAN for these + * sort of corruptions. + * + * EROFS: If we check BTRFS_FS_STATE_ERROR and fail out with a return error, we + * need to use EROFS for this case. We will have no idea of the + * original failure, that will have been reported at the time we tripped + * over the error. Each subsequent error that doesn't have any context + * of the original error should use EROFS when handling BTRFS_FS_STATE_ERROR. + */ +const char * __attribute_const__ btrfs_decode_error(int error) +{ + char *errstr = "unknown"; + + switch (error) { + case -ENOENT: /* -2 */ + errstr = "No such entry"; + break; + case -EIO: /* -5 */ + errstr = "IO failure"; + break; + case -ENOMEM: /* -12*/ + errstr = "Out of memory"; + break; + case -EEXIST: /* -17 */ + errstr = "Object already exists"; + break; + case -ENOSPC: /* -28 */ + errstr = "No space left"; + break; + case -EROFS: /* -30 */ + errstr = "Readonly filesystem"; + break; + case -EOPNOTSUPP: /* -95 */ + errstr = "Operation not supported"; + break; + case -EUCLEAN: /* -117 */ + errstr = "Filesystem corrupted"; + break; + case -EDQUOT: /* -122 */ + errstr = "Quota exceeded"; + break; + } + + return errstr; +} + +/* + * Decodes expected errors from the caller and invokes the appropriate error + * response. + */ +__cold +void __btrfs_handle_fs_error(struct btrfs_fs_info *fs_info, const char *function, + unsigned int line, int error, const char *fmt, ...) +{ + struct super_block *sb = fs_info->sb; +#ifdef CONFIG_PRINTK + char statestr[STATE_STRING_BUF_LEN]; + const char *errstr; +#endif + +#ifdef CONFIG_PRINTK_INDEX + printk_index_subsys_emit( + "BTRFS: error (device %s%s) in %s:%d: errno=%d %s", KERN_CRIT, fmt); +#endif + + /* + * Special case: if the error is EROFS, and we're already under + * SB_RDONLY, then it is safe here. + */ + if (error == -EROFS && sb_rdonly(sb)) + return; + +#ifdef CONFIG_PRINTK + errstr = btrfs_decode_error(error); + btrfs_state_to_string(fs_info, statestr); + if (fmt) { + struct va_format vaf; + va_list args; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + + pr_crit("BTRFS: error (device %s%s) in %s:%d: errno=%d %s (%pV)\n", + sb->s_id, statestr, function, line, error, errstr, &vaf); + va_end(args); + } else { + pr_crit("BTRFS: error (device %s%s) in %s:%d: errno=%d %s\n", + sb->s_id, statestr, function, line, error, errstr); + } +#endif + + /* + * Today we only save the error info to memory. Long term we'll also + * send it down to the disk. + */ + WRITE_ONCE(fs_info->fs_error, error); + + /* Don't go through full error handling during mount. */ + if (!(sb->s_flags & SB_BORN)) + return; + + if (sb_rdonly(sb)) + return; + + btrfs_discard_stop(fs_info); + + /* Handle error by forcing the filesystem readonly. */ + btrfs_set_sb_rdonly(sb); + btrfs_info(fs_info, "forced readonly"); + /* + * Note that a running device replace operation is not canceled here + * although there is no way to update the progress. It would add the + * risk of a deadlock, therefore the canceling is omitted. The only + * penalty is that some I/O remains active until the procedure + * completes. The next time when the filesystem is mounted writable + * again, the device replace operation continues. + */ +} + +#ifdef CONFIG_PRINTK +static const char * const logtypes[] = { + "emergency", + "alert", + "critical", + "error", + "warning", + "notice", + "info", + "debug", +}; + +/* + * Use one ratelimit state per log level so that a flood of less important + * messages doesn't cause more important ones to be dropped. + */ +static struct ratelimit_state printk_limits[] = { + RATELIMIT_STATE_INIT(printk_limits[0], DEFAULT_RATELIMIT_INTERVAL, 100), + RATELIMIT_STATE_INIT(printk_limits[1], DEFAULT_RATELIMIT_INTERVAL, 100), + RATELIMIT_STATE_INIT(printk_limits[2], DEFAULT_RATELIMIT_INTERVAL, 100), + RATELIMIT_STATE_INIT(printk_limits[3], DEFAULT_RATELIMIT_INTERVAL, 100), + RATELIMIT_STATE_INIT(printk_limits[4], DEFAULT_RATELIMIT_INTERVAL, 100), + RATELIMIT_STATE_INIT(printk_limits[5], DEFAULT_RATELIMIT_INTERVAL, 100), + RATELIMIT_STATE_INIT(printk_limits[6], DEFAULT_RATELIMIT_INTERVAL, 100), + RATELIMIT_STATE_INIT(printk_limits[7], DEFAULT_RATELIMIT_INTERVAL, 100), +}; + +__printf(3, 4) __cold +void _btrfs_printk(const struct btrfs_fs_info *fs_info, unsigned int level, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + const char *type = logtypes[level]; + struct ratelimit_state *ratelimit = &printk_limits[level]; + +#ifdef CONFIG_PRINTK_INDEX + printk_index_subsys_emit("%sBTRFS %s (device %s): ", NULL, fmt); +#endif + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + + /* Do not ratelimit if CONFIG_BTRFS_DEBUG is enabled. */ + if (IS_ENABLED(CONFIG_BTRFS_DEBUG) || __ratelimit(ratelimit)) { + if (fs_info) { + char statestr[STATE_STRING_BUF_LEN]; + + btrfs_state_to_string(fs_info, statestr); + _printk(KERN_SOH "%dBTRFS %s (device %s%s): %pV\n", level, type, + fs_info->sb->s_id, statestr, &vaf); + } else { + _printk(KERN_SOH "%dBTRFS %s: %pV\n", level, type, &vaf); + } + } + + va_end(args); +} +#endif + +#if BITS_PER_LONG == 32 +void __cold btrfs_warn_32bit_limit(struct btrfs_fs_info *fs_info) +{ + if (!test_and_set_bit(BTRFS_FS_32BIT_WARN, &fs_info->flags)) { + btrfs_warn(fs_info, "reaching 32bit limit for logical addresses"); + btrfs_warn(fs_info, +"due to page cache limit on 32bit systems, btrfs can't access metadata at or beyond %lluT", + BTRFS_32BIT_MAX_FILE_SIZE >> 40); + btrfs_warn(fs_info, + "please consider upgrading to 64bit kernel/hardware"); + } +} + +void __cold btrfs_err_32bit_limit(struct btrfs_fs_info *fs_info) +{ + if (!test_and_set_bit(BTRFS_FS_32BIT_ERROR, &fs_info->flags)) { + btrfs_err(fs_info, "reached 32bit limit for logical addresses"); + btrfs_err(fs_info, +"due to page cache limit on 32bit systems, metadata beyond %lluT can't be accessed", + BTRFS_32BIT_MAX_FILE_SIZE >> 40); + btrfs_err(fs_info, + "please consider upgrading to 64bit kernel/hardware"); + } +} +#endif + +/* + * Decode unexpected, fatal errors from the caller, issue an alert, and either + * panic or BUGs, depending on mount options. + */ +__cold +void __btrfs_panic(const struct btrfs_fs_info *fs_info, const char *function, + unsigned int line, int error, const char *fmt, ...) +{ + char *s_id = ""; + struct va_format vaf = { .fmt = fmt }; + va_list args; + + if (fs_info) + s_id = fs_info->sb->s_id; + + va_start(args, fmt); + vaf.va = &args; + + if (fs_info && (btrfs_test_opt(fs_info, PANIC_ON_FATAL_ERROR))) + panic(KERN_CRIT "BTRFS panic (device %s) in %s:%d: %pV (errno=%d %pe)\n", + s_id, function, line, &vaf, error, ERR_PTR(error)); + + btrfs_crit(fs_info, "panic in %s:%d: %pV (errno=%d %pe)", + function, line, &vaf, error, ERR_PTR(error)); + va_end(args); + /* Caller calls BUG() */ +} diff --git a/fs/btrfs/messages.h b/fs/btrfs/messages.h new file mode 100644 index 000000000..556d4e79c --- /dev/null +++ b/fs/btrfs/messages.h @@ -0,0 +1,207 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_MESSAGES_H +#define BTRFS_MESSAGES_H + +#include +#include +#include + +struct btrfs_fs_info; + +/* + * We want to be able to override this in btrfs-progs. + */ +#ifdef __KERNEL__ + +static inline __printf(2, 3) __cold +void btrfs_no_printk(const struct btrfs_fs_info *fs_info, const char *fmt, ...) +{ +} + +#endif + +#ifdef CONFIG_PRINTK + +__printf(3, 4) __cold +void _btrfs_printk(const struct btrfs_fs_info *fs_info, unsigned int level, const char *fmt, ...); + +#else + +#define btrfs_printk_in_rcu(fs_info, level, fmt, args...) \ + btrfs_no_printk(fs_info, fmt, ##args) + +#define btrfs_printk_rl_in_rcu(fs_info, level, fmt, args...) \ + btrfs_no_printk(fs_info, fmt, ##args) + +#endif + +/* + * Print a message with filesystem info, enclosed in RCU protection. + */ +#define btrfs_crit(fs_info, fmt, args...) \ + btrfs_printk_in_rcu(fs_info, LOGLEVEL_CRIT, fmt, ##args) +#define btrfs_err(fs_info, fmt, args...) \ + btrfs_printk_in_rcu(fs_info, LOGLEVEL_ERR, fmt, ##args) +#define btrfs_warn(fs_info, fmt, args...) \ + btrfs_printk_in_rcu(fs_info, LOGLEVEL_WARNING, fmt, ##args) +#define btrfs_info(fs_info, fmt, args...) \ + btrfs_printk_in_rcu(fs_info, LOGLEVEL_INFO, fmt, ##args) + +/* + * Wrappers that use a ratelimited printk + */ +#define btrfs_crit_rl(fs_info, fmt, args...) \ + btrfs_printk_rl_in_rcu(fs_info, LOGLEVEL_CRIT, fmt, ##args) +#define btrfs_err_rl(fs_info, fmt, args...) \ + btrfs_printk_rl_in_rcu(fs_info, LOGLEVEL_ERR, fmt, ##args) +#define btrfs_warn_rl(fs_info, fmt, args...) \ + btrfs_printk_rl_in_rcu(fs_info, LOGLEVEL_WARNING, fmt, ##args) +#define btrfs_info_rl(fs_info, fmt, args...) \ + btrfs_printk_rl_in_rcu(fs_info, LOGLEVEL_INFO, fmt, ##args) + +#if defined(CONFIG_DYNAMIC_DEBUG) +#define btrfs_debug(fs_info, fmt, args...) \ + _dynamic_func_call_no_desc(fmt, btrfs_printk_in_rcu, \ + fs_info, LOGLEVEL_DEBUG, fmt, ##args) +#define btrfs_debug_rl(fs_info, fmt, args...) \ + _dynamic_func_call_no_desc(fmt, btrfs_printk_rl_in_rcu, \ + fs_info, LOGLEVEL_DEBUG, fmt, ##args) +#elif defined(DEBUG) +#define btrfs_debug(fs_info, fmt, args...) \ + btrfs_printk_in_rcu(fs_info, LOGLEVEL_DEBUG, fmt, ##args) +#define btrfs_debug_rl(fs_info, fmt, args...) \ + btrfs_printk_rl_in_rcu(fs_info, LOGLEVEl_DEBUG, fmt, ##args) +#else +/* When printk() is no_printk(), expand to no-op. */ +#define btrfs_debug(fs_info, fmt, args...) do { (void)(fs_info); } while(0) +#define btrfs_debug_rl(fs_info, fmt, args...) do { (void)(fs_info); } while(0) +#endif + +#ifdef CONFIG_PRINTK + +#define btrfs_printk_in_rcu(fs_info, level, fmt, args...) \ +do { \ + rcu_read_lock(); \ + _btrfs_printk(fs_info, level, fmt, ##args); \ + rcu_read_unlock(); \ +} while (0) + +#define btrfs_printk_rl_in_rcu(fs_info, level, fmt, args...) \ +do { \ + static DEFINE_RATELIMIT_STATE(_rs, \ + DEFAULT_RATELIMIT_INTERVAL, \ + DEFAULT_RATELIMIT_BURST); \ + \ + rcu_read_lock(); \ + if (__ratelimit(&_rs)) \ + _btrfs_printk(fs_info, level, fmt, ##args); \ + rcu_read_unlock(); \ +} while (0) + +#endif + +#ifdef CONFIG_BTRFS_ASSERT + +__printf(1, 2) +static inline void verify_assert_printk_format(const char *fmt, ...) { + /* Stub to verify the assertion format string. */ +} + +/* Take the first token if any. */ +#define __FIRST_ARG(_, ...) _ +/* + * Skip the first token and return the rest, if it's empty the comma is dropped. + * As ##__VA_ARGS__ cannot be at the beginning of the macro the __VA_OPT__ is needed + * and supported since GCC 8 and Clang 12. + */ +#define __REST_ARGS(_, ... ) __VA_OPT__(,) __VA_ARGS__ + +/* + * Assertion with optional printk() format. + * + * Accepted syntax: + * ASSERT(condition); + * ASSERT(condition, "string"); + * ASSERT(condition, "variable=%d", variable); + * + * How it works: + * - if there's no format string, ""[0] evaluates at compile time to 0 and the + * true branch is executed + * - any non-empty format string with the "" prefix evaluates to != 0 at + * compile time and the false branch is executed + * - stringified condition is printed as %s so we don't accidentally mix format + * strings (the % operator) + * - there can be only one printk() call, so the format strings and arguments are + * spliced together: + * DEFAULT_FMT [USER_FMT], DEFAULT_ARGS [, USER_ARGS] + * - comma between DEFAULT_ARGS and USER_ARGS is handled by preprocessor + * (requires __VA_OPT__ support) + * - otherwise we could use __VA_OPT(,) __VA_ARGS__ for the 2nd+ argument of args, + */ +#define ASSERT(cond, args...) \ +do { \ + verify_assert_printk_format("check the format string" args); \ + if (!likely(cond)) { \ + if (("" __FIRST_ARG(args) [0]) == 0) { \ + pr_err("assertion failed: %s, in %s:%d\n", \ + #cond, __FILE__, __LINE__); \ + } else { \ + pr_err("assertion failed: %s, in %s:%d (" __FIRST_ARG(args) ")\n", \ + #cond, __FILE__, __LINE__ __REST_ARGS(args)); \ + } \ + BUG(); \ + } \ +} while(0) + +#else +/* Compile check the @cond expression but don't generate any code. */ +#define ASSERT(cond, args...) BUILD_BUG_ON_INVALID(cond) +#endif + +#ifdef CONFIG_BTRFS_DEBUG +/* Verbose warning only under debug build. */ +#define DEBUG_WARN(args...) WARN(1, KERN_ERR args) +#else +#define DEBUG_WARN(...) do {} while(0) +#endif + +__printf(5, 6) +__cold +void __btrfs_handle_fs_error(struct btrfs_fs_info *fs_info, const char *function, + unsigned int line, int error, const char *fmt, ...); + +const char * __attribute_const__ btrfs_decode_error(int error); + +#define btrfs_handle_fs_error(fs_info, error, fmt, args...) \ + __btrfs_handle_fs_error((fs_info), __func__, __LINE__, \ + (error), fmt, ##args) + +__printf(5, 6) +__cold +void __btrfs_panic(const struct btrfs_fs_info *fs_info, const char *function, + unsigned int line, int error, const char *fmt, ...); +/* + * If BTRFS_MOUNT_PANIC_ON_FATAL_ERROR is in mount_opt, __btrfs_panic + * will panic(). Otherwise we BUG() here. + */ +#define btrfs_panic(fs_info, error, fmt, args...) \ +do { \ + __btrfs_panic(fs_info, __func__, __LINE__, error, fmt, ##args); \ + BUG(); \ +} while (0) + +#if BITS_PER_LONG == 32 +#define BTRFS_32BIT_MAX_FILE_SIZE (((u64)ULONG_MAX + 1) << PAGE_SHIFT) +/* + * The warning threshold is 5/8th of the MAX_LFS_FILESIZE that limits the logical + * addresses of extents. + * + * For 4K page size it's about 10T, for 64K it's 160T. + */ +#define BTRFS_32BIT_EARLY_WARN_THRESHOLD (BTRFS_32BIT_MAX_FILE_SIZE * 5 / 8) +void btrfs_warn_32bit_limit(struct btrfs_fs_info *fs_info); +void btrfs_err_32bit_limit(struct btrfs_fs_info *fs_info); +#endif + +#endif diff --git a/fs/btrfs/misc.h b/fs/btrfs/misc.h new file mode 100644 index 000000000..802060943 --- /dev/null +++ b/fs/btrfs/misc.h @@ -0,0 +1,227 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_MISC_H +#define BTRFS_MISC_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Convenience macros to define a pointer with the __free(kfree) and + * __free(kvfree) cleanup attributes and initialized to NULL. + */ +#define AUTO_KFREE(name) *name __free(kfree) = NULL +#define AUTO_KVFREE(name) *name __free(kvfree) = NULL + +/* + * Enumerate bits using enum autoincrement. Define the @name as the n-th bit. + */ +#define ENUM_BIT(name) \ + __ ## name ## _BIT, \ + name = (1U << __ ## name ## _BIT), \ + __ ## name ## _SEQ = __ ## name ## _BIT + +static inline phys_addr_t bio_iter_phys(const struct bio *bio, + const struct bvec_iter *iter) +{ + struct bio_vec bv = bio_iter_iovec(bio, *iter); + + return bvec_phys(&bv); +} + +/* + * Iterate bio using btrfs block size. + * + * This will handle large folio and highmem. + * + * @paddr: Physical memory address of each iteration + * @bio: The bio to iterate + * @iter: The bvec_iter (pointer) to use. + * @blocksize: The blocksize to iterate. + * + * This requires all folios in the bio to cover at least one block. + */ +#define btrfs_bio_for_each_block(paddr, bio, iter, blocksize) \ + for (; (iter)->bi_size && \ + (paddr = bio_iter_phys((bio), (iter)), 1); \ + bio_advance_iter_single((bio), (iter), (blocksize))) + +/* Can only be called on a non-cloned bio. */ +static inline u32 bio_get_size(struct bio *bio) +{ + struct bio_vec *bvec; + u32 ret = 0; + int i; + + bio_for_each_bvec_all(bvec, bio, i) + ret += bvec->bv_len; + return ret; +} + +/* Initialize a bvec_iter to the size of the specified bio. */ +static inline struct bvec_iter init_bvec_iter_for_bio(struct bio *bio) +{ + const u32 bio_size = bio_get_size(bio); + + return (struct bvec_iter) { + .bi_sector = 0, + .bi_size = bio_size, + .bi_idx = 0, + .bi_offset = 0, + }; +} + +#define btrfs_bio_for_each_block_all(paddr, bio, blocksize) \ + for (struct bvec_iter iter = init_bvec_iter_for_bio(bio); \ + (iter).bi_size && \ + (paddr = bio_iter_phys((bio), &(iter)), 1); \ + bio_advance_iter_single((bio), &(iter), (blocksize))) + +static inline void cond_wake_up(struct wait_queue_head *wq) +{ + /* + * This implies a full smp_mb barrier, see comments for + * waitqueue_active why. + */ + if (wq_has_sleeper(wq)) + wake_up(wq); +} + +static inline void cond_wake_up_nomb(struct wait_queue_head *wq) +{ + /* + * Special case for conditional wakeup where the barrier required for + * waitqueue_active is implied by some of the preceding code. Eg. one + * of such atomic operations (atomic_dec_and_return, ...), or a + * unlock/lock sequence, etc. + */ + if (waitqueue_active(wq)) + wake_up(wq); +} + +static inline u64 mult_perc(u64 num, u32 percent) +{ + return div_u64(num * percent, 100); +} +/* Copy of is_power_of_two that is 64bit safe */ +static inline bool is_power_of_two_u64(u64 n) +{ + return n != 0 && (n & (n - 1)) == 0; +} + +static inline bool has_single_bit_set(u64 n) +{ + return is_power_of_two_u64(n); +} + +/* + * Simple bytenr based rb_tree relate structures + * + * Any structure wants to use bytenr as single search index should have their + * structure start with these members. + */ +struct rb_simple_node { + struct rb_node rb_node; + u64 bytenr; +}; + +static inline struct rb_node *rb_simple_search(const struct rb_root *root, u64 bytenr) +{ + struct rb_node *node = root->rb_node; + struct rb_simple_node *entry; + + while (node) { + entry = rb_entry(node, struct rb_simple_node, rb_node); + + if (bytenr < entry->bytenr) + node = node->rb_left; + else if (bytenr > entry->bytenr) + node = node->rb_right; + else + return node; + } + return NULL; +} + +/* + * Search @root from an entry that starts or comes after @bytenr. + * + * @root: the root to search. + * @bytenr: bytenr to search from. + * + * Return the rb_node that start at or after @bytenr. If there is no entry at + * or after @bytner return NULL. + */ +static inline struct rb_node *rb_simple_search_first(const struct rb_root *root, + u64 bytenr) +{ + struct rb_node *node = root->rb_node, *ret = NULL; + struct rb_simple_node *entry, *ret_entry = NULL; + + while (node) { + entry = rb_entry(node, struct rb_simple_node, rb_node); + + if (bytenr < entry->bytenr) { + if (!ret || entry->bytenr < ret_entry->bytenr) { + ret = node; + ret_entry = entry; + } + + node = node->rb_left; + } else if (bytenr > entry->bytenr) { + node = node->rb_right; + } else { + return node; + } + } + + return ret; +} + +static int rb_simple_node_bytenr_cmp(struct rb_node *new, const struct rb_node *existing) +{ + struct rb_simple_node *new_entry = rb_entry(new, struct rb_simple_node, rb_node); + struct rb_simple_node *existing_entry = rb_entry(existing, struct rb_simple_node, rb_node); + + if (new_entry->bytenr < existing_entry->bytenr) + return -1; + else if (new_entry->bytenr > existing_entry->bytenr) + return 1; + + return 0; +} + +static inline struct rb_node *rb_simple_insert(struct rb_root *root, + struct rb_simple_node *simple_node) +{ + return rb_find_add(&simple_node->rb_node, root, rb_simple_node_bytenr_cmp); +} + +static inline bool bitmap_test_range_all_set(const unsigned long *addr, + unsigned long start, + unsigned long nbits) +{ + unsigned long found_zero; + + found_zero = find_next_zero_bit(addr, start + nbits, start); + return (found_zero == start + nbits); +} + +static inline bool bitmap_test_range_all_zero(const unsigned long *addr, + unsigned long start, + unsigned long nbits) +{ + unsigned long found_set; + + found_set = find_next_bit(addr, start + nbits, start); + return (found_set == start + nbits); +} + +#endif diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c new file mode 100644 index 000000000..b32d4eabe --- /dev/null +++ b/fs/btrfs/ordered-data.c @@ -0,0 +1,1371 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include "messages.h" +#include "misc.h" +#include "ctree.h" +#include "transaction.h" +#include "btrfs_inode.h" +#include "extent_io.h" +#include "disk-io.h" +#include "compression.h" +#include "delalloc-space.h" +#include "qgroup.h" +#include "subpage.h" +#include "file.h" +#include "block-group.h" + +static struct kmem_cache *btrfs_ordered_extent_cache; + +static u64 entry_end(struct btrfs_ordered_extent *entry) +{ + if (entry->file_offset + entry->num_bytes < entry->file_offset) + return (u64)-1; + return entry->file_offset + entry->num_bytes; +} + +/* returns NULL if the insertion worked, or it returns the node it did find + * in the tree + */ +static struct rb_node *tree_insert(struct rb_root *root, u64 file_offset, + struct rb_node *node) +{ + struct rb_node **p = &root->rb_node; + struct rb_node *parent = NULL; + struct btrfs_ordered_extent *entry; + + while (*p) { + parent = *p; + entry = rb_entry(parent, struct btrfs_ordered_extent, rb_node); + + if (file_offset < entry->file_offset) + p = &(*p)->rb_left; + else if (file_offset >= entry_end(entry)) + p = &(*p)->rb_right; + else + return parent; + } + + rb_link_node(node, parent, p); + rb_insert_color(node, root); + return NULL; +} + +/* + * look for a given offset in the tree, and if it can't be found return the + * first lesser offset + */ +static struct rb_node *__tree_search(struct rb_root *root, u64 file_offset, + struct rb_node **prev_ret) +{ + struct rb_node *n = root->rb_node; + struct rb_node *prev = NULL; + struct rb_node *test; + struct btrfs_ordered_extent *entry; + struct btrfs_ordered_extent *prev_entry = NULL; + + while (n) { + entry = rb_entry(n, struct btrfs_ordered_extent, rb_node); + prev = n; + prev_entry = entry; + + if (file_offset < entry->file_offset) + n = n->rb_left; + else if (file_offset >= entry_end(entry)) + n = n->rb_right; + else + return n; + } + if (!prev_ret) + return NULL; + + while (prev && file_offset >= entry_end(prev_entry)) { + test = rb_next(prev); + if (!test) + break; + prev_entry = rb_entry(test, struct btrfs_ordered_extent, + rb_node); + if (file_offset < entry_end(prev_entry)) + break; + + prev = test; + } + if (prev) + prev_entry = rb_entry(prev, struct btrfs_ordered_extent, + rb_node); + while (prev && file_offset < entry_end(prev_entry)) { + test = rb_prev(prev); + if (!test) + break; + prev_entry = rb_entry(test, struct btrfs_ordered_extent, + rb_node); + prev = test; + } + *prev_ret = prev; + return NULL; +} + +static int btrfs_range_overlaps(struct btrfs_ordered_extent *entry, u64 file_offset, + u64 len) +{ + if (file_offset + len <= entry->file_offset || + entry->file_offset + entry->num_bytes <= file_offset) + return 0; + return 1; +} + +/* + * look find the first ordered struct that has this offset, otherwise + * the first one less than this offset + */ +static inline struct rb_node *ordered_tree_search(struct btrfs_inode *inode, + u64 file_offset) +{ + struct rb_node *prev = NULL; + struct rb_node *ret; + struct btrfs_ordered_extent *entry; + + if (inode->ordered_tree_last) { + entry = rb_entry(inode->ordered_tree_last, struct btrfs_ordered_extent, + rb_node); + if (in_range(file_offset, entry->file_offset, entry->num_bytes)) + return inode->ordered_tree_last; + } + ret = __tree_search(&inode->ordered_tree, file_offset, &prev); + if (!ret) + ret = prev; + if (ret) + inode->ordered_tree_last = ret; + return ret; +} + +static struct btrfs_ordered_extent *alloc_ordered_extent( + struct btrfs_inode *inode, u64 file_offset, u64 num_bytes, + u64 ram_bytes, u64 disk_bytenr, u64 disk_num_bytes, + u64 offset, unsigned long flags, int compress_type) +{ + struct btrfs_ordered_extent *entry; + int ret; + u64 qgroup_rsv = 0; + const bool is_nocow = (flags & + ((1U << BTRFS_ORDERED_NOCOW) | (1U << BTRFS_ORDERED_PREALLOC))); + + /* Only one type flag can be set. */ + ASSERT(has_single_bit_set(flags & BTRFS_ORDERED_EXCLUSIVE_FLAGS), + "flags=0x%lx", flags); + + /* DIRECT cannot be set with COMPRESSED nor ENCODED. */ + if (test_bit(BTRFS_ORDERED_DIRECT, &flags)) { + ASSERT(!test_bit(BTRFS_ORDERED_COMPRESSED, &flags)); + ASSERT(!test_bit(BTRFS_ORDERED_ENCODED, &flags)); + } + + /* ENCODED must be set with COMPRESSED. */ + if (test_bit(BTRFS_ORDERED_ENCODED, &flags)) + ASSERT(test_bit(BTRFS_ORDERED_COMPRESSED, &flags)); + + /* + * For a NOCOW write we can free the qgroup reserve right now. For a COW + * one we transfer the reserved space from the inode's iotree into the + * ordered extent by calling btrfs_qgroup_release_data() and tracking + * the qgroup reserved amount in the ordered extent, so that later after + * completing the ordered extent, when running the data delayed ref it + * creates, we free the reserved data with btrfs_qgroup_free_refroot(). + */ + if (is_nocow) + ret = btrfs_qgroup_free_data(inode, NULL, file_offset, num_bytes, &qgroup_rsv); + else + ret = btrfs_qgroup_release_data(inode, file_offset, num_bytes, &qgroup_rsv); + + if (ret < 0) + return ERR_PTR(ret); + + entry = kmem_cache_zalloc(btrfs_ordered_extent_cache, GFP_NOFS); + if (!entry) { + entry = ERR_PTR(-ENOMEM); + goto out; + } + + entry->file_offset = file_offset; + entry->num_bytes = num_bytes; + entry->ram_bytes = ram_bytes; + entry->disk_bytenr = disk_bytenr; + entry->disk_num_bytes = disk_num_bytes; + entry->offset = offset; + entry->bytes_left = num_bytes; + if (WARN_ON_ONCE(!igrab(&inode->vfs_inode))) { + kmem_cache_free(btrfs_ordered_extent_cache, entry); + entry = ERR_PTR(-ESTALE); + goto out; + } + entry->inode = inode; + entry->compress_type = compress_type; + entry->truncated_len = (u64)-1; + entry->qgroup_rsv = qgroup_rsv; + entry->flags = flags; + refcount_set(&entry->refs, 1); + init_waitqueue_head(&entry->wait); + INIT_LIST_HEAD(&entry->csum_list); + INIT_LIST_HEAD(&entry->log_list); + INIT_LIST_HEAD(&entry->root_extent_list); + INIT_LIST_HEAD(&entry->work_list); + INIT_LIST_HEAD(&entry->bioc_list); + init_completion(&entry->completion); + + /* + * We don't need the count_max_extents here, we can assume that all of + * that work has been done at higher layers, so this is truly the + * smallest the extent is going to get. + */ + spin_lock(&inode->lock); + btrfs_mod_outstanding_extents(inode, 1); + spin_unlock(&inode->lock); + +out: + if (IS_ERR(entry) && !is_nocow) + btrfs_qgroup_free_refroot(inode->root->fs_info, + btrfs_root_id(inode->root), + qgroup_rsv, BTRFS_QGROUP_RSV_DATA); + + return entry; +} + +static void insert_ordered_extent(struct btrfs_ordered_extent *entry) +{ + struct btrfs_inode *inode = entry->inode; + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct rb_node *node; + + trace_btrfs_ordered_extent_add(inode, entry); + + percpu_counter_add_batch(&fs_info->ordered_bytes, entry->num_bytes, + fs_info->delalloc_batch); + + /* One ref for the tree. */ + refcount_inc(&entry->refs); + + spin_lock(&inode->ordered_tree_lock); + node = tree_insert(&inode->ordered_tree, entry->file_offset, + &entry->rb_node); + if (unlikely(node)) { + struct btrfs_ordered_extent *exist = + rb_entry(node, struct btrfs_ordered_extent, rb_node); + + btrfs_panic(fs_info, -EEXIST, +"overlapping ordered extents, existing oe file_offset %llu num_bytes %llu flags 0x%lx, new oe file_offset %llu num_bytes %llu flags 0x%lx", + exist->file_offset, exist->num_bytes, exist->flags, + entry->file_offset, entry->num_bytes, entry->flags); + } + spin_unlock(&inode->ordered_tree_lock); + + spin_lock(&root->ordered_extent_lock); + list_add_tail(&entry->root_extent_list, + &root->ordered_extents); + root->nr_ordered_extents++; + if (root->nr_ordered_extents == 1) { + spin_lock(&fs_info->ordered_root_lock); + BUG_ON(!list_empty(&root->ordered_root)); + list_add_tail(&root->ordered_root, &fs_info->ordered_roots); + spin_unlock(&fs_info->ordered_root_lock); + } + spin_unlock(&root->ordered_extent_lock); +} + +/* + * Add an ordered extent to the per-inode tree. + * + * @inode: Inode that this extent is for. + * @file_offset: Logical offset in file where the extent starts. + * @num_bytes: Logical length of extent in file. + * @ram_bytes: Full length of unencoded data. + * @disk_bytenr: Offset of extent on disk. + * @disk_num_bytes: Size of extent on disk. + * @offset: Offset into unencoded data where file data starts. + * @flags: Flags specifying type of extent (1U << BTRFS_ORDERED_*). + * @compress_type: Compression algorithm used for data. + * + * Most of these parameters correspond to &struct btrfs_file_extent_item. The + * tree is given a single reference on the ordered extent that was inserted, and + * the returned pointer is given a second reference. + * + * Return: the new ordered extent or error pointer. + */ +struct btrfs_ordered_extent *btrfs_alloc_ordered_extent( + struct btrfs_inode *inode, u64 file_offset, + const struct btrfs_file_extent *file_extent, unsigned long flags) +{ + struct btrfs_ordered_extent *entry; + + ASSERT((flags & ~BTRFS_ORDERED_TYPE_FLAGS) == 0, "flags=0x%lx", flags); + + /* + * For regular writes, we just use the members in @file_extent. + * + * For NOCOW, we don't really care about the numbers except @start and + * file_extent->num_bytes, as we won't insert a file extent item at all. + * + * For PREALLOC, we do not use ordered extent members, but + * btrfs_mark_extent_written() handles everything. + * + * So here we always pass 0 as offset for NOCOW/PREALLOC ordered extents, + * or btrfs_split_ordered_extent() cannot handle it correctly. + */ + if (flags & ((1U << BTRFS_ORDERED_NOCOW) | (1U << BTRFS_ORDERED_PREALLOC))) + entry = alloc_ordered_extent(inode, file_offset, + file_extent->num_bytes, + file_extent->num_bytes, + file_extent->disk_bytenr + file_extent->offset, + file_extent->num_bytes, 0, flags, + file_extent->compression); + else + entry = alloc_ordered_extent(inode, file_offset, + file_extent->num_bytes, + file_extent->ram_bytes, + file_extent->disk_bytenr, + file_extent->disk_num_bytes, + file_extent->offset, flags, + file_extent->compression); + if (!IS_ERR(entry)) + insert_ordered_extent(entry); + return entry; +} + +/* + * Add a struct btrfs_ordered_sum into the list of checksums to be inserted + * when an ordered extent is finished. If the list covers more than one + * ordered extent, it is split across multiples. + */ +void btrfs_add_ordered_sum(struct btrfs_ordered_extent *entry, + struct btrfs_ordered_sum *sum) +{ + struct btrfs_inode *inode = entry->inode; + + spin_lock(&inode->ordered_tree_lock); + list_add_tail(&sum->list, &entry->csum_list); + spin_unlock(&inode->ordered_tree_lock); +} + +void btrfs_mark_ordered_extent_error(struct btrfs_ordered_extent *ordered) +{ + if (!test_and_set_bit(BTRFS_ORDERED_IOERR, &ordered->flags)) + mapping_set_error(ordered->inode->vfs_inode.i_mapping, -EIO); +} + +void btrfs_mark_ordered_extent_truncated(struct btrfs_ordered_extent *ordered, + u64 truncate_len) +{ + struct btrfs_inode *inode = ordered->inode; + + ASSERT(truncate_len <= ordered->num_bytes); + spin_lock(&inode->ordered_tree_lock); + set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags); + ordered->truncated_len = min(ordered->truncated_len, truncate_len); + spin_unlock(&inode->ordered_tree_lock); +} + +static void finish_ordered_fn(struct btrfs_work *work) +{ + struct btrfs_ordered_extent *ordered_extent; + + ordered_extent = container_of(work, struct btrfs_ordered_extent, work); + btrfs_finish_ordered_io(ordered_extent); +} + +static bool can_finish_ordered_extent(struct btrfs_ordered_extent *ordered, + u64 file_offset, u64 len, bool uptodate) +{ + struct btrfs_inode *inode = ordered->inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + + lockdep_assert_held(&inode->ordered_tree_lock); + + /* Now we're fine to update the accounting. */ + if (WARN_ON_ONCE(len > ordered->bytes_left)) { + btrfs_crit(fs_info, +"bad ordered extent accounting, root=%llu ino=%llu OE offset=%llu OE len=%llu to_dec=%llu left=%llu", + btrfs_root_id(inode->root), btrfs_ino(inode), + ordered->file_offset, ordered->num_bytes, + len, ordered->bytes_left); + ordered->bytes_left = 0; + } else { + ordered->bytes_left -= len; + } + + if (!uptodate) + btrfs_mark_ordered_extent_error(ordered); + + if (ordered->bytes_left) + return false; + + /* + * All the IO of the ordered extent is finished, we need to queue + * the finish_func to be executed. + */ + set_bit(BTRFS_ORDERED_IO_DONE, &ordered->flags); + cond_wake_up(&ordered->wait); + refcount_inc(&ordered->refs); + trace_btrfs_ordered_extent_mark_finished(inode, ordered); + return true; +} + +static void btrfs_queue_ordered_fn(struct btrfs_ordered_extent *ordered) +{ + struct btrfs_inode *inode = ordered->inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct btrfs_workqueue *wq = btrfs_is_free_space_inode(inode) ? + fs_info->endio_freespace_worker : fs_info->endio_write_workers; + + btrfs_init_work(&ordered->work, finish_ordered_fn, NULL); + btrfs_queue_work(wq, &ordered->work); +} + +void btrfs_finish_ordered_extent(struct btrfs_ordered_extent *ordered, + u64 file_offset, u64 len, bool uptodate) +{ + struct btrfs_inode *inode = ordered->inode; + bool ret; + + trace_btrfs_finish_ordered_extent(inode, file_offset, len, uptodate); + + spin_lock(&inode->ordered_tree_lock); + ret = can_finish_ordered_extent(ordered, file_offset, len, + uptodate); + spin_unlock(&inode->ordered_tree_lock); + + /* + * If this is a COW write it means we created new extent maps for the + * range and they point to unwritten locations if we got an error either + * before submitting a bio or during IO. + * + * We have marked the ordered extent with BTRFS_ORDERED_IOERR, and we + * are queuing its completion below. During completion, at + * btrfs_finish_one_ordered(), we will drop the extent maps for the + * unwritten extents. + * + * However because completion runs in a work queue we can end up having + * a fast fsync running before that. In the case of direct IO, once we + * unlock the inode the fsync might start, and we queue the completion + * before unlocking the inode. In the case of buffered IO when writeback + * finishes (end_bbio_data_write()) we queue the completion, so if the + * writeback was triggered by a fast fsync, the fsync might start + * logging before ordered extent completion runs in the work queue. + * + * The fast fsync will log file extent items based on the extent maps it + * finds, so if by the time it collects extent maps the ordered extent + * completion didn't happen yet, it will log file extent items that + * point to unwritten extents, resulting in a corruption if a crash + * happens and the log tree is replayed. Note that a fast fsync does not + * wait for completion of ordered extents in order to reduce latency. + * + * Set a flag in the inode so that the next fast fsync will wait for + * ordered extents to complete before starting to log. + */ + if (!uptodate && !test_bit(BTRFS_ORDERED_NOCOW, &ordered->flags)) + set_bit(BTRFS_INODE_COW_WRITE_ERROR, &inode->runtime_flags); + + if (ret) + btrfs_queue_ordered_fn(ordered); +} + +/* + * Mark all ordered extents io inside the specified range finished. + * + * @folio: The involved folio for the operation. + * For uncompressed buffered IO, the folio status also needs to be + * updated to indicate whether the pending ordered io is finished. + * Can be NULL for direct IO and compressed write. + * For these cases, callers are ensured they won't execute the + * endio function twice. + * + * This function is called for endio, thus the range must have ordered + * extent(s) covering it. + */ +void btrfs_mark_ordered_io_finished(struct btrfs_inode *inode, + u64 file_offset, u64 num_bytes, bool uptodate) +{ + struct rb_node *node; + struct btrfs_ordered_extent *entry = NULL; + u64 cur = file_offset; + const u64 end = file_offset + num_bytes; + + trace_btrfs_writepage_end_io_hook(inode, file_offset, end - 1, uptodate); + + spin_lock(&inode->ordered_tree_lock); + while (cur < end) { + u64 entry_end; + u64 this_end; + u64 len; + + node = ordered_tree_search(inode, cur); + /* No ordered extents at all */ + if (!node) + break; + + entry = rb_entry(node, struct btrfs_ordered_extent, rb_node); + entry_end = entry->file_offset + entry->num_bytes; + /* + * |<-- OE --->| | + * cur + * Go to next OE. + */ + if (cur >= entry_end) { + node = rb_next(node); + /* No more ordered extents, exit */ + if (!node) + break; + entry = rb_entry(node, struct btrfs_ordered_extent, + rb_node); + + /* Go to next ordered extent and continue */ + cur = entry->file_offset; + continue; + } + /* + * | |<--- OE --->| + * cur + * Go to the start of OE. + */ + if (cur < entry->file_offset) { + cur = entry->file_offset; + continue; + } + + /* + * Now we are definitely inside one ordered extent. + * + * |<--- OE --->| + * | + * cur + */ + this_end = min(entry_end, end); + len = this_end - cur; + ASSERT(len < U32_MAX); + + if (can_finish_ordered_extent(entry, cur, len, uptodate)) { + spin_unlock(&inode->ordered_tree_lock); + btrfs_queue_ordered_fn(entry); + spin_lock(&inode->ordered_tree_lock); + } + cur += len; + } + spin_unlock(&inode->ordered_tree_lock); +} + +/* + * Finish IO for one ordered extent across a given range. The range can only + * contain one ordered extent. + * + * @cached: The cached ordered extent. If not NULL, we can skip the tree + * search and use the ordered extent directly. + * Will be also used to store the finished ordered extent. + * @file_offset: File offset for the finished IO + * @io_size: Length of the finish IO range + * + * Return true if the ordered extent is finished in the range, and update + * @cached. + * Return false otherwise. + * + * NOTE: The range can NOT cross multiple ordered extents. + * Thus caller should ensure the range doesn't cross ordered extents. + */ +bool btrfs_dec_test_ordered_pending(struct btrfs_inode *inode, + struct btrfs_ordered_extent **cached, + u64 file_offset, u64 io_size) +{ + struct rb_node *node; + struct btrfs_ordered_extent *entry = NULL; + bool finished = false; + + spin_lock(&inode->ordered_tree_lock); + if (cached && *cached) { + entry = *cached; + goto have_entry; + } + + node = ordered_tree_search(inode, file_offset); + if (!node) + goto out; + + entry = rb_entry(node, struct btrfs_ordered_extent, rb_node); +have_entry: + if (!in_range(file_offset, entry->file_offset, entry->num_bytes)) + goto out; + + if (io_size > entry->bytes_left) + btrfs_crit(inode->root->fs_info, + "bad ordered accounting left %llu size %llu", + entry->bytes_left, io_size); + + entry->bytes_left -= io_size; + + if (entry->bytes_left == 0) { + /* + * Ensure only one caller can set the flag and finished_ret + * accordingly + */ + finished = !test_and_set_bit(BTRFS_ORDERED_IO_DONE, &entry->flags); + /* test_and_set_bit implies a barrier */ + cond_wake_up_nomb(&entry->wait); + } +out: + if (finished && cached && entry) { + *cached = entry; + refcount_inc(&entry->refs); + trace_btrfs_ordered_extent_dec_test_pending(inode, entry); + } + spin_unlock(&inode->ordered_tree_lock); + return finished; +} + +/* + * used to drop a reference on an ordered extent. This will free + * the extent if the last reference is dropped + */ +void btrfs_put_ordered_extent(struct btrfs_ordered_extent *entry) +{ + trace_btrfs_ordered_extent_put(entry->inode, entry); + + if (refcount_dec_and_test(&entry->refs)) { + struct btrfs_ordered_sum *sum; + struct btrfs_ordered_sum *tmp; + + ASSERT(list_empty(&entry->root_extent_list)); + ASSERT(list_empty(&entry->log_list)); + ASSERT(RB_EMPTY_NODE(&entry->rb_node)); + btrfs_add_delayed_iput(entry->inode); + list_for_each_entry_safe(sum, tmp, &entry->csum_list, list) + kvfree(sum); + kmem_cache_free(btrfs_ordered_extent_cache, entry); + } +} + +/* + * remove an ordered extent from the tree. No references are dropped + * and waiters are woken up. + */ +void btrfs_remove_ordered_extent(struct btrfs_ordered_extent *entry) +{ + struct btrfs_inode *btrfs_inode = entry->inode; + struct btrfs_root *root = btrfs_inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct rb_node *node; + bool pending; + bool freespace_inode; + + /* + * If this is a free space inode the thread has not acquired the ordered + * extents lockdep map. + */ + freespace_inode = btrfs_is_free_space_inode(btrfs_inode); + + btrfs_lockdep_acquire(fs_info, btrfs_trans_pending_ordered); + /* This is paired with alloc_ordered_extent(). */ + spin_lock(&btrfs_inode->lock); + btrfs_mod_outstanding_extents(btrfs_inode, -1); + spin_unlock(&btrfs_inode->lock); + if (root != fs_info->tree_root) { + u64 release; + + if (test_bit(BTRFS_ORDERED_ENCODED, &entry->flags)) + release = entry->disk_num_bytes; + else + release = entry->num_bytes; + btrfs_delalloc_release_metadata(btrfs_inode, release, + test_bit(BTRFS_ORDERED_IOERR, + &entry->flags)); + } + + percpu_counter_add_batch(&fs_info->ordered_bytes, -entry->num_bytes, + fs_info->delalloc_batch); + + spin_lock(&btrfs_inode->ordered_tree_lock); + node = &entry->rb_node; + rb_erase(node, &btrfs_inode->ordered_tree); + RB_CLEAR_NODE(node); + if (btrfs_inode->ordered_tree_last == node) + btrfs_inode->ordered_tree_last = NULL; + set_bit(BTRFS_ORDERED_COMPLETE, &entry->flags); + pending = test_and_clear_bit(BTRFS_ORDERED_PENDING, &entry->flags); + spin_unlock(&btrfs_inode->ordered_tree_lock); + + /* + * The current running transaction is waiting on us, we need to let it + * know that we're complete and wake it up. + */ + if (pending) { + struct btrfs_transaction *trans; + + /* + * The checks for trans are just a formality, it should be set, + * but if it isn't we don't want to deref/assert under the spin + * lock, so be nice and check if trans is set, but ASSERT() so + * if it isn't set a developer will notice. + */ + spin_lock(&fs_info->trans_lock); + trans = fs_info->running_transaction; + if (trans) + refcount_inc(&trans->use_count); + spin_unlock(&fs_info->trans_lock); + + ASSERT(trans || BTRFS_FS_ERROR(fs_info)); + if (trans) { + if (atomic_dec_and_test(&trans->pending_ordered)) + wake_up(&trans->pending_wait); + btrfs_put_transaction(trans); + } + } + + btrfs_lockdep_release(fs_info, btrfs_trans_pending_ordered); + + spin_lock(&root->ordered_extent_lock); + list_del_init(&entry->root_extent_list); + root->nr_ordered_extents--; + + trace_btrfs_ordered_extent_remove(btrfs_inode, entry); + + if (!root->nr_ordered_extents) { + spin_lock(&fs_info->ordered_root_lock); + BUG_ON(list_empty(&root->ordered_root)); + list_del_init(&root->ordered_root); + spin_unlock(&fs_info->ordered_root_lock); + } + spin_unlock(&root->ordered_extent_lock); + wake_up(&entry->wait); + if (!freespace_inode) + btrfs_lockdep_release(fs_info, btrfs_ordered_extent); +} + +static void btrfs_run_ordered_extent_work(struct btrfs_work *work) +{ + struct btrfs_ordered_extent *ordered; + + ordered = container_of(work, struct btrfs_ordered_extent, flush_work); + btrfs_start_ordered_extent(ordered); + complete(&ordered->completion); +} + +/* + * Wait for all the ordered extents in a root. Use @bg as range or do whole + * range if it's NULL. + */ +u64 btrfs_wait_ordered_extents(struct btrfs_root *root, u64 nr, + const struct btrfs_block_group *bg) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + LIST_HEAD(splice); + LIST_HEAD(skipped); + LIST_HEAD(works); + struct btrfs_ordered_extent *ordered, *next; + u64 count = 0; + u64 range_start, range_len; + u64 range_end; + + if (bg) { + range_start = bg->start; + range_len = bg->length; + } else { + range_start = 0; + range_len = U64_MAX; + } + range_end = range_start + range_len; + + mutex_lock(&root->ordered_extent_mutex); + spin_lock(&root->ordered_extent_lock); + list_splice_init(&root->ordered_extents, &splice); + while (!list_empty(&splice) && nr) { + ordered = list_first_entry(&splice, struct btrfs_ordered_extent, + root_extent_list); + + if (range_end <= ordered->disk_bytenr || + ordered->disk_bytenr + ordered->disk_num_bytes <= range_start) { + list_move_tail(&ordered->root_extent_list, &skipped); + cond_resched_lock(&root->ordered_extent_lock); + continue; + } + + list_move_tail(&ordered->root_extent_list, + &root->ordered_extents); + refcount_inc(&ordered->refs); + spin_unlock(&root->ordered_extent_lock); + + btrfs_init_work(&ordered->flush_work, + btrfs_run_ordered_extent_work, NULL); + list_add_tail(&ordered->work_list, &works); + btrfs_queue_work(fs_info->flush_workers, &ordered->flush_work); + + cond_resched(); + if (nr != U64_MAX) + nr--; + count++; + spin_lock(&root->ordered_extent_lock); + } + list_splice_tail(&skipped, &root->ordered_extents); + list_splice_tail(&splice, &root->ordered_extents); + spin_unlock(&root->ordered_extent_lock); + + list_for_each_entry_safe(ordered, next, &works, work_list) { + list_del_init(&ordered->work_list); + wait_for_completion(&ordered->completion); + btrfs_put_ordered_extent(ordered); + cond_resched(); + } + mutex_unlock(&root->ordered_extent_mutex); + + return count; +} + +/* + * Wait for @nr ordered extents that intersect the @bg, or the whole range of + * the filesystem if @bg is NULL. + */ +void btrfs_wait_ordered_roots(struct btrfs_fs_info *fs_info, u64 nr, + const struct btrfs_block_group *bg) +{ + struct btrfs_root *root; + LIST_HEAD(splice); + u64 done; + + mutex_lock(&fs_info->ordered_operations_mutex); + spin_lock(&fs_info->ordered_root_lock); + list_splice_init(&fs_info->ordered_roots, &splice); + while (!list_empty(&splice) && nr) { + root = list_first_entry(&splice, struct btrfs_root, + ordered_root); + root = btrfs_grab_root(root); + BUG_ON(!root); + list_move_tail(&root->ordered_root, + &fs_info->ordered_roots); + spin_unlock(&fs_info->ordered_root_lock); + + done = btrfs_wait_ordered_extents(root, nr, bg); + btrfs_put_root(root); + + if (nr != U64_MAX) + nr -= done; + + spin_lock(&fs_info->ordered_root_lock); + } + list_splice_tail(&splice, &fs_info->ordered_roots); + spin_unlock(&fs_info->ordered_root_lock); + mutex_unlock(&fs_info->ordered_operations_mutex); +} + +/* + * Start IO and wait for a given ordered extent to finish. + * + * Wait on page writeback for all the pages in the extent but not in + * [@nowriteback_start, @nowriteback_start + @nowriteback_len) and the + * IO completion code to insert metadata into the btree corresponding to the extent. + */ +void btrfs_start_ordered_extent_nowriteback(struct btrfs_ordered_extent *entry, + u64 nowriteback_start, u32 nowriteback_len) +{ + u64 start = entry->file_offset; + u64 end = start + entry->num_bytes - 1; + struct btrfs_inode *inode = entry->inode; + bool freespace_inode; + + trace_btrfs_ordered_extent_start(inode, entry); + + /* + * If this is a free space inode do not take the ordered extents lockdep + * map. + */ + freespace_inode = btrfs_is_free_space_inode(inode); + + /* + * pages in the range can be dirty, clean or writeback. We + * start IO on any dirty ones so the wait doesn't stall waiting + * for the flusher thread to find them + */ + if (!test_bit(BTRFS_ORDERED_DIRECT, &entry->flags)) { + if (!nowriteback_len) { + filemap_fdatawrite_range(inode->vfs_inode.i_mapping, start, end); + } else { + if (start < nowriteback_start) + filemap_fdatawrite_range(inode->vfs_inode.i_mapping, start, + nowriteback_start - 1); + if (nowriteback_start + nowriteback_len < end) + filemap_fdatawrite_range(inode->vfs_inode.i_mapping, + nowriteback_start + nowriteback_len, + end); + } + } + + if (!freespace_inode) + btrfs_might_wait_for_event(inode->root->fs_info, btrfs_ordered_extent); + wait_event(entry->wait, test_bit(BTRFS_ORDERED_COMPLETE, &entry->flags)); +} + +/* + * Used to wait on ordered extents across a large range of bytes. + */ +int btrfs_wait_ordered_range(struct btrfs_inode *inode, u64 start, u64 len) +{ + int ret = 0; + int ret_wb = 0; + u64 end; + u64 orig_end; + struct btrfs_ordered_extent *ordered; + + if (start + len < start) { + orig_end = OFFSET_MAX; + } else { + orig_end = start + len - 1; + if (orig_end > OFFSET_MAX) + orig_end = OFFSET_MAX; + } + + /* start IO across the range first to instantiate any delalloc + * extents + */ + ret = btrfs_fdatawrite_range(inode, start, orig_end); + if (ret) + return ret; + + /* + * If we have a writeback error don't return immediately. Wait first + * for any ordered extents that haven't completed yet. This is to make + * sure no one can dirty the same page ranges and call writepages() + * before the ordered extents complete - to avoid failures (-EEXIST) + * when adding the new ordered extents to the ordered tree. + */ + ret_wb = filemap_fdatawait_range(inode->vfs_inode.i_mapping, start, orig_end); + + end = orig_end; + while (1) { + ordered = btrfs_lookup_first_ordered_extent(inode, end); + if (!ordered) + break; + if (ordered->file_offset > orig_end) { + btrfs_put_ordered_extent(ordered); + break; + } + if (ordered->file_offset + ordered->num_bytes <= start) { + btrfs_put_ordered_extent(ordered); + break; + } + btrfs_start_ordered_extent(ordered); + end = ordered->file_offset; + /* + * If the ordered extent had an error save the error but don't + * exit without waiting first for all other ordered extents in + * the range to complete. + */ + if (test_bit(BTRFS_ORDERED_IOERR, &ordered->flags)) + ret = -EIO; + btrfs_put_ordered_extent(ordered); + if (end == 0 || end == start) + break; + end--; + } + return ret_wb ? ret_wb : ret; +} + +/* + * find an ordered extent corresponding to file_offset. return NULL if + * nothing is found, otherwise take a reference on the extent and return it + */ +struct btrfs_ordered_extent *btrfs_lookup_ordered_extent(struct btrfs_inode *inode, + u64 file_offset) +{ + struct rb_node *node; + struct btrfs_ordered_extent *entry = NULL; + + spin_lock(&inode->ordered_tree_lock); + node = ordered_tree_search(inode, file_offset); + if (!node) + goto out; + + entry = rb_entry(node, struct btrfs_ordered_extent, rb_node); + if (!in_range(file_offset, entry->file_offset, entry->num_bytes)) + entry = NULL; + if (entry) { + refcount_inc(&entry->refs); + trace_btrfs_ordered_extent_lookup(inode, entry); + } +out: + spin_unlock(&inode->ordered_tree_lock); + return entry; +} + +/* Since the DIO code tries to lock a wide area we need to look for any ordered + * extents that exist in the range, rather than just the start of the range. + */ +struct btrfs_ordered_extent *btrfs_lookup_ordered_range( + struct btrfs_inode *inode, u64 file_offset, u64 len) +{ + struct rb_node *node; + struct btrfs_ordered_extent *entry = NULL; + + spin_lock(&inode->ordered_tree_lock); + node = ordered_tree_search(inode, file_offset); + if (!node) { + node = ordered_tree_search(inode, file_offset + len); + if (!node) + goto out; + } + + while (1) { + entry = rb_entry(node, struct btrfs_ordered_extent, rb_node); + if (btrfs_range_overlaps(entry, file_offset, len)) + break; + + if (entry->file_offset >= file_offset + len) { + entry = NULL; + break; + } + entry = NULL; + node = rb_next(node); + if (!node) + break; + } +out: + if (entry) { + refcount_inc(&entry->refs); + trace_btrfs_ordered_extent_lookup_range(inode, entry); + } + spin_unlock(&inode->ordered_tree_lock); + return entry; +} + +/* + * Adds all ordered extents to the given list. The list ends up sorted by the + * file_offset of the ordered extents. + */ +void btrfs_get_ordered_extents_for_logging(struct btrfs_inode *inode, + struct list_head *list) +{ + struct rb_node *n; + + btrfs_assert_inode_locked(inode); + + spin_lock(&inode->ordered_tree_lock); + for (n = rb_first(&inode->ordered_tree); n; n = rb_next(n)) { + struct btrfs_ordered_extent *ordered; + + ordered = rb_entry(n, struct btrfs_ordered_extent, rb_node); + + if (test_bit(BTRFS_ORDERED_LOGGED, &ordered->flags)) + continue; + + ASSERT(list_empty(&ordered->log_list)); + list_add_tail(&ordered->log_list, list); + refcount_inc(&ordered->refs); + trace_btrfs_ordered_extent_lookup_for_logging(inode, ordered); + } + spin_unlock(&inode->ordered_tree_lock); +} + +/* + * lookup and return any extent before 'file_offset'. NULL is returned + * if none is found + */ +struct btrfs_ordered_extent * +btrfs_lookup_first_ordered_extent(struct btrfs_inode *inode, u64 file_offset) +{ + struct rb_node *node; + struct btrfs_ordered_extent *entry = NULL; + + spin_lock(&inode->ordered_tree_lock); + node = ordered_tree_search(inode, file_offset); + if (!node) + goto out; + + entry = rb_entry(node, struct btrfs_ordered_extent, rb_node); + refcount_inc(&entry->refs); + trace_btrfs_ordered_extent_lookup_first(inode, entry); +out: + spin_unlock(&inode->ordered_tree_lock); + return entry; +} + +/* + * Lookup the first ordered extent that overlaps the range + * [@file_offset, @file_offset + @len). + * + * The difference between this and btrfs_lookup_first_ordered_extent() is + * that this one won't return any ordered extent that does not overlap the range. + * And the difference against btrfs_lookup_ordered_extent() is, this function + * ensures the first ordered extent gets returned. + */ +struct btrfs_ordered_extent *btrfs_lookup_first_ordered_range( + struct btrfs_inode *inode, u64 file_offset, u64 len) +{ + struct rb_node *node; + struct rb_node *cur; + struct rb_node *prev; + struct rb_node *next; + struct btrfs_ordered_extent *entry = NULL; + + spin_lock(&inode->ordered_tree_lock); + node = inode->ordered_tree.rb_node; + /* + * Here we don't want to use tree_search() which will use tree->last + * and screw up the search order. + * And __tree_search() can't return the adjacent ordered extents + * either, thus here we do our own search. + */ + while (node) { + entry = rb_entry(node, struct btrfs_ordered_extent, rb_node); + + if (file_offset < entry->file_offset) { + node = node->rb_left; + } else if (file_offset >= entry_end(entry)) { + node = node->rb_right; + } else { + /* + * Direct hit, got an ordered extent that starts at + * @file_offset + */ + goto out; + } + } + if (!entry) { + /* Empty tree */ + goto out; + } + + cur = &entry->rb_node; + /* We got an entry around @file_offset, check adjacent entries */ + if (entry->file_offset < file_offset) { + prev = cur; + next = rb_next(cur); + } else { + prev = rb_prev(cur); + next = cur; + } + if (prev) { + entry = rb_entry(prev, struct btrfs_ordered_extent, rb_node); + if (btrfs_range_overlaps(entry, file_offset, len)) + goto out; + } + if (next) { + entry = rb_entry(next, struct btrfs_ordered_extent, rb_node); + if (btrfs_range_overlaps(entry, file_offset, len)) + goto out; + } + /* No ordered extent in the range */ + entry = NULL; +out: + if (entry) { + refcount_inc(&entry->refs); + trace_btrfs_ordered_extent_lookup_first_range(inode, entry); + } + + spin_unlock(&inode->ordered_tree_lock); + return entry; +} + +/* + * Lock the passed range and ensures all pending ordered extents in it are run + * to completion. + * + * @inode: Inode whose ordered tree is to be searched + * @start: Beginning of range to flush + * @end: Last byte of range to lock + * @cached_state: If passed, will return the extent state responsible for the + * locked range. It's the caller's responsibility to free the + * cached state. + * + * Always return with the given range locked, ensuring after it's called no + * order extent can be pending. + */ +void btrfs_lock_and_flush_ordered_range(struct btrfs_inode *inode, u64 start, + u64 end, + struct extent_state **cached_state) +{ + struct btrfs_ordered_extent *ordered; + struct extent_state *cache = NULL; + struct extent_state **cachedp = &cache; + + if (cached_state) + cachedp = cached_state; + + while (1) { + btrfs_lock_extent(&inode->io_tree, start, end, cachedp); + ordered = btrfs_lookup_ordered_range(inode, start, + end - start + 1); + if (!ordered) { + /* + * If no external cached_state has been passed then + * decrement the extra ref taken for cachedp since we + * aren't exposing it outside of this function + */ + if (!cached_state) + refcount_dec(&cache->refs); + break; + } + btrfs_unlock_extent(&inode->io_tree, start, end, cachedp); + btrfs_start_ordered_extent(ordered); + btrfs_put_ordered_extent(ordered); + } +} + +/* + * Lock the passed range and ensure all pending ordered extents in it are run + * to completion in nowait mode. + * + * Return true if btrfs_lock_ordered_range does not return any extents, + * otherwise false. + */ +bool btrfs_try_lock_ordered_range(struct btrfs_inode *inode, u64 start, u64 end, + struct extent_state **cached_state) +{ + struct btrfs_ordered_extent *ordered; + + if (!btrfs_try_lock_extent(&inode->io_tree, start, end, cached_state)) + return false; + + ordered = btrfs_lookup_ordered_range(inode, start, end - start + 1); + if (!ordered) + return true; + + btrfs_put_ordered_extent(ordered); + btrfs_unlock_extent(&inode->io_tree, start, end, cached_state); + + return false; +} + +/* Split out a new ordered extent for this first @len bytes of @ordered. */ +struct btrfs_ordered_extent *btrfs_split_ordered_extent( + struct btrfs_ordered_extent *ordered, u64 len) +{ + struct btrfs_inode *inode = ordered->inode; + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + u64 file_offset = ordered->file_offset; + u64 disk_bytenr = ordered->disk_bytenr; + unsigned long flags = ordered->flags; + struct btrfs_ordered_sum *sum, *tmpsum; + struct btrfs_ordered_extent *new; + struct rb_node *node; + u64 offset = 0; + + trace_btrfs_ordered_extent_split(inode, ordered); + + ASSERT(!(flags & (1U << BTRFS_ORDERED_COMPRESSED)), "flags=0x%lx", flags); + + /* + * The entire bio must be covered by the ordered extent, but we can't + * reduce the original extent to a zero length either. + */ + if (WARN_ON_ONCE(len >= ordered->num_bytes)) + return ERR_PTR(-EINVAL); + /* + * If our ordered extent had an error there's no point in continuing. + * The error may have come from a transaction abort done either by this + * task or some other concurrent task, and the transaction abort path + * iterates over all existing ordered extents and sets the flag + * BTRFS_ORDERED_IOERR on them. + */ + if (unlikely(flags & (1U << BTRFS_ORDERED_IOERR))) { + const int fs_error = BTRFS_FS_ERROR(fs_info); + + return fs_error ? ERR_PTR(fs_error) : ERR_PTR(-EIO); + } + /* We cannot split partially completed ordered extents. */ + if (ordered->bytes_left) { + ASSERT(!(flags & ~BTRFS_ORDERED_TYPE_FLAGS), "flags=0x%lx", flags); + if (WARN_ON_ONCE(ordered->bytes_left != ordered->disk_num_bytes)) + return ERR_PTR(-EINVAL); + } + /* We cannot split a compressed ordered extent. */ + if (WARN_ON_ONCE(ordered->disk_num_bytes != ordered->num_bytes)) + return ERR_PTR(-EINVAL); + + new = alloc_ordered_extent(inode, file_offset, len, len, disk_bytenr, + len, 0, flags, ordered->compress_type); + if (IS_ERR(new)) + return new; + + /* One ref for the tree. */ + refcount_inc(&new->refs); + + /* + * Take the root's ordered_extent_lock to avoid a race with + * btrfs_wait_ordered_extents() when updating the disk_bytenr and + * disk_num_bytes fields of the ordered extent below. + * + * There's no concern about a previous caller of + * btrfs_wait_ordered_extents() getting the trimmed ordered extent + * before we insert the new one, because even if it gets the ordered + * extent before it's trimmed and the new one inserted, right before it + * uses it or during its use, the ordered extent might have been + * trimmed in the meanwhile, and it missed the new ordered extent. + * There's no way around this and it's harmless for current use cases, + * so we take the root's ordered_extent_lock to fix that race during + * trimming and silence tools like KCSAN. + */ + spin_lock_irq(&root->ordered_extent_lock); + spin_lock(&inode->ordered_tree_lock); + + /* + * We don't have overlapping ordered extents (that would imply double + * allocation of extents) and we checked above that the split length + * does not cross the ordered extent's num_bytes field, so there's + * no need to remove it and re-insert it in the tree. + */ + ordered->file_offset += len; + ordered->disk_bytenr += len; + ordered->num_bytes -= len; + ordered->disk_num_bytes -= len; + ordered->ram_bytes -= len; + + if (test_bit(BTRFS_ORDERED_IO_DONE, &ordered->flags)) { + ASSERT(ordered->bytes_left == 0, "ordered->bytes_left=%llu", + ordered->bytes_left); + new->bytes_left = 0; + } else { + ordered->bytes_left -= len; + } + + if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags)) { + if (ordered->truncated_len > len) { + ordered->truncated_len -= len; + } else { + new->truncated_len = ordered->truncated_len; + ordered->truncated_len = 0; + } + } + + list_for_each_entry_safe(sum, tmpsum, &ordered->csum_list, list) { + if (offset == len) + break; + list_move_tail(&sum->list, &new->csum_list); + offset += sum->len; + } + + node = tree_insert(&inode->ordered_tree, new->file_offset, &new->rb_node); + if (unlikely(node)) + btrfs_panic(fs_info, -EEXIST, + "inconsistency in ordered tree at offset %llu after split", + new->file_offset); + spin_unlock(&inode->ordered_tree_lock); + + list_add_tail(&new->root_extent_list, &root->ordered_extents); + root->nr_ordered_extents++; + spin_unlock_irq(&root->ordered_extent_lock); + return new; +} + +int __init ordered_data_init(void) +{ + btrfs_ordered_extent_cache = KMEM_CACHE(btrfs_ordered_extent, 0); + if (!btrfs_ordered_extent_cache) + return -ENOMEM; + + return 0; +} + +void __cold ordered_data_exit(void) +{ + kmem_cache_destroy(btrfs_ordered_extent_cache); +} diff --git a/fs/btrfs/ordered-data.h b/fs/btrfs/ordered-data.h new file mode 100644 index 000000000..8d5d5ba1e --- /dev/null +++ b/fs/btrfs/ordered-data.h @@ -0,0 +1,234 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#ifndef BTRFS_ORDERED_DATA_H +#define BTRFS_ORDERED_DATA_H + +#include +#include +#include +#include +#include +#include +#include "async-thread.h" + +struct inode; +struct page; +struct extent_state; +struct btrfs_block_group; +struct btrfs_inode; +struct btrfs_root; +struct btrfs_fs_info; + +struct btrfs_ordered_sum { + /* + * Logical start address and length for of the blocks covered by + * the sums array. + */ + u64 logical; + u32 len; + + struct list_head list; + /* last field is a variable length array of csums */ + u8 sums[]; +}; + +/* + * Bits for btrfs_ordered_extent::flags. + * + * BTRFS_ORDERED_IO_DONE is set when all of the blocks are written. + * It is used to make sure metadata is inserted into the tree only once + * per extent. + * + * BTRFS_ORDERED_COMPLETE is set when the extent is removed from the + * rbtree, just before waking any waiters. It is used to indicate the + * IO is done and any metadata is inserted into the tree. + */ +enum { + /* Extra status bits for ordered extents */ + + /* Set when all the pages are written. */ + BTRFS_ORDERED_IO_DONE, + /* Set when removed from the tree. */ + BTRFS_ORDERED_COMPLETE, + /* We had an io error when writing this out. */ + BTRFS_ORDERED_IOERR, + /* Set when we have to truncate an extent. */ + BTRFS_ORDERED_TRUNCATED, + /* Used during fsync to track already logged extents. */ + BTRFS_ORDERED_LOGGED, + /* We have already logged all the csums of the ordered extent. */ + BTRFS_ORDERED_LOGGED_CSUM, + /* We wait for this extent to complete in the current transaction. */ + BTRFS_ORDERED_PENDING, + + /* + * Different types for ordered extents, one and only one of these types + * need to be set when creating ordered extent. + * + * REGULAR: For regular non-compressed COW write + * NOCOW: For NOCOW write into existing non-hole extent + * PREALLOC: For NOCOW write into preallocated extent + * COMPRESSED: For compressed COW write + */ + BTRFS_ORDERED_REGULAR, + BTRFS_ORDERED_NOCOW, + BTRFS_ORDERED_PREALLOC, + BTRFS_ORDERED_COMPRESSED, + + /* Extra bit for encoded write, must be set with COMPRESSED. */ + BTRFS_ORDERED_ENCODED, + + /* + * Extra bit for direct io, can only be set for + * REGULAR/NOCOW/PREALLOC. Must not be set for COMPRESSED nor ENCODED. + */ + BTRFS_ORDERED_DIRECT, + + BTRFS_ORDERED_NR_FLAGS, +}; +static_assert(BTRFS_ORDERED_NR_FLAGS <= BITS_PER_LONG); + +/* One and only one flag can be set. */ +#define BTRFS_ORDERED_EXCLUSIVE_FLAGS ((1UL << BTRFS_ORDERED_REGULAR) | \ + (1UL << BTRFS_ORDERED_NOCOW) | \ + (1UL << BTRFS_ORDERED_PREALLOC) | \ + (1UL << BTRFS_ORDERED_COMPRESSED)) + +/* BTRFS_ORDERED_* flags that specify the type of the extent. */ +#define BTRFS_ORDERED_TYPE_FLAGS (BTRFS_ORDERED_EXCLUSIVE_FLAGS | \ + (1UL << BTRFS_ORDERED_DIRECT) | \ + (1UL << BTRFS_ORDERED_ENCODED)) + +struct btrfs_ordered_extent { + /* logical offset in the file */ + u64 file_offset; + + /* + * These fields directly correspond to the same fields in + * btrfs_file_extent_item. + */ + u64 num_bytes; + u64 ram_bytes; + u64 disk_bytenr; + u64 disk_num_bytes; + u64 offset; + + /* number of bytes that still need writing */ + u64 bytes_left; + + /* + * If we get truncated we need to adjust the file extent we enter for + * this ordered extent so that we do not expose stale data. + */ + u64 truncated_len; + + /* flags (described above) */ + unsigned long flags; + + /* compression algorithm */ + int compress_type; + + /* Qgroup reserved space */ + int qgroup_rsv; + + /* reference count */ + refcount_t refs; + + /* the inode we belong to */ + struct btrfs_inode *inode; + + /* list of checksums for insertion when the extent io is done */ + struct list_head csum_list; + + /* used for fast fsyncs */ + struct list_head log_list; + + /* used to wait for the BTRFS_ORDERED_COMPLETE bit */ + wait_queue_head_t wait; + + /* our friendly rbtree entry */ + struct rb_node rb_node; + + /* a per root list of all the pending ordered extents */ + struct list_head root_extent_list; + + struct btrfs_work work; + + struct completion completion; + struct btrfs_work flush_work; + struct list_head work_list; + + struct list_head bioc_list; +}; + +int btrfs_finish_one_ordered(struct btrfs_ordered_extent *ordered_extent); +int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent); + +void btrfs_put_ordered_extent(struct btrfs_ordered_extent *entry); +void btrfs_remove_ordered_extent(struct btrfs_ordered_extent *entry); +void btrfs_finish_ordered_extent(struct btrfs_ordered_extent *ordered, + u64 file_offset, u64 len, bool uptodate); +void btrfs_mark_ordered_io_finished(struct btrfs_inode *inode, + u64 file_offset, u64 num_bytes, bool uptodate); +bool btrfs_dec_test_ordered_pending(struct btrfs_inode *inode, + struct btrfs_ordered_extent **cached, + u64 file_offset, u64 io_size); + +/* + * This represents details about the target file extent item of a write operation. + */ +struct btrfs_file_extent { + u64 disk_bytenr; + u64 disk_num_bytes; + u64 num_bytes; + u64 ram_bytes; + u64 offset; + u8 compression; +}; + +struct btrfs_ordered_extent *btrfs_alloc_ordered_extent( + struct btrfs_inode *inode, u64 file_offset, + const struct btrfs_file_extent *file_extent, unsigned long flags); +void btrfs_add_ordered_sum(struct btrfs_ordered_extent *entry, + struct btrfs_ordered_sum *sum); +struct btrfs_ordered_extent *btrfs_lookup_ordered_extent(struct btrfs_inode *inode, + u64 file_offset); +void btrfs_start_ordered_extent_nowriteback(struct btrfs_ordered_extent *entry, + u64 nowriteback_start, u32 nowriteback_len); +static inline void btrfs_start_ordered_extent(struct btrfs_ordered_extent *entry) +{ + return btrfs_start_ordered_extent_nowriteback(entry, 0, 0); +} + +int btrfs_wait_ordered_range(struct btrfs_inode *inode, u64 start, u64 len); +struct btrfs_ordered_extent * +btrfs_lookup_first_ordered_extent(struct btrfs_inode *inode, u64 file_offset); +struct btrfs_ordered_extent *btrfs_lookup_first_ordered_range( + struct btrfs_inode *inode, u64 file_offset, u64 len); +struct btrfs_ordered_extent *btrfs_lookup_ordered_range( + struct btrfs_inode *inode, + u64 file_offset, + u64 len); +void btrfs_get_ordered_extents_for_logging(struct btrfs_inode *inode, + struct list_head *list); +u64 btrfs_wait_ordered_extents(struct btrfs_root *root, u64 nr, + const struct btrfs_block_group *bg); +void btrfs_wait_ordered_roots(struct btrfs_fs_info *fs_info, u64 nr, + const struct btrfs_block_group *bg); +void btrfs_lock_and_flush_ordered_range(struct btrfs_inode *inode, u64 start, + u64 end, + struct extent_state **cached_state); +bool btrfs_try_lock_ordered_range(struct btrfs_inode *inode, u64 start, u64 end, + struct extent_state **cached_state); +struct btrfs_ordered_extent *btrfs_split_ordered_extent( + struct btrfs_ordered_extent *ordered, u64 len); +void btrfs_mark_ordered_extent_error(struct btrfs_ordered_extent *ordered); +void btrfs_mark_ordered_extent_truncated(struct btrfs_ordered_extent *ordered, + u64 truncate_len); +int __init ordered_data_init(void); +void __cold ordered_data_exit(void); + +#endif diff --git a/fs/btrfs/orphan.c b/fs/btrfs/orphan.c new file mode 100644 index 000000000..9f3ad1241 --- /dev/null +++ b/fs/btrfs/orphan.c @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2008 Red Hat. All rights reserved. + */ + +#include "ctree.h" +#include "orphan.h" + +int btrfs_insert_orphan_item(struct btrfs_trans_handle *trans, + struct btrfs_root *root, u64 offset) +{ + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + + key.objectid = BTRFS_ORPHAN_OBJECTID; + key.type = BTRFS_ORPHAN_ITEM_KEY; + key.offset = offset; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + return btrfs_insert_empty_item(trans, root, path, &key, 0); +} + +int btrfs_del_orphan_item(struct btrfs_trans_handle *trans, + struct btrfs_root *root, u64 offset) +{ + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + int ret = 0; + + key.objectid = BTRFS_ORPHAN_OBJECTID; + key.type = BTRFS_ORPHAN_ITEM_KEY; + key.offset = offset; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + ret = btrfs_search_slot(trans, root, &key, path, -1, 1); + if (ret < 0) + return ret; + if (ret) + return -ENOENT; + + return btrfs_del_item(trans, root, path); +} diff --git a/fs/btrfs/orphan.h b/fs/btrfs/orphan.h new file mode 100644 index 000000000..aa54a88a6 --- /dev/null +++ b/fs/btrfs/orphan.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_ORPHAN_H +#define BTRFS_ORPHAN_H + +#include + +struct btrfs_trans_handle; +struct btrfs_root; + +int btrfs_insert_orphan_item(struct btrfs_trans_handle *trans, + struct btrfs_root *root, u64 offset); +int btrfs_del_orphan_item(struct btrfs_trans_handle *trans, + struct btrfs_root *root, u64 offset); + +#endif diff --git a/fs/btrfs/print-tree.c b/fs/btrfs/print-tree.c new file mode 100644 index 000000000..53e726119 --- /dev/null +++ b/fs/btrfs/print-tree.c @@ -0,0 +1,639 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#include "messages.h" +#include "ctree.h" +#include "disk-io.h" +#include "file-item.h" +#include "print-tree.h" +#include "accessors.h" +#include "tree-checker.h" +#include "volumes.h" +#include "raid-stripe-tree.h" + +/* + * Large enough buffer size for the stringification of any key type yet short + * enough to use the stack and avoid allocations. + */ +#define KEY_TYPE_BUF_SIZE 32 + +struct root_name_map { + u64 id; + const char *name; +}; + +static const struct root_name_map root_map[] = { + { BTRFS_ROOT_TREE_OBJECTID, "ROOT_TREE" }, + { BTRFS_EXTENT_TREE_OBJECTID, "EXTENT_TREE" }, + { BTRFS_CHUNK_TREE_OBJECTID, "CHUNK_TREE" }, + { BTRFS_DEV_TREE_OBJECTID, "DEV_TREE" }, + { BTRFS_FS_TREE_OBJECTID, "FS_TREE" }, + { BTRFS_CSUM_TREE_OBJECTID, "CSUM_TREE" }, + { BTRFS_TREE_LOG_OBJECTID, "TREE_LOG" }, + { BTRFS_QUOTA_TREE_OBJECTID, "QUOTA_TREE" }, + { BTRFS_UUID_TREE_OBJECTID, "UUID_TREE" }, + { BTRFS_FREE_SPACE_TREE_OBJECTID, "FREE_SPACE_TREE" }, + { BTRFS_BLOCK_GROUP_TREE_OBJECTID, "BLOCK_GROUP_TREE" }, + { BTRFS_DATA_RELOC_TREE_OBJECTID, "DATA_RELOC_TREE" }, + { BTRFS_RAID_STRIPE_TREE_OBJECTID, "RAID_STRIPE_TREE" }, + { BTRFS_REMAP_TREE_OBJECTID, "REMAP_TREE" }, +}; + +const char *btrfs_root_name(const struct btrfs_key *key, char *buf) +{ + int i; + + if (key->objectid == BTRFS_TREE_RELOC_OBJECTID) { + snprintf(buf, BTRFS_ROOT_NAME_BUF_LEN, + "TREE_RELOC offset=%llu", key->offset); + return buf; + } + + for (i = 0; i < ARRAY_SIZE(root_map); i++) { + if (root_map[i].id == key->objectid) + return root_map[i].name; + } + + snprintf(buf, BTRFS_ROOT_NAME_BUF_LEN, "%llu", key->objectid); + return buf; +} + +static void print_chunk(const struct extent_buffer *eb, struct btrfs_chunk *chunk) +{ + int num_stripes = btrfs_chunk_num_stripes(eb, chunk); + int i; + pr_info("\t\tchunk length %llu owner %llu type %llu num_stripes %d\n", + btrfs_chunk_length(eb, chunk), btrfs_chunk_owner(eb, chunk), + btrfs_chunk_type(eb, chunk), num_stripes); + for (i = 0 ; i < num_stripes ; i++) { + pr_info("\t\t\tstripe %d devid %llu offset %llu\n", i, + btrfs_stripe_devid_nr(eb, chunk, i), + btrfs_stripe_offset_nr(eb, chunk, i)); + } +} +static void print_dev_item(const struct extent_buffer *eb, + struct btrfs_dev_item *dev_item) +{ + pr_info("\t\tdev item devid %llu total_bytes %llu bytes used %llu\n", + btrfs_device_id(eb, dev_item), + btrfs_device_total_bytes(eb, dev_item), + btrfs_device_bytes_used(eb, dev_item)); +} +static void print_extent_data_ref(const struct extent_buffer *eb, + struct btrfs_extent_data_ref *ref) +{ + pr_cont("extent data backref root %llu objectid %llu offset %llu count %u\n", + btrfs_extent_data_ref_root(eb, ref), + btrfs_extent_data_ref_objectid(eb, ref), + btrfs_extent_data_ref_offset(eb, ref), + btrfs_extent_data_ref_count(eb, ref)); +} + +static void print_extent_owner_ref(const struct extent_buffer *eb, + const struct btrfs_extent_owner_ref *ref) +{ + ASSERT(btrfs_fs_incompat(eb->fs_info, SIMPLE_QUOTA)); + pr_cont("extent data owner root %llu\n", btrfs_extent_owner_ref_root_id(eb, ref)); +} + +static void print_extent_item(const struct extent_buffer *eb, int slot, int type) +{ + struct btrfs_extent_item *ei; + struct btrfs_extent_inline_ref *iref; + struct btrfs_extent_data_ref *dref; + struct btrfs_shared_data_ref *sref; + struct btrfs_extent_owner_ref *oref; + struct btrfs_disk_key key; + unsigned long end; + unsigned long ptr; + u32 item_size = btrfs_item_size(eb, slot); + u64 flags; + u64 offset; + int ref_index = 0; + + if (unlikely(item_size < sizeof(*ei))) { + btrfs_err(eb->fs_info, + "unexpected extent item size, has %u expect >= %zu", + item_size, sizeof(*ei)); + return; + } + + ei = btrfs_item_ptr(eb, slot, struct btrfs_extent_item); + flags = btrfs_extent_flags(eb, ei); + + pr_info("\t\textent refs %llu gen %llu flags %llu\n", + btrfs_extent_refs(eb, ei), btrfs_extent_generation(eb, ei), + flags); + + if ((type == BTRFS_EXTENT_ITEM_KEY) && + flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) { + struct btrfs_tree_block_info *info; + info = (struct btrfs_tree_block_info *)(ei + 1); + btrfs_tree_block_key(eb, info, &key); + pr_info("\t\ttree block key " BTRFS_KEY_FMT " level %d\n", + btrfs_disk_key_objectid(&key), key.type, + btrfs_disk_key_offset(&key), + btrfs_tree_block_level(eb, info)); + iref = (struct btrfs_extent_inline_ref *)(info + 1); + } else { + iref = (struct btrfs_extent_inline_ref *)(ei + 1); + } + + ptr = (unsigned long)iref; + end = (unsigned long)ei + item_size; + while (ptr < end) { + iref = (struct btrfs_extent_inline_ref *)ptr; + type = btrfs_extent_inline_ref_type(eb, iref); + offset = btrfs_extent_inline_ref_offset(eb, iref); + pr_info("\t\tref#%d: ", ref_index++); + switch (type) { + case BTRFS_TREE_BLOCK_REF_KEY: + pr_cont("tree block backref root %llu\n", offset); + break; + case BTRFS_SHARED_BLOCK_REF_KEY: + pr_cont("shared block backref parent %llu\n", offset); + /* + * offset is supposed to be a tree block which + * must be aligned to nodesize. + */ + if (!IS_ALIGNED(offset, eb->fs_info->sectorsize)) + pr_info( + "\t\t\t(parent %llu not aligned to sectorsize %u)\n", + offset, eb->fs_info->sectorsize); + break; + case BTRFS_EXTENT_DATA_REF_KEY: + dref = (struct btrfs_extent_data_ref *)(&iref->offset); + print_extent_data_ref(eb, dref); + break; + case BTRFS_SHARED_DATA_REF_KEY: + sref = (struct btrfs_shared_data_ref *)(iref + 1); + pr_cont("shared data backref parent %llu count %u\n", + offset, btrfs_shared_data_ref_count(eb, sref)); + /* + * Offset is supposed to be a tree block which must be + * aligned to sectorsize. + */ + if (!IS_ALIGNED(offset, eb->fs_info->sectorsize)) + pr_info( + "\t\t\t(parent %llu not aligned to sectorsize %u)\n", + offset, eb->fs_info->sectorsize); + break; + case BTRFS_EXTENT_OWNER_REF_KEY: + oref = (struct btrfs_extent_owner_ref *)(&iref->offset); + print_extent_owner_ref(eb, oref); + break; + default: + pr_cont("(extent %llu has INVALID ref type %d)\n", + eb->start, type); + return; + } + ptr += btrfs_extent_inline_ref_size(type); + } + WARN_ON(ptr > end); +} + +static void print_uuid_item(const struct extent_buffer *l, unsigned long offset, + u32 item_size) +{ + if (!IS_ALIGNED(item_size, sizeof(u64))) { + btrfs_warn(l->fs_info, "uuid item with illegal size %lu", + (unsigned long)item_size); + return; + } + while (item_size) { + __le64 subvol_id; + + read_extent_buffer(l, &subvol_id, offset, sizeof(subvol_id)); + pr_info("\t\tsubvol_id %llu\n", le64_to_cpu(subvol_id)); + item_size -= sizeof(u64); + offset += sizeof(u64); + } +} + +static void print_raid_stripe_key(const struct extent_buffer *eb, u32 item_size, + struct btrfs_stripe_extent *stripe) +{ + const int num_stripes = btrfs_num_raid_stripes(item_size); + + for (int i = 0; i < num_stripes; i++) + pr_info("\t\t\tstride %d devid %llu physical %llu\n", + i, btrfs_raid_stride_devid(eb, &stripe->strides[i]), + btrfs_raid_stride_physical(eb, &stripe->strides[i])); +} + +/* + * Helper to output refs and locking status of extent buffer. Useful to debug + * race condition related problems. + */ +static void print_eb_refs_lock(const struct extent_buffer *eb) +{ +#ifdef CONFIG_BTRFS_DEBUG + btrfs_info(eb->fs_info, "refs %u lock_owner %u current %u", + refcount_read(&eb->refs), eb->lock_owner, current->pid); +#endif +} + +static void print_timespec(const struct extent_buffer *eb, + struct btrfs_timespec *timespec, + const char *prefix, const char *suffix) +{ + const u64 secs = btrfs_timespec_sec(eb, timespec); + const u32 nsecs = btrfs_timespec_nsec(eb, timespec); + + pr_info("%s%llu.%u%s", prefix, secs, nsecs, suffix); +} + +static void print_inode_item(const struct extent_buffer *eb, int i) +{ + struct btrfs_inode_item *ii = btrfs_item_ptr(eb, i, struct btrfs_inode_item); + + pr_info("\t\tinode generation %llu transid %llu size %llu nbytes %llu\n", + btrfs_inode_generation(eb, ii), btrfs_inode_transid(eb, ii), + btrfs_inode_size(eb, ii), btrfs_inode_nbytes(eb, ii)); + pr_info("\t\tblock group %llu mode %o links %u uid %u gid %u\n", + btrfs_inode_block_group(eb, ii), btrfs_inode_mode(eb, ii), + btrfs_inode_nlink(eb, ii), btrfs_inode_uid(eb, ii), + btrfs_inode_gid(eb, ii)); + pr_info("\t\trdev %llu sequence %llu flags 0x%llx\n", + btrfs_inode_rdev(eb, ii), btrfs_inode_sequence(eb, ii), + btrfs_inode_flags(eb, ii)); + print_timespec(eb, &ii->atime, "\t\tatime ", "\n"); + print_timespec(eb, &ii->ctime, "\t\tctime ", "\n"); + print_timespec(eb, &ii->mtime, "\t\tmtime ", "\n"); + print_timespec(eb, &ii->otime, "\t\totime ", "\n"); +} + +static void print_dir_item(const struct extent_buffer *eb, int i) +{ + const u32 size = btrfs_item_size(eb, i); + struct btrfs_dir_item *di = btrfs_item_ptr(eb, i, struct btrfs_dir_item); + u32 cur = 0; + + while (cur < size) { + const u32 name_len = btrfs_dir_name_len(eb, di); + const u32 data_len = btrfs_dir_data_len(eb, di); + const u32 len = sizeof(*di) + name_len + data_len; + struct btrfs_key location; + + btrfs_dir_item_key_to_cpu(eb, di, &location); + pr_info("\t\tlocation key " BTRFS_KEY_FMT " type %d\n", + BTRFS_KEY_FMT_VALUE(&location), btrfs_dir_ftype(eb, di)); + pr_info("\t\ttransid %llu data_len %u name_len %u\n", + btrfs_dir_transid(eb, di), data_len, name_len); + di = (struct btrfs_dir_item *)((char *)di + len); + cur += len; + } +} + +static void print_inode_ref_item(const struct extent_buffer *eb, int i) +{ + const u32 size = btrfs_item_size(eb, i); + struct btrfs_inode_ref *ref = btrfs_item_ptr(eb, i, struct btrfs_inode_ref); + u32 cur = 0; + + while (cur < size) { + const u64 index = btrfs_inode_ref_index(eb, ref); + const u32 name_len = btrfs_inode_ref_name_len(eb, ref); + const u32 len = sizeof(*ref) + name_len; + + pr_info("\t\tindex %llu name_len %u\n", index, name_len); + ref = (struct btrfs_inode_ref *)((char *)ref + len); + cur += len; + } +} + +static void print_inode_extref_item(const struct extent_buffer *eb, int i) +{ + const u32 size = btrfs_item_size(eb, i); + struct btrfs_inode_extref *extref; + u32 cur = 0; + + extref = btrfs_item_ptr(eb, i, struct btrfs_inode_extref); + while (cur < size) { + const u64 index = btrfs_inode_extref_index(eb, extref); + const u32 name_len = btrfs_inode_extref_name_len(eb, extref); + const u64 parent = btrfs_inode_extref_parent(eb, extref); + const u32 len = sizeof(*extref) + name_len; + + pr_info("\t\tindex %llu parent %llu name_len %u\n", + index, parent, name_len); + extref = (struct btrfs_inode_extref *)((char *)extref + len); + cur += len; + } +} + +static void print_dir_log_index_item(const struct extent_buffer *eb, int i) +{ + struct btrfs_dir_log_item *dlog; + + dlog = btrfs_item_ptr(eb, i, struct btrfs_dir_log_item); + pr_info("\t\tdir log end %llu\n", btrfs_dir_log_end(eb, dlog)); +} + +static void print_extent_csum(const struct extent_buffer *eb, int i) +{ + const struct btrfs_fs_info *fs_info = eb->fs_info; + const u32 size = btrfs_item_size(eb, i); + const u32 csum_bytes = (size / fs_info->csum_size) * fs_info->sectorsize; + struct btrfs_key key; + + btrfs_item_key_to_cpu(eb, &key, i); + pr_info("\t\trange start %llu end %llu length %u\n", + key.offset, key.offset + csum_bytes, csum_bytes); +} + +static void print_file_extent_item(const struct extent_buffer *eb, int i) +{ + struct btrfs_file_extent_item *fi; + + fi = btrfs_item_ptr(eb, i, struct btrfs_file_extent_item); + pr_info("\t\tgeneration %llu type %hhu\n", + btrfs_file_extent_generation(eb, fi), + btrfs_file_extent_type(eb, fi)); + + if (btrfs_file_extent_type(eb, fi) == BTRFS_FILE_EXTENT_INLINE) { + pr_info("\t\tinline extent data size %u ram_bytes %llu compression %hhu\n", + btrfs_file_extent_inline_item_len(eb, i), + btrfs_file_extent_ram_bytes(eb, fi), + btrfs_file_extent_compression(eb, fi)); + return; + } + + pr_info("\t\textent data disk bytenr %llu nr %llu\n", + btrfs_file_extent_disk_bytenr(eb, fi), + btrfs_file_extent_disk_num_bytes(eb, fi)); + pr_info("\t\textent data offset %llu nr %llu ram %llu\n", + btrfs_file_extent_offset(eb, fi), + btrfs_file_extent_num_bytes(eb, fi), + btrfs_file_extent_ram_bytes(eb, fi)); + pr_info("\t\textent compression %hhu\n", + btrfs_file_extent_compression(eb, fi)); +} + +static void key_type_string(const struct btrfs_key *key, char *buf, int buf_size) +{ + static const char *key_to_str[256] = { + [BTRFS_INODE_ITEM_KEY] = "INODE_ITEM", + [BTRFS_INODE_REF_KEY] = "INODE_REF", + [BTRFS_INODE_EXTREF_KEY] = "INODE_EXTREF", + [BTRFS_DIR_ITEM_KEY] = "DIR_ITEM", + [BTRFS_DIR_INDEX_KEY] = "DIR_INDEX", + [BTRFS_DIR_LOG_ITEM_KEY] = "DIR_LOG_ITEM", + [BTRFS_DIR_LOG_INDEX_KEY] = "DIR_LOG_INDEX", + [BTRFS_XATTR_ITEM_KEY] = "XATTR_ITEM", + [BTRFS_VERITY_DESC_ITEM_KEY] = "VERITY_DESC_ITEM", + [BTRFS_VERITY_MERKLE_ITEM_KEY] = "VERITY_MERKLE_ITEM", + [BTRFS_ORPHAN_ITEM_KEY] = "ORPHAN_ITEM", + [BTRFS_ROOT_ITEM_KEY] = "ROOT_ITEM", + [BTRFS_ROOT_REF_KEY] = "ROOT_REF", + [BTRFS_ROOT_BACKREF_KEY] = "ROOT_BACKREF", + [BTRFS_EXTENT_ITEM_KEY] = "EXTENT_ITEM", + [BTRFS_METADATA_ITEM_KEY] = "METADATA_ITEM", + [BTRFS_TREE_BLOCK_REF_KEY] = "TREE_BLOCK_REF", + [BTRFS_SHARED_BLOCK_REF_KEY] = "SHARED_BLOCK_REF", + [BTRFS_EXTENT_DATA_REF_KEY] = "EXTENT_DATA_REF", + [BTRFS_SHARED_DATA_REF_KEY] = "SHARED_DATA_REF", + [BTRFS_EXTENT_OWNER_REF_KEY] = "EXTENT_OWNER_REF", + [BTRFS_EXTENT_CSUM_KEY] = "EXTENT_CSUM", + [BTRFS_EXTENT_DATA_KEY] = "EXTENT_DATA", + [BTRFS_BLOCK_GROUP_ITEM_KEY] = "BLOCK_GROUP_ITEM", + [BTRFS_FREE_SPACE_INFO_KEY] = "FREE_SPACE_INFO", + [BTRFS_FREE_SPACE_EXTENT_KEY] = "FREE_SPACE_EXTENT", + [BTRFS_FREE_SPACE_BITMAP_KEY] = "FREE_SPACE_BITMAP", + [BTRFS_CHUNK_ITEM_KEY] = "CHUNK_ITEM", + [BTRFS_DEV_ITEM_KEY] = "DEV_ITEM", + [BTRFS_DEV_EXTENT_KEY] = "DEV_EXTENT", + [BTRFS_TEMPORARY_ITEM_KEY] = "TEMPORARY_ITEM", + [BTRFS_DEV_REPLACE_KEY] = "DEV_REPLACE", + [BTRFS_STRING_ITEM_KEY] = "STRING_ITEM", + [BTRFS_QGROUP_STATUS_KEY] = "QGROUP_STATUS", + [BTRFS_QGROUP_RELATION_KEY] = "QGROUP_RELATION", + [BTRFS_QGROUP_INFO_KEY] = "QGROUP_INFO", + [BTRFS_QGROUP_LIMIT_KEY] = "QGROUP_LIMIT", + [BTRFS_PERSISTENT_ITEM_KEY] = "PERSISTENT_ITEM", + [BTRFS_UUID_KEY_SUBVOL] = "UUID_KEY_SUBVOL", + [BTRFS_UUID_KEY_RECEIVED_SUBVOL] = "UUID_KEY_RECEIVED_SUBVOL", + [BTRFS_RAID_STRIPE_KEY] = "RAID_STRIPE", + [BTRFS_IDENTITY_REMAP_KEY] = "IDENTITY_REMAP", + [BTRFS_REMAP_KEY] = "REMAP", + [BTRFS_REMAP_BACKREF_KEY] = "REMAP_BACKREF", + }; + + if (key->type == 0 && key->objectid == BTRFS_FREE_SPACE_OBJECTID) + scnprintf(buf, buf_size, "UNTYPED"); + else if (key_to_str[key->type]) + scnprintf(buf, buf_size, "%s", key_to_str[key->type]); + else + scnprintf(buf, buf_size, "UNKNOWN.%d", key->type); +} + +void btrfs_print_leaf(const struct extent_buffer *l) +{ + struct btrfs_fs_info *fs_info; + int i; + u32 type, nr; + struct btrfs_root_item *ri; + struct btrfs_block_group_item *bi; + struct btrfs_extent_data_ref *dref; + struct btrfs_shared_data_ref *sref; + struct btrfs_dev_extent *dev_extent; + struct btrfs_remap_item *remap; + struct btrfs_key key; + + if (!l) + return; + + fs_info = l->fs_info; + nr = btrfs_header_nritems(l); + + btrfs_info(fs_info, + "leaf %llu gen %llu total ptrs %d free space %d owner %lld", + btrfs_header_bytenr(l), btrfs_header_generation(l), nr, + btrfs_leaf_free_space(l), (s64)btrfs_header_owner(l)); + print_eb_refs_lock(l); + for (i = 0 ; i < nr ; i++) { + char key_buf[KEY_TYPE_BUF_SIZE]; + + btrfs_item_key_to_cpu(l, &key, i); + type = key.type; + key_type_string(&key, key_buf, KEY_TYPE_BUF_SIZE); + + pr_info("\titem %d key (%llu %s %llu) itemoff %d itemsize %d\n", + i, key.objectid, key_buf, key.offset, + btrfs_item_offset(l, i), btrfs_item_size(l, i)); + switch (type) { + case BTRFS_INODE_ITEM_KEY: + print_inode_item(l, i); + break; + case BTRFS_INODE_REF_KEY: + print_inode_ref_item(l, i); + break; + case BTRFS_INODE_EXTREF_KEY: + print_inode_extref_item(l, i); + break; + case BTRFS_DIR_ITEM_KEY: + case BTRFS_DIR_INDEX_KEY: + case BTRFS_XATTR_ITEM_KEY: + print_dir_item(l, i); + break; + case BTRFS_DIR_LOG_INDEX_KEY: + print_dir_log_index_item(l, i); + break; + case BTRFS_EXTENT_CSUM_KEY: + print_extent_csum(l, i); + break; + case BTRFS_ROOT_ITEM_KEY: + ri = btrfs_item_ptr(l, i, struct btrfs_root_item); + pr_info("\t\troot data bytenr %llu refs %u\n", + btrfs_disk_root_bytenr(l, ri), + btrfs_disk_root_refs(l, ri)); + break; + case BTRFS_EXTENT_ITEM_KEY: + case BTRFS_METADATA_ITEM_KEY: + print_extent_item(l, i, type); + break; + case BTRFS_TREE_BLOCK_REF_KEY: + pr_info("\t\ttree block backref\n"); + break; + case BTRFS_SHARED_BLOCK_REF_KEY: + pr_info("\t\tshared block backref\n"); + break; + case BTRFS_EXTENT_DATA_REF_KEY: + dref = btrfs_item_ptr(l, i, + struct btrfs_extent_data_ref); + print_extent_data_ref(l, dref); + break; + case BTRFS_SHARED_DATA_REF_KEY: + sref = btrfs_item_ptr(l, i, + struct btrfs_shared_data_ref); + pr_info("\t\tshared data backref count %u\n", + btrfs_shared_data_ref_count(l, sref)); + break; + case BTRFS_EXTENT_DATA_KEY: + print_file_extent_item(l, i); + break; + case BTRFS_BLOCK_GROUP_ITEM_KEY: + bi = btrfs_item_ptr(l, i, + struct btrfs_block_group_item); + pr_info( + "\t\tblock group used %llu chunk_objectid %llu flags %llu\n", + btrfs_block_group_used(l, bi), + btrfs_block_group_chunk_objectid(l, bi), + btrfs_block_group_flags(l, bi)); + break; + case BTRFS_CHUNK_ITEM_KEY: + print_chunk(l, btrfs_item_ptr(l, i, + struct btrfs_chunk)); + break; + case BTRFS_DEV_ITEM_KEY: + print_dev_item(l, btrfs_item_ptr(l, i, + struct btrfs_dev_item)); + break; + case BTRFS_DEV_EXTENT_KEY: + dev_extent = btrfs_item_ptr(l, i, + struct btrfs_dev_extent); + pr_info("\t\tdev extent chunk_tree %llu\n\t\tchunk objectid %llu chunk offset %llu length %llu\n", + btrfs_dev_extent_chunk_tree(l, dev_extent), + btrfs_dev_extent_chunk_objectid(l, dev_extent), + btrfs_dev_extent_chunk_offset(l, dev_extent), + btrfs_dev_extent_length(l, dev_extent)); + break; + case BTRFS_PERSISTENT_ITEM_KEY: + pr_info("\t\tpersistent item objectid %llu offset %llu\n", + key.objectid, key.offset); + switch (key.objectid) { + case BTRFS_DEV_STATS_OBJECTID: + pr_info("\t\tdevice stats\n"); + break; + default: + pr_info("\t\tunknown persistent item\n"); + } + break; + case BTRFS_TEMPORARY_ITEM_KEY: + pr_info("\t\ttemporary item objectid %llu offset %llu\n", + key.objectid, key.offset); + switch (key.objectid) { + case BTRFS_BALANCE_OBJECTID: + pr_info("\t\tbalance status\n"); + break; + default: + pr_info("\t\tunknown temporary item\n"); + } + break; + case BTRFS_DEV_REPLACE_KEY: + pr_info("\t\tdev replace\n"); + break; + case BTRFS_UUID_KEY_SUBVOL: + case BTRFS_UUID_KEY_RECEIVED_SUBVOL: + print_uuid_item(l, btrfs_item_ptr_offset(l, i), + btrfs_item_size(l, i)); + break; + case BTRFS_RAID_STRIPE_KEY: + print_raid_stripe_key(l, btrfs_item_size(l, i), + btrfs_item_ptr(l, i, struct btrfs_stripe_extent)); + break; + case BTRFS_REMAP_KEY: + case BTRFS_REMAP_BACKREF_KEY: + remap = btrfs_item_ptr(l, i, struct btrfs_remap_item); + pr_info("\t\taddress %llu\n", btrfs_remap_address(l, remap)); + break; + } + } +} + +void btrfs_print_tree(const struct extent_buffer *c, bool follow) +{ + struct btrfs_fs_info *fs_info; + int i; u32 nr; + struct btrfs_key key; + int level; + + if (!c) + return; + fs_info = c->fs_info; + nr = btrfs_header_nritems(c); + level = btrfs_header_level(c); + if (level == 0) { + btrfs_print_leaf(c); + return; + } + btrfs_info(fs_info, + "node %llu level %d gen %llu total ptrs %d free spc %u owner %lld", + btrfs_header_bytenr(c), level, btrfs_header_generation(c), + nr, (u32)BTRFS_NODEPTRS_PER_BLOCK(fs_info) - nr, + (s64)btrfs_header_owner(c)); + print_eb_refs_lock(c); + for (i = 0; i < nr; i++) { + btrfs_node_key_to_cpu(c, &key, i); + pr_info("\tkey %d " BTRFS_KEY_FMT " block %llu gen %llu\n", + i, BTRFS_KEY_FMT_VALUE(&key), btrfs_node_blockptr(c, i), + btrfs_node_ptr_generation(c, i)); + } + if (!follow) + return; + for (i = 0; i < nr; i++) { + struct btrfs_tree_parent_check check = { + .level = level - 1, + .transid = btrfs_node_ptr_generation(c, i), + .owner_root = btrfs_header_owner(c), + .has_first_key = true + }; + struct extent_buffer *next; + + btrfs_node_key_to_cpu(c, &check.first_key, i); + next = read_tree_block(fs_info, btrfs_node_blockptr(c, i), &check); + if (IS_ERR(next)) + continue; + + if (btrfs_is_leaf(next) && + level != 1) + BUG(); + if (btrfs_header_level(next) != + level - 1) + BUG(); + btrfs_print_tree(next, follow); + free_extent_buffer(next); + } +} diff --git a/fs/btrfs/print-tree.h b/fs/btrfs/print-tree.h new file mode 100644 index 000000000..d0e620bf5 --- /dev/null +++ b/fs/btrfs/print-tree.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#ifndef BTRFS_PRINT_TREE_H +#define BTRFS_PRINT_TREE_H + +#include + +/* Buffer size to contain tree name and possibly additional data (offset) */ +#define BTRFS_ROOT_NAME_BUF_LEN 48 + +struct extent_buffer; +struct btrfs_key; + +void btrfs_print_leaf(const struct extent_buffer *l); +void btrfs_print_tree(const struct extent_buffer *c, bool follow); +const char *btrfs_root_name(const struct btrfs_key *key, char *buf); + +#endif diff --git a/fs/btrfs/props.c b/fs/btrfs/props.c new file mode 100644 index 000000000..bb77d4637 --- /dev/null +++ b/fs/btrfs/props.c @@ -0,0 +1,485 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2014 Filipe David Borba Manana + */ + +#include +#include +#include "messages.h" +#include "props.h" +#include "btrfs_inode.h" +#include "transaction.h" +#include "ctree.h" +#include "xattr.h" +#include "compression.h" +#include "space-info.h" +#include "fs.h" +#include "accessors.h" +#include "super.h" +#include "dir-item.h" + +#define BTRFS_PROP_HANDLERS_HT_BITS 8 +static DEFINE_HASHTABLE(prop_handlers_ht, BTRFS_PROP_HANDLERS_HT_BITS); + +struct prop_handler { + struct hlist_node node; + const char *xattr_name; + int (*validate)(const struct btrfs_inode *inode, const char *value, + size_t len); + int (*apply)(struct btrfs_inode *inode, const char *value, size_t len); + const char *(*extract)(const struct btrfs_inode *inode); + bool (*ignore)(const struct btrfs_inode *inode); + int inheritable; +}; + +static const struct hlist_head *find_prop_handlers_by_hash(const u64 hash) +{ + struct hlist_head *h; + + h = &prop_handlers_ht[hash_min(hash, BTRFS_PROP_HANDLERS_HT_BITS)]; + if (hlist_empty(h)) + return NULL; + + return h; +} + +static const struct prop_handler * +find_prop_handler(const char *name, + const struct hlist_head *handlers) +{ + struct prop_handler *h; + + if (!handlers) { + u64 hash = btrfs_name_hash(name, strlen(name)); + + handlers = find_prop_handlers_by_hash(hash); + if (!handlers) + return NULL; + } + + hlist_for_each_entry(h, handlers, node) + if (!strcmp(h->xattr_name, name)) + return h; + + return NULL; +} + +int btrfs_validate_prop(const struct btrfs_inode *inode, const char *name, + const char *value, size_t value_len) +{ + const struct prop_handler *handler; + + if (strlen(name) <= XATTR_BTRFS_PREFIX_LEN) + return -EINVAL; + + handler = find_prop_handler(name, NULL); + if (!handler) + return -EINVAL; + + if (value_len == 0) + return 0; + + return handler->validate(inode, value, value_len); +} + +/* + * Check if a property should be ignored (not set) for an inode. + * + * @inode: The target inode. + * @name: The property's name. + * + * The caller must be sure the given property name is valid, for example by + * having previously called btrfs_validate_prop(). + * + * Returns: true if the property should be ignored for the given inode + * false if the property must not be ignored for the given inode + */ +bool btrfs_ignore_prop(const struct btrfs_inode *inode, const char *name) +{ + const struct prop_handler *handler; + + handler = find_prop_handler(name, NULL); + ASSERT(handler != NULL); + + return handler->ignore(inode); +} + +int btrfs_set_prop(struct btrfs_trans_handle *trans, struct btrfs_inode *inode, + const char *name, const char *value, size_t value_len, + int flags) +{ + const struct prop_handler *handler; + int ret; + + handler = find_prop_handler(name, NULL); + if (!handler) + return -EINVAL; + + if (value_len == 0) { + ret = btrfs_setxattr(trans, &inode->vfs_inode, handler->xattr_name, + NULL, 0, flags); + if (ret) + return ret; + + ret = handler->apply(inode, NULL, 0); + ASSERT(ret == 0); + + return ret; + } + + ret = handler->validate(inode, value, value_len); + if (ret) + return ret; + ret = btrfs_setxattr(trans, &inode->vfs_inode, handler->xattr_name, value, + value_len, flags); + if (ret) + return ret; + ret = handler->apply(inode, value, value_len); + /* We validated before, so it should not fail here. */ + ASSERT(ret == 0); + if (unlikely(ret)) { + int ret2; + + /* Try to delete xattr, if not possible abort transaction. */ + ret2 = btrfs_setxattr(trans, &inode->vfs_inode, handler->xattr_name, + NULL, 0, flags); + if (unlikely(ret2)) + btrfs_abort_transaction(trans, ret2); + return ret; + } + + set_bit(BTRFS_INODE_HAS_PROPS, &inode->runtime_flags); + + return 0; +} + +static int iterate_object_props(struct btrfs_root *root, + struct btrfs_path *path, + u64 objectid, + void (*iterator)(void *, + const struct prop_handler *, + const char *, + size_t), + void *ctx) +{ + int ret; + char *name_buf = NULL; + char *value_buf = NULL; + int name_buf_len = 0; + int value_buf_len = 0; + + while (1) { + struct btrfs_key key; + struct btrfs_dir_item *di; + struct extent_buffer *leaf; + u32 total_len, cur, this_len; + int slot; + const struct hlist_head *handlers; + + slot = path->slots[0]; + leaf = path->nodes[0]; + + if (slot >= btrfs_header_nritems(leaf)) { + ret = btrfs_next_leaf(root, path); + if (ret < 0) + goto out; + else if (ret > 0) + break; + continue; + } + + btrfs_item_key_to_cpu(leaf, &key, slot); + if (key.objectid != objectid) + break; + if (key.type != BTRFS_XATTR_ITEM_KEY) + break; + + handlers = find_prop_handlers_by_hash(key.offset); + if (!handlers) + goto next_slot; + + di = btrfs_item_ptr(leaf, slot, struct btrfs_dir_item); + cur = 0; + total_len = btrfs_item_size(leaf, slot); + + while (cur < total_len) { + u32 name_len = btrfs_dir_name_len(leaf, di); + u32 data_len = btrfs_dir_data_len(leaf, di); + unsigned long name_ptr, data_ptr; + const struct prop_handler *handler; + + this_len = sizeof(*di) + name_len + data_len; + name_ptr = (unsigned long)(di + 1); + data_ptr = name_ptr + name_len; + + if (name_len <= XATTR_BTRFS_PREFIX_LEN || + memcmp_extent_buffer(leaf, XATTR_BTRFS_PREFIX, + name_ptr, + XATTR_BTRFS_PREFIX_LEN)) + goto next_dir_item; + + if (name_len >= name_buf_len) { + kfree(name_buf); + name_buf_len = name_len + 1; + name_buf = kmalloc(name_buf_len, GFP_NOFS); + if (!name_buf) { + ret = -ENOMEM; + goto out; + } + } + read_extent_buffer(leaf, name_buf, name_ptr, name_len); + name_buf[name_len] = '\0'; + + handler = find_prop_handler(name_buf, handlers); + if (!handler) + goto next_dir_item; + + if (data_len > value_buf_len) { + kfree(value_buf); + value_buf_len = data_len; + value_buf = kmalloc(data_len, GFP_NOFS); + if (!value_buf) { + ret = -ENOMEM; + goto out; + } + } + read_extent_buffer(leaf, value_buf, data_ptr, data_len); + + iterator(ctx, handler, value_buf, data_len); +next_dir_item: + cur += this_len; + di = (struct btrfs_dir_item *)((char *) di + this_len); + } + +next_slot: + path->slots[0]++; + } + + ret = 0; +out: + btrfs_release_path(path); + kfree(name_buf); + kfree(value_buf); + + return ret; +} + +static void inode_prop_iterator(void *ctx, + const struct prop_handler *handler, + const char *value, + size_t len) +{ + struct inode *inode = ctx; + struct btrfs_root *root = BTRFS_I(inode)->root; + int ret; + + ret = handler->apply(BTRFS_I(inode), value, len); + if (unlikely(ret)) + btrfs_warn(root->fs_info, + "error applying prop %s to ino %llu (root %llu): %d", + handler->xattr_name, btrfs_ino(BTRFS_I(inode)), + btrfs_root_id(root), ret); + else + set_bit(BTRFS_INODE_HAS_PROPS, &BTRFS_I(inode)->runtime_flags); +} + +int btrfs_load_inode_props(struct btrfs_inode *inode, struct btrfs_path *path) +{ + struct btrfs_root *root = inode->root; + u64 ino = btrfs_ino(inode); + + return iterate_object_props(root, path, ino, inode_prop_iterator, + &inode->vfs_inode); +} + +static int prop_compression_validate(const struct btrfs_inode *inode, + const char *value, size_t len) +{ + if (!btrfs_inode_can_compress(inode)) + return -EINVAL; + + if (!value) + return 0; + + if (btrfs_compress_is_valid_type(value, len)) + return 0; + + if ((len == 2 && strncmp("no", value, 2) == 0) || + (len == 4 && strncmp("none", value, 4) == 0)) + return 0; + + return -EINVAL; +} + +static int prop_compression_apply(struct btrfs_inode *inode, const char *value, + size_t len) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + int type; + + /* Reset to defaults */ + if (len == 0) { + inode->flags &= ~BTRFS_INODE_COMPRESS; + inode->flags &= ~BTRFS_INODE_NOCOMPRESS; + inode->prop_compress = BTRFS_COMPRESS_NONE; + return 0; + } + + /* Set NOCOMPRESS flag */ + if ((len == 2 && strncmp("no", value, 2) == 0) || + (len == 4 && strncmp("none", value, 4) == 0)) { + inode->flags |= BTRFS_INODE_NOCOMPRESS; + inode->flags &= ~BTRFS_INODE_COMPRESS; + inode->prop_compress = BTRFS_COMPRESS_NONE; + + return 0; + } + + if (!strncmp("lzo", value, 3)) { + type = BTRFS_COMPRESS_LZO; + btrfs_set_fs_incompat(fs_info, COMPRESS_LZO); + } else if (!strncmp("zlib", value, 4)) { + type = BTRFS_COMPRESS_ZLIB; + } else if (!strncmp("zstd", value, 4)) { + type = BTRFS_COMPRESS_ZSTD; + btrfs_set_fs_incompat(fs_info, COMPRESS_ZSTD); + } else { + return -EINVAL; + } + + inode->flags &= ~BTRFS_INODE_NOCOMPRESS; + inode->flags |= BTRFS_INODE_COMPRESS; + inode->prop_compress = type; + + return 0; +} + +static bool prop_compression_ignore(const struct btrfs_inode *inode) +{ + /* + * Compression only has effect for regular files, and for directories + * we set it just to propagate it to new files created inside them. + * Everything else (symlinks, devices, sockets, fifos) is pointless as + * it will do nothing, so don't waste metadata space on a compression + * xattr for anything that is neither a file nor a directory. + */ + if (!S_ISREG(inode->vfs_inode.i_mode) && + !S_ISDIR(inode->vfs_inode.i_mode)) + return true; + + return false; +} + +static const char *prop_compression_extract(const struct btrfs_inode *inode) +{ + switch (inode->prop_compress) { + case BTRFS_COMPRESS_ZLIB: + case BTRFS_COMPRESS_LZO: + case BTRFS_COMPRESS_ZSTD: + return btrfs_compress_type2str(inode->prop_compress); + default: + break; + } + + return NULL; +} + +static struct prop_handler prop_handlers[] = { + { + .xattr_name = XATTR_BTRFS_PREFIX "compression", + .validate = prop_compression_validate, + .apply = prop_compression_apply, + .extract = prop_compression_extract, + .ignore = prop_compression_ignore, + .inheritable = 1 + }, +}; + +int btrfs_inode_inherit_props(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + const struct btrfs_inode *parent) +{ + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + int ret; + int i; + bool need_reserve = false; + + if (!test_bit(BTRFS_INODE_HAS_PROPS, &parent->runtime_flags)) + return 0; + + for (i = 0; i < ARRAY_SIZE(prop_handlers); i++) { + const struct prop_handler *h = &prop_handlers[i]; + const char *value; + u64 num_bytes = 0; + + if (!h->inheritable) + continue; + + if (h->ignore(inode)) + continue; + + value = h->extract(parent); + if (!value) + continue; + + /* + * This is not strictly necessary as the property should be + * valid, but in case it isn't, don't propagate it further. + */ + ret = h->validate(inode, value, strlen(value)); + if (ret) + continue; + + /* + * Currently callers should be reserving 1 item for properties, + * since we only have 1 property that we currently support. If + * we add more in the future we need to try and reserve more + * space for them. But we should also revisit how we do space + * reservations if we do add more properties in the future. + */ + if (need_reserve) { + num_bytes = btrfs_calc_insert_metadata_size(fs_info, 1); + ret = btrfs_block_rsv_add(fs_info, trans->block_rsv, + num_bytes, + BTRFS_RESERVE_NO_FLUSH); + if (ret) + return ret; + } + + ret = btrfs_setxattr(trans, &inode->vfs_inode, h->xattr_name, value, + strlen(value), 0); + if (!ret) { + ret = h->apply(inode, value, strlen(value)); + if (ret) + btrfs_setxattr(trans, &inode->vfs_inode, h->xattr_name, + NULL, 0, 0); + else + set_bit(BTRFS_INODE_HAS_PROPS, &inode->runtime_flags); + } + + if (need_reserve) { + btrfs_block_rsv_release(fs_info, trans->block_rsv, + num_bytes, NULL); + if (ret) + return ret; + } + need_reserve = true; + } + + return 0; +} + +int __init btrfs_props_init(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(prop_handlers); i++) { + struct prop_handler *p = &prop_handlers[i]; + u64 h = btrfs_name_hash(p->xattr_name, strlen(p->xattr_name)); + + hash_add(prop_handlers_ht, &p->node, h); + } + return 0; +} + diff --git a/fs/btrfs/props.h b/fs/btrfs/props.h new file mode 100644 index 000000000..15d9a025c --- /dev/null +++ b/fs/btrfs/props.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2014 Filipe David Borba Manana + */ + +#ifndef BTRFS_PROPS_H +#define BTRFS_PROPS_H + +#include +#include + +struct btrfs_inode; +struct btrfs_path; +struct btrfs_trans_handle; + +int __init btrfs_props_init(void); + +int btrfs_set_prop(struct btrfs_trans_handle *trans, struct btrfs_inode *inode, + const char *name, const char *value, size_t value_len, + int flags); +int btrfs_validate_prop(const struct btrfs_inode *inode, const char *name, + const char *value, size_t value_len); +bool btrfs_ignore_prop(const struct btrfs_inode *inode, const char *name); + +int btrfs_load_inode_props(struct btrfs_inode *inode, struct btrfs_path *path); + +int btrfs_inode_inherit_props(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + const struct btrfs_inode *dir); + +#endif diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c new file mode 100644 index 000000000..f68b696b4 --- /dev/null +++ b/fs/btrfs/qgroup.c @@ -0,0 +1,4993 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2011 STRATO. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ctree.h" +#include "transaction.h" +#include "disk-io.h" +#include "locking.h" +#include "ulist.h" +#include "backref.h" +#include "extent_io.h" +#include "qgroup.h" +#include "block-group.h" +#include "sysfs.h" +#include "tree-mod-log.h" +#include "fs.h" +#include "accessors.h" +#include "extent-tree.h" +#include "root-tree.h" +#include "tree-checker.h" + +enum btrfs_qgroup_mode btrfs_qgroup_mode(const struct btrfs_fs_info *fs_info) +{ + if (!test_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags)) + return BTRFS_QGROUP_MODE_DISABLED; + if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_SIMPLE_MODE) + return BTRFS_QGROUP_MODE_SIMPLE; + return BTRFS_QGROUP_MODE_FULL; +} + +bool btrfs_qgroup_enabled(const struct btrfs_fs_info *fs_info) +{ + return btrfs_qgroup_mode(fs_info) != BTRFS_QGROUP_MODE_DISABLED; +} + +bool btrfs_qgroup_full_accounting(const struct btrfs_fs_info *fs_info) +{ + return btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_FULL; +} + +/* + * Helpers to access qgroup reservation + * + * Callers should ensure the lock context and type are valid + */ + +static u64 qgroup_rsv_total(const struct btrfs_qgroup *qgroup) +{ + u64 ret = 0; + int i; + + for (i = 0; i < BTRFS_QGROUP_RSV_LAST; i++) + ret += qgroup->rsv.values[i]; + + return ret; +} + +#ifdef CONFIG_BTRFS_DEBUG +static const char *qgroup_rsv_type_str(enum btrfs_qgroup_rsv_type type) +{ + if (type == BTRFS_QGROUP_RSV_DATA) + return "data"; + if (type == BTRFS_QGROUP_RSV_META_PERTRANS) + return "meta_pertrans"; + if (type == BTRFS_QGROUP_RSV_META_PREALLOC) + return "meta_prealloc"; + return NULL; +} +#endif + +static void qgroup_rsv_add(struct btrfs_fs_info *fs_info, + struct btrfs_qgroup *qgroup, u64 num_bytes, + enum btrfs_qgroup_rsv_type type) +{ + trace_btrfs_qgroup_update_reserve(fs_info, qgroup, num_bytes, type); + qgroup->rsv.values[type] += num_bytes; +} + +static void qgroup_rsv_release(struct btrfs_fs_info *fs_info, + struct btrfs_qgroup *qgroup, u64 num_bytes, + enum btrfs_qgroup_rsv_type type) +{ + trace_btrfs_qgroup_update_reserve(fs_info, qgroup, -(s64)num_bytes, type); + if (qgroup->rsv.values[type] >= num_bytes) { + qgroup->rsv.values[type] -= num_bytes; + return; + } +#ifdef CONFIG_BTRFS_DEBUG + WARN_RATELIMIT(1, + "qgroup %llu %s reserved space underflow, have %llu to free %llu", + qgroup->qgroupid, qgroup_rsv_type_str(type), + qgroup->rsv.values[type], num_bytes); +#endif + qgroup->rsv.values[type] = 0; +} + +static void qgroup_rsv_add_by_qgroup(struct btrfs_fs_info *fs_info, + struct btrfs_qgroup *dest, + const struct btrfs_qgroup *src) +{ + int i; + + for (i = 0; i < BTRFS_QGROUP_RSV_LAST; i++) + qgroup_rsv_add(fs_info, dest, src->rsv.values[i], i); +} + +static void qgroup_rsv_release_by_qgroup(struct btrfs_fs_info *fs_info, + struct btrfs_qgroup *dest, + const struct btrfs_qgroup *src) +{ + int i; + + for (i = 0; i < BTRFS_QGROUP_RSV_LAST; i++) + qgroup_rsv_release(fs_info, dest, src->rsv.values[i], i); +} + +static void btrfs_qgroup_update_old_refcnt(struct btrfs_qgroup *qg, u64 seq, + int mod) +{ + if (qg->old_refcnt < seq) + qg->old_refcnt = seq; + qg->old_refcnt += mod; +} + +static void btrfs_qgroup_update_new_refcnt(struct btrfs_qgroup *qg, u64 seq, + int mod) +{ + if (qg->new_refcnt < seq) + qg->new_refcnt = seq; + qg->new_refcnt += mod; +} + +static inline u64 btrfs_qgroup_get_old_refcnt(const struct btrfs_qgroup *qg, u64 seq) +{ + if (qg->old_refcnt < seq) + return 0; + return qg->old_refcnt - seq; +} + +static inline u64 btrfs_qgroup_get_new_refcnt(const struct btrfs_qgroup *qg, u64 seq) +{ + if (qg->new_refcnt < seq) + return 0; + return qg->new_refcnt - seq; +} + +static int +qgroup_rescan_init(struct btrfs_fs_info *fs_info, u64 progress_objectid, + int init_flags); +static void qgroup_rescan_zero_tracking(struct btrfs_fs_info *fs_info); + +static int btrfs_qgroup_qgroupid_key_cmp(const void *key, const struct rb_node *node) +{ + const u64 *qgroupid = key; + const struct btrfs_qgroup *qgroup = rb_entry(node, struct btrfs_qgroup, node); + + if (qgroup->qgroupid < *qgroupid) + return -1; + else if (qgroup->qgroupid > *qgroupid) + return 1; + + return 0; +} + +/* must be called with qgroup_ioctl_lock held */ +static struct btrfs_qgroup *find_qgroup_rb(const struct btrfs_fs_info *fs_info, + u64 qgroupid) +{ + struct rb_node *node; + + node = rb_find(&qgroupid, &fs_info->qgroup_tree, btrfs_qgroup_qgroupid_key_cmp); + return rb_entry_safe(node, struct btrfs_qgroup, node); +} + +static int btrfs_qgroup_qgroupid_cmp(struct rb_node *new, const struct rb_node *existing) +{ + const struct btrfs_qgroup *new_qgroup = rb_entry(new, struct btrfs_qgroup, node); + + return btrfs_qgroup_qgroupid_key_cmp(&new_qgroup->qgroupid, existing); +} + +/* + * Add qgroup to the filesystem's qgroup tree. + * + * Must be called with qgroup_lock held and @prealloc preallocated. + * + * The control on the lifespan of @prealloc would be transferred to this + * function, thus caller should no longer touch @prealloc. + */ +static struct btrfs_qgroup *add_qgroup_rb(struct btrfs_fs_info *fs_info, + struct btrfs_qgroup *prealloc, + u64 qgroupid) +{ + struct rb_node *node; + + /* Caller must have pre-allocated @prealloc. */ + ASSERT(prealloc); + + prealloc->qgroupid = qgroupid; + node = rb_find_add(&prealloc->node, &fs_info->qgroup_tree, btrfs_qgroup_qgroupid_cmp); + if (node) { + kfree(prealloc); + return rb_entry(node, struct btrfs_qgroup, node); + } + + INIT_LIST_HEAD(&prealloc->groups); + INIT_LIST_HEAD(&prealloc->members); + INIT_LIST_HEAD(&prealloc->dirty); + INIT_LIST_HEAD(&prealloc->iterator); + INIT_LIST_HEAD(&prealloc->nested_iterator); + + return prealloc; +} + +static void __del_qgroup_rb(struct btrfs_qgroup *qgroup) +{ + struct btrfs_qgroup_list *list; + + list_del(&qgroup->dirty); + while (!list_empty(&qgroup->groups)) { + list = list_first_entry(&qgroup->groups, + struct btrfs_qgroup_list, next_group); + list_del(&list->next_group); + list_del(&list->next_member); + kfree(list); + } + + while (!list_empty(&qgroup->members)) { + list = list_first_entry(&qgroup->members, + struct btrfs_qgroup_list, next_member); + list_del(&list->next_group); + list_del(&list->next_member); + kfree(list); + } +} + +/* must be called with qgroup_lock held */ +static int del_qgroup_rb(struct btrfs_fs_info *fs_info, u64 qgroupid) +{ + struct btrfs_qgroup *qgroup = find_qgroup_rb(fs_info, qgroupid); + + if (!qgroup) + return -ENOENT; + + rb_erase(&qgroup->node, &fs_info->qgroup_tree); + __del_qgroup_rb(qgroup); + return 0; +} + +/* + * Add relation specified by two qgroups. + * + * Must be called with qgroup_lock held, the ownership of @prealloc is + * transferred to this function and caller should not touch it anymore. + * + * Return: 0 on success + * -ENOENT if one of the qgroups is NULL + * <0 other errors + */ +static int __add_relation_rb(struct btrfs_qgroup_list *prealloc, + struct btrfs_qgroup *member, + struct btrfs_qgroup *parent) +{ + if (!member || !parent) { + kfree(prealloc); + return -ENOENT; + } + + prealloc->group = parent; + prealloc->member = member; + list_add_tail(&prealloc->next_group, &member->groups); + list_add_tail(&prealloc->next_member, &parent->members); + + return 0; +} + +/* + * Add relation specified by two qgroup ids. + * + * Must be called with qgroup_lock held. + * + * Return: 0 on success + * -ENOENT if one of the ids does not exist + * <0 other errors + */ +static int add_relation_rb(struct btrfs_fs_info *fs_info, + struct btrfs_qgroup_list *prealloc, + u64 memberid, u64 parentid) +{ + struct btrfs_qgroup *member; + struct btrfs_qgroup *parent; + + member = find_qgroup_rb(fs_info, memberid); + parent = find_qgroup_rb(fs_info, parentid); + + return __add_relation_rb(prealloc, member, parent); +} + +/* Must be called with qgroup_lock held */ +static int del_relation_rb(struct btrfs_fs_info *fs_info, + u64 memberid, u64 parentid) +{ + struct btrfs_qgroup *member; + struct btrfs_qgroup *parent; + struct btrfs_qgroup_list *list; + + member = find_qgroup_rb(fs_info, memberid); + parent = find_qgroup_rb(fs_info, parentid); + if (!member || !parent) + return -ENOENT; + + list_for_each_entry(list, &member->groups, next_group) { + if (list->group == parent) { + list_del(&list->next_group); + list_del(&list->next_member); + kfree(list); + return 0; + } + } + return -ENOENT; +} + +#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS +int btrfs_verify_qgroup_counts(const struct btrfs_fs_info *fs_info, u64 qgroupid, + u64 rfer, u64 excl) +{ + struct btrfs_qgroup *qgroup; + + qgroup = find_qgroup_rb(fs_info, qgroupid); + if (!qgroup) + return -EINVAL; + if (qgroup->rfer != rfer || qgroup->excl != excl) + return -EINVAL; + return 0; +} +#endif + +static bool squota_check_parent_usage(struct btrfs_fs_info *fs_info, struct btrfs_qgroup *parent) +{ + u64 excl_sum = 0; + u64 rfer_sum = 0; + u64 excl_cmpr_sum = 0; + u64 rfer_cmpr_sum = 0; + struct btrfs_qgroup_list *glist; + int nr_members = 0; + bool mismatch; + + if (btrfs_qgroup_mode(fs_info) != BTRFS_QGROUP_MODE_SIMPLE) + return false; + if (btrfs_qgroup_level(parent->qgroupid) == 0) + return false; + + /* Eligible parent qgroup. Squota; level > 0; empty members list. */ + list_for_each_entry(glist, &parent->members, next_member) { + excl_sum += glist->member->excl; + rfer_sum += glist->member->rfer; + excl_cmpr_sum += glist->member->excl_cmpr; + rfer_cmpr_sum += glist->member->rfer_cmpr; + nr_members++; + } + mismatch = (parent->excl != excl_sum || parent->rfer != rfer_sum || + parent->excl_cmpr != excl_cmpr_sum || parent->rfer_cmpr != rfer_cmpr_sum); + + WARN(mismatch, + "parent squota qgroup " BTRFS_QGROUP_FMT " has mismatched usage from its %d members. " + "%llu %llu %llu %llu vs %llu %llu %llu %llu\n", + BTRFS_QGROUP_FMT_VALUE(parent), nr_members, parent->excl, + parent->rfer, parent->excl_cmpr, parent->rfer_cmpr, excl_sum, + rfer_sum, excl_cmpr_sum, rfer_cmpr_sum); + return mismatch; +} + +__printf(2, 3) +static void qgroup_mark_inconsistent(struct btrfs_fs_info *fs_info, const char *fmt, ...) +{ + const u64 old_flags = fs_info->qgroup_flags; + + if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE) + return; + fs_info->qgroup_flags |= (BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT | + BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN | + BTRFS_QGROUP_RUNTIME_FLAG_NO_ACCOUNTING); + if (!(old_flags & BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT)) { + struct va_format vaf; + va_list args; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + + btrfs_warn_rl(fs_info, "qgroup marked inconsistent, %pV", &vaf); + va_end(args); + } +} + +static void qgroup_read_enable_gen(struct btrfs_fs_info *fs_info, + struct extent_buffer *leaf, int slot, + struct btrfs_qgroup_status_item *ptr) +{ + ASSERT(btrfs_fs_incompat(fs_info, SIMPLE_QUOTA)); + ASSERT(btrfs_item_size(leaf, slot) >= sizeof(*ptr)); + fs_info->qgroup_enable_gen = btrfs_qgroup_status_enable_gen(leaf, ptr); +} + +/* + * The full config is read in one go, only called from open_ctree() + * It doesn't use any locking, as at this point we're still single-threaded + */ +int btrfs_read_qgroup_config(struct btrfs_fs_info *fs_info) +{ + struct btrfs_key key; + struct btrfs_key found_key; + struct btrfs_root *quota_root = fs_info->quota_root; + struct btrfs_path *path = NULL; + struct extent_buffer *l; + int slot; + int ret = 0; + u64 flags = 0; + u64 rescan_progress = 0; + + if (!fs_info->quota_root) + return 0; + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto out; + } + + ret = btrfs_sysfs_add_qgroups(fs_info); + if (ret < 0) + goto out; + /* default this to quota off, in case no status key is found */ + fs_info->qgroup_flags = 0; + + /* + * pass 1: read status, all qgroup infos and limits + */ + key.objectid = 0; + key.type = 0; + key.offset = 0; + ret = btrfs_search_slot_for_read(quota_root, &key, path, 1, 1); + if (ret) + goto out; + + while (1) { + struct btrfs_qgroup *qgroup; + + slot = path->slots[0]; + l = path->nodes[0]; + btrfs_item_key_to_cpu(l, &found_key, slot); + + if (found_key.type == BTRFS_QGROUP_STATUS_KEY) { + struct btrfs_qgroup_status_item *ptr; + + ptr = btrfs_item_ptr(l, slot, + struct btrfs_qgroup_status_item); + + if (btrfs_qgroup_status_version(l, ptr) != + BTRFS_QGROUP_STATUS_VERSION) { + btrfs_err(fs_info, + "old qgroup version, quota disabled"); + goto out; + } + fs_info->qgroup_flags = btrfs_qgroup_status_flags(l, ptr); + if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_SIMPLE_MODE) + qgroup_read_enable_gen(fs_info, l, slot, ptr); + else if (btrfs_qgroup_status_generation(l, ptr) != fs_info->generation) + qgroup_mark_inconsistent(fs_info, "qgroup generation mismatch"); + rescan_progress = btrfs_qgroup_status_rescan(l, ptr); + goto next1; + } + + if (found_key.type != BTRFS_QGROUP_INFO_KEY && + found_key.type != BTRFS_QGROUP_LIMIT_KEY) + goto next1; + + qgroup = find_qgroup_rb(fs_info, found_key.offset); + if ((qgroup && found_key.type == BTRFS_QGROUP_INFO_KEY) || + (!qgroup && found_key.type == BTRFS_QGROUP_LIMIT_KEY)) + qgroup_mark_inconsistent(fs_info, "inconsistent qgroup config"); + if (!qgroup) { + struct btrfs_qgroup *prealloc; + struct btrfs_root *tree_root = fs_info->tree_root; + + prealloc = kzalloc_obj(*prealloc); + if (!prealloc) { + ret = -ENOMEM; + goto out; + } + qgroup = add_qgroup_rb(fs_info, prealloc, found_key.offset); + /* + * If a qgroup exists for a subvolume ID, it is possible + * that subvolume has been deleted, in which case + * reusing that ID would lead to incorrect accounting. + * + * Ensure that we skip any such subvol ids. + * + * We don't need to lock because this is only called + * during mount before we start doing things like creating + * subvolumes. + */ + if (btrfs_is_fstree(qgroup->qgroupid) && + qgroup->qgroupid > tree_root->free_objectid) + /* + * Don't need to check against BTRFS_LAST_FREE_OBJECTID, + * as it will get checked on the next call to + * btrfs_get_free_objectid. + */ + tree_root->free_objectid = qgroup->qgroupid + 1; + } + ret = btrfs_sysfs_add_one_qgroup(fs_info, qgroup); + if (ret < 0) + goto out; + + switch (found_key.type) { + case BTRFS_QGROUP_INFO_KEY: { + struct btrfs_qgroup_info_item *ptr; + + ptr = btrfs_item_ptr(l, slot, + struct btrfs_qgroup_info_item); + qgroup->rfer = btrfs_qgroup_info_rfer(l, ptr); + qgroup->rfer_cmpr = btrfs_qgroup_info_rfer_cmpr(l, ptr); + qgroup->excl = btrfs_qgroup_info_excl(l, ptr); + qgroup->excl_cmpr = btrfs_qgroup_info_excl_cmpr(l, ptr); + /* generation currently unused */ + break; + } + case BTRFS_QGROUP_LIMIT_KEY: { + struct btrfs_qgroup_limit_item *ptr; + + ptr = btrfs_item_ptr(l, slot, + struct btrfs_qgroup_limit_item); + qgroup->lim_flags = btrfs_qgroup_limit_flags(l, ptr); + qgroup->max_rfer = btrfs_qgroup_limit_max_rfer(l, ptr); + qgroup->max_excl = btrfs_qgroup_limit_max_excl(l, ptr); + qgroup->rsv_rfer = btrfs_qgroup_limit_rsv_rfer(l, ptr); + qgroup->rsv_excl = btrfs_qgroup_limit_rsv_excl(l, ptr); + break; + } + } +next1: + ret = btrfs_next_item(quota_root, path); + if (ret < 0) + goto out; + if (ret) + break; + } + btrfs_release_path(path); + + /* + * pass 2: read all qgroup relations + */ + key.objectid = 0; + key.type = BTRFS_QGROUP_RELATION_KEY; + key.offset = 0; + ret = btrfs_search_slot_for_read(quota_root, &key, path, 1, 0); + if (ret) + goto out; + while (1) { + struct btrfs_qgroup_list *list = NULL; + + slot = path->slots[0]; + l = path->nodes[0]; + btrfs_item_key_to_cpu(l, &found_key, slot); + + if (found_key.type != BTRFS_QGROUP_RELATION_KEY) + goto next2; + + if (found_key.objectid > found_key.offset) { + /* parent <- member, not needed to build config */ + /* FIXME should we omit the key completely? */ + goto next2; + } + + list = kzalloc_obj(*list); + if (!list) { + ret = -ENOMEM; + goto out; + } + ret = add_relation_rb(fs_info, list, found_key.objectid, + found_key.offset); + list = NULL; + if (ret == -ENOENT) { + btrfs_warn(fs_info, + "orphan qgroup relation 0x%llx->0x%llx", + found_key.objectid, found_key.offset); + ret = 0; /* ignore the error */ + } + if (ret) + goto out; +next2: + ret = btrfs_next_item(quota_root, path); + if (ret < 0) + goto out; + if (ret) + break; + } +out: + btrfs_free_path(path); + fs_info->qgroup_flags |= flags; + if (ret >= 0) { + if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_ON) + set_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags); + if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_RESCAN) + ret = qgroup_rescan_init(fs_info, rescan_progress, 0); + } else { + fs_info->qgroup_flags &= ~BTRFS_QGROUP_STATUS_FLAG_RESCAN; + btrfs_sysfs_del_qgroups(fs_info); + } + + return ret < 0 ? ret : 0; +} + +/* + * Called in close_ctree() when quota is still enabled. This verifies we don't + * leak some reserved space. + * + * Return false if no reserved space is left. + * Return true if some reserved space is leaked. + */ +bool btrfs_check_quota_leak(const struct btrfs_fs_info *fs_info) +{ + struct rb_node *node; + bool ret = false; + + if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_DISABLED) + return ret; + /* + * Since we're unmounting, there is no race and no need to grab qgroup + * lock. And here we don't go post-order to provide a more user + * friendly sorted result. + */ + for (node = rb_first(&fs_info->qgroup_tree); node; node = rb_next(node)) { + struct btrfs_qgroup *qgroup; + int i; + + qgroup = rb_entry(node, struct btrfs_qgroup, node); + for (i = 0; i < BTRFS_QGROUP_RSV_LAST; i++) { + if (qgroup->rsv.values[i]) { + ret = true; + btrfs_warn(fs_info, + "qgroup " BTRFS_QGROUP_FMT " has unreleased space, type %d rsv %llu", + BTRFS_QGROUP_FMT_VALUE(qgroup), + i, qgroup->rsv.values[i]); + } + } + } + return ret; +} + +/* + * This is called from close_ctree() or open_ctree() or btrfs_quota_disable(), + * first two are in single-threaded paths. + */ +void btrfs_free_qgroup_config(struct btrfs_fs_info *fs_info) +{ + struct rb_node *n; + struct btrfs_qgroup *qgroup; + + /* + * btrfs_quota_disable() can be called concurrently with + * btrfs_qgroup_rescan() -> qgroup_rescan_zero_tracking(), so take the + * lock. + */ + spin_lock(&fs_info->qgroup_lock); + while ((n = rb_first(&fs_info->qgroup_tree))) { + qgroup = rb_entry(n, struct btrfs_qgroup, node); + rb_erase(n, &fs_info->qgroup_tree); + __del_qgroup_rb(qgroup); + spin_unlock(&fs_info->qgroup_lock); + btrfs_sysfs_del_one_qgroup(fs_info, qgroup); + kfree(qgroup); + spin_lock(&fs_info->qgroup_lock); + } + spin_unlock(&fs_info->qgroup_lock); + + btrfs_sysfs_del_qgroups(fs_info); +} + +static int add_qgroup_relation_item(struct btrfs_trans_handle *trans, u64 src, + u64 dst) +{ + struct btrfs_root *quota_root = trans->fs_info->quota_root; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = src; + key.type = BTRFS_QGROUP_RELATION_KEY; + key.offset = dst; + + return btrfs_insert_empty_item(trans, quota_root, path, &key, 0); +} + +static int del_qgroup_relation_item(struct btrfs_trans_handle *trans, u64 src, + u64 dst) +{ + int ret; + struct btrfs_root *quota_root = trans->fs_info->quota_root; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = src; + key.type = BTRFS_QGROUP_RELATION_KEY; + key.offset = dst; + + ret = btrfs_search_slot(trans, quota_root, &key, path, -1, 1); + if (ret < 0) + return ret; + + if (ret > 0) + return -ENOENT; + + return btrfs_del_item(trans, quota_root, path); +} + +static int add_qgroup_item(struct btrfs_trans_handle *trans, + struct btrfs_root *quota_root, u64 qgroupid) +{ + int ret; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_qgroup_info_item *qgroup_info; + struct btrfs_qgroup_limit_item *qgroup_limit; + struct extent_buffer *leaf; + struct btrfs_key key; + + if (btrfs_is_testing(quota_root->fs_info)) + return 0; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = 0; + key.type = BTRFS_QGROUP_INFO_KEY; + key.offset = qgroupid; + + /* + * Avoid a transaction abort by catching -EEXIST here. In that + * case, we proceed by re-initializing the existing structure + * on disk. + */ + + ret = btrfs_insert_empty_item(trans, quota_root, path, &key, + sizeof(*qgroup_info)); + if (ret && ret != -EEXIST) + return ret; + + leaf = path->nodes[0]; + qgroup_info = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_qgroup_info_item); + btrfs_set_qgroup_info_generation(leaf, qgroup_info, trans->transid); + btrfs_set_qgroup_info_rfer(leaf, qgroup_info, 0); + btrfs_set_qgroup_info_rfer_cmpr(leaf, qgroup_info, 0); + btrfs_set_qgroup_info_excl(leaf, qgroup_info, 0); + btrfs_set_qgroup_info_excl_cmpr(leaf, qgroup_info, 0); + + btrfs_release_path(path); + + key.type = BTRFS_QGROUP_LIMIT_KEY; + ret = btrfs_insert_empty_item(trans, quota_root, path, &key, + sizeof(*qgroup_limit)); + if (ret && ret != -EEXIST) + return ret; + + leaf = path->nodes[0]; + qgroup_limit = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_qgroup_limit_item); + btrfs_set_qgroup_limit_flags(leaf, qgroup_limit, 0); + btrfs_set_qgroup_limit_max_rfer(leaf, qgroup_limit, 0); + btrfs_set_qgroup_limit_max_excl(leaf, qgroup_limit, 0); + btrfs_set_qgroup_limit_rsv_rfer(leaf, qgroup_limit, 0); + btrfs_set_qgroup_limit_rsv_excl(leaf, qgroup_limit, 0); + + return 0; +} + +static int del_qgroup_item(struct btrfs_trans_handle *trans, u64 qgroupid) +{ + int ret; + struct btrfs_root *quota_root = trans->fs_info->quota_root; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = 0; + key.type = BTRFS_QGROUP_INFO_KEY; + key.offset = qgroupid; + ret = btrfs_search_slot(trans, quota_root, &key, path, -1, 1); + if (ret < 0) + return ret; + + if (ret > 0) + return -ENOENT; + + ret = btrfs_del_item(trans, quota_root, path); + if (ret) + return ret; + + btrfs_release_path(path); + + key.type = BTRFS_QGROUP_LIMIT_KEY; + ret = btrfs_search_slot(trans, quota_root, &key, path, -1, 1); + if (ret < 0) + return ret; + + if (ret > 0) + return -ENOENT; + + return btrfs_del_item(trans, quota_root, path); +} + +static int update_qgroup_limit_item(struct btrfs_trans_handle *trans, + struct btrfs_qgroup *qgroup) +{ + struct btrfs_root *quota_root = trans->fs_info->quota_root; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + struct extent_buffer *l; + struct btrfs_qgroup_limit_item *qgroup_limit; + int ret; + int slot; + + key.objectid = 0; + key.type = BTRFS_QGROUP_LIMIT_KEY; + key.offset = qgroup->qgroupid; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + ret = btrfs_search_slot(trans, quota_root, &key, path, 0, 1); + if (ret > 0) + ret = -ENOENT; + + if (ret) + return ret; + + l = path->nodes[0]; + slot = path->slots[0]; + qgroup_limit = btrfs_item_ptr(l, slot, struct btrfs_qgroup_limit_item); + btrfs_set_qgroup_limit_flags(l, qgroup_limit, qgroup->lim_flags); + btrfs_set_qgroup_limit_max_rfer(l, qgroup_limit, qgroup->max_rfer); + btrfs_set_qgroup_limit_max_excl(l, qgroup_limit, qgroup->max_excl); + btrfs_set_qgroup_limit_rsv_rfer(l, qgroup_limit, qgroup->rsv_rfer); + btrfs_set_qgroup_limit_rsv_excl(l, qgroup_limit, qgroup->rsv_excl); + + return ret; +} + +static int update_qgroup_info_item(struct btrfs_trans_handle *trans, + struct btrfs_qgroup *qgroup) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *quota_root = fs_info->quota_root; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + struct extent_buffer *l; + struct btrfs_qgroup_info_item *qgroup_info; + int ret; + int slot; + + if (btrfs_is_testing(fs_info)) + return 0; + + key.objectid = 0; + key.type = BTRFS_QGROUP_INFO_KEY; + key.offset = qgroup->qgroupid; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + ret = btrfs_search_slot(trans, quota_root, &key, path, 0, 1); + if (ret > 0) + ret = -ENOENT; + + if (ret) + return ret; + + l = path->nodes[0]; + slot = path->slots[0]; + qgroup_info = btrfs_item_ptr(l, slot, struct btrfs_qgroup_info_item); + btrfs_set_qgroup_info_generation(l, qgroup_info, trans->transid); + btrfs_set_qgroup_info_rfer(l, qgroup_info, qgroup->rfer); + btrfs_set_qgroup_info_rfer_cmpr(l, qgroup_info, qgroup->rfer_cmpr); + btrfs_set_qgroup_info_excl(l, qgroup_info, qgroup->excl); + btrfs_set_qgroup_info_excl_cmpr(l, qgroup_info, qgroup->excl_cmpr); + + return ret; +} + +static int update_qgroup_status_item(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *quota_root = fs_info->quota_root; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + struct extent_buffer *l; + struct btrfs_qgroup_status_item *ptr; + int ret; + int slot; + + key.objectid = 0; + key.type = BTRFS_QGROUP_STATUS_KEY; + key.offset = 0; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + ret = btrfs_search_slot(trans, quota_root, &key, path, 0, 1); + if (ret > 0) + ret = -ENOENT; + + if (ret) + return ret; + + l = path->nodes[0]; + slot = path->slots[0]; + ptr = btrfs_item_ptr(l, slot, struct btrfs_qgroup_status_item); + btrfs_set_qgroup_status_flags(l, ptr, fs_info->qgroup_flags & + BTRFS_QGROUP_STATUS_FLAGS_MASK); + btrfs_set_qgroup_status_generation(l, ptr, trans->transid); + btrfs_set_qgroup_status_rescan(l, ptr, + fs_info->qgroup_rescan_progress.objectid); + + return ret; +} + +/* + * called with qgroup_lock held + */ +static int btrfs_clean_quota_tree(struct btrfs_trans_handle *trans, + struct btrfs_root *root) +{ + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + struct extent_buffer *leaf = NULL; + int ret; + int nr = 0; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = 0; + key.type = 0; + key.offset = 0; + + while (1) { + ret = btrfs_search_slot(trans, root, &key, path, -1, 1); + if (ret < 0) + return ret; + leaf = path->nodes[0]; + nr = btrfs_header_nritems(leaf); + if (!nr) + break; + /* + * delete the leaf one by one + * since the whole tree is going + * to be deleted. + */ + path->slots[0] = 0; + ret = btrfs_del_items(trans, root, path, 0, nr); + if (ret) + return ret; + + btrfs_release_path(path); + } + + return 0; +} + +int btrfs_quota_enable(struct btrfs_fs_info *fs_info, + struct btrfs_ioctl_quota_ctl_args *quota_ctl_args) +{ + struct btrfs_root *quota_root; + struct btrfs_root *tree_root = fs_info->tree_root; + struct btrfs_path *path = NULL; + struct btrfs_qgroup_status_item *ptr; + struct extent_buffer *leaf; + struct btrfs_key key; + struct btrfs_key found_key; + struct btrfs_qgroup *qgroup = NULL; + struct btrfs_qgroup *prealloc = NULL; + struct btrfs_trans_handle *trans = NULL; + const bool simple = (quota_ctl_args->cmd == BTRFS_QUOTA_CTL_ENABLE_SIMPLE_QUOTA); + int ret = 0; + int slot; + + /* + * We need to have subvol_sem write locked, to prevent races between + * concurrent tasks trying to enable quotas, because we will unlock + * and relock qgroup_ioctl_lock before setting fs_info->quota_root + * and before setting BTRFS_FS_QUOTA_ENABLED. + */ + lockdep_assert_held_write(&fs_info->subvol_sem); + + if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) { + btrfs_err(fs_info, + "qgroups are currently unsupported in extent tree v2"); + return -EINVAL; + } + + mutex_lock(&fs_info->qgroup_ioctl_lock); + if (fs_info->quota_root) + goto out; + + ret = btrfs_sysfs_add_qgroups(fs_info); + if (ret < 0) + goto out; + + /* + * Unlock qgroup_ioctl_lock before starting the transaction. This is to + * avoid lock acquisition inversion problems (reported by lockdep) between + * qgroup_ioctl_lock and the vfs freeze semaphores, acquired when we + * start a transaction. + * After we started the transaction lock qgroup_ioctl_lock again and + * check if someone else created the quota root in the meanwhile. If so, + * just return success and release the transaction handle. + * + * Also we don't need to worry about someone else calling + * btrfs_sysfs_add_qgroups() after we unlock and getting an error because + * that function returns 0 (success) when the sysfs entries already exist. + */ + mutex_unlock(&fs_info->qgroup_ioctl_lock); + + /* + * 1 for quota root item + * 1 for BTRFS_QGROUP_STATUS item + * + * Yet we also need 2*n items for a QGROUP_INFO/QGROUP_LIMIT items + * per subvolume. However those are not currently reserved since it + * would be a lot of overkill. + */ + trans = btrfs_start_transaction(tree_root, 2); + + mutex_lock(&fs_info->qgroup_ioctl_lock); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + trans = NULL; + goto out; + } + + if (fs_info->quota_root) + goto out; + + /* + * initially create the quota tree + */ + quota_root = btrfs_create_tree(trans, BTRFS_QUOTA_TREE_OBJECTID); + if (IS_ERR(quota_root)) { + ret = PTR_ERR(quota_root); + btrfs_abort_transaction(trans, ret); + goto out; + } + + path = btrfs_alloc_path(); + if (unlikely(!path)) { + ret = -ENOMEM; + btrfs_abort_transaction(trans, ret); + goto out_free_root; + } + + key.objectid = 0; + key.type = BTRFS_QGROUP_STATUS_KEY; + key.offset = 0; + + ret = btrfs_insert_empty_item(trans, quota_root, path, &key, + sizeof(*ptr)); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out_free_path; + } + + leaf = path->nodes[0]; + ptr = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_qgroup_status_item); + btrfs_set_qgroup_status_generation(leaf, ptr, trans->transid); + btrfs_set_qgroup_status_version(leaf, ptr, BTRFS_QGROUP_STATUS_VERSION); + fs_info->qgroup_flags = BTRFS_QGROUP_STATUS_FLAG_ON; + if (simple) { + fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_SIMPLE_MODE; + btrfs_set_fs_incompat(fs_info, SIMPLE_QUOTA); + /* + * Set the enable generation to the next transaction, as we cannot + * ensure that extents written during this transaction will see any + * state we have set here. So we should treat all extents of the + * transaction as coming in before squotas was enabled. + */ + btrfs_set_qgroup_status_enable_gen(leaf, ptr, trans->transid + 1); + } else { + fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT; + } + btrfs_set_qgroup_status_flags(leaf, ptr, fs_info->qgroup_flags & + BTRFS_QGROUP_STATUS_FLAGS_MASK); + btrfs_set_qgroup_status_rescan(leaf, ptr, 0); + + key.objectid = 0; + key.type = BTRFS_ROOT_REF_KEY; + key.offset = 0; + + btrfs_release_path(path); + ret = btrfs_search_slot_for_read(tree_root, &key, path, 1, 0); + if (ret > 0) + goto out_add_root; + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + goto out_free_path; + } + + while (1) { + slot = path->slots[0]; + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &found_key, slot); + + if (found_key.type == BTRFS_ROOT_REF_KEY) { + + /* Release locks on tree_root before we access quota_root */ + btrfs_release_path(path); + + /* We should not have a stray @prealloc pointer. */ + ASSERT(prealloc == NULL); + prealloc = kzalloc_obj(*prealloc, GFP_NOFS); + if (unlikely(!prealloc)) { + ret = -ENOMEM; + btrfs_abort_transaction(trans, ret); + goto out_free_path; + } + + ret = add_qgroup_item(trans, quota_root, + found_key.offset); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out_free_path; + } + + qgroup = add_qgroup_rb(fs_info, prealloc, found_key.offset); + prealloc = NULL; + ret = btrfs_sysfs_add_one_qgroup(fs_info, qgroup); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + goto out_free_path; + } + ret = btrfs_search_slot_for_read(tree_root, &found_key, + path, 1, 0); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + goto out_free_path; + } + if (ret > 0) { + /* + * Shouldn't happen because the key should still + * be there (return 0), but in case it does it + * means we have reached the end of the tree - + * there are no more leaves with items that have + * a key greater than or equals to @found_key, + * so just stop the search loop. + */ + break; + } + } + ret = btrfs_next_item(tree_root, path); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + goto out_free_path; + } + if (ret) + break; + } + +out_add_root: + btrfs_release_path(path); + ret = add_qgroup_item(trans, quota_root, BTRFS_FS_TREE_OBJECTID); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out_free_path; + } + + ASSERT(prealloc == NULL); + prealloc = kzalloc_obj(*prealloc, GFP_NOFS); + if (!prealloc) { + ret = -ENOMEM; + goto out_free_path; + } + qgroup = add_qgroup_rb(fs_info, prealloc, BTRFS_FS_TREE_OBJECTID); + prealloc = NULL; + ret = btrfs_sysfs_add_one_qgroup(fs_info, qgroup); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + goto out_free_path; + } + + /* + * Set fs_info->qgroup_enable_gen and BTRFS_FS_SQUOTA_ENABLING + * under the transaction handle. We want to ensure that all extents in + * the next transaction definitely see them. + */ + if (simple) { + fs_info->qgroup_enable_gen = trans->transid + 1; + set_bit(BTRFS_FS_SQUOTA_ENABLING, &fs_info->flags); + } + + mutex_unlock(&fs_info->qgroup_ioctl_lock); + /* + * Commit the transaction while not holding qgroup_ioctl_lock, to avoid + * a deadlock with tasks concurrently doing other qgroup operations, such + * adding/removing qgroups or adding/deleting qgroup relations for example, + * because all qgroup operations first start or join a transaction and then + * lock the qgroup_ioctl_lock mutex. + * We are safe from a concurrent task trying to enable quotas, by calling + * this function, since we are serialized by fs_info->subvol_sem. + */ + ret = btrfs_commit_transaction(trans); + trans = NULL; + + mutex_lock(&fs_info->qgroup_ioctl_lock); + if (ret) { + if (simple) { + clear_bit(BTRFS_FS_SQUOTA_ENABLING, &fs_info->flags); + fs_info->qgroup_enable_gen = 0; + } + goto out_free_path; + } + + /* + * Set quota enabled flag after committing the transaction, to avoid + * deadlocks on fs_info->qgroup_ioctl_lock with concurrent snapshot + * creation. + */ + spin_lock(&fs_info->qgroup_lock); + fs_info->quota_root = quota_root; + set_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags); + if (simple) + clear_bit(BTRFS_FS_SQUOTA_ENABLING, &fs_info->flags); + spin_unlock(&fs_info->qgroup_lock); + + /* Skip rescan for simple qgroups. */ + if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE) + goto out_free_path; + + ret = qgroup_rescan_init(fs_info, 0, 1); + if (!ret) { + qgroup_rescan_zero_tracking(fs_info); + fs_info->qgroup_rescan_running = true; + btrfs_queue_work(fs_info->qgroup_rescan_workers, + &fs_info->qgroup_rescan_work); + } else { + /* + * We have set both BTRFS_FS_QUOTA_ENABLED and + * BTRFS_QGROUP_STATUS_FLAG_ON, so we can only fail with + * -EINPROGRESS. That can happen because someone started the + * rescan worker by calling quota rescan ioctl before we + * attempted to initialize the rescan worker. Failure due to + * quotas disabled in the meanwhile is not possible, because + * we are holding a write lock on fs_info->subvol_sem, which + * is also acquired when disabling quotas. + * Ignore such error, and any other error would need to undo + * everything we did in the transaction we just committed. + */ + ASSERT(ret == -EINPROGRESS); + ret = 0; + } + +out_free_path: + btrfs_free_path(path); +out_free_root: + if (ret) + btrfs_put_root(quota_root); +out: + if (ret) + btrfs_sysfs_del_qgroups(fs_info); + mutex_unlock(&fs_info->qgroup_ioctl_lock); + if (ret && trans) + btrfs_end_transaction(trans); + else if (trans) + ret = btrfs_end_transaction(trans); + kfree(prealloc); + return ret; +} + +/* + * It is possible to have outstanding ordered extents which reserved bytes + * before we disabled. We need to fully flush delalloc, ordered extents, and a + * commit to ensure that we don't leak such reservations, only to have them + * come back if we re-enable. + * + * - enable simple quotas + * - reserve space + * - release it, store rsv_bytes in OE + * - disable quotas + * - enable simple quotas (qgroup rsv are all 0) + * - OE finishes + * - run delayed refs + * - free rsv_bytes, resulting in miscounting or even underflow + */ +static int flush_reservations(struct btrfs_fs_info *fs_info) +{ + int ret; + + ret = btrfs_start_delalloc_roots(fs_info, LONG_MAX, false); + if (ret) + return ret; + btrfs_wait_ordered_roots(fs_info, U64_MAX, NULL); + + return btrfs_commit_current_transaction(fs_info->tree_root); +} + +int btrfs_quota_disable(struct btrfs_fs_info *fs_info) +{ + struct btrfs_root *quota_root = NULL; + struct btrfs_trans_handle *trans = NULL; + int ret = 0; + + /* + * We need to have subvol_sem write locked to prevent races with + * snapshot creation. + */ + lockdep_assert_held_write(&fs_info->subvol_sem); + + /* + * Relocation will mess with backrefs, so make sure we have the + * cleaner_mutex held to protect us from relocate. + */ + lockdep_assert_held(&fs_info->cleaner_mutex); + + mutex_lock(&fs_info->qgroup_ioctl_lock); + if (!fs_info->quota_root) + goto out; + + /* + * Unlock the qgroup_ioctl_lock mutex before waiting for the rescan worker to + * complete. Otherwise we can deadlock because btrfs_remove_qgroup() needs + * to lock that mutex while holding a transaction handle and the rescan + * worker needs to commit a transaction. + */ + mutex_unlock(&fs_info->qgroup_ioctl_lock); + + /* + * Request qgroup rescan worker to complete and wait for it. This wait + * must be done before transaction start for quota disable since it may + * deadlock with transaction by the qgroup rescan worker. + */ + clear_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags); + btrfs_qgroup_wait_for_completion(fs_info, false); + + /* + * We have nothing held here and no trans handle, just return the error + * if there is one and set back the quota enabled bit since we didn't + * actually disable quotas. + */ + ret = flush_reservations(fs_info); + if (ret) { + set_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags); + return ret; + } + + /* + * 1 For the root item + * + * We should also reserve enough items for the quota tree deletion in + * btrfs_clean_quota_tree but this is not done. + * + * Also, we must always start a transaction without holding the mutex + * qgroup_ioctl_lock, see btrfs_quota_enable(). + */ + trans = btrfs_start_transaction(fs_info->tree_root, 1); + + mutex_lock(&fs_info->qgroup_ioctl_lock); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + trans = NULL; + set_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags); + goto out; + } + + if (!fs_info->quota_root) + goto out; + + spin_lock(&fs_info->qgroup_lock); + quota_root = fs_info->quota_root; + fs_info->quota_root = NULL; + fs_info->qgroup_flags &= ~BTRFS_QGROUP_STATUS_FLAG_ON; + fs_info->qgroup_flags &= ~BTRFS_QGROUP_STATUS_FLAG_SIMPLE_MODE; + fs_info->qgroup_drop_subtree_thres = BTRFS_QGROUP_DROP_SUBTREE_THRES_DEFAULT; + spin_unlock(&fs_info->qgroup_lock); + + btrfs_free_qgroup_config(fs_info); + + ret = btrfs_clean_quota_tree(trans, quota_root); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + + ret = btrfs_del_root(trans, "a_root->root_key); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + + spin_lock(&fs_info->trans_lock); + list_del("a_root->dirty_list); + spin_unlock(&fs_info->trans_lock); + + btrfs_tree_lock(quota_root->node); + btrfs_clear_buffer_dirty(trans, quota_root->node); + btrfs_tree_unlock(quota_root->node); + ret = btrfs_free_tree_block(trans, btrfs_root_id(quota_root), + quota_root->node, 0, 1); + + if (ret < 0) + btrfs_abort_transaction(trans, ret); + +out: + btrfs_put_root(quota_root); + mutex_unlock(&fs_info->qgroup_ioctl_lock); + if (ret && trans) + btrfs_end_transaction(trans); + else if (trans) + ret = btrfs_commit_transaction(trans); + return ret; +} + +static void qgroup_dirty(struct btrfs_fs_info *fs_info, + struct btrfs_qgroup *qgroup) +{ + if (list_empty(&qgroup->dirty)) + list_add(&qgroup->dirty, &fs_info->dirty_qgroups); +} + +static void qgroup_iterator_add(struct list_head *head, struct btrfs_qgroup *qgroup) +{ + if (!list_empty(&qgroup->iterator)) + return; + + list_add_tail(&qgroup->iterator, head); +} + +static void qgroup_iterator_clean(struct list_head *head) +{ + while (!list_empty(head)) { + struct btrfs_qgroup *qgroup; + + qgroup = list_first_entry(head, struct btrfs_qgroup, iterator); + list_del_init(&qgroup->iterator); + } +} + +/* + * The easy accounting, we're updating qgroup relationship whose child qgroup + * only has exclusive extents. + * + * In this case, all exclusive extents will also be exclusive for parent, so + * excl/rfer just get added/removed. + * + * So is qgroup reservation space, which should also be added/removed to + * parent. + * Or when child tries to release reservation space, parent will underflow its + * reservation (for relationship adding case). + * + * Caller should hold fs_info->qgroup_lock. + */ +static int __qgroup_excl_accounting(struct btrfs_fs_info *fs_info, u64 ref_root, + struct btrfs_qgroup *src, int sign) +{ + struct btrfs_qgroup *qgroup; + LIST_HEAD(qgroup_list); + u64 num_bytes = src->excl; + u64 num_bytes_cmpr = src->excl_cmpr; + int ret = 0; + + qgroup = find_qgroup_rb(fs_info, ref_root); + if (!qgroup) + goto out; + + qgroup_iterator_add(&qgroup_list, qgroup); + list_for_each_entry(qgroup, &qgroup_list, iterator) { + struct btrfs_qgroup_list *glist; + + qgroup->rfer += sign * num_bytes; + qgroup->rfer_cmpr += sign * num_bytes_cmpr; + + WARN_ON(sign < 0 && qgroup->excl < num_bytes); + WARN_ON(sign < 0 && qgroup->excl_cmpr < num_bytes_cmpr); + qgroup->excl += sign * num_bytes; + qgroup->excl_cmpr += sign * num_bytes_cmpr; + + if (sign > 0) + qgroup_rsv_add_by_qgroup(fs_info, qgroup, src); + else + qgroup_rsv_release_by_qgroup(fs_info, qgroup, src); + qgroup_dirty(fs_info, qgroup); + + /* Append parent qgroups to @qgroup_list. */ + list_for_each_entry(glist, &qgroup->groups, next_group) + qgroup_iterator_add(&qgroup_list, glist->group); + } + ret = 0; +out: + qgroup_iterator_clean(&qgroup_list); + return ret; +} + + +/* + * Quick path for updating qgroup with only excl refs. + * + * In that case, just update all parent will be enough. + * Or we needs to do a full rescan. + * Caller should also hold fs_info->qgroup_lock. + * + * Return 0 for quick update, return >0 for need to full rescan + * and mark INCONSISTENT flag. + * Return < 0 for other error. + */ +static int quick_update_accounting(struct btrfs_fs_info *fs_info, + u64 src, u64 dst, int sign) +{ + struct btrfs_qgroup *qgroup; + int ret = 1; + + qgroup = find_qgroup_rb(fs_info, src); + if (!qgroup) + goto out; + if (qgroup->excl == qgroup->rfer) { + ret = __qgroup_excl_accounting(fs_info, dst, qgroup, sign); + if (ret < 0) + goto out; + ret = 0; + } +out: + if (ret) + fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT; + return ret; +} + +/* + * Add relation between @src and @dst qgroup. The @prealloc is allocated by the + * callers and transferred here (either used or freed on error). + */ +int btrfs_add_qgroup_relation(struct btrfs_trans_handle *trans, u64 src, u64 dst, + struct btrfs_qgroup_list *prealloc) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_qgroup *parent; + struct btrfs_qgroup *member; + struct btrfs_qgroup_list *list; + int ret = 0; + + ASSERT(prealloc); + + /* Check the level of src and dst first */ + if (btrfs_qgroup_level(src) >= btrfs_qgroup_level(dst)) { + kfree(prealloc); + return -EINVAL; + } + + mutex_lock(&fs_info->qgroup_ioctl_lock); + if (!fs_info->quota_root) { + ret = -ENOTCONN; + goto out; + } + member = find_qgroup_rb(fs_info, src); + parent = find_qgroup_rb(fs_info, dst); + if (!member || !parent) { + ret = -EINVAL; + goto out; + } + + /* check if such qgroup relation exist firstly */ + list_for_each_entry(list, &member->groups, next_group) { + if (list->group == parent) { + ret = -EEXIST; + goto out; + } + } + + ret = add_qgroup_relation_item(trans, src, dst); + if (ret) + goto out; + + ret = add_qgroup_relation_item(trans, dst, src); + if (ret) { + del_qgroup_relation_item(trans, src, dst); + goto out; + } + + spin_lock(&fs_info->qgroup_lock); + ret = __add_relation_rb(prealloc, member, parent); + prealloc = NULL; + if (ret < 0) { + spin_unlock(&fs_info->qgroup_lock); + goto out; + } + ret = quick_update_accounting(fs_info, src, dst, 1); + squota_check_parent_usage(fs_info, parent); + spin_unlock(&fs_info->qgroup_lock); +out: + kfree(prealloc); + mutex_unlock(&fs_info->qgroup_ioctl_lock); + return ret; +} + +static int __del_qgroup_relation(struct btrfs_trans_handle *trans, u64 src, + u64 dst) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_qgroup *parent; + struct btrfs_qgroup *member; + struct btrfs_qgroup_list *list; + bool found = false; + int ret = 0; + int ret2; + + if (!fs_info->quota_root) + return -ENOTCONN; + + member = find_qgroup_rb(fs_info, src); + parent = find_qgroup_rb(fs_info, dst); + /* + * The parent/member pair doesn't exist, then try to delete the dead + * relation items only. + */ + if (!member || !parent) + goto delete_item; + + /* check if such qgroup relation exist firstly */ + list_for_each_entry(list, &member->groups, next_group) { + if (list->group == parent) { + found = true; + break; + } + } + +delete_item: + ret = del_qgroup_relation_item(trans, src, dst); + if (ret < 0 && ret != -ENOENT) + return ret; + ret2 = del_qgroup_relation_item(trans, dst, src); + if (ret2 < 0 && ret2 != -ENOENT) + return ret2; + + /* At least one deletion succeeded, return 0 */ + if (!ret || !ret2) + ret = 0; + + if (found) { + spin_lock(&fs_info->qgroup_lock); + del_relation_rb(fs_info, src, dst); + ret = quick_update_accounting(fs_info, src, dst, -1); + ASSERT(parent); + squota_check_parent_usage(fs_info, parent); + spin_unlock(&fs_info->qgroup_lock); + } + + return ret; +} + +int btrfs_del_qgroup_relation(struct btrfs_trans_handle *trans, u64 src, + u64 dst) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + int ret = 0; + + mutex_lock(&fs_info->qgroup_ioctl_lock); + ret = __del_qgroup_relation(trans, src, dst); + mutex_unlock(&fs_info->qgroup_ioctl_lock); + + return ret; +} + +int btrfs_create_qgroup(struct btrfs_trans_handle *trans, u64 qgroupid) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *quota_root; + struct btrfs_qgroup *qgroup; + struct btrfs_qgroup *prealloc = NULL; + int ret = 0; + + mutex_lock(&fs_info->qgroup_ioctl_lock); + if (!fs_info->quota_root) { + ret = -ENOTCONN; + goto out; + } + quota_root = fs_info->quota_root; + qgroup = find_qgroup_rb(fs_info, qgroupid); + if (qgroup) { + ret = -EEXIST; + goto out; + } + + prealloc = kzalloc_obj(*prealloc, GFP_NOFS); + if (!prealloc) { + ret = -ENOMEM; + goto out; + } + + ret = add_qgroup_item(trans, quota_root, qgroupid); + if (ret) + goto out; + + spin_lock(&fs_info->qgroup_lock); + qgroup = add_qgroup_rb(fs_info, prealloc, qgroupid); + spin_unlock(&fs_info->qgroup_lock); + prealloc = NULL; + + ret = btrfs_sysfs_add_one_qgroup(fs_info, qgroup); +out: + mutex_unlock(&fs_info->qgroup_ioctl_lock); + kfree(prealloc); + return ret; +} + +static bool can_delete_parent_qgroup(struct btrfs_fs_info *fs_info, struct btrfs_qgroup *qgroup) +{ + ASSERT(btrfs_qgroup_level(qgroup->qgroupid)); + if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE) + squota_check_parent_usage(fs_info, qgroup); + return list_empty(&qgroup->members); +} + +/* + * Because a shared extent can outlive its owning subvolume, we cannot delete a + * subvol squota qgroup until all of the extents it owns are gone, even if the + * subvolume itself has been deleted. + */ +static bool can_delete_squota_subvol_qgroup(struct btrfs_fs_info *fs_info, + struct btrfs_qgroup *qgroup) +{ + ASSERT(btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE); + ASSERT(btrfs_qgroup_level(qgroup->qgroupid) == 0); + + return !(qgroup->rfer || qgroup->excl || qgroup->rfer_cmpr || qgroup->excl_cmpr); +} + +/* + * Return 0 if we can not delete the qgroup (not empty or has children etc). + * Return >0 if we can delete the qgroup. + * Return <0 for other errors during tree search. + */ +static int can_delete_qgroup(struct btrfs_fs_info *fs_info, struct btrfs_qgroup *qgroup) +{ + struct btrfs_key key; + BTRFS_PATH_AUTO_FREE(path); + int ret; + + /* For higher level qgroup, we can only delete it if it has no child. */ + if (btrfs_qgroup_level(qgroup->qgroupid)) + return can_delete_parent_qgroup(fs_info, qgroup); + + /* + * For level-0 qgroups, we can only delete it if it has no subvolume + * for it. + * This means even a subvolume is unlinked but not yet fully dropped, + * we can not delete the qgroup. + */ + key.objectid = qgroup->qgroupid; + key.type = BTRFS_ROOT_ITEM_KEY; + key.offset = -1ULL; + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + /* + * Any subvol qgroup, regardless of mode, cannot be deleted if the + * subvol still exists. + */ + ret = btrfs_find_root(fs_info->tree_root, &key, path, NULL, NULL); + /* + * btrfs_find_root returns <0 on error, 0 if found, and >0 if not, + * so the "found" and "error" cases match our desired return values. + */ + if (ret <= 0) + return ret; + + /* Squotas require additional checks, even if the subvol is deleted. */ + if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE) + return can_delete_squota_subvol_qgroup(fs_info, qgroup); + return 1; +} + +int btrfs_remove_qgroup(struct btrfs_trans_handle *trans, u64 qgroupid) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_qgroup *qgroup; + struct btrfs_qgroup_list *list; + int ret = 0; + + mutex_lock(&fs_info->qgroup_ioctl_lock); + if (!fs_info->quota_root) { + ret = -ENOTCONN; + goto out; + } + + qgroup = find_qgroup_rb(fs_info, qgroupid); + if (!qgroup) { + ret = -ENOENT; + goto out; + } + + ret = can_delete_qgroup(fs_info, qgroup); + if (ret < 0) + goto out; + if (ret == 0) { + ret = -EBUSY; + goto out; + } + + /* Check if there are no children of this qgroup */ + if (!list_empty(&qgroup->members)) { + ret = -EBUSY; + goto out; + } + + ret = del_qgroup_item(trans, qgroupid); + if (ret && ret != -ENOENT) + goto out; + + while (!list_empty(&qgroup->groups)) { + list = list_first_entry(&qgroup->groups, + struct btrfs_qgroup_list, next_group); + ret = __del_qgroup_relation(trans, qgroupid, + list->group->qgroupid); + if (ret) + goto out; + } + + spin_lock(&fs_info->qgroup_lock); + /* + * Warn on reserved space. The subvolume should has no child nor + * corresponding subvolume. + * Thus its reserved space should all be zero, no matter if qgroup + * is consistent or the mode. + */ + if (unlikely(qgroup->rsv.values[BTRFS_QGROUP_RSV_DATA] || + qgroup->rsv.values[BTRFS_QGROUP_RSV_META_PREALLOC] || + qgroup->rsv.values[BTRFS_QGROUP_RSV_META_PERTRANS])) { + DEBUG_WARN(); + btrfs_warn_rl(fs_info, +"to be deleted qgroup " BTRFS_QGROUP_FMT " has non-zero numbers, data %llu meta prealloc %llu meta pertrans %llu", + BTRFS_QGROUP_FMT_VALUE(qgroup), + qgroup->rsv.values[BTRFS_QGROUP_RSV_DATA], + qgroup->rsv.values[BTRFS_QGROUP_RSV_META_PREALLOC], + qgroup->rsv.values[BTRFS_QGROUP_RSV_META_PERTRANS]); + + } + /* + * The same for rfer/excl numbers, but that's only if our qgroup is + * consistent and if it's in regular qgroup mode. + * For simple mode it's not as accurate thus we can hit non-zero values + * very frequently. + */ + if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_FULL && + !(fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT)) { + if (unlikely(qgroup->rfer || qgroup->excl || + qgroup->rfer_cmpr || qgroup->excl_cmpr)) { + DEBUG_WARN(); + qgroup_mark_inconsistent(fs_info, +"to be deleted qgroup " BTRFS_QGROUP_FMT " has non-zero numbers, rfer %llu rfer_cmpr %llu excl %llu excl_cmpr %llu", + BTRFS_QGROUP_FMT_VALUE(qgroup), + qgroup->rfer, qgroup->rfer_cmpr, + qgroup->excl, qgroup->excl_cmpr); + } + } + del_qgroup_rb(fs_info, qgroupid); + spin_unlock(&fs_info->qgroup_lock); + + /* + * Remove the qgroup from sysfs now without holding the qgroup_lock + * spinlock, since the sysfs_remove_group() function needs to take + * the mutex kernfs_mutex through kernfs_remove_by_name_ns(). + */ + btrfs_sysfs_del_one_qgroup(fs_info, qgroup); + kfree(qgroup); +out: + mutex_unlock(&fs_info->qgroup_ioctl_lock); + return ret; +} + +int btrfs_qgroup_cleanup_dropped_subvolume(struct btrfs_fs_info *fs_info, u64 subvolid) +{ + struct btrfs_trans_handle *trans; + int ret; + + if (!btrfs_is_fstree(subvolid) || !btrfs_qgroup_enabled(fs_info) || + !fs_info->quota_root) + return 0; + + /* + * Commit current transaction to make sure all the rfer/excl numbers + * get updated. + */ + ret = btrfs_commit_current_transaction(fs_info->quota_root); + if (ret < 0) + return ret; + + /* Start new trans to delete the qgroup info and limit items. */ + trans = btrfs_start_transaction(fs_info->quota_root, 2); + if (IS_ERR(trans)) + return PTR_ERR(trans); + ret = btrfs_remove_qgroup(trans, subvolid); + btrfs_end_transaction(trans); + /* + * It's squota and the subvolume still has numbers needed for future + * accounting, in this case we can not delete it. Just skip it. + * + * Or the qgroup is already removed by a qgroup rescan. For both cases we're + * safe to ignore them. + */ + if (ret == -EBUSY || ret == -ENOENT) + ret = 0; + return ret; +} + +int btrfs_limit_qgroup(struct btrfs_trans_handle *trans, u64 qgroupid, + struct btrfs_qgroup_limit *limit) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_qgroup *qgroup; + int ret = 0; + /* Sometimes we would want to clear the limit on this qgroup. + * To meet this requirement, we treat the -1 as a special value + * which tell kernel to clear the limit on this qgroup. + */ + const u64 CLEAR_VALUE = -1; + + mutex_lock(&fs_info->qgroup_ioctl_lock); + if (!fs_info->quota_root) { + ret = -ENOTCONN; + goto out; + } + + qgroup = find_qgroup_rb(fs_info, qgroupid); + if (!qgroup) { + ret = -ENOENT; + goto out; + } + + spin_lock(&fs_info->qgroup_lock); + if (limit->flags & BTRFS_QGROUP_LIMIT_MAX_RFER) { + if (limit->max_rfer == CLEAR_VALUE) { + qgroup->lim_flags &= ~BTRFS_QGROUP_LIMIT_MAX_RFER; + limit->flags &= ~BTRFS_QGROUP_LIMIT_MAX_RFER; + qgroup->max_rfer = 0; + } else { + qgroup->max_rfer = limit->max_rfer; + } + } + if (limit->flags & BTRFS_QGROUP_LIMIT_MAX_EXCL) { + if (limit->max_excl == CLEAR_VALUE) { + qgroup->lim_flags &= ~BTRFS_QGROUP_LIMIT_MAX_EXCL; + limit->flags &= ~BTRFS_QGROUP_LIMIT_MAX_EXCL; + qgroup->max_excl = 0; + } else { + qgroup->max_excl = limit->max_excl; + } + } + if (limit->flags & BTRFS_QGROUP_LIMIT_RSV_RFER) { + if (limit->rsv_rfer == CLEAR_VALUE) { + qgroup->lim_flags &= ~BTRFS_QGROUP_LIMIT_RSV_RFER; + limit->flags &= ~BTRFS_QGROUP_LIMIT_RSV_RFER; + qgroup->rsv_rfer = 0; + } else { + qgroup->rsv_rfer = limit->rsv_rfer; + } + } + if (limit->flags & BTRFS_QGROUP_LIMIT_RSV_EXCL) { + if (limit->rsv_excl == CLEAR_VALUE) { + qgroup->lim_flags &= ~BTRFS_QGROUP_LIMIT_RSV_EXCL; + limit->flags &= ~BTRFS_QGROUP_LIMIT_RSV_EXCL; + qgroup->rsv_excl = 0; + } else { + qgroup->rsv_excl = limit->rsv_excl; + } + } + qgroup->lim_flags |= limit->flags; + + spin_unlock(&fs_info->qgroup_lock); + + ret = update_qgroup_limit_item(trans, qgroup); + if (ret) + qgroup_mark_inconsistent(fs_info, "qgroup item update error %d", ret); + +out: + mutex_unlock(&fs_info->qgroup_ioctl_lock); + return ret; +} + +/* + * Inform qgroup to trace one dirty extent, its info is recorded in @record. + * So qgroup can account it at transaction committing time. + * + * No lock version, caller must acquire delayed ref lock and allocated memory, + * then call btrfs_qgroup_trace_extent_post() after exiting lock context. + * + * Return 0 for success insert + * Return >0 for existing record, caller can free @record safely. + * Return <0 for insertion failure, caller can free @record safely. + */ +int btrfs_qgroup_trace_extent_nolock(struct btrfs_fs_info *fs_info, + struct btrfs_delayed_ref_root *delayed_refs, + struct btrfs_qgroup_extent_record *record, + u64 bytenr) +{ + struct btrfs_qgroup_extent_record *existing, *ret; + const unsigned long index = (bytenr >> fs_info->sectorsize_bits); + + if (!btrfs_qgroup_full_accounting(fs_info)) + return 1; + +#if BITS_PER_LONG == 32 + if (bytenr >= MAX_LFS_FILESIZE) { + btrfs_err_rl(fs_info, +"qgroup record for extent at %llu is beyond 32bit page cache and xarray index limit", + bytenr); + btrfs_err_32bit_limit(fs_info); + return -EOVERFLOW; + } +#endif + + trace_btrfs_qgroup_trace_extent(fs_info, record, bytenr); + + xa_lock(&delayed_refs->dirty_extents); + existing = xa_load(&delayed_refs->dirty_extents, index); + if (existing) { + if (record->data_rsv && !existing->data_rsv) { + existing->data_rsv = record->data_rsv; + existing->data_rsv_refroot = record->data_rsv_refroot; + } + xa_unlock(&delayed_refs->dirty_extents); + return 1; + } + + ret = __xa_store(&delayed_refs->dirty_extents, index, record, GFP_ATOMIC); + xa_unlock(&delayed_refs->dirty_extents); + if (xa_is_err(ret)) { + qgroup_mark_inconsistent(fs_info, "xarray insert error: %d", xa_err(ret)); + return xa_err(ret); + } + + return 0; +} + +/* + * Post handler after qgroup_trace_extent_nolock(). + * + * NOTE: Current qgroup does the expensive backref walk at transaction + * committing time with TRANS_STATE_COMMIT_DOING, this blocks incoming + * new transaction. + * This is designed to allow btrfs_find_all_roots() to get correct new_roots + * result. + * + * However for old_roots there is no need to do backref walk at that time, + * since we search commit roots to walk backref and result will always be + * correct. + * + * Due to the nature of no lock version, we can't do backref there. + * So we must call btrfs_qgroup_trace_extent_post() after exiting + * spinlock context. + * + * TODO: If we can fix and prove btrfs_find_all_roots() can get correct result + * using current root, then we can move all expensive backref walk out of + * transaction committing, but not now as qgroup accounting will be wrong again. + */ +int btrfs_qgroup_trace_extent_post(struct btrfs_trans_handle *trans, + struct btrfs_qgroup_extent_record *qrecord, + u64 bytenr) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_backref_walk_ctx ctx = { + .bytenr = bytenr, + .fs_info = fs_info, + }; + int ret; + + if (!btrfs_qgroup_full_accounting(fs_info)) + return 0; + /* + * We are always called in a context where we are already holding a + * transaction handle. Often we are called when adding a data delayed + * reference from btrfs_truncate_inode_items() (truncating or unlinking), + * in which case we will be holding a write lock on extent buffer from a + * subvolume tree. In this case we can't allow btrfs_find_all_roots() to + * acquire fs_info->commit_root_sem, because that is a higher level lock + * that must be acquired before locking any extent buffers. + * + * So we want btrfs_find_all_roots() to not acquire the commit_root_sem + * but we can't pass it a non-NULL transaction handle, because otherwise + * it would not use commit roots and would lock extent buffers, causing + * a deadlock if it ends up trying to read lock the same extent buffer + * that was previously write locked at btrfs_truncate_inode_items(). + * + * So pass a NULL transaction handle to btrfs_find_all_roots() and + * explicitly tell it to not acquire the commit_root_sem - if we are + * holding a transaction handle we don't need its protection. + */ + ASSERT(trans != NULL); + + if (fs_info->qgroup_flags & BTRFS_QGROUP_RUNTIME_FLAG_NO_ACCOUNTING) + return 0; + + ret = btrfs_find_all_roots(&ctx, true); + if (ret < 0) { + qgroup_mark_inconsistent(fs_info, + "error accounting new delayed refs extent: %d", ret); + return 0; + } + + /* + * Here we don't need to get the lock of + * trans->transaction->delayed_refs, since inserted qrecord won't + * be deleted, only qrecord->node may be modified (new qrecord insert) + * + * So modifying qrecord->old_roots is safe here + */ + qrecord->old_roots = ctx.roots; + return 0; +} + +/* + * Inform qgroup to trace one dirty extent, specified by @bytenr and + * @num_bytes. + * So qgroup can account it at commit trans time. + * + * Better encapsulated version, with memory allocation and backref walk for + * commit roots. + * So this can sleep. + * + * Return 0 if the operation is done. + * Return <0 for error, like memory allocation failure or invalid parameter + * (NULL trans) + */ +int btrfs_qgroup_trace_extent(struct btrfs_trans_handle *trans, u64 bytenr, + u64 num_bytes) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_qgroup_extent_record *record; + struct btrfs_delayed_ref_root *delayed_refs = &trans->transaction->delayed_refs; + const unsigned long index = (bytenr >> fs_info->sectorsize_bits); + int ret; + + if (!btrfs_qgroup_full_accounting(fs_info) || bytenr == 0 || num_bytes == 0) + return 0; + record = kzalloc_obj(*record, GFP_NOFS); + if (!record) + return -ENOMEM; + + if (xa_reserve(&delayed_refs->dirty_extents, index, GFP_NOFS)) { + kfree(record); + return -ENOMEM; + } + + record->num_bytes = num_bytes; + + ret = btrfs_qgroup_trace_extent_nolock(fs_info, delayed_refs, record, bytenr); + if (ret) { + /* Clean up if insertion fails or item exists. */ + xa_release(&delayed_refs->dirty_extents, index); + kfree(record); + return 0; + } + return btrfs_qgroup_trace_extent_post(trans, record, bytenr); +} + +/* + * Inform qgroup to trace all leaf items of data + * + * Return 0 for success + * Return <0 for error(ENOMEM) + */ +int btrfs_qgroup_trace_leaf_items(struct btrfs_trans_handle *trans, + struct extent_buffer *eb) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + int nr = btrfs_header_nritems(eb); + int i, extent_type, ret; + struct btrfs_key key; + struct btrfs_file_extent_item *fi; + u64 bytenr, num_bytes; + + /* We can be called directly from walk_up_proc() */ + if (!btrfs_qgroup_full_accounting(fs_info)) + return 0; + + for (i = 0; i < nr; i++) { + btrfs_item_key_to_cpu(eb, &key, i); + + if (key.type != BTRFS_EXTENT_DATA_KEY) + continue; + + fi = btrfs_item_ptr(eb, i, struct btrfs_file_extent_item); + /* filter out non qgroup-accountable extents */ + extent_type = btrfs_file_extent_type(eb, fi); + + if (extent_type == BTRFS_FILE_EXTENT_INLINE) + continue; + + bytenr = btrfs_file_extent_disk_bytenr(eb, fi); + if (!bytenr) + continue; + + num_bytes = btrfs_file_extent_disk_num_bytes(eb, fi); + + ret = btrfs_qgroup_trace_extent(trans, bytenr, num_bytes); + if (ret) + return ret; + } + cond_resched(); + return 0; +} + +/* + * Walk up the tree from the bottom, freeing leaves and any interior + * nodes which have had all slots visited. If a node (leaf or + * interior) is freed, the node above it will have it's slot + * incremented. The root node will never be freed. + * + * At the end of this function, we should have a path which has all + * slots incremented to the next position for a search. If we need to + * read a new node it will be NULL and the node above it will have the + * correct slot selected for a later read. + * + * If we increment the root nodes slot counter past the number of + * elements, 1 is returned to signal completion of the search. + */ +static int adjust_slots_upwards(struct btrfs_path *path, int root_level) +{ + int level = 0; + int nr, slot; + struct extent_buffer *eb; + + if (root_level == 0) + return 1; + + while (level <= root_level) { + eb = path->nodes[level]; + nr = btrfs_header_nritems(eb); + path->slots[level]++; + slot = path->slots[level]; + if (slot >= nr || level == 0) { + /* + * Don't free the root - we will detect this + * condition after our loop and return a + * positive value for caller to stop walking the tree. + */ + if (level != root_level) { + btrfs_tree_unlock_rw(eb, path->locks[level]); + path->locks[level] = 0; + + free_extent_buffer(eb); + path->nodes[level] = NULL; + path->slots[level] = 0; + } + } else { + /* + * We have a valid slot to walk back down + * from. Stop here so caller can process these + * new nodes. + */ + break; + } + + level++; + } + + eb = path->nodes[root_level]; + if (path->slots[root_level] >= btrfs_header_nritems(eb)) + return 1; + + return 0; +} + +/* + * Helper function to trace a subtree tree block swap. + * + * The swap will happen in highest tree block, but there may be a lot of + * tree blocks involved. + * + * For example: + * OO = Old tree blocks + * NN = New tree blocks allocated during balance + * + * File tree (257) Reloc tree for 257 + * L2 OO NN + * / \ / \ + * L1 OO OO (a) OO NN (a) + * / \ / \ / \ / \ + * L0 OO OO OO OO OO OO NN NN + * (b) (c) (b) (c) + * + * When calling qgroup_trace_extent_swap(), we will pass: + * @src_eb = OO(a) + * @dst_path = [ nodes[1] = NN(a), nodes[0] = NN(c) ] + * @dst_level = 0 + * @root_level = 1 + * + * In that case, qgroup_trace_extent_swap() will search from OO(a) to + * reach OO(c), then mark both OO(c) and NN(c) as qgroup dirty. + * + * The main work of qgroup_trace_extent_swap() can be split into 3 parts: + * + * 1) Tree search from @src_eb + * It should acts as a simplified btrfs_search_slot(). + * The key for search can be extracted from @dst_path->nodes[dst_level] + * (first key). + * + * 2) Mark the final tree blocks in @src_path and @dst_path qgroup dirty + * NOTE: In above case, OO(a) and NN(a) won't be marked qgroup dirty. + * They should be marked during previous (@dst_level = 1) iteration. + * + * 3) Mark file extents in leaves dirty + * We don't have good way to pick out new file extents only. + * So we still follow the old method by scanning all file extents in + * the leave. + * + * This function can free us from keeping two paths, thus later we only need + * to care about how to iterate all new tree blocks in reloc tree. + */ +static int qgroup_trace_extent_swap(struct btrfs_trans_handle* trans, + struct extent_buffer *src_eb, + struct btrfs_path *dst_path, + int dst_level, int root_level, + bool trace_leaf) +{ + struct btrfs_key key; + BTRFS_PATH_AUTO_FREE(src_path); + struct btrfs_fs_info *fs_info = trans->fs_info; + u32 nodesize = fs_info->nodesize; + int cur_level = root_level; + int ret; + + BUG_ON(dst_level > root_level); + /* Level mismatch */ + if (btrfs_header_level(src_eb) != root_level) + return -EINVAL; + + src_path = btrfs_alloc_path(); + if (!src_path) + return -ENOMEM; + + if (dst_level) + btrfs_node_key_to_cpu(dst_path->nodes[dst_level], &key, 0); + else + btrfs_item_key_to_cpu(dst_path->nodes[dst_level], &key, 0); + + /* For src_path */ + refcount_inc(&src_eb->refs); + src_path->nodes[root_level] = src_eb; + src_path->slots[root_level] = dst_path->slots[root_level]; + src_path->locks[root_level] = 0; + + /* A simplified version of btrfs_search_slot() */ + while (cur_level >= dst_level) { + struct btrfs_key src_key; + struct btrfs_key dst_key; + + if (src_path->nodes[cur_level] == NULL) { + struct extent_buffer *eb; + int parent_slot; + + eb = src_path->nodes[cur_level + 1]; + parent_slot = src_path->slots[cur_level + 1]; + + eb = btrfs_read_node_slot(eb, parent_slot); + if (IS_ERR(eb)) + return PTR_ERR(eb); + + src_path->nodes[cur_level] = eb; + + btrfs_tree_read_lock(eb); + src_path->locks[cur_level] = BTRFS_READ_LOCK; + } + + src_path->slots[cur_level] = dst_path->slots[cur_level]; + if (cur_level) { + btrfs_node_key_to_cpu(dst_path->nodes[cur_level], + &dst_key, dst_path->slots[cur_level]); + btrfs_node_key_to_cpu(src_path->nodes[cur_level], + &src_key, src_path->slots[cur_level]); + } else { + btrfs_item_key_to_cpu(dst_path->nodes[cur_level], + &dst_key, dst_path->slots[cur_level]); + btrfs_item_key_to_cpu(src_path->nodes[cur_level], + &src_key, src_path->slots[cur_level]); + } + /* Content mismatch, something went wrong */ + if (btrfs_comp_cpu_keys(&dst_key, &src_key)) + return -ENOENT; + cur_level--; + } + + /* + * Now both @dst_path and @src_path have been populated, record the tree + * blocks for qgroup accounting. + */ + ret = btrfs_qgroup_trace_extent(trans, src_path->nodes[dst_level]->start, + nodesize); + if (ret < 0) + return ret; + ret = btrfs_qgroup_trace_extent(trans, dst_path->nodes[dst_level]->start, + nodesize); + if (ret < 0) + return ret; + + /* Record leaf file extents */ + if (dst_level == 0 && trace_leaf) { + ret = btrfs_qgroup_trace_leaf_items(trans, src_path->nodes[0]); + if (ret < 0) + return ret; + ret = btrfs_qgroup_trace_leaf_items(trans, dst_path->nodes[0]); + } + + return ret; +} + +/* + * Helper function to do recursive generation-aware depth-first search, to + * locate all new tree blocks in a subtree of reloc tree. + * + * E.g. (OO = Old tree blocks, NN = New tree blocks, whose gen == last_snapshot) + * reloc tree + * L2 NN (a) + * / \ + * L1 OO NN (b) + * / \ / \ + * L0 OO OO OO NN + * (c) (d) + * If we pass: + * @dst_path = [ nodes[1] = NN(b), nodes[0] = NULL ], + * @cur_level = 1 + * @root_level = 1 + * + * We will iterate through tree blocks NN(b), NN(d) and info qgroup to trace + * above tree blocks along with their counter parts in file tree. + * While during search, old tree blocks OO(c) will be skipped as tree block swap + * won't affect OO(c). + */ +static int qgroup_trace_new_subtree_blocks(struct btrfs_trans_handle* trans, + struct extent_buffer *src_eb, + struct btrfs_path *dst_path, + int cur_level, int root_level, + u64 last_snapshot, bool trace_leaf) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct extent_buffer *eb; + bool need_cleanup = false; + int ret = 0; + int i; + + /* Level sanity check */ + if (unlikely(cur_level < 0 || cur_level >= BTRFS_MAX_LEVEL - 1 || + root_level < 0 || root_level >= BTRFS_MAX_LEVEL - 1 || + root_level < cur_level)) { + btrfs_err_rl(fs_info, + "%s: bad levels, cur_level=%d root_level=%d", + __func__, cur_level, root_level); + return -EUCLEAN; + } + + /* Read the tree block if needed */ + if (dst_path->nodes[cur_level] == NULL) { + int parent_slot; + u64 child_gen; + + /* + * dst_path->nodes[root_level] must be initialized before + * calling this function. + */ + if (unlikely(cur_level == root_level)) { + btrfs_err_rl(fs_info, + "%s: dst_path->nodes[%d] not initialized, root_level=%d cur_level=%d", + __func__, root_level, root_level, cur_level); + return -EUCLEAN; + } + + /* + * We need to get child blockptr/gen from parent before we can + * read it. + */ + eb = dst_path->nodes[cur_level + 1]; + parent_slot = dst_path->slots[cur_level + 1]; + child_gen = btrfs_node_ptr_generation(eb, parent_slot); + + /* This node is old, no need to trace */ + if (child_gen < last_snapshot) + return ret; + + eb = btrfs_read_node_slot(eb, parent_slot); + if (IS_ERR(eb)) + return PTR_ERR(eb); + + dst_path->nodes[cur_level] = eb; + dst_path->slots[cur_level] = 0; + + btrfs_tree_read_lock(eb); + dst_path->locks[cur_level] = BTRFS_READ_LOCK; + need_cleanup = true; + } + + /* Now record this tree block and its counter part for qgroups */ + ret = qgroup_trace_extent_swap(trans, src_eb, dst_path, cur_level, + root_level, trace_leaf); + if (ret < 0) + goto cleanup; + + eb = dst_path->nodes[cur_level]; + + if (cur_level > 0) { + /* Iterate all child tree blocks */ + for (i = 0; i < btrfs_header_nritems(eb); i++) { + /* Skip old tree blocks as they won't be swapped */ + if (btrfs_node_ptr_generation(eb, i) < last_snapshot) + continue; + dst_path->slots[cur_level] = i; + + /* Recursive call (at most 7 times) */ + ret = qgroup_trace_new_subtree_blocks(trans, src_eb, + dst_path, cur_level - 1, root_level, + last_snapshot, trace_leaf); + if (ret < 0) + goto cleanup; + } + } + +cleanup: + if (need_cleanup) { + /* Clean up */ + btrfs_tree_unlock_rw(dst_path->nodes[cur_level], + dst_path->locks[cur_level]); + free_extent_buffer(dst_path->nodes[cur_level]); + dst_path->nodes[cur_level] = NULL; + dst_path->slots[cur_level] = 0; + dst_path->locks[cur_level] = 0; + } + + return ret; +} + +static int qgroup_trace_subtree_swap(struct btrfs_trans_handle *trans, + struct extent_buffer *src_eb, + struct extent_buffer *dst_eb, + u64 last_snapshot, bool trace_leaf) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_path *dst_path = NULL; + int level; + int ret; + + if (!btrfs_qgroup_full_accounting(fs_info)) + return 0; + + /* Wrong parameter order */ + if (unlikely(btrfs_header_generation(src_eb) > btrfs_header_generation(dst_eb))) { + btrfs_err_rl(fs_info, + "%s: bad parameter order, src_gen=%llu dst_gen=%llu", __func__, + btrfs_header_generation(src_eb), + btrfs_header_generation(dst_eb)); + return -EUCLEAN; + } + + if (unlikely(!extent_buffer_uptodate(src_eb) || !extent_buffer_uptodate(dst_eb))) { + ret = -EIO; + goto out; + } + + level = btrfs_header_level(dst_eb); + dst_path = btrfs_alloc_path(); + if (!dst_path) { + ret = -ENOMEM; + goto out; + } + /* For dst_path */ + refcount_inc(&dst_eb->refs); + dst_path->nodes[level] = dst_eb; + dst_path->slots[level] = 0; + dst_path->locks[level] = 0; + + /* Do the generation aware breadth-first search */ + ret = qgroup_trace_new_subtree_blocks(trans, src_eb, dst_path, level, + level, last_snapshot, trace_leaf); + if (ret < 0) + goto out; + ret = 0; + +out: + btrfs_free_path(dst_path); + if (ret < 0) + qgroup_mark_inconsistent(fs_info, "%s error: %d", __func__, ret); + return ret; +} + +/* + * Inform qgroup to trace a whole subtree, including all its child tree + * blocks and data. + * The root tree block is specified by @root_eb. + * + * Normally used by relocation(tree block swap) and subvolume deletion. + * + * Return 0 for success + * Return <0 for error(ENOMEM or tree search error) + */ +int btrfs_qgroup_trace_subtree(struct btrfs_trans_handle *trans, + struct extent_buffer *root_eb, + u64 root_gen, int root_level) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + int ret = 0; + int level; + u8 drop_subptree_thres; + struct extent_buffer *eb = root_eb; + BTRFS_PATH_AUTO_FREE(path); + + ASSERT(0 <= root_level && root_level < BTRFS_MAX_LEVEL); + ASSERT(root_eb != NULL); + + if (!btrfs_qgroup_full_accounting(fs_info)) + return 0; + + spin_lock(&fs_info->qgroup_lock); + drop_subptree_thres = fs_info->qgroup_drop_subtree_thres; + spin_unlock(&fs_info->qgroup_lock); + + /* + * This function only gets called for snapshot drop, if we hit a high + * node here, it means we are going to change ownership for quite a lot + * of extents, which will greatly slow down btrfs_commit_transaction(). + * + * So here if we find a high tree here, we just skip the accounting and + * mark qgroup inconsistent. + */ + if (root_level >= drop_subptree_thres) { + qgroup_mark_inconsistent(fs_info, "subtree level reached threshold"); + return 0; + } + + if (!extent_buffer_uptodate(root_eb)) { + struct btrfs_tree_parent_check check = { + .transid = root_gen, + .level = root_level + }; + + ret = btrfs_read_extent_buffer(root_eb, &check); + if (ret) + return ret; + } + + if (root_level == 0) + return btrfs_qgroup_trace_leaf_items(trans, root_eb); + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + /* + * Walk down the tree. Missing extent blocks are filled in as + * we go. Metadata is accounted every time we read a new + * extent block. + * + * When we reach a leaf, we account for file extent items in it, + * walk back up the tree (adjusting slot pointers as we go) + * and restart the search process. + */ + refcount_inc(&root_eb->refs); /* For path */ + path->nodes[root_level] = root_eb; + path->slots[root_level] = 0; + path->locks[root_level] = 0; /* so release_path doesn't try to unlock */ +walk_down: + level = root_level; + while (level >= 0) { + if (path->nodes[level] == NULL) { + int parent_slot; + u64 child_bytenr; + + /* + * We need to get child blockptr from parent before we + * can read it. + */ + eb = path->nodes[level + 1]; + parent_slot = path->slots[level + 1]; + child_bytenr = btrfs_node_blockptr(eb, parent_slot); + + eb = btrfs_read_node_slot(eb, parent_slot); + if (IS_ERR(eb)) + return PTR_ERR(eb); + + path->nodes[level] = eb; + path->slots[level] = 0; + + btrfs_tree_read_lock(eb); + path->locks[level] = BTRFS_READ_LOCK; + + ret = btrfs_qgroup_trace_extent(trans, child_bytenr, + fs_info->nodesize); + if (ret) + return ret; + } + + if (level == 0) { + ret = btrfs_qgroup_trace_leaf_items(trans, + path->nodes[level]); + if (ret) + return ret; + + /* Nonzero return here means we completed our search */ + ret = adjust_slots_upwards(path, root_level); + if (ret) + break; + + /* Restart search with new slots */ + goto walk_down; + } + + level--; + } + + return 0; +} + +static void qgroup_iterator_nested_add(struct list_head *head, struct btrfs_qgroup *qgroup) +{ + if (!list_empty(&qgroup->nested_iterator)) + return; + + list_add_tail(&qgroup->nested_iterator, head); +} + +static void qgroup_iterator_nested_clean(struct list_head *head) +{ + while (!list_empty(head)) { + struct btrfs_qgroup *qgroup; + + qgroup = list_first_entry(head, struct btrfs_qgroup, nested_iterator); + list_del_init(&qgroup->nested_iterator); + } +} + +/* + * Walk all of the roots that points to the bytenr and adjust their refcnts. + */ +static void qgroup_update_refcnt(struct btrfs_fs_info *fs_info, + struct ulist *roots, struct list_head *qgroups, + u64 seq, bool update_old) +{ + struct ulist_node *unode; + struct ulist_iterator uiter; + struct btrfs_qgroup *qg; + + if (!roots) + return; + ULIST_ITER_INIT(&uiter); + while ((unode = ulist_next(roots, &uiter))) { + LIST_HEAD(tmp); + + qg = find_qgroup_rb(fs_info, unode->val); + if (!qg) + continue; + + qgroup_iterator_nested_add(qgroups, qg); + qgroup_iterator_add(&tmp, qg); + list_for_each_entry(qg, &tmp, iterator) { + struct btrfs_qgroup_list *glist; + + if (update_old) + btrfs_qgroup_update_old_refcnt(qg, seq, 1); + else + btrfs_qgroup_update_new_refcnt(qg, seq, 1); + + list_for_each_entry(glist, &qg->groups, next_group) { + qgroup_iterator_nested_add(qgroups, glist->group); + qgroup_iterator_add(&tmp, glist->group); + } + } + qgroup_iterator_clean(&tmp); + } +} + +/* + * Update qgroup rfer/excl counters. + * Rfer update is easy, codes can explain themselves. + * + * Excl update is tricky, the update is split into 2 parts. + * Part 1: Possible exclusive <-> sharing detect: + * | A | !A | + * ------------------------------------- + * B | * | - | + * ------------------------------------- + * !B | + | ** | + * ------------------------------------- + * + * Conditions: + * A: cur_old_roots < nr_old_roots (not exclusive before) + * !A: cur_old_roots == nr_old_roots (possible exclusive before) + * B: cur_new_roots < nr_new_roots (not exclusive now) + * !B: cur_new_roots == nr_new_roots (possible exclusive now) + * + * Results: + * +: Possible sharing -> exclusive -: Possible exclusive -> sharing + * *: Definitely not changed. **: Possible unchanged. + * + * For !A and !B condition, the exception is cur_old/new_roots == 0 case. + * + * To make the logic clear, we first use condition A and B to split + * combination into 4 results. + * + * Then, for result "+" and "-", check old/new_roots == 0 case, as in them + * only on variant maybe 0. + * + * Lastly, check result **, since there are 2 variants maybe 0, split them + * again(2x2). + * But this time we don't need to consider other things, the codes and logic + * is easy to understand now. + */ +static void qgroup_update_counters(struct btrfs_fs_info *fs_info, + struct list_head *qgroups, u64 nr_old_roots, + u64 nr_new_roots, u64 num_bytes, u64 seq) +{ + struct btrfs_qgroup *qg; + + list_for_each_entry(qg, qgroups, nested_iterator) { + u64 cur_new_count, cur_old_count; + bool dirty = false; + + cur_old_count = btrfs_qgroup_get_old_refcnt(qg, seq); + cur_new_count = btrfs_qgroup_get_new_refcnt(qg, seq); + + trace_btrfs_qgroup_update_counters(fs_info, qg, cur_old_count, + cur_new_count); + + /* Rfer update part */ + if (cur_old_count == 0 && cur_new_count > 0) { + qg->rfer += num_bytes; + qg->rfer_cmpr += num_bytes; + dirty = true; + } + if (cur_old_count > 0 && cur_new_count == 0) { + qg->rfer -= num_bytes; + qg->rfer_cmpr -= num_bytes; + dirty = true; + } + + /* Excl update part */ + /* Exclusive/none -> shared case */ + if (cur_old_count == nr_old_roots && + cur_new_count < nr_new_roots) { + /* Exclusive -> shared */ + if (cur_old_count != 0) { + qg->excl -= num_bytes; + qg->excl_cmpr -= num_bytes; + dirty = true; + } + } + + /* Shared -> exclusive/none case */ + if (cur_old_count < nr_old_roots && + cur_new_count == nr_new_roots) { + /* Shared->exclusive */ + if (cur_new_count != 0) { + qg->excl += num_bytes; + qg->excl_cmpr += num_bytes; + dirty = true; + } + } + + /* Exclusive/none -> exclusive/none case */ + if (cur_old_count == nr_old_roots && + cur_new_count == nr_new_roots) { + if (cur_old_count == 0) { + /* None -> exclusive/none */ + + if (cur_new_count != 0) { + /* None -> exclusive */ + qg->excl += num_bytes; + qg->excl_cmpr += num_bytes; + dirty = true; + } + /* None -> none, nothing changed */ + } else { + /* Exclusive -> exclusive/none */ + + if (cur_new_count == 0) { + /* Exclusive -> none */ + qg->excl -= num_bytes; + qg->excl_cmpr -= num_bytes; + dirty = true; + } + /* Exclusive -> exclusive, nothing changed */ + } + } + + if (dirty) + qgroup_dirty(fs_info, qg); + } +} + +/* + * Check if the @roots potentially is a list of fs tree roots + * + * Return 0 for definitely not a fs/subvol tree roots ulist + * Return 1 for possible fs/subvol tree roots in the list (considering an empty + * one as well) + */ +static int maybe_fs_roots(struct ulist *roots) +{ + struct ulist_node *unode; + struct ulist_iterator uiter; + + /* Empty one, still possible for fs roots */ + if (!roots || roots->nnodes == 0) + return 1; + + ULIST_ITER_INIT(&uiter); + unode = ulist_next(roots, &uiter); + if (!unode) + return 1; + + /* + * If it contains fs tree roots, then it must belong to fs/subvol + * trees. + * If it contains a non-fs tree, it won't be shared with fs/subvol trees. + */ + return btrfs_is_fstree(unode->val); +} + +int btrfs_qgroup_account_extent(struct btrfs_trans_handle *trans, u64 bytenr, + u64 num_bytes, struct ulist *old_roots, + struct ulist *new_roots) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + LIST_HEAD(qgroups); + u64 seq; + u64 nr_new_roots = 0; + u64 nr_old_roots = 0; + int ret = 0; + + /* + * If quotas get disabled meanwhile, the resources need to be freed and + * we can't just exit here. + */ + if (!btrfs_qgroup_full_accounting(fs_info) || + fs_info->qgroup_flags & BTRFS_QGROUP_RUNTIME_FLAG_NO_ACCOUNTING) + goto out_free; + + if (new_roots) { + if (!maybe_fs_roots(new_roots)) + goto out_free; + nr_new_roots = new_roots->nnodes; + } + if (old_roots) { + if (!maybe_fs_roots(old_roots)) + goto out_free; + nr_old_roots = old_roots->nnodes; + } + + /* Quick exit, either not fs tree roots, or won't affect any qgroup */ + if (nr_old_roots == 0 && nr_new_roots == 0) + goto out_free; + + trace_btrfs_qgroup_account_extent(fs_info, trans->transid, bytenr, + num_bytes, nr_old_roots, nr_new_roots); + + mutex_lock(&fs_info->qgroup_rescan_lock); + if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_RESCAN) { + if (fs_info->qgroup_rescan_progress.objectid <= bytenr) { + mutex_unlock(&fs_info->qgroup_rescan_lock); + ret = 0; + goto out_free; + } + } + mutex_unlock(&fs_info->qgroup_rescan_lock); + + spin_lock(&fs_info->qgroup_lock); + seq = fs_info->qgroup_seq; + + /* Update old refcnts using old_roots */ + qgroup_update_refcnt(fs_info, old_roots, &qgroups, seq, true); + + /* Update new refcnts using new_roots */ + qgroup_update_refcnt(fs_info, new_roots, &qgroups, seq, false); + + qgroup_update_counters(fs_info, &qgroups, nr_old_roots, nr_new_roots, + num_bytes, seq); + + /* + * We're done using the iterator, release all its qgroups while holding + * fs_info->qgroup_lock so that we don't race with btrfs_remove_qgroup() + * and trigger use-after-free accesses to qgroups. + */ + qgroup_iterator_nested_clean(&qgroups); + + /* + * Bump qgroup_seq to avoid seq overlap + */ + fs_info->qgroup_seq += max(nr_old_roots, nr_new_roots) + 1; + spin_unlock(&fs_info->qgroup_lock); +out_free: + ulist_free(old_roots); + ulist_free(new_roots); + return ret; +} + +int btrfs_qgroup_account_extents(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_qgroup_extent_record *record; + struct btrfs_delayed_ref_root *delayed_refs; + struct ulist *new_roots = NULL; + unsigned long index; + u64 num_dirty_extents = 0; + u64 qgroup_to_skip; + int ret = 0; + + if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE) + return 0; + + delayed_refs = &trans->transaction->delayed_refs; + qgroup_to_skip = delayed_refs->qgroup_to_skip; + xa_for_each(&delayed_refs->dirty_extents, index, record) { + const u64 bytenr = (((u64)index) << fs_info->sectorsize_bits); + + num_dirty_extents++; + trace_btrfs_qgroup_account_extents(fs_info, record, bytenr); + + if (!ret && !(fs_info->qgroup_flags & + BTRFS_QGROUP_RUNTIME_FLAG_NO_ACCOUNTING)) { + struct btrfs_backref_walk_ctx ctx = { 0 }; + + ctx.bytenr = bytenr; + ctx.fs_info = fs_info; + + /* + * Old roots should be searched when inserting qgroup + * extent record. + * + * But for INCONSISTENT (NO_ACCOUNTING) -> rescan case, + * we may have some record inserted during + * NO_ACCOUNTING (thus no old_roots populated), but + * later we start rescan, which clears NO_ACCOUNTING, + * leaving some inserted records without old_roots + * populated. + * + * Those cases are rare and should not cause too much + * time spent during commit_transaction(). + */ + if (!record->old_roots) { + /* Search commit root to find old_roots */ + ret = btrfs_find_all_roots(&ctx, false); + if (ret < 0) + goto cleanup; + record->old_roots = ctx.roots; + ctx.roots = NULL; + } + + /* + * Use BTRFS_SEQ_LAST as time_seq to do special search, + * which doesn't lock tree or delayed_refs and search + * current root. It's safe inside commit_transaction(). + */ + ctx.trans = trans; + ctx.time_seq = BTRFS_SEQ_LAST; + ret = btrfs_find_all_roots(&ctx, false); + if (ret < 0) + goto cleanup; + new_roots = ctx.roots; + if (qgroup_to_skip) { + ulist_del(new_roots, qgroup_to_skip, 0); + ulist_del(record->old_roots, qgroup_to_skip, + 0); + } + ret = btrfs_qgroup_account_extent(trans, bytenr, + record->num_bytes, + record->old_roots, + new_roots); + record->old_roots = NULL; + new_roots = NULL; + } + /* Free the reserved data space */ + btrfs_qgroup_free_refroot(fs_info, + record->data_rsv_refroot, + record->data_rsv, + BTRFS_QGROUP_RSV_DATA); +cleanup: + ulist_free(record->old_roots); + ulist_free(new_roots); + new_roots = NULL; + xa_erase(&delayed_refs->dirty_extents, index); + kfree(record); + + } + trace_btrfs_qgroup_num_dirty_extents(fs_info, trans->transid, num_dirty_extents); + return ret; +} + +/* + * Writes all changed qgroups to disk. + * Called by the transaction commit path and the qgroup assign ioctl. + */ +int btrfs_run_qgroups(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + int ret = 0; + + /* + * In case we are called from the qgroup assign ioctl, assert that we + * are holding the qgroup_ioctl_lock, otherwise we can race with a quota + * disable operation (ioctl) and access a freed quota root. + */ + if (trans->transaction->state != TRANS_STATE_COMMIT_DOING) + lockdep_assert_held(&fs_info->qgroup_ioctl_lock); + + if (!fs_info->quota_root) + return ret; + + spin_lock(&fs_info->qgroup_lock); + while (!list_empty(&fs_info->dirty_qgroups)) { + struct btrfs_qgroup *qgroup; + qgroup = list_first_entry(&fs_info->dirty_qgroups, + struct btrfs_qgroup, dirty); + list_del_init(&qgroup->dirty); + spin_unlock(&fs_info->qgroup_lock); + ret = update_qgroup_info_item(trans, qgroup); + if (ret) + qgroup_mark_inconsistent(fs_info, + "qgroup info item update error %d", ret); + ret = update_qgroup_limit_item(trans, qgroup); + if (ret) + qgroup_mark_inconsistent(fs_info, + "qgroup limit item update error %d", ret); + spin_lock(&fs_info->qgroup_lock); + } + if (btrfs_qgroup_enabled(fs_info)) + fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_ON; + else + fs_info->qgroup_flags &= ~BTRFS_QGROUP_STATUS_FLAG_ON; + spin_unlock(&fs_info->qgroup_lock); + + ret = update_qgroup_status_item(trans); + if (ret) + qgroup_mark_inconsistent(fs_info, + "qgroup status item update error %d", ret); + + return ret; +} + +int btrfs_qgroup_check_inherit(struct btrfs_fs_info *fs_info, + struct btrfs_qgroup_inherit *inherit, + size_t size) +{ + if (inherit->flags & ~BTRFS_QGROUP_INHERIT_FLAGS_SUPP) + return -EOPNOTSUPP; + if (size < sizeof(*inherit) || size > PAGE_SIZE) + return -EINVAL; + + /* + * In the past we allowed btrfs_qgroup_inherit to specify to copy + * rfer/excl numbers directly from other qgroups. This behavior has + * been disabled in userspace for a very long time, but here we should + * also disable it in kernel, as this behavior is known to mark qgroup + * inconsistent, and a rescan would wipe out the changes anyway. + * + * Reject any btrfs_qgroup_inherit with num_ref_copies or num_excl_copies. + */ + if (inherit->num_ref_copies > 0 || inherit->num_excl_copies > 0) + return -EINVAL; + + if (size != struct_size(inherit, qgroups, inherit->num_qgroups)) + return -EINVAL; + + /* + * Skip the inherit source qgroups check if qgroup is not enabled. + * Qgroup can still be later enabled causing problems, but in that case + * btrfs_qgroup_inherit() would just ignore those invalid ones. + */ + if (!btrfs_qgroup_enabled(fs_info)) + return 0; + + /* + * Now check all the remaining qgroups, they should all: + * + * - Exist + * - Be higher level qgroups. + */ + for (int i = 0; i < inherit->num_qgroups; i++) { + struct btrfs_qgroup *qgroup; + u64 qgroupid = inherit->qgroups[i]; + + if (btrfs_qgroup_level(qgroupid) == 0) + return -EINVAL; + + spin_lock(&fs_info->qgroup_lock); + qgroup = find_qgroup_rb(fs_info, qgroupid); + if (!qgroup) { + spin_unlock(&fs_info->qgroup_lock); + return -ENOENT; + } + spin_unlock(&fs_info->qgroup_lock); + } + return 0; +} + +static int qgroup_auto_inherit(struct btrfs_fs_info *fs_info, + u64 inode_rootid, + struct btrfs_qgroup_inherit **inherit) +{ + int i = 0; + u64 num_qgroups = 0; + struct btrfs_qgroup *inode_qg; + struct btrfs_qgroup_list *qg_list; + struct btrfs_qgroup_inherit *res; + size_t struct_sz; + u64 *qgids; + + if (*inherit) + return -EEXIST; + + inode_qg = find_qgroup_rb(fs_info, inode_rootid); + if (!inode_qg) + return -ENOENT; + + num_qgroups = list_count_nodes(&inode_qg->groups); + + if (!num_qgroups) + return 0; + + struct_sz = struct_size(res, qgroups, num_qgroups); + if (struct_sz == SIZE_MAX) + return -ERANGE; + + res = kzalloc(struct_sz, GFP_NOFS); + if (!res) + return -ENOMEM; + res->num_qgroups = num_qgroups; + qgids = res->qgroups; + + list_for_each_entry(qg_list, &inode_qg->groups, next_group) + qgids[i++] = qg_list->group->qgroupid; + + *inherit = res; + return 0; +} + +/* + * Check if we can skip rescan when inheriting qgroups. If @src has a single + * @parent, and that @parent is owning all its bytes exclusively, we can skip + * the full rescan, by just adding nodesize to the @parent's excl/rfer. + * + * Return <0 for fatal errors (like srcid/parentid has no qgroup). + * Return 0 if a quick inherit is done. + * Return >0 if a quick inherit is not possible, and a full rescan is needed. + */ +static int qgroup_snapshot_quick_inherit(struct btrfs_fs_info *fs_info, + u64 srcid, u64 parentid) +{ + struct btrfs_qgroup *src; + struct btrfs_qgroup *parent; + struct btrfs_qgroup *qgroup; + struct btrfs_qgroup_list *list; + LIST_HEAD(qgroup_list); + const u32 nodesize = fs_info->nodesize; + int nr_parents = 0; + + if (btrfs_qgroup_mode(fs_info) != BTRFS_QGROUP_MODE_FULL) + return 0; + + src = find_qgroup_rb(fs_info, srcid); + if (!src) + return -ENOENT; + parent = find_qgroup_rb(fs_info, parentid); + if (!parent) + return -ENOENT; + + /* + * Source has no parent qgroup, but our new qgroup would have one. + * Qgroup numbers would become inconsistent. + */ + if (list_empty(&src->groups)) + return 1; + + list_for_each_entry(list, &src->groups, next_group) { + /* The parent is not the same, quick update is not possible. */ + if (list->group->qgroupid != parentid) + return 1; + nr_parents++; + /* + * More than one parent qgroup, we can't be sure about accounting + * consistency. + */ + if (nr_parents > 1) + return 1; + } + + /* + * The parent is not exclusively owning all its bytes. We're not sure + * if the source has any bytes not fully owned by the parent. + */ + if (parent->excl != parent->rfer) + return 1; + + qgroup_iterator_add(&qgroup_list, parent); + list_for_each_entry(qgroup, &qgroup_list, iterator) { + qgroup->rfer += nodesize; + qgroup->rfer_cmpr += nodesize; + qgroup->excl += nodesize; + qgroup->excl_cmpr += nodesize; + qgroup_dirty(fs_info, qgroup); + + /* Append parent qgroups to @qgroup_list. */ + list_for_each_entry(list, &qgroup->groups, next_group) + qgroup_iterator_add(&qgroup_list, list->group); + } + qgroup_iterator_clean(&qgroup_list); + return 0; +} + +/* + * Copy the accounting information between qgroups. This is necessary + * when a snapshot or a subvolume is created. Throwing an error will + * cause a transaction abort so we take extra care here to only error + * when a readonly fs is a reasonable outcome. + */ +int btrfs_qgroup_inherit(struct btrfs_trans_handle *trans, u64 srcid, + u64 objectid, u64 inode_rootid, + struct btrfs_qgroup_inherit *inherit) +{ + int ret = 0; + u64 *i_qgroups; + bool committing = false; + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *quota_root; + struct btrfs_qgroup *srcgroup; + struct btrfs_qgroup *dstgroup; + struct btrfs_qgroup *prealloc; + struct btrfs_qgroup_list **qlist_prealloc = NULL; + bool free_inherit = false; + bool need_rescan = false; + u32 level_size = 0; + u64 nums; + + if (!btrfs_qgroup_enabled(fs_info)) + return 0; + + prealloc = kzalloc_obj(*prealloc, GFP_NOFS); + if (!prealloc) + return -ENOMEM; + + /* + * There are only two callers of this function. + * + * One in create_subvol() in the ioctl context, which needs to hold + * the qgroup_ioctl_lock. + * + * The other one in create_pending_snapshot() where no other qgroup + * code can modify the fs as they all need to either start a new trans + * or hold a trans handler, thus we don't need to hold + * qgroup_ioctl_lock. + * This would avoid long and complex lock chain and make lockdep happy. + */ + spin_lock(&fs_info->trans_lock); + if (trans->transaction->state == TRANS_STATE_COMMIT_DOING) + committing = true; + spin_unlock(&fs_info->trans_lock); + + if (!committing) + mutex_lock(&fs_info->qgroup_ioctl_lock); + + quota_root = fs_info->quota_root; + if (!quota_root) { + ret = -EINVAL; + goto out; + } + + if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE && !inherit) { + ret = qgroup_auto_inherit(fs_info, inode_rootid, &inherit); + if (ret) + goto out; + free_inherit = true; + } + + if (inherit) { + i_qgroups = (u64 *)(inherit + 1); + nums = inherit->num_qgroups + 2 * inherit->num_ref_copies + + 2 * inherit->num_excl_copies; + for (int i = 0; i < nums; i++) { + srcgroup = find_qgroup_rb(fs_info, *i_qgroups); + + /* + * Zero out invalid groups so we can ignore + * them later. + */ + if (!srcgroup || + ((srcgroup->qgroupid >> 48) <= (objectid >> 48))) + *i_qgroups = 0ULL; + + ++i_qgroups; + } + } + + /* + * create a tracking group for the subvol itself + */ + ret = add_qgroup_item(trans, quota_root, objectid); + if (ret) + goto out; + + /* + * add qgroup to all inherited groups + */ + if (inherit) { + i_qgroups = (u64 *)(inherit + 1); + for (int i = 0; i < inherit->num_qgroups; i++, i_qgroups++) { + if (*i_qgroups == 0) + continue; + ret = add_qgroup_relation_item(trans, objectid, + *i_qgroups); + if (ret && ret != -EEXIST) + goto out; + ret = add_qgroup_relation_item(trans, *i_qgroups, + objectid); + if (ret && ret != -EEXIST) + goto out; + } + ret = 0; + + qlist_prealloc = kzalloc_objs(struct btrfs_qgroup_list *, + inherit->num_qgroups, GFP_NOFS); + if (!qlist_prealloc) { + ret = -ENOMEM; + goto out; + } + for (int i = 0; i < inherit->num_qgroups; i++) { + qlist_prealloc[i] = kzalloc_obj(struct btrfs_qgroup_list, + GFP_NOFS); + if (!qlist_prealloc[i]) { + ret = -ENOMEM; + goto out; + } + } + } + + spin_lock(&fs_info->qgroup_lock); + + dstgroup = add_qgroup_rb(fs_info, prealloc, objectid); + prealloc = NULL; + + if (inherit && inherit->flags & BTRFS_QGROUP_INHERIT_SET_LIMITS) { + dstgroup->lim_flags = inherit->lim.flags; + dstgroup->max_rfer = inherit->lim.max_rfer; + dstgroup->max_excl = inherit->lim.max_excl; + dstgroup->rsv_rfer = inherit->lim.rsv_rfer; + dstgroup->rsv_excl = inherit->lim.rsv_excl; + + qgroup_dirty(fs_info, dstgroup); + } + + if (srcid && btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_FULL) { + srcgroup = find_qgroup_rb(fs_info, srcid); + if (!srcgroup) + goto unlock; + + /* + * We call inherit after we clone the root in order to make sure + * our counts don't go crazy, so at this point the only + * difference between the two roots should be the root node. + */ + level_size = fs_info->nodesize; + dstgroup->rfer = srcgroup->rfer; + dstgroup->rfer_cmpr = srcgroup->rfer_cmpr; + dstgroup->excl = level_size; + dstgroup->excl_cmpr = level_size; + srcgroup->excl = level_size; + srcgroup->excl_cmpr = level_size; + + /* inherit the limit info */ + dstgroup->lim_flags = srcgroup->lim_flags; + dstgroup->max_rfer = srcgroup->max_rfer; + dstgroup->max_excl = srcgroup->max_excl; + dstgroup->rsv_rfer = srcgroup->rsv_rfer; + dstgroup->rsv_excl = srcgroup->rsv_excl; + + qgroup_dirty(fs_info, dstgroup); + qgroup_dirty(fs_info, srcgroup); + + /* + * If the source qgroup has parent but the new one doesn't, + * we need a full rescan. + */ + if (!inherit && !list_empty(&srcgroup->groups)) + need_rescan = true; + } + + if (!inherit) + goto unlock; + + i_qgroups = (u64 *)(inherit + 1); + for (int i = 0; i < inherit->num_qgroups; i++) { + if (*i_qgroups) { + ret = add_relation_rb(fs_info, qlist_prealloc[i], objectid, + *i_qgroups); + qlist_prealloc[i] = NULL; + if (ret) + goto unlock; + } + if (srcid) { + /* Check if we can do a quick inherit. */ + ret = qgroup_snapshot_quick_inherit(fs_info, srcid, *i_qgroups); + if (ret < 0) + goto unlock; + if (ret > 0) + need_rescan = true; + ret = 0; + } + ++i_qgroups; + } + + for (int i = 0; i < inherit->num_ref_copies; i++, i_qgroups += 2) { + struct btrfs_qgroup *src; + struct btrfs_qgroup *dst; + + if (!i_qgroups[0] || !i_qgroups[1]) + continue; + + src = find_qgroup_rb(fs_info, i_qgroups[0]); + dst = find_qgroup_rb(fs_info, i_qgroups[1]); + + if (!src || !dst) { + ret = -EINVAL; + goto unlock; + } + + dst->rfer = src->rfer - level_size; + dst->rfer_cmpr = src->rfer_cmpr - level_size; + + /* Manually tweaking numbers certainly needs a rescan */ + need_rescan = true; + } + for (int i = 0; i < inherit->num_excl_copies; i++, i_qgroups += 2) { + struct btrfs_qgroup *src; + struct btrfs_qgroup *dst; + + if (!i_qgroups[0] || !i_qgroups[1]) + continue; + + src = find_qgroup_rb(fs_info, i_qgroups[0]); + dst = find_qgroup_rb(fs_info, i_qgroups[1]); + + if (!src || !dst) { + ret = -EINVAL; + goto unlock; + } + + dst->excl = src->excl + level_size; + dst->excl_cmpr = src->excl_cmpr + level_size; + need_rescan = true; + } + +unlock: + spin_unlock(&fs_info->qgroup_lock); + if (!ret) + ret = btrfs_sysfs_add_one_qgroup(fs_info, dstgroup); +out: + if (!committing) + mutex_unlock(&fs_info->qgroup_ioctl_lock); + if (need_rescan) + qgroup_mark_inconsistent(fs_info, "qgroup inherit needs a rescan"); + if (qlist_prealloc) { + for (int i = 0; i < inherit->num_qgroups; i++) + kfree(qlist_prealloc[i]); + kfree(qlist_prealloc); + } + if (free_inherit) + kfree(inherit); + kfree(prealloc); + return ret; +} + +static bool qgroup_check_limits(const struct btrfs_qgroup *qg, u64 num_bytes) +{ + if ((qg->lim_flags & BTRFS_QGROUP_LIMIT_MAX_RFER) && + qgroup_rsv_total(qg) + (s64)qg->rfer + num_bytes > qg->max_rfer) + return false; + + if ((qg->lim_flags & BTRFS_QGROUP_LIMIT_MAX_EXCL) && + qgroup_rsv_total(qg) + (s64)qg->excl + num_bytes > qg->max_excl) + return false; + + return true; +} + +static int qgroup_reserve(struct btrfs_root *root, u64 num_bytes, bool enforce, + enum btrfs_qgroup_rsv_type type) +{ + struct btrfs_qgroup *qgroup; + struct btrfs_fs_info *fs_info = root->fs_info; + u64 ref_root = btrfs_root_id(root); + int ret = 0; + LIST_HEAD(qgroup_list); + + if (!btrfs_is_fstree(ref_root)) + return 0; + + if (num_bytes == 0) + return 0; + + if (test_bit(BTRFS_FS_QUOTA_OVERRIDE, &fs_info->flags) && + capable(CAP_SYS_RESOURCE)) + enforce = false; + + spin_lock(&fs_info->qgroup_lock); + if (!fs_info->quota_root) + goto out; + + qgroup = find_qgroup_rb(fs_info, ref_root); + if (!qgroup) + goto out; + + qgroup_iterator_add(&qgroup_list, qgroup); + list_for_each_entry(qgroup, &qgroup_list, iterator) { + struct btrfs_qgroup_list *glist; + + if (enforce && !qgroup_check_limits(qgroup, num_bytes)) { + ret = -EDQUOT; + goto out; + } + + list_for_each_entry(glist, &qgroup->groups, next_group) + qgroup_iterator_add(&qgroup_list, glist->group); + } + + ret = 0; + /* + * no limits exceeded, now record the reservation into all qgroups + */ + list_for_each_entry(qgroup, &qgroup_list, iterator) + qgroup_rsv_add(fs_info, qgroup, num_bytes, type); + +out: + qgroup_iterator_clean(&qgroup_list); + spin_unlock(&fs_info->qgroup_lock); + return ret; +} + +/* + * Free @num_bytes of reserved space with @type for qgroup. (Normally level 0 + * qgroup). + * + * Will handle all higher level qgroup too. + * + * NOTE: If @num_bytes is (u64)-1, this means to free all bytes of this qgroup. + * This special case is only used for META_PERTRANS type. + */ +void btrfs_qgroup_free_refroot(struct btrfs_fs_info *fs_info, + u64 ref_root, u64 num_bytes, + enum btrfs_qgroup_rsv_type type) +{ + struct btrfs_qgroup *qgroup; + LIST_HEAD(qgroup_list); + + if (!btrfs_is_fstree(ref_root)) + return; + + if (num_bytes == 0) + return; + + if (num_bytes == (u64)-1 && type != BTRFS_QGROUP_RSV_META_PERTRANS) { + WARN(1, "%s: Invalid type to free", __func__); + return; + } + spin_lock(&fs_info->qgroup_lock); + + if (!fs_info->quota_root) + goto out; + + qgroup = find_qgroup_rb(fs_info, ref_root); + if (!qgroup) + goto out; + + if (num_bytes == (u64)-1) + /* + * We're freeing all pertrans rsv, get reserved value from + * level 0 qgroup as real num_bytes to free. + */ + num_bytes = qgroup->rsv.values[type]; + + qgroup_iterator_add(&qgroup_list, qgroup); + list_for_each_entry(qgroup, &qgroup_list, iterator) { + struct btrfs_qgroup_list *glist; + + qgroup_rsv_release(fs_info, qgroup, num_bytes, type); + list_for_each_entry(glist, &qgroup->groups, next_group) { + qgroup_iterator_add(&qgroup_list, glist->group); + } + } +out: + qgroup_iterator_clean(&qgroup_list); + spin_unlock(&fs_info->qgroup_lock); +} + +/* + * Check if the leaf is the last leaf. Which means all node pointers + * are at their last position. + */ +static bool is_last_leaf(struct btrfs_path *path) +{ + int i; + + for (i = 1; i < BTRFS_MAX_LEVEL && path->nodes[i]; i++) { + if (path->slots[i] != btrfs_header_nritems(path->nodes[i]) - 1) + return false; + } + return true; +} + +/* + * returns < 0 on error, 0 when more leafs are to be scanned. + * returns 1 when done. + */ +static int qgroup_rescan_leaf(struct btrfs_trans_handle *trans, + struct btrfs_path *path) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *extent_root; + struct btrfs_key found; + struct extent_buffer *scratch_leaf = NULL; + u64 num_bytes; + bool done; + int slot; + int ret; + + if (!btrfs_qgroup_full_accounting(fs_info)) + return 1; + + mutex_lock(&fs_info->qgroup_rescan_lock); + extent_root = btrfs_extent_root(fs_info, + fs_info->qgroup_rescan_progress.objectid); + if (unlikely(!extent_root)) { + btrfs_err(fs_info, + "missing extent root for extent at bytenr %llu", + fs_info->qgroup_rescan_progress.objectid); + mutex_unlock(&fs_info->qgroup_rescan_lock); + return -EUCLEAN; + } + + ret = btrfs_search_slot_for_read(extent_root, + &fs_info->qgroup_rescan_progress, + path, 1, 0); + + btrfs_debug(fs_info, + "current progress key " BTRFS_KEY_FMT ", search_slot ret %d", + BTRFS_KEY_FMT_VALUE(&fs_info->qgroup_rescan_progress), ret); + + if (ret) { + /* + * The rescan is about to end, we will not be scanning any + * further blocks. We cannot unset the RESCAN flag here, because + * we want to commit the transaction if everything went well. + * To make the live accounting work in this phase, we set our + * scan progress pointer such that every real extent objectid + * will be smaller. + */ + fs_info->qgroup_rescan_progress.objectid = (u64)-1; + btrfs_release_path(path); + mutex_unlock(&fs_info->qgroup_rescan_lock); + return ret; + } + done = is_last_leaf(path); + + btrfs_item_key_to_cpu(path->nodes[0], &found, + btrfs_header_nritems(path->nodes[0]) - 1); + fs_info->qgroup_rescan_progress.objectid = found.objectid + 1; + + scratch_leaf = btrfs_clone_extent_buffer(path->nodes[0]); + if (!scratch_leaf) { + ret = -ENOMEM; + mutex_unlock(&fs_info->qgroup_rescan_lock); + goto out; + } + slot = path->slots[0]; + btrfs_release_path(path); + mutex_unlock(&fs_info->qgroup_rescan_lock); + + for (; slot < btrfs_header_nritems(scratch_leaf); ++slot) { + struct btrfs_backref_walk_ctx ctx = { 0 }; + + btrfs_item_key_to_cpu(scratch_leaf, &found, slot); + if (found.type != BTRFS_EXTENT_ITEM_KEY && + found.type != BTRFS_METADATA_ITEM_KEY) + continue; + if (found.type == BTRFS_METADATA_ITEM_KEY) + num_bytes = fs_info->nodesize; + else + num_bytes = found.offset; + + ctx.bytenr = found.objectid; + ctx.fs_info = fs_info; + + ret = btrfs_find_all_roots(&ctx, false); + if (ret < 0) + goto out; + /* For rescan, just pass old_roots as NULL */ + ret = btrfs_qgroup_account_extent(trans, found.objectid, + num_bytes, NULL, ctx.roots); + if (ret < 0) + goto out; + } +out: + if (scratch_leaf) + free_extent_buffer(scratch_leaf); + + if (done && !ret) { + ret = 1; + fs_info->qgroup_rescan_progress.objectid = (u64)-1; + } + return ret; +} + +static bool rescan_should_stop(struct btrfs_fs_info *fs_info) +{ + if (btrfs_fs_closing(fs_info)) + return true; + if (test_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state)) + return true; + if (!btrfs_qgroup_enabled(fs_info)) + return true; + if (fs_info->qgroup_flags & BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN) + return true; + return false; +} + +static void btrfs_qgroup_rescan_worker(struct btrfs_work *work) +{ + struct btrfs_fs_info *fs_info = container_of(work, struct btrfs_fs_info, + qgroup_rescan_work); + struct btrfs_path *path; + struct btrfs_trans_handle *trans = NULL; + int ret = 0; + bool stopped = false; + bool did_leaf_rescans = false; + + if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE) + return; + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto out; + } + /* + * Rescan should only search for commit root, and any later difference + * should be recorded by qgroup + */ + path->search_commit_root = true; + path->skip_locking = true; + + while (!ret && !(stopped = rescan_should_stop(fs_info))) { + trans = btrfs_start_transaction(fs_info->fs_root, 0); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + break; + } + + ret = qgroup_rescan_leaf(trans, path); + did_leaf_rescans = true; + + if (ret > 0) + btrfs_commit_transaction(trans); + else + btrfs_end_transaction(trans); + } + +out: + btrfs_free_path(path); + + mutex_lock(&fs_info->qgroup_rescan_lock); + if (ret > 0 && + fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT) { + fs_info->qgroup_flags &= ~BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT; + } else if (ret < 0 || stopped) { + fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT; + } + mutex_unlock(&fs_info->qgroup_rescan_lock); + + /* + * Only update status, since the previous part has already updated the + * qgroup info, and only if we did any actual work. This also prevents + * race with a concurrent quota disable, which has already set + * fs_info->quota_root to NULL and cleared BTRFS_FS_QUOTA_ENABLED at + * btrfs_quota_disable(). + */ + if (did_leaf_rescans) { + trans = btrfs_start_transaction(fs_info->quota_root, 1); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + trans = NULL; + btrfs_err(fs_info, + "fail to start transaction for status update: %pe", + ERR_PTR(ret)); + } + } else { + trans = NULL; + } + + mutex_lock(&fs_info->qgroup_rescan_lock); + if (!stopped || + fs_info->qgroup_flags & BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN) + fs_info->qgroup_flags &= ~BTRFS_QGROUP_STATUS_FLAG_RESCAN; + if (trans) { + int ret2 = update_qgroup_status_item(trans); + + if (ret2 < 0) { + ret = ret2; + btrfs_err(fs_info, "fail to update qgroup status: %pe", ERR_PTR(ret)); + } + } + fs_info->qgroup_rescan_running = false; + fs_info->qgroup_flags &= ~BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN; + complete_all(&fs_info->qgroup_rescan_completion); + mutex_unlock(&fs_info->qgroup_rescan_lock); + + if (!trans) + return; + + btrfs_end_transaction(trans); + + if (stopped) { + btrfs_info(fs_info, "qgroup scan paused"); + } else if (fs_info->qgroup_flags & BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN) { + btrfs_info(fs_info, "qgroup scan cancelled"); + } else if (ret >= 0) { + btrfs_info(fs_info, "qgroup scan completed%s", + ret > 0 ? " (inconsistency flag cleared)" : ""); + } else { + btrfs_err(fs_info, "qgroup scan failed with %pe", ERR_PTR(ret)); + } +} + +/* + * Checks that (a) no rescan is running and (b) quota is enabled. Allocates all + * memory required for the rescan context. + */ +static int +qgroup_rescan_init(struct btrfs_fs_info *fs_info, u64 progress_objectid, + int init_flags) +{ + int ret = 0; + + if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE) { + btrfs_warn(fs_info, "qgroup rescan init failed, running in simple mode"); + return -EINVAL; + } + + if (!init_flags) { + /* we're resuming qgroup rescan at mount time */ + if (!(fs_info->qgroup_flags & + BTRFS_QGROUP_STATUS_FLAG_RESCAN)) { + btrfs_debug(fs_info, + "qgroup rescan init failed, qgroup rescan is not queued"); + ret = -EINVAL; + } else if (!(fs_info->qgroup_flags & + BTRFS_QGROUP_STATUS_FLAG_ON)) { + btrfs_debug(fs_info, + "qgroup rescan init failed, qgroup is not enabled"); + ret = -ENOTCONN; + } + + if (ret) + return ret; + } + + mutex_lock(&fs_info->qgroup_rescan_lock); + + if (init_flags) { + if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_RESCAN) { + ret = -EINPROGRESS; + } else if (!(fs_info->qgroup_flags & + BTRFS_QGROUP_STATUS_FLAG_ON)) { + btrfs_debug(fs_info, + "qgroup rescan init failed, qgroup is not enabled"); + ret = -ENOTCONN; + } else if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_DISABLED) { + /* Quota disable is in progress */ + ret = -EBUSY; + } + + if (ret) { + mutex_unlock(&fs_info->qgroup_rescan_lock); + return ret; + } + fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_RESCAN; + } + + memset(&fs_info->qgroup_rescan_progress, 0, + sizeof(fs_info->qgroup_rescan_progress)); + fs_info->qgroup_flags &= ~(BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN | + BTRFS_QGROUP_RUNTIME_FLAG_NO_ACCOUNTING); + fs_info->qgroup_rescan_progress.objectid = progress_objectid; + init_completion(&fs_info->qgroup_rescan_completion); + mutex_unlock(&fs_info->qgroup_rescan_lock); + + btrfs_init_work(&fs_info->qgroup_rescan_work, + btrfs_qgroup_rescan_worker, NULL); + return 0; +} + +static void +qgroup_rescan_zero_tracking(struct btrfs_fs_info *fs_info) +{ + struct rb_node *n; + struct btrfs_qgroup *qgroup; + + spin_lock(&fs_info->qgroup_lock); + /* clear all current qgroup tracking information */ + for (n = rb_first(&fs_info->qgroup_tree); n; n = rb_next(n)) { + qgroup = rb_entry(n, struct btrfs_qgroup, node); + qgroup->rfer = 0; + qgroup->rfer_cmpr = 0; + qgroup->excl = 0; + qgroup->excl_cmpr = 0; + qgroup_dirty(fs_info, qgroup); + } + spin_unlock(&fs_info->qgroup_lock); +} + +int +btrfs_qgroup_rescan(struct btrfs_fs_info *fs_info) +{ + int ret = 0; + + ret = qgroup_rescan_init(fs_info, 0, 1); + if (ret) + return ret; + + /* + * We have set the rescan_progress to 0, which means no more + * delayed refs will be accounted by btrfs_qgroup_account_ref. + * However, btrfs_qgroup_account_ref may be right after its call + * to btrfs_find_all_roots, in which case it would still do the + * accounting. + * To solve this, we're committing the transaction, which will + * ensure we run all delayed refs and only after that, we are + * going to clear all tracking information for a clean start. + */ + + ret = btrfs_commit_current_transaction(fs_info->fs_root); + if (ret) { + fs_info->qgroup_flags &= ~BTRFS_QGROUP_STATUS_FLAG_RESCAN; + return ret; + } + + qgroup_rescan_zero_tracking(fs_info); + + mutex_lock(&fs_info->qgroup_rescan_lock); + /* + * The rescan worker is only for full accounting qgroups, check if it's + * enabled as it is pointless to queue it otherwise. A concurrent quota + * disable may also have just cleared BTRFS_FS_QUOTA_ENABLED. + */ + if (btrfs_qgroup_full_accounting(fs_info)) { + fs_info->qgroup_rescan_running = true; + btrfs_queue_work(fs_info->qgroup_rescan_workers, + &fs_info->qgroup_rescan_work); + } else { + ret = -ENOTCONN; + } + mutex_unlock(&fs_info->qgroup_rescan_lock); + + return ret; +} + +int btrfs_qgroup_wait_for_completion(struct btrfs_fs_info *fs_info, + bool interruptible) +{ + int running; + int ret = 0; + + mutex_lock(&fs_info->qgroup_rescan_lock); + running = fs_info->qgroup_rescan_running; + mutex_unlock(&fs_info->qgroup_rescan_lock); + + if (!running) + return 0; + + if (interruptible) + ret = wait_for_completion_interruptible( + &fs_info->qgroup_rescan_completion); + else + wait_for_completion(&fs_info->qgroup_rescan_completion); + + return ret; +} + +/* + * this is only called from open_ctree where we're still single threaded, thus + * locking is omitted here. + */ +void +btrfs_qgroup_rescan_resume(struct btrfs_fs_info *fs_info) +{ + if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_RESCAN) { + mutex_lock(&fs_info->qgroup_rescan_lock); + fs_info->qgroup_rescan_running = true; + btrfs_queue_work(fs_info->qgroup_rescan_workers, + &fs_info->qgroup_rescan_work); + mutex_unlock(&fs_info->qgroup_rescan_lock); + } +} + +#define rbtree_iterate_from_safe(node, next, start) \ + for (node = start; node && ({ next = rb_next(node); 1;}); node = next) + +static int qgroup_unreserve_range(struct btrfs_inode *inode, + struct extent_changeset *reserved, u64 start, + u64 len) +{ + struct rb_node *node; + struct rb_node *next; + struct ulist_node *entry; + int ret = 0; + + node = reserved->range_changed.root.rb_node; + if (!node) + return 0; + while (node) { + entry = rb_entry(node, struct ulist_node, rb_node); + if (entry->val < start) + node = node->rb_right; + else + node = node->rb_left; + } + + if (entry->val > start && rb_prev(&entry->rb_node)) + entry = rb_entry(rb_prev(&entry->rb_node), struct ulist_node, + rb_node); + + rbtree_iterate_from_safe(node, next, &entry->rb_node) { + u64 entry_start; + u64 entry_end; + u64 entry_len; + int clear_ret; + + entry = rb_entry(node, struct ulist_node, rb_node); + entry_start = entry->val; + entry_end = entry->aux; + entry_len = entry_end - entry_start + 1; + + if (entry_start >= start + len) + break; + if (entry_start + entry_len <= start) + continue; + /* + * Now the entry is in [start, start + len), revert the + * EXTENT_QGROUP_RESERVED bit. + */ + clear_ret = btrfs_clear_extent_bit(&inode->io_tree, entry_start, entry_end, + EXTENT_QGROUP_RESERVED, NULL); + if (!ret && clear_ret < 0) + ret = clear_ret; + + ulist_del(&reserved->range_changed, entry->val, entry->aux); + if (likely(reserved->bytes_changed >= entry_len)) { + reserved->bytes_changed -= entry_len; + } else { + WARN_ON(1); + reserved->bytes_changed = 0; + } + } + + return ret; +} + +/* + * Try to free some space for qgroup. + * + * For qgroup, there are only 3 ways to free qgroup space: + * - Flush nodatacow write + * Any nodatacow write will free its reserved data space at run_delalloc_range(). + * In theory, we should only flush nodatacow inodes, but it's not yet + * possible, so we need to flush the whole root. + * + * - Wait for ordered extents + * When ordered extents are finished, their reserved metadata is finally + * converted to per_trans status, which can be freed by later commit + * transaction. + * + * - Commit transaction + * This would free the meta_per_trans space. + * In theory this shouldn't provide much space, but any more qgroup space + * is needed. + */ +static int try_flush_qgroup(struct btrfs_root *root) +{ + int ret; + + /* Can't hold an open transaction or we run the risk of deadlocking. */ + ASSERT(current->journal_info == NULL); + if (WARN_ON(current->journal_info)) + return 0; + + /* + * We don't want to run flush again and again, so if there is a running + * one, we won't try to start a new flush, but exit directly. + */ + if (test_and_set_bit(BTRFS_ROOT_QGROUP_FLUSHING, &root->state)) { + wait_event(root->qgroup_flush_wait, + !test_bit(BTRFS_ROOT_QGROUP_FLUSHING, &root->state)); + return 0; + } + + ret = btrfs_start_delalloc_snapshot(root, true); + if (ret < 0) + goto out; + btrfs_wait_ordered_extents(root, U64_MAX, NULL); + + /* + * After waiting for ordered extents run delayed iputs in order to free + * space from unlinked files before committing the current transaction, + * as ordered extents may have been holding the last reference of an + * inode and they add a delayed iput when they complete. + */ + btrfs_run_delayed_iputs(root->fs_info); + btrfs_wait_on_delayed_iputs(root->fs_info); + + ret = btrfs_commit_current_transaction(root); +out: + clear_bit(BTRFS_ROOT_QGROUP_FLUSHING, &root->state); + wake_up(&root->qgroup_flush_wait); + return ret; +} + +static int qgroup_reserve_data(struct btrfs_inode *inode, + struct extent_changeset **reserved_ret, u64 start, + u64 len) +{ + struct btrfs_root *root = inode->root; + struct extent_changeset *reserved; + bool new_reserved = false; + u64 orig_reserved; + u64 to_reserve; + int ret; + + if (btrfs_qgroup_mode(root->fs_info) == BTRFS_QGROUP_MODE_DISABLED || + !btrfs_is_fstree(btrfs_root_id(root)) || len == 0) + return 0; + + /* @reserved parameter is mandatory for qgroup */ + if (WARN_ON(!reserved_ret)) + return -EINVAL; + if (!*reserved_ret) { + new_reserved = true; + *reserved_ret = extent_changeset_alloc(); + if (!*reserved_ret) + return -ENOMEM; + } + reserved = *reserved_ret; + /* Record already reserved space */ + orig_reserved = reserved->bytes_changed; + ret = btrfs_set_record_extent_bits(&inode->io_tree, start, + start + len - 1, EXTENT_QGROUP_RESERVED, + reserved); + + /* Newly reserved space */ + to_reserve = reserved->bytes_changed - orig_reserved; + trace_btrfs_qgroup_reserve_data(&inode->vfs_inode, start, len, + to_reserve, QGROUP_RESERVE); + if (ret < 0) + goto out; + ret = qgroup_reserve(root, to_reserve, true, BTRFS_QGROUP_RSV_DATA); + if (ret < 0) + goto cleanup; + + return ret; + +cleanup: + qgroup_unreserve_range(inode, reserved, start, len); +out: + if (new_reserved) { + extent_changeset_free(reserved); + *reserved_ret = NULL; + } + return ret; +} + +/* + * Reserve qgroup space for range [start, start + len). + * + * This function will either reserve space from related qgroups or do nothing + * if the range is already reserved. + * + * Return 0 for successful reservation + * Return <0 for error (including -EQUOT) + * + * NOTE: This function may sleep for memory allocation, dirty page flushing and + * commit transaction. So caller should not hold any dirty page locked. + */ +int btrfs_qgroup_reserve_data(struct btrfs_inode *inode, + struct extent_changeset **reserved_ret, u64 start, + u64 len) +{ + int ret; + + ret = qgroup_reserve_data(inode, reserved_ret, start, len); + if (ret <= 0 && ret != -EDQUOT) + return ret; + + ret = try_flush_qgroup(inode->root); + if (ret < 0) + return ret; + return qgroup_reserve_data(inode, reserved_ret, start, len); +} + +/* Free ranges specified by @reserved, normally in error path */ +static int qgroup_free_reserved_data(struct btrfs_inode *inode, + struct extent_changeset *reserved, + u64 start, u64 len, u64 *freed_ret) +{ + struct btrfs_root *root = inode->root; + struct ulist_node *unode; + struct ulist_iterator uiter; + struct extent_changeset changeset; + const u32 sectorsize = root->fs_info->sectorsize; + const u64 aligned_start = round_down(start, sectorsize); + const u64 aligned_len = round_up(start + len, sectorsize) - aligned_start; + u64 freed = 0; + int ret; + + extent_changeset_init_bytes_only(&changeset); + + ULIST_ITER_INIT(&uiter); + while ((unode = ulist_next(&reserved->range_changed, &uiter))) { + u64 range_start = unode->val; + /* unode->aux is the inclusive end */ + u64 range_len = unode->aux - range_start + 1; + u64 free_start; + u64 free_len; + + extent_changeset_release(&changeset); + + /* + * Only free the range within + * [aligned_start, aligned_start + aligned_len). + */ + if (range_start >= aligned_start + aligned_len || + range_start + range_len <= aligned_start) + continue; + free_start = max(range_start, aligned_start); + free_len = min(aligned_start + aligned_len, range_start + range_len) - + free_start; + /* + * TODO: To also modify reserved->ranges_reserved to reflect + * the modification. + * + * However as long as we free qgroup reserved according to + * EXTENT_QGROUP_RESERVED, we won't double free. + * So not need to rush. + */ + ret = btrfs_clear_record_extent_bits(&inode->io_tree, free_start, + free_start + free_len - 1, + EXTENT_QGROUP_RESERVED, + &changeset); + if (ret < 0) + goto out; + freed += changeset.bytes_changed; + } + btrfs_qgroup_free_refroot(root->fs_info, btrfs_root_id(root), freed, + BTRFS_QGROUP_RSV_DATA); + if (freed_ret) + *freed_ret = freed; + ret = 0; +out: + extent_changeset_release(&changeset); + return ret; +} + +static int __btrfs_qgroup_release_data(struct btrfs_inode *inode, + struct extent_changeset *reserved, u64 start, u64 len, + u64 *released, int free) +{ + struct extent_changeset changeset; + int trace_op = QGROUP_RELEASE; + int ret; + + if (btrfs_qgroup_mode(inode->root->fs_info) == BTRFS_QGROUP_MODE_DISABLED) { + return btrfs_clear_record_extent_bits(&inode->io_tree, start, + start + len - 1, + EXTENT_QGROUP_RESERVED, NULL); + } + + /* In release case, we shouldn't have @reserved */ + WARN_ON(!free && reserved); + if (free && reserved) + return qgroup_free_reserved_data(inode, reserved, start, len, released); + extent_changeset_init_bytes_only(&changeset); + ret = btrfs_clear_record_extent_bits(&inode->io_tree, start, start + len - 1, + EXTENT_QGROUP_RESERVED, &changeset); + if (ret < 0) + goto out; + + if (free) + trace_op = QGROUP_FREE; + trace_btrfs_qgroup_release_data(&inode->vfs_inode, start, len, + changeset.bytes_changed, trace_op); + if (free) + btrfs_qgroup_free_refroot(inode->root->fs_info, + btrfs_root_id(inode->root), + changeset.bytes_changed, BTRFS_QGROUP_RSV_DATA); + if (released) + *released = changeset.bytes_changed; +out: + extent_changeset_release(&changeset); + return ret; +} + +/* + * Free a reserved space range from io_tree and related qgroups + * + * Should be called when a range of pages get invalidated before reaching disk. + * Or for error cleanup case. + * if @reserved is given, only reserved range in [@start, @start + @len) will + * be freed. + * + * For data written to disk, use btrfs_qgroup_release_data(). + * + * NOTE: This function may sleep for memory allocation. + */ +int btrfs_qgroup_free_data(struct btrfs_inode *inode, + struct extent_changeset *reserved, + u64 start, u64 len, u64 *freed) +{ + return __btrfs_qgroup_release_data(inode, reserved, start, len, freed, 1); +} + +/* + * Release a reserved space range from io_tree only. + * + * Should be called when a range of pages get written to disk and corresponding + * FILE_EXTENT is inserted into corresponding root. + * + * Since new qgroup accounting framework will only update qgroup numbers at + * commit_transaction() time, its reserved space shouldn't be freed from + * related qgroups. + * + * But we should release the range from io_tree, to allow further write to be + * COWed. + * + * NOTE: This function may sleep for memory allocation. + */ +int btrfs_qgroup_release_data(struct btrfs_inode *inode, u64 start, u64 len, u64 *released) +{ + return __btrfs_qgroup_release_data(inode, NULL, start, len, released, 0); +} + +static void add_root_meta_rsv(struct btrfs_root *root, int num_bytes, + enum btrfs_qgroup_rsv_type type) +{ + if (type != BTRFS_QGROUP_RSV_META_PREALLOC && + type != BTRFS_QGROUP_RSV_META_PERTRANS) + return; + if (num_bytes == 0) + return; + + spin_lock(&root->qgroup_meta_rsv_lock); + if (type == BTRFS_QGROUP_RSV_META_PREALLOC) + root->qgroup_meta_rsv_prealloc += num_bytes; + else + root->qgroup_meta_rsv_pertrans += num_bytes; + spin_unlock(&root->qgroup_meta_rsv_lock); +} + +static int sub_root_meta_rsv(struct btrfs_root *root, int num_bytes, + enum btrfs_qgroup_rsv_type type) +{ + if (type != BTRFS_QGROUP_RSV_META_PREALLOC && + type != BTRFS_QGROUP_RSV_META_PERTRANS) + return 0; + if (num_bytes == 0) + return 0; + + spin_lock(&root->qgroup_meta_rsv_lock); + if (type == BTRFS_QGROUP_RSV_META_PREALLOC) { + num_bytes = min_t(u64, root->qgroup_meta_rsv_prealloc, + num_bytes); + root->qgroup_meta_rsv_prealloc -= num_bytes; + } else { + num_bytes = min_t(u64, root->qgroup_meta_rsv_pertrans, + num_bytes); + root->qgroup_meta_rsv_pertrans -= num_bytes; + } + spin_unlock(&root->qgroup_meta_rsv_lock); + return num_bytes; +} + +static int btrfs_qgroup_reserve_meta(struct btrfs_root *root, int num_bytes, + enum btrfs_qgroup_rsv_type type, bool enforce) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + int ret; + + if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_DISABLED || + !btrfs_is_fstree(btrfs_root_id(root)) || num_bytes == 0) + return 0; + + BUG_ON(num_bytes != round_down(num_bytes, fs_info->nodesize)); + trace_btrfs_qgroup_meta_reserve(root, (s64)num_bytes, type); + ret = qgroup_reserve(root, num_bytes, enforce, type); + if (ret < 0) + return ret; + /* + * Record what we have reserved into root. + * + * To avoid quota disabled->enabled underflow. + * In that case, we may try to free space we haven't reserved + * (since quota was disabled), so record what we reserved into root. + * And ensure later release won't underflow this number. + */ + add_root_meta_rsv(root, num_bytes, type); + return ret; +} + +int btrfs_qgroup_reserve_meta_prealloc(struct btrfs_root *root, int num_bytes, + bool enforce, bool noflush) +{ + int ret; + + ret = btrfs_qgroup_reserve_meta(root, num_bytes, + BTRFS_QGROUP_RSV_META_PREALLOC, enforce); + if ((ret <= 0 && ret != -EDQUOT) || noflush) + return ret; + + ret = try_flush_qgroup(root); + if (ret < 0) + return ret; + return btrfs_qgroup_reserve_meta(root, num_bytes, + BTRFS_QGROUP_RSV_META_PREALLOC, enforce); +} + +/* + * Per-transaction meta reservation should be all freed at transaction commit + * time + */ +void btrfs_qgroup_free_meta_all_pertrans(struct btrfs_root *root) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + + if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_DISABLED || + !btrfs_is_fstree(btrfs_root_id(root))) + return; + + /* TODO: Update trace point to handle such free */ + trace_btrfs_qgroup_meta_free_all_pertrans(root); + /* Special value -1 means to free all reserved space */ + btrfs_qgroup_free_refroot(fs_info, btrfs_root_id(root), (u64)-1, + BTRFS_QGROUP_RSV_META_PERTRANS); +} + +void btrfs_qgroup_free_meta_prealloc(struct btrfs_root *root, int num_bytes) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + + if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_DISABLED || + !btrfs_is_fstree(btrfs_root_id(root))) + return; + + /* + * reservation for META_PREALLOC can happen before quota is enabled, + * which can lead to underflow. + * Here ensure we will only free what we really have reserved. + */ + num_bytes = sub_root_meta_rsv(root, num_bytes, + BTRFS_QGROUP_RSV_META_PREALLOC); + BUG_ON(num_bytes != round_down(num_bytes, fs_info->nodesize)); + trace_btrfs_qgroup_meta_reserve(root, -(s64)num_bytes, + BTRFS_QGROUP_RSV_META_PREALLOC); + btrfs_qgroup_free_refroot(fs_info, btrfs_root_id(root), num_bytes, + BTRFS_QGROUP_RSV_META_PREALLOC); +} + +static void qgroup_convert_meta(struct btrfs_fs_info *fs_info, u64 ref_root, + int num_bytes) +{ + struct btrfs_qgroup *qgroup; + LIST_HEAD(qgroup_list); + + if (num_bytes == 0) + return; + if (!fs_info->quota_root) + return; + + spin_lock(&fs_info->qgroup_lock); + qgroup = find_qgroup_rb(fs_info, ref_root); + if (!qgroup) + goto out; + + qgroup_iterator_add(&qgroup_list, qgroup); + list_for_each_entry(qgroup, &qgroup_list, iterator) { + struct btrfs_qgroup_list *glist; + + qgroup_rsv_release(fs_info, qgroup, num_bytes, + BTRFS_QGROUP_RSV_META_PREALLOC); + if (!sb_rdonly(fs_info->sb)) + qgroup_rsv_add(fs_info, qgroup, num_bytes, + BTRFS_QGROUP_RSV_META_PERTRANS); + + list_for_each_entry(glist, &qgroup->groups, next_group) + qgroup_iterator_add(&qgroup_list, glist->group); + } +out: + qgroup_iterator_clean(&qgroup_list); + spin_unlock(&fs_info->qgroup_lock); +} + +/* + * Convert @num_bytes of META_PREALLOCATED reservation to META_PERTRANS. + * + * This is called when preallocated meta reservation needs to be used. + * Normally after btrfs_join_transaction() call. + */ +void btrfs_qgroup_convert_reserved_meta(struct btrfs_root *root, int num_bytes) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + + if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_DISABLED || + !btrfs_is_fstree(btrfs_root_id(root))) + return; + /* Same as btrfs_qgroup_free_meta_prealloc() */ + num_bytes = sub_root_meta_rsv(root, num_bytes, + BTRFS_QGROUP_RSV_META_PREALLOC); + trace_btrfs_qgroup_meta_convert(root, num_bytes); + qgroup_convert_meta(fs_info, btrfs_root_id(root), num_bytes); + if (!sb_rdonly(fs_info->sb)) + add_root_meta_rsv(root, num_bytes, BTRFS_QGROUP_RSV_META_PERTRANS); +} + +/* + * Check qgroup reserved space leaking, normally at destroy inode + * time + */ +void btrfs_qgroup_check_reserved_leak(struct btrfs_inode *inode) +{ + struct extent_changeset changeset; + struct ulist_node *unode; + struct ulist_iterator iter; + int ret; + + extent_changeset_init(&changeset); + ret = btrfs_clear_record_extent_bits(&inode->io_tree, 0, (u64)-1, + EXTENT_QGROUP_RESERVED, &changeset); + + WARN_ON(ret < 0); + if (WARN_ON(changeset.bytes_changed)) { + ASSERT(extent_changeset_tracks_ranges(&changeset)); + ULIST_ITER_INIT(&iter); + while ((unode = ulist_next(&changeset.range_changed, &iter))) { + btrfs_warn(inode->root->fs_info, + "leaking qgroup reserved space, ino: %llu, start: %llu, end: %llu", + btrfs_ino(inode), unode->val, unode->aux); + } + btrfs_qgroup_free_refroot(inode->root->fs_info, + btrfs_root_id(inode->root), + changeset.bytes_changed, BTRFS_QGROUP_RSV_DATA); + + } + extent_changeset_release(&changeset); +} + +void btrfs_qgroup_init_swapped_blocks( + struct btrfs_qgroup_swapped_blocks *swapped_blocks) +{ + int i; + + spin_lock_init(&swapped_blocks->lock); + for (i = 0; i < BTRFS_MAX_LEVEL; i++) + swapped_blocks->blocks[i] = RB_ROOT; + swapped_blocks->swapped = false; +} + +/* + * Delete all swapped blocks record of @root. + * Every record here means we skipped a full subtree scan for qgroup. + * + * Gets called when committing one transaction. + */ +void btrfs_qgroup_clean_swapped_blocks(struct btrfs_root *root) +{ + struct btrfs_qgroup_swapped_blocks *swapped_blocks; + int i; + + swapped_blocks = &root->swapped_blocks; + + spin_lock(&swapped_blocks->lock); + if (!swapped_blocks->swapped) + goto out; + for (i = 0; i < BTRFS_MAX_LEVEL; i++) { + struct rb_root *cur_root = &swapped_blocks->blocks[i]; + struct btrfs_qgroup_swapped_block *entry; + struct btrfs_qgroup_swapped_block *next; + + rbtree_postorder_for_each_entry_safe(entry, next, cur_root, + node) + kfree(entry); + swapped_blocks->blocks[i] = RB_ROOT; + } + swapped_blocks->swapped = false; +out: + spin_unlock(&swapped_blocks->lock); +} + +static int qgroup_swapped_block_bytenr_key_cmp(const void *key, const struct rb_node *node) +{ + const u64 *bytenr = key; + const struct btrfs_qgroup_swapped_block *block = rb_entry(node, + struct btrfs_qgroup_swapped_block, node); + + if (block->subvol_bytenr < *bytenr) + return -1; + else if (block->subvol_bytenr > *bytenr) + return 1; + + return 0; +} + +static int qgroup_swapped_block_bytenr_cmp(struct rb_node *new, const struct rb_node *existing) +{ + const struct btrfs_qgroup_swapped_block *new_block = rb_entry(new, + struct btrfs_qgroup_swapped_block, node); + + return qgroup_swapped_block_bytenr_key_cmp(&new_block->subvol_bytenr, existing); +} + +/* + * Add subtree roots record into @subvol_root. + * + * @subvol_root: tree root of the subvolume tree get swapped + * @bg: block group under balance + * @subvol_parent/slot: pointer to the subtree root in subvolume tree + * @reloc_parent/slot: pointer to the subtree root in reloc tree + * BOTH POINTERS ARE BEFORE TREE SWAP + * @last_snapshot: last snapshot generation of the subvolume tree + */ +int btrfs_qgroup_add_swapped_blocks(struct btrfs_root *subvol_root, + struct btrfs_block_group *bg, + struct extent_buffer *subvol_parent, int subvol_slot, + struct extent_buffer *reloc_parent, int reloc_slot, + u64 last_snapshot) +{ + struct btrfs_fs_info *fs_info = subvol_root->fs_info; + struct btrfs_qgroup_swapped_blocks *blocks = &subvol_root->swapped_blocks; + struct btrfs_qgroup_swapped_block *block; + struct rb_node *node; + int level = btrfs_header_level(subvol_parent) - 1; + int ret = 0; + + if (!btrfs_qgroup_full_accounting(fs_info)) + return 0; + + if (unlikely(btrfs_node_ptr_generation(subvol_parent, subvol_slot) > + btrfs_node_ptr_generation(reloc_parent, reloc_slot))) { + btrfs_err_rl(fs_info, + "%s: bad parameter order, subvol_gen=%llu reloc_gen=%llu", + __func__, + btrfs_node_ptr_generation(subvol_parent, subvol_slot), + btrfs_node_ptr_generation(reloc_parent, reloc_slot)); + return -EUCLEAN; + } + + block = kmalloc_obj(*block, GFP_NOFS); + if (!block) { + ret = -ENOMEM; + goto out; + } + + /* + * @reloc_parent/slot is still before swap, while @block is going to + * record the bytenr after swap, so we do the swap here. + */ + block->subvol_bytenr = btrfs_node_blockptr(reloc_parent, reloc_slot); + block->subvol_generation = btrfs_node_ptr_generation(reloc_parent, + reloc_slot); + block->reloc_bytenr = btrfs_node_blockptr(subvol_parent, subvol_slot); + block->reloc_generation = btrfs_node_ptr_generation(subvol_parent, + subvol_slot); + block->last_snapshot = last_snapshot; + block->level = level; + + /* + * If we have bg == NULL, we're called from btrfs_recover_relocation(), + * no one else can modify tree blocks thus we qgroup will not change + * no matter the value of trace_leaf. + */ + if (bg && bg->flags & BTRFS_BLOCK_GROUP_DATA) + block->trace_leaf = true; + else + block->trace_leaf = false; + btrfs_node_key_to_cpu(reloc_parent, &block->first_key, reloc_slot); + + /* Insert @block into @blocks */ + spin_lock(&blocks->lock); + node = rb_find_add(&block->node, &blocks->blocks[level], qgroup_swapped_block_bytenr_cmp); + if (node) { + struct btrfs_qgroup_swapped_block *entry; + + entry = rb_entry(node, struct btrfs_qgroup_swapped_block, node); + + if (unlikely(entry->subvol_generation != block->subvol_generation || + entry->reloc_bytenr != block->reloc_bytenr || + entry->reloc_generation != block->reloc_generation)) { + /* + * Duplicated but mismatch entry found. Shouldn't happen. + * Marking qgroup inconsistent should be enough for end + * users. + */ + DEBUG_WARN("duplicated but mismatched entry found"); + ret = -EEXIST; + } + kfree(block); + goto out_unlock; + } + blocks->swapped = true; +out_unlock: + spin_unlock(&blocks->lock); +out: + if (ret < 0) + qgroup_mark_inconsistent(fs_info, "%s error: %d", __func__, ret); + return ret; +} + +/* + * Check if the tree block is a subtree root, and if so do the needed + * delayed subtree trace for qgroup. + * + * This is called during btrfs_cow_block(). + */ +int btrfs_qgroup_trace_subtree_after_cow(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct extent_buffer *subvol_eb) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_tree_parent_check check = { 0 }; + struct btrfs_qgroup_swapped_blocks *blocks = &root->swapped_blocks; + struct btrfs_qgroup_swapped_block AUTO_KFREE(block); + struct extent_buffer *reloc_eb = NULL; + struct rb_node *node; + bool swapped = false; + int level = btrfs_header_level(subvol_eb); + int ret = 0; + int i; + + if (!btrfs_qgroup_full_accounting(fs_info)) + return 0; + if (!btrfs_is_fstree(btrfs_root_id(root)) || !root->reloc_root) + return 0; + + spin_lock(&blocks->lock); + if (!blocks->swapped) { + spin_unlock(&blocks->lock); + return 0; + } + node = rb_find(&subvol_eb->start, &blocks->blocks[level], + qgroup_swapped_block_bytenr_key_cmp); + if (!node) { + spin_unlock(&blocks->lock); + goto out; + } + block = rb_entry(node, struct btrfs_qgroup_swapped_block, node); + + /* Found one, remove it from @blocks first and update blocks->swapped */ + rb_erase(&block->node, &blocks->blocks[level]); + for (i = 0; i < BTRFS_MAX_LEVEL; i++) { + if (RB_EMPTY_ROOT(&blocks->blocks[i])) { + swapped = true; + break; + } + } + blocks->swapped = swapped; + spin_unlock(&blocks->lock); + + check.level = block->level; + check.transid = block->reloc_generation; + check.has_first_key = true; + memcpy(&check.first_key, &block->first_key, sizeof(check.first_key)); + + /* Read out reloc subtree root */ + reloc_eb = read_tree_block(fs_info, block->reloc_bytenr, &check); + if (IS_ERR(reloc_eb)) { + ret = PTR_ERR(reloc_eb); + reloc_eb = NULL; + goto free_out; + } + + ret = qgroup_trace_subtree_swap(trans, reloc_eb, subvol_eb, + block->last_snapshot, block->trace_leaf); +free_out: + free_extent_buffer(reloc_eb); +out: + if (ret < 0) { + qgroup_mark_inconsistent(fs_info, + "failed to account subtree at bytenr %llu: %d", + subvol_eb->start, ret); + } + return ret; +} + +void btrfs_qgroup_destroy_extent_records(struct btrfs_transaction *trans) +{ + struct btrfs_qgroup_extent_record *entry; + unsigned long index; + + xa_for_each(&trans->delayed_refs.dirty_extents, index, entry) { + ulist_free(entry->old_roots); + kfree(entry); + } + xa_destroy(&trans->delayed_refs.dirty_extents); +} + +int btrfs_record_squota_delta(struct btrfs_fs_info *fs_info, + const struct btrfs_squota_delta *delta) +{ + int ret; + struct btrfs_qgroup *qgroup; + struct btrfs_qgroup *qg; + LIST_HEAD(qgroup_list); + u64 root = delta->root; + u64 num_bytes = delta->num_bytes; + const int sign = (delta->is_inc ? 1 : -1); + + if (btrfs_qgroup_mode(fs_info) != BTRFS_QGROUP_MODE_SIMPLE && + !test_bit(BTRFS_FS_SQUOTA_ENABLING, &fs_info->flags)) + return 0; + + if (!btrfs_is_fstree(root)) + return 0; + + /* If the extent predates enabling quotas, don't count it. */ + if (delta->generation < fs_info->qgroup_enable_gen) + return 0; + + spin_lock(&fs_info->qgroup_lock); + qgroup = find_qgroup_rb(fs_info, root); + if (WARN_ON_ONCE(!qgroup)) { + btrfs_warn(fs_info, "squota failed to find qgroup for root %llu", root); + ret = 0; + goto out; + } + + ret = 0; + qgroup_iterator_add(&qgroup_list, qgroup); + list_for_each_entry(qg, &qgroup_list, iterator) { + struct btrfs_qgroup_list *glist; + + ASSERT(qg->excl == qg->rfer); + if (WARN_ON_ONCE(sign < 0 && qg->excl < num_bytes)) { + btrfs_warn(fs_info, + "squota underflow qg " BTRFS_QGROUP_FMT " excl %llu num_bytes %llu", + BTRFS_QGROUP_FMT_VALUE(qg), + qg->excl, num_bytes); + qg->excl = 0; + qg->rfer = 0; + } else { + qg->excl += num_bytes * sign; + qg->rfer += num_bytes * sign; + } + qgroup_dirty(fs_info, qg); + + list_for_each_entry(glist, &qg->groups, next_group) + qgroup_iterator_add(&qgroup_list, glist->group); + } + qgroup_iterator_clean(&qgroup_list); + +out: + spin_unlock(&fs_info->qgroup_lock); + return ret; +} diff --git a/fs/btrfs/qgroup.h b/fs/btrfs/qgroup.h new file mode 100644 index 000000000..80dd2dacd --- /dev/null +++ b/fs/btrfs/qgroup.h @@ -0,0 +1,421 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2014 Facebook. All rights reserved. + */ + +#ifndef BTRFS_QGROUP_H +#define BTRFS_QGROUP_H + +#include +#include +#include +#include +#include +#include + +struct extent_buffer; +struct extent_changeset; +struct btrfs_delayed_extent_op; +struct btrfs_fs_info; +struct btrfs_root; +struct btrfs_ioctl_quota_ctl_args; +struct btrfs_trans_handle; +struct btrfs_delayed_ref_root; +struct btrfs_inode; +struct btrfs_transaction; +struct btrfs_block_group; +struct btrfs_qgroup_swapped_blocks; + +/* + * Btrfs qgroup overview + * + * Btrfs qgroup splits into 3 main part: + * 1) Reserve + * Reserve metadata/data space for incoming operations + * Affect how qgroup limit works + * + * 2) Trace + * Tell btrfs qgroup to trace dirty extents. + * + * Dirty extents including: + * - Newly allocated extents + * - Extents going to be deleted (in this trans) + * - Extents whose owner is going to be modified + * + * This is the main part affects whether qgroup numbers will stay + * consistent. + * Btrfs qgroup can trace clean extents and won't cause any problem, + * but it will consume extra CPU time, it should be avoided if possible. + * + * 3) Account + * Btrfs qgroup will updates its numbers, based on dirty extents traced + * in previous step. + * + * Normally at qgroup rescan and transaction commit time. + */ + +/* + * Special performance optimization for balance. + * + * For balance, we need to swap subtree of subvolume and reloc trees. + * In theory, we need to trace all subtree blocks of both subvolume and reloc + * trees, since their owner has changed during such swap. + * + * However since balance has ensured that both subtrees are containing the + * same contents and have the same tree structures, such swap won't cause + * qgroup number change. + * + * But there is a race window between subtree swap and transaction commit, + * during that window, if we increase/decrease tree level or merge/split tree + * blocks, we still need to trace the original subtrees. + * + * So for balance, we use a delayed subtree tracing, whose workflow is: + * + * 1) Record the subtree root block get swapped. + * + * During subtree swap: + * O = Old tree blocks + * N = New tree blocks + * reloc tree subvolume tree X + * Root Root + * / \ / \ + * NA OB OA OB + * / | | \ / | | \ + * NC ND OE OF OC OD OE OF + * + * In this case, NA and OA are going to be swapped, record (NA, OA) into + * subvolume tree X. + * + * 2) After subtree swap. + * reloc tree subvolume tree X + * Root Root + * / \ / \ + * OA OB NA OB + * / | | \ / | | \ + * OC OD OE OF NC ND OE OF + * + * 3a) COW happens for OB + * If we are going to COW tree block OB, we check OB's bytenr against + * tree X's swapped_blocks structure. + * If it doesn't fit any, nothing will happen. + * + * 3b) COW happens for NA + * Check NA's bytenr against tree X's swapped_blocks, and get a hit. + * Then we do subtree scan on both subtrees OA and NA. + * Resulting 6 tree blocks to be scanned (OA, OC, OD, NA, NC, ND). + * + * Then no matter what we do to subvolume tree X, qgroup numbers will + * still be correct. + * Then NA's record gets removed from X's swapped_blocks. + * + * 4) Transaction commit + * Any record in X's swapped_blocks gets removed, since there is no + * modification to the swapped subtrees, no need to trigger heavy qgroup + * subtree rescan for them. + */ + +/* + * These flags share the flags field of the btrfs_qgroup_status_item with the + * persisted flags defined in btrfs_tree.h. + * + * To minimize the chance of collision with new persisted status flags, these + * count backwards from the MSB. + */ +#define BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN (1ULL << 63) +#define BTRFS_QGROUP_RUNTIME_FLAG_NO_ACCOUNTING (1ULL << 62) + +#define BTRFS_QGROUP_DROP_SUBTREE_THRES_DEFAULT (3) + +/* + * Record a dirty extent, and info qgroup to update quota on it + */ +struct btrfs_qgroup_extent_record { + /* + * The bytenr of the extent is given by its index in the dirty_extents + * xarray of struct btrfs_delayed_ref_root left shifted by + * fs_info->sectorsize_bits. + */ + + u64 num_bytes; + + /* + * For qgroup reserved data space freeing. + * + * @data_rsv_refroot and @data_rsv will be recorded after + * BTRFS_ADD_DELAYED_EXTENT is called. + * And will be used to free reserved qgroup space at + * transaction commit time. + */ + u32 data_rsv; /* reserved data space needs to be freed */ + u64 data_rsv_refroot; /* which root the reserved data belongs to */ + struct ulist *old_roots; +}; + +struct btrfs_qgroup_swapped_block { + struct rb_node node; + + int level; + bool trace_leaf; + + /* bytenr/generation of the tree block in subvolume tree after swap */ + u64 subvol_bytenr; + u64 subvol_generation; + + /* bytenr/generation of the tree block in reloc tree after swap */ + u64 reloc_bytenr; + u64 reloc_generation; + + u64 last_snapshot; + struct btrfs_key first_key; +}; + +/* + * Qgroup reservation types: + * + * DATA: + * space reserved for data + * + * META_PERTRANS: + * Space reserved for metadata (per-transaction) + * Due to the fact that qgroup data is only updated at transaction commit + * time, reserved space for metadata must be kept until transaction + * commits. + * Any metadata reserved that are used in btrfs_start_transaction() should + * be of this type. + * + * META_PREALLOC: + * There are cases where metadata space is reserved before starting + * transaction, and then btrfs_join_transaction() to get a trans handle. + * Any metadata reserved for such usage should be of this type. + * And after join_transaction() part (or all) of such reservation should + * be converted into META_PERTRANS. + */ +enum btrfs_qgroup_rsv_type { + BTRFS_QGROUP_RSV_DATA, + BTRFS_QGROUP_RSV_META_PERTRANS, + BTRFS_QGROUP_RSV_META_PREALLOC, + BTRFS_QGROUP_RSV_LAST, +}; + +/* + * Represents how many bytes we have reserved for this qgroup. + * + * Each type should have different reservation behavior. + * E.g, data follows its io_tree flag modification, while + * *currently* meta is just reserve-and-clear during transaction. + * + * TODO: Add new type for reservation which can survive transaction commit. + * Current metadata reservation behavior is not suitable for such case. + */ +struct btrfs_qgroup_rsv { + u64 values[BTRFS_QGROUP_RSV_LAST]; +}; + +/* + * one struct for each qgroup, organized in fs_info->qgroup_tree. + */ +struct btrfs_qgroup { + u64 qgroupid; + + /* + * state + */ + u64 rfer; /* referenced */ + u64 rfer_cmpr; /* referenced compressed */ + u64 excl; /* exclusive */ + u64 excl_cmpr; /* exclusive compressed */ + + /* + * limits + */ + u64 lim_flags; /* which limits are set */ + u64 max_rfer; + u64 max_excl; + u64 rsv_rfer; + u64 rsv_excl; + + /* + * reservation tracking + */ + struct btrfs_qgroup_rsv rsv; + + /* + * lists + */ + struct list_head groups; /* groups this group is member of */ + struct list_head members; /* groups that are members of this group */ + struct list_head dirty; /* dirty groups */ + + /* + * For qgroup iteration usage. + * + * The iteration list should always be empty until qgroup_iterator_add() + * is called. And should be reset to empty after the iteration is + * finished. + */ + struct list_head iterator; + + /* + * For nested iterator usage. + * + * Here we support at most one level of nested iterator calls like: + * + * LIST_HEAD(all_qgroups); + * { + * LIST_HEAD(local_qgroups); + * qgroup_iterator_add(local_qgroups, qg); + * qgroup_iterator_nested_add(all_qgroups, qg); + * do_some_work(local_qgroups); + * qgroup_iterator_clean(local_qgroups); + * } + * do_some_work(all_qgroups); + * qgroup_iterator_nested_clean(all_qgroups); + */ + struct list_head nested_iterator; + struct rb_node node; /* tree of qgroups */ + + /* + * temp variables for accounting operations + * Refer to qgroup_shared_accounting() for details. + */ + u64 old_refcnt; + u64 new_refcnt; + + /* + * Sysfs kobjectid + */ + struct kobject kobj; +}; + +/* Glue structure to represent the relations between qgroups. */ +struct btrfs_qgroup_list { + struct list_head next_group; + struct list_head next_member; + struct btrfs_qgroup *group; + struct btrfs_qgroup *member; +}; + +struct btrfs_squota_delta { + /* The fstree root this delta counts against. */ + u64 root; + /* The number of bytes in the extent being counted. */ + u64 num_bytes; + /* The generation the extent was created in. */ + u64 generation; + /* Whether we are using or freeing the extent. */ + bool is_inc; + /* Whether the extent is data or metadata. */ + bool is_data; +}; + +static inline u64 btrfs_qgroup_subvolid(u64 qgroupid) +{ + return (qgroupid & ((1ULL << BTRFS_QGROUP_LEVEL_SHIFT) - 1)); +} + +/* + * For qgroup event trace points only + */ +enum { + ENUM_BIT(QGROUP_RESERVE), + ENUM_BIT(QGROUP_RELEASE), + ENUM_BIT(QGROUP_FREE), +}; + +enum btrfs_qgroup_mode { + BTRFS_QGROUP_MODE_DISABLED, + BTRFS_QGROUP_MODE_FULL, + BTRFS_QGROUP_MODE_SIMPLE +}; + +enum btrfs_qgroup_mode btrfs_qgroup_mode(const struct btrfs_fs_info *fs_info); +bool btrfs_qgroup_enabled(const struct btrfs_fs_info *fs_info); +bool btrfs_qgroup_full_accounting(const struct btrfs_fs_info *fs_info); +int btrfs_quota_enable(struct btrfs_fs_info *fs_info, + struct btrfs_ioctl_quota_ctl_args *quota_ctl_args); +int btrfs_quota_disable(struct btrfs_fs_info *fs_info); +int btrfs_qgroup_rescan(struct btrfs_fs_info *fs_info); +void btrfs_qgroup_rescan_resume(struct btrfs_fs_info *fs_info); +int btrfs_qgroup_wait_for_completion(struct btrfs_fs_info *fs_info, + bool interruptible); +int btrfs_add_qgroup_relation(struct btrfs_trans_handle *trans, u64 src, u64 dst, + struct btrfs_qgroup_list *prealloc); +int btrfs_del_qgroup_relation(struct btrfs_trans_handle *trans, u64 src, + u64 dst); +int btrfs_create_qgroup(struct btrfs_trans_handle *trans, u64 qgroupid); +int btrfs_remove_qgroup(struct btrfs_trans_handle *trans, u64 qgroupid); +int btrfs_qgroup_cleanup_dropped_subvolume(struct btrfs_fs_info *fs_info, u64 subvolid); +int btrfs_limit_qgroup(struct btrfs_trans_handle *trans, u64 qgroupid, + struct btrfs_qgroup_limit *limit); +int btrfs_read_qgroup_config(struct btrfs_fs_info *fs_info); +void btrfs_free_qgroup_config(struct btrfs_fs_info *fs_info); + +int btrfs_qgroup_trace_extent_nolock( + struct btrfs_fs_info *fs_info, + struct btrfs_delayed_ref_root *delayed_refs, + struct btrfs_qgroup_extent_record *record, + u64 bytenr); +int btrfs_qgroup_trace_extent_post(struct btrfs_trans_handle *trans, + struct btrfs_qgroup_extent_record *qrecord, + u64 bytenr); +int btrfs_qgroup_trace_extent(struct btrfs_trans_handle *trans, u64 bytenr, + u64 num_bytes); +int btrfs_qgroup_trace_leaf_items(struct btrfs_trans_handle *trans, + struct extent_buffer *eb); +int btrfs_qgroup_trace_subtree(struct btrfs_trans_handle *trans, + struct extent_buffer *root_eb, + u64 root_gen, int root_level); +int btrfs_qgroup_account_extent(struct btrfs_trans_handle *trans, u64 bytenr, + u64 num_bytes, struct ulist *old_roots, + struct ulist *new_roots); +int btrfs_qgroup_account_extents(struct btrfs_trans_handle *trans); +int btrfs_run_qgroups(struct btrfs_trans_handle *trans); +int btrfs_qgroup_check_inherit(struct btrfs_fs_info *fs_info, + struct btrfs_qgroup_inherit *inherit, + size_t size); +int btrfs_qgroup_inherit(struct btrfs_trans_handle *trans, u64 srcid, + u64 objectid, u64 inode_rootid, + struct btrfs_qgroup_inherit *inherit); +void btrfs_qgroup_free_refroot(struct btrfs_fs_info *fs_info, + u64 ref_root, u64 num_bytes, + enum btrfs_qgroup_rsv_type type); + +#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS +int btrfs_verify_qgroup_counts(const struct btrfs_fs_info *fs_info, u64 qgroupid, + u64 rfer, u64 excl); +#endif + +/* New io_tree based accurate qgroup reserve API */ +int btrfs_qgroup_reserve_data(struct btrfs_inode *inode, + struct extent_changeset **reserved, u64 start, u64 len); +int btrfs_qgroup_release_data(struct btrfs_inode *inode, u64 start, u64 len, u64 *released); +int btrfs_qgroup_free_data(struct btrfs_inode *inode, + struct extent_changeset *reserved, u64 start, + u64 len, u64 *freed); +int btrfs_qgroup_reserve_meta_prealloc(struct btrfs_root *root, int num_bytes, + bool enforce, bool noflush); +/* Pre-allocated meta reservation can be freed at need */ +void btrfs_qgroup_free_meta_prealloc(struct btrfs_root *root, int num_bytes); + +void btrfs_qgroup_free_meta_all_pertrans(struct btrfs_root *root); +void btrfs_qgroup_convert_reserved_meta(struct btrfs_root *root, int num_bytes); +void btrfs_qgroup_check_reserved_leak(struct btrfs_inode *inode); + +/* btrfs_qgroup_swapped_blocks related functions */ +void btrfs_qgroup_init_swapped_blocks( + struct btrfs_qgroup_swapped_blocks *swapped_blocks); + +void btrfs_qgroup_clean_swapped_blocks(struct btrfs_root *root); +int btrfs_qgroup_add_swapped_blocks(struct btrfs_root *subvol_root, + struct btrfs_block_group *bg, + struct extent_buffer *subvol_parent, int subvol_slot, + struct extent_buffer *reloc_parent, int reloc_slot, + u64 last_snapshot); +int btrfs_qgroup_trace_subtree_after_cow(struct btrfs_trans_handle *trans, + struct btrfs_root *root, struct extent_buffer *eb); +void btrfs_qgroup_destroy_extent_records(struct btrfs_transaction *trans); +bool btrfs_check_quota_leak(const struct btrfs_fs_info *fs_info); +int btrfs_record_squota_delta(struct btrfs_fs_info *fs_info, + const struct btrfs_squota_delta *delta); + +#endif diff --git a/fs/btrfs/raid-stripe-tree.c b/fs/btrfs/raid-stripe-tree.c new file mode 100644 index 000000000..b210371ce --- /dev/null +++ b/fs/btrfs/raid-stripe-tree.c @@ -0,0 +1,511 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2023 Western Digital Corporation or its affiliates. + */ + +#include +#include "ctree.h" +#include "fs.h" +#include "accessors.h" +#include "transaction.h" +#include "disk-io.h" +#include "raid-stripe-tree.h" +#include "volumes.h" +#include "print-tree.h" + +static int btrfs_partially_delete_raid_extent(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + const struct btrfs_key *oldkey, + u64 newlen, u64 frontpad) +{ + struct btrfs_root *stripe_root = trans->fs_info->stripe_root; + struct btrfs_stripe_extent *extent, AUTO_KFREE(newitem); + struct extent_buffer *leaf; + int slot; + size_t item_size; + struct btrfs_key newkey = { + .objectid = oldkey->objectid + frontpad, + .type = BTRFS_RAID_STRIPE_KEY, + .offset = newlen, + }; + int ret; + + ASSERT(newlen > 0); + ASSERT(oldkey->type == BTRFS_RAID_STRIPE_KEY); + + leaf = path->nodes[0]; + slot = path->slots[0]; + item_size = btrfs_item_size(leaf, slot); + + newitem = kzalloc(item_size, GFP_NOFS); + if (!newitem) + return -ENOMEM; + + extent = btrfs_item_ptr(leaf, slot, struct btrfs_stripe_extent); + + for (int i = 0; i < btrfs_num_raid_stripes(item_size); i++) { + struct btrfs_raid_stride *stride = &extent->strides[i]; + u64 devid; + u64 phys; + + devid = btrfs_raid_stride_devid(leaf, stride); + btrfs_set_stack_raid_stride_devid(&newitem->strides[i], devid); + phys = btrfs_raid_stride_physical(leaf, stride) + frontpad; + btrfs_set_stack_raid_stride_physical(&newitem->strides[i], phys); + } + + ret = btrfs_del_item(trans, stripe_root, path); + if (ret) + return ret; + + btrfs_release_path(path); + return btrfs_insert_item(trans, stripe_root, &newkey, newitem, item_size); +} + +int btrfs_delete_raid_extent(struct btrfs_trans_handle *trans, u64 start, u64 length) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *stripe_root = fs_info->stripe_root; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + struct extent_buffer *leaf; + u64 found_start; + u64 found_end; + u64 end = start + length; + int slot; + int ret; + + if (!btrfs_fs_incompat(fs_info, RAID_STRIPE_TREE) || !stripe_root) + return 0; + + if (!btrfs_is_testing(fs_info)) { + struct btrfs_chunk_map *map; + bool use_rst; + + map = btrfs_find_chunk_map(fs_info, start, length); + if (!map) + return -EINVAL; + use_rst = btrfs_need_stripe_tree_update(fs_info, map->type); + btrfs_free_chunk_map(map); + if (!use_rst) + return 0; + } + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + while (1) { + key.objectid = start; + key.type = BTRFS_RAID_STRIPE_KEY; + key.offset = (u64)-1; + + ret = btrfs_search_slot(trans, stripe_root, &key, path, -1, 1); + if (ret < 0) + break; + + /* + * Search with offset=(u64)-1 ensures we land on the correct + * leaf even when the target entry is the first item on a leaf. + * Since no real entry has offset=(u64)-1, ret is always 1 and + * slot points past the last entry with objectid==start (or + * past the end of the leaf if that entry is the last item). + * Back up one slot to find the actual entry. + */ + if (path->slots[0] == 0) { + /* No entry with objectid <= start exists. */ + ret = 0; + break; + } + path->slots[0]--; + + leaf = path->nodes[0]; + slot = path->slots[0]; + btrfs_item_key_to_cpu(leaf, &key, slot); + found_start = key.objectid; + found_end = found_start + key.offset; + ret = 0; + + /* + * The stripe extent starts before the range we want to delete, + * but the range spans more than one stripe extent: + * + * |--- RAID Stripe Extent ---||--- RAID Stripe Extent ---| + * |--- keep ---|--- drop ---| + * + * This means we have to get the previous item, truncate its + * length and then restart the search. + */ + if (found_start > start) { + if (slot == 0) { + ret = btrfs_previous_item(stripe_root, path, 0, + BTRFS_RAID_STRIPE_KEY); + if (ret) { + if (ret > 0) + ret = -ENOENT; + break; + } + } else { + path->slots[0]--; + } + + leaf = path->nodes[0]; + slot = path->slots[0]; + btrfs_item_key_to_cpu(leaf, &key, slot); + found_start = key.objectid; + found_end = found_start + key.offset; + if (found_start > start || found_end <= start) { + ret = -ENOENT; + break; + } + } + + if (key.type != BTRFS_RAID_STRIPE_KEY) + break; + + /* That stripe ends before we start, we're done. */ + if (found_end <= start) + break; + + trace_btrfs_raid_extent_delete(fs_info, start, end, + found_start, found_end); + + /* + * The stripe extent starts before the range we want to delete + * and ends after the range we want to delete, i.e. we're + * punching a hole in the stripe extent: + * + * |--- RAID Stripe Extent ---| + * | keep |--- drop ---| keep | + * + * This means we need to a) truncate the existing item and b) + * create a second item for the remaining range. + */ + if (found_start < start && found_end > end) { + size_t item_size; + u64 diff_start = start - found_start; + u64 diff_end = found_end - end; + struct btrfs_stripe_extent *extent; + struct btrfs_key newkey = { + .objectid = end, + .type = BTRFS_RAID_STRIPE_KEY, + .offset = diff_end, + }; + + /* The "right" item. */ + ret = btrfs_duplicate_item(trans, stripe_root, path, &newkey); + if (ret == -EAGAIN) { + btrfs_release_path(path); + continue; + } + if (ret) + break; + + /* + * btrfs_duplicate_item() may have triggered a leaf + * split via setup_leaf_for_split(), so we must refresh + * our leaf pointer from the path. + */ + leaf = path->nodes[0]; + item_size = btrfs_item_size(leaf, path->slots[0]); + extent = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_stripe_extent); + + for (int i = 0; i < btrfs_num_raid_stripes(item_size); i++) { + struct btrfs_raid_stride *stride = &extent->strides[i]; + u64 phys; + + phys = btrfs_raid_stride_physical(leaf, stride); + phys += diff_start + length; + btrfs_set_raid_stride_physical(leaf, stride, phys); + } + + /* The "left" item. */ + path->slots[0]--; + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + ret = btrfs_partially_delete_raid_extent(trans, path, + &key, + diff_start, 0); + break; + } + + /* + * The stripe extent starts before the range we want to delete: + * + * |--- RAID Stripe Extent ---| + * |--- keep ---|--- drop ---| + * + * This means we have to duplicate the tree item, truncate the + * length to the new size and then re-insert the item. + */ + if (found_start < start) { + u64 diff_start = start - found_start; + + ret = btrfs_partially_delete_raid_extent(trans, path, + &key, + diff_start, 0); + if (ret) + break; + + start += (key.offset - diff_start); + length -= (key.offset - diff_start); + if (length == 0) + break; + + btrfs_release_path(path); + continue; + } + + /* + * The stripe extent ends after the range we want to delete: + * + * |--- RAID Stripe Extent ---| + * |--- drop ---|--- keep ---| + * + * This means we have to duplicate the tree item, truncate the + * length to the new size and then re-insert the item. + */ + if (found_end > end) { + u64 diff_end = found_end - end; + + ret = btrfs_partially_delete_raid_extent(trans, path, + &key, + key.offset - length, + length); + ASSERT(key.offset - diff_end == length, + "key.offset=%llu diff_end=%llu length=%llu", + key.offset, diff_end, length); + break; + } + + /* Finally we can delete the whole item, no more special cases. */ + ret = btrfs_del_item(trans, stripe_root, path); + if (ret) + break; + + start += key.offset; + length -= key.offset; + if (length == 0) + break; + + btrfs_release_path(path); + } + + return ret; +} + +static int update_raid_extent_item(struct btrfs_trans_handle *trans, + struct btrfs_key *key, + struct btrfs_stripe_extent *stripe_extent, + const size_t item_size) +{ + BTRFS_PATH_AUTO_FREE(path); + struct extent_buffer *leaf; + int ret; + int slot; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + ret = btrfs_search_slot(trans, trans->fs_info->stripe_root, key, path, + 0, 1); + if (ret) + return (ret == 1 ? ret : -EINVAL); + + leaf = path->nodes[0]; + slot = path->slots[0]; + + write_extent_buffer(leaf, stripe_extent, btrfs_item_ptr_offset(leaf, slot), + item_size); + + return ret; +} + +EXPORT_FOR_TESTS +int btrfs_insert_one_raid_extent(struct btrfs_trans_handle *trans, + struct btrfs_io_context *bioc) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_key stripe_key; + struct btrfs_root *stripe_root = fs_info->stripe_root; + const int num_stripes = btrfs_bg_type_to_factor(bioc->map_type); + struct btrfs_stripe_extent AUTO_KFREE(stripe_extent); + const size_t item_size = struct_size(stripe_extent, strides, num_stripes); + int ret; + + stripe_extent = kzalloc(item_size, GFP_NOFS); + if (unlikely(!stripe_extent)) { + btrfs_abort_transaction(trans, -ENOMEM); + btrfs_end_transaction(trans); + return -ENOMEM; + } + + trace_btrfs_insert_one_raid_extent(fs_info, bioc->logical, bioc->size, + num_stripes); + for (int i = 0; i < num_stripes; i++) { + u64 devid = bioc->stripes[i].dev->devid; + u64 physical = bioc->stripes[i].physical; + struct btrfs_raid_stride *raid_stride = &stripe_extent->strides[i]; + + btrfs_set_stack_raid_stride_devid(raid_stride, devid); + btrfs_set_stack_raid_stride_physical(raid_stride, physical); + } + + stripe_key.objectid = bioc->logical; + stripe_key.type = BTRFS_RAID_STRIPE_KEY; + stripe_key.offset = bioc->size; + + ret = btrfs_insert_item(trans, stripe_root, &stripe_key, stripe_extent, + item_size); + if (ret == -EEXIST) { + ret = update_raid_extent_item(trans, &stripe_key, stripe_extent, + item_size); + if (ret) + btrfs_abort_transaction(trans, ret); + } else if (ret) { + btrfs_abort_transaction(trans, ret); + } + + return ret; +} + +int btrfs_insert_raid_extent(struct btrfs_trans_handle *trans, + struct btrfs_ordered_extent *ordered_extent) +{ + struct btrfs_io_context *bioc; + int ret; + + if (!btrfs_fs_incompat(trans->fs_info, RAID_STRIPE_TREE)) + return 0; + + list_for_each_entry(bioc, &ordered_extent->bioc_list, rst_ordered_entry) { + ret = btrfs_insert_one_raid_extent(trans, bioc); + if (ret) + return ret; + } + + while (!list_empty(&ordered_extent->bioc_list)) { + bioc = list_first_entry(&ordered_extent->bioc_list, + typeof(*bioc), rst_ordered_entry); + list_del(&bioc->rst_ordered_entry); + btrfs_put_bioc(bioc); + } + + return 0; +} + +int btrfs_get_raid_extent_offset(struct btrfs_fs_info *fs_info, + u64 logical, u64 *length, u64 map_type, + u32 stripe_index, struct btrfs_io_stripe *stripe) +{ + struct btrfs_root *stripe_root = fs_info->stripe_root; + struct btrfs_stripe_extent *stripe_extent; + struct btrfs_key stripe_key; + struct btrfs_key found_key; + BTRFS_PATH_AUTO_FREE(path); + struct extent_buffer *leaf; + const u64 end = logical + *length; + int num_stripes; + u64 offset; + u64 found_logical; + u64 found_length; + u64 found_end; + int slot; + int ret; + + if (unlikely(!stripe_root)) { + btrfs_err_rl(fs_info, "missing raid stripe tree root for logical %llu", + logical); + return -EUCLEAN; + } + + stripe_key.objectid = logical; + stripe_key.type = BTRFS_RAID_STRIPE_KEY; + stripe_key.offset = 0; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + if (stripe->rst_search_commit_root) { + path->skip_locking = true; + path->search_commit_root = true; + } + + ret = btrfs_search_slot(NULL, stripe_root, &stripe_key, path, 0, 0); + if (ret < 0) + return ret; + if (ret) { + if (path->slots[0] != 0) + path->slots[0]--; + } + + while (1) { + leaf = path->nodes[0]; + slot = path->slots[0]; + + btrfs_item_key_to_cpu(leaf, &found_key, slot); + found_logical = found_key.objectid; + found_length = found_key.offset; + found_end = found_logical + found_length; + + if (found_logical > end) { + ret = -ENODATA; + goto out; + } + + if (in_range(logical, found_logical, found_length)) + break; + + ret = btrfs_next_item(stripe_root, path); + if (ret) + goto out; + } + + offset = logical - found_logical; + + /* + * If we have a logically contiguous, but physically non-continuous + * range, we need to split the bio. Record the length after which we + * must split the bio. + */ + if (end > found_end) + *length -= end - found_end; + + num_stripes = btrfs_num_raid_stripes(btrfs_item_size(leaf, slot)); + stripe_extent = btrfs_item_ptr(leaf, slot, struct btrfs_stripe_extent); + + for (int i = 0; i < num_stripes; i++) { + struct btrfs_raid_stride *stride = &stripe_extent->strides[i]; + u64 devid = btrfs_raid_stride_devid(leaf, stride); + u64 physical = btrfs_raid_stride_physical(leaf, stride); + + if (devid != stripe->dev->devid) + continue; + + if ((map_type & BTRFS_BLOCK_GROUP_DUP) && stripe_index != i) + continue; + + stripe->physical = physical + offset; + + trace_btrfs_get_raid_extent_offset(fs_info, logical, *length, + stripe->physical, devid); + + return 0; + } + + /* If we're here, we haven't found the requested devid in the stripe. */ + ret = -ENODATA; +out: + if (ret > 0) + ret = -ENODATA; + if (ret && ret != -EIO && !stripe->rst_search_commit_root) { + btrfs_debug(fs_info, + "cannot find raid-stripe for logical [%llu, %llu] devid %llu, profile %s", + logical, logical + *length, stripe->dev->devid, + btrfs_bg_type_to_raid_name(map_type)); + } + + return ret; +} diff --git a/fs/btrfs/raid-stripe-tree.h b/fs/btrfs/raid-stripe-tree.h new file mode 100644 index 000000000..69942ad43 --- /dev/null +++ b/fs/btrfs/raid-stripe-tree.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023 Western Digital Corporation or its affiliates. + */ + +#ifndef BTRFS_RAID_STRIPE_TREE_H +#define BTRFS_RAID_STRIPE_TREE_H + +#include +#include +#include "fs.h" +#include "accessors.h" + +#define BTRFS_RST_SUPP_BLOCK_GROUP_MASK (BTRFS_BLOCK_GROUP_DUP | \ + BTRFS_BLOCK_GROUP_RAID1_MASK | \ + BTRFS_BLOCK_GROUP_RAID0 | \ + BTRFS_BLOCK_GROUP_RAID10) + +struct btrfs_io_context; +struct btrfs_io_stripe; +struct btrfs_fs_info; +struct btrfs_ordered_extent; +struct btrfs_trans_handle; + +int btrfs_delete_raid_extent(struct btrfs_trans_handle *trans, u64 start, u64 length); +int btrfs_get_raid_extent_offset(struct btrfs_fs_info *fs_info, + u64 logical, u64 *length, u64 map_type, + u32 stripe_index, struct btrfs_io_stripe *stripe); +int btrfs_insert_raid_extent(struct btrfs_trans_handle *trans, + struct btrfs_ordered_extent *ordered_extent); + +#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS +int btrfs_insert_one_raid_extent(struct btrfs_trans_handle *trans, + struct btrfs_io_context *bioc); +#endif + +static inline bool btrfs_need_stripe_tree_update(struct btrfs_fs_info *fs_info, + u64 map_type) +{ + u64 type = map_type & BTRFS_BLOCK_GROUP_TYPE_MASK; + u64 profile = map_type & BTRFS_BLOCK_GROUP_PROFILE_MASK; + + if (!btrfs_fs_incompat(fs_info, RAID_STRIPE_TREE)) + return false; + + if (type != BTRFS_BLOCK_GROUP_DATA) + return false; + + if (profile & BTRFS_RST_SUPP_BLOCK_GROUP_MASK) + return true; + + return false; +} + +static inline int btrfs_num_raid_stripes(u32 item_size) +{ + return item_size / sizeof(struct btrfs_raid_stride); +} + +#endif diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c new file mode 100644 index 000000000..1ee52a9dc --- /dev/null +++ b/fs/btrfs/raid56.c @@ -0,0 +1,3035 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2012 Fusion-io All rights reserved. + * Copyright (C) 2012 Intel Corp. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "messages.h" +#include "ctree.h" +#include "disk-io.h" +#include "volumes.h" +#include "raid56.h" +#include "async-thread.h" +#include "file-item.h" +#include "btrfs_inode.h" + +/* set when additional merges to this rbio are not allowed */ +#define RBIO_RMW_LOCKED_BIT 1 + +/* + * set when this rbio is sitting in the hash, but it is just a cache + * of past RMW + */ +#define RBIO_CACHE_BIT 2 + +/* + * set when it is safe to trust the stripe_pages for caching + */ +#define RBIO_CACHE_READY_BIT 3 + +#define RBIO_CACHE_SIZE 1024 + +#define BTRFS_STRIPE_HASH_TABLE_BITS 11 + +static void dump_bioc(const struct btrfs_fs_info *fs_info, const struct btrfs_io_context *bioc) +{ + if (unlikely(!bioc)) { + btrfs_crit(fs_info, "bioc=NULL"); + return; + } + btrfs_crit(fs_info, +"bioc logical=%llu full_stripe=%llu size=%llu map_type=0x%llx mirror=%u replace_nr_stripes=%u replace_stripe_src=%d num_stripes=%u", + bioc->logical, bioc->full_stripe_logical, bioc->size, + bioc->map_type, bioc->mirror_num, bioc->replace_nr_stripes, + bioc->replace_stripe_src, bioc->num_stripes); + for (int i = 0; i < bioc->num_stripes; i++) { + btrfs_crit(fs_info, " nr=%d devid=%llu physical=%llu", + i, bioc->stripes[i].dev->devid, + bioc->stripes[i].physical); + } +} + +static void btrfs_dump_rbio(const struct btrfs_fs_info *fs_info, + const struct btrfs_raid_bio *rbio) +{ + if (!IS_ENABLED(CONFIG_BTRFS_ASSERT)) + return; + + dump_bioc(fs_info, rbio->bioc); + btrfs_crit(fs_info, +"rbio flags=0x%lx nr_sectors=%u nr_data=%u real_stripes=%u stripe_nsectors=%u sector_nsteps=%u scrubp=%u dbitmap=0x%lx", + rbio->flags, rbio->nr_sectors, rbio->nr_data, + rbio->real_stripes, rbio->stripe_nsectors, + rbio->sector_nsteps, rbio->scrubp, rbio->dbitmap); +} + +#define ASSERT_RBIO(expr, rbio) \ +({ \ + if (IS_ENABLED(CONFIG_BTRFS_ASSERT) && unlikely(!(expr))) { \ + const struct btrfs_fs_info *__fs_info = (rbio)->bioc ? \ + (rbio)->bioc->fs_info : NULL; \ + \ + btrfs_dump_rbio(__fs_info, (rbio)); \ + } \ + ASSERT((expr)); \ +}) + +#define ASSERT_RBIO_STRIPE(expr, rbio, stripe_nr) \ +({ \ + if (IS_ENABLED(CONFIG_BTRFS_ASSERT) && unlikely(!(expr))) { \ + const struct btrfs_fs_info *__fs_info = (rbio)->bioc ? \ + (rbio)->bioc->fs_info : NULL; \ + \ + btrfs_dump_rbio(__fs_info, (rbio)); \ + btrfs_crit(__fs_info, "stripe_nr=%d", (stripe_nr)); \ + } \ + ASSERT((expr)); \ +}) + +#define ASSERT_RBIO_SECTOR(expr, rbio, sector_nr) \ +({ \ + if (IS_ENABLED(CONFIG_BTRFS_ASSERT) && unlikely(!(expr))) { \ + const struct btrfs_fs_info *__fs_info = (rbio)->bioc ? \ + (rbio)->bioc->fs_info : NULL; \ + \ + btrfs_dump_rbio(__fs_info, (rbio)); \ + btrfs_crit(__fs_info, "sector_nr=%d", (sector_nr)); \ + } \ + ASSERT((expr)); \ +}) + +#define ASSERT_RBIO_LOGICAL(expr, rbio, logical) \ +({ \ + if (IS_ENABLED(CONFIG_BTRFS_ASSERT) && unlikely(!(expr))) { \ + const struct btrfs_fs_info *__fs_info = (rbio)->bioc ? \ + (rbio)->bioc->fs_info : NULL; \ + \ + btrfs_dump_rbio(__fs_info, (rbio)); \ + btrfs_crit(__fs_info, "logical=%llu", (logical)); \ + } \ + ASSERT((expr)); \ +}) + +/* Used by the raid56 code to lock stripes for read/modify/write */ +struct btrfs_stripe_hash { + struct list_head hash_list; + spinlock_t lock; +}; + +/* Used by the raid56 code to lock stripes for read/modify/write */ +struct btrfs_stripe_hash_table { + struct list_head stripe_cache; + spinlock_t cache_lock; + int cache_size; + struct btrfs_stripe_hash table[]; +}; + +/* + * The PFN may still be valid, but our paddrs should always be block size + * aligned, thus such -1 paddr is definitely not a valid one. + */ +#define INVALID_PADDR (~(phys_addr_t)0) + +static void rmw_rbio_work(struct work_struct *work); +static void rmw_rbio_work_locked(struct work_struct *work); +static void index_rbio_pages(struct btrfs_raid_bio *rbio); +static int alloc_rbio_pages(struct btrfs_raid_bio *rbio); + +static int finish_parity_scrub(struct btrfs_raid_bio *rbio); +static void scrub_rbio_work_locked(struct work_struct *work); + +static void free_raid_bio_pointers(struct btrfs_raid_bio *rbio) +{ + bitmap_free(rbio->error_bitmap); + bitmap_free(rbio->stripe_uptodate_bitmap); + kfree(rbio->stripe_pages); + kfree(rbio->bio_paddrs); + kfree(rbio->stripe_paddrs); + kfree(rbio->finish_pointers); +} + +static void free_raid_bio(struct btrfs_raid_bio *rbio) +{ + int i; + + if (!refcount_dec_and_test(&rbio->refs)) + return; + + WARN_ON(!list_empty(&rbio->stripe_cache)); + WARN_ON(!list_empty(&rbio->hash_list)); + WARN_ON(!bio_list_empty(&rbio->bio_list)); + + for (i = 0; i < rbio->nr_pages; i++) { + if (rbio->stripe_pages[i]) { + __free_page(rbio->stripe_pages[i]); + rbio->stripe_pages[i] = NULL; + } + } + + btrfs_put_bioc(rbio->bioc); + free_raid_bio_pointers(rbio); + kfree(rbio); +} + +static void start_async_work(struct btrfs_raid_bio *rbio, work_func_t work_func) +{ + INIT_WORK(&rbio->work, work_func); + queue_work(rbio->bioc->fs_info->rmw_workers, &rbio->work); +} + +/* + * the stripe hash table is used for locking, and to collect + * bios in hopes of making a full stripe + */ +int btrfs_alloc_stripe_hash_table(struct btrfs_fs_info *info) +{ + struct btrfs_stripe_hash_table *table; + struct btrfs_stripe_hash_table *x; + struct btrfs_stripe_hash *cur; + struct btrfs_stripe_hash *h; + unsigned int num_entries = 1U << BTRFS_STRIPE_HASH_TABLE_BITS; + + if (info->stripe_hash_table) + return 0; + + /* + * The table is large, starting with order 4 and can go as high as + * order 7 in case lock debugging is turned on. + * + * Try harder to allocate and fallback to vmalloc to lower the chance + * of a failing mount. + */ + table = kvzalloc_flex(*table, table, num_entries); + if (!table) + return -ENOMEM; + + spin_lock_init(&table->cache_lock); + INIT_LIST_HEAD(&table->stripe_cache); + + h = table->table; + + for (unsigned int i = 0; i < num_entries; i++) { + cur = h + i; + INIT_LIST_HEAD(&cur->hash_list); + spin_lock_init(&cur->lock); + } + + x = cmpxchg(&info->stripe_hash_table, NULL, table); + kvfree(x); + return 0; +} + +static void memcpy_from_bio_to_stripe(struct btrfs_raid_bio *rbio, unsigned int sector_nr) +{ + const u32 step = min(rbio->bioc->fs_info->sectorsize, PAGE_SIZE); + + ASSERT(sector_nr < rbio->nr_sectors); + for (int i = 0; i < rbio->sector_nsteps; i++) { + unsigned int index = sector_nr * rbio->sector_nsteps + i; + phys_addr_t dst = rbio->stripe_paddrs[index]; + phys_addr_t src = rbio->bio_paddrs[index]; + + ASSERT(dst != INVALID_PADDR); + ASSERT(src != INVALID_PADDR); + + memcpy_page(phys_to_page(dst), offset_in_page(dst), + phys_to_page(src), offset_in_page(src), step); + } +} + +/* + * caching an rbio means to copy anything from the + * bio_sectors array into the stripe_pages array. We + * use the page uptodate bit in the stripe cache array + * to indicate if it has valid data + * + * once the caching is done, we set the cache ready + * bit. + */ +static void cache_rbio_pages(struct btrfs_raid_bio *rbio) +{ + int i; + int ret; + + ret = alloc_rbio_pages(rbio); + if (ret) + return; + + for (i = 0; i < rbio->nr_sectors; i++) { + /* Some range not covered by bio (partial write), skip it */ + if (rbio->bio_paddrs[i * rbio->sector_nsteps] == INVALID_PADDR) { + /* + * Even if the sector is not covered by bio, if it is + * a data sector it should still be uptodate as it is + * read from disk. + */ + if (i < rbio->nr_data * rbio->stripe_nsectors) + ASSERT(test_bit(i, rbio->stripe_uptodate_bitmap)); + continue; + } + + memcpy_from_bio_to_stripe(rbio, i); + set_bit(i, rbio->stripe_uptodate_bitmap); + } + set_bit(RBIO_CACHE_READY_BIT, &rbio->flags); +} + +/* + * we hash on the first logical address of the stripe + */ +static int rbio_bucket(struct btrfs_raid_bio *rbio) +{ + u64 num = rbio->bioc->full_stripe_logical; + + /* + * we shift down quite a bit. We're using byte + * addressing, and most of the lower bits are zeros. + * This tends to upset hash_64, and it consistently + * returns just one or two different values. + * + * shifting off the lower bits fixes things. + */ + return hash_64(num >> 16, BTRFS_STRIPE_HASH_TABLE_BITS); +} + +/* Get the sector number of the first sector covered by @page_nr. */ +static u32 page_nr_to_sector_nr(struct btrfs_raid_bio *rbio, unsigned int page_nr) +{ + u32 sector_nr; + + ASSERT(page_nr < rbio->nr_pages); + + sector_nr = (page_nr << PAGE_SHIFT) >> rbio->bioc->fs_info->sectorsize_bits; + ASSERT(sector_nr < rbio->nr_sectors); + return sector_nr; +} + +/* + * Get the number of sectors covered by @page_nr. + * + * For bs > ps cases, the result will always be 1. + * For bs <= ps cases, the result will be ps / bs. + */ +static u32 page_nr_to_num_sectors(struct btrfs_raid_bio *rbio, unsigned int page_nr) +{ + struct btrfs_fs_info *fs_info = rbio->bioc->fs_info; + u32 nr_sectors; + + ASSERT(page_nr < rbio->nr_pages); + + nr_sectors = round_up(PAGE_SIZE, fs_info->sectorsize) >> fs_info->sectorsize_bits; + ASSERT(nr_sectors > 0); + return nr_sectors; +} + +static __maybe_unused bool full_page_sectors_uptodate(struct btrfs_raid_bio *rbio, + unsigned int page_nr) +{ + const u32 sector_nr = page_nr_to_sector_nr(rbio, page_nr); + const u32 nr_bits = page_nr_to_num_sectors(rbio, page_nr); + int i; + + ASSERT(page_nr < rbio->nr_pages); + ASSERT(sector_nr + nr_bits < rbio->nr_sectors); + + for (i = sector_nr; i < sector_nr + nr_bits; i++) { + if (!test_bit(i, rbio->stripe_uptodate_bitmap)) + return false; + } + return true; +} + +/* + * Update the stripe_sectors[] array to use correct page and pgoff + * + * Should be called every time any page pointer in stripes_pages[] got modified. + */ +static void index_stripe_sectors(struct btrfs_raid_bio *rbio) +{ + const u32 step = min(rbio->bioc->fs_info->sectorsize, PAGE_SIZE); + u32 offset; + int i; + + for (i = 0, offset = 0; i < rbio->nr_sectors * rbio->sector_nsteps; + i++, offset += step) { + int page_index = offset >> PAGE_SHIFT; + + ASSERT(page_index < rbio->nr_pages); + if (!rbio->stripe_pages[page_index]) + continue; + + rbio->stripe_paddrs[i] = page_to_phys(rbio->stripe_pages[page_index]) + + offset_in_page(offset); + } +} + +static void steal_rbio_page(struct btrfs_raid_bio *src, + struct btrfs_raid_bio *dest, int page_nr) +{ + const u32 sector_nr = page_nr_to_sector_nr(src, page_nr); + const u32 nr_bits = page_nr_to_num_sectors(src, page_nr); + + ASSERT(page_nr < src->nr_pages); + ASSERT(sector_nr + nr_bits < src->nr_sectors); + + if (dest->stripe_pages[page_nr]) + __free_page(dest->stripe_pages[page_nr]); + dest->stripe_pages[page_nr] = src->stripe_pages[page_nr]; + src->stripe_pages[page_nr] = NULL; + + /* Also update the stripe_uptodate_bitmap bits. */ + bitmap_set(dest->stripe_uptodate_bitmap, sector_nr, nr_bits); +} + +static bool is_data_stripe_page(struct btrfs_raid_bio *rbio, int page_nr) +{ + const int sector_nr = page_nr_to_sector_nr(rbio, page_nr); + + /* + * We have ensured PAGE_SIZE is aligned with sectorsize, thus + * we won't have a page which is half data half parity. + * + * Thus if the first sector of the page belongs to data stripes, then + * the full page belongs to data stripes. + */ + return (sector_nr < rbio->nr_data * rbio->stripe_nsectors); +} + +/* + * Stealing an rbio means taking all the uptodate pages from the stripe array + * in the source rbio and putting them into the destination rbio. + * + * This will also update the involved stripe_sectors[] which are referring to + * the old pages. + */ +static void steal_rbio(struct btrfs_raid_bio *src, struct btrfs_raid_bio *dest) +{ + int i; + + if (!test_bit(RBIO_CACHE_READY_BIT, &src->flags)) + return; + + for (i = 0; i < dest->nr_pages; i++) { + struct page *p = src->stripe_pages[i]; + + /* + * We don't need to steal P/Q pages as they will always be + * regenerated for RMW or full write anyway. + */ + if (!is_data_stripe_page(src, i)) + continue; + + /* + * If @src already has RBIO_CACHE_READY_BIT, it should have + * all data stripe pages present and uptodate. + */ + ASSERT(p); + ASSERT(full_page_sectors_uptodate(src, i)); + steal_rbio_page(src, dest, i); + } + index_stripe_sectors(dest); + index_stripe_sectors(src); +} + +/* + * merging means we take the bio_list from the victim and + * splice it into the destination. The victim should + * be discarded afterwards. + * + * must be called with dest->rbio_list_lock held + */ +static void merge_rbio(struct btrfs_raid_bio *dest, + struct btrfs_raid_bio *victim) +{ + bio_list_merge_init(&dest->bio_list, &victim->bio_list); + dest->bio_list_bytes += victim->bio_list_bytes; + /* Also inherit the bitmaps from @victim. */ + bitmap_or(&dest->dbitmap, &victim->dbitmap, &dest->dbitmap, + dest->stripe_nsectors); +} + +/* + * used to prune items that are in the cache. The caller + * must hold the hash table lock. + */ +static void __remove_rbio_from_cache(struct btrfs_raid_bio *rbio) +{ + int bucket = rbio_bucket(rbio); + struct btrfs_stripe_hash_table *table; + struct btrfs_stripe_hash *h; + bool freeit = false; + + /* + * check the bit again under the hash table lock. + */ + if (!test_bit(RBIO_CACHE_BIT, &rbio->flags)) + return; + + table = rbio->bioc->fs_info->stripe_hash_table; + h = table->table + bucket; + + /* hold the lock for the bucket because we may be + * removing it from the hash table + */ + spin_lock(&h->lock); + + /* + * hold the lock for the bio list because we need + * to make sure the bio list is empty + */ + spin_lock(&rbio->bio_list_lock); + + if (test_and_clear_bit(RBIO_CACHE_BIT, &rbio->flags)) { + list_del_init(&rbio->stripe_cache); + table->cache_size -= 1; + freeit = true; + + /* if the bio list isn't empty, this rbio is + * still involved in an IO. We take it out + * of the cache list, and drop the ref that + * was held for the list. + * + * If the bio_list was empty, we also remove + * the rbio from the hash_table, and drop + * the corresponding ref + */ + if (bio_list_empty(&rbio->bio_list)) { + if (!list_empty(&rbio->hash_list)) { + list_del_init(&rbio->hash_list); + refcount_dec(&rbio->refs); + BUG_ON(!list_empty(&rbio->plug_list)); + } + } + } + + spin_unlock(&rbio->bio_list_lock); + spin_unlock(&h->lock); + + if (freeit) + free_raid_bio(rbio); +} + +/* + * prune a given rbio from the cache + */ +static void remove_rbio_from_cache(struct btrfs_raid_bio *rbio) +{ + struct btrfs_stripe_hash_table *table; + + if (!test_bit(RBIO_CACHE_BIT, &rbio->flags)) + return; + + table = rbio->bioc->fs_info->stripe_hash_table; + + spin_lock(&table->cache_lock); + __remove_rbio_from_cache(rbio); + spin_unlock(&table->cache_lock); +} + +/* + * remove everything in the cache + */ +static void btrfs_clear_rbio_cache(struct btrfs_fs_info *info) +{ + struct btrfs_stripe_hash_table *table; + struct btrfs_raid_bio *rbio; + + table = info->stripe_hash_table; + + spin_lock(&table->cache_lock); + while (!list_empty(&table->stripe_cache)) { + rbio = list_first_entry(&table->stripe_cache, + struct btrfs_raid_bio, stripe_cache); + __remove_rbio_from_cache(rbio); + } + spin_unlock(&table->cache_lock); +} + +/* + * remove all cached entries and free the hash table + * used by unmount + */ +void btrfs_free_stripe_hash_table(struct btrfs_fs_info *info) +{ + if (!info->stripe_hash_table) + return; + btrfs_clear_rbio_cache(info); + kvfree(info->stripe_hash_table); + info->stripe_hash_table = NULL; +} + +/* + * insert an rbio into the stripe cache. It + * must have already been prepared by calling + * cache_rbio_pages + * + * If this rbio was already cached, it gets + * moved to the front of the lru. + * + * If the size of the rbio cache is too big, we + * prune an item. + */ +static void cache_rbio(struct btrfs_raid_bio *rbio) +{ + struct btrfs_stripe_hash_table *table; + + if (!test_bit(RBIO_CACHE_READY_BIT, &rbio->flags)) + return; + + table = rbio->bioc->fs_info->stripe_hash_table; + + spin_lock(&table->cache_lock); + spin_lock(&rbio->bio_list_lock); + + /* bump our ref if we were not in the list before */ + if (!test_and_set_bit(RBIO_CACHE_BIT, &rbio->flags)) + refcount_inc(&rbio->refs); + + if (!list_empty(&rbio->stripe_cache)){ + list_move(&rbio->stripe_cache, &table->stripe_cache); + } else { + list_add(&rbio->stripe_cache, &table->stripe_cache); + table->cache_size += 1; + } + + spin_unlock(&rbio->bio_list_lock); + + if (table->cache_size > RBIO_CACHE_SIZE) { + struct btrfs_raid_bio *found; + + found = list_last_entry(&table->stripe_cache, + struct btrfs_raid_bio, + stripe_cache); + + if (found != rbio) + __remove_rbio_from_cache(found); + } + + spin_unlock(&table->cache_lock); +} + +/* + * Returns true if the bio list inside this rbio covers an entire stripe (no + * rmw required). + */ +static int rbio_is_full(struct btrfs_raid_bio *rbio) +{ + unsigned long size = rbio->bio_list_bytes; + int ret = 1; + + spin_lock(&rbio->bio_list_lock); + if (size != rbio->nr_data * BTRFS_STRIPE_LEN) + ret = 0; + BUG_ON(size > rbio->nr_data * BTRFS_STRIPE_LEN); + spin_unlock(&rbio->bio_list_lock); + + return ret; +} + +/* + * returns 1 if it is safe to merge two rbios together. + * The merging is safe if the two rbios correspond to + * the same stripe and if they are both going in the same + * direction (read vs write), and if neither one is + * locked for final IO + * + * The caller is responsible for locking such that + * rmw_locked is safe to test + */ +static int rbio_can_merge(struct btrfs_raid_bio *last, + struct btrfs_raid_bio *cur) +{ + if (test_bit(RBIO_RMW_LOCKED_BIT, &last->flags) || + test_bit(RBIO_RMW_LOCKED_BIT, &cur->flags)) + return 0; + + /* + * we can't merge with cached rbios, since the + * idea is that when we merge the destination + * rbio is going to run our IO for us. We can + * steal from cached rbios though, other functions + * handle that. + */ + if (test_bit(RBIO_CACHE_BIT, &last->flags) || + test_bit(RBIO_CACHE_BIT, &cur->flags)) + return 0; + + if (last->bioc->full_stripe_logical != cur->bioc->full_stripe_logical) + return 0; + + /* we can't merge with different operations */ + if (last->operation != cur->operation) + return 0; + /* + * We've need read the full stripe from the drive. + * check and repair the parity and write the new results. + * + * We're not allowed to add any new bios to the + * bio list here, anyone else that wants to + * change this stripe needs to do their own rmw. + */ + if (last->operation == BTRFS_RBIO_PARITY_SCRUB) + return 0; + + if (last->operation == BTRFS_RBIO_READ_REBUILD) + return 0; + + return 1; +} + +/* Return the sector index for @stripe_nr and @sector_nr. */ +static unsigned int rbio_sector_index(const struct btrfs_raid_bio *rbio, + unsigned int stripe_nr, + unsigned int sector_nr) +{ + unsigned int ret; + + ASSERT_RBIO_STRIPE(stripe_nr < rbio->real_stripes, rbio, stripe_nr); + ASSERT_RBIO_SECTOR(sector_nr < rbio->stripe_nsectors, rbio, sector_nr); + + ret = stripe_nr * rbio->stripe_nsectors + sector_nr; + ASSERT(ret < rbio->nr_sectors); + return ret; +} + +/* Return the paddr array index for @stripe_nr, @sector_nr and @step_nr. */ +static unsigned int rbio_paddr_index(const struct btrfs_raid_bio *rbio, + unsigned int stripe_nr, + unsigned int sector_nr, + unsigned int step_nr) +{ + unsigned int ret; + + ASSERT_RBIO_SECTOR(step_nr < rbio->sector_nsteps, rbio, step_nr); + + ret = rbio_sector_index(rbio, stripe_nr, sector_nr) * rbio->sector_nsteps + step_nr; + ASSERT(ret < rbio->nr_sectors * rbio->sector_nsteps); + return ret; +} + +static phys_addr_t rbio_stripe_paddr(const struct btrfs_raid_bio *rbio, + unsigned int stripe_nr, unsigned int sector_nr, + unsigned int step_nr) +{ + return rbio->stripe_paddrs[rbio_paddr_index(rbio, stripe_nr, sector_nr, step_nr)]; +} + +static phys_addr_t rbio_pstripe_paddr(const struct btrfs_raid_bio *rbio, + unsigned int sector_nr, unsigned int step_nr) +{ + return rbio_stripe_paddr(rbio, rbio->nr_data, sector_nr, step_nr); +} + +static phys_addr_t rbio_qstripe_paddr(const struct btrfs_raid_bio *rbio, + unsigned int sector_nr, unsigned int step_nr) +{ + if (rbio->nr_data + 1 == rbio->real_stripes) + return INVALID_PADDR; + return rbio_stripe_paddr(rbio, rbio->nr_data + 1, sector_nr, step_nr); +} + +/* Return a paddr pointer into the rbio::stripe_paddrs[] for the specified sector. */ +static phys_addr_t *rbio_stripe_paddrs(const struct btrfs_raid_bio *rbio, + unsigned int stripe_nr, unsigned int sector_nr) +{ + return &rbio->stripe_paddrs[rbio_paddr_index(rbio, stripe_nr, sector_nr, 0)]; +} + +/* + * The first stripe in the table for a logical address + * has the lock. rbios are added in one of three ways: + * + * 1) Nobody has the stripe locked yet. The rbio is given + * the lock and 0 is returned. The caller must start the IO + * themselves. + * + * 2) Someone has the stripe locked, but we're able to merge + * with the lock owner. The rbio is freed and the IO will + * start automatically along with the existing rbio. 1 is returned. + * + * 3) Someone has the stripe locked, but we're not able to merge. + * The rbio is added to the lock owner's plug list, or merged into + * an rbio already on the plug list. When the lock owner unlocks, + * the next rbio on the list is run and the IO is started automatically. + * 1 is returned + * + * If we return 0, the caller still owns the rbio and must continue with + * IO submission. If we return 1, the caller must assume the rbio has + * already been freed. + */ +static noinline int lock_stripe_add(struct btrfs_raid_bio *rbio) +{ + struct btrfs_stripe_hash *h; + struct btrfs_raid_bio *cur; + struct btrfs_raid_bio *pending; + struct btrfs_raid_bio *freeit = NULL; + struct btrfs_raid_bio *cache_drop = NULL; + int ret = 0; + + h = rbio->bioc->fs_info->stripe_hash_table->table + rbio_bucket(rbio); + + spin_lock(&h->lock); + list_for_each_entry(cur, &h->hash_list, hash_list) { + if (cur->bioc->full_stripe_logical != rbio->bioc->full_stripe_logical) + continue; + + spin_lock(&cur->bio_list_lock); + + /* Can we steal this cached rbio's pages? */ + if (bio_list_empty(&cur->bio_list) && + list_empty(&cur->plug_list) && + test_bit(RBIO_CACHE_BIT, &cur->flags) && + !test_bit(RBIO_RMW_LOCKED_BIT, &cur->flags)) { + list_del_init(&cur->hash_list); + refcount_dec(&cur->refs); + + steal_rbio(cur, rbio); + cache_drop = cur; + spin_unlock(&cur->bio_list_lock); + + goto lockit; + } + + /* Can we merge into the lock owner? */ + if (rbio_can_merge(cur, rbio)) { + merge_rbio(cur, rbio); + spin_unlock(&cur->bio_list_lock); + freeit = rbio; + ret = 1; + goto out; + } + + + /* + * We couldn't merge with the running rbio, see if we can merge + * with the pending ones. We don't have to check for rmw_locked + * because there is no way they are inside finish_rmw right now + */ + list_for_each_entry(pending, &cur->plug_list, plug_list) { + if (rbio_can_merge(pending, rbio)) { + merge_rbio(pending, rbio); + spin_unlock(&cur->bio_list_lock); + freeit = rbio; + ret = 1; + goto out; + } + } + + /* + * No merging, put us on the tail of the plug list, our rbio + * will be started with the currently running rbio unlocks + */ + list_add_tail(&rbio->plug_list, &cur->plug_list); + spin_unlock(&cur->bio_list_lock); + ret = 1; + goto out; + } +lockit: + refcount_inc(&rbio->refs); + list_add(&rbio->hash_list, &h->hash_list); +out: + spin_unlock(&h->lock); + if (cache_drop) + remove_rbio_from_cache(cache_drop); + if (freeit) + free_raid_bio(freeit); + return ret; +} + +static void recover_rbio_work_locked(struct work_struct *work); + +/* + * called as rmw or parity rebuild is completed. If the plug list has more + * rbios waiting for this stripe, the next one on the list will be started + */ +static noinline void unlock_stripe(struct btrfs_raid_bio *rbio) +{ + int bucket; + struct btrfs_stripe_hash *h; + bool keep_cache = false; + + bucket = rbio_bucket(rbio); + h = rbio->bioc->fs_info->stripe_hash_table->table + bucket; + + if (list_empty(&rbio->plug_list)) + cache_rbio(rbio); + + spin_lock(&h->lock); + spin_lock(&rbio->bio_list_lock); + + if (!list_empty(&rbio->hash_list)) { + /* + * if we're still cached and there is no other IO + * to perform, just leave this rbio here for others + * to steal from later + */ + if (list_empty(&rbio->plug_list) && + test_bit(RBIO_CACHE_BIT, &rbio->flags)) { + keep_cache = true; + clear_bit(RBIO_RMW_LOCKED_BIT, &rbio->flags); + BUG_ON(!bio_list_empty(&rbio->bio_list)); + goto done; + } + + list_del_init(&rbio->hash_list); + refcount_dec(&rbio->refs); + + /* + * we use the plug list to hold all the rbios + * waiting for the chance to lock this stripe. + * hand the lock over to one of them. + */ + if (!list_empty(&rbio->plug_list)) { + struct btrfs_raid_bio *next; + struct list_head *head = rbio->plug_list.next; + + next = list_entry(head, struct btrfs_raid_bio, + plug_list); + + list_del_init(&rbio->plug_list); + + list_add(&next->hash_list, &h->hash_list); + refcount_inc(&next->refs); + spin_unlock(&rbio->bio_list_lock); + spin_unlock(&h->lock); + + if (next->operation == BTRFS_RBIO_READ_REBUILD) { + start_async_work(next, recover_rbio_work_locked); + } else if (next->operation == BTRFS_RBIO_WRITE) { + steal_rbio(rbio, next); + start_async_work(next, rmw_rbio_work_locked); + } else if (next->operation == BTRFS_RBIO_PARITY_SCRUB) { + steal_rbio(rbio, next); + start_async_work(next, scrub_rbio_work_locked); + } + + goto done_nolock; + } + } +done: + spin_unlock(&rbio->bio_list_lock); + spin_unlock(&h->lock); + +done_nolock: + if (!keep_cache) + remove_rbio_from_cache(rbio); +} + +static void rbio_endio_bio_list(struct bio *cur, blk_status_t status) +{ + struct bio *next; + + while (cur) { + next = cur->bi_next; + cur->bi_next = NULL; + cur->bi_status = status; + bio_endio(cur); + cur = next; + } +} + +/* + * this frees the rbio and runs through all the bios in the + * bio_list and calls end_io on them + */ +static void rbio_orig_end_io(struct btrfs_raid_bio *rbio, blk_status_t status) +{ + struct bio *cur = bio_list_get(&rbio->bio_list); + struct bio *extra; + + kfree(rbio->csum_buf); + bitmap_free(rbio->csum_bitmap); + rbio->csum_buf = NULL; + rbio->csum_bitmap = NULL; + + /* + * Clear the data bitmap, as the rbio may be cached for later usage. + * do this before before unlock_stripe() so there will be no new bio + * for this bio. + */ + bitmap_clear(&rbio->dbitmap, 0, rbio->stripe_nsectors); + + /* + * At this moment, rbio->bio_list is empty, however since rbio does not + * always have RBIO_RMW_LOCKED_BIT set and rbio is still linked on the + * hash list, rbio may be merged with others so that rbio->bio_list + * becomes non-empty. + * Once unlock_stripe() is done, rbio->bio_list will not be updated any + * more and we can call bio_endio() on all queued bios. + */ + unlock_stripe(rbio); + extra = bio_list_get(&rbio->bio_list); + free_raid_bio(rbio); + + rbio_endio_bio_list(cur, status); + if (extra) + rbio_endio_bio_list(extra, status); +} + +/* + * Get paddr pointer for the sector specified by its @stripe_nr and @sector_nr. + * + * @rbio: The raid bio + * @stripe_nr: Stripe number, valid range [0, real_stripe) + * @sector_nr: Sector number inside the stripe, + * valid range [0, stripe_nsectors) + * @bio_list_only: Whether to use sectors inside the bio list only. + * + * The read/modify/write code wants to reuse the original bio page as much + * as possible, and only use stripe_sectors as fallback. + * + * Return NULL if bio_list_only is set but the specified sector has no + * coresponding bio. + */ +static phys_addr_t *sector_paddrs_in_rbio(struct btrfs_raid_bio *rbio, + int stripe_nr, int sector_nr, + bool bio_list_only) +{ + phys_addr_t *ret = NULL; + const int index = rbio_paddr_index(rbio, stripe_nr, sector_nr, 0); + + ASSERT(index >= 0 && index < rbio->nr_sectors * rbio->sector_nsteps); + + scoped_guard(spinlock, &rbio->bio_list_lock) { + if (rbio->bio_paddrs[index] != INVALID_PADDR || bio_list_only) { + /* Don't return sector without a valid page pointer */ + if (rbio->bio_paddrs[index] != INVALID_PADDR) + ret = &rbio->bio_paddrs[index]; + return ret; + } + } + return &rbio->stripe_paddrs[index]; +} + +/* + * Similar to sector_paddr_in_rbio(), but with extra consideration for + * bs > ps cases, where we can have multiple steps for a fs block. + */ +static phys_addr_t sector_paddr_in_rbio(struct btrfs_raid_bio *rbio, + int stripe_nr, int sector_nr, int step_nr, + bool bio_list_only) +{ + phys_addr_t ret = INVALID_PADDR; + const int index = rbio_paddr_index(rbio, stripe_nr, sector_nr, step_nr); + + ASSERT(index >= 0 && index < rbio->nr_sectors * rbio->sector_nsteps); + + scoped_guard(spinlock, &rbio->bio_list_lock) { + if (rbio->bio_paddrs[index] != INVALID_PADDR || bio_list_only) { + /* Don't return sector without a valid page pointer */ + if (rbio->bio_paddrs[index] != INVALID_PADDR) + ret = rbio->bio_paddrs[index]; + return ret; + } + } + return rbio->stripe_paddrs[index]; +} + +/* + * allocation and initial setup for the btrfs_raid_bio. Not + * this does not allocate any pages for rbio->pages. + */ +static struct btrfs_raid_bio *alloc_rbio(struct btrfs_fs_info *fs_info, + struct btrfs_io_context *bioc) +{ + const unsigned int real_stripes = bioc->num_stripes - bioc->replace_nr_stripes; + const unsigned int stripe_npages = BTRFS_STRIPE_LEN >> PAGE_SHIFT; + const unsigned int num_pages = stripe_npages * real_stripes; + const unsigned int stripe_nsectors = + BTRFS_STRIPE_LEN >> fs_info->sectorsize_bits; + const unsigned int num_sectors = stripe_nsectors * real_stripes; + const unsigned int step = min(fs_info->sectorsize, PAGE_SIZE); + const unsigned int sector_nsteps = fs_info->sectorsize / step; + struct btrfs_raid_bio *rbio; + + /* + * For bs <= ps cases, ps must be aligned to bs. + * For bs > ps cases, bs must be aligned to ps. + */ + ASSERT(IS_ALIGNED(PAGE_SIZE, fs_info->sectorsize) || + IS_ALIGNED(fs_info->sectorsize, PAGE_SIZE)); + /* + * Our current stripe len should be fixed to 64k thus stripe_nsectors + * (at most 16) should be no larger than BITS_PER_LONG. + */ + ASSERT(stripe_nsectors <= BITS_PER_LONG); + + /* + * Real stripes must be between 2 (2 disks RAID5, aka RAID1) and 256 + * (limited by u8). + */ + ASSERT(real_stripes >= 2); + ASSERT(real_stripes <= U8_MAX); + + rbio = kzalloc_obj(*rbio, GFP_NOFS); + if (!rbio) + return ERR_PTR(-ENOMEM); + rbio->stripe_pages = kzalloc_objs(struct page *, num_pages, GFP_NOFS); + rbio->bio_paddrs = kzalloc_objs(phys_addr_t, + num_sectors * sector_nsteps, GFP_NOFS); + rbio->stripe_paddrs = kzalloc_objs(phys_addr_t, + num_sectors * sector_nsteps, + GFP_NOFS); + rbio->finish_pointers = kcalloc(real_stripes, sizeof(void *), GFP_NOFS); + rbio->error_bitmap = bitmap_zalloc(num_sectors, GFP_NOFS); + rbio->stripe_uptodate_bitmap = bitmap_zalloc(num_sectors, GFP_NOFS); + + if (!rbio->stripe_pages || !rbio->bio_paddrs || !rbio->stripe_paddrs || + !rbio->finish_pointers || !rbio->error_bitmap || !rbio->stripe_uptodate_bitmap) { + free_raid_bio_pointers(rbio); + kfree(rbio); + return ERR_PTR(-ENOMEM); + } + for (int i = 0; i < num_sectors * sector_nsteps; i++) { + rbio->stripe_paddrs[i] = INVALID_PADDR; + rbio->bio_paddrs[i] = INVALID_PADDR; + } + + bio_list_init(&rbio->bio_list); + init_waitqueue_head(&rbio->io_wait); + INIT_LIST_HEAD(&rbio->plug_list); + spin_lock_init(&rbio->bio_list_lock); + INIT_LIST_HEAD(&rbio->stripe_cache); + INIT_LIST_HEAD(&rbio->hash_list); + btrfs_get_bioc(bioc); + rbio->bioc = bioc; + rbio->nr_pages = num_pages; + rbio->nr_sectors = num_sectors; + rbio->real_stripes = real_stripes; + rbio->stripe_npages = stripe_npages; + rbio->stripe_nsectors = stripe_nsectors; + rbio->sector_nsteps = sector_nsteps; + refcount_set(&rbio->refs, 1); + atomic_set(&rbio->stripes_pending, 0); + + ASSERT(btrfs_nr_parity_stripes(bioc->map_type)); + rbio->nr_data = real_stripes - btrfs_nr_parity_stripes(bioc->map_type); + ASSERT(rbio->nr_data > 0); + + return rbio; +} + +/* allocate pages for all the stripes in the bio, including parity */ +static int alloc_rbio_pages(struct btrfs_raid_bio *rbio) +{ + int ret; + + ret = btrfs_alloc_page_array(rbio->nr_pages, rbio->stripe_pages, GFP_NOFS); + if (ret < 0) + return ret; + /* Mapping all sectors */ + index_stripe_sectors(rbio); + return 0; +} + +/* only allocate pages for p/q stripes */ +static int alloc_rbio_parity_pages(struct btrfs_raid_bio *rbio) +{ + const int data_pages = rbio->nr_data * rbio->stripe_npages; + int ret; + + ret = btrfs_alloc_page_array(rbio->nr_pages - data_pages, + rbio->stripe_pages + data_pages, GFP_NOFS); + if (ret < 0) + return ret; + + index_stripe_sectors(rbio); + return 0; +} + +/* + * Return the total number of errors found in the vertical stripe of @sector_nr. + * + * @faila and @failb will also be updated to the first and second stripe + * number of the errors. + */ +static int get_rbio_vertical_errors(struct btrfs_raid_bio *rbio, int sector_nr, + int *faila, int *failb) +{ + int stripe_nr; + int found_errors = 0; + + if (faila || failb) { + /* + * Both @faila and @failb should be valid pointers if any of + * them is specified. + */ + ASSERT(faila && failb); + *faila = -1; + *failb = -1; + } + + for (stripe_nr = 0; stripe_nr < rbio->real_stripes; stripe_nr++) { + int total_sector_nr = stripe_nr * rbio->stripe_nsectors + sector_nr; + + if (test_bit(total_sector_nr, rbio->error_bitmap)) { + found_errors++; + if (faila) { + /* Update faila and failb. */ + if (*faila < 0) + *faila = stripe_nr; + else if (*failb < 0) + *failb = stripe_nr; + } + } + } + return found_errors; +} + +static int bio_add_paddrs(struct bio *bio, phys_addr_t *paddrs, unsigned int nr_steps, + unsigned int step) +{ + int added = 0; + int ret; + + for (int i = 0; i < nr_steps; i++) { + ret = bio_add_page(bio, phys_to_page(paddrs[i]), step, + offset_in_page(paddrs[i])); + if (ret != step) + goto revert; + added += ret; + } + return added; +revert: + /* + * We don't need to revert the bvec, as the bio will be submitted immediately, + * as long as the size is reduced the extra bvec will not be accessed. + */ + bio->bi_iter.bi_size -= added; + return 0; +} + +/* + * Add a single sector @sector into our list of bios for IO. + * + * Return 0 if everything went well. + * Return <0 for error, and no byte will be added to @rbio. + */ +static int rbio_add_io_paddrs(struct btrfs_raid_bio *rbio, struct bio_list *bio_list, + phys_addr_t *paddrs, unsigned int stripe_nr, + unsigned int sector_nr, enum req_op op) +{ + const u32 sectorsize = rbio->bioc->fs_info->sectorsize; + const u32 step = min(sectorsize, PAGE_SIZE); + struct bio *last = bio_list->tail; + int ret; + struct bio *bio; + struct btrfs_io_stripe *stripe; + u64 disk_start; + + /* + * Note: here stripe_nr has taken device replace into consideration, + * thus it can be larger than rbio->real_stripe. + * So here we check against bioc->num_stripes, not rbio->real_stripes. + */ + ASSERT_RBIO_STRIPE(stripe_nr >= 0 && stripe_nr < rbio->bioc->num_stripes, + rbio, stripe_nr); + ASSERT_RBIO_SECTOR(sector_nr >= 0 && sector_nr < rbio->stripe_nsectors, + rbio, sector_nr); + ASSERT(paddrs != NULL); + + stripe = &rbio->bioc->stripes[stripe_nr]; + disk_start = stripe->physical + sector_nr * sectorsize; + + /* if the device is missing, just fail this stripe */ + if (!stripe->dev->bdev) { + int found_errors; + + set_bit(stripe_nr * rbio->stripe_nsectors + sector_nr, + rbio->error_bitmap); + + /* Check if we have reached tolerance early. */ + found_errors = get_rbio_vertical_errors(rbio, sector_nr, + NULL, NULL); + if (unlikely(found_errors > rbio->bioc->max_errors)) + return -EIO; + return 0; + } + + /* see if we can add this page onto our existing bio */ + if (last) { + u64 last_end = last->bi_iter.bi_sector << SECTOR_SHIFT; + last_end += last->bi_iter.bi_size; + + /* + * we can't merge these if they are from different + * devices or if they are not contiguous + */ + if (last_end == disk_start && !last->bi_status && + last->bi_bdev == stripe->dev->bdev) { + ret = bio_add_paddrs(last, paddrs, rbio->sector_nsteps, step); + if (ret == sectorsize) + return 0; + } + } + + /* put a new bio on the list */ + bio = bio_alloc(stripe->dev->bdev, + max(BTRFS_STRIPE_LEN >> PAGE_SHIFT, 1), + op, GFP_NOFS); + bio->bi_iter.bi_sector = disk_start >> SECTOR_SHIFT; + bio->bi_private = rbio; + + ret = bio_add_paddrs(bio, paddrs, rbio->sector_nsteps, step); + ASSERT(ret == sectorsize); + bio_list_add(bio_list, bio); + return 0; +} + +static void index_one_bio(struct btrfs_raid_bio *rbio, struct bio *bio) +{ + struct btrfs_fs_info *fs_info = rbio->bioc->fs_info; + const u32 step = min(fs_info->sectorsize, PAGE_SIZE); + const u32 step_bits = min(fs_info->sectorsize_bits, PAGE_SHIFT); + struct bvec_iter iter = bio->bi_iter; + phys_addr_t paddr; + u32 offset = (bio->bi_iter.bi_sector << SECTOR_SHIFT) - + rbio->bioc->full_stripe_logical; + + btrfs_bio_for_each_block(paddr, bio, &iter, step) { + unsigned int index = (offset >> step_bits); + + rbio->bio_paddrs[index] = paddr; + offset += step; + } +} + +/* + * helper function to walk our bio list and populate the bio_pages array with + * the result. This seems expensive, but it is faster than constantly + * searching through the bio list as we setup the IO in finish_rmw or stripe + * reconstruction. + * + * This must be called before you trust the answers from page_in_rbio + */ +static void index_rbio_pages(struct btrfs_raid_bio *rbio) +{ + struct bio *bio; + + spin_lock(&rbio->bio_list_lock); + bio_list_for_each(bio, &rbio->bio_list) + index_one_bio(rbio, bio); + + spin_unlock(&rbio->bio_list_lock); +} + +static void bio_get_trace_info(struct btrfs_raid_bio *rbio, struct bio *bio, + struct raid56_bio_trace_info *trace_info) +{ + const struct btrfs_io_context *bioc = rbio->bioc; + int i; + + ASSERT(bioc); + + /* We rely on bio->bi_bdev to find the stripe number. */ + if (!bio->bi_bdev) + goto not_found; + + for (i = 0; i < bioc->num_stripes; i++) { + if (bio->bi_bdev != bioc->stripes[i].dev->bdev) + continue; + trace_info->stripe_nr = i; + trace_info->devid = bioc->stripes[i].dev->devid; + trace_info->offset = (bio->bi_iter.bi_sector << SECTOR_SHIFT) - + bioc->stripes[i].physical; + return; + } + +not_found: + trace_info->devid = -1; + trace_info->offset = -1; + trace_info->stripe_nr = -1; +} + +static inline void bio_list_put(struct bio_list *bio_list) +{ + struct bio *bio; + + while ((bio = bio_list_pop(bio_list))) + bio_put(bio); +} + +static void assert_rbio(struct btrfs_raid_bio *rbio) +{ + if (!IS_ENABLED(CONFIG_BTRFS_ASSERT)) + return; + + /* + * At least two stripes (2 disks RAID5), and since real_stripes is U8, + * we won't go beyond 256 disks anyway. + */ + ASSERT_RBIO(rbio->real_stripes >= 2, rbio); + ASSERT_RBIO(rbio->nr_data > 0, rbio); + + /* + * This is another check to make sure nr data stripes is smaller + * than total stripes. + */ + ASSERT_RBIO(rbio->nr_data < rbio->real_stripes, rbio); +} + +static inline void *kmap_local_paddr(phys_addr_t paddr) +{ + /* The sector pointer must have a page mapped to it. */ + ASSERT(paddr != INVALID_PADDR); + + return kmap_local_page(phys_to_page(paddr)) + offset_in_page(paddr); +} + +static void generate_pq_vertical_step(struct btrfs_raid_bio *rbio, unsigned int sector_nr, + unsigned int step_nr) +{ + void **pointers = rbio->finish_pointers; + const u32 step = min(rbio->bioc->fs_info->sectorsize, PAGE_SIZE); + int stripe; + const bool has_qstripe = rbio->bioc->map_type & BTRFS_BLOCK_GROUP_RAID6; + + /* First collect one sector from each data stripe */ + for (stripe = 0; stripe < rbio->nr_data; stripe++) + pointers[stripe] = kmap_local_paddr( + sector_paddr_in_rbio(rbio, stripe, sector_nr, step_nr, 0)); + + /* Then add the parity stripe */ + pointers[stripe++] = kmap_local_paddr(rbio_pstripe_paddr(rbio, sector_nr, step_nr)); + + if (has_qstripe) { + /* + * RAID6, add the qstripe and call the library function + * to fill in our p/q + */ + pointers[stripe++] = kmap_local_paddr( + rbio_qstripe_paddr(rbio, sector_nr, step_nr)); + + assert_rbio(rbio); + raid6_gen_syndrome(rbio->real_stripes, step, pointers); + } else { + /* raid5 */ + memcpy(pointers[rbio->nr_data], pointers[0], step); + xor_gen(pointers[rbio->nr_data], pointers + 1, rbio->nr_data - 1, + step); + } + for (stripe = stripe - 1; stripe >= 0; stripe--) + kunmap_local(pointers[stripe]); +} + +/* Generate PQ for one vertical stripe. */ +static void generate_pq_vertical(struct btrfs_raid_bio *rbio, int sectornr) +{ + const bool has_qstripe = (rbio->bioc->map_type & BTRFS_BLOCK_GROUP_RAID6); + + for (int i = 0; i < rbio->sector_nsteps; i++) + generate_pq_vertical_step(rbio, sectornr, i); + + set_bit(rbio_sector_index(rbio, rbio->nr_data, sectornr), + rbio->stripe_uptodate_bitmap); + if (has_qstripe) + set_bit(rbio_sector_index(rbio, rbio->nr_data + 1, sectornr), + rbio->stripe_uptodate_bitmap); +} + +static int rmw_assemble_write_bios(struct btrfs_raid_bio *rbio, + struct bio_list *bio_list) +{ + /* The total sector number inside the full stripe. */ + int total_sector_nr; + int sectornr; + int stripe; + int ret; + + ASSERT(bio_list_size(bio_list) == 0); + + /* We should have at least one data sector. */ + ASSERT(bitmap_weight(&rbio->dbitmap, rbio->stripe_nsectors)); + + /* + * Reset errors, as we may have errors inherited from from degraded + * write. + */ + bitmap_clear(rbio->error_bitmap, 0, rbio->nr_sectors); + + /* + * Start assembly. Make bios for everything from the higher layers (the + * bio_list in our rbio) and our P/Q. Ignore everything else. + */ + for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors; + total_sector_nr++) { + phys_addr_t *paddrs; + + stripe = total_sector_nr / rbio->stripe_nsectors; + sectornr = total_sector_nr % rbio->stripe_nsectors; + + /* This vertical stripe has no data, skip it. */ + if (!test_bit(sectornr, &rbio->dbitmap)) + continue; + + if (stripe < rbio->nr_data) { + paddrs = sector_paddrs_in_rbio(rbio, stripe, sectornr, 1); + if (paddrs == NULL) + continue; + } else { + paddrs = rbio_stripe_paddrs(rbio, stripe, sectornr); + } + + ret = rbio_add_io_paddrs(rbio, bio_list, paddrs, stripe, + sectornr, REQ_OP_WRITE); + if (ret) + goto error; + } + + if (likely(!rbio->bioc->replace_nr_stripes)) + return 0; + + /* + * Make a copy for the replace target device. + * + * Thus the source stripe number (in replace_stripe_src) should be valid. + */ + ASSERT(rbio->bioc->replace_stripe_src >= 0); + + for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors; + total_sector_nr++) { + phys_addr_t *paddrs; + + stripe = total_sector_nr / rbio->stripe_nsectors; + sectornr = total_sector_nr % rbio->stripe_nsectors; + + /* + * For RAID56, there is only one device that can be replaced, + * and replace_stripe_src[0] indicates the stripe number we + * need to copy from. + */ + if (stripe != rbio->bioc->replace_stripe_src) { + /* + * We can skip the whole stripe completely, note + * total_sector_nr will be increased by one anyway. + */ + ASSERT(sectornr == 0); + total_sector_nr += rbio->stripe_nsectors - 1; + continue; + } + + /* This vertical stripe has no data, skip it. */ + if (!test_bit(sectornr, &rbio->dbitmap)) + continue; + + if (stripe < rbio->nr_data) { + paddrs = sector_paddrs_in_rbio(rbio, stripe, sectornr, 1); + if (paddrs == NULL) + continue; + } else { + paddrs = rbio_stripe_paddrs(rbio, stripe, sectornr); + } + + ret = rbio_add_io_paddrs(rbio, bio_list, paddrs, + rbio->real_stripes, + sectornr, REQ_OP_WRITE); + if (ret) + goto error; + } + + return 0; +error: + bio_list_put(bio_list); + return -EIO; +} + +static void set_rbio_range_error(struct btrfs_raid_bio *rbio, struct bio *bio) +{ + struct btrfs_fs_info *fs_info = rbio->bioc->fs_info; + u32 offset = (bio->bi_iter.bi_sector << SECTOR_SHIFT) - + rbio->bioc->full_stripe_logical; + int total_nr_sector = offset >> fs_info->sectorsize_bits; + + ASSERT(total_nr_sector < rbio->nr_data * rbio->stripe_nsectors); + + bitmap_set(rbio->error_bitmap, total_nr_sector, + bio->bi_iter.bi_size >> fs_info->sectorsize_bits); + + /* + * Special handling for raid56_alloc_missing_rbio() used by + * scrub/replace. Unlike call path in raid56_parity_recover(), they + * pass an empty bio here. Thus we have to find out the missing device + * and mark the stripe error instead. + */ + if (bio->bi_iter.bi_size == 0) { + bool found_missing = false; + int stripe_nr; + + for (stripe_nr = 0; stripe_nr < rbio->real_stripes; stripe_nr++) { + if (!rbio->bioc->stripes[stripe_nr].dev->bdev) { + found_missing = true; + bitmap_set(rbio->error_bitmap, + stripe_nr * rbio->stripe_nsectors, + rbio->stripe_nsectors); + } + } + ASSERT(found_missing); + } +} + +/* + * Return the index inside the rbio->stripe_sectors[] array. + * + * Return -1 if not found. + */ +static int find_stripe_sector_nr(struct btrfs_raid_bio *rbio, phys_addr_t paddr) +{ + for (int i = 0; i < rbio->nr_sectors; i++) { + if (rbio->stripe_paddrs[i * rbio->sector_nsteps] == paddr) + return i; + } + return -1; +} + +/* + * this sets each page in the bio uptodate. It should only be used on private + * rbio pages, nothing that comes in from the higher layers + */ +static void set_bio_pages_uptodate(struct btrfs_raid_bio *rbio, struct bio *bio) +{ + const u32 sectorsize = rbio->bioc->fs_info->sectorsize; + const u32 step = min(sectorsize, PAGE_SIZE); + u32 offset = 0; + phys_addr_t paddr; + + ASSERT(!bio_flagged(bio, BIO_CLONED)); + + btrfs_bio_for_each_block_all(paddr, bio, step) { + /* Hitting the first step of a sector. */ + if (IS_ALIGNED(offset, sectorsize)) { + int sector_nr = find_stripe_sector_nr(rbio, paddr); + + ASSERT(sector_nr >= 0); + if (sector_nr >= 0) + set_bit(sector_nr, rbio->stripe_uptodate_bitmap); + } + offset += step; + } +} + +static int get_bio_sector_nr(struct btrfs_raid_bio *rbio, struct bio *bio) +{ + phys_addr_t bvec_paddr = bvec_phys(bio_first_bvec_all(bio)); + int i; + + for (i = 0; i < rbio->nr_sectors; i++) { + if (rbio->stripe_paddrs[i * rbio->sector_nsteps] == bvec_paddr) + break; + if (rbio->bio_paddrs[i * rbio->sector_nsteps] == bvec_paddr) + break; + } + ASSERT(i < rbio->nr_sectors); + return i; +} + +static void rbio_update_error_bitmap(struct btrfs_raid_bio *rbio, struct bio *bio) +{ + int total_sector_nr = get_bio_sector_nr(rbio, bio); + const u32 bio_size = bio_get_size(bio); + + /* + * Since we can have multiple bios touching the error_bitmap, we cannot + * call bitmap_set() without protection. + * + * Instead use set_bit() for each bit, as set_bit() itself is atomic. + */ + for (int i = total_sector_nr; i < total_sector_nr + + (bio_size >> rbio->bioc->fs_info->sectorsize_bits); i++) + set_bit(i, rbio->error_bitmap); +} + +/* Verify the data sectors at read time. */ +static void verify_bio_data_sectors(struct btrfs_raid_bio *rbio, + struct bio *bio) +{ + struct btrfs_fs_info *fs_info = rbio->bioc->fs_info; + const u32 step = min(fs_info->sectorsize, PAGE_SIZE); + const u32 nr_steps = rbio->sector_nsteps; + int total_sector_nr = get_bio_sector_nr(rbio, bio); + u32 offset = 0; + phys_addr_t paddrs[BTRFS_MAX_BLOCKSIZE / PAGE_SIZE]; + phys_addr_t paddr; + + /* No data csum for the whole stripe, no need to verify. */ + if (!rbio->csum_bitmap || !rbio->csum_buf) + return; + + /* P/Q stripes, they have no data csum to verify against. */ + if (total_sector_nr >= rbio->nr_data * rbio->stripe_nsectors) + return; + + btrfs_bio_for_each_block_all(paddr, bio, step) { + u8 csum_buf[BTRFS_CSUM_SIZE]; + u8 *expected_csum; + + paddrs[(offset / step) % nr_steps] = paddr; + offset += step; + + /* Not yet covering the full fs block, continue to the next step. */ + if (!IS_ALIGNED(offset, fs_info->sectorsize)) + continue; + + /* No csum for this sector, skip to the next sector. */ + if (!test_bit(total_sector_nr, rbio->csum_bitmap)) { + total_sector_nr++; + continue; + } + + expected_csum = rbio->csum_buf + total_sector_nr * fs_info->csum_size; + btrfs_calculate_block_csum_pages(fs_info, paddrs, csum_buf); + if (unlikely(memcmp(csum_buf, expected_csum, fs_info->csum_size) != 0)) + set_bit(total_sector_nr, rbio->error_bitmap); + total_sector_nr++; + } +} + +static void raid_wait_read_end_io(struct bio *bio) +{ + struct btrfs_raid_bio *rbio = bio->bi_private; + + if (bio->bi_status) { + rbio_update_error_bitmap(rbio, bio); + } else { + set_bio_pages_uptodate(rbio, bio); + verify_bio_data_sectors(rbio, bio); + } + + bio_put(bio); + if (atomic_dec_and_test(&rbio->stripes_pending)) + wake_up(&rbio->io_wait); +} + +static void submit_read_wait_bio_list(struct btrfs_raid_bio *rbio, + struct bio_list *bio_list) +{ + struct bio *bio; + + atomic_set(&rbio->stripes_pending, bio_list_size(bio_list)); + while ((bio = bio_list_pop(bio_list))) { + bio->bi_end_io = raid_wait_read_end_io; + + if (trace_raid56_read_enabled()) { + struct raid56_bio_trace_info trace_info = { 0 }; + + bio_get_trace_info(rbio, bio, &trace_info); + trace_call__raid56_read(rbio, bio, &trace_info); + } + submit_bio(bio); + } + + wait_event(rbio->io_wait, atomic_read(&rbio->stripes_pending) == 0); +} + +static int alloc_rbio_data_pages(struct btrfs_raid_bio *rbio) +{ + const int data_pages = rbio->nr_data * rbio->stripe_npages; + int ret; + + ret = btrfs_alloc_page_array(data_pages, rbio->stripe_pages, GFP_NOFS); + if (ret < 0) + return ret; + + index_stripe_sectors(rbio); + return 0; +} + +/* + * We use plugging call backs to collect full stripes. + * Any time we get a partial stripe write while plugged + * we collect it into a list. When the unplug comes down, + * we sort the list by logical block number and merge + * everything we can into the same rbios + */ +struct btrfs_plug_cb { + struct blk_plug_cb cb; + struct btrfs_fs_info *info; + struct list_head rbio_list; +}; + +/* + * rbios on the plug list are sorted for easier merging. + */ +static int plug_cmp(void *priv, const struct list_head *a, + const struct list_head *b) +{ + const struct btrfs_raid_bio *ra = container_of(a, struct btrfs_raid_bio, + plug_list); + const struct btrfs_raid_bio *rb = container_of(b, struct btrfs_raid_bio, + plug_list); + u64 a_sector = ra->bio_list.head->bi_iter.bi_sector; + u64 b_sector = rb->bio_list.head->bi_iter.bi_sector; + + if (a_sector < b_sector) + return -1; + if (a_sector > b_sector) + return 1; + return 0; +} + +static void raid_unplug(struct blk_plug_cb *cb, bool from_schedule) +{ + struct btrfs_plug_cb *plug = container_of(cb, struct btrfs_plug_cb, cb); + struct btrfs_raid_bio *cur; + struct btrfs_raid_bio *last = NULL; + + list_sort(NULL, &plug->rbio_list, plug_cmp); + + while (!list_empty(&plug->rbio_list)) { + cur = list_first_entry(&plug->rbio_list, + struct btrfs_raid_bio, plug_list); + list_del_init(&cur->plug_list); + + if (rbio_is_full(cur)) { + /* We have a full stripe, queue it down. */ + start_async_work(cur, rmw_rbio_work); + continue; + } + if (last) { + if (rbio_can_merge(last, cur)) { + merge_rbio(last, cur); + free_raid_bio(cur); + continue; + } + start_async_work(last, rmw_rbio_work); + } + last = cur; + } + if (last) + start_async_work(last, rmw_rbio_work); + kfree(plug); +} + +/* Add the original bio into rbio->bio_list, and update rbio::dbitmap. */ +static void rbio_add_bio(struct btrfs_raid_bio *rbio, struct bio *orig_bio) +{ + const struct btrfs_fs_info *fs_info = rbio->bioc->fs_info; + const u64 orig_logical = orig_bio->bi_iter.bi_sector << SECTOR_SHIFT; + const u64 full_stripe_start = rbio->bioc->full_stripe_logical; + const u32 orig_len = orig_bio->bi_iter.bi_size; + const u32 sectorsize = fs_info->sectorsize; + u64 cur_logical; + + ASSERT_RBIO_LOGICAL(orig_logical >= full_stripe_start && + orig_logical + orig_len <= full_stripe_start + + rbio->nr_data * BTRFS_STRIPE_LEN, + rbio, orig_logical); + + bio_list_add(&rbio->bio_list, orig_bio); + rbio->bio_list_bytes += orig_bio->bi_iter.bi_size; + + /* Update the dbitmap. */ + for (cur_logical = orig_logical; cur_logical < orig_logical + orig_len; + cur_logical += sectorsize) { + int bit = ((u32)(cur_logical - full_stripe_start) >> + fs_info->sectorsize_bits) % rbio->stripe_nsectors; + + set_bit(bit, &rbio->dbitmap); + } +} + +/* + * our main entry point for writes from the rest of the FS. + */ +void raid56_parity_write(struct bio *bio, struct btrfs_io_context *bioc) +{ + struct btrfs_fs_info *fs_info = bioc->fs_info; + struct btrfs_raid_bio *rbio; + struct btrfs_plug_cb *plug = NULL; + struct blk_plug_cb *cb; + + rbio = alloc_rbio(fs_info, bioc); + if (IS_ERR(rbio)) { + bio->bi_status = errno_to_blk_status(PTR_ERR(rbio)); + bio_endio(bio); + return; + } + rbio->operation = BTRFS_RBIO_WRITE; + rbio_add_bio(rbio, bio); + + /* + * Don't plug on full rbios, just get them out the door + * as quickly as we can + */ + if (!rbio_is_full(rbio)) { + cb = blk_check_plugged(raid_unplug, fs_info, sizeof(*plug)); + if (cb) { + plug = container_of(cb, struct btrfs_plug_cb, cb); + if (!plug->info) { + plug->info = fs_info; + INIT_LIST_HEAD(&plug->rbio_list); + } + list_add_tail(&rbio->plug_list, &plug->rbio_list); + return; + } + } + + /* + * Either we don't have any existing plug, or we're doing a full stripe, + * queue the rmw work now. + */ + start_async_work(rbio, rmw_rbio_work); +} + +static int verify_one_sector(struct btrfs_raid_bio *rbio, + int stripe_nr, int sector_nr) +{ + struct btrfs_fs_info *fs_info = rbio->bioc->fs_info; + phys_addr_t *paddrs; + u8 csum_buf[BTRFS_CSUM_SIZE]; + u8 *csum_expected; + + if (!rbio->csum_bitmap || !rbio->csum_buf) + return 0; + + /* No way to verify P/Q as they are not covered by data csum. */ + if (stripe_nr >= rbio->nr_data) + return 0; + /* + * If we're rebuilding a read, we have to use pages from the + * bio list if possible. + */ + if (rbio->operation == BTRFS_RBIO_READ_REBUILD) { + paddrs = sector_paddrs_in_rbio(rbio, stripe_nr, sector_nr, 0); + } else { + paddrs = rbio_stripe_paddrs(rbio, stripe_nr, sector_nr); + } + + csum_expected = rbio->csum_buf + + (stripe_nr * rbio->stripe_nsectors + sector_nr) * + fs_info->csum_size; + btrfs_calculate_block_csum_pages(fs_info, paddrs, csum_buf); + if (unlikely(memcmp(csum_buf, csum_expected, fs_info->csum_size) != 0)) + return -EIO; + return 0; +} + +static void recover_vertical_step(struct btrfs_raid_bio *rbio, + unsigned int sector_nr, + unsigned int step_nr, + int faila, int failb, + void **pointers, void **unmap_array) +{ + struct btrfs_fs_info *fs_info = rbio->bioc->fs_info; + const u32 step = min(fs_info->sectorsize, PAGE_SIZE); + int stripe_nr; + + ASSERT(step_nr < rbio->sector_nsteps); + ASSERT(sector_nr < rbio->stripe_nsectors); + + /* + * Setup our array of pointers with sectors from each stripe + * + * NOTE: store a duplicate array of pointers to preserve the + * pointer order. + */ + for (stripe_nr = 0; stripe_nr < rbio->real_stripes; stripe_nr++) { + phys_addr_t paddr; + + /* + * If we're rebuilding a read, we have to use pages from the + * bio list if possible. + */ + if (rbio->operation == BTRFS_RBIO_READ_REBUILD) { + paddr = sector_paddr_in_rbio(rbio, stripe_nr, sector_nr, step_nr, 0); + } else { + paddr = rbio_stripe_paddr(rbio, stripe_nr, sector_nr, step_nr); + } + pointers[stripe_nr] = kmap_local_paddr(paddr); + unmap_array[stripe_nr] = pointers[stripe_nr]; + } + + /* All raid6 handling here */ + if (rbio->bioc->map_type & BTRFS_BLOCK_GROUP_RAID6) { + /* Single failure, rebuild from parity raid5 style */ + if (failb < 0) { + if (faila == rbio->nr_data) + /* + * Just the P stripe has failed, without + * a bad data or Q stripe. + * We have nothing to do, just skip the + * recovery for this stripe. + */ + goto cleanup; + /* + * a single failure in raid6 is rebuilt + * in the pstripe code below + */ + goto pstripe; + } + + /* + * If the q stripe is failed, do a pstripe reconstruction from + * the xors. + * If both the q stripe and the P stripe are failed, we're + * here due to a crc mismatch and we can't give them the + * data they want. + */ + if (failb == rbio->real_stripes - 1) { + if (faila == rbio->real_stripes - 2) + /* + * Only P and Q are corrupted. + * We only care about data stripes recovery, + * can skip this vertical stripe. + */ + goto cleanup; + /* + * Otherwise we have one bad data stripe and + * a good P stripe. raid5! + */ + goto pstripe; + } + + if (failb == rbio->real_stripes - 2) { + raid6_recov_datap(rbio->real_stripes, step, + faila, pointers); + } else { + raid6_recov_2data(rbio->real_stripes, step, + faila, failb, pointers); + } + } else { + void *p; + + /* Rebuild from P stripe here (raid5 or raid6). */ + ASSERT(failb == -1); +pstripe: + /* Copy parity block into failed block to start with */ + memcpy(pointers[faila], pointers[rbio->nr_data], step); + + /* Rearrange the pointer array */ + p = pointers[faila]; + for (stripe_nr = faila; stripe_nr < rbio->nr_data - 1; + stripe_nr++) + pointers[stripe_nr] = pointers[stripe_nr + 1]; + pointers[rbio->nr_data - 1] = p; + + /* Xor in the rest */ + xor_gen(p, pointers, rbio->nr_data - 1, step); + } + +cleanup: + for (stripe_nr = rbio->real_stripes - 1; stripe_nr >= 0; stripe_nr--) + kunmap_local(unmap_array[stripe_nr]); +} + +/* + * Recover a vertical stripe specified by @sector_nr. + * @*pointers are the pre-allocated pointers by the caller, so we don't + * need to allocate/free the pointers again and again. + */ +static int recover_vertical(struct btrfs_raid_bio *rbio, int sector_nr, + void **pointers, void **unmap_array) +{ + int found_errors; + int faila; + int failb; + int ret = 0; + + /* + * Now we just use bitmap to mark the horizontal stripes in + * which we have data when doing parity scrub. + */ + if (rbio->operation == BTRFS_RBIO_PARITY_SCRUB && + !test_bit(sector_nr, &rbio->dbitmap)) + return 0; + + found_errors = get_rbio_vertical_errors(rbio, sector_nr, &faila, + &failb); + /* + * No errors in the vertical stripe, skip it. Can happen for recovery + * which only part of a stripe failed csum check. + */ + if (!found_errors) + return 0; + + if (unlikely(found_errors > rbio->bioc->max_errors)) + return -EIO; + + for (int i = 0; i < rbio->sector_nsteps; i++) + recover_vertical_step(rbio, sector_nr, i, faila, failb, + pointers, unmap_array); + if (faila >= 0) { + ret = verify_one_sector(rbio, faila, sector_nr); + if (ret < 0) + return ret; + + set_bit(rbio_sector_index(rbio, faila, sector_nr), + rbio->stripe_uptodate_bitmap); + } + if (failb >= 0) { + ret = verify_one_sector(rbio, failb, sector_nr); + if (ret < 0) + return ret; + + set_bit(rbio_sector_index(rbio, failb, sector_nr), + rbio->stripe_uptodate_bitmap); + } + return ret; +} + +static int recover_sectors(struct btrfs_raid_bio *rbio) +{ + void **pointers = NULL; + void **unmap_array = NULL; + int sectornr; + int ret = 0; + + /* + * @pointers array stores the pointer for each sector. + * + * @unmap_array stores copy of pointers that does not get reordered + * during reconstruction so that kunmap_local works. + */ + pointers = kzalloc_objs(void *, rbio->real_stripes, GFP_NOFS); + unmap_array = kzalloc_objs(void *, rbio->real_stripes, GFP_NOFS); + if (!pointers || !unmap_array) { + ret = -ENOMEM; + goto out; + } + + if (rbio->operation == BTRFS_RBIO_READ_REBUILD) { + spin_lock(&rbio->bio_list_lock); + set_bit(RBIO_RMW_LOCKED_BIT, &rbio->flags); + spin_unlock(&rbio->bio_list_lock); + } + + index_rbio_pages(rbio); + + for (sectornr = 0; sectornr < rbio->stripe_nsectors; sectornr++) { + ret = recover_vertical(rbio, sectornr, pointers, unmap_array); + if (ret < 0) + break; + } + +out: + kfree(pointers); + kfree(unmap_array); + return ret; +} + +static void recover_rbio(struct btrfs_raid_bio *rbio) +{ + struct bio_list bio_list = BIO_EMPTY_LIST; + int total_sector_nr; + int ret = 0; + + /* + * Either we're doing recover for a read failure or degraded write, + * caller should have set error bitmap correctly. + */ + ASSERT(bitmap_weight(rbio->error_bitmap, rbio->nr_sectors)); + + /* For recovery, we need to read all sectors including P/Q. */ + ret = alloc_rbio_pages(rbio); + if (ret < 0) + goto out; + + index_rbio_pages(rbio); + + /* + * Read everything that hasn't failed. However this time we will + * not trust any cached sector. + * As we may read out some stale data but higher layer is not reading + * that stale part. + * + * So here we always re-read everything in recovery path. + */ + for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors; + total_sector_nr++) { + int stripe = total_sector_nr / rbio->stripe_nsectors; + int sectornr = total_sector_nr % rbio->stripe_nsectors; + phys_addr_t *paddrs; + + /* + * Skip the range which has error. It can be a range which is + * marked error (for csum mismatch), or it can be a missing + * device. + */ + if (!rbio->bioc->stripes[stripe].dev->bdev || + test_bit(total_sector_nr, rbio->error_bitmap)) { + /* + * Also set the error bit for missing device, which + * may not yet have its error bit set. + */ + set_bit(total_sector_nr, rbio->error_bitmap); + continue; + } + + paddrs = rbio_stripe_paddrs(rbio, stripe, sectornr); + ret = rbio_add_io_paddrs(rbio, &bio_list, paddrs, stripe, + sectornr, REQ_OP_READ); + if (ret < 0) { + bio_list_put(&bio_list); + goto out; + } + } + + submit_read_wait_bio_list(rbio, &bio_list); + ret = recover_sectors(rbio); +out: + rbio_orig_end_io(rbio, errno_to_blk_status(ret)); +} + +static void recover_rbio_work(struct work_struct *work) +{ + struct btrfs_raid_bio *rbio; + + rbio = container_of(work, struct btrfs_raid_bio, work); + if (!lock_stripe_add(rbio)) + recover_rbio(rbio); +} + +static void recover_rbio_work_locked(struct work_struct *work) +{ + recover_rbio(container_of(work, struct btrfs_raid_bio, work)); +} + +static void set_rbio_raid6_extra_error(struct btrfs_raid_bio *rbio, int mirror_num) +{ + bool found = false; + int sector_nr; + + /* + * This is for RAID6 extra recovery tries, thus mirror number should + * be large than 2. + * Mirror 1 means read from data stripes. Mirror 2 means rebuild using + * RAID5 methods. + */ + ASSERT(mirror_num > 2); + for (sector_nr = 0; sector_nr < rbio->stripe_nsectors; sector_nr++) { + int found_errors; + int faila; + int failb; + + found_errors = get_rbio_vertical_errors(rbio, sector_nr, + &faila, &failb); + /* This vertical stripe doesn't have errors. */ + if (!found_errors) + continue; + + /* + * If we found errors, there should be only one error marked + * by previous set_rbio_range_error(). + */ + ASSERT(found_errors == 1); + found = true; + + /* Now select another stripe to mark as error. */ + failb = rbio->real_stripes - (mirror_num - 1); + if (failb <= faila) + failb--; + + /* Set the extra bit in error bitmap. */ + if (failb >= 0) + set_bit(failb * rbio->stripe_nsectors + sector_nr, + rbio->error_bitmap); + } + + /* We should found at least one vertical stripe with error.*/ + ASSERT(found); +} + +/* + * the main entry point for reads from the higher layers. This + * is really only called when the normal read path had a failure, + * so we assume the bio they send down corresponds to a failed part + * of the drive. + */ +void raid56_parity_recover(struct bio *bio, struct btrfs_io_context *bioc, + int mirror_num) +{ + struct btrfs_fs_info *fs_info = bioc->fs_info; + struct btrfs_raid_bio *rbio; + + rbio = alloc_rbio(fs_info, bioc); + if (IS_ERR(rbio)) { + bio->bi_status = errno_to_blk_status(PTR_ERR(rbio)); + bio_endio(bio); + return; + } + + rbio->operation = BTRFS_RBIO_READ_REBUILD; + rbio_add_bio(rbio, bio); + + set_rbio_range_error(rbio, bio); + + /* + * Loop retry: + * for 'mirror == 2', reconstruct from all other stripes. + * for 'mirror_num > 2', select a stripe to fail on every retry. + */ + if (mirror_num > 2) + set_rbio_raid6_extra_error(rbio, mirror_num); + + start_async_work(rbio, recover_rbio_work); +} + +static void fill_data_csums(struct btrfs_raid_bio *rbio) +{ + struct btrfs_fs_info *fs_info = rbio->bioc->fs_info; + struct btrfs_root *csum_root; + const u64 start = rbio->bioc->full_stripe_logical; + const u32 len = (rbio->nr_data * rbio->stripe_nsectors) << + fs_info->sectorsize_bits; + int ret; + + /* The rbio should not have its csum buffer initialized. */ + ASSERT(!rbio->csum_buf && !rbio->csum_bitmap); + + /* + * Skip the csum search if: + * + * - The rbio doesn't belong to data block groups + * Then we are doing IO for tree blocks, no need to search csums. + * + * - The rbio belongs to mixed block groups + * This is to avoid deadlock, as we're already holding the full + * stripe lock, if we trigger a metadata read, and it needs to do + * raid56 recovery, we will deadlock. + */ + if (!(rbio->bioc->map_type & BTRFS_BLOCK_GROUP_DATA) || + rbio->bioc->map_type & BTRFS_BLOCK_GROUP_METADATA) + return; + + rbio->csum_buf = kzalloc(rbio->nr_data * rbio->stripe_nsectors * + fs_info->csum_size, GFP_NOFS); + rbio->csum_bitmap = bitmap_zalloc(rbio->nr_data * rbio->stripe_nsectors, + GFP_NOFS); + if (!rbio->csum_buf || !rbio->csum_bitmap) { + ret = -ENOMEM; + goto error; + } + + csum_root = btrfs_csum_root(fs_info, rbio->bioc->full_stripe_logical); + if (unlikely(!csum_root)) { + btrfs_err(fs_info, + "missing csum root for extent at bytenr %llu", + rbio->bioc->full_stripe_logical); + ret = -EUCLEAN; + goto error; + } + + ret = btrfs_lookup_csums_bitmap(csum_root, NULL, start, start + len - 1, + rbio->csum_buf, rbio->csum_bitmap); + if (ret < 0) + goto error; + if (bitmap_empty(rbio->csum_bitmap, len >> fs_info->sectorsize_bits)) + goto no_csum; + return; + +error: + /* + * We failed to allocate memory or grab the csum, but it's not fatal, + * we can still continue. But better to warn users that RMW is no + * longer safe for this particular sub-stripe write. + */ + btrfs_warn_rl(fs_info, +"sub-stripe write for full stripe %llu is not safe, failed to get csum: %d", + rbio->bioc->full_stripe_logical, ret); +no_csum: + kfree(rbio->csum_buf); + bitmap_free(rbio->csum_bitmap); + rbio->csum_buf = NULL; + rbio->csum_bitmap = NULL; +} + +static int rmw_read_wait_recover(struct btrfs_raid_bio *rbio) +{ + struct bio_list bio_list = BIO_EMPTY_LIST; + int total_sector_nr; + int ret = 0; + + /* + * Fill the data csums we need for data verification. We need to fill + * the csum_bitmap/csum_buf first, as our endio function will try to + * verify the data sectors. + */ + fill_data_csums(rbio); + + /* + * Build a list of bios to read all sectors (including data and P/Q). + * + * This behavior is to compensate the later csum verification and recovery. + */ + for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors; + total_sector_nr++) { + int stripe = total_sector_nr / rbio->stripe_nsectors; + int sectornr = total_sector_nr % rbio->stripe_nsectors; + phys_addr_t *paddrs; + + paddrs = rbio_stripe_paddrs(rbio, stripe, sectornr); + ret = rbio_add_io_paddrs(rbio, &bio_list, paddrs, stripe, + sectornr, REQ_OP_READ); + if (ret) { + bio_list_put(&bio_list); + return ret; + } + } + + /* + * We may or may not have any corrupted sectors (including missing dev + * and csum mismatch), just let recover_sectors() to handle them all. + */ + submit_read_wait_bio_list(rbio, &bio_list); + return recover_sectors(rbio); +} + +static void raid_wait_write_end_io(struct bio *bio) +{ + struct btrfs_raid_bio *rbio = bio->bi_private; + + if (bio->bi_status) + rbio_update_error_bitmap(rbio, bio); + bio_put(bio); + if (atomic_dec_and_test(&rbio->stripes_pending)) + wake_up(&rbio->io_wait); +} + +static void submit_write_bios(struct btrfs_raid_bio *rbio, + struct bio_list *bio_list) +{ + struct bio *bio; + + atomic_set(&rbio->stripes_pending, bio_list_size(bio_list)); + while ((bio = bio_list_pop(bio_list))) { + bio->bi_end_io = raid_wait_write_end_io; + + if (trace_raid56_write_enabled()) { + struct raid56_bio_trace_info trace_info = { 0 }; + + bio_get_trace_info(rbio, bio, &trace_info); + trace_call__raid56_write(rbio, bio, &trace_info); + } + submit_bio(bio); + } +} + +/* + * To determine if we need to read any sector from the disk. + * Should only be utilized in RMW path, to skip cached rbio. + */ +static bool need_read_stripe_sectors(struct btrfs_raid_bio *rbio) +{ + int i; + + for (i = 0; i < rbio->nr_data * rbio->stripe_nsectors; i++) { + phys_addr_t paddr = rbio->stripe_paddrs[i * rbio->sector_nsteps]; + + /* + * We have a sector which doesn't have page nor uptodate, + * thus this rbio can not be cached one, as cached one must + * have all its data sectors present and uptodate. + */ + if (paddr == INVALID_PADDR || + !test_bit(i, rbio->stripe_uptodate_bitmap)) + return true; + } + return false; +} + +static void rmw_rbio(struct btrfs_raid_bio *rbio) +{ + struct bio_list bio_list; + int sectornr; + int ret = 0; + + /* + * Allocate the pages for parity first, as P/Q pages will always be + * needed for both full-stripe and sub-stripe writes. + */ + ret = alloc_rbio_parity_pages(rbio); + if (ret < 0) + goto out; + + /* + * Either full stripe write, or we have every data sector already + * cached, can go to write path immediately. + */ + if (!rbio_is_full(rbio) && need_read_stripe_sectors(rbio)) { + /* + * Now we're doing sub-stripe write, also need all data stripes + * to do the full RMW. + */ + ret = alloc_rbio_data_pages(rbio); + if (ret < 0) + goto out; + + index_rbio_pages(rbio); + + ret = rmw_read_wait_recover(rbio); + if (ret < 0) + goto out; + } + + /* + * At this stage we're not allowed to add any new bios to the + * bio list any more, anyone else that wants to change this stripe + * needs to do their own rmw. + */ + spin_lock(&rbio->bio_list_lock); + set_bit(RBIO_RMW_LOCKED_BIT, &rbio->flags); + spin_unlock(&rbio->bio_list_lock); + + bitmap_clear(rbio->error_bitmap, 0, rbio->nr_sectors); + + index_rbio_pages(rbio); + + /* + * We don't cache full rbios because we're assuming + * the higher layers are unlikely to use this area of + * the disk again soon. If they do use it again, + * hopefully they will send another full bio. + */ + if (!rbio_is_full(rbio)) + cache_rbio_pages(rbio); + else + clear_bit(RBIO_CACHE_READY_BIT, &rbio->flags); + + for (sectornr = 0; sectornr < rbio->stripe_nsectors; sectornr++) + generate_pq_vertical(rbio, sectornr); + + bio_list_init(&bio_list); + ret = rmw_assemble_write_bios(rbio, &bio_list); + if (ret < 0) + goto out; + + /* We should have at least one bio assembled. */ + ASSERT(bio_list_size(&bio_list)); + submit_write_bios(rbio, &bio_list); + wait_event(rbio->io_wait, atomic_read(&rbio->stripes_pending) == 0); + + /* We may have more errors than our tolerance during the read. */ + for (sectornr = 0; sectornr < rbio->stripe_nsectors; sectornr++) { + int found_errors; + + found_errors = get_rbio_vertical_errors(rbio, sectornr, NULL, NULL); + if (unlikely(found_errors > rbio->bioc->max_errors)) { + ret = -EIO; + break; + } + } +out: + rbio_orig_end_io(rbio, errno_to_blk_status(ret)); +} + +static void rmw_rbio_work(struct work_struct *work) +{ + struct btrfs_raid_bio *rbio; + + rbio = container_of(work, struct btrfs_raid_bio, work); + if (lock_stripe_add(rbio) == 0) + rmw_rbio(rbio); +} + +static void rmw_rbio_work_locked(struct work_struct *work) +{ + rmw_rbio(container_of(work, struct btrfs_raid_bio, work)); +} + +/* + * The following code is used to scrub/replace the parity stripe + * + * Caller must have already increased bio_counter for getting @bioc. + * + * Note: We need make sure all the pages that add into the scrub/replace + * raid bio are correct and not be changed during the scrub/replace. That + * is those pages just hold metadata or file data with checksum. + */ + +struct btrfs_raid_bio *raid56_parity_alloc_scrub_rbio(struct bio *bio, + struct btrfs_io_context *bioc, + struct btrfs_device *scrub_dev, + unsigned long *dbitmap, int stripe_nsectors) +{ + struct btrfs_fs_info *fs_info = bioc->fs_info; + struct btrfs_raid_bio *rbio; + int i; + + rbio = alloc_rbio(fs_info, bioc); + if (IS_ERR(rbio)) + return NULL; + bio_list_add(&rbio->bio_list, bio); + /* + * This is a special bio which is used to hold the completion handler + * and make the scrub rbio is similar to the other types + */ + ASSERT(!bio->bi_iter.bi_size); + rbio->operation = BTRFS_RBIO_PARITY_SCRUB; + + /* + * After mapping bioc with BTRFS_MAP_WRITE, parities have been sorted + * to the end position, so this search can start from the first parity + * stripe. + */ + for (i = rbio->nr_data; i < rbio->real_stripes; i++) { + if (bioc->stripes[i].dev == scrub_dev) { + rbio->scrubp = i; + break; + } + } + ASSERT_RBIO_STRIPE(i < rbio->real_stripes, rbio, i); + + bitmap_copy(&rbio->dbitmap, dbitmap, stripe_nsectors); + return rbio; +} + +static int alloc_rbio_sector_pages(struct btrfs_raid_bio *rbio, + int sector_nr) +{ + const u32 step = min(PAGE_SIZE, rbio->bioc->fs_info->sectorsize); + const u32 base = sector_nr * rbio->sector_nsteps; + + for (int i = base; i < base + rbio->sector_nsteps; i++) { + const unsigned int page_index = (i * step) >> PAGE_SHIFT; + struct page *page; + + if (rbio->stripe_pages[page_index]) + continue; + page = alloc_page(GFP_NOFS); + if (!page) + return -ENOMEM; + rbio->stripe_pages[page_index] = page; + } + return 0; +} + +/* + * We just scrub the parity that we have correct data on the same horizontal, + * so we needn't allocate all pages for all the stripes. + */ +static int alloc_rbio_essential_pages(struct btrfs_raid_bio *rbio) +{ + int total_sector_nr; + + for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors; + total_sector_nr++) { + int sectornr = total_sector_nr % rbio->stripe_nsectors; + int ret; + + if (!test_bit(sectornr, &rbio->dbitmap)) + continue; + ret = alloc_rbio_sector_pages(rbio, total_sector_nr); + if (ret < 0) + return ret; + } + index_stripe_sectors(rbio); + return 0; +} + +/* Return true if the content of the step matches the caclulated one. */ +static bool verify_one_parity_step(struct btrfs_raid_bio *rbio, + void *pointers[], unsigned int sector_nr, + unsigned int step_nr) +{ + const unsigned int nr_data = rbio->nr_data; + const bool has_qstripe = (rbio->real_stripes - rbio->nr_data == 2); + const u32 step = min(rbio->bioc->fs_info->sectorsize, PAGE_SIZE); + void *parity; + bool ret = false; + + ASSERT(step_nr < rbio->sector_nsteps); + + /* First collect one page from each data stripe. */ + for (int stripe = 0; stripe < nr_data; stripe++) + pointers[stripe] = kmap_local_paddr( + sector_paddr_in_rbio(rbio, stripe, sector_nr, + step_nr, 0)); + + if (has_qstripe) { + assert_rbio(rbio); + /* RAID6, call the library function to fill in our P/Q. */ + raid6_gen_syndrome(rbio->real_stripes, step, pointers); + } else { + /* RAID5. */ + memcpy(pointers[nr_data], pointers[0], step); + xor_gen(pointers[nr_data], pointers + 1, nr_data - 1, step); + } + + /* Check scrubbing parity and repair it. */ + parity = kmap_local_paddr(rbio_stripe_paddr(rbio, rbio->scrubp, sector_nr, step_nr)); + if (memcmp(parity, pointers[rbio->scrubp], step) != 0) + memcpy(parity, pointers[rbio->scrubp], step); + else + ret = true; + kunmap_local(parity); + + for (int stripe = nr_data - 1; stripe >= 0; stripe--) + kunmap_local(pointers[stripe]); + return ret; +} + +/* + * The @pointers array should have the P/Q parity already mapped. + */ +static void verify_one_parity_sector(struct btrfs_raid_bio *rbio, + void *pointers[], unsigned int sector_nr) +{ + bool found_error = false; + + for (int step_nr = 0; step_nr < rbio->sector_nsteps; step_nr++) { + bool match; + + match = verify_one_parity_step(rbio, pointers, sector_nr, step_nr); + if (!match) + found_error = true; + } + if (!found_error) + bitmap_clear(&rbio->dbitmap, sector_nr, 1); +} + +static int finish_parity_scrub(struct btrfs_raid_bio *rbio) +{ + struct btrfs_io_context *bioc = rbio->bioc; + void **pointers = rbio->finish_pointers; + unsigned long *pbitmap = &rbio->finish_pbitmap; + int nr_data = rbio->nr_data; + int sectornr; + bool has_qstripe; + struct page *page; + phys_addr_t p_paddr = INVALID_PADDR; + phys_addr_t q_paddr = INVALID_PADDR; + struct bio_list bio_list; + bool is_replace = false; + int ret; + + bio_list_init(&bio_list); + + if (rbio->real_stripes - rbio->nr_data == 1) + has_qstripe = false; + else if (rbio->real_stripes - rbio->nr_data == 2) + has_qstripe = true; + else + BUG(); + + /* + * Replace is running and our P/Q stripe is being replaced, then we + * need to duplicate the final write to replace target. + */ + if (bioc->replace_nr_stripes && bioc->replace_stripe_src == rbio->scrubp) { + is_replace = true; + bitmap_copy(pbitmap, &rbio->dbitmap, rbio->stripe_nsectors); + } + + /* + * Because the higher layers(scrubber) are unlikely to + * use this area of the disk again soon, so don't cache + * it. + */ + clear_bit(RBIO_CACHE_READY_BIT, &rbio->flags); + + page = alloc_page(GFP_NOFS); + if (!page) + return -ENOMEM; + p_paddr = page_to_phys(page); + page = NULL; + pointers[nr_data] = kmap_local_paddr(p_paddr); + + if (has_qstripe) { + /* RAID6, allocate and map temp space for the Q stripe */ + page = alloc_page(GFP_NOFS); + if (!page) { + __free_page(phys_to_page(p_paddr)); + p_paddr = INVALID_PADDR; + return -ENOMEM; + } + q_paddr = page_to_phys(page); + page = NULL; + pointers[rbio->real_stripes - 1] = kmap_local_paddr(q_paddr); + } + + bitmap_clear(rbio->error_bitmap, 0, rbio->nr_sectors); + + /* Map the parity stripe just once */ + + for_each_set_bit(sectornr, &rbio->dbitmap, rbio->stripe_nsectors) + verify_one_parity_sector(rbio, pointers, sectornr); + + kunmap_local(pointers[nr_data]); + __free_page(phys_to_page(p_paddr)); + p_paddr = INVALID_PADDR; + if (q_paddr != INVALID_PADDR) { + __free_page(phys_to_page(q_paddr)); + q_paddr = INVALID_PADDR; + } + + /* + * time to start writing. Make bios for everything from the + * higher layers (the bio_list in our rbio) and our p/q. Ignore + * everything else. + */ + for_each_set_bit(sectornr, &rbio->dbitmap, rbio->stripe_nsectors) { + phys_addr_t *paddrs; + + paddrs = rbio_stripe_paddrs(rbio, rbio->scrubp, sectornr); + ret = rbio_add_io_paddrs(rbio, &bio_list, paddrs, rbio->scrubp, + sectornr, REQ_OP_WRITE); + if (ret) + goto cleanup; + } + + if (!is_replace) + goto submit_write; + + /* + * Replace is running and our parity stripe needs to be duplicated to + * the target device. Check we have a valid source stripe number. + */ + ASSERT_RBIO(rbio->bioc->replace_stripe_src >= 0, rbio); + for_each_set_bit(sectornr, pbitmap, rbio->stripe_nsectors) { + phys_addr_t *paddrs; + + paddrs = rbio_stripe_paddrs(rbio, rbio->scrubp, sectornr); + ret = rbio_add_io_paddrs(rbio, &bio_list, paddrs, rbio->real_stripes, + sectornr, REQ_OP_WRITE); + if (ret) + goto cleanup; + } + +submit_write: + submit_write_bios(rbio, &bio_list); + return 0; + +cleanup: + bio_list_put(&bio_list); + return ret; +} + +static inline int is_data_stripe(struct btrfs_raid_bio *rbio, int stripe) +{ + if (stripe >= 0 && stripe < rbio->nr_data) + return 1; + return 0; +} + +static int recover_scrub_rbio(struct btrfs_raid_bio *rbio) +{ + void **pointers = NULL; + void **unmap_array = NULL; + int sector_nr; + int ret = 0; + + /* + * @pointers array stores the pointer for each sector. + * + * @unmap_array stores copy of pointers that does not get reordered + * during reconstruction so that kunmap_local works. + */ + pointers = kzalloc_objs(void *, rbio->real_stripes, GFP_NOFS); + unmap_array = kzalloc_objs(void *, rbio->real_stripes, GFP_NOFS); + if (!pointers || !unmap_array) { + ret = -ENOMEM; + goto out; + } + + for (sector_nr = 0; sector_nr < rbio->stripe_nsectors; sector_nr++) { + int dfail = 0, failp = -1; + int faila; + int failb; + int found_errors; + + found_errors = get_rbio_vertical_errors(rbio, sector_nr, + &faila, &failb); + if (unlikely(found_errors > rbio->bioc->max_errors)) { + ret = -EIO; + goto out; + } + if (found_errors == 0) + continue; + + /* We should have at least one error here. */ + ASSERT(faila >= 0 || failb >= 0); + + if (is_data_stripe(rbio, faila)) + dfail++; + else if (is_parity_stripe(faila)) + failp = faila; + + if (is_data_stripe(rbio, failb)) + dfail++; + else if (is_parity_stripe(failb)) + failp = failb; + /* + * Because we can not use a scrubbing parity to repair the + * data, so the capability of the repair is declined. (In the + * case of RAID5, we can not repair anything.) + */ + if (unlikely(dfail > rbio->bioc->max_errors - 1)) { + ret = -EIO; + goto out; + } + /* + * If all data is good, only parity is correctly, just repair + * the parity, no need to recover data stripes. + */ + if (dfail == 0) + continue; + + /* + * Here means we got one corrupted data stripe and one + * corrupted parity on RAID6, if the corrupted parity is + * scrubbing parity, luckily, use the other one to repair the + * data, or we can not repair the data stripe. + */ + if (unlikely(failp != rbio->scrubp)) { + ret = -EIO; + goto out; + } + + ret = recover_vertical(rbio, sector_nr, pointers, unmap_array); + if (ret < 0) + goto out; + } +out: + kfree(pointers); + kfree(unmap_array); + return ret; +} + +static int scrub_assemble_read_bios(struct btrfs_raid_bio *rbio) +{ + struct bio_list bio_list = BIO_EMPTY_LIST; + int total_sector_nr; + int ret = 0; + + /* Build a list of bios to read all the missing parts. */ + for (total_sector_nr = 0; total_sector_nr < rbio->nr_sectors; + total_sector_nr++) { + int sectornr = total_sector_nr % rbio->stripe_nsectors; + int stripe = total_sector_nr / rbio->stripe_nsectors; + phys_addr_t *paddrs; + + /* No data in the vertical stripe, no need to read. */ + if (!test_bit(sectornr, &rbio->dbitmap)) + continue; + + /* + * A parity-scrub rbio carries no data in its bio list: the + * only bio there is the empty completion bio added by + * raid56_parity_alloc_scrub_rbio(). Every sector is read + * from the stripe, so only assert that invariant here. + */ + ASSERT(!sector_paddrs_in_rbio(rbio, stripe, sectornr, 1)); + + paddrs = rbio_stripe_paddrs(rbio, stripe, sectornr); + /* + * The bio cache may have handed us an uptodate sector. If so, + * use it. + */ + if (test_bit(rbio_sector_index(rbio, stripe, sectornr), + rbio->stripe_uptodate_bitmap)) + continue; + + ret = rbio_add_io_paddrs(rbio, &bio_list, paddrs, stripe, + sectornr, REQ_OP_READ); + if (ret) { + bio_list_put(&bio_list); + return ret; + } + } + + submit_read_wait_bio_list(rbio, &bio_list); + return 0; +} + +static void scrub_rbio(struct btrfs_raid_bio *rbio) +{ + int sector_nr; + int ret; + + ret = alloc_rbio_essential_pages(rbio); + if (ret) + goto out; + + bitmap_clear(rbio->error_bitmap, 0, rbio->nr_sectors); + + ret = scrub_assemble_read_bios(rbio); + if (ret < 0) + goto out; + + /* We may have some failures, recover the failed sectors first. */ + ret = recover_scrub_rbio(rbio); + if (ret < 0) + goto out; + + /* + * We have every sector properly prepared. Can finish the scrub + * and writeback the good content. + */ + ret = finish_parity_scrub(rbio); + wait_event(rbio->io_wait, atomic_read(&rbio->stripes_pending) == 0); + for (sector_nr = 0; sector_nr < rbio->stripe_nsectors; sector_nr++) { + int found_errors; + + found_errors = get_rbio_vertical_errors(rbio, sector_nr, NULL, NULL); + if (unlikely(found_errors > rbio->bioc->max_errors)) { + ret = -EIO; + break; + } + } +out: + rbio_orig_end_io(rbio, errno_to_blk_status(ret)); +} + +static void scrub_rbio_work_locked(struct work_struct *work) +{ + scrub_rbio(container_of(work, struct btrfs_raid_bio, work)); +} + +void raid56_parity_submit_scrub_rbio(struct btrfs_raid_bio *rbio) +{ + if (!lock_stripe_add(rbio)) + start_async_work(rbio, scrub_rbio_work_locked); +} + +/* + * This is for scrub call sites where we already have correct data contents. + * This allows us to avoid reading data stripes again. + * + * Unfortunately here we have to do folio copy, other than reusing the pages. + * This is due to the fact rbio has its own page management for its cache. + */ +void raid56_parity_cache_data_folios(struct btrfs_raid_bio *rbio, + void *vaddr, u64 data_logical) +{ + struct btrfs_fs_info *fs_info = rbio->bioc->fs_info; + const u64 offset_in_full_stripe = data_logical - + rbio->bioc->full_stripe_logical; + int ret; + + /* + * If we hit ENOMEM temporarily, but later at + * raid56_parity_submit_scrub_rbio() time it succeeded, we just do + * the extra read, not a big deal. + * + * If we hit ENOMEM later at raid56_parity_submit_scrub_rbio() time, + * the bio would got proper error number set. + */ + ret = alloc_rbio_data_pages(rbio); + if (ret < 0) + return; + + /* data_logical must be at stripe boundary and inside the full stripe. */ + ASSERT(IS_ALIGNED(offset_in_full_stripe, BTRFS_STRIPE_LEN)); + ASSERT(offset_in_full_stripe < (rbio->nr_data << BTRFS_STRIPE_LEN_SHIFT)); + + for (unsigned int cur_off = offset_in_full_stripe; + cur_off < offset_in_full_stripe + BTRFS_STRIPE_LEN; + cur_off += PAGE_SIZE) { + const unsigned int pindex = cur_off >> PAGE_SHIFT; + + ASSERT(cur_off - offset_in_full_stripe + PAGE_SIZE <= BTRFS_STRIPE_LEN); + memcpy_to_page(rbio->stripe_pages[pindex], 0, + vaddr + cur_off - offset_in_full_stripe, PAGE_SIZE); + } + bitmap_set(rbio->stripe_uptodate_bitmap, + offset_in_full_stripe >> fs_info->sectorsize_bits, + BTRFS_STRIPE_LEN >> fs_info->sectorsize_bits); +} diff --git a/fs/btrfs/raid56.h b/fs/btrfs/raid56.h new file mode 100644 index 000000000..854264819 --- /dev/null +++ b/fs/btrfs/raid56.h @@ -0,0 +1,291 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2012 Fusion-io All rights reserved. + * Copyright (C) 2012 Intel Corp. All rights reserved. + */ + +#ifndef BTRFS_RAID56_H +#define BTRFS_RAID56_H + +#include +#include +#include +#include +#include +#include +#include "volumes.h" + +struct page; +struct btrfs_fs_info; + +enum btrfs_rbio_ops { + BTRFS_RBIO_WRITE, + BTRFS_RBIO_READ_REBUILD, + BTRFS_RBIO_PARITY_SCRUB, +}; + +/* + * Overview of btrfs_raid_bio. + * + * One btrfs_raid_bio represents a full stripe of RAID56, including both data + * and P/Q stripes. For now, each data and P/Q stripe is of a fixed length (64K). + * + * One btrfs_raid_bio can have one or more bios from higher layer, covering + * part or all of the data stripes. + * + * [PAGES FROM HIGHER LAYER BIOS] + * Higher layer bios are in the btrfs_raid_bio::bio_list. + * + * Pages from the bio_list are represented like the following: + * + * bio_list: |<- Bio 1 ->| |<- Bio 2 ->| ... + * bio_paddrs: [0] [1] [2] [3] [4] [5] ... + * + * If there is a bio covering a sector (one btrfs fs block), the corresponding + * pointer in btrfs_raid_bio::bio_paddrs[] will point to the physical address + * (with the offset inside the page) of the corresponding bio. + * + * If there is no bio covering a sector, then btrfs_raid_bio::bio_paddrs[i] will + * be INVALID_PADDR. + * + * The length of each entry in bio_paddrs[] is a step (aka, min(sectorsize, PAGE_SIZE)). + * + * [PAGES FOR INTERNAL USAGES] + * Pages not covered by any bio or belonging to P/Q stripes are stored in + * btrfs_raid_bio::stripe_pages[] and stripe_paddrs[], like the following: + * + * stripe_pages: |<- Page 0 ->|<- Page 1 ->| ... + * stripe_paddrs: [0] [1] [2] [3] [4] ... + * + * stripe_pages[] array stores all the pages covering the full stripe, including + * data and P/Q pages. + * stripe_pages[0] is the first page of the first data stripe. + * stripe_pages[BTRFS_STRIPE_LEN / PAGE_SIZE] is the first page of the second + * data stripe. + * + * Some pointers inside stripe_pages[] can be NULL, e.g. for a full stripe write + * (the bio covers all data stripes) there is no need to allocate pages for + * data stripes (can grab from bio_paddrs[]). + * + * If the corresponding page of stripe_paddrs[i] is not allocated, the value of + * stripe_paddrs[i] will be INVALID_PADDR. + * + * The length of each entry in stripe_paddrs[] is a step. + * + * [LOCATING A SECTOR] + * To locate a sector for IO, we need the following info: + * + * - stripe_nr + * Starts from 0 (representing the first data stripe), ends at + * @nr_data (RAID5, P stripe) or @nr_data + 1 (RAID6, Q stripe). + * + * - sector_nr + * Starts from 0 (representing the first sector of the stripe), ends + * at BTRFS_STRIPE_LEN / sectorsize - 1. + * + * - step_nr + * A step is min(sector_size, PAGE_SIZE). + * + * Starts from 0 (representing the first step of the sector), ends + * at @sector_nsteps - 1. + * + * For most call sites they do not need to bother this parameter. + * It is for bs > ps support and only for vertical stripe related works. + * (e.g. RMW/recover) + * + * - from which array + * Whether grabbing from stripe_paddrs[] (aka, internal pages) or from the + * bio_paddrs[] (aka, from the higher layer bios). + * + * For IO, a physical address is returned, so that we can extract the page and + * the offset inside the page for IO. + * A special value INVALID_PADDR represents when the physical address is invalid, + * normally meaning there is no page allocated for the specified sector. + */ +struct btrfs_raid_bio { + struct btrfs_io_context *bioc; + + /* + * While we're doing RMW on a stripe we put it into a hash table so we + * can lock the stripe and merge more rbios into it. + */ + struct list_head hash_list; + + /* LRU list for the stripe cache */ + struct list_head stripe_cache; + + /* For scheduling work in the helper threads */ + struct work_struct work; + + /* + * bio_list and bio_list_lock are used to add more bios into the stripe + * in hopes of avoiding the full RMW + */ + struct bio_list bio_list; + spinlock_t bio_list_lock; + + /* + * Also protected by the bio_list_lock, the plug list is used by the + * plugging code to collect partial bios while plugged. The stripe + * locking code also uses it to hand off the stripe lock to the next + * pending IO. + */ + struct list_head plug_list; + + /* Flags that tell us if it is safe to merge with this bio. */ + unsigned long flags; + + /* + * Set if we're doing a parity rebuild for a read from higher up, which + * is handled differently from a parity rebuild as part of RMW. + */ + enum btrfs_rbio_ops operation; + + /* How many pages there are for the full stripe including P/Q */ + u16 nr_pages; + + /* How many sectors there are for the full stripe including P/Q */ + u16 nr_sectors; + + /* Number of data stripes (no p/q) */ + u8 nr_data; + + /* Number of all stripes (including P/Q) */ + u8 real_stripes; + + /* How many pages there are for each stripe */ + u8 stripe_npages; + + /* How many sectors there are for each stripe */ + u8 stripe_nsectors; + + /* + * How many steps there are for one sector. + * + * For bs > ps cases, it's sectorsize / PAGE_SIZE. + * For bs <= ps cases, it's always 1. + */ + u8 sector_nsteps; + + /* Stripe number that we're scrubbing */ + u8 scrubp; + + /* + * Size of all the bios in the bio_list. This helps us decide if the + * rbio maps to a full stripe or not. + */ + int bio_list_bytes; + + refcount_t refs; + + atomic_t stripes_pending; + + wait_queue_head_t io_wait; + + /* Bitmap to record which horizontal stripe has data */ + unsigned long dbitmap; + + /* Allocated with stripe_nsectors-many bits for finish_*() calls */ + unsigned long finish_pbitmap; + + /* + * These are two arrays of pointers. We allocate the rbio big enough + * to hold them both and setup their locations when the rbio is + * allocated. + */ + + /* + * Pointers to pages that we allocated for reading/writing stripes + * directly from the disk (including P/Q). + */ + struct page **stripe_pages; + + /* Pointers to the sectors in the bio_list, for faster lookup */ + phys_addr_t *bio_paddrs; + + /* Pointers to the sectors in the stripe_pages[]. */ + phys_addr_t *stripe_paddrs; + + /* Each set bit means the corresponding sector in stripe_sectors[] is uptodate. */ + unsigned long *stripe_uptodate_bitmap; + + /* Allocated with real_stripes-many pointers for finish_*() calls */ + void **finish_pointers; + + /* + * The bitmap recording where IO errors happened. + * Each bit is corresponding to one sector in either bio_sectors[] or + * stripe_sectors[] array. + */ + unsigned long *error_bitmap; + + /* + * Checksum buffer if the rbio is for data. The buffer should cover + * all data sectors (excluding P/Q sectors). + */ + u8 *csum_buf; + + /* + * Each bit represents if the corresponding sector has data csum found. + * Should only cover data sectors (excluding P/Q sectors). + */ + unsigned long *csum_bitmap; +}; + +/* + * For trace event usage only. Records useful debug info for each bio submitted + * by RAID56 to each physical device. + * + * No matter signed or not, (-1) is always the one indicating we can not grab + * the proper stripe number. + */ +struct raid56_bio_trace_info { + u64 devid; + + /* The offset inside the stripe. (<= STRIPE_LEN) */ + u32 offset; + + /* + * Stripe number. + * 0 is the first data stripe, and nr_data for P stripe, + * nr_data + 1 for Q stripe. + * >= real_stripes for + */ + u8 stripe_nr; +}; + +static inline int nr_data_stripes(const struct btrfs_chunk_map *map) +{ + return map->num_stripes - btrfs_nr_parity_stripes(map->type); +} + +static inline int nr_bioc_data_stripes(const struct btrfs_io_context *bioc) +{ + return bioc->num_stripes - btrfs_nr_parity_stripes(bioc->map_type); +} + +#define RAID5_P_STRIPE ((u64)-2) +#define RAID6_Q_STRIPE ((u64)-1) + +#define is_parity_stripe(x) (((x) == RAID5_P_STRIPE) || \ + ((x) == RAID6_Q_STRIPE)) + +struct btrfs_device; + +void raid56_parity_recover(struct bio *bio, struct btrfs_io_context *bioc, + int mirror_num); +void raid56_parity_write(struct bio *bio, struct btrfs_io_context *bioc); + +struct btrfs_raid_bio *raid56_parity_alloc_scrub_rbio(struct bio *bio, + struct btrfs_io_context *bioc, + struct btrfs_device *scrub_dev, + unsigned long *dbitmap, int stripe_nsectors); +void raid56_parity_submit_scrub_rbio(struct btrfs_raid_bio *rbio); + +void raid56_parity_cache_data_folios(struct btrfs_raid_bio *rbio, + void *vaddr, u64 data_logical); + +int btrfs_alloc_stripe_hash_table(struct btrfs_fs_info *info); +void btrfs_free_stripe_hash_table(struct btrfs_fs_info *info); + +#endif diff --git a/fs/btrfs/ref-verify.c b/fs/btrfs/ref-verify.c new file mode 100644 index 000000000..f78369ff2 --- /dev/null +++ b/fs/btrfs/ref-verify.c @@ -0,0 +1,1025 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2014 Facebook. All rights reserved. + */ + +#include +#include +#include "messages.h" +#include "ctree.h" +#include "disk-io.h" +#include "locking.h" +#include "delayed-ref.h" +#include "ref-verify.h" +#include "fs.h" +#include "accessors.h" + +/* + * Used to keep track the roots and number of refs each root has for a given + * bytenr. This just tracks the number of direct references, no shared + * references. + */ +struct root_entry { + u64 root_objectid; + u64 num_refs; + struct rb_node node; +}; + +/* + * These are meant to represent what should exist in the extent tree, these can + * be used to verify the extent tree is consistent as these should all match + * what the extent tree says. + */ +struct ref_entry { + u64 root_objectid; + u64 parent; + u64 owner; + u64 offset; + u64 num_refs; + struct rb_node node; +}; + +#define MAX_TRACE 16 + +/* + * Whenever we add/remove a reference we record the action. The action maps + * back to the delayed ref action. We hold the ref we are changing in the + * action so we can account for the history properly, and we record the root we + * were called with since it could be different from ref_root. We also store + * stack traces because that's how I roll. + */ +struct ref_action { + int action; + u64 root; + struct ref_entry ref; + struct list_head list; + unsigned long trace[MAX_TRACE]; + unsigned int trace_len; +}; + +/* + * One of these for every block we reference, it holds the roots and references + * to it as well as all of the ref actions that have occurred to it. We never + * free it until we unmount the file system in order to make sure re-allocations + * are happening properly. + */ +struct block_entry { + u64 bytenr; + u64 len; + u64 num_refs; + int metadata; + int from_disk; + struct rb_root roots; + struct rb_root refs; + struct rb_node node; + struct list_head actions; +}; + +static int block_entry_bytenr_key_cmp(const void *key, const struct rb_node *node) +{ + const u64 *bytenr = key; + const struct block_entry *entry = rb_entry(node, struct block_entry, node); + + if (entry->bytenr < *bytenr) + return 1; + else if (entry->bytenr > *bytenr) + return -1; + + return 0; +} + +static int block_entry_bytenr_cmp(struct rb_node *new, const struct rb_node *existing) +{ + const struct block_entry *new_entry = rb_entry(new, struct block_entry, node); + + return block_entry_bytenr_key_cmp(&new_entry->bytenr, existing); +} + +static struct block_entry *insert_block_entry(struct rb_root *root, + struct block_entry *be) +{ + struct rb_node *node; + + node = rb_find_add(&be->node, root, block_entry_bytenr_cmp); + return rb_entry_safe(node, struct block_entry, node); +} + +static struct block_entry *lookup_block_entry(struct rb_root *root, u64 bytenr) +{ + struct rb_node *node; + + node = rb_find(&bytenr, root, block_entry_bytenr_key_cmp); + return rb_entry_safe(node, struct block_entry, node); +} + +static int root_entry_root_objectid_key_cmp(const void *key, const struct rb_node *node) +{ + const u64 *objectid = key; + const struct root_entry *entry = rb_entry(node, struct root_entry, node); + + if (entry->root_objectid < *objectid) + return 1; + else if (entry->root_objectid > *objectid) + return -1; + + return 0; +} + +static int root_entry_root_objectid_cmp(struct rb_node *new, const struct rb_node *existing) +{ + const struct root_entry *new_entry = rb_entry(new, struct root_entry, node); + + return root_entry_root_objectid_key_cmp(&new_entry->root_objectid, existing); +} + +static struct root_entry *insert_root_entry(struct rb_root *root, + struct root_entry *re) +{ + struct rb_node *node; + + node = rb_find_add(&re->node, root, root_entry_root_objectid_cmp); + return rb_entry_safe(node, struct root_entry, node); +} + +static int comp_refs(struct ref_entry *ref1, struct ref_entry *ref2) +{ + if (ref1->root_objectid < ref2->root_objectid) + return -1; + if (ref1->root_objectid > ref2->root_objectid) + return 1; + if (ref1->parent < ref2->parent) + return -1; + if (ref1->parent > ref2->parent) + return 1; + if (ref1->owner < ref2->owner) + return -1; + if (ref1->owner > ref2->owner) + return 1; + if (ref1->offset < ref2->offset) + return -1; + if (ref1->offset > ref2->offset) + return 1; + return 0; +} + +static int ref_entry_cmp(struct rb_node *new, const struct rb_node *existing) +{ + struct ref_entry *new_entry = rb_entry(new, struct ref_entry, node); + struct ref_entry *existing_entry = rb_entry(existing, struct ref_entry, node); + + return comp_refs(new_entry, existing_entry); +} + +static struct ref_entry *insert_ref_entry(struct rb_root *root, + struct ref_entry *ref) +{ + struct rb_node *node; + + node = rb_find_add(&ref->node, root, ref_entry_cmp); + return rb_entry_safe(node, struct ref_entry, node); +} + +static struct root_entry *lookup_root_entry(struct rb_root *root, u64 objectid) +{ + struct rb_node *node; + + node = rb_find(&objectid, root, root_entry_root_objectid_key_cmp); + return rb_entry_safe(node, struct root_entry, node); +} + +#ifdef CONFIG_STACKTRACE +static void __save_stack_trace(struct ref_action *ra) +{ + ra->trace_len = stack_trace_save(ra->trace, MAX_TRACE, 2); +} + +static void __print_stack_trace(struct btrfs_fs_info *fs_info, + struct ref_action *ra) +{ + if (ra->trace_len == 0) { + btrfs_err(fs_info, " ref-verify: no stacktrace"); + return; + } + stack_trace_print(ra->trace, ra->trace_len, 2); +} +#else +static inline void __save_stack_trace(struct ref_action *ra) +{ +} + +static inline void __print_stack_trace(struct btrfs_fs_info *fs_info, + struct ref_action *ra) +{ + btrfs_err(fs_info, " ref-verify: no stacktrace support"); +} +#endif + +static void free_block_entry(struct block_entry *be) +{ + struct root_entry *re; + struct ref_entry *ref; + struct ref_action *ra; + struct rb_node *n; + + while ((n = rb_first(&be->roots))) { + re = rb_entry(n, struct root_entry, node); + rb_erase(&re->node, &be->roots); + kfree(re); + } + + while((n = rb_first(&be->refs))) { + ref = rb_entry(n, struct ref_entry, node); + rb_erase(&ref->node, &be->refs); + kfree(ref); + } + + while (!list_empty(&be->actions)) { + ra = list_first_entry(&be->actions, struct ref_action, + list); + list_del(&ra->list); + kfree(ra); + } + kfree(be); +} + +static struct block_entry *add_block_entry(struct btrfs_fs_info *fs_info, + u64 bytenr, u64 len, + u64 root_objectid) +{ + struct block_entry *be = NULL, *exist; + struct root_entry *re = NULL; + + re = kzalloc_obj(struct root_entry, GFP_NOFS); + be = kzalloc_obj(struct block_entry, GFP_NOFS); + if (!be || !re) { + kfree(re); + kfree(be); + return ERR_PTR(-ENOMEM); + } + be->bytenr = bytenr; + be->len = len; + + re->root_objectid = root_objectid; + re->num_refs = 0; + + spin_lock(&fs_info->ref_verify_lock); + exist = insert_block_entry(&fs_info->block_tree, be); + if (exist) { + if (root_objectid) { + struct root_entry *exist_re; + + exist_re = insert_root_entry(&exist->roots, re); + if (exist_re) + kfree(re); + } else { + kfree(re); + } + kfree(be); + return exist; + } + + be->num_refs = 0; + be->metadata = 0; + be->from_disk = 0; + be->roots = RB_ROOT; + be->refs = RB_ROOT; + INIT_LIST_HEAD(&be->actions); + if (root_objectid) + insert_root_entry(&be->roots, re); + else + kfree(re); + return be; +} + +static int add_tree_block(struct btrfs_fs_info *fs_info, u64 ref_root, + u64 parent, u64 bytenr, int level) +{ + struct block_entry *be; + struct root_entry *re; + struct ref_entry *ref = NULL, *exist; + + ref = kmalloc_obj(struct ref_entry, GFP_NOFS); + if (!ref) + return -ENOMEM; + + if (parent) + ref->root_objectid = 0; + else + ref->root_objectid = ref_root; + ref->parent = parent; + ref->owner = level; + ref->offset = 0; + ref->num_refs = 1; + + be = add_block_entry(fs_info, bytenr, fs_info->nodesize, ref_root); + if (IS_ERR(be)) { + kfree(ref); + return PTR_ERR(be); + } + be->num_refs++; + be->from_disk = 1; + be->metadata = 1; + + if (!parent) { + ASSERT(ref_root); + re = lookup_root_entry(&be->roots, ref_root); + ASSERT(re); + re->num_refs++; + } + exist = insert_ref_entry(&be->refs, ref); + if (exist) { + exist->num_refs++; + kfree(ref); + } + spin_unlock(&fs_info->ref_verify_lock); + + return 0; +} + +static int add_shared_data_ref(struct btrfs_fs_info *fs_info, + u64 parent, u32 num_refs, u64 bytenr, + u64 num_bytes) +{ + struct block_entry *be; + struct ref_entry *ref; + + ref = kzalloc_obj(struct ref_entry, GFP_NOFS); + if (!ref) + return -ENOMEM; + be = add_block_entry(fs_info, bytenr, num_bytes, 0); + if (IS_ERR(be)) { + kfree(ref); + return PTR_ERR(be); + } + be->num_refs += num_refs; + + ref->parent = parent; + ref->num_refs = num_refs; + if (insert_ref_entry(&be->refs, ref)) { + spin_unlock(&fs_info->ref_verify_lock); + btrfs_err(fs_info, "existing shared ref when reading from disk?"); + kfree(ref); + return -EINVAL; + } + spin_unlock(&fs_info->ref_verify_lock); + return 0; +} + +static int add_extent_data_ref(struct btrfs_fs_info *fs_info, + struct extent_buffer *leaf, + struct btrfs_extent_data_ref *dref, + u64 bytenr, u64 num_bytes) +{ + struct block_entry *be; + struct ref_entry *ref; + struct root_entry *re; + u64 ref_root = btrfs_extent_data_ref_root(leaf, dref); + u64 owner = btrfs_extent_data_ref_objectid(leaf, dref); + u64 offset = btrfs_extent_data_ref_offset(leaf, dref); + u32 num_refs = btrfs_extent_data_ref_count(leaf, dref); + + ref = kzalloc_obj(struct ref_entry, GFP_NOFS); + if (!ref) + return -ENOMEM; + be = add_block_entry(fs_info, bytenr, num_bytes, ref_root); + if (IS_ERR(be)) { + kfree(ref); + return PTR_ERR(be); + } + be->num_refs += num_refs; + + ref->parent = 0; + ref->owner = owner; + ref->root_objectid = ref_root; + ref->offset = offset; + ref->num_refs = num_refs; + if (insert_ref_entry(&be->refs, ref)) { + spin_unlock(&fs_info->ref_verify_lock); + btrfs_err(fs_info, "existing ref when reading from disk?"); + kfree(ref); + return -EINVAL; + } + + re = lookup_root_entry(&be->roots, ref_root); + if (!re) { + spin_unlock(&fs_info->ref_verify_lock); + btrfs_err(fs_info, "missing root in new block entry?"); + return -EINVAL; + } + re->num_refs += num_refs; + spin_unlock(&fs_info->ref_verify_lock); + return 0; +} + +static int process_extent_item(struct btrfs_fs_info *fs_info, + struct btrfs_path *path, struct btrfs_key *key, + int slot, int *tree_block_level) +{ + struct btrfs_extent_item *ei; + struct btrfs_extent_inline_ref *iref; + struct btrfs_extent_data_ref *dref; + struct btrfs_shared_data_ref *sref; + struct extent_buffer *leaf = path->nodes[0]; + u32 item_size = btrfs_item_size(leaf, slot); + unsigned long end, ptr; + u64 offset, flags, count; + int type; + int ret = 0; + + ei = btrfs_item_ptr(leaf, slot, struct btrfs_extent_item); + flags = btrfs_extent_flags(leaf, ei); + + if ((key->type == BTRFS_EXTENT_ITEM_KEY) && + flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) { + struct btrfs_tree_block_info *info; + + info = (struct btrfs_tree_block_info *)(ei + 1); + *tree_block_level = btrfs_tree_block_level(leaf, info); + iref = (struct btrfs_extent_inline_ref *)(info + 1); + } else { + if (key->type == BTRFS_METADATA_ITEM_KEY) + *tree_block_level = key->offset; + iref = (struct btrfs_extent_inline_ref *)(ei + 1); + } + + ptr = (unsigned long)iref; + end = (unsigned long)ei + item_size; + while (ptr < end) { + iref = (struct btrfs_extent_inline_ref *)ptr; + type = btrfs_extent_inline_ref_type(leaf, iref); + offset = btrfs_extent_inline_ref_offset(leaf, iref); + switch (type) { + case BTRFS_TREE_BLOCK_REF_KEY: + ret = add_tree_block(fs_info, offset, 0, key->objectid, + *tree_block_level); + break; + case BTRFS_SHARED_BLOCK_REF_KEY: + ret = add_tree_block(fs_info, 0, offset, key->objectid, + *tree_block_level); + break; + case BTRFS_EXTENT_DATA_REF_KEY: + dref = (struct btrfs_extent_data_ref *)(&iref->offset); + ret = add_extent_data_ref(fs_info, leaf, dref, + key->objectid, key->offset); + break; + case BTRFS_SHARED_DATA_REF_KEY: + sref = (struct btrfs_shared_data_ref *)(iref + 1); + count = btrfs_shared_data_ref_count(leaf, sref); + ret = add_shared_data_ref(fs_info, offset, count, + key->objectid, key->offset); + break; + case BTRFS_EXTENT_OWNER_REF_KEY: + if (!btrfs_fs_incompat(fs_info, SIMPLE_QUOTA)) { + btrfs_err(fs_info, + "found extent owner ref without simple quotas enabled"); + ret = -EINVAL; + } + break; + default: + btrfs_err(fs_info, "invalid key type in iref"); + ret = -EINVAL; + break; + } + if (ret) + break; + ptr += btrfs_extent_inline_ref_size(type); + } + return ret; +} + +static int process_leaf(struct btrfs_root *root, + struct btrfs_path *path, u64 *bytenr, u64 *num_bytes, + int *tree_block_level) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct extent_buffer *leaf = path->nodes[0]; + struct btrfs_extent_data_ref *dref; + struct btrfs_shared_data_ref *sref; + u32 count; + int i = 0, ret = 0; + struct btrfs_key key; + int nritems = btrfs_header_nritems(leaf); + + for (i = 0; i < nritems; i++) { + btrfs_item_key_to_cpu(leaf, &key, i); + switch (key.type) { + case BTRFS_EXTENT_ITEM_KEY: + *num_bytes = key.offset; + fallthrough; + case BTRFS_METADATA_ITEM_KEY: + *bytenr = key.objectid; + ret = process_extent_item(fs_info, path, &key, i, + tree_block_level); + break; + case BTRFS_TREE_BLOCK_REF_KEY: + ret = add_tree_block(fs_info, key.offset, 0, + key.objectid, *tree_block_level); + break; + case BTRFS_SHARED_BLOCK_REF_KEY: + ret = add_tree_block(fs_info, 0, key.offset, + key.objectid, *tree_block_level); + break; + case BTRFS_EXTENT_DATA_REF_KEY: + dref = btrfs_item_ptr(leaf, i, + struct btrfs_extent_data_ref); + ret = add_extent_data_ref(fs_info, leaf, dref, *bytenr, + *num_bytes); + break; + case BTRFS_SHARED_DATA_REF_KEY: + sref = btrfs_item_ptr(leaf, i, + struct btrfs_shared_data_ref); + count = btrfs_shared_data_ref_count(leaf, sref); + ret = add_shared_data_ref(fs_info, key.offset, count, + *bytenr, *num_bytes); + break; + default: + break; + } + if (ret) + break; + } + return ret; +} + +/* Walk down to the leaf from the given level */ +static int walk_down_tree(struct btrfs_root *root, struct btrfs_path *path, + int level, u64 *bytenr, u64 *num_bytes, + int *tree_block_level) +{ + struct extent_buffer *eb; + int ret = 0; + + while (level >= 0) { + if (level) { + eb = btrfs_read_node_slot(path->nodes[level], + path->slots[level]); + if (IS_ERR(eb)) + return PTR_ERR(eb); + btrfs_tree_read_lock(eb); + path->nodes[level-1] = eb; + path->slots[level-1] = 0; + path->locks[level-1] = BTRFS_READ_LOCK; + } else { + ret = process_leaf(root, path, bytenr, num_bytes, + tree_block_level); + if (ret) + break; + } + level--; + } + return ret; +} + +/* Walk up to the next node that needs to be processed */ +static int walk_up_tree(struct btrfs_path *path, int *level) +{ + int l; + + for (l = 0; l < BTRFS_MAX_LEVEL; l++) { + if (!path->nodes[l]) + continue; + if (l) { + path->slots[l]++; + if (path->slots[l] < + btrfs_header_nritems(path->nodes[l])) { + *level = l; + return 0; + } + } + btrfs_tree_unlock_rw(path->nodes[l], path->locks[l]); + free_extent_buffer(path->nodes[l]); + path->nodes[l] = NULL; + path->slots[l] = 0; + path->locks[l] = 0; + } + + return 1; +} + +static void dump_ref_action(struct btrfs_fs_info *fs_info, + struct ref_action *ra) +{ + btrfs_err(fs_info, +" Ref action %d, root %llu, ref_root %llu, parent %llu, owner %llu, offset %llu, num_refs %llu", + ra->action, ra->root, ra->ref.root_objectid, ra->ref.parent, + ra->ref.owner, ra->ref.offset, ra->ref.num_refs); + __print_stack_trace(fs_info, ra); +} + +/* + * Dumps all the information from the block entry to printk, it's going to be + * awesome. + */ +static void dump_block_entry(struct btrfs_fs_info *fs_info, + struct block_entry *be) +{ + struct ref_entry *ref; + struct root_entry *re; + struct ref_action *ra; + struct rb_node *n; + + btrfs_err(fs_info, +"dumping block entry [%llu %llu], num_refs %llu, metadata %d, from disk %d", + be->bytenr, be->len, be->num_refs, be->metadata, + be->from_disk); + + for (n = rb_first(&be->refs); n; n = rb_next(n)) { + ref = rb_entry(n, struct ref_entry, node); + btrfs_err(fs_info, +" ref root %llu, parent %llu, owner %llu, offset %llu, num_refs %llu", + ref->root_objectid, ref->parent, ref->owner, + ref->offset, ref->num_refs); + } + + for (n = rb_first(&be->roots); n; n = rb_next(n)) { + re = rb_entry(n, struct root_entry, node); + btrfs_err(fs_info, " root entry %llu, num_refs %llu", + re->root_objectid, re->num_refs); + } + + list_for_each_entry(ra, &be->actions, list) + dump_ref_action(fs_info, ra); +} + +/* + * Called when we modify a ref for a bytenr. + * + * This will add an action item to the given bytenr and do sanity checks to make + * sure we haven't messed something up. If we are making a new allocation and + * this block entry has history we will delete all previous actions as long as + * our sanity checks pass as they are no longer needed. + */ +int btrfs_ref_tree_mod(struct btrfs_fs_info *fs_info, + const struct btrfs_ref *generic_ref) +{ + struct ref_entry *ref = NULL, *exist; + struct ref_action *ra = NULL; + struct block_entry *be = NULL; + struct root_entry *re = NULL; + int action = generic_ref->action; + int ret = 0; + bool metadata; + u64 bytenr = generic_ref->bytenr; + u64 num_bytes = generic_ref->num_bytes; + u64 parent = generic_ref->parent; + u64 ref_root = 0; + u64 owner = 0; + u64 offset = 0; + + if (!btrfs_test_opt(fs_info, REF_VERIFY)) + return 0; + + if (generic_ref->type == BTRFS_REF_METADATA) { + if (!parent) + ref_root = generic_ref->ref_root; + owner = generic_ref->tree_ref.level; + } else if (!parent) { + ref_root = generic_ref->ref_root; + owner = generic_ref->data_ref.objectid; + offset = generic_ref->data_ref.offset; + } + metadata = owner < BTRFS_FIRST_FREE_OBJECTID; + + ref = kzalloc_obj(struct ref_entry, GFP_NOFS); + ra = kmalloc_obj(struct ref_action, GFP_NOFS); + if (!ra || !ref) { + kfree(ref); + kfree(ra); + ret = -ENOMEM; + goto out; + } + + ref->parent = parent; + ref->owner = owner; + ref->root_objectid = ref_root; + ref->offset = offset; + ref->num_refs = (action == BTRFS_DROP_DELAYED_REF) ? -1 : 1; + + memcpy(&ra->ref, ref, sizeof(struct ref_entry)); + /* + * Save the extra info from the delayed ref in the ref action to make it + * easier to figure out what is happening. The real ref's we add to the + * ref tree need to reflect what we save on disk so it matches any + * on-disk refs we pre-loaded. + */ + ra->ref.owner = owner; + ra->ref.offset = offset; + ra->ref.root_objectid = ref_root; + __save_stack_trace(ra); + + INIT_LIST_HEAD(&ra->list); + ra->action = action; + ra->root = generic_ref->real_root; + + /* + * This is an allocation, preallocate the block_entry in case we haven't + * used it before. + */ + ret = -EINVAL; + if (action == BTRFS_ADD_DELAYED_EXTENT) { + /* + * For subvol_create we'll just pass in whatever the parent root + * is and the new root objectid, so let's not treat the passed + * in root as if it really has a ref for this bytenr. + */ + be = add_block_entry(fs_info, bytenr, num_bytes, ref_root); + if (IS_ERR(be)) { + kfree(ref); + kfree(ra); + ret = PTR_ERR(be); + goto out; + } + be->num_refs++; + if (metadata) + be->metadata = 1; + + if (be->num_refs != 1) { + btrfs_err(fs_info, + "re-allocated a block that still has references to it!"); + dump_block_entry(fs_info, be); + dump_ref_action(fs_info, ra); + kfree(ref); + kfree(ra); + goto out_unlock; + } + + while (!list_empty(&be->actions)) { + struct ref_action *tmp; + + tmp = list_first_entry(&be->actions, struct ref_action, + list); + list_del(&tmp->list); + kfree(tmp); + } + } else { + struct root_entry *tmp; + + if (!parent) { + re = kmalloc_obj(struct root_entry, GFP_NOFS); + if (!re) { + kfree(ref); + kfree(ra); + ret = -ENOMEM; + goto out; + } + /* + * This is the root that is modifying us, so it's the + * one we want to lookup below when we modify the + * re->num_refs. + */ + ref_root = generic_ref->real_root; + re->root_objectid = generic_ref->real_root; + re->num_refs = 0; + } + + spin_lock(&fs_info->ref_verify_lock); + be = lookup_block_entry(&fs_info->block_tree, bytenr); + if (!be) { + btrfs_err(fs_info, +"trying to do action %d to bytenr %llu num_bytes %llu but there is no existing entry!", + action, bytenr, num_bytes); + dump_ref_action(fs_info, ra); + kfree(ref); + kfree(ra); + kfree(re); + goto out_unlock; + } else if (be->num_refs == 0) { + btrfs_err(fs_info, + "trying to do action %d for a bytenr that has 0 total references", + action); + dump_block_entry(fs_info, be); + dump_ref_action(fs_info, ra); + kfree(ref); + kfree(ra); + kfree(re); + goto out_unlock; + } + + if (!parent) { + tmp = insert_root_entry(&be->roots, re); + if (tmp) { + kfree(re); + re = tmp; + } + } + } + + exist = insert_ref_entry(&be->refs, ref); + if (exist) { + if (action == BTRFS_DROP_DELAYED_REF) { + if (exist->num_refs == 0) { + btrfs_err(fs_info, +"dropping a ref for a existing root that doesn't have a ref on the block"); + dump_block_entry(fs_info, be); + dump_ref_action(fs_info, ra); + kfree(ref); + kfree(ra); + goto out_unlock; + } + exist->num_refs--; + if (exist->num_refs == 0) { + rb_erase(&exist->node, &be->refs); + kfree(exist); + } + } else if (!be->metadata) { + exist->num_refs++; + } else { + btrfs_err(fs_info, +"attempting to add another ref for an existing ref on a tree block"); + dump_block_entry(fs_info, be); + dump_ref_action(fs_info, ra); + kfree(ref); + kfree(ra); + goto out_unlock; + } + kfree(ref); + } else { + if (action == BTRFS_DROP_DELAYED_REF) { + btrfs_err(fs_info, +"dropping a ref for a root that doesn't have a ref on the block"); + dump_block_entry(fs_info, be); + dump_ref_action(fs_info, ra); + rb_erase(&ref->node, &be->refs); + kfree(ref); + kfree(ra); + goto out_unlock; + } + } + + if (!parent && !re) { + re = lookup_root_entry(&be->roots, ref_root); + if (!re) { + /* + * This shouldn't happen because we will add our re + * above when we lookup the be with !parent, but just in + * case catch this case so we don't panic because I + * didn't think of some other corner case. + */ + btrfs_err(fs_info, "failed to find root %llu for %llu", + generic_ref->real_root, be->bytenr); + dump_block_entry(fs_info, be); + dump_ref_action(fs_info, ra); + kfree(ra); + goto out_unlock; + } + } + if (action == BTRFS_DROP_DELAYED_REF) { + if (re) + re->num_refs--; + be->num_refs--; + } else if (action == BTRFS_ADD_DELAYED_REF) { + be->num_refs++; + if (re) + re->num_refs++; + } + list_add_tail(&ra->list, &be->actions); + ret = 0; +out_unlock: + spin_unlock(&fs_info->ref_verify_lock); +out: + if (ret) { + btrfs_free_ref_cache(fs_info); + btrfs_clear_opt(fs_info->mount_opt, REF_VERIFY); + } + return ret; +} + +/* Free up the ref cache */ +void btrfs_free_ref_cache(struct btrfs_fs_info *fs_info) +{ + struct block_entry *be; + struct rb_node *n; + + if (!btrfs_test_opt(fs_info, REF_VERIFY)) + return; + + spin_lock(&fs_info->ref_verify_lock); + while ((n = rb_first(&fs_info->block_tree))) { + be = rb_entry(n, struct block_entry, node); + rb_erase(&be->node, &fs_info->block_tree); + free_block_entry(be); + cond_resched_lock(&fs_info->ref_verify_lock); + } + spin_unlock(&fs_info->ref_verify_lock); +} + +void btrfs_free_ref_tree_range(struct btrfs_fs_info *fs_info, u64 start, + u64 len) +{ + struct block_entry *be = NULL, *entry; + struct rb_node *n; + + if (!btrfs_test_opt(fs_info, REF_VERIFY)) + return; + + spin_lock(&fs_info->ref_verify_lock); + n = fs_info->block_tree.rb_node; + while (n) { + entry = rb_entry(n, struct block_entry, node); + if (entry->bytenr < start) { + n = n->rb_right; + } else if (entry->bytenr > start) { + n = n->rb_left; + } else { + be = entry; + break; + } + /* We want to get as close to start as possible */ + if (be == NULL || + (entry->bytenr < start && be->bytenr > start) || + (entry->bytenr < start && entry->bytenr > be->bytenr)) + be = entry; + } + + /* + * Could have an empty block group, maybe have something to check for + * this case to verify we were actually empty? + */ + if (!be) { + spin_unlock(&fs_info->ref_verify_lock); + return; + } + + n = &be->node; + while (n) { + be = rb_entry(n, struct block_entry, node); + n = rb_next(n); + if (be->bytenr < start && be->bytenr + be->len > start) { + btrfs_err(fs_info, + "block entry overlaps a block group [%llu,%llu]!", + start, len); + dump_block_entry(fs_info, be); + continue; + } + if (be->bytenr < start) + continue; + if (be->bytenr >= start + len) + break; + if (be->bytenr + be->len > start + len) { + btrfs_err(fs_info, + "block entry overlaps a block group [%llu,%llu]!", + start, len); + dump_block_entry(fs_info, be); + } + rb_erase(&be->node, &fs_info->block_tree); + free_block_entry(be); + } + spin_unlock(&fs_info->ref_verify_lock); +} + +/* Walk down all roots and build the ref tree, meant to be called at mount */ +int btrfs_build_ref_tree(struct btrfs_fs_info *fs_info) +{ + struct btrfs_root *extent_root; + BTRFS_PATH_AUTO_FREE(path); + struct extent_buffer *eb; + int tree_block_level = 0; + u64 bytenr = 0, num_bytes = 0; + int ret, level; + + if (!btrfs_test_opt(fs_info, REF_VERIFY)) + return 0; + + extent_root = btrfs_extent_root(fs_info, 0); + /* If the extent tree is damaged we cannot ignore it (IGNOREBADROOTS). */ + if (!extent_root) { + btrfs_warn(fs_info, "ref-verify: extent tree not available, disabling"); + btrfs_clear_opt(fs_info->mount_opt, REF_VERIFY); + return 0; + } + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + eb = btrfs_read_lock_root_node(extent_root); + level = btrfs_header_level(eb); + path->nodes[level] = eb; + path->slots[level] = 0; + path->locks[level] = BTRFS_READ_LOCK; + + while (1) { + /* + * We have to keep track of the bytenr/num_bytes we last hit + * because we could have run out of space for an inline ref, and + * would have had to added a ref key item which may appear on a + * different leaf from the original extent item. + */ + ret = walk_down_tree(extent_root, path, level, + &bytenr, &num_bytes, &tree_block_level); + if (ret) + break; + ret = walk_up_tree(path, &level); + if (ret < 0) + break; + if (ret > 0) { + ret = 0; + break; + } + } + if (ret) { + btrfs_free_ref_cache(fs_info); + btrfs_clear_opt(fs_info->mount_opt, REF_VERIFY); + } + return ret; +} diff --git a/fs/btrfs/ref-verify.h b/fs/btrfs/ref-verify.h new file mode 100644 index 000000000..1ce544d53 --- /dev/null +++ b/fs/btrfs/ref-verify.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2014 Facebook. All rights reserved. + */ + +#ifndef BTRFS_REF_VERIFY_H +#define BTRFS_REF_VERIFY_H + +#include +#include + +struct btrfs_fs_info; +struct btrfs_ref; + +#ifdef CONFIG_BTRFS_DEBUG + +#include + +int btrfs_build_ref_tree(struct btrfs_fs_info *fs_info); +void btrfs_free_ref_cache(struct btrfs_fs_info *fs_info); +int btrfs_ref_tree_mod(struct btrfs_fs_info *fs_info, + const struct btrfs_ref *generic_ref); +void btrfs_free_ref_tree_range(struct btrfs_fs_info *fs_info, u64 start, + u64 len); + +static inline void btrfs_init_ref_verify(struct btrfs_fs_info *fs_info) +{ + spin_lock_init(&fs_info->ref_verify_lock); + fs_info->block_tree = RB_ROOT; +} +#else +static inline int btrfs_build_ref_tree(struct btrfs_fs_info *fs_info) +{ + return 0; +} + +static inline void btrfs_free_ref_cache(struct btrfs_fs_info *fs_info) +{ +} + +static inline int btrfs_ref_tree_mod(struct btrfs_fs_info *fs_info, + const struct btrfs_ref *generic_ref) +{ + return 0; +} + +static inline void btrfs_free_ref_tree_range(struct btrfs_fs_info *fs_info, + u64 start, u64 len) +{ +} + +static inline void btrfs_init_ref_verify(struct btrfs_fs_info *fs_info) +{ +} + +#endif /* CONFIG_BTRFS_DEBUG */ + +#endif diff --git a/fs/btrfs/reflink.c b/fs/btrfs/reflink.c new file mode 100644 index 000000000..d2a410191 --- /dev/null +++ b/fs/btrfs/reflink.c @@ -0,0 +1,983 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include "ctree.h" +#include "fs.h" +#include "messages.h" +#include "compression.h" +#include "delalloc-space.h" +#include "disk-io.h" +#include "reflink.h" +#include "transaction.h" +#include "subpage.h" +#include "accessors.h" +#include "file-item.h" +#include "file.h" +#include "super.h" + +#define BTRFS_MAX_DEDUPE_LEN SZ_16M + +static int clone_finish_inode_update(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + u64 endoff, + const u64 destoff, + const u64 olen, + bool no_time_update) +{ + struct inode *vfs_inode = &inode->vfs_inode; + int ret; + + inode_inc_iversion(vfs_inode); + if (!no_time_update) + inode_set_mtime_to_ts(vfs_inode, inode_set_ctime_current(vfs_inode)); + + /* + * We round up to the block size at eof when determining which + * extents to clone above, but shouldn't round up the file size. + */ + if (endoff > destoff + olen) + endoff = destoff + olen; + if (endoff > vfs_inode->i_size) { + i_size_write(vfs_inode, endoff); + btrfs_inode_safe_disk_i_size_write(inode, 0); + } + + ret = btrfs_update_inode(trans, inode); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + btrfs_end_transaction(trans); + return ret; + } + return btrfs_end_transaction(trans); +} + +static int copy_inline_to_page(struct btrfs_inode *inode, + const u64 file_offset, + char *inline_data, + const u64 size, + const u64 datal, + const u8 comp_type) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + const u32 block_size = fs_info->sectorsize; + const u64 range_end = file_offset + block_size - 1; + const size_t inline_size = size - btrfs_file_extent_calc_inline_size(0); + char *data_start = inline_data + btrfs_file_extent_calc_inline_size(0); + struct extent_changeset *data_reserved = NULL; + struct folio *folio = NULL; + struct address_space *mapping = inode->vfs_inode.i_mapping; + int ret; + + ASSERT(IS_ALIGNED(file_offset, block_size), "file_offset=%llu block_size=%u", + file_offset, block_size); + + /* + * We have flushed and locked the ranges of the source and destination + * inodes, we also have locked the inodes, so we are safe to do a + * reservation here. Also we must not do the reservation while holding + * a transaction open, otherwise we would deadlock. + */ + ret = btrfs_delalloc_reserve_space(inode, &data_reserved, file_offset, + block_size); + if (ret) + goto out; + + folio = __filemap_get_folio(mapping, file_offset >> PAGE_SHIFT, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, + btrfs_alloc_write_mask(mapping)); + if (IS_ERR(folio)) { + ret = PTR_ERR(folio); + goto out_unlock; + } + + ret = set_folio_extent_mapped(folio); + if (ret < 0) + goto out_unlock; + + ret = btrfs_reset_extent_delalloc(inode, file_offset, range_end, 0, NULL); + if (ret) + goto out_unlock; + + /* + * After dirtying the page our caller will need to start a transaction, + * and if we are low on metadata free space, that can cause flushing of + * delalloc for all inodes in order to get metadata space released. + * However we are holding the range locked for the whole duration of + * the clone/dedupe operation, so we may deadlock if that happens and no + * other task releases enough space. So mark this inode as not being + * possible to flush to avoid such deadlock. We will clear that flag + * when we finish cloning all extents, since a transaction is started + * after finding each extent to clone. + */ + set_bit(BTRFS_INODE_NO_DELALLOC_FLUSH, &inode->runtime_flags); + + if (comp_type == BTRFS_COMPRESS_NONE) { + memcpy_to_folio(folio, offset_in_folio(folio, file_offset), data_start, + datal); + } else { + ret = btrfs_decompress(comp_type, data_start, folio, + offset_in_folio(folio, file_offset), + inline_size, datal); + if (ret) + goto out_unlock; + flush_dcache_folio(folio); + } + + /* + * If our inline data is smaller then the block/page size, then the + * remaining of the block/page is equivalent to zeroes. We had something + * like the following done: + * + * $ xfs_io -f -c "pwrite -S 0xab 0 500" file + * $ sync # (or fsync) + * $ xfs_io -c "falloc 0 4K" file + * $ xfs_io -c "pwrite -S 0xcd 4K 4K" + * + * So what's in the range [500, 4095] corresponds to zeroes. + */ + if (datal < block_size) + folio_zero_range(folio, datal, block_size - datal); + + btrfs_folio_set_uptodate(fs_info, folio, file_offset, block_size); + btrfs_folio_set_dirty(fs_info, folio, file_offset, block_size); +out_unlock: + if (!IS_ERR(folio)) { + folio_unlock(folio); + folio_put(folio); + } + if (ret) + btrfs_delalloc_release_space(inode, data_reserved, file_offset, + block_size, true); + btrfs_delalloc_release_extents(inode, block_size); +out: + extent_changeset_free(data_reserved); + + return ret; +} + +/* + * Deal with cloning of inline extents. We try to copy the inline extent from + * the source inode to destination inode when possible. When not possible we + * copy the inline extent's data into the respective page of the inode. + */ +static int clone_copy_inline_extent(struct btrfs_inode *inode, + struct btrfs_path *path, + struct btrfs_key *new_key, + const u64 drop_start, + const u64 datal, + const u64 size, + const u8 comp_type, + char *inline_data, + struct btrfs_trans_handle **trans_out) +{ + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + const u64 aligned_end = ALIGN(new_key->offset + datal, + fs_info->sectorsize); + struct btrfs_trans_handle *trans = NULL; + struct btrfs_drop_extents_args drop_args = { 0 }; + int ret; + struct btrfs_key key; + bool copied_inline_to_page = false; + + if (new_key->offset > 0) { + ret = copy_inline_to_page(inode, new_key->offset, + inline_data, size, datal, comp_type); + copied_inline_to_page = (ret == 0); + goto out; + } + + key.objectid = btrfs_ino(inode); + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = 0; + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) { + return ret; + } else if (ret > 0) { + if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) { + ret = btrfs_next_leaf(root, path); + if (ret < 0) + return ret; + else if (ret > 0) + goto copy_inline_extent; + } + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + if (key.objectid == btrfs_ino(inode) && + key.type == BTRFS_EXTENT_DATA_KEY) { + /* + * There's an implicit hole at file offset 0, copy the + * inline extent's data to the page. + */ + ASSERT(key.offset > 0); + goto copy_to_page; + } + } else if (i_size_read(&inode->vfs_inode) <= datal) { + struct btrfs_file_extent_item *ei; + + ei = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_file_extent_item); + /* + * If it's an inline extent replace it with the source inline + * extent, otherwise copy the source inline extent data into + * the respective page at the destination inode. + */ + if (btrfs_file_extent_type(path->nodes[0], ei) == + BTRFS_FILE_EXTENT_INLINE) + goto copy_inline_extent; + + goto copy_to_page; + } + +copy_inline_extent: + /* + * We have no extent items, or we have an extent at offset 0 which may + * or may not be inlined. All these cases are dealt the same way. + */ + if (i_size_read(&inode->vfs_inode) > datal) { + /* + * At the destination offset 0 we have either a hole, a regular + * extent or an inline extent larger then the one we want to + * clone. Deal with all these cases by copying the inline extent + * data into the respective page at the destination inode. + */ + goto copy_to_page; + } + + /* + * Release path before starting a new transaction so we don't hold locks + * that would confuse lockdep. + */ + btrfs_release_path(path); + /* + * If we end up here it means were copy the inline extent into a leaf + * of the destination inode. We know we will drop or adjust at most one + * extent item in the destination root. + * + * 1 unit - adjusting old extent (we may have to split it) + * 1 unit - add new extent + * 1 unit - inode update + */ + trans = btrfs_start_transaction(root, 3); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + trans = NULL; + goto out; + } + drop_args.path = path; + drop_args.start = drop_start; + drop_args.end = aligned_end; + drop_args.drop_cache = true; + ret = btrfs_drop_extents(trans, root, inode, &drop_args); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + ret = btrfs_insert_empty_item(trans, root, path, new_key, size); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + + write_extent_buffer(path->nodes[0], inline_data, + btrfs_item_ptr_offset(path->nodes[0], + path->slots[0]), + size); + btrfs_update_inode_bytes(inode, datal, drop_args.bytes_found); + btrfs_set_inode_full_sync(inode); + ret = btrfs_inode_set_file_extent_range(inode, 0, aligned_end); + if (unlikely(ret)) + btrfs_abort_transaction(trans, ret); +out: + if (!ret && !trans) { + if (copied_inline_to_page && + new_key->offset + datal > i_size_read(&inode->vfs_inode)) { + /* + * If we copied the inline extent data to a page/folio + * beyond the i_size of the destination inode, then we + * need to increase the i_size before we start a + * transaction to update the inode item. This is to + * prevent a deadlock when the flushoncommit mount + * option is used, which happens like this: + * + * 1) Task A clones an inline extent from inode X to an + * offset of inode Y that is beyond Y's current + * i_size. This means we copied the inline extent's + * data to a folio of inode Y that is beyond its EOF, + * using the call above to copy_inline_to_page(); + * + * 2) Task B starts a transaction commit and calls + * btrfs_start_delalloc_flush() to flush delalloc; + * + * 3) The delalloc flushing sees the new dirty folio of + * inode Y and when it attempts to flush it, it ends + * up at extent_writepage() and sees that the offset + * of the folio is beyond the i_size of inode Y, so + * it attempts to invalidate the folio by calling + * folio_invalidate(), which ends up at btrfs' folio + * invalidate callback - btrfs_invalidate_folio(). + * There it tries to lock the folio's range in inode + * Y's extent io tree, but it blocks since it's + * currently locked by task A - during reflink we + * lock the inodes and the source and destination + * ranges after flushing all delalloc and waiting for + * ordered extent completion - after that we don't + * expect to have dirty folios in the ranges, the + * exception is if we have to copy an inline extent's + * data (because the destination offset is not zero); + * + * 4) Task A then does the 'goto out' below and attempts + * to start a transaction to update the inode item, + * and then it's blocked since the current + * transaction is in the TRANS_STATE_COMMIT_START + * state. Therefore task A has to wait for the + * current transaction to become unblocked (its + * state >= TRANS_STATE_UNBLOCKED). + * + * This leads to a deadlock - the task committing the + * transaction waiting for the delalloc flushing which + * is blocked during folio invalidation on the inode's + * extent lock and the reflink task waiting for the + * current transaction to be unblocked so that it can + * start a new one to update the inode item (while + * holding the extent lock). + */ + i_size_write(&inode->vfs_inode, new_key->offset + datal); + } + /* + * No transaction here means we copied the inline extent into a + * page of the destination inode. + * + * 1 unit to update inode item + */ + trans = btrfs_start_transaction(root, 1); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + trans = NULL; + } + } + if (ret && trans) + btrfs_end_transaction(trans); + if (!ret) + *trans_out = trans; + + return ret; + +copy_to_page: + /* + * Release our path because we don't need it anymore and also because + * copy_inline_to_page() needs to reserve data and metadata, which may + * need to flush delalloc when we are low on available space and + * therefore cause a deadlock if writeback of an inline extent needs to + * write to the same leaf or an ordered extent completion needs to write + * to the same leaf. + */ + btrfs_release_path(path); + + ret = copy_inline_to_page(inode, new_key->offset, + inline_data, size, datal, comp_type); + copied_inline_to_page = (ret == 0); + + goto out; +} + +/* + * Clone a range from inode file to another. + * + * @src: Inode to clone from + * @inode: Inode to clone to + * @off: Offset within source to start clone from + * @olen: Original length, passed by user, of range to clone + * @olen_aligned: Block-aligned value of olen + * @destoff: Offset within @inode to start clone + * @no_time_update: Whether to update mtime/ctime on the target inode + */ +static int btrfs_clone(struct btrfs_inode *src, struct btrfs_inode *inode, + const u64 off, const u64 olen, const u64 olen_aligned, + const u64 destoff, bool no_time_update) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + BTRFS_PATH_AUTO_FREE(path); + struct extent_buffer *leaf; + struct btrfs_trans_handle *trans; + char AUTO_KVFREE(buf); + struct btrfs_key key; + u32 nritems; + int slot; + int ret; + const u64 len = olen_aligned; + u64 last_dest_end = destoff; + u64 prev_extent_end = off; + + ret = -ENOMEM; + buf = kvmalloc(fs_info->nodesize, GFP_KERNEL); + if (!buf) + return ret; + + path = btrfs_alloc_path(); + if (!path) + return ret; + + path->reada = READA_FORWARD; + /* Clone data */ + key.objectid = btrfs_ino(src); + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = off; + + while (1) { + struct btrfs_file_extent_item *extent; + u64 extent_gen; + int type; + u32 size; + struct btrfs_key new_key; + u64 disko = 0, diskl = 0; + u64 datao = 0, datal = 0; + u8 comp; + u64 drop_start; + + /* Note the key will change type as we walk through the tree */ + ret = btrfs_search_slot(NULL, src->root, &key, path, 0, 0); + if (ret < 0) + goto out; + /* + * First search, if no extent item that starts at offset off was + * found but the previous item is an extent item, it's possible + * it might overlap our target range, therefore process it. + */ + if (key.offset == off && ret > 0 && path->slots[0] > 0) { + btrfs_item_key_to_cpu(path->nodes[0], &key, + path->slots[0] - 1); + if (key.type == BTRFS_EXTENT_DATA_KEY) + path->slots[0]--; + } + + nritems = btrfs_header_nritems(path->nodes[0]); +process_slot: + if (path->slots[0] >= nritems) { + ret = btrfs_next_leaf(src->root, path); + if (ret < 0) + goto out; + if (ret > 0) + break; + nritems = btrfs_header_nritems(path->nodes[0]); + } + leaf = path->nodes[0]; + slot = path->slots[0]; + + btrfs_item_key_to_cpu(leaf, &key, slot); + if (key.type > BTRFS_EXTENT_DATA_KEY || key.objectid != btrfs_ino(src)) + break; + + ASSERT(key.type == BTRFS_EXTENT_DATA_KEY, "key.type=%u", key.type); + + extent = btrfs_item_ptr(leaf, slot, + struct btrfs_file_extent_item); + extent_gen = btrfs_file_extent_generation(leaf, extent); + comp = btrfs_file_extent_compression(leaf, extent); + type = btrfs_file_extent_type(leaf, extent); + if (type == BTRFS_FILE_EXTENT_REG || + type == BTRFS_FILE_EXTENT_PREALLOC) { + disko = btrfs_file_extent_disk_bytenr(leaf, extent); + diskl = btrfs_file_extent_disk_num_bytes(leaf, extent); + datao = btrfs_file_extent_offset(leaf, extent); + datal = btrfs_file_extent_num_bytes(leaf, extent); + } else if (type == BTRFS_FILE_EXTENT_INLINE) { + /* Take upper bound, may be compressed */ + datal = btrfs_file_extent_ram_bytes(leaf, extent); + } + + /* + * The first search might have left us at an extent item that + * ends before our target range's start, can happen if we have + * holes and NO_HOLES feature enabled. + * + * Subsequent searches may leave us on a file range we have + * processed before - this happens due to a race with ordered + * extent completion for a file range that is outside our source + * range, but that range was part of a file extent item that + * also covered a leading part of our source range. + */ + if (key.offset + datal <= prev_extent_end) { + path->slots[0]++; + goto process_slot; + } else if (key.offset >= off + len) { + break; + } + + prev_extent_end = key.offset + datal; + size = btrfs_item_size(leaf, slot); + read_extent_buffer(leaf, buf, btrfs_item_ptr_offset(leaf, slot), + size); + + btrfs_release_path(path); + + memcpy(&new_key, &key, sizeof(new_key)); + new_key.objectid = btrfs_ino(inode); + if (off <= key.offset) + new_key.offset = key.offset + destoff - off; + else + new_key.offset = destoff; + + /* + * Deal with a hole that doesn't have an extent item that + * represents it (NO_HOLES feature enabled). + * This hole is either in the middle of the cloning range or at + * the beginning (fully overlaps it or partially overlaps it). + */ + if (new_key.offset != last_dest_end) + drop_start = last_dest_end; + else + drop_start = new_key.offset; + + if (type == BTRFS_FILE_EXTENT_REG || + type == BTRFS_FILE_EXTENT_PREALLOC) { + struct btrfs_replace_extent_info clone_info; + + /* + * a | --- range to clone ---| b + * | ------------- extent ------------- | + */ + + /* Subtract range b */ + if (key.offset + datal > off + len) + datal = off + len - key.offset; + + /* Subtract range a */ + if (off > key.offset) { + datao += off - key.offset; + datal -= off - key.offset; + } + + clone_info.disk_offset = disko; + clone_info.disk_len = diskl; + clone_info.data_offset = datao; + clone_info.data_len = datal; + clone_info.file_offset = new_key.offset; + clone_info.extent_buf = buf; + clone_info.is_new_extent = false; + clone_info.update_times = !no_time_update; + ret = btrfs_replace_file_extents(inode, path, + drop_start, new_key.offset + datal - 1, + &clone_info, &trans); + if (ret) + goto out; + } else { + ASSERT(type == BTRFS_FILE_EXTENT_INLINE); + /* + * Inline extents always have to start at file offset 0 + * and can never be bigger then the sector size. We can + * never clone only parts of an inline extent, since all + * reflink operations must start at a sector size aligned + * offset, and the length must be aligned too or end at + * the i_size (which implies the whole inlined data). + */ + ASSERT(key.offset == 0); + ASSERT(datal <= fs_info->sectorsize); + if (WARN_ON(type != BTRFS_FILE_EXTENT_INLINE) || + WARN_ON(key.offset != 0) || + WARN_ON(datal > fs_info->sectorsize)) { + ret = -EUCLEAN; + goto out; + } + + ret = clone_copy_inline_extent(inode, path, &new_key, + drop_start, datal, size, + comp, buf, &trans); + if (ret) + goto out; + } + + btrfs_release_path(path); + + /* + * Whenever we share an extent we update the last_reflink_trans + * of each inode to the current transaction. This is needed to + * make sure fsync does not log multiple checksum items with + * overlapping ranges (because some extent items might refer + * only to sections of the original extent). For the destination + * inode we do this regardless of the generation of the extents + * or even if they are inline extents or explicit holes, to make + * sure a full fsync does not skip them. For the source inode, + * we only need to update last_reflink_trans in case it's a new + * extent that is not a hole or an inline extent, to deal with + * the checksums problem on fsync. + */ + if (extent_gen == trans->transid && disko > 0) + src->last_reflink_trans = trans->transid; + + inode->last_reflink_trans = trans->transid; + + last_dest_end = ALIGN(new_key.offset + datal, + fs_info->sectorsize); + ret = clone_finish_inode_update(trans, inode, last_dest_end, + destoff, olen, no_time_update); + if (ret) + goto out; + if (new_key.offset + datal >= destoff + len) + break; + + btrfs_release_path(path); + key.offset = prev_extent_end; + + if (fatal_signal_pending(current)) { + ret = -EINTR; + goto out; + } + + cond_resched(); + } + ret = 0; + + if (last_dest_end < destoff + len) { + /* + * We have an implicit hole that fully or partially overlaps our + * cloning range at its end. This means that we either have the + * NO_HOLES feature enabled or the implicit hole happened due to + * mixing buffered and direct IO writes against this file. + */ + btrfs_release_path(path); + + /* + * When using NO_HOLES and we are cloning a range that covers + * only a hole (no extents) into a range beyond the current + * i_size, punching a hole in the target range will not create + * an extent map defining a hole, because the range starts at or + * beyond current i_size. If the file previously had an i_size + * greater than the new i_size set by this clone operation, we + * need to make sure the next fsync is a full fsync, so that it + * detects and logs a hole covering a range from the current + * i_size to the new i_size. If the clone range covers extents, + * besides a hole, then we know the full sync flag was already + * set by previous calls to btrfs_replace_file_extents() that + * replaced file extent items. + */ + if (last_dest_end >= i_size_read(&inode->vfs_inode)) + btrfs_set_inode_full_sync(inode); + + ret = btrfs_replace_file_extents(inode, path, + last_dest_end, destoff + len - 1, NULL, &trans); + if (ret) + goto out; + + ret = clone_finish_inode_update(trans, inode, destoff + len, + destoff, olen, no_time_update); + } + +out: + clear_bit(BTRFS_INODE_NO_DELALLOC_FLUSH, &inode->runtime_flags); + + return ret; +} + +static void btrfs_double_mmap_lock(struct btrfs_inode *inode1, struct btrfs_inode *inode2) +{ + if (inode1 < inode2) + swap(inode1, inode2); + down_write(&inode1->i_mmap_lock); + down_write_nested(&inode2->i_mmap_lock, SINGLE_DEPTH_NESTING); +} + +static void btrfs_double_mmap_unlock(struct btrfs_inode *inode1, struct btrfs_inode *inode2) +{ + up_write(&inode1->i_mmap_lock); + up_write(&inode2->i_mmap_lock); +} + +static int btrfs_extent_same_range(struct btrfs_inode *src, u64 loff, u64 len, + struct btrfs_inode *dst, u64 dst_loff) +{ + struct extent_state *cached_state = NULL; + struct btrfs_fs_info *fs_info = src->root->fs_info; + const u32 bs = fs_info->sectorsize; + const u64 end = round_up(dst_loff + len, bs) - 1; + int ret; + + /* + * Lock destination range to serialize with concurrent readahead(), and + * we are safe from concurrency with relocation of source extents + * because we have already locked the inode's i_mmap_lock in exclusive + * mode. + */ + btrfs_lock_extent(&dst->io_tree, dst_loff, end, &cached_state); + ret = btrfs_clone(src, dst, loff, len, ALIGN(len, bs), dst_loff, true); + btrfs_unlock_extent(&dst->io_tree, dst_loff, end, &cached_state); + + btrfs_btree_balance_dirty(fs_info); + + return ret; +} + +static int btrfs_extent_same(struct btrfs_inode *src, u64 loff, u64 olen, + struct btrfs_inode *dst, u64 dst_loff) +{ + int ret = 0; + u64 i, tail_len, chunk_count; + struct btrfs_root *root_dst = dst->root; + + spin_lock(&root_dst->root_item_lock); + if (root_dst->send_in_progress) { + btrfs_warn_rl(root_dst->fs_info, +"cannot deduplicate to root %llu while send operations are using it (%d in progress)", + btrfs_root_id(root_dst), + root_dst->send_in_progress); + spin_unlock(&root_dst->root_item_lock); + return -EAGAIN; + } + root_dst->dedupe_in_progress++; + spin_unlock(&root_dst->root_item_lock); + + tail_len = olen % BTRFS_MAX_DEDUPE_LEN; + chunk_count = div_u64(olen, BTRFS_MAX_DEDUPE_LEN); + + for (i = 0; i < chunk_count; i++) { + ret = btrfs_extent_same_range(src, loff, BTRFS_MAX_DEDUPE_LEN, + dst, dst_loff); + if (ret) + goto out; + + loff += BTRFS_MAX_DEDUPE_LEN; + dst_loff += BTRFS_MAX_DEDUPE_LEN; + } + + if (tail_len > 0) + ret = btrfs_extent_same_range(src, loff, tail_len, dst, dst_loff); +out: + spin_lock(&root_dst->root_item_lock); + root_dst->dedupe_in_progress--; + spin_unlock(&root_dst->root_item_lock); + + return ret; +} + +static noinline int btrfs_clone_files(struct file *file, struct file *file_src, + u64 off, u64 olen, u64 destoff) +{ + struct extent_state *cached_state = NULL; + struct btrfs_inode *inode = BTRFS_I(file_inode(file)); + struct btrfs_inode *src = BTRFS_I(file_inode(file_src)); + struct btrfs_fs_info *fs_info = inode->root->fs_info; + const u64 src_isize = src->vfs_inode.i_size; + const u64 inode_isize = inode->vfs_inode.i_size; + int ret; + u64 len = olen; + const u32 bs = fs_info->sectorsize; + u64 end; + + /* + * VFS's generic_remap_file_range_prep() protects us from cloning the + * eof block into the middle of a file, which would result in corruption + * if the file size is not blocksize aligned. So we don't need to check + * for that case here. + */ + if (off + len == src_isize) + len = ALIGN(src_isize, bs) - off; + + if (destoff > inode_isize) { + const u64 wb_start = ALIGN_DOWN(inode_isize, bs); + + ret = btrfs_cont_expand(inode, inode_isize, destoff); + if (ret) + return ret; + /* + * We may have truncated the last block if the inode's size is + * not sector size aligned, so we need to wait for writeback to + * complete before proceeding further, otherwise we can race + * with cloning and attempt to increment a reference to an + * extent that no longer exists (writeback completed right after + * we found the previous extent covering eof and before we + * attempted to increment its reference count). + */ + ret = btrfs_wait_ordered_range(inode, wb_start, destoff - wb_start); + if (ret) + return ret; + } + + /* + * Lock destination range to serialize with concurrent readahead(), and + * we are safe from concurrency with relocation of source extents + * because we have already locked the inode's i_mmap_lock in exclusive + * mode. + */ + end = round_up(destoff + len, bs) - 1; + btrfs_lock_extent(&inode->io_tree, destoff, end, &cached_state); + ret = btrfs_clone(src, inode, off, olen, len, destoff, false); + btrfs_unlock_extent(&inode->io_tree, destoff, end, &cached_state); + if (ret < 0) + return ret; + + /* + * We may have copied an inline extent into a page of the destination + * range. So flush delalloc and wait for ordered extent completion. + * This is to ensure the invalidation below does not fail, as if for + * example it finds a dirty folio, our folio release callback + * (btrfs_release_folio()) returns false, which makes the invalidation + * return an -EBUSY error. We can't ignore such failures since they + * could come from some range other than the copied inline extent's + * destination range and we have no way to know that. + */ + ret = btrfs_wait_ordered_range(inode, destoff, len); + if (ret < 0) + return ret; + + /* + * Invalidate page cache so that future reads will see the cloned data + * immediately and not the previous data. + */ + ret = filemap_invalidate_inode(&inode->vfs_inode, false, destoff, end); + if (ret < 0) + return ret; + + btrfs_btree_balance_dirty(fs_info); + + return 0; +} + +static int btrfs_remap_file_range_prep(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + loff_t *len, unsigned int remap_flags) +{ + struct btrfs_inode *inode_in = BTRFS_I(file_inode(file_in)); + struct btrfs_inode *inode_out = BTRFS_I(file_inode(file_out)); + const u32 bs = inode_out->root->fs_info->sectorsize; + u64 wb_len; + int ret; + + if (!(remap_flags & REMAP_FILE_DEDUP)) { + struct btrfs_root *root_out = inode_out->root; + + if (btrfs_root_readonly(root_out)) + return -EROFS; + + ASSERT(inode_in->vfs_inode.i_sb == inode_out->vfs_inode.i_sb); + } + + /* Can only reflink encrypted files if both files are encrypted. */ + if (IS_ENCRYPTED(&inode_in->vfs_inode) != IS_ENCRYPTED(&inode_out->vfs_inode)) + return -EINVAL; + + /* Don't make the dst file partly checksummed */ + if ((inode_in->flags & BTRFS_INODE_NODATASUM) != + (inode_out->flags & BTRFS_INODE_NODATASUM)) { + return -EINVAL; + } + + /* + * Now that the inodes are locked, we need to start writeback ourselves + * and can not rely on the writeback from the VFS's generic helper + * generic_remap_file_range_prep() because: + * + * 1) For compression we must call filemap_fdatawrite_range() range + * twice (btrfs_fdatawrite_range() does it for us), and the generic + * helper only calls it once; + * + * 2) filemap_fdatawrite_range(), called by the generic helper only + * waits for the writeback to complete, i.e. for IO to be done, and + * not for the ordered extents to complete. We need to wait for them + * to complete so that new file extent items are in the fs tree. + */ + if (*len == 0 && !(remap_flags & REMAP_FILE_DEDUP)) + wb_len = ALIGN(inode_in->vfs_inode.i_size, bs) - ALIGN_DOWN(pos_in, bs); + else + wb_len = ALIGN(*len, bs); + + /* + * Workaround to make sure NOCOW buffered write reach disk as NOCOW. + * + * Btrfs' back references do not have a block level granularity, they + * work at the whole extent level. + * NOCOW buffered write without data space reserved may not be able + * to fall back to CoW due to lack of data space, thus could cause + * data loss. + * + * Here we take a shortcut by flushing the whole inode, so that all + * nocow write should reach disk as nocow before we increase the + * reference of the extent. We could do better by only flushing NOCOW + * data, but that needs extra accounting. + * + * Also we don't need to check ASYNC_EXTENT, as async extent will be + * CoWed anyway, not affecting nocow part. + */ + ret = filemap_flush(inode_in->vfs_inode.i_mapping); + if (ret < 0) + return ret; + + ret = btrfs_wait_ordered_range(inode_in, ALIGN_DOWN(pos_in, bs), wb_len); + if (ret < 0) + return ret; + ret = btrfs_wait_ordered_range(inode_out, ALIGN_DOWN(pos_out, bs), wb_len); + if (ret < 0) + return ret; + + return generic_remap_file_range_prep(file_in, pos_in, file_out, pos_out, + len, remap_flags); +} + +static bool file_sync_write(const struct file *file) +{ + if (file->f_flags & (__O_SYNC | O_DSYNC)) + return true; + if (IS_SYNC(file_inode(file))) + return true; + + return false; +} + +loff_t btrfs_remap_file_range(struct file *src_file, loff_t off, + struct file *dst_file, loff_t destoff, loff_t len, + unsigned int remap_flags) +{ + struct btrfs_inode *src_inode = BTRFS_I(file_inode(src_file)); + struct btrfs_inode *dst_inode = BTRFS_I(file_inode(dst_file)); + bool same_inode = dst_inode == src_inode; + int ret; + + if (btrfs_is_shutdown(src_inode->root->fs_info)) + return -EIO; + + if (remap_flags & ~(REMAP_FILE_DEDUP | REMAP_FILE_ADVISORY)) + return -EINVAL; + + if (same_inode) { + btrfs_inode_lock(src_inode, BTRFS_ILOCK_MMAP); + } else { + lock_two_nondirectories(&src_inode->vfs_inode, &dst_inode->vfs_inode); + btrfs_double_mmap_lock(src_inode, dst_inode); + } + + ret = btrfs_remap_file_range_prep(src_file, off, dst_file, destoff, + &len, remap_flags); + if (ret < 0 || len == 0) + goto out_unlock; + + if (remap_flags & REMAP_FILE_DEDUP) + ret = btrfs_extent_same(src_inode, off, len, dst_inode, destoff); + else + ret = btrfs_clone_files(dst_file, src_file, off, len, destoff); + +out_unlock: + if (same_inode) { + btrfs_inode_unlock(src_inode, BTRFS_ILOCK_MMAP); + } else { + btrfs_double_mmap_unlock(src_inode, dst_inode); + unlock_two_nondirectories(&src_inode->vfs_inode, + &dst_inode->vfs_inode); + } + + /* + * If either the source or the destination file was opened with O_SYNC, + * O_DSYNC or has the S_SYNC attribute, fsync both the destination and + * source files/ranges, so that after a successful return (0) followed + * by a power failure results in the reflinked data to be readable from + * both files/ranges. + */ + if (ret == 0 && len > 0 && + (file_sync_write(src_file) || file_sync_write(dst_file))) { + ret = btrfs_sync_file(src_file, off, off + len - 1, 0); + if (ret == 0) + ret = btrfs_sync_file(dst_file, destoff, + destoff + len - 1, 0); + } + + return ret < 0 ? ret : len; +} diff --git a/fs/btrfs/reflink.h b/fs/btrfs/reflink.h new file mode 100644 index 000000000..1e291f7d8 --- /dev/null +++ b/fs/btrfs/reflink.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_REFLINK_H +#define BTRFS_REFLINK_H + +#include + +struct file; + +loff_t btrfs_remap_file_range(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + loff_t len, unsigned int remap_flags); + +#endif /* BTRFS_REFLINK_H */ diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c new file mode 100644 index 000000000..da54db75e --- /dev/null +++ b/fs/btrfs/relocation.c @@ -0,0 +1,6283 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2009 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "ctree.h" +#include "disk-io.h" +#include "transaction.h" +#include "volumes.h" +#include "locking.h" +#include "btrfs_inode.h" +#include "async-thread.h" +#include "free-space-cache.h" +#include "qgroup.h" +#include "print-tree.h" +#include "delalloc-space.h" +#include "block-group.h" +#include "backref.h" +#include "misc.h" +#include "subpage.h" +#include "zoned.h" +#include "inode-item.h" +#include "space-info.h" +#include "fs.h" +#include "accessors.h" +#include "extent-tree.h" +#include "root-tree.h" +#include "file-item.h" +#include "relocation.h" +#include "super.h" +#include "tree-checker.h" +#include "raid-stripe-tree.h" +#include "free-space-tree.h" + +/* + * Relocation overview + * + * [What does relocation do] + * + * The objective of relocation is to relocate all extents of the target block + * group to other block groups. + * This is utilized by resize (shrink only), profile converting, compacting + * space, or balance routine to spread chunks over devices. + * + * Before | After + * ------------------------------------------------------------------ + * BG A: 10 data extents | BG A: deleted + * BG B: 2 data extents | BG B: 10 data extents (2 old + 8 relocated) + * BG C: 1 extents | BG C: 3 data extents (1 old + 2 relocated) + * + * [How does relocation work] + * + * 1. Mark the target block group read-only + * New extents won't be allocated from the target block group. + * + * 2.1 Record each extent in the target block group + * To build a proper map of extents to be relocated. + * + * 2.2 Build data reloc tree and reloc trees + * Data reloc tree will contain an inode, recording all newly relocated + * data extents. + * There will be only one data reloc tree for one data block group. + * + * Reloc tree will be a special snapshot of its source tree, containing + * relocated tree blocks. + * Each tree referring to a tree block in target block group will get its + * reloc tree built. + * + * 2.3 Swap source tree with its corresponding reloc tree + * Each involved tree only refers to new extents after swap. + * + * 3. Cleanup reloc trees and data reloc tree. + * As old extents in the target block group are still referenced by reloc + * trees, we need to clean them up before really freeing the target block + * group. + * + * The main complexity is in steps 2.2 and 2.3. + * + * The entry point of relocation is relocate_block_group() function. + */ + +#define RELOCATION_RESERVED_NODES 256 +/* + * map address of tree root to tree + */ +struct mapping_node { + union { + /* Use rb_simple_node for search/insert */ + struct { + struct rb_node rb_node; + u64 bytenr; + }; + + struct rb_simple_node simple_node; + }; + void *data; +}; + +struct mapping_tree { + struct rb_root rb_root; + spinlock_t lock; +}; + +/* + * present a tree block to process + */ +struct tree_block { + union { + /* Use rb_simple_node for search/insert */ + struct { + struct rb_node rb_node; + u64 bytenr; + }; + + struct rb_simple_node simple_node; + }; + u64 owner; + struct btrfs_key key; + u8 level; + bool key_ready; +}; + +#define MAX_EXTENTS 128 + +struct file_extent_cluster { + u64 start; + u64 end; + u64 boundary[MAX_EXTENTS]; + unsigned int nr; + u64 owning_root; +}; + +/* Stages of data relocation. */ +enum reloc_stage { + MOVE_DATA_EXTENTS, + UPDATE_DATA_PTRS +}; + +struct reloc_control { + /* block group to relocate */ + struct btrfs_block_group *block_group; + /* extent tree */ + struct btrfs_root *extent_root; + /* inode for moving data */ + struct inode *data_inode; + + struct btrfs_block_rsv *block_rsv; + + struct btrfs_backref_cache backref_cache; + + struct file_extent_cluster cluster; + /* tree blocks have been processed */ + struct extent_io_tree processed_blocks; + /* map start of tree root to corresponding reloc tree */ + struct mapping_tree reloc_root_tree; + /* list of reloc trees */ + struct list_head reloc_roots; + /* list of subvolume trees that get relocated */ + struct list_head dirty_subvol_roots; + /* size of metadata reservation for merging reloc trees */ + u64 merging_rsv_size; + /* size of relocated tree nodes */ + u64 nodes_relocated; + /* reserved size for block group relocation*/ + u64 reserved_bytes; + + u64 search_start; + u64 extents_found; + + enum reloc_stage stage; + bool create_reloc_tree; + bool merge_reloc_tree; + bool found_file_extent; + + refcount_t refs; +}; + +static struct reloc_control *get_reloc_control(struct btrfs_fs_info *fs_info) +{ + struct reloc_control *rc; + + /* Quick path, avoid lock contention on fs_info->reloc_ctl_lock. */ + if (!data_race(fs_info->reloc_ctl)) + return NULL; + + spin_lock(&fs_info->reloc_ctl_lock); + rc = fs_info->reloc_ctl; + if (rc) + refcount_inc(&rc->refs); + spin_unlock(&fs_info->reloc_ctl_lock); + + return rc; +} + +static void __del_reloc_root(struct btrfs_root *root); + +static noinline_for_stack void free_reloc_roots(struct list_head *list) +{ + struct btrfs_root *reloc_root, *tmp; + + list_for_each_entry_safe(reloc_root, tmp, list, root_list) + __del_reloc_root(reloc_root); +} + +static void put_reloc_control(struct reloc_control *rc) +{ + if (refcount_dec_and_test(&rc->refs)) { + struct mapping_node *node, *tmp; + + if (rc->extent_root) + ASSERT(rc->extent_root->fs_info->reloc_ctl != rc); + + free_reloc_roots(&rc->reloc_roots); + rbtree_postorder_for_each_entry_safe(node, tmp, + &rc->reloc_root_tree.rb_root, + rb_node) + kfree(node); + + if (rc->block_group) + btrfs_put_block_group(rc->block_group); + + kfree(rc); + } +} + +/* Helper to delete the 'address of tree root -> reloc tree' mapping. */ +static void __del_reloc_root(struct btrfs_root *root) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct rb_node *rb_node; + struct mapping_node AUTO_KFREE(node); + struct reloc_control *rc; + bool put_ref = false; + + rc = get_reloc_control(fs_info); + if (rc && root->node) { + spin_lock(&rc->reloc_root_tree.lock); + rb_node = rb_simple_search(&rc->reloc_root_tree.rb_root, + root->commit_root->start); + if (rb_node) { + node = rb_entry(rb_node, struct mapping_node, rb_node); + rb_erase(&node->rb_node, &rc->reloc_root_tree.rb_root); + RB_CLEAR_NODE(&node->rb_node); + } + spin_unlock(&rc->reloc_root_tree.lock); + ASSERT(!node || (struct btrfs_root *)node->data == root); + } + + /* + * We only put the reloc root here if it's on the list. There's a lot + * of places where the pattern is to splice the rc->reloc_roots, process + * the reloc roots, and then add the reloc root back onto + * rc->reloc_roots. If we call __del_reloc_root while it's off of the + * list we don't want the reference being dropped, because the guy + * messing with the list is in charge of the reference. + */ + spin_lock(&fs_info->trans_lock); + if (!list_empty(&root->root_list)) { + put_ref = true; + list_del_init(&root->root_list); + } + spin_unlock(&fs_info->trans_lock); + if (put_ref) + btrfs_put_root(root); + if (rc) + put_reloc_control(rc); +} + +static void mark_block_processed(struct reloc_control *rc, + struct btrfs_backref_node *node) +{ + u32 blocksize; + + if (node->level == 0 || + in_range(node->bytenr, rc->block_group->start, + rc->block_group->length)) { + blocksize = rc->extent_root->fs_info->nodesize; + btrfs_set_extent_bit(&rc->processed_blocks, node->bytenr, + node->bytenr + blocksize - 1, EXTENT_DIRTY, + NULL); + } + node->processed = 1; +} + +/* + * walk up backref nodes until reach node presents tree root + */ +static struct btrfs_backref_node *walk_up_backref( + struct btrfs_backref_node *node, + struct btrfs_backref_edge *edges[], int *index) +{ + struct btrfs_backref_edge *edge; + int idx = *index; + + while (!list_empty(&node->upper)) { + edge = list_first_entry(&node->upper, struct btrfs_backref_edge, + list[LOWER]); + edges[idx++] = edge; + node = edge->node[UPPER]; + } + BUG_ON(node->detached); + *index = idx; + return node; +} + +/* + * walk down backref nodes to find start of next reference path + */ +static struct btrfs_backref_node *walk_down_backref( + struct btrfs_backref_edge *edges[], int *index) +{ + struct btrfs_backref_edge *edge; + struct btrfs_backref_node *lower; + int idx = *index; + + while (idx > 0) { + edge = edges[idx - 1]; + lower = edge->node[LOWER]; + if (list_is_last(&edge->list[LOWER], &lower->upper)) { + idx--; + continue; + } + edge = list_first_entry(&edge->list[LOWER], struct btrfs_backref_edge, + list[LOWER]); + edges[idx - 1] = edge; + *index = idx; + return edge->node[UPPER]; + } + *index = 0; + return NULL; +} + +static bool reloc_root_is_dead(const struct btrfs_root *root) +{ + if (test_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state)) + return true; + /* + * Pairs with set_bit/clear_bit in clear_reloc_root() and + * btrfs_update_reloc_root(). We need to see the updated bit before + * trying to access root->reloc_root in our callers. + */ + smp_rmb(); + + return false; +} + +/* + * Check if this subvolume tree has valid reloc tree. + * + * Reloc tree after swap is considered dead, thus not considered as valid. + * This is enough for most callers, as they don't distinguish dead reloc root + * from no reloc root. But btrfs_should_ignore_reloc_root() below is a + * special case. + */ +static bool have_reloc_root(const struct btrfs_root *root) +{ + if (reloc_root_is_dead(root)) + return false; + if (!root->reloc_root) + return false; + return true; +} + +bool btrfs_should_ignore_reloc_root(const struct btrfs_root *root) +{ + struct btrfs_root *reloc_root; + + if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) + return false; + + /* This root has been merged with its reloc tree, we can ignore it */ + if (reloc_root_is_dead(root)) + return true; + + reloc_root = root->reloc_root; + if (!reloc_root) + return false; + + if (btrfs_header_generation(reloc_root->commit_root) == + root->fs_info->running_transaction->transid) + return false; + /* + * If there is reloc tree and it was created in previous transaction + * backref lookup can find the reloc tree, so backref node for the fs + * tree root is useless for relocation. + */ + return true; +} + +/* + * find reloc tree by address of tree root + */ +struct btrfs_root *find_reloc_root(struct btrfs_fs_info *fs_info, u64 bytenr) +{ + struct reloc_control *rc = fs_info->reloc_ctl; + struct rb_node *rb_node; + struct mapping_node *node; + struct btrfs_root *root = NULL; + + ASSERT(rc); + spin_lock(&rc->reloc_root_tree.lock); + rb_node = rb_simple_search(&rc->reloc_root_tree.rb_root, bytenr); + if (rb_node) { + node = rb_entry(rb_node, struct mapping_node, rb_node); + root = node->data; + } + spin_unlock(&rc->reloc_root_tree.lock); + return btrfs_grab_root(root); +} + +/* + * For useless nodes, do two major clean ups: + * + * - Cleanup the children edges and nodes + * If child node is also orphan (no parent) during cleanup, then the child + * node will also be cleaned up. + * + * - Freeing up leaves (level 0), keeps nodes detached + * For nodes, the node is still cached as "detached" + * + * Return false if @node is not in the @useless_nodes list. + * Return true if @node is in the @useless_nodes list. + */ +static bool handle_useless_nodes(struct reloc_control *rc, + struct btrfs_backref_node *node) +{ + struct btrfs_backref_cache *cache = &rc->backref_cache; + struct list_head *useless_node = &cache->useless_node; + bool ret = false; + + while (!list_empty(useless_node)) { + struct btrfs_backref_node *cur; + + cur = list_first_entry(useless_node, struct btrfs_backref_node, + list); + list_del_init(&cur->list); + + /* Only tree root nodes can be added to @useless_nodes */ + ASSERT(list_empty(&cur->upper)); + + if (cur == node) + ret = true; + + /* Cleanup the lower edges */ + while (!list_empty(&cur->lower)) { + struct btrfs_backref_edge *edge; + struct btrfs_backref_node *lower; + + edge = list_first_entry(&cur->lower, struct btrfs_backref_edge, + list[UPPER]); + list_del(&edge->list[UPPER]); + list_del(&edge->list[LOWER]); + lower = edge->node[LOWER]; + btrfs_backref_free_edge(cache, edge); + + /* Child node is also orphan, queue for cleanup */ + if (list_empty(&lower->upper)) + list_add(&lower->list, useless_node); + } + /* Mark this block processed for relocation */ + mark_block_processed(rc, cur); + + /* + * Backref nodes for tree leaves are deleted from the cache. + * Backref nodes for upper level tree blocks are left in the + * cache to avoid unnecessary backref lookup. + */ + if (cur->level > 0) { + cur->detached = 1; + } else { + rb_erase(&cur->rb_node, &cache->rb_root); + btrfs_backref_free_node(cache, cur); + } + } + return ret; +} + +/* + * Build backref tree for a given tree block. Root of the backref tree + * corresponds the tree block, leaves of the backref tree correspond roots of + * b-trees that reference the tree block. + * + * The basic idea of this function is check backrefs of a given block to find + * upper level blocks that reference the block, and then check backrefs of + * these upper level blocks recursively. The recursion stops when tree root is + * reached or backrefs for the block is cached. + * + * NOTE: if we find that backrefs for a block are cached, we know backrefs for + * all upper level blocks that directly/indirectly reference the block are also + * cached. + */ +static noinline_for_stack struct btrfs_backref_node *build_backref_tree( + struct btrfs_trans_handle *trans, + struct reloc_control *rc, struct btrfs_key *node_key, + int level, u64 bytenr) +{ + struct btrfs_backref_iter iter; + struct btrfs_backref_cache *cache = &rc->backref_cache; + /* For searching parent of TREE_BLOCK_REF */ + struct btrfs_path *path; + struct btrfs_backref_node *cur; + struct btrfs_backref_node *node = NULL; + struct btrfs_backref_edge *edge; + int ret; + + ret = btrfs_backref_iter_init(&iter); + if (ret < 0) + return ERR_PTR(ret); + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto out; + } + + node = btrfs_backref_alloc_node(cache, bytenr, level); + if (!node) { + ret = -ENOMEM; + goto out; + } + + cur = node; + + /* Breadth-first search to build backref cache */ + do { + ret = btrfs_backref_add_tree_node(trans, cache, path, &iter, + node_key, cur); + if (ret < 0) + goto out; + + edge = list_first_entry_or_null(&cache->pending_edge, + struct btrfs_backref_edge, list[UPPER]); + /* + * The pending list isn't empty, take the first block to + * process + */ + if (edge) { + list_del_init(&edge->list[UPPER]); + cur = edge->node[UPPER]; + } + } while (edge); + + /* Finish the upper linkage of newly added edges/nodes */ + ret = btrfs_backref_finish_upper_links(cache, node); + if (ret < 0) + goto out; + + if (handle_useless_nodes(rc, node)) + node = NULL; +out: + btrfs_free_path(iter.path); + btrfs_free_path(path); + if (ret) { + btrfs_backref_error_cleanup(cache, node); + return ERR_PTR(ret); + } + ASSERT(!node || !node->detached); + ASSERT(list_empty(&cache->useless_node) && + list_empty(&cache->pending_edge)); + return node; +} + +/* + * helper to add 'address of tree root -> reloc tree' mapping + */ +static int __add_reloc_root(struct btrfs_root *root, struct reloc_control *rc) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct rb_node *rb_node; + struct mapping_node *node; + + node = kmalloc_obj(*node, GFP_NOFS); + if (!node) + return -ENOMEM; + + node->bytenr = root->commit_root->start; + node->data = root; + + spin_lock(&rc->reloc_root_tree.lock); + rb_node = rb_simple_insert(&rc->reloc_root_tree.rb_root, &node->simple_node); + spin_unlock(&rc->reloc_root_tree.lock); + if (rb_node) { + btrfs_err(fs_info, + "Duplicate root found for start=%llu while inserting into relocation tree", + node->bytenr); + kfree(node); + return -EEXIST; + } + + list_add_tail(&root->root_list, &rc->reloc_roots); + return 0; +} + +/* + * helper to update the 'address of tree root -> reloc tree' + * mapping + */ +static int __update_reloc_root(struct btrfs_root *root) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct rb_node *rb_node; + struct mapping_node *node = NULL; + struct reloc_control *rc = fs_info->reloc_ctl; + + spin_lock(&rc->reloc_root_tree.lock); + rb_node = rb_simple_search(&rc->reloc_root_tree.rb_root, + root->commit_root->start); + if (rb_node) { + node = rb_entry(rb_node, struct mapping_node, rb_node); + rb_erase(&node->rb_node, &rc->reloc_root_tree.rb_root); + } + spin_unlock(&rc->reloc_root_tree.lock); + + if (!node) + return 0; + BUG_ON((struct btrfs_root *)node->data != root); + + spin_lock(&rc->reloc_root_tree.lock); + node->bytenr = root->node->start; + rb_node = rb_simple_insert(&rc->reloc_root_tree.rb_root, &node->simple_node); + spin_unlock(&rc->reloc_root_tree.lock); + if (rb_node) + btrfs_backref_panic(fs_info, node->bytenr, -EEXIST); + return 0; +} + +static struct btrfs_root *create_reloc_root(struct btrfs_trans_handle *trans, + struct btrfs_root *root, u64 objectid) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_root *reloc_root; + struct extent_buffer *eb; + struct btrfs_root_item AUTO_KFREE(root_item); + struct btrfs_key root_key; + int ret = 0; + + root_item = kmalloc_obj(*root_item, GFP_NOFS); + if (!root_item) + return ERR_PTR(-ENOMEM); + + root_key.objectid = BTRFS_TREE_RELOC_OBJECTID; + root_key.type = BTRFS_ROOT_ITEM_KEY; + root_key.offset = objectid; + + if (btrfs_root_id(root) == objectid) { + u64 commit_root_gen; + + /* + * Relocation will wait for cleaner thread, and any half-dropped + * subvolume will be fully cleaned up at mount time. + * So here we shouldn't hit a subvolume with non-zero drop_progress. + * + * If this isn't the case, error out since it can make us attempt to + * drop references for extents that were already dropped before. + */ + if (unlikely(btrfs_disk_key_objectid(&root->root_item.drop_progress))) { + struct btrfs_key cpu_key; + + btrfs_disk_key_to_cpu(&cpu_key, &root->root_item.drop_progress); + btrfs_err(fs_info, + "cannot relocate partially dropped subvolume %llu, drop progress key " BTRFS_KEY_FMT, + objectid, BTRFS_KEY_FMT_VALUE(&cpu_key)); + return ERR_PTR(-EUCLEAN); + } + + /* called by btrfs_init_reloc_root */ + ret = btrfs_copy_root(trans, root, root->commit_root, &eb, + BTRFS_TREE_RELOC_OBJECTID); + if (ret) + return ERR_PTR(ret); + + /* + * Set the last_snapshot field to the generation of the commit + * root - like this ctree.c:btrfs_block_can_be_shared() behaves + * correctly (returns true) when the relocation root is created + * either inside the critical section of a transaction commit + * (through transaction.c:qgroup_account_snapshot()) and when + * it's created before the transaction commit is started. + */ + commit_root_gen = btrfs_header_generation(root->commit_root); + btrfs_set_root_last_snapshot(&root->root_item, commit_root_gen); + } else { + /* + * called by btrfs_reloc_post_snapshot_hook. + * the source tree is a reloc tree, all tree blocks + * modified after it was created have RELOC flag + * set in their headers. so it's OK to not update + * the 'last_snapshot'. + */ + ret = btrfs_copy_root(trans, root, root->node, &eb, + BTRFS_TREE_RELOC_OBJECTID); + if (ret) + return ERR_PTR(ret); + } + + /* + * We have changed references at this point, we must abort the + * transaction if anything fails (i.e. 'goto abort'). + */ + + memcpy(root_item, &root->root_item, sizeof(*root_item)); + btrfs_set_root_bytenr(root_item, eb->start); + btrfs_set_root_level(root_item, btrfs_header_level(eb)); + btrfs_set_root_generation(root_item, trans->transid); + + if (btrfs_root_id(root) == objectid) { + btrfs_set_root_refs(root_item, 0); + memset(&root_item->drop_progress, 0, + sizeof(struct btrfs_disk_key)); + btrfs_set_root_drop_level(root_item, 0); + } + + btrfs_tree_unlock(eb); + free_extent_buffer(eb); + + ret = btrfs_insert_root(trans, fs_info->tree_root, + &root_key, root_item); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ERR_PTR(ret); + } + + reloc_root = btrfs_read_tree_root(fs_info->tree_root, &root_key); + if (IS_ERR(reloc_root)) { + btrfs_abort_transaction(trans, PTR_ERR(reloc_root)); + return ERR_CAST(reloc_root); + } + set_bit(BTRFS_ROOT_SHAREABLE, &reloc_root->state); + btrfs_set_root_last_trans(reloc_root, trans->transid); + return reloc_root; +} + +/* + * create reloc tree for a given fs tree. reloc tree is just a + * snapshot of the fs tree with special root objectid. + * + * The reloc_root comes out of here with two references, one for + * root->reloc_root, and another for being on the rc->reloc_roots list. + */ +int btrfs_init_reloc_root(struct btrfs_trans_handle *trans, + struct btrfs_root *root) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_root *reloc_root; + struct reloc_control *rc; + struct btrfs_block_rsv *rsv; + bool clear_rsv = false; + int ret = 0; + + rc = get_reloc_control(fs_info); + if (!rc) + return 0; + + /* + * The subvolume has reloc tree but the swap is finished, no need to + * create/update the dead reloc tree + */ + if (reloc_root_is_dead(root)) + goto out; + + /* + * This is subtle but important. We do not do + * record_root_in_transaction for reloc roots, instead we record their + * corresponding fs root, and then here we update the last trans for the + * reloc root. This means that we have to do this for the entire life + * of the reloc root, regardless of which stage of the relocation we are + * in. + */ + if (root->reloc_root) { + btrfs_set_root_last_trans(root->reloc_root, trans->transid); + goto out; + } + + /* + * We are merging reloc roots, we do not need new reloc trees. Also + * reloc trees never need their own reloc tree. + */ + if (!rc->create_reloc_tree || btrfs_root_id(root) == BTRFS_TREE_RELOC_OBJECTID) + goto out; + + if (!trans->reloc_reserved) { + rsv = trans->block_rsv; + trans->block_rsv = rc->block_rsv; + clear_rsv = true; + } + reloc_root = create_reloc_root(trans, root, btrfs_root_id(root)); + if (clear_rsv) + trans->block_rsv = rsv; + if (IS_ERR(reloc_root)) { + ret = PTR_ERR(reloc_root); + goto out; + } + + ret = __add_reloc_root(reloc_root, rc); + ASSERT(ret != -EEXIST); + if (ret) { + /* Pairs with create_reloc_root */ + btrfs_put_root(reloc_root); + goto out; + } + root->reloc_root = btrfs_grab_root(reloc_root); +out: + put_reloc_control(rc); + + return ret; +} + +/* + * update root item of reloc tree + */ +int btrfs_update_reloc_root(struct btrfs_trans_handle *trans, + struct btrfs_root *root) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_root *reloc_root; + struct btrfs_root_item *root_item; + struct reloc_control *rc; + int ret; + + if (!have_reloc_root(root)) + return 0; + + reloc_root = root->reloc_root; + root_item = &reloc_root->root_item; + + /* + * We are probably ok here, but __del_reloc_root() will drop its ref of + * the root. We have the ref for root->reloc_root, but just in case + * hold it while we update the reloc root. + */ + btrfs_grab_root(reloc_root); + + rc = get_reloc_control(fs_info); + /* root->reloc_root will stay until current relocation finished */ + if (rc && rc->merge_reloc_tree && btrfs_root_refs(root_item) == 0) { + set_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state); + /* + * Mark the tree as dead before we change reloc_root so + * have_reloc_root will not touch it from now on. + */ + smp_wmb(); + __del_reloc_root(reloc_root); + } + + if (reloc_root->commit_root != reloc_root->node) { + __update_reloc_root(reloc_root); + btrfs_set_root_node(root_item, reloc_root->node); + free_extent_buffer(reloc_root->commit_root); + reloc_root->commit_root = btrfs_root_node(reloc_root); + } + + ret = btrfs_update_root(trans, fs_info->tree_root, + &reloc_root->root_key, root_item); + btrfs_put_root(reloc_root); + if (rc) + put_reloc_control(rc); + + return ret; +} + +/* + * get new location of data + */ +static int get_new_location(struct inode *reloc_inode, u64 *new_bytenr, + u64 bytenr, u64 num_bytes) +{ + struct btrfs_root *root = BTRFS_I(reloc_inode)->root; + struct btrfs_fs_info *fs_info = root->fs_info; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_file_extent_item *fi; + struct extent_buffer *leaf; + int ret; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + bytenr -= BTRFS_I(reloc_inode)->reloc_block_group_start; + ret = btrfs_lookup_file_extent(NULL, root, path, + btrfs_ino(BTRFS_I(reloc_inode)), bytenr, 0); + if (ret < 0) + return ret; + if (ret > 0) + return -ENOENT; + + leaf = path->nodes[0]; + fi = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_file_extent_item); + if (unlikely(btrfs_file_extent_type(leaf, fi) == BTRFS_FILE_EXTENT_INLINE)) { + btrfs_print_leaf(leaf); + btrfs_err(fs_info, + "unexpected inline file extent item for data reloc inode %llu key offset %llu", + btrfs_ino(BTRFS_I(reloc_inode)), bytenr); + return -EUCLEAN; + } + + /* + * The cluster-boundary key searched above is always written by + * relocation with offset 0: either by insert_prealloc_file_extent() + * (memsets the stack item to 0) or by the front portion of a partial + * writeback (offset=0 by construction). A non-zero value here means + * the on-disk leaf does not match what relocation wrote, i.e. + * corruption. The other encoding fields are caught earlier by + * tree-checker's check_extent_data_item(). + */ + if (unlikely(btrfs_file_extent_offset(leaf, fi))) { + btrfs_print_leaf(leaf); + btrfs_err(fs_info, +"unexpected non-zero offset in file extent item for data reloc inode %llu key offset %llu offset %llu", + btrfs_ino(BTRFS_I(reloc_inode)), bytenr, + btrfs_file_extent_offset(leaf, fi)); + return -EUCLEAN; + } + + if (num_bytes != btrfs_file_extent_disk_num_bytes(leaf, fi)) + return -EINVAL; + + *new_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi); + return 0; +} + +/* + * update file extent items in the tree leaf to point to + * the new locations. + */ +static noinline_for_stack +int replace_file_extents(struct btrfs_trans_handle *trans, + struct reloc_control *rc, + struct btrfs_root *root, + struct extent_buffer *leaf) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_key key; + struct btrfs_file_extent_item *fi; + struct btrfs_inode *inode = NULL; + u64 parent; + u64 bytenr; + u64 new_bytenr = 0; + u64 num_bytes; + u64 end; + u32 nritems; + u32 i; + int ret = 0; + bool first = true; + + if (rc->stage != UPDATE_DATA_PTRS) + return 0; + + /* reloc trees always use full backref */ + if (btrfs_root_id(root) == BTRFS_TREE_RELOC_OBJECTID) + parent = leaf->start; + else + parent = 0; + + nritems = btrfs_header_nritems(leaf); + for (i = 0; i < nritems; i++) { + struct btrfs_ref ref = { 0 }; + + cond_resched(); + btrfs_item_key_to_cpu(leaf, &key, i); + if (key.type != BTRFS_EXTENT_DATA_KEY) + continue; + fi = btrfs_item_ptr(leaf, i, struct btrfs_file_extent_item); + if (btrfs_file_extent_type(leaf, fi) == + BTRFS_FILE_EXTENT_INLINE) + continue; + bytenr = btrfs_file_extent_disk_bytenr(leaf, fi); + num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi); + if (bytenr == 0) + continue; + if (!in_range(bytenr, rc->block_group->start, + rc->block_group->length)) + continue; + + /* + * if we are modifying block in fs tree, wait for read_folio + * to complete and drop the extent cache + */ + if (btrfs_root_id(root) != BTRFS_TREE_RELOC_OBJECTID) { + if (first) { + inode = btrfs_find_first_inode(root, key.objectid); + first = false; + } else if (inode && btrfs_ino(inode) < key.objectid) { + btrfs_add_delayed_iput(inode); + inode = btrfs_find_first_inode(root, key.objectid); + } + if (inode && btrfs_ino(inode) == key.objectid) { + struct extent_state *cached_state = NULL; + + end = key.offset + + btrfs_file_extent_num_bytes(leaf, fi); + WARN_ON(!IS_ALIGNED(key.offset, + fs_info->sectorsize)); + WARN_ON(!IS_ALIGNED(end, fs_info->sectorsize)); + end--; + /* Take mmap lock to serialize with reflinks. */ + if (!down_read_trylock(&inode->i_mmap_lock)) + continue; + ret = btrfs_try_lock_extent(&inode->io_tree, key.offset, + end, &cached_state); + if (!ret) { + up_read(&inode->i_mmap_lock); + continue; + } + + btrfs_drop_extent_map_range(inode, key.offset, end, true); + btrfs_unlock_extent(&inode->io_tree, key.offset, end, + &cached_state); + up_read(&inode->i_mmap_lock); + } + } + + ret = get_new_location(rc->data_inode, &new_bytenr, + bytenr, num_bytes); + if (ret) { + /* + * Don't have to abort since we've not changed anything + * in the file extent yet. + */ + break; + } + + btrfs_set_file_extent_disk_bytenr(leaf, fi, new_bytenr); + + key.offset -= btrfs_file_extent_offset(leaf, fi); + ref.action = BTRFS_ADD_DELAYED_REF; + ref.bytenr = new_bytenr; + ref.num_bytes = num_bytes; + ref.parent = parent; + ref.owning_root = btrfs_root_id(root); + ref.ref_root = btrfs_header_owner(leaf); + btrfs_init_data_ref(&ref, key.objectid, key.offset, + btrfs_root_id(root), false); + ret = btrfs_inc_extent_ref(trans, &ref); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + break; + } + + ref.action = BTRFS_DROP_DELAYED_REF; + ref.bytenr = bytenr; + ref.num_bytes = num_bytes; + ref.parent = parent; + ref.owning_root = btrfs_root_id(root); + ref.ref_root = btrfs_header_owner(leaf); + btrfs_init_data_ref(&ref, key.objectid, key.offset, + btrfs_root_id(root), false); + ret = btrfs_free_extent(trans, &ref); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + break; + } + } + if (inode) + btrfs_add_delayed_iput(inode); + return ret; +} + +static noinline_for_stack int memcmp_node_keys(const struct extent_buffer *eb, + int slot, const struct btrfs_path *path, + int level) +{ + struct btrfs_disk_key key1; + struct btrfs_disk_key key2; + btrfs_node_key(eb, &key1, slot); + btrfs_node_key(path->nodes[level], &key2, path->slots[level]); + return memcmp(&key1, &key2, sizeof(key1)); +} + +/* + * try to replace tree blocks in fs tree with the new blocks + * in reloc tree. tree blocks haven't been modified since the + * reloc tree was create can be replaced. + * + * if a block was replaced, level of the block + 1 is returned. + * if no block got replaced, 0 is returned. if there are other + * errors, a negative error number is returned. + */ +static noinline_for_stack +int replace_path(struct btrfs_trans_handle *trans, struct reloc_control *rc, + struct btrfs_root *dest, struct btrfs_root *src, + struct btrfs_path *path, struct btrfs_key *next_key, + int lowest_level, int max_level) +{ + struct btrfs_fs_info *fs_info = dest->fs_info; + struct extent_buffer *eb; + struct extent_buffer *parent; + struct btrfs_ref ref = { 0 }; + struct btrfs_key key; + u64 old_bytenr; + u64 new_bytenr; + u64 old_ptr_gen; + u64 new_ptr_gen; + u64 last_snapshot; + u32 blocksize; + bool cow = false; + int level; + int ret; + int slot; + + ASSERT(btrfs_root_id(src) == BTRFS_TREE_RELOC_OBJECTID); + ASSERT(btrfs_root_id(dest) != BTRFS_TREE_RELOC_OBJECTID); + + last_snapshot = btrfs_root_last_snapshot(&src->root_item); +again: + slot = path->slots[lowest_level]; + btrfs_node_key_to_cpu(path->nodes[lowest_level], &key, slot); + + eb = btrfs_lock_root_node(dest); + level = btrfs_header_level(eb); + + if (level < lowest_level) { + btrfs_tree_unlock(eb); + free_extent_buffer(eb); + return 0; + } + + if (cow) { + ret = btrfs_cow_block(trans, dest, eb, NULL, 0, &eb, + BTRFS_NESTING_COW); + if (ret) { + btrfs_tree_unlock(eb); + free_extent_buffer(eb); + return ret; + } + } + + if (next_key) { + next_key->objectid = (u64)-1; + next_key->type = (u8)-1; + next_key->offset = (u64)-1; + } + + parent = eb; + while (1) { + level = btrfs_header_level(parent); + ASSERT(level >= lowest_level); + + ret = btrfs_bin_search(parent, 0, &key, &slot); + if (ret < 0) + break; + if (ret && slot > 0) + slot--; + + if (next_key && slot + 1 < btrfs_header_nritems(parent)) + btrfs_node_key_to_cpu(parent, next_key, slot + 1); + + old_bytenr = btrfs_node_blockptr(parent, slot); + blocksize = fs_info->nodesize; + old_ptr_gen = btrfs_node_ptr_generation(parent, slot); + + if (level <= max_level) { + eb = path->nodes[level]; + new_bytenr = btrfs_node_blockptr(eb, + path->slots[level]); + new_ptr_gen = btrfs_node_ptr_generation(eb, + path->slots[level]); + } else { + new_bytenr = 0; + new_ptr_gen = 0; + } + + if (WARN_ON(new_bytenr > 0 && new_bytenr == old_bytenr)) { + ret = level; + break; + } + + if (new_bytenr == 0 || old_ptr_gen > last_snapshot || + memcmp_node_keys(parent, slot, path, level)) { + if (level <= lowest_level) { + ret = 0; + break; + } + + eb = btrfs_read_node_slot(parent, slot); + if (IS_ERR(eb)) { + ret = PTR_ERR(eb); + break; + } + btrfs_tree_lock(eb); + if (cow) { + ret = btrfs_cow_block(trans, dest, eb, parent, + slot, &eb, + BTRFS_NESTING_COW); + if (ret) { + btrfs_tree_unlock(eb); + free_extent_buffer(eb); + break; + } + } + + btrfs_tree_unlock(parent); + free_extent_buffer(parent); + + parent = eb; + continue; + } + + if (!cow) { + btrfs_tree_unlock(parent); + free_extent_buffer(parent); + cow = true; + goto again; + } + + btrfs_node_key_to_cpu(path->nodes[level], &key, + path->slots[level]); + btrfs_release_path(path); + + path->lowest_level = level; + set_bit(BTRFS_ROOT_RESET_LOCKDEP_CLASS, &src->state); + ret = btrfs_search_slot(trans, src, &key, path, 0, 1); + clear_bit(BTRFS_ROOT_RESET_LOCKDEP_CLASS, &src->state); + path->lowest_level = 0; + if (ret) { + if (ret > 0) + ret = -ENOENT; + break; + } + + /* + * Info qgroup to trace both subtrees. + * + * We must trace both trees. + * 1) Tree reloc subtree + * If not traced, we will leak data numbers + * 2) Fs subtree + * If not traced, we will double count old data + * + * We don't scan the subtree right now, but only record + * the swapped tree blocks. + * The real subtree rescan is delayed until we have new + * CoW on the subtree root node before transaction commit. + */ + ret = btrfs_qgroup_add_swapped_blocks(dest, + rc->block_group, parent, slot, + path->nodes[level], path->slots[level], + last_snapshot); + if (ret < 0) + break; + /* + * swap blocks in fs tree and reloc tree. + */ + btrfs_set_node_blockptr(parent, slot, new_bytenr); + btrfs_set_node_ptr_generation(parent, slot, new_ptr_gen); + + btrfs_set_node_blockptr(path->nodes[level], + path->slots[level], old_bytenr); + btrfs_set_node_ptr_generation(path->nodes[level], + path->slots[level], old_ptr_gen); + + ref.action = BTRFS_ADD_DELAYED_REF; + ref.bytenr = old_bytenr; + ref.num_bytes = blocksize; + ref.parent = path->nodes[level]->start; + ref.owning_root = btrfs_root_id(src); + ref.ref_root = btrfs_root_id(src); + btrfs_init_tree_ref(&ref, level - 1, 0, true); + ret = btrfs_inc_extent_ref(trans, &ref); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + break; + } + + ref.action = BTRFS_ADD_DELAYED_REF; + ref.bytenr = new_bytenr; + ref.num_bytes = blocksize; + ref.parent = 0; + ref.owning_root = btrfs_root_id(dest); + ref.ref_root = btrfs_root_id(dest); + btrfs_init_tree_ref(&ref, level - 1, 0, true); + ret = btrfs_inc_extent_ref(trans, &ref); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + break; + } + + /* We don't know the real owning_root, use 0. */ + ref.action = BTRFS_DROP_DELAYED_REF; + ref.bytenr = new_bytenr; + ref.num_bytes = blocksize; + ref.parent = path->nodes[level]->start; + ref.owning_root = 0; + ref.ref_root = btrfs_root_id(src); + btrfs_init_tree_ref(&ref, level - 1, 0, true); + ret = btrfs_free_extent(trans, &ref); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + break; + } + + /* We don't know the real owning_root, use 0. */ + ref.action = BTRFS_DROP_DELAYED_REF; + ref.bytenr = old_bytenr; + ref.num_bytes = blocksize; + ref.parent = 0; + ref.owning_root = 0; + ref.ref_root = btrfs_root_id(dest); + btrfs_init_tree_ref(&ref, level - 1, 0, true); + ret = btrfs_free_extent(trans, &ref); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + break; + } + + btrfs_unlock_up_safe(path, 0); + + ret = level; + break; + } + btrfs_tree_unlock(parent); + free_extent_buffer(parent); + return ret; +} + +/* + * helper to find next relocated block in reloc tree + */ +static noinline_for_stack +int walk_up_reloc_tree(struct btrfs_root *root, struct btrfs_path *path, + int *level) +{ + struct extent_buffer *eb; + int i; + u64 last_snapshot; + u32 nritems; + + last_snapshot = btrfs_root_last_snapshot(&root->root_item); + + for (i = 0; i < *level; i++) { + free_extent_buffer(path->nodes[i]); + path->nodes[i] = NULL; + } + + for (i = *level; i < BTRFS_MAX_LEVEL && path->nodes[i]; i++) { + eb = path->nodes[i]; + nritems = btrfs_header_nritems(eb); + while (path->slots[i] + 1 < nritems) { + path->slots[i]++; + if (btrfs_node_ptr_generation(eb, path->slots[i]) <= + last_snapshot) + continue; + + *level = i; + return 0; + } + free_extent_buffer(path->nodes[i]); + path->nodes[i] = NULL; + } + return 1; +} + +/* + * walk down reloc tree to find relocated block of lowest level + */ +static noinline_for_stack +int walk_down_reloc_tree(struct btrfs_root *root, struct btrfs_path *path, + int *level) +{ + struct extent_buffer *eb = NULL; + int i; + u64 ptr_gen = 0; + u64 last_snapshot; + u32 nritems; + + last_snapshot = btrfs_root_last_snapshot(&root->root_item); + + for (i = *level; i > 0; i--) { + eb = path->nodes[i]; + nritems = btrfs_header_nritems(eb); + while (path->slots[i] < nritems) { + ptr_gen = btrfs_node_ptr_generation(eb, path->slots[i]); + if (ptr_gen > last_snapshot) + break; + path->slots[i]++; + } + if (path->slots[i] >= nritems) { + if (i == *level) + break; + *level = i + 1; + return 0; + } + if (i == 1) { + *level = i; + return 0; + } + + eb = btrfs_read_node_slot(eb, path->slots[i]); + if (IS_ERR(eb)) + return PTR_ERR(eb); + BUG_ON(btrfs_header_level(eb) != i - 1); + path->nodes[i - 1] = eb; + path->slots[i - 1] = 0; + } + return 1; +} + +/* + * invalidate extent cache for file extents whose key in range of + * [min_key, max_key) + */ +static int invalidate_extent_cache(struct btrfs_root *root, + const struct btrfs_key *min_key, + const struct btrfs_key *max_key) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_inode *inode = NULL; + u64 objectid; + u64 start, end; + u64 ino; + + objectid = min_key->objectid; + while (1) { + struct extent_state *cached_state = NULL; + + cond_resched(); + if (inode) + iput(&inode->vfs_inode); + + if (objectid > max_key->objectid) + break; + + inode = btrfs_find_first_inode(root, objectid); + if (!inode) + break; + ino = btrfs_ino(inode); + + if (ino > max_key->objectid) { + iput(&inode->vfs_inode); + break; + } + + objectid = ino + 1; + if (!S_ISREG(inode->vfs_inode.i_mode)) + continue; + + if (unlikely(min_key->objectid == ino)) { + if (min_key->type > BTRFS_EXTENT_DATA_KEY) + continue; + if (min_key->type < BTRFS_EXTENT_DATA_KEY) + start = 0; + else { + start = min_key->offset; + WARN_ON(!IS_ALIGNED(start, fs_info->sectorsize)); + } + } else { + start = 0; + } + + if (unlikely(max_key->objectid == ino)) { + if (max_key->type < BTRFS_EXTENT_DATA_KEY) + continue; + if (max_key->type > BTRFS_EXTENT_DATA_KEY) { + end = (u64)-1; + } else { + if (max_key->offset == 0) + continue; + end = max_key->offset; + WARN_ON(!IS_ALIGNED(end, fs_info->sectorsize)); + end--; + } + } else { + end = (u64)-1; + } + + /* the lock_extent waits for read_folio to complete */ + btrfs_lock_extent(&inode->io_tree, start, end, &cached_state); + btrfs_drop_extent_map_range(inode, start, end, true); + btrfs_unlock_extent(&inode->io_tree, start, end, &cached_state); + } + return 0; +} + +static int find_next_key(struct btrfs_path *path, int level, + struct btrfs_key *key) + +{ + while (level < BTRFS_MAX_LEVEL) { + if (!path->nodes[level]) + break; + if (path->slots[level] + 1 < + btrfs_header_nritems(path->nodes[level])) { + btrfs_node_key_to_cpu(path->nodes[level], key, + path->slots[level] + 1); + return 0; + } + level++; + } + return 1; +} + +/* + * Insert current subvolume into reloc_control::dirty_subvol_roots + */ +static int insert_dirty_subvol(struct btrfs_trans_handle *trans, + struct reloc_control *rc, + struct btrfs_root *root) +{ + struct btrfs_root *reloc_root = root->reloc_root; + struct btrfs_root_item *reloc_root_item; + int ret; + + /* @root must be a subvolume tree root with a valid reloc tree */ + ASSERT(btrfs_root_id(root) != BTRFS_TREE_RELOC_OBJECTID); + ASSERT(reloc_root); + + reloc_root_item = &reloc_root->root_item; + memset(&reloc_root_item->drop_progress, 0, + sizeof(reloc_root_item->drop_progress)); + btrfs_set_root_drop_level(reloc_root_item, 0); + btrfs_set_root_refs(reloc_root_item, 0); + ret = btrfs_update_reloc_root(trans, root); + if (ret) + return ret; + + if (list_empty(&root->reloc_dirty_list)) { + btrfs_grab_root(root); + list_add_tail(&root->reloc_dirty_list, &rc->dirty_subvol_roots); + } + + return 0; +} + +static void clear_reloc_root(struct btrfs_root *root) +{ + root->reloc_root = NULL; + /* + * Need barrier to ensure clear_bit() only happens after + * root->reloc_root = NULL. Pairs with have_reloc_root(). + */ + smp_wmb(); + clear_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state); +} + +/* Drop the reloc trees of a relocation that is being deferred and retried. */ +static void abort_reloc_roots(struct reloc_control *rc, struct list_head *list) +{ + struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; + struct btrfs_root *reloc_root, *tmp; + + list_for_each_entry_safe(reloc_root, tmp, list, root_list) { + struct btrfs_root *root; + + root = btrfs_get_fs_root(fs_info, reloc_root->root_key.offset, false); + if (!IS_ERR(root)) { + if (root->reloc_root == reloc_root) { + clear_reloc_root(root); + btrfs_put_root(reloc_root); + } + btrfs_put_root(root); + } + + btrfs_set_root_refs(&reloc_root->root_item, 0); + memset(&reloc_root->root_item.drop_progress, 0, sizeof(struct btrfs_disk_key)); + btrfs_set_root_drop_level(&reloc_root->root_item, 0); + + list_del_init(&reloc_root->root_list); + list_add_tail(&reloc_root->reloc_dirty_list, &rc->dirty_subvol_roots); + } +} + +static int clean_dirty_subvols(struct reloc_control *rc) +{ + struct btrfs_root *root; + struct btrfs_root *next; + int ret = 0; + int ret2; + + list_for_each_entry_safe(root, next, &rc->dirty_subvol_roots, + reloc_dirty_list) { + if (btrfs_root_id(root) != BTRFS_TREE_RELOC_OBJECTID) { + /* Merged subvolume, cleanup its reloc root */ + struct btrfs_root *reloc_root = root->reloc_root; + + list_del_init(&root->reloc_dirty_list); + clear_reloc_root(root); + if (reloc_root) { + /* + * btrfs_drop_snapshot drops our ref we hold for + * ->reloc_root. If it fails however we must + * drop the ref ourselves. + */ + ret2 = btrfs_drop_snapshot(reloc_root, false, true); + if (ret2 < 0) { + btrfs_put_root(reloc_root); + if (!ret) + ret = ret2; + } + } + btrfs_put_root(root); + } else { + /* Orphan reloc tree, just clean it up */ + ret2 = btrfs_drop_snapshot(root, false, true); + if (ret2 < 0) { + btrfs_put_root(root); + if (!ret) + ret = ret2; + } + } + } + return ret; +} + +/* + * merge the relocated tree blocks in reloc tree with corresponding + * fs tree. + */ +static noinline_for_stack int merge_reloc_root(struct reloc_control *rc, + struct btrfs_root *root) +{ + struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; + struct btrfs_key key; + struct btrfs_key next_key; + struct btrfs_trans_handle *trans = NULL; + struct btrfs_root *reloc_root; + struct btrfs_root_item *root_item; + struct btrfs_path *path; + struct extent_buffer *leaf; + int reserve_level; + int level; + int max_level; + bool replaced = false; + int ret = 0; + u32 min_reserved; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + path->reada = READA_FORWARD; + + reloc_root = root->reloc_root; + root_item = &reloc_root->root_item; + + if (btrfs_disk_key_objectid(&root_item->drop_progress) == 0) { + level = btrfs_root_level(root_item); + refcount_inc(&reloc_root->node->refs); + path->nodes[level] = reloc_root->node; + path->slots[level] = 0; + } else { + btrfs_disk_key_to_cpu(&key, &root_item->drop_progress); + + level = btrfs_root_drop_level(root_item); + BUG_ON(level == 0); + path->lowest_level = level; + ret = btrfs_search_slot(NULL, reloc_root, &key, path, 0, 0); + path->lowest_level = 0; + if (ret < 0) { + btrfs_free_path(path); + return ret; + } + + btrfs_node_key_to_cpu(path->nodes[level], &next_key, + path->slots[level]); + WARN_ON(memcmp(&key, &next_key, sizeof(key))); + + btrfs_unlock_up_safe(path, 0); + } + + /* + * In merge_reloc_root(), we modify the upper level pointer to swap the + * tree blocks between reloc tree and subvolume tree. Thus for tree + * block COW, we COW at most from level 1 to root level for each tree. + * + * Thus the needed metadata size is at most root_level * nodesize, + * and * 2 since we have two trees to COW. + */ + reserve_level = max_t(int, 1, btrfs_root_level(root_item)); + min_reserved = (reserve_level << fs_info->nodesize_bits) * 2; + memset(&next_key, 0, sizeof(next_key)); + + while (1) { + ret = btrfs_block_rsv_refill(fs_info, rc->block_rsv, + min_reserved, + BTRFS_RESERVE_FLUSH_LIMIT); + if (ret) + goto out; + trans = btrfs_start_transaction(root, 0); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + trans = NULL; + goto out; + } + + /* + * At this point we no longer have a reloc_control, so we can't + * depend on btrfs_init_reloc_root to update our last_trans. + * + * But that's ok, we started the trans handle on our + * corresponding fs_root, which means it's been added to the + * dirty list. At commit time we'll still call + * btrfs_update_reloc_root() and update our root item + * appropriately. + */ + btrfs_set_root_last_trans(reloc_root, trans->transid); + trans->block_rsv = rc->block_rsv; + + replaced = false; + max_level = level; + + ret = walk_down_reloc_tree(reloc_root, path, &level); + if (ret < 0) + goto out; + if (ret > 0) + break; + + if (!find_next_key(path, level, &key) && + btrfs_comp_cpu_keys(&next_key, &key) >= 0) { + ret = 0; + } else { + ret = replace_path(trans, rc, root, reloc_root, path, + &next_key, level, max_level); + } + if (ret < 0) + goto out; + if (ret > 0) { + level = ret; + btrfs_node_key_to_cpu(path->nodes[level], &key, + path->slots[level]); + replaced = true; + } + + ret = walk_up_reloc_tree(reloc_root, path, &level); + if (ret > 0) + break; + + BUG_ON(level == 0); + /* + * save the merging progress in the drop_progress. + * this is OK since root refs == 1 in this case. + */ + btrfs_node_key(path->nodes[level], &root_item->drop_progress, + path->slots[level]); + btrfs_set_root_drop_level(root_item, level); + + btrfs_end_transaction_throttle(trans); + trans = NULL; + + btrfs_btree_balance_dirty(fs_info); + + if (replaced && rc->stage == UPDATE_DATA_PTRS) + invalidate_extent_cache(root, &key, &next_key); + } + + /* + * handle the case only one block in the fs tree need to be + * relocated and the block is tree root. + */ + leaf = btrfs_lock_root_node(root); + ret = btrfs_cow_block(trans, root, leaf, NULL, 0, &leaf, + BTRFS_NESTING_COW); + btrfs_tree_unlock(leaf); + free_extent_buffer(leaf); +out: + btrfs_free_path(path); + + if (ret == 0) { + ret = insert_dirty_subvol(trans, rc, root); + if (ret) + btrfs_abort_transaction(trans, ret); + } + + if (trans) + btrfs_end_transaction_throttle(trans); + + btrfs_btree_balance_dirty(fs_info); + + if (replaced && rc->stage == UPDATE_DATA_PTRS) + invalidate_extent_cache(root, &key, &next_key); + + return ret; +} + +static noinline_for_stack +int prepare_to_merge(struct reloc_control *rc, int err) +{ + struct btrfs_root *root = rc->extent_root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_root *reloc_root; + struct btrfs_trans_handle *trans; + LIST_HEAD(reloc_roots); + u64 num_bytes = 0; + int ret; + + mutex_lock(&fs_info->reloc_mutex); + rc->merging_rsv_size += fs_info->nodesize * (BTRFS_MAX_LEVEL - 1) * 2; + rc->merging_rsv_size += rc->nodes_relocated * 2; + mutex_unlock(&fs_info->reloc_mutex); + +again: + if (!err) { + num_bytes = rc->merging_rsv_size; + ret = btrfs_block_rsv_add(fs_info, rc->block_rsv, num_bytes, + BTRFS_RESERVE_FLUSH_ALL); + if (ret) + err = ret; + } + + trans = btrfs_join_transaction(rc->extent_root); + if (IS_ERR(trans)) { + if (!err) + btrfs_block_rsv_release(fs_info, rc->block_rsv, + num_bytes, NULL); + return PTR_ERR(trans); + } + + if (!err) { + if (num_bytes != rc->merging_rsv_size) { + btrfs_end_transaction(trans); + btrfs_block_rsv_release(fs_info, rc->block_rsv, + num_bytes, NULL); + goto again; + } + } + + rc->merge_reloc_tree = true; + + while (!list_empty(&rc->reloc_roots)) { + reloc_root = list_first_entry(&rc->reloc_roots, + struct btrfs_root, root_list); + list_del_init(&reloc_root->root_list); + + root = btrfs_get_fs_root(fs_info, reloc_root->root_key.offset, + false); + if (IS_ERR(root)) { + /* + * Even if we have an error we need this reloc root + * back on our list so we can clean up properly. + */ + list_add(&reloc_root->root_list, &reloc_roots); + btrfs_abort_transaction(trans, (int)PTR_ERR(root)); + if (!err) + err = PTR_ERR(root); + break; + } + + if (unlikely(root->reloc_root != reloc_root)) { + if (root->reloc_root) { + btrfs_err(fs_info, +"reloc tree mismatch, root %lld has reloc root key (%lld %u %llu) gen %llu, expect reloc root key (%lld %u %llu) gen %llu", + btrfs_root_id(root), + btrfs_root_id(root->reloc_root), + root->reloc_root->root_key.type, + root->reloc_root->root_key.offset, + btrfs_root_generation( + &root->reloc_root->root_item), + btrfs_root_id(reloc_root), + reloc_root->root_key.type, + reloc_root->root_key.offset, + btrfs_root_generation( + &reloc_root->root_item)); + } else { + btrfs_err(fs_info, +"reloc tree mismatch, root %lld has no reloc root, expect reloc root key (%lld %u %llu) gen %llu", + btrfs_root_id(root), + btrfs_root_id(reloc_root), + reloc_root->root_key.type, + reloc_root->root_key.offset, + btrfs_root_generation( + &reloc_root->root_item)); + } + list_add(&reloc_root->root_list, &reloc_roots); + btrfs_put_root(root); + btrfs_abort_transaction(trans, -EUCLEAN); + if (!err) + err = -EUCLEAN; + break; + } + + /* + * set reference count to 1, so btrfs_recover_relocation + * knows it should resumes merging + */ + if (!err) + btrfs_set_root_refs(&reloc_root->root_item, 1); + ret = btrfs_update_reloc_root(trans, root); + + /* + * Even if we have an error we need this reloc root back on our + * list so we can clean up properly. + */ + list_add(&reloc_root->root_list, &reloc_roots); + btrfs_put_root(root); + + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + if (!err) + err = ret; + break; + } + } + + list_splice(&reloc_roots, &rc->reloc_roots); + + if (!err) + err = btrfs_commit_transaction(trans); + else + btrfs_end_transaction(trans); + return err; +} + +static noinline_for_stack int merge_reloc_roots(struct reloc_control *rc) +{ + struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; + struct btrfs_root *root; + struct btrfs_root *reloc_root; + LIST_HEAD(reloc_roots); + bool found = false; + int ret = 0; +again: + root = rc->extent_root; + + /* + * this serializes us with btrfs_record_root_in_transaction, + * we have to make sure nobody is in the middle of + * adding their roots to the list while we are + * doing this splice + */ + mutex_lock(&fs_info->reloc_mutex); + list_splice_init(&rc->reloc_roots, &reloc_roots); + mutex_unlock(&fs_info->reloc_mutex); + + while (!list_empty(&reloc_roots)) { + found = true; + reloc_root = list_first_entry(&reloc_roots, struct btrfs_root, root_list); + + root = btrfs_get_fs_root(fs_info, reloc_root->root_key.offset, + false); + if (btrfs_root_refs(&reloc_root->root_item) > 0) { + if (WARN_ON(IS_ERR(root))) { + /* + * For recovery we read the fs roots on mount, + * and if we didn't find the root then we marked + * the reloc root as a garbage root. For normal + * relocation obviously the root should exist in + * memory. However there's no reason we can't + * handle the error properly here just in case. + */ + ret = PTR_ERR(root); + goto out; + } + if (WARN_ON(root->reloc_root != reloc_root)) { + /* + * This can happen if on-disk metadata has some + * corruption, e.g. bad reloc tree key offset. + */ + ret = -EINVAL; + btrfs_put_root(root); + goto out; + } + ret = merge_reloc_root(rc, root); + if (ret) { + /* + * Clear the reloc root since below we will call + * free_reloc_roots(), otherwise we leave + * root->reloc_root pointing to a freed reloc + * root and trigger a use-after-free during + * unmount or elsewhere. + */ + clear_reloc_root(root); + btrfs_put_root(root); + /* + * We are adding the reloc_root to the local + * reloc_roots list, so we add a ref for this + * list which will be dropped below by the call + * to free_reloc_roots(). + */ + if (list_empty(&reloc_root->root_list)) { + list_add_tail(&reloc_root->root_list, + &reloc_roots); + btrfs_grab_root(reloc_root); + } + /* Now drop the ref for root->reloc_root. */ + btrfs_put_root(reloc_root); + goto out; + } + btrfs_put_root(root); + } else { + if (!IS_ERR(root)) { + if (root->reloc_root == reloc_root) { + clear_reloc_root(root); + /* Drop the ref for root->reloc_root. */ + btrfs_put_root(reloc_root); + } + btrfs_put_root(root); + } + + list_del_init(&reloc_root->root_list); + /* Don't forget to queue this reloc root for cleanup */ + list_add_tail(&reloc_root->reloc_dirty_list, + &rc->dirty_subvol_roots); + } + } + + if (found) { + found = false; + goto again; + } +out: + if (btrfs_is_zoned(fs_info) && ret == -EAGAIN) { + abort_reloc_roots(rc, &reloc_roots); + + /* New reloc root may be added. */ + mutex_lock(&fs_info->reloc_mutex); + list_splice_init(&rc->reloc_roots, &reloc_roots); + mutex_unlock(&fs_info->reloc_mutex); + abort_reloc_roots(rc, &reloc_roots); + } else if (ret) { + btrfs_handle_fs_error(fs_info, ret, NULL); + free_reloc_roots(&reloc_roots); + + /* new reloc root may be added */ + mutex_lock(&fs_info->reloc_mutex); + list_splice_init(&rc->reloc_roots, &reloc_roots); + mutex_unlock(&fs_info->reloc_mutex); + free_reloc_roots(&reloc_roots); + } + + /* + * We used to have + * + * BUG_ON(!RB_EMPTY_ROOT(&rc->reloc_root_tree.rb_root)); + * + * here, but it's wrong. If we fail to start the transaction in + * prepare_to_merge() we will have only 0 ref reloc roots, none of which + * have actually been removed from the reloc_root_tree rb tree. This is + * fine because we're bailing here, and we hold a reference on the root + * for the list that holds it, so these roots will be cleaned up when we + * do the reloc_dirty_list afterwards. Meanwhile the root->reloc_root + * will be cleaned up on unmount. + * + * The remaining nodes will be cleaned up by put_reloc_control(). + */ + return ret; +} + +static void free_block_list(struct rb_root *blocks) +{ + struct tree_block *block; + struct rb_node *rb_node; + while ((rb_node = rb_first(blocks))) { + block = rb_entry(rb_node, struct tree_block, rb_node); + rb_erase(rb_node, blocks); + kfree(block); + } +} + +static int record_reloc_root_in_trans(struct btrfs_trans_handle *trans, + struct btrfs_root *reloc_root) +{ + struct btrfs_fs_info *fs_info = reloc_root->fs_info; + struct btrfs_root *root; + int ret; + + if (btrfs_get_root_last_trans(reloc_root) == trans->transid) + return 0; + + root = btrfs_get_fs_root(fs_info, reloc_root->root_key.offset, false); + + /* + * This should succeed, since we can't have a reloc root without having + * already looked up the actual root and created the reloc root for this + * root. + * + * However if there's some sort of corruption where we have a ref to a + * reloc root without a corresponding root this could return ENOENT. + */ + if (IS_ERR(root)) { + DEBUG_WARN("error %ld reading root for reloc root", PTR_ERR(root)); + return PTR_ERR(root); + } + if (unlikely(root->reloc_root != reloc_root)) { + DEBUG_WARN("unexpected reloc root found"); + btrfs_err(fs_info, + "root %llu has two reloc roots associated with it", + reloc_root->root_key.offset); + btrfs_put_root(root); + return -EUCLEAN; + } + ret = btrfs_record_root_in_trans(trans, root); + btrfs_put_root(root); + + return ret; +} + +static noinline_for_stack +struct btrfs_root *select_reloc_root(struct btrfs_trans_handle *trans, + struct reloc_control *rc, + struct btrfs_backref_node *node, + struct btrfs_backref_edge *edges[]) +{ + struct btrfs_backref_node *next; + struct btrfs_root *root; + int index = 0; + int ret; + + next = walk_up_backref(node, edges, &index); + root = next->root; + + /* + * If there is no root, then our references for this block are + * incomplete, as we should be able to walk all the way up to a block + * that is owned by a root. + * + * This path is only for SHAREABLE roots, so if we come upon a + * non-SHAREABLE root then we have backrefs that resolve improperly. + * + * Both of these cases indicate file system corruption, or a bug in the + * backref walking code. + */ + if (unlikely(!root)) { + btrfs_err(trans->fs_info, + "bytenr %llu doesn't have a backref path ending in a root", + node->bytenr); + return ERR_PTR(-EUCLEAN); + } + if (unlikely(!test_bit(BTRFS_ROOT_SHAREABLE, &root->state))) { + btrfs_err(trans->fs_info, + "bytenr %llu has multiple refs with one ending in a non-shareable root", + node->bytenr); + return ERR_PTR(-EUCLEAN); + } + + if (btrfs_root_id(root) == BTRFS_TREE_RELOC_OBJECTID) { + ret = record_reloc_root_in_trans(trans, root); + if (ret) + return ERR_PTR(ret); + goto found; + } + + ret = btrfs_record_root_in_trans(trans, root); + if (ret) + return ERR_PTR(ret); + root = root->reloc_root; + + /* + * We could have raced with another thread which failed, so + * root->reloc_root may not be set, return ENOENT in this case. + */ + if (!root) + return ERR_PTR(-ENOENT); + + if (unlikely(next->new_bytenr)) { + /* + * We just created the reloc root, so we shouldn't have + * ->new_bytenr set yet. If it is then we have multiple roots + * pointing at the same bytenr which indicates corruption, or + * we've made a mistake in the backref walking code. + */ + ASSERT(next->new_bytenr == 0); + btrfs_err(trans->fs_info, + "bytenr %llu possibly has multiple roots pointing at the same bytenr %llu", + node->bytenr, next->bytenr); + return ERR_PTR(-EUCLEAN); + } + + next->new_bytenr = root->node->start; + btrfs_put_root(next->root); + next->root = btrfs_grab_root(root); + ASSERT(next->root); + mark_block_processed(rc, next); +found: + next = node; + /* setup backref node path for btrfs_reloc_cow_block */ + while (1) { + rc->backref_cache.path[next->level] = next; + if (--index < 0) + break; + next = edges[index]->node[UPPER]; + } + return root; +} + +/* + * Select a tree root for relocation. + * + * Return NULL if the block is not shareable. We should use do_relocation() in + * this case. + * + * Return a tree root pointer if the block is shareable. + * Return -ENOENT if the block is root of reloc tree. + */ +static noinline_for_stack +struct btrfs_root *select_one_root(struct btrfs_backref_node *node) +{ + struct btrfs_backref_node *next; + struct btrfs_root *root; + struct btrfs_root *fs_root = NULL; + struct btrfs_backref_edge *edges[BTRFS_MAX_LEVEL - 1]; + int index = 0; + + next = node; + while (1) { + cond_resched(); + next = walk_up_backref(next, edges, &index); + root = next->root; + + /* + * This can occur if we have incomplete extent refs leading all + * the way up a particular path, in this case return -EUCLEAN. + */ + if (unlikely(!root)) + return ERR_PTR(-EUCLEAN); + + /* No other choice for non-shareable tree */ + if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) + return root; + + if (btrfs_root_id(root) != BTRFS_TREE_RELOC_OBJECTID) + fs_root = root; + + if (next != node) + return NULL; + + next = walk_down_backref(edges, &index); + if (!next || next->level <= node->level) + break; + } + + if (!fs_root) + return ERR_PTR(-ENOENT); + return fs_root; +} + +static noinline_for_stack u64 calcu_metadata_size(struct reloc_control *rc, + struct btrfs_backref_node *node) +{ + struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; + struct btrfs_backref_node *next = node; + struct btrfs_backref_edge *edge; + struct btrfs_backref_edge *edges[BTRFS_MAX_LEVEL - 1]; + u64 num_bytes = 0; + int index = 0; + + BUG_ON(node->processed); + + while (next) { + cond_resched(); + while (1) { + if (next->processed) + break; + + num_bytes += fs_info->nodesize; + + if (list_empty(&next->upper)) + break; + + edge = list_first_entry(&next->upper, struct btrfs_backref_edge, + list[LOWER]); + edges[index++] = edge; + next = edge->node[UPPER]; + } + next = walk_down_backref(edges, &index); + } + return num_bytes; +} + +static int refill_metadata_space(struct btrfs_trans_handle *trans, + struct reloc_control *rc, u64 num_bytes) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + int ret; + + trans->block_rsv = rc->block_rsv; + rc->reserved_bytes += num_bytes; + + /* + * We are under a transaction here so we can only do limited flushing. + * If we get an enospc just kick back -EAGAIN so we know to drop the + * transaction and try to refill when we can flush all the things. + */ + ret = btrfs_block_rsv_refill(fs_info, rc->block_rsv, num_bytes, + BTRFS_RESERVE_FLUSH_LIMIT); + if (ret) { + u64 tmp = fs_info->nodesize * RELOCATION_RESERVED_NODES; + + while (tmp <= rc->reserved_bytes) + tmp <<= 1; + /* + * only one thread can access block_rsv at this point, + * so we don't need hold lock to protect block_rsv. + * we expand more reservation size here to allow enough + * space for relocation and we will return earlier in + * enospc case. + */ + rc->block_rsv->size = tmp + fs_info->nodesize * + RELOCATION_RESERVED_NODES; + return -EAGAIN; + } + + return 0; +} + +static int reserve_metadata_space(struct btrfs_trans_handle *trans, + struct reloc_control *rc, + struct btrfs_backref_node *node) +{ + u64 num_bytes; + + num_bytes = calcu_metadata_size(rc, node) * 2; + return refill_metadata_space(trans, rc, num_bytes); +} + +/* + * relocate a block tree, and then update pointers in upper level + * blocks that reference the block to point to the new location. + * + * if called by link_to_upper, the block has already been relocated. + * in that case this function just updates pointers. + */ +static int do_relocation(struct btrfs_trans_handle *trans, + struct reloc_control *rc, + struct btrfs_backref_node *node, + struct btrfs_key *key, + struct btrfs_path *path, int lowest) +{ + struct btrfs_backref_node *upper; + struct btrfs_backref_edge *edge; + struct btrfs_backref_edge *edges[BTRFS_MAX_LEVEL - 1]; + struct btrfs_root *root; + struct extent_buffer *eb; + u32 blocksize; + u64 bytenr; + int slot; + int ret = 0; + + /* + * If we are lowest then this is the first time we're processing this + * block, and thus shouldn't have an eb associated with it yet. + */ + ASSERT(!lowest || !node->eb); + + path->lowest_level = node->level + 1; + rc->backref_cache.path[node->level] = node; + list_for_each_entry(edge, &node->upper, list[LOWER]) { + cond_resched(); + + upper = edge->node[UPPER]; + root = select_reloc_root(trans, rc, upper, edges); + if (IS_ERR(root)) { + ret = PTR_ERR(root); + goto next; + } + + if (upper->eb && !upper->locked) { + if (!lowest) { + ret = btrfs_bin_search(upper->eb, 0, key, &slot); + if (ret < 0) + goto next; + BUG_ON(ret); + bytenr = btrfs_node_blockptr(upper->eb, slot); + if (node->eb->start == bytenr) + goto next; + } + btrfs_backref_drop_node_buffer(upper); + } + + if (!upper->eb) { + ret = btrfs_search_slot(trans, root, key, path, 0, 1); + if (ret) { + if (ret > 0) + ret = -ENOENT; + + btrfs_release_path(path); + break; + } + + if (!upper->eb) { + upper->eb = path->nodes[upper->level]; + path->nodes[upper->level] = NULL; + } else { + BUG_ON(upper->eb != path->nodes[upper->level]); + } + + upper->locked = 1; + path->locks[upper->level] = 0; + + slot = path->slots[upper->level]; + btrfs_release_path(path); + } else { + ret = btrfs_bin_search(upper->eb, 0, key, &slot); + if (ret < 0) + goto next; + BUG_ON(ret); + } + + bytenr = btrfs_node_blockptr(upper->eb, slot); + if (lowest) { + if (unlikely(bytenr != node->bytenr)) { + btrfs_err(root->fs_info, + "lowest leaf/node mismatch: bytenr %llu node->bytenr %llu slot %d upper %llu", + bytenr, node->bytenr, slot, + upper->eb->start); + ret = -EIO; + goto next; + } + } else { + if (node->eb->start == bytenr) + goto next; + } + + blocksize = root->fs_info->nodesize; + eb = btrfs_read_node_slot(upper->eb, slot); + if (IS_ERR(eb)) { + ret = PTR_ERR(eb); + goto next; + } + btrfs_tree_lock(eb); + + if (!node->eb) { + ret = btrfs_cow_block(trans, root, eb, upper->eb, + slot, &eb, BTRFS_NESTING_COW); + btrfs_tree_unlock(eb); + free_extent_buffer(eb); + if (ret < 0) + goto next; + /* + * We've just COWed this block, it should have updated + * the correct backref node entry. + */ + ASSERT(node->eb == eb); + } else { + struct btrfs_ref ref = { + .action = BTRFS_ADD_DELAYED_REF, + .bytenr = node->eb->start, + .num_bytes = blocksize, + .parent = upper->eb->start, + .owning_root = btrfs_header_owner(upper->eb), + .ref_root = btrfs_header_owner(upper->eb), + }; + + btrfs_set_node_blockptr(upper->eb, slot, + node->eb->start); + btrfs_set_node_ptr_generation(upper->eb, slot, + trans->transid); + btrfs_mark_buffer_dirty(trans, upper->eb); + + btrfs_init_tree_ref(&ref, node->level, + btrfs_root_id(root), false); + ret = btrfs_inc_extent_ref(trans, &ref); + if (!ret) + ret = btrfs_drop_subtree(trans, root, eb, + upper->eb); + if (unlikely(ret)) + btrfs_abort_transaction(trans, ret); + } +next: + if (!upper->pending) + btrfs_backref_drop_node_buffer(upper); + else + btrfs_backref_unlock_node_buffer(upper); + if (ret) + break; + } + + if (!ret && node->pending) { + btrfs_backref_drop_node_buffer(node); + list_del_init(&node->list); + node->pending = 0; + } + + path->lowest_level = 0; + + /* + * We should have allocated all of our space in the block rsv and thus + * shouldn't ENOSPC. + */ + ASSERT(ret != -ENOSPC); + return ret; +} + +static int link_to_upper(struct btrfs_trans_handle *trans, + struct reloc_control *rc, + struct btrfs_backref_node *node, + struct btrfs_path *path) +{ + struct btrfs_key key; + + btrfs_node_key_to_cpu(node->eb, &key, 0); + return do_relocation(trans, rc, node, &key, path, 0); +} + +static int finish_pending_nodes(struct btrfs_trans_handle *trans, + struct reloc_control *rc, + struct btrfs_path *path, int err) +{ + LIST_HEAD(list); + struct btrfs_backref_cache *cache = &rc->backref_cache; + struct btrfs_backref_node *node; + int level; + int ret; + + for (level = 0; level < BTRFS_MAX_LEVEL; level++) { + while (!list_empty(&cache->pending[level])) { + node = list_first_entry(&cache->pending[level], + struct btrfs_backref_node, list); + list_move_tail(&node->list, &list); + BUG_ON(!node->pending); + + if (!err) { + ret = link_to_upper(trans, rc, node, path); + if (ret < 0) + err = ret; + } + } + list_splice_init(&list, &cache->pending[level]); + } + return err; +} + +/* + * mark a block and all blocks directly/indirectly reference the block + * as processed. + */ +static void update_processed_blocks(struct reloc_control *rc, + struct btrfs_backref_node *node) +{ + struct btrfs_backref_node *next = node; + struct btrfs_backref_edge *edge; + struct btrfs_backref_edge *edges[BTRFS_MAX_LEVEL - 1]; + int index = 0; + + while (next) { + cond_resched(); + while (1) { + if (next->processed) + break; + + mark_block_processed(rc, next); + + if (list_empty(&next->upper)) + break; + + edge = list_first_entry(&next->upper, struct btrfs_backref_edge, + list[LOWER]); + edges[index++] = edge; + next = edge->node[UPPER]; + } + next = walk_down_backref(edges, &index); + } +} + +static int tree_block_processed(u64 bytenr, struct reloc_control *rc) +{ + u32 blocksize = rc->extent_root->fs_info->nodesize; + + if (btrfs_test_range_bit(&rc->processed_blocks, bytenr, + bytenr + blocksize - 1, EXTENT_DIRTY, NULL)) + return 1; + return 0; +} + +static int get_tree_block_key(struct btrfs_fs_info *fs_info, + struct tree_block *block) +{ + struct btrfs_tree_parent_check check = { + .level = block->level, + .owner_root = block->owner, + .transid = block->key.offset + }; + struct extent_buffer *eb; + + eb = read_tree_block(fs_info, block->bytenr, &check); + if (IS_ERR(eb)) + return PTR_ERR(eb); + + if (block->level == 0) + btrfs_item_key_to_cpu(eb, &block->key, 0); + else + btrfs_node_key_to_cpu(eb, &block->key, 0); + free_extent_buffer(eb); + block->key_ready = true; + return 0; +} + +/* + * helper function to relocate a tree block + */ +static int relocate_tree_block(struct btrfs_trans_handle *trans, + struct reloc_control *rc, + struct btrfs_backref_node *node, + struct btrfs_key *key, + struct btrfs_path *path) +{ + struct btrfs_root *root; + int ret = 0; + + if (!node) + return 0; + + /* + * If we fail here we want to drop our backref_node because we are going + * to start over and regenerate the tree for it. + */ + ret = reserve_metadata_space(trans, rc, node); + if (ret) + goto out; + + BUG_ON(node->processed); + root = select_one_root(node); + if (IS_ERR(root)) { + ret = PTR_ERR(root); + + /* See explanation in select_one_root for the -EUCLEAN case. */ + ASSERT(ret == -ENOENT); + if (ret == -ENOENT) { + ret = 0; + update_processed_blocks(rc, node); + } + goto out; + } + + if (root) { + if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) { + /* + * This block was the root block of a root, and this is + * the first time we're processing the block and thus it + * should not have had the ->new_bytenr modified. + * + * However in the case of corruption we could have + * multiple refs pointing to the same block improperly, + * and thus we would trip over these checks. ASSERT() + * for the developer case, because it could indicate a + * bug in the backref code, however error out for a + * normal user in the case of corruption. + */ + ASSERT(node->new_bytenr == 0); + if (unlikely(node->new_bytenr)) { + btrfs_err(root->fs_info, + "bytenr %llu has improper references to it", + node->bytenr); + ret = -EUCLEAN; + goto out; + } + ret = btrfs_record_root_in_trans(trans, root); + if (ret) + goto out; + /* + * Another thread could have failed, need to check if we + * have reloc_root actually set. + */ + if (!root->reloc_root) { + ret = -ENOENT; + goto out; + } + root = root->reloc_root; + node->new_bytenr = root->node->start; + btrfs_put_root(node->root); + node->root = btrfs_grab_root(root); + ASSERT(node->root); + } else { + btrfs_err(root->fs_info, + "bytenr %llu resolved to a non-shareable root", + node->bytenr); + ret = -EUCLEAN; + goto out; + } + if (!ret) + update_processed_blocks(rc, node); + } else { + ret = do_relocation(trans, rc, node, key, path, 1); + } +out: + if (ret || node->level == 0) + btrfs_backref_cleanup_node(&rc->backref_cache, node); + return ret; +} + +static int relocate_cowonly_block(struct btrfs_trans_handle *trans, + struct reloc_control *rc, struct tree_block *block, + struct btrfs_path *path) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *root; + u64 num_bytes; + int nr_levels; + int ret; + + root = btrfs_get_fs_root(fs_info, block->owner, true); + if (IS_ERR(root)) + return PTR_ERR(root); + + nr_levels = max(btrfs_header_level(root->node) - block->level, 0) + 1; + + num_bytes = (nr_levels << fs_info->nodesize_bits); + ret = refill_metadata_space(trans, rc, num_bytes); + if (ret) { + btrfs_put_root(root); + return ret; + } + path->lowest_level = block->level; + if (root == root->fs_info->chunk_root) + btrfs_reserve_chunk_metadata(trans, false); + + ret = btrfs_search_slot(trans, root, &block->key, path, 0, 1); + path->lowest_level = 0; + btrfs_release_path(path); + + if (root == root->fs_info->chunk_root) + btrfs_trans_release_chunk_metadata(trans); + if (ret > 0) + ret = 0; + btrfs_put_root(root); + + return ret; +} + +/* + * relocate a list of blocks + */ +static noinline_for_stack +int relocate_tree_blocks(struct btrfs_trans_handle *trans, + struct reloc_control *rc, struct rb_root *blocks) +{ + struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; + struct btrfs_backref_node *node; + struct btrfs_path *path; + struct tree_block *block; + struct tree_block *next; + int ret = 0; + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto out_free_blocks; + } + + /* Kick in readahead for tree blocks with missing keys */ + rbtree_postorder_for_each_entry_safe(block, next, blocks, rb_node) { + if (!block->key_ready) + btrfs_readahead_tree_block(fs_info, block->bytenr, + block->owner, 0, + block->level, NULL); + } + + /* Get first keys */ + rbtree_postorder_for_each_entry_safe(block, next, blocks, rb_node) { + if (!block->key_ready) { + ret = get_tree_block_key(fs_info, block); + if (ret) + goto out_free_path; + } + } + + /* Do tree relocation */ + rbtree_postorder_for_each_entry_safe(block, next, blocks, rb_node) { + /* + * For COWonly blocks, or the data reloc tree, we only need to + * COW down to the block, there's no need to generate a backref + * tree. + */ + if (block->owner && + (!btrfs_is_fstree(block->owner) || + block->owner == BTRFS_DATA_RELOC_TREE_OBJECTID)) { + ret = relocate_cowonly_block(trans, rc, block, path); + if (ret) + break; + continue; + } + + node = build_backref_tree(trans, rc, &block->key, + block->level, block->bytenr); + if (IS_ERR(node)) { + ret = PTR_ERR(node); + goto out; + } + + ret = relocate_tree_block(trans, rc, node, &block->key, + path); + if (ret < 0) + break; + } +out: + ret = finish_pending_nodes(trans, rc, path, ret); + +out_free_path: + btrfs_free_path(path); +out_free_blocks: + free_block_list(blocks); + return ret; +} + +static noinline_for_stack int prealloc_file_extent_cluster(struct reloc_control *rc) +{ + const struct file_extent_cluster *cluster = &rc->cluster; + struct btrfs_inode *inode = BTRFS_I(rc->data_inode); + u64 alloc_hint = 0; + u64 start; + u64 end; + u64 offset = inode->reloc_block_group_start; + u64 num_bytes; + int nr; + int ret = 0; + u64 prealloc_start = cluster->start - offset; + u64 prealloc_end = cluster->end - offset; + u64 cur_offset = prealloc_start; + + /* + * For blocksize < folio size case (either bs < page size or large folios), + * beyond i_size, all blocks are filled with zero. + * + * If the current cluster covers the above range, btrfs_do_readpage() + * will skip the read, and relocate_one_folio() will later writeback + * the padding zeros as new data, causing data corruption. + * + * Here we have to invalidate the cache covering our cluster. + */ + ret = filemap_invalidate_inode(&inode->vfs_inode, true, prealloc_start, + prealloc_end); + if (ret < 0) + return ret; + + BUG_ON(cluster->start != cluster->boundary[0]); + ret = btrfs_alloc_data_chunk_ondemand(inode, + prealloc_end + 1 - prealloc_start); + if (ret) + return ret; + + btrfs_inode_lock(inode, 0); + for (nr = 0; nr < cluster->nr; nr++) { + struct extent_state *cached_state = NULL; + + start = cluster->boundary[nr] - offset; + if (nr + 1 < cluster->nr) + end = cluster->boundary[nr + 1] - 1 - offset; + else + end = cluster->end - offset; + + btrfs_lock_extent(&inode->io_tree, start, end, &cached_state); + num_bytes = end + 1 - start; + ret = btrfs_prealloc_file_range(&inode->vfs_inode, 0, start, + num_bytes, num_bytes, + end + 1, &alloc_hint); + cur_offset = end + 1; + btrfs_unlock_extent(&inode->io_tree, start, end, &cached_state); + if (ret) + break; + } + btrfs_inode_unlock(inode, 0); + + if (cur_offset < prealloc_end) + btrfs_free_reserved_data_space_noquota(inode, + prealloc_end + 1 - cur_offset); + return ret; +} + +static noinline_for_stack int setup_relocation_extent_mapping(struct reloc_control *rc) +{ + struct btrfs_inode *inode = BTRFS_I(rc->data_inode); + struct extent_map *em; + struct extent_state *cached_state = NULL; + u64 offset = inode->reloc_block_group_start; + u64 start = rc->cluster.start - offset; + u64 end = rc->cluster.end - offset; + int ret = 0; + + em = btrfs_alloc_extent_map(); + if (!em) + return -ENOMEM; + + em->start = start; + em->len = end + 1 - start; + em->disk_bytenr = rc->cluster.start; + em->disk_num_bytes = em->len; + em->ram_bytes = em->len; + em->flags |= EXTENT_FLAG_PINNED; + + btrfs_lock_extent(&inode->io_tree, start, end, &cached_state); + ret = btrfs_replace_extent_map_range(inode, em, false); + btrfs_unlock_extent(&inode->io_tree, start, end, &cached_state); + btrfs_free_extent_map(em); + + return ret; +} + +/* + * Allow error injection to test balance/relocation cancellation + */ +noinline int btrfs_should_cancel_balance(const struct btrfs_fs_info *fs_info) +{ + return atomic_read(&fs_info->balance_cancel_req) || + atomic_read(&fs_info->reloc_cancel_req) || + fatal_signal_pending(current); +} +ALLOW_ERROR_INJECTION(btrfs_should_cancel_balance, TRUE); + +static u64 get_cluster_boundary_end(const struct file_extent_cluster *cluster, + int cluster_nr) +{ + /* Last extent, use cluster end directly */ + if (cluster_nr >= cluster->nr - 1) + return cluster->end; + + /* Use next boundary start*/ + return cluster->boundary[cluster_nr + 1] - 1; +} + +static int relocate_one_folio(struct reloc_control *rc, + struct file_ra_state *ra, + int *cluster_nr, u64 *file_offset_ret) +{ + const struct file_extent_cluster *cluster = &rc->cluster; + struct inode *inode = rc->data_inode; + struct btrfs_fs_info *fs_info = inode_to_fs_info(inode); + const u64 orig_file_offset = *file_offset_ret; + u64 offset = BTRFS_I(inode)->reloc_block_group_start; + const pgoff_t last_index = (cluster->end - offset) >> PAGE_SHIFT; + const pgoff_t index = orig_file_offset >> PAGE_SHIFT; + gfp_t mask = btrfs_alloc_write_mask(inode->i_mapping); + struct folio *folio; + u64 folio_start; + u64 folio_end; + u64 cur; + int ret; + const bool use_rst = btrfs_need_stripe_tree_update(fs_info, rc->block_group->flags); + + ASSERT(index <= last_index); +again: + folio = filemap_lock_folio(inode->i_mapping, index); + if (IS_ERR(folio)) { + + /* + * On relocation we're doing readahead on the relocation inode, + * but if the filesystem is backed by a RAID stripe tree we can + * get ENOENT (e.g. due to preallocated extents not being + * mapped in the RST) from the lookup. + * + * But readahead doesn't handle the error and submits invalid + * reads to the device, causing a assertion failures. + */ + if (!use_rst) + page_cache_sync_readahead(inode->i_mapping, ra, NULL, + index, last_index + 1 - index); + folio = __filemap_get_folio(inode->i_mapping, index, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, + mask); + if (IS_ERR(folio)) + return PTR_ERR(folio); + } + + if (folio_test_readahead(folio) && !use_rst) + page_cache_async_readahead(inode->i_mapping, ra, NULL, + folio, last_index + 1 - index); + + if (!folio_test_uptodate(folio)) { + btrfs_read_folio(NULL, folio); + folio_lock(folio); + if (unlikely(!folio_test_uptodate(folio))) { + ret = -EIO; + goto release_folio; + } + if (folio->mapping != inode->i_mapping) { + folio_unlock(folio); + folio_put(folio); + goto again; + } + } + + /* + * We could have lost folio private when we dropped the lock to read the + * folio above, make sure we set_folio_extent_mapped() here so we have any + * of the subpage blocksize stuff we need in place. + */ + ret = set_folio_extent_mapped(folio); + if (ret < 0) + goto release_folio; + + folio_start = folio_pos(folio); + folio_end = folio_start + folio_size(folio) - 1; + + /* + * Start from the cluster, as for subpage case, the cluster can start + * inside the folio. + */ + cur = max(folio_start, cluster->boundary[*cluster_nr] - offset); + while (cur <= folio_end) { + struct extent_state *cached_state = NULL; + u64 extent_start = cluster->boundary[*cluster_nr] - offset; + u64 extent_end = get_cluster_boundary_end(cluster, + *cluster_nr) - offset; + u64 clamped_start = max(folio_start, extent_start); + u64 clamped_end = min(folio_end, extent_end); + u32 clamped_len = clamped_end + 1 - clamped_start; + + /* Reserve metadata for this range */ + ret = btrfs_delalloc_reserve_metadata(BTRFS_I(inode), + clamped_len, clamped_len, + false); + if (ret) + goto release_folio; + + /* Mark the range delalloc and dirty for later writeback */ + btrfs_lock_extent(&BTRFS_I(inode)->io_tree, clamped_start, + clamped_end, &cached_state); + ret = btrfs_set_extent_delalloc(BTRFS_I(inode), clamped_start, + clamped_end, 0, &cached_state); + if (ret) { + btrfs_clear_extent_bit(&BTRFS_I(inode)->io_tree, + clamped_start, clamped_end, + EXTENT_LOCKED | EXTENT_BOUNDARY, + &cached_state); + btrfs_delalloc_release_metadata(BTRFS_I(inode), + clamped_len, true); + btrfs_delalloc_release_extents(BTRFS_I(inode), + clamped_len); + goto release_folio; + } + btrfs_folio_set_dirty(fs_info, folio, clamped_start, clamped_len); + + /* + * Set the boundary if it's inside the folio. + * Data relocation requires the destination extents to have the + * same size as the source. + * EXTENT_BOUNDARY bit prevents current extent from being merged + * with previous extent. + */ + if (in_range(cluster->boundary[*cluster_nr] - offset, + folio_start, folio_size(folio))) { + u64 boundary_start = cluster->boundary[*cluster_nr] - + offset; + u64 boundary_end = boundary_start + + fs_info->sectorsize - 1; + + btrfs_set_extent_bit(&BTRFS_I(inode)->io_tree, + boundary_start, boundary_end, + EXTENT_BOUNDARY, NULL); + } + btrfs_unlock_extent(&BTRFS_I(inode)->io_tree, clamped_start, clamped_end, + &cached_state); + btrfs_delalloc_release_extents(BTRFS_I(inode), clamped_len); + cur += clamped_len; + + /* Crossed extent end, go to next extent */ + if (cur >= extent_end) { + (*cluster_nr)++; + /* Just finished the last extent of the cluster, exit. */ + if (*cluster_nr >= cluster->nr) + break; + } + } + folio_unlock(folio); + folio_put(folio); + + balance_dirty_pages_ratelimited(inode->i_mapping); + btrfs_throttle(fs_info); + if (btrfs_should_cancel_balance(fs_info)) + ret = -ECANCELED; + *file_offset_ret = folio_end + 1; + return ret; + +release_folio: + folio_unlock(folio); + folio_put(folio); + return ret; +} + +static int relocate_file_extent_cluster(struct reloc_control *rc) +{ + struct inode *inode = rc->data_inode; + const struct file_extent_cluster *cluster = &rc->cluster; + u64 offset = BTRFS_I(inode)->reloc_block_group_start; + u64 cur_file_offset = cluster->start - offset; + struct file_ra_state AUTO_KFREE(ra); + int cluster_nr = 0; + int ret = 0; + + if (!cluster->nr) + return 0; + + ra = kzalloc_obj(*ra, GFP_NOFS); + if (!ra) + return -ENOMEM; + + ret = prealloc_file_extent_cluster(rc); + if (ret) + return ret; + + file_ra_state_init(ra, inode->i_mapping); + + ret = setup_relocation_extent_mapping(rc); + if (ret) + return ret; + + while (cur_file_offset < cluster->end - offset) { + ret = relocate_one_folio(rc, ra, &cluster_nr, &cur_file_offset); + if (ret) + break; + } + if (ret == 0) + WARN_ON(cluster_nr != cluster->nr); + return ret; +} + +static noinline_for_stack int relocate_data_extent(struct reloc_control *rc, + const struct btrfs_key *extent_key) +{ + struct inode *inode = rc->data_inode; + struct file_extent_cluster *cluster = &rc->cluster; + int ret; + struct btrfs_root *root = BTRFS_I(inode)->root; + + if (cluster->nr > 0 && extent_key->objectid != cluster->end + 1) { + ret = relocate_file_extent_cluster(rc); + if (ret) + return ret; + cluster->nr = 0; + } + + /* + * Under simple quotas, we set root->relocation_src_root when we find + * the extent. If adjacent extents have different owners, we can't merge + * them while relocating. Handle this by storing the owning root that + * started a cluster and if we see an extent from a different root break + * cluster formation (just like the above case of non-adjacent extents). + * + * Without simple quotas, relocation_src_root is always 0, so we should + * never see a mismatch, and it should have no effect on relocation + * clusters. + */ + if (cluster->nr > 0 && cluster->owning_root != root->relocation_src_root) { + u64 tmp = root->relocation_src_root; + + /* + * root->relocation_src_root is the state that actually affects + * the preallocation we do here, so set it to the root owning + * the cluster we need to relocate. + */ + root->relocation_src_root = cluster->owning_root; + ret = relocate_file_extent_cluster(rc); + if (ret) + return ret; + cluster->nr = 0; + /* And reset it back for the current extent's owning root. */ + root->relocation_src_root = tmp; + } + + if (!cluster->nr) { + cluster->start = extent_key->objectid; + cluster->owning_root = root->relocation_src_root; + } + else + BUG_ON(cluster->nr >= MAX_EXTENTS); + cluster->end = extent_key->objectid + extent_key->offset - 1; + cluster->boundary[cluster->nr] = extent_key->objectid; + cluster->nr++; + + if (cluster->nr >= MAX_EXTENTS) { + ret = relocate_file_extent_cluster(rc); + if (ret) + return ret; + cluster->nr = 0; + } + return 0; +} + +/* + * helper to add a tree block to the list. + * the major work is getting the generation and level of the block + */ +static int add_tree_block(struct reloc_control *rc, + const struct btrfs_key *extent_key, + struct btrfs_path *path, + struct rb_root *blocks) +{ + struct extent_buffer *eb; + struct btrfs_extent_item *ei; + struct btrfs_tree_block_info *bi; + struct tree_block *block; + struct rb_node *rb_node; + u32 item_size; + int level = -1; + u64 generation; + u64 owner = 0; + + eb = path->nodes[0]; + item_size = btrfs_item_size(eb, path->slots[0]); + + if (extent_key->type == BTRFS_METADATA_ITEM_KEY || + item_size >= sizeof(*ei) + sizeof(*bi)) { + unsigned long ptr = 0, end; + + ei = btrfs_item_ptr(eb, path->slots[0], + struct btrfs_extent_item); + end = (unsigned long)ei + item_size; + if (extent_key->type == BTRFS_EXTENT_ITEM_KEY) { + bi = (struct btrfs_tree_block_info *)(ei + 1); + level = btrfs_tree_block_level(eb, bi); + ptr = (unsigned long)(bi + 1); + } else { + level = (int)extent_key->offset; + ptr = (unsigned long)(ei + 1); + } + generation = btrfs_extent_generation(eb, ei); + + /* + * We're reading random blocks without knowing their owner ahead + * of time. This is ok most of the time, as all reloc roots and + * fs roots have the same lock type. However normal trees do + * not, and the only way to know ahead of time is to read the + * inline ref offset. We know it's an fs root if + * + * 1. There's more than one ref. + * 2. There's a SHARED_DATA_REF_KEY set. + * 3. FULL_BACKREF is set on the flags. + * + * Otherwise it's safe to assume that the ref offset == the + * owner of this block, so we can use that when calling + * read_tree_block. + */ + if (btrfs_extent_refs(eb, ei) == 1 && + !(btrfs_extent_flags(eb, ei) & + BTRFS_BLOCK_FLAG_FULL_BACKREF) && + ptr < end) { + struct btrfs_extent_inline_ref *iref; + int type; + + iref = (struct btrfs_extent_inline_ref *)ptr; + type = btrfs_get_extent_inline_ref_type(eb, iref, + BTRFS_REF_TYPE_BLOCK); + if (type == BTRFS_REF_TYPE_INVALID) + return -EINVAL; + if (type == BTRFS_TREE_BLOCK_REF_KEY) + owner = btrfs_extent_inline_ref_offset(eb, iref); + } + } else { + btrfs_print_leaf(eb); + btrfs_err(rc->block_group->fs_info, + "unrecognized tree backref at tree block %llu slot %u", + eb->start, path->slots[0]); + btrfs_release_path(path); + return -EUCLEAN; + } + + btrfs_release_path(path); + + BUG_ON(level == -1); + + block = kmalloc_obj(*block, GFP_NOFS); + if (!block) + return -ENOMEM; + + block->bytenr = extent_key->objectid; + block->key.objectid = rc->extent_root->fs_info->nodesize; + block->key.offset = generation; + block->level = level; + block->key_ready = false; + block->owner = owner; + + rb_node = rb_simple_insert(blocks, &block->simple_node); + if (rb_node) + btrfs_backref_panic(rc->extent_root->fs_info, block->bytenr, + -EEXIST); + + return 0; +} + +/* + * helper to add tree blocks for backref of type BTRFS_SHARED_DATA_REF_KEY + */ +static int __add_tree_block(struct reloc_control *rc, + u64 bytenr, u32 blocksize, + struct rb_root *blocks) +{ + struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + int ret; + bool skinny = btrfs_fs_incompat(fs_info, SKINNY_METADATA); + + if (tree_block_processed(bytenr, rc)) + return 0; + + if (rb_simple_search(blocks, bytenr)) + return 0; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; +again: + key.objectid = bytenr; + if (skinny) { + key.type = BTRFS_METADATA_ITEM_KEY; + key.offset = (u64)-1; + } else { + key.type = BTRFS_EXTENT_ITEM_KEY; + key.offset = blocksize; + } + + path->search_commit_root = true; + path->skip_locking = true; + ret = btrfs_search_slot(NULL, rc->extent_root, &key, path, 0, 0); + if (ret < 0) + return ret; + + if (ret > 0 && skinny) { + if (path->slots[0]) { + path->slots[0]--; + btrfs_item_key_to_cpu(path->nodes[0], &key, + path->slots[0]); + if (key.objectid == bytenr && + (key.type == BTRFS_METADATA_ITEM_KEY || + (key.type == BTRFS_EXTENT_ITEM_KEY && + key.offset == blocksize))) + ret = 0; + } + + if (ret) { + skinny = false; + btrfs_release_path(path); + goto again; + } + } + if (WARN_ON(ret)) { + ASSERT(ret == 1); + btrfs_print_leaf(path->nodes[0]); + btrfs_err(fs_info, + "tree block extent item (%llu) is not found in extent tree", + bytenr); + return -EINVAL; + } + + return add_tree_block(rc, &key, path, blocks); +} + +static int delete_block_group_cache(struct btrfs_block_group *block_group, + struct inode *inode, + u64 ino) +{ + struct btrfs_fs_info *fs_info = block_group->fs_info; + struct btrfs_root *root = fs_info->tree_root; + struct btrfs_trans_handle *trans; + struct btrfs_inode *btrfs_inode; + int ret = 0; + + if (inode) + goto truncate; + + btrfs_inode = btrfs_iget(ino, root); + if (IS_ERR(btrfs_inode)) + return -ENOENT; + inode = &btrfs_inode->vfs_inode; + +truncate: + ret = btrfs_check_trunc_cache_free_space(fs_info, + &fs_info->global_block_rsv); + if (ret) + goto out; + + trans = btrfs_join_transaction(root); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto out; + } + + ret = btrfs_truncate_free_space_cache(trans, block_group, inode); + + btrfs_end_transaction(trans); + btrfs_btree_balance_dirty(fs_info); +out: + iput(inode); + return ret; +} + +/* + * Locate the free space cache EXTENT_DATA in root tree leaf and delete the + * cache inode, to avoid free space cache data extent blocking data relocation. + */ +static int delete_v1_space_cache(struct extent_buffer *leaf, + struct btrfs_block_group *block_group, + u64 data_bytenr) +{ + u64 space_cache_ino; + struct btrfs_file_extent_item *ei; + struct btrfs_key key; + bool found = false; + int i; + + if (btrfs_header_owner(leaf) != BTRFS_ROOT_TREE_OBJECTID) + return 0; + + for (i = 0; i < btrfs_header_nritems(leaf); i++) { + u8 type; + + btrfs_item_key_to_cpu(leaf, &key, i); + if (key.type != BTRFS_EXTENT_DATA_KEY) + continue; + ei = btrfs_item_ptr(leaf, i, struct btrfs_file_extent_item); + type = btrfs_file_extent_type(leaf, ei); + + if ((type == BTRFS_FILE_EXTENT_REG || + type == BTRFS_FILE_EXTENT_PREALLOC) && + btrfs_file_extent_disk_bytenr(leaf, ei) == data_bytenr) { + found = true; + space_cache_ino = key.objectid; + break; + } + } + if (!found) + return -ENOENT; + + return delete_block_group_cache(block_group, NULL, space_cache_ino); +} + +/* + * helper to find all tree blocks that reference a given data extent + */ +static noinline_for_stack int add_data_references(struct reloc_control *rc, + const struct btrfs_key *extent_key, + struct btrfs_path *path, + struct rb_root *blocks) +{ + struct btrfs_backref_walk_ctx ctx = { 0 }; + struct ulist_iterator leaf_uiter; + struct ulist_node *ref_node = NULL; + const u32 blocksize = rc->extent_root->fs_info->nodesize; + int ret = 0; + + btrfs_release_path(path); + + ctx.bytenr = extent_key->objectid; + ctx.skip_inode_ref_list = true; + ctx.fs_info = rc->extent_root->fs_info; + + ret = btrfs_find_all_leafs(&ctx); + if (ret < 0) + return ret; + + ULIST_ITER_INIT(&leaf_uiter); + while ((ref_node = ulist_next(ctx.refs, &leaf_uiter))) { + struct btrfs_tree_parent_check check = { 0 }; + struct extent_buffer *eb; + + eb = read_tree_block(ctx.fs_info, ref_node->val, &check); + if (IS_ERR(eb)) { + ret = PTR_ERR(eb); + break; + } + ret = delete_v1_space_cache(eb, rc->block_group, + extent_key->objectid); + free_extent_buffer(eb); + if (ret < 0) + break; + ret = __add_tree_block(rc, ref_node->val, blocksize, blocks); + if (ret < 0) + break; + } + if (ret < 0) + free_block_list(blocks); + ulist_free(ctx.refs); + return ret; +} + +/* + * helper to find next unprocessed extent + */ +static noinline_for_stack +int find_next_extent(struct reloc_control *rc, struct btrfs_path *path, + struct btrfs_key *extent_key) +{ + struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; + struct btrfs_key key; + struct extent_buffer *leaf; + u64 start, end, last; + int ret; + + last = rc->block_group->start + rc->block_group->length; + while (1) { + bool block_found; + + cond_resched(); + if (rc->search_start >= last) { + ret = 1; + break; + } + + key.objectid = rc->search_start; + key.type = BTRFS_EXTENT_ITEM_KEY; + key.offset = 0; + + path->search_commit_root = true; + path->skip_locking = true; + ret = btrfs_search_slot(NULL, rc->extent_root, &key, path, + 0, 0); + if (ret < 0) + break; +next: + leaf = path->nodes[0]; + if (path->slots[0] >= btrfs_header_nritems(leaf)) { + ret = btrfs_next_leaf(rc->extent_root, path); + if (ret != 0) + break; + leaf = path->nodes[0]; + } + + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + if (key.objectid >= last) { + ret = 1; + break; + } + + if (key.type != BTRFS_EXTENT_ITEM_KEY && + key.type != BTRFS_METADATA_ITEM_KEY) { + path->slots[0]++; + goto next; + } + + if (key.type == BTRFS_EXTENT_ITEM_KEY && + key.objectid + key.offset <= rc->search_start) { + path->slots[0]++; + goto next; + } + + if (key.type == BTRFS_METADATA_ITEM_KEY && + key.objectid + fs_info->nodesize <= + rc->search_start) { + path->slots[0]++; + goto next; + } + + block_found = btrfs_find_first_extent_bit(&rc->processed_blocks, + key.objectid, &start, &end, + EXTENT_DIRTY, NULL); + + if (block_found && start <= key.objectid) { + btrfs_release_path(path); + rc->search_start = end + 1; + } else { + if (key.type == BTRFS_EXTENT_ITEM_KEY) + rc->search_start = key.objectid + key.offset; + else + rc->search_start = key.objectid + + fs_info->nodesize; + memcpy(extent_key, &key, sizeof(key)); + return 0; + } + } + btrfs_release_path(path); + return ret; +} + +static void set_reloc_control(struct reloc_control *rc) +{ + struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; + + mutex_lock(&fs_info->reloc_mutex); + spin_lock(&fs_info->reloc_ctl_lock); + fs_info->reloc_ctl = rc; + spin_unlock(&fs_info->reloc_ctl_lock); + mutex_unlock(&fs_info->reloc_mutex); +} + +static void unset_reloc_control(struct reloc_control *rc) +{ + struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; + + mutex_lock(&fs_info->reloc_mutex); + spin_lock(&fs_info->reloc_ctl_lock); + fs_info->reloc_ctl = NULL; + spin_unlock(&fs_info->reloc_ctl_lock); + mutex_unlock(&fs_info->reloc_mutex); +} + +static noinline_for_stack +int prepare_to_relocate(struct reloc_control *rc) +{ + struct btrfs_trans_handle *trans; + int ret; + + rc->block_rsv = btrfs_alloc_block_rsv(rc->extent_root->fs_info, + BTRFS_BLOCK_RSV_TEMP); + if (!rc->block_rsv) + return -ENOMEM; + + memset(&rc->cluster, 0, sizeof(rc->cluster)); + rc->search_start = rc->block_group->start; + rc->extents_found = 0; + rc->nodes_relocated = 0; + rc->merging_rsv_size = 0; + rc->reserved_bytes = 0; + rc->block_rsv->size = rc->extent_root->fs_info->nodesize * + RELOCATION_RESERVED_NODES; + ret = btrfs_block_rsv_refill(rc->extent_root->fs_info, + rc->block_rsv, rc->block_rsv->size, + BTRFS_RESERVE_FLUSH_ALL); + if (ret) + return ret; + + rc->create_reloc_tree = true; + set_reloc_control(rc); + + trans = btrfs_join_transaction(rc->extent_root); + if (IS_ERR(trans)) { + unset_reloc_control(rc); + /* + * extent tree is not a ref_cow tree and has no reloc_root to + * cleanup. And callers are responsible to free the above + * block rsv. + */ + return PTR_ERR(trans); + } + + ret = btrfs_commit_transaction(trans); + if (ret) + unset_reloc_control(rc); + + return ret; +} + +static noinline_for_stack int relocate_block_group(struct reloc_control *rc) +{ + struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; + struct rb_root blocks = RB_ROOT; + struct btrfs_key key; + struct btrfs_trans_handle *trans = NULL; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_extent_item *ei; + u64 flags; + int ret; + int err = 0; + int progress = 0; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + path->reada = READA_FORWARD; + + ret = prepare_to_relocate(rc); + if (ret) { + err = ret; + goto out_free; + } + + while (1) { + rc->reserved_bytes = 0; + ret = btrfs_block_rsv_refill(fs_info, rc->block_rsv, + rc->block_rsv->size, + BTRFS_RESERVE_FLUSH_ALL); + if (ret) { + err = ret; + break; + } + progress++; + trans = btrfs_start_transaction(rc->extent_root, 0); + if (IS_ERR(trans)) { + err = PTR_ERR(trans); + trans = NULL; + break; + } +restart: + if (rc->backref_cache.last_trans != trans->transid) + btrfs_backref_release_cache(&rc->backref_cache); + rc->backref_cache.last_trans = trans->transid; + + ret = find_next_extent(rc, path, &key); + if (ret < 0) + err = ret; + if (ret != 0) + break; + + rc->extents_found++; + + ei = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_extent_item); + flags = btrfs_extent_flags(path->nodes[0], ei); + + /* + * If we are relocating a simple quota owned extent item, we + * need to note the owner on the reloc data root so that when + * we allocate the replacement item, we can attribute it to the + * correct eventual owner (rather than the reloc data root). + */ + if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE) { + struct btrfs_root *root = BTRFS_I(rc->data_inode)->root; + u64 owning_root_id = btrfs_get_extent_owner_root(fs_info, + path->nodes[0], + path->slots[0]); + + root->relocation_src_root = owning_root_id; + } + + if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) { + ret = add_tree_block(rc, &key, path, &blocks); + } else if (rc->stage == UPDATE_DATA_PTRS && + (flags & BTRFS_EXTENT_FLAG_DATA)) { + ret = add_data_references(rc, &key, path, &blocks); + } else { + btrfs_release_path(path); + ret = 0; + } + if (ret < 0) { + err = ret; + break; + } + + if (!RB_EMPTY_ROOT(&blocks)) { + ret = relocate_tree_blocks(trans, rc, &blocks); + if (ret < 0) { + if (ret != -EAGAIN) { + err = ret; + break; + } + rc->extents_found--; + rc->search_start = key.objectid; + } + } + + btrfs_end_transaction_throttle(trans); + btrfs_btree_balance_dirty(fs_info); + trans = NULL; + + if (rc->stage == MOVE_DATA_EXTENTS && + (flags & BTRFS_EXTENT_FLAG_DATA)) { + rc->found_file_extent = true; + ret = relocate_data_extent(rc, &key); + if (ret < 0) { + err = ret; + break; + } + } + if (btrfs_should_cancel_balance(fs_info)) { + err = -ECANCELED; + break; + } + } + if (trans && progress && err == -ENOSPC) { + ret = btrfs_force_chunk_alloc(trans, rc->block_group->flags); + if (ret == 1) { + err = 0; + progress = 0; + goto restart; + } + } + + btrfs_release_path(path); + btrfs_clear_extent_bit(&rc->processed_blocks, 0, (u64)-1, EXTENT_DIRTY, NULL); + + if (trans) { + btrfs_end_transaction_throttle(trans); + btrfs_btree_balance_dirty(fs_info); + } + + if (!err && !btrfs_fs_incompat(fs_info, REMAP_TREE)) { + ret = relocate_file_extent_cluster(rc); + if (ret < 0) + err = ret; + } + + rc->create_reloc_tree = false; + set_reloc_control(rc); + + btrfs_backref_release_cache(&rc->backref_cache); + btrfs_block_rsv_release(fs_info, rc->block_rsv, (u64)-1, NULL); + + /* + * Even in the case when the relocation is cancelled, we should all go + * through prepare_to_merge() and merge_reloc_roots(). + * + * For error (including cancelled balance), prepare_to_merge() will + * mark all reloc trees orphan, then queue them for cleanup in + * merge_reloc_roots() + */ + err = prepare_to_merge(rc, err); + + ret = merge_reloc_roots(rc); + if (ret && !err) + err = ret; + + rc->merge_reloc_tree = false; + unset_reloc_control(rc); + btrfs_block_rsv_release(fs_info, rc->block_rsv, (u64)-1, NULL); + + /* get rid of pinned extents */ + ret = btrfs_commit_current_transaction(rc->extent_root); + if (ret && !err) + err = ret; +out_free: + ret = clean_dirty_subvols(rc); + if (ret < 0 && !err) + err = ret; + btrfs_free_block_rsv(fs_info, rc->block_rsv); + return err; +} + +static int __insert_orphan_inode(struct btrfs_trans_handle *trans, + struct btrfs_root *root, u64 objectid) +{ + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_inode_item *item; + struct extent_buffer *leaf; + int ret; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + ret = btrfs_insert_empty_inode(trans, root, path, objectid); + if (ret) + return ret; + + leaf = path->nodes[0]; + item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_inode_item); + memzero_extent_buffer(leaf, (unsigned long)item, sizeof(*item)); + btrfs_set_inode_generation(leaf, item, 1); + btrfs_set_inode_size(leaf, item, 0); + btrfs_set_inode_mode(leaf, item, S_IFREG | 0600); + btrfs_set_inode_flags(leaf, item, BTRFS_INODE_NOCOMPRESS | + BTRFS_INODE_PREALLOC); + return 0; +} + +static void delete_orphan_inode(struct btrfs_trans_handle *trans, + struct btrfs_root *root, u64 objectid) +{ + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + int ret = 0; + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto out; + } + + key.objectid = objectid; + key.type = BTRFS_INODE_ITEM_KEY; + key.offset = 0; + ret = btrfs_search_slot(trans, root, &key, path, -1, 1); + if (ret) { + if (ret > 0) + ret = -ENOENT; + goto out; + } + ret = btrfs_del_item(trans, root, path); +out: + if (ret) + btrfs_abort_transaction(trans, ret); +} + +/* + * helper to create inode for data relocation. + * the inode is in data relocation tree and its link count is 0 + */ +static noinline_for_stack struct inode *create_reloc_inode( + const struct btrfs_block_group *group) +{ + struct btrfs_fs_info *fs_info = group->fs_info; + struct btrfs_inode *inode = NULL; + struct btrfs_trans_handle *trans; + struct btrfs_root *root; + u64 objectid; + int ret = 0; + + root = btrfs_grab_root(fs_info->data_reloc_root); + trans = btrfs_start_transaction(root, 6); + if (IS_ERR(trans)) { + btrfs_put_root(root); + return ERR_CAST(trans); + } + + ret = btrfs_get_free_objectid(root, &objectid); + if (ret) + goto out; + + ret = __insert_orphan_inode(trans, root, objectid); + if (ret) + goto out; + + inode = btrfs_iget(objectid, root); + if (IS_ERR(inode)) { + delete_orphan_inode(trans, root, objectid); + ret = PTR_ERR(inode); + inode = NULL; + goto out; + } + inode->reloc_block_group_start = group->start; + + ret = btrfs_orphan_add(trans, inode); +out: + btrfs_put_root(root); + btrfs_end_transaction(trans); + btrfs_btree_balance_dirty(fs_info); + if (ret) { + if (inode) + iput(&inode->vfs_inode); + return ERR_PTR(ret); + } + return &inode->vfs_inode; +} + +/* + * Mark start of chunk relocation that is cancellable. Check if the cancellation + * has been requested meanwhile and don't start in that case. + * NOTE: if this returns an error, reloc_chunk_end() must not be called. + * + * Return: + * 0 success + * -EINPROGRESS operation is already in progress, that's probably a bug + * -ECANCELED cancellation request was set before the operation started + */ +static int reloc_chunk_start(struct btrfs_fs_info *fs_info) +{ + if (test_and_set_bit(BTRFS_FS_RELOC_RUNNING, &fs_info->flags)) { + /* This should not happen */ + btrfs_err(fs_info, "reloc already running, cannot start"); + return -EINPROGRESS; + } + + if (atomic_read(&fs_info->reloc_cancel_req) > 0) { + btrfs_info(fs_info, "chunk relocation canceled on start"); + /* On cancel, clear all requests. */ + clear_and_wake_up_bit(BTRFS_FS_RELOC_RUNNING, &fs_info->flags); + atomic_set(&fs_info->reloc_cancel_req, 0); + return -ECANCELED; + } + return 0; +} + +/* + * Mark end of chunk relocation that is cancellable and wake any waiters. + * NOTE: call only if a previous call to reloc_chunk_start() succeeded. + */ +static void reloc_chunk_end(struct btrfs_fs_info *fs_info) +{ + ASSERT(test_bit(BTRFS_FS_RELOC_RUNNING, &fs_info->flags)); + /* Requested after start, clear bit first so any waiters can continue */ + if (atomic_read(&fs_info->reloc_cancel_req) > 0) + btrfs_info(fs_info, "chunk relocation canceled during operation"); + clear_and_wake_up_bit(BTRFS_FS_RELOC_RUNNING, &fs_info->flags); + atomic_set(&fs_info->reloc_cancel_req, 0); +} + +static struct reloc_control *alloc_reloc_control(struct btrfs_fs_info *fs_info) +{ + struct reloc_control *rc; + + rc = kzalloc_obj(*rc, GFP_NOFS); + if (!rc) + return NULL; + + INIT_LIST_HEAD(&rc->reloc_roots); + INIT_LIST_HEAD(&rc->dirty_subvol_roots); + btrfs_backref_init_cache(fs_info, &rc->backref_cache, true); + rc->reloc_root_tree.rb_root = RB_ROOT; + spin_lock_init(&rc->reloc_root_tree.lock); + btrfs_extent_io_tree_init(fs_info, &rc->processed_blocks, IO_TREE_RELOC_BLOCKS); + refcount_set(&rc->refs, 1); + + return rc; +} + +/* + * Print the block group being relocated + */ +static void describe_relocation(struct btrfs_block_group *block_group) +{ + char buf[128] = "NONE"; + + btrfs_describe_block_groups(block_group->flags, buf, sizeof(buf)); + + btrfs_info(block_group->fs_info, "relocating block group %llu flags %s", + block_group->start, buf); +} + +static const char *stage_to_string(enum reloc_stage stage) +{ + if (stage == MOVE_DATA_EXTENTS) + return "move data extents"; + if (stage == UPDATE_DATA_PTRS) + return "update data pointers"; + return "unknown"; +} + +static int add_remap_tree_entries(struct btrfs_trans_handle *trans, struct btrfs_path *path, + struct btrfs_key *entries, unsigned int num_entries) +{ + int ret; + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_item_batch batch; + u32 *data_sizes; + u32 max_items; + + max_items = BTRFS_LEAF_DATA_SIZE(trans->fs_info) / sizeof(struct btrfs_item); + + data_sizes = kzalloc_objs(u32, min_t(u32, num_entries, max_items), GFP_NOFS); + if (!data_sizes) + return -ENOMEM; + + while (true) { + batch.keys = entries; + batch.data_sizes = data_sizes; + batch.total_data_size = 0; + batch.nr = min_t(u32, num_entries, max_items); + + ret = btrfs_insert_empty_items(trans, fs_info->remap_root, path, &batch); + btrfs_release_path(path); + + if (ret || num_entries <= max_items) + break; + + num_entries -= max_items; + entries += max_items; + } + + kfree(data_sizes); + + return ret; +} + +struct space_run { + u64 start; + u64 end; +}; + +static void parse_bitmap(u64 block_size, const unsigned long *bitmap, + unsigned long size, u64 address, struct space_run *space_runs, + unsigned int *num_space_runs) +{ + unsigned long pos, end; + u64 run_start, run_length; + + pos = find_first_bit(bitmap, size); + if (pos == size) + return; + + while (true) { + end = find_next_zero_bit(bitmap, size, pos); + + run_start = address + (pos * block_size); + run_length = (end - pos) * block_size; + + if (*num_space_runs != 0 && + space_runs[*num_space_runs - 1].end == run_start) { + space_runs[*num_space_runs - 1].end += run_length; + } else { + space_runs[*num_space_runs].start = run_start; + space_runs[*num_space_runs].end = run_start + run_length; + + (*num_space_runs)++; + } + + if (end == size) + break; + + pos = find_next_bit(bitmap, size, end + 1); + if (pos == size) + break; + } +} + +static void adjust_block_group_remap_bytes(struct btrfs_trans_handle *trans, + struct btrfs_block_group *bg, s64 diff) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + bool bg_already_dirty = true; + bool mark_unused = false; + + spin_lock(&bg->lock); + bg->remap_bytes += diff; + if (bg->used == 0 && bg->remap_bytes == 0) + mark_unused = true; + spin_unlock(&bg->lock); + + if (mark_unused) + btrfs_mark_bg_unused(bg); + + spin_lock(&trans->transaction->dirty_bgs_lock); + if (list_empty(&bg->dirty_list)) { + list_add_tail(&bg->dirty_list, &trans->transaction->dirty_bgs); + bg_already_dirty = false; + btrfs_get_block_group(bg); + } + spin_unlock(&trans->transaction->dirty_bgs_lock); + + /* Modified block groups are accounted for in the delayed_refs_rsv. */ + if (!bg_already_dirty) + btrfs_inc_delayed_refs_rsv_bg_updates(fs_info); +} + +/* Private structure for I/O from copy_remapped_data(). */ +struct reloc_io_private { + struct completion done; + refcount_t pending_refs; + blk_status_t status; +}; + +static void reloc_endio(struct btrfs_bio *bbio) +{ + struct reloc_io_private *priv = bbio->private; + + if (bbio->bio.bi_status) + WRITE_ONCE(priv->status, bbio->bio.bi_status); + + if (refcount_dec_and_test(&priv->pending_refs)) + complete(&priv->done); + + bio_put(&bbio->bio); +} + +static int copy_remapped_data_io(struct btrfs_fs_info *fs_info, + struct reloc_io_private *priv, + struct page **pages, u64 addr, u64 length, + blk_opf_t op) +{ + struct btrfs_bio *bbio; + int i; + + init_completion(&priv->done); + refcount_set(&priv->pending_refs, 1); + priv->status = 0; + + bbio = btrfs_bio_alloc(BIO_MAX_VECS, op, BTRFS_I(fs_info->btree_inode), + addr, reloc_endio, priv); + bbio->bio.bi_iter.bi_sector = (addr >> SECTOR_SHIFT); + bbio->is_remap = true; + + i = 0; + do { + size_t bytes = min_t(u64, length, PAGE_SIZE); + + if (bio_add_page(&bbio->bio, pages[i], bytes, 0) < bytes) { + refcount_inc(&priv->pending_refs); + btrfs_submit_bbio(bbio, 0); + + bbio = btrfs_bio_alloc(BIO_MAX_VECS, op, + BTRFS_I(fs_info->btree_inode), + addr, reloc_endio, priv); + bbio->bio.bi_iter.bi_sector = (addr >> SECTOR_SHIFT); + bbio->is_remap = true; + continue; + } + + i++; + addr += bytes; + length -= bytes; + } while (length); + + refcount_inc(&priv->pending_refs); + btrfs_submit_bbio(bbio, 0); + + if (!refcount_dec_and_test(&priv->pending_refs)) + wait_for_completion_io(&priv->done); + + return blk_status_to_errno(READ_ONCE(priv->status)); +} + +static int copy_remapped_data(struct btrfs_fs_info *fs_info, u64 old_addr, + u64 new_addr, u64 length) +{ + int ret; + const u64 copy_len = min_t(u64, length, SZ_1M); + struct page **pages; + struct reloc_io_private priv; + const unsigned int nr_pages = DIV_ROUND_UP(copy_len, PAGE_SIZE); + + pages = kzalloc_objs(struct page *, nr_pages, GFP_NOFS); + if (!pages) + return -ENOMEM; + + ret = btrfs_alloc_page_array(nr_pages, pages, GFP_NOFS); + if (ret) { + ret = -ENOMEM; + goto end; + } + + /* Copy 1MB at a time, to avoid using too much memory. */ + do { + u64 to_copy = min_t(u64, length, copy_len); + + /* Limit to one bio. */ + to_copy = min_t(u64, to_copy, BIO_MAX_VECS << PAGE_SHIFT); + + ret = copy_remapped_data_io(fs_info, &priv, pages, old_addr, + to_copy, REQ_OP_READ); + if (ret) + goto end; + + ret = copy_remapped_data_io(fs_info, &priv, pages, new_addr, + to_copy, REQ_OP_WRITE); + if (ret) + goto end; + + if (to_copy == length) + break; + + old_addr += to_copy; + new_addr += to_copy; + length -= to_copy; + } while (true); + + ret = 0; +end: + for (int i = 0; i < nr_pages; i++) { + if (pages[i]) + __free_page(pages[i]); + } + kfree(pages); + + return ret; +} + +static int add_remap_item(struct btrfs_trans_handle *trans, + struct btrfs_path *path, u64 new_addr, u64 length, + u64 old_addr) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_remap_item remap = { 0 }; + struct btrfs_key key; + struct extent_buffer *leaf; + int ret; + + key.objectid = old_addr; + key.type = BTRFS_REMAP_KEY; + key.offset = length; + + ret = btrfs_insert_empty_item(trans, fs_info->remap_root, path, + &key, sizeof(struct btrfs_remap_item)); + if (ret) + return ret; + + leaf = path->nodes[0]; + btrfs_set_stack_remap_address(&remap, new_addr); + write_extent_buffer(leaf, &remap, btrfs_item_ptr_offset(leaf, path->slots[0]), + sizeof(struct btrfs_remap_item)); + + btrfs_release_path(path); + + return 0; +} + +static int add_remap_backref_item(struct btrfs_trans_handle *trans, + struct btrfs_path *path, u64 new_addr, + u64 length, u64 old_addr) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_remap_item remap = { 0 }; + struct btrfs_key key; + struct extent_buffer *leaf; + int ret; + + key.objectid = new_addr; + key.type = BTRFS_REMAP_BACKREF_KEY; + key.offset = length; + + ret = btrfs_insert_empty_item(trans, fs_info->remap_root, path, &key, + sizeof(struct btrfs_remap_item)); + if (ret) + return ret; + + leaf = path->nodes[0]; + btrfs_set_stack_remap_address(&remap, old_addr); + write_extent_buffer(leaf, &remap, btrfs_item_ptr_offset(leaf, path->slots[0]), + sizeof(struct btrfs_remap_item)); + + btrfs_release_path(path); + + return 0; +} + +static int move_existing_remap(struct btrfs_fs_info *fs_info, + struct btrfs_path *path, + struct btrfs_block_group *bg, u64 new_addr, + u64 length, u64 old_addr) +{ + struct btrfs_trans_handle *trans; + struct extent_buffer *leaf; + struct btrfs_remap_item *remap_ptr; + struct btrfs_remap_item remap = { 0 }; + struct btrfs_key key, ins; + u64 dest_addr, dest_length, min_size; + struct btrfs_block_group *dest_bg; + int ret; + const bool is_data = (bg->flags & BTRFS_BLOCK_GROUP_DATA); + struct btrfs_space_info *sinfo = bg->space_info; + bool mutex_taken = false; + bool bg_needs_free_space; + + spin_lock(&sinfo->lock); + btrfs_space_info_update_bytes_may_use(sinfo, length); + spin_unlock(&sinfo->lock); + + if (is_data) + min_size = fs_info->sectorsize; + else + min_size = fs_info->nodesize; + + ret = btrfs_reserve_extent(fs_info->fs_root, length, length, min_size, + 0, 0, &ins, is_data, false); + if (unlikely(ret)) { + spin_lock(&sinfo->lock); + btrfs_space_info_update_bytes_may_use(sinfo, -length); + spin_unlock(&sinfo->lock); + return ret; + } + + if (ins.offset < length) { + spin_lock(&sinfo->lock); + btrfs_space_info_update_bytes_may_use(sinfo, ins.offset - length); + spin_unlock(&sinfo->lock); + } + + dest_addr = ins.objectid; + dest_length = ins.offset; + + dest_bg = btrfs_lookup_block_group(fs_info, dest_addr); + + if (!is_data && !IS_ALIGNED(dest_length, fs_info->nodesize)) { + u64 new_length = ALIGN_DOWN(dest_length, fs_info->nodesize); + + btrfs_free_reserved_extent(fs_info, dest_addr + new_length, + dest_length - new_length, 0); + + dest_length = new_length; + } + + trans = btrfs_join_transaction(fs_info->remap_root); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + trans = NULL; + goto end; + } + + mutex_lock(&fs_info->remap_mutex); + mutex_taken = true; + + /* Find old remap entry. */ + key.objectid = old_addr; + key.type = BTRFS_REMAP_KEY; + key.offset = length; + + ret = btrfs_search_slot(trans, fs_info->remap_root, &key, path, 0, 1); + if (ret == 1) { + /* + * Not a problem if the remap entry wasn't found: that means + * that another transaction has deallocated the data. + * move_existing_remaps() loops until the BG contains no + * remaps, so we can just return 0 in this case. + */ + btrfs_release_path(path); + ret = 0; + goto end; + } else if (unlikely(ret)) { + goto end; + } + + ret = copy_remapped_data(fs_info, new_addr, dest_addr, dest_length); + if (unlikely(ret)) + goto end; + + /* Change data of old remap entry. */ + leaf = path->nodes[0]; + remap_ptr = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_remap_item); + btrfs_set_remap_address(leaf, remap_ptr, dest_addr); + btrfs_mark_buffer_dirty(trans, leaf); + + if (dest_length != length) { + key.offset = dest_length; + btrfs_set_item_key_safe(trans, path, &key); + } + + btrfs_release_path(path); + + if (dest_length != length) { + /* Add remap item for remainder. */ + ret = add_remap_item(trans, path, new_addr + dest_length, + length - dest_length, old_addr + dest_length); + if (unlikely(ret)) + goto end; + } + + /* Change or remove old backref. */ + key.objectid = new_addr; + key.type = BTRFS_REMAP_BACKREF_KEY; + key.offset = length; + + ret = btrfs_search_slot(trans, fs_info->remap_root, &key, path, -1, 1); + if (unlikely(ret)) { + if (ret == 1) { + btrfs_release_path(path); + ret = -ENOENT; + } + goto end; + } + + leaf = path->nodes[0]; + + if (dest_length == length) { + ret = btrfs_del_item(trans, fs_info->remap_root, path); + if (unlikely(ret)) { + btrfs_release_path(path); + goto end; + } + } else { + key.objectid += dest_length; + key.offset -= dest_length; + btrfs_set_item_key_safe(trans, path, &key); + btrfs_set_stack_remap_address(&remap, old_addr + dest_length); + + write_extent_buffer(leaf, &remap, + btrfs_item_ptr_offset(leaf, path->slots[0]), + sizeof(struct btrfs_remap_item)); + } + + btrfs_release_path(path); + + /* Add new backref. */ + ret = add_remap_backref_item(trans, path, dest_addr, dest_length, old_addr); + if (unlikely(ret)) + goto end; + + adjust_block_group_remap_bytes(trans, bg, -dest_length); + + ret = btrfs_add_to_free_space_tree(trans, new_addr, dest_length); + if (unlikely(ret)) + goto end; + + adjust_block_group_remap_bytes(trans, dest_bg, dest_length); + + mutex_lock(&dest_bg->free_space_lock); + bg_needs_free_space = test_bit(BLOCK_GROUP_FLAG_NEEDS_FREE_SPACE, + &dest_bg->runtime_flags); + mutex_unlock(&dest_bg->free_space_lock); + + if (bg_needs_free_space) { + ret = btrfs_add_block_group_free_space(trans, dest_bg); + if (unlikely(ret)) + goto end; + } + + ret = btrfs_remove_from_free_space_tree(trans, dest_addr, dest_length); + if (unlikely(ret)) { + btrfs_remove_from_free_space_tree(trans, new_addr, dest_length); + goto end; + } + + ret = 0; + +end: + if (mutex_taken) + mutex_unlock(&fs_info->remap_mutex); + + btrfs_dec_block_group_reservations(fs_info, dest_addr); + + if (unlikely(ret)) { + btrfs_free_reserved_extent(fs_info, dest_addr, dest_length, 0); + + if (trans) { + btrfs_abort_transaction(trans, ret); + btrfs_end_transaction(trans); + } + } else { + btrfs_free_reserved_bytes(dest_bg, dest_length, 0); + + ret = btrfs_commit_transaction(trans); + } + + btrfs_put_block_group(dest_bg); + + return ret; +} + +static int move_existing_remaps(struct btrfs_fs_info *fs_info, + struct btrfs_block_group *bg, + struct btrfs_path *path) +{ + int ret; + struct btrfs_key key; + struct extent_buffer *leaf; + struct btrfs_remap_item *remap; + u64 old_addr; + + /* Look for backrefs in remap tree. */ + while (bg->remap_bytes > 0) { + key.objectid = bg->start; + key.type = BTRFS_REMAP_BACKREF_KEY; + key.offset = 0; + + ret = btrfs_search_slot(NULL, fs_info->remap_root, &key, path, 0, 0); + if (ret < 0) + return ret; + + leaf = path->nodes[0]; + + if (path->slots[0] >= btrfs_header_nritems(leaf)) { + ret = btrfs_next_leaf(fs_info->remap_root, path); + if (ret < 0) { + btrfs_release_path(path); + return ret; + } + + if (ret) { + btrfs_release_path(path); + break; + } + + leaf = path->nodes[0]; + } + + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + + if (key.type != BTRFS_REMAP_BACKREF_KEY) { + path->slots[0]++; + + if (path->slots[0] >= btrfs_header_nritems(leaf)) { + ret = btrfs_next_leaf(fs_info->remap_root, path); + if (ret < 0) { + btrfs_release_path(path); + return ret; + } + + if (ret) { + btrfs_release_path(path); + break; + } + + leaf = path->nodes[0]; + } + + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + } + + remap = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_remap_item); + old_addr = btrfs_remap_address(leaf, remap); + + btrfs_release_path(path); + + ret = move_existing_remap(fs_info, path, bg, key.objectid, + key.offset, old_addr); + if (ret) + return ret; + } + + ASSERT(bg->remap_bytes == 0); + + return 0; +} + +static int create_remap_tree_entries(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + struct btrfs_block_group *bg) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_free_space_info *fsi; + struct btrfs_key key, found_key; + struct extent_buffer *leaf; + struct btrfs_root *space_root; + u32 extent_count; + struct space_run *space_runs = NULL; + unsigned int num_space_runs = 0; + struct btrfs_key *entries = NULL; + unsigned int max_entries, num_entries; + int ret; + + mutex_lock(&bg->free_space_lock); + + if (test_bit(BLOCK_GROUP_FLAG_NEEDS_FREE_SPACE, &bg->runtime_flags)) { + mutex_unlock(&bg->free_space_lock); + + ret = btrfs_add_block_group_free_space(trans, bg); + if (ret) + return ret; + + mutex_lock(&bg->free_space_lock); + } + + fsi = btrfs_search_free_space_info(trans, bg, path, 0); + if (IS_ERR(fsi)) { + mutex_unlock(&bg->free_space_lock); + return PTR_ERR(fsi); + } + + extent_count = btrfs_free_space_extent_count(path->nodes[0], fsi); + + btrfs_release_path(path); + + space_runs = kmalloc_objs(*space_runs, extent_count, GFP_NOFS); + if (!space_runs) { + mutex_unlock(&bg->free_space_lock); + return -ENOMEM; + } + + key.objectid = bg->start; + key.type = 0; + key.offset = 0; + + space_root = btrfs_free_space_root(bg); + + ret = btrfs_search_slot(trans, space_root, &key, path, 0, 0); + if (ret < 0) { + mutex_unlock(&bg->free_space_lock); + goto out; + } + + ret = 0; + + while (true) { + leaf = path->nodes[0]; + + btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); + + if (found_key.objectid >= bg->start + bg->length) + break; + + if (found_key.type == BTRFS_FREE_SPACE_EXTENT_KEY) { + if (num_space_runs != 0 && + space_runs[num_space_runs - 1].end == found_key.objectid) { + space_runs[num_space_runs - 1].end = + found_key.objectid + found_key.offset; + } else { + ASSERT(num_space_runs < extent_count); + + space_runs[num_space_runs].start = found_key.objectid; + space_runs[num_space_runs].end = + found_key.objectid + found_key.offset; + + num_space_runs++; + } + } else if (found_key.type == BTRFS_FREE_SPACE_BITMAP_KEY) { + void *bitmap; + unsigned long offset; + u32 data_size; + + offset = btrfs_item_ptr_offset(leaf, path->slots[0]); + data_size = btrfs_item_size(leaf, path->slots[0]); + + if (data_size != 0) { + bitmap = kmalloc(data_size, GFP_NOFS); + if (!bitmap) { + mutex_unlock(&bg->free_space_lock); + ret = -ENOMEM; + goto out; + } + + read_extent_buffer(leaf, bitmap, offset, data_size); + + parse_bitmap(fs_info->sectorsize, bitmap, + data_size * BITS_PER_BYTE, + found_key.objectid, space_runs, + &num_space_runs); + + ASSERT(num_space_runs <= extent_count); + + kfree(bitmap); + } + } + + path->slots[0]++; + + if (path->slots[0] >= btrfs_header_nritems(leaf)) { + ret = btrfs_next_leaf(space_root, path); + if (ret != 0) { + if (ret == 1) + ret = 0; + break; + } + leaf = path->nodes[0]; + } + } + + btrfs_release_path(path); + + mutex_unlock(&bg->free_space_lock); + + max_entries = extent_count + 2; + entries = kmalloc_objs(*entries, max_entries, GFP_NOFS); + if (!entries) { + ret = -ENOMEM; + goto out; + } + + num_entries = 0; + + if (num_space_runs == 0) { + entries[num_entries].objectid = bg->start; + entries[num_entries].type = BTRFS_IDENTITY_REMAP_KEY; + entries[num_entries].offset = bg->length; + num_entries++; + } else { + if (space_runs[0].start > bg->start) { + entries[num_entries].objectid = bg->start; + entries[num_entries].type = BTRFS_IDENTITY_REMAP_KEY; + entries[num_entries].offset = space_runs[0].start - bg->start; + num_entries++; + } + + for (unsigned int i = 1; i < num_space_runs; i++) { + entries[num_entries].objectid = space_runs[i - 1].end; + entries[num_entries].type = BTRFS_IDENTITY_REMAP_KEY; + entries[num_entries].offset = + space_runs[i].start - space_runs[i - 1].end; + num_entries++; + } + + if (space_runs[num_space_runs - 1].end < bg->start + bg->length) { + entries[num_entries].objectid = + space_runs[num_space_runs - 1].end; + entries[num_entries].type = BTRFS_IDENTITY_REMAP_KEY; + entries[num_entries].offset = + bg->start + bg->length - space_runs[num_space_runs - 1].end; + num_entries++; + } + + if (num_entries == 0) + goto out; + } + + bg->identity_remap_count = num_entries; + + ret = add_remap_tree_entries(trans, path, entries, num_entries); + +out: + kfree(entries); + kfree(space_runs); + + return ret; +} + +static int find_next_identity_remap(struct btrfs_trans_handle *trans, + struct btrfs_path *path, u64 bg_end, + u64 last_start, u64 *start, u64 *length) +{ + int ret; + struct btrfs_key key, found_key; + struct btrfs_root *remap_root = trans->fs_info->remap_root; + struct extent_buffer *leaf; + + key.objectid = last_start; + key.type = BTRFS_IDENTITY_REMAP_KEY; + key.offset = 0; + + ret = btrfs_search_slot(trans, remap_root, &key, path, 0, 0); + if (ret < 0) + goto out; + + leaf = path->nodes[0]; + while (true) { + if (path->slots[0] >= btrfs_header_nritems(leaf)) { + ret = btrfs_next_leaf(remap_root, path); + + if (ret != 0) { + if (ret == 1) + ret = -ENOENT; + goto out; + } + + leaf = path->nodes[0]; + } + + btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); + + if (found_key.objectid >= bg_end) { + ret = -ENOENT; + goto out; + } + + if (found_key.type == BTRFS_IDENTITY_REMAP_KEY) { + *start = found_key.objectid; + *length = found_key.offset; + ret = 0; + goto out; + } + + path->slots[0]++; + } + +out: + btrfs_release_path(path); + + return ret; +} + +static int remove_chunk_stripes(struct btrfs_trans_handle *trans, + struct btrfs_chunk_map *chunk_map, + struct btrfs_path *path) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_key key; + struct extent_buffer *leaf; + struct btrfs_chunk *chunk; + int ret; + + key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID; + key.type = BTRFS_CHUNK_ITEM_KEY; + key.offset = chunk_map->start; + + btrfs_reserve_chunk_metadata(trans, false); + + ret = btrfs_search_slot(trans, fs_info->chunk_root, &key, path, 0, 1); + if (ret) { + if (ret == 1) { + btrfs_release_path(path); + ret = -ENOENT; + } + btrfs_trans_release_chunk_metadata(trans); + return ret; + } + + leaf = path->nodes[0]; + + chunk = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_chunk); + btrfs_set_chunk_num_stripes(leaf, chunk, 0); + btrfs_set_chunk_sub_stripes(leaf, chunk, 0); + + btrfs_truncate_item(trans, path, offsetof(struct btrfs_chunk, stripe), 1); + + btrfs_mark_buffer_dirty(trans, leaf); + + btrfs_release_path(path); + btrfs_trans_release_chunk_metadata(trans); + + return 0; +} + +int btrfs_last_identity_remap_gone(struct btrfs_chunk_map *chunk_map, + struct btrfs_block_group *bg) +{ + struct btrfs_fs_info *fs_info = bg->fs_info; + struct btrfs_trans_handle *trans; + int ret; + unsigned int num_items; + BTRFS_PATH_AUTO_FREE(path); + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + /* + * One item for each entry we're removing in the dev extents tree, and + * another for each device. DUP chunks are all on one device, + * everything else has one device per stripe. + */ + if (bg->flags & BTRFS_BLOCK_GROUP_DUP) + num_items = chunk_map->num_stripes + 1; + else + num_items = 2 * chunk_map->num_stripes; + + trans = btrfs_start_transaction_fallback_global_rsv(fs_info->tree_root, num_items); + if (IS_ERR(trans)) + return PTR_ERR(trans); + + ret = btrfs_remove_dev_extents(trans, chunk_map); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + btrfs_end_transaction(trans); + return ret; + } + + mutex_lock(&trans->fs_info->chunk_mutex); + for (unsigned int i = 0; i < chunk_map->num_stripes; i++) { + ret = btrfs_update_device(trans, chunk_map->stripes[i].dev); + if (unlikely(ret)) { + mutex_unlock(&trans->fs_info->chunk_mutex); + btrfs_abort_transaction(trans, ret); + btrfs_end_transaction(trans); + return ret; + } + } + mutex_unlock(&trans->fs_info->chunk_mutex); + + write_lock(&trans->fs_info->mapping_tree_lock); + btrfs_chunk_map_device_clear_bits(chunk_map, CHUNK_ALLOCATED); + write_unlock(&trans->fs_info->mapping_tree_lock); + + btrfs_remove_bg_from_sinfo(bg); + + spin_lock(&bg->lock); + clear_bit(BLOCK_GROUP_FLAG_STRIPE_REMOVAL_PENDING, &bg->runtime_flags); + spin_unlock(&bg->lock); + + ret = remove_chunk_stripes(trans, chunk_map, path); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + btrfs_end_transaction(trans); + return ret; + } + + ret = btrfs_commit_transaction(trans); + if (ret) + return ret; + + return 0; +} + +static void adjust_identity_remap_count(struct btrfs_trans_handle *trans, + struct btrfs_block_group *bg, int delta) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + bool bg_already_dirty = true; + bool mark_fully_remapped = false; + + WARN_ON(delta < 0 && -delta > bg->identity_remap_count); + + spin_lock(&bg->lock); + + bg->identity_remap_count += delta; + + if (bg->identity_remap_count == 0 && + !test_bit(BLOCK_GROUP_FLAG_FULLY_REMAPPED, &bg->runtime_flags)) { + set_bit(BLOCK_GROUP_FLAG_FULLY_REMAPPED, &bg->runtime_flags); + mark_fully_remapped = true; + } + + spin_unlock(&bg->lock); + + spin_lock(&trans->transaction->dirty_bgs_lock); + if (list_empty(&bg->dirty_list)) { + list_add_tail(&bg->dirty_list, &trans->transaction->dirty_bgs); + bg_already_dirty = false; + btrfs_get_block_group(bg); + } + spin_unlock(&trans->transaction->dirty_bgs_lock); + + /* Modified block groups are accounted for in the delayed_refs_rsv. */ + if (!bg_already_dirty) + btrfs_inc_delayed_refs_rsv_bg_updates(fs_info); + + if (mark_fully_remapped) + btrfs_mark_bg_fully_remapped(bg, trans); +} + +static int add_remap_entry(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + struct btrfs_block_group *src_bg, u64 old_addr, + u64 new_addr, u64 length) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_key key, new_key; + int ret; + int identity_count_delta = 0; + + key.objectid = old_addr; + key.type = (u8)-1; + key.offset = (u64)-1; + + ret = btrfs_search_slot(trans, fs_info->remap_root, &key, path, -1, 1); + if (ret < 0) + goto end; + + if (path->slots[0] == 0) { + ret = -ENOENT; + goto end; + } + + path->slots[0]--; + + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + + if (key.type != BTRFS_IDENTITY_REMAP_KEY || + key.objectid > old_addr || + key.objectid + key.offset <= old_addr) { + ret = -ENOENT; + goto end; + } + + /* Shorten or delete identity mapping entry. */ + if (key.objectid == old_addr) { + ret = btrfs_del_item(trans, fs_info->remap_root, path); + if (ret) + goto end; + + identity_count_delta--; + } else { + new_key.objectid = key.objectid; + new_key.type = BTRFS_IDENTITY_REMAP_KEY; + new_key.offset = old_addr - key.objectid; + + btrfs_set_item_key_safe(trans, path, &new_key); + } + + btrfs_release_path(path); + + /* Create new remap entry. */ + ret = add_remap_item(trans, path, new_addr, length, old_addr); + if (ret) + goto end; + + /* Add entry for remainder of identity mapping, if necessary. */ + if (key.objectid + key.offset != old_addr + length) { + new_key.objectid = old_addr + length; + new_key.type = BTRFS_IDENTITY_REMAP_KEY; + new_key.offset = key.objectid + key.offset - old_addr - length; + + ret = btrfs_insert_empty_item(trans, fs_info->remap_root, + path, &new_key, 0); + if (ret) + goto end; + + btrfs_release_path(path); + + identity_count_delta++; + } + + /* Add backref. */ + ret = add_remap_backref_item(trans, path, new_addr, length, old_addr); + if (ret) + goto end; + + if (identity_count_delta != 0) + adjust_identity_remap_count(trans, src_bg, identity_count_delta); + +end: + btrfs_release_path(path); + + return ret; +} + +static int mark_chunk_remapped(struct btrfs_trans_handle *trans, + struct btrfs_path *path, u64 start) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_chunk_map *chunk_map; + struct btrfs_key key; + u64 type; + int ret; + struct extent_buffer *leaf; + struct btrfs_chunk *chunk; + + read_lock(&fs_info->mapping_tree_lock); + + chunk_map = btrfs_find_chunk_map_nolock(fs_info, start, 1); + if (!chunk_map) { + read_unlock(&fs_info->mapping_tree_lock); + return -ENOENT; + } + + chunk_map->type |= BTRFS_BLOCK_GROUP_REMAPPED; + type = chunk_map->type; + + read_unlock(&fs_info->mapping_tree_lock); + + key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID; + key.type = BTRFS_CHUNK_ITEM_KEY; + key.offset = start; + + ret = btrfs_search_slot(trans, fs_info->chunk_root, &key, path, 0, 1); + if (ret == 1) { + ret = -ENOENT; + goto end; + } else if (ret < 0) + goto end; + + leaf = path->nodes[0]; + + chunk = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_chunk); + btrfs_set_chunk_type(leaf, chunk, type); + btrfs_mark_buffer_dirty(trans, leaf); + + ret = 0; +end: + btrfs_free_chunk_map(chunk_map); + btrfs_release_path(path); + + return ret; +} + +static int do_remap_reloc_trans(struct btrfs_fs_info *fs_info, + struct btrfs_block_group *src_bg, + struct btrfs_path *path, u64 *last_start) +{ + struct btrfs_trans_handle *trans; + struct btrfs_root *extent_root; + struct btrfs_key ins; + struct btrfs_block_group *dest_bg = NULL; + u64 start = 0, remap_length = 0; + u64 length, new_addr, min_size; + int ret; + const bool is_data = (src_bg->flags & BTRFS_BLOCK_GROUP_DATA); + bool no_more = false; + bool made_reservation = false, bg_needs_free_space; + struct btrfs_space_info *sinfo = src_bg->space_info; + + extent_root = btrfs_extent_root(fs_info, src_bg->start); + if (unlikely(!extent_root)) { + btrfs_err(fs_info, + "missing extent root for block group at offset %llu", + src_bg->start); + return -EUCLEAN; + } + + trans = btrfs_start_transaction(extent_root, 0); + if (IS_ERR(trans)) + return PTR_ERR(trans); + + mutex_lock(&fs_info->remap_mutex); + + ret = find_next_identity_remap(trans, path, src_bg->start + src_bg->length, + *last_start, &start, &remap_length); + if (ret == -ENOENT) { + no_more = true; + goto next; + } else if (ret) { + mutex_unlock(&fs_info->remap_mutex); + btrfs_end_transaction(trans); + return ret; + } + + /* Try to reserve enough space for block. */ + spin_lock(&sinfo->lock); + btrfs_space_info_update_bytes_may_use(sinfo, remap_length); + spin_unlock(&sinfo->lock); + + if (is_data) + min_size = fs_info->sectorsize; + else + min_size = fs_info->nodesize; + + /* + * We're using btrfs_reserve_extent() to allocate a contiguous + * logical address range, but this will become a remap item rather than + * an extent in the extent tree. + * + * Short allocations are fine: it means that we chop off the beginning + * of the identity remap that we're processing, and will tackle the + * rest of it the next time round. + */ + ret = btrfs_reserve_extent(fs_info->fs_root, remap_length, remap_length, + min_size, 0, 0, &ins, is_data, false); + if (ret) { + spin_lock(&sinfo->lock); + btrfs_space_info_update_bytes_may_use(sinfo, -remap_length); + spin_unlock(&sinfo->lock); + + mutex_unlock(&fs_info->remap_mutex); + btrfs_end_transaction(trans); + return ret; + } + + if (ins.offset < remap_length) { + spin_lock(&sinfo->lock); + btrfs_space_info_update_bytes_may_use(sinfo, ins.offset - remap_length); + spin_unlock(&sinfo->lock); + } + + made_reservation = true; + + new_addr = ins.objectid; + length = ins.offset; + + if (!is_data && !IS_ALIGNED(length, fs_info->nodesize)) { + u64 new_length = ALIGN_DOWN(length, fs_info->nodesize); + + btrfs_free_reserved_extent(fs_info, new_addr + new_length, + length - new_length, 0); + + length = new_length; + } + + dest_bg = btrfs_lookup_block_group(fs_info, new_addr); + + mutex_lock(&dest_bg->free_space_lock); + bg_needs_free_space = test_bit(BLOCK_GROUP_FLAG_NEEDS_FREE_SPACE, + &dest_bg->runtime_flags); + mutex_unlock(&dest_bg->free_space_lock); + + if (bg_needs_free_space) { + ret = btrfs_add_block_group_free_space(trans, dest_bg); + if (ret) { + btrfs_abort_transaction(trans, ret); + goto fail; + } + } + + ret = copy_remapped_data(fs_info, start, new_addr, length); + if (ret) { + btrfs_abort_transaction(trans, ret); + goto fail; + } + + ret = btrfs_remove_from_free_space_tree(trans, new_addr, length); + if (ret) { + btrfs_abort_transaction(trans, ret); + goto fail; + } + + ret = add_remap_entry(trans, path, src_bg, start, new_addr, length); + if (ret) { + btrfs_abort_transaction(trans, ret); + goto fail; + } + + adjust_block_group_remap_bytes(trans, dest_bg, length); + btrfs_free_reserved_bytes(dest_bg, length, 0); + + spin_lock(&sinfo->lock); + sinfo->bytes_readonly += length; + spin_unlock(&sinfo->lock); + +next: + if (dest_bg) + btrfs_put_block_group(dest_bg); + + if (made_reservation) + btrfs_dec_block_group_reservations(fs_info, new_addr); + + mutex_unlock(&fs_info->remap_mutex); + + if (src_bg->identity_remap_count == 0) { + bool mark_fully_remapped = false; + + spin_lock(&src_bg->lock); + if (!test_bit(BLOCK_GROUP_FLAG_FULLY_REMAPPED, &src_bg->runtime_flags)) { + mark_fully_remapped = true; + set_bit(BLOCK_GROUP_FLAG_FULLY_REMAPPED, &src_bg->runtime_flags); + } + spin_unlock(&src_bg->lock); + + if (mark_fully_remapped) + btrfs_mark_bg_fully_remapped(src_bg, trans); + } + + ret = btrfs_end_transaction(trans); + if (ret) + return ret; + + if (no_more) + return 1; + + *last_start = start; + + return 0; + +fail: + if (dest_bg) + btrfs_put_block_group(dest_bg); + + btrfs_free_reserved_extent(fs_info, new_addr, length, 0); + + mutex_unlock(&fs_info->remap_mutex); + btrfs_end_transaction(trans); + + return ret; +} + +static int do_remap_reloc(struct btrfs_fs_info *fs_info, struct btrfs_path *path, + struct btrfs_block_group *bg) +{ + u64 last_start = bg->start; + int ret; + + while (true) { + ret = do_remap_reloc_trans(fs_info, bg, path, &last_start); + if (ret) { + if (ret == 1) + ret = 0; + break; + } + } + + return ret; +} + +int btrfs_translate_remap(struct btrfs_fs_info *fs_info, u64 *logical, u64 *length) +{ + int ret; + struct btrfs_key key, found_key; + struct extent_buffer *leaf; + struct btrfs_remap_item *remap; + BTRFS_PATH_AUTO_FREE(path); + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = *logical; + key.type = (u8)-1; + key.offset = (u64)-1; + + ret = btrfs_search_slot(NULL, fs_info->remap_root, &key, path, 0, 0); + if (ret < 0) + return ret; + + leaf = path->nodes[0]; + if (path->slots[0] == 0) + return -ENOENT; + + path->slots[0]--; + + btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); + + if (found_key.type != BTRFS_REMAP_KEY && + found_key.type != BTRFS_IDENTITY_REMAP_KEY) { + return -ENOENT; + } + + if (found_key.objectid > *logical || + found_key.objectid + found_key.offset <= *logical) { + return -ENOENT; + } + + if (*logical + *length > found_key.objectid + found_key.offset) + *length = found_key.objectid + found_key.offset - *logical; + + if (found_key.type == BTRFS_IDENTITY_REMAP_KEY) + return 0; + + remap = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_remap_item); + *logical += btrfs_remap_address(leaf, remap) - found_key.objectid; + + return 0; +} + +static int start_block_group_remapping(struct btrfs_fs_info *fs_info, + struct btrfs_path *path, + struct btrfs_block_group *bg) +{ + struct btrfs_trans_handle *trans; + bool bg_already_dirty = true; + int ret, ret2; + + ret = btrfs_cache_block_group(bg, true); + if (ret) + return ret; + + trans = btrfs_start_transaction(fs_info->remap_root, 0); + if (IS_ERR(trans)) + return PTR_ERR(trans); + + /* We need to run delayed refs, to make sure FST is up to date. */ + ret = btrfs_run_delayed_refs(trans, U64_MAX); + if (ret) { + btrfs_end_transaction(trans); + return ret; + } + + mutex_lock(&fs_info->remap_mutex); + + if (bg->flags & BTRFS_BLOCK_GROUP_REMAPPED) { + ret = 0; + goto end; + } + + ret = create_remap_tree_entries(trans, path, bg); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto end; + } + + spin_lock(&bg->lock); + bg->flags |= BTRFS_BLOCK_GROUP_REMAPPED; + spin_unlock(&bg->lock); + + spin_lock(&trans->transaction->dirty_bgs_lock); + if (list_empty(&bg->dirty_list)) { + list_add_tail(&bg->dirty_list, &trans->transaction->dirty_bgs); + bg_already_dirty = false; + btrfs_get_block_group(bg); + } + spin_unlock(&trans->transaction->dirty_bgs_lock); + + /* Modified block groups are accounted for in the delayed_refs_rsv. */ + if (!bg_already_dirty) + btrfs_inc_delayed_refs_rsv_bg_updates(fs_info); + + ret = mark_chunk_remapped(trans, path, bg->start); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto end; + } + + ret = btrfs_remove_block_group_free_space(trans, bg); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto end; + } + + btrfs_remove_free_space_cache(bg); + +end: + mutex_unlock(&fs_info->remap_mutex); + + ret2 = btrfs_end_transaction(trans); + if (!ret) + ret = ret2; + + return ret; +} + +static int do_nonremap_reloc(struct btrfs_fs_info *fs_info, bool verbose, + struct reloc_control *rc) +{ + int ret; + + while (1) { + enum reloc_stage finishes_stage; + + mutex_lock(&fs_info->cleaner_mutex); + ret = relocate_block_group(rc); + mutex_unlock(&fs_info->cleaner_mutex); + + finishes_stage = rc->stage; + /* + * We may have gotten ENOSPC after we already dirtied some + * extents. If writeout happens while we're relocating a + * different block group we could end up hitting the + * BUG_ON(rc->stage == UPDATE_DATA_PTRS) in + * btrfs_reloc_cow_block. Make sure we write everything out + * properly so we don't trip over this problem, and then break + * out of the loop if we hit an error. + */ + if (rc->stage == MOVE_DATA_EXTENTS && rc->found_file_extent) { + int wb_ret; + + wb_ret = btrfs_wait_ordered_range(BTRFS_I(rc->data_inode), + 0, (u64)-1); + if (wb_ret && ret == 0) + ret = wb_ret; + invalidate_mapping_pages(rc->data_inode->i_mapping, 0, -1); + rc->stage = UPDATE_DATA_PTRS; + } + + if (ret < 0) + return ret; + + if (rc->extents_found == 0) + break; + + if (verbose) + btrfs_info(fs_info, "found %llu extents, stage: %s", + rc->extents_found, stage_to_string(finishes_stage)); + } + + WARN_ON(rc->block_group->pinned > 0); + WARN_ON(rc->block_group->reserved > 0); + WARN_ON(rc->block_group->used > 0); + + return 0; +} + +/* + * function to relocate all extents in a block group. + */ +int btrfs_relocate_block_group(struct btrfs_fs_info *fs_info, u64 group_start, + bool verbose) +{ + struct btrfs_block_group *bg; + struct btrfs_root *extent_root = btrfs_extent_root(fs_info, group_start); + struct reloc_control *rc; + struct inode *inode; + struct btrfs_path *path = NULL; + int ret; + bool bg_is_ro = false; + + if (unlikely(!extent_root)) { + btrfs_err(fs_info, + "missing extent root for block group at offset %llu", + group_start); + return -EUCLEAN; + } + + /* + * This only gets set if we had a half-deleted snapshot on mount. We + * cannot allow relocation to start while we're still trying to clean up + * these pending deletions. + */ + ret = wait_on_bit(&fs_info->flags, BTRFS_FS_UNFINISHED_DROPS, TASK_INTERRUPTIBLE); + if (ret) + return ret; + + /* We may have been woken up by close_ctree, so bail if we're closing. */ + if (btrfs_fs_closing(fs_info)) + return -EINTR; + + bg = btrfs_lookup_block_group(fs_info, group_start); + if (!bg) + return -ENOENT; + + /* + * Relocation of a data block group creates ordered extents. Without + * sb_start_write(), we can freeze the filesystem while unfinished + * ordered extents are left. Such ordered extents can cause a deadlock + * e.g. when syncfs() is waiting for their completion but they can't + * finish because they block when joining a transaction, due to the + * fact that the freeze locks are being held in write mode. + */ + if (bg->flags & BTRFS_BLOCK_GROUP_DATA) + ASSERT(sb_write_started(fs_info->sb)); + + if (btrfs_pinned_by_swapfile(fs_info, bg)) { + btrfs_put_block_group(bg); + return -ETXTBSY; + } + + rc = alloc_reloc_control(fs_info); + if (!rc) { + btrfs_put_block_group(bg); + return -ENOMEM; + } + + rc->extent_root = extent_root; + /* Block group ref now owned by rc, put_reloc_control() will drop it. */ + rc->block_group = bg; + + ret = reloc_chunk_start(fs_info); + if (ret < 0) + goto out_put_rc; + + ret = btrfs_inc_block_group_ro(rc->block_group, true); + if (ret) + goto out; + bg_is_ro = true; + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto out; + } + + inode = lookup_free_space_inode(rc->block_group, path); + btrfs_release_path(path); + + if (!IS_ERR(inode)) + ret = delete_block_group_cache(rc->block_group, inode, 0); + else + ret = PTR_ERR(inode); + + if (ret && ret != -ENOENT) + goto out; + + if (!btrfs_fs_incompat(fs_info, REMAP_TREE)) { + rc->data_inode = create_reloc_inode(rc->block_group); + if (IS_ERR(rc->data_inode)) { + ret = PTR_ERR(rc->data_inode); + rc->data_inode = NULL; + goto out; + } + } + + if (verbose) + describe_relocation(rc->block_group); + + btrfs_wait_block_group_reservations(rc->block_group); + btrfs_wait_nocow_writers(rc->block_group); + btrfs_wait_ordered_roots(fs_info, U64_MAX, rc->block_group); + + ret = btrfs_zone_finish(rc->block_group); + WARN_ON(ret && ret != -EAGAIN); + + if (should_relocate_using_remap_tree(bg)) { + if (bg->remap_bytes != 0) { + ret = move_existing_remaps(fs_info, bg, path); + if (ret) + goto out; + } + ret = start_block_group_remapping(fs_info, path, bg); + if (ret) + goto out; + + ret = do_remap_reloc(fs_info, path, rc->block_group); + if (ret) + goto out; + + btrfs_delete_unused_bgs(fs_info); + } else { + ret = do_nonremap_reloc(fs_info, verbose, rc); + } + +out: + if (ret && bg_is_ro) + btrfs_dec_block_group_ro(rc->block_group); + if (!btrfs_fs_incompat(fs_info, REMAP_TREE)) + iput(rc->data_inode); + btrfs_free_path(path); + reloc_chunk_end(fs_info); +out_put_rc: + put_reloc_control(rc); + return ret; +} + +static noinline_for_stack int mark_garbage_root(struct btrfs_root *root) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_trans_handle *trans; + int ret, err; + + trans = btrfs_start_transaction(fs_info->tree_root, 0); + if (IS_ERR(trans)) + return PTR_ERR(trans); + + memset(&root->root_item.drop_progress, 0, + sizeof(root->root_item.drop_progress)); + btrfs_set_root_drop_level(&root->root_item, 0); + btrfs_set_root_refs(&root->root_item, 0); + ret = btrfs_update_root(trans, fs_info->tree_root, + &root->root_key, &root->root_item); + + err = btrfs_end_transaction(trans); + if (err) + return err; + return ret; +} + +static void release_recovered_fs_roots(struct list_head *roots, bool drop_reloc_refs) +{ + struct btrfs_root *root; + struct btrfs_root *next; + + list_for_each_entry_safe(root, next, roots, reloc_dirty_list) { + list_del_init(&root->reloc_dirty_list); + if (drop_reloc_refs) { + struct btrfs_root *reloc_root = root->reloc_root; + + ASSERT(reloc_root); + root->reloc_root = NULL; + btrfs_put_root(reloc_root); + } + btrfs_put_root(root); + } +} + +/* + * recover relocation interrupted by system crash. + * + * this function resumes merging reloc trees with corresponding fs trees. + * this is important for keeping the sharing of tree blocks + */ +int btrfs_recover_relocation(struct btrfs_fs_info *fs_info) +{ + LIST_HEAD(reloc_roots); + LIST_HEAD(recovered_roots); + struct btrfs_key key; + struct btrfs_root *fs_root; + struct btrfs_root *reloc_root; + struct btrfs_path *path; + struct extent_buffer *leaf; + struct reloc_control *rc = NULL; + struct btrfs_trans_handle *trans; + int ret2; + int ret = 0; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + path->reada = READA_BACK; + + key.objectid = BTRFS_TREE_RELOC_OBJECTID; + key.type = BTRFS_ROOT_ITEM_KEY; + key.offset = (u64)-1; + + while (1) { + ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, + path, 0, 0); + if (ret < 0) + goto out; + if (ret > 0) { + if (path->slots[0] == 0) + break; + path->slots[0]--; + } + ret = 0; + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + btrfs_release_path(path); + + if (key.objectid != BTRFS_TREE_RELOC_OBJECTID || + key.type != BTRFS_ROOT_ITEM_KEY) + break; + + reloc_root = btrfs_read_tree_root(fs_info->tree_root, &key); + if (IS_ERR(reloc_root)) { + ret = PTR_ERR(reloc_root); + goto out; + } + + set_bit(BTRFS_ROOT_SHAREABLE, &reloc_root->state); + list_add(&reloc_root->root_list, &reloc_roots); + + if (btrfs_root_refs(&reloc_root->root_item) > 0) { + fs_root = btrfs_get_fs_root(fs_info, + reloc_root->root_key.offset, false); + if (IS_ERR(fs_root)) { + ret = PTR_ERR(fs_root); + if (ret != -ENOENT) + goto out; + ret = mark_garbage_root(reloc_root); + if (ret < 0) + goto out; + ret = 0; + } else { + btrfs_put_root(fs_root); + } + } + + if (key.offset == 0) + break; + + key.offset--; + } + btrfs_release_path(path); + + if (list_empty(&reloc_roots)) + goto out; + + rc = alloc_reloc_control(fs_info); + if (!rc) { + ret = -ENOMEM; + goto out; + } + + rc->extent_root = btrfs_extent_root(fs_info, 0); + if (unlikely(!rc->extent_root)) { + btrfs_err(fs_info, "missing extent root for extent at bytenr 0"); + ret = -EUCLEAN; + goto out; + } + + ret = reloc_chunk_start(fs_info); + if (ret < 0) + goto out_end; + + set_reloc_control(rc); + + trans = btrfs_join_transaction(rc->extent_root); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto out_unset; + } + + rc->merge_reloc_tree = true; + + while (!list_empty(&reloc_roots)) { + reloc_root = list_first_entry(&reloc_roots, struct btrfs_root, root_list); + list_del(&reloc_root->root_list); + + if (btrfs_root_refs(&reloc_root->root_item) == 0) { + list_add_tail(&reloc_root->root_list, + &rc->reloc_roots); + continue; + } + + fs_root = btrfs_get_fs_root(fs_info, reloc_root->root_key.offset, + false); + if (IS_ERR(fs_root)) { + ret = PTR_ERR(fs_root); + list_add_tail(&reloc_root->root_list, &reloc_roots); + btrfs_end_transaction(trans); + goto out_drop_reloc_refs; + } + + ret = __add_reloc_root(reloc_root, rc); + ASSERT(ret != -EEXIST); + if (ret) { + list_add_tail(&reloc_root->root_list, &reloc_roots); + btrfs_put_root(fs_root); + btrfs_end_transaction(trans); + goto out_drop_reloc_refs; + } + ASSERT(list_empty(&fs_root->reloc_dirty_list)); + fs_root->reloc_root = btrfs_grab_root(reloc_root); + list_add_tail(&fs_root->reloc_dirty_list, &recovered_roots); + } + + ret = btrfs_commit_transaction(trans); + if (ret) + goto out_drop_reloc_refs; + release_recovered_fs_roots(&recovered_roots, false); + + ret = merge_reloc_roots(rc); + if (ret) + goto out_unset; + + unset_reloc_control(rc); + + trans = btrfs_join_transaction(rc->extent_root); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto out_clean; + } + ret = btrfs_commit_transaction(trans); +out_clean: + ret2 = clean_dirty_subvols(rc); + if (ret2 < 0 && !ret) + ret = ret2; +out_drop_reloc_refs: + release_recovered_fs_roots(&recovered_roots, true); +out_unset: + unset_reloc_control(rc); + reloc_chunk_end(fs_info); +out_end: + put_reloc_control(rc); +out: + free_reloc_roots(&reloc_roots); + + btrfs_free_path(path); + + if (ret == 0 && !btrfs_fs_incompat(fs_info, REMAP_TREE)) { + /* cleanup orphan inode in data relocation tree */ + fs_root = btrfs_grab_root(fs_info->data_reloc_root); + ASSERT(fs_root); + ret = btrfs_orphan_cleanup(fs_root); + btrfs_put_root(fs_root); + } + return ret; +} + +/* + * helper to add ordered checksum for data relocation. + * + * cloning checksum properly handles the nodatasum extents. + * it also saves CPU time to re-calculate the checksum. + */ +int btrfs_reloc_clone_csums(struct btrfs_ordered_extent *ordered) +{ + struct btrfs_inode *inode = ordered->inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + u64 disk_bytenr = ordered->file_offset + inode->reloc_block_group_start; + struct btrfs_root *csum_root = btrfs_csum_root(fs_info, disk_bytenr); + LIST_HEAD(list); + int ret; + + if (unlikely(!csum_root)) { + btrfs_mark_ordered_extent_error(ordered); + btrfs_err(fs_info, + "missing csum root for extent at bytenr %llu", + disk_bytenr); + return -EUCLEAN; + } + + ret = btrfs_lookup_csums_list(csum_root, disk_bytenr, + disk_bytenr + ordered->num_bytes - 1, + &list, false); + if (ret < 0) { + btrfs_mark_ordered_extent_error(ordered); + return ret; + } + + while (!list_empty(&list)) { + struct btrfs_ordered_sum *sums = + list_first_entry(&list, struct btrfs_ordered_sum, list); + + list_del_init(&sums->list); + + /* + * We need to offset the new_bytenr based on where the csum is. + * We need to do this because we will read in entire prealloc + * extents but we may have written to say the middle of the + * prealloc extent, so we need to make sure the csum goes with + * the right disk offset. + * + * We can do this because the data reloc inode refers strictly + * to the on disk bytes, so we don't have to worry about + * disk_len vs real len like with real inodes since it's all + * disk length. + */ + sums->logical = ordered->disk_bytenr + sums->logical - disk_bytenr; + btrfs_add_ordered_sum(ordered, sums); + } + + return 0; +} + +int btrfs_reloc_cow_block(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + const struct extent_buffer *buf, + struct extent_buffer *cow) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct reloc_control *rc; + struct btrfs_backref_node *node; + bool first_cow = false; + int level; + int ret = 0; + + rc = get_reloc_control(fs_info); + if (!rc) + return 0; + + BUG_ON(rc->stage == UPDATE_DATA_PTRS && btrfs_is_data_reloc_root(root)); + + level = btrfs_header_level(buf); + if (btrfs_header_generation(buf) <= + btrfs_root_last_snapshot(&root->root_item)) + first_cow = true; + + if (btrfs_root_id(root) == BTRFS_TREE_RELOC_OBJECTID && rc->create_reloc_tree) { + WARN_ON(!first_cow && level == 0); + + node = rc->backref_cache.path[level]; + + /* + * If node->bytenr != buf->start and node->new_bytenr != + * buf->start then we've got the wrong backref node for what we + * expected to see here and the cache is incorrect. + */ + if (unlikely(node->bytenr != buf->start && node->new_bytenr != buf->start)) { + btrfs_err(fs_info, +"bytenr %llu was found but our backref cache was expecting %llu or %llu", + buf->start, node->bytenr, node->new_bytenr); + ret = -EUCLEAN; + goto out; + } + + btrfs_backref_drop_node_buffer(node); + refcount_inc(&cow->refs); + node->eb = cow; + node->new_bytenr = cow->start; + + if (!node->pending) { + list_move_tail(&node->list, + &rc->backref_cache.pending[level]); + node->pending = 1; + } + + if (first_cow) + mark_block_processed(rc, node); + + if (first_cow && level > 0) + rc->nodes_relocated += buf->len; + } + + if (level == 0 && first_cow && rc->stage == UPDATE_DATA_PTRS) + ret = replace_file_extents(trans, rc, root, cow); +out: + put_reloc_control(rc); + + return ret; +} + +/* + * called before creating snapshot. it calculates metadata reservation + * required for relocating tree blocks in the snapshot + */ +void btrfs_reloc_pre_snapshot(struct btrfs_pending_snapshot *pending, + u64 *bytes_to_reserve) +{ + struct btrfs_root *root = pending->root; + struct reloc_control *rc = root->fs_info->reloc_ctl; + + if (!rc || !have_reloc_root(root)) + return; + + if (!rc->merge_reloc_tree) + return; + + root = root->reloc_root; + BUG_ON(btrfs_root_refs(&root->root_item) == 0); + /* + * relocation is in the stage of merging trees. the space + * used by merging a reloc tree is twice the size of + * relocated tree nodes in the worst case. half for cowing + * the reloc tree, half for cowing the fs tree. the space + * used by cowing the reloc tree will be freed after the + * tree is dropped. if we create snapshot, cowing the fs + * tree may use more space than it frees. so we need + * reserve extra space. + */ + *bytes_to_reserve += rc->nodes_relocated; +} + +/* + * called after snapshot is created. migrate block reservation + * and create reloc root for the newly created snapshot + * + * This is similar to btrfs_init_reloc_root(), we come out of here with two + * references held on the reloc_root, one for root->reloc_root and one for + * rc->reloc_roots. + */ +int btrfs_reloc_post_snapshot(struct btrfs_trans_handle *trans, + struct btrfs_pending_snapshot *pending) +{ + struct btrfs_root *root = pending->root; + struct btrfs_root *reloc_root; + struct btrfs_root *new_root; + struct reloc_control *rc; + int ret = 0; + + rc = get_reloc_control(trans->fs_info); + if (!rc) + return 0; + + if (!have_reloc_root(root)) + goto out; + + rc->merging_rsv_size += rc->nodes_relocated; + + if (rc->merge_reloc_tree) { + ret = btrfs_block_rsv_migrate(&pending->block_rsv, + rc->block_rsv, + rc->nodes_relocated, true); + if (ret) + goto out; + } + + new_root = pending->snap; + reloc_root = create_reloc_root(trans, root->reloc_root, btrfs_root_id(new_root)); + if (IS_ERR(reloc_root)) { + ret = PTR_ERR(reloc_root); + goto out; + } + + ret = __add_reloc_root(reloc_root, rc); + ASSERT(ret != -EEXIST); + if (ret) { + /* Pairs with create_reloc_root */ + btrfs_put_root(reloc_root); + goto out; + } + new_root->reloc_root = btrfs_grab_root(reloc_root); +out: + put_reloc_control(rc); + + return ret; +} + +/* + * Get the current bytenr for the block group which is being relocated. + * + * Return U64_MAX if no running relocation. + */ +u64 btrfs_get_reloc_bg_bytenr(struct btrfs_fs_info *fs_info) +{ + u64 logical = U64_MAX; + + mutex_lock(&fs_info->reloc_mutex); + if (fs_info->reloc_ctl && fs_info->reloc_ctl->block_group) + logical = fs_info->reloc_ctl->block_group->start; + mutex_unlock(&fs_info->reloc_mutex); + + return logical; +} + +static int insert_remap_item(struct btrfs_trans_handle *trans, struct btrfs_path *path, + u64 old_addr, u64 length, u64 new_addr) +{ + int ret; + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_key key; + struct btrfs_remap_item remap = { 0 }; + + if (old_addr == new_addr) { + /* Add new identity remap item. */ + key.objectid = old_addr; + key.type = BTRFS_IDENTITY_REMAP_KEY; + key.offset = length; + + ret = btrfs_insert_empty_item(trans, fs_info->remap_root, path, + &key, 0); + if (ret) + return ret; + } else { + /* Add new remap item. */ + key.objectid = old_addr; + key.type = BTRFS_REMAP_KEY; + key.offset = length; + + ret = btrfs_insert_empty_item(trans, fs_info->remap_root, + path, &key, sizeof(struct btrfs_remap_item)); + if (ret) + return ret; + + btrfs_set_stack_remap_address(&remap, new_addr); + + write_extent_buffer(path->nodes[0], &remap, + btrfs_item_ptr_offset(path->nodes[0], path->slots[0]), + sizeof(struct btrfs_remap_item)); + + btrfs_release_path(path); + + /* Add new backref item. */ + key.objectid = new_addr; + key.type = BTRFS_REMAP_BACKREF_KEY; + key.offset = length; + + ret = btrfs_insert_empty_item(trans, fs_info->remap_root, + path, &key, + sizeof(struct btrfs_remap_item)); + if (ret) + return ret; + + btrfs_set_stack_remap_address(&remap, old_addr); + + write_extent_buffer(path->nodes[0], &remap, + btrfs_item_ptr_offset(path->nodes[0], path->slots[0]), + sizeof(struct btrfs_remap_item)); + } + + btrfs_release_path(path); + + return 0; +} + +/* + * Punch a hole in the remap item or identity remap item pointed to by path, + * for the range [hole_start, hole_start + hole_length). + */ +static int remove_range_from_remap_tree(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + struct btrfs_block_group *bg, + u64 hole_start, u64 hole_length) +{ + int ret; + struct btrfs_fs_info *fs_info = trans->fs_info; + struct extent_buffer *leaf = path->nodes[0]; + struct btrfs_key key; + u64 hole_end, new_addr, remap_start, remap_length, remap_end; + u64 overlap_length; + bool is_identity_remap; + int identity_count_delta = 0; + + hole_end = hole_start + hole_length; + + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + + is_identity_remap = (key.type == BTRFS_IDENTITY_REMAP_KEY); + + remap_start = key.objectid; + remap_length = key.offset; + remap_end = remap_start + remap_length; + + if (is_identity_remap) { + new_addr = remap_start; + } else { + struct btrfs_remap_item *remap_ptr; + + remap_ptr = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_remap_item); + new_addr = btrfs_remap_address(leaf, remap_ptr); + } + + /* Delete old item. */ + ret = btrfs_del_item(trans, fs_info->remap_root, path); + btrfs_release_path(path); + if (ret) + return ret; + + if (is_identity_remap) { + identity_count_delta = -1; + } else { + /* Remove backref. */ + key.objectid = new_addr; + key.type = BTRFS_REMAP_BACKREF_KEY; + key.offset = remap_length; + + ret = btrfs_search_slot(trans, fs_info->remap_root, &key, path, -1, 1); + if (ret) { + if (ret == 1) { + btrfs_release_path(path); + ret = -ENOENT; + } + return ret; + } + + ret = btrfs_del_item(trans, fs_info->remap_root, path); + + btrfs_release_path(path); + + if (ret) + return ret; + } + + /* If hole_start > remap_start, re-add the start of the remap item. */ + if (hole_start > remap_start) { + ret = insert_remap_item(trans, path, remap_start, + hole_start - remap_start, new_addr); + if (ret) + return ret; + + if (is_identity_remap) + identity_count_delta++; + } + + /* If hole_end < remap_end, re-add the end of the remap item. */ + if (hole_end < remap_end) { + ret = insert_remap_item(trans, path, hole_end, + remap_end - hole_end, + hole_end - remap_start + new_addr); + if (ret) + return ret; + + if (is_identity_remap) + identity_count_delta++; + } + + if (identity_count_delta != 0) + adjust_identity_remap_count(trans, bg, identity_count_delta); + + overlap_length = min_t(u64, hole_end, remap_end) - + max_t(u64, hole_start, remap_start); + + if (!is_identity_remap) { + struct btrfs_block_group *dest_bg; + + dest_bg = btrfs_lookup_block_group(fs_info, new_addr); + if (unlikely(!dest_bg)) + return -EUCLEAN; + + adjust_block_group_remap_bytes(trans, dest_bg, -overlap_length); + btrfs_put_block_group(dest_bg); + ret = btrfs_add_to_free_space_tree(trans, + hole_start - remap_start + new_addr, + overlap_length); + if (ret) + return ret; + } + + ret = overlap_length; + + return ret; +} + +/* + * Return 1 if remove_range_from_remap_tree() has been called successfully, + * 0 if block group wasn't remapped, and a negative number on error. + */ +int btrfs_remove_extent_from_remap_tree(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + u64 bytenr, u64 num_bytes) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_key key, found_key; + struct extent_buffer *leaf; + struct btrfs_block_group *bg; + int ret, length; + + if (!(btrfs_super_incompat_flags(fs_info->super_copy) & + BTRFS_FEATURE_INCOMPAT_REMAP_TREE)) + return 0; + + bg = btrfs_lookup_block_group(fs_info, bytenr); + if (!bg) + return 0; + + mutex_lock(&fs_info->remap_mutex); + + if (!(bg->flags & BTRFS_BLOCK_GROUP_REMAPPED)) { + mutex_unlock(&fs_info->remap_mutex); + btrfs_put_block_group(bg); + return 0; + } + + do { + key.objectid = bytenr; + key.type = (u8)-1; + key.offset = (u64)-1; + + ret = btrfs_search_slot(trans, fs_info->remap_root, &key, path, -1, 1); + if (ret < 0) + goto end; + + leaf = path->nodes[0]; + if (path->slots[0] == 0) { + ret = -ENOENT; + goto end; + } + + path->slots[0]--; + + btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); + + if (found_key.type != BTRFS_IDENTITY_REMAP_KEY && + found_key.type != BTRFS_REMAP_KEY) { + ret = -ENOENT; + goto end; + } + + if (bytenr < found_key.objectid || + bytenr >= found_key.objectid + found_key.offset) { + ret = -ENOENT; + goto end; + } + + length = remove_range_from_remap_tree(trans, path, bg, bytenr, num_bytes); + if (length < 0) { + ret = length; + goto end; + } + + bytenr += length; + num_bytes -= length; + } while (num_bytes > 0); + + ret = 1; + +end: + mutex_unlock(&fs_info->remap_mutex); + + btrfs_put_block_group(bg); + btrfs_release_path(path); + + return ret; +} diff --git a/fs/btrfs/relocation.h b/fs/btrfs/relocation.h new file mode 100644 index 000000000..bb7a86e7d --- /dev/null +++ b/fs/btrfs/relocation.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_RELOCATION_H +#define BTRFS_RELOCATION_H + +#include + +struct extent_buffer; +struct btrfs_fs_info; +struct btrfs_root; +struct btrfs_trans_handle; +struct btrfs_ordered_extent; +struct btrfs_pending_snapshot; + +static inline bool should_relocate_using_remap_tree(const struct btrfs_block_group *bg) +{ + if (!btrfs_fs_incompat(bg->fs_info, REMAP_TREE)) + return false; + + if (bg->flags & (BTRFS_BLOCK_GROUP_SYSTEM | BTRFS_BLOCK_GROUP_METADATA_REMAP)) + return false; + + return true; +} + +int btrfs_relocate_block_group(struct btrfs_fs_info *fs_info, u64 group_start, + bool verbose); +int btrfs_init_reloc_root(struct btrfs_trans_handle *trans, struct btrfs_root *root); +int btrfs_update_reloc_root(struct btrfs_trans_handle *trans, + struct btrfs_root *root); +int btrfs_recover_relocation(struct btrfs_fs_info *fs_info); +int btrfs_reloc_clone_csums(struct btrfs_ordered_extent *ordered); +int btrfs_reloc_cow_block(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + const struct extent_buffer *buf, + struct extent_buffer *cow); +void btrfs_reloc_pre_snapshot(struct btrfs_pending_snapshot *pending, + u64 *bytes_to_reserve); +int btrfs_reloc_post_snapshot(struct btrfs_trans_handle *trans, + struct btrfs_pending_snapshot *pending); +int btrfs_should_cancel_balance(const struct btrfs_fs_info *fs_info); +struct btrfs_root *find_reloc_root(struct btrfs_fs_info *fs_info, u64 bytenr); +bool btrfs_should_ignore_reloc_root(const struct btrfs_root *root); +u64 btrfs_get_reloc_bg_bytenr(struct btrfs_fs_info *fs_info); +int btrfs_translate_remap(struct btrfs_fs_info *fs_info, u64 *logical, u64 *length); +int btrfs_remove_extent_from_remap_tree(struct btrfs_trans_handle *trans, + struct btrfs_path *path, + u64 bytenr, u64 num_bytes); +int btrfs_last_identity_remap_gone(struct btrfs_chunk_map *chunk_map, + struct btrfs_block_group *bg); + +#endif diff --git a/fs/btrfs/root-tree.c b/fs/btrfs/root-tree.c new file mode 100644 index 000000000..2e4c3efbd --- /dev/null +++ b/fs/btrfs/root-tree.c @@ -0,0 +1,528 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#include +#include +#include "ctree.h" +#include "fs.h" +#include "messages.h" +#include "transaction.h" +#include "disk-io.h" +#include "qgroup.h" +#include "space-info.h" +#include "accessors.h" +#include "root-tree.h" +#include "orphan.h" + +/* + * Read a root item from the tree. In case we detect a root item smaller then + * sizeof(root_item), we know it's an old version of the root structure and + * initialize all new fields to zero. The same happens if we detect mismatching + * generation numbers as then we know the root was once mounted with an older + * kernel that was not aware of the root item structure change. + */ +static void btrfs_read_root_item(struct extent_buffer *eb, int slot, + struct btrfs_root_item *item) +{ + u32 len; + bool need_reset = false; + + len = btrfs_item_size(eb, slot); + read_extent_buffer(eb, item, btrfs_item_ptr_offset(eb, slot), + min_t(u32, len, sizeof(*item))); + if (len < sizeof(*item)) + need_reset = true; + if (!need_reset && btrfs_root_generation(item) + != btrfs_root_generation_v2(item)) { + if (btrfs_root_generation_v2(item) != 0) { + btrfs_warn(eb->fs_info, + "mismatching generation and generation_v2 found in root item. This root was probably mounted with an older kernel. Resetting all new fields."); + } + need_reset = true; + } + if (need_reset) { + /* Clear all members from generation_v2 onwards. */ + memset_startat(item, 0, generation_v2); + generate_random_guid(item->uuid); + } +} + +/* + * Lookup the root by the key. + * + * root: the root of the root tree + * search_key: the key to search + * path: the path we search + * root_item: the root item of the tree we look for + * root_key: the root key of the tree we look for + * + * If ->offset of 'search_key' is -1ULL, it means we are not sure the offset + * of the search key, just lookup the root with the highest offset for a + * given objectid. + * + * If we find something return 0, otherwise > 0, < 0 on error. + */ +int btrfs_find_root(struct btrfs_root *root, const struct btrfs_key *search_key, + struct btrfs_path *path, struct btrfs_root_item *root_item, + struct btrfs_key *root_key) +{ + struct btrfs_key found_key; + struct extent_buffer *l; + int ret; + int slot; + + ret = btrfs_search_slot(NULL, root, search_key, path, 0, 0); + if (ret < 0) + return ret; + + if (search_key->offset != -1ULL) { /* the search key is exact */ + if (ret > 0) + goto out; + } else { + /* + * Key with offset -1 found, there would have to exist a root + * with such id, but this is out of the valid range. + */ + if (unlikely(ret == 0)) { + ret = -EUCLEAN; + goto out; + } + if (path->slots[0] == 0) + goto out; + path->slots[0]--; + ret = 0; + } + + l = path->nodes[0]; + slot = path->slots[0]; + + btrfs_item_key_to_cpu(l, &found_key, slot); + if (found_key.objectid != search_key->objectid || + found_key.type != BTRFS_ROOT_ITEM_KEY) { + ret = 1; + goto out; + } + + if (root_item) + btrfs_read_root_item(l, slot, root_item); + if (root_key) + memcpy(root_key, &found_key, sizeof(found_key)); +out: + btrfs_release_path(path); + return ret; +} + +void btrfs_set_root_node(struct btrfs_root_item *item, + struct extent_buffer *node) +{ + btrfs_set_root_bytenr(item, node->start); + btrfs_set_root_level(item, btrfs_header_level(node)); + btrfs_set_root_generation(item, btrfs_header_generation(node)); +} + +/* + * copy the data in 'item' into the btree + */ +int btrfs_update_root(struct btrfs_trans_handle *trans, struct btrfs_root + *root, struct btrfs_key *key, struct btrfs_root_item + *item) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + BTRFS_PATH_AUTO_FREE(path); + struct extent_buffer *l; + int ret; + int slot; + unsigned long ptr; + u32 old_len; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + ret = btrfs_search_slot(trans, root, key, path, 0, 1); + if (ret < 0) + return ret; + + if (unlikely(ret > 0)) { + btrfs_crit(fs_info, + "unable to find root key " BTRFS_KEY_FMT " in tree %llu", + BTRFS_KEY_FMT_VALUE(key), btrfs_root_id(root)); + ret = -EUCLEAN; + btrfs_abort_transaction(trans, ret); + return ret; + } + + l = path->nodes[0]; + slot = path->slots[0]; + ptr = btrfs_item_ptr_offset(l, slot); + old_len = btrfs_item_size(l, slot); + + /* + * If this is the first time we update the root item which originated + * from an older kernel, we need to enlarge the item size to make room + * for the added fields. + */ + if (old_len < sizeof(*item)) { + btrfs_release_path(path); + ret = btrfs_search_slot(trans, root, key, path, + -1, 1); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + + ret = btrfs_del_item(trans, root, path); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + btrfs_release_path(path); + ret = btrfs_insert_empty_item(trans, root, path, + key, sizeof(*item)); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + l = path->nodes[0]; + slot = path->slots[0]; + ptr = btrfs_item_ptr_offset(l, slot); + } + + /* + * Update generation_v2 so at the next mount we know the new root + * fields are valid. + */ + btrfs_set_root_generation_v2(item, btrfs_root_generation(item)); + + write_extent_buffer(l, item, ptr, sizeof(*item)); + return ret; +} + +int btrfs_insert_root(struct btrfs_trans_handle *trans, struct btrfs_root *root, + const struct btrfs_key *key, struct btrfs_root_item *item) +{ + /* + * Make sure generation v1 and v2 match. See update_root for details. + */ + btrfs_set_root_generation_v2(item, btrfs_root_generation(item)); + return btrfs_insert_item(trans, root, key, item, sizeof(*item)); +} + +int btrfs_find_orphan_roots(struct btrfs_fs_info *fs_info) +{ + struct btrfs_root *tree_root = fs_info->tree_root; + struct extent_buffer *leaf; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + struct btrfs_root *root; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = BTRFS_ORPHAN_OBJECTID; + key.type = BTRFS_ORPHAN_ITEM_KEY; + key.offset = 0; + + while (1) { + u64 root_objectid; + int ret; + + ret = btrfs_search_slot(NULL, tree_root, &key, path, 0, 0); + if (ret < 0) + return ret; + + leaf = path->nodes[0]; + if (path->slots[0] >= btrfs_header_nritems(leaf)) { + ret = btrfs_next_leaf(tree_root, path); + if (ret < 0) + return ret; + else if (ret > 0) + return 0; + leaf = path->nodes[0]; + } + + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + btrfs_release_path(path); + + if (key.objectid != BTRFS_ORPHAN_OBJECTID || + key.type != BTRFS_ORPHAN_ITEM_KEY) + return 0; + + root_objectid = key.offset; + key.offset++; + + root = btrfs_get_fs_root(fs_info, root_objectid, false); + ret = PTR_ERR_OR_ZERO(root); + if (ret && ret != -ENOENT) { + return ret; + } else if (ret == -ENOENT) { + struct btrfs_trans_handle *trans; + + trans = btrfs_join_transaction(tree_root); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + btrfs_err(fs_info, + "failed to join transaction to delete orphan item: %pe", + ERR_PTR(ret)); + return ret; + } + ret = btrfs_del_orphan_item(trans, tree_root, root_objectid); + btrfs_end_transaction(trans); + if (ret) { + btrfs_err(fs_info, + "failed to delete root orphan item: %pe", ERR_PTR(ret)); + return ret; + } + continue; + } + + WARN_ON(!test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state)); + if (btrfs_root_refs(&root->root_item) == 0) { + struct btrfs_key drop_key; + + btrfs_disk_key_to_cpu(&drop_key, &root->root_item.drop_progress); + /* + * If we have a non-zero drop_progress then we know we + * made it partly through deleting this snapshot, and + * thus we need to make sure we block any balance from + * happening until this snapshot is completely dropped. + */ + if (drop_key.objectid != 0 || drop_key.type != 0 || + drop_key.offset != 0) { + set_bit(BTRFS_FS_UNFINISHED_DROPS, &fs_info->flags); + set_bit(BTRFS_ROOT_UNFINISHED_DROP, &root->state); + } + + set_bit(BTRFS_ROOT_DEAD_TREE, &root->state); + btrfs_add_dead_root(root); + } + btrfs_put_root(root); + } + + return 0; +} + +/* drop the root item for 'key' from the tree root */ +int btrfs_del_root(struct btrfs_trans_handle *trans, + const struct btrfs_key *key) +{ + struct btrfs_root *root = trans->fs_info->tree_root; + BTRFS_PATH_AUTO_FREE(path); + int ret; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + ret = btrfs_search_slot(trans, root, key, path, -1, 1); + if (ret < 0) + return ret; + if (unlikely(ret > 0)) + /* The root must exist but we did not find it by the key. */ + return -EUCLEAN; + + return btrfs_del_item(trans, root, path); +} + +int btrfs_del_root_ref(struct btrfs_trans_handle *trans, u64 root_id, + u64 ref_id, u64 dirid, u64 *sequence, + const struct fscrypt_str *name) +{ + struct btrfs_root *tree_root = trans->fs_info->tree_root; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_root_ref *ref; + struct extent_buffer *leaf; + struct btrfs_key key; + unsigned long ptr; + int ret; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = root_id; + key.type = BTRFS_ROOT_BACKREF_KEY; + key.offset = ref_id; +again: + ret = btrfs_search_slot(trans, tree_root, &key, path, -1, 1); + if (ret < 0) { + return ret; + } else if (ret == 0) { + leaf = path->nodes[0]; + ref = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_root_ref); + ptr = (unsigned long)(ref + 1); + if ((btrfs_root_ref_dirid(leaf, ref) != dirid) || + (btrfs_root_ref_name_len(leaf, ref) != name->len) || + memcmp_extent_buffer(leaf, name->name, ptr, name->len)) + return -ENOENT; + + *sequence = btrfs_root_ref_sequence(leaf, ref); + + ret = btrfs_del_item(trans, tree_root, path); + if (ret) + return ret; + } else { + return -ENOENT; + } + + if (key.type == BTRFS_ROOT_BACKREF_KEY) { + btrfs_release_path(path); + key.objectid = ref_id; + key.type = BTRFS_ROOT_REF_KEY; + key.offset = root_id; + goto again; + } + + return ret; +} + +/* + * add a btrfs_root_ref item. type is either BTRFS_ROOT_REF_KEY + * or BTRFS_ROOT_BACKREF_KEY. + * + * The dirid, sequence, name and name_len refer to the directory entry + * that is referencing the root. + * + * For a forward ref, the root_id is the id of the tree referencing + * the root and ref_id is the id of the subvol or snapshot. + * + * For a back ref the root_id is the id of the subvol or snapshot and + * ref_id is the id of the tree referencing it. + * + * Will return 0, -ENOMEM, or anything from the CoW path + */ +int btrfs_add_root_ref(struct btrfs_trans_handle *trans, u64 root_id, + u64 ref_id, u64 dirid, u64 sequence, + const struct fscrypt_str *name) +{ + struct btrfs_root *tree_root = trans->fs_info->tree_root; + struct btrfs_key key; + int ret; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_root_ref *ref; + struct extent_buffer *leaf; + unsigned long ptr; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = root_id; + key.type = BTRFS_ROOT_BACKREF_KEY; + key.offset = ref_id; +again: + ret = btrfs_insert_empty_item(trans, tree_root, path, &key, + sizeof(*ref) + name->len); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + + leaf = path->nodes[0]; + ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref); + btrfs_set_root_ref_dirid(leaf, ref, dirid); + btrfs_set_root_ref_sequence(leaf, ref, sequence); + btrfs_set_root_ref_name_len(leaf, ref, name->len); + ptr = (unsigned long)(ref + 1); + write_extent_buffer(leaf, name->name, ptr, name->len); + + if (key.type == BTRFS_ROOT_BACKREF_KEY) { + btrfs_release_path(path); + key.objectid = ref_id; + key.type = BTRFS_ROOT_REF_KEY; + key.offset = root_id; + goto again; + } + + return 0; +} + +/* + * Old btrfs forgets to init root_item->flags and root_item->byte_limit + * for subvolumes. To work around this problem, we steal a bit from + * root_item->inode_item->flags, and use it to indicate if those fields + * have been properly initialized. + */ +void btrfs_check_and_init_root_item(struct btrfs_root_item *root_item) +{ + u64 inode_flags = btrfs_stack_inode_flags(&root_item->inode); + + if (!(inode_flags & BTRFS_INODE_ROOT_ITEM_INIT)) { + inode_flags |= BTRFS_INODE_ROOT_ITEM_INIT; + btrfs_set_stack_inode_flags(&root_item->inode, inode_flags); + btrfs_set_root_flags(root_item, 0); + btrfs_set_root_limit(root_item, 0); + } +} + +void btrfs_update_root_times(struct btrfs_trans_handle *trans, + struct btrfs_root *root) +{ + struct btrfs_root_item *item = &root->root_item; + struct timespec64 ct; + + ktime_get_real_ts64(&ct); + spin_lock(&root->root_item_lock); + btrfs_set_root_ctransid(item, trans->transid); + btrfs_set_stack_timespec_sec(&item->ctime, ct.tv_sec); + btrfs_set_stack_timespec_nsec(&item->ctime, ct.tv_nsec); + spin_unlock(&root->root_item_lock); +} + +/* + * Reserve space for subvolume operation. + * + * root: the root of the parent directory + * rsv: block reservation + * items: the number of items that we need do reservation + * use_global_rsv: allow fallback to the global block reservation + * + * This function is used to reserve the space for snapshot/subvolume + * creation and deletion. Those operations are different with the + * common file/directory operations, they change two fs/file trees + * and root tree, the number of items that the qgroup reserves is + * different with the free space reservation. So we can not use + * the space reservation mechanism in start_transaction(). + */ +int btrfs_subvolume_reserve_metadata(struct btrfs_root *root, + struct btrfs_block_rsv *rsv, int items, + bool use_global_rsv) +{ + u64 qgroup_num_bytes = 0; + u64 num_bytes; + int ret; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; + + if (btrfs_qgroup_enabled(fs_info)) { + /* One for parent inode, two for dir entries */ + qgroup_num_bytes = 3 * fs_info->nodesize; + ret = btrfs_qgroup_reserve_meta_prealloc(root, + qgroup_num_bytes, true, + false); + if (ret) + return ret; + } + + num_bytes = btrfs_calc_insert_metadata_size(fs_info, items); + rsv->space_info = btrfs_find_space_info(fs_info, + BTRFS_BLOCK_GROUP_METADATA); + ret = btrfs_block_rsv_add(fs_info, rsv, num_bytes, + BTRFS_RESERVE_FLUSH_ALL); + + if (ret == -ENOSPC && use_global_rsv) + ret = btrfs_block_rsv_migrate(global_rsv, rsv, num_bytes, true); + + if (ret && qgroup_num_bytes) + btrfs_qgroup_free_meta_prealloc(root, qgroup_num_bytes); + + if (!ret) { + spin_lock(&rsv->lock); + rsv->qgroup_rsv_reserved += qgroup_num_bytes; + spin_unlock(&rsv->lock); + } + return ret; +} diff --git a/fs/btrfs/root-tree.h b/fs/btrfs/root-tree.h new file mode 100644 index 000000000..8f5739e73 --- /dev/null +++ b/fs/btrfs/root-tree.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_ROOT_TREE_H +#define BTRFS_ROOT_TREE_H + +#include + +struct fscrypt_str; +struct extent_buffer; +struct btrfs_key; +struct btrfs_root; +struct btrfs_root_item; +struct btrfs_path; +struct btrfs_fs_info; +struct btrfs_block_rsv; +struct btrfs_trans_handle; + +int btrfs_subvolume_reserve_metadata(struct btrfs_root *root, + struct btrfs_block_rsv *rsv, + int nitems, bool use_global_rsv); +int btrfs_add_root_ref(struct btrfs_trans_handle *trans, u64 root_id, + u64 ref_id, u64 dirid, u64 sequence, + const struct fscrypt_str *name); +int btrfs_del_root_ref(struct btrfs_trans_handle *trans, u64 root_id, + u64 ref_id, u64 dirid, u64 *sequence, + const struct fscrypt_str *name); +int btrfs_del_root(struct btrfs_trans_handle *trans, const struct btrfs_key *key); +int btrfs_insert_root(struct btrfs_trans_handle *trans, struct btrfs_root *root, + const struct btrfs_key *key, + struct btrfs_root_item *item); +int btrfs_update_root(struct btrfs_trans_handle *trans, struct btrfs_root *root, + struct btrfs_key *key, struct btrfs_root_item *item); +int btrfs_find_root(struct btrfs_root *root, const struct btrfs_key *search_key, + struct btrfs_path *path, struct btrfs_root_item *root_item, + struct btrfs_key *root_key); +int btrfs_find_orphan_roots(struct btrfs_fs_info *fs_info); +void btrfs_set_root_node(struct btrfs_root_item *item, + struct extent_buffer *node); +void btrfs_check_and_init_root_item(struct btrfs_root_item *item); +void btrfs_update_root_times(struct btrfs_trans_handle *trans, struct btrfs_root *root); + +#endif diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c new file mode 100644 index 000000000..f209e75f0 --- /dev/null +++ b/fs/btrfs/scrub.c @@ -0,0 +1,3293 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2011, 2012 STRATO. All rights reserved. + */ + +#include +#include +#include +#include "ctree.h" +#include "discard.h" +#include "volumes.h" +#include "disk-io.h" +#include "ordered-data.h" +#include "transaction.h" +#include "backref.h" +#include "extent_io.h" +#include "dev-replace.h" +#include "raid56.h" +#include "block-group.h" +#include "zoned.h" +#include "fs.h" +#include "accessors.h" +#include "file-item.h" +#include "scrub.h" +#include "raid-stripe-tree.h" + +/* + * This is only the first step towards a full-features scrub. It reads all + * extent and super block and verifies the checksums. In case a bad checksum + * is found or the extent cannot be read, good data will be written back if + * any can be found. + * + * Future enhancements: + * - In case an unrepairable extent is encountered, track which files are + * affected and report them + * - track and record media errors, throw out bad devices + * - add a mode to also read unallocated space + */ + +struct scrub_ctx; + +/* + * The following value only influences the performance. + * + * This determines how many stripes would be submitted in one go, + * which is 512KiB (BTRFS_STRIPE_LEN * SCRUB_STRIPES_PER_GROUP). + */ +#define SCRUB_STRIPES_PER_GROUP 8 + +/* + * How many groups we have for each sctx. + * + * This would be 8M per device, the same value as the old scrub in-flight bios + * size limit. + */ +#define SCRUB_GROUPS_PER_SCTX 16 + +#define SCRUB_TOTAL_STRIPES (SCRUB_GROUPS_PER_SCTX * SCRUB_STRIPES_PER_GROUP) + +/* Represent one sector and its needed info to verify the content. */ +struct scrub_sector_verification { + union { + /* + * Csum pointer for data csum verification. Should point to a + * sector csum inside scrub_stripe::csums. + * + * NULL if this data sector has no csum. + */ + u8 *csum; + + /* + * Extra info for metadata verification. All sectors inside a + * tree block share the same generation. + */ + u64 generation; + }; +}; + +enum scrub_stripe_flags { + /* Set when @mirror_num, @dev, @physical and @logical are set. */ + SCRUB_STRIPE_FLAG_INITIALIZED, + + /* Set when the read-repair is finished. */ + SCRUB_STRIPE_FLAG_REPAIR_DONE, + + /* + * Set for data stripes if it's triggered from P/Q stripe. + * During such scrub, we should not report errors in data stripes, nor + * update the accounting. + */ + SCRUB_STRIPE_FLAG_NO_REPORT, +}; + +/* + * We have multiple bitmaps for one scrub_stripe. + * However each bitmap has at most (BTRFS_STRIPE_LEN / blocksize) bits, + * which is normally 16, and much smaller than BITS_PER_LONG (32 or 64). + * + * So to reduce memory usage for each scrub_stripe, we pack those bitmaps + * into a larger one. + * + * These enum records where the sub-bitmap are inside the larger one. + * Each subbitmap starts at scrub_bitmap_nr_##name * nr_sectors bit. + */ +enum { + /* Which blocks are covered by extent items. */ + scrub_bitmap_nr_has_extent = 0, + + /* Which blocks are metadata. */ + scrub_bitmap_nr_is_metadata, + + /* + * Which blocks have errors, including IO, csum, and metadata + * errors. + * This sub-bitmap is the OR results of the next few error related + * sub-bitmaps. + */ + scrub_bitmap_nr_error, + scrub_bitmap_nr_io_error, + scrub_bitmap_nr_csum_error, + scrub_bitmap_nr_meta_error, + scrub_bitmap_nr_meta_gen_error, + scrub_bitmap_nr_last, +}; + +/* + * Represent one contiguous range with a length of BTRFS_STRIPE_LEN. + */ +struct scrub_stripe { + struct scrub_ctx *sctx; + struct btrfs_block_group *bg; + struct scrub_sector_verification *sectors; + struct btrfs_device *dev; + + void *buffer; + + u64 logical; + u64 physical; + + u16 mirror_num; + + /* Should be BTRFS_STRIPE_LEN / sectorsize. */ + u16 nr_sectors; + + /* + * How many data/meta extents are in this stripe. Only for scrub status + * reporting purposes. + */ + u16 nr_data_extents; + u16 nr_meta_extents; + + atomic_t pending_io; + wait_queue_head_t io_wait; + wait_queue_head_t repair_wait; + + /* + * Indicate the states of the stripe. Bits are defined in + * scrub_stripe_flags enum. + */ + unsigned long state; + + /* The large bitmap contains all the sub-bitmaps. */ + unsigned long bitmaps[BITS_TO_LONGS(scrub_bitmap_nr_last * + (BTRFS_STRIPE_LEN / BTRFS_MIN_BLOCKSIZE))]; + + /* + * For writeback (repair or replace) error reporting. + * This one is protected by a spinlock, thus can not be packed into + * the larger bitmap. + */ + unsigned long write_error_bitmap; + + /* Writeback can be concurrent, thus we need to protect the bitmap. */ + spinlock_t write_error_lock; + + /* + * Checksum for the whole stripe if this stripe is inside a data block + * group. + */ + u8 *csums; + + struct work_struct work; +}; + +struct scrub_ctx { + struct scrub_stripe stripes[SCRUB_TOTAL_STRIPES]; + struct scrub_stripe *raid56_data_stripes; + struct btrfs_fs_info *fs_info; + struct btrfs_path extent_path; + struct btrfs_path csum_path; + int first_free; + int cur_stripe; + atomic_t cancel_req; + int readonly; + + /* State of IO submission throttling affecting the associated device */ + ktime_t throttle_deadline; + u64 throttle_sent; + + bool is_dev_replace; + u64 write_pointer; + + struct mutex wr_lock; + struct btrfs_device *wr_tgtdev; + + /* + * statistics + */ + struct btrfs_scrub_progress stat; + spinlock_t stat_lock; + + /* + * Use a ref counter to avoid use-after-free issues. Scrub workers + * decrement bios_in_flight and workers_pending and then do a wakeup + * on the list_wait wait queue. We must ensure the main scrub task + * doesn't free the scrub context before or while the workers are + * doing the wakeup() call. + */ + refcount_t refs; +}; + +static_assert(BTRFS_STRIPE_LEN >= PAGE_SIZE); +static_assert(IS_ALIGNED(BTRFS_STRIPE_LEN, PAGE_SIZE)); + +#define scrub_calc_start_bit(stripe, name, block_nr) \ +({ \ + unsigned int __start_bit; \ + \ + ASSERT(block_nr < stripe->nr_sectors, \ + "nr_sectors=%u block_nr=%u", stripe->nr_sectors, block_nr); \ + __start_bit = scrub_bitmap_nr_##name * stripe->nr_sectors + block_nr; \ + __start_bit; \ +}) + +#define IMPLEMENT_SCRUB_BITMAP_OPS(name) \ +static inline void scrub_bitmap_set_##name(struct scrub_stripe *stripe, \ + unsigned int block_nr, \ + unsigned int nr_blocks) \ +{ \ + const unsigned int start_bit = scrub_calc_start_bit(stripe, \ + name, block_nr); \ + \ + bitmap_set(stripe->bitmaps, start_bit, nr_blocks); \ +} \ +static inline void scrub_bitmap_clear_##name(struct scrub_stripe *stripe, \ + unsigned int block_nr, \ + unsigned int nr_blocks) \ +{ \ + const unsigned int start_bit = scrub_calc_start_bit(stripe, name, \ + block_nr); \ + \ + bitmap_clear(stripe->bitmaps, start_bit, nr_blocks); \ +} \ +static inline bool scrub_bitmap_test_bit_##name(struct scrub_stripe *stripe, \ + unsigned int block_nr) \ +{ \ + const unsigned int start_bit = scrub_calc_start_bit(stripe, name, \ + block_nr); \ + \ + return test_bit(start_bit, stripe->bitmaps); \ +} \ +static inline void scrub_bitmap_set_bit_##name(struct scrub_stripe *stripe, \ + unsigned int block_nr) \ +{ \ + const unsigned int start_bit = scrub_calc_start_bit(stripe, name, \ + block_nr); \ + \ + set_bit(start_bit, stripe->bitmaps); \ +} \ +static inline void scrub_bitmap_clear_bit_##name(struct scrub_stripe *stripe, \ + unsigned int block_nr) \ +{ \ + const unsigned int start_bit = scrub_calc_start_bit(stripe, name, \ + block_nr); \ + \ + clear_bit(start_bit, stripe->bitmaps); \ +} \ +static inline unsigned long scrub_bitmap_read_##name(struct scrub_stripe *stripe) \ +{ \ + const unsigned int nr_blocks = stripe->nr_sectors; \ + \ + ASSERT(nr_blocks > 0 && nr_blocks <= BITS_PER_LONG, \ + "nr_blocks=%u BITS_PER_LONG=%u", \ + nr_blocks, BITS_PER_LONG); \ + \ + return bitmap_read(stripe->bitmaps, nr_blocks * scrub_bitmap_nr_##name, \ + stripe->nr_sectors); \ +} \ +static inline bool scrub_bitmap_empty_##name(struct scrub_stripe *stripe) \ +{ \ + unsigned long bitmap = scrub_bitmap_read_##name(stripe); \ + \ + return bitmap_empty(&bitmap, stripe->nr_sectors); \ +} \ +static inline unsigned int scrub_bitmap_weight_##name(struct scrub_stripe *stripe) \ +{ \ + unsigned long bitmap = scrub_bitmap_read_##name(stripe); \ + \ + return bitmap_weight(&bitmap, stripe->nr_sectors); \ +} +IMPLEMENT_SCRUB_BITMAP_OPS(has_extent); +IMPLEMENT_SCRUB_BITMAP_OPS(is_metadata); +IMPLEMENT_SCRUB_BITMAP_OPS(error); +IMPLEMENT_SCRUB_BITMAP_OPS(io_error); +IMPLEMENT_SCRUB_BITMAP_OPS(csum_error); +IMPLEMENT_SCRUB_BITMAP_OPS(meta_error); +IMPLEMENT_SCRUB_BITMAP_OPS(meta_gen_error); + +struct scrub_warning { + struct btrfs_path *path; + u64 extent_item_size; + const char *errstr; + u64 physical; + u64 logical; + struct btrfs_device *dev; +}; + +struct scrub_error_records { + /* + * Bitmap recording which blocks hit errors (IO/csum/...) during the + * initial read. + */ + unsigned long init_error_bitmap; + + unsigned int nr_io_errors; + unsigned int nr_csum_errors; + unsigned int nr_meta_errors; + unsigned int nr_meta_gen_errors; +}; + +static void release_scrub_stripe(struct scrub_stripe *stripe) +{ + if (!stripe) + return; + + kvfree(stripe->buffer); + kfree(stripe->sectors); + kfree(stripe->csums); + stripe->buffer = NULL; + stripe->sectors = NULL; + stripe->csums = NULL; + stripe->sctx = NULL; + stripe->state = 0; +} + +static int init_scrub_stripe(struct btrfs_fs_info *fs_info, + struct scrub_stripe *stripe) +{ + memset(stripe, 0, sizeof(*stripe)); + + stripe->nr_sectors = BTRFS_STRIPE_LEN >> fs_info->sectorsize_bits; + stripe->state = 0; + + init_waitqueue_head(&stripe->io_wait); + init_waitqueue_head(&stripe->repair_wait); + atomic_set(&stripe->pending_io, 0); + spin_lock_init(&stripe->write_error_lock); + + stripe->buffer = kvmalloc(BTRFS_STRIPE_LEN, GFP_NOFS); + if (!stripe->buffer) + goto error; + + stripe->sectors = kzalloc_objs(struct scrub_sector_verification, + stripe->nr_sectors); + if (!stripe->sectors) + goto error; + + stripe->csums = kcalloc(BTRFS_STRIPE_LEN >> fs_info->sectorsize_bits, + fs_info->csum_size, GFP_KERNEL); + if (!stripe->csums) + goto error; + return 0; +error: + release_scrub_stripe(stripe); + return -ENOMEM; +} + +static void wait_scrub_stripe_io(struct scrub_stripe *stripe) +{ + wait_event(stripe->io_wait, atomic_read(&stripe->pending_io) == 0); +} + +static void scrub_put_ctx(struct scrub_ctx *sctx); + +static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info) +{ + while (atomic_read(&fs_info->scrub_pause_req)) { + mutex_unlock(&fs_info->scrub_lock); + wait_event(fs_info->scrub_pause_wait, + atomic_read(&fs_info->scrub_pause_req) == 0); + mutex_lock(&fs_info->scrub_lock); + } +} + +static void scrub_pause_on(struct btrfs_fs_info *fs_info) +{ + atomic_inc(&fs_info->scrubs_paused); + wake_up(&fs_info->scrub_pause_wait); +} + +static void scrub_pause_off(struct btrfs_fs_info *fs_info) +{ + mutex_lock(&fs_info->scrub_lock); + __scrub_blocked_if_needed(fs_info); + atomic_dec(&fs_info->scrubs_paused); + mutex_unlock(&fs_info->scrub_lock); + + wake_up(&fs_info->scrub_pause_wait); +} + +static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info) +{ + scrub_pause_on(fs_info); + scrub_pause_off(fs_info); +} + +static noinline_for_stack void scrub_free_ctx(struct scrub_ctx *sctx) +{ + int i; + + if (!sctx) + return; + + for (i = 0; i < SCRUB_TOTAL_STRIPES; i++) + release_scrub_stripe(&sctx->stripes[i]); + + kvfree(sctx); +} + +static void scrub_put_ctx(struct scrub_ctx *sctx) +{ + if (refcount_dec_and_test(&sctx->refs)) + scrub_free_ctx(sctx); +} + +static noinline_for_stack struct scrub_ctx *scrub_setup_ctx( + struct btrfs_fs_info *fs_info, bool is_dev_replace) +{ + struct scrub_ctx *sctx; + int i; + + /* Since sctx has inline 128 stripes, it can go beyond 64K easily. Use + * kvzalloc(). + */ + sctx = kvzalloc_obj(*sctx); + if (!sctx) + goto nomem; + refcount_set(&sctx->refs, 1); + sctx->is_dev_replace = is_dev_replace; + sctx->fs_info = fs_info; + sctx->extent_path.search_commit_root = true; + sctx->extent_path.skip_locking = true; + sctx->csum_path.search_commit_root = true; + sctx->csum_path.skip_locking = true; + for (i = 0; i < SCRUB_TOTAL_STRIPES; i++) { + int ret; + + ret = init_scrub_stripe(fs_info, &sctx->stripes[i]); + if (ret < 0) + goto nomem; + sctx->stripes[i].sctx = sctx; + } + sctx->first_free = 0; + atomic_set(&sctx->cancel_req, 0); + + spin_lock_init(&sctx->stat_lock); + sctx->throttle_deadline = 0; + + mutex_init(&sctx->wr_lock); + if (is_dev_replace) { + WARN_ON(!fs_info->dev_replace.tgtdev); + sctx->wr_tgtdev = fs_info->dev_replace.tgtdev; + } + + return sctx; + +nomem: + scrub_free_ctx(sctx); + return ERR_PTR(-ENOMEM); +} + +static int scrub_print_warning_inode(u64 inum, u64 offset, u64 num_bytes, + u64 root, void *warn_ctx) +{ + u32 nlink; + int ret; + int i; + unsigned nofs_flag; + struct extent_buffer *eb; + struct btrfs_inode_item *inode_item; + struct scrub_warning *swarn = warn_ctx; + struct btrfs_fs_info *fs_info = swarn->dev->fs_info; + struct inode_fs_paths *ipath __free(inode_fs_paths) = NULL; + struct btrfs_root *local_root; + struct btrfs_key key; + + local_root = btrfs_get_fs_root(fs_info, root, true); + if (IS_ERR(local_root)) { + ret = PTR_ERR(local_root); + goto err; + } + + /* + * this makes the path point to (inum INODE_ITEM ioff) + */ + key.objectid = inum; + key.type = BTRFS_INODE_ITEM_KEY; + key.offset = 0; + + ret = btrfs_search_slot(NULL, local_root, &key, swarn->path, 0, 0); + if (ret) { + btrfs_put_root(local_root); + btrfs_release_path(swarn->path); + goto err; + } + + eb = swarn->path->nodes[0]; + inode_item = btrfs_item_ptr(eb, swarn->path->slots[0], + struct btrfs_inode_item); + nlink = btrfs_inode_nlink(eb, inode_item); + btrfs_release_path(swarn->path); + + /* + * init_path might indirectly call vmalloc, or use GFP_KERNEL. Scrub + * uses GFP_NOFS in this context, so we keep it consistent but it does + * not seem to be strictly necessary. + */ + nofs_flag = memalloc_nofs_save(); + ipath = init_ipath(4096, local_root, swarn->path); + memalloc_nofs_restore(nofs_flag); + if (IS_ERR(ipath)) { + btrfs_put_root(local_root); + ret = PTR_ERR(ipath); + ipath = NULL; + goto err; + } + ret = paths_from_inode(inum, ipath); + + if (ret < 0) + goto err; + + /* + * we deliberately ignore the bit ipath might have been too small to + * hold all of the paths here + */ + for (i = 0; i < ipath->fspath->elem_cnt; ++i) + btrfs_warn(fs_info, +"scrub: %s at logical %llu on dev %s, physical %llu root %llu inode %llu offset %llu length %u links %u (path: %s)", + swarn->errstr, swarn->logical, + btrfs_dev_name(swarn->dev), + swarn->physical, + root, inum, offset, + fs_info->sectorsize, nlink, + (char *)(unsigned long)ipath->fspath->val[i]); + + btrfs_put_root(local_root); + return 0; + +err: + btrfs_warn(fs_info, + "scrub: %s at logical %llu on dev %s, physical %llu root %llu inode %llu offset %llu: path resolving failed with ret=%d", + swarn->errstr, swarn->logical, + btrfs_dev_name(swarn->dev), + swarn->physical, + root, inum, offset, ret); + + return 0; +} + +static void scrub_print_common_warning(const char *errstr, struct btrfs_device *dev, + bool is_super, u64 logical, u64 physical) +{ + struct btrfs_fs_info *fs_info = dev->fs_info; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key found_key; + struct extent_buffer *eb; + struct btrfs_extent_item *ei; + struct scrub_warning swarn; + u64 flags = 0; + u32 item_size; + int ret; + + /* Super block error, no need to search extent tree. */ + if (is_super) { + btrfs_warn(fs_info, "scrub: %s on device %s, physical %llu", + errstr, btrfs_dev_name(dev), physical); + return; + } + path = btrfs_alloc_path(); + if (!path) + return; + + swarn.physical = physical; + swarn.logical = logical; + swarn.errstr = errstr; + swarn.dev = NULL; + + ret = extent_from_logical(fs_info, swarn.logical, path, &found_key, + &flags); + if (ret < 0) + return; + + swarn.extent_item_size = found_key.offset; + + eb = path->nodes[0]; + ei = btrfs_item_ptr(eb, path->slots[0], struct btrfs_extent_item); + item_size = btrfs_item_size(eb, path->slots[0]); + + if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) { + unsigned long ptr = 0; + u8 ref_level; + u64 ref_root; + + while (true) { + ret = tree_backref_for_extent(&ptr, eb, &found_key, ei, + item_size, &ref_root, + &ref_level); + if (ret < 0) { + btrfs_warn(fs_info, + "scrub: failed to resolve tree backref for logical %llu: %d", + swarn.logical, ret); + break; + } + if (ret > 0) + break; + btrfs_warn(fs_info, +"scrub: %s at logical %llu on dev %s, physical %llu: metadata %s (level %d) in tree %llu", + errstr, swarn.logical, btrfs_dev_name(dev), + swarn.physical, (ref_level ? "node" : "leaf"), + ref_level, ref_root); + } + btrfs_release_path(path); + } else { + struct btrfs_backref_walk_ctx ctx = { 0 }; + + btrfs_release_path(path); + + ctx.bytenr = found_key.objectid; + ctx.extent_item_pos = swarn.logical - found_key.objectid; + ctx.fs_info = fs_info; + + swarn.path = path; + swarn.dev = dev; + + iterate_extent_inodes(&ctx, true, scrub_print_warning_inode, &swarn); + } +} + +static int fill_writer_pointer_gap(struct scrub_ctx *sctx, u64 physical) +{ + int ret = 0; + u64 length; + + if (!btrfs_is_zoned(sctx->fs_info)) + return 0; + + if (!btrfs_dev_is_sequential(sctx->wr_tgtdev, physical)) + return 0; + + if (sctx->write_pointer < physical) { + length = physical - sctx->write_pointer; + + ret = btrfs_zoned_issue_zeroout(sctx->wr_tgtdev, + sctx->write_pointer, length); + if (!ret) + sctx->write_pointer = physical; + } + return ret; +} + +/* + * Unlike the existing csum which is based on paddr, this version is fully on + * vaddr, so no extra per-page iteration needed. + */ +static void scrub_calc_vaddr_csum(struct btrfs_fs_info *fs_info, + void *vaddr, unsigned int len, u8 *dest) +{ + struct btrfs_csum_ctx csum; + + btrfs_csum_init(&csum, fs_info->csum_type); + btrfs_csum_update(&csum, vaddr, len); + btrfs_csum_final(&csum, dest); +} + +static void scrub_verify_one_metadata(struct scrub_stripe *stripe, int sector_nr) +{ + struct btrfs_fs_info *fs_info = stripe->bg->fs_info; + const u32 sectors_per_tree = fs_info->nodesize >> fs_info->sectorsize_bits; + const u64 logical = stripe->logical + (sector_nr << fs_info->sectorsize_bits); + void *first_vaddr = stripe->buffer + (sector_nr << fs_info->sectorsize_bits); + struct btrfs_header *header = first_vaddr; + u8 calculated_csum[BTRFS_CSUM_SIZE]; + + if (logical != btrfs_stack_header_bytenr(header)) { + scrub_bitmap_set_meta_error(stripe, sector_nr, sectors_per_tree); + scrub_bitmap_set_error(stripe, sector_nr, sectors_per_tree); + btrfs_warn_rl(fs_info, + "scrub: tree block %llu mirror %u has bad bytenr, has %llu want %llu", + logical, stripe->mirror_num, + btrfs_stack_header_bytenr(header), logical); + return; + } + if (memcmp(header->fsid, fs_info->fs_devices->metadata_uuid, + BTRFS_FSID_SIZE) != 0) { + scrub_bitmap_set_meta_error(stripe, sector_nr, sectors_per_tree); + scrub_bitmap_set_error(stripe, sector_nr, sectors_per_tree); + btrfs_warn_rl(fs_info, + "scrub: tree block %llu mirror %u has bad fsid, has %pU want %pU", + logical, stripe->mirror_num, + header->fsid, fs_info->fs_devices->metadata_uuid); + return; + } + if (memcmp(header->chunk_tree_uuid, fs_info->chunk_tree_uuid, + BTRFS_UUID_SIZE) != 0) { + scrub_bitmap_set_meta_error(stripe, sector_nr, sectors_per_tree); + scrub_bitmap_set_error(stripe, sector_nr, sectors_per_tree); + btrfs_warn_rl(fs_info, + "scrub: tree block %llu mirror %u has bad chunk tree uuid, has %pU want %pU", + logical, stripe->mirror_num, + header->chunk_tree_uuid, fs_info->chunk_tree_uuid); + return; + } + + /* Now check tree block csum. */ + scrub_calc_vaddr_csum(fs_info, first_vaddr + BTRFS_CSUM_SIZE, + fs_info->nodesize - BTRFS_CSUM_SIZE, calculated_csum); + if (memcmp(calculated_csum, header->csum, fs_info->csum_size) != 0) { + scrub_bitmap_set_meta_error(stripe, sector_nr, sectors_per_tree); + scrub_bitmap_set_error(stripe, sector_nr, sectors_per_tree); + btrfs_warn_rl(fs_info, +"scrub: tree block %llu mirror %u has bad csum, has " BTRFS_CSUM_FMT " want " BTRFS_CSUM_FMT, + logical, stripe->mirror_num, + BTRFS_CSUM_FMT_VALUE(fs_info->csum_size, header->csum), + BTRFS_CSUM_FMT_VALUE(fs_info->csum_size, calculated_csum)); + return; + } + if (stripe->sectors[sector_nr].generation != + btrfs_stack_header_generation(header)) { + scrub_bitmap_set_meta_gen_error(stripe, sector_nr, sectors_per_tree); + scrub_bitmap_set_error(stripe, sector_nr, sectors_per_tree); + btrfs_warn_rl(fs_info, + "scrub: tree block %llu mirror %u has bad generation, has %llu want %llu", + logical, stripe->mirror_num, + btrfs_stack_header_generation(header), + stripe->sectors[sector_nr].generation); + return; + } + scrub_bitmap_clear_error(stripe, sector_nr, sectors_per_tree); + scrub_bitmap_clear_csum_error(stripe, sector_nr, sectors_per_tree); + scrub_bitmap_clear_meta_error(stripe, sector_nr, sectors_per_tree); + scrub_bitmap_clear_meta_gen_error(stripe, sector_nr, sectors_per_tree); +} + +static void scrub_verify_one_sector(struct scrub_stripe *stripe, int sector_nr) +{ + struct btrfs_fs_info *fs_info = stripe->bg->fs_info; + struct scrub_sector_verification *sector = &stripe->sectors[sector_nr]; + const u32 sectors_per_tree = fs_info->nodesize >> fs_info->sectorsize_bits; + u8 csum_buf[BTRFS_CSUM_SIZE]; + + ASSERT(sector_nr >= 0 && sector_nr < stripe->nr_sectors); + + /* Sector not utilized, skip it. */ + if (!scrub_bitmap_test_bit_has_extent(stripe, sector_nr)) + return; + + /* IO error, no need to check. */ + if (scrub_bitmap_test_bit_io_error(stripe, sector_nr)) + return; + + /* Metadata, verify the full tree block. */ + if (scrub_bitmap_test_bit_is_metadata(stripe, sector_nr)) { + /* + * Check if the tree block crosses the stripe boundary. If + * crossed the boundary, we cannot verify it but only give a + * warning. + * + * This can only happen on a very old filesystem where chunks + * are not ensured to be stripe aligned. + */ + if (unlikely(sector_nr + sectors_per_tree > stripe->nr_sectors)) { + btrfs_warn_rl(fs_info, + "scrub: tree block at %llu crosses stripe boundary %llu", + stripe->logical + + (sector_nr << fs_info->sectorsize_bits), + stripe->logical); + return; + } + scrub_verify_one_metadata(stripe, sector_nr); + return; + } + + /* + * Data is easier, we just verify the data csum (if we have it). For + * cases without csum, we have no other choice but to trust it. + */ + if (!sector->csum) { + scrub_bitmap_clear_bit_error(stripe, sector_nr); + return; + } + + scrub_calc_vaddr_csum(fs_info, + stripe->buffer + (sector_nr << fs_info->sectorsize_bits), + fs_info->sectorsize, csum_buf); + if (memcmp(csum_buf, sector->csum, fs_info->csum_size)) { + scrub_bitmap_set_bit_csum_error(stripe, sector_nr); + scrub_bitmap_set_bit_error(stripe, sector_nr); + } else { + scrub_bitmap_clear_bit_csum_error(stripe, sector_nr); + scrub_bitmap_clear_bit_error(stripe, sector_nr); + } +} + +/* Verify specified sectors of a stripe. */ +static void scrub_verify_one_stripe(struct scrub_stripe *stripe, unsigned long bitmap) +{ + struct btrfs_fs_info *fs_info = stripe->bg->fs_info; + const u32 sectors_per_tree = fs_info->nodesize >> fs_info->sectorsize_bits; + int sector_nr; + + for_each_set_bit(sector_nr, &bitmap, stripe->nr_sectors) { + scrub_verify_one_sector(stripe, sector_nr); + if (scrub_bitmap_test_bit_is_metadata(stripe, sector_nr)) + sector_nr += sectors_per_tree - 1; + } +} + +static unsigned int calc_sector_number(const struct btrfs_bio *bbio) +{ + const struct scrub_stripe *stripe = bbio->private; + const struct btrfs_fs_info *fs_info = stripe->bg->fs_info; + + /* Scrub bbios all have their @file_offset set to the logical bytenr. */ + ASSERT(bbio->file_offset >= stripe->logical && + bbio->file_offset < stripe->logical + (stripe->nr_sectors << + fs_info->sectorsize_bits), + "scrub bio logical=%llu stripe logical=%llu stripe len=%u", + bbio->file_offset, stripe->logical, + stripe->nr_sectors << fs_info->sectorsize_bits); + return (bbio->file_offset - stripe->logical) >> fs_info->sectorsize_bits; +} + +/* + * Common handling of read endio. + * + * The bbio will be released, so no more access to @bbio after this function. + */ +static void scrub_read_endio_common(struct btrfs_bio *bbio) +{ + struct scrub_stripe *stripe = bbio->private; + struct btrfs_fs_info *fs_info = stripe->bg->fs_info; + unsigned int sector_nr = calc_sector_number(bbio); + const u32 bio_size = bio_get_size(&bbio->bio); + const u32 sectors = bio_size >> fs_info->sectorsize_bits; + + + /* + * For vmallocated space, readers need to call invalidate_kernel_vmap_range() + * to manage the coherency between kernel mapping and devie space mapping. + */ + if (is_vmalloc_addr(stripe->buffer)) + invalidate_kernel_vmap_range( + stripe->buffer + (sector_nr << fs_info->sectorsize_bits), + bio_size); + + if (bbio->bio.bi_status) { + scrub_bitmap_set_io_error(stripe, sector_nr, sectors); + scrub_bitmap_set_error(stripe, sector_nr, sectors); + } else { + scrub_bitmap_clear_io_error(stripe, sector_nr, sectors); + } + bio_put(&bbio->bio); +} + +/* + * Repair read is different to the regular read: + * + * - Only reads the failed sectors + * - May have extra blocksize limits + */ +static void scrub_repair_read_endio(struct btrfs_bio *bbio) +{ + struct scrub_stripe *stripe = bbio->private; + + scrub_read_endio_common(bbio); + + if (atomic_dec_and_test(&stripe->pending_io)) + wake_up(&stripe->io_wait); +} + +static int calc_next_mirror(int mirror, int num_copies) +{ + ASSERT(mirror <= num_copies); + return (mirror + 1 > num_copies) ? 1 : mirror + 1; +} + +static void scrub_bio_add_sector(struct btrfs_bio *bbio, struct scrub_stripe *stripe, + int sector_nr) +{ + struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; + const u32 offset = sector_nr << fs_info->sectorsize_bits; + int ret; + + ASSERT(offset + fs_info->sectorsize <= BTRFS_STRIPE_LEN); + + if (is_vmalloc_addr(stripe->buffer)) { + ret = bio_add_vmalloc(&bbio->bio, stripe->buffer + offset, fs_info->sectorsize); + ASSERT(ret == true); + return; + } + ret = bio_add_page(&bbio->bio, virt_to_page(stripe->buffer + offset), + fs_info->sectorsize, offset_in_page(stripe->buffer + offset)); + ASSERT(ret == fs_info->sectorsize); +} + +static struct btrfs_bio *alloc_scrub_bbio(struct btrfs_fs_info *fs_info, + blk_opf_t opf, + u64 logical, + btrfs_bio_end_io_t end_io, void *private) +{ + struct btrfs_bio *bbio; + + /* + * Stripe->buffer is allocated by kvmalloc(), which can be pages at + * different physical addresses, we have to ensure the bbio is large + * enough to contain the full stripe. + */ + bbio = btrfs_bio_alloc(BTRFS_STRIPE_LEN >> PAGE_SHIFT, opf, + BTRFS_I(fs_info->btree_inode), + logical, end_io, private); + bbio->is_scrub = true; + bbio->bio.bi_iter.bi_sector = logical >> SECTOR_SHIFT; + return bbio; +} + +static void scrub_stripe_submit_repair_read(struct scrub_stripe *stripe, + int mirror, int blocksize, bool wait) +{ + struct btrfs_fs_info *fs_info = stripe->bg->fs_info; + struct btrfs_bio *bbio = NULL; + const unsigned long old_error_bitmap = scrub_bitmap_read_error(stripe); + int i; + + ASSERT(stripe->mirror_num >= 1, "stripe->mirror_num=%d", stripe->mirror_num); + ASSERT(atomic_read(&stripe->pending_io) == 0, + "atomic_read(&stripe->pending_io)=%d", atomic_read(&stripe->pending_io)); + + for_each_set_bit(i, &old_error_bitmap, stripe->nr_sectors) { + /* The current sector cannot be merged, submit the bio. */ + if (bbio && ((i > 0 && !test_bit(i - 1, &old_error_bitmap)) || + bbio->bio.bi_iter.bi_size >= blocksize)) { + ASSERT(bbio->bio.bi_iter.bi_size); + atomic_inc(&stripe->pending_io); + btrfs_submit_bbio(bbio, mirror); + if (wait) + wait_scrub_stripe_io(stripe); + bbio = NULL; + } + + if (!bbio) + bbio = alloc_scrub_bbio(fs_info, REQ_OP_READ, + stripe->logical + (i << fs_info->sectorsize_bits), + scrub_repair_read_endio, stripe); + + scrub_bio_add_sector(bbio, stripe, i); + } + if (bbio) { + ASSERT(bbio->bio.bi_iter.bi_size); + atomic_inc(&stripe->pending_io); + btrfs_submit_bbio(bbio, mirror); + if (wait) + wait_scrub_stripe_io(stripe); + } +} + +static void scrub_stripe_report_errors(struct scrub_ctx *sctx, + struct scrub_stripe *stripe, + const struct scrub_error_records *errors) +{ + static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + struct btrfs_fs_info *fs_info = sctx->fs_info; + struct btrfs_device *dev = NULL; + const unsigned long extent_bitmap = scrub_bitmap_read_has_extent(stripe); + const unsigned long error_bitmap = scrub_bitmap_read_error(stripe); + u64 physical = 0; + int nr_data_sectors = 0; + int nr_meta_sectors = 0; + int nr_nodatacsum_sectors = 0; + int nr_repaired_sectors = 0; + int sector_nr; + + if (test_bit(SCRUB_STRIPE_FLAG_NO_REPORT, &stripe->state)) + return; + + /* + * Init needed infos for error reporting. + * + * Although our scrub_stripe infrastructure is mostly based on btrfs_submit_bio() + * thus no need for dev/physical, error reporting still needs dev and physical. + */ + if (!bitmap_empty(&errors->init_error_bitmap, stripe->nr_sectors)) { + u64 mapped_len = fs_info->sectorsize; + struct btrfs_io_context *bioc = NULL; + int stripe_index = stripe->mirror_num - 1; + int ret; + + /* For scrub, our mirror_num should always start at 1. */ + ASSERT(stripe->mirror_num >= 1, "stripe->mirror_num=%d", stripe->mirror_num); + ret = btrfs_map_block(fs_info, BTRFS_MAP_GET_READ_MIRRORS, + stripe->logical, &mapped_len, &bioc, + NULL, NULL); + /* + * If we failed, dev will be NULL, and later detailed reports + * will just be skipped. + */ + if (ret < 0) + goto skip; + physical = bioc->stripes[stripe_index].physical; + dev = bioc->stripes[stripe_index].dev; + btrfs_put_bioc(bioc); + } + +skip: + for_each_set_bit(sector_nr, &extent_bitmap, stripe->nr_sectors) { + bool repaired = false; + + if (scrub_bitmap_test_bit_is_metadata(stripe, sector_nr)) { + nr_meta_sectors++; + } else { + nr_data_sectors++; + if (!stripe->sectors[sector_nr].csum) + nr_nodatacsum_sectors++; + } + + if (test_bit(sector_nr, &errors->init_error_bitmap) && + !test_bit(sector_nr, &error_bitmap)) { + nr_repaired_sectors++; + repaired = true; + } + + /* Good sector from the beginning, nothing need to be done. */ + if (!test_bit(sector_nr, &errors->init_error_bitmap)) + continue; + + /* + * Report error for the corrupted sectors. If repaired, just + * output the message of repaired message. + */ + if (repaired) { + if (dev) { + btrfs_err_rl(fs_info, + "scrub: fixed up error at logical %llu on dev %s physical %llu", + stripe->logical, btrfs_dev_name(dev), + physical); + } else { + btrfs_err_rl(fs_info, + "scrub: fixed up error at logical %llu on mirror %u", + stripe->logical, stripe->mirror_num); + } + continue; + } + + /* The remaining are all for unrepaired. */ + if (dev) { + btrfs_err_rl(fs_info, +"scrub: unable to fixup (regular) error at logical %llu on dev %s physical %llu", + stripe->logical, btrfs_dev_name(dev), + physical); + } else { + btrfs_err_rl(fs_info, + "scrub: unable to fixup (regular) error at logical %llu on mirror %u", + stripe->logical, stripe->mirror_num); + } + + if (scrub_bitmap_test_bit_io_error(stripe, sector_nr)) + if (__ratelimit(&rs) && dev) + scrub_print_common_warning("i/o error", dev, false, + stripe->logical, physical); + if (scrub_bitmap_test_bit_csum_error(stripe, sector_nr)) + if (__ratelimit(&rs) && dev) + scrub_print_common_warning("checksum error", dev, false, + stripe->logical, physical); + if (scrub_bitmap_test_bit_meta_error(stripe, sector_nr)) + if (__ratelimit(&rs) && dev) + scrub_print_common_warning("header error", dev, false, + stripe->logical, physical); + if (scrub_bitmap_test_bit_meta_gen_error(stripe, sector_nr)) + if (__ratelimit(&rs) && dev) + scrub_print_common_warning("generation error", dev, false, + stripe->logical, physical); + } + + /* Update the device stats. */ + for (int i = 0; i < errors->nr_io_errors; i++) + btrfs_dev_stat_inc_and_print(stripe->dev, BTRFS_DEV_STAT_READ_ERRS); + for (int i = 0; i < errors->nr_csum_errors; i++) + btrfs_dev_stat_inc_and_print(stripe->dev, BTRFS_DEV_STAT_CORRUPTION_ERRS); + /* Generation mismatch error is based on each metadata, not each block. */ + for (int i = 0; i < errors->nr_meta_gen_errors; + i += (fs_info->nodesize >> fs_info->sectorsize_bits)) + btrfs_dev_stat_inc_and_print(stripe->dev, BTRFS_DEV_STAT_GENERATION_ERRS); + + spin_lock(&sctx->stat_lock); + sctx->stat.data_extents_scrubbed += stripe->nr_data_extents; + sctx->stat.tree_extents_scrubbed += stripe->nr_meta_extents; + sctx->stat.data_bytes_scrubbed += nr_data_sectors << fs_info->sectorsize_bits; + sctx->stat.tree_bytes_scrubbed += nr_meta_sectors << fs_info->sectorsize_bits; + sctx->stat.no_csum += nr_nodatacsum_sectors; + sctx->stat.read_errors += errors->nr_io_errors; + sctx->stat.csum_errors += errors->nr_csum_errors; + sctx->stat.verify_errors += errors->nr_meta_errors + + errors->nr_meta_gen_errors; + sctx->stat.uncorrectable_errors += + bitmap_weight(&error_bitmap, stripe->nr_sectors); + sctx->stat.corrected_errors += nr_repaired_sectors; + spin_unlock(&sctx->stat_lock); +} + +static void scrub_write_sectors(struct scrub_ctx *sctx, struct scrub_stripe *stripe, + unsigned long write_bitmap, bool dev_replace); + +/* + * The main entrance for all read related scrub work, including: + * + * - Wait for the initial read to finish + * - Verify and locate any bad sectors + * - Go through the remaining mirrors and try to read as large blocksize as + * possible + * - Go through all mirrors (including the failed mirror) sector-by-sector + * - Submit writeback for repaired sectors + * + * Writeback for dev-replace does not happen here, it needs extra + * synchronization for zoned devices. + */ +static void scrub_stripe_read_repair_worker(struct work_struct *work) +{ + struct scrub_stripe *stripe = container_of(work, struct scrub_stripe, work); + struct scrub_ctx *sctx = stripe->sctx; + struct btrfs_fs_info *fs_info = sctx->fs_info; + struct scrub_error_records errors = { 0 }; + int num_copies = btrfs_num_copies(fs_info, stripe->bg->start, + stripe->bg->length); + unsigned long repaired; + unsigned long error; + int mirror; + int i; + + ASSERT(stripe->mirror_num >= 1, "stripe->mirror_num=%d", stripe->mirror_num); + + wait_scrub_stripe_io(stripe); + scrub_verify_one_stripe(stripe, scrub_bitmap_read_has_extent(stripe)); + /* Save the initial failed bitmap for later repair and report usage. */ + errors.init_error_bitmap = scrub_bitmap_read_error(stripe); + errors.nr_io_errors = scrub_bitmap_weight_io_error(stripe); + errors.nr_csum_errors = scrub_bitmap_weight_csum_error(stripe); + errors.nr_meta_errors = scrub_bitmap_weight_meta_error(stripe); + errors.nr_meta_gen_errors = scrub_bitmap_weight_meta_gen_error(stripe); + + if (bitmap_empty(&errors.init_error_bitmap, stripe->nr_sectors)) + goto out; + + /* + * Try all remaining mirrors. + * + * Here we still try to read as large block as possible, as this is + * faster and we have extra safety nets to rely on. + */ + for (mirror = calc_next_mirror(stripe->mirror_num, num_copies); + mirror != stripe->mirror_num; + mirror = calc_next_mirror(mirror, num_copies)) { + const unsigned long old_error_bitmap = scrub_bitmap_read_error(stripe); + + scrub_stripe_submit_repair_read(stripe, mirror, + BTRFS_STRIPE_LEN, false); + wait_scrub_stripe_io(stripe); + scrub_verify_one_stripe(stripe, old_error_bitmap); + if (scrub_bitmap_empty_error(stripe)) + goto out; + } + + /* + * Last safety net, try re-checking all mirrors, including the failed + * one, sector-by-sector. + * + * As if one sector failed the drive's internal csum, the whole read + * containing the offending sector would be marked as error. + * Thus here we do sector-by-sector read. + * + * This can be slow, thus we only try it as the last resort. + */ + + for (i = 0, mirror = stripe->mirror_num; + i < num_copies; + i++, mirror = calc_next_mirror(mirror, num_copies)) { + const unsigned long old_error_bitmap = scrub_bitmap_read_error(stripe); + + scrub_stripe_submit_repair_read(stripe, mirror, + fs_info->sectorsize, true); + wait_scrub_stripe_io(stripe); + scrub_verify_one_stripe(stripe, old_error_bitmap); + if (scrub_bitmap_empty_error(stripe)) + goto out; + } +out: + error = scrub_bitmap_read_error(stripe); + /* + * Submit the repaired sectors. For zoned case, we cannot do repair + * in-place, but queue the bg to be relocated. + */ + bitmap_andnot(&repaired, &errors.init_error_bitmap, &error, + stripe->nr_sectors); + if (!sctx->readonly && !bitmap_empty(&repaired, stripe->nr_sectors)) { + if (btrfs_is_zoned(fs_info)) { + btrfs_repair_one_zone(fs_info, sctx->stripes[0].bg->start); + } else { + scrub_write_sectors(sctx, stripe, repaired, false); + wait_scrub_stripe_io(stripe); + } + } + + scrub_stripe_report_errors(sctx, stripe, &errors); + set_bit(SCRUB_STRIPE_FLAG_REPAIR_DONE, &stripe->state); + wake_up(&stripe->repair_wait); +} + +static void scrub_read_endio(struct btrfs_bio *bbio) +{ + struct scrub_stripe *stripe = bbio->private; + + scrub_read_endio_common(bbio); + + if (atomic_dec_and_test(&stripe->pending_io)) { + wake_up(&stripe->io_wait); + INIT_WORK(&stripe->work, scrub_stripe_read_repair_worker); + queue_work(stripe->bg->fs_info->scrub_workers, &stripe->work); + } +} + +static void scrub_write_endio(struct btrfs_bio *bbio) +{ + struct scrub_stripe *stripe = bbio->private; + struct btrfs_fs_info *fs_info = stripe->bg->fs_info; + unsigned int sector_nr = calc_sector_number(bbio); + const u32 bio_size = bio_get_size(&bbio->bio); + + if (bbio->bio.bi_status) { + unsigned long flags; + + spin_lock_irqsave(&stripe->write_error_lock, flags); + bitmap_set(&stripe->write_error_bitmap, sector_nr, + bio_size >> fs_info->sectorsize_bits); + spin_unlock_irqrestore(&stripe->write_error_lock, flags); + for (int i = 0; i < (bio_size >> fs_info->sectorsize_bits); i++) + btrfs_dev_stat_inc_and_print(stripe->dev, + BTRFS_DEV_STAT_WRITE_ERRS); + } + bio_put(&bbio->bio); + + if (atomic_dec_and_test(&stripe->pending_io)) + wake_up(&stripe->io_wait); +} + +static void scrub_submit_write_bio(struct scrub_ctx *sctx, + struct scrub_stripe *stripe, + struct btrfs_bio *bbio, bool dev_replace) +{ + struct btrfs_fs_info *fs_info = sctx->fs_info; + u32 bio_len = bbio->bio.bi_iter.bi_size; + u32 bio_off = (bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT) - + stripe->logical; + + fill_writer_pointer_gap(sctx, stripe->physical + bio_off); + atomic_inc(&stripe->pending_io); + btrfs_submit_repair_write(bbio, stripe->mirror_num, dev_replace); + if (!btrfs_is_zoned(fs_info)) + return; + /* + * For zoned writeback, queue depth must be 1, thus we must wait for + * the write to finish before the next write. + */ + wait_scrub_stripe_io(stripe); + + /* + * And also need to update the write pointer if write finished + * successfully. + */ + if (!test_bit(bio_off >> fs_info->sectorsize_bits, + &stripe->write_error_bitmap)) + sctx->write_pointer += bio_len; +} + +/* + * Submit the write bio(s) for the sectors specified by @write_bitmap. + * + * Here we utilize btrfs_submit_repair_write(), which has some extra benefits: + * + * - Only needs logical bytenr and mirror_num + * Just like the scrub read path + * + * - Would only result in writes to the specified mirror + * Unlike the regular writeback path, which would write back to all stripes + * + * - Handle dev-replace and read-repair writeback differently + */ +static void scrub_write_sectors(struct scrub_ctx *sctx, struct scrub_stripe *stripe, + unsigned long write_bitmap, bool dev_replace) +{ + struct btrfs_fs_info *fs_info = stripe->bg->fs_info; + struct btrfs_bio *bbio = NULL; + int sector_nr; + + for_each_set_bit(sector_nr, &write_bitmap, stripe->nr_sectors) { + /* We should only writeback sectors covered by an extent. */ + ASSERT(scrub_bitmap_test_bit_has_extent(stripe, sector_nr)); + + /* Cannot merge with previous sector, submit the current one. */ + if (bbio && sector_nr && !test_bit(sector_nr - 1, &write_bitmap)) { + scrub_submit_write_bio(sctx, stripe, bbio, dev_replace); + bbio = NULL; + } + if (!bbio) + bbio = alloc_scrub_bbio(fs_info, REQ_OP_WRITE, + stripe->logical + (sector_nr << fs_info->sectorsize_bits), + scrub_write_endio, stripe); + scrub_bio_add_sector(bbio, stripe, sector_nr); + } + if (bbio) + scrub_submit_write_bio(sctx, stripe, bbio, dev_replace); +} + +/* + * Throttling of IO submission, bandwidth-limit based, the timeslice is 1 + * second. Limit can be set via /sys/fs/UUID/devinfo/devid/scrub_speed_max. + */ +static void scrub_throttle_dev_io(struct scrub_ctx *sctx, struct btrfs_device *device, + unsigned int bio_size) +{ + const int time_slice = 1000; + s64 delta; + ktime_t now; + u32 div; + u64 bwlimit; + + bwlimit = READ_ONCE(device->scrub_speed_max); + if (bwlimit == 0) + return; + + /* + * Slice is divided into intervals when the IO is submitted, adjust by + * bwlimit and maximum of 64 intervals. + */ + div = clamp(bwlimit / (16 * 1024 * 1024), 1, 64); + + /* Start new epoch, set deadline */ + now = ktime_get(); + if (sctx->throttle_deadline == 0) { + sctx->throttle_deadline = ktime_add_ms(now, time_slice / div); + sctx->throttle_sent = 0; + } + + /* Still in the time to send? */ + if (ktime_before(now, sctx->throttle_deadline)) { + /* If current bio is within the limit, send it */ + sctx->throttle_sent += bio_size; + if (sctx->throttle_sent <= div_u64(bwlimit, div)) + return; + + /* We're over the limit, sleep until the rest of the slice */ + delta = ktime_ms_delta(sctx->throttle_deadline, now); + } else { + /* New request after deadline, start new epoch */ + delta = 0; + } + + if (delta) { + long timeout; + + timeout = div_u64(delta * HZ, 1000); + schedule_timeout_interruptible(timeout); + } + + /* Next call will start the deadline period */ + sctx->throttle_deadline = 0; +} + +/* + * Given a physical address, this will calculate it's + * logical offset. if this is a parity stripe, it will return + * the most left data stripe's logical offset. + * + * return 0 if it is a data stripe, 1 means parity stripe. + */ +static int get_raid56_logic_offset(u64 physical, int num, + struct btrfs_chunk_map *map, u64 *offset, + u64 *stripe_start) +{ + int i; + int j = 0; + u64 last_offset; + const int data_stripes = nr_data_stripes(map); + + last_offset = (physical - map->stripes[num].physical) * data_stripes; + if (stripe_start) + *stripe_start = last_offset; + + *offset = last_offset; + for (i = 0; i < data_stripes; i++) { + u32 stripe_nr; + u32 stripe_index; + u32 rot; + + *offset = last_offset + btrfs_stripe_nr_to_offset(i); + + stripe_nr = (u32)(*offset >> BTRFS_STRIPE_LEN_SHIFT) / data_stripes; + + /* Work out the disk rotation on this stripe-set */ + rot = stripe_nr % map->num_stripes; + /* calculate which stripe this data locates */ + rot += i; + stripe_index = rot % map->num_stripes; + if (stripe_index == num) + return 0; + if (stripe_index < num) + j++; + } + *offset = last_offset + btrfs_stripe_nr_to_offset(j); + return 1; +} + +/* + * Return 0 if the extent item range covers any byte of the range. + * Return <0 if the extent item is before @search_start. + * Return >0 if the extent item is after @start_start + @search_len. + */ +static int compare_extent_item_range(struct btrfs_path *path, + u64 search_start, u64 search_len) +{ + struct btrfs_fs_info *fs_info = path->nodes[0]->fs_info; + u64 len; + struct btrfs_key key; + + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + ASSERT(key.type == BTRFS_EXTENT_ITEM_KEY || + key.type == BTRFS_METADATA_ITEM_KEY, "key.type=%u", key.type); + if (key.type == BTRFS_METADATA_ITEM_KEY) + len = fs_info->nodesize; + else + len = key.offset; + + if (key.objectid + len <= search_start) + return -1; + if (key.objectid >= search_start + search_len) + return 1; + return 0; +} + +/* + * Locate one extent item which covers any byte in range + * [@search_start, @search_start + @search_length) + * + * If the path is not initialized, we will initialize the search by doing + * a btrfs_search_slot(). + * If the path is already initialized, we will use the path as the initial + * slot, to avoid duplicated btrfs_search_slot() calls. + * + * NOTE: If an extent item starts before @search_start, we will still + * return the extent item. This is for data extent crossing stripe boundary. + * + * Return 0 if we found such extent item, and @path will point to the extent item. + * Return >0 if no such extent item can be found, and @path will be released. + * Return <0 if hit fatal error, and @path will be released. + */ +static int find_first_extent_item(struct btrfs_root *extent_root, + struct btrfs_path *path, + u64 search_start, u64 search_len) +{ + struct btrfs_fs_info *fs_info = extent_root->fs_info; + struct btrfs_key key; + int ret; + + /* Continue using the existing path */ + if (path->nodes[0]) + goto search_forward; + + key.objectid = search_start; + if (btrfs_fs_incompat(fs_info, SKINNY_METADATA)) + key.type = BTRFS_METADATA_ITEM_KEY; + else + key.type = BTRFS_EXTENT_ITEM_KEY; + key.offset = (u64)-1; + + ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0); + if (ret < 0) + return ret; + if (unlikely(ret == 0)) { + /* + * Key with offset -1 found, there would have to exist an extent + * item with such offset, but this is out of the valid range. + */ + btrfs_release_path(path); + return -EUCLEAN; + } + + /* + * Here we intentionally pass 0 as @min_objectid, as there could be + * an extent item starting before @search_start. + */ + ret = btrfs_previous_extent_item(extent_root, path, 0); + if (ret < 0) + return ret; + /* + * No matter whether we have found an extent item, the next loop will + * properly do every check on the key. + */ +search_forward: + while (true) { + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + if (key.objectid >= search_start + search_len) + break; + if (key.type != BTRFS_METADATA_ITEM_KEY && + key.type != BTRFS_EXTENT_ITEM_KEY) + goto next; + + ret = compare_extent_item_range(path, search_start, search_len); + if (ret == 0) + return ret; + if (ret > 0) + break; +next: + ret = btrfs_next_item(extent_root, path); + if (ret) { + /* Either no more items or a fatal error. */ + btrfs_release_path(path); + return ret; + } + } + btrfs_release_path(path); + return 1; +} + +static void get_extent_info(struct btrfs_path *path, u64 *extent_start_ret, + u64 *size_ret, u64 *flags_ret, u64 *generation_ret) +{ + struct btrfs_key key; + struct btrfs_extent_item *ei; + + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + ASSERT(key.type == BTRFS_METADATA_ITEM_KEY || + key.type == BTRFS_EXTENT_ITEM_KEY, "key.type=%u", key.type); + *extent_start_ret = key.objectid; + if (key.type == BTRFS_METADATA_ITEM_KEY) + *size_ret = path->nodes[0]->fs_info->nodesize; + else + *size_ret = key.offset; + ei = btrfs_item_ptr(path->nodes[0], path->slots[0], struct btrfs_extent_item); + *flags_ret = btrfs_extent_flags(path->nodes[0], ei); + *generation_ret = btrfs_extent_generation(path->nodes[0], ei); +} + +static int sync_write_pointer_for_zoned(struct scrub_ctx *sctx, u64 logical, + u64 physical, u64 physical_end) +{ + struct btrfs_fs_info *fs_info = sctx->fs_info; + int ret = 0; + + if (!btrfs_is_zoned(fs_info)) + return 0; + + mutex_lock(&sctx->wr_lock); + if (sctx->write_pointer < physical_end) { + ret = btrfs_sync_zone_write_pointer(sctx->wr_tgtdev, logical, + physical, + sctx->write_pointer); + if (ret) + btrfs_err(fs_info, "scrub: zoned: failed to recover write pointer"); + } + mutex_unlock(&sctx->wr_lock); + btrfs_dev_clear_zone_empty(sctx->wr_tgtdev, physical); + + return ret; +} + +static void fill_one_extent_info(struct btrfs_fs_info *fs_info, + struct scrub_stripe *stripe, + u64 extent_start, u64 extent_len, + u64 extent_flags, u64 extent_gen) +{ + for (u64 cur_logical = max(stripe->logical, extent_start); + cur_logical < min(stripe->logical + BTRFS_STRIPE_LEN, + extent_start + extent_len); + cur_logical += fs_info->sectorsize) { + const int nr_sector = (cur_logical - stripe->logical) >> + fs_info->sectorsize_bits; + struct scrub_sector_verification *sector = + &stripe->sectors[nr_sector]; + + scrub_bitmap_set_bit_has_extent(stripe, nr_sector); + if (extent_flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) { + scrub_bitmap_set_bit_is_metadata(stripe, nr_sector); + sector->generation = extent_gen; + } + } +} + +static void scrub_stripe_reset_bitmaps(struct scrub_stripe *stripe) +{ + ASSERT(stripe->nr_sectors); + bitmap_zero(stripe->bitmaps, scrub_bitmap_nr_last * stripe->nr_sectors); +} + +/* + * Locate one stripe which has at least one extent in its range. + * + * Return 0 if found such stripe, and store its info into @stripe. + * Return >0 if there is no such stripe in the specified range. + * Return <0 for error. + */ +static int scrub_find_fill_first_stripe(struct btrfs_block_group *bg, + struct btrfs_path *extent_path, + struct btrfs_path *csum_path, + struct btrfs_device *dev, u64 physical, + int mirror_num, u64 logical_start, + u32 logical_len, + struct scrub_stripe *stripe) +{ + struct btrfs_fs_info *fs_info = bg->fs_info; + struct btrfs_root *extent_root = btrfs_extent_root(fs_info, bg->start); + struct btrfs_root *csum_root = btrfs_csum_root(fs_info, bg->start); + const u64 logical_end = logical_start + logical_len; + u64 cur_logical = logical_start; + u64 stripe_end; + u64 extent_start; + u64 extent_len; + u64 extent_flags; + u64 extent_gen; + int ret; + + if (unlikely(!extent_root || !csum_root)) { + btrfs_err(fs_info, "scrub: no valid extent or csum root found"); + return -EUCLEAN; + } + memset(stripe->sectors, 0, sizeof(struct scrub_sector_verification) * + stripe->nr_sectors); + scrub_stripe_reset_bitmaps(stripe); + + /* The range must be inside the bg. */ + ASSERT(logical_start >= bg->start && logical_end <= btrfs_block_group_end(bg), + "bg->start=%llu logical_start=%llu logical_end=%llu end=%llu", + bg->start, logical_start, logical_end, btrfs_block_group_end(bg)); + + ret = find_first_extent_item(extent_root, extent_path, logical_start, + logical_len); + /* Either error or not found. */ + if (ret) + return ret; + get_extent_info(extent_path, &extent_start, &extent_len, &extent_flags, + &extent_gen); + if (extent_flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) + stripe->nr_meta_extents++; + if (extent_flags & BTRFS_EXTENT_FLAG_DATA) + stripe->nr_data_extents++; + cur_logical = max(extent_start, cur_logical); + + /* + * Round down to stripe boundary. + * + * The extra calculation against bg->start is to handle block groups + * whose logical bytenr is not BTRFS_STRIPE_LEN aligned. + */ + stripe->logical = round_down(cur_logical - bg->start, BTRFS_STRIPE_LEN) + + bg->start; + stripe->physical = physical + stripe->logical - logical_start; + stripe->dev = dev; + stripe->bg = bg; + stripe->mirror_num = mirror_num; + stripe_end = stripe->logical + BTRFS_STRIPE_LEN - 1; + + /* Fill the first extent info into stripe->sectors[] array. */ + fill_one_extent_info(fs_info, stripe, extent_start, extent_len, + extent_flags, extent_gen); + cur_logical = extent_start + extent_len; + + /* Fill the extent info for the remaining sectors. */ + while (cur_logical <= stripe_end) { + ret = find_first_extent_item(extent_root, extent_path, cur_logical, + stripe_end - cur_logical + 1); + if (ret < 0) + return ret; + if (ret > 0) { + ret = 0; + break; + } + get_extent_info(extent_path, &extent_start, &extent_len, + &extent_flags, &extent_gen); + if (extent_flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) + stripe->nr_meta_extents++; + if (extent_flags & BTRFS_EXTENT_FLAG_DATA) + stripe->nr_data_extents++; + fill_one_extent_info(fs_info, stripe, extent_start, extent_len, + extent_flags, extent_gen); + cur_logical = extent_start + extent_len; + } + + /* Now fill the data csum. */ + if (bg->flags & BTRFS_BLOCK_GROUP_DATA) { + int sector_nr; + unsigned long csum_bitmap = 0; + + /* Csum space should have already been allocated. */ + ASSERT(stripe->csums); + + /* + * Our csum bitmap should be large enough, as BTRFS_STRIPE_LEN + * should contain at most 16 sectors. + */ + ASSERT(BITS_PER_LONG >= BTRFS_STRIPE_LEN >> fs_info->sectorsize_bits); + + ret = btrfs_lookup_csums_bitmap(csum_root, csum_path, + stripe->logical, stripe_end, + stripe->csums, &csum_bitmap); + if (ret < 0) + return ret; + if (ret > 0) + ret = 0; + + for_each_set_bit(sector_nr, &csum_bitmap, stripe->nr_sectors) { + stripe->sectors[sector_nr].csum = stripe->csums + + sector_nr * fs_info->csum_size; + } + } + set_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &stripe->state); + + return ret; +} + +static void scrub_reset_stripe(struct scrub_stripe *stripe) +{ + scrub_stripe_reset_bitmaps(stripe); + + stripe->nr_meta_extents = 0; + stripe->nr_data_extents = 0; + stripe->state = 0; + + for (int i = 0; i < stripe->nr_sectors; i++) { + stripe->sectors[i].csum = NULL; + stripe->sectors[i].generation = 0; + } +} + +static u32 stripe_length(const struct scrub_stripe *stripe) +{ + ASSERT(stripe->bg); + + return min(BTRFS_STRIPE_LEN, + stripe->bg->start + stripe->bg->length - stripe->logical); +} + +static void scrub_submit_extent_sector_read(struct scrub_stripe *stripe) +{ + struct btrfs_fs_info *fs_info = stripe->bg->fs_info; + struct btrfs_bio *bbio = NULL; + unsigned int nr_sectors = stripe_length(stripe) >> fs_info->sectorsize_bits; + const unsigned long has_extent = scrub_bitmap_read_has_extent(stripe); + u64 stripe_len = BTRFS_STRIPE_LEN; + int mirror = stripe->mirror_num; + int i; + + atomic_inc(&stripe->pending_io); + + for_each_set_bit(i, &has_extent, stripe->nr_sectors) { + /* We're beyond the chunk boundary, no need to read anymore. */ + if (i >= nr_sectors) + break; + + /* The current sector cannot be merged, submit the bio. */ + if (bbio && + ((i > 0 && !test_bit(i - 1, &has_extent)) || + bbio->bio.bi_iter.bi_size >= stripe_len)) { + ASSERT(bbio->bio.bi_iter.bi_size); + atomic_inc(&stripe->pending_io); + btrfs_submit_bbio(bbio, mirror); + bbio = NULL; + } + + if (!bbio) { + struct btrfs_io_stripe io_stripe = {}; + struct btrfs_io_context *bioc = NULL; + const u64 logical = stripe->logical + + (i << fs_info->sectorsize_bits); + int ret; + + io_stripe.rst_search_commit_root = true; + stripe_len = (nr_sectors - i) << fs_info->sectorsize_bits; + /* + * For RST cases, we need to manually split the bbio to + * follow the RST boundary. + */ + ret = btrfs_map_block(fs_info, BTRFS_MAP_READ, logical, + &stripe_len, &bioc, &io_stripe, &mirror); + btrfs_put_bioc(bioc); + if (ret < 0) { + if (ret != -ENODATA) { + /* + * Earlier btrfs_get_raid_extent_offset() + * returned -ENODATA, which means there's + * no entry for the corresponding range + * in the stripe tree. But if it's in + * the extent tree, then it's a preallocated + * extent and not an error. + */ + scrub_bitmap_set_bit_io_error(stripe, i); + scrub_bitmap_set_bit_error(stripe, i); + } + continue; + } + + bbio = alloc_scrub_bbio(fs_info, REQ_OP_READ, + logical, scrub_read_endio, stripe); + } + + scrub_bio_add_sector(bbio, stripe, i); + } + + if (bbio) { + ASSERT(bbio->bio.bi_iter.bi_size); + atomic_inc(&stripe->pending_io); + btrfs_submit_bbio(bbio, mirror); + } + + if (atomic_dec_and_test(&stripe->pending_io)) { + wake_up(&stripe->io_wait); + INIT_WORK(&stripe->work, scrub_stripe_read_repair_worker); + queue_work(stripe->bg->fs_info->scrub_workers, &stripe->work); + } +} + +static void scrub_submit_initial_read(struct scrub_ctx *sctx, + struct scrub_stripe *stripe) +{ + struct btrfs_fs_info *fs_info = sctx->fs_info; + struct btrfs_bio *bbio; + unsigned int nr_sectors = stripe_length(stripe) >> fs_info->sectorsize_bits; + int mirror = stripe->mirror_num; + + ASSERT(stripe->bg); + ASSERT(stripe->mirror_num > 0); + ASSERT(test_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &stripe->state)); + + if (btrfs_need_stripe_tree_update(fs_info, stripe->bg->flags)) { + scrub_submit_extent_sector_read(stripe); + return; + } + + bbio = alloc_scrub_bbio(fs_info, REQ_OP_READ, + stripe->logical, scrub_read_endio, stripe); + /* Read the whole range inside the chunk boundary. */ + for (unsigned int cur = 0; cur < nr_sectors; cur++) + scrub_bio_add_sector(bbio, stripe, cur); + atomic_inc(&stripe->pending_io); + + /* + * For dev-replace, either user asks to avoid the source dev, or + * the device is missing, we try the next mirror instead. + */ + if (sctx->is_dev_replace && + (fs_info->dev_replace.cont_reading_from_srcdev_mode == + BTRFS_DEV_REPLACE_ITEM_CONT_READING_FROM_SRCDEV_MODE_AVOID || + !stripe->dev->bdev)) { + int num_copies = btrfs_num_copies(fs_info, stripe->bg->start, + stripe->bg->length); + + mirror = calc_next_mirror(mirror, num_copies); + } + btrfs_submit_bbio(bbio, mirror); +} + +static bool stripe_has_metadata_error(struct scrub_stripe *stripe) +{ + const unsigned long error = scrub_bitmap_read_error(stripe); + int i; + + for_each_set_bit(i, &error, stripe->nr_sectors) { + if (scrub_bitmap_test_bit_is_metadata(stripe, i)) { + struct btrfs_fs_info *fs_info = stripe->bg->fs_info; + + btrfs_err(fs_info, + "scrub: stripe %llu has unrepaired metadata sector at logical %llu", + stripe->logical, + stripe->logical + (i << fs_info->sectorsize_bits)); + return true; + } + } + return false; +} + +static void submit_initial_group_read(struct scrub_ctx *sctx, + unsigned int first_slot, + unsigned int nr_stripes) +{ + struct blk_plug plug; + + ASSERT(first_slot < SCRUB_TOTAL_STRIPES); + ASSERT(first_slot + nr_stripes <= SCRUB_TOTAL_STRIPES); + + scrub_throttle_dev_io(sctx, sctx->stripes[0].dev, + btrfs_stripe_nr_to_offset(nr_stripes)); + blk_start_plug(&plug); + for (int i = 0; i < nr_stripes; i++) { + struct scrub_stripe *stripe = &sctx->stripes[first_slot + i]; + + /* Those stripes should be initialized. */ + ASSERT(test_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &stripe->state)); + scrub_submit_initial_read(sctx, stripe); + } + blk_finish_plug(&plug); +} + +static int flush_scrub_stripes(struct scrub_ctx *sctx) +{ + struct btrfs_fs_info *fs_info = sctx->fs_info; + struct scrub_stripe *stripe; + const int nr_stripes = sctx->cur_stripe; + int ret = 0; + + if (!nr_stripes) + return 0; + + ASSERT(test_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &sctx->stripes[0].state)); + + /* Submit the stripes which are populated but not submitted. */ + if (nr_stripes % SCRUB_STRIPES_PER_GROUP) { + const int first_slot = round_down(nr_stripes, SCRUB_STRIPES_PER_GROUP); + + submit_initial_group_read(sctx, first_slot, nr_stripes - first_slot); + } + + for (int i = 0; i < nr_stripes; i++) { + stripe = &sctx->stripes[i]; + + wait_event(stripe->repair_wait, + test_bit(SCRUB_STRIPE_FLAG_REPAIR_DONE, &stripe->state)); + } + + /* Submit for dev-replace. */ + if (sctx->is_dev_replace) { + /* + * For dev-replace, if we know there is something wrong with + * metadata, we should immediately abort. + */ + for (int i = 0; i < nr_stripes; i++) { + if (unlikely(stripe_has_metadata_error(&sctx->stripes[i]))) { + ret = -EIO; + goto out; + } + } + for (int i = 0; i < nr_stripes; i++) { + unsigned long good; + unsigned long has_extent; + unsigned long error; + + stripe = &sctx->stripes[i]; + + ASSERT(stripe->dev == fs_info->dev_replace.srcdev); + + has_extent = scrub_bitmap_read_has_extent(stripe); + error = scrub_bitmap_read_error(stripe); + bitmap_andnot(&good, &has_extent, &error, stripe->nr_sectors); + scrub_write_sectors(sctx, stripe, good, true); + } + } + + /* Wait for the above writebacks to finish. */ + for (int i = 0; i < nr_stripes; i++) { + stripe = &sctx->stripes[i]; + + wait_scrub_stripe_io(stripe); + spin_lock(&sctx->stat_lock); + sctx->stat.last_physical = stripe->physical + stripe_length(stripe); + spin_unlock(&sctx->stat_lock); + scrub_reset_stripe(stripe); + } +out: + sctx->cur_stripe = 0; + return ret; +} + +static void raid56_scrub_wait_endio(struct bio *bio) +{ + complete(bio->bi_private); +} + +static int queue_scrub_stripe(struct scrub_ctx *sctx, struct btrfs_block_group *bg, + struct btrfs_device *dev, int mirror_num, + u64 logical, u32 length, u64 physical, + u64 *found_logical_ret) +{ + struct scrub_stripe *stripe; + int ret; + + /* + * There should always be one slot left, as caller filling the last + * slot should flush them all. + */ + ASSERT(sctx->cur_stripe < SCRUB_TOTAL_STRIPES); + + /* @found_logical_ret must be specified. */ + ASSERT(found_logical_ret); + + stripe = &sctx->stripes[sctx->cur_stripe]; + scrub_reset_stripe(stripe); + ret = scrub_find_fill_first_stripe(bg, &sctx->extent_path, + &sctx->csum_path, dev, physical, + mirror_num, logical, length, stripe); + /* Either >0 as no more extents or <0 for error. */ + if (ret) + return ret; + *found_logical_ret = stripe->logical; + sctx->cur_stripe++; + + /* We filled one group, submit it. */ + if (sctx->cur_stripe % SCRUB_STRIPES_PER_GROUP == 0) { + const int first_slot = sctx->cur_stripe - SCRUB_STRIPES_PER_GROUP; + + submit_initial_group_read(sctx, first_slot, SCRUB_STRIPES_PER_GROUP); + } + + /* Last slot used, flush them all. */ + if (sctx->cur_stripe == SCRUB_TOTAL_STRIPES) + return flush_scrub_stripes(sctx); + return 0; +} + +/* + * Return 0 if we should not cancel the scrub. + * Return <0 if we need to cancel the scrub, returned value will + * indicate the reason: + * - -ECANCELED - Being explicitly canceled through ioctl. + * - -EINTR - Being interrupted by signal or fs/process freezing. + */ +static int should_cancel_scrub(const struct scrub_ctx *sctx) +{ + struct btrfs_fs_info *fs_info = sctx->fs_info; + + if (atomic_read(&fs_info->scrub_cancel_req) || + atomic_read(&sctx->cancel_req)) + return -ECANCELED; + + /* + * The user (e.g. fsfreeze command) or power management (PM) + * suspend/hibernate can freeze the fs. And PM suspend/hibernate will + * also freeze all user processes. + * + * A user process can only be frozen when it is in user space, thus we + * have to cancel the run so that the process can return to the user + * space. + * + * Furthermore we have to check both filesystem and process freezing, + * as PM can be configured to freeze the filesystems before processes. + * + * If we only check fs freezing, then suspend without fs freezing + * will timeout, as the process is still in kernel space. + * + * If we only check process freezing, then suspend with fs freezing + * will timeout, as the running scrub will prevent the fs from being frozen. + */ + if (fs_info->sb->s_writers.frozen > SB_UNFROZEN || + freezing(current) || signal_pending(current)) + return -EINTR; + return 0; +} + +static int scrub_raid56_cached_parity(struct scrub_ctx *sctx, + struct btrfs_device *scrub_dev, + struct btrfs_chunk_map *map, + u64 full_stripe_start, + unsigned long *extent_bitmap) +{ + DECLARE_COMPLETION_ONSTACK(io_done); + struct btrfs_fs_info *fs_info = sctx->fs_info; + struct btrfs_io_context *bioc = NULL; + struct btrfs_raid_bio *rbio; + struct bio bio; + const int data_stripes = nr_data_stripes(map); + u64 length = btrfs_stripe_nr_to_offset(data_stripes); + int ret; + + bio_init(&bio, NULL, NULL, 0, REQ_OP_READ); + bio.bi_iter.bi_sector = full_stripe_start >> SECTOR_SHIFT; + bio.bi_private = &io_done; + bio.bi_end_io = raid56_scrub_wait_endio; + + btrfs_bio_counter_inc_blocked(fs_info); + ret = btrfs_map_block(fs_info, BTRFS_MAP_WRITE, full_stripe_start, + &length, &bioc, NULL, NULL); + if (ret < 0) + goto out; + /* For RAID56 write there must be an @bioc allocated. */ + ASSERT(bioc); + rbio = raid56_parity_alloc_scrub_rbio(&bio, bioc, scrub_dev, extent_bitmap, + BTRFS_STRIPE_LEN >> fs_info->sectorsize_bits); + btrfs_put_bioc(bioc); + if (!rbio) { + ret = -ENOMEM; + goto out; + } + /* Use the recovered stripes as cache to avoid read them from disk again. */ + for (int i = 0; i < data_stripes; i++) { + struct scrub_stripe *stripe = &sctx->raid56_data_stripes[i]; + + raid56_parity_cache_data_folios(rbio, stripe->buffer, + full_stripe_start + (i << BTRFS_STRIPE_LEN_SHIFT)); + } + raid56_parity_submit_scrub_rbio(rbio); + wait_for_completion_io(&io_done); + ret = blk_status_to_errno(bio.bi_status); +out: + btrfs_bio_counter_dec(fs_info); + bio_uninit(&bio); + return ret; +} + +static int scrub_raid56_parity_stripe(struct scrub_ctx *sctx, + struct btrfs_device *scrub_dev, + struct btrfs_block_group *bg, + struct btrfs_chunk_map *map, + u64 full_stripe_start) +{ + struct btrfs_fs_info *fs_info = sctx->fs_info; + BTRFS_PATH_AUTO_RELEASE(extent_path); + BTRFS_PATH_AUTO_RELEASE(csum_path); + struct scrub_stripe *stripe; + bool all_empty = true; + const int data_stripes = nr_data_stripes(map); + unsigned long extent_bitmap = 0; + int ret; + + ASSERT(sctx->raid56_data_stripes); + + ret = should_cancel_scrub(sctx); + if (ret < 0) + return ret; + + if (atomic_read(&fs_info->scrub_pause_req)) + scrub_blocked_if_needed(fs_info); + + spin_lock(&bg->lock); + if (test_bit(BLOCK_GROUP_FLAG_REMOVED, &bg->runtime_flags)) { + spin_unlock(&bg->lock); + return 0; + } + spin_unlock(&bg->lock); + + /* + * For data stripe search, we cannot reuse the same extent/csum paths, + * as the data stripe bytenr may be smaller than previous extent. Thus + * we have to use our own extent/csum paths. + */ + extent_path.search_commit_root = true; + extent_path.skip_locking = true; + csum_path.search_commit_root = true; + csum_path.skip_locking = true; + + for (int i = 0; i < data_stripes; i++) { + int stripe_index; + int rot; + u64 physical; + + stripe = &sctx->raid56_data_stripes[i]; + rot = div_u64(full_stripe_start - bg->start, + data_stripes) >> BTRFS_STRIPE_LEN_SHIFT; + stripe_index = (i + rot) % map->num_stripes; + physical = map->stripes[stripe_index].physical + + btrfs_stripe_nr_to_offset(rot); + + scrub_reset_stripe(stripe); + set_bit(SCRUB_STRIPE_FLAG_NO_REPORT, &stripe->state); + ret = scrub_find_fill_first_stripe(bg, &extent_path, &csum_path, + map->stripes[stripe_index].dev, physical, 1, + full_stripe_start + btrfs_stripe_nr_to_offset(i), + BTRFS_STRIPE_LEN, stripe); + if (ret < 0) + return ret; + /* + * No extent in this data stripe, need to manually mark them + * initialized to make later read submission happy. + */ + if (ret > 0) { + stripe->logical = full_stripe_start + + btrfs_stripe_nr_to_offset(i); + stripe->dev = map->stripes[stripe_index].dev; + stripe->mirror_num = 1; + set_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &stripe->state); + } + } + + /* Check if all data stripes are empty. */ + for (int i = 0; i < data_stripes; i++) { + stripe = &sctx->raid56_data_stripes[i]; + if (!scrub_bitmap_empty_has_extent(stripe)) { + all_empty = false; + break; + } + } + if (all_empty) + return 0; + + for (int i = 0; i < data_stripes; i++) { + stripe = &sctx->raid56_data_stripes[i]; + scrub_submit_initial_read(sctx, stripe); + } + for (int i = 0; i < data_stripes; i++) { + stripe = &sctx->raid56_data_stripes[i]; + + wait_event(stripe->repair_wait, + test_bit(SCRUB_STRIPE_FLAG_REPAIR_DONE, &stripe->state)); + } + /* For now, no zoned support for RAID56. */ + ASSERT(!btrfs_is_zoned(sctx->fs_info)); + + /* + * Now all data stripes are properly verified. Check if we have any + * unrepaired, if so abort immediately or we could further corrupt the + * P/Q stripes. + * + * During the loop, also populate extent_bitmap. + */ + for (int i = 0; i < data_stripes; i++) { + unsigned long error; + unsigned long has_extent; + + stripe = &sctx->raid56_data_stripes[i]; + + error = scrub_bitmap_read_error(stripe); + has_extent = scrub_bitmap_read_has_extent(stripe); + + /* + * We should only check the errors where there is an extent. + * As we may hit an empty data stripe while it's missing. + */ + bitmap_and(&error, &error, &has_extent, stripe->nr_sectors); + if (unlikely(!bitmap_empty(&error, stripe->nr_sectors))) { + btrfs_err(fs_info, +"scrub: unrepaired sectors detected, full stripe %llu data stripe %u errors %*pbl", + full_stripe_start, i, stripe->nr_sectors, + &error); + return ret; + } + bitmap_or(&extent_bitmap, &extent_bitmap, &has_extent, + stripe->nr_sectors); + } + + /* Now we can check and regenerate the P/Q stripe. */ + return scrub_raid56_cached_parity(sctx, scrub_dev, map, full_stripe_start, + &extent_bitmap); +} + +/* + * Scrub one range which can only has simple mirror based profile. + * (Including all range in SINGLE/DUP/RAID1/RAID1C*, and each stripe in + * RAID0/RAID10). + * + * Since we may need to handle a subset of block group, we need @logical_start + * and @logical_length parameter. + */ +static int scrub_simple_mirror(struct scrub_ctx *sctx, + struct btrfs_block_group *bg, + u64 logical_start, u64 logical_length, + struct btrfs_device *device, + u64 physical, int mirror_num) +{ + struct btrfs_fs_info *fs_info = sctx->fs_info; + const u64 logical_end = logical_start + logical_length; + u64 cur_logical = logical_start; + int ret = 0; + + /* The range must be inside the bg */ + ASSERT(logical_start >= bg->start && logical_end <= btrfs_block_group_end(bg)); + + /* Go through each extent items inside the logical range */ + while (cur_logical < logical_end) { + u64 found_logical = U64_MAX; + u64 cur_physical = physical + cur_logical - logical_start; + + ret = should_cancel_scrub(sctx); + if (ret < 0) + break; + + if (atomic_read(&fs_info->scrub_pause_req)) + scrub_blocked_if_needed(fs_info); + + spin_lock(&bg->lock); + if (test_bit(BLOCK_GROUP_FLAG_REMOVED, &bg->runtime_flags)) { + spin_unlock(&bg->lock); + ret = 0; + break; + } + spin_unlock(&bg->lock); + + ret = queue_scrub_stripe(sctx, bg, device, mirror_num, + cur_logical, logical_end - cur_logical, + cur_physical, &found_logical); + if (ret > 0) { + /* No more extent, just update the accounting */ + spin_lock(&sctx->stat_lock); + sctx->stat.last_physical = physical + logical_length; + spin_unlock(&sctx->stat_lock); + ret = 0; + break; + } + if (ret < 0) + break; + + /* queue_scrub_stripe() returned 0, @found_logical must be updated. */ + ASSERT(found_logical != U64_MAX); + cur_logical = found_logical + BTRFS_STRIPE_LEN; + + /* Don't hold CPU for too long time */ + cond_resched(); + } + return ret; +} + +/* Calculate the full stripe length for simple stripe based profiles */ +static u64 simple_stripe_full_stripe_len(const struct btrfs_chunk_map *map) +{ + ASSERT(map->type & (BTRFS_BLOCK_GROUP_RAID0 | + BTRFS_BLOCK_GROUP_RAID10)); + + return btrfs_stripe_nr_to_offset(map->num_stripes / map->sub_stripes); +} + +/* Get the logical bytenr for the stripe */ +static u64 simple_stripe_get_logical(struct btrfs_chunk_map *map, + struct btrfs_block_group *bg, + int stripe_index) +{ + ASSERT(map->type & (BTRFS_BLOCK_GROUP_RAID0 | + BTRFS_BLOCK_GROUP_RAID10)); + ASSERT(stripe_index < map->num_stripes); + + /* + * (stripe_index / sub_stripes) gives how many data stripes we need to + * skip. + */ + return btrfs_stripe_nr_to_offset(stripe_index / map->sub_stripes) + + bg->start; +} + +/* Get the mirror number for the stripe */ +static int simple_stripe_mirror_num(struct btrfs_chunk_map *map, int stripe_index) +{ + ASSERT(map->type & (BTRFS_BLOCK_GROUP_RAID0 | + BTRFS_BLOCK_GROUP_RAID10)); + ASSERT(stripe_index < map->num_stripes); + + /* For RAID0, it's fixed to 1, for RAID10 it's 0,1,0,1... */ + return stripe_index % map->sub_stripes + 1; +} + +static int scrub_simple_stripe(struct scrub_ctx *sctx, + struct btrfs_block_group *bg, + struct btrfs_chunk_map *map, + struct btrfs_device *device, + int stripe_index) +{ + const u64 logical_increment = simple_stripe_full_stripe_len(map); + const u64 orig_logical = simple_stripe_get_logical(map, bg, stripe_index); + const u64 orig_physical = map->stripes[stripe_index].physical; + const u64 end = btrfs_block_group_end(bg); + const int mirror_num = simple_stripe_mirror_num(map, stripe_index); + u64 cur_logical = orig_logical; + u64 cur_physical = orig_physical; + int ret = 0; + + while (cur_logical < end) { + /* + * Inside each stripe, RAID0 is just SINGLE, and RAID10 is + * just RAID1, so we can reuse scrub_simple_mirror() to scrub + * this stripe. + */ + ret = scrub_simple_mirror(sctx, bg, cur_logical, + BTRFS_STRIPE_LEN, device, cur_physical, + mirror_num); + if (ret) + return ret; + /* Skip to next stripe which belongs to the target device */ + cur_logical += logical_increment; + /* For physical offset, we just go to next stripe */ + cur_physical += BTRFS_STRIPE_LEN; + } + return ret; +} + +static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx, + struct btrfs_block_group *bg, + struct btrfs_chunk_map *map, + struct btrfs_device *scrub_dev, + int stripe_index) +{ + struct btrfs_fs_info *fs_info = sctx->fs_info; + const u64 profile = map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK; + const u64 chunk_logical = bg->start; + int ret; + int ret2; + u64 physical = map->stripes[stripe_index].physical; + const u64 dev_stripe_len = btrfs_calc_stripe_length(map); + const u64 physical_end = physical + dev_stripe_len; + u64 logical; + u64 logic_end; + /* The logical increment after finishing one stripe */ + u64 increment; + /* Offset inside the chunk */ + u64 offset; + u64 stripe_logical; + + /* Extent_path should be released by now. */ + ASSERT(sctx->extent_path.nodes[0] == NULL); + + scrub_blocked_if_needed(fs_info); + + if (sctx->is_dev_replace && + btrfs_dev_is_sequential(sctx->wr_tgtdev, physical)) { + mutex_lock(&sctx->wr_lock); + sctx->write_pointer = physical; + mutex_unlock(&sctx->wr_lock); + } + + /* Prepare the extra data stripes used by RAID56. */ + if (profile & BTRFS_BLOCK_GROUP_RAID56_MASK) { + ASSERT(sctx->raid56_data_stripes == NULL); + + sctx->raid56_data_stripes = kzalloc_objs(struct scrub_stripe, + nr_data_stripes(map)); + if (!sctx->raid56_data_stripes) { + ret = -ENOMEM; + goto out; + } + for (int i = 0; i < nr_data_stripes(map); i++) { + ret = init_scrub_stripe(fs_info, + &sctx->raid56_data_stripes[i]); + if (ret < 0) + goto out; + sctx->raid56_data_stripes[i].bg = bg; + sctx->raid56_data_stripes[i].sctx = sctx; + } + } + /* + * There used to be a big double loop to handle all profiles using the + * same routine, which grows larger and more gross over time. + * + * So here we handle each profile differently, so simpler profiles + * have simpler scrubbing function. + */ + if (!(profile & (BTRFS_BLOCK_GROUP_RAID0 | BTRFS_BLOCK_GROUP_RAID10 | + BTRFS_BLOCK_GROUP_RAID56_MASK))) { + /* + * Above check rules out all complex profile, the remaining + * profiles are SINGLE|DUP|RAID1|RAID1C*, which is simple + * mirrored duplication without stripe. + * + * Only @physical and @mirror_num needs to calculated using + * @stripe_index. + */ + ret = scrub_simple_mirror(sctx, bg, bg->start, bg->length, + scrub_dev, map->stripes[stripe_index].physical, + stripe_index + 1); + offset = 0; + goto out; + } + if (profile & (BTRFS_BLOCK_GROUP_RAID0 | BTRFS_BLOCK_GROUP_RAID10)) { + ret = scrub_simple_stripe(sctx, bg, map, scrub_dev, stripe_index); + offset = btrfs_stripe_nr_to_offset(stripe_index / map->sub_stripes); + goto out; + } + + /* Only RAID56 goes through the old code */ + ASSERT(map->type & BTRFS_BLOCK_GROUP_RAID56_MASK); + ret = 0; + + /* Calculate the logical end of the stripe */ + get_raid56_logic_offset(physical_end, stripe_index, + map, &logic_end, NULL); + logic_end += chunk_logical; + + /* Initialize @offset in case we need to go to out: label */ + get_raid56_logic_offset(physical, stripe_index, map, &offset, NULL); + increment = btrfs_stripe_nr_to_offset(nr_data_stripes(map)); + + /* + * Due to the rotation, for RAID56 it's better to iterate each stripe + * using their physical offset. + */ + while (physical < physical_end) { + ret = get_raid56_logic_offset(physical, stripe_index, map, + &logical, &stripe_logical); + logical += chunk_logical; + if (ret) { + /* it is parity strip */ + stripe_logical += chunk_logical; + ret = scrub_raid56_parity_stripe(sctx, scrub_dev, bg, + map, stripe_logical); + spin_lock(&sctx->stat_lock); + sctx->stat.last_physical = min(physical + BTRFS_STRIPE_LEN, + physical_end); + spin_unlock(&sctx->stat_lock); + if (ret) + goto out; + goto next; + } + + /* + * Now we're at a data stripe, scrub each extents in the range. + * + * At this stage, if we ignore the repair part, inside each data + * stripe it is no different than SINGLE profile. + * We can reuse scrub_simple_mirror() here, as the repair part + * is still based on @mirror_num. + */ + ret = scrub_simple_mirror(sctx, bg, logical, BTRFS_STRIPE_LEN, + scrub_dev, physical, 1); + if (ret < 0) + goto out; +next: + logical += increment; + physical += BTRFS_STRIPE_LEN; + spin_lock(&sctx->stat_lock); + sctx->stat.last_physical = physical; + spin_unlock(&sctx->stat_lock); + } +out: + ret2 = flush_scrub_stripes(sctx); + if (!ret) + ret = ret2; + btrfs_release_path(&sctx->extent_path); + btrfs_release_path(&sctx->csum_path); + + if (sctx->raid56_data_stripes) { + for (int i = 0; i < nr_data_stripes(map); i++) + release_scrub_stripe(&sctx->raid56_data_stripes[i]); + kfree(sctx->raid56_data_stripes); + sctx->raid56_data_stripes = NULL; + } + + if (sctx->is_dev_replace && ret >= 0) { + ret2 = sync_write_pointer_for_zoned(sctx, + chunk_logical + offset, + map->stripes[stripe_index].physical, + physical_end); + if (ret2) + ret = ret2; + } + + return ret < 0 ? ret : 0; +} + +static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx, + struct btrfs_block_group *bg, + struct btrfs_device *scrub_dev, + u64 dev_offset, + u64 dev_extent_len) +{ + struct btrfs_fs_info *fs_info = sctx->fs_info; + struct btrfs_chunk_map *map; + int i; + int ret = 0; + + map = btrfs_find_chunk_map(fs_info, bg->start, bg->length); + if (!map) { + /* + * Might have been an unused block group deleted by the cleaner + * kthread or relocation. + */ + spin_lock(&bg->lock); + if (!test_bit(BLOCK_GROUP_FLAG_REMOVED, &bg->runtime_flags)) + ret = -EINVAL; + spin_unlock(&bg->lock); + + return ret; + } + if (map->start != bg->start) + goto out; + if (map->chunk_len < dev_extent_len) + goto out; + + for (i = 0; i < map->num_stripes; ++i) { + if (map->stripes[i].dev->bdev == scrub_dev->bdev && + map->stripes[i].physical == dev_offset) { + ret = scrub_stripe(sctx, bg, map, scrub_dev, i); + if (ret) + goto out; + } + } +out: + btrfs_free_chunk_map(map); + + return ret; +} + +static int finish_extent_writes_for_zoned(struct btrfs_root *root, + struct btrfs_block_group *cache) +{ + struct btrfs_fs_info *fs_info = cache->fs_info; + + if (!btrfs_is_zoned(fs_info)) + return 0; + + btrfs_wait_block_group_reservations(cache); + btrfs_wait_nocow_writers(cache); + btrfs_wait_ordered_roots(fs_info, U64_MAX, cache); + + return btrfs_commit_current_transaction(root); +} + +static noinline_for_stack +int scrub_enumerate_chunks(struct scrub_ctx *sctx, + struct btrfs_device *scrub_dev, u64 start, u64 end) +{ + struct btrfs_dev_extent *dev_extent = NULL; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_fs_info *fs_info = sctx->fs_info; + struct btrfs_root *root = fs_info->dev_root; + u64 chunk_offset; + int ret = 0; + int ro_set; + int slot; + struct extent_buffer *l; + struct btrfs_key key; + struct btrfs_key found_key; + struct btrfs_block_group *cache; + struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + path->reada = READA_FORWARD; + path->search_commit_root = true; + path->skip_locking = true; + + key.objectid = scrub_dev->devid; + key.type = BTRFS_DEV_EXTENT_KEY; + key.offset = 0ull; + + while (1) { + u64 dev_extent_len; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + break; + if (ret > 0) { + if (path->slots[0] >= + btrfs_header_nritems(path->nodes[0])) { + ret = btrfs_next_leaf(root, path); + if (ret < 0) + break; + if (ret > 0) { + ret = 0; + break; + } + } else { + ret = 0; + } + } + + l = path->nodes[0]; + slot = path->slots[0]; + + btrfs_item_key_to_cpu(l, &found_key, slot); + + if (found_key.objectid != scrub_dev->devid) + break; + + if (found_key.type != BTRFS_DEV_EXTENT_KEY) + break; + + if (found_key.offset >= end) + break; + + if (found_key.offset < key.offset) + break; + + dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent); + dev_extent_len = btrfs_dev_extent_length(l, dev_extent); + + if (found_key.offset + dev_extent_len <= start) + goto skip; + + chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent); + + /* + * get a reference on the corresponding block group to prevent + * the chunk from going away while we scrub it + */ + cache = btrfs_lookup_block_group(fs_info, chunk_offset); + + /* some chunks are removed but not committed to disk yet, + * continue scrubbing */ + if (!cache) + goto skip; + + ASSERT(cache->start <= chunk_offset); + /* + * We are using the commit root to search for device extents, so + * that means we could have found a device extent item from a + * block group that was deleted in the current transaction. The + * logical start offset of the deleted block group, stored at + * @chunk_offset, might be part of the logical address range of + * a new block group (which uses different physical extents). + * In this case btrfs_lookup_block_group() has returned the new + * block group, and its start address is less than @chunk_offset. + * + * We skip such new block groups, because it's pointless to + * process them, as we won't find their extents because we search + * for them using the commit root of the extent tree. For a device + * replace it's also fine to skip it, we won't miss copying them + * to the target device because we have the write duplication + * setup through the regular write path (by btrfs_map_block()), + * and we have committed a transaction when we started the device + * replace, right after setting up the device replace state. + */ + if (cache->start < chunk_offset) { + btrfs_put_block_group(cache); + goto skip; + } + + if (sctx->is_dev_replace && btrfs_is_zoned(fs_info)) { + if (!test_bit(BLOCK_GROUP_FLAG_TO_COPY, &cache->runtime_flags)) { + btrfs_put_block_group(cache); + goto skip; + } + } + + /* + * Make sure that while we are scrubbing the corresponding block + * group doesn't get its logical address and its device extents + * reused for another block group, which can possibly be of a + * different type and different profile. We do this to prevent + * false error detections and crashes due to bogus attempts to + * repair extents. + */ + spin_lock(&cache->lock); + if (test_bit(BLOCK_GROUP_FLAG_REMOVED, &cache->runtime_flags)) { + spin_unlock(&cache->lock); + btrfs_put_block_group(cache); + goto skip; + } + btrfs_freeze_block_group(cache); + spin_unlock(&cache->lock); + + /* + * we need call btrfs_inc_block_group_ro() with scrubs_paused, + * to avoid deadlock caused by: + * btrfs_inc_block_group_ro() + * -> btrfs_wait_for_commit() + * -> btrfs_commit_transaction() + * -> btrfs_scrub_pause() + */ + scrub_pause_on(fs_info); + + /* + * Don't do chunk preallocation for scrub. + * + * This is especially important for SYSTEM bgs, or we can hit + * -EFBIG from btrfs_finish_chunk_alloc() like: + * 1. The only SYSTEM bg is marked RO. + * Since SYSTEM bg is small, that's pretty common. + * 2. New SYSTEM bg will be allocated + * Due to regular version will allocate new chunk. + * 3. New SYSTEM bg is empty and will get cleaned up + * Before cleanup really happens, it's marked RO again. + * 4. Empty SYSTEM bg get scrubbed + * We go back to 2. + * + * This can easily boost the amount of SYSTEM chunks if cleaner + * thread can't be triggered fast enough, and use up all space + * of btrfs_super_block::sys_chunk_array + * + * While for dev replace, we need to try our best to mark block + * group RO, to prevent race between: + * - Write duplication + * Contains latest data + * - Scrub copy + * Contains data from commit tree + * + * If target block group is not marked RO, nocow writes can + * be overwritten by scrub copy, causing data corruption. + * So for dev-replace, it's not allowed to continue if a block + * group is not RO. + */ + ret = btrfs_inc_block_group_ro(cache, sctx->is_dev_replace); + if (!ret && sctx->is_dev_replace) { + ret = finish_extent_writes_for_zoned(root, cache); + if (ret) { + btrfs_dec_block_group_ro(cache); + scrub_pause_off(fs_info); + btrfs_put_block_group(cache); + break; + } + } + + if (ret == 0) { + ro_set = 1; + } else if (ret == -ENOSPC && !sctx->is_dev_replace && + !(cache->flags & BTRFS_BLOCK_GROUP_RAID56_MASK)) { + /* + * btrfs_inc_block_group_ro return -ENOSPC when it + * failed in creating new chunk for metadata. + * It is not a problem for scrub, because + * metadata are always cowed, and our scrub paused + * commit_transactions. + * + * For RAID56 chunks, we have to mark them read-only + * for scrub, as later we would use our own cache + * out of RAID56 realm. + * Thus we want the RAID56 bg to be marked RO to + * prevent RMW from screwing up out cache. + */ + ro_set = 0; + } else if (ret == -ETXTBSY) { + btrfs_warn(fs_info, + "scrub: skipping scrub of block group %llu due to active swapfile", + cache->start); + scrub_pause_off(fs_info); + ret = 0; + goto skip_unfreeze; + } else { + btrfs_warn(fs_info, "scrub: failed setting block group ro: %d", + ret); + btrfs_unfreeze_block_group(cache); + btrfs_put_block_group(cache); + scrub_pause_off(fs_info); + break; + } + + /* + * Now the target block is marked RO, wait for nocow writes to + * finish before dev-replace. + * COW is fine, as COW never overwrites extents in commit tree. + */ + if (sctx->is_dev_replace) { + btrfs_wait_nocow_writers(cache); + btrfs_wait_ordered_roots(fs_info, U64_MAX, cache); + } + + scrub_pause_off(fs_info); + down_write(&dev_replace->rwsem); + dev_replace->cursor_right = found_key.offset + dev_extent_len; + dev_replace->cursor_left = found_key.offset; + dev_replace->item_needs_writeback = 1; + up_write(&dev_replace->rwsem); + + ret = scrub_chunk(sctx, cache, scrub_dev, found_key.offset, + dev_extent_len); + if (sctx->is_dev_replace && + !btrfs_finish_block_group_to_copy(dev_replace->srcdev, + cache, found_key.offset)) + ro_set = 0; + + down_write(&dev_replace->rwsem); + dev_replace->cursor_left = dev_replace->cursor_right; + dev_replace->item_needs_writeback = 1; + up_write(&dev_replace->rwsem); + + if (ro_set) + btrfs_dec_block_group_ro(cache); + + /* + * We might have prevented the cleaner kthread from deleting + * this block group if it was already unused because we raced + * and set it to RO mode first. So add it back to the unused + * list, otherwise it might not ever be deleted unless a manual + * balance is triggered or it becomes used and unused again. + */ + spin_lock(&cache->lock); + if (!test_bit(BLOCK_GROUP_FLAG_REMOVED, &cache->runtime_flags) && + !cache->ro && cache->reserved == 0 && cache->used == 0) { + spin_unlock(&cache->lock); + if (btrfs_test_opt(fs_info, DISCARD_ASYNC)) + btrfs_discard_queue_work(&fs_info->discard_ctl, + cache); + else + btrfs_mark_bg_unused(cache); + } else { + spin_unlock(&cache->lock); + } +skip_unfreeze: + btrfs_unfreeze_block_group(cache); + btrfs_put_block_group(cache); + if (ret) + break; + if (unlikely(sctx->is_dev_replace && + atomic64_read(&dev_replace->num_write_errors) > 0)) { + ret = -EIO; + break; + } + if (sctx->stat.malloc_errors > 0) { + ret = -ENOMEM; + break; + } +skip: + key.offset = found_key.offset + dev_extent_len; + btrfs_release_path(path); + } + + return ret; +} + +static int scrub_one_super(struct scrub_ctx *sctx, struct btrfs_device *dev, + struct page *page, u64 physical, u64 generation) +{ + struct btrfs_fs_info *fs_info = sctx->fs_info; + struct btrfs_super_block *sb = page_address(page); + int ret; + + ret = bdev_rw_virt(dev->bdev, physical >> SECTOR_SHIFT, sb, + BTRFS_SUPER_INFO_SIZE, REQ_OP_READ); + if (ret < 0) + return ret; + ret = btrfs_check_super_csum(fs_info, sb); + if (unlikely(ret != 0)) { + btrfs_err_rl(fs_info, + "scrub: super block at physical %llu devid %llu has bad csum", + physical, dev->devid); + return -EIO; + } + if (unlikely(btrfs_super_generation(sb) != generation)) { + btrfs_err_rl(fs_info, +"scrub: super block at physical %llu devid %llu has bad generation %llu expect %llu", + physical, dev->devid, + btrfs_super_generation(sb), generation); + return -EUCLEAN; + } + + return btrfs_validate_super(fs_info, sb, -1); +} + +static noinline_for_stack int scrub_supers(struct scrub_ctx *sctx, + struct btrfs_device *scrub_dev) +{ + int i; + u64 bytenr; + u64 gen; + int ret = 0; + struct page *page; + struct btrfs_fs_info *fs_info = sctx->fs_info; + + if (unlikely(BTRFS_FS_ERROR(fs_info))) + return -EROFS; + + page = alloc_page(GFP_KERNEL); + if (!page) { + spin_lock(&sctx->stat_lock); + sctx->stat.malloc_errors++; + spin_unlock(&sctx->stat_lock); + return -ENOMEM; + } + + /* Seed devices of a new filesystem has their own generation. */ + if (scrub_dev->fs_devices != fs_info->fs_devices) + gen = scrub_dev->generation; + else + gen = btrfs_get_last_trans_committed(fs_info); + + for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) { + ret = btrfs_sb_log_location(scrub_dev, i, 0, &bytenr); + if (ret == -ENOENT) + break; + + if (ret) { + spin_lock(&sctx->stat_lock); + sctx->stat.super_errors++; + spin_unlock(&sctx->stat_lock); + continue; + } + + if (bytenr + BTRFS_SUPER_INFO_SIZE > + scrub_dev->commit_total_bytes) + break; + if (!btrfs_check_super_location(scrub_dev, bytenr)) + continue; + + ret = scrub_one_super(sctx, scrub_dev, page, bytenr, gen); + if (ret) { + spin_lock(&sctx->stat_lock); + sctx->stat.super_errors++; + spin_unlock(&sctx->stat_lock); + } + } + __free_page(page); + return 0; +} + +static void scrub_workers_put(struct btrfs_fs_info *fs_info) +{ + if (refcount_dec_and_mutex_lock(&fs_info->scrub_workers_refcnt, + &fs_info->scrub_lock)) { + struct workqueue_struct *scrub_workers = fs_info->scrub_workers; + + fs_info->scrub_workers = NULL; + mutex_unlock(&fs_info->scrub_lock); + + if (scrub_workers) + destroy_workqueue(scrub_workers); + } +} + +/* + * get a reference count on fs_info->scrub_workers. start worker if necessary + */ +static noinline_for_stack int scrub_workers_get(struct btrfs_fs_info *fs_info) +{ + struct workqueue_struct *scrub_workers = NULL; + unsigned int flags = WQ_FREEZABLE | WQ_UNBOUND; + int max_active = fs_info->thread_pool_size; + int ret = -ENOMEM; + + if (refcount_inc_not_zero(&fs_info->scrub_workers_refcnt)) + return 0; + + scrub_workers = alloc_workqueue("btrfs-scrub", flags, max_active); + if (!scrub_workers) + return -ENOMEM; + + mutex_lock(&fs_info->scrub_lock); + if (refcount_read(&fs_info->scrub_workers_refcnt) == 0) { + ASSERT(fs_info->scrub_workers == NULL); + fs_info->scrub_workers = scrub_workers; + refcount_set(&fs_info->scrub_workers_refcnt, 1); + mutex_unlock(&fs_info->scrub_lock); + return 0; + } + /* Other thread raced in and created the workers for us */ + refcount_inc(&fs_info->scrub_workers_refcnt); + mutex_unlock(&fs_info->scrub_lock); + + ret = 0; + + destroy_workqueue(scrub_workers); + return ret; +} + +int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start, + u64 end, struct btrfs_scrub_progress *progress, + bool readonly, bool is_dev_replace) +{ + struct btrfs_dev_lookup_args args = { .devid = devid }; + struct scrub_ctx *sctx; + int ret; + struct btrfs_device *dev; + unsigned int nofs_flag; + bool need_commit = false; + + /* Set the basic fallback @last_physical before we got a sctx. */ + if (progress) + progress->last_physical = start; + + if (btrfs_fs_closing(fs_info)) + return -EAGAIN; + + /* At mount time we have ensured nodesize is in the range of [4K, 64K]. */ + ASSERT(fs_info->nodesize <= BTRFS_STRIPE_LEN); + + /* Allocate outside of device_list_mutex */ + sctx = scrub_setup_ctx(fs_info, is_dev_replace); + if (IS_ERR(sctx)) + return PTR_ERR(sctx); + sctx->stat.last_physical = start; + + ret = scrub_workers_get(fs_info); + if (ret) + goto out_free_ctx; + + mutex_lock(&fs_info->fs_devices->device_list_mutex); + dev = btrfs_find_device(fs_info->fs_devices, &args); + if (!dev || (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) && + !is_dev_replace)) { + mutex_unlock(&fs_info->fs_devices->device_list_mutex); + ret = -ENODEV; + goto out; + } + + if (!is_dev_replace && !readonly && + !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) { + mutex_unlock(&fs_info->fs_devices->device_list_mutex); + btrfs_err(fs_info, + "scrub: devid %llu: filesystem on %s is not writable", + devid, btrfs_dev_name(dev)); + ret = -EROFS; + goto out; + } + + mutex_lock(&fs_info->scrub_lock); + if (unlikely(!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) || + test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &dev->dev_state))) { + mutex_unlock(&fs_info->scrub_lock); + mutex_unlock(&fs_info->fs_devices->device_list_mutex); + ret = -EIO; + goto out; + } + + down_read(&fs_info->dev_replace.rwsem); + if (dev->scrub_ctx || + (!is_dev_replace && + btrfs_dev_replace_is_ongoing(&fs_info->dev_replace))) { + up_read(&fs_info->dev_replace.rwsem); + mutex_unlock(&fs_info->scrub_lock); + mutex_unlock(&fs_info->fs_devices->device_list_mutex); + ret = -EINPROGRESS; + goto out; + } + up_read(&fs_info->dev_replace.rwsem); + + sctx->readonly = readonly; + dev->scrub_ctx = sctx; + mutex_unlock(&fs_info->fs_devices->device_list_mutex); + + /* + * checking @scrub_pause_req here, we can avoid + * race between committing transaction and scrubbing. + */ + __scrub_blocked_if_needed(fs_info); + atomic_inc(&fs_info->scrubs_running); + mutex_unlock(&fs_info->scrub_lock); + + /* + * In order to avoid deadlock with reclaim when there is a transaction + * trying to pause scrub, make sure we use GFP_NOFS for all the + * allocations done at btrfs_scrub_sectors() and scrub_sectors_for_parity() + * invoked by our callees. The pausing request is done when the + * transaction commit starts, and it blocks the transaction until scrub + * is paused (done at specific points at scrub_stripe() or right above + * before incrementing fs_info->scrubs_running). + */ + nofs_flag = memalloc_nofs_save(); + if (!is_dev_replace) { + u64 old_super_errors; + + spin_lock(&sctx->stat_lock); + old_super_errors = sctx->stat.super_errors; + spin_unlock(&sctx->stat_lock); + + btrfs_info(fs_info, "scrub: started on devid %llu", devid); + /* + * by holding device list mutex, we can + * kick off writing super in log tree sync. + */ + mutex_lock(&fs_info->fs_devices->device_list_mutex); + ret = scrub_supers(sctx, dev); + mutex_unlock(&fs_info->fs_devices->device_list_mutex); + + spin_lock(&sctx->stat_lock); + /* + * Super block errors found, but we can not commit transaction + * at current context, since btrfs_commit_transaction() needs + * to pause the current running scrub (hold by ourselves). + */ + if (sctx->stat.super_errors > old_super_errors && !sctx->readonly) + need_commit = true; + spin_unlock(&sctx->stat_lock); + } + + if (!ret) + ret = scrub_enumerate_chunks(sctx, dev, start, end); + memalloc_nofs_restore(nofs_flag); + + atomic_dec(&fs_info->scrubs_running); + wake_up(&fs_info->scrub_pause_wait); + + if (progress) + memcpy(progress, &sctx->stat, sizeof(*progress)); + + if (!is_dev_replace) + btrfs_info(fs_info, "scrub: %s on devid %llu with status: %d", + ret ? "not finished" : "finished", devid, ret); + + mutex_lock(&fs_info->scrub_lock); + dev->scrub_ctx = NULL; + mutex_unlock(&fs_info->scrub_lock); + + scrub_workers_put(fs_info); + scrub_put_ctx(sctx); + + /* + * We found some super block errors before, now try to force a + * transaction commit, as scrub has finished. + */ + if (need_commit) { + struct btrfs_trans_handle *trans; + + trans = btrfs_start_transaction(fs_info->tree_root, 0); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + btrfs_err(fs_info, + "scrub: failed to start transaction to fix super block errors: %d", ret); + return ret; + } + ret = btrfs_commit_transaction(trans); + if (ret < 0) + btrfs_err(fs_info, + "scrub: failed to commit transaction to fix super block errors: %d", ret); + } + return ret; +out: + scrub_workers_put(fs_info); +out_free_ctx: + scrub_free_ctx(sctx); + + return ret; +} + +void btrfs_scrub_pause(struct btrfs_fs_info *fs_info) +{ + mutex_lock(&fs_info->scrub_lock); + atomic_inc(&fs_info->scrub_pause_req); + while (atomic_read(&fs_info->scrubs_paused) != + atomic_read(&fs_info->scrubs_running)) { + mutex_unlock(&fs_info->scrub_lock); + wait_event(fs_info->scrub_pause_wait, + atomic_read(&fs_info->scrubs_paused) == + atomic_read(&fs_info->scrubs_running)); + mutex_lock(&fs_info->scrub_lock); + } + mutex_unlock(&fs_info->scrub_lock); +} + +void btrfs_scrub_continue(struct btrfs_fs_info *fs_info) +{ + atomic_dec(&fs_info->scrub_pause_req); + wake_up(&fs_info->scrub_pause_wait); +} + +int btrfs_scrub_cancel(struct btrfs_fs_info *fs_info) +{ + mutex_lock(&fs_info->scrub_lock); + if (!atomic_read(&fs_info->scrubs_running)) { + mutex_unlock(&fs_info->scrub_lock); + return -ENOTCONN; + } + + atomic_inc(&fs_info->scrub_cancel_req); + while (atomic_read(&fs_info->scrubs_running)) { + mutex_unlock(&fs_info->scrub_lock); + wait_event(fs_info->scrub_pause_wait, + atomic_read(&fs_info->scrubs_running) == 0); + mutex_lock(&fs_info->scrub_lock); + } + atomic_dec(&fs_info->scrub_cancel_req); + mutex_unlock(&fs_info->scrub_lock); + + return 0; +} + +int btrfs_scrub_cancel_dev(struct btrfs_device *dev) +{ + struct btrfs_fs_info *fs_info = dev->fs_info; + struct scrub_ctx *sctx; + + mutex_lock(&fs_info->scrub_lock); + sctx = dev->scrub_ctx; + if (!sctx) { + mutex_unlock(&fs_info->scrub_lock); + return -ENOTCONN; + } + atomic_inc(&sctx->cancel_req); + while (dev->scrub_ctx) { + mutex_unlock(&fs_info->scrub_lock); + wait_event(fs_info->scrub_pause_wait, + dev->scrub_ctx == NULL); + mutex_lock(&fs_info->scrub_lock); + } + mutex_unlock(&fs_info->scrub_lock); + + return 0; +} + +int btrfs_scrub_progress(struct btrfs_fs_info *fs_info, u64 devid, + struct btrfs_scrub_progress *progress) +{ + struct btrfs_dev_lookup_args args = { .devid = devid }; + struct btrfs_device *dev; + struct scrub_ctx *sctx = NULL; + + mutex_lock(&fs_info->fs_devices->device_list_mutex); + dev = btrfs_find_device(fs_info->fs_devices, &args); + if (dev) + sctx = dev->scrub_ctx; + if (sctx) + memcpy(progress, &sctx->stat, sizeof(*progress)); + mutex_unlock(&fs_info->fs_devices->device_list_mutex); + + return dev ? (sctx ? 0 : -ENOTCONN) : -ENODEV; +} diff --git a/fs/btrfs/scrub.h b/fs/btrfs/scrub.h new file mode 100644 index 000000000..aa68b6eba --- /dev/null +++ b/fs/btrfs/scrub.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_SCRUB_H +#define BTRFS_SCRUB_H + +#include + +struct btrfs_fs_info; +struct btrfs_device; +struct btrfs_scrub_progress; + +int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start, + u64 end, struct btrfs_scrub_progress *progress, + bool readonly, bool is_dev_replace); +void btrfs_scrub_pause(struct btrfs_fs_info *fs_info); +void btrfs_scrub_continue(struct btrfs_fs_info *fs_info); +int btrfs_scrub_cancel(struct btrfs_fs_info *info); +int btrfs_scrub_cancel_dev(struct btrfs_device *dev); +int btrfs_scrub_progress(struct btrfs_fs_info *fs_info, u64 devid, + struct btrfs_scrub_progress *progress); + +#endif diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c new file mode 100644 index 000000000..dca357016 --- /dev/null +++ b/fs/btrfs/send.c @@ -0,0 +1,8297 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2012 Alexander Block. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "send.h" +#include "ctree.h" +#include "backref.h" +#include "locking.h" +#include "disk-io.h" +#include "btrfs_inode.h" +#include "transaction.h" +#include "compression.h" +#include "print-tree.h" +#include "accessors.h" +#include "dir-item.h" +#include "file-item.h" +#include "ioctl.h" +#include "verity.h" +#include "lru_cache.h" + +/* + * Maximum number of references an extent can have in order for us to attempt to + * issue clone operations instead of write operations. This currently exists to + * avoid hitting limitations of the backreference walking code (taking a lot of + * time and using too much memory for extents with large number of references). + */ +#define SEND_MAX_EXTENT_REFS 1024 + +/* + * A fs_path is a helper to dynamically build path names with unknown size. + * It reallocates the internal buffer on demand. + * It allows fast adding of path elements on the right side (normal path) and + * fast adding to the left side (reversed path). A reversed path can also be + * unreversed if needed. + * + * The definition of struct fs_path relies on -fms-extensions to allow + * including a tagged struct as an anonymous member. + */ +struct __fs_path { + char *start; + char *end; + + char *buf; + unsigned short buf_len:15; + unsigned short reversed:1; +}; +static_assert(sizeof(struct __fs_path) < 256); +struct fs_path { + struct __fs_path; + /* + * Average path length does not exceed 200 bytes, we'll have + * better packing in the slab and higher chance to satisfy + * an allocation later during send. + */ + char inline_buf[256 - sizeof(struct __fs_path)]; +}; +#define FS_PATH_INLINE_SIZE \ + sizeof_field(struct fs_path, inline_buf) + +static void fs_path_free(struct fs_path *p); +DEFINE_FREE(fs_path_free, struct fs_path *, fs_path_free(_T)) + +/* reused for each extent */ +struct clone_root { + struct btrfs_root *root; + u64 ino; + u64 offset; + u64 num_bytes; + bool found_ref; +}; + +#define SEND_MAX_NAME_CACHE_SIZE 256 + +/* + * Limit the root_ids array of struct backref_cache_entry to 17 elements. + * This makes the size of a cache entry to be exactly 192 bytes on x86_64, which + * can be satisfied from the kmalloc-192 slab, without wasting any space. + * The most common case is to have a single root for cloning, which corresponds + * to the send root. Having the user specify more than 16 clone roots is not + * common, and in such rare cases we simply don't use caching if the number of + * cloning roots that lead down to a leaf is more than 17. + */ +#define SEND_MAX_BACKREF_CACHE_ROOTS 17 + +/* + * Max number of entries in the cache. + * With SEND_MAX_BACKREF_CACHE_ROOTS as 17, the size in bytes, excluding + * maple tree's internal nodes, is 24K. + */ +#define SEND_MAX_BACKREF_CACHE_SIZE 128 + +/* + * A backref cache entry maps a leaf to a list of IDs of roots from which the + * leaf is accessible and we can use for clone operations. + * With SEND_MAX_BACKREF_CACHE_ROOTS as 12, each cache entry is 128 bytes (on + * x86_64). + */ +struct backref_cache_entry { + struct btrfs_lru_cache_entry entry; + u64 root_ids[SEND_MAX_BACKREF_CACHE_ROOTS]; + /* Number of valid elements in the root_ids array. */ + int num_roots; +}; + +/* See the comment at lru_cache.h about struct btrfs_lru_cache_entry. */ +static_assert(offsetof(struct backref_cache_entry, entry) == 0); + +/* + * Max number of entries in the cache that stores directories that were already + * created. The cache uses raw struct btrfs_lru_cache_entry entries, so it uses + * at most 4096 bytes - sizeof(struct btrfs_lru_cache_entry) is 48 bytes, but + * the kmalloc-64 slab is used, so we get 4096 bytes (64 bytes * 64). + */ +#define SEND_MAX_DIR_CREATED_CACHE_SIZE 64 + +/* + * Maximum number of entries in the cache that stores utimes values for directories. + * The cache uses raw struct btrfs_lru_cache_entry entries, so it uses at most + * 4096 bytes - sizeof(struct btrfs_lru_cache_entry) is 48 bytes, but the + * kmalloc-64 slab is used, so we get 4096 bytes (64 bytes * 64). + */ +#define SEND_MAX_DIR_UTIMES_CACHE_SIZE 64 + +struct send_ctx { + struct file *send_filp; + loff_t send_off; + char *send_buf; + u32 send_size; + u32 send_max_size; + /* + * Whether BTRFS_SEND_A_DATA attribute was already added to current + * command (since protocol v2, data must be the last attribute). + */ + bool put_data; + struct page **send_buf_pages; + u64 flags; /* 'flags' member of btrfs_ioctl_send_args is u64 */ + /* Protocol version compatibility requested */ + u32 proto; + + struct btrfs_root *send_root; + struct btrfs_root *parent_root; + struct clone_root *clone_roots; + int clone_roots_cnt; + + /* current state of the compare_tree call */ + struct btrfs_path *left_path; + struct btrfs_path *right_path; + struct btrfs_key *cmp_key; + + /* + * Keep track of the generation of the last transaction that was used + * for relocating a block group. This is periodically checked in order + * to detect if a relocation happened since the last check, so that we + * don't operate on stale extent buffers for nodes (level >= 1) or on + * stale disk_bytenr values of file extent items. + */ + u64 last_reloc_trans; + + /* + * infos of the currently processed inode. In case of deleted inodes, + * these are the values from the deleted inode. + */ + u64 cur_ino; + u64 cur_inode_gen; + u64 cur_inode_size; + u64 cur_inode_mode; + u64 cur_inode_rdev; + u64 cur_inode_last_extent; + u64 cur_inode_next_write_offset; + bool cur_inode_new; + bool cur_inode_new_gen; + bool cur_inode_deleted; + bool ignore_cur_inode; + bool cur_inode_needs_verity; + void *verity_descriptor; + + u64 send_progress; + + struct list_head new_refs; + struct list_head deleted_refs; + + struct btrfs_lru_cache name_cache; + + /* + * The inode we are currently processing. It's not NULL only when we + * need to issue write commands for data extents from this inode. + */ + struct inode *cur_inode; + struct file_ra_state ra; + u64 page_cache_clear_start; + bool clean_page_cache; + + /* + * We process inodes by their increasing order, so if before an + * incremental send we reverse the parent/child relationship of + * directories such that a directory with a lower inode number was + * the parent of a directory with a higher inode number, and the one + * becoming the new parent got renamed too, we can't rename/move the + * directory with lower inode number when we finish processing it - we + * must process the directory with higher inode number first, then + * rename/move it and then rename/move the directory with lower inode + * number. Example follows. + * + * Tree state when the first send was performed: + * + * . + * |-- a (ino 257) + * |-- b (ino 258) + * | + * | + * |-- c (ino 259) + * | |-- d (ino 260) + * | + * |-- c2 (ino 261) + * + * Tree state when the second (incremental) send is performed: + * + * . + * |-- a (ino 257) + * |-- b (ino 258) + * |-- c2 (ino 261) + * |-- d2 (ino 260) + * |-- cc (ino 259) + * + * The sequence of steps that lead to the second state was: + * + * mv /a/b/c/d /a/b/c2/d2 + * mv /a/b/c /a/b/c2/d2/cc + * + * "c" has lower inode number, but we can't move it (2nd mv operation) + * before we move "d", which has higher inode number. + * + * So we just memorize which move/rename operations must be performed + * later when their respective parent is processed and moved/renamed. + */ + + /* Indexed by parent directory inode number. */ + struct rb_root pending_dir_moves; + + /* + * Reverse index, indexed by the inode number of a directory that + * is waiting for the move/rename of its immediate parent before its + * own move/rename can be performed. + */ + struct rb_root waiting_dir_moves; + + /* + * A directory that is going to be rm'ed might have a child directory + * which is in the pending directory moves index above. In this case, + * the directory can only be removed after the move/rename of its child + * is performed. Example: + * + * Parent snapshot: + * + * . (ino 256) + * |-- a/ (ino 257) + * |-- b/ (ino 258) + * |-- c/ (ino 259) + * | |-- x/ (ino 260) + * | + * |-- y/ (ino 261) + * + * Send snapshot: + * + * . (ino 256) + * |-- a/ (ino 257) + * |-- b/ (ino 258) + * |-- YY/ (ino 261) + * |-- x/ (ino 260) + * + * Sequence of steps that lead to the send snapshot: + * rm -f /a/b/c/foo.txt + * mv /a/b/y /a/b/YY + * mv /a/b/c/x /a/b/YY + * rmdir /a/b/c + * + * When the child is processed, its move/rename is delayed until its + * parent is processed (as explained above), but all other operations + * like update utimes, chown, chgrp, etc, are performed and the paths + * that it uses for those operations must use the orphanized name of + * its parent (the directory we're going to rm later), so we need to + * memorize that name. + * + * Indexed by the inode number of the directory to be deleted. + */ + struct rb_root orphan_dirs; + + struct rb_root rbtree_new_refs; + struct rb_root rbtree_deleted_refs; + + struct btrfs_lru_cache backref_cache; + u64 backref_cache_last_reloc_trans; + + struct btrfs_lru_cache dir_created_cache; + struct btrfs_lru_cache dir_utimes_cache; + + struct fs_path cur_inode_path; +}; + +struct pending_dir_move { + struct rb_node node; + struct list_head list; + u64 parent_ino; + u64 ino; + u64 gen; + struct list_head update_refs; +}; + +struct waiting_dir_move { + struct rb_node node; + u64 ino; + /* + * There might be some directory that could not be removed because it + * was waiting for this directory inode to be moved first. Therefore + * after this directory is moved, we can try to rmdir the ino rmdir_ino. + */ + u64 rmdir_ino; + u64 rmdir_gen; + bool orphanized; +}; + +struct orphan_dir_info { + struct rb_node node; + u64 ino; + u64 gen; + u64 last_dir_index_offset; + u64 dir_high_seq_ino; +}; + +struct name_cache_entry { + /* + * The key in the entry is an inode number, and the generation matches + * the inode's generation. + */ + struct btrfs_lru_cache_entry entry; + u64 parent_ino; + u64 parent_gen; + int ret; + int need_later_update; + /* Name length without NUL terminator. */ + int name_len; + /* Not NUL terminated. */ + char name[] __counted_by(name_len) __nonstring; +}; + +/* See the comment at lru_cache.h about struct btrfs_lru_cache_entry. */ +static_assert(offsetof(struct name_cache_entry, entry) == 0); + +#define ADVANCE 1 +#define ADVANCE_ONLY_NEXT -1 + +enum btrfs_compare_tree_result { + BTRFS_COMPARE_TREE_NEW, + BTRFS_COMPARE_TREE_DELETED, + BTRFS_COMPARE_TREE_CHANGED, + BTRFS_COMPARE_TREE_SAME, +}; + +__cold +static void inconsistent_snapshot_error(struct send_ctx *sctx, + enum btrfs_compare_tree_result result, + const char *what) +{ + const char *result_string; + + switch (result) { + case BTRFS_COMPARE_TREE_NEW: + result_string = "new"; + break; + case BTRFS_COMPARE_TREE_DELETED: + result_string = "deleted"; + break; + case BTRFS_COMPARE_TREE_CHANGED: + result_string = "updated"; + break; + case BTRFS_COMPARE_TREE_SAME: + DEBUG_WARN("no change between trees"); + result_string = "unchanged"; + break; + default: + DEBUG_WARN("unexpected comparison result %d", result); + result_string = "unexpected"; + } + + btrfs_err(sctx->send_root->fs_info, + "Send: inconsistent snapshot, found %s %s for inode %llu without updated inode item, send root is %llu, parent root is %llu", + result_string, what, sctx->cmp_key->objectid, + btrfs_root_id(sctx->send_root), + (sctx->parent_root ? btrfs_root_id(sctx->parent_root) : 0)); +} + +__maybe_unused +static bool proto_cmd_ok(const struct send_ctx *sctx, int cmd) +{ + switch (sctx->proto) { + case 1: return cmd <= BTRFS_SEND_C_MAX_V1; + case 2: return cmd <= BTRFS_SEND_C_MAX_V2; + case 3: return cmd <= BTRFS_SEND_C_MAX_V3; + default: return false; + } +} + +static int is_waiting_for_move(struct send_ctx *sctx, u64 ino); + +static struct waiting_dir_move * +get_waiting_dir_move(struct send_ctx *sctx, u64 ino); + +static int is_waiting_for_rm(struct send_ctx *sctx, u64 dir_ino, u64 gen); + +static int need_send_hole(struct send_ctx *sctx) +{ + return (sctx->parent_root && !sctx->cur_inode_new && + !sctx->cur_inode_new_gen && !sctx->cur_inode_deleted && + S_ISREG(sctx->cur_inode_mode)); +} + +static void fs_path_reset(struct fs_path *p) +{ + if (p->reversed) + p->start = p->buf + p->buf_len - 1; + else + p->start = p->buf; + + p->end = p->start; + *p->start = 0; +} + +static void init_path(struct fs_path *p) +{ + p->reversed = 0; + p->buf = p->inline_buf; + p->buf_len = FS_PATH_INLINE_SIZE; + fs_path_reset(p); +} + +static struct fs_path *fs_path_alloc(void) +{ + struct fs_path *p; + + p = kmalloc_obj(*p); + if (!p) + return NULL; + init_path(p); + return p; +} + +static struct fs_path *fs_path_alloc_reversed(void) +{ + struct fs_path *p; + + p = fs_path_alloc(); + if (!p) + return NULL; + p->reversed = 1; + fs_path_reset(p); + return p; +} + +static void fs_path_free(struct fs_path *p) +{ + if (!p) + return; + if (p->buf != p->inline_buf) + kfree(p->buf); + kfree(p); +} + +static inline int fs_path_len(const struct fs_path *p) +{ + return p->end - p->start; +} + +static int fs_path_ensure_buf(struct fs_path *p, int len) +{ + char *tmp_buf; + int path_len; + int old_buf_len; + + len++; + + if (p->buf_len >= len) + return 0; + + if (WARN_ON(len > PATH_MAX)) + return -ENAMETOOLONG; + + path_len = fs_path_len(p); + old_buf_len = p->buf_len; + + /* + * Allocate to the next largest kmalloc bucket size, to let + * the fast path happen most of the time. + */ + len = kmalloc_size_roundup(len); + /* + * First time the inline_buf does not suffice + */ + if (p->buf == p->inline_buf) { + tmp_buf = kmalloc(len, GFP_KERNEL); + if (tmp_buf) + memcpy(tmp_buf, p->buf, old_buf_len); + } else { + tmp_buf = krealloc(p->buf, len, GFP_KERNEL); + } + if (!tmp_buf) + return -ENOMEM; + p->buf = tmp_buf; + p->buf_len = len; + + if (p->reversed) { + tmp_buf = p->buf + old_buf_len - path_len - 1; + p->end = p->buf + p->buf_len - 1; + p->start = p->end - path_len; + memmove(p->start, tmp_buf, path_len + 1); + } else { + p->start = p->buf; + p->end = p->start + path_len; + } + return 0; +} + +static int fs_path_prepare_for_add(struct fs_path *p, int name_len, + char **prepared) +{ + int ret; + int new_len; + + new_len = fs_path_len(p) + name_len; + if (p->start != p->end) + new_len++; + ret = fs_path_ensure_buf(p, new_len); + if (ret < 0) + return ret; + + if (p->reversed) { + if (p->start != p->end) + *--p->start = '/'; + p->start -= name_len; + *prepared = p->start; + } else { + if (p->start != p->end) + *p->end++ = '/'; + *prepared = p->end; + p->end += name_len; + *p->end = 0; + } + + return 0; +} + +static int fs_path_add(struct fs_path *p, const char *name, int name_len) +{ + int ret; + char *prepared; + + ret = fs_path_prepare_for_add(p, name_len, &prepared); + if (ret < 0) + return ret; + memcpy(prepared, name, name_len); + + return 0; +} + +static inline int fs_path_add_path(struct fs_path *p, const struct fs_path *p2) +{ + return fs_path_add(p, p2->start, fs_path_len(p2)); +} + +static int fs_path_add_from_extent_buffer(struct fs_path *p, + struct extent_buffer *eb, + unsigned long off, int len) +{ + int ret; + char *prepared; + + ret = fs_path_prepare_for_add(p, len, &prepared); + if (ret < 0) + return ret; + + read_extent_buffer(eb, prepared, off, len); + + return 0; +} + +static int fs_path_copy(struct fs_path *p, struct fs_path *from) +{ + p->reversed = from->reversed; + fs_path_reset(p); + + return fs_path_add_path(p, from); +} + +static void fs_path_unreverse(struct fs_path *p) +{ + char *tmp; + int len; + + if (!p->reversed) + return; + + tmp = p->start; + len = fs_path_len(p); + p->start = p->buf; + p->end = p->start + len; + memmove(p->start, tmp, len + 1); + p->reversed = 0; +} + +static inline bool is_current_inode_path(const struct send_ctx *sctx, + const struct fs_path *path) +{ + /* Paths are always nul terminated. */ + return (strcmp(path->start, sctx->cur_inode_path.start) == 0); +} + +static struct btrfs_path *alloc_path_for_send(void) +{ + struct btrfs_path *path; + + path = btrfs_alloc_path(); + if (!path) + return NULL; + path->search_commit_root = true; + path->skip_locking = true; + path->need_commit_sem = true; + return path; +} + +static int write_buf(struct file *filp, const void *buf, u32 len, loff_t *off) +{ + int ret; + u32 pos = 0; + + while (pos < len) { + ret = kernel_write(filp, buf + pos, len - pos, off); + if (ret < 0) + return ret; + if (unlikely(ret == 0)) + return -EIO; + pos += ret; + } + + return 0; +} + +static int tlv_put(struct send_ctx *sctx, u16 attr, const void *data, int len) +{ + struct btrfs_tlv_header *hdr; + int total_len = sizeof(*hdr) + len; + int left = sctx->send_max_size - sctx->send_size; + + if (WARN_ON_ONCE(sctx->put_data)) + return -EINVAL; + + if (unlikely(left < total_len)) + return -EOVERFLOW; + + hdr = (struct btrfs_tlv_header *) (sctx->send_buf + sctx->send_size); + put_unaligned_le16(attr, &hdr->tlv_type); + put_unaligned_le16(len, &hdr->tlv_len); + memcpy(hdr + 1, data, len); + sctx->send_size += total_len; + + return 0; +} + +#define TLV_PUT_DEFINE_INT(bits) \ + static int tlv_put_u##bits(struct send_ctx *sctx, \ + u##bits attr, u##bits value) \ + { \ + __le##bits __tmp = cpu_to_le##bits(value); \ + return tlv_put(sctx, attr, &__tmp, sizeof(__tmp)); \ + } + +TLV_PUT_DEFINE_INT(8) +TLV_PUT_DEFINE_INT(32) +TLV_PUT_DEFINE_INT(64) + +static int tlv_put_string(struct send_ctx *sctx, u16 attr, + const char *str, int len) +{ + if (len == -1) + len = strlen(str); + return tlv_put(sctx, attr, str, len); +} + +static int tlv_put_uuid(struct send_ctx *sctx, u16 attr, + const u8 *uuid) +{ + return tlv_put(sctx, attr, uuid, BTRFS_UUID_SIZE); +} + +static int tlv_put_btrfs_timespec(struct send_ctx *sctx, u16 attr, + struct extent_buffer *eb, + struct btrfs_timespec *ts) +{ + struct btrfs_timespec bts; + read_extent_buffer(eb, &bts, (unsigned long)ts, sizeof(bts)); + return tlv_put(sctx, attr, &bts, sizeof(bts)); +} + + +#define TLV_PUT(sctx, attrtype, data, attrlen) \ + do { \ + ret = tlv_put(sctx, attrtype, data, attrlen); \ + if (ret < 0) \ + goto tlv_put_failure; \ + } while (0) + +#define TLV_PUT_INT(sctx, attrtype, bits, value) \ + do { \ + ret = tlv_put_u##bits(sctx, attrtype, value); \ + if (ret < 0) \ + goto tlv_put_failure; \ + } while (0) + +#define TLV_PUT_U8(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 8, data) +#define TLV_PUT_U16(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 16, data) +#define TLV_PUT_U32(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 32, data) +#define TLV_PUT_U64(sctx, attrtype, data) TLV_PUT_INT(sctx, attrtype, 64, data) +#define TLV_PUT_STRING(sctx, attrtype, str, len) \ + do { \ + ret = tlv_put_string(sctx, attrtype, str, len); \ + if (ret < 0) \ + goto tlv_put_failure; \ + } while (0) +#define TLV_PUT_PATH(sctx, attrtype, p) \ + do { \ + ret = tlv_put_string(sctx, attrtype, p->start, \ + fs_path_len((p))); \ + if (ret < 0) \ + goto tlv_put_failure; \ + } while(0) +#define TLV_PUT_UUID(sctx, attrtype, uuid) \ + do { \ + ret = tlv_put_uuid(sctx, attrtype, uuid); \ + if (ret < 0) \ + goto tlv_put_failure; \ + } while (0) +#define TLV_PUT_BTRFS_TIMESPEC(sctx, attrtype, eb, ts) \ + do { \ + ret = tlv_put_btrfs_timespec(sctx, attrtype, eb, ts); \ + if (ret < 0) \ + goto tlv_put_failure; \ + } while (0) + +static int send_header(struct send_ctx *sctx) +{ + struct btrfs_stream_header hdr; + + strscpy(hdr.magic, BTRFS_SEND_STREAM_MAGIC); + hdr.version = cpu_to_le32(sctx->proto); + return write_buf(sctx->send_filp, &hdr, sizeof(hdr), + &sctx->send_off); +} + +/* + * For each command/item we want to send to userspace, we call this function. + */ +static int begin_cmd(struct send_ctx *sctx, int cmd) +{ + struct btrfs_cmd_header *hdr; + + if (WARN_ON(!sctx->send_buf)) + return -EINVAL; + + if (unlikely(sctx->send_size != 0)) { + btrfs_err(sctx->send_root->fs_info, + "send: command header buffer not empty cmd %d offset %llu", + cmd, sctx->send_off); + return -EINVAL; + } + + sctx->send_size += sizeof(*hdr); + hdr = (struct btrfs_cmd_header *)sctx->send_buf; + put_unaligned_le16(cmd, &hdr->cmd); + + return 0; +} + +static int send_cmd(struct send_ctx *sctx) +{ + int ret; + struct btrfs_cmd_header *hdr; + u32 crc; + + hdr = (struct btrfs_cmd_header *)sctx->send_buf; + put_unaligned_le32(sctx->send_size - sizeof(*hdr), &hdr->len); + put_unaligned_le32(0, &hdr->crc); + + crc = crc32c(0, (unsigned char *)sctx->send_buf, sctx->send_size); + put_unaligned_le32(crc, &hdr->crc); + + ret = write_buf(sctx->send_filp, sctx->send_buf, sctx->send_size, + &sctx->send_off); + + sctx->send_size = 0; + sctx->put_data = false; + + return ret; +} + +/* + * Sends a move instruction to user space + */ +static int send_rename(struct send_ctx *sctx, + struct fs_path *from, struct fs_path *to) +{ + int ret; + + ret = begin_cmd(sctx, BTRFS_SEND_C_RENAME); + if (ret < 0) + return ret; + + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, from); + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_TO, to); + + ret = send_cmd(sctx); + +tlv_put_failure: + return ret; +} + +/* + * Sends a link instruction to user space + */ +static int send_link(struct send_ctx *sctx, + struct fs_path *path, struct fs_path *lnk) +{ + int ret; + + ret = begin_cmd(sctx, BTRFS_SEND_C_LINK); + if (ret < 0) + return ret; + + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path); + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_LINK, lnk); + + ret = send_cmd(sctx); + +tlv_put_failure: + return ret; +} + +/* + * Sends an unlink instruction to user space + */ +static int send_unlink(struct send_ctx *sctx, struct fs_path *path) +{ + int ret; + + ret = begin_cmd(sctx, BTRFS_SEND_C_UNLINK); + if (ret < 0) + return ret; + + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path); + + ret = send_cmd(sctx); + +tlv_put_failure: + return ret; +} + +/* + * Sends a rmdir instruction to user space + */ +static int send_rmdir(struct send_ctx *sctx, struct fs_path *path) +{ + int ret; + + ret = begin_cmd(sctx, BTRFS_SEND_C_RMDIR); + if (ret < 0) + return ret; + + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path); + + ret = send_cmd(sctx); + +tlv_put_failure: + return ret; +} + +struct btrfs_inode_info { + u64 size; + u64 gen; + u64 mode; + u64 uid; + u64 gid; + u64 rdev; + u64 fileattr; + u64 nlink; +}; + +/* + * Helper function to retrieve some fields from an inode item. + */ +static int get_inode_info(struct btrfs_root *root, u64 ino, + struct btrfs_inode_info *info) +{ + int ret; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_inode_item *ii; + struct btrfs_key key; + + path = alloc_path_for_send(); + if (!path) + return -ENOMEM; + + key.objectid = ino; + key.type = BTRFS_INODE_ITEM_KEY; + key.offset = 0; + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret) { + if (ret > 0) + ret = -ENOENT; + return ret; + } + + if (!info) + return 0; + + ii = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_inode_item); + info->size = btrfs_inode_size(path->nodes[0], ii); + info->gen = btrfs_inode_generation(path->nodes[0], ii); + info->mode = btrfs_inode_mode(path->nodes[0], ii); + info->uid = btrfs_inode_uid(path->nodes[0], ii); + info->gid = btrfs_inode_gid(path->nodes[0], ii); + info->rdev = btrfs_inode_rdev(path->nodes[0], ii); + info->nlink = btrfs_inode_nlink(path->nodes[0], ii); + /* + * Transfer the unchanged u64 value of btrfs_inode_item::flags, that's + * otherwise logically split to 32/32 parts. + */ + info->fileattr = btrfs_inode_flags(path->nodes[0], ii); + + return 0; +} + +static int get_inode_gen(struct btrfs_root *root, u64 ino, u64 *gen) +{ + int ret; + struct btrfs_inode_info info = { 0 }; + + ASSERT(gen); + + ret = get_inode_info(root, ino, &info); + *gen = info.gen; + return ret; +} + +typedef int (*iterate_inode_ref_t)(u64 dir, struct fs_path *p, void *ctx); + +/* + * Helper function to iterate the entries in ONE btrfs_inode_ref or + * btrfs_inode_extref. + * The iterate callback may return a non zero value to stop iteration. This can + * be a negative value for error codes or 1 to simply stop it. + * + * path must point to the INODE_REF or INODE_EXTREF when called. + */ +static int iterate_inode_ref(struct btrfs_root *root, struct btrfs_path *path, + struct btrfs_key *found_key, bool resolve, + iterate_inode_ref_t iterate, void *ctx) +{ + struct extent_buffer *eb = path->nodes[0]; + struct btrfs_inode_ref *iref; + struct btrfs_inode_extref *extref; + BTRFS_PATH_AUTO_FREE(tmp_path); + struct fs_path *p __free(fs_path_free) = NULL; + u32 cur = 0; + u32 total; + int slot = path->slots[0]; + u32 name_len; + char *start; + int ret = 0; + u64 dir; + unsigned long name_off; + unsigned long elem_size; + unsigned long ptr; + + p = fs_path_alloc_reversed(); + if (!p) + return -ENOMEM; + + tmp_path = alloc_path_for_send(); + if (!tmp_path) + return -ENOMEM; + + if (found_key->type == BTRFS_INODE_REF_KEY) { + ptr = (unsigned long)btrfs_item_ptr(eb, slot, + struct btrfs_inode_ref); + total = btrfs_item_size(eb, slot); + elem_size = sizeof(*iref); + } else { + ptr = btrfs_item_ptr_offset(eb, slot); + total = btrfs_item_size(eb, slot); + elem_size = sizeof(*extref); + } + + while (cur < total) { + fs_path_reset(p); + + if (found_key->type == BTRFS_INODE_REF_KEY) { + iref = (struct btrfs_inode_ref *)(ptr + cur); + name_len = btrfs_inode_ref_name_len(eb, iref); + name_off = (unsigned long)(iref + 1); + dir = found_key->offset; + } else { + extref = (struct btrfs_inode_extref *)(ptr + cur); + name_len = btrfs_inode_extref_name_len(eb, extref); + name_off = (unsigned long)&extref->name; + dir = btrfs_inode_extref_parent(eb, extref); + } + + if (resolve) { + start = btrfs_ref_to_path(root, tmp_path, name_len, + name_off, eb, dir, + p->buf, p->buf_len); + if (IS_ERR(start)) + return PTR_ERR(start); + + if (start < p->buf) { + /* overflow , try again with larger buffer */ + ret = fs_path_ensure_buf(p, + p->buf_len + p->buf - start); + if (ret < 0) + return ret; + start = btrfs_ref_to_path(root, tmp_path, + name_len, name_off, + eb, dir, + p->buf, p->buf_len); + if (IS_ERR(start)) + return PTR_ERR(start); + + if (unlikely(start < p->buf)) { + btrfs_err(root->fs_info, + "send: path ref buffer underflow for key " BTRFS_KEY_FMT, + BTRFS_KEY_FMT_VALUE(found_key)); + return -EINVAL; + } + } + p->start = start; + } else { + ret = fs_path_add_from_extent_buffer(p, eb, name_off, + name_len); + if (ret < 0) + return ret; + } + + cur += elem_size + name_len; + ret = iterate(dir, p, ctx); + if (ret) + return ret; + } + + return ret; +} + +typedef int (*iterate_dir_item_t)(int num, struct btrfs_key *di_key, + const char *name, int name_len, + const char *data, int data_len, + void *ctx); + +/* + * Helper function to iterate the entries in ONE btrfs_dir_item. + * The iterate callback may return a non zero value to stop iteration. This can + * be a negative value for error codes or 1 to simply stop it. + * + * path must point to the dir item when called. + */ +static int iterate_dir_item(struct btrfs_root *root, struct btrfs_path *path, + iterate_dir_item_t iterate, void *ctx) +{ + int ret = 0; + struct extent_buffer *eb; + struct btrfs_dir_item *di; + struct btrfs_key di_key; + char *buf = NULL; + int buf_len; + u32 name_len; + u32 data_len; + u32 cur; + u32 len; + u32 total; + int slot; + int num; + + /* + * Start with a small buffer (1 page). If later we end up needing more + * space, which can happen for xattrs on a fs with a leaf size greater + * than the page size, attempt to increase the buffer. Typically xattr + * values are small. + */ + buf_len = PATH_MAX; + buf = kmalloc(buf_len, GFP_KERNEL); + if (!buf) { + ret = -ENOMEM; + goto out; + } + + eb = path->nodes[0]; + slot = path->slots[0]; + di = btrfs_item_ptr(eb, slot, struct btrfs_dir_item); + cur = 0; + len = 0; + total = btrfs_item_size(eb, slot); + + num = 0; + while (cur < total) { + name_len = btrfs_dir_name_len(eb, di); + data_len = btrfs_dir_data_len(eb, di); + btrfs_dir_item_key_to_cpu(eb, di, &di_key); + + if (btrfs_dir_ftype(eb, di) == BTRFS_FT_XATTR) { + if (unlikely(name_len > XATTR_NAME_MAX)) { + ret = -ENAMETOOLONG; + goto out; + } + if (unlikely(name_len + data_len > + BTRFS_MAX_XATTR_SIZE(root->fs_info))) { + ret = -E2BIG; + goto out; + } + } else { + /* + * Path too long + */ + if (unlikely(name_len + data_len > PATH_MAX)) { + ret = -ENAMETOOLONG; + goto out; + } + } + + if (name_len + data_len > buf_len) { + buf_len = name_len + data_len; + if (is_vmalloc_addr(buf)) { + vfree(buf); + buf = NULL; + } else { + char *tmp = krealloc(buf, buf_len, + GFP_KERNEL | __GFP_NOWARN); + + if (!tmp) + kfree(buf); + buf = tmp; + } + if (!buf) { + buf = kvmalloc(buf_len, GFP_KERNEL); + if (!buf) { + ret = -ENOMEM; + goto out; + } + } + } + + read_extent_buffer(eb, buf, (unsigned long)(di + 1), + name_len + data_len); + + len = sizeof(*di) + name_len + data_len; + di = (struct btrfs_dir_item *)((char *)di + len); + cur += len; + + ret = iterate(num, &di_key, buf, name_len, buf + name_len, + data_len, ctx); + if (ret < 0) + goto out; + if (ret) { + ret = 0; + goto out; + } + + num++; + } + +out: + kvfree(buf); + return ret; +} + +static int __copy_first_ref(u64 dir, struct fs_path *p, void *ctx) +{ + int ret; + struct fs_path *pt = ctx; + + ret = fs_path_copy(pt, p); + if (ret < 0) + return ret; + + /* we want the first only */ + return 1; +} + +/* + * Retrieve the first path of an inode. If an inode has more then one + * ref/hardlink, this is ignored. + */ +static int get_inode_path(struct btrfs_root *root, + u64 ino, struct fs_path *path) +{ + int ret; + struct btrfs_key key, found_key; + BTRFS_PATH_AUTO_FREE(p); + + p = alloc_path_for_send(); + if (!p) + return -ENOMEM; + + fs_path_reset(path); + + key.objectid = ino; + key.type = BTRFS_INODE_REF_KEY; + key.offset = 0; + + ret = btrfs_search_slot_for_read(root, &key, p, 1, 0); + if (ret < 0) + return ret; + if (ret) + return 1; + + btrfs_item_key_to_cpu(p->nodes[0], &found_key, p->slots[0]); + if (found_key.objectid != ino || + (found_key.type != BTRFS_INODE_REF_KEY && + found_key.type != BTRFS_INODE_EXTREF_KEY)) + return -ENOENT; + + ret = iterate_inode_ref(root, p, &found_key, true, __copy_first_ref, path); + if (ret < 0) + return ret; + return 0; +} + +struct backref_ctx { + struct send_ctx *sctx; + + /* number of total found references */ + u64 found; + + /* + * used for clones found in send_root. clones found behind cur_objectid + * and cur_offset are not considered as allowed clones. + */ + u64 cur_objectid; + u64 cur_offset; + + /* may be truncated in case it's the last extent in a file */ + u64 extent_len; + + /* The bytenr the file extent item we are processing refers to. */ + u64 bytenr; + /* The owner (root id) of the data backref for the current extent. */ + u64 backref_owner; + /* The offset of the data backref for the current extent. */ + u64 backref_offset; +}; + +static int __clone_root_cmp_bsearch(const void *key, const void *elt) +{ + u64 root = (u64)(uintptr_t)key; + const struct clone_root *cr = elt; + + if (root < btrfs_root_id(cr->root)) + return -1; + if (root > btrfs_root_id(cr->root)) + return 1; + return 0; +} + +static int __clone_root_cmp_sort(const void *e1, const void *e2) +{ + const struct clone_root *cr1 = e1; + const struct clone_root *cr2 = e2; + + if (btrfs_root_id(cr1->root) < btrfs_root_id(cr2->root)) + return -1; + if (btrfs_root_id(cr1->root) > btrfs_root_id(cr2->root)) + return 1; + return 0; +} + +/* + * Called for every backref that is found for the current extent. + * Results are collected in sctx->clone_roots->ino/offset. + */ +static int iterate_backrefs(u64 ino, u64 offset, u64 num_bytes, u64 root_id, + void *ctx_) +{ + struct backref_ctx *bctx = ctx_; + struct clone_root *clone_root; + + /* First check if the root is in the list of accepted clone sources */ + clone_root = bsearch((void *)(uintptr_t)root_id, bctx->sctx->clone_roots, + bctx->sctx->clone_roots_cnt, + sizeof(struct clone_root), + __clone_root_cmp_bsearch); + if (!clone_root) + return 0; + + /* This is our own reference, bail out as we can't clone from it. */ + if (clone_root->root == bctx->sctx->send_root && + ino == bctx->cur_objectid && + offset == bctx->cur_offset) + return 0; + + /* + * Make sure we don't consider clones from send_root that are + * behind the current inode/offset. + */ + if (clone_root->root == bctx->sctx->send_root) { + /* + * If the source inode was not yet processed we can't issue a + * clone operation, as the source extent does not exist yet at + * the destination of the stream. + */ + if (ino > bctx->cur_objectid) + return 0; + /* + * We clone from the inode currently being sent as long as the + * source extent is already processed, otherwise we could try + * to clone from an extent that does not exist yet at the + * destination of the stream. + */ + if (ino == bctx->cur_objectid && + offset + bctx->extent_len > + bctx->sctx->cur_inode_next_write_offset) + return 0; + } + + bctx->found++; + clone_root->found_ref = true; + + /* + * If the given backref refers to a file extent item with a larger + * number of bytes than what we found before, use the new one so that + * we clone more optimally and end up doing less writes and getting + * less exclusive, non-shared extents at the destination. + */ + if (num_bytes > clone_root->num_bytes) { + clone_root->ino = ino; + clone_root->offset = offset; + clone_root->num_bytes = num_bytes; + + /* + * Found a perfect candidate, so there's no need to continue + * backref walking. + */ + if (num_bytes >= bctx->extent_len) + return BTRFS_ITERATE_EXTENT_INODES_STOP; + } + + return 0; +} + +static bool lookup_backref_cache(u64 leaf_bytenr, void *ctx, + const u64 **root_ids_ret, int *root_count_ret) +{ + struct backref_ctx *bctx = ctx; + struct send_ctx *sctx = bctx->sctx; + struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; + const u64 key = leaf_bytenr >> fs_info->nodesize_bits; + struct btrfs_lru_cache_entry *raw_entry; + struct backref_cache_entry *entry; + + if (sctx->backref_cache.size == 0) + return false; + + /* + * If relocation happened since we first filled the cache, then we must + * empty the cache and can not use it, because even though we operate on + * read-only roots, their leaves and nodes may have been reallocated and + * now be used for different nodes/leaves of the same tree or some other + * tree. + * + * We are called from iterate_extent_inodes() while either holding a + * transaction handle or holding fs_info->commit_root_sem, so no need + * to take any lock here. + */ + if (fs_info->last_reloc_trans > sctx->backref_cache_last_reloc_trans) { + btrfs_lru_cache_clear(&sctx->backref_cache); + return false; + } + + raw_entry = btrfs_lru_cache_lookup(&sctx->backref_cache, key, 0); + if (!raw_entry) + return false; + + entry = container_of(raw_entry, struct backref_cache_entry, entry); + *root_ids_ret = entry->root_ids; + *root_count_ret = entry->num_roots; + + return true; +} + +static void store_backref_cache(u64 leaf_bytenr, const struct ulist *root_ids, + void *ctx) +{ + struct backref_ctx *bctx = ctx; + struct send_ctx *sctx = bctx->sctx; + struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; + struct backref_cache_entry *new_entry; + struct ulist_iterator uiter; + struct ulist_node *node; + int ret; + + /* + * We're called while holding a transaction handle or while holding + * fs_info->commit_root_sem (at iterate_extent_inodes()), so must do a + * NOFS allocation. + */ + new_entry = kmalloc_obj(struct backref_cache_entry, GFP_NOFS); + /* No worries, cache is optional. */ + if (!new_entry) + return; + + new_entry->entry.key = leaf_bytenr >> fs_info->nodesize_bits; + new_entry->entry.gen = 0; + new_entry->num_roots = 0; + ULIST_ITER_INIT(&uiter); + while ((node = ulist_next(root_ids, &uiter)) != NULL) { + const u64 root_id = node->val; + struct clone_root *root; + + root = bsearch((void *)(uintptr_t)root_id, sctx->clone_roots, + sctx->clone_roots_cnt, sizeof(struct clone_root), + __clone_root_cmp_bsearch); + if (!root) + continue; + + /* Too many roots, just exit, no worries as caching is optional. */ + if (new_entry->num_roots >= SEND_MAX_BACKREF_CACHE_ROOTS) { + kfree(new_entry); + return; + } + + new_entry->root_ids[new_entry->num_roots] = root_id; + new_entry->num_roots++; + } + + /* + * We may have not added any roots to the new cache entry, which means + * none of the roots is part of the list of roots from which we are + * allowed to clone. Cache the new entry as it's still useful to avoid + * backref walking to determine which roots have a path to the leaf. + * + * Also use GFP_NOFS because we're called while holding a transaction + * handle or while holding fs_info->commit_root_sem. + */ + ret = btrfs_lru_cache_store(&sctx->backref_cache, &new_entry->entry, + GFP_NOFS); + ASSERT(ret == 0 || ret == -ENOMEM); + if (ret) { + /* Caching is optional, no worries. */ + kfree(new_entry); + return; + } + + /* + * We are called from iterate_extent_inodes() while either holding a + * transaction handle or holding fs_info->commit_root_sem, so no need + * to take any lock here. + */ + if (sctx->backref_cache.size == 1) + sctx->backref_cache_last_reloc_trans = fs_info->last_reloc_trans; +} + +static int check_extent_item(u64 bytenr, const struct btrfs_extent_item *ei, + const struct extent_buffer *leaf, void *ctx) +{ + const u64 refs = btrfs_extent_refs(leaf, ei); + const struct backref_ctx *bctx = ctx; + const struct send_ctx *sctx = bctx->sctx; + + if (bytenr == bctx->bytenr) { + const u64 flags = btrfs_extent_flags(leaf, ei); + + if (WARN_ON(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)) + return -EUCLEAN; + + /* + * If we have only one reference and only the send root as a + * clone source - meaning no clone roots were given in the + * struct btrfs_ioctl_send_args passed to the send ioctl - then + * it's our reference and there's no point in doing backref + * walking which is expensive, so exit early. + */ + if (refs == 1 && sctx->clone_roots_cnt == 1) + return -ENOENT; + } + + /* + * Backreference walking (iterate_extent_inodes() below) is currently + * too expensive when an extent has a large number of references, both + * in time spent and used memory. So for now just fallback to write + * operations instead of clone operations when an extent has more than + * a certain amount of references. + */ + if (refs > SEND_MAX_EXTENT_REFS) + return -ENOENT; + + return 0; +} + +static bool skip_self_data_ref(u64 root, u64 ino, u64 offset, void *ctx) +{ + const struct backref_ctx *bctx = ctx; + + if (ino == bctx->cur_objectid && + root == bctx->backref_owner && + offset == bctx->backref_offset) + return true; + + return false; +} + +/* + * Given an inode, offset and extent item, it finds a good clone for a clone + * instruction. Returns -ENOENT when none could be found. The function makes + * sure that the returned clone is usable at the point where sending is at the + * moment. This means, that no clones are accepted which lie behind the current + * inode+offset. + * + * path must point to the extent item when called. + */ +static int find_extent_clone(struct send_ctx *sctx, + struct btrfs_path *path, + u64 ino, u64 data_offset, + u64 ino_size, + struct clone_root **found) +{ + struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; + int ret; + int extent_type; + u64 disk_byte; + u64 num_bytes; + struct btrfs_file_extent_item *fi; + struct extent_buffer *eb = path->nodes[0]; + struct backref_ctx backref_ctx = { 0 }; + struct btrfs_backref_walk_ctx backref_walk_ctx = { 0 }; + struct clone_root *cur_clone_root; + int compressed; + u32 i; + + /* + * With fallocate we can get prealloc extents beyond the inode's i_size, + * so we don't do anything here because clone operations can not clone + * to a range beyond i_size without increasing the i_size of the + * destination inode. + */ + if (data_offset >= ino_size) + return 0; + + fi = btrfs_item_ptr(eb, path->slots[0], struct btrfs_file_extent_item); + extent_type = btrfs_file_extent_type(eb, fi); + if (extent_type == BTRFS_FILE_EXTENT_INLINE) + return -ENOENT; + + disk_byte = btrfs_file_extent_disk_bytenr(eb, fi); + if (disk_byte == 0) + return -ENOENT; + + compressed = btrfs_file_extent_compression(eb, fi); + num_bytes = btrfs_file_extent_num_bytes(eb, fi); + + /* + * Setup the clone roots. + */ + for (i = 0; i < sctx->clone_roots_cnt; i++) { + cur_clone_root = sctx->clone_roots + i; + cur_clone_root->ino = (u64)-1; + cur_clone_root->offset = 0; + cur_clone_root->num_bytes = 0; + cur_clone_root->found_ref = false; + } + + backref_ctx.sctx = sctx; + backref_ctx.cur_objectid = ino; + backref_ctx.cur_offset = data_offset; + backref_ctx.bytenr = disk_byte; + /* + * Use the header owner and not the send root's id, because in case of a + * snapshot we can have shared subtrees. + */ + backref_ctx.backref_owner = btrfs_header_owner(eb); + backref_ctx.backref_offset = data_offset - btrfs_file_extent_offset(eb, fi); + + /* + * The last extent of a file may be too large due to page alignment. + * We need to adjust extent_len in this case so that the checks in + * iterate_backrefs() work. + */ + if (data_offset + num_bytes >= ino_size) + backref_ctx.extent_len = ino_size - data_offset; + else + backref_ctx.extent_len = num_bytes; + + /* + * Now collect all backrefs. + */ + backref_walk_ctx.bytenr = disk_byte; + if (compressed == BTRFS_COMPRESS_NONE) + backref_walk_ctx.extent_item_pos = btrfs_file_extent_offset(eb, fi); + backref_walk_ctx.fs_info = fs_info; + backref_walk_ctx.cache_lookup = lookup_backref_cache; + backref_walk_ctx.cache_store = store_backref_cache; + backref_walk_ctx.indirect_ref_iterator = iterate_backrefs; + backref_walk_ctx.check_extent_item = check_extent_item; + backref_walk_ctx.user_ctx = &backref_ctx; + + /* + * If have a single clone root, then it's the send root and we can tell + * the backref walking code to skip our own backref and not resolve it, + * since we can not use it for cloning - the source and destination + * ranges can't overlap and in case the leaf is shared through a subtree + * due to snapshots, we can't use those other roots since they are not + * in the list of clone roots. + */ + if (sctx->clone_roots_cnt == 1) + backref_walk_ctx.skip_data_ref = skip_self_data_ref; + + ret = iterate_extent_inodes(&backref_walk_ctx, true, iterate_backrefs, + &backref_ctx); + if (ret < 0) + return ret; + + down_read(&fs_info->commit_root_sem); + if (fs_info->last_reloc_trans > sctx->last_reloc_trans) { + /* + * A transaction commit for a transaction in which block group + * relocation was done just happened. + * The disk_bytenr of the file extent item we processed is + * possibly stale, referring to the extent's location before + * relocation. So act as if we haven't found any clone sources + * and fallback to write commands, which will read the correct + * data from the new extent location. Otherwise we will fail + * below because we haven't found our own back reference or we + * could be getting incorrect sources in case the old extent + * was already reallocated after the relocation. + */ + up_read(&fs_info->commit_root_sem); + return -ENOENT; + } + up_read(&fs_info->commit_root_sem); + + if (!backref_ctx.found) + return -ENOENT; + + cur_clone_root = NULL; + for (i = 0; i < sctx->clone_roots_cnt; i++) { + struct clone_root *clone_root = &sctx->clone_roots[i]; + + if (!clone_root->found_ref) + continue; + + /* + * Choose the root from which we can clone more bytes, to + * minimize write operations and therefore have more extent + * sharing at the destination (the same as in the source). + */ + if (!cur_clone_root || + clone_root->num_bytes > cur_clone_root->num_bytes) { + cur_clone_root = clone_root; + + /* + * We found an optimal clone candidate (any inode from + * any root is fine), so we're done. + */ + if (clone_root->num_bytes >= backref_ctx.extent_len) + break; + } + } + + if (cur_clone_root) { + *found = cur_clone_root; + ret = 0; + } else { + ret = -ENOENT; + } + + return ret; +} + +static int read_symlink(struct btrfs_root *root, + u64 ino, + struct fs_path *dest) +{ + int ret; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + struct btrfs_file_extent_item *ei; + u8 type; + u8 compression; + unsigned long off; + int len; + + path = alloc_path_for_send(); + if (!path) + return -ENOMEM; + + key.objectid = ino; + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = 0; + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + return ret; + if (unlikely(ret)) { + /* + * An empty symlink inode. Can happen in rare error paths when + * creating a symlink (transaction committed before the inode + * eviction handler removed the symlink inode items and a crash + * happened in between or the subvol was snapshotted in between). + * Print an informative message to dmesg/syslog so that the user + * can delete the symlink. + */ + btrfs_err(root->fs_info, + "Found empty symlink inode %llu at root %llu", + ino, btrfs_root_id(root)); + return -EIO; + } + + ei = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_file_extent_item); + type = btrfs_file_extent_type(path->nodes[0], ei); + if (unlikely(type != BTRFS_FILE_EXTENT_INLINE)) { + ret = -EUCLEAN; + btrfs_crit(root->fs_info, +"send: found symlink extent that is not inline, ino %llu root %llu extent type %d", + ino, btrfs_root_id(root), type); + return ret; + } + compression = btrfs_file_extent_compression(path->nodes[0], ei); + if (unlikely(compression != BTRFS_COMPRESS_NONE)) { + ret = -EUCLEAN; + btrfs_crit(root->fs_info, +"send: found symlink extent with compression, ino %llu root %llu compression type %d", + ino, btrfs_root_id(root), compression); + return ret; + } + + off = btrfs_file_extent_inline_start(ei); + len = btrfs_file_extent_ram_bytes(path->nodes[0], ei); + + return fs_path_add_from_extent_buffer(dest, path->nodes[0], off, len); +} + +/* + * Helper function to generate a file name that is unique in the root of + * send_root and parent_root. This is used to generate names for orphan inodes. + */ +static int gen_unique_name(struct send_ctx *sctx, + u64 ino, u64 gen, + struct fs_path *dest) +{ + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_dir_item *di; + char tmp[64]; + int len; + u64 idx = 0; + + path = alloc_path_for_send(); + if (!path) + return -ENOMEM; + + while (1) { + struct fscrypt_str tmp_name; + + len = snprintf(tmp, sizeof(tmp), "o%llu-%llu-%llu", + ino, gen, idx); + ASSERT(len < sizeof(tmp)); + tmp_name.name = tmp; + tmp_name.len = len; + + di = btrfs_lookup_dir_item(NULL, sctx->send_root, + path, BTRFS_FIRST_FREE_OBJECTID, + &tmp_name, 0); + btrfs_release_path(path); + if (IS_ERR(di)) + return PTR_ERR(di); + + if (di) { + /* not unique, try again */ + idx++; + continue; + } + + if (!sctx->parent_root) { + /* unique */ + break; + } + + di = btrfs_lookup_dir_item(NULL, sctx->parent_root, + path, BTRFS_FIRST_FREE_OBJECTID, + &tmp_name, 0); + btrfs_release_path(path); + if (IS_ERR(di)) + return PTR_ERR(di); + + if (di) { + /* not unique, try again */ + idx++; + continue; + } + /* unique */ + break; + } + + return fs_path_add(dest, tmp, len); +} + +enum inode_state { + inode_state_no_change, + inode_state_will_create, + inode_state_did_create, + inode_state_will_delete, + inode_state_did_delete, +}; + +static int get_cur_inode_state(struct send_ctx *sctx, u64 ino, u64 gen, + u64 *send_gen, u64 *parent_gen) +{ + int ret; + int left_ret; + int right_ret; + u64 left_gen; + u64 right_gen = 0; + struct btrfs_inode_info info; + + ret = get_inode_info(sctx->send_root, ino, &info); + if (ret < 0 && ret != -ENOENT) + return ret; + left_ret = (info.nlink == 0) ? -ENOENT : ret; + left_gen = info.gen; + if (send_gen) + *send_gen = ((left_ret == -ENOENT) ? 0 : info.gen); + + if (!sctx->parent_root) { + right_ret = -ENOENT; + } else { + ret = get_inode_info(sctx->parent_root, ino, &info); + if (ret < 0 && ret != -ENOENT) + return ret; + right_ret = (info.nlink == 0) ? -ENOENT : ret; + right_gen = info.gen; + if (parent_gen) + *parent_gen = ((right_ret == -ENOENT) ? 0 : info.gen); + } + + if (!left_ret && !right_ret) { + if (left_gen == gen && right_gen == gen) { + ret = inode_state_no_change; + } else if (left_gen == gen) { + if (ino < sctx->send_progress) + ret = inode_state_did_create; + else + ret = inode_state_will_create; + } else if (right_gen == gen) { + if (ino < sctx->send_progress) + ret = inode_state_did_delete; + else + ret = inode_state_will_delete; + } else { + ret = -ENOENT; + } + } else if (!left_ret) { + if (left_gen == gen) { + if (ino < sctx->send_progress) + ret = inode_state_did_create; + else + ret = inode_state_will_create; + } else { + ret = -ENOENT; + } + } else if (!right_ret) { + if (right_gen == gen) { + if (ino < sctx->send_progress) + ret = inode_state_did_delete; + else + ret = inode_state_will_delete; + } else { + ret = -ENOENT; + } + } else { + ret = -ENOENT; + } + + return ret; +} + +static int is_inode_existent(struct send_ctx *sctx, u64 ino, u64 gen, + u64 *send_gen, u64 *parent_gen) +{ + int ret; + + if (ino == BTRFS_FIRST_FREE_OBJECTID) + return 1; + + ret = get_cur_inode_state(sctx, ino, gen, send_gen, parent_gen); + if (ret < 0) + return ret; + + if (ret == inode_state_no_change || + ret == inode_state_did_create || + ret == inode_state_will_delete) + return 1; + + return 0; +} + +/* + * Helper function to lookup a dir item in a dir. + */ +static int lookup_dir_item_inode(struct btrfs_root *root, + u64 dir, const char *name, int name_len, + u64 *found_inode) +{ + int ret = 0; + struct btrfs_dir_item *di; + struct btrfs_key key; + BTRFS_PATH_AUTO_FREE(path); + struct fscrypt_str name_str = FSTR_INIT((char *)name, name_len); + + path = alloc_path_for_send(); + if (!path) + return -ENOMEM; + + di = btrfs_lookup_dir_item(NULL, root, path, dir, &name_str, 0); + if (IS_ERR_OR_NULL(di)) + return di ? PTR_ERR(di) : -ENOENT; + + btrfs_dir_item_key_to_cpu(path->nodes[0], di, &key); + if (key.type == BTRFS_ROOT_ITEM_KEY) + return -ENOENT; + + *found_inode = key.objectid; + + return ret; +} + +/* + * Looks up the first btrfs_inode_ref of a given ino. It returns the parent dir, + * generation of the parent dir and the name of the dir entry. + */ +static int get_first_ref(struct btrfs_root *root, u64 ino, + u64 *dir, u64 *dir_gen, struct fs_path *name) +{ + int ret; + struct btrfs_key key; + struct btrfs_key found_key; + BTRFS_PATH_AUTO_FREE(path); + int len; + u64 parent_dir; + + path = alloc_path_for_send(); + if (!path) + return -ENOMEM; + + key.objectid = ino; + key.type = BTRFS_INODE_REF_KEY; + key.offset = 0; + + ret = btrfs_search_slot_for_read(root, &key, path, 1, 0); + if (ret < 0) + return ret; + if (!ret) + btrfs_item_key_to_cpu(path->nodes[0], &found_key, + path->slots[0]); + if (ret || found_key.objectid != ino || + (found_key.type != BTRFS_INODE_REF_KEY && + found_key.type != BTRFS_INODE_EXTREF_KEY)) + return -ENOENT; + + if (found_key.type == BTRFS_INODE_REF_KEY) { + struct btrfs_inode_ref *iref; + iref = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_inode_ref); + len = btrfs_inode_ref_name_len(path->nodes[0], iref); + ret = fs_path_add_from_extent_buffer(name, path->nodes[0], + (unsigned long)(iref + 1), + len); + parent_dir = found_key.offset; + } else { + struct btrfs_inode_extref *extref; + extref = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_inode_extref); + len = btrfs_inode_extref_name_len(path->nodes[0], extref); + ret = fs_path_add_from_extent_buffer(name, path->nodes[0], + (unsigned long)&extref->name, len); + parent_dir = btrfs_inode_extref_parent(path->nodes[0], extref); + } + if (ret < 0) + return ret; + btrfs_release_path(path); + + if (dir_gen) { + ret = get_inode_gen(root, parent_dir, dir_gen); + if (ret < 0) + return ret; + } + + *dir = parent_dir; + + return ret; +} + +static int is_first_ref(struct btrfs_root *root, + u64 ino, u64 dir, + const char *name, int name_len) +{ + int ret; + struct fs_path *tmp_name __free(fs_path_free) = NULL; + u64 tmp_dir; + + tmp_name = fs_path_alloc(); + if (!tmp_name) + return -ENOMEM; + + ret = get_first_ref(root, ino, &tmp_dir, NULL, tmp_name); + if (ret < 0) + return ret; + + if (dir != tmp_dir || name_len != fs_path_len(tmp_name)) + return 0; + + ret = !memcmp(tmp_name->start, name, name_len); + + return ret; +} + +/* + * Used by process_recorded_refs to determine if a new ref would overwrite an + * already existing ref. In case it detects an overwrite, it returns the + * inode/gen in who_ino/who_gen. + * When an overwrite is detected, process_recorded_refs does proper orphanizing + * to make sure later references to the overwritten inode are possible. + * Orphanizing is however only required for the first ref of an inode. + * process_recorded_refs does an additional is_first_ref check to see if + * orphanizing is really required. + */ +static int will_overwrite_ref(struct send_ctx *sctx, u64 dir, u64 dir_gen, + const char *name, int name_len, + u64 *who_ino, u64 *who_gen, u64 *who_mode) +{ + int ret; + u64 parent_root_dir_gen; + u64 other_inode = 0; + struct btrfs_inode_info info; + + if (!sctx->parent_root) + return 0; + + ret = is_inode_existent(sctx, dir, dir_gen, NULL, &parent_root_dir_gen); + if (ret <= 0) + return 0; + + /* + * If we have a parent root we need to verify that the parent dir was + * not deleted and then re-created, if it was then we have no overwrite + * and we can just unlink this entry. + * + * @parent_root_dir_gen was set to 0 if the inode does not exist in the + * parent root. + */ + if (sctx->parent_root && dir != BTRFS_FIRST_FREE_OBJECTID && + parent_root_dir_gen != dir_gen) + return 0; + + ret = lookup_dir_item_inode(sctx->parent_root, dir, name, name_len, + &other_inode); + if (ret == -ENOENT) + return 0; + else if (ret < 0) + return ret; + + /* + * Check if the overwritten ref was already processed. If yes, the ref + * was already unlinked/moved, so we can safely assume that we will not + * overwrite anything at this point in time. + */ + if (other_inode > sctx->send_progress || + is_waiting_for_move(sctx, other_inode)) { + ret = get_inode_info(sctx->parent_root, other_inode, &info); + if (ret < 0) + return ret; + + *who_ino = other_inode; + *who_gen = info.gen; + *who_mode = info.mode; + return 1; + } + + return 0; +} + +/* + * Checks if the ref was overwritten by an already processed inode. This is + * used by __get_cur_name_and_parent to find out if the ref was orphanized and + * thus the orphan name needs be used. + * process_recorded_refs also uses it to avoid unlinking of refs that were + * overwritten. + */ +static int did_overwrite_ref(struct send_ctx *sctx, + u64 dir, u64 dir_gen, + u64 ino, u64 ino_gen, + const char *name, int name_len) +{ + int ret; + u64 ow_inode; + u64 ow_gen = 0; + u64 send_root_dir_gen; + + if (!sctx->parent_root) + return 0; + + ret = is_inode_existent(sctx, dir, dir_gen, &send_root_dir_gen, NULL); + if (ret <= 0) + return ret; + + /* + * @send_root_dir_gen was set to 0 if the inode does not exist in the + * send root. + */ + if (dir != BTRFS_FIRST_FREE_OBJECTID && send_root_dir_gen != dir_gen) + return 0; + + /* check if the ref was overwritten by another ref */ + ret = lookup_dir_item_inode(sctx->send_root, dir, name, name_len, + &ow_inode); + if (ret == -ENOENT) { + /* was never and will never be overwritten */ + return 0; + } else if (ret < 0) { + return ret; + } + + if (ow_inode == ino) { + ret = get_inode_gen(sctx->send_root, ow_inode, &ow_gen); + if (ret < 0) + return ret; + + /* It's the same inode, so no overwrite happened. */ + if (ow_gen == ino_gen) + return 0; + } + + /* + * We know that it is or will be overwritten. Check this now. + * The current inode being processed might have been the one that caused + * inode 'ino' to be orphanized, therefore check if ow_inode matches + * the current inode being processed. + */ + if (ow_inode < sctx->send_progress) + return 1; + + if (ino != sctx->cur_ino && ow_inode == sctx->cur_ino) { + if (ow_gen == 0) { + ret = get_inode_gen(sctx->send_root, ow_inode, &ow_gen); + if (ret < 0) + return ret; + } + if (ow_gen == sctx->cur_inode_gen) + return 1; + } + + return 0; +} + +/* + * Same as did_overwrite_ref, but also checks if it is the first ref of an inode + * that got overwritten. This is used by process_recorded_refs to determine + * if it has to use the path as returned by get_cur_path or the orphan name. + */ +static int did_overwrite_first_ref(struct send_ctx *sctx, u64 ino, u64 gen) +{ + int ret; + struct fs_path *name __free(fs_path_free) = NULL; + u64 dir; + u64 dir_gen; + + if (!sctx->parent_root) + return 0; + + name = fs_path_alloc(); + if (!name) + return -ENOMEM; + + ret = get_first_ref(sctx->parent_root, ino, &dir, &dir_gen, name); + if (ret < 0) + return ret; + + return did_overwrite_ref(sctx, dir, dir_gen, ino, gen, + name->start, fs_path_len(name)); +} + +static inline struct name_cache_entry *name_cache_search(struct send_ctx *sctx, + u64 ino, u64 gen) +{ + struct btrfs_lru_cache_entry *entry; + + entry = btrfs_lru_cache_lookup(&sctx->name_cache, ino, gen); + if (!entry) + return NULL; + + return container_of(entry, struct name_cache_entry, entry); +} + +/* + * Used by get_cur_path for each ref up to the root. + * Returns 0 if it succeeded. + * Returns 1 if the inode is not existent or got overwritten. In that case, the + * name is an orphan name. This instructs get_cur_path to stop iterating. If 1 + * is returned, parent_ino/parent_gen are not guaranteed to be valid. + * Returns <0 in case of error. + */ +static int __get_cur_name_and_parent(struct send_ctx *sctx, + u64 ino, u64 gen, + u64 *parent_ino, + u64 *parent_gen, + struct fs_path *dest) +{ + int ret; + int nce_ret; + struct name_cache_entry *nce; + + /* + * First check if we already did a call to this function with the same + * ino/gen. If yes, check if the cache entry is still up-to-date. If yes + * return the cached result. + */ + nce = name_cache_search(sctx, ino, gen); + if (nce) { + if (ino < sctx->send_progress && nce->need_later_update) { + btrfs_lru_cache_remove(&sctx->name_cache, &nce->entry); + nce = NULL; + } else { + *parent_ino = nce->parent_ino; + *parent_gen = nce->parent_gen; + ret = fs_path_add(dest, nce->name, nce->name_len); + if (ret < 0) + return ret; + return nce->ret; + } + } + + /* + * If the inode is not existent yet, add the orphan name and return 1. + * This should only happen for the parent dir that we determine in + * record_new_ref_if_needed(). + */ + ret = is_inode_existent(sctx, ino, gen, NULL, NULL); + if (ret < 0) + return ret; + + if (!ret) { + ret = gen_unique_name(sctx, ino, gen, dest); + if (ret < 0) + return ret; + ret = 1; + goto out_cache; + } + + /* + * Depending on whether the inode was already processed or not, use + * send_root or parent_root for ref lookup. + */ + if (ino < sctx->send_progress) + ret = get_first_ref(sctx->send_root, ino, + parent_ino, parent_gen, dest); + else + ret = get_first_ref(sctx->parent_root, ino, + parent_ino, parent_gen, dest); + if (ret < 0) + return ret; + + /* + * Check if the ref was overwritten by an inode's ref that was processed + * earlier. If yes, treat as orphan and return 1. + */ + ret = did_overwrite_ref(sctx, *parent_ino, *parent_gen, ino, gen, + dest->start, fs_path_len(dest)); + if (ret < 0) + return ret; + if (ret) { + fs_path_reset(dest); + ret = gen_unique_name(sctx, ino, gen, dest); + if (ret < 0) + return ret; + ret = 1; + } + +out_cache: + /* + * Store the result of the lookup in the name cache. + */ + nce = kmalloc(sizeof(*nce) + fs_path_len(dest), GFP_KERNEL); + if (!nce) + return -ENOMEM; + + nce->entry.key = ino; + nce->entry.gen = gen; + nce->parent_ino = *parent_ino; + nce->parent_gen = *parent_gen; + nce->name_len = fs_path_len(dest); + nce->ret = ret; + memcpy(nce->name, dest->start, nce->name_len); + + if (ino < sctx->send_progress) + nce->need_later_update = 0; + else + nce->need_later_update = 1; + + nce_ret = btrfs_lru_cache_store(&sctx->name_cache, &nce->entry, GFP_KERNEL); + if (nce_ret < 0) { + kfree(nce); + return nce_ret; + } + + return ret; +} + +/* + * Magic happens here. This function returns the first ref to an inode as it + * would look like while receiving the stream at this point in time. + * We walk the path up to the root. For every inode in between, we check if it + * was already processed/sent. If yes, we continue with the parent as found + * in send_root. If not, we continue with the parent as found in parent_root. + * If we encounter an inode that was deleted at this point in time, we use the + * inodes "orphan" name instead of the real name and stop. Same with new inodes + * that were not created yet and overwritten inodes/refs. + * + * When do we have orphan inodes: + * 1. When an inode is freshly created and thus no valid refs are available yet + * 2. When a directory lost all it's refs (deleted) but still has dir items + * inside which were not processed yet (pending for move/delete). If anyone + * tried to get the path to the dir items, it would get a path inside that + * orphan directory. + * 3. When an inode is moved around or gets new links, it may overwrite the ref + * of an unprocessed inode. If in that case the first ref would be + * overwritten, the overwritten inode gets "orphanized". Later when we + * process this overwritten inode, it is restored at a new place by moving + * the orphan inode. + * + * sctx->send_progress tells this function at which point in time receiving + * would be. + */ +static int get_cur_path(struct send_ctx *sctx, u64 ino, u64 gen, + struct fs_path *dest) +{ + int ret = 0; + struct fs_path *name __free(fs_path_free) = NULL; + u64 parent_inode = 0; + u64 parent_gen = 0; + bool stop = false; + const bool is_cur_inode = (ino == sctx->cur_ino && gen == sctx->cur_inode_gen); + + if (is_cur_inode && fs_path_len(&sctx->cur_inode_path) > 0) { + if (dest != &sctx->cur_inode_path) + return fs_path_copy(dest, &sctx->cur_inode_path); + + return 0; + } + + name = fs_path_alloc(); + if (!name) + return -ENOMEM; + + dest->reversed = 1; + fs_path_reset(dest); + + while (!stop && ino != BTRFS_FIRST_FREE_OBJECTID) { + struct waiting_dir_move *wdm; + + fs_path_reset(name); + + if (is_waiting_for_rm(sctx, ino, gen)) { + ret = gen_unique_name(sctx, ino, gen, name); + if (ret < 0) + goto out; + ret = fs_path_add_path(dest, name); + break; + } + + wdm = get_waiting_dir_move(sctx, ino); + if (wdm && wdm->orphanized) { + ret = gen_unique_name(sctx, ino, gen, name); + stop = true; + } else if (wdm) { + ret = get_first_ref(sctx->parent_root, ino, + &parent_inode, &parent_gen, name); + } else { + ret = __get_cur_name_and_parent(sctx, ino, gen, + &parent_inode, + &parent_gen, name); + if (ret) + stop = true; + } + + if (ret < 0) + goto out; + + ret = fs_path_add_path(dest, name); + if (ret < 0) + goto out; + + ino = parent_inode; + gen = parent_gen; + } + +out: + if (!ret) { + fs_path_unreverse(dest); + if (is_cur_inode && dest != &sctx->cur_inode_path) + ret = fs_path_copy(&sctx->cur_inode_path, dest); + } + + return ret; +} + +/* + * Sends a BTRFS_SEND_C_SUBVOL command/item to userspace + */ +static int send_subvol_begin(struct send_ctx *sctx) +{ + int ret; + struct btrfs_root *send_root = sctx->send_root; + struct btrfs_root *parent_root = sctx->parent_root; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + struct btrfs_root_ref *ref; + struct extent_buffer *leaf; + char AUTO_KFREE(name); + int namelen; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + name = kmalloc(BTRFS_PATH_NAME_MAX, GFP_KERNEL); + if (!name) + return -ENOMEM; + + key.objectid = btrfs_root_id(send_root); + key.type = BTRFS_ROOT_BACKREF_KEY; + key.offset = 0; + + ret = btrfs_search_slot_for_read(send_root->fs_info->tree_root, + &key, path, 1, 0); + if (ret < 0) + return ret; + if (ret) + return -ENOENT; + + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + if (key.type != BTRFS_ROOT_BACKREF_KEY || + key.objectid != btrfs_root_id(send_root)) { + return -ENOENT; + } + ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref); + namelen = btrfs_root_ref_name_len(leaf, ref); + read_extent_buffer(leaf, name, (unsigned long)(ref + 1), namelen); + btrfs_release_path(path); + + if (parent_root) { + ret = begin_cmd(sctx, BTRFS_SEND_C_SNAPSHOT); + if (ret < 0) + return ret; + } else { + ret = begin_cmd(sctx, BTRFS_SEND_C_SUBVOL); + if (ret < 0) + return ret; + } + + TLV_PUT_STRING(sctx, BTRFS_SEND_A_PATH, name, namelen); + + if (!btrfs_is_empty_uuid(sctx->send_root->root_item.received_uuid)) + TLV_PUT_UUID(sctx, BTRFS_SEND_A_UUID, + sctx->send_root->root_item.received_uuid); + else + TLV_PUT_UUID(sctx, BTRFS_SEND_A_UUID, + sctx->send_root->root_item.uuid); + + TLV_PUT_U64(sctx, BTRFS_SEND_A_CTRANSID, + btrfs_root_ctransid(&sctx->send_root->root_item)); + if (parent_root) { + if (!btrfs_is_empty_uuid(parent_root->root_item.received_uuid)) + TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID, + parent_root->root_item.received_uuid); + else + TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID, + parent_root->root_item.uuid); + TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_CTRANSID, + btrfs_root_ctransid(&sctx->parent_root->root_item)); + } + + ret = send_cmd(sctx); + +tlv_put_failure: + return ret; +} + +static struct fs_path *get_cur_inode_path(struct send_ctx *sctx) +{ + if (fs_path_len(&sctx->cur_inode_path) == 0) { + int ret; + + ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, + &sctx->cur_inode_path); + if (ret < 0) + return ERR_PTR(ret); + } + + return &sctx->cur_inode_path; +} + +static struct fs_path *get_path_for_command(struct send_ctx *sctx, u64 ino, u64 gen) +{ + struct fs_path *path; + int ret; + + if (ino == sctx->cur_ino && gen == sctx->cur_inode_gen) + return get_cur_inode_path(sctx); + + path = fs_path_alloc(); + if (!path) + return ERR_PTR(-ENOMEM); + + ret = get_cur_path(sctx, ino, gen, path); + if (ret < 0) { + fs_path_free(path); + return ERR_PTR(ret); + } + + return path; +} + +static void free_path_for_command(const struct send_ctx *sctx, struct fs_path *path) +{ + if (path != &sctx->cur_inode_path) + fs_path_free(path); +} + +static int send_truncate(struct send_ctx *sctx, u64 ino, u64 gen, u64 size) +{ + int ret = 0; + struct fs_path *p; + + p = get_path_for_command(sctx, ino, gen); + if (IS_ERR(p)) + return PTR_ERR(p); + + ret = begin_cmd(sctx, BTRFS_SEND_C_TRUNCATE); + if (ret < 0) + goto out; + + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); + TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, size); + + ret = send_cmd(sctx); + +tlv_put_failure: +out: + free_path_for_command(sctx, p); + return ret; +} + +static int send_chmod(struct send_ctx *sctx, u64 ino, u64 gen, u64 mode) +{ + int ret = 0; + struct fs_path *p; + + p = get_path_for_command(sctx, ino, gen); + if (IS_ERR(p)) + return PTR_ERR(p); + + ret = begin_cmd(sctx, BTRFS_SEND_C_CHMOD); + if (ret < 0) + goto out; + + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); + TLV_PUT_U64(sctx, BTRFS_SEND_A_MODE, mode & 07777); + + ret = send_cmd(sctx); + +tlv_put_failure: +out: + free_path_for_command(sctx, p); + return ret; +} + +static int send_fileattr(struct send_ctx *sctx, u64 ino, u64 gen, u64 fileattr) +{ + int ret = 0; + struct fs_path *p; + + if (sctx->proto < 2) + return 0; + + p = get_path_for_command(sctx, ino, gen); + if (IS_ERR(p)) + return PTR_ERR(p); + + ret = begin_cmd(sctx, BTRFS_SEND_C_FILEATTR); + if (ret < 0) + goto out; + + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); + TLV_PUT_U64(sctx, BTRFS_SEND_A_FILEATTR, fileattr); + + ret = send_cmd(sctx); + +tlv_put_failure: +out: + free_path_for_command(sctx, p); + return ret; +} + +static int send_chown(struct send_ctx *sctx, u64 ino, u64 gen, u64 uid, u64 gid) +{ + int ret = 0; + struct fs_path *p; + + p = get_path_for_command(sctx, ino, gen); + if (IS_ERR(p)) + return PTR_ERR(p); + + ret = begin_cmd(sctx, BTRFS_SEND_C_CHOWN); + if (ret < 0) + goto out; + + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); + TLV_PUT_U64(sctx, BTRFS_SEND_A_UID, uid); + TLV_PUT_U64(sctx, BTRFS_SEND_A_GID, gid); + + ret = send_cmd(sctx); + +tlv_put_failure: +out: + free_path_for_command(sctx, p); + return ret; +} + +static int send_utimes(struct send_ctx *sctx, u64 ino, u64 gen) +{ + int ret = 0; + struct fs_path *p = NULL; + struct btrfs_inode_item *ii; + BTRFS_PATH_AUTO_FREE(path); + struct extent_buffer *eb; + struct btrfs_key key; + int slot; + + p = get_path_for_command(sctx, ino, gen); + if (IS_ERR(p)) + return PTR_ERR(p); + + path = alloc_path_for_send(); + if (!path) { + ret = -ENOMEM; + goto out; + } + + key.objectid = ino; + key.type = BTRFS_INODE_ITEM_KEY; + key.offset = 0; + ret = btrfs_search_slot(NULL, sctx->send_root, &key, path, 0, 0); + if (ret > 0) + ret = -ENOENT; + if (ret < 0) + goto out; + + eb = path->nodes[0]; + slot = path->slots[0]; + ii = btrfs_item_ptr(eb, slot, struct btrfs_inode_item); + + ret = begin_cmd(sctx, BTRFS_SEND_C_UTIMES); + if (ret < 0) + goto out; + + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); + TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_ATIME, eb, &ii->atime); + TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_MTIME, eb, &ii->mtime); + TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_CTIME, eb, &ii->ctime); + if (sctx->proto >= 2) + TLV_PUT_BTRFS_TIMESPEC(sctx, BTRFS_SEND_A_OTIME, eb, &ii->otime); + + ret = send_cmd(sctx); + +tlv_put_failure: +out: + free_path_for_command(sctx, p); + return ret; +} + +/* + * If the cache is full, we can't remove entries from it and do a call to + * send_utimes() for each respective inode, because we might be finishing + * processing an inode that is a directory and it just got renamed, and existing + * entries in the cache may refer to inodes that have the directory in their + * full path - in which case we would generate outdated paths (pre-rename) + * for the inodes that the cache entries point to. Instead of pruning the + * cache when inserting, do it after we finish processing each inode at + * finish_inode_if_needed(). + */ +static int cache_dir_utimes(struct send_ctx *sctx, u64 dir, u64 gen) +{ + struct btrfs_lru_cache_entry *entry; + int ret; + + entry = btrfs_lru_cache_lookup(&sctx->dir_utimes_cache, dir, gen); + if (entry != NULL) + return 0; + + /* Caching is optional, don't fail if we can't allocate memory. */ + entry = kmalloc_obj(*entry); + if (!entry) + return send_utimes(sctx, dir, gen); + + entry->key = dir; + entry->gen = gen; + + ret = btrfs_lru_cache_store(&sctx->dir_utimes_cache, entry, GFP_KERNEL); + ASSERT(ret != -EEXIST); + if (ret) { + kfree(entry); + return send_utimes(sctx, dir, gen); + } + + return 0; +} + +static int trim_dir_utimes_cache(struct send_ctx *sctx) +{ + while (sctx->dir_utimes_cache.size > SEND_MAX_DIR_UTIMES_CACHE_SIZE) { + struct btrfs_lru_cache_entry *lru; + int ret; + + lru = btrfs_lru_cache_lru_entry(&sctx->dir_utimes_cache); + ASSERT(lru != NULL); + + ret = send_utimes(sctx, lru->key, lru->gen); + if (ret) + return ret; + + btrfs_lru_cache_remove(&sctx->dir_utimes_cache, lru); + } + + return 0; +} + +/* + * Sends a BTRFS_SEND_C_MKXXX or SYMLINK command to user space. We don't have + * a valid path yet because we did not process the refs yet. So, the inode + * is created as orphan. + */ +static int send_create_inode(struct send_ctx *sctx, u64 ino) +{ + int ret = 0; + struct fs_path *p __free(fs_path_free) = NULL; + int cmd; + struct btrfs_inode_info info; + u64 gen; + u64 mode; + u64 rdev; + + p = fs_path_alloc(); + if (!p) + return -ENOMEM; + + if (ino != sctx->cur_ino) { + ret = get_inode_info(sctx->send_root, ino, &info); + if (ret < 0) + return ret; + gen = info.gen; + mode = info.mode; + rdev = info.rdev; + } else { + gen = sctx->cur_inode_gen; + mode = sctx->cur_inode_mode; + rdev = sctx->cur_inode_rdev; + } + + if (S_ISREG(mode)) { + cmd = BTRFS_SEND_C_MKFILE; + } else if (S_ISDIR(mode)) { + cmd = BTRFS_SEND_C_MKDIR; + } else if (S_ISLNK(mode)) { + cmd = BTRFS_SEND_C_SYMLINK; + } else if (S_ISCHR(mode) || S_ISBLK(mode)) { + cmd = BTRFS_SEND_C_MKNOD; + } else if (S_ISFIFO(mode)) { + cmd = BTRFS_SEND_C_MKFIFO; + } else if (S_ISSOCK(mode)) { + cmd = BTRFS_SEND_C_MKSOCK; + } else { + btrfs_warn(sctx->send_root->fs_info, "unexpected inode type %o", + (int)(mode & S_IFMT)); + return -EOPNOTSUPP; + } + + ret = begin_cmd(sctx, cmd); + if (ret < 0) + return ret; + + ret = gen_unique_name(sctx, ino, gen, p); + if (ret < 0) + return ret; + + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); + TLV_PUT_U64(sctx, BTRFS_SEND_A_INO, ino); + + if (S_ISLNK(mode)) { + fs_path_reset(p); + ret = read_symlink(sctx->send_root, ino, p); + if (ret < 0) + return ret; + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH_LINK, p); + } else if (S_ISCHR(mode) || S_ISBLK(mode) || + S_ISFIFO(mode) || S_ISSOCK(mode)) { + TLV_PUT_U64(sctx, BTRFS_SEND_A_RDEV, new_encode_dev(rdev)); + TLV_PUT_U64(sctx, BTRFS_SEND_A_MODE, mode); + } + + ret = send_cmd(sctx); + if (ret < 0) + return ret; + +tlv_put_failure: + return ret; +} + +static void cache_dir_created(struct send_ctx *sctx, u64 dir) +{ + struct btrfs_lru_cache_entry *entry; + int ret; + + /* Caching is optional, ignore any failures. */ + entry = kmalloc_obj(*entry); + if (!entry) + return; + + entry->key = dir; + entry->gen = 0; + ret = btrfs_lru_cache_store(&sctx->dir_created_cache, entry, GFP_KERNEL); + if (ret < 0) + kfree(entry); +} + +/* + * We need some special handling for inodes that get processed before the parent + * directory got created. See process_recorded_refs for details. + * This function does the check if we already created the dir out of order. + */ +static int did_create_dir(struct send_ctx *sctx, u64 dir) +{ + int ret = 0; + int iter_ret = 0; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + struct btrfs_key found_key; + struct btrfs_key di_key; + struct btrfs_dir_item *di; + + if (btrfs_lru_cache_lookup(&sctx->dir_created_cache, dir, 0)) + return 1; + + path = alloc_path_for_send(); + if (!path) + return -ENOMEM; + + key.objectid = dir; + key.type = BTRFS_DIR_INDEX_KEY; + key.offset = 0; + + btrfs_for_each_slot(sctx->send_root, &key, &found_key, path, iter_ret) { + struct extent_buffer *eb = path->nodes[0]; + + if (found_key.objectid != key.objectid || + found_key.type != key.type) { + ret = 0; + break; + } + + di = btrfs_item_ptr(eb, path->slots[0], struct btrfs_dir_item); + btrfs_dir_item_key_to_cpu(eb, di, &di_key); + + if (di_key.type != BTRFS_ROOT_ITEM_KEY && + di_key.objectid < sctx->send_progress) { + ret = 1; + cache_dir_created(sctx, dir); + break; + } + } + /* Catch error found during iteration */ + if (iter_ret < 0) + ret = iter_ret; + + return ret; +} + +/* + * Only creates the inode if it is: + * 1. Not a directory + * 2. Or a directory which was not created already due to out of order + * directories. See did_create_dir and process_recorded_refs for details. + */ +static int send_create_inode_if_needed(struct send_ctx *sctx) +{ + int ret; + + if (S_ISDIR(sctx->cur_inode_mode)) { + ret = did_create_dir(sctx, sctx->cur_ino); + if (ret < 0) + return ret; + else if (ret > 0) + return 0; + } + + ret = send_create_inode(sctx, sctx->cur_ino); + + if (ret == 0 && S_ISDIR(sctx->cur_inode_mode)) + cache_dir_created(sctx, sctx->cur_ino); + + return ret; +} + +struct recorded_ref { + struct list_head list; + char *name; + struct fs_path *full_path; + u64 dir; + u64 dir_gen; + int name_len; + struct rb_node node; + struct rb_root *root; +}; + +static struct recorded_ref *recorded_ref_alloc(void) +{ + struct recorded_ref *ref; + + ref = kzalloc_obj(*ref); + if (!ref) + return NULL; + RB_CLEAR_NODE(&ref->node); + INIT_LIST_HEAD(&ref->list); + return ref; +} + +static void recorded_ref_free(struct recorded_ref *ref) +{ + if (!ref) + return; + if (!RB_EMPTY_NODE(&ref->node)) + rb_erase(&ref->node, ref->root); + list_del(&ref->list); + fs_path_free(ref->full_path); + kfree(ref); +} + +static void set_ref_path(struct recorded_ref *ref, struct fs_path *path) +{ + ref->full_path = path; + ref->name = (char *)kbasename(ref->full_path->start); + ref->name_len = ref->full_path->end - ref->name; +} + +static int dup_ref(struct recorded_ref *ref, struct list_head *list) +{ + struct recorded_ref *new; + + new = recorded_ref_alloc(); + if (!new) + return -ENOMEM; + + new->dir = ref->dir; + new->dir_gen = ref->dir_gen; + list_add_tail(&new->list, list); + return 0; +} + +static void __free_recorded_refs(struct list_head *head) +{ + struct recorded_ref *cur; + + while (!list_empty(head)) { + cur = list_first_entry(head, struct recorded_ref, list); + recorded_ref_free(cur); + } +} + +static void free_recorded_refs(struct send_ctx *sctx) +{ + __free_recorded_refs(&sctx->new_refs); + __free_recorded_refs(&sctx->deleted_refs); +} + +/* + * Renames/moves a file/dir to its orphan name. Used when the first + * ref of an unprocessed inode gets overwritten and for all non empty + * directories. + */ +static int orphanize_inode(struct send_ctx *sctx, u64 ino, u64 gen, + struct fs_path *path) +{ + int ret; + struct fs_path *orphan __free(fs_path_free) = NULL; + + orphan = fs_path_alloc(); + if (!orphan) + return -ENOMEM; + + ret = gen_unique_name(sctx, ino, gen, orphan); + if (ret < 0) + return ret; + + ret = send_rename(sctx, path, orphan); + if (ret < 0) + return ret; + + if (ino == sctx->cur_ino && gen == sctx->cur_inode_gen) + ret = fs_path_copy(&sctx->cur_inode_path, orphan); + + return ret; +} + +static struct orphan_dir_info *add_orphan_dir_info(struct send_ctx *sctx, + u64 dir_ino, u64 dir_gen) +{ + struct rb_node **p = &sctx->orphan_dirs.rb_node; + struct rb_node *parent = NULL; + struct orphan_dir_info *entry, *odi; + + while (*p) { + parent = *p; + entry = rb_entry(parent, struct orphan_dir_info, node); + if (dir_ino < entry->ino) + p = &(*p)->rb_left; + else if (dir_ino > entry->ino) + p = &(*p)->rb_right; + else if (dir_gen < entry->gen) + p = &(*p)->rb_left; + else if (dir_gen > entry->gen) + p = &(*p)->rb_right; + else + return entry; + } + + odi = kmalloc_obj(*odi); + if (!odi) + return ERR_PTR(-ENOMEM); + odi->ino = dir_ino; + odi->gen = dir_gen; + odi->last_dir_index_offset = 0; + odi->dir_high_seq_ino = 0; + + rb_link_node(&odi->node, parent, p); + rb_insert_color(&odi->node, &sctx->orphan_dirs); + return odi; +} + +static struct orphan_dir_info *get_orphan_dir_info(struct send_ctx *sctx, + u64 dir_ino, u64 gen) +{ + struct rb_node *n = sctx->orphan_dirs.rb_node; + struct orphan_dir_info *entry; + + while (n) { + entry = rb_entry(n, struct orphan_dir_info, node); + if (dir_ino < entry->ino) + n = n->rb_left; + else if (dir_ino > entry->ino) + n = n->rb_right; + else if (gen < entry->gen) + n = n->rb_left; + else if (gen > entry->gen) + n = n->rb_right; + else + return entry; + } + return NULL; +} + +static int is_waiting_for_rm(struct send_ctx *sctx, u64 dir_ino, u64 gen) +{ + struct orphan_dir_info *odi = get_orphan_dir_info(sctx, dir_ino, gen); + + return odi != NULL; +} + +static void free_orphan_dir_info(struct send_ctx *sctx, + struct orphan_dir_info *odi) +{ + if (!odi) + return; + rb_erase(&odi->node, &sctx->orphan_dirs); + kfree(odi); +} + +/* + * Returns 1 if a directory can be removed at this point in time. + * We check this by iterating all dir items and checking if the inode behind + * the dir item was already processed. + */ +static int can_rmdir(struct send_ctx *sctx, u64 dir, u64 dir_gen) +{ + int ret = 0; + int iter_ret = 0; + struct btrfs_root *root = sctx->parent_root; + struct btrfs_path *path; + struct btrfs_key key; + struct btrfs_key found_key; + struct btrfs_key loc; + struct btrfs_dir_item *di; + struct orphan_dir_info *odi = NULL; + u64 dir_high_seq_ino = 0; + u64 last_dir_index_offset = 0; + + /* + * Don't try to rmdir the top/root subvolume dir. + */ + if (dir == BTRFS_FIRST_FREE_OBJECTID) + return 0; + + odi = get_orphan_dir_info(sctx, dir, dir_gen); + if (odi && sctx->cur_ino < odi->dir_high_seq_ino) + return 0; + + path = alloc_path_for_send(); + if (!path) + return -ENOMEM; + + if (!odi) { + /* + * Find the inode number associated with the last dir index + * entry. This is very likely the inode with the highest number + * of all inodes that have an entry in the directory. We can + * then use it to avoid future calls to can_rmdir(), when + * processing inodes with a lower number, from having to search + * the parent root b+tree for dir index keys. + */ + key.objectid = dir; + key.type = BTRFS_DIR_INDEX_KEY; + key.offset = (u64)-1; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) { + goto out; + } else if (ret > 0) { + /* Can't happen, the root is never empty. */ + ASSERT(path->slots[0] > 0); + if (WARN_ON(path->slots[0] == 0)) { + ret = -EUCLEAN; + goto out; + } + path->slots[0]--; + } + + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + if (key.objectid != dir || key.type != BTRFS_DIR_INDEX_KEY) { + /* No index keys, dir can be removed. */ + ret = 1; + goto out; + } + + di = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_dir_item); + btrfs_dir_item_key_to_cpu(path->nodes[0], di, &loc); + dir_high_seq_ino = loc.objectid; + if (sctx->cur_ino < dir_high_seq_ino) { + ret = 0; + goto out; + } + + btrfs_release_path(path); + } + + key.objectid = dir; + key.type = BTRFS_DIR_INDEX_KEY; + key.offset = (odi ? odi->last_dir_index_offset : 0); + + btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) { + struct waiting_dir_move *dm; + + if (found_key.objectid != key.objectid || + found_key.type != key.type) + break; + + di = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_dir_item); + btrfs_dir_item_key_to_cpu(path->nodes[0], di, &loc); + + dir_high_seq_ino = max(dir_high_seq_ino, loc.objectid); + last_dir_index_offset = found_key.offset; + + dm = get_waiting_dir_move(sctx, loc.objectid); + if (dm) { + dm->rmdir_ino = dir; + dm->rmdir_gen = dir_gen; + ret = 0; + goto out; + } + + if (loc.objectid > sctx->cur_ino) { + ret = 0; + goto out; + } + } + if (iter_ret < 0) { + ret = iter_ret; + goto out; + } + free_orphan_dir_info(sctx, odi); + + ret = 1; + +out: + btrfs_free_path(path); + + if (ret) + return ret; + + if (!odi) { + odi = add_orphan_dir_info(sctx, dir, dir_gen); + if (IS_ERR(odi)) + return PTR_ERR(odi); + + odi->gen = dir_gen; + } + + odi->last_dir_index_offset = last_dir_index_offset; + odi->dir_high_seq_ino = max(odi->dir_high_seq_ino, dir_high_seq_ino); + + return 0; +} + +static int is_waiting_for_move(struct send_ctx *sctx, u64 ino) +{ + struct waiting_dir_move *entry = get_waiting_dir_move(sctx, ino); + + return entry != NULL; +} + +static int add_waiting_dir_move(struct send_ctx *sctx, u64 ino, bool orphanized) +{ + struct rb_node **p = &sctx->waiting_dir_moves.rb_node; + struct rb_node *parent = NULL; + struct waiting_dir_move *entry, *dm; + + dm = kmalloc_obj(*dm); + if (!dm) + return -ENOMEM; + dm->ino = ino; + dm->rmdir_ino = 0; + dm->rmdir_gen = 0; + dm->orphanized = orphanized; + + while (*p) { + parent = *p; + entry = rb_entry(parent, struct waiting_dir_move, node); + if (ino < entry->ino) { + p = &(*p)->rb_left; + } else if (ino > entry->ino) { + p = &(*p)->rb_right; + } else { + kfree(dm); + return -EEXIST; + } + } + + rb_link_node(&dm->node, parent, p); + rb_insert_color(&dm->node, &sctx->waiting_dir_moves); + return 0; +} + +static struct waiting_dir_move * +get_waiting_dir_move(struct send_ctx *sctx, u64 ino) +{ + struct rb_node *n = sctx->waiting_dir_moves.rb_node; + struct waiting_dir_move *entry; + + while (n) { + entry = rb_entry(n, struct waiting_dir_move, node); + if (ino < entry->ino) + n = n->rb_left; + else if (ino > entry->ino) + n = n->rb_right; + else + return entry; + } + return NULL; +} + +static void free_waiting_dir_move(struct send_ctx *sctx, + struct waiting_dir_move *dm) +{ + if (!dm) + return; + rb_erase(&dm->node, &sctx->waiting_dir_moves); + kfree(dm); +} + +static int add_pending_dir_move(struct send_ctx *sctx, + u64 ino, + u64 ino_gen, + u64 parent_ino, + struct list_head *new_refs, + struct list_head *deleted_refs, + const bool is_orphan) +{ + struct rb_node **p = &sctx->pending_dir_moves.rb_node; + struct rb_node *parent = NULL; + struct pending_dir_move *entry = NULL, *pm; + struct recorded_ref *cur; + bool exists = false; + int ret; + + pm = kmalloc_obj(*pm); + if (!pm) + return -ENOMEM; + pm->parent_ino = parent_ino; + pm->ino = ino; + pm->gen = ino_gen; + INIT_LIST_HEAD(&pm->list); + INIT_LIST_HEAD(&pm->update_refs); + RB_CLEAR_NODE(&pm->node); + + while (*p) { + parent = *p; + entry = rb_entry(parent, struct pending_dir_move, node); + if (parent_ino < entry->parent_ino) { + p = &(*p)->rb_left; + } else if (parent_ino > entry->parent_ino) { + p = &(*p)->rb_right; + } else { + exists = true; + break; + } + } + + list_for_each_entry(cur, deleted_refs, list) { + ret = dup_ref(cur, &pm->update_refs); + if (ret < 0) + goto out; + } + list_for_each_entry(cur, new_refs, list) { + ret = dup_ref(cur, &pm->update_refs); + if (ret < 0) + goto out; + } + + ret = add_waiting_dir_move(sctx, pm->ino, is_orphan); + if (ret) + goto out; + + if (exists) { + list_add_tail(&pm->list, &entry->list); + } else { + rb_link_node(&pm->node, parent, p); + rb_insert_color(&pm->node, &sctx->pending_dir_moves); + } + ret = 0; +out: + if (ret) { + __free_recorded_refs(&pm->update_refs); + kfree(pm); + } + return ret; +} + +static struct pending_dir_move *get_pending_dir_moves(struct send_ctx *sctx, + u64 parent_ino) +{ + struct rb_node *n = sctx->pending_dir_moves.rb_node; + struct pending_dir_move *entry; + + while (n) { + entry = rb_entry(n, struct pending_dir_move, node); + if (parent_ino < entry->parent_ino) + n = n->rb_left; + else if (parent_ino > entry->parent_ino) + n = n->rb_right; + else + return entry; + } + return NULL; +} + +static int path_loop(struct send_ctx *sctx, struct fs_path *name, + u64 ino, u64 gen, u64 *ancestor_ino) +{ + int ret = 0; + u64 parent_inode = 0; + u64 parent_gen = 0; + u64 start_ino = ino; + + *ancestor_ino = 0; + while (ino != BTRFS_FIRST_FREE_OBJECTID) { + fs_path_reset(name); + + if (is_waiting_for_rm(sctx, ino, gen)) + break; + if (is_waiting_for_move(sctx, ino)) { + if (*ancestor_ino == 0) + *ancestor_ino = ino; + ret = get_first_ref(sctx->parent_root, ino, + &parent_inode, &parent_gen, name); + } else { + ret = __get_cur_name_and_parent(sctx, ino, gen, + &parent_inode, + &parent_gen, name); + if (ret > 0) { + ret = 0; + break; + } + } + if (ret < 0) + break; + if (parent_inode == start_ino) { + ret = 1; + if (*ancestor_ino == 0) + *ancestor_ino = ino; + break; + } + ino = parent_inode; + gen = parent_gen; + } + return ret; +} + +static int apply_dir_move(struct send_ctx *sctx, struct pending_dir_move *pm) +{ + struct fs_path *from_path __free(fs_path_free) = NULL; + struct fs_path *to_path __free(fs_path_free) = NULL; + struct fs_path *name __free(fs_path_free) = NULL; + u64 orig_progress = sctx->send_progress; + struct recorded_ref *cur; + u64 parent_ino, parent_gen; + struct waiting_dir_move *dm = NULL; + u64 rmdir_ino = 0; + u64 rmdir_gen; + u64 ancestor; + bool is_orphan; + int ret; + + name = fs_path_alloc(); + from_path = fs_path_alloc(); + if (!name || !from_path) + return -ENOMEM; + + dm = get_waiting_dir_move(sctx, pm->ino); + ASSERT(dm); + rmdir_ino = dm->rmdir_ino; + rmdir_gen = dm->rmdir_gen; + is_orphan = dm->orphanized; + free_waiting_dir_move(sctx, dm); + + if (is_orphan) { + ret = gen_unique_name(sctx, pm->ino, + pm->gen, from_path); + } else { + ret = get_first_ref(sctx->parent_root, pm->ino, + &parent_ino, &parent_gen, name); + if (ret < 0) + goto out; + ret = get_cur_path(sctx, parent_ino, parent_gen, + from_path); + if (ret < 0) + goto out; + ret = fs_path_add_path(from_path, name); + } + if (ret < 0) + goto out; + + sctx->send_progress = sctx->cur_ino + 1; + ret = path_loop(sctx, name, pm->ino, pm->gen, &ancestor); + if (ret < 0) + goto out; + if (ret) { + LIST_HEAD(deleted_refs); + ASSERT(ancestor > BTRFS_FIRST_FREE_OBJECTID); + ret = add_pending_dir_move(sctx, pm->ino, pm->gen, ancestor, + &pm->update_refs, &deleted_refs, + is_orphan); + if (ret < 0) + goto out; + if (rmdir_ino) { + dm = get_waiting_dir_move(sctx, pm->ino); + ASSERT(dm); + dm->rmdir_ino = rmdir_ino; + dm->rmdir_gen = rmdir_gen; + } + goto out; + } + fs_path_reset(name); + to_path = name; + name = NULL; + ret = get_cur_path(sctx, pm->ino, pm->gen, to_path); + if (ret < 0) + goto out; + + ret = send_rename(sctx, from_path, to_path); + if (ret < 0) + goto out; + + if (rmdir_ino) { + struct orphan_dir_info *odi; + u64 gen; + + odi = get_orphan_dir_info(sctx, rmdir_ino, rmdir_gen); + if (!odi) { + /* already deleted */ + goto finish; + } + gen = odi->gen; + + ret = can_rmdir(sctx, rmdir_ino, gen); + if (ret < 0) + goto out; + if (!ret) + goto finish; + + name = fs_path_alloc(); + if (!name) { + ret = -ENOMEM; + goto out; + } + ret = get_cur_path(sctx, rmdir_ino, gen, name); + if (ret < 0) + goto out; + ret = send_rmdir(sctx, name); + if (ret < 0) + goto out; + } + +finish: + ret = cache_dir_utimes(sctx, pm->ino, pm->gen); + if (ret < 0) + goto out; + + /* + * After rename/move, need to update the utimes of both new parent(s) + * and old parent(s). + */ + list_for_each_entry(cur, &pm->update_refs, list) { + /* + * The parent inode might have been deleted in the send snapshot + */ + ret = get_inode_info(sctx->send_root, cur->dir, NULL); + if (ret == -ENOENT) { + ret = 0; + continue; + } + if (ret < 0) + goto out; + + ret = cache_dir_utimes(sctx, cur->dir, cur->dir_gen); + if (ret < 0) + goto out; + } + +out: + sctx->send_progress = orig_progress; + + return ret; +} + +static void free_pending_move(struct send_ctx *sctx, struct pending_dir_move *m) +{ + if (!list_empty(&m->list)) + list_del(&m->list); + if (!RB_EMPTY_NODE(&m->node)) + rb_erase(&m->node, &sctx->pending_dir_moves); + __free_recorded_refs(&m->update_refs); + kfree(m); +} + +static void tail_append_pending_moves(struct send_ctx *sctx, + struct pending_dir_move *moves, + struct list_head *stack) +{ + if (list_empty(&moves->list)) { + list_add_tail(&moves->list, stack); + } else { + LIST_HEAD(list); + list_splice_init(&moves->list, &list); + list_add_tail(&moves->list, stack); + list_splice_tail(&list, stack); + } + if (!RB_EMPTY_NODE(&moves->node)) { + rb_erase(&moves->node, &sctx->pending_dir_moves); + RB_CLEAR_NODE(&moves->node); + } +} + +static int apply_children_dir_moves(struct send_ctx *sctx) +{ + struct pending_dir_move *pm; + LIST_HEAD(stack); + u64 parent_ino = sctx->cur_ino; + int ret = 0; + + pm = get_pending_dir_moves(sctx, parent_ino); + if (!pm) + return 0; + + tail_append_pending_moves(sctx, pm, &stack); + + while (!list_empty(&stack)) { + pm = list_first_entry(&stack, struct pending_dir_move, list); + parent_ino = pm->ino; + ret = apply_dir_move(sctx, pm); + free_pending_move(sctx, pm); + if (ret) + goto out; + pm = get_pending_dir_moves(sctx, parent_ino); + if (pm) + tail_append_pending_moves(sctx, pm, &stack); + } + return 0; + +out: + while (!list_empty(&stack)) { + pm = list_first_entry(&stack, struct pending_dir_move, list); + free_pending_move(sctx, pm); + } + return ret; +} + +/* + * We might need to delay a directory rename even when no ancestor directory + * (in the send root) with a higher inode number than ours (sctx->cur_ino) was + * renamed. This happens when we rename a directory to the old name (the name + * in the parent root) of some other unrelated directory that got its rename + * delayed due to some ancestor with higher number that got renamed. + * + * Example: + * + * Parent snapshot: + * . (ino 256) + * |---- a/ (ino 257) + * | |---- file (ino 260) + * | + * |---- b/ (ino 258) + * |---- c/ (ino 259) + * + * Send snapshot: + * . (ino 256) + * |---- a/ (ino 258) + * |---- x/ (ino 259) + * |---- y/ (ino 257) + * |----- file (ino 260) + * + * Here we can not rename 258 from 'b' to 'a' without the rename of inode 257 + * from 'a' to 'x/y' happening first, which in turn depends on the rename of + * inode 259 from 'c' to 'x'. So the order of rename commands the send stream + * must issue is: + * + * 1 - rename 259 from 'c' to 'x' + * 2 - rename 257 from 'a' to 'x/y' + * 3 - rename 258 from 'b' to 'a' + * + * Returns 1 if the rename of sctx->cur_ino needs to be delayed, 0 if it can + * be done right away and < 0 on error. + */ +static int wait_for_dest_dir_move(struct send_ctx *sctx, + struct recorded_ref *parent_ref, + const bool is_orphan) +{ + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + struct btrfs_key di_key; + struct btrfs_dir_item *di; + u64 left_gen; + u64 right_gen; + int ret = 0; + struct waiting_dir_move *wdm; + + if (RB_EMPTY_ROOT(&sctx->waiting_dir_moves)) + return 0; + + path = alloc_path_for_send(); + if (!path) + return -ENOMEM; + + key.objectid = parent_ref->dir; + key.type = BTRFS_DIR_ITEM_KEY; + key.offset = btrfs_name_hash(parent_ref->name, parent_ref->name_len); + + ret = btrfs_search_slot(NULL, sctx->parent_root, &key, path, 0, 0); + if (ret < 0) + return ret; + if (ret > 0) + return 0; + + di = btrfs_match_dir_item_name(path, parent_ref->name, + parent_ref->name_len); + if (!di) + return 0; + /* + * di_key.objectid has the number of the inode that has a dentry in the + * parent directory with the same name that sctx->cur_ino is being + * renamed to. We need to check if that inode is in the send root as + * well and if it is currently marked as an inode with a pending rename, + * if it is, we need to delay the rename of sctx->cur_ino as well, so + * that it happens after that other inode is renamed. + */ + btrfs_dir_item_key_to_cpu(path->nodes[0], di, &di_key); + if (di_key.type != BTRFS_INODE_ITEM_KEY) + return 0; + + ret = get_inode_gen(sctx->parent_root, di_key.objectid, &left_gen); + if (ret < 0) + return ret; + ret = get_inode_gen(sctx->send_root, di_key.objectid, &right_gen); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + return ret; + } + + /* Different inode, no need to delay the rename of sctx->cur_ino */ + if (right_gen != left_gen) + return 0; + + wdm = get_waiting_dir_move(sctx, di_key.objectid); + if (wdm && !wdm->orphanized) { + ret = add_pending_dir_move(sctx, + sctx->cur_ino, + sctx->cur_inode_gen, + di_key.objectid, + &sctx->new_refs, + &sctx->deleted_refs, + is_orphan); + if (!ret) + ret = 1; + } + return ret; +} + +/* + * Check if inode ino2, or any of its ancestors, is inode ino1. + * Return 1 if true, 0 if false and < 0 on error. + */ +static int check_ino_in_path(struct btrfs_root *root, + const u64 ino1, + const u64 ino1_gen, + const u64 ino2, + const u64 ino2_gen, + struct fs_path *fs_path) +{ + u64 ino = ino2; + + if (ino1 == ino2) + return ino1_gen == ino2_gen; + + while (ino > BTRFS_FIRST_FREE_OBJECTID) { + u64 parent; + u64 parent_gen; + int ret; + + fs_path_reset(fs_path); + ret = get_first_ref(root, ino, &parent, &parent_gen, fs_path); + if (ret < 0) + return ret; + if (parent == ino1) + return parent_gen == ino1_gen; + ino = parent; + } + return 0; +} + +/* + * Check if inode ino1 is an ancestor of inode ino2 in the given root for any + * possible path (in case ino2 is not a directory and has multiple hard links). + * Return 1 if true, 0 if false and < 0 on error. + */ +static int is_ancestor(struct btrfs_root *root, + const u64 ino1, + const u64 ino1_gen, + const u64 ino2, + struct fs_path *fs_path) +{ + bool free_fs_path = false; + int ret = 0; + int iter_ret = 0; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + + if (!fs_path) { + fs_path = fs_path_alloc(); + if (!fs_path) + return -ENOMEM; + free_fs_path = true; + } + + path = alloc_path_for_send(); + if (!path) { + ret = -ENOMEM; + goto out; + } + + key.objectid = ino2; + key.type = BTRFS_INODE_REF_KEY; + key.offset = 0; + + btrfs_for_each_slot(root, &key, &key, path, iter_ret) { + struct extent_buffer *leaf = path->nodes[0]; + int slot = path->slots[0]; + u32 cur_offset = 0; + u32 item_size; + + if (key.objectid != ino2) + break; + if (key.type != BTRFS_INODE_REF_KEY && + key.type != BTRFS_INODE_EXTREF_KEY) + break; + + item_size = btrfs_item_size(leaf, slot); + while (cur_offset < item_size) { + u64 parent; + u64 parent_gen; + + if (key.type == BTRFS_INODE_EXTREF_KEY) { + unsigned long ptr; + struct btrfs_inode_extref *extref; + + ptr = btrfs_item_ptr_offset(leaf, slot); + extref = (struct btrfs_inode_extref *) + (ptr + cur_offset); + parent = btrfs_inode_extref_parent(leaf, + extref); + cur_offset += sizeof(*extref); + cur_offset += btrfs_inode_extref_name_len(leaf, + extref); + } else { + parent = key.offset; + cur_offset = item_size; + } + + ret = get_inode_gen(root, parent, &parent_gen); + if (ret < 0) + goto out; + ret = check_ino_in_path(root, ino1, ino1_gen, + parent, parent_gen, fs_path); + if (ret) + goto out; + } + } + ret = 0; + if (iter_ret < 0) + ret = iter_ret; + +out: + if (free_fs_path) + fs_path_free(fs_path); + return ret; +} + +static int wait_for_parent_move(struct send_ctx *sctx, + struct recorded_ref *parent_ref, + const bool is_orphan) +{ + int ret = 0; + u64 ino = parent_ref->dir; + u64 ino_gen = parent_ref->dir_gen; + u64 parent_ino_before, parent_ino_after; + struct fs_path *path_before = NULL; + struct fs_path *path_after = NULL; + int len1, len2; + + path_after = fs_path_alloc(); + path_before = fs_path_alloc(); + if (!path_after || !path_before) { + ret = -ENOMEM; + goto out; + } + + /* + * Our current directory inode may not yet be renamed/moved because some + * ancestor (immediate or not) has to be renamed/moved first. So find if + * such ancestor exists and make sure our own rename/move happens after + * that ancestor is processed to avoid path build infinite loops (done + * at get_cur_path()). + */ + while (ino > BTRFS_FIRST_FREE_OBJECTID) { + u64 parent_ino_after_gen; + + if (is_waiting_for_move(sctx, ino)) { + /* + * If the current inode is an ancestor of ino in the + * parent root, we need to delay the rename of the + * current inode, otherwise don't delayed the rename + * because we can end up with a circular dependency + * of renames, resulting in some directories never + * getting the respective rename operations issued in + * the send stream or getting into infinite path build + * loops. + */ + ret = is_ancestor(sctx->parent_root, + sctx->cur_ino, sctx->cur_inode_gen, + ino, path_before); + if (ret) + break; + } + + fs_path_reset(path_before); + fs_path_reset(path_after); + + ret = get_first_ref(sctx->send_root, ino, &parent_ino_after, + &parent_ino_after_gen, path_after); + if (ret < 0) + goto out; + ret = get_first_ref(sctx->parent_root, ino, &parent_ino_before, + NULL, path_before); + if (ret < 0 && ret != -ENOENT) { + goto out; + } else if (ret == -ENOENT) { + ret = 0; + break; + } + + len1 = fs_path_len(path_before); + len2 = fs_path_len(path_after); + if (ino > sctx->cur_ino && + (parent_ino_before != parent_ino_after || len1 != len2 || + memcmp(path_before->start, path_after->start, len1))) { + u64 parent_ino_gen; + + ret = get_inode_gen(sctx->parent_root, ino, &parent_ino_gen); + if (ret < 0) + goto out; + if (ino_gen == parent_ino_gen) { + ret = 1; + break; + } + } + ino = parent_ino_after; + ino_gen = parent_ino_after_gen; + } + +out: + fs_path_free(path_before); + fs_path_free(path_after); + + if (ret == 1) { + ret = add_pending_dir_move(sctx, + sctx->cur_ino, + sctx->cur_inode_gen, + ino, + &sctx->new_refs, + &sctx->deleted_refs, + is_orphan); + if (!ret) + ret = 1; + } + + return ret; +} + +static int update_ref_path(struct send_ctx *sctx, struct recorded_ref *ref) +{ + int ret; + struct fs_path *new_path; + + /* + * Our reference's name member points to its full_path member string, so + * we use here a new path. + */ + new_path = fs_path_alloc(); + if (!new_path) + return -ENOMEM; + + ret = get_cur_path(sctx, ref->dir, ref->dir_gen, new_path); + if (ret < 0) { + fs_path_free(new_path); + return ret; + } + ret = fs_path_add(new_path, ref->name, ref->name_len); + if (ret < 0) { + fs_path_free(new_path); + return ret; + } + + fs_path_free(ref->full_path); + set_ref_path(ref, new_path); + + return 0; +} + +/* + * When processing the new references for an inode we may orphanize an existing + * directory inode because its old name conflicts with one of the new references + * of the current inode. Later, when processing another new reference of our + * inode, we might need to orphanize another inode, but the path we have in the + * reference reflects the pre-orphanization name of the directory we previously + * orphanized. For example: + * + * parent snapshot looks like: + * + * . (ino 256) + * |----- f1 (ino 257) + * |----- f2 (ino 258) + * |----- d1/ (ino 259) + * |----- d2/ (ino 260) + * + * send snapshot looks like: + * + * . (ino 256) + * |----- d1 (ino 258) + * |----- f2/ (ino 259) + * |----- f2_link/ (ino 260) + * | |----- f1 (ino 257) + * | + * |----- d2 (ino 258) + * + * When processing inode 257 we compute the name for inode 259 as "d1", and we + * cache it in the name cache. Later when we start processing inode 258, when + * collecting all its new references we set a full path of "d1/d2" for its new + * reference with name "d2". When we start processing the new references we + * start by processing the new reference with name "d1", and this results in + * orphanizing inode 259, since its old reference causes a conflict. Then we + * move on the next new reference, with name "d2", and we find out we must + * orphanize inode 260, as its old reference conflicts with ours - but for the + * orphanization we use a source path corresponding to the path we stored in the + * new reference, which is "d1/d2" and not "o259-6-0/d2" - this makes the + * receiver fail since the path component "d1/" no longer exists, it was renamed + * to "o259-6-0/" when processing the previous new reference. So in this case we + * must recompute the path in the new reference and use it for the new + * orphanization operation. + */ +static int refresh_ref_path(struct send_ctx *sctx, struct recorded_ref *ref) +{ + char AUTO_KFREE(name); + int ret; + + name = kmemdup(ref->name, ref->name_len, GFP_KERNEL); + if (!name) + return -ENOMEM; + + fs_path_reset(ref->full_path); + ret = get_cur_path(sctx, ref->dir, ref->dir_gen, ref->full_path); + if (ret < 0) + return ret; + + ret = fs_path_add(ref->full_path, name, ref->name_len); + if (ret < 0) + return ret; + + /* Update the reference's base name pointer. */ + set_ref_path(ref, ref->full_path); + + return 0; +} + +static int rbtree_check_dir_ref_comp(const void *k, const struct rb_node *node) +{ + const struct recorded_ref *data = k; + const struct recorded_ref *ref = rb_entry(node, struct recorded_ref, node); + + if (data->dir > ref->dir) + return 1; + if (data->dir < ref->dir) + return -1; + if (data->dir_gen > ref->dir_gen) + return 1; + if (data->dir_gen < ref->dir_gen) + return -1; + return 0; +} + +static bool rbtree_check_dir_ref_less(struct rb_node *node, const struct rb_node *parent) +{ + const struct recorded_ref *entry = rb_entry(node, struct recorded_ref, node); + + return rbtree_check_dir_ref_comp(entry, parent) < 0; +} + +static int record_check_dir_ref_in_tree(struct rb_root *root, + struct recorded_ref *ref, struct list_head *list) +{ + struct recorded_ref *tmp_ref; + int ret; + + if (rb_find(ref, root, rbtree_check_dir_ref_comp)) + return 0; + + ret = dup_ref(ref, list); + if (ret < 0) + return ret; + + tmp_ref = list_last_entry(list, struct recorded_ref, list); + rb_add(&tmp_ref->node, root, rbtree_check_dir_ref_less); + tmp_ref->root = root; + return 0; +} + +static int rename_current_inode(struct send_ctx *sctx, + struct fs_path *current_path, + struct fs_path *new_path) +{ + int ret; + + ret = send_rename(sctx, current_path, new_path); + if (ret < 0) + return ret; + + ret = fs_path_copy(&sctx->cur_inode_path, new_path); + if (ret < 0) + return ret; + + return fs_path_copy(current_path, new_path); +} + +/* + * This does all the move/link/unlink/rmdir magic. + */ +static int process_recorded_refs(struct send_ctx *sctx, bool *pending_move) +{ + struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; + int ret = 0; + struct recorded_ref *cur; + struct recorded_ref *cur2; + LIST_HEAD(check_dirs); + struct rb_root rbtree_check_dirs = RB_ROOT; + struct fs_path *valid_path = NULL; + u64 ow_inode = 0; + u64 ow_gen; + u64 ow_mode; + bool did_overwrite = false; + bool is_orphan = false; + bool can_rename = true; + bool orphanized_dir = false; + bool orphanized_ancestor = false; + + /* + * This should never happen as the root dir always has the same ref + * which is always '..' + */ + if (unlikely(sctx->cur_ino <= BTRFS_FIRST_FREE_OBJECTID)) { + btrfs_err(fs_info, + "send: unexpected inode %llu in process_recorded_refs()", + sctx->cur_ino); + ret = -EINVAL; + goto out; + } + + valid_path = fs_path_alloc(); + if (!valid_path) { + ret = -ENOMEM; + goto out; + } + + /* + * First, check if the first ref of the current inode was overwritten + * before. If yes, we know that the current inode was already orphanized + * and thus use the orphan name. If not, we can use get_cur_path to + * get the path of the first ref as it would like while receiving at + * this point in time. + * New inodes are always orphan at the beginning, so force to use the + * orphan name in this case. + * The first ref is stored in valid_path and will be updated if it + * gets moved around. + */ + if (!sctx->cur_inode_new) { + ret = did_overwrite_first_ref(sctx, sctx->cur_ino, + sctx->cur_inode_gen); + if (ret < 0) + goto out; + if (ret) + did_overwrite = true; + } + if (sctx->cur_inode_new || did_overwrite) { + ret = gen_unique_name(sctx, sctx->cur_ino, + sctx->cur_inode_gen, valid_path); + if (ret < 0) + goto out; + is_orphan = true; + } else { + ret = get_cur_path(sctx, sctx->cur_ino, sctx->cur_inode_gen, + valid_path); + if (ret < 0) + goto out; + } + + /* + * Before doing any rename and link operations, do a first pass on the + * new references to orphanize any unprocessed inodes that may have a + * reference that conflicts with one of the new references of the current + * inode. This needs to happen first because a new reference may conflict + * with the old reference of a parent directory, so we must make sure + * that the path used for link and rename commands don't use an + * orphanized name when an ancestor was not yet orphanized. + * + * Example: + * + * Parent snapshot: + * + * . (ino 256) + * |----- testdir/ (ino 259) + * | |----- a (ino 257) + * | + * |----- b (ino 258) + * + * Send snapshot: + * + * . (ino 256) + * |----- testdir_2/ (ino 259) + * | |----- a (ino 260) + * | + * |----- testdir (ino 257) + * |----- b (ino 257) + * |----- b2 (ino 258) + * + * Processing the new reference for inode 257 with name "b" may happen + * before processing the new reference with name "testdir". If so, we + * must make sure that by the time we send a link command to create the + * hard link "b", inode 259 was already orphanized, since the generated + * path in "valid_path" already contains the orphanized name for 259. + * We are processing inode 257, so only later when processing 259 we do + * the rename operation to change its temporary (orphanized) name to + * "testdir_2". + */ + list_for_each_entry(cur, &sctx->new_refs, list) { + ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL); + if (ret < 0) + goto out; + if (ret == inode_state_will_create) + continue; + + /* + * Check if this new ref would overwrite the first ref of another + * unprocessed inode. If yes, orphanize the overwritten inode. + * If we find an overwritten ref that is not the first ref, + * simply unlink it. + */ + ret = will_overwrite_ref(sctx, cur->dir, cur->dir_gen, + cur->name, cur->name_len, + &ow_inode, &ow_gen, &ow_mode); + if (ret < 0) + goto out; + if (ret) { + ret = is_first_ref(sctx->parent_root, + ow_inode, cur->dir, cur->name, + cur->name_len); + if (ret < 0) + goto out; + if (ret) { + struct name_cache_entry *nce; + struct waiting_dir_move *wdm; + + if (orphanized_dir) { + ret = refresh_ref_path(sctx, cur); + if (ret < 0) + goto out; + } + + ret = orphanize_inode(sctx, ow_inode, ow_gen, + cur->full_path); + if (ret < 0) + goto out; + if (S_ISDIR(ow_mode)) + orphanized_dir = true; + + /* + * If ow_inode has its rename operation delayed + * make sure that its orphanized name is used in + * the source path when performing its rename + * operation. + */ + wdm = get_waiting_dir_move(sctx, ow_inode); + if (wdm) + wdm->orphanized = true; + + /* + * Make sure we clear our orphanized inode's + * name from the name cache. This is because the + * inode ow_inode might be an ancestor of some + * other inode that will be orphanized as well + * later and has an inode number greater than + * sctx->send_progress. We need to prevent + * future name lookups from using the old name + * and get instead the orphan name. + */ + nce = name_cache_search(sctx, ow_inode, ow_gen); + if (nce) + btrfs_lru_cache_remove(&sctx->name_cache, + &nce->entry); + + /* + * ow_inode might currently be an ancestor of + * cur_ino, therefore compute valid_path (the + * current path of cur_ino) again because it + * might contain the pre-orphanization name of + * ow_inode, which is no longer valid. + */ + ret = is_ancestor(sctx->parent_root, + ow_inode, ow_gen, + sctx->cur_ino, NULL); + if (ret > 0) { + orphanized_ancestor = true; + fs_path_reset(valid_path); + fs_path_reset(&sctx->cur_inode_path); + ret = get_cur_path(sctx, sctx->cur_ino, + sctx->cur_inode_gen, + valid_path); + } + if (ret < 0) + goto out; + } else { + /* + * If we previously orphanized a directory that + * collided with a new reference that we already + * processed, recompute the current path because + * that directory may be part of the path. + */ + if (orphanized_dir) { + ret = refresh_ref_path(sctx, cur); + if (ret < 0) + goto out; + } + ret = send_unlink(sctx, cur->full_path); + if (ret < 0) + goto out; + } + } + + } + + list_for_each_entry(cur, &sctx->new_refs, list) { + /* + * We may have refs where the parent directory does not exist + * yet. This happens if the parent directories inum is higher + * than the current inum. To handle this case, we create the + * parent directory out of order. But we need to check if this + * did already happen before due to other refs in the same dir. + */ + ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL); + if (ret < 0) + goto out; + if (ret == inode_state_will_create) { + ret = 0; + /* + * First check if any of the current inodes refs did + * already create the dir. + */ + list_for_each_entry(cur2, &sctx->new_refs, list) { + if (cur == cur2) + break; + if (cur2->dir == cur->dir) { + ret = 1; + break; + } + } + + /* + * If that did not happen, check if a previous inode + * did already create the dir. + */ + if (!ret) + ret = did_create_dir(sctx, cur->dir); + if (ret < 0) + goto out; + if (!ret) { + ret = send_create_inode(sctx, cur->dir); + if (ret < 0) + goto out; + cache_dir_created(sctx, cur->dir); + } + } + + if (S_ISDIR(sctx->cur_inode_mode) && sctx->parent_root) { + ret = wait_for_dest_dir_move(sctx, cur, is_orphan); + if (ret < 0) + goto out; + if (ret == 1) { + can_rename = false; + *pending_move = true; + } + } + + if (S_ISDIR(sctx->cur_inode_mode) && sctx->parent_root && + can_rename) { + ret = wait_for_parent_move(sctx, cur, is_orphan); + if (ret < 0) + goto out; + if (ret == 1) { + can_rename = false; + *pending_move = true; + } + } + + /* + * link/move the ref to the new place. If we have an orphan + * inode, move it and update valid_path. If not, link or move + * it depending on the inode mode. + */ + if (is_orphan && can_rename) { + ret = rename_current_inode(sctx, valid_path, cur->full_path); + if (ret < 0) + goto out; + is_orphan = false; + } else if (can_rename) { + if (S_ISDIR(sctx->cur_inode_mode)) { + /* + * Dirs can't be linked, so move it. For moved + * dirs, we always have one new and one deleted + * ref. The deleted ref is ignored later. + */ + ret = rename_current_inode(sctx, valid_path, + cur->full_path); + if (ret < 0) + goto out; + } else { + /* + * We might have previously orphanized an inode + * which is an ancestor of our current inode, + * so our reference's full path, which was + * computed before any such orphanizations, must + * be updated. + */ + if (orphanized_dir) { + ret = update_ref_path(sctx, cur); + if (ret < 0) + goto out; + } + ret = send_link(sctx, cur->full_path, + valid_path); + if (ret < 0) + goto out; + } + } + ret = record_check_dir_ref_in_tree(&rbtree_check_dirs, cur, &check_dirs); + if (ret < 0) + goto out; + } + + if (S_ISDIR(sctx->cur_inode_mode) && sctx->cur_inode_deleted) { + /* + * Check if we can already rmdir the directory. If not, + * orphanize it. For every dir item inside that gets deleted + * later, we do this check again and rmdir it then if possible. + * See the use of check_dirs for more details. + */ + ret = can_rmdir(sctx, sctx->cur_ino, sctx->cur_inode_gen); + if (ret < 0) + goto out; + if (ret) { + ret = send_rmdir(sctx, valid_path); + if (ret < 0) + goto out; + } else if (!is_orphan) { + ret = orphanize_inode(sctx, sctx->cur_ino, + sctx->cur_inode_gen, valid_path); + if (ret < 0) + goto out; + is_orphan = true; + } + + list_for_each_entry(cur, &sctx->deleted_refs, list) { + ret = record_check_dir_ref_in_tree(&rbtree_check_dirs, cur, &check_dirs); + if (ret < 0) + goto out; + } + } else if (S_ISDIR(sctx->cur_inode_mode) && + !list_empty(&sctx->deleted_refs)) { + /* + * We have a moved dir. Add the old parent to check_dirs + */ + cur = list_first_entry(&sctx->deleted_refs, struct recorded_ref, list); + ret = record_check_dir_ref_in_tree(&rbtree_check_dirs, cur, &check_dirs); + if (ret < 0) + goto out; + } else if (!S_ISDIR(sctx->cur_inode_mode)) { + /* + * We have a non dir inode. Go through all deleted refs and + * unlink them if they were not already overwritten by other + * inodes. + */ + list_for_each_entry(cur, &sctx->deleted_refs, list) { + ret = did_overwrite_ref(sctx, cur->dir, cur->dir_gen, + sctx->cur_ino, sctx->cur_inode_gen, + cur->name, cur->name_len); + if (ret < 0) + goto out; + if (!ret) { + /* + * If we orphanized any ancestor before, we need + * to recompute the full path for deleted names, + * since any such path was computed before we + * processed any references and orphanized any + * ancestor inode. + */ + if (orphanized_ancestor) { + ret = update_ref_path(sctx, cur); + if (ret < 0) + goto out; + } + ret = send_unlink(sctx, cur->full_path); + if (ret < 0) + goto out; + if (is_current_inode_path(sctx, cur->full_path)) + fs_path_reset(&sctx->cur_inode_path); + } + ret = record_check_dir_ref_in_tree(&rbtree_check_dirs, cur, &check_dirs); + if (ret < 0) + goto out; + } + /* + * If the inode is still orphan, unlink the orphan. This may + * happen when a previous inode did overwrite the first ref + * of this inode and no new refs were added for the current + * inode. Unlinking does not mean that the inode is deleted in + * all cases. There may still be links to this inode in other + * places. + */ + if (is_orphan) { + ret = send_unlink(sctx, valid_path); + if (ret < 0) + goto out; + } + } + + /* + * We did collect all parent dirs where cur_inode was once located. We + * now go through all these dirs and check if they are pending for + * deletion and if it's finally possible to perform the rmdir now. + * We also update the inode stats of the parent dirs here. + */ + list_for_each_entry(cur, &check_dirs, list) { + /* + * In case we had refs into dirs that were not processed yet, + * we don't need to do the utime and rmdir logic for these dirs. + * The dir will be processed later. + */ + if (cur->dir > sctx->cur_ino) + continue; + + ret = get_cur_inode_state(sctx, cur->dir, cur->dir_gen, NULL, NULL); + if (ret < 0) + goto out; + + if (ret == inode_state_did_create || + ret == inode_state_no_change) { + ret = cache_dir_utimes(sctx, cur->dir, cur->dir_gen); + if (ret < 0) + goto out; + } else if (ret == inode_state_did_delete) { + ret = can_rmdir(sctx, cur->dir, cur->dir_gen); + if (ret < 0) + goto out; + if (ret) { + ret = get_cur_path(sctx, cur->dir, + cur->dir_gen, valid_path); + if (ret < 0) + goto out; + ret = send_rmdir(sctx, valid_path); + if (ret < 0) + goto out; + } + } + } + + ret = 0; + +out: + __free_recorded_refs(&check_dirs); + free_recorded_refs(sctx); + fs_path_free(valid_path); + return ret; +} + +static int rbtree_ref_comp(const void *k, const struct rb_node *node) +{ + const struct recorded_ref *data = k; + const struct recorded_ref *ref = rb_entry(node, struct recorded_ref, node); + + if (data->dir > ref->dir) + return 1; + if (data->dir < ref->dir) + return -1; + if (data->dir_gen > ref->dir_gen) + return 1; + if (data->dir_gen < ref->dir_gen) + return -1; + if (data->name_len > ref->name_len) + return 1; + if (data->name_len < ref->name_len) + return -1; + return strcmp(data->name, ref->name); +} + +static bool rbtree_ref_less(struct rb_node *node, const struct rb_node *parent) +{ + const struct recorded_ref *entry = rb_entry(node, struct recorded_ref, node); + + return rbtree_ref_comp(entry, parent) < 0; +} + +static int record_ref_in_tree(struct rb_root *root, struct list_head *refs, + struct fs_path *name, u64 dir, u64 dir_gen, + struct send_ctx *sctx) +{ + int ret = 0; + struct fs_path *path = NULL; + struct recorded_ref *ref = NULL; + + path = fs_path_alloc(); + if (!path) { + ret = -ENOMEM; + goto out; + } + + ref = recorded_ref_alloc(); + if (!ref) { + ret = -ENOMEM; + goto out; + } + + ret = get_cur_path(sctx, dir, dir_gen, path); + if (ret < 0) + goto out; + ret = fs_path_add_path(path, name); + if (ret < 0) + goto out; + + ref->dir = dir; + ref->dir_gen = dir_gen; + set_ref_path(ref, path); + list_add_tail(&ref->list, refs); + rb_add(&ref->node, root, rbtree_ref_less); + ref->root = root; +out: + if (ret) { + if (path && (!ref || !ref->full_path)) + fs_path_free(path); + recorded_ref_free(ref); + } + return ret; +} + +static int record_new_ref_if_needed(u64 dir, struct fs_path *name, void *ctx) +{ + int ret; + struct send_ctx *sctx = ctx; + struct rb_node *node = NULL; + struct recorded_ref data; + struct recorded_ref *ref; + u64 dir_gen; + + ret = get_inode_gen(sctx->send_root, dir, &dir_gen); + if (ret < 0) + return ret; + + data.dir = dir; + data.dir_gen = dir_gen; + set_ref_path(&data, name); + node = rb_find(&data, &sctx->rbtree_deleted_refs, rbtree_ref_comp); + if (node) { + ref = rb_entry(node, struct recorded_ref, node); + recorded_ref_free(ref); + } else { + ret = record_ref_in_tree(&sctx->rbtree_new_refs, + &sctx->new_refs, name, dir, dir_gen, + sctx); + } + + return ret; +} + +static int record_deleted_ref_if_needed(u64 dir, struct fs_path *name, void *ctx) +{ + int ret; + struct send_ctx *sctx = ctx; + struct rb_node *node = NULL; + struct recorded_ref data; + struct recorded_ref *ref; + u64 dir_gen; + + ret = get_inode_gen(sctx->parent_root, dir, &dir_gen); + if (ret < 0) + return ret; + + data.dir = dir; + data.dir_gen = dir_gen; + set_ref_path(&data, name); + node = rb_find(&data, &sctx->rbtree_new_refs, rbtree_ref_comp); + if (node) { + ref = rb_entry(node, struct recorded_ref, node); + recorded_ref_free(ref); + } else { + ret = record_ref_in_tree(&sctx->rbtree_deleted_refs, + &sctx->deleted_refs, name, dir, + dir_gen, sctx); + } + + return ret; +} + +static int record_new_ref(struct send_ctx *sctx) +{ + int ret; + + ret = iterate_inode_ref(sctx->send_root, sctx->left_path, sctx->cmp_key, + false, record_new_ref_if_needed, sctx); + if (ret < 0) + return ret; + + return 0; +} + +static int record_deleted_ref(struct send_ctx *sctx) +{ + int ret; + + ret = iterate_inode_ref(sctx->parent_root, sctx->right_path, sctx->cmp_key, + false, record_deleted_ref_if_needed, sctx); + if (ret < 0) + return ret; + + return 0; +} + +static int record_changed_ref(struct send_ctx *sctx) +{ + int ret; + + ret = iterate_inode_ref(sctx->send_root, sctx->left_path, sctx->cmp_key, + false, record_new_ref_if_needed, sctx); + if (ret < 0) + return ret; + ret = iterate_inode_ref(sctx->parent_root, sctx->right_path, sctx->cmp_key, + false, record_deleted_ref_if_needed, sctx); + if (ret < 0) + return ret; + + return 0; +} + +/* + * Record and process all refs at once. Needed when an inode changes the + * generation number, which means that it was deleted and recreated. + */ +static int process_all_refs(struct send_ctx *sctx, + enum btrfs_compare_tree_result cmd) +{ + int ret = 0; + int iter_ret = 0; + struct btrfs_root *root; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + struct btrfs_key found_key; + iterate_inode_ref_t cb; + bool pending_move = false; + + path = alloc_path_for_send(); + if (!path) + return -ENOMEM; + + if (cmd == BTRFS_COMPARE_TREE_NEW) { + root = sctx->send_root; + cb = record_new_ref_if_needed; + } else if (cmd == BTRFS_COMPARE_TREE_DELETED) { + root = sctx->parent_root; + cb = record_deleted_ref_if_needed; + } else { + btrfs_err(sctx->send_root->fs_info, + "Wrong command %d in process_all_refs", cmd); + return -EINVAL; + } + + key.objectid = sctx->cmp_key->objectid; + key.type = BTRFS_INODE_REF_KEY; + key.offset = 0; + btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) { + if (found_key.objectid != key.objectid || + (found_key.type != BTRFS_INODE_REF_KEY && + found_key.type != BTRFS_INODE_EXTREF_KEY)) + break; + + ret = iterate_inode_ref(root, path, &found_key, false, cb, sctx); + if (ret < 0) + return ret; + } + /* Catch error found during iteration */ + if (iter_ret < 0) + return iter_ret; + + btrfs_release_path(path); + + /* + * We don't actually care about pending_move as we are simply + * re-creating this inode and will be rename'ing it into place once we + * rename the parent directory. + */ + return process_recorded_refs(sctx, &pending_move); +} + +static int send_set_xattr(struct send_ctx *sctx, + const char *name, int name_len, + const char *data, int data_len) +{ + struct fs_path *path; + int ret; + + path = get_cur_inode_path(sctx); + if (IS_ERR(path)) + return PTR_ERR(path); + + ret = begin_cmd(sctx, BTRFS_SEND_C_SET_XATTR); + if (ret < 0) + return ret; + + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path); + TLV_PUT_STRING(sctx, BTRFS_SEND_A_XATTR_NAME, name, name_len); + TLV_PUT(sctx, BTRFS_SEND_A_XATTR_DATA, data, data_len); + + ret = send_cmd(sctx); + +tlv_put_failure: + return ret; +} + +static int send_remove_xattr(struct send_ctx *sctx, + struct fs_path *path, + const char *name, int name_len) +{ + int ret; + + ret = begin_cmd(sctx, BTRFS_SEND_C_REMOVE_XATTR); + if (ret < 0) + return ret; + + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path); + TLV_PUT_STRING(sctx, BTRFS_SEND_A_XATTR_NAME, name, name_len); + + ret = send_cmd(sctx); + +tlv_put_failure: + return ret; +} + +static int __process_new_xattr(int num, struct btrfs_key *di_key, + const char *name, int name_len, const char *data, + int data_len, void *ctx) +{ + struct send_ctx *sctx = ctx; + struct posix_acl_xattr_header dummy_acl; + + /* Capabilities are emitted by finish_inode_if_needed */ + if (!strncmp(name, XATTR_NAME_CAPS, name_len)) + return 0; + + /* + * This hack is needed because empty acls are stored as zero byte + * data in xattrs. Problem with that is, that receiving these zero byte + * acls will fail later. To fix this, we send a dummy acl list that + * only contains the version number and no entries. + */ + if (!strncmp(name, XATTR_NAME_POSIX_ACL_ACCESS, name_len) || + !strncmp(name, XATTR_NAME_POSIX_ACL_DEFAULT, name_len)) { + if (data_len == 0) { + dummy_acl.a_version = + cpu_to_le32(POSIX_ACL_XATTR_VERSION); + data = (char *)&dummy_acl; + data_len = sizeof(dummy_acl); + } + } + + return send_set_xattr(sctx, name, name_len, data, data_len); +} + +static int __process_deleted_xattr(int num, struct btrfs_key *di_key, + const char *name, int name_len, + const char *data, int data_len, void *ctx) +{ + struct send_ctx *sctx = ctx; + struct fs_path *p; + + p = get_cur_inode_path(sctx); + if (IS_ERR(p)) + return PTR_ERR(p); + + return send_remove_xattr(sctx, p, name, name_len); +} + +static int process_new_xattr(struct send_ctx *sctx) +{ + return iterate_dir_item(sctx->send_root, sctx->left_path, + __process_new_xattr, sctx); +} + +static int process_deleted_xattr(struct send_ctx *sctx) +{ + return iterate_dir_item(sctx->parent_root, sctx->right_path, + __process_deleted_xattr, sctx); +} + +struct find_xattr_ctx { + const char *name; + int name_len; + int found_idx; + char *found_data; + int found_data_len; + bool copy_data; +}; + +static int __find_xattr(int num, struct btrfs_key *di_key, const char *name, + int name_len, const char *data, int data_len, void *vctx) +{ + struct find_xattr_ctx *ctx = vctx; + + if (name_len == ctx->name_len && + strncmp(name, ctx->name, name_len) == 0) { + ctx->found_idx = num; + ctx->found_data_len = data_len; + if (ctx->copy_data) { + ctx->found_data = kmemdup(data, data_len, GFP_KERNEL); + if (!ctx->found_data) + return -ENOMEM; + } + return 1; + } + return 0; +} + +static int find_xattr(struct btrfs_root *root, + struct btrfs_path *path, + struct btrfs_key *key, + const char *name, int name_len, + char **data, int *data_len) +{ + int ret; + struct find_xattr_ctx ctx; + + ctx.name = name; + ctx.name_len = name_len; + ctx.found_idx = -1; + ctx.found_data = NULL; + ctx.found_data_len = 0; + ctx.copy_data = (data != NULL); + + ret = iterate_dir_item(root, path, __find_xattr, &ctx); + if (ret < 0) + return ret; + + if (ctx.found_idx == -1) + return -ENOENT; + if (data) { + *data = ctx.found_data; + *data_len = ctx.found_data_len; + } else { + ASSERT(ctx.found_data == NULL); + } + return ctx.found_idx; +} + + +static int __process_changed_new_xattr(int num, struct btrfs_key *di_key, + const char *name, int name_len, + const char *data, int data_len, + void *ctx) +{ + int ret; + struct send_ctx *sctx = ctx; + char AUTO_KFREE(found_data); + int found_data_len = 0; + + ret = find_xattr(sctx->parent_root, sctx->right_path, + sctx->cmp_key, name, name_len, &found_data, + &found_data_len); + if (ret == -ENOENT) { + ret = __process_new_xattr(num, di_key, name, name_len, data, + data_len, ctx); + } else if (ret >= 0) { + if (data_len != found_data_len || + memcmp(data, found_data, data_len)) { + ret = __process_new_xattr(num, di_key, name, name_len, + data, data_len, ctx); + } else { + ret = 0; + } + } + + return ret; +} + +static int __process_changed_deleted_xattr(int num, struct btrfs_key *di_key, + const char *name, int name_len, + const char *data, int data_len, + void *ctx) +{ + int ret; + struct send_ctx *sctx = ctx; + + ret = find_xattr(sctx->send_root, sctx->left_path, sctx->cmp_key, + name, name_len, NULL, NULL); + if (ret == -ENOENT) + ret = __process_deleted_xattr(num, di_key, name, name_len, data, + data_len, ctx); + else if (ret >= 0) + ret = 0; + + return ret; +} + +static int process_changed_xattr(struct send_ctx *sctx) +{ + int ret; + + ret = iterate_dir_item(sctx->send_root, sctx->left_path, + __process_changed_new_xattr, sctx); + if (ret < 0) + return ret; + + return iterate_dir_item(sctx->parent_root, sctx->right_path, + __process_changed_deleted_xattr, sctx); +} + +static int process_all_new_xattrs(struct send_ctx *sctx) +{ + int ret = 0; + int iter_ret = 0; + struct btrfs_root *root; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + struct btrfs_key found_key; + + path = alloc_path_for_send(); + if (!path) + return -ENOMEM; + + root = sctx->send_root; + + key.objectid = sctx->cmp_key->objectid; + key.type = BTRFS_XATTR_ITEM_KEY; + key.offset = 0; + btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) { + if (found_key.objectid != key.objectid || + found_key.type != key.type) { + ret = 0; + break; + } + + ret = iterate_dir_item(root, path, __process_new_xattr, sctx); + if (ret < 0) + break; + } + /* Catch error found during iteration */ + if (iter_ret < 0) + ret = iter_ret; + + return ret; +} + +static int send_verity(struct send_ctx *sctx, struct fs_path *path, + struct fsverity_descriptor *desc) +{ + int ret; + + ret = begin_cmd(sctx, BTRFS_SEND_C_ENABLE_VERITY); + if (ret < 0) + return ret; + + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path); + TLV_PUT_U8(sctx, BTRFS_SEND_A_VERITY_ALGORITHM, + le8_to_cpu(desc->hash_algorithm)); + TLV_PUT_U32(sctx, BTRFS_SEND_A_VERITY_BLOCK_SIZE, + 1U << le8_to_cpu(desc->log_blocksize)); + TLV_PUT(sctx, BTRFS_SEND_A_VERITY_SALT_DATA, desc->salt, + le8_to_cpu(desc->salt_size)); + TLV_PUT(sctx, BTRFS_SEND_A_VERITY_SIG_DATA, desc->signature, + le32_to_cpu(desc->sig_size)); + + ret = send_cmd(sctx); + +tlv_put_failure: + return ret; +} + +static int process_verity(struct send_ctx *sctx) +{ + int ret = 0; + struct btrfs_inode *inode; + struct fs_path *p; + + inode = btrfs_iget(sctx->cur_ino, sctx->send_root); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + ret = btrfs_get_verity_descriptor(&inode->vfs_inode, NULL, 0); + if (ret < 0) + goto iput; + + if (unlikely(ret > FS_VERITY_MAX_DESCRIPTOR_SIZE)) { + ret = -EMSGSIZE; + goto iput; + } + if (!sctx->verity_descriptor) { + sctx->verity_descriptor = kvmalloc(FS_VERITY_MAX_DESCRIPTOR_SIZE, + GFP_KERNEL); + if (!sctx->verity_descriptor) { + ret = -ENOMEM; + goto iput; + } + } + + ret = btrfs_get_verity_descriptor(&inode->vfs_inode, sctx->verity_descriptor, ret); + if (ret < 0) + goto iput; + + p = get_cur_inode_path(sctx); + if (IS_ERR(p)) { + ret = PTR_ERR(p); + goto iput; + } + + ret = send_verity(sctx, p, sctx->verity_descriptor); +iput: + iput(&inode->vfs_inode); + return ret; +} + +static inline u64 max_send_read_size(const struct send_ctx *sctx) +{ + return sctx->send_max_size - SZ_16K; +} + +static int put_data_header(struct send_ctx *sctx, u32 len) +{ + if (WARN_ON_ONCE(sctx->put_data)) + return -EINVAL; + sctx->put_data = true; + if (sctx->proto >= 2) { + /* + * Since v2, the data attribute header doesn't include a length, + * it is implicitly to the end of the command. + */ + if (unlikely(sctx->send_max_size - sctx->send_size < sizeof(__le16) + len)) + return -EOVERFLOW; + put_unaligned_le16(BTRFS_SEND_A_DATA, sctx->send_buf + sctx->send_size); + sctx->send_size += sizeof(__le16); + } else { + struct btrfs_tlv_header *hdr; + + if (unlikely(sctx->send_max_size - sctx->send_size < sizeof(*hdr) + len)) + return -EOVERFLOW; + hdr = (struct btrfs_tlv_header *)(sctx->send_buf + sctx->send_size); + put_unaligned_le16(BTRFS_SEND_A_DATA, &hdr->tlv_type); + put_unaligned_le16(len, &hdr->tlv_len); + sctx->send_size += sizeof(*hdr); + } + return 0; +} + +static int put_file_data(struct send_ctx *sctx, u64 offset, u32 len) +{ + struct btrfs_root *root = sctx->send_root; + struct btrfs_fs_info *fs_info = root->fs_info; + u64 cur = offset; + const u64 end = offset + len; + const pgoff_t last_index = ((end - 1) >> PAGE_SHIFT); + struct address_space *mapping = sctx->cur_inode->i_mapping; + int ret; + + ret = put_data_header(sctx, len); + if (ret) + return ret; + + while (cur < end) { + pgoff_t index = (cur >> PAGE_SHIFT); + unsigned int cur_len; + unsigned int pg_offset; + struct folio *folio; + + folio = filemap_lock_folio(mapping, index); + if (IS_ERR(folio)) { + page_cache_sync_readahead(mapping, + &sctx->ra, NULL, index, + last_index + 1 - index); + + folio = filemap_grab_folio(mapping, index); + if (IS_ERR(folio)) { + ret = PTR_ERR(folio); + break; + } + } + pg_offset = offset_in_folio(folio, cur); + cur_len = min_t(unsigned int, end - cur, folio_size(folio) - pg_offset); + + if (folio_test_readahead(folio)) + page_cache_async_readahead(mapping, &sctx->ra, NULL, folio, + last_index + 1 - index); + + if (!folio_test_uptodate(folio)) { + btrfs_read_folio(NULL, folio); + folio_lock(folio); + if (unlikely(!folio_test_uptodate(folio))) { + folio_unlock(folio); + btrfs_err(fs_info, + "send: IO error at offset %llu for inode %llu root %llu", + folio_pos(folio), sctx->cur_ino, + btrfs_root_id(sctx->send_root)); + folio_put(folio); + ret = -EIO; + break; + } + if (folio->mapping != mapping) { + folio_unlock(folio); + folio_put(folio); + continue; + } + } + + memcpy_from_folio(sctx->send_buf + sctx->send_size, folio, + pg_offset, cur_len); + folio_unlock(folio); + folio_put(folio); + cur += cur_len; + sctx->send_size += cur_len; + } + + return ret; +} + +/* + * Read some bytes from the current inode/file and send a write command to + * user space. + */ +static int send_write(struct send_ctx *sctx, u64 offset, u32 len) +{ + int ret = 0; + struct fs_path *p; + + p = get_cur_inode_path(sctx); + if (IS_ERR(p)) + return PTR_ERR(p); + + ret = begin_cmd(sctx, BTRFS_SEND_C_WRITE); + if (ret < 0) + return ret; + + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); + TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset); + ret = put_file_data(sctx, offset, len); + if (ret < 0) + return ret; + + ret = send_cmd(sctx); + +tlv_put_failure: + return ret; +} + +/* + * Send a clone command to user space. + */ +static int send_clone(struct send_ctx *sctx, + u64 offset, u32 len, + struct clone_root *clone_root) +{ + int ret = 0; + struct fs_path *p; + struct fs_path *cur_inode_path; + u64 gen; + + cur_inode_path = get_cur_inode_path(sctx); + if (IS_ERR(cur_inode_path)) + return PTR_ERR(cur_inode_path); + + p = fs_path_alloc(); + if (!p) + return -ENOMEM; + + ret = begin_cmd(sctx, BTRFS_SEND_C_CLONE); + if (ret < 0) + goto out; + + TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset); + TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_LEN, len); + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, cur_inode_path); + + if (clone_root->root == sctx->send_root) { + ret = get_inode_gen(sctx->send_root, clone_root->ino, &gen); + if (ret < 0) + goto out; + ret = get_cur_path(sctx, clone_root->ino, gen, p); + } else { + ret = get_inode_path(clone_root->root, clone_root->ino, p); + } + if (ret < 0) + goto out; + + /* + * If the parent we're using has a received_uuid set then use that as + * our clone source as that is what we will look for when doing a + * receive. + * + * This covers the case that we create a snapshot off of a received + * subvolume and then use that as the parent and try to receive on a + * different host. + */ + if (!btrfs_is_empty_uuid(clone_root->root->root_item.received_uuid)) + TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID, + clone_root->root->root_item.received_uuid); + else + TLV_PUT_UUID(sctx, BTRFS_SEND_A_CLONE_UUID, + clone_root->root->root_item.uuid); + TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_CTRANSID, + btrfs_root_ctransid(&clone_root->root->root_item)); + TLV_PUT_PATH(sctx, BTRFS_SEND_A_CLONE_PATH, p); + TLV_PUT_U64(sctx, BTRFS_SEND_A_CLONE_OFFSET, + clone_root->offset); + + ret = send_cmd(sctx); + +tlv_put_failure: +out: + fs_path_free(p); + return ret; +} + +/* + * Send an update extent command to user space. + */ +static int send_update_extent(struct send_ctx *sctx, + u64 offset, u32 len) +{ + int ret = 0; + struct fs_path *p; + + p = get_cur_inode_path(sctx); + if (IS_ERR(p)) + return PTR_ERR(p); + + ret = begin_cmd(sctx, BTRFS_SEND_C_UPDATE_EXTENT); + if (ret < 0) + return ret; + + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); + TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset); + TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, len); + + ret = send_cmd(sctx); + +tlv_put_failure: + return ret; +} + +static int send_fallocate(struct send_ctx *sctx, u32 mode, u64 offset, u64 len) +{ + struct fs_path *path; + int ret; + + path = get_cur_inode_path(sctx); + if (IS_ERR(path)) + return PTR_ERR(path); + + ret = begin_cmd(sctx, BTRFS_SEND_C_FALLOCATE); + if (ret < 0) + return ret; + + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, path); + TLV_PUT_U32(sctx, BTRFS_SEND_A_FALLOCATE_MODE, mode); + TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset); + TLV_PUT_U64(sctx, BTRFS_SEND_A_SIZE, len); + + ret = send_cmd(sctx); + +tlv_put_failure: + return ret; +} + +static int send_hole(struct send_ctx *sctx, u64 end) +{ + struct fs_path *p = NULL; + u64 read_size = max_send_read_size(sctx); + u64 offset = sctx->cur_inode_last_extent; + int ret = 0; + + /* + * Starting with send stream v2 we have fallocate and can use it to + * punch holes instead of sending writes full of zeroes. + */ + if (proto_cmd_ok(sctx, BTRFS_SEND_C_FALLOCATE)) + return send_fallocate(sctx, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + offset, end - offset); + + /* + * A hole that starts at EOF or beyond it. Since we do not yet support + * fallocate (for extent preallocation and hole punching), sending a + * write of zeroes starting at EOF or beyond would later require issuing + * a truncate operation which would undo the write and achieve nothing. + */ + if (offset >= sctx->cur_inode_size) + return 0; + + /* + * Don't go beyond the inode's i_size due to prealloc extents that start + * after the i_size. + */ + end = min_t(u64, end, sctx->cur_inode_size); + + if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA) + return send_update_extent(sctx, offset, end - offset); + + p = get_cur_inode_path(sctx); + if (IS_ERR(p)) + return PTR_ERR(p); + + while (offset < end) { + u64 len = min(end - offset, read_size); + + ret = begin_cmd(sctx, BTRFS_SEND_C_WRITE); + if (ret < 0) + break; + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, p); + TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset); + ret = put_data_header(sctx, len); + if (ret < 0) + break; + memset(sctx->send_buf + sctx->send_size, 0, len); + sctx->send_size += len; + ret = send_cmd(sctx); + if (ret < 0) + break; + offset += len; + } + sctx->cur_inode_next_write_offset = offset; +tlv_put_failure: + return ret; +} + +static int send_encoded_inline_extent(struct send_ctx *sctx, + struct btrfs_path *path, u64 offset, + u64 len) +{ + struct btrfs_fs_info *fs_info = sctx->send_root->fs_info; + struct fs_path *fspath; + struct extent_buffer *leaf = path->nodes[0]; + struct btrfs_key key; + struct btrfs_file_extent_item *ei; + u64 ram_bytes; + size_t inline_size; + int ret; + + fspath = get_cur_inode_path(sctx); + if (IS_ERR(fspath)) + return PTR_ERR(fspath); + + ret = begin_cmd(sctx, BTRFS_SEND_C_ENCODED_WRITE); + if (ret < 0) + return ret; + + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); + ram_bytes = btrfs_file_extent_ram_bytes(leaf, ei); + inline_size = btrfs_file_extent_inline_item_len(leaf, path->slots[0]); + + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, fspath); + TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset); + TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_FILE_LEN, + min(key.offset + ram_bytes - offset, len)); + TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_LEN, ram_bytes); + TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_OFFSET, offset - key.offset); + ret = btrfs_encoded_io_compression_from_extent(fs_info, + btrfs_file_extent_compression(leaf, ei)); + if (ret < 0) + return ret; + TLV_PUT_U32(sctx, BTRFS_SEND_A_COMPRESSION, ret); + + ret = put_data_header(sctx, inline_size); + if (ret < 0) + return ret; + read_extent_buffer(leaf, sctx->send_buf + sctx->send_size, + btrfs_file_extent_inline_start(ei), inline_size); + sctx->send_size += inline_size; + + ret = send_cmd(sctx); + +tlv_put_failure: + return ret; +} + +static int send_encoded_extent(struct send_ctx *sctx, struct btrfs_path *path, + u64 offset, u64 len) +{ + struct btrfs_root *root = sctx->send_root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_inode *inode; + struct fs_path *fspath; + struct extent_buffer *leaf = path->nodes[0]; + struct btrfs_key key; + struct btrfs_file_extent_item *ei; + u64 disk_bytenr, disk_num_bytes; + u32 data_offset; + struct btrfs_cmd_header *hdr; + u32 crc; + int ret; + + inode = btrfs_iget(sctx->cur_ino, root); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + fspath = get_cur_inode_path(sctx); + if (IS_ERR(fspath)) { + ret = PTR_ERR(fspath); + goto out; + } + + ret = begin_cmd(sctx, BTRFS_SEND_C_ENCODED_WRITE); + if (ret < 0) + goto out; + + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_file_extent_item); + disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, ei); + disk_num_bytes = btrfs_file_extent_disk_num_bytes(leaf, ei); + + TLV_PUT_PATH(sctx, BTRFS_SEND_A_PATH, fspath); + TLV_PUT_U64(sctx, BTRFS_SEND_A_FILE_OFFSET, offset); + TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_FILE_LEN, + min(key.offset + btrfs_file_extent_num_bytes(leaf, ei) - offset, + len)); + TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_LEN, + btrfs_file_extent_ram_bytes(leaf, ei)); + TLV_PUT_U64(sctx, BTRFS_SEND_A_UNENCODED_OFFSET, + offset - key.offset + btrfs_file_extent_offset(leaf, ei)); + ret = btrfs_encoded_io_compression_from_extent(fs_info, + btrfs_file_extent_compression(leaf, ei)); + if (ret < 0) + goto out; + TLV_PUT_U32(sctx, BTRFS_SEND_A_COMPRESSION, ret); + TLV_PUT_U32(sctx, BTRFS_SEND_A_ENCRYPTION, 0); + + ret = put_data_header(sctx, disk_num_bytes); + if (ret < 0) + goto out; + + /* + * We want to do I/O directly into the send buffer, so get the next page + * boundary in the send buffer. This means that there may be a gap + * between the beginning of the command and the file data. + */ + data_offset = PAGE_ALIGN(sctx->send_size); + if (unlikely(data_offset > sctx->send_max_size || + sctx->send_max_size - data_offset < disk_num_bytes)) { + ret = -EOVERFLOW; + goto out; + } + + /* + * Note that send_buf is a mapping of send_buf_pages, so this is really + * reading into send_buf. + */ + ret = btrfs_encoded_read_regular_fill_pages(inode, + disk_bytenr, disk_num_bytes, + sctx->send_buf_pages + + (data_offset >> PAGE_SHIFT), + NULL); + if (ret) + goto out; + + hdr = (struct btrfs_cmd_header *)sctx->send_buf; + hdr->len = cpu_to_le32(sctx->send_size + disk_num_bytes - sizeof(*hdr)); + hdr->crc = 0; + crc = crc32c(0, sctx->send_buf, sctx->send_size); + crc = crc32c(crc, sctx->send_buf + data_offset, disk_num_bytes); + hdr->crc = cpu_to_le32(crc); + + ret = write_buf(sctx->send_filp, sctx->send_buf, sctx->send_size, + &sctx->send_off); + if (!ret) { + ret = write_buf(sctx->send_filp, sctx->send_buf + data_offset, + disk_num_bytes, &sctx->send_off); + } + sctx->send_size = 0; + sctx->put_data = false; + +tlv_put_failure: +out: + iput(&inode->vfs_inode); + return ret; +} + +static int send_extent_data(struct send_ctx *sctx, struct btrfs_path *path, + const u64 offset, const u64 len) +{ + const u64 end = offset + len; + struct extent_buffer *leaf = path->nodes[0]; + struct btrfs_file_extent_item *ei; + u64 read_size = max_send_read_size(sctx); + u64 sent = 0; + + if (sctx->flags & BTRFS_SEND_FLAG_NO_FILE_DATA) + return send_update_extent(sctx, offset, len); + + ei = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_file_extent_item); + if ((sctx->flags & BTRFS_SEND_FLAG_COMPRESSED) && + btrfs_file_extent_compression(leaf, ei) != BTRFS_COMPRESS_NONE) { + bool is_inline = (btrfs_file_extent_type(leaf, ei) == + BTRFS_FILE_EXTENT_INLINE); + + /* + * Send the compressed extent unless the compressed data is + * larger than the decompressed data. This can happen if we're + * not sending the entire extent, either because it has been + * partially overwritten/truncated or because this is a part of + * the extent that we couldn't clone in clone_range(). + */ + if (is_inline && + btrfs_file_extent_inline_item_len(leaf, + path->slots[0]) <= len) { + return send_encoded_inline_extent(sctx, path, offset, + len); + } else if (!is_inline && + btrfs_file_extent_disk_num_bytes(leaf, ei) <= len) { + return send_encoded_extent(sctx, path, offset, len); + } + } + + if (sctx->cur_inode == NULL) { + struct btrfs_inode *btrfs_inode; + struct btrfs_root *root = sctx->send_root; + + btrfs_inode = btrfs_iget(sctx->cur_ino, root); + if (IS_ERR(btrfs_inode)) + return PTR_ERR(btrfs_inode); + + sctx->cur_inode = &btrfs_inode->vfs_inode; + memset(&sctx->ra, 0, sizeof(struct file_ra_state)); + file_ra_state_init(&sctx->ra, sctx->cur_inode->i_mapping); + + /* + * It's very likely there are no pages from this inode in the page + * cache, so after reading extents and sending their data, we clean + * the page cache to avoid trashing the page cache (adding pressure + * to the page cache and forcing eviction of other data more useful + * for applications). + * + * We decide if we should clean the page cache simply by checking + * if the inode's mapping nrpages is 0 when we first open it, and + * not by using something like filemap_range_has_page() before + * reading an extent because when we ask the readahead code to + * read a given file range, it may (and almost always does) read + * pages from beyond that range (see the documentation for + * page_cache_sync_readahead()), so it would not be reliable, + * because after reading the first extent future calls to + * filemap_range_has_page() would return true because the readahead + * on the previous extent resulted in reading pages of the current + * extent as well. + */ + sctx->clean_page_cache = (sctx->cur_inode->i_mapping->nrpages == 0); + sctx->page_cache_clear_start = round_down(offset, PAGE_SIZE); + } + + while (sent < len) { + u64 size = min(len - sent, read_size); + int ret; + + ret = send_write(sctx, offset + sent, size); + if (ret < 0) + return ret; + sent += size; + } + + if (sctx->clean_page_cache && PAGE_ALIGNED(end)) { + /* + * Always operate only on ranges that are a multiple of the page + * size. This is not only to prevent zeroing parts of a page in + * the case of subpage sector size, but also to guarantee we evict + * pages, as passing a range that is smaller than page size does + * not evict the respective page (only zeroes part of its content). + * + * Always start from the end offset of the last range cleared. + * This is because the readahead code may (and very often does) + * reads pages beyond the range we request for readahead. So if + * we have an extent layout like this: + * + * [ extent A ] [ extent B ] [ extent C ] + * + * When we ask page_cache_sync_readahead() to read extent A, it + * may also trigger reads for pages of extent B. If we are doing + * an incremental send and extent B has not changed between the + * parent and send snapshots, some or all of its pages may end + * up being read and placed in the page cache. So when truncating + * the page cache we always start from the end offset of the + * previously processed extent up to the end of the current + * extent. + */ + truncate_inode_pages_range(&sctx->cur_inode->i_data, + sctx->page_cache_clear_start, + end - 1); + sctx->page_cache_clear_start = end; + } + + return 0; +} + +/* + * Search for a capability xattr related to sctx->cur_ino. If the capability is + * found, call send_set_xattr function to emit it. + * + * Return 0 if there isn't a capability, or when the capability was emitted + * successfully, or < 0 if an error occurred. + */ +static int send_capabilities(struct send_ctx *sctx) +{ + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_dir_item *di; + struct extent_buffer *leaf; + unsigned long data_ptr; + char AUTO_KFREE(buf); + int buf_len; + int ret = 0; + + path = alloc_path_for_send(); + if (!path) + return -ENOMEM; + + di = btrfs_lookup_xattr(NULL, sctx->send_root, path, sctx->cur_ino, + XATTR_NAME_CAPS, strlen(XATTR_NAME_CAPS), 0); + if (!di) { + /* There is no xattr for this inode */ + return 0; + } else if (IS_ERR(di)) { + return PTR_ERR(di); + } + + leaf = path->nodes[0]; + buf_len = btrfs_dir_data_len(leaf, di); + + buf = kmalloc(buf_len, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + data_ptr = (unsigned long)(di + 1) + btrfs_dir_name_len(leaf, di); + read_extent_buffer(leaf, buf, data_ptr, buf_len); + + ret = send_set_xattr(sctx, XATTR_NAME_CAPS, + strlen(XATTR_NAME_CAPS), buf, buf_len); + return ret; +} + +static int clone_range(struct send_ctx *sctx, struct btrfs_path *dst_path, + struct clone_root *clone_root, const u64 disk_byte, + u64 data_offset, u64 offset, u64 len) +{ + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + int ret; + struct btrfs_inode_info info; + u64 clone_src_i_size = 0; + + /* + * Prevent cloning from a zero offset with a length matching the sector + * size because in some scenarios this will make the receiver fail. + * + * For example, if in the source filesystem the extent at offset 0 + * has a length of sectorsize and it was written using direct IO, then + * it can never be an inline extent (even if compression is enabled). + * Then this extent can be cloned in the original filesystem to a non + * zero file offset, but it may not be possible to clone in the + * destination filesystem because it can be inlined due to compression + * on the destination filesystem (as the receiver's write operations are + * always done using buffered IO). The same happens when the original + * filesystem does not have compression enabled but the destination + * filesystem has. + */ + if (clone_root->offset == 0 && + len == sctx->send_root->fs_info->sectorsize) + return send_extent_data(sctx, dst_path, offset, len); + + path = alloc_path_for_send(); + if (!path) + return -ENOMEM; + + /* + * There are inodes that have extents that lie behind its i_size. Don't + * accept clones from these extents. + */ + ret = get_inode_info(clone_root->root, clone_root->ino, &info); + btrfs_release_path(path); + if (ret < 0) + return ret; + clone_src_i_size = info.size; + + /* + * We can't send a clone operation for the entire range if we find + * extent items in the respective range in the source file that + * refer to different extents or if we find holes. + * So check for that and do a mix of clone and regular write/copy + * operations if needed. + * + * Example: + * + * mkfs.btrfs -f /dev/sda + * mount /dev/sda /mnt + * xfs_io -f -c "pwrite -S 0xaa 0K 100K" /mnt/foo + * cp --reflink=always /mnt/foo /mnt/bar + * xfs_io -c "pwrite -S 0xbb 50K 50K" /mnt/foo + * btrfs subvolume snapshot -r /mnt /mnt/snap + * + * If when we send the snapshot and we are processing file bar (which + * has a higher inode number than foo) we blindly send a clone operation + * for the [0, 100K[ range from foo to bar, the receiver ends up getting + * a file bar that matches the content of file foo - iow, doesn't match + * the content from bar in the original filesystem. + */ + key.objectid = clone_root->ino; + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = clone_root->offset; + ret = btrfs_search_slot(NULL, clone_root->root, &key, path, 0, 0); + if (ret < 0) + return ret; + if (ret > 0 && path->slots[0] > 0) { + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0] - 1); + if (key.objectid == clone_root->ino && + key.type == BTRFS_EXTENT_DATA_KEY) + path->slots[0]--; + } + + while (true) { + struct extent_buffer *leaf = path->nodes[0]; + int slot = path->slots[0]; + struct btrfs_file_extent_item *ei; + u8 type; + u64 ext_len; + u64 clone_len; + u64 clone_data_offset; + bool crossed_src_i_size = false; + + if (slot >= btrfs_header_nritems(leaf)) { + ret = btrfs_next_leaf(clone_root->root, path); + if (ret < 0) + return ret; + else if (ret > 0) + break; + continue; + } + + btrfs_item_key_to_cpu(leaf, &key, slot); + + /* + * We might have an implicit trailing hole (NO_HOLES feature + * enabled). We deal with it after leaving this loop. + */ + if (key.objectid != clone_root->ino || + key.type != BTRFS_EXTENT_DATA_KEY) + break; + + ei = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item); + type = btrfs_file_extent_type(leaf, ei); + if (type == BTRFS_FILE_EXTENT_INLINE) { + ext_len = btrfs_file_extent_ram_bytes(leaf, ei); + ext_len = PAGE_ALIGN(ext_len); + } else { + ext_len = btrfs_file_extent_num_bytes(leaf, ei); + } + + if (key.offset + ext_len <= clone_root->offset) + goto next; + + if (key.offset > clone_root->offset) { + /* Implicit hole, NO_HOLES feature enabled. */ + u64 hole_len = key.offset - clone_root->offset; + + if (hole_len > len) + hole_len = len; + ret = send_extent_data(sctx, dst_path, offset, + hole_len); + if (ret < 0) + return ret; + + len -= hole_len; + if (len == 0) + break; + offset += hole_len; + clone_root->offset += hole_len; + data_offset += hole_len; + } + + if (key.offset >= clone_root->offset + len) + break; + + if (key.offset >= clone_src_i_size) + break; + + if (key.offset + ext_len > clone_src_i_size) { + ext_len = clone_src_i_size - key.offset; + crossed_src_i_size = true; + } + + clone_data_offset = btrfs_file_extent_offset(leaf, ei); + if (btrfs_file_extent_disk_bytenr(leaf, ei) == disk_byte) { + clone_root->offset = key.offset; + if (clone_data_offset < data_offset && + clone_data_offset + ext_len > data_offset) { + u64 extent_offset; + + extent_offset = data_offset - clone_data_offset; + ext_len -= extent_offset; + clone_data_offset += extent_offset; + clone_root->offset += extent_offset; + } + } + + clone_len = min_t(u64, ext_len, len); + + if (btrfs_file_extent_disk_bytenr(leaf, ei) == disk_byte && + clone_data_offset == data_offset) { + const u64 src_end = clone_root->offset + clone_len; + const u64 sectorsize = SZ_64K; + + /* + * We can't clone the last block, when its size is not + * sector size aligned, into the middle of a file. If we + * do so, the receiver will get a failure (-EINVAL) when + * trying to clone or will silently corrupt the data in + * the destination file if it's on a kernel without the + * fix introduced by commit ac765f83f1397646 + * ("Btrfs: fix data corruption due to cloning of eof + * block). + * + * So issue a clone of the aligned down range plus a + * regular write for the eof block, if we hit that case. + * + * Also, we use the maximum possible sector size, 64K, + * because we don't know what's the sector size of the + * filesystem that receives the stream, so we have to + * assume the largest possible sector size. + */ + if (src_end == clone_src_i_size && + !IS_ALIGNED(src_end, sectorsize) && + offset + clone_len < sctx->cur_inode_size) { + u64 slen; + + slen = ALIGN_DOWN(src_end - clone_root->offset, + sectorsize); + if (slen > 0) { + ret = send_clone(sctx, offset, slen, + clone_root); + if (ret < 0) + return ret; + } + ret = send_extent_data(sctx, dst_path, + offset + slen, + clone_len - slen); + } else { + ret = send_clone(sctx, offset, clone_len, + clone_root); + } + } else if (crossed_src_i_size && clone_len < len) { + /* + * If we are at i_size of the clone source inode and we + * can not clone from it, terminate the loop. This is + * to avoid sending two write operations, one with a + * length matching clone_len and the final one after + * this loop with a length of len - clone_len. + * + * When using encoded writes (BTRFS_SEND_FLAG_COMPRESSED + * was passed to the send ioctl), this helps avoid + * sending an encoded write for an offset that is not + * sector size aligned, in case the i_size of the source + * inode is not sector size aligned. That will make the + * receiver fallback to decompression of the data and + * writing it using regular buffered IO, therefore while + * not incorrect, it's not optimal due decompression and + * possible re-compression at the receiver. + */ + break; + } else { + ret = send_extent_data(sctx, dst_path, offset, + clone_len); + } + + if (ret < 0) + return ret; + + len -= clone_len; + if (len == 0) + break; + offset += clone_len; + clone_root->offset += clone_len; + + /* + * If we are cloning from the file we are currently processing, + * and using the send root as the clone root, we must stop once + * the current clone offset reaches the current eof of the file + * at the receiver, otherwise we would issue an invalid clone + * operation (source range going beyond eof) and cause the + * receiver to fail. So if we reach the current eof, bail out + * and fallback to a regular write. + */ + if (clone_root->root == sctx->send_root && + clone_root->ino == sctx->cur_ino && + clone_root->offset >= sctx->cur_inode_next_write_offset) + break; + + data_offset += clone_len; +next: + path->slots[0]++; + } + + if (len > 0) + ret = send_extent_data(sctx, dst_path, offset, len); + else + ret = 0; + return ret; +} + +static int send_write_or_clone(struct send_ctx *sctx, + struct btrfs_path *path, + struct btrfs_key *key, + struct clone_root *clone_root) +{ + int ret = 0; + u64 offset = key->offset; + u64 end; + const u32 bs = sctx->send_root->fs_info->sectorsize; + struct btrfs_file_extent_item *ei; + u64 disk_byte; + u64 data_offset; + u64 num_bytes; + struct btrfs_inode_info info = { 0 }; + + end = min_t(u64, btrfs_file_extent_end(path), sctx->cur_inode_size); + if (offset >= end) + return 0; + + num_bytes = end - offset; + + if (!clone_root) + goto write_data; + + if (IS_ALIGNED(end, bs)) + goto clone_data; + + /* + * If the extent end is not aligned, we can clone if the extent ends at + * the i_size of the inode and the clone range ends at the i_size of the + * source inode, otherwise the clone operation fails with -EINVAL. + */ + if (end != sctx->cur_inode_size) + goto write_data; + + ret = get_inode_info(clone_root->root, clone_root->ino, &info); + if (ret < 0) + return ret; + + if (clone_root->offset + num_bytes == info.size) { + /* + * The final size of our file matches the end offset, but it may + * be that its current size is larger, so we have to truncate it + * to any value between the start offset of the range and the + * final i_size, otherwise the clone operation is invalid + * because it's unaligned and it ends before the current EOF. + * We do this truncate to the final i_size when we finish + * processing the inode, but it's too late by then. And here we + * truncate to the start offset of the range because it's always + * sector size aligned while if it were the final i_size it + * would result in dirtying part of a page, filling part of a + * page with zeroes and then having the clone operation at the + * receiver trigger IO and wait for it due to the dirty page. + */ + if (sctx->parent_root != NULL) { + ret = send_truncate(sctx, sctx->cur_ino, + sctx->cur_inode_gen, offset); + if (ret < 0) + return ret; + } + goto clone_data; + } + +write_data: + ret = send_extent_data(sctx, path, offset, num_bytes); + sctx->cur_inode_next_write_offset = end; + return ret; + +clone_data: + ei = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_file_extent_item); + disk_byte = btrfs_file_extent_disk_bytenr(path->nodes[0], ei); + data_offset = btrfs_file_extent_offset(path->nodes[0], ei); + ret = clone_range(sctx, path, clone_root, disk_byte, data_offset, offset, + num_bytes); + sctx->cur_inode_next_write_offset = end; + return ret; +} + +static int is_extent_unchanged(struct send_ctx *sctx, + struct btrfs_path *left_path, + struct btrfs_key *ekey) +{ + int ret = 0; + struct btrfs_key key; + BTRFS_PATH_AUTO_FREE(path); + struct extent_buffer *eb; + int slot; + struct btrfs_key found_key; + struct btrfs_file_extent_item *ei; + u64 left_disknr; + u64 right_disknr; + u64 left_offset; + u64 right_offset; + u64 left_offset_fixed; + u64 left_len; + u64 right_len; + u64 left_gen; + u64 right_gen; + u8 left_type; + u8 right_type; + + path = alloc_path_for_send(); + if (!path) + return -ENOMEM; + + eb = left_path->nodes[0]; + slot = left_path->slots[0]; + ei = btrfs_item_ptr(eb, slot, struct btrfs_file_extent_item); + left_type = btrfs_file_extent_type(eb, ei); + + if (left_type != BTRFS_FILE_EXTENT_REG) + return 0; + + left_disknr = btrfs_file_extent_disk_bytenr(eb, ei); + left_len = btrfs_file_extent_num_bytes(eb, ei); + left_offset = btrfs_file_extent_offset(eb, ei); + left_gen = btrfs_file_extent_generation(eb, ei); + + /* + * Following comments will refer to these graphics. L is the left + * extents which we are checking at the moment. 1-8 are the right + * extents that we iterate. + * + * |-----L-----| + * |-1-|-2a-|-3-|-4-|-5-|-6-| + * + * |-----L-----| + * |--1--|-2b-|...(same as above) + * + * Alternative situation. Happens on files where extents got split. + * |-----L-----| + * |-----------7-----------|-6-| + * + * Alternative situation. Happens on files which got larger. + * |-----L-----| + * |-8-| + * Nothing follows after 8. + */ + + key.objectid = ekey->objectid; + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = ekey->offset; + ret = btrfs_search_slot_for_read(sctx->parent_root, &key, path, 0, 0); + if (ret < 0) + return ret; + if (ret) + return 0; + + /* + * Handle special case where the right side has no extents at all. + */ + eb = path->nodes[0]; + slot = path->slots[0]; + btrfs_item_key_to_cpu(eb, &found_key, slot); + if (found_key.objectid != key.objectid || + found_key.type != key.type) + /* If we're a hole then just pretend nothing changed */ + return (left_disknr ? 0 : 1); + + /* + * We're now on 2a, 2b or 7. + */ + key = found_key; + while (key.offset < ekey->offset + left_len) { + ei = btrfs_item_ptr(eb, slot, struct btrfs_file_extent_item); + right_type = btrfs_file_extent_type(eb, ei); + if (right_type != BTRFS_FILE_EXTENT_REG && + right_type != BTRFS_FILE_EXTENT_INLINE) + return 0; + + if (right_type == BTRFS_FILE_EXTENT_INLINE) { + right_len = btrfs_file_extent_ram_bytes(eb, ei); + right_len = PAGE_ALIGN(right_len); + } else { + right_len = btrfs_file_extent_num_bytes(eb, ei); + } + + /* + * Are we at extent 8? If yes, we know the extent is changed. + * This may only happen on the first iteration. + */ + if (found_key.offset + right_len <= ekey->offset) + /* If we're a hole just pretend nothing changed */ + return (left_disknr ? 0 : 1); + + /* + * We just wanted to see if when we have an inline extent, what + * follows it is a regular extent (wanted to check the above + * condition for inline extents too). This should normally not + * happen but it's possible for example when we have an inline + * compressed extent representing data with a size matching + * the page size (currently the same as sector size). + */ + if (right_type == BTRFS_FILE_EXTENT_INLINE) + return 0; + + right_disknr = btrfs_file_extent_disk_bytenr(eb, ei); + right_offset = btrfs_file_extent_offset(eb, ei); + right_gen = btrfs_file_extent_generation(eb, ei); + + left_offset_fixed = left_offset; + if (key.offset < ekey->offset) { + /* Fix the right offset for 2a and 7. */ + right_offset += ekey->offset - key.offset; + } else { + /* Fix the left offset for all behind 2a and 2b */ + left_offset_fixed += key.offset - ekey->offset; + } + + /* + * Check if we have the same extent. + */ + if (left_disknr != right_disknr || + left_offset_fixed != right_offset || + left_gen != right_gen) + return 0; + + /* + * Go to the next extent. + */ + ret = btrfs_next_item(sctx->parent_root, path); + if (ret < 0) + return ret; + if (!ret) { + eb = path->nodes[0]; + slot = path->slots[0]; + btrfs_item_key_to_cpu(eb, &found_key, slot); + } + if (ret || found_key.objectid != key.objectid || + found_key.type != key.type) { + key.offset += right_len; + break; + } + if (found_key.offset != key.offset + right_len) + return 0; + + key = found_key; + } + + /* + * We're now behind the left extent (treat as unchanged) or at the end + * of the right side (treat as changed). + */ + if (key.offset >= ekey->offset + left_len) + ret = 1; + else + ret = 0; + + return ret; +} + +static int get_last_extent(struct send_ctx *sctx, u64 offset) +{ + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_root *root = sctx->send_root; + struct btrfs_key key; + int ret; + + path = alloc_path_for_send(); + if (!path) + return -ENOMEM; + + sctx->cur_inode_last_extent = 0; + + key.objectid = sctx->cur_ino; + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = offset; + ret = btrfs_search_slot_for_read(root, &key, path, 0, 1); + if (ret < 0) + return ret; + ret = 0; + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + if (key.objectid != sctx->cur_ino || key.type != BTRFS_EXTENT_DATA_KEY) + return ret; + + sctx->cur_inode_last_extent = btrfs_file_extent_end(path); + return ret; +} + +static int range_is_hole_in_parent(struct send_ctx *sctx, + const u64 start, + const u64 end) +{ + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + struct btrfs_root *root = sctx->parent_root; + u64 search_start = start; + int ret; + + path = alloc_path_for_send(); + if (!path) + return -ENOMEM; + + key.objectid = sctx->cur_ino; + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = search_start; + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + return ret; + if (ret > 0 && path->slots[0] > 0) + path->slots[0]--; + + while (search_start < end) { + struct extent_buffer *leaf = path->nodes[0]; + int slot = path->slots[0]; + struct btrfs_file_extent_item *fi; + u64 extent_end; + + if (slot >= btrfs_header_nritems(leaf)) { + ret = btrfs_next_leaf(root, path); + if (ret < 0) + return ret; + if (ret > 0) + break; + continue; + } + + btrfs_item_key_to_cpu(leaf, &key, slot); + if (key.objectid < sctx->cur_ino || + key.type < BTRFS_EXTENT_DATA_KEY) + goto next; + if (key.objectid > sctx->cur_ino || + key.type > BTRFS_EXTENT_DATA_KEY || + key.offset >= end) + break; + + fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item); + extent_end = btrfs_file_extent_end(path); + if (extent_end <= start) + goto next; + if (btrfs_file_extent_type(leaf, fi) == BTRFS_FILE_EXTENT_INLINE) + return 0; + if (btrfs_file_extent_disk_bytenr(leaf, fi) == 0) { + search_start = extent_end; + goto next; + } + return 0; +next: + path->slots[0]++; + } + return 1; +} + +static int maybe_send_hole(struct send_ctx *sctx, struct btrfs_path *path, + struct btrfs_key *key) +{ + int ret = 0; + + if (sctx->cur_ino != key->objectid || !need_send_hole(sctx)) + return 0; + + /* + * Get last extent's end offset (exclusive) if we haven't determined it + * yet (we're processing the first file extent item that is new), or if + * we're at the first slot of a leaf and the last extent's end is less + * than the current extent's offset, because we might have skipped + * entire leaves that contained only file extent items for our current + * inode. These leaves have a generation number smaller (older) than the + * one in the current leaf and the leaf our last extent came from, and + * are located between these 2 leaves. + */ + if ((sctx->cur_inode_last_extent == (u64)-1) || + (path->slots[0] == 0 && sctx->cur_inode_last_extent < key->offset)) { + ret = get_last_extent(sctx, key->offset - 1); + if (ret) + return ret; + } + + if (sctx->cur_inode_last_extent < key->offset) { + ret = range_is_hole_in_parent(sctx, + sctx->cur_inode_last_extent, + key->offset); + if (ret < 0) + return ret; + else if (ret == 0) + ret = send_hole(sctx, key->offset); + else + ret = 0; + } + sctx->cur_inode_last_extent = btrfs_file_extent_end(path); + return ret; +} + +static int process_extent(struct send_ctx *sctx, + struct btrfs_path *path, + struct btrfs_key *key) +{ + struct clone_root *found_clone = NULL; + int ret = 0; + + if (S_ISLNK(sctx->cur_inode_mode)) + return 0; + + if (sctx->parent_root && !sctx->cur_inode_new) { + ret = is_extent_unchanged(sctx, path, key); + if (ret < 0) + return ret; + if (ret) + goto out_hole; + } else { + struct btrfs_file_extent_item *ei; + u8 type; + + ei = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_file_extent_item); + type = btrfs_file_extent_type(path->nodes[0], ei); + if (type == BTRFS_FILE_EXTENT_PREALLOC || + type == BTRFS_FILE_EXTENT_REG) { + /* + * The send spec does not have a prealloc command yet, + * so just leave a hole for prealloc'ed extents until + * we have enough commands queued up to justify rev'ing + * the send spec. + */ + if (type == BTRFS_FILE_EXTENT_PREALLOC) + return 0; + + /* Have a hole, just skip it. */ + if (btrfs_file_extent_disk_bytenr(path->nodes[0], ei) == 0) + return 0; + } + } + + ret = find_extent_clone(sctx, path, key->objectid, key->offset, + sctx->cur_inode_size, &found_clone); + if (ret != -ENOENT && ret < 0) + return ret; + + ret = send_write_or_clone(sctx, path, key, found_clone); + if (ret) + return ret; +out_hole: + return maybe_send_hole(sctx, path, key); +} + +static int process_all_extents(struct send_ctx *sctx) +{ + int ret = 0; + int iter_ret = 0; + struct btrfs_root *root; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + struct btrfs_key found_key; + + root = sctx->send_root; + path = alloc_path_for_send(); + if (!path) + return -ENOMEM; + + key.objectid = sctx->cmp_key->objectid; + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = 0; + btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) { + if (found_key.objectid != key.objectid || + found_key.type != key.type) { + ret = 0; + break; + } + + ret = process_extent(sctx, path, &found_key); + if (ret < 0) + break; + } + /* Catch error found during iteration */ + if (iter_ret < 0) + ret = iter_ret; + + return ret; +} + +static int process_recorded_refs_if_needed(struct send_ctx *sctx, bool at_end, + bool *pending_move, bool *refs_processed) +{ + int ret; + + if (sctx->cur_ino == 0) + return 0; + + if (!at_end && sctx->cur_ino == sctx->cmp_key->objectid && + sctx->cmp_key->type <= BTRFS_INODE_EXTREF_KEY) + return 0; + + if (list_empty(&sctx->new_refs) && list_empty(&sctx->deleted_refs)) + return 0; + + ret = process_recorded_refs(sctx, pending_move); + if (ret < 0) + return ret; + + *refs_processed = true; + return 0; +} + +static int finish_inode_if_needed(struct send_ctx *sctx, bool at_end) +{ + int ret = 0; + struct btrfs_inode_info info; + u64 left_mode; + u64 left_uid; + u64 left_gid; + u64 left_fileattr; + u64 right_mode; + u64 right_uid; + u64 right_gid; + u64 right_fileattr; + bool need_chmod = false; + bool need_chown = false; + bool need_fileattr = false; + bool need_truncate = true; + bool pending_move = false; + bool refs_processed = false; + + if (sctx->ignore_cur_inode) + return 0; + + ret = process_recorded_refs_if_needed(sctx, at_end, &pending_move, + &refs_processed); + if (ret < 0) + goto out; + + /* + * We have processed the refs and thus need to advance send_progress. + * Now, calls to get_cur_xxx will take the updated refs of the current + * inode into account. + * + * On the other hand, if our current inode is a directory and couldn't + * be moved/renamed because its parent was renamed/moved too and it has + * a higher inode number, we can only move/rename our current inode + * after we moved/renamed its parent. Therefore in this case operate on + * the old path (pre move/rename) of our current inode, and the + * move/rename will be performed later. + */ + if (refs_processed && !pending_move) + sctx->send_progress = sctx->cur_ino + 1; + + if (sctx->cur_ino == 0 || sctx->cur_inode_deleted) + goto out; + if (!at_end && sctx->cmp_key->objectid == sctx->cur_ino) + goto out; + ret = get_inode_info(sctx->send_root, sctx->cur_ino, &info); + if (ret < 0) + goto out; + left_mode = info.mode; + left_uid = info.uid; + left_gid = info.gid; + left_fileattr = info.fileattr; + + if (!sctx->parent_root || sctx->cur_inode_new) { + need_chown = true; + if (!S_ISLNK(sctx->cur_inode_mode)) + need_chmod = true; + if (sctx->cur_inode_next_write_offset == sctx->cur_inode_size) + need_truncate = false; + } else { + u64 old_size; + + ret = get_inode_info(sctx->parent_root, sctx->cur_ino, &info); + if (ret < 0) + goto out; + old_size = info.size; + right_mode = info.mode; + right_uid = info.uid; + right_gid = info.gid; + right_fileattr = info.fileattr; + + if (left_uid != right_uid || left_gid != right_gid) + need_chown = true; + if (!S_ISLNK(sctx->cur_inode_mode) && left_mode != right_mode) + need_chmod = true; + if (!S_ISLNK(sctx->cur_inode_mode) && left_fileattr != right_fileattr) + need_fileattr = true; + if ((old_size == sctx->cur_inode_size) || + (sctx->cur_inode_size > old_size && + sctx->cur_inode_next_write_offset == sctx->cur_inode_size)) + need_truncate = false; + } + + if (S_ISREG(sctx->cur_inode_mode)) { + if (need_send_hole(sctx)) { + if (sctx->cur_inode_last_extent == (u64)-1 || + sctx->cur_inode_last_extent < + sctx->cur_inode_size) { + ret = get_last_extent(sctx, (u64)-1); + if (ret) + goto out; + } + if (sctx->cur_inode_last_extent < sctx->cur_inode_size) { + ret = range_is_hole_in_parent(sctx, + sctx->cur_inode_last_extent, + sctx->cur_inode_size); + if (ret < 0) { + goto out; + } else if (ret == 0) { + ret = send_hole(sctx, sctx->cur_inode_size); + if (ret < 0) + goto out; + } else { + /* Range is already a hole, skip. */ + ret = 0; + } + } + } + if (need_truncate) { + ret = send_truncate(sctx, sctx->cur_ino, + sctx->cur_inode_gen, + sctx->cur_inode_size); + if (ret < 0) + goto out; + } + } + + if (need_chown) { + ret = send_chown(sctx, sctx->cur_ino, sctx->cur_inode_gen, + left_uid, left_gid); + if (ret < 0) + goto out; + } + if (need_chmod) { + ret = send_chmod(sctx, sctx->cur_ino, sctx->cur_inode_gen, + left_mode); + if (ret < 0) + goto out; + } + if (need_fileattr) { + ret = send_fileattr(sctx, sctx->cur_ino, sctx->cur_inode_gen, + left_fileattr); + if (ret < 0) + goto out; + } + + if (proto_cmd_ok(sctx, BTRFS_SEND_C_ENABLE_VERITY) + && sctx->cur_inode_needs_verity) { + ret = process_verity(sctx); + if (ret < 0) + goto out; + } + + ret = send_capabilities(sctx); + if (ret < 0) + goto out; + + /* + * If other directory inodes depended on our current directory + * inode's move/rename, now do their move/rename operations. + */ + if (!is_waiting_for_move(sctx, sctx->cur_ino)) { + ret = apply_children_dir_moves(sctx); + if (ret) + goto out; + /* + * Need to send that every time, no matter if it actually + * changed between the two trees as we have done changes to + * the inode before. If our inode is a directory and it's + * waiting to be moved/renamed, we will send its utimes when + * it's moved/renamed, therefore we don't need to do it here. + */ + sctx->send_progress = sctx->cur_ino + 1; + + /* + * If the current inode is a non-empty directory, delay issuing + * the utimes command for it, as it's very likely we have inodes + * with an higher number inside it. We want to issue the utimes + * command only after adding all dentries to it. + */ + if (S_ISDIR(sctx->cur_inode_mode) && sctx->cur_inode_size > 0) + ret = cache_dir_utimes(sctx, sctx->cur_ino, sctx->cur_inode_gen); + else + ret = send_utimes(sctx, sctx->cur_ino, sctx->cur_inode_gen); + + if (ret < 0) + goto out; + } + +out: + if (!ret) + ret = trim_dir_utimes_cache(sctx); + + return ret; +} + +static void close_current_inode(struct send_ctx *sctx) +{ + u64 i_size; + + if (sctx->cur_inode == NULL) + return; + + i_size = i_size_read(sctx->cur_inode); + + /* + * If we are doing an incremental send, we may have extents between the + * last processed extent and the i_size that have not been processed + * because they haven't changed but we may have read some of their pages + * through readahead, see the comments at send_extent_data(). + */ + if (sctx->clean_page_cache && sctx->page_cache_clear_start < i_size) + truncate_inode_pages_range(&sctx->cur_inode->i_data, + sctx->page_cache_clear_start, + round_up(i_size, PAGE_SIZE) - 1); + + iput(sctx->cur_inode); + sctx->cur_inode = NULL; +} + +static int changed_inode(struct send_ctx *sctx, + enum btrfs_compare_tree_result result) +{ + int ret; + struct btrfs_key *key = sctx->cmp_key; + struct btrfs_inode_item *left_ii = NULL; + struct btrfs_inode_item *right_ii = NULL; + u64 left_gen = 0; + u64 right_gen = 0; + + close_current_inode(sctx); + + sctx->cur_ino = key->objectid; + sctx->cur_inode_new_gen = false; + sctx->cur_inode_last_extent = (u64)-1; + sctx->cur_inode_next_write_offset = 0; + sctx->ignore_cur_inode = false; + fs_path_reset(&sctx->cur_inode_path); + + /* + * Set send_progress to current inode. This will tell all get_cur_xxx + * functions that the current inode's refs are not updated yet. Later, + * when process_recorded_refs is finished, it is set to cur_ino + 1. + */ + sctx->send_progress = sctx->cur_ino; + + if (result == BTRFS_COMPARE_TREE_NEW || + result == BTRFS_COMPARE_TREE_CHANGED) { + left_ii = btrfs_item_ptr(sctx->left_path->nodes[0], + sctx->left_path->slots[0], + struct btrfs_inode_item); + left_gen = btrfs_inode_generation(sctx->left_path->nodes[0], + left_ii); + } else { + right_ii = btrfs_item_ptr(sctx->right_path->nodes[0], + sctx->right_path->slots[0], + struct btrfs_inode_item); + right_gen = btrfs_inode_generation(sctx->right_path->nodes[0], + right_ii); + } + if (result == BTRFS_COMPARE_TREE_CHANGED) { + right_ii = btrfs_item_ptr(sctx->right_path->nodes[0], + sctx->right_path->slots[0], + struct btrfs_inode_item); + + right_gen = btrfs_inode_generation(sctx->right_path->nodes[0], + right_ii); + + /* + * The cur_ino = root dir case is special here. We can't treat + * the inode as deleted+reused because it would generate a + * stream that tries to delete/mkdir the root dir. + */ + if (left_gen != right_gen && + sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID) + sctx->cur_inode_new_gen = true; + } + + /* + * Normally we do not find inodes with a link count of zero (orphans) + * because the most common case is to create a snapshot and use it + * for a send operation. However other less common use cases involve + * using a subvolume and send it after turning it to RO mode just + * after deleting all hard links of a file while holding an open + * file descriptor against it or turning a RO snapshot into RW mode, + * keep an open file descriptor against a file, delete it and then + * turn the snapshot back to RO mode before using it for a send + * operation. The former is what the receiver operation does. + * Therefore, if we want to send these snapshots soon after they're + * received, we need to handle orphan inodes as well. Moreover, orphans + * can appear not only in the send snapshot but also in the parent + * snapshot. Here are several cases: + * + * Case 1: BTRFS_COMPARE_TREE_NEW + * | send snapshot | action + * -------------------------------- + * nlink | 0 | ignore + * + * Case 2: BTRFS_COMPARE_TREE_DELETED + * | parent snapshot | action + * ---------------------------------- + * nlink | 0 | as usual + * Note: No unlinks will be sent because there're no paths for it. + * + * Case 3: BTRFS_COMPARE_TREE_CHANGED + * | | parent snapshot | send snapshot | action + * ----------------------------------------------------------------------- + * subcase 1 | nlink | 0 | 0 | ignore + * subcase 2 | nlink | >0 | 0 | new_gen(deletion) + * subcase 3 | nlink | 0 | >0 | new_gen(creation) + * + */ + if (result == BTRFS_COMPARE_TREE_NEW) { + if (btrfs_inode_nlink(sctx->left_path->nodes[0], left_ii) == 0) { + sctx->ignore_cur_inode = true; + return 0; + } + sctx->cur_inode_gen = left_gen; + sctx->cur_inode_new = true; + sctx->cur_inode_deleted = false; + sctx->cur_inode_size = btrfs_inode_size( + sctx->left_path->nodes[0], left_ii); + sctx->cur_inode_mode = btrfs_inode_mode( + sctx->left_path->nodes[0], left_ii); + sctx->cur_inode_rdev = btrfs_inode_rdev( + sctx->left_path->nodes[0], left_ii); + if (sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID) + ret = send_create_inode_if_needed(sctx); + } else if (result == BTRFS_COMPARE_TREE_DELETED) { + sctx->cur_inode_gen = right_gen; + sctx->cur_inode_new = false; + sctx->cur_inode_deleted = true; + sctx->cur_inode_size = btrfs_inode_size( + sctx->right_path->nodes[0], right_ii); + sctx->cur_inode_mode = btrfs_inode_mode( + sctx->right_path->nodes[0], right_ii); + } else if (result == BTRFS_COMPARE_TREE_CHANGED) { + u32 new_nlinks, old_nlinks; + + new_nlinks = btrfs_inode_nlink(sctx->left_path->nodes[0], left_ii); + old_nlinks = btrfs_inode_nlink(sctx->right_path->nodes[0], right_ii); + if (new_nlinks == 0 && old_nlinks == 0) { + sctx->ignore_cur_inode = true; + return 0; + } else if (new_nlinks == 0 || old_nlinks == 0) { + sctx->cur_inode_new_gen = 1; + } + /* + * We need to do some special handling in case the inode was + * reported as changed with a changed generation number. This + * means that the original inode was deleted and new inode + * reused the same inum. So we have to treat the old inode as + * deleted and the new one as new. + */ + if (sctx->cur_inode_new_gen) { + /* + * First, process the inode as if it was deleted. + */ + if (old_nlinks > 0) { + sctx->cur_inode_gen = right_gen; + sctx->cur_inode_new = false; + sctx->cur_inode_deleted = true; + sctx->cur_inode_size = btrfs_inode_size( + sctx->right_path->nodes[0], right_ii); + sctx->cur_inode_mode = btrfs_inode_mode( + sctx->right_path->nodes[0], right_ii); + ret = process_all_refs(sctx, + BTRFS_COMPARE_TREE_DELETED); + if (ret < 0) + return ret; + } + + /* + * Now process the inode as if it was new. + */ + if (new_nlinks > 0) { + sctx->cur_inode_gen = left_gen; + sctx->cur_inode_new = true; + sctx->cur_inode_deleted = false; + sctx->cur_inode_size = btrfs_inode_size( + sctx->left_path->nodes[0], + left_ii); + sctx->cur_inode_mode = btrfs_inode_mode( + sctx->left_path->nodes[0], + left_ii); + sctx->cur_inode_rdev = btrfs_inode_rdev( + sctx->left_path->nodes[0], + left_ii); + ret = send_create_inode_if_needed(sctx); + if (ret < 0) + return ret; + + ret = process_all_refs(sctx, BTRFS_COMPARE_TREE_NEW); + if (ret < 0) + return ret; + /* + * Advance send_progress now as we did not get + * into process_recorded_refs_if_needed in the + * new_gen case. + */ + sctx->send_progress = sctx->cur_ino + 1; + + /* + * Now process all extents and xattrs of the + * inode as if they were all new. + */ + ret = process_all_extents(sctx); + if (ret < 0) + return ret; + ret = process_all_new_xattrs(sctx); + if (ret < 0) + return ret; + } + } else { + sctx->cur_inode_gen = left_gen; + sctx->cur_inode_new = false; + sctx->cur_inode_new_gen = false; + sctx->cur_inode_deleted = false; + sctx->cur_inode_size = btrfs_inode_size( + sctx->left_path->nodes[0], left_ii); + sctx->cur_inode_mode = btrfs_inode_mode( + sctx->left_path->nodes[0], left_ii); + } + } + + return 0; +} + +/* + * We have to process new refs before deleted refs, but compare_trees gives us + * the new and deleted refs mixed. To fix this, we record the new/deleted refs + * first and later process them in process_recorded_refs. + * For the cur_inode_new_gen case, we skip recording completely because + * changed_inode did already initiate processing of refs. The reason for this is + * that in this case, compare_tree actually compares the refs of 2 different + * inodes. To fix this, process_all_refs is used in changed_inode to handle all + * refs of the right tree as deleted and all refs of the left tree as new. + */ +static int changed_ref(struct send_ctx *sctx, + enum btrfs_compare_tree_result result) +{ + int ret = 0; + + if (unlikely(sctx->cur_ino != sctx->cmp_key->objectid)) { + inconsistent_snapshot_error(sctx, result, "reference"); + return -EIO; + } + + if (!sctx->cur_inode_new_gen && + sctx->cur_ino != BTRFS_FIRST_FREE_OBJECTID) { + if (result == BTRFS_COMPARE_TREE_NEW) + ret = record_new_ref(sctx); + else if (result == BTRFS_COMPARE_TREE_DELETED) + ret = record_deleted_ref(sctx); + else if (result == BTRFS_COMPARE_TREE_CHANGED) + ret = record_changed_ref(sctx); + } + + return ret; +} + +/* + * Process new/deleted/changed xattrs. We skip processing in the + * cur_inode_new_gen case because changed_inode did already initiate processing + * of xattrs. The reason is the same as in changed_ref + */ +static int changed_xattr(struct send_ctx *sctx, + enum btrfs_compare_tree_result result) +{ + int ret = 0; + + if (unlikely(sctx->cur_ino != sctx->cmp_key->objectid)) { + inconsistent_snapshot_error(sctx, result, "xattr"); + return -EIO; + } + + if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) { + if (result == BTRFS_COMPARE_TREE_NEW) + ret = process_new_xattr(sctx); + else if (result == BTRFS_COMPARE_TREE_DELETED) + ret = process_deleted_xattr(sctx); + else if (result == BTRFS_COMPARE_TREE_CHANGED) + ret = process_changed_xattr(sctx); + } + + return ret; +} + +/* + * Process new/deleted/changed extents. We skip processing in the + * cur_inode_new_gen case because changed_inode did already initiate processing + * of extents. The reason is the same as in changed_ref + */ +static int changed_extent(struct send_ctx *sctx, + enum btrfs_compare_tree_result result) +{ + int ret = 0; + + /* + * We have found an extent item that changed without the inode item + * having changed. This can happen either after relocation (where the + * disk_bytenr of an extent item is replaced at + * relocation.c:replace_file_extents()) or after deduplication into a + * file in both the parent and send snapshots (where an extent item can + * get modified or replaced with a new one). Note that deduplication + * updates the inode item, but it only changes the iversion (sequence + * field in the inode item) of the inode, so if a file is deduplicated + * the same amount of times in both the parent and send snapshots, its + * iversion becomes the same in both snapshots, whence the inode item is + * the same on both snapshots. + */ + if (sctx->cur_ino != sctx->cmp_key->objectid) + return 0; + + if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) { + if (result != BTRFS_COMPARE_TREE_DELETED) + ret = process_extent(sctx, sctx->left_path, + sctx->cmp_key); + } + + return ret; +} + +static int changed_verity(struct send_ctx *sctx, enum btrfs_compare_tree_result result) +{ + if (!sctx->cur_inode_new_gen && !sctx->cur_inode_deleted) { + if (result == BTRFS_COMPARE_TREE_NEW) + sctx->cur_inode_needs_verity = true; + } + return 0; +} + +static int dir_changed(struct send_ctx *sctx, u64 dir) +{ + u64 orig_gen, new_gen; + int ret; + + ret = get_inode_gen(sctx->send_root, dir, &new_gen); + if (ret) + return ret; + + ret = get_inode_gen(sctx->parent_root, dir, &orig_gen); + if (ret) + return ret; + + return (orig_gen != new_gen) ? 1 : 0; +} + +static int compare_refs(struct send_ctx *sctx, struct btrfs_path *path, + struct btrfs_key *key) +{ + struct btrfs_inode_extref *extref; + struct extent_buffer *leaf; + u64 dirid = 0, last_dirid = 0; + unsigned long ptr; + u32 item_size; + u32 cur_offset = 0; + int ref_name_len; + + /* Easy case, just check this one dirid */ + if (key->type == BTRFS_INODE_REF_KEY) { + dirid = key->offset; + + return dir_changed(sctx, dirid); + } + + leaf = path->nodes[0]; + item_size = btrfs_item_size(leaf, path->slots[0]); + ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); + while (cur_offset < item_size) { + int ret; + + extref = (struct btrfs_inode_extref *)(ptr + + cur_offset); + dirid = btrfs_inode_extref_parent(leaf, extref); + ref_name_len = btrfs_inode_extref_name_len(leaf, extref); + cur_offset += ref_name_len + sizeof(*extref); + if (dirid == last_dirid) + continue; + ret = dir_changed(sctx, dirid); + if (ret) + return ret; + last_dirid = dirid; + } + return 0; +} + +/* + * Updates compare related fields in sctx and simply forwards to the actual + * changed_xxx functions. + */ +static int changed_cb(struct btrfs_path *left_path, + struct btrfs_path *right_path, + struct btrfs_key *key, + enum btrfs_compare_tree_result result, + struct send_ctx *sctx) +{ + int ret; + + /* + * We can not hold the commit root semaphore here. This is because in + * the case of sending and receiving to the same filesystem, using a + * pipe, could result in a deadlock: + * + * 1) The task running send blocks on the pipe because it's full; + * + * 2) The task running receive, which is the only consumer of the pipe, + * is waiting for a transaction commit (for example due to a space + * reservation when doing a write or triggering a transaction commit + * when creating a subvolume); + * + * 3) The transaction is waiting to write lock the commit root semaphore, + * but can not acquire it since it's being held at 1). + * + * Down this call chain we write to the pipe through kernel_write(). + * The same type of problem can also happen when sending to a file that + * is stored in the same filesystem - when reserving space for a write + * into the file, we can trigger a transaction commit. + * + * Our caller has supplied us with clones of leaves from the send and + * parent roots, so we're safe here from a concurrent relocation and + * further reallocation of metadata extents while we are here. Below we + * also assert that the leaves are clones. + */ + lockdep_assert_not_held(&sctx->send_root->fs_info->commit_root_sem); + + /* + * We always have a send root, so left_path is never NULL. We will not + * have a leaf when we have reached the end of the send root but have + * not yet reached the end of the parent root. + */ + if (left_path->nodes[0]) + ASSERT(test_bit(EXTENT_BUFFER_UNMAPPED, + &left_path->nodes[0]->bflags)); + /* + * When doing a full send we don't have a parent root, so right_path is + * NULL. When doing an incremental send, we may have reached the end of + * the parent root already, so we don't have a leaf at right_path. + */ + if (right_path && right_path->nodes[0]) + ASSERT(test_bit(EXTENT_BUFFER_UNMAPPED, + &right_path->nodes[0]->bflags)); + + if (result == BTRFS_COMPARE_TREE_SAME) { + if (key->type == BTRFS_INODE_REF_KEY || + key->type == BTRFS_INODE_EXTREF_KEY) { + ret = compare_refs(sctx, left_path, key); + if (!ret) + return 0; + if (ret < 0) + return ret; + } else if (key->type == BTRFS_EXTENT_DATA_KEY) { + return maybe_send_hole(sctx, left_path, key); + } else { + return 0; + } + result = BTRFS_COMPARE_TREE_CHANGED; + } + + sctx->left_path = left_path; + sctx->right_path = right_path; + sctx->cmp_key = key; + + ret = finish_inode_if_needed(sctx, false); + if (ret < 0) + return ret; + + /* Ignore non-FS objects */ + if (key->objectid == BTRFS_FREE_INO_OBJECTID || + key->objectid == BTRFS_FREE_SPACE_OBJECTID) + return 0; + + if (key->type == BTRFS_INODE_ITEM_KEY) { + ret = changed_inode(sctx, result); + } else if (!sctx->ignore_cur_inode) { + if (key->type == BTRFS_INODE_REF_KEY || + key->type == BTRFS_INODE_EXTREF_KEY) + ret = changed_ref(sctx, result); + else if (key->type == BTRFS_XATTR_ITEM_KEY) + ret = changed_xattr(sctx, result); + else if (key->type == BTRFS_EXTENT_DATA_KEY) + ret = changed_extent(sctx, result); + else if (key->type == BTRFS_VERITY_DESC_ITEM_KEY && + key->offset == 0) + ret = changed_verity(sctx, result); + } + + return ret; +} + +static int search_key_again(const struct send_ctx *sctx, + struct btrfs_root *root, + struct btrfs_path *path, + const struct btrfs_key *key) +{ + int ret; + + if (!path->need_commit_sem) + lockdep_assert_held_read(&root->fs_info->commit_root_sem); + + /* + * Roots used for send operations are readonly and no one can add, + * update or remove keys from them, so we should be able to find our + * key again. The only exception is deduplication, which can operate on + * readonly roots and add, update or remove keys to/from them - but at + * the moment we don't allow it to run in parallel with send. + */ + ret = btrfs_search_slot(NULL, root, key, path, 0, 0); + ASSERT(ret <= 0); + if (unlikely(ret > 0)) { + btrfs_print_tree(path->nodes[path->lowest_level], false); + btrfs_err(root->fs_info, +"send: key " BTRFS_KEY_FMT" not found in %s root %llu, lowest_level %d, slot %d", + BTRFS_KEY_FMT_VALUE(key), + (root == sctx->parent_root ? "parent" : "send"), + btrfs_root_id(root), path->lowest_level, + path->slots[path->lowest_level]); + return -EUCLEAN; + } + + return ret; +} + +static int full_send_tree(struct send_ctx *sctx) +{ + int ret; + struct btrfs_root *send_root = sctx->send_root; + struct btrfs_key key; + struct btrfs_fs_info *fs_info = send_root->fs_info; + BTRFS_PATH_AUTO_FREE(path); + + path = alloc_path_for_send(); + if (!path) + return -ENOMEM; + path->reada = READA_FORWARD_ALWAYS; + + key.objectid = BTRFS_FIRST_FREE_OBJECTID; + key.type = BTRFS_INODE_ITEM_KEY; + key.offset = 0; + + down_read(&fs_info->commit_root_sem); + sctx->last_reloc_trans = fs_info->last_reloc_trans; + up_read(&fs_info->commit_root_sem); + + ret = btrfs_search_slot_for_read(send_root, &key, path, 1, 0); + if (ret < 0) + return ret; + if (ret) + goto out_finish; + + while (1) { + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + + ret = changed_cb(path, NULL, &key, + BTRFS_COMPARE_TREE_NEW, sctx); + if (ret < 0) + return ret; + + down_read(&fs_info->commit_root_sem); + if (fs_info->last_reloc_trans > sctx->last_reloc_trans) { + sctx->last_reloc_trans = fs_info->last_reloc_trans; + up_read(&fs_info->commit_root_sem); + /* + * A transaction used for relocating a block group was + * committed or is about to finish its commit. Release + * our path (leaf) and restart the search, so that we + * avoid operating on any file extent items that are + * stale, with a disk_bytenr that reflects a pre + * relocation value. This way we avoid as much as + * possible to fallback to regular writes when checking + * if we can clone file ranges. + */ + btrfs_release_path(path); + ret = search_key_again(sctx, send_root, path, &key); + if (ret < 0) + return ret; + } else { + up_read(&fs_info->commit_root_sem); + } + + ret = btrfs_next_item(send_root, path); + if (ret < 0) + return ret; + if (ret) { + ret = 0; + break; + } + } + +out_finish: + return finish_inode_if_needed(sctx, true); +} + +static int replace_node_with_clone(struct btrfs_path *path, int level) +{ + struct extent_buffer *clone; + + clone = btrfs_clone_extent_buffer(path->nodes[level]); + if (!clone) + return -ENOMEM; + + free_extent_buffer(path->nodes[level]); + path->nodes[level] = clone; + + return 0; +} + +static int tree_move_down(struct btrfs_path *path, int *level, u64 reada_min_gen) +{ + struct extent_buffer *eb; + struct extent_buffer *parent = path->nodes[*level]; + int slot = path->slots[*level]; + const int nritems = btrfs_header_nritems(parent); + u64 reada_max; + u64 reada_done = 0; + + lockdep_assert_held_read(&parent->fs_info->commit_root_sem); + ASSERT(*level != 0); + + eb = btrfs_read_node_slot(parent, slot); + if (IS_ERR(eb)) + return PTR_ERR(eb); + + /* + * Trigger readahead for the next leaves we will process, so that it is + * very likely that when we need them they are already in memory and we + * will not block on disk IO. For nodes we only do readahead for one, + * since the time window between processing nodes is typically larger. + */ + reada_max = (*level == 1 ? SZ_128K : eb->fs_info->nodesize); + + for (slot++; slot < nritems && reada_done < reada_max; slot++) { + if (btrfs_node_ptr_generation(parent, slot) > reada_min_gen) { + btrfs_readahead_node_child(parent, slot); + reada_done += eb->fs_info->nodesize; + } + } + + path->nodes[*level - 1] = eb; + path->slots[*level - 1] = 0; + (*level)--; + + if (*level == 0) + return replace_node_with_clone(path, 0); + + return 0; +} + +static int tree_move_next_or_upnext(struct btrfs_path *path, + int *level, int root_level) +{ + int ret = 0; + int nritems; + nritems = btrfs_header_nritems(path->nodes[*level]); + + path->slots[*level]++; + + while (path->slots[*level] >= nritems) { + if (*level == root_level) { + path->slots[*level] = nritems - 1; + return -1; + } + + /* move upnext */ + path->slots[*level] = 0; + free_extent_buffer(path->nodes[*level]); + path->nodes[*level] = NULL; + (*level)++; + path->slots[*level]++; + + nritems = btrfs_header_nritems(path->nodes[*level]); + ret = 1; + } + return ret; +} + +/* + * Returns 1 if it had to move up and next. 0 is returned if it moved only next + * or down. + */ +static int tree_advance(struct btrfs_path *path, + int *level, int root_level, + int allow_down, + struct btrfs_key *key, + u64 reada_min_gen) +{ + int ret; + + if (*level == 0 || !allow_down) { + ret = tree_move_next_or_upnext(path, level, root_level); + } else { + ret = tree_move_down(path, level, reada_min_gen); + } + + /* + * Even if we have reached the end of a tree, ret is -1, update the key + * anyway, so that in case we need to restart due to a block group + * relocation, we can assert that the last key of the root node still + * exists in the tree. + */ + if (*level == 0) + btrfs_item_key_to_cpu(path->nodes[*level], key, + path->slots[*level]); + else + btrfs_node_key_to_cpu(path->nodes[*level], key, + path->slots[*level]); + + return ret; +} + +static int tree_compare_item(struct btrfs_path *left_path, + struct btrfs_path *right_path, + char *tmp_buf) +{ + int cmp; + int len1, len2; + unsigned long off1, off2; + + len1 = btrfs_item_size(left_path->nodes[0], left_path->slots[0]); + len2 = btrfs_item_size(right_path->nodes[0], right_path->slots[0]); + if (len1 != len2) + return 1; + + off1 = btrfs_item_ptr_offset(left_path->nodes[0], left_path->slots[0]); + off2 = btrfs_item_ptr_offset(right_path->nodes[0], + right_path->slots[0]); + + read_extent_buffer(left_path->nodes[0], tmp_buf, off1, len1); + + cmp = memcmp_extent_buffer(right_path->nodes[0], tmp_buf, off2, len1); + if (cmp) + return 1; + return 0; +} + +/* + * A transaction used for relocating a block group was committed or is about to + * finish its commit. Release our paths and restart the search, so that we are + * not using stale extent buffers: + * + * 1) For levels > 0, we are only holding references of extent buffers, without + * any locks on them, which does not prevent them from having been relocated + * and reallocated after the last time we released the commit root semaphore. + * The exception are the root nodes, for which we always have a clone, see + * the comment at btrfs_compare_trees(); + * + * 2) For leaves, level 0, we are holding copies (clones) of extent buffers, so + * we are safe from the concurrent relocation and reallocation. However they + * can have file extent items with a pre relocation disk_bytenr value, so we + * restart the start from the current commit roots and clone the new leaves so + * that we get the post relocation disk_bytenr values. Not doing so, could + * make us clone the wrong data in case there are new extents using the old + * disk_bytenr that happen to be shared. + */ +static int restart_after_relocation(struct btrfs_path *left_path, + struct btrfs_path *right_path, + const struct btrfs_key *left_key, + const struct btrfs_key *right_key, + int left_level, + int right_level, + const struct send_ctx *sctx) +{ + int root_level; + int ret; + + lockdep_assert_held_read(&sctx->send_root->fs_info->commit_root_sem); + + btrfs_release_path(left_path); + btrfs_release_path(right_path); + + /* + * Since keys can not be added or removed to/from our roots because they + * are readonly and we do not allow deduplication to run in parallel + * (which can add, remove or change keys), the layout of the trees should + * not change. + */ + left_path->lowest_level = left_level; + ret = search_key_again(sctx, sctx->send_root, left_path, left_key); + if (ret < 0) + return ret; + + right_path->lowest_level = right_level; + ret = search_key_again(sctx, sctx->parent_root, right_path, right_key); + if (ret < 0) + return ret; + + /* + * If the lowest level nodes are leaves, clone them so that they can be + * safely used by changed_cb() while not under the protection of the + * commit root semaphore, even if relocation and reallocation happens in + * parallel. + */ + if (left_level == 0) { + ret = replace_node_with_clone(left_path, 0); + if (ret < 0) + return ret; + } + + if (right_level == 0) { + ret = replace_node_with_clone(right_path, 0); + if (ret < 0) + return ret; + } + + /* + * Now clone the root nodes (unless they happen to be the leaves we have + * already cloned). This is to protect against concurrent snapshotting of + * the send and parent roots (see the comment at btrfs_compare_trees()). + */ + root_level = btrfs_header_level(sctx->send_root->commit_root); + if (root_level > 0) { + ret = replace_node_with_clone(left_path, root_level); + if (ret < 0) + return ret; + } + + root_level = btrfs_header_level(sctx->parent_root->commit_root); + if (root_level > 0) { + ret = replace_node_with_clone(right_path, root_level); + if (ret < 0) + return ret; + } + + return 0; +} + +/* + * This function compares two trees and calls the provided callback for + * every changed/new/deleted item it finds. + * If shared tree blocks are encountered, whole subtrees are skipped, making + * the compare pretty fast on snapshotted subvolumes. + * + * This currently works on commit roots only. As commit roots are read only, + * we don't do any locking. The commit roots are protected with transactions. + * Transactions are ended and rejoined when a commit is tried in between. + * + * This function checks for modifications done to the trees while comparing. + * If it detects a change, it aborts immediately. + */ +static int btrfs_compare_trees(struct btrfs_root *left_root, + struct btrfs_root *right_root, struct send_ctx *sctx) +{ + struct btrfs_fs_info *fs_info = left_root->fs_info; + int ret; + int cmp; + BTRFS_PATH_AUTO_FREE(left_path); + BTRFS_PATH_AUTO_FREE(right_path); + struct btrfs_key left_key; + struct btrfs_key right_key; + char *tmp_buf = NULL; + int left_root_level; + int right_root_level; + int left_level; + int right_level; + int left_end_reached = 0; + int right_end_reached = 0; + int advance_left = 0; + int advance_right = 0; + u64 left_blockptr; + u64 right_blockptr; + u64 left_gen; + u64 right_gen; + u64 reada_min_gen; + + left_path = btrfs_alloc_path(); + if (!left_path) { + ret = -ENOMEM; + goto out; + } + right_path = btrfs_alloc_path(); + if (!right_path) { + ret = -ENOMEM; + goto out; + } + + tmp_buf = kvmalloc(fs_info->nodesize, GFP_KERNEL); + if (!tmp_buf) { + ret = -ENOMEM; + goto out; + } + + left_path->search_commit_root = true; + left_path->skip_locking = true; + right_path->search_commit_root = true; + right_path->skip_locking = true; + + /* + * Strategy: Go to the first items of both trees. Then do + * + * If both trees are at level 0 + * Compare keys of current items + * If left < right treat left item as new, advance left tree + * and repeat + * If left > right treat right item as deleted, advance right tree + * and repeat + * If left == right do deep compare of items, treat as changed if + * needed, advance both trees and repeat + * If both trees are at the same level but not at level 0 + * Compare keys of current nodes/leafs + * If left < right advance left tree and repeat + * If left > right advance right tree and repeat + * If left == right compare blockptrs of the next nodes/leafs + * If they match advance both trees but stay at the same level + * and repeat + * If they don't match advance both trees while allowing to go + * deeper and repeat + * If tree levels are different + * Advance the tree that needs it and repeat + * + * Advancing a tree means: + * If we are at level 0, try to go to the next slot. If that's not + * possible, go one level up and repeat. Stop when we found a level + * where we could go to the next slot. We may at this point be on a + * node or a leaf. + * + * If we are not at level 0 and not on shared tree blocks, go one + * level deeper. + * + * If we are not at level 0 and on shared tree blocks, go one slot to + * the right if possible or go up and right. + */ + + down_read(&fs_info->commit_root_sem); + left_level = btrfs_header_level(left_root->commit_root); + left_root_level = left_level; + /* + * We clone the root node of the send and parent roots to prevent races + * with snapshot creation of these roots. Snapshot creation COWs the + * root node of a tree, so after the transaction is committed the old + * extent can be reallocated while this send operation is still ongoing. + * So we clone them, under the commit root semaphore, to be race free. + */ + left_path->nodes[left_level] = + btrfs_clone_extent_buffer(left_root->commit_root); + if (!left_path->nodes[left_level]) { + ret = -ENOMEM; + goto out_unlock; + } + + right_level = btrfs_header_level(right_root->commit_root); + right_root_level = right_level; + right_path->nodes[right_level] = + btrfs_clone_extent_buffer(right_root->commit_root); + if (!right_path->nodes[right_level]) { + ret = -ENOMEM; + goto out_unlock; + } + /* + * Our right root is the parent root, while the left root is the "send" + * root. We know that all new nodes/leaves in the left root must have + * a generation greater than the right root's generation, so we trigger + * readahead for those nodes and leaves of the left root, as we know we + * will need to read them at some point. + */ + reada_min_gen = btrfs_header_generation(right_root->commit_root); + + if (left_level == 0) + btrfs_item_key_to_cpu(left_path->nodes[left_level], + &left_key, left_path->slots[left_level]); + else + btrfs_node_key_to_cpu(left_path->nodes[left_level], + &left_key, left_path->slots[left_level]); + if (right_level == 0) + btrfs_item_key_to_cpu(right_path->nodes[right_level], + &right_key, right_path->slots[right_level]); + else + btrfs_node_key_to_cpu(right_path->nodes[right_level], + &right_key, right_path->slots[right_level]); + + sctx->last_reloc_trans = fs_info->last_reloc_trans; + + while (1) { + if (need_resched() || + rwsem_is_contended(&fs_info->commit_root_sem)) { + up_read(&fs_info->commit_root_sem); + cond_resched(); + down_read(&fs_info->commit_root_sem); + } + + if (fs_info->last_reloc_trans > sctx->last_reloc_trans) { + ret = restart_after_relocation(left_path, right_path, + &left_key, &right_key, + left_level, right_level, + sctx); + if (ret < 0) + goto out_unlock; + sctx->last_reloc_trans = fs_info->last_reloc_trans; + } + + if (advance_left && !left_end_reached) { + ret = tree_advance(left_path, &left_level, + left_root_level, + advance_left != ADVANCE_ONLY_NEXT, + &left_key, reada_min_gen); + if (ret == -1) + left_end_reached = ADVANCE; + else if (ret < 0) + goto out_unlock; + advance_left = 0; + } + if (advance_right && !right_end_reached) { + ret = tree_advance(right_path, &right_level, + right_root_level, + advance_right != ADVANCE_ONLY_NEXT, + &right_key, reada_min_gen); + if (ret == -1) + right_end_reached = ADVANCE; + else if (ret < 0) + goto out_unlock; + advance_right = 0; + } + + if (left_end_reached && right_end_reached) { + ret = 0; + goto out_unlock; + } else if (left_end_reached) { + if (right_level == 0) { + up_read(&fs_info->commit_root_sem); + ret = changed_cb(left_path, right_path, + &right_key, + BTRFS_COMPARE_TREE_DELETED, + sctx); + if (ret < 0) + goto out; + down_read(&fs_info->commit_root_sem); + } + advance_right = ADVANCE; + continue; + } else if (right_end_reached) { + if (left_level == 0) { + up_read(&fs_info->commit_root_sem); + ret = changed_cb(left_path, right_path, + &left_key, + BTRFS_COMPARE_TREE_NEW, + sctx); + if (ret < 0) + goto out; + down_read(&fs_info->commit_root_sem); + } + advance_left = ADVANCE; + continue; + } + + if (left_level == 0 && right_level == 0) { + up_read(&fs_info->commit_root_sem); + cmp = btrfs_comp_cpu_keys(&left_key, &right_key); + if (cmp < 0) { + ret = changed_cb(left_path, right_path, + &left_key, + BTRFS_COMPARE_TREE_NEW, + sctx); + advance_left = ADVANCE; + } else if (cmp > 0) { + ret = changed_cb(left_path, right_path, + &right_key, + BTRFS_COMPARE_TREE_DELETED, + sctx); + advance_right = ADVANCE; + } else { + enum btrfs_compare_tree_result result; + + WARN_ON(!extent_buffer_uptodate(left_path->nodes[0])); + ret = tree_compare_item(left_path, right_path, + tmp_buf); + if (ret) + result = BTRFS_COMPARE_TREE_CHANGED; + else + result = BTRFS_COMPARE_TREE_SAME; + ret = changed_cb(left_path, right_path, + &left_key, result, sctx); + advance_left = ADVANCE; + advance_right = ADVANCE; + } + + if (ret < 0) + goto out; + down_read(&fs_info->commit_root_sem); + } else if (left_level == right_level) { + cmp = btrfs_comp_cpu_keys(&left_key, &right_key); + if (cmp < 0) { + advance_left = ADVANCE; + } else if (cmp > 0) { + advance_right = ADVANCE; + } else { + left_blockptr = btrfs_node_blockptr( + left_path->nodes[left_level], + left_path->slots[left_level]); + right_blockptr = btrfs_node_blockptr( + right_path->nodes[right_level], + right_path->slots[right_level]); + left_gen = btrfs_node_ptr_generation( + left_path->nodes[left_level], + left_path->slots[left_level]); + right_gen = btrfs_node_ptr_generation( + right_path->nodes[right_level], + right_path->slots[right_level]); + if (left_blockptr == right_blockptr && + left_gen == right_gen) { + /* + * As we're on a shared block, don't + * allow to go deeper. + */ + advance_left = ADVANCE_ONLY_NEXT; + advance_right = ADVANCE_ONLY_NEXT; + } else { + advance_left = ADVANCE; + advance_right = ADVANCE; + } + } + } else if (left_level < right_level) { + advance_right = ADVANCE; + } else { + advance_left = ADVANCE; + } + } + +out_unlock: + up_read(&fs_info->commit_root_sem); +out: + kvfree(tmp_buf); + return ret; +} + +static int send_subvol(struct send_ctx *sctx) +{ + int ret; + + if (!(sctx->flags & BTRFS_SEND_FLAG_OMIT_STREAM_HEADER)) { + ret = send_header(sctx); + if (ret < 0) + goto out; + } + + ret = send_subvol_begin(sctx); + if (ret < 0) + goto out; + + if (sctx->parent_root) { + ret = btrfs_compare_trees(sctx->send_root, sctx->parent_root, sctx); + if (ret < 0) + goto out; + ret = finish_inode_if_needed(sctx, true); + if (ret < 0) + goto out; + } else { + ret = full_send_tree(sctx); + if (ret < 0) + goto out; + } + +out: + free_recorded_refs(sctx); + return ret; +} + +/* + * If orphan cleanup did remove any orphans from a root, it means the tree + * was modified and therefore the commit root is not the same as the current + * root anymore. This is a problem, because send uses the commit root and + * therefore can see inode items that don't exist in the current root anymore, + * and for example make calls to btrfs_iget, which will do tree lookups based + * on the current root and not on the commit root. Those lookups will fail, + * returning a -ESTALE error, and making send fail with that error. So make + * sure a send does not see any orphans we have just removed, and that it will + * see the same inodes regardless of whether a transaction commit happened + * before it started (meaning that the commit root will be the same as the + * current root) or not. + */ +static int ensure_commit_roots_uptodate(struct send_ctx *sctx) +{ + struct btrfs_root *root = sctx->parent_root; + + if (root && root->node != root->commit_root) + return btrfs_commit_current_transaction(root); + + for (int i = 0; i < sctx->clone_roots_cnt; i++) { + root = sctx->clone_roots[i].root; + if (root->node != root->commit_root) + return btrfs_commit_current_transaction(root); + } + + return 0; +} + +/* + * Make sure any existing delalloc is flushed for any root used by a send + * operation so that we do not miss any data and we do not race with writeback + * finishing and changing a tree while send is using the tree. This could + * happen if a subvolume is in RW mode, has delalloc, is turned to RO mode and + * a send operation then uses the subvolume. + * After flushing delalloc ensure_commit_roots_uptodate() must be called. + */ +static int flush_delalloc_roots(struct send_ctx *sctx) +{ + struct btrfs_root *root = sctx->parent_root; + int ret; + int i; + + if (root) { + ret = btrfs_start_delalloc_snapshot(root, false); + if (ret) + return ret; + btrfs_wait_ordered_extents(root, U64_MAX, NULL); + } + + for (i = 0; i < sctx->clone_roots_cnt; i++) { + root = sctx->clone_roots[i].root; + ret = btrfs_start_delalloc_snapshot(root, false); + if (ret) + return ret; + btrfs_wait_ordered_extents(root, U64_MAX, NULL); + } + + return 0; +} + +static void btrfs_root_dec_send_in_progress(struct btrfs_root* root) +{ + spin_lock(&root->root_item_lock); + root->send_in_progress--; + /* + * Not much left to do, we don't know why it's unbalanced and + * can't blindly reset it to 0. + */ + if (root->send_in_progress < 0) + btrfs_err(root->fs_info, + "send_in_progress unbalanced %d root %llu", + root->send_in_progress, btrfs_root_id(root)); + spin_unlock(&root->root_item_lock); +} + +static void dedupe_in_progress_warn(const struct btrfs_root *root) +{ + btrfs_warn_rl(root->fs_info, +"cannot use root %llu for send while deduplications on it are in progress (%d in progress)", + btrfs_root_id(root), root->dedupe_in_progress); +} + +long btrfs_ioctl_send(struct btrfs_root *send_root, const struct btrfs_ioctl_send_args *arg) +{ + int ret = 0; + struct btrfs_fs_info *fs_info = send_root->fs_info; + struct btrfs_root *clone_root; + struct send_ctx *sctx = NULL; + u32 i; + u64 *clone_sources_tmp = NULL; + int clone_sources_to_rollback = 0; + size_t alloc_size; + bool sort_clone_roots = false; + struct btrfs_lru_cache_entry *entry; + struct btrfs_lru_cache_entry *tmp; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + /* + * The subvolume must remain read-only during send, protect against + * making it RW. This also protects against deletion. + */ + spin_lock(&send_root->root_item_lock); + /* + * Unlikely but possible, if the subvolume is marked for deletion but + * is slow to remove the directory entry, send can still be started. + */ + if (btrfs_root_dead(send_root)) { + spin_unlock(&send_root->root_item_lock); + return -EPERM; + } + /* Userspace tools do the checks and warn the user if it's not RO. */ + if (!btrfs_root_readonly(send_root)) { + spin_unlock(&send_root->root_item_lock); + return -EPERM; + } + if (send_root->dedupe_in_progress) { + dedupe_in_progress_warn(send_root); + spin_unlock(&send_root->root_item_lock); + return -EAGAIN; + } + send_root->send_in_progress++; + spin_unlock(&send_root->root_item_lock); + + /* + * Check that we don't overflow at later allocations, we request + * clone_sources_count + 1 items, and compare to unsigned long inside + * access_ok. Also set an upper limit for allocation size so this can't + * easily exhaust memory. Max number of clone sources is about 200K. + */ + if (arg->clone_sources_count > SZ_8M / sizeof(struct clone_root)) { + ret = -EINVAL; + goto out; + } + + if (arg->flags & ~BTRFS_SEND_FLAG_MASK) { + ret = -EOPNOTSUPP; + goto out; + } + + sctx = kzalloc_obj(struct send_ctx); + if (!sctx) { + ret = -ENOMEM; + goto out; + } + + init_path(&sctx->cur_inode_path); + INIT_LIST_HEAD(&sctx->new_refs); + INIT_LIST_HEAD(&sctx->deleted_refs); + + btrfs_lru_cache_init(&sctx->name_cache, SEND_MAX_NAME_CACHE_SIZE); + btrfs_lru_cache_init(&sctx->backref_cache, SEND_MAX_BACKREF_CACHE_SIZE); + btrfs_lru_cache_init(&sctx->dir_created_cache, + SEND_MAX_DIR_CREATED_CACHE_SIZE); + /* + * This cache is periodically trimmed to a fixed size elsewhere, see + * cache_dir_utimes() and trim_dir_utimes_cache(). + */ + btrfs_lru_cache_init(&sctx->dir_utimes_cache, 0); + + sctx->pending_dir_moves = RB_ROOT; + sctx->waiting_dir_moves = RB_ROOT; + sctx->orphan_dirs = RB_ROOT; + sctx->rbtree_new_refs = RB_ROOT; + sctx->rbtree_deleted_refs = RB_ROOT; + + sctx->flags = arg->flags; + + if (arg->flags & BTRFS_SEND_FLAG_VERSION) { + if (arg->version > BTRFS_SEND_STREAM_VERSION) { + ret = -EPROTO; + goto out; + } + /* Zero means "use the highest version" */ + sctx->proto = arg->version ?: BTRFS_SEND_STREAM_VERSION; + } else { + sctx->proto = 1; + } + if ((arg->flags & BTRFS_SEND_FLAG_COMPRESSED) && sctx->proto < 2) { + ret = -EINVAL; + goto out; + } + + sctx->send_filp = fget(arg->send_fd); + if (!sctx->send_filp || !(sctx->send_filp->f_mode & FMODE_WRITE)) { + ret = -EBADF; + goto out; + } + + sctx->send_root = send_root; + sctx->clone_roots_cnt = arg->clone_sources_count; + + if (sctx->proto >= 2) { + u32 send_buf_num_pages; + + sctx->send_max_size = BTRFS_SEND_BUF_SIZE_V2; + sctx->send_buf = vmalloc(sctx->send_max_size); + if (!sctx->send_buf) { + ret = -ENOMEM; + goto out; + } + send_buf_num_pages = sctx->send_max_size >> PAGE_SHIFT; + sctx->send_buf_pages = kzalloc_objs(*sctx->send_buf_pages, + send_buf_num_pages); + if (!sctx->send_buf_pages) { + ret = -ENOMEM; + goto out; + } + for (i = 0; i < send_buf_num_pages; i++) { + sctx->send_buf_pages[i] = + vmalloc_to_page(sctx->send_buf + (i << PAGE_SHIFT)); + } + } else { + sctx->send_max_size = BTRFS_SEND_BUF_SIZE_V1; + sctx->send_buf = kvmalloc(sctx->send_max_size, GFP_KERNEL); + } + if (!sctx->send_buf) { + ret = -ENOMEM; + goto out; + } + + sctx->clone_roots = kvzalloc_objs(*sctx->clone_roots, + arg->clone_sources_count + 1); + if (!sctx->clone_roots) { + ret = -ENOMEM; + goto out; + } + + alloc_size = array_size(sizeof(*arg->clone_sources), + arg->clone_sources_count); + + if (arg->clone_sources_count) { + clone_sources_tmp = kvmalloc(alloc_size, GFP_KERNEL); + if (!clone_sources_tmp) { + ret = -ENOMEM; + goto out; + } + + ret = copy_from_user(clone_sources_tmp, arg->clone_sources, + alloc_size); + if (ret) { + ret = -EFAULT; + goto out; + } + + for (i = 0; i < arg->clone_sources_count; i++) { + clone_root = btrfs_get_fs_root(fs_info, + clone_sources_tmp[i], true); + if (IS_ERR(clone_root)) { + ret = PTR_ERR(clone_root); + goto out; + } + spin_lock(&clone_root->root_item_lock); + if (!btrfs_root_readonly(clone_root) || + btrfs_root_dead(clone_root)) { + spin_unlock(&clone_root->root_item_lock); + btrfs_put_root(clone_root); + ret = -EPERM; + goto out; + } + if (clone_root->dedupe_in_progress) { + dedupe_in_progress_warn(clone_root); + spin_unlock(&clone_root->root_item_lock); + btrfs_put_root(clone_root); + ret = -EAGAIN; + goto out; + } + clone_root->send_in_progress++; + spin_unlock(&clone_root->root_item_lock); + + sctx->clone_roots[i].root = clone_root; + clone_sources_to_rollback = i + 1; + } + kvfree(clone_sources_tmp); + clone_sources_tmp = NULL; + } + + if (arg->parent_root) { + sctx->parent_root = btrfs_get_fs_root(fs_info, arg->parent_root, + true); + if (IS_ERR(sctx->parent_root)) { + ret = PTR_ERR(sctx->parent_root); + goto out; + } + + spin_lock(&sctx->parent_root->root_item_lock); + sctx->parent_root->send_in_progress++; + if (!btrfs_root_readonly(sctx->parent_root) || + btrfs_root_dead(sctx->parent_root)) { + spin_unlock(&sctx->parent_root->root_item_lock); + ret = -EPERM; + goto out; + } + if (sctx->parent_root->dedupe_in_progress) { + dedupe_in_progress_warn(sctx->parent_root); + spin_unlock(&sctx->parent_root->root_item_lock); + ret = -EAGAIN; + goto out; + } + spin_unlock(&sctx->parent_root->root_item_lock); + } + + /* + * Clones from send_root are allowed, but only if the clone source + * is behind the current send position. This is checked while searching + * for possible clone sources. + */ + sctx->clone_roots[sctx->clone_roots_cnt++].root = + btrfs_grab_root(sctx->send_root); + + /* We do a bsearch later */ + sort(sctx->clone_roots, sctx->clone_roots_cnt, + sizeof(*sctx->clone_roots), __clone_root_cmp_sort, + NULL); + sort_clone_roots = true; + + ret = flush_delalloc_roots(sctx); + if (ret) + goto out; + + ret = ensure_commit_roots_uptodate(sctx); + if (ret) + goto out; + + ret = send_subvol(sctx); + if (ret < 0) + goto out; + + btrfs_lru_cache_for_each_entry_safe(&sctx->dir_utimes_cache, entry, tmp) { + ret = send_utimes(sctx, entry->key, entry->gen); + if (ret < 0) + goto out; + btrfs_lru_cache_remove(&sctx->dir_utimes_cache, entry); + } + + if (!(sctx->flags & BTRFS_SEND_FLAG_OMIT_END_CMD)) { + ret = begin_cmd(sctx, BTRFS_SEND_C_END); + if (ret < 0) + goto out; + ret = send_cmd(sctx); + if (ret < 0) + goto out; + } + +out: + WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->pending_dir_moves)); + while (sctx && !RB_EMPTY_ROOT(&sctx->pending_dir_moves)) { + struct rb_node *n; + struct pending_dir_move *pm; + + n = rb_first(&sctx->pending_dir_moves); + pm = rb_entry(n, struct pending_dir_move, node); + while (!list_empty(&pm->list)) { + struct pending_dir_move *pm2; + + pm2 = list_first_entry(&pm->list, + struct pending_dir_move, list); + free_pending_move(sctx, pm2); + } + free_pending_move(sctx, pm); + } + + WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->waiting_dir_moves)); + while (sctx && !RB_EMPTY_ROOT(&sctx->waiting_dir_moves)) { + struct rb_node *n; + struct waiting_dir_move *dm; + + n = rb_first(&sctx->waiting_dir_moves); + dm = rb_entry(n, struct waiting_dir_move, node); + rb_erase(&dm->node, &sctx->waiting_dir_moves); + kfree(dm); + } + + WARN_ON(sctx && !ret && !RB_EMPTY_ROOT(&sctx->orphan_dirs)); + while (sctx && !RB_EMPTY_ROOT(&sctx->orphan_dirs)) { + struct rb_node *n; + struct orphan_dir_info *odi; + + n = rb_first(&sctx->orphan_dirs); + odi = rb_entry(n, struct orphan_dir_info, node); + free_orphan_dir_info(sctx, odi); + } + + if (sort_clone_roots) { + for (i = 0; sctx && i < sctx->clone_roots_cnt; i++) { + btrfs_root_dec_send_in_progress( + sctx->clone_roots[i].root); + btrfs_put_root(sctx->clone_roots[i].root); + } + } else { + for (i = 0; sctx && i < clone_sources_to_rollback; i++) { + btrfs_root_dec_send_in_progress( + sctx->clone_roots[i].root); + btrfs_put_root(sctx->clone_roots[i].root); + } + + btrfs_root_dec_send_in_progress(send_root); + } + if (sctx && !IS_ERR_OR_NULL(sctx->parent_root)) { + btrfs_root_dec_send_in_progress(sctx->parent_root); + btrfs_put_root(sctx->parent_root); + } + + kvfree(clone_sources_tmp); + + if (sctx) { + if (sctx->send_filp) + fput(sctx->send_filp); + + kvfree(sctx->clone_roots); + kfree(sctx->send_buf_pages); + kvfree(sctx->send_buf); + kvfree(sctx->verity_descriptor); + + close_current_inode(sctx); + + btrfs_lru_cache_clear(&sctx->name_cache); + btrfs_lru_cache_clear(&sctx->backref_cache); + btrfs_lru_cache_clear(&sctx->dir_created_cache); + btrfs_lru_cache_clear(&sctx->dir_utimes_cache); + + if (sctx->cur_inode_path.buf != sctx->cur_inode_path.inline_buf) + kfree(sctx->cur_inode_path.buf); + + kfree(sctx); + } + + return ret; +} diff --git a/fs/btrfs/send.h b/fs/btrfs/send.h new file mode 100644 index 000000000..652bb28f6 --- /dev/null +++ b/fs/btrfs/send.h @@ -0,0 +1,187 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2012 Alexander Block. All rights reserved. + * Copyright (C) 2012 STRATO. All rights reserved. + */ + +#ifndef BTRFS_SEND_H +#define BTRFS_SEND_H + +#include +#include +#include + +struct btrfs_root; +struct btrfs_ioctl_send_args; + +#define BTRFS_SEND_STREAM_MAGIC "btrfs-stream" +/* Conditional support for the upcoming protocol version. */ +#ifdef CONFIG_BTRFS_EXPERIMENTAL +#define BTRFS_SEND_STREAM_VERSION 3 +#else +#define BTRFS_SEND_STREAM_VERSION 2 +#endif + +/* + * In send stream v1, no command is larger than 64K. In send stream v2, no + * limit should be assumed, the buffer size is set to be a header with + * compressed extent size. + */ +#define BTRFS_SEND_BUF_SIZE_V1 SZ_64K +#define BTRFS_SEND_BUF_SIZE_V2 ALIGN(SZ_16K + BTRFS_MAX_COMPRESSED, PAGE_SIZE) + +enum btrfs_tlv_type { + BTRFS_TLV_U8, + BTRFS_TLV_U16, + BTRFS_TLV_U32, + BTRFS_TLV_U64, + BTRFS_TLV_BINARY, + BTRFS_TLV_STRING, + BTRFS_TLV_UUID, + BTRFS_TLV_TIMESPEC, +}; + +struct btrfs_stream_header { + char magic[sizeof(BTRFS_SEND_STREAM_MAGIC)]; + __le32 version; +} __attribute__ ((__packed__)); + +struct btrfs_cmd_header { + /* len excluding the header */ + __le32 len; + __le16 cmd; + /* crc including the header with zero crc field */ + __le32 crc; +} __attribute__ ((__packed__)); + +struct btrfs_tlv_header { + __le16 tlv_type; + /* len excluding the header */ + __le16 tlv_len; +} __attribute__ ((__packed__)); + +/* commands */ +enum btrfs_send_cmd { + BTRFS_SEND_C_UNSPEC = 0, + + /* Version 1 */ + BTRFS_SEND_C_SUBVOL = 1, + BTRFS_SEND_C_SNAPSHOT = 2, + + BTRFS_SEND_C_MKFILE = 3, + BTRFS_SEND_C_MKDIR = 4, + BTRFS_SEND_C_MKNOD = 5, + BTRFS_SEND_C_MKFIFO = 6, + BTRFS_SEND_C_MKSOCK = 7, + BTRFS_SEND_C_SYMLINK = 8, + + BTRFS_SEND_C_RENAME = 9, + BTRFS_SEND_C_LINK = 10, + BTRFS_SEND_C_UNLINK = 11, + BTRFS_SEND_C_RMDIR = 12, + + BTRFS_SEND_C_SET_XATTR = 13, + BTRFS_SEND_C_REMOVE_XATTR = 14, + + BTRFS_SEND_C_WRITE = 15, + BTRFS_SEND_C_CLONE = 16, + + BTRFS_SEND_C_TRUNCATE = 17, + BTRFS_SEND_C_CHMOD = 18, + BTRFS_SEND_C_CHOWN = 19, + BTRFS_SEND_C_UTIMES = 20, + + BTRFS_SEND_C_END = 21, + BTRFS_SEND_C_UPDATE_EXTENT = 22, + BTRFS_SEND_C_MAX_V1 = 22, + + /* Version 2 */ + BTRFS_SEND_C_FALLOCATE = 23, + BTRFS_SEND_C_FILEATTR = 24, + BTRFS_SEND_C_ENCODED_WRITE = 25, + BTRFS_SEND_C_MAX_V2 = 25, + + /* Version 3 */ + BTRFS_SEND_C_ENABLE_VERITY = 26, + BTRFS_SEND_C_MAX_V3 = 26, + /* End */ + BTRFS_SEND_C_MAX = 26, +}; + +/* attributes in send stream */ +enum { + BTRFS_SEND_A_UNSPEC = 0, + + /* Version 1 */ + BTRFS_SEND_A_UUID = 1, + BTRFS_SEND_A_CTRANSID = 2, + + BTRFS_SEND_A_INO = 3, + BTRFS_SEND_A_SIZE = 4, + BTRFS_SEND_A_MODE = 5, + BTRFS_SEND_A_UID = 6, + BTRFS_SEND_A_GID = 7, + BTRFS_SEND_A_RDEV = 8, + BTRFS_SEND_A_CTIME = 9, + BTRFS_SEND_A_MTIME = 10, + BTRFS_SEND_A_ATIME = 11, + BTRFS_SEND_A_OTIME = 12, + + BTRFS_SEND_A_XATTR_NAME = 13, + BTRFS_SEND_A_XATTR_DATA = 14, + + BTRFS_SEND_A_PATH = 15, + BTRFS_SEND_A_PATH_TO = 16, + BTRFS_SEND_A_PATH_LINK = 17, + + BTRFS_SEND_A_FILE_OFFSET = 18, + /* + * As of send stream v2, this attribute is special: it must be the last + * attribute in a command, its header contains only the type, and its + * length is implicitly the remaining length of the command. + */ + BTRFS_SEND_A_DATA = 19, + + BTRFS_SEND_A_CLONE_UUID = 20, + BTRFS_SEND_A_CLONE_CTRANSID = 21, + BTRFS_SEND_A_CLONE_PATH = 22, + BTRFS_SEND_A_CLONE_OFFSET = 23, + BTRFS_SEND_A_CLONE_LEN = 24, + + BTRFS_SEND_A_MAX_V1 = 24, + + /* Version 2 */ + BTRFS_SEND_A_FALLOCATE_MODE = 25, + + /* + * File attributes from the FS_*_FL namespace (i_flags, xflags), + * translated to BTRFS_INODE_* bits (BTRFS_INODE_FLAG_MASK) and stored + * in btrfs_inode_item::flags (represented by btrfs_inode::flags and + * btrfs_inode::ro_flags). + */ + BTRFS_SEND_A_FILEATTR = 26, + + BTRFS_SEND_A_UNENCODED_FILE_LEN = 27, + BTRFS_SEND_A_UNENCODED_LEN = 28, + BTRFS_SEND_A_UNENCODED_OFFSET = 29, + /* + * COMPRESSION and ENCRYPTION default to NONE (0) if omitted from + * BTRFS_SEND_C_ENCODED_WRITE. + */ + BTRFS_SEND_A_COMPRESSION = 30, + BTRFS_SEND_A_ENCRYPTION = 31, + BTRFS_SEND_A_MAX_V2 = 31, + + /* Version 3 */ + BTRFS_SEND_A_VERITY_ALGORITHM = 32, + BTRFS_SEND_A_VERITY_BLOCK_SIZE = 33, + BTRFS_SEND_A_VERITY_SALT_DATA = 34, + BTRFS_SEND_A_VERITY_SIG_DATA = 35, + BTRFS_SEND_A_MAX_V3 = 35, + + __BTRFS_SEND_A_MAX = 35, +}; + +long btrfs_ioctl_send(struct btrfs_root *send_root, const struct btrfs_ioctl_send_args *arg); + +#endif diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c new file mode 100644 index 000000000..39a28e1be --- /dev/null +++ b/fs/btrfs/space-info.c @@ -0,0 +1,2267 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include "misc.h" +#include "ctree.h" +#include "space-info.h" +#include "sysfs.h" +#include "volumes.h" +#include "free-space-cache.h" +#include "ordered-data.h" +#include "transaction.h" +#include "block-group.h" +#include "fs.h" +#include "accessors.h" +#include "extent-tree.h" +#include "zoned.h" +#include "delayed-inode.h" + +/* + * HOW DOES SPACE RESERVATION WORK + * + * If you want to know about delalloc specifically, there is a separate comment + * for that with the delalloc code. This comment is about how the whole system + * works generally. + * + * BASIC CONCEPTS + * + * 1) space_info. This is the ultimate arbiter of how much space we can use. + * There's a description of the bytes_ fields with the struct declaration, + * refer to that for specifics on each field. Suffice it to say that for + * reservations we care about total_bytes - SUM(space_info->bytes_) when + * determining if there is space to make an allocation. There is a space_info + * for METADATA, SYSTEM, and DATA areas. + * + * 2) block_rsv's. These are basically buckets for every different type of + * metadata reservation we have. You can see the comment in the block_rsv + * code on the rules for each type, but generally block_rsv->reserved is how + * much space is accounted for in space_info->bytes_may_use. + * + * 3) btrfs_calc*_size. These are the worst case calculations we used based + * on the number of items we will want to modify. We have one for changing + * items, and one for inserting new items. Generally we use these helpers to + * determine the size of the block reserves, and then use the actual bytes + * values to adjust the space_info counters. + * + * MAKING RESERVATIONS, THE NORMAL CASE + * + * We call into either btrfs_reserve_data_bytes() or + * btrfs_reserve_metadata_bytes(), depending on which we're looking for, with + * num_bytes we want to reserve. + * + * ->reserve + * space_info->bytes_may_use += num_bytes + * + * ->extent allocation + * Call btrfs_add_reserved_bytes() which does + * space_info->bytes_may_use -= num_bytes + * space_info->bytes_reserved += extent_bytes + * + * ->insert reference + * Call btrfs_update_block_group() which does + * space_info->bytes_reserved -= extent_bytes + * space_info->bytes_used += extent_bytes + * + * MAKING RESERVATIONS, FLUSHING NORMALLY (non-priority) + * + * Assume we are unable to simply make the reservation because we do not have + * enough space + * + * -> reserve_bytes + * create a reserve_ticket with ->bytes set to our reservation, add it to + * the tail of space_info->tickets, kick async flush thread + * + * ->handle_reserve_ticket + * wait on ticket->wait for ->bytes to be reduced to 0, or ->error to be set + * on the ticket. + * + * -> btrfs_async_reclaim_metadata_space/btrfs_async_reclaim_data_space + * Flushes various things attempting to free up space. + * + * -> btrfs_try_granting_tickets() + * This is called by anything that either subtracts space from + * space_info->bytes_may_use, ->bytes_pinned, etc, or adds to the + * space_info->total_bytes. This loops through the ->priority_tickets and + * then the ->tickets list checking to see if the reservation can be + * completed. If it can the space is added to space_info->bytes_may_use and + * the ticket is woken up. + * + * -> ticket wakeup + * Check if ->bytes == 0, if it does we got our reservation and we can carry + * on, if not return the appropriate error (ENOSPC, but can be EINTR if we + * were interrupted.) + * + * MAKING RESERVATIONS, FLUSHING HIGH PRIORITY + * + * Same as the above, except we add ourselves to the + * space_info->priority_tickets, and we do not use ticket->wait, we simply + * call flush_space() ourselves for the states that are safe for us to call + * without deadlocking and hope for the best. + * + * THE FLUSHING STATES + * + * Generally speaking we will have two cases for each state, a "nice" state + * and a "ALL THE THINGS" state. In btrfs we delay a lot of work in order to + * reduce the locking over head on the various trees, and even to keep from + * doing any work at all in the case of delayed refs. Each of these delayed + * things however hold reservations, and so letting them run allows us to + * reclaim space so we can make new reservations. + * + * FLUSH_DELAYED_ITEMS + * Every inode has a delayed item to update the inode. Take a simple write + * for example, we would update the inode item at write time to update the + * mtime, and then again at finish_ordered_io() time in order to update the + * isize or bytes. We keep these delayed items to coalesce these operations + * into a single operation done on demand. These are an easy way to reclaim + * metadata space. + * + * FLUSH_DELALLOC + * Look at the delalloc comment to get an idea of how much space is reserved + * for delayed allocation. We can reclaim some of this space simply by + * running delalloc, but usually we need to wait for ordered extents to + * reclaim the bulk of this space. + * + * FLUSH_DELAYED_REFS + * We have a block reserve for the outstanding delayed refs space, and every + * delayed ref operation holds a reservation. Running these is a quick way + * to reclaim space, but we want to hold this until the end because COW can + * churn a lot and we can avoid making some extent tree modifications if we + * are able to delay for as long as possible. + * + * RECLAIM_ZONES + * This state only works for the zoned mode. In zoned mode, we cannot reuse + * regions that have once been allocated and then been freed until we reset + * the zone, due to the sequential write requirement. The RECLAIM_ZONES state + * calls the reclaim machinery, evacuating the still valid data in these + * block-groups and relocates it to the data_reloc_bg. Afterwards these + * block-groups get deleted and the transaction is committed. This frees up + * space to use for new allocations. + * + * RESET_ZONES + * This state works only for the zoned mode. On the zoned mode, we cannot + * reuse once allocated then freed region until we reset the zone, due to + * the sequential write zone requirement. The RESET_ZONES state resets the + * zones of an unused block group and let us reuse the space. The reusing + * is faster than removing the block group and allocating another block + * group on the zones. + * + * ALLOC_CHUNK + * We will skip this the first time through space reservation, because of + * overcommit and we don't want to have a lot of useless metadata space when + * our worst case reservations will likely never come true. + * + * RUN_DELAYED_IPUTS + * If we're freeing inodes we're likely freeing checksums, file extent + * items, and extent tree items. Loads of space could be freed up by these + * operations, however they won't be usable until the transaction commits. + * + * COMMIT_TRANS + * This will commit the transaction. Historically we had a lot of logic + * surrounding whether or not we'd commit the transaction, but this waits born + * out of a pre-tickets era where we could end up committing the transaction + * thousands of times in a row without making progress. Now thanks to our + * ticketing system we know if we're not making progress and can error + * everybody out after a few commits rather than burning the disk hoping for + * a different answer. + * + * OVERCOMMIT + * + * Because we hold so many reservations for metadata we will allow you to + * reserve more space than is currently free in the currently allocate + * metadata space. This only happens with metadata, data does not allow + * overcommitting. + * + * You can see the current logic for when we allow overcommit in + * btrfs_can_overcommit(), but it only applies to unallocated space. If there + * is no unallocated space to be had, all reservations are kept within the + * free space in the allocated metadata chunks. + * + * Because of overcommitting, you generally want to use the + * btrfs_can_overcommit() logic for metadata allocations, as it does the right + * thing with or without extra unallocated space. + */ + +struct reserve_ticket { + u64 bytes; + int error; + bool steal; + struct list_head list; + wait_queue_head_t wait; + spinlock_t lock; +}; + +/* + * after adding space to the filesystem, we need to clear the full flags + * on all the space infos. + */ +void btrfs_clear_space_info_full(struct btrfs_fs_info *info) +{ + struct list_head *head = &info->space_info; + struct btrfs_space_info *found; + + list_for_each_entry(found, head, list) + found->full = false; +} + +/* + * Block groups with more than this value (percents) of unusable space will be + * scheduled for background reclaim. + */ +#define BTRFS_DEFAULT_ZONED_RECLAIM_THRESH (75) + +#define BTRFS_UNALLOC_BLOCK_GROUP_TARGET (10ULL) + +#define BTRFS_ZONED_SYNC_RECLAIM_BATCH (5) + +/* + * Calculate chunk size depending on volume type (regular or zoned). + */ +static u64 calc_chunk_size(const struct btrfs_fs_info *fs_info, u64 flags) +{ + if (btrfs_is_zoned(fs_info)) + return fs_info->zone_size; + + ASSERT(flags & BTRFS_BLOCK_GROUP_TYPE_MASK, "flags=%llu", flags); + + if (flags & BTRFS_BLOCK_GROUP_DATA) + return BTRFS_MAX_DATA_CHUNK_SIZE; + else if (flags & (BTRFS_BLOCK_GROUP_SYSTEM | BTRFS_BLOCK_GROUP_METADATA_REMAP)) + return SZ_32M; + + /* Handle BTRFS_BLOCK_GROUP_METADATA */ + if (fs_info->fs_devices->total_rw_bytes > 50ULL * SZ_1G) + return SZ_1G; + + return SZ_256M; +} + +/* + * Update default chunk size. + */ +void btrfs_update_space_info_chunk_size(struct btrfs_space_info *space_info, + u64 chunk_size) +{ + WRITE_ONCE(space_info->chunk_size, chunk_size); +} + +static void init_space_info(struct btrfs_fs_info *info, + struct btrfs_space_info *space_info, u64 flags) +{ + space_info->fs_info = info; + for (int i = 0; i < BTRFS_NR_RAID_TYPES; i++) + INIT_LIST_HEAD(&space_info->block_groups[i]); + init_rwsem(&space_info->groups_sem); + spin_lock_init(&space_info->lock); + space_info->flags = flags & BTRFS_BLOCK_GROUP_TYPE_MASK; + space_info->force_alloc = CHUNK_ALLOC_NO_FORCE; + INIT_LIST_HEAD(&space_info->ro_bgs); + INIT_LIST_HEAD(&space_info->tickets); + INIT_LIST_HEAD(&space_info->priority_tickets); + space_info->clamp = 1; + btrfs_update_space_info_chunk_size(space_info, calc_chunk_size(info, flags)); + space_info->subgroup_id = BTRFS_SUB_GROUP_PRIMARY; + + if (btrfs_is_zoned(info)) + space_info->bg_reclaim_threshold = BTRFS_DEFAULT_ZONED_RECLAIM_THRESH; +} + +static int create_space_info_sub_group(struct btrfs_space_info *parent, u64 flags, + enum btrfs_space_info_sub_group id, int index) +{ + struct btrfs_fs_info *fs_info = parent->fs_info; + struct btrfs_space_info *sub_group; + int ret; + + ASSERT(parent->subgroup_id == BTRFS_SUB_GROUP_PRIMARY, + "parent->subgroup_id=%d", parent->subgroup_id); + ASSERT(id != BTRFS_SUB_GROUP_PRIMARY, "id=%d", id); + + sub_group = kzalloc_obj(*sub_group, GFP_NOFS); + if (!sub_group) + return -ENOMEM; + + init_space_info(fs_info, sub_group, flags); + parent->sub_group[index] = sub_group; + sub_group->parent = parent; + sub_group->subgroup_id = id; + + ret = btrfs_sysfs_add_space_info_type(sub_group); + if (ret) + parent->sub_group[index] = NULL; + return ret; +} + +static int create_space_info(struct btrfs_fs_info *info, u64 flags) +{ + + struct btrfs_space_info *space_info; + int ret = 0; + + space_info = kzalloc_obj(*space_info, GFP_NOFS); + if (!space_info) + return -ENOMEM; + + init_space_info(info, space_info, flags); + + if (btrfs_is_zoned(info)) { + if (flags & BTRFS_BLOCK_GROUP_DATA) + ret = create_space_info_sub_group(space_info, flags, + BTRFS_SUB_GROUP_DATA_RELOC, + 0); + else if (flags & BTRFS_BLOCK_GROUP_METADATA) + ret = create_space_info_sub_group(space_info, flags, + BTRFS_SUB_GROUP_TREELOG, + 0); + + if (ret) + goto out_free; + } + + ret = btrfs_sysfs_add_space_info_type(space_info); + if (ret) + return ret; + + list_add(&space_info->list, &info->space_info); + if (flags & BTRFS_BLOCK_GROUP_DATA) + info->data_sinfo = space_info; + + return ret; + +out_free: + kfree(space_info); + return ret; +} + +int btrfs_init_space_info(struct btrfs_fs_info *fs_info) +{ + struct btrfs_super_block *disk_super; + u64 features; + u64 flags; + bool mixed = false; + int ret; + + disk_super = fs_info->super_copy; + if (!btrfs_super_root(disk_super)) + return -EINVAL; + + features = btrfs_super_incompat_flags(disk_super); + if (features & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS) + mixed = true; + + flags = BTRFS_BLOCK_GROUP_SYSTEM; + ret = create_space_info(fs_info, flags); + if (ret) + return ret; + + if (mixed) { + flags = BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA; + ret = create_space_info(fs_info, flags); + if (ret) + return ret; + } else { + flags = BTRFS_BLOCK_GROUP_METADATA; + ret = create_space_info(fs_info, flags); + if (ret) + return ret; + + flags = BTRFS_BLOCK_GROUP_DATA; + ret = create_space_info(fs_info, flags); + if (ret) + return ret; + } + + if (features & BTRFS_FEATURE_INCOMPAT_REMAP_TREE) { + flags = BTRFS_BLOCK_GROUP_METADATA_REMAP; + ret = create_space_info(fs_info, flags); + } + + return ret; +} + +void btrfs_add_bg_to_space_info(struct btrfs_fs_info *info, + struct btrfs_block_group *block_group) +{ + struct btrfs_space_info *space_info = block_group->space_info; + int factor, index; + + factor = btrfs_bg_type_to_factor(block_group->flags); + + spin_lock(&space_info->lock); + + if (!(block_group->flags & BTRFS_BLOCK_GROUP_REMAPPED) || + block_group->identity_remap_count != 0) { + space_info->total_bytes += block_group->length; + space_info->disk_total += block_group->length * factor; + } + + space_info->bytes_used += block_group->used; + space_info->disk_used += block_group->used * factor; + space_info->bytes_readonly += block_group->bytes_super; + btrfs_space_info_update_bytes_zone_unusable(space_info, block_group->zone_unusable); + if (block_group->length > 0) + space_info->full = false; + btrfs_try_granting_tickets(space_info); + spin_unlock(&space_info->lock); + + block_group->space_info = space_info; + + index = btrfs_bg_flags_to_raid_index(block_group->flags); + down_write(&space_info->groups_sem); + list_add_tail(&block_group->list, &space_info->block_groups[index]); + up_write(&space_info->groups_sem); +} + +struct btrfs_space_info *btrfs_find_space_info(const struct btrfs_fs_info *info, + u64 flags) +{ + const struct list_head *head = &info->space_info; + struct btrfs_space_info *found; + + flags &= BTRFS_BLOCK_GROUP_TYPE_MASK; + + list_for_each_entry(found, head, list) { + if (found->flags & flags) + return found; + } + return NULL; +} + +static u64 calc_effective_data_chunk_size(const struct btrfs_fs_info *fs_info) +{ + struct btrfs_space_info *data_sinfo; + u64 data_chunk_size; + + /* + * Calculate the data_chunk_size, space_info->chunk_size is the + * "optimal" chunk size based on the fs size. However when we actually + * allocate the chunk we will strip this down further, making it no + * more than 10% of the disk or 1G, whichever is smaller. + * + * On the zoned mode, we need to use zone_size (= data_sinfo->chunk_size) + * as it is. + */ + data_sinfo = btrfs_find_space_info(fs_info, BTRFS_BLOCK_GROUP_DATA); + if (btrfs_is_zoned(fs_info)) + return data_sinfo->chunk_size; + data_chunk_size = min(data_sinfo->chunk_size, + mult_perc(fs_info->fs_devices->total_rw_bytes, 10)); + return min_t(u64, data_chunk_size, SZ_1G); +} + +static u64 calc_available_free_space(const struct btrfs_space_info *space_info, + enum btrfs_reserve_flush_enum flush) +{ + struct btrfs_fs_info *fs_info = space_info->fs_info; + bool has_per_profile; + u64 profile; + u64 avail; + u64 data_chunk_size; + int factor; + + if (space_info->flags & BTRFS_BLOCK_GROUP_SYSTEM) + profile = btrfs_system_alloc_profile(fs_info); + else + profile = btrfs_metadata_alloc_profile(fs_info); + + has_per_profile = btrfs_get_per_profile_avail(fs_info, profile, &avail); + if (!has_per_profile) { + avail = atomic64_read(&fs_info->free_chunk_space); + + /* + * If we have dup, raid1 or raid10 then only half of the free + * space is actually usable. For raid56, the space info used + * doesn't include the parity drive, so we don't have to + * change the math + */ + factor = btrfs_bg_type_to_factor(profile); + avail = div_u64(avail, factor); + if (avail == 0) + return 0; + } + data_chunk_size = calc_effective_data_chunk_size(fs_info); + + /* + * Since data allocations immediately use block groups as part of the + * reservation, because we assume that data reservations will == actual + * usage, we could potentially overcommit and then immediately have that + * available space used by a data allocation, which could put us in a + * bind when we get close to filling the file system. + * + * To handle this simply remove the data_chunk_size from the available + * space. If we are relatively empty this won't affect our ability to + * overcommit much, and if we're very close to full it'll keep us from + * getting into a position where we've given ourselves very little + * metadata wiggle room. + */ + if (avail <= data_chunk_size) + return 0; + avail -= data_chunk_size; + + /* + * If we aren't flushing all things, let us overcommit up to + * 1/2th of the space. If we can flush, don't let us overcommit + * too much, let it overcommit up to 1/64th of the space. + */ + if (flush == BTRFS_RESERVE_FLUSH_ALL || flush == BTRFS_RESERVE_FLUSH_ALL_STEAL) + avail >>= 6; + else + avail >>= 1; + + /* + * On the zoned mode, we always allocate one zone as one chunk. + * Returning non-zone size aligned bytes here will result in + * less pressure for the async metadata reclaim process, and it + * will over-commit too much leading to ENOSPC. Align down to the + * zone size to avoid that. + */ + if (btrfs_is_zoned(fs_info)) + avail = ALIGN_DOWN(avail, fs_info->zone_size); + + return avail; +} + +static inline bool check_can_overcommit(const struct btrfs_space_info *space_info, + u64 space_info_used_bytes, u64 bytes, + enum btrfs_reserve_flush_enum flush) +{ + const u64 avail = calc_available_free_space(space_info, flush); + + return (space_info_used_bytes + bytes < space_info->total_bytes + avail); +} + +static inline bool can_overcommit(const struct btrfs_space_info *space_info, + u64 space_info_used_bytes, u64 bytes, + enum btrfs_reserve_flush_enum flush) +{ + /* Don't overcommit when in mixed mode. */ + if (space_info->flags & BTRFS_BLOCK_GROUP_DATA) + return false; + + return check_can_overcommit(space_info, space_info_used_bytes, bytes, flush); +} + +bool btrfs_can_overcommit(const struct btrfs_space_info *space_info, u64 bytes, + enum btrfs_reserve_flush_enum flush) +{ + u64 used; + + /* Don't overcommit when in mixed mode */ + if (space_info->flags & BTRFS_BLOCK_GROUP_DATA) + return false; + + used = btrfs_space_info_used(space_info, true); + + return check_can_overcommit(space_info, used, bytes, flush); +} + +static void remove_ticket(struct btrfs_space_info *space_info, + struct reserve_ticket *ticket, int error) +{ + lockdep_assert_held(&space_info->lock); + + if (!list_empty(&ticket->list)) { + list_del_init(&ticket->list); + ASSERT(space_info->reclaim_size >= ticket->bytes, + "space_info->reclaim_size=%llu ticket->bytes=%llu", + space_info->reclaim_size, ticket->bytes); + space_info->reclaim_size -= ticket->bytes; + } + + spin_lock(&ticket->lock); + /* + * If we are called from a task waiting on the ticket, it may happen + * that before it sets an error on the ticket, a reclaim task was able + * to satisfy the ticket. In that case ignore the error. + */ + if (error && ticket->bytes > 0) + ticket->error = error; + else + ticket->bytes = 0; + + wake_up(&ticket->wait); + spin_unlock(&ticket->lock); +} + +/* + * This is for space we already have accounted in space_info->bytes_may_use, so + * basically when we're returning space from block_rsv's. + */ +void btrfs_try_granting_tickets(struct btrfs_space_info *space_info) +{ + struct list_head *head; + enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_NO_FLUSH; + u64 used = btrfs_space_info_used(space_info, true); + + lockdep_assert_held(&space_info->lock); + + head = &space_info->priority_tickets; +again: + while (!list_empty(head)) { + struct reserve_ticket *ticket; + u64 used_after; + + ticket = list_first_entry(head, struct reserve_ticket, list); + used_after = used + ticket->bytes; + + /* Check and see if our ticket can be satisfied now. */ + if (used_after <= space_info->total_bytes || + can_overcommit(space_info, used, ticket->bytes, flush)) { + btrfs_space_info_update_bytes_may_use(space_info, ticket->bytes); + remove_ticket(space_info, ticket, 0); + space_info->tickets_id++; + used = used_after; + } else { + break; + } + } + + if (head == &space_info->priority_tickets) { + head = &space_info->tickets; + flush = BTRFS_RESERVE_FLUSH_ALL; + goto again; + } +} + +#define DUMP_BLOCK_RSV(fs_info, rsv_name) \ +do { \ + struct btrfs_block_rsv *__rsv = &(fs_info)->rsv_name; \ + spin_lock(&__rsv->lock); \ + btrfs_info(fs_info, #rsv_name ": size %llu reserved %llu", \ + __rsv->size, __rsv->reserved); \ + spin_unlock(&__rsv->lock); \ +} while (0) + +static void dump_global_block_rsv(struct btrfs_fs_info *fs_info) +{ + DUMP_BLOCK_RSV(fs_info, global_block_rsv); + DUMP_BLOCK_RSV(fs_info, trans_block_rsv); + DUMP_BLOCK_RSV(fs_info, chunk_block_rsv); + DUMP_BLOCK_RSV(fs_info, remap_block_rsv); + DUMP_BLOCK_RSV(fs_info, delayed_block_rsv); + DUMP_BLOCK_RSV(fs_info, delayed_refs_rsv); +} + +static void __btrfs_dump_space_info(const struct btrfs_space_info *info) +{ + const struct btrfs_fs_info *fs_info = info->fs_info; + const char *flag_str = btrfs_space_info_type_str(info); + lockdep_assert_held(&info->lock); + + /* The free space could be negative in case of overcommit */ + btrfs_info(fs_info, + "space_info %s (sub-group id %d) has %lld free, is %sfull", + flag_str, info->subgroup_id, + (s64)(info->total_bytes - btrfs_space_info_used(info, true)), + info->full ? "" : "not "); + btrfs_info(fs_info, +"space_info total=%llu, used=%llu, pinned=%llu, reserved=%llu, may_use=%llu, readonly=%llu zone_unusable=%llu", + info->total_bytes, info->bytes_used, info->bytes_pinned, + info->bytes_reserved, info->bytes_may_use, + info->bytes_readonly, info->bytes_zone_unusable); +} + +void btrfs_dump_space_info(struct btrfs_space_info *info, u64 bytes, + bool dump_block_groups) +{ + struct btrfs_fs_info *fs_info = info->fs_info; + struct btrfs_block_group *cache; + u64 total_avail = 0; + int index = 0; + + spin_lock(&info->lock); + __btrfs_dump_space_info(info); + dump_global_block_rsv(fs_info); + spin_unlock(&info->lock); + + if (!dump_block_groups) + return; + + down_read(&info->groups_sem); +again: + list_for_each_entry(cache, &info->block_groups[index], list) { + u64 avail; + + spin_lock(&cache->lock); + avail = btrfs_block_group_available_space(cache); + btrfs_info(fs_info, +"block group %llu has %llu bytes, %llu used %llu pinned %llu reserved %llu delalloc %llu super %llu zone_unusable (%llu bytes available) %s", + cache->start, cache->length, cache->used, cache->pinned, + cache->reserved, cache->delalloc_bytes, + cache->bytes_super, cache->zone_unusable, + avail, cache->ro ? "[readonly]" : ""); + spin_unlock(&cache->lock); + btrfs_dump_free_space(cache, bytes); + total_avail += avail; + } + if (++index < BTRFS_NR_RAID_TYPES) + goto again; + up_read(&info->groups_sem); + + btrfs_info(fs_info, "%llu bytes available across all block groups", total_avail); +} + +static inline u64 calc_reclaim_items_nr(const struct btrfs_fs_info *fs_info, + u64 to_reclaim) +{ + u64 bytes; + u64 nr; + + bytes = btrfs_calc_insert_metadata_size(fs_info, 1); + nr = div64_u64(to_reclaim, bytes); + if (!nr) + nr = 1; + return nr; +} + +/* + * shrink metadata reservation for delalloc + */ +static void shrink_delalloc(struct btrfs_space_info *space_info, + u64 to_reclaim, bool wait_ordered, + bool for_preempt) +{ + struct btrfs_fs_info *fs_info = space_info->fs_info; + struct btrfs_trans_handle *trans; + u64 delalloc_bytes; + u64 ordered_bytes; + u64 items; + long time_left; + int loops; + + delalloc_bytes = percpu_counter_sum_positive(&fs_info->delalloc_bytes); + ordered_bytes = percpu_counter_sum_positive(&fs_info->ordered_bytes); + if (delalloc_bytes == 0 && ordered_bytes == 0) + return; + + /* Calc the number of the pages we need flush for space reservation */ + if (to_reclaim == U64_MAX) { + items = U64_MAX; + } else { + /* + * to_reclaim is set to however much metadata we need to + * reclaim, but reclaiming that much data doesn't really track + * exactly. What we really want to do is reclaim full inode's + * worth of reservations, however that's not available to us + * here. We will take a fraction of the delalloc bytes for our + * flushing loops and hope for the best. Delalloc will expand + * the amount we write to cover an entire dirty extent, which + * will reclaim the metadata reservation for that range. If + * it's not enough subsequent flush stages will be more + * aggressive. + */ + to_reclaim = max(to_reclaim, delalloc_bytes >> 3); + items = calc_reclaim_items_nr(fs_info, to_reclaim) * 2; + } + + trans = current->journal_info; + + /* + * If we are doing more ordered than delalloc we need to just wait on + * ordered extents, otherwise we'll waste time trying to flush delalloc + * that likely won't give us the space back we need. + */ + if (ordered_bytes > delalloc_bytes && !for_preempt) + wait_ordered = true; + + loops = 0; + while ((delalloc_bytes || ordered_bytes) && loops < 3) { + u64 temp = min(delalloc_bytes, to_reclaim) >> PAGE_SHIFT; + long nr_pages = min_t(u64, temp, LONG_MAX); + int async_pages; + + btrfs_start_delalloc_roots(fs_info, nr_pages, true); + + /* + * We need to make sure any outstanding async pages are now + * processed before we continue. This is because things like + * sync_inode() try to be smart and skip writing if the inode is + * marked clean. We don't use filemap_fwrite for flushing + * because we want to control how many pages we write out at a + * time, thus this is the only safe way to make sure we've + * waited for outstanding compressed workers to have started + * their jobs and thus have ordered extents set up properly. + * + * This exists because we do not want to wait for each + * individual inode to finish its async work, we simply want to + * start the IO on everybody, and then come back here and wait + * for all of the async work to catch up. Once we're done with + * that we know we'll have ordered extents for everything and we + * can decide if we wait for that or not. + * + * If we choose to replace this in the future, make absolutely + * sure that the proper waiting is being done in the async case, + * as there have been bugs in that area before. + */ + async_pages = atomic_read(&fs_info->async_delalloc_pages); + if (!async_pages) + goto skip_async; + + /* + * We don't want to wait forever, if we wrote less pages in this + * loop than we have outstanding, only wait for that number of + * pages, otherwise we can wait for all async pages to finish + * before continuing. + */ + if (async_pages > nr_pages) + async_pages -= nr_pages; + else + async_pages = 0; + wait_event(fs_info->async_submit_wait, + atomic_read(&fs_info->async_delalloc_pages) <= + async_pages); +skip_async: + loops++; + if (wait_ordered && !trans) { + btrfs_wait_ordered_roots(fs_info, items, NULL); + } else { + time_left = schedule_timeout_killable(1); + if (time_left) + break; + } + + /* + * If we are for preemption we just want a one-shot of delalloc + * flushing so we can stop flushing if we decide we don't need + * to anymore. + */ + if (for_preempt) + break; + + spin_lock(&space_info->lock); + if (list_empty(&space_info->tickets) && + list_empty(&space_info->priority_tickets)) { + spin_unlock(&space_info->lock); + break; + } + spin_unlock(&space_info->lock); + + delalloc_bytes = percpu_counter_sum_positive( + &fs_info->delalloc_bytes); + ordered_bytes = percpu_counter_sum_positive( + &fs_info->ordered_bytes); + } +} + +/* + * Try to flush some data based on policy set by @state. This is only advisory + * and may fail for various reasons. The caller is supposed to examine the + * state of @space_info to detect the outcome. + */ +static void flush_space(struct btrfs_space_info *space_info, u64 num_bytes, + enum btrfs_flush_state state, bool for_preempt) +{ + struct btrfs_fs_info *fs_info = space_info->fs_info; + struct btrfs_root *root = fs_info->tree_root; + struct btrfs_trans_handle *trans; + int nr; + int ret = 0; + + switch (state) { + case FLUSH_DELAYED_ITEMS_NR: + case FLUSH_DELAYED_ITEMS: + if (state == FLUSH_DELAYED_ITEMS_NR) + nr = calc_reclaim_items_nr(fs_info, num_bytes) * 2; + else + nr = -1; + + trans = btrfs_join_transaction_nostart(root); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + if (ret == -ENOENT) + ret = 0; + break; + } + ret = btrfs_run_delayed_items_nr(trans, nr); + btrfs_end_transaction(trans); + break; + case FLUSH_DELALLOC: + case FLUSH_DELALLOC_WAIT: + case FLUSH_DELALLOC_FULL: + if (state == FLUSH_DELALLOC_FULL) + num_bytes = U64_MAX; + shrink_delalloc(space_info, num_bytes, + state != FLUSH_DELALLOC, for_preempt); + break; + case FLUSH_DELAYED_REFS_NR: + case FLUSH_DELAYED_REFS: + trans = btrfs_join_transaction_nostart(root); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + if (ret == -ENOENT) + ret = 0; + break; + } + if (state == FLUSH_DELAYED_REFS_NR) + btrfs_run_delayed_refs(trans, num_bytes); + else + btrfs_run_delayed_refs(trans, 0); + btrfs_end_transaction(trans); + break; + case ALLOC_CHUNK: + case ALLOC_CHUNK_FORCE: + trans = btrfs_join_transaction(root); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + break; + } + ret = btrfs_chunk_alloc(trans, space_info, + btrfs_get_alloc_profile(fs_info, space_info->flags), + (state == ALLOC_CHUNK) ? CHUNK_ALLOC_NO_FORCE : + CHUNK_ALLOC_FORCE); + btrfs_end_transaction(trans); + + if (ret > 0 || ret == -ENOSPC) + ret = 0; + break; + case RECLAIM_ZONES: + if (btrfs_is_zoned(fs_info)) { + btrfs_reclaim_sweep(fs_info); + btrfs_delete_unused_bgs(fs_info); + btrfs_reclaim_block_groups(fs_info, + BTRFS_ZONED_SYNC_RECLAIM_BATCH); + ASSERT(current->journal_info == NULL); + ret = btrfs_commit_current_transaction(root); + } else { + ret = 0; + } + break; + case RUN_DELAYED_IPUTS: + /* + * If we have pending delayed iputs then we could free up a + * bunch of pinned space, so make sure we run the iputs before + * we do our pinned bytes check below. + */ + btrfs_run_delayed_iputs(fs_info); + btrfs_wait_on_delayed_iputs(fs_info); + break; + case COMMIT_TRANS: + ASSERT(current->journal_info == NULL); + /* + * We don't want to start a new transaction, just attach to the + * current one or wait it fully commits in case its commit is + * happening at the moment. Note: we don't use a nostart join + * because that does not wait for a transaction to fully commit + * (only for it to be unblocked, state TRANS_STATE_UNBLOCKED). + */ + ret = btrfs_commit_current_transaction(root); + break; + case RESET_ZONES: + ret = btrfs_reset_unused_block_groups(space_info, num_bytes); + break; + default: + ret = -ENOSPC; + break; + } + + trace_btrfs_flush_space(fs_info, space_info->flags, num_bytes, state, + ret, for_preempt); + return; +} + +static u64 btrfs_calc_reclaim_metadata_size(const struct btrfs_space_info *space_info) +{ + u64 used; + u64 avail; + u64 to_reclaim = space_info->reclaim_size; + + lockdep_assert_held(&space_info->lock); + + avail = calc_available_free_space(space_info, BTRFS_RESERVE_FLUSH_ALL); + used = btrfs_space_info_used(space_info, true); + + /* + * We may be flushing because suddenly we have less space than we had + * before, and now we're well over-committed based on our current free + * space. If that's the case add in our overage so we make sure to put + * appropriate pressure on the flushing state machine. + */ + if (space_info->total_bytes + avail < used) + to_reclaim += used - (space_info->total_bytes + avail); + + return to_reclaim; +} + +static bool need_preemptive_reclaim(const struct btrfs_space_info *space_info) +{ + struct btrfs_fs_info *fs_info = space_info->fs_info; + const u64 global_rsv_size = btrfs_block_rsv_reserved(&fs_info->global_block_rsv); + u64 ordered, delalloc; + u64 thresh; + u64 used; + + lockdep_assert_held(&space_info->lock); + + /* + * We have tickets queued, bail so we don't compete with the async + * flushers. + */ + if (space_info->reclaim_size) + return false; + + thresh = mult_perc(space_info->total_bytes, 90); + + /* If we're just plain full then async reclaim just slows us down. */ + if ((space_info->bytes_used + space_info->bytes_reserved + + global_rsv_size) >= thresh) + return false; + + used = space_info->bytes_may_use + space_info->bytes_pinned; + + /* The total flushable belongs to the global rsv, don't flush. */ + if (global_rsv_size >= used) + return false; + + /* + * 128MiB is 1/4 of the maximum global rsv size. If we have less than + * that devoted to other reservations then there's no sense in flushing, + * we don't have a lot of things that need flushing. + */ + if (used - global_rsv_size <= SZ_128M) + return false; + + /* + * If we have over half of the free space occupied by reservations or + * pinned then we want to start flushing. + * + * We do not do the traditional thing here, which is to say + * + * if (used >= ((total_bytes + avail) / 2)) + * return 1; + * + * because this doesn't quite work how we want. If we had more than 50% + * of the space_info used by bytes_used and we had 0 available we'd just + * constantly run the background flusher. Instead we want it to kick in + * if our reclaimable space exceeds our clamped free space. + * + * Our clamping range is 2^1 -> 2^8. Practically speaking that means + * the following: + * + * Amount of RAM Minimum threshold Maximum threshold + * + * 256GiB 1GiB 128GiB + * 128GiB 512MiB 64GiB + * 64GiB 256MiB 32GiB + * 32GiB 128MiB 16GiB + * 16GiB 64MiB 8GiB + * + * These are the range our thresholds will fall in, corresponding to how + * much delalloc we need for the background flusher to kick in. + */ + + thresh = calc_available_free_space(space_info, BTRFS_RESERVE_FLUSH_ALL); + used = space_info->bytes_used + space_info->bytes_reserved + + space_info->bytes_readonly + global_rsv_size; + if (used < space_info->total_bytes) + thresh += space_info->total_bytes - used; + thresh >>= space_info->clamp; + + used = space_info->bytes_pinned; + + /* + * If we have more ordered bytes than delalloc bytes then we're either + * doing a lot of DIO, or we simply don't have a lot of delalloc waiting + * around. Preemptive flushing is only useful in that it can free up + * space before tickets need to wait for things to finish. In the case + * of ordered extents, preemptively waiting on ordered extents gets us + * nothing, if our reservations are tied up in ordered extents we'll + * simply have to slow down writers by forcing them to wait on ordered + * extents. + * + * In the case that ordered is larger than delalloc, only include the + * block reserves that we would actually be able to directly reclaim + * from. In this case if we're heavy on metadata operations this will + * clearly be heavy enough to warrant preemptive flushing. In the case + * of heavy DIO or ordered reservations, preemptive flushing will just + * waste time and cause us to slow down. + * + * We want to make sure we truly are maxed out on ordered however, so + * cut ordered in half, and if it's still higher than delalloc then we + * can keep flushing. This is to avoid the case where we start + * flushing, and now delalloc == ordered and we stop preemptively + * flushing when we could still have several gigs of delalloc to flush. + */ + ordered = percpu_counter_read_positive(&fs_info->ordered_bytes) >> 1; + delalloc = percpu_counter_read_positive(&fs_info->delalloc_bytes); + if (ordered >= delalloc) + used += btrfs_block_rsv_reserved(&fs_info->delayed_refs_rsv) + + btrfs_block_rsv_reserved(&fs_info->delayed_block_rsv); + else + used += space_info->bytes_may_use - global_rsv_size; + + return (used >= thresh && !btrfs_fs_closing(fs_info) && + !test_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state)); +} + +static bool steal_from_global_rsv(struct btrfs_space_info *space_info, + struct reserve_ticket *ticket) +{ + struct btrfs_fs_info *fs_info = space_info->fs_info; + struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; + u64 min_bytes; + + lockdep_assert_held(&space_info->lock); + + if (!ticket->steal) + return false; + + if (global_rsv->space_info != space_info) + return false; + + spin_lock(&global_rsv->lock); + min_bytes = mult_perc(global_rsv->size, 10); + if (global_rsv->reserved < min_bytes + ticket->bytes) { + spin_unlock(&global_rsv->lock); + return false; + } + global_rsv->reserved -= ticket->bytes; + if (global_rsv->reserved < global_rsv->size) + global_rsv->full = false; + spin_unlock(&global_rsv->lock); + + remove_ticket(space_info, ticket, 0); + space_info->tickets_id++; + + return true; +} + +/* + * We've exhausted our flushing, start failing tickets. + * + * @space_info - the space info we were flushing + * + * We call this when we've exhausted our flushing ability and haven't made + * progress in satisfying tickets. The reservation code handles tickets in + * order, so if there is a large ticket first and then smaller ones we could + * very well satisfy the smaller tickets. This will attempt to wake up any + * tickets in the list to catch this case. + * + * This function returns true if it was able to make progress by clearing out + * other tickets, or if it stumbles across a ticket that was smaller than the + * first ticket. + */ +static bool maybe_fail_all_tickets(struct btrfs_space_info *space_info) +{ + struct btrfs_fs_info *fs_info = space_info->fs_info; + struct reserve_ticket *ticket; + u64 tickets_id = space_info->tickets_id; + const int abort_error = BTRFS_FS_ERROR(fs_info); + + trace_btrfs_fail_all_tickets(fs_info, space_info); + + if (btrfs_test_opt(fs_info, ENOSPC_DEBUG)) { + btrfs_info(fs_info, "cannot satisfy tickets, dumping space info"); + __btrfs_dump_space_info(space_info); + } + + while (!list_empty(&space_info->tickets) && + tickets_id == space_info->tickets_id) { + ticket = list_first_entry(&space_info->tickets, + struct reserve_ticket, list); + if (unlikely(abort_error)) { + remove_ticket(space_info, ticket, abort_error); + } else { + if (steal_from_global_rsv(space_info, ticket)) + return true; + + if (btrfs_test_opt(fs_info, ENOSPC_DEBUG)) + btrfs_info(fs_info, "failing ticket with %llu bytes", + ticket->bytes); + + remove_ticket(space_info, ticket, -ENOSPC); + + /* + * We're just throwing tickets away, so more flushing may + * not trip over btrfs_try_granting_tickets, so we need + * to call it here to see if we can make progress with + * the next ticket in the list. + */ + btrfs_try_granting_tickets(space_info); + } + } + return (tickets_id != space_info->tickets_id); +} + +static void do_async_reclaim_metadata_space(struct btrfs_space_info *space_info) +{ + struct btrfs_fs_info *fs_info = space_info->fs_info; + u64 to_reclaim; + enum btrfs_flush_state flush_state; + int commit_cycles = 0; + u64 last_tickets_id; + enum btrfs_flush_state final_state; + + if (btrfs_is_zoned(fs_info)) + final_state = RESET_ZONES; + else + final_state = COMMIT_TRANS; + + spin_lock(&space_info->lock); + to_reclaim = btrfs_calc_reclaim_metadata_size(space_info); + if (!to_reclaim) { + space_info->flush = false; + spin_unlock(&space_info->lock); + return; + } + last_tickets_id = space_info->tickets_id; + spin_unlock(&space_info->lock); + + flush_state = FLUSH_DELAYED_ITEMS_NR; + do { + flush_space(space_info, to_reclaim, flush_state, false); + spin_lock(&space_info->lock); + if (list_empty(&space_info->tickets)) { + space_info->flush = false; + spin_unlock(&space_info->lock); + return; + } + to_reclaim = btrfs_calc_reclaim_metadata_size(space_info); + if (last_tickets_id == space_info->tickets_id) { + flush_state++; + } else { + last_tickets_id = space_info->tickets_id; + flush_state = FLUSH_DELAYED_ITEMS_NR; + if (commit_cycles) + commit_cycles--; + } + + /* + * We do not want to empty the system of delalloc unless we're + * under heavy pressure, so allow one trip through the flushing + * logic before we start doing a FLUSH_DELALLOC_FULL. + */ + if (flush_state == FLUSH_DELALLOC_FULL && !commit_cycles) + flush_state++; + + /* + * We don't want to force a chunk allocation until we've tried + * pretty hard to reclaim space. Think of the case where we + * freed up a bunch of space and so have a lot of pinned space + * to reclaim. We would rather use that than possibly create a + * underutilized metadata chunk. So if this is our first run + * through the flushing state machine skip ALLOC_CHUNK_FORCE and + * commit the transaction. If nothing has changed the next go + * around then we can force a chunk allocation. + */ + if (flush_state == ALLOC_CHUNK_FORCE && !commit_cycles) + flush_state++; + + if (flush_state > final_state) { + commit_cycles++; + if (commit_cycles > 2) { + if (maybe_fail_all_tickets(space_info)) { + flush_state = FLUSH_DELAYED_ITEMS_NR; + commit_cycles--; + } else { + space_info->flush = false; + } + } else { + flush_state = FLUSH_DELAYED_ITEMS_NR; + } + } + spin_unlock(&space_info->lock); + } while (flush_state <= final_state); +} + +/* + * This is for normal flushers, it can wait as much time as needed. We will + * loop and continuously try to flush as long as we are making progress. We + * count progress as clearing off tickets each time we have to loop. + */ +static void btrfs_async_reclaim_metadata_space(struct work_struct *work) +{ + struct btrfs_fs_info *fs_info; + struct btrfs_space_info *space_info; + + fs_info = container_of(work, struct btrfs_fs_info, async_reclaim_work); + space_info = btrfs_find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA); + do_async_reclaim_metadata_space(space_info); + for (int i = 0; i < BTRFS_SPACE_INFO_SUB_GROUP_MAX; i++) { + if (space_info->sub_group[i]) + do_async_reclaim_metadata_space(space_info->sub_group[i]); + } +} + +/* + * This handles pre-flushing of metadata space before we get to the point that + * we need to start blocking threads on tickets. The logic here is different + * from the other flush paths because it doesn't rely on tickets to tell us how + * much we need to flush, instead it attempts to keep us below the 80% full + * watermark of space by flushing whichever reservation pool is currently the + * largest. + */ +static void btrfs_preempt_reclaim_metadata_space(struct work_struct *work) +{ + struct btrfs_fs_info *fs_info; + struct btrfs_space_info *space_info; + struct btrfs_block_rsv *delayed_block_rsv; + struct btrfs_block_rsv *delayed_refs_rsv; + struct btrfs_block_rsv *global_rsv; + struct btrfs_block_rsv *trans_rsv; + int loops = 0; + + fs_info = container_of(work, struct btrfs_fs_info, + preempt_reclaim_work); + space_info = btrfs_find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA); + delayed_block_rsv = &fs_info->delayed_block_rsv; + delayed_refs_rsv = &fs_info->delayed_refs_rsv; + global_rsv = &fs_info->global_block_rsv; + trans_rsv = &fs_info->trans_block_rsv; + + spin_lock(&space_info->lock); + while (need_preemptive_reclaim(space_info)) { + enum btrfs_flush_state flush; + u64 delalloc_size = 0; + u64 to_reclaim, block_rsv_size; + const u64 global_rsv_size = btrfs_block_rsv_reserved(global_rsv); + const u64 bytes_may_use = space_info->bytes_may_use; + const u64 bytes_pinned = space_info->bytes_pinned; + + spin_unlock(&space_info->lock); + /* + * We don't have a precise counter for the metadata being + * reserved for delalloc, so we'll approximate it by subtracting + * out the block rsv's space from the bytes_may_use. If that + * amount is higher than the individual reserves, then we can + * assume it's tied up in delalloc reservations. + */ + block_rsv_size = global_rsv_size + + btrfs_block_rsv_reserved(delayed_block_rsv) + + btrfs_block_rsv_reserved(delayed_refs_rsv) + + btrfs_block_rsv_reserved(trans_rsv); + if (block_rsv_size < bytes_may_use) + delalloc_size = bytes_may_use - block_rsv_size; + + /* + * We don't want to include the global_rsv in our calculation, + * because that's space we can't touch. Subtract it from the + * block_rsv_size for the next checks. + */ + block_rsv_size -= global_rsv_size; + + /* + * We really want to avoid flushing delalloc too much, as it + * could result in poor allocation patterns, so only flush it if + * it's larger than the rest of the pools combined. + */ + if (delalloc_size > block_rsv_size) { + to_reclaim = delalloc_size; + flush = FLUSH_DELALLOC; + } else if (bytes_pinned > + (btrfs_block_rsv_reserved(delayed_block_rsv) + + btrfs_block_rsv_reserved(delayed_refs_rsv))) { + to_reclaim = bytes_pinned; + flush = COMMIT_TRANS; + } else if (btrfs_block_rsv_reserved(delayed_block_rsv) > + btrfs_block_rsv_reserved(delayed_refs_rsv)) { + to_reclaim = btrfs_block_rsv_reserved(delayed_block_rsv); + flush = FLUSH_DELAYED_ITEMS_NR; + } else { + to_reclaim = btrfs_block_rsv_reserved(delayed_refs_rsv); + flush = FLUSH_DELAYED_REFS_NR; + } + + loops++; + + /* + * We don't want to reclaim everything, just a portion, so scale + * down the to_reclaim by 1/4. If it takes us down to 0, + * reclaim 1 items worth. + */ + to_reclaim >>= 2; + if (!to_reclaim) + to_reclaim = btrfs_calc_insert_metadata_size(fs_info, 1); + flush_space(space_info, to_reclaim, flush, true); + cond_resched(); + spin_lock(&space_info->lock); + } + + /* We only went through once, back off our clamping. */ + if (loops == 1 && !space_info->reclaim_size) + space_info->clamp = max(1, space_info->clamp - 1); + trace_btrfs_done_preemptive_reclaim(fs_info, space_info); + spin_unlock(&space_info->lock); +} + +/* + * FLUSH_DELALLOC_WAIT: + * Space is freed from flushing delalloc in one of two ways. + * + * 1) compression is on and we allocate less space than we reserved + * 2) we are overwriting existing space + * + * For #1 that extra space is reclaimed as soon as the delalloc pages are + * COWed, by way of btrfs_add_reserved_bytes() which adds the actual extent + * length to ->bytes_reserved, and subtracts the reserved space from + * ->bytes_may_use. + * + * For #2 this is trickier. Once the ordered extent runs we will drop the + * extent in the range we are overwriting, which creates a delayed ref for + * that freed extent. This however is not reclaimed until the transaction + * commits, thus the next stages. + * + * RUN_DELAYED_IPUTS + * If we are freeing inodes, we want to make sure all delayed iputs have + * completed, because they could have been on an inode with i_nlink == 0, and + * thus have been truncated and freed up space. But again this space is not + * immediately reusable, it comes in the form of a delayed ref, which must be + * run and then the transaction must be committed. + * + * COMMIT_TRANS + * This is where we reclaim all of the pinned space generated by running the + * iputs + * + * RECLAIM_ZONES + * This state only works for the zoned mode. We scan the block groups in the + * reclaim_bgs_list and check if we can relocate them. If yes perform the + * relocation to garbage collect the zone. On each of these runs + * BTRFS_ZONED_SYNC_RECLAIM_BATCH (5) block-groups will be reclaimed, after all + * unused block-groups have been deleted. + * + * RESET_ZONES + * This state works only for the zoned mode. We scan the unused block group + * list and reset the zones and reuse the block group. + * + * ALLOC_CHUNK_FORCE + * For data we start with alloc chunk force, however we could have been full + * before, and then the transaction commit could have freed new block groups, + * so if we now have space to allocate do the force chunk allocation. + */ +static const enum btrfs_flush_state data_flush_states[] = { + FLUSH_DELALLOC_FULL, + RUN_DELAYED_IPUTS, + COMMIT_TRANS, + RECLAIM_ZONES, + RESET_ZONES, + ALLOC_CHUNK_FORCE, +}; + +static void do_async_reclaim_data_space(struct btrfs_space_info *space_info) +{ + struct btrfs_fs_info *fs_info = space_info->fs_info; + u64 last_tickets_id; + enum btrfs_flush_state flush_state = 0; + + spin_lock(&space_info->lock); + if (list_empty(&space_info->tickets)) { + space_info->flush = false; + spin_unlock(&space_info->lock); + return; + } + last_tickets_id = space_info->tickets_id; + spin_unlock(&space_info->lock); + + while (!space_info->full) { + flush_space(space_info, U64_MAX, ALLOC_CHUNK_FORCE, false); + spin_lock(&space_info->lock); + if (list_empty(&space_info->tickets)) { + space_info->flush = false; + spin_unlock(&space_info->lock); + return; + } + + /* Something happened, fail everything and bail. */ + if (unlikely(BTRFS_FS_ERROR(fs_info))) + goto aborted_fs; + last_tickets_id = space_info->tickets_id; + spin_unlock(&space_info->lock); + } + + while (flush_state < ARRAY_SIZE(data_flush_states)) { + flush_space(space_info, U64_MAX, + data_flush_states[flush_state], false); + spin_lock(&space_info->lock); + if (list_empty(&space_info->tickets)) { + space_info->flush = false; + spin_unlock(&space_info->lock); + return; + } + + if (last_tickets_id == space_info->tickets_id) { + flush_state++; + } else { + last_tickets_id = space_info->tickets_id; + flush_state = 0; + } + + if (flush_state >= ARRAY_SIZE(data_flush_states)) { + if (space_info->full) { + if (maybe_fail_all_tickets(space_info)) + flush_state = 0; + else + space_info->flush = false; + } else { + flush_state = 0; + } + + /* Something happened, fail everything and bail. */ + if (unlikely(BTRFS_FS_ERROR(fs_info))) + goto aborted_fs; + + } + spin_unlock(&space_info->lock); + } + return; + +aborted_fs: + maybe_fail_all_tickets(space_info); + space_info->flush = false; + spin_unlock(&space_info->lock); +} + +static void btrfs_async_reclaim_data_space(struct work_struct *work) +{ + struct btrfs_fs_info *fs_info; + struct btrfs_space_info *space_info; + + fs_info = container_of(work, struct btrfs_fs_info, async_data_reclaim_work); + space_info = fs_info->data_sinfo; + do_async_reclaim_data_space(space_info); + for (int i = 0; i < BTRFS_SPACE_INFO_SUB_GROUP_MAX; i++) + if (space_info->sub_group[i]) + do_async_reclaim_data_space(space_info->sub_group[i]); +} + +void btrfs_init_async_reclaim_work(struct btrfs_fs_info *fs_info) +{ + INIT_WORK(&fs_info->async_reclaim_work, btrfs_async_reclaim_metadata_space); + INIT_WORK(&fs_info->async_data_reclaim_work, btrfs_async_reclaim_data_space); + INIT_WORK(&fs_info->preempt_reclaim_work, + btrfs_preempt_reclaim_metadata_space); +} + +static const enum btrfs_flush_state priority_flush_states[] = { + FLUSH_DELAYED_ITEMS_NR, + FLUSH_DELAYED_ITEMS, + RESET_ZONES, + ALLOC_CHUNK, +}; + +static const enum btrfs_flush_state evict_flush_states[] = { + FLUSH_DELAYED_ITEMS_NR, + FLUSH_DELAYED_ITEMS, + FLUSH_DELAYED_REFS_NR, + FLUSH_DELAYED_REFS, + FLUSH_DELALLOC, + FLUSH_DELALLOC_WAIT, + FLUSH_DELALLOC_FULL, + ALLOC_CHUNK, + COMMIT_TRANS, + RESET_ZONES, +}; + +static bool is_ticket_served(struct reserve_ticket *ticket) +{ + bool ret; + + spin_lock(&ticket->lock); + ret = (ticket->bytes == 0); + spin_unlock(&ticket->lock); + + return ret; +} + +static void priority_reclaim_metadata_space(struct btrfs_space_info *space_info, + struct reserve_ticket *ticket, + const enum btrfs_flush_state *states, + int states_nr) +{ + struct btrfs_fs_info *fs_info = space_info->fs_info; + u64 to_reclaim; + int flush_state = 0; + + /* + * This is the priority reclaim path, so to_reclaim could be >0 still + * because we may have only satisfied the priority tickets and still + * left non priority tickets on the list. We would then have + * to_reclaim but ->bytes == 0. + */ + if (is_ticket_served(ticket)) + return; + + spin_lock(&space_info->lock); + to_reclaim = btrfs_calc_reclaim_metadata_size(space_info); + spin_unlock(&space_info->lock); + + while (flush_state < states_nr) { + flush_space(space_info, to_reclaim, states[flush_state], false); + if (is_ticket_served(ticket)) + return; + flush_state++; + } + + spin_lock(&space_info->lock); + /* + * Attempt to steal from the global rsv if we can, except if the fs was + * turned into error mode due to a transaction abort when flushing space + * above, in that case fail with the abort error instead of returning + * success to the caller if we can steal from the global rsv - this is + * just to have caller fail immediately instead of later when trying to + * modify the fs, making it easier to debug -ENOSPC problems. + */ + if (unlikely(BTRFS_FS_ERROR(fs_info))) + remove_ticket(space_info, ticket, BTRFS_FS_ERROR(fs_info)); + else if (!steal_from_global_rsv(space_info, ticket)) + remove_ticket(space_info, ticket, -ENOSPC); + + /* + * We must run try_granting_tickets here because we could be a large + * ticket in front of a smaller ticket that can now be satisfied with + * the available space. + */ + btrfs_try_granting_tickets(space_info); + spin_unlock(&space_info->lock); +} + +static void priority_reclaim_data_space(struct btrfs_space_info *space_info, + struct reserve_ticket *ticket) +{ + /* We could have been granted before we got here. */ + if (is_ticket_served(ticket)) + return; + + spin_lock(&space_info->lock); + while (!space_info->full) { + spin_unlock(&space_info->lock); + flush_space(space_info, U64_MAX, ALLOC_CHUNK_FORCE, false); + if (is_ticket_served(ticket)) + return; + spin_lock(&space_info->lock); + } + + remove_ticket(space_info, ticket, -ENOSPC); + btrfs_try_granting_tickets(space_info); + spin_unlock(&space_info->lock); +} + +static void wait_reserve_ticket(struct btrfs_space_info *space_info, + struct reserve_ticket *ticket) + +{ + DEFINE_WAIT(wait); + + spin_lock(&ticket->lock); + while (ticket->bytes > 0 && ticket->error == 0) { + int ret; + + ret = prepare_to_wait_event(&ticket->wait, &wait, TASK_KILLABLE); + spin_unlock(&ticket->lock); + if (ret) { + /* + * Delete us from the list. After we unlock the space + * info, we don't want the async reclaim job to reserve + * space for this ticket. If that would happen, then the + * ticket's task would not known that space was reserved + * despite getting an error, resulting in a space leak + * (bytes_may_use counter of our space_info). + */ + spin_lock(&space_info->lock); + remove_ticket(space_info, ticket, -EINTR); + spin_unlock(&space_info->lock); + return; + } + + schedule(); + + finish_wait(&ticket->wait, &wait); + spin_lock(&ticket->lock); + } + spin_unlock(&ticket->lock); +} + +/* + * Do the appropriate flushing and waiting for a ticket. + * + * @space_info: space info for the reservation + * @ticket: ticket for the reservation + * @start_ns: timestamp when the reservation started + * @orig_bytes: amount of bytes originally reserved + * @flush: how much we can flush + * + * This does the work of figuring out how to flush for the ticket, waiting for + * the reservation, and returning the appropriate error if there is one. + */ +static int handle_reserve_ticket(struct btrfs_space_info *space_info, + struct reserve_ticket *ticket, + u64 start_ns, u64 orig_bytes, + enum btrfs_reserve_flush_enum flush) +{ + int ret; + + switch (flush) { + case BTRFS_RESERVE_FLUSH_DATA: + case BTRFS_RESERVE_FLUSH_ALL: + case BTRFS_RESERVE_FLUSH_ALL_STEAL: + wait_reserve_ticket(space_info, ticket); + break; + case BTRFS_RESERVE_FLUSH_LIMIT: + priority_reclaim_metadata_space(space_info, ticket, + priority_flush_states, + ARRAY_SIZE(priority_flush_states)); + break; + case BTRFS_RESERVE_FLUSH_EVICT: + priority_reclaim_metadata_space(space_info, ticket, + evict_flush_states, + ARRAY_SIZE(evict_flush_states)); + break; + case BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE: + case BTRFS_RESERVE_FLUSH_ZONED_RELOCATION: + priority_reclaim_data_space(space_info, ticket); + break; + default: + ASSERT(0, "flush=%d", flush); + break; + } + + ret = ticket->error; + ASSERT(list_empty(&ticket->list)); + /* + * Check that we can't have an error set if the reservation succeeded, + * as that would confuse tasks and lead them to error out without + * releasing reserved space (if an error happens the expectation is that + * space wasn't reserved at all). + */ + ASSERT(!(ticket->bytes == 0 && ticket->error), + "ticket->bytes=%llu ticket->error=%d", ticket->bytes, ticket->error); + trace_btrfs_reserve_ticket(space_info->fs_info, space_info->flags, + orig_bytes, start_ns, flush, ticket->error); + return ret; +} + +/* + * This returns true if this flush state will go through the ordinary flushing + * code. + */ +static inline bool is_normal_flushing(enum btrfs_reserve_flush_enum flush) +{ + return (flush == BTRFS_RESERVE_FLUSH_ALL) || + (flush == BTRFS_RESERVE_FLUSH_ALL_STEAL); +} + +static inline void maybe_clamp_preempt(struct btrfs_space_info *space_info) +{ + struct btrfs_fs_info *fs_info = space_info->fs_info; + u64 ordered = percpu_counter_sum_positive(&fs_info->ordered_bytes); + u64 delalloc = percpu_counter_sum_positive(&fs_info->delalloc_bytes); + + /* + * If we're heavy on ordered operations then clamping won't help us. We + * need to clamp specifically to keep up with dirty'ing buffered + * writers, because there's not a 1:1 correlation of writing delalloc + * and freeing space, like there is with flushing delayed refs or + * delayed nodes. If we're already more ordered than delalloc then + * we're keeping up, otherwise we aren't and should probably clamp. + */ + if (ordered < delalloc) + space_info->clamp = min(space_info->clamp + 1, 8); +} + +static inline bool can_steal(enum btrfs_reserve_flush_enum flush) +{ + return (flush == BTRFS_RESERVE_FLUSH_ALL_STEAL || + flush == BTRFS_RESERVE_FLUSH_EVICT); +} + +/* + * NO_FLUSH and FLUSH_EMERGENCY don't want to create a ticket, they just want to + * fail as quickly as possible. + */ +static inline bool can_ticket(enum btrfs_reserve_flush_enum flush) +{ + return (flush != BTRFS_RESERVE_NO_FLUSH && + flush != BTRFS_RESERVE_FLUSH_EMERGENCY); +} + +/* + * Try to reserve bytes from the block_rsv's space. + * + * @space_info: space info we want to allocate from + * @orig_bytes: number of bytes we want + * @flush: whether or not we can flush to make our reservation + * + * This will reserve orig_bytes number of bytes from the space info associated + * with the block_rsv. If there is not enough space it will make an attempt to + * flush out space to make room. It will do this by flushing delalloc if + * possible or committing the transaction. If flush is 0 then no attempts to + * regain reservations will be made and this will fail if there is not enough + * space already. + */ +static int reserve_bytes(struct btrfs_space_info *space_info, u64 orig_bytes, + enum btrfs_reserve_flush_enum flush) +{ + struct btrfs_fs_info *fs_info = space_info->fs_info; + struct work_struct *async_work; + struct reserve_ticket ticket; + u64 start_ns = 0; + u64 used; + int ret = -ENOSPC; + bool pending_tickets; + + ASSERT(orig_bytes, "orig_bytes=%llu", orig_bytes); + /* + * If have a transaction handle (current->journal_info != NULL), then + * the flush method can not be neither BTRFS_RESERVE_FLUSH_ALL* nor + * BTRFS_RESERVE_FLUSH_EVICT, as we could deadlock because those + * flushing methods can trigger transaction commits. + */ + if (current->journal_info) { + /* One assert per line for easier debugging. */ + ASSERT(flush != BTRFS_RESERVE_FLUSH_ALL, "flush=%d", flush); + ASSERT(flush != BTRFS_RESERVE_FLUSH_ALL_STEAL, "flush=%d", flush); + ASSERT(flush != BTRFS_RESERVE_FLUSH_EVICT, "flush=%d", flush); + } + + if (flush == BTRFS_RESERVE_FLUSH_DATA) + async_work = &fs_info->async_data_reclaim_work; + else + async_work = &fs_info->async_reclaim_work; + + spin_lock(&space_info->lock); + used = btrfs_space_info_used(space_info, true); + + /* + * We don't want NO_FLUSH allocations to jump everybody, they can + * generally handle ENOSPC in a different way, so treat them the same as + * normal flushers when it comes to skipping pending tickets. + */ + if (is_normal_flushing(flush) || (flush == BTRFS_RESERVE_NO_FLUSH)) + pending_tickets = !list_empty(&space_info->tickets) || + !list_empty(&space_info->priority_tickets); + else + pending_tickets = !list_empty(&space_info->priority_tickets); + + /* + * Carry on if we have enough space (short-circuit) OR call + * can_overcommit() to ensure we can overcommit to continue. + */ + if (!pending_tickets && + ((used + orig_bytes <= space_info->total_bytes) || + can_overcommit(space_info, used, orig_bytes, flush))) { + btrfs_space_info_update_bytes_may_use(space_info, orig_bytes); + ret = 0; + } + + /* + * Things are dire, we need to make a reservation so we don't abort. We + * will let this reservation go through as long as we have actual space + * left to allocate for the block. + */ + if (ret && unlikely(flush == BTRFS_RESERVE_FLUSH_EMERGENCY)) { + used -= space_info->bytes_may_use; + if (used + orig_bytes <= space_info->total_bytes) { + btrfs_space_info_update_bytes_may_use(space_info, orig_bytes); + ret = 0; + } + } + + /* + * If we couldn't make a reservation then setup our reservation ticket + * and kick the async worker if it's not already running. + * + * If we are a priority flusher then we just need to add our ticket to + * the list and we will do our own flushing further down. + */ + if (ret && can_ticket(flush)) { + ticket.bytes = orig_bytes; + ticket.error = 0; + space_info->reclaim_size += ticket.bytes; + init_waitqueue_head(&ticket.wait); + spin_lock_init(&ticket.lock); + ticket.steal = can_steal(flush); + if (trace_btrfs_reserve_ticket_enabled()) + start_ns = ktime_get_ns(); + + if (flush == BTRFS_RESERVE_FLUSH_ALL || + flush == BTRFS_RESERVE_FLUSH_ALL_STEAL || + flush == BTRFS_RESERVE_FLUSH_DATA) { + list_add_tail(&ticket.list, &space_info->tickets); + if (!space_info->flush) { + /* + * We were forced to add a reserve ticket, so + * our preemptive flushing is unable to keep + * up. Clamp down on the threshold for the + * preemptive flushing in order to keep up with + * the workload. + */ + maybe_clamp_preempt(space_info); + + space_info->flush = true; + trace_btrfs_trigger_flush(fs_info, + space_info->flags, + orig_bytes, flush, + "enospc"); + queue_work(system_dfl_wq, async_work); + } + } else { + list_add_tail(&ticket.list, + &space_info->priority_tickets); + } + } else if (!ret && space_info->flags & BTRFS_BLOCK_GROUP_METADATA) { + /* + * We will do the space reservation dance during log replay, + * which means we won't have fs_info->fs_root set, so don't do + * the async reclaim as we will panic. + */ + if (!test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags) && + !work_busy(&fs_info->preempt_reclaim_work) && + need_preemptive_reclaim(space_info)) { + trace_btrfs_trigger_flush(fs_info, space_info->flags, + orig_bytes, flush, "preempt"); + queue_work(system_dfl_wq, + &fs_info->preempt_reclaim_work); + } + } + spin_unlock(&space_info->lock); + if (!ret || !can_ticket(flush)) + return ret; + + return handle_reserve_ticket(space_info, &ticket, start_ns, orig_bytes, flush); +} + +/* + * Try to reserve metadata bytes from the block_rsv's space. + * + * @space_info: the space_info we're allocating for + * @orig_bytes: number of bytes we want + * @flush: whether or not we can flush to make our reservation + * + * This will reserve orig_bytes number of bytes from the space info associated + * with the block_rsv. If there is not enough space it will make an attempt to + * flush out space to make room. It will do this by flushing delalloc if + * possible or committing the transaction. If flush is 0 then no attempts to + * regain reservations will be made and this will fail if there is not enough + * space already. + */ +int btrfs_reserve_metadata_bytes(struct btrfs_space_info *space_info, + u64 orig_bytes, + enum btrfs_reserve_flush_enum flush) +{ + int ret; + + ret = reserve_bytes(space_info, orig_bytes, flush); + if (ret == -ENOSPC) { + struct btrfs_fs_info *fs_info = space_info->fs_info; + + trace_btrfs_space_reservation(fs_info, "space_info:enospc", + space_info->flags, orig_bytes, 1); + + if (btrfs_test_opt(fs_info, ENOSPC_DEBUG)) + btrfs_dump_space_info(space_info, orig_bytes, false); + } + return ret; +} + +/* + * Try to reserve data bytes for an allocation. + * + * @space_info: the space_info we're allocating for + * @bytes: number of bytes we need + * @flush: how we are allowed to flush + * + * This will reserve bytes from the data space info. If there is not enough + * space then we will attempt to flush space as specified by flush. + */ +int btrfs_reserve_data_bytes(struct btrfs_space_info *space_info, u64 bytes, + enum btrfs_reserve_flush_enum flush) +{ + struct btrfs_fs_info *fs_info = space_info->fs_info; + int ret; + + ASSERT(flush == BTRFS_RESERVE_FLUSH_DATA || + flush == BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE || + flush == BTRFS_RESERVE_FLUSH_ZONED_RELOCATION || + flush == BTRFS_RESERVE_NO_FLUSH, "flush=%d", flush); + ASSERT(!current->journal_info || flush != BTRFS_RESERVE_FLUSH_DATA, + "current->journal_info=0x%lx flush=%d", + (unsigned long)current->journal_info, flush); + + ret = reserve_bytes(space_info, bytes, flush); + if (ret == -ENOSPC) { + trace_btrfs_space_reservation(fs_info, "space_info:enospc", + space_info->flags, bytes, 1); + if (btrfs_test_opt(fs_info, ENOSPC_DEBUG)) + btrfs_dump_space_info(space_info, bytes, false); + } + return ret; +} + +/* Dump all the space infos when we abort a transaction due to ENOSPC. */ +__cold void btrfs_dump_space_info_for_trans_abort(struct btrfs_fs_info *fs_info) +{ + struct btrfs_space_info *space_info; + + btrfs_info(fs_info, "dumping space info:"); + list_for_each_entry(space_info, &fs_info->space_info, list) { + spin_lock(&space_info->lock); + __btrfs_dump_space_info(space_info); + spin_unlock(&space_info->lock); + } + dump_global_block_rsv(fs_info); +} + +/* + * Account the unused space of all the readonly block group in the space_info. + * takes mirrors into account. + */ +u64 btrfs_account_ro_block_groups_free_space(struct btrfs_space_info *sinfo) +{ + struct btrfs_block_group *block_group; + u64 free_bytes = 0; + int factor; + + /* It's df, we don't care if it's racy */ + if (data_race(list_empty(&sinfo->ro_bgs))) + return 0; + + spin_lock(&sinfo->lock); + list_for_each_entry(block_group, &sinfo->ro_bgs, ro_list) { + spin_lock(&block_group->lock); + + if (!block_group->ro) { + spin_unlock(&block_group->lock); + continue; + } + + factor = btrfs_bg_type_to_factor(block_group->flags); + free_bytes += (block_group->length - + block_group->used) * factor; + + spin_unlock(&block_group->lock); + } + spin_unlock(&sinfo->lock); + + return free_bytes; +} + +static u64 calc_pct_ratio(u64 x, u64 y) +{ + int ret; + + if (!y) + return 0; +again: + ret = check_mul_overflow(100, x, &x); + if (ret) + goto lose_precision; + return div64_u64(x, y); +lose_precision: + x >>= 10; + y >>= 10; + if (!y) + y = 1; + goto again; +} + +/* + * A reasonable buffer for unallocated space is 10 data block_groups. + * If we claw this back repeatedly, we can still achieve efficient + * utilization when near full, and not do too much reclaim while + * always maintaining a solid buffer for workloads that quickly + * allocate and pressure the unallocated space. + */ +static u64 calc_unalloc_target(struct btrfs_fs_info *fs_info) +{ + u64 chunk_sz = calc_effective_data_chunk_size(fs_info); + + return BTRFS_UNALLOC_BLOCK_GROUP_TARGET * chunk_sz; +} + +/* + * The fundamental goal of automatic reclaim is to protect the filesystem's + * unallocated space and thus minimize the probability of the filesystem going + * read only when a metadata allocation failure causes a transaction abort. + * + * However, relocations happen into the space_info's unused space, therefore + * automatic reclaim must also back off as that space runs low. There is no + * value in doing trivial "relocations" of re-writing the same block group + * into a fresh one. + * + * Furthermore, we want to avoid doing too much reclaim even if there are good + * candidates. This is because the allocator is pretty good at filling up the + * holes with writes. So we want to do just enough reclaim to try and stay + * safe from running out of unallocated space but not be wasteful about it. + * + * Therefore, the dynamic reclaim threshold is calculated as follows: + * - calculate a target unallocated amount of 5 block group sized chunks + * - ratchet up the intensity of reclaim depending on how far we are from + * that target by using a formula of unalloc / target to set the threshold. + * + * Typically with 10 block groups as the target, the discrete values this comes + * out to are 0, 10, 20, ... , 80, 90, and 99. + */ +static int calc_dynamic_reclaim_threshold(const struct btrfs_space_info *space_info) +{ + struct btrfs_fs_info *fs_info = space_info->fs_info; + u64 unalloc = atomic64_read(&fs_info->free_chunk_space); + u64 target = calc_unalloc_target(fs_info); + u64 alloc = space_info->total_bytes; + u64 used = btrfs_space_info_used(space_info, false); + u64 unused = alloc - used; + u64 want = target > unalloc ? target - unalloc : 0; + u64 data_chunk_size = calc_effective_data_chunk_size(fs_info); + + /* If we have no unused space, don't bother, it won't work anyway. */ + if (unused < data_chunk_size) + return 0; + + /* Cast to int is OK because want <= target. */ + return calc_pct_ratio(want, target); +} + +int btrfs_calc_reclaim_threshold(const struct btrfs_space_info *space_info) +{ + lockdep_assert_held(&space_info->lock); + + if (READ_ONCE(space_info->dynamic_reclaim)) + return calc_dynamic_reclaim_threshold(space_info); + return READ_ONCE(space_info->bg_reclaim_threshold); +} + +/* + * Under "urgent" reclaim, we will reclaim even fresh block groups that have + * recently seen successful allocations, as we are desperate to reclaim + * whatever we can to avoid ENOSPC in a transaction leading to a readonly fs. + */ +static bool is_reclaim_urgent(struct btrfs_space_info *space_info) +{ + struct btrfs_fs_info *fs_info = space_info->fs_info; + u64 unalloc = atomic64_read(&fs_info->free_chunk_space); + u64 data_chunk_size = calc_effective_data_chunk_size(fs_info); + + return unalloc < data_chunk_size; +} + +static bool do_reclaim_sweep(struct btrfs_space_info *space_info, int raid) +{ + struct btrfs_block_group *bg; + int thresh_pct; + bool will_reclaim = false; + bool urgent; + + spin_lock(&space_info->lock); + urgent = is_reclaim_urgent(space_info); + thresh_pct = btrfs_calc_reclaim_threshold(space_info); + spin_unlock(&space_info->lock); + + down_read(&space_info->groups_sem); +again: + list_for_each_entry(bg, &space_info->block_groups[raid], list) { + u64 thresh; + bool reclaim = false; + + btrfs_get_block_group(bg); + spin_lock(&bg->lock); + thresh = mult_perc(bg->length, thresh_pct); + if (bg->used < thresh && bg->reclaim_mark) { + will_reclaim = true; + reclaim = true; + } + bg->reclaim_mark = true; + spin_unlock(&bg->lock); + if (reclaim) + btrfs_mark_bg_to_reclaim(bg); + btrfs_put_block_group(bg); + } + + /* + * In situations where we are very motivated to reclaim (low unalloc) + * use two passes to make the reclaim mark check best effort. + * + * If we have any staler groups, we don't touch the fresher ones, but if we + * really need a block group, do take a fresh one. + */ + if (!will_reclaim && urgent) { + urgent = false; + goto again; + } + + up_read(&space_info->groups_sem); + return will_reclaim; +} + +void btrfs_space_info_update_reclaimable(struct btrfs_space_info *space_info, s64 bytes) +{ + u64 chunk_sz = calc_effective_data_chunk_size(space_info->fs_info); + + lockdep_assert_held(&space_info->lock); + space_info->reclaimable_bytes += bytes; + + if (space_info->reclaimable_bytes > 0 && + space_info->reclaimable_bytes >= chunk_sz) + btrfs_set_periodic_reclaim_ready(space_info, true); +} + +void btrfs_set_periodic_reclaim_ready(struct btrfs_space_info *space_info, bool ready) +{ + lockdep_assert_held(&space_info->lock); + if (!READ_ONCE(space_info->periodic_reclaim)) + return; + if (ready != space_info->periodic_reclaim_ready) { + space_info->periodic_reclaim_ready = ready; + if (!ready) + space_info->reclaimable_bytes = 0; + } +} + +static bool btrfs_should_periodic_reclaim(struct btrfs_space_info *space_info) +{ + bool ret; + + if (space_info->flags & BTRFS_BLOCK_GROUP_SYSTEM) + return false; + if (!READ_ONCE(space_info->periodic_reclaim)) + return false; + + spin_lock(&space_info->lock); + ret = space_info->periodic_reclaim_ready; + spin_unlock(&space_info->lock); + + return ret; +} + +void btrfs_reclaim_sweep(const struct btrfs_fs_info *fs_info) +{ + int raid; + struct btrfs_space_info *space_info; + + list_for_each_entry(space_info, &fs_info->space_info, list) { + if (!btrfs_should_periodic_reclaim(space_info)) + continue; + for (raid = 0; raid < BTRFS_NR_RAID_TYPES; raid++) { + if (do_reclaim_sweep(space_info, raid)) { + spin_lock(&space_info->lock); + btrfs_set_periodic_reclaim_ready(space_info, false); + spin_unlock(&space_info->lock); + } + } + } +} + +void btrfs_return_free_space(struct btrfs_space_info *space_info, u64 len) +{ + struct btrfs_fs_info *fs_info = space_info->fs_info; + struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; + + lockdep_assert_held(&space_info->lock); + + /* Prioritize the global reservation to receive the freed space. */ + if (global_rsv->space_info != space_info) + goto grant; + + spin_lock(&global_rsv->lock); + if (!global_rsv->full) { + u64 to_add = min(len, global_rsv->size - global_rsv->reserved); + + global_rsv->reserved += to_add; + btrfs_space_info_update_bytes_may_use(space_info, to_add); + if (global_rsv->reserved >= global_rsv->size) + global_rsv->full = true; + len -= to_add; + } + spin_unlock(&global_rsv->lock); + +grant: + /* Add to any tickets we may have. */ + if (len) + btrfs_try_granting_tickets(space_info); +} diff --git a/fs/btrfs/space-info.h b/fs/btrfs/space-info.h new file mode 100644 index 000000000..aa836e8a9 --- /dev/null +++ b/fs/btrfs/space-info.h @@ -0,0 +1,355 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_SPACE_INFO_H +#define BTRFS_SPACE_INFO_H + +#include +#include +#include +#include +#include +#include +#include +#include "volumes.h" + +struct btrfs_fs_info; +struct btrfs_block_group; + +/* + * Different levels for to flush space when doing space reservations. + * + * The higher the level, the more methods we try to reclaim space. + */ +enum btrfs_reserve_flush_enum { + /* + * Used when we can't flush or don't need: + * + * 1) We are holding a transaction handle open, so we can't flush as + * that could deadlock. + * + * 2) For a nowait write we don't want to block when reserving delalloc. + * + * 3) Joining a transaction or attaching a transaction, we don't want + * to wait and we don't need to reserve anything (any needed space + * was reserved before in a dedicated block reserve, or we rely on + * the global block reserve, see btrfs_init_root_block_rsv()). + * + * 4) Starting a transaction when we don't need to reserve space, as + * we don't need it because we previously reserved in a dedicated + * block reserve or rely on the global block reserve, like the above + * case. + */ + BTRFS_RESERVE_NO_FLUSH, + + /* + * Flush space by: + * - Running delayed inode items + * - Allocating a new chunk + */ + BTRFS_RESERVE_FLUSH_LIMIT, + + /* + * Flush space by: + * - Running delayed inode items + * - Running delayed refs + * - Running delalloc and waiting for ordered extents + * - Allocating a new chunk + * - Committing transaction + */ + BTRFS_RESERVE_FLUSH_EVICT, + + /* + * Flush space by above mentioned methods and by: + * - Running delayed iputs + * - Committing transaction + * + * Can be interrupted by a fatal signal. + */ + BTRFS_RESERVE_FLUSH_DATA, + BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE, + BTRFS_RESERVE_FLUSH_ALL, + + /* + * Pretty much the same as FLUSH_ALL, but can also steal space from + * global rsv. + * + * Can be interrupted by a fatal signal. + */ + BTRFS_RESERVE_FLUSH_ALL_STEAL, + + /* + * This is for relocation on zoned filesystems only. We need to use + * priority flushing for this, because otherwise we can deadlock on + * waiting for a ticket, that cannot be granted, because we cannot do + * any allocations. + * + * Apart from being specific to zoned relocation, it is equal to + * BTRFS_FLUSH_FREE_SPACE_INODE. + */ + BTRFS_RESERVE_FLUSH_ZONED_RELOCATION, + + /* + * This is for btrfs_use_block_rsv only. We have exhausted our block + * rsv and our global block rsv. This can happen for things like + * delalloc where we are overwriting a lot of extents with a single + * extent and didn't reserve enough space. Alternatively it can happen + * with delalloc where we reserve 1 extents worth for a large extent but + * fragmentation leads to multiple extents being created. This will + * give us the reservation in the case of + * + * if (num_bytes < (space_info->total_bytes - + * btrfs_space_info_used(space_info, false)) + * + * Which ignores bytes_may_use. This is potentially dangerous, but our + * reservation system is generally pessimistic so is able to absorb this + * style of mistake. + */ + BTRFS_RESERVE_FLUSH_EMERGENCY, +}; + +/* + * Please be aware that the order of enum values will be the order of the reclaim + * process in btrfs_async_reclaim_metadata_space(). + */ +enum btrfs_flush_state { + FLUSH_DELAYED_ITEMS_NR = 1, + FLUSH_DELAYED_ITEMS = 2, + FLUSH_DELAYED_REFS_NR = 3, + FLUSH_DELAYED_REFS = 4, + FLUSH_DELALLOC = 5, + FLUSH_DELALLOC_WAIT = 6, + FLUSH_DELALLOC_FULL = 7, + ALLOC_CHUNK = 8, + ALLOC_CHUNK_FORCE = 9, + RUN_DELAYED_IPUTS = 10, + COMMIT_TRANS = 11, + RESET_ZONES = 12, + RECLAIM_ZONES = 13, +}; + +enum btrfs_space_info_sub_group { + BTRFS_SUB_GROUP_PRIMARY, + BTRFS_SUB_GROUP_DATA_RELOC, + BTRFS_SUB_GROUP_TREELOG, +}; + +#define BTRFS_SPACE_INFO_SUB_GROUP_MAX 1 +struct btrfs_space_info { + struct btrfs_fs_info *fs_info; + struct btrfs_space_info *parent; + struct btrfs_space_info *sub_group[BTRFS_SPACE_INFO_SUB_GROUP_MAX]; + int subgroup_id; + spinlock_t lock; + + u64 total_bytes; /* total bytes in the space, + this doesn't take mirrors into account */ + u64 bytes_used; /* total bytes used, + this doesn't take mirrors into account */ + u64 bytes_pinned; /* total bytes pinned, will be freed when the + transaction finishes */ + u64 bytes_reserved; /* total bytes the allocator has reserved for + current allocations */ + u64 bytes_may_use; /* number of bytes that may be used for + delalloc/allocations */ + u64 bytes_readonly; /* total bytes that are read only */ + u64 bytes_zone_unusable; /* total bytes that are unusable until + resetting the device zone */ + + u64 max_extent_size; /* This will hold the maximum extent size of + the space info if we had an ENOSPC in the + allocator. */ + /* Chunk size in bytes */ + u64 chunk_size; + + /* + * Once a block group drops below this threshold (percents) we'll + * schedule it for reclaim. + */ + int bg_reclaim_threshold; + + int clamp; /* Used to scale our threshold for preemptive + flushing. The value is >> clamp, so turns + out to be a 2^clamp divisor. */ + + bool full; /* indicates that we cannot allocate any more + chunks for this space */ + bool chunk_alloc; /* set if we are allocating a chunk */ + + bool flush; /* set if we are trying to make space */ + + unsigned int force_alloc; /* set if we need to force a chunk + alloc for this space */ + + u64 disk_used; /* total bytes used on disk */ + u64 disk_total; /* total bytes on disk, takes mirrors into + account */ + + u64 flags; + + struct list_head list; + /* Protected by the spinlock 'lock'. */ + struct list_head ro_bgs; + struct list_head priority_tickets; + struct list_head tickets; + + /* + * Size of space that needs to be reclaimed in order to satisfy pending + * tickets + */ + u64 reclaim_size; + + /* + * tickets_id just indicates the next ticket will be handled, so note + * it's not stored per ticket. + */ + u64 tickets_id; + + struct rw_semaphore groups_sem; + /* for block groups in our same type */ + struct list_head block_groups[BTRFS_NR_RAID_TYPES]; + + struct kobject kobj; + struct kobject *block_group_kobjs[BTRFS_NR_RAID_TYPES]; + + /* + * Monotonically increasing counter of block group reclaim attempts + * Exposed in /sys/fs//allocation//reclaim_count + */ + u64 reclaim_count; + + /* + * Monotonically increasing counter of reclaimed bytes + * Exposed in /sys/fs//allocation//reclaim_bytes + */ + u64 reclaim_bytes; + + /* + * Monotonically increasing counter of reclaim errors + * Exposed in /sys/fs//allocation//reclaim_errors + */ + u64 reclaim_errors; + + /* + * If true, use the dynamic relocation threshold, instead of the + * fixed bg_reclaim_threshold. + */ + bool dynamic_reclaim; + + /* + * Periodically check all block groups against the reclaim + * threshold in the cleaner thread. + */ + bool periodic_reclaim; + + /* + * Periodic reclaim should be a no-op if a space_info hasn't + * freed any space since the last time we tried. + */ + bool periodic_reclaim_ready; + + /* + * Net bytes freed or allocated since the last reclaim pass. + */ + s64 reclaimable_bytes; +}; + +static inline bool btrfs_mixed_space_info(const struct btrfs_space_info *space_info) +{ + return ((space_info->flags & BTRFS_BLOCK_GROUP_METADATA) && + (space_info->flags & BTRFS_BLOCK_GROUP_DATA)); +} + +/* + * + * Declare a helper function to detect underflow of various space info members + */ +#define DECLARE_SPACE_INFO_UPDATE(name, trace_name) \ +static inline void \ +btrfs_space_info_update_##name(struct btrfs_space_info *sinfo, \ + s64 bytes) \ +{ \ + struct btrfs_fs_info *fs_info = sinfo->fs_info; \ + const u64 abs_bytes = (bytes < 0) ? -bytes : bytes; \ + lockdep_assert_held(&sinfo->lock); \ + trace_update_##name(fs_info, sinfo, sinfo->name, bytes); \ + trace_btrfs_space_reservation(fs_info, trace_name, \ + sinfo->flags, abs_bytes, \ + bytes > 0); \ + if (bytes < 0 && sinfo->name < -bytes) { \ + WARN_ON(1); \ + sinfo->name = 0; \ + return; \ + } \ + sinfo->name += bytes; \ +} + +DECLARE_SPACE_INFO_UPDATE(bytes_may_use, "space_info"); +DECLARE_SPACE_INFO_UPDATE(bytes_pinned, "pinned"); +DECLARE_SPACE_INFO_UPDATE(bytes_zone_unusable, "zone_unusable"); + +static inline u64 btrfs_space_info_used(const struct btrfs_space_info *s_info, + bool may_use_included) +{ + lockdep_assert_held(&s_info->lock); + + return s_info->bytes_used + s_info->bytes_reserved + + s_info->bytes_pinned + s_info->bytes_readonly + + s_info->bytes_zone_unusable + + (may_use_included ? s_info->bytes_may_use : 0); +} + +int btrfs_init_space_info(struct btrfs_fs_info *fs_info); +void btrfs_add_bg_to_space_info(struct btrfs_fs_info *info, + struct btrfs_block_group *block_group); +void btrfs_update_space_info_chunk_size(struct btrfs_space_info *space_info, + u64 chunk_size); +struct btrfs_space_info *btrfs_find_space_info(const struct btrfs_fs_info *info, + u64 flags); +void btrfs_clear_space_info_full(struct btrfs_fs_info *info); +void btrfs_dump_space_info(struct btrfs_space_info *info, u64 bytes, + bool dump_block_groups); +int btrfs_reserve_metadata_bytes(struct btrfs_space_info *space_info, + u64 orig_bytes, + enum btrfs_reserve_flush_enum flush); +void btrfs_try_granting_tickets(struct btrfs_space_info *space_info); +bool btrfs_can_overcommit(const struct btrfs_space_info *space_info, u64 bytes, + enum btrfs_reserve_flush_enum flush); + +static inline void btrfs_space_info_free_bytes_may_use( + struct btrfs_space_info *space_info, + u64 num_bytes) +{ + spin_lock(&space_info->lock); + btrfs_space_info_update_bytes_may_use(space_info, -num_bytes); + btrfs_try_granting_tickets(space_info); + spin_unlock(&space_info->lock); +} +int btrfs_reserve_data_bytes(struct btrfs_space_info *space_info, u64 bytes, + enum btrfs_reserve_flush_enum flush); +void btrfs_dump_space_info_for_trans_abort(struct btrfs_fs_info *fs_info); +void btrfs_init_async_reclaim_work(struct btrfs_fs_info *fs_info); +u64 btrfs_account_ro_block_groups_free_space(struct btrfs_space_info *sinfo); + +void btrfs_space_info_update_reclaimable(struct btrfs_space_info *space_info, s64 bytes); +void btrfs_set_periodic_reclaim_ready(struct btrfs_space_info *space_info, bool ready); +int btrfs_calc_reclaim_threshold(const struct btrfs_space_info *space_info); +void btrfs_reclaim_sweep(const struct btrfs_fs_info *fs_info); +void btrfs_return_free_space(struct btrfs_space_info *space_info, u64 len); + +static inline const char *btrfs_space_info_type_str(const struct btrfs_space_info *space_info) +{ + switch (space_info->flags) { + case BTRFS_BLOCK_GROUP_SYSTEM: + return "SYSTEM"; + case BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA: + return "DATA+METADATA"; + case BTRFS_BLOCK_GROUP_DATA: + return "DATA"; + case BTRFS_BLOCK_GROUP_METADATA: + return "METADATA"; + default: + return "UNKNOWN"; + } +} + +#endif /* BTRFS_SPACE_INFO_H */ diff --git a/fs/btrfs/subpage.c b/fs/btrfs/subpage.c new file mode 100644 index 000000000..ebf18efe1 --- /dev/null +++ b/fs/btrfs/subpage.c @@ -0,0 +1,993 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include "messages.h" +#include "subpage.h" +#include "btrfs_inode.h" + +/* + * Subpage (block size < folio size) support overview: + * + * Limitations: + * + * - Metadata must be fully aligned to node size + * So when nodesize <= page size, the metadata can never cross folio boundaries. + * + * - Only support blocks per folio <= min(BTRFS_MAX_FOLIO_SIZE / fs block size, + * BTRFS_MAX_BLOCKS_PER_FOLIO) + * This is to ensure we can afford an on-stack bitmap, without the need to allocate + * bitmap memory at runtime. + * + * Implementation: + * + * - Common + * Both metadata and data will use a new structure, btrfs_folio_state, to + * record the status of each sector inside a page. This provides the extra + * granularity needed. + * + * - Metadata + * Since we have multiple tree blocks inside one page, we can't rely on page + * locking anymore, or we will have greatly reduced concurrency or even + * deadlocks (hold one tree lock while trying to lock another tree lock in + * the same page). + * + * Thus for metadata locking, subpage support relies on io_tree locking only. + * This means a slightly higher tree locking latency. + */ + +int btrfs_attach_folio_state(const struct btrfs_fs_info *fs_info, + struct folio *folio, enum btrfs_folio_type type) +{ + struct btrfs_folio_state *bfs; + + /* For metadata we don't support large folio yet. */ + if (type == BTRFS_SUBPAGE_METADATA) + ASSERT(!folio_test_large(folio)); + + /* + * We have cases like a dummy extent buffer page, which is not mapped + * and doesn't need to be locked. + */ + if (folio->mapping) + ASSERT(folio_test_locked(folio)); + + /* Either not subpage, or the folio already has private attached. */ + if (folio_test_private(folio)) + return 0; + if (type == BTRFS_SUBPAGE_METADATA && !btrfs_meta_is_subpage(fs_info)) + return 0; + if (type == BTRFS_SUBPAGE_DATA && !btrfs_is_subpage(fs_info, folio)) + return 0; + + bfs = btrfs_alloc_folio_state(fs_info, folio_size(folio), type, GFP_NOFS); + if (IS_ERR(bfs)) + return PTR_ERR(bfs); + + folio_attach_private(folio, bfs); + return 0; +} + +void btrfs_detach_folio_state(const struct btrfs_fs_info *fs_info, struct folio *folio, + enum btrfs_folio_type type) +{ + struct btrfs_folio_state *bfs; + + /* Either not subpage, or the folio already has private attached. */ + if (!folio_test_private(folio)) + return; + if (type == BTRFS_SUBPAGE_METADATA && !btrfs_meta_is_subpage(fs_info)) + return; + if (type == BTRFS_SUBPAGE_DATA && !btrfs_is_subpage(fs_info, folio)) + return; + + bfs = folio_detach_private(folio); + ASSERT(bfs); + btrfs_free_folio_state(bfs); +} + +struct btrfs_folio_state *btrfs_alloc_folio_state(const struct btrfs_fs_info *fs_info, + size_t fsize, enum btrfs_folio_type type, + gfp_t gfp) +{ + struct btrfs_folio_state *ret; + unsigned int real_size; + + ASSERT(fs_info->sectorsize < fsize); + + real_size = struct_size(ret, bitmaps, + BITS_TO_LONGS(btrfs_bitmap_nr_max * + (fsize >> fs_info->sectorsize_bits))); + ret = kzalloc(real_size, gfp); + if (!ret) + return ERR_PTR(-ENOMEM); + + spin_lock_init(&ret->lock); + if (type == BTRFS_SUBPAGE_METADATA) + atomic_set(&ret->eb_refs, 0); + else + atomic_set(&ret->nr_locked, 0); + return ret; +} + +/* + * Increase the eb_refs of current subpage. + * + * This is important for eb allocation, to prevent race with last eb freeing + * of the same page. + * With the eb_refs increased before the eb inserted into radix tree, + * detach_extent_buffer_page() won't detach the folio private while we're still + * allocating the extent buffer. + */ +void btrfs_folio_inc_eb_refs(const struct btrfs_fs_info *fs_info, struct folio *folio) +{ + struct btrfs_folio_state *bfs; + + if (!btrfs_meta_is_subpage(fs_info)) + return; + + ASSERT(folio_test_private(folio) && folio->mapping); + lockdep_assert_held(&folio->mapping->i_private_lock); + + bfs = folio_get_private(folio); + atomic_inc(&bfs->eb_refs); +} + +void btrfs_folio_dec_eb_refs(const struct btrfs_fs_info *fs_info, struct folio *folio) +{ + struct btrfs_folio_state *bfs; + + if (!btrfs_meta_is_subpage(fs_info)) + return; + + ASSERT(folio_test_private(folio) && folio->mapping); + lockdep_assert_held(&folio->mapping->i_private_lock); + + bfs = folio_get_private(folio); + ASSERT(atomic_read(&bfs->eb_refs)); + atomic_dec(&bfs->eb_refs); +} + +static void btrfs_subpage_assert(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len) +{ + /* Basic checks */ + ASSERT(folio_test_private(folio) && folio_get_private(folio)); + ASSERT(IS_ALIGNED(start, fs_info->sectorsize) && + IS_ALIGNED(len, fs_info->sectorsize), "start=%llu len=%u", start, len); + /* + * The range check only works for mapped page, we can still have + * unmapped page like dummy extent buffer pages. + */ + if (folio->mapping) + ASSERT(folio_pos(folio) <= start && + start + len <= folio_next_pos(folio), + "start=%llu len=%u folio_pos=%llu folio_size=%zu", + start, len, folio_pos(folio), folio_size(folio)); +} + +#define subpage_calc_start_bit(fs_info, folio, name, start, len) \ +({ \ + unsigned int __start_bit; \ + const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ + \ + btrfs_subpage_assert(fs_info, folio, start, len); \ + __start_bit = offset_in_folio(folio, start) >> fs_info->sectorsize_bits; \ + __start_bit += __bpf * btrfs_bitmap_nr_##name; \ + __start_bit; \ +}) + +static void btrfs_subpage_clamp_range(struct folio *folio, u64 *start, u32 *len) +{ + u64 orig_start = *start; + u32 orig_len = *len; + + *start = max_t(u64, folio_pos(folio), orig_start); + /* + * For certain call sites like btrfs_drop_pages(), we may have pages + * beyond the target range. In that case, just set @len to 0, subpage + * helpers can handle @len == 0 without any problem. + */ + if (folio_pos(folio) >= orig_start + orig_len) + *len = 0; + else + *len = min_t(u64, folio_next_pos(folio), orig_start + orig_len) - *start; +} + +static bool btrfs_subpage_end_and_test_lock(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len) +{ + struct btrfs_folio_state *bfs = folio_get_private(folio); + const int nbits = (len >> fs_info->sectorsize_bits); + unsigned long flags; + bool last; + + btrfs_subpage_assert(fs_info, folio, start, len); + + spin_lock_irqsave(&bfs->lock, flags); + /* + * We have call sites passing @lock_page into + * extent_clear_unlock_delalloc() for compression path. + * + * This @locked_page is locked by plain lock_page(), thus its + * subpage::locked is 0. Handle them in a special way. + */ + if (atomic_read(&bfs->nr_locked) == 0) { + spin_unlock_irqrestore(&bfs->lock, flags); + return true; + } + ASSERT(atomic_read(&bfs->nr_locked) >= nbits, + "atomic_read(&bfs->nr_locked)=%d nbits=%d", + atomic_read(&bfs->nr_locked), nbits); + last = atomic_sub_and_test(nbits, &bfs->nr_locked); + spin_unlock_irqrestore(&bfs->lock, flags); + return last; +} + +/* + * Handle different locked folios: + * + * - Non-subpage folio + * Just unlock it. + * + * - folio locked but without any subpage locked + * This happens either before writepage_delalloc() or the delalloc range is + * already handled by previous folio. + * We can simple unlock it. + * + * - folio locked with subpage range locked. + * We go through the locked sectors inside the range and clear their locked + * bitmap, reduce the writer lock number, and unlock the page if that's + * the last locked range. + */ +void btrfs_folio_end_lock(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len) +{ + struct btrfs_folio_state *bfs = folio_get_private(folio); + + ASSERT(folio_test_locked(folio)); + + if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, folio)) { + folio_unlock(folio); + return; + } + + /* + * For subpage case, there are two types of locked page. With or + * without locked number. + * + * Since we own the page lock, no one else could touch subpage::locked + * and we are safe to do several atomic operations without spinlock. + */ + if (atomic_read(&bfs->nr_locked) == 0) { + /* No subpage lock, locked by plain lock_page(). */ + folio_unlock(folio); + return; + } + + btrfs_subpage_clamp_range(folio, &start, &len); + if (btrfs_subpage_end_and_test_lock(fs_info, folio, start, len)) + folio_unlock(folio); +} + +void btrfs_folio_end_lock_bitmap(const struct btrfs_fs_info *fs_info, + struct folio *folio, unsigned long *bitmap) +{ + struct btrfs_folio_state *bfs = folio_get_private(folio); + const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio); + const unsigned int nbits = bitmap_weight(bitmap, blocks_per_folio); + unsigned long flags; + bool last = false; + + if (!btrfs_is_subpage(fs_info, folio)) { + folio_unlock(folio); + return; + } + + if (atomic_read(&bfs->nr_locked) == 0) { + /* No subpage lock, locked by plain lock_page(). */ + folio_unlock(folio); + return; + } + + spin_lock_irqsave(&bfs->lock, flags); + ASSERT(atomic_read(&bfs->nr_locked) >= nbits, + "atomic_read(&bfs->nr_locked)=%d nbits=%d", + atomic_read(&bfs->nr_locked), nbits); + last = atomic_sub_and_test(nbits, &bfs->nr_locked); + spin_unlock_irqrestore(&bfs->lock, flags); + if (last) + folio_unlock(folio); +} + +#define subpage_test_bitmap_all_set(fs_info, folio, name) \ +({ \ + struct btrfs_folio_state *__bfs = folio_get_private(folio); \ + const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ + \ + bitmap_test_range_all_set(__bfs->bitmaps, \ + __bpf * btrfs_bitmap_nr_##name, __bpf); \ +}) + +#define subpage_test_bitmap_all_zero(fs_info, folio, name) \ +({ \ + struct btrfs_folio_state *__bfs = folio_get_private(folio); \ + const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ + \ + bitmap_test_range_all_zero(__bfs->bitmaps, \ + __bpf * btrfs_bitmap_nr_##name, __bpf); \ +}) + +void btrfs_subpage_set_uptodate(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len) +{ + struct btrfs_folio_state *bfs = folio_get_private(folio); + unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, + uptodate, start, len); + unsigned long flags; + + spin_lock_irqsave(&bfs->lock, flags); + bitmap_set(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits); + if (subpage_test_bitmap_all_set(fs_info, folio, uptodate)) + folio_mark_uptodate(folio); + spin_unlock_irqrestore(&bfs->lock, flags); +} + +void btrfs_subpage_clear_uptodate(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len) +{ + struct btrfs_folio_state *bfs = folio_get_private(folio); + unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, + uptodate, start, len); + unsigned long flags; + + spin_lock_irqsave(&bfs->lock, flags); + bitmap_clear(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits); + folio_clear_uptodate(folio); + spin_unlock_irqrestore(&bfs->lock, flags); +} + +/* + * folio_mark_dirty() for a folio we are dirtying with a space reservation. + * + * Dirtiers without a reservation use btrfs_data_dirty_folio(). + */ +static void btrfs_folio_mark_dirty(struct folio *folio) +{ + struct address_space *mapping = folio_mapping(folio); + + if (!mapping || !mapping->host || !is_data_inode(BTRFS_I(mapping->host))) { + folio_mark_dirty(folio); + return; + } + if (folio_test_reclaim(folio)) + folio_clear_reclaim(folio); + filemap_dirty_folio(mapping, folio); +} + +/* + * The set helper of the dirty ops, so it only runs for folios without a + * fixup bitmap: for those the folio flag is the whole fixup state, and this + * reserving write covers the block, so retire it. Metadata never has the + * flag set and only pays the test. + */ +static void btrfs_folio_mark_dirty_reserved(struct folio *folio) +{ + if (folio_test_fixup_pending(folio)) + folio_clear_fixup_pending(folio); + btrfs_folio_mark_dirty(folio); +} + +void btrfs_subpage_set_dirty(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len) +{ + struct btrfs_folio_state *bfs = folio_get_private(folio); + unsigned int dirty_bit = subpage_calc_start_bit(fs_info, folio, + dirty, start, len); + unsigned int fixup_bit = subpage_calc_start_bit(fs_info, folio, + fixup, start, len); + const unsigned int nbits = len >> fs_info->sectorsize_bits; + unsigned long flags; + + spin_lock_irqsave(&bfs->lock, flags); + bitmap_set(bfs->bitmaps, dirty_bit, nbits); + /* Proper dirtying obviates the need for fixup. */ + bitmap_clear(bfs->bitmaps, fixup_bit, nbits); + if (folio_test_fixup_pending(folio) && + subpage_test_bitmap_all_zero(fs_info, folio, fixup)) + folio_clear_fixup_pending(folio); + spin_unlock_irqrestore(&bfs->lock, flags); + btrfs_folio_mark_dirty(folio); +} + +static void folio_clear_tags(struct folio *folio) +{ + struct address_space *mapping = folio_mapping(folio); + XA_STATE(xas, &mapping->i_pages, folio->index); + unsigned long flags; + + ASSERT(folio_test_locked(folio)); + ASSERT(mapping); + ASSERT(mapping_use_writeback_tags(mapping)); + + xas_lock_irqsave(&xas, flags); + xas_load(&xas); + xas_clear_mark(&xas, PAGECACHE_TAG_DIRTY); + xas_clear_mark(&xas, PAGECACHE_TAG_TOWRITE); + xas_unlock_irqrestore(&xas, flags); +} + +/* + * Extra clear_and_test function for subpage dirty bitmap. + * + * Return true if we're the last bits in the dirty_bitmap and clear the + * dirty_bitmap. + * Return false otherwise. + * + * NOTE: Callers should manually clear page dirty for true case, as we have + * extra handling for tree blocks. + */ +bool btrfs_subpage_clear_and_test_dirty(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len) +{ + struct btrfs_folio_state *bfs = folio_get_private(folio); + unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, + dirty, start, len); + unsigned long flags; + bool last = false; + + spin_lock_irqsave(&bfs->lock, flags); + bitmap_clear(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits); + if (subpage_test_bitmap_all_zero(fs_info, folio, dirty)) + last = true; + spin_unlock_irqrestore(&bfs->lock, flags); + return last; +} + +void btrfs_subpage_clear_dirty(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len) +{ + bool last; + + last = btrfs_subpage_clear_and_test_dirty(fs_info, folio, start, len); + if (last) + folio_clear_dirty_for_io(folio); +} + +void btrfs_subpage_set_writeback(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len) +{ + struct btrfs_folio_state *bfs = folio_get_private(folio); + unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, + writeback, start, len); + unsigned long flags; + + spin_lock_irqsave(&bfs->lock, flags); + bitmap_set(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits); + + /* + * Don't clear the TOWRITE tag when starting writeback on a still-dirty + * folio. Doing so can cause WB_SYNC_ALL writepages() to overlook it, + * assume writeback is complete, and exit too early — violating sync + * ordering guarantees. + * + * Instead we manually clear the DIRTY and TOWRITE tags after the folio + * is no longer dirty. + */ + if (!folio_test_writeback(folio)) + __folio_start_writeback(folio, true); + if (!folio_test_dirty(folio)) + folio_clear_tags(folio); + spin_unlock_irqrestore(&bfs->lock, flags); +} + +void btrfs_subpage_clear_writeback(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len) +{ + struct btrfs_folio_state *bfs = folio_get_private(folio); + unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, + writeback, start, len); + unsigned long flags; + + spin_lock_irqsave(&bfs->lock, flags); + bitmap_clear(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits); + if (subpage_test_bitmap_all_zero(fs_info, folio, writeback)) { + ASSERT(folio_test_writeback(folio)); + folio_end_writeback(folio); + } + spin_unlock_irqrestore(&bfs->lock, flags); +} + +void btrfs_subpage_clear_fixup(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len) +{ + struct btrfs_folio_state *bfs = folio_get_private(folio); + unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, + fixup, start, len); + unsigned long flags; + + spin_lock_irqsave(&bfs->lock, flags); + bitmap_clear(bfs->bitmaps, start_bit, len >> fs_info->sectorsize_bits); + if (subpage_test_bitmap_all_zero(fs_info, folio, fixup)) + folio_clear_fixup_pending(folio); + spin_unlock_irqrestore(&bfs->lock, flags); +} + +/* + * In one pass under bfs->lock, mark every block with a clear dirty bit in the + * range both dirty and needing fixup. + * + * Only called from the dirty_folio callback, which owns the folio-level + * dirty flag; calling folio_mark_dirty() here would recurse. + * + * The folio fixup flag and bits are both set under bfs->lock so that a + * writeback pass observing the new bits also observes the flag. + */ +static void btrfs_subpage_set_fixup_dirty(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len) +{ + struct btrfs_folio_state *bfs = folio_get_private(folio); + unsigned int dirty_bit = subpage_calc_start_bit(fs_info, folio, + dirty, start, len); + unsigned int fixup_bit = subpage_calc_start_bit(fs_info, folio, + fixup, start, len); + const unsigned int nbits = len >> fs_info->sectorsize_bits; + unsigned long flags; + bool marked = false; + + spin_lock_irqsave(&bfs->lock, flags); + for (unsigned int i = 0; i < nbits; i++) { + if (test_bit(dirty_bit + i, bfs->bitmaps)) + continue; + set_bit(dirty_bit + i, bfs->bitmaps); + set_bit(fixup_bit + i, bfs->bitmaps); + marked = true; + } + if (marked) + folio_set_fixup_pending(folio); + spin_unlock_irqrestore(&bfs->lock, flags); +} + +/* + * Mark the still-clean blocks of a folio dirty and needing fixup, for + * btrfs_data_dirty_folio(). + * + * A subpage block size folio that is not uptodate is left alone: its clean + * blocks may hold content that was never read in, which must not be marked + * dirty. + */ +void btrfs_folio_set_fixup_dirty(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len) +{ + if (!btrfs_is_subpage(fs_info, folio)) { + if (!folio_test_dirty(folio)) + folio_set_fixup_pending(folio); + return; + } + if (!folio_test_uptodate(folio)) + return; + btrfs_subpage_set_fixup_dirty(fs_info, folio, start, len); +} + +/* + * Drop the fixup blocks inside the range: clear both their fixup and dirty + * bits. + * + * Fixup blocks carry no space reservation, so their fixup and dirty bits + * must be dropped together. Clearing only the fixup bit would leave a + * dirty block without a reservation which is not a valid state. + * + * Returns true if the folio has no dirty blocks left. + */ +static bool btrfs_subpage_clear_fixup_dirty(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len) +{ + struct btrfs_folio_state *bfs = folio_get_private(folio); + unsigned int dirty_bit = subpage_calc_start_bit(fs_info, folio, + dirty, start, len); + unsigned int fixup_bit = subpage_calc_start_bit(fs_info, folio, + fixup, start, len); + const unsigned int nbits = len >> fs_info->sectorsize_bits; + unsigned long flags; + bool last; + + spin_lock_irqsave(&bfs->lock, flags); + for (unsigned int i = 0; i < nbits; i++) { + if (!test_bit(fixup_bit + i, bfs->bitmaps)) + continue; + clear_bit(fixup_bit + i, bfs->bitmaps); + clear_bit(dirty_bit + i, bfs->bitmaps); + } + if (subpage_test_bitmap_all_zero(fs_info, folio, fixup)) + folio_clear_fixup_pending(folio); + last = subpage_test_bitmap_all_zero(fs_info, folio, dirty); + spin_unlock_irqrestore(&bfs->lock, flags); + return last; +} + +/* + * Drop the fixup blocks inside the range, for callers discarding their data: + * btrfs_invalidate_folio() and the writepage fixup worker's error path. + * + * Callers that have just reserved space for a block want + * btrfs_folio_clear_fixup() instead - there the block stays dirty and gets + * written. + * + * The range can be byte-granular (an unaligned truncate through + * btrfs_invalidate_folio()); only blocks fully inside it are dropped, as a + * partially covered block still holds live data outside the range. For + * single-block folios the folio flag is the fixup state, so it is dropped + * only when the range covers the whole folio. + */ +void btrfs_folio_clear_fixup_dirty(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len) +{ + u64 aligned_start; + u64 aligned_end; + + /* The folio flag is set whenever any fixup bitmap bit is. */ + if (!folio_test_fixup_pending(folio)) + return; + if (!btrfs_is_subpage(fs_info, folio)) { + if (start <= folio_pos(folio) && + start + len >= folio_next_pos(folio)) { + folio_clear_fixup_pending(folio); + folio_clear_dirty_for_io(folio); + } + return; + } + btrfs_subpage_clamp_range(folio, &start, &len); + aligned_start = round_up(start, fs_info->sectorsize); + aligned_end = round_down(start + len, fs_info->sectorsize); + if (aligned_end <= aligned_start) + return; + if (btrfs_subpage_clear_fixup_dirty(fs_info, folio, aligned_start, + aligned_end - aligned_start)) + folio_clear_dirty_for_io(folio); +} + +bool btrfs_folio_test_fixup(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len) +{ + if (!btrfs_is_subpage(fs_info, folio)) + return folio_test_fixup_pending(folio); + return btrfs_subpage_test_fixup(fs_info, folio, start, len); +} + +void btrfs_folio_clear_fixup(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len) +{ + if (!btrfs_is_subpage(fs_info, folio)) { + folio_clear_fixup_pending(folio); + return; + } + btrfs_subpage_clear_fixup(fs_info, folio, start, len); +} + +/* + * Unlike set/clear which is dependent on each page status, for test all bits + * are tested in the same way. + */ +#define IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(name) \ +bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info, \ + struct folio *folio, u64 start, u32 len) \ +{ \ + struct btrfs_folio_state *bfs = folio_get_private(folio); \ + unsigned int start_bit = subpage_calc_start_bit(fs_info, folio, \ + name, start, len); \ + unsigned long flags; \ + bool ret; \ + \ + spin_lock_irqsave(&bfs->lock, flags); \ + ret = bitmap_test_range_all_set(bfs->bitmaps, start_bit, \ + len >> fs_info->sectorsize_bits); \ + spin_unlock_irqrestore(&bfs->lock, flags); \ + return ret; \ +} +IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(uptodate); +IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(dirty); +IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(writeback); +IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(fixup); + +/* + * Note that, in selftests (extent-io-tests), we can have empty fs_info passed + * in. We only test sectorsize == PAGE_SIZE cases so far, thus we can fall + * back to regular sectorsize branch. + */ +#define IMPLEMENT_BTRFS_PAGE_OPS(name, folio_set_func, \ + folio_clear_func, folio_test_func) \ +void btrfs_folio_set_##name(const struct btrfs_fs_info *fs_info, \ + struct folio *folio, u64 start, u32 len) \ +{ \ + if (unlikely(!fs_info) || \ + !btrfs_is_subpage(fs_info, folio)) { \ + folio_set_func(folio); \ + return; \ + } \ + btrfs_subpage_set_##name(fs_info, folio, start, len); \ +} \ +void btrfs_folio_clear_##name(const struct btrfs_fs_info *fs_info, \ + struct folio *folio, u64 start, u32 len) \ +{ \ + if (unlikely(!fs_info) || \ + !btrfs_is_subpage(fs_info, folio)) { \ + folio_clear_func(folio); \ + return; \ + } \ + btrfs_subpage_clear_##name(fs_info, folio, start, len); \ +} \ +bool btrfs_folio_test_##name(const struct btrfs_fs_info *fs_info, \ + struct folio *folio, u64 start, u32 len) \ +{ \ + if (unlikely(!fs_info) || \ + !btrfs_is_subpage(fs_info, folio)) \ + return folio_test_func(folio); \ + return btrfs_subpage_test_##name(fs_info, folio, start, len); \ +} \ +void btrfs_folio_clamp_set_##name(const struct btrfs_fs_info *fs_info, \ + struct folio *folio, u64 start, u32 len) \ +{ \ + if (unlikely(!fs_info) || \ + !btrfs_is_subpage(fs_info, folio)) { \ + folio_set_func(folio); \ + return; \ + } \ + btrfs_subpage_clamp_range(folio, &start, &len); \ + btrfs_subpage_set_##name(fs_info, folio, start, len); \ +} \ +void btrfs_folio_clamp_clear_##name(const struct btrfs_fs_info *fs_info, \ + struct folio *folio, u64 start, u32 len) \ +{ \ + if (unlikely(!fs_info) || \ + !btrfs_is_subpage(fs_info, folio)) { \ + folio_clear_func(folio); \ + return; \ + } \ + btrfs_subpage_clamp_range(folio, &start, &len); \ + btrfs_subpage_clear_##name(fs_info, folio, start, len); \ +} \ +bool btrfs_folio_clamp_test_##name(const struct btrfs_fs_info *fs_info, \ + struct folio *folio, u64 start, u32 len) \ +{ \ + if (unlikely(!fs_info) || \ + !btrfs_is_subpage(fs_info, folio)) \ + return folio_test_func(folio); \ + btrfs_subpage_clamp_range(folio, &start, &len); \ + return btrfs_subpage_test_##name(fs_info, folio, start, len); \ +} \ +void btrfs_meta_folio_set_##name(struct folio *folio, const struct extent_buffer *eb) \ +{ \ + if (!btrfs_meta_is_subpage(eb->fs_info)) { \ + folio_set_func(folio); \ + return; \ + } \ + btrfs_subpage_set_##name(eb->fs_info, folio, eb->start, eb->len); \ +} \ +void btrfs_meta_folio_clear_##name(struct folio *folio, const struct extent_buffer *eb) \ +{ \ + if (!btrfs_meta_is_subpage(eb->fs_info)) { \ + folio_clear_func(folio); \ + return; \ + } \ + btrfs_subpage_clear_##name(eb->fs_info, folio, eb->start, eb->len); \ +} \ +bool btrfs_meta_folio_test_##name(struct folio *folio, const struct extent_buffer *eb) \ +{ \ + if (!btrfs_meta_is_subpage(eb->fs_info)) \ + return folio_test_func(folio); \ + return btrfs_subpage_test_##name(eb->fs_info, folio, eb->start, eb->len); \ +} +IMPLEMENT_BTRFS_PAGE_OPS(uptodate, folio_mark_uptodate, folio_clear_uptodate, + folio_test_uptodate); +IMPLEMENT_BTRFS_PAGE_OPS(dirty, btrfs_folio_mark_dirty_reserved, + folio_clear_dirty_for_io, folio_test_dirty); +IMPLEMENT_BTRFS_PAGE_OPS(writeback, folio_start_writeback, folio_end_writeback, + folio_test_writeback); + +#define DEFINE_GET_SUBPAGE_BITMAP(name) \ +static inline unsigned long get_bitmap_value_##name( \ + const struct btrfs_fs_info *fs_info, \ + struct folio *folio) \ +{ \ + const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ + const struct btrfs_folio_state *__bfs = folio_get_private(folio); \ + unsigned long value; \ + \ + ASSERT(__bpf <= BITS_PER_LONG); \ + value = bitmap_read(__bfs->bitmaps, __bpf * btrfs_bitmap_nr_##name, \ + __bpf); \ + return value; \ +} \ +static inline const unsigned long *get_bitmap_pointer_##name( \ + const struct btrfs_fs_info *fs_info, \ + struct folio *folio) \ +{ \ + const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ + struct btrfs_folio_state *__bfs = folio_get_private(folio); \ + unsigned long *pointer; \ + \ + ASSERT(__bpf >= BITS_PER_LONG); \ + ASSERT(IS_ALIGNED(__bpf, BITS_PER_LONG)); \ + pointer = __bfs->bitmaps + (BIT_WORD(__bpf) * btrfs_bitmap_nr_##name); \ + return pointer; \ +} + +DEFINE_GET_SUBPAGE_BITMAP(uptodate); +DEFINE_GET_SUBPAGE_BITMAP(dirty); +DEFINE_GET_SUBPAGE_BITMAP(writeback); + +#define SUBPAGE_DUMP_BITMAP(fs_info, folio, name, start, len) \ +{ \ + const unsigned int __bpf = btrfs_blocks_per_folio(fs_info, folio); \ + \ + if (__bpf <= BITS_PER_LONG) { \ + unsigned long bitmap = get_bitmap_value_##name(fs_info, folio); \ + \ + btrfs_warn(fs_info, \ + "dumping bitmap start=%llu len=%u folio=%llu " #name "_bitmap=%*pbl", \ + start, len, folio_pos(folio), __bpf, &bitmap); \ + } else { \ + btrfs_warn(fs_info, \ + "dumping bitmap start=%llu len=%u folio=%llu " #name "_bitmap=%*pbl", \ + start, len, folio_pos(folio), __bpf, \ + get_bitmap_pointer_##name(fs_info, folio)); \ + } \ +} + +/* + * Make sure not only the page dirty bit is cleared, but also subpage dirty bit + * is cleared. + */ +void btrfs_folio_assert_not_dirty(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len) +{ + struct btrfs_folio_state *bfs; + unsigned int start_bit; + unsigned int nbits; + unsigned long flags; + + if (!IS_ENABLED(CONFIG_BTRFS_ASSERT)) + return; + + if (!btrfs_is_subpage(fs_info, folio)) { + ASSERT(!folio_test_dirty(folio)); + return; + } + + start_bit = subpage_calc_start_bit(fs_info, folio, dirty, start, len); + nbits = len >> fs_info->sectorsize_bits; + bfs = folio_get_private(folio); + ASSERT(bfs); + spin_lock_irqsave(&bfs->lock, flags); + if (unlikely(!bitmap_test_range_all_zero(bfs->bitmaps, start_bit, nbits))) { + SUBPAGE_DUMP_BITMAP(fs_info, folio, dirty, start, len); + ASSERT(bitmap_test_range_all_zero(bfs->bitmaps, start_bit, nbits)); + } + ASSERT(bitmap_test_range_all_zero(bfs->bitmaps, start_bit, nbits)); + spin_unlock_irqrestore(&bfs->lock, flags); +} + +/* + * This is for folio already locked by plain lock_page()/folio_lock(), which + * doesn't have any subpage awareness. + * + * This populates the involved subpage ranges so that subpage helpers can + * properly unlock them. + */ +void btrfs_folio_set_lock(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len) +{ + struct btrfs_folio_state *bfs; + unsigned long flags; + unsigned int nbits; + int ret; + + ASSERT(folio_test_locked(folio)); + if (unlikely(!fs_info) || !btrfs_is_subpage(fs_info, folio)) + return; + + bfs = folio_get_private(folio); + nbits = len >> fs_info->sectorsize_bits; + spin_lock_irqsave(&bfs->lock, flags); + ret = atomic_add_return(nbits, &bfs->nr_locked); + ASSERT(ret <= btrfs_blocks_per_folio(fs_info, folio)); + spin_unlock_irqrestore(&bfs->lock, flags); +} + +/* + * Clear the dirty flag for the folio. + * + * If the affected folio is no longer dirty, return true. Otherwise return false. + */ +bool btrfs_meta_folio_clear_and_test_dirty(struct folio *folio, const struct extent_buffer *eb) +{ + bool last; + + if (!btrfs_meta_is_subpage(eb->fs_info)) { + folio_clear_dirty_for_io(folio); + return true; + } + + last = btrfs_subpage_clear_and_test_dirty(eb->fs_info, folio, eb->start, eb->len); + if (last) { + folio_clear_dirty_for_io(folio); + return true; + } + return false; +} + +void __cold btrfs_subpage_dump_bitmap(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len) +{ + struct btrfs_folio_state *bfs; + const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio); + unsigned long flags; + + ASSERT(folio_test_private(folio) && folio_get_private(folio)); + ASSERT(blocks_per_folio > 1); + bfs = folio_get_private(folio); + + dump_page(folio_page(folio, 0), "btrfs folio state dump"); + + if (blocks_per_folio <= BITS_PER_LONG) { + unsigned long uptodate; + unsigned long dirty; + unsigned long writeback; + + spin_lock_irqsave(&bfs->lock, flags); + uptodate = get_bitmap_value_uptodate(fs_info, folio); + dirty = get_bitmap_value_dirty(fs_info, folio); + writeback = get_bitmap_value_writeback(fs_info, folio); + + spin_unlock_irqrestore(&bfs->lock, flags); + + btrfs_warn(fs_info, +"start=%llu len=%u page=%llu, bitmaps uptodate=%*pbl dirty=%*pbl writeback=%*pbl", + start, len, folio_pos(folio), + blocks_per_folio, &uptodate, + blocks_per_folio, &dirty, + blocks_per_folio, &writeback); + return; + } + + spin_lock_irqsave(&bfs->lock, flags); + btrfs_warn(fs_info, +"start=%llu len=%u page=%llu, bitmaps uptodate=%*pbl dirty=%*pbl writeback=%*pbl", + start, len, folio_pos(folio), + blocks_per_folio, get_bitmap_pointer_uptodate(fs_info, folio), + blocks_per_folio, get_bitmap_pointer_dirty(fs_info, folio), + blocks_per_folio, get_bitmap_pointer_writeback(fs_info, folio)); + spin_unlock_irqrestore(&bfs->lock, flags); +} + +void btrfs_copy_subpage_dirty_bitmap(struct btrfs_fs_info *fs_info, + struct folio *folio, + unsigned long *dst) +{ + struct btrfs_folio_state *bfs; + const unsigned int blocks_per_folio = btrfs_blocks_per_folio(fs_info, folio); + unsigned long flags; + unsigned long value; + + if (blocks_per_folio == 1) { + value = 1; + bitmap_copy(dst, &value, 1); + return; + } + + ASSERT(folio_test_private(folio) && folio_get_private(folio)); + ASSERT(blocks_per_folio > 1); + bfs = folio_get_private(folio); + + if (blocks_per_folio <= BITS_PER_LONG) { + spin_lock_irqsave(&bfs->lock, flags); + value = bitmap_read(bfs->bitmaps, btrfs_bitmap_nr_dirty * blocks_per_folio, + blocks_per_folio); + spin_unlock_irqrestore(&bfs->lock, flags); + bitmap_copy(dst, &value, blocks_per_folio); + return; + } + spin_lock_irqsave(&bfs->lock, flags); + bitmap_copy(dst, get_bitmap_pointer_dirty(fs_info, folio), + blocks_per_folio); + spin_unlock_irqrestore(&bfs->lock, flags); +} diff --git a/fs/btrfs/subpage.h b/fs/btrfs/subpage.h new file mode 100644 index 000000000..9b106a73d --- /dev/null +++ b/fs/btrfs/subpage.h @@ -0,0 +1,225 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_SUBPAGE_H +#define BTRFS_SUBPAGE_H + +#include +#include +#include +#include "btrfs_inode.h" + +struct address_space; +struct folio; + +/* + * Extra info for subpage bitmap. + * + * For subpage we pack all uptodate/dirty/writeback/fixup bitmaps into + * one larger bitmap. + * + * This structure records how they are organized in the bitmap: + * + * /- uptodate /- dirty /- writeback /- fixup + * | | | | + * v v v v + * |u|u|u|u|........|u|u|d|d|.......|d|d|w|w|.....|w|w|f|f|.....|f|f| + * |< sectors_per_page >| + * + * Unlike regular macro-like enums, here we do not go upper-case names, as + * these names will be utilized in various macros to define function names. + */ +enum { + btrfs_bitmap_nr_uptodate = 0, + btrfs_bitmap_nr_dirty, + + /* + * This can be changed to atomic eventually. But this change will rely + * on the async delalloc range rework for locked bitmap. As async + * delalloc can unlock its range and mark blocks writeback at random + * timing. + */ + btrfs_bitmap_nr_writeback, + + /* + * Blocks dirtied by the dirty_folio callback instead of a reserving + * write path (e.g. set_page_dirty_lock() on a GUP pin). They have + * no space reservation and need the writepage fixup before they can + * be submitted. + */ + btrfs_bitmap_nr_fixup, + + btrfs_bitmap_nr_max +}; + +/* + * Structure to trace status of each sector inside a page, attached to + * page::private for both data and metadata inodes. + */ +struct btrfs_folio_state { + /* Common members for both data and metadata pages */ + spinlock_t lock; + union { + /* + * Structures only used by metadata + * + * @eb_refs should only be operated under private_lock, as it + * manages whether the btrfs_folio_state can be detached. + */ + atomic_t eb_refs; + + /* + * Structures only used by data, + * + * How many sectors inside the page is locked. + */ + atomic_t nr_locked; + }; + unsigned long bitmaps[]; +}; + +enum btrfs_folio_type { + BTRFS_SUBPAGE_METADATA, + BTRFS_SUBPAGE_DATA, +}; + +/* + * Subpage support for metadata is more complex, as we can have dummy extent + * buffers, where folios have no mapping to determine the owning inode. + * + * Thankfully we only need to check if node size is smaller than page size. + * Even with larger folio support, we will only allocate a folio as large as + * node size. + * Thus if nodesize < PAGE_SIZE, we know metadata needs need to subpage routine. + */ +static inline bool btrfs_meta_is_subpage(const struct btrfs_fs_info *fs_info) +{ + return fs_info->nodesize < PAGE_SIZE; +} +static inline bool btrfs_is_subpage(const struct btrfs_fs_info *fs_info, + struct folio *folio) +{ + if (folio->mapping && folio->mapping->host) + ASSERT(is_data_inode(BTRFS_I(folio->mapping->host))); + return fs_info->sectorsize < folio_size(folio); +} + +int btrfs_attach_folio_state(const struct btrfs_fs_info *fs_info, + struct folio *folio, enum btrfs_folio_type type); +void btrfs_detach_folio_state(const struct btrfs_fs_info *fs_info, struct folio *folio, + enum btrfs_folio_type type); + +/* Allocate additional data where page represents more than one sector */ +struct btrfs_folio_state *btrfs_alloc_folio_state(const struct btrfs_fs_info *fs_info, + size_t fsize, enum btrfs_folio_type type, + gfp_t gfp); +static inline void btrfs_free_folio_state(struct btrfs_folio_state *bfs) +{ + kfree(bfs); +} + +void btrfs_folio_inc_eb_refs(const struct btrfs_fs_info *fs_info, struct folio *folio); +void btrfs_folio_dec_eb_refs(const struct btrfs_fs_info *fs_info, struct folio *folio); + +void btrfs_folio_end_lock(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len); +void btrfs_folio_set_lock(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len); +void btrfs_folio_end_lock_bitmap(const struct btrfs_fs_info *fs_info, + struct folio *folio, unsigned long *bitmap); +/* + * Template for subpage related operations. + * + * btrfs_subpage_*() are for call sites where the folio has subpage attached and + * the range is ensured to be inside the folio's single page. + * + * btrfs_folio_*() are for call sites where the page can either be subpage + * specific or regular folios. The function will handle both cases. + * But the range still needs to be inside one single page. + * + * btrfs_folio_clamp_*() are similar to btrfs_folio_*(), except the range doesn't + * need to be inside the page. Those functions will truncate the range + * automatically. + * + * Both btrfs_folio_*() and btrfs_folio_clamp_*() are for data folios. + * + * For metadata, one should use btrfs_meta_folio_*() helpers instead, and there + * is no clamp version for metadata helpers, as we either go subpage + * (nodesize < PAGE_SIZE) or go regular folio helpers (nodesize >= PAGE_SIZE, + * and our folio is never larger than nodesize). + */ +#define DECLARE_BTRFS_SUBPAGE_OPS(name) \ +void btrfs_subpage_set_##name(const struct btrfs_fs_info *fs_info, \ + struct folio *folio, u64 start, u32 len); \ +void btrfs_subpage_clear_##name(const struct btrfs_fs_info *fs_info, \ + struct folio *folio, u64 start, u32 len); \ +bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info, \ + struct folio *folio, u64 start, u32 len); \ +void btrfs_folio_set_##name(const struct btrfs_fs_info *fs_info, \ + struct folio *folio, u64 start, u32 len); \ +void btrfs_folio_clear_##name(const struct btrfs_fs_info *fs_info, \ + struct folio *folio, u64 start, u32 len); \ +bool btrfs_folio_test_##name(const struct btrfs_fs_info *fs_info, \ + struct folio *folio, u64 start, u32 len); \ +void btrfs_folio_clamp_set_##name(const struct btrfs_fs_info *fs_info, \ + struct folio *folio, u64 start, u32 len); \ +void btrfs_folio_clamp_clear_##name(const struct btrfs_fs_info *fs_info, \ + struct folio *folio, u64 start, u32 len); \ +bool btrfs_folio_clamp_test_##name(const struct btrfs_fs_info *fs_info, \ + struct folio *folio, u64 start, u32 len); \ +void btrfs_meta_folio_set_##name(struct folio *folio, const struct extent_buffer *eb); \ +void btrfs_meta_folio_clear_##name(struct folio *folio, const struct extent_buffer *eb); \ +bool btrfs_meta_folio_test_##name(struct folio *folio, const struct extent_buffer *eb); + +DECLARE_BTRFS_SUBPAGE_OPS(uptodate); +DECLARE_BTRFS_SUBPAGE_OPS(dirty); +DECLARE_BTRFS_SUBPAGE_OPS(writeback); + +/* + * Fixup bit helpers. + * + * The fixup bit is data-only and has no plain set helper (setting happens + * together with dirtying in btrfs_subpage_set_fixup_dirty()), so it does not + * go through DECLARE_BTRFS_SUBPAGE_OPS(). For single-block folios the + * folio_*_fixup_pending() flag takes the place of the bitmap. + */ +void btrfs_subpage_clear_fixup(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len); +bool btrfs_subpage_test_fixup(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len); +bool btrfs_folio_test_fixup(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len); +void btrfs_folio_set_fixup_dirty(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len); +/* For a block that just got its space reserved; it stays dirty. */ +void btrfs_folio_clear_fixup(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len); +/* For callers discarding the data; clears the dirty bits too. */ +void btrfs_folio_clear_fixup_dirty(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len); + +/* + * Helper for error cleanup, where a folio will have its dirty flag cleared, + * with writeback started and finished. + */ +static inline void btrfs_folio_clamp_finish_io(struct btrfs_fs_info *fs_info, + struct folio *locked_folio, + u64 start, u32 len) +{ + btrfs_folio_clamp_clear_dirty(fs_info, locked_folio, start, len); + btrfs_folio_clamp_set_writeback(fs_info, locked_folio, start, len); + btrfs_folio_clamp_clear_writeback(fs_info, locked_folio, start, len); +} + +bool btrfs_subpage_clear_and_test_dirty(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len); + +void btrfs_folio_assert_not_dirty(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len); +bool btrfs_meta_folio_clear_and_test_dirty(struct folio *folio, const struct extent_buffer *eb); +void btrfs_copy_subpage_dirty_bitmap(struct btrfs_fs_info *fs_info, + struct folio *folio, + unsigned long *dst); +void __cold btrfs_subpage_dump_bitmap(const struct btrfs_fs_info *fs_info, + struct folio *folio, u64 start, u32 len); + +#endif diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c new file mode 100644 index 000000000..464129b1b --- /dev/null +++ b/fs/btrfs/super.c @@ -0,0 +1,2744 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "messages.h" +#include "delayed-inode.h" +#include "ctree.h" +#include "disk-io.h" +#include "transaction.h" +#include "btrfs_inode.h" +#include "direct-io.h" +#include "props.h" +#include "xattr.h" +#include "bio.h" +#include "export.h" +#include "compression.h" +#include "dev-replace.h" +#include "free-space-cache.h" +#include "backref.h" +#include "space-info.h" +#include "sysfs.h" +#include "zoned.h" +#include "tests/btrfs-tests.h" +#include "block-group.h" +#include "discard.h" +#include "qgroup.h" +#include "raid56.h" +#include "fs.h" +#include "accessors.h" +#include "defrag.h" +#include "dir-item.h" +#include "ioctl.h" +#include "scrub.h" +#include "verity.h" +#include "super.h" +#include "extent-tree.h" +#include "tree-log.h" +#define CREATE_TRACE_POINTS +#include + +static const struct super_operations btrfs_super_ops; +static struct file_system_type btrfs_fs_type; + +static void btrfs_put_super(struct super_block *sb) +{ + struct btrfs_fs_info *fs_info = btrfs_sb(sb); + + btrfs_info(fs_info, "last unmount of filesystem %pU", fs_info->fs_devices->fsid); + close_ctree(fs_info); +} + +/* Store the mount options related information. */ +struct btrfs_fs_context { + char *subvol_name; + u64 subvol_objectid; + u64 max_inline; + u32 commit_interval; + u32 metadata_ratio; + u32 thread_pool_size; + unsigned long long mount_opt; + unsigned long compress_type:4; + int compress_level; + refcount_t refs; +}; + +static void btrfs_emit_options(struct btrfs_fs_info *info, + struct btrfs_fs_context *old); + +enum { + Opt_acl, + Opt_clear_cache, + Opt_commit_interval, + Opt_compress, + Opt_compress_force, + Opt_compress_force_type, + Opt_compress_type, + Opt_degraded, + Opt_device, + Opt_fatal_errors, + Opt_flushoncommit, + Opt_max_inline, + Opt_barrier, + Opt_datacow, + Opt_datasum, + Opt_defrag, + Opt_discard, + Opt_discard_mode, + Opt_ratio, + Opt_rescan_uuid_tree, + Opt_skip_balance, + Opt_space_cache, + Opt_space_cache_version, + Opt_ssd, + Opt_ssd_spread, + Opt_subvol, + Opt_subvol_empty, + Opt_subvolid, + Opt_thread_pool, + Opt_treelog, + Opt_user_subvol_rm_allowed, + Opt_norecovery, + + /* Rescue options */ + Opt_rescue, + + /* Debugging options */ + Opt_enospc_debug, +#ifdef CONFIG_BTRFS_DEBUG + Opt_fragment, Opt_fragment_data, Opt_fragment_metadata, Opt_fragment_all, + Opt_ref_verify, + Opt_ref_tracker, +#endif + Opt_err, +}; + +enum { + Opt_fatal_errors_panic, + Opt_fatal_errors_bug, +}; + +static const struct constant_table btrfs_parameter_fatal_errors[] = { + { "panic", Opt_fatal_errors_panic }, + { "bug", Opt_fatal_errors_bug }, + {} +}; + +enum { + Opt_discard_sync, + Opt_discard_async, +}; + +static const struct constant_table btrfs_parameter_discard[] = { + { "sync", Opt_discard_sync }, + { "async", Opt_discard_async }, + {} +}; + +enum { + Opt_space_cache_v1, + Opt_space_cache_v2, +}; + +static const struct constant_table btrfs_parameter_space_cache[] = { + { "v1", Opt_space_cache_v1 }, + { "v2", Opt_space_cache_v2 }, + {} +}; + +enum { + Opt_rescue_usebackuproot, + Opt_rescue_nologreplay, + Opt_rescue_ignorebadroots, + Opt_rescue_ignoredatacsums, + Opt_rescue_ignoremetacsums, + Opt_rescue_ignoresuperflags, + Opt_rescue_parameter_all, +}; + +static const struct constant_table btrfs_parameter_rescue[] = { + { "usebackuproot", Opt_rescue_usebackuproot }, + { "nologreplay", Opt_rescue_nologreplay }, + { "ignorebadroots", Opt_rescue_ignorebadroots }, + { "ibadroots", Opt_rescue_ignorebadroots }, + { "ignoredatacsums", Opt_rescue_ignoredatacsums }, + { "ignoremetacsums", Opt_rescue_ignoremetacsums}, + { "ignoresuperflags", Opt_rescue_ignoresuperflags}, + { "idatacsums", Opt_rescue_ignoredatacsums }, + { "imetacsums", Opt_rescue_ignoremetacsums}, + { "isuperflags", Opt_rescue_ignoresuperflags}, + { "all", Opt_rescue_parameter_all }, + {} +}; + +#ifdef CONFIG_BTRFS_DEBUG +enum { + Opt_fragment_parameter_data, + Opt_fragment_parameter_metadata, + Opt_fragment_parameter_all, +}; + +static const struct constant_table btrfs_parameter_fragment[] = { + { "data", Opt_fragment_parameter_data }, + { "metadata", Opt_fragment_parameter_metadata }, + { "all", Opt_fragment_parameter_all }, + {} +}; +#endif + +static const struct fs_parameter_spec btrfs_fs_parameters[] = { + fsparam_flag_no("acl", Opt_acl), + fsparam_flag_no("autodefrag", Opt_defrag), + fsparam_flag_no("barrier", Opt_barrier), + fsparam_flag("clear_cache", Opt_clear_cache), + fsparam_u32("commit", Opt_commit_interval), + fsparam_flag("compress", Opt_compress), + fsparam_string("compress", Opt_compress_type), + fsparam_flag("compress-force", Opt_compress_force), + fsparam_string("compress-force", Opt_compress_force_type), + fsparam_flag_no("datacow", Opt_datacow), + fsparam_flag_no("datasum", Opt_datasum), + fsparam_flag("degraded", Opt_degraded), + fsparam_string("device", Opt_device), + fsparam_flag_no("discard", Opt_discard), + fsparam_enum("discard", Opt_discard_mode, btrfs_parameter_discard), + fsparam_enum("fatal_errors", Opt_fatal_errors, btrfs_parameter_fatal_errors), + fsparam_flag_no("flushoncommit", Opt_flushoncommit), + fsparam_string("max_inline", Opt_max_inline), + fsparam_u32("metadata_ratio", Opt_ratio), + fsparam_flag("rescan_uuid_tree", Opt_rescan_uuid_tree), + fsparam_flag("skip_balance", Opt_skip_balance), + fsparam_flag_no("space_cache", Opt_space_cache), + fsparam_enum("space_cache", Opt_space_cache_version, btrfs_parameter_space_cache), + fsparam_flag_no("ssd", Opt_ssd), + fsparam_flag_no("ssd_spread", Opt_ssd_spread), + fsparam_string("subvol", Opt_subvol), + fsparam_flag("subvol=", Opt_subvol_empty), + fsparam_u64("subvolid", Opt_subvolid), + fsparam_u32("thread_pool", Opt_thread_pool), + fsparam_flag_no("treelog", Opt_treelog), + fsparam_flag("user_subvol_rm_allowed", Opt_user_subvol_rm_allowed), + + /* Rescue options. */ + fsparam_enum("rescue", Opt_rescue, btrfs_parameter_rescue), + /* For compatibility only, alias for "rescue=nologreplay". */ + fsparam_flag("norecovery", Opt_norecovery), + + /* Debugging options. */ + fsparam_flag_no("enospc_debug", Opt_enospc_debug), +#ifdef CONFIG_BTRFS_DEBUG + fsparam_enum("fragment", Opt_fragment, btrfs_parameter_fragment), + fsparam_flag("ref_tracker", Opt_ref_tracker), + fsparam_flag("ref_verify", Opt_ref_verify), +#endif + {} +}; + +static bool btrfs_match_compress_type(const char *string, const char *type, bool may_have_level) +{ + const int len = strlen(type); + + return (strncmp(string, type, len) == 0) && + ((may_have_level && string[len] == ':') || string[len] == '\0'); +} + +static int btrfs_parse_compress(struct btrfs_fs_context *ctx, + const struct fs_parameter *param, int opt) +{ + const char *string = param->string; + int ret; + + /* + * Provide the same semantics as older kernels that don't use fs + * context, specifying the "compress" option clears "force-compress" + * without the need to pass "compress-force=[no|none]" before + * specifying "compress". + */ + if (opt != Opt_compress_force && opt != Opt_compress_force_type) + btrfs_clear_opt(ctx->mount_opt, FORCE_COMPRESS); + + if (opt == Opt_compress || opt == Opt_compress_force) { + ctx->compress_type = BTRFS_COMPRESS_ZLIB; + ctx->compress_level = BTRFS_ZLIB_DEFAULT_LEVEL; + btrfs_set_opt(ctx->mount_opt, COMPRESS); + btrfs_clear_opt(ctx->mount_opt, NODATACOW); + btrfs_clear_opt(ctx->mount_opt, NODATASUM); + } else if (btrfs_match_compress_type(string, "zlib", true)) { + ctx->compress_type = BTRFS_COMPRESS_ZLIB; + ret = btrfs_compress_str2level(BTRFS_COMPRESS_ZLIB, string + 4, + &ctx->compress_level); + if (ret < 0) + goto error; + btrfs_set_opt(ctx->mount_opt, COMPRESS); + btrfs_clear_opt(ctx->mount_opt, NODATACOW); + btrfs_clear_opt(ctx->mount_opt, NODATASUM); + } else if (btrfs_match_compress_type(string, "lzo", true)) { + ctx->compress_type = BTRFS_COMPRESS_LZO; + ret = btrfs_compress_str2level(BTRFS_COMPRESS_LZO, string + 3, + &ctx->compress_level); + if (ret < 0) + goto error; + if (string[3] == ':' && string[4]) + btrfs_warn(NULL, "Compression level ignored for LZO"); + btrfs_set_opt(ctx->mount_opt, COMPRESS); + btrfs_clear_opt(ctx->mount_opt, NODATACOW); + btrfs_clear_opt(ctx->mount_opt, NODATASUM); + } else if (btrfs_match_compress_type(string, "zstd", true)) { + ctx->compress_type = BTRFS_COMPRESS_ZSTD; + ret = btrfs_compress_str2level(BTRFS_COMPRESS_ZSTD, string + 4, + &ctx->compress_level); + if (ret < 0) + goto error; + btrfs_set_opt(ctx->mount_opt, COMPRESS); + btrfs_clear_opt(ctx->mount_opt, NODATACOW); + btrfs_clear_opt(ctx->mount_opt, NODATASUM); + } else if (btrfs_match_compress_type(string, "no", false) || + btrfs_match_compress_type(string, "none", false)) { + ctx->compress_level = 0; + ctx->compress_type = 0; + btrfs_clear_opt(ctx->mount_opt, COMPRESS); + btrfs_clear_opt(ctx->mount_opt, FORCE_COMPRESS); + } else { + ret = -EINVAL; + goto error; + } + return 0; +error: + btrfs_err(NULL, "failed to parse compression option '%s'", string); + return ret; + +} + +static int btrfs_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct btrfs_fs_context *ctx = fc->fs_private; + struct fs_parse_result result; + int opt; + + opt = fs_parse(fc, btrfs_fs_parameters, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_degraded: + btrfs_set_opt(ctx->mount_opt, DEGRADED); + break; + case Opt_subvol_empty: + /* + * This exists because we used to allow it on accident, so we're + * keeping it to maintain ABI. See 37becec95ac3 ("Btrfs: allow + * empty subvol= again"). + */ + break; + case Opt_subvol: + kfree(ctx->subvol_name); + ctx->subvol_name = kstrdup(param->string, GFP_KERNEL); + if (!ctx->subvol_name) + return -ENOMEM; + break; + case Opt_subvolid: + ctx->subvol_objectid = result.uint_64; + + /* subvolid=0 means give me the original fs_tree. */ + if (!ctx->subvol_objectid) + ctx->subvol_objectid = BTRFS_FS_TREE_OBJECTID; + break; + case Opt_device: { + struct btrfs_device *device; + + mutex_lock(&uuid_mutex); + device = btrfs_scan_one_device(param->string, false); + mutex_unlock(&uuid_mutex); + if (IS_ERR(device)) + return PTR_ERR(device); + break; + } + case Opt_datasum: + if (result.negated) { + btrfs_set_opt(ctx->mount_opt, NODATASUM); + } else { + btrfs_clear_opt(ctx->mount_opt, NODATACOW); + btrfs_clear_opt(ctx->mount_opt, NODATASUM); + } + break; + case Opt_datacow: + if (result.negated) { + btrfs_clear_opt(ctx->mount_opt, COMPRESS); + btrfs_clear_opt(ctx->mount_opt, FORCE_COMPRESS); + btrfs_set_opt(ctx->mount_opt, NODATACOW); + btrfs_set_opt(ctx->mount_opt, NODATASUM); + } else { + btrfs_clear_opt(ctx->mount_opt, NODATACOW); + } + break; + case Opt_compress_force: + case Opt_compress_force_type: + btrfs_set_opt(ctx->mount_opt, FORCE_COMPRESS); + fallthrough; + case Opt_compress: + case Opt_compress_type: + if (btrfs_parse_compress(ctx, param, opt)) + return -EINVAL; + break; + case Opt_ssd: + if (result.negated) { + btrfs_set_opt(ctx->mount_opt, NOSSD); + btrfs_clear_opt(ctx->mount_opt, SSD); + btrfs_clear_opt(ctx->mount_opt, SSD_SPREAD); + } else { + btrfs_set_opt(ctx->mount_opt, SSD); + btrfs_clear_opt(ctx->mount_opt, NOSSD); + } + break; + case Opt_ssd_spread: + if (result.negated) { + btrfs_clear_opt(ctx->mount_opt, SSD_SPREAD); + } else { + btrfs_set_opt(ctx->mount_opt, SSD); + btrfs_set_opt(ctx->mount_opt, SSD_SPREAD); + btrfs_clear_opt(ctx->mount_opt, NOSSD); + } + break; + case Opt_barrier: + if (result.negated) + btrfs_set_opt(ctx->mount_opt, NOBARRIER); + else + btrfs_clear_opt(ctx->mount_opt, NOBARRIER); + break; + case Opt_thread_pool: + if (result.uint_32 == 0) { + btrfs_err(NULL, "invalid value 0 for thread_pool"); + return -EINVAL; + } + ctx->thread_pool_size = result.uint_32; + break; + case Opt_max_inline: + ctx->max_inline = memparse(param->string, NULL); + break; + case Opt_acl: + if (result.negated) { + fc->sb_flags &= ~SB_POSIXACL; + } else { +#ifdef CONFIG_BTRFS_FS_POSIX_ACL + fc->sb_flags |= SB_POSIXACL; +#else + btrfs_err(NULL, "support for ACL not compiled in"); + return -EINVAL; +#endif + } + /* + * VFS limits the ability to toggle ACL on and off via remount, + * despite every file system allowing this. This seems to be + * an oversight since we all do, but it'll fail if we're + * remounting. So don't set the mask here, we'll check it in + * btrfs_reconfigure and do the toggling ourselves. + */ + if (fc->purpose != FS_CONTEXT_FOR_RECONFIGURE) + fc->sb_flags_mask |= SB_POSIXACL; + break; + case Opt_treelog: + if (result.negated) + btrfs_set_opt(ctx->mount_opt, NOTREELOG); + else + btrfs_clear_opt(ctx->mount_opt, NOTREELOG); + break; + case Opt_norecovery: + btrfs_info(NULL, +"'norecovery' is for compatibility only, recommended to use 'rescue=nologreplay'"); + btrfs_set_opt(ctx->mount_opt, NOLOGREPLAY); + break; + case Opt_flushoncommit: + if (result.negated) + btrfs_clear_opt(ctx->mount_opt, FLUSHONCOMMIT); + else + btrfs_set_opt(ctx->mount_opt, FLUSHONCOMMIT); + break; + case Opt_ratio: + ctx->metadata_ratio = result.uint_32; + break; + case Opt_discard: + if (result.negated) { + btrfs_clear_opt(ctx->mount_opt, DISCARD_SYNC); + btrfs_clear_opt(ctx->mount_opt, DISCARD_ASYNC); + btrfs_set_opt(ctx->mount_opt, NODISCARD); + } else { + btrfs_set_opt(ctx->mount_opt, DISCARD_SYNC); + btrfs_clear_opt(ctx->mount_opt, DISCARD_ASYNC); + } + break; + case Opt_discard_mode: + switch (result.uint_32) { + case Opt_discard_sync: + btrfs_clear_opt(ctx->mount_opt, DISCARD_ASYNC); + btrfs_set_opt(ctx->mount_opt, DISCARD_SYNC); + break; + case Opt_discard_async: + btrfs_clear_opt(ctx->mount_opt, DISCARD_SYNC); + btrfs_set_opt(ctx->mount_opt, DISCARD_ASYNC); + break; + default: + btrfs_err(NULL, "unrecognized discard mode value %s", + param->key); + return -EINVAL; + } + btrfs_clear_opt(ctx->mount_opt, NODISCARD); + break; + case Opt_space_cache: + if (!result.negated) + btrfs_warn(NULL, + "v1 space cache is deprecated, falling back to no space cache"); + btrfs_set_opt(ctx->mount_opt, NOSPACECACHE); + btrfs_clear_opt(ctx->mount_opt, SPACE_CACHE); + btrfs_clear_opt(ctx->mount_opt, FREE_SPACE_TREE); + break; + case Opt_space_cache_version: + switch (result.uint_32) { + case Opt_space_cache_v1: + btrfs_warn(NULL, + "v1 space cache is deprecated, falling back to no space cache"); + btrfs_set_opt(ctx->mount_opt, NOSPACECACHE); + btrfs_clear_opt(ctx->mount_opt, SPACE_CACHE); + btrfs_clear_opt(ctx->mount_opt, FREE_SPACE_TREE); + break; + case Opt_space_cache_v2: + btrfs_clear_opt(ctx->mount_opt, SPACE_CACHE); + btrfs_set_opt(ctx->mount_opt, FREE_SPACE_TREE); + break; + default: + btrfs_err(NULL, "unrecognized space_cache value %s", + param->key); + return -EINVAL; + } + break; + case Opt_rescan_uuid_tree: + btrfs_set_opt(ctx->mount_opt, RESCAN_UUID_TREE); + break; + case Opt_clear_cache: + btrfs_set_opt(ctx->mount_opt, CLEAR_CACHE); + break; + case Opt_user_subvol_rm_allowed: + btrfs_set_opt(ctx->mount_opt, USER_SUBVOL_RM_ALLOWED); + break; + case Opt_enospc_debug: + if (result.negated) + btrfs_clear_opt(ctx->mount_opt, ENOSPC_DEBUG); + else + btrfs_set_opt(ctx->mount_opt, ENOSPC_DEBUG); + break; + case Opt_defrag: + if (result.negated) + btrfs_clear_opt(ctx->mount_opt, AUTO_DEFRAG); + else + btrfs_set_opt(ctx->mount_opt, AUTO_DEFRAG); + break; + case Opt_skip_balance: + btrfs_set_opt(ctx->mount_opt, SKIP_BALANCE); + break; + case Opt_fatal_errors: + switch (result.uint_32) { + case Opt_fatal_errors_panic: + btrfs_set_opt(ctx->mount_opt, PANIC_ON_FATAL_ERROR); + break; + case Opt_fatal_errors_bug: + btrfs_clear_opt(ctx->mount_opt, PANIC_ON_FATAL_ERROR); + break; + default: + btrfs_err(NULL, "unrecognized fatal_errors value %s", + param->key); + return -EINVAL; + } + break; + case Opt_commit_interval: + ctx->commit_interval = result.uint_32; + if (ctx->commit_interval > BTRFS_WARNING_COMMIT_INTERVAL) { + btrfs_warn(NULL, "excessive commit interval %u, use with care", + ctx->commit_interval); + } + if (ctx->commit_interval == 0) + ctx->commit_interval = BTRFS_DEFAULT_COMMIT_INTERVAL; + break; + case Opt_rescue: + switch (result.uint_32) { + case Opt_rescue_usebackuproot: + btrfs_set_opt(ctx->mount_opt, USEBACKUPROOT); + break; + case Opt_rescue_nologreplay: + btrfs_set_opt(ctx->mount_opt, NOLOGREPLAY); + break; + case Opt_rescue_ignorebadroots: + btrfs_set_opt(ctx->mount_opt, IGNOREBADROOTS); + break; + case Opt_rescue_ignoredatacsums: + btrfs_set_opt(ctx->mount_opt, IGNOREDATACSUMS); + break; + case Opt_rescue_ignoremetacsums: + btrfs_set_opt(ctx->mount_opt, IGNOREMETACSUMS); + break; + case Opt_rescue_ignoresuperflags: + btrfs_set_opt(ctx->mount_opt, IGNORESUPERFLAGS); + break; + case Opt_rescue_parameter_all: + btrfs_set_opt(ctx->mount_opt, IGNOREDATACSUMS); + btrfs_set_opt(ctx->mount_opt, IGNOREMETACSUMS); + btrfs_set_opt(ctx->mount_opt, IGNORESUPERFLAGS); + btrfs_set_opt(ctx->mount_opt, IGNOREBADROOTS); + btrfs_set_opt(ctx->mount_opt, NOLOGREPLAY); + btrfs_set_opt(ctx->mount_opt, USEBACKUPROOT); + break; + default: + btrfs_info(NULL, "unrecognized rescue option '%s'", + param->key); + return -EINVAL; + } + break; +#ifdef CONFIG_BTRFS_DEBUG + case Opt_fragment: + switch (result.uint_32) { + case Opt_fragment_parameter_all: + btrfs_set_opt(ctx->mount_opt, FRAGMENT_DATA); + btrfs_set_opt(ctx->mount_opt, FRAGMENT_METADATA); + break; + case Opt_fragment_parameter_metadata: + btrfs_set_opt(ctx->mount_opt, FRAGMENT_METADATA); + break; + case Opt_fragment_parameter_data: + btrfs_set_opt(ctx->mount_opt, FRAGMENT_DATA); + break; + default: + btrfs_info(NULL, "unrecognized fragment option '%s'", + param->key); + return -EINVAL; + } + break; + case Opt_ref_verify: + btrfs_set_opt(ctx->mount_opt, REF_VERIFY); + break; + case Opt_ref_tracker: + btrfs_set_opt(ctx->mount_opt, REF_TRACKER); + break; +#endif + default: + btrfs_err(NULL, "unrecognized mount option '%s'", param->key); + return -EINVAL; + } + + return 0; +} + +/* + * Some options only have meaning at mount time and shouldn't persist across + * remounts, or be displayed. Clear these at the end of mount and remount code + * paths. + */ +static void btrfs_clear_oneshot_options(struct btrfs_fs_info *fs_info) +{ + btrfs_clear_opt(fs_info->mount_opt, CLEAR_CACHE); + btrfs_clear_opt(fs_info->mount_opt, NOSPACECACHE); +} + +static bool check_ro_option(const struct btrfs_fs_info *fs_info, + unsigned long long mount_opt, unsigned long long opt, + const char *opt_name) +{ + if (mount_opt & opt) { + btrfs_err(fs_info, "%s must be used with ro mount option", + opt_name); + return true; + } + return false; +} + +bool btrfs_check_options(const struct btrfs_fs_info *info, + unsigned long long *mount_opt, + unsigned long flags) +{ + bool ret = true; + + if (!(flags & SB_RDONLY) && + (check_ro_option(info, *mount_opt, BTRFS_MOUNT_USEBACKUPROOT, "usebackuproot") || + check_ro_option(info, *mount_opt, BTRFS_MOUNT_NOLOGREPLAY, "nologreplay") || + check_ro_option(info, *mount_opt, BTRFS_MOUNT_IGNOREBADROOTS, "ignorebadroots") || + check_ro_option(info, *mount_opt, BTRFS_MOUNT_IGNOREDATACSUMS, "ignoredatacsums") || + check_ro_option(info, *mount_opt, BTRFS_MOUNT_IGNOREMETACSUMS, "ignoremetacsums") || + check_ro_option(info, *mount_opt, BTRFS_MOUNT_IGNORESUPERFLAGS, "ignoresuperflags"))) + ret = false; + + if (btrfs_fs_compat_ro(info, FREE_SPACE_TREE) && + !btrfs_raw_test_opt(*mount_opt, FREE_SPACE_TREE) && + !btrfs_raw_test_opt(*mount_opt, CLEAR_CACHE)) { + btrfs_err(info, "cannot disable free-space-tree"); + ret = false; + } + if (btrfs_fs_compat_ro(info, BLOCK_GROUP_TREE) && + !btrfs_raw_test_opt(*mount_opt, FREE_SPACE_TREE)) { + btrfs_err(info, "cannot disable free-space-tree with block-group-tree feature"); + ret = false; + } + + if (btrfs_check_mountopts_zoned(info, mount_opt)) + ret = false; + + if (!test_bit(BTRFS_FS_STATE_REMOUNTING, &info->fs_state)) { + if (btrfs_raw_test_opt(*mount_opt, SPACE_CACHE)) { + btrfs_warn(info, +"space cache v1 is being deprecated and will be removed in a future release, please use -o space_cache=v2"); + } + } + + return ret; +} + +/* + * This is subtle, we only call this during open_ctree(). We need to pre-load + * the mount options with the on-disk settings. Before the new mount API took + * effect we would do this on mount and remount. With the new mount API we'll + * only do this on the initial mount. + * + * This isn't a change in behavior, because we're using the current state of the + * file system to set the current mount options. If you mounted with special + * options to disable these features and then remounted we wouldn't revert the + * settings, because mounting without these features cleared the on-disk + * settings, so this being called on re-mount is not needed. + */ +void btrfs_set_free_space_cache_settings(struct btrfs_fs_info *fs_info) +{ + if (fs_info->sectorsize != PAGE_SIZE && btrfs_test_opt(fs_info, SPACE_CACHE)) { + btrfs_info(fs_info, + "forcing free space tree for sector size %u with page size %lu", + fs_info->sectorsize, PAGE_SIZE); + btrfs_clear_opt(fs_info->mount_opt, SPACE_CACHE); + btrfs_set_opt(fs_info->mount_opt, FREE_SPACE_TREE); + } + + /* + * At this point our mount options are populated, so we only mess with + * these settings if we don't have any settings already. + */ + if (btrfs_test_opt(fs_info, FREE_SPACE_TREE)) + return; + + if (btrfs_is_zoned(fs_info) && + btrfs_free_space_cache_v1_active(fs_info)) { + btrfs_info(fs_info, "zoned: clearing existing space cache"); + btrfs_set_super_cache_generation(fs_info->super_copy, 0); + return; + } + + if (btrfs_test_opt(fs_info, SPACE_CACHE)) + return; + + if (btrfs_test_opt(fs_info, NOSPACECACHE)) + return; + + /* + * At this point we don't have explicit options set by the user, set + * them ourselves based on the state of the file system. + */ + if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) + btrfs_set_opt(fs_info->mount_opt, FREE_SPACE_TREE); + else if (btrfs_free_space_cache_v1_active(fs_info)) + btrfs_set_opt(fs_info->mount_opt, SPACE_CACHE); +} + +static void set_device_specific_options(struct btrfs_fs_info *fs_info) +{ + if (!btrfs_test_opt(fs_info, NOSSD) && + !fs_info->fs_devices->rotating) + btrfs_set_opt(fs_info->mount_opt, SSD); + + /* + * For devices supporting discard turn on discard=async automatically, + * unless it's already set or disabled. This could be turned off by + * nodiscard for the same mount. + * + * The zoned mode piggy backs on the discard functionality for + * resetting a zone. There is no reason to delay the zone reset as it is + * fast enough. So, do not enable async discard for zoned mode. + */ + if (!(btrfs_test_opt(fs_info, DISCARD_SYNC) || + btrfs_test_opt(fs_info, DISCARD_ASYNC) || + btrfs_test_opt(fs_info, NODISCARD)) && + fs_info->fs_devices->discardable && + !btrfs_is_zoned(fs_info)) + btrfs_set_opt(fs_info->mount_opt, DISCARD_ASYNC); +} + +char *btrfs_get_subvol_name_from_objectid(struct btrfs_fs_info *fs_info, + u64 subvol_objectid) +{ + struct btrfs_root *root = fs_info->tree_root; + struct btrfs_root *fs_root = NULL; + struct btrfs_root_ref *root_ref; + struct btrfs_inode_ref *inode_ref; + struct btrfs_key key; + BTRFS_PATH_AUTO_FREE(path); + char *name = NULL, *ptr; + u64 dirid; + int len; + int ret; + + path = btrfs_alloc_path(); + if (!path) + return ERR_PTR(-ENOMEM); + + name = kmalloc(PATH_MAX, GFP_KERNEL); + if (!name) { + ret = -ENOMEM; + goto err; + } + ptr = name + PATH_MAX - 1; + ptr[0] = '\0'; + + /* + * Walk up the subvolume trees in the tree of tree roots by root + * backrefs until we hit the top-level subvolume. + */ + while (subvol_objectid != BTRFS_FS_TREE_OBJECTID) { + key.objectid = subvol_objectid; + key.type = BTRFS_ROOT_BACKREF_KEY; + key.offset = (u64)-1; + + ret = btrfs_search_backwards(root, &key, path); + if (ret < 0) { + goto err; + } else if (ret > 0) { + ret = -ENOENT; + goto err; + } + + subvol_objectid = key.offset; + + root_ref = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_root_ref); + len = btrfs_root_ref_name_len(path->nodes[0], root_ref); + ptr -= len + 1; + if (ptr < name) { + ret = -ENAMETOOLONG; + goto err; + } + read_extent_buffer(path->nodes[0], ptr + 1, + (unsigned long)(root_ref + 1), len); + ptr[0] = '/'; + dirid = btrfs_root_ref_dirid(path->nodes[0], root_ref); + btrfs_release_path(path); + + fs_root = btrfs_get_fs_root(fs_info, subvol_objectid, true); + if (IS_ERR(fs_root)) { + ret = PTR_ERR(fs_root); + fs_root = NULL; + goto err; + } + + /* + * Walk up the filesystem tree by inode refs until we hit the + * root directory. + */ + while (dirid != BTRFS_FIRST_FREE_OBJECTID) { + key.objectid = dirid; + key.type = BTRFS_INODE_REF_KEY; + key.offset = (u64)-1; + + ret = btrfs_search_backwards(fs_root, &key, path); + if (ret < 0) { + goto err; + } else if (ret > 0) { + ret = -ENOENT; + goto err; + } + + dirid = key.offset; + + inode_ref = btrfs_item_ptr(path->nodes[0], + path->slots[0], + struct btrfs_inode_ref); + len = btrfs_inode_ref_name_len(path->nodes[0], + inode_ref); + ptr -= len + 1; + if (ptr < name) { + ret = -ENAMETOOLONG; + goto err; + } + read_extent_buffer(path->nodes[0], ptr + 1, + (unsigned long)(inode_ref + 1), len); + ptr[0] = '/'; + btrfs_release_path(path); + } + btrfs_put_root(fs_root); + fs_root = NULL; + } + + if (ptr == name + PATH_MAX - 1) { + name[0] = '/'; + name[1] = '\0'; + } else { + memmove(name, ptr, name + PATH_MAX - ptr); + } + return name; + +err: + btrfs_put_root(fs_root); + kfree(name); + return ERR_PTR(ret); +} + +static int get_default_subvol_objectid(struct btrfs_fs_info *fs_info, u64 *objectid) +{ + struct btrfs_root *root = fs_info->tree_root; + struct btrfs_dir_item *di; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key location; + struct fscrypt_str name = FSTR_INIT("default", 7); + u64 dir_id; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + /* + * Find the "default" dir item which points to the root item that we + * will mount by default if we haven't been given a specific subvolume + * to mount. + */ + dir_id = btrfs_super_root_dir(fs_info->super_copy); + di = btrfs_lookup_dir_item(NULL, root, path, dir_id, &name, 0); + if (IS_ERR(di)) { + return PTR_ERR(di); + } + if (!di) { + /* + * Ok the default dir item isn't there. This is weird since + * it's always been there, but don't freak out, just try and + * mount the top-level subvolume. + */ + *objectid = BTRFS_FS_TREE_OBJECTID; + return 0; + } + + btrfs_dir_item_key_to_cpu(path->nodes[0], di, &location); + *objectid = location.objectid; + return 0; +} + +static int btrfs_fill_super(struct super_block *sb, + struct btrfs_fs_devices *fs_devices) +{ + struct btrfs_inode *inode; + struct btrfs_fs_info *fs_info = btrfs_sb(sb); + int ret; + + sb->s_maxbytes = MAX_LFS_FILESIZE; + sb->s_magic = BTRFS_SUPER_MAGIC; + sb->s_op = &btrfs_super_ops; + set_default_d_op(sb, &btrfs_dentry_operations); + sb->s_export_op = &btrfs_export_ops; +#ifdef CONFIG_FS_VERITY + sb->s_vop = &btrfs_verityops; +#endif + sb->s_xattr = btrfs_xattr_handlers; + sb->s_time_gran = 1; + sb->s_iflags |= SB_I_CGROUPWB | SB_I_ALLOW_HSM; + + ret = super_setup_bdi(sb); + if (ret) { + btrfs_err(fs_info, "super_setup_bdi failed"); + return ret; + } + + ret = open_ctree(sb, fs_devices); + if (ret) { + btrfs_err(fs_info, "open_ctree failed: %pe", ERR_PTR(ret)); + return ret; + } + + btrfs_emit_options(fs_info, NULL); + + inode = btrfs_iget(BTRFS_FIRST_FREE_OBJECTID, fs_info->fs_root); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + btrfs_handle_fs_error(fs_info, ret, NULL); + goto fail_close; + } + + sb->s_root = d_make_root(&inode->vfs_inode); + if (!sb->s_root) { + ret = -ENOMEM; + goto fail_close; + } + + sb->s_flags |= SB_ACTIVE; + return 0; + +fail_close: + close_ctree(fs_info); + return ret; +} + +int btrfs_sync_fs(struct super_block *sb, int wait) +{ + struct btrfs_trans_handle *trans; + struct btrfs_fs_info *fs_info = btrfs_sb(sb); + struct btrfs_root *root = fs_info->tree_root; + + trace_btrfs_sync_fs(fs_info, wait); + + if (!wait) { + filemap_flush(fs_info->btree_inode->i_mapping); + return 0; + } + + btrfs_wait_ordered_roots(fs_info, U64_MAX, NULL); + + trans = btrfs_attach_transaction_barrier(root); + if (IS_ERR(trans)) { + /* no transaction, don't bother */ + if (PTR_ERR(trans) == -ENOENT) { + /* + * Exit unless we have some pending changes + * that need to go through commit + */ + if (!test_bit(BTRFS_FS_NEED_TRANS_COMMIT, + &fs_info->flags)) + return 0; + /* + * A non-blocking test if the fs is frozen. We must not + * start a new transaction here otherwise a deadlock + * happens. The pending operations are delayed to the + * next commit after thawing. + */ + if (sb_start_write_trylock(sb)) + sb_end_write(sb); + else + return 0; + trans = btrfs_start_transaction(root, 0); + } + if (IS_ERR(trans)) + return PTR_ERR(trans); + } + return btrfs_commit_transaction(trans); +} + +static void print_rescue_option(struct seq_file *seq, const char *s, bool *printed) +{ + seq_printf(seq, "%s%s", (*printed) ? ":" : ",rescue=", s); + *printed = true; +} + +static int btrfs_show_options(struct seq_file *seq, struct dentry *dentry) +{ + struct btrfs_fs_info *info = btrfs_sb(dentry->d_sb); + const char *compress_type; + const char *subvol_name; + bool printed = false; + + if (btrfs_test_opt(info, DEGRADED)) + seq_puts(seq, ",degraded"); + if (btrfs_test_opt(info, NODATASUM)) + seq_puts(seq, ",nodatasum"); + if (btrfs_test_opt(info, NODATACOW)) + seq_puts(seq, ",nodatacow"); + if (btrfs_test_opt(info, NOBARRIER)) + seq_puts(seq, ",nobarrier"); + if (info->max_inline != BTRFS_DEFAULT_MAX_INLINE) + seq_printf(seq, ",max_inline=%llu", info->max_inline); + if (info->thread_pool_size != min_t(unsigned long, + num_online_cpus() + 2, 8)) + seq_printf(seq, ",thread_pool=%u", info->thread_pool_size); + if (btrfs_test_opt(info, COMPRESS)) { + compress_type = btrfs_compress_type2str(info->compress_type); + if (btrfs_test_opt(info, FORCE_COMPRESS)) + seq_printf(seq, ",compress-force=%s", compress_type); + else + seq_printf(seq, ",compress=%s", compress_type); + if (info->compress_level && info->compress_type != BTRFS_COMPRESS_LZO) + seq_printf(seq, ":%d", info->compress_level); + } + if (btrfs_test_opt(info, NOSSD)) + seq_puts(seq, ",nossd"); + if (btrfs_test_opt(info, SSD_SPREAD)) + seq_puts(seq, ",ssd_spread"); + else if (btrfs_test_opt(info, SSD)) + seq_puts(seq, ",ssd"); + if (btrfs_test_opt(info, NOTREELOG)) + seq_puts(seq, ",notreelog"); + if (btrfs_test_opt(info, NOLOGREPLAY)) + print_rescue_option(seq, "nologreplay", &printed); + if (btrfs_test_opt(info, USEBACKUPROOT)) + print_rescue_option(seq, "usebackuproot", &printed); + if (btrfs_test_opt(info, IGNOREBADROOTS)) + print_rescue_option(seq, "ignorebadroots", &printed); + if (btrfs_test_opt(info, IGNOREDATACSUMS)) + print_rescue_option(seq, "ignoredatacsums", &printed); + if (btrfs_test_opt(info, IGNOREMETACSUMS)) + print_rescue_option(seq, "ignoremetacsums", &printed); + if (btrfs_test_opt(info, IGNORESUPERFLAGS)) + print_rescue_option(seq, "ignoresuperflags", &printed); + if (btrfs_test_opt(info, FLUSHONCOMMIT)) + seq_puts(seq, ",flushoncommit"); + if (btrfs_test_opt(info, DISCARD_SYNC)) + seq_puts(seq, ",discard"); + if (btrfs_test_opt(info, DISCARD_ASYNC)) + seq_puts(seq, ",discard=async"); + if (!(info->sb->s_flags & SB_POSIXACL)) + seq_puts(seq, ",noacl"); + if (btrfs_free_space_cache_v1_active(info)) + seq_puts(seq, ",space_cache"); + else if (btrfs_fs_compat_ro(info, FREE_SPACE_TREE)) + seq_puts(seq, ",space_cache=v2"); + else + seq_puts(seq, ",nospace_cache"); + if (btrfs_test_opt(info, RESCAN_UUID_TREE)) + seq_puts(seq, ",rescan_uuid_tree"); + if (btrfs_test_opt(info, CLEAR_CACHE)) + seq_puts(seq, ",clear_cache"); + if (btrfs_test_opt(info, USER_SUBVOL_RM_ALLOWED)) + seq_puts(seq, ",user_subvol_rm_allowed"); + if (btrfs_test_opt(info, ENOSPC_DEBUG)) + seq_puts(seq, ",enospc_debug"); + if (btrfs_test_opt(info, AUTO_DEFRAG)) + seq_puts(seq, ",autodefrag"); + if (btrfs_test_opt(info, SKIP_BALANCE)) + seq_puts(seq, ",skip_balance"); + if (info->metadata_ratio) + seq_printf(seq, ",metadata_ratio=%u", info->metadata_ratio); + if (btrfs_test_opt(info, PANIC_ON_FATAL_ERROR)) + seq_puts(seq, ",fatal_errors=panic"); + if (info->commit_interval != BTRFS_DEFAULT_COMMIT_INTERVAL) + seq_printf(seq, ",commit=%u", info->commit_interval); +#ifdef CONFIG_BTRFS_DEBUG + if (btrfs_test_opt(info, FRAGMENT_DATA)) + seq_puts(seq, ",fragment=data"); + if (btrfs_test_opt(info, FRAGMENT_METADATA)) + seq_puts(seq, ",fragment=metadata"); +#endif + if (btrfs_test_opt(info, REF_VERIFY)) + seq_puts(seq, ",ref_verify"); + if (btrfs_test_opt(info, REF_TRACKER)) + seq_puts(seq, ",ref_tracker"); + seq_printf(seq, ",subvolid=%llu", btrfs_root_id(BTRFS_I(d_inode(dentry))->root)); + subvol_name = btrfs_get_subvol_name_from_objectid(info, + btrfs_root_id(BTRFS_I(d_inode(dentry))->root)); + if (!IS_ERR(subvol_name)) { + seq_show_option(seq, "subvol", subvol_name); + kfree(subvol_name); + } + return 0; +} + +/* + * subvolumes are identified by ino 256 + */ +static inline bool is_subvolume_inode(struct inode *inode) +{ + if (inode && inode->i_ino == BTRFS_FIRST_FREE_OBJECTID) + return true; + return false; +} + +static struct dentry *mount_subvol(const char *subvol_name, u64 subvol_objectid, + struct vfsmount *mnt) +{ + struct dentry *root; + int ret; + + if (!subvol_name) { + if (!subvol_objectid) { + ret = get_default_subvol_objectid(btrfs_sb(mnt->mnt_sb), + &subvol_objectid); + if (ret) { + root = ERR_PTR(ret); + goto out; + } + } + subvol_name = btrfs_get_subvol_name_from_objectid( + btrfs_sb(mnt->mnt_sb), subvol_objectid); + if (IS_ERR(subvol_name)) { + root = ERR_CAST(subvol_name); + subvol_name = NULL; + goto out; + } + + } + + root = mount_subtree(mnt, subvol_name); + /* mount_subtree() drops our reference on the vfsmount. */ + mnt = NULL; + + if (!IS_ERR(root)) { + struct super_block *s = root->d_sb; + struct btrfs_fs_info *fs_info = btrfs_sb(s); + struct inode *root_inode = d_inode(root); + u64 root_objectid = btrfs_root_id(BTRFS_I(root_inode)->root); + + ret = 0; + if (!is_subvolume_inode(root_inode)) { + btrfs_err(fs_info, "'%s' is not a valid subvolume", + subvol_name); + ret = -EINVAL; + } + if (subvol_objectid && root_objectid != subvol_objectid) { + /* + * This will also catch a race condition where a + * subvolume which was passed by ID is renamed and + * another subvolume is renamed over the old location. + */ + btrfs_err(fs_info, + "subvol '%s' does not match subvolid %llu", + subvol_name, subvol_objectid); + ret = -EINVAL; + } + if (ret) { + dput(root); + root = ERR_PTR(ret); + deactivate_locked_super(s); + } + } + +out: + mntput(mnt); + kfree(subvol_name); + return root; +} + +static void btrfs_resize_thread_pool(struct btrfs_fs_info *fs_info, + u32 new_pool_size, u32 old_pool_size) +{ + if (new_pool_size == old_pool_size) + return; + + fs_info->thread_pool_size = new_pool_size; + + btrfs_info(fs_info, "resize thread pool %d -> %d", + old_pool_size, new_pool_size); + + btrfs_workqueue_set_max(fs_info->workers, new_pool_size); + btrfs_workqueue_set_max(fs_info->delalloc_workers, new_pool_size); + btrfs_workqueue_set_max(fs_info->caching_workers, new_pool_size); + workqueue_set_max_active(fs_info->endio_workers, new_pool_size); + workqueue_set_max_active(fs_info->endio_meta_workers, new_pool_size); + btrfs_workqueue_set_max(fs_info->endio_write_workers, new_pool_size); + btrfs_workqueue_set_max(fs_info->endio_freespace_worker, new_pool_size); + btrfs_workqueue_set_max(fs_info->delayed_workers, new_pool_size); +} + +static inline void btrfs_remount_begin(struct btrfs_fs_info *fs_info, + unsigned long long old_opts, int flags) +{ + if (btrfs_raw_test_opt(old_opts, AUTO_DEFRAG) && + (!btrfs_raw_test_opt(fs_info->mount_opt, AUTO_DEFRAG) || + (flags & SB_RDONLY))) { + /* wait for any defraggers to finish */ + wait_event(fs_info->transaction_wait, + (atomic_read(&fs_info->defrag_running) == 0)); + if (flags & SB_RDONLY) + sync_filesystem(fs_info->sb); + } +} + +static inline void btrfs_remount_cleanup(struct btrfs_fs_info *fs_info, + unsigned long long old_opts) +{ + const bool cache_opt = btrfs_test_opt(fs_info, SPACE_CACHE); + + /* + * We need to cleanup all defraggable inodes if the autodefragment is + * close or the filesystem is read only. + */ + if (btrfs_raw_test_opt(old_opts, AUTO_DEFRAG) && + (!btrfs_raw_test_opt(fs_info->mount_opt, AUTO_DEFRAG) || sb_rdonly(fs_info->sb))) { + btrfs_cleanup_defrag_inodes(fs_info); + } + + /* If we toggled discard async */ + if (!btrfs_raw_test_opt(old_opts, DISCARD_ASYNC) && + btrfs_test_opt(fs_info, DISCARD_ASYNC)) + btrfs_discard_resume(fs_info); + else if (btrfs_raw_test_opt(old_opts, DISCARD_ASYNC) && + !btrfs_test_opt(fs_info, DISCARD_ASYNC)) + btrfs_discard_cleanup(fs_info); + + /* If we toggled space cache */ + if (cache_opt != btrfs_free_space_cache_v1_active(fs_info)) + btrfs_set_free_space_cache_v1_active(fs_info, cache_opt); +} + +static int btrfs_remount_rw(struct btrfs_fs_info *fs_info) +{ + int ret; + + if (unlikely(BTRFS_FS_ERROR(fs_info))) { + btrfs_err(fs_info, + "remounting read-write after error is not allowed"); + return -EINVAL; + } + + if (fs_info->fs_devices->rw_devices == 0) + return -EACCES; + + if (!btrfs_check_rw_degradable(fs_info, NULL)) { + btrfs_warn(fs_info, + "too many missing devices, writable remount is not allowed"); + return -EACCES; + } + + if (btrfs_super_log_root(fs_info->super_copy) != 0) { + btrfs_warn(fs_info, + "mount required to replay tree-log, cannot remount read-write"); + return -EINVAL; + } + + /* + * NOTE: when remounting with a change that does writes, don't put it + * anywhere above this point, as we are not sure to be safe to write + * until we pass the above checks. + */ + ret = btrfs_start_pre_rw_mount(fs_info); + if (ret) + return ret; + + btrfs_clear_sb_rdonly(fs_info->sb); + + set_bit(BTRFS_FS_OPEN, &fs_info->flags); + + /* + * If we've gone from readonly -> read-write, we need to get our + * sync/async discard lists in the right state. + */ + btrfs_discard_resume(fs_info); + + return 0; +} + +static int btrfs_remount_ro(struct btrfs_fs_info *fs_info) +{ + /* + * This also happens on 'umount -rf' or on shutdown, when the + * filesystem is busy. + */ + cancel_work_sync(&fs_info->async_reclaim_work); + cancel_work_sync(&fs_info->async_data_reclaim_work); + + btrfs_discard_cleanup(fs_info); + + /* Wait for the uuid_scan task to finish */ + down(&fs_info->uuid_tree_rescan_sem); + /* Avoid complains from lockdep et al. */ + up(&fs_info->uuid_tree_rescan_sem); + + btrfs_set_sb_rdonly(fs_info->sb); + + /* + * Setting SB_RDONLY will put the cleaner thread to sleep at the next + * loop if it's already active. If it's already asleep, we'll leave + * unused block groups on disk until we're mounted read-write again + * unless we clean them up here. + */ + btrfs_delete_unused_bgs(fs_info); + + /* + * The cleaner task could be already running before we set the flag + * BTRFS_FS_STATE_RO (and SB_RDONLY in the superblock). We must make + * sure that after we finish the remount, i.e. after we call + * btrfs_commit_super(), the cleaner can no longer start a transaction + * - either because it was dropping a dead root, running delayed iputs + * or deleting an unused block group (the cleaner picked a block + * group from the list of unused block groups before we were able to + * in the previous call to btrfs_delete_unused_bgs()). + */ + wait_on_bit(&fs_info->flags, BTRFS_FS_CLEANER_RUNNING, TASK_UNINTERRUPTIBLE); + + /* + * We've set the superblock to RO mode, so we might have made the + * cleaner task sleep without running all pending delayed iputs. Go + * through all the delayed iputs here, so that if an unmount happens + * without remounting RW we don't end up at finishing close_ctree() + * with a non-empty list of delayed iputs. + */ + btrfs_run_delayed_iputs(fs_info); + + btrfs_dev_replace_suspend_for_unmount(fs_info); + btrfs_scrub_cancel(fs_info); + btrfs_pause_balance(fs_info); + + /* + * Pause the qgroup rescan worker if it is running. We don't want it to + * be still running after we are in RO mode, as after that, by the time + * we unmount, it might have left a transaction open, so we would leak + * the transaction and/or crash. + */ + btrfs_qgroup_wait_for_completion(fs_info, false); + + return btrfs_commit_super(fs_info); +} + +static void btrfs_ctx_to_info(struct btrfs_fs_info *fs_info, struct btrfs_fs_context *ctx) +{ + fs_info->max_inline = ctx->max_inline; + fs_info->commit_interval = ctx->commit_interval; + fs_info->metadata_ratio = ctx->metadata_ratio; + fs_info->thread_pool_size = ctx->thread_pool_size; + fs_info->mount_opt = ctx->mount_opt; + fs_info->compress_type = ctx->compress_type; + fs_info->compress_level = ctx->compress_level; +} + +static void btrfs_info_to_ctx(struct btrfs_fs_info *fs_info, struct btrfs_fs_context *ctx) +{ + ctx->max_inline = fs_info->max_inline; + ctx->commit_interval = fs_info->commit_interval; + ctx->metadata_ratio = fs_info->metadata_ratio; + ctx->thread_pool_size = fs_info->thread_pool_size; + ctx->mount_opt = fs_info->mount_opt; + ctx->compress_type = fs_info->compress_type; + ctx->compress_level = fs_info->compress_level; +} + +#define btrfs_info_if_set(fs_info, old_ctx, opt, fmt, args...) \ +do { \ + if ((!old_ctx || !btrfs_raw_test_opt(old_ctx->mount_opt, opt)) && \ + btrfs_raw_test_opt(fs_info->mount_opt, opt)) \ + btrfs_info(fs_info, fmt, ##args); \ +} while (0) + +#define btrfs_info_if_unset(fs_info, old_ctx, opt, fmt, args...) \ +do { \ + if ((old_ctx && btrfs_raw_test_opt(old_ctx->mount_opt, opt)) && \ + !btrfs_raw_test_opt(fs_info->mount_opt, opt)) \ + btrfs_info(fs_info, fmt, ##args); \ +} while (0) + +static void btrfs_emit_options(struct btrfs_fs_info *info, + struct btrfs_fs_context *old) +{ + btrfs_info_if_set(info, old, NODATASUM, "setting nodatasum"); + btrfs_info_if_set(info, old, DEGRADED, "allowing degraded mounts"); + btrfs_info_if_set(info, old, NODATACOW, "setting nodatacow"); + btrfs_info_if_set(info, old, SSD, "enabling ssd optimizations"); + btrfs_info_if_set(info, old, SSD_SPREAD, "using spread ssd allocation scheme"); + btrfs_info_if_set(info, old, NOBARRIER, "turning off barriers"); + btrfs_info_if_set(info, old, NOTREELOG, "disabling tree log"); + btrfs_info_if_set(info, old, NOLOGREPLAY, "disabling log replay at mount time"); + btrfs_info_if_set(info, old, FLUSHONCOMMIT, "turning on flush-on-commit"); + btrfs_info_if_set(info, old, DISCARD_SYNC, "turning on sync discard"); + btrfs_info_if_set(info, old, DISCARD_ASYNC, "turning on async discard"); + btrfs_info_if_set(info, old, FREE_SPACE_TREE, "enabling free space tree"); + btrfs_info_if_set(info, old, SPACE_CACHE, "enabling disk space caching"); + btrfs_info_if_set(info, old, CLEAR_CACHE, "force clearing of disk cache"); + btrfs_info_if_set(info, old, AUTO_DEFRAG, "enabling auto defrag"); + btrfs_info_if_set(info, old, FRAGMENT_DATA, "fragmenting data"); + btrfs_info_if_set(info, old, FRAGMENT_METADATA, "fragmenting metadata"); + btrfs_info_if_set(info, old, REF_VERIFY, "doing ref verification"); + btrfs_info_if_set(info, old, USEBACKUPROOT, "trying to use backup root at mount time"); + btrfs_info_if_set(info, old, IGNOREBADROOTS, "ignoring bad roots"); + btrfs_info_if_set(info, old, IGNOREDATACSUMS, "ignoring data csums"); + btrfs_info_if_set(info, old, IGNOREMETACSUMS, "ignoring meta csums"); + btrfs_info_if_set(info, old, IGNORESUPERFLAGS, "ignoring unknown super block flags"); + + btrfs_info_if_unset(info, old, NODATASUM, "setting datasum"); + btrfs_info_if_unset(info, old, NODATACOW, "setting datacow"); + btrfs_info_if_unset(info, old, SSD, "not using ssd optimizations"); + btrfs_info_if_unset(info, old, SSD_SPREAD, "not using spread ssd allocation scheme"); + btrfs_info_if_unset(info, old, NOBARRIER, "turning on barriers"); + btrfs_info_if_unset(info, old, NOTREELOG, "enabling tree log"); + btrfs_info_if_unset(info, old, SPACE_CACHE, "disabling disk space caching"); + btrfs_info_if_unset(info, old, FREE_SPACE_TREE, "disabling free space tree"); + btrfs_info_if_unset(info, old, AUTO_DEFRAG, "disabling auto defrag"); + btrfs_info_if_unset(info, old, COMPRESS, "use no compression"); + + /* Did the compression settings change? */ + if (btrfs_test_opt(info, COMPRESS) && + (!old || + old->compress_type != info->compress_type || + old->compress_level != info->compress_level || + (!btrfs_raw_test_opt(old->mount_opt, FORCE_COMPRESS) && + btrfs_raw_test_opt(info->mount_opt, FORCE_COMPRESS)))) { + const char *compress_type = btrfs_compress_type2str(info->compress_type); + + btrfs_info(info, "%s %s compression, level %d", + btrfs_test_opt(info, FORCE_COMPRESS) ? "force" : "use", + compress_type, info->compress_level); + } + + if (info->max_inline != BTRFS_DEFAULT_MAX_INLINE) + btrfs_info(info, "max_inline set to %llu", info->max_inline); +} + +static int btrfs_reconfigure(struct fs_context *fc) +{ + struct super_block *sb = fc->root->d_sb; + struct btrfs_fs_info *fs_info = btrfs_sb(sb); + struct btrfs_fs_context *ctx = fc->fs_private; + struct btrfs_fs_context old_ctx; + int ret = 0; + bool mount_reconfigure = (fc->s_fs_info != NULL); + + btrfs_info_to_ctx(fs_info, &old_ctx); + + /* + * This is our "bind mount" trick, we don't want to allow the user to do + * anything other than mount a different ro/rw and a different subvol, + * all of the mount options should be maintained. + */ + if (mount_reconfigure) + ctx->mount_opt = old_ctx.mount_opt; + + sync_filesystem(sb); + set_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state); + + if (!btrfs_check_options(fs_info, &ctx->mount_opt, fc->sb_flags)) { + ret = -EINVAL; + goto restore; + } + + ret = btrfs_check_features(fs_info, !(fc->sb_flags & SB_RDONLY)); + if (ret < 0) + goto restore; + + btrfs_ctx_to_info(fs_info, ctx); + btrfs_remount_begin(fs_info, old_ctx.mount_opt, fc->sb_flags); + btrfs_resize_thread_pool(fs_info, fs_info->thread_pool_size, + old_ctx.thread_pool_size); + + if ((bool)btrfs_test_opt(fs_info, FREE_SPACE_TREE) != + (bool)btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE) && + (!sb_rdonly(sb) || (fc->sb_flags & SB_RDONLY))) { + btrfs_warn(fs_info, + "remount supports changing free space tree only from RO to RW"); + /* Make sure free space cache options match the state on disk. */ + if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE)) { + btrfs_set_opt(fs_info->mount_opt, FREE_SPACE_TREE); + btrfs_clear_opt(fs_info->mount_opt, SPACE_CACHE); + } + if (btrfs_free_space_cache_v1_active(fs_info)) { + btrfs_clear_opt(fs_info->mount_opt, FREE_SPACE_TREE); + btrfs_set_opt(fs_info->mount_opt, SPACE_CACHE); + } + } + + ret = 0; + if (!sb_rdonly(sb) && (fc->sb_flags & SB_RDONLY)) + ret = btrfs_remount_ro(fs_info); + else if (sb_rdonly(sb) && !(fc->sb_flags & SB_RDONLY)) + ret = btrfs_remount_rw(fs_info); + if (ret) + goto restore; + + /* + * If we set the mask during the parameter parsing VFS would reject the + * remount. Here we can set the mask and the value will be updated + * appropriately. + */ + if ((fc->sb_flags & SB_POSIXACL) != (sb->s_flags & SB_POSIXACL)) + fc->sb_flags_mask |= SB_POSIXACL; + + btrfs_emit_options(fs_info, &old_ctx); + wake_up_process(fs_info->transaction_kthread); + btrfs_remount_cleanup(fs_info, old_ctx.mount_opt); + btrfs_clear_oneshot_options(fs_info); + clear_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state); + + return 0; +restore: + btrfs_ctx_to_info(fs_info, &old_ctx); + btrfs_remount_cleanup(fs_info, old_ctx.mount_opt); + clear_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state); + return ret; +} + +/* Used to sort the devices by max_avail(descending sort) */ +static int btrfs_cmp_device_free_bytes(const void *a, const void *b) +{ + const struct btrfs_device_info *dev_info1 = a; + const struct btrfs_device_info *dev_info2 = b; + + if (dev_info1->max_avail > dev_info2->max_avail) + return -1; + else if (dev_info1->max_avail < dev_info2->max_avail) + return 1; + return 0; +} + +/* + * sort the devices by max_avail, in which max free extent size of each device + * is stored.(Descending Sort) + */ +static inline void btrfs_descending_sort_devices( + struct btrfs_device_info *devices, + size_t nr_devices) +{ + sort(devices, nr_devices, sizeof(struct btrfs_device_info), + btrfs_cmp_device_free_bytes, NULL); +} + +/* + * The helper to calc the free space on the devices that can be used to store + * file data. + */ +static inline int btrfs_calc_avail_data_space(struct btrfs_fs_info *fs_info, + u64 *free_bytes) +{ + struct btrfs_device_info AUTO_KFREE(devices_info); + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; + struct btrfs_device *device; + u64 type; + u64 avail_space; + u64 min_stripe_size; + int num_stripes = 1; + int i = 0, nr_devices; + const struct btrfs_raid_attr *rattr; + + /* + * We aren't under the device list lock, so this is racy-ish, but good + * enough for our purposes. + */ + nr_devices = fs_info->fs_devices->open_devices; + if (!nr_devices) { + smp_mb(); + nr_devices = fs_info->fs_devices->open_devices; + ASSERT(nr_devices); + if (!nr_devices) { + *free_bytes = 0; + return 0; + } + } + + devices_info = kmalloc_objs(*devices_info, nr_devices); + if (!devices_info) + return -ENOMEM; + + /* calc min stripe number for data space allocation */ + type = btrfs_data_alloc_profile(fs_info); + rattr = &btrfs_raid_array[btrfs_bg_flags_to_raid_index(type)]; + + if (type & BTRFS_BLOCK_GROUP_RAID0) + num_stripes = nr_devices; + else if (type & BTRFS_BLOCK_GROUP_RAID1_MASK) + num_stripes = rattr->ncopies; + else if (type & BTRFS_BLOCK_GROUP_RAID10) + num_stripes = 4; + + /* Adjust for more than 1 stripe per device */ + min_stripe_size = rattr->dev_stripes * BTRFS_STRIPE_LEN; + + rcu_read_lock(); + list_for_each_entry_rcu(device, &fs_devices->devices, dev_list) { + if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, + &device->dev_state) || + !device->bdev || + test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) + continue; + + if (i >= nr_devices) + break; + + avail_space = device->total_bytes - device->bytes_used; + + /* align with stripe_len */ + avail_space = rounddown(avail_space, BTRFS_STRIPE_LEN); + + /* + * Ensure we have at least min_stripe_size on top of the + * reserved space on the device. + */ + if (avail_space <= BTRFS_DEVICE_RANGE_RESERVED + min_stripe_size) + continue; + + avail_space -= BTRFS_DEVICE_RANGE_RESERVED; + + devices_info[i].dev = device; + devices_info[i].max_avail = avail_space; + + i++; + } + rcu_read_unlock(); + + nr_devices = i; + + btrfs_descending_sort_devices(devices_info, nr_devices); + + i = nr_devices - 1; + avail_space = 0; + while (nr_devices >= rattr->devs_min) { + num_stripes = min(num_stripes, nr_devices); + + if (devices_info[i].max_avail >= min_stripe_size) { + int j; + u64 alloc_size; + + avail_space += devices_info[i].max_avail * num_stripes; + alloc_size = devices_info[i].max_avail; + for (j = i + 1 - num_stripes; j <= i; j++) + devices_info[j].max_avail -= alloc_size; + } + i--; + nr_devices--; + } + + *free_bytes = avail_space; + return 0; +} + +/* + * Calculate numbers for 'df', pessimistic in case of mixed raid profiles. + * + * If there's a redundant raid level at DATA block groups, use the respective + * multiplier to scale the sizes. + * + * Unused device space usage is based on simulating the chunk allocator + * algorithm that respects the device sizes and order of allocations. This is + * a close approximation of the actual use but there are other factors that may + * change the result (like a new metadata chunk). + * + * If metadata is exhausted, f_bavail will be 0. + */ +static int btrfs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct btrfs_fs_info *fs_info = btrfs_sb(dentry->d_sb); + struct btrfs_super_block *disk_super = fs_info->super_copy; + struct btrfs_space_info *found; + u64 total_used = 0; + u64 total_free_data = 0; + u64 total_free_meta = 0; + u32 bits = fs_info->sectorsize_bits; + __be32 *fsid; + unsigned factor = 1; + struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv; + int ret; + u64 thresh = 0; + bool mixed = false; + __kernel_fsid_t f_fsid; + + list_for_each_entry(found, &fs_info->space_info, list) { + if (found->flags & BTRFS_BLOCK_GROUP_DATA && + found->subgroup_id != BTRFS_SUB_GROUP_DATA_RELOC) { + int i; + + total_free_data += found->disk_total - found->disk_used; + total_free_data -= + btrfs_account_ro_block_groups_free_space(found); + + for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) { + if (!list_empty(&found->block_groups[i])) + factor = btrfs_bg_type_to_factor( + btrfs_raid_array[i].bg_flag); + } + } + + /* + * Metadata in mixed block group profiles are accounted in data + */ + if (!mixed && found->flags & BTRFS_BLOCK_GROUP_METADATA) { + if (found->flags & BTRFS_BLOCK_GROUP_DATA) + mixed = true; + else + total_free_meta += found->disk_total - + found->disk_used; + } + + total_used += found->disk_used; + } + + buf->f_blocks = div_u64(btrfs_super_total_bytes(disk_super), factor); + buf->f_blocks >>= bits; + buf->f_bfree = buf->f_blocks - (div_u64(total_used, factor) >> bits); + + /* Account global block reserve as used, it's in logical size already */ + spin_lock(&block_rsv->lock); + /* Mixed block groups accounting is not byte-accurate, avoid overflow */ + if (buf->f_bfree >= block_rsv->size >> bits) + buf->f_bfree -= block_rsv->size >> bits; + else + buf->f_bfree = 0; + spin_unlock(&block_rsv->lock); + + buf->f_bavail = div_u64(total_free_data, factor); + ret = btrfs_calc_avail_data_space(fs_info, &total_free_data); + if (ret) + return ret; + buf->f_bavail += div_u64(total_free_data, factor); + buf->f_bavail = buf->f_bavail >> bits; + + /* + * We calculate the remaining metadata space minus global reserve. If + * this is (supposedly) smaller than zero, there's no space. But this + * does not hold in practice, the exhausted state happens where's still + * some positive delta. So we apply some guesswork and compare the + * delta to a 4M threshold. (Practically observed delta was ~2M.) + * + * We probably cannot calculate the exact threshold value because this + * depends on the internal reservations requested by various + * operations, so some operations that consume a few metadata will + * succeed even if the Avail is zero. But this is better than the other + * way around. + */ + thresh = SZ_4M; + + /* + * We only want to claim there's no available space if we can no longer + * allocate chunks for our metadata profile and our global reserve will + * not fit in the free metadata space. If we aren't ->full then we + * still can allocate chunks and thus are fine using the currently + * calculated f_bavail. + */ + if (!mixed && block_rsv->space_info->full && + (total_free_meta < thresh || total_free_meta - thresh < block_rsv->size)) + buf->f_bavail = 0; + + buf->f_type = BTRFS_SUPER_MAGIC; + buf->f_bsize = fs_info->sectorsize; + buf->f_namelen = BTRFS_NAME_LEN; + + /* + * fs_devices->fsid is dynamically generated when temp_fsid is active + * to support cloned filesystems. Use the original on-disk fsid instead, + * as it remains consistent across mount cycles. + */ + if (fs_info->fs_devices->temp_fsid) + fsid = (__be32 *)fs_info->super_copy->fsid; + else + fsid = (__be32 *)fs_info->fs_devices->fsid; + + /* + * We treat it as constant endianness (it doesn't matter _which_) + * because we want the fsid to come out the same whether mounted + * on a big-endian or little-endian host. + */ + f_fsid.val[0] = be32_to_cpu(fsid[0]) ^ be32_to_cpu(fsid[2]); + f_fsid.val[1] = be32_to_cpu(fsid[1]) ^ be32_to_cpu(fsid[3]); + + /* Mask in the root object ID too, to disambiguate subvols */ + f_fsid.val[0] ^= btrfs_root_id(BTRFS_I(d_inode(dentry))->root) >> 32; + f_fsid.val[1] ^= btrfs_root_id(BTRFS_I(d_inode(dentry))->root); + + /* Hash dev_t to avoid f_fsid collision with cloned filesystems. */ + if (fs_info->fs_devices->total_devices == 1) { + __kernel_fsid_t dev_fsid = + u64_to_fsid(huge_encode_dev(fs_info->fs_devices->latest_dev->bdev->bd_dev)); + + f_fsid.val[0] ^= dev_fsid.val[1]; + f_fsid.val[1] ^= dev_fsid.val[0]; + } + + memcpy(&buf->f_fsid, &f_fsid, sizeof(f_fsid)); + + return 0; +} + +static int btrfs_fc_test_super(struct super_block *sb, struct fs_context *fc) +{ + struct btrfs_fs_info *p = fc->s_fs_info; + struct btrfs_fs_info *fs_info = btrfs_sb(sb); + + return fs_info->fs_devices == p->fs_devices; +} + +static int btrfs_get_tree_super(struct fs_context *fc) +{ + struct btrfs_fs_info *fs_info = fc->s_fs_info; + struct btrfs_fs_context *ctx = fc->fs_private; + struct btrfs_fs_devices *fs_devices = NULL; + struct btrfs_device *device; + struct super_block *sb; + blk_mode_t mode = sb_open_mode(fc->sb_flags); + int ret; + + btrfs_ctx_to_info(fs_info, ctx); + mutex_lock(&uuid_mutex); + + /* + * With 'true' passed to btrfs_scan_one_device() (mount time) we expect + * either a valid device or an error. + */ + device = btrfs_scan_one_device(fc->source, true); + ASSERT(device != NULL); + if (IS_ERR(device)) { + mutex_unlock(&uuid_mutex); + return PTR_ERR(device); + } + fs_devices = device->fs_devices; + /* + * We cannot hold uuid_mutex calling sget_fc(), it will lead to a + * locking order reversal with s_umount. + * + * So here we increase the holding number of fs_devices, this will ensure + * the fs_devices itself won't be freed. + */ + btrfs_fs_devices_inc_holding(fs_devices); + fs_info->fs_devices = fs_devices; + mutex_unlock(&uuid_mutex); + + fc->sb_flags |= SB_NOSEC; + + sb = sget_fc(fc, btrfs_fc_test_super, set_anon_super_fc); + if (IS_ERR(sb)) { + mutex_lock(&uuid_mutex); + btrfs_fs_devices_dec_holding(fs_devices); + /* + * Since the fs_devices is not opened, it can be freed at any + * time after unlocking uuid_mutex. We need to avoid double + * free through put_fs_context()->btrfs_free_fs_info(). + * So here we reset fs_info->fs_devices to NULL, and let the + * regular fs_devices reclaim path to handle it. + * + * This applies to all later branches where no fs_devices is + * opened. + */ + fs_info->fs_devices = NULL; + mutex_unlock(&uuid_mutex); + return PTR_ERR(sb); + } + + if (sb->s_root) { + /* + * Not the first mount of the fs thus got an existing super block. + * Will reuse the returned super block, fs_info and fs_devices. + * + * fc->s_fs_info is not touched and will be later freed by + * put_fs_context() through btrfs_free_fs_context(). + */ + ASSERT(fc->s_fs_info == fs_info); + + mutex_lock(&uuid_mutex); + btrfs_fs_devices_dec_holding(fs_devices); + fs_info->fs_devices = NULL; + mutex_unlock(&uuid_mutex); + /* + * At this stage we may have RO flag mismatch between + * fc->sb_flags and sb->s_flags. Caller should detect such + * mismatch and reconfigure with sb->s_umount rwsem held if + * needed. + */ + } else { + struct block_device *bdev; + + /* + * The first mount of the fs thus a new superblock, fc->s_fs_info + * must be NULL, and the ownership of our fs_info and fs_devices is + * transferred to the super block. + */ + ASSERT(fc->s_fs_info == NULL); + + mutex_lock(&uuid_mutex); + btrfs_fs_devices_dec_holding(fs_devices); + ret = btrfs_open_devices(fs_devices, mode, sb); + if (ret < 0) + fs_info->fs_devices = NULL; + mutex_unlock(&uuid_mutex); + if (ret < 0) { + deactivate_locked_super(sb); + return ret; + } + if (!(fc->sb_flags & SB_RDONLY) && fs_devices->rw_devices == 0) { + deactivate_locked_super(sb); + return -EACCES; + } + set_device_specific_options(fs_info); + bdev = fs_devices->latest_dev->bdev; + snprintf(sb->s_id, sizeof(sb->s_id), "%pg", bdev); + shrinker_debugfs_rename(sb->s_shrink, "sb-btrfs:%s", sb->s_id); + ret = btrfs_fill_super(sb, fs_devices); + if (ret) { + deactivate_locked_super(sb); + return ret; + } + } + + btrfs_clear_oneshot_options(fs_info); + + fc->root = dget(sb->s_root); + return 0; +} + +/* + * Ever since commit 0723a0473fb4 ("btrfs: allow mounting btrfs subvolumes + * with different ro/rw options") the following works: + * + * (i) mount /dev/sda3 -o subvol=foo,ro /mnt/foo + * (ii) mount /dev/sda3 -o subvol=bar,rw /mnt/bar + * + * which looks nice and innocent but is actually pretty intricate and deserves + * a long comment. + * + * On another filesystem a subvolume mount is close to something like: + * + * (iii) # create rw superblock + initial mount + * mount -t xfs /dev/sdb /opt/ + * + * # create ro bind mount + * mount --bind -o ro /opt/foo /mnt/foo + * + * # unmount initial mount + * umount /opt + * + * Of course, there's some special subvolume sauce and there's the fact that the + * sb->s_root dentry is really swapped after mount_subtree(). But conceptually + * it's very close and will help us understand the issue. + * + * The old mount API didn't cleanly distinguish between a mount being made ro + * and a superblock being made ro. The only way to change the ro state of + * either object was by passing ms_rdonly. If a new mount was created via + * mount(2) such as: + * + * mount("/dev/sdb", "/mnt", "xfs", ms_rdonly, null); + * + * the MS_RDONLY flag being specified had two effects: + * + * (1) MNT_READONLY was raised -> the resulting mount got + * @mnt->mnt_flags |= MNT_READONLY raised. + * + * (2) MS_RDONLY was passed to the filesystem's mount method and the filesystems + * made the superblock ro. Note, how SB_RDONLY has the same value as + * ms_rdonly and is raised whenever MS_RDONLY is passed through mount(2). + * + * Creating a subtree mount via (iii) ends up leaving a rw superblock with a + * subtree mounted ro. + * + * But consider the effect on the old mount API on btrfs subvolume mounting + * which combines the distinct step in (iii) into a single step. + * + * By issuing (i) both the mount and the superblock are turned ro. Now when (ii) + * is issued the superblock is ro and thus even if the mount created for (ii) is + * rw it wouldn't help. Hence, btrfs needed to transition the superblock from ro + * to rw for (ii) which it did using an internal remount call. + * + * IOW, subvolume mounting was inherently complicated due to the ambiguity of + * MS_RDONLY in mount(2). Note, this ambiguity has mount(8) always translate + * "ro" to MS_RDONLY. IOW, in both (i) and (ii) "ro" becomes MS_RDONLY when + * passed by mount(8) to mount(2). + * + * Enter the new mount API. The new mount API disambiguates making a mount ro + * and making a superblock ro. + * + * (3) To turn a mount ro the MOUNT_ATTR_ONLY flag can be used with either + * fsmount() or mount_setattr() this is a pure VFS level change for a + * specific mount or mount tree that is never seen by the filesystem itself. + * + * (4) To turn a superblock ro the "ro" flag must be used with + * fsconfig(FSCONFIG_SET_FLAG, "ro"). This option is seen by the filesystem + * in fc->sb_flags. + * + * But, currently the util-linux mount command already utilizes the new mount + * API and is still setting fsconfig(FSCONFIG_SET_FLAG, "ro") no matter if it's + * btrfs or not, setting the whole super block RO. To make per-subvolume mounting + * work with different options work we need to keep backward compatibility. + */ +static int btrfs_reconfigure_for_mount(struct fs_context *fc) +{ + int ret = 0; + + if (!(fc->sb_flags & SB_RDONLY) && (fc->root->d_sb->s_flags & SB_RDONLY)) + ret = btrfs_reconfigure(fc); + + return ret; +} + +static int btrfs_get_tree_subvol(struct fs_context *fc) +{ + struct btrfs_fs_info *fs_info = NULL; + struct btrfs_fs_context *ctx = fc->fs_private; + struct fs_context *dup_fc; + struct dentry *dentry; + struct vfsmount *mnt; + int ret = 0; + + /* + * Setup a dummy root and fs_info for test/set super. This is because + * we don't actually fill this stuff out until open_ctree, but we need + * then open_ctree will properly initialize the file system specific + * settings later. btrfs_init_fs_info initializes the static elements + * of the fs_info (locks and such) to make cleanup easier if we find a + * superblock with our given fs_devices later on at sget_fc() time. + */ + fs_info = kvzalloc_obj(struct btrfs_fs_info); + if (!fs_info) + return -ENOMEM; + + fs_info->super_copy = kzalloc(BTRFS_SUPER_INFO_SIZE, GFP_KERNEL); + fs_info->super_for_commit = kzalloc(BTRFS_SUPER_INFO_SIZE, GFP_KERNEL); + if (!fs_info->super_copy || !fs_info->super_for_commit) { + /* + * Dont call btrfs_free_fs_info() to free it as it's still + * initialized partially. + */ + kfree(fs_info->super_copy); + kfree(fs_info->super_for_commit); + kvfree(fs_info); + return -ENOMEM; + } + btrfs_init_fs_info(fs_info); + + dup_fc = vfs_dup_fs_context(fc); + if (IS_ERR(dup_fc)) { + btrfs_free_fs_info(fs_info); + return PTR_ERR(dup_fc); + } + + /* + * When we do the sget_fc this gets transferred to the sb, so we only + * need to set it on the dup_fc as this is what creates the super block. + */ + dup_fc->s_fs_info = fs_info; + + ret = btrfs_get_tree_super(dup_fc); + if (ret) + goto error; + + ret = btrfs_reconfigure_for_mount(dup_fc); + up_write(&dup_fc->root->d_sb->s_umount); + if (ret) + goto error; + mnt = vfs_create_mount(dup_fc); + put_fs_context(dup_fc); + if (IS_ERR(mnt)) + return PTR_ERR(mnt); + + /* + * This free's ->subvol_name, because if it isn't set we have to + * allocate a buffer to hold the subvol_name, so we just drop our + * reference to it here. + */ + dentry = mount_subvol(ctx->subvol_name, ctx->subvol_objectid, mnt); + ctx->subvol_name = NULL; + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + + fc->root = dentry; + return 0; +error: + put_fs_context(dup_fc); + return ret; +} + +static int btrfs_get_tree(struct fs_context *fc) +{ + ASSERT(fc->s_fs_info == NULL); + + return btrfs_get_tree_subvol(fc); +} + +static void btrfs_kill_super(struct super_block *sb) +{ + struct btrfs_fs_info *fs_info = btrfs_sb(sb); + kill_anon_super(sb); + btrfs_free_fs_info(fs_info); +} + +static void btrfs_free_fs_context(struct fs_context *fc) +{ + struct btrfs_fs_context *ctx = fc->fs_private; + struct btrfs_fs_info *fs_info = fc->s_fs_info; + + if (fs_info) + btrfs_free_fs_info(fs_info); + + if (ctx && refcount_dec_and_test(&ctx->refs)) { + kfree(ctx->subvol_name); + kfree(ctx); + } +} + +static int btrfs_dup_fs_context(struct fs_context *fc, struct fs_context *src_fc) +{ + struct btrfs_fs_context *ctx = src_fc->fs_private; + + /* + * Give a ref to our ctx to this dup, as we want to keep it around for + * our original fc so we can have the subvolume name or objectid. + * + * We unset ->source in the original fc because the dup needs it for + * mounting, and then once we free the dup it'll free ->source, so we + * need to make sure we're only pointing to it in one fc. + */ + refcount_inc(&ctx->refs); + fc->fs_private = ctx; + fc->source = src_fc->source; + src_fc->source = NULL; + return 0; +} + +static const struct fs_context_operations btrfs_fs_context_ops = { + .parse_param = btrfs_parse_param, + .reconfigure = btrfs_reconfigure, + .get_tree = btrfs_get_tree, + .dup = btrfs_dup_fs_context, + .free = btrfs_free_fs_context, +}; + +static int btrfs_init_fs_context(struct fs_context *fc) +{ + struct btrfs_fs_context *ctx; + + ctx = kzalloc_obj(struct btrfs_fs_context); + if (!ctx) + return -ENOMEM; + + refcount_set(&ctx->refs, 1); + fc->fs_private = ctx; + fc->ops = &btrfs_fs_context_ops; + + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE) { + btrfs_info_to_ctx(btrfs_sb(fc->root->d_sb), ctx); + } else { + ctx->thread_pool_size = + min_t(unsigned long, num_online_cpus() + 2, 8); + ctx->max_inline = BTRFS_DEFAULT_MAX_INLINE; + ctx->commit_interval = BTRFS_DEFAULT_COMMIT_INTERVAL; + } + +#ifdef CONFIG_BTRFS_FS_POSIX_ACL + fc->sb_flags |= SB_POSIXACL; +#endif + fc->sb_flags |= SB_I_VERSION; + + return 0; +} + +static struct file_system_type btrfs_fs_type = { + .owner = THIS_MODULE, + .name = "btrfs", + .init_fs_context = btrfs_init_fs_context, + .parameters = btrfs_fs_parameters, + .kill_sb = btrfs_kill_super, + .fs_flags = FS_REQUIRES_DEV | FS_BINARY_MOUNTDATA | + FS_ALLOW_IDMAP | FS_MGTIME, + }; + +MODULE_ALIAS_FS("btrfs"); + +static int btrfs_control_open(struct inode *inode, struct file *file) +{ + /* + * The control file's private_data is used to hold the + * transaction when it is started and is used to keep + * track of whether a transaction is already in progress. + */ + file->private_data = NULL; + return 0; +} + +/* + * Used by /dev/btrfs-control for devices ioctls. + */ +static long btrfs_control_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct btrfs_ioctl_vol_args *vol; + struct btrfs_device *device = NULL; + dev_t devt = 0; + int ret = -ENOTTY; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + vol = memdup_user((void __user *)arg, sizeof(*vol)); + if (IS_ERR(vol)) + return PTR_ERR(vol); + ret = btrfs_check_ioctl_vol_args_path(vol); + if (ret < 0) + goto out; + + switch (cmd) { + case BTRFS_IOC_SCAN_DEV: + mutex_lock(&uuid_mutex); + /* + * Scanning outside of mount can return NULL which would turn + * into 0 error code. + */ + device = btrfs_scan_one_device(vol->name, false); + ret = PTR_ERR_OR_ZERO(device); + mutex_unlock(&uuid_mutex); + break; + case BTRFS_IOC_FORGET_DEV: + if (vol->name[0] != 0) { + ret = lookup_bdev(vol->name, &devt); + if (ret) + break; + } + ret = btrfs_forget_devices(devt); + break; + case BTRFS_IOC_DEVICES_READY: + mutex_lock(&uuid_mutex); + /* + * Scanning outside of mount can return NULL which would turn + * into 0 error code. + */ + device = btrfs_scan_one_device(vol->name, false); + if (IS_ERR_OR_NULL(device)) { + mutex_unlock(&uuid_mutex); + ret = PTR_ERR_OR_ZERO(device); + break; + } + ret = !(device->fs_devices->num_devices == + device->fs_devices->total_devices); + mutex_unlock(&uuid_mutex); + break; + case BTRFS_IOC_GET_SUPPORTED_FEATURES: + ret = btrfs_ioctl_get_supported_features((void __user*)arg); + break; + } + +out: + kfree(vol); + return ret; +} + +static int btrfs_freeze(struct super_block *sb) +{ + struct btrfs_fs_info *fs_info = btrfs_sb(sb); + + set_bit(BTRFS_FS_FROZEN, &fs_info->flags); + /* + * We don't need a barrier here, we'll wait for any transaction that + * could be in progress on other threads (and do delayed iputs that + * we want to avoid on a frozen filesystem), or do the commit + * ourselves. + */ + return btrfs_commit_current_transaction(fs_info->tree_root); +} + +static int check_dev_super(struct btrfs_device *dev) +{ + struct btrfs_fs_info *fs_info = dev->fs_info; + struct btrfs_super_block *sb; + u64 last_trans; + u16 csum_type; + int ret = 0; + + /* This should be called with fs still frozen. */ + ASSERT(test_bit(BTRFS_FS_FROZEN, &fs_info->flags)); + + /* Missing dev, no need to check. */ + if (!dev->bdev) + return 0; + + /* Only need to check the primary super block. */ + sb = btrfs_read_disk_super(dev->bdev, 0, true); + if (IS_ERR(sb)) + return PTR_ERR(sb); + + /* Verify the checksum. */ + csum_type = btrfs_super_csum_type(sb); + if (unlikely(csum_type != btrfs_super_csum_type(fs_info->super_copy))) { + btrfs_err(fs_info, "csum type changed, has %u expect %u", + csum_type, btrfs_super_csum_type(fs_info->super_copy)); + ret = -EUCLEAN; + goto out; + } + + if (unlikely(btrfs_check_super_csum(fs_info, sb))) { + btrfs_err(fs_info, "csum for on-disk super block no longer matches"); + ret = -EUCLEAN; + goto out; + } + + /* Btrfs_validate_super() includes fsid check against super->fsid. */ + ret = btrfs_validate_super(fs_info, sb, 0); + if (ret < 0) + goto out; + + last_trans = btrfs_get_last_trans_committed(fs_info); + if (unlikely(btrfs_super_generation(sb) != last_trans)) { + btrfs_err(fs_info, "transid mismatch, has %llu expect %llu", + btrfs_super_generation(sb), last_trans); + ret = -EUCLEAN; + goto out; + } +out: + btrfs_release_disk_super(sb); + return ret; +} + +static int btrfs_unfreeze(struct super_block *sb) +{ + struct btrfs_fs_info *fs_info = btrfs_sb(sb); + struct btrfs_device *device; + int ret = 0; + + /* + * Make sure the fs is not changed by accident (like hibernation then + * modified by other OS). + * If we found anything wrong, we mark the fs error immediately. + * + * And since the fs is frozen, no one can modify the fs yet, thus + * we don't need to hold device_list_mutex. + */ + list_for_each_entry(device, &fs_info->fs_devices->devices, dev_list) { + ret = check_dev_super(device); + if (ret < 0) { + btrfs_handle_fs_error(fs_info, ret, + "super block on devid %llu got modified unexpectedly", + device->devid); + break; + } + } + clear_bit(BTRFS_FS_FROZEN, &fs_info->flags); + + /* + * We still return 0, to allow VFS layer to unfreeze the fs even the + * above checks failed. Since the fs is either fine or read-only, we're + * safe to continue, without causing further damage. + */ + return 0; +} + +static int btrfs_show_devname(struct seq_file *m, struct dentry *root) +{ + struct btrfs_fs_info *fs_info = btrfs_sb(root->d_sb); + + /* + * There should be always a valid pointer in latest_dev, it may be stale + * for a short moment in case it's being deleted but still valid until + * the end of RCU grace period. + */ + rcu_read_lock(); + seq_escape(m, btrfs_dev_name(fs_info->fs_devices->latest_dev), " \t\n\\"); + rcu_read_unlock(); + + return 0; +} + +static long btrfs_nr_cached_objects(struct super_block *sb, struct shrink_control *sc) +{ + struct btrfs_fs_info *fs_info = btrfs_sb(sb); + const s64 nr = percpu_counter_read_positive(&fs_info->evictable_extent_maps); + + /* + * The evictable extent map counter is filesystem-global and does not + * honour sc->memcg, so it is only meaningful on the global (kswapd or + * root direct reclaim) shrink path. Skip the per-memcg iterations of + * shrink_slab_memcg() to avoid queueing duplicate global work. + */ + if (!mem_cgroup_shrink_is_root(sc)) + return 0; + + trace_btrfs_extent_map_shrinker_count(fs_info, nr); + + return nr; +} + +static long btrfs_free_cached_objects(struct super_block *sb, struct shrink_control *sc) +{ + const long nr_to_scan = min_t(unsigned long, LONG_MAX, sc->nr_to_scan); + struct btrfs_fs_info *fs_info = btrfs_sb(sb); + + btrfs_free_extent_maps(fs_info, nr_to_scan); + + /* The extent map shrinker runs asynchronously, so always return 0. */ + return 0; +} + +static int btrfs_remove_bdev(struct super_block *sb, struct block_device *bdev) +{ + struct btrfs_fs_info *fs_info = btrfs_sb(sb); + struct btrfs_device *device; + struct btrfs_dev_lookup_args lookup_args = { .devt = bdev->bd_dev }; + bool can_rw; + + mutex_lock(&fs_info->fs_devices->device_list_mutex); + device = btrfs_find_device(fs_info->fs_devices, &lookup_args); + if (!device) { + mutex_unlock(&fs_info->fs_devices->device_list_mutex); + /* Device not found, should not affect the running fs, just give a warning. */ + btrfs_warn(fs_info, "unable to find btrfs device for block device '%pg'", bdev); + return 0; + } + /* + * The to-be-removed device is already missing? + * + * That's weird but no special handling needed and can exit right now. + */ + if (unlikely(test_and_set_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state))) { + mutex_unlock(&fs_info->fs_devices->device_list_mutex); + btrfs_warn(fs_info, "btrfs device id %llu is already missing", device->devid); + return 0; + } + + device->fs_devices->missing_devices++; + if (test_and_clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { + list_del_init(&device->dev_alloc_list); + WARN_ON(device->fs_devices->rw_devices < 1); + device->fs_devices->rw_devices--; + } + can_rw = btrfs_check_rw_degradable(fs_info, device); + mutex_unlock(&fs_info->fs_devices->device_list_mutex); + /* + * Now device is considered missing, btrfs_device_name() won't give a + * meaningful result anymore, so only output the devid. + */ + if (unlikely(!can_rw)) { + btrfs_crit(fs_info, + "btrfs device id %llu has gone missing, can not maintain read-write", + device->devid); + return -EIO; + } + btrfs_warn(fs_info, + "btrfs device id %llu has gone missing, continue as degraded", + device->devid); + btrfs_set_opt(fs_info->mount_opt, DEGRADED); + return 0; +} + +static void btrfs_shutdown(struct super_block *sb) +{ + struct btrfs_fs_info *fs_info = btrfs_sb(sb); + + btrfs_force_shutdown(fs_info); +} + +static int btrfs_show_stats(struct seq_file *seq, struct dentry *root) +{ + struct btrfs_fs_info *fs_info = btrfs_sb(root->d_sb); + + if (btrfs_is_zoned(fs_info)) { + btrfs_show_zoned_stats(fs_info, seq); + return 0; + } + + return 0; +} + +static const struct super_operations btrfs_super_ops = { + .drop_inode = btrfs_drop_inode, + .evict_inode = btrfs_evict_inode, + .put_super = btrfs_put_super, + .sync_fs = btrfs_sync_fs, + .show_options = btrfs_show_options, + .show_devname = btrfs_show_devname, + .alloc_inode = btrfs_alloc_inode, + .destroy_inode = btrfs_destroy_inode, + .free_inode = btrfs_free_inode, + .statfs = btrfs_statfs, + .freeze_fs = btrfs_freeze, + .unfreeze_fs = btrfs_unfreeze, + .nr_cached_objects = btrfs_nr_cached_objects, + .free_cached_objects = btrfs_free_cached_objects, + .show_stats = btrfs_show_stats, + .remove_bdev = btrfs_remove_bdev, + .shutdown = btrfs_shutdown, +}; + +static const struct file_operations btrfs_ctl_fops = { + .open = btrfs_control_open, + .unlocked_ioctl = btrfs_control_ioctl, + .compat_ioctl = compat_ptr_ioctl, + .owner = THIS_MODULE, + .llseek = noop_llseek, +}; + +static struct miscdevice btrfs_misc = { + .minor = BTRFS_MINOR, + .name = "btrfs-control", + .fops = &btrfs_ctl_fops +}; + +MODULE_ALIAS_MISCDEV(BTRFS_MINOR); +MODULE_ALIAS("devname:btrfs-control"); + +static int __init btrfs_interface_init(void) +{ + return misc_register(&btrfs_misc); +} + +static __cold void btrfs_interface_exit(void) +{ + misc_deregister(&btrfs_misc); +} + +static int __init btrfs_print_mod_info(void) +{ + static const char options[] = "" +#ifdef CONFIG_BTRFS_EXPERIMENTAL + ", experimental=on" +#endif +#ifdef CONFIG_BTRFS_DEBUG + ", debug=on" +#endif +#ifdef CONFIG_BTRFS_ASSERT + ", assert=on" +#endif +#ifdef CONFIG_BLK_DEV_ZONED + ", zoned=yes" +#else + ", zoned=no" +#endif +#ifdef CONFIG_FS_VERITY + ", fsverity=yes" +#else + ", fsverity=no" +#endif + ; + +#ifdef CONFIG_BTRFS_EXPERIMENTAL + if (btrfs_get_mod_read_policy() == NULL) + pr_info("Btrfs loaded%s\n", options); + else + pr_info("Btrfs loaded%s, read_policy=%s\n", + options, btrfs_get_mod_read_policy()); +#else + pr_info("Btrfs loaded%s\n", options); +#endif + + return 0; +} + +static int register_btrfs(void) +{ + return register_filesystem(&btrfs_fs_type); +} + +static void unregister_btrfs(void) +{ + unregister_filesystem(&btrfs_fs_type); +} + +/* Helper structure for long init/exit functions. */ +struct init_sequence { + int (*init_func)(void); + /* Can be NULL if the init_func doesn't need cleanup. */ + void (*exit_func)(void); +}; + +static const struct init_sequence mod_init_seq[] = { + { + .init_func = btrfs_props_init, + .exit_func = NULL, + }, { + .init_func = btrfs_init_sysfs, + .exit_func = btrfs_exit_sysfs, + }, { + .init_func = btrfs_init_compress, + .exit_func = btrfs_exit_compress, + }, { + .init_func = btrfs_init_block_group, + .exit_func = btrfs_exit_block_group, + }, { + .init_func = btrfs_init_cachep, + .exit_func = btrfs_destroy_cachep, + }, { + .init_func = btrfs_init_dio, + .exit_func = btrfs_destroy_dio, + }, { + .init_func = btrfs_transaction_init, + .exit_func = btrfs_transaction_exit, + }, { + .init_func = btrfs_ctree_init, + .exit_func = btrfs_ctree_exit, + }, { + .init_func = btrfs_free_space_init, + .exit_func = btrfs_free_space_exit, + }, { + .init_func = btrfs_extent_state_init_cachep, + .exit_func = btrfs_extent_state_free_cachep, + }, { + .init_func = extent_buffer_init_cachep, + .exit_func = extent_buffer_free_cachep, + }, { + .init_func = btrfs_bioset_init, + .exit_func = btrfs_bioset_exit, + }, { + .init_func = btrfs_extent_map_init, + .exit_func = btrfs_extent_map_exit, +#ifdef CONFIG_BTRFS_EXPERIMENTAL + }, { + .init_func = btrfs_read_policy_init, + .exit_func = NULL, +#endif + }, { + .init_func = ordered_data_init, + .exit_func = ordered_data_exit, + }, { + .init_func = btrfs_delayed_inode_init, + .exit_func = btrfs_delayed_inode_exit, + }, { + .init_func = btrfs_auto_defrag_init, + .exit_func = btrfs_auto_defrag_exit, + }, { + .init_func = btrfs_delayed_ref_init, + .exit_func = btrfs_delayed_ref_exit, + }, { + .init_func = btrfs_prelim_ref_init, + .exit_func = btrfs_prelim_ref_exit, + }, { + .init_func = btrfs_interface_init, + .exit_func = btrfs_interface_exit, + }, { + .init_func = btrfs_print_mod_info, + .exit_func = NULL, + }, { + .init_func = btrfs_run_sanity_tests, + .exit_func = NULL, + }, { + .init_func = register_btrfs, + .exit_func = unregister_btrfs, + } +}; + +static bool mod_init_result[ARRAY_SIZE(mod_init_seq)]; + +static __always_inline void btrfs_exit_btrfs_fs(void) +{ + int i; + + for (i = ARRAY_SIZE(mod_init_seq) - 1; i >= 0; i--) { + if (!mod_init_result[i]) + continue; + if (mod_init_seq[i].exit_func) + mod_init_seq[i].exit_func(); + mod_init_result[i] = false; + } +} + +static void __exit exit_btrfs_fs(void) +{ + btrfs_exit_btrfs_fs(); + btrfs_cleanup_fs_uuids(); +} + +static int __init init_btrfs_fs(void) +{ + int ret; + int i; + + for (i = 0; i < ARRAY_SIZE(mod_init_seq); i++) { + ASSERT(!mod_init_result[i]); + ret = mod_init_seq[i].init_func(); + if (ret < 0) { + btrfs_exit_btrfs_fs(); + return ret; + } + mod_init_result[i] = true; + } + return 0; +} + +late_initcall(init_btrfs_fs); +module_exit(exit_btrfs_fs) + +MODULE_DESCRIPTION("B-Tree File System (BTRFS)"); +MODULE_LICENSE("GPL"); diff --git a/fs/btrfs/super.h b/fs/btrfs/super.h new file mode 100644 index 000000000..f85f8a8a7 --- /dev/null +++ b/fs/btrfs/super.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_SUPER_H +#define BTRFS_SUPER_H + +#include +#include +#include "fs.h" + +struct super_block; +struct btrfs_fs_info; + +bool btrfs_check_options(const struct btrfs_fs_info *info, + unsigned long long *mount_opt, + unsigned long flags); +int btrfs_sync_fs(struct super_block *sb, int wait); +char *btrfs_get_subvol_name_from_objectid(struct btrfs_fs_info *fs_info, + u64 subvol_objectid); +void btrfs_set_free_space_cache_settings(struct btrfs_fs_info *fs_info); + +static inline struct btrfs_fs_info *btrfs_sb(const struct super_block *sb) +{ + return sb->s_fs_info; +} + +static inline void btrfs_set_sb_rdonly(struct super_block *sb) +{ + sb->s_flags |= SB_RDONLY; + set_bit(BTRFS_FS_STATE_RO, &btrfs_sb(sb)->fs_state); +} + +static inline void btrfs_clear_sb_rdonly(struct super_block *sb) +{ + sb->s_flags &= ~SB_RDONLY; + clear_bit(BTRFS_FS_STATE_RO, &btrfs_sb(sb)->fs_state); +} + +#endif diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c new file mode 100644 index 000000000..39cb01ee4 --- /dev/null +++ b/fs/btrfs/sysfs.c @@ -0,0 +1,2702 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "messages.h" +#include "ctree.h" +#include "discard.h" +#include "disk-io.h" +#include "send.h" +#include "transaction.h" +#include "sysfs.h" +#include "volumes.h" +#include "space-info.h" +#include "block-group.h" +#include "qgroup.h" +#include "misc.h" +#include "fs.h" +#include "accessors.h" + +/* + * Structure name Path + * -------------------------------------------------------------------------- + * btrfs_supported_static_feature_attrs /sys/fs/btrfs/features + * btrfs_supported_feature_attrs /sys/fs/btrfs/features and + * /sys/fs/btrfs//features + * btrfs_attrs /sys/fs/btrfs/ + * devid_attrs /sys/fs/btrfs//devinfo/ + * allocation_attrs /sys/fs/btrfs//allocation + * qgroup_attrs /sys/fs/btrfs//qgroups/_ + * space_info_attrs /sys/fs/btrfs//allocation/ + * raid_attrs /sys/fs/btrfs//allocation// + * discard_attrs /sys/fs/btrfs//discard + * + * When built with BTRFS_CONFIG_DEBUG: + * + * btrfs_debug_feature_attrs /sys/fs/btrfs/debug + * btrfs_debug_mount_attrs /sys/fs/btrfs//debug + */ + +struct btrfs_feature_attr { + struct kobj_attribute kobj_attr; + enum btrfs_feature_set feature_set; + u64 feature_bit; +}; + +/* For raid type sysfs entries */ +struct raid_kobject { + u64 flags; + struct kobject kobj; +}; + +#define __INIT_KOBJ_ATTR(_name, _mode, _show, _store) \ +{ \ + .attr = { .name = __stringify(_name), .mode = _mode }, \ + .show = _show, \ + .store = _store, \ +} + +#define BTRFS_ATTR_W(_prefix, _name, _store) \ + static struct kobj_attribute btrfs_attr_##_prefix##_##_name = \ + __INIT_KOBJ_ATTR(_name, 0200, NULL, _store) + +#define BTRFS_ATTR_RW(_prefix, _name, _show, _store) \ + static struct kobj_attribute btrfs_attr_##_prefix##_##_name = \ + __INIT_KOBJ_ATTR(_name, 0644, _show, _store) + +#define BTRFS_ATTR(_prefix, _name, _show) \ + static struct kobj_attribute btrfs_attr_##_prefix##_##_name = \ + __INIT_KOBJ_ATTR(_name, 0444, _show, NULL) + +#define BTRFS_ATTR_PTR(_prefix, _name) \ + (&btrfs_attr_##_prefix##_##_name.attr) + +#define BTRFS_FEAT_ATTR(_name, _feature_set, _feature_prefix, _feature_bit) \ +static struct btrfs_feature_attr btrfs_attr_features_##_name = { \ + .kobj_attr = __INIT_KOBJ_ATTR(_name, S_IRUGO, \ + btrfs_feature_attr_show, \ + btrfs_feature_attr_store), \ + .feature_set = _feature_set, \ + .feature_bit = _feature_prefix ##_## _feature_bit, \ +} +#define BTRFS_FEAT_ATTR_PTR(_name) \ + (&btrfs_attr_features_##_name.kobj_attr.attr) + +#define BTRFS_FEAT_ATTR_COMPAT(name, feature) \ + BTRFS_FEAT_ATTR(name, FEAT_COMPAT, BTRFS_FEATURE_COMPAT, feature) +#define BTRFS_FEAT_ATTR_COMPAT_RO(name, feature) \ + BTRFS_FEAT_ATTR(name, FEAT_COMPAT_RO, BTRFS_FEATURE_COMPAT_RO, feature) +#define BTRFS_FEAT_ATTR_INCOMPAT(name, feature) \ + BTRFS_FEAT_ATTR(name, FEAT_INCOMPAT, BTRFS_FEATURE_INCOMPAT, feature) + +static inline struct btrfs_fs_info *to_fs_info(struct kobject *kobj); +static inline struct btrfs_fs_devices *to_fs_devs(struct kobject *kobj); +static struct kobject *get_btrfs_kobj(struct kobject *kobj); + +static struct btrfs_feature_attr *to_btrfs_feature_attr(struct kobj_attribute *a) +{ + return container_of(a, struct btrfs_feature_attr, kobj_attr); +} + +static struct kobj_attribute *attr_to_btrfs_attr(struct attribute *attr) +{ + return container_of(attr, struct kobj_attribute, attr); +} + +static struct btrfs_feature_attr *attr_to_btrfs_feature_attr( + struct attribute *attr) +{ + return to_btrfs_feature_attr(attr_to_btrfs_attr(attr)); +} + +static u64 get_features(struct btrfs_fs_info *fs_info, + enum btrfs_feature_set set) +{ + struct btrfs_super_block *disk_super = fs_info->super_copy; + if (set == FEAT_COMPAT) + return btrfs_super_compat_flags(disk_super); + else if (set == FEAT_COMPAT_RO) + return btrfs_super_compat_ro_flags(disk_super); + else + return btrfs_super_incompat_flags(disk_super); +} + +static void set_features(struct btrfs_fs_info *fs_info, + enum btrfs_feature_set set, u64 features) +{ + struct btrfs_super_block *disk_super = fs_info->super_copy; + if (set == FEAT_COMPAT) + btrfs_set_super_compat_flags(disk_super, features); + else if (set == FEAT_COMPAT_RO) + btrfs_set_super_compat_ro_flags(disk_super, features); + else + btrfs_set_super_incompat_flags(disk_super, features); +} + +static int can_modify_feature(struct btrfs_feature_attr *fa) +{ + int val = 0; + u64 set, clear; + switch (fa->feature_set) { + case FEAT_COMPAT: + set = BTRFS_FEATURE_COMPAT_SAFE_SET; + clear = BTRFS_FEATURE_COMPAT_SAFE_CLEAR; + break; + case FEAT_COMPAT_RO: + set = BTRFS_FEATURE_COMPAT_RO_SAFE_SET; + clear = BTRFS_FEATURE_COMPAT_RO_SAFE_CLEAR; + break; + case FEAT_INCOMPAT: + set = BTRFS_FEATURE_INCOMPAT_SAFE_SET; + clear = BTRFS_FEATURE_INCOMPAT_SAFE_CLEAR; + break; + default: + btrfs_warn(NULL, "sysfs: unknown feature set %d", fa->feature_set); + return 0; + } + + if (set & fa->feature_bit) + val |= 1; + if (clear & fa->feature_bit) + val |= 2; + + return val; +} + +static ssize_t btrfs_feature_attr_show(struct kobject *kobj, + struct kobj_attribute *a, char *buf) +{ + int val = 0; + struct btrfs_fs_info *fs_info = to_fs_info(kobj); + struct btrfs_feature_attr *fa = to_btrfs_feature_attr(a); + if (fs_info) { + u64 features = get_features(fs_info, fa->feature_set); + if (features & fa->feature_bit) + val = 1; + } else + val = can_modify_feature(fa); + + return sysfs_emit(buf, "%d\n", val); +} + +static ssize_t btrfs_feature_attr_store(struct kobject *kobj, + struct kobj_attribute *a, + const char *buf, size_t count) +{ + struct btrfs_fs_info *fs_info; + struct btrfs_feature_attr *fa = to_btrfs_feature_attr(a); + u64 features, set, clear; + unsigned long val; + int ret; + + fs_info = to_fs_info(kobj); + if (!fs_info) + return -EPERM; + + if (sb_rdonly(fs_info->sb)) + return -EROFS; + + ret = kstrtoul(skip_spaces(buf), 0, &val); + if (ret) + return ret; + + if (fa->feature_set == FEAT_COMPAT) { + set = BTRFS_FEATURE_COMPAT_SAFE_SET; + clear = BTRFS_FEATURE_COMPAT_SAFE_CLEAR; + } else if (fa->feature_set == FEAT_COMPAT_RO) { + set = BTRFS_FEATURE_COMPAT_RO_SAFE_SET; + clear = BTRFS_FEATURE_COMPAT_RO_SAFE_CLEAR; + } else { + set = BTRFS_FEATURE_INCOMPAT_SAFE_SET; + clear = BTRFS_FEATURE_INCOMPAT_SAFE_CLEAR; + } + + features = get_features(fs_info, fa->feature_set); + + /* Nothing to do */ + if ((val && (features & fa->feature_bit)) || + (!val && !(features & fa->feature_bit))) + return count; + + if ((val && !(set & fa->feature_bit)) || + (!val && !(clear & fa->feature_bit))) { + btrfs_info(fs_info, + "%sabling feature %s on mounted fs is not supported.", + val ? "En" : "Dis", fa->kobj_attr.attr.name); + return -EPERM; + } + + btrfs_info(fs_info, "%s %s feature flag", + val ? "Setting" : "Clearing", fa->kobj_attr.attr.name); + + spin_lock(&fs_info->super_lock); + features = get_features(fs_info, fa->feature_set); + if (val) + features |= fa->feature_bit; + else + features &= ~fa->feature_bit; + set_features(fs_info, fa->feature_set, features); + spin_unlock(&fs_info->super_lock); + + /* + * We don't want to do full transaction commit from inside sysfs + */ + set_bit(BTRFS_FS_NEED_TRANS_COMMIT, &fs_info->flags); + wake_up_process(fs_info->transaction_kthread); + + return count; +} + +static umode_t btrfs_feature_visible(struct kobject *kobj, + struct attribute *attr, int unused) +{ + struct btrfs_fs_info *fs_info = to_fs_info(kobj); + umode_t mode = attr->mode; + + if (fs_info) { + struct btrfs_feature_attr *fa; + u64 features; + + fa = attr_to_btrfs_feature_attr(attr); + features = get_features(fs_info, fa->feature_set); + + if (can_modify_feature(fa)) + mode |= S_IWUSR; + else if (!(features & fa->feature_bit)) + mode = 0; + } + + return mode; +} + +BTRFS_FEAT_ATTR_INCOMPAT(default_subvol, DEFAULT_SUBVOL); +BTRFS_FEAT_ATTR_INCOMPAT(mixed_groups, MIXED_GROUPS); +BTRFS_FEAT_ATTR_INCOMPAT(compress_lzo, COMPRESS_LZO); +BTRFS_FEAT_ATTR_INCOMPAT(compress_zstd, COMPRESS_ZSTD); +BTRFS_FEAT_ATTR_INCOMPAT(extended_iref, EXTENDED_IREF); +BTRFS_FEAT_ATTR_INCOMPAT(raid56, RAID56); +BTRFS_FEAT_ATTR_INCOMPAT(skinny_metadata, SKINNY_METADATA); +BTRFS_FEAT_ATTR_INCOMPAT(no_holes, NO_HOLES); +BTRFS_FEAT_ATTR_INCOMPAT(metadata_uuid, METADATA_UUID); +BTRFS_FEAT_ATTR_COMPAT_RO(free_space_tree, FREE_SPACE_TREE); +BTRFS_FEAT_ATTR_COMPAT_RO(block_group_tree, BLOCK_GROUP_TREE); +BTRFS_FEAT_ATTR_INCOMPAT(raid1c34, RAID1C34); +BTRFS_FEAT_ATTR_INCOMPAT(simple_quota, SIMPLE_QUOTA); +#ifdef CONFIG_BLK_DEV_ZONED +BTRFS_FEAT_ATTR_INCOMPAT(zoned, ZONED); +#endif +#ifdef CONFIG_BTRFS_EXPERIMENTAL +/* Remove once support for extent tree v2 is feature complete */ +BTRFS_FEAT_ATTR_INCOMPAT(extent_tree_v2, EXTENT_TREE_V2); +/* Remove once support for raid stripe tree is feature complete. */ +BTRFS_FEAT_ATTR_INCOMPAT(raid_stripe_tree, RAID_STRIPE_TREE); +/* Remove once support for remap tree is feature complete. */ +BTRFS_FEAT_ATTR_INCOMPAT(remap_tree, REMAP_TREE); +#endif +#ifdef CONFIG_FS_VERITY +BTRFS_FEAT_ATTR_COMPAT_RO(verity, VERITY); +#endif + +/* + * Features which depend on feature bits and may differ between each fs. + * + * /sys/fs/btrfs/features - all available features implemented by this version + * /sys/fs/btrfs/UUID/features - features of the fs which are enabled or + * can be changed on a mounted filesystem. + */ +static struct attribute *btrfs_supported_feature_attrs[] = { + BTRFS_FEAT_ATTR_PTR(default_subvol), + BTRFS_FEAT_ATTR_PTR(mixed_groups), + BTRFS_FEAT_ATTR_PTR(compress_lzo), + BTRFS_FEAT_ATTR_PTR(compress_zstd), + BTRFS_FEAT_ATTR_PTR(extended_iref), + BTRFS_FEAT_ATTR_PTR(raid56), + BTRFS_FEAT_ATTR_PTR(skinny_metadata), + BTRFS_FEAT_ATTR_PTR(no_holes), + BTRFS_FEAT_ATTR_PTR(metadata_uuid), + BTRFS_FEAT_ATTR_PTR(free_space_tree), + BTRFS_FEAT_ATTR_PTR(raid1c34), + BTRFS_FEAT_ATTR_PTR(block_group_tree), + BTRFS_FEAT_ATTR_PTR(simple_quota), +#ifdef CONFIG_BLK_DEV_ZONED + BTRFS_FEAT_ATTR_PTR(zoned), +#endif +#ifdef CONFIG_BTRFS_EXPERIMENTAL + BTRFS_FEAT_ATTR_PTR(extent_tree_v2), + BTRFS_FEAT_ATTR_PTR(raid_stripe_tree), + BTRFS_FEAT_ATTR_PTR(remap_tree), +#endif +#ifdef CONFIG_FS_VERITY + BTRFS_FEAT_ATTR_PTR(verity), +#endif + NULL +}; + +static const struct attribute_group btrfs_feature_attr_group = { + .name = "features", + .is_visible = btrfs_feature_visible, + .attrs = btrfs_supported_feature_attrs, +}; + +static ssize_t rmdir_subvol_show(struct kobject *kobj, + struct kobj_attribute *ka, char *buf) +{ + return sysfs_emit(buf, "0\n"); +} +BTRFS_ATTR(static_feature, rmdir_subvol, rmdir_subvol_show); + +static ssize_t supported_checksums_show(struct kobject *kobj, + struct kobj_attribute *a, char *buf) +{ + ssize_t ret = 0; + int i; + + for (i = 0; i < btrfs_get_num_csums(); i++) { + /* + * This "trick" only works as long as 'enum btrfs_csum_type' has + * no holes in it + */ + ret += sysfs_emit_at(buf, ret, "%s%s", (i == 0 ? "" : " "), + btrfs_super_csum_name(i)); + + } + + ret += sysfs_emit_at(buf, ret, "\n"); + return ret; +} +BTRFS_ATTR(static_feature, supported_checksums, supported_checksums_show); + +static ssize_t send_stream_version_show(struct kobject *kobj, + struct kobj_attribute *ka, char *buf) +{ + return sysfs_emit(buf, "%d\n", BTRFS_SEND_STREAM_VERSION); +} +BTRFS_ATTR(static_feature, send_stream_version, send_stream_version_show); + +static const char *rescue_opts[] = { + "usebackuproot", + "nologreplay", + "ignorebadroots", + "ignoredatacsums", + "ignoremetacsums", + "ignoresuperflags", + "all", +}; + +static ssize_t supported_rescue_options_show(struct kobject *kobj, + struct kobj_attribute *a, + char *buf) +{ + ssize_t ret = 0; + int i; + + for (i = 0; i < ARRAY_SIZE(rescue_opts); i++) + ret += sysfs_emit_at(buf, ret, "%s%s", (i ? " " : ""), rescue_opts[i]); + ret += sysfs_emit_at(buf, ret, "\n"); + return ret; +} +BTRFS_ATTR(static_feature, supported_rescue_options, + supported_rescue_options_show); + +static ssize_t supported_sectorsizes_show(struct kobject *kobj, + struct kobj_attribute *a, + char *buf) +{ + ssize_t ret = 0; + bool has_output = false; + + for (u32 cur = BTRFS_MIN_BLOCKSIZE; cur <= BTRFS_MAX_BLOCKSIZE; cur *= 2) { + if (!btrfs_supported_blocksize(cur)) + continue; + if (has_output) + ret += sysfs_emit_at(buf, ret, " "); + ret += sysfs_emit_at(buf, ret, "%u", cur); + has_output = true; + } + ret += sysfs_emit_at(buf, ret, "\n"); + return ret; +} +BTRFS_ATTR(static_feature, supported_sectorsizes, + supported_sectorsizes_show); + +static ssize_t acl_show(struct kobject *kobj, struct kobj_attribute *a, char *buf) +{ + return sysfs_emit(buf, "%d\n", IS_ENABLED(CONFIG_BTRFS_FS_POSIX_ACL)); +} +BTRFS_ATTR(static_feature, acl, acl_show); + +static ssize_t temp_fsid_supported_show(struct kobject *kobj, + struct kobj_attribute *a, char *buf) +{ + return sysfs_emit(buf, "0\n"); +} +BTRFS_ATTR(static_feature, temp_fsid, temp_fsid_supported_show); + +/* + * Features which only depend on kernel version. + * + * These are listed in /sys/fs/btrfs/features along with + * btrfs_supported_feature_attrs. + */ +static struct attribute *btrfs_supported_static_feature_attrs[] = { + BTRFS_ATTR_PTR(static_feature, acl), + BTRFS_ATTR_PTR(static_feature, rmdir_subvol), + BTRFS_ATTR_PTR(static_feature, supported_checksums), + BTRFS_ATTR_PTR(static_feature, send_stream_version), + BTRFS_ATTR_PTR(static_feature, supported_rescue_options), + BTRFS_ATTR_PTR(static_feature, supported_sectorsizes), + BTRFS_ATTR_PTR(static_feature, temp_fsid), + NULL +}; + +static const struct attribute_group btrfs_static_feature_attr_group = { + .name = "features", + .attrs = btrfs_supported_static_feature_attrs, +}; + +/* + * Discard statistics and tunables + */ +#define discard_to_fs_info(_kobj) to_fs_info(get_btrfs_kobj(_kobj)) + +static ssize_t btrfs_discardable_bytes_show(struct kobject *kobj, + struct kobj_attribute *a, + char *buf) +{ + struct btrfs_fs_info *fs_info = discard_to_fs_info(kobj); + + return sysfs_emit(buf, "%lld\n", + atomic64_read(&fs_info->discard_ctl.discardable_bytes)); +} +BTRFS_ATTR(discard, discardable_bytes, btrfs_discardable_bytes_show); + +static ssize_t btrfs_discardable_extents_show(struct kobject *kobj, + struct kobj_attribute *a, + char *buf) +{ + struct btrfs_fs_info *fs_info = discard_to_fs_info(kobj); + + return sysfs_emit(buf, "%d\n", + atomic_read(&fs_info->discard_ctl.discardable_extents)); +} +BTRFS_ATTR(discard, discardable_extents, btrfs_discardable_extents_show); + +static ssize_t btrfs_discard_bitmap_bytes_show(struct kobject *kobj, + struct kobj_attribute *a, + char *buf) +{ + struct btrfs_fs_info *fs_info = discard_to_fs_info(kobj); + + return sysfs_emit(buf, "%llu\n", + fs_info->discard_ctl.discard_bitmap_bytes); +} +BTRFS_ATTR(discard, discard_bitmap_bytes, btrfs_discard_bitmap_bytes_show); + +static ssize_t btrfs_discard_bytes_saved_show(struct kobject *kobj, + struct kobj_attribute *a, + char *buf) +{ + struct btrfs_fs_info *fs_info = discard_to_fs_info(kobj); + + return sysfs_emit(buf, "%lld\n", + atomic64_read(&fs_info->discard_ctl.discard_bytes_saved)); +} +BTRFS_ATTR(discard, discard_bytes_saved, btrfs_discard_bytes_saved_show); + +static ssize_t btrfs_discard_extent_bytes_show(struct kobject *kobj, + struct kobj_attribute *a, + char *buf) +{ + struct btrfs_fs_info *fs_info = discard_to_fs_info(kobj); + + return sysfs_emit(buf, "%llu\n", + fs_info->discard_ctl.discard_extent_bytes); +} +BTRFS_ATTR(discard, discard_extent_bytes, btrfs_discard_extent_bytes_show); + +static ssize_t btrfs_discard_iops_limit_show(struct kobject *kobj, + struct kobj_attribute *a, + char *buf) +{ + struct btrfs_fs_info *fs_info = discard_to_fs_info(kobj); + + return sysfs_emit(buf, "%u\n", + READ_ONCE(fs_info->discard_ctl.iops_limit)); +} + +static ssize_t btrfs_discard_iops_limit_store(struct kobject *kobj, + struct kobj_attribute *a, + const char *buf, size_t len) +{ + struct btrfs_fs_info *fs_info = discard_to_fs_info(kobj); + struct btrfs_discard_ctl *discard_ctl = &fs_info->discard_ctl; + u32 iops_limit; + int ret; + + ret = kstrtou32(buf, 10, &iops_limit); + if (ret) + return -EINVAL; + + WRITE_ONCE(discard_ctl->iops_limit, iops_limit); + btrfs_discard_calc_delay(discard_ctl); + btrfs_discard_schedule_work(discard_ctl, true); + return len; +} +BTRFS_ATTR_RW(discard, iops_limit, btrfs_discard_iops_limit_show, + btrfs_discard_iops_limit_store); + +static ssize_t btrfs_discard_kbps_limit_show(struct kobject *kobj, + struct kobj_attribute *a, + char *buf) +{ + struct btrfs_fs_info *fs_info = discard_to_fs_info(kobj); + + return sysfs_emit(buf, "%u\n", + READ_ONCE(fs_info->discard_ctl.kbps_limit)); +} + +static ssize_t btrfs_discard_kbps_limit_store(struct kobject *kobj, + struct kobj_attribute *a, + const char *buf, size_t len) +{ + struct btrfs_fs_info *fs_info = discard_to_fs_info(kobj); + struct btrfs_discard_ctl *discard_ctl = &fs_info->discard_ctl; + u32 kbps_limit; + int ret; + + ret = kstrtou32(buf, 10, &kbps_limit); + if (ret) + return -EINVAL; + + WRITE_ONCE(discard_ctl->kbps_limit, kbps_limit); + btrfs_discard_schedule_work(discard_ctl, true); + return len; +} +BTRFS_ATTR_RW(discard, kbps_limit, btrfs_discard_kbps_limit_show, + btrfs_discard_kbps_limit_store); + +static ssize_t btrfs_discard_max_discard_size_show(struct kobject *kobj, + struct kobj_attribute *a, + char *buf) +{ + struct btrfs_fs_info *fs_info = discard_to_fs_info(kobj); + + return sysfs_emit(buf, "%llu\n", + READ_ONCE(fs_info->discard_ctl.max_discard_size)); +} + +static ssize_t btrfs_discard_max_discard_size_store(struct kobject *kobj, + struct kobj_attribute *a, + const char *buf, size_t len) +{ + struct btrfs_fs_info *fs_info = discard_to_fs_info(kobj); + struct btrfs_discard_ctl *discard_ctl = &fs_info->discard_ctl; + u64 max_discard_size; + int ret; + + ret = kstrtou64(buf, 10, &max_discard_size); + if (ret) + return -EINVAL; + + WRITE_ONCE(discard_ctl->max_discard_size, max_discard_size); + + return len; +} +BTRFS_ATTR_RW(discard, max_discard_size, btrfs_discard_max_discard_size_show, + btrfs_discard_max_discard_size_store); + +/* + * Per-filesystem stats for discard (when mounted with discard=async). + * + * Path: /sys/fs/btrfs//discard/ + */ +static const struct attribute *discard_attrs[] = { + BTRFS_ATTR_PTR(discard, discardable_bytes), + BTRFS_ATTR_PTR(discard, discardable_extents), + BTRFS_ATTR_PTR(discard, discard_bitmap_bytes), + BTRFS_ATTR_PTR(discard, discard_bytes_saved), + BTRFS_ATTR_PTR(discard, discard_extent_bytes), + BTRFS_ATTR_PTR(discard, iops_limit), + BTRFS_ATTR_PTR(discard, kbps_limit), + BTRFS_ATTR_PTR(discard, max_discard_size), + NULL, +}; + +#ifdef CONFIG_BTRFS_DEBUG + +/* + * Per-filesystem runtime debugging exported via sysfs. + * + * Path: /sys/fs/btrfs/UUID/debug/ + */ +static const struct attribute *btrfs_debug_mount_attrs[] = { + NULL, +}; + +/* + * Runtime debugging exported via sysfs, applies to all mounted filesystems. + * + * Path: /sys/fs/btrfs/debug + */ +static struct attribute *btrfs_debug_feature_attrs[] = { + NULL +}; + +static const struct attribute_group btrfs_debug_feature_attr_group = { + .name = "debug", + .attrs = btrfs_debug_feature_attrs, +}; + +#endif + +static ssize_t btrfs_show_u64(u64 *value_ptr, spinlock_t *lock, char *buf) +{ + u64 val; + if (lock) + spin_lock(lock); + val = *value_ptr; + if (lock) + spin_unlock(lock); + return sysfs_emit(buf, "%llu\n", val); +} + +static ssize_t global_rsv_size_show(struct kobject *kobj, + struct kobj_attribute *ka, char *buf) +{ + struct btrfs_fs_info *fs_info = to_fs_info(kobj->parent); + struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv; + return btrfs_show_u64(&block_rsv->size, &block_rsv->lock, buf); +} +BTRFS_ATTR(allocation, global_rsv_size, global_rsv_size_show); + +static ssize_t global_rsv_reserved_show(struct kobject *kobj, + struct kobj_attribute *a, char *buf) +{ + struct btrfs_fs_info *fs_info = to_fs_info(kobj->parent); + struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv; + return btrfs_show_u64(&block_rsv->reserved, &block_rsv->lock, buf); +} +BTRFS_ATTR(allocation, global_rsv_reserved, global_rsv_reserved_show); + +#define to_space_info(_kobj) container_of(_kobj, struct btrfs_space_info, kobj) +#define to_raid_kobj(_kobj) container_of(_kobj, struct raid_kobject, kobj) + +static ssize_t raid_bytes_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf); +BTRFS_ATTR(raid, total_bytes, raid_bytes_show); +BTRFS_ATTR(raid, used_bytes, raid_bytes_show); + +static ssize_t raid_bytes_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) + +{ + struct btrfs_space_info *sinfo = to_space_info(kobj->parent); + struct btrfs_block_group *block_group; + int index = btrfs_bg_flags_to_raid_index(to_raid_kobj(kobj)->flags); + u64 val = 0; + + down_read(&sinfo->groups_sem); + list_for_each_entry(block_group, &sinfo->block_groups[index], list) { + if (&attr->attr == BTRFS_ATTR_PTR(raid, total_bytes)) + val += block_group->length; + else + val += block_group->used; + } + up_read(&sinfo->groups_sem); + return sysfs_emit(buf, "%llu\n", val); +} + +/* + * Allocation information about block group profiles. + * + * Path: /sys/fs/btrfs//allocation/// + */ +static struct attribute *raid_attrs[] = { + BTRFS_ATTR_PTR(raid, total_bytes), + BTRFS_ATTR_PTR(raid, used_bytes), + NULL +}; +ATTRIBUTE_GROUPS(raid); + +static void release_raid_kobj(struct kobject *kobj) +{ + kfree(to_raid_kobj(kobj)); +} + +static const struct kobj_type btrfs_raid_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .release = release_raid_kobj, + .default_groups = raid_groups, +}; + +#define SPACE_INFO_ATTR(field) \ +static ssize_t btrfs_space_info_show_##field(struct kobject *kobj, \ + struct kobj_attribute *a, \ + char *buf) \ +{ \ + struct btrfs_space_info *sinfo = to_space_info(kobj); \ + return btrfs_show_u64(&sinfo->field, &sinfo->lock, buf); \ +} \ +BTRFS_ATTR(space_info, field, btrfs_space_info_show_##field) + +static ssize_t btrfs_chunk_size_show(struct kobject *kobj, + struct kobj_attribute *a, char *buf) +{ + struct btrfs_space_info *sinfo = to_space_info(kobj); + + return sysfs_emit(buf, "%llu\n", READ_ONCE(sinfo->chunk_size)); +} + +/* + * Store new chunk size in space info. Can be called on a read-only filesystem. + * + * If the new chunk size value is larger than 10% of free space it is reduced + * to match that limit. Alignment must be to 256M and the system chunk size + * cannot be set. + */ +static ssize_t btrfs_chunk_size_store(struct kobject *kobj, + struct kobj_attribute *a, + const char *buf, size_t len) +{ + struct btrfs_space_info *space_info = to_space_info(kobj); + struct btrfs_fs_info *fs_info = to_fs_info(get_btrfs_kobj(kobj)); + char *retptr; + u64 val; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (!fs_info->fs_devices) + return -EINVAL; + + if (btrfs_is_zoned(fs_info)) + return -EINVAL; + + /* System block type must not be changed. */ + if (space_info->flags & BTRFS_BLOCK_GROUP_SYSTEM) + return -EPERM; + + val = memparse(buf, &retptr); + /* There could be trailing '\n', also catch any typos after the value */ + retptr = skip_spaces(retptr); + if (*retptr != 0 || val == 0) + return -EINVAL; + + val = min(val, BTRFS_MAX_DATA_CHUNK_SIZE); + + /* Limit stripe size to 10% of available space. */ + val = min(mult_perc(fs_info->fs_devices->total_rw_bytes, 10), val); + + /* Must be multiple of 256M. */ + val &= ~((u64)SZ_256M - 1); + + /* Must be at least 256M. */ + if (val < SZ_256M) + return -EINVAL; + + btrfs_update_space_info_chunk_size(space_info, val); + + return len; +} + +static ssize_t btrfs_size_classes_show(struct kobject *kobj, + struct kobj_attribute *a, char *buf) +{ + struct btrfs_space_info *sinfo = to_space_info(kobj); + struct btrfs_block_group *bg; + u32 none = 0; + u32 small = 0; + u32 medium = 0; + u32 large = 0; + + for (int i = 0; i < BTRFS_NR_RAID_TYPES; ++i) { + down_read(&sinfo->groups_sem); + list_for_each_entry(bg, &sinfo->block_groups[i], list) { + if (!btrfs_block_group_should_use_size_class(bg)) + continue; + switch (bg->size_class) { + case BTRFS_BG_SZ_NONE: + none++; + break; + case BTRFS_BG_SZ_SMALL: + small++; + break; + case BTRFS_BG_SZ_MEDIUM: + medium++; + break; + case BTRFS_BG_SZ_LARGE: + large++; + break; + } + } + up_read(&sinfo->groups_sem); + } + return sysfs_emit(buf, "none %u\n" + "small %u\n" + "medium %u\n" + "large %u\n", + none, small, medium, large); +} + +#ifdef CONFIG_BTRFS_DEBUG +/* + * Request chunk allocation with current chunk size. + */ +static ssize_t btrfs_force_chunk_alloc_store(struct kobject *kobj, + struct kobj_attribute *a, + const char *buf, size_t len) +{ + struct btrfs_space_info *space_info = to_space_info(kobj); + struct btrfs_fs_info *fs_info = to_fs_info(get_btrfs_kobj(kobj)); + struct btrfs_trans_handle *trans; + bool val; + int ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (sb_rdonly(fs_info->sb)) + return -EROFS; + + ret = kstrtobool(buf, &val); + if (ret) + return ret; + + if (!val) + return -EINVAL; + + /* + * This is unsafe to be called from sysfs context and may cause + * unexpected problems. + */ + trans = btrfs_start_transaction(fs_info->tree_root, 0); + if (IS_ERR(trans)) + return PTR_ERR(trans); + ret = btrfs_force_chunk_alloc(trans, space_info->flags); + btrfs_end_transaction(trans); + + if (ret == 1) + return len; + + return -ENOSPC; +} +BTRFS_ATTR_W(space_info, force_chunk_alloc, btrfs_force_chunk_alloc_store); + +#endif + +SPACE_INFO_ATTR(flags); +SPACE_INFO_ATTR(total_bytes); +SPACE_INFO_ATTR(bytes_used); +SPACE_INFO_ATTR(bytes_pinned); +SPACE_INFO_ATTR(bytes_reserved); +SPACE_INFO_ATTR(bytes_may_use); +SPACE_INFO_ATTR(bytes_readonly); +SPACE_INFO_ATTR(bytes_zone_unusable); +SPACE_INFO_ATTR(disk_used); +SPACE_INFO_ATTR(disk_total); +SPACE_INFO_ATTR(reclaim_count); +SPACE_INFO_ATTR(reclaim_bytes); +SPACE_INFO_ATTR(reclaim_errors); +BTRFS_ATTR_RW(space_info, chunk_size, btrfs_chunk_size_show, btrfs_chunk_size_store); +BTRFS_ATTR(space_info, size_classes, btrfs_size_classes_show); + +static ssize_t btrfs_sinfo_bg_reclaim_threshold_show(struct kobject *kobj, + struct kobj_attribute *a, + char *buf) +{ + struct btrfs_space_info *space_info = to_space_info(kobj); + ssize_t ret; + + spin_lock(&space_info->lock); + ret = sysfs_emit(buf, "%d\n", btrfs_calc_reclaim_threshold(space_info)); + spin_unlock(&space_info->lock); + return ret; +} + +static ssize_t btrfs_sinfo_bg_reclaim_threshold_store(struct kobject *kobj, + struct kobj_attribute *a, + const char *buf, size_t len) +{ + struct btrfs_space_info *space_info = to_space_info(kobj); + int thresh; + int ret; + + if (READ_ONCE(space_info->dynamic_reclaim)) + return -EINVAL; + + ret = kstrtoint(buf, 10, &thresh); + if (ret) + return ret; + + if (thresh < 0 || thresh > 100) + return -EINVAL; + + WRITE_ONCE(space_info->bg_reclaim_threshold, thresh); + + return len; +} + +BTRFS_ATTR_RW(space_info, bg_reclaim_threshold, + btrfs_sinfo_bg_reclaim_threshold_show, + btrfs_sinfo_bg_reclaim_threshold_store); + +static ssize_t btrfs_sinfo_dynamic_reclaim_show(struct kobject *kobj, + struct kobj_attribute *a, + char *buf) +{ + struct btrfs_space_info *space_info = to_space_info(kobj); + + return sysfs_emit(buf, "%d\n", READ_ONCE(space_info->dynamic_reclaim)); +} + +static ssize_t btrfs_sinfo_dynamic_reclaim_store(struct kobject *kobj, + struct kobj_attribute *a, + const char *buf, size_t len) +{ + struct btrfs_space_info *space_info = to_space_info(kobj); + int dynamic_reclaim; + int ret; + + ret = kstrtoint(buf, 10, &dynamic_reclaim); + if (ret) + return ret; + + if (dynamic_reclaim < 0) + return -EINVAL; + + WRITE_ONCE(space_info->dynamic_reclaim, dynamic_reclaim != 0); + + return len; +} + +BTRFS_ATTR_RW(space_info, dynamic_reclaim, + btrfs_sinfo_dynamic_reclaim_show, + btrfs_sinfo_dynamic_reclaim_store); + +static ssize_t btrfs_sinfo_periodic_reclaim_show(struct kobject *kobj, + struct kobj_attribute *a, + char *buf) +{ + struct btrfs_space_info *space_info = to_space_info(kobj); + + return sysfs_emit(buf, "%d\n", READ_ONCE(space_info->periodic_reclaim)); +} + +static ssize_t btrfs_sinfo_periodic_reclaim_store(struct kobject *kobj, + struct kobj_attribute *a, + const char *buf, size_t len) +{ + struct btrfs_space_info *space_info = to_space_info(kobj); + int periodic_reclaim; + int ret; + + ret = kstrtoint(buf, 10, &periodic_reclaim); + if (ret) + return ret; + + if (periodic_reclaim < 0) + return -EINVAL; + + WRITE_ONCE(space_info->periodic_reclaim, periodic_reclaim != 0); + + return len; +} + +BTRFS_ATTR_RW(space_info, periodic_reclaim, + btrfs_sinfo_periodic_reclaim_show, + btrfs_sinfo_periodic_reclaim_store); + +/* + * Allocation information about block group types. + * + * Path: /sys/fs/btrfs//allocation// + */ +static struct attribute *space_info_attrs[] = { + BTRFS_ATTR_PTR(space_info, flags), + BTRFS_ATTR_PTR(space_info, total_bytes), + BTRFS_ATTR_PTR(space_info, bytes_used), + BTRFS_ATTR_PTR(space_info, bytes_pinned), + BTRFS_ATTR_PTR(space_info, bytes_reserved), + BTRFS_ATTR_PTR(space_info, bytes_may_use), + BTRFS_ATTR_PTR(space_info, bytes_readonly), + BTRFS_ATTR_PTR(space_info, bytes_zone_unusable), + BTRFS_ATTR_PTR(space_info, disk_used), + BTRFS_ATTR_PTR(space_info, disk_total), + BTRFS_ATTR_PTR(space_info, bg_reclaim_threshold), + BTRFS_ATTR_PTR(space_info, dynamic_reclaim), + BTRFS_ATTR_PTR(space_info, chunk_size), + BTRFS_ATTR_PTR(space_info, size_classes), + BTRFS_ATTR_PTR(space_info, reclaim_count), + BTRFS_ATTR_PTR(space_info, reclaim_bytes), + BTRFS_ATTR_PTR(space_info, reclaim_errors), + BTRFS_ATTR_PTR(space_info, periodic_reclaim), +#ifdef CONFIG_BTRFS_DEBUG + BTRFS_ATTR_PTR(space_info, force_chunk_alloc), +#endif + NULL, +}; +ATTRIBUTE_GROUPS(space_info); + +static void space_info_release(struct kobject *kobj) +{ + struct btrfs_space_info *sinfo = to_space_info(kobj); + kfree(sinfo); +} + +static const struct kobj_type space_info_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .release = space_info_release, + .default_groups = space_info_groups, +}; + +/* + * Allocation information about block groups. + * + * Path: /sys/fs/btrfs//allocation/ + */ +static const struct attribute *allocation_attrs[] = { + BTRFS_ATTR_PTR(allocation, global_rsv_reserved), + BTRFS_ATTR_PTR(allocation, global_rsv_size), + NULL, +}; + +static ssize_t btrfs_label_show(struct kobject *kobj, + struct kobj_attribute *a, char *buf) +{ + struct btrfs_fs_info *fs_info = to_fs_info(kobj); + char *label = fs_info->super_copy->label; + ssize_t ret; + + spin_lock(&fs_info->super_lock); + ret = sysfs_emit(buf, label[0] ? "%s\n" : "%s", label); + spin_unlock(&fs_info->super_lock); + + return ret; +} + +static ssize_t btrfs_label_store(struct kobject *kobj, + struct kobj_attribute *a, + const char *buf, size_t len) +{ + struct btrfs_fs_info *fs_info = to_fs_info(kobj); + size_t p_len; + + if (!fs_info) + return -EPERM; + + if (sb_rdonly(fs_info->sb)) + return -EROFS; + + /* + * p_len is the len until the first occurrence of either + * '\n' or '\0' + */ + p_len = strcspn(buf, "\n"); + + if (p_len >= BTRFS_LABEL_SIZE) + return -EINVAL; + + spin_lock(&fs_info->super_lock); + memset(fs_info->super_copy->label, 0, BTRFS_LABEL_SIZE); + memcpy(fs_info->super_copy->label, buf, p_len); + spin_unlock(&fs_info->super_lock); + + /* + * We don't want to do full transaction commit from inside sysfs + */ + set_bit(BTRFS_FS_NEED_TRANS_COMMIT, &fs_info->flags); + wake_up_process(fs_info->transaction_kthread); + + return len; +} +BTRFS_ATTR_RW(, label, btrfs_label_show, btrfs_label_store); + +static ssize_t btrfs_nodesize_show(struct kobject *kobj, + struct kobj_attribute *a, char *buf) +{ + struct btrfs_fs_info *fs_info = to_fs_info(kobj); + + return sysfs_emit(buf, "%u\n", fs_info->nodesize); +} + +BTRFS_ATTR(, nodesize, btrfs_nodesize_show); + +static ssize_t btrfs_sectorsize_show(struct kobject *kobj, + struct kobj_attribute *a, char *buf) +{ + struct btrfs_fs_info *fs_info = to_fs_info(kobj); + + return sysfs_emit(buf, "%u\n", fs_info->sectorsize); +} + +BTRFS_ATTR(, sectorsize, btrfs_sectorsize_show); + +static ssize_t btrfs_commit_stats_show(struct kobject *kobj, + struct kobj_attribute *a, char *buf) +{ + struct btrfs_fs_info *fs_info = to_fs_info(kobj); + u64 now = ktime_get_ns(); + u64 start_time = fs_info->commit_stats.critical_section_start_time; + u64 pending = 0; + + if (start_time) + pending = now - start_time; + + return sysfs_emit(buf, + "commits %llu\n" + "cur_commit_ms %llu\n" + "last_commit_ms %llu\n" + "max_commit_ms %llu\n" + "total_commit_ms %llu\n", + fs_info->commit_stats.commit_count, + div_u64(pending, NSEC_PER_MSEC), + div_u64(fs_info->commit_stats.last_commit_dur, NSEC_PER_MSEC), + div_u64(fs_info->commit_stats.max_commit_dur, NSEC_PER_MSEC), + div_u64(fs_info->commit_stats.total_commit_dur, NSEC_PER_MSEC)); +} + +static ssize_t btrfs_commit_stats_store(struct kobject *kobj, + struct kobj_attribute *a, + const char *buf, size_t len) +{ + struct btrfs_fs_info *fs_info = to_fs_info(kobj); + unsigned long val; + int ret; + + if (!fs_info) + return -EPERM; + + if (!capable(CAP_SYS_RESOURCE)) + return -EPERM; + + ret = kstrtoul(buf, 10, &val); + if (ret) + return ret; + if (val) + return -EINVAL; + + WRITE_ONCE(fs_info->commit_stats.max_commit_dur, 0); + + return len; +} +BTRFS_ATTR_RW(, commit_stats, btrfs_commit_stats_show, btrfs_commit_stats_store); + +static ssize_t btrfs_clone_alignment_show(struct kobject *kobj, + struct kobj_attribute *a, char *buf) +{ + struct btrfs_fs_info *fs_info = to_fs_info(kobj); + + return sysfs_emit(buf, "%u\n", fs_info->sectorsize); +} + +BTRFS_ATTR(, clone_alignment, btrfs_clone_alignment_show); + +static ssize_t quota_override_show(struct kobject *kobj, + struct kobj_attribute *a, char *buf) +{ + struct btrfs_fs_info *fs_info = to_fs_info(kobj); + int quota_override; + + quota_override = test_bit(BTRFS_FS_QUOTA_OVERRIDE, &fs_info->flags); + return sysfs_emit(buf, "%d\n", quota_override); +} + +static ssize_t quota_override_store(struct kobject *kobj, + struct kobj_attribute *a, + const char *buf, size_t len) +{ + struct btrfs_fs_info *fs_info = to_fs_info(kobj); + unsigned long knob; + int ret; + + if (!fs_info) + return -EPERM; + + if (!capable(CAP_SYS_RESOURCE)) + return -EPERM; + + ret = kstrtoul(buf, 10, &knob); + if (ret) + return ret; + if (knob > 1) + return -EINVAL; + + if (knob) + set_bit(BTRFS_FS_QUOTA_OVERRIDE, &fs_info->flags); + else + clear_bit(BTRFS_FS_QUOTA_OVERRIDE, &fs_info->flags); + + return len; +} + +BTRFS_ATTR_RW(, quota_override, quota_override_show, quota_override_store); + +static ssize_t btrfs_metadata_uuid_show(struct kobject *kobj, + struct kobj_attribute *a, char *buf) +{ + struct btrfs_fs_info *fs_info = to_fs_info(kobj); + + return sysfs_emit(buf, "%pU\n", fs_info->fs_devices->metadata_uuid); +} + +BTRFS_ATTR(, metadata_uuid, btrfs_metadata_uuid_show); + +static ssize_t btrfs_checksum_show(struct kobject *kobj, + struct kobj_attribute *a, char *buf) +{ + struct btrfs_fs_info *fs_info = to_fs_info(kobj); + u16 csum_type = btrfs_super_csum_type(fs_info->super_copy); + const char *csum_name = btrfs_super_csum_name(csum_type); + + return sysfs_emit(buf, "%s (%s-lib)\n", csum_name, csum_name); +} + +BTRFS_ATTR(, checksum, btrfs_checksum_show); + +static ssize_t btrfs_exclusive_operation_show(struct kobject *kobj, + struct kobj_attribute *a, char *buf) +{ + struct btrfs_fs_info *fs_info = to_fs_info(kobj); + const char *str; + + switch (READ_ONCE(fs_info->exclusive_operation)) { + case BTRFS_EXCLOP_NONE: + str = "none\n"; + break; + case BTRFS_EXCLOP_BALANCE: + str = "balance\n"; + break; + case BTRFS_EXCLOP_BALANCE_PAUSED: + str = "balance paused\n"; + break; + case BTRFS_EXCLOP_DEV_ADD: + str = "device add\n"; + break; + case BTRFS_EXCLOP_DEV_REMOVE: + str = "device remove\n"; + break; + case BTRFS_EXCLOP_DEV_REPLACE: + str = "device replace\n"; + break; + case BTRFS_EXCLOP_RESIZE: + str = "resize\n"; + break; + case BTRFS_EXCLOP_SWAP_ACTIVATE: + str = "swap activate\n"; + break; + default: + str = "UNKNOWN\n"; + break; + } + return sysfs_emit(buf, "%s", str); +} +BTRFS_ATTR(, exclusive_operation, btrfs_exclusive_operation_show); + +static ssize_t btrfs_generation_show(struct kobject *kobj, + struct kobj_attribute *a, char *buf) +{ + struct btrfs_fs_info *fs_info = to_fs_info(kobj); + + return sysfs_emit(buf, "%llu\n", btrfs_get_fs_generation(fs_info)); +} +BTRFS_ATTR(, generation, btrfs_generation_show); + +static ssize_t btrfs_temp_fsid_show(struct kobject *kobj, + struct kobj_attribute *a, char *buf) +{ + struct btrfs_fs_info *fs_info = to_fs_info(kobj); + + return sysfs_emit(buf, "%d\n", fs_info->fs_devices->temp_fsid); +} +BTRFS_ATTR(, temp_fsid, btrfs_temp_fsid_show); + +static const char *btrfs_read_policy_name[] = { + "pid", +#ifdef CONFIG_BTRFS_EXPERIMENTAL + "round-robin", + "devid", +#endif +}; + +#ifdef CONFIG_BTRFS_EXPERIMENTAL + +/* Global module configuration parameters. */ +static char *read_policy; +char *btrfs_get_mod_read_policy(void) +{ + return read_policy; +} + +/* Set perms to 0, disable /sys/module/btrfs/parameters/read_policy interface. */ +module_param(read_policy, charp, 0); +MODULE_PARM_DESC(read_policy, +"Global read policy: pid (default), round-robin[:], devid[:]"); +#endif + +int btrfs_read_policy_to_enum(const char *str, s64 *value_ret) +{ + char param[32]; + char __maybe_unused *value_str; + + if (!str || strlen(str) == 0) + return 0; + + strscpy(param, str); + +#ifdef CONFIG_BTRFS_EXPERIMENTAL + /* Separate value from input in policy:value format. */ + value_str = strchr(param, ':'); + if (value_str) { + char *retptr; + + *value_str = 0; + value_str++; + if (!value_ret) + return -EINVAL; + + *value_ret = memparse(value_str, &retptr); + /* There could be any trailing typos after the value. */ + retptr = skip_spaces(retptr); + if (*retptr != 0 || *value_ret <= 0) + return -EINVAL; + } +#endif + + return sysfs_match_string(btrfs_read_policy_name, param); +} + +#ifdef CONFIG_BTRFS_EXPERIMENTAL +int __init btrfs_read_policy_init(void) +{ + s64 value; + + if (btrfs_read_policy_to_enum(read_policy, &value) == -EINVAL) { + btrfs_err(NULL, "invalid read policy or value %s", read_policy); + return -EINVAL; + } + + return 0; +} +#endif + +static ssize_t btrfs_read_policy_show(struct kobject *kobj, + struct kobj_attribute *a, char *buf) +{ + struct btrfs_fs_devices *fs_devices = to_fs_devs(kobj); + const enum btrfs_read_policy policy = READ_ONCE(fs_devices->read_policy); + ssize_t ret = 0; + int i; + + for (i = 0; i < BTRFS_NR_READ_POLICY; i++) { + if (ret != 0) + ret += sysfs_emit_at(buf, ret, " "); + + if (i == policy) + ret += sysfs_emit_at(buf, ret, "["); + + ret += sysfs_emit_at(buf, ret, "%s", btrfs_read_policy_name[i]); + +#ifdef CONFIG_BTRFS_EXPERIMENTAL + if (i == BTRFS_READ_POLICY_RR) + ret += sysfs_emit_at(buf, ret, ":%u", + READ_ONCE(fs_devices->rr_min_contig_read)); + + if (i == BTRFS_READ_POLICY_DEVID) + ret += sysfs_emit_at(buf, ret, ":%llu", + READ_ONCE(fs_devices->read_devid)); +#endif + if (i == policy) + ret += sysfs_emit_at(buf, ret, "]"); + } + + ret += sysfs_emit_at(buf, ret, "\n"); + + return ret; +} + +static ssize_t btrfs_read_policy_store(struct kobject *kobj, + struct kobj_attribute *a, + const char *buf, size_t len) +{ + struct btrfs_fs_devices *fs_devices = to_fs_devs(kobj); + int index; + s64 value = -1; + + index = btrfs_read_policy_to_enum(buf, &value); + if (index < 0) + return -EINVAL; + +#ifdef CONFIG_BTRFS_EXPERIMENTAL + /* If moving from RR then disable collecting fs stats. */ + if (fs_devices->read_policy == BTRFS_READ_POLICY_RR && index != BTRFS_READ_POLICY_RR) + fs_devices->collect_fs_stats = false; + + if (index == BTRFS_READ_POLICY_RR) { + if (value != -1) { + const u32 sectorsize = fs_devices->fs_info->sectorsize; + + if (!IS_ALIGNED(value, sectorsize)) { + u64 temp_value = round_up(value, sectorsize); + + btrfs_debug(fs_devices->fs_info, +"read_policy: min contig read %lld should be multiple of sectorsize %u, rounded to %llu", + value, sectorsize, temp_value); + value = temp_value; + } + } else { + value = BTRFS_DEFAULT_RR_MIN_CONTIG_READ; + } + + if (index != READ_ONCE(fs_devices->read_policy) || + value != READ_ONCE(fs_devices->rr_min_contig_read)) { + WRITE_ONCE(fs_devices->read_policy, index); + WRITE_ONCE(fs_devices->rr_min_contig_read, value); + + btrfs_info(fs_devices->fs_info, "read policy set to '%s:%lld'", + btrfs_read_policy_name[index], value); + } + + fs_devices->collect_fs_stats = true; + + return len; + } + + if (index == BTRFS_READ_POLICY_DEVID) { + if (value != -1) { + BTRFS_DEV_LOOKUP_ARGS(args); + + /* Validate input devid. */ + args.devid = value; + if (btrfs_find_device(fs_devices, &args) == NULL) + return -EINVAL; + } else { + /* Set default devid to the devid of the latest device. */ + value = fs_devices->latest_dev->devid; + } + + if (index != READ_ONCE(fs_devices->read_policy) || + value != READ_ONCE(fs_devices->read_devid)) { + WRITE_ONCE(fs_devices->read_policy, index); + WRITE_ONCE(fs_devices->read_devid, value); + + btrfs_info(fs_devices->fs_info, "read policy set to '%s:%llu'", + btrfs_read_policy_name[index], value); + } + + return len; + } +#endif + if (index != READ_ONCE(fs_devices->read_policy)) { + WRITE_ONCE(fs_devices->read_policy, index); + btrfs_info(fs_devices->fs_info, "read policy set to '%s'", + btrfs_read_policy_name[index]); + } + + return len; +} +BTRFS_ATTR_RW(, read_policy, btrfs_read_policy_show, btrfs_read_policy_store); + +static ssize_t btrfs_bg_reclaim_threshold_show(struct kobject *kobj, + struct kobj_attribute *a, + char *buf) +{ + struct btrfs_fs_info *fs_info = to_fs_info(kobj); + + return sysfs_emit(buf, "%d\n", READ_ONCE(fs_info->bg_reclaim_threshold)); +} + +static ssize_t btrfs_bg_reclaim_threshold_store(struct kobject *kobj, + struct kobj_attribute *a, + const char *buf, size_t len) +{ + struct btrfs_fs_info *fs_info = to_fs_info(kobj); + int thresh; + int ret; + + ret = kstrtoint(buf, 10, &thresh); + if (ret) + return ret; + +#ifdef CONFIG_BTRFS_DEBUG + if (thresh != 0 && (thresh > 100)) + return -EINVAL; +#else + if (thresh != 0 && (thresh <= 50 || thresh > 100)) + return -EINVAL; +#endif + + WRITE_ONCE(fs_info->bg_reclaim_threshold, thresh); + + return len; +} +BTRFS_ATTR_RW(, bg_reclaim_threshold, btrfs_bg_reclaim_threshold_show, + btrfs_bg_reclaim_threshold_store); + +/* + * Per-filesystem information and stats. + * + * Path: /sys/fs/btrfs// + */ +static const struct attribute *btrfs_attrs[] = { + BTRFS_ATTR_PTR(, label), + BTRFS_ATTR_PTR(, nodesize), + BTRFS_ATTR_PTR(, sectorsize), + BTRFS_ATTR_PTR(, clone_alignment), + BTRFS_ATTR_PTR(, quota_override), + BTRFS_ATTR_PTR(, metadata_uuid), + BTRFS_ATTR_PTR(, checksum), + BTRFS_ATTR_PTR(, exclusive_operation), + BTRFS_ATTR_PTR(, generation), + BTRFS_ATTR_PTR(, read_policy), + BTRFS_ATTR_PTR(, bg_reclaim_threshold), + BTRFS_ATTR_PTR(, commit_stats), + BTRFS_ATTR_PTR(, temp_fsid), + NULL, +}; + +static void btrfs_release_fsid_kobj(struct kobject *kobj) +{ + struct btrfs_fs_devices *fs_devs = to_fs_devs(kobj); + + memset(&fs_devs->fsid_kobj, 0, sizeof(struct kobject)); + complete(&fs_devs->kobj_unregister); +} + +static const struct kobj_type btrfs_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .release = btrfs_release_fsid_kobj, +}; + +static inline struct btrfs_fs_devices *to_fs_devs(struct kobject *kobj) +{ + if (kobj->ktype != &btrfs_ktype) + return NULL; + return container_of(kobj, struct btrfs_fs_devices, fsid_kobj); +} + +static inline struct btrfs_fs_info *to_fs_info(struct kobject *kobj) +{ + if (kobj->ktype != &btrfs_ktype) + return NULL; + return to_fs_devs(kobj)->fs_info; +} + +static struct kobject *get_btrfs_kobj(struct kobject *kobj) +{ + while (kobj) { + if (kobj->ktype == &btrfs_ktype) + return kobj; + kobj = kobj->parent; + } + return NULL; +} + +#define NUM_FEATURE_BITS 64 +#define BTRFS_FEATURE_NAME_MAX 13 +static char btrfs_unknown_feature_names[FEAT_MAX][NUM_FEATURE_BITS][BTRFS_FEATURE_NAME_MAX]; +static struct btrfs_feature_attr btrfs_feature_attrs[FEAT_MAX][NUM_FEATURE_BITS]; + +static_assert(ARRAY_SIZE(btrfs_unknown_feature_names) == + ARRAY_SIZE(btrfs_feature_attrs)); +static_assert(ARRAY_SIZE(btrfs_unknown_feature_names[0]) == + ARRAY_SIZE(btrfs_feature_attrs[0])); + +static const u64 supported_feature_masks[FEAT_MAX] = { + [FEAT_COMPAT] = BTRFS_FEATURE_COMPAT_SUPP, + [FEAT_COMPAT_RO] = BTRFS_FEATURE_COMPAT_RO_SUPP, + [FEAT_INCOMPAT] = BTRFS_FEATURE_INCOMPAT_SUPP, +}; + +static int addrm_unknown_feature_attrs(struct btrfs_fs_info *fs_info, bool add) +{ + int set; + + for (set = 0; set < FEAT_MAX; set++) { + int i; + struct attribute *attrs[2]; + struct attribute_group agroup = { + .name = "features", + .attrs = attrs, + }; + u64 features = get_features(fs_info, set); + features &= ~supported_feature_masks[set]; + + if (!features) + continue; + + attrs[1] = NULL; + for (i = 0; i < NUM_FEATURE_BITS; i++) { + struct btrfs_feature_attr *fa; + + if (!(features & (1ULL << i))) + continue; + + fa = &btrfs_feature_attrs[set][i]; + attrs[0] = &fa->kobj_attr.attr; + if (add) { + int ret; + ret = sysfs_merge_group(&fs_info->fs_devices->fsid_kobj, + &agroup); + if (ret) + return ret; + } else + sysfs_unmerge_group(&fs_info->fs_devices->fsid_kobj, + &agroup); + } + + } + return 0; +} + +static void __btrfs_sysfs_remove_fsid(struct btrfs_fs_devices *fs_devs) +{ + if (fs_devs->devinfo_kobj) { + kobject_del(fs_devs->devinfo_kobj); + kobject_put(fs_devs->devinfo_kobj); + fs_devs->devinfo_kobj = NULL; + } + + if (fs_devs->devices_kobj) { + kobject_del(fs_devs->devices_kobj); + kobject_put(fs_devs->devices_kobj); + fs_devs->devices_kobj = NULL; + } + + if (fs_devs->fsid_kobj.state_initialized) { + kobject_del(&fs_devs->fsid_kobj); + kobject_put(&fs_devs->fsid_kobj); + wait_for_completion(&fs_devs->kobj_unregister); + } +} + +/* when fs_devs is NULL it will remove all fsid kobject */ +void btrfs_sysfs_remove_fsid(struct btrfs_fs_devices *fs_devs) +{ + struct list_head *fs_uuids = btrfs_get_fs_uuids(); + + if (fs_devs) { + __btrfs_sysfs_remove_fsid(fs_devs); + return; + } + + list_for_each_entry(fs_devs, fs_uuids, fs_list) { + __btrfs_sysfs_remove_fsid(fs_devs); + } +} + +static void btrfs_sysfs_remove_fs_devices(struct btrfs_fs_devices *fs_devices) +{ + struct btrfs_device *device; + struct btrfs_fs_devices *seed; + + list_for_each_entry(device, &fs_devices->devices, dev_list) + btrfs_sysfs_remove_device(device); + + list_for_each_entry(seed, &fs_devices->seed_list, seed_list) { + list_for_each_entry(device, &seed->devices, dev_list) + btrfs_sysfs_remove_device(device); + } +} + +void btrfs_sysfs_remove_mounted(struct btrfs_fs_info *fs_info) +{ + struct kobject *fsid_kobj = &fs_info->fs_devices->fsid_kobj; + + sysfs_remove_link(fsid_kobj, "bdi"); + + if (fs_info->space_info_kobj) { + sysfs_remove_files(fs_info->space_info_kobj, allocation_attrs); + kobject_del(fs_info->space_info_kobj); + kobject_put(fs_info->space_info_kobj); + } + if (fs_info->discard_kobj) { + sysfs_remove_files(fs_info->discard_kobj, discard_attrs); + kobject_del(fs_info->discard_kobj); + kobject_put(fs_info->discard_kobj); + } +#ifdef CONFIG_BTRFS_DEBUG + if (fs_info->debug_kobj) { + sysfs_remove_files(fs_info->debug_kobj, btrfs_debug_mount_attrs); + kobject_del(fs_info->debug_kobj); + kobject_put(fs_info->debug_kobj); + } +#endif + addrm_unknown_feature_attrs(fs_info, false); + sysfs_remove_group(fsid_kobj, &btrfs_feature_attr_group); + sysfs_remove_files(fsid_kobj, btrfs_attrs); + btrfs_sysfs_remove_fs_devices(fs_info->fs_devices); +} + +static const char * const btrfs_feature_set_names[FEAT_MAX] = { + [FEAT_COMPAT] = "compat", + [FEAT_COMPAT_RO] = "compat_ro", + [FEAT_INCOMPAT] = "incompat", +}; + +const char *btrfs_feature_set_name(enum btrfs_feature_set set) +{ + return btrfs_feature_set_names[set]; +} + +char *btrfs_printable_features(enum btrfs_feature_set set, u64 flags) +{ + size_t bufsize = 4096; /* safe max, 64 names * 64 bytes */ + int len = 0; + int i; + char *str; + + str = kmalloc(bufsize, GFP_KERNEL); + if (!str) + return str; + + for (i = 0; i < ARRAY_SIZE(btrfs_feature_attrs[set]); i++) { + const char *name; + + if (!(flags & (1ULL << i))) + continue; + + name = btrfs_feature_attrs[set][i].kobj_attr.attr.name; + len += scnprintf(str + len, bufsize - len, "%s%s", + len ? "," : "", name); + } + + return str; +} + +static void init_feature_attrs(void) +{ + struct btrfs_feature_attr *fa; + int set, i; + + memset(btrfs_feature_attrs, 0, sizeof(btrfs_feature_attrs)); + memset(btrfs_unknown_feature_names, 0, + sizeof(btrfs_unknown_feature_names)); + + for (i = 0; btrfs_supported_feature_attrs[i]; i++) { + struct btrfs_feature_attr *sfa; + struct attribute *a = btrfs_supported_feature_attrs[i]; + int bit; + sfa = attr_to_btrfs_feature_attr(a); + bit = ilog2(sfa->feature_bit); + fa = &btrfs_feature_attrs[sfa->feature_set][bit]; + + fa->kobj_attr.attr.name = sfa->kobj_attr.attr.name; + } + + for (set = 0; set < FEAT_MAX; set++) { + for (i = 0; i < ARRAY_SIZE(btrfs_feature_attrs[set]); i++) { + char *name = btrfs_unknown_feature_names[set][i]; + fa = &btrfs_feature_attrs[set][i]; + + if (fa->kobj_attr.attr.name) + continue; + + snprintf(name, BTRFS_FEATURE_NAME_MAX, "%s:%u", + btrfs_feature_set_names[set], i); + + fa->kobj_attr.attr.name = name; + fa->kobj_attr.attr.mode = S_IRUGO; + fa->feature_set = set; + fa->feature_bit = 1ULL << i; + } + } +} + +/* + * Create a sysfs entry for a given block group type at path + * /sys/fs/btrfs/UUID/allocation/data/TYPE + */ +void btrfs_sysfs_add_block_group_type(struct btrfs_block_group *cache) +{ + struct btrfs_fs_info *fs_info = cache->fs_info; + struct btrfs_space_info *space_info = cache->space_info; + struct raid_kobject *rkobj; + const int index = btrfs_bg_flags_to_raid_index(cache->flags); + unsigned int nofs_flag; + int ret; + + /* + * Setup a NOFS context because kobject_add(), deep in its call chain, + * does GFP_KERNEL allocations, and we are often called in a context + * where if reclaim is triggered we can deadlock (we are either holding + * a transaction handle or some lock required for a transaction + * commit). + */ + nofs_flag = memalloc_nofs_save(); + + rkobj = kzalloc_obj(*rkobj, GFP_NOFS); + if (!rkobj) { + memalloc_nofs_restore(nofs_flag); + btrfs_warn(cache->fs_info, + "couldn't alloc memory for raid level kobject"); + return; + } + + rkobj->flags = cache->flags; + kobject_init(&rkobj->kobj, &btrfs_raid_ktype); + + /* + * We call this either on mount, or if we've created a block group for a + * new index type while running (i.e. when restriping). The running + * case is tricky because we could race with other threads, so we need + * to have this check to make sure we didn't already init the kobject. + * + * We don't have to protect on the free side because it only happens on + * unmount. + */ + spin_lock(&space_info->lock); + if (space_info->block_group_kobjs[index]) { + spin_unlock(&space_info->lock); + kobject_put(&rkobj->kobj); + return; + } else { + space_info->block_group_kobjs[index] = &rkobj->kobj; + } + spin_unlock(&space_info->lock); + + ret = kobject_add(&rkobj->kobj, &space_info->kobj, "%s", + btrfs_bg_type_to_raid_name(rkobj->flags)); + memalloc_nofs_restore(nofs_flag); + if (ret) { + spin_lock(&space_info->lock); + space_info->block_group_kobjs[index] = NULL; + spin_unlock(&space_info->lock); + kobject_put(&rkobj->kobj); + btrfs_warn(fs_info, + "failed to add kobject for block cache, ignoring"); + return; + } +} + +/* + * Remove sysfs directories for all block group types of a given space info and + * the space info as well + */ +void btrfs_sysfs_remove_space_info(struct btrfs_space_info *space_info) +{ + int i; + + for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) { + struct kobject *kobj; + + kobj = space_info->block_group_kobjs[i]; + space_info->block_group_kobjs[i] = NULL; + if (kobj) { + kobject_del(kobj); + kobject_put(kobj); + } + } + kobject_del(&space_info->kobj); + kobject_put(&space_info->kobj); +} + +static const char *alloc_name(struct btrfs_space_info *space_info) +{ + u64 flags = space_info->flags; + + switch (flags) { + case BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA: + return "mixed"; + case BTRFS_BLOCK_GROUP_METADATA: + switch (space_info->subgroup_id) { + case BTRFS_SUB_GROUP_PRIMARY: + return "metadata"; + case BTRFS_SUB_GROUP_TREELOG: + return "metadata-treelog"; + default: + WARN_ON_ONCE(1); + return "metadata (unknown sub-group)"; + } + case BTRFS_BLOCK_GROUP_DATA: + switch (space_info->subgroup_id) { + case BTRFS_SUB_GROUP_PRIMARY: + return "data"; + case BTRFS_SUB_GROUP_DATA_RELOC: + return "data-reloc"; + default: + WARN_ON_ONCE(1); + return "data (unknown sub-group)"; + } + case BTRFS_BLOCK_GROUP_SYSTEM: + ASSERT(space_info->subgroup_id == BTRFS_SUB_GROUP_PRIMARY); + return "system"; + case BTRFS_BLOCK_GROUP_METADATA_REMAP: + return "metadata-remap"; + default: + WARN_ON(1); + return "invalid-combination"; + } +} + +/* + * Create a sysfs entry for a space info type at path + * /sys/fs/btrfs/UUID/allocation/TYPE + */ +int btrfs_sysfs_add_space_info_type(struct btrfs_space_info *space_info) +{ + int ret; + + ret = kobject_init_and_add(&space_info->kobj, &space_info_ktype, + space_info->fs_info->space_info_kobj, "%s", + alloc_name(space_info)); + if (ret) { + kobject_put(&space_info->kobj); + return ret; + } + + return 0; +} + +void btrfs_sysfs_remove_device(struct btrfs_device *device) +{ + struct kobject *devices_kobj; + + /* + * Seed fs_devices devices_kobj aren't used, fetch kobject from the + * fs_info::fs_devices. + */ + devices_kobj = device->fs_info->fs_devices->devices_kobj; + ASSERT(devices_kobj); + + if (device->bdev) + sysfs_remove_link(devices_kobj, bdev_kobj(device->bdev)->name); + + if (device->devid_kobj.state_initialized) { + kobject_del(&device->devid_kobj); + kobject_put(&device->devid_kobj); + wait_for_completion(&device->kobj_unregister); + } +} + +static ssize_t btrfs_devinfo_in_fs_metadata_show(struct kobject *kobj, + struct kobj_attribute *a, + char *buf) +{ + int val; + struct btrfs_device *device = container_of(kobj, struct btrfs_device, + devid_kobj); + + val = !!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state); + + return sysfs_emit(buf, "%d\n", val); +} +BTRFS_ATTR(devid, in_fs_metadata, btrfs_devinfo_in_fs_metadata_show); + +static ssize_t btrfs_devinfo_missing_show(struct kobject *kobj, + struct kobj_attribute *a, char *buf) +{ + int val; + struct btrfs_device *device = container_of(kobj, struct btrfs_device, + devid_kobj); + + val = !!test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state); + + return sysfs_emit(buf, "%d\n", val); +} +BTRFS_ATTR(devid, missing, btrfs_devinfo_missing_show); + +static ssize_t btrfs_devinfo_replace_target_show(struct kobject *kobj, + struct kobj_attribute *a, + char *buf) +{ + int val; + struct btrfs_device *device = container_of(kobj, struct btrfs_device, + devid_kobj); + + val = !!test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state); + + return sysfs_emit(buf, "%d\n", val); +} +BTRFS_ATTR(devid, replace_target, btrfs_devinfo_replace_target_show); + +static ssize_t btrfs_devinfo_scrub_speed_max_show(struct kobject *kobj, + struct kobj_attribute *a, + char *buf) +{ + struct btrfs_device *device = container_of(kobj, struct btrfs_device, + devid_kobj); + + return sysfs_emit(buf, "%llu\n", READ_ONCE(device->scrub_speed_max)); +} + +static ssize_t btrfs_devinfo_scrub_speed_max_store(struct kobject *kobj, + struct kobj_attribute *a, + const char *buf, size_t len) +{ + struct btrfs_device *device = container_of(kobj, struct btrfs_device, + devid_kobj); + char *endptr; + unsigned long long limit; + + limit = memparse(buf, &endptr); + /* There could be trailing '\n', also catch any typos after the value. */ + endptr = skip_spaces(endptr); + if (*endptr != 0) + return -EINVAL; + WRITE_ONCE(device->scrub_speed_max, limit); + return len; +} +BTRFS_ATTR_RW(devid, scrub_speed_max, btrfs_devinfo_scrub_speed_max_show, + btrfs_devinfo_scrub_speed_max_store); + +static ssize_t btrfs_devinfo_writeable_show(struct kobject *kobj, + struct kobj_attribute *a, char *buf) +{ + int val; + struct btrfs_device *device = container_of(kobj, struct btrfs_device, + devid_kobj); + + val = !!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state); + + return sysfs_emit(buf, "%d\n", val); +} +BTRFS_ATTR(devid, writeable, btrfs_devinfo_writeable_show); + +static ssize_t btrfs_devinfo_fsid_show(struct kobject *kobj, + struct kobj_attribute *a, char *buf) +{ + struct btrfs_device *device = container_of(kobj, struct btrfs_device, + devid_kobj); + + return sysfs_emit(buf, "%pU\n", device->fs_devices->fsid); +} +BTRFS_ATTR(devid, fsid, btrfs_devinfo_fsid_show); + +static ssize_t btrfs_devinfo_error_stats_show(struct kobject *kobj, + struct kobj_attribute *a, char *buf) +{ + struct btrfs_device *device = container_of(kobj, struct btrfs_device, + devid_kobj); + + if (!device->dev_stats_valid) + return sysfs_emit(buf, "invalid\n"); + + /* + * Print all at once so we get a snapshot of all values from the same + * time. Keep them in sync and in order of definition of + * btrfs_dev_stat_values. + */ + return sysfs_emit(buf, + "write_errs %d\n" + "read_errs %d\n" + "flush_errs %d\n" + "corruption_errs %d\n" + "generation_errs %d\n", + btrfs_dev_stat_read(device, BTRFS_DEV_STAT_WRITE_ERRS), + btrfs_dev_stat_read(device, BTRFS_DEV_STAT_READ_ERRS), + btrfs_dev_stat_read(device, BTRFS_DEV_STAT_FLUSH_ERRS), + btrfs_dev_stat_read(device, BTRFS_DEV_STAT_CORRUPTION_ERRS), + btrfs_dev_stat_read(device, BTRFS_DEV_STAT_GENERATION_ERRS)); +} +BTRFS_ATTR(devid, error_stats, btrfs_devinfo_error_stats_show); + +/* + * Information about one device. + * + * Path: /sys/fs/btrfs//devinfo// + */ +static struct attribute *devid_attrs[] = { + BTRFS_ATTR_PTR(devid, error_stats), + BTRFS_ATTR_PTR(devid, fsid), + BTRFS_ATTR_PTR(devid, in_fs_metadata), + BTRFS_ATTR_PTR(devid, missing), + BTRFS_ATTR_PTR(devid, replace_target), + BTRFS_ATTR_PTR(devid, scrub_speed_max), + BTRFS_ATTR_PTR(devid, writeable), + NULL +}; +ATTRIBUTE_GROUPS(devid); + +static void btrfs_release_devid_kobj(struct kobject *kobj) +{ + struct btrfs_device *device = container_of(kobj, struct btrfs_device, + devid_kobj); + + memset(&device->devid_kobj, 0, sizeof(struct kobject)); + complete(&device->kobj_unregister); +} + +static const struct kobj_type devid_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .default_groups = devid_groups, + .release = btrfs_release_devid_kobj, +}; + +int btrfs_sysfs_add_device(struct btrfs_device *device) +{ + int ret; + unsigned int nofs_flag; + struct kobject *devices_kobj; + struct kobject *devinfo_kobj; + + /* + * Make sure we use the fs_info::fs_devices to fetch the kobjects even + * for the seed fs_devices + */ + devices_kobj = device->fs_info->fs_devices->devices_kobj; + devinfo_kobj = device->fs_info->fs_devices->devinfo_kobj; + ASSERT(devices_kobj); + ASSERT(devinfo_kobj); + + nofs_flag = memalloc_nofs_save(); + + if (device->bdev) { + struct kobject *disk_kobj = bdev_kobj(device->bdev); + + ret = sysfs_create_link(devices_kobj, disk_kobj, disk_kobj->name); + if (ret) { + btrfs_warn(device->fs_info, + "creating sysfs device link for devid %llu failed: %d", + device->devid, ret); + goto out; + } + } + + init_completion(&device->kobj_unregister); + ret = kobject_init_and_add(&device->devid_kobj, &devid_ktype, + devinfo_kobj, "%llu", device->devid); + if (ret) { + kobject_put(&device->devid_kobj); + btrfs_warn(device->fs_info, + "devinfo init for devid %llu failed: %d", + device->devid, ret); + } + +out: + memalloc_nofs_restore(nofs_flag); + return ret; +} + +static int btrfs_sysfs_add_fs_devices(struct btrfs_fs_devices *fs_devices) +{ + int ret; + struct btrfs_device *device; + struct btrfs_fs_devices *seed; + + list_for_each_entry(device, &fs_devices->devices, dev_list) { + ret = btrfs_sysfs_add_device(device); + if (ret) + goto fail; + } + + list_for_each_entry(seed, &fs_devices->seed_list, seed_list) { + list_for_each_entry(device, &seed->devices, dev_list) { + ret = btrfs_sysfs_add_device(device); + if (ret) + goto fail; + } + } + + return 0; + +fail: + btrfs_sysfs_remove_fs_devices(fs_devices); + return ret; +} + +void btrfs_kobject_uevent(struct block_device *bdev, enum kobject_action action) +{ + int ret; + + ret = kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, action); + if (ret) + btrfs_warn(NULL, "sending event %d to kobject: '%s' (%p): failed", + action, kobject_name(&disk_to_dev(bdev->bd_disk)->kobj), + &disk_to_dev(bdev->bd_disk)->kobj); +} + +void btrfs_sysfs_update_sprout_fsid(struct btrfs_fs_devices *fs_devices) + +{ + char fsid_buf[BTRFS_UUID_UNPARSED_SIZE]; + + /* + * Sprouting changes fsid of the mounted filesystem, rename the fsid + * directory + */ + snprintf(fsid_buf, BTRFS_UUID_UNPARSED_SIZE, "%pU", fs_devices->fsid); + if (kobject_rename(&fs_devices->fsid_kobj, fsid_buf)) + btrfs_warn(fs_devices->fs_info, + "sysfs: failed to create fsid for sprout"); +} + +void btrfs_sysfs_update_devid(struct btrfs_device *device) +{ + char tmp[24]; + + snprintf(tmp, sizeof(tmp), "%llu", device->devid); + + if (kobject_rename(&device->devid_kobj, tmp)) + btrfs_warn(device->fs_devices->fs_info, + "sysfs: failed to update devid for %llu", + device->devid); +} + +/* /sys/fs/btrfs/ entry */ +static struct kset *btrfs_kset; + +/* + * Creates: + * /sys/fs/btrfs/UUID + * + * Can be called by the device discovery thread. + */ +int btrfs_sysfs_add_fsid(struct btrfs_fs_devices *fs_devs) +{ + int ret; + + init_completion(&fs_devs->kobj_unregister); + fs_devs->fsid_kobj.kset = btrfs_kset; + ret = kobject_init_and_add(&fs_devs->fsid_kobj, &btrfs_ktype, NULL, + "%pU", fs_devs->fsid); + if (ret) { + kobject_put(&fs_devs->fsid_kobj); + return ret; + } + + fs_devs->devices_kobj = kobject_create_and_add("devices", + &fs_devs->fsid_kobj); + if (!fs_devs->devices_kobj) { + btrfs_err(fs_devs->fs_info, + "failed to init sysfs device interface"); + btrfs_sysfs_remove_fsid(fs_devs); + return -ENOMEM; + } + + fs_devs->devinfo_kobj = kobject_create_and_add("devinfo", + &fs_devs->fsid_kobj); + if (!fs_devs->devinfo_kobj) { + btrfs_err(fs_devs->fs_info, + "failed to init sysfs devinfo kobject"); + btrfs_sysfs_remove_fsid(fs_devs); + return -ENOMEM; + } + + return 0; +} + +int btrfs_sysfs_add_mounted(struct btrfs_fs_info *fs_info) +{ + int ret; + struct btrfs_fs_devices *fs_devs = fs_info->fs_devices; + struct kobject *fsid_kobj = &fs_devs->fsid_kobj; + + ret = btrfs_sysfs_add_fs_devices(fs_devs); + if (ret) + return ret; + + ret = sysfs_create_files(fsid_kobj, btrfs_attrs); + if (ret) { + btrfs_sysfs_remove_fs_devices(fs_devs); + return ret; + } + + ret = sysfs_create_group(fsid_kobj, &btrfs_feature_attr_group); + if (ret) + goto failure; + +#ifdef CONFIG_BTRFS_DEBUG + fs_info->debug_kobj = kobject_create_and_add("debug", fsid_kobj); + if (!fs_info->debug_kobj) { + ret = -ENOMEM; + goto failure; + } + + ret = sysfs_create_files(fs_info->debug_kobj, btrfs_debug_mount_attrs); + if (ret) + goto failure; +#endif + + /* Discard directory */ + fs_info->discard_kobj = kobject_create_and_add("discard", fsid_kobj); + if (!fs_info->discard_kobj) { + ret = -ENOMEM; + goto failure; + } + + ret = sysfs_create_files(fs_info->discard_kobj, discard_attrs); + if (ret) + goto failure; + + ret = addrm_unknown_feature_attrs(fs_info, true); + if (ret) + goto failure; + + ret = sysfs_create_link(fsid_kobj, &fs_info->sb->s_bdi->dev->kobj, "bdi"); + if (ret) + goto failure; + + fs_info->space_info_kobj = kobject_create_and_add("allocation", + fsid_kobj); + if (!fs_info->space_info_kobj) { + ret = -ENOMEM; + goto failure; + } + + ret = sysfs_create_files(fs_info->space_info_kobj, allocation_attrs); + if (ret) + goto failure; + + return 0; +failure: + btrfs_sysfs_remove_mounted(fs_info); + return ret; +} + +static ssize_t qgroup_enabled_show(struct kobject *qgroups_kobj, + struct kobj_attribute *a, + char *buf) +{ + struct btrfs_fs_info *fs_info = to_fs_info(qgroups_kobj->parent); + bool enabled; + + spin_lock(&fs_info->qgroup_lock); + enabled = fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_ON; + spin_unlock(&fs_info->qgroup_lock); + + return sysfs_emit(buf, "%d\n", enabled); +} +BTRFS_ATTR(qgroups, enabled, qgroup_enabled_show); + +static ssize_t qgroup_mode_show(struct kobject *qgroups_kobj, + struct kobj_attribute *a, + char *buf) +{ + struct btrfs_fs_info *fs_info = to_fs_info(qgroups_kobj->parent); + ssize_t ret = 0; + + spin_lock(&fs_info->qgroup_lock); + ASSERT(btrfs_qgroup_enabled(fs_info)); + switch (btrfs_qgroup_mode(fs_info)) { + case BTRFS_QGROUP_MODE_FULL: + ret = sysfs_emit(buf, "qgroup\n"); + break; + case BTRFS_QGROUP_MODE_SIMPLE: + ret = sysfs_emit(buf, "squota\n"); + break; + default: + btrfs_warn(fs_info, "unexpected qgroup mode %d\n", + btrfs_qgroup_mode(fs_info)); + break; + } + spin_unlock(&fs_info->qgroup_lock); + + return ret; +} +BTRFS_ATTR(qgroups, mode, qgroup_mode_show); + +static ssize_t qgroup_inconsistent_show(struct kobject *qgroups_kobj, + struct kobj_attribute *a, + char *buf) +{ + struct btrfs_fs_info *fs_info = to_fs_info(qgroups_kobj->parent); + bool inconsistent; + + spin_lock(&fs_info->qgroup_lock); + inconsistent = (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT); + spin_unlock(&fs_info->qgroup_lock); + + return sysfs_emit(buf, "%d\n", inconsistent); +} +BTRFS_ATTR(qgroups, inconsistent, qgroup_inconsistent_show); + +static ssize_t qgroup_drop_subtree_thres_show(struct kobject *qgroups_kobj, + struct kobj_attribute *a, + char *buf) +{ + struct btrfs_fs_info *fs_info = to_fs_info(qgroups_kobj->parent); + u8 result; + + spin_lock(&fs_info->qgroup_lock); + result = fs_info->qgroup_drop_subtree_thres; + spin_unlock(&fs_info->qgroup_lock); + + return sysfs_emit(buf, "%d\n", result); +} + +static ssize_t qgroup_drop_subtree_thres_store(struct kobject *qgroups_kobj, + struct kobj_attribute *a, + const char *buf, size_t len) +{ + struct btrfs_fs_info *fs_info = to_fs_info(qgroups_kobj->parent); + u8 new_thres; + int ret; + + ret = kstrtou8(buf, 10, &new_thres); + if (ret) + return -EINVAL; + + if (new_thres > BTRFS_MAX_LEVEL) + return -EINVAL; + + spin_lock(&fs_info->qgroup_lock); + fs_info->qgroup_drop_subtree_thres = new_thres; + spin_unlock(&fs_info->qgroup_lock); + + return len; +} +BTRFS_ATTR_RW(qgroups, drop_subtree_threshold, qgroup_drop_subtree_thres_show, + qgroup_drop_subtree_thres_store); + +/* + * Qgroups global info + * + * Path: /sys/fs/btrfs//qgroups/ + */ +static struct attribute *qgroups_attrs[] = { + BTRFS_ATTR_PTR(qgroups, enabled), + BTRFS_ATTR_PTR(qgroups, inconsistent), + BTRFS_ATTR_PTR(qgroups, drop_subtree_threshold), + BTRFS_ATTR_PTR(qgroups, mode), + NULL +}; +ATTRIBUTE_GROUPS(qgroups); + +static void qgroups_release(struct kobject *kobj) +{ + kfree(kobj); +} + +static const struct kobj_type qgroups_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .default_groups = qgroups_groups, + .release = qgroups_release, +}; + +static inline struct btrfs_fs_info *qgroup_kobj_to_fs_info(struct kobject *kobj) +{ + return to_fs_info(kobj->parent->parent); +} + +#define QGROUP_ATTR(_member, _show_name) \ +static ssize_t btrfs_qgroup_show_##_member(struct kobject *qgroup_kobj, \ + struct kobj_attribute *a, \ + char *buf) \ +{ \ + struct btrfs_fs_info *fs_info = qgroup_kobj_to_fs_info(qgroup_kobj); \ + struct btrfs_qgroup *qgroup = container_of(qgroup_kobj, \ + struct btrfs_qgroup, kobj); \ + return btrfs_show_u64(&qgroup->_member, &fs_info->qgroup_lock, buf); \ +} \ +BTRFS_ATTR(qgroup, _show_name, btrfs_qgroup_show_##_member) + +#define QGROUP_RSV_ATTR(_name, _type) \ +static ssize_t btrfs_qgroup_rsv_show_##_name(struct kobject *qgroup_kobj, \ + struct kobj_attribute *a, \ + char *buf) \ +{ \ + struct btrfs_fs_info *fs_info = qgroup_kobj_to_fs_info(qgroup_kobj); \ + struct btrfs_qgroup *qgroup = container_of(qgroup_kobj, \ + struct btrfs_qgroup, kobj); \ + return btrfs_show_u64(&qgroup->rsv.values[_type], \ + &fs_info->qgroup_lock, buf); \ +} \ +BTRFS_ATTR(qgroup, rsv_##_name, btrfs_qgroup_rsv_show_##_name) + +QGROUP_ATTR(rfer, referenced); +QGROUP_ATTR(excl, exclusive); +QGROUP_ATTR(max_rfer, max_referenced); +QGROUP_ATTR(max_excl, max_exclusive); +QGROUP_ATTR(lim_flags, limit_flags); +QGROUP_RSV_ATTR(data, BTRFS_QGROUP_RSV_DATA); +QGROUP_RSV_ATTR(meta_pertrans, BTRFS_QGROUP_RSV_META_PERTRANS); +QGROUP_RSV_ATTR(meta_prealloc, BTRFS_QGROUP_RSV_META_PREALLOC); + +/* + * Qgroup information. + * + * Path: /sys/fs/btrfs//qgroups/_/ + */ +static struct attribute *qgroup_attrs[] = { + BTRFS_ATTR_PTR(qgroup, referenced), + BTRFS_ATTR_PTR(qgroup, exclusive), + BTRFS_ATTR_PTR(qgroup, max_referenced), + BTRFS_ATTR_PTR(qgroup, max_exclusive), + BTRFS_ATTR_PTR(qgroup, limit_flags), + BTRFS_ATTR_PTR(qgroup, rsv_data), + BTRFS_ATTR_PTR(qgroup, rsv_meta_pertrans), + BTRFS_ATTR_PTR(qgroup, rsv_meta_prealloc), + NULL +}; +ATTRIBUTE_GROUPS(qgroup); + +static void qgroup_release(struct kobject *kobj) +{ + struct btrfs_qgroup *qgroup = container_of(kobj, struct btrfs_qgroup, kobj); + + memset(&qgroup->kobj, 0, sizeof(*kobj)); +} + +static const struct kobj_type qgroup_ktype = { + .sysfs_ops = &kobj_sysfs_ops, + .release = qgroup_release, + .default_groups = qgroup_groups, +}; + +int btrfs_sysfs_add_one_qgroup(struct btrfs_fs_info *fs_info, + struct btrfs_qgroup *qgroup) +{ + struct kobject *qgroups_kobj = fs_info->qgroups_kobj; + int ret; + + if (btrfs_is_testing(fs_info)) + return 0; + if (qgroup->kobj.state_initialized) + return 0; + if (!qgroups_kobj) + return -EINVAL; + + ret = kobject_init_and_add(&qgroup->kobj, &qgroup_ktype, qgroups_kobj, + "%hu_%llu", btrfs_qgroup_level(qgroup->qgroupid), + btrfs_qgroup_subvolid(qgroup->qgroupid)); + if (ret < 0) + kobject_put(&qgroup->kobj); + + return ret; +} + +void btrfs_sysfs_del_qgroups(struct btrfs_fs_info *fs_info) +{ + struct btrfs_qgroup *qgroup; + struct btrfs_qgroup *next; + + if (btrfs_is_testing(fs_info)) + return; + + rbtree_postorder_for_each_entry_safe(qgroup, next, + &fs_info->qgroup_tree, node) + btrfs_sysfs_del_one_qgroup(fs_info, qgroup); + if (fs_info->qgroups_kobj) { + kobject_del(fs_info->qgroups_kobj); + kobject_put(fs_info->qgroups_kobj); + fs_info->qgroups_kobj = NULL; + } +} + +/* Called when qgroups get initialized, thus there is no need for locking */ +int btrfs_sysfs_add_qgroups(struct btrfs_fs_info *fs_info) +{ + struct kobject *fsid_kobj = &fs_info->fs_devices->fsid_kobj; + struct btrfs_qgroup *qgroup; + struct btrfs_qgroup *next; + int ret = 0; + + if (btrfs_is_testing(fs_info)) + return 0; + + ASSERT(fsid_kobj); + if (fs_info->qgroups_kobj) + return 0; + + fs_info->qgroups_kobj = kzalloc_obj(struct kobject); + if (!fs_info->qgroups_kobj) + return -ENOMEM; + + ret = kobject_init_and_add(fs_info->qgroups_kobj, &qgroups_ktype, + fsid_kobj, "qgroups"); + if (ret < 0) + goto out; + + rbtree_postorder_for_each_entry_safe(qgroup, next, + &fs_info->qgroup_tree, node) { + ret = btrfs_sysfs_add_one_qgroup(fs_info, qgroup); + if (ret < 0) + goto out; + } + +out: + if (ret < 0) + btrfs_sysfs_del_qgroups(fs_info); + return ret; +} + +void btrfs_sysfs_del_one_qgroup(struct btrfs_fs_info *fs_info, + struct btrfs_qgroup *qgroup) +{ + if (btrfs_is_testing(fs_info)) + return; + + if (qgroup->kobj.state_initialized) { + kobject_del(&qgroup->kobj); + kobject_put(&qgroup->kobj); + } +} + +/* + * Change per-fs features in /sys/fs/btrfs/UUID/features to match current + * values in superblock. Call after any changes to incompat/compat_ro flags + */ +void btrfs_sysfs_feature_update(struct btrfs_fs_info *fs_info) +{ + struct kobject *fsid_kobj; + int ret; + + if (!fs_info) + return; + + fsid_kobj = &fs_info->fs_devices->fsid_kobj; + if (!fsid_kobj->state_initialized) + return; + + ret = sysfs_update_group(fsid_kobj, &btrfs_feature_attr_group); + if (ret < 0) + btrfs_warn(fs_info, + "failed to update /sys/fs/btrfs/%pU/features: %d", + fs_info->fs_devices->fsid, ret); +} + +int __init btrfs_init_sysfs(void) +{ + int ret; + + btrfs_kset = kset_create_and_add("btrfs", NULL, fs_kobj); + if (!btrfs_kset) + return -ENOMEM; + + init_feature_attrs(); + ret = sysfs_create_group(&btrfs_kset->kobj, &btrfs_feature_attr_group); + if (ret) + goto out2; + ret = sysfs_merge_group(&btrfs_kset->kobj, + &btrfs_static_feature_attr_group); + if (ret) + goto out_remove_group; + +#ifdef CONFIG_BTRFS_DEBUG + ret = sysfs_create_group(&btrfs_kset->kobj, &btrfs_debug_feature_attr_group); + if (ret) { + sysfs_unmerge_group(&btrfs_kset->kobj, + &btrfs_static_feature_attr_group); + goto out_remove_group; + } +#endif + + return 0; + +out_remove_group: + sysfs_remove_group(&btrfs_kset->kobj, &btrfs_feature_attr_group); +out2: + kset_unregister(btrfs_kset); + + return ret; +} + +void __cold btrfs_exit_sysfs(void) +{ + sysfs_unmerge_group(&btrfs_kset->kobj, + &btrfs_static_feature_attr_group); + sysfs_remove_group(&btrfs_kset->kobj, &btrfs_feature_attr_group); +#ifdef CONFIG_BTRFS_DEBUG + sysfs_remove_group(&btrfs_kset->kobj, &btrfs_debug_feature_attr_group); +#endif + kset_unregister(btrfs_kset); +} diff --git a/fs/btrfs/sysfs.h b/fs/btrfs/sysfs.h new file mode 100644 index 000000000..05498e534 --- /dev/null +++ b/fs/btrfs/sysfs.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_SYSFS_H +#define BTRFS_SYSFS_H + +#include +#include +#include + +struct block_device; +struct btrfs_fs_info; +struct btrfs_device; +struct btrfs_fs_devices; +struct btrfs_block_group; +struct btrfs_space_info; +struct btrfs_qgroup; + +enum btrfs_feature_set { + FEAT_COMPAT, + FEAT_COMPAT_RO, + FEAT_INCOMPAT, + FEAT_MAX +}; + +char *btrfs_printable_features(enum btrfs_feature_set set, u64 flags); +const char *btrfs_feature_set_name(enum btrfs_feature_set set); +int btrfs_sysfs_add_device(struct btrfs_device *device); +void btrfs_sysfs_remove_device(struct btrfs_device *device); +int btrfs_sysfs_add_fsid(struct btrfs_fs_devices *fs_devs); +void btrfs_sysfs_remove_fsid(struct btrfs_fs_devices *fs_devs); +void btrfs_sysfs_update_sprout_fsid(struct btrfs_fs_devices *fs_devices); +void btrfs_sysfs_feature_update(struct btrfs_fs_info *fs_info); +void btrfs_kobject_uevent(struct block_device *bdev, enum kobject_action action); + +int __init btrfs_init_sysfs(void); +void __cold btrfs_exit_sysfs(void); +int btrfs_sysfs_add_mounted(struct btrfs_fs_info *fs_info); +void btrfs_sysfs_remove_mounted(struct btrfs_fs_info *fs_info); +void btrfs_sysfs_add_block_group_type(struct btrfs_block_group *cache); +int btrfs_sysfs_add_space_info_type(struct btrfs_space_info *space_info); +void btrfs_sysfs_remove_space_info(struct btrfs_space_info *space_info); +void btrfs_sysfs_update_devid(struct btrfs_device *device); + +int btrfs_sysfs_add_one_qgroup(struct btrfs_fs_info *fs_info, + struct btrfs_qgroup *qgroup); +void btrfs_sysfs_del_qgroups(struct btrfs_fs_info *fs_info); +int btrfs_sysfs_add_qgroups(struct btrfs_fs_info *fs_info); +void btrfs_sysfs_del_one_qgroup(struct btrfs_fs_info *fs_info, + struct btrfs_qgroup *qgroup); +int btrfs_read_policy_to_enum(const char *str, s64 *value); + +#ifdef CONFIG_BTRFS_EXPERIMENTAL +int __init btrfs_read_policy_init(void); +char *btrfs_get_mod_read_policy(void); +#endif + +#endif diff --git a/fs/btrfs/tests/btrfs-tests.c b/fs/btrfs/tests/btrfs-tests.c new file mode 100644 index 000000000..6287d9403 --- /dev/null +++ b/fs/btrfs/tests/btrfs-tests.c @@ -0,0 +1,314 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2013 Fusion IO. All rights reserved. + */ + +#include +#include +#include +#include +#include "btrfs-tests.h" +#include "../ctree.h" +#include "../free-space-cache.h" +#include "../free-space-tree.h" +#include "../transaction.h" +#include "../volumes.h" +#include "../disk-io.h" +#include "../qgroup.h" +#include "../block-group.h" +#include "../fs.h" + +static struct vfsmount *test_mnt = NULL; + +const char *test_error[] = { + [TEST_ALLOC_FS_INFO] = "cannot allocate fs_info", + [TEST_ALLOC_ROOT] = "cannot allocate root", + [TEST_ALLOC_EXTENT_BUFFER] = "cannot extent buffer", + [TEST_ALLOC_PATH] = "cannot allocate path", + [TEST_ALLOC_INODE] = "cannot allocate inode", + [TEST_ALLOC_BLOCK_GROUP] = "cannot allocate block group", + [TEST_ALLOC_EXTENT_MAP] = "cannot allocate extent map", + [TEST_ALLOC_CHUNK_MAP] = "cannot allocate chunk map", + [TEST_ALLOC_IO_CONTEXT] = "cannot allocate io context", + [TEST_ALLOC_TRANSACTION] = "cannot allocate transaction", +}; + +static const struct super_operations btrfs_test_super_ops = { + .alloc_inode = btrfs_alloc_inode, + .destroy_inode = btrfs_test_destroy_inode, +}; + + +static int btrfs_test_init_fs_context(struct fs_context *fc) +{ + struct pseudo_fs_context *ctx = init_pseudo(fc, BTRFS_TEST_MAGIC); + if (!ctx) + return -ENOMEM; + ctx->ops = &btrfs_test_super_ops; + return 0; +} + +static struct file_system_type test_type = { + .name = "btrfs_test_fs", + .init_fs_context = btrfs_test_init_fs_context, + .kill_sb = kill_anon_super, +}; + +struct inode *btrfs_new_test_inode(void) +{ + struct inode *inode; + + inode = new_inode(test_mnt->mnt_sb); + if (!inode) + return NULL; + + inode->i_mode = S_IFREG; + btrfs_set_inode_number(BTRFS_I(inode), BTRFS_FIRST_FREE_OBJECTID); + inode_init_owner(&nop_mnt_idmap, inode, NULL, S_IFREG); + + return inode; +} + +static int btrfs_init_test_fs(void) +{ + int ret; + + ret = register_filesystem(&test_type); + if (ret) { + printk(KERN_ERR "btrfs: cannot register test file system\n"); + return ret; + } + + test_mnt = kern_mount(&test_type); + if (IS_ERR(test_mnt)) { + printk(KERN_ERR "btrfs: cannot mount test file system\n"); + unregister_filesystem(&test_type); + return PTR_ERR(test_mnt); + } + return 0; +} + +static void btrfs_destroy_test_fs(void) +{ + kern_unmount(test_mnt); + unregister_filesystem(&test_type); +} + +struct btrfs_device *btrfs_alloc_dummy_device(struct btrfs_fs_info *fs_info) +{ + struct btrfs_device *dev; + + dev = kzalloc_obj(*dev); + if (!dev) + return ERR_PTR(-ENOMEM); + + btrfs_extent_io_tree_init(fs_info, &dev->alloc_state, 0); + INIT_LIST_HEAD(&dev->dev_list); + list_add(&dev->dev_list, &fs_info->fs_devices->devices); + + return dev; +} + +static void btrfs_free_dummy_device(struct btrfs_device *dev) +{ + btrfs_extent_io_tree_release(&dev->alloc_state); + kfree(dev); +} + +struct btrfs_fs_info *btrfs_alloc_dummy_fs_info(u32 nodesize, u32 sectorsize) +{ + struct btrfs_fs_info *fs_info = kzalloc_obj(struct btrfs_fs_info); + + if (!fs_info) + return fs_info; + fs_info->fs_devices = kzalloc_obj(struct btrfs_fs_devices); + if (!fs_info->fs_devices) { + kfree(fs_info); + return NULL; + } + INIT_LIST_HEAD(&fs_info->fs_devices->devices); + + fs_info->super_copy = kzalloc_obj(struct btrfs_super_block); + if (!fs_info->super_copy) { + kfree(fs_info->fs_devices); + kfree(fs_info); + return NULL; + } + + btrfs_init_fs_info(fs_info); + + fs_info->nodesize = nodesize; + fs_info->sectorsize = sectorsize; + fs_info->sectorsize_bits = ilog2(sectorsize); + + /* CRC32C csum size. */ + fs_info->csum_size = 4; + fs_info->csums_per_leaf = BTRFS_MAX_ITEM_SIZE(fs_info) / + fs_info->csum_size; + fs_info->use_bitmap = btrfs_use_bitmap; + set_bit(BTRFS_FS_STATE_DUMMY_FS_INFO, &fs_info->fs_state); + + test_mnt->mnt_sb->s_fs_info = fs_info; + + return fs_info; +} + +void btrfs_free_dummy_fs_info(struct btrfs_fs_info *fs_info) +{ + struct btrfs_device *dev, *tmp; + struct extent_buffer *eb; + unsigned long index; + + if (!fs_info) + return; + + if (WARN_ON(!btrfs_is_testing(fs_info))) + return; + + test_mnt->mnt_sb->s_fs_info = NULL; + + xa_lock_irq(&fs_info->buffer_tree); + xa_for_each(&fs_info->buffer_tree, index, eb) { + xa_unlock_irq(&fs_info->buffer_tree); + free_extent_buffer(eb); + xa_lock_irq(&fs_info->buffer_tree); + } + xa_unlock_irq(&fs_info->buffer_tree); + + btrfs_mapping_tree_free(fs_info); + list_for_each_entry_safe(dev, tmp, &fs_info->fs_devices->devices, + dev_list) { + btrfs_free_dummy_device(dev); + } + btrfs_free_qgroup_config(fs_info); + btrfs_free_fs_roots(fs_info); + kfree(fs_info->super_copy); + btrfs_check_leaked_roots(fs_info); + btrfs_extent_buffer_leak_debug_check(fs_info); + kfree(fs_info->fs_devices); + kfree(fs_info); +} + +void btrfs_free_dummy_root(struct btrfs_root *root) +{ + if (IS_ERR_OR_NULL(root)) + return; + /* Will be freed by btrfs_free_fs_roots */ + if (WARN_ON(test_bit(BTRFS_ROOT_IN_RADIX, &root->state))) + return; + btrfs_global_root_delete(root); + btrfs_put_root(root); +} + +struct btrfs_block_group * +btrfs_alloc_dummy_block_group(struct btrfs_fs_info *fs_info, + unsigned long length) +{ + struct btrfs_block_group *cache; + + cache = kzalloc_obj(*cache); + if (!cache) + return NULL; + cache->free_space_ctl = kzalloc_obj(*cache->free_space_ctl); + if (!cache->free_space_ctl) { + kfree(cache); + return NULL; + } + + cache->start = 0; + cache->length = length; + cache->full_stripe_len = fs_info->sectorsize; + cache->fs_info = fs_info; + + INIT_LIST_HEAD(&cache->list); + INIT_LIST_HEAD(&cache->cluster_list); + INIT_LIST_HEAD(&cache->bg_list); + btrfs_init_free_space_ctl(cache, cache->free_space_ctl); + mutex_init(&cache->free_space_lock); + + return cache; +} + +void btrfs_free_dummy_block_group(struct btrfs_block_group *cache) +{ + if (!cache) + return; + btrfs_remove_free_space_cache(cache); + kfree(cache->free_space_ctl); + kfree(cache); +} + +void btrfs_init_dummy_transaction(struct btrfs_transaction *trans, struct btrfs_fs_info *fs_info) +{ + memset(trans, 0, sizeof(*trans)); + trans->fs_info = fs_info; + xa_init(&trans->delayed_refs.head_refs); + xa_init(&trans->delayed_refs.dirty_extents); + spin_lock_init(&trans->delayed_refs.lock); +} + +void btrfs_init_dummy_trans(struct btrfs_trans_handle *trans, + struct btrfs_fs_info *fs_info) +{ + memset(trans, 0, sizeof(*trans)); + trans->transid = 1; + trans->type = __TRANS_DUMMY; + trans->fs_info = fs_info; +} + +int btrfs_run_sanity_tests(void) +{ + int ret, i; + u32 sectorsize, nodesize; + u32 test_sectorsize[] = { + PAGE_SIZE, + }; + ret = btrfs_init_test_fs(); + if (ret) + return ret; + for (i = 0; i < ARRAY_SIZE(test_sectorsize); i++) { + sectorsize = test_sectorsize[i]; + for (nodesize = sectorsize; + nodesize <= BTRFS_MAX_METADATA_BLOCKSIZE; + nodesize <<= 1) { + pr_info("BTRFS: selftest: sectorsize: %u nodesize: %u\n", + sectorsize, nodesize); + ret = btrfs_test_free_space_cache(sectorsize, nodesize); + if (ret) + goto out; + ret = btrfs_test_extent_buffer_operations(sectorsize, + nodesize); + if (ret) + goto out; + ret = btrfs_test_extent_io(sectorsize, nodesize); + if (ret) + goto out; + ret = btrfs_test_inodes(sectorsize, nodesize); + if (ret) + goto out; + ret = btrfs_test_qgroups(sectorsize, nodesize); + if (ret) + goto out; + ret = btrfs_test_free_space_tree(sectorsize, nodesize); + if (ret) + goto out; + ret = btrfs_test_raid_stripe_tree(sectorsize, nodesize); + if (ret) + goto out; + ret = btrfs_test_delayed_refs(sectorsize, nodesize); + if (ret) + goto out; + ret = btrfs_test_chunk_allocation(sectorsize, nodesize); + if (ret) + goto out; + } + } + ret = btrfs_test_extent_map(); + if (ret) + goto out; + ret = btrfs_test_zoned(); + +out: + btrfs_destroy_test_fs(); + return ret; +} diff --git a/fs/btrfs/tests/btrfs-tests.h b/fs/btrfs/tests/btrfs-tests.h new file mode 100644 index 000000000..cea58fe84 --- /dev/null +++ b/fs/btrfs/tests/btrfs-tests.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2013 Fusion IO. All rights reserved. + */ + +#ifndef BTRFS_TESTS_H +#define BTRFS_TESTS_H + +#include +#include + +#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS + +int btrfs_run_sanity_tests(void); + +#define test_msg(fmt, ...) pr_info("BTRFS: selftest: " fmt "\n", ##__VA_ARGS__) +#define test_err(fmt, ...) pr_err("BTRFS: selftest: %s:%d " fmt "\n", \ + __FILE__, __LINE__, ##__VA_ARGS__) + +#define test_std_err(index) test_err("%s", test_error[index]) + +enum { + TEST_ALLOC_FS_INFO, + TEST_ALLOC_ROOT, + TEST_ALLOC_EXTENT_BUFFER, + TEST_ALLOC_PATH, + TEST_ALLOC_INODE, + TEST_ALLOC_BLOCK_GROUP, + TEST_ALLOC_EXTENT_MAP, + TEST_ALLOC_CHUNK_MAP, + TEST_ALLOC_IO_CONTEXT, + TEST_ALLOC_TRANSACTION, +}; + +extern const char *test_error[]; + +struct btrfs_root; +struct btrfs_trans_handle; +struct btrfs_transaction; + +int btrfs_test_extent_buffer_operations(u32 sectorsize, u32 nodesize); +int btrfs_test_free_space_cache(u32 sectorsize, u32 nodesize); +int btrfs_test_extent_io(u32 sectorsize, u32 nodesize); +int btrfs_test_inodes(u32 sectorsize, u32 nodesize); +int btrfs_test_qgroups(u32 sectorsize, u32 nodesize); +int btrfs_test_free_space_tree(u32 sectorsize, u32 nodesize); +int btrfs_test_raid_stripe_tree(u32 sectorsize, u32 nodesize); +int btrfs_test_extent_map(void); +int btrfs_test_delayed_refs(u32 sectorsize, u32 nodesize); +int btrfs_test_chunk_allocation(u32 sectorsize, u32 nodesize); +struct inode *btrfs_new_test_inode(void); +struct btrfs_fs_info *btrfs_alloc_dummy_fs_info(u32 nodesize, u32 sectorsize); +void btrfs_free_dummy_fs_info(struct btrfs_fs_info *fs_info); +DEFINE_FREE(btrfs_free_dummy_fs_info, struct btrfs_fs_info *, + btrfs_free_dummy_fs_info(_T)) +void btrfs_free_dummy_root(struct btrfs_root *root); +struct btrfs_block_group * +btrfs_alloc_dummy_block_group(struct btrfs_fs_info *fs_info, unsigned long length); +void btrfs_free_dummy_block_group(struct btrfs_block_group *cache); +DEFINE_FREE(btrfs_free_dummy_block_group, struct btrfs_block_group *, + btrfs_free_dummy_block_group(_T)); +void btrfs_init_dummy_trans(struct btrfs_trans_handle *trans, + struct btrfs_fs_info *fs_info); +void btrfs_init_dummy_transaction(struct btrfs_transaction *trans, struct btrfs_fs_info *fs_info); +struct btrfs_device *btrfs_alloc_dummy_device(struct btrfs_fs_info *fs_info); + +#ifdef CONFIG_BLK_DEV_ZONED +int btrfs_test_zoned(void); +#else +static inline int btrfs_test_zoned(void) +{ + return 0; +} +#endif + +#else +static inline int btrfs_run_sanity_tests(void) +{ + return 0; +} +#endif + +#endif diff --git a/fs/btrfs/tests/chunk-allocation-tests.c b/fs/btrfs/tests/chunk-allocation-tests.c new file mode 100644 index 000000000..9beb0602f --- /dev/null +++ b/fs/btrfs/tests/chunk-allocation-tests.c @@ -0,0 +1,476 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2026 Meta. All rights reserved. + */ + +#include +#include "btrfs-tests.h" +#include "../volumes.h" +#include "../disk-io.h" +#include "../extent-io-tree.h" + +/* + * Tests for chunk allocator pending extent internals. + * These two functions form the core of searching the chunk allocation pending + * extent bitmap and have relatively easily definable semantics, so unit + * testing them can help ensure the correctness of chunk allocation. + */ + +/* + * Describes the inputs to the system and expected results + * when testing btrfs_find_hole_in_pending_extents(). + */ +struct pending_extent_test_case { + const char *name; + /* Input range to search. */ + u64 hole_start; + u64 hole_len; + /* The size of hole we are searching for. */ + u64 min_hole_size; + /* + * Pending extents to set up (up to 2 for up to 3 holes) + * If len == 0, then it is skipped. + */ + struct { + u64 start; + u64 len; + } pending_extents[2]; + /* Expected outputs. */ + bool expected_found; + u64 expected_start; + u64 expected_len; +}; + +static const struct pending_extent_test_case find_hole_tests[] = { + { + .name = "no pending extents", + .hole_start = 0, + .hole_len = 10ULL * SZ_1G, + .min_hole_size = SZ_1G, + .pending_extents = { }, + .expected_found = true, + .expected_start = 0, + .expected_len = 10ULL * SZ_1G, + }, + { + .name = "pending extent at start of range", + .hole_start = 0, + .hole_len = 10ULL * SZ_1G, + .min_hole_size = SZ_1G, + .pending_extents = { + { .start = 0, .len = SZ_1G }, + }, + .expected_found = true, + .expected_start = SZ_1G, + .expected_len = 9ULL * SZ_1G, + }, + { + .name = "pending extent overlapping start of range", + .hole_start = SZ_1G, + .hole_len = 9ULL * SZ_1G, + .min_hole_size = SZ_1G, + .pending_extents = { + { .start = 0, .len = SZ_2G }, + }, + .expected_found = true, + .expected_start = SZ_2G, + .expected_len = 8ULL * SZ_1G, + }, + { + .name = "two holes; first hole is exactly big enough", + .hole_start = 0, + .hole_len = 10ULL * SZ_1G, + .min_hole_size = SZ_1G, + .pending_extents = { + { .start = SZ_1G, .len = SZ_1G }, + }, + .expected_found = true, + .expected_start = 0, + .expected_len = SZ_1G, + }, + { + .name = "two holes; first hole is big enough", + .hole_start = 0, + .hole_len = 10ULL * SZ_1G, + .min_hole_size = SZ_1G, + .pending_extents = { + { .start = SZ_2G, .len = SZ_1G }, + }, + .expected_found = true, + .expected_start = 0, + .expected_len = SZ_2G, + }, + { + .name = "two holes; second hole is big enough", + .hole_start = 0, + .hole_len = 10ULL * SZ_1G, + .min_hole_size = SZ_2G, + .pending_extents = { + { .start = SZ_1G, .len = SZ_1G }, + }, + .expected_found = true, + .expected_start = SZ_2G, + .expected_len = 8ULL * SZ_1G, + }, + { + .name = "three holes; first hole big enough", + .hole_start = 0, + .hole_len = 10ULL * SZ_1G, + .min_hole_size = SZ_2G, + .pending_extents = { + { .start = SZ_2G, .len = SZ_1G }, + { .start = 4ULL * SZ_1G, .len = SZ_1G }, + }, + .expected_found = true, + .expected_start = 0, + .expected_len = SZ_2G, + }, + { + .name = "three holes; second hole big enough", + .hole_start = 0, + .hole_len = 10ULL * SZ_1G, + .min_hole_size = SZ_2G, + .pending_extents = { + { .start = SZ_1G, .len = SZ_1G }, + { .start = 5ULL * SZ_1G, .len = SZ_1G }, + }, + .expected_found = true, + .expected_start = SZ_2G, + .expected_len = 3ULL * SZ_1G, + }, + { + .name = "three holes; third hole big enough", + .hole_start = 0, + .hole_len = 10ULL * SZ_1G, + .min_hole_size = SZ_2G, + .pending_extents = { + { .start = SZ_1G, .len = SZ_1G }, + { .start = 3ULL * SZ_1G, .len = 5ULL * SZ_1G }, + }, + .expected_found = true, + .expected_start = 8ULL * SZ_1G, + .expected_len = SZ_2G, + }, + { + .name = "three holes; all holes too small", + .hole_start = 0, + .hole_len = 10ULL * SZ_1G, + .min_hole_size = SZ_2G, + .pending_extents = { + { .start = SZ_1G, .len = SZ_1G }, + { .start = 3ULL * SZ_1G, .len = 6ULL * SZ_1G }, + }, + .expected_found = false, + .expected_start = 0, + .expected_len = SZ_1G, + }, + { + .name = "three holes; all holes too small; first biggest", + .hole_start = 0, + .hole_len = 10ULL * SZ_1G, + .min_hole_size = 3ULL * SZ_1G, + .pending_extents = { + { .start = SZ_2G, .len = SZ_1G }, + { .start = 4ULL * SZ_1G, .len = 5ULL * SZ_1G }, + }, + .expected_found = false, + .expected_start = 0, + .expected_len = SZ_2G, + }, + { + .name = "three holes; all holes too small; second biggest", + .hole_start = 0, + .hole_len = 10ULL * SZ_1G, + .min_hole_size = 3ULL * SZ_1G, + .pending_extents = { + { .start = SZ_1G, .len = SZ_1G }, + { .start = 4ULL * SZ_1G, .len = 5ULL * SZ_1G }, + }, + .expected_found = false, + .expected_start = SZ_2G, + .expected_len = SZ_2G, + }, + { + .name = "three holes; all holes too small; third biggest", + .hole_start = 0, + .hole_len = 10ULL * SZ_1G, + .min_hole_size = 3ULL * SZ_1G, + .pending_extents = { + { .start = SZ_1G, .len = SZ_1G }, + { .start = 3ULL * SZ_1G, .len = 5ULL * SZ_1G }, + }, + .expected_found = false, + .expected_start = 8ULL * SZ_1G, + .expected_len = SZ_2G, + }, + { + .name = "hole entirely allocated by pending", + .hole_start = 0, + .hole_len = 10ULL * SZ_1G, + .min_hole_size = SZ_1G, + .pending_extents = { + { .start = 0, .len = 10ULL * SZ_1G }, + }, + .expected_found = false, + .expected_start = 10ULL * SZ_1G, + .expected_len = 0, + }, + { + .name = "pending extent at end of range", + .hole_start = 0, + .hole_len = 10ULL * SZ_1G, + .min_hole_size = SZ_1G, + .pending_extents = { + { .start = 9ULL * SZ_1G, .len = SZ_2G }, + }, + .expected_found = true, + .expected_start = 0, + .expected_len = 9ULL * SZ_1G, + }, + { + .name = "zero length input", + .hole_start = SZ_1G, + .hole_len = 0, + .min_hole_size = SZ_1G, + .pending_extents = { }, + .expected_found = false, + .expected_start = SZ_1G, + .expected_len = 0, + }, +}; + +static int test_find_hole_in_pending(u32 sectorsize, u32 nodesize) +{ + struct btrfs_fs_info *fs_info; + struct btrfs_device *device; + int ret = 0; + + test_msg("running find_hole_in_pending_extents tests"); + + fs_info = btrfs_alloc_dummy_fs_info(nodesize, sectorsize); + if (!fs_info) { + test_std_err(TEST_ALLOC_FS_INFO); + return -ENOMEM; + } + + device = btrfs_alloc_dummy_device(fs_info); + if (IS_ERR(device)) { + test_err("failed to allocate dummy device"); + ret = PTR_ERR(device); + goto out_free_fs_info; + } + device->fs_info = fs_info; + + for (int i = 0; i < ARRAY_SIZE(find_hole_tests); i++) { + const struct pending_extent_test_case *test_case = &find_hole_tests[i]; + u64 hole_start = test_case->hole_start; + u64 hole_len = test_case->hole_len; + bool found; + + for (int j = 0; j < ARRAY_SIZE(test_case->pending_extents); j++) { + u64 start = test_case->pending_extents[j].start; + u64 len = test_case->pending_extents[j].len; + + if (!len) + continue; + btrfs_set_extent_bit(&device->alloc_state, + start, start + len - 1, + CHUNK_ALLOCATED, NULL); + } + + mutex_lock(&fs_info->chunk_mutex); + found = btrfs_find_hole_in_pending_extents(device, &hole_start, &hole_len, + test_case->min_hole_size); + mutex_unlock(&fs_info->chunk_mutex); + + if (found != test_case->expected_found) { + test_err("%s: expected found=%d, got found=%d", + test_case->name, test_case->expected_found, found); + ret = -EINVAL; + goto out_clear_pending_extents; + } + if (hole_start != test_case->expected_start || + hole_len != test_case->expected_len) { + test_err("%s: expected [%llu, %llu), got [%llu, %llu)", + test_case->name, test_case->expected_start, + test_case->expected_start + + test_case->expected_len, + hole_start, hole_start + hole_len); + ret = -EINVAL; + goto out_clear_pending_extents; + } +out_clear_pending_extents: + btrfs_clear_extent_bit(&device->alloc_state, 0, (u64)-1, + CHUNK_ALLOCATED, NULL); + if (ret) + break; + } + +out_free_fs_info: + btrfs_free_dummy_fs_info(fs_info); + return ret; +} + +/* + * Describes the inputs to the system and expected results + * when testing btrfs_first_pending_extent(). + */ +struct first_pending_test_case { + const char *name; + /* The range to look for a pending extent in. */ + u64 hole_start; + u64 hole_len; + /* The pending extent to look for. */ + struct { + u64 start; + u64 len; + } pending_extent; + /* Expected outputs. */ + bool expected_found; + u64 expected_pending_start; + u64 expected_pending_end; +}; + +static const struct first_pending_test_case first_pending_tests[] = { + { + .name = "no pending extent", + .hole_start = 0, + .hole_len = 10ULL * SZ_1G, + .pending_extent = { 0, 0 }, + .expected_found = false, + }, + { + .name = "pending extent at search start", + .hole_start = SZ_1G, + .hole_len = 9ULL * SZ_1G, + .pending_extent = { SZ_1G, SZ_1G }, + .expected_found = true, + .expected_pending_start = SZ_1G, + .expected_pending_end = SZ_2G - 1, + }, + { + .name = "pending extent overlapping search start", + .hole_start = SZ_1G, + .hole_len = 9ULL * SZ_1G, + .pending_extent = { 0, SZ_2G }, + .expected_found = true, + .expected_pending_start = 0, + .expected_pending_end = SZ_2G - 1, + }, + { + .name = "pending extent inside search range", + .hole_start = 0, + .hole_len = 10ULL * SZ_1G, + .pending_extent = { SZ_2G, SZ_1G }, + .expected_found = true, + .expected_pending_start = SZ_2G, + .expected_pending_end = 3ULL * SZ_1G - 1, + }, + { + .name = "pending extent outside search range", + .hole_start = 0, + .hole_len = SZ_1G, + .pending_extent = { SZ_2G, SZ_1G }, + .expected_found = false, + }, + { + .name = "pending extent overlapping end of search range", + .hole_start = 0, + .hole_len = SZ_2G, + .pending_extent = { SZ_1G, SZ_2G }, + .expected_found = true, + .expected_pending_start = SZ_1G, + .expected_pending_end = 3ULL * SZ_1G - 1, + }, +}; + +static int test_first_pending_extent(u32 sectorsize, u32 nodesize) +{ + struct btrfs_fs_info *fs_info; + struct btrfs_device *device; + int ret = 0; + + test_msg("running first_pending_extent tests"); + + fs_info = btrfs_alloc_dummy_fs_info(nodesize, sectorsize); + if (!fs_info) { + test_std_err(TEST_ALLOC_FS_INFO); + return -ENOMEM; + } + + device = btrfs_alloc_dummy_device(fs_info); + if (IS_ERR(device)) { + test_err("failed to allocate dummy device"); + ret = PTR_ERR(device); + goto out_free_fs_info; + } + + device->fs_info = fs_info; + + for (int i = 0; i < ARRAY_SIZE(first_pending_tests); i++) { + const struct first_pending_test_case *test_case = &first_pending_tests[i]; + u64 start = test_case->pending_extent.start; + u64 len = test_case->pending_extent.len; + u64 pending_start, pending_end; + bool found; + + if (len) { + btrfs_set_extent_bit(&device->alloc_state, + start, start + len - 1, + CHUNK_ALLOCATED, NULL); + } + + mutex_lock(&fs_info->chunk_mutex); + found = btrfs_first_pending_extent(device, test_case->hole_start, + test_case->hole_len, + &pending_start, &pending_end); + mutex_unlock(&fs_info->chunk_mutex); + + if (found != test_case->expected_found) { + test_err("%s: expected found=%d, got found=%d", + test_case->name, test_case->expected_found, found); + ret = -EINVAL; + goto out_clear_pending_extents; + } + if (!found) + goto out_clear_pending_extents; + + if (pending_start != test_case->expected_pending_start || + pending_end != test_case->expected_pending_end) { + test_err("%s: expected pending [%llu, %llu], got [%llu, %llu]", + test_case->name, + test_case->expected_pending_start, + test_case->expected_pending_end, + pending_start, pending_end); + ret = -EINVAL; + goto out_clear_pending_extents; + } + +out_clear_pending_extents: + btrfs_clear_extent_bit(&device->alloc_state, 0, (u64)-1, + CHUNK_ALLOCATED, NULL); + if (ret) + break; + } + +out_free_fs_info: + btrfs_free_dummy_fs_info(fs_info); + return ret; +} + +int btrfs_test_chunk_allocation(u32 sectorsize, u32 nodesize) +{ + int ret; + + test_msg("running chunk allocation tests"); + + ret = test_first_pending_extent(sectorsize, nodesize); + if (ret) + return ret; + + ret = test_find_hole_in_pending(sectorsize, nodesize); + if (ret) + return ret; + + return 0; +} diff --git a/fs/btrfs/tests/delayed-refs-tests.c b/fs/btrfs/tests/delayed-refs-tests.c new file mode 100644 index 000000000..ccab45d84 --- /dev/null +++ b/fs/btrfs/tests/delayed-refs-tests.c @@ -0,0 +1,1016 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include "btrfs-tests.h" +#include "../transaction.h" +#include "../delayed-ref.h" +#include "../extent-tree.h" + +#define FAKE_ROOT_OBJECTID 256 +#define FAKE_BYTENR 0 +#define FAKE_LEVEL 1 +#define FAKE_INO 256 +#define FAKE_FILE_OFFSET 0 +#define FAKE_PARENT SZ_1M + +struct ref_head_check { + u64 bytenr; + u64 num_bytes; + int ref_mod; + int total_ref_mod; + int must_insert; +}; + +struct ref_node_check { + u64 bytenr; + u64 num_bytes; + int ref_mod; + enum btrfs_delayed_ref_action action; + u8 type; + u64 parent; + u64 root; + u64 owner; + u64 offset; +}; + +static enum btrfs_ref_type ref_type_from_disk_ref_type(u8 type) +{ + if ((type == BTRFS_TREE_BLOCK_REF_KEY) || + (type == BTRFS_SHARED_BLOCK_REF_KEY)) + return BTRFS_REF_METADATA; + return BTRFS_REF_DATA; +} + +static void delete_delayed_ref_head(struct btrfs_trans_handle *trans, + struct btrfs_delayed_ref_head *head) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_delayed_ref_root *delayed_refs = + &trans->transaction->delayed_refs; + + spin_lock(&delayed_refs->lock); + spin_lock(&head->lock); + btrfs_delete_ref_head(fs_info, delayed_refs, head); + spin_unlock(&head->lock); + spin_unlock(&delayed_refs->lock); + + btrfs_delayed_ref_unlock(head); + btrfs_put_delayed_ref_head(head); +} + +static void delete_delayed_ref_node(struct btrfs_delayed_ref_head *head, + struct btrfs_delayed_ref_node *node) +{ + rb_erase_cached(&node->ref_node, &head->ref_tree); + RB_CLEAR_NODE(&node->ref_node); + if (!list_empty(&node->add_list)) + list_del_init(&node->add_list); + btrfs_put_delayed_ref(node); +} + +static int validate_ref_head(struct btrfs_delayed_ref_head *head, + struct ref_head_check *check) +{ + if (head->bytenr != check->bytenr) { + test_err("invalid bytenr have: %llu want: %llu", head->bytenr, + check->bytenr); + return -EINVAL; + } + + if (head->num_bytes != check->num_bytes) { + test_err("invalid num_bytes have: %llu want: %llu", + head->num_bytes, check->num_bytes); + return -EINVAL; + } + + if (head->ref_mod != check->ref_mod) { + test_err("invalid ref_mod have: %d want: %d", head->ref_mod, + check->ref_mod); + return -EINVAL; + } + + if (head->total_ref_mod != check->total_ref_mod) { + test_err("invalid total_ref_mod have: %d want: %d", + head->total_ref_mod, check->total_ref_mod); + return -EINVAL; + } + + if (head->must_insert_reserved != check->must_insert) { + test_err("invalid must_insert have: %d want: %d", + head->must_insert_reserved, check->must_insert); + return -EINVAL; + } + + return 0; +} + +static int validate_ref_node(struct btrfs_delayed_ref_node *node, + struct ref_node_check *check) +{ + if (node->bytenr != check->bytenr) { + test_err("invalid bytenr have: %llu want: %llu", node->bytenr, + check->bytenr); + return -EINVAL; + } + + if (node->num_bytes != check->num_bytes) { + test_err("invalid num_bytes have: %llu want: %llu", + node->num_bytes, check->num_bytes); + return -EINVAL; + } + + if (node->ref_mod != check->ref_mod) { + test_err("invalid ref_mod have: %d want: %d", node->ref_mod, + check->ref_mod); + return -EINVAL; + } + + if (node->action != check->action) { + test_err("invalid action have: %d want: %d", node->action, + check->action); + return -EINVAL; + } + + if (node->parent != check->parent) { + test_err("invalid parent have: %llu want: %llu", node->parent, + check->parent); + return -EINVAL; + } + + if (node->ref_root != check->root) { + test_err("invalid root have: %llu want: %llu", node->ref_root, + check->root); + return -EINVAL; + } + + if (node->type != check->type) { + test_err("invalid type have: %d want: %d", node->type, + check->type); + return -EINVAL; + } + + if (btrfs_delayed_ref_owner(node) != check->owner) { + test_err("invalid owner have: %llu want: %llu", + btrfs_delayed_ref_owner(node), check->owner); + return -EINVAL; + } + + if (btrfs_delayed_ref_offset(node) != check->offset) { + test_err("invalid offset have: %llu want: %llu", + btrfs_delayed_ref_offset(node), check->offset); + return -EINVAL; + } + + return 0; +} + +static int simple_test(struct btrfs_trans_handle *trans, + struct ref_head_check *head_check, + struct ref_node_check *node_check) +{ + struct btrfs_delayed_ref_root *delayed_refs = + &trans->transaction->delayed_refs; + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_delayed_ref_head *head; + struct btrfs_delayed_ref_node *node; + struct btrfs_ref ref = { + .type = ref_type_from_disk_ref_type(node_check->type), + .action = node_check->action, + .parent = node_check->parent, + .ref_root = node_check->root, + .bytenr = node_check->bytenr, + .num_bytes = fs_info->nodesize, + }; + int ret; + + if (ref.type == BTRFS_REF_METADATA) + btrfs_init_tree_ref(&ref, node_check->owner, node_check->root, + false); + else + btrfs_init_data_ref(&ref, node_check->owner, node_check->offset, + node_check->root, true); + + if (ref.type == BTRFS_REF_METADATA) + ret = btrfs_add_delayed_tree_ref(trans, &ref, NULL); + else + ret = btrfs_add_delayed_data_ref(trans, &ref, 0); + if (ret) { + test_err("failed ref action %d", ret); + return ret; + } + + head = btrfs_select_ref_head(fs_info, delayed_refs); + if (IS_ERR_OR_NULL(head)) { + if (IS_ERR(head)) + test_err("failed to select delayed ref head: %ld", + PTR_ERR(head)); + else + test_err("failed to find delayed ref head"); + return -EINVAL; + } + + ret = -EINVAL; + if (validate_ref_head(head, head_check)) + goto out; + + spin_lock(&head->lock); + node = btrfs_select_delayed_ref(head); + spin_unlock(&head->lock); + if (!node) { + test_err("failed to select delayed ref"); + goto out; + } + + if (validate_ref_node(node, node_check)) + goto out; + ret = 0; +out: + btrfs_unselect_ref_head(delayed_refs, head); + btrfs_destroy_delayed_refs(trans->transaction); + return ret; +} + +/* + * These are simple tests, make sure that our btrfs_ref's get turned into the + * appropriate btrfs_delayed_ref_node based on their settings and action. + */ +static int simple_tests(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct ref_head_check head_check = { + .bytenr = FAKE_BYTENR, + .num_bytes = fs_info->nodesize, + .ref_mod = 1, + .total_ref_mod = 1, + }; + struct ref_node_check node_check = { + .bytenr = FAKE_BYTENR, + .num_bytes = fs_info->nodesize, + .ref_mod = 1, + .action = BTRFS_ADD_DELAYED_REF, + .type = BTRFS_TREE_BLOCK_REF_KEY, + .parent = 0, + .root = FAKE_ROOT_OBJECTID, + .owner = FAKE_LEVEL, + .offset = 0, + }; + + if (simple_test(trans, &head_check, &node_check)) { + test_err("single add tree block failed"); + return -EINVAL; + } + + node_check.type = BTRFS_EXTENT_DATA_REF_KEY; + node_check.owner = FAKE_INO; + node_check.offset = FAKE_FILE_OFFSET; + + if (simple_test(trans, &head_check, &node_check)) { + test_err("single add extent data failed"); + return -EINVAL; + } + + node_check.parent = FAKE_PARENT; + node_check.type = BTRFS_SHARED_BLOCK_REF_KEY; + node_check.owner = FAKE_LEVEL; + node_check.offset = 0; + + if (simple_test(trans, &head_check, &node_check)) { + test_err("single add shared block failed"); + return -EINVAL; + } + + node_check.type = BTRFS_SHARED_DATA_REF_KEY; + node_check.owner = FAKE_INO; + node_check.offset = FAKE_FILE_OFFSET; + + if (simple_test(trans, &head_check, &node_check)) { + test_err("single add shared data failed"); + return -EINVAL; + } + + head_check.ref_mod = -1; + head_check.total_ref_mod = -1; + node_check.action = BTRFS_DROP_DELAYED_REF; + node_check.type = BTRFS_TREE_BLOCK_REF_KEY; + node_check.owner = FAKE_LEVEL; + node_check.offset = 0; + node_check.parent = 0; + + if (simple_test(trans, &head_check, &node_check)) { + test_err("single drop tree block failed"); + return -EINVAL; + } + + node_check.type = BTRFS_EXTENT_DATA_REF_KEY; + node_check.owner = FAKE_INO; + node_check.offset = FAKE_FILE_OFFSET; + + if (simple_test(trans, &head_check, &node_check)) { + test_err("single drop extent data failed"); + return -EINVAL; + } + + node_check.parent = FAKE_PARENT; + node_check.type = BTRFS_SHARED_BLOCK_REF_KEY; + node_check.owner = FAKE_LEVEL; + node_check.offset = 0; + if (simple_test(trans, &head_check, &node_check)) { + test_err("single drop shared block failed"); + return -EINVAL; + } + + node_check.type = BTRFS_SHARED_DATA_REF_KEY; + node_check.owner = FAKE_INO; + node_check.offset = FAKE_FILE_OFFSET; + if (simple_test(trans, &head_check, &node_check)) { + test_err("single drop shared data failed"); + return -EINVAL; + } + + return 0; +} + +/* + * Merge tests, validate that we do delayed ref merging properly, the ref counts + * all end up properly, and delayed refs are deleted once they're no longer + * needed. + */ +static int merge_tests(struct btrfs_trans_handle *trans, + enum btrfs_ref_type type) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_delayed_ref_head *head = NULL; + struct btrfs_delayed_ref_node *node; + struct btrfs_ref ref = { + .type = type, + .action = BTRFS_ADD_DELAYED_REF, + .parent = 0, + .ref_root = FAKE_ROOT_OBJECTID, + .bytenr = FAKE_BYTENR, + .num_bytes = fs_info->nodesize, + }; + struct ref_head_check head_check = { + .bytenr = FAKE_BYTENR, + .num_bytes = fs_info->nodesize, + .ref_mod = 0, + .total_ref_mod = 0, + }; + struct ref_node_check node_check = { + .bytenr = FAKE_BYTENR, + .num_bytes = fs_info->nodesize, + .ref_mod = 2, + .action = BTRFS_ADD_DELAYED_REF, + .parent = 0, + .root = FAKE_ROOT_OBJECTID, + }; + int ret; + + /* + * First add a ref and then drop it, make sure we get a head ref with a + * 0 total ref mod and no nodes. + */ + if (type == BTRFS_REF_METADATA) { + node_check.type = BTRFS_TREE_BLOCK_REF_KEY; + node_check.owner = FAKE_LEVEL; + btrfs_init_tree_ref(&ref, FAKE_LEVEL, FAKE_ROOT_OBJECTID, false); + } else { + node_check.type = BTRFS_EXTENT_DATA_REF_KEY; + node_check.owner = FAKE_INO; + node_check.offset = FAKE_FILE_OFFSET; + btrfs_init_data_ref(&ref, FAKE_INO, FAKE_FILE_OFFSET, + FAKE_ROOT_OBJECTID, true); + } + + if (type == BTRFS_REF_METADATA) + ret = btrfs_add_delayed_tree_ref(trans, &ref, NULL); + else + ret = btrfs_add_delayed_data_ref(trans, &ref, 0); + if (ret) { + test_err("failed ref action %d", ret); + return ret; + } + + ref.action = BTRFS_DROP_DELAYED_REF; + if (type == BTRFS_REF_METADATA) + ret = btrfs_add_delayed_tree_ref(trans, &ref, NULL); + else + ret = btrfs_add_delayed_data_ref(trans, &ref, 0); + if (ret) { + test_err("failed ref action %d", ret); + goto out; + } + + head = btrfs_select_ref_head(fs_info, &trans->transaction->delayed_refs); + if (IS_ERR_OR_NULL(head)) { + if (IS_ERR(head)) + test_err("failed to select delayed ref head: %ld", + PTR_ERR(head)); + else + test_err("failed to find delayed ref head"); + goto out; + } + + ret = -EINVAL; + if (validate_ref_head(head, &head_check)) { + test_err("single add and drop failed"); + goto out; + } + + spin_lock(&head->lock); + node = btrfs_select_delayed_ref(head); + spin_unlock(&head->lock); + if (node) { + test_err("found node when none should exist"); + goto out; + } + + delete_delayed_ref_head(trans, head); + head = NULL; + + /* + * Add a ref, then add another ref, make sure we get a head ref with a + * 2 total ref mod and 1 node. + */ + ref.action = BTRFS_ADD_DELAYED_REF; + if (type == BTRFS_REF_METADATA) + ret = btrfs_add_delayed_tree_ref(trans, &ref, NULL); + else + ret = btrfs_add_delayed_data_ref(trans, &ref, 0); + if (ret) { + test_err("failed ref action %d", ret); + goto out; + } + + if (type == BTRFS_REF_METADATA) + ret = btrfs_add_delayed_tree_ref(trans, &ref, NULL); + else + ret = btrfs_add_delayed_data_ref(trans, &ref, 0); + if (ret) { + test_err("failed ref action %d", ret); + goto out; + } + + head = btrfs_select_ref_head(fs_info, &trans->transaction->delayed_refs); + if (IS_ERR_OR_NULL(head)) { + if (IS_ERR(head)) + test_err("failed to select delayed ref head: %ld", + PTR_ERR(head)); + else + test_err("failed to find delayed ref head"); + goto out; + } + + head_check.ref_mod = 2; + head_check.total_ref_mod = 2; + ret = -EINVAL; + if (validate_ref_head(head, &head_check)) { + test_err("double add failed"); + goto out; + } + + spin_lock(&head->lock); + node = btrfs_select_delayed_ref(head); + spin_unlock(&head->lock); + if (!node) { + test_err("failed to select delayed ref"); + goto out; + } + + if (validate_ref_node(node, &node_check)) { + test_err("node check failed"); + goto out; + } + + delete_delayed_ref_node(head, node); + + spin_lock(&head->lock); + node = btrfs_select_delayed_ref(head); + spin_unlock(&head->lock); + if (node) { + test_err("found node when none should exist"); + goto out; + } + delete_delayed_ref_head(trans, head); + head = NULL; + + /* Add two drop refs, make sure they are merged properly. */ + ref.action = BTRFS_DROP_DELAYED_REF; + if (type == BTRFS_REF_METADATA) + ret = btrfs_add_delayed_tree_ref(trans, &ref, NULL); + else + ret = btrfs_add_delayed_data_ref(trans, &ref, 0); + if (ret) { + test_err("failed ref action %d", ret); + goto out; + } + + if (type == BTRFS_REF_METADATA) + ret = btrfs_add_delayed_tree_ref(trans, &ref, NULL); + else + ret = btrfs_add_delayed_data_ref(trans, &ref, 0); + if (ret) { + test_err("failed ref action %d", ret); + goto out; + } + + head = btrfs_select_ref_head(fs_info, &trans->transaction->delayed_refs); + if (IS_ERR_OR_NULL(head)) { + if (IS_ERR(head)) + test_err("failed to select delayed ref head: %ld", + PTR_ERR(head)); + else + test_err("failed to find delayed ref head"); + goto out; + } + + head_check.ref_mod = -2; + head_check.total_ref_mod = -2; + ret = -EINVAL; + if (validate_ref_head(head, &head_check)) { + test_err("double drop failed"); + goto out; + } + + node_check.action = BTRFS_DROP_DELAYED_REF; + spin_lock(&head->lock); + node = btrfs_select_delayed_ref(head); + spin_unlock(&head->lock); + if (!node) { + test_err("failed to select delayed ref"); + goto out; + } + + if (validate_ref_node(node, &node_check)) { + test_err("node check failed"); + goto out; + } + + delete_delayed_ref_node(head, node); + + spin_lock(&head->lock); + node = btrfs_select_delayed_ref(head); + spin_unlock(&head->lock); + if (node) { + test_err("found node when none should exist"); + goto out; + } + delete_delayed_ref_head(trans, head); + head = NULL; + + /* Add multiple refs, then drop until we go negative again. */ + ref.action = BTRFS_ADD_DELAYED_REF; + for (int i = 0; i < 10; i++) { + if (type == BTRFS_REF_METADATA) + ret = btrfs_add_delayed_tree_ref(trans, &ref, NULL); + else + ret = btrfs_add_delayed_data_ref(trans, &ref, 0); + if (ret) { + test_err("failed ref action %d", ret); + goto out; + } + } + + ref.action = BTRFS_DROP_DELAYED_REF; + for (int i = 0; i < 12; i++) { + if (type == BTRFS_REF_METADATA) + ret = btrfs_add_delayed_tree_ref(trans, &ref, NULL); + else + ret = btrfs_add_delayed_data_ref(trans, &ref, 0); + if (ret) { + test_err("failed ref action %d", ret); + goto out; + } + } + + head = btrfs_select_ref_head(fs_info, &trans->transaction->delayed_refs); + if (IS_ERR_OR_NULL(head)) { + if (IS_ERR(head)) + test_err("failed to select delayed ref head: %ld", + PTR_ERR(head)); + else + test_err("failed to find delayed ref head"); + ret = -EINVAL; + goto out; + } + + head_check.ref_mod = -2; + head_check.total_ref_mod = -2; + ret = -EINVAL; + if (validate_ref_head(head, &head_check)) { + test_err("double drop failed"); + goto out; + } + + spin_lock(&head->lock); + node = btrfs_select_delayed_ref(head); + spin_unlock(&head->lock); + if (!node) { + test_err("failed to select delayed ref"); + goto out; + } + + if (validate_ref_node(node, &node_check)) { + test_err("node check failed"); + goto out; + } + + delete_delayed_ref_node(head, node); + + spin_lock(&head->lock); + node = btrfs_select_delayed_ref(head); + spin_unlock(&head->lock); + if (node) { + test_err("found node when none should exist"); + goto out; + } + + delete_delayed_ref_head(trans, head); + head = NULL; + + /* Drop multiple refs, then add until we go positive again. */ + ref.action = BTRFS_DROP_DELAYED_REF; + for (int i = 0; i < 10; i++) { + if (type == BTRFS_REF_METADATA) + ret = btrfs_add_delayed_tree_ref(trans, &ref, NULL); + else + ret = btrfs_add_delayed_data_ref(trans, &ref, 0); + if (ret) { + test_err("failed ref action %d", ret); + goto out; + } + } + + ref.action = BTRFS_ADD_DELAYED_REF; + for (int i = 0; i < 12; i++) { + if (type == BTRFS_REF_METADATA) + ret = btrfs_add_delayed_tree_ref(trans, &ref, NULL); + else + ret = btrfs_add_delayed_data_ref(trans, &ref, 0); + if (ret) { + test_err("failed ref action %d", ret); + goto out; + } + } + + head = btrfs_select_ref_head(fs_info, &trans->transaction->delayed_refs); + if (IS_ERR_OR_NULL(head)) { + if (IS_ERR(head)) + test_err("failed to select delayed ref head: %ld", + PTR_ERR(head)); + else + test_err("failed to find delayed ref head"); + ret = -EINVAL; + goto out; + } + + head_check.ref_mod = 2; + head_check.total_ref_mod = 2; + ret = -EINVAL; + if (validate_ref_head(head, &head_check)) { + test_err("add and drop to positive failed"); + goto out; + } + + node_check.action = BTRFS_ADD_DELAYED_REF; + spin_lock(&head->lock); + node = btrfs_select_delayed_ref(head); + spin_unlock(&head->lock); + if (!node) { + test_err("failed to select delayed ref"); + goto out; + } + + if (validate_ref_node(node, &node_check)) { + test_err("node check failed"); + goto out; + } + + delete_delayed_ref_node(head, node); + + spin_lock(&head->lock); + node = btrfs_select_delayed_ref(head); + spin_unlock(&head->lock); + if (node) { + test_err("found node when none should exist"); + goto out; + } + delete_delayed_ref_head(trans, head); + head = NULL; + + /* + * Add a bunch of refs with different roots and parents, then drop them + * all, make sure everything is properly merged. + */ + ref.action = BTRFS_ADD_DELAYED_REF; + for (int i = 0; i < 50; i++) { + if (!(i % 2)) { + ref.parent = 0; + ref.ref_root = FAKE_ROOT_OBJECTID + i; + } else { + ref.parent = FAKE_PARENT + (i * fs_info->nodesize); + } + if (type == BTRFS_REF_METADATA) + ret = btrfs_add_delayed_tree_ref(trans, &ref, NULL); + else + ret = btrfs_add_delayed_data_ref(trans, &ref, 0); + if (ret) { + test_err("failed ref action %d", ret); + goto out; + } + } + + ref.action = BTRFS_DROP_DELAYED_REF; + for (int i = 0; i < 50; i++) { + if (!(i % 2)) { + ref.parent = 0; + ref.ref_root = FAKE_ROOT_OBJECTID + i; + } else { + ref.parent = FAKE_PARENT + (i * fs_info->nodesize); + } + if (type == BTRFS_REF_METADATA) + ret = btrfs_add_delayed_tree_ref(trans, &ref, NULL); + else + ret = btrfs_add_delayed_data_ref(trans, &ref, 0); + if (ret) { + test_err("failed ref action %d", ret); + goto out; + } + } + + head = btrfs_select_ref_head(fs_info, &trans->transaction->delayed_refs); + if (IS_ERR_OR_NULL(head)) { + if (IS_ERR(head)) + test_err("failed to select delayed ref head: %ld", + PTR_ERR(head)); + else + test_err("failed to find delayed ref head"); + ret = -EINVAL; + goto out; + } + + head_check.ref_mod = 0; + head_check.total_ref_mod = 0; + ret = -EINVAL; + if (validate_ref_head(head, &head_check)) { + test_err("add and drop multiple failed"); + goto out; + } + + spin_lock(&head->lock); + node = btrfs_select_delayed_ref(head); + spin_unlock(&head->lock); + if (node) { + test_err("found node when none should exist"); + goto out; + } + ret = 0; +out: + if (!IS_ERR_OR_NULL(head)) + btrfs_unselect_ref_head(&trans->transaction->delayed_refs, head); + btrfs_destroy_delayed_refs(trans->transaction); + return ret; +} + +/* + * Basic test to validate we always get the add operations first followed by any + * delete operations. + */ +static int select_delayed_refs_test(struct btrfs_trans_handle *trans) +{ + struct btrfs_delayed_ref_root *delayed_refs = + &trans->transaction->delayed_refs; + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_delayed_ref_head *head = NULL; + struct btrfs_delayed_ref_node *node; + struct btrfs_ref ref = { + .type = BTRFS_REF_METADATA, + .action = BTRFS_DROP_DELAYED_REF, + .parent = 0, + .ref_root = FAKE_ROOT_OBJECTID, + .bytenr = FAKE_BYTENR, + .num_bytes = fs_info->nodesize, + }; + struct ref_head_check head_check = { + .bytenr = FAKE_BYTENR, + .num_bytes = fs_info->nodesize, + .ref_mod = 0, + .total_ref_mod = 0, + }; + struct ref_node_check node_check = { + .bytenr = FAKE_BYTENR, + .num_bytes = fs_info->nodesize, + .ref_mod = 1, + .action = BTRFS_ADD_DELAYED_REF, + .type = BTRFS_TREE_BLOCK_REF_KEY, + .parent = 0, + .owner = FAKE_LEVEL, + .offset = 0, + }; + int ret; + + /* Add the drop first. */ + btrfs_init_tree_ref(&ref, FAKE_LEVEL, FAKE_ROOT_OBJECTID, false); + ret = btrfs_add_delayed_tree_ref(trans, &ref, NULL); + if (ret) { + test_err("failed ref action %d", ret); + return ret; + } + + /* + * Now add the add, and make it a different root so it's logically later + * in the rb tree. + */ + ref.action = BTRFS_ADD_DELAYED_REF; + ref.ref_root = FAKE_ROOT_OBJECTID + 1; + ret = btrfs_add_delayed_tree_ref(trans, &ref, NULL); + if (ret) { + test_err("failed ref action %d", ret); + goto out; + } + + head = btrfs_select_ref_head(fs_info, delayed_refs); + if (IS_ERR_OR_NULL(head)) { + if (IS_ERR(head)) + test_err("failed to select delayed ref head: %ld", + PTR_ERR(head)); + else + test_err("failed to find delayed ref head"); + ret = -EINVAL; + head = NULL; + goto out; + } + + ret = -EINVAL; + if (validate_ref_head(head, &head_check)) { + test_err("head check failed"); + goto out; + } + + spin_lock(&head->lock); + node = btrfs_select_delayed_ref(head); + spin_unlock(&head->lock); + if (!node) { + test_err("failed to select delayed ref"); + goto out; + } + + node_check.root = FAKE_ROOT_OBJECTID + 1; + if (validate_ref_node(node, &node_check)) { + test_err("node check failed"); + goto out; + } + delete_delayed_ref_node(head, node); + + spin_lock(&head->lock); + node = btrfs_select_delayed_ref(head); + spin_unlock(&head->lock); + if (!node) { + test_err("failed to select delayed ref"); + goto out; + } + + node_check.action = BTRFS_DROP_DELAYED_REF; + node_check.root = FAKE_ROOT_OBJECTID; + if (validate_ref_node(node, &node_check)) { + test_err("node check failed"); + goto out; + } + delete_delayed_ref_node(head, node); + delete_delayed_ref_head(trans, head); + head = NULL; + + /* + * Now we're going to do the same thing, but we're going to have an add + * that gets deleted because of a merge, and make sure we still have + * another add in place. + */ + ref.action = BTRFS_DROP_DELAYED_REF; + ref.ref_root = FAKE_ROOT_OBJECTID; + ret = btrfs_add_delayed_tree_ref(trans, &ref, NULL); + if (ret) { + test_err("failed ref action %d", ret); + goto out; + } + + ref.action = BTRFS_ADD_DELAYED_REF; + ref.ref_root = FAKE_ROOT_OBJECTID + 1; + ret = btrfs_add_delayed_tree_ref(trans, &ref, NULL); + if (ret) { + test_err("failed ref action %d", ret); + goto out; + } + + ref.action = BTRFS_DROP_DELAYED_REF; + ret = btrfs_add_delayed_tree_ref(trans, &ref, NULL); + if (ret) { + test_err("failed ref action %d", ret); + goto out; + } + + ref.action = BTRFS_ADD_DELAYED_REF; + ref.ref_root = FAKE_ROOT_OBJECTID + 2; + ret = btrfs_add_delayed_tree_ref(trans, &ref, NULL); + if (ret) { + test_err("failed ref action %d", ret); + goto out; + } + + head = btrfs_select_ref_head(fs_info, delayed_refs); + if (IS_ERR_OR_NULL(head)) { + if (IS_ERR(head)) + test_err("failed to select delayed ref head: %ld", + PTR_ERR(head)); + else + test_err("failed to find delayed ref head"); + ret = -EINVAL; + head = NULL; + goto out; + } + + ret = -EINVAL; + if (validate_ref_head(head, &head_check)) { + test_err("head check failed"); + goto out; + } + + spin_lock(&head->lock); + node = btrfs_select_delayed_ref(head); + spin_unlock(&head->lock); + if (!node) { + test_err("failed to select delayed ref"); + goto out; + } + + node_check.action = BTRFS_ADD_DELAYED_REF; + node_check.root = FAKE_ROOT_OBJECTID + 2; + if (validate_ref_node(node, &node_check)) { + test_err("node check failed"); + goto out; + } + delete_delayed_ref_node(head, node); + + spin_lock(&head->lock); + node = btrfs_select_delayed_ref(head); + spin_unlock(&head->lock); + if (!node) { + test_err("failed to select delayed ref"); + goto out; + } + + node_check.action = BTRFS_DROP_DELAYED_REF; + node_check.root = FAKE_ROOT_OBJECTID; + if (validate_ref_node(node, &node_check)) { + test_err("node check failed"); + goto out; + } + delete_delayed_ref_node(head, node); + ret = 0; +out: + if (head) + btrfs_unselect_ref_head(delayed_refs, head); + btrfs_destroy_delayed_refs(trans->transaction); + return ret; +} + +int btrfs_test_delayed_refs(u32 sectorsize, u32 nodesize) +{ + struct btrfs_transaction *transaction; + struct btrfs_trans_handle trans; + struct btrfs_fs_info *fs_info; + int ret; + + test_msg("running delayed refs tests"); + + fs_info = btrfs_alloc_dummy_fs_info(nodesize, sectorsize); + if (!fs_info) { + test_std_err(TEST_ALLOC_FS_INFO); + return -ENOMEM; + } + transaction = kmalloc_obj(*transaction); + if (!transaction) { + test_std_err(TEST_ALLOC_TRANSACTION); + ret = -ENOMEM; + goto out_free_fs_info; + } + btrfs_init_dummy_trans(&trans, fs_info); + btrfs_init_dummy_transaction(transaction, fs_info); + trans.transaction = transaction; + + ret = simple_tests(&trans); + if (!ret) { + test_msg("running delayed refs merge tests on metadata refs"); + ret = merge_tests(&trans, BTRFS_REF_METADATA); + } + + if (!ret) { + test_msg("running delayed refs merge tests on data refs"); + ret = merge_tests(&trans, BTRFS_REF_DATA); + } + + if (!ret) + ret = select_delayed_refs_test(&trans); + + kfree(transaction); +out_free_fs_info: + btrfs_free_dummy_fs_info(fs_info); + return ret; +} diff --git a/fs/btrfs/tests/extent-buffer-tests.c b/fs/btrfs/tests/extent-buffer-tests.c new file mode 100644 index 000000000..6a43a64ba --- /dev/null +++ b/fs/btrfs/tests/extent-buffer-tests.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2013 Fusion IO. All rights reserved. + */ + +#include +#include "btrfs-tests.h" +#include "../ctree.h" +#include "../extent_io.h" +#include "../disk-io.h" +#include "../accessors.h" + +static int test_btrfs_split_item(u32 sectorsize, u32 nodesize) +{ + struct btrfs_fs_info *fs_info; + struct btrfs_path *path = NULL; + struct btrfs_root *root = NULL; + struct extent_buffer *eb; + char *value = "mary had a little lamb"; + char *split1 = "mary had a little"; + char *split2 = " lamb"; + char *split3 = "mary"; + char *split4 = " had a little"; + char buf[32]; + struct btrfs_key key; + u32 value_len = strlen(value); + int ret = 0; + + test_msg("running btrfs_split_item tests"); + + fs_info = btrfs_alloc_dummy_fs_info(nodesize, sectorsize); + if (!fs_info) { + test_std_err(TEST_ALLOC_FS_INFO); + return -ENOMEM; + } + + root = btrfs_alloc_dummy_root(fs_info); + if (IS_ERR(root)) { + test_std_err(TEST_ALLOC_ROOT); + ret = PTR_ERR(root); + goto out; + } + + path = btrfs_alloc_path(); + if (!path) { + test_std_err(TEST_ALLOC_PATH); + ret = -ENOMEM; + goto out; + } + + eb = alloc_dummy_extent_buffer(fs_info, nodesize); + path->nodes[0] = eb; + if (!eb) { + test_std_err(TEST_ALLOC_EXTENT_BUFFER); + ret = -ENOMEM; + goto out; + } + path->slots[0] = 0; + + key.objectid = 0; + key.type = BTRFS_EXTENT_CSUM_KEY; + key.offset = 0; + + /* + * Passing a NULL trans handle is fine here, we have a dummy root eb + * and the tree is a single node (level 0). + */ + btrfs_setup_item_for_insert(NULL, root, path, &key, value_len); + write_extent_buffer(eb, value, btrfs_item_ptr_offset(eb, 0), + value_len); + + key.offset = 3; + + /* + * Passing NULL trans here should be safe because we have plenty of + * space in this leaf to split the item without having to split the + * leaf. + */ + ret = btrfs_split_item(NULL, root, path, &key, 17); + if (ret) { + test_err("split item failed %d", ret); + goto out; + } + + /* + * Read the first slot, it should have the original key and contain only + * 'mary had a little' + */ + btrfs_item_key_to_cpu(eb, &key, 0); + if (key.objectid != 0 || key.type != BTRFS_EXTENT_CSUM_KEY || + key.offset != 0) { + test_err("invalid key at slot 0"); + ret = -EINVAL; + goto out; + } + + if (btrfs_item_size(eb, 0) != strlen(split1)) { + test_err("invalid len in the first split"); + ret = -EINVAL; + goto out; + } + + read_extent_buffer(eb, buf, btrfs_item_ptr_offset(eb, 0), + strlen(split1)); + if (memcmp(buf, split1, strlen(split1))) { + test_err( +"data in the buffer doesn't match what it should in the first split have='%.*s' want '%s'", + (int)strlen(split1), buf, split1); + ret = -EINVAL; + goto out; + } + + btrfs_item_key_to_cpu(eb, &key, 1); + if (key.objectid != 0 || key.type != BTRFS_EXTENT_CSUM_KEY || + key.offset != 3) { + test_err("invalid key at slot 1"); + ret = -EINVAL; + goto out; + } + + if (btrfs_item_size(eb, 1) != strlen(split2)) { + test_err("invalid len in the second split"); + ret = -EINVAL; + goto out; + } + + read_extent_buffer(eb, buf, btrfs_item_ptr_offset(eb, 1), + strlen(split2)); + if (memcmp(buf, split2, strlen(split2))) { + test_err( + "data in the buffer doesn't match what it should in the second split"); + ret = -EINVAL; + goto out; + } + + key.offset = 1; + /* Do it again so we test memmoving the other items in the leaf */ + ret = btrfs_split_item(NULL, root, path, &key, 4); + if (ret) { + test_err("second split item failed %d", ret); + goto out; + } + + btrfs_item_key_to_cpu(eb, &key, 0); + if (key.objectid != 0 || key.type != BTRFS_EXTENT_CSUM_KEY || + key.offset != 0) { + test_err("invalid key at slot 0"); + ret = -EINVAL; + goto out; + } + + if (btrfs_item_size(eb, 0) != strlen(split3)) { + test_err("invalid len in the first split"); + ret = -EINVAL; + goto out; + } + + read_extent_buffer(eb, buf, btrfs_item_ptr_offset(eb, 0), + strlen(split3)); + if (memcmp(buf, split3, strlen(split3))) { + test_err( + "data in the buffer doesn't match what it should in the third split"); + ret = -EINVAL; + goto out; + } + + btrfs_item_key_to_cpu(eb, &key, 1); + if (key.objectid != 0 || key.type != BTRFS_EXTENT_CSUM_KEY || + key.offset != 1) { + test_err("invalid key at slot 1"); + ret = -EINVAL; + goto out; + } + + if (btrfs_item_size(eb, 1) != strlen(split4)) { + test_err("invalid len in the second split"); + ret = -EINVAL; + goto out; + } + + read_extent_buffer(eb, buf, btrfs_item_ptr_offset(eb, 1), + strlen(split4)); + if (memcmp(buf, split4, strlen(split4))) { + test_err( + "data in the buffer doesn't match what it should in the fourth split"); + ret = -EINVAL; + goto out; + } + + btrfs_item_key_to_cpu(eb, &key, 2); + if (key.objectid != 0 || key.type != BTRFS_EXTENT_CSUM_KEY || + key.offset != 3) { + test_err("invalid key at slot 2"); + ret = -EINVAL; + goto out; + } + + if (btrfs_item_size(eb, 2) != strlen(split2)) { + test_err("invalid len in the second split"); + ret = -EINVAL; + goto out; + } + + read_extent_buffer(eb, buf, btrfs_item_ptr_offset(eb, 2), + strlen(split2)); + if (memcmp(buf, split2, strlen(split2))) { + test_err( + "data in the buffer doesn't match what it should in the last chunk"); + ret = -EINVAL; + goto out; + } +out: + btrfs_free_path(path); + btrfs_free_dummy_root(root); + btrfs_free_dummy_fs_info(fs_info); + return ret; +} + +int btrfs_test_extent_buffer_operations(u32 sectorsize, u32 nodesize) +{ + test_msg("running extent buffer operation tests"); + return test_btrfs_split_item(sectorsize, nodesize); +} diff --git a/fs/btrfs/tests/extent-io-tests.c b/fs/btrfs/tests/extent-io-tests.c new file mode 100644 index 000000000..b2aacf846 --- /dev/null +++ b/fs/btrfs/tests/extent-io-tests.c @@ -0,0 +1,830 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2013 Fusion IO. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include "btrfs-tests.h" +#include "../ctree.h" +#include "../extent_io.h" +#include "../disk-io.h" +#include "../btrfs_inode.h" + +#define PROCESS_UNLOCK (1U << 0) +#define PROCESS_RELEASE (1U << 1) +#define PROCESS_TEST_LOCKED (1U << 2) + +static noinline int process_page_range(struct inode *inode, u64 start, u64 end, + unsigned long flags) +{ + int ret; + struct folio_batch fbatch; + pgoff_t index = start >> PAGE_SHIFT; + pgoff_t end_index = end >> PAGE_SHIFT; + int i; + int count = 0; + int loops = 0; + + folio_batch_init(&fbatch); + + while (index <= end_index) { + ret = filemap_get_folios_contig(inode->i_mapping, &index, + end_index, &fbatch); + for (i = 0; i < ret; i++) { + struct folio *folio = fbatch.folios[i]; + + if (flags & PROCESS_TEST_LOCKED && + !folio_test_locked(folio)) + count++; + if (flags & PROCESS_UNLOCK && folio_test_locked(folio)) + folio_unlock(folio); + if (flags & PROCESS_RELEASE) + folio_put(folio); + } + folio_batch_release(&fbatch); + cond_resched(); + loops++; + if (loops > 100000) { + printk(KERN_ERR + "stuck in a loop, start %llu, end %llu, ret %d\n", + start, end, ret); + break; + } + } + + return count; +} + +#define STATE_FLAG_STR_LEN 256 + +#define PRINT_ONE_FLAG(state, dest, cur, name) \ +({ \ + if (state->state & EXTENT_##name) \ + cur += scnprintf(dest + cur, STATE_FLAG_STR_LEN - cur, \ + "%s" #name, cur == 0 ? "" : "|"); \ +}) + +static void extent_flag_to_str(const struct extent_state *state, char *dest) +{ + int cur = 0; + + dest[0] = 0; + PRINT_ONE_FLAG(state, dest, cur, DIRTY); + PRINT_ONE_FLAG(state, dest, cur, LOCKED); + PRINT_ONE_FLAG(state, dest, cur, DIRTY_LOG1); + PRINT_ONE_FLAG(state, dest, cur, DIRTY_LOG2); + PRINT_ONE_FLAG(state, dest, cur, DELALLOC); + PRINT_ONE_FLAG(state, dest, cur, DEFRAG); + PRINT_ONE_FLAG(state, dest, cur, BOUNDARY); + PRINT_ONE_FLAG(state, dest, cur, NODATASUM); + PRINT_ONE_FLAG(state, dest, cur, CLEAR_META_RESV); + PRINT_ONE_FLAG(state, dest, cur, NEED_WAIT); + PRINT_ONE_FLAG(state, dest, cur, NORESERVE); + PRINT_ONE_FLAG(state, dest, cur, QGROUP_RESERVED); + PRINT_ONE_FLAG(state, dest, cur, CLEAR_DATA_RESV); +} + +static void dump_extent_io_tree(const struct extent_io_tree *tree) +{ + struct rb_node *node; + char flags_str[STATE_FLAG_STR_LEN]; + + node = rb_first(&tree->state); + test_msg("io tree content:"); + while (node) { + struct extent_state *state; + + state = rb_entry(node, struct extent_state, rb_node); + extent_flag_to_str(state, flags_str); + test_msg(" start=%llu len=%llu flags=%s", state->start, + state->end + 1 - state->start, flags_str); + node = rb_next(node); + } +} + +static int test_find_delalloc(u32 sectorsize, u32 nodesize) +{ + struct btrfs_fs_info *fs_info; + struct btrfs_root *root = NULL; + struct inode *inode = NULL; + struct extent_io_tree *tmp; + struct page *page; + struct page *locked_page = NULL; + /* In this test we need at least 2 file extents at its maximum size */ + u64 max_bytes = BTRFS_MAX_EXTENT_SIZE; + u64 total_dirty = 2 * max_bytes; + u64 start, end, test_start; + bool found; + int ret = -EINVAL; + + test_msg("running find delalloc tests"); + + fs_info = btrfs_alloc_dummy_fs_info(nodesize, sectorsize); + if (!fs_info) { + test_std_err(TEST_ALLOC_FS_INFO); + return -ENOMEM; + } + + root = btrfs_alloc_dummy_root(fs_info); + if (IS_ERR(root)) { + test_std_err(TEST_ALLOC_ROOT); + ret = PTR_ERR(root); + goto out; + } + + inode = btrfs_new_test_inode(); + if (!inode) { + test_std_err(TEST_ALLOC_INODE); + ret = -ENOMEM; + goto out; + } + tmp = &BTRFS_I(inode)->io_tree; + BTRFS_I(inode)->root = root; + + /* + * Passing NULL as we don't have fs_info but tracepoints are not used + * at this point + */ + btrfs_extent_io_tree_init(NULL, tmp, IO_TREE_SELFTEST); + + /* + * First go through and create and mark all of our pages dirty, we pin + * everything to make sure our pages don't get evicted and screw up our + * test. + */ + for (pgoff_t index = 0; index < (total_dirty >> PAGE_SHIFT); index++) { + page = find_or_create_page(inode->i_mapping, index, GFP_KERNEL); + if (!page) { + test_err("failed to allocate test page"); + ret = -ENOMEM; + goto out; + } + SetPageDirty(page); + if (index) { + unlock_page(page); + } else { + get_page(page); + locked_page = page; + } + } + + /* Test this scenario + * |--- delalloc ---| + * |--- search ---| + */ + btrfs_set_extent_bit(tmp, 0, sectorsize - 1, EXTENT_DELALLOC, NULL); + start = 0; + end = start + PAGE_SIZE - 1; + found = find_lock_delalloc_range(inode, page_folio(locked_page), &start, + &end); + if (!found) { + test_err("should have found at least one delalloc"); + goto out_bits; + } + if (start != 0 || end != (sectorsize - 1)) { + test_err("expected start 0 end %u, got start %llu end %llu", + sectorsize - 1, start, end); + goto out_bits; + } + btrfs_unlock_extent(tmp, start, end, NULL); + unlock_page(locked_page); + put_page(locked_page); + + /* + * Test this scenario + * + * |--- delalloc ---| + * |--- search ---| + */ + test_start = SZ_64M; + locked_page = find_lock_page(inode->i_mapping, + test_start >> PAGE_SHIFT); + if (!locked_page) { + test_err("couldn't find the locked page"); + goto out_bits; + } + btrfs_set_extent_bit(tmp, sectorsize, max_bytes - 1, EXTENT_DELALLOC, NULL); + start = test_start; + end = start + PAGE_SIZE - 1; + found = find_lock_delalloc_range(inode, page_folio(locked_page), &start, + &end); + if (!found) { + test_err("couldn't find delalloc in our range"); + goto out_bits; + } + if (start != test_start || end != max_bytes - 1) { + test_err("expected start %llu end %llu, got start %llu, end %llu", + test_start, max_bytes - 1, start, end); + goto out_bits; + } + if (process_page_range(inode, start, end, + PROCESS_TEST_LOCKED | PROCESS_UNLOCK)) { + test_err("there were unlocked pages in the range"); + goto out_bits; + } + btrfs_unlock_extent(tmp, start, end, NULL); + /* locked_page was unlocked above */ + put_page(locked_page); + + /* + * Test this scenario + * |--- delalloc ---| + * |--- search ---| + */ + test_start = max_bytes + sectorsize; + locked_page = find_lock_page(inode->i_mapping, test_start >> + PAGE_SHIFT); + if (!locked_page) { + test_err("couldn't find the locked page"); + goto out_bits; + } + start = test_start; + end = start + PAGE_SIZE - 1; + found = find_lock_delalloc_range(inode, page_folio(locked_page), &start, + &end); + if (found) { + test_err("found range when we shouldn't have"); + goto out_bits; + } + if (end != test_start + PAGE_SIZE - 1) { + test_err("did not return the proper end offset"); + goto out_bits; + } + + /* + * Test this scenario + * [------- delalloc -------| + * [max_bytes]|-- search--| + * + * We are re-using our test_start from above since it works out well. + */ + btrfs_set_extent_bit(tmp, max_bytes, total_dirty - 1, EXTENT_DELALLOC, NULL); + start = test_start; + end = start + PAGE_SIZE - 1; + found = find_lock_delalloc_range(inode, page_folio(locked_page), &start, + &end); + if (!found) { + test_err("didn't find our range"); + goto out_bits; + } + if (start != test_start || end != total_dirty - 1) { + test_err("expected start %llu end %llu, got start %llu end %llu", + test_start, total_dirty - 1, start, end); + goto out_bits; + } + if (process_page_range(inode, start, end, + PROCESS_TEST_LOCKED | PROCESS_UNLOCK)) { + test_err("pages in range were not all locked"); + goto out_bits; + } + btrfs_unlock_extent(tmp, start, end, NULL); + + /* + * Now to test where we run into a page that is no longer dirty in the + * range we want to find. + */ + page = find_get_page(inode->i_mapping, + (max_bytes + SZ_1M) >> PAGE_SHIFT); + if (!page) { + test_err("couldn't find our page"); + goto out_bits; + } + ClearPageDirty(page); + put_page(page); + + /* We unlocked it in the previous test */ + lock_page(locked_page); + start = test_start; + end = start + PAGE_SIZE - 1; + /* + * Currently if we fail to find dirty pages in the delalloc range we + * will adjust max_bytes down to PAGE_SIZE and then re-search. If + * this changes at any point in the future we will need to fix this + * tests expected behavior. + */ + found = find_lock_delalloc_range(inode, page_folio(locked_page), &start, + &end); + if (!found) { + test_err("didn't find our range"); + goto out_bits; + } + if (start != test_start && end != test_start + PAGE_SIZE - 1) { + test_err("expected start %llu end %llu, got start %llu end %llu", + test_start, test_start + PAGE_SIZE - 1, start, end); + goto out_bits; + } + if (process_page_range(inode, start, end, PROCESS_TEST_LOCKED | + PROCESS_UNLOCK)) { + test_err("pages in range were not all locked"); + goto out_bits; + } + ret = 0; +out_bits: + if (ret) + dump_extent_io_tree(tmp); + btrfs_clear_extent_bit(tmp, 0, total_dirty - 1, (unsigned)-1, NULL); +out: + if (locked_page) + put_page(locked_page); + process_page_range(inode, 0, total_dirty - 1, + PROCESS_UNLOCK | PROCESS_RELEASE); + iput(inode); + btrfs_free_dummy_root(root); + btrfs_free_dummy_fs_info(fs_info); + return ret; +} + +static int check_eb_bitmap(unsigned long *bitmap, struct extent_buffer *eb) +{ + unsigned long i; + + for (i = 0; i < eb->len * BITS_PER_BYTE; i++) { + bool bit_set, bit1_set; + + bit_set = test_bit(i, bitmap); + bit1_set = extent_buffer_test_bit(eb, 0, i); + if (bit1_set != bit_set) { + u8 has; + u8 expect; + + read_extent_buffer(eb, &has, i / BITS_PER_BYTE, 1); + expect = bitmap_get_value8(bitmap, ALIGN(i, BITS_PER_BYTE)); + + test_err( + "bits do not match, start byte 0 bit %lu, byte %lu has 0x%02x expect 0x%02x", + i, i / BITS_PER_BYTE, has, expect); + return -EINVAL; + } + + bit1_set = extent_buffer_test_bit(eb, i / BITS_PER_BYTE, + i % BITS_PER_BYTE); + if (bit1_set != bit_set) { + u8 has; + u8 expect; + + read_extent_buffer(eb, &has, i / BITS_PER_BYTE, 1); + expect = bitmap_get_value8(bitmap, ALIGN(i, BITS_PER_BYTE)); + + test_err( + "bits do not match, start byte %lu bit %lu, byte %lu has 0x%02x expect 0x%02x", + i / BITS_PER_BYTE, i % BITS_PER_BYTE, + i / BITS_PER_BYTE, has, expect); + return -EINVAL; + } + } + return 0; +} + +static int test_bitmap_set(const char *name, unsigned long *bitmap, + struct extent_buffer *eb, + unsigned long byte_start, unsigned long bit_start, + unsigned long bit_len) +{ + int ret; + + bitmap_set(bitmap, byte_start * BITS_PER_BYTE + bit_start, bit_len); + extent_buffer_bitmap_set(eb, byte_start, bit_start, bit_len); + ret = check_eb_bitmap(bitmap, eb); + if (ret < 0) + test_err("%s test failed", name); + return ret; +} + +static int test_bitmap_clear(const char *name, unsigned long *bitmap, + struct extent_buffer *eb, + unsigned long byte_start, unsigned long bit_start, + unsigned long bit_len) +{ + int ret; + + bitmap_clear(bitmap, byte_start * BITS_PER_BYTE + bit_start, bit_len); + extent_buffer_bitmap_clear(eb, byte_start, bit_start, bit_len); + ret = check_eb_bitmap(bitmap, eb); + if (ret < 0) + test_err("%s test failed", name); + return ret; +} +static int __test_eb_bitmaps(unsigned long *bitmap, struct extent_buffer *eb) +{ + unsigned long i, j; + unsigned long byte_len = eb->len; + u32 x; + int ret; + + ret = test_bitmap_clear("clear all run 1", bitmap, eb, 0, 0, + byte_len * BITS_PER_BYTE); + if (ret < 0) + return ret; + + ret = test_bitmap_set("set all", bitmap, eb, 0, 0, byte_len * BITS_PER_BYTE); + if (ret < 0) + return ret; + + ret = test_bitmap_clear("clear all run 2", bitmap, eb, 0, 0, + byte_len * BITS_PER_BYTE); + if (ret < 0) + return ret; + + ret = test_bitmap_set("same byte set", bitmap, eb, 0, 2, 4); + if (ret < 0) + return ret; + + ret = test_bitmap_clear("same byte partial clear", bitmap, eb, 0, 4, 1); + if (ret < 0) + return ret; + + ret = test_bitmap_set("cross byte set", bitmap, eb, 2, 4, 8); + if (ret < 0) + return ret; + + ret = test_bitmap_set("cross multi byte set", bitmap, eb, 4, 4, 24); + if (ret < 0) + return ret; + + ret = test_bitmap_clear("cross byte clear", bitmap, eb, 2, 6, 4); + if (ret < 0) + return ret; + + ret = test_bitmap_clear("cross multi byte clear", bitmap, eb, 4, 6, 20); + if (ret < 0) + return ret; + + /* Straddling pages test */ + if (byte_len > PAGE_SIZE) { + ret = test_bitmap_set("cross page set", bitmap, eb, + PAGE_SIZE - sizeof(long) / 2, 0, + sizeof(long) * BITS_PER_BYTE); + if (ret < 0) + return ret; + + ret = test_bitmap_set("cross page set all", bitmap, eb, 0, 0, + byte_len * BITS_PER_BYTE); + if (ret < 0) + return ret; + + ret = test_bitmap_clear("cross page clear", bitmap, eb, + PAGE_SIZE - sizeof(long) / 2, 0, + sizeof(long) * BITS_PER_BYTE); + if (ret < 0) + return ret; + } + + /* + * Generate a wonky pseudo-random bit pattern for the sake of not using + * something repetitive that could miss some hypothetical off-by-n bug. + */ + x = 0; + ret = test_bitmap_clear("clear all run 3", bitmap, eb, 0, 0, + byte_len * BITS_PER_BYTE); + if (ret < 0) + return ret; + + for (i = 0; i < byte_len * BITS_PER_BYTE / 32; i++) { + x = (0x19660dULL * (u64)x + 0x3c6ef35fULL) & 0xffffffffU; + for (j = 0; j < 32; j++) { + if (x & (1U << j)) { + bitmap_set(bitmap, i * 32 + j, 1); + extent_buffer_bitmap_set(eb, 0, i * 32 + j, 1); + } + } + } + + ret = check_eb_bitmap(bitmap, eb); + if (ret) { + test_err("random bit pattern failed"); + return ret; + } + + return 0; +} + +static int test_eb_bitmaps(u32 sectorsize, u32 nodesize) +{ + struct btrfs_fs_info *fs_info; + unsigned long AUTO_KFREE(bitmap); + struct extent_buffer *eb = NULL; + int ret; + + test_msg("running extent buffer bitmap tests"); + + fs_info = btrfs_alloc_dummy_fs_info(nodesize, sectorsize); + if (!fs_info) { + test_std_err(TEST_ALLOC_FS_INFO); + return -ENOMEM; + } + + bitmap = kmalloc(nodesize, GFP_KERNEL); + if (!bitmap) { + test_err("couldn't allocate test bitmap"); + ret = -ENOMEM; + goto out; + } + + eb = alloc_dummy_extent_buffer(fs_info, 0); + if (!eb) { + test_std_err(TEST_ALLOC_ROOT); + ret = -ENOMEM; + goto out; + } + + ret = __test_eb_bitmaps(bitmap, eb); + if (ret) + goto out; + + free_extent_buffer(eb); + + /* + * Test again for case where the tree block is sectorsize aligned but + * not nodesize aligned. + */ + eb = alloc_dummy_extent_buffer(fs_info, sectorsize); + if (!eb) { + test_std_err(TEST_ALLOC_ROOT); + ret = -ENOMEM; + goto out; + } + + ret = __test_eb_bitmaps(bitmap, eb); +out: + free_extent_buffer(eb); + btrfs_free_dummy_fs_info(fs_info); + return ret; +} + +static int test_find_first_clear_extent_bit(void) +{ + struct extent_io_tree tree; + u64 start, end; + int ret = -EINVAL; + + test_msg("running find_first_clear_extent_bit test"); + + btrfs_extent_io_tree_init(NULL, &tree, IO_TREE_SELFTEST); + + /* Test correct handling of empty tree */ + btrfs_find_first_clear_extent_bit(&tree, 0, &start, &end, CHUNK_TRIMMED); + if (start != 0 || end != -1) { + test_err( + "error getting a range from completely empty tree: start %llu end %llu", + start, end); + goto out; + } + /* + * Set 1M-4M alloc/discard and 32M-64M thus leaving a hole between + * 4M-32M + */ + btrfs_set_extent_bit(&tree, SZ_1M, SZ_4M - 1, + CHUNK_TRIMMED | CHUNK_ALLOCATED, NULL); + + btrfs_find_first_clear_extent_bit(&tree, SZ_512K, &start, &end, + CHUNK_TRIMMED | CHUNK_ALLOCATED); + + if (start != 0 || end != SZ_1M - 1) { + test_err("error finding beginning range: start %llu end %llu", + start, end); + goto out; + } + + /* Now add 32M-64M so that we have a hole between 4M-32M */ + btrfs_set_extent_bit(&tree, SZ_32M, SZ_64M - 1, + CHUNK_TRIMMED | CHUNK_ALLOCATED, NULL); + + /* + * Request first hole starting at 12M, we should get 4M-32M + */ + btrfs_find_first_clear_extent_bit(&tree, 12 * SZ_1M, &start, &end, + CHUNK_TRIMMED | CHUNK_ALLOCATED); + + if (start != SZ_4M || end != SZ_32M - 1) { + test_err("error finding trimmed range: start %llu end %llu", + start, end); + goto out; + } + + /* + * Search in the middle of allocated range, should get the next one + * available, which happens to be unallocated -> 4M-32M + */ + btrfs_find_first_clear_extent_bit(&tree, SZ_2M, &start, &end, + CHUNK_TRIMMED | CHUNK_ALLOCATED); + + if (start != SZ_4M || end != SZ_32M - 1) { + test_err("error finding next unalloc range: start %llu end %llu", + start, end); + goto out; + } + + /* + * Set 64M-72M with CHUNK_ALLOC flag, then search for CHUNK_TRIMMED flag + * being unset in this range, we should get the entry in range 64M-72M + */ + btrfs_set_extent_bit(&tree, SZ_64M, SZ_64M + SZ_8M - 1, CHUNK_ALLOCATED, NULL); + btrfs_find_first_clear_extent_bit(&tree, SZ_64M + SZ_1M, &start, &end, + CHUNK_TRIMMED); + + if (start != SZ_64M || end != SZ_64M + SZ_8M - 1) { + test_err("error finding exact range: start %llu end %llu", + start, end); + goto out; + } + + btrfs_find_first_clear_extent_bit(&tree, SZ_64M - SZ_8M, &start, &end, + CHUNK_TRIMMED); + + /* + * Search in the middle of set range whose immediate neighbour doesn't + * have the bits set so it must be returned + */ + if (start != SZ_64M || end != SZ_64M + SZ_8M - 1) { + test_err("error finding next alloc range: start %llu end %llu", + start, end); + goto out; + } + + /* + * Search beyond any known range, shall return after last known range + * and end should be -1 + */ + btrfs_find_first_clear_extent_bit(&tree, -1, &start, &end, CHUNK_TRIMMED); + if (start != SZ_64M + SZ_8M || end != -1) { + test_err( + "error handling beyond end of range search: start %llu end %llu", + start, end); + goto out; + } + + ret = 0; +out: + if (ret) + dump_extent_io_tree(&tree); + btrfs_clear_extent_bit(&tree, 0, (u64)-1, CHUNK_TRIMMED | CHUNK_ALLOCATED, NULL); + + return ret; +} + +static void dump_eb_and_memory_contents(struct extent_buffer *eb, void *memory, + const char *test_name) +{ + for (int i = 0; i < eb->len; i++) { + struct page *page = folio_page(eb->folios[i >> PAGE_SHIFT], 0); + void *addr = page_address(page) + offset_in_page(i); + + if (memcmp(addr, memory + i, 1) != 0) { + test_err("%s failed", test_name); + test_err("eb and memory diffs at byte %u, eb has 0x%02x memory has 0x%02x", + i, *(u8 *)addr, *(u8 *)(memory + i)); + return; + } + } +} + +static int verify_eb_and_memory(struct extent_buffer *eb, void *memory, + const char *test_name) +{ + for (int i = 0; i < (eb->len >> PAGE_SHIFT); i++) { + void *eb_addr = folio_address(eb->folios[i]); + + if (memcmp(memory + (i << PAGE_SHIFT), eb_addr, PAGE_SIZE) != 0) { + dump_eb_and_memory_contents(eb, memory, test_name); + return -EUCLEAN; + } + } + return 0; +} + +/* + * Init both memory and extent buffer contents to the same randomly generated + * contents. + */ +static void init_eb_and_memory(struct extent_buffer *eb, void *memory) +{ + get_random_bytes(memory, eb->len); + write_extent_buffer(eb, memory, 0, eb->len); +} + +static int test_eb_mem_ops(u32 sectorsize, u32 nodesize) +{ + struct btrfs_fs_info *fs_info; + struct extent_buffer *eb = NULL; + void *memory = NULL; + int ret; + + test_msg("running extent buffer memory operation tests"); + + fs_info = btrfs_alloc_dummy_fs_info(nodesize, sectorsize); + if (!fs_info) { + test_std_err(TEST_ALLOC_FS_INFO); + return -ENOMEM; + } + + memory = kvzalloc(nodesize, GFP_KERNEL); + if (!memory) { + test_err("failed to allocate memory"); + ret = -ENOMEM; + goto out; + } + + eb = alloc_dummy_extent_buffer(fs_info, SZ_1M); + if (!eb) { + test_std_err(TEST_ALLOC_EXTENT_BUFFER); + ret = -ENOMEM; + goto out; + } + + init_eb_and_memory(eb, memory); + ret = verify_eb_and_memory(eb, memory, "full eb write"); + if (ret < 0) + goto out; + + memcpy(memory, memory + 16, 16); + memcpy_extent_buffer(eb, 0, 16, 16); + ret = verify_eb_and_memory(eb, memory, "same page non-overlapping memcpy 1"); + if (ret < 0) + goto out; + + memcpy(memory, memory + 2048, 16); + memcpy_extent_buffer(eb, 0, 2048, 16); + ret = verify_eb_and_memory(eb, memory, "same page non-overlapping memcpy 2"); + if (ret < 0) + goto out; + memcpy(memory, memory + 2048, 2048); + memcpy_extent_buffer(eb, 0, 2048, 2048); + ret = verify_eb_and_memory(eb, memory, "same page non-overlapping memcpy 3"); + if (ret < 0) + goto out; + + memmove(memory + 512, memory + 256, 512); + memmove_extent_buffer(eb, 512, 256, 512); + ret = verify_eb_and_memory(eb, memory, "same page overlapping memcpy 1"); + if (ret < 0) + goto out; + + memmove(memory + 2048, memory + 512, 2048); + memmove_extent_buffer(eb, 2048, 512, 2048); + ret = verify_eb_and_memory(eb, memory, "same page overlapping memcpy 2"); + if (ret < 0) + goto out; + memmove(memory + 512, memory + 2048, 2048); + memmove_extent_buffer(eb, 512, 2048, 2048); + ret = verify_eb_and_memory(eb, memory, "same page overlapping memcpy 3"); + if (ret < 0) + goto out; + + if (nodesize > PAGE_SIZE) { + memcpy(memory, memory + 4096 - 128, 256); + memcpy_extent_buffer(eb, 0, 4096 - 128, 256); + ret = verify_eb_and_memory(eb, memory, "cross page non-overlapping memcpy 1"); + if (ret < 0) + goto out; + + memcpy(memory + 4096 - 128, memory + 4096 + 128, 256); + memcpy_extent_buffer(eb, 4096 - 128, 4096 + 128, 256); + ret = verify_eb_and_memory(eb, memory, "cross page non-overlapping memcpy 2"); + if (ret < 0) + goto out; + + memmove(memory + 4096 - 128, memory + 4096 - 64, 256); + memmove_extent_buffer(eb, 4096 - 128, 4096 - 64, 256); + ret = verify_eb_and_memory(eb, memory, "cross page overlapping memcpy 1"); + if (ret < 0) + goto out; + + memmove(memory + 4096 - 64, memory + 4096 - 128, 256); + memmove_extent_buffer(eb, 4096 - 64, 4096 - 128, 256); + ret = verify_eb_and_memory(eb, memory, "cross page overlapping memcpy 2"); + if (ret < 0) + goto out; + } +out: + free_extent_buffer(eb); + kvfree(memory); + btrfs_free_dummy_fs_info(fs_info); + return ret; +} + +int btrfs_test_extent_io(u32 sectorsize, u32 nodesize) +{ + int ret; + + test_msg("running extent I/O tests"); + + ret = test_find_delalloc(sectorsize, nodesize); + if (ret) + goto out; + + ret = test_find_first_clear_extent_bit(); + if (ret) + goto out; + + ret = test_eb_bitmaps(sectorsize, nodesize); + if (ret) + goto out; + + ret = test_eb_mem_ops(sectorsize, nodesize); +out: + return ret; +} diff --git a/fs/btrfs/tests/extent-map-tests.c b/fs/btrfs/tests/extent-map-tests.c new file mode 100644 index 000000000..811f36d41 --- /dev/null +++ b/fs/btrfs/tests/extent-map-tests.c @@ -0,0 +1,1202 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2017 Oracle. All rights reserved. + */ + +#include +#include "btrfs-tests.h" +#include "../ctree.h" +#include "../btrfs_inode.h" +#include "../volumes.h" +#include "../disk-io.h" +#include "../block-group.h" + +static int free_extent_map_tree(struct btrfs_inode *inode) +{ + struct extent_map_tree *em_tree = &inode->extent_tree; + struct extent_map *em; + struct rb_node *node; + int ret = 0; + + write_lock(&em_tree->lock); + while (!RB_EMPTY_ROOT(&em_tree->root)) { + node = rb_first(&em_tree->root); + em = rb_entry(node, struct extent_map, rb_node); + btrfs_remove_extent_mapping(inode, em); + +#ifdef CONFIG_BTRFS_DEBUG + if (refcount_read(&em->refs) != 1) { + ret = -EINVAL; + test_err( +"em leak: em (start %llu len %llu disk_bytenr %llu disk_num_bytes %llu offset %llu) refs %d", + em->start, em->len, em->disk_bytenr, + em->disk_num_bytes, em->offset, + refcount_read(&em->refs)); + + refcount_set(&em->refs, 1); + } +#endif + btrfs_free_extent_map(em); + } + write_unlock(&em_tree->lock); + + return ret; +} + +/* + * Test scenario: + * + * Suppose that no extent map has been loaded into memory yet, there is a file + * extent [0, 16K), followed by another file extent [16K, 20K), two dio reads + * are entering btrfs_get_extent() concurrently, t1 is reading [8K, 16K), t2 is + * reading [0, 8K) + * + * t1 t2 + * btrfs_get_extent() btrfs_get_extent() + * -> lookup_extent_mapping() ->lookup_extent_mapping() + * -> add_extent_mapping(0, 16K) + * -> return em + * ->add_extent_mapping(0, 16K) + * -> #handle -EEXIST + */ +static int test_case_1(struct btrfs_fs_info *fs_info, struct btrfs_inode *inode) +{ + struct extent_map_tree *em_tree = &inode->extent_tree; + struct extent_map *em; + u64 start = 0; + u64 len = SZ_8K; + int ret; + int ret2; + + em = btrfs_alloc_extent_map(); + if (!em) { + test_std_err(TEST_ALLOC_EXTENT_MAP); + return -ENOMEM; + } + + /* Add [0, 16K) */ + em->start = 0; + em->len = SZ_16K; + em->disk_bytenr = 0; + em->disk_num_bytes = SZ_16K; + em->ram_bytes = SZ_16K; + write_lock(&em_tree->lock); + ret = btrfs_add_extent_mapping(inode, &em, em->start, em->len); + write_unlock(&em_tree->lock); + if (ret < 0) { + test_err("cannot add extent range [0, 16K)"); + goto out; + } + btrfs_free_extent_map(em); + + /* Add [16K, 20K) following [0, 16K) */ + em = btrfs_alloc_extent_map(); + if (!em) { + test_std_err(TEST_ALLOC_EXTENT_MAP); + ret = -ENOMEM; + goto out; + } + + em->start = SZ_16K; + em->len = SZ_4K; + em->disk_bytenr = SZ_32K; /* avoid merging */ + em->disk_num_bytes = SZ_4K; + em->ram_bytes = SZ_4K; + write_lock(&em_tree->lock); + ret = btrfs_add_extent_mapping(inode, &em, em->start, em->len); + write_unlock(&em_tree->lock); + if (ret < 0) { + test_err("cannot add extent range [16K, 20K)"); + goto out; + } + btrfs_free_extent_map(em); + + em = btrfs_alloc_extent_map(); + if (!em) { + test_std_err(TEST_ALLOC_EXTENT_MAP); + ret = -ENOMEM; + goto out; + } + + /* Add [0, 8K), should return [0, 16K) instead. */ + em->start = start; + em->len = len; + em->disk_bytenr = start; + em->disk_num_bytes = len; + em->ram_bytes = len; + write_lock(&em_tree->lock); + ret = btrfs_add_extent_mapping(inode, &em, em->start, em->len); + write_unlock(&em_tree->lock); + if (ret) { + test_err("case1 [%llu %llu]: ret %d", start, start + len, ret); + goto out; + } + if (!em) { + test_err("case1 [%llu %llu]: no extent map returned", + start, start + len); + ret = -ENOENT; + goto out; + } + if (em->start != 0 || btrfs_extent_map_end(em) != SZ_16K || + em->disk_bytenr != 0 || em->disk_num_bytes != SZ_16K) { + test_err( +"case1 [%llu %llu]: ret %d return a wrong em (start %llu len %llu disk_bytenr %llu disk_num_bytes %llu", + start, start + len, ret, em->start, em->len, + em->disk_bytenr, em->disk_num_bytes); + ret = -EINVAL; + } + btrfs_free_extent_map(em); +out: + ret2 = free_extent_map_tree(inode); + if (ret == 0) + ret = ret2; + + return ret; +} + +/* + * Test scenario: + * + * Reading the inline ending up with EEXIST, ie. read an inline + * extent and discard page cache and read it again. + */ +static int test_case_2(struct btrfs_fs_info *fs_info, struct btrfs_inode *inode) +{ + struct extent_map_tree *em_tree = &inode->extent_tree; + struct extent_map *em; + int ret; + int ret2; + + em = btrfs_alloc_extent_map(); + if (!em) { + test_std_err(TEST_ALLOC_EXTENT_MAP); + return -ENOMEM; + } + + /* + * Add [0, 1K) which is inlined. And the extent map length must + * be one block. + */ + em->start = 0; + em->len = SZ_4K; + em->disk_bytenr = EXTENT_MAP_INLINE; + em->disk_num_bytes = 0; + em->ram_bytes = SZ_1K; + write_lock(&em_tree->lock); + ret = btrfs_add_extent_mapping(inode, &em, em->start, em->len); + write_unlock(&em_tree->lock); + if (ret < 0) { + test_err("cannot add extent range [0, 1K)"); + goto out; + } + btrfs_free_extent_map(em); + + /* Add [4K, 8K) following [0, 1K) */ + em = btrfs_alloc_extent_map(); + if (!em) { + test_std_err(TEST_ALLOC_EXTENT_MAP); + ret = -ENOMEM; + goto out; + } + + em->start = SZ_4K; + em->len = SZ_4K; + em->disk_bytenr = SZ_4K; + em->disk_num_bytes = SZ_4K; + em->ram_bytes = SZ_4K; + write_lock(&em_tree->lock); + ret = btrfs_add_extent_mapping(inode, &em, em->start, em->len); + write_unlock(&em_tree->lock); + if (ret < 0) { + test_err("cannot add extent range [4K, 8K)"); + goto out; + } + btrfs_free_extent_map(em); + + em = btrfs_alloc_extent_map(); + if (!em) { + test_std_err(TEST_ALLOC_EXTENT_MAP); + ret = -ENOMEM; + goto out; + } + + /* Add [0, 1K) */ + em->start = 0; + em->len = SZ_4K; + em->disk_bytenr = EXTENT_MAP_INLINE; + em->disk_num_bytes = 0; + em->ram_bytes = SZ_1K; + write_lock(&em_tree->lock); + ret = btrfs_add_extent_mapping(inode, &em, em->start, em->len); + write_unlock(&em_tree->lock); + if (ret) { + test_err("case2 [0 1K]: ret %d", ret); + goto out; + } + if (!em) { + test_err("case2 [0 1K]: no extent map returned"); + ret = -ENOENT; + goto out; + } + if (em->start != 0 || btrfs_extent_map_end(em) != SZ_4K || + em->disk_bytenr != EXTENT_MAP_INLINE) { + test_err( +"case2 [0 1K]: ret %d return a wrong em (start %llu len %llu disk_bytenr %llu", + ret, em->start, em->len, em->disk_bytenr); + ret = -EINVAL; + } + btrfs_free_extent_map(em); +out: + ret2 = free_extent_map_tree(inode); + if (ret == 0) + ret = ret2; + + return ret; +} + +static int __test_case_3(struct btrfs_fs_info *fs_info, + struct btrfs_inode *inode, u64 start) +{ + struct extent_map_tree *em_tree = &inode->extent_tree; + struct extent_map *em; + u64 len = SZ_4K; + int ret; + int ret2; + + em = btrfs_alloc_extent_map(); + if (!em) { + test_std_err(TEST_ALLOC_EXTENT_MAP); + return -ENOMEM; + } + + /* Add [4K, 8K) */ + em->start = SZ_4K; + em->len = SZ_4K; + em->disk_bytenr = SZ_4K; + em->disk_num_bytes = SZ_4K; + em->ram_bytes = SZ_4K; + write_lock(&em_tree->lock); + ret = btrfs_add_extent_mapping(inode, &em, em->start, em->len); + write_unlock(&em_tree->lock); + if (ret < 0) { + test_err("cannot add extent range [4K, 8K)"); + goto out; + } + btrfs_free_extent_map(em); + + em = btrfs_alloc_extent_map(); + if (!em) { + test_std_err(TEST_ALLOC_EXTENT_MAP); + ret = -ENOMEM; + goto out; + } + + /* Add [0, 16K) */ + em->start = 0; + em->len = SZ_16K; + em->disk_bytenr = 0; + em->disk_num_bytes = SZ_16K; + em->ram_bytes = SZ_16K; + write_lock(&em_tree->lock); + ret = btrfs_add_extent_mapping(inode, &em, start, len); + write_unlock(&em_tree->lock); + if (ret) { + test_err("case3 [%llu %llu): ret %d", + start, start + len, ret); + goto out; + } + if (!em) { + test_err("case3 [%llu %llu): no extent map returned", + start, start + len); + ret = -ENOENT; + goto out; + } + /* + * Since bytes within em are contiguous, em->block_start is identical to + * em->start. + */ + if (start < em->start || start + len > btrfs_extent_map_end(em) || + em->start != btrfs_extent_map_block_start(em)) { + test_err( +"case3 [%llu %llu): ret %d em (start %llu len %llu disk_bytenr %llu block_len %llu)", + start, start + len, ret, em->start, em->len, + em->disk_bytenr, em->disk_num_bytes); + ret = -EINVAL; + } + btrfs_free_extent_map(em); +out: + ret2 = free_extent_map_tree(inode); + if (ret == 0) + ret = ret2; + + return ret; +} + +/* + * Test scenario: + * + * Suppose that no extent map has been loaded into memory yet. + * There is a file extent [0, 16K), two jobs are running concurrently + * against it, t1 is buffered writing to [4K, 8K) and t2 is doing dio + * read from [0, 4K) or [8K, 12K) or [12K, 16K). + * + * t1 goes ahead of t2 and adds em [4K, 8K) into tree. + * + * t1 t2 + * cow_file_range() btrfs_get_extent() + * -> lookup_extent_mapping() + * -> add_extent_mapping() + * -> add_extent_mapping() + */ +static int test_case_3(struct btrfs_fs_info *fs_info, struct btrfs_inode *inode) +{ + int ret; + + ret = __test_case_3(fs_info, inode, 0); + if (ret) + return ret; + ret = __test_case_3(fs_info, inode, SZ_8K); + if (ret) + return ret; + ret = __test_case_3(fs_info, inode, (12 * SZ_1K)); + + return ret; +} + +static int __test_case_4(struct btrfs_fs_info *fs_info, + struct btrfs_inode *inode, u64 start) +{ + struct extent_map_tree *em_tree = &inode->extent_tree; + struct extent_map *em; + u64 len = SZ_4K; + int ret; + int ret2; + + em = btrfs_alloc_extent_map(); + if (!em) { + test_std_err(TEST_ALLOC_EXTENT_MAP); + return -ENOMEM; + } + + /* Add [0K, 8K) */ + em->start = 0; + em->len = SZ_8K; + em->disk_bytenr = 0; + em->disk_num_bytes = SZ_8K; + em->ram_bytes = SZ_8K; + write_lock(&em_tree->lock); + ret = btrfs_add_extent_mapping(inode, &em, em->start, em->len); + write_unlock(&em_tree->lock); + if (ret < 0) { + test_err("cannot add extent range [0, 8K)"); + goto out; + } + btrfs_free_extent_map(em); + + em = btrfs_alloc_extent_map(); + if (!em) { + test_std_err(TEST_ALLOC_EXTENT_MAP); + ret = -ENOMEM; + goto out; + } + + /* Add [8K, 32K) */ + em->start = SZ_8K; + em->len = 24 * SZ_1K; + em->disk_bytenr = SZ_16K; /* avoid merging */ + em->disk_num_bytes = 24 * SZ_1K; + em->ram_bytes = 24 * SZ_1K; + write_lock(&em_tree->lock); + ret = btrfs_add_extent_mapping(inode, &em, em->start, em->len); + write_unlock(&em_tree->lock); + if (ret < 0) { + test_err("cannot add extent range [8K, 32K)"); + goto out; + } + btrfs_free_extent_map(em); + + em = btrfs_alloc_extent_map(); + if (!em) { + test_std_err(TEST_ALLOC_EXTENT_MAP); + ret = -ENOMEM; + goto out; + } + /* Add [0K, 32K) */ + em->start = 0; + em->len = SZ_32K; + em->disk_bytenr = 0; + em->disk_num_bytes = SZ_32K; + em->ram_bytes = SZ_32K; + write_lock(&em_tree->lock); + ret = btrfs_add_extent_mapping(inode, &em, start, len); + write_unlock(&em_tree->lock); + if (ret) { + test_err("case4 [%llu %llu): ret %d", + start, start + len, ret); + goto out; + } + if (!em) { + test_err("case4 [%llu %llu): no extent map returned", + start, start + len); + ret = -ENOENT; + goto out; + } + if (start < em->start || start + len > btrfs_extent_map_end(em)) { + test_err( +"case4 [%llu %llu): ret %d, added wrong em (start %llu len %llu disk_bytenr %llu disk_num_bytes %llu)", + start, start + len, ret, em->start, em->len, + em->disk_bytenr, em->disk_num_bytes); + ret = -EINVAL; + } + btrfs_free_extent_map(em); +out: + ret2 = free_extent_map_tree(inode); + if (ret == 0) + ret = ret2; + + return ret; +} + +/* + * Test scenario: + * + * Suppose that no extent map has been loaded into memory yet. + * There is a file extent [0, 32K), two jobs are running concurrently + * against it, t1 is doing dio write to [8K, 32K) and t2 is doing dio + * read from [0, 4K) or [4K, 8K). + * + * t1 goes ahead of t2 and splits em [0, 32K) to em [0K, 8K) and [8K 32K). + * + * t1 t2 + * btrfs_get_blocks_direct() btrfs_get_blocks_direct() + * -> btrfs_get_extent() -> btrfs_get_extent() + * -> lookup_extent_mapping() + * -> add_extent_mapping() -> lookup_extent_mapping() + * # load [0, 32K) + * -> btrfs_new_extent_direct() + * -> btrfs_drop_extent_cache() + * # split [0, 32K) + * -> add_extent_mapping() + * # add [8K, 32K) + * -> add_extent_mapping() + * # handle -EEXIST when adding + * # [0, 32K) + */ +static int test_case_4(struct btrfs_fs_info *fs_info, struct btrfs_inode *inode) +{ + int ret; + + ret = __test_case_4(fs_info, inode, 0); + if (ret) + return ret; + ret = __test_case_4(fs_info, inode, SZ_4K); + + return ret; +} + +static int add_compressed_extent(struct btrfs_inode *inode, + u64 start, u64 len, u64 block_start) +{ + struct extent_map_tree *em_tree = &inode->extent_tree; + struct extent_map *em; + int ret; + + em = btrfs_alloc_extent_map(); + if (!em) { + test_std_err(TEST_ALLOC_EXTENT_MAP); + return -ENOMEM; + } + + em->start = start; + em->len = len; + em->disk_bytenr = block_start; + em->disk_num_bytes = SZ_4K; + em->ram_bytes = len; + em->flags |= EXTENT_FLAG_COMPRESS_ZLIB; + write_lock(&em_tree->lock); + ret = btrfs_add_extent_mapping(inode, &em, em->start, em->len); + write_unlock(&em_tree->lock); + btrfs_free_extent_map(em); + if (ret < 0) { + test_err("cannot add extent map [%llu, %llu)", start, start + len); + return ret; + } + + return 0; +} + +struct extent_range { + u64 start; + u64 len; +}; + +/* The valid states of the tree after every drop, as described below. */ +struct extent_range valid_ranges[][7] = { + { + { .start = 0, .len = SZ_8K }, /* [0, 8K) */ + { .start = SZ_4K * 3, .len = SZ_4K * 3}, /* [12k, 24k) */ + { .start = SZ_4K * 6, .len = SZ_4K * 3}, /* [24k, 36k) */ + { .start = SZ_32K + SZ_4K, .len = SZ_4K}, /* [36k, 40k) */ + { .start = SZ_4K * 10, .len = SZ_4K * 6}, /* [40k, 64k) */ + }, + { + { .start = 0, .len = SZ_8K }, /* [0, 8K) */ + { .start = SZ_4K * 5, .len = SZ_4K}, /* [20k, 24k) */ + { .start = SZ_4K * 6, .len = SZ_4K * 3}, /* [24k, 36k) */ + { .start = SZ_32K + SZ_4K, .len = SZ_4K}, /* [36k, 40k) */ + { .start = SZ_4K * 10, .len = SZ_4K * 6}, /* [40k, 64k) */ + }, + { + { .start = 0, .len = SZ_8K }, /* [0, 8K) */ + { .start = SZ_4K * 5, .len = SZ_4K}, /* [20k, 24k) */ + { .start = SZ_4K * 6, .len = SZ_4K}, /* [24k, 28k) */ + { .start = SZ_32K, .len = SZ_4K}, /* [32k, 36k) */ + { .start = SZ_32K + SZ_4K, .len = SZ_4K}, /* [36k, 40k) */ + { .start = SZ_4K * 10, .len = SZ_4K * 6}, /* [40k, 64k) */ + }, + { + { .start = 0, .len = SZ_8K}, /* [0, 8K) */ + { .start = SZ_4K * 5, .len = SZ_4K}, /* [20k, 24k) */ + { .start = SZ_4K * 6, .len = SZ_4K}, /* [24k, 28k) */ + } +}; + +static int validate_range(struct extent_map_tree *em_tree, int index) +{ + struct rb_node *n; + int i; + + for (i = 0, n = rb_first(&em_tree->root); + valid_ranges[index][i].len && n; + i++, n = rb_next(n)) { + struct extent_map *entry = rb_entry(n, struct extent_map, rb_node); + + if (entry->start != valid_ranges[index][i].start) { + test_err("mapping has start %llu expected %llu", + entry->start, valid_ranges[index][i].start); + return -EINVAL; + } + + if (entry->len != valid_ranges[index][i].len) { + test_err("mapping has len %llu expected %llu", + entry->len, valid_ranges[index][i].len); + return -EINVAL; + } + } + + /* + * We exited because we don't have any more entries in the extent_map + * but we still expect more valid entries. + */ + if (valid_ranges[index][i].len) { + test_err("missing an entry"); + return -EINVAL; + } + + /* We exited the loop but still have entries in the extent map. */ + if (n) { + test_err("we have a left over entry in the extent map we didn't expect"); + return -EINVAL; + } + + return 0; +} + +/* + * Test scenario: + * + * Test the various edge cases of btrfs_drop_extent_map_range, create the + * following ranges + * + * [0, 12k)[12k, 24k)[24k, 36k)[36k, 40k)[40k,64k) + * + * And then we'll drop: + * + * [8k, 12k) - test the single front split + * [12k, 20k) - test the single back split + * [28k, 32k) - test the double split + * [32k, 64k) - test whole em dropping + * + * They'll have the EXTENT_FLAG_COMPRESSED flag set to keep the em tree from + * merging the em's. + */ +static int test_case_5(struct btrfs_fs_info *fs_info, struct btrfs_inode *inode) +{ + u64 start, end; + int ret; + int ret2; + + test_msg("Running btrfs_drop_extent_map_range tests"); + + /* [0, 12k) */ + ret = add_compressed_extent(inode, 0, SZ_4K * 3, 0); + if (ret) { + test_err("cannot add extent range [0, 12K)"); + goto out; + } + + /* [12k, 24k) */ + ret = add_compressed_extent(inode, SZ_4K * 3, SZ_4K * 3, SZ_4K); + if (ret) { + test_err("cannot add extent range [12k, 24k)"); + goto out; + } + + /* [24k, 36k) */ + ret = add_compressed_extent(inode, SZ_4K * 6, SZ_4K * 3, SZ_8K); + if (ret) { + test_err("cannot add extent range [12k, 24k)"); + goto out; + } + + /* [36k, 40k) */ + ret = add_compressed_extent(inode, SZ_32K + SZ_4K, SZ_4K, SZ_4K * 3); + if (ret) { + test_err("cannot add extent range [12k, 24k)"); + goto out; + } + + /* [40k, 64k) */ + ret = add_compressed_extent(inode, SZ_4K * 10, SZ_4K * 6, SZ_16K); + if (ret) { + test_err("cannot add extent range [12k, 24k)"); + goto out; + } + + /* Drop [8k, 12k) */ + start = SZ_8K; + end = (3 * SZ_4K) - 1; + btrfs_drop_extent_map_range(inode, start, end, false); + ret = validate_range(&inode->extent_tree, 0); + if (ret) + goto out; + + /* Drop [12k, 20k) */ + start = SZ_4K * 3; + end = SZ_16K + SZ_4K - 1; + btrfs_drop_extent_map_range(inode, start, end, false); + ret = validate_range(&inode->extent_tree, 1); + if (ret) + goto out; + + /* Drop [28k, 32k) */ + start = SZ_32K - SZ_4K; + end = SZ_32K - 1; + btrfs_drop_extent_map_range(inode, start, end, false); + ret = validate_range(&inode->extent_tree, 2); + if (ret) + goto out; + + /* Drop [32k, 64k) */ + start = SZ_32K; + end = SZ_64K - 1; + btrfs_drop_extent_map_range(inode, start, end, false); + ret = validate_range(&inode->extent_tree, 3); + if (ret) + goto out; +out: + ret2 = free_extent_map_tree(inode); + if (ret == 0) + ret = ret2; + + return ret; +} + +/* + * Test the btrfs_add_extent_mapping helper which will attempt to create an em + * for areas between two existing ems. Validate it doesn't do this when there + * are two unmerged em's side by side. + */ +static int test_case_6(struct btrfs_fs_info *fs_info, struct btrfs_inode *inode) +{ + struct extent_map_tree *em_tree = &inode->extent_tree; + struct extent_map *em = NULL; + int ret; + int ret2; + + ret = add_compressed_extent(inode, 0, SZ_4K, 0); + if (ret) + goto out; + + ret = add_compressed_extent(inode, SZ_4K, SZ_4K, 0); + if (ret) + goto out; + + em = btrfs_alloc_extent_map(); + if (!em) { + test_std_err(TEST_ALLOC_EXTENT_MAP); + ret = -ENOMEM; + goto out; + } + + em->start = SZ_4K; + em->len = SZ_4K; + em->disk_bytenr = SZ_16K; + em->disk_num_bytes = SZ_16K; + em->ram_bytes = SZ_16K; + write_lock(&em_tree->lock); + ret = btrfs_add_extent_mapping(inode, &em, 0, SZ_8K); + write_unlock(&em_tree->lock); + + if (ret != 0) { + test_err("got an error when adding our em: %d", ret); + goto out; + } + + ret = -EINVAL; + if (em->start != 0) { + test_err("unexpected em->start at %llu, wanted 0", em->start); + goto out; + } + if (em->len != SZ_4K) { + test_err("unexpected em->len %llu, expected 4K", em->len); + goto out; + } + ret = 0; +out: + btrfs_free_extent_map(em); + ret2 = free_extent_map_tree(inode); + if (ret == 0) + ret = ret2; + + return ret; +} + +/* + * Regression test for btrfs_drop_extent_map_range. Calling with skip_pinned == + * true would mess up the start/end calculations and subsequent splits would be + * incorrect. + */ +static int test_case_7(struct btrfs_fs_info *fs_info, struct btrfs_inode *inode) +{ + struct extent_map_tree *em_tree = &inode->extent_tree; + struct extent_map *em; + int ret; + int ret2; + + test_msg("Running btrfs_drop_extent_cache with pinned"); + + em = btrfs_alloc_extent_map(); + if (!em) { + test_std_err(TEST_ALLOC_EXTENT_MAP); + return -ENOMEM; + } + + /* [0, 16K), pinned */ + em->start = 0; + em->len = SZ_16K; + em->disk_bytenr = 0; + em->disk_num_bytes = SZ_4K; + em->ram_bytes = SZ_16K; + em->flags |= (EXTENT_FLAG_PINNED | EXTENT_FLAG_COMPRESS_ZLIB); + write_lock(&em_tree->lock); + ret = btrfs_add_extent_mapping(inode, &em, em->start, em->len); + write_unlock(&em_tree->lock); + if (ret < 0) { + test_err("couldn't add extent map"); + goto out; + } + btrfs_free_extent_map(em); + + em = btrfs_alloc_extent_map(); + if (!em) { + test_std_err(TEST_ALLOC_EXTENT_MAP); + ret = -ENOMEM; + goto out; + } + + /* [32K, 48K), not pinned */ + em->start = SZ_32K; + em->len = SZ_16K; + em->disk_bytenr = SZ_32K; + em->disk_num_bytes = SZ_16K; + em->ram_bytes = SZ_16K; + write_lock(&em_tree->lock); + ret = btrfs_add_extent_mapping(inode, &em, em->start, em->len); + write_unlock(&em_tree->lock); + if (ret < 0) { + test_err("couldn't add extent map"); + goto out; + } + btrfs_free_extent_map(em); + + /* + * Drop [0, 36K) This should skip the [0, 4K) extent and then split the + * [32K, 48K) extent. + */ + btrfs_drop_extent_map_range(inode, 0, (36 * SZ_1K) - 1, true); + + /* Make sure our extent maps look sane. */ + ret = -EINVAL; + + em = btrfs_lookup_extent_mapping(em_tree, 0, SZ_16K); + if (!em) { + test_err("didn't find an em at 0 as expected"); + goto out; + } + + if (em->start != 0) { + test_err("em->start is %llu, expected 0", em->start); + goto out; + } + + if (em->len != SZ_16K) { + test_err("em->len is %llu, expected 16K", em->len); + goto out; + } + + btrfs_free_extent_map(em); + + read_lock(&em_tree->lock); + em = btrfs_lookup_extent_mapping(em_tree, SZ_16K, SZ_16K); + read_unlock(&em_tree->lock); + if (em) { + test_err("found an em when we weren't expecting one"); + goto out; + } + + read_lock(&em_tree->lock); + em = btrfs_lookup_extent_mapping(em_tree, SZ_32K, SZ_16K); + read_unlock(&em_tree->lock); + if (!em) { + test_err("didn't find an em at 32K as expected"); + goto out; + } + + if (em->start != (36 * SZ_1K)) { + test_err("em->start is %llu, expected 36K", em->start); + goto out; + } + + if (em->len != (12 * SZ_1K)) { + test_err("em->len is %llu, expected 12K", em->len); + goto out; + } + + if (btrfs_extent_map_block_start(em) != SZ_32K + SZ_4K) { + test_err("em->block_start is %llu, expected 36K", + btrfs_extent_map_block_start(em)); + goto out; + } + + btrfs_free_extent_map(em); + + read_lock(&em_tree->lock); + em = btrfs_lookup_extent_mapping(em_tree, 48 * SZ_1K, (u64)-1); + read_unlock(&em_tree->lock); + if (em) { + test_err("found an unexpected em above 48K"); + goto out; + } + + ret = 0; +out: + btrfs_free_extent_map(em); + /* Unpin our extent to prevent warning when removing it below. */ + ret2 = btrfs_unpin_extent_cache(inode, 0, SZ_16K, 0); + if (ret == 0) + ret = ret2; + ret2 = free_extent_map_tree(inode); + if (ret == 0) + ret = ret2; + + return ret; +} + +/* + * Test a regression for compressed extent map adjustment when we attempt to + * add an extent map that is partially overlapped by another existing extent + * map. The resulting extent map offset was left unchanged despite having + * incremented its start offset. + */ +static int test_case_8(struct btrfs_fs_info *fs_info, struct btrfs_inode *inode) +{ + struct extent_map_tree *em_tree = &inode->extent_tree; + struct extent_map *em; + int ret; + int ret2; + + em = btrfs_alloc_extent_map(); + if (!em) { + test_std_err(TEST_ALLOC_EXTENT_MAP); + return -ENOMEM; + } + + /* Compressed extent for the file range [120K, 128K). */ + em->start = SZ_1K * 120; + em->len = SZ_8K; + em->disk_num_bytes = SZ_4K; + em->ram_bytes = SZ_8K; + em->flags |= EXTENT_FLAG_COMPRESS_ZLIB; + write_lock(&em_tree->lock); + ret = btrfs_add_extent_mapping(inode, &em, em->start, em->len); + write_unlock(&em_tree->lock); + btrfs_free_extent_map(em); + if (ret < 0) { + test_err("couldn't add extent map for range [120K, 128K)"); + goto out; + } + + em = btrfs_alloc_extent_map(); + if (!em) { + test_std_err(TEST_ALLOC_EXTENT_MAP); + ret = -ENOMEM; + goto out; + } + + /* + * Compressed extent for the file range [108K, 144K), which overlaps + * with the [120K, 128K) we previously inserted. + */ + em->start = SZ_1K * 108; + em->len = SZ_1K * 36; + em->disk_num_bytes = SZ_4K; + em->ram_bytes = SZ_1K * 36; + em->flags |= EXTENT_FLAG_COMPRESS_ZLIB; + + /* + * Try to add the extent map but with a search range of [140K, 144K), + * this should succeed and adjust the extent map to the range + * [128K, 144K), with a length of 16K and an offset of 20K. + * + * This simulates a scenario where in the subvolume tree of an inode we + * have a compressed file extent item for the range [108K, 144K) and we + * have an overlapping compressed extent map for the range [120K, 128K), + * which was created by an encoded write, but its ordered extent was not + * yet completed, so the subvolume tree doesn't have yet the file extent + * item for that range - we only have the extent map in the inode's + * extent map tree. + */ + write_lock(&em_tree->lock); + ret = btrfs_add_extent_mapping(inode, &em, SZ_1K * 140, SZ_4K); + write_unlock(&em_tree->lock); + btrfs_free_extent_map(em); + if (ret < 0) { + test_err("couldn't add extent map for range [108K, 144K)"); + goto out; + } + + if (em->start != SZ_128K) { + test_err("unexpected extent map start %llu (should be 128K)", em->start); + ret = -EINVAL; + goto out; + } + if (em->len != SZ_16K) { + test_err("unexpected extent map length %llu (should be 16K)", em->len); + ret = -EINVAL; + goto out; + } + if (em->offset != SZ_1K * 20) { + test_err("unexpected extent map offset %llu (should be 20K)", em->offset); + ret = -EINVAL; + goto out; + } +out: + ret2 = free_extent_map_tree(inode); + if (ret == 0) + ret = ret2; + + return ret; +} + +struct rmap_test_vector { + u64 raid_type; + u64 physical_start; + u64 data_stripe_size; + u64 num_data_stripes; + u64 num_stripes; + /* Assume we won't have more than 5 physical stripes */ + u64 data_stripe_phys_start[5]; + bool expected_mapped_addr; + /* Physical to logical addresses */ + u64 mapped_logical[5]; +}; + +static int test_rmap_block(struct btrfs_fs_info *fs_info, + struct rmap_test_vector *test) +{ + struct btrfs_chunk_map *map; + u64 AUTO_KFREE(logical); + int i, out_ndaddrs, out_stripe_len; + int ret; + + map = btrfs_alloc_chunk_map(test->num_stripes, GFP_KERNEL); + if (!map) { + test_std_err(TEST_ALLOC_CHUNK_MAP); + return -ENOMEM; + } + + /* Start at 4GiB logical address */ + map->start = SZ_4G; + map->chunk_len = test->data_stripe_size * test->num_data_stripes; + map->stripe_size = test->data_stripe_size; + map->num_stripes = test->num_stripes; + map->type = test->raid_type; + + for (i = 0; i < map->num_stripes; i++) { + struct btrfs_device *dev = btrfs_alloc_dummy_device(fs_info); + + if (IS_ERR(dev)) { + test_err("cannot allocate device"); + ret = PTR_ERR(dev); + goto out; + } + map->stripes[i].dev = dev; + map->stripes[i].physical = test->data_stripe_phys_start[i]; + } + + ret = btrfs_add_chunk_map(fs_info, map); + if (ret) { + test_err("error adding chunk map to mapping tree"); + btrfs_free_chunk_map(map); + return ret; + } + + ret = btrfs_rmap_block(fs_info, map->start, btrfs_sb_offset(1), + &logical, &out_ndaddrs, &out_stripe_len); + if (ret || (out_ndaddrs == 0 && test->expected_mapped_addr)) { + test_err("didn't rmap anything but expected %d", + test->expected_mapped_addr); + goto out; + } + + if (out_stripe_len != BTRFS_STRIPE_LEN) { + test_err("calculated stripe length doesn't match"); + ret = -EINVAL; + goto out; + } + + if (out_ndaddrs != test->expected_mapped_addr) { + for (i = 0; i < out_ndaddrs; i++) + test_msg("mapped %llu", logical[i]); + test_err("unexpected number of mapped addresses: %d", out_ndaddrs); + ret = -EINVAL; + goto out; + } + + for (i = 0; i < out_ndaddrs; i++) { + if (logical[i] != test->mapped_logical[i]) { + test_err("unexpected logical address mapped"); + ret = -EINVAL; + goto out; + } + } + + ret = 0; +out: + btrfs_remove_chunk_map(fs_info, map); + return ret; +} + +int btrfs_test_extent_map(void) +{ + struct btrfs_fs_info *fs_info = NULL; + struct inode *inode; + struct btrfs_root *root = NULL; + int ret = 0, i; + struct rmap_test_vector rmap_tests[] = { + { + /* + * Test a chunk with 2 data stripes one of which + * intersects the physical address of the super block + * is correctly recognized. + */ + .raid_type = BTRFS_BLOCK_GROUP_RAID1, + .physical_start = SZ_64M - SZ_4M, + .data_stripe_size = SZ_256M, + .num_data_stripes = 2, + .num_stripes = 2, + .data_stripe_phys_start = + {SZ_64M - SZ_4M, SZ_64M - SZ_4M + SZ_256M}, + .expected_mapped_addr = true, + .mapped_logical= {SZ_4G + SZ_4M} + }, + { + /* + * Test that out-of-range physical addresses are + * ignored + */ + + /* SINGLE chunk type */ + .raid_type = 0, + .physical_start = SZ_4G, + .data_stripe_size = SZ_256M, + .num_data_stripes = 1, + .num_stripes = 1, + .data_stripe_phys_start = {SZ_256M}, + .expected_mapped_addr = false, + .mapped_logical = {0} + } + }; + + test_msg("running extent_map tests"); + + /* + * Note: the fs_info is not set up completely, we only need + * fs_info::fsid for the tracepoint. + * + * And all the immediate numbers are based on 4K blocksize, + * thus we have to use 4K as sectorsize no matter the page size. + */ + fs_info = btrfs_alloc_dummy_fs_info(SZ_4K, SZ_4K); + if (!fs_info) { + test_std_err(TEST_ALLOC_FS_INFO); + return -ENOMEM; + } + + inode = btrfs_new_test_inode(); + if (!inode) { + test_std_err(TEST_ALLOC_INODE); + ret = -ENOMEM; + goto out; + } + + root = btrfs_alloc_dummy_root(fs_info); + if (IS_ERR(root)) { + test_std_err(TEST_ALLOC_ROOT); + ret = PTR_ERR(root); + root = NULL; + goto out; + } + + BTRFS_I(inode)->root = root; + + ret = test_case_1(fs_info, BTRFS_I(inode)); + if (ret) + goto out; + ret = test_case_2(fs_info, BTRFS_I(inode)); + if (ret) + goto out; + ret = test_case_3(fs_info, BTRFS_I(inode)); + if (ret) + goto out; + ret = test_case_4(fs_info, BTRFS_I(inode)); + if (ret) + goto out; + ret = test_case_5(fs_info, BTRFS_I(inode)); + if (ret) + goto out; + ret = test_case_6(fs_info, BTRFS_I(inode)); + if (ret) + goto out; + ret = test_case_7(fs_info, BTRFS_I(inode)); + if (ret) + goto out; + ret = test_case_8(fs_info, BTRFS_I(inode)); + if (ret) + goto out; + + test_msg("running rmap tests"); + for (i = 0; i < ARRAY_SIZE(rmap_tests); i++) { + ret = test_rmap_block(fs_info, &rmap_tests[i]); + if (ret) + goto out; + } + +out: + iput(inode); + btrfs_free_dummy_root(root); + btrfs_free_dummy_fs_info(fs_info); + + return ret; +} diff --git a/fs/btrfs/tests/free-space-tests.c b/fs/btrfs/tests/free-space-tests.c new file mode 100644 index 000000000..0425b3b68 --- /dev/null +++ b/fs/btrfs/tests/free-space-tests.c @@ -0,0 +1,1059 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2013 Fusion IO. All rights reserved. + */ + +#include +#include "btrfs-tests.h" +#include "../ctree.h" +#include "../disk-io.h" +#include "../free-space-cache.h" +#include "../block-group.h" + +#define BITS_PER_BITMAP (PAGE_SIZE * 8UL) + +/* + * This test just does basic sanity checking, making sure we can add an extent + * entry and remove space from either end and the middle, and make sure we can + * remove space that covers adjacent extent entries. + */ +static int test_extents(struct btrfs_block_group *cache) +{ + int ret = 0; + + test_msg("running extent only tests"); + + /* First just make sure we can remove an entire entry */ + ret = btrfs_add_free_space(cache, 0, SZ_4M); + if (ret) { + test_err("error adding initial extents %d", ret); + return ret; + } + + ret = btrfs_remove_free_space(cache, 0, SZ_4M); + if (ret) { + test_err("error removing extent %d", ret); + return ret; + } + + if (test_check_exists(cache, 0, SZ_4M)) { + test_err("full remove left some lingering space"); + return -1; + } + + /* Ok edge and middle cases now */ + ret = btrfs_add_free_space(cache, 0, SZ_4M); + if (ret) { + test_err("error adding half extent %d", ret); + return ret; + } + + ret = btrfs_remove_free_space(cache, 3 * SZ_1M, SZ_1M); + if (ret) { + test_err("error removing tail end %d", ret); + return ret; + } + + ret = btrfs_remove_free_space(cache, 0, SZ_1M); + if (ret) { + test_err("error removing front end %d", ret); + return ret; + } + + ret = btrfs_remove_free_space(cache, SZ_2M, 4096); + if (ret) { + test_err("error removing middle piece %d", ret); + return ret; + } + + if (test_check_exists(cache, 0, SZ_1M)) { + test_err("still have space at the front"); + return -1; + } + + if (test_check_exists(cache, SZ_2M, 4096)) { + test_err("still have space in the middle"); + return -1; + } + + if (test_check_exists(cache, 3 * SZ_1M, SZ_1M)) { + test_err("still have space at the end"); + return -1; + } + + /* Cleanup */ + btrfs_remove_free_space_cache(cache); + + return 0; +} + +static int test_bitmaps(struct btrfs_block_group *cache, u32 sectorsize) +{ + u64 next_bitmap_offset; + int ret; + + test_msg("running bitmap only tests"); + + ret = test_add_free_space_entry(cache, 0, SZ_4M, 1); + if (ret) { + test_err("couldn't create a bitmap entry %d", ret); + return ret; + } + + ret = btrfs_remove_free_space(cache, 0, SZ_4M); + if (ret) { + test_err("error removing bitmap full range %d", ret); + return ret; + } + + if (test_check_exists(cache, 0, SZ_4M)) { + test_err("left some space in bitmap"); + return -1; + } + + ret = test_add_free_space_entry(cache, 0, SZ_4M, 1); + if (ret) { + test_err("couldn't add to our bitmap entry %d", ret); + return ret; + } + + ret = btrfs_remove_free_space(cache, SZ_1M, SZ_2M); + if (ret) { + test_err("couldn't remove middle chunk %d", ret); + return ret; + } + + /* + * The first bitmap we have starts at offset 0 so the next one is just + * at the end of the first bitmap. + */ + next_bitmap_offset = (u64)(BITS_PER_BITMAP * sectorsize); + + /* Test a bit straddling two bitmaps */ + ret = test_add_free_space_entry(cache, next_bitmap_offset - SZ_2M, + SZ_4M, 1); + if (ret) { + test_err("couldn't add space that straddles two bitmaps %d", + ret); + return ret; + } + + ret = btrfs_remove_free_space(cache, next_bitmap_offset - SZ_1M, SZ_2M); + if (ret) { + test_err("couldn't remove overlapping space %d", ret); + return ret; + } + + if (test_check_exists(cache, next_bitmap_offset - SZ_1M, SZ_2M)) { + test_err("left some space when removing overlapping"); + return -1; + } + + btrfs_remove_free_space_cache(cache); + + return 0; +} + +/* This is the high grade jackassery */ +static int test_bitmaps_and_extents(struct btrfs_block_group *cache, + u32 sectorsize) +{ + u64 bitmap_offset = (u64)(BITS_PER_BITMAP * sectorsize); + int ret; + + test_msg("running bitmap and extent tests"); + + /* + * First let's do something simple, an extent at the same offset as the + * bitmap, but the free space completely in the extent and then + * completely in the bitmap. + */ + ret = test_add_free_space_entry(cache, SZ_4M, SZ_1M, 1); + if (ret) { + test_err("couldn't create bitmap entry %d", ret); + return ret; + } + + ret = test_add_free_space_entry(cache, 0, SZ_1M, 0); + if (ret) { + test_err("couldn't add extent entry %d", ret); + return ret; + } + + ret = btrfs_remove_free_space(cache, 0, SZ_1M); + if (ret) { + test_err("couldn't remove extent entry %d", ret); + return ret; + } + + if (test_check_exists(cache, 0, SZ_1M)) { + test_err("left remnants after our remove"); + return -1; + } + + /* Now to add back the extent entry and remove from the bitmap */ + ret = test_add_free_space_entry(cache, 0, SZ_1M, 0); + if (ret) { + test_err("couldn't re-add extent entry %d", ret); + return ret; + } + + ret = btrfs_remove_free_space(cache, SZ_4M, SZ_1M); + if (ret) { + test_err("couldn't remove from bitmap %d", ret); + return ret; + } + + if (test_check_exists(cache, SZ_4M, SZ_1M)) { + test_err("left remnants in the bitmap"); + return -1; + } + + /* + * Ok so a little more evil, extent entry and bitmap at the same offset, + * removing an overlapping chunk. + */ + ret = test_add_free_space_entry(cache, SZ_1M, SZ_4M, 1); + if (ret) { + test_err("couldn't add to a bitmap %d", ret); + return ret; + } + + ret = btrfs_remove_free_space(cache, SZ_512K, 3 * SZ_1M); + if (ret) { + test_err("couldn't remove overlapping space %d", ret); + return ret; + } + + if (test_check_exists(cache, SZ_512K, 3 * SZ_1M)) { + test_err("left over pieces after removing overlapping"); + return -1; + } + + btrfs_remove_free_space_cache(cache); + + /* Now with the extent entry offset into the bitmap */ + ret = test_add_free_space_entry(cache, SZ_4M, SZ_4M, 1); + if (ret) { + test_err("couldn't add space to the bitmap %d", ret); + return ret; + } + + ret = test_add_free_space_entry(cache, SZ_2M, SZ_2M, 0); + if (ret) { + test_err("couldn't add extent to the cache %d", ret); + return ret; + } + + ret = btrfs_remove_free_space(cache, 3 * SZ_1M, SZ_4M); + if (ret) { + test_err("problem removing overlapping space %d", ret); + return ret; + } + + if (test_check_exists(cache, 3 * SZ_1M, SZ_4M)) { + test_err("left something behind when removing space"); + return -1; + } + + /* + * This has blown up in the past, the extent entry starts before the + * bitmap entry, but we're trying to remove an offset that falls + * completely within the bitmap range and is in both the extent entry + * and the bitmap entry, looks like this + * + * [ extent ] + * [ bitmap ] + * [ del ] + */ + btrfs_remove_free_space_cache(cache); + ret = test_add_free_space_entry(cache, bitmap_offset + SZ_4M, SZ_4M, 1); + if (ret) { + test_err("couldn't add bitmap %d", ret); + return ret; + } + + ret = test_add_free_space_entry(cache, bitmap_offset - SZ_1M, + 5 * SZ_1M, 0); + if (ret) { + test_err("couldn't add extent entry %d", ret); + return ret; + } + + ret = btrfs_remove_free_space(cache, bitmap_offset + SZ_1M, 5 * SZ_1M); + if (ret) { + test_err("failed to free our space %d", ret); + return ret; + } + + if (test_check_exists(cache, bitmap_offset + SZ_1M, 5 * SZ_1M)) { + test_err("left stuff over"); + return -1; + } + + btrfs_remove_free_space_cache(cache); + + /* + * This blew up before, we have part of the free space in a bitmap and + * then the entirety of the rest of the space in an extent. This used + * to return -EAGAIN back from btrfs_remove_extent, make sure this + * doesn't happen. + */ + ret = test_add_free_space_entry(cache, SZ_1M, SZ_2M, 1); + if (ret) { + test_err("couldn't add bitmap entry %d", ret); + return ret; + } + + ret = test_add_free_space_entry(cache, 3 * SZ_1M, SZ_1M, 0); + if (ret) { + test_err("couldn't add extent entry %d", ret); + return ret; + } + + ret = btrfs_remove_free_space(cache, SZ_1M, 3 * SZ_1M); + if (ret) { + test_err("error removing bitmap and extent overlapping %d", ret); + return ret; + } + + btrfs_remove_free_space_cache(cache); + return 0; +} + +/* Used by test_steal_space_from_bitmap_to_extent(). */ +static bool test_use_bitmap(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info) +{ + return ctl->free_extents > 0; +} + +/* Used by test_steal_space_from_bitmap_to_extent(). */ +static int +check_num_extents_and_bitmaps(const struct btrfs_block_group *cache, + const int num_extents, + const int num_bitmaps) +{ + if (cache->free_space_ctl->free_extents != num_extents) { + test_err( + "incorrect # of extent entries in the cache: %d, expected %d", + cache->free_space_ctl->free_extents, num_extents); + return -EINVAL; + } + if (cache->free_space_ctl->total_bitmaps != num_bitmaps) { + test_err( + "incorrect # of extent entries in the cache: %d, expected %d", + cache->free_space_ctl->total_bitmaps, num_bitmaps); + return -EINVAL; + } + return 0; +} + +/* Used by test_steal_space_from_bitmap_to_extent(). */ +static int check_cache_empty(struct btrfs_block_group *cache) +{ + u64 offset; + u64 max_extent_size; + + /* + * Now lets confirm that there's absolutely no free space left to + * allocate. + */ + if (cache->free_space_ctl->free_space != 0) { + test_err("cache free space is not 0"); + return -EINVAL; + } + + /* And any allocation request, no matter how small, should fail now. */ + offset = btrfs_find_space_for_alloc(cache, 0, 4096, 0, + &max_extent_size); + if (offset != 0) { + test_err("space allocation did not fail, returned offset: %llu", + offset); + return -EINVAL; + } + + /* And no extent nor bitmap entries in the cache anymore. */ + return check_num_extents_and_bitmaps(cache, 0, 0); +} + +/* + * Before we were able to steal free space from a bitmap entry to an extent + * entry, we could end up with 2 entries representing a contiguous free space. + * One would be an extent entry and the other a bitmap entry. Since in order + * to allocate space to a caller we use only 1 entry, we couldn't return that + * whole range to the caller if it was requested. This forced the caller to + * either assume ENOSPC or perform several smaller space allocations, which + * wasn't optimal as they could be spread all over the block group while under + * concurrency (extra overhead and fragmentation). + * + * This stealing approach is beneficial, since we always prefer to allocate + * from extent entries, both for clustered and non-clustered allocation + * requests. + */ +static int +test_steal_space_from_bitmap_to_extent(struct btrfs_block_group *cache, + u32 sectorsize) +{ + int ret; + u64 offset; + u64 max_extent_size; + bool (*orig_use_bitmap)(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info); + + test_msg("running space stealing from bitmap to extent tests"); + + /* + * For this test, we want to ensure we end up with an extent entry + * immediately adjacent to a bitmap entry, where the bitmap starts + * at an offset where the extent entry ends. We keep adding and + * removing free space to reach into this state, but to get there + * we need to reach a point where marking new free space doesn't + * result in adding new extent entries or merging the new space + * with existing extent entries - the space ends up being marked + * in an existing bitmap that covers the new free space range. + * + * To get there, we need to reach the threshold defined set at + * cache->free_space_ctl->extents_thresh, which currently is + * 256 extents on a x86_64 system at least, and a few other + * conditions (check free_space_cache.c). Instead of making the + * test much longer and complicated, use a "use_bitmap" operation + * that forces use of bitmaps as soon as we have at least 1 + * extent entry. + */ + orig_use_bitmap = cache->fs_info->use_bitmap; + cache->fs_info->use_bitmap = test_use_bitmap; + + /* + * Extent entry covering free space range [128Mb - 256Kb, 128Mb - 128Kb[ + */ + ret = test_add_free_space_entry(cache, SZ_128M - SZ_256K, SZ_128K, 0); + if (ret) { + test_err("couldn't add extent entry %d", ret); + return ret; + } + + /* Bitmap entry covering free space range [128Mb + 512Kb, 256Mb[ */ + ret = test_add_free_space_entry(cache, SZ_128M + SZ_512K, + SZ_128M - SZ_512K, 1); + if (ret) { + test_err("couldn't add bitmap entry %d", ret); + return ret; + } + + ret = check_num_extents_and_bitmaps(cache, 2, 1); + if (ret) + return ret; + + /* + * Now make only the first 256Kb of the bitmap marked as free, so that + * we end up with only the following ranges marked as free space: + * + * [128Mb - 256Kb, 128Mb - 128Kb[ + * [128Mb + 512Kb, 128Mb + 768Kb[ + */ + ret = btrfs_remove_free_space(cache, + SZ_128M + 768 * SZ_1K, + SZ_128M - 768 * SZ_1K); + if (ret) { + test_err("failed to free part of bitmap space %d", ret); + return ret; + } + + /* Confirm that only those 2 ranges are marked as free. */ + if (!test_check_exists(cache, SZ_128M - SZ_256K, SZ_128K)) { + test_err("free space range missing"); + return -ENOENT; + } + if (!test_check_exists(cache, SZ_128M + SZ_512K, SZ_256K)) { + test_err("free space range missing"); + return -ENOENT; + } + + /* + * Confirm that the bitmap range [128Mb + 768Kb, 256Mb[ isn't marked + * as free anymore. + */ + if (test_check_exists(cache, SZ_128M + 768 * SZ_1K, + SZ_128M - 768 * SZ_1K)) { + test_err("bitmap region not removed from space cache"); + return -EINVAL; + } + + /* + * Confirm that the region [128Mb + 256Kb, 128Mb + 512Kb[, which is + * covered by the bitmap, isn't marked as free. + */ + if (test_check_exists(cache, SZ_128M + SZ_256K, SZ_256K)) { + test_err("invalid bitmap region marked as free"); + return -EINVAL; + } + + /* + * Confirm that the region [128Mb, 128Mb + 256Kb[, which is covered + * by the bitmap too, isn't marked as free either. + */ + if (test_check_exists(cache, SZ_128M, SZ_256K)) { + test_err("invalid bitmap region marked as free"); + return -EINVAL; + } + + /* + * Now lets mark the region [128Mb, 128Mb + 512Kb[ as free too. But, + * lets make sure the free space cache marks it as free in the bitmap, + * and doesn't insert a new extent entry to represent this region. + */ + ret = btrfs_add_free_space(cache, SZ_128M, SZ_512K); + if (ret) { + test_err("error adding free space: %d", ret); + return ret; + } + /* Confirm the region is marked as free. */ + if (!test_check_exists(cache, SZ_128M, SZ_512K)) { + test_err("bitmap region not marked as free"); + return -ENOENT; + } + + /* + * Confirm that no new extent entries or bitmap entries were added to + * the cache after adding that free space region. + */ + ret = check_num_extents_and_bitmaps(cache, 2, 1); + if (ret) + return ret; + + /* + * Now lets add a small free space region to the right of the previous + * one, which is not contiguous with it and is part of the bitmap too. + * The goal is to test that the bitmap entry space stealing doesn't + * steal this space region. + */ + ret = btrfs_add_free_space(cache, SZ_128M + SZ_16M, sectorsize); + if (ret) { + test_err("error adding free space: %d", ret); + return ret; + } + + /* + * Confirm that no new extent entries or bitmap entries were added to + * the cache after adding that free space region. + */ + ret = check_num_extents_and_bitmaps(cache, 2, 1); + if (ret) + return ret; + + /* + * Now mark the region [128Mb - 128Kb, 128Mb[ as free too. This will + * expand the range covered by the existing extent entry that represents + * the free space [128Mb - 256Kb, 128Mb - 128Kb[. + */ + ret = btrfs_add_free_space(cache, SZ_128M - SZ_128K, SZ_128K); + if (ret) { + test_err("error adding free space: %d", ret); + return ret; + } + /* Confirm the region is marked as free. */ + if (!test_check_exists(cache, SZ_128M - SZ_128K, SZ_128K)) { + test_err("extent region not marked as free"); + return -ENOENT; + } + + /* + * Confirm that our extent entry didn't stole all free space from the + * bitmap, because of the small 4Kb free space region. + */ + ret = check_num_extents_and_bitmaps(cache, 2, 1); + if (ret) + return ret; + + /* + * So now we have the range [128Mb - 256Kb, 128Mb + 768Kb[ as free + * space. Without stealing bitmap free space into extent entry space, + * we would have all this free space represented by 2 entries in the + * cache: + * + * extent entry covering range: [128Mb - 256Kb, 128Mb[ + * bitmap entry covering range: [128Mb, 128Mb + 768Kb[ + * + * Attempting to allocate the whole free space (1Mb) would fail, because + * we can't allocate from multiple entries. + * With the bitmap free space stealing, we get a single extent entry + * that represents the 1Mb free space, and therefore we're able to + * allocate the whole free space at once. + */ + if (!test_check_exists(cache, SZ_128M - SZ_256K, SZ_1M)) { + test_err("expected region not marked as free"); + return -ENOENT; + } + + if (cache->free_space_ctl->free_space != (SZ_1M + sectorsize)) { + test_err("cache free space is not 1Mb + %u", sectorsize); + return -EINVAL; + } + + offset = btrfs_find_space_for_alloc(cache, + 0, SZ_1M, 0, + &max_extent_size); + if (offset != (SZ_128M - SZ_256K)) { + test_err( + "failed to allocate 1Mb from space cache, returned offset is: %llu", + offset); + return -EINVAL; + } + + /* + * All that remains is a sectorsize free space region in a bitmap. + * Confirm. + */ + ret = check_num_extents_and_bitmaps(cache, 1, 1); + if (ret) + return ret; + + if (cache->free_space_ctl->free_space != sectorsize) { + test_err("cache free space is not %u", sectorsize); + return -EINVAL; + } + + offset = btrfs_find_space_for_alloc(cache, + 0, sectorsize, 0, + &max_extent_size); + if (offset != (SZ_128M + SZ_16M)) { + test_err("failed to allocate %u, returned offset : %llu", + sectorsize, offset); + return -EINVAL; + } + + ret = check_cache_empty(cache); + if (ret) + return ret; + + btrfs_remove_free_space_cache(cache); + + /* + * Now test a similar scenario, but where our extent entry is located + * to the right of the bitmap entry, so that we can check that stealing + * space from a bitmap to the front of an extent entry works. + */ + + /* + * Extent entry covering free space range [128Mb + 128Kb, 128Mb + 256Kb[ + */ + ret = test_add_free_space_entry(cache, SZ_128M + SZ_128K, SZ_128K, 0); + if (ret) { + test_err("couldn't add extent entry %d", ret); + return ret; + } + + /* Bitmap entry covering free space range [0, 128Mb - 512Kb[ */ + ret = test_add_free_space_entry(cache, 0, SZ_128M - SZ_512K, 1); + if (ret) { + test_err("couldn't add bitmap entry %d", ret); + return ret; + } + + ret = check_num_extents_and_bitmaps(cache, 2, 1); + if (ret) + return ret; + + /* + * Now make only the last 256Kb of the bitmap marked as free, so that + * we end up with only the following ranges marked as free space: + * + * [128Mb + 128b, 128Mb + 256Kb[ + * [128Mb - 768Kb, 128Mb - 512Kb[ + */ + ret = btrfs_remove_free_space(cache, 0, SZ_128M - 768 * SZ_1K); + if (ret) { + test_err("failed to free part of bitmap space %d", ret); + return ret; + } + + /* Confirm that only those 2 ranges are marked as free. */ + if (!test_check_exists(cache, SZ_128M + SZ_128K, SZ_128K)) { + test_err("free space range missing"); + return -ENOENT; + } + if (!test_check_exists(cache, SZ_128M - 768 * SZ_1K, SZ_256K)) { + test_err("free space range missing"); + return -ENOENT; + } + + /* + * Confirm that the bitmap range [0, 128Mb - 768Kb[ isn't marked + * as free anymore. + */ + if (test_check_exists(cache, 0, SZ_128M - 768 * SZ_1K)) { + test_err("bitmap region not removed from space cache"); + return -EINVAL; + } + + /* + * Confirm that the region [128Mb - 512Kb, 128Mb[, which is + * covered by the bitmap, isn't marked as free. + */ + if (test_check_exists(cache, SZ_128M - SZ_512K, SZ_512K)) { + test_err("invalid bitmap region marked as free"); + return -EINVAL; + } + + /* + * Now lets mark the region [128Mb - 512Kb, 128Mb[ as free too. But, + * lets make sure the free space cache marks it as free in the bitmap, + * and doesn't insert a new extent entry to represent this region. + */ + ret = btrfs_add_free_space(cache, SZ_128M - SZ_512K, SZ_512K); + if (ret) { + test_err("error adding free space: %d", ret); + return ret; + } + /* Confirm the region is marked as free. */ + if (!test_check_exists(cache, SZ_128M - SZ_512K, SZ_512K)) { + test_err("bitmap region not marked as free"); + return -ENOENT; + } + + /* + * Confirm that no new extent entries or bitmap entries were added to + * the cache after adding that free space region. + */ + ret = check_num_extents_and_bitmaps(cache, 2, 1); + if (ret) + return ret; + + /* + * Now lets add a small free space region to the left of the previous + * one, which is not contiguous with it and is part of the bitmap too. + * The goal is to test that the bitmap entry space stealing doesn't + * steal this space region. + */ + ret = btrfs_add_free_space(cache, SZ_32M, 2 * sectorsize); + if (ret) { + test_err("error adding free space: %d", ret); + return ret; + } + + /* + * Now mark the region [128Mb, 128Mb + 128Kb[ as free too. This will + * expand the range covered by the existing extent entry that represents + * the free space [128Mb + 128Kb, 128Mb + 256Kb[. + */ + ret = btrfs_add_free_space(cache, SZ_128M, SZ_128K); + if (ret) { + test_err("error adding free space: %d", ret); + return ret; + } + /* Confirm the region is marked as free. */ + if (!test_check_exists(cache, SZ_128M, SZ_128K)) { + test_err("extent region not marked as free"); + return -ENOENT; + } + + /* + * Confirm that our extent entry didn't stole all free space from the + * bitmap, because of the small 2 * sectorsize free space region. + */ + ret = check_num_extents_and_bitmaps(cache, 2, 1); + if (ret) + return ret; + + /* + * So now we have the range [128Mb - 768Kb, 128Mb + 256Kb[ as free + * space. Without stealing bitmap free space into extent entry space, + * we would have all this free space represented by 2 entries in the + * cache: + * + * extent entry covering range: [128Mb, 128Mb + 256Kb[ + * bitmap entry covering range: [128Mb - 768Kb, 128Mb[ + * + * Attempting to allocate the whole free space (1Mb) would fail, because + * we can't allocate from multiple entries. + * With the bitmap free space stealing, we get a single extent entry + * that represents the 1Mb free space, and therefore we're able to + * allocate the whole free space at once. + */ + if (!test_check_exists(cache, SZ_128M - 768 * SZ_1K, SZ_1M)) { + test_err("expected region not marked as free"); + return -ENOENT; + } + + if (cache->free_space_ctl->free_space != (SZ_1M + 2 * sectorsize)) { + test_err("cache free space is not 1Mb + %u", 2 * sectorsize); + return -EINVAL; + } + + offset = btrfs_find_space_for_alloc(cache, 0, SZ_1M, 0, + &max_extent_size); + if (offset != (SZ_128M - 768 * SZ_1K)) { + test_err( + "failed to allocate 1Mb from space cache, returned offset is: %llu", + offset); + return -EINVAL; + } + + /* + * All that remains is 2 * sectorsize free space region + * in a bitmap. Confirm. + */ + ret = check_num_extents_and_bitmaps(cache, 1, 1); + if (ret) + return ret; + + if (cache->free_space_ctl->free_space != 2 * sectorsize) { + test_err("cache free space is not %u", 2 * sectorsize); + return -EINVAL; + } + + offset = btrfs_find_space_for_alloc(cache, + 0, 2 * sectorsize, 0, + &max_extent_size); + if (offset != SZ_32M) { + test_err("failed to allocate %u, offset: %llu", + 2 * sectorsize, offset); + return -EINVAL; + } + + ret = check_cache_empty(cache); + if (ret) + return ret; + + cache->fs_info->use_bitmap = orig_use_bitmap; + btrfs_remove_free_space_cache(cache); + + return 0; +} + +static bool bytes_index_use_bitmap(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info) +{ + return true; +} + +static int test_bytes_index(struct btrfs_block_group *cache, u32 sectorsize) +{ + bool (*orig_use_bitmap)(struct btrfs_free_space_ctl *ctl, + struct btrfs_free_space *info); + struct btrfs_free_space_ctl *ctl = cache->free_space_ctl; + struct btrfs_free_space *entry; + struct rb_node *node; + u64 offset, max_extent_size, bytes; + int ret, i; + + test_msg("running bytes index tests"); + + /* First just validate that it does everything in order. */ + offset = 0; + for (i = 0; i < 10; i++) { + bytes = (i + 1) * SZ_1M; + ret = test_add_free_space_entry(cache, offset, bytes, 0); + if (ret) { + test_err("couldn't add extent entry %d\n", ret); + return ret; + } + offset += bytes + sectorsize; + } + + for (node = rb_first_cached(&ctl->free_space_bytes), i = 9; node; + node = rb_next(node), i--) { + entry = rb_entry(node, struct btrfs_free_space, bytes_index); + bytes = (i + 1) * SZ_1M; + if (entry->bytes != bytes) { + test_err("invalid bytes index order, found %llu expected %llu", + entry->bytes, bytes); + return -EINVAL; + } + } + + /* Now validate bitmaps do the correct thing. */ + btrfs_remove_free_space_cache(cache); + for (i = 0; i < 2; i++) { + offset = i * BITS_PER_BITMAP * sectorsize; + bytes = (i + 1) * SZ_1M; + ret = test_add_free_space_entry(cache, offset, bytes, 1); + if (ret) { + test_err("couldn't add bitmap entry"); + return ret; + } + } + + for (node = rb_first_cached(&ctl->free_space_bytes), i = 1; node; + node = rb_next(node), i--) { + entry = rb_entry(node, struct btrfs_free_space, bytes_index); + bytes = (i + 1) * SZ_1M; + if (entry->bytes != bytes) { + test_err("invalid bytes index order, found %llu expected %llu", + entry->bytes, bytes); + return -EINVAL; + } + } + + /* Now validate bitmaps with different ->max_extent_size. */ + btrfs_remove_free_space_cache(cache); + orig_use_bitmap = cache->fs_info->use_bitmap; + cache->fs_info->use_bitmap = bytes_index_use_bitmap; + + ret = test_add_free_space_entry(cache, 0, sectorsize, 1); + if (ret) { + test_err("couldn't add bitmap entry"); + return ret; + } + + offset = BITS_PER_BITMAP * sectorsize; + ret = test_add_free_space_entry(cache, offset, sectorsize, 1); + if (ret) { + test_err("couldn't add bitmap_entry"); + return ret; + } + + /* + * Now set a bunch of sectorsize extents in the first entry so it's + * ->bytes is large. + */ + for (i = 2; i < 20; i += 2) { + offset = sectorsize * i; + ret = btrfs_add_free_space(cache, offset, sectorsize); + if (ret) { + test_err("error populating sparse bitmap %d", ret); + return ret; + } + } + + /* + * Now set a contiguous extent in the second bitmap so its + * ->max_extent_size is larger than the first bitmaps. + */ + offset = (BITS_PER_BITMAP * sectorsize) + sectorsize; + ret = btrfs_add_free_space(cache, offset, sectorsize); + if (ret) { + test_err("error adding contiguous extent %d", ret); + return ret; + } + + /* + * Since we don't set ->max_extent_size unless we search everything + * should be indexed on bytes. + */ + entry = rb_entry(rb_first_cached(&ctl->free_space_bytes), + struct btrfs_free_space, bytes_index); + if (entry->bytes != (10 * sectorsize)) { + test_err("error, wrong entry in the first slot in bytes_index"); + return -EINVAL; + } + + max_extent_size = 0; + offset = btrfs_find_space_for_alloc(cache, cache->start, sectorsize * 3, + 0, &max_extent_size); + if (offset != 0) { + test_err("found space to alloc even though we don't have enough space"); + return -EINVAL; + } + + if (max_extent_size != (2 * sectorsize)) { + test_err("got the wrong max_extent size %llu expected %llu", + max_extent_size, (unsigned long long)(2 * sectorsize)); + return -EINVAL; + } + + /* + * The search should have re-arranged the bytes index to use the + * ->max_extent_size, validate it's now what we expect it to be. + */ + entry = rb_entry(rb_first_cached(&ctl->free_space_bytes), + struct btrfs_free_space, bytes_index); + if (entry->bytes != (2 * sectorsize)) { + test_err("error, the bytes index wasn't recalculated properly"); + return -EINVAL; + } + + /* Add another sectorsize to re-arrange the tree back to ->bytes. */ + offset = (BITS_PER_BITMAP * sectorsize) - sectorsize; + ret = btrfs_add_free_space(cache, offset, sectorsize); + if (ret) { + test_err("error adding extent to the sparse entry %d", ret); + return ret; + } + + entry = rb_entry(rb_first_cached(&ctl->free_space_bytes), + struct btrfs_free_space, bytes_index); + if (entry->bytes != (11 * sectorsize)) { + test_err("error, wrong entry in the first slot in bytes_index"); + return -EINVAL; + } + + /* + * Now make sure we find our correct entry after searching that will + * result in a re-arranging of the tree. + */ + max_extent_size = 0; + offset = btrfs_find_space_for_alloc(cache, cache->start, sectorsize * 2, + 0, &max_extent_size); + if (offset != (BITS_PER_BITMAP * sectorsize)) { + test_err("error, found %llu instead of %llu for our alloc", + offset, + (unsigned long long)(BITS_PER_BITMAP * sectorsize)); + return -EINVAL; + } + + cache->fs_info->use_bitmap = orig_use_bitmap; + btrfs_remove_free_space_cache(cache); + return 0; +} + +int btrfs_test_free_space_cache(u32 sectorsize, u32 nodesize) +{ + struct btrfs_fs_info *fs_info; + struct btrfs_block_group *cache; + struct btrfs_root *root = NULL; + int ret = -ENOMEM; + + test_msg("running btrfs free space cache tests"); + fs_info = btrfs_alloc_dummy_fs_info(nodesize, sectorsize); + if (!fs_info) { + test_std_err(TEST_ALLOC_FS_INFO); + return -ENOMEM; + } + + /* + * For ppc64 (with 64k page size), bytes per bitmap might be + * larger than 1G. To make bitmap test available in ppc64, + * alloc dummy block group whose size cross bitmaps. + */ + cache = btrfs_alloc_dummy_block_group(fs_info, + BITS_PER_BITMAP * sectorsize + PAGE_SIZE); + if (!cache) { + test_std_err(TEST_ALLOC_BLOCK_GROUP); + btrfs_free_dummy_fs_info(fs_info); + return 0; + } + + root = btrfs_alloc_dummy_root(fs_info); + if (IS_ERR(root)) { + test_std_err(TEST_ALLOC_ROOT); + ret = PTR_ERR(root); + goto out; + } + + root->root_key.objectid = BTRFS_EXTENT_TREE_OBJECTID; + root->root_key.type = BTRFS_ROOT_ITEM_KEY; + root->root_key.offset = 0; + btrfs_global_root_insert(root); + + ret = test_extents(cache); + if (ret) + goto out; + ret = test_bitmaps(cache, sectorsize); + if (ret) + goto out; + ret = test_bitmaps_and_extents(cache, sectorsize); + if (ret) + goto out; + + ret = test_steal_space_from_bitmap_to_extent(cache, sectorsize); + if (ret) + goto out; + ret = test_bytes_index(cache, sectorsize); +out: + btrfs_free_dummy_block_group(cache); + btrfs_free_dummy_root(root); + btrfs_free_dummy_fs_info(fs_info); + return ret; +} diff --git a/fs/btrfs/tests/free-space-tree-tests.c b/fs/btrfs/tests/free-space-tree-tests.c new file mode 100644 index 000000000..8dee057f4 --- /dev/null +++ b/fs/btrfs/tests/free-space-tree-tests.c @@ -0,0 +1,586 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015 Facebook. All rights reserved. + */ + +#include +#include "btrfs-tests.h" +#include "../ctree.h" +#include "../disk-io.h" +#include "../free-space-tree.h" +#include "../transaction.h" +#include "../block-group.h" +#include "../accessors.h" + +struct free_space_extent { + u64 start; + u64 length; +}; + +static int __check_free_space_extents(struct btrfs_trans_handle *trans, + struct btrfs_fs_info *fs_info, + struct btrfs_block_group *cache, + struct btrfs_path *path, + const struct free_space_extent * const extents, + unsigned int num_extents) +{ + struct btrfs_free_space_info *info; + struct btrfs_key key; + int prev_bit = 0, bit; + u64 extent_start = 0, offset, end; + u32 flags, extent_count; + unsigned int i; + int ret; + + info = btrfs_search_free_space_info(trans, cache, path, 0); + if (IS_ERR(info)) { + test_err("could not find free space info"); + ret = PTR_ERR(info); + goto out; + } + flags = btrfs_free_space_flags(path->nodes[0], info); + extent_count = btrfs_free_space_extent_count(path->nodes[0], info); + + if (extent_count != num_extents) { + test_err("extent count is wrong"); + ret = -EINVAL; + goto out; + } + if (flags & BTRFS_FREE_SPACE_USING_BITMAPS) { + if (path->slots[0] != 0) + goto invalid; + end = btrfs_block_group_end(cache); + i = 0; + while (++path->slots[0] < btrfs_header_nritems(path->nodes[0])) { + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + if (key.type != BTRFS_FREE_SPACE_BITMAP_KEY) + goto invalid; + offset = key.objectid; + while (offset < key.objectid + key.offset) { + bit = btrfs_free_space_test_bit(cache, path, offset); + if (prev_bit == 0 && bit == 1) { + extent_start = offset; + } else if (prev_bit == 1 && bit == 0) { + if (i >= num_extents || + extent_start != extents[i].start || + offset - extent_start != extents[i].length) + goto invalid; + i++; + } + prev_bit = bit; + offset += fs_info->sectorsize; + } + } + if (prev_bit == 1) { + if (i >= num_extents || + extent_start != extents[i].start || + end - extent_start != extents[i].length) + goto invalid; + i++; + } + if (i != num_extents) + goto invalid; + } else { + if (btrfs_header_nritems(path->nodes[0]) != num_extents + 1 || + path->slots[0] != 0) + goto invalid; + for (i = 0; i < num_extents; i++) { + path->slots[0]++; + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + if (key.type != BTRFS_FREE_SPACE_EXTENT_KEY || + key.objectid != extents[i].start || + key.offset != extents[i].length) + goto invalid; + } + } + + ret = 0; +out: + btrfs_release_path(path); + return ret; +invalid: + test_err("free space tree is invalid"); + ret = -EINVAL; + goto out; +} + +static int check_free_space_extents(struct btrfs_trans_handle *trans, + struct btrfs_fs_info *fs_info, + struct btrfs_block_group *cache, + struct btrfs_path *path, + const struct free_space_extent * const extents, + unsigned int num_extents) +{ + struct btrfs_free_space_info *info; + u32 flags; + int ret; + + info = btrfs_search_free_space_info(trans, cache, path, 0); + if (IS_ERR(info)) { + test_err("could not find free space info"); + btrfs_release_path(path); + return PTR_ERR(info); + } + flags = btrfs_free_space_flags(path->nodes[0], info); + btrfs_release_path(path); + + ret = __check_free_space_extents(trans, fs_info, cache, path, extents, + num_extents); + if (ret) + return ret; + + /* Flip it to the other format and check that for good measure. */ + if (flags & BTRFS_FREE_SPACE_USING_BITMAPS) { + ret = btrfs_convert_free_space_to_extents(trans, cache, path); + if (ret) { + test_err("could not convert to extents"); + return ret; + } + } else { + ret = btrfs_convert_free_space_to_bitmaps(trans, cache, path); + if (ret) { + test_err("could not convert to bitmaps"); + return ret; + } + } + return __check_free_space_extents(trans, fs_info, cache, path, extents, + num_extents); +} + +static int test_empty_block_group(struct btrfs_trans_handle *trans, + struct btrfs_fs_info *fs_info, + struct btrfs_block_group *cache, + struct btrfs_path *path, + u32 alignment) +{ + const struct free_space_extent extents[] = { + {cache->start, cache->length}, + }; + + return check_free_space_extents(trans, fs_info, cache, path, + extents, ARRAY_SIZE(extents)); +} + +static int test_remove_all(struct btrfs_trans_handle *trans, + struct btrfs_fs_info *fs_info, + struct btrfs_block_group *cache, + struct btrfs_path *path, + u32 alignment) +{ + const struct free_space_extent extents[] = {}; + int ret; + + ret = __btrfs_remove_from_free_space_tree(trans, cache, path, + cache->start, cache->length); + if (ret) { + test_err("could not remove free space"); + return ret; + } + + return check_free_space_extents(trans, fs_info, cache, path, + extents, ARRAY_SIZE(extents)); +} + +static int test_remove_beginning(struct btrfs_trans_handle *trans, + struct btrfs_fs_info *fs_info, + struct btrfs_block_group *cache, + struct btrfs_path *path, + u32 alignment) +{ + const struct free_space_extent extents[] = { + {cache->start + alignment, cache->length - alignment}, + }; + int ret; + + ret = __btrfs_remove_from_free_space_tree(trans, cache, path, + cache->start, alignment); + if (ret) { + test_err("could not remove free space"); + return ret; + } + + return check_free_space_extents(trans, fs_info, cache, path, + extents, ARRAY_SIZE(extents)); + +} + +static int test_remove_end(struct btrfs_trans_handle *trans, + struct btrfs_fs_info *fs_info, + struct btrfs_block_group *cache, + struct btrfs_path *path, + u32 alignment) +{ + const struct free_space_extent extents[] = { + {cache->start, cache->length - alignment}, + }; + int ret; + + ret = __btrfs_remove_from_free_space_tree(trans, cache, path, + btrfs_block_group_end(cache) - alignment, + alignment); + if (ret) { + test_err("could not remove free space"); + return ret; + } + + return check_free_space_extents(trans, fs_info, cache, path, + extents, ARRAY_SIZE(extents)); +} + +static int test_remove_middle(struct btrfs_trans_handle *trans, + struct btrfs_fs_info *fs_info, + struct btrfs_block_group *cache, + struct btrfs_path *path, + u32 alignment) +{ + const struct free_space_extent extents[] = { + {cache->start, alignment}, + {cache->start + 2 * alignment, cache->length - 2 * alignment}, + }; + int ret; + + ret = __btrfs_remove_from_free_space_tree(trans, cache, path, + cache->start + alignment, + alignment); + if (ret) { + test_err("could not remove free space"); + return ret; + } + + return check_free_space_extents(trans, fs_info, cache, path, + extents, ARRAY_SIZE(extents)); +} + +static int test_merge_left(struct btrfs_trans_handle *trans, + struct btrfs_fs_info *fs_info, + struct btrfs_block_group *cache, + struct btrfs_path *path, + u32 alignment) +{ + const struct free_space_extent extents[] = { + {cache->start, 2 * alignment}, + }; + int ret; + + ret = __btrfs_remove_from_free_space_tree(trans, cache, path, + cache->start, cache->length); + if (ret) { + test_err("could not remove free space"); + return ret; + } + + ret = __btrfs_add_to_free_space_tree(trans, cache, path, cache->start, + alignment); + if (ret) { + test_err("could not add free space"); + return ret; + } + + ret = __btrfs_add_to_free_space_tree(trans, cache, path, + cache->start + alignment, alignment); + if (ret) { + test_err("could not add free space"); + return ret; + } + + return check_free_space_extents(trans, fs_info, cache, path, + extents, ARRAY_SIZE(extents)); +} + +static int test_merge_right(struct btrfs_trans_handle *trans, + struct btrfs_fs_info *fs_info, + struct btrfs_block_group *cache, + struct btrfs_path *path, + u32 alignment) +{ + const struct free_space_extent extents[] = { + {cache->start + alignment, 2 * alignment}, + }; + int ret; + + ret = __btrfs_remove_from_free_space_tree(trans, cache, path, + cache->start, cache->length); + if (ret) { + test_err("could not remove free space"); + return ret; + } + + ret = __btrfs_add_to_free_space_tree(trans, cache, path, + cache->start + 2 * alignment, + alignment); + if (ret) { + test_err("could not add free space"); + return ret; + } + + ret = __btrfs_add_to_free_space_tree(trans, cache, path, + cache->start + alignment, alignment); + if (ret) { + test_err("could not add free space"); + return ret; + } + + return check_free_space_extents(trans, fs_info, cache, path, + extents, ARRAY_SIZE(extents)); +} + +static int test_merge_both(struct btrfs_trans_handle *trans, + struct btrfs_fs_info *fs_info, + struct btrfs_block_group *cache, + struct btrfs_path *path, + u32 alignment) +{ + const struct free_space_extent extents[] = { + {cache->start, 3 * alignment}, + }; + int ret; + + ret = __btrfs_remove_from_free_space_tree(trans, cache, path, + cache->start, cache->length); + if (ret) { + test_err("could not remove free space"); + return ret; + } + + ret = __btrfs_add_to_free_space_tree(trans, cache, path, cache->start, + alignment); + if (ret) { + test_err("could not add free space"); + return ret; + } + + ret = __btrfs_add_to_free_space_tree(trans, cache, path, + cache->start + 2 * alignment, alignment); + if (ret) { + test_err("could not add free space"); + return ret; + } + + ret = __btrfs_add_to_free_space_tree(trans, cache, path, + cache->start + alignment, alignment); + if (ret) { + test_err("could not add free space"); + return ret; + } + + return check_free_space_extents(trans, fs_info, cache, path, + extents, ARRAY_SIZE(extents)); +} + +static int test_merge_none(struct btrfs_trans_handle *trans, + struct btrfs_fs_info *fs_info, + struct btrfs_block_group *cache, + struct btrfs_path *path, + u32 alignment) +{ + const struct free_space_extent extents[] = { + {cache->start, alignment}, + {cache->start + 2 * alignment, alignment}, + {cache->start + 4 * alignment, alignment}, + }; + int ret; + + ret = __btrfs_remove_from_free_space_tree(trans, cache, path, + cache->start, cache->length); + if (ret) { + test_err("could not remove free space"); + return ret; + } + + ret = __btrfs_add_to_free_space_tree(trans, cache, path, cache->start, + alignment); + if (ret) { + test_err("could not add free space"); + return ret; + } + + ret = __btrfs_add_to_free_space_tree(trans, cache, path, + cache->start + 4 * alignment, alignment); + if (ret) { + test_err("could not add free space"); + return ret; + } + + ret = __btrfs_add_to_free_space_tree(trans, cache, path, + cache->start + 2 * alignment, alignment); + if (ret) { + test_err("could not add free space"); + return ret; + } + + return check_free_space_extents(trans, fs_info, cache, path, + extents, ARRAY_SIZE(extents)); +} + +typedef int (*test_func_t)(struct btrfs_trans_handle *, + struct btrfs_fs_info *, + struct btrfs_block_group *, + struct btrfs_path *, + u32 alignment); + +static int run_test(test_func_t test_func, int bitmaps, u32 sectorsize, + u32 nodesize, u32 alignment) +{ + struct btrfs_fs_info *fs_info; + struct btrfs_root *root = NULL; + struct btrfs_block_group *cache = NULL; + struct btrfs_trans_handle trans; + struct btrfs_path *path = NULL; + int ret; + + fs_info = btrfs_alloc_dummy_fs_info(nodesize, sectorsize); + if (!fs_info) { + test_std_err(TEST_ALLOC_FS_INFO); + ret = -ENOMEM; + goto out; + } + + root = btrfs_alloc_dummy_root(fs_info); + if (IS_ERR(root)) { + test_std_err(TEST_ALLOC_ROOT); + ret = PTR_ERR(root); + goto out; + } + + btrfs_set_super_compat_ro_flags(root->fs_info->super_copy, + BTRFS_FEATURE_COMPAT_RO_FREE_SPACE_TREE); + root->root_key.objectid = BTRFS_FREE_SPACE_TREE_OBJECTID; + root->root_key.type = BTRFS_ROOT_ITEM_KEY; + root->root_key.offset = 0; + btrfs_global_root_insert(root); + root->fs_info->tree_root = root; + + root->node = alloc_test_extent_buffer(root->fs_info, nodesize); + if (IS_ERR(root->node)) { + test_std_err(TEST_ALLOC_EXTENT_BUFFER); + ret = PTR_ERR(root->node); + goto out; + } + btrfs_set_header_level(root->node, 0); + btrfs_set_header_nritems(root->node, 0); + root->alloc_bytenr += 2 * nodesize; + + cache = btrfs_alloc_dummy_block_group(fs_info, 8 * alignment); + if (!cache) { + test_std_err(TEST_ALLOC_BLOCK_GROUP); + ret = -ENOMEM; + goto out; + } + cache->bitmap_low_thresh = 0; + cache->bitmap_high_thresh = (u32)-1; + set_bit(BLOCK_GROUP_FLAG_NEEDS_FREE_SPACE, &cache->runtime_flags); + cache->fs_info = root->fs_info; + + btrfs_init_dummy_trans(&trans, root->fs_info); + + path = btrfs_alloc_path(); + if (!path) { + test_std_err(TEST_ALLOC_ROOT); + ret = -ENOMEM; + goto out; + } + + ret = btrfs_add_block_group_free_space(&trans, cache); + if (ret) { + test_err("could not add block group free space"); + goto out; + } + + if (bitmaps) { + ret = btrfs_convert_free_space_to_bitmaps(&trans, cache, path); + if (ret) { + test_err("could not convert block group to bitmaps"); + goto out; + } + } + + ret = test_func(&trans, root->fs_info, cache, path, alignment); + if (ret) + goto out; + + ret = btrfs_remove_block_group_free_space(&trans, cache); + if (ret) { + test_err("could not remove block group free space"); + goto out; + } + + if (btrfs_header_nritems(root->node) != 0) { + test_err("free space tree has leftover items"); + ret = -EINVAL; + goto out; + } + + ret = 0; +out: + btrfs_free_path(path); + btrfs_free_dummy_block_group(cache); + btrfs_free_dummy_root(root); + btrfs_free_dummy_fs_info(fs_info); + return ret; +} + +static int run_test_both_formats(test_func_t test_func, u32 sectorsize, + u32 nodesize, u32 alignment) +{ + int test_ret = 0; + int ret; + + ret = run_test(test_func, 0, sectorsize, nodesize, alignment); + if (ret) { + test_err( + "%ps failed with extents, sectorsize=%u, nodesize=%u, alignment=%u", + test_func, sectorsize, nodesize, alignment); + test_ret = ret; + } + + ret = run_test(test_func, 1, sectorsize, nodesize, alignment); + if (ret) { + test_err( + "%ps failed with bitmaps, sectorsize=%u, nodesize=%u, alignment=%u", + test_func, sectorsize, nodesize, alignment); + test_ret = ret; + } + + return test_ret; +} + +int btrfs_test_free_space_tree(u32 sectorsize, u32 nodesize) +{ + test_func_t tests[] = { + test_empty_block_group, + test_remove_all, + test_remove_beginning, + test_remove_end, + test_remove_middle, + test_merge_left, + test_merge_right, + test_merge_both, + test_merge_none, + }; + u32 bitmap_alignment; + int test_ret = 0; + int i; + + /* + * Align some operations to a page to flush out bugs in the extent + * buffer bitmap handling of highmem. + */ + bitmap_alignment = BTRFS_FREE_SPACE_BITMAP_BITS * PAGE_SIZE; + + test_msg("running free space tree tests"); + for (i = 0; i < ARRAY_SIZE(tests); i++) { + int ret; + + ret = run_test_both_formats(tests[i], sectorsize, nodesize, + sectorsize); + if (ret) + test_ret = ret; + + ret = run_test_both_formats(tests[i], sectorsize, nodesize, + bitmap_alignment); + if (ret) + test_ret = ret; + } + + return test_ret; +} diff --git a/fs/btrfs/tests/inode-tests.c b/fs/btrfs/tests/inode-tests.c new file mode 100644 index 000000000..b04fbcaf0 --- /dev/null +++ b/fs/btrfs/tests/inode-tests.c @@ -0,0 +1,1095 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2013 Fusion IO. All rights reserved. + */ + +#include +#include "btrfs-tests.h" +#include "../ctree.h" +#include "../btrfs_inode.h" +#include "../disk-io.h" +#include "../extent_io.h" +#include "../volumes.h" +#include "../compression.h" +#include "../accessors.h" + +static void insert_extent(struct btrfs_root *root, u64 start, u64 len, + u64 ram_bytes, u64 offset, u64 disk_bytenr, + u64 disk_len, u32 type, u8 compression, int slot) +{ + struct btrfs_path path; + struct btrfs_file_extent_item *fi; + struct extent_buffer *leaf = root->node; + struct btrfs_key key; + u32 value_len = sizeof(struct btrfs_file_extent_item); + + if (type == BTRFS_FILE_EXTENT_INLINE) + value_len += len; + memset(&path, 0, sizeof(path)); + + path.nodes[0] = leaf; + path.slots[0] = slot; + + key.objectid = BTRFS_FIRST_FREE_OBJECTID; + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = start; + + /* + * Passing a NULL trans handle is fine here, we have a dummy root eb + * and the tree is a single node (level 0). + */ + btrfs_setup_item_for_insert(NULL, root, &path, &key, value_len); + fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item); + btrfs_set_file_extent_generation(leaf, fi, 1); + btrfs_set_file_extent_type(leaf, fi, type); + btrfs_set_file_extent_disk_bytenr(leaf, fi, disk_bytenr); + btrfs_set_file_extent_disk_num_bytes(leaf, fi, disk_len); + btrfs_set_file_extent_offset(leaf, fi, offset); + btrfs_set_file_extent_num_bytes(leaf, fi, len); + btrfs_set_file_extent_ram_bytes(leaf, fi, ram_bytes); + btrfs_set_file_extent_compression(leaf, fi, compression); + btrfs_set_file_extent_encryption(leaf, fi, 0); + btrfs_set_file_extent_other_encoding(leaf, fi, 0); +} + +static void insert_inode_item_key(struct btrfs_root *root) +{ + struct btrfs_path path; + struct extent_buffer *leaf = root->node; + struct btrfs_key key; + u32 value_len = 0; + + memset(&path, 0, sizeof(path)); + + path.nodes[0] = leaf; + path.slots[0] = 0; + + key.objectid = BTRFS_INODE_ITEM_KEY; + key.type = BTRFS_INODE_ITEM_KEY; + key.offset = 0; + + /* + * Passing a NULL trans handle is fine here, we have a dummy root eb + * and the tree is a single node (level 0). + */ + btrfs_setup_item_for_insert(NULL, root, &path, &key, value_len); +} + +/* + * Build the most complicated map of extents the earth has ever seen. We want + * this so we can test all of the corner cases of btrfs_get_extent. Here is a + * diagram of how the extents will look though this may not be possible we still + * want to make sure everything acts normally (the last number is not inclusive) + * + * The numbers are using 4K fs block size as an example, the real test will scale + * all the extent maps (except the inlined one) according to the block size. + * + * [ 0 - 6 ][ 6 - 4K ][ 4K - 8K ][ 8K - 12K ] + * [ inline ][ implied hole ][ regular ][ regular1 split ] + * + * [ 12K - 16K ][ 16K - 24K ][ 24K - 28K ][ 28K - 32K ][ 32K - 36K ] + * [ hole ][ regular1 split ][ prealloc ][ prealloc1 ][ prealloc1 written ] + * + * [ 36K - 44K ][ 44K - 52K ][ 52K - 56K ][ 56K - 60K ][ 60K - 68 K ] + * [ prealloc1 ][ compressed ][ compressed1 ][ regular ][ compressed1 ] + * + * [ 68K - 72K ][ 72K - 84K ][ 84K - 88K ] + * [ regular ][ hole but no extent ][ regular ] + */ +static void setup_file_extents(struct btrfs_root *root, u32 sectorsize) +{ + int slot = 0; + u64 disk_bytenr = SZ_1M; + u64 offset = 0; + + /* + * Start 0, length 6, inlined. + * + * Tree-checker has strict limits on inline extents that they can only + * exist at file offset 0, thus we can only have one inline file extent + * at most. + */ + insert_extent(root, offset, 6, 6, 0, 0, 0, BTRFS_FILE_EXTENT_INLINE, 0, + slot); + slot++; + offset = sectorsize; + + /* Start 1 * blocksize, length 1 * blocksize, regular. */ + insert_extent(root, offset, sectorsize, sectorsize, 0, + disk_bytenr, sectorsize, BTRFS_FILE_EXTENT_REG, 0, slot); + slot++; + + /* We don't want the regular em merged with the next one. */ + disk_bytenr += 2 * sectorsize; + offset += sectorsize; + + /* + * Start 2 * blocksize, length 1 * blocksize, regular. + * + * Now for 3 extents that were split from a hole punch so we test + * offsets properly. + */ + insert_extent(root, offset, sectorsize, 4 * sectorsize, 0, disk_bytenr, + 4 * sectorsize, BTRFS_FILE_EXTENT_REG, 0, slot); + slot++; + offset += sectorsize; + + /* Start 3 * blocksize, length 1 * blocksize, regular, explicit hole. */ + insert_extent(root, offset, sectorsize, sectorsize, 0, 0, 0, + BTRFS_FILE_EXTENT_REG, 0, slot); + slot++; + offset += sectorsize; + + /* Start 4 * blocksize, length 2 * blocksize, regular. */ + insert_extent(root, offset, 2 * sectorsize, 4 * sectorsize, + 2 * sectorsize, disk_bytenr, 4 * sectorsize, + BTRFS_FILE_EXTENT_REG, 0, slot); + slot++; + offset += 2 * sectorsize; + disk_bytenr += 4 * sectorsize; + + /* Start 6 * blocksize, length 1 * blocksize, preallocated. */ + insert_extent(root, offset, sectorsize, sectorsize, 0, disk_bytenr, + sectorsize, BTRFS_FILE_EXTENT_PREALLOC, 0, slot); + slot++; + offset += sectorsize; + + /* + * We want to jack up disk_bytenr a little more so the em stuff doesn't + * merge our records. + */ + disk_bytenr += 2 * sectorsize; + + /* + * Start 7 * blocksize, length 1 * blocksize, prealloc. + * + * Now for a partially written prealloc extent, basically the same as + * the hole punch example above. Ram_bytes never changes when you mark + * extents written btw. + */ + insert_extent(root, offset, sectorsize, 4 * sectorsize, 0, disk_bytenr, + 4 * sectorsize, BTRFS_FILE_EXTENT_PREALLOC, 0, slot); + slot++; + offset += sectorsize; + + /* Start 8 * blocksize, length 1 * blocksize, regular. */ + insert_extent(root, offset, sectorsize, 4 * sectorsize, sectorsize, + disk_bytenr, 4 * sectorsize, BTRFS_FILE_EXTENT_REG, 0, + slot); + slot++; + offset += sectorsize; + + /* Start 9 * blocksize, length 2 * blocksize, prealloc. */ + insert_extent(root, offset, 2 * sectorsize, 4 * sectorsize, + 2 * sectorsize, disk_bytenr, 4 * sectorsize, + BTRFS_FILE_EXTENT_PREALLOC, 0, slot); + slot++; + offset += 2 * sectorsize; + disk_bytenr += 4 * sectorsize; + + /* Start 11 * blocksize, length 2 * blocksize, regular. */ + insert_extent(root, offset, 2 * sectorsize, 2 * sectorsize, 0, + disk_bytenr, sectorsize, BTRFS_FILE_EXTENT_REG, + BTRFS_COMPRESS_ZLIB, slot); + slot++; + offset += 2 * sectorsize; + /* No merges */ + disk_bytenr += 2 * sectorsize; + + /* Start 13 * blocksize, length 1 * blocksize, regular. */ + insert_extent(root, offset, sectorsize, 4 * sectorsize, 0, disk_bytenr, + sectorsize, BTRFS_FILE_EXTENT_REG, + BTRFS_COMPRESS_ZLIB, slot); + slot++; + offset += sectorsize; + + /* Start 14 * blocksize, length 1 * blocksize, regular. */ + insert_extent(root, offset, sectorsize, sectorsize, 0, + disk_bytenr + sectorsize, sectorsize, + BTRFS_FILE_EXTENT_REG, 0, slot); + slot++; + offset += sectorsize; + + /* Start 15 * blocksize, length 2 * blocksize, regular. */ + insert_extent(root, offset, 2 * sectorsize, 4 * sectorsize, + 2 * sectorsize, disk_bytenr, sectorsize, + BTRFS_FILE_EXTENT_REG, BTRFS_COMPRESS_ZLIB, slot); + slot++; + offset += 2 * sectorsize; + disk_bytenr += 2 * sectorsize; + + /* Start 17 * blocksize, length 1 * blocksize, regular. */ + insert_extent(root, offset, sectorsize, sectorsize, 0, disk_bytenr, + sectorsize, BTRFS_FILE_EXTENT_REG, 0, slot); + slot++; + offset += 4 * sectorsize; + disk_bytenr += sectorsize; + + /* + * Start 18 * blocksize, length 3 * blocksize, implied hole (aka no + * file extent item). + * + * Start 21 * blocksize, length 1 * blocksize, regular. + */ + insert_extent(root, offset, sectorsize, sectorsize, 0, disk_bytenr, + sectorsize, BTRFS_FILE_EXTENT_REG, 0, slot); +} + +static u32 prealloc_only = 0; +static u32 compressed_only = 0; +static u32 vacancy_only = 0; + +static noinline int test_btrfs_get_extent(u32 sectorsize, u32 nodesize) +{ + struct btrfs_fs_info *fs_info = NULL; + struct inode *inode = NULL; + struct btrfs_root *root = NULL; + struct extent_map *em = NULL; + u64 orig_start; + u64 disk_bytenr; + u64 offset; + int ret = -ENOMEM; + + test_msg("running btrfs_get_extent tests"); + + inode = btrfs_new_test_inode(); + if (!inode) { + test_std_err(TEST_ALLOC_INODE); + return ret; + } + + fs_info = btrfs_alloc_dummy_fs_info(nodesize, sectorsize); + if (!fs_info) { + test_std_err(TEST_ALLOC_FS_INFO); + goto out; + } + + root = btrfs_alloc_dummy_root(fs_info); + if (IS_ERR(root)) { + test_std_err(TEST_ALLOC_ROOT); + goto out; + } + + root->node = alloc_dummy_extent_buffer(fs_info, nodesize); + if (!root->node) { + test_std_err(TEST_ALLOC_ROOT); + goto out; + } + + btrfs_set_header_nritems(root->node, 0); + btrfs_set_header_level(root->node, 0); + ret = -EINVAL; + + /* First with no extents */ + BTRFS_I(inode)->root = root; + em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, sectorsize); + if (IS_ERR(em)) { + em = NULL; + test_err("got an error when we shouldn't have"); + goto out; + } + if (em->disk_bytenr != EXTENT_MAP_HOLE) { + test_err("expected a hole, got %llu", em->disk_bytenr); + goto out; + } + btrfs_free_extent_map(em); + btrfs_drop_extent_map_range(BTRFS_I(inode), 0, (u64)-1, false); + + /* + * All of the magic numbers are based on the mapping setup in + * setup_file_extents, so if you change anything there you need to + * update the comment and update the expected values below. + */ + setup_file_extents(root, sectorsize); + + em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, (u64)-1); + if (IS_ERR(em)) { + test_err("got an error when we shouldn't have"); + goto out; + } + if (em->disk_bytenr != EXTENT_MAP_INLINE) { + test_err("expected an inline, got %llu", em->disk_bytenr); + goto out; + } + + /* + * For inline extent, we always round up the em to sectorsize, as + * they are either: + * + * a) a hidden hole + * The range will be zeroed at inline extent read time. + * + * b) a file extent with unaligned bytenr + * Tree checker will reject it. + */ + if (em->start != 0 || em->len != sectorsize) { + test_err( + "unexpected extent wanted start 0 len %u, got start %llu len %llu", + sectorsize, em->start, em->len); + goto out; + } + if (em->flags != 0) { + test_err("unexpected flags set, want 0 have %u", em->flags); + goto out; + } + /* + * We don't test anything else for inline since it doesn't get set + * unless we have a page for it to write into. Maybe we should change + * this? + */ + offset = btrfs_extent_map_end(em); + btrfs_free_extent_map(em); + + /* Regular extent */ + em = btrfs_get_extent(BTRFS_I(inode), NULL, offset, sectorsize); + if (IS_ERR(em)) { + test_err("got an error when we shouldn't have"); + goto out; + } + if (em->disk_bytenr >= EXTENT_MAP_LAST_BYTE) { + test_err("expected a real extent, got %llu", em->disk_bytenr); + goto out; + } + if (em->start != offset || em->len != sectorsize) { + test_err( + "unexpected extent wanted start %llu len %u, got start %llu len %llu", + offset, sectorsize, em->start, em->len); + goto out; + } + if (em->flags != 0) { + test_err("unexpected flags set, want 0 have %u", em->flags); + goto out; + } + if (em->offset != 0) { + test_err("wrong offset, want 0, have %llu", em->offset); + goto out; + } + offset = btrfs_extent_map_end(em); + btrfs_free_extent_map(em); + + /* The next 3 are split extents */ + em = btrfs_get_extent(BTRFS_I(inode), NULL, offset, sectorsize); + if (IS_ERR(em)) { + test_err("got an error when we shouldn't have"); + goto out; + } + if (em->disk_bytenr >= EXTENT_MAP_LAST_BYTE) { + test_err("expected a real extent, got %llu", em->disk_bytenr); + goto out; + } + if (em->start != offset || em->len != sectorsize) { + test_err( + "unexpected extent start %llu len %u, got start %llu len %llu", + offset, sectorsize, em->start, em->len); + goto out; + } + if (em->flags != 0) { + test_err("unexpected flags set, want 0 have %u", em->flags); + goto out; + } + if (em->offset != 0) { + test_err("wrong offset, want 0, have %llu", em->offset); + goto out; + } + disk_bytenr = btrfs_extent_map_block_start(em); + orig_start = em->start; + offset = btrfs_extent_map_end(em); + btrfs_free_extent_map(em); + + em = btrfs_get_extent(BTRFS_I(inode), NULL, offset, sectorsize); + if (IS_ERR(em)) { + test_err("got an error when we shouldn't have"); + goto out; + } + if (em->disk_bytenr != EXTENT_MAP_HOLE) { + test_err("expected a hole, got %llu", em->disk_bytenr); + goto out; + } + if (em->start != offset || em->len != sectorsize) { + test_err( + "unexpected extent wanted start %llu len %u, got start %llu len %llu", + offset, sectorsize, em->start, em->len); + goto out; + } + if (em->flags != 0) { + test_err("unexpected flags set, want 0 have %u", em->flags); + goto out; + } + offset = btrfs_extent_map_end(em); + btrfs_free_extent_map(em); + + em = btrfs_get_extent(BTRFS_I(inode), NULL, offset, sectorsize); + if (IS_ERR(em)) { + test_err("got an error when we shouldn't have"); + goto out; + } + if (em->disk_bytenr >= EXTENT_MAP_LAST_BYTE) { + test_err("expected a real extent, got %llu", em->disk_bytenr); + goto out; + } + if (em->start != offset || em->len != 2 * sectorsize) { + test_err( + "unexpected extent wanted start %llu len %u, got start %llu len %llu", + offset, 2 * sectorsize, em->start, em->len); + goto out; + } + if (em->flags != 0) { + test_err("unexpected flags set, want 0 have %u", em->flags); + goto out; + } + if (em->start - em->offset != orig_start) { + test_err("wrong offset, em->start=%llu em->offset=%llu orig_start=%llu", + em->start, em->offset, orig_start); + goto out; + } + disk_bytenr += (em->start - orig_start); + if (btrfs_extent_map_block_start(em) != disk_bytenr) { + test_err("wrong block start, want %llu, have %llu", + disk_bytenr, btrfs_extent_map_block_start(em)); + goto out; + } + offset = btrfs_extent_map_end(em); + btrfs_free_extent_map(em); + + /* Prealloc extent */ + em = btrfs_get_extent(BTRFS_I(inode), NULL, offset, sectorsize); + if (IS_ERR(em)) { + test_err("got an error when we shouldn't have"); + goto out; + } + if (em->disk_bytenr >= EXTENT_MAP_LAST_BYTE) { + test_err("expected a real extent, got %llu", em->disk_bytenr); + goto out; + } + if (em->start != offset || em->len != sectorsize) { + test_err( + "unexpected extent wanted start %llu len %u, got start %llu len %llu", + offset, sectorsize, em->start, em->len); + goto out; + } + if (em->flags != prealloc_only) { + test_err("unexpected flags set, want %u have %u", + prealloc_only, em->flags); + goto out; + } + if (em->offset != 0) { + test_err("wrong offset, want 0, have %llu", em->offset); + goto out; + } + offset = btrfs_extent_map_end(em); + btrfs_free_extent_map(em); + + /* The next 3 are a half written prealloc extent */ + em = btrfs_get_extent(BTRFS_I(inode), NULL, offset, sectorsize); + if (IS_ERR(em)) { + test_err("got an error when we shouldn't have"); + goto out; + } + if (em->disk_bytenr >= EXTENT_MAP_LAST_BYTE) { + test_err("expected a real extent, got %llu", em->disk_bytenr); + goto out; + } + if (em->start != offset || em->len != sectorsize) { + test_err( + "unexpected extent wanted start %llu len %u, got start %llu len %llu", + offset, sectorsize, em->start, em->len); + goto out; + } + if (em->flags != prealloc_only) { + test_err("unexpected flags set, want %u have %u", + prealloc_only, em->flags); + goto out; + } + if (em->offset != 0) { + test_err("wrong offset, want 0, have %llu", em->offset); + goto out; + } + disk_bytenr = btrfs_extent_map_block_start(em); + orig_start = em->start; + offset = btrfs_extent_map_end(em); + btrfs_free_extent_map(em); + + em = btrfs_get_extent(BTRFS_I(inode), NULL, offset, sectorsize); + if (IS_ERR(em)) { + test_err("got an error when we shouldn't have"); + goto out; + } + if (em->disk_bytenr >= EXTENT_MAP_HOLE) { + test_err("expected a real extent, got %llu", em->disk_bytenr); + goto out; + } + if (em->start != offset || em->len != sectorsize) { + test_err( + "unexpected extent wanted start %llu len %u, got start %llu len %llu", + offset, sectorsize, em->start, em->len); + goto out; + } + if (em->flags != 0) { + test_err("unexpected flags set, want 0 have %u", em->flags); + goto out; + } + if (em->start - em->offset != orig_start) { + test_err("unexpected offset, wanted %llu, have %llu", + em->start - orig_start, em->offset); + goto out; + } + if (btrfs_extent_map_block_start(em) != disk_bytenr + em->offset) { + test_err("unexpected block start, wanted %llu, have %llu", + disk_bytenr + em->offset, btrfs_extent_map_block_start(em)); + goto out; + } + offset = btrfs_extent_map_end(em); + btrfs_free_extent_map(em); + + em = btrfs_get_extent(BTRFS_I(inode), NULL, offset, sectorsize); + if (IS_ERR(em)) { + test_err("got an error when we shouldn't have"); + goto out; + } + if (em->disk_bytenr >= EXTENT_MAP_LAST_BYTE) { + test_err("expected a real extent, got %llu", em->disk_bytenr); + goto out; + } + if (em->start != offset || em->len != 2 * sectorsize) { + test_err( + "unexpected extent wanted start %llu len %u, got start %llu len %llu", + offset, 2 * sectorsize, em->start, em->len); + goto out; + } + if (em->flags != prealloc_only) { + test_err("unexpected flags set, want %u have %u", + prealloc_only, em->flags); + goto out; + } + if (em->start - em->offset != orig_start) { + test_err("wrong offset, em->start=%llu em->offset=%llu orig_start=%llu", + em->start, em->offset, orig_start); + goto out; + } + if (btrfs_extent_map_block_start(em) != disk_bytenr + em->offset) { + test_err("unexpected block start, wanted %llu, have %llu", + disk_bytenr + em->offset, btrfs_extent_map_block_start(em)); + goto out; + } + offset = btrfs_extent_map_end(em); + btrfs_free_extent_map(em); + + /* Now for the compressed extent */ + em = btrfs_get_extent(BTRFS_I(inode), NULL, offset, sectorsize); + if (IS_ERR(em)) { + test_err("got an error when we shouldn't have"); + goto out; + } + if (em->disk_bytenr >= EXTENT_MAP_LAST_BYTE) { + test_err("expected a real extent, got %llu", em->disk_bytenr); + goto out; + } + if (em->start != offset || em->len != 2 * sectorsize) { + test_err( + "unexpected extent wanted start %llu len %u, got start %llu len %llu", + offset, 2 * sectorsize, em->start, em->len); + goto out; + } + if (em->flags != compressed_only) { + test_err("unexpected flags set, want %u have %u", + compressed_only, em->flags); + goto out; + } + if (em->offset != 0) { + test_err("wrong offset, want 0, have %llu", em->offset); + goto out; + } + if (btrfs_extent_map_compression(em) != BTRFS_COMPRESS_ZLIB) { + test_err("unexpected compress type, wanted %d, got %d", + BTRFS_COMPRESS_ZLIB, btrfs_extent_map_compression(em)); + goto out; + } + offset = btrfs_extent_map_end(em); + btrfs_free_extent_map(em); + + /* Split compressed extent */ + em = btrfs_get_extent(BTRFS_I(inode), NULL, offset, sectorsize); + if (IS_ERR(em)) { + test_err("got an error when we shouldn't have"); + goto out; + } + if (em->disk_bytenr >= EXTENT_MAP_LAST_BYTE) { + test_err("expected a real extent, got %llu", em->disk_bytenr); + goto out; + } + if (em->start != offset || em->len != sectorsize) { + test_err( + "unexpected extent wanted start %llu len %u, got start %llu len %llu", + offset, sectorsize, em->start, em->len); + goto out; + } + if (em->flags != compressed_only) { + test_err("unexpected flags set, want %u have %u", + compressed_only, em->flags); + goto out; + } + if (em->offset != 0) { + test_err("wrong offset, want 0, have %llu", em->offset); + goto out; + } + if (btrfs_extent_map_compression(em) != BTRFS_COMPRESS_ZLIB) { + test_err("unexpected compress type, wanted %d, got %d", + BTRFS_COMPRESS_ZLIB, btrfs_extent_map_compression(em)); + goto out; + } + disk_bytenr = btrfs_extent_map_block_start(em); + orig_start = em->start; + offset = btrfs_extent_map_end(em); + btrfs_free_extent_map(em); + + em = btrfs_get_extent(BTRFS_I(inode), NULL, offset, sectorsize); + if (IS_ERR(em)) { + test_err("got an error when we shouldn't have"); + goto out; + } + if (em->disk_bytenr >= EXTENT_MAP_LAST_BYTE) { + test_err("expected a real extent, got %llu", em->disk_bytenr); + goto out; + } + if (em->start != offset || em->len != sectorsize) { + test_err( + "unexpected extent wanted start %llu len %u, got start %llu len %llu", + offset, sectorsize, em->start, em->len); + goto out; + } + if (em->flags != 0) { + test_err("unexpected flags set, want 0 have %u", em->flags); + goto out; + } + if (em->offset != 0) { + test_err("wrong offset, want 0, have %llu", em->offset); + goto out; + } + offset = btrfs_extent_map_end(em); + btrfs_free_extent_map(em); + + em = btrfs_get_extent(BTRFS_I(inode), NULL, offset, sectorsize); + if (IS_ERR(em)) { + test_err("got an error when we shouldn't have"); + goto out; + } + if (btrfs_extent_map_block_start(em) != disk_bytenr) { + test_err("block start does not match, want %llu got %llu", + disk_bytenr, btrfs_extent_map_block_start(em)); + goto out; + } + if (em->start != offset || em->len != 2 * sectorsize) { + test_err( + "unexpected extent wanted start %llu len %u, got start %llu len %llu", + offset, 2 * sectorsize, em->start, em->len); + goto out; + } + if (em->flags != compressed_only) { + test_err("unexpected flags set, want %u have %u", + compressed_only, em->flags); + goto out; + } + if (em->start - em->offset != orig_start) { + test_err("wrong offset, em->start=%llu em->offset=%llu orig_start=%llu", + em->start, em->offset, orig_start); + goto out; + } + if (btrfs_extent_map_compression(em) != BTRFS_COMPRESS_ZLIB) { + test_err("unexpected compress type, wanted %d, got %d", + BTRFS_COMPRESS_ZLIB, btrfs_extent_map_compression(em)); + goto out; + } + offset = btrfs_extent_map_end(em); + btrfs_free_extent_map(em); + + /* A hole between regular extents but no hole extent */ + em = btrfs_get_extent(BTRFS_I(inode), NULL, offset + 6, sectorsize); + if (IS_ERR(em)) { + test_err("got an error when we shouldn't have"); + goto out; + } + if (em->disk_bytenr >= EXTENT_MAP_LAST_BYTE) { + test_err("expected a real extent, got %llu", em->disk_bytenr); + goto out; + } + if (em->start != offset || em->len != sectorsize) { + test_err( + "unexpected extent wanted start %llu len %u, got start %llu len %llu", + offset, sectorsize, em->start, em->len); + goto out; + } + if (em->flags != 0) { + test_err("unexpected flags set, want 0 have %u", em->flags); + goto out; + } + if (em->offset != 0) { + test_err("wrong offset, want 0, have %llu", em->offset); + goto out; + } + offset = btrfs_extent_map_end(em); + btrfs_free_extent_map(em); + + em = btrfs_get_extent(BTRFS_I(inode), NULL, offset, SZ_4M); + if (IS_ERR(em)) { + test_err("got an error when we shouldn't have"); + goto out; + } + if (em->disk_bytenr != EXTENT_MAP_HOLE) { + test_err("expected a hole extent, got %llu", em->disk_bytenr); + goto out; + } + /* + * Currently we just return a length that we requested rather than the + * length of the actual hole, if this changes we'll have to change this + * test. + */ + if (em->start != offset || em->len != 3 * sectorsize) { + test_err( + "unexpected extent wanted start %llu len %u, got start %llu len %llu", + offset, 3 * sectorsize, em->start, em->len); + goto out; + } + if (em->flags != vacancy_only) { + test_err("unexpected flags set, want %u have %u", + vacancy_only, em->flags); + goto out; + } + if (em->offset != 0) { + test_err("wrong offset, want 0, have %llu", em->offset); + goto out; + } + offset = btrfs_extent_map_end(em); + btrfs_free_extent_map(em); + + em = btrfs_get_extent(BTRFS_I(inode), NULL, offset, sectorsize); + if (IS_ERR(em)) { + test_err("got an error when we shouldn't have"); + goto out; + } + if (em->disk_bytenr >= EXTENT_MAP_LAST_BYTE) { + test_err("expected a real extent, got %llu", em->disk_bytenr); + goto out; + } + if (em->start != offset || em->len != sectorsize) { + test_err( + "unexpected extent wanted start %llu len %u, got start %llu len %llu", + offset, sectorsize, em->start, em->len); + goto out; + } + if (em->flags != 0) { + test_err("unexpected flags set, want 0 have %u", em->flags); + goto out; + } + if (em->offset != 0) { + test_err("wrong orig offset, want 0, have %llu", em->offset); + goto out; + } + ret = 0; +out: + if (!IS_ERR(em)) + btrfs_free_extent_map(em); + iput(inode); + btrfs_free_dummy_root(root); + btrfs_free_dummy_fs_info(fs_info); + return ret; +} + +static int test_hole_first(u32 sectorsize, u32 nodesize) +{ + struct btrfs_fs_info *fs_info = NULL; + struct inode *inode = NULL; + struct btrfs_root *root = NULL; + struct extent_map *em = NULL; + int ret = -ENOMEM; + + test_msg("running hole first btrfs_get_extent test"); + + inode = btrfs_new_test_inode(); + if (!inode) { + test_std_err(TEST_ALLOC_INODE); + return ret; + } + + fs_info = btrfs_alloc_dummy_fs_info(nodesize, sectorsize); + if (!fs_info) { + test_std_err(TEST_ALLOC_FS_INFO); + goto out; + } + + root = btrfs_alloc_dummy_root(fs_info); + if (IS_ERR(root)) { + test_std_err(TEST_ALLOC_ROOT); + goto out; + } + + root->node = alloc_dummy_extent_buffer(fs_info, nodesize); + if (!root->node) { + test_std_err(TEST_ALLOC_ROOT); + goto out; + } + + btrfs_set_header_nritems(root->node, 0); + btrfs_set_header_level(root->node, 0); + BTRFS_I(inode)->root = root; + ret = -EINVAL; + + /* + * Need a blank inode item here just so we don't confuse + * btrfs_get_extent. + */ + insert_inode_item_key(root); + insert_extent(root, sectorsize, sectorsize, sectorsize, 0, sectorsize, + sectorsize, BTRFS_FILE_EXTENT_REG, 0, 1); + em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, 2 * sectorsize); + if (IS_ERR(em)) { + test_err("got an error when we shouldn't have"); + goto out; + } + if (em->disk_bytenr != EXTENT_MAP_HOLE) { + test_err("expected a hole, got %llu", em->disk_bytenr); + goto out; + } + if (em->start != 0 || em->len != sectorsize) { + test_err( + "unexpected extent wanted start 0 len %u, got start %llu len %llu", + sectorsize, em->start, em->len); + goto out; + } + if (em->flags != vacancy_only) { + test_err("wrong flags, wanted %u, have %u", vacancy_only, + em->flags); + goto out; + } + btrfs_free_extent_map(em); + + em = btrfs_get_extent(BTRFS_I(inode), NULL, sectorsize, 2 * sectorsize); + if (IS_ERR(em)) { + test_err("got an error when we shouldn't have"); + goto out; + } + if (btrfs_extent_map_block_start(em) != sectorsize) { + test_err("expected a real extent, got %llu", + btrfs_extent_map_block_start(em)); + goto out; + } + if (em->start != sectorsize || em->len != sectorsize) { + test_err( + "unexpected extent wanted start %u len %u, got start %llu len %llu", + sectorsize, sectorsize, em->start, em->len); + goto out; + } + if (em->flags != 0) { + test_err("unexpected flags set, wanted 0 got %u", + em->flags); + goto out; + } + ret = 0; +out: + if (!IS_ERR(em)) + btrfs_free_extent_map(em); + iput(inode); + btrfs_free_dummy_root(root); + btrfs_free_dummy_fs_info(fs_info); + return ret; +} + +static int test_extent_accounting(u32 sectorsize, u32 nodesize) +{ + struct btrfs_fs_info *fs_info = NULL; + struct inode *inode = NULL; + struct btrfs_root *root = NULL; + int ret = -ENOMEM; + + test_msg("running outstanding_extents tests"); + + inode = btrfs_new_test_inode(); + if (!inode) { + test_std_err(TEST_ALLOC_INODE); + return ret; + } + + fs_info = btrfs_alloc_dummy_fs_info(nodesize, sectorsize); + if (!fs_info) { + test_std_err(TEST_ALLOC_FS_INFO); + goto out; + } + + root = btrfs_alloc_dummy_root(fs_info); + if (IS_ERR(root)) { + test_std_err(TEST_ALLOC_ROOT); + goto out; + } + + BTRFS_I(inode)->root = root; + + /* [BTRFS_MAX_EXTENT_SIZE] */ + ret = btrfs_set_extent_delalloc(BTRFS_I(inode), 0, + BTRFS_MAX_EXTENT_SIZE - 1, 0, NULL); + if (ret) { + test_err("btrfs_set_extent_delalloc returned %d", ret); + goto out; + } + if (BTRFS_I(inode)->outstanding_extents != 1) { + ret = -EINVAL; + test_err("miscount, wanted 1, got %u", + BTRFS_I(inode)->outstanding_extents); + goto out; + } + + /* [BTRFS_MAX_EXTENT_SIZE][sectorsize] */ + ret = btrfs_set_extent_delalloc(BTRFS_I(inode), BTRFS_MAX_EXTENT_SIZE, + BTRFS_MAX_EXTENT_SIZE + sectorsize - 1, + 0, NULL); + if (ret) { + test_err("btrfs_set_extent_delalloc returned %d", ret); + goto out; + } + if (BTRFS_I(inode)->outstanding_extents != 2) { + ret = -EINVAL; + test_err("miscount, wanted 2, got %u", + BTRFS_I(inode)->outstanding_extents); + goto out; + } + + /* [BTRFS_MAX_EXTENT_SIZE/2][sectorsize HOLE][the rest] */ + ret = btrfs_clear_extent_bit(&BTRFS_I(inode)->io_tree, + BTRFS_MAX_EXTENT_SIZE >> 1, + (BTRFS_MAX_EXTENT_SIZE >> 1) + sectorsize - 1, + EXTENT_DELALLOC | EXTENT_DELALLOC_NEW, NULL); + if (ret) { + test_err("clear_extent_bit returned %d", ret); + goto out; + } + if (BTRFS_I(inode)->outstanding_extents != 2) { + ret = -EINVAL; + test_err("miscount, wanted 2, got %u", + BTRFS_I(inode)->outstanding_extents); + goto out; + } + + /* [BTRFS_MAX_EXTENT_SIZE][sectorsize] */ + ret = btrfs_set_extent_delalloc(BTRFS_I(inode), BTRFS_MAX_EXTENT_SIZE >> 1, + (BTRFS_MAX_EXTENT_SIZE >> 1) + + sectorsize - 1, + 0, NULL); + if (ret) { + test_err("btrfs_set_extent_delalloc returned %d", ret); + goto out; + } + if (BTRFS_I(inode)->outstanding_extents != 2) { + ret = -EINVAL; + test_err("miscount, wanted 2, got %u", + BTRFS_I(inode)->outstanding_extents); + goto out; + } + + /* + * [BTRFS_MAX_EXTENT_SIZE+sectorsize][sectorsize HOLE][BTRFS_MAX_EXTENT_SIZE+sectorsize] + */ + ret = btrfs_set_extent_delalloc(BTRFS_I(inode), + BTRFS_MAX_EXTENT_SIZE + 2 * sectorsize, + (BTRFS_MAX_EXTENT_SIZE << 1) + 3 * sectorsize - 1, + 0, NULL); + if (ret) { + test_err("btrfs_set_extent_delalloc returned %d", ret); + goto out; + } + if (BTRFS_I(inode)->outstanding_extents != 4) { + ret = -EINVAL; + test_err("miscount, wanted 4, got %u", + BTRFS_I(inode)->outstanding_extents); + goto out; + } + + /* + * [BTRFS_MAX_EXTENT_SIZE+sectorsize][sectorsize][BTRFS_MAX_EXTENT_SIZE+sectorsize] + */ + ret = btrfs_set_extent_delalloc(BTRFS_I(inode), + BTRFS_MAX_EXTENT_SIZE + sectorsize, + BTRFS_MAX_EXTENT_SIZE + 2 * sectorsize - 1, 0, NULL); + if (ret) { + test_err("btrfs_set_extent_delalloc returned %d", ret); + goto out; + } + if (BTRFS_I(inode)->outstanding_extents != 3) { + ret = -EINVAL; + test_err("miscount, wanted 3, got %u", + BTRFS_I(inode)->outstanding_extents); + goto out; + } + + /* [BTRFS_MAX_EXTENT_SIZE+4k][4K HOLE][BTRFS_MAX_EXTENT_SIZE+4k] */ + ret = btrfs_clear_extent_bit(&BTRFS_I(inode)->io_tree, + BTRFS_MAX_EXTENT_SIZE + sectorsize, + BTRFS_MAX_EXTENT_SIZE + 2 * sectorsize - 1, + EXTENT_DELALLOC | EXTENT_DELALLOC_NEW, NULL); + if (ret) { + test_err("clear_extent_bit returned %d", ret); + goto out; + } + if (BTRFS_I(inode)->outstanding_extents != 4) { + ret = -EINVAL; + test_err("miscount, wanted 4, got %u", + BTRFS_I(inode)->outstanding_extents); + goto out; + } + + /* + * Refill the hole again just for good measure, because I thought it + * might fail and I'd rather satisfy my paranoia at this point. + */ + ret = btrfs_set_extent_delalloc(BTRFS_I(inode), + BTRFS_MAX_EXTENT_SIZE + sectorsize, + BTRFS_MAX_EXTENT_SIZE + 2 * sectorsize - 1, 0, NULL); + if (ret) { + test_err("btrfs_set_extent_delalloc returned %d", ret); + goto out; + } + if (BTRFS_I(inode)->outstanding_extents != 3) { + ret = -EINVAL; + test_err("miscount, wanted 3, got %u", + BTRFS_I(inode)->outstanding_extents); + goto out; + } + + /* Empty */ + ret = btrfs_clear_extent_bit(&BTRFS_I(inode)->io_tree, 0, (u64)-1, + EXTENT_DELALLOC | EXTENT_DELALLOC_NEW, NULL); + if (ret) { + test_err("clear_extent_bit returned %d", ret); + goto out; + } + if (BTRFS_I(inode)->outstanding_extents) { + ret = -EINVAL; + test_err("miscount, wanted 0, got %u", + BTRFS_I(inode)->outstanding_extents); + goto out; + } + ret = 0; +out: + if (ret) + btrfs_clear_extent_bit(&BTRFS_I(inode)->io_tree, 0, (u64)-1, + EXTENT_DELALLOC | EXTENT_DELALLOC_NEW, NULL); + iput(inode); + btrfs_free_dummy_root(root); + btrfs_free_dummy_fs_info(fs_info); + return ret; +} + +int btrfs_test_inodes(u32 sectorsize, u32 nodesize) +{ + int ret; + + test_msg("running inode tests"); + + compressed_only |= EXTENT_FLAG_COMPRESS_ZLIB; + prealloc_only |= EXTENT_FLAG_PREALLOC; + + ret = test_btrfs_get_extent(sectorsize, nodesize); + if (ret) + return ret; + ret = test_hole_first(sectorsize, nodesize); + if (ret) + return ret; + return test_extent_accounting(sectorsize, nodesize); +} diff --git a/fs/btrfs/tests/qgroup-tests.c b/fs/btrfs/tests/qgroup-tests.c new file mode 100644 index 000000000..0fcc31bee --- /dev/null +++ b/fs/btrfs/tests/qgroup-tests.c @@ -0,0 +1,550 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2013 Facebook. All rights reserved. + */ + +#include +#include "btrfs-tests.h" +#include "../ctree.h" +#include "../transaction.h" +#include "../disk-io.h" +#include "../qgroup.h" +#include "../backref.h" +#include "../fs.h" +#include "../accessors.h" + +static int insert_normal_tree_ref(struct btrfs_root *root, u64 bytenr, + u64 num_bytes, u64 parent, u64 root_objectid) +{ + struct btrfs_trans_handle trans; + struct btrfs_extent_item *item; + struct btrfs_extent_inline_ref *iref; + struct btrfs_tree_block_info *block_info; + BTRFS_PATH_AUTO_FREE(path); + struct extent_buffer *leaf; + struct btrfs_key ins; + u32 size = sizeof(*item) + sizeof(*iref) + sizeof(*block_info); + int ret; + + btrfs_init_dummy_trans(&trans, NULL); + + ins.objectid = bytenr; + ins.type = BTRFS_EXTENT_ITEM_KEY; + ins.offset = num_bytes; + + path = btrfs_alloc_path(); + if (!path) { + test_std_err(TEST_ALLOC_ROOT); + return -ENOMEM; + } + + ret = btrfs_insert_empty_item(&trans, root, path, &ins, size); + if (ret) { + test_err("couldn't insert ref %d", ret); + return ret; + } + + leaf = path->nodes[0]; + item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); + btrfs_set_extent_refs(leaf, item, 1); + btrfs_set_extent_generation(leaf, item, 1); + btrfs_set_extent_flags(leaf, item, BTRFS_EXTENT_FLAG_TREE_BLOCK); + block_info = (struct btrfs_tree_block_info *)(item + 1); + btrfs_set_tree_block_level(leaf, block_info, 0); + iref = (struct btrfs_extent_inline_ref *)(block_info + 1); + if (parent > 0) { + btrfs_set_extent_inline_ref_type(leaf, iref, + BTRFS_SHARED_BLOCK_REF_KEY); + btrfs_set_extent_inline_ref_offset(leaf, iref, parent); + } else { + btrfs_set_extent_inline_ref_type(leaf, iref, BTRFS_TREE_BLOCK_REF_KEY); + btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid); + } + return 0; +} + +static int add_tree_ref(struct btrfs_root *root, u64 bytenr, u64 num_bytes, + u64 parent, u64 root_objectid) +{ + struct btrfs_trans_handle trans; + struct btrfs_extent_item *item; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + u64 refs; + int ret; + + btrfs_init_dummy_trans(&trans, NULL); + + key.objectid = bytenr; + key.type = BTRFS_EXTENT_ITEM_KEY; + key.offset = num_bytes; + + path = btrfs_alloc_path(); + if (!path) { + test_std_err(TEST_ALLOC_ROOT); + return -ENOMEM; + } + + ret = btrfs_search_slot(&trans, root, &key, path, 0, 1); + if (ret) { + test_err("couldn't find extent ref"); + return ret; + } + + item = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_extent_item); + refs = btrfs_extent_refs(path->nodes[0], item); + btrfs_set_extent_refs(path->nodes[0], item, refs + 1); + btrfs_release_path(path); + + key.objectid = bytenr; + if (parent) { + key.type = BTRFS_SHARED_BLOCK_REF_KEY; + key.offset = parent; + } else { + key.type = BTRFS_TREE_BLOCK_REF_KEY; + key.offset = root_objectid; + } + + ret = btrfs_insert_empty_item(&trans, root, path, &key, 0); + if (ret) + test_err("failed to insert backref"); + return ret; +} + +static int remove_extent_item(struct btrfs_root *root, u64 bytenr, + u64 num_bytes) +{ + struct btrfs_trans_handle trans; + struct btrfs_key key; + BTRFS_PATH_AUTO_FREE(path); + int ret; + + btrfs_init_dummy_trans(&trans, NULL); + + key.objectid = bytenr; + key.type = BTRFS_EXTENT_ITEM_KEY; + key.offset = num_bytes; + + path = btrfs_alloc_path(); + if (!path) { + test_std_err(TEST_ALLOC_ROOT); + return -ENOMEM; + } + + ret = btrfs_search_slot(&trans, root, &key, path, -1, 1); + if (ret) { + test_err("didn't find our key %d", ret); + return ret; + } + btrfs_del_item(&trans, root, path); + return 0; +} + +static int remove_extent_ref(struct btrfs_root *root, u64 bytenr, + u64 num_bytes, u64 parent, u64 root_objectid) +{ + struct btrfs_trans_handle trans; + struct btrfs_extent_item *item; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + u64 refs; + int ret; + + btrfs_init_dummy_trans(&trans, NULL); + + key.objectid = bytenr; + key.type = BTRFS_EXTENT_ITEM_KEY; + key.offset = num_bytes; + + path = btrfs_alloc_path(); + if (!path) { + test_std_err(TEST_ALLOC_ROOT); + return -ENOMEM; + } + + ret = btrfs_search_slot(&trans, root, &key, path, 0, 1); + if (ret) { + test_err("couldn't find extent ref"); + return ret; + } + + item = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_extent_item); + refs = btrfs_extent_refs(path->nodes[0], item); + btrfs_set_extent_refs(path->nodes[0], item, refs - 1); + btrfs_release_path(path); + + key.objectid = bytenr; + if (parent) { + key.type = BTRFS_SHARED_BLOCK_REF_KEY; + key.offset = parent; + } else { + key.type = BTRFS_TREE_BLOCK_REF_KEY; + key.offset = root_objectid; + } + + ret = btrfs_search_slot(&trans, root, &key, path, -1, 1); + if (ret) { + test_err("couldn't find backref %d", ret); + return ret; + } + btrfs_del_item(&trans, root, path); + return ret; +} + +static int test_no_shared_qgroup(struct btrfs_root *root, + u32 sectorsize, u32 nodesize) +{ + struct btrfs_backref_walk_ctx ctx = { 0 }; + struct btrfs_trans_handle trans; + struct btrfs_fs_info *fs_info = root->fs_info; + struct ulist *old_roots = NULL; + struct ulist *new_roots = NULL; + int ret; + + btrfs_init_dummy_trans(&trans, fs_info); + + test_msg("running qgroup add/remove tests"); + ret = btrfs_create_qgroup(&trans, BTRFS_FS_TREE_OBJECTID); + if (ret) { + test_err("couldn't create a qgroup %d", ret); + return ret; + } + + ctx.bytenr = nodesize; + ctx.trans = &trans; + ctx.fs_info = fs_info; + + /* + * Since the test trans doesn't have the complicated delayed refs, + * we can only call btrfs_qgroup_account_extent() directly to test + * quota. + */ + ret = btrfs_find_all_roots(&ctx, false); + if (ret) { + test_err("couldn't find old roots: %d", ret); + return ret; + } + old_roots = ctx.roots; + ctx.roots = NULL; + + ret = insert_normal_tree_ref(root, nodesize, nodesize, 0, + BTRFS_FS_TREE_OBJECTID); + if (ret) { + ulist_free(old_roots); + return ret; + } + + ret = btrfs_find_all_roots(&ctx, false); + if (ret) { + ulist_free(old_roots); + test_err("couldn't find old roots: %d", ret); + return ret; + } + new_roots = ctx.roots; + ctx.roots = NULL; + + ret = btrfs_qgroup_account_extent(&trans, nodesize, nodesize, old_roots, + new_roots); + if (ret) { + test_err("couldn't account space for a qgroup %d", ret); + return ret; + } + + /* btrfs_qgroup_account_extent() always frees the ulists passed to it. */ + old_roots = NULL; + new_roots = NULL; + + if (btrfs_verify_qgroup_counts(fs_info, BTRFS_FS_TREE_OBJECTID, + nodesize, nodesize)) { + test_err("qgroup counts didn't match expected values"); + return -EINVAL; + } + + ret = btrfs_find_all_roots(&ctx, false); + if (ret) { + test_err("couldn't find old roots: %d", ret); + return ret; + } + old_roots = ctx.roots; + ctx.roots = NULL; + + ret = remove_extent_item(root, nodesize, nodesize); + if (ret) { + ulist_free(old_roots); + return -EINVAL; + } + + ret = btrfs_find_all_roots(&ctx, false); + if (ret) { + ulist_free(old_roots); + test_err("couldn't find old roots: %d", ret); + return ret; + } + new_roots = ctx.roots; + ctx.roots = NULL; + + ret = btrfs_qgroup_account_extent(&trans, nodesize, nodesize, old_roots, + new_roots); + if (ret) { + test_err("couldn't account space for a qgroup %d", ret); + return -EINVAL; + } + + if (btrfs_verify_qgroup_counts(fs_info, BTRFS_FS_TREE_OBJECTID, 0, 0)) { + test_err("qgroup counts didn't match expected values"); + return -EINVAL; + } + + return 0; +} + +/* + * Add a ref for two different roots to make sure the shared value comes out + * right, also remove one of the roots and make sure the exclusive count is + * adjusted properly. + */ +static int test_multiple_refs(struct btrfs_root *root, + u32 sectorsize, u32 nodesize) +{ + struct btrfs_backref_walk_ctx ctx = { 0 }; + struct btrfs_trans_handle trans; + struct btrfs_fs_info *fs_info = root->fs_info; + struct ulist *old_roots = NULL; + struct ulist *new_roots = NULL; + int ret; + + btrfs_init_dummy_trans(&trans, fs_info); + + test_msg("running qgroup multiple refs test"); + + /* + * We have BTRFS_FS_TREE_OBJECTID created already from the + * previous test. + */ + ret = btrfs_create_qgroup(&trans, BTRFS_FIRST_FREE_OBJECTID); + if (ret) { + test_err("couldn't create a qgroup %d", ret); + return ret; + } + + ctx.bytenr = nodesize; + ctx.trans = &trans; + ctx.fs_info = fs_info; + + ret = btrfs_find_all_roots(&ctx, false); + if (ret) { + test_err("couldn't find old roots: %d", ret); + return ret; + } + old_roots = ctx.roots; + ctx.roots = NULL; + + ret = insert_normal_tree_ref(root, nodesize, nodesize, 0, + BTRFS_FS_TREE_OBJECTID); + if (ret) { + ulist_free(old_roots); + return ret; + } + + ret = btrfs_find_all_roots(&ctx, false); + if (ret) { + ulist_free(old_roots); + test_err("couldn't find old roots: %d", ret); + return ret; + } + new_roots = ctx.roots; + ctx.roots = NULL; + + ret = btrfs_qgroup_account_extent(&trans, nodesize, nodesize, old_roots, + new_roots); + if (ret) { + test_err("couldn't account space for a qgroup %d", ret); + return ret; + } + + if (btrfs_verify_qgroup_counts(fs_info, BTRFS_FS_TREE_OBJECTID, + nodesize, nodesize)) { + test_err("qgroup counts didn't match expected values"); + return -EINVAL; + } + + ret = btrfs_find_all_roots(&ctx, false); + if (ret) { + test_err("couldn't find old roots: %d", ret); + return ret; + } + old_roots = ctx.roots; + ctx.roots = NULL; + + ret = add_tree_ref(root, nodesize, nodesize, 0, + BTRFS_FIRST_FREE_OBJECTID); + if (ret) { + ulist_free(old_roots); + return ret; + } + + ret = btrfs_find_all_roots(&ctx, false); + if (ret) { + ulist_free(old_roots); + test_err("couldn't find old roots: %d", ret); + return ret; + } + new_roots = ctx.roots; + ctx.roots = NULL; + + ret = btrfs_qgroup_account_extent(&trans, nodesize, nodesize, old_roots, + new_roots); + if (ret) { + test_err("couldn't account space for a qgroup %d", ret); + return ret; + } + + if (btrfs_verify_qgroup_counts(fs_info, BTRFS_FS_TREE_OBJECTID, + nodesize, 0)) { + test_err("qgroup counts didn't match expected values"); + return -EINVAL; + } + + if (btrfs_verify_qgroup_counts(fs_info, BTRFS_FIRST_FREE_OBJECTID, + nodesize, 0)) { + test_err("qgroup counts didn't match expected values"); + return -EINVAL; + } + + ret = btrfs_find_all_roots(&ctx, false); + if (ret) { + test_err("couldn't find old roots: %d", ret); + return ret; + } + old_roots = ctx.roots; + ctx.roots = NULL; + + ret = remove_extent_ref(root, nodesize, nodesize, 0, + BTRFS_FIRST_FREE_OBJECTID); + if (ret) { + ulist_free(old_roots); + return ret; + } + + ret = btrfs_find_all_roots(&ctx, false); + if (ret) { + ulist_free(old_roots); + test_err("couldn't find old roots: %d", ret); + return ret; + } + new_roots = ctx.roots; + ctx.roots = NULL; + + ret = btrfs_qgroup_account_extent(&trans, nodesize, nodesize, old_roots, + new_roots); + if (ret) { + test_err("couldn't account space for a qgroup %d", ret); + return ret; + } + + if (btrfs_verify_qgroup_counts(fs_info, BTRFS_FIRST_FREE_OBJECTID, + 0, 0)) { + test_err("qgroup counts didn't match expected values"); + return -EINVAL; + } + + if (btrfs_verify_qgroup_counts(fs_info, BTRFS_FS_TREE_OBJECTID, + nodesize, nodesize)) { + test_err("qgroup counts didn't match expected values"); + return -EINVAL; + } + + return 0; +} + +int btrfs_test_qgroups(u32 sectorsize, u32 nodesize) +{ + struct btrfs_fs_info *fs_info = NULL; + struct btrfs_root *root; + struct btrfs_root *tmp_root; + int ret = 0; + + fs_info = btrfs_alloc_dummy_fs_info(nodesize, sectorsize); + if (!fs_info) { + test_std_err(TEST_ALLOC_FS_INFO); + return -ENOMEM; + } + + root = btrfs_alloc_dummy_root(fs_info); + if (IS_ERR(root)) { + test_std_err(TEST_ALLOC_ROOT); + ret = PTR_ERR(root); + goto out; + } + + /* We are using this root as our extent root */ + root->root_key.objectid = BTRFS_EXTENT_TREE_OBJECTID; + root->root_key.type = BTRFS_ROOT_ITEM_KEY; + root->root_key.offset = 0; + btrfs_global_root_insert(root); + + /* + * Some of the paths we test assume we have a filled out fs_info, so we + * just need to add the root in there so we don't panic. + */ + root->fs_info->tree_root = root; + root->fs_info->quota_root = root; + set_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags); + + /* + * Can't use bytenr 0, some things freak out + * *cough*backref walking code*cough* + */ + root->node = alloc_test_extent_buffer(root->fs_info, nodesize); + if (IS_ERR(root->node)) { + test_err("couldn't allocate dummy buffer"); + ret = PTR_ERR(root->node); + goto out; + } + btrfs_set_header_level(root->node, 0); + btrfs_set_header_nritems(root->node, 0); + root->alloc_bytenr += 2 * nodesize; + + tmp_root = btrfs_alloc_dummy_root(fs_info); + if (IS_ERR(tmp_root)) { + test_std_err(TEST_ALLOC_ROOT); + ret = PTR_ERR(tmp_root); + goto out; + } + + tmp_root->root_key.objectid = BTRFS_FS_TREE_OBJECTID; + root->fs_info->fs_root = tmp_root; + ret = btrfs_insert_fs_root(root->fs_info, tmp_root); + btrfs_put_root(tmp_root); + if (ret) { + test_err("couldn't insert fs root %d", ret); + goto out; + } + + tmp_root = btrfs_alloc_dummy_root(fs_info); + if (IS_ERR(tmp_root)) { + test_std_err(TEST_ALLOC_ROOT); + ret = PTR_ERR(tmp_root); + goto out; + } + + tmp_root->root_key.objectid = BTRFS_FIRST_FREE_OBJECTID; + ret = btrfs_insert_fs_root(root->fs_info, tmp_root); + btrfs_put_root(tmp_root); + if (ret) { + test_err("couldn't insert subvolume root %d", ret); + goto out; + } + + test_msg("running qgroup tests"); + ret = test_no_shared_qgroup(root, sectorsize, nodesize); + if (ret) + goto out; + ret = test_multiple_refs(root, sectorsize, nodesize); +out: + btrfs_free_dummy_root(root); + btrfs_free_dummy_fs_info(fs_info); + return ret; +} diff --git a/fs/btrfs/tests/raid-stripe-tree-tests.c b/fs/btrfs/tests/raid-stripe-tree-tests.c new file mode 100644 index 000000000..a7bc58a5c --- /dev/null +++ b/fs/btrfs/tests/raid-stripe-tree-tests.c @@ -0,0 +1,1161 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2024 Western Digital Corporation or its affiliates. + */ + +#include +#include "../fs.h" +#include "../disk-io.h" +#include "../transaction.h" +#include "../volumes.h" +#include "../raid-stripe-tree.h" +#include "btrfs-tests.h" + +#define RST_TEST_NUM_DEVICES (2) +#define RST_TEST_RAID1_TYPE (BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_RAID1) + +#define SZ_48K (SZ_32K + SZ_16K) + +typedef int (*test_func_t)(struct btrfs_trans_handle *trans); + +static struct btrfs_device *btrfs_device_by_devid(struct btrfs_fs_devices *fs_devices, + u64 devid) +{ + struct btrfs_device *dev; + + list_for_each_entry(dev, &fs_devices->devices, dev_list) { + if (dev->devid == devid) + return dev; + } + + return NULL; +} + +/* + * Test creating a range of three extents and then punch a hole in the middle, + * deleting all of the middle extents and partially deleting the "book ends". + */ +static int test_punch_hole_3extents(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_io_context *bioc; + struct btrfs_io_stripe io_stripe = { 0 }; + u64 map_type = RST_TEST_RAID1_TYPE; + u64 logical1 = SZ_1M; + u64 len1 = SZ_1M; + u64 logical2 = logical1 + len1; + u64 len2 = SZ_1M; + u64 logical3 = logical2 + len2; + u64 len3 = SZ_1M; + u64 hole_start = logical1 + SZ_256K; + u64 hole_len = SZ_2M; + int ret; + + bioc = alloc_btrfs_io_context(fs_info, logical1, RST_TEST_NUM_DEVICES); + if (!bioc) { + test_std_err(TEST_ALLOC_IO_CONTEXT); + ret = -ENOMEM; + goto out; + } + + io_stripe.dev = btrfs_device_by_devid(fs_info->fs_devices, 0); + + /* Prepare for the test, 1st create 3 x 1M extents. */ + bioc->map_type = map_type; + bioc->size = len1; + + for (int i = 0; i < RST_TEST_NUM_DEVICES; i++) { + struct btrfs_io_stripe *stripe = &bioc->stripes[i]; + + stripe->dev = btrfs_device_by_devid(fs_info->fs_devices, i); + if (!stripe->dev) { + test_err("cannot find device with devid %d", i); + ret = -EINVAL; + goto out; + } + + stripe->physical = logical1 + i * SZ_1G; + } + + ret = btrfs_insert_one_raid_extent(trans, bioc); + if (ret) { + test_err("inserting RAID extent failed: %d", ret); + goto out; + } + + bioc->logical = logical2; + bioc->size = len2; + for (int i = 0; i < RST_TEST_NUM_DEVICES; i++) { + struct btrfs_io_stripe *stripe = &bioc->stripes[i]; + + stripe->dev = btrfs_device_by_devid(fs_info->fs_devices, i); + if (!stripe->dev) { + test_err("cannot find device with devid %d", i); + ret = -EINVAL; + goto out; + } + + stripe->physical = logical2 + i * SZ_1G; + } + + ret = btrfs_insert_one_raid_extent(trans, bioc); + if (ret) { + test_err("inserting RAID extent failed: %d", ret); + goto out; + } + + bioc->logical = logical3; + bioc->size = len3; + for (int i = 0; i < RST_TEST_NUM_DEVICES; i++) { + struct btrfs_io_stripe *stripe = &bioc->stripes[i]; + + stripe->dev = btrfs_device_by_devid(fs_info->fs_devices, i); + if (!stripe->dev) { + test_err("cannot find device with devid %d", i); + ret = -EINVAL; + goto out; + } + + stripe->physical = logical3 + i * SZ_1G; + } + + ret = btrfs_insert_one_raid_extent(trans, bioc); + if (ret) { + test_err("inserting RAID extent failed: %d", ret); + goto out; + } + + /* + * Delete a range starting at logical1 + 256K and 2M in length. Extent + * 1 is truncated to 256k length, extent 2 is completely dropped and + * extent 3 is moved 256K to the right. + */ + ret = btrfs_delete_raid_extent(trans, hole_start, hole_len); + if (ret) { + test_err("deleting RAID extent [%llu, %llu] failed", + hole_start, hole_start + hole_len); + goto out; + } + + /* Get the first extent and check its size. */ + ret = btrfs_get_raid_extent_offset(fs_info, logical1, &len1, map_type, + 0, &io_stripe); + if (ret) { + test_err("lookup of RAID extent [%llu, %llu] failed", + logical1, logical1 + len1); + goto out; + } + + if (io_stripe.physical != logical1) { + test_err("invalid physical address, expected %llu, got %llu", + logical1, io_stripe.physical); + ret = -EINVAL; + goto out; + } + + if (len1 != SZ_256K) { + test_err("invalid stripe length, expected %llu, got %llu", + (u64)SZ_256K, len1); + ret = -EINVAL; + goto out; + } + + /* Get the second extent and check it's absent. */ + ret = btrfs_get_raid_extent_offset(fs_info, logical2, &len2, map_type, + 0, &io_stripe); + if (ret != -ENODATA) { + test_err("lookup of RAID extent [%llu, %llu] succeeded should fail", + logical2, logical2 + len2); + ret = -EINVAL; + goto out; + } + + /* Get the third extent and check its size. */ + logical3 += SZ_256K; + ret = btrfs_get_raid_extent_offset(fs_info, logical3, &len3, map_type, + 0, &io_stripe); + if (ret) { + test_err("lookup of RAID extent [%llu, %llu] failed", + logical3, logical3 + len3); + goto out; + } + + if (io_stripe.physical != logical3) { + test_err("invalid physical address, expected %llu, got %llu", + logical3 + SZ_256K, io_stripe.physical); + ret = -EINVAL; + goto out; + } + + if (len3 != SZ_1M - SZ_256K) { + test_err("invalid stripe length, expected %llu, got %llu", + (u64)SZ_1M - SZ_256K, len3); + ret = -EINVAL; + goto out; + } + + ret = btrfs_delete_raid_extent(trans, logical1, len1); + if (ret) { + test_err("deleting RAID extent [%llu, %llu] failed", + logical1, logical1 + len1); + goto out; + } + + ret = btrfs_delete_raid_extent(trans, logical3, len3); + if (ret) { + test_err("deleting RAID extent [%llu, %llu] failed", + logical1, logical1 + len1); + goto out; + } + +out: + btrfs_put_bioc(bioc); + return ret; +} + +static int test_delete_two_extents(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_io_context *bioc; + struct btrfs_io_stripe io_stripe = { 0 }; + u64 map_type = RST_TEST_RAID1_TYPE; + u64 logical1 = SZ_1M; + u64 len1 = SZ_1M; + u64 logical2 = logical1 + len1; + u64 len2 = SZ_1M; + u64 logical3 = logical2 + len2; + u64 len3 = SZ_1M; + int ret; + + bioc = alloc_btrfs_io_context(fs_info, logical1, RST_TEST_NUM_DEVICES); + if (!bioc) { + test_std_err(TEST_ALLOC_IO_CONTEXT); + ret = -ENOMEM; + goto out; + } + + io_stripe.dev = btrfs_device_by_devid(fs_info->fs_devices, 0); + + /* Prepare for the test, 1st create 3 x 1M extents. */ + bioc->map_type = map_type; + bioc->size = len1; + + for (int i = 0; i < RST_TEST_NUM_DEVICES; i++) { + struct btrfs_io_stripe *stripe = &bioc->stripes[i]; + + stripe->dev = btrfs_device_by_devid(fs_info->fs_devices, i); + if (!stripe->dev) { + test_err("cannot find device with devid %d", i); + ret = -EINVAL; + goto out; + } + + stripe->physical = logical1 + i * SZ_1G; + } + + ret = btrfs_insert_one_raid_extent(trans, bioc); + if (ret) { + test_err("inserting RAID extent failed: %d", ret); + goto out; + } + + bioc->logical = logical2; + bioc->size = len2; + for (int i = 0; i < RST_TEST_NUM_DEVICES; i++) { + struct btrfs_io_stripe *stripe = &bioc->stripes[i]; + + stripe->dev = btrfs_device_by_devid(fs_info->fs_devices, i); + if (!stripe->dev) { + test_err("cannot find device with devid %d", i); + ret = -EINVAL; + goto out; + } + + stripe->physical = logical2 + i * SZ_1G; + } + + ret = btrfs_insert_one_raid_extent(trans, bioc); + if (ret) { + test_err("inserting RAID extent failed: %d", ret); + goto out; + } + + bioc->logical = logical3; + bioc->size = len3; + for (int i = 0; i < RST_TEST_NUM_DEVICES; i++) { + struct btrfs_io_stripe *stripe = &bioc->stripes[i]; + + stripe->dev = btrfs_device_by_devid(fs_info->fs_devices, i); + if (!stripe->dev) { + test_err("cannot find device with devid %d", i); + ret = -EINVAL; + goto out; + } + + stripe->physical = logical3 + i * SZ_1G; + } + + ret = btrfs_insert_one_raid_extent(trans, bioc); + if (ret) { + test_err("inserting RAID extent failed: %d", ret); + goto out; + } + + /* + * Delete a range starting at logical1 and 2M in length. Extents 1 + * and 2 are dropped and extent 3 is kept as is. + */ + ret = btrfs_delete_raid_extent(trans, logical1, len1 + len2); + if (ret) { + test_err("deleting RAID extent [%llu, %llu] failed", + logical1, logical1 + len1 + len2); + goto out; + } + + ret = btrfs_get_raid_extent_offset(fs_info, logical1, &len1, map_type, + 0, &io_stripe); + if (ret != -ENODATA) { + test_err("lookup of RAID extent [%llu, %llu] succeeded, should fail", + logical1, len1); + goto out; + } + + ret = btrfs_get_raid_extent_offset(fs_info, logical2, &len2, map_type, + 0, &io_stripe); + if (ret != -ENODATA) { + test_err("lookup of RAID extent [%llu, %llu] succeeded, should fail", + logical2, len2); + goto out; + } + + ret = btrfs_get_raid_extent_offset(fs_info, logical3, &len3, map_type, + 0, &io_stripe); + if (ret) { + test_err("lookup of RAID extent [%llu, %llu] failed", + logical3, len3); + goto out; + } + + if (io_stripe.physical != logical3) { + test_err("invalid physical address, expected %llu, got %llu", + logical3, io_stripe.physical); + ret = -EINVAL; + goto out; + } + + if (len3 != SZ_1M) { + test_err("invalid stripe length, expected %llu, got %llu", + (u64)SZ_1M, len3); + ret = -EINVAL; + goto out; + } + + ret = btrfs_delete_raid_extent(trans, logical3, len3); +out: + btrfs_put_bioc(bioc); + return ret; +} + +/* Test punching a hole into a single RAID stripe-extent. */ +static int test_punch_hole(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_io_context *bioc; + struct btrfs_io_stripe io_stripe = { 0 }; + u64 map_type = RST_TEST_RAID1_TYPE; + u64 logical1 = SZ_1M; + u64 hole_start = logical1 + SZ_32K; + u64 hole_len = SZ_64K; + u64 logical2 = hole_start + hole_len; + u64 len = SZ_1M; + u64 len1 = SZ_32K; + u64 len2 = len - len1 - hole_len; + int ret; + + bioc = alloc_btrfs_io_context(fs_info, logical1, RST_TEST_NUM_DEVICES); + if (!bioc) { + test_std_err(TEST_ALLOC_IO_CONTEXT); + ret = -ENOMEM; + goto out; + } + + io_stripe.dev = btrfs_device_by_devid(fs_info->fs_devices, 0); + bioc->map_type = map_type; + bioc->size = len; + + for (int i = 0; i < RST_TEST_NUM_DEVICES; i++) { + struct btrfs_io_stripe *stripe = &bioc->stripes[i]; + + stripe->dev = btrfs_device_by_devid(fs_info->fs_devices, i); + if (!stripe->dev) { + test_err("cannot find device with devid %d", i); + ret = -EINVAL; + goto out; + } + + stripe->physical = logical1 + i * SZ_1G; + } + + ret = btrfs_insert_one_raid_extent(trans, bioc); + if (ret) { + test_err("inserting RAID extent failed: %d", ret); + goto out; + } + + ret = btrfs_get_raid_extent_offset(fs_info, logical1, &len, map_type, 0, + &io_stripe); + if (ret) { + test_err("lookup of RAID extent [%llu, %llu] failed", logical1, + logical1 + len); + goto out; + } + + if (io_stripe.physical != logical1) { + test_err("invalid physical address, expected %llu got %llu", + logical1, io_stripe.physical); + ret = -EINVAL; + goto out; + } + + if (len != SZ_1M) { + test_err("invalid stripe length, expected %llu got %llu", + (u64)SZ_1M, len); + ret = -EINVAL; + goto out; + } + + ret = btrfs_delete_raid_extent(trans, hole_start, hole_len); + if (ret) { + test_err("deleting RAID extent [%llu, %llu] failed", + hole_start, hole_start + hole_len); + goto out; + } + + ret = btrfs_get_raid_extent_offset(fs_info, logical1, &len1, map_type, + 0, &io_stripe); + if (ret) { + test_err("lookup of RAID extent [%llu, %llu] failed", + logical1, logical1 + len1); + goto out; + } + + if (io_stripe.physical != logical1) { + test_err("invalid physical address, expected %llu, got %llu", + logical1, io_stripe.physical); + ret = -EINVAL; + goto out; + } + + if (len1 != SZ_32K) { + test_err("invalid stripe length, expected %llu, got %llu", + (u64)SZ_32K, len1); + ret = -EINVAL; + goto out; + } + + ret = btrfs_get_raid_extent_offset(fs_info, logical2, &len2, map_type, + 0, &io_stripe); + if (ret) { + test_err("lookup of RAID extent [%llu, %llu] failed", logical2, + logical2 + len2); + goto out; + } + + if (io_stripe.physical != logical2) { + test_err("invalid physical address, expected %llu, got %llu", + logical2, io_stripe.physical); + ret = -EINVAL; + goto out; + } + + if (len2 != len - len1 - hole_len) { + test_err("invalid length, expected %llu, got %llu", + len - len1 - hole_len, len2); + ret = -EINVAL; + goto out; + } + + /* Check for the absence of the hole. */ + ret = btrfs_get_raid_extent_offset(fs_info, hole_start, &hole_len, + map_type, 0, &io_stripe); + if (ret != -ENODATA) { + ret = -EINVAL; + test_err("lookup of RAID extent [%llu, %llu] succeeded, should fail", + hole_start, hole_start + SZ_64K); + goto out; + } + + ret = btrfs_delete_raid_extent(trans, logical1, len1); + if (ret) + goto out; + + ret = btrfs_delete_raid_extent(trans, logical2, len2); +out: + btrfs_put_bioc(bioc); + return ret; +} + +/* + * Test a 1M RST write that spans two adjacent RST items on disk and then + * delete a portion starting in the first item and spanning into the second + * item. This is similar to test_front_delete(), but spanning multiple items. + */ +static int test_front_delete_prev_item(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_io_context *bioc; + struct btrfs_io_stripe io_stripe = { 0 }; + u64 map_type = RST_TEST_RAID1_TYPE; + u64 logical1 = SZ_1M; + u64 logical2 = SZ_2M; + u64 len = SZ_1M; + int ret; + + bioc = alloc_btrfs_io_context(fs_info, logical1, RST_TEST_NUM_DEVICES); + if (!bioc) { + test_std_err(TEST_ALLOC_IO_CONTEXT); + ret = -ENOMEM; + goto out; + } + + io_stripe.dev = btrfs_device_by_devid(fs_info->fs_devices, 0); + bioc->map_type = map_type; + bioc->size = len; + + /* Insert RAID extent 1. */ + for (int i = 0; i < RST_TEST_NUM_DEVICES; i++) { + struct btrfs_io_stripe *stripe = &bioc->stripes[i]; + + stripe->dev = btrfs_device_by_devid(fs_info->fs_devices, i); + if (!stripe->dev) { + test_err("cannot find device with devid %d", i); + ret = -EINVAL; + goto out; + } + + stripe->physical = logical1 + i * SZ_1G; + } + + ret = btrfs_insert_one_raid_extent(trans, bioc); + if (ret) { + test_err("inserting RAID extent failed: %d", ret); + goto out; + } + + bioc->logical = logical2; + /* Insert RAID extent 2, directly adjacent to it. */ + for (int i = 0; i < RST_TEST_NUM_DEVICES; i++) { + struct btrfs_io_stripe *stripe = &bioc->stripes[i]; + + stripe->dev = btrfs_device_by_devid(fs_info->fs_devices, i); + if (!stripe->dev) { + test_err("cannot find device with devid %d", i); + ret = -EINVAL; + goto out; + } + + stripe->physical = logical2 + i * SZ_1G; + } + + ret = btrfs_insert_one_raid_extent(trans, bioc); + if (ret) { + test_err("inserting RAID extent failed: %d", ret); + goto out; + } + + ret = btrfs_delete_raid_extent(trans, logical1 + SZ_512K, SZ_1M); + if (ret) { + test_err("deleting RAID extent [%llu, %llu] failed", + logical1 + SZ_512K, (u64)SZ_1M); + goto out; + } + + /* Verify item 1 is truncated to 512K. */ + ret = btrfs_get_raid_extent_offset(fs_info, logical1, &len, map_type, 0, + &io_stripe); + if (ret) { + test_err("lookup of RAID extent [%llu, %llu] failed", logical1, + logical1 + len); + goto out; + } + + if (io_stripe.physical != logical1) { + test_err("invalid physical address, expected %llu got %llu", + logical1, io_stripe.physical); + ret = -EINVAL; + goto out; + } + + if (len != SZ_512K) { + test_err("invalid stripe length, expected %llu got %llu", + (u64)SZ_512K, len); + ret = -EINVAL; + goto out; + } + + /* Verify item 2's start is moved by 512K. */ + ret = btrfs_get_raid_extent_offset(fs_info, logical2 + SZ_512K, &len, + map_type, 0, &io_stripe); + if (ret) { + test_err("lookup of RAID extent [%llu, %llu] failed", + logical2 + SZ_512K, logical2 + len); + goto out; + } + + if (io_stripe.physical != logical2 + SZ_512K) { + test_err("invalid physical address, expected %llu got %llu", + logical2 + SZ_512K, io_stripe.physical); + ret = -EINVAL; + goto out; + } + + if (len != SZ_512K) { + test_err("invalid stripe length, expected %llu got %llu", + (u64)SZ_512K, len); + ret = -EINVAL; + goto out; + } + + /* Verify there's a hole at [1M+512K, 2M+512K] . */ + len = SZ_1M; + ret = btrfs_get_raid_extent_offset(fs_info, logical1 + SZ_512K, &len, + map_type, 0, &io_stripe); + if (ret != -ENODATA) { + test_err("lookup of RAID [%llu, %llu] succeeded, should fail", + logical1 + SZ_512K, logical1 + SZ_512K + len); + goto out; + } + + /* Clean up after us. */ + ret = btrfs_delete_raid_extent(trans, logical1, SZ_512K); + if (ret) + goto out; + + ret = btrfs_delete_raid_extent(trans, logical2 + SZ_512K, SZ_512K); + +out: + btrfs_put_bioc(bioc); + return ret; +} + +/* + * Test a 64K RST write on a 2 disk RAID1 at a logical address of 1M and then + * delete the 1st 32K, making the new start address 1M+32K. + */ +static int test_front_delete(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_io_context *bioc; + struct btrfs_io_stripe io_stripe = { 0 }; + u64 map_type = RST_TEST_RAID1_TYPE; + u64 logical = SZ_1M; + u64 len = SZ_64K; + int ret; + + bioc = alloc_btrfs_io_context(fs_info, logical, RST_TEST_NUM_DEVICES); + if (!bioc) { + test_std_err(TEST_ALLOC_IO_CONTEXT); + ret = -ENOMEM; + goto out; + } + + io_stripe.dev = btrfs_device_by_devid(fs_info->fs_devices, 0); + bioc->map_type = map_type; + bioc->size = len; + + for (int i = 0; i < RST_TEST_NUM_DEVICES; i++) { + struct btrfs_io_stripe *stripe = &bioc->stripes[i]; + + stripe->dev = btrfs_device_by_devid(fs_info->fs_devices, i); + if (!stripe->dev) { + test_err("cannot find device with devid %d", i); + ret = -EINVAL; + goto out; + } + + stripe->physical = logical + i * SZ_1G; + } + + ret = btrfs_insert_one_raid_extent(trans, bioc); + if (ret) { + test_err("inserting RAID extent failed: %d", ret); + goto out; + } + + ret = btrfs_get_raid_extent_offset(fs_info, logical, &len, map_type, 0, &io_stripe); + if (ret) { + test_err("lookup of RAID extent [%llu, %llu] failed", logical, + logical + len); + goto out; + } + + if (io_stripe.physical != logical) { + test_err("invalid physical address, expected %llu got %llu", + logical, io_stripe.physical); + ret = -EINVAL; + goto out; + } + + if (len != SZ_64K) { + test_err("invalid stripe length, expected %llu got %llu", + (u64)SZ_64K, len); + ret = -EINVAL; + goto out; + } + + ret = btrfs_delete_raid_extent(trans, logical, SZ_16K); + if (ret) { + test_err("deleting RAID extent [%llu, %llu] failed", logical, + logical + SZ_16K); + goto out; + } + + len -= SZ_16K; + ret = btrfs_get_raid_extent_offset(fs_info, logical + SZ_16K, &len, + map_type, 0, &io_stripe); + if (ret) { + test_err("lookup of RAID extent [%llu, %llu] failed", + logical + SZ_16K, logical + SZ_64K); + goto out; + } + + if (io_stripe.physical != logical + SZ_16K) { + test_err("invalid physical address, expected %llu, got %llu", + logical + SZ_16K, io_stripe.physical); + ret = -EINVAL; + goto out; + } + + if (len != SZ_48K) { + test_err("invalid stripe length, expected %llu, got %llu", + (u64)SZ_48K, len); + ret = -EINVAL; + goto out; + } + + ret = btrfs_get_raid_extent_offset(fs_info, logical, &len, map_type, 0, &io_stripe); + if (ret != -ENODATA) { + ret = -EINVAL; + test_err("lookup of RAID extent [%llu, %llu] succeeded, should fail", + logical, logical + SZ_16K); + goto out; + } + + ret = btrfs_delete_raid_extent(trans, logical + SZ_16K, SZ_48K); +out: + btrfs_put_bioc(bioc); + return ret; +} + +/* + * Test a 64K RST write on a 2 disk RAID1 at a logical address of 1M and then + * truncate the stripe extent down to 32K. + */ +static int test_tail_delete(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_io_context *bioc; + struct btrfs_io_stripe io_stripe = { 0 }; + u64 map_type = RST_TEST_RAID1_TYPE; + u64 logical = SZ_1M; + u64 len = SZ_64K; + int ret; + + bioc = alloc_btrfs_io_context(fs_info, logical, RST_TEST_NUM_DEVICES); + if (!bioc) { + test_std_err(TEST_ALLOC_IO_CONTEXT); + ret = -ENOMEM; + goto out; + } + + io_stripe.dev = btrfs_device_by_devid(fs_info->fs_devices, 0); + bioc->map_type = map_type; + bioc->size = len; + + for (int i = 0; i < RST_TEST_NUM_DEVICES; i++) { + struct btrfs_io_stripe *stripe = &bioc->stripes[i]; + + stripe->dev = btrfs_device_by_devid(fs_info->fs_devices, i); + if (!stripe->dev) { + test_err("cannot find device with devid %d", i); + ret = -EINVAL; + goto out; + } + + stripe->physical = logical + i * SZ_1G; + } + + ret = btrfs_insert_one_raid_extent(trans, bioc); + if (ret) { + test_err("inserting RAID extent failed: %d", ret); + goto out; + } + + io_stripe.dev = btrfs_device_by_devid(fs_info->fs_devices, 0); + if (!io_stripe.dev) { + ret = -EINVAL; + goto out; + } + + ret = btrfs_get_raid_extent_offset(fs_info, logical, &len, map_type, 0, &io_stripe); + if (ret) { + test_err("lookup of RAID extent [%llu, %llu] failed", logical, + logical + len); + goto out; + } + + if (io_stripe.physical != logical) { + test_err("invalid physical address, expected %llu got %llu", + logical, io_stripe.physical); + ret = -EINVAL; + goto out; + } + + if (len != SZ_64K) { + test_err("invalid stripe length, expected %llu got %llu", + (u64)SZ_64K, len); + ret = -EINVAL; + goto out; + } + + ret = btrfs_delete_raid_extent(trans, logical + SZ_48K, SZ_16K); + if (ret) { + test_err("deleting RAID extent [%llu, %llu] failed", + logical + SZ_48K, logical + SZ_64K); + goto out; + } + + len = SZ_48K; + ret = btrfs_get_raid_extent_offset(fs_info, logical, &len, map_type, 0, &io_stripe); + if (ret) { + test_err("lookup of RAID extent [%llu, %llu] failed", logical, + logical + len); + goto out; + } + + if (io_stripe.physical != logical) { + test_err("invalid physical address, expected %llu, got %llu", + logical, io_stripe.physical); + ret = -EINVAL; + goto out; + } + + if (len != SZ_48K) { + test_err("invalid stripe length, expected %llu, got %llu", + (u64)SZ_48K, len); + ret = -EINVAL; + goto out; + } + + len = SZ_16K; + ret = btrfs_get_raid_extent_offset(fs_info, logical + SZ_48K, &len, + map_type, 0, &io_stripe); + if (ret != -ENODATA) { + test_err("lookup of RAID extent [%llu, %llu] succeeded should fail", + logical + SZ_48K, logical + SZ_64K); + ret = -EINVAL; + goto out; + } + + ret = btrfs_delete_raid_extent(trans, logical, len); + if (ret) + test_err("deleting RAID extent [%llu, %llu] failed", logical, + logical + len); + +out: + btrfs_put_bioc(bioc); + return ret; +} + +/* + * Test a 64K RST write on a 2 disk RAID1 at a logical address of 1M and then + * overwrite the whole range giving it new physical address at an offset of 1G. + * The intent of this test is to exercise the 'update_raid_extent_item()' + * function called be btrfs_insert_one_raid_extent(). + */ +static int test_create_update_delete(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_io_context *bioc; + struct btrfs_io_stripe io_stripe = { 0 }; + u64 map_type = RST_TEST_RAID1_TYPE; + u64 logical = SZ_1M; + u64 len = SZ_64K; + int ret; + + bioc = alloc_btrfs_io_context(fs_info, logical, RST_TEST_NUM_DEVICES); + if (!bioc) { + test_std_err(TEST_ALLOC_IO_CONTEXT); + ret = -ENOMEM; + goto out; + } + + io_stripe.dev = btrfs_device_by_devid(fs_info->fs_devices, 0); + bioc->map_type = map_type; + bioc->size = len; + + for (int i = 0; i < RST_TEST_NUM_DEVICES; i++) { + struct btrfs_io_stripe *stripe = &bioc->stripes[i]; + + stripe->dev = btrfs_device_by_devid(fs_info->fs_devices, i); + if (!stripe->dev) { + test_err("cannot find device with devid %d", i); + ret = -EINVAL; + goto out; + } + + stripe->physical = logical + i * SZ_1G; + } + + ret = btrfs_insert_one_raid_extent(trans, bioc); + if (ret) { + test_err("inserting RAID extent failed: %d", ret); + goto out; + } + + io_stripe.dev = btrfs_device_by_devid(fs_info->fs_devices, 0); + if (!io_stripe.dev) { + ret = -EINVAL; + goto out; + } + + ret = btrfs_get_raid_extent_offset(fs_info, logical, &len, map_type, 0, &io_stripe); + if (ret) { + test_err("lookup of RAID extent [%llu, %llu] failed", logical, + logical + len); + goto out; + } + + if (io_stripe.physical != logical) { + test_err("invalid physical address, expected %llu got %llu", + logical, io_stripe.physical); + ret = -EINVAL; + goto out; + } + + if (len != SZ_64K) { + test_err("invalid stripe length, expected %llu got %llu", + (u64)SZ_64K, len); + ret = -EINVAL; + goto out; + } + + for (int i = 0; i < RST_TEST_NUM_DEVICES; i++) { + struct btrfs_io_stripe *stripe = &bioc->stripes[i]; + + stripe->dev = btrfs_device_by_devid(fs_info->fs_devices, i); + if (!stripe->dev) { + test_err("cannot find device with devid %d", i); + ret = -EINVAL; + goto out; + } + + stripe->physical = SZ_1G + logical + i * SZ_1G; + } + + ret = btrfs_insert_one_raid_extent(trans, bioc); + if (ret) { + test_err("updating RAID extent failed: %d", ret); + goto out; + } + + ret = btrfs_get_raid_extent_offset(fs_info, logical, &len, map_type, 0, &io_stripe); + if (ret) { + test_err("lookup of RAID extent [%llu, %llu] failed", logical, + logical + len); + goto out; + } + + if (io_stripe.physical != logical + SZ_1G) { + test_err("invalid physical address, expected %llu, got %llu", + logical + SZ_1G, io_stripe.physical); + ret = -EINVAL; + goto out; + } + + if (len != SZ_64K) { + test_err("invalid stripe length, expected %llu, got %llu", + (u64)SZ_64K, len); + ret = -EINVAL; + goto out; + } + + ret = btrfs_delete_raid_extent(trans, logical, len); + if (ret) + test_err("deleting RAID extent [%llu, %llu] failed", logical, + logical + len); + +out: + btrfs_put_bioc(bioc); + return ret; +} + +/* + * Test a simple 64K RST write on a 2 disk RAID1 at a logical address of 1M. + * The "physical" copy on device 0 is at 1M, on device 1 it is at 1G+1M. + */ +static int test_simple_create_delete(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_io_context *bioc; + struct btrfs_io_stripe io_stripe = { 0 }; + u64 map_type = RST_TEST_RAID1_TYPE; + u64 logical = SZ_1M; + u64 len = SZ_64K; + int ret; + + bioc = alloc_btrfs_io_context(fs_info, logical, RST_TEST_NUM_DEVICES); + if (!bioc) { + test_std_err(TEST_ALLOC_IO_CONTEXT); + ret = -ENOMEM; + goto out; + } + + bioc->map_type = map_type; + bioc->size = SZ_64K; + + for (int i = 0; i < RST_TEST_NUM_DEVICES; i++) { + struct btrfs_io_stripe *stripe = &bioc->stripes[i]; + + stripe->dev = btrfs_device_by_devid(fs_info->fs_devices, i); + if (!stripe->dev) { + test_err("cannot find device with devid %d", i); + ret = -EINVAL; + goto out; + } + + stripe->physical = logical + i * SZ_1G; + } + + ret = btrfs_insert_one_raid_extent(trans, bioc); + if (ret) { + test_err("inserting RAID extent failed: %d", ret); + goto out; + } + + io_stripe.dev = btrfs_device_by_devid(fs_info->fs_devices, 0); + if (!io_stripe.dev) { + ret = -EINVAL; + goto out; + } + + ret = btrfs_get_raid_extent_offset(fs_info, logical, &len, map_type, 0, &io_stripe); + if (ret) { + test_err("lookup of RAID extent [%llu, %llu] failed", logical, + logical + len); + goto out; + } + + if (io_stripe.physical != logical) { + test_err("invalid physical address, expected %llu got %llu", + logical, io_stripe.physical); + ret = -EINVAL; + goto out; + } + + if (len != SZ_64K) { + test_err("invalid stripe length, expected %llu got %llu", + (u64)SZ_64K, len); + ret = -EINVAL; + goto out; + } + + ret = btrfs_delete_raid_extent(trans, logical, len); + if (ret) + test_err("deleting RAID extent [%llu, %llu] failed", logical, + logical + len); + +out: + btrfs_put_bioc(bioc); + return ret; +} + +static const test_func_t tests[] = { + test_simple_create_delete, + test_create_update_delete, + test_tail_delete, + test_front_delete, + test_front_delete_prev_item, + test_punch_hole, + test_punch_hole_3extents, + test_delete_two_extents, +}; + +static int run_test(test_func_t test, u32 sectorsize, u32 nodesize) +{ + struct btrfs_trans_handle trans; + struct btrfs_fs_info *fs_info; + struct btrfs_root *root = NULL; + int ret; + + fs_info = btrfs_alloc_dummy_fs_info(sectorsize, nodesize); + if (!fs_info) { + test_std_err(TEST_ALLOC_FS_INFO); + ret = -ENOMEM; + goto out; + } + + root = btrfs_alloc_dummy_root(fs_info); + if (IS_ERR(root)) { + test_std_err(TEST_ALLOC_ROOT); + ret = PTR_ERR(root); + goto out; + } + btrfs_set_super_incompat_flags(root->fs_info->super_copy, + BTRFS_FEATURE_INCOMPAT_RAID_STRIPE_TREE); + root->root_key.objectid = BTRFS_RAID_STRIPE_TREE_OBJECTID; + root->root_key.type = BTRFS_ROOT_ITEM_KEY; + root->root_key.offset = 0; + fs_info->stripe_root = root; + root->fs_info->tree_root = root; + + root->node = alloc_test_extent_buffer(root->fs_info, nodesize); + if (IS_ERR(root->node)) { + test_std_err(TEST_ALLOC_EXTENT_BUFFER); + ret = PTR_ERR(root->node); + goto out; + } + btrfs_set_header_level(root->node, 0); + btrfs_set_header_nritems(root->node, 0); + root->alloc_bytenr += 2 * nodesize; + + for (int i = 0; i < RST_TEST_NUM_DEVICES; i++) { + struct btrfs_device *dev; + + dev = btrfs_alloc_dummy_device(fs_info); + if (IS_ERR(dev)) { + test_err("cannot allocate device"); + ret = PTR_ERR(dev); + goto out; + } + dev->devid = i; + } + + btrfs_init_dummy_trans(&trans, root->fs_info); + ret = test(&trans); + if (ret) + goto out; + +out: + btrfs_free_dummy_root(root); + btrfs_free_dummy_fs_info(fs_info); + + return ret; +} + +int btrfs_test_raid_stripe_tree(u32 sectorsize, u32 nodesize) +{ + int ret = 0; + + test_msg("running raid-stripe-tree tests"); + for (int i = 0; i < ARRAY_SIZE(tests); i++) { + ret = run_test(tests[i], sectorsize, nodesize); + if (ret) { + test_err("test-case %ps failed with %d\n", tests[i], ret); + goto out; + } + } + +out: + return ret; +} diff --git a/fs/btrfs/tests/zoned-tests.c b/fs/btrfs/tests/zoned-tests.c new file mode 100644 index 000000000..2bc3b14ba --- /dev/null +++ b/fs/btrfs/tests/zoned-tests.c @@ -0,0 +1,675 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2026 Western Digital. All rights reserved. + */ + +#include +#include + +#include "btrfs-tests.h" +#include "../space-info.h" +#include "../volumes.h" +#include "../zoned.h" + +#define WP_MISSING_DEV ((u64)-1) +#define WP_CONVENTIONAL ((u64)-2) +#define ZONE_SIZE SZ_256M + +#define HALF_STRIPE_LEN (BTRFS_STRIPE_LEN >> 1) + +struct load_zone_info_test_vector { + u64 raid_type; + u64 num_stripes; + u64 alloc_offsets[8]; + u64 last_alloc; + u64 bg_length; + bool degraded; + + int expected_result; + u64 expected_alloc_offset; + + const char *description; +}; + +struct zone_info { + u64 physical; + u64 capacity; + u64 alloc_offset; +}; + +static int test_load_zone_info(struct btrfs_fs_info *fs_info, + const struct load_zone_info_test_vector *test) +{ + struct btrfs_block_group *bg __free(btrfs_free_dummy_block_group) = NULL; + struct btrfs_chunk_map *map __free(btrfs_free_chunk_map) = NULL; + struct zone_info AUTO_KFREE(zone_info); + unsigned long AUTO_KFREE(active); + int ret; + + bg = btrfs_alloc_dummy_block_group(fs_info, test->bg_length); + if (!bg) { + test_std_err(TEST_ALLOC_BLOCK_GROUP); + return -ENOMEM; + } + + map = btrfs_alloc_chunk_map(test->num_stripes, GFP_KERNEL); + if (!map) { + test_std_err(TEST_ALLOC_EXTENT_MAP); + return -ENOMEM; + } + + zone_info = kzalloc_objs(*zone_info, test->num_stripes, GFP_KERNEL); + if (!zone_info) { + test_err("cannot allocate zone info"); + return -ENOMEM; + } + + active = bitmap_zalloc(test->num_stripes, GFP_KERNEL); + if (!zone_info) { + test_err("cannot allocate active bitmap"); + return -ENOMEM; + } + + map->type = test->raid_type; + map->num_stripes = test->num_stripes; + if (test->raid_type == BTRFS_BLOCK_GROUP_RAID10) + map->sub_stripes = 2; + for (int i = 0; i < test->num_stripes; i++) { + zone_info[i].physical = 0; + zone_info[i].alloc_offset = test->alloc_offsets[i]; + zone_info[i].capacity = ZONE_SIZE; + if (zone_info[i].alloc_offset && zone_info[i].alloc_offset < ZONE_SIZE) + __set_bit(i, active); + } + if (test->degraded) + btrfs_set_opt(fs_info->mount_opt, DEGRADED); + else + btrfs_clear_opt(fs_info->mount_opt, DEGRADED); + + ret = btrfs_load_block_group_by_raid_type(bg, map, zone_info, active, + test->last_alloc); + + if (ret != test->expected_result) { + test_err("unexpected return value: ret %d expected %d", ret, + test->expected_result); + return -EINVAL; + } + + if (!ret && bg->alloc_offset != test->expected_alloc_offset) { + test_err("unexpected alloc_offset: alloc_offset %llu expected %llu", + bg->alloc_offset, test->expected_alloc_offset); + return -EINVAL; + } + + return 0; +} + +static const struct load_zone_info_test_vector load_zone_info_tests[] = { + /* SINGLE */ + { + .description = "SINGLE: load write pointer from sequential zone", + .raid_type = 0, + .num_stripes = 1, + .alloc_offsets = { + SZ_1M, + }, + .expected_alloc_offset = SZ_1M, + }, + /* + * SINGLE block group on a conventional zone sets last_alloc outside of + * btrfs_load_block_group_*(). Do not test that case. + */ + + /* DUP */ + /* Normal case */ + { + .description = "DUP: having matching write pointers", + .raid_type = BTRFS_BLOCK_GROUP_DUP, + .num_stripes = 2, + .alloc_offsets = { + SZ_1M, SZ_1M, + }, + .expected_alloc_offset = SZ_1M, + }, + /* + * One sequential zone and one conventional zone, having matching + * last_alloc. + */ + { + .description = "DUP: seq zone and conv zone, matching last_alloc", + .raid_type = BTRFS_BLOCK_GROUP_DUP, + .num_stripes = 2, + .alloc_offsets = { + SZ_1M, WP_CONVENTIONAL, + }, + .last_alloc = SZ_1M, + .expected_alloc_offset = SZ_1M, + }, + /* + * One sequential and one conventional zone, but having smaller + * last_alloc than write pointer. + */ + { + .description = "DUP: seq zone and conv zone, smaller last_alloc", + .raid_type = BTRFS_BLOCK_GROUP_DUP, + .num_stripes = 2, + .alloc_offsets = { + SZ_1M, WP_CONVENTIONAL, + }, + .last_alloc = 0, + .expected_alloc_offset = SZ_1M, + }, + /* Error case: having different write pointers. */ + { + .description = "DUP: fail: different write pointers", + .raid_type = BTRFS_BLOCK_GROUP_DUP, + .num_stripes = 2, + .alloc_offsets = { + SZ_1M, SZ_2M, + }, + .expected_result = -EIO, + }, + /* Error case: partial missing device should not happen on DUP. */ + { + .description = "DUP: fail: missing device", + .raid_type = BTRFS_BLOCK_GROUP_DUP, + .num_stripes = 2, + .alloc_offsets = { + SZ_1M, WP_MISSING_DEV, + }, + .expected_result = -EIO, + }, + /* + * Error case: one sequential and one conventional zone, but having larger + * last_alloc than write pointer. + */ + { + .description = "DUP: fail: seq zone and conv zone, larger last_alloc", + .raid_type = BTRFS_BLOCK_GROUP_DUP, + .num_stripes = 2, + .alloc_offsets = { + SZ_1M, WP_CONVENTIONAL, + }, + .last_alloc = SZ_2M, + .expected_result = -EIO, + }, + + /* RAID1 */ + /* Normal case */ + { + .description = "RAID1: having matching write pointers", + .raid_type = BTRFS_BLOCK_GROUP_RAID1, + .num_stripes = 2, + .alloc_offsets = { + SZ_1M, SZ_1M, + }, + .expected_alloc_offset = SZ_1M, + }, + /* + * One sequential zone and one conventional zone, having matching + * last_alloc. + */ + { + .description = "RAID1: seq zone and conv zone, matching last_alloc", + .raid_type = BTRFS_BLOCK_GROUP_RAID1, + .num_stripes = 2, + .alloc_offsets = { + SZ_1M, WP_CONVENTIONAL, + }, + .last_alloc = SZ_1M, + .expected_alloc_offset = SZ_1M, + }, + /* + * One sequential and one conventional zone, but having smaller + * last_alloc than write pointer. + */ + { + .description = "RAID1: seq zone and conv zone, smaller last_alloc", + .raid_type = BTRFS_BLOCK_GROUP_RAID1, + .num_stripes = 2, + .alloc_offsets = { + SZ_1M, WP_CONVENTIONAL, + }, + .last_alloc = 0, + .expected_alloc_offset = SZ_1M, + }, + /* Partial missing device should be recovered on DEGRADED mount */ + { + .description = "RAID1: fail: missing device on DEGRADED", + .raid_type = BTRFS_BLOCK_GROUP_RAID1, + .num_stripes = 2, + .alloc_offsets = { + SZ_1M, WP_MISSING_DEV, + }, + .degraded = true, + .expected_alloc_offset = SZ_1M, + }, + /* Error case: having different write pointers. */ + { + .description = "RAID1: fail: different write pointers", + .raid_type = BTRFS_BLOCK_GROUP_RAID1, + .num_stripes = 2, + .alloc_offsets = { + SZ_1M, SZ_2M, + }, + .expected_result = -EIO, + }, + /* + * Partial missing device is not allowed on non-DEGRADED mount never happen + * as it is rejected beforehand. + */ + /* + * Error case: one sequential and one conventional zone, but having larger + * last_alloc than write pointer. + */ + { + .description = "RAID1: fail: seq zone and conv zone, larger last_alloc", + .raid_type = BTRFS_BLOCK_GROUP_RAID1, + .num_stripes = 2, + .alloc_offsets = { + SZ_1M, WP_CONVENTIONAL, + }, + .last_alloc = SZ_2M, + .expected_result = -EIO, + }, + + /* RAID0 */ + /* Normal case */ + { + .description = "RAID0: initial partial write", + .raid_type = BTRFS_BLOCK_GROUP_RAID0, + .num_stripes = 4, + .alloc_offsets = { + HALF_STRIPE_LEN, 0, 0, 0, + }, + .expected_alloc_offset = HALF_STRIPE_LEN, + }, + { + .description = "RAID0: while in second stripe", + .raid_type = BTRFS_BLOCK_GROUP_RAID0, + .num_stripes = 4, + .alloc_offsets = { + BTRFS_STRIPE_LEN * 2, BTRFS_STRIPE_LEN + HALF_STRIPE_LEN, + BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN, + }, + .expected_alloc_offset = BTRFS_STRIPE_LEN * 5 + HALF_STRIPE_LEN, + }, + { + .description = "RAID0: one stripe advanced", + .raid_type = BTRFS_BLOCK_GROUP_RAID0, + .num_stripes = 2, + .alloc_offsets = { + SZ_1M + BTRFS_STRIPE_LEN, SZ_1M, + }, + .expected_alloc_offset = SZ_2M + BTRFS_STRIPE_LEN, + }, + /* Error case: having different write pointers. */ + { + .description = "RAID0: fail: disordered stripes", + .raid_type = BTRFS_BLOCK_GROUP_RAID0, + .num_stripes = 4, + .alloc_offsets = { + BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN * 2, + BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN, + }, + .expected_result = -EIO, + }, + { + .description = "RAID0: fail: far distance", + .raid_type = BTRFS_BLOCK_GROUP_RAID0, + .num_stripes = 4, + .alloc_offsets = { + BTRFS_STRIPE_LEN * 3, BTRFS_STRIPE_LEN, + BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN, + }, + .expected_result = -EIO, + }, + { + .description = "RAID0: fail: too many partial write", + .raid_type = BTRFS_BLOCK_GROUP_RAID0, + .num_stripes = 4, + .alloc_offsets = { + HALF_STRIPE_LEN, HALF_STRIPE_LEN, 0, 0, + }, + .expected_result = -EIO, + }, + /* + * Error case: Partial missing device is not allowed even on non-DEGRADED + * mount. + */ + { + .description = "RAID0: fail: missing device on DEGRADED", + .raid_type = BTRFS_BLOCK_GROUP_RAID0, + .num_stripes = 2, + .alloc_offsets = { + SZ_1M, WP_MISSING_DEV, + }, + .degraded = true, + .expected_result = -EIO, + }, + + /* + * One sequential zone and one conventional zone, having matching + * last_alloc. + */ + { + .description = "RAID0: seq zone and conv zone, partially written stripe", + .raid_type = BTRFS_BLOCK_GROUP_RAID0, + .num_stripes = 2, + .alloc_offsets = { + SZ_1M, WP_CONVENTIONAL, + }, + .last_alloc = SZ_2M - SZ_4K, + .expected_alloc_offset = SZ_2M - SZ_4K, + }, + { + .description = "RAID0: conv zone and seq zone, partially written stripe", + .raid_type = BTRFS_BLOCK_GROUP_RAID0, + .num_stripes = 2, + .alloc_offsets = { + WP_CONVENTIONAL, SZ_1M, + }, + .last_alloc = SZ_2M + SZ_4K, + .expected_alloc_offset = SZ_2M + SZ_4K, + }, + /* + * Error case: one sequential and one conventional zone, but having larger + * last_alloc than write pointer. + */ + { + .description = "RAID0: fail: seq zone and conv zone, larger last_alloc", + .raid_type = BTRFS_BLOCK_GROUP_RAID0, + .num_stripes = 2, + .alloc_offsets = { + SZ_1M, WP_CONVENTIONAL, + }, + .last_alloc = SZ_2M + BTRFS_STRIPE_LEN * 2, + .expected_result = -EIO, + }, + + /* RAID0, 4 stripes with seq zones and conv zones. */ + { + .description = "RAID0: stripes [2, 2, ?, ?] last_alloc = 6", + .raid_type = BTRFS_BLOCK_GROUP_RAID0, + .num_stripes = 4, + .alloc_offsets = { + BTRFS_STRIPE_LEN * 2, BTRFS_STRIPE_LEN * 2, + WP_CONVENTIONAL, WP_CONVENTIONAL, + }, + .last_alloc = BTRFS_STRIPE_LEN * 6, + .expected_alloc_offset = BTRFS_STRIPE_LEN * 6, + }, + { + .description = "RAID0: stripes [2, 2, ?, ?] last_alloc = 7.5", + .raid_type = BTRFS_BLOCK_GROUP_RAID0, + .num_stripes = 4, + .alloc_offsets = { + BTRFS_STRIPE_LEN * 2, BTRFS_STRIPE_LEN * 2, + WP_CONVENTIONAL, WP_CONVENTIONAL, + }, + .last_alloc = BTRFS_STRIPE_LEN * 7 + HALF_STRIPE_LEN, + .expected_alloc_offset = BTRFS_STRIPE_LEN * 7 + HALF_STRIPE_LEN, + }, + { + .description = "RAID0: stripes [3, ?, ?, ?] last_alloc = 1", + .raid_type = BTRFS_BLOCK_GROUP_RAID0, + .num_stripes = 4, + .alloc_offsets = { + BTRFS_STRIPE_LEN * 3, WP_CONVENTIONAL, + WP_CONVENTIONAL, WP_CONVENTIONAL, + }, + .last_alloc = BTRFS_STRIPE_LEN, + .expected_alloc_offset = BTRFS_STRIPE_LEN * 9, + }, + { + .description = "RAID0: stripes [2, ?, 1, ?] last_alloc = 5", + .raid_type = BTRFS_BLOCK_GROUP_RAID0, + .num_stripes = 4, + .alloc_offsets = { + BTRFS_STRIPE_LEN * 2, WP_CONVENTIONAL, + BTRFS_STRIPE_LEN, WP_CONVENTIONAL, + }, + .last_alloc = BTRFS_STRIPE_LEN * 5, + .expected_alloc_offset = BTRFS_STRIPE_LEN * 5, + }, + { + .description = "RAID0: fail: stripes [2, ?, 1, ?] last_alloc = 7", + .raid_type = BTRFS_BLOCK_GROUP_RAID0, + .num_stripes = 4, + .alloc_offsets = { + BTRFS_STRIPE_LEN * 2, WP_CONVENTIONAL, + BTRFS_STRIPE_LEN, WP_CONVENTIONAL, + }, + .last_alloc = BTRFS_STRIPE_LEN * 7, + .expected_result = -EIO, + }, + + /* RAID10 */ + /* Normal case */ + { + .description = "RAID10: initial partial write", + .raid_type = BTRFS_BLOCK_GROUP_RAID10, + .num_stripes = 4, + .alloc_offsets = { + HALF_STRIPE_LEN, HALF_STRIPE_LEN, 0, 0, + }, + .expected_alloc_offset = HALF_STRIPE_LEN, + }, + { + .description = "RAID10: while in second stripe", + .raid_type = BTRFS_BLOCK_GROUP_RAID10, + .num_stripes = 8, + .alloc_offsets = { + BTRFS_STRIPE_LEN * 2, BTRFS_STRIPE_LEN * 2, + BTRFS_STRIPE_LEN + HALF_STRIPE_LEN, + BTRFS_STRIPE_LEN + HALF_STRIPE_LEN, + BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN, + BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN, + }, + .expected_alloc_offset = BTRFS_STRIPE_LEN * 5 + HALF_STRIPE_LEN, + }, + { + .description = "RAID10: one stripe advanced", + .raid_type = BTRFS_BLOCK_GROUP_RAID10, + .num_stripes = 4, + .alloc_offsets = { + SZ_1M + BTRFS_STRIPE_LEN, SZ_1M + BTRFS_STRIPE_LEN, + SZ_1M, SZ_1M, + }, + .expected_alloc_offset = SZ_2M + BTRFS_STRIPE_LEN, + }, + { + .description = "RAID10: one stripe advanced, with conventional zone", + .raid_type = BTRFS_BLOCK_GROUP_RAID10, + .num_stripes = 4, + .alloc_offsets = { + SZ_1M + BTRFS_STRIPE_LEN, WP_CONVENTIONAL, + WP_CONVENTIONAL, SZ_1M, + }, + .expected_alloc_offset = SZ_2M + BTRFS_STRIPE_LEN, + }, + /* Error case: having different write pointers. */ + { + .description = "RAID10: fail: disordered stripes", + .raid_type = BTRFS_BLOCK_GROUP_RAID10, + .num_stripes = 8, + .alloc_offsets = { + BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN, + BTRFS_STRIPE_LEN * 2, BTRFS_STRIPE_LEN * 2, + BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN, + BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN, + }, + .expected_result = -EIO, + }, + { + .description = "RAID10: fail: far distance", + .raid_type = BTRFS_BLOCK_GROUP_RAID10, + .num_stripes = 8, + .alloc_offsets = { + BTRFS_STRIPE_LEN * 3, BTRFS_STRIPE_LEN * 3, + BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN, + BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN, + BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN, + }, + .expected_result = -EIO, + }, + { + .description = "RAID10: fail: too many partial write", + .raid_type = BTRFS_BLOCK_GROUP_RAID10, + .num_stripes = 8, + .alloc_offsets = { + HALF_STRIPE_LEN, HALF_STRIPE_LEN, + HALF_STRIPE_LEN, HALF_STRIPE_LEN, + 0, 0, 0, 0, + }, + .expected_result = -EIO, + }, + /* + * Error case: Partial missing device in RAID0 level is not allowed even on + * non-DEGRADED mount. + */ + { + .description = "RAID10: fail: missing device on DEGRADED", + .raid_type = BTRFS_BLOCK_GROUP_RAID10, + .num_stripes = 4, + .alloc_offsets = { + SZ_1M, SZ_1M, + WP_MISSING_DEV, WP_MISSING_DEV, + }, + .degraded = true, + .expected_result = -EIO, + }, + + /* + * One sequential zone and one conventional zone, having matching + * last_alloc. + */ + { + .description = "RAID10: seq zone and conv zone, partially written stripe", + .raid_type = BTRFS_BLOCK_GROUP_RAID10, + .num_stripes = 4, + .alloc_offsets = { + SZ_1M, SZ_1M, + WP_CONVENTIONAL, WP_CONVENTIONAL, + }, + .last_alloc = SZ_2M - SZ_4K, + .expected_alloc_offset = SZ_2M - SZ_4K, + }, + { + .description = "RAID10: conv zone and seq zone, partially written stripe", + .raid_type = BTRFS_BLOCK_GROUP_RAID10, + .num_stripes = 4, + .alloc_offsets = { + WP_CONVENTIONAL, WP_CONVENTIONAL, + SZ_1M, SZ_1M, + }, + .last_alloc = SZ_2M + SZ_4K, + .expected_alloc_offset = SZ_2M + SZ_4K, + }, + /* + * Error case: one sequential and one conventional zone, but having larger + * last_alloc than write pointer. + */ + { + .description = "RAID10: fail: seq zone and conv zone, larger last_alloc", + .raid_type = BTRFS_BLOCK_GROUP_RAID10, + .num_stripes = 4, + .alloc_offsets = { + SZ_1M, SZ_1M, + WP_CONVENTIONAL, WP_CONVENTIONAL, + }, + .last_alloc = SZ_2M + BTRFS_STRIPE_LEN * 2, + .expected_result = -EIO, + }, + + /* RAID10, 4 stripes with seq zones and conv zones. */ + { + .description = "RAID10: stripes [2, 2, ?, ?] last_alloc = 6", + .raid_type = BTRFS_BLOCK_GROUP_RAID10, + .num_stripes = 8, + .alloc_offsets = { + BTRFS_STRIPE_LEN * 2, BTRFS_STRIPE_LEN * 2, + BTRFS_STRIPE_LEN * 2, BTRFS_STRIPE_LEN * 2, + WP_CONVENTIONAL, WP_CONVENTIONAL, + WP_CONVENTIONAL, WP_CONVENTIONAL, + }, + .last_alloc = BTRFS_STRIPE_LEN * 6, + .expected_alloc_offset = BTRFS_STRIPE_LEN * 6, + }, + { + .description = "RAID10: stripes [2, 2, ?, ?] last_alloc = 7.5", + .raid_type = BTRFS_BLOCK_GROUP_RAID10, + .num_stripes = 8, + .alloc_offsets = { + BTRFS_STRIPE_LEN * 2, BTRFS_STRIPE_LEN * 2, + BTRFS_STRIPE_LEN * 2, BTRFS_STRIPE_LEN * 2, + WP_CONVENTIONAL, WP_CONVENTIONAL, + WP_CONVENTIONAL, WP_CONVENTIONAL, + }, + .last_alloc = BTRFS_STRIPE_LEN * 7 + HALF_STRIPE_LEN, + .expected_alloc_offset = BTRFS_STRIPE_LEN * 7 + HALF_STRIPE_LEN, + }, + { + .description = "RAID10: stripes [3, ?, ?, ?] last_alloc = 1", + .raid_type = BTRFS_BLOCK_GROUP_RAID10, + .num_stripes = 8, + .alloc_offsets = { + BTRFS_STRIPE_LEN * 3, BTRFS_STRIPE_LEN * 3, + WP_CONVENTIONAL, WP_CONVENTIONAL, + WP_CONVENTIONAL, WP_CONVENTIONAL, + WP_CONVENTIONAL, WP_CONVENTIONAL, + }, + .last_alloc = BTRFS_STRIPE_LEN, + .expected_alloc_offset = BTRFS_STRIPE_LEN * 9, + }, + { + .description = "RAID10: stripes [2, ?, 1, ?] last_alloc = 5", + .raid_type = BTRFS_BLOCK_GROUP_RAID10, + .num_stripes = 8, + .alloc_offsets = { + BTRFS_STRIPE_LEN * 2, BTRFS_STRIPE_LEN * 2, + WP_CONVENTIONAL, WP_CONVENTIONAL, + BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN, + WP_CONVENTIONAL, WP_CONVENTIONAL, + }, + .last_alloc = BTRFS_STRIPE_LEN * 5, + .expected_alloc_offset = BTRFS_STRIPE_LEN * 5, + }, + { + .description = "RAID10: fail: stripes [2, ?, 1, ?] last_alloc = 7", + .raid_type = BTRFS_BLOCK_GROUP_RAID10, + .num_stripes = 8, + .alloc_offsets = { + BTRFS_STRIPE_LEN * 2, BTRFS_STRIPE_LEN * 2, + WP_CONVENTIONAL, WP_CONVENTIONAL, + BTRFS_STRIPE_LEN, BTRFS_STRIPE_LEN, + WP_CONVENTIONAL, WP_CONVENTIONAL, + }, + .last_alloc = BTRFS_STRIPE_LEN * 7, + .expected_result = -EIO, + }, +}; + +int btrfs_test_zoned(void) +{ + struct btrfs_fs_info *fs_info __free(btrfs_free_dummy_fs_info) = NULL; + int ret; + + test_msg("running zoned tests (error messages are expected)"); + + fs_info = btrfs_alloc_dummy_fs_info(PAGE_SIZE, PAGE_SIZE); + if (!fs_info) { + test_std_err(TEST_ALLOC_FS_INFO); + return -ENOMEM; + } + + for (int i = 0; i < ARRAY_SIZE(load_zone_info_tests); i++) { + ret = test_load_zone_info(fs_info, &load_zone_info_tests[i]); + if (ret) { + test_err("test case \"%s\" failed", load_zone_info_tests[i].description); + return ret; + } + } + + return 0; +} diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c new file mode 100644 index 000000000..bafc62cf5 --- /dev/null +++ b/fs/btrfs/transaction.c @@ -0,0 +1,2768 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "misc.h" +#include "ctree.h" +#include "disk-io.h" +#include "extent_io.h" +#include "transaction.h" +#include "locking.h" +#include "tree-log.h" +#include "volumes.h" +#include "dev-replace.h" +#include "qgroup.h" +#include "block-group.h" +#include "space-info.h" +#include "fs.h" +#include "accessors.h" +#include "extent-tree.h" +#include "root-tree.h" +#include "dir-item.h" +#include "uuid-tree.h" +#include "ioctl.h" +#include "relocation.h" +#include "scrub.h" +#include "ordered-data.h" +#include "delayed-inode.h" + +static struct kmem_cache *btrfs_trans_handle_cachep; + +/* + * Transaction states and transitions + * + * No running transaction (fs tree blocks are not modified) + * | + * | To next stage: + * | Call start_transaction() variants. Except btrfs_join_transaction_nostart(). + * V + * Transaction N [[TRANS_STATE_RUNNING]] + * | + * | New trans handles can be attached to transaction N by calling all + * | start_transaction() variants. + * | + * | To next stage: + * | Call btrfs_commit_transaction() on any trans handle attached to + * | transaction N + * V + * Transaction N [[TRANS_STATE_COMMIT_PREP]] + * | + * | If there are simultaneous calls to btrfs_commit_transaction() one will win + * | the race and the rest will wait for the winner to commit the transaction. + * | + * | The winner will wait for previous running transaction to completely finish + * | if there is one. + * | + * Transaction N [[TRANS_STATE_COMMIT_START]] + * | + * | Then one of the following happens: + * | - Wait for all other trans handle holders to release. + * | The btrfs_commit_transaction() caller will do the commit work. + * | - Wait for current transaction to be committed by others. + * | Other btrfs_commit_transaction() caller will do the commit work. + * | + * | At this stage, only btrfs_join_transaction*() variants can attach + * | to this running transaction. + * | All other variants will wait for current one to finish and attach to + * | transaction N+1. + * | + * | To next stage: + * | Caller is chosen to commit transaction N, and all other trans handle + * | haven been released. + * V + * Transaction N [[TRANS_STATE_COMMIT_DOING]] + * | + * | The heavy lifting transaction work is started. + * | From running delayed refs (modifying extent tree) to creating pending + * | snapshots, running qgroups. + * | In short, modify supporting trees to reflect modifications of subvolume + * | trees. + * | + * | At this stage, all start_transaction() calls will wait for this + * | transaction to finish and attach to transaction N+1. + * | + * | To next stage: + * | Until all supporting trees are updated. + * V + * Transaction N [[TRANS_STATE_UNBLOCKED]] + * | Transaction N+1 + * | All needed trees are modified, thus we only [[TRANS_STATE_RUNNING]] + * | need to write them back to disk and update | + * | super blocks. | + * | | + * | At this stage, new transaction is allowed to | + * | start. | + * | All new start_transaction() calls will be | + * | attached to transid N+1. | + * | | + * | To next stage: | + * | Until all tree blocks and super blocks are | + * | written to block devices | + * V | + * Transaction N [[TRANS_STATE_COMPLETED]] V + * All tree blocks and super blocks are written. Transaction N+1 + * This transaction is finished and all its [[TRANS_STATE_COMMIT_START]] + * data structures will be cleaned up. | Life goes on + */ +static const unsigned int btrfs_blocked_trans_types[TRANS_STATE_MAX] = { + [TRANS_STATE_RUNNING] = 0U, + [TRANS_STATE_COMMIT_PREP] = 0U, + [TRANS_STATE_COMMIT_START] = (__TRANS_START | __TRANS_ATTACH), + [TRANS_STATE_COMMIT_DOING] = (__TRANS_START | + __TRANS_ATTACH | + __TRANS_JOIN | + __TRANS_JOIN_NOSTART), + [TRANS_STATE_UNBLOCKED] = (__TRANS_START | + __TRANS_ATTACH | + __TRANS_JOIN | + __TRANS_JOIN_NOLOCK | + __TRANS_JOIN_NOSTART), + [TRANS_STATE_SUPER_COMMITTED] = (__TRANS_START | + __TRANS_ATTACH | + __TRANS_JOIN | + __TRANS_JOIN_NOLOCK | + __TRANS_JOIN_NOSTART), + [TRANS_STATE_COMPLETED] = (__TRANS_START | + __TRANS_ATTACH | + __TRANS_JOIN | + __TRANS_JOIN_NOLOCK | + __TRANS_JOIN_NOSTART), +}; + +void btrfs_put_transaction(struct btrfs_transaction *transaction) +{ + if (refcount_dec_and_test(&transaction->use_count)) { + BUG_ON(!list_empty(&transaction->list)); + WARN_ON(!xa_empty(&transaction->delayed_refs.head_refs)); + WARN_ON(!xa_empty(&transaction->delayed_refs.dirty_extents)); + if (transaction->delayed_refs.pending_csums) + btrfs_err(transaction->fs_info, + "pending csums is %llu", + transaction->delayed_refs.pending_csums); + /* + * If any block groups are found in ->deleted_bgs then it's + * because the transaction was aborted and a commit did not + * happen (things failed before writing the new superblock + * and calling btrfs_finish_extent_commit()), so we can not + * discard the physical locations of the block groups. + */ + while (!list_empty(&transaction->deleted_bgs)) { + struct btrfs_block_group *cache; + + cache = list_first_entry(&transaction->deleted_bgs, + struct btrfs_block_group, + bg_list); + /* + * Not strictly necessary to lock, as no other task will be using a + * block_group on the deleted_bgs list during a transaction abort. + */ + spin_lock(&transaction->fs_info->unused_bgs_lock); + list_del_init(&cache->bg_list); + spin_unlock(&transaction->fs_info->unused_bgs_lock); + btrfs_unfreeze_block_group(cache); + btrfs_put_block_group(cache); + } + WARN_ON(!list_empty(&transaction->dev_update_list)); + kfree(transaction); + } +} + +static noinline void switch_commit_roots(struct btrfs_trans_handle *trans) +{ + struct btrfs_transaction *cur_trans = trans->transaction; + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *root, *tmp; + + /* + * At this point no one can be using this transaction to modify any tree + * and no one can start another transaction to modify any tree either. + */ + ASSERT(cur_trans->state == TRANS_STATE_COMMIT_DOING, + "cur_trans->state=%d", cur_trans->state); + + down_write(&fs_info->commit_root_sem); + + if (test_bit(BTRFS_FS_RELOC_RUNNING, &fs_info->flags)) + fs_info->last_reloc_trans = trans->transid; + + list_for_each_entry_safe(root, tmp, &cur_trans->switch_commits, + dirty_list) { + list_del_init(&root->dirty_list); + free_extent_buffer(root->commit_root); + root->commit_root = btrfs_root_node(root); + btrfs_extent_io_tree_release(&root->dirty_log_pages); + btrfs_qgroup_clean_swapped_blocks(root); + } + + /* We can free old roots now. */ + spin_lock(&cur_trans->dropped_roots_lock); + while (!list_empty(&cur_trans->dropped_roots)) { + root = list_first_entry(&cur_trans->dropped_roots, + struct btrfs_root, root_list); + list_del_init(&root->root_list); + spin_unlock(&cur_trans->dropped_roots_lock); + btrfs_free_log(trans, root); + btrfs_drop_and_free_fs_root(fs_info, root); + spin_lock(&cur_trans->dropped_roots_lock); + } + spin_unlock(&cur_trans->dropped_roots_lock); + + up_write(&fs_info->commit_root_sem); +} + +static inline void extwriter_counter_inc(struct btrfs_transaction *trans, + unsigned int type) +{ + if (type & TRANS_EXTWRITERS) + atomic_inc(&trans->num_extwriters); +} + +static inline void extwriter_counter_dec(struct btrfs_transaction *trans, + unsigned int type) +{ + if (type & TRANS_EXTWRITERS) + atomic_dec(&trans->num_extwriters); +} + +static inline void extwriter_counter_init(struct btrfs_transaction *trans, + unsigned int type) +{ + atomic_set(&trans->num_extwriters, ((type & TRANS_EXTWRITERS) ? 1 : 0)); +} + +static inline int extwriter_counter_read(struct btrfs_transaction *trans) +{ + return atomic_read(&trans->num_extwriters); +} + +/* + * To be called after doing the chunk btree updates right after allocating a new + * chunk (after btrfs_chunk_alloc_add_chunk_item() is called), when removing a + * chunk after all chunk btree updates and after finishing the second phase of + * chunk allocation (btrfs_create_pending_block_groups()) in case some block + * group had its chunk item insertion delayed to the second phase. + */ +void btrfs_trans_release_chunk_metadata(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + + if (!trans->chunk_bytes_reserved) + return; + + btrfs_block_rsv_release(fs_info, &fs_info->chunk_block_rsv, + trans->chunk_bytes_reserved, NULL); + trans->chunk_bytes_reserved = 0; +} + +/* + * either allocate a new transaction or hop into the existing one + */ +static noinline int join_transaction(struct btrfs_fs_info *fs_info, + unsigned int type) +{ + struct btrfs_transaction *cur_trans; + + spin_lock(&fs_info->trans_lock); +loop: + /* The file system has been taken offline. No new transactions. */ + if (unlikely(BTRFS_FS_ERROR(fs_info))) { + spin_unlock(&fs_info->trans_lock); + return -EROFS; + } + + cur_trans = fs_info->running_transaction; + if (cur_trans) { + if (TRANS_ABORTED(cur_trans)) { + const int abort_error = cur_trans->aborted; + + spin_unlock(&fs_info->trans_lock); + return abort_error; + } + if (btrfs_blocked_trans_types[cur_trans->state] & type) { + spin_unlock(&fs_info->trans_lock); + return -EBUSY; + } + refcount_inc(&cur_trans->use_count); + atomic_inc(&cur_trans->num_writers); + extwriter_counter_inc(cur_trans, type); + spin_unlock(&fs_info->trans_lock); + btrfs_lockdep_acquire(fs_info, btrfs_trans_num_writers); + btrfs_lockdep_acquire(fs_info, btrfs_trans_num_extwriters); + return 0; + } + spin_unlock(&fs_info->trans_lock); + + /* + * If we are ATTACH or TRANS_JOIN_NOSTART, we just want to catch the + * current transaction, and commit it. If there is no transaction, just + * return ENOENT. + */ + if (type == TRANS_ATTACH || type == TRANS_JOIN_NOSTART) + return -ENOENT; + + /* + * JOIN_NOLOCK only happens during the transaction commit, so + * it is impossible that ->running_transaction is NULL + */ + BUG_ON(type == TRANS_JOIN_NOLOCK); + + cur_trans = kmalloc_obj(*cur_trans, GFP_NOFS); + if (!cur_trans) + return -ENOMEM; + + btrfs_lockdep_acquire(fs_info, btrfs_trans_num_writers); + btrfs_lockdep_acquire(fs_info, btrfs_trans_num_extwriters); + + spin_lock(&fs_info->trans_lock); + if (fs_info->running_transaction) { + /* + * someone started a transaction after we unlocked. Make sure + * to redo the checks above + */ + btrfs_lockdep_release(fs_info, btrfs_trans_num_extwriters); + btrfs_lockdep_release(fs_info, btrfs_trans_num_writers); + kfree(cur_trans); + goto loop; + } else if (unlikely(BTRFS_FS_ERROR(fs_info))) { + spin_unlock(&fs_info->trans_lock); + btrfs_lockdep_release(fs_info, btrfs_trans_num_extwriters); + btrfs_lockdep_release(fs_info, btrfs_trans_num_writers); + kfree(cur_trans); + return -EROFS; + } + + cur_trans->fs_info = fs_info; + atomic_set(&cur_trans->pending_ordered, 0); + init_waitqueue_head(&cur_trans->pending_wait); + atomic_set(&cur_trans->num_writers, 1); + extwriter_counter_init(cur_trans, type); + init_waitqueue_head(&cur_trans->writer_wait); + init_waitqueue_head(&cur_trans->commit_wait); + cur_trans->state = TRANS_STATE_RUNNING; + /* + * One for this trans handle, one so it will live on until we + * commit the transaction. + */ + refcount_set(&cur_trans->use_count, 2); + cur_trans->flags = 0; + cur_trans->start_time = ktime_get_seconds(); + + memset(&cur_trans->delayed_refs, 0, sizeof(cur_trans->delayed_refs)); + + xa_init(&cur_trans->delayed_refs.head_refs); + xa_init(&cur_trans->delayed_refs.dirty_extents); + + /* + * although the tree mod log is per file system and not per transaction, + * the log must never go across transaction boundaries. + */ + smp_mb(); + if (!list_empty(&fs_info->tree_mod_seq_list)) + WARN(1, KERN_ERR "BTRFS: tree_mod_seq_list not empty when creating a fresh transaction\n"); + if (!RB_EMPTY_ROOT(&fs_info->tree_mod_log)) + WARN(1, KERN_ERR "BTRFS: tree_mod_log rb tree not empty when creating a fresh transaction\n"); + atomic64_set(&fs_info->tree_mod_seq, 0); + + spin_lock_init(&cur_trans->delayed_refs.lock); + + INIT_LIST_HEAD(&cur_trans->pending_snapshots); + INIT_LIST_HEAD(&cur_trans->dev_update_list); + INIT_LIST_HEAD(&cur_trans->switch_commits); + INIT_LIST_HEAD(&cur_trans->dirty_bgs); + INIT_LIST_HEAD(&cur_trans->io_bgs); + INIT_LIST_HEAD(&cur_trans->dropped_roots); + mutex_init(&cur_trans->cache_write_mutex); + spin_lock_init(&cur_trans->dirty_bgs_lock); + INIT_LIST_HEAD(&cur_trans->deleted_bgs); + spin_lock_init(&cur_trans->dropped_roots_lock); + list_add_tail(&cur_trans->list, &fs_info->trans_list); + btrfs_extent_io_tree_init(fs_info, &cur_trans->dirty_pages, + IO_TREE_TRANS_DIRTY_PAGES); + btrfs_extent_io_tree_init(fs_info, &cur_trans->pinned_extents, + IO_TREE_FS_PINNED_EXTENTS); + btrfs_set_fs_generation(fs_info, fs_info->generation + 1); + cur_trans->transid = fs_info->generation; + fs_info->running_transaction = cur_trans; + cur_trans->aborted = 0; + trace_btrfs_transaction_start(cur_trans); + spin_unlock(&fs_info->trans_lock); + + return 0; +} + +/* + * This does all the record keeping required to make sure that a shareable root + * is properly recorded in a given transaction. This is required to make sure + * the old root from before we joined the transaction is deleted when the + * transaction commits. + */ +static int record_root_in_trans(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + bool force) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + int ret = 0; + + if ((test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && + btrfs_get_root_last_trans(root) < trans->transid) || force) { + WARN_ON(!force && root->commit_root != root->node); + + /* + * see below for IN_TRANS_SETUP usage rules + * we have the reloc mutex held now, so there + * is only one writer in this function + */ + set_bit(BTRFS_ROOT_IN_TRANS_SETUP, &root->state); + + /* make sure readers find IN_TRANS_SETUP before + * they find our root->last_trans update + */ + smp_wmb(); + + spin_lock(&fs_info->fs_roots_radix_lock); + if (btrfs_get_root_last_trans(root) == trans->transid && !force) { + spin_unlock(&fs_info->fs_roots_radix_lock); + return 0; + } + radix_tree_tag_set(&fs_info->fs_roots_radix, + (unsigned long)btrfs_root_id(root), + BTRFS_ROOT_TRANS_TAG); + spin_unlock(&fs_info->fs_roots_radix_lock); + btrfs_set_root_last_trans(root, trans->transid); + + /* this is pretty tricky. We don't want to + * take the relocation lock in btrfs_record_root_in_trans + * unless we're really doing the first setup for this root in + * this transaction. + * + * Normally we'd use root->last_trans as a flag to decide + * if we want to take the expensive mutex. + * + * But, we have to set root->last_trans before we + * init the relocation root, otherwise, we trip over warnings + * in ctree.c. The solution used here is to flag ourselves + * with root IN_TRANS_SETUP. When this is 1, we're still + * fixing up the reloc trees and everyone must wait. + * + * When this is zero, they can trust root->last_trans and fly + * through btrfs_record_root_in_trans without having to take the + * lock. smp_wmb() makes sure that all the writes above are + * done before we pop in the zero below + */ + ret = btrfs_init_reloc_root(trans, root); + smp_mb__before_atomic(); + clear_bit(BTRFS_ROOT_IN_TRANS_SETUP, &root->state); + } + return ret; +} + + +void btrfs_add_dropped_root(struct btrfs_trans_handle *trans, + struct btrfs_root *root) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_transaction *cur_trans = trans->transaction; + + /* Add ourselves to the transaction dropped list */ + spin_lock(&cur_trans->dropped_roots_lock); + list_add_tail(&root->root_list, &cur_trans->dropped_roots); + spin_unlock(&cur_trans->dropped_roots_lock); + + /* Make sure we don't try to update the root at commit time */ + spin_lock(&fs_info->fs_roots_radix_lock); + radix_tree_tag_clear(&fs_info->fs_roots_radix, + (unsigned long)btrfs_root_id(root), + BTRFS_ROOT_TRANS_TAG); + spin_unlock(&fs_info->fs_roots_radix_lock); +} + +int btrfs_record_root_in_trans(struct btrfs_trans_handle *trans, + struct btrfs_root *root) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + int ret; + + if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state)) + return 0; + + /* + * see record_root_in_trans for comments about IN_TRANS_SETUP usage + * and barriers + */ + smp_rmb(); + if (btrfs_get_root_last_trans(root) == trans->transid && + !test_bit(BTRFS_ROOT_IN_TRANS_SETUP, &root->state)) + return 0; + + mutex_lock(&fs_info->reloc_mutex); + ret = record_root_in_trans(trans, root, false); + mutex_unlock(&fs_info->reloc_mutex); + + return ret; +} + +static inline int is_transaction_blocked(struct btrfs_transaction *trans) +{ + return (trans->state >= TRANS_STATE_COMMIT_START && + trans->state < TRANS_STATE_UNBLOCKED && + !TRANS_ABORTED(trans)); +} + +/* wait for commit against the current transaction to become unblocked + * when this is done, it is safe to start a new transaction, but the current + * transaction might not be fully on disk. + */ +static void wait_current_trans(struct btrfs_fs_info *fs_info, unsigned int type) +{ + struct btrfs_transaction *cur_trans; + + spin_lock(&fs_info->trans_lock); + cur_trans = fs_info->running_transaction; + if (cur_trans && is_transaction_blocked(cur_trans) && + (btrfs_blocked_trans_types[cur_trans->state] & type)) { + refcount_inc(&cur_trans->use_count); + spin_unlock(&fs_info->trans_lock); + + btrfs_might_wait_for_state(fs_info, BTRFS_LOCKDEP_TRANS_UNBLOCKED); + wait_event(fs_info->transaction_wait, + cur_trans->state >= TRANS_STATE_UNBLOCKED || + TRANS_ABORTED(cur_trans)); + btrfs_put_transaction(cur_trans); + } else { + spin_unlock(&fs_info->trans_lock); + } +} + +static bool may_wait_transaction(struct btrfs_fs_info *fs_info, int type) +{ + if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) + return false; + + if (type == TRANS_START) + return true; + + return false; +} + +static inline bool need_reserve_reloc_root(struct btrfs_root *root) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + + if (!fs_info->reloc_ctl || + !test_bit(BTRFS_ROOT_SHAREABLE, &root->state) || + btrfs_root_id(root) == BTRFS_TREE_RELOC_OBJECTID || + root->reloc_root) + return false; + + return true; +} + +static int btrfs_reserve_trans_metadata(struct btrfs_fs_info *fs_info, + enum btrfs_reserve_flush_enum flush, + u64 num_bytes, + u64 *delayed_refs_bytes) +{ + struct btrfs_space_info *si = fs_info->trans_block_rsv.space_info; + u64 bytes = num_bytes + *delayed_refs_bytes; + int ret; + + /* + * We want to reserve all the bytes we may need all at once, so we only + * do 1 enospc flushing cycle per transaction start. + */ + ret = btrfs_reserve_metadata_bytes(si, bytes, flush); + + /* + * If we are an emergency flush, which can steal from the global block + * reserve, then attempt to not reserve space for the delayed refs, as + * we will consume space for them from the global block reserve. + */ + if (ret && flush == BTRFS_RESERVE_FLUSH_ALL_STEAL) { + bytes -= *delayed_refs_bytes; + *delayed_refs_bytes = 0; + ret = btrfs_reserve_metadata_bytes(si, bytes, flush); + } + + return ret; +} + +static struct btrfs_trans_handle * +start_transaction(struct btrfs_root *root, unsigned int num_items, + unsigned int type, enum btrfs_reserve_flush_enum flush, + bool enforce_qgroups) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_block_rsv *delayed_refs_rsv = &fs_info->delayed_refs_rsv; + struct btrfs_block_rsv *trans_rsv = &fs_info->trans_block_rsv; + struct btrfs_trans_handle *h; + struct btrfs_transaction *cur_trans; + u64 num_bytes = 0; + u64 qgroup_reserved = 0; + u64 delayed_refs_bytes = 0; + bool reloc_reserved = false; + bool do_chunk_alloc = false; + int ret; + + if (unlikely(BTRFS_FS_ERROR(fs_info))) + return ERR_PTR(-EROFS); + + if (current->journal_info) { + WARN_ON(type & TRANS_EXTWRITERS); + h = current->journal_info; + refcount_inc(&h->use_count); + WARN_ON(refcount_read(&h->use_count) > 2); + h->orig_rsv = h->block_rsv; + h->block_rsv = NULL; + goto got_it; + } + + /* + * Do the reservation before we join the transaction so we can do all + * the appropriate flushing if need be. + */ + if (num_items && root != fs_info->chunk_root) { + qgroup_reserved = (num_items << fs_info->nodesize_bits); + /* + * Use prealloc for now, as there might be a currently running + * transaction that could free this reserved space prematurely + * by committing. + */ + ret = btrfs_qgroup_reserve_meta_prealloc(root, qgroup_reserved, + enforce_qgroups, false); + if (ret) + return ERR_PTR(ret); + + num_bytes = btrfs_calc_insert_metadata_size(fs_info, num_items); + /* + * If we plan to insert/update/delete "num_items" from a btree, + * we will also generate delayed refs for extent buffers in the + * respective btree paths, so reserve space for the delayed refs + * that will be generated by the caller as it modifies btrees. + * Try to reserve them to avoid excessive use of the global + * block reserve. + */ + delayed_refs_bytes = btrfs_calc_delayed_ref_bytes(fs_info, num_items); + + /* + * Do the reservation for the relocation root creation + */ + if (need_reserve_reloc_root(root)) { + num_bytes += fs_info->nodesize; + reloc_reserved = true; + } + + ret = btrfs_reserve_trans_metadata(fs_info, flush, num_bytes, + &delayed_refs_bytes); + if (ret) + goto reserve_fail; + + btrfs_block_rsv_add_bytes(trans_rsv, num_bytes, true); + + if (trans_rsv->space_info->force_alloc) + do_chunk_alloc = true; + } else if (num_items == 0 && flush == BTRFS_RESERVE_FLUSH_ALL && + !btrfs_block_rsv_full(delayed_refs_rsv)) { + /* + * Some people call with btrfs_start_transaction(root, 0) + * because they can be throttled, but have some other mechanism + * for reserving space. We still want these guys to refill the + * delayed block_rsv so just add 1 items worth of reservation + * here. + */ + ret = btrfs_delayed_refs_rsv_refill(fs_info, flush); + if (ret == -EAGAIN) { + ASSERT(btrfs_is_zoned(fs_info)); + ret = btrfs_commit_current_transaction(root); + if (ret) + goto reserve_fail; + ret = btrfs_delayed_refs_rsv_refill(fs_info, flush); + } + + if (ret) + goto reserve_fail; + } +again: + h = kmem_cache_zalloc(btrfs_trans_handle_cachep, GFP_NOFS); + if (!h) { + ret = -ENOMEM; + goto alloc_fail; + } + + /* + * If we are JOIN_NOLOCK we're already committing a transaction and + * waiting on this guy, so we don't need to do the sb_start_intwrite + * because we're already holding a ref. We need this because we could + * have raced in and did an fsync() on a file which can kick a commit + * and then we deadlock with somebody doing a freeze. + * + * If we are ATTACH, it means we just want to catch the current + * transaction and commit it, so we needn't do sb_start_intwrite(). + */ + if (type & __TRANS_FREEZABLE) + sb_start_intwrite(fs_info->sb); + + if (may_wait_transaction(fs_info, type)) + wait_current_trans(fs_info, type); + + do { + ret = join_transaction(fs_info, type); + if (ret == -EBUSY) { + wait_current_trans(fs_info, type); + if (unlikely(type == TRANS_ATTACH || + type == TRANS_JOIN_NOSTART)) + ret = -ENOENT; + } + } while (ret == -EBUSY); + + if (ret < 0) + goto join_fail; + + cur_trans = fs_info->running_transaction; + + h->transid = cur_trans->transid; + h->transaction = cur_trans; + refcount_set(&h->use_count, 1); + h->fs_info = root->fs_info; + + h->type = type; + INIT_LIST_HEAD(&h->new_bgs); + btrfs_init_metadata_block_rsv(fs_info, &h->delayed_rsv, BTRFS_BLOCK_RSV_DELREFS); + + smp_mb(); + if (cur_trans->state >= TRANS_STATE_COMMIT_START && + may_wait_transaction(fs_info, type)) { + current->journal_info = h; + btrfs_commit_transaction(h); + goto again; + } + + if (num_bytes) { + trace_btrfs_space_reservation(fs_info, "transaction", + h->transid, num_bytes, 1); + h->block_rsv = trans_rsv; + h->bytes_reserved = num_bytes; + if (delayed_refs_bytes > 0) { + trace_btrfs_space_reservation(fs_info, + "local_delayed_refs_rsv", + h->transid, + delayed_refs_bytes, 1); + h->delayed_refs_bytes_reserved = delayed_refs_bytes; + btrfs_block_rsv_add_bytes(&h->delayed_rsv, delayed_refs_bytes, true); + delayed_refs_bytes = 0; + } + h->reloc_reserved = reloc_reserved; + } + +got_it: + if (!current->journal_info) + current->journal_info = h; + + /* + * If the space_info is marked ALLOC_FORCE then we'll get upgraded to + * ALLOC_FORCE the first run through, and then we won't allocate for + * anybody else who races in later. We don't care about the return + * value here. + */ + if (do_chunk_alloc && num_bytes) { + struct btrfs_space_info *space_info = h->block_rsv->space_info; + u64 flags = space_info->flags; + + btrfs_chunk_alloc(h, space_info, btrfs_get_alloc_profile(fs_info, flags), + CHUNK_ALLOC_NO_FORCE); + } + + /* + * btrfs_record_root_in_trans() needs to alloc new extents, and may + * call btrfs_join_transaction() while we're also starting a + * transaction. + * + * Thus it need to be called after current->journal_info initialized, + * or we can deadlock. + */ + ret = btrfs_record_root_in_trans(h, root); + if (ret) { + /* + * The transaction handle is fully initialized and linked with + * other structures so it needs to be ended in case of errors, + * not just freed. + */ + btrfs_end_transaction(h); + goto reserve_fail; + } + /* + * Now that we have found a transaction to be a part of, convert the + * qgroup reservation from prealloc to pertrans. A different transaction + * can't race in and free our pertrans out from under us. + */ + if (qgroup_reserved) + btrfs_qgroup_convert_reserved_meta(root, qgroup_reserved); + + return h; + +join_fail: + if (type & __TRANS_FREEZABLE) + sb_end_intwrite(fs_info->sb); + kmem_cache_free(btrfs_trans_handle_cachep, h); +alloc_fail: + if (num_bytes) + btrfs_block_rsv_release(fs_info, trans_rsv, num_bytes, NULL); + if (delayed_refs_bytes) + btrfs_space_info_free_bytes_may_use(trans_rsv->space_info, delayed_refs_bytes); +reserve_fail: + btrfs_qgroup_free_meta_prealloc(root, qgroup_reserved); + return ERR_PTR(ret); +} + +struct btrfs_trans_handle *btrfs_start_transaction(struct btrfs_root *root, + unsigned int num_items) +{ + return start_transaction(root, num_items, TRANS_START, + BTRFS_RESERVE_FLUSH_ALL, true); +} + +struct btrfs_trans_handle *btrfs_start_transaction_fallback_global_rsv( + struct btrfs_root *root, + unsigned int num_items) +{ + return start_transaction(root, num_items, TRANS_START, + BTRFS_RESERVE_FLUSH_ALL_STEAL, false); +} + +struct btrfs_trans_handle *btrfs_join_transaction(struct btrfs_root *root) +{ + return start_transaction(root, 0, TRANS_JOIN, BTRFS_RESERVE_NO_FLUSH, + true); +} + +struct btrfs_trans_handle *btrfs_join_transaction_spacecache(struct btrfs_root *root) +{ + return start_transaction(root, 0, TRANS_JOIN_NOLOCK, + BTRFS_RESERVE_NO_FLUSH, true); +} + +/* + * Similar to regular join but it never starts a transaction when none is + * running or when there's a running one at a state >= TRANS_STATE_UNBLOCKED. + * This is similar to btrfs_attach_transaction() but it allows the join to + * happen if the transaction commit already started but it's not yet in the + * "doing" phase (the state is < TRANS_STATE_COMMIT_DOING). + */ +struct btrfs_trans_handle *btrfs_join_transaction_nostart(struct btrfs_root *root) +{ + return start_transaction(root, 0, TRANS_JOIN_NOSTART, + BTRFS_RESERVE_NO_FLUSH, true); +} + +/* + * Catch the running transaction. + * + * It is used when we want to commit the current the transaction, but + * don't want to start a new one. + * + * Note: If this function return -ENOENT, it just means there is no + * running transaction. But it is possible that the inactive transaction + * is still in the memory, not fully on disk. If you hope there is no + * inactive transaction in the fs when -ENOENT is returned, you should + * invoke + * btrfs_attach_transaction_barrier() + */ +struct btrfs_trans_handle *btrfs_attach_transaction(struct btrfs_root *root) +{ + return start_transaction(root, 0, TRANS_ATTACH, + BTRFS_RESERVE_NO_FLUSH, true); +} + +/* + * Catch the running transaction. + * + * It is similar to the above function, the difference is this one + * will wait for all the inactive transactions until they fully + * complete. + */ +struct btrfs_trans_handle * +btrfs_attach_transaction_barrier(struct btrfs_root *root) +{ + struct btrfs_trans_handle *trans; + + trans = start_transaction(root, 0, TRANS_ATTACH, + BTRFS_RESERVE_NO_FLUSH, true); + if (trans == ERR_PTR(-ENOENT)) { + int ret; + + ret = btrfs_wait_for_commit(root->fs_info, 0); + if (ret) + return ERR_PTR(ret); + } + + return trans; +} + +/* Wait for a transaction commit to reach at least the given state. */ +static noinline void wait_for_commit(struct btrfs_transaction *commit, + const enum btrfs_trans_state min_state) +{ + struct btrfs_fs_info *fs_info = commit->fs_info; + u64 transid = commit->transid; + bool put = false; + + /* + * At the moment this function is called with min_state either being + * TRANS_STATE_COMPLETED or TRANS_STATE_SUPER_COMMITTED. + */ + if (min_state == TRANS_STATE_COMPLETED) + btrfs_might_wait_for_state(fs_info, BTRFS_LOCKDEP_TRANS_COMPLETED); + else + btrfs_might_wait_for_state(fs_info, BTRFS_LOCKDEP_TRANS_SUPER_COMMITTED); + + while (1) { + wait_event(commit->commit_wait, commit->state >= min_state); + if (put) + btrfs_put_transaction(commit); + + if (min_state < TRANS_STATE_COMPLETED) + break; + + /* + * A transaction isn't really completed until all of the + * previous transactions are completed, but with fsync we can + * end up with SUPER_COMMITTED transactions before a COMPLETED + * transaction. Wait for those. + */ + + spin_lock(&fs_info->trans_lock); + commit = list_first_entry_or_null(&fs_info->trans_list, + struct btrfs_transaction, + list); + if (!commit || commit->transid > transid) { + spin_unlock(&fs_info->trans_lock); + break; + } + refcount_inc(&commit->use_count); + put = true; + spin_unlock(&fs_info->trans_lock); + } +} + +int btrfs_wait_for_commit(struct btrfs_fs_info *fs_info, u64 transid) +{ + struct btrfs_transaction *cur_trans = NULL, *t; + int ret = 0; + + if (transid) { + if (transid <= btrfs_get_last_trans_committed(fs_info)) + return 0; + + /* find specified transaction */ + spin_lock(&fs_info->trans_lock); + list_for_each_entry(t, &fs_info->trans_list, list) { + if (t->transid == transid) { + cur_trans = t; + refcount_inc(&cur_trans->use_count); + ret = 0; + break; + } + if (t->transid > transid) { + ret = 0; + break; + } + } + spin_unlock(&fs_info->trans_lock); + + /* + * The specified transaction doesn't exist, or we + * raced with btrfs_commit_transaction + */ + if (!cur_trans) { + if (transid > btrfs_get_last_trans_committed(fs_info)) + ret = -EINVAL; + return ret; + } + } else { + /* find newest transaction that is committing | committed */ + spin_lock(&fs_info->trans_lock); + list_for_each_entry_reverse(t, &fs_info->trans_list, + list) { + if (t->state >= TRANS_STATE_COMMIT_START) { + if (t->state == TRANS_STATE_COMPLETED) + break; + cur_trans = t; + refcount_inc(&cur_trans->use_count); + break; + } + } + spin_unlock(&fs_info->trans_lock); + /* Nothing committing or committed. */ + if (!cur_trans) + return ret; + } + + wait_for_commit(cur_trans, TRANS_STATE_COMPLETED); + ret = cur_trans->aborted; + btrfs_put_transaction(cur_trans); + + return ret; +} + +void btrfs_throttle(struct btrfs_fs_info *fs_info) +{ + wait_current_trans(fs_info, TRANS_START); +} + +bool btrfs_should_end_transaction(struct btrfs_trans_handle *trans) +{ + struct btrfs_transaction *cur_trans = trans->transaction; + + if (cur_trans->state >= TRANS_STATE_COMMIT_START || + test_bit(BTRFS_DELAYED_REFS_FLUSHING, &cur_trans->delayed_refs.flags)) + return true; + + if (btrfs_check_space_for_delayed_refs(trans->fs_info)) + return true; + + return !!btrfs_block_rsv_check(&trans->fs_info->global_block_rsv, 50); +} + +static void btrfs_trans_release_metadata(struct btrfs_trans_handle *trans) + +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + + if (!trans->block_rsv) { + ASSERT(trans->bytes_reserved == 0, + "trans->bytes_reserved=%llu", trans->bytes_reserved); + ASSERT(trans->delayed_refs_bytes_reserved == 0, + "trans->delayed_refs_bytes_reserved=%llu", + trans->delayed_refs_bytes_reserved); + return; + } + + if (!trans->bytes_reserved) { + ASSERT(trans->delayed_refs_bytes_reserved == 0, + "trans->delayed_refs_bytes_reserved=%llu", + trans->delayed_refs_bytes_reserved); + return; + } + + ASSERT(trans->block_rsv == &fs_info->trans_block_rsv); + trace_btrfs_space_reservation(fs_info, "transaction", + trans->transid, trans->bytes_reserved, 0); + btrfs_block_rsv_release(fs_info, trans->block_rsv, + trans->bytes_reserved, NULL); + trans->bytes_reserved = 0; + + if (!trans->delayed_refs_bytes_reserved) + return; + + trace_btrfs_space_reservation(fs_info, "local_delayed_refs_rsv", + trans->transid, + trans->delayed_refs_bytes_reserved, 0); + btrfs_block_rsv_release(fs_info, &trans->delayed_rsv, + trans->delayed_refs_bytes_reserved, NULL); + trans->delayed_refs_bytes_reserved = 0; +} + +static int __btrfs_end_transaction(struct btrfs_trans_handle *trans, + int throttle) +{ + struct btrfs_fs_info *info = trans->fs_info; + struct btrfs_transaction *cur_trans = trans->transaction; + int ret = 0; + + if (refcount_read(&trans->use_count) > 1) { + refcount_dec(&trans->use_count); + trans->block_rsv = trans->orig_rsv; + return 0; + } + + btrfs_trans_release_metadata(trans); + trans->block_rsv = NULL; + + btrfs_create_pending_block_groups(trans); + + btrfs_trans_release_chunk_metadata(trans); + + if (trans->type & __TRANS_FREEZABLE) + sb_end_intwrite(info->sb); + + /* + * Uninhibit extent buffer writeback before decrementing num_writers, + * since the decrement wakes the committing thread which needs all + * buffers uninhibited to write them to disk. + */ + btrfs_uninhibit_all_eb_writeback(trans); + + WARN_ON(cur_trans != info->running_transaction); + WARN_ON(atomic_read(&cur_trans->num_writers) < 1); + atomic_dec(&cur_trans->num_writers); + extwriter_counter_dec(cur_trans, trans->type); + + cond_wake_up(&cur_trans->writer_wait); + + btrfs_lockdep_release(info, btrfs_trans_num_extwriters); + btrfs_lockdep_release(info, btrfs_trans_num_writers); + + btrfs_put_transaction(cur_trans); + + if (current->journal_info == trans) + current->journal_info = NULL; + + if (throttle) + btrfs_run_delayed_iputs(info); + + if (unlikely(TRANS_ABORTED(trans) || BTRFS_FS_ERROR(info))) { + wake_up_process(info->transaction_kthread); + if (TRANS_ABORTED(trans)) + ret = trans->aborted; + else + ret = -EROFS; + } + + kmem_cache_free(btrfs_trans_handle_cachep, trans); + return ret; +} + +int btrfs_end_transaction(struct btrfs_trans_handle *trans) +{ + return __btrfs_end_transaction(trans, 0); +} + +int btrfs_end_transaction_throttle(struct btrfs_trans_handle *trans) +{ + return __btrfs_end_transaction(trans, 1); +} + +/* + * when btree blocks are allocated, they have some corresponding bits set for + * them in one of two extent_io trees. This is used to make sure all of + * those extents are sent to disk but does not wait on them + */ +int btrfs_write_marked_extents(struct btrfs_fs_info *fs_info, + struct extent_io_tree *dirty_pages, int mark) +{ + int ret = 0; + struct address_space *mapping = fs_info->btree_inode->i_mapping; + struct extent_state *cached_state = NULL; + u64 start = 0; + u64 end; + + while (btrfs_find_first_extent_bit(dirty_pages, start, &start, &end, + mark, &cached_state)) { + bool wait_writeback = false; + + ret = btrfs_convert_extent_bit(dirty_pages, start, end, + EXTENT_NEED_WAIT, + mark, &cached_state); + /* + * convert_extent_bit can return -ENOMEM, which is most of the + * time a temporary error. So when it happens, ignore the error + * and wait for writeback of this range to finish - because we + * failed to set the bit EXTENT_NEED_WAIT for the range, a call + * to __btrfs_wait_marked_extents() would not know that + * writeback for this range started and therefore wouldn't + * wait for it to finish - we don't want to commit a + * superblock that points to btree nodes/leafs for which + * writeback hasn't finished yet (and without errors). + * We cleanup any entries left in the io tree when committing + * the transaction (through extent_io_tree_release()). + */ + if (ret == -ENOMEM) { + ret = 0; + wait_writeback = true; + } + if (!ret) + ret = filemap_fdatawrite_range(mapping, start, end); + if (!ret && wait_writeback) + btrfs_btree_wait_writeback_range(fs_info, start, end); + btrfs_free_extent_state(cached_state); + if (ret) + break; + cached_state = NULL; + cond_resched(); + start = end + 1; + } + return ret; +} + +/* + * when btree blocks are allocated, they have some corresponding bits set for + * them in one of two extent_io trees. This is used to make sure all of + * those extents are on disk for transaction or log commit. We wait + * on all the pages and clear them from the dirty pages state tree + */ +static int __btrfs_wait_marked_extents(struct btrfs_fs_info *fs_info, + struct extent_io_tree *dirty_pages) +{ + struct extent_state *cached_state = NULL; + u64 start = 0; + u64 end; + int ret = 0; + + while (btrfs_find_first_extent_bit(dirty_pages, start, &start, &end, + EXTENT_NEED_WAIT, &cached_state)) { + /* + * Ignore -ENOMEM errors returned by clear_extent_bit(). + * When committing the transaction, we'll remove any entries + * left in the io tree. For a log commit, we don't remove them + * after committing the log because the tree can be accessed + * concurrently - we do it only at transaction commit time when + * it's safe to do it (through extent_io_tree_release()). + */ + ret = btrfs_clear_extent_bit(dirty_pages, start, end, + EXTENT_NEED_WAIT, &cached_state); + if (ret == -ENOMEM) + ret = 0; + if (!ret) + btrfs_btree_wait_writeback_range(fs_info, start, end); + btrfs_free_extent_state(cached_state); + if (ret) + break; + cached_state = NULL; + cond_resched(); + start = end + 1; + } + return ret; +} + +static int btrfs_wait_extents(struct btrfs_fs_info *fs_info, + struct extent_io_tree *dirty_pages) +{ + bool errors = false; + int ret; + + ret = __btrfs_wait_marked_extents(fs_info, dirty_pages); + if (test_and_clear_bit(BTRFS_FS_BTREE_ERR, &fs_info->flags)) + errors = true; + + if (errors && !ret) + ret = -EIO; + return ret; +} + +int btrfs_wait_tree_log_extents(struct btrfs_root *log_root, int mark) +{ + struct btrfs_fs_info *fs_info = log_root->fs_info; + struct extent_io_tree *dirty_pages = &log_root->dirty_log_pages; + bool errors = false; + int ret; + + ASSERT(btrfs_root_id(log_root) == BTRFS_TREE_LOG_OBJECTID, + "root_id(log_root)=%llu", btrfs_root_id(log_root)); + + ret = __btrfs_wait_marked_extents(fs_info, dirty_pages); + if ((mark & EXTENT_DIRTY_LOG1) && + test_and_clear_bit(BTRFS_FS_LOG1_ERR, &fs_info->flags)) + errors = true; + + if ((mark & EXTENT_DIRTY_LOG2) && + test_and_clear_bit(BTRFS_FS_LOG2_ERR, &fs_info->flags)) + errors = true; + + if (errors && !ret) + ret = -EIO; + return ret; +} + +/* + * When btree blocks are allocated the corresponding extents are marked dirty. + * This function ensures such extents are persisted on disk for transaction or + * log commit. + * + * @trans: transaction whose dirty pages we'd like to write + */ +static int btrfs_write_and_wait_transaction(struct btrfs_trans_handle *trans) +{ + int ret; + int ret2; + struct extent_io_tree *dirty_pages = &trans->transaction->dirty_pages; + struct btrfs_fs_info *fs_info = trans->fs_info; + struct blk_plug plug; + + blk_start_plug(&plug); + ret = btrfs_write_marked_extents(fs_info, dirty_pages, EXTENT_DIRTY); + blk_finish_plug(&plug); + ret2 = btrfs_wait_extents(fs_info, dirty_pages); + + if (ret) + return ret; + if (ret2) + return ret2; + + btrfs_extent_io_tree_release(&trans->transaction->dirty_pages); + return 0; +} + +/* + * this is used to update the root pointer in the tree of tree roots. + * + * But, in the case of the extent allocation tree, updating the root + * pointer may allocate blocks which may change the root of the extent + * allocation tree. + * + * So, this loops and repeats and makes sure the cowonly root didn't + * change while the root pointer was being updated in the metadata. + */ +static int update_cowonly_root(struct btrfs_trans_handle *trans, + struct btrfs_root *root) +{ + int ret; + u64 old_root_bytenr; + u64 old_root_used; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_root *tree_root = fs_info->tree_root; + + old_root_used = btrfs_root_used(&root->root_item); + + while (1) { + old_root_bytenr = btrfs_root_bytenr(&root->root_item); + if (old_root_bytenr == root->node->start && + old_root_used == btrfs_root_used(&root->root_item)) + break; + + btrfs_set_root_node(&root->root_item, root->node); + ret = btrfs_update_root(trans, tree_root, + &root->root_key, + &root->root_item); + if (ret) + return ret; + + old_root_used = btrfs_root_used(&root->root_item); + } + + return 0; +} + +/* + * update all the cowonly tree roots on disk + * + * The error handling in this function may not be obvious. Any of the + * failures will cause the file system to go offline. We still need + * to clean up the delayed refs. + */ +static noinline int commit_cowonly_roots(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct list_head *dirty_bgs = &trans->transaction->dirty_bgs; + struct list_head *io_bgs = &trans->transaction->io_bgs; + struct extent_buffer *eb; + int ret; + + /* + * At this point no one can be using this transaction to modify any tree + * and no one can start another transaction to modify any tree either. + */ + ASSERT(trans->transaction->state == TRANS_STATE_COMMIT_DOING, + "trans->transaction->state=%d", trans->transaction->state); + + eb = btrfs_lock_root_node(fs_info->tree_root); + ret = btrfs_cow_block(trans, fs_info->tree_root, eb, NULL, + 0, &eb, BTRFS_NESTING_COW); + btrfs_tree_unlock(eb); + free_extent_buffer(eb); + + if (ret) + return ret; + + ret = btrfs_run_dev_stats(trans); + if (ret) + return ret; + ret = btrfs_run_dev_replace(trans); + if (ret) + return ret; + ret = btrfs_run_qgroups(trans); + if (ret) + return ret; + + ret = btrfs_setup_space_cache(trans); + if (ret) + return ret; + +again: + while (!list_empty(&fs_info->dirty_cowonly_roots)) { + struct btrfs_root *root; + + root = list_first_entry(&fs_info->dirty_cowonly_roots, + struct btrfs_root, dirty_list); + clear_bit(BTRFS_ROOT_DIRTY, &root->state); + list_move_tail(&root->dirty_list, + &trans->transaction->switch_commits); + + ret = update_cowonly_root(trans, root); + if (ret) + return ret; + } + + /* Now flush any delayed refs generated by updating all of the roots */ + ret = btrfs_run_delayed_refs(trans, U64_MAX); + if (ret) + return ret; + + while (!list_empty(dirty_bgs) || !list_empty(io_bgs)) { + ret = btrfs_write_dirty_block_groups(trans); + if (ret) + return ret; + + /* + * We're writing the dirty block groups, which could generate + * delayed refs, which could generate more dirty block groups, + * so we want to keep this flushing in this loop to make sure + * everything gets run. + */ + ret = btrfs_run_delayed_refs(trans, U64_MAX); + if (ret) + return ret; + } + + if (!list_empty(&fs_info->dirty_cowonly_roots)) + goto again; + + /* Update dev-replace pointer once everything is committed */ + fs_info->dev_replace.committed_cursor_left = + fs_info->dev_replace.cursor_left_last_write_of_item; + + return 0; +} + +/* + * If we had a pending drop we need to see if there are any others left in our + * dead roots list, and if not clear our bit and wake any waiters. + */ +void btrfs_maybe_wake_unfinished_drop(struct btrfs_fs_info *fs_info) +{ + /* + * We put the drop in progress roots at the front of the list, so if the + * first entry doesn't have UNFINISHED_DROP set we can wake everybody + * up. + */ + spin_lock(&fs_info->trans_lock); + if (!list_empty(&fs_info->dead_roots)) { + struct btrfs_root *root = list_first_entry(&fs_info->dead_roots, + struct btrfs_root, + root_list); + if (test_bit(BTRFS_ROOT_UNFINISHED_DROP, &root->state)) { + spin_unlock(&fs_info->trans_lock); + return; + } + } + spin_unlock(&fs_info->trans_lock); + + btrfs_wake_unfinished_drop(fs_info); +} + +/* + * dead roots are old snapshots that need to be deleted. This allocates + * a dirty root struct and adds it into the list of dead roots that need to + * be deleted + */ +void btrfs_add_dead_root(struct btrfs_root *root) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + + spin_lock(&fs_info->trans_lock); + if (list_empty(&root->root_list)) { + btrfs_grab_root(root); + + /* We want to process the partially complete drops first. */ + if (test_bit(BTRFS_ROOT_UNFINISHED_DROP, &root->state)) + list_add(&root->root_list, &fs_info->dead_roots); + else + list_add_tail(&root->root_list, &fs_info->dead_roots); + } + spin_unlock(&fs_info->trans_lock); +} + +/* + * Update each subvolume root and its relocation root, if it exists, in the tree + * of tree roots. Also free log roots if they exist. + */ +static noinline int commit_fs_roots(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *gang[8]; + int i; + int ret; + + /* + * At this point no one can be using this transaction to modify any tree + * and no one can start another transaction to modify any tree either. + */ + ASSERT(trans->transaction->state == TRANS_STATE_COMMIT_DOING, + "trans->transaction->state=%d", trans->transaction->state); + + spin_lock(&fs_info->fs_roots_radix_lock); + while (1) { + ret = radix_tree_gang_lookup_tag(&fs_info->fs_roots_radix, + (void **)gang, 0, + ARRAY_SIZE(gang), + BTRFS_ROOT_TRANS_TAG); + if (ret == 0) + break; + for (i = 0; i < ret; i++) { + struct btrfs_root *root = gang[i]; + int ret2; + + /* + * At this point we can neither have tasks logging inodes + * from a root nor trying to commit a log tree. + */ + ASSERT(atomic_read(&root->log_writers) == 0, + "atomic_read(&root->log_writers)=%d", + atomic_read(&root->log_writers)); + ASSERT(!root->log_commit[0]); + ASSERT(!root->log_commit[1]); + + radix_tree_tag_clear(&fs_info->fs_roots_radix, + (unsigned long)btrfs_root_id(root), + BTRFS_ROOT_TRANS_TAG); + btrfs_qgroup_free_meta_all_pertrans(root); + spin_unlock(&fs_info->fs_roots_radix_lock); + + btrfs_free_log(trans, root); + ret2 = btrfs_update_reloc_root(trans, root); + if (unlikely(ret2)) + return ret2; + + /* see comments in should_cow_block() */ + clear_bit(BTRFS_ROOT_FORCE_COW, &root->state); + smp_mb__after_atomic(); + + if (root->commit_root != root->node) { + list_add_tail(&root->dirty_list, + &trans->transaction->switch_commits); + btrfs_set_root_node(&root->root_item, + root->node); + } + + ret2 = btrfs_update_root(trans, fs_info->tree_root, + &root->root_key, + &root->root_item); + if (unlikely(ret2)) + return ret2; + spin_lock(&fs_info->fs_roots_radix_lock); + } + } + spin_unlock(&fs_info->fs_roots_radix_lock); + return 0; +} + +/* + * Do all special snapshot related qgroup dirty hack. + * + * Will do all needed qgroup inherit and dirty hack like switch commit + * roots inside one transaction and write all btree into disk, to make + * qgroup works. + */ +static int qgroup_account_snapshot(struct btrfs_trans_handle *trans, + struct btrfs_root *src, + struct btrfs_root *parent, + struct btrfs_qgroup_inherit *inherit, + u64 dst_objectid) +{ + struct btrfs_fs_info *fs_info = src->fs_info; + int ret; + + /* + * Save some performance in the case that qgroups are not enabled. If + * this check races with the ioctl, rescan will kick in anyway. + */ + if (!btrfs_qgroup_full_accounting(fs_info)) + return 0; + + /* + * Ensure dirty @src will be committed. Or, after coming + * commit_fs_roots() and switch_commit_roots(), any dirty but not + * recorded root will never be updated again, causing an outdated root + * item. + */ + ret = record_root_in_trans(trans, src, true); + if (ret) + return ret; + + /* + * btrfs_qgroup_inherit relies on a consistent view of the usage for the + * src root, so we must run the delayed refs here. + * + * However this isn't particularly fool proof, because there's no + * synchronization keeping us from changing the tree after this point + * before we do the qgroup_inherit, or even from making changes while + * we're doing the qgroup_inherit. But that's a problem for the future, + * for now flush the delayed refs to narrow the race window where the + * qgroup counters could end up wrong. + */ + ret = btrfs_run_delayed_refs(trans, U64_MAX); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + + ret = commit_fs_roots(trans); + if (ret) + return ret; + ret = btrfs_qgroup_account_extents(trans); + if (ret < 0) + return ret; + + /* Now qgroup are all updated, we can inherit it to new qgroups */ + ret = btrfs_qgroup_inherit(trans, btrfs_root_id(src), dst_objectid, + btrfs_root_id(parent), inherit); + if (ret < 0) + return ret; + + /* + * Now we do a simplified commit transaction, which will: + * 1) commit all subvolume and extent tree + * To ensure all subvolume and extent tree have a valid + * commit_root to accounting later insert_dir_item() + * 2) write all btree blocks onto disk + * This is to make sure later btree modification will be cowed + * Or commit_root can be populated and cause wrong qgroup numbers + * In this simplified commit, we don't really care about other trees + * like chunk and root tree, as they won't affect qgroup. + * And we don't write super to avoid half committed status. + */ + ret = commit_cowonly_roots(trans); + if (ret) + return ret; + switch_commit_roots(trans); + ret = btrfs_write_and_wait_transaction(trans); + if (unlikely(ret)) { + btrfs_err(fs_info, +"error while writing out transaction during qgroup snapshot accounting: %pe", ERR_PTR(ret)); + return ret; + } + + /* + * Force parent root to be updated, as we recorded it before so its + * last_trans == cur_transid. + * Or it won't be committed again onto disk after later + * insert_dir_item() + */ + return record_root_in_trans(trans, parent, true); +} + +/* + * new snapshots need to be created at a very specific time in the + * transaction commit. This does the actual creation. + * + * Note: + * If the error which may affect the commitment of the current transaction + * happens, we should return the error number. If the error which just affect + * the creation of the pending snapshots, just return 0. + */ +static noinline int create_pending_snapshot(struct btrfs_trans_handle *trans, + struct btrfs_pending_snapshot *pending) +{ + + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_key key; + struct btrfs_root_item *new_root_item; + struct btrfs_root *tree_root = fs_info->tree_root; + struct btrfs_root *root = pending->root; + struct btrfs_root *parent_root; + struct btrfs_block_rsv *rsv; + struct btrfs_inode *parent_inode = pending->dir; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_dir_item *dir_item; + struct extent_buffer *tmp; + struct extent_buffer *root_eb; + struct timespec64 cur_time; + int ret = 0; + u64 to_reserve = 0; + u64 index = 0; + u64 objectid; + u64 root_flags; + unsigned int nofs_flags; + struct fscrypt_name fname; + + ASSERT(pending->path); + path = pending->path; + + ASSERT(pending->root_item); + new_root_item = pending->root_item; + + /* + * We're inside a transaction and must make sure that any potential + * allocations with GFP_KERNEL in fscrypt won't recurse back to + * filesystem. + */ + nofs_flags = memalloc_nofs_save(); + pending->error = fscrypt_setup_filename(&parent_inode->vfs_inode, + &pending->dentry->d_name, 0, + &fname); + memalloc_nofs_restore(nofs_flags); + if (unlikely(pending->error)) + goto free_pending; + + pending->error = btrfs_get_free_objectid(tree_root, &objectid); + if (unlikely(pending->error)) + goto free_fname; + + /* + * Make qgroup to skip current new snapshot's qgroupid, as it is + * accounted by later btrfs_qgroup_inherit(). + */ + btrfs_set_skip_qgroup(trans, objectid); + + btrfs_reloc_pre_snapshot(pending, &to_reserve); + + if (to_reserve > 0) { + pending->error = btrfs_block_rsv_add(fs_info, + &pending->block_rsv, + to_reserve, + BTRFS_RESERVE_NO_FLUSH); + if (unlikely(pending->error)) + goto clear_skip_qgroup; + } + + rsv = trans->block_rsv; + trans->block_rsv = &pending->block_rsv; + trans->bytes_reserved = trans->block_rsv->reserved; + trace_btrfs_space_reservation(fs_info, "transaction", + trans->transid, + trans->bytes_reserved, 1); + parent_root = parent_inode->root; + ret = record_root_in_trans(trans, parent_root, false); + if (unlikely(ret)) + goto fail; + cur_time = current_time(&parent_inode->vfs_inode); + + /* + * insert the directory item + */ + ret = btrfs_set_inode_index(parent_inode, &index); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto fail; + } + + /* check if there is a file/dir which has the same name. */ + dir_item = btrfs_lookup_dir_item(NULL, parent_root, path, + btrfs_ino(parent_inode), + &fname.disk_name, 0); + if (!IS_ERR_OR_NULL(dir_item)) { + pending->error = -EEXIST; + goto dir_item_existed; + } else if (IS_ERR(dir_item)) { + ret = PTR_ERR(dir_item); + btrfs_abort_transaction(trans, ret); + goto fail; + } + btrfs_release_path(path); + + ret = btrfs_create_qgroup(trans, objectid); + if (ret && ret != -EEXIST) { + if (unlikely(ret != -ENOTCONN || btrfs_qgroup_enabled(fs_info))) { + btrfs_abort_transaction(trans, ret); + goto fail; + } + } + + /* + * pull in the delayed directory update + * and the delayed inode item + * otherwise we corrupt the FS during + * snapshot + */ + ret = btrfs_run_delayed_items(trans); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto fail; + } + + ret = record_root_in_trans(trans, root, false); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto fail; + } + btrfs_set_root_last_snapshot(&root->root_item, trans->transid); + memcpy(new_root_item, &root->root_item, sizeof(*new_root_item)); + btrfs_check_and_init_root_item(new_root_item); + + root_flags = btrfs_root_flags(new_root_item); + if (pending->readonly) + root_flags |= BTRFS_ROOT_SUBVOL_RDONLY; + else + root_flags &= ~BTRFS_ROOT_SUBVOL_RDONLY; + btrfs_set_root_flags(new_root_item, root_flags); + + btrfs_set_root_generation_v2(new_root_item, + trans->transid); + generate_random_guid(new_root_item->uuid); + memcpy(new_root_item->parent_uuid, root->root_item.uuid, + BTRFS_UUID_SIZE); + if (!(root_flags & BTRFS_ROOT_SUBVOL_RDONLY)) { + memset(new_root_item->received_uuid, 0, + sizeof(new_root_item->received_uuid)); + memset(&new_root_item->stime, 0, sizeof(new_root_item->stime)); + memset(&new_root_item->rtime, 0, sizeof(new_root_item->rtime)); + btrfs_set_root_stransid(new_root_item, 0); + btrfs_set_root_rtransid(new_root_item, 0); + } + btrfs_set_stack_timespec_sec(&new_root_item->otime, cur_time.tv_sec); + btrfs_set_stack_timespec_nsec(&new_root_item->otime, cur_time.tv_nsec); + btrfs_set_root_otransid(new_root_item, trans->transid); + + root_eb = btrfs_lock_root_node(root); + ret = btrfs_copy_root(trans, root, root_eb, &tmp, objectid); + btrfs_tree_unlock(root_eb); + free_extent_buffer(root_eb); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto fail; + } + /* see comments in should_cow_block() */ + set_bit(BTRFS_ROOT_FORCE_COW, &root->state); + smp_mb__after_atomic(); + + btrfs_set_root_node(new_root_item, tmp); + /* record when the snapshot was created in key.offset */ + key.objectid = objectid; + key.type = BTRFS_ROOT_ITEM_KEY; + key.offset = trans->transid; + ret = btrfs_insert_root(trans, tree_root, &key, new_root_item); + btrfs_tree_unlock(tmp); + free_extent_buffer(tmp); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto fail; + } + + /* + * insert root back/forward references + */ + ret = btrfs_add_root_ref(trans, objectid, + btrfs_root_id(parent_root), + btrfs_ino(parent_inode), index, + &fname.disk_name); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto fail; + } + + key.offset = (u64)-1; + pending->snap = btrfs_get_new_fs_root(fs_info, objectid, &pending->anon_dev); + if (IS_ERR(pending->snap)) { + ret = PTR_ERR(pending->snap); + pending->snap = NULL; + btrfs_abort_transaction(trans, ret); + goto fail; + } + + ret = btrfs_reloc_post_snapshot(trans, pending); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto fail; + } + + /* + * Do special qgroup accounting for snapshot, as we do some qgroup + * snapshot hack to do fast snapshot. + * To co-operate with that hack, we do hack again. + * Or snapshot will be greatly slowed down by a subtree qgroup rescan + */ + if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_FULL) + ret = qgroup_account_snapshot(trans, root, parent_root, + pending->inherit, objectid); + else if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE) + ret = btrfs_qgroup_inherit(trans, btrfs_root_id(root), objectid, + btrfs_root_id(parent_root), pending->inherit); + if (unlikely(ret < 0)) + goto fail; + + ret = btrfs_insert_dir_item(trans, &fname.disk_name, + parent_inode, &key, BTRFS_FT_DIR, + index); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto fail; + } + + btrfs_i_size_write(parent_inode, parent_inode->vfs_inode.i_size + + fname.disk_name.len * 2); + inode_set_mtime_to_ts(&parent_inode->vfs_inode, + inode_set_ctime_current(&parent_inode->vfs_inode)); + ret = btrfs_update_inode_fallback(trans, parent_inode); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto fail; + } + ret = btrfs_uuid_tree_add(trans, new_root_item->uuid, + BTRFS_UUID_KEY_SUBVOL, + objectid); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto fail; + } + if (!btrfs_is_empty_uuid(new_root_item->received_uuid)) { + ret = btrfs_uuid_tree_add(trans, new_root_item->received_uuid, + BTRFS_UUID_KEY_RECEIVED_SUBVOL, + objectid); + /* + * We are creating of lot of snapshots of the same root that was + * received (has a received UUID) and reached a leaf's limit for + * an item. We can safely ignore this and avoid a transaction + * abort. A deletion of this snapshot will still work since we + * ignore if an item with a BTRFS_UUID_KEY_RECEIVED_SUBVOL key + * is missing (see btrfs_delete_subvolume()). Send/receive will + * work too since it peeks the first root id from the existing + * item (it could peek any), and in case it's missing it + * falls back to search by BTRFS_UUID_KEY_SUBVOL keys. + * Creation of a snapshot does not require CAP_SYS_ADMIN, so + * we don't want users triggering transaction aborts, either + * intentionally or not. + */ + if (ret == -EOVERFLOW) + ret = 0; + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto fail; + } + } + +fail: + pending->error = ret; +dir_item_existed: + trans->block_rsv = rsv; + trans->bytes_reserved = 0; +clear_skip_qgroup: + btrfs_clear_skip_qgroup(trans); +free_fname: + fscrypt_free_filename(&fname); +free_pending: + kfree(new_root_item); + pending->root_item = NULL; + pending->path = NULL; + + return ret; +} + +/* + * create all the snapshots we've scheduled for creation + */ +static noinline int create_pending_snapshots(struct btrfs_trans_handle *trans) +{ + struct btrfs_pending_snapshot *pending, *next; + struct list_head *head = &trans->transaction->pending_snapshots; + int ret = 0; + + list_for_each_entry_safe(pending, next, head, list) { + list_del(&pending->list); + ret = create_pending_snapshot(trans, pending); + if (unlikely(ret)) + break; + } + return ret; +} + +static void update_super_roots(struct btrfs_fs_info *fs_info) +{ + struct btrfs_root_item *root_item; + struct btrfs_super_block *super; + + super = fs_info->super_copy; + + root_item = &fs_info->chunk_root->root_item; + super->chunk_root = root_item->bytenr; + super->chunk_root_generation = root_item->generation; + super->chunk_root_level = root_item->level; + + root_item = &fs_info->tree_root->root_item; + super->root = root_item->bytenr; + super->generation = root_item->generation; + super->root_level = root_item->level; + if (btrfs_test_opt(fs_info, SPACE_CACHE)) + super->cache_generation = root_item->generation; + else if (test_bit(BTRFS_FS_CLEANUP_SPACE_CACHE_V1, &fs_info->flags)) + super->cache_generation = 0; + if (test_bit(BTRFS_FS_UPDATE_UUID_TREE_GEN, &fs_info->flags)) + super->uuid_tree_generation = root_item->generation; + + if (btrfs_fs_incompat(fs_info, REMAP_TREE)) { + root_item = &fs_info->remap_root->root_item; + super->remap_root = root_item->bytenr; + super->remap_root_generation = root_item->generation; + super->remap_root_level = root_item->level; + } +} + +int btrfs_transaction_blocked(struct btrfs_fs_info *info) +{ + struct btrfs_transaction *trans; + int ret = 0; + + spin_lock(&info->trans_lock); + trans = info->running_transaction; + if (trans) + ret = is_transaction_blocked(trans); + spin_unlock(&info->trans_lock); + return ret; +} + +void btrfs_commit_transaction_async(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_transaction *cur_trans; + + /* Kick the transaction kthread. */ + set_bit(BTRFS_FS_COMMIT_TRANS, &fs_info->flags); + wake_up_process(fs_info->transaction_kthread); + + /* take transaction reference */ + cur_trans = trans->transaction; + refcount_inc(&cur_trans->use_count); + + btrfs_end_transaction(trans); + + /* + * Wait for the current transaction commit to start and block + * subsequent transaction joins + */ + btrfs_might_wait_for_state(fs_info, BTRFS_LOCKDEP_TRANS_COMMIT_PREP); + wait_event(fs_info->transaction_blocked_wait, + cur_trans->state >= TRANS_STATE_COMMIT_START || + TRANS_ABORTED(cur_trans)); + btrfs_put_transaction(cur_trans); +} + +/* + * If there is a running transaction commit it or if it's already committing, + * wait for its commit to complete. Does not start and commit a new transaction + * if there isn't any running. + */ +int btrfs_commit_current_transaction(struct btrfs_root *root) +{ + struct btrfs_trans_handle *trans; + + trans = btrfs_attach_transaction_barrier(root); + if (IS_ERR(trans)) { + int ret = PTR_ERR(trans); + + return (ret == -ENOENT) ? 0 : ret; + } + + return btrfs_commit_transaction(trans); +} + +static void cleanup_transaction(struct btrfs_trans_handle *trans, int err) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_transaction *cur_trans = trans->transaction; + + WARN_ON(refcount_read(&trans->use_count) > 1); + + btrfs_abort_transaction(trans, err); + + spin_lock(&fs_info->trans_lock); + + /* + * If the transaction is removed from the list, it means this + * transaction has been committed successfully, so it is impossible + * to call the cleanup function. + */ + BUG_ON(list_empty(&cur_trans->list)); + + if (cur_trans == fs_info->running_transaction) { + cur_trans->state = TRANS_STATE_COMMIT_DOING; + spin_unlock(&fs_info->trans_lock); + + /* + * The thread has already released the lockdep map as reader + * already in btrfs_commit_transaction(). + */ + btrfs_might_wait_for_event(fs_info, btrfs_trans_num_writers); + wait_event(cur_trans->writer_wait, + atomic_read(&cur_trans->num_writers) == 1); + + spin_lock(&fs_info->trans_lock); + } + + /* + * Now that we know no one else is still using the transaction we can + * remove the transaction from the list of transactions. This avoids + * the transaction kthread from cleaning up the transaction while some + * other task is still using it, which could result in a use-after-free + * on things like log trees, as it forces the transaction kthread to + * wait for this transaction to be cleaned up by us. + */ + list_del_init(&cur_trans->list); + + spin_unlock(&fs_info->trans_lock); + + btrfs_cleanup_one_transaction(trans->transaction); + + spin_lock(&fs_info->trans_lock); + if (cur_trans == fs_info->running_transaction) + fs_info->running_transaction = NULL; + spin_unlock(&fs_info->trans_lock); + + if (trans->type & __TRANS_FREEZABLE) + sb_end_intwrite(fs_info->sb); + btrfs_put_transaction(cur_trans); + btrfs_put_transaction(cur_trans); + + trace_btrfs_transaction_commit(trans); + + if (current->journal_info == trans) + current->journal_info = NULL; + + /* + * If relocation is running, we can't cancel scrub because that will + * result in a deadlock. Before relocating a block group, relocation + * pauses scrub, then starts and commits a transaction before unpausing + * scrub. If the transaction commit is being done by the relocation + * task or triggered by another task and the relocation task is waiting + * for the commit, and we end up here due to an error in the commit + * path, then calling btrfs_scrub_cancel() will deadlock, as we are + * asking for scrub to stop while having it asked to be paused higher + * above in relocation code. + */ + if (!test_bit(BTRFS_FS_RELOC_RUNNING, &fs_info->flags)) + btrfs_scrub_cancel(fs_info); + + btrfs_uninhibit_all_eb_writeback(trans); + kmem_cache_free(btrfs_trans_handle_cachep, trans); +} + +/* + * Release reserved delayed ref space of all pending block groups of the + * transaction and remove them from the list + */ +static void btrfs_cleanup_pending_block_groups(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_block_group *block_group, *tmp; + + list_for_each_entry_safe(block_group, tmp, &trans->new_bgs, bg_list) { + btrfs_dec_delayed_refs_rsv_bg_inserts(fs_info); + /* + * Not strictly necessary to lock, as no other task will be using a + * block_group on the new_bgs list during a transaction abort. + */ + spin_lock(&fs_info->unused_bgs_lock); + list_del_init(&block_group->bg_list); + btrfs_put_block_group(block_group); + spin_unlock(&fs_info->unused_bgs_lock); + } +} + +static inline int btrfs_start_delalloc_flush(struct btrfs_fs_info *fs_info) +{ + /* + * We use try_to_writeback_inodes_sb() here because if we used + * btrfs_start_delalloc_roots we would deadlock with fs freeze. + * Currently are holding the fs freeze lock, if we do an async flush + * we'll do btrfs_join_transaction() and deadlock because we need to + * wait for the fs freeze lock. Using the direct flushing we benefit + * from already being in a transaction and our join_transaction doesn't + * have to re-take the fs freeze lock. + * + * Note that try_to_writeback_inodes_sb() will only trigger writeback + * if it can read lock sb->s_umount. It will always be able to lock it, + * except when the filesystem is being unmounted or being frozen, but in + * those cases sync_filesystem() is called, which results in calling + * writeback_inodes_sb() while holding a write lock on sb->s_umount. + * Note that we don't call writeback_inodes_sb() directly, because it + * will emit a warning if sb->s_umount is not locked. + */ + if (btrfs_test_opt(fs_info, FLUSHONCOMMIT)) + try_to_writeback_inodes_sb(fs_info->sb, WB_REASON_SYNC); + return 0; +} + +static inline void btrfs_wait_delalloc_flush(struct btrfs_fs_info *fs_info) +{ + if (btrfs_test_opt(fs_info, FLUSHONCOMMIT)) + btrfs_wait_ordered_roots(fs_info, U64_MAX, NULL); +} + +/* + * Add a pending snapshot associated with the given transaction handle to the + * respective handle. This must be called after the transaction commit started + * and while holding fs_info->trans_lock. + * This serves to guarantee a caller of btrfs_commit_transaction() that it can + * safely free the pending snapshot pointer in case btrfs_commit_transaction() + * returns an error. + */ +static void add_pending_snapshot(struct btrfs_trans_handle *trans) +{ + struct btrfs_transaction *cur_trans = trans->transaction; + + if (!trans->pending_snapshot) + return; + + lockdep_assert_held(&trans->fs_info->trans_lock); + ASSERT(cur_trans->state >= TRANS_STATE_COMMIT_PREP, + "cur_trans->state=%d", cur_trans->state); + + list_add(&trans->pending_snapshot->list, &cur_trans->pending_snapshots); +} + +static void update_commit_stats(struct btrfs_fs_info *fs_info) +{ + ktime_t now = ktime_get_ns(); + ktime_t interval = now - fs_info->commit_stats.critical_section_start_time; + + ASSERT(fs_info->commit_stats.critical_section_start_time); + + fs_info->commit_stats.commit_count++; + fs_info->commit_stats.last_commit_dur = interval; + fs_info->commit_stats.max_commit_dur = + max_t(u64, fs_info->commit_stats.max_commit_dur, interval); + fs_info->commit_stats.total_commit_dur += interval; + fs_info->commit_stats.critical_section_start_time = 0; +} + +int btrfs_commit_transaction(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_transaction *cur_trans = trans->transaction; + struct btrfs_transaction *prev_trans = NULL; + int ret; + + ASSERT(refcount_read(&trans->use_count) == 1, + "refcount_read(&trans->use_count)=%d", refcount_read(&trans->use_count)); + btrfs_trans_state_lockdep_acquire(fs_info, BTRFS_LOCKDEP_TRANS_COMMIT_PREP); + + clear_bit(BTRFS_FS_NEED_TRANS_COMMIT, &fs_info->flags); + + /* Stop the commit early if ->aborted is set */ + if (TRANS_ABORTED(cur_trans)) { + ret = cur_trans->aborted; + goto lockdep_trans_commit_start_release; + } + + btrfs_trans_release_metadata(trans); + trans->block_rsv = NULL; + + /* + * We only want one transaction commit doing the flushing so we do not + * waste a bunch of time on lock contention on the extent root node. + */ + if (!test_and_set_bit(BTRFS_DELAYED_REFS_FLUSHING, + &cur_trans->delayed_refs.flags)) { + /* + * Make a pass through all the delayed refs we have so far. + * Any running threads may add more while we are here. + */ + ret = btrfs_run_delayed_refs(trans, 0); + if (ret) + goto lockdep_trans_commit_start_release; + } + + btrfs_create_pending_block_groups(trans); + + if (!test_bit(BTRFS_TRANS_DIRTY_BG_RUN, &cur_trans->flags)) { + bool run_it = false; + + /* this mutex is also taken before trying to set + * block groups readonly. We need to make sure + * that nobody has set a block group readonly + * after a extents from that block group have been + * allocated for cache files. btrfs_set_block_group_ro + * will wait for the transaction to commit if it + * finds BTRFS_TRANS_DIRTY_BG_RUN set. + * + * The BTRFS_TRANS_DIRTY_BG_RUN flag is also used to make sure + * only one process starts all the block group IO. It wouldn't + * hurt to have more than one go through, but there's no + * real advantage to it either. + */ + mutex_lock(&fs_info->ro_block_group_mutex); + if (!test_and_set_bit(BTRFS_TRANS_DIRTY_BG_RUN, + &cur_trans->flags)) + run_it = true; + mutex_unlock(&fs_info->ro_block_group_mutex); + + if (run_it) { + ret = btrfs_start_dirty_block_groups(trans); + if (unlikely(ret)) + goto lockdep_trans_commit_start_release; + } + } + + spin_lock(&fs_info->trans_lock); + if (cur_trans->state >= TRANS_STATE_COMMIT_PREP) { + enum btrfs_trans_state want_state = TRANS_STATE_COMPLETED; + + add_pending_snapshot(trans); + + spin_unlock(&fs_info->trans_lock); + refcount_inc(&cur_trans->use_count); + + if (trans->in_fsync) + want_state = TRANS_STATE_SUPER_COMMITTED; + + btrfs_trans_state_lockdep_release(fs_info, + BTRFS_LOCKDEP_TRANS_COMMIT_PREP); + ret = btrfs_end_transaction(trans); + wait_for_commit(cur_trans, want_state); + + if (TRANS_ABORTED(cur_trans)) + ret = cur_trans->aborted; + + btrfs_put_transaction(cur_trans); + + return ret; + } + + cur_trans->state = TRANS_STATE_COMMIT_PREP; + trace_btrfs_transaction_commit(trans); + wake_up(&fs_info->transaction_blocked_wait); + btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_COMMIT_PREP); + + if (!list_is_first(&cur_trans->list, &fs_info->trans_list)) { + enum btrfs_trans_state want_state = TRANS_STATE_COMPLETED; + + if (trans->in_fsync) + want_state = TRANS_STATE_SUPER_COMMITTED; + + prev_trans = list_prev_entry(cur_trans, list); + if (prev_trans->state < want_state) { + refcount_inc(&prev_trans->use_count); + spin_unlock(&fs_info->trans_lock); + + wait_for_commit(prev_trans, want_state); + + ret = READ_ONCE(prev_trans->aborted); + + btrfs_put_transaction(prev_trans); + if (unlikely(ret)) + goto lockdep_release; + spin_lock(&fs_info->trans_lock); + } + } else { + /* + * The previous transaction was aborted and was already removed + * from the list of transactions at fs_info->trans_list. So we + * abort to prevent writing a new superblock that reflects a + * corrupt state (pointing to trees with unwritten nodes/leafs). + */ + if (unlikely(BTRFS_FS_ERROR(fs_info))) { + spin_unlock(&fs_info->trans_lock); + ret = -EROFS; + goto lockdep_release; + } + } + + cur_trans->state = TRANS_STATE_COMMIT_START; + trace_btrfs_transaction_commit(trans); + wake_up(&fs_info->transaction_blocked_wait); + spin_unlock(&fs_info->trans_lock); + + /* + * Get the time spent on the work done by the commit thread and not + * the time spent waiting on a previous commit + */ + fs_info->commit_stats.critical_section_start_time = ktime_get_ns(); + extwriter_counter_dec(cur_trans, trans->type); + + ret = btrfs_start_delalloc_flush(fs_info); + if (unlikely(ret)) + goto lockdep_release; + + ret = btrfs_run_delayed_items(trans); + if (unlikely(ret)) + goto lockdep_release; + + /* + * The thread has started/joined the transaction thus it holds the + * lockdep map as a reader. It has to release it before acquiring the + * lockdep map as a writer. + */ + btrfs_lockdep_release(fs_info, btrfs_trans_num_extwriters); + btrfs_might_wait_for_event(fs_info, btrfs_trans_num_extwriters); + wait_event(cur_trans->writer_wait, + extwriter_counter_read(cur_trans) == 0); + + /* some pending stuffs might be added after the previous flush. */ + ret = btrfs_run_delayed_items(trans); + if (unlikely(ret)) { + btrfs_lockdep_release(fs_info, btrfs_trans_num_writers); + goto cleanup_transaction; + } + + btrfs_wait_delalloc_flush(fs_info); + + /* + * Wait for all ordered extents started by a fast fsync that joined this + * transaction. Otherwise if this transaction commits before the ordered + * extents complete we lose logged data after a power failure. + */ + btrfs_might_wait_for_event(fs_info, btrfs_trans_pending_ordered); + wait_event(cur_trans->pending_wait, + atomic_read(&cur_trans->pending_ordered) == 0); + + btrfs_scrub_pause(fs_info); + /* + * Ok now we need to make sure to block out any other joins while we + * commit the transaction. We could have started a join before setting + * COMMIT_DOING so make sure to wait for num_writers to == 1 again. + */ + spin_lock(&fs_info->trans_lock); + add_pending_snapshot(trans); + cur_trans->state = TRANS_STATE_COMMIT_DOING; + trace_btrfs_transaction_commit(trans); + spin_unlock(&fs_info->trans_lock); + + /* + * The thread has started/joined the transaction thus it holds the + * lockdep map as a reader. It has to release it before acquiring the + * lockdep map as a writer. + */ + btrfs_lockdep_release(fs_info, btrfs_trans_num_writers); + btrfs_might_wait_for_event(fs_info, btrfs_trans_num_writers); + wait_event(cur_trans->writer_wait, + atomic_read(&cur_trans->num_writers) == 1); + + /* + * Make lockdep happy by acquiring the state locks after + * btrfs_trans_num_writers is released. If we acquired the state locks + * before releasing the btrfs_trans_num_writers lock then lockdep would + * complain because we did not follow the reverse order unlocking rule. + */ + btrfs_trans_state_lockdep_acquire(fs_info, BTRFS_LOCKDEP_TRANS_COMPLETED); + btrfs_trans_state_lockdep_acquire(fs_info, BTRFS_LOCKDEP_TRANS_SUPER_COMMITTED); + btrfs_trans_state_lockdep_acquire(fs_info, BTRFS_LOCKDEP_TRANS_UNBLOCKED); + + /* + * We've started the commit, clear the flag in case we were triggered to + * do an async commit but somebody else started before the transaction + * kthread could do the work. + */ + clear_bit(BTRFS_FS_COMMIT_TRANS, &fs_info->flags); + + if (TRANS_ABORTED(cur_trans)) { + ret = cur_trans->aborted; + btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_UNBLOCKED); + goto scrub_continue; + } + /* + * the reloc mutex makes sure that we stop + * the balancing code from coming in and moving + * extents around in the middle of the commit + */ + mutex_lock(&fs_info->reloc_mutex); + + /* + * We needn't worry about the delayed items because we will + * deal with them in create_pending_snapshot(), which is the + * core function of the snapshot creation. + */ + ret = create_pending_snapshots(trans); + if (unlikely(ret)) + goto unlock_reloc; + + /* + * We insert the dir indexes of the snapshots and update the inode + * of the snapshots' parents after the snapshot creation, so there + * are some delayed items which are not dealt with. Now deal with + * them. + * + * We needn't worry that this operation will corrupt the snapshots, + * because all the tree which are snapshotted will be forced to COW + * the nodes and leaves. + */ + ret = btrfs_run_delayed_items(trans); + if (unlikely(ret)) + goto unlock_reloc; + + ret = btrfs_run_delayed_refs(trans, U64_MAX); + if (unlikely(ret)) + goto unlock_reloc; + + /* + * make sure none of the code above managed to slip in a + * delayed item + */ + btrfs_assert_delayed_root_empty(fs_info); + + WARN_ON(cur_trans != trans->transaction); + + ret = commit_fs_roots(trans); + if (unlikely(ret)) + goto unlock_reloc; + + /* commit_fs_roots gets rid of all the tree log roots, it is now + * safe to free the root of tree log roots + */ + btrfs_free_log_root_tree(trans, fs_info); + + /* + * Since fs roots are all committed, we can get a quite accurate + * new_roots. So let's do quota accounting. + */ + ret = btrfs_qgroup_account_extents(trans); + if (unlikely(ret < 0)) + goto unlock_reloc; + + ret = commit_cowonly_roots(trans); + if (unlikely(ret)) + goto unlock_reloc; + + /* + * The tasks which save the space cache and inode cache may also + * update ->aborted, check it. + */ + if (TRANS_ABORTED(cur_trans)) { + ret = cur_trans->aborted; + goto unlock_reloc; + } + + cur_trans = fs_info->running_transaction; + + btrfs_set_root_node(&fs_info->tree_root->root_item, + fs_info->tree_root->node); + list_add_tail(&fs_info->tree_root->dirty_list, + &cur_trans->switch_commits); + + btrfs_set_root_node(&fs_info->chunk_root->root_item, + fs_info->chunk_root->node); + list_add_tail(&fs_info->chunk_root->dirty_list, + &cur_trans->switch_commits); + + switch_commit_roots(trans); + + ASSERT(list_empty(&cur_trans->dirty_bgs)); + ASSERT(list_empty(&cur_trans->io_bgs)); + update_super_roots(fs_info); + + btrfs_set_super_log_root(fs_info->super_copy, 0); + btrfs_set_super_log_root_level(fs_info->super_copy, 0); + memcpy(fs_info->super_for_commit, fs_info->super_copy, + sizeof(*fs_info->super_copy)); + + btrfs_commit_device_sizes(cur_trans); + + clear_bit(BTRFS_FS_LOG1_ERR, &fs_info->flags); + clear_bit(BTRFS_FS_LOG2_ERR, &fs_info->flags); + + btrfs_trans_release_chunk_metadata(trans); + + /* + * Before changing the transaction state to TRANS_STATE_UNBLOCKED and + * setting fs_info->running_transaction to NULL, lock tree_log_mutex to + * make sure that before we commit our superblock, no other task can + * start a new transaction and commit a log tree before we commit our + * superblock. Anyone trying to commit a log tree locks this mutex before + * writing its superblock. + */ + mutex_lock(&fs_info->tree_log_mutex); + + spin_lock(&fs_info->trans_lock); + cur_trans->state = TRANS_STATE_UNBLOCKED; + trace_btrfs_transaction_commit(trans); + fs_info->running_transaction = NULL; + spin_unlock(&fs_info->trans_lock); + mutex_unlock(&fs_info->reloc_mutex); + + wake_up(&fs_info->transaction_wait); + btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_UNBLOCKED); + + /* If we have features changed, wake up the cleaner to update sysfs. */ + if (test_bit(BTRFS_FS_FEATURE_CHANGED, &fs_info->flags) && + fs_info->cleaner_kthread) + wake_up_process(fs_info->cleaner_kthread); + + /* + * Uninhibit writeback on all extent buffers inhibited during this + * transaction before writing them to disk. Inhibiting prevented + * writeback while the transaction was building, but now we need + * them written. + */ + btrfs_uninhibit_all_eb_writeback(trans); + + ret = btrfs_write_and_wait_transaction(trans); + if (unlikely(ret)) { + btrfs_err(fs_info, "error while writing out transaction: %pe", ERR_PTR(ret)); + mutex_unlock(&fs_info->tree_log_mutex); + goto scrub_continue; + } + + ret = write_all_supers(trans); + /* + * the super is written, we can safely allow the tree-loggers + * to go about their business + */ + mutex_unlock(&fs_info->tree_log_mutex); + if (unlikely(ret)) + goto scrub_continue; + + update_commit_stats(fs_info); + /* + * We needn't acquire the lock here because there is no other task + * which can change it. + */ + cur_trans->state = TRANS_STATE_SUPER_COMMITTED; + trace_btrfs_transaction_commit(trans); + wake_up(&cur_trans->commit_wait); + btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_SUPER_COMMITTED); + + ret = btrfs_finish_extent_commit(trans); + if (unlikely(ret)) + goto scrub_continue; + + if (test_bit(BTRFS_TRANS_HAVE_FREE_BGS, &cur_trans->flags)) + btrfs_clear_space_info_full(fs_info); + + btrfs_set_last_trans_committed(fs_info, cur_trans->transid); + /* + * We needn't acquire the lock here because there is no other task + * which can change it. + */ + cur_trans->state = TRANS_STATE_COMPLETED; + trace_btrfs_transaction_commit(trans); + wake_up(&cur_trans->commit_wait); + btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_COMPLETED); + + spin_lock(&fs_info->trans_lock); + list_del_init(&cur_trans->list); + spin_unlock(&fs_info->trans_lock); + + btrfs_put_transaction(cur_trans); + btrfs_put_transaction(cur_trans); + + if (trans->type & __TRANS_FREEZABLE) + sb_end_intwrite(fs_info->sb); + + btrfs_scrub_continue(fs_info); + + if (current->journal_info == trans) + current->journal_info = NULL; + + kmem_cache_free(btrfs_trans_handle_cachep, trans); + + return ret; + +unlock_reloc: + mutex_unlock(&fs_info->reloc_mutex); + btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_UNBLOCKED); +scrub_continue: + btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_SUPER_COMMITTED); + btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_COMPLETED); + btrfs_scrub_continue(fs_info); +cleanup_transaction: + btrfs_trans_release_metadata(trans); + btrfs_cleanup_pending_block_groups(trans); + btrfs_trans_release_chunk_metadata(trans); + trans->block_rsv = NULL; + btrfs_warn(fs_info, "Skipping commit of aborted transaction."); + cleanup_transaction(trans, ret); + + return ret; + +lockdep_release: + btrfs_lockdep_release(fs_info, btrfs_trans_num_extwriters); + btrfs_lockdep_release(fs_info, btrfs_trans_num_writers); + goto cleanup_transaction; + +lockdep_trans_commit_start_release: + btrfs_trans_state_lockdep_release(fs_info, BTRFS_LOCKDEP_TRANS_COMMIT_PREP); + btrfs_end_transaction(trans); + return ret; +} + +/* + * return < 0 if error + * 0 if there are no more dead_roots at the time of call + * 1 there are more to be processed, call me again + * + * The return value indicates there are certainly more snapshots to delete, but + * if there comes a new one during processing, it may return 0. We don't mind, + * because btrfs_commit_super will poke cleaner thread and it will process it a + * few seconds later. + */ +int btrfs_clean_one_deleted_snapshot(struct btrfs_fs_info *fs_info) +{ + struct btrfs_root *root; + int ret; + + spin_lock(&fs_info->trans_lock); + if (list_empty(&fs_info->dead_roots)) { + spin_unlock(&fs_info->trans_lock); + return 0; + } + root = list_first_entry(&fs_info->dead_roots, + struct btrfs_root, root_list); + list_del_init(&root->root_list); + spin_unlock(&fs_info->trans_lock); + + btrfs_debug(fs_info, "cleaner removing %llu", btrfs_root_id(root)); + + btrfs_kill_all_delayed_nodes(root); + + if (btrfs_header_backref_rev(root->node) < + BTRFS_MIXED_BACKREF_REV) + ret = btrfs_drop_snapshot(root, false, false); + else + ret = btrfs_drop_snapshot(root, true, false); + + btrfs_put_root(root); + return (ret < 0) ? 0 : 1; +} + +/* + * We only mark the transaction aborted and then set the file system read-only. + * This will prevent new transactions from starting or trying to join this + * one. + * + * This means that error recovery at the call site is limited to freeing + * any local memory allocations and passing the error code up without + * further cleanup. The transaction should complete as it normally would + * in the call path but will return -EIO. + * + * We'll complete the cleanup in btrfs_end_transaction and + * btrfs_commit_transaction. + * + * Note: the parameter @error encodes whether the transactin abort was first hit + * (setting the FS_ERROR state bit in btrfs_abort_transaction()) + * - positive number - first hit + * - negative number - abort after it was already done + */ +void __cold __btrfs_abort_transaction(struct btrfs_trans_handle *trans, + const char *function, + unsigned int line, int error) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + bool first_hit = false; + + if (error > 0) { + error = -error; + first_hit = true; + } + + WRITE_ONCE(trans->aborted, error); + WRITE_ONCE(trans->transaction->aborted, error); + trace_btrfs_transaction_abort(trans); + if (first_hit) { + btrfs_err(fs_info, "Transaction %llu aborted (%pe)", + trans->transid, ERR_PTR(error)); + if (error == -ENOSPC) + btrfs_dump_space_info_for_trans_abort(fs_info); + } + /* Wake up anybody who may be waiting on this transaction */ + wake_up(&fs_info->transaction_wait); + wake_up(&fs_info->transaction_blocked_wait); + __btrfs_handle_fs_error(fs_info, function, line, error, NULL); +} + +int __init btrfs_transaction_init(void) +{ + btrfs_trans_handle_cachep = KMEM_CACHE(btrfs_trans_handle, SLAB_TEMPORARY); + if (!btrfs_trans_handle_cachep) + return -ENOMEM; + return 0; +} + +void __cold btrfs_transaction_exit(void) +{ + kmem_cache_destroy(btrfs_trans_handle_cachep); +} diff --git a/fs/btrfs/transaction.h b/fs/btrfs/transaction.h new file mode 100644 index 000000000..3a57f227b --- /dev/null +++ b/fs/btrfs/transaction.h @@ -0,0 +1,348 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#ifndef BTRFS_TRANSACTION_H +#define BTRFS_TRANSACTION_H + +#include +#include +#include +#include +#include +#include +#include +#include "btrfs_inode.h" +#include "delayed-ref.h" + +struct dentry; +struct inode; +struct btrfs_pending_snapshot; +struct btrfs_fs_info; +struct btrfs_root_item; +struct btrfs_root; +struct btrfs_path; +struct extent_buffer; + +/* + * Signal that a direct IO write is in progress, to avoid deadlock for sync + * direct IO writes when fsync is called during the direct IO write path. + */ +#define BTRFS_TRANS_DIO_WRITE_STUB ((void *) 1) + +/* Radix-tree tag for roots that are part of the transaction. */ +#define BTRFS_ROOT_TRANS_TAG 0 + +enum btrfs_trans_state { + TRANS_STATE_RUNNING, + TRANS_STATE_COMMIT_PREP, + TRANS_STATE_COMMIT_START, + TRANS_STATE_COMMIT_DOING, + TRANS_STATE_UNBLOCKED, + TRANS_STATE_SUPER_COMMITTED, + TRANS_STATE_COMPLETED, + TRANS_STATE_MAX, +}; + +#define BTRFS_TRANS_HAVE_FREE_BGS 0 +#define BTRFS_TRANS_DIRTY_BG_RUN 1 +#define BTRFS_TRANS_CACHE_ENOSPC 2 + +struct btrfs_transaction { + u64 transid; + /* + * total external writers(USERSPACE/START/ATTACH) in this + * transaction, it must be zero before the transaction is + * being committed + */ + atomic_t num_extwriters; + /* + * total writers in this transaction, it must be zero before the + * transaction can end + */ + atomic_t num_writers; + refcount_t use_count; + + unsigned long flags; + + /* Be protected by fs_info->trans_lock when we want to change it. */ + enum btrfs_trans_state state; + int aborted; + struct list_head list; + struct extent_io_tree dirty_pages; + time64_t start_time; + wait_queue_head_t writer_wait; + wait_queue_head_t commit_wait; + struct list_head pending_snapshots; + struct list_head dev_update_list; + struct list_head switch_commits; + struct list_head dirty_bgs; + + /* + * There is no explicit lock which protects io_bgs, rather its + * consistency is implied by the fact that all the sites which modify + * it do so under some form of transaction critical section, namely: + * + * - btrfs_start_dirty_block_groups - This function can only ever be + * run by one of the transaction committers. Refer to + * BTRFS_TRANS_DIRTY_BG_RUN usage in btrfs_commit_transaction + * + * - btrfs_write_dirty_blockgroups - this is called by + * commit_cowonly_roots from transaction critical section + * (TRANS_STATE_COMMIT_DOING) + * + * - btrfs_cleanup_dirty_bgs - called on transaction abort + */ + struct list_head io_bgs; + struct list_head dropped_roots; + struct extent_io_tree pinned_extents; + + /* + * we need to make sure block group deletion doesn't race with + * free space cache writeout. This mutex keeps them from stomping + * on each other + */ + struct mutex cache_write_mutex; + spinlock_t dirty_bgs_lock; + /* Protected by spin lock fs_info->unused_bgs_lock. */ + struct list_head deleted_bgs; + spinlock_t dropped_roots_lock; + struct btrfs_delayed_ref_root delayed_refs; + struct btrfs_fs_info *fs_info; + + /* + * Number of ordered extents the transaction must wait for before + * committing. These are ordered extents started by a fast fsync. + */ + atomic_t pending_ordered; + wait_queue_head_t pending_wait; +}; + +enum { + ENUM_BIT(__TRANS_FREEZABLE), + ENUM_BIT(__TRANS_START), + ENUM_BIT(__TRANS_ATTACH), + ENUM_BIT(__TRANS_JOIN), + ENUM_BIT(__TRANS_JOIN_NOLOCK), + ENUM_BIT(__TRANS_DUMMY), + ENUM_BIT(__TRANS_JOIN_NOSTART), +}; + +#define TRANS_START (__TRANS_START | __TRANS_FREEZABLE) +#define TRANS_ATTACH (__TRANS_ATTACH) +#define TRANS_JOIN (__TRANS_JOIN | __TRANS_FREEZABLE) +#define TRANS_JOIN_NOLOCK (__TRANS_JOIN_NOLOCK) +#define TRANS_JOIN_NOSTART (__TRANS_JOIN_NOSTART) + +#define TRANS_EXTWRITERS (__TRANS_START | __TRANS_ATTACH) + +/* + * Number of extent buffers a transaction handle tracks for writeback + * inhibition. The CLOCK reference bits pack into a u32 so this must not exceed + * 32, and keeping it a power of two lets the compiler reduce the CLOCK hand + * modulo to a mask. + */ +#define BTRFS_INHIBITED_EBS_SLOTS 8 + +static_assert(BTRFS_INHIBITED_EBS_SLOTS <= 32); +static_assert(BTRFS_INHIBITED_EBS_SLOTS != 0 && + (BTRFS_INHIBITED_EBS_SLOTS & (BTRFS_INHIBITED_EBS_SLOTS - 1)) == 0); + +struct btrfs_trans_handle { + u64 transid; + u64 bytes_reserved; + u64 delayed_refs_bytes_reserved; + u64 chunk_bytes_reserved; + unsigned long delayed_ref_updates; + unsigned long delayed_ref_csum_deletions; + struct btrfs_transaction *transaction; + struct btrfs_block_rsv *block_rsv; + struct btrfs_block_rsv *orig_rsv; + /* Set by a task that wants to create a snapshot. */ + struct btrfs_pending_snapshot *pending_snapshot; + refcount_t use_count; + unsigned int type; + /* + * Error code of transaction abort, set outside of locks and must use + * the READ_ONCE/WRITE_ONCE access + */ + short aborted; + bool adding_csums; + bool allocating_chunk; + bool removing_chunk; + bool reloc_reserved; + bool in_fsync; + struct btrfs_fs_info *fs_info; + struct list_head new_bgs; + struct btrfs_block_rsv delayed_rsv; + + /* Extent buffers this handle has inhibited writeback on. */ + struct extent_buffer *inhibited_ebs[BTRFS_INHIBITED_EBS_SLOTS]; + /* CLOCK reference bit per slot. */ + u32 inhibited_ebs_referenced; + u32 nr_inhibited_ebs; + /* CLOCK hand. */ + u32 inhibited_ebs_hand; +}; + +/* + * The abort status can be changed between calls and is not protected by locks. + * This accepts btrfs_transaction and btrfs_trans_handle as types. Once it's + * set to a non-zero value it does not change, so the macro should be in checks + * but is not necessary for further reads of the value. + */ +#define TRANS_ABORTED(trans) (unlikely(READ_ONCE((trans)->aborted))) + +struct btrfs_pending_snapshot { + struct dentry *dentry; + struct btrfs_inode *dir; + struct btrfs_root *root; + struct btrfs_root_item *root_item; + struct btrfs_root *snap; + struct btrfs_qgroup_inherit *inherit; + struct btrfs_path *path; + /* block reservation for the operation */ + struct btrfs_block_rsv block_rsv; + /* extra metadata reservation for relocation */ + int error; + /* Preallocated anonymous block device number */ + dev_t anon_dev; + bool readonly; + struct list_head list; +}; + +static inline void btrfs_set_inode_last_trans(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode) +{ + spin_lock(&inode->lock); + inode->last_trans = trans->transaction->transid; + inode->last_sub_trans = btrfs_get_root_log_transid(inode->root); + inode->last_log_commit = inode->last_sub_trans - 1; + spin_unlock(&inode->lock); +} + +/* + * Make qgroup codes to skip given qgroupid, means the old/new_roots for + * qgroup won't contain the qgroupid in it. + */ +static inline void btrfs_set_skip_qgroup(struct btrfs_trans_handle *trans, + u64 qgroupid) +{ + struct btrfs_delayed_ref_root *delayed_refs; + + delayed_refs = &trans->transaction->delayed_refs; + WARN_ON(delayed_refs->qgroup_to_skip); + delayed_refs->qgroup_to_skip = qgroupid; +} + +static inline void btrfs_clear_skip_qgroup(struct btrfs_trans_handle *trans) +{ + struct btrfs_delayed_ref_root *delayed_refs; + + delayed_refs = &trans->transaction->delayed_refs; + WARN_ON(!delayed_refs->qgroup_to_skip); + delayed_refs->qgroup_to_skip = 0; +} + +/* + * We want the transaction abort to print stack trace only for errors where the + * cause could be a bug, eg. due to ENOSPC, and not for common errors that are + * caused by external factors. + */ +static inline bool btrfs_abort_should_print_stack(int error) +{ + switch (error) { + case -EIO: + case -EROFS: + case -ENOMEM: + return false; + } + return true; +} + +/* + * Compile-time and run-time verification of error passed to transaction abort. + * Direct constants will be caught at compile time, errors read from variables + * can be caught only at run-time and will warn under debugging config. + * + * How verification works: + * - accepted builtin constants are all -EIO and such + * - for compile-time check, invalid condition produces a negative-sized array + * type, valid zero-sized + * - when a variable is passed as error the first check is a no-op + * - with enabled debugging, the second array type size is constructed from the + * real variable value, valid condition produces array of size 1 + * - sizeof(type) does not generate any code + */ +#define VERIFY_NEGATIVE_ERROR(error) \ +do { \ + (void)sizeof(char[-!(__builtin_constant_p(error) ? (error) < 0 : 1)]); \ + if (IS_ENABLED(CONFIG_BTRFS_DEBUG)) { \ + if (sizeof(char[(error) < 0]) != 1) \ + DEBUG_WARN("error >= 0 passed to btrfs_abort_transaction()"); \ + } \ +} while(0) + +/* + * Call btrfs_abort_transaction() as early as possible when an error condition + * is detected, that way the exact stack trace is reported for some errors. + * + * Error number must be negative as it encodes wheather it's the first abort. + */ +#define btrfs_abort_transaction(trans, error) \ +do { \ + int __error = (error); \ + \ + VERIFY_NEGATIVE_ERROR(error); \ + /* Report first abort since mount */ \ + if (!test_and_set_bit(BTRFS_FS_STATE_TRANS_ABORTED, \ + &((trans)->fs_info->fs_state))) { \ + WARN_ON(btrfs_abort_should_print_stack(__error)); \ + __error = -__error; \ + } \ + __btrfs_abort_transaction((trans), __func__, \ + __LINE__, __error); \ +} while (0) + +int btrfs_end_transaction(struct btrfs_trans_handle *trans); +struct btrfs_trans_handle *btrfs_start_transaction(struct btrfs_root *root, + unsigned int num_items); +struct btrfs_trans_handle *btrfs_start_transaction_fallback_global_rsv( + struct btrfs_root *root, + unsigned int num_items); +struct btrfs_trans_handle *btrfs_join_transaction(struct btrfs_root *root); +struct btrfs_trans_handle *btrfs_join_transaction_spacecache(struct btrfs_root *root); +struct btrfs_trans_handle *btrfs_join_transaction_nostart(struct btrfs_root *root); +struct btrfs_trans_handle *btrfs_attach_transaction(struct btrfs_root *root); +struct btrfs_trans_handle *btrfs_attach_transaction_barrier( + struct btrfs_root *root); +int btrfs_wait_for_commit(struct btrfs_fs_info *fs_info, u64 transid); + +void btrfs_add_dead_root(struct btrfs_root *root); +void btrfs_maybe_wake_unfinished_drop(struct btrfs_fs_info *fs_info); +int btrfs_clean_one_deleted_snapshot(struct btrfs_fs_info *fs_info); +int btrfs_commit_transaction(struct btrfs_trans_handle *trans); +void btrfs_commit_transaction_async(struct btrfs_trans_handle *trans); +int btrfs_commit_current_transaction(struct btrfs_root *root); +int btrfs_end_transaction_throttle(struct btrfs_trans_handle *trans); +bool btrfs_should_end_transaction(struct btrfs_trans_handle *trans); +void btrfs_throttle(struct btrfs_fs_info *fs_info); +int btrfs_record_root_in_trans(struct btrfs_trans_handle *trans, + struct btrfs_root *root); +int btrfs_write_marked_extents(struct btrfs_fs_info *fs_info, + struct extent_io_tree *dirty_pages, int mark); +int btrfs_wait_tree_log_extents(struct btrfs_root *root, int mark); +int btrfs_transaction_blocked(struct btrfs_fs_info *info); +void btrfs_put_transaction(struct btrfs_transaction *transaction); +void btrfs_add_dropped_root(struct btrfs_trans_handle *trans, + struct btrfs_root *root); +void btrfs_trans_release_chunk_metadata(struct btrfs_trans_handle *trans); +void __cold __btrfs_abort_transaction(struct btrfs_trans_handle *trans, + const char *function, + unsigned int line, int error); + +int __init btrfs_transaction_init(void); +void __cold btrfs_transaction_exit(void); + +#endif diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c new file mode 100644 index 000000000..0ce91396b --- /dev/null +++ b/fs/btrfs/tree-checker.c @@ -0,0 +1,2742 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) Qu Wenruo 2017. All rights reserved. + */ + +/* + * The module is used to catch unexpected/corrupted tree block data. + * Such behavior can be caused either by a fuzzed image or bugs. + * + * The objective is to do leaf/node validation checks when tree block is read + * from disk, and check *every* possible member, so other code won't + * need to checking them again. + * + * Due to the potential and unwanted damage, every checker needs to be + * carefully reviewed otherwise so it does not prevent mount of valid images. + */ + +#include +#include +#include +#include "messages.h" +#include "ctree.h" +#include "tree-checker.h" +#include "compression.h" +#include "volumes.h" +#include "misc.h" +#include "fs.h" +#include "accessors.h" +#include "file-item.h" +#include "inode-item.h" +#include "dir-item.h" +#include "extent-tree.h" + +/* + * Error message should follow the following format: + * corrupt : , [, ] + * + * @type: leaf or node + * @identifier: the necessary info to locate the leaf/node. + * It's recommended to decode key.objecitd/offset if it's + * meaningful. + * @reason: describe the error + * @bad_value: optional, it's recommended to output bad value and its + * expected value (range). + * + * Since comma is used to separate the components, only space is allowed + * inside each component. + */ + +/* + * Append generic "corrupt leaf/node root=%llu block=%llu slot=%d: " to @fmt. + * Allows callers to customize the output. + */ +__printf(3, 4) +__cold +static void generic_err(const struct extent_buffer *eb, int slot, + const char *fmt, ...) +{ + const struct btrfs_fs_info *fs_info = eb->fs_info; + struct va_format vaf; + va_list args; + + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + dump_page(folio_page(eb->folios[0], 0), "eb page dump"); + btrfs_crit(fs_info, + "corrupt %s: root=%llu block=%llu slot=%d, %pV", + btrfs_header_level(eb) == 0 ? "leaf" : "node", + btrfs_header_owner(eb), btrfs_header_bytenr(eb), slot, &vaf); + va_end(args); +} + +/* + * Customized reporter for extent data item, since its key objectid and + * offset has its own meaning. + */ +__printf(3, 4) +__cold +static void file_extent_err(const struct extent_buffer *eb, int slot, + const char *fmt, ...) +{ + const struct btrfs_fs_info *fs_info = eb->fs_info; + struct btrfs_key key; + struct va_format vaf; + va_list args; + + btrfs_item_key_to_cpu(eb, &key, slot); + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + dump_page(folio_page(eb->folios[0], 0), "eb page dump"); + btrfs_crit(fs_info, + "corrupt %s: root=%llu block=%llu slot=%d ino=%llu file_offset=%llu, %pV", + btrfs_header_level(eb) == 0 ? "leaf" : "node", + btrfs_header_owner(eb), btrfs_header_bytenr(eb), slot, + key.objectid, key.offset, &vaf); + va_end(args); +} + +/* + * Return 0 if the btrfs_file_extent_##name is aligned to @alignment + * Else return 1 + */ +#define CHECK_FE_ALIGNED(leaf, slot, fi, name, alignment) \ +({ \ + if (unlikely(!IS_ALIGNED(btrfs_file_extent_##name((leaf), (fi)), \ + (alignment)))) \ + file_extent_err((leaf), (slot), \ + "invalid %s for file extent, have %llu, should be aligned to %u", \ + (#name), btrfs_file_extent_##name((leaf), (fi)), \ + (alignment)); \ + (!IS_ALIGNED(btrfs_file_extent_##name((leaf), (fi)), (alignment))); \ +}) + +static u64 file_extent_end(struct extent_buffer *leaf, + struct btrfs_key *key, + struct btrfs_file_extent_item *extent) +{ + u64 end; + u64 len; + + if (btrfs_file_extent_type(leaf, extent) == BTRFS_FILE_EXTENT_INLINE) { + len = btrfs_file_extent_ram_bytes(leaf, extent); + end = ALIGN(key->offset + len, leaf->fs_info->sectorsize); + } else { + len = btrfs_file_extent_num_bytes(leaf, extent); + end = key->offset + len; + } + return end; +} + +/* + * Customized report for dir_item, the only new important information is + * key->objectid, which represents inode number + */ +__printf(3, 4) +__cold +static void dir_item_err(const struct extent_buffer *eb, int slot, + const char *fmt, ...) +{ + const struct btrfs_fs_info *fs_info = eb->fs_info; + struct btrfs_key key; + struct va_format vaf; + va_list args; + + btrfs_item_key_to_cpu(eb, &key, slot); + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + dump_page(folio_page(eb->folios[0], 0), "eb page dump"); + btrfs_crit(fs_info, + "corrupt %s: root=%llu block=%llu slot=%d ino=%llu, %pV", + btrfs_header_level(eb) == 0 ? "leaf" : "node", + btrfs_header_owner(eb), btrfs_header_bytenr(eb), slot, + key.objectid, &vaf); + va_end(args); +} + +/* + * This functions checks prev_key->objectid, to ensure current key and prev_key + * share the same objectid as inode number. + * + * This is to detect missing INODE_ITEM in subvolume trees. + * + * Return true if everything is OK or we don't need to check. + * Return false if anything is wrong. + */ +static bool check_prev_ino(struct extent_buffer *leaf, + struct btrfs_key *key, int slot, + struct btrfs_key *prev_key) +{ + /* No prev key, skip check */ + if (slot == 0) + return true; + + /* Only these key->types needs to be checked */ + ASSERT(key->type == BTRFS_XATTR_ITEM_KEY || + key->type == BTRFS_INODE_REF_KEY || + key->type == BTRFS_INODE_EXTREF_KEY || + key->type == BTRFS_DIR_INDEX_KEY || + key->type == BTRFS_DIR_ITEM_KEY || + key->type == BTRFS_EXTENT_DATA_KEY, "key->type=%u", key->type); + + /* + * Only subvolume trees along with their reloc trees need this check. + * Things like log tree doesn't follow this ino requirement. + */ + if (!btrfs_is_fstree(btrfs_header_owner(leaf))) + return true; + + if (key->objectid == prev_key->objectid) + return true; + + /* Error found */ + dir_item_err(leaf, slot, + "invalid previous key objectid, have %llu expect %llu", + prev_key->objectid, key->objectid); + return false; +} +static int check_extent_data_item(struct extent_buffer *leaf, + struct btrfs_key *key, int slot, + struct btrfs_key *prev_key) +{ + struct btrfs_fs_info *fs_info = leaf->fs_info; + struct btrfs_file_extent_item *fi; + u32 sectorsize = fs_info->sectorsize; + u32 item_size = btrfs_item_size(leaf, slot); + u64 extent_end; + + if (unlikely(!IS_ALIGNED(key->offset, sectorsize))) { + file_extent_err(leaf, slot, +"unaligned file_offset for file extent, have %llu should be aligned to %u", + key->offset, sectorsize); + return -EUCLEAN; + } + + /* + * Previous key must have the same key->objectid (ino). + * It can be XATTR_ITEM, INODE_ITEM or just another EXTENT_DATA. + * But if objectids mismatch, it means we have a missing + * INODE_ITEM. + */ + if (unlikely(!check_prev_ino(leaf, key, slot, prev_key))) + return -EUCLEAN; + + fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item); + + /* + * Make sure the item contains at least inline header, so the file + * extent type is not some garbage. + */ + if (unlikely(item_size < BTRFS_FILE_EXTENT_INLINE_DATA_START)) { + file_extent_err(leaf, slot, + "invalid item size, have %u expect [%zu, %u)", + item_size, BTRFS_FILE_EXTENT_INLINE_DATA_START, + SZ_4K); + return -EUCLEAN; + } + if (unlikely(btrfs_file_extent_type(leaf, fi) >= + BTRFS_NR_FILE_EXTENT_TYPES)) { + file_extent_err(leaf, slot, + "invalid type for file extent, have %u expect range [0, %u]", + btrfs_file_extent_type(leaf, fi), + BTRFS_NR_FILE_EXTENT_TYPES - 1); + return -EUCLEAN; + } + + /* + * Support for new compression/encryption must introduce incompat flag, + * and must be caught in open_ctree(). + */ + if (unlikely(btrfs_file_extent_compression(leaf, fi) >= + BTRFS_NR_COMPRESS_TYPES)) { + file_extent_err(leaf, slot, + "invalid compression for file extent, have %u expect range [0, %u]", + btrfs_file_extent_compression(leaf, fi), + BTRFS_NR_COMPRESS_TYPES - 1); + return -EUCLEAN; + } + if (unlikely(btrfs_file_extent_encryption(leaf, fi))) { + file_extent_err(leaf, slot, + "invalid encryption for file extent, have %u expect 0", + btrfs_file_extent_encryption(leaf, fi)); + return -EUCLEAN; + } + if (btrfs_file_extent_type(leaf, fi) == BTRFS_FILE_EXTENT_INLINE) { + /* Inline extent must have 0 as key offset */ + if (unlikely(key->offset)) { + file_extent_err(leaf, slot, + "invalid file_offset for inline file extent, have %llu expect 0", + key->offset); + return -EUCLEAN; + } + + /* Compressed inline extent has no on-disk size, skip it */ + if (btrfs_file_extent_compression(leaf, fi) != + BTRFS_COMPRESS_NONE) + return 0; + + /* Uncompressed inline extent size must match item size */ + if (unlikely(item_size != BTRFS_FILE_EXTENT_INLINE_DATA_START + + btrfs_file_extent_ram_bytes(leaf, fi))) { + file_extent_err(leaf, slot, + "invalid ram_bytes for uncompressed inline extent, have %u expect %llu", + item_size, BTRFS_FILE_EXTENT_INLINE_DATA_START + + btrfs_file_extent_ram_bytes(leaf, fi)); + return -EUCLEAN; + } + return 0; + } + + /* + * For the data reloc tree, file extent items are written by + * relocation's own paths. The data reloc inode is created with + * BTRFS_INODE_NOCOMPRESS, so insert_ordered_extent_file_extent() + * always leaves the compression field at 0. Encryption and + * other_encoding are reserved-and-zero in btrfs. A non-zero value + * for any of these means the leaf decoded from disk does not match + * what the kernel wrote, i.e. on-disk corruption. + * + * The file_extent_item's offset field is NOT a universal invariant + * here: partial-PREALLOC writebacks legitimately produce REG items + * with non-zero offset at non-boundary keys. The offset check is + * performed at the call site in get_new_location(), which only + * inspects cluster-boundary keys where offset is always 0. + */ + if (unlikely(btrfs_header_owner(leaf) == BTRFS_DATA_RELOC_TREE_OBJECTID && + (btrfs_file_extent_compression(leaf, fi) || + btrfs_file_extent_encryption(leaf, fi) || + btrfs_file_extent_other_encoding(leaf, fi)))) { + file_extent_err(leaf, slot, +"invalid encoding fields for data reloc tree, compression=%u encryption=%u other_encoding=%u", + btrfs_file_extent_compression(leaf, fi), + btrfs_file_extent_encryption(leaf, fi), + btrfs_file_extent_other_encoding(leaf, fi)); + return -EUCLEAN; + } + + /* Regular or preallocated extent has fixed item size */ + if (unlikely(item_size != sizeof(*fi))) { + file_extent_err(leaf, slot, + "invalid item size for reg/prealloc file extent, have %u expect %zu", + item_size, sizeof(*fi)); + return -EUCLEAN; + } + if (unlikely(CHECK_FE_ALIGNED(leaf, slot, fi, ram_bytes, sectorsize) || + CHECK_FE_ALIGNED(leaf, slot, fi, disk_bytenr, sectorsize) || + CHECK_FE_ALIGNED(leaf, slot, fi, disk_num_bytes, sectorsize) || + CHECK_FE_ALIGNED(leaf, slot, fi, offset, sectorsize) || + CHECK_FE_ALIGNED(leaf, slot, fi, num_bytes, sectorsize))) + return -EUCLEAN; + + /* Catch extent end overflow */ + if (unlikely(check_add_overflow(btrfs_file_extent_num_bytes(leaf, fi), + key->offset, &extent_end))) { + file_extent_err(leaf, slot, + "extent end overflow, have file offset %llu extent num bytes %llu", + key->offset, + btrfs_file_extent_num_bytes(leaf, fi)); + return -EUCLEAN; + } + + /* + * Check that no two consecutive file extent items, in the same leaf, + * present ranges that overlap each other. + */ + if (slot > 0 && + prev_key->objectid == key->objectid && + prev_key->type == BTRFS_EXTENT_DATA_KEY) { + struct btrfs_file_extent_item *prev_fi; + u64 prev_end; + + prev_fi = btrfs_item_ptr(leaf, slot - 1, + struct btrfs_file_extent_item); + prev_end = file_extent_end(leaf, prev_key, prev_fi); + if (unlikely(prev_end > key->offset)) { + file_extent_err(leaf, slot - 1, +"file extent end range (%llu) goes beyond start offset (%llu) of the next file extent", + prev_end, key->offset); + return -EUCLEAN; + } + } + + /* + * For non-compressed data extents, ram_bytes should match its + * disk_num_bytes. + * However we do not really utilize ram_bytes in this case, so this check + * is only optional for DEBUG builds for developers to catch the + * unexpected behaviors. + */ + if (IS_ENABLED(CONFIG_BTRFS_DEBUG) && + btrfs_file_extent_compression(leaf, fi) == BTRFS_COMPRESS_NONE && + btrfs_file_extent_disk_bytenr(leaf, fi)) { + if (WARN_ON(btrfs_file_extent_ram_bytes(leaf, fi) != + btrfs_file_extent_disk_num_bytes(leaf, fi))) + file_extent_err(leaf, slot, +"mismatch ram_bytes (%llu) and disk_num_bytes (%llu) for non-compressed extent", + btrfs_file_extent_ram_bytes(leaf, fi), + btrfs_file_extent_disk_num_bytes(leaf, fi)); + } + + return 0; +} + +static int check_csum_item(struct extent_buffer *leaf, struct btrfs_key *key, + int slot, struct btrfs_key *prev_key) +{ + struct btrfs_fs_info *fs_info = leaf->fs_info; + u32 sectorsize = fs_info->sectorsize; + const u32 csumsize = fs_info->csum_size; + + if (unlikely(key->objectid != BTRFS_EXTENT_CSUM_OBJECTID)) { + generic_err(leaf, slot, + "invalid key objectid for csum item, have %llu expect %llu", + key->objectid, BTRFS_EXTENT_CSUM_OBJECTID); + return -EUCLEAN; + } + if (unlikely(!IS_ALIGNED(key->offset, sectorsize))) { + generic_err(leaf, slot, + "unaligned key offset for csum item, have %llu should be aligned to %u", + key->offset, sectorsize); + return -EUCLEAN; + } + if (unlikely(!IS_ALIGNED(btrfs_item_size(leaf, slot), csumsize))) { + generic_err(leaf, slot, + "unaligned item size for csum item, have %u should be aligned to %u", + btrfs_item_size(leaf, slot), csumsize); + return -EUCLEAN; + } + if (slot > 0 && prev_key->type == BTRFS_EXTENT_CSUM_KEY) { + u64 prev_csum_end; + u32 prev_item_size; + + prev_item_size = btrfs_item_size(leaf, slot - 1); + prev_csum_end = (prev_item_size / csumsize) * sectorsize; + prev_csum_end += prev_key->offset; + if (unlikely(prev_csum_end > key->offset)) { + generic_err(leaf, slot - 1, +"csum end range (%llu) goes beyond the start range (%llu) of the next csum item", + prev_csum_end, key->offset); + return -EUCLEAN; + } + } + return 0; +} + +/* Inode item error output has the same format as dir_item_err() */ +#define inode_item_err(eb, slot, fmt, ...) \ + dir_item_err(eb, slot, fmt, __VA_ARGS__) + +static int check_inode_key(struct extent_buffer *leaf, struct btrfs_key *key, + int slot) +{ + struct btrfs_key item_key; + bool is_inode_item; + + btrfs_item_key_to_cpu(leaf, &item_key, slot); + is_inode_item = (item_key.type == BTRFS_INODE_ITEM_KEY); + + /* For XATTR_ITEM, location key should be all 0 */ + if (item_key.type == BTRFS_XATTR_ITEM_KEY) { + if (unlikely(key->objectid != 0 || key->type != 0 || + key->offset != 0)) + return -EUCLEAN; + return 0; + } + + if (unlikely((key->objectid < BTRFS_FIRST_FREE_OBJECTID || + key->objectid > BTRFS_LAST_FREE_OBJECTID) && + key->objectid != BTRFS_ROOT_TREE_DIR_OBJECTID && + key->objectid != BTRFS_FREE_INO_OBJECTID)) { + if (is_inode_item) { + generic_err(leaf, slot, + "invalid key objectid: has %llu expect %llu or [%llu, %llu] or %llu", + key->objectid, BTRFS_ROOT_TREE_DIR_OBJECTID, + BTRFS_FIRST_FREE_OBJECTID, + BTRFS_LAST_FREE_OBJECTID, + BTRFS_FREE_INO_OBJECTID); + } else { + dir_item_err(leaf, slot, +"invalid location key objectid: has %llu expect %llu or [%llu, %llu] or %llu", + key->objectid, BTRFS_ROOT_TREE_DIR_OBJECTID, + BTRFS_FIRST_FREE_OBJECTID, + BTRFS_LAST_FREE_OBJECTID, + BTRFS_FREE_INO_OBJECTID); + } + return -EUCLEAN; + } + if (unlikely(key->offset != 0)) { + if (is_inode_item) + inode_item_err(leaf, slot, + "invalid key offset: has %llu expect 0", + key->offset); + else + dir_item_err(leaf, slot, + "invalid location key offset:has %llu expect 0", + key->offset); + return -EUCLEAN; + } + return 0; +} + +static int check_root_key(struct extent_buffer *leaf, struct btrfs_key *key, + int slot) +{ + struct btrfs_key item_key; + bool is_root_item; + + btrfs_item_key_to_cpu(leaf, &item_key, slot); + is_root_item = (item_key.type == BTRFS_ROOT_ITEM_KEY); + + /* + * Bad rootid for reloc trees. + * + * Reloc trees are only for subvolume trees, other trees only need + * to be COWed to be relocated. + */ + if (unlikely(is_root_item && key->objectid == BTRFS_TREE_RELOC_OBJECTID && + !btrfs_is_fstree(key->offset))) { + generic_err(leaf, slot, + "invalid reloc tree for root %lld, root id is not a subvolume tree", + key->offset); + return -EUCLEAN; + } + + /* No such tree id */ + if (unlikely(key->objectid == 0)) { + if (is_root_item) + generic_err(leaf, slot, "invalid root id 0"); + else + dir_item_err(leaf, slot, + "invalid location key root id 0"); + return -EUCLEAN; + } + + /* DIR_ITEM/INDEX/INODE_REF is not allowed to point to non-fs trees */ + if (unlikely(!btrfs_is_fstree(key->objectid) && !is_root_item)) { + dir_item_err(leaf, slot, + "invalid location key objectid, have %llu expect [%llu, %llu]", + key->objectid, BTRFS_FIRST_FREE_OBJECTID, + BTRFS_LAST_FREE_OBJECTID); + return -EUCLEAN; + } + + /* + * ROOT_ITEM with non-zero offset means this is a snapshot, created at + * @offset transid. + * Furthermore, for location key in DIR_ITEM, its offset is always -1. + * + * So here we only check offset for reloc tree whose key->offset must + * be a valid tree. + */ + if (unlikely(key->objectid == BTRFS_TREE_RELOC_OBJECTID && + key->offset == 0)) { + generic_err(leaf, slot, "invalid root id 0 for reloc tree"); + return -EUCLEAN; + } + return 0; +} + +static int check_dir_item(struct extent_buffer *leaf, + struct btrfs_key *key, struct btrfs_key *prev_key, + int slot) +{ + struct btrfs_fs_info *fs_info = leaf->fs_info; + struct btrfs_dir_item *di; + u32 item_size = btrfs_item_size(leaf, slot); + u32 cur = 0; + + if (unlikely(!check_prev_ino(leaf, key, slot, prev_key))) + return -EUCLEAN; + + di = btrfs_item_ptr(leaf, slot, struct btrfs_dir_item); + while (cur < item_size) { + struct btrfs_key location_key; + u32 name_len; + u32 data_len; + u32 max_name_len; + u32 total_size; + u32 name_hash; + u8 dir_type; + int ret; + + /* header itself should not cross item boundary */ + if (unlikely(cur + sizeof(*di) > item_size)) { + dir_item_err(leaf, slot, + "dir item header crosses item boundary, have %zu boundary %u", + cur + sizeof(*di), item_size); + return -EUCLEAN; + } + + /* Location key check */ + btrfs_dir_item_key_to_cpu(leaf, di, &location_key); + if (location_key.type == BTRFS_ROOT_ITEM_KEY) { + ret = check_root_key(leaf, &location_key, slot); + if (unlikely(ret < 0)) + return ret; + } else if (location_key.type == BTRFS_INODE_ITEM_KEY || + location_key.type == 0) { + ret = check_inode_key(leaf, &location_key, slot); + if (unlikely(ret < 0)) + return ret; + } else { + dir_item_err(leaf, slot, + "invalid location key type, have %u, expect %u or %u", + location_key.type, BTRFS_ROOT_ITEM_KEY, + BTRFS_INODE_ITEM_KEY); + return -EUCLEAN; + } + + /* dir type check */ + dir_type = btrfs_dir_ftype(leaf, di); + if (unlikely(dir_type <= BTRFS_FT_UNKNOWN || + dir_type >= BTRFS_FT_MAX)) { + dir_item_err(leaf, slot, + "invalid dir item type, have %u expect (0, %u)", + dir_type, BTRFS_FT_MAX); + return -EUCLEAN; + } + + if (unlikely(key->type == BTRFS_XATTR_ITEM_KEY && + dir_type != BTRFS_FT_XATTR)) { + dir_item_err(leaf, slot, + "invalid dir item type for XATTR key, have %u expect %u", + dir_type, BTRFS_FT_XATTR); + return -EUCLEAN; + } + if (unlikely(dir_type == BTRFS_FT_XATTR && + key->type != BTRFS_XATTR_ITEM_KEY)) { + dir_item_err(leaf, slot, + "xattr dir type found for non-XATTR key"); + return -EUCLEAN; + } + if (dir_type == BTRFS_FT_XATTR) + max_name_len = XATTR_NAME_MAX; + else + max_name_len = BTRFS_NAME_LEN; + + /* Name/data length check */ + name_len = btrfs_dir_name_len(leaf, di); + data_len = btrfs_dir_data_len(leaf, di); + if (unlikely(name_len > max_name_len)) { + dir_item_err(leaf, slot, + "dir item name len too long, have %u max %u", + name_len, max_name_len); + return -EUCLEAN; + } + if (unlikely(name_len + data_len > BTRFS_MAX_XATTR_SIZE(fs_info))) { + dir_item_err(leaf, slot, + "dir item name and data len too long, have %u max %u", + name_len + data_len, + BTRFS_MAX_XATTR_SIZE(fs_info)); + return -EUCLEAN; + } + + if (unlikely(data_len && dir_type != BTRFS_FT_XATTR)) { + dir_item_err(leaf, slot, + "dir item with invalid data len, have %u expect 0", + data_len); + return -EUCLEAN; + } + + total_size = sizeof(*di) + name_len + data_len; + + /* header and name/data should not cross item boundary */ + if (unlikely(cur + total_size > item_size)) { + dir_item_err(leaf, slot, + "dir item data crosses item boundary, have %u boundary %u", + cur + total_size, item_size); + return -EUCLEAN; + } + + /* + * Special check for XATTR/DIR_ITEM, as key->offset is name + * hash, should match its name + */ + if (key->type == BTRFS_DIR_ITEM_KEY || + key->type == BTRFS_XATTR_ITEM_KEY) { + char namebuf[MAX(BTRFS_NAME_LEN, XATTR_NAME_MAX)]; + + read_extent_buffer(leaf, namebuf, + (unsigned long)(di + 1), name_len); + name_hash = btrfs_name_hash(namebuf, name_len); + if (unlikely(key->offset != name_hash)) { + dir_item_err(leaf, slot, + "name hash mismatch with key, have 0x%016x expect 0x%016llx", + name_hash, key->offset); + return -EUCLEAN; + } + } + cur += total_size; + di = (struct btrfs_dir_item *)((void *)di + total_size); + } + return 0; +} + +__printf(3, 4) +__cold +static void block_group_err(const struct extent_buffer *eb, int slot, + const char *fmt, ...) +{ + const struct btrfs_fs_info *fs_info = eb->fs_info; + struct btrfs_key key; + struct va_format vaf; + va_list args; + + btrfs_item_key_to_cpu(eb, &key, slot); + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + dump_page(folio_page(eb->folios[0], 0), "eb page dump"); + btrfs_crit(fs_info, + "corrupt %s: root=%llu block=%llu slot=%d bg_start=%llu bg_len=%llu, %pV", + btrfs_header_level(eb) == 0 ? "leaf" : "node", + btrfs_header_owner(eb), btrfs_header_bytenr(eb), slot, + key.objectid, key.offset, &vaf); + va_end(args); +} + +static int check_block_group_item(struct extent_buffer *leaf, + struct btrfs_key *key, int slot) +{ + struct btrfs_fs_info *fs_info = leaf->fs_info; + struct btrfs_block_group_item bgi; + u32 item_size = btrfs_item_size(leaf, slot); + u64 chunk_objectid; + u64 flags; + u64 type; + size_t exp_size; + + /* + * Here we don't really care about alignment since extent allocator can + * handle it. We care more about the size. + */ + if (unlikely(key->offset == 0)) { + block_group_err(leaf, slot, + "invalid block group size 0"); + return -EUCLEAN; + } + + if (btrfs_fs_incompat(fs_info, REMAP_TREE)) + exp_size = sizeof(struct btrfs_block_group_item_v2); + else + exp_size = sizeof(struct btrfs_block_group_item); + + if (unlikely(item_size != exp_size)) { + block_group_err(leaf, slot, + "invalid item size, have %u expect %zu", + item_size, exp_size); + return -EUCLEAN; + } + + read_extent_buffer(leaf, &bgi, btrfs_item_ptr_offset(leaf, slot), + sizeof(bgi)); + chunk_objectid = btrfs_stack_block_group_chunk_objectid(&bgi); + if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) { + /* + * We don't init the nr_global_roots until we load the global + * roots, so this could be 0 at mount time. If it's 0 we'll + * just assume we're fine, and later we'll check against our + * actual value. + */ + if (unlikely(fs_info->nr_global_roots && + chunk_objectid >= fs_info->nr_global_roots)) { + block_group_err(leaf, slot, + "invalid block group global root id, have %llu, needs to be <= %llu", + chunk_objectid, + fs_info->nr_global_roots); + return -EUCLEAN; + } + } else if (unlikely(chunk_objectid != BTRFS_FIRST_CHUNK_TREE_OBJECTID)) { + block_group_err(leaf, slot, + "invalid block group chunk objectid, have %llu expect %llu", + btrfs_stack_block_group_chunk_objectid(&bgi), + BTRFS_FIRST_CHUNK_TREE_OBJECTID); + return -EUCLEAN; + } + + if (unlikely(btrfs_stack_block_group_used(&bgi) > key->offset)) { + block_group_err(leaf, slot, + "invalid block group used, have %llu expect [0, %llu)", + btrfs_stack_block_group_used(&bgi), key->offset); + return -EUCLEAN; + } + + flags = btrfs_stack_block_group_flags(&bgi); + if (unlikely(hweight64(flags & BTRFS_BLOCK_GROUP_PROFILE_MASK) > 1)) { + block_group_err(leaf, slot, +"invalid profile flags, have 0x%llx (%lu bits set) expect no more than 1 bit set", + flags & BTRFS_BLOCK_GROUP_PROFILE_MASK, + hweight64(flags & BTRFS_BLOCK_GROUP_PROFILE_MASK)); + return -EUCLEAN; + } + + if (unlikely(flags & BTRFS_BLOCK_GROUP_METADATA_REMAP && + !btrfs_fs_incompat(fs_info, REMAP_TREE))) { + block_group_err(leaf, slot, +"invalid flags, have 0x%llx (METADATA_REMAP flag set) but no remap-tree incompat flag", + flags); + return -EUCLEAN; + } + + type = flags & BTRFS_BLOCK_GROUP_TYPE_MASK; + if (unlikely(type != BTRFS_BLOCK_GROUP_DATA && + type != BTRFS_BLOCK_GROUP_METADATA && + type != BTRFS_BLOCK_GROUP_SYSTEM && + type != BTRFS_BLOCK_GROUP_METADATA_REMAP && + type != (BTRFS_BLOCK_GROUP_METADATA | + BTRFS_BLOCK_GROUP_DATA))) { + block_group_err(leaf, slot, +"invalid type, have 0x%llx (%lu bits set) expect either 0x%llx, 0x%llx, 0x%llx, 0x%llx or 0x%llx", + type, hweight64(type), + BTRFS_BLOCK_GROUP_DATA, BTRFS_BLOCK_GROUP_METADATA, + BTRFS_BLOCK_GROUP_SYSTEM, BTRFS_BLOCK_GROUP_METADATA_REMAP, + BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA); + return -EUCLEAN; + } + + if (unlikely(!btrfs_fs_incompat(fs_info, REMAP_TREE) && + type == BTRFS_BLOCK_GROUP_METADATA_REMAP)) { + block_group_err(leaf, slot, + "invalid type, METADATA_REMAP set but REMAP_TREE incompat flag not set"); + return -EUCLEAN; + } + + if (unlikely(!btrfs_fs_incompat(fs_info, REMAP_TREE) && + flags & BTRFS_BLOCK_GROUP_REMAPPED)) { + block_group_err(leaf, slot, + "invalid flags, REMAPPED set but REMAP_TREE incompat flag not set"); + return -EUCLEAN; + } + + if (item_size == sizeof(struct btrfs_block_group_item_v2)) { + struct btrfs_block_group_item_v2 *bgi2; + u64 remap_bytes; + u32 identity_remap_count; + + bgi2 = btrfs_item_ptr(leaf, slot, struct btrfs_block_group_item_v2); + remap_bytes = btrfs_block_group_v2_remap_bytes(leaf, bgi2); + + if (unlikely(remap_bytes > key->offset)) { + block_group_err(leaf, slot, + "invalid remap_bytes, have %llu expect [0, %llu]", + remap_bytes, key->offset); + return -EUCLEAN; + } + + identity_remap_count = btrfs_block_group_v2_identity_remap_count(leaf, bgi2); + if (unlikely((u64)identity_remap_count > + key->offset >> fs_info->sectorsize_bits)) { + block_group_err(leaf, slot, + "invalid identity_remap_count, have %u expect [0, %llu]", + identity_remap_count, + key->offset >> fs_info->sectorsize_bits); + return -EUCLEAN; + } + } + + return 0; +} + +__printf(5, 6) +__cold +static void chunk_err(const struct btrfs_fs_info *fs_info, + const struct extent_buffer *leaf, + const struct btrfs_chunk *chunk, u64 logical, + const char *fmt, ...) +{ + bool is_sb = !leaf; + struct va_format vaf; + va_list args; + int i; + int slot = -1; + + if (!is_sb) { + /* + * Get the slot number by iterating through all slots, this + * would provide better readability. + */ + for (i = 0; i < btrfs_header_nritems(leaf); i++) { + if (btrfs_item_ptr_offset(leaf, i) == + (unsigned long)chunk) { + slot = i; + break; + } + } + } + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + + if (is_sb) + btrfs_crit(fs_info, + "corrupt superblock syschunk array: chunk_start=%llu, %pV", + logical, &vaf); + else + btrfs_crit(fs_info, + "corrupt leaf: root=%llu block=%llu slot=%d chunk_start=%llu, %pV", + BTRFS_CHUNK_TREE_OBJECTID, leaf->start, slot, + logical, &vaf); + va_end(args); +} + +static bool valid_stripe_count(u64 profile, u16 num_stripes, u16 sub_stripes) +{ + switch (profile) { + case BTRFS_BLOCK_GROUP_RAID0: + return true; + case BTRFS_BLOCK_GROUP_RAID10: + return sub_stripes == btrfs_raid_array[BTRFS_RAID_RAID10].sub_stripes; + case BTRFS_BLOCK_GROUP_RAID1: + return num_stripes == btrfs_raid_array[BTRFS_RAID_RAID1].devs_min; + case BTRFS_BLOCK_GROUP_RAID1C3: + return num_stripes == btrfs_raid_array[BTRFS_RAID_RAID1C3].devs_min; + case BTRFS_BLOCK_GROUP_RAID1C4: + return num_stripes == btrfs_raid_array[BTRFS_RAID_RAID1C4].devs_min; + case BTRFS_BLOCK_GROUP_RAID5: + return num_stripes >= btrfs_raid_array[BTRFS_RAID_RAID5].devs_min; + case BTRFS_BLOCK_GROUP_RAID6: + return num_stripes >= btrfs_raid_array[BTRFS_RAID_RAID6].devs_min; + case BTRFS_BLOCK_GROUP_DUP: + return num_stripes == btrfs_raid_array[BTRFS_RAID_DUP].dev_stripes; + case 0: /* SINGLE */ + return num_stripes == btrfs_raid_array[BTRFS_RAID_SINGLE].dev_stripes; + default: + BUG(); + } +} + +/* + * The common chunk check which could also work on super block sys chunk array. + * + * If @leaf is NULL, then @chunk must be an on-stack chunk item. + * (For superblock sys_chunk array, and fs_info->sectorsize is unreliable) + * + * Return -EUCLEAN if anything is corrupted. + * Return 0 if everything is OK. + */ +int btrfs_check_chunk_valid(const struct btrfs_fs_info *fs_info, + const struct extent_buffer *leaf, + const struct btrfs_chunk *chunk, u64 logical, + u32 sectorsize) +{ + u64 length; + u64 chunk_end; + u64 stripe_len; + u16 num_stripes; + u16 sub_stripes; + u64 type; + u64 features; + u32 chunk_sector_size; + bool mixed = false; + bool remapped; + int raid_index; + int nparity; + int ncopies; + + if (leaf) { + length = btrfs_chunk_length(leaf, chunk); + stripe_len = btrfs_chunk_stripe_len(leaf, chunk); + num_stripes = btrfs_chunk_num_stripes(leaf, chunk); + sub_stripes = btrfs_chunk_sub_stripes(leaf, chunk); + type = btrfs_chunk_type(leaf, chunk); + chunk_sector_size = btrfs_chunk_sector_size(leaf, chunk); + } else { + length = btrfs_stack_chunk_length(chunk); + stripe_len = btrfs_stack_chunk_stripe_len(chunk); + num_stripes = btrfs_stack_chunk_num_stripes(chunk); + sub_stripes = btrfs_stack_chunk_sub_stripes(chunk); + type = btrfs_stack_chunk_type(chunk); + chunk_sector_size = btrfs_stack_chunk_sector_size(chunk); + } + raid_index = btrfs_bg_flags_to_raid_index(type); + ncopies = btrfs_raid_array[raid_index].ncopies; + nparity = btrfs_raid_array[raid_index].nparity; + remapped = (type & BTRFS_BLOCK_GROUP_REMAPPED); + + if (unlikely(!remapped && !num_stripes)) { + chunk_err(fs_info, leaf, chunk, logical, + "invalid chunk num_stripes, have %u", num_stripes); + return -EUCLEAN; + } + if (unlikely(num_stripes != 0 && num_stripes < ncopies)) { + chunk_err(fs_info, leaf, chunk, logical, + "invalid chunk num_stripes < ncopies, have %u < %d", + num_stripes, ncopies); + return -EUCLEAN; + } + if (unlikely(nparity && num_stripes == nparity)) { + chunk_err(fs_info, leaf, chunk, logical, + "invalid chunk num_stripes == nparity, have %u == %d", + num_stripes, nparity); + return -EUCLEAN; + } + if (unlikely(!IS_ALIGNED(logical, sectorsize))) { + chunk_err(fs_info, leaf, chunk, logical, + "invalid chunk logical, have %llu should aligned to %u", + logical, sectorsize); + return -EUCLEAN; + } + if (unlikely(chunk_sector_size != sectorsize)) { + chunk_err(fs_info, leaf, chunk, logical, + "invalid chunk sectorsize, have %u expect %u", + chunk_sector_size, sectorsize); + return -EUCLEAN; + } + if (unlikely(!length || !IS_ALIGNED(length, sectorsize))) { + chunk_err(fs_info, leaf, chunk, logical, + "invalid chunk length, have %llu", length); + return -EUCLEAN; + } + if (unlikely(check_add_overflow(logical, length, &chunk_end))) { + chunk_err(fs_info, leaf, chunk, logical, +"invalid chunk logical start and length, have logical start %llu length %llu", + logical, length); + return -EUCLEAN; + } + if (unlikely(!is_power_of_2(stripe_len) || stripe_len != BTRFS_STRIPE_LEN)) { + chunk_err(fs_info, leaf, chunk, logical, + "invalid chunk stripe length: %llu", + stripe_len); + return -EUCLEAN; + } + /* + * We artificially limit the chunk size, so that the number of stripes + * inside a chunk can be fit into a U32. The current limit (256G) is + * way too large for real world usage anyway, and it's also much larger + * than our existing limit (10G). + * + * Thus it should be a good way to catch obvious bitflips. + */ + if (unlikely(length >= btrfs_stripe_nr_to_offset(U32_MAX))) { + chunk_err(fs_info, leaf, chunk, logical, + "chunk length too large: have %llu limit %llu", + length, btrfs_stripe_nr_to_offset(U32_MAX)); + return -EUCLEAN; + } + if (unlikely(type & ~BTRFS_BLOCK_GROUP_VALID)) { + chunk_err(fs_info, leaf, chunk, logical, + "unrecognized chunk type: 0x%llx", + type & ~BTRFS_BLOCK_GROUP_VALID); + return -EUCLEAN; + } + + if (unlikely(!has_single_bit_set(type & BTRFS_BLOCK_GROUP_PROFILE_MASK) && + (type & BTRFS_BLOCK_GROUP_PROFILE_MASK) != 0)) { + chunk_err(fs_info, leaf, chunk, logical, + "invalid chunk profile flag: 0x%llx, expect 0 or 1 bit set", + type & BTRFS_BLOCK_GROUP_PROFILE_MASK); + return -EUCLEAN; + } + if (unlikely((type & BTRFS_BLOCK_GROUP_TYPE_MASK) == 0)) { + chunk_err(fs_info, leaf, chunk, logical, + "missing chunk type flag, have 0x%llx one bit must be set in 0x%llx", + type, BTRFS_BLOCK_GROUP_TYPE_MASK); + return -EUCLEAN; + } + + if (unlikely((type & BTRFS_BLOCK_GROUP_SYSTEM) && + (type & (BTRFS_BLOCK_GROUP_METADATA | + BTRFS_BLOCK_GROUP_DATA)))) { + chunk_err(fs_info, leaf, chunk, logical, + "system chunk with data or metadata type: 0x%llx", + type); + return -EUCLEAN; + } + + features = btrfs_super_incompat_flags(fs_info->super_copy); + if (features & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS) + mixed = true; + + if (!mixed) { + if (unlikely((type & BTRFS_BLOCK_GROUP_METADATA) && + (type & BTRFS_BLOCK_GROUP_DATA))) { + chunk_err(fs_info, leaf, chunk, logical, + "mixed chunk type in non-mixed mode: 0x%llx", type); + return -EUCLEAN; + } + } + + if (unlikely((type & BTRFS_BLOCK_GROUP_METADATA_REMAP) && + !(features & BTRFS_FEATURE_INCOMPAT_REMAP_TREE))) { + chunk_err(fs_info, leaf, chunk, logical, + "METADATA_REMAP chunk type without REMAP_TREE incompat bit"); + return -EUCLEAN; + } + + if (unlikely(remapped && + !(features & BTRFS_FEATURE_INCOMPAT_REMAP_TREE))) { + chunk_err(fs_info, leaf, chunk, logical, + "REMAPPED chunk flag without REMAP_TREE incompat bit"); + return -EUCLEAN; + } + + if (!remapped && + !valid_stripe_count(type & BTRFS_BLOCK_GROUP_PROFILE_MASK, + num_stripes, sub_stripes)) { + chunk_err(fs_info, leaf, chunk, logical, + "invalid num_stripes:sub_stripes %u:%u for profile %llu", + num_stripes, sub_stripes, + type & BTRFS_BLOCK_GROUP_PROFILE_MASK); + return -EUCLEAN; + } + + return 0; +} + +/* + * Enhanced version of chunk item checker. + * + * The common btrfs_check_chunk_valid() doesn't check item size since it needs + * to work on super block sys_chunk_array which doesn't have full item ptr. + */ +static int check_leaf_chunk_item(struct extent_buffer *leaf, + struct btrfs_chunk *chunk, + struct btrfs_key *key, int slot) +{ + struct btrfs_fs_info *fs_info = leaf->fs_info; + int num_stripes; + + if (unlikely(btrfs_item_size(leaf, slot) < offsetof(struct btrfs_chunk, stripe))) { + chunk_err(fs_info, leaf, chunk, key->offset, + "invalid chunk item size: have %u expect [%zu, %u)", + btrfs_item_size(leaf, slot), + offsetof(struct btrfs_chunk, stripe), + BTRFS_LEAF_DATA_SIZE(fs_info)); + return -EUCLEAN; + } + + num_stripes = btrfs_chunk_num_stripes(leaf, chunk); + /* Let btrfs_check_chunk_valid() handle this error type */ + if (num_stripes == 0) + goto out; + + if (unlikely(btrfs_chunk_item_size(num_stripes) != + btrfs_item_size(leaf, slot))) { + chunk_err(fs_info, leaf, chunk, key->offset, + "invalid chunk item size: have %u expect %lu", + btrfs_item_size(leaf, slot), + btrfs_chunk_item_size(num_stripes)); + return -EUCLEAN; + } +out: + return btrfs_check_chunk_valid(fs_info, leaf, chunk, key->offset, + fs_info->sectorsize); +} + +__printf(3, 4) +__cold +static void dev_item_err(const struct extent_buffer *eb, int slot, + const char *fmt, ...) +{ + struct btrfs_key key; + struct va_format vaf; + va_list args; + + btrfs_item_key_to_cpu(eb, &key, slot); + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + dump_page(folio_page(eb->folios[0], 0), "eb page dump"); + btrfs_crit(eb->fs_info, + "corrupt %s: root=%llu block=%llu slot=%d devid=%llu %pV", + btrfs_header_level(eb) == 0 ? "leaf" : "node", + btrfs_header_owner(eb), btrfs_header_bytenr(eb), slot, + key.objectid, &vaf); + va_end(args); +} + +static int check_dev_item(struct extent_buffer *leaf, + struct btrfs_key *key, int slot) +{ + struct btrfs_dev_item *ditem; + const u32 item_size = btrfs_item_size(leaf, slot); + + if (unlikely(key->objectid != BTRFS_DEV_ITEMS_OBJECTID)) { + dev_item_err(leaf, slot, + "invalid objectid: has=%llu expect=%llu", + key->objectid, BTRFS_DEV_ITEMS_OBJECTID); + return -EUCLEAN; + } + + if (unlikely(item_size != sizeof(*ditem))) { + dev_item_err(leaf, slot, "invalid item size: has %u expect %zu", + item_size, sizeof(*ditem)); + return -EUCLEAN; + } + + ditem = btrfs_item_ptr(leaf, slot, struct btrfs_dev_item); + if (unlikely(btrfs_device_id(leaf, ditem) != key->offset)) { + dev_item_err(leaf, slot, + "devid mismatch: key has=%llu item has=%llu", + key->offset, btrfs_device_id(leaf, ditem)); + return -EUCLEAN; + } + + /* + * For device total_bytes, we don't have reliable way to check it, as + * it can be 0 for device removal. Device size check can only be done + * by dev extents check. + */ + if (unlikely(btrfs_device_bytes_used(leaf, ditem) > + btrfs_device_total_bytes(leaf, ditem))) { + dev_item_err(leaf, slot, + "invalid bytes used: have %llu expect [0, %llu]", + btrfs_device_bytes_used(leaf, ditem), + btrfs_device_total_bytes(leaf, ditem)); + return -EUCLEAN; + } + /* + * Remaining members like io_align/type/gen/dev_group aren't really + * utilized. Skip them to make later usage of them easier. + */ + return 0; +} + +static int check_inode_item(struct extent_buffer *leaf, + struct btrfs_key *key, int slot) +{ + struct btrfs_fs_info *fs_info = leaf->fs_info; + struct btrfs_inode_item *iitem; + u64 super_gen = btrfs_super_generation(fs_info->super_copy); + u32 valid_mask = (S_IFMT | S_ISUID | S_ISGID | S_ISVTX | 0777); + const u32 item_size = btrfs_item_size(leaf, slot); + u32 mode; + int ret; + u32 flags; + u32 ro_flags; + + ret = check_inode_key(leaf, key, slot); + if (unlikely(ret < 0)) + return ret; + + if (unlikely(item_size != sizeof(*iitem))) { + generic_err(leaf, slot, "invalid item size: has %u expect %zu", + item_size, sizeof(*iitem)); + return -EUCLEAN; + } + + iitem = btrfs_item_ptr(leaf, slot, struct btrfs_inode_item); + + /* Here we use super block generation + 1 to handle log tree */ + if (unlikely(btrfs_inode_generation(leaf, iitem) > super_gen + 1)) { + inode_item_err(leaf, slot, + "invalid inode generation: has %llu expect (0, %llu]", + btrfs_inode_generation(leaf, iitem), + super_gen + 1); + return -EUCLEAN; + } + /* Note for ROOT_TREE_DIR_ITEM, mkfs could set its transid 0 */ + if (unlikely(btrfs_inode_transid(leaf, iitem) > super_gen + 1)) { + inode_item_err(leaf, slot, + "invalid inode transid: has %llu expect [0, %llu]", + btrfs_inode_transid(leaf, iitem), super_gen + 1); + return -EUCLEAN; + } + + /* + * For size and nbytes it's better not to be too strict, as for dir + * item its size/nbytes can easily get wrong, but doesn't affect + * anything in the fs. So here we skip the check. + */ + mode = btrfs_inode_mode(leaf, iitem); + if (unlikely(mode & ~valid_mask)) { + inode_item_err(leaf, slot, + "unknown mode bit detected: 0x%x", + mode & ~valid_mask); + return -EUCLEAN; + } + + /* + * S_IFMT is not bit mapped so we can't completely rely on + * is_power_of_2/has_single_bit_set, but it can save us from checking + * FIFO/CHR/DIR/REG. Only needs to check BLK, LNK and SOCKS + */ + if (!has_single_bit_set(mode & S_IFMT)) { + if (unlikely(!S_ISLNK(mode) && !S_ISBLK(mode) && !S_ISSOCK(mode))) { + inode_item_err(leaf, slot, + "invalid mode: has 0%o expect valid S_IF* bit(s)", + mode & S_IFMT); + return -EUCLEAN; + } + } + if (unlikely(S_ISDIR(mode) && btrfs_inode_nlink(leaf, iitem) > 1)) { + inode_item_err(leaf, slot, + "invalid nlink: has %u expect no more than 1 for dir", + btrfs_inode_nlink(leaf, iitem)); + return -EUCLEAN; + } + btrfs_inode_split_flags(btrfs_inode_flags(leaf, iitem), &flags, &ro_flags); + if (unlikely(flags & ~BTRFS_INODE_FLAG_MASK)) { + inode_item_err(leaf, slot, + "unknown incompat flags detected: 0x%x", flags); + return -EUCLEAN; + } + if (unlikely(!sb_rdonly(fs_info->sb) && + (ro_flags & ~BTRFS_INODE_RO_FLAG_MASK))) { + inode_item_err(leaf, slot, + "unknown ro-compat flags detected on writeable mount: 0x%x", + ro_flags); + return -EUCLEAN; + } + return 0; +} + +static int check_root_item(struct extent_buffer *leaf, struct btrfs_key *key, + int slot) +{ + struct btrfs_fs_info *fs_info = leaf->fs_info; + struct btrfs_root_item ri = { 0 }; + const u64 valid_root_flags = BTRFS_ROOT_SUBVOL_RDONLY | + BTRFS_ROOT_SUBVOL_DEAD; + int ret; + + ret = check_root_key(leaf, key, slot); + if (unlikely(ret < 0)) + return ret; + + if (unlikely(btrfs_item_size(leaf, slot) != sizeof(ri) && + btrfs_item_size(leaf, slot) != + btrfs_legacy_root_item_size())) { + generic_err(leaf, slot, + "invalid root item size, have %u expect %zu or %u", + btrfs_item_size(leaf, slot), sizeof(ri), + btrfs_legacy_root_item_size()); + return -EUCLEAN; + } + + /* + * For legacy root item, the members starting at generation_v2 will be + * all filled with 0. + * And since we allow generation_v2 as 0, it will still pass the check. + */ + read_extent_buffer(leaf, &ri, btrfs_item_ptr_offset(leaf, slot), + btrfs_item_size(leaf, slot)); + + /* Generation related */ + if (unlikely(btrfs_root_generation(&ri) > + btrfs_super_generation(fs_info->super_copy) + 1)) { + generic_err(leaf, slot, + "invalid root generation, have %llu expect (0, %llu]", + btrfs_root_generation(&ri), + btrfs_super_generation(fs_info->super_copy) + 1); + return -EUCLEAN; + } + if (unlikely(btrfs_root_generation_v2(&ri) > + btrfs_super_generation(fs_info->super_copy) + 1)) { + generic_err(leaf, slot, + "invalid root v2 generation, have %llu expect (0, %llu]", + btrfs_root_generation_v2(&ri), + btrfs_super_generation(fs_info->super_copy) + 1); + return -EUCLEAN; + } + if (unlikely(btrfs_root_last_snapshot(&ri) > + btrfs_super_generation(fs_info->super_copy) + 1)) { + generic_err(leaf, slot, + "invalid root last_snapshot, have %llu expect (0, %llu]", + btrfs_root_last_snapshot(&ri), + btrfs_super_generation(fs_info->super_copy) + 1); + return -EUCLEAN; + } + + /* Alignment and level check */ + if (unlikely(!IS_ALIGNED(btrfs_root_bytenr(&ri), fs_info->sectorsize))) { + generic_err(leaf, slot, + "invalid root bytenr, have %llu expect to be aligned to %u", + btrfs_root_bytenr(&ri), fs_info->sectorsize); + return -EUCLEAN; + } + if (unlikely(btrfs_root_level(&ri) >= BTRFS_MAX_LEVEL)) { + generic_err(leaf, slot, + "invalid root level, have %u expect [0, %u]", + btrfs_root_level(&ri), BTRFS_MAX_LEVEL - 1); + return -EUCLEAN; + } + if (unlikely(btrfs_root_drop_level(&ri) >= BTRFS_MAX_LEVEL)) { + generic_err(leaf, slot, + "invalid root drop_level, have %u expect [0, %u]", + btrfs_root_drop_level(&ri), BTRFS_MAX_LEVEL - 1); + return -EUCLEAN; + } + /* + * If drop_progress.objectid is non-zero, a btrfs_drop_snapshot() was + * interrupted and the resume point was recorded in drop_progress and + * drop_level. In that case drop_level must be >= 1: level 0 is the + * leaf level and drop_snapshot never saves a checkpoint there (it + * only records checkpoints at internal node levels in DROP_REFERENCE + * stage). A zero drop_level combined with a non-zero drop_progress + * objectid indicates on-disk corruption and would cause a BUG_ON in + * merge_reloc_root() and btrfs_drop_snapshot() at mount time. + */ + if (unlikely(btrfs_disk_key_objectid(&ri.drop_progress) != 0 && + btrfs_root_drop_level(&ri) == 0)) { + generic_err(leaf, slot, + "invalid root drop_level 0 with non-zero drop_progress objectid %llu", + btrfs_disk_key_objectid(&ri.drop_progress)); + return -EUCLEAN; + } + + /* Flags check */ + if (unlikely(btrfs_root_flags(&ri) & ~valid_root_flags)) { + generic_err(leaf, slot, + "invalid root flags, have 0x%llx expect mask 0x%llx", + btrfs_root_flags(&ri), valid_root_flags); + return -EUCLEAN; + } + return 0; +} + +static int check_root_ref(struct extent_buffer *leaf, struct btrfs_key *key, int slot) +{ + struct btrfs_root_ref *rref; + u32 item_size = btrfs_item_size(leaf, slot); + u32 name_len; + + if (unlikely(item_size <= sizeof(*rref))) { + generic_err(leaf, slot, + "invalid root ref item size for key type %u, have %u expect > %zu", + key->type, item_size, sizeof(*rref)); + return -EUCLEAN; + } + + rref = btrfs_item_ptr(leaf, slot, struct btrfs_root_ref); + name_len = btrfs_root_ref_name_len(leaf, rref); + if (unlikely(name_len > BTRFS_NAME_LEN)) { + generic_err(leaf, slot, + "root ref name too long for key type %u, have %u max %u", + key->type, name_len, BTRFS_NAME_LEN); + return -EUCLEAN; + } + if (unlikely(item_size != sizeof(*rref) + name_len)) { + generic_err(leaf, slot, + "invalid root ref item size for key type %u, have %u expect %zu", + key->type, item_size, sizeof(*rref) + name_len); + return -EUCLEAN; + } + + return 0; +} + +__printf(3,4) +__cold +static void extent_err(const struct extent_buffer *eb, int slot, + const char *fmt, ...) +{ + struct btrfs_key key; + struct va_format vaf; + va_list args; + u64 bytenr; + u64 len; + + btrfs_item_key_to_cpu(eb, &key, slot); + bytenr = key.objectid; + if (key.type == BTRFS_METADATA_ITEM_KEY || + key.type == BTRFS_TREE_BLOCK_REF_KEY || + key.type == BTRFS_SHARED_BLOCK_REF_KEY) + len = eb->fs_info->nodesize; + else + len = key.offset; + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + dump_page(folio_page(eb->folios[0], 0), "eb page dump"); + btrfs_crit(eb->fs_info, + "corrupt %s: block=%llu slot=%d extent bytenr=%llu len=%llu %pV", + btrfs_header_level(eb) == 0 ? "leaf" : "node", + eb->start, slot, bytenr, len, &vaf); + va_end(args); +} + +static bool is_valid_dref_root(u64 rootid) +{ + /* + * The following tree root objectids are allowed to have a data backref: + * - subvolume trees + * - data reloc tree + * - tree root + * For v1 space cache + */ + return btrfs_is_fstree(rootid) || rootid == BTRFS_DATA_RELOC_TREE_OBJECTID || + rootid == BTRFS_ROOT_TREE_OBJECTID; +} + +static int check_extent_item(struct extent_buffer *leaf, + struct btrfs_key *key, int slot, + struct btrfs_key *prev_key) +{ + struct btrfs_fs_info *fs_info = leaf->fs_info; + struct btrfs_extent_item *ei; + bool is_tree_block = false; + unsigned long ptr; /* Current pointer inside inline refs */ + unsigned long end; /* Extent item end */ + const u32 item_size = btrfs_item_size(leaf, slot); + u8 last_type = 0; + u64 last_seq = U64_MAX; + u64 flags; + u64 generation; + u64 total_refs; /* Total refs in btrfs_extent_item */ + u64 inline_refs = 0; /* found total inline refs */ + + if (unlikely(key->type == BTRFS_METADATA_ITEM_KEY && + !btrfs_fs_incompat(fs_info, SKINNY_METADATA))) { + generic_err(leaf, slot, +"invalid key type, METADATA_ITEM type invalid when SKINNY_METADATA feature disabled"); + return -EUCLEAN; + } + /* key->objectid is the bytenr for both key types */ + if (unlikely(!IS_ALIGNED(key->objectid, fs_info->sectorsize))) { + generic_err(leaf, slot, + "invalid key objectid, have %llu expect to be aligned to %u", + key->objectid, fs_info->sectorsize); + return -EUCLEAN; + } + + /* key->offset is tree level for METADATA_ITEM_KEY */ + if (unlikely(key->type == BTRFS_METADATA_ITEM_KEY && + key->offset >= BTRFS_MAX_LEVEL)) { + extent_err(leaf, slot, + "invalid tree level, have %llu expect [0, %u]", + key->offset, BTRFS_MAX_LEVEL - 1); + return -EUCLEAN; + } + + /* + * EXTENT/METADATA_ITEM consists of: + * 1) One btrfs_extent_item + * Records the total refs, type and generation of the extent. + * + * 2) One btrfs_tree_block_info (for EXTENT_ITEM and tree backref only) + * Records the first key and level of the tree block. + * + * 2) Zero or more btrfs_extent_inline_ref(s) + * Each inline ref has one btrfs_extent_inline_ref shows: + * 2.1) The ref type, one of the 4 + * TREE_BLOCK_REF Tree block only + * SHARED_BLOCK_REF Tree block only + * EXTENT_DATA_REF Data only + * SHARED_DATA_REF Data only + * 2.2) Ref type specific data + * Either using btrfs_extent_inline_ref::offset, or specific + * data structure. + * + * All above inline items should follow the order: + * + * - All btrfs_extent_inline_ref::type should be in an ascending + * order + * + * - Within the same type, the items should follow a descending + * order by their sequence number. The sequence number is + * determined by: + * * btrfs_extent_inline_ref::offset for all types other than + * EXTENT_DATA_REF + * * hash_extent_data_ref() for EXTENT_DATA_REF + */ + if (unlikely(item_size < sizeof(*ei))) { + extent_err(leaf, slot, + "invalid item size, have %u expect [%zu, %u)", + item_size, sizeof(*ei), + BTRFS_LEAF_DATA_SIZE(fs_info)); + return -EUCLEAN; + } + end = item_size + btrfs_item_ptr_offset(leaf, slot); + + /* Checks against extent_item */ + ei = btrfs_item_ptr(leaf, slot, struct btrfs_extent_item); + flags = btrfs_extent_flags(leaf, ei); + total_refs = btrfs_extent_refs(leaf, ei); + generation = btrfs_extent_generation(leaf, ei); + if (unlikely(generation > + btrfs_super_generation(fs_info->super_copy) + 1)) { + extent_err(leaf, slot, + "invalid generation, have %llu expect (0, %llu]", + generation, + btrfs_super_generation(fs_info->super_copy) + 1); + return -EUCLEAN; + } + if (unlikely(!has_single_bit_set(flags & (BTRFS_EXTENT_FLAG_DATA | + BTRFS_EXTENT_FLAG_TREE_BLOCK)))) { + extent_err(leaf, slot, + "invalid extent flag, have 0x%llx expect 1 bit set in 0x%llx", + flags, BTRFS_EXTENT_FLAG_DATA | + BTRFS_EXTENT_FLAG_TREE_BLOCK); + return -EUCLEAN; + } + is_tree_block = !!(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK); + if (is_tree_block) { + if (unlikely(key->type == BTRFS_EXTENT_ITEM_KEY && + key->offset != fs_info->nodesize)) { + extent_err(leaf, slot, + "invalid extent length, have %llu expect %u", + key->offset, fs_info->nodesize); + return -EUCLEAN; + } + } else { + if (unlikely(key->type != BTRFS_EXTENT_ITEM_KEY)) { + extent_err(leaf, slot, + "invalid key type, have %u expect %u for data backref", + key->type, BTRFS_EXTENT_ITEM_KEY); + return -EUCLEAN; + } + if (unlikely(!IS_ALIGNED(key->offset, fs_info->sectorsize))) { + extent_err(leaf, slot, + "invalid extent length, have %llu expect aligned to %u", + key->offset, fs_info->sectorsize); + return -EUCLEAN; + } + if (unlikely(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF)) { + extent_err(leaf, slot, + "invalid extent flag, data has full backref set"); + return -EUCLEAN; + } + } + ptr = (unsigned long)(struct btrfs_extent_item *)(ei + 1); + + /* Check the special case of btrfs_tree_block_info */ + if (is_tree_block && key->type != BTRFS_METADATA_ITEM_KEY) { + struct btrfs_tree_block_info *info; + + info = (struct btrfs_tree_block_info *)ptr; + if (unlikely(btrfs_tree_block_level(leaf, info) >= BTRFS_MAX_LEVEL)) { + extent_err(leaf, slot, + "invalid tree block info level, have %u expect [0, %u]", + btrfs_tree_block_level(leaf, info), + BTRFS_MAX_LEVEL - 1); + return -EUCLEAN; + } + ptr = (unsigned long)(struct btrfs_tree_block_info *)(info + 1); + } + + /* Check inline refs */ + while (ptr < end) { + struct btrfs_extent_inline_ref *iref; + struct btrfs_extent_data_ref *dref; + struct btrfs_shared_data_ref *sref; + u64 seq; + u64 dref_root; + u64 dref_objectid; + u64 dref_offset; + u64 inline_offset; + u8 inline_type; + + if (unlikely(ptr + sizeof(*iref) > end)) { + extent_err(leaf, slot, +"inline ref item overflows extent item, ptr %lu iref size %zu end %lu", + ptr, sizeof(*iref), end); + return -EUCLEAN; + } + iref = (struct btrfs_extent_inline_ref *)ptr; + inline_type = btrfs_extent_inline_ref_type(leaf, iref); + inline_offset = btrfs_extent_inline_ref_offset(leaf, iref); + seq = inline_offset; + if (unlikely(ptr + btrfs_extent_inline_ref_size(inline_type) > end)) { + extent_err(leaf, slot, +"inline ref item overflows extent item, ptr %lu iref size %u end %lu", + ptr, btrfs_extent_inline_ref_size(inline_type), end); + return -EUCLEAN; + } + + switch (inline_type) { + /* inline_offset is subvolid of the owner, no need to check */ + case BTRFS_TREE_BLOCK_REF_KEY: + inline_refs++; + break; + /* Contains parent bytenr */ + case BTRFS_SHARED_BLOCK_REF_KEY: + if (unlikely(!IS_ALIGNED(inline_offset, + fs_info->sectorsize))) { + extent_err(leaf, slot, + "invalid tree parent bytenr, have %llu expect aligned to %u", + inline_offset, fs_info->sectorsize); + return -EUCLEAN; + } + inline_refs++; + break; + /* + * Contains owner subvolid, owner key objectid, adjusted offset. + * The only obvious corruption can happen in that offset. + */ + case BTRFS_EXTENT_DATA_REF_KEY: + dref = (struct btrfs_extent_data_ref *)(&iref->offset); + dref_root = btrfs_extent_data_ref_root(leaf, dref); + dref_objectid = btrfs_extent_data_ref_objectid(leaf, dref); + dref_offset = btrfs_extent_data_ref_offset(leaf, dref); + seq = hash_extent_data_ref( + btrfs_extent_data_ref_root(leaf, dref), + btrfs_extent_data_ref_objectid(leaf, dref), + btrfs_extent_data_ref_offset(leaf, dref)); + if (unlikely(!is_valid_dref_root(dref_root))) { + extent_err(leaf, slot, + "invalid data ref root value %llu", + dref_root); + return -EUCLEAN; + } + if (unlikely(dref_objectid < BTRFS_FIRST_FREE_OBJECTID || + dref_objectid > BTRFS_LAST_FREE_OBJECTID)) { + extent_err(leaf, slot, + "invalid data ref objectid value %llu", + dref_objectid); + return -EUCLEAN; + } + if (unlikely(!IS_ALIGNED(dref_offset, + fs_info->sectorsize))) { + extent_err(leaf, slot, + "invalid data ref offset, have %llu expect aligned to %u", + dref_offset, fs_info->sectorsize); + return -EUCLEAN; + } + if (unlikely(btrfs_extent_data_ref_count(leaf, dref) == 0)) { + extent_err(leaf, slot, + "invalid data ref count, should have non-zero value"); + return -EUCLEAN; + } + inline_refs += btrfs_extent_data_ref_count(leaf, dref); + break; + /* Contains parent bytenr and ref count */ + case BTRFS_SHARED_DATA_REF_KEY: + sref = (struct btrfs_shared_data_ref *)(iref + 1); + if (unlikely(!IS_ALIGNED(inline_offset, + fs_info->sectorsize))) { + extent_err(leaf, slot, + "invalid data parent bytenr, have %llu expect aligned to %u", + inline_offset, fs_info->sectorsize); + return -EUCLEAN; + } + if (unlikely(btrfs_shared_data_ref_count(leaf, sref) == 0)) { + extent_err(leaf, slot, + "invalid shared data ref count, should have non-zero value"); + return -EUCLEAN; + } + inline_refs += btrfs_shared_data_ref_count(leaf, sref); + break; + case BTRFS_EXTENT_OWNER_REF_KEY: + WARN_ON(!btrfs_fs_incompat(fs_info, SIMPLE_QUOTA)); + break; + default: + extent_err(leaf, slot, "unknown inline ref type: %u", + inline_type); + return -EUCLEAN; + } + if (unlikely(inline_type < last_type)) { + extent_err(leaf, slot, + "inline ref out-of-order: has type %u, prev type %u", + inline_type, last_type); + return -EUCLEAN; + } + /* Type changed, allow the sequence starts from U64_MAX again. */ + if (inline_type > last_type) + last_seq = U64_MAX; + if (unlikely(seq > last_seq)) { + extent_err(leaf, slot, +"inline ref out-of-order: has type %u offset %llu seq 0x%llx, prev type %u seq 0x%llx", + inline_type, inline_offset, seq, + last_type, last_seq); + return -EUCLEAN; + } + last_type = inline_type; + last_seq = seq; + ptr += btrfs_extent_inline_ref_size(inline_type); + } + /* No padding is allowed */ + if (unlikely(ptr != end)) { + extent_err(leaf, slot, + "invalid extent item size, padding bytes found"); + return -EUCLEAN; + } + + /* Finally, check the inline refs against total refs */ + if (unlikely(inline_refs > total_refs)) { + extent_err(leaf, slot, + "invalid extent refs, have %llu expect >= inline %llu", + total_refs, inline_refs); + return -EUCLEAN; + } + + if ((prev_key->type == BTRFS_EXTENT_ITEM_KEY) || + (prev_key->type == BTRFS_METADATA_ITEM_KEY)) { + u64 prev_end = prev_key->objectid; + + if (prev_key->type == BTRFS_METADATA_ITEM_KEY) + prev_end += fs_info->nodesize; + else + prev_end += prev_key->offset; + + if (unlikely(prev_end > key->objectid)) { + extent_err(leaf, slot, + "previous extent " BTRFS_KEY_FMT " overlaps current extent " BTRFS_KEY_FMT, + BTRFS_KEY_FMT_VALUE(prev_key), + BTRFS_KEY_FMT_VALUE(key)); + return -EUCLEAN; + } + } + + return 0; +} + +static int check_simple_keyed_refs(struct extent_buffer *leaf, + struct btrfs_key *key, int slot) +{ + u32 expect_item_size = 0; + + if (key->type == BTRFS_SHARED_DATA_REF_KEY) { + struct btrfs_shared_data_ref *sref; + + sref = btrfs_item_ptr(leaf, slot, struct btrfs_shared_data_ref); + if (unlikely(btrfs_shared_data_ref_count(leaf, sref) == 0)) { + extent_err(leaf, slot, + "invalid shared data backref count, should have non-zero value"); + return -EUCLEAN; + } + + expect_item_size = sizeof(struct btrfs_shared_data_ref); + } + + if (unlikely(btrfs_item_size(leaf, slot) != expect_item_size)) { + generic_err(leaf, slot, + "invalid item size, have %u expect %u for key type %u", + btrfs_item_size(leaf, slot), + expect_item_size, key->type); + return -EUCLEAN; + } + if (unlikely(!IS_ALIGNED(key->objectid, leaf->fs_info->sectorsize))) { + generic_err(leaf, slot, +"invalid key objectid for shared block ref, have %llu expect aligned to %u", + key->objectid, leaf->fs_info->sectorsize); + return -EUCLEAN; + } + if (unlikely(key->type != BTRFS_TREE_BLOCK_REF_KEY && + !IS_ALIGNED(key->offset, leaf->fs_info->sectorsize))) { + extent_err(leaf, slot, + "invalid tree parent bytenr, have %llu expect aligned to %u", + key->offset, leaf->fs_info->sectorsize); + return -EUCLEAN; + } + return 0; +} + +static int check_extent_data_ref(struct extent_buffer *leaf, + struct btrfs_key *key, int slot) +{ + struct btrfs_extent_data_ref *dref; + unsigned long ptr = btrfs_item_ptr_offset(leaf, slot); + const unsigned long end = ptr + btrfs_item_size(leaf, slot); + + if (unlikely(btrfs_item_size(leaf, slot) % sizeof(*dref) != 0)) { + generic_err(leaf, slot, + "invalid item size, have %u expect aligned to %zu for key type %u", + btrfs_item_size(leaf, slot), + sizeof(*dref), key->type); + return -EUCLEAN; + } + if (unlikely(!IS_ALIGNED(key->objectid, leaf->fs_info->sectorsize))) { + generic_err(leaf, slot, +"invalid key objectid for shared block ref, have %llu expect aligned to %u", + key->objectid, leaf->fs_info->sectorsize); + return -EUCLEAN; + } + for (; ptr < end; ptr += sizeof(*dref)) { + u64 root; + u64 objectid; + u64 offset; + + /* + * We cannot check the extent_data_ref hash due to possible + * overflow from the leaf due to hash collisions. + */ + dref = (struct btrfs_extent_data_ref *)ptr; + root = btrfs_extent_data_ref_root(leaf, dref); + objectid = btrfs_extent_data_ref_objectid(leaf, dref); + offset = btrfs_extent_data_ref_offset(leaf, dref); + if (unlikely(!is_valid_dref_root(root))) { + extent_err(leaf, slot, + "invalid extent data backref root value %llu", + root); + return -EUCLEAN; + } + if (unlikely(objectid < BTRFS_FIRST_FREE_OBJECTID || + objectid > BTRFS_LAST_FREE_OBJECTID)) { + extent_err(leaf, slot, + "invalid extent data backref objectid value %llu", + objectid); + return -EUCLEAN; + } + if (unlikely(!IS_ALIGNED(offset, leaf->fs_info->sectorsize))) { + extent_err(leaf, slot, + "invalid extent data backref offset, have %llu expect aligned to %u", + offset, leaf->fs_info->sectorsize); + return -EUCLEAN; + } + if (unlikely(btrfs_extent_data_ref_count(leaf, dref) == 0)) { + extent_err(leaf, slot, + "invalid extent data backref count, should have non-zero value"); + return -EUCLEAN; + } + } + return 0; +} + +#define inode_ref_err(eb, slot, fmt, args...) \ + inode_item_err(eb, slot, fmt, ##args) +static int check_inode_ref(struct extent_buffer *leaf, + struct btrfs_key *key, struct btrfs_key *prev_key, + int slot) +{ + struct btrfs_inode_ref *iref; + unsigned long ptr; + unsigned long end; + + if (unlikely(!check_prev_ino(leaf, key, slot, prev_key))) + return -EUCLEAN; + /* namelen can't be 0, so item_size == sizeof() is also invalid */ + if (unlikely(btrfs_item_size(leaf, slot) <= sizeof(*iref))) { + inode_ref_err(leaf, slot, + "invalid item size, have %u expect (%zu, %u)", + btrfs_item_size(leaf, slot), + sizeof(*iref), BTRFS_LEAF_DATA_SIZE(leaf->fs_info)); + return -EUCLEAN; + } + + ptr = btrfs_item_ptr_offset(leaf, slot); + end = ptr + btrfs_item_size(leaf, slot); + while (ptr < end) { + u16 namelen; + + if (unlikely(ptr + sizeof(*iref) > end)) { + inode_ref_err(leaf, slot, + "inode ref overflow, ptr %lu end %lu inode_ref_size %zu", + ptr, end, sizeof(*iref)); + return -EUCLEAN; + } + + iref = (struct btrfs_inode_ref *)ptr; + namelen = btrfs_inode_ref_name_len(leaf, iref); + if (unlikely(namelen == 0 || namelen > BTRFS_NAME_LEN)) { + inode_ref_err(leaf, slot, + "invalid inode ref name length, has %u expect [1, %u]", + namelen, BTRFS_NAME_LEN); + return -EUCLEAN; + } + if (unlikely(ptr + sizeof(*iref) + namelen > end)) { + inode_ref_err(leaf, slot, + "inode ref overflow, ptr %lu end %lu namelen %u", + ptr, end, namelen); + return -EUCLEAN; + } + + /* + * NOTE: In theory we should record all found index numbers + * to find any duplicated indexes, but that will be too time + * consuming for inodes with too many hard links. + */ + ptr += sizeof(*iref) + namelen; + } + return 0; +} + +static int check_inode_extref(struct extent_buffer *leaf, + struct btrfs_key *key, struct btrfs_key *prev_key, + int slot) +{ + unsigned long ptr = btrfs_item_ptr_offset(leaf, slot); + unsigned long end = ptr + btrfs_item_size(leaf, slot); + + if (unlikely(!check_prev_ino(leaf, key, slot, prev_key))) + return -EUCLEAN; + + while (ptr < end) { + struct btrfs_inode_extref *extref = (struct btrfs_inode_extref *)ptr; + u16 namelen; + + if (unlikely(ptr + sizeof(*extref) > end)) { + inode_ref_err(leaf, slot, + "inode extref overflow, ptr %lu end %lu inode_extref size %zu", + ptr, end, sizeof(*extref)); + return -EUCLEAN; + } + + namelen = btrfs_inode_extref_name_len(leaf, extref); + if (unlikely(ptr + sizeof(*extref) + namelen > end)) { + inode_ref_err(leaf, slot, + "inode extref overflow, ptr %lu end %lu namelen %u", + ptr, end, namelen); + return -EUCLEAN; + } + ptr += sizeof(*extref) + namelen; + } + return 0; +} + +static int check_raid_stripe_extent(const struct extent_buffer *leaf, + const struct btrfs_key *key, int slot) +{ + if (unlikely(!IS_ALIGNED(key->objectid, leaf->fs_info->sectorsize))) { + generic_err(leaf, slot, +"invalid key objectid for raid stripe extent, have %llu expect aligned to %u", + key->objectid, leaf->fs_info->sectorsize); + return -EUCLEAN; + } + + if (unlikely(!btrfs_fs_incompat(leaf->fs_info, RAID_STRIPE_TREE))) { + generic_err(leaf, slot, + "RAID_STRIPE_EXTENT present but RAID_STRIPE_TREE incompat bit unset"); + return -EUCLEAN; + } + + return 0; +} + +static int check_remap_key(const struct extent_buffer *leaf, + const struct btrfs_key *key, int slot) +{ + const u32 item_size = btrfs_item_size(leaf, slot); + const u32 sectorsize = leaf->fs_info->sectorsize; + u64 end; + + if (unlikely(!btrfs_fs_incompat(leaf->fs_info, REMAP_TREE))) { + generic_err(leaf, slot, + "remap key type %u present but REMAP_TREE incompat bit unset", + key->type); + return -EUCLEAN; + } + + switch (key->type) { + case BTRFS_IDENTITY_REMAP_KEY: + if (unlikely(item_size != 0)) { + generic_err(leaf, slot, + "invalid item size for IDENTITY_REMAP, have %u expect 0", + item_size); + return -EUCLEAN; + } + break; + case BTRFS_REMAP_KEY: + case BTRFS_REMAP_BACKREF_KEY: + if (unlikely(item_size != sizeof(struct btrfs_remap_item))) { + generic_err(leaf, slot, + "invalid item size for remap key type %u, have %u expect %zu", + key->type, item_size, + sizeof(struct btrfs_remap_item)); + return -EUCLEAN; + } + break; + } + + if (unlikely(key->offset == 0)) { + generic_err(leaf, slot, + "invalid remap key length, have 0 expect nonzero"); + return -EUCLEAN; + } + + if (unlikely(!IS_ALIGNED(key->objectid, sectorsize))) { + generic_err(leaf, slot, + "invalid remap key objectid, have %llu expect aligned to %u", + key->objectid, sectorsize); + return -EUCLEAN; + } + + if (unlikely(!IS_ALIGNED(key->offset, sectorsize))) { + generic_err(leaf, slot, + "invalid remap key offset (length), have %llu expect aligned to %u", + key->offset, sectorsize); + return -EUCLEAN; + } + + if (unlikely(check_add_overflow(key->objectid, key->offset, &end))) { + generic_err(leaf, slot, + "remap key overflow, objectid %llu + offset %llu wraps", + key->objectid, key->offset); + return -EUCLEAN; + } + + return 0; +} + +static int check_dev_extent_item(const struct extent_buffer *leaf, + const struct btrfs_key *key, + int slot, + struct btrfs_key *prev_key) +{ + struct btrfs_dev_extent *de; + const u32 sectorsize = leaf->fs_info->sectorsize; + + de = btrfs_item_ptr(leaf, slot, struct btrfs_dev_extent); + /* Basic fixed member checks. */ + if (unlikely(btrfs_dev_extent_chunk_tree(leaf, de) != + BTRFS_CHUNK_TREE_OBJECTID)) { + generic_err(leaf, slot, + "invalid dev extent chunk tree id, has %llu expect %llu", + btrfs_dev_extent_chunk_tree(leaf, de), + BTRFS_CHUNK_TREE_OBJECTID); + return -EUCLEAN; + } + if (unlikely(btrfs_dev_extent_chunk_objectid(leaf, de) != + BTRFS_FIRST_CHUNK_TREE_OBJECTID)) { + generic_err(leaf, slot, + "invalid dev extent chunk objectid, has %llu expect %llu", + btrfs_dev_extent_chunk_objectid(leaf, de), + BTRFS_FIRST_CHUNK_TREE_OBJECTID); + return -EUCLEAN; + } + /* Alignment check. */ + if (unlikely(!IS_ALIGNED(key->offset, sectorsize))) { + generic_err(leaf, slot, + "invalid dev extent key.offset, has %llu not aligned to %u", + key->offset, sectorsize); + return -EUCLEAN; + } + if (unlikely(!IS_ALIGNED(btrfs_dev_extent_chunk_offset(leaf, de), + sectorsize))) { + generic_err(leaf, slot, + "invalid dev extent chunk offset, has %llu not aligned to %u", + btrfs_dev_extent_chunk_objectid(leaf, de), + sectorsize); + return -EUCLEAN; + } + if (unlikely(!IS_ALIGNED(btrfs_dev_extent_length(leaf, de), + sectorsize))) { + generic_err(leaf, slot, + "invalid dev extent length, has %llu not aligned to %u", + btrfs_dev_extent_length(leaf, de), sectorsize); + return -EUCLEAN; + } + /* Overlap check with previous dev extent. */ + if (slot && prev_key->objectid == key->objectid && + prev_key->type == key->type) { + struct btrfs_dev_extent *prev_de; + u64 prev_len; + + prev_de = btrfs_item_ptr(leaf, slot - 1, struct btrfs_dev_extent); + prev_len = btrfs_dev_extent_length(leaf, prev_de); + if (unlikely(prev_key->offset + prev_len > key->offset)) { + generic_err(leaf, slot, + "dev extent overlap, prev offset %llu len %llu current offset %llu", + prev_key->offset, prev_len, key->offset); + return -EUCLEAN; + } + } + return 0; +} + +static int check_free_space_info(struct extent_buffer *leaf, struct btrfs_key *key, + int slot) +{ + struct btrfs_fs_info *fs_info = leaf->fs_info; + struct btrfs_free_space_info *fsi; + const u32 blocksize = fs_info->sectorsize; + u64 end; + u32 flags; + + if (unlikely(!IS_ALIGNED(key->objectid, blocksize))) { + generic_err(leaf, slot, + "free space info key objectid is not aligned to %u, has " BTRFS_KEY_FMT, + blocksize, BTRFS_KEY_FMT_VALUE(key)); + return -EUCLEAN; + } + if (unlikely(!IS_ALIGNED(key->offset, blocksize))) { + generic_err(leaf, slot, + "free space info key offset is not aligned to %u, has " BTRFS_KEY_FMT, + blocksize, BTRFS_KEY_FMT_VALUE(key)); + return -EUCLEAN; + } + if (unlikely(check_add_overflow(key->objectid, key->offset, &end))) { + generic_err(leaf, slot, + "free space info key overflows, has " BTRFS_KEY_FMT, + BTRFS_KEY_FMT_VALUE(key)); + return -EUCLEAN; + } + if (unlikely(btrfs_item_size(leaf, slot) != + sizeof(struct btrfs_free_space_info))) { + generic_err(leaf, slot, + "invalid item size for free space info, has %u expect %zu", + btrfs_item_size(leaf, slot), + sizeof(struct btrfs_free_space_info)); + return -EUCLEAN; + } + fsi = btrfs_item_ptr(leaf, slot, struct btrfs_free_space_info); + flags = btrfs_free_space_flags(leaf, fsi); + if (unlikely(flags & ~BTRFS_FREE_SPACE_FLAGS_MASK)) { + generic_err(leaf, slot, + "unknown flags for free space info, has 0x%x valid mask 0x%lx", + flags, BTRFS_FREE_SPACE_FLAGS_MASK); + return -EUCLEAN; + } + if (unlikely(btrfs_free_space_extent_count(leaf, fsi) > + key->offset >> fs_info->sectorsize_bits)) { + generic_err(leaf, slot, + "suspicious extent count, has %u max valid %llu", + btrfs_free_space_extent_count(leaf, fsi), + key->offset >> fs_info->sectorsize_bits); + return -EUCLEAN; + } + return 0; +} + +static int check_free_space_common_key(struct extent_buffer *leaf, struct btrfs_key *key, int slot, + struct btrfs_key *prev_key) +{ + struct btrfs_fs_info *fs_info = leaf->fs_info; + const u32 blocksize = fs_info->sectorsize; + const char *type_str = (key->type == BTRFS_FREE_SPACE_EXTENT_KEY) ? "extent" : "bitmap"; + u64 end; + + if (unlikely(!IS_ALIGNED(key->objectid, blocksize))) { + generic_err(leaf, slot, + "free space %s key objectid is not aligned to %u, has " BTRFS_KEY_FMT, + type_str, blocksize, BTRFS_KEY_FMT_VALUE(key)); + return -EUCLEAN; + } + if (unlikely(!IS_ALIGNED(key->offset, blocksize))) { + generic_err(leaf, slot, + "free space %s key offset is not aligned to %u, has " BTRFS_KEY_FMT, + type_str, blocksize, BTRFS_KEY_FMT_VALUE(key)); + return -EUCLEAN; + } + if (unlikely(key->offset == 0)) { + generic_err(leaf, slot, "free space %s length is 0", type_str); + return -EUCLEAN; + } + if (unlikely(check_add_overflow(key->objectid, key->offset, &end))) { + generic_err(leaf, slot, + "free space %s end overflow, have objectid %llu offset %llu", + type_str, key->objectid, key->offset); + return -EUCLEAN; + } + if (slot == 0) + return 0; + + /* + * Make sure the current key is inside the block group, and matching + * the expected info type. + */ + if (prev_key->type == BTRFS_FREE_SPACE_INFO_KEY) { + struct btrfs_free_space_info *fsi; + u32 info_flags; + + if (unlikely(key->objectid < prev_key->objectid || + key->objectid + key->offset > prev_key->objectid + prev_key->offset)) { + generic_err(leaf, slot, +"free space %s is not inside the space info, prev key " BTRFS_KEY_FMT " current key " BTRFS_KEY_FMT, + type_str, BTRFS_KEY_FMT_VALUE(prev_key), + BTRFS_KEY_FMT_VALUE(key)); + return -EUCLEAN; + } + fsi = btrfs_item_ptr(leaf, slot - 1, struct btrfs_free_space_info); + info_flags = btrfs_free_space_flags(leaf, fsi); + if (unlikely((info_flags == BTRFS_FREE_SPACE_USING_BITMAPS && + key->type == BTRFS_FREE_SPACE_EXTENT_KEY) || + (info_flags != BTRFS_FREE_SPACE_USING_BITMAPS && + key->type == BTRFS_FREE_SPACE_BITMAP_KEY))) { + generic_err(leaf, slot, +"free space %s key type is not matching the type of space info, key type %u space info flags %u", + type_str, key->type, info_flags); + return -EUCLEAN; + } + return 0; + } + /* + * Previous key should be either FREE_SPACE_EXTENT or FREE_SPACE_BITMAP. + * Inside the same block group the key type should match each other, and + * no overlaps. + */ + if (unlikely(key->type != prev_key->type)) { + generic_err(leaf, slot, +"free space %s key type is not matching the type of previous key, key type %u prev key type %u", + type_str, key->type, prev_key->type); + return -EUCLEAN; + } + if (unlikely(prev_key->objectid + prev_key->offset > key->objectid)) { + generic_err(leaf, slot, +"free space %s key overlaps previous key, prev key " BTRFS_KEY_FMT " current key " BTRFS_KEY_FMT, + type_str, BTRFS_KEY_FMT_VALUE(prev_key), + BTRFS_KEY_FMT_VALUE(key)); + return -EUCLEAN; + } + return 0; +} + +static int check_free_space_extent(struct extent_buffer *leaf, struct btrfs_key *key, int slot, + struct btrfs_key *prev_key) +{ + int ret; + + ret = check_free_space_common_key(leaf, key, slot, prev_key); + if (unlikely(ret < 0)) + return ret; + + if (unlikely(btrfs_item_size(leaf, slot) != 0)) { + generic_err(leaf, slot, + "invalid item size for free space info, has %u expect 0", + btrfs_item_size(leaf, slot)); + return -EUCLEAN; + } + return 0; +} + +static int check_free_space_bitmap(struct extent_buffer *leaf, + struct btrfs_key *key, int slot, + struct btrfs_key *prev_key) +{ + struct btrfs_fs_info *fs_info = leaf->fs_info; + u32 expected_item_size; + int ret; + + ret = check_free_space_common_key(leaf, key, slot, prev_key); + if (unlikely(ret < 0)) + return ret; + + /* + * The item must hold exactly the right number of bitmap bytes for the + * range described by key->offset. A mismatch means the item was + * truncated or the key is corrupt; either way the bitmap data is not + * safe to access. + */ + expected_item_size = DIV_ROUND_UP(key->offset >> fs_info->sectorsize_bits, + BITS_PER_BYTE); + if (unlikely(btrfs_item_size(leaf, slot) != expected_item_size)) { + generic_err(leaf, slot, + "invalid item size for free space bitmap, has %u expect %u", + btrfs_item_size(leaf, slot), expected_item_size); + return -EUCLEAN; + } + return 0; +} + +/* + * Common point to switch the item-specific validation. + */ +static enum btrfs_tree_block_status check_leaf_item(struct extent_buffer *leaf, + struct btrfs_key *key, + int slot, + struct btrfs_key *prev_key) +{ + int ret = 0; + struct btrfs_chunk *chunk; + + switch (key->type) { + case BTRFS_EXTENT_DATA_KEY: + ret = check_extent_data_item(leaf, key, slot, prev_key); + break; + case BTRFS_EXTENT_CSUM_KEY: + ret = check_csum_item(leaf, key, slot, prev_key); + break; + case BTRFS_DIR_ITEM_KEY: + case BTRFS_DIR_INDEX_KEY: + case BTRFS_XATTR_ITEM_KEY: + ret = check_dir_item(leaf, key, prev_key, slot); + break; + case BTRFS_INODE_REF_KEY: + ret = check_inode_ref(leaf, key, prev_key, slot); + break; + case BTRFS_INODE_EXTREF_KEY: + ret = check_inode_extref(leaf, key, prev_key, slot); + break; + case BTRFS_BLOCK_GROUP_ITEM_KEY: + ret = check_block_group_item(leaf, key, slot); + break; + case BTRFS_CHUNK_ITEM_KEY: + chunk = btrfs_item_ptr(leaf, slot, struct btrfs_chunk); + ret = check_leaf_chunk_item(leaf, chunk, key, slot); + break; + case BTRFS_DEV_ITEM_KEY: + ret = check_dev_item(leaf, key, slot); + break; + case BTRFS_DEV_EXTENT_KEY: + ret = check_dev_extent_item(leaf, key, slot, prev_key); + break; + case BTRFS_INODE_ITEM_KEY: + ret = check_inode_item(leaf, key, slot); + break; + case BTRFS_ROOT_ITEM_KEY: + ret = check_root_item(leaf, key, slot); + break; + case BTRFS_ROOT_REF_KEY: + case BTRFS_ROOT_BACKREF_KEY: + ret = check_root_ref(leaf, key, slot); + break; + case BTRFS_EXTENT_ITEM_KEY: + case BTRFS_METADATA_ITEM_KEY: + ret = check_extent_item(leaf, key, slot, prev_key); + break; + case BTRFS_TREE_BLOCK_REF_KEY: + case BTRFS_SHARED_DATA_REF_KEY: + case BTRFS_SHARED_BLOCK_REF_KEY: + ret = check_simple_keyed_refs(leaf, key, slot); + break; + case BTRFS_EXTENT_DATA_REF_KEY: + ret = check_extent_data_ref(leaf, key, slot); + break; + case BTRFS_RAID_STRIPE_KEY: + ret = check_raid_stripe_extent(leaf, key, slot); + break; + case BTRFS_FREE_SPACE_INFO_KEY: + ret = check_free_space_info(leaf, key, slot); + break; + case BTRFS_FREE_SPACE_EXTENT_KEY: + ret = check_free_space_extent(leaf, key, slot, prev_key); + break; + case BTRFS_FREE_SPACE_BITMAP_KEY: + ret = check_free_space_bitmap(leaf, key, slot, prev_key); + break; + case BTRFS_IDENTITY_REMAP_KEY: + case BTRFS_REMAP_KEY: + case BTRFS_REMAP_BACKREF_KEY: + ret = check_remap_key(leaf, key, slot); + break; + } + + if (unlikely(ret)) + return BTRFS_TREE_BLOCK_INVALID_ITEM; + return BTRFS_TREE_BLOCK_CLEAN; +} + +enum btrfs_tree_block_status __btrfs_check_leaf(struct extent_buffer *leaf) +{ + struct btrfs_fs_info *fs_info = leaf->fs_info; + /* No valid key type is 0, so all key should be larger than this key */ + struct btrfs_key prev_key = {0, 0, 0}; + struct btrfs_key key; + u32 nritems = btrfs_header_nritems(leaf); + int slot; + + if (unlikely(btrfs_header_level(leaf) != 0)) { + generic_err(leaf, 0, + "invalid level for leaf, have %d expect 0", + btrfs_header_level(leaf)); + return BTRFS_TREE_BLOCK_INVALID_LEVEL; + } + + if (unlikely(!btrfs_header_flag(leaf, BTRFS_HEADER_FLAG_WRITTEN))) { + generic_err(leaf, 0, "invalid flag for leaf, WRITTEN not set"); + return BTRFS_TREE_BLOCK_WRITTEN_NOT_SET; + } + + /* + * Extent buffers from a relocation tree have a owner field that + * corresponds to the subvolume tree they are based on. So just from an + * extent buffer alone we can not find out what is the id of the + * corresponding subvolume tree, so we can not figure out if the extent + * buffer corresponds to the root of the relocation tree or not. So + * skip this check for relocation trees. + */ + if (nritems == 0 && !btrfs_header_flag(leaf, BTRFS_HEADER_FLAG_RELOC)) { + u64 owner = btrfs_header_owner(leaf); + + /* These trees must never be empty */ + if (unlikely(owner == BTRFS_ROOT_TREE_OBJECTID || + owner == BTRFS_CHUNK_TREE_OBJECTID || + owner == BTRFS_DEV_TREE_OBJECTID || + owner == BTRFS_FS_TREE_OBJECTID || + owner == BTRFS_DATA_RELOC_TREE_OBJECTID)) { + generic_err(leaf, 0, + "invalid root, root %llu must never be empty", + owner); + return BTRFS_TREE_BLOCK_INVALID_NRITEMS; + } + + /* Unknown tree */ + if (unlikely(owner == 0)) { + generic_err(leaf, 0, + "invalid owner, root 0 is not defined"); + return BTRFS_TREE_BLOCK_INVALID_OWNER; + } + + /* EXTENT_TREE_V2 can have empty extent trees. */ + if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) + return BTRFS_TREE_BLOCK_CLEAN; + + if (unlikely(owner == BTRFS_EXTENT_TREE_OBJECTID)) { + generic_err(leaf, 0, + "invalid root, root %llu must never be empty", + owner); + return BTRFS_TREE_BLOCK_INVALID_NRITEMS; + } + + return BTRFS_TREE_BLOCK_CLEAN; + } + + if (unlikely(nritems == 0)) + return BTRFS_TREE_BLOCK_CLEAN; + + /* + * Check the following things to make sure this is a good leaf, and + * leaf users won't need to bother with similar sanity checks: + * + * 1) key ordering + * 2) item offset and size + * No overlap, no hole, all inside the leaf. + * 3) item content + * If possible, do comprehensive sanity check. + * NOTE: All checks must only rely on the item data itself. + */ + for (slot = 0; slot < nritems; slot++) { + u32 item_end_expected; + u64 item_data_end; + enum btrfs_tree_block_status ret; + + btrfs_item_key_to_cpu(leaf, &key, slot); + + /* Make sure the keys are in the right order */ + if (unlikely(btrfs_comp_cpu_keys(&prev_key, &key) >= 0)) { + generic_err(leaf, slot, + "bad key order, prev " BTRFS_KEY_FMT " current " BTRFS_KEY_FMT, + BTRFS_KEY_FMT_VALUE(&prev_key), + BTRFS_KEY_FMT_VALUE(&key)); + return BTRFS_TREE_BLOCK_BAD_KEY_ORDER; + } + + item_data_end = (u64)btrfs_item_offset(leaf, slot) + + btrfs_item_size(leaf, slot); + /* + * Make sure the offset and ends are right, remember that the + * item data starts at the end of the leaf and grows towards the + * front. + */ + if (slot == 0) + item_end_expected = BTRFS_LEAF_DATA_SIZE(fs_info); + else + item_end_expected = btrfs_item_offset(leaf, + slot - 1); + if (unlikely(item_data_end != item_end_expected)) { + generic_err(leaf, slot, + "unexpected item end, have %llu expect %u", + item_data_end, item_end_expected); + return BTRFS_TREE_BLOCK_INVALID_OFFSETS; + } + + /* + * Check to make sure that we don't point outside of the leaf, + * just in case all the items are consistent to each other, but + * all point outside of the leaf. + */ + if (unlikely(item_data_end > BTRFS_LEAF_DATA_SIZE(fs_info))) { + generic_err(leaf, slot, + "slot end outside of leaf, have %llu expect range [0, %u]", + item_data_end, BTRFS_LEAF_DATA_SIZE(fs_info)); + return BTRFS_TREE_BLOCK_INVALID_OFFSETS; + } + + /* Also check if the item pointer overlaps with btrfs item. */ + if (unlikely(btrfs_item_ptr_offset(leaf, slot) < + btrfs_item_nr_offset(leaf, slot) + sizeof(struct btrfs_item))) { + generic_err(leaf, slot, + "slot overlaps with its data, item end %lu data start %lu", + btrfs_item_nr_offset(leaf, slot) + + sizeof(struct btrfs_item), + btrfs_item_ptr_offset(leaf, slot)); + return BTRFS_TREE_BLOCK_INVALID_OFFSETS; + } + + /* Check if the item size and content meet other criteria. */ + ret = check_leaf_item(leaf, &key, slot, &prev_key); + if (unlikely(ret != BTRFS_TREE_BLOCK_CLEAN)) + return ret; + + prev_key.objectid = key.objectid; + prev_key.type = key.type; + prev_key.offset = key.offset; + } + + return BTRFS_TREE_BLOCK_CLEAN; +} + +int btrfs_check_leaf(struct extent_buffer *leaf) +{ + enum btrfs_tree_block_status ret; + + ret = __btrfs_check_leaf(leaf); + if (unlikely(ret != BTRFS_TREE_BLOCK_CLEAN)) + return -EUCLEAN; + return 0; +} +ALLOW_ERROR_INJECTION(btrfs_check_leaf, ERRNO); + +enum btrfs_tree_block_status __btrfs_check_node(struct extent_buffer *node) +{ + struct btrfs_fs_info *fs_info = node->fs_info; + unsigned long nr = btrfs_header_nritems(node); + struct btrfs_key key, next_key; + int slot; + int level = btrfs_header_level(node); + u64 bytenr; + + if (unlikely(!btrfs_header_flag(node, BTRFS_HEADER_FLAG_WRITTEN))) { + generic_err(node, 0, "invalid flag for node, WRITTEN not set"); + return BTRFS_TREE_BLOCK_WRITTEN_NOT_SET; + } + + if (unlikely(level <= 0 || level >= BTRFS_MAX_LEVEL)) { + generic_err(node, 0, + "invalid level for node, have %d expect [1, %d]", + level, BTRFS_MAX_LEVEL - 1); + return BTRFS_TREE_BLOCK_INVALID_LEVEL; + } + if (unlikely(nr == 0 || nr > BTRFS_NODEPTRS_PER_BLOCK(fs_info))) { + btrfs_crit(fs_info, +"corrupt node: root=%llu block=%llu, nritems too %s, have %lu expect range [1,%u]", + btrfs_header_owner(node), node->start, + nr == 0 ? "small" : "large", nr, + BTRFS_NODEPTRS_PER_BLOCK(fs_info)); + return BTRFS_TREE_BLOCK_INVALID_NRITEMS; + } + + for (slot = 0; slot < nr - 1; slot++) { + bytenr = btrfs_node_blockptr(node, slot); + btrfs_node_key_to_cpu(node, &key, slot); + btrfs_node_key_to_cpu(node, &next_key, slot + 1); + + if (unlikely(!bytenr)) { + generic_err(node, slot, + "invalid NULL node pointer"); + return BTRFS_TREE_BLOCK_INVALID_BLOCKPTR; + } + if (unlikely(!IS_ALIGNED(bytenr, fs_info->sectorsize))) { + generic_err(node, slot, + "unaligned pointer, have %llu should be aligned to %u", + bytenr, fs_info->sectorsize); + return BTRFS_TREE_BLOCK_INVALID_BLOCKPTR; + } + + if (unlikely(btrfs_comp_cpu_keys(&key, &next_key) >= 0)) { + generic_err(node, slot, + "bad key order, current " BTRFS_KEY_FMT " next " BTRFS_KEY_FMT, + BTRFS_KEY_FMT_VALUE(&key), + BTRFS_KEY_FMT_VALUE(&next_key)); + return BTRFS_TREE_BLOCK_BAD_KEY_ORDER; + } + } + return BTRFS_TREE_BLOCK_CLEAN; +} + +int btrfs_check_node(struct extent_buffer *node) +{ + enum btrfs_tree_block_status ret; + + ret = __btrfs_check_node(node); + if (unlikely(ret != BTRFS_TREE_BLOCK_CLEAN)) + return -EUCLEAN; + return 0; +} +ALLOW_ERROR_INJECTION(btrfs_check_node, ERRNO); + +int btrfs_check_eb_owner(const struct extent_buffer *eb, u64 root_owner) +{ + const bool is_subvol = btrfs_is_fstree(root_owner); + const u64 eb_owner = btrfs_header_owner(eb); + + /* + * Skip dummy fs, as selftests don't create unique ebs for each dummy + * root. + */ + if (btrfs_is_testing(eb->fs_info)) + return 0; + /* + * There are several call sites (backref walking, qgroup, and data + * reloc) passing 0 as @root_owner, as they are not holding the + * tree root. In that case, we can not do a reliable ownership check, + * so just exit. + */ + if (root_owner == 0) + return 0; + /* + * These trees use key.offset as their owner, our callers don't have + * the extra capacity to pass key.offset here. So we just skip them. + */ + if (root_owner == BTRFS_TREE_LOG_OBJECTID || + root_owner == BTRFS_TREE_RELOC_OBJECTID) + return 0; + + if (!is_subvol) { + /* For non-subvolume trees, the eb owner should match root owner */ + if (unlikely(root_owner != eb_owner)) { + btrfs_crit(eb->fs_info, +"corrupted %s, root=%llu block=%llu owner mismatch, have %llu expect %llu", + btrfs_header_level(eb) == 0 ? "leaf" : "node", + root_owner, btrfs_header_bytenr(eb), eb_owner, + root_owner); + return -EUCLEAN; + } + return 0; + } + + /* + * For subvolume trees, owners can mismatch, but they should all belong + * to subvolume trees. + */ + if (unlikely(is_subvol != btrfs_is_fstree(eb_owner))) { + btrfs_crit(eb->fs_info, +"corrupted %s, root=%llu block=%llu owner mismatch, have %llu expect [%llu, %llu]", + btrfs_header_level(eb) == 0 ? "leaf" : "node", + root_owner, btrfs_header_bytenr(eb), eb_owner, + BTRFS_FIRST_FREE_OBJECTID, BTRFS_LAST_FREE_OBJECTID); + return -EUCLEAN; + } + return 0; +} + +int btrfs_verify_level_key(struct extent_buffer *eb, + const struct btrfs_tree_parent_check *check) +{ + struct btrfs_fs_info *fs_info = eb->fs_info; + int found_level; + struct btrfs_key found_key; + int ret; + + found_level = btrfs_header_level(eb); + if (unlikely(found_level != check->level)) { + DEBUG_WARN(); + btrfs_err(fs_info, +"tree level mismatch detected, bytenr=%llu level expected=%u has=%u", + eb->start, check->level, found_level); + return -EUCLEAN; + } + + if (!check->has_first_key) + return 0; + + /* + * For live tree block (new tree blocks in current transaction), + * we need proper lock context to avoid race, which is impossible here. + * So we only checks tree blocks which is read from disk, whose + * generation <= fs_info->last_trans_committed. + */ + if (btrfs_header_generation(eb) > btrfs_get_last_trans_committed(fs_info)) + return 0; + + /* We have @first_key, so this @eb must have at least one item */ + if (unlikely(btrfs_header_nritems(eb) == 0)) { + btrfs_err(fs_info, + "invalid tree nritems, bytenr=%llu nritems=0 expect >0", + eb->start); + DEBUG_WARN(); + return -EUCLEAN; + } + + if (found_level) + btrfs_node_key_to_cpu(eb, &found_key, 0); + else + btrfs_item_key_to_cpu(eb, &found_key, 0); + + ret = btrfs_comp_cpu_keys(&check->first_key, &found_key); + if (unlikely(ret)) { + DEBUG_WARN(); + btrfs_err(fs_info, +"tree first key mismatch detected, bytenr=%llu parent_transid=%llu key expected=(%llu,%u,%llu) has=(%llu,%u,%llu)", + eb->start, check->transid, check->first_key.objectid, + check->first_key.type, check->first_key.offset, + found_key.objectid, found_key.type, + found_key.offset); + } + return ret; +} diff --git a/fs/btrfs/tree-checker.h b/fs/btrfs/tree-checker.h new file mode 100644 index 000000000..833e2fd98 --- /dev/null +++ b/fs/btrfs/tree-checker.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) Qu Wenruo 2017. All rights reserved. + */ + +#ifndef BTRFS_TREE_CHECKER_H +#define BTRFS_TREE_CHECKER_H + +#include +#include + +struct extent_buffer; +struct btrfs_fs_info; +struct btrfs_chunk; +struct btrfs_key; + +/* All the extra info needed to verify the parentness of a tree block. */ +struct btrfs_tree_parent_check { + /* + * The owner check against the tree block. + * + * Can be 0 to skip the owner check. + */ + u64 owner_root; + + /* + * Expected transid, can be 0 to skip the check, but such skip + * should only be utilized for backref walk related code. + */ + u64 transid; + + /* + * The expected first key. + * + * This check can be skipped if @has_first_key is false, such skip + * can happen for case where we don't have the parent node key, + * e.g. reading the tree root, doing backref walk. + */ + struct btrfs_key first_key; + bool has_first_key; + + /* The expected level. Should always be set. */ + u8 level; +}; + +enum btrfs_tree_block_status { + BTRFS_TREE_BLOCK_CLEAN, + BTRFS_TREE_BLOCK_INVALID_NRITEMS, + BTRFS_TREE_BLOCK_INVALID_PARENT_KEY, + BTRFS_TREE_BLOCK_BAD_KEY_ORDER, + BTRFS_TREE_BLOCK_INVALID_LEVEL, + BTRFS_TREE_BLOCK_INVALID_FREE_SPACE, + BTRFS_TREE_BLOCK_INVALID_OFFSETS, + BTRFS_TREE_BLOCK_INVALID_BLOCKPTR, + BTRFS_TREE_BLOCK_INVALID_ITEM, + BTRFS_TREE_BLOCK_INVALID_OWNER, + BTRFS_TREE_BLOCK_WRITTEN_NOT_SET, +}; + + +#define BTRFS_BLOCK_GROUP_VALID (BTRFS_BLOCK_GROUP_TYPE_MASK | \ + BTRFS_BLOCK_GROUP_PROFILE_MASK | \ + BTRFS_BLOCK_GROUP_REMAPPED) + +/* + * Exported simply for btrfs-progs which wants to have the + * btrfs_tree_block_status return codes. + */ +enum btrfs_tree_block_status __btrfs_check_leaf(struct extent_buffer *leaf); +enum btrfs_tree_block_status __btrfs_check_node(struct extent_buffer *node); + +int btrfs_check_leaf(struct extent_buffer *leaf); +int btrfs_check_node(struct extent_buffer *node); + +int btrfs_check_chunk_valid(const struct btrfs_fs_info *fs_info, + const struct extent_buffer *leaf, + const struct btrfs_chunk *chunk, u64 logical, + u32 sectorsize); +int btrfs_check_eb_owner(const struct extent_buffer *eb, u64 root_owner); +int btrfs_verify_level_key(struct extent_buffer *eb, + const struct btrfs_tree_parent_check *check); + +#endif diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c new file mode 100644 index 000000000..7ba7b6098 --- /dev/null +++ b/fs/btrfs/tree-log.c @@ -0,0 +1,8201 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2008 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include "misc.h" +#include "ctree.h" +#include "tree-log.h" +#include "disk-io.h" +#include "locking.h" +#include "backref.h" +#include "compression.h" +#include "qgroup.h" +#include "block-group.h" +#include "space-info.h" +#include "inode-item.h" +#include "fs.h" +#include "accessors.h" +#include "extent-tree.h" +#include "root-tree.h" +#include "dir-item.h" +#include "file-item.h" +#include "file.h" +#include "orphan.h" +#include "print-tree.h" +#include "tree-checker.h" +#include "delayed-inode.h" + +#define MAX_CONFLICT_INODES 10 + +/* + * directory trouble cases + * + * 1) on rename or unlink, if the inode being unlinked isn't in the fsync + * log, we must force a full commit before doing an fsync of the directory + * where the unlink was done. + * ---> record transid of last unlink/rename per directory + * + * mkdir foo/some_dir + * normal commit + * rename foo/some_dir foo2/some_dir + * mkdir foo/some_dir + * fsync foo/some_dir/some_file + * + * The fsync above will unlink the original some_dir without recording + * it in its new location (foo2). After a crash, some_dir will be gone + * unless the fsync of some_file forces a full commit + * + * 2) we must log any new names for any file or dir that is in the fsync + * log. ---> check inode while renaming/linking. + * + * 2a) we must log any new names for any file or dir during rename + * when the directory they are being removed from was logged. + * ---> check inode and old parent dir during rename + * + * 2a is actually the more important variant. With the extra logging + * a crash might unlink the old name without recreating the new one + * + * 3) after a crash, we must go through any directories with a link count + * of zero and redo the rm -rf + * + * mkdir f1/foo + * normal commit + * rm -rf f1/foo + * fsync(f1) + * + * The directory f1 was fully removed from the FS, but fsync was never + * called on f1, only its parent dir. After a crash the rm -rf must + * be replayed. This must be able to recurse down the entire + * directory tree. The inode link count fixup code takes care of the + * ugly details. + */ + +/* + * stages for the tree walking. The first + * stage (0) is to only pin down the blocks we find + * the second stage (1) is to make sure that all the inodes + * we find in the log are created in the subvolume. + * + * The last stage is to deal with directories and links and extents + * and all the other fun semantics + */ +enum { + LOG_WALK_PIN_ONLY, + LOG_WALK_REPLAY_INODES, + LOG_WALK_REPLAY_DIR_INDEX, + LOG_WALK_REPLAY_ALL, +}; + +/* + * The walk control struct is used to pass state down the chain when processing + * the log tree. The stage field tells us which part of the log tree processing + * we are currently doing. + */ +struct walk_control { + /* + * Signal that we are freeing the metadata extents of a log tree. + * This is used at transaction commit time while freeing a log tree. + */ + bool free; + + /* + * Signal that we are pinning the metadata extents of a log tree and the + * data extents its leaves point to (if using mixed block groups). + * This happens in the first stage of log replay to ensure that during + * replay, while we are modifying subvolume trees, we don't overwrite + * the metadata extents of log trees. + */ + bool pin; + + /* What stage of the replay code we're currently in. */ + int stage; + + /* + * Ignore any items from the inode currently being processed. Needs + * to be set every time we find a BTRFS_INODE_ITEM_KEY. + */ + bool ignore_cur_inode; + + /* + * The root we are currently replaying to. This is NULL for the replay + * stage LOG_WALK_PIN_ONLY. + */ + struct btrfs_root *root; + + /* The log tree we are currently processing (not NULL for any stage). */ + struct btrfs_root *log; + + /* The transaction handle used for replaying all log trees. */ + struct btrfs_trans_handle *trans; + + /* + * The function that gets used to process blocks we find in the tree. + * Note the extent_buffer might not be up to date when it is passed in, + * and it must be checked or read if you need the data inside it. + */ + int (*process_func)(struct extent_buffer *eb, + struct walk_control *wc, u64 gen, int level); + + /* + * The following are used only when stage is >= LOG_WALK_REPLAY_INODES + * and by the replay_one_buffer() callback. + */ + + /* The current log leaf being processed. */ + struct extent_buffer *log_leaf; + /* The key being processed of the current log leaf. */ + struct btrfs_key log_key; + /* The slot being processed of the current log leaf. */ + int log_slot; + + /* A path used for searches and modifications to subvolume trees. */ + struct btrfs_path *subvol_path; +}; + +static void do_abort_log_replay(struct walk_control *wc, const char *function, + unsigned int line, int error, const char *fmt, ...) +{ + struct btrfs_fs_info *fs_info = wc->trans->fs_info; + struct va_format vaf; + va_list args; + + /* + * Do nothing if we already aborted, to avoid dumping leaves again which + * can be verbose. Further more, only the first call is useful since it + * is where we have a problem. Note that we do not use the flag + * BTRFS_FS_STATE_TRANS_ABORTED because log replay calls functions that + * are outside of tree-log.c that can abort transactions (such as + * btrfs_add_link() for example), so if that happens we still want to + * dump all log replay specific information below. + */ + if (test_and_set_bit(BTRFS_FS_STATE_LOG_REPLAY_ABORTED, &fs_info->fs_state)) + return; + + btrfs_abort_transaction(wc->trans, error); + + if (wc->subvol_path && wc->subvol_path->nodes[0]) { + btrfs_crit(fs_info, + "subvolume (root %llu) leaf currently being processed:", + btrfs_root_id(wc->root)); + btrfs_print_leaf(wc->subvol_path->nodes[0]); + } + + if (wc->log_leaf) { + btrfs_crit(fs_info, +"log tree (for root %llu) leaf currently being processed (slot %d key " BTRFS_KEY_FMT "):", + btrfs_root_id(wc->root), wc->log_slot, + BTRFS_KEY_FMT_VALUE(&wc->log_key)); + btrfs_print_leaf(wc->log_leaf); + } + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + + btrfs_crit(fs_info, + "log replay failed in %s:%u for root %llu, stage %d, with error %d: %pV", + function, line, btrfs_root_id(wc->root), wc->stage, error, &vaf); + + va_end(args); +} + +/* + * Use this for aborting a transaction during log replay while we are down the + * call chain of replay_one_buffer(), so that we get a lot more useful + * information for debugging issues when compared to a plain call to + * btrfs_abort_transaction(). + */ +#define btrfs_abort_log_replay(wc, error, fmt, args...) \ + do_abort_log_replay((wc), __func__, __LINE__, (error), fmt, ##args) + +static int btrfs_log_inode(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + enum btrfs_log_mode log_mode, + struct btrfs_log_ctx *ctx); +static int link_to_fixup_dir(struct walk_control *wc, u64 objectid); +static noinline int replay_dir_deletes(struct walk_control *wc, + u64 dirid, bool del_all); +static bool wait_log_commit(struct btrfs_root *root, int transid); + +/* + * tree logging is a special write ahead log used to make sure that + * fsyncs and O_SYNCs can happen without doing full tree commits. + * + * Full tree commits are expensive because they require commonly + * modified blocks to be recowed, creating many dirty pages in the + * extent tree an 4x-6x higher write load than ext3. + * + * Instead of doing a tree commit on every fsync, we use the + * key ranges and transaction ids to find items for a given file or directory + * that have changed in this transaction. Those items are copied into + * a special tree (one per subvolume root), that tree is written to disk + * and then the fsync is considered complete. + * + * After a crash, items are copied out of the log-tree back into the + * subvolume tree. Any file data extents found are recorded in the extent + * allocation tree, and the log-tree freed. + * + * The log tree is read three times, once to pin down all the extents it is + * using in ram and once, once to create all the inodes logged in the tree + * and once to do all the other items. + */ + +static struct btrfs_inode *btrfs_iget_logging(u64 objectid, struct btrfs_root *root) +{ + unsigned int nofs_flag; + struct btrfs_inode *inode; + + /* Only meant to be called for subvolume roots and not for log roots. */ + ASSERT(btrfs_is_fstree(btrfs_root_id(root)), "root_id=%llu", btrfs_root_id(root)); + + /* + * We're holding a transaction handle whether we are logging or + * replaying a log tree, so we must make sure NOFS semantics apply + * because btrfs_alloc_inode() may be triggered and it uses GFP_KERNEL + * to allocate an inode, which can recurse back into the filesystem and + * attempt a transaction commit, resulting in a deadlock. + */ + nofs_flag = memalloc_nofs_save(); + inode = btrfs_iget(objectid, root); + memalloc_nofs_restore(nofs_flag); + + return inode; +} + +/* + * start a sub transaction and setup the log tree + * this increments the log tree writer count to make the people + * syncing the tree wait for us to finish + */ +static int start_log_trans(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_log_ctx *ctx) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_root *tree_root = fs_info->tree_root; + const bool zoned = btrfs_is_zoned(fs_info); + int ret = 0; + bool created = false; + + /* + * First check if the log root tree was already created. If not, create + * it before locking the root's log_mutex, just to keep lockdep happy. + */ + if (!test_bit(BTRFS_ROOT_HAS_LOG_TREE, &tree_root->state)) { + mutex_lock(&tree_root->log_mutex); + if (!fs_info->log_root_tree) { + ret = btrfs_init_log_root_tree(trans, fs_info); + if (!ret) { + set_bit(BTRFS_ROOT_HAS_LOG_TREE, &tree_root->state); + created = true; + } + } + mutex_unlock(&tree_root->log_mutex); + if (ret) + return ret; + } + + mutex_lock(&root->log_mutex); + +again: + if (root->log_root) { + if (btrfs_need_log_full_commit(trans)) { + ret = BTRFS_LOG_FORCE_COMMIT; + goto out; + } + + if (zoned && wait_log_commit(root, root->log_transid - 1)) + goto again; + } else { + /* + * This means fs_info->log_root_tree was already created + * for some other FS trees. Do the full commit not to mix + * nodes from multiple log transactions to do sequential + * writing. + */ + if (zoned && !created) { + ret = BTRFS_LOG_FORCE_COMMIT; + goto out; + } + + ret = btrfs_add_log_tree(trans, root); + if (ret) + goto out; + + set_bit(BTRFS_ROOT_HAS_LOG_TREE, &root->state); + } + + atomic_inc(&root->log_writers); + if (!ctx->logging_new_name) { + int index = root->log_transid % 2; + list_add_tail(&ctx->list, &root->log_ctxs[index]); + ctx->log_transid = root->log_transid; + } + +out: + mutex_unlock(&root->log_mutex); + return ret; +} + +/* + * returns 0 if there was a log transaction running and we were able + * to join, or returns -ENOENT if there were not transactions + * in progress + */ +static int join_running_log_trans(struct btrfs_root *root) +{ + const bool zoned = btrfs_is_zoned(root->fs_info); + int ret = -ENOENT; + + if (!test_bit(BTRFS_ROOT_HAS_LOG_TREE, &root->state)) + return ret; + + mutex_lock(&root->log_mutex); +again: + if (root->log_root) { + ret = 0; + if (zoned && wait_log_commit(root, root->log_transid - 1)) + goto again; + atomic_inc(&root->log_writers); + } + mutex_unlock(&root->log_mutex); + return ret; +} + +/* + * This either makes the current running log transaction wait + * until you call btrfs_end_log_trans() or it makes any future + * log transactions wait until you call btrfs_end_log_trans() + */ +void btrfs_pin_log_trans(struct btrfs_root *root) +{ + atomic_inc(&root->log_writers); +} + +/* + * indicate we're done making changes to the log tree + * and wake up anyone waiting to do a sync + */ +void btrfs_end_log_trans(struct btrfs_root *root) +{ + if (atomic_dec_and_test(&root->log_writers)) { + /* atomic_dec_and_test implies a barrier */ + cond_wake_up_nomb(&root->log_writer_wait); + } +} + +/* + * process_func used to pin down extents, write them or wait on them + */ +static int process_one_buffer(struct extent_buffer *eb, + struct walk_control *wc, u64 gen, int level) +{ + struct btrfs_root *log = wc->log; + struct btrfs_trans_handle *trans = wc->trans; + struct btrfs_fs_info *fs_info = log->fs_info; + int ret = 0; + + /* + * If this fs is mixed then we need to be able to process the leaves to + * pin down any logged extents, so we have to read the block. + */ + if (btrfs_fs_incompat(fs_info, MIXED_GROUPS)) { + struct btrfs_tree_parent_check check = { + .level = level, + .transid = gen + }; + + ret = btrfs_read_extent_buffer(eb, &check); + if (unlikely(ret)) { + if (trans) + btrfs_abort_transaction(trans, ret); + else + btrfs_handle_fs_error(fs_info, ret, NULL); + return ret; + } + } + + if (wc->pin) { + ASSERT(trans != NULL); + ret = btrfs_pin_extent_for_log_replay(trans, eb); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + return ret; + } + + if (btrfs_buffer_uptodate(eb, gen, NULL) && level == 0) { + ret = btrfs_exclude_logged_extents(eb); + if (ret) + btrfs_abort_transaction(trans, ret); + } + } + return ret; +} + +/* + * Item overwrite used by log replay. The given log tree leaf, slot and key + * from the walk_control structure all refer to the source data we are copying + * out. + * + * The given root is for the tree we are copying into, and path is a scratch + * path for use in this function (it should be released on entry and will be + * released on exit). + * + * If the key is already in the destination tree the existing item is + * overwritten. If the existing item isn't big enough, it is extended. + * If it is too large, it is truncated. + * + * If the key isn't in the destination yet, a new item is inserted. + */ +static int overwrite_item(struct walk_control *wc) +{ + struct btrfs_trans_handle *trans = wc->trans; + struct btrfs_root *root = wc->root; + int ret; + u32 item_size; + u64 saved_i_size = 0; + int save_old_i_size = 0; + unsigned long src_ptr; + unsigned long dst_ptr; + struct extent_buffer *dst_eb; + int dst_slot; + const bool is_inode_item = (wc->log_key.type == BTRFS_INODE_ITEM_KEY); + + /* + * This is only used during log replay, so the root is always from a + * fs/subvolume tree. In case we ever need to support a log root, then + * we'll have to clone the leaf in the path, release the path and use + * the leaf before writing into the log tree. See the comments at + * copy_items() for more details. + */ + ASSERT(btrfs_root_id(root) != BTRFS_TREE_LOG_OBJECTID); + + item_size = btrfs_item_size(wc->log_leaf, wc->log_slot); + src_ptr = btrfs_item_ptr_offset(wc->log_leaf, wc->log_slot); + + /* Look for the key in the destination tree. */ + ret = btrfs_search_slot(NULL, root, &wc->log_key, wc->subvol_path, 0, 0); + if (ret < 0) { + btrfs_abort_log_replay(wc, ret, + "failed to search subvolume tree for key " BTRFS_KEY_FMT " root %llu", + BTRFS_KEY_FMT_VALUE(&wc->log_key), + btrfs_root_id(root)); + return ret; + } + + dst_eb = wc->subvol_path->nodes[0]; + dst_slot = wc->subvol_path->slots[0]; + + if (ret == 0) { + char *src_copy; + const u32 dst_size = btrfs_item_size(dst_eb, dst_slot); + + if (dst_size != item_size) + goto insert; + + if (item_size == 0) { + btrfs_release_path(wc->subvol_path); + return 0; + } + src_copy = kmalloc(item_size, GFP_NOFS); + if (!src_copy) { + btrfs_abort_log_replay(wc, -ENOMEM, + "failed to allocate memory for log leaf item"); + return -ENOMEM; + } + + read_extent_buffer(wc->log_leaf, src_copy, src_ptr, item_size); + dst_ptr = btrfs_item_ptr_offset(dst_eb, dst_slot); + ret = memcmp_extent_buffer(dst_eb, src_copy, dst_ptr, item_size); + + kfree(src_copy); + /* + * they have the same contents, just return, this saves + * us from cowing blocks in the destination tree and doing + * extra writes that may not have been done by a previous + * sync + */ + if (ret == 0) { + btrfs_release_path(wc->subvol_path); + return 0; + } + + /* + * We need to load the old nbytes into the inode so when we + * replay the extents we've logged we get the right nbytes. + */ + if (is_inode_item) { + struct btrfs_inode_item *item; + u64 nbytes; + u32 mode; + + item = btrfs_item_ptr(dst_eb, dst_slot, + struct btrfs_inode_item); + nbytes = btrfs_inode_nbytes(dst_eb, item); + item = btrfs_item_ptr(wc->log_leaf, wc->log_slot, + struct btrfs_inode_item); + btrfs_set_inode_nbytes(wc->log_leaf, item, nbytes); + + /* + * If this is a directory we need to reset the i_size to + * 0 so that we can set it up properly when replaying + * the rest of the items in this log. + */ + mode = btrfs_inode_mode(wc->log_leaf, item); + if (S_ISDIR(mode)) + btrfs_set_inode_size(wc->log_leaf, item, 0); + } + } else if (is_inode_item) { + struct btrfs_inode_item *item; + u32 mode; + + /* + * New inode, set nbytes to 0 so that the nbytes comes out + * properly when we replay the extents. + */ + item = btrfs_item_ptr(wc->log_leaf, wc->log_slot, struct btrfs_inode_item); + btrfs_set_inode_nbytes(wc->log_leaf, item, 0); + + /* + * If this is a directory we need to reset the i_size to 0 so + * that we can set it up properly when replaying the rest of + * the items in this log. + */ + mode = btrfs_inode_mode(wc->log_leaf, item); + if (S_ISDIR(mode)) + btrfs_set_inode_size(wc->log_leaf, item, 0); + } +insert: + btrfs_release_path(wc->subvol_path); + /* try to insert the key into the destination tree */ + wc->subvol_path->skip_release_on_error = true; + ret = btrfs_insert_empty_item(trans, root, wc->subvol_path, &wc->log_key, item_size); + wc->subvol_path->skip_release_on_error = false; + + dst_eb = wc->subvol_path->nodes[0]; + dst_slot = wc->subvol_path->slots[0]; + + /* make sure any existing item is the correct size */ + if (ret == -EEXIST || ret == -EOVERFLOW) { + const u32 found_size = btrfs_item_size(dst_eb, dst_slot); + + if (found_size > item_size) + btrfs_truncate_item(trans, wc->subvol_path, item_size, 1); + else if (found_size < item_size) + btrfs_extend_item(trans, wc->subvol_path, item_size - found_size); + } else if (ret) { + btrfs_abort_log_replay(wc, ret, + "failed to insert item for key " BTRFS_KEY_FMT, + BTRFS_KEY_FMT_VALUE(&wc->log_key)); + return ret; + } + dst_ptr = btrfs_item_ptr_offset(dst_eb, dst_slot); + + /* don't overwrite an existing inode if the generation number + * was logged as zero. This is done when the tree logging code + * is just logging an inode to make sure it exists after recovery. + * + * Also, don't overwrite i_size on directories during replay. + * log replay inserts and removes directory items based on the + * state of the tree found in the subvolume, and i_size is modified + * as it goes + */ + if (is_inode_item && ret == -EEXIST) { + struct btrfs_inode_item *src_item; + struct btrfs_inode_item *dst_item; + + src_item = (struct btrfs_inode_item *)src_ptr; + dst_item = (struct btrfs_inode_item *)dst_ptr; + + if (btrfs_inode_generation(wc->log_leaf, src_item) == 0) { + const u64 ino_size = btrfs_inode_size(wc->log_leaf, src_item); + + /* + * For regular files an ino_size == 0 is used only when + * logging that an inode exists, as part of a directory + * fsync, and the inode wasn't fsynced before. In this + * case don't set the size of the inode in the fs/subvol + * tree, otherwise we would be throwing valid data away. + */ + if (S_ISREG(btrfs_inode_mode(wc->log_leaf, src_item)) && + S_ISREG(btrfs_inode_mode(dst_eb, dst_item)) && + ino_size != 0) + btrfs_set_inode_size(dst_eb, dst_item, ino_size); + goto no_copy; + } + + if (S_ISDIR(btrfs_inode_mode(wc->log_leaf, src_item)) && + S_ISDIR(btrfs_inode_mode(dst_eb, dst_item))) { + save_old_i_size = 1; + saved_i_size = btrfs_inode_size(dst_eb, dst_item); + } + } + + copy_extent_buffer(dst_eb, wc->log_leaf, dst_ptr, src_ptr, item_size); + + if (save_old_i_size) { + struct btrfs_inode_item *dst_item; + + dst_item = (struct btrfs_inode_item *)dst_ptr; + btrfs_set_inode_size(dst_eb, dst_item, saved_i_size); + } + + /* make sure the generation is filled in */ + if (is_inode_item) { + struct btrfs_inode_item *dst_item; + + dst_item = (struct btrfs_inode_item *)dst_ptr; + if (btrfs_inode_generation(dst_eb, dst_item) == 0) + btrfs_set_inode_generation(dst_eb, dst_item, trans->transid); + } +no_copy: + btrfs_release_path(wc->subvol_path); + return 0; +} + +static int read_alloc_one_name(struct extent_buffer *eb, void *start, int len, + struct fscrypt_str *name) +{ + char *buf; + + buf = kmalloc(len, GFP_NOFS); + if (!buf) + return -ENOMEM; + + read_extent_buffer(eb, buf, (unsigned long)start, len); + name->name = buf; + name->len = len; + return 0; +} + +/* replays a single extent in 'eb' at 'slot' with 'key' into the + * subvolume 'root'. path is released on entry and should be released + * on exit. + * + * extents in the log tree have not been allocated out of the extent + * tree yet. So, this completes the allocation, taking a reference + * as required if the extent already exists or creating a new extent + * if it isn't in the extent allocation tree yet. + * + * The extent is inserted into the file, dropping any existing extents + * from the file that overlap the new one. + */ +static noinline int replay_one_extent(struct walk_control *wc) +{ + struct btrfs_trans_handle *trans = wc->trans; + struct btrfs_root *root = wc->root; + struct btrfs_drop_extents_args drop_args = { 0 }; + struct btrfs_fs_info *fs_info = root->fs_info; + int found_type; + u64 extent_end; + const u64 start = wc->log_key.offset; + u64 nbytes = 0; + u64 csum_start; + u64 csum_end; + LIST_HEAD(ordered_sums); + u64 offset; + unsigned long dest_offset; + struct btrfs_key ins; + struct btrfs_file_extent_item *item; + struct btrfs_inode *inode = NULL; + int ret = 0; + + item = btrfs_item_ptr(wc->log_leaf, wc->log_slot, struct btrfs_file_extent_item); + found_type = btrfs_file_extent_type(wc->log_leaf, item); + + if (found_type == BTRFS_FILE_EXTENT_REG || + found_type == BTRFS_FILE_EXTENT_PREALLOC) { + extent_end = start + btrfs_file_extent_num_bytes(wc->log_leaf, item); + /* Holes don't take up space. */ + if (btrfs_file_extent_disk_bytenr(wc->log_leaf, item) != 0) + nbytes = btrfs_file_extent_num_bytes(wc->log_leaf, item); + } else if (found_type == BTRFS_FILE_EXTENT_INLINE) { + nbytes = btrfs_file_extent_ram_bytes(wc->log_leaf, item); + extent_end = ALIGN(start + nbytes, fs_info->sectorsize); + } else { + btrfs_abort_log_replay(wc, -EUCLEAN, + "unexpected extent type=%d root=%llu inode=%llu offset=%llu", + found_type, btrfs_root_id(root), + wc->log_key.objectid, wc->log_key.offset); + return -EUCLEAN; + } + + inode = btrfs_iget_logging(wc->log_key.objectid, root); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + btrfs_abort_log_replay(wc, ret, + "failed to get inode %llu for root %llu", + wc->log_key.objectid, btrfs_root_id(root)); + return ret; + } + + /* + * first check to see if we already have this extent in the + * file. This must be done before the btrfs_drop_extents run + * so we don't try to drop this extent. + */ + ret = btrfs_lookup_file_extent(trans, root, wc->subvol_path, + btrfs_ino(inode), start, 0); + + if (ret == 0 && + (found_type == BTRFS_FILE_EXTENT_REG || + found_type == BTRFS_FILE_EXTENT_PREALLOC)) { + struct extent_buffer *leaf = wc->subvol_path->nodes[0]; + struct btrfs_file_extent_item existing; + unsigned long ptr; + + ptr = btrfs_item_ptr_offset(leaf, wc->subvol_path->slots[0]); + read_extent_buffer(leaf, &existing, ptr, sizeof(existing)); + + /* + * we already have a pointer to this exact extent, + * we don't have to do anything + */ + if (memcmp_extent_buffer(wc->log_leaf, &existing, (unsigned long)item, + sizeof(existing)) == 0) { + btrfs_release_path(wc->subvol_path); + goto out; + } + } + btrfs_release_path(wc->subvol_path); + + /* drop any overlapping extents */ + drop_args.start = start; + drop_args.end = extent_end; + drop_args.drop_cache = true; + drop_args.path = wc->subvol_path; + ret = btrfs_drop_extents(trans, root, inode, &drop_args); + if (ret) { + btrfs_abort_log_replay(wc, ret, + "failed to drop extents for inode %llu range [%llu, %llu) root %llu", + wc->log_key.objectid, start, extent_end, + btrfs_root_id(root)); + goto out; + } + + if (found_type == BTRFS_FILE_EXTENT_INLINE) { + /* inline extents are easy, we just overwrite them */ + ret = overwrite_item(wc); + if (ret) + goto out; + goto update_inode; + } + + /* + * If not an inline extent, it can only be a regular or prealloc one. + * We have checked that above and returned -EUCLEAN if not. + */ + + /* A hole and NO_HOLES feature enabled, nothing else to do. */ + if (btrfs_file_extent_disk_bytenr(wc->log_leaf, item) == 0 && + btrfs_fs_incompat(fs_info, NO_HOLES)) + goto update_inode; + + ret = btrfs_insert_empty_item(trans, root, wc->subvol_path, + &wc->log_key, sizeof(*item)); + if (ret) { + btrfs_abort_log_replay(wc, ret, + "failed to insert item with key " BTRFS_KEY_FMT " root %llu", + BTRFS_KEY_FMT_VALUE(&wc->log_key), + btrfs_root_id(root)); + goto out; + } + dest_offset = btrfs_item_ptr_offset(wc->subvol_path->nodes[0], + wc->subvol_path->slots[0]); + copy_extent_buffer(wc->subvol_path->nodes[0], wc->log_leaf, dest_offset, + (unsigned long)item, sizeof(*item)); + + /* + * We have an explicit hole and NO_HOLES is not enabled. We have added + * the hole file extent item to the subvolume tree, so we don't have + * anything else to do other than update the file extent item range and + * update the inode item. + */ + if (btrfs_file_extent_disk_bytenr(wc->log_leaf, item) == 0) { + btrfs_release_path(wc->subvol_path); + goto update_inode; + } + + ins.objectid = btrfs_file_extent_disk_bytenr(wc->log_leaf, item); + ins.type = BTRFS_EXTENT_ITEM_KEY; + ins.offset = btrfs_file_extent_disk_num_bytes(wc->log_leaf, item); + offset = wc->log_key.offset - btrfs_file_extent_offset(wc->log_leaf, item); + + /* + * Manually record dirty extent, as here we did a shallow file extent + * item copy and skip normal backref update, but modifying extent tree + * all by ourselves. So need to manually record dirty extent for qgroup, + * as the owner of the file extent changed from log tree (doesn't affect + * qgroup) to fs/file tree (affects qgroup). + */ + ret = btrfs_qgroup_trace_extent(trans, ins.objectid, ins.offset); + if (ret < 0) { + btrfs_abort_log_replay(wc, ret, +"failed to trace extent for bytenr %llu disk_num_bytes %llu inode %llu root %llu", + ins.objectid, ins.offset, + wc->log_key.objectid, btrfs_root_id(root)); + goto out; + } + + /* + * Is this extent already allocated in the extent tree? + * If so, just add a reference. + */ + ret = btrfs_lookup_data_extent(fs_info, ins.objectid, ins.offset); + if (ret < 0) { + btrfs_abort_log_replay(wc, ret, +"failed to lookup data extent for bytenr %llu disk_num_bytes %llu inode %llu root %llu", + ins.objectid, ins.offset, + wc->log_key.objectid, btrfs_root_id(root)); + goto out; + } else if (ret == 0) { + struct btrfs_ref ref = { + .action = BTRFS_ADD_DELAYED_REF, + .bytenr = ins.objectid, + .num_bytes = ins.offset, + .owning_root = btrfs_root_id(root), + .ref_root = btrfs_root_id(root), + }; + + btrfs_init_data_ref(&ref, wc->log_key.objectid, offset, 0, false); + ret = btrfs_inc_extent_ref(trans, &ref); + if (ret) { + btrfs_abort_log_replay(wc, ret, +"failed to increment data extent for bytenr %llu disk_num_bytes %llu inode %llu root %llu", + ins.objectid, ins.offset, + wc->log_key.objectid, + btrfs_root_id(root)); + goto out; + } + } else { + /* Insert the extent pointer in the extent tree. */ + ret = btrfs_alloc_logged_file_extent(trans, btrfs_root_id(root), + wc->log_key.objectid, offset, &ins); + if (ret) { + btrfs_abort_log_replay(wc, ret, +"failed to allocate logged data extent for bytenr %llu disk_num_bytes %llu offset %llu inode %llu root %llu", + ins.objectid, ins.offset, offset, + wc->log_key.objectid, btrfs_root_id(root)); + goto out; + } + } + + btrfs_release_path(wc->subvol_path); + + if (btrfs_file_extent_compression(wc->log_leaf, item)) { + csum_start = ins.objectid; + csum_end = csum_start + ins.offset; + } else { + csum_start = ins.objectid + btrfs_file_extent_offset(wc->log_leaf, item); + csum_end = csum_start + btrfs_file_extent_num_bytes(wc->log_leaf, item); + } + + ret = btrfs_lookup_csums_list(root->log_root, csum_start, csum_end - 1, + &ordered_sums, false); + if (ret < 0) { + btrfs_abort_log_replay(wc, ret, + "failed to lookups csums for range [%llu, %llu) inode %llu root %llu", + csum_start, csum_end, wc->log_key.objectid, + btrfs_root_id(root)); + goto out; + } + ret = 0; + /* + * Now delete all existing cums in the csum root that cover our range. + * We do this because we can have an extent that is completely + * referenced by one file extent item and partially referenced by + * another file extent item (like after using the clone or extent_same + * ioctls). In this case if we end up doing the replay of the one that + * partially references the extent first, and we do not do the csum + * deletion below, we can get 2 csum items in the csum tree that overlap + * each other. For example, imagine our log has the two following file + * extent items: + * + * key (257 EXTENT_DATA 409600) + * extent data disk byte 12845056 nr 102400 + * extent data offset 20480 nr 20480 ram 102400 + * + * key (257 EXTENT_DATA 819200) + * extent data disk byte 12845056 nr 102400 + * extent data offset 0 nr 102400 ram 102400 + * + * Where the second one fully references the 100K extent that starts at + * disk byte 12845056, and the log tree has a single csum item that + * covers the entire range of the extent: + * + * key (EXTENT_CSUM EXTENT_CSUM 12845056) itemsize 100 + * + * After the first file extent item is replayed, the csum tree gets the + * following csum item: + * + * key (EXTENT_CSUM EXTENT_CSUM 12865536) itemsize 20 + * + * Which covers the 20K sub-range starting at offset 20K of our extent. + * Now when we replay the second file extent item, if we do not delete + * existing csum items that cover any of its blocks, we end up getting + * two csum items in our csum tree that overlap each other: + * + * key (EXTENT_CSUM EXTENT_CSUM 12845056) itemsize 100 + * key (EXTENT_CSUM EXTENT_CSUM 12865536) itemsize 20 + * + * Which is a problem, because after this anyone trying to lookup for + * the checksum of any block of our extent starting at an offset of 40K + * or higher, will end up looking at the second csum item only, which + * does not contain the checksum for any block starting at offset 40K or + * higher of our extent. + */ + while (!list_empty(&ordered_sums)) { + struct btrfs_ordered_sum *sums; + struct btrfs_root *csum_root; + + sums = list_first_entry(&ordered_sums, struct btrfs_ordered_sum, list); + csum_root = btrfs_csum_root(fs_info, sums->logical); + if (unlikely(!csum_root)) { + btrfs_err(fs_info, + "missing csum root for extent at bytenr %llu", + sums->logical); + ret = -EUCLEAN; + } + + if (!ret) { + ret = btrfs_del_csums(trans, csum_root, sums->logical, + sums->len); + if (ret) + btrfs_abort_log_replay(wc, ret, + "failed to delete csums for range [%llu, %llu) inode %llu root %llu", + sums->logical, + sums->logical + sums->len, + wc->log_key.objectid, + btrfs_root_id(root)); + } + if (!ret) { + ret = btrfs_insert_data_csums(trans, csum_root, sums); + if (ret) + btrfs_abort_log_replay(wc, ret, + "failed to add csums for range [%llu, %llu) inode %llu root %llu", + sums->logical, + sums->logical + sums->len, + wc->log_key.objectid, + btrfs_root_id(root)); + } + list_del(&sums->list); + kfree(sums); + } + if (ret) + goto out; + +update_inode: + ret = btrfs_inode_set_file_extent_range(inode, start, extent_end - start); + if (ret) { + btrfs_abort_log_replay(wc, ret, + "failed to set file extent range [%llu, %llu) inode %llu root %llu", + start, extent_end, wc->log_key.objectid, + btrfs_root_id(root)); + goto out; + } + + btrfs_update_inode_bytes(inode, nbytes, drop_args.bytes_found); + ret = btrfs_update_inode(trans, inode); + if (ret) + btrfs_abort_log_replay(wc, ret, + "failed to update inode %llu root %llu", + wc->log_key.objectid, btrfs_root_id(root)); +out: + iput(&inode->vfs_inode); + return ret; +} + +static int unlink_inode_for_log_replay(struct walk_control *wc, + struct btrfs_inode *dir, + struct btrfs_inode *inode, + const struct fscrypt_str *name) +{ + struct btrfs_trans_handle *trans = wc->trans; + int ret; + + ret = btrfs_unlink_inode(trans, dir, inode, name); + if (ret) { + btrfs_abort_log_replay(wc, ret, + "failed to unlink inode %llu parent dir %llu name %.*s root %llu", + btrfs_ino(inode), btrfs_ino(dir), name->len, + name->name, btrfs_root_id(inode->root)); + return ret; + } + /* + * Whenever we need to check if a name exists or not, we check the + * fs/subvolume tree. So after an unlink we must run delayed items, so + * that future checks for a name during log replay see that the name + * does not exists anymore. + */ + ret = btrfs_run_delayed_items(trans); + if (ret) + btrfs_abort_log_replay(wc, ret, +"failed to run delayed items current inode %llu parent dir %llu name %.*s root %llu", + btrfs_ino(inode), btrfs_ino(dir), name->len, + name->name, btrfs_root_id(inode->root)); + + return ret; +} + +/* + * when cleaning up conflicts between the directory names in the + * subvolume, directory names in the log and directory names in the + * inode back references, we may have to unlink inodes from directories. + * + * This is a helper function to do the unlink of a specific directory + * item + */ +static noinline int drop_one_dir_item(struct walk_control *wc, + struct btrfs_inode *dir, + struct btrfs_dir_item *di) +{ + struct btrfs_root *root = dir->root; + struct btrfs_inode *inode; + struct fscrypt_str name; + struct extent_buffer *leaf = wc->subvol_path->nodes[0]; + struct btrfs_key location; + int ret; + + btrfs_dir_item_key_to_cpu(leaf, di, &location); + ret = read_alloc_one_name(leaf, di + 1, btrfs_dir_name_len(leaf, di), &name); + if (ret) { + btrfs_abort_log_replay(wc, ret, + "failed to allocate name for dir %llu root %llu", + btrfs_ino(dir), btrfs_root_id(root)); + return ret; + } + + btrfs_release_path(wc->subvol_path); + + inode = btrfs_iget_logging(location.objectid, root); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + btrfs_abort_log_replay(wc, ret, + "failed to open inode %llu parent dir %llu name %.*s root %llu", + location.objectid, btrfs_ino(dir), + name.len, name.name, btrfs_root_id(root)); + inode = NULL; + goto out; + } + + ret = link_to_fixup_dir(wc, location.objectid); + if (ret) + goto out; + + ret = unlink_inode_for_log_replay(wc, dir, inode, &name); +out: + kfree(name.name); + if (inode) + iput(&inode->vfs_inode); + return ret; +} + +/* + * See if a given name and sequence number found in an inode back reference are + * already in a directory and correctly point to this inode. + * + * Returns: < 0 on error, 0 if the directory entry does not exists and 1 if it + * exists. + */ +static noinline int inode_in_dir(struct btrfs_root *root, + struct btrfs_path *path, + u64 dirid, u64 objectid, u64 index, + struct fscrypt_str *name) +{ + struct btrfs_dir_item *di; + struct btrfs_key location; + int ret = 0; + + di = btrfs_lookup_dir_index_item(NULL, root, path, dirid, + index, name, 0); + if (IS_ERR(di)) { + ret = PTR_ERR(di); + goto out; + } else if (di) { + btrfs_dir_item_key_to_cpu(path->nodes[0], di, &location); + if (location.objectid != objectid) + goto out; + } else { + goto out; + } + + btrfs_release_path(path); + di = btrfs_lookup_dir_item(NULL, root, path, dirid, name, 0); + if (IS_ERR(di)) { + ret = PTR_ERR(di); + goto out; + } else if (di) { + btrfs_dir_item_key_to_cpu(path->nodes[0], di, &location); + if (location.objectid == objectid) + ret = 1; + } +out: + btrfs_release_path(path); + return ret; +} + +/* + * helper function to check a log tree for a named back reference in + * an inode. This is used to decide if a back reference that is + * found in the subvolume conflicts with what we find in the log. + * + * inode backreferences may have multiple refs in a single item, + * during replay we process one reference at a time, and we don't + * want to delete valid links to a file from the subvolume if that + * link is also in the log. + */ +static noinline int backref_in_log(struct btrfs_root *log, + struct btrfs_key *key, + u64 ref_objectid, + const struct fscrypt_str *name) +{ + BTRFS_PATH_AUTO_FREE(path); + int ret; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + ret = btrfs_search_slot(NULL, log, key, path, 0, 0); + if (ret < 0) + return ret; + if (ret == 1) + return 0; + + if (key->type == BTRFS_INODE_EXTREF_KEY) + ret = !!btrfs_find_name_in_ext_backref(path->nodes[0], + path->slots[0], + ref_objectid, name); + else + ret = !!btrfs_find_name_in_backref(path->nodes[0], + path->slots[0], name); + return ret; +} + +static int unlink_refs_not_in_log(struct walk_control *wc, + struct btrfs_key *search_key, + struct btrfs_inode *dir, + struct btrfs_inode *inode) +{ + struct extent_buffer *leaf = wc->subvol_path->nodes[0]; + unsigned long ptr; + unsigned long ptr_end; + + /* + * Check all the names in this back reference to see if they are in the + * log. If so, we allow them to stay otherwise they must be unlinked as + * a conflict. + */ + ptr = btrfs_item_ptr_offset(leaf, wc->subvol_path->slots[0]); + ptr_end = ptr + btrfs_item_size(leaf, wc->subvol_path->slots[0]); + while (ptr < ptr_end) { + struct fscrypt_str victim_name; + struct btrfs_inode_ref *victim_ref; + int ret; + + victim_ref = (struct btrfs_inode_ref *)ptr; + ret = read_alloc_one_name(leaf, (victim_ref + 1), + btrfs_inode_ref_name_len(leaf, victim_ref), + &victim_name); + if (ret) { + btrfs_abort_log_replay(wc, ret, + "failed to allocate name for inode %llu parent dir %llu root %llu", + btrfs_ino(inode), btrfs_ino(dir), + btrfs_root_id(inode->root)); + return ret; + } + + ret = backref_in_log(wc->log, search_key, btrfs_ino(dir), &victim_name); + if (ret) { + if (ret < 0) { + btrfs_abort_log_replay(wc, ret, +"failed to check if backref is in log tree for inode %llu parent dir %llu name %.*s root %llu", + btrfs_ino(inode), btrfs_ino(dir), + victim_name.len, victim_name.name, + btrfs_root_id(inode->root)); + kfree(victim_name.name); + return ret; + } + kfree(victim_name.name); + ptr = (unsigned long)(victim_ref + 1) + victim_name.len; + continue; + } + + inc_nlink(&inode->vfs_inode); + btrfs_release_path(wc->subvol_path); + + ret = unlink_inode_for_log_replay(wc, dir, inode, &victim_name); + kfree(victim_name.name); + if (ret) + return ret; + return -EAGAIN; + } + + return 0; +} + +static int unlink_extrefs_not_in_log(struct walk_control *wc, + struct btrfs_key *search_key, + struct btrfs_inode *dir, + struct btrfs_inode *inode) +{ + struct extent_buffer *leaf = wc->subvol_path->nodes[0]; + const unsigned long base = btrfs_item_ptr_offset(leaf, wc->subvol_path->slots[0]); + const u32 item_size = btrfs_item_size(leaf, wc->subvol_path->slots[0]); + u32 cur_offset = 0; + + while (cur_offset < item_size) { + struct btrfs_root *log_root = wc->log; + struct btrfs_inode_extref *extref; + struct fscrypt_str victim_name; + int ret; + + extref = (struct btrfs_inode_extref *)(base + cur_offset); + victim_name.len = btrfs_inode_extref_name_len(leaf, extref); + + if (btrfs_inode_extref_parent(leaf, extref) != btrfs_ino(dir)) + goto next; + + ret = read_alloc_one_name(leaf, &extref->name, victim_name.len, + &victim_name); + if (ret) { + btrfs_abort_log_replay(wc, ret, + "failed to allocate name for inode %llu parent dir %llu root %llu", + btrfs_ino(inode), btrfs_ino(dir), + btrfs_root_id(inode->root)); + return ret; + } + + search_key->objectid = btrfs_ino(inode); + search_key->type = BTRFS_INODE_EXTREF_KEY; + search_key->offset = btrfs_extref_hash(btrfs_ino(dir), + victim_name.name, + victim_name.len); + ret = backref_in_log(log_root, search_key, btrfs_ino(dir), &victim_name); + if (ret) { + if (ret < 0) { + btrfs_abort_log_replay(wc, ret, +"failed to check if backref is in log tree for inode %llu parent dir %llu name %.*s root %llu", + btrfs_ino(inode), btrfs_ino(dir), + victim_name.len, victim_name.name, + btrfs_root_id(inode->root)); + kfree(victim_name.name); + return ret; + } + kfree(victim_name.name); +next: + cur_offset += victim_name.len + sizeof(*extref); + continue; + } + + inc_nlink(&inode->vfs_inode); + btrfs_release_path(wc->subvol_path); + + ret = unlink_inode_for_log_replay(wc, dir, inode, &victim_name); + kfree(victim_name.name); + if (ret) + return ret; + return -EAGAIN; + } + + return 0; +} + +static inline int __add_inode_ref(struct walk_control *wc, + struct btrfs_inode *dir, + struct btrfs_inode *inode, + u64 ref_index, struct fscrypt_str *name) +{ + int ret; + struct btrfs_trans_handle *trans = wc->trans; + struct btrfs_root *root = wc->root; + struct btrfs_dir_item *di; + struct btrfs_key search_key; + struct btrfs_inode_extref *extref; + +again: + /* Search old style refs */ + search_key.objectid = btrfs_ino(inode); + search_key.type = BTRFS_INODE_REF_KEY; + search_key.offset = btrfs_ino(dir); + ret = btrfs_search_slot(NULL, root, &search_key, wc->subvol_path, 0, 0); + if (ret < 0) { + btrfs_abort_log_replay(wc, ret, + "failed to search subvolume tree for key " BTRFS_KEY_FMT " root %llu", + BTRFS_KEY_FMT_VALUE(&search_key), + btrfs_root_id(root)); + return ret; + } else if (ret == 0) { + /* + * Are we trying to overwrite a back ref for the root directory? + * If so, we're done. + */ + if (search_key.objectid == search_key.offset) + return 1; + + ret = unlink_refs_not_in_log(wc, &search_key, dir, inode); + if (ret == -EAGAIN) + goto again; + else if (ret) + return ret; + } + btrfs_release_path(wc->subvol_path); + + /* Same search but for extended refs */ + extref = btrfs_lookup_inode_extref(root, wc->subvol_path, name, + btrfs_ino(inode), btrfs_ino(dir)); + if (IS_ERR(extref)) { + return PTR_ERR(extref); + } else if (extref) { + ret = unlink_extrefs_not_in_log(wc, &search_key, dir, inode); + if (ret == -EAGAIN) + goto again; + else if (ret) + return ret; + } + btrfs_release_path(wc->subvol_path); + + /* look for a conflicting sequence number */ + di = btrfs_lookup_dir_index_item(trans, root, wc->subvol_path, btrfs_ino(dir), + ref_index, name, 0); + if (IS_ERR(di)) { + ret = PTR_ERR(di); + btrfs_abort_log_replay(wc, ret, +"failed to lookup dir index item for dir %llu ref_index %llu name %.*s root %llu", + btrfs_ino(dir), ref_index, name->len, + name->name, btrfs_root_id(root)); + return ret; + } else if (di) { + ret = drop_one_dir_item(wc, dir, di); + if (ret) + return ret; + } + btrfs_release_path(wc->subvol_path); + + /* look for a conflicting name */ + di = btrfs_lookup_dir_item(trans, root, wc->subvol_path, btrfs_ino(dir), name, 0); + if (IS_ERR(di)) { + ret = PTR_ERR(di); + btrfs_abort_log_replay(wc, ret, + "failed to lookup dir item for dir %llu name %.*s root %llu", + btrfs_ino(dir), name->len, name->name, + btrfs_root_id(root)); + return ret; + } else if (di) { + ret = drop_one_dir_item(wc, dir, di); + if (ret) + return ret; + } + btrfs_release_path(wc->subvol_path); + + return 0; +} + +static int extref_get_fields(struct extent_buffer *eb, unsigned long ref_ptr, + struct fscrypt_str *name, u64 *index, + u64 *parent_objectid) +{ + struct btrfs_inode_extref *extref; + int ret; + + extref = (struct btrfs_inode_extref *)ref_ptr; + + ret = read_alloc_one_name(eb, &extref->name, + btrfs_inode_extref_name_len(eb, extref), name); + if (ret) + return ret; + + if (index) + *index = btrfs_inode_extref_index(eb, extref); + if (parent_objectid) + *parent_objectid = btrfs_inode_extref_parent(eb, extref); + + return 0; +} + +static int ref_get_fields(struct extent_buffer *eb, unsigned long ref_ptr, + struct fscrypt_str *name, u64 *index) +{ + struct btrfs_inode_ref *ref; + int ret; + + ref = (struct btrfs_inode_ref *)ref_ptr; + + ret = read_alloc_one_name(eb, ref + 1, btrfs_inode_ref_name_len(eb, ref), + name); + if (ret) + return ret; + + if (index) + *index = btrfs_inode_ref_index(eb, ref); + + return 0; +} + +/* + * Take an inode reference item from the log tree and iterate all names from the + * inode reference item in the subvolume tree with the same key (if it exists). + * For any name that is not in the inode reference item from the log tree, do a + * proper unlink of that name (that is, remove its entry from the inode + * reference item and both dir index keys). + */ +static int unlink_old_inode_refs(struct walk_control *wc, struct btrfs_inode *inode) +{ + struct btrfs_root *root = wc->root; + int ret; + unsigned long ref_ptr; + unsigned long ref_end; + struct extent_buffer *eb; + +again: + btrfs_release_path(wc->subvol_path); + ret = btrfs_search_slot(NULL, root, &wc->log_key, wc->subvol_path, 0, 0); + if (ret > 0) { + ret = 0; + goto out; + } + if (ret < 0) { + btrfs_abort_log_replay(wc, ret, + "failed to search subvolume tree for key " BTRFS_KEY_FMT " root %llu", + BTRFS_KEY_FMT_VALUE(&wc->log_key), + btrfs_root_id(root)); + goto out; + } + + eb = wc->subvol_path->nodes[0]; + ref_ptr = btrfs_item_ptr_offset(eb, wc->subvol_path->slots[0]); + ref_end = ref_ptr + btrfs_item_size(eb, wc->subvol_path->slots[0]); + while (ref_ptr < ref_end) { + struct fscrypt_str name; + u64 parent_id; + + if (wc->log_key.type == BTRFS_INODE_EXTREF_KEY) { + ret = extref_get_fields(eb, ref_ptr, &name, + NULL, &parent_id); + if (ret) { + btrfs_abort_log_replay(wc, ret, + "failed to get extref details for inode %llu root %llu", + btrfs_ino(inode), + btrfs_root_id(root)); + goto out; + } + } else { + parent_id = wc->log_key.offset; + ret = ref_get_fields(eb, ref_ptr, &name, NULL); + if (ret) { + btrfs_abort_log_replay(wc, ret, + "failed to get ref details for inode %llu parent_id %llu root %llu", + btrfs_ino(inode), parent_id, + btrfs_root_id(root)); + goto out; + } + } + + if (wc->log_key.type == BTRFS_INODE_EXTREF_KEY) + ret = !!btrfs_find_name_in_ext_backref(wc->log_leaf, wc->log_slot, + parent_id, &name); + else + ret = !!btrfs_find_name_in_backref(wc->log_leaf, wc->log_slot, + &name); + + if (!ret) { + struct btrfs_inode *dir; + + btrfs_release_path(wc->subvol_path); + dir = btrfs_iget_logging(parent_id, root); + if (IS_ERR(dir)) { + ret = PTR_ERR(dir); + kfree(name.name); + btrfs_abort_log_replay(wc, ret, + "failed to lookup dir inode %llu root %llu", + parent_id, btrfs_root_id(root)); + goto out; + } + ret = unlink_inode_for_log_replay(wc, dir, inode, &name); + kfree(name.name); + iput(&dir->vfs_inode); + if (ret) + goto out; + goto again; + } + + kfree(name.name); + ref_ptr += name.len; + if (wc->log_key.type == BTRFS_INODE_EXTREF_KEY) + ref_ptr += sizeof(struct btrfs_inode_extref); + else + ref_ptr += sizeof(struct btrfs_inode_ref); + } + ret = 0; + out: + btrfs_release_path(wc->subvol_path); + return ret; +} + +/* + * Replay one inode back reference item found in the log tree. + * Path is for temporary use by this function (it should be released on return). + */ +static noinline int add_inode_ref(struct walk_control *wc) +{ + struct btrfs_trans_handle *trans = wc->trans; + struct btrfs_root *root = wc->root; + struct btrfs_inode *dir = NULL; + struct btrfs_inode *inode = NULL; + unsigned long ref_ptr; + unsigned long ref_end; + struct fscrypt_str name = { 0 }; + int ret; + const bool is_extref_item = (wc->log_key.type == BTRFS_INODE_EXTREF_KEY); + u64 parent_objectid; + u64 inode_objectid; + u64 ref_index = 0; + int ref_struct_size; + + ref_ptr = btrfs_item_ptr_offset(wc->log_leaf, wc->log_slot); + ref_end = ref_ptr + btrfs_item_size(wc->log_leaf, wc->log_slot); + + if (is_extref_item) { + struct btrfs_inode_extref *r; + + ref_struct_size = sizeof(struct btrfs_inode_extref); + r = (struct btrfs_inode_extref *)ref_ptr; + parent_objectid = btrfs_inode_extref_parent(wc->log_leaf, r); + } else { + ref_struct_size = sizeof(struct btrfs_inode_ref); + parent_objectid = wc->log_key.offset; + } + inode_objectid = wc->log_key.objectid; + + /* + * it is possible that we didn't log all the parent directories + * for a given inode. If we don't find the dir, just don't + * copy the back ref in. The link count fixup code will take + * care of the rest + */ + dir = btrfs_iget_logging(parent_objectid, root); + if (IS_ERR(dir)) { + ret = PTR_ERR(dir); + if (ret == -ENOENT) + ret = 0; + else + btrfs_abort_log_replay(wc, ret, + "failed to lookup dir inode %llu root %llu", + parent_objectid, btrfs_root_id(root)); + dir = NULL; + goto out; + } + + inode = btrfs_iget_logging(inode_objectid, root); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + btrfs_abort_log_replay(wc, ret, + "failed to lookup inode %llu root %llu", + inode_objectid, btrfs_root_id(root)); + inode = NULL; + goto out; + } + + while (ref_ptr < ref_end) { + if (is_extref_item) { + ret = extref_get_fields(wc->log_leaf, ref_ptr, &name, + &ref_index, &parent_objectid); + if (ret) { + btrfs_abort_log_replay(wc, ret, + "failed to get extref details for inode %llu root %llu", + btrfs_ino(inode), + btrfs_root_id(root)); + goto out; + } + /* + * parent object can change from one array + * item to another. + */ + if (!dir) { + dir = btrfs_iget_logging(parent_objectid, root); + if (IS_ERR(dir)) { + ret = PTR_ERR(dir); + dir = NULL; + /* + * A new parent dir may have not been + * logged and not exist in the subvolume + * tree, see the comment above before + * the loop when getting the first + * parent dir. + */ + if (ret == -ENOENT) { + /* + * The next extref may refer to + * another parent dir that + * exists, so continue. + */ + ret = 0; + goto next; + } else { + btrfs_abort_log_replay(wc, ret, + "failed to lookup dir inode %llu root %llu", + parent_objectid, + btrfs_root_id(root)); + } + goto out; + } + } + } else { + ret = ref_get_fields(wc->log_leaf, ref_ptr, &name, &ref_index); + if (ret) { + btrfs_abort_log_replay(wc, ret, + "failed to get ref details for inode %llu parent_objectid %llu root %llu", + btrfs_ino(inode), + parent_objectid, + btrfs_root_id(root)); + goto out; + } + } + + ret = inode_in_dir(root, wc->subvol_path, btrfs_ino(dir), + btrfs_ino(inode), ref_index, &name); + if (ret < 0) { + btrfs_abort_log_replay(wc, ret, +"failed to check if inode %llu is in dir %llu ref_index %llu name %.*s root %llu", + btrfs_ino(inode), btrfs_ino(dir), + ref_index, name.len, name.name, + btrfs_root_id(root)); + goto out; + } else if (ret == 0) { + /* + * look for a conflicting back reference in the + * metadata. if we find one we have to unlink that name + * of the file before we add our new link. Later on, we + * overwrite any existing back reference, and we don't + * want to create dangling pointers in the directory. + */ + ret = __add_inode_ref(wc, dir, inode, ref_index, &name); + if (ret) { + if (ret == 1) + ret = 0; + goto out; + } + + /* insert our name */ + ret = btrfs_add_link(trans, dir, inode, &name, false, ref_index); + if (ret) { + btrfs_abort_log_replay(wc, ret, +"failed to add link for inode %llu in dir %llu ref_index %llu name %.*s root %llu", + btrfs_ino(inode), + btrfs_ino(dir), ref_index, + name.len, name.name, + btrfs_root_id(root)); + goto out; + } + + ret = btrfs_update_inode(trans, inode); + if (ret) { + btrfs_abort_log_replay(wc, ret, + "failed to update inode %llu root %llu", + btrfs_ino(inode), + btrfs_root_id(root)); + goto out; + } + } + /* Else, ret == 1, we already have a perfect match, we're done. */ + +next: + ref_ptr = (unsigned long)(ref_ptr + ref_struct_size) + name.len; + kfree(name.name); + name.name = NULL; + if (is_extref_item && dir) { + iput(&dir->vfs_inode); + dir = NULL; + } + } + + /* + * Before we overwrite the inode reference item in the subvolume tree + * with the item from the log tree, we must unlink all names from the + * parent directory that are in the subvolume's tree inode reference + * item, otherwise we end up with an inconsistent subvolume tree where + * dir index entries exist for a name but there is no inode reference + * item with the same name. + */ + ret = unlink_old_inode_refs(wc, inode); + if (ret) + goto out; + + /* finally write the back reference in the inode */ + ret = overwrite_item(wc); +out: + btrfs_release_path(wc->subvol_path); + kfree(name.name); + if (dir) + iput(&dir->vfs_inode); + if (inode) + iput(&inode->vfs_inode); + return ret; +} + +static int count_inode_extrefs(struct btrfs_inode *inode, struct btrfs_path *path) +{ + int ret = 0; + int name_len; + unsigned int nlink = 0; + u32 item_size; + u32 cur_offset = 0; + u64 inode_objectid = btrfs_ino(inode); + u64 offset = 0; + unsigned long ptr; + struct btrfs_inode_extref *extref; + struct extent_buffer *leaf; + + while (1) { + ret = btrfs_find_one_extref(inode->root, inode_objectid, offset, + path, &extref, &offset); + if (ret) + break; + + leaf = path->nodes[0]; + item_size = btrfs_item_size(leaf, path->slots[0]); + ptr = btrfs_item_ptr_offset(leaf, path->slots[0]); + cur_offset = 0; + + while (cur_offset < item_size) { + extref = (struct btrfs_inode_extref *) (ptr + cur_offset); + name_len = btrfs_inode_extref_name_len(leaf, extref); + + nlink++; + + cur_offset += name_len + sizeof(*extref); + } + + offset++; + btrfs_release_path(path); + } + btrfs_release_path(path); + + if (ret < 0 && ret != -ENOENT) + return ret; + return nlink; +} + +static int count_inode_refs(struct btrfs_inode *inode, struct btrfs_path *path) +{ + int ret; + struct btrfs_key key; + unsigned int nlink = 0; + unsigned long ptr; + unsigned long ptr_end; + int name_len; + u64 ino = btrfs_ino(inode); + + key.objectid = ino; + key.type = BTRFS_INODE_REF_KEY; + key.offset = (u64)-1; + + while (1) { + ret = btrfs_search_slot(NULL, inode->root, &key, path, 0, 0); + if (ret < 0) + break; + if (ret > 0) { + if (path->slots[0] == 0) + break; + path->slots[0]--; + } +process_slot: + btrfs_item_key_to_cpu(path->nodes[0], &key, + path->slots[0]); + if (key.objectid != ino || + key.type != BTRFS_INODE_REF_KEY) + break; + ptr = btrfs_item_ptr_offset(path->nodes[0], path->slots[0]); + ptr_end = ptr + btrfs_item_size(path->nodes[0], + path->slots[0]); + while (ptr < ptr_end) { + struct btrfs_inode_ref *ref; + + ref = (struct btrfs_inode_ref *)ptr; + name_len = btrfs_inode_ref_name_len(path->nodes[0], + ref); + ptr = (unsigned long)(ref + 1) + name_len; + nlink++; + } + + if (key.offset == 0) + break; + if (path->slots[0] > 0) { + path->slots[0]--; + goto process_slot; + } + key.offset--; + btrfs_release_path(path); + } + btrfs_release_path(path); + + return nlink; +} + +/* + * There are a few corners where the link count of the file can't + * be properly maintained during replay. So, instead of adding + * lots of complexity to the log code, we just scan the backrefs + * for any file that has been through replay. + * + * The scan will update the link count on the inode to reflect the + * number of back refs found. If it goes down to zero, the iput + * will free the inode. + */ +static noinline int fixup_inode_link_count(struct walk_control *wc, + struct btrfs_inode *inode) +{ + struct btrfs_trans_handle *trans = wc->trans; + struct btrfs_root *root = inode->root; + int ret; + u64 nlink = 0; + const u64 ino = btrfs_ino(inode); + + ret = count_inode_refs(inode, wc->subvol_path); + if (ret < 0) + goto out; + + nlink = ret; + + ret = count_inode_extrefs(inode, wc->subvol_path); + if (ret < 0) + goto out; + + nlink += ret; + + ret = 0; + + if (nlink != inode->vfs_inode.i_nlink) { + set_nlink(&inode->vfs_inode, nlink); + ret = btrfs_update_inode(trans, inode); + if (ret) + goto out; + } + if (S_ISDIR(inode->vfs_inode.i_mode)) + inode->index_cnt = (u64)-1; + + if (inode->vfs_inode.i_nlink == 0) { + if (S_ISDIR(inode->vfs_inode.i_mode)) { + ret = replay_dir_deletes(wc, ino, true); + if (ret) + goto out; + } + ret = btrfs_insert_orphan_item(trans, root, ino); + if (ret == -EEXIST) + ret = 0; + } + +out: + btrfs_release_path(wc->subvol_path); + return ret; +} + +static noinline int fixup_inode_link_counts(struct walk_control *wc) +{ + int ret; + struct btrfs_key key; + + key.objectid = BTRFS_TREE_LOG_FIXUP_OBJECTID; + key.type = BTRFS_ORPHAN_ITEM_KEY; + key.offset = (u64)-1; + while (1) { + struct btrfs_trans_handle *trans = wc->trans; + struct btrfs_root *root = wc->root; + struct btrfs_inode *inode; + + ret = btrfs_search_slot(trans, root, &key, wc->subvol_path, -1, 1); + if (ret < 0) + break; + + if (ret == 1) { + ret = 0; + if (wc->subvol_path->slots[0] == 0) + break; + wc->subvol_path->slots[0]--; + } + + btrfs_item_key_to_cpu(wc->subvol_path->nodes[0], &key, wc->subvol_path->slots[0]); + if (key.objectid != BTRFS_TREE_LOG_FIXUP_OBJECTID || + key.type != BTRFS_ORPHAN_ITEM_KEY) + break; + + ret = btrfs_del_item(trans, root, wc->subvol_path); + if (ret) + break; + + btrfs_release_path(wc->subvol_path); + inode = btrfs_iget_logging(key.offset, root); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + break; + } + + ret = fixup_inode_link_count(wc, inode); + iput(&inode->vfs_inode); + if (ret) + break; + + /* + * fixup on a directory may create new entries, + * make sure we always look for the highest possible + * offset + */ + key.offset = (u64)-1; + } + btrfs_release_path(wc->subvol_path); + return ret; +} + + +/* + * record a given inode in the fixup dir so we can check its link + * count when replay is done. The link count is incremented here + * so the inode won't go away until we check it + */ +static noinline int link_to_fixup_dir(struct walk_control *wc, u64 objectid) +{ + struct btrfs_trans_handle *trans = wc->trans; + struct btrfs_root *root = wc->root; + struct btrfs_key key; + int ret = 0; + struct btrfs_inode *inode; + struct inode *vfs_inode; + + inode = btrfs_iget_logging(objectid, root); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + btrfs_abort_log_replay(wc, ret, + "failed to lookup inode %llu root %llu", + objectid, btrfs_root_id(root)); + return ret; + } + + vfs_inode = &inode->vfs_inode; + key.objectid = BTRFS_TREE_LOG_FIXUP_OBJECTID; + key.type = BTRFS_ORPHAN_ITEM_KEY; + key.offset = objectid; + + ret = btrfs_insert_empty_item(trans, root, wc->subvol_path, &key, 0); + + btrfs_release_path(wc->subvol_path); + if (ret == 0) { + if (!vfs_inode->i_nlink) + set_nlink(vfs_inode, 1); + else + inc_nlink(vfs_inode); + ret = btrfs_update_inode(trans, inode); + if (ret) + btrfs_abort_log_replay(wc, ret, + "failed to update inode %llu root %llu", + objectid, btrfs_root_id(root)); + } else if (ret == -EEXIST) { + ret = 0; + } else { + btrfs_abort_log_replay(wc, ret, + "failed to insert fixup item for inode %llu root %llu", + objectid, btrfs_root_id(root)); + } + iput(vfs_inode); + + return ret; +} + +/* + * when replaying the log for a directory, we only insert names + * for inodes that actually exist. This means an fsync on a directory + * does not implicitly fsync all the new files in it + */ +static noinline int insert_one_name(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + u64 dirid, u64 index, + const struct fscrypt_str *name, + struct btrfs_key *location) +{ + struct btrfs_inode *inode; + struct btrfs_inode *dir; + int ret; + + inode = btrfs_iget_logging(location->objectid, root); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + dir = btrfs_iget_logging(dirid, root); + if (IS_ERR(dir)) { + iput(&inode->vfs_inode); + return PTR_ERR(dir); + } + + ret = btrfs_add_link(trans, dir, inode, name, true, index); + + /* FIXME, put inode into FIXUP list */ + + iput(&inode->vfs_inode); + iput(&dir->vfs_inode); + return ret; +} + +static int delete_conflicting_dir_entry(struct walk_control *wc, + struct btrfs_inode *dir, + struct btrfs_dir_item *dst_di, + const struct btrfs_key *log_key, + u8 log_flags, + bool exists) +{ + struct btrfs_key found_key; + + btrfs_dir_item_key_to_cpu(wc->subvol_path->nodes[0], dst_di, &found_key); + /* The existing dentry points to the same inode, don't delete it. */ + if (found_key.objectid == log_key->objectid && + found_key.type == log_key->type && + found_key.offset == log_key->offset && + btrfs_dir_flags(wc->subvol_path->nodes[0], dst_di) == log_flags) + return 1; + + /* + * Don't drop the conflicting directory entry if the inode for the new + * entry doesn't exist. + */ + if (!exists) + return 0; + + return drop_one_dir_item(wc, dir, dst_di); +} + +/* + * take a single entry in a log directory item and replay it into + * the subvolume. + * + * if a conflicting item exists in the subdirectory already, + * the inode it points to is unlinked and put into the link count + * fix up tree. + * + * If a name from the log points to a file or directory that does + * not exist in the FS, it is skipped. fsyncs on directories + * do not force down inodes inside that directory, just changes to the + * names or unlinks in a directory. + * + * Returns < 0 on error, 0 if the name wasn't replayed (dentry points to a + * non-existing inode) and 1 if the name was replayed. + */ +static noinline int replay_one_name(struct walk_control *wc, struct btrfs_dir_item *di) +{ + struct btrfs_trans_handle *trans = wc->trans; + struct btrfs_root *root = wc->root; + struct fscrypt_str name = { 0 }; + struct btrfs_dir_item *dir_dst_di; + struct btrfs_dir_item *index_dst_di; + bool dir_dst_matches = false; + bool index_dst_matches = false; + struct btrfs_key log_key; + struct btrfs_key search_key; + struct btrfs_inode *dir; + u8 log_flags; + bool exists; + int ret; + bool update_size = true; + bool name_added = false; + + dir = btrfs_iget_logging(wc->log_key.objectid, root); + if (IS_ERR(dir)) { + ret = PTR_ERR(dir); + btrfs_abort_log_replay(wc, ret, + "failed to lookup dir inode %llu root %llu", + wc->log_key.objectid, btrfs_root_id(root)); + return ret; + } + + ret = read_alloc_one_name(wc->log_leaf, di + 1, + btrfs_dir_name_len(wc->log_leaf, di), &name); + if (ret) { + btrfs_abort_log_replay(wc, ret, + "failed to allocate name for dir %llu root %llu", + btrfs_ino(dir), btrfs_root_id(root)); + goto out; + } + + log_flags = btrfs_dir_flags(wc->log_leaf, di); + btrfs_dir_item_key_to_cpu(wc->log_leaf, di, &log_key); + ret = btrfs_lookup_inode(trans, root, wc->subvol_path, &log_key, 0); + btrfs_release_path(wc->subvol_path); + if (ret < 0) { + btrfs_abort_log_replay(wc, ret, + "failed to lookup inode %llu root %llu", + log_key.objectid, btrfs_root_id(root)); + goto out; + } + exists = (ret == 0); + ret = 0; + + dir_dst_di = btrfs_lookup_dir_item(trans, root, wc->subvol_path, + wc->log_key.objectid, &name, 1); + if (IS_ERR(dir_dst_di)) { + ret = PTR_ERR(dir_dst_di); + btrfs_abort_log_replay(wc, ret, + "failed to lookup dir item for dir %llu name %.*s root %llu", + wc->log_key.objectid, name.len, name.name, + btrfs_root_id(root)); + goto out; + } else if (dir_dst_di) { + ret = delete_conflicting_dir_entry(wc, dir, dir_dst_di, + &log_key, log_flags, exists); + if (ret < 0) { + btrfs_abort_log_replay(wc, ret, + "failed to delete conflicting entry for dir %llu name %.*s root %llu", + btrfs_ino(dir), name.len, name.name, + btrfs_root_id(root)); + goto out; + } + dir_dst_matches = (ret == 1); + } + + btrfs_release_path(wc->subvol_path); + + index_dst_di = btrfs_lookup_dir_index_item(trans, root, wc->subvol_path, + wc->log_key.objectid, + wc->log_key.offset, &name, 1); + if (IS_ERR(index_dst_di)) { + ret = PTR_ERR(index_dst_di); + btrfs_abort_log_replay(wc, ret, + "failed to lookup dir index item for dir %llu name %.*s root %llu", + wc->log_key.objectid, name.len, name.name, + btrfs_root_id(root)); + goto out; + } else if (index_dst_di) { + ret = delete_conflicting_dir_entry(wc, dir, index_dst_di, + &log_key, log_flags, exists); + if (ret < 0) { + btrfs_abort_log_replay(wc, ret, + "failed to delete conflicting entry for dir %llu name %.*s root %llu", + btrfs_ino(dir), name.len, name.name, + btrfs_root_id(root)); + goto out; + } + index_dst_matches = (ret == 1); + } + + btrfs_release_path(wc->subvol_path); + + if (dir_dst_matches && index_dst_matches) { + ret = 0; + update_size = false; + goto out; + } + + /* + * Check if the inode reference exists in the log for the given name, + * inode and parent inode + */ + search_key.objectid = log_key.objectid; + search_key.type = BTRFS_INODE_REF_KEY; + search_key.offset = wc->log_key.objectid; + ret = backref_in_log(root->log_root, &search_key, 0, &name); + if (ret < 0) { + btrfs_abort_log_replay(wc, ret, +"failed to check if ref item is logged for inode %llu dir %llu name %.*s root %llu", + search_key.objectid, btrfs_ino(dir), + name.len, name.name, btrfs_root_id(root)); + goto out; + } else if (ret) { + /* The dentry will be added later. */ + ret = 0; + update_size = false; + goto out; + } + + search_key.objectid = log_key.objectid; + search_key.type = BTRFS_INODE_EXTREF_KEY; + search_key.offset = btrfs_extref_hash(wc->log_key.objectid, name.name, name.len); + ret = backref_in_log(root->log_root, &search_key, wc->log_key.objectid, &name); + if (ret < 0) { + btrfs_abort_log_replay(wc, ret, +"failed to check if extref item is logged for inode %llu dir %llu name %.*s root %llu", + search_key.objectid, btrfs_ino(dir), + name.len, name.name, btrfs_root_id(root)); + goto out; + } else if (ret) { + /* The dentry will be added later. */ + ret = 0; + update_size = false; + goto out; + } + ret = insert_one_name(trans, root, wc->log_key.objectid, wc->log_key.offset, + &name, &log_key); + if (ret && ret != -ENOENT && ret != -EEXIST) { + btrfs_abort_log_replay(wc, ret, + "failed to insert name %.*s for inode %llu dir %llu root %llu", + name.len, name.name, log_key.objectid, + btrfs_ino(dir), btrfs_root_id(root)); + goto out; + } + if (!ret) + name_added = true; + update_size = false; + ret = 0; + +out: + if (!ret && update_size) { + btrfs_i_size_write(dir, dir->vfs_inode.i_size + name.len * 2); + ret = btrfs_update_inode(trans, dir); + if (ret) + btrfs_abort_log_replay(wc, ret, + "failed to update dir inode %llu root %llu", + btrfs_ino(dir), btrfs_root_id(root)); + } + kfree(name.name); + iput(&dir->vfs_inode); + if (!ret && name_added) + ret = 1; + return ret; +} + +/* Replay one dir item from a BTRFS_DIR_INDEX_KEY key. */ +static noinline int replay_one_dir_item(struct walk_control *wc) +{ + int ret; + struct btrfs_dir_item *di; + + /* We only log dir index keys, which only contain a single dir item. */ + ASSERT(wc->log_key.type == BTRFS_DIR_INDEX_KEY, + "wc->log_key.type=%u", wc->log_key.type); + + di = btrfs_item_ptr(wc->log_leaf, wc->log_slot, struct btrfs_dir_item); + ret = replay_one_name(wc, di); + if (ret < 0) + return ret; + + /* + * If this entry refers to a non-directory (directories can not have a + * link count > 1) and it was added in the transaction that was not + * committed, make sure we fixup the link count of the inode the entry + * points to. Otherwise something like the following would result in a + * directory pointing to an inode with a wrong link that does not account + * for this dir entry: + * + * mkdir testdir + * touch testdir/foo + * touch testdir/bar + * sync + * + * ln testdir/bar testdir/bar_link + * ln testdir/foo testdir/foo_link + * xfs_io -c "fsync" testdir/bar + * + * + * + * mount fs, log replay happens + * + * File foo would remain with a link count of 1 when it has two entries + * pointing to it in the directory testdir. This would make it impossible + * to ever delete the parent directory has it would result in stale + * dentries that can never be deleted. + */ + if (ret == 1 && btrfs_dir_ftype(wc->log_leaf, di) != BTRFS_FT_DIR) { + struct btrfs_key di_key; + + btrfs_dir_item_key_to_cpu(wc->log_leaf, di, &di_key); + ret = link_to_fixup_dir(wc, di_key.objectid); + } + + return ret; +} + +/* + * directory replay has two parts. There are the standard directory + * items in the log copied from the subvolume, and range items + * created in the log while the subvolume was logged. + * + * The range items tell us which parts of the key space the log + * is authoritative for. During replay, if a key in the subvolume + * directory is in a logged range item, but not actually in the log + * that means it was deleted from the directory before the fsync + * and should be removed. + */ +static noinline int find_dir_range(struct btrfs_root *root, + struct btrfs_path *path, + u64 dirid, + u64 *start_ret, u64 *end_ret) +{ + struct btrfs_key key; + u64 found_end; + struct btrfs_dir_log_item *item; + int ret; + int nritems; + + if (*start_ret == (u64)-1) + return 1; + + key.objectid = dirid; + key.type = BTRFS_DIR_LOG_INDEX_KEY; + key.offset = *start_ret; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + goto out; + if (ret > 0) { + if (path->slots[0] == 0) + goto out; + path->slots[0]--; + } + if (ret != 0) + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + + if (key.type != BTRFS_DIR_LOG_INDEX_KEY || key.objectid != dirid) { + ret = 1; + goto next; + } + item = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_dir_log_item); + found_end = btrfs_dir_log_end(path->nodes[0], item); + + if (*start_ret >= key.offset && *start_ret <= found_end) { + ret = 0; + *start_ret = key.offset; + *end_ret = found_end; + goto out; + } + ret = 1; +next: + /* check the next slot in the tree to see if it is a valid item */ + nritems = btrfs_header_nritems(path->nodes[0]); + path->slots[0]++; + if (path->slots[0] >= nritems) { + ret = btrfs_next_leaf(root, path); + if (ret) + goto out; + } + + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + + if (key.type != BTRFS_DIR_LOG_INDEX_KEY || key.objectid != dirid) { + ret = 1; + goto out; + } + item = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_dir_log_item); + found_end = btrfs_dir_log_end(path->nodes[0], item); + *start_ret = key.offset; + *end_ret = found_end; + ret = 0; +out: + btrfs_release_path(path); + return ret; +} + +/* + * this looks for a given directory item in the log. If the directory + * item is not in the log, the item is removed and the inode it points + * to is unlinked + */ +static noinline int check_item_in_log(struct walk_control *wc, + struct btrfs_path *log_path, + struct btrfs_inode *dir, + struct btrfs_key *dir_key, + bool force_remove) +{ + struct btrfs_trans_handle *trans = wc->trans; + struct btrfs_root *root = dir->root; + int ret; + struct extent_buffer *eb; + int slot; + struct btrfs_dir_item *di; + struct fscrypt_str name = { 0 }; + struct btrfs_inode *inode = NULL; + struct btrfs_key location; + + /* + * Currently we only log dir index keys. Even if we replay a log created + * by an older kernel that logged both dir index and dir item keys, all + * we need to do is process the dir index keys, we (and our caller) can + * safely ignore dir item keys (key type BTRFS_DIR_ITEM_KEY). + */ + ASSERT(dir_key->type == BTRFS_DIR_INDEX_KEY, "dir_key->type=%u", dir_key->type); + + eb = wc->subvol_path->nodes[0]; + slot = wc->subvol_path->slots[0]; + di = btrfs_item_ptr(eb, slot, struct btrfs_dir_item); + ret = read_alloc_one_name(eb, di + 1, btrfs_dir_name_len(eb, di), &name); + if (ret) { + btrfs_abort_log_replay(wc, ret, + "failed to allocate name for dir %llu index %llu root %llu", + btrfs_ino(dir), dir_key->offset, + btrfs_root_id(root)); + goto out; + } + + if (!force_remove) { + struct btrfs_dir_item *log_di; + + log_di = btrfs_lookup_dir_index_item(trans, wc->log, log_path, + dir_key->objectid, + dir_key->offset, &name, 0); + if (IS_ERR(log_di)) { + ret = PTR_ERR(log_di); + btrfs_abort_log_replay(wc, ret, + "failed to lookup dir index item for dir %llu index %llu name %.*s root %llu", + btrfs_ino(dir), dir_key->offset, + name.len, name.name, + btrfs_root_id(root)); + goto out; + } else if (log_di) { + /* The dentry exists in the log, we have nothing to do. */ + ret = 0; + goto out; + } + } + + btrfs_dir_item_key_to_cpu(eb, di, &location); + btrfs_release_path(wc->subvol_path); + btrfs_release_path(log_path); + inode = btrfs_iget_logging(location.objectid, root); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + inode = NULL; + btrfs_abort_log_replay(wc, ret, + "failed to lookup inode %llu root %llu", + location.objectid, btrfs_root_id(root)); + goto out; + } + + ret = link_to_fixup_dir(wc, location.objectid); + if (ret) + goto out; + + inc_nlink(&inode->vfs_inode); + ret = unlink_inode_for_log_replay(wc, dir, inode, &name); + /* + * Unlike dir item keys, dir index keys can only have one name (entry) in + * them, as there are no key collisions since each key has a unique offset + * (an index number), so we're done. + */ +out: + btrfs_release_path(wc->subvol_path); + btrfs_release_path(log_path); + kfree(name.name); + if (inode) + iput(&inode->vfs_inode); + return ret; +} + +static int replay_xattr_deletes(struct walk_control *wc) +{ + struct btrfs_trans_handle *trans = wc->trans; + struct btrfs_root *root = wc->root; + struct btrfs_root *log = wc->log; + struct btrfs_key search_key; + BTRFS_PATH_AUTO_FREE(log_path); + const u64 ino = wc->log_key.objectid; + int nritems; + int ret; + + log_path = btrfs_alloc_path(); + if (!log_path) { + btrfs_abort_log_replay(wc, -ENOMEM, "failed to allocate path"); + return -ENOMEM; + } + + search_key.objectid = ino; + search_key.type = BTRFS_XATTR_ITEM_KEY; + search_key.offset = 0; +again: + ret = btrfs_search_slot(NULL, root, &search_key, wc->subvol_path, 0, 0); + if (ret < 0) { + btrfs_abort_log_replay(wc, ret, + "failed to search xattrs for inode %llu root %llu", + ino, btrfs_root_id(root)); + goto out; + } +process_leaf: + nritems = btrfs_header_nritems(wc->subvol_path->nodes[0]); + for (int i = wc->subvol_path->slots[0]; i < nritems; i++) { + struct btrfs_key key; + struct btrfs_dir_item *di; + struct btrfs_dir_item *log_di; + u32 total_size; + u32 cur; + + btrfs_item_key_to_cpu(wc->subvol_path->nodes[0], &key, i); + if (key.objectid != ino || key.type != BTRFS_XATTR_ITEM_KEY) { + ret = 0; + goto out; + } + + di = btrfs_item_ptr(wc->subvol_path->nodes[0], i, struct btrfs_dir_item); + total_size = btrfs_item_size(wc->subvol_path->nodes[0], i); + cur = 0; + while (cur < total_size) { + u16 name_len = btrfs_dir_name_len(wc->subvol_path->nodes[0], di); + u16 data_len = btrfs_dir_data_len(wc->subvol_path->nodes[0], di); + u32 this_len = sizeof(*di) + name_len + data_len; + char *name; + + name = kmalloc(name_len, GFP_NOFS); + if (!name) { + ret = -ENOMEM; + btrfs_abort_log_replay(wc, ret, + "failed to allocate memory for name of length %u", + name_len); + goto out; + } + read_extent_buffer(wc->subvol_path->nodes[0], name, + (unsigned long)(di + 1), name_len); + + log_di = btrfs_lookup_xattr(NULL, log, log_path, ino, + name, name_len, 0); + btrfs_release_path(log_path); + if (!log_di) { + /* Doesn't exist in log tree, so delete it. */ + btrfs_release_path(wc->subvol_path); + di = btrfs_lookup_xattr(trans, root, wc->subvol_path, ino, + name, name_len, -1); + if (IS_ERR(di)) { + ret = PTR_ERR(di); + btrfs_abort_log_replay(wc, ret, + "failed to lookup xattr with name %.*s for inode %llu root %llu", + name_len, name, ino, + btrfs_root_id(root)); + kfree(name); + goto out; + } + ASSERT(di); + ret = btrfs_delete_one_dir_name(trans, root, + wc->subvol_path, di); + if (ret) { + btrfs_abort_log_replay(wc, ret, + "failed to delete xattr with name %.*s for inode %llu root %llu", + name_len, name, ino, + btrfs_root_id(root)); + kfree(name); + goto out; + } + btrfs_release_path(wc->subvol_path); + kfree(name); + search_key = key; + goto again; + } + if (IS_ERR(log_di)) { + ret = PTR_ERR(log_di); + btrfs_abort_log_replay(wc, ret, + "failed to lookup xattr in log tree with name %.*s for inode %llu root %llu", + name_len, name, ino, + btrfs_root_id(root)); + kfree(name); + goto out; + } + kfree(name); + cur += this_len; + di = (struct btrfs_dir_item *)((char *)di + this_len); + } + } + ret = btrfs_next_leaf(root, wc->subvol_path); + if (ret > 0) + ret = 0; + else if (ret == 0) + goto process_leaf; + else + btrfs_abort_log_replay(wc, ret, + "failed to get next leaf in subvolume root %llu", + btrfs_root_id(root)); +out: + btrfs_release_path(wc->subvol_path); + return ret; +} + + +/* + * deletion replay happens before we copy any new directory items + * out of the log or out of backreferences from inodes. It + * scans the log to find ranges of keys that log is authoritative for, + * and then scans the directory to find items in those ranges that are + * not present in the log. + * + * Anything we don't find in the log is unlinked and removed from the + * directory. + */ +static noinline int replay_dir_deletes(struct walk_control *wc, + u64 dirid, bool del_all) +{ + struct btrfs_root *root = wc->root; + struct btrfs_root *log = (del_all ? NULL : wc->log); + u64 range_start; + u64 range_end; + int ret = 0; + struct btrfs_key dir_key; + struct btrfs_key found_key; + BTRFS_PATH_AUTO_FREE(log_path); + struct btrfs_inode *dir; + + dir_key.objectid = dirid; + dir_key.type = BTRFS_DIR_INDEX_KEY; + log_path = btrfs_alloc_path(); + if (!log_path) { + btrfs_abort_log_replay(wc, -ENOMEM, "failed to allocate path"); + return -ENOMEM; + } + + dir = btrfs_iget_logging(dirid, root); + /* + * It isn't an error if the inode isn't there, that can happen because + * we replay the deletes before we copy in the inode item from the log. + */ + if (IS_ERR(dir)) { + ret = PTR_ERR(dir); + if (ret == -ENOENT) + ret = 0; + else + btrfs_abort_log_replay(wc, ret, + "failed to lookup dir inode %llu root %llu", + dirid, btrfs_root_id(root)); + return ret; + } + + range_start = 0; + range_end = 0; + while (1) { + if (del_all) + range_end = (u64)-1; + else { + ret = find_dir_range(log, wc->subvol_path, dirid, + &range_start, &range_end); + if (ret < 0) { + btrfs_abort_log_replay(wc, ret, + "failed to find range for dir %llu in log tree root %llu", + dirid, btrfs_root_id(root)); + goto out; + } else if (ret > 0) { + break; + } + } + + dir_key.offset = range_start; + while (1) { + int nritems; + ret = btrfs_search_slot(NULL, root, &dir_key, + wc->subvol_path, 0, 0); + if (ret < 0) { + btrfs_abort_log_replay(wc, ret, + "failed to search root %llu for key " BTRFS_KEY_FMT, + btrfs_root_id(root), + BTRFS_KEY_FMT_VALUE(&dir_key)); + goto out; + } + + nritems = btrfs_header_nritems(wc->subvol_path->nodes[0]); + if (wc->subvol_path->slots[0] >= nritems) { + ret = btrfs_next_leaf(root, wc->subvol_path); + if (ret == 1) { + break; + } else if (ret < 0) { + btrfs_abort_log_replay(wc, ret, + "failed to get next leaf in subvolume root %llu", + btrfs_root_id(root)); + goto out; + } + } + btrfs_item_key_to_cpu(wc->subvol_path->nodes[0], &found_key, + wc->subvol_path->slots[0]); + if (found_key.objectid != dirid || + found_key.type != dir_key.type) { + ret = 0; + goto out; + } + + if (found_key.offset > range_end) + break; + + ret = check_item_in_log(wc, log_path, dir, &found_key, del_all); + if (ret) + goto out; + if (found_key.offset == (u64)-1) + break; + dir_key.offset = found_key.offset + 1; + } + btrfs_release_path(wc->subvol_path); + if (range_end == (u64)-1) + break; + range_start = range_end + 1; + } + ret = 0; +out: + btrfs_release_path(wc->subvol_path); + iput(&dir->vfs_inode); + return ret; +} + +/* + * the process_func used to replay items from the log tree. This + * gets called in two different stages. The first stage just looks + * for inodes and makes sure they are all copied into the subvolume. + * + * The second stage copies all the other item types from the log into + * the subvolume. The two stage approach is slower, but gets rid of + * lots of complexity around inodes referencing other inodes that exist + * only in the log (references come from either directory items or inode + * back refs). + */ +static int replay_one_buffer(struct extent_buffer *eb, + struct walk_control *wc, u64 gen, int level) +{ + int nritems; + struct btrfs_tree_parent_check check = { + .transid = gen, + .level = level + }; + struct btrfs_root *root = wc->root; + struct btrfs_trans_handle *trans = wc->trans; + int ret; + + if (level != 0) + return 0; + + /* + * Set to NULL since it was not yet read and in case we abort log replay + * on error, we have no valid log tree leaf to dump. + */ + wc->log_leaf = NULL; + ret = btrfs_read_extent_buffer(eb, &check); + if (ret) { + btrfs_abort_log_replay(wc, ret, + "failed to read log tree leaf %llu for root %llu", + eb->start, btrfs_root_id(root)); + return ret; + } + + ASSERT(wc->subvol_path == NULL); + wc->subvol_path = btrfs_alloc_path(); + if (!wc->subvol_path) { + btrfs_abort_log_replay(wc, -ENOMEM, "failed to allocate path"); + return -ENOMEM; + } + + wc->log_leaf = eb; + + nritems = btrfs_header_nritems(eb); + for (wc->log_slot = 0; wc->log_slot < nritems; wc->log_slot++) { + struct btrfs_inode_item *inode_item = NULL; + + btrfs_item_key_to_cpu(eb, &wc->log_key, wc->log_slot); + + if (wc->log_key.type == BTRFS_INODE_ITEM_KEY) { + inode_item = btrfs_item_ptr(eb, wc->log_slot, + struct btrfs_inode_item); + /* + * An inode with no links is either: + * + * 1) A tmpfile (O_TMPFILE) that got fsync'ed and never + * got linked before the fsync, skip it, as replaying + * it is pointless since it would be deleted later. + * We skip logging tmpfiles, but it's always possible + * we are replaying a log created with a kernel that + * used to log tmpfiles; + * + * 2) A non-tmpfile which got its last link deleted + * while holding an open fd on it and later got + * fsynced through that fd. We always log the + * parent inodes when inode->last_unlink_trans is + * set to the current transaction, so ignore all the + * inode items for this inode. We will delete the + * inode when processing the parent directory with + * replay_dir_deletes(). + */ + if (btrfs_inode_nlink(eb, inode_item) == 0) { + wc->ignore_cur_inode = true; + continue; + } else { + wc->ignore_cur_inode = false; + } + } + + /* Inode keys are done during the first stage. */ + if (wc->log_key.type == BTRFS_INODE_ITEM_KEY && + wc->stage == LOG_WALK_REPLAY_INODES) { + u32 mode; + + ret = replay_xattr_deletes(wc); + if (ret) + break; + mode = btrfs_inode_mode(eb, inode_item); + if (S_ISDIR(mode)) { + ret = replay_dir_deletes(wc, wc->log_key.objectid, false); + if (ret) + break; + } + ret = overwrite_item(wc); + if (ret) + break; + + /* + * Before replaying extents, truncate the inode to its + * size. We need to do it now and not after log replay + * because before an fsync we can have prealloc extents + * added beyond the inode's i_size. If we did it after, + * through orphan cleanup for example, we would drop + * those prealloc extents just after replaying them. + */ + if (S_ISREG(mode)) { + struct btrfs_drop_extents_args drop_args = { 0 }; + struct btrfs_inode *inode; + u64 from; + + inode = btrfs_iget_logging(wc->log_key.objectid, root); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + btrfs_abort_log_replay(wc, ret, + "failed to lookup inode %llu root %llu", + wc->log_key.objectid, + btrfs_root_id(root)); + break; + } + from = ALIGN(i_size_read(&inode->vfs_inode), + root->fs_info->sectorsize); + drop_args.start = from; + drop_args.end = (u64)-1; + drop_args.drop_cache = true; + drop_args.path = wc->subvol_path; + ret = btrfs_drop_extents(trans, root, inode, &drop_args); + if (ret) { + btrfs_abort_log_replay(wc, ret, + "failed to drop extents for inode %llu root %llu offset %llu", + btrfs_ino(inode), + btrfs_root_id(root), + from); + } else { + inode_sub_bytes(&inode->vfs_inode, + drop_args.bytes_found); + /* Update the inode's nbytes. */ + ret = btrfs_update_inode(trans, inode); + if (ret) + btrfs_abort_log_replay(wc, ret, + "failed to update inode %llu root %llu", + btrfs_ino(inode), + btrfs_root_id(root)); + } + iput(&inode->vfs_inode); + if (ret) + break; + } + + ret = link_to_fixup_dir(wc, wc->log_key.objectid); + if (ret) + break; + } + + if (wc->ignore_cur_inode) + continue; + + if (wc->log_key.type == BTRFS_DIR_INDEX_KEY && + wc->stage == LOG_WALK_REPLAY_DIR_INDEX) { + ret = replay_one_dir_item(wc); + if (ret) + break; + } + + if (wc->stage < LOG_WALK_REPLAY_ALL) + continue; + + /* these keys are simply copied */ + if (wc->log_key.type == BTRFS_XATTR_ITEM_KEY) { + ret = overwrite_item(wc); + if (ret) + break; + } else if (wc->log_key.type == BTRFS_INODE_REF_KEY || + wc->log_key.type == BTRFS_INODE_EXTREF_KEY) { + ret = add_inode_ref(wc); + if (ret) + break; + } else if (wc->log_key.type == BTRFS_EXTENT_DATA_KEY) { + ret = replay_one_extent(wc); + if (ret) + break; + } + /* + * We don't log BTRFS_DIR_ITEM_KEY keys anymore, only the + * BTRFS_DIR_INDEX_KEY items which we use to derive the + * BTRFS_DIR_ITEM_KEY items. If we are replaying a log from an + * older kernel with such keys, ignore them. + */ + } + btrfs_free_path(wc->subvol_path); + wc->subvol_path = NULL; + return ret; +} + +static int clean_log_buffer(struct btrfs_trans_handle *trans, + struct extent_buffer *eb) +{ + struct btrfs_fs_info *fs_info = eb->fs_info; + struct btrfs_block_group *bg; + + btrfs_tree_lock(eb); + btrfs_clear_buffer_dirty(trans, eb); + wait_on_extent_buffer_writeback(eb); + btrfs_tree_unlock(eb); + + if (trans) { + int ret; + + ret = btrfs_pin_reserved_extent(trans, eb); + if (ret) + btrfs_abort_transaction(trans, ret); + return ret; + } + + bg = btrfs_lookup_block_group(fs_info, eb->start); + if (!bg) { + btrfs_err(fs_info, "unable to find block group for %llu", eb->start); + btrfs_handle_fs_error(fs_info, -ENOENT, NULL); + return -ENOENT; + } + + spin_lock(&bg->space_info->lock); + spin_lock(&bg->lock); + bg->reserved -= fs_info->nodesize; + bg->space_info->bytes_reserved -= fs_info->nodesize; + spin_unlock(&bg->lock); + spin_unlock(&bg->space_info->lock); + + btrfs_put_block_group(bg); + + return 0; +} + +static noinline int walk_down_log_tree(struct btrfs_path *path, int *level, + struct walk_control *wc) +{ + struct btrfs_trans_handle *trans = wc->trans; + struct btrfs_fs_info *fs_info = wc->log->fs_info; + struct btrfs_eb_prealloc pa = { 0 }; + u64 bytenr; + u64 ptr_gen; + struct extent_buffer *next; + struct extent_buffer *cur; + int ret = 0; + + while (*level > 0) { + struct btrfs_tree_parent_check check = { 0 }; + + cur = path->nodes[*level]; + + WARN_ON(btrfs_header_level(cur) != *level); + + if (path->slots[*level] >= + btrfs_header_nritems(cur)) + break; + + bytenr = btrfs_node_blockptr(cur, path->slots[*level]); + ptr_gen = btrfs_node_ptr_generation(cur, path->slots[*level]); + check.transid = ptr_gen; + check.level = *level - 1; + check.has_first_key = true; + btrfs_node_key_to_cpu(cur, &check.first_key, path->slots[*level]); + + next = btrfs_find_create_tree_block(fs_info, &pa, bytenr, + btrfs_header_owner(cur), + *level - 1); + if (IS_ERR(next)) { + ret = PTR_ERR(next); + if (trans) + btrfs_abort_transaction(trans, ret); + else + btrfs_handle_fs_error(fs_info, ret, NULL); + return ret; + } + + if (*level == 1) { + ret = wc->process_func(next, wc, ptr_gen, *level - 1); + if (ret) { + free_extent_buffer(next); + return ret; + } + + path->slots[*level]++; + if (wc->free) { + ret = btrfs_read_extent_buffer(next, &check); + if (ret) { + free_extent_buffer(next); + if (trans) + btrfs_abort_transaction(trans, ret); + else + btrfs_handle_fs_error(fs_info, ret, NULL); + return ret; + } + + ret = clean_log_buffer(trans, next); + if (ret) { + free_extent_buffer(next); + return ret; + } + } + free_extent_buffer(next); + continue; + } + ret = btrfs_read_extent_buffer(next, &check); + if (ret) { + free_extent_buffer(next); + if (trans) + btrfs_abort_transaction(trans, ret); + else + btrfs_handle_fs_error(fs_info, ret, NULL); + return ret; + } + + if (path->nodes[*level-1]) + free_extent_buffer(path->nodes[*level-1]); + path->nodes[*level-1] = next; + *level = btrfs_header_level(next); + path->slots[*level] = 0; + cond_resched(); + } + path->slots[*level] = btrfs_header_nritems(path->nodes[*level]); + + cond_resched(); + return 0; +} + +static noinline int walk_up_log_tree(struct btrfs_path *path, int *level, + struct walk_control *wc) +{ + int i; + int slot; + int ret; + + for (i = *level; i < BTRFS_MAX_LEVEL - 1 && path->nodes[i]; i++) { + slot = path->slots[i]; + if (slot + 1 < btrfs_header_nritems(path->nodes[i])) { + path->slots[i]++; + *level = i; + WARN_ON(*level == 0); + return 0; + } else { + ret = wc->process_func(path->nodes[*level], wc, + btrfs_header_generation(path->nodes[*level]), + *level); + if (ret) + return ret; + + if (wc->free) { + ret = clean_log_buffer(wc->trans, path->nodes[*level]); + if (ret) + return ret; + } + free_extent_buffer(path->nodes[*level]); + path->nodes[*level] = NULL; + *level = i + 1; + } + } + return 1; +} + +/* + * drop the reference count on the tree rooted at 'snap'. This traverses + * the tree freeing any blocks that have a ref count of zero after being + * decremented. + */ +static int walk_log_tree(struct walk_control *wc) +{ + struct btrfs_root *log = wc->log; + int ret = 0; + int wret; + int level; + BTRFS_PATH_AUTO_FREE(path); + int orig_level; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + level = btrfs_header_level(log->node); + orig_level = level; + path->nodes[level] = log->node; + refcount_inc(&log->node->refs); + path->slots[level] = 0; + + while (1) { + wret = walk_down_log_tree(path, &level, wc); + if (wret > 0) + break; + if (wret < 0) + return wret; + + wret = walk_up_log_tree(path, &level, wc); + if (wret > 0) + break; + if (wret < 0) + return wret; + } + + /* was the root node processed? if not, catch it here */ + if (path->nodes[orig_level]) { + ret = wc->process_func(path->nodes[orig_level], wc, + btrfs_header_generation(path->nodes[orig_level]), + orig_level); + if (ret) + return ret; + if (wc->free) + ret = clean_log_buffer(wc->trans, path->nodes[orig_level]); + } + + return ret; +} + +/* + * helper function to update the item for a given subvolumes log root + * in the tree of log roots + */ +static int update_log_root(struct btrfs_trans_handle *trans, + struct btrfs_root *log, + struct btrfs_root_item *root_item) +{ + struct btrfs_fs_info *fs_info = log->fs_info; + int ret; + + if (log->log_transid == 1) { + /* insert root item on the first sync */ + ret = btrfs_insert_root(trans, fs_info->log_root_tree, + &log->root_key, root_item); + } else { + ret = btrfs_update_root(trans, fs_info->log_root_tree, + &log->root_key, root_item); + } + return ret; +} + +/* Returns true if we had to wait, false otherwise. */ +static bool wait_log_commit(struct btrfs_root *root, int transid) +{ + DEFINE_WAIT(wait); + const int index = (transid >= 0 ? transid % 2 : -transid % 2); + + if (!root->log_commit[index]) + return false; + + /* + * we only allow two pending log transactions at a time, + * so we know that if ours is more than 2 older than the + * current transaction, we're done + */ + for (;;) { + prepare_to_wait(&root->log_commit_wait[index], + &wait, TASK_UNINTERRUPTIBLE); + + mutex_unlock(&root->log_mutex); + schedule(); + mutex_lock(&root->log_mutex); + + if (!(root->log_transid_committed < transid && + root->log_commit[index])) + break; + } + finish_wait(&root->log_commit_wait[index], &wait); + + return true; +} + +static void wait_for_writer(struct btrfs_root *root) +{ + DEFINE_WAIT(wait); + + for (;;) { + prepare_to_wait(&root->log_writer_wait, &wait, + TASK_UNINTERRUPTIBLE); + if (!atomic_read(&root->log_writers)) + break; + + mutex_unlock(&root->log_mutex); + schedule(); + mutex_lock(&root->log_mutex); + } + finish_wait(&root->log_writer_wait, &wait); +} + +void btrfs_init_log_ctx(struct btrfs_log_ctx *ctx, struct btrfs_inode *inode) +{ + ctx->log_ret = 0; + ctx->log_transid = 0; + ctx->log_new_dentries = false; + ctx->logging_new_name = false; + ctx->logging_new_delayed_dentries = false; + ctx->logged_before = false; + ctx->inode = inode; + INIT_LIST_HEAD(&ctx->list); + INIT_LIST_HEAD(&ctx->ordered_extents); + INIT_LIST_HEAD(&ctx->conflict_inodes); + ctx->num_conflict_inodes = 0; + ctx->logging_conflict_inodes = false; + ctx->scratch_eb = NULL; +} + +void btrfs_init_log_ctx_scratch_eb(struct btrfs_log_ctx *ctx) +{ + struct btrfs_inode *inode = ctx->inode; + + if (!test_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags) && + !test_bit(BTRFS_INODE_COPY_EVERYTHING, &inode->runtime_flags)) + return; + + /* + * Don't care about allocation failure. This is just for optimization, + * if we fail to allocate here, we will try again later if needed. + */ + ctx->scratch_eb = alloc_dummy_extent_buffer(inode->root->fs_info, 0); +} + +void btrfs_release_log_ctx_extents(struct btrfs_log_ctx *ctx) +{ + struct btrfs_ordered_extent *ordered; + struct btrfs_ordered_extent *tmp; + + btrfs_assert_inode_locked(ctx->inode); + + list_for_each_entry_safe(ordered, tmp, &ctx->ordered_extents, log_list) { + list_del_init(&ordered->log_list); + btrfs_put_ordered_extent(ordered); + } +} + + +static inline void btrfs_remove_log_ctx(struct btrfs_root *root, + struct btrfs_log_ctx *ctx) +{ + mutex_lock(&root->log_mutex); + list_del_init(&ctx->list); + mutex_unlock(&root->log_mutex); +} + +/* + * Invoked in log mutex context, or be sure there is no other task which + * can access the list. + */ +static inline void btrfs_remove_all_log_ctxs(struct btrfs_root *root, + int index, int error) +{ + struct btrfs_log_ctx *ctx; + struct btrfs_log_ctx *safe; + + list_for_each_entry_safe(ctx, safe, &root->log_ctxs[index], list) { + list_del_init(&ctx->list); + ctx->log_ret = error; + } +} + +/* + * Sends a given tree log down to the disk and updates the super blocks to + * record it. When this call is done, you know that any inodes previously + * logged are safely on disk only if it returns 0. + * + * Any other return value means you need to call btrfs_commit_transaction. + * Some of the edge cases for fsyncing directories that have had unlinks + * or renames done in the past mean that sometimes the only safe + * fsync is to commit the whole FS. When btrfs_sync_log returns -EAGAIN, + * that has happened. + */ +int btrfs_sync_log(struct btrfs_trans_handle *trans, + struct btrfs_root *root, struct btrfs_log_ctx *ctx) +{ + int mark; + int ret; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_root *log = root->log_root; + struct btrfs_root *log_root_tree = fs_info->log_root_tree; + struct btrfs_root_item new_root_item; + int log_transid = ctx->log_transid; + int index1 = log_transid % 2; + int index2; + struct btrfs_log_ctx root_log_ctx; + struct blk_plug plug; + u64 log_root_start; + u64 log_root_level; + + mutex_lock(&root->log_mutex); + trace_btrfs_sync_log_enter(trans, root, ctx); + if (root->log_transid_committed >= log_transid) { + trace_btrfs_sync_log_exit(trans, root, ctx, ctx->log_ret); + mutex_unlock(&root->log_mutex); + return ctx->log_ret; + } + + if (wait_log_commit(root, log_transid)) { + trace_btrfs_sync_log_exit(trans, root, ctx, ctx->log_ret); + mutex_unlock(&root->log_mutex); + return ctx->log_ret; + } + ASSERT(log_transid == root->log_transid, + "log_transid=%d root->log_transid=%d", log_transid, root->log_transid); + root->log_commit[index1] = true; + + /* wait for previous tree log sync to complete */ + wait_log_commit(root, log_transid - 1); + + wait_for_writer(root); + + /* bail out if we need to do a full commit */ + if (btrfs_need_log_full_commit(trans)) { + ret = BTRFS_LOG_FORCE_COMMIT; + trace_btrfs_sync_log_exit(trans, root, ctx, ret); + mutex_unlock(&root->log_mutex); + goto out; + } + + if (log_transid % 2 == 0) + mark = EXTENT_DIRTY_LOG1; + else + mark = EXTENT_DIRTY_LOG2; + + /* we start IO on all the marked extents here, but we don't actually + * wait for them until later. + */ + blk_start_plug(&plug); + ret = btrfs_write_marked_extents(fs_info, &log->dirty_log_pages, mark); + /* + * -EAGAIN happens when someone, e.g., a concurrent transaction + * commit, writes a dirty extent in this tree-log commit. This + * concurrent write will create a hole writing out the extents, + * and we cannot proceed on a zoned filesystem, requiring + * sequential writing. While we can bail out to a full commit + * here, but we can continue hoping the concurrent writing fills + * the hole. + */ + if (ret == -EAGAIN && btrfs_is_zoned(fs_info)) + ret = 0; + if (ret) { + trace_btrfs_sync_log_exit(trans, root, ctx, ret); + blk_finish_plug(&plug); + btrfs_set_log_full_commit(trans); + mutex_unlock(&root->log_mutex); + goto out; + } + + /* + * We _must_ update under the root->log_mutex in order to make sure we + * have a consistent view of the log root we are trying to commit at + * this moment. + * + * We _must_ copy this into a local copy, because we are not holding the + * log_root_tree->log_mutex yet. This is important because when we + * commit the log_root_tree we must have a consistent view of the + * log_root_tree when we update the super block to point at the + * log_root_tree bytenr. If we update the log_root_tree here we'll race + * with the commit and possibly point at the new block which we may not + * have written out. + */ + btrfs_set_root_node(&log->root_item, log->node); + memcpy(&new_root_item, &log->root_item, sizeof(new_root_item)); + + btrfs_set_root_log_transid(root, root->log_transid + 1); + log->log_transid = root->log_transid; + /* + * IO has been started, blocks of the log tree have WRITTEN flag set + * in their headers. new modifications of the log will be written to + * new positions. so it's safe to allow log writers to go in. + */ + mutex_unlock(&root->log_mutex); + + if (btrfs_is_zoned(fs_info)) { + mutex_lock(&fs_info->tree_root->log_mutex); + if (!log_root_tree->node) { + ret = btrfs_alloc_log_tree_node(trans, log_root_tree); + if (ret) { + trace_btrfs_sync_log_exit(trans, root, ctx, ret); + mutex_unlock(&fs_info->tree_root->log_mutex); + blk_finish_plug(&plug); + goto out; + } + } + mutex_unlock(&fs_info->tree_root->log_mutex); + } + + btrfs_init_log_ctx(&root_log_ctx, NULL); + + mutex_lock(&log_root_tree->log_mutex); + + index2 = log_root_tree->log_transid % 2; + list_add_tail(&root_log_ctx.list, &log_root_tree->log_ctxs[index2]); + root_log_ctx.log_transid = log_root_tree->log_transid; + + /* + * Now we are safe to update the log_root_tree because we're under the + * log_mutex, and we're a current writer so we're holding the commit + * open until we drop the log_mutex. + */ + ret = update_log_root(trans, log, &new_root_item); + if (ret) { + trace_btrfs_sync_log_exit(trans, root, ctx, ret); + list_del_init(&root_log_ctx.list); + blk_finish_plug(&plug); + btrfs_set_log_full_commit(trans); + if (ret != -ENOSPC) + btrfs_err(fs_info, + "failed to update log for root %llu ret %d", + btrfs_root_id(root), ret); + btrfs_wait_tree_log_extents(log, mark); + mutex_unlock(&log_root_tree->log_mutex); + goto out; + } + + if (log_root_tree->log_transid_committed >= root_log_ctx.log_transid) { + blk_finish_plug(&plug); + list_del_init(&root_log_ctx.list); + mutex_unlock(&log_root_tree->log_mutex); + ret = root_log_ctx.log_ret; + trace_btrfs_sync_log_exit(trans, root, ctx, ret); + goto out; + } + + if (log_root_tree->log_commit[index2]) { + blk_finish_plug(&plug); + ret = btrfs_wait_tree_log_extents(log, mark); + wait_log_commit(log_root_tree, + root_log_ctx.log_transid); + mutex_unlock(&log_root_tree->log_mutex); + if (!ret) + ret = root_log_ctx.log_ret; + trace_btrfs_sync_log_exit(trans, root, ctx, ret); + goto out; + } + ASSERT(root_log_ctx.log_transid == log_root_tree->log_transid, + "root_log_ctx.log_transid=%d log_root_tree->log_transid=%d", + root_log_ctx.log_transid, log_root_tree->log_transid); + log_root_tree->log_commit[index2] = true; + + wait_log_commit(log_root_tree, root_log_ctx.log_transid - 1); + + /* + * now that we've moved on to the tree of log tree roots, + * check the full commit flag again + */ + if (btrfs_need_log_full_commit(trans)) { + blk_finish_plug(&plug); + btrfs_wait_tree_log_extents(log, mark); + mutex_unlock(&log_root_tree->log_mutex); + ret = BTRFS_LOG_FORCE_COMMIT; + trace_btrfs_sync_log_exit(trans, root, ctx, ret); + goto out_wake_log_root; + } + + ret = btrfs_write_marked_extents(fs_info, + &log_root_tree->dirty_log_pages, + EXTENT_DIRTY_LOG1 | EXTENT_DIRTY_LOG2); + blk_finish_plug(&plug); + /* + * As described above, -EAGAIN indicates a hole in the extents. We + * cannot wait for these write outs since the waiting cause a + * deadlock. Bail out to the full commit instead. + */ + if (ret == -EAGAIN && btrfs_is_zoned(fs_info)) { + trace_btrfs_sync_log_exit(trans, root, ctx, ret); + btrfs_set_log_full_commit(trans); + btrfs_wait_tree_log_extents(log, mark); + mutex_unlock(&log_root_tree->log_mutex); + goto out_wake_log_root; + } else if (ret) { + trace_btrfs_sync_log_exit(trans, root, ctx, ret); + btrfs_set_log_full_commit(trans); + mutex_unlock(&log_root_tree->log_mutex); + goto out_wake_log_root; + } + ret = btrfs_wait_tree_log_extents(log, mark); + if (!ret) + ret = btrfs_wait_tree_log_extents(log_root_tree, + EXTENT_DIRTY_LOG1 | EXTENT_DIRTY_LOG2); + if (ret) { + trace_btrfs_sync_log_exit(trans, root, ctx, ret); + btrfs_set_log_full_commit(trans); + mutex_unlock(&log_root_tree->log_mutex); + goto out_wake_log_root; + } + + log_root_start = log_root_tree->node->start; + log_root_level = btrfs_header_level(log_root_tree->node); + log_root_tree->log_transid++; + mutex_unlock(&log_root_tree->log_mutex); + + /* + * Here we are guaranteed that nobody is going to write the superblock + * for the current transaction before us and that neither we do write + * our superblock before the previous transaction finishes its commit + * and writes its superblock, because: + * + * 1) We are holding a handle on the current transaction, so no body + * can commit it until we release the handle; + * + * 2) Before writing our superblock we acquire the tree_log_mutex, so + * if the previous transaction is still committing, and hasn't yet + * written its superblock, we wait for it to do it, because a + * transaction commit acquires the tree_log_mutex when the commit + * begins and releases it only after writing its superblock. + */ + mutex_lock(&fs_info->tree_log_mutex); + + /* + * The previous transaction writeout phase could have failed, and thus + * marked the fs in an error state. We must not commit here, as we + * could have updated our generation in the super_for_commit and + * writing the super here would result in transid mismatches. If there + * is an error here just bail. + */ + if (unlikely(BTRFS_FS_ERROR(fs_info))) { + ret = -EIO; + trace_btrfs_sync_log_exit(trans, root, ctx, ret); + btrfs_set_log_full_commit(trans); + btrfs_abort_transaction(trans, ret); + mutex_unlock(&fs_info->tree_log_mutex); + goto out_wake_log_root; + } + + btrfs_set_super_log_root(fs_info->super_for_commit, log_root_start); + btrfs_set_super_log_root_level(fs_info->super_for_commit, log_root_level); + ret = write_all_supers(trans); + mutex_unlock(&fs_info->tree_log_mutex); + if (unlikely(ret)) { + trace_btrfs_sync_log_exit(trans, root, ctx, ret); + btrfs_set_log_full_commit(trans); + btrfs_abort_transaction(trans, ret); + goto out_wake_log_root; + } + + /* + * We know there can only be one task here, since we have not yet set + * root->log_commit[index1] to false and any task attempting to sync the + * log must wait for the previous log transaction to commit if it's + * still in progress or wait for the current log transaction commit if + * someone else already started it. We use <= and not < because the + * first log transaction has an ID of 0. + */ + ASSERT(btrfs_get_root_last_log_commit(root) <= log_transid, + "last_log_commit(root)=%d log_transid=%d", + btrfs_get_root_last_log_commit(root), log_transid); + btrfs_set_root_last_log_commit(root, log_transid); + +out_wake_log_root: + mutex_lock(&log_root_tree->log_mutex); + btrfs_remove_all_log_ctxs(log_root_tree, index2, ret); + + log_root_tree->log_transid_committed++; + log_root_tree->log_commit[index2] = false; + mutex_unlock(&log_root_tree->log_mutex); + + /* + * The barrier before waitqueue_active (in cond_wake_up) is needed so + * all the updates above are seen by the woken threads. It might not be + * necessary, but proving that seems to be hard. + */ + cond_wake_up(&log_root_tree->log_commit_wait[index2]); +out: + mutex_lock(&root->log_mutex); + btrfs_remove_all_log_ctxs(root, index1, ret); + root->log_transid_committed++; + root->log_commit[index1] = false; + mutex_unlock(&root->log_mutex); + + /* + * The barrier before waitqueue_active (in cond_wake_up) is needed so + * all the updates above are seen by the woken threads. It might not be + * necessary, but proving that seems to be hard. + */ + cond_wake_up(&root->log_commit_wait[index1]); + return ret; +} + +static void free_log_tree(struct btrfs_trans_handle *trans, + struct btrfs_root *log) +{ + int ret; + struct walk_control wc = { + .free = true, + .process_func = process_one_buffer, + .log = log, + .trans = trans, + }; + + if (log->node) { + ret = walk_log_tree(&wc); + if (ret) { + /* + * We weren't able to traverse the entire log tree, the + * typical scenario is getting an -EIO when reading an + * extent buffer of the tree, due to a previous writeback + * failure of it. + */ + set_bit(BTRFS_FS_STATE_LOG_CLEANUP_ERROR, + &log->fs_info->fs_state); + + /* + * Some extent buffers of the log tree may still be dirty + * and not yet written back to storage, because we may + * have updates to a log tree without syncing a log tree, + * such as during rename and link operations. So flush + * them out and wait for their writeback to complete, so + * that we properly cleanup their state and pages. + */ + btrfs_write_marked_extents(log->fs_info, + &log->dirty_log_pages, + EXTENT_DIRTY_LOG1 | EXTENT_DIRTY_LOG2); + btrfs_wait_tree_log_extents(log, + EXTENT_DIRTY_LOG1 | EXTENT_DIRTY_LOG2); + + if (trans) + btrfs_abort_transaction(trans, ret); + else + btrfs_handle_fs_error(log->fs_info, ret, NULL); + } + } + + btrfs_extent_io_tree_release(&log->dirty_log_pages); + btrfs_extent_io_tree_release(&log->log_csum_range); + + btrfs_put_root(log); +} + +/* + * free all the extents used by the tree log. This should be called + * at commit time of the full transaction + */ +void btrfs_free_log(struct btrfs_trans_handle *trans, struct btrfs_root *root) +{ + if (root->log_root) { + free_log_tree(trans, root->log_root); + root->log_root = NULL; + clear_bit(BTRFS_ROOT_HAS_LOG_TREE, &root->state); + } +} + +void btrfs_free_log_root_tree(struct btrfs_trans_handle *trans, struct btrfs_fs_info *fs_info) +{ + if (fs_info->log_root_tree) { + free_log_tree(trans, fs_info->log_root_tree); + fs_info->log_root_tree = NULL; + clear_bit(BTRFS_ROOT_HAS_LOG_TREE, &fs_info->tree_root->state); + } +} + +static bool mark_inode_as_not_logged(const struct btrfs_trans_handle *trans, + struct btrfs_inode *inode) +{ + bool ret = false; + + /* + * Do this only if ->logged_trans is still 0 to prevent races with + * concurrent logging as we may see the inode not logged when + * inode_logged() is called but it gets logged after inode_logged() did + * not find it in the log tree and we end up setting ->logged_trans to a + * value less than trans->transid after the concurrent logging task has + * set it to trans->transid. As a consequence, subsequent rename, unlink + * and link operations may end up not logging new names and removing old + * names from the log. + */ + spin_lock(&inode->lock); + if (inode->logged_trans == 0) + inode->logged_trans = trans->transid - 1; + else if (inode->logged_trans == trans->transid) + ret = true; + spin_unlock(&inode->lock); + + return ret; +} + +/* + * Check if an inode was logged in the current transaction. This correctly deals + * with the case where the inode was logged but has a logged_trans of 0, which + * happens if the inode is evicted and loaded again, as logged_trans is an in + * memory only field (not persisted). + * + * Returns 1 if the inode was logged before in the transaction, 0 if it was not, + * and < 0 on error. + */ +static int inode_logged(const struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_path *path_in) +{ + struct btrfs_path *path = path_in; + struct btrfs_key key; + int ret; + + /* + * Quick lockless call, since once ->logged_trans is set to the current + * transaction, we never set it to a lower value anywhere else. + */ + if (data_race(inode->logged_trans) == trans->transid) + return 1; + + /* + * If logged_trans is not 0 and not trans->transid, then we know the + * inode was not logged in this transaction, so we can return false + * right away. We take the lock to avoid a race caused by load/store + * tearing with a concurrent btrfs_log_inode() call or a concurrent task + * in this function further below - an update to trans->transid can be + * teared into two 32 bits updates for example, in which case we could + * see a positive value that is not trans->transid and assume the inode + * was not logged when it was. + */ + spin_lock(&inode->lock); + if (inode->logged_trans == trans->transid) { + spin_unlock(&inode->lock); + return 1; + } else if (inode->logged_trans > 0) { + spin_unlock(&inode->lock); + return 0; + } + spin_unlock(&inode->lock); + + /* + * If no log tree was created for this root in this transaction, then + * the inode can not have been logged in this transaction. In that case + * set logged_trans to anything greater than 0 and less than the current + * transaction's ID, to avoid the search below in a future call in case + * a log tree gets created after this. + */ + if (!test_bit(BTRFS_ROOT_HAS_LOG_TREE, &inode->root->state)) + return mark_inode_as_not_logged(trans, inode); + + /* + * We have a log tree and the inode's logged_trans is 0. We can't tell + * for sure if the inode was logged before in this transaction by looking + * only at logged_trans. We could be pessimistic and assume it was, but + * that can lead to unnecessarily logging an inode during rename and link + * operations, and then further updating the log in followup rename and + * link operations, specially if it's a directory, which adds latency + * visible to applications doing a series of rename or link operations. + * + * A logged_trans of 0 here can mean several things: + * + * 1) The inode was never logged since the filesystem was mounted, and may + * or may have not been evicted and loaded again; + * + * 2) The inode was logged in a previous transaction, then evicted and + * then loaded again; + * + * 3) The inode was logged in the current transaction, then evicted and + * then loaded again. + * + * For cases 1) and 2) we don't want to return true, but we need to detect + * case 3) and return true. So we do a search in the log root for the inode + * item. + */ + key.objectid = btrfs_ino(inode); + key.type = BTRFS_INODE_ITEM_KEY; + key.offset = 0; + + if (!path) { + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + } + + ret = btrfs_search_slot(NULL, inode->root->log_root, &key, path, 0, 0); + + if (path_in) + btrfs_release_path(path); + else + btrfs_free_path(path); + + /* + * Logging an inode always results in logging its inode item. So if we + * did not find the item we know the inode was not logged for sure. + */ + if (ret < 0) { + return ret; + } else if (ret > 0) { + /* + * Set logged_trans to a value greater than 0 and less then the + * current transaction to avoid doing the search in future calls. + */ + return mark_inode_as_not_logged(trans, inode); + } + + /* + * The inode was previously logged and then evicted, set logged_trans to + * the current transaction's ID, to avoid future tree searches as long as + * the inode is not evicted again. + */ + spin_lock(&inode->lock); + inode->logged_trans = trans->transid; + spin_unlock(&inode->lock); + + return 1; +} + +/* + * Delete a directory entry from the log if it exists. + * + * Returns < 0 on error + * 1 if the entry does not exists + * 0 if the entry existed and was successfully deleted + */ +static int del_logged_dentry(struct btrfs_trans_handle *trans, + struct btrfs_root *log, + struct btrfs_path *path, + u64 dir_ino, + const struct fscrypt_str *name, + u64 index) +{ + struct btrfs_dir_item *di; + + /* + * We only log dir index items of a directory, so we don't need to look + * for dir item keys. + */ + di = btrfs_lookup_dir_index_item(trans, log, path, dir_ino, + index, name, -1); + if (IS_ERR(di)) + return PTR_ERR(di); + else if (!di) + return 1; + + /* + * We do not need to update the size field of the directory's + * inode item because on log replay we update the field to reflect + * all existing entries in the directory (see overwrite_item()). + */ + return btrfs_del_item(trans, log, path); +} + +/* + * If both a file and directory are logged, and unlinks or renames are + * mixed in, we have a few interesting corners: + * + * create file X in dir Y + * link file X to X.link in dir Y + * fsync file X + * unlink file X but leave X.link + * fsync dir Y + * + * After a crash we would expect only X.link to exist. But file X + * didn't get fsync'd again so the log has back refs for X and X.link. + * + * We solve this by removing directory entries and inode backrefs from the + * log when a file that was logged in the current transaction is + * unlinked. Any later fsync will include the updated log entries, and + * we'll be able to reconstruct the proper directory items from backrefs. + * + * This optimizations allows us to avoid relogging the entire inode + * or the entire directory. + */ +void btrfs_del_dir_entries_in_log(struct btrfs_trans_handle *trans, + const struct fscrypt_str *name, + struct btrfs_inode *dir, u64 index) +{ + struct btrfs_root *root = dir->root; + BTRFS_PATH_AUTO_FREE(path); + int ret; + + ret = inode_logged(trans, dir, NULL); + if (ret == 0) + return; + if (ret < 0) { + btrfs_set_log_full_commit(trans); + return; + } + + path = btrfs_alloc_path(); + if (!path) { + btrfs_set_log_full_commit(trans); + return; + } + + ret = join_running_log_trans(root); + ASSERT(ret == 0, "join_running_log_trans() ret=%d", ret); + if (WARN_ON(ret)) + return; + + mutex_lock(&dir->log_mutex); + + ret = del_logged_dentry(trans, root->log_root, path, btrfs_ino(dir), + name, index); + mutex_unlock(&dir->log_mutex); + if (ret < 0) + btrfs_set_log_full_commit(trans); + btrfs_end_log_trans(root); +} + +/* see comments for btrfs_del_dir_entries_in_log */ +void btrfs_del_inode_ref_in_log(struct btrfs_trans_handle *trans, + const struct fscrypt_str *name, + struct btrfs_inode *inode, + struct btrfs_inode *dir) +{ + struct btrfs_root *root = dir->root; + int ret; + + ret = inode_logged(trans, inode, NULL); + if (ret == 0) + return; + else if (ret < 0) { + btrfs_set_log_full_commit(trans); + return; + } + + ret = join_running_log_trans(root); + ASSERT(ret == 0, "join_running_log_trans() ret=%d", ret); + if (WARN_ON(ret)) + return; + mutex_lock(&inode->log_mutex); + + ret = btrfs_del_inode_ref(trans, root->log_root, name, btrfs_ino(inode), + btrfs_ino(dir), NULL); + mutex_unlock(&inode->log_mutex); + if (ret < 0 && ret != -ENOENT) + btrfs_set_log_full_commit(trans); + btrfs_end_log_trans(root); +} + +/* + * creates a range item in the log for 'dirid'. first_offset and + * last_offset tell us which parts of the key space the log should + * be considered authoritative for. + */ +static noinline int insert_dir_log_key(struct btrfs_trans_handle *trans, + struct btrfs_root *log, + struct btrfs_path *path, + u64 dirid, + u64 first_offset, u64 last_offset) +{ + int ret; + struct btrfs_key key; + struct btrfs_dir_log_item *item; + + key.objectid = dirid; + key.type = BTRFS_DIR_LOG_INDEX_KEY; + key.offset = first_offset; + ret = btrfs_insert_empty_item(trans, log, path, &key, sizeof(*item)); + /* + * -EEXIST is fine and can happen sporadically when we are logging a + * directory and have concurrent insertions in the subvolume's tree for + * items from other inodes and that result in pushing off some dir items + * from one leaf to another in order to accommodate for the new items. + * This results in logging the same dir index range key. + */ + if (ret && ret != -EEXIST) + return ret; + + item = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_dir_log_item); + if (ret == -EEXIST) { + const u64 curr_end = btrfs_dir_log_end(path->nodes[0], item); + + /* + * btrfs_del_dir_entries_in_log() might have been called during + * an unlink between the initial insertion of this key and the + * current update, or we might be logging a single entry deletion + * during a rename, so set the new last_offset to the max value. + */ + last_offset = max(last_offset, curr_end); + } + btrfs_set_dir_log_end(path->nodes[0], item, last_offset); + btrfs_release_path(path); + return 0; +} + +static int flush_dir_items_batch(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct extent_buffer *src, + struct btrfs_path *dst_path, + int start_slot, + int count) +{ + struct btrfs_root *log = inode->root->log_root; + char AUTO_KFREE(ins_data); + struct btrfs_item_batch batch; + struct extent_buffer *dst; + unsigned long src_offset; + unsigned long dst_offset; + u64 last_index; + struct btrfs_key key; + u32 item_size; + int ret; + int i; + + ASSERT(count > 0, "count=%d", count); + batch.nr = count; + + if (count == 1) { + btrfs_item_key_to_cpu(src, &key, start_slot); + item_size = btrfs_item_size(src, start_slot); + batch.keys = &key; + batch.data_sizes = &item_size; + batch.total_data_size = item_size; + } else { + struct btrfs_key *ins_keys; + u32 *ins_sizes; + + ins_data = kmalloc_array(count, sizeof(u32) + sizeof(struct btrfs_key), GFP_NOFS); + if (!ins_data) + return -ENOMEM; + + ins_sizes = (u32 *)ins_data; + ins_keys = (struct btrfs_key *)(ins_data + count * sizeof(u32)); + batch.keys = ins_keys; + batch.data_sizes = ins_sizes; + batch.total_data_size = 0; + + for (i = 0; i < count; i++) { + const int slot = start_slot + i; + + btrfs_item_key_to_cpu(src, &ins_keys[i], slot); + ins_sizes[i] = btrfs_item_size(src, slot); + batch.total_data_size += ins_sizes[i]; + } + } + + ret = btrfs_insert_empty_items(trans, log, dst_path, &batch); + if (ret) + return ret; + + dst = dst_path->nodes[0]; + /* + * Copy all the items in bulk, in a single copy operation. Item data is + * organized such that it's placed at the end of a leaf and from right + * to left. For example, the data for the second item ends at an offset + * that matches the offset where the data for the first item starts, the + * data for the third item ends at an offset that matches the offset + * where the data of the second items starts, and so on. + * Therefore our source and destination start offsets for copy match the + * offsets of the last items (highest slots). + */ + dst_offset = btrfs_item_ptr_offset(dst, dst_path->slots[0] + count - 1); + src_offset = btrfs_item_ptr_offset(src, start_slot + count - 1); + copy_extent_buffer(dst, src, dst_offset, src_offset, batch.total_data_size); + btrfs_release_path(dst_path); + + last_index = batch.keys[count - 1].offset; + ASSERT(last_index > inode->last_dir_index_offset, + "last_index=%llu inode->last_dir_index_offset=%llu", + last_index, inode->last_dir_index_offset); + + /* + * If for some unexpected reason the last item's index is not greater + * than the last index we logged, warn and force a transaction commit. + */ + if (WARN_ON(last_index <= inode->last_dir_index_offset)) + ret = BTRFS_LOG_FORCE_COMMIT; + else + inode->last_dir_index_offset = last_index; + + if (btrfs_get_first_dir_index_to_log(inode) == 0) + btrfs_set_first_dir_index_to_log(inode, batch.keys[0].offset); + + return ret; +} + +static int clone_leaf(struct btrfs_path *path, struct btrfs_log_ctx *ctx) +{ + const int slot = path->slots[0]; + + if (ctx->scratch_eb) { + copy_extent_buffer_full(ctx->scratch_eb, path->nodes[0]); + } else { + ctx->scratch_eb = btrfs_clone_extent_buffer(path->nodes[0]); + if (!ctx->scratch_eb) + return -ENOMEM; + } + + btrfs_release_path(path); + path->nodes[0] = ctx->scratch_eb; + path->slots[0] = slot; + /* + * Add extra ref to scratch eb so that it is not freed when callers + * release the path, so we can reuse it later if needed. + */ + refcount_inc(&ctx->scratch_eb->refs); + + return 0; +} + +static int process_dir_items_leaf(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_path *path, + struct btrfs_path *dst_path, + struct btrfs_log_ctx *ctx, + u64 *last_old_dentry_offset) +{ + struct btrfs_root *log = inode->root->log_root; + struct extent_buffer *src; + const int nritems = btrfs_header_nritems(path->nodes[0]); + const u64 ino = btrfs_ino(inode); + bool last_found = false; + int batch_start = 0; + int batch_size = 0; + int ret; + + /* + * We need to clone the leaf, release the read lock on it, and use the + * clone before modifying the log tree. See the comment at copy_items() + * about why we need to do this. + */ + ret = clone_leaf(path, ctx); + if (ret < 0) + return ret; + + src = path->nodes[0]; + + for (int i = path->slots[0]; i < nritems; i++) { + struct btrfs_dir_item *di; + struct btrfs_key key; + + btrfs_item_key_to_cpu(src, &key, i); + + if (key.objectid != ino || key.type != BTRFS_DIR_INDEX_KEY) { + last_found = true; + break; + } + + di = btrfs_item_ptr(src, i, struct btrfs_dir_item); + + /* + * Skip ranges of items that consist only of dir item keys created + * in past transactions. However if we find a gap, we must log a + * dir index range item for that gap, so that index keys in that + * gap are deleted during log replay. + */ + if (btrfs_dir_transid(src, di) < trans->transid) { + if (key.offset > *last_old_dentry_offset + 1) { + ret = insert_dir_log_key(trans, log, dst_path, + ino, *last_old_dentry_offset + 1, + key.offset - 1); + if (ret < 0) + return ret; + } + + *last_old_dentry_offset = key.offset; + continue; + } + + /* If we logged this dir index item before, we can skip it. */ + if (key.offset <= inode->last_dir_index_offset) + continue; + + /* + * We must make sure that when we log a directory entry, the + * corresponding inode, after log replay, has a matching link + * count. For example: + * + * touch foo + * mkdir mydir + * sync + * ln foo mydir/bar + * xfs_io -c "fsync" mydir + * + * + * + * Would result in a fsync log that when replayed, our file inode + * would have a link count of 1, but we get two directory entries + * pointing to the same inode. After removing one of the names, + * it would not be possible to remove the other name, which + * resulted always in stale file handle errors, and would not be + * possible to rmdir the parent directory, since its i_size could + * never be decremented to the value BTRFS_EMPTY_DIR_SIZE, + * resulting in -ENOTEMPTY errors. + */ + if (!ctx->log_new_dentries) { + struct btrfs_key di_key; + + btrfs_dir_item_key_to_cpu(src, di, &di_key); + if (di_key.type != BTRFS_ROOT_ITEM_KEY) + ctx->log_new_dentries = true; + } + + if (batch_size == 0) + batch_start = i; + batch_size++; + } + + if (batch_size > 0) { + ret = flush_dir_items_batch(trans, inode, src, dst_path, + batch_start, batch_size); + if (ret < 0) + return ret; + } + + return last_found ? 1 : 0; +} + +/* + * log all the items included in the current transaction for a given + * directory. This also creates the range items in the log tree required + * to replay anything deleted before the fsync + */ +static noinline int log_dir_items(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_path *path, + struct btrfs_path *dst_path, + struct btrfs_log_ctx *ctx, + u64 min_offset, u64 *last_offset_ret) +{ + struct btrfs_key min_key; + struct btrfs_root *root = inode->root; + struct btrfs_root *log = root->log_root; + int ret; + u64 last_old_dentry_offset = min_offset - 1; + u64 last_offset = (u64)-1; + u64 ino = btrfs_ino(inode); + + min_key.objectid = ino; + min_key.type = BTRFS_DIR_INDEX_KEY; + min_key.offset = min_offset; + + ret = btrfs_search_forward(root, &min_key, path, trans->transid); + + /* + * we didn't find anything from this transaction, see if there + * is anything at all + */ + if (ret != 0 || min_key.objectid != ino || + min_key.type != BTRFS_DIR_INDEX_KEY) { + min_key.objectid = ino; + min_key.type = BTRFS_DIR_INDEX_KEY; + min_key.offset = (u64)-1; + btrfs_release_path(path); + ret = btrfs_search_slot(NULL, root, &min_key, path, 0, 0); + if (ret < 0) { + btrfs_release_path(path); + return ret; + } + ret = btrfs_previous_item(root, path, ino, BTRFS_DIR_INDEX_KEY); + + /* if ret == 0 there are items for this type, + * create a range to tell us the last key of this type. + * otherwise, there are no items in this directory after + * *min_offset, and we create a range to indicate that. + */ + if (ret == 0) { + struct btrfs_key tmp; + + btrfs_item_key_to_cpu(path->nodes[0], &tmp, + path->slots[0]); + if (tmp.type == BTRFS_DIR_INDEX_KEY) + last_old_dentry_offset = tmp.offset; + } else if (ret > 0) { + ret = 0; + } + + goto done; + } + + /* go backward to find any previous key */ + ret = btrfs_previous_item(root, path, ino, BTRFS_DIR_INDEX_KEY); + if (ret == 0) { + struct btrfs_key tmp; + + btrfs_item_key_to_cpu(path->nodes[0], &tmp, path->slots[0]); + /* + * The dir index key before the first one we found that needs to + * be logged might be in a previous leaf, and there might be a + * gap between these keys, meaning that we had deletions that + * happened. So the key range item we log (key type + * BTRFS_DIR_LOG_INDEX_KEY) must cover a range that starts at the + * previous key's offset plus 1, so that those deletes are replayed. + */ + if (tmp.type == BTRFS_DIR_INDEX_KEY) + last_old_dentry_offset = tmp.offset; + } else if (ret < 0) { + goto done; + } + + btrfs_release_path(path); + + /* + * Find the first key from this transaction again or the one we were at + * in the loop below in case we had to reschedule. We may be logging the + * directory without holding its VFS lock, which happen when logging new + * dentries (through log_new_dir_dentries()) or in some cases when we + * need to log the parent directory of an inode. This means a dir index + * key might be deleted from the inode's root, and therefore we may not + * find it anymore. If we can't find it, just move to the next key. We + * can not bail out and ignore, because if we do that we will simply + * not log dir index keys that come after the one that was just deleted + * and we can end up logging a dir index range that ends at (u64)-1 + * (@last_offset is initialized to that), resulting in removing dir + * entries we should not remove at log replay time. + */ +search: + ret = btrfs_search_slot(NULL, root, &min_key, path, 0, 0); + if (ret > 0) { + ret = btrfs_next_item(root, path); + if (ret > 0) { + /* There are no more keys in the inode's root. */ + ret = 0; + goto done; + } + } + if (ret < 0) + goto done; + + /* + * we have a block from this transaction, log every item in it + * from our directory + */ + while (1) { + ret = process_dir_items_leaf(trans, inode, path, dst_path, ctx, + &last_old_dentry_offset); + if (ret != 0) { + if (ret > 0) + ret = 0; + goto done; + } + path->slots[0] = btrfs_header_nritems(path->nodes[0]); + + /* + * look ahead to the next item and see if it is also + * from this directory and from this transaction + */ + ret = btrfs_next_leaf(root, path); + if (ret) { + if (ret == 1) { + last_offset = (u64)-1; + ret = 0; + } + goto done; + } + btrfs_item_key_to_cpu(path->nodes[0], &min_key, path->slots[0]); + if (min_key.objectid != ino || min_key.type != BTRFS_DIR_INDEX_KEY) { + last_offset = (u64)-1; + goto done; + } + if (btrfs_header_generation(path->nodes[0]) != trans->transid) { + /* + * The next leaf was not changed in the current transaction + * and has at least one dir index key. + * We check for the next key because there might have been + * one or more deletions between the last key we logged and + * that next key. So the key range item we log (key type + * BTRFS_DIR_LOG_INDEX_KEY) must end at the next key's + * offset minus 1, so that those deletes are replayed. + */ + last_offset = min_key.offset - 1; + goto done; + } + if (need_resched()) { + btrfs_release_path(path); + cond_resched(); + goto search; + } + } +done: + btrfs_release_path(path); + btrfs_release_path(dst_path); + + if (ret == 0) { + *last_offset_ret = last_offset; + /* + * In case the leaf was changed in the current transaction but + * all its dir items are from a past transaction, the last item + * in the leaf is a dir item and there's no gap between that last + * dir item and the first one on the next leaf (which did not + * change in the current transaction), then we don't need to log + * a range, last_old_dentry_offset is == to last_offset. + */ + ASSERT(last_old_dentry_offset <= last_offset, + "last_old_dentry_offset=%llu last_offset=%llu", + last_old_dentry_offset, last_offset); + if (last_old_dentry_offset < last_offset) + ret = insert_dir_log_key(trans, log, path, ino, + last_old_dentry_offset + 1, + last_offset); + } + + return ret; +} + +/* + * If the inode was logged before and it was evicted, then its + * last_dir_index_offset is 0, so we don't know the value of the last index + * key offset. If that's the case, search for it and update the inode. This + * is to avoid lookups in the log tree every time we try to insert a dir index + * key from a leaf changed in the current transaction, and to allow us to always + * do batch insertions of dir index keys. + */ +static int update_last_dir_index_offset(struct btrfs_inode *inode, + struct btrfs_path *path, + const struct btrfs_log_ctx *ctx) +{ + const u64 ino = btrfs_ino(inode); + struct btrfs_key key; + int ret; + + lockdep_assert_held(&inode->log_mutex); + + if (inode->last_dir_index_offset != 0) + return 0; + + if (!ctx->logged_before) { + inode->last_dir_index_offset = BTRFS_DIR_START_INDEX - 1; + return 0; + } + + key.objectid = ino; + key.type = BTRFS_DIR_INDEX_KEY; + key.offset = (u64)-1; + + ret = btrfs_search_slot(NULL, inode->root->log_root, &key, path, 0, 0); + /* + * An error happened or we actually have an index key with an offset + * value of (u64)-1. Bail out, we're done. + */ + if (ret <= 0) + goto out; + + ret = 0; + inode->last_dir_index_offset = BTRFS_DIR_START_INDEX - 1; + + /* + * No dir index items, bail out and leave last_dir_index_offset with + * the value right before the first valid index value. + */ + if (path->slots[0] == 0) + goto out; + + /* + * btrfs_search_slot() left us at one slot beyond the slot with the last + * index key, or beyond the last key of the directory that is not an + * index key. If we have an index key before, set last_dir_index_offset + * to its offset value, otherwise leave it with a value right before the + * first valid index value, as it means we have an empty directory. + */ + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0] - 1); + if (key.objectid == ino && key.type == BTRFS_DIR_INDEX_KEY) + inode->last_dir_index_offset = key.offset; + +out: + btrfs_release_path(path); + + return ret; +} + +/* + * logging directories is very similar to logging inodes, We find all the items + * from the current transaction and write them to the log. + * + * The recovery code scans the directory in the subvolume, and if it finds a + * key in the range logged that is not present in the log tree, then it means + * that dir entry was unlinked during the transaction. + * + * In order for that scan to work, we must include one key smaller than + * the smallest logged by this transaction and one key larger than the largest + * key logged by this transaction. + */ +static noinline int log_directory_changes(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_path *path, + struct btrfs_path *dst_path, + struct btrfs_log_ctx *ctx) +{ + u64 min_key; + u64 max_key; + int ret; + + ret = update_last_dir_index_offset(inode, path, ctx); + if (ret) + return ret; + + min_key = BTRFS_DIR_START_INDEX; + max_key = 0; + + while (1) { + ret = log_dir_items(trans, inode, path, dst_path, + ctx, min_key, &max_key); + if (ret) + return ret; + if (max_key == (u64)-1) + break; + min_key = max_key + 1; + } + + return 0; +} + +/* + * a helper function to drop items from the log before we relog an + * inode. max_key_type indicates the highest item type to remove. + * This cannot be run for file data extents because it does not + * free the extents they point to. + */ +static int drop_inode_items(struct btrfs_trans_handle *trans, + struct btrfs_root *log, + struct btrfs_path *path, + struct btrfs_inode *inode, + int max_key_type) +{ + int ret; + struct btrfs_key key; + struct btrfs_key found_key; + int start_slot; + + key.objectid = btrfs_ino(inode); + key.type = max_key_type; + key.offset = (u64)-1; + + while (1) { + ret = btrfs_search_slot(trans, log, &key, path, -1, 1); + if (ret < 0) { + break; + } else if (ret > 0) { + if (path->slots[0] == 0) + break; + path->slots[0]--; + } + + btrfs_item_key_to_cpu(path->nodes[0], &found_key, + path->slots[0]); + + if (found_key.objectid != key.objectid) + break; + + found_key.offset = 0; + found_key.type = 0; + ret = btrfs_bin_search(path->nodes[0], 0, &found_key, &start_slot); + if (ret < 0) + break; + + ret = btrfs_del_items(trans, log, path, start_slot, + path->slots[0] - start_slot + 1); + /* + * If start slot isn't 0 then we don't need to re-search, we've + * found the last guy with the objectid in this tree. + */ + if (ret || start_slot != 0) + break; + btrfs_release_path(path); + } + btrfs_release_path(path); + if (ret > 0) + ret = 0; + return ret; +} + +static int truncate_inode_items(struct btrfs_trans_handle *trans, + struct btrfs_root *log_root, + struct btrfs_inode *inode, + u64 new_size, u32 min_type) +{ + struct btrfs_truncate_control control = { + .new_size = new_size, + .ino = btrfs_ino(inode), + .min_type = min_type, + .skip_ref_updates = true, + }; + + return btrfs_truncate_inode_items(trans, log_root, &control); +} + +static void fill_inode_item(struct btrfs_trans_handle *trans, + struct extent_buffer *leaf, + struct btrfs_inode_item *item, + struct btrfs_inode *inode, bool log_inode_only, + u64 logged_isize) +{ + struct inode *vfs_inode = &inode->vfs_inode; + u64 gen = inode->generation; + u64 flags; + + if (log_inode_only) { + /* + * Set the generation to zero so the recover code can tell the + * difference between a logging just to say 'this inode exists' + * and a logging to say 'update this inode with these values'. + * But only if the inode was not already logged before. + * We access ->logged_trans directly since it was already set + * up in the call chain by btrfs_log_inode(), and data_race() + * to avoid false alerts from KCSAN and since it was set already + * and one can set it to 0 since that only happens on eviction + * and we are holding a ref on the inode. + */ + ASSERT(data_race(inode->logged_trans) > 0); + if (data_race(inode->logged_trans) < trans->transid) + gen = 0; + + btrfs_set_inode_size(leaf, item, logged_isize); + } else { + btrfs_set_inode_size(leaf, item, vfs_inode->i_size); + } + + btrfs_set_inode_generation(leaf, item, gen); + + btrfs_set_inode_uid(leaf, item, i_uid_read(vfs_inode)); + btrfs_set_inode_gid(leaf, item, i_gid_read(vfs_inode)); + btrfs_set_inode_mode(leaf, item, vfs_inode->i_mode); + btrfs_set_inode_nlink(leaf, item, vfs_inode->i_nlink); + + btrfs_set_timespec_sec(leaf, &item->atime, inode_get_atime_sec(vfs_inode)); + btrfs_set_timespec_nsec(leaf, &item->atime, inode_get_atime_nsec(vfs_inode)); + + btrfs_set_timespec_sec(leaf, &item->mtime, inode_get_mtime_sec(vfs_inode)); + btrfs_set_timespec_nsec(leaf, &item->mtime, inode_get_mtime_nsec(vfs_inode)); + + btrfs_set_timespec_sec(leaf, &item->ctime, inode_get_ctime_sec(vfs_inode)); + btrfs_set_timespec_nsec(leaf, &item->ctime, inode_get_ctime_nsec(vfs_inode)); + + btrfs_set_timespec_sec(leaf, &item->otime, inode->i_otime_sec); + btrfs_set_timespec_nsec(leaf, &item->otime, inode->i_otime_nsec); + + /* + * We do not need to set the nbytes field, in fact during a fast fsync + * its value may not even be correct, since a fast fsync does not wait + * for ordered extent completion, which is where we update nbytes, it + * only waits for writeback to complete. During log replay as we find + * file extent items and replay them, we adjust the nbytes field of the + * inode item in subvolume tree as needed (see overwrite_item()). + */ + + btrfs_set_inode_sequence(leaf, item, inode_peek_iversion(vfs_inode)); + btrfs_set_inode_transid(leaf, item, trans->transid); + btrfs_set_inode_rdev(leaf, item, vfs_inode->i_rdev); + flags = btrfs_inode_combine_flags(inode->flags, inode->ro_flags); + btrfs_set_inode_flags(leaf, item, flags); + btrfs_set_inode_block_group(leaf, item, 0); +} + +static int log_inode_item(struct btrfs_trans_handle *trans, + struct btrfs_root *log, struct btrfs_path *path, + struct btrfs_inode *inode, bool inode_item_dropped) +{ + struct btrfs_inode_item *inode_item; + struct btrfs_key key; + int ret; + + btrfs_get_inode_key(inode, &key); + /* + * If we are doing a fast fsync and the inode was logged before in the + * current transaction, then we know the inode was previously logged and + * it exists in the log tree. For performance reasons, in this case use + * btrfs_search_slot() directly with ins_len set to 0 so that we never + * attempt a write lock on the leaf's parent, which adds unnecessary lock + * contention in case there are concurrent fsyncs for other inodes of the + * same subvolume. Using btrfs_insert_empty_item() when the inode item + * already exists can also result in unnecessarily splitting a leaf. + */ + if (!inode_item_dropped && inode->logged_trans == trans->transid) { + ret = btrfs_search_slot(trans, log, &key, path, 0, 1); + ASSERT(ret <= 0); + if (ret > 0) + ret = -ENOENT; + } else { + /* + * This means it is the first fsync in the current transaction, + * so the inode item is not in the log and we need to insert it. + * We can never get -EEXIST because we are only called for a fast + * fsync and in case an inode eviction happens after the inode was + * logged before in the current transaction, when we load again + * the inode, we set BTRFS_INODE_NEEDS_FULL_SYNC on its runtime + * flags and set ->logged_trans to 0. + */ + ret = btrfs_insert_empty_item(trans, log, path, &key, + sizeof(*inode_item)); + ASSERT(ret != -EEXIST); + } + if (ret) + return ret; + inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_inode_item); + fill_inode_item(trans, path->nodes[0], inode_item, inode, false, 0); + btrfs_release_path(path); + return 0; +} + +static int log_csums(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_root *log_root, + struct btrfs_ordered_sum *sums) +{ + const u64 lock_end = sums->logical + sums->len - 1; + struct extent_state *cached_state = NULL; + int ret; + + /* + * If this inode was not used for reflink operations in the current + * transaction with new extents, then do the fast path, no need to + * worry about logging checksum items with overlapping ranges. + */ + if (inode->last_reflink_trans < trans->transid) + return btrfs_insert_data_csums(trans, log_root, sums); + + /* + * Serialize logging for checksums. This is to avoid racing with the + * same checksum being logged by another task that is logging another + * file which happens to refer to the same extent as well. Such races + * can leave checksum items in the log with overlapping ranges. + */ + ret = btrfs_lock_extent(&log_root->log_csum_range, sums->logical, lock_end, + &cached_state); + if (ret) + return ret; + /* + * Due to extent cloning, we might have logged a csum item that covers a + * subrange of a cloned extent, and later we can end up logging a csum + * item for a larger subrange of the same extent or the entire range. + * This would leave csum items in the log tree that cover the same range + * and break the searches for checksums in the log tree, resulting in + * some checksums missing in the fs/subvolume tree. So just delete (or + * trim and adjust) any existing csum items in the log for this range. + */ + ret = btrfs_del_csums(trans, log_root, sums->logical, sums->len); + if (!ret) + ret = btrfs_insert_data_csums(trans, log_root, sums); + + btrfs_unlock_extent(&log_root->log_csum_range, sums->logical, lock_end, + &cached_state); + + return ret; +} + +static noinline int copy_items(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_path *dst_path, + struct btrfs_path *src_path, + int start_slot, int nr, enum btrfs_log_mode log_mode, + u64 logged_isize, struct btrfs_log_ctx *ctx) +{ + struct btrfs_root *log = inode->root->log_root; + struct btrfs_file_extent_item *extent; + struct extent_buffer *src; + int ret; + struct btrfs_key *ins_keys; + u32 *ins_sizes; + struct btrfs_item_batch batch; + char AUTO_KFREE(ins_data); + int dst_index; + const bool skip_csum = (inode->flags & BTRFS_INODE_NODATASUM); + const u64 i_size = i_size_read(&inode->vfs_inode); + + /* + * To keep lockdep happy and avoid deadlocks, clone the source leaf and + * use the clone. This is because otherwise we would be changing the log + * tree, to insert items from the subvolume tree or insert csum items, + * while holding a read lock on a leaf from the subvolume tree, which + * creates a nasty lock dependency when COWing log tree nodes/leaves: + * + * 1) Modifying the log tree triggers an extent buffer allocation while + * holding a write lock on a parent extent buffer from the log tree. + * Allocating the pages for an extent buffer, or the extent buffer + * struct, can trigger inode eviction and finally the inode eviction + * will trigger a release/remove of a delayed node, which requires + * taking the delayed node's mutex; + * + * 2) Allocating a metadata extent for a log tree can trigger the async + * reclaim thread and make us wait for it to release enough space and + * unblock our reservation ticket. The reclaim thread can start + * flushing delayed items, and that in turn results in the need to + * lock delayed node mutexes and in the need to write lock extent + * buffers of a subvolume tree - all this while holding a write lock + * on the parent extent buffer in the log tree. + * + * So one task in scenario 1) running in parallel with another task in + * scenario 2) could lead to a deadlock, one wanting to lock a delayed + * node mutex while having a read lock on a leaf from the subvolume, + * while the other is holding the delayed node's mutex and wants to + * write lock the same subvolume leaf for flushing delayed items. + */ + ret = clone_leaf(src_path, ctx); + if (ret < 0) + return ret; + + src = src_path->nodes[0]; + + ins_data = kmalloc_array(nr, sizeof(struct btrfs_key) + sizeof(u32), GFP_NOFS); + if (!ins_data) + return -ENOMEM; + + ins_sizes = (u32 *)ins_data; + ins_keys = (struct btrfs_key *)(ins_data + nr * sizeof(u32)); + batch.keys = ins_keys; + batch.data_sizes = ins_sizes; + batch.total_data_size = 0; + batch.nr = 0; + + dst_index = 0; + for (int i = 0; i < nr; i++) { + const int src_slot = start_slot + i; + struct btrfs_root *csum_root; + struct btrfs_ordered_sum *sums; + struct btrfs_ordered_sum *sums_next; + LIST_HEAD(ordered_sums); + u64 disk_bytenr; + u64 disk_num_bytes; + u64 extent_offset; + u64 extent_num_bytes; + bool is_old_extent; + + btrfs_item_key_to_cpu(src, &ins_keys[dst_index], src_slot); + + if (ins_keys[dst_index].type != BTRFS_EXTENT_DATA_KEY) + goto add_to_batch; + + extent = btrfs_item_ptr(src, src_slot, + struct btrfs_file_extent_item); + + is_old_extent = (btrfs_file_extent_generation(src, extent) < + trans->transid); + + /* + * Don't copy extents from past generations. That would make us + * log a lot more metadata for common cases like doing only a + * few random writes into a file and then fsync it for the first + * time or after the full sync flag is set on the inode. We can + * get leaves full of extent items, most of which are from past + * generations, so we can skip them - as long as the inode has + * not been the target of a reflink operation in this transaction, + * as in that case it might have had file extent items with old + * generations copied into it. We also must always log prealloc + * extents that start at or beyond eof, otherwise we would lose + * them on log replay. + */ + if (is_old_extent && + ins_keys[dst_index].offset < i_size && + inode->last_reflink_trans < trans->transid) + continue; + + if (skip_csum) + goto add_to_batch; + + /* Only regular extents have checksums. */ + if (btrfs_file_extent_type(src, extent) != BTRFS_FILE_EXTENT_REG) + goto add_to_batch; + + /* + * If it's an extent created in a past transaction, then its + * checksums are already accessible from the committed csum tree, + * no need to log them. + */ + if (is_old_extent) + goto add_to_batch; + + disk_bytenr = btrfs_file_extent_disk_bytenr(src, extent); + /* If it's an explicit hole, there are no checksums. */ + if (disk_bytenr == 0) + goto add_to_batch; + + disk_num_bytes = btrfs_file_extent_disk_num_bytes(src, extent); + + if (btrfs_file_extent_compression(src, extent)) { + extent_offset = 0; + extent_num_bytes = disk_num_bytes; + } else { + extent_offset = btrfs_file_extent_offset(src, extent); + extent_num_bytes = btrfs_file_extent_num_bytes(src, extent); + } + + csum_root = btrfs_csum_root(trans->fs_info, disk_bytenr); + if (unlikely(!csum_root)) { + btrfs_err(trans->fs_info, + "missing csum root for extent at bytenr %llu", + disk_bytenr); + return -EUCLEAN; + } + + disk_bytenr += extent_offset; + ret = btrfs_lookup_csums_list(csum_root, disk_bytenr, + disk_bytenr + extent_num_bytes - 1, + &ordered_sums, false); + if (ret < 0) + return ret; + ret = 0; + + list_for_each_entry_safe(sums, sums_next, &ordered_sums, list) { + if (!ret) + ret = log_csums(trans, inode, log, sums); + list_del(&sums->list); + kfree(sums); + } + if (ret) + return ret; + +add_to_batch: + ins_sizes[dst_index] = btrfs_item_size(src, src_slot); + batch.total_data_size += ins_sizes[dst_index]; + batch.nr++; + dst_index++; + } + + /* + * We have a leaf full of old extent items that don't need to be logged, + * so we don't need to do anything. + */ + if (batch.nr == 0) + return 0; + + ret = btrfs_insert_empty_items(trans, log, dst_path, &batch); + if (ret) + return ret; + + dst_index = 0; + for (int i = 0; i < nr; i++) { + const int src_slot = start_slot + i; + const int dst_slot = dst_path->slots[0] + dst_index; + struct btrfs_key key; + unsigned long src_offset; + unsigned long dst_offset; + + /* + * We're done, all the remaining items in the source leaf + * correspond to old file extent items. + */ + if (dst_index >= batch.nr) + break; + + btrfs_item_key_to_cpu(src, &key, src_slot); + + if (key.type != BTRFS_EXTENT_DATA_KEY) + goto copy_item; + + extent = btrfs_item_ptr(src, src_slot, + struct btrfs_file_extent_item); + + /* See the comment in the previous loop, same logic. */ + if (btrfs_file_extent_generation(src, extent) < trans->transid && + key.offset < i_size && + inode->last_reflink_trans < trans->transid) + continue; + +copy_item: + dst_offset = btrfs_item_ptr_offset(dst_path->nodes[0], dst_slot); + src_offset = btrfs_item_ptr_offset(src, src_slot); + + if (key.type == BTRFS_INODE_ITEM_KEY) { + struct btrfs_inode_item *inode_item; + + inode_item = btrfs_item_ptr(dst_path->nodes[0], dst_slot, + struct btrfs_inode_item); + fill_inode_item(trans, dst_path->nodes[0], inode_item, + inode, log_mode == LOG_INODE_EXISTS, + logged_isize); + } else { + copy_extent_buffer(dst_path->nodes[0], src, dst_offset, + src_offset, ins_sizes[dst_index]); + } + + dst_index++; + } + + btrfs_release_path(dst_path); + + return ret; +} + +static int extent_cmp(void *priv, const struct list_head *a, + const struct list_head *b) +{ + const struct extent_map *em1, *em2; + + em1 = list_entry(a, struct extent_map, list); + em2 = list_entry(b, struct extent_map, list); + + if (em1->start < em2->start) + return -1; + else if (em1->start > em2->start) + return 1; + return 0; +} + +static int log_extent_csums(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_root *log_root, + const struct extent_map *em, + struct btrfs_log_ctx *ctx) +{ + struct btrfs_ordered_extent *ordered; + struct btrfs_root *csum_root; + u64 block_start; + u64 csum_offset; + u64 csum_len; + u64 mod_start = em->start; + u64 mod_len = em->len; + LIST_HEAD(ordered_sums); + int ret = 0; + + if (inode->flags & BTRFS_INODE_NODATASUM || + (em->flags & EXTENT_FLAG_PREALLOC) || + em->disk_bytenr == EXTENT_MAP_HOLE) + return 0; + + list_for_each_entry(ordered, &ctx->ordered_extents, log_list) { + const u64 ordered_end = ordered->file_offset + ordered->num_bytes; + const u64 mod_end = mod_start + mod_len; + struct btrfs_ordered_sum *sums; + + if (mod_len == 0) + break; + + if (ordered_end <= mod_start) + continue; + if (mod_end <= ordered->file_offset) + break; + + /* + * We are going to copy all the csums on this ordered extent, so + * go ahead and adjust mod_start and mod_len in case this ordered + * extent has already been logged. + */ + if (ordered->file_offset > mod_start) { + if (ordered_end >= mod_end) + mod_len = ordered->file_offset - mod_start; + /* + * If we have this case + * + * |--------- logged extent ---------| + * |----- ordered extent ----| + * + * Just don't mess with mod_start and mod_len, we'll + * just end up logging more csums than we need and it + * will be ok. + */ + } else { + if (ordered_end < mod_end) { + mod_len = mod_end - ordered_end; + mod_start = ordered_end; + } else { + mod_len = 0; + } + } + + /* + * To keep us from looping for the above case of an ordered + * extent that falls inside of the logged extent. + */ + if (test_and_set_bit(BTRFS_ORDERED_LOGGED_CSUM, &ordered->flags)) + continue; + + list_for_each_entry(sums, &ordered->csum_list, list) { + ret = log_csums(trans, inode, log_root, sums); + if (ret) + return ret; + } + } + + /* We're done, found all csums in the ordered extents. */ + if (mod_len == 0) + return 0; + + /* If we're compressed we have to save the entire range of csums. */ + if (btrfs_extent_map_is_compressed(em)) { + csum_offset = 0; + csum_len = em->disk_num_bytes; + } else { + csum_offset = mod_start - em->start; + csum_len = mod_len; + } + + /* block start is already adjusted for the file extent offset. */ + block_start = btrfs_extent_map_block_start(em); + csum_root = btrfs_csum_root(trans->fs_info, block_start); + if (unlikely(!csum_root)) { + btrfs_err(trans->fs_info, + "missing csum root for extent at bytenr %llu", + block_start); + return -EUCLEAN; + } + + ret = btrfs_lookup_csums_list(csum_root, block_start + csum_offset, + block_start + csum_offset + csum_len - 1, + &ordered_sums, false); + if (ret < 0) + return ret; + ret = 0; + + while (!list_empty(&ordered_sums)) { + struct btrfs_ordered_sum *sums = list_first_entry(&ordered_sums, + struct btrfs_ordered_sum, + list); + if (!ret) + ret = log_csums(trans, inode, log_root, sums); + list_del(&sums->list); + kfree(sums); + } + + return ret; +} + +static int log_one_extent(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + const struct extent_map *em, + struct btrfs_path *path, + struct btrfs_log_ctx *ctx) +{ + struct btrfs_drop_extents_args drop_args = { 0 }; + struct btrfs_root *log = inode->root->log_root; + struct btrfs_file_extent_item fi = { 0 }; + struct extent_buffer *leaf; + struct btrfs_key key; + enum btrfs_compression_type compress_type; + u64 extent_offset = em->offset; + u64 block_start = btrfs_extent_map_block_start(em); + u64 block_len; + int ret; + + btrfs_set_stack_file_extent_generation(&fi, trans->transid); + if (em->flags & EXTENT_FLAG_PREALLOC) + btrfs_set_stack_file_extent_type(&fi, BTRFS_FILE_EXTENT_PREALLOC); + else + btrfs_set_stack_file_extent_type(&fi, BTRFS_FILE_EXTENT_REG); + + block_len = em->disk_num_bytes; + compress_type = btrfs_extent_map_compression(em); + if (compress_type != BTRFS_COMPRESS_NONE) { + btrfs_set_stack_file_extent_disk_bytenr(&fi, block_start); + btrfs_set_stack_file_extent_disk_num_bytes(&fi, block_len); + } else if (em->disk_bytenr < EXTENT_MAP_LAST_BYTE) { + btrfs_set_stack_file_extent_disk_bytenr(&fi, block_start - extent_offset); + btrfs_set_stack_file_extent_disk_num_bytes(&fi, block_len); + } + + btrfs_set_stack_file_extent_offset(&fi, extent_offset); + btrfs_set_stack_file_extent_num_bytes(&fi, em->len); + btrfs_set_stack_file_extent_ram_bytes(&fi, em->ram_bytes); + btrfs_set_stack_file_extent_compression(&fi, compress_type); + + ret = log_extent_csums(trans, inode, log, em, ctx); + if (ret) + return ret; + + /* + * If this is the first time we are logging the inode in the current + * transaction, we can avoid btrfs_drop_extents(), which is expensive + * because it does a deletion search, which always acquires write locks + * for extent buffers at levels 2, 1 and 0. This not only wastes time + * but also adds significant contention in a log tree, since log trees + * are small, with a root at level 2 or 3 at most, due to their short + * life span. + */ + if (ctx->logged_before) { + drop_args.path = path; + drop_args.start = em->start; + drop_args.end = btrfs_extent_map_end(em); + drop_args.replace_extent = true; + drop_args.extent_item_size = sizeof(fi); + ret = btrfs_drop_extents(trans, log, inode, &drop_args); + if (ret) + return ret; + } + + if (!drop_args.extent_inserted) { + key.objectid = btrfs_ino(inode); + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = em->start; + + ret = btrfs_insert_empty_item(trans, log, path, &key, + sizeof(fi)); + if (ret) + return ret; + } + leaf = path->nodes[0]; + write_extent_buffer(leaf, &fi, + btrfs_item_ptr_offset(leaf, path->slots[0]), + sizeof(fi)); + + btrfs_release_path(path); + + return ret; +} + +/* + * Log all prealloc extents beyond the inode's i_size to make sure we do not + * lose them after doing a full/fast fsync and replaying the log. We scan the + * subvolume's root instead of iterating the inode's extent map tree because + * otherwise we can log incorrect extent items based on extent map conversion. + * That can happen due to the fact that extent maps are merged when they + * are not in the extent map tree's list of modified extents. + */ +static int btrfs_log_prealloc_extents(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_path *path, + struct btrfs_log_ctx *ctx) +{ + struct btrfs_root *root = inode->root; + struct btrfs_key key; + const u64 i_size = i_size_read(&inode->vfs_inode); + const u64 ino = btrfs_ino(inode); + BTRFS_PATH_AUTO_FREE(dst_path); + bool dropped_extents = false; + u64 truncate_offset = i_size; + struct extent_buffer *leaf; + int slot; + int ins_nr = 0; + int start_slot = 0; + int ret; + + if (!(inode->flags & BTRFS_INODE_PREALLOC)) + return 0; + + key.objectid = ino; + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = i_size; + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + goto out; + + /* + * We must check if there is a prealloc extent that starts before the + * i_size and crosses the i_size boundary. This is to ensure later we + * truncate down to the end of that extent and not to the i_size, as + * otherwise we end up losing part of the prealloc extent after a log + * replay and with an implicit hole if there is another prealloc extent + * that starts at an offset beyond i_size. + */ + ret = btrfs_previous_item(root, path, ino, BTRFS_EXTENT_DATA_KEY); + if (ret < 0) + goto out; + + if (ret == 0) { + struct btrfs_file_extent_item *ei; + + leaf = path->nodes[0]; + slot = path->slots[0]; + ei = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item); + + if (btrfs_file_extent_type(leaf, ei) == + BTRFS_FILE_EXTENT_PREALLOC) { + u64 extent_end; + + btrfs_item_key_to_cpu(leaf, &key, slot); + extent_end = key.offset + + btrfs_file_extent_num_bytes(leaf, ei); + + if (extent_end > i_size) + truncate_offset = extent_end; + } + } else { + ret = 0; + } + + while (true) { + leaf = path->nodes[0]; + slot = path->slots[0]; + + if (slot >= btrfs_header_nritems(leaf)) { + if (ins_nr > 0) { + ret = copy_items(trans, inode, dst_path, path, + start_slot, ins_nr, 1, 0, ctx); + if (ret < 0) + goto out; + ins_nr = 0; + } + ret = btrfs_next_leaf(root, path); + if (ret < 0) + goto out; + if (ret > 0) { + ret = 0; + break; + } + continue; + } + + btrfs_item_key_to_cpu(leaf, &key, slot); + if (key.objectid > ino) + break; + if (WARN_ON_ONCE(key.objectid < ino) || + key.type < BTRFS_EXTENT_DATA_KEY || + key.offset < i_size) { + path->slots[0]++; + continue; + } + /* + * Avoid overlapping items in the log tree. The first time we + * get here, get rid of everything from a past fsync. After + * that, if the current extent starts before the end of the last + * extent we copied, truncate the last one. This can happen if + * an ordered extent completion modifies the subvolume tree + * while btrfs_next_leaf() has the tree unlocked. + */ + if (!dropped_extents || key.offset < truncate_offset) { + ret = truncate_inode_items(trans, root->log_root, inode, + min(key.offset, truncate_offset), + BTRFS_EXTENT_DATA_KEY); + if (ret) + goto out; + dropped_extents = true; + } + truncate_offset = btrfs_file_extent_end(path); + if (ins_nr == 0) + start_slot = slot; + ins_nr++; + path->slots[0]++; + if (!dst_path) { + dst_path = btrfs_alloc_path(); + if (!dst_path) { + ret = -ENOMEM; + goto out; + } + } + } + if (ins_nr > 0) + ret = copy_items(trans, inode, dst_path, path, + start_slot, ins_nr, 1, 0, ctx); +out: + btrfs_release_path(path); + return ret; +} + +static int btrfs_log_changed_extents(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_path *path, + struct btrfs_log_ctx *ctx) +{ + struct btrfs_ordered_extent *ordered; + struct btrfs_ordered_extent *tmp; + struct extent_map *em, *n; + LIST_HEAD(extents); + struct extent_map_tree *tree = &inode->extent_tree; + int ret = 0; + int num = 0; + + write_lock(&tree->lock); + + list_for_each_entry_safe(em, n, &tree->modified_extents, list) { + list_del_init(&em->list); + /* + * Just an arbitrary number, this can be really CPU intensive + * once we start getting a lot of extents, and really once we + * have a bunch of extents we just want to commit since it will + * be faster. + */ + if (++num > 32768) { + list_del_init(&tree->modified_extents); + ret = -EFBIG; + goto process; + } + + if (em->generation < trans->transid) + continue; + + /* We log prealloc extents beyond eof later. */ + if ((em->flags & EXTENT_FLAG_PREALLOC) && + em->start >= i_size_read(&inode->vfs_inode)) + continue; + + /* Need a ref to keep it from getting evicted from cache */ + refcount_inc(&em->refs); + em->flags |= EXTENT_FLAG_LOGGING; + list_add_tail(&em->list, &extents); + num++; + } + + list_sort(NULL, &extents, extent_cmp); +process: + while (!list_empty(&extents)) { + em = list_first_entry(&extents, struct extent_map, list); + + list_del_init(&em->list); + + /* + * If we had an error we just need to delete everybody from our + * private list. + */ + if (ret) { + btrfs_clear_em_logging(inode, em); + btrfs_free_extent_map(em); + continue; + } + + write_unlock(&tree->lock); + + ret = log_one_extent(trans, inode, em, path, ctx); + write_lock(&tree->lock); + btrfs_clear_em_logging(inode, em); + btrfs_free_extent_map(em); + } + WARN_ON(!list_empty(&extents)); + write_unlock(&tree->lock); + + if (!ret) + ret = btrfs_log_prealloc_extents(trans, inode, path, ctx); + if (ret) + return ret; + + /* + * We have logged all extents successfully, now make sure the commit of + * the current transaction waits for the ordered extents to complete + * before it commits and wipes out the log trees, otherwise we would + * lose data if an ordered extents completes after the transaction + * commits and a power failure happens after the transaction commit. + */ + list_for_each_entry_safe(ordered, tmp, &ctx->ordered_extents, log_list) { + list_del_init(&ordered->log_list); + set_bit(BTRFS_ORDERED_LOGGED, &ordered->flags); + + if (!test_bit(BTRFS_ORDERED_COMPLETE, &ordered->flags)) { + spin_lock(&inode->ordered_tree_lock); + if (!test_bit(BTRFS_ORDERED_COMPLETE, &ordered->flags)) { + set_bit(BTRFS_ORDERED_PENDING, &ordered->flags); + atomic_inc(&trans->transaction->pending_ordered); + } + spin_unlock(&inode->ordered_tree_lock); + } + btrfs_put_ordered_extent(ordered); + } + + return 0; +} + +static int get_inode_size_to_log(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_path *path, u64 *size_ret) +{ + struct btrfs_key key; + struct btrfs_inode_item *item; + int ret; + + key.objectid = btrfs_ino(inode); + key.type = BTRFS_INODE_ITEM_KEY; + key.offset = 0; + + /* + * Our caller called inode_logged(), so logged_trans is up to date. + * Use data_race() to silence any warning from KCSAN. Once logged_trans + * is set, it can only be reset to 0 after inode eviction. + */ + if (data_race(inode->logged_trans) == trans->transid) { + ret = btrfs_search_slot(NULL, inode->root->log_root, &key, path, 0, 0); + } else if (inode->generation < trans->transid) { + path->search_commit_root = true; + path->skip_locking = true; + ret = btrfs_search_slot(NULL, inode->root, &key, path, 0, 0); + path->search_commit_root = false; + path->skip_locking = false; + + } else { + *size_ret = 0; + return 0; + } + + /* + * If the inode was logged before or is from a past transaction, then + * its inode item must exist in the log root or in the commit root. + */ + ASSERT(ret <= 0); + if (WARN_ON_ONCE(ret > 0)) + ret = -ENOENT; + + if (ret < 0) + return ret; + + item = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_inode_item); + *size_ret = btrfs_inode_size(path->nodes[0], item); + /* + * If the in-memory inode's i_size is smaller then the inode size stored + * in the btree, return the inode's i_size, so that we get a correct + * inode size after replaying the log when before a power failure we had + * a shrinking truncate followed by addition of a new name (rename / new + * hard link). Otherwise return the inode size from the btree, to avoid + * data loss when replaying a log due to previously doing a write that + * expands the inode's size and logging a new name immediately after. + */ + if (*size_ret > inode->vfs_inode.i_size) + *size_ret = inode->vfs_inode.i_size; + + btrfs_release_path(path); + return 0; +} + +/* + * At the moment we always log all xattrs. This is to figure out at log replay + * time which xattrs must have their deletion replayed. If a xattr is missing + * in the log tree and exists in the fs/subvol tree, we delete it. This is + * because if a xattr is deleted, the inode is fsynced and a power failure + * happens, causing the log to be replayed the next time the fs is mounted, + * we want the xattr to not exist anymore (same behaviour as other filesystems + * with a journal, ext3/4, xfs, f2fs, etc). + */ +static int btrfs_log_all_xattrs(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_path *path, + struct btrfs_path *dst_path, + struct btrfs_log_ctx *ctx) +{ + struct btrfs_root *root = inode->root; + int ret; + struct btrfs_key key; + const u64 ino = btrfs_ino(inode); + int ins_nr = 0; + int start_slot = 0; + bool found_xattrs = false; + + if (test_bit(BTRFS_INODE_NO_XATTRS, &inode->runtime_flags)) + return 0; + + key.objectid = ino; + key.type = BTRFS_XATTR_ITEM_KEY; + key.offset = 0; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + return ret; + + while (true) { + int slot = path->slots[0]; + struct extent_buffer *leaf = path->nodes[0]; + int nritems = btrfs_header_nritems(leaf); + + if (slot >= nritems) { + if (ins_nr > 0) { + ret = copy_items(trans, inode, dst_path, path, + start_slot, ins_nr, 1, 0, ctx); + if (ret < 0) + return ret; + ins_nr = 0; + } + ret = btrfs_next_leaf(root, path); + if (ret < 0) + return ret; + else if (ret > 0) + break; + continue; + } + + btrfs_item_key_to_cpu(leaf, &key, slot); + if (key.objectid != ino || key.type != BTRFS_XATTR_ITEM_KEY) + break; + + if (ins_nr == 0) + start_slot = slot; + ins_nr++; + path->slots[0]++; + found_xattrs = true; + cond_resched(); + } + if (ins_nr > 0) { + ret = copy_items(trans, inode, dst_path, path, + start_slot, ins_nr, 1, 0, ctx); + if (ret < 0) + return ret; + } + + if (!found_xattrs) + set_bit(BTRFS_INODE_NO_XATTRS, &inode->runtime_flags); + + return 0; +} + +/* + * When using the NO_HOLES feature if we punched a hole that causes the + * deletion of entire leafs or all the extent items of the first leaf (the one + * that contains the inode item and references) we may end up not processing + * any extents, because there are no leafs with a generation matching the + * current transaction that have extent items for our inode. So we need to find + * if any holes exist and then log them. We also need to log holes after any + * truncate operation that changes the inode's size. + */ +static int btrfs_log_holes(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_path *path) +{ + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_key key; + const u64 ino = btrfs_ino(inode); + const u64 i_size = i_size_read(&inode->vfs_inode); + u64 prev_extent_end = 0; + int ret; + + if (!btrfs_fs_incompat(fs_info, NO_HOLES) || i_size == 0) + return 0; + + /* + * If there are no prealloc extents (which can be located past i_size), + * and disk space used is greater than or equals to i_size, then there + * are no holes. + */ + if (!(inode->flags & BTRFS_INODE_PREALLOC) && + i_size <= inode_get_bytes(&inode->vfs_inode)) + return 0; + + key.objectid = ino; + key.type = BTRFS_EXTENT_DATA_KEY; + key.offset = 0; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + return ret; + + while (true) { + struct extent_buffer *leaf = path->nodes[0]; + + if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) { + ret = btrfs_next_leaf(root, path); + if (ret < 0) + return ret; + if (ret > 0) { + ret = 0; + break; + } + leaf = path->nodes[0]; + } + + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + if (key.objectid != ino || key.type != BTRFS_EXTENT_DATA_KEY) + break; + + /* We have a hole, log it. */ + if (prev_extent_end < key.offset) { + const u64 hole_len = key.offset - prev_extent_end; + + /* + * Release the path to avoid deadlocks with other code + * paths that search the root while holding locks on + * leafs from the log root. + */ + btrfs_release_path(path); + ret = btrfs_insert_hole_extent(trans, root->log_root, + ino, prev_extent_end, + hole_len); + if (ret < 0) + return ret; + + /* + * Search for the same key again in the root. Since it's + * an extent item and we are holding the inode lock, the + * key must still exist. If it doesn't just emit warning + * and return an error to fall back to a transaction + * commit. + */ + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + return ret; + if (WARN_ON(ret > 0)) + return -ENOENT; + leaf = path->nodes[0]; + } + + prev_extent_end = btrfs_file_extent_end(path); + path->slots[0]++; + cond_resched(); + } + + if (prev_extent_end < i_size) { + u64 hole_len; + + btrfs_release_path(path); + hole_len = ALIGN(i_size - prev_extent_end, fs_info->sectorsize); + ret = btrfs_insert_hole_extent(trans, root->log_root, ino, + prev_extent_end, hole_len); + if (ret < 0) + return ret; + } + + return 0; +} + +/* + * When we are logging a new inode X, check if it doesn't have a reference that + * matches the reference from some other inode Y created in a past transaction + * and that was renamed in the current transaction. If we don't do this, then at + * log replay time we can lose inode Y (and all its files if it's a directory): + * + * mkdir /mnt/x + * echo "hello world" > /mnt/x/foobar + * sync + * mv /mnt/x /mnt/y + * mkdir /mnt/x # or touch /mnt/x + * xfs_io -c fsync /mnt/x + * + * mount fs, trigger log replay + * + * After the log replay procedure, we would lose the first directory and all its + * files (file foobar). + * For the case where inode Y is not a directory we simply end up losing it: + * + * echo "123" > /mnt/foo + * sync + * mv /mnt/foo /mnt/bar + * echo "abc" > /mnt/foo + * xfs_io -c fsync /mnt/foo + * + * + * We also need this for cases where a snapshot entry is replaced by some other + * entry (file or directory) otherwise we end up with an unreplayable log due to + * attempts to delete the snapshot entry (entry of type BTRFS_ROOT_ITEM_KEY) as + * if it were a regular entry: + * + * mkdir /mnt/x + * btrfs subvolume snapshot /mnt /mnt/x/snap + * btrfs subvolume delete /mnt/x/snap + * rmdir /mnt/x + * mkdir /mnt/x + * fsync /mnt/x or fsync some new file inside it + * + * + * The snapshot delete, rmdir of x, mkdir of a new x and the fsync all happen in + * the same transaction. + */ +static int btrfs_check_ref_name_override(struct extent_buffer *eb, + const int slot, + const struct btrfs_key *key, + struct btrfs_inode *inode, + u64 *other_ino, u64 *other_parent) +{ + BTRFS_PATH_AUTO_FREE(search_path); + char AUTO_KFREE(name); + u32 name_len = 0; + u32 item_size = btrfs_item_size(eb, slot); + u32 cur_offset = 0; + unsigned long ptr = btrfs_item_ptr_offset(eb, slot); + + search_path = btrfs_alloc_path(); + if (!search_path) + return -ENOMEM; + search_path->search_commit_root = true; + search_path->skip_locking = true; + + while (cur_offset < item_size) { + u64 parent; + u32 this_name_len; + u32 this_len; + unsigned long name_ptr; + struct btrfs_dir_item *di; + struct fscrypt_str name_str; + + if (key->type == BTRFS_INODE_REF_KEY) { + struct btrfs_inode_ref *iref; + + iref = (struct btrfs_inode_ref *)(ptr + cur_offset); + parent = key->offset; + this_name_len = btrfs_inode_ref_name_len(eb, iref); + name_ptr = (unsigned long)(iref + 1); + this_len = sizeof(*iref) + this_name_len; + } else { + struct btrfs_inode_extref *extref; + + extref = (struct btrfs_inode_extref *)(ptr + + cur_offset); + parent = btrfs_inode_extref_parent(eb, extref); + this_name_len = btrfs_inode_extref_name_len(eb, extref); + name_ptr = (unsigned long)&extref->name; + this_len = sizeof(*extref) + this_name_len; + } + + if (this_name_len > name_len) { + char *new_name; + + new_name = krealloc(name, this_name_len, GFP_NOFS); + if (!new_name) + return -ENOMEM; + name_len = this_name_len; + name = new_name; + } + + read_extent_buffer(eb, name, name_ptr, this_name_len); + + name_str.name = name; + name_str.len = this_name_len; + di = btrfs_lookup_dir_item(NULL, inode->root, search_path, + parent, &name_str, 0); + if (!IS_ERR_OR_NULL(di)) { + struct btrfs_key di_key; + + btrfs_dir_item_key_to_cpu(search_path->nodes[0], + di, &di_key); + if (di_key.type == BTRFS_INODE_ITEM_KEY) { + if (di_key.objectid != key->objectid) { + *other_ino = di_key.objectid; + *other_parent = parent; + return 1; + } else { + return 0; + } + } else { + return -EAGAIN; + } + } else if (IS_ERR(di)) { + return PTR_ERR(di); + } + btrfs_release_path(search_path); + + cur_offset += this_len; + } + + return 0; +} + +/* + * Check if we need to log an inode. This is used in contexts where while + * logging an inode we need to log another inode (either that it exists or in + * full mode). This is used instead of btrfs_inode_in_log() because the later + * requires the inode to be in the log and have the log transaction committed, + * while here we do not care if the log transaction was already committed - our + * caller will commit the log later - and we want to avoid logging an inode + * multiple times when multiple tasks have joined the same log transaction. + */ +static bool need_log_inode(const struct btrfs_trans_handle *trans, + struct btrfs_inode *inode) +{ + /* + * If a directory was not modified, no dentries added or removed, we can + * and should avoid logging it. + */ + if (S_ISDIR(inode->vfs_inode.i_mode) && inode->last_trans < trans->transid) + return false; + + /* + * If this inode does not have new/updated/deleted xattrs since the last + * time it was logged and is flagged as logged in the current transaction, + * we can skip logging it. As for new/deleted names, those are updated in + * the log by link/unlink/rename operations. + * In case the inode was logged and then evicted and reloaded, its + * logged_trans will be 0, in which case we have to fully log it since + * logged_trans is a transient field, not persisted. + */ + if (inode_logged(trans, inode, NULL) == 1 && + !test_bit(BTRFS_INODE_COPY_EVERYTHING, &inode->runtime_flags)) + return false; + + return true; +} + +struct btrfs_dir_list { + u64 ino; + struct list_head list; +}; + +/* + * Log the inodes of the new dentries of a directory. + * See process_dir_items_leaf() for details about why it is needed. + * This is a recursive operation - if an existing dentry corresponds to a + * directory, that directory's new entries are logged too (same behaviour as + * ext3/4, xfs, f2fs, nilfs2). Note that when logging the inodes + * the dentries point to we do not acquire their VFS lock, otherwise lockdep + * complains about the following circular lock dependency / possible deadlock: + * + * CPU0 CPU1 + * ---- ---- + * lock(&type->i_mutex_dir_key#3/2); + * lock(sb_internal#2); + * lock(&type->i_mutex_dir_key#3/2); + * lock(&sb->s_type->i_mutex_key#14); + * + * Where sb_internal is the lock (a counter that works as a lock) acquired by + * sb_start_intwrite() in btrfs_start_transaction(). + * Not acquiring the VFS lock of the inodes is still safe because: + * + * 1) For regular files we log with a mode of LOG_INODE_EXISTS. It's possible + * that while logging the inode new references (names) are added or removed + * from the inode, leaving the logged inode item with a link count that does + * not match the number of logged inode reference items. This is fine because + * at log replay time we compute the real number of links and correct the + * link count in the inode item (see replay_one_buffer() and + * link_to_fixup_dir()); + * + * 2) For directories we log with a mode of LOG_INODE_ALL. It's possible that + * while logging the inode's items new index items (key type + * BTRFS_DIR_INDEX_KEY) are added to fs/subvol tree and the logged inode item + * has a size that doesn't match the sum of the lengths of all the logged + * names - this is ok, not a problem, because at log replay time we set the + * directory's i_size to the correct value (see replay_one_name() and + * overwrite_item()). + */ +static int log_new_dir_dentries(struct btrfs_trans_handle *trans, + struct btrfs_inode *start_inode, + struct btrfs_log_ctx *ctx) +{ + struct btrfs_root *root = start_inode->root; + struct btrfs_path *path; + LIST_HEAD(dir_list); + struct btrfs_dir_list *dir_elem; + u64 ino = btrfs_ino(start_inode); + struct btrfs_inode *curr_inode = start_inode; + int ret = 0; + + trace_btrfs_log_new_dir_dentries_enter(trans, start_inode); + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto out; + } + + /* Pairs with btrfs_add_delayed_iput below. */ + ihold(&curr_inode->vfs_inode); + + while (true) { + struct btrfs_key key; + struct btrfs_key found_key; + u64 next_index; + bool continue_curr_inode = true; + int iter_ret; + + key.objectid = ino; + key.type = BTRFS_DIR_INDEX_KEY; + key.offset = btrfs_get_first_dir_index_to_log(curr_inode); + next_index = key.offset; +again: + btrfs_for_each_slot(root->log_root, &key, &found_key, path, iter_ret) { + struct extent_buffer *leaf = path->nodes[0]; + struct btrfs_dir_item *di; + struct btrfs_key di_key; + struct btrfs_inode *di_inode; + int log_mode = LOG_INODE_EXISTS; + int type; + + if (found_key.objectid != ino || + found_key.type != BTRFS_DIR_INDEX_KEY) { + continue_curr_inode = false; + break; + } + + next_index = found_key.offset + 1; + + di = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dir_item); + type = btrfs_dir_ftype(leaf, di); + if (btrfs_dir_transid(leaf, di) < trans->transid) + continue; + btrfs_dir_item_key_to_cpu(leaf, di, &di_key); + if (di_key.type == BTRFS_ROOT_ITEM_KEY) + continue; + + btrfs_release_path(path); + di_inode = btrfs_iget_logging(di_key.objectid, root); + if (IS_ERR(di_inode)) { + ret = PTR_ERR(di_inode); + goto out; + } + + if (!need_log_inode(trans, di_inode)) { + btrfs_add_delayed_iput(di_inode); + break; + } + + ctx->log_new_dentries = false; + if (type == BTRFS_FT_DIR) + log_mode = LOG_INODE_ALL; + ret = btrfs_log_inode(trans, di_inode, log_mode, ctx); + btrfs_add_delayed_iput(di_inode); + if (ret) + goto out; + if (ctx->log_new_dentries) { + dir_elem = kmalloc_obj(*dir_elem, GFP_NOFS); + if (!dir_elem) { + ret = -ENOMEM; + goto out; + } + dir_elem->ino = di_key.objectid; + list_add_tail(&dir_elem->list, &dir_list); + } + break; + } + + btrfs_release_path(path); + + if (iter_ret < 0) { + ret = iter_ret; + goto out; + } else if (iter_ret > 0) { + continue_curr_inode = false; + } else { + key = found_key; + } + + if (continue_curr_inode && key.offset < (u64)-1) { + key.offset++; + goto again; + } + + btrfs_set_first_dir_index_to_log(curr_inode, next_index); + + if (list_empty(&dir_list)) + break; + + dir_elem = list_first_entry(&dir_list, struct btrfs_dir_list, list); + ino = dir_elem->ino; + list_del(&dir_elem->list); + kfree(dir_elem); + + btrfs_add_delayed_iput(curr_inode); + + curr_inode = btrfs_iget_logging(ino, root); + if (IS_ERR(curr_inode)) { + ret = PTR_ERR(curr_inode); + curr_inode = NULL; + break; + } + } +out: + btrfs_free_path(path); + if (curr_inode) + btrfs_add_delayed_iput(curr_inode); + + if (ret) { + struct btrfs_dir_list *next; + + list_for_each_entry_safe(dir_elem, next, &dir_list, list) + kfree(dir_elem); + } + + trace_btrfs_log_new_dir_dentries_exit(trans, start_inode, ret); + + return ret; +} + +struct btrfs_ino_list { + u64 ino; + u64 parent; + struct list_head list; +}; + +static void free_conflicting_inodes(struct btrfs_log_ctx *ctx) +{ + struct btrfs_ino_list *curr; + struct btrfs_ino_list *next; + + list_for_each_entry_safe(curr, next, &ctx->conflict_inodes, list) { + list_del(&curr->list); + kfree(curr); + } +} + +static int conflicting_inode_is_dir(struct btrfs_root *root, u64 ino, + struct btrfs_path *path) +{ + struct btrfs_key key; + int ret; + + key.objectid = ino; + key.type = BTRFS_INODE_ITEM_KEY; + key.offset = 0; + + path->search_commit_root = true; + path->skip_locking = true; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (WARN_ON_ONCE(ret > 0)) { + /* + * We have previously found the inode through the commit root + * so this should not happen. If it does, just error out and + * fallback to a transaction commit. + */ + ret = -ENOENT; + } else if (ret == 0) { + struct btrfs_inode_item *item; + + item = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_inode_item); + if (S_ISDIR(btrfs_inode_mode(path->nodes[0], item))) + ret = 1; + } + + btrfs_release_path(path); + path->search_commit_root = false; + path->skip_locking = false; + + return ret; +} + +static bool can_log_conflicting_inode(const struct btrfs_trans_handle *trans, + const struct btrfs_inode *inode) +{ + if (!S_ISDIR(inode->vfs_inode.i_mode)) + return true; + + if (inode->last_unlink_trans < trans->transid) + return true; + + /* + * If this is a directory and its unlink_trans is not from a past + * transaction then we must fallback to a transaction commit in order + * to avoid getting a directory with 2 hard links after log replay. + * + * This happens if a directory A is renamed, moved from one parent + * directory to another one, a new file is created in the old parent + * directory with the old name of our directory A, the new file is + * fsynced, then we moved the new file to some other parent directory + * and fsync again the new file. This results in a log tree where we + * logged that directory A existed, with the INODE_REF item for the + * new location but without having logged its old parent inode, so + * that on log replay we add a new link for the new location but the + * old link remains, resulting in a link count of 2. + */ + return false; +} + +static int add_conflicting_inode(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + u64 ino, u64 parent, + struct btrfs_log_ctx *ctx) +{ + struct btrfs_ino_list *ino_elem; + struct btrfs_inode *inode; + int ret = 0; + + trace_btrfs_add_conflicting_inode_enter(trans, ctx, ino, parent); + + /* + * It's rare to have a lot of conflicting inodes, in practice it is not + * common to have more than 1 or 2. We don't want to collect too many, + * as we could end up logging too many inodes (even if only in + * LOG_INODE_EXISTS mode) and slow down other fsyncs or transaction + * commits. + */ + if (ctx->num_conflict_inodes >= MAX_CONFLICT_INODES) { + ret = BTRFS_LOG_FORCE_COMMIT; + goto out; + } + + inode = btrfs_iget_logging(ino, root); + /* + * If the other inode that had a conflicting dir entry was deleted in + * the current transaction then we either: + * + * 1) Log the parent directory (later after adding it to the list) if + * the inode is a directory. This is because it may be a deleted + * subvolume/snapshot or it may be a regular directory that had + * deleted subvolumes/snapshots (or subdirectories that had them), + * and at the moment we can't deal with dropping subvolumes/snapshots + * during log replay. So we just log the parent, which will result in + * a fallback to a transaction commit if we are dealing with those + * cases (last_unlink_trans will match the current transaction); + * + * 2) Do nothing if it's not a directory. During log replay we simply + * unlink the conflicting dentry from the parent directory and then + * add the dentry for our inode. Like this we can avoid logging the + * parent directory (and maybe fallback to a transaction commit in + * case it has a last_unlink_trans == trans->transid, due to moving + * some inode from it to some other directory). + */ + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + if (ret != -ENOENT) + goto out; + + ret = conflicting_inode_is_dir(root, ino, path); + /* Not a directory or we got an error. */ + if (ret <= 0) + goto out; + + /* Conflicting inode is a directory, so we'll log its parent. */ + ino_elem = kmalloc_obj(*ino_elem, GFP_NOFS); + if (!ino_elem) { + ret = -ENOMEM; + goto out; + } + ino_elem->ino = ino; + ino_elem->parent = parent; + list_add_tail(&ino_elem->list, &ctx->conflict_inodes); + ctx->num_conflict_inodes++; + ret = 0; + goto out; + } + + /* + * If the inode was already logged skip it - otherwise we can hit an + * infinite loop. Example: + * + * From the commit root (previous transaction) we have the following + * inodes: + * + * inode 257 a directory + * inode 258 with references "zz" and "zz_link" on inode 257 + * inode 259 with reference "a" on inode 257 + * + * And in the current (uncommitted) transaction we have: + * + * inode 257 a directory, unchanged + * inode 258 with references "a" and "a2" on inode 257 + * inode 259 with reference "zz_link" on inode 257 + * inode 261 with reference "zz" on inode 257 + * + * When logging inode 261 the following infinite loop could + * happen if we don't skip already logged inodes: + * + * - we detect inode 258 as a conflicting inode, with inode 261 + * on reference "zz", and log it; + * + * - we detect inode 259 as a conflicting inode, with inode 258 + * on reference "a", and log it; + * + * - we detect inode 258 as a conflicting inode, with inode 259 + * on reference "zz_link", and log it - again! After this we + * repeat the above steps forever. + * + * Here we can use need_log_inode() because we only need to log the + * inode in LOG_INODE_EXISTS mode and rename operations update the log, + * so that the log ends up with the new name and without the old name. + */ + if (!need_log_inode(trans, inode)) { + btrfs_add_delayed_iput(inode); + goto out; + } + + if (!can_log_conflicting_inode(trans, inode)) { + btrfs_add_delayed_iput(inode); + ret = BTRFS_LOG_FORCE_COMMIT; + goto out; + } + + btrfs_add_delayed_iput(inode); + + ino_elem = kmalloc_obj(*ino_elem, GFP_NOFS); + if (!ino_elem) { + ret = -ENOMEM; + goto out; + } + ino_elem->ino = ino; + ino_elem->parent = parent; + list_add_tail(&ino_elem->list, &ctx->conflict_inodes); + ctx->num_conflict_inodes++; + +out: + trace_btrfs_add_conflicting_inode_exit(trans, ctx, ino, parent, ret); + + return ret; +} + +static int log_conflicting_inodes(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_log_ctx *ctx) +{ + const bool orig_log_new_dentries = ctx->log_new_dentries; + int ret = 0; + + /* + * Conflicting inodes are logged by the first call to btrfs_log_inode(), + * otherwise we could have unbounded recursion of btrfs_log_inode() + * calls. This check guarantees we can have only 1 level of recursion. + */ + if (ctx->logging_conflict_inodes) + return 0; + + /* + * Avoid any work if no conflicting inodes and emitting the trace event + * which only adds noise and it's useless if there are no inodes. + */ + if (list_empty(&ctx->conflict_inodes)) + return 0; + + ctx->logging_conflict_inodes = true; + trace_btrfs_log_conflicting_inodes_enter(trans, ctx); + + /* + * New conflicting inodes may be found and added to the list while we + * are logging a conflicting inode, so keep iterating while the list is + * not empty. + */ + while (!list_empty(&ctx->conflict_inodes)) { + struct btrfs_ino_list *curr; + struct btrfs_inode *inode; + u64 ino; + u64 parent; + + curr = list_first_entry(&ctx->conflict_inodes, + struct btrfs_ino_list, list); + ino = curr->ino; + parent = curr->parent; + list_del(&curr->list); + kfree(curr); + + inode = btrfs_iget_logging(ino, root); + /* + * If the other inode that had a conflicting dir entry was + * deleted in the current transaction, we need to log its parent + * directory. See the comment at add_conflicting_inode(). + */ + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + if (ret != -ENOENT) + break; + + inode = btrfs_iget_logging(parent, root); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + break; + } + + if (!can_log_conflicting_inode(trans, inode)) { + btrfs_add_delayed_iput(inode); + ret = BTRFS_LOG_FORCE_COMMIT; + break; + } + + /* + * Always log the directory, we cannot make this + * conditional on need_log_inode() because the directory + * might have been logged in LOG_INODE_EXISTS mode or + * the dir index of the conflicting inode is not in a + * dir index key range logged for the directory. So we + * must make sure the deletion is recorded. + */ + ctx->log_new_dentries = false; + ret = btrfs_log_inode(trans, inode, LOG_INODE_ALL, ctx); + if (!ret && ctx->log_new_dentries) + ret = log_new_dir_dentries(trans, inode, ctx); + + btrfs_add_delayed_iput(inode); + if (ret) + break; + continue; + } + + /* + * Here we can use need_log_inode() because we only need to log + * the inode in LOG_INODE_EXISTS mode and rename operations + * update the log, so that the log ends up with the new name and + * without the old name. + * + * We did this check at add_conflicting_inode(), but here we do + * it again because if some other task logged the inode after + * that, we can avoid doing it again. + */ + if (!need_log_inode(trans, inode)) { + btrfs_add_delayed_iput(inode); + continue; + } + + /* + * We are safe logging the other inode without acquiring its + * lock as long as we log with the LOG_INODE_EXISTS mode. We + * are safe against concurrent renames of the other inode as + * well because during a rename we pin the log and update the + * log with the new name before we unpin it. + */ + ret = btrfs_log_inode(trans, inode, LOG_INODE_EXISTS, ctx); + btrfs_add_delayed_iput(inode); + if (ret) + break; + } + + ctx->log_new_dentries = orig_log_new_dentries; + ctx->logging_conflict_inodes = false; + if (ret) + free_conflicting_inodes(ctx); + trace_btrfs_log_conflicting_inodes_exit(trans, ctx, ret); + + return ret; +} + +static int copy_inode_items_to_log(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_key *min_key, + const struct btrfs_key *max_key, + struct btrfs_path *path, + struct btrfs_path *dst_path, + const u64 logged_isize, + const enum btrfs_log_mode log_mode, + struct btrfs_log_ctx *ctx, + bool *need_log_inode_item) +{ + const u64 i_size = i_size_read(&inode->vfs_inode); + struct btrfs_root *root = inode->root; + int ins_start_slot = 0; + int ins_nr = 0; + int ret; + + while (1) { + ret = btrfs_search_forward(root, min_key, path, trans->transid); + if (ret < 0) + return ret; + if (ret > 0) { + ret = 0; + break; + } +again: + /* Note, ins_nr might be > 0 here, cleanup outside the loop */ + if (min_key->objectid != max_key->objectid) + break; + if (min_key->type > max_key->type) + break; + + if (min_key->type == BTRFS_INODE_ITEM_KEY) { + *need_log_inode_item = false; + } else if (min_key->type == BTRFS_EXTENT_DATA_KEY && + min_key->offset >= i_size) { + /* + * Extents at and beyond eof are logged with + * btrfs_log_prealloc_extents(). + * Only regular files have BTRFS_EXTENT_DATA_KEY keys, + * and no keys greater than that, so bail out. + */ + break; + } else if (min_key->type == BTRFS_INODE_REF_KEY || + min_key->type == BTRFS_INODE_EXTREF_KEY) { + u64 other_ino = 0; + u64 other_parent = 0; + + ret = btrfs_check_ref_name_override(path->nodes[0], + path->slots[0], min_key, inode, + &other_ino, &other_parent); + if (ret < 0) { + return ret; + } else if (ret > 0 && + other_ino != btrfs_ino(ctx->inode)) { + if (ins_nr > 0) { + ins_nr++; + } else { + ins_nr = 1; + ins_start_slot = path->slots[0]; + } + ret = copy_items(trans, inode, dst_path, path, + ins_start_slot, ins_nr, + log_mode, logged_isize, ctx); + if (ret < 0) + return ret; + ins_nr = 0; + + btrfs_release_path(path); + ret = add_conflicting_inode(trans, root, path, + other_ino, + other_parent, ctx); + if (ret) + return ret; + goto next_key; + } + } else if (min_key->type == BTRFS_XATTR_ITEM_KEY) { + /* Skip xattrs, logged later with btrfs_log_all_xattrs() */ + if (ins_nr == 0) + goto next_slot; + ret = copy_items(trans, inode, dst_path, path, + ins_start_slot, + ins_nr, log_mode, logged_isize, ctx); + if (ret < 0) + return ret; + ins_nr = 0; + goto next_slot; + } + + if (ins_nr && ins_start_slot + ins_nr == path->slots[0]) { + ins_nr++; + goto next_slot; + } else if (!ins_nr) { + ins_start_slot = path->slots[0]; + ins_nr = 1; + goto next_slot; + } + + ret = copy_items(trans, inode, dst_path, path, ins_start_slot, + ins_nr, log_mode, logged_isize, ctx); + if (ret < 0) + return ret; + ins_nr = 1; + ins_start_slot = path->slots[0]; +next_slot: + path->slots[0]++; + if (path->slots[0] < btrfs_header_nritems(path->nodes[0])) { + btrfs_item_key_to_cpu(path->nodes[0], min_key, + path->slots[0]); + goto again; + } + if (ins_nr) { + ret = copy_items(trans, inode, dst_path, path, + ins_start_slot, ins_nr, log_mode, + logged_isize, ctx); + if (ret < 0) + return ret; + ins_nr = 0; + } + btrfs_release_path(path); +next_key: + if (min_key->offset < (u64)-1) { + min_key->offset++; + } else if (min_key->type < max_key->type) { + min_key->type++; + min_key->offset = 0; + } else { + break; + } + + /* + * We may process many leaves full of items for our inode, so + * avoid monopolizing a cpu for too long by rescheduling while + * not holding locks on any tree. + */ + cond_resched(); + } + if (ins_nr) { + ret = copy_items(trans, inode, dst_path, path, ins_start_slot, + ins_nr, log_mode, logged_isize, ctx); + if (ret) + return ret; + } + + if (log_mode == LOG_INODE_ALL && S_ISREG(inode->vfs_inode.i_mode)) { + /* + * Release the path because otherwise we might attempt to double + * lock the same leaf with btrfs_log_prealloc_extents() below. + */ + btrfs_release_path(path); + ret = btrfs_log_prealloc_extents(trans, inode, dst_path, ctx); + } + + return ret; +} + +static int insert_delayed_items_batch(struct btrfs_trans_handle *trans, + struct btrfs_root *log, + struct btrfs_path *path, + const struct btrfs_item_batch *batch, + const struct btrfs_delayed_item *first_item) +{ + const struct btrfs_delayed_item *curr = first_item; + int ret; + + ret = btrfs_insert_empty_items(trans, log, path, batch); + if (ret) + return ret; + + for (int i = 0; i < batch->nr; i++) { + char *data_ptr; + + data_ptr = btrfs_item_ptr(path->nodes[0], path->slots[0], char); + write_extent_buffer(path->nodes[0], &curr->data, + (unsigned long)data_ptr, curr->data_len); + curr = list_next_entry(curr, log_list); + path->slots[0]++; + } + + btrfs_release_path(path); + + return 0; +} + +static int log_delayed_insertion_items(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_path *path, + const struct list_head *delayed_ins_list, + struct btrfs_log_ctx *ctx) +{ + /* 195 (4095 bytes of keys and sizes) fits in a single 4K page. */ + const int max_batch_size = 195; + const int leaf_data_size = BTRFS_LEAF_DATA_SIZE(trans->fs_info); + const u64 ino = btrfs_ino(inode); + struct btrfs_root *log = inode->root->log_root; + struct btrfs_item_batch batch = { + .nr = 0, + .total_data_size = 0, + }; + const struct btrfs_delayed_item *first = NULL; + const struct btrfs_delayed_item *curr; + char *ins_data; + struct btrfs_key *ins_keys; + u32 *ins_sizes; + u64 curr_batch_size = 0; + int batch_idx = 0; + int ret; + + /* We are adding dir index items to the log tree. */ + lockdep_assert_held(&inode->log_mutex); + + /* + * We collect delayed items before copying index keys from the subvolume + * to the log tree. However just after we collected them, they may have + * been flushed (all of them or just some of them), and therefore we + * could have copied them from the subvolume tree to the log tree. + * So find the first delayed item that was not yet logged (they are + * sorted by index number). + */ + list_for_each_entry(curr, delayed_ins_list, log_list) { + if (curr->index > inode->last_dir_index_offset) { + first = curr; + break; + } + } + + /* Empty list or all delayed items were already logged. */ + if (!first) + return 0; + + ins_data = kmalloc_array(max_batch_size, sizeof(u32) + sizeof(struct btrfs_key), GFP_NOFS); + if (!ins_data) + return -ENOMEM; + ins_sizes = (u32 *)ins_data; + batch.data_sizes = ins_sizes; + ins_keys = (struct btrfs_key *)(ins_data + max_batch_size * sizeof(u32)); + batch.keys = ins_keys; + + curr = first; + while (!list_entry_is_head(curr, delayed_ins_list, log_list)) { + const u32 curr_size = curr->data_len + sizeof(struct btrfs_item); + + if (curr_batch_size + curr_size > leaf_data_size || + batch.nr == max_batch_size) { + ret = insert_delayed_items_batch(trans, log, path, + &batch, first); + if (ret) + goto out; + batch_idx = 0; + batch.nr = 0; + batch.total_data_size = 0; + curr_batch_size = 0; + first = curr; + } + + ins_sizes[batch_idx] = curr->data_len; + ins_keys[batch_idx].objectid = ino; + ins_keys[batch_idx].type = BTRFS_DIR_INDEX_KEY; + ins_keys[batch_idx].offset = curr->index; + curr_batch_size += curr_size; + batch.total_data_size += curr->data_len; + batch.nr++; + batch_idx++; + curr = list_next_entry(curr, log_list); + } + + ASSERT(batch.nr >= 1, "batch.nr=%d", batch.nr); + ret = insert_delayed_items_batch(trans, log, path, &batch, first); + + curr = list_last_entry(delayed_ins_list, struct btrfs_delayed_item, + log_list); + inode->last_dir_index_offset = curr->index; +out: + kfree(ins_data); + + return ret; +} + +static int log_delayed_deletions_full(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_path *path, + const struct list_head *delayed_del_list, + struct btrfs_log_ctx *ctx) +{ + const u64 ino = btrfs_ino(inode); + const struct btrfs_delayed_item *curr; + + curr = list_first_entry(delayed_del_list, struct btrfs_delayed_item, + log_list); + + while (!list_entry_is_head(curr, delayed_del_list, log_list)) { + u64 first_dir_index = curr->index; + u64 last_dir_index; + const struct btrfs_delayed_item *next; + int ret; + + /* + * Find a range of consecutive dir index items to delete. Like + * this we log a single dir range item spanning several contiguous + * dir items instead of logging one range item per dir index item. + */ + next = list_next_entry(curr, log_list); + while (!list_entry_is_head(next, delayed_del_list, log_list)) { + if (next->index != curr->index + 1) + break; + curr = next; + next = list_next_entry(next, log_list); + } + + last_dir_index = curr->index; + ASSERT(last_dir_index >= first_dir_index, + "last_dir_index=%llu first_dir_index=%llu", + last_dir_index, first_dir_index); + + ret = insert_dir_log_key(trans, inode->root->log_root, path, + ino, first_dir_index, last_dir_index); + if (ret) + return ret; + curr = list_next_entry(curr, log_list); + } + + return 0; +} + +static int batch_delete_dir_index_items(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_path *path, + const struct list_head *delayed_del_list, + const struct btrfs_delayed_item *first, + const struct btrfs_delayed_item **last_ret) +{ + const struct btrfs_delayed_item *next; + struct extent_buffer *leaf = path->nodes[0]; + const int last_slot = btrfs_header_nritems(leaf) - 1; + int slot = path->slots[0] + 1; + const u64 ino = btrfs_ino(inode); + + next = list_next_entry(first, log_list); + + while (slot < last_slot && + !list_entry_is_head(next, delayed_del_list, log_list)) { + struct btrfs_key key; + + btrfs_item_key_to_cpu(leaf, &key, slot); + if (key.objectid != ino || + key.type != BTRFS_DIR_INDEX_KEY || + key.offset != next->index) + break; + + slot++; + *last_ret = next; + next = list_next_entry(next, log_list); + } + + return btrfs_del_items(trans, inode->root->log_root, path, + path->slots[0], slot - path->slots[0]); +} + +static int log_delayed_deletions_incremental(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_path *path, + const struct list_head *delayed_del_list, + struct btrfs_log_ctx *ctx) +{ + struct btrfs_root *log = inode->root->log_root; + const struct btrfs_delayed_item *curr; + u64 last_range_start = 0; + u64 last_range_end = 0; + struct btrfs_key key; + + key.objectid = btrfs_ino(inode); + key.type = BTRFS_DIR_INDEX_KEY; + curr = list_first_entry(delayed_del_list, struct btrfs_delayed_item, + log_list); + + while (!list_entry_is_head(curr, delayed_del_list, log_list)) { + const struct btrfs_delayed_item *last = curr; + u64 first_dir_index = curr->index; + u64 last_dir_index; + bool deleted_items = false; + int ret; + + key.offset = curr->index; + ret = btrfs_search_slot(trans, log, &key, path, -1, 1); + if (ret < 0) { + return ret; + } else if (ret == 0) { + ret = batch_delete_dir_index_items(trans, inode, path, + delayed_del_list, curr, + &last); + if (ret) + return ret; + deleted_items = true; + } + + btrfs_release_path(path); + + /* + * If we deleted items from the leaf, it means we have a range + * item logging their range, so no need to add one or update an + * existing one. Otherwise we have to log a dir range item. + */ + if (deleted_items) + goto next_batch; + + last_dir_index = last->index; + ASSERT(last_dir_index >= first_dir_index, + "last_dir_index=%llu first_dir_index=%llu", + last_dir_index, first_dir_index); + /* + * If this range starts right after where the previous one ends, + * then we want to reuse the previous range item and change its + * end offset to the end of this range. This is just to minimize + * leaf space usage, by avoiding adding a new range item. + */ + if (last_range_end != 0 && first_dir_index == last_range_end + 1) + first_dir_index = last_range_start; + + ret = insert_dir_log_key(trans, log, path, key.objectid, + first_dir_index, last_dir_index); + if (ret) + return ret; + + last_range_start = first_dir_index; + last_range_end = last_dir_index; +next_batch: + curr = list_next_entry(last, log_list); + } + + return 0; +} + +static int log_delayed_deletion_items(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_path *path, + const struct list_head *delayed_del_list, + struct btrfs_log_ctx *ctx) +{ + /* + * We are deleting dir index items from the log tree or adding range + * items to it. + */ + lockdep_assert_held(&inode->log_mutex); + + if (list_empty(delayed_del_list)) + return 0; + + if (ctx->logged_before) + return log_delayed_deletions_incremental(trans, inode, path, + delayed_del_list, ctx); + + return log_delayed_deletions_full(trans, inode, path, delayed_del_list, + ctx); +} + +/* + * Similar logic as for log_new_dir_dentries(), but it iterates over the delayed + * items instead of the subvolume tree. + */ +static int log_new_delayed_dentries(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + const struct list_head *delayed_ins_list, + struct btrfs_log_ctx *ctx) +{ + const bool orig_log_new_dentries = ctx->log_new_dentries; + struct btrfs_delayed_item *item; + int ret = 0; + + /* + * No need for the log mutex, plus to avoid potential deadlocks or + * lockdep annotations due to nesting of delayed inode mutexes and log + * mutexes. + */ + lockdep_assert_not_held(&inode->log_mutex); + + ASSERT(!ctx->logging_new_delayed_dentries); + + /* + * Return early if empty list, avoid emitting redundant trace events + * that generate noise only. + */ + if (list_empty(delayed_ins_list)) + return 0; + + trace_btrfs_log_new_delayed_dentries_enter(trans, inode); + ctx->logging_new_delayed_dentries = true; + + list_for_each_entry(item, delayed_ins_list, log_list) { + struct btrfs_dir_item *dir_item; + struct btrfs_inode *di_inode; + struct btrfs_key key; + int log_mode = LOG_INODE_EXISTS; + + dir_item = (struct btrfs_dir_item *)item->data; + btrfs_disk_key_to_cpu(&key, &dir_item->location); + + if (key.type == BTRFS_ROOT_ITEM_KEY) + continue; + + di_inode = btrfs_iget_logging(key.objectid, inode->root); + if (IS_ERR(di_inode)) { + ret = PTR_ERR(di_inode); + break; + } + + if (!need_log_inode(trans, di_inode)) { + btrfs_add_delayed_iput(di_inode); + continue; + } + + if (btrfs_stack_dir_ftype(dir_item) == BTRFS_FT_DIR) + log_mode = LOG_INODE_ALL; + + ctx->log_new_dentries = false; + ret = btrfs_log_inode(trans, di_inode, log_mode, ctx); + + if (!ret && ctx->log_new_dentries) + ret = log_new_dir_dentries(trans, di_inode, ctx); + + btrfs_add_delayed_iput(di_inode); + + if (ret) + break; + } + + ctx->log_new_dentries = orig_log_new_dentries; + ctx->logging_new_delayed_dentries = false; + trace_btrfs_log_new_delayed_dentries_exit(trans, inode, ret); + + return ret; +} + +/* log a single inode in the tree log. + * At least one parent directory for this inode must exist in the tree + * or be logged already. + * + * Any items from this inode changed by the current transaction are copied + * to the log tree. An extra reference is taken on any extents in this + * file, allowing us to avoid a whole pile of corner cases around logging + * blocks that have been removed from the tree. + * + * See LOG_INODE_ALL and related defines for a description of what inode_only + * does. + * + * This handles both files and directories. + */ +static int btrfs_log_inode(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + enum btrfs_log_mode log_mode, + struct btrfs_log_ctx *ctx) +{ + struct btrfs_path *path; + struct btrfs_path *dst_path = NULL; + struct btrfs_key min_key; + struct btrfs_key max_key; + struct btrfs_root *log = inode->root->log_root; + int ret; + bool fast_search = false; + u64 ino = btrfs_ino(inode); + struct extent_map_tree *em_tree = &inode->extent_tree; + u64 logged_isize = 0; + bool need_log_inode_item = true; + bool xattrs_logged = false; + bool inode_item_dropped = true; + bool full_dir_logging = false; + LIST_HEAD(delayed_ins_list); + LIST_HEAD(delayed_del_list); + + trace_btrfs_log_inode_enter(trans, inode, ctx, log_mode); + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto out; + } + dst_path = btrfs_alloc_path(); + if (!dst_path) { + ret = -ENOMEM; + goto out; + } + + min_key.objectid = ino; + min_key.type = BTRFS_INODE_ITEM_KEY; + min_key.offset = 0; + + max_key.objectid = ino; + + + /* today the code can only do partial logging of directories */ + if (S_ISDIR(inode->vfs_inode.i_mode) || + (!test_bit(BTRFS_INODE_NEEDS_FULL_SYNC, + &inode->runtime_flags) && + log_mode >= LOG_INODE_EXISTS)) + max_key.type = BTRFS_XATTR_ITEM_KEY; + else + max_key.type = (u8)-1; + max_key.offset = (u64)-1; + + if (S_ISDIR(inode->vfs_inode.i_mode) && log_mode == LOG_INODE_ALL) + full_dir_logging = true; + + /* + * If we are logging a directory while we are logging dentries of the + * delayed items of some other inode, then we need to flush the delayed + * items of this directory and not log the delayed items directly. This + * is to prevent more than one level of recursion into btrfs_log_inode() + * by having something like this: + * + * $ mkdir -p a/b/c/d/e/f/g/h/... + * $ xfs_io -c "fsync" a + * + * Where all directories in the path did not exist before and are + * created in the current transaction. + * So in such a case we directly log the delayed items of the main + * directory ("a") without flushing them first, while for each of its + * subdirectories we flush their delayed items before logging them. + * This prevents a potential unbounded recursion like this: + * + * btrfs_log_inode() + * log_new_delayed_dentries() + * btrfs_log_inode() + * log_new_delayed_dentries() + * btrfs_log_inode() + * log_new_delayed_dentries() + * (...) + * + * We have thresholds for the maximum number of delayed items to have in + * memory, and once they are hit, the items are flushed asynchronously. + * However the limit is quite high, so lets prevent deep levels of + * recursion to happen by limiting the maximum depth to be 1. + */ + if (full_dir_logging && ctx->logging_new_delayed_dentries) { + ret = btrfs_commit_inode_delayed_items(trans, inode); + if (ret) + goto out; + } + + mutex_lock(&inode->log_mutex); + + /* + * For symlinks, we must always log their content, which is stored in an + * inline extent, otherwise we could end up with an empty symlink after + * log replay, which is invalid on linux (symlink(2) returns -ENOENT if + * one attempts to create an empty symlink). + * We don't need to worry about flushing delalloc, because when we create + * the inline extent when the symlink is created (we never have delalloc + * for symlinks). + */ + if (S_ISLNK(inode->vfs_inode.i_mode)) + log_mode = LOG_INODE_ALL; + + /* + * Before logging the inode item, cache the value returned by + * inode_logged(), because after that we have the need to figure out if + * the inode was previously logged in this transaction. + */ + ret = inode_logged(trans, inode, path); + if (ret < 0) + goto out_unlock; + ctx->logged_before = (ret == 1); + ret = 0; + + /* + * This is for cases where logging a directory could result in losing a + * a file after replaying the log. For example, if we move a file from a + * directory A to a directory B, then fsync directory A, we have no way + * to known the file was moved from A to B, so logging just A would + * result in losing the file after a log replay. + */ + if (full_dir_logging && inode->last_unlink_trans >= trans->transid) { + ret = BTRFS_LOG_FORCE_COMMIT; + goto out_unlock; + } + + /* + * a brute force approach to making sure we get the most uptodate + * copies of everything. + */ + if (S_ISDIR(inode->vfs_inode.i_mode)) { + clear_bit(BTRFS_INODE_COPY_EVERYTHING, &inode->runtime_flags); + if (ctx->logged_before) + ret = drop_inode_items(trans, log, path, inode, + BTRFS_XATTR_ITEM_KEY); + } else { + if (log_mode == LOG_INODE_EXISTS) { + /* + * Make sure the new inode item we write to the log has + * the same isize as the current one (if it exists). + * This is necessary to prevent data loss after log + * replay, and also to prevent doing a wrong expanding + * truncate - for e.g. create file, write 4K into offset + * 0, fsync, write 4K into offset 4096, add hard link, + * fsync some other file (to sync log), power fail - if + * we use the inode's current i_size, after log replay + * we get a 8Kb file, with the last 4Kb extent as a hole + * (zeroes), as if an expanding truncate happened, + * instead of getting a file of 4Kb only. + */ + ret = get_inode_size_to_log(trans, inode, path, &logged_isize); + if (ret) + goto out_unlock; + } + if (test_bit(BTRFS_INODE_NEEDS_FULL_SYNC, + &inode->runtime_flags)) { + if (log_mode == LOG_INODE_EXISTS) { + max_key.type = BTRFS_XATTR_ITEM_KEY; + if (ctx->logged_before) + ret = drop_inode_items(trans, log, path, + inode, max_key.type); + } else { + clear_bit(BTRFS_INODE_NEEDS_FULL_SYNC, + &inode->runtime_flags); + clear_bit(BTRFS_INODE_COPY_EVERYTHING, + &inode->runtime_flags); + if (ctx->logged_before) + ret = truncate_inode_items(trans, log, + inode, 0, 0); + } + } else if (test_and_clear_bit(BTRFS_INODE_COPY_EVERYTHING, + &inode->runtime_flags) || + log_mode == LOG_INODE_EXISTS) { + if (log_mode == LOG_INODE_ALL) + fast_search = true; + max_key.type = BTRFS_XATTR_ITEM_KEY; + if (ctx->logged_before) + ret = drop_inode_items(trans, log, path, inode, + max_key.type); + } else { + if (log_mode == LOG_INODE_ALL) + fast_search = true; + inode_item_dropped = false; + goto log_extents; + } + + } + if (ret) + goto out_unlock; + + /* + * If we are logging a directory in full mode, collect the delayed items + * before iterating the subvolume tree, so that we don't miss any new + * dir index items in case they get flushed while or right after we are + * iterating the subvolume tree. + */ + if (full_dir_logging && !ctx->logging_new_delayed_dentries) + btrfs_log_get_delayed_items(inode, &delayed_ins_list, + &delayed_del_list); + + /* + * If we are fsyncing a file with 0 hard links, then commit the delayed + * inode because the last inode ref (or extref) item may still be in the + * subvolume tree and if we log it the file will still exist after a log + * replay. So commit the delayed inode to delete that last ref and we + * skip logging it. + */ + if (inode->vfs_inode.i_nlink == 0) { + ret = btrfs_commit_inode_delayed_inode(inode); + if (ret) + goto out_unlock; + } + + ret = copy_inode_items_to_log(trans, inode, &min_key, &max_key, + path, dst_path, logged_isize, + log_mode, ctx, &need_log_inode_item); + if (ret) + goto out_unlock; + + btrfs_release_path(path); + btrfs_release_path(dst_path); + ret = btrfs_log_all_xattrs(trans, inode, path, dst_path, ctx); + if (ret) + goto out_unlock; + xattrs_logged = true; + if (max_key.type >= BTRFS_EXTENT_DATA_KEY && !fast_search) { + btrfs_release_path(path); + btrfs_release_path(dst_path); + ret = btrfs_log_holes(trans, inode, path); + if (ret) + goto out_unlock; + } +log_extents: + btrfs_release_path(path); + btrfs_release_path(dst_path); + if (need_log_inode_item) { + ret = log_inode_item(trans, log, dst_path, inode, inode_item_dropped); + if (ret) + goto out_unlock; + /* + * If we are doing a fast fsync and the inode was logged before + * in this transaction, we don't need to log the xattrs because + * they were logged before. If xattrs were added, changed or + * deleted since the last time we logged the inode, then we have + * already logged them because the inode had the runtime flag + * BTRFS_INODE_COPY_EVERYTHING set. + */ + if (!xattrs_logged && inode->logged_trans < trans->transid) { + ret = btrfs_log_all_xattrs(trans, inode, path, dst_path, ctx); + if (ret) + goto out_unlock; + btrfs_release_path(path); + } + } + if (fast_search) { + ret = btrfs_log_changed_extents(trans, inode, dst_path, ctx); + if (ret) + goto out_unlock; + } else if (log_mode == LOG_INODE_ALL) { + struct extent_map *em, *n; + + write_lock(&em_tree->lock); + list_for_each_entry_safe(em, n, &em_tree->modified_extents, list) + list_del_init(&em->list); + write_unlock(&em_tree->lock); + } + + if (full_dir_logging) { + ret = log_directory_changes(trans, inode, path, dst_path, ctx); + if (ret) + goto out_unlock; + ret = log_delayed_insertion_items(trans, inode, path, + &delayed_ins_list, ctx); + if (ret) + goto out_unlock; + ret = log_delayed_deletion_items(trans, inode, path, + &delayed_del_list, ctx); + if (ret) + goto out_unlock; + } + + spin_lock(&inode->lock); + inode->logged_trans = trans->transid; + /* + * Don't update last_log_commit if we logged that an inode exists. + * We do this for three reasons: + * + * 1) We might have had buffered writes to this inode that were + * flushed and had their ordered extents completed in this + * transaction, but we did not previously log the inode with + * LOG_INODE_ALL. Later the inode was evicted and after that + * it was loaded again and this LOG_INODE_EXISTS log operation + * happened. We must make sure that if an explicit fsync against + * the inode is performed later, it logs the new extents, an + * updated inode item, etc, and syncs the log. The same logic + * applies to direct IO writes instead of buffered writes. + * + * 2) When we log the inode with LOG_INODE_EXISTS, its inode item + * is logged with an i_size of 0 or whatever value was logged + * before. If later the i_size of the inode is increased by a + * truncate operation, the log is synced through an fsync of + * some other inode and then finally an explicit fsync against + * this inode is made, we must make sure this fsync logs the + * inode with the new i_size, the hole between old i_size and + * the new i_size, and syncs the log. + * + * 3) If we are logging that an ancestor inode exists as part of + * logging a new name from a link or rename operation, don't update + * its last_log_commit - otherwise if an explicit fsync is made + * against an ancestor, the fsync considers the inode in the log + * and doesn't sync the log, resulting in the ancestor missing after + * a power failure unless the log was synced as part of an fsync + * against any other unrelated inode. + */ + if (!ctx->logging_new_name && log_mode != LOG_INODE_EXISTS) + inode->last_log_commit = inode->last_sub_trans; + spin_unlock(&inode->lock); + + /* + * Reset the last_reflink_trans so that the next fsync does not need to + * go through the slower path when logging extents and their checksums. + */ + if (log_mode == LOG_INODE_ALL) + inode->last_reflink_trans = 0; + +out_unlock: + mutex_unlock(&inode->log_mutex); +out: + btrfs_free_path(path); + btrfs_free_path(dst_path); + + if (ret) + free_conflicting_inodes(ctx); + else + ret = log_conflicting_inodes(trans, inode->root, ctx); + + if (full_dir_logging && !ctx->logging_new_delayed_dentries) { + if (!ret) + ret = log_new_delayed_dentries(trans, inode, + &delayed_ins_list, ctx); + + btrfs_log_put_delayed_items(inode, &delayed_ins_list, + &delayed_del_list); + } + + trace_btrfs_log_inode_exit(trans, inode, ret); + + return ret; +} + +static int btrfs_log_all_parents(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct btrfs_log_ctx *ctx) +{ + int ret; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + struct btrfs_root *root = inode->root; + const u64 ino = btrfs_ino(inode); + + trace_btrfs_log_all_parents_enter(trans, inode); + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto out; + } + path->skip_locking = true; + path->search_commit_root = true; + + key.objectid = ino; + key.type = BTRFS_INODE_REF_KEY; + key.offset = 0; + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + goto out; + + while (true) { + struct extent_buffer *leaf = path->nodes[0]; + int slot = path->slots[0]; + u32 cur_offset = 0; + u32 item_size; + unsigned long ptr; + + if (slot >= btrfs_header_nritems(leaf)) { + ret = btrfs_next_leaf(root, path); + if (ret < 0) + goto out; + if (ret > 0) { + ret = 0; + break; + } + continue; + } + + btrfs_item_key_to_cpu(leaf, &key, slot); + /* BTRFS_INODE_EXTREF_KEY is BTRFS_INODE_REF_KEY + 1 */ + if (key.objectid != ino || key.type > BTRFS_INODE_EXTREF_KEY) + break; + + item_size = btrfs_item_size(leaf, slot); + ptr = btrfs_item_ptr_offset(leaf, slot); + while (cur_offset < item_size) { + u64 dir_id; + struct btrfs_inode *dir_inode; + + if (key.type == BTRFS_INODE_EXTREF_KEY) { + struct btrfs_inode_extref *extref; + + extref = (struct btrfs_inode_extref *) + (ptr + cur_offset); + dir_id = btrfs_inode_extref_parent(leaf, extref); + cur_offset += sizeof(*extref); + cur_offset += btrfs_inode_extref_name_len(leaf, + extref); + } else { + dir_id = key.offset; + cur_offset = item_size; + } + + dir_inode = btrfs_iget_logging(dir_id, root); + /* + * If the parent inode was deleted, return an error to + * fallback to a transaction commit. This is to prevent + * getting an inode that was moved from one parent A to + * a parent B, got its former parent A deleted and then + * it got fsync'ed, from existing at both parents after + * a log replay (and the old parent still existing). + * Example: + * + * mkdir /mnt/A + * mkdir /mnt/B + * touch /mnt/B/bar + * sync + * mv /mnt/B/bar /mnt/A/bar + * mv -T /mnt/A /mnt/B + * fsync /mnt/B/bar + * + * + * If we ignore the old parent B which got deleted, + * after a log replay we would have file bar linked + * at both parents and the old parent B would still + * exist. + */ + if (IS_ERR(dir_inode)) { + ret = PTR_ERR(dir_inode); + goto out; + } + + if (!need_log_inode(trans, dir_inode)) { + btrfs_add_delayed_iput(dir_inode); + continue; + } + + ctx->log_new_dentries = false; + ret = btrfs_log_inode(trans, dir_inode, LOG_INODE_ALL, ctx); + if (!ret && ctx->log_new_dentries) + ret = log_new_dir_dentries(trans, dir_inode, ctx); + btrfs_add_delayed_iput(dir_inode); + if (ret) + goto out; + } + path->slots[0]++; + } +out: + trace_btrfs_log_all_parents_exit(trans, inode, ret); + + return ret; +} + +static int log_new_ancestors(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + struct btrfs_log_ctx *ctx) +{ + struct btrfs_key found_key; + + btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]); + + while (true) { + struct extent_buffer *leaf; + int slot; + struct btrfs_key search_key; + struct btrfs_inode *inode; + u64 ino; + int ret = 0; + + btrfs_release_path(path); + + ino = found_key.offset; + + search_key.objectid = found_key.offset; + search_key.type = BTRFS_INODE_ITEM_KEY; + search_key.offset = 0; + inode = btrfs_iget_logging(ino, root); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + if (inode->generation >= trans->transid && + need_log_inode(trans, inode)) + ret = btrfs_log_inode(trans, inode, LOG_INODE_EXISTS, ctx); + btrfs_add_delayed_iput(inode); + if (ret) + return ret; + + if (search_key.objectid == BTRFS_FIRST_FREE_OBJECTID) + break; + + search_key.type = BTRFS_INODE_REF_KEY; + ret = btrfs_search_slot(NULL, root, &search_key, path, 0, 0); + if (ret < 0) + return ret; + + leaf = path->nodes[0]; + slot = path->slots[0]; + if (slot >= btrfs_header_nritems(leaf)) { + ret = btrfs_next_leaf(root, path); + if (ret < 0) + return ret; + else if (ret > 0) + return -ENOENT; + leaf = path->nodes[0]; + slot = path->slots[0]; + } + + btrfs_item_key_to_cpu(leaf, &found_key, slot); + if (found_key.objectid != search_key.objectid || + found_key.type != BTRFS_INODE_REF_KEY) + return -ENOENT; + } + return 0; +} + +static int log_new_ancestors_fast(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct dentry *parent, + struct btrfs_log_ctx *ctx) +{ + struct btrfs_root *root = inode->root; + struct dentry *old_parent = NULL; + struct super_block *sb = inode->vfs_inode.i_sb; + int ret = 0; + + while (true) { + if (!parent || d_really_is_negative(parent) || + sb != parent->d_sb) + break; + + inode = BTRFS_I(d_inode(parent)); + if (root != inode->root) + break; + + if (inode->generation >= trans->transid && + need_log_inode(trans, inode)) { + ret = btrfs_log_inode(trans, inode, + LOG_INODE_EXISTS, ctx); + if (ret) + break; + } + if (IS_ROOT(parent)) + break; + + parent = dget_parent(parent); + dput(old_parent); + old_parent = parent; + } + dput(old_parent); + + return ret; +} + +static int log_all_new_ancestors(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct dentry *parent, + struct btrfs_log_ctx *ctx) +{ + struct btrfs_root *root = inode->root; + const u64 ino = btrfs_ino(inode); + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key search_key; + int ret; + + trace_btrfs_log_all_new_ancestors_enter(trans, inode); + + /* + * For a single hard link case, go through a fast path that does not + * need to iterate the fs/subvolume tree. + */ + if (inode->vfs_inode.i_nlink < 2) { + ret = log_new_ancestors_fast(trans, inode, parent, ctx); + goto out; + } + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto out; + } + + search_key.objectid = ino; + search_key.type = BTRFS_INODE_REF_KEY; + search_key.offset = 0; +again: + ret = btrfs_search_slot(NULL, root, &search_key, path, 0, 0); + if (ret < 0) + goto out; + if (ret == 0) + path->slots[0]++; + + while (true) { + struct extent_buffer *leaf = path->nodes[0]; + int slot = path->slots[0]; + struct btrfs_key found_key; + + if (slot >= btrfs_header_nritems(leaf)) { + ret = btrfs_next_leaf(root, path); + if (ret < 0) + goto out; + if (ret > 0) { + ret = 0; + break; + } + continue; + } + + btrfs_item_key_to_cpu(leaf, &found_key, slot); + if (found_key.objectid != ino || + found_key.type > BTRFS_INODE_EXTREF_KEY) + break; + + /* + * Don't deal with extended references because they are rare + * cases and too complex to deal with (we would need to keep + * track of which subitem we are processing for each item in + * this loop, etc). So just return some error to fallback to + * a transaction commit. + */ + if (found_key.type == BTRFS_INODE_EXTREF_KEY) { + ret = -EMLINK; + goto out; + } + + /* + * Logging ancestors needs to do more searches on the fs/subvol + * tree, so it releases the path as needed to avoid deadlocks. + * Keep track of the last inode ref key and resume from that key + * after logging all new ancestors for the current hard link. + */ + memcpy(&search_key, &found_key, sizeof(search_key)); + + ret = log_new_ancestors(trans, root, path, ctx); + if (ret) + goto out; + btrfs_release_path(path); + goto again; + } +out: + trace_btrfs_log_all_new_ancestors_exit(trans, inode, ret); + return ret; +} + +/* + * helper function around btrfs_log_inode to make sure newly created + * parent directories also end up in the log. A minimal inode and backref + * only logging is done of any parent directories that are older than + * the last committed transaction + */ +static int btrfs_log_inode_parent(struct btrfs_trans_handle *trans, + struct btrfs_inode *inode, + struct dentry *parent, + enum btrfs_log_mode log_mode, + struct btrfs_log_ctx *ctx) +{ + struct btrfs_root *root = inode->root; + struct btrfs_fs_info *fs_info = root->fs_info; + int ret = 0; + bool log_dentries; + + trace_btrfs_log_inode_parent_enter(trans, inode); + + if (btrfs_test_opt(fs_info, NOTREELOG)) { + ret = BTRFS_LOG_FORCE_COMMIT; + goto out; + } + + if (btrfs_root_refs(&root->root_item) == 0) { + ret = BTRFS_LOG_FORCE_COMMIT; + goto out; + } + + /* + * If we're logging an inode from a subvolume created in the current + * transaction we must force a commit since the root is not persisted. + */ + if (btrfs_root_generation(&root->root_item) == trans->transid) { + ret = BTRFS_LOG_FORCE_COMMIT; + goto out; + } + + /* Skip already logged inodes and without new extents. */ + if (btrfs_inode_in_log(inode, trans->transid) && + list_empty(&ctx->ordered_extents)) { + ret = BTRFS_NO_LOG_SYNC; + goto out; + } + + ret = start_log_trans(trans, root, ctx); + if (ret) + goto out; + + ret = btrfs_log_inode(trans, inode, log_mode, ctx); + if (ret) + goto end_trans; + + /* + * for regular files, if its inode is already on disk, we don't + * have to worry about the parents at all. This is because + * we can use the last_unlink_trans field to record renames + * and other fun in this file. + */ + if (S_ISREG(inode->vfs_inode.i_mode) && + inode->generation < trans->transid && + inode->last_unlink_trans < trans->transid) { + ret = 0; + goto end_trans; + } + + /* + * Track if we need to log dentries because ctx->log_new_dentries can + * be modified in the call chains below. + */ + log_dentries = ctx->log_new_dentries; + + /* + * On unlink we must make sure all our current and old parent directory + * inodes are fully logged. This is to prevent leaving dangling + * directory index entries in directories that were our parents but are + * not anymore. Not doing this results in old parent directory being + * impossible to delete after log replay (rmdir will always fail with + * error -ENOTEMPTY). + * + * Example 1: + * + * mkdir testdir + * touch testdir/foo + * ln testdir/foo testdir/bar + * sync + * unlink testdir/bar + * xfs_io -c fsync testdir/foo + * + * mount fs, triggers log replay + * + * If we don't log the parent directory (testdir), after log replay the + * directory still has an entry pointing to the file inode using the bar + * name, but a matching BTRFS_INODE_[REF|EXTREF]_KEY does not exist and + * the file inode has a link count of 1. + * + * Example 2: + * + * mkdir testdir + * touch foo + * ln foo testdir/foo2 + * ln foo testdir/foo3 + * sync + * unlink testdir/foo3 + * xfs_io -c fsync foo + * + * mount fs, triggers log replay + * + * Similar as the first example, after log replay the parent directory + * testdir still has an entry pointing to the inode file with name foo3 + * but the file inode does not have a matching BTRFS_INODE_REF_KEY item + * and has a link count of 2. + */ + if (inode->last_unlink_trans >= trans->transid) { + ret = btrfs_log_all_parents(trans, inode, ctx); + if (ret) + goto end_trans; + } + + ret = log_all_new_ancestors(trans, inode, parent, ctx); + if (ret) + goto end_trans; + + if (log_dentries) + ret = log_new_dir_dentries(trans, inode, ctx); +end_trans: + if (ret < 0) { + btrfs_set_log_full_commit(trans); + ret = BTRFS_LOG_FORCE_COMMIT; + } + + if (ret) + btrfs_remove_log_ctx(root, ctx); + btrfs_end_log_trans(root); + +out: + trace_btrfs_log_inode_parent_exit(trans, inode, ret); + + return ret; +} + +/* + * it is not safe to log dentry if the chunk root has added new + * chunks. This returns 0 if the dentry was logged, and 1 otherwise. + * If this returns 1, you must commit the transaction to safely get your + * data on disk. + */ +int btrfs_log_dentry_safe(struct btrfs_trans_handle *trans, + struct dentry *dentry, + struct btrfs_log_ctx *ctx) +{ + struct dentry *parent = dget_parent(dentry); + int ret; + + ret = btrfs_log_inode_parent(trans, BTRFS_I(d_inode(dentry)), parent, + LOG_INODE_ALL, ctx); + dput(parent); + + return ret; +} + +/* + * should be called during mount to recover any replay any log trees + * from the FS + */ +int btrfs_recover_log_trees(struct btrfs_root *log_root_tree) +{ + int ret; + struct btrfs_path *path; + struct btrfs_trans_handle *trans; + struct btrfs_key key; + struct btrfs_fs_info *fs_info = log_root_tree->fs_info; + struct walk_control wc = { + .process_func = process_one_buffer, + .stage = LOG_WALK_PIN_ONLY, + }; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + set_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags); + + trans = btrfs_start_transaction(fs_info->tree_root, 0); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto error; + } + + wc.trans = trans; + wc.pin = true; + wc.log = log_root_tree; + + ret = walk_log_tree(&wc); + wc.log = NULL; + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto error; + } + +again: + key.objectid = BTRFS_TREE_LOG_OBJECTID; + key.type = BTRFS_ROOT_ITEM_KEY; + key.offset = (u64)-1; + + while (1) { + struct btrfs_key found_key; + + ret = btrfs_search_slot(NULL, log_root_tree, &key, path, 0, 0); + + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + goto error; + } + if (ret > 0) { + if (path->slots[0] == 0) + break; + path->slots[0]--; + } + btrfs_item_key_to_cpu(path->nodes[0], &found_key, + path->slots[0]); + btrfs_release_path(path); + if (found_key.objectid != BTRFS_TREE_LOG_OBJECTID) + break; + + wc.log = btrfs_read_tree_root(log_root_tree, &found_key); + if (IS_ERR(wc.log)) { + ret = PTR_ERR(wc.log); + wc.log = NULL; + btrfs_abort_transaction(trans, ret); + goto error; + } + + wc.root = btrfs_get_fs_root(fs_info, found_key.offset, true); + if (IS_ERR(wc.root)) { + ret = PTR_ERR(wc.root); + wc.root = NULL; + if (unlikely(ret != -ENOENT)) { + btrfs_abort_transaction(trans, ret); + goto error; + } + + /* + * We didn't find the subvol, likely because it was + * deleted. This is ok, simply skip this log and go to + * the next one. + * + * We need to exclude the root because we can't have + * other log replays overwriting this log as we'll read + * it back in a few more times. This will keep our + * block from being modified, and we'll just bail for + * each subsequent pass. + */ + ret = btrfs_pin_extent_for_log_replay(trans, wc.log->node); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto error; + } + goto next; + } + + wc.root->log_root = wc.log; + ret = btrfs_record_root_in_trans(trans, wc.root); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto next; + } + + ret = walk_log_tree(&wc); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto next; + } + + if (wc.stage == LOG_WALK_REPLAY_ALL) { + struct btrfs_root *root = wc.root; + + wc.subvol_path = path; + ret = fixup_inode_link_counts(&wc); + wc.subvol_path = NULL; + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto next; + } + /* + * We have just replayed everything, and the highest + * objectid of fs roots probably has changed in case + * some inode_item's got replayed. + * + * root->objectid_mutex is not acquired as log replay + * could only happen during mount. + */ + ret = btrfs_init_root_free_objectid(root); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto next; + } + } +next: + if (wc.root) { + wc.root->log_root = NULL; + btrfs_put_root(wc.root); + } + btrfs_put_root(wc.log); + wc.log = NULL; + + if (ret) + goto error; + if (found_key.offset == 0) + break; + key.offset = found_key.offset - 1; + } + btrfs_release_path(path); + + /* step one is to pin it all, step two is to replay just inodes */ + if (wc.pin) { + wc.pin = false; + wc.process_func = replay_one_buffer; + wc.stage = LOG_WALK_REPLAY_INODES; + goto again; + } + /* step three is to replay everything */ + if (wc.stage < LOG_WALK_REPLAY_ALL) { + wc.stage++; + goto again; + } + + btrfs_free_path(path); + + /* step 4: commit the transaction, which also unpins the blocks */ + ret = btrfs_commit_transaction(trans); + if (ret) + return ret; + + clear_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags); + + return 0; +error: + if (wc.trans) + btrfs_end_transaction(wc.trans); + btrfs_put_root(wc.log); + clear_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags); + btrfs_free_path(path); + return ret; +} + +/* + * there are some corner cases where we want to force a full + * commit instead of allowing a directory to be logged. + * + * They revolve around files there were unlinked from the directory, and + * this function updates the parent directory so that a full commit is + * properly done if it is fsync'd later after the unlinks are done. + * + * Must be called before the unlink operations (updates to the subvolume tree, + * inodes, etc) are done. + */ +void btrfs_record_unlink_dir(struct btrfs_trans_handle *trans, + struct btrfs_inode *dir, struct btrfs_inode *inode, + bool for_rename) +{ + trace_btrfs_record_unlink_dir(trans, dir, inode, for_rename); + + /* + * when we're logging a file, if it hasn't been renamed + * or unlinked, and its inode is fully committed on disk, + * we don't have to worry about walking up the directory chain + * to log its parents. + * + * So, we use the last_unlink_trans field to put this transid + * into the file. When the file is logged we check it and + * don't log the parents if the file is fully on disk. + */ + mutex_lock(&inode->log_mutex); + inode->last_unlink_trans = trans->transid; + mutex_unlock(&inode->log_mutex); + + if (!for_rename) + return; + + /* + * If this directory was already logged, any new names will be logged + * with btrfs_log_new_name() and old names will be deleted from the log + * tree with btrfs_del_dir_entries_in_log() or with + * btrfs_del_inode_ref_in_log(). + */ + if (inode_logged(trans, dir, NULL) == 1) + return; + + /* + * If the inode we're about to unlink was logged before, the log will be + * properly updated with the new name with btrfs_log_new_name() and the + * old name removed with btrfs_del_dir_entries_in_log() or with + * btrfs_del_inode_ref_in_log(). + */ + if (inode_logged(trans, inode, NULL) == 1) + return; + + /* + * when renaming files across directories, if the directory + * there we're unlinking from gets fsync'd later on, there's + * no way to find the destination directory later and fsync it + * properly. So, we have to be conservative and force commits + * so the new name gets discovered. + */ + mutex_lock(&dir->log_mutex); + dir->last_unlink_trans = trans->transid; + mutex_unlock(&dir->log_mutex); +} + +/* + * Make sure that if someone attempts to fsync the parent directory of a deleted + * snapshot, it ends up triggering a transaction commit. This is to guarantee + * that after replaying the log tree of the parent directory's root we will not + * see the snapshot anymore and at log replay time we will not see any log tree + * corresponding to the deleted snapshot's root, which could lead to replaying + * it after replaying the log tree of the parent directory (which would replay + * the snapshot delete operation). + * + * Must be called before the actual snapshot destroy operation (updates to the + * parent root and tree of tree roots trees, etc) are done. + */ +void btrfs_record_snapshot_destroy(struct btrfs_trans_handle *trans, + struct btrfs_inode *dir) +{ + trace_btrfs_record_snapshot_destroy(trans, dir); + + mutex_lock(&dir->log_mutex); + dir->last_unlink_trans = trans->transid; + mutex_unlock(&dir->log_mutex); +} + +/* + * Call this when creating a subvolume in a directory. + * Because we don't commit a transaction when creating a subvolume, we can't + * allow the directory pointing to the subvolume to be logged with an entry that + * points to an unpersisted root if we are still in the transaction used to + * create the subvolume, so make any attempt to log the directory to result in a + * full log sync. + * Also we don't need to worry with renames, since btrfs_rename() marks the log + * for full commit when renaming a subvolume. + * + * Must be called before creating the subvolume entry in its parent directory. + */ +void btrfs_record_new_subvolume(const struct btrfs_trans_handle *trans, + struct btrfs_inode *dir) +{ + trace_btrfs_record_new_subvolume(trans, dir); + + mutex_lock(&dir->log_mutex); + dir->last_unlink_trans = trans->transid; + mutex_unlock(&dir->log_mutex); +} + +/* + * Update the log after adding a new name for an inode. + * + * @trans: Transaction handle. + * @old_dentry: The dentry associated with the old name and the old + * parent directory. + * @old_dir: The inode of the previous parent directory for the case + * of a rename. For a link operation, it must be NULL. + * @old_dir_index: The index number associated with the old name, meaningful + * only for rename operations (when @old_dir is not NULL). + * Ignored for link operations. + * @parent: The dentry associated with the directory under which the + * new name is located. + * + * Call this after adding a new name for an inode, as a result of a link or + * rename operation, and it will properly update the log to reflect the new name. + */ +void btrfs_log_new_name(struct btrfs_trans_handle *trans, + struct dentry *old_dentry, struct btrfs_inode *old_dir, + u64 old_dir_index, struct dentry *parent) +{ + struct btrfs_inode *inode = BTRFS_I(d_inode(old_dentry)); + struct btrfs_root *root = inode->root; + struct btrfs_log_ctx ctx; + bool log_pinned = false; + int ret; + + trace_btrfs_log_new_name_enter(trans, inode, old_dir, old_dir_index); + + /* The inode has a new name (ref/extref), so make sure we log it. */ + set_bit(BTRFS_INODE_COPY_EVERYTHING, &inode->runtime_flags); + + btrfs_init_log_ctx(&ctx, inode); + ctx.logging_new_name = true; + + /* + * this will force the logging code to walk the dentry chain + * up for the file + */ + if (!S_ISDIR(inode->vfs_inode.i_mode)) + inode->last_unlink_trans = trans->transid; + + /* + * if this inode hasn't been logged and directory we're renaming it + * from hasn't been logged, we don't need to log it + */ + ret = inode_logged(trans, inode, NULL); + if (ret < 0) { + goto out; + } else if (ret == 0) { + if (!old_dir) + goto out; + /* + * If the inode was not logged and we are doing a rename (old_dir is not + * NULL), check if old_dir was logged - if it was not we can return and + * do nothing. + */ + ret = inode_logged(trans, old_dir, NULL); + if (ret < 0) + goto out; + else if (ret == 0) + goto out; + } + ret = 0; + + /* + * Now that we know we need to update the log, allocate the scratch eb + * for the context before joining a log transaction below, as this can + * take time and therefore we could delay log commits from other tasks. + */ + btrfs_init_log_ctx_scratch_eb(&ctx); + + /* + * If we are doing a rename (old_dir is not NULL) from a directory that + * was previously logged, make sure that on log replay we get the old + * dir entry deleted. This is needed because we will also log the new + * name of the renamed inode, so we need to make sure that after log + * replay we don't end up with both the new and old dir entries existing. + */ + if (old_dir && old_dir->logged_trans == trans->transid) { + struct btrfs_root *log = old_dir->root->log_root; + struct btrfs_path *path; + struct fscrypt_name fname; + + ASSERT(old_dir_index >= BTRFS_DIR_START_INDEX, + "old_dir_index=%llu", old_dir_index); + + ret = fscrypt_setup_filename(&old_dir->vfs_inode, + &old_dentry->d_name, 0, &fname); + if (ret) + goto out; + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + fscrypt_free_filename(&fname); + goto out; + } + + /* + * We have two inodes to update in the log, the old directory and + * the inode that got renamed, so we must pin the log to prevent + * anyone from syncing the log until we have updated both inodes + * in the log. + */ + ret = join_running_log_trans(root); + /* + * At least one of the inodes was logged before, so this should + * not fail, but if it does, it's not serious, just bail out and + * mark the log for a full commit. + */ + if (WARN_ON_ONCE(ret < 0)) { + btrfs_free_path(path); + fscrypt_free_filename(&fname); + goto out; + } + + log_pinned = true; + + /* + * Other concurrent task might be logging the old directory, + * as it can be triggered when logging other inode that had or + * still has a dentry in the old directory. We lock the old + * directory's log_mutex to ensure the deletion of the old + * name is persisted, because during directory logging we + * delete all BTRFS_DIR_LOG_INDEX_KEY keys and the deletion of + * the old name's dir index item is in the delayed items, so + * it could be missed by an in progress directory logging. + */ + mutex_lock(&old_dir->log_mutex); + ret = del_logged_dentry(trans, log, path, btrfs_ino(old_dir), + &fname.disk_name, old_dir_index); + if (ret > 0) { + /* + * The dentry does not exist in the log, so record its + * deletion. + */ + btrfs_release_path(path); + ret = insert_dir_log_key(trans, log, path, + btrfs_ino(old_dir), + old_dir_index, old_dir_index); + } + mutex_unlock(&old_dir->log_mutex); + + btrfs_free_path(path); + fscrypt_free_filename(&fname); + if (ret < 0) + goto out; + } + + /* + * We don't care about the return value. If we fail to log the new name + * then we know the next attempt to sync the log will fallback to a full + * transaction commit (due to a call to btrfs_set_log_full_commit()), so + * we don't need to worry about getting a log committed that has an + * inconsistent state after a rename operation. + */ + btrfs_log_inode_parent(trans, inode, parent, LOG_INODE_EXISTS, &ctx); + ASSERT(list_empty(&ctx.conflict_inodes)); +out: + trace_btrfs_log_new_name_exit(trans, inode, old_dir, ret); + /* + * If an error happened mark the log for a full commit because it's not + * consistent and up to date or we couldn't find out if one of the + * inodes was logged before in this transaction. Do it before unpinning + * the log, to avoid any races with someone else trying to commit it. + */ + if (ret < 0) + btrfs_set_log_full_commit(trans); + if (log_pinned) + btrfs_end_log_trans(root); + free_extent_buffer(ctx.scratch_eb); +} + diff --git a/fs/btrfs/tree-log.h b/fs/btrfs/tree-log.h new file mode 100644 index 000000000..81ab5eeeb --- /dev/null +++ b/fs/btrfs/tree-log.h @@ -0,0 +1,107 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2008 Oracle. All rights reserved. + */ + +#ifndef BTRFS_TREE_LOG_H +#define BTRFS_TREE_LOG_H + +#include +#include +#include +#include "transaction.h" + +enum btrfs_log_mode { + /* Log everything about an inode. */ + LOG_INODE_ALL, + /* Log just enough to recreate the inode during log replay. */ + LOG_INODE_EXISTS, +}; + +struct inode; +struct dentry; +struct btrfs_ordered_extent; +struct btrfs_root; +struct btrfs_trans_handle; + +/* return value for btrfs_log_dentry_safe that means we don't need to log it at all */ +#define BTRFS_NO_LOG_SYNC 256 + +/* + * We can't use the tree log for whatever reason, force a transaction commit. + * We use a negative value because there are functions through the logging code + * that need to return an error (< 0 value), false (0) or true (1). Any negative + * value will do, as it will cause the log to be marked for a full sync. + */ +#define BTRFS_LOG_FORCE_COMMIT (-(MAX_ERRNO + 1)) + +struct btrfs_log_ctx { + int log_ret; + int log_transid; + bool log_new_dentries; + bool logging_new_name; + bool logging_new_delayed_dentries; + /* Indicate if the inode being logged was logged before. */ + bool logged_before; + struct btrfs_inode *inode; + struct list_head list; + /* Only used for fast fsyncs. */ + struct list_head ordered_extents; + struct list_head conflict_inodes; + int num_conflict_inodes; + bool logging_conflict_inodes; + /* + * Used for fsyncs that need to copy items from the subvolume tree to + * the log tree (full sync flag set or copy everything flag set) to + * avoid allocating a temporary extent buffer while holding a lock on + * an extent buffer of the subvolume tree and under the log transaction. + * Also helps to avoid allocating and freeing a temporary extent buffer + * in case we need to process multiple leaves from the subvolume tree. + */ + struct extent_buffer *scratch_eb; +}; + +void btrfs_init_log_ctx(struct btrfs_log_ctx *ctx, struct btrfs_inode *inode); +void btrfs_init_log_ctx_scratch_eb(struct btrfs_log_ctx *ctx); +void btrfs_release_log_ctx_extents(struct btrfs_log_ctx *ctx); + +static inline void btrfs_set_log_full_commit(struct btrfs_trans_handle *trans) +{ + WRITE_ONCE(trans->fs_info->last_trans_log_full_commit, trans->transid); +} + +static inline int btrfs_need_log_full_commit(struct btrfs_trans_handle *trans) +{ + return READ_ONCE(trans->fs_info->last_trans_log_full_commit) == + trans->transid; +} + +int btrfs_sync_log(struct btrfs_trans_handle *trans, + struct btrfs_root *root, struct btrfs_log_ctx *ctx); +void btrfs_free_log(struct btrfs_trans_handle *trans, struct btrfs_root *root); +void btrfs_free_log_root_tree(struct btrfs_trans_handle *trans, struct btrfs_fs_info *fs_info); +int btrfs_recover_log_trees(struct btrfs_root *tree_root); +int btrfs_log_dentry_safe(struct btrfs_trans_handle *trans, + struct dentry *dentry, + struct btrfs_log_ctx *ctx); +void btrfs_del_dir_entries_in_log(struct btrfs_trans_handle *trans, + const struct fscrypt_str *name, + struct btrfs_inode *dir, u64 index); +void btrfs_del_inode_ref_in_log(struct btrfs_trans_handle *trans, + const struct fscrypt_str *name, + struct btrfs_inode *inode, + struct btrfs_inode *dir); +void btrfs_end_log_trans(struct btrfs_root *root); +void btrfs_pin_log_trans(struct btrfs_root *root); +void btrfs_record_unlink_dir(struct btrfs_trans_handle *trans, + struct btrfs_inode *dir, struct btrfs_inode *inode, + bool for_rename); +void btrfs_record_snapshot_destroy(struct btrfs_trans_handle *trans, + struct btrfs_inode *dir); +void btrfs_record_new_subvolume(const struct btrfs_trans_handle *trans, + struct btrfs_inode *dir); +void btrfs_log_new_name(struct btrfs_trans_handle *trans, + struct dentry *old_dentry, struct btrfs_inode *old_dir, + u64 old_dir_index, struct dentry *parent); + +#endif diff --git a/fs/btrfs/tree-mod-log.c b/fs/btrfs/tree-mod-log.c new file mode 100644 index 000000000..a8094928f --- /dev/null +++ b/fs/btrfs/tree-mod-log.c @@ -0,0 +1,1146 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "messages.h" +#include "tree-mod-log.h" +#include "disk-io.h" +#include "fs.h" +#include "accessors.h" +#include "tree-checker.h" + +struct tree_mod_root { + u64 logical; + u8 level; +}; + +struct tree_mod_elem { + struct rb_node node; + u64 logical; + u64 seq; + enum btrfs_mod_log_op op; + + /* + * This is used for BTRFS_MOD_LOG_KEY_* and BTRFS_MOD_LOG_MOVE_KEYS + * operations. + */ + int slot; + + /* This is used for BTRFS_MOD_LOG_KEY* and BTRFS_MOD_LOG_ROOT_REPLACE. */ + u64 generation; + + union { + /* + * This is used for the following op types: + * + * BTRFS_MOD_LOG_KEY_REMOVE_WHILE_FREEING + * BTRFS_MOD_LOG_KEY_REMOVE_WHILE_MOVING + * BTRFS_MOD_LOG_KEY_REMOVE + * BTRFS_MOD_LOG_KEY_REPLACE + */ + struct { + struct btrfs_disk_key key; + u64 blockptr; + } slot_change; + + /* This is used for op == BTRFS_MOD_LOG_MOVE_KEYS. */ + struct { + int dst_slot; + int nr_items; + } move; + + /* This is used for op == BTRFS_MOD_LOG_ROOT_REPLACE. */ + struct tree_mod_root old_root; + }; +}; + +/* + * Pull a new tree mod seq number for our operation. + */ +static u64 btrfs_inc_tree_mod_seq(struct btrfs_fs_info *fs_info) +{ + return atomic64_inc_return(&fs_info->tree_mod_seq); +} + +/* + * This adds a new blocker to the tree mod log's blocker list if the @elem + * passed does not already have a sequence number set. So when a caller expects + * to record tree modifications, it should ensure to set elem->seq to zero + * before calling btrfs_get_tree_mod_seq. + * Returns a fresh, unused tree log modification sequence number, even if no new + * blocker was added. + */ +u64 btrfs_get_tree_mod_seq(struct btrfs_fs_info *fs_info, + struct btrfs_seq_list *elem) +{ + write_lock(&fs_info->tree_mod_log_lock); + if (!elem->seq) { + elem->seq = btrfs_inc_tree_mod_seq(fs_info); + list_add_tail(&elem->list, &fs_info->tree_mod_seq_list); + set_bit(BTRFS_FS_TREE_MOD_LOG_USERS, &fs_info->flags); + } + write_unlock(&fs_info->tree_mod_log_lock); + + return elem->seq; +} + +void btrfs_put_tree_mod_seq(struct btrfs_fs_info *fs_info, + struct btrfs_seq_list *elem) +{ + struct rb_root *tm_root; + struct rb_node *node; + struct rb_node *next; + struct tree_mod_elem *tm; + u64 min_seq = BTRFS_SEQ_LAST; + u64 seq_putting = elem->seq; + + if (!seq_putting) + return; + + write_lock(&fs_info->tree_mod_log_lock); + list_del(&elem->list); + elem->seq = 0; + + if (list_empty(&fs_info->tree_mod_seq_list)) { + clear_bit(BTRFS_FS_TREE_MOD_LOG_USERS, &fs_info->flags); + } else { + struct btrfs_seq_list *first; + + first = list_first_entry(&fs_info->tree_mod_seq_list, + struct btrfs_seq_list, list); + if (seq_putting > first->seq) { + /* + * Blocker with lower sequence number exists, we cannot + * remove anything from the log. + */ + write_unlock(&fs_info->tree_mod_log_lock); + return; + } + min_seq = first->seq; + } + + /* + * Anything that's lower than the lowest existing (read: blocked) + * sequence number can be removed from the tree. + */ + tm_root = &fs_info->tree_mod_log; + for (node = rb_first(tm_root); node; node = next) { + next = rb_next(node); + tm = rb_entry(node, struct tree_mod_elem, node); + if (tm->seq >= min_seq) + continue; + rb_erase(node, tm_root); + kfree(tm); + } + write_unlock(&fs_info->tree_mod_log_lock); +} + +/* + * Key order of the log: + * node/leaf start address -> sequence + * + * The 'start address' is the logical address of the *new* root node for root + * replace operations, or the logical address of the affected block for all + * other operations. + */ +static noinline int tree_mod_log_insert(struct btrfs_fs_info *fs_info, + struct tree_mod_elem *tm) +{ + struct rb_root *tm_root; + struct rb_node **new; + struct rb_node *parent = NULL; + struct tree_mod_elem *cur; + + lockdep_assert_held_write(&fs_info->tree_mod_log_lock); + + tm->seq = btrfs_inc_tree_mod_seq(fs_info); + + tm_root = &fs_info->tree_mod_log; + new = &tm_root->rb_node; + while (*new) { + cur = rb_entry(*new, struct tree_mod_elem, node); + parent = *new; + if (cur->logical < tm->logical) + new = &((*new)->rb_left); + else if (cur->logical > tm->logical) + new = &((*new)->rb_right); + else if (cur->seq < tm->seq) + new = &((*new)->rb_left); + else if (cur->seq > tm->seq) + new = &((*new)->rb_right); + else + return -EEXIST; + } + + rb_link_node(&tm->node, parent, new); + rb_insert_color(&tm->node, tm_root); + return 0; +} + +static inline bool skip_eb_logging(const struct extent_buffer *eb) +{ + const u64 owner = btrfs_header_owner(eb); + + if (btrfs_header_level(eb) == 0) + return true; + + /* + * Tree mod logging exists so that there's a consistent view of the + * extents and backrefs of inodes even if while a task is iterating over + * them other tasks are modifying subvolume trees and the extent tree + * (including running delayed refs). So we only need to log extent + * buffers from the extent tree and subvolume trees. + */ + + if (owner == BTRFS_EXTENT_TREE_OBJECTID) + return false; + + if (btrfs_is_fstree(owner)) + return false; + + return true; +} + +/* + * Determines if logging can be omitted. Returns true if it can. Otherwise, it + * returns false with the tree_mod_log_lock acquired. The caller must hold + * this until all tree mod log insertions are recorded in the rb tree and then + * write unlock fs_info::tree_mod_log_lock. + */ +static bool tree_mod_dont_log(struct btrfs_fs_info *fs_info, const struct extent_buffer *eb) +{ + if (!test_bit(BTRFS_FS_TREE_MOD_LOG_USERS, &fs_info->flags)) + return true; + if (eb && skip_eb_logging(eb)) + return true; + + write_lock(&fs_info->tree_mod_log_lock); + if (list_empty(&(fs_info)->tree_mod_seq_list)) { + write_unlock(&fs_info->tree_mod_log_lock); + return true; + } + + return false; +} + +/* Similar to tree_mod_dont_log, but doesn't acquire any locks. */ +static bool tree_mod_need_log(const struct btrfs_fs_info *fs_info, + const struct extent_buffer *eb) +{ + if (!test_bit(BTRFS_FS_TREE_MOD_LOG_USERS, &fs_info->flags)) + return false; + if (eb && skip_eb_logging(eb)) + return false; + + return true; +} + +static struct tree_mod_elem *alloc_tree_mod_elem(const struct extent_buffer *eb, + int slot, + enum btrfs_mod_log_op op) +{ + struct tree_mod_elem *tm; + + /* Can't be one of these types, due to union in struct tree_mod_elem. */ + ASSERT(op != BTRFS_MOD_LOG_MOVE_KEYS); + ASSERT(op != BTRFS_MOD_LOG_ROOT_REPLACE); + + tm = kzalloc_obj(*tm, GFP_NOFS); + if (!tm) + return NULL; + + tm->logical = eb->start; + btrfs_node_key(eb, &tm->slot_change.key, slot); + tm->slot_change.blockptr = btrfs_node_blockptr(eb, slot); + tm->op = op; + tm->slot = slot; + tm->generation = btrfs_node_ptr_generation(eb, slot); + RB_CLEAR_NODE(&tm->node); + + return tm; +} + +int btrfs_tree_mod_log_insert_key(const struct extent_buffer *eb, int slot, + enum btrfs_mod_log_op op) +{ + struct tree_mod_elem *tm; + int ret = 0; + + if (!tree_mod_need_log(eb->fs_info, eb)) + return 0; + + tm = alloc_tree_mod_elem(eb, slot, op); + if (!tm) + ret = -ENOMEM; + + if (tree_mod_dont_log(eb->fs_info, eb)) { + kfree(tm); + /* + * Don't error if we failed to allocate memory because we don't + * need to log. + */ + return 0; + } else if (ret != 0) { + /* + * We previously failed to allocate memory and we need to log, + * so we have to fail. + */ + goto out_unlock; + } + + ret = tree_mod_log_insert(eb->fs_info, tm); +out_unlock: + write_unlock(&eb->fs_info->tree_mod_log_lock); + if (ret) + kfree(tm); + + return ret; +} + +static struct tree_mod_elem *tree_mod_log_alloc_move(const struct extent_buffer *eb, + int dst_slot, int src_slot, + int nr_items) +{ + struct tree_mod_elem *tm; + + tm = kzalloc_obj(*tm, GFP_NOFS); + if (!tm) + return ERR_PTR(-ENOMEM); + + tm->logical = eb->start; + tm->slot = src_slot; + tm->move.dst_slot = dst_slot; + tm->move.nr_items = nr_items; + tm->op = BTRFS_MOD_LOG_MOVE_KEYS; + RB_CLEAR_NODE(&tm->node); + + return tm; +} + +int btrfs_tree_mod_log_insert_move(const struct extent_buffer *eb, + int dst_slot, int src_slot, + int nr_items) +{ + struct tree_mod_elem *tm = NULL; + struct tree_mod_elem **tm_list = NULL; + int ret = 0; + int i; + bool locked = false; + + if (!tree_mod_need_log(eb->fs_info, eb)) + return 0; + + tm_list = kzalloc_objs(struct tree_mod_elem *, nr_items, GFP_NOFS); + if (!tm_list) { + ret = -ENOMEM; + goto lock; + } + + tm = tree_mod_log_alloc_move(eb, dst_slot, src_slot, nr_items); + if (IS_ERR(tm)) { + ret = PTR_ERR(tm); + tm = NULL; + goto lock; + } + + for (i = 0; i + dst_slot < src_slot && i < nr_items; i++) { + tm_list[i] = alloc_tree_mod_elem(eb, i + dst_slot, + BTRFS_MOD_LOG_KEY_REMOVE_WHILE_MOVING); + if (!tm_list[i]) { + ret = -ENOMEM; + goto lock; + } + } + +lock: + if (tree_mod_dont_log(eb->fs_info, eb)) { + /* + * Don't error if we failed to allocate memory because we don't + * need to log. + */ + ret = 0; + goto free_tms; + } + locked = true; + + /* + * We previously failed to allocate memory and we need to log, so we + * have to fail. + */ + if (ret != 0) + goto free_tms; + + /* + * When we override something during the move, we log these removals. + * This can only happen when we move towards the beginning of the + * buffer, i.e. dst_slot < src_slot. + */ + for (i = 0; i + dst_slot < src_slot && i < nr_items; i++) { + ret = tree_mod_log_insert(eb->fs_info, tm_list[i]); + if (ret) + goto free_tms; + } + + ret = tree_mod_log_insert(eb->fs_info, tm); + if (ret) + goto free_tms; + write_unlock(&eb->fs_info->tree_mod_log_lock); + kfree(tm_list); + + return 0; + +free_tms: + if (tm_list) { + for (i = 0; i < nr_items; i++) { + if (tm_list[i] && !RB_EMPTY_NODE(&tm_list[i]->node)) + rb_erase(&tm_list[i]->node, &eb->fs_info->tree_mod_log); + kfree(tm_list[i]); + } + } + if (locked) + write_unlock(&eb->fs_info->tree_mod_log_lock); + kfree(tm_list); + kfree(tm); + + return ret; +} + +static int tree_mod_log_free_eb(struct btrfs_fs_info *fs_info, + struct tree_mod_elem **tm_list, + int nritems) +{ + int i, j; + int ret; + + for (i = nritems - 1; i >= 0; i--) { + ret = tree_mod_log_insert(fs_info, tm_list[i]); + if (ret) { + for (j = nritems - 1; j > i; j--) + rb_erase(&tm_list[j]->node, + &fs_info->tree_mod_log); + return ret; + } + } + + return 0; +} + +int btrfs_tree_mod_log_insert_root(struct extent_buffer *old_root, + struct extent_buffer *new_root, + bool log_removal) +{ + struct btrfs_fs_info *fs_info = old_root->fs_info; + struct tree_mod_elem *tm = NULL; + struct tree_mod_elem **tm_list = NULL; + int nritems = 0; + int ret = 0; + int i; + + if (!tree_mod_need_log(fs_info, NULL)) + return 0; + + if (log_removal && btrfs_header_level(old_root) > 0) { + nritems = btrfs_header_nritems(old_root); + tm_list = kzalloc_objs(struct tree_mod_elem *, nritems, + GFP_NOFS); + if (!tm_list) { + ret = -ENOMEM; + goto lock; + } + for (i = 0; i < nritems; i++) { + tm_list[i] = alloc_tree_mod_elem(old_root, i, + BTRFS_MOD_LOG_KEY_REMOVE_WHILE_FREEING); + if (!tm_list[i]) { + ret = -ENOMEM; + goto lock; + } + } + } + + tm = kzalloc_obj(*tm, GFP_NOFS); + if (!tm) { + ret = -ENOMEM; + goto lock; + } + + tm->logical = new_root->start; + tm->old_root.logical = old_root->start; + tm->old_root.level = btrfs_header_level(old_root); + tm->generation = btrfs_header_generation(old_root); + tm->op = BTRFS_MOD_LOG_ROOT_REPLACE; + +lock: + if (tree_mod_dont_log(fs_info, NULL)) { + /* + * Don't error if we failed to allocate memory because we don't + * need to log. + */ + ret = 0; + goto free_tms; + } else if (ret != 0) { + /* + * We previously failed to allocate memory and we need to log, + * so we have to fail. + */ + goto out_unlock; + } + + if (tm_list) + ret = tree_mod_log_free_eb(fs_info, tm_list, nritems); + if (!ret) + ret = tree_mod_log_insert(fs_info, tm); + +out_unlock: + write_unlock(&fs_info->tree_mod_log_lock); + if (ret) + goto free_tms; + kfree(tm_list); + + return ret; + +free_tms: + if (tm_list) { + for (i = 0; i < nritems; i++) + kfree(tm_list[i]); + kfree(tm_list); + } + kfree(tm); + + return ret; +} + +static struct tree_mod_elem *__tree_mod_log_search(struct btrfs_fs_info *fs_info, + u64 start, u64 min_seq, + bool smallest) +{ + struct rb_root *tm_root; + struct rb_node *node; + struct tree_mod_elem *cur = NULL; + struct tree_mod_elem *found = NULL; + + read_lock(&fs_info->tree_mod_log_lock); + tm_root = &fs_info->tree_mod_log; + node = tm_root->rb_node; + while (node) { + cur = rb_entry(node, struct tree_mod_elem, node); + if (cur->logical < start) { + node = node->rb_left; + } else if (cur->logical > start) { + node = node->rb_right; + } else if (cur->seq < min_seq) { + node = node->rb_left; + } else if (!smallest) { + /* We want the node with the highest seq */ + if (found) + BUG_ON(found->seq > cur->seq); + found = cur; + node = node->rb_left; + } else if (cur->seq > min_seq) { + /* We want the node with the smallest seq */ + if (found) + BUG_ON(found->seq < cur->seq); + found = cur; + node = node->rb_right; + } else { + found = cur; + break; + } + } + read_unlock(&fs_info->tree_mod_log_lock); + + return found; +} + +/* + * This returns the element from the log with the smallest time sequence + * value that's in the log (the oldest log item). Any element with a time + * sequence lower than min_seq will be ignored. + */ +static struct tree_mod_elem *tree_mod_log_search_oldest(struct btrfs_fs_info *fs_info, + u64 start, u64 min_seq) +{ + return __tree_mod_log_search(fs_info, start, min_seq, true); +} + +/* + * This returns the element from the log with the largest time sequence + * value that's in the log (the most recent log item). Any element with + * a time sequence lower than min_seq will be ignored. + */ +static struct tree_mod_elem *tree_mod_log_search(struct btrfs_fs_info *fs_info, + u64 start, u64 min_seq) +{ + return __tree_mod_log_search(fs_info, start, min_seq, false); +} + +int btrfs_tree_mod_log_eb_copy(struct extent_buffer *dst, + const struct extent_buffer *src, + unsigned long dst_offset, + unsigned long src_offset, + int nr_items) +{ + struct btrfs_fs_info *fs_info = dst->fs_info; + int ret = 0; + struct tree_mod_elem **tm_list = NULL; + struct tree_mod_elem **tm_list_add = NULL; + struct tree_mod_elem **tm_list_rem = NULL; + int i; + bool locked = false; + struct tree_mod_elem *dst_move_tm = NULL; + struct tree_mod_elem *src_move_tm = NULL; + u32 dst_move_nr_items = btrfs_header_nritems(dst) - dst_offset; + u32 src_move_nr_items = btrfs_header_nritems(src) - (src_offset + nr_items); + + if (!tree_mod_need_log(fs_info, NULL)) + return 0; + + if (btrfs_header_level(dst) == 0 && btrfs_header_level(src) == 0) + return 0; + + tm_list = kzalloc_objs(struct tree_mod_elem *, nr_items * 2, GFP_NOFS); + if (!tm_list) { + ret = -ENOMEM; + goto lock; + } + + if (dst_move_nr_items) { + dst_move_tm = tree_mod_log_alloc_move(dst, dst_offset + nr_items, + dst_offset, dst_move_nr_items); + if (IS_ERR(dst_move_tm)) { + ret = PTR_ERR(dst_move_tm); + dst_move_tm = NULL; + goto lock; + } + } + if (src_move_nr_items) { + src_move_tm = tree_mod_log_alloc_move(src, src_offset, + src_offset + nr_items, + src_move_nr_items); + if (IS_ERR(src_move_tm)) { + ret = PTR_ERR(src_move_tm); + src_move_tm = NULL; + goto lock; + } + } + + tm_list_add = tm_list; + tm_list_rem = tm_list + nr_items; + for (i = 0; i < nr_items; i++) { + tm_list_rem[i] = alloc_tree_mod_elem(src, i + src_offset, + BTRFS_MOD_LOG_KEY_REMOVE); + if (!tm_list_rem[i]) { + ret = -ENOMEM; + goto lock; + } + + tm_list_add[i] = alloc_tree_mod_elem(dst, i + dst_offset, + BTRFS_MOD_LOG_KEY_ADD); + if (!tm_list_add[i]) { + ret = -ENOMEM; + goto lock; + } + } + +lock: + if (tree_mod_dont_log(fs_info, NULL)) { + /* + * Don't error if we failed to allocate memory because we don't + * need to log. + */ + ret = 0; + goto free_tms; + } + locked = true; + + /* + * We previously failed to allocate memory and we need to log, so we + * have to fail. + */ + if (ret != 0) + goto free_tms; + + if (dst_move_tm) { + ret = tree_mod_log_insert(fs_info, dst_move_tm); + if (ret) + goto free_tms; + } + for (i = 0; i < nr_items; i++) { + ret = tree_mod_log_insert(fs_info, tm_list_rem[i]); + if (ret) + goto free_tms; + ret = tree_mod_log_insert(fs_info, tm_list_add[i]); + if (ret) + goto free_tms; + } + if (src_move_tm) { + ret = tree_mod_log_insert(fs_info, src_move_tm); + if (ret) + goto free_tms; + } + + write_unlock(&fs_info->tree_mod_log_lock); + kfree(tm_list); + + return 0; + +free_tms: + if (dst_move_tm && !RB_EMPTY_NODE(&dst_move_tm->node)) + rb_erase(&dst_move_tm->node, &fs_info->tree_mod_log); + kfree(dst_move_tm); + if (src_move_tm && !RB_EMPTY_NODE(&src_move_tm->node)) + rb_erase(&src_move_tm->node, &fs_info->tree_mod_log); + kfree(src_move_tm); + if (tm_list) { + for (i = 0; i < nr_items * 2; i++) { + if (tm_list[i] && !RB_EMPTY_NODE(&tm_list[i]->node)) + rb_erase(&tm_list[i]->node, &fs_info->tree_mod_log); + kfree(tm_list[i]); + } + } + if (locked) + write_unlock(&fs_info->tree_mod_log_lock); + kfree(tm_list); + + return ret; +} + +int btrfs_tree_mod_log_free_eb(struct extent_buffer *eb) +{ + struct tree_mod_elem **tm_list = NULL; + int nritems = 0; + int i; + int ret = 0; + + if (!tree_mod_need_log(eb->fs_info, eb)) + return 0; + + nritems = btrfs_header_nritems(eb); + tm_list = kzalloc_objs(struct tree_mod_elem *, nritems, GFP_NOFS); + if (!tm_list) { + ret = -ENOMEM; + goto lock; + } + + for (i = 0; i < nritems; i++) { + tm_list[i] = alloc_tree_mod_elem(eb, i, + BTRFS_MOD_LOG_KEY_REMOVE_WHILE_FREEING); + if (!tm_list[i]) { + ret = -ENOMEM; + goto lock; + } + } + +lock: + if (tree_mod_dont_log(eb->fs_info, eb)) { + /* + * Don't error if we failed to allocate memory because we don't + * need to log. + */ + ret = 0; + goto free_tms; + } else if (ret != 0) { + /* + * We previously failed to allocate memory and we need to log, + * so we have to fail. + */ + goto out_unlock; + } + + ret = tree_mod_log_free_eb(eb->fs_info, tm_list, nritems); +out_unlock: + write_unlock(&eb->fs_info->tree_mod_log_lock); + if (ret) + goto free_tms; + kfree(tm_list); + + return 0; + +free_tms: + if (tm_list) { + for (i = 0; i < nritems; i++) + kfree(tm_list[i]); + kfree(tm_list); + } + + return ret; +} + +/* + * Returns the logical address of the oldest predecessor of the given root. + * Entries older than time_seq are ignored. + */ +static struct tree_mod_elem *tree_mod_log_oldest_root(struct extent_buffer *eb_root, + u64 time_seq) +{ + struct tree_mod_elem *tm; + struct tree_mod_elem *found = NULL; + u64 root_logical = eb_root->start; + bool looped = false; + + if (!time_seq) + return NULL; + + /* + * The very last operation that's logged for a root is the replacement + * operation (if it is replaced at all). This has the logical address + * of the *new* root, making it the very first operation that's logged + * for this root. + */ + while (1) { + tm = tree_mod_log_search_oldest(eb_root->fs_info, root_logical, + time_seq); + if (!looped && !tm) + return NULL; + /* + * If there are no tree operation for the oldest root, we simply + * return it. This should only happen if that (old) root is at + * level 0. + */ + if (!tm) + break; + + /* + * If there's an operation that's not a root replacement, we + * found the oldest version of our root. Normally, we'll find a + * BTRFS_MOD_LOG_KEY_REMOVE_WHILE_FREEING operation here. + */ + if (tm->op != BTRFS_MOD_LOG_ROOT_REPLACE) + break; + + found = tm; + root_logical = tm->old_root.logical; + looped = true; + } + + /* If there's no old root to return, return what we found instead */ + if (!found) + found = tm; + + return found; +} + + +/* + * tm is a pointer to the first operation to rewind within eb. Then, all + * previous operations will be rewound (until we reach something older than + * time_seq). + */ +static void tree_mod_log_rewind(struct btrfs_fs_info *fs_info, + struct extent_buffer *eb, + u64 time_seq, + struct tree_mod_elem *first_tm) +{ + u32 n; + struct rb_node *next; + struct tree_mod_elem *tm = first_tm; + unsigned long o_dst; + unsigned long o_src; + unsigned long p_size = sizeof(struct btrfs_key_ptr); + /* + * max_slot tracks the maximum valid slot of the rewind eb at every + * step of the rewind. This is in contrast with 'n' which eventually + * matches the number of items, but can be wrong during moves or if + * removes overlap on already valid slots (which is probably separately + * a bug). We do this to validate the offsets of memmoves for rewinding + * moves and detect invalid memmoves. + * + * Since a rewind eb can start empty, max_slot is a signed integer with + * a special meaning for -1, which is that no slot is valid to move out + * of. Any other negative value is invalid. + */ + int max_slot; + int move_src_end_slot; + int move_dst_end_slot; + + n = btrfs_header_nritems(eb); + max_slot = n - 1; + read_lock(&fs_info->tree_mod_log_lock); + while (tm && tm->seq >= time_seq) { + ASSERT(max_slot >= -1); + /* + * All the operations are recorded with the operator used for + * the modification. As we're going backwards, we do the + * opposite of each operation here. + */ + switch (tm->op) { + case BTRFS_MOD_LOG_KEY_REMOVE_WHILE_FREEING: + BUG_ON(tm->slot < n); + fallthrough; + case BTRFS_MOD_LOG_KEY_REMOVE_WHILE_MOVING: + case BTRFS_MOD_LOG_KEY_REMOVE: + btrfs_set_node_key(eb, &tm->slot_change.key, tm->slot); + btrfs_set_node_blockptr(eb, tm->slot, tm->slot_change.blockptr); + btrfs_set_node_ptr_generation(eb, tm->slot, + tm->generation); + n++; + if (tm->slot > max_slot) + max_slot = tm->slot; + break; + case BTRFS_MOD_LOG_KEY_REPLACE: + BUG_ON(tm->slot >= n); + btrfs_set_node_key(eb, &tm->slot_change.key, tm->slot); + btrfs_set_node_blockptr(eb, tm->slot, tm->slot_change.blockptr); + btrfs_set_node_ptr_generation(eb, tm->slot, + tm->generation); + break; + case BTRFS_MOD_LOG_KEY_ADD: + /* + * It is possible we could have already removed keys + * behind the known max slot, so this will be an + * overestimate. In practice, the copy operation + * inserts them in increasing order, and overestimating + * just means we miss some warnings, so it's OK. It + * isn't worth carefully tracking the full array of + * valid slots to check against when moving. + */ + if (tm->slot == max_slot) + max_slot--; + /* if a move operation is needed it's in the log */ + n--; + break; + case BTRFS_MOD_LOG_MOVE_KEYS: + ASSERT(tm->move.nr_items > 0); + move_src_end_slot = tm->move.dst_slot + tm->move.nr_items - 1; + move_dst_end_slot = tm->slot + tm->move.nr_items - 1; + o_dst = btrfs_node_key_ptr_offset(eb, tm->slot); + o_src = btrfs_node_key_ptr_offset(eb, tm->move.dst_slot); + if (WARN_ON(move_src_end_slot > max_slot || + tm->move.nr_items <= 0)) { + btrfs_warn(fs_info, +"move from invalid tree mod log slot eb %llu slot %d dst_slot %d nr_items %d seq %llu n %u max_slot %d", + eb->start, tm->slot, + tm->move.dst_slot, tm->move.nr_items, + tm->seq, n, max_slot); + } + memmove_extent_buffer(eb, o_dst, o_src, + tm->move.nr_items * p_size); + max_slot = move_dst_end_slot; + break; + case BTRFS_MOD_LOG_ROOT_REPLACE: + /* + * This operation is special. For roots, this must be + * handled explicitly before rewinding. + * For non-roots, this operation may exist if the node + * was a root: root A -> child B; then A gets empty and + * B is promoted to the new root. In the mod log, we'll + * have a root-replace operation for B, a tree block + * that is no root. We simply ignore that operation. + */ + break; + } + next = rb_next(&tm->node); + if (!next) + break; + tm = rb_entry(next, struct tree_mod_elem, node); + if (tm->logical != first_tm->logical) + break; + } + read_unlock(&fs_info->tree_mod_log_lock); + btrfs_set_header_nritems(eb, n); +} + +/* + * Called with eb read locked. If the buffer cannot be rewound, the same buffer + * is returned. If rewind operations happen, a fresh buffer is returned. The + * returned buffer is always read-locked. If the returned buffer is not the + * input buffer, the lock on the input buffer is released and the input buffer + * is freed (its refcount is decremented). + */ +struct extent_buffer *btrfs_tree_mod_log_rewind(struct btrfs_fs_info *fs_info, + struct extent_buffer *eb, + u64 time_seq) +{ + struct extent_buffer *eb_rewin; + struct tree_mod_elem *tm; + + if (!time_seq) + return eb; + + if (btrfs_header_level(eb) == 0) + return eb; + + tm = tree_mod_log_search(fs_info, eb->start, time_seq); + if (!tm) + return eb; + + if (tm->op == BTRFS_MOD_LOG_KEY_REMOVE_WHILE_FREEING) { + BUG_ON(tm->slot != 0); + eb_rewin = alloc_dummy_extent_buffer(fs_info, eb->start); + if (!eb_rewin) { + btrfs_tree_read_unlock(eb); + free_extent_buffer(eb); + return NULL; + } + btrfs_set_header_bytenr(eb_rewin, eb->start); + btrfs_set_header_backref_rev(eb_rewin, + btrfs_header_backref_rev(eb)); + btrfs_set_header_owner(eb_rewin, btrfs_header_owner(eb)); + btrfs_set_header_level(eb_rewin, btrfs_header_level(eb)); + } else { + eb_rewin = btrfs_clone_extent_buffer(eb); + if (!eb_rewin) { + btrfs_tree_read_unlock(eb); + free_extent_buffer(eb); + return NULL; + } + } + + btrfs_tree_read_unlock(eb); + free_extent_buffer(eb); + + btrfs_set_buffer_lockdep_class(btrfs_header_owner(eb_rewin), + eb_rewin, btrfs_header_level(eb_rewin)); + btrfs_tree_read_lock(eb_rewin); + tree_mod_log_rewind(fs_info, eb_rewin, time_seq, tm); + WARN_ON(btrfs_header_nritems(eb_rewin) > + BTRFS_NODEPTRS_PER_BLOCK(fs_info)); + + return eb_rewin; +} + +/* + * Rewind the state of @root's root node to the given @time_seq value. + * If there are no changes, the current root->root_node is returned. If anything + * changed in between, there's a fresh buffer allocated on which the rewind + * operations are done. In any case, the returned buffer is read locked. + * Returns NULL on error (with no locks held). + */ +struct extent_buffer *btrfs_get_old_root(struct btrfs_root *root, u64 time_seq) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct tree_mod_elem *tm; + struct extent_buffer *eb = NULL; + struct extent_buffer *eb_root; + u64 eb_root_owner = 0; + struct extent_buffer *old; + struct tree_mod_root *old_root = NULL; + u64 old_generation = 0; + u64 logical; + int level; + + eb_root = btrfs_read_lock_root_node(root); + tm = tree_mod_log_oldest_root(eb_root, time_seq); + if (!tm) + return eb_root; + + if (tm->op == BTRFS_MOD_LOG_ROOT_REPLACE) { + old_root = &tm->old_root; + old_generation = tm->generation; + logical = old_root->logical; + level = old_root->level; + } else { + logical = eb_root->start; + level = btrfs_header_level(eb_root); + } + + tm = tree_mod_log_search(fs_info, logical, time_seq); + if (old_root && tm && tm->op != BTRFS_MOD_LOG_KEY_REMOVE_WHILE_FREEING) { + struct btrfs_tree_parent_check check = { 0 }; + + btrfs_tree_read_unlock(eb_root); + free_extent_buffer(eb_root); + + check.level = level; + check.owner_root = btrfs_root_id(root); + + old = read_tree_block(fs_info, logical, &check); + if (WARN_ON(IS_ERR(old))) { + btrfs_warn(fs_info, + "failed to read tree block %llu from get_old_root: %ld", + logical, PTR_ERR(old)); + } else { + struct tree_mod_elem *tm2; + + btrfs_tree_read_lock(old); + eb = btrfs_clone_extent_buffer(old); + /* + * After the lookup for the most recent tree mod operation + * above and before we locked and cloned the extent buffer + * 'old', a new tree mod log operation may have been added. + * So lookup for a more recent one to make sure the number + * of mod log operations we replay is consistent with the + * number of items we have in the cloned extent buffer, + * otherwise we can hit a BUG_ON when rewinding the extent + * buffer. + */ + tm2 = tree_mod_log_search(fs_info, logical, time_seq); + btrfs_tree_read_unlock(old); + free_extent_buffer(old); + ASSERT(tm2); + ASSERT(tm2 == tm || tm2->seq > tm->seq); + if (!tm2 || tm2->seq < tm->seq) { + free_extent_buffer(eb); + return NULL; + } + tm = tm2; + } + } else if (old_root) { + eb_root_owner = btrfs_header_owner(eb_root); + btrfs_tree_read_unlock(eb_root); + free_extent_buffer(eb_root); + eb = alloc_dummy_extent_buffer(fs_info, logical); + } else { + eb = btrfs_clone_extent_buffer(eb_root); + btrfs_tree_read_unlock(eb_root); + free_extent_buffer(eb_root); + } + + if (!eb) + return NULL; + if (old_root) { + btrfs_set_header_bytenr(eb, eb->start); + btrfs_set_header_backref_rev(eb, BTRFS_MIXED_BACKREF_REV); + btrfs_set_header_owner(eb, eb_root_owner); + btrfs_set_header_level(eb, old_root->level); + btrfs_set_header_generation(eb, old_generation); + } + btrfs_set_buffer_lockdep_class(btrfs_header_owner(eb), eb, + btrfs_header_level(eb)); + btrfs_tree_read_lock(eb); + if (tm) + tree_mod_log_rewind(fs_info, eb, time_seq, tm); + else + WARN_ON(btrfs_header_level(eb) != 0); + WARN_ON(btrfs_header_nritems(eb) > BTRFS_NODEPTRS_PER_BLOCK(fs_info)); + + return eb; +} + +int btrfs_old_root_level(struct btrfs_root *root, u64 time_seq) +{ + struct tree_mod_elem *tm; + int level; + struct extent_buffer *eb_root = btrfs_root_node(root); + + tm = tree_mod_log_oldest_root(eb_root, time_seq); + if (tm && tm->op == BTRFS_MOD_LOG_ROOT_REPLACE) + level = tm->old_root.level; + else + level = btrfs_header_level(eb_root); + + free_extent_buffer(eb_root); + + return level; +} + +/* + * Return the lowest sequence number in the tree modification log. + * + * Return the sequence number of the oldest tree modification log user, which + * corresponds to the lowest sequence number of all existing users. If there are + * no users it returns 0. + */ +u64 btrfs_tree_mod_log_lowest_seq(struct btrfs_fs_info *fs_info) +{ + u64 ret = 0; + + read_lock(&fs_info->tree_mod_log_lock); + if (!list_empty(&fs_info->tree_mod_seq_list)) { + struct btrfs_seq_list *elem; + + elem = list_first_entry(&fs_info->tree_mod_seq_list, + struct btrfs_seq_list, list); + ret = elem->seq; + } + read_unlock(&fs_info->tree_mod_log_lock); + + return ret; +} diff --git a/fs/btrfs/tree-mod-log.h b/fs/btrfs/tree-mod-log.h new file mode 100644 index 000000000..1c1256604 --- /dev/null +++ b/fs/btrfs/tree-mod-log.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_TREE_MOD_LOG_H +#define BTRFS_TREE_MOD_LOG_H + +#include + +struct extent_buffer; +struct btrfs_fs_info; +struct btrfs_path; +struct btrfs_root; +struct btrfs_seq_list; + +/* Represents a tree mod log user. */ +struct btrfs_seq_list { + struct list_head list; + u64 seq; +}; + +#define BTRFS_SEQ_LIST_INIT(name) { .list = LIST_HEAD_INIT((name).list), .seq = 0 } +#define BTRFS_SEQ_LAST ((u64)-1) + +enum btrfs_mod_log_op { + BTRFS_MOD_LOG_KEY_REPLACE, + BTRFS_MOD_LOG_KEY_ADD, + BTRFS_MOD_LOG_KEY_REMOVE, + BTRFS_MOD_LOG_KEY_REMOVE_WHILE_FREEING, + BTRFS_MOD_LOG_KEY_REMOVE_WHILE_MOVING, + BTRFS_MOD_LOG_MOVE_KEYS, + BTRFS_MOD_LOG_ROOT_REPLACE, +}; + +u64 btrfs_get_tree_mod_seq(struct btrfs_fs_info *fs_info, + struct btrfs_seq_list *elem); +void btrfs_put_tree_mod_seq(struct btrfs_fs_info *fs_info, + struct btrfs_seq_list *elem); +int btrfs_tree_mod_log_insert_root(struct extent_buffer *old_root, + struct extent_buffer *new_root, + bool log_removal); +int btrfs_tree_mod_log_insert_key(const struct extent_buffer *eb, int slot, + enum btrfs_mod_log_op op); +int btrfs_tree_mod_log_free_eb(struct extent_buffer *eb); +struct extent_buffer *btrfs_tree_mod_log_rewind(struct btrfs_fs_info *fs_info, + struct extent_buffer *eb, + u64 time_seq); +struct extent_buffer *btrfs_get_old_root(struct btrfs_root *root, u64 time_seq); +int btrfs_old_root_level(struct btrfs_root *root, u64 time_seq); +int btrfs_tree_mod_log_eb_copy(struct extent_buffer *dst, + const struct extent_buffer *src, + unsigned long dst_offset, + unsigned long src_offset, + int nr_items); +int btrfs_tree_mod_log_insert_move(const struct extent_buffer *eb, + int dst_slot, int src_slot, + int nr_items); +u64 btrfs_tree_mod_log_lowest_seq(struct btrfs_fs_info *fs_info); + +#endif diff --git a/fs/btrfs/ulist.c b/fs/btrfs/ulist.c new file mode 100644 index 000000000..7898076ce --- /dev/null +++ b/fs/btrfs/ulist.c @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2011 STRATO AG + * written by Arne Jansen + */ + +#include +#include "messages.h" +#include "ulist.h" + +/* + * ulist is a generic data structure to hold a collection of unique u64 + * values. The only operations it supports is adding to the list and + * enumerating it. + * It is possible to store an auxiliary value along with the key. + * + * A sample usage for ulists is the enumeration of directed graphs without + * visiting a node twice. The pseudo-code could look like this: + * + * ulist = ulist_alloc(); + * ulist_add(ulist, root); + * ULIST_ITER_INIT(&uiter); + * + * while ((elem = ulist_next(ulist, &uiter)) { + * for (all child nodes n in elem) + * ulist_add(ulist, n); + * do something useful with the node; + * } + * ulist_free(ulist); + * + * This assumes the graph nodes are addressable by u64. This stems from the + * usage for tree enumeration in btrfs, where the logical addresses are + * 64 bit. + * + * It is also useful for tree enumeration which could be done elegantly + * recursively, but is not possible due to kernel stack limitations. The + * loop would be similar to the above. + */ + +/* + * Freshly initialize a ulist. + * + * @ulist: the ulist to initialize + * + * Note: don't use this function to init an already used ulist, use + * ulist_reinit instead. + */ +void ulist_init(struct ulist *ulist) +{ + INIT_LIST_HEAD(&ulist->nodes); + ulist->root = RB_ROOT; + ulist->nnodes = 0; + ulist->prealloc = NULL; +} + +/* + * Free up additionally allocated memory for the ulist. + * + * @ulist: the ulist from which to free the additional memory + * + * This is useful in cases where the base 'struct ulist' has been statically + * allocated. + */ +void ulist_release(struct ulist *ulist) +{ + struct ulist_node *node; + struct ulist_node *next; + + list_for_each_entry_safe(node, next, &ulist->nodes, list) { + kfree(node); + } + kfree(ulist->prealloc); + ulist->prealloc = NULL; + ulist->root = RB_ROOT; + INIT_LIST_HEAD(&ulist->nodes); +} + +/* + * Prepare a ulist for reuse. + * + * @ulist: ulist to be reused + * + * Free up all additional memory allocated for the list elements and reinit + * the ulist. + */ +void ulist_reinit(struct ulist *ulist) +{ + ulist_release(ulist); + ulist_init(ulist); +} + +/* + * Dynamically allocate a ulist. + * + * @gfp_mask: allocation flags to for base allocation + * + * The allocated ulist will be returned in an initialized state. + */ +struct ulist *ulist_alloc(gfp_t gfp_mask) +{ + struct ulist *ulist = kmalloc_obj(*ulist, gfp_mask); + + if (!ulist) + return NULL; + + ulist_init(ulist); + + return ulist; +} + +void ulist_prealloc(struct ulist *ulist, gfp_t gfp_mask) +{ + if (!ulist->prealloc) + ulist->prealloc = kzalloc_obj(*ulist->prealloc, gfp_mask); +} + +/* + * Free dynamically allocated ulist. + * + * @ulist: ulist to free + * + * It is not necessary to call ulist_release before. + */ +void ulist_free(struct ulist *ulist) +{ + if (!ulist) + return; + ulist_release(ulist); + kfree(ulist); +} + +static int ulist_node_val_key_cmp(const void *key, const struct rb_node *node) +{ + const u64 *val = key; + const struct ulist_node *unode = rb_entry(node, struct ulist_node, rb_node); + + if (unode->val < *val) + return 1; + else if (unode->val > *val) + return -1; + + return 0; +} + +static struct ulist_node *ulist_rbtree_search(struct ulist *ulist, u64 val) +{ + struct rb_node *node; + + node = rb_find(&val, &ulist->root, ulist_node_val_key_cmp); + return rb_entry_safe(node, struct ulist_node, rb_node); +} + +static void ulist_rbtree_erase(struct ulist *ulist, struct ulist_node *node) +{ + rb_erase(&node->rb_node, &ulist->root); + list_del(&node->list); + kfree(node); + BUG_ON(ulist->nnodes == 0); + ulist->nnodes--; +} + +static int ulist_node_val_cmp(struct rb_node *new, const struct rb_node *existing) +{ + const struct ulist_node *unode = rb_entry(new, struct ulist_node, rb_node); + + return ulist_node_val_key_cmp(&unode->val, existing); +} + +static int ulist_rbtree_insert(struct ulist *ulist, struct ulist_node *ins) +{ + struct rb_node *node; + + node = rb_find_add(&ins->rb_node, &ulist->root, ulist_node_val_cmp); + if (node) + return -EEXIST; + return 0; +} + +/* + * Add an element to the ulist. + * + * @ulist: ulist to add the element to + * @val: value to add to ulist + * @aux: auxiliary value to store along with val + * @gfp_mask: flags to use for allocation + * + * Note: locking must be provided by the caller. In case of rwlocks write + * locking is needed + * + * Add an element to a ulist. The @val will only be added if it doesn't + * already exist. If it is added, the auxiliary value @aux is stored along with + * it. In case @val already exists in the ulist, @aux is ignored, even if + * it differs from the already stored value. + * + * ulist_add returns 0 if @val already exists in ulist and 1 if @val has been + * inserted. + * In case of allocation failure -ENOMEM is returned and the ulist stays + * unaltered. + */ +int ulist_add(struct ulist *ulist, u64 val, u64 aux, gfp_t gfp_mask) +{ + return ulist_add_merge(ulist, val, aux, NULL, gfp_mask); +} + +int ulist_add_merge(struct ulist *ulist, u64 val, u64 aux, + u64 *old_aux, gfp_t gfp_mask) +{ + int ret; + struct ulist_node *node; + + node = ulist_rbtree_search(ulist, val); + if (node) { + if (old_aux) + *old_aux = node->aux; + return 0; + } + + if (ulist->prealloc) { + node = ulist->prealloc; + ulist->prealloc = NULL; + } else { + node = kmalloc_obj(*node, gfp_mask); + if (!node) + return -ENOMEM; + } + + node->val = val; + node->aux = aux; + + ret = ulist_rbtree_insert(ulist, node); + ASSERT(!ret); + list_add_tail(&node->list, &ulist->nodes); + ulist->nnodes++; + + return 1; +} + +/* + * Delete one node from ulist. + * + * @ulist: ulist to remove node from + * @val: value to delete + * @aux: aux to delete + * + * The deletion will only be done when *BOTH* val and aux matches. + * Return 0 for successful delete. + * Return > 0 for not found. + */ +int ulist_del(struct ulist *ulist, u64 val, u64 aux) +{ + struct ulist_node *node; + + node = ulist_rbtree_search(ulist, val); + /* Not found */ + if (!node) + return 1; + + if (node->aux != aux) + return 1; + + /* Found and delete */ + ulist_rbtree_erase(ulist, node); + return 0; +} + +/* + * Iterate ulist. + * + * @ulist: ulist to iterate + * @uiter: iterator variable, initialized with ULIST_ITER_INIT(&iterator) + * + * Note: locking must be provided by the caller. In case of rwlocks only read + * locking is needed + * + * This function is used to iterate an ulist. + * It returns the next element from the ulist or %NULL when the + * end is reached. No guarantee is made with respect to the order in which + * the elements are returned. They might neither be returned in order of + * addition nor in ascending order. + * It is allowed to call ulist_add during an enumeration. Newly added items + * are guaranteed to show up in the running enumeration. + */ +struct ulist_node *ulist_next(const struct ulist *ulist, struct ulist_iterator *uiter) +{ + struct ulist_node *node; + + if (list_empty(&ulist->nodes)) + return NULL; + if (uiter->cur_list && uiter->cur_list->next == &ulist->nodes) + return NULL; + if (uiter->cur_list) { + uiter->cur_list = uiter->cur_list->next; + } else { + uiter->cur_list = ulist->nodes.next; + } + node = list_entry(uiter->cur_list, struct ulist_node, list); + return node; +} diff --git a/fs/btrfs/ulist.h b/fs/btrfs/ulist.h new file mode 100644 index 000000000..c62a372f1 --- /dev/null +++ b/fs/btrfs/ulist.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2011 STRATO AG + * written by Arne Jansen + */ + +#ifndef BTRFS_ULIST_H +#define BTRFS_ULIST_H + +#include +#include +#include + +/* + * ulist is a generic data structure to hold a collection of unique u64 + * values. The only operations it supports is adding to the list and + * enumerating it. + * It is possible to store an auxiliary value along with the key. + * + */ +struct ulist_iterator { + struct list_head *cur_list; /* hint to start search */ +}; + +/* + * element of the list + */ +struct ulist_node { + u64 val; /* value to store */ + u64 aux; /* auxiliary value saved along with the val */ + + struct list_head list; /* used to link node */ + struct rb_node rb_node; /* used to speed up search */ +}; + +struct ulist { + /* + * number of elements stored in list + */ + unsigned long nnodes; + + struct list_head nodes; + struct rb_root root; + struct ulist_node *prealloc; +}; + +void ulist_init(struct ulist *ulist); +void ulist_release(struct ulist *ulist); +void ulist_reinit(struct ulist *ulist); +struct ulist *ulist_alloc(gfp_t gfp_mask); +void ulist_prealloc(struct ulist *ulist, gfp_t mask); +void ulist_free(struct ulist *ulist); +int ulist_add(struct ulist *ulist, u64 val, u64 aux, gfp_t gfp_mask); +int ulist_add_merge(struct ulist *ulist, u64 val, u64 aux, + u64 *old_aux, gfp_t gfp_mask); +int ulist_del(struct ulist *ulist, u64 val, u64 aux); + +/* just like ulist_add_merge() but take a pointer for the aux data */ +static inline int ulist_add_merge_ptr(struct ulist *ulist, u64 val, void *aux, + void **old_aux, gfp_t gfp_mask) +{ +#if BITS_PER_LONG == 32 + u64 old64 = (uintptr_t)*old_aux; + int ret = ulist_add_merge(ulist, val, (uintptr_t)aux, &old64, gfp_mask); + *old_aux = (void *)((uintptr_t)old64); + return ret; +#else + return ulist_add_merge(ulist, val, (u64)aux, (u64 *)old_aux, gfp_mask); +#endif +} + +struct ulist_node *ulist_next(const struct ulist *ulist, + struct ulist_iterator *uiter); + +#define ULIST_ITER_INIT(uiter) ((uiter)->cur_list = NULL) + +#endif diff --git a/fs/btrfs/uuid-tree.c b/fs/btrfs/uuid-tree.c new file mode 100644 index 000000000..467dff721 --- /dev/null +++ b/fs/btrfs/uuid-tree.c @@ -0,0 +1,566 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) STRATO AG 2013. All rights reserved. + */ + +#include +#include +#include +#include "messages.h" +#include "ctree.h" +#include "transaction.h" +#include "disk-io.h" +#include "fs.h" +#include "accessors.h" +#include "uuid-tree.h" +#include "ioctl.h" + +static void btrfs_uuid_to_key(const u8 *uuid, u8 type, struct btrfs_key *key) +{ + key->type = type; + key->objectid = get_unaligned_le64(uuid); + key->offset = get_unaligned_le64(uuid + sizeof(u64)); +} + +/* return -ENOENT for !found, < 0 for errors, or 0 if an item was found */ +static int btrfs_uuid_tree_lookup(struct btrfs_root *uuid_root, const u8 *uuid, + u8 type, u64 subid) +{ + int ret; + BTRFS_PATH_AUTO_FREE(path); + struct extent_buffer *eb; + int slot; + u32 item_size; + unsigned long offset; + struct btrfs_key key; + + if (WARN_ON_ONCE(!uuid_root)) + return -EINVAL; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + btrfs_uuid_to_key(uuid, type, &key); + ret = btrfs_search_slot(NULL, uuid_root, &key, path, 0, 0); + if (ret < 0) + return ret; + if (ret > 0) + return -ENOENT; + + eb = path->nodes[0]; + slot = path->slots[0]; + item_size = btrfs_item_size(eb, slot); + offset = btrfs_item_ptr_offset(eb, slot); + ret = -ENOENT; + + if (!IS_ALIGNED(item_size, sizeof(u64))) { + btrfs_warn(uuid_root->fs_info, + "uuid item with illegal size %lu!", + (unsigned long)item_size); + return ret; + } + while (item_size) { + __le64 data; + + read_extent_buffer(eb, &data, offset, sizeof(data)); + if (le64_to_cpu(data) == subid) { + ret = 0; + break; + } + offset += sizeof(data); + item_size -= sizeof(data); + } + + return ret; +} + +int btrfs_uuid_tree_add(struct btrfs_trans_handle *trans, const u8 *uuid, u8 type, + u64 subid_cpu) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *uuid_root = fs_info->uuid_root; + int ret; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + struct extent_buffer *eb; + int slot; + unsigned long offset; + __le64 subid_le; + + ret = btrfs_uuid_tree_lookup(uuid_root, uuid, type, subid_cpu); + if (ret != -ENOENT) + return ret; + + btrfs_uuid_to_key(uuid, type, &key); + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + ret = btrfs_insert_empty_item(trans, uuid_root, path, &key, + sizeof(subid_le)); + if (ret == 0) { + /* Add an item for the type for the first time */ + eb = path->nodes[0]; + slot = path->slots[0]; + offset = btrfs_item_ptr_offset(eb, slot); + } else if (ret == -EEXIST) { + /* + * An item with that type already exists. + * Extend the item and store the new subid at the end. + */ + btrfs_extend_item(trans, path, sizeof(subid_le)); + eb = path->nodes[0]; + slot = path->slots[0]; + offset = btrfs_item_ptr_offset(eb, slot); + offset += btrfs_item_size(eb, slot) - sizeof(subid_le); + } else { + btrfs_warn(fs_info, + "insert uuid item failed %d (0x%016llx, 0x%016llx) type %u!", + ret, key.objectid, key.offset, type); + return ret; + } + + subid_le = cpu_to_le64(subid_cpu); + write_extent_buffer(eb, &subid_le, offset, sizeof(subid_le)); + return 0; +} + +int btrfs_uuid_tree_remove(struct btrfs_trans_handle *trans, const u8 *uuid, u8 type, + u64 subid) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *uuid_root = fs_info->uuid_root; + int ret; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + struct extent_buffer *eb; + int slot; + unsigned long offset; + u32 item_size; + unsigned long move_dst; + unsigned long move_src; + unsigned long move_len; + + if (WARN_ON_ONCE(!uuid_root)) + return -EINVAL; + + btrfs_uuid_to_key(uuid, type, &key); + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + ret = btrfs_search_slot(trans, uuid_root, &key, path, -1, 1); + if (ret < 0) { + btrfs_warn(fs_info, "error %d while searching for uuid item!", + ret); + return ret; + } + if (ret > 0) + return -ENOENT; + + eb = path->nodes[0]; + slot = path->slots[0]; + offset = btrfs_item_ptr_offset(eb, slot); + item_size = btrfs_item_size(eb, slot); + if (!IS_ALIGNED(item_size, sizeof(u64))) { + btrfs_warn(fs_info, "uuid item with illegal size %lu!", + (unsigned long)item_size); + return -ENOENT; + } + while (item_size) { + __le64 read_subid; + + read_extent_buffer(eb, &read_subid, offset, sizeof(read_subid)); + if (le64_to_cpu(read_subid) == subid) + break; + offset += sizeof(read_subid); + item_size -= sizeof(read_subid); + } + + if (!item_size) + return -ENOENT; + + item_size = btrfs_item_size(eb, slot); + if (item_size == sizeof(subid)) + return btrfs_del_item(trans, uuid_root, path); + + move_dst = offset; + move_src = offset + sizeof(subid); + move_len = item_size - (move_src - btrfs_item_ptr_offset(eb, slot)); + memmove_extent_buffer(eb, move_dst, move_src, move_len); + btrfs_truncate_item(trans, path, item_size - sizeof(subid), 1); + + return 0; +} + +/* + * Check if we can add one root ID to a UUID key. + * If the key does not yet exists, we can, otherwise only if extended item does + * not exceeds the maximum item size permitted by the leaf size. + * + * Returns 0 on success, negative value on error. + */ +int btrfs_uuid_tree_check_overflow(struct btrfs_fs_info *fs_info, + const u8 *uuid, u8 type) +{ + BTRFS_PATH_AUTO_FREE(path); + int ret; + u32 item_size; + struct btrfs_key key; + + if (WARN_ON_ONCE(!fs_info->uuid_root)) + return -EINVAL; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + btrfs_uuid_to_key(uuid, type, &key); + ret = btrfs_search_slot(NULL, fs_info->uuid_root, &key, path, 0, 0); + if (ret < 0) + return ret; + if (ret > 0) + return 0; + + item_size = btrfs_item_size(path->nodes[0], path->slots[0]); + + if (sizeof(struct btrfs_item) + item_size + sizeof(u64) > + BTRFS_LEAF_DATA_SIZE(fs_info)) + return -EOVERFLOW; + + return 0; +} + +static int btrfs_uuid_iter_rem(struct btrfs_root *uuid_root, u8 *uuid, u8 type, + u64 subid) +{ + struct btrfs_trans_handle *trans; + int ret; + + /* 1 - for the uuid item */ + trans = btrfs_start_transaction(uuid_root, 1); + if (IS_ERR(trans)) + return PTR_ERR(trans); + + ret = btrfs_uuid_tree_remove(trans, uuid, type, subid); + btrfs_end_transaction(trans); + return ret; +} + +/* + * Check if there's an matching subvolume for given UUID + * + * Return: + * 0 check succeeded, the entry is not outdated + * > 0 if the check failed, the caller should remove the entry + * < 0 if an error occurred + */ +static int btrfs_check_uuid_tree_entry(struct btrfs_fs_info *fs_info, + const u8 *uuid, u8 type, u64 subvolid) +{ + int ret = 0; + struct btrfs_root *subvol_root; + + if (type != BTRFS_UUID_KEY_SUBVOL && + type != BTRFS_UUID_KEY_RECEIVED_SUBVOL) + return 0; + + subvol_root = btrfs_get_fs_root(fs_info, subvolid, true); + if (IS_ERR(subvol_root)) { + ret = PTR_ERR(subvol_root); + if (ret == -ENOENT) + return 1; + return ret; + } + + switch (type) { + case BTRFS_UUID_KEY_SUBVOL: + if (memcmp(uuid, subvol_root->root_item.uuid, BTRFS_UUID_SIZE)) + ret = 1; + break; + case BTRFS_UUID_KEY_RECEIVED_SUBVOL: + if (memcmp(uuid, subvol_root->root_item.received_uuid, + BTRFS_UUID_SIZE)) + ret = 1; + break; + } + btrfs_put_root(subvol_root); + + return ret; +} + +int btrfs_uuid_tree_iterate(struct btrfs_fs_info *fs_info) +{ + struct btrfs_root *root = fs_info->uuid_root; + struct btrfs_key key; + BTRFS_PATH_AUTO_FREE(path); + int ret = 0; + struct extent_buffer *leaf; + int slot; + u32 item_size; + unsigned long offset; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = 0; + key.type = 0; + key.offset = 0; + +again_search_slot: + ret = btrfs_search_forward(root, &key, path, BTRFS_OLDEST_GENERATION); + if (ret < 0) + return ret; + if (ret > 0) + return 0; + + while (1) { + if (btrfs_fs_closing(fs_info)) + return -EINTR; + + cond_resched(); + leaf = path->nodes[0]; + slot = path->slots[0]; + btrfs_item_key_to_cpu(leaf, &key, slot); + + if (key.type != BTRFS_UUID_KEY_SUBVOL && + key.type != BTRFS_UUID_KEY_RECEIVED_SUBVOL) + goto skip; + + offset = btrfs_item_ptr_offset(leaf, slot); + item_size = btrfs_item_size(leaf, slot); + if (!IS_ALIGNED(item_size, sizeof(u64))) { + btrfs_warn(fs_info, + "uuid item with illegal size %lu!", + (unsigned long)item_size); + goto skip; + } + while (item_size) { + u8 uuid[BTRFS_UUID_SIZE]; + __le64 subid_le; + u64 subid_cpu; + + put_unaligned_le64(key.objectid, uuid); + put_unaligned_le64(key.offset, uuid + sizeof(u64)); + read_extent_buffer(leaf, &subid_le, offset, + sizeof(subid_le)); + subid_cpu = le64_to_cpu(subid_le); + ret = btrfs_check_uuid_tree_entry(fs_info, uuid, + key.type, subid_cpu); + if (ret < 0) + return ret; + if (ret > 0) { + btrfs_release_path(path); + ret = btrfs_uuid_iter_rem(root, uuid, key.type, + subid_cpu); + if (ret == 0) { + /* + * this might look inefficient, but the + * justification is that it is an + * exception that check_func returns 1, + * and that in the regular case only one + * entry per UUID exists. + */ + goto again_search_slot; + } + if (ret < 0 && ret != -ENOENT) + return ret; + key.offset++; + goto again_search_slot; + } + item_size -= sizeof(subid_le); + offset += sizeof(subid_le); + } + +skip: + ret = btrfs_next_item(root, path); + if (ret == 0) + continue; + else if (ret > 0) + ret = 0; + break; + } + + return ret; +} + +int btrfs_uuid_scan_kthread(void *data) +{ + struct btrfs_fs_info *fs_info = data; + struct btrfs_root *root = fs_info->tree_root; + struct btrfs_key key; + struct btrfs_path *path = NULL; + int ret = 0; + struct extent_buffer *eb; + int slot; + struct btrfs_root_item root_item; + u32 item_size; + struct btrfs_trans_handle *trans = NULL; + bool closing = false; + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto out; + } + + key.objectid = 0; + key.type = BTRFS_ROOT_ITEM_KEY; + key.offset = 0; + + while (1) { + if (btrfs_fs_closing(fs_info)) { + closing = true; + break; + } + ret = btrfs_search_forward(root, &key, path, + BTRFS_OLDEST_GENERATION); + if (ret) { + if (ret > 0) + ret = 0; + break; + } + + if (key.type != BTRFS_ROOT_ITEM_KEY || + (key.objectid < BTRFS_FIRST_FREE_OBJECTID && + key.objectid != BTRFS_FS_TREE_OBJECTID) || + key.objectid > BTRFS_LAST_FREE_OBJECTID) + goto skip; + + eb = path->nodes[0]; + slot = path->slots[0]; + item_size = btrfs_item_size(eb, slot); + if (item_size < sizeof(root_item)) + goto skip; + + read_extent_buffer(eb, &root_item, + btrfs_item_ptr_offset(eb, slot), + (int)sizeof(root_item)); + if (btrfs_root_refs(&root_item) == 0) + goto skip; + + if (!btrfs_is_empty_uuid(root_item.uuid) || + !btrfs_is_empty_uuid(root_item.received_uuid)) { + if (trans) + goto update_tree; + + btrfs_release_path(path); + /* + * 1 - subvol uuid item + * 1 - received_subvol uuid item + */ + trans = btrfs_start_transaction(fs_info->uuid_root, 2); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + break; + } + continue; + } else { + goto skip; + } +update_tree: + btrfs_release_path(path); + if (!btrfs_is_empty_uuid(root_item.uuid)) { + ret = btrfs_uuid_tree_add(trans, root_item.uuid, + BTRFS_UUID_KEY_SUBVOL, + key.objectid); + if (ret < 0) { + btrfs_warn(fs_info, "uuid_tree_add failed %d", + ret); + break; + } + } + + if (!btrfs_is_empty_uuid(root_item.received_uuid)) { + ret = btrfs_uuid_tree_add(trans, + root_item.received_uuid, + BTRFS_UUID_KEY_RECEIVED_SUBVOL, + key.objectid); + if (ret < 0) { + btrfs_warn(fs_info, "uuid_tree_add failed %d", + ret); + break; + } + } + +skip: + btrfs_release_path(path); + if (trans) { + ret = btrfs_end_transaction(trans); + trans = NULL; + if (ret) + break; + } + + if (key.offset < (u64)-1) { + key.offset++; + } else if (key.type < BTRFS_ROOT_ITEM_KEY) { + key.offset = 0; + key.type = BTRFS_ROOT_ITEM_KEY; + } else if (key.objectid < (u64)-1) { + key.offset = 0; + key.type = BTRFS_ROOT_ITEM_KEY; + key.objectid++; + } else { + break; + } + cond_resched(); + } + +out: + btrfs_free_path(path); + if (!IS_ERR_OR_NULL(trans)) + btrfs_end_transaction(trans); + if (ret) + btrfs_warn(fs_info, "btrfs_uuid_scan_kthread failed %d", ret); + else if (!closing) + set_bit(BTRFS_FS_UPDATE_UUID_TREE_GEN, &fs_info->flags); + up(&fs_info->uuid_tree_rescan_sem); + return 0; +} + +int btrfs_create_uuid_tree(struct btrfs_fs_info *fs_info) +{ + struct btrfs_trans_handle *trans; + struct btrfs_root *tree_root = fs_info->tree_root; + struct btrfs_root *uuid_root; + struct task_struct *task; + int ret; + + /* + * 1 - root node + * 1 - root item + */ + trans = btrfs_start_transaction(tree_root, 2); + if (IS_ERR(trans)) + return PTR_ERR(trans); + + uuid_root = btrfs_create_tree(trans, BTRFS_UUID_TREE_OBJECTID); + if (IS_ERR(uuid_root)) { + ret = PTR_ERR(uuid_root); + btrfs_abort_transaction(trans, ret); + btrfs_end_transaction(trans); + return ret; + } + + fs_info->uuid_root = uuid_root; + + ret = btrfs_commit_transaction(trans); + if (ret) + return ret; + + down(&fs_info->uuid_tree_rescan_sem); + task = kthread_run(btrfs_uuid_scan_kthread, fs_info, "btrfs-uuid"); + if (IS_ERR(task)) { + /* fs_info->update_uuid_tree_gen remains 0 in all error case */ + btrfs_warn(fs_info, "failed to start uuid_scan task"); + up(&fs_info->uuid_tree_rescan_sem); + return PTR_ERR(task); + } + + return 0; +} diff --git a/fs/btrfs/uuid-tree.h b/fs/btrfs/uuid-tree.h new file mode 100644 index 000000000..02b235a36 --- /dev/null +++ b/fs/btrfs/uuid-tree.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_UUID_TREE_H +#define BTRFS_UUID_TREE_H + +#include + +struct btrfs_trans_handle; +struct btrfs_fs_info; + +int btrfs_uuid_tree_add(struct btrfs_trans_handle *trans, const u8 *uuid, u8 type, + u64 subid); +int btrfs_uuid_tree_remove(struct btrfs_trans_handle *trans, const u8 *uuid, u8 type, + u64 subid); +int btrfs_uuid_tree_check_overflow(struct btrfs_fs_info *fs_info, + const u8 *uuid, u8 type); +int btrfs_uuid_tree_iterate(struct btrfs_fs_info *fs_info); +int btrfs_create_uuid_tree(struct btrfs_fs_info *fs_info); +int btrfs_uuid_scan_kthread(void *data); + +#endif diff --git a/fs/btrfs/verity.c b/fs/btrfs/verity.c new file mode 100644 index 000000000..4e0ab5842 --- /dev/null +++ b/fs/btrfs/verity.c @@ -0,0 +1,806 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "messages.h" +#include "ctree.h" +#include "btrfs_inode.h" +#include "transaction.h" +#include "locking.h" +#include "fs.h" +#include "accessors.h" +#include "ioctl.h" +#include "verity.h" +#include "orphan.h" + +/* + * Implementation of the interface defined in struct fsverity_operations. + * + * The main question is how and where to store the verity descriptor and the + * Merkle tree. We store both in dedicated btree items in the filesystem tree, + * together with the rest of the inode metadata. This means we'll need to do + * extra work to encrypt them once encryption is supported in btrfs, but btrfs + * has a lot of careful code around i_size and it seems better to make a new key + * type than try and adjust all of our expectations for i_size. + * + * Note that this differs from the implementation in ext4 and f2fs, where + * this data is stored as if it were in the file, but past EOF. However, btrfs + * does not have a widespread mechanism for caching opaque metadata pages, so we + * do pretend that the Merkle tree pages themselves are past EOF for the + * purposes of caching them (as opposed to creating a virtual inode). + * + * fs verity items are stored under two different key types on disk. + * The descriptor items: + * [ inode objectid, BTRFS_VERITY_DESC_ITEM_KEY, offset ] + * + * At offset 0, we store a btrfs_verity_descriptor_item which tracks the + * size of the descriptor item and some extra data for encryption. + * Starting at offset 1, these hold the generic fs verity descriptor. + * The latter are opaque to btrfs, we just read and write them as a blob for + * the higher level verity code. The most common descriptor size is 256 bytes. + * + * The merkle tree items: + * [ inode objectid, BTRFS_VERITY_MERKLE_ITEM_KEY, offset ] + * + * These also start at offset 0, and correspond to the merkle tree bytes. + * So when fsverity asks for page 0 of the merkle tree, we pull up one page + * starting at offset 0 for this key type. These are also opaque to btrfs, + * we're blindly storing whatever fsverity sends down. + * + * Another important consideration is the fact that the Merkle tree data scales + * linearly with the size of the file (with 4K pages/blocks and SHA-256, it's + * ~1/127th the size) so for large files, writing the tree can be a lengthy + * operation. For that reason, we guard the whole enable verity operation + * (between begin_enable_verity and end_enable_verity) with an orphan item. + * Again, because the data can be pretty large, it's quite possible that we + * could run out of space writing it, so we try our best to handle errors by + * stopping and rolling back rather than aborting the victim transaction. + */ + +#define MERKLE_START_ALIGN 65536 + +/* + * Compute the logical file offset where we cache the Merkle tree. + * + * @inode: inode of the verity file + * + * For the purposes of caching the Merkle tree pages, as required by + * fs-verity, it is convenient to do size computations in terms of a file + * offset, rather than in terms of page indices. + * + * Use 64K to be sure it's past the last page in the file, even with 64K pages. + * That rounding operation itself can overflow loff_t, so we do it in u64 and + * check. + * + * Returns the file offset on success, negative error code on failure. + */ +static loff_t merkle_file_pos(const struct inode *inode) +{ + u64 sz = inode->i_size; + u64 rounded = round_up(sz, MERKLE_START_ALIGN); + + if (rounded > inode->i_sb->s_maxbytes) + return -EFBIG; + + return rounded; +} + +/* + * Drop all the items for this inode with this key_type. + * + * @inode: inode to drop items for + * @key_type: type of items to drop (BTRFS_VERITY_DESC_ITEM or + * BTRFS_VERITY_MERKLE_ITEM) + * + * Before doing a verity enable we cleanup any existing verity items. + * This is also used to clean up if a verity enable failed half way through. + * + * Returns number of dropped items on success, negative error code on failure. + */ +static int drop_verity_items(struct btrfs_inode *inode, u8 key_type) +{ + struct btrfs_trans_handle *trans; + struct btrfs_root *root = inode->root; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + int count = 0; + int ret; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + while (1) { + /* 1 for the item being dropped */ + trans = btrfs_start_transaction(root, 1); + if (IS_ERR(trans)) + return PTR_ERR(trans); + + /* + * Walk backwards through all the items until we find one that + * isn't from our key type or objectid + */ + key.objectid = btrfs_ino(inode); + key.type = key_type; + key.offset = (u64)-1; + + ret = btrfs_search_slot(trans, root, &key, path, -1, 1); + if (ret > 0) { + ret = 0; + /* No more keys of this type, we're done */ + if (path->slots[0] == 0) + break; + path->slots[0]--; + } else if (ret < 0) { + btrfs_end_transaction(trans); + return ret; + } + + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + + /* No more keys of this type, we're done */ + if (key.objectid != btrfs_ino(inode) || key.type != key_type) + break; + + /* + * This shouldn't be a performance sensitive function because + * it's not used as part of truncate. If it ever becomes + * perf sensitive, change this to walk forward and bulk delete + * items + */ + ret = btrfs_del_items(trans, root, path, path->slots[0], 1); + if (ret) { + btrfs_end_transaction(trans); + return ret; + } + count++; + btrfs_release_path(path); + btrfs_end_transaction(trans); + } + btrfs_end_transaction(trans); + return count; +} + +/* + * Drop all verity items + * + * @inode: inode to drop verity items for + * + * In most contexts where we are dropping verity items, we want to do it for all + * the types of verity items, not a particular one. + * + * Returns: 0 on success, negative error code on failure. + */ +int btrfs_drop_verity_items(struct btrfs_inode *inode) +{ + int ret; + + ret = drop_verity_items(inode, BTRFS_VERITY_DESC_ITEM_KEY); + if (ret < 0) + return ret; + ret = drop_verity_items(inode, BTRFS_VERITY_MERKLE_ITEM_KEY); + if (ret < 0) + return ret; + + return 0; +} + +/* + * Insert and write inode items with a given key type and offset. + * + * @inode: inode to insert for + * @key_type: key type to insert + * @offset: item offset to insert at + * @src: source data to write + * @len: length of source data to write + * + * Write len bytes from src into items of up to 2K length. + * The inserted items will have key (ino, key_type, offset + off) where off is + * consecutively increasing from 0 up to the last item ending at offset + len. + * + * Returns 0 on success and a negative error code on failure. + */ +static int write_key_bytes(struct btrfs_inode *inode, u8 key_type, u64 offset, + const char *src, u64 len) +{ + struct btrfs_trans_handle *trans; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_root *root = inode->root; + struct extent_buffer *leaf; + struct btrfs_key key; + unsigned long copy_bytes; + unsigned long src_offset = 0; + void *data; + int ret = 0; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + while (len > 0) { + /* 1 for the new item being inserted */ + trans = btrfs_start_transaction(root, 1); + if (IS_ERR(trans)) + return PTR_ERR(trans); + + key.objectid = btrfs_ino(inode); + key.type = key_type; + key.offset = offset; + + /* + * Insert 2K at a time mostly to be friendly for smaller leaf + * size filesystems + */ + copy_bytes = min_t(u64, len, 2048); + + ret = btrfs_insert_empty_item(trans, root, path, &key, copy_bytes); + if (ret) { + btrfs_end_transaction(trans); + break; + } + + leaf = path->nodes[0]; + + data = btrfs_item_ptr(leaf, path->slots[0], void); + write_extent_buffer(leaf, src + src_offset, + (unsigned long)data, copy_bytes); + offset += copy_bytes; + src_offset += copy_bytes; + len -= copy_bytes; + + btrfs_release_path(path); + btrfs_end_transaction(trans); + } + + return ret; +} + +/* + * Read inode items of the given key type and offset from the btree. + * + * @inode: inode to read items of + * @key_type: key type to read + * @offset: item offset to read from + * @dest: Buffer to read into. This parameter has slightly tricky + * semantics. If it is NULL, the function will not do any copying + * and will just return the size of all the items up to len bytes. + * If dest_page is passed, then the function will kmap_local the + * page and ignore dest, but it must still be non-NULL to avoid the + * counting-only behavior. + * @len: length in bytes to read + * @dest_folio: copy into this folio instead of the dest buffer + * + * Helper function to read items from the btree. This returns the number of + * bytes read or < 0 for errors. We can return short reads if the items don't + * exist on disk or aren't big enough to fill the desired length. Supports + * reading into a provided buffer (dest) or into the page cache + * + * Returns number of bytes read or a negative error code on failure. + */ +static int read_key_bytes(struct btrfs_inode *inode, u8 key_type, u64 offset, + char *dest, u64 len, struct folio *dest_folio) +{ + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_root *root = inode->root; + struct extent_buffer *leaf; + struct btrfs_key key; + u64 item_end; + u64 copy_end; + int copied = 0; + u32 copy_offset; + unsigned long copy_bytes; + unsigned long dest_offset = 0; + void *data; + char *kaddr = dest; + int ret; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + if (dest_folio) + path->reada = READA_FORWARD; + + key.objectid = btrfs_ino(inode); + key.type = key_type; + key.offset = offset; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) { + goto out; + } else if (ret > 0) { + ret = 0; + if (path->slots[0] == 0) + goto out; + path->slots[0]--; + } + + while (len > 0) { + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + + if (key.objectid != btrfs_ino(inode) || key.type != key_type) + break; + + item_end = btrfs_item_size(leaf, path->slots[0]) + key.offset; + + if (copied > 0) { + /* + * Once we've copied something, we want all of the items + * to be sequential + */ + if (key.offset != offset) + break; + } else { + /* + * Our initial offset might be in the middle of an + * item. Make sure it all makes sense. + */ + if (key.offset > offset) + break; + if (item_end <= offset) + break; + } + + /* desc = NULL to just sum all the item lengths */ + if (!dest) + copy_end = item_end; + else + copy_end = min(offset + len, item_end); + + /* Number of bytes in this item we want to copy */ + copy_bytes = copy_end - offset; + + /* Offset from the start of item for copying */ + copy_offset = offset - key.offset; + + if (dest) { + if (dest_folio) + kaddr = kmap_local_folio(dest_folio, 0); + + data = btrfs_item_ptr(leaf, path->slots[0], void); + read_extent_buffer(leaf, kaddr + dest_offset, + (unsigned long)data + copy_offset, + copy_bytes); + + if (dest_folio) + kunmap_local(kaddr); + } + + offset += copy_bytes; + dest_offset += copy_bytes; + len -= copy_bytes; + copied += copy_bytes; + + path->slots[0]++; + if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) { + /* + * We've reached the last slot in this leaf and we need + * to go to the next leaf. + */ + ret = btrfs_next_leaf(root, path); + if (ret < 0) { + break; + } else if (ret > 0) { + ret = 0; + break; + } + } + } +out: + if (!ret) + ret = copied; + return ret; +} + +/* + * Delete an fsverity orphan + * + * @trans: transaction to do the delete in + * @inode: inode to orphan + * + * Capture verity orphan specific logic that is repeated in the couple places + * we delete verity orphans. Specifically, handling ENOENT and ignoring inodes + * with 0 links. + * + * Returns zero on success or a negative error code on failure. + */ +static int del_orphan(struct btrfs_trans_handle *trans, struct btrfs_inode *inode) +{ + struct btrfs_root *root = inode->root; + int ret; + + /* + * If the inode has no links, it is either already unlinked, or was + * created with O_TMPFILE. In either case, it should have an orphan from + * that other operation. Rather than reference count the orphans, we + * simply ignore them here, because we only invoke the verity path in + * the orphan logic when i_nlink is 1. + */ + if (!inode->vfs_inode.i_nlink) + return 0; + + ret = btrfs_del_orphan_item(trans, root, btrfs_ino(inode)); + if (ret == -ENOENT) + ret = 0; + return ret; +} + +/* + * Rollback in-progress verity if we encounter an error. + * + * @inode: inode verity had an error for + * + * We try to handle recoverable errors while enabling verity by rolling it back + * and just failing the operation, rather than having an fs level error no + * matter what. However, any error in rollback is unrecoverable. + * + * Returns 0 on success, negative error code on failure. + */ +static int rollback_verity(struct btrfs_inode *inode) +{ + struct btrfs_trans_handle *trans = NULL; + struct btrfs_root *root = inode->root; + int ret; + + btrfs_assert_inode_locked(inode); + truncate_inode_pages(inode->vfs_inode.i_mapping, inode->vfs_inode.i_size); + clear_bit(BTRFS_INODE_VERITY_IN_PROGRESS, &inode->runtime_flags); + ret = btrfs_drop_verity_items(inode); + if (ret) { + btrfs_handle_fs_error(root->fs_info, ret, + "failed to drop verity items in rollback %llu", + inode->vfs_inode.i_ino); + goto out; + } + + /* + * 1 for updating the inode flag + * 1 for deleting the orphan + */ + trans = btrfs_start_transaction(root, 2); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + trans = NULL; + btrfs_handle_fs_error(root->fs_info, ret, + "failed to start transaction in verity rollback %llu", + inode->vfs_inode.i_ino); + goto out; + } + inode->ro_flags &= ~BTRFS_INODE_RO_VERITY; + btrfs_sync_inode_flags_to_i_flags(inode); + ret = btrfs_update_inode(trans, inode); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + ret = del_orphan(trans, inode); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } +out: + if (trans) + btrfs_end_transaction(trans); + return ret; +} + +/* + * Finalize making the file a valid verity file + * + * @inode: inode to be marked as verity + * @desc: contents of the verity descriptor to write (not NULL) + * @desc_size: size of the verity descriptor + * + * Do the actual work of finalizing verity after successfully writing the Merkle + * tree: + * + * - write out the descriptor items + * - mark the inode with the verity flag + * - delete the orphan item + * - mark the ro compat bit + * - clear the in progress bit + * + * Returns 0 on success, negative error code on failure. + */ +static int finish_verity(struct btrfs_inode *inode, const void *desc, + size_t desc_size) +{ + struct btrfs_trans_handle *trans = NULL; + struct btrfs_root *root = inode->root; + struct btrfs_verity_descriptor_item item; + int ret; + + /* Write out the descriptor item */ + memset(&item, 0, sizeof(item)); + btrfs_set_stack_verity_descriptor_size(&item, desc_size); + ret = write_key_bytes(inode, BTRFS_VERITY_DESC_ITEM_KEY, 0, + (const char *)&item, sizeof(item)); + if (ret) + return ret; + + /* Write out the descriptor itself */ + ret = write_key_bytes(inode, BTRFS_VERITY_DESC_ITEM_KEY, 1, + desc, desc_size); + if (ret) + return ret; + + /* + * 1 for updating the inode flag + * 1 for deleting the orphan + */ + trans = btrfs_start_transaction(root, 2); + if (IS_ERR(trans)) + return PTR_ERR(trans); + inode->ro_flags |= BTRFS_INODE_RO_VERITY; + btrfs_sync_inode_flags_to_i_flags(inode); + ret = btrfs_update_inode(trans, inode); + if (ret) + goto end_trans; + ret = del_orphan(trans, inode); + if (ret) + goto end_trans; + clear_bit(BTRFS_INODE_VERITY_IN_PROGRESS, &inode->runtime_flags); + btrfs_set_fs_compat_ro(root->fs_info, VERITY); +end_trans: + btrfs_end_transaction(trans); + return ret; + +} + +/* + * fsverity op that begins enabling verity. + * + * @filp: file to enable verity on + * + * Begin enabling fsverity for the file. We drop any existing verity items, add + * an orphan and set the in progress bit. + * + * Returns 0 on success, negative error code on failure. + */ +static int btrfs_begin_enable_verity(struct file *filp) +{ + struct btrfs_inode *inode = BTRFS_I(file_inode(filp)); + struct btrfs_root *root = inode->root; + struct btrfs_trans_handle *trans; + int ret; + + btrfs_assert_inode_locked(inode); + + if (IS_ENCRYPTED(&inode->vfs_inode)) + return -EOPNOTSUPP; + + if (test_bit(BTRFS_INODE_VERITY_IN_PROGRESS, &inode->runtime_flags)) + return -EBUSY; + + /* + * This should almost never do anything, but theoretically, it's + * possible that we failed to enable verity on a file, then were + * interrupted or failed while rolling back, failed to cleanup the + * orphan, and finally attempt to enable verity again. + */ + ret = btrfs_drop_verity_items(inode); + if (ret) + return ret; + + /* 1 for the orphan item */ + trans = btrfs_start_transaction(root, 1); + if (IS_ERR(trans)) + return PTR_ERR(trans); + + ret = btrfs_orphan_add(trans, inode); + if (!ret) + set_bit(BTRFS_INODE_VERITY_IN_PROGRESS, &inode->runtime_flags); + btrfs_end_transaction(trans); + + return 0; +} + +/* + * fsverity op that ends enabling verity. + * + * @filp: file we are finishing enabling verity on + * @desc: verity descriptor to write out (NULL in error conditions) + * @desc_size: size of the verity descriptor (variable with signatures) + * @merkle_tree_size: size of the merkle tree in bytes + * + * If desc is null, then VFS is signaling an error occurred during verity + * enable, and we should try to rollback. Otherwise, attempt to finish verity. + * + * Returns 0 on success, negative error code on error. + */ +static int btrfs_end_enable_verity(struct file *filp, const void *desc, + size_t desc_size, u64 merkle_tree_size) +{ + struct btrfs_inode *inode = BTRFS_I(file_inode(filp)); + int ret = 0; + int rollback_ret; + + btrfs_assert_inode_locked(inode); + + if (desc == NULL) + goto rollback; + + ret = finish_verity(inode, desc, desc_size); + if (ret) + goto rollback; + return ret; + +rollback: + rollback_ret = rollback_verity(inode); + if (rollback_ret) + btrfs_err(inode->root->fs_info, + "failed to rollback verity items: %pe", ERR_PTR(rollback_ret)); + return ret; +} + +/* + * fsverity op that gets the struct fsverity_descriptor. + * + * @inode: inode to get the descriptor of + * @buf: output buffer for the descriptor contents + * @buf_size: size of the output buffer. 0 to query the size + * + * fsverity does a two pass setup for reading the descriptor, in the first pass + * it calls with buf_size = 0 to query the size of the descriptor, and then in + * the second pass it actually reads the descriptor off disk. + * + * Returns the size on success or a negative error code on failure. + */ +int btrfs_get_verity_descriptor(struct inode *inode, void *buf, size_t buf_size) +{ + u64 true_size; + int ret = 0; + struct btrfs_verity_descriptor_item item; + + memset(&item, 0, sizeof(item)); + ret = read_key_bytes(BTRFS_I(inode), BTRFS_VERITY_DESC_ITEM_KEY, 0, + (char *)&item, sizeof(item), NULL); + if (ret < 0) + return ret; + + if (unlikely(item.reserved[0] != 0 || item.reserved[1] != 0)) + return -EUCLEAN; + + true_size = btrfs_stack_verity_descriptor_size(&item); + if (unlikely(true_size > INT_MAX)) + return -EUCLEAN; + + if (buf_size == 0) + return true_size; + if (buf_size < true_size) + return -ERANGE; + + ret = read_key_bytes(BTRFS_I(inode), BTRFS_VERITY_DESC_ITEM_KEY, 1, + buf, buf_size, NULL); + if (ret < 0) + return ret; + if (ret != true_size) + return -EIO; + + return true_size; +} + +/* + * fsverity op that reads and caches a merkle tree page. + * + * @inode: inode to read a merkle tree page for + * @index: page index relative to the start of the merkle tree + * + * The Merkle tree is stored in the filesystem btree, but its pages are cached + * with a logical position past EOF in the inode's mapping. + * + * Returns the page we read, or an ERR_PTR on error. + */ +static struct page *btrfs_read_merkle_tree_page(struct inode *inode, + pgoff_t index) +{ + struct folio *folio; + u64 off = (u64)index << PAGE_SHIFT; + loff_t merkle_pos = merkle_file_pos(inode); + int ret; + + if (merkle_pos < 0) + return ERR_PTR(merkle_pos); + if (merkle_pos > inode->i_sb->s_maxbytes - off - PAGE_SIZE) + return ERR_PTR(-EFBIG); + index += merkle_pos >> PAGE_SHIFT; +again: + folio = __filemap_get_folio(inode->i_mapping, index, FGP_ACCESSED, 0); + if (!IS_ERR(folio)) { + if (folio_test_uptodate(folio)) + goto out; + + folio_lock(folio); + /* Folio was truncated from mapping. */ + if (!folio->mapping) { + folio_unlock(folio); + folio_put(folio); + goto again; + } + /* Another reader may have filled the folio while we waited. */ + if (folio_test_uptodate(folio)) { + folio_unlock(folio); + goto out; + } + goto read_folio; + } + + folio = filemap_alloc_folio(mapping_gfp_constraint(inode->i_mapping, ~__GFP_FS), + 0, NULL); + if (!folio) + return ERR_PTR(-ENOMEM); + + ret = filemap_add_folio(inode->i_mapping, folio, index, GFP_NOFS); + if (ret) { + folio_put(folio); + /* Did someone else insert a folio here? */ + if (ret == -EEXIST) + goto again; + return ERR_PTR(ret); + } + +read_folio: + /* + * Merkle item keys are indexed from byte 0 in the merkle tree. + * They have the form: + * + * [ inode objectid, BTRFS_MERKLE_ITEM_KEY, offset in bytes ] + */ + ret = read_key_bytes(BTRFS_I(inode), BTRFS_VERITY_MERKLE_ITEM_KEY, off, + folio_address(folio), PAGE_SIZE, folio); + if (ret < 0) { + folio_unlock(folio); + folio_put(folio); + return ERR_PTR(ret); + } + if (ret < PAGE_SIZE) + folio_zero_segment(folio, ret, PAGE_SIZE); + + folio_mark_uptodate(folio); + folio_unlock(folio); + +out: + return folio_file_page(folio, index); +} + +/* + * fsverity op that writes a Merkle tree block into the btree. + * + * @file: file to write a Merkle tree block for + * @buf: Merkle tree block to write + * @pos: the position of the block in the Merkle tree (in bytes) + * @size: the Merkle tree block size (in bytes) + * + * Returns 0 on success or negative error code on failure + */ +static int btrfs_write_merkle_tree_block(struct file *file, const void *buf, + u64 pos, unsigned int size) +{ + struct inode *inode = file_inode(file); + loff_t merkle_pos = merkle_file_pos(inode); + + if (merkle_pos < 0) + return merkle_pos; + if (merkle_pos > inode->i_sb->s_maxbytes - pos - size) + return -EFBIG; + + return write_key_bytes(BTRFS_I(inode), BTRFS_VERITY_MERKLE_ITEM_KEY, + pos, buf, size); +} + +const struct fsverity_operations btrfs_verityops = { + .begin_enable_verity = btrfs_begin_enable_verity, + .end_enable_verity = btrfs_end_enable_verity, + .get_verity_descriptor = btrfs_get_verity_descriptor, + .read_merkle_tree_page = btrfs_read_merkle_tree_page, + .write_merkle_tree_block = btrfs_write_merkle_tree_block, +}; diff --git a/fs/btrfs/verity.h b/fs/btrfs/verity.h new file mode 100644 index 000000000..d696659e4 --- /dev/null +++ b/fs/btrfs/verity.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_VERITY_H +#define BTRFS_VERITY_H + +struct inode; +struct btrfs_inode; + +#ifdef CONFIG_FS_VERITY + +#include + +extern const struct fsverity_operations btrfs_verityops; + +int btrfs_drop_verity_items(struct btrfs_inode *inode); +int btrfs_get_verity_descriptor(struct inode *inode, void *buf, size_t buf_size); + +#else + +#include + +static inline int btrfs_drop_verity_items(struct btrfs_inode *inode) +{ + return 0; +} + +static inline int btrfs_get_verity_descriptor(struct inode *inode, void *buf, + size_t buf_size) +{ + return -EPERM; +} + +#endif + +#endif diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c new file mode 100644 index 000000000..9b66eb584 --- /dev/null +++ b/fs/btrfs/volumes.c @@ -0,0 +1,9034 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "misc.h" +#include "disk-io.h" +#include "extent-tree.h" +#include "transaction.h" +#include "volumes.h" +#include "raid56.h" +#include "dev-replace.h" +#include "sysfs.h" +#include "tree-checker.h" +#include "space-info.h" +#include "block-group.h" +#include "discard.h" +#include "zoned.h" +#include "fs.h" +#include "accessors.h" +#include "uuid-tree.h" +#include "ioctl.h" +#include "relocation.h" +#include "scrub.h" +#include "super.h" +#include "raid-stripe-tree.h" + +#define BTRFS_BLOCK_GROUP_STRIPE_MASK (BTRFS_BLOCK_GROUP_RAID0 | \ + BTRFS_BLOCK_GROUP_RAID10 | \ + BTRFS_BLOCK_GROUP_RAID56_MASK) + +struct btrfs_io_geometry { + u32 stripe_index; + u32 stripe_nr; + int mirror_num; + int num_stripes; + u64 stripe_offset; + u64 raid56_full_stripe_start; + int max_errors; + enum btrfs_map_op op; + bool use_rst; +}; + +const struct btrfs_raid_attr btrfs_raid_array[BTRFS_NR_RAID_TYPES] = { + [BTRFS_RAID_RAID10] = { + .sub_stripes = 2, + .dev_stripes = 1, + .devs_max = 0, /* 0 == as many as possible */ + .devs_min = 2, + .tolerated_failures = 1, + .devs_increment = 2, + .ncopies = 2, + .nparity = 0, + .raid_name = "raid10", + .bg_flag = BTRFS_BLOCK_GROUP_RAID10, + .mindev_error = BTRFS_ERROR_DEV_RAID10_MIN_NOT_MET, + }, + [BTRFS_RAID_RAID1] = { + .sub_stripes = 1, + .dev_stripes = 1, + .devs_max = 2, + .devs_min = 2, + .tolerated_failures = 1, + .devs_increment = 2, + .ncopies = 2, + .nparity = 0, + .raid_name = "raid1", + .bg_flag = BTRFS_BLOCK_GROUP_RAID1, + .mindev_error = BTRFS_ERROR_DEV_RAID1_MIN_NOT_MET, + }, + [BTRFS_RAID_RAID1C3] = { + .sub_stripes = 1, + .dev_stripes = 1, + .devs_max = 3, + .devs_min = 3, + .tolerated_failures = 2, + .devs_increment = 3, + .ncopies = 3, + .nparity = 0, + .raid_name = "raid1c3", + .bg_flag = BTRFS_BLOCK_GROUP_RAID1C3, + .mindev_error = BTRFS_ERROR_DEV_RAID1C3_MIN_NOT_MET, + }, + [BTRFS_RAID_RAID1C4] = { + .sub_stripes = 1, + .dev_stripes = 1, + .devs_max = 4, + .devs_min = 4, + .tolerated_failures = 3, + .devs_increment = 4, + .ncopies = 4, + .nparity = 0, + .raid_name = "raid1c4", + .bg_flag = BTRFS_BLOCK_GROUP_RAID1C4, + .mindev_error = BTRFS_ERROR_DEV_RAID1C4_MIN_NOT_MET, + }, + [BTRFS_RAID_DUP] = { + .sub_stripes = 1, + .dev_stripes = 2, + .devs_max = 1, + .devs_min = 1, + .tolerated_failures = 0, + .devs_increment = 1, + .ncopies = 2, + .nparity = 0, + .raid_name = "dup", + .bg_flag = BTRFS_BLOCK_GROUP_DUP, + .mindev_error = 0, + }, + [BTRFS_RAID_RAID0] = { + .sub_stripes = 1, + .dev_stripes = 1, + .devs_max = 0, + .devs_min = 1, + .tolerated_failures = 0, + .devs_increment = 1, + .ncopies = 1, + .nparity = 0, + .raid_name = "raid0", + .bg_flag = BTRFS_BLOCK_GROUP_RAID0, + .mindev_error = 0, + }, + [BTRFS_RAID_SINGLE] = { + .sub_stripes = 1, + .dev_stripes = 1, + .devs_max = 1, + .devs_min = 1, + .tolerated_failures = 0, + .devs_increment = 1, + .ncopies = 1, + .nparity = 0, + .raid_name = "single", + .bg_flag = 0, + .mindev_error = 0, + }, + [BTRFS_RAID_RAID5] = { + .sub_stripes = 1, + .dev_stripes = 1, + .devs_max = 0, + .devs_min = 2, + .tolerated_failures = 1, + .devs_increment = 1, + .ncopies = 1, + .nparity = 1, + .raid_name = "raid5", + .bg_flag = BTRFS_BLOCK_GROUP_RAID5, + .mindev_error = BTRFS_ERROR_DEV_RAID5_MIN_NOT_MET, + }, + [BTRFS_RAID_RAID6] = { + .sub_stripes = 1, + .dev_stripes = 1, + .devs_max = 0, + .devs_min = 3, + .tolerated_failures = 2, + .devs_increment = 1, + .ncopies = 1, + .nparity = 2, + .raid_name = "raid6", + .bg_flag = BTRFS_BLOCK_GROUP_RAID6, + .mindev_error = BTRFS_ERROR_DEV_RAID6_MIN_NOT_MET, + }, +}; + +/* + * Convert block group flags (BTRFS_BLOCK_GROUP_*) to btrfs_raid_types, which + * can be used as index to access btrfs_raid_array[]. + */ +enum btrfs_raid_types __attribute_const__ btrfs_bg_flags_to_raid_index(u64 flags) +{ + const u64 profile = (flags & BTRFS_BLOCK_GROUP_PROFILE_MASK); + + if (!profile) + return BTRFS_RAID_SINGLE; + + return BTRFS_BG_FLAG_TO_INDEX(profile); +} + +const char *btrfs_bg_type_to_raid_name(u64 flags) +{ + const int index = btrfs_bg_flags_to_raid_index(flags); + + if (index >= BTRFS_NR_RAID_TYPES) + return NULL; + + return btrfs_raid_array[index].raid_name; +} + +int btrfs_nr_parity_stripes(u64 type) +{ + enum btrfs_raid_types index = btrfs_bg_flags_to_raid_index(type); + + return btrfs_raid_array[index].nparity; +} + +/* + * Fill @buf with textual description of @bg_flags, no more than @size_buf + * bytes including terminating null byte. + */ +void btrfs_describe_block_groups(u64 bg_flags, char *buf, u32 size_buf) +{ + int i; + int ret; + char *bp = buf; + u64 flags = bg_flags; + u32 size_bp = size_buf; + + if (!flags) + return; + +#define DESCRIBE_FLAG(flag, desc) \ + do { \ + if (flags & (flag)) { \ + ret = snprintf(bp, size_bp, "%s|", (desc)); \ + if (ret < 0 || ret >= size_bp) \ + goto out_overflow; \ + size_bp -= ret; \ + bp += ret; \ + flags &= ~(flag); \ + } \ + } while (0) + + DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_DATA, "data"); + DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_SYSTEM, "system"); + DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_METADATA, "metadata"); + /* Block groups containing the remap tree. */ + DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_METADATA_REMAP, "metadata-remap"); + /* Block group that has been remapped. */ + DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_REMAPPED, "remapped"); + + DESCRIBE_FLAG(BTRFS_AVAIL_ALLOC_BIT_SINGLE, "single"); + for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) + DESCRIBE_FLAG(btrfs_raid_array[i].bg_flag, + btrfs_raid_array[i].raid_name); +#undef DESCRIBE_FLAG + + if (flags) { + ret = snprintf(bp, size_bp, "0x%llx|", flags); + size_bp -= ret; + } + + if (size_bp < size_buf) + buf[size_buf - size_bp - 1] = '\0'; /* remove last | */ + + /* + * The text is trimmed, it's up to the caller to provide sufficiently + * large buffer + */ +out_overflow:; +} + +static int init_first_rw_device(struct btrfs_trans_handle *trans); +static int btrfs_relocate_sys_chunks(struct btrfs_fs_info *fs_info); +static void btrfs_dev_stat_print_on_load(struct btrfs_device *device); + +/* + * Device locking + * ============== + * + * There are several mutexes that protect manipulation of devices and low-level + * structures like chunks but not block groups, extents or files + * + * uuid_mutex (global lock) + * ------------------------ + * protects the fs_uuids list that tracks all per-fs fs_devices, resulting from + * the SCAN_DEV ioctl registration or from mount either implicitly (the first + * device) or requested by the device= mount option + * + * the mutex can be very coarse and can cover long-running operations + * + * protects: updates to fs_devices counters like missing devices, rw devices, + * seeding, structure cloning, opening/closing devices at mount/umount time + * + * global::fs_devs - add, remove, updates to the global list + * + * does not protect: manipulation of the fs_devices::devices list in general + * but in mount context it could be used to exclude list modifications by eg. + * scan ioctl + * + * btrfs_device::name - renames (write side), read is RCU + * + * fs_devices::device_list_mutex (per-fs, with RCU) + * ------------------------------------------------ + * protects updates to fs_devices::devices, ie. adding and deleting + * + * simple list traversal with read-only actions can be done with RCU protection + * + * may be used to exclude some operations from running concurrently without any + * modifications to the list (see write_all_supers) + * + * Is not required at mount and close times, because our device list is + * protected by the uuid_mutex at that point. + * + * balance_mutex + * ------------- + * protects balance structures (status, state) and context accessed from + * several places (internally, ioctl) + * + * chunk_mutex + * ----------- + * protects chunks, adding or removing during allocation, trim or when a new + * device is added/removed. Additionally it also protects post_commit_list of + * individual devices, since they can be added to the transaction's + * post_commit_list only with chunk_mutex held. + * + * cleaner_mutex + * ------------- + * a big lock that is held by the cleaner thread and prevents running subvolume + * cleaning together with relocation or delayed iputs + * + * + * Lock nesting + * ============ + * + * uuid_mutex + * device_list_mutex + * chunk_mutex + * balance_mutex + * + * + * Exclusive operations + * ==================== + * + * Maintains the exclusivity of the following operations that apply to the + * whole filesystem and cannot run in parallel. + * + * - Balance (*) + * - Device add + * - Device remove + * - Device replace (*) + * - Resize + * + * The device operations (as above) can be in one of the following states: + * + * - Running state + * - Paused state + * - Completed state + * + * Only device operations marked with (*) can go into the Paused state for the + * following reasons: + * + * - ioctl (only Balance can be Paused through ioctl) + * - filesystem remounted as read-only + * - filesystem unmounted and mounted as read-only + * - system power-cycle and filesystem mounted as read-only + * - filesystem or device errors leading to forced read-only + * + * The status of exclusive operation is set and cleared atomically. + * During the course of Paused state, fs_info::exclusive_operation remains set. + * A device operation in Paused or Running state can be canceled or resumed + * either by ioctl (Balance only) or when remounted as read-write. + * The exclusive status is cleared when the device operation is canceled or + * completed. + */ + +DEFINE_MUTEX(uuid_mutex); +static LIST_HEAD(fs_uuids); +struct list_head * __attribute_const__ btrfs_get_fs_uuids(void) +{ + return &fs_uuids; +} + +/* + * Allocate new btrfs_fs_devices structure identified by a fsid. + * + * @fsid: if not NULL, copy the UUID to fs_devices::fsid and to + * fs_devices::metadata_fsid + * + * Return a pointer to a new struct btrfs_fs_devices on success, or ERR_PTR(). + * The returned struct is not linked onto any lists and can be destroyed with + * kfree() right away. + */ +static struct btrfs_fs_devices *alloc_fs_devices(const u8 *fsid) +{ + struct btrfs_fs_devices *fs_devs; + + fs_devs = kzalloc_obj(*fs_devs); + if (!fs_devs) + return ERR_PTR(-ENOMEM); + + mutex_init(&fs_devs->device_list_mutex); + + INIT_LIST_HEAD(&fs_devs->devices); + INIT_LIST_HEAD(&fs_devs->alloc_list); + INIT_LIST_HEAD(&fs_devs->fs_list); + INIT_LIST_HEAD(&fs_devs->seed_list); + spin_lock_init(&fs_devs->per_profile_lock); + + if (fsid) { + memcpy(fs_devs->fsid, fsid, BTRFS_FSID_SIZE); + memcpy(fs_devs->metadata_uuid, fsid, BTRFS_FSID_SIZE); + } + + return fs_devs; +} + +static void btrfs_free_device(struct btrfs_device *device) +{ + WARN_ON(!list_empty(&device->post_commit_list)); + /* + * No need to call kfree_rcu() nor do RCU lock/unlock, nothing is + * reading the device name. + */ + kfree(rcu_dereference_raw(device->name)); + btrfs_extent_io_tree_release(&device->alloc_state); + btrfs_destroy_dev_zone_info(device); + kfree(device); +} + +static void free_fs_devices(struct btrfs_fs_devices *fs_devices) +{ + struct btrfs_device *device; + + WARN_ON(fs_devices->opened); + WARN_ON(fs_devices->holding); + while (!list_empty(&fs_devices->devices)) { + device = list_first_entry(&fs_devices->devices, + struct btrfs_device, dev_list); + list_del(&device->dev_list); + btrfs_free_device(device); + } + kfree(fs_devices); +} + +void __exit btrfs_cleanup_fs_uuids(void) +{ + struct btrfs_fs_devices *fs_devices; + + while (!list_empty(&fs_uuids)) { + fs_devices = list_first_entry(&fs_uuids, struct btrfs_fs_devices, + fs_list); + list_del(&fs_devices->fs_list); + free_fs_devices(fs_devices); + } +} + +static bool match_fsid_fs_devices(const struct btrfs_fs_devices *fs_devices, + const u8 *fsid, const u8 *metadata_fsid) +{ + if (memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE) != 0) + return false; + + if (!metadata_fsid) + return true; + + if (memcmp(metadata_fsid, fs_devices->metadata_uuid, BTRFS_FSID_SIZE) != 0) + return false; + + return true; +} + +static noinline struct btrfs_fs_devices *find_fsid( + const u8 *fsid, const u8 *metadata_fsid) +{ + struct btrfs_fs_devices *fs_devices; + + ASSERT(fsid); + + /* Handle non-split brain cases */ + list_for_each_entry(fs_devices, &fs_uuids, fs_list) { + if (match_fsid_fs_devices(fs_devices, fsid, metadata_fsid)) + return fs_devices; + } + return NULL; +} + +static int +btrfs_get_bdev_and_sb(const char *device_path, blk_mode_t flags, void *holder, + int flush, struct file **bdev_file, + struct btrfs_super_block **disk_super) +{ + struct block_device *bdev; + int ret; + + if (holder) + *bdev_file = fs_bdev_file_open_by_path(device_path, flags, + holder, holder); + else + *bdev_file = bdev_file_open_by_path(device_path, flags, NULL, + NULL); + + if (IS_ERR(*bdev_file)) { + ret = PTR_ERR(*bdev_file); + btrfs_err(NULL, "failed to open device for path %s with flags 0x%x: %d", + device_path, flags, ret); + goto error; + } + bdev = file_bdev(*bdev_file); + + if (flush) + sync_blockdev(bdev); + if (holder) { + ret = set_blocksize(*bdev_file, BTRFS_BDEV_BLOCKSIZE); + if (ret) { + fs_bdev_file_release(*bdev_file, holder); + goto error; + } + } + invalidate_bdev(bdev); + *disk_super = btrfs_read_disk_super(bdev, 0, false); + if (IS_ERR(*disk_super)) { + ret = PTR_ERR(*disk_super); + if (holder) + fs_bdev_file_release(*bdev_file, holder); + else + bdev_fput(*bdev_file); + goto error; + } + + return 0; + +error: + *disk_super = NULL; + *bdev_file = NULL; + return ret; +} + +/* + * Search and remove all stale devices (which are not mounted). When both + * inputs are NULL, it will search and release all stale devices. + * + * @devt: Optional. When provided will it release all unmounted devices + * matching this devt only. + * @skip_device: Optional. Will skip this device when searching for the stale + * devices. + * + * Return: 0 for success or if @devt is 0. + * -EBUSY if @devt is a mounted device. + * -ENOENT if @devt does not match any device in the list. + */ +static int btrfs_free_stale_devices(dev_t devt, struct btrfs_device *skip_device) +{ + struct btrfs_fs_devices *fs_devices, *tmp_fs_devices; + struct btrfs_device *device, *tmp_device; + int ret; + bool freed = false; + + lockdep_assert_held(&uuid_mutex); + + /* Return good status if there is no instance of devt. */ + ret = 0; + list_for_each_entry_safe(fs_devices, tmp_fs_devices, &fs_uuids, fs_list) { + + mutex_lock(&fs_devices->device_list_mutex); + list_for_each_entry_safe(device, tmp_device, + &fs_devices->devices, dev_list) { + if (skip_device && skip_device == device) + continue; + if (devt && devt != device->devt) + continue; + if (fs_devices->opened || fs_devices->holding) { + if (devt) + ret = -EBUSY; + break; + } + + /* delete the stale device */ + fs_devices->num_devices--; + list_del(&device->dev_list); + btrfs_free_device(device); + + freed = true; + } + mutex_unlock(&fs_devices->device_list_mutex); + + if (fs_devices->num_devices == 0) { + btrfs_sysfs_remove_fsid(fs_devices); + list_del(&fs_devices->fs_list); + free_fs_devices(fs_devices); + } + } + + /* If there is at least one freed device return 0. */ + if (freed) + return 0; + + return ret; +} + +static struct btrfs_fs_devices *find_fsid_by_device( + struct btrfs_super_block *disk_super, + dev_t devt, bool *same_fsid_diff_dev) +{ + struct btrfs_fs_devices *fsid_fs_devices; + struct btrfs_fs_devices *devt_fs_devices; + const bool has_metadata_uuid = (btrfs_super_incompat_flags(disk_super) & + BTRFS_FEATURE_INCOMPAT_METADATA_UUID); + bool found_by_devt = false; + + /* Find the fs_device by the usual method, if found use it. */ + fsid_fs_devices = find_fsid(disk_super->fsid, + has_metadata_uuid ? disk_super->metadata_uuid : NULL); + + /* The temp_fsid feature is supported only with single device filesystem. */ + if (btrfs_super_num_devices(disk_super) != 1) + return fsid_fs_devices; + + /* + * A seed device is an integral component of the sprout device, which + * functions as a multi-device filesystem. So, temp-fsid feature is + * not supported. + */ + if (btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_SEEDING) + return fsid_fs_devices; + + /* Try to find a fs_devices by matching devt. */ + list_for_each_entry(devt_fs_devices, &fs_uuids, fs_list) { + struct btrfs_device *device; + + list_for_each_entry(device, &devt_fs_devices->devices, dev_list) { + if (device->devt == devt) { + found_by_devt = true; + break; + } + } + if (found_by_devt) + break; + } + + if (found_by_devt) { + /* Existing device. */ + if (fsid_fs_devices == NULL) { + if (devt_fs_devices->opened == 0) { + /* Stale device. */ + return NULL; + } else { + /* temp_fsid is mounting a subvol. */ + return devt_fs_devices; + } + } else { + /* Regular or temp_fsid device mounting a subvol. */ + return devt_fs_devices; + } + } else { + /* New device. */ + if (fsid_fs_devices == NULL) { + return NULL; + } else { + /* sb::fsid is already used create a new temp_fsid. */ + *same_fsid_diff_dev = true; + return NULL; + } + } + + /* Not reached. */ +} + +/* + * This is only used on mount, and we are protected from competing things + * messing with our fs_devices by the uuid_mutex, thus we do not need the + * fs_devices->device_list_mutex here. + */ +static int btrfs_open_one_device(struct btrfs_fs_devices *fs_devices, + struct btrfs_device *device, blk_mode_t flags, + void *holder) +{ + struct file *bdev_file; + struct btrfs_super_block *disk_super; + u64 devid; + int ret; + + if (device->bdev) + return -EINVAL; + if (!device->name) + return -EINVAL; + + ret = btrfs_get_bdev_and_sb(rcu_dereference_raw(device->name), flags, holder, 1, + &bdev_file, &disk_super); + if (ret) + return ret; + + devid = btrfs_stack_device_id(&disk_super->dev_item); + if (devid != device->devid) + goto error_free_page; + + if (memcmp(device->uuid, disk_super->dev_item.uuid, BTRFS_UUID_SIZE)) + goto error_free_page; + + device->generation = btrfs_super_generation(disk_super); + + if (btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_SEEDING) { + if (btrfs_super_incompat_flags(disk_super) & + BTRFS_FEATURE_INCOMPAT_METADATA_UUID) { + btrfs_err(NULL, + "invalid seeding and uuid-changed device detected"); + goto error_free_page; + } + + clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state); + fs_devices->seeding = true; + } else { + if (bdev_read_only(file_bdev(bdev_file))) + clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state); + else + set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state); + } + + if (bdev_rot(file_bdev(bdev_file))) + fs_devices->rotating = true; + + if (bdev_max_discard_sectors(file_bdev(bdev_file))) + fs_devices->discardable = true; + + device->bdev_file = bdev_file; + device->bdev = file_bdev(bdev_file); + clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state); + + if (device->devt != device->bdev->bd_dev) { + btrfs_warn(NULL, + "device %s maj:min changed from %d:%d to %d:%d", + rcu_dereference_raw(device->name), MAJOR(device->devt), + MINOR(device->devt), MAJOR(device->bdev->bd_dev), + MINOR(device->bdev->bd_dev)); + + device->devt = device->bdev->bd_dev; + } + + fs_devices->open_devices++; + if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) && + device->devid != BTRFS_DEV_REPLACE_DEVID) { + fs_devices->rw_devices++; + list_add_tail(&device->dev_alloc_list, &fs_devices->alloc_list); + } + btrfs_release_disk_super(disk_super); + + return 0; + +error_free_page: + btrfs_release_disk_super(disk_super); + fs_bdev_file_release(bdev_file, holder); + + return -EINVAL; +} + +const u8 *btrfs_sb_fsid_ptr(const struct btrfs_super_block *sb) +{ + bool has_metadata_uuid = (btrfs_super_incompat_flags(sb) & + BTRFS_FEATURE_INCOMPAT_METADATA_UUID); + + return has_metadata_uuid ? sb->metadata_uuid : sb->fsid; +} + +/* + * Add new device to list of registered devices + * + * Returns: + * device pointer which was just added or updated when successful + * error pointer when failed + */ +static noinline struct btrfs_device *device_list_add(const char *path, + struct btrfs_super_block *disk_super, + bool *new_device_added) +{ + struct btrfs_device *device; + struct btrfs_fs_devices *fs_devices = NULL; + const char *name; + u64 found_transid = btrfs_super_generation(disk_super); + u64 devid = btrfs_stack_device_id(&disk_super->dev_item); + dev_t path_devt; + int ret; + bool same_fsid_diff_dev = false; + bool has_metadata_uuid = (btrfs_super_incompat_flags(disk_super) & + BTRFS_FEATURE_INCOMPAT_METADATA_UUID); + + if (btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_CHANGING_FSID_V2) { + btrfs_err(NULL, +"device %s has incomplete metadata_uuid change, please use btrfstune to complete", + path); + return ERR_PTR(-EAGAIN); + } + + ret = lookup_bdev(path, &path_devt); + if (ret) { + btrfs_err(NULL, "failed to lookup block device for path %s: %d", + path, ret); + return ERR_PTR(ret); + } + + fs_devices = find_fsid_by_device(disk_super, path_devt, &same_fsid_diff_dev); + + if (!fs_devices) { + fs_devices = alloc_fs_devices(disk_super->fsid); + if (IS_ERR(fs_devices)) + return ERR_CAST(fs_devices); + + if (has_metadata_uuid) + memcpy(fs_devices->metadata_uuid, + disk_super->metadata_uuid, BTRFS_FSID_SIZE); + + if (same_fsid_diff_dev) { + generate_random_uuid(fs_devices->fsid); + fs_devices->temp_fsid = true; + btrfs_info(NULL, "device %s (%d:%d) using temp-fsid %pU", + path, MAJOR(path_devt), MINOR(path_devt), + fs_devices->fsid); + } + + mutex_lock(&fs_devices->device_list_mutex); + list_add(&fs_devices->fs_list, &fs_uuids); + + device = NULL; + } else { + struct btrfs_dev_lookup_args args = { + .devid = devid, + .uuid = disk_super->dev_item.uuid, + }; + + mutex_lock(&fs_devices->device_list_mutex); + device = btrfs_find_device(fs_devices, &args); + + if (found_transid > fs_devices->latest_generation) { + memcpy(fs_devices->fsid, disk_super->fsid, + BTRFS_FSID_SIZE); + memcpy(fs_devices->metadata_uuid, + btrfs_sb_fsid_ptr(disk_super), BTRFS_FSID_SIZE); + } + } + + if (!device) { + unsigned int nofs_flag; + + if (fs_devices->opened) { + btrfs_err(NULL, +"device %s (%d:%d) belongs to fsid %pU, and the fs is already mounted, scanned by %s (%d)", + path, MAJOR(path_devt), MINOR(path_devt), + fs_devices->fsid, current->comm, + task_pid_nr(current)); + mutex_unlock(&fs_devices->device_list_mutex); + return ERR_PTR(-EBUSY); + } + + nofs_flag = memalloc_nofs_save(); + device = btrfs_alloc_device(NULL, &devid, + disk_super->dev_item.uuid, path); + memalloc_nofs_restore(nofs_flag); + if (IS_ERR(device)) { + mutex_unlock(&fs_devices->device_list_mutex); + /* we can safely leave the fs_devices entry around */ + return device; + } + + device->devt = path_devt; + + list_add_rcu(&device->dev_list, &fs_devices->devices); + fs_devices->num_devices++; + + device->fs_devices = fs_devices; + *new_device_added = true; + + if (disk_super->label[0]) + pr_info( +"BTRFS: device label %s devid %llu transid %llu %s (%d:%d) scanned by %s (%d)\n", + disk_super->label, devid, found_transid, path, + MAJOR(path_devt), MINOR(path_devt), + current->comm, task_pid_nr(current)); + else + pr_info( +"BTRFS: device fsid %pU devid %llu transid %llu %s (%d:%d) scanned by %s (%d)\n", + disk_super->fsid, devid, found_transid, path, + MAJOR(path_devt), MINOR(path_devt), + current->comm, task_pid_nr(current)); + + } else if (!device->name || device->devt != path_devt) { + const char *old_name; + + /* + * When FS is already mounted. + * 1. If you are here and if the device->name is NULL that + * means this device was missing at time of FS mount. + * 2. If you are here and if the device->name is different + * from 'path' that means either + * a. The same device disappeared and reappeared with + * different name. or + * b. The missing-disk-which-was-replaced, has + * reappeared now. + * + * We must allow 1 and 2a above. But 2b would be a spurious + * and unintentional. + * + * Further in case of 1 and 2a above, the disk at 'path' + * would have missed some transaction when it was away and + * in case of 2a the stale bdev has to be updated as well. + * 2b must not be allowed at all time. + */ + + /* + * For now, we do allow update to btrfs_fs_device through the + * btrfs dev scan cli after FS has been mounted. We're still + * tracking a problem where systems fail mount by subvolume id + * when we reject replacement on a mounted FS. + */ + if (!fs_devices->opened && found_transid < device->generation) { + /* + * That is if the FS is _not_ mounted and if you + * are here, that means there is more than one + * disk with same uuid and devid.We keep the one + * with larger generation number or the last-in if + * generation are equal. + */ + mutex_unlock(&fs_devices->device_list_mutex); + btrfs_err(NULL, +"device %s already registered with a higher generation, found %llu expect %llu", + path, found_transid, device->generation); + return ERR_PTR(-EEXIST); + } + + /* + * We are going to replace the device path for a given devid, + * make sure it's the same device if the device is mounted + * + * NOTE: the device->fs_info may not be reliable here so pass + * in a NULL to message helpers instead. This avoids a possible + * use-after-free when the fs_info and fs_info->sb are already + * torn down. + */ + if (device->bdev) { + if (device->devt != path_devt) { + mutex_unlock(&fs_devices->device_list_mutex); + btrfs_warn(NULL, + "duplicate device %s devid %llu generation %llu scanned by %s (%d)", + path, devid, found_transid, + current->comm, + task_pid_nr(current)); + return ERR_PTR(-EEXIST); + } + btrfs_info(NULL, + "devid %llu device path %s changed to %s scanned by %s (%d)", + devid, btrfs_dev_name(device), + path, current->comm, + task_pid_nr(current)); + } else { + btrfs_info(NULL, + "missing devid %llu re-appeared at %s scanned by %s (%d)", + devid, path, current->comm, + task_pid_nr(current)); + } + + name = kstrdup(path, GFP_NOFS); + if (!name) { + mutex_unlock(&fs_devices->device_list_mutex); + return ERR_PTR(-ENOMEM); + } + rcu_read_lock(); + old_name = rcu_dereference(device->name); + rcu_read_unlock(); + rcu_assign_pointer(device->name, name); + kfree_rcu_mightsleep(old_name); + + if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) { + fs_devices->missing_devices--; + clear_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state); + } + device->devt = path_devt; + } + + /* + * Unmount does not free the btrfs_device struct but would zero + * generation along with most of the other members. So just update + * it back. We need it to pick the disk with largest generation + * (as above). + */ + if (!fs_devices->opened) { + device->generation = found_transid; + fs_devices->latest_generation = max_t(u64, found_transid, + fs_devices->latest_generation); + } + + fs_devices->total_devices = btrfs_super_num_devices(disk_super); + + mutex_unlock(&fs_devices->device_list_mutex); + return device; +} + +static struct btrfs_fs_devices *clone_fs_devices(struct btrfs_fs_devices *orig) +{ + struct btrfs_fs_devices *fs_devices; + struct btrfs_device *device; + struct btrfs_device *orig_dev; + int ret = 0; + + lockdep_assert_held(&uuid_mutex); + + fs_devices = alloc_fs_devices(orig->fsid); + if (IS_ERR(fs_devices)) + return fs_devices; + + fs_devices->total_devices = orig->total_devices; + + list_for_each_entry(orig_dev, &orig->devices, dev_list) { + const char *dev_path = NULL; + + /* + * This is ok to do without RCU read locked because we hold the + * uuid mutex so nothing we touch in here is going to disappear. + */ + if (orig_dev->name) + dev_path = rcu_dereference_raw(orig_dev->name); + + device = btrfs_alloc_device(NULL, &orig_dev->devid, + orig_dev->uuid, dev_path); + if (IS_ERR(device)) { + ret = PTR_ERR(device); + goto error; + } + + if (orig_dev->zone_info) { + struct btrfs_zoned_device_info *zone_info; + + zone_info = btrfs_clone_dev_zone_info(orig_dev); + if (!zone_info) { + btrfs_free_device(device); + ret = -ENOMEM; + goto error; + } + device->zone_info = zone_info; + } + + list_add(&device->dev_list, &fs_devices->devices); + device->fs_devices = fs_devices; + fs_devices->num_devices++; + } + return fs_devices; +error: + free_fs_devices(fs_devices); + return ERR_PTR(ret); +} + +static void __btrfs_free_extra_devids(struct btrfs_fs_devices *fs_devices, + struct btrfs_device **latest_dev) +{ + struct btrfs_device *device, *next; + + /* This is the initialized path, it is safe to release the devices. */ + list_for_each_entry_safe(device, next, &fs_devices->devices, dev_list) { + if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state)) { + if (!test_bit(BTRFS_DEV_STATE_REPLACE_TGT, + &device->dev_state) && + !test_bit(BTRFS_DEV_STATE_MISSING, + &device->dev_state) && + (!*latest_dev || + device->generation > (*latest_dev)->generation)) { + *latest_dev = device; + } + continue; + } + + /* + * We have already validated the presence of BTRFS_DEV_REPLACE_DEVID, + * in btrfs_init_dev_replace() so just continue. + */ + if (device->devid == BTRFS_DEV_REPLACE_DEVID) + continue; + + if (device->bdev_file) { + fs_bdev_file_release(device->bdev_file, device->bdev_file->private_data); + device->bdev = NULL; + device->bdev_file = NULL; + fs_devices->open_devices--; + } + if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { + list_del_init(&device->dev_alloc_list); + clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state); + fs_devices->rw_devices--; + } + list_del_init(&device->dev_list); + fs_devices->num_devices--; + btrfs_free_device(device); + } + +} + +/* + * After we have read the system tree and know devids belonging to this + * filesystem, remove the device which does not belong there. + */ +void btrfs_free_extra_devids(struct btrfs_fs_devices *fs_devices) +{ + struct btrfs_device *latest_dev = NULL; + struct btrfs_fs_devices *seed_dev; + + mutex_lock(&uuid_mutex); + __btrfs_free_extra_devids(fs_devices, &latest_dev); + + list_for_each_entry(seed_dev, &fs_devices->seed_list, seed_list) + __btrfs_free_extra_devids(seed_dev, &latest_dev); + + fs_devices->latest_dev = latest_dev; + + mutex_unlock(&uuid_mutex); +} + +/* Release a device that was made unfreezable for a membership change. */ +void btrfs_release_device_allow_freeze(struct file *bdev_file) +{ + struct super_block *sb = bdev_file->private_data; + + /* Unregister before re-allowing (strand-safe); file still open (UAF-safe). */ + fs_bdev_unregister(bdev_file, sb); + bdev_allow_freeze(file_bdev(bdev_file)); + bdev_fput(bdev_file); +} + +static void btrfs_close_bdev(struct btrfs_device *device, bool allow_freeze) +{ + if (!device->bdev) + return; + + if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { + sync_blockdev(device->bdev); + invalidate_bdev(device->bdev); + } + + /* @allow_freeze undoes a replace-time deny; unmount-close was never denied. */ + if (allow_freeze) + btrfs_release_device_allow_freeze(device->bdev_file); + else + fs_bdev_file_release(device->bdev_file, + device->bdev_file->private_data); +} + +static void btrfs_close_one_device(struct btrfs_device *device) +{ + struct btrfs_fs_devices *fs_devices = device->fs_devices; + + if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) && + device->devid != BTRFS_DEV_REPLACE_DEVID) { + list_del_init(&device->dev_alloc_list); + fs_devices->rw_devices--; + } + + if (device->devid == BTRFS_DEV_REPLACE_DEVID) + clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state); + + if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) { + clear_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state); + fs_devices->missing_devices--; + } + + btrfs_close_bdev(device, false); + if (device->bdev) { + fs_devices->open_devices--; + device->bdev = NULL; + device->bdev_file = NULL; + } + clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state); + btrfs_destroy_dev_zone_info(device); + + device->fs_info = NULL; + atomic_set(&device->dev_stats_ccnt, 0); + btrfs_extent_io_tree_release(&device->alloc_state); + + /* + * Reset the flush error record. We might have a transient flush error + * in this mount, and if so we aborted the current transaction and set + * the fs to an error state, guaranteeing no super blocks can be further + * committed. However that error might be transient and if we unmount the + * filesystem and mount it again, we should allow the mount to succeed + * (btrfs_check_rw_degradable() should not fail) - if after mounting the + * filesystem again we still get flush errors, then we will again abort + * any transaction and set the error state, guaranteeing no commits of + * unsafe super blocks. + */ + clear_bit(BTRFS_DEV_STATE_FLUSH_FAILED, &device->dev_state); + + /* Verify the device is back in a pristine state */ + WARN_ON(test_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state)); + WARN_ON(test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)); + WARN_ON(!list_empty(&device->dev_alloc_list)); + WARN_ON(!list_empty(&device->post_commit_list)); +} + +static void close_fs_devices(struct btrfs_fs_devices *fs_devices) +{ + struct btrfs_device *device, *tmp; + + lockdep_assert_held(&uuid_mutex); + + if (--fs_devices->opened > 0) + return; + + list_for_each_entry_safe(device, tmp, &fs_devices->devices, dev_list) + btrfs_close_one_device(device); + + WARN_ON(fs_devices->open_devices); + WARN_ON(fs_devices->rw_devices); + fs_devices->opened = 0; + fs_devices->seeding = false; + fs_devices->fs_info = NULL; +} + +void btrfs_close_devices(struct btrfs_fs_devices *fs_devices) +{ + LIST_HEAD(list); + struct btrfs_fs_devices *tmp; + + mutex_lock(&uuid_mutex); + close_fs_devices(fs_devices); + if (!fs_devices->opened && !fs_devices->holding) { + list_splice_init(&fs_devices->seed_list, &list); + + /* + * If the struct btrfs_fs_devices is not assembled with any + * other device, it can be re-initialized during the next mount + * without the needing device-scan step. Therefore, it can be + * fully freed. + */ + if (fs_devices->num_devices == 1) { + list_del(&fs_devices->fs_list); + free_fs_devices(fs_devices); + } + } + + + list_for_each_entry_safe(fs_devices, tmp, &list, seed_list) { + close_fs_devices(fs_devices); + list_del(&fs_devices->seed_list); + free_fs_devices(fs_devices); + } + mutex_unlock(&uuid_mutex); +} + +static int open_fs_devices(struct btrfs_fs_devices *fs_devices, + blk_mode_t flags, void *holder) +{ + struct btrfs_device *device; + struct btrfs_device *latest_dev = NULL; + struct btrfs_device *tmp_device; + s64 __maybe_unused value = 0; + int ret = 0; + + list_for_each_entry_safe(device, tmp_device, &fs_devices->devices, + dev_list) { + int ret2; + + ret2 = btrfs_open_one_device(fs_devices, device, flags, holder); + if (ret2 == 0 && + (!latest_dev || device->generation > latest_dev->generation)) { + latest_dev = device; + } else if (ret2 == -ENODATA) { + fs_devices->num_devices--; + list_del(&device->dev_list); + btrfs_free_device(device); + } + if (ret == 0 && ret2 != 0) + ret = ret2; + } + + if (fs_devices->open_devices == 0) { + if (ret) + return ret; + return -EINVAL; + } + + fs_devices->opened = 1; + fs_devices->latest_dev = latest_dev; + fs_devices->total_rw_bytes = 0; + fs_devices->chunk_alloc_policy = BTRFS_CHUNK_ALLOC_REGULAR; +#ifdef CONFIG_BTRFS_EXPERIMENTAL + fs_devices->rr_min_contig_read = BTRFS_DEFAULT_RR_MIN_CONTIG_READ; + fs_devices->read_devid = latest_dev->devid; + fs_devices->read_policy = btrfs_read_policy_to_enum(btrfs_get_mod_read_policy(), + &value); + if (fs_devices->read_policy == BTRFS_READ_POLICY_RR) + fs_devices->collect_fs_stats = true; + + if (value) { + if (fs_devices->read_policy == BTRFS_READ_POLICY_RR) + fs_devices->rr_min_contig_read = value; + if (fs_devices->read_policy == BTRFS_READ_POLICY_DEVID) + fs_devices->read_devid = value; + } +#else + fs_devices->read_policy = BTRFS_READ_POLICY_PID; +#endif + + return 0; +} + +static int devid_cmp(void *priv, const struct list_head *a, + const struct list_head *b) +{ + const struct btrfs_device *dev1, *dev2; + + dev1 = list_entry(a, struct btrfs_device, dev_list); + dev2 = list_entry(b, struct btrfs_device, dev_list); + + if (dev1->devid < dev2->devid) + return -1; + else if (dev1->devid > dev2->devid) + return 1; + return 0; +} + +int btrfs_open_devices(struct btrfs_fs_devices *fs_devices, + blk_mode_t flags, void *holder) +{ + int ret; + + lockdep_assert_held(&uuid_mutex); + /* + * The device_list_mutex cannot be taken here in case opening the + * underlying device takes further locks like open_mutex. + * + * We also don't need the lock here as this is called during mount and + * exclusion is provided by uuid_mutex + */ + + if (fs_devices->opened) { + fs_devices->opened++; + ret = 0; + } else { + list_sort(NULL, &fs_devices->devices, devid_cmp); + ret = open_fs_devices(fs_devices, flags, holder); + } + + return ret; +} + +void btrfs_release_disk_super(struct btrfs_super_block *super) +{ + struct page *page = virt_to_page(super); + + put_page(page); +} + +struct btrfs_super_block *btrfs_read_disk_super(struct block_device *bdev, + int copy_num, bool drop_cache) +{ + struct btrfs_super_block *super; + struct page *page; + u64 bytenr, bytenr_orig; + struct address_space *mapping = bdev->bd_mapping; + int ret; + + bytenr_orig = btrfs_sb_offset(copy_num); + ret = btrfs_sb_log_location_bdev(bdev, copy_num, READ, &bytenr); + if (ret < 0) { + if (ret == -ENOENT) + ret = -EINVAL; + return ERR_PTR(ret); + } + + if (bytenr + BTRFS_SUPER_INFO_SIZE >= bdev_nr_bytes(bdev)) + return ERR_PTR(-EINVAL); + + if (drop_cache) { + /* This should only be called with the primary sb. */ + ASSERT(copy_num == 0); + + /* + * Drop the page of the primary superblock, so later read will + * always read from the device. + */ + invalidate_inode_pages2_range(mapping, bytenr >> PAGE_SHIFT, + (bytenr + BTRFS_SUPER_INFO_SIZE) >> PAGE_SHIFT); + } + + filemap_invalidate_lock_shared(mapping); + page = read_cache_page_gfp(mapping, bytenr >> PAGE_SHIFT, GFP_NOFS); + filemap_invalidate_unlock_shared(mapping); + if (IS_ERR(page)) + return ERR_CAST(page); + + super = page_address(page); + if (btrfs_super_magic(super) != BTRFS_MAGIC || + btrfs_super_bytenr(super) != bytenr_orig) { + btrfs_release_disk_super(super); + return ERR_PTR(-EINVAL); + } + + /* + * Make sure the last byte of label is properly NUL terminated. We use + * '%s' to print the label, if not properly NUL terminated we can access + * beyond the label. + */ + if (super->label[0] && super->label[BTRFS_LABEL_SIZE - 1]) + super->label[BTRFS_LABEL_SIZE - 1] = 0; + + return super; +} + +int btrfs_forget_devices(dev_t devt) +{ + int ret; + + mutex_lock(&uuid_mutex); + ret = btrfs_free_stale_devices(devt, NULL); + mutex_unlock(&uuid_mutex); + + return ret; +} + +static bool btrfs_skip_registration(struct btrfs_super_block *disk_super, + const char *path, dev_t devt, + bool mount_arg_dev) +{ + struct btrfs_fs_devices *fs_devices; + + /* + * Do not skip device registration for mounted devices with matching + * maj:min but different paths. Booting without initrd relies on + * /dev/root initially, later replaced with the actual root device. + * A successful scan ensures grub2-probe selects the correct device. + */ + list_for_each_entry(fs_devices, &fs_uuids, fs_list) { + struct btrfs_device *device; + + mutex_lock(&fs_devices->device_list_mutex); + + if (!fs_devices->opened) { + mutex_unlock(&fs_devices->device_list_mutex); + continue; + } + + list_for_each_entry(device, &fs_devices->devices, dev_list) { + if (device->bdev && (device->bdev->bd_dev == devt) && + strcmp(rcu_dereference_raw(device->name), path) != 0) { + mutex_unlock(&fs_devices->device_list_mutex); + + /* Do not skip registration. */ + return false; + } + } + mutex_unlock(&fs_devices->device_list_mutex); + } + + if (!mount_arg_dev && btrfs_super_num_devices(disk_super) == 1 && + !(btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_SEEDING)) + return true; + + return false; +} + +/* + * Look for a btrfs signature on a device. This may be called out of the mount path + * and we are not allowed to call set_blocksize during the scan. The superblock + * is read via pagecache. + * + * With @mount_arg_dev it's a scan during mount time that will always register + * the device or return an error. Multi-device and seeding devices are registered + * in both cases. + */ +struct btrfs_device *btrfs_scan_one_device(const char *path, + bool mount_arg_dev) +{ + struct btrfs_super_block *disk_super; + bool new_device_added = false; + struct btrfs_device *device = NULL; + struct file *bdev_file; + dev_t devt; + + lockdep_assert_held(&uuid_mutex); + + /* + * Avoid an exclusive open here, as the systemd-udev may initiate the + * device scan which may race with the user's mount or mkfs command, + * resulting in failure. + * Since the device scan is solely for reading purposes, there is no + * need for an exclusive open. Additionally, the devices are read again + * during the mount process. It is ok to get some inconsistent + * values temporarily, as the device paths of the fsid are the only + * required information for assembling the volume. + */ + bdev_file = bdev_file_open_by_path(path, BLK_OPEN_READ, NULL, NULL); + if (IS_ERR(bdev_file)) + return ERR_CAST(bdev_file); + + disk_super = btrfs_read_disk_super(file_bdev(bdev_file), 0, false); + if (IS_ERR(disk_super)) { + device = ERR_CAST(disk_super); + goto error_bdev_put; + } + + devt = file_bdev(bdev_file)->bd_dev; + if (btrfs_skip_registration(disk_super, path, devt, mount_arg_dev)) { + btrfs_debug(NULL, "skip registering single non-seed device %s (%d:%d)", + path, MAJOR(devt), MINOR(devt)); + + btrfs_free_stale_devices(devt, NULL); + + device = NULL; + goto free_disk_super; + } + + device = device_list_add(path, disk_super, &new_device_added); + if (!IS_ERR(device) && new_device_added) + btrfs_free_stale_devices(device->devt, device); + +free_disk_super: + btrfs_release_disk_super(disk_super); + +error_bdev_put: + bdev_fput(bdev_file); + + return device; +} + +/* + * Find the first pending extent intersecting a range. + * + * @device: the device to search + * @start: start of the range to check + * @len: length of the range to check + * @pending_start: output pointer for the start of the found pending extent + * @pending_end: output pointer for the end of the found pending extent (inclusive) + * + * Search for a pending chunk allocation that intersects the half-open range + * [start, start + len). + * + * Return: true if a pending extent was found, false otherwise. + * If the return value is true, store the first pending extent in + * [*pending_start, *pending_end]. Otherwise, the two output variables + * may still be modified, to something outside the range and should not + * be used. + */ +bool btrfs_first_pending_extent(struct btrfs_device *device, u64 start, u64 len, + u64 *pending_start, u64 *pending_end) +{ + lockdep_assert_held(&device->fs_info->chunk_mutex); + + if (btrfs_find_first_extent_bit(&device->alloc_state, start, + pending_start, pending_end, + CHUNK_ALLOCATED, NULL)) { + + if (in_range(*pending_start, start, len) || + in_range(start, *pending_start, *pending_end + 1 - *pending_start)) { + return true; + } + } + return false; +} + +/* + * Find the first real hole accounting for pending extents. + * + * @device: the device containing the candidate hole + * @start: input/output pointer for the hole start position + * @len: input/output pointer for the hole length + * @min_hole_size: the size of hole we are looking for + * + * Given a potential hole specified by [*start, *start + *len), check for pending + * chunk allocations within that range. If pending extents are found, the hole is + * adjusted to represent the first true free space that is large enough when + * accounting for pending chunks. + * + * Note that this function must handle various cases involving non consecutive + * pending extents. + * + * Returns: true if a suitable hole was found and false otherwise. + * If the return value is true, then *start and *len are set to represent the hole. + * If the return value is false, then *start is set to the largest hole we + * found and *len is set to its length. + * If there are no holes at all, then *start is set to the end of the range and + * *len is set to 0. + */ +bool btrfs_find_hole_in_pending_extents(struct btrfs_device *device, u64 *start, + u64 *len, u64 min_hole_size) +{ + u64 pending_start, pending_end; + u64 end; + u64 max_hole_start = 0; + u64 max_hole_len = 0; + + lockdep_assert_held(&device->fs_info->chunk_mutex); + + if (*len == 0) + return false; + + end = *start + *len - 1; + + /* + * Loop until we either see a large enough hole or check every pending + * extent overlapping the candidate hole. + * At every hole that we observe, record it if it is the new max. + * At the end of the iteration, set the output variables to the max hole. + */ + while (true) { + if (btrfs_first_pending_extent(device, *start, *len, &pending_start, &pending_end)) { + /* + * Case 1: the pending extent overlaps the start of + * candidate hole. That means the true hole is after the + * pending extent, but we need to find the next pending + * extent to properly size the hole. In the next loop, + * we will reduce to case 2 or 3. + * e.g., + * + * |----pending A----| real hole |----pending B----| + * | candidate hole | + * *start end + */ + if (pending_start <= *start) { + *start = pending_end + 1; + goto next; + } + /* + * Case 2: The pending extent starts after *start (and overlaps + * [*start, end), so the first hole just goes up to the start + * of the pending extent. + * e.g., + * + * | real hole |----pending A----| + * | candidate hole | + * *start end + */ + *len = pending_start - *start; + if (*len > max_hole_len) { + max_hole_start = *start; + max_hole_len = *len; + } + if (*len >= min_hole_size) + break; + /* + * If the hole wasn't big enough, then we advance past + * the pending extent and keep looking. + */ + *start = pending_end + 1; + goto next; + } else { + /* + * Case 3: There is no pending extent overlapping the + * range [*start, *start + *len - 1], so the only remaining + * hole is the remaining range. + * e.g., + * + * | candidate hole | + * | real hole | + * *start end + */ + + if (*len > max_hole_len) { + max_hole_start = *start; + max_hole_len = *len; + } + break; + } +next: + if (*start > end) + break; + *len = end - *start + 1; + } + if (max_hole_len) { + *start = max_hole_start; + *len = max_hole_len; + } else { + *start = end + 1; + *len = 0; + } + return max_hole_len >= min_hole_size; +} + +static u64 dev_extent_search_start(struct btrfs_device *device) +{ + switch (device->fs_devices->chunk_alloc_policy) { + default: + btrfs_warn_unknown_chunk_allocation(device->fs_devices->chunk_alloc_policy); + fallthrough; + case BTRFS_CHUNK_ALLOC_REGULAR: + return BTRFS_DEVICE_RANGE_RESERVED; + case BTRFS_CHUNK_ALLOC_ZONED: + /* + * We don't care about the starting region like regular + * allocator, because we anyway use/reserve the first two zones + * for superblock logging. + */ + return 0; + } +} + +static bool dev_extent_hole_check_zoned(struct btrfs_device *device, + u64 *hole_start, u64 *hole_size, + u64 num_bytes) +{ + u64 zone_size = device->zone_info->zone_size; + u64 pos; + int ret; + bool changed = false; + + ASSERT(IS_ALIGNED(*hole_start, zone_size), + "hole_start=%llu zone_size=%llu", *hole_start, zone_size); + + while (*hole_size > 0) { + pos = btrfs_find_allocatable_zones(device, *hole_start, + *hole_start + *hole_size, + num_bytes); + if (pos != *hole_start) { + *hole_size = *hole_start + *hole_size - pos; + *hole_start = pos; + changed = true; + if (*hole_size < num_bytes) + break; + } + + ret = btrfs_ensure_empty_zones(device, pos, num_bytes); + + /* Range is ensured to be empty */ + if (!ret) + return changed; + + /* Given hole range was invalid (outside of device) */ + if (ret == -ERANGE) { + *hole_start += *hole_size; + *hole_size = 0; + return true; + } + + *hole_start += zone_size; + *hole_size -= zone_size; + changed = true; + } + + return changed; +} + +/* + * Validate and adjust a hole for chunk allocation + * + * @device: the device containing the candidate hole + * @hole_start: input/output pointer for the hole start position + * @hole_size: input/output pointer for the hole size + * @num_bytes: minimum allocation size required + * + * Check if the specified hole is suitable for allocation and adjust it if + * necessary. The hole may be modified to skip over pending chunk allocations + * and to satisfy stricter zoned requirements on zoned filesystems. + * + * For regular (non-zoned) allocation, if the hole after adjustment is smaller + * than @num_bytes, the search continues past additional pending extents until + * either a sufficiently large hole is found or no more pending extents exist. + * + * Return: true if a suitable hole was found and false otherwise. + * If the return value is true, then *hole_start and *hole_size are set to + * represent the hole we found. + * If the return value is false, then *hole_start is set to the largest + * hole we found and *hole_size is set to its length. + * If there are no holes at all, then *hole_start is set to the end of the range + * and *hole_size is set to 0. + */ +static bool dev_extent_hole_check(struct btrfs_device *device, u64 *hole_start, + u64 *hole_size, u64 num_bytes) +{ + bool found = false; + const u64 hole_end = *hole_start + *hole_size - 1; + + ASSERT(*hole_size > 0); + +again: + *hole_size = hole_end - *hole_start + 1; + found = btrfs_find_hole_in_pending_extents(device, hole_start, hole_size, num_bytes); + if (!found) + return found; + ASSERT(*hole_size >= num_bytes); + + switch (device->fs_devices->chunk_alloc_policy) { + default: + btrfs_warn_unknown_chunk_allocation(device->fs_devices->chunk_alloc_policy); + fallthrough; + case BTRFS_CHUNK_ALLOC_REGULAR: + return found; + case BTRFS_CHUNK_ALLOC_ZONED: + if (dev_extent_hole_check_zoned(device, hole_start, hole_size, num_bytes)) + goto again; + break; + } + + return found; +} + +/* + * Find free space in the specified device. + * + * @device: the device which we search the free space in + * @num_bytes: the size of the free space that we need + * @search_start: the position from which to begin the search + * @start: store the start of the free space. + * @len: the size of the free space. that we find, or the size + * of the max free space if we don't find suitable free space + * + * This does a pretty simple search, the expectation is that it is called very + * infrequently and that a given device has a small number of extents. + * + * @start is used to store the start of the free space if we find. But if we + * don't find suitable free space, it will be used to store the start position + * of the max free space. + * + * @len is used to store the size of the free space that we find. + * But if we don't find suitable free space, it is used to store the size of + * the max free space. + * + * NOTE: This function will search *commit* root of device tree, and does extra + * check to ensure dev extents are not double allocated. + * This makes the function safe to allocate dev extents but may not report + * correct usable device space, as device extent freed in current transaction + * is not reported as available. + */ +static int find_free_dev_extent(struct btrfs_device *device, u64 num_bytes, + u64 *start, u64 *len) +{ + struct btrfs_fs_info *fs_info = device->fs_info; + struct btrfs_root *root = fs_info->dev_root; + struct btrfs_key key; + struct btrfs_dev_extent *dev_extent; + BTRFS_PATH_AUTO_FREE(path); + u64 search_start; + u64 hole_size; + u64 max_hole_start; + u64 max_hole_size = 0; + u64 extent_end; + u64 search_end = device->total_bytes; + int ret; + int slot; + struct extent_buffer *l; + + search_start = dev_extent_search_start(device); + max_hole_start = search_start; + + WARN_ON(device->zone_info && + !IS_ALIGNED(num_bytes, device->zone_info->zone_size)); + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto out; + } + + if (search_start >= search_end || + test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) { + ret = -ENOSPC; + goto out; + } + + path->reada = READA_FORWARD; + path->search_commit_root = true; + path->skip_locking = true; + + key.objectid = device->devid; + key.type = BTRFS_DEV_EXTENT_KEY; + key.offset = search_start; + + ret = btrfs_search_backwards(root, &key, path); + if (ret < 0) + goto out; + + while (search_start < search_end) { + l = path->nodes[0]; + slot = path->slots[0]; + if (slot >= btrfs_header_nritems(l)) { + ret = btrfs_next_leaf(root, path); + if (ret == 0) + continue; + if (ret < 0) + goto out; + + break; + } + btrfs_item_key_to_cpu(l, &key, slot); + + if (key.objectid < device->devid) + goto next; + + if (key.objectid > device->devid) + break; + + if (key.type != BTRFS_DEV_EXTENT_KEY) + goto next; + + if (key.offset > search_end) + break; + + if (key.offset > search_start) { + hole_size = key.offset - search_start; + dev_extent_hole_check(device, &search_start, &hole_size, + num_bytes); + + if (hole_size > max_hole_size) { + max_hole_start = search_start; + max_hole_size = hole_size; + } + + /* + * If this free space is greater than which we need, + * it must be the max free space that we have found + * until now, so max_hole_start must point to the start + * of this free space and the length of this free space + * is stored in max_hole_size. Thus, we return + * max_hole_start and max_hole_size and go back to the + * caller. + */ + if (hole_size >= num_bytes) { + ret = 0; + goto out; + } + } + + dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent); + extent_end = key.offset + btrfs_dev_extent_length(l, + dev_extent); + if (extent_end > search_start) + search_start = extent_end; +next: + path->slots[0]++; + cond_resched(); + } + + /* + * At this point, search_start should be the end of + * allocated dev extents, and when shrinking the device, + * search_end may be smaller than search_start. + */ + if (search_end > search_start) { + hole_size = search_end - search_start; + dev_extent_hole_check(device, &search_start, &hole_size, num_bytes); + + if (hole_size > max_hole_size) { + max_hole_start = search_start; + max_hole_size = hole_size; + } + } + + /* See above. */ + if (max_hole_size < num_bytes) + ret = -ENOSPC; + else + ret = 0; + + ASSERT(max_hole_start + max_hole_size <= search_end, + "max_hole_start=%llu max_hole_size=%llu search_end=%llu", + max_hole_start, max_hole_size, search_end); +out: + *start = max_hole_start; + if (len) + *len = max_hole_size; + return ret; +} + +static int btrfs_free_dev_extent(struct btrfs_trans_handle *trans, + struct btrfs_device *device, + u64 start, u64 *dev_extent_len) +{ + struct btrfs_fs_info *fs_info = device->fs_info; + struct btrfs_root *root = fs_info->dev_root; + int ret; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + struct btrfs_key found_key; + struct extent_buffer *leaf = NULL; + struct btrfs_dev_extent *extent = NULL; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = device->devid; + key.type = BTRFS_DEV_EXTENT_KEY; + key.offset = start; +again: + ret = btrfs_search_slot(trans, root, &key, path, -1, 1); + if (ret > 0) { + ret = btrfs_previous_item(root, path, key.objectid, + BTRFS_DEV_EXTENT_KEY); + if (ret) + return ret; + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); + extent = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_dev_extent); + BUG_ON(found_key.offset > start || found_key.offset + + btrfs_dev_extent_length(leaf, extent) < start); + key = found_key; + btrfs_release_path(path); + goto again; + } else if (ret == 0) { + leaf = path->nodes[0]; + extent = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_dev_extent); + } else { + return ret; + } + + *dev_extent_len = btrfs_dev_extent_length(leaf, extent); + + ret = btrfs_del_item(trans, root, path); + if (ret == 0) + set_bit(BTRFS_TRANS_HAVE_FREE_BGS, &trans->transaction->flags); + return ret; +} + +static u64 find_next_chunk(struct btrfs_fs_info *fs_info) +{ + struct rb_node *n; + u64 ret = 0; + + read_lock(&fs_info->mapping_tree_lock); + n = rb_last(&fs_info->mapping_tree.rb_root); + if (n) { + struct btrfs_chunk_map *map; + + map = rb_entry(n, struct btrfs_chunk_map, rb_node); + ret = map->start + map->chunk_len; + } + read_unlock(&fs_info->mapping_tree_lock); + + return ret; +} + +static noinline int find_next_devid(struct btrfs_fs_info *fs_info, + u64 *devid_ret) +{ + int ret; + struct btrfs_key key; + struct btrfs_key found_key; + BTRFS_PATH_AUTO_FREE(path); + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = BTRFS_DEV_ITEMS_OBJECTID; + key.type = BTRFS_DEV_ITEM_KEY; + key.offset = (u64)-1; + + ret = btrfs_search_slot(NULL, fs_info->chunk_root, &key, path, 0, 0); + if (ret < 0) + return ret; + + if (unlikely(ret == 0)) { + /* Corruption */ + btrfs_err(fs_info, "corrupted chunk tree devid -1 matched"); + return -EUCLEAN; + } + + ret = btrfs_previous_item(fs_info->chunk_root, path, + BTRFS_DEV_ITEMS_OBJECTID, + BTRFS_DEV_ITEM_KEY); + if (ret) { + *devid_ret = 1; + } else { + btrfs_item_key_to_cpu(path->nodes[0], &found_key, + path->slots[0]); + *devid_ret = found_key.offset + 1; + } + return 0; +} + +/* + * the device information is stored in the chunk root + * the btrfs_device struct should be fully filled in + */ +static int btrfs_add_dev_item(struct btrfs_trans_handle *trans, + struct btrfs_device *device) +{ + int ret; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_dev_item *dev_item; + struct extent_buffer *leaf; + struct btrfs_key key; + unsigned long ptr; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = BTRFS_DEV_ITEMS_OBJECTID; + key.type = BTRFS_DEV_ITEM_KEY; + key.offset = device->devid; + + btrfs_reserve_chunk_metadata(trans, true); + ret = btrfs_insert_empty_item(trans, trans->fs_info->chunk_root, path, + &key, sizeof(*dev_item)); + btrfs_trans_release_chunk_metadata(trans); + if (ret) + return ret; + + leaf = path->nodes[0]; + dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item); + + btrfs_set_device_id(leaf, dev_item, device->devid); + btrfs_set_device_generation(leaf, dev_item, 0); + btrfs_set_device_type(leaf, dev_item, device->type); + btrfs_set_device_io_align(leaf, dev_item, device->io_align); + btrfs_set_device_io_width(leaf, dev_item, device->io_width); + btrfs_set_device_sector_size(leaf, dev_item, device->sector_size); + btrfs_set_device_total_bytes(leaf, dev_item, + btrfs_device_get_disk_total_bytes(device)); + btrfs_set_device_bytes_used(leaf, dev_item, + btrfs_device_get_bytes_used(device)); + btrfs_set_device_group(leaf, dev_item, 0); + btrfs_set_device_seek_speed(leaf, dev_item, 0); + btrfs_set_device_bandwidth(leaf, dev_item, 0); + btrfs_set_device_start_offset(leaf, dev_item, 0); + + ptr = btrfs_device_uuid(dev_item); + write_extent_buffer(leaf, device->uuid, ptr, BTRFS_UUID_SIZE); + ptr = btrfs_device_fsid(dev_item); + write_extent_buffer(leaf, trans->fs_info->fs_devices->metadata_uuid, + ptr, BTRFS_FSID_SIZE); + + return 0; +} + +/* + * Function to update ctime/mtime for a given device path. + * Mainly used for ctime/mtime based probe like libblkid. + * + * We don't care about errors here, this is just to be kind to userspace. + */ +static void update_dev_time(const char *device_path) +{ + struct path path; + int err; + + if (tsk_is_kthread(current)) { + scoped_with_init_fs() + err = kern_path(device_path, LOOKUP_FOLLOW, &path); + } else { + err = kern_path(device_path, LOOKUP_FOLLOW, &path); + } + + if (!err) { + vfs_utimes(&path, NULL); + path_put(&path); + } +} + +static int btrfs_rm_dev_item(struct btrfs_trans_handle *trans, + struct btrfs_device *device) +{ + struct btrfs_root *root = device->fs_info->chunk_root; + int ret; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = BTRFS_DEV_ITEMS_OBJECTID; + key.type = BTRFS_DEV_ITEM_KEY; + key.offset = device->devid; + + btrfs_reserve_chunk_metadata(trans, false); + ret = btrfs_search_slot(trans, root, &key, path, -1, 1); + btrfs_trans_release_chunk_metadata(trans); + if (ret > 0) + return -ENOENT; + if (ret < 0) + return ret; + + return btrfs_del_item(trans, root, path); +} + +/* + * Verify that @num_devices satisfies the RAID profile constraints in the whole + * filesystem. It's up to the caller to adjust that number regarding eg. device + * replace. + */ +static int btrfs_check_raid_min_devices(struct btrfs_fs_info *fs_info, + u64 num_devices) +{ + u64 all_avail; + unsigned seq; + int i; + + do { + seq = read_seqbegin(&fs_info->profiles_lock); + + all_avail = fs_info->avail_data_alloc_bits | + fs_info->avail_system_alloc_bits | + fs_info->avail_metadata_alloc_bits; + } while (read_seqretry(&fs_info->profiles_lock, seq)); + + for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) { + if (!(all_avail & btrfs_raid_array[i].bg_flag)) + continue; + + if (num_devices < btrfs_raid_array[i].devs_min) + return btrfs_raid_array[i].mindev_error; + } + + return 0; +} + +static struct btrfs_device * btrfs_find_next_active_device( + struct btrfs_fs_devices *fs_devs, struct btrfs_device *device) +{ + struct btrfs_device *next_device; + + list_for_each_entry(next_device, &fs_devs->devices, dev_list) { + if (next_device != device && + !test_bit(BTRFS_DEV_STATE_MISSING, &next_device->dev_state) + && next_device->bdev) + return next_device; + } + + return NULL; +} + +/* + * Helper function to check if the given device is part of s_bdev / latest_dev + * and replace it with the provided or the next active device, in the context + * where this function called, there should be always be another device (or + * this_dev) which is active. + */ +void __cold btrfs_assign_next_active_device(struct btrfs_device *device, + struct btrfs_device *next_device) +{ + struct btrfs_fs_info *fs_info = device->fs_info; + + if (!next_device) + next_device = btrfs_find_next_active_device(fs_info->fs_devices, + device); + ASSERT(next_device); + + if (fs_info->sb->s_bdev && + (fs_info->sb->s_bdev == device->bdev)) + fs_info->sb->s_bdev = next_device->bdev; + + if (fs_info->fs_devices->latest_dev->bdev == device->bdev) + fs_info->fs_devices->latest_dev = next_device; +} + +/* + * Return btrfs_fs_devices::num_devices excluding the device that's being + * currently replaced. + */ +static u64 btrfs_num_devices(struct btrfs_fs_info *fs_info) +{ + u64 num_devices = fs_info->fs_devices->num_devices; + + down_read(&fs_info->dev_replace.rwsem); + if (btrfs_dev_replace_is_ongoing(&fs_info->dev_replace)) { + ASSERT(num_devices > 1, "num_devices=%llu", num_devices); + num_devices--; + } + up_read(&fs_info->dev_replace.rwsem); + + return num_devices; +} + +static void btrfs_scratch_superblock(struct btrfs_fs_info *fs_info, + struct block_device *bdev, int copy_num) +{ + struct btrfs_super_block *disk_super; + const size_t len = sizeof(disk_super->magic); + const u64 bytenr = btrfs_sb_offset(copy_num); + int ret; + + disk_super = btrfs_read_disk_super(bdev, copy_num, false); + if (IS_ERR(disk_super)) + return; + + memset(&disk_super->magic, 0, len); + folio_mark_dirty(virt_to_folio(disk_super)); + btrfs_release_disk_super(disk_super); + + ret = sync_blockdev_range(bdev, bytenr, bytenr + len - 1); + if (ret) + btrfs_warn(fs_info, "error clearing superblock number %d (%d)", + copy_num, ret); +} + +void btrfs_scratch_superblocks(struct btrfs_fs_info *fs_info, struct btrfs_device *device) +{ + int copy_num; + struct block_device *bdev = device->bdev; + + if (!bdev) + return; + + for (copy_num = 0; copy_num < BTRFS_SUPER_MIRROR_MAX; copy_num++) { + if (bdev_is_zoned(bdev)) + btrfs_reset_sb_log_zones(bdev, copy_num); + else + btrfs_scratch_superblock(fs_info, bdev, copy_num); + } + + /* Notify udev that device has changed */ + btrfs_kobject_uevent(bdev, KOBJ_CHANGE); + + /* Update ctime/mtime for device path for libblkid */ + update_dev_time(rcu_dereference_raw(device->name)); +} + +int btrfs_remove_dev_stat_item(struct btrfs_trans_handle *trans, u64 devid) +{ + BTRFS_PATH_AUTO_RELEASE(path); + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *dev_root = fs_info->dev_root; + struct btrfs_key key; + int ret; + + key.objectid = BTRFS_DEV_STATS_OBJECTID; + key.type = BTRFS_PERSISTENT_ITEM_KEY; + key.offset = devid; + + ret = btrfs_search_slot(trans, dev_root, &key, &path, -1, 1); + if (ret < 0) { + btrfs_warn(fs_info, + "error %d while searching for dev_stats item for devid %llu", + ret, devid); + return ret; + } + /* The dev stats item does not exist, nothing to bother. */ + if (ret > 0) + return 0; + ret = btrfs_del_item(trans, dev_root, &path); + if (ret < 0) { + btrfs_warn(fs_info, + "error %d while deleting dev_stats item for devid %llu", + ret, devid); + return ret; + } + return 0; +} + +int btrfs_rm_device(struct btrfs_fs_info *fs_info, + struct btrfs_dev_lookup_args *args, + struct file **bdev_file) +{ + struct btrfs_trans_handle *trans; + struct btrfs_device *device; + struct btrfs_fs_devices *cur_devices; + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; + u64 num_devices; + int ret = 0; + + if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) { + btrfs_err(fs_info, "device remove not supported on extent tree v2 yet"); + return -EINVAL; + } + + /* + * The device list in fs_devices is accessed without locks (neither + * uuid_mutex nor device_list_mutex) as it won't change on a mounted + * filesystem and another device rm cannot run. + */ + num_devices = btrfs_num_devices(fs_info); + + ret = btrfs_check_raid_min_devices(fs_info, num_devices - 1); + if (ret) + return ret; + + device = btrfs_find_device(fs_info->fs_devices, args); + if (!device) { + if (args->missing) + ret = BTRFS_ERROR_DEV_MISSING_NOT_FOUND; + else + ret = -ENOENT; + return ret; + } + + if (btrfs_pinned_by_swapfile(fs_info, device)) { + btrfs_warn(fs_info, + "cannot remove device %s (devid %llu) due to active swapfile", + btrfs_dev_name(device), device->devid); + return -ETXTBSY; + } + + if (test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) + return BTRFS_ERROR_DEV_TGT_REPLACE; + + if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) && + fs_info->fs_devices->rw_devices == 1) + return BTRFS_ERROR_DEV_ONLY_WRITABLE; + + /* Removal and freezing are mutually exclusive; refuse if frozen now. */ + if (device->bdev) { + ret = bdev_deny_freeze(device->bdev); + if (ret) + return ret; + } + + if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { + mutex_lock(&fs_info->chunk_mutex); + list_del_init(&device->dev_alloc_list); + device->fs_devices->rw_devices--; + btrfs_update_per_profile_avail(fs_info); + mutex_unlock(&fs_info->chunk_mutex); + } + + ret = btrfs_shrink_device(device, 0); + if (ret) + goto error_undo; + + trans = btrfs_start_transaction(fs_info->chunk_root, 0); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto error_undo; + } + + ret = btrfs_rm_dev_item(trans, device); + if (unlikely(ret)) { + /* Any error in dev item removal is critical */ + btrfs_crit(fs_info, + "failed to remove device item for devid %llu: %d", + device->devid, ret); + btrfs_abort_transaction(trans, ret); + btrfs_end_transaction(trans); + if (device->bdev) + bdev_allow_freeze(device->bdev); + return ret; + } + + ret = btrfs_remove_dev_stat_item(trans, device->devid); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + btrfs_end_transaction(trans); + return ret; + } + clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state); + btrfs_scrub_cancel_dev(device); + + /* + * the device list mutex makes sure that we don't change + * the device list while someone else is writing out all + * the device supers. Whoever is writing all supers, should + * lock the device list mutex before getting the number of + * devices in the super block (super_copy). Conversely, + * whoever updates the number of devices in the super block + * (super_copy) should hold the device list mutex. + */ + + /* + * In normal cases the cur_devices == fs_devices. But in case + * of deleting a seed device, the cur_devices should point to + * its own fs_devices listed under the fs_devices->seed_list. + */ + cur_devices = device->fs_devices; + mutex_lock(&fs_devices->device_list_mutex); + list_del_rcu(&device->dev_list); + + cur_devices->num_devices--; + cur_devices->total_devices--; + /* Update total_devices of the parent fs_devices if it's seed */ + if (cur_devices != fs_devices) + fs_devices->total_devices--; + + if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) + cur_devices->missing_devices--; + + btrfs_assign_next_active_device(device, NULL); + + if (device->bdev_file) { + cur_devices->open_devices--; + /* remove sysfs entry */ + btrfs_sysfs_remove_device(device); + } + + num_devices = btrfs_super_num_devices(fs_info->super_copy) - 1; + btrfs_set_super_num_devices(fs_info->super_copy, num_devices); + mutex_unlock(&fs_devices->device_list_mutex); + + /* + * At this point, the device is zero sized and detached from the + * devices list. All that's left is to zero out the old supers and + * free the device. + * + * We cannot call btrfs_close_bdev() here because we're holding the sb + * write lock, and bdev_fput() on the block device will pull in the + * ->open_mutex on the block device and it's dependencies. Instead + * just flush the device and let the caller do the final bdev_release. + */ + if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { + btrfs_scratch_superblocks(fs_info, device); + if (device->bdev) { + sync_blockdev(device->bdev); + invalidate_bdev(device->bdev); + } + } + + *bdev_file = device->bdev_file; + synchronize_rcu(); + btrfs_free_device(device); + + /* + * This can happen if cur_devices is the private seed devices list. We + * cannot call close_fs_devices() here because it expects the uuid_mutex + * to be held, but in fact we don't need that for the private + * seed_devices, we can simply decrement cur_devices->opened and then + * remove it from our list and free the fs_devices. + */ + if (cur_devices->num_devices == 0) { + list_del_init(&cur_devices->seed_list); + ASSERT(cur_devices->opened == 1, "opened=%d", cur_devices->opened); + cur_devices->opened--; + free_fs_devices(cur_devices); + } + + return btrfs_commit_transaction(trans); + +error_undo: + if (device->bdev) + bdev_allow_freeze(device->bdev); + if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { + mutex_lock(&fs_info->chunk_mutex); + list_add(&device->dev_alloc_list, + &fs_devices->alloc_list); + device->fs_devices->rw_devices++; + btrfs_update_per_profile_avail(fs_info); + mutex_unlock(&fs_info->chunk_mutex); + } + return ret; +} + +void btrfs_rm_dev_replace_remove_srcdev(struct btrfs_device *srcdev) +{ + struct btrfs_fs_devices *fs_devices; + + lockdep_assert_held(&srcdev->fs_info->fs_devices->device_list_mutex); + + /* + * in case of fs with no seed, srcdev->fs_devices will point + * to fs_devices of fs_info. However when the dev being replaced is + * a seed dev it will point to the seed's local fs_devices. In short + * srcdev will have its correct fs_devices in both the cases. + */ + fs_devices = srcdev->fs_devices; + + list_del_rcu(&srcdev->dev_list); + list_del(&srcdev->dev_alloc_list); + fs_devices->num_devices--; + if (test_bit(BTRFS_DEV_STATE_MISSING, &srcdev->dev_state)) + fs_devices->missing_devices--; + + if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &srcdev->dev_state)) + fs_devices->rw_devices--; + + if (srcdev->bdev) + fs_devices->open_devices--; +} + +void btrfs_rm_dev_replace_free_srcdev(struct btrfs_device *srcdev) +{ + struct btrfs_fs_devices *fs_devices = srcdev->fs_devices; + + mutex_lock(&uuid_mutex); + + /* The source was made unfreezable for the replace; undo it. */ + btrfs_close_bdev(srcdev, true); + synchronize_rcu(); + btrfs_free_device(srcdev); + + /* if this is no devs we rather delete the fs_devices */ + if (!fs_devices->num_devices) { + /* + * On a mounted FS, num_devices can't be zero unless it's a + * seed. In case of a seed device being replaced, the replace + * target added to the sprout FS, so there will be no more + * device left under the seed FS. + */ + ASSERT(fs_devices->seeding); + + list_del_init(&fs_devices->seed_list); + close_fs_devices(fs_devices); + free_fs_devices(fs_devices); + } + mutex_unlock(&uuid_mutex); +} + +void btrfs_destroy_dev_replace_tgtdev(struct btrfs_device *tgtdev, + bool allow_freeze) +{ + struct btrfs_fs_devices *fs_devices = tgtdev->fs_info->fs_devices; + + mutex_lock(&fs_devices->device_list_mutex); + + btrfs_sysfs_remove_device(tgtdev); + + if (tgtdev->bdev) + fs_devices->open_devices--; + + fs_devices->num_devices--; + + btrfs_assign_next_active_device(tgtdev, NULL); + + list_del_rcu(&tgtdev->dev_list); + + mutex_unlock(&fs_devices->device_list_mutex); + + btrfs_scratch_superblocks(tgtdev->fs_info, tgtdev); + + btrfs_close_bdev(tgtdev, allow_freeze); + synchronize_rcu(); + btrfs_free_device(tgtdev); +} + +/* + * Populate args from device at path. + * + * @fs_info: the filesystem + * @args: the args to populate + * @path: the path to the device + * + * This will read the super block of the device at @path and populate @args with + * the devid, fsid, and uuid. This is meant to be used for ioctls that need to + * lookup a device to operate on, but need to do it before we take any locks. + * This properly handles the special case of "missing" that a user may pass in, + * and does some basic sanity checks. The caller must make sure that @path is + * properly NUL terminated before calling in, and must call + * btrfs_put_dev_args_from_path() in order to free up the temporary fsid and + * uuid buffers. + * + * Return: 0 for success, -errno for failure + */ +int btrfs_get_dev_args_from_path(struct btrfs_fs_info *fs_info, + struct btrfs_dev_lookup_args *args, + const char *path) +{ + struct btrfs_super_block *disk_super; + struct file *bdev_file; + int ret; + + if (!path || !path[0]) + return -EINVAL; + if (!strcmp(path, "missing")) { + args->missing = true; + return 0; + } + + args->uuid = kzalloc(BTRFS_UUID_SIZE, GFP_KERNEL); + args->fsid = kzalloc(BTRFS_FSID_SIZE, GFP_KERNEL); + if (!args->uuid || !args->fsid) { + btrfs_put_dev_args_from_path(args); + return -ENOMEM; + } + + ret = btrfs_get_bdev_and_sb(path, BLK_OPEN_READ, NULL, 0, + &bdev_file, &disk_super); + if (ret) { + btrfs_put_dev_args_from_path(args); + return ret; + } + + args->devid = btrfs_stack_device_id(&disk_super->dev_item); + memcpy(args->uuid, disk_super->dev_item.uuid, BTRFS_UUID_SIZE); + if (btrfs_fs_incompat(fs_info, METADATA_UUID)) + memcpy(args->fsid, disk_super->metadata_uuid, BTRFS_FSID_SIZE); + else + memcpy(args->fsid, disk_super->fsid, BTRFS_FSID_SIZE); + btrfs_release_disk_super(disk_super); + bdev_fput(bdev_file); + return 0; +} + +/* + * Only use this jointly with btrfs_get_dev_args_from_path() because we will + * allocate our ->uuid and ->fsid pointers, everybody else uses local variables + * that don't need to be freed. + */ +void btrfs_put_dev_args_from_path(struct btrfs_dev_lookup_args *args) +{ + kfree(args->uuid); + kfree(args->fsid); + args->uuid = NULL; + args->fsid = NULL; +} + +struct btrfs_device *btrfs_find_device_by_devspec( + struct btrfs_fs_info *fs_info, u64 devid, + const char *device_path) +{ + BTRFS_DEV_LOOKUP_ARGS(args); + struct btrfs_device *device; + int ret; + + if (devid) { + args.devid = devid; + device = btrfs_find_device(fs_info->fs_devices, &args); + if (!device) + return ERR_PTR(-ENOENT); + return device; + } + + ret = btrfs_get_dev_args_from_path(fs_info, &args, device_path); + if (ret) + return ERR_PTR(ret); + device = btrfs_find_device(fs_info->fs_devices, &args); + btrfs_put_dev_args_from_path(&args); + if (!device) + return ERR_PTR(-ENOENT); + return device; +} + +static struct btrfs_fs_devices *btrfs_init_sprout(struct btrfs_fs_info *fs_info) +{ + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; + struct btrfs_fs_devices *old_devices; + struct btrfs_fs_devices *seed_devices; + + lockdep_assert_held(&uuid_mutex); + if (!fs_devices->seeding) + return ERR_PTR(-EINVAL); + + /* + * Private copy of the seed devices, anchored at + * fs_info->fs_devices->seed_list + */ + seed_devices = alloc_fs_devices(NULL); + if (IS_ERR(seed_devices)) + return seed_devices; + + /* + * It's necessary to retain a copy of the original seed fs_devices in + * fs_uuids so that filesystems which have been seeded can successfully + * reference the seed device from open_seed_devices. This also supports + * multiple fs seed. + */ + old_devices = clone_fs_devices(fs_devices); + if (IS_ERR(old_devices)) { + kfree(seed_devices); + return old_devices; + } + + list_add(&old_devices->fs_list, &fs_uuids); + + memcpy(seed_devices, fs_devices, sizeof(*seed_devices)); + seed_devices->opened = 1; + INIT_LIST_HEAD(&seed_devices->devices); + INIT_LIST_HEAD(&seed_devices->alloc_list); + mutex_init(&seed_devices->device_list_mutex); + + return seed_devices; +} + +/* + * Splice seed devices into the sprout fs_devices. + * Generate a new fsid for the sprouted read-write filesystem. + */ +static void btrfs_setup_sprout(struct btrfs_fs_info *fs_info, + struct btrfs_fs_devices *seed_devices) +{ + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; + struct btrfs_super_block *disk_super = fs_info->super_copy; + struct btrfs_device *device; + u64 super_flags; + + /* + * We are updating the fsid, the thread leading to device_list_add() + * could race, so uuid_mutex is needed. + */ + lockdep_assert_held(&uuid_mutex); + + /* + * The threads listed below may traverse dev_list but can do that without + * device_list_mutex: + * - All device ops and balance - as we are in btrfs_exclop_start. + * - Various dev_list readers - are using RCU. + * - btrfs_ioctl_fitrim() - is using RCU. + * + * For-read threads as below are using device_list_mutex: + * - Readonly scrub btrfs_scrub_dev() + * - Readonly scrub btrfs_scrub_progress() + * - btrfs_get_dev_stats() + */ + lockdep_assert_held(&fs_devices->device_list_mutex); + + list_splice_init_rcu(&fs_devices->devices, &seed_devices->devices, + synchronize_rcu); + list_for_each_entry(device, &seed_devices->devices, dev_list) + device->fs_devices = seed_devices; + + fs_devices->seeding = false; + fs_devices->num_devices = 0; + fs_devices->open_devices = 0; + fs_devices->missing_devices = 0; + fs_devices->rotating = false; + list_add(&seed_devices->seed_list, &fs_devices->seed_list); + + generate_random_uuid(fs_devices->fsid); + memcpy(fs_devices->metadata_uuid, fs_devices->fsid, BTRFS_FSID_SIZE); + memcpy(disk_super->fsid, fs_devices->fsid, BTRFS_FSID_SIZE); + + super_flags = btrfs_super_flags(disk_super) & + ~BTRFS_SUPER_FLAG_SEEDING; + btrfs_set_super_flags(disk_super, super_flags); +} + +/* + * Store the expected generation for seed devices in device items. + */ +static int btrfs_finish_sprout(struct btrfs_trans_handle *trans) +{ + BTRFS_DEV_LOOKUP_ARGS(args); + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *root = fs_info->chunk_root; + BTRFS_PATH_AUTO_FREE(path); + struct extent_buffer *leaf; + struct btrfs_dev_item *dev_item; + struct btrfs_device *device; + struct btrfs_key key; + u8 fs_uuid[BTRFS_FSID_SIZE]; + u8 dev_uuid[BTRFS_UUID_SIZE]; + int ret; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = BTRFS_DEV_ITEMS_OBJECTID; + key.type = BTRFS_DEV_ITEM_KEY; + key.offset = 0; + + while (1) { + btrfs_reserve_chunk_metadata(trans, false); + ret = btrfs_search_slot(trans, root, &key, path, 0, 1); + btrfs_trans_release_chunk_metadata(trans); + if (ret < 0) + return ret; + + leaf = path->nodes[0]; +next_slot: + if (path->slots[0] >= btrfs_header_nritems(leaf)) { + ret = btrfs_next_leaf(root, path); + if (ret > 0) + break; + if (ret < 0) + return ret; + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + btrfs_release_path(path); + continue; + } + + btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); + if (key.objectid != BTRFS_DEV_ITEMS_OBJECTID || + key.type != BTRFS_DEV_ITEM_KEY) + break; + + dev_item = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_dev_item); + args.devid = btrfs_device_id(leaf, dev_item); + read_extent_buffer(leaf, dev_uuid, btrfs_device_uuid(dev_item), + BTRFS_UUID_SIZE); + read_extent_buffer(leaf, fs_uuid, btrfs_device_fsid(dev_item), + BTRFS_FSID_SIZE); + args.uuid = dev_uuid; + args.fsid = fs_uuid; + device = btrfs_find_device(fs_info->fs_devices, &args); + BUG_ON(!device); /* Logic error */ + + if (device->fs_devices->seeding) + btrfs_set_device_generation(leaf, dev_item, + device->generation); + + path->slots[0]++; + goto next_slot; + } + return 0; +} + +/* + * Open @path for @sb with freezing denied before the holder claim is published, + * so a racing bdev_freeze() can never reach a claim a device add or replace may + * still abort. The deny is taken on a throwaway non-holder probe open, then the + * holder is opened by the probe's dev_t. Balanced by the caller. + */ +struct file *btrfs_open_device_deny_freeze(const char *path, + struct super_block *sb) +{ + struct file *probe_file, *bdev_file; + int ret; + + /* WRITE so bdev_file_open_by_path() rejects a read-only device. */ + probe_file = bdev_file_open_by_path(path, BLK_OPEN_WRITE, NULL, NULL); + if (IS_ERR(probe_file)) + return probe_file; + + ret = bdev_deny_freeze(file_bdev(probe_file)); + if (ret) { + bdev_fput(probe_file); + return ERR_PTR(ret); + } + + bdev_file = fs_bdev_file_open_by_dev(file_bdev(probe_file)->bd_dev, + BLK_OPEN_WRITE, sb, sb); + if (IS_ERR(bdev_file)) + bdev_allow_freeze(file_bdev(probe_file)); + bdev_fput(probe_file); + return bdev_file; +} + +int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path) +{ + struct btrfs_root *root = fs_info->dev_root; + struct btrfs_trans_handle *trans; + struct btrfs_device *device; + struct file *bdev_file; + struct super_block *sb = fs_info->sb; + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; + struct btrfs_fs_devices *seed_devices = NULL; + u64 orig_super_total_bytes; + u64 orig_super_num_devices; + int ret = 0; + bool seeding_dev = false; + bool locked = false; + + if (sb_rdonly(sb) && !fs_devices->seeding) + return -EROFS; + + /* Forbid freezing until the device is a committed member (or unwound). */ + bdev_file = btrfs_open_device_deny_freeze(device_path, fs_info->sb); + if (IS_ERR(bdev_file)) + return PTR_ERR(bdev_file); + + if (!btrfs_check_device_zone_type(fs_info, file_bdev(bdev_file))) { + ret = -EINVAL; + goto error; + } + + if (bdev_nr_bytes(file_bdev(bdev_file)) <= BTRFS_DEVICE_RANGE_RESERVED) { + ret = -EINVAL; + goto error; + } + + if (fs_devices->seeding) { + seeding_dev = true; + down_write(&sb->s_umount); + mutex_lock(&uuid_mutex); + locked = true; + } + + sync_blockdev(file_bdev(bdev_file)); + + rcu_read_lock(); + list_for_each_entry_rcu(device, &fs_devices->devices, dev_list) { + if (device->bdev == file_bdev(bdev_file)) { + ret = -EEXIST; + rcu_read_unlock(); + goto error; + } + } + rcu_read_unlock(); + + device = btrfs_alloc_device(fs_info, NULL, NULL, device_path); + if (IS_ERR(device)) { + /* we can safely leave the fs_devices entry around */ + ret = PTR_ERR(device); + goto error; + } + + device->fs_info = fs_info; + device->bdev_file = bdev_file; + device->bdev = file_bdev(bdev_file); + ret = lookup_bdev(device_path, &device->devt); + if (ret) + goto error_free_device; + + ret = btrfs_get_dev_zone_info(device, false); + if (ret) + goto error_free_device; + + trans = btrfs_start_transaction(root, 0); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto error_free_zone; + } + + set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state); + device->generation = trans->transid; + device->io_width = fs_info->sectorsize; + device->io_align = fs_info->sectorsize; + device->sector_size = fs_info->sectorsize; + device->total_bytes = + round_down(bdev_nr_bytes(device->bdev), fs_info->sectorsize); + device->disk_total_bytes = device->total_bytes; + device->commit_total_bytes = device->total_bytes; + set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state); + clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state); + + /* + * Increase dev_stats_ccnt so that corresponding DEV_STATS item can be + * created at the next transaction commit. + */ + atomic_inc(&device->dev_stats_ccnt); + device->dev_stats_valid = 1; + set_blocksize(device->bdev_file, BTRFS_BDEV_BLOCKSIZE); + + if (seeding_dev) { + /* GFP_KERNEL allocation must not be under device_list_mutex */ + seed_devices = btrfs_init_sprout(fs_info); + if (IS_ERR(seed_devices)) { + ret = PTR_ERR(seed_devices); + btrfs_abort_transaction(trans, ret); + goto error_trans; + } + } + + mutex_lock(&fs_devices->device_list_mutex); + if (seeding_dev) { + btrfs_setup_sprout(fs_info, seed_devices); + btrfs_assign_next_active_device(fs_info->fs_devices->latest_dev, + device); + } + + device->fs_devices = fs_devices; + + mutex_lock(&fs_info->chunk_mutex); + list_add_rcu(&device->dev_list, &fs_devices->devices); + list_add(&device->dev_alloc_list, &fs_devices->alloc_list); + fs_devices->num_devices++; + fs_devices->open_devices++; + fs_devices->rw_devices++; + fs_devices->total_devices++; + fs_devices->total_rw_bytes += device->total_bytes; + + atomic64_add(device->total_bytes, &fs_info->free_chunk_space); + + if (bdev_rot(device->bdev)) + fs_devices->rotating = true; + + orig_super_total_bytes = btrfs_super_total_bytes(fs_info->super_copy); + btrfs_set_super_total_bytes(fs_info->super_copy, + round_down(orig_super_total_bytes + device->total_bytes, + fs_info->sectorsize)); + + orig_super_num_devices = btrfs_super_num_devices(fs_info->super_copy); + btrfs_set_super_num_devices(fs_info->super_copy, + orig_super_num_devices + 1); + + /* + * we've got more storage, clear any full flags on the space + * infos + */ + btrfs_clear_space_info_full(fs_info); + + btrfs_update_per_profile_avail(fs_info); + mutex_unlock(&fs_info->chunk_mutex); + + /* Add sysfs device entry */ + btrfs_sysfs_add_device(device); + + mutex_unlock(&fs_devices->device_list_mutex); + + if (seeding_dev) { + mutex_lock(&fs_info->chunk_mutex); + ret = init_first_rw_device(trans); + btrfs_update_per_profile_avail(fs_info); + mutex_unlock(&fs_info->chunk_mutex); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto error_sysfs; + } + } + + ret = btrfs_add_dev_item(trans, device); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto error_sysfs; + } + + if (seeding_dev) { + ret = btrfs_finish_sprout(trans); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto error_sysfs; + } + + /* + * fs_devices now represents the newly sprouted filesystem and + * its fsid has been changed by btrfs_sprout_splice(). + */ + btrfs_sysfs_update_sprout_fsid(fs_devices); + } + + ret = btrfs_commit_transaction(trans); + + if (seeding_dev) { + mutex_unlock(&uuid_mutex); + up_write(&sb->s_umount); + locked = false; + + if (ret) { /* transaction commit */ + bdev_allow_freeze(file_bdev(bdev_file)); + return ret; + } + + ret = btrfs_relocate_sys_chunks(fs_info); + if (ret < 0) + btrfs_handle_fs_error(fs_info, ret, + "Failed to relocate sys chunks after device initialization. This can be fixed using the \"btrfs balance\" command."); + trans = btrfs_attach_transaction(root); + if (IS_ERR(trans)) { + if (PTR_ERR(trans) == -ENOENT) { + bdev_allow_freeze(file_bdev(bdev_file)); + return 0; + } + ret = PTR_ERR(trans); + trans = NULL; + goto error_sysfs; + } + ret = btrfs_commit_transaction(trans); + } + + /* + * Now that we have written a new super block to this device, check all + * other fs_devices list if device_path alienates any other scanned + * device. + * We can ignore the return value as it typically returns -EINVAL and + * only succeeds if the device was an alien. + */ + btrfs_forget_devices(device->devt); + + /* Update ctime/mtime for blkid or udev */ + update_dev_time(device_path); + + bdev_allow_freeze(file_bdev(bdev_file)); + return ret; + +error_sysfs: + btrfs_sysfs_remove_device(device); + mutex_lock(&fs_info->fs_devices->device_list_mutex); + mutex_lock(&fs_info->chunk_mutex); + list_del_rcu(&device->dev_list); + list_del(&device->dev_alloc_list); + fs_info->fs_devices->num_devices--; + fs_info->fs_devices->open_devices--; + fs_info->fs_devices->rw_devices--; + fs_info->fs_devices->total_devices--; + fs_info->fs_devices->total_rw_bytes -= device->total_bytes; + atomic64_sub(device->total_bytes, &fs_info->free_chunk_space); + btrfs_set_super_total_bytes(fs_info->super_copy, + orig_super_total_bytes); + btrfs_set_super_num_devices(fs_info->super_copy, + orig_super_num_devices); + btrfs_update_per_profile_avail(fs_info); + mutex_unlock(&fs_info->chunk_mutex); + mutex_unlock(&fs_info->fs_devices->device_list_mutex); +error_trans: + if (trans) + btrfs_end_transaction(trans); +error_free_zone: + btrfs_destroy_dev_zone_info(device); +error_free_device: + btrfs_free_device(device); +error: + btrfs_release_device_allow_freeze(bdev_file); + if (locked) { + mutex_unlock(&uuid_mutex); + up_write(&sb->s_umount); + } + return ret; +} + +int btrfs_update_device(struct btrfs_trans_handle *trans, struct btrfs_device *device) +{ + int ret; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_root *root = device->fs_info->chunk_root; + struct btrfs_dev_item *dev_item; + struct extent_buffer *leaf; + struct btrfs_key key; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = BTRFS_DEV_ITEMS_OBJECTID; + key.type = BTRFS_DEV_ITEM_KEY; + key.offset = device->devid; + + ret = btrfs_search_slot(trans, root, &key, path, 0, 1); + if (ret < 0) + return ret; + + if (ret > 0) + return -ENOENT; + + leaf = path->nodes[0]; + dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item); + + btrfs_set_device_id(leaf, dev_item, device->devid); + btrfs_set_device_type(leaf, dev_item, device->type); + btrfs_set_device_io_align(leaf, dev_item, device->io_align); + btrfs_set_device_io_width(leaf, dev_item, device->io_width); + btrfs_set_device_sector_size(leaf, dev_item, device->sector_size); + btrfs_set_device_total_bytes(leaf, dev_item, + btrfs_device_get_disk_total_bytes(device)); + btrfs_set_device_bytes_used(leaf, dev_item, + btrfs_device_get_bytes_used(device)); + return ret; +} + +int btrfs_grow_device(struct btrfs_trans_handle *trans, + struct btrfs_device *device, u64 new_size) +{ + struct btrfs_fs_info *fs_info = device->fs_info; + struct btrfs_super_block *super_copy = fs_info->super_copy; + u64 old_total; + u64 diff; + int ret; + + if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) + return -EACCES; + + new_size = round_down(new_size, fs_info->sectorsize); + + mutex_lock(&fs_info->chunk_mutex); + old_total = btrfs_super_total_bytes(super_copy); + diff = round_down(new_size - device->total_bytes, fs_info->sectorsize); + + if (new_size <= device->total_bytes || + test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) { + mutex_unlock(&fs_info->chunk_mutex); + return -EINVAL; + } + + btrfs_set_super_total_bytes(super_copy, + round_down(old_total + diff, fs_info->sectorsize)); + device->fs_devices->total_rw_bytes += diff; + atomic64_add(diff, &fs_info->free_chunk_space); + + btrfs_device_set_total_bytes(device, new_size); + btrfs_device_set_disk_total_bytes(device, new_size); + btrfs_clear_space_info_full(device->fs_info); + if (list_empty(&device->post_commit_list)) + list_add_tail(&device->post_commit_list, + &trans->transaction->dev_update_list); + btrfs_update_per_profile_avail(fs_info); + mutex_unlock(&fs_info->chunk_mutex); + + btrfs_reserve_chunk_metadata(trans, false); + ret = btrfs_update_device(trans, device); + btrfs_trans_release_chunk_metadata(trans); + + return ret; +} + +static int btrfs_free_chunk(struct btrfs_trans_handle *trans, u64 chunk_offset) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *root = fs_info->chunk_root; + int ret; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID; + key.type = BTRFS_CHUNK_ITEM_KEY; + key.offset = chunk_offset; + + ret = btrfs_search_slot(trans, root, &key, path, -1, 1); + if (ret < 0) + return ret; + if (unlikely(ret > 0)) { + /* Logic error or corruption */ + btrfs_err(fs_info, "failed to lookup chunk %llu when freeing", + chunk_offset); + btrfs_abort_transaction(trans, -ENOENT); + return -EUCLEAN; + } + + ret = btrfs_del_item(trans, root, path); + if (unlikely(ret < 0)) { + btrfs_err(fs_info, "failed to delete chunk %llu item", chunk_offset); + btrfs_abort_transaction(trans, ret); + return ret; + } + return ret; +} + +static int btrfs_del_sys_chunk(struct btrfs_fs_info *fs_info, u64 chunk_offset) +{ + struct btrfs_super_block *super_copy = fs_info->super_copy; + struct btrfs_disk_key *disk_key; + struct btrfs_chunk *chunk; + u8 *ptr; + int ret = 0; + u32 num_stripes; + u32 array_size; + u32 len = 0; + u32 cur; + struct btrfs_key key; + + lockdep_assert_held(&fs_info->chunk_mutex); + array_size = btrfs_super_sys_array_size(super_copy); + + ptr = super_copy->sys_chunk_array; + cur = 0; + + while (cur < array_size) { + disk_key = (struct btrfs_disk_key *)ptr; + btrfs_disk_key_to_cpu(&key, disk_key); + + len = sizeof(*disk_key); + + if (key.type == BTRFS_CHUNK_ITEM_KEY) { + chunk = (struct btrfs_chunk *)(ptr + len); + num_stripes = btrfs_stack_chunk_num_stripes(chunk); + len += btrfs_chunk_item_size(num_stripes); + } else { + ret = -EIO; + break; + } + if (key.objectid == BTRFS_FIRST_CHUNK_TREE_OBJECTID && + key.offset == chunk_offset) { + memmove(ptr, ptr + len, array_size - (cur + len)); + array_size -= len; + btrfs_set_super_sys_array_size(super_copy, array_size); + } else { + ptr += len; + cur += len; + } + } + return ret; +} + +struct btrfs_chunk_map *btrfs_find_chunk_map_nolock(struct btrfs_fs_info *fs_info, + u64 logical, u64 length) +{ + struct rb_node *node = fs_info->mapping_tree.rb_root.rb_node; + struct rb_node *prev = NULL; + struct rb_node *orig_prev; + struct btrfs_chunk_map *map; + struct btrfs_chunk_map *prev_map = NULL; + + while (node) { + map = rb_entry(node, struct btrfs_chunk_map, rb_node); + prev = node; + prev_map = map; + + if (logical < map->start) { + node = node->rb_left; + } else if (logical >= map->start + map->chunk_len) { + node = node->rb_right; + } else { + refcount_inc(&map->refs); + return map; + } + } + + if (!prev) + return NULL; + + orig_prev = prev; + while (prev && logical >= prev_map->start + prev_map->chunk_len) { + prev = rb_next(prev); + prev_map = rb_entry(prev, struct btrfs_chunk_map, rb_node); + } + + if (!prev) { + prev = orig_prev; + prev_map = rb_entry(prev, struct btrfs_chunk_map, rb_node); + while (prev && logical < prev_map->start) { + prev = rb_prev(prev); + prev_map = rb_entry(prev, struct btrfs_chunk_map, rb_node); + } + } + + if (prev) { + u64 end = logical + length; + + /* + * Caller can pass a U64_MAX length when it wants to get any + * chunk starting at an offset of 'logical' or higher, so deal + * with underflow by resetting the end offset to U64_MAX. + */ + if (end < logical) + end = U64_MAX; + + if (end > prev_map->start && + logical < prev_map->start + prev_map->chunk_len) { + refcount_inc(&prev_map->refs); + return prev_map; + } + } + + return NULL; +} + +struct btrfs_chunk_map *btrfs_find_chunk_map(struct btrfs_fs_info *fs_info, + u64 logical, u64 length) +{ + struct btrfs_chunk_map *map; + + read_lock(&fs_info->mapping_tree_lock); + map = btrfs_find_chunk_map_nolock(fs_info, logical, length); + read_unlock(&fs_info->mapping_tree_lock); + + return map; +} + +/* + * Find the mapping containing the given logical extent. + * + * @logical: Logical block offset in bytes. + * @length: Length of extent in bytes. + * + * Return: Chunk mapping or ERR_PTR. + */ +struct btrfs_chunk_map *btrfs_get_chunk_map(struct btrfs_fs_info *fs_info, + u64 logical, u64 length) +{ + struct btrfs_chunk_map *map; + + map = btrfs_find_chunk_map(fs_info, logical, length); + + if (unlikely(!map)) { + btrfs_crit(fs_info, + "unable to find chunk map for logical %llu length %llu", + logical, length); + return ERR_PTR(-EINVAL); + } + + if (unlikely(map->start > logical || map->start + map->chunk_len <= logical)) { + btrfs_crit(fs_info, + "found a bad chunk map, wanted %llu-%llu, found %llu-%llu", + logical, logical + length, map->start, + map->start + map->chunk_len); + btrfs_free_chunk_map(map); + return ERR_PTR(-EINVAL); + } + + /* Callers are responsible for dropping the reference. */ + return map; +} + +static int remove_chunk_item(struct btrfs_trans_handle *trans, + struct btrfs_chunk_map *map, u64 chunk_offset) +{ + int i; + + /* + * Removing chunk items and updating the device items in the chunks btree + * requires holding the chunk_mutex. + * See the comment at btrfs_chunk_alloc() for the details. + */ + lockdep_assert_held(&trans->fs_info->chunk_mutex); + + for (i = 0; i < map->num_stripes; i++) { + int ret; + + ret = btrfs_update_device(trans, map->stripes[i].dev); + if (ret) + return ret; + } + + return btrfs_free_chunk(trans, chunk_offset); +} + +int btrfs_remove_dev_extents(struct btrfs_trans_handle *trans, struct btrfs_chunk_map *map) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; + u64 dev_extent_len = 0; + int i, ret = 0; + + /* + * First delete the device extent items from the devices btree. + * We take the device_list_mutex to avoid racing with the finishing phase + * of a device replace operation. See the comment below before acquiring + * fs_info->chunk_mutex. Note that here we do not acquire the chunk_mutex + * because that can result in a deadlock when deleting the device extent + * items from the devices btree - COWing an extent buffer from the btree + * may result in allocating a new metadata chunk, which would attempt to + * lock again fs_info->chunk_mutex. + */ + mutex_lock(&fs_devices->device_list_mutex); + for (i = 0; i < map->num_stripes; i++) { + struct btrfs_device *device = map->stripes[i].dev; + ret = btrfs_free_dev_extent(trans, device, + map->stripes[i].physical, + &dev_extent_len); + if (unlikely(ret)) { + mutex_unlock(&fs_devices->device_list_mutex); + btrfs_abort_transaction(trans, ret); + return ret; + } + + if (device->bytes_used > 0) { + mutex_lock(&fs_info->chunk_mutex); + btrfs_device_set_bytes_used(device, + device->bytes_used - dev_extent_len); + atomic64_add(dev_extent_len, &fs_info->free_chunk_space); + btrfs_clear_space_info_full(fs_info); + + if (list_empty(&device->post_commit_list)) { + list_add_tail(&device->post_commit_list, + &trans->transaction->dev_update_list); + } + + mutex_unlock(&fs_info->chunk_mutex); + } + } + mutex_unlock(&fs_devices->device_list_mutex); + + return 0; +} + +int btrfs_remove_chunk(struct btrfs_trans_handle *trans, u64 chunk_offset) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_chunk_map *map; + int ret; + + map = btrfs_get_chunk_map(fs_info, chunk_offset, 1); + if (IS_ERR(map)) { + DEBUG_WARN("errr %ld reading chunk map at offset %llu", + PTR_ERR(map), chunk_offset); + return PTR_ERR(map); + } + + ret = btrfs_remove_dev_extents(trans, map); + if (ret) + goto out; + + /* + * We acquire fs_info->chunk_mutex for 2 reasons: + * + * 1) Just like with the first phase of the chunk allocation, we must + * reserve system space, do all chunk btree updates and deletions, and + * update the system chunk array in the superblock while holding this + * mutex. This is for similar reasons as explained on the comment at + * the top of btrfs_chunk_alloc(); + * + * 2) Prevent races with the final phase of a device replace operation + * that replaces the device object associated with the map's stripes, + * because the device object's id can change at any time during that + * final phase of the device replace operation + * (dev-replace.c:btrfs_dev_replace_finishing()), so we could grab the + * replaced device and then see it with an ID of + * BTRFS_DEV_REPLACE_DEVID, which would cause a failure when updating + * the device item, which does not exists on the chunk btree. + * The finishing phase of device replace acquires both the + * device_list_mutex and the chunk_mutex, in that order, so we are + * safe by just acquiring the chunk_mutex. + */ + trans->removing_chunk = true; + mutex_lock(&fs_info->chunk_mutex); + + check_system_chunk(trans, map->type); + + ret = remove_chunk_item(trans, map, chunk_offset); + /* + * Normally we should not get -ENOSPC since we reserved space before + * through the call to check_system_chunk(). + * + * Despite our system space_info having enough free space, we may not + * be able to allocate extents from its block groups, because all have + * an incompatible profile, which will force us to allocate a new system + * block group with the right profile, or right after we called + * check_system_space() above, a scrub turned the only system block group + * with enough free space into RO mode. + * This is explained with more detail at do_chunk_alloc(). + * + * So if we get -ENOSPC, allocate a new system chunk and retry once. + */ + if (ret == -ENOSPC) { + const u64 sys_flags = btrfs_system_alloc_profile(fs_info); + struct btrfs_block_group *sys_bg; + struct btrfs_space_info *space_info; + + space_info = btrfs_find_space_info(fs_info, sys_flags); + if (unlikely(!space_info)) { + ret = -EINVAL; + btrfs_abort_transaction(trans, ret); + goto out; + } + + sys_bg = btrfs_create_chunk(trans, space_info, sys_flags); + if (IS_ERR(sys_bg)) { + ret = PTR_ERR(sys_bg); + btrfs_abort_transaction(trans, ret); + goto out; + } + + ret = btrfs_chunk_alloc_add_chunk_item(trans, sys_bg); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + + ret = remove_chunk_item(trans, map, chunk_offset); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + } else if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + + trace_btrfs_chunk_free(fs_info, map, chunk_offset, map->chunk_len); + + if (map->type & BTRFS_BLOCK_GROUP_SYSTEM) { + ret = btrfs_del_sys_chunk(fs_info, chunk_offset); + if (unlikely(ret)) { + btrfs_abort_transaction(trans, ret); + goto out; + } + } + + btrfs_update_per_profile_avail(fs_info); + mutex_unlock(&fs_info->chunk_mutex); + trans->removing_chunk = false; + + /* + * We are done with chunk btree updates and deletions, so release the + * system space we previously reserved (with check_system_chunk()). + */ + btrfs_trans_release_chunk_metadata(trans); + + /* On error, btrfs_remove_block_group() aborts the transaction. */ + ret = btrfs_remove_block_group(trans, map); + if (unlikely(ret)) + ASSERT(BTRFS_FS_ERROR(fs_info) != 0); + +out: + if (trans->removing_chunk) { + mutex_unlock(&fs_info->chunk_mutex); + trans->removing_chunk = false; + } + /* once for us */ + btrfs_free_chunk_map(map); + return ret; +} + +static int btrfs_relocate_chunk_finish(struct btrfs_fs_info *fs_info, + struct btrfs_block_group *bg) +{ + struct btrfs_root *root = fs_info->chunk_root; + struct btrfs_trans_handle *trans; + u64 length; + int ret; + + btrfs_discard_cancel_work(&fs_info->discard_ctl, bg); + length = bg->length; + btrfs_put_block_group(bg); + + /* + * On a zoned file system, discard the whole block group, this will + * trigger a REQ_OP_ZONE_RESET operation on the device zone. If + * resetting the zone fails, don't treat it as a fatal problem from the + * filesystem's point of view. + */ + if (btrfs_is_zoned(fs_info)) { + ret = btrfs_discard_extent(fs_info, bg->start, length, NULL, true); + if (ret) + btrfs_info(fs_info, "failed to reset zone %llu after relocation", + bg->start); + } + + trans = btrfs_start_trans_remove_block_group(root->fs_info, bg->start); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + btrfs_handle_fs_error(root->fs_info, ret, NULL); + return ret; + } + + /* Step two, delete the device extents and the chunk tree entries. */ + ret = btrfs_remove_chunk(trans, bg->start); + btrfs_end_transaction(trans); + + return ret; +} + +int btrfs_relocate_chunk(struct btrfs_fs_info *fs_info, u64 chunk_offset, bool verbose) +{ + struct btrfs_block_group *block_group; + int ret; + + if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) { + btrfs_err(fs_info, + "relocate: not supported on extent tree v2 yet"); + return -EINVAL; + } + + /* + * Prevent races with automatic removal of unused block groups. + * After we relocate and before we remove the chunk with offset + * chunk_offset, automatic removal of the block group can kick in, + * resulting in a failure when calling btrfs_remove_chunk() below. + * + * Make sure to acquire this mutex before doing a tree search (dev + * or chunk trees) to find chunks. Otherwise the cleaner kthread might + * call btrfs_remove_chunk() (through btrfs_delete_unused_bgs()) after + * we release the path used to search the chunk/dev tree and before + * the current task acquires this mutex and calls us. + */ + lockdep_assert_held(&fs_info->reclaim_bgs_lock); + + /* step one, relocate all the extents inside this chunk */ + btrfs_scrub_pause(fs_info); + ret = btrfs_relocate_block_group(fs_info, chunk_offset, verbose); + btrfs_scrub_continue(fs_info); + if (ret) { + /* + * If we had a transaction abort, stop all running scrubs. + * See transaction.c:cleanup_transaction() why we do it here. + */ + if (unlikely(BTRFS_FS_ERROR(fs_info))) + btrfs_scrub_cancel(fs_info); + return ret; + } + + block_group = btrfs_lookup_block_group(fs_info, chunk_offset); + if (!block_group) + return -ENOENT; + + if (should_relocate_using_remap_tree(block_group)) { + /* If we're relocating using the remap tree we're now done. */ + btrfs_put_block_group(block_group); + ret = 0; + } else { + ret = btrfs_relocate_chunk_finish(fs_info, block_group); + } + + return ret; +} + +static int btrfs_relocate_sys_chunks(struct btrfs_fs_info *fs_info) +{ + struct btrfs_root *chunk_root = fs_info->chunk_root; + BTRFS_PATH_AUTO_FREE(path); + struct extent_buffer *leaf; + struct btrfs_chunk *chunk; + struct btrfs_key key; + struct btrfs_key found_key; + u64 chunk_type; + bool retried = false; + int failed = 0; + int ret; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + +again: + key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID; + key.type = BTRFS_CHUNK_ITEM_KEY; + key.offset = (u64)-1; + + while (1) { + mutex_lock(&fs_info->reclaim_bgs_lock); + ret = btrfs_search_slot(NULL, chunk_root, &key, path, 0, 0); + if (ret < 0) { + mutex_unlock(&fs_info->reclaim_bgs_lock); + return ret; + } + if (unlikely(ret == 0)) { + /* + * On the first search we would find chunk tree with + * offset -1, which is not possible. On subsequent + * loops this would find an existing item on an invalid + * offset (one less than the previous one, wrong + * alignment and size). + */ + mutex_unlock(&fs_info->reclaim_bgs_lock); + return -EUCLEAN; + } + + ret = btrfs_previous_item(chunk_root, path, key.objectid, + key.type); + if (ret) + mutex_unlock(&fs_info->reclaim_bgs_lock); + if (ret < 0) + return ret; + if (ret > 0) + break; + + leaf = path->nodes[0]; + btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); + + chunk = btrfs_item_ptr(leaf, path->slots[0], + struct btrfs_chunk); + chunk_type = btrfs_chunk_type(leaf, chunk); + btrfs_release_path(path); + + if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM) { + ret = btrfs_relocate_chunk(fs_info, found_key.offset, + true); + if (ret == -ENOSPC) + failed++; + else + BUG_ON(ret); + } + mutex_unlock(&fs_info->reclaim_bgs_lock); + + if (found_key.offset == 0) + break; + key.offset = found_key.offset - 1; + } + ret = 0; + if (failed && !retried) { + failed = 0; + retried = true; + goto again; + } else if (WARN_ON(failed && retried)) { + ret = -ENOSPC; + } + return ret; +} + +/* + * return 1 : allocate a data chunk successfully, + * return <0: errors during allocating a data chunk, + * return 0 : no need to allocate a data chunk. + */ +static int btrfs_may_alloc_data_chunk(struct btrfs_fs_info *fs_info, + u64 chunk_offset) +{ + struct btrfs_block_group *cache; + u64 bytes_used; + u64 chunk_type; + + cache = btrfs_lookup_block_group(fs_info, chunk_offset); + if (unlikely(!cache)) { + btrfs_err(fs_info, "balance: chunk at bytenr %llu has no corresponding block group", + chunk_offset); + return -EUCLEAN; + } + chunk_type = cache->flags; + btrfs_put_block_group(cache); + + if (!(chunk_type & BTRFS_BLOCK_GROUP_DATA)) + return 0; + + spin_lock(&fs_info->data_sinfo->lock); + bytes_used = fs_info->data_sinfo->bytes_used; + spin_unlock(&fs_info->data_sinfo->lock); + + if (!bytes_used) { + struct btrfs_trans_handle *trans; + int ret; + + trans = btrfs_join_transaction(fs_info->tree_root); + if (IS_ERR(trans)) + return PTR_ERR(trans); + + ret = btrfs_force_chunk_alloc(trans, BTRFS_BLOCK_GROUP_DATA); + btrfs_end_transaction(trans); + if (ret < 0) + return ret; + return 1; + } + + return 0; +} + +static void btrfs_disk_balance_args_to_cpu(struct btrfs_balance_args *cpu, + const struct btrfs_disk_balance_args *disk) +{ + memset(cpu, 0, sizeof(*cpu)); + + cpu->profiles = le64_to_cpu(disk->profiles); + cpu->usage = le64_to_cpu(disk->usage); + cpu->devid = le64_to_cpu(disk->devid); + cpu->pstart = le64_to_cpu(disk->pstart); + cpu->pend = le64_to_cpu(disk->pend); + cpu->vstart = le64_to_cpu(disk->vstart); + cpu->vend = le64_to_cpu(disk->vend); + cpu->target = le64_to_cpu(disk->target); + cpu->flags = le64_to_cpu(disk->flags); + cpu->limit = le64_to_cpu(disk->limit); + cpu->stripes_min = le32_to_cpu(disk->stripes_min); + cpu->stripes_max = le32_to_cpu(disk->stripes_max); +} + +static void btrfs_cpu_balance_args_to_disk(struct btrfs_disk_balance_args *disk, + const struct btrfs_balance_args *cpu) +{ + memset(disk, 0, sizeof(*disk)); + + disk->profiles = cpu_to_le64(cpu->profiles); + disk->usage = cpu_to_le64(cpu->usage); + disk->devid = cpu_to_le64(cpu->devid); + disk->pstart = cpu_to_le64(cpu->pstart); + disk->pend = cpu_to_le64(cpu->pend); + disk->vstart = cpu_to_le64(cpu->vstart); + disk->vend = cpu_to_le64(cpu->vend); + disk->target = cpu_to_le64(cpu->target); + disk->flags = cpu_to_le64(cpu->flags); + disk->limit = cpu_to_le64(cpu->limit); + disk->stripes_min = cpu_to_le32(cpu->stripes_min); + disk->stripes_max = cpu_to_le32(cpu->stripes_max); +} + +static int insert_balance_item(struct btrfs_fs_info *fs_info, + struct btrfs_balance_control *bctl) +{ + struct btrfs_root *root = fs_info->tree_root; + struct btrfs_trans_handle *trans; + struct btrfs_balance_item *item; + struct btrfs_disk_balance_args disk_bargs; + struct btrfs_path *path; + struct extent_buffer *leaf; + struct btrfs_key key; + int ret; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + trans = btrfs_start_transaction(root, 0); + if (IS_ERR(trans)) { + btrfs_free_path(path); + return PTR_ERR(trans); + } + + key.objectid = BTRFS_BALANCE_OBJECTID; + key.type = BTRFS_TEMPORARY_ITEM_KEY; + key.offset = 0; + + ret = btrfs_insert_empty_item(trans, root, path, &key, + sizeof(*item)); + if (ret) + goto out; + + leaf = path->nodes[0]; + item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_balance_item); + + memzero_extent_buffer(leaf, (unsigned long)item, sizeof(*item)); + + btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->data); + btrfs_set_balance_data(leaf, item, &disk_bargs); + btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->meta); + btrfs_set_balance_meta(leaf, item, &disk_bargs); + btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->sys); + btrfs_set_balance_sys(leaf, item, &disk_bargs); + btrfs_set_balance_flags(leaf, item, bctl->flags); +out: + btrfs_free_path(path); + if (ret == 0) + ret = btrfs_commit_transaction(trans); + else + btrfs_end_transaction(trans); + + return ret; +} + +static int del_balance_item(struct btrfs_fs_info *fs_info) +{ + struct btrfs_root *root = fs_info->tree_root; + struct btrfs_trans_handle *trans; + struct btrfs_path *path; + struct btrfs_key key; + int ret; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + trans = btrfs_start_transaction_fallback_global_rsv(root, 0); + if (IS_ERR(trans)) { + btrfs_free_path(path); + return PTR_ERR(trans); + } + + key.objectid = BTRFS_BALANCE_OBJECTID; + key.type = BTRFS_TEMPORARY_ITEM_KEY; + key.offset = 0; + + ret = btrfs_search_slot(trans, root, &key, path, -1, 1); + if (ret < 0) + goto out; + if (ret > 0) { + ret = -ENOENT; + goto out; + } + + ret = btrfs_del_item(trans, root, path); +out: + btrfs_free_path(path); + if (ret == 0) + ret = btrfs_commit_transaction(trans); + else + btrfs_end_transaction(trans); + + return ret; +} + +/* + * This is a heuristic used to reduce the number of chunks balanced on + * resume after balance was interrupted. + */ +static void update_balance_args(struct btrfs_balance_control *bctl) +{ + /* + * Turn on soft mode for chunk types that were being converted. + */ + if (bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) + bctl->data.flags |= BTRFS_BALANCE_ARGS_SOFT; + if (bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) + bctl->sys.flags |= BTRFS_BALANCE_ARGS_SOFT; + if (bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) + bctl->meta.flags |= BTRFS_BALANCE_ARGS_SOFT; + + /* + * Turn on usage filter if is not already used. The idea is + * that chunks that we have already balanced should be + * reasonably full. Don't do it for chunks that are being + * converted - that will keep us from relocating unconverted + * (albeit full) chunks. + */ + if (!(bctl->data.flags & BTRFS_BALANCE_ARGS_USAGE) && + !(bctl->data.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) && + !(bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT)) { + bctl->data.flags |= BTRFS_BALANCE_ARGS_USAGE; + bctl->data.usage = 90; + } + if (!(bctl->sys.flags & BTRFS_BALANCE_ARGS_USAGE) && + !(bctl->sys.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) && + !(bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT)) { + bctl->sys.flags |= BTRFS_BALANCE_ARGS_USAGE; + bctl->sys.usage = 90; + } + if (!(bctl->meta.flags & BTRFS_BALANCE_ARGS_USAGE) && + !(bctl->meta.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) && + !(bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT)) { + bctl->meta.flags |= BTRFS_BALANCE_ARGS_USAGE; + bctl->meta.usage = 90; + } +} + +/* + * Clear the balance status in fs_info and delete the balance item from disk. + */ +static void reset_balance_state(struct btrfs_fs_info *fs_info) +{ + struct btrfs_balance_control *bctl = fs_info->balance_ctl; + int ret; + + ASSERT(fs_info->balance_ctl); + + spin_lock(&fs_info->balance_lock); + fs_info->balance_ctl = NULL; + spin_unlock(&fs_info->balance_lock); + + kfree(bctl); + ret = del_balance_item(fs_info); + if (ret) + btrfs_handle_fs_error(fs_info, ret, NULL); +} + +/* + * Balance filters. Return 1 if chunk should be filtered out + * (should not be balanced). + */ +static bool chunk_profiles_filter(u64 chunk_type, struct btrfs_balance_args *bargs) +{ + chunk_type = chunk_to_extended(chunk_type) & + BTRFS_EXTENDED_PROFILE_MASK; + + if (bargs->profiles & chunk_type) + return false; + + return true; +} + +static int chunk_usage_range_filter(struct btrfs_fs_info *fs_info, u64 chunk_offset, + struct btrfs_balance_args *bargs) +{ + struct btrfs_block_group *cache; + u64 chunk_used; + u64 user_thresh_min; + u64 user_thresh_max; + int ret = 1; + + cache = btrfs_lookup_block_group(fs_info, chunk_offset); + if (unlikely(!cache)) { + btrfs_err(fs_info, "balance: chunk at bytenr %llu has no corresponding block group", + chunk_offset); + return -EUCLEAN; + } + chunk_used = cache->used; + + if (bargs->usage_min == 0) + user_thresh_min = 0; + else + user_thresh_min = mult_perc(cache->length, bargs->usage_min); + + if (bargs->usage_max == 0) + user_thresh_max = 1; + else if (bargs->usage_max > 100) + user_thresh_max = cache->length; + else + user_thresh_max = mult_perc(cache->length, bargs->usage_max); + + if (user_thresh_min <= chunk_used && chunk_used < user_thresh_max) + ret = 0; + + btrfs_put_block_group(cache); + return ret; +} + +static int chunk_usage_filter(struct btrfs_fs_info *fs_info, u64 chunk_offset, + struct btrfs_balance_args *bargs) +{ + struct btrfs_block_group *cache; + u64 chunk_used, user_thresh; + int ret = 1; + + cache = btrfs_lookup_block_group(fs_info, chunk_offset); + if (unlikely(!cache)) { + btrfs_err(fs_info, "balance: chunk at bytenr %llu has no corresponding block group", + chunk_offset); + return -EUCLEAN; + } + chunk_used = cache->used; + + if (bargs->usage_min == 0) + user_thresh = 1; + else if (bargs->usage > 100) + user_thresh = cache->length; + else + user_thresh = mult_perc(cache->length, bargs->usage); + + if (chunk_used < user_thresh) + ret = 0; + + btrfs_put_block_group(cache); + return ret; +} + +static bool chunk_devid_filter(struct extent_buffer *leaf, struct btrfs_chunk *chunk, + struct btrfs_balance_args *bargs) +{ + struct btrfs_stripe *stripe; + int num_stripes = btrfs_chunk_num_stripes(leaf, chunk); + int i; + + for (i = 0; i < num_stripes; i++) { + stripe = btrfs_stripe_nr(chunk, i); + if (btrfs_stripe_devid(leaf, stripe) == bargs->devid) + return false; + } + + return true; +} + +static u64 calc_data_stripes(u64 type, int num_stripes) +{ + const int index = btrfs_bg_flags_to_raid_index(type); + const int ncopies = btrfs_raid_array[index].ncopies; + const int nparity = btrfs_raid_array[index].nparity; + + return (num_stripes - nparity) / ncopies; +} + +/* [pstart, pend) */ +static bool chunk_drange_filter(struct extent_buffer *leaf, struct btrfs_chunk *chunk, + struct btrfs_balance_args *bargs) +{ + struct btrfs_stripe *stripe; + int num_stripes = btrfs_chunk_num_stripes(leaf, chunk); + u64 stripe_offset; + u64 stripe_length; + u64 type; + int factor; + int i; + + if (!(bargs->flags & BTRFS_BALANCE_ARGS_DEVID)) + return false; + + type = btrfs_chunk_type(leaf, chunk); + factor = calc_data_stripes(type, num_stripes); + + for (i = 0; i < num_stripes; i++) { + stripe = btrfs_stripe_nr(chunk, i); + if (btrfs_stripe_devid(leaf, stripe) != bargs->devid) + continue; + + stripe_offset = btrfs_stripe_offset(leaf, stripe); + stripe_length = btrfs_chunk_length(leaf, chunk); + stripe_length = div_u64(stripe_length, factor); + + if (stripe_offset < bargs->pend && + stripe_offset + stripe_length > bargs->pstart) + return false; + } + + return true; +} + +/* [vstart, vend) */ +static bool chunk_vrange_filter(struct extent_buffer *leaf, struct btrfs_chunk *chunk, + u64 chunk_offset, struct btrfs_balance_args *bargs) +{ + if (chunk_offset < bargs->vend && + chunk_offset + btrfs_chunk_length(leaf, chunk) > bargs->vstart) + /* at least part of the chunk is inside this vrange */ + return false; + + return true; +} + +static bool chunk_stripes_range_filter(struct extent_buffer *leaf, + struct btrfs_chunk *chunk, + struct btrfs_balance_args *bargs) +{ + int num_stripes = btrfs_chunk_num_stripes(leaf, chunk); + + if (bargs->stripes_min <= num_stripes + && num_stripes <= bargs->stripes_max) + return false; + + return true; +} + +static bool chunk_soft_convert_filter(u64 chunk_type, struct btrfs_balance_args *bargs) +{ + if (!(bargs->flags & BTRFS_BALANCE_ARGS_CONVERT)) + return false; + + chunk_type = chunk_to_extended(chunk_type) & + BTRFS_EXTENDED_PROFILE_MASK; + + if (bargs->target == chunk_type) + return true; + + return false; +} + +static int should_balance_chunk(struct extent_buffer *leaf, struct btrfs_chunk *chunk, + u64 chunk_offset) +{ + struct btrfs_fs_info *fs_info = leaf->fs_info; + struct btrfs_balance_control *bctl = fs_info->balance_ctl; + struct btrfs_balance_args *bargs = NULL; + u64 chunk_type = btrfs_chunk_type(leaf, chunk); + + /* Treat METADATA_REMAP chunks as METADATA. */ + if (chunk_type & BTRFS_BLOCK_GROUP_METADATA_REMAP) { + chunk_type &= ~BTRFS_BLOCK_GROUP_METADATA_REMAP; + chunk_type |= BTRFS_BLOCK_GROUP_METADATA; + } + + /* type filter */ + if (!((chunk_type & BTRFS_BLOCK_GROUP_TYPE_MASK) & + (bctl->flags & BTRFS_BALANCE_TYPE_MASK))) { + return false; + } + + if (chunk_type & BTRFS_BLOCK_GROUP_DATA) + bargs = &bctl->data; + else if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM) + bargs = &bctl->sys; + else if (chunk_type & BTRFS_BLOCK_GROUP_METADATA) + bargs = &bctl->meta; + + /* profiles filter */ + if ((bargs->flags & BTRFS_BALANCE_ARGS_PROFILES) && + chunk_profiles_filter(chunk_type, bargs)) { + return false; + } + + /* usage filter */ + if (bargs->flags & BTRFS_BALANCE_ARGS_USAGE) { + int ret2; + + ret2 = chunk_usage_filter(fs_info, chunk_offset, bargs); + if (ret2 < 0) + return ret2; + if (ret2) + return false; + } else if (bargs->flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) { + int ret2; + + ret2 = chunk_usage_range_filter(fs_info, chunk_offset, bargs); + if (ret2 < 0) + return ret2; + if (ret2) + return false; + } + + /* devid filter */ + if ((bargs->flags & BTRFS_BALANCE_ARGS_DEVID) && + chunk_devid_filter(leaf, chunk, bargs)) { + return false; + } + + /* drange filter, makes sense only with devid filter */ + if ((bargs->flags & BTRFS_BALANCE_ARGS_DRANGE) && + chunk_drange_filter(leaf, chunk, bargs)) { + return false; + } + + /* vrange filter */ + if ((bargs->flags & BTRFS_BALANCE_ARGS_VRANGE) && + chunk_vrange_filter(leaf, chunk, chunk_offset, bargs)) { + return false; + } + + /* stripes filter */ + if ((bargs->flags & BTRFS_BALANCE_ARGS_STRIPES_RANGE) && + chunk_stripes_range_filter(leaf, chunk, bargs)) { + return false; + } + + /* soft profile changing mode */ + if ((bargs->flags & BTRFS_BALANCE_ARGS_SOFT) && + chunk_soft_convert_filter(chunk_type, bargs)) { + return false; + } + + /* + * limited by count, must be the last filter + */ + if ((bargs->flags & BTRFS_BALANCE_ARGS_LIMIT)) { + if (bargs->limit == 0) + return false; + else + bargs->limit--; + } else if ((bargs->flags & BTRFS_BALANCE_ARGS_LIMIT_RANGE)) { + /* + * Same logic as the 'limit' filter; the minimum cannot be + * determined here because we do not have the global information + * about the count of all chunks that satisfy the filters. + */ + if (bargs->limit_max == 0) + return false; + else + bargs->limit_max--; + } + + return true; +} + +struct remap_chunk_info { + struct list_head list; + u64 offset; + struct btrfs_block_group *bg; + bool made_ro; +}; + +static int cow_remap_tree(struct btrfs_trans_handle *trans, struct btrfs_path *path) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_key key = { 0 }; + int ret; + + ret = btrfs_search_slot(trans, fs_info->remap_root, &key, path, 0, 1); + if (ret < 0) + return ret; + + while (true) { + ret = btrfs_next_leaf(fs_info->remap_root, path); + if (ret < 0) { + return ret; + } else if (ret > 0) { + ret = 0; + break; + } + + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + + btrfs_release_path(path); + + ret = btrfs_search_slot(trans, fs_info->remap_root, &key, path, 0, 1); + if (ret < 0) + break; + } + + return ret; +} + +static int balance_remap_chunks(struct btrfs_fs_info *fs_info, struct btrfs_path *path, + struct list_head *chunks) +{ + struct remap_chunk_info *rci, *tmp; + struct btrfs_trans_handle *trans; + int ret; + + list_for_each_entry_safe(rci, tmp, chunks, list) { + rci->bg = btrfs_lookup_block_group(fs_info, rci->offset); + if (!rci->bg) { + list_del(&rci->list); + kfree(rci); + continue; + } + + ret = btrfs_inc_block_group_ro(rci->bg, false); + if (ret) + goto end; + + rci->made_ro = true; + } + + if (list_empty(chunks)) + return 0; + + trans = btrfs_start_transaction(fs_info->remap_root, 0); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto end; + } + + mutex_lock(&fs_info->remap_mutex); + ret = cow_remap_tree(trans, path); + mutex_unlock(&fs_info->remap_mutex); + + btrfs_release_path(path); + btrfs_commit_transaction(trans); + +end: + while (!list_empty(chunks)) { + bool is_unused; + struct btrfs_block_group *bg; + + rci = list_first_entry(chunks, struct remap_chunk_info, list); + + bg = rci->bg; + if (bg) { + /* + * This is a bit racy and the 'used' status can change + * but this is not a problem as later functions will + * verify it again. + */ + spin_lock(&bg->lock); + is_unused = !btrfs_is_block_group_used(bg); + spin_unlock(&bg->lock); + + if (is_unused) + btrfs_mark_bg_unused(bg); + + if (rci->made_ro) + btrfs_dec_block_group_ro(bg); + + btrfs_put_block_group(bg); + } + + list_del(&rci->list); + kfree(rci); + } + + return ret; +} + +static int __btrfs_balance(struct btrfs_fs_info *fs_info) +{ + struct btrfs_balance_control *bctl = fs_info->balance_ctl; + struct btrfs_root *chunk_root = fs_info->chunk_root; + u64 chunk_type; + struct btrfs_chunk *chunk; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + struct btrfs_key found_key; + struct extent_buffer *leaf; + int slot; + int ret; + int enospc_errors = 0; + bool counting = true; + /* The single value limit and min/max limits use the same bytes in the */ + u64 limit_data = bctl->data.limit; + u64 limit_meta = bctl->meta.limit; + u64 limit_sys = bctl->sys.limit; + u32 count_data = 0; + u32 count_meta = 0; + u32 count_sys = 0; + bool chunk_reserved = false; + struct remap_chunk_info *rci; + unsigned int num_remap_chunks = 0; + LIST_HEAD(remap_chunks); + + path = btrfs_alloc_path(); + if (!path) { + ret = -ENOMEM; + goto error; + } + + /* zero out stat counters */ + spin_lock(&fs_info->balance_lock); + memset(&bctl->stat, 0, sizeof(bctl->stat)); + spin_unlock(&fs_info->balance_lock); +again: + if (!counting) { + /* + * The single value limit and min/max limits use the same bytes + * in the + */ + bctl->data.limit = limit_data; + bctl->meta.limit = limit_meta; + bctl->sys.limit = limit_sys; + } + key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID; + key.type = BTRFS_CHUNK_ITEM_KEY; + key.offset = (u64)-1; + + while (1) { + if ((!counting && atomic_read(&fs_info->balance_pause_req)) || + atomic_read(&fs_info->balance_cancel_req)) { + ret = -ECANCELED; + goto error; + } + + mutex_lock(&fs_info->reclaim_bgs_lock); + ret = btrfs_search_slot(NULL, chunk_root, &key, path, 0, 0); + if (ret < 0) { + mutex_unlock(&fs_info->reclaim_bgs_lock); + goto error; + } + + /* + * this shouldn't happen, it means the last relocate + * failed + */ + if (unlikely(ret == 0)) { + btrfs_err(fs_info, + "unexpected exact match of CHUNK_ITEM in chunk tree, offset 0x%llx", + key.offset); + mutex_unlock(&fs_info->reclaim_bgs_lock); + ret = -EUCLEAN; + goto error; + } + + ret = btrfs_previous_item(chunk_root, path, 0, + BTRFS_CHUNK_ITEM_KEY); + if (ret) { + mutex_unlock(&fs_info->reclaim_bgs_lock); + ret = 0; + break; + } + + leaf = path->nodes[0]; + slot = path->slots[0]; + btrfs_item_key_to_cpu(leaf, &found_key, slot); + + if (found_key.objectid != key.objectid) { + mutex_unlock(&fs_info->reclaim_bgs_lock); + break; + } + + chunk = btrfs_item_ptr(leaf, slot, struct btrfs_chunk); + chunk_type = btrfs_chunk_type(leaf, chunk); + + /* Check if chunk has already been fully relocated. */ + if (chunk_type & BTRFS_BLOCK_GROUP_REMAPPED && + btrfs_chunk_num_stripes(leaf, chunk) == 0) { + btrfs_release_path(path); + mutex_unlock(&fs_info->reclaim_bgs_lock); + goto loop; + } + + if (!counting) { + spin_lock(&fs_info->balance_lock); + bctl->stat.considered++; + spin_unlock(&fs_info->balance_lock); + } + + ret = should_balance_chunk(leaf, chunk, found_key.offset); + + btrfs_release_path(path); + if (ret < 0) { + mutex_unlock(&fs_info->reclaim_bgs_lock); + goto error; + } + if (!ret) { + mutex_unlock(&fs_info->reclaim_bgs_lock); + goto loop; + } + + if (counting) { + mutex_unlock(&fs_info->reclaim_bgs_lock); + spin_lock(&fs_info->balance_lock); + bctl->stat.expected++; + spin_unlock(&fs_info->balance_lock); + + if (chunk_type & BTRFS_BLOCK_GROUP_DATA) + count_data++; + else if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM) + count_sys++; + else if (chunk_type & (BTRFS_BLOCK_GROUP_METADATA | + BTRFS_BLOCK_GROUP_METADATA_REMAP)) + count_meta++; + + goto loop; + } + + /* + * Apply limit_min filter, no need to check if the LIMITS + * filter is used, limit_min is 0 by default + */ + if (((chunk_type & BTRFS_BLOCK_GROUP_DATA) && + count_data < bctl->data.limit_min) + || ((chunk_type & BTRFS_BLOCK_GROUP_METADATA) && + count_meta < bctl->meta.limit_min) + || ((chunk_type & BTRFS_BLOCK_GROUP_SYSTEM) && + count_sys < bctl->sys.limit_min)) { + mutex_unlock(&fs_info->reclaim_bgs_lock); + goto loop; + } + + /* + * Balancing METADATA_REMAP chunks takes place separately - add + * the details to a list so it can be processed later. + */ + if (chunk_type & BTRFS_BLOCK_GROUP_METADATA_REMAP) { + mutex_unlock(&fs_info->reclaim_bgs_lock); + + rci = kmalloc_obj(struct remap_chunk_info, GFP_NOFS); + if (!rci) { + ret = -ENOMEM; + goto error; + } + + rci->offset = found_key.offset; + rci->bg = NULL; + rci->made_ro = false; + list_add_tail(&rci->list, &remap_chunks); + + num_remap_chunks++; + + goto loop; + } + + if (!chunk_reserved) { + /* + * We may be relocating the only data chunk we have, + * which could potentially end up with losing data's + * raid profile, so lets allocate an empty one in + * advance. + */ + ret = btrfs_may_alloc_data_chunk(fs_info, + found_key.offset); + if (ret < 0) { + mutex_unlock(&fs_info->reclaim_bgs_lock); + goto error; + } else if (ret == 1) { + chunk_reserved = true; + } + } + + ret = btrfs_relocate_chunk(fs_info, found_key.offset, true); + mutex_unlock(&fs_info->reclaim_bgs_lock); + if (ret == -ENOSPC) { + enospc_errors++; + } else if (ret == -ETXTBSY) { + btrfs_warn(fs_info, + "skipping relocation of block group %llu due to active swapfile", + found_key.offset); + ret = 0; + } else if (ret) { + goto error; + } else { + spin_lock(&fs_info->balance_lock); + bctl->stat.completed++; + spin_unlock(&fs_info->balance_lock); + } +loop: + if (found_key.offset == 0) + break; + key.offset = found_key.offset - 1; + } + + btrfs_release_path(path); + + if (counting) { + counting = false; + goto again; + } + + if (!list_empty(&remap_chunks)) { + ret = balance_remap_chunks(fs_info, path, &remap_chunks); + if (ret == -ENOSPC) + enospc_errors++; + + if (!ret) { + spin_lock(&fs_info->balance_lock); + bctl->stat.completed += num_remap_chunks; + spin_unlock(&fs_info->balance_lock); + } + } +error: + if (enospc_errors) { + btrfs_info(fs_info, "%d enospc errors during balance", + enospc_errors); + if (!ret) + ret = -ENOSPC; + } + + return ret; +} + +/* + * See if a given profile is valid and reduced. + * + * @flags: profile to validate + * @extended: if true @flags is treated as an extended profile + */ +static int alloc_profile_is_valid(u64 flags, bool extended) +{ + u64 mask = (extended ? BTRFS_EXTENDED_PROFILE_MASK : + BTRFS_BLOCK_GROUP_PROFILE_MASK); + + flags &= ~BTRFS_BLOCK_GROUP_TYPE_MASK; + + /* 1) check that all other bits are zeroed */ + if (flags & ~mask) + return 0; + + /* 2) see if profile is reduced */ + if (flags == 0) + return !extended; /* "0" is valid for usual profiles */ + + return has_single_bit_set(flags); +} + +/* + * Validate target profile against allowed profiles and return true if it's OK. + * Otherwise print the error message and return false. + */ +static inline int validate_convert_profile(struct btrfs_fs_info *fs_info, + const struct btrfs_balance_args *bargs, + u64 allowed, const char *type) +{ + if (!(bargs->flags & BTRFS_BALANCE_ARGS_CONVERT)) + return true; + + /* Profile is valid and does not have bits outside of the allowed set */ + if (alloc_profile_is_valid(bargs->target, 1) && + (bargs->target & ~allowed) == 0) + return true; + + btrfs_err(fs_info, "balance: invalid convert %s profile %s", + type, btrfs_bg_type_to_raid_name(bargs->target)); + return false; +} + +/* + * Fill @buf with textual description of balance filter flags @bargs, up to + * @size_buf including the terminating null. The output may be trimmed if it + * does not fit into the provided buffer. + */ +static void describe_balance_args(struct btrfs_balance_args *bargs, char *buf, + u32 size_buf) +{ + int ret; + u32 size_bp = size_buf; + char *bp = buf; + u64 flags = bargs->flags; + char tmp_buf[128] = {'\0'}; + + if (!flags) + return; + +#define CHECK_APPEND_NOARG(a) \ + do { \ + ret = snprintf(bp, size_bp, (a)); \ + if (ret < 0 || ret >= size_bp) \ + goto out_overflow; \ + size_bp -= ret; \ + bp += ret; \ + } while (0) + +#define CHECK_APPEND_1ARG(a, v1) \ + do { \ + ret = snprintf(bp, size_bp, (a), (v1)); \ + if (ret < 0 || ret >= size_bp) \ + goto out_overflow; \ + size_bp -= ret; \ + bp += ret; \ + } while (0) + +#define CHECK_APPEND_2ARG(a, v1, v2) \ + do { \ + ret = snprintf(bp, size_bp, (a), (v1), (v2)); \ + if (ret < 0 || ret >= size_bp) \ + goto out_overflow; \ + size_bp -= ret; \ + bp += ret; \ + } while (0) + + if (flags & BTRFS_BALANCE_ARGS_CONVERT) + CHECK_APPEND_1ARG("convert=%s,", + btrfs_bg_type_to_raid_name(bargs->target)); + + if (flags & BTRFS_BALANCE_ARGS_SOFT) + CHECK_APPEND_NOARG("soft,"); + + if (flags & BTRFS_BALANCE_ARGS_PROFILES) { + btrfs_describe_block_groups(bargs->profiles, tmp_buf, + sizeof(tmp_buf)); + CHECK_APPEND_1ARG("profiles=%s,", tmp_buf); + } + + if (flags & BTRFS_BALANCE_ARGS_USAGE) + CHECK_APPEND_1ARG("usage=%llu,", bargs->usage); + + if (flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) + CHECK_APPEND_2ARG("usage=%u..%u,", + bargs->usage_min, bargs->usage_max); + + if (flags & BTRFS_BALANCE_ARGS_DEVID) + CHECK_APPEND_1ARG("devid=%llu,", bargs->devid); + + if (flags & BTRFS_BALANCE_ARGS_DRANGE) + CHECK_APPEND_2ARG("drange=%llu..%llu,", + bargs->pstart, bargs->pend); + + if (flags & BTRFS_BALANCE_ARGS_VRANGE) + CHECK_APPEND_2ARG("vrange=%llu..%llu,", + bargs->vstart, bargs->vend); + + if (flags & BTRFS_BALANCE_ARGS_LIMIT) + CHECK_APPEND_1ARG("limit=%llu,", bargs->limit); + + if (flags & BTRFS_BALANCE_ARGS_LIMIT_RANGE) + CHECK_APPEND_2ARG("limit=%u..%u,", + bargs->limit_min, bargs->limit_max); + + if (flags & BTRFS_BALANCE_ARGS_STRIPES_RANGE) + CHECK_APPEND_2ARG("stripes=%u..%u,", + bargs->stripes_min, bargs->stripes_max); + +#undef CHECK_APPEND_2ARG +#undef CHECK_APPEND_1ARG +#undef CHECK_APPEND_NOARG + +out_overflow: + + if (size_bp < size_buf) + buf[size_buf - size_bp - 1] = '\0'; /* remove last , */ + else + buf[0] = '\0'; +} + +static void describe_balance_start_or_resume(struct btrfs_fs_info *fs_info) +{ + u32 size_buf = 1024; + char tmp_buf[192] = {'\0'}; + char AUTO_KFREE(buf); + char *bp; + u32 size_bp = size_buf; + int ret; + struct btrfs_balance_control *bctl = fs_info->balance_ctl; + + buf = kzalloc(size_buf, GFP_KERNEL); + if (!buf) + return; + + bp = buf; + +#define CHECK_APPEND_1ARG(a, v1) \ + do { \ + ret = snprintf(bp, size_bp, (a), (v1)); \ + if (ret < 0 || ret >= size_bp) \ + goto out_overflow; \ + size_bp -= ret; \ + bp += ret; \ + } while (0) + + if (bctl->flags & BTRFS_BALANCE_FORCE) + CHECK_APPEND_1ARG("%s", "-f "); + + if (bctl->flags & BTRFS_BALANCE_DATA) { + describe_balance_args(&bctl->data, tmp_buf, sizeof(tmp_buf)); + CHECK_APPEND_1ARG("-d%s ", tmp_buf); + } + + if (bctl->flags & BTRFS_BALANCE_METADATA) { + describe_balance_args(&bctl->meta, tmp_buf, sizeof(tmp_buf)); + CHECK_APPEND_1ARG("-m%s ", tmp_buf); + } + + if (bctl->flags & BTRFS_BALANCE_SYSTEM) { + describe_balance_args(&bctl->sys, tmp_buf, sizeof(tmp_buf)); + CHECK_APPEND_1ARG("-s%s ", tmp_buf); + } + +#undef CHECK_APPEND_1ARG + +out_overflow: + + if (size_bp < size_buf) + buf[size_buf - size_bp - 1] = '\0'; /* remove last " " */ + btrfs_info(fs_info, "balance: %s %s", + (bctl->flags & BTRFS_BALANCE_RESUME) ? + "resume" : "start", buf); +} + +/* + * Should be called with balance mutex held + */ +int btrfs_balance(struct btrfs_fs_info *fs_info, + struct btrfs_balance_control *bctl, + struct btrfs_ioctl_balance_args *bargs) +{ + u64 meta_target, data_target; + u64 allowed; + bool mixed = false; + int ret; + u64 num_devices; + unsigned seq; + bool reducing_redundancy; + bool paused = false; + int i; + + if (btrfs_fs_closing(fs_info) || + atomic_read(&fs_info->balance_pause_req) || + btrfs_should_cancel_balance(fs_info)) { + ret = -EINVAL; + goto out; + } + + allowed = btrfs_super_incompat_flags(fs_info->super_copy); + if (allowed & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS) + mixed = true; + + /* + * In case of mixed groups both data and meta should be picked, + * and identical options should be given for both of them. + */ + allowed = BTRFS_BALANCE_DATA | BTRFS_BALANCE_METADATA; + if (mixed && (bctl->flags & allowed)) { + if (!(bctl->flags & BTRFS_BALANCE_DATA) || + !(bctl->flags & BTRFS_BALANCE_METADATA) || + memcmp(&bctl->data, &bctl->meta, sizeof(bctl->data))) { + btrfs_err(fs_info, + "balance: mixed groups data and metadata options must be the same"); + ret = -EINVAL; + goto out; + } + } + + /* + * rw_devices will not change at the moment, device add/delete/replace + * are exclusive + */ + num_devices = fs_info->fs_devices->rw_devices; + + /* + * SINGLE profile on-disk has no profile bit, but in-memory we have a + * special bit for it, to make it easier to distinguish. Thus we need + * to set it manually, or balance would refuse the profile. + */ + allowed = BTRFS_AVAIL_ALLOC_BIT_SINGLE; + for (i = 0; i < ARRAY_SIZE(btrfs_raid_array); i++) + if (num_devices >= btrfs_raid_array[i].devs_min) + allowed |= btrfs_raid_array[i].bg_flag; + + if (!validate_convert_profile(fs_info, &bctl->data, allowed, "data") || + !validate_convert_profile(fs_info, &bctl->meta, allowed, "metadata") || + !validate_convert_profile(fs_info, &bctl->sys, allowed, "system")) { + ret = -EINVAL; + goto out; + } + + /* + * Allow to reduce metadata or system integrity only if force set for + * profiles with redundancy (copies, parity) + */ + allowed = 0; + for (i = 0; i < ARRAY_SIZE(btrfs_raid_array); i++) { + if (btrfs_raid_array[i].ncopies >= 2 || + btrfs_raid_array[i].tolerated_failures >= 1) + allowed |= btrfs_raid_array[i].bg_flag; + } + do { + seq = read_seqbegin(&fs_info->profiles_lock); + + if (((bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) && + (fs_info->avail_system_alloc_bits & allowed) && + !(bctl->sys.target & allowed)) || + ((bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) && + (fs_info->avail_metadata_alloc_bits & allowed) && + !(bctl->meta.target & allowed))) + reducing_redundancy = true; + else + reducing_redundancy = false; + + /* if we're not converting, the target field is uninitialized */ + meta_target = (bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) ? + bctl->meta.target : fs_info->avail_metadata_alloc_bits; + data_target = (bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) ? + bctl->data.target : fs_info->avail_data_alloc_bits; + } while (read_seqretry(&fs_info->profiles_lock, seq)); + + if (reducing_redundancy) { + if (bctl->flags & BTRFS_BALANCE_FORCE) { + btrfs_info(fs_info, + "balance: force reducing metadata redundancy"); + } else { + btrfs_err(fs_info, + "balance: reduces metadata redundancy, use --force if you want this"); + ret = -EINVAL; + goto out; + } + } + + if (btrfs_get_num_tolerated_disk_barrier_failures(meta_target) < + btrfs_get_num_tolerated_disk_barrier_failures(data_target)) { + btrfs_warn(fs_info, + "balance: metadata profile %s has lower redundancy than data profile %s", + btrfs_bg_type_to_raid_name(meta_target), + btrfs_bg_type_to_raid_name(data_target)); + } + + ret = insert_balance_item(fs_info, bctl); + if (ret && ret != -EEXIST) + goto out; + + if (!(bctl->flags & BTRFS_BALANCE_RESUME)) { + BUG_ON(ret == -EEXIST); + BUG_ON(fs_info->balance_ctl); + spin_lock(&fs_info->balance_lock); + fs_info->balance_ctl = bctl; + spin_unlock(&fs_info->balance_lock); + } else { + BUG_ON(ret != -EEXIST); + spin_lock(&fs_info->balance_lock); + update_balance_args(bctl); + spin_unlock(&fs_info->balance_lock); + } + + ASSERT(!test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)); + set_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags); + describe_balance_start_or_resume(fs_info); + mutex_unlock(&fs_info->balance_mutex); + + ret = __btrfs_balance(fs_info); + + mutex_lock(&fs_info->balance_mutex); + if (ret == -ECANCELED && atomic_read(&fs_info->balance_pause_req)) { + btrfs_info(fs_info, "balance: paused"); + btrfs_exclop_balance(fs_info, BTRFS_EXCLOP_BALANCE_PAUSED); + paused = true; + } + /* + * Balance can be canceled by: + * + * - Regular cancel request + * Then ret == -ECANCELED and balance_cancel_req > 0 + * + * - Fatal signal to "btrfs" process + * Either the signal caught by wait_reserve_ticket() and callers + * got -EINTR, or caught by btrfs_should_cancel_balance() and + * got -ECANCELED. + * Either way, in this case balance_cancel_req = 0, and + * ret == -EINTR or ret == -ECANCELED. + * + * So here we only check the return value to catch canceled balance. + */ + else if (ret == -ECANCELED || ret == -EINTR) + btrfs_info(fs_info, "balance: canceled"); + else + btrfs_info(fs_info, "balance: ended with status: %d", ret); + + clear_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags); + + if (bargs) { + memset(bargs, 0, sizeof(*bargs)); + btrfs_update_ioctl_balance_args(fs_info, bargs); + } + + /* We didn't pause, we can clean everything up. */ + if (!paused) { + reset_balance_state(fs_info); + btrfs_exclop_finish(fs_info); + } + + wake_up(&fs_info->balance_wait_q); + + return ret; +out: + if (bctl->flags & BTRFS_BALANCE_RESUME) + reset_balance_state(fs_info); + else + kfree(bctl); + btrfs_exclop_finish(fs_info); + + return ret; +} + +static int balance_kthread(void *data) +{ + struct btrfs_fs_info *fs_info = data; + int ret = 0; + + guard(super_write)(fs_info->sb); + + mutex_lock(&fs_info->balance_mutex); + if (fs_info->balance_ctl) + ret = btrfs_balance(fs_info, fs_info->balance_ctl, NULL); + mutex_unlock(&fs_info->balance_mutex); + + return ret; +} + +int btrfs_resume_balance_async(struct btrfs_fs_info *fs_info) +{ + struct task_struct *tsk; + + mutex_lock(&fs_info->balance_mutex); + if (!fs_info->balance_ctl) { + mutex_unlock(&fs_info->balance_mutex); + return 0; + } + mutex_unlock(&fs_info->balance_mutex); + + if (btrfs_test_opt(fs_info, SKIP_BALANCE)) { + btrfs_info(fs_info, "balance: resume skipped"); + return 0; + } + + spin_lock(&fs_info->super_lock); + ASSERT(fs_info->exclusive_operation == BTRFS_EXCLOP_BALANCE_PAUSED, + "exclusive_operation=%d", fs_info->exclusive_operation); + fs_info->exclusive_operation = BTRFS_EXCLOP_BALANCE; + spin_unlock(&fs_info->super_lock); + /* + * A ro->rw remount sequence should continue with the paused balance + * regardless of who pauses it, system or the user as of now, so set + * the resume flag. + */ + spin_lock(&fs_info->balance_lock); + fs_info->balance_ctl->flags |= BTRFS_BALANCE_RESUME; + spin_unlock(&fs_info->balance_lock); + + tsk = kthread_run(balance_kthread, fs_info, "btrfs-balance"); + return PTR_ERR_OR_ZERO(tsk); +} + +int btrfs_recover_balance(struct btrfs_fs_info *fs_info) +{ + struct btrfs_balance_control *bctl; + struct btrfs_balance_item *item; + struct btrfs_disk_balance_args disk_bargs; + BTRFS_PATH_AUTO_FREE(path); + struct extent_buffer *leaf; + struct btrfs_key key; + int ret; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = BTRFS_BALANCE_OBJECTID; + key.type = BTRFS_TEMPORARY_ITEM_KEY; + key.offset = 0; + + ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0); + if (ret < 0) + return ret; + if (ret > 0) { /* ret = -ENOENT; */ + return 0; + } + + bctl = kzalloc_obj(*bctl, GFP_NOFS); + if (!bctl) + return -ENOMEM; + + leaf = path->nodes[0]; + item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_balance_item); + + bctl->flags = btrfs_balance_flags(leaf, item); + bctl->flags |= BTRFS_BALANCE_RESUME; + + btrfs_balance_data(leaf, item, &disk_bargs); + btrfs_disk_balance_args_to_cpu(&bctl->data, &disk_bargs); + btrfs_balance_meta(leaf, item, &disk_bargs); + btrfs_disk_balance_args_to_cpu(&bctl->meta, &disk_bargs); + btrfs_balance_sys(leaf, item, &disk_bargs); + btrfs_disk_balance_args_to_cpu(&bctl->sys, &disk_bargs); + + /* + * This should never happen, as the paused balance state is recovered + * during mount without any chance of other exclusive ops to collide. + * + * This gives the exclusive op status to balance and keeps in paused + * state until user intervention (cancel or umount). If the ownership + * cannot be assigned, show a message but do not fail. The balance + * is in a paused state and must have fs_info::balance_ctl properly + * set up. + */ + if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_BALANCE_PAUSED)) + btrfs_warn(fs_info, + "balance: cannot set exclusive op status, resume manually"); + + btrfs_release_path(path); + + mutex_lock(&fs_info->balance_mutex); + BUG_ON(fs_info->balance_ctl); + spin_lock(&fs_info->balance_lock); + fs_info->balance_ctl = bctl; + spin_unlock(&fs_info->balance_lock); + mutex_unlock(&fs_info->balance_mutex); + return ret; +} + +int btrfs_pause_balance(struct btrfs_fs_info *fs_info) +{ + int ret = 0; + + mutex_lock(&fs_info->balance_mutex); + if (!fs_info->balance_ctl) { + mutex_unlock(&fs_info->balance_mutex); + return -ENOTCONN; + } + + if (test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) { + atomic_inc(&fs_info->balance_pause_req); + mutex_unlock(&fs_info->balance_mutex); + + wait_event(fs_info->balance_wait_q, + !test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)); + + mutex_lock(&fs_info->balance_mutex); + /* we are good with balance_ctl ripped off from under us */ + BUG_ON(test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)); + atomic_dec(&fs_info->balance_pause_req); + } else { + ret = -ENOTCONN; + } + + mutex_unlock(&fs_info->balance_mutex); + return ret; +} + +int btrfs_cancel_balance(struct btrfs_fs_info *fs_info) +{ + mutex_lock(&fs_info->balance_mutex); + if (!fs_info->balance_ctl) { + mutex_unlock(&fs_info->balance_mutex); + return -ENOTCONN; + } + + /* + * A paused balance with the item stored on disk can be resumed at + * mount time if the mount is read-write. Otherwise it's still paused + * and we must not allow cancelling as it deletes the item. + */ + if (sb_rdonly(fs_info->sb)) { + mutex_unlock(&fs_info->balance_mutex); + return -EROFS; + } + + atomic_inc(&fs_info->balance_cancel_req); + /* + * if we are running just wait and return, balance item is + * deleted in btrfs_balance in this case + */ + if (test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) { + mutex_unlock(&fs_info->balance_mutex); + wait_event(fs_info->balance_wait_q, + !test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)); + mutex_lock(&fs_info->balance_mutex); + } else { + mutex_unlock(&fs_info->balance_mutex); + /* + * Lock released to allow other waiters to continue, we'll + * reexamine the status again. + */ + mutex_lock(&fs_info->balance_mutex); + + if (fs_info->balance_ctl) { + reset_balance_state(fs_info); + btrfs_exclop_finish(fs_info); + btrfs_info(fs_info, "balance: canceled"); + } + } + + ASSERT(!test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)); + atomic_dec(&fs_info->balance_cancel_req); + mutex_unlock(&fs_info->balance_mutex); + return 0; +} + +/* + * shrinking a device means finding all of the device extents past + * the new size, and then following the back refs to the chunks. + * The chunk relocation code actually frees the device extent + */ +int btrfs_shrink_device(struct btrfs_device *device, u64 new_size) +{ + struct btrfs_fs_info *fs_info = device->fs_info; + struct btrfs_root *root = fs_info->dev_root; + struct btrfs_trans_handle *trans; + struct btrfs_dev_extent *dev_extent = NULL; + struct btrfs_path *path; + u64 length; + u64 chunk_offset; + int ret; + int slot; + int failed = 0; + bool retried = false; + struct extent_buffer *l; + struct btrfs_key key; + struct btrfs_super_block *super_copy = fs_info->super_copy; + u64 old_total = btrfs_super_total_bytes(super_copy); + u64 old_size = btrfs_device_get_total_bytes(device); + u64 diff; + u64 start; + u64 free_diff = 0; + u64 pending_start, pending_end; + + new_size = round_down(new_size, fs_info->sectorsize); + start = new_size; + diff = round_down(old_size - new_size, fs_info->sectorsize); + + if (test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) + return -EINVAL; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + path->reada = READA_BACK; + + trans = btrfs_start_transaction(root, 0); + if (IS_ERR(trans)) { + btrfs_free_path(path); + return PTR_ERR(trans); + } + + mutex_lock(&fs_info->chunk_mutex); + + btrfs_device_set_total_bytes(device, new_size); + if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { + device->fs_devices->total_rw_bytes -= diff; + + /* + * The new free_chunk_space is new_size - used, so we have to + * subtract the delta of the old free_chunk_space which included + * old_size - used. If used > new_size then just subtract this + * entire device's free space. + */ + if (device->bytes_used < new_size) + free_diff = (old_size - device->bytes_used) - + (new_size - device->bytes_used); + else + free_diff = old_size - device->bytes_used; + atomic64_sub(free_diff, &fs_info->free_chunk_space); + } + + btrfs_update_per_profile_avail(fs_info); + /* + * Once the device's size has been set to the new size, ensure all + * in-memory chunks are synced to disk so that the loop below sees them + * and relocates them accordingly. + */ + if (btrfs_first_pending_extent(device, start, diff, &pending_start, &pending_end)) { + mutex_unlock(&fs_info->chunk_mutex); + ret = btrfs_commit_transaction(trans); + if (ret) + goto done; + } else { + mutex_unlock(&fs_info->chunk_mutex); + btrfs_end_transaction(trans); + } + +again: + key.objectid = device->devid; + key.type = BTRFS_DEV_EXTENT_KEY; + key.offset = (u64)-1; + + do { + mutex_lock(&fs_info->reclaim_bgs_lock); + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) { + mutex_unlock(&fs_info->reclaim_bgs_lock); + goto done; + } + + ret = btrfs_previous_item(root, path, 0, key.type); + if (ret) { + mutex_unlock(&fs_info->reclaim_bgs_lock); + if (ret < 0) + goto done; + ret = 0; + btrfs_release_path(path); + break; + } + + l = path->nodes[0]; + slot = path->slots[0]; + btrfs_item_key_to_cpu(l, &key, path->slots[0]); + + if (key.objectid != device->devid) { + mutex_unlock(&fs_info->reclaim_bgs_lock); + btrfs_release_path(path); + break; + } + + dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent); + length = btrfs_dev_extent_length(l, dev_extent); + + if (key.offset + length <= new_size) { + mutex_unlock(&fs_info->reclaim_bgs_lock); + btrfs_release_path(path); + break; + } + + chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent); + btrfs_release_path(path); + + /* + * We may be relocating the only data chunk we have, + * which could potentially end up with losing data's + * raid profile, so lets allocate an empty one in + * advance. + */ + ret = btrfs_may_alloc_data_chunk(fs_info, chunk_offset); + if (ret < 0) { + mutex_unlock(&fs_info->reclaim_bgs_lock); + goto done; + } + + ret = btrfs_relocate_chunk(fs_info, chunk_offset, true); + mutex_unlock(&fs_info->reclaim_bgs_lock); + if (ret == -ENOSPC) { + failed++; + } else if (ret) { + if (ret == -ETXTBSY) { + btrfs_warn(fs_info, + "could not shrink block group %llu due to active swapfile", + chunk_offset); + } + goto done; + } + } while (key.offset-- > 0); + + if (failed && !retried) { + failed = 0; + retried = true; + goto again; + } else if (failed && retried) { + ret = -ENOSPC; + goto done; + } + + /* Shrinking succeeded, else we would be at "done". */ + trans = btrfs_start_transaction(root, 0); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto done; + } + + mutex_lock(&fs_info->chunk_mutex); + /* Clear all state bits beyond the shrunk device size */ + btrfs_clear_extent_bit(&device->alloc_state, new_size, (u64)-1, + CHUNK_STATE_MASK, NULL); + + btrfs_device_set_disk_total_bytes(device, new_size); + if (list_empty(&device->post_commit_list)) + list_add_tail(&device->post_commit_list, + &trans->transaction->dev_update_list); + + WARN_ON(diff > old_total); + btrfs_set_super_total_bytes(super_copy, + round_down(old_total - diff, fs_info->sectorsize)); + btrfs_update_per_profile_avail(fs_info); + mutex_unlock(&fs_info->chunk_mutex); + + btrfs_reserve_chunk_metadata(trans, false); + /* Now btrfs_update_device() will change the on-disk size. */ + ret = btrfs_update_device(trans, device); + btrfs_trans_release_chunk_metadata(trans); + if (unlikely(ret < 0)) { + btrfs_abort_transaction(trans, ret); + btrfs_end_transaction(trans); + } else { + ret = btrfs_commit_transaction(trans); + } +done: + btrfs_free_path(path); + if (ret) { + mutex_lock(&fs_info->chunk_mutex); + btrfs_device_set_total_bytes(device, old_size); + if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { + device->fs_devices->total_rw_bytes += diff; + atomic64_add(free_diff, &fs_info->free_chunk_space); + } + mutex_unlock(&fs_info->chunk_mutex); + } + return ret; +} + +static int btrfs_add_system_chunk(struct btrfs_fs_info *fs_info, + struct btrfs_key *key, + struct btrfs_chunk *chunk, int item_size) +{ + struct btrfs_super_block *super_copy = fs_info->super_copy; + struct btrfs_disk_key disk_key; + u32 array_size; + u8 *ptr; + + lockdep_assert_held(&fs_info->chunk_mutex); + + array_size = btrfs_super_sys_array_size(super_copy); + if (array_size + item_size + sizeof(disk_key) + > BTRFS_SYSTEM_CHUNK_ARRAY_SIZE) + return -EFBIG; + + ptr = super_copy->sys_chunk_array + array_size; + btrfs_cpu_key_to_disk(&disk_key, key); + memcpy(ptr, &disk_key, sizeof(disk_key)); + ptr += sizeof(disk_key); + memcpy(ptr, chunk, item_size); + item_size += sizeof(disk_key); + btrfs_set_super_sys_array_size(super_copy, array_size + item_size); + + return 0; +} + +/* + * sort the devices in descending order by max_avail, total_avail + */ +static int btrfs_cmp_device_info(const void *a, const void *b) +{ + const struct btrfs_device_info *di_a = a; + const struct btrfs_device_info *di_b = b; + + if (di_a->max_avail > di_b->max_avail) + return -1; + if (di_a->max_avail < di_b->max_avail) + return 1; + if (di_a->total_avail > di_b->total_avail) + return -1; + if (di_a->total_avail < di_b->total_avail) + return 1; + return 0; +} + +/* + * Return 0 if we allocated any virtual(*) chunk, and restore the size to + * @allocated. + * Return -ENOSPC if we have no more space to allocate virtual chunk + * + * *: A virtual chunk is a chunk that only exists during per-profile available + * estimation. + * Those numbers won't really take on-disk space, but only to emulate + * chunk allocator behavior to get accurate estimation on available space. + * + * Another difference is, a virtual chunk has no size limit and doesn't care + * about holes in the device tree, allowing us to exhaust device space + * much faster. + */ +static int alloc_virtual_chunk(struct btrfs_fs_info *fs_info, + struct btrfs_device_info *devices_info, + enum btrfs_raid_types type, + u64 *allocated) +{ + const struct btrfs_raid_attr *raid_attr = &btrfs_raid_array[type]; + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; + struct btrfs_device *device; + u64 stripe_size; + int ndevs = 0; + + lockdep_assert_held(&fs_info->chunk_mutex); + + /* Go through devices to collect their unallocated space. */ + list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) { + u64 avail; + + if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, + &device->dev_state) || + test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) + continue; + + if (device->total_bytes > device->bytes_used + + device->per_profile_allocated) + avail = device->total_bytes - device->bytes_used - + device->per_profile_allocated; + else + avail = 0; + + avail = round_down(avail, fs_info->sectorsize); + + /* And exclude the [0, 1M) reserved space. */ + if (avail > BTRFS_DEVICE_RANGE_RESERVED) + avail -= BTRFS_DEVICE_RANGE_RESERVED; + else + avail = 0; + + /* + * Not enough to support a single stripe, this device + * can not be utilized for chunk allocation. + */ + if (avail < BTRFS_STRIPE_LEN) + continue; + + /* + * Unlike chunk allocator, we don't care about stripe or hole + * size, so here we use @avail directly. + */ + devices_info[ndevs].dev_offset = 0; + devices_info[ndevs].total_avail = avail; + devices_info[ndevs].max_avail = avail; + devices_info[ndevs].dev = device; + ++ndevs; + } + sort(devices_info, ndevs, sizeof(struct btrfs_device_info), + btrfs_cmp_device_info, NULL); + ndevs = rounddown(ndevs, raid_attr->devs_increment); + if (ndevs < raid_attr->devs_min) + return -ENOSPC; + if (raid_attr->devs_max) + ndevs = min(ndevs, (int)raid_attr->devs_max); + else + ndevs = min(ndevs, (int)BTRFS_MAX_DEVS(fs_info)); + + /* + * Stripe size will be determined by the device with the least + * unallocated space. + */ + stripe_size = devices_info[ndevs - 1].total_avail; + + for (int i = 0; i < ndevs; i++) + devices_info[i].dev->per_profile_allocated += stripe_size; + *allocated = div_u64(stripe_size * (ndevs - raid_attr->nparity), + raid_attr->ncopies); + return 0; +} + +static int calc_one_profile_avail(struct btrfs_fs_info *fs_info, + enum btrfs_raid_types type, + u64 *result_ret) +{ + struct btrfs_device_info *devices_info = NULL; + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; + struct btrfs_device *device; + u64 allocated; + u64 result = 0; + int ret = 0; + + lockdep_assert_held(&fs_info->chunk_mutex); + ASSERT(type >= 0 && type < BTRFS_NR_RAID_TYPES); + + /* Not enough devices, quick exit, just update the result. */ + if (fs_devices->rw_devices < btrfs_raid_array[type].devs_min) { + ret = -ENOSPC; + goto out; + } + + devices_info = kzalloc_objs(*devices_info, fs_devices->rw_devices, GFP_NOFS); + if (!devices_info) { + ret = -ENOMEM; + goto out; + } + /* Clear virtual chunk used space for each device. */ + list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) + device->per_profile_allocated = 0; + + while (!alloc_virtual_chunk(fs_info, devices_info, type, &allocated)) + result += allocated; + +out: + kfree(devices_info); + if (ret < 0 && ret != -ENOSPC) + return ret; + *result_ret = result; + return 0; +} + +/* Update the per-profile available space array. */ +void btrfs_update_per_profile_avail(struct btrfs_fs_info *fs_info) +{ + u64 results[BTRFS_NR_RAID_TYPES]; + int ret; + + /* + * Zoned is more complex as we can not simply get the amount of + * available space for each device. + */ + if (btrfs_is_zoned(fs_info)) + goto error; + + for (int i = 0; i < BTRFS_NR_RAID_TYPES; i++) { + ret = calc_one_profile_avail(fs_info, i, &results[i]); + if (ret < 0) + goto error; + } + + spin_lock(&fs_info->fs_devices->per_profile_lock); + for (int i = 0; i < BTRFS_NR_RAID_TYPES; i++) + fs_info->fs_devices->per_profile_avail[i] = results[i]; + spin_unlock(&fs_info->fs_devices->per_profile_lock); + return; +error: + spin_lock(&fs_info->fs_devices->per_profile_lock); + for (int i = 0; i < BTRFS_NR_RAID_TYPES; i++) + fs_info->fs_devices->per_profile_avail[i] = U64_MAX; + spin_unlock(&fs_info->fs_devices->per_profile_lock); +} + +static void check_raid56_incompat_flag(struct btrfs_fs_info *info, u64 type) +{ + if (!(type & BTRFS_BLOCK_GROUP_RAID56_MASK)) + return; + + btrfs_set_fs_incompat(info, RAID56); +} + +static void check_raid1c34_incompat_flag(struct btrfs_fs_info *info, u64 type) +{ + if (!(type & (BTRFS_BLOCK_GROUP_RAID1C3 | BTRFS_BLOCK_GROUP_RAID1C4))) + return; + + btrfs_set_fs_incompat(info, RAID1C34); +} + +/* + * Structure used internally for btrfs_create_chunk() function. + * Wraps needed parameters. + */ +struct alloc_chunk_ctl { + u64 start; + u64 type; + /* Total number of stripes to allocate */ + int num_stripes; + /* sub_stripes info for map */ + int sub_stripes; + /* Stripes per device */ + int dev_stripes; + /* Maximum number of devices to use */ + int devs_max; + /* Minimum number of devices to use */ + int devs_min; + /* ndevs has to be a multiple of this */ + int devs_increment; + /* Number of copies */ + int ncopies; + /* Number of stripes worth of bytes to store parity information */ + int nparity; + u64 max_stripe_size; + u64 max_chunk_size; + u64 dev_extent_min; + u64 stripe_size; + u64 chunk_size; + int ndevs; + /* Space_info the block group is going to belong. */ + struct btrfs_space_info *space_info; +}; + +static void init_alloc_chunk_ctl_policy_regular( + struct btrfs_fs_devices *fs_devices, + struct alloc_chunk_ctl *ctl) +{ + struct btrfs_space_info *space_info; + + space_info = btrfs_find_space_info(fs_devices->fs_info, ctl->type); + ASSERT(space_info); + + ctl->max_chunk_size = READ_ONCE(space_info->chunk_size); + ctl->max_stripe_size = min_t(u64, ctl->max_chunk_size, SZ_1G); + + if (ctl->type & BTRFS_BLOCK_GROUP_SYSTEM) + ctl->devs_max = min_t(int, ctl->devs_max, BTRFS_MAX_DEVS_SYS_CHUNK); + + /* We don't want a chunk larger than 10% of writable space */ + ctl->max_chunk_size = min(mult_perc(fs_devices->total_rw_bytes, 10), + ctl->max_chunk_size); + ctl->dev_extent_min = btrfs_stripe_nr_to_offset(ctl->dev_stripes); +} + +static void init_alloc_chunk_ctl_policy_zoned( + struct btrfs_fs_devices *fs_devices, + struct alloc_chunk_ctl *ctl) +{ + u64 zone_size = fs_devices->fs_info->zone_size; + u64 limit; + int min_num_stripes = ctl->devs_min * ctl->dev_stripes; + int min_data_stripes = (min_num_stripes - ctl->nparity) / ctl->ncopies; + u64 min_chunk_size = min_data_stripes * zone_size; + u64 type = ctl->type; + + ctl->max_stripe_size = zone_size; + if (type & BTRFS_BLOCK_GROUP_DATA) { + ctl->max_chunk_size = round_down(BTRFS_MAX_DATA_CHUNK_SIZE, + zone_size); + } else if (type & BTRFS_BLOCK_GROUP_METADATA) { + ctl->max_chunk_size = ctl->max_stripe_size; + } else if (type & BTRFS_BLOCK_GROUP_SYSTEM) { + ctl->max_chunk_size = 2 * ctl->max_stripe_size; + ctl->devs_max = min_t(int, ctl->devs_max, + BTRFS_MAX_DEVS_SYS_CHUNK); + } else { + BUG(); + } + + /* We don't want a chunk larger than 10% of writable space */ + limit = max(round_down(mult_perc(fs_devices->total_rw_bytes, 10), + zone_size), + min_chunk_size); + ctl->max_chunk_size = min(limit, ctl->max_chunk_size); + ctl->dev_extent_min = zone_size * ctl->dev_stripes; +} + +static void init_alloc_chunk_ctl(struct btrfs_fs_devices *fs_devices, + struct alloc_chunk_ctl *ctl) +{ + int index = btrfs_bg_flags_to_raid_index(ctl->type); + + ctl->sub_stripes = btrfs_raid_array[index].sub_stripes; + ctl->dev_stripes = btrfs_raid_array[index].dev_stripes; + ctl->devs_max = btrfs_raid_array[index].devs_max; + if (!ctl->devs_max) + ctl->devs_max = BTRFS_MAX_DEVS(fs_devices->fs_info); + ctl->devs_min = btrfs_raid_array[index].devs_min; + ctl->devs_increment = btrfs_raid_array[index].devs_increment; + ctl->ncopies = btrfs_raid_array[index].ncopies; + ctl->nparity = btrfs_raid_array[index].nparity; + ctl->ndevs = 0; + + switch (fs_devices->chunk_alloc_policy) { + default: + btrfs_warn_unknown_chunk_allocation(fs_devices->chunk_alloc_policy); + fallthrough; + case BTRFS_CHUNK_ALLOC_REGULAR: + init_alloc_chunk_ctl_policy_regular(fs_devices, ctl); + break; + case BTRFS_CHUNK_ALLOC_ZONED: + init_alloc_chunk_ctl_policy_zoned(fs_devices, ctl); + break; + } +} + +static int gather_device_info(struct btrfs_fs_devices *fs_devices, + struct alloc_chunk_ctl *ctl, + struct btrfs_device_info *devices_info) +{ + struct btrfs_fs_info *info = fs_devices->fs_info; + struct btrfs_device *device; + u64 total_avail; + u64 dev_extent_want = ctl->max_stripe_size * ctl->dev_stripes; + int ret; + int ndevs = 0; + u64 max_avail; + u64 dev_offset; + + /* + * in the first pass through the devices list, we gather information + * about the available holes on each device. + */ + list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) { + if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { + WARN(1, KERN_ERR + "BTRFS: read-only device in alloc_list\n"); + continue; + } + + if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, + &device->dev_state) || + test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) + continue; + + if (device->total_bytes > device->bytes_used) + total_avail = device->total_bytes - device->bytes_used; + else + total_avail = 0; + + /* If there is no space on this device, skip it. */ + if (total_avail < ctl->dev_extent_min) + continue; + + ret = find_free_dev_extent(device, dev_extent_want, &dev_offset, + &max_avail); + if (ret && ret != -ENOSPC) + return ret; + + if (ret == 0) + max_avail = dev_extent_want; + + if (max_avail < ctl->dev_extent_min) { + if (btrfs_test_opt(info, ENOSPC_DEBUG)) + btrfs_debug(info, + "%s: devid %llu has no free space, have=%llu want=%llu", + __func__, device->devid, max_avail, + ctl->dev_extent_min); + continue; + } + + if (ndevs == fs_devices->rw_devices) { + WARN(1, "%s: found more than %llu devices\n", + __func__, fs_devices->rw_devices); + break; + } + devices_info[ndevs].dev_offset = dev_offset; + devices_info[ndevs].max_avail = max_avail; + devices_info[ndevs].total_avail = total_avail; + devices_info[ndevs].dev = device; + ++ndevs; + } + ctl->ndevs = ndevs; + + /* + * now sort the devices by hole size / available space + */ + sort(devices_info, ndevs, sizeof(struct btrfs_device_info), + btrfs_cmp_device_info, NULL); + + return 0; +} + +static int decide_stripe_size_regular(struct alloc_chunk_ctl *ctl, + struct btrfs_device_info *devices_info) +{ + /* Number of stripes that count for block group size */ + int data_stripes; + + /* + * The primary goal is to maximize the number of stripes, so use as + * many devices as possible, even if the stripes are not maximum sized. + * + * The DUP profile stores more than one stripe per device, the + * max_avail is the total size so we have to adjust. + */ + ctl->stripe_size = div_u64(devices_info[ctl->ndevs - 1].max_avail, + ctl->dev_stripes); + ctl->num_stripes = ctl->ndevs * ctl->dev_stripes; + + /* This will have to be fixed for RAID1 and RAID10 over more drives */ + data_stripes = (ctl->num_stripes - ctl->nparity) / ctl->ncopies; + + /* + * Use the number of data stripes to figure out how big this chunk is + * really going to be in terms of logical address space, and compare + * that answer with the max chunk size. If it's higher, we try to + * reduce stripe_size. + */ + if (ctl->stripe_size * data_stripes > ctl->max_chunk_size) { + /* + * Reduce stripe_size, round it up to a 16MB boundary again and + * then use it, unless it ends up being even bigger than the + * previous value we had already. + */ + ctl->stripe_size = min(round_up(div_u64(ctl->max_chunk_size, + data_stripes), SZ_16M), + ctl->stripe_size); + } + + /* Stripe size should not go beyond 1G. */ + ctl->stripe_size = min_t(u64, ctl->stripe_size, SZ_1G); + + /* Align to BTRFS_STRIPE_LEN */ + ctl->stripe_size = round_down(ctl->stripe_size, BTRFS_STRIPE_LEN); + ctl->chunk_size = ctl->stripe_size * data_stripes; + + return 0; +} + +static int decide_stripe_size_zoned(struct alloc_chunk_ctl *ctl, + struct btrfs_device_info *devices_info) +{ + u64 zone_size = devices_info[0].dev->zone_info->zone_size; + /* Number of stripes that count for block group size */ + int data_stripes; + + /* + * It should hold because: + * dev_extent_min == dev_extent_want == zone_size * dev_stripes + */ + ASSERT(devices_info[ctl->ndevs - 1].max_avail == ctl->dev_extent_min, + "ndevs=%d max_avail=%llu dev_extent_min=%llu", ctl->ndevs, + devices_info[ctl->ndevs - 1].max_avail, ctl->dev_extent_min); + + ctl->stripe_size = zone_size; + ctl->num_stripes = ctl->ndevs * ctl->dev_stripes; + data_stripes = (ctl->num_stripes - ctl->nparity) / ctl->ncopies; + + /* stripe_size is fixed in zoned filesystem. Reduce ndevs instead. */ + if (ctl->stripe_size * data_stripes > ctl->max_chunk_size) { + ctl->ndevs = div_u64(div_u64(ctl->max_chunk_size * ctl->ncopies, + ctl->stripe_size) + ctl->nparity, + ctl->dev_stripes); + ctl->num_stripes = ctl->ndevs * ctl->dev_stripes; + data_stripes = (ctl->num_stripes - ctl->nparity) / ctl->ncopies; + ASSERT(ctl->stripe_size * data_stripes <= ctl->max_chunk_size, + "stripe_size=%llu data_stripes=%d max_chunk_size=%llu", + ctl->stripe_size, data_stripes, ctl->max_chunk_size); + } + + ctl->chunk_size = ctl->stripe_size * data_stripes; + + return 0; +} + +static int decide_stripe_size(struct btrfs_fs_devices *fs_devices, + struct alloc_chunk_ctl *ctl, + struct btrfs_device_info *devices_info) +{ + struct btrfs_fs_info *info = fs_devices->fs_info; + + /* + * Round down to number of usable stripes, devs_increment can be any + * number so we can't use round_down() that requires power of 2, while + * rounddown is safe. + */ + ctl->ndevs = rounddown(ctl->ndevs, ctl->devs_increment); + + if (ctl->ndevs < ctl->devs_min) { + if (btrfs_test_opt(info, ENOSPC_DEBUG)) { + btrfs_debug(info, + "%s: not enough devices with free space: have=%d minimum required=%d", + __func__, ctl->ndevs, ctl->devs_min); + } + return -ENOSPC; + } + + ctl->ndevs = min(ctl->ndevs, ctl->devs_max); + + switch (fs_devices->chunk_alloc_policy) { + default: + btrfs_warn_unknown_chunk_allocation(fs_devices->chunk_alloc_policy); + fallthrough; + case BTRFS_CHUNK_ALLOC_REGULAR: + return decide_stripe_size_regular(ctl, devices_info); + case BTRFS_CHUNK_ALLOC_ZONED: + return decide_stripe_size_zoned(ctl, devices_info); + } +} + +static void chunk_map_device_set_bits(struct btrfs_chunk_map *map, unsigned int bits) +{ + for (int i = 0; i < map->num_stripes; i++) { + struct btrfs_io_stripe *stripe = &map->stripes[i]; + struct btrfs_device *device = stripe->dev; + + btrfs_set_extent_bit(&device->alloc_state, stripe->physical, + stripe->physical + map->stripe_size - 1, + bits | EXTENT_NOWAIT, NULL); + } +} + +void btrfs_chunk_map_device_clear_bits(struct btrfs_chunk_map *map, unsigned int bits) +{ + for (int i = 0; i < map->num_stripes; i++) { + struct btrfs_io_stripe *stripe = &map->stripes[i]; + struct btrfs_device *device = stripe->dev; + + btrfs_clear_extent_bit(&device->alloc_state, stripe->physical, + stripe->physical + map->stripe_size - 1, + bits | EXTENT_NOWAIT, NULL); + } +} + +void btrfs_remove_chunk_map(struct btrfs_fs_info *fs_info, struct btrfs_chunk_map *map) +{ + write_lock(&fs_info->mapping_tree_lock); + rb_erase_cached(&map->rb_node, &fs_info->mapping_tree); + RB_CLEAR_NODE(&map->rb_node); + btrfs_chunk_map_device_clear_bits(map, CHUNK_ALLOCATED); + write_unlock(&fs_info->mapping_tree_lock); + + /* Once for the tree reference. */ + btrfs_free_chunk_map(map); +} + +static int btrfs_chunk_map_cmp(const struct rb_node *new, + const struct rb_node *exist) +{ + const struct btrfs_chunk_map *new_map = + rb_entry(new, struct btrfs_chunk_map, rb_node); + const struct btrfs_chunk_map *exist_map = + rb_entry(exist, struct btrfs_chunk_map, rb_node); + + if (new_map->start == exist_map->start) + return 0; + if (new_map->start < exist_map->start) + return -1; + return 1; +} + +EXPORT_FOR_TESTS +int btrfs_add_chunk_map(struct btrfs_fs_info *fs_info, struct btrfs_chunk_map *map) +{ + struct rb_node *exist; + + write_lock(&fs_info->mapping_tree_lock); + exist = rb_find_add_cached(&map->rb_node, &fs_info->mapping_tree, + btrfs_chunk_map_cmp); + + if (exist) { + write_unlock(&fs_info->mapping_tree_lock); + return -EEXIST; + } + chunk_map_device_set_bits(map, CHUNK_ALLOCATED); + btrfs_chunk_map_device_clear_bits(map, CHUNK_TRIMMED); + write_unlock(&fs_info->mapping_tree_lock); + + return 0; +} + +EXPORT_FOR_TESTS +struct btrfs_chunk_map *btrfs_alloc_chunk_map(int num_stripes, gfp_t gfp) +{ + struct btrfs_chunk_map *map; + + map = kmalloc(btrfs_chunk_map_size(num_stripes), gfp); + if (!map) + return NULL; + + refcount_set(&map->refs, 1); + RB_CLEAR_NODE(&map->rb_node); + + return map; +} + +static void set_real_chunk_type(struct btrfs_chunk_map *map) +{ + map->type = map->on_disk_type; + if (likely((map->on_disk_type & BTRFS_BLOCK_GROUP_RAID56_MASK) == 0 || + nr_data_stripes(map) > 1)) + return; + if (map->on_disk_type & BTRFS_BLOCK_GROUP_RAID5) + map->type |= BTRFS_BLOCK_GROUP_RAID1; + else + map->type |= BTRFS_BLOCK_GROUP_RAID1C3; + map->type &= ~BTRFS_BLOCK_GROUP_RAID56_MASK; +} + +static struct btrfs_block_group *create_chunk(struct btrfs_trans_handle *trans, + struct alloc_chunk_ctl *ctl, + struct btrfs_device_info *devices_info) +{ + struct btrfs_fs_info *info = trans->fs_info; + struct btrfs_chunk_map *map; + struct btrfs_block_group *block_group; + u64 start = ctl->start; + u64 type = ctl->type; + int ret; + + map = btrfs_alloc_chunk_map(ctl->num_stripes, GFP_NOFS); + if (!map) + return ERR_PTR(-ENOMEM); + + map->start = start; + map->chunk_len = ctl->chunk_size; + map->stripe_size = ctl->stripe_size; + map->on_disk_type = type; + map->sub_stripes = ctl->sub_stripes; + map->num_stripes = ctl->num_stripes; + set_real_chunk_type(map); + + for (int i = 0; i < ctl->ndevs; i++) { + for (int j = 0; j < ctl->dev_stripes; j++) { + int s = i * ctl->dev_stripes + j; + map->stripes[s].dev = devices_info[i].dev; + map->stripes[s].physical = devices_info[i].dev_offset + + j * ctl->stripe_size; + } + } + + trace_btrfs_chunk_alloc(info, map, start, ctl->chunk_size); + + ret = btrfs_add_chunk_map(info, map); + if (ret) { + btrfs_free_chunk_map(map); + return ERR_PTR(ret); + } + + block_group = btrfs_make_block_group(trans, ctl->space_info, type, start, + ctl->chunk_size); + if (IS_ERR(block_group)) { + btrfs_remove_chunk_map(info, map); + return block_group; + } + + for (int i = 0; i < map->num_stripes; i++) { + struct btrfs_device *dev = map->stripes[i].dev; + + btrfs_device_set_bytes_used(dev, + dev->bytes_used + ctl->stripe_size); + if (list_empty(&dev->post_commit_list)) + list_add_tail(&dev->post_commit_list, + &trans->transaction->dev_update_list); + } + + atomic64_sub(ctl->stripe_size * map->num_stripes, + &info->free_chunk_space); + + check_raid56_incompat_flag(info, type); + check_raid1c34_incompat_flag(info, type); + + btrfs_update_per_profile_avail(info); + + return block_group; +} + +struct btrfs_block_group *btrfs_create_chunk(struct btrfs_trans_handle *trans, + struct btrfs_space_info *space_info, + u64 type) +{ + struct btrfs_fs_info *info = trans->fs_info; + struct btrfs_fs_devices *fs_devices = info->fs_devices; + struct btrfs_device_info AUTO_KFREE(devices_info); + struct alloc_chunk_ctl ctl; + int ret; + + lockdep_assert_held(&info->chunk_mutex); + + if (unlikely(!alloc_profile_is_valid(type, 0))) { + DEBUG_WARN("invalid alloc profile for type %llu", type); + return ERR_PTR(-EINVAL); + } + + if (list_empty(&fs_devices->alloc_list)) { + if (btrfs_test_opt(info, ENOSPC_DEBUG)) + btrfs_debug(info, "%s: no writable device", __func__); + return ERR_PTR(-ENOSPC); + } + + if (unlikely(!(type & BTRFS_BLOCK_GROUP_TYPE_MASK))) { + btrfs_err(info, "invalid chunk type 0x%llx requested", type); + DEBUG_WARN(); + return ERR_PTR(-EINVAL); + } + + ctl.start = find_next_chunk(info); + ctl.type = type; + ctl.space_info = space_info; + init_alloc_chunk_ctl(fs_devices, &ctl); + + devices_info = kzalloc_objs(*devices_info, fs_devices->rw_devices, GFP_NOFS); + if (!devices_info) + return ERR_PTR(-ENOMEM); + + ret = gather_device_info(fs_devices, &ctl, devices_info); + if (ret < 0) + return ERR_PTR(ret); + + ret = decide_stripe_size(fs_devices, &ctl, devices_info); + if (ret < 0) + return ERR_PTR(ret); + + return create_chunk(trans, &ctl, devices_info); +} + +/* + * This function, btrfs_chunk_alloc_add_chunk_item(), typically belongs to the + * phase 1 of chunk allocation. It belongs to phase 2 only when allocating system + * chunks. + * + * See the comment at btrfs_chunk_alloc() for details about the chunk allocation + * phases. + */ +int btrfs_chunk_alloc_add_chunk_item(struct btrfs_trans_handle *trans, + struct btrfs_block_group *bg) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *chunk_root = fs_info->chunk_root; + struct btrfs_key key; + struct btrfs_chunk *chunk; + struct btrfs_stripe *stripe; + struct btrfs_chunk_map *map; + size_t item_size; + int i; + int ret; + + /* + * We take the chunk_mutex for 2 reasons: + * + * 1) Updates and insertions in the chunk btree must be done while holding + * the chunk_mutex, as well as updating the system chunk array in the + * superblock. See the comment on top of btrfs_chunk_alloc() for the + * details; + * + * 2) To prevent races with the final phase of a device replace operation + * that replaces the device object associated with the map's stripes, + * because the device object's id can change at any time during that + * final phase of the device replace operation + * (dev-replace.c:btrfs_dev_replace_finishing()), so we could grab the + * replaced device and then see it with an ID of BTRFS_DEV_REPLACE_DEVID, + * which would cause a failure when updating the device item, which does + * not exists, or persisting a stripe of the chunk item with such ID. + * Here we can't use the device_list_mutex because our caller already + * has locked the chunk_mutex, and the final phase of device replace + * acquires both mutexes - first the device_list_mutex and then the + * chunk_mutex. Using any of those two mutexes protects us from a + * concurrent device replace. + */ + lockdep_assert_held(&fs_info->chunk_mutex); + + map = btrfs_get_chunk_map(fs_info, bg->start, bg->length); + if (IS_ERR(map)) { + ret = PTR_ERR(map); + btrfs_abort_transaction(trans, ret); + return ret; + } + + item_size = btrfs_chunk_item_size(map->num_stripes); + + chunk = kzalloc(item_size, GFP_NOFS); + if (unlikely(!chunk)) { + ret = -ENOMEM; + btrfs_abort_transaction(trans, ret); + goto out; + } + + for (i = 0; i < map->num_stripes; i++) { + struct btrfs_device *device = map->stripes[i].dev; + + ret = btrfs_update_device(trans, device); + if (ret) + goto out; + } + + stripe = &chunk->stripe; + for (i = 0; i < map->num_stripes; i++) { + struct btrfs_device *device = map->stripes[i].dev; + const u64 dev_offset = map->stripes[i].physical; + + btrfs_set_stack_stripe_devid(stripe, device->devid); + btrfs_set_stack_stripe_offset(stripe, dev_offset); + memcpy(stripe->dev_uuid, device->uuid, BTRFS_UUID_SIZE); + stripe++; + } + + btrfs_set_stack_chunk_length(chunk, bg->length); + btrfs_set_stack_chunk_owner(chunk, BTRFS_EXTENT_TREE_OBJECTID); + btrfs_set_stack_chunk_stripe_len(chunk, BTRFS_STRIPE_LEN); + btrfs_set_stack_chunk_type(chunk, map->on_disk_type); + btrfs_set_stack_chunk_num_stripes(chunk, map->num_stripes); + btrfs_set_stack_chunk_io_align(chunk, BTRFS_STRIPE_LEN); + btrfs_set_stack_chunk_io_width(chunk, BTRFS_STRIPE_LEN); + btrfs_set_stack_chunk_sector_size(chunk, fs_info->sectorsize); + btrfs_set_stack_chunk_sub_stripes(chunk, map->sub_stripes); + + key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID; + key.type = BTRFS_CHUNK_ITEM_KEY; + key.offset = bg->start; + + ret = btrfs_insert_item(trans, chunk_root, &key, chunk, item_size); + if (ret) + goto out; + + set_bit(BLOCK_GROUP_FLAG_CHUNK_ITEM_INSERTED, &bg->runtime_flags); + + if (map->type & BTRFS_BLOCK_GROUP_SYSTEM) { + ret = btrfs_add_system_chunk(fs_info, &key, chunk, item_size); + if (ret) + goto out; + } + +out: + kfree(chunk); + btrfs_free_chunk_map(map); + return ret; +} + +static noinline int init_first_rw_device(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + u64 alloc_profile; + struct btrfs_block_group *meta_bg; + struct btrfs_space_info *meta_space_info; + struct btrfs_block_group *sys_bg; + struct btrfs_space_info *sys_space_info; + + /* + * When adding a new device for sprouting, the seed device is read-only + * so we must first allocate a metadata and a system chunk. But before + * adding the block group items to the extent, device and chunk btrees, + * we must first: + * + * 1) Create both chunks without doing any changes to the btrees, as + * otherwise we would get -ENOSPC since the block groups from the + * seed device are read-only; + * + * 2) Add the device item for the new sprout device - finishing the setup + * of a new block group requires updating the device item in the chunk + * btree, so it must exist when we attempt to do it. The previous step + * ensures this does not fail with -ENOSPC. + * + * After that we can add the block group items to their btrees: + * update existing device item in the chunk btree, add a new block group + * item to the extent btree, add a new chunk item to the chunk btree and + * finally add the new device extent items to the devices btree. + */ + + alloc_profile = btrfs_metadata_alloc_profile(fs_info); + meta_space_info = btrfs_find_space_info(fs_info, alloc_profile); + if (unlikely(!meta_space_info)) { + DEBUG_WARN(); + return -EINVAL; + } + meta_bg = btrfs_create_chunk(trans, meta_space_info, alloc_profile); + if (IS_ERR(meta_bg)) + return PTR_ERR(meta_bg); + + alloc_profile = btrfs_system_alloc_profile(fs_info); + sys_space_info = btrfs_find_space_info(fs_info, alloc_profile); + if (unlikely(!sys_space_info)) { + DEBUG_WARN(); + return -EINVAL; + } + sys_bg = btrfs_create_chunk(trans, sys_space_info, alloc_profile); + if (IS_ERR(sys_bg)) + return PTR_ERR(sys_bg); + + return 0; +} + +static inline int btrfs_chunk_max_errors(struct btrfs_chunk_map *map) +{ + const int index = btrfs_bg_flags_to_raid_index(map->type); + + return btrfs_raid_array[index].tolerated_failures; +} + +bool btrfs_chunk_writeable(struct btrfs_fs_info *fs_info, u64 chunk_offset) +{ + struct btrfs_chunk_map *map; + int miss_ndevs = 0; + int i; + bool ret = true; + + map = btrfs_get_chunk_map(fs_info, chunk_offset, 1); + if (IS_ERR(map)) + return false; + + for (i = 0; i < map->num_stripes; i++) { + if (test_bit(BTRFS_DEV_STATE_MISSING, + &map->stripes[i].dev->dev_state)) { + miss_ndevs++; + continue; + } + if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, + &map->stripes[i].dev->dev_state)) { + ret = false; + goto end; + } + } + + /* + * If the number of missing devices is larger than max errors, we can + * not write the data into that chunk successfully. + */ + if (miss_ndevs > btrfs_chunk_max_errors(map)) + ret = false; +end: + btrfs_free_chunk_map(map); + return ret; +} + +void btrfs_mapping_tree_free(struct btrfs_fs_info *fs_info) +{ + write_lock(&fs_info->mapping_tree_lock); + while (!RB_EMPTY_ROOT(&fs_info->mapping_tree.rb_root)) { + struct btrfs_chunk_map *map; + struct rb_node *node; + + node = rb_first_cached(&fs_info->mapping_tree); + map = rb_entry(node, struct btrfs_chunk_map, rb_node); + rb_erase_cached(&map->rb_node, &fs_info->mapping_tree); + RB_CLEAR_NODE(&map->rb_node); + btrfs_chunk_map_device_clear_bits(map, CHUNK_ALLOCATED); + /* Once for the tree ref. */ + btrfs_free_chunk_map(map); + cond_resched_rwlock_write(&fs_info->mapping_tree_lock); + } + write_unlock(&fs_info->mapping_tree_lock); +} + +static int btrfs_chunk_map_num_copies(const struct btrfs_chunk_map *map) +{ + enum btrfs_raid_types index = btrfs_bg_flags_to_raid_index(map->type); + + if (map->type & BTRFS_BLOCK_GROUP_RAID5) + return 2; + + /* + * There could be two corrupted data stripes, we need to loop retry in + * order to rebuild the correct data. + * + * Fail a stripe at a time on every retry except the stripe under + * reconstruction. + */ + if (map->type & BTRFS_BLOCK_GROUP_RAID6) + return map->num_stripes; + + /* Non-RAID56, use their ncopies from btrfs_raid_array. */ + return btrfs_raid_array[index].ncopies; +} + +int btrfs_num_copies(struct btrfs_fs_info *fs_info, u64 logical, u64 len) +{ + struct btrfs_chunk_map *map; + int ret; + + map = btrfs_get_chunk_map(fs_info, logical, len); + if (IS_ERR(map)) + /* + * We could return errors for these cases, but that could get + * ugly and we'd probably do the same thing which is just not do + * anything else and exit, so return 1 so the callers don't try + * to use other copies. + */ + return 1; + + ret = btrfs_chunk_map_num_copies(map); + btrfs_free_chunk_map(map); + return ret; +} + +unsigned long btrfs_full_stripe_len(struct btrfs_fs_info *fs_info, + u64 logical) +{ + struct btrfs_chunk_map *map; + unsigned long len = fs_info->sectorsize; + + if (!btrfs_fs_incompat(fs_info, RAID56)) + return len; + + map = btrfs_get_chunk_map(fs_info, logical, len); + + if (!WARN_ON(IS_ERR(map))) { + if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) + len = btrfs_stripe_nr_to_offset(nr_data_stripes(map)); + btrfs_free_chunk_map(map); + } + return len; +} + +#ifdef CONFIG_BTRFS_EXPERIMENTAL +static int btrfs_read_preferred(struct btrfs_chunk_map *map, int first, int num_stripes) +{ + for (int index = first; index < first + num_stripes; index++) { + const struct btrfs_device *device = map->stripes[index].dev; + + if (device->devid == READ_ONCE(device->fs_devices->read_devid)) + return index; + } + + /* If no read-preferred device is set use the first stripe. */ + return first; +} + +struct stripe_mirror { + u64 devid; + int num; +}; + +static int btrfs_cmp_devid(const void *a, const void *b) +{ + const struct stripe_mirror *s1 = (const struct stripe_mirror *)a; + const struct stripe_mirror *s2 = (const struct stripe_mirror *)b; + + if (s1->devid < s2->devid) + return -1; + if (s1->devid > s2->devid) + return 1; + return 0; +} + +/* + * Select a stripe for reading using the round-robin algorithm. + * + * 1. Compute the read cycle as the total sectors read divided by the minimum + * sectors per device. + * 2. Determine the stripe number for the current read by taking the modulus + * of the read cycle with the total number of stripes: + * + * stripe index = (total sectors / min sectors per dev) % num stripes + * + * The calculated stripe index is then used to select the corresponding device + * from the list of devices, which is ordered by devid. + */ +static int btrfs_read_rr(const struct btrfs_chunk_map *map, int first, int num_stripes) +{ + struct stripe_mirror stripes[BTRFS_RAID1_MAX_MIRRORS] = { 0 }; + struct btrfs_device *device = map->stripes[first].dev; + struct btrfs_fs_info *fs_info = device->fs_devices->fs_info; + unsigned int read_cycle; + unsigned int total_reads; + unsigned int min_reads_per_dev; + + total_reads = percpu_counter_sum(&fs_info->stats_read_blocks); + min_reads_per_dev = READ_ONCE(fs_info->fs_devices->rr_min_contig_read) >> + fs_info->sectorsize_bits; + + for (int index = 0, i = first; i < first + num_stripes; i++) { + stripes[index].devid = map->stripes[i].dev->devid; + stripes[index].num = i; + index++; + } + sort(stripes, num_stripes, sizeof(struct stripe_mirror), + btrfs_cmp_devid, NULL); + + read_cycle = total_reads / min_reads_per_dev; + return stripes[read_cycle % num_stripes].num; +} +#endif + +static int find_live_mirror(struct btrfs_fs_info *fs_info, + struct btrfs_chunk_map *map, int first, + bool dev_replace_is_ongoing) +{ + const enum btrfs_read_policy policy = READ_ONCE(fs_info->fs_devices->read_policy); + int i; + int num_stripes; + int preferred_mirror; + int tolerance; + struct btrfs_device *srcdev; + + ASSERT((map->type & (BTRFS_BLOCK_GROUP_RAID1_MASK | BTRFS_BLOCK_GROUP_RAID10)), + "type=%llu", map->type); + + if (map->type & BTRFS_BLOCK_GROUP_RAID10) + num_stripes = map->sub_stripes; + else + num_stripes = map->num_stripes; + + switch (policy) { + default: + /* Shouldn't happen, just warn and use pid instead of failing */ + btrfs_warn_rl(fs_info, "unknown read_policy type %u, reset to pid", + policy); + WRITE_ONCE(fs_info->fs_devices->read_policy, BTRFS_READ_POLICY_PID); + fallthrough; + case BTRFS_READ_POLICY_PID: + preferred_mirror = first + (current->pid % num_stripes); + break; +#ifdef CONFIG_BTRFS_EXPERIMENTAL + case BTRFS_READ_POLICY_RR: + preferred_mirror = btrfs_read_rr(map, first, num_stripes); + break; + case BTRFS_READ_POLICY_DEVID: + preferred_mirror = btrfs_read_preferred(map, first, num_stripes); + break; +#endif + } + + if (dev_replace_is_ongoing && + fs_info->dev_replace.cont_reading_from_srcdev_mode == + BTRFS_DEV_REPLACE_ITEM_CONT_READING_FROM_SRCDEV_MODE_AVOID) + srcdev = fs_info->dev_replace.srcdev; + else + srcdev = NULL; + + /* + * try to avoid the drive that is the source drive for a + * dev-replace procedure, only choose it if no other non-missing + * mirror is available + */ + for (tolerance = 0; tolerance < 2; tolerance++) { + if (map->stripes[preferred_mirror].dev->bdev && + (tolerance || map->stripes[preferred_mirror].dev != srcdev)) + return preferred_mirror; + for (i = first; i < first + num_stripes; i++) { + if (map->stripes[i].dev->bdev && + (tolerance || map->stripes[i].dev != srcdev)) + return i; + } + } + + /* we couldn't find one that doesn't fail. Just return something + * and the io error handling code will clean up eventually + */ + return preferred_mirror; +} + +EXPORT_FOR_TESTS +struct btrfs_io_context *alloc_btrfs_io_context(struct btrfs_fs_info *fs_info, + u64 logical, u16 total_stripes) +{ + struct btrfs_io_context *bioc; + + bioc = kzalloc_flex(*bioc, stripes, total_stripes, GFP_NOFS); + + if (!bioc) + return NULL; + + refcount_set(&bioc->refs, 1); + + bioc->fs_info = fs_info; + bioc->replace_stripe_src = -1; + bioc->full_stripe_logical = (u64)-1; + bioc->logical = logical; + + return bioc; +} + +void btrfs_get_bioc(struct btrfs_io_context *bioc) +{ + WARN_ON(!refcount_read(&bioc->refs)); + refcount_inc(&bioc->refs); +} + +void btrfs_put_bioc(struct btrfs_io_context *bioc) +{ + if (!bioc) + return; + if (refcount_dec_and_test(&bioc->refs)) + kfree(bioc); +} + +/* + * Please note that, discard won't be sent to target device of device + * replace. + */ +struct btrfs_discard_stripe *btrfs_map_discard(struct btrfs_fs_info *fs_info, + u64 logical, u64 *length_ret, + u32 *num_stripes, bool do_remap) +{ + struct btrfs_chunk_map *map; + struct btrfs_discard_stripe *stripes; + u64 length = *length_ret; + u64 offset; + u32 stripe_nr; + u32 stripe_nr_end; + u32 stripe_cnt; + u64 stripe_end_offset; + u64 stripe_offset; + u32 stripe_index; + u32 factor = 0; + u32 sub_stripes = 0; + u32 stripes_per_dev = 0; + u32 remaining_stripes = 0; + u32 last_stripe = 0; + int ret; + int i; + + map = btrfs_get_chunk_map(fs_info, logical, length); + if (IS_ERR(map)) + return ERR_CAST(map); + + if (do_remap && (map->type & BTRFS_BLOCK_GROUP_REMAPPED)) { + u64 new_logical = logical; + + ret = btrfs_translate_remap(fs_info, &new_logical, &length); + if (ret) + goto out_free_map; + + if (new_logical != logical) { + btrfs_free_chunk_map(map); + + map = btrfs_get_chunk_map(fs_info, new_logical, length); + if (IS_ERR(map)) + return ERR_CAST(map); + + logical = new_logical; + } + } + + /* we don't discard raid56 yet */ + if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) { + ret = -EOPNOTSUPP; + goto out_free_map; + } + + offset = logical - map->start; + length = min_t(u64, map->start + map->chunk_len - logical, length); + *length_ret = length; + + /* + * stripe_nr counts the total number of stripes we have to stride + * to get to this block + */ + stripe_nr = offset >> BTRFS_STRIPE_LEN_SHIFT; + + /* stripe_offset is the offset of this block in its stripe */ + stripe_offset = offset - btrfs_stripe_nr_to_offset(stripe_nr); + + stripe_nr_end = round_up(offset + length, BTRFS_STRIPE_LEN) >> + BTRFS_STRIPE_LEN_SHIFT; + stripe_cnt = stripe_nr_end - stripe_nr; + stripe_end_offset = btrfs_stripe_nr_to_offset(stripe_nr_end) - + (offset + length); + /* + * after this, stripe_nr is the number of stripes on this + * device we have to walk to find the data, and stripe_index is + * the number of our device in the stripe array + */ + *num_stripes = 1; + stripe_index = 0; + if (map->type & (BTRFS_BLOCK_GROUP_RAID0 | + BTRFS_BLOCK_GROUP_RAID10)) { + if (map->type & BTRFS_BLOCK_GROUP_RAID0) + sub_stripes = 1; + else + sub_stripes = map->sub_stripes; + + factor = map->num_stripes / sub_stripes; + *num_stripes = min_t(u64, map->num_stripes, + sub_stripes * stripe_cnt); + stripe_index = stripe_nr % factor; + stripe_nr /= factor; + stripe_index *= sub_stripes; + + remaining_stripes = stripe_cnt % factor; + stripes_per_dev = stripe_cnt / factor; + last_stripe = ((stripe_nr_end - 1) % factor) * sub_stripes; + } else if (map->type & (BTRFS_BLOCK_GROUP_RAID1_MASK | + BTRFS_BLOCK_GROUP_DUP)) { + *num_stripes = map->num_stripes; + } else { + stripe_index = stripe_nr % map->num_stripes; + stripe_nr /= map->num_stripes; + } + + stripes = kzalloc_objs(*stripes, *num_stripes, GFP_NOFS); + if (!stripes) { + ret = -ENOMEM; + goto out_free_map; + } + + for (i = 0; i < *num_stripes; i++) { + stripes[i].physical = + map->stripes[stripe_index].physical + + stripe_offset + btrfs_stripe_nr_to_offset(stripe_nr); + stripes[i].dev = map->stripes[stripe_index].dev; + + if (map->type & (BTRFS_BLOCK_GROUP_RAID0 | + BTRFS_BLOCK_GROUP_RAID10)) { + stripes[i].length = btrfs_stripe_nr_to_offset(stripes_per_dev); + + if (i / sub_stripes < remaining_stripes) + stripes[i].length += BTRFS_STRIPE_LEN; + + /* + * Special for the first stripe and + * the last stripe: + * + * |-------|...|-------| + * |----------| + * off end_off + */ + if (i < sub_stripes) + stripes[i].length -= stripe_offset; + + if (stripe_index >= last_stripe && + stripe_index <= (last_stripe + + sub_stripes - 1)) + stripes[i].length -= stripe_end_offset; + + if (i == sub_stripes - 1) + stripe_offset = 0; + } else { + stripes[i].length = length; + } + + stripe_index++; + if (stripe_index == map->num_stripes) { + stripe_index = 0; + stripe_nr++; + } + } + + btrfs_free_chunk_map(map); + return stripes; +out_free_map: + btrfs_free_chunk_map(map); + return ERR_PTR(ret); +} + +static bool is_block_group_to_copy(struct btrfs_fs_info *fs_info, u64 logical) +{ + struct btrfs_block_group *cache; + bool ret; + + /* Non zoned filesystem does not use "to_copy" flag */ + if (!btrfs_is_zoned(fs_info)) + return false; + + cache = btrfs_lookup_block_group(fs_info, logical); + + ret = test_bit(BLOCK_GROUP_FLAG_TO_COPY, &cache->runtime_flags); + + btrfs_put_block_group(cache); + return ret; +} + +static void handle_ops_on_dev_replace(struct btrfs_io_context *bioc, + struct btrfs_dev_replace *dev_replace, + u64 logical, + struct btrfs_io_geometry *io_geom) +{ + u64 srcdev_devid = dev_replace->srcdev->devid; + /* + * At this stage, num_stripes is still the real number of stripes, + * excluding the duplicated stripes. + */ + int num_stripes = io_geom->num_stripes; + int max_errors = io_geom->max_errors; + int nr_extra_stripes = 0; + int i; + + /* + * A block group which has "to_copy" set will eventually be copied by + * the dev-replace process. We can avoid cloning IO here. + */ + if (is_block_group_to_copy(dev_replace->srcdev->fs_info, logical)) + return; + + /* + * Duplicate the write operations while the dev-replace procedure is + * running. Since the copying of the old disk to the new disk takes + * place at run time while the filesystem is mounted writable, the + * regular write operations to the old disk have to be duplicated to go + * to the new disk as well. + * + * Note that device->missing is handled by the caller, and that the + * write to the old disk is already set up in the stripes array. + */ + for (i = 0; i < num_stripes; i++) { + struct btrfs_io_stripe *old = &bioc->stripes[i]; + struct btrfs_io_stripe *new = &bioc->stripes[num_stripes + nr_extra_stripes]; + + if (old->dev->devid != srcdev_devid) + continue; + + new->physical = old->physical; + new->dev = dev_replace->tgtdev; + if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) + bioc->replace_stripe_src = i; + nr_extra_stripes++; + } + + /* We can only have at most 2 extra nr_stripes (for DUP). */ + ASSERT(nr_extra_stripes <= 2, "nr_extra_stripes=%d", nr_extra_stripes); + /* + * For GET_READ_MIRRORS, we can only return at most 1 extra stripe for + * replace. + * If we have 2 extra stripes, only choose the one with smaller physical. + */ + if (io_geom->op == BTRFS_MAP_GET_READ_MIRRORS && nr_extra_stripes == 2) { + struct btrfs_io_stripe *first = &bioc->stripes[num_stripes]; + struct btrfs_io_stripe *second = &bioc->stripes[num_stripes + 1]; + + /* Only DUP can have two extra stripes. */ + ASSERT(bioc->map_type & BTRFS_BLOCK_GROUP_DUP, + "map_type=%llu", bioc->map_type); + + /* + * Swap the last stripe stripes and reduce @nr_extra_stripes. + * The extra stripe would still be there, but won't be accessed. + */ + if (first->physical > second->physical) { + swap(second->physical, first->physical); + swap(second->dev, first->dev); + nr_extra_stripes--; + } + } + + io_geom->num_stripes = num_stripes + nr_extra_stripes; + io_geom->max_errors = max_errors + nr_extra_stripes; + bioc->replace_nr_stripes = nr_extra_stripes; +} + +static u64 btrfs_max_io_len(struct btrfs_chunk_map *map, u64 offset, + struct btrfs_io_geometry *io_geom) +{ + /* + * Stripe_nr is the stripe where this block falls. stripe_offset is + * the offset of this block in its stripe. + */ + io_geom->stripe_offset = offset & BTRFS_STRIPE_LEN_MASK; + io_geom->stripe_nr = offset >> BTRFS_STRIPE_LEN_SHIFT; + ASSERT(io_geom->stripe_offset < U32_MAX, + "stripe_offset=%llu", io_geom->stripe_offset); + + if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) { + unsigned long full_stripe_len = + btrfs_stripe_nr_to_offset(nr_data_stripes(map)); + + /* + * For full stripe start, we use previously calculated + * @stripe_nr. Align it to nr_data_stripes, then multiply with + * STRIPE_LEN. + * + * By this we can avoid u64 division completely. And we have + * to go rounddown(), not round_down(), as nr_data_stripes is + * not ensured to be power of 2. + */ + io_geom->raid56_full_stripe_start = btrfs_stripe_nr_to_offset( + rounddown(io_geom->stripe_nr, nr_data_stripes(map))); + + ASSERT(io_geom->raid56_full_stripe_start + full_stripe_len > offset, + "raid56_full_stripe_start=%llu full_stripe_len=%lu offset=%llu", + io_geom->raid56_full_stripe_start, full_stripe_len, offset); + ASSERT(io_geom->raid56_full_stripe_start <= offset, + "raid56_full_stripe_start=%llu offset=%llu", + io_geom->raid56_full_stripe_start, offset); + /* + * For writes to RAID56, allow to write a full stripe set, but + * no straddling of stripe sets. + */ + if (io_geom->op == BTRFS_MAP_WRITE) + return full_stripe_len - (offset - io_geom->raid56_full_stripe_start); + } + + /* + * For other RAID types and for RAID56 reads, allow a single stripe (on + * a single disk). + */ + if (map->type & BTRFS_BLOCK_GROUP_STRIPE_MASK) + return BTRFS_STRIPE_LEN - io_geom->stripe_offset; + return U64_MAX; +} + +static int set_io_stripe(struct btrfs_fs_info *fs_info, u64 logical, + u64 *length, struct btrfs_io_stripe *dst, + struct btrfs_chunk_map *map, + struct btrfs_io_geometry *io_geom) +{ + dst->dev = map->stripes[io_geom->stripe_index].dev; + + if (io_geom->op == BTRFS_MAP_READ && io_geom->use_rst) + return btrfs_get_raid_extent_offset(fs_info, logical, length, + map->type, + io_geom->stripe_index, dst); + + dst->physical = map->stripes[io_geom->stripe_index].physical + + io_geom->stripe_offset + + btrfs_stripe_nr_to_offset(io_geom->stripe_nr); + return 0; +} + +static bool is_single_device_io(struct btrfs_fs_info *fs_info, + const struct btrfs_io_stripe *smap, + const struct btrfs_chunk_map *map, + int num_alloc_stripes, + struct btrfs_io_geometry *io_geom) +{ + if (!smap) + return false; + + if (num_alloc_stripes != 1) + return false; + + if (io_geom->use_rst && io_geom->op != BTRFS_MAP_READ) + return false; + + if ((map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) && io_geom->mirror_num > 1) + return false; + + return true; +} + +static void map_blocks_raid0(const struct btrfs_chunk_map *map, + struct btrfs_io_geometry *io_geom) +{ + io_geom->stripe_index = io_geom->stripe_nr % map->num_stripes; + io_geom->stripe_nr /= map->num_stripes; + if (io_geom->op == BTRFS_MAP_READ) + io_geom->mirror_num = 1; +} + +static void map_blocks_raid1(struct btrfs_fs_info *fs_info, + struct btrfs_chunk_map *map, + struct btrfs_io_geometry *io_geom, + bool dev_replace_is_ongoing) +{ + if (io_geom->op != BTRFS_MAP_READ) { + io_geom->num_stripes = map->num_stripes; + return; + } + + if (io_geom->mirror_num) { + io_geom->stripe_index = io_geom->mirror_num - 1; + return; + } + + io_geom->stripe_index = find_live_mirror(fs_info, map, 0, + dev_replace_is_ongoing); + io_geom->mirror_num = io_geom->stripe_index + 1; +} + +static void map_blocks_dup(const struct btrfs_chunk_map *map, + struct btrfs_io_geometry *io_geom) +{ + if (io_geom->op != BTRFS_MAP_READ) { + io_geom->num_stripes = map->num_stripes; + return; + } + + if (io_geom->mirror_num) { + io_geom->stripe_index = io_geom->mirror_num - 1; + return; + } + + io_geom->mirror_num = 1; +} + +static void map_blocks_raid10(struct btrfs_fs_info *fs_info, + struct btrfs_chunk_map *map, + struct btrfs_io_geometry *io_geom, + bool dev_replace_is_ongoing) +{ + u32 factor = map->num_stripes / map->sub_stripes; + int old_stripe_index; + + io_geom->stripe_index = (io_geom->stripe_nr % factor) * map->sub_stripes; + io_geom->stripe_nr /= factor; + + if (io_geom->op != BTRFS_MAP_READ) { + io_geom->num_stripes = map->sub_stripes; + return; + } + + if (io_geom->mirror_num) { + io_geom->stripe_index += io_geom->mirror_num - 1; + return; + } + + old_stripe_index = io_geom->stripe_index; + io_geom->stripe_index = find_live_mirror(fs_info, map, + io_geom->stripe_index, + dev_replace_is_ongoing); + io_geom->mirror_num = io_geom->stripe_index - old_stripe_index + 1; +} + +static void map_blocks_raid56_write(struct btrfs_chunk_map *map, + struct btrfs_io_geometry *io_geom, + u64 logical, u64 *length) +{ + int data_stripes = nr_data_stripes(map); + + /* + * Needs full stripe mapping. + * + * Push stripe_nr back to the start of the full stripe For those cases + * needing a full stripe, @stripe_nr is the full stripe number. + * + * Originally we go raid56_full_stripe_start / full_stripe_len, but + * that can be expensive. Here we just divide @stripe_nr with + * @data_stripes. + */ + io_geom->stripe_nr /= data_stripes; + + /* RAID[56] write or recovery. Return all stripes */ + io_geom->num_stripes = map->num_stripes; + io_geom->max_errors = btrfs_chunk_max_errors(map); + + /* Return the length to the full stripe end. */ + *length = min(logical + *length, + io_geom->raid56_full_stripe_start + map->start + + btrfs_stripe_nr_to_offset(data_stripes)) - + logical; + io_geom->stripe_index = 0; + io_geom->stripe_offset = 0; +} + +static void map_blocks_raid56_read(struct btrfs_chunk_map *map, + struct btrfs_io_geometry *io_geom) +{ + int data_stripes = nr_data_stripes(map); + + ASSERT(io_geom->mirror_num <= 1, "mirror_num=%d", io_geom->mirror_num); + /* Just grab the data stripe directly. */ + io_geom->stripe_index = io_geom->stripe_nr % data_stripes; + io_geom->stripe_nr /= data_stripes; + + /* We distribute the parity blocks across stripes. */ + io_geom->stripe_index = + (io_geom->stripe_nr + io_geom->stripe_index) % map->num_stripes; + + if (io_geom->op == BTRFS_MAP_READ && io_geom->mirror_num < 1) + io_geom->mirror_num = 1; +} + +static void map_blocks_single(const struct btrfs_chunk_map *map, + struct btrfs_io_geometry *io_geom) +{ + io_geom->stripe_index = io_geom->stripe_nr % map->num_stripes; + io_geom->stripe_nr /= map->num_stripes; + io_geom->mirror_num = io_geom->stripe_index + 1; +} + +/* + * Map one logical range to one or more physical ranges. + * + * @length: (Mandatory) mapped length of this run. + * One logical range can be split into different segments + * due to factors like zones and RAID0/5/6/10 stripe + * boundaries. + * + * @bioc_ret: (Mandatory) returned btrfs_io_context structure. + * which has one or more physical ranges (btrfs_io_stripe) + * recorded inside. + * Caller should call btrfs_put_bioc() to free it after use. + * + * @smap: (Optional) single physical range optimization. + * If the map request can be fulfilled by one single + * physical range, and this is parameter is not NULL, + * then @bioc_ret would be NULL, and @smap would be + * updated. + * + * @mirror_num_ret: (Mandatory) returned mirror number if the original + * value is 0. + * + * Mirror number 0 means to choose any live mirrors. + * + * For non-RAID56 profiles, non-zero mirror_num means + * the Nth mirror. (e.g. mirror_num 1 means the first + * copy). + * + * For RAID56 profile, mirror 1 means rebuild from P and + * the remaining data stripes. + * + * For RAID6 profile, mirror > 2 means mark another + * data/P stripe error and rebuild from the remaining + * stripes.. + */ +int btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op, + u64 logical, u64 *length, + struct btrfs_io_context **bioc_ret, + struct btrfs_io_stripe *smap, int *mirror_num_ret) +{ + struct btrfs_chunk_map *map; + struct btrfs_io_geometry io_geom = { 0 }; + u64 map_offset; + int ret = 0; + int num_copies; + struct btrfs_io_context *bioc = NULL; + struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; + bool dev_replace_is_ongoing = false; + u16 num_alloc_stripes; + u64 max_len; + + ASSERT(bioc_ret); + + io_geom.mirror_num = (mirror_num_ret ? *mirror_num_ret : 0); + io_geom.num_stripes = 1; + io_geom.stripe_index = 0; + io_geom.op = op; + + map = btrfs_get_chunk_map(fs_info, logical, *length); + if (IS_ERR(map)) + return PTR_ERR(map); + + if (map->type & BTRFS_BLOCK_GROUP_REMAPPED) { + u64 new_logical = logical; + + ret = btrfs_translate_remap(fs_info, &new_logical, length); + if (ret) + goto out; + + if (new_logical != logical) { + btrfs_free_chunk_map(map); + + map = btrfs_get_chunk_map(fs_info, new_logical, *length); + if (IS_ERR(map)) + return PTR_ERR(map); + + logical = new_logical; + } + } + + num_copies = btrfs_chunk_map_num_copies(map); + if (io_geom.mirror_num > num_copies) { + ret = -EINVAL; + goto out; + } + + map_offset = logical - map->start; + io_geom.raid56_full_stripe_start = (u64)-1; + max_len = btrfs_max_io_len(map, map_offset, &io_geom); + *length = min_t(u64, map->chunk_len - map_offset, max_len); + io_geom.use_rst = btrfs_need_stripe_tree_update(fs_info, map->type); + + if (dev_replace->replace_task != current) + down_read(&dev_replace->rwsem); + + dev_replace_is_ongoing = btrfs_dev_replace_is_ongoing(dev_replace); + /* + * Hold the semaphore for read during the whole operation, write is + * requested at commit time but must wait. + */ + if (!dev_replace_is_ongoing && dev_replace->replace_task != current) + up_read(&dev_replace->rwsem); + + switch (map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) { + case BTRFS_BLOCK_GROUP_RAID0: + map_blocks_raid0(map, &io_geom); + break; + case BTRFS_BLOCK_GROUP_RAID1: + case BTRFS_BLOCK_GROUP_RAID1C3: + case BTRFS_BLOCK_GROUP_RAID1C4: + map_blocks_raid1(fs_info, map, &io_geom, dev_replace_is_ongoing); + break; + case BTRFS_BLOCK_GROUP_DUP: + map_blocks_dup(map, &io_geom); + break; + case BTRFS_BLOCK_GROUP_RAID10: + map_blocks_raid10(fs_info, map, &io_geom, dev_replace_is_ongoing); + break; + case BTRFS_BLOCK_GROUP_RAID5: + case BTRFS_BLOCK_GROUP_RAID6: + if (op != BTRFS_MAP_READ || io_geom.mirror_num > 1) + map_blocks_raid56_write(map, &io_geom, logical, length); + else + map_blocks_raid56_read(map, &io_geom); + break; + default: + /* + * After this, stripe_nr is the number of stripes on this + * device we have to walk to find the data, and stripe_index is + * the number of our device in the stripe array + */ + map_blocks_single(map, &io_geom); + break; + } + if (io_geom.stripe_index >= map->num_stripes) { + btrfs_crit(fs_info, + "stripe index math went horribly wrong, got stripe_index=%u, num_stripes=%u", + io_geom.stripe_index, map->num_stripes); + ret = -EINVAL; + goto out; + } + + num_alloc_stripes = io_geom.num_stripes; + if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL && + op != BTRFS_MAP_READ) + /* + * For replace case, we need to add extra stripes for extra + * duplicated stripes. + * + * For both WRITE and GET_READ_MIRRORS, we may have at most + * 2 more stripes (DUP types, otherwise 1). + */ + num_alloc_stripes += 2; + + /* + * If this I/O maps to a single device, try to return the device and + * physical block information on the stack instead of allocating an + * I/O context structure. + */ + if (is_single_device_io(fs_info, smap, map, num_alloc_stripes, &io_geom)) { + ret = set_io_stripe(fs_info, logical, length, smap, map, &io_geom); + if (mirror_num_ret) + *mirror_num_ret = io_geom.mirror_num; + *bioc_ret = NULL; + goto out; + } + + bioc = alloc_btrfs_io_context(fs_info, logical, num_alloc_stripes); + if (!bioc) { + ret = -ENOMEM; + goto out; + } + bioc->map_type = map->type; + bioc->use_rst = io_geom.use_rst; + + /* + * For RAID56 full map, we need to make sure the stripes[] follows the + * rule that data stripes are all ordered, then followed with P and Q + * (if we have). + * + * It's still mostly the same as other profiles, just with extra rotation. + */ + if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK && + (op != BTRFS_MAP_READ || io_geom.mirror_num > 1)) { + /* + * For RAID56 @stripe_nr is already the number of full stripes + * before us, which is also the rotation value (needs to modulo + * with num_stripes). + * + * In this case, we just add @stripe_nr with @i, then do the + * modulo, to reduce one modulo call. + */ + bioc->full_stripe_logical = map->start + + btrfs_stripe_nr_to_offset(io_geom.stripe_nr * + nr_data_stripes(map)); + for (int i = 0; i < io_geom.num_stripes; i++) { + struct btrfs_io_stripe *dst = &bioc->stripes[i]; + u32 stripe_index; + + stripe_index = (i + io_geom.stripe_nr) % io_geom.num_stripes; + dst->dev = map->stripes[stripe_index].dev; + dst->physical = + map->stripes[stripe_index].physical + + io_geom.stripe_offset + + btrfs_stripe_nr_to_offset(io_geom.stripe_nr); + } + } else { + /* + * For all other non-RAID56 profiles, just copy the target + * stripe into the bioc. + */ + for (int i = 0; i < io_geom.num_stripes; i++) { + ret = set_io_stripe(fs_info, logical, length, + &bioc->stripes[i], map, &io_geom); + if (ret < 0) + break; + io_geom.stripe_index++; + } + } + + if (ret) { + *bioc_ret = NULL; + btrfs_put_bioc(bioc); + goto out; + } + + if (op != BTRFS_MAP_READ) + io_geom.max_errors = btrfs_chunk_max_errors(map); + + if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL && + op != BTRFS_MAP_READ) { + handle_ops_on_dev_replace(bioc, dev_replace, logical, &io_geom); + } + + *bioc_ret = bioc; + bioc->num_stripes = io_geom.num_stripes; + bioc->max_errors = io_geom.max_errors; + bioc->mirror_num = io_geom.mirror_num; + +out: + if (dev_replace_is_ongoing && dev_replace->replace_task != current) { + lockdep_assert_held(&dev_replace->rwsem); + /* Unlock and let waiting writers proceed */ + up_read(&dev_replace->rwsem); + } + btrfs_free_chunk_map(map); + return ret; +} + +static bool dev_args_match_fs_devices(const struct btrfs_dev_lookup_args *args, + const struct btrfs_fs_devices *fs_devices) +{ + if (args->fsid == NULL) + return true; + if (memcmp(fs_devices->metadata_uuid, args->fsid, BTRFS_FSID_SIZE) == 0) + return true; + return false; +} + +static bool dev_args_match_device(const struct btrfs_dev_lookup_args *args, + const struct btrfs_device *device) +{ + if (args->devt) + return device->devt == args->devt; + if (args->missing) { + if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state) && + !device->bdev) + return true; + return false; + } + + if (device->devid != args->devid) + return false; + if (args->uuid && memcmp(device->uuid, args->uuid, BTRFS_UUID_SIZE) != 0) + return false; + return true; +} + +/* + * Find a device specified by @devid or @uuid in the list of @fs_devices, or + * return NULL. + * + * If devid and uuid are both specified, the match must be exact, otherwise + * only devid is used. + */ +struct btrfs_device *btrfs_find_device(const struct btrfs_fs_devices *fs_devices, + const struct btrfs_dev_lookup_args *args) +{ + struct btrfs_device *device; + struct btrfs_fs_devices *seed_devs; + + if (dev_args_match_fs_devices(args, fs_devices)) { + list_for_each_entry(device, &fs_devices->devices, dev_list) { + if (dev_args_match_device(args, device)) + return device; + } + } + + list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) { + if (!dev_args_match_fs_devices(args, seed_devs)) + continue; + list_for_each_entry(device, &seed_devs->devices, dev_list) { + if (dev_args_match_device(args, device)) + return device; + } + } + + return NULL; +} + +static struct btrfs_device *add_missing_dev(struct btrfs_fs_devices *fs_devices, + u64 devid, u8 *dev_uuid) +{ + struct btrfs_device *device; + unsigned int nofs_flag; + + /* + * We call this under the chunk_mutex, so we want to use NOFS for this + * allocation, however we don't want to change btrfs_alloc_device() to + * always do NOFS because we use it in a lot of other GFP_KERNEL safe + * places. + */ + + nofs_flag = memalloc_nofs_save(); + device = btrfs_alloc_device(NULL, &devid, dev_uuid, NULL); + memalloc_nofs_restore(nofs_flag); + if (IS_ERR(device)) + return device; + + list_add(&device->dev_list, &fs_devices->devices); + device->fs_devices = fs_devices; + fs_devices->num_devices++; + + set_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state); + fs_devices->missing_devices++; + + return device; +} + +/* + * Allocate new device struct, set up devid and UUID. + * + * @fs_info: used only for generating a new devid, can be NULL if + * devid is provided (i.e. @devid != NULL). + * @devid: a pointer to devid for this device. If NULL a new devid + * is generated. + * @uuid: a pointer to UUID for this device. If NULL a new UUID + * is generated. + * @path: a pointer to device path if available, NULL otherwise. + * + * Return: a pointer to a new &struct btrfs_device on success; ERR_PTR() + * on error. Returned struct is not linked onto any lists and must be + * destroyed with btrfs_free_device. + */ +struct btrfs_device *btrfs_alloc_device(struct btrfs_fs_info *fs_info, + const u64 *devid, const u8 *uuid, + const char *path) +{ + struct btrfs_device *dev; + u64 tmp; + + if (WARN_ON(!devid && !fs_info)) + return ERR_PTR(-EINVAL); + + dev = kzalloc_obj(*dev); + if (!dev) + return ERR_PTR(-ENOMEM); + + INIT_LIST_HEAD(&dev->dev_list); + INIT_LIST_HEAD(&dev->dev_alloc_list); + INIT_LIST_HEAD(&dev->post_commit_list); + + atomic_set(&dev->dev_stats_ccnt, 0); + btrfs_device_data_ordered_init(dev); + btrfs_extent_io_tree_init(fs_info, &dev->alloc_state, IO_TREE_DEVICE_ALLOC_STATE); + + if (devid) + tmp = *devid; + else { + int ret; + + ret = find_next_devid(fs_info, &tmp); + if (ret) { + btrfs_free_device(dev); + return ERR_PTR(ret); + } + } + dev->devid = tmp; + + if (uuid) + memcpy(dev->uuid, uuid, BTRFS_UUID_SIZE); + else + generate_random_uuid(dev->uuid); + + if (path) { + const char *name; + + name = kstrdup(path, GFP_KERNEL); + if (!name) { + btrfs_free_device(dev); + return ERR_PTR(-ENOMEM); + } + rcu_assign_pointer(dev->name, name); + } + + return dev; +} + +static void btrfs_report_missing_device(struct btrfs_fs_info *fs_info, + u64 devid, u8 *uuid, bool error) +{ + if (error) + btrfs_err_rl(fs_info, "devid %llu uuid %pU is missing", + devid, uuid); + else + btrfs_warn_rl(fs_info, "devid %llu uuid %pU is missing", + devid, uuid); +} + +u64 btrfs_calc_stripe_length(const struct btrfs_chunk_map *map) +{ + const int data_stripes = calc_data_stripes(map->type, map->num_stripes); + + return div_u64(map->chunk_len, data_stripes); +} + +#if BITS_PER_LONG == 32 +/* + * Due to page cache limit, metadata beyond BTRFS_32BIT_MAX_FILE_SIZE + * can't be accessed on 32bit systems. + * + * This function do mount time check to reject the fs if it already has + * metadata chunk beyond that limit. + */ +static int check_32bit_meta_chunk(struct btrfs_fs_info *fs_info, + u64 logical, u64 length, u64 type) +{ + if (!(type & BTRFS_BLOCK_GROUP_METADATA)) + return 0; + + if (logical + length < MAX_LFS_FILESIZE) + return 0; + + btrfs_err_32bit_limit(fs_info); + return -EOVERFLOW; +} + +/* + * This is to give early warning for any metadata chunk reaching + * BTRFS_32BIT_EARLY_WARN_THRESHOLD. + * Although we can still access the metadata, it's not going to be possible + * once the limit is reached. + */ +static void warn_32bit_meta_chunk(struct btrfs_fs_info *fs_info, + u64 logical, u64 length, u64 type) +{ + if (!(type & BTRFS_BLOCK_GROUP_METADATA)) + return; + + if (logical + length < BTRFS_32BIT_EARLY_WARN_THRESHOLD) + return; + + btrfs_warn_32bit_limit(fs_info); +} +#endif + +static struct btrfs_device *handle_missing_device(struct btrfs_fs_info *fs_info, + u64 devid, u8 *uuid) +{ + struct btrfs_device *dev; + + if (!btrfs_test_opt(fs_info, DEGRADED)) { + btrfs_report_missing_device(fs_info, devid, uuid, true); + return ERR_PTR(-ENOENT); + } + + dev = add_missing_dev(fs_info->fs_devices, devid, uuid); + if (IS_ERR(dev)) { + btrfs_err(fs_info, "failed to init missing device %llu: %ld", + devid, PTR_ERR(dev)); + return dev; + } + btrfs_report_missing_device(fs_info, devid, uuid, false); + + return dev; +} + +static int read_one_chunk(struct btrfs_key *key, struct extent_buffer *leaf, + struct btrfs_chunk *chunk) +{ + BTRFS_DEV_LOOKUP_ARGS(args); + struct btrfs_fs_info *fs_info = leaf->fs_info; + struct btrfs_chunk_map *map; + u64 logical; + u64 length; + u64 devid; + u64 type; + u8 uuid[BTRFS_UUID_SIZE]; + int index; + int num_stripes; + int ret; + int i; + + logical = key->offset; + length = btrfs_chunk_length(leaf, chunk); + type = btrfs_chunk_type(leaf, chunk); + index = btrfs_bg_flags_to_raid_index(type); + num_stripes = btrfs_chunk_num_stripes(leaf, chunk); + +#if BITS_PER_LONG == 32 + ret = check_32bit_meta_chunk(fs_info, logical, length, type); + if (ret < 0) + return ret; + warn_32bit_meta_chunk(fs_info, logical, length, type); +#endif + + map = btrfs_find_chunk_map(fs_info, logical, 1); + + /* already mapped? */ + if (map && map->start <= logical && map->start + map->chunk_len > logical) { + btrfs_free_chunk_map(map); + return 0; + } else if (map) { + btrfs_free_chunk_map(map); + } + + map = btrfs_alloc_chunk_map(num_stripes, GFP_NOFS); + if (!map) + return -ENOMEM; + + map->start = logical; + map->chunk_len = length; + map->num_stripes = num_stripes; + map->on_disk_type = type; + /* + * We can't use the sub_stripes value, as for profiles other than + * RAID10, they may have 0 as sub_stripes for filesystems created by + * older mkfs (sub_stripes = btrfs_raid_array[index].sub_stripes; + map->verified_stripes = 0; + set_real_chunk_type(map); + + if (num_stripes > 0) + map->stripe_size = btrfs_calc_stripe_length(map); + else + map->stripe_size = 0; + + for (i = 0; i < num_stripes; i++) { + map->stripes[i].physical = + btrfs_stripe_offset_nr(leaf, chunk, i); + devid = btrfs_stripe_devid_nr(leaf, chunk, i); + args.devid = devid; + read_extent_buffer(leaf, uuid, (unsigned long) + btrfs_stripe_dev_uuid_nr(chunk, i), + BTRFS_UUID_SIZE); + args.uuid = uuid; + map->stripes[i].dev = btrfs_find_device(fs_info->fs_devices, &args); + if (!map->stripes[i].dev) { + map->stripes[i].dev = handle_missing_device(fs_info, + devid, uuid); + if (IS_ERR(map->stripes[i].dev)) { + ret = PTR_ERR(map->stripes[i].dev); + btrfs_free_chunk_map(map); + return ret; + } + } + + set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, + &(map->stripes[i].dev->dev_state)); + } + + ret = btrfs_add_chunk_map(fs_info, map); + if (ret < 0) { + btrfs_err(fs_info, + "failed to add chunk map, start=%llu len=%llu: %d", + map->start, map->chunk_len, ret); + btrfs_free_chunk_map(map); + } + + return ret; +} + +static void fill_device_from_item(struct extent_buffer *leaf, + struct btrfs_dev_item *dev_item, + struct btrfs_device *device) +{ + unsigned long ptr; + + device->devid = btrfs_device_id(leaf, dev_item); + device->disk_total_bytes = btrfs_device_total_bytes(leaf, dev_item); + device->total_bytes = device->disk_total_bytes; + device->commit_total_bytes = device->disk_total_bytes; + device->bytes_used = btrfs_device_bytes_used(leaf, dev_item); + device->commit_bytes_used = device->bytes_used; + device->type = btrfs_device_type(leaf, dev_item); + device->io_align = btrfs_device_io_align(leaf, dev_item); + device->io_width = btrfs_device_io_width(leaf, dev_item); + device->sector_size = btrfs_device_sector_size(leaf, dev_item); + WARN_ON(device->devid == BTRFS_DEV_REPLACE_DEVID); + clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state); + + ptr = btrfs_device_uuid(dev_item); + read_extent_buffer(leaf, device->uuid, ptr, BTRFS_UUID_SIZE); +} + +static struct btrfs_fs_devices *open_seed_devices(struct btrfs_fs_info *fs_info, + u8 *fsid) +{ + struct btrfs_fs_devices *fs_devices; + int ret; + + lockdep_assert_held(&uuid_mutex); + ASSERT(fsid); + + /* This will match only for multi-device seed fs */ + list_for_each_entry(fs_devices, &fs_info->fs_devices->seed_list, seed_list) + if (!memcmp(fs_devices->fsid, fsid, BTRFS_FSID_SIZE)) + return fs_devices; + + + fs_devices = find_fsid(fsid, NULL); + if (!fs_devices) { + if (!btrfs_test_opt(fs_info, DEGRADED)) { + btrfs_err(fs_info, + "failed to find fsid %pU when attempting to open seed devices", + fsid); + return ERR_PTR(-ENOENT); + } + + fs_devices = alloc_fs_devices(fsid); + if (IS_ERR(fs_devices)) + return fs_devices; + + fs_devices->seeding = true; + fs_devices->opened = 1; + list_add(&fs_devices->seed_list, &fs_info->fs_devices->seed_list); + return fs_devices; + } + + /* + * Upon first call for a seed fs fsid, just create a private copy of the + * respective fs_devices and anchor it at fs_info->fs_devices->seed_list + */ + fs_devices = clone_fs_devices(fs_devices); + if (IS_ERR(fs_devices)) + return fs_devices; + + ret = open_fs_devices(fs_devices, BLK_OPEN_READ, fs_info->sb); + if (ret) { + free_fs_devices(fs_devices); + return ERR_PTR(ret); + } + + if (!fs_devices->seeding) { + close_fs_devices(fs_devices); + free_fs_devices(fs_devices); + return ERR_PTR(-EINVAL); + } + + list_add(&fs_devices->seed_list, &fs_info->fs_devices->seed_list); + + return fs_devices; +} + +static int read_one_dev(struct extent_buffer *leaf, + struct btrfs_dev_item *dev_item) +{ + BTRFS_DEV_LOOKUP_ARGS(args); + struct btrfs_fs_info *fs_info = leaf->fs_info; + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; + struct btrfs_device *device; + u64 devid; + u8 fs_uuid[BTRFS_FSID_SIZE]; + u8 dev_uuid[BTRFS_UUID_SIZE]; + + devid = btrfs_device_id(leaf, dev_item); + args.devid = devid; + read_extent_buffer(leaf, dev_uuid, btrfs_device_uuid(dev_item), + BTRFS_UUID_SIZE); + read_extent_buffer(leaf, fs_uuid, btrfs_device_fsid(dev_item), + BTRFS_FSID_SIZE); + args.uuid = dev_uuid; + args.fsid = fs_uuid; + + if (memcmp(fs_uuid, fs_devices->metadata_uuid, BTRFS_FSID_SIZE)) { + fs_devices = open_seed_devices(fs_info, fs_uuid); + if (IS_ERR(fs_devices)) + return PTR_ERR(fs_devices); + } + + device = btrfs_find_device(fs_info->fs_devices, &args); + if (!device) { + if (!btrfs_test_opt(fs_info, DEGRADED)) { + btrfs_report_missing_device(fs_info, devid, + dev_uuid, true); + return -ENOENT; + } + + device = add_missing_dev(fs_devices, devid, dev_uuid); + if (IS_ERR(device)) { + btrfs_err(fs_info, + "failed to add missing dev %llu: %ld", + devid, PTR_ERR(device)); + return PTR_ERR(device); + } + btrfs_report_missing_device(fs_info, devid, dev_uuid, false); + } else { + if (!device->bdev) { + if (!btrfs_test_opt(fs_info, DEGRADED)) { + btrfs_report_missing_device(fs_info, + devid, dev_uuid, true); + return -ENOENT; + } + btrfs_report_missing_device(fs_info, devid, + dev_uuid, false); + } + + if (!device->bdev && + !test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) { + /* + * this happens when a device that was properly setup + * in the device info lists suddenly goes bad. + * device->bdev is NULL, and so we have to set + * device->missing to one here + */ + device->fs_devices->missing_devices++; + set_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state); + } + + /* Move the device to its own fs_devices */ + if (device->fs_devices != fs_devices) { + ASSERT(test_bit(BTRFS_DEV_STATE_MISSING, + &device->dev_state)); + + list_move(&device->dev_list, &fs_devices->devices); + device->fs_devices->num_devices--; + fs_devices->num_devices++; + + device->fs_devices->missing_devices--; + fs_devices->missing_devices++; + + device->fs_devices = fs_devices; + } + } + + if (device->fs_devices != fs_info->fs_devices) { + BUG_ON(test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)); + if (device->generation != + btrfs_device_generation(leaf, dev_item)) + return -EINVAL; + } + + fill_device_from_item(leaf, dev_item, device); + if (device->bdev) { + u64 max_total_bytes = bdev_nr_bytes(device->bdev); + + if (device->total_bytes > max_total_bytes) { + btrfs_err(fs_info, + "device total_bytes should be at most %llu but found %llu", + max_total_bytes, device->total_bytes); + return -EINVAL; + } + } + set_bit(BTRFS_DEV_STATE_ITEM_FOUND, &device->dev_state); + set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state); + if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) && + !test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) { + device->fs_devices->total_rw_bytes += device->total_bytes; + atomic64_add(device->total_bytes - device->bytes_used, + &fs_info->free_chunk_space); + } + + return 0; +} + +int btrfs_read_sys_array(struct btrfs_fs_info *fs_info) +{ + struct btrfs_super_block *super_copy = fs_info->super_copy; + struct extent_buffer *sb; + u8 *array_ptr; + unsigned long sb_array_offset; + int ret = 0; + u32 array_size; + u32 cur_offset; + struct btrfs_key key; + + ASSERT(BTRFS_SUPER_INFO_SIZE <= fs_info->nodesize); + + /* + * We allocated a dummy extent, just to use extent buffer accessors. + * There will be unused space after BTRFS_SUPER_INFO_SIZE, but + * that's fine, we will not go beyond system chunk array anyway. + */ + sb = alloc_dummy_extent_buffer(fs_info, BTRFS_SUPER_INFO_OFFSET); + if (!sb) + return -ENOMEM; + set_extent_buffer_uptodate(sb); + + write_extent_buffer(sb, super_copy, 0, BTRFS_SUPER_INFO_SIZE); + array_size = btrfs_super_sys_array_size(super_copy); + + array_ptr = super_copy->sys_chunk_array; + sb_array_offset = offsetof(struct btrfs_super_block, sys_chunk_array); + cur_offset = 0; + + while (cur_offset < array_size) { + struct btrfs_chunk *chunk; + struct btrfs_disk_key *disk_key = (struct btrfs_disk_key *)array_ptr; + u32 len = sizeof(*disk_key); + + /* + * The sys_chunk_array has been already verified at super block + * read time. Only do ASSERT()s for basic checks. + */ + ASSERT(cur_offset + len <= array_size); + + btrfs_disk_key_to_cpu(&key, disk_key); + + array_ptr += len; + sb_array_offset += len; + cur_offset += len; + + ASSERT(key.type == BTRFS_CHUNK_ITEM_KEY); + + chunk = (struct btrfs_chunk *)sb_array_offset; + ASSERT(btrfs_chunk_type(sb, chunk) & BTRFS_BLOCK_GROUP_SYSTEM); + + len = btrfs_chunk_item_size(btrfs_chunk_num_stripes(sb, chunk)); + + ASSERT(cur_offset + len <= array_size); + + ret = read_one_chunk(&key, sb, chunk); + if (ret) + break; + + array_ptr += len; + sb_array_offset += len; + cur_offset += len; + } + clear_extent_buffer_uptodate(sb); + free_extent_buffer_stale(sb); + return ret; +} + +/* + * Check if all chunks in the fs are OK for read-write degraded mount + * + * If the @failing_dev is specified, it's accounted as missing. + * + * Return true if all chunks meet the minimal RW mount requirements. + * Return false if any chunk doesn't meet the minimal RW mount requirements. + */ +bool btrfs_check_rw_degradable(struct btrfs_fs_info *fs_info, + struct btrfs_device *failing_dev) +{ + struct btrfs_chunk_map *map; + u64 next_start; + bool ret = true; + + map = btrfs_find_chunk_map(fs_info, 0, U64_MAX); + /* No chunk at all? Return false anyway */ + if (!map) + return false; + + while (map) { + int missing = 0; + int max_tolerated; + int i; + + max_tolerated = + btrfs_get_num_tolerated_disk_barrier_failures( + map->type); + for (i = 0; i < map->num_stripes; i++) { + struct btrfs_device *dev = map->stripes[i].dev; + + if (!dev || !dev->bdev || + test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) || + test_bit(BTRFS_DEV_STATE_FLUSH_FAILED, &dev->dev_state)) + missing++; + else if (failing_dev && failing_dev == dev) + missing++; + } + if (missing > max_tolerated) { + if (!failing_dev) + btrfs_warn(fs_info, + "chunk %llu missing %d devices, max tolerance is %d for writable mount", + map->start, missing, max_tolerated); + btrfs_free_chunk_map(map); + return false; + } + next_start = map->start + map->chunk_len; + btrfs_free_chunk_map(map); + + map = btrfs_find_chunk_map(fs_info, next_start, U64_MAX - next_start); + } + + return ret; +} + +static void readahead_tree_node_children(struct extent_buffer *node) +{ + int i; + const int nr_items = btrfs_header_nritems(node); + + for (i = 0; i < nr_items; i++) + btrfs_readahead_node_child(node, i); +} + +int btrfs_read_chunk_tree(struct btrfs_fs_info *fs_info) +{ + struct btrfs_root *root = fs_info->chunk_root; + BTRFS_PATH_AUTO_FREE(path); + struct extent_buffer *leaf; + struct btrfs_key key; + struct btrfs_key found_key; + int ret; + int slot; + int iter_ret = 0; + u64 total_dev = 0; + u64 last_ra_node = 0; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + /* + * uuid_mutex is needed only if we are mounting a sprout FS + * otherwise we don't need it. + */ + mutex_lock(&uuid_mutex); + + /* + * It is possible for mount and umount to race in such a way that + * we execute this code path, but open_fs_devices failed to clear + * total_rw_bytes. We certainly want it cleared before reading the + * device items, so clear it here. + */ + fs_info->fs_devices->total_rw_bytes = 0; + + /* + * Lockdep complains about possible circular locking dependency between + * a disk's open_mutex (struct gendisk.open_mutex), the rw semaphores + * used for freeze protection of a fs (struct super_block.s_writers), + * which we take when starting a transaction, and extent buffers of the + * chunk tree if we call read_one_dev() while holding a lock on an + * extent buffer of the chunk tree. Since we are mounting the filesystem + * and at this point there can't be any concurrent task modifying the + * chunk tree, to keep it simple, just skip locking on the chunk tree. + */ + ASSERT(!test_bit(BTRFS_FS_OPEN, &fs_info->flags)); + path->skip_locking = true; + + /* + * Read all device items, and then all the chunk items. All + * device items are found before any chunk item (their object id + * is smaller than the lowest possible object id for a chunk + * item - BTRFS_FIRST_CHUNK_TREE_OBJECTID). + */ + key.objectid = BTRFS_DEV_ITEMS_OBJECTID; + key.type = 0; + key.offset = 0; + btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) { + struct extent_buffer *node = path->nodes[1]; + + leaf = path->nodes[0]; + slot = path->slots[0]; + + if (node) { + if (last_ra_node != node->start) { + readahead_tree_node_children(node); + last_ra_node = node->start; + } + } + if (found_key.type == BTRFS_DEV_ITEM_KEY) { + struct btrfs_dev_item *dev_item; + dev_item = btrfs_item_ptr(leaf, slot, + struct btrfs_dev_item); + ret = read_one_dev(leaf, dev_item); + if (ret) + goto error; + total_dev++; + } else if (found_key.type == BTRFS_CHUNK_ITEM_KEY) { + struct btrfs_chunk *chunk; + + /* + * We are only called at mount time, so no need to take + * fs_info->chunk_mutex. Plus, to avoid lockdep warnings, + * we always lock first fs_info->chunk_mutex before + * acquiring any locks on the chunk tree. This is a + * requirement for chunk allocation, see the comment on + * top of btrfs_chunk_alloc() for details. + */ + chunk = btrfs_item_ptr(leaf, slot, struct btrfs_chunk); + ret = read_one_chunk(&found_key, leaf, chunk); + if (ret) + goto error; + } + } + /* Catch error found during iteration */ + if (iter_ret < 0) { + ret = iter_ret; + goto error; + } + + /* + * After loading chunk tree, we've got all device information, + * do another round of validation checks. + */ + if (total_dev != fs_info->fs_devices->total_devices) { + btrfs_warn(fs_info, +"super block num_devices %llu mismatch with DEV_ITEM count %llu, will be repaired on next transaction commit", + btrfs_super_num_devices(fs_info->super_copy), + total_dev); + fs_info->fs_devices->total_devices = total_dev; + btrfs_set_super_num_devices(fs_info->super_copy, total_dev); + } + if (btrfs_super_total_bytes(fs_info->super_copy) < + fs_info->fs_devices->total_rw_bytes) { + btrfs_err(fs_info, + "super_total_bytes %llu mismatch with fs_devices total_rw_bytes %llu", + btrfs_super_total_bytes(fs_info->super_copy), + fs_info->fs_devices->total_rw_bytes); + ret = -EINVAL; + goto error; + } + ret = 0; +error: + mutex_unlock(&uuid_mutex); + return ret; +} + +int btrfs_init_devices_late(struct btrfs_fs_info *fs_info) +{ + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs; + struct btrfs_device *device; + int ret = 0; + + mutex_lock(&fs_devices->device_list_mutex); + list_for_each_entry(device, &fs_devices->devices, dev_list) + device->fs_info = fs_info; + + list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) { + list_for_each_entry(device, &seed_devs->devices, dev_list) { + device->fs_info = fs_info; + ret = btrfs_get_dev_zone_info(device, false); + if (ret) + break; + } + + seed_devs->fs_info = fs_info; + } + mutex_unlock(&fs_devices->device_list_mutex); + + return ret; +} + +static u64 btrfs_dev_stats_value(const struct extent_buffer *eb, + const struct btrfs_dev_stats_item *ptr, + int index) +{ + u64 val; + + read_extent_buffer(eb, &val, + offsetof(struct btrfs_dev_stats_item, values) + + ((unsigned long)ptr) + (index * sizeof(u64)), + sizeof(val)); + return val; +} + +static void btrfs_set_dev_stats_value(struct extent_buffer *eb, + struct btrfs_dev_stats_item *ptr, + int index, u64 val) +{ + write_extent_buffer(eb, &val, + offsetof(struct btrfs_dev_stats_item, values) + + ((unsigned long)ptr) + (index * sizeof(u64)), + sizeof(val)); +} + +static int btrfs_device_init_dev_stats(struct btrfs_device *device, + struct btrfs_path *path) +{ + struct btrfs_dev_stats_item *ptr; + struct extent_buffer *eb; + struct btrfs_key key; + int item_size; + int i, ret, slot; + + if (!device->fs_info->dev_root) + return 0; + + key.objectid = BTRFS_DEV_STATS_OBJECTID; + key.type = BTRFS_PERSISTENT_ITEM_KEY; + key.offset = device->devid; + ret = btrfs_search_slot(NULL, device->fs_info->dev_root, &key, path, 0, 0); + if (ret) { + for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) + btrfs_dev_stat_set(device, i, 0); + device->dev_stats_valid = 1; + btrfs_release_path(path); + return ret < 0 ? ret : 0; + } + slot = path->slots[0]; + eb = path->nodes[0]; + item_size = btrfs_item_size(eb, slot); + + ptr = btrfs_item_ptr(eb, slot, struct btrfs_dev_stats_item); + + for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) { + if (item_size >= (1 + i) * sizeof(__le64)) + atomic_set(device->dev_stat_values + i, + btrfs_dev_stats_value(eb, ptr, i)); + else + btrfs_dev_stat_set(device, i, 0); + } + + device->dev_stats_valid = 1; + btrfs_dev_stat_print_on_load(device); + btrfs_release_path(path); + + return 0; +} + +int btrfs_init_dev_stats(struct btrfs_fs_info *fs_info) +{ + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs; + struct btrfs_device *device; + BTRFS_PATH_AUTO_FREE(path); + int ret = 0; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + mutex_lock(&fs_devices->device_list_mutex); + list_for_each_entry(device, &fs_devices->devices, dev_list) { + ret = btrfs_device_init_dev_stats(device, path); + if (ret) + goto out; + } + list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) { + list_for_each_entry(device, &seed_devs->devices, dev_list) { + ret = btrfs_device_init_dev_stats(device, path); + if (ret) + goto out; + } + } +out: + mutex_unlock(&fs_devices->device_list_mutex); + return ret; +} + +int btrfs_init_writeback_bio_size(struct btrfs_fs_info *fs_info) +{ + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; + struct btrfs_device *device; + u32 writeback_bio_size = fs_info->sectorsize; + + mutex_lock(&fs_devices->device_list_mutex); + /* + * Let's take maximum over optimal request sizes for all devices. For + * RAID profiles writeback will submit stripe (64k) sized bios anyway + * so our value doesn't matter and for simple profiles this is a good + * approximation of sensible IO chunking. + */ + list_for_each_entry(device, &fs_devices->devices, dev_list) { + struct request_queue *queue; + unsigned int io_opt; + + if (!device->bdev || test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) + continue; + queue = bdev_get_queue(device->bdev); + io_opt = queue_io_opt(queue) ? : + queue_max_sectors(queue) << SECTOR_SHIFT; + writeback_bio_size = max(writeback_bio_size, io_opt); + } + mutex_unlock(&fs_devices->device_list_mutex); + + fs_info->writeback_bio_size = writeback_bio_size; + + return 0; +} + +static int update_dev_stat_item(struct btrfs_trans_handle *trans, + struct btrfs_device *device) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_root *dev_root = fs_info->dev_root; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + struct extent_buffer *eb; + struct btrfs_dev_stats_item *ptr; + int ret; + int i; + + key.objectid = BTRFS_DEV_STATS_OBJECTID; + key.type = BTRFS_PERSISTENT_ITEM_KEY; + key.offset = device->devid; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + ret = btrfs_search_slot(trans, dev_root, &key, path, -1, 1); + if (ret < 0) { + btrfs_warn(fs_info, + "error %d while searching for dev_stats item for device %s", + ret, btrfs_dev_name(device)); + return ret; + } + + if (ret == 0 && + btrfs_item_size(path->nodes[0], path->slots[0]) < sizeof(*ptr)) { + /* need to delete old one and insert a new one */ + ret = btrfs_del_item(trans, dev_root, path); + if (ret != 0) { + btrfs_warn(fs_info, + "delete too small dev_stats item for device %s failed %d", + btrfs_dev_name(device), ret); + return ret; + } + ret = 1; + } + + if (ret == 1) { + /* need to insert a new item */ + btrfs_release_path(path); + ret = btrfs_insert_empty_item(trans, dev_root, path, + &key, sizeof(*ptr)); + if (ret < 0) { + btrfs_warn(fs_info, + "insert dev_stats item for device %s failed %d", + btrfs_dev_name(device), ret); + return ret; + } + } + + eb = path->nodes[0]; + ptr = btrfs_item_ptr(eb, path->slots[0], struct btrfs_dev_stats_item); + for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) + btrfs_set_dev_stats_value(eb, ptr, i, + btrfs_dev_stat_read(device, i)); + return ret; +} + +/* + * called from commit_transaction. Writes all changed device stats to disk. + */ +int btrfs_run_dev_stats(struct btrfs_trans_handle *trans) +{ + struct btrfs_fs_info *fs_info = trans->fs_info; + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; + struct btrfs_device *device; + int stats_cnt; + int ret = 0; + bool need_update_dev_stats = false; + + /* + * Do an initial pass using RCU to see if we need to update any dev + * stats item. This is to avoid taking the device_list_mutex which is + * acquired by the fitrim operation and can take a while since it does + * discard operations while holding that mutex. Most of the time, if + * we are on a healthy filesystem, we don't have new stat updates, so + * this avoids blocking on that mutex, which is specially important + * because we are called during the critical section of a transaction + * commit, therefore blocking new transactions from starting while + * discard is running. + * + * Also note that adding/removing devices also requires starting a + * transaction, and since we are called from the critical section of a + * transaction commit, no one can be concurrently adding or removing a + * device. + */ + rcu_read_lock(); + list_for_each_entry_rcu(device, &fs_devices->devices, dev_list) { + if (device->dev_stats_valid && + atomic_read(&device->dev_stats_ccnt) != 0) { + need_update_dev_stats = true; + break; + } + } + rcu_read_unlock(); + + if (!need_update_dev_stats) + return 0; + + mutex_lock(&fs_devices->device_list_mutex); + list_for_each_entry(device, &fs_devices->devices, dev_list) { + stats_cnt = atomic_read(&device->dev_stats_ccnt); + if (!device->dev_stats_valid || stats_cnt == 0) + continue; + + + /* + * There is a LOAD-LOAD control dependency between the value of + * dev_stats_ccnt and updating the on-disk values which requires + * reading the in-memory counters. Such control dependencies + * require explicit read memory barriers. + * + * This memory barriers pairs with smp_mb__before_atomic in + * btrfs_dev_stat_inc/btrfs_dev_stat_set and with the full + * barrier implied by atomic_xchg in + * btrfs_dev_stats_read_and_reset + */ + smp_rmb(); + + ret = update_dev_stat_item(trans, device); + if (ret) + break; + atomic_sub(stats_cnt, &device->dev_stats_ccnt); + } + mutex_unlock(&fs_devices->device_list_mutex); + + return ret; +} + +void btrfs_dev_stat_inc_and_print(struct btrfs_device *dev, int index) +{ + btrfs_dev_stat_inc(dev, index); + + if (!dev->dev_stats_valid) + return; + btrfs_err_rl(dev->fs_info, + "bdev %s errs: wr %u, rd %u, flush %u, corrupt %u, gen %u", + btrfs_dev_name(dev), + btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_WRITE_ERRS), + btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_READ_ERRS), + btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_FLUSH_ERRS), + btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS), + btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_GENERATION_ERRS)); +} + +static void btrfs_dev_stat_print_on_load(struct btrfs_device *dev) +{ + int i; + + for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) + if (btrfs_dev_stat_read(dev, i) != 0) + break; + if (i == BTRFS_DEV_STAT_VALUES_MAX) + return; /* all values == 0, suppress message */ + + btrfs_info(dev->fs_info, + "bdev %s errs: wr %u, rd %u, flush %u, corrupt %u, gen %u", + btrfs_dev_name(dev), + btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_WRITE_ERRS), + btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_READ_ERRS), + btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_FLUSH_ERRS), + btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS), + btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_GENERATION_ERRS)); +} + +int btrfs_get_dev_stats(struct btrfs_fs_info *fs_info, + struct btrfs_ioctl_get_dev_stats *stats) +{ + BTRFS_DEV_LOOKUP_ARGS(args); + struct btrfs_device *dev; + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; + int i; + + mutex_lock(&fs_devices->device_list_mutex); + args.devid = stats->devid; + dev = btrfs_find_device(fs_info->fs_devices, &args); + mutex_unlock(&fs_devices->device_list_mutex); + + if (!dev) { + btrfs_warn(fs_info, "get dev_stats failed, device not found"); + return -ENODEV; + } else if (!dev->dev_stats_valid) { + btrfs_warn(fs_info, "get dev_stats failed, not yet valid"); + return -ENODEV; + } else if (stats->flags & BTRFS_DEV_STATS_RESET) { + for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) { + if (stats->nr_items > i) + stats->values[i] = + btrfs_dev_stat_read_and_reset(dev, i); + else + btrfs_dev_stat_set(dev, i, 0); + } + btrfs_info(fs_info, "device stats zeroed by %s (%d)", + current->comm, task_pid_nr(current)); + } else { + for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) + if (stats->nr_items > i) + stats->values[i] = btrfs_dev_stat_read(dev, i); + } + if (stats->nr_items > BTRFS_DEV_STAT_VALUES_MAX) + stats->nr_items = BTRFS_DEV_STAT_VALUES_MAX; + return 0; +} + +/* + * Update the size and bytes used for each device where it changed. This is + * delayed since we would otherwise get errors while writing out the + * superblocks. + * + * Must be invoked during transaction commit. + */ +void btrfs_commit_device_sizes(struct btrfs_transaction *trans) +{ + struct btrfs_device *curr, *next; + + ASSERT(trans->state == TRANS_STATE_COMMIT_DOING, "state=%d" , trans->state); + + if (list_empty(&trans->dev_update_list)) + return; + + /* + * We don't need the device_list_mutex here. This list is owned by the + * transaction and the transaction must complete before the device is + * released. + */ + mutex_lock(&trans->fs_info->chunk_mutex); + list_for_each_entry_safe(curr, next, &trans->dev_update_list, + post_commit_list) { + list_del_init(&curr->post_commit_list); + curr->commit_total_bytes = curr->disk_total_bytes; + curr->commit_bytes_used = curr->bytes_used; + } + mutex_unlock(&trans->fs_info->chunk_mutex); +} + +/* + * Multiplicity factor for simple profiles: DUP, RAID1-like and RAID10. + */ +int btrfs_bg_type_to_factor(u64 flags) +{ + const int index = btrfs_bg_flags_to_raid_index(flags); + + return btrfs_raid_array[index].ncopies; +} + +static int verify_one_dev_extent(struct btrfs_fs_info *fs_info, + u64 chunk_offset, u64 devid, + u64 physical_offset, u64 physical_len) +{ + struct btrfs_dev_lookup_args args = { .devid = devid }; + struct btrfs_chunk_map *map; + struct btrfs_device *dev; + u64 stripe_len; + bool found = false; + int ret = 0; + int i; + + map = btrfs_find_chunk_map(fs_info, chunk_offset, 1); + if (unlikely(!map)) { + btrfs_err(fs_info, +"dev extent physical offset %llu on devid %llu doesn't have corresponding chunk", + physical_offset, devid); + ret = -EUCLEAN; + goto out; + } + + stripe_len = btrfs_calc_stripe_length(map); + if (unlikely(physical_len != stripe_len)) { + btrfs_err(fs_info, +"dev extent physical offset %llu on devid %llu length doesn't match chunk %llu, have %llu expect %llu", + physical_offset, devid, map->start, physical_len, + stripe_len); + ret = -EUCLEAN; + goto out; + } + + /* + * Very old mkfs.btrfs (before v4.15) will not respect the reserved + * space. Although kernel can handle it without problem, better to warn + * the users. + */ + if (physical_offset < BTRFS_DEVICE_RANGE_RESERVED) + btrfs_warn(fs_info, + "devid %llu physical %llu len %llu inside the reserved space", + devid, physical_offset, physical_len); + + for (i = 0; i < map->num_stripes; i++) { + if (unlikely(map->stripes[i].dev->devid == devid && + map->stripes[i].physical == physical_offset)) { + found = true; + if (map->verified_stripes >= map->num_stripes) { + btrfs_err(fs_info, + "too many dev extents for chunk %llu found", + map->start); + ret = -EUCLEAN; + goto out; + } + map->verified_stripes++; + break; + } + } + if (unlikely(!found)) { + btrfs_err(fs_info, + "dev extent physical offset %llu devid %llu has no corresponding chunk", + physical_offset, devid); + ret = -EUCLEAN; + } + + /* Make sure no dev extent is beyond device boundary */ + dev = btrfs_find_device(fs_info->fs_devices, &args); + if (unlikely(!dev)) { + btrfs_err(fs_info, "failed to find devid %llu", devid); + ret = -EUCLEAN; + goto out; + } + + if (unlikely(physical_offset + physical_len > dev->disk_total_bytes)) { + btrfs_err(fs_info, +"dev extent devid %llu physical offset %llu len %llu is beyond device boundary %llu", + devid, physical_offset, physical_len, + dev->disk_total_bytes); + ret = -EUCLEAN; + goto out; + } + + if (dev->zone_info) { + u64 zone_size = dev->zone_info->zone_size; + + if (unlikely(!IS_ALIGNED(physical_offset, zone_size) || + !IS_ALIGNED(physical_len, zone_size))) { + btrfs_err(fs_info, +"zoned: dev extent devid %llu physical offset %llu len %llu is not aligned to device zone", + devid, physical_offset, physical_len); + ret = -EUCLEAN; + goto out; + } + } + +out: + btrfs_free_chunk_map(map); + return ret; +} + +static int verify_chunk_dev_extent_mapping(struct btrfs_fs_info *fs_info) +{ + struct rb_node *node; + int ret = 0; + + read_lock(&fs_info->mapping_tree_lock); + for (node = rb_first_cached(&fs_info->mapping_tree); node; node = rb_next(node)) { + struct btrfs_chunk_map *map; + + map = rb_entry(node, struct btrfs_chunk_map, rb_node); + if (unlikely(map->num_stripes != map->verified_stripes)) { + btrfs_err(fs_info, + "chunk %llu has missing dev extent, have %d expect %d", + map->start, map->verified_stripes, map->num_stripes); + ret = -EUCLEAN; + goto out; + } + } +out: + read_unlock(&fs_info->mapping_tree_lock); + return ret; +} + +/* + * Ensure that all dev extents are mapped to correct chunk, otherwise + * later chunk allocation/free would cause unexpected behavior. + * + * NOTE: This will iterate through the whole device tree, which should be of + * the same size level as the chunk tree. This slightly increases mount time. + */ +int btrfs_verify_dev_extents(struct btrfs_fs_info *fs_info) +{ + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_root *root = fs_info->dev_root; + struct btrfs_key key; + u64 prev_devid = 0; + u64 prev_dev_ext_end = 0; + int ret = 0; + + /* + * We don't have a dev_root because we mounted with ignorebadroots and + * failed to load the root, so we want to skip the verification in this + * case for sure. + * + * However if the dev root is fine, but the tree itself is corrupted + * we'd still fail to mount. This verification is only to make sure + * writes can happen safely, so instead just bypass this check + * completely in the case of IGNOREBADROOTS. + */ + if (btrfs_test_opt(fs_info, IGNOREBADROOTS)) + return 0; + + key.objectid = 1; + key.type = BTRFS_DEV_EXTENT_KEY; + key.offset = 0; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + path->reada = READA_FORWARD_ALWAYS; + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + return ret; + + if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) { + ret = btrfs_next_leaf(root, path); + if (ret < 0) + return ret; + /* No dev extents at all? Not good */ + if (unlikely(ret > 0)) + return -EUCLEAN; + } + while (1) { + struct extent_buffer *leaf = path->nodes[0]; + struct btrfs_dev_extent *dext; + int slot = path->slots[0]; + u64 chunk_offset; + u64 physical_offset; + u64 physical_len; + u64 devid; + + btrfs_item_key_to_cpu(leaf, &key, slot); + if (key.type != BTRFS_DEV_EXTENT_KEY) + break; + devid = key.objectid; + physical_offset = key.offset; + + dext = btrfs_item_ptr(leaf, slot, struct btrfs_dev_extent); + chunk_offset = btrfs_dev_extent_chunk_offset(leaf, dext); + physical_len = btrfs_dev_extent_length(leaf, dext); + + /* Check if this dev extent overlaps with the previous one */ + if (unlikely(devid == prev_devid && physical_offset < prev_dev_ext_end)) { + btrfs_err(fs_info, +"dev extent devid %llu physical offset %llu overlap with previous dev extent end %llu", + devid, physical_offset, prev_dev_ext_end); + return -EUCLEAN; + } + + ret = verify_one_dev_extent(fs_info, chunk_offset, devid, + physical_offset, physical_len); + if (ret < 0) + return ret; + prev_devid = devid; + prev_dev_ext_end = physical_offset + physical_len; + + ret = btrfs_next_item(root, path); + if (ret < 0) + return ret; + if (ret > 0) { + ret = 0; + break; + } + } + + /* Ensure all chunks have corresponding dev extents */ + ret = verify_chunk_dev_extent_mapping(fs_info); + if (ret < 0) + return ret; + + mutex_lock(&fs_info->chunk_mutex); + btrfs_update_per_profile_avail(fs_info); + mutex_unlock(&fs_info->chunk_mutex); + return 0; +} + +/* + * Ensure that all devices registered in the fs have their device items in the + * chunk tree. + * + * Return true if unexpected device is found. + * Return false otherwise. + */ +bool btrfs_verify_dev_items(const struct btrfs_fs_info *fs_info) +{ + struct btrfs_fs_devices *seed_devs; + struct btrfs_device *dev; + bool ret = false; + + mutex_lock(&uuid_mutex); + list_for_each_entry(dev, &fs_info->fs_devices->devices, dev_list) { + /* + * Replace target dev item (devid 0) is not inserted into chunk tree. + * So skip the DEV_STATE_ITEM check. + */ + if (dev->devid != BTRFS_DEV_REPLACE_DEVID && + !test_bit(BTRFS_DEV_STATE_ITEM_FOUND, &dev->dev_state)) { + btrfs_err(fs_info, + "devid %llu path %s is registered but not found in chunk tree", + dev->devid, btrfs_dev_name(dev)); + ret = true; + } + } + list_for_each_entry(seed_devs, &fs_info->fs_devices->seed_list, seed_list) { + list_for_each_entry(dev, &seed_devs->devices, dev_list) { + if (!test_bit(BTRFS_DEV_STATE_ITEM_FOUND, &dev->dev_state)) { + btrfs_err(fs_info, + "devid %llu path %s is registered but not found in chunk tree", + dev->devid, btrfs_dev_name(dev)); + ret = true; + } + } + } + mutex_unlock(&uuid_mutex); + if (ret) + btrfs_err(fs_info, +"remove the above devices or use 'btrfs device scan --forget ' to unregister them before mount"); + return ret; +} + +/* + * Check whether the given block group or device is pinned by any inode being + * used as a swapfile. + */ +bool btrfs_pinned_by_swapfile(struct btrfs_fs_info *fs_info, void *ptr) +{ + struct btrfs_swapfile_pin *sp; + struct rb_node *node; + + spin_lock(&fs_info->swapfile_pins_lock); + node = fs_info->swapfile_pins.rb_node; + while (node) { + sp = rb_entry(node, struct btrfs_swapfile_pin, node); + if (ptr < sp->ptr) + node = node->rb_left; + else if (ptr > sp->ptr) + node = node->rb_right; + else + break; + } + spin_unlock(&fs_info->swapfile_pins_lock); + return node != NULL; +} + +static int relocating_repair_kthread(void *data) +{ + struct btrfs_block_group *cache = data; + struct btrfs_fs_info *fs_info = cache->fs_info; + u64 target; + int ret = 0; + + target = cache->start; + btrfs_put_block_group(cache); + + guard(super_write)(fs_info->sb); + + if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_BALANCE)) { + btrfs_info(fs_info, + "zoned: skip relocating block group %llu to repair: EBUSY", + target); + return -EBUSY; + } + + mutex_lock(&fs_info->reclaim_bgs_lock); + + /* Ensure block group still exists */ + cache = btrfs_lookup_block_group(fs_info, target); + if (!cache) + goto out; + + if (!test_bit(BLOCK_GROUP_FLAG_RELOCATING_REPAIR, &cache->runtime_flags)) + goto out; + + ret = btrfs_may_alloc_data_chunk(fs_info, target); + if (ret < 0) + goto out; + + btrfs_info(fs_info, + "zoned: relocating block group %llu to repair IO failure", + target); + ret = btrfs_relocate_chunk(fs_info, target, true); + +out: + if (cache) + btrfs_put_block_group(cache); + mutex_unlock(&fs_info->reclaim_bgs_lock); + btrfs_exclop_finish(fs_info); + + return ret; +} + +bool btrfs_repair_one_zone(struct btrfs_fs_info *fs_info, u64 logical) +{ + struct btrfs_block_group *cache; + + if (!btrfs_is_zoned(fs_info)) + return false; + + /* Do not attempt to repair in degraded state */ + if (btrfs_test_opt(fs_info, DEGRADED)) + return true; + + cache = btrfs_lookup_block_group(fs_info, logical); + if (!cache) + return true; + + if (test_and_set_bit(BLOCK_GROUP_FLAG_RELOCATING_REPAIR, &cache->runtime_flags)) { + btrfs_put_block_group(cache); + return true; + } + + kthread_run(relocating_repair_kthread, cache, + "btrfs-relocating-repair"); + + return true; +} + +static void map_raid56_repair_block(struct btrfs_io_context *bioc, + struct btrfs_io_stripe *smap, + u64 logical) +{ + int data_stripes = nr_bioc_data_stripes(bioc); + int i; + + for (i = 0; i < data_stripes; i++) { + u64 stripe_start = bioc->full_stripe_logical + + btrfs_stripe_nr_to_offset(i); + + if (logical >= stripe_start && + logical < stripe_start + BTRFS_STRIPE_LEN) + break; + } + ASSERT(i < data_stripes, "i=%d data_stripes=%d", i, data_stripes); + smap->dev = bioc->stripes[i].dev; + smap->physical = bioc->stripes[i].physical + + ((logical - bioc->full_stripe_logical) & + BTRFS_STRIPE_LEN_MASK); +} + +/* + * Map a repair write into a single device. + * + * A repair write is triggered by read time repair or scrub, which would only + * update the contents of a single device. + * Not update any other mirrors nor go through RMW path. + * + * Callers should ensure: + * + * - Call btrfs_bio_counter_inc_blocked() first + * - The range does not cross stripe boundary + * - Has a valid @mirror_num passed in. + */ +int btrfs_map_repair_block(struct btrfs_fs_info *fs_info, + struct btrfs_io_stripe *smap, u64 logical, + u32 length, int mirror_num) +{ + struct btrfs_io_context *bioc = NULL; + u64 map_length = length; + int mirror_ret = mirror_num; + int ret; + + ASSERT(mirror_num > 0, "mirror_num=%d", mirror_num); + + ret = btrfs_map_block(fs_info, BTRFS_MAP_WRITE, logical, &map_length, + &bioc, smap, &mirror_ret); + if (ret < 0) + return ret; + + /* The map range should not cross stripe boundary. */ + ASSERT(map_length >= length, "map_length=%llu length=%u", map_length, length); + + /* Already mapped to single stripe. */ + if (!bioc) + goto out; + + /* Map the RAID56 multi-stripe writes to a single one. */ + if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) { + map_raid56_repair_block(bioc, smap, logical); + goto out; + } + + ASSERT(mirror_num <= bioc->num_stripes, + "mirror_num=%d num_stripes=%d", mirror_num, bioc->num_stripes); + smap->dev = bioc->stripes[mirror_num - 1].dev; + smap->physical = bioc->stripes[mirror_num - 1].physical; +out: + btrfs_put_bioc(bioc); + ASSERT(smap->dev); + return 0; +} diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h new file mode 100644 index 000000000..0415d74ca --- /dev/null +++ b/fs/btrfs/volumes.h @@ -0,0 +1,953 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#ifndef BTRFS_VOLUMES_H +#define BTRFS_VOLUMES_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "messages.h" +#include "extent-io-tree.h" +#include "fs.h" + +struct block_device; +struct bdev_handle; +struct btrfs_fs_info; +struct btrfs_block_group; +struct btrfs_trans_handle; +struct btrfs_transaction; +struct btrfs_zoned_device_info; +struct btrfs_space_info; + +#define BTRFS_MAX_DATA_CHUNK_SIZE (10ULL * SZ_1G) + +/* + * Arbitrary maximum size of one discard request to limit potentially long time + * spent in blkdev_issue_discard(). + */ +#define BTRFS_MAX_DISCARD_CHUNK_SIZE (SZ_1G) + +extern struct mutex uuid_mutex; + +#define BTRFS_STRIPE_LEN SZ_64K +#define BTRFS_STRIPE_LEN_SHIFT (16) +#define BTRFS_STRIPE_LEN_MASK (BTRFS_STRIPE_LEN - 1) + +static_assert(ilog2(BTRFS_STRIPE_LEN) == BTRFS_STRIPE_LEN_SHIFT); + +/* Used by sanity check for btrfs_raid_types. */ +#define const_ffs(n) (__builtin_ctzll(n) + 1) + +/* + * The conversion from BTRFS_BLOCK_GROUP_* bits to btrfs_raid_type requires + * RAID0 always to be the lowest profile bit. + * Although it's part of on-disk format and should never change, do extra + * compile-time sanity checks. + */ +static_assert(const_ffs(BTRFS_BLOCK_GROUP_RAID0) < + const_ffs(BTRFS_BLOCK_GROUP_PROFILE_MASK & ~BTRFS_BLOCK_GROUP_RAID0)); + +/* ilog2() can handle both constants and variables */ +#define BTRFS_BG_FLAG_TO_INDEX(profile) \ + ilog2((profile) >> (ilog2(BTRFS_BLOCK_GROUP_RAID0) - 1)) + +enum btrfs_raid_types { + /* SINGLE is the special one as it doesn't have on-disk bit. */ + BTRFS_RAID_SINGLE = 0, + + BTRFS_RAID_RAID0 = BTRFS_BG_FLAG_TO_INDEX(BTRFS_BLOCK_GROUP_RAID0), + BTRFS_RAID_RAID1 = BTRFS_BG_FLAG_TO_INDEX(BTRFS_BLOCK_GROUP_RAID1), + BTRFS_RAID_DUP = BTRFS_BG_FLAG_TO_INDEX(BTRFS_BLOCK_GROUP_DUP), + BTRFS_RAID_RAID10 = BTRFS_BG_FLAG_TO_INDEX(BTRFS_BLOCK_GROUP_RAID10), + BTRFS_RAID_RAID5 = BTRFS_BG_FLAG_TO_INDEX(BTRFS_BLOCK_GROUP_RAID5), + BTRFS_RAID_RAID6 = BTRFS_BG_FLAG_TO_INDEX(BTRFS_BLOCK_GROUP_RAID6), + BTRFS_RAID_RAID1C3 = BTRFS_BG_FLAG_TO_INDEX(BTRFS_BLOCK_GROUP_RAID1C3), + BTRFS_RAID_RAID1C4 = BTRFS_BG_FLAG_TO_INDEX(BTRFS_BLOCK_GROUP_RAID1C4), + + BTRFS_NR_RAID_TYPES +}; + +static_assert(BTRFS_RAID_RAID0 == 1); +static_assert(BTRFS_RAID_RAID1 == 2); +static_assert(BTRFS_RAID_DUP == 3); +static_assert(BTRFS_RAID_RAID10 == 4); +static_assert(BTRFS_RAID_RAID5 == 5); +static_assert(BTRFS_RAID_RAID6 == 6); +static_assert(BTRFS_RAID_RAID1C3 == 7); +static_assert(BTRFS_RAID_RAID1C4 == 8); + +/* + * Use sequence counter to get consistent device stat data on + * 32-bit processors. + */ +#if BITS_PER_LONG==32 && defined(CONFIG_SMP) +#include +#define __BTRFS_NEED_DEVICE_DATA_ORDERED +#define btrfs_device_data_ordered_init(device) \ + seqcount_init(&device->data_seqcount) +#else +#define btrfs_device_data_ordered_init(device) do { } while (0) +#endif + +#define BTRFS_DEV_STATE_WRITEABLE (0) +#define BTRFS_DEV_STATE_IN_FS_METADATA (1) +#define BTRFS_DEV_STATE_MISSING (2) +#define BTRFS_DEV_STATE_REPLACE_TGT (3) +#define BTRFS_DEV_STATE_FLUSH_SENT (4) +#define BTRFS_DEV_STATE_NO_READA (5) +#define BTRFS_DEV_STATE_FLUSH_FAILED (6) + +/* Set when the device item is found in chunk tree, used to catch unexpected registered device. */ +#define BTRFS_DEV_STATE_ITEM_FOUND (7) + +/* Special value encoding failure to write primary super block. */ +#define BTRFS_SUPER_PRIMARY_WRITE_ERROR (INT_MAX / 2) + +struct btrfs_fs_devices; + +struct btrfs_device { + struct list_head dev_list; /* device_list_mutex */ + struct list_head dev_alloc_list; /* chunk mutex */ + struct list_head post_commit_list; /* chunk mutex */ + struct btrfs_fs_devices *fs_devices; + struct btrfs_fs_info *fs_info; + + /* Device path or NULL if missing. */ + const char __rcu *name; + + u64 generation; + + struct file *bdev_file; + struct block_device *bdev; + + struct btrfs_zoned_device_info *zone_info; + + unsigned long dev_state; + +#ifdef __BTRFS_NEED_DEVICE_DATA_ORDERED + seqcount_t data_seqcount; +#endif + + /* the internal btrfs device id */ + u64 devid; + + /* size of the device in memory */ + u64 total_bytes; + + /* size of the device on disk */ + u64 disk_total_bytes; + + /* bytes used */ + u64 bytes_used; + + /* optimal io alignment for this device */ + u32 io_align; + + /* optimal io width for this device */ + u32 io_width; + /* type and info about this device */ + u64 type; + + /* + * Counter of super block write errors, values larger than + * BTRFS_SUPER_PRIMARY_WRITE_ERROR encode primary super block write failure. + */ + atomic_t sb_write_errors; + + /* minimal io size for this device */ + u32 sector_size; + + /* physical drive uuid (or lvm uuid) */ + u8 uuid[BTRFS_UUID_SIZE]; + + /* + * size of the device on the current transaction + * + * This variant is update when committing the transaction, + * and protected by chunk mutex + */ + u64 commit_total_bytes; + + /* bytes used on the current transaction */ + u64 commit_bytes_used; + + /* Bio used for flushing device barriers */ + struct bio flush_bio; + struct completion flush_wait; + + /* per-device scrub information */ + struct scrub_ctx *scrub_ctx; + + /* disk I/O failure stats. For detailed description refer to + * enum btrfs_dev_stat_values in ioctl.h */ + int dev_stats_valid; + + /* Counter to record the change of device stats */ + atomic_t dev_stats_ccnt; + atomic_t dev_stat_values[BTRFS_DEV_STAT_VALUES_MAX]; + + /* + * Device's major-minor number. Must be set even if the device is not + * opened (bdev == NULL), unless the device is missing. + */ + dev_t devt; + + struct extent_io_tree alloc_state; + + struct completion kobj_unregister; + /* For sysfs/FSID/devinfo/devid/ */ + struct kobject devid_kobj; + + /* Bandwidth limit for scrub, in bytes */ + u64 scrub_speed_max; + + /* + * A temporary number of allocated space during per-profile + * available space calculation. + */ + u64 per_profile_allocated; +}; + +/* + * Block group or device which contains an active swapfile. Used for preventing + * unsafe operations while a swapfile is active. + * + * These are sorted on (ptr, inode) (note that a block group or device can + * contain more than one swapfile). We compare the pointer values because we + * don't actually care what the object is, we just need a quick check whether + * the object exists in the rbtree. + */ +struct btrfs_swapfile_pin { + struct rb_node node; + void *ptr; + struct inode *inode; + /* + * If true, ptr points to a struct btrfs_block_group. Otherwise, ptr + * points to a struct btrfs_device. + */ + bool is_block_group; + /* + * Only used when 'is_block_group' is true and it is the number of + * extents used by a swapfile for this block group ('ptr' field). + */ + int bg_extent_count; +}; + +/* + * If we read those variants at the context of their own lock, we needn't + * use the following helpers, reading them directly is safe. + */ +#if BITS_PER_LONG==32 && defined(CONFIG_SMP) +#define BTRFS_DEVICE_GETSET_FUNCS(name) \ +static inline u64 \ +btrfs_device_get_##name(const struct btrfs_device *dev) \ +{ \ + u64 size; \ + unsigned int seq; \ + \ + do { \ + seq = read_seqcount_begin(&dev->data_seqcount); \ + size = dev->name; \ + } while (read_seqcount_retry(&dev->data_seqcount, seq)); \ + return size; \ +} \ + \ +static inline void \ +btrfs_device_set_##name(struct btrfs_device *dev, u64 size) \ +{ \ + preempt_disable(); \ + write_seqcount_begin(&dev->data_seqcount); \ + dev->name = size; \ + write_seqcount_end(&dev->data_seqcount); \ + preempt_enable(); \ +} +#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION) +#define BTRFS_DEVICE_GETSET_FUNCS(name) \ +static inline u64 \ +btrfs_device_get_##name(const struct btrfs_device *dev) \ +{ \ + u64 size; \ + \ + preempt_disable(); \ + size = dev->name; \ + preempt_enable(); \ + return size; \ +} \ + \ +static inline void \ +btrfs_device_set_##name(struct btrfs_device *dev, u64 size) \ +{ \ + preempt_disable(); \ + dev->name = size; \ + preempt_enable(); \ +} +#else +#define BTRFS_DEVICE_GETSET_FUNCS(name) \ +static inline u64 \ +btrfs_device_get_##name(const struct btrfs_device *dev) \ +{ \ + return dev->name; \ +} \ + \ +static inline void \ +btrfs_device_set_##name(struct btrfs_device *dev, u64 size) \ +{ \ + dev->name = size; \ +} +#endif + +BTRFS_DEVICE_GETSET_FUNCS(total_bytes); +BTRFS_DEVICE_GETSET_FUNCS(disk_total_bytes); +BTRFS_DEVICE_GETSET_FUNCS(bytes_used); + +enum btrfs_chunk_allocation_policy { + BTRFS_CHUNK_ALLOC_REGULAR, + BTRFS_CHUNK_ALLOC_ZONED, +}; + +#define BTRFS_DEFAULT_RR_MIN_CONTIG_READ (SZ_256K) +/* Keep in sync with raid_attr table, current maximum is RAID1C4. */ +#define BTRFS_RAID1_MAX_MIRRORS (4) +/* + * Read policies for mirrored block group profiles, read picks the stripe based + * on these policies. + */ +enum btrfs_read_policy { + /* Use process PID to choose the stripe */ + BTRFS_READ_POLICY_PID, +#ifdef CONFIG_BTRFS_EXPERIMENTAL + /* Balancing RAID1 reads across all striped devices (round-robin). */ + BTRFS_READ_POLICY_RR, + /* Read from a specific device. */ + BTRFS_READ_POLICY_DEVID, +#endif + BTRFS_NR_READ_POLICY, +}; + +struct btrfs_fs_devices { + u8 fsid[BTRFS_FSID_SIZE]; /* FS specific uuid */ + + /* + * UUID written into the btree blocks: + * + * - If metadata_uuid != fsid then super block must have + * BTRFS_FEATURE_INCOMPAT_METADATA_UUID flag set. + * + * - Following shall be true at all times: + * - metadata_uuid == btrfs_header::fsid + * - metadata_uuid == btrfs_dev_item::fsid + * + * - Relations between fsid and metadata_uuid in sb and fs_devices: + * - Normal: + * fs_devices->fsid == fs_devices->metadata_uuid == sb->fsid + * sb->metadata_uuid == 0 + * + * - When the BTRFS_FEATURE_INCOMPAT_METADATA_UUID flag is set: + * fs_devices->fsid == sb->fsid + * fs_devices->metadata_uuid == sb->metadata_uuid + * + * - When in-memory fs_devices->temp_fsid is true + * fs_devices->fsid = random + * fs_devices->metadata_uuid == sb->fsid + */ + u8 metadata_uuid[BTRFS_FSID_SIZE]; + + struct list_head fs_list; + + /* + * Number of devices under this fsid including missing and + * replace-target device and excludes seed devices. + */ + u64 num_devices; + + /* + * The number of devices that successfully opened, including + * replace-target, excludes seed devices. + */ + u64 open_devices; + + /* The number of devices that are under the chunk allocation list. */ + u64 rw_devices; + + /* Count of missing devices under this fsid excluding seed device. */ + u64 missing_devices; + u64 total_rw_bytes; + + /* + * Count of devices from btrfs_super_block::num_devices for this fsid, + * which includes the seed device, excludes the transient replace-target + * device. + */ + u64 total_devices; + + /* Highest generation number of seen devices */ + u64 latest_generation; + + /* + * The mount device or a device with highest generation after removal + * or replace. + */ + struct btrfs_device *latest_dev; + + /* + * All of the devices in the filesystem, protected by a mutex so we can + * safely walk it to write out the super blocks without worrying about + * adding/removing by the multi-device code. Scrubbing super block can + * kick off supers writing by holding this mutex lock. + */ + struct mutex device_list_mutex; + + /* List of all devices, protected by device_list_mutex */ + struct list_head devices; + + /* Devices which can satisfy space allocation. Protected by * chunk_mutex. */ + struct list_head alloc_list; + + struct list_head seed_list; + + /* Count fs-devices opened. */ + int opened; + + /* + * Counter of the processes that are holding this fs_devices but not + * yet opened. + * This is for mounting handling, as we can only open the fs_devices + * after a super block is created. But we cannot take uuid_mutex + * during sget_fc(), thus we have to hold the fs_devices (meaning it + * cannot be released) until a super block is returned. + */ + int holding; + + /* Set when we find or add a device that doesn't have the nonrot flag set. */ + bool rotating; + /* Devices support TRIM/discard commands. */ + bool discardable; + /* The filesystem is a seed filesystem. */ + bool seeding; + /* The mount needs to use a randomly generated fsid. */ + bool temp_fsid; + /* Enable/disable the filesystem stats tracking. */ + bool collect_fs_stats; + + struct btrfs_fs_info *fs_info; + /* sysfs kobjects */ + struct kobject fsid_kobj; + struct kobject *devices_kobj; + struct kobject *devinfo_kobj; + struct completion kobj_unregister; + + enum btrfs_chunk_allocation_policy chunk_alloc_policy; + + /* Policy used to read the mirrored stripes. */ + enum btrfs_read_policy read_policy; + +#ifdef CONFIG_BTRFS_EXPERIMENTAL + /* + * Minimum contiguous reads before switching to next device, the unit + * is one block/sectorsize. + */ + u32 rr_min_contig_read; + + /* Device to be used for reading in case of RAID1. */ + u64 read_devid; +#endif + + /* + * Each value indicates the available space for that profile. + * U64_MAX means the estimation is unavailable. + * + * Protected by per_profile_lock; + */ + u64 per_profile_avail[BTRFS_NR_RAID_TYPES]; + spinlock_t per_profile_lock; +}; + +#define BTRFS_MAX_DEVS(info) ((BTRFS_MAX_ITEM_SIZE(info) \ + - sizeof(struct btrfs_chunk)) \ + / sizeof(struct btrfs_stripe) + 1) + +#define BTRFS_MAX_DEVS_SYS_CHUNK ((BTRFS_SYSTEM_CHUNK_ARRAY_SIZE \ + - 2 * sizeof(struct btrfs_disk_key) \ + - 2 * sizeof(struct btrfs_chunk)) \ + / sizeof(struct btrfs_stripe) + 1) + +struct btrfs_io_stripe { + struct btrfs_device *dev; + /* Block mapping. */ + u64 physical; + bool rst_search_commit_root; + /* For the endio handler. */ + struct btrfs_io_context *bioc; +}; + +struct btrfs_discard_stripe { + struct btrfs_device *dev; + u64 physical; + u64 length; +}; + +/* + * Context for IO submission for device stripe. + * + * - Track the unfinished mirrors for mirror based profiles + * Mirror based profiles are SINGLE/DUP/RAID1/RAID10. + * + * - Contain the logical -> physical mapping info + * Used by submit_stripe_bio() for mapping logical bio + * into physical device address. + * + * - Contain device replace info + * Used by handle_ops_on_dev_replace() to copy logical bios + * into the new device. + * + * - Contain RAID56 full stripe logical bytenrs + */ +struct btrfs_io_context { + refcount_t refs; + struct btrfs_fs_info *fs_info; + /* Taken from struct btrfs_chunk_map::type. */ + u64 map_type; + struct bio *orig_bio; + atomic_t error; + u16 max_errors; + bool use_rst; + + u64 logical; + u64 size; + /* Raid stripe tree ordered entry. */ + struct list_head rst_ordered_entry; + + /* + * The total number of stripes, including the extra duplicated + * stripe for replace. + */ + u16 num_stripes; + + /* + * The mirror_num of this bioc. + * + * This is for reads which use 0 as mirror_num, thus we should return a + * valid mirror_num (>0) for the reader. + */ + u16 mirror_num; + + /* + * The following two members are for dev-replace case only. + * + * @replace_nr_stripes: Number of duplicated stripes which need to be + * written to replace target. + * Should be <= 2 (2 for DUP, otherwise <= 1). + * @replace_stripe_src: The array indicates where the duplicated stripes + * are from. + * + * The @replace_stripe_src[] array is mostly for RAID56 cases. + * As non-RAID56 stripes share the same contents of the mapped range, + * thus no need to bother where the duplicated ones are from. + * + * But for RAID56 case, all stripes contain different contents, thus + * we need a way to know the mapping. + * + * There is an example for the two members, using a RAID5 write: + * + * num_stripes: 4 (3 + 1 duplicated write) + * stripes[0]: dev = devid 1, physical = X + * stripes[1]: dev = devid 2, physical = Y + * stripes[2]: dev = devid 3, physical = Z + * stripes[3]: dev = devid 0, physical = Y + * + * replace_nr_stripes = 1 + * replace_stripe_src = 1 <- Means stripes[1] is involved in replace. + * The duplicated stripe index would be + * (@num_stripes - 1). + * + * Note, that we can still have cases replace_nr_stripes = 2 for DUP. + * In that case, all stripes share the same content, thus we don't + * need to bother @replace_stripe_src value at all. + */ + u16 replace_nr_stripes; + s16 replace_stripe_src; + /* + * Logical bytenr of the full stripe start, only for RAID56 cases. + * + * When this value is set to other than (u64)-1, the stripes[] should + * follow this pattern: + * + * (real_stripes = num_stripes - replace_nr_stripes) + * (data_stripes = (is_raid6) ? (real_stripes - 2) : (real_stripes - 1)) + * + * stripes[0]: The first data stripe + * stripes[1]: The second data stripe + * ... + * stripes[data_stripes - 1]: The last data stripe + * stripes[data_stripes]: The P stripe + * stripes[data_stripes + 1]: The Q stripe (only for RAID6). + */ + u64 full_stripe_logical; + struct btrfs_io_stripe stripes[]; +}; + +struct btrfs_device_info { + struct btrfs_device *dev; + u64 dev_offset; + u64 max_avail; + u64 total_avail; +}; + +struct btrfs_raid_attr { + u8 sub_stripes; /* sub_stripes info for map */ + u8 dev_stripes; /* stripes per dev */ + u8 devs_max; /* max devs to use */ + u8 devs_min; /* min devs needed */ + u8 tolerated_failures; /* max tolerated fail devs */ + u8 devs_increment; /* ndevs has to be a multiple of this */ + u8 ncopies; /* how many copies to data has */ + u8 nparity; /* number of stripes worth of bytes to store + * parity information */ + u8 mindev_error; /* error code if min devs requisite is unmet */ + const char raid_name[8]; /* name of the raid */ + u64 bg_flag; /* block group flag of the raid */ +}; + +extern const struct btrfs_raid_attr btrfs_raid_array[BTRFS_NR_RAID_TYPES]; + +struct btrfs_chunk_map { + struct rb_node rb_node; + /* For mount time dev extent verification. */ + int verified_stripes; + refcount_t refs; + u64 start; + u64 chunk_len; + u64 stripe_size; + /* + * The real type that is utilized during logical address mapping. + * + * For most profiles it matches @on_disk_type, but for single-data-RAID56, + * the real type will be set to RAID1/RAID1C3, to avoid unsupported + * operations from raid56 lib. + */ + u64 type; + u64 on_disk_type; + int num_stripes; + int sub_stripes; + struct btrfs_io_stripe stripes[]; +}; + +#define btrfs_chunk_map_size(n) (sizeof(struct btrfs_chunk_map) + \ + (sizeof(struct btrfs_io_stripe) * (n))) + +static inline void btrfs_free_chunk_map(struct btrfs_chunk_map *map) +{ + if (map && refcount_dec_and_test(&map->refs)) { + ASSERT(RB_EMPTY_NODE(&map->rb_node)); + kfree(map); + } +} +DEFINE_FREE(btrfs_free_chunk_map, struct btrfs_chunk_map *, btrfs_free_chunk_map(_T)) + +struct btrfs_balance_control { + struct btrfs_balance_args data; + struct btrfs_balance_args meta; + struct btrfs_balance_args sys; + + u64 flags; + + struct btrfs_balance_progress stat; +}; + +/* + * Search for a given device by the set parameters + */ +struct btrfs_dev_lookup_args { + u64 devid; + u8 *uuid; + u8 *fsid; + /* + * If devt is specified, all other members will be ignored as it is + * enough to uniquely locate a device. + */ + dev_t devt; + bool missing; +}; + +/* We have to initialize to -1 because BTRFS_DEV_REPLACE_DEVID is 0 */ +#define BTRFS_DEV_LOOKUP_ARGS_INIT { .devid = (u64)-1 } + +#define BTRFS_DEV_LOOKUP_ARGS(name) \ + struct btrfs_dev_lookup_args name = BTRFS_DEV_LOOKUP_ARGS_INIT + +enum btrfs_map_op { + BTRFS_MAP_READ, + BTRFS_MAP_WRITE, + BTRFS_MAP_GET_READ_MIRRORS, +}; + +static inline enum btrfs_map_op btrfs_op(const struct bio *bio) +{ + switch (bio_op(bio)) { + case REQ_OP_WRITE: + case REQ_OP_ZONE_APPEND: + return BTRFS_MAP_WRITE; + default: + WARN_ON_ONCE(1); + fallthrough; + case REQ_OP_READ: + return BTRFS_MAP_READ; + } +} + +static inline unsigned long btrfs_chunk_item_size(int num_stripes) +{ + ASSERT(num_stripes); + return sizeof(struct btrfs_chunk) + + sizeof(struct btrfs_stripe) * (num_stripes - 1); +} + +/* + * Do the type safe conversion from stripe_nr to offset inside the chunk. + * + * @stripe_nr is u32, with left shift it can overflow u32 for chunks larger + * than 4G. This does the proper type cast to avoid overflow. + */ +static inline u64 btrfs_stripe_nr_to_offset(u32 stripe_nr) +{ + return (u64)stripe_nr << BTRFS_STRIPE_LEN_SHIFT; +} + +void btrfs_get_bioc(struct btrfs_io_context *bioc); +void btrfs_put_bioc(struct btrfs_io_context *bioc); +int btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op, + u64 logical, u64 *length, + struct btrfs_io_context **bioc_ret, + struct btrfs_io_stripe *smap, int *mirror_num_ret); +int btrfs_map_repair_block(struct btrfs_fs_info *fs_info, + struct btrfs_io_stripe *smap, u64 logical, + u32 length, int mirror_num); +struct btrfs_discard_stripe *btrfs_map_discard(struct btrfs_fs_info *fs_info, + u64 logical, u64 *length_ret, + u32 *num_stripes, bool do_remap); +int btrfs_read_sys_array(struct btrfs_fs_info *fs_info); +int btrfs_read_chunk_tree(struct btrfs_fs_info *fs_info); +struct btrfs_block_group *btrfs_create_chunk(struct btrfs_trans_handle *trans, + struct btrfs_space_info *space_info, + u64 type); +void btrfs_mapping_tree_free(struct btrfs_fs_info *fs_info); +int btrfs_open_devices(struct btrfs_fs_devices *fs_devices, + blk_mode_t flags, void *holder); +struct btrfs_device *btrfs_scan_one_device(const char *path, bool mount_arg_dev); +int btrfs_forget_devices(dev_t devt); +void btrfs_close_devices(struct btrfs_fs_devices *fs_devices); +void btrfs_release_device_allow_freeze(struct file *bdev_file); +void btrfs_free_extra_devids(struct btrfs_fs_devices *fs_devices); +void btrfs_assign_next_active_device(struct btrfs_device *device, + struct btrfs_device *this_dev); +struct btrfs_device *btrfs_find_device_by_devspec(struct btrfs_fs_info *fs_info, + u64 devid, + const char *devpath); +int btrfs_get_dev_args_from_path(struct btrfs_fs_info *fs_info, + struct btrfs_dev_lookup_args *args, + const char *path); +struct btrfs_device *btrfs_alloc_device(struct btrfs_fs_info *fs_info, + const u64 *devid, const u8 *uuid, + const char *path); +void btrfs_put_dev_args_from_path(struct btrfs_dev_lookup_args *args); +int btrfs_rm_device(struct btrfs_fs_info *fs_info, + struct btrfs_dev_lookup_args *args, + struct file **bdev_file); +void __exit btrfs_cleanup_fs_uuids(void); +int btrfs_num_copies(struct btrfs_fs_info *fs_info, u64 logical, u64 len); +int btrfs_grow_device(struct btrfs_trans_handle *trans, + struct btrfs_device *device, u64 new_size); +struct btrfs_device *btrfs_find_device(const struct btrfs_fs_devices *fs_devices, + const struct btrfs_dev_lookup_args *args); +int btrfs_shrink_device(struct btrfs_device *device, u64 new_size); +int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *path); +struct file *btrfs_open_device_deny_freeze(const char *path, + struct super_block *sb); +int btrfs_balance(struct btrfs_fs_info *fs_info, + struct btrfs_balance_control *bctl, + struct btrfs_ioctl_balance_args *bargs); +void btrfs_describe_block_groups(u64 flags, char *buf, u32 size_buf); +int btrfs_resume_balance_async(struct btrfs_fs_info *fs_info); +int btrfs_recover_balance(struct btrfs_fs_info *fs_info); +int btrfs_pause_balance(struct btrfs_fs_info *fs_info); +int btrfs_relocate_chunk(struct btrfs_fs_info *fs_info, u64 chunk_offset, + bool verbose); +int btrfs_cancel_balance(struct btrfs_fs_info *fs_info); +bool btrfs_chunk_writeable(struct btrfs_fs_info *fs_info, u64 chunk_offset); +void btrfs_dev_stat_inc_and_print(struct btrfs_device *dev, int index); +int btrfs_get_dev_stats(struct btrfs_fs_info *fs_info, + struct btrfs_ioctl_get_dev_stats *stats); +int btrfs_init_devices_late(struct btrfs_fs_info *fs_info); +int btrfs_init_dev_stats(struct btrfs_fs_info *fs_info); +int btrfs_init_writeback_bio_size(struct btrfs_fs_info *fs_info); +int btrfs_run_dev_stats(struct btrfs_trans_handle *trans); +void btrfs_rm_dev_replace_remove_srcdev(struct btrfs_device *srcdev); +void btrfs_rm_dev_replace_free_srcdev(struct btrfs_device *srcdev); +void btrfs_destroy_dev_replace_tgtdev(struct btrfs_device *tgtdev, + bool allow_freeze); +unsigned long btrfs_full_stripe_len(struct btrfs_fs_info *fs_info, + u64 logical); +u64 btrfs_calc_stripe_length(const struct btrfs_chunk_map *map); +int btrfs_nr_parity_stripes(u64 type); +int btrfs_chunk_alloc_add_chunk_item(struct btrfs_trans_handle *trans, + struct btrfs_block_group *bg); +int btrfs_remove_dev_extents(struct btrfs_trans_handle *trans, struct btrfs_chunk_map *map); +int btrfs_remove_chunk(struct btrfs_trans_handle *trans, u64 chunk_offset); + +#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS +struct btrfs_chunk_map *btrfs_alloc_chunk_map(int num_stripes, gfp_t gfp); +int btrfs_add_chunk_map(struct btrfs_fs_info *fs_info, struct btrfs_chunk_map *map); +#endif + +struct btrfs_chunk_map *btrfs_find_chunk_map(struct btrfs_fs_info *fs_info, + u64 logical, u64 length); +struct btrfs_chunk_map *btrfs_find_chunk_map_nolock(struct btrfs_fs_info *fs_info, + u64 logical, u64 length); +struct btrfs_chunk_map *btrfs_get_chunk_map(struct btrfs_fs_info *fs_info, + u64 logical, u64 length); +void btrfs_remove_chunk_map(struct btrfs_fs_info *fs_info, struct btrfs_chunk_map *map); +struct btrfs_super_block *btrfs_read_disk_super(struct block_device *bdev, + int copy_num, bool drop_cache); +void btrfs_release_disk_super(struct btrfs_super_block *super); + +static inline void btrfs_dev_stat_inc(struct btrfs_device *dev, + int index) +{ + atomic_inc(dev->dev_stat_values + index); + /* + * This memory barrier orders stores updating statistics before stores + * updating dev_stats_ccnt. + * + * It pairs with smp_rmb() in btrfs_run_dev_stats(). + */ + smp_mb__before_atomic(); + atomic_inc(&dev->dev_stats_ccnt); +} + +static inline int btrfs_dev_stat_read(struct btrfs_device *dev, + int index) +{ + return atomic_read(dev->dev_stat_values + index); +} + +static inline int btrfs_dev_stat_read_and_reset(struct btrfs_device *dev, + int index) +{ + int ret; + + ret = atomic_xchg(dev->dev_stat_values + index, 0); + /* + * atomic_xchg implies a full memory barriers as per atomic_t.txt: + * - RMW operations that have a return value are fully ordered; + * + * This implicit memory barriers is paired with the smp_rmb in + * btrfs_run_dev_stats + */ + atomic_inc(&dev->dev_stats_ccnt); + return ret; +} + +static inline void btrfs_dev_stat_set(struct btrfs_device *dev, + int index, unsigned long val) +{ + atomic_set(dev->dev_stat_values + index, val); + /* + * This memory barrier orders stores updating statistics before stores + * updating dev_stats_ccnt. + * + * It pairs with smp_rmb() in btrfs_run_dev_stats(). + */ + smp_mb__before_atomic(); + atomic_inc(&dev->dev_stats_ccnt); +} + +static inline const char *btrfs_dev_name(const struct btrfs_device *device) +{ + if (!device || test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) + return ""; + else + return rcu_dereference(device->name); +} + +static inline void btrfs_warn_unknown_chunk_allocation(enum btrfs_chunk_allocation_policy pol) +{ + WARN_ONCE(1, "unknown allocation policy %d, fallback to regular", pol); +} + +static inline void btrfs_fs_devices_inc_holding(struct btrfs_fs_devices *fs_devices) +{ + lockdep_assert_held(&uuid_mutex); + ASSERT(fs_devices->holding >= 0); + fs_devices->holding++; +} + +static inline void btrfs_fs_devices_dec_holding(struct btrfs_fs_devices *fs_devices) +{ + lockdep_assert_held(&uuid_mutex); + ASSERT(fs_devices->holding > 0); + fs_devices->holding--; +} + +void btrfs_commit_device_sizes(struct btrfs_transaction *trans); + +struct list_head * __attribute_const__ btrfs_get_fs_uuids(void); +bool btrfs_check_rw_degradable(struct btrfs_fs_info *fs_info, + struct btrfs_device *failing_dev); +void btrfs_scratch_superblocks(struct btrfs_fs_info *fs_info, struct btrfs_device *device); + +enum btrfs_raid_types __attribute_const__ btrfs_bg_flags_to_raid_index(u64 flags); +int btrfs_bg_type_to_factor(u64 flags); +const char *btrfs_bg_type_to_raid_name(u64 flags); +int btrfs_verify_dev_extents(struct btrfs_fs_info *fs_info); +bool btrfs_verify_dev_items(const struct btrfs_fs_info *fs_info); +void btrfs_update_per_profile_avail(struct btrfs_fs_info *fs_info); + +static inline bool btrfs_get_per_profile_avail(struct btrfs_fs_info *fs_info, + u64 profile, u64 *avail_ret) +{ + enum btrfs_raid_types index = btrfs_bg_flags_to_raid_index(profile); + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; + bool uptodate = false; + + spin_lock(&fs_devices->per_profile_lock); + if (fs_devices->per_profile_avail[index] != U64_MAX) { + uptodate = true; + *avail_ret = fs_devices->per_profile_avail[index]; + } + spin_unlock(&fs_info->fs_devices->per_profile_lock); + return uptodate; +} + +bool btrfs_repair_one_zone(struct btrfs_fs_info *fs_info, u64 logical); + +bool btrfs_pinned_by_swapfile(struct btrfs_fs_info *fs_info, void *ptr); +const u8 *btrfs_sb_fsid_ptr(const struct btrfs_super_block *sb); +int btrfs_update_device(struct btrfs_trans_handle *trans, struct btrfs_device *device); +void btrfs_chunk_map_device_clear_bits(struct btrfs_chunk_map *map, unsigned int bits); + +bool btrfs_first_pending_extent(struct btrfs_device *device, u64 start, u64 len, + u64 *pending_start, u64 *pending_end); +bool btrfs_find_hole_in_pending_extents(struct btrfs_device *device, + u64 *start, u64 *len, u64 min_hole_size); +int btrfs_remove_dev_stat_item(struct btrfs_trans_handle *trans, u64 devid); + +#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS +struct btrfs_io_context *alloc_btrfs_io_context(struct btrfs_fs_info *fs_info, + u64 logical, u16 total_stripes); +#endif + +#endif diff --git a/fs/btrfs/xattr.c b/fs/btrfs/xattr.c new file mode 100644 index 000000000..ab55d10bd --- /dev/null +++ b/fs/btrfs/xattr.c @@ -0,0 +1,527 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2007 Red Hat. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ctree.h" +#include "fs.h" +#include "messages.h" +#include "btrfs_inode.h" +#include "transaction.h" +#include "xattr.h" +#include "disk-io.h" +#include "props.h" +#include "locking.h" +#include "accessors.h" +#include "dir-item.h" + +int btrfs_getxattr(const struct inode *inode, const char *name, + void *buffer, size_t size) +{ + struct btrfs_dir_item *di; + struct btrfs_root *root = BTRFS_I(inode)->root; + BTRFS_PATH_AUTO_FREE(path); + struct extent_buffer *leaf; + unsigned long data_ptr; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + /* lookup the xattr by name */ + di = btrfs_lookup_xattr(NULL, root, path, btrfs_ino(BTRFS_I(inode)), + name, strlen(name), 0); + if (!di) + return -ENODATA; + if (IS_ERR(di)) + return PTR_ERR(di); + + leaf = path->nodes[0]; + /* if size is 0, that means we want the size of the attr */ + if (!size) + return btrfs_dir_data_len(leaf, di); + + /* now get the data out of our dir_item */ + if (btrfs_dir_data_len(leaf, di) > size) + return -ERANGE; + + /* + * The way things are packed into the leaf is like this + * |struct btrfs_dir_item|name|data| + * where name is the xattr name, so security.foo, and data is the + * content of the xattr. data_ptr points to the location in memory + * where the data starts in the in memory leaf + */ + data_ptr = (unsigned long)((char *)(di + 1) + + btrfs_dir_name_len(leaf, di)); + read_extent_buffer(leaf, buffer, data_ptr, + btrfs_dir_data_len(leaf, di)); + return btrfs_dir_data_len(leaf, di); +} + +int btrfs_setxattr(struct btrfs_trans_handle *trans, struct inode *inode, + const char *name, const void *value, size_t size, int flags) +{ + struct btrfs_dir_item *di = NULL; + struct btrfs_root *root = BTRFS_I(inode)->root; + BTRFS_PATH_AUTO_FREE(path); + size_t name_len = strlen(name); + int ret = 0; + + ASSERT(trans); + + if (name_len + size > BTRFS_MAX_XATTR_SIZE(root->fs_info)) + return -ENOSPC; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + path->skip_release_on_error = true; + + if (!value) { + di = btrfs_lookup_xattr(trans, root, path, + btrfs_ino(BTRFS_I(inode)), name, name_len, -1); + if (!di && (flags & XATTR_REPLACE)) + ret = -ENODATA; + else if (IS_ERR(di)) + ret = PTR_ERR(di); + else if (di) + ret = btrfs_delete_one_dir_name(trans, root, path, di); + goto out; + } + + /* + * For a replace we can't just do the insert blindly. + * Do a lookup first (read-only btrfs_search_slot), and return if xattr + * doesn't exist. If it exists, fall down below to the insert/replace + * path - we can't race with a concurrent xattr delete, because the VFS + * locks the inode's i_mutex before calling setxattr or removexattr. + */ + if (flags & XATTR_REPLACE) { + btrfs_assert_inode_locked(BTRFS_I(inode)); + di = btrfs_lookup_xattr(NULL, root, path, + btrfs_ino(BTRFS_I(inode)), name, name_len, 0); + if (!di) + ret = -ENODATA; + else if (IS_ERR(di)) + ret = PTR_ERR(di); + if (ret) + goto out; + btrfs_release_path(path); + di = NULL; + } + + ret = btrfs_insert_xattr_item(trans, root, path, btrfs_ino(BTRFS_I(inode)), + name, name_len, value, size); + if (ret == -EOVERFLOW) { + /* + * We have an existing item in a leaf, split_leaf couldn't + * expand it. That item might have or not a dir_item that + * matches our target xattr, so lets check. + */ + ret = 0; + btrfs_assert_tree_write_locked(path->nodes[0]); + di = btrfs_match_dir_item_name(path, name, name_len); + if (!di && !(flags & XATTR_REPLACE)) { + ret = -ENOSPC; + goto out; + } + } else if (ret == -EEXIST) { + ret = 0; + di = btrfs_match_dir_item_name(path, name, name_len); + ASSERT(di); /* logic error */ + } else if (ret) { + goto out; + } + + if (di && (flags & XATTR_CREATE)) { + ret = -EEXIST; + goto out; + } + + if (di) { + /* + * We're doing a replace, and it must be atomic, that is, at + * any point in time we have either the old or the new xattr + * value in the tree. We don't want readers (getxattr and + * listxattrs) to miss a value, this is specially important + * for ACLs. + */ + const int slot = path->slots[0]; + struct extent_buffer *leaf = path->nodes[0]; + const u16 old_data_len = btrfs_dir_data_len(leaf, di); + const u32 item_size = btrfs_item_size(leaf, slot); + const u32 data_size = sizeof(*di) + name_len + size; + unsigned long data_ptr; + char *ptr; + + if (size > old_data_len) { + if (btrfs_leaf_free_space(leaf) < + (size - old_data_len)) { + ret = -ENOSPC; + goto out; + } + } + + if (old_data_len + name_len + sizeof(*di) == item_size) { + /* No other xattrs packed in the same leaf item. */ + if (size > old_data_len) + btrfs_extend_item(trans, path, size - old_data_len); + else if (size < old_data_len) + btrfs_truncate_item(trans, path, data_size, 1); + } else { + /* There are other xattrs packed in the same item. */ + ret = btrfs_delete_one_dir_name(trans, root, path, di); + if (ret) + goto out; + btrfs_extend_item(trans, path, data_size); + } + + ptr = btrfs_item_ptr(leaf, slot, char); + ptr += btrfs_item_size(leaf, slot) - data_size; + di = (struct btrfs_dir_item *)ptr; + btrfs_set_dir_data_len(leaf, di, size); + data_ptr = ((unsigned long)(di + 1)) + name_len; + write_extent_buffer(leaf, value, data_ptr, size); + } else { + /* + * Insert, and we had space for the xattr, so path->slots[0] is + * where our xattr dir_item is and btrfs_insert_xattr_item() + * filled it. + */ + } +out: + if (!ret) { + set_bit(BTRFS_INODE_COPY_EVERYTHING, + &BTRFS_I(inode)->runtime_flags); + clear_bit(BTRFS_INODE_NO_XATTRS, &BTRFS_I(inode)->runtime_flags); + } + return ret; +} + +/* + * @value: "" makes the attribute to empty, NULL removes it + */ +int btrfs_setxattr_trans(struct inode *inode, const char *name, + const void *value, size_t size, int flags) +{ + struct btrfs_root *root = BTRFS_I(inode)->root; + struct btrfs_trans_handle *trans; + const bool start_trans = (current->journal_info == NULL); + int ret; + + if (start_trans) { + /* + * 1 unit for inserting/updating/deleting the xattr + * 1 unit for the inode item update + */ + trans = btrfs_start_transaction(root, 2); + if (IS_ERR(trans)) + return PTR_ERR(trans); + } else { + /* + * This can happen when smack is enabled and a directory is being + * created. It happens through d_instantiate_new(), which calls + * smack_d_instantiate(), which in turn calls __vfs_setxattr() to + * set the transmute xattr (XATTR_NAME_SMACKTRANSMUTE) on the + * inode. We have already reserved space for the xattr and inode + * update at btrfs_mkdir(), so just use the transaction handle. + * We don't join or start a transaction, as that will reset the + * block_rsv of the handle and trigger a warning for the start + * case. + */ + ASSERT(strncmp(name, XATTR_SECURITY_PREFIX, + XATTR_SECURITY_PREFIX_LEN) == 0); + trans = current->journal_info; + } + + ret = btrfs_setxattr(trans, inode, name, value, size, flags); + if (ret) + goto out; + + inode_inc_iversion(inode); + inode_set_ctime_current(inode); + ret = btrfs_update_inode(trans, BTRFS_I(inode)); + if (ret) + btrfs_abort_transaction(trans, ret); +out: + if (start_trans) + btrfs_end_transaction(trans); + return ret; +} + +ssize_t btrfs_listxattr(struct dentry *dentry, char *buffer, size_t size) +{ + struct btrfs_key found_key; + struct btrfs_key key; + struct inode *inode = d_inode(dentry); + struct btrfs_root *root = BTRFS_I(inode)->root; + BTRFS_PATH_AUTO_FREE(path); + int iter_ret = 0; + int ret = 0; + size_t total_size = 0, size_left = size; + + /* + * ok we want all objects associated with this id. + * NOTE: we set key.offset = 0; because we want to start with the + * first xattr that we find and walk forward + */ + key.objectid = btrfs_ino(BTRFS_I(inode)); + key.type = BTRFS_XATTR_ITEM_KEY; + key.offset = 0; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + path->reada = READA_FORWARD; + + /* search for our xattrs */ + btrfs_for_each_slot(root, &key, &found_key, path, iter_ret) { + struct extent_buffer *leaf; + int slot; + struct btrfs_dir_item *di; + u32 item_size; + u32 cur; + + leaf = path->nodes[0]; + slot = path->slots[0]; + + /* check to make sure this item is what we want */ + if (found_key.objectid != key.objectid) + break; + if (found_key.type > BTRFS_XATTR_ITEM_KEY) + break; + if (found_key.type < BTRFS_XATTR_ITEM_KEY) + continue; + + di = btrfs_item_ptr(leaf, slot, struct btrfs_dir_item); + item_size = btrfs_item_size(leaf, slot); + cur = 0; + while (cur < item_size) { + u16 name_len = btrfs_dir_name_len(leaf, di); + u16 data_len = btrfs_dir_data_len(leaf, di); + u32 this_len = sizeof(*di) + name_len + data_len; + unsigned long name_ptr = (unsigned long)(di + 1); + + total_size += name_len + 1; + /* + * We are just looking for how big our buffer needs to + * be. + */ + if (!size) + goto next; + + if (!buffer || (name_len + 1) > size_left) { + iter_ret = -ERANGE; + break; + } + + read_extent_buffer(leaf, buffer, name_ptr, name_len); + buffer[name_len] = '\0'; + + size_left -= name_len + 1; + buffer += name_len + 1; +next: + cur += this_len; + di = (struct btrfs_dir_item *)((char *)di + this_len); + } + } + + if (iter_ret < 0) + ret = iter_ret; + else + ret = total_size; + + return ret; +} + +static int btrfs_xattr_handler_get(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + name = xattr_full_name(handler, name); + return btrfs_getxattr(inode, name, buffer, size); +} + +static int btrfs_xattr_handler_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *buffer, + size_t size, int flags) +{ + if (btrfs_root_readonly(BTRFS_I(inode)->root)) + return -EROFS; + + name = xattr_full_name(handler, name); + return btrfs_setxattr_trans(inode, name, buffer, size, flags); +} + +static int btrfs_xattr_handler_get_security(const struct xattr_handler *handler, + struct dentry *unused, + struct inode *inode, + const char *name, void *buffer, + size_t size) +{ + int ret; + bool is_cap = false; + + name = xattr_full_name(handler, name); + + /* + * security.capability doesn't cache the results, so calls into us + * constantly to see if there's a capability xattr. Cache the result + * here in order to avoid wasting time doing lookups for xattrs we know + * don't exist. + */ + if (strcmp(name, XATTR_NAME_CAPS) == 0) { + is_cap = true; + if (test_bit(BTRFS_INODE_NO_CAP_XATTR, &BTRFS_I(inode)->runtime_flags)) + return -ENODATA; + } + + ret = btrfs_getxattr(inode, name, buffer, size); + if (ret == -ENODATA && is_cap) + set_bit(BTRFS_INODE_NO_CAP_XATTR, &BTRFS_I(inode)->runtime_flags); + return ret; +} + +static int btrfs_xattr_handler_set_security(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, + struct inode *inode, + const char *name, + const void *buffer, + size_t size, int flags) +{ + if (btrfs_root_readonly(BTRFS_I(inode)->root)) + return -EROFS; + + name = xattr_full_name(handler, name); + if (strcmp(name, XATTR_NAME_CAPS) == 0) + clear_bit(BTRFS_INODE_NO_CAP_XATTR, &BTRFS_I(inode)->runtime_flags); + + return btrfs_setxattr_trans(inode, name, buffer, size, flags); +} + +static int btrfs_xattr_handler_set_prop(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +{ + int ret; + struct btrfs_trans_handle *trans; + struct btrfs_root *root = BTRFS_I(inode)->root; + + name = xattr_full_name(handler, name); + ret = btrfs_validate_prop(BTRFS_I(inode), name, value, size); + if (ret) + return ret; + + if (btrfs_ignore_prop(BTRFS_I(inode), name)) + return 0; + + trans = btrfs_start_transaction(root, 2); + if (IS_ERR(trans)) + return PTR_ERR(trans); + + ret = btrfs_set_prop(trans, BTRFS_I(inode), name, value, size, flags); + if (!ret) { + inode_inc_iversion(inode); + inode_set_ctime_current(inode); + ret = btrfs_update_inode(trans, BTRFS_I(inode)); + if (ret) + btrfs_abort_transaction(trans, ret); + } + + btrfs_end_transaction(trans); + + return ret; +} + +static const struct xattr_handler btrfs_security_xattr_handler = { + .prefix = XATTR_SECURITY_PREFIX, + .get = btrfs_xattr_handler_get_security, + .set = btrfs_xattr_handler_set_security, +}; + +static const struct xattr_handler btrfs_trusted_xattr_handler = { + .prefix = XATTR_TRUSTED_PREFIX, + .get = btrfs_xattr_handler_get, + .set = btrfs_xattr_handler_set, +}; + +static const struct xattr_handler btrfs_user_xattr_handler = { + .prefix = XATTR_USER_PREFIX, + .get = btrfs_xattr_handler_get, + .set = btrfs_xattr_handler_set, +}; + +static const struct xattr_handler btrfs_btrfs_xattr_handler = { + .prefix = XATTR_BTRFS_PREFIX, + .get = btrfs_xattr_handler_get, + .set = btrfs_xattr_handler_set_prop, +}; + +const struct xattr_handler * const btrfs_xattr_handlers[] = { + &btrfs_security_xattr_handler, + &btrfs_trusted_xattr_handler, + &btrfs_user_xattr_handler, + &btrfs_btrfs_xattr_handler, + NULL, +}; + +static int btrfs_initxattrs(struct inode *inode, + const struct xattr *xattr_array, void *fs_private) +{ + struct btrfs_trans_handle *trans = fs_private; + const struct xattr *xattr; + unsigned int nofs_flag; + char *name; + int ret = 0; + + /* + * We're holding a transaction handle, so use a NOFS memory allocation + * context to avoid deadlock if reclaim happens. + */ + nofs_flag = memalloc_nofs_save(); + for (xattr = xattr_array; xattr->name != NULL; xattr++) { + const size_t name_len = XATTR_SECURITY_PREFIX_LEN + + strlen(xattr->name) + 1; + + name = kmalloc(name_len, GFP_KERNEL); + if (!name) { + ret = -ENOMEM; + break; + } + scnprintf(name, name_len, "%s%s", XATTR_SECURITY_PREFIX, xattr->name); + + if (strcmp(name, XATTR_NAME_CAPS) == 0) + clear_bit(BTRFS_INODE_NO_CAP_XATTR, &BTRFS_I(inode)->runtime_flags); + + ret = btrfs_setxattr(trans, inode, name, xattr->value, + xattr->value_len, 0); + kfree(name); + if (ret < 0) + break; + } + memalloc_nofs_restore(nofs_flag); + return ret; +} + +int btrfs_xattr_security_init(struct btrfs_trans_handle *trans, + struct inode *inode, struct inode *dir, + const struct qstr *qstr) +{ + return security_inode_init_security(inode, dir, qstr, + &btrfs_initxattrs, trans); +} diff --git a/fs/btrfs/xattr.h b/fs/btrfs/xattr.h new file mode 100644 index 000000000..0ce10e4ec --- /dev/null +++ b/fs/btrfs/xattr.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2007 Red Hat. All rights reserved. + */ + +#ifndef BTRFS_XATTR_H +#define BTRFS_XATTR_H + +#include + +struct dentry; +struct inode; +struct qstr; +struct xattr_handler; +struct btrfs_trans_handle; + +extern const struct xattr_handler * const btrfs_xattr_handlers[]; + +int btrfs_getxattr(const struct inode *inode, const char *name, + void *buffer, size_t size); +int btrfs_setxattr(struct btrfs_trans_handle *trans, struct inode *inode, + const char *name, const void *value, size_t size, int flags); +int btrfs_setxattr_trans(struct inode *inode, const char *name, + const void *value, size_t size, int flags); +ssize_t btrfs_listxattr(struct dentry *dentry, char *buffer, size_t size); + +int btrfs_xattr_security_init(struct btrfs_trans_handle *trans, + struct inode *inode, struct inode *dir, + const struct qstr *qstr); + +#endif diff --git a/fs/btrfs/zlib.c b/fs/btrfs/zlib.c new file mode 100644 index 000000000..486b52db5 --- /dev/null +++ b/fs/btrfs/zlib.c @@ -0,0 +1,514 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2008 Oracle. All rights reserved. + * + * Based on jffs2 zlib code: + * Copyright © 2001-2007 Red Hat, Inc. + * Created by David Woodhouse + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "btrfs_inode.h" +#include "compression.h" +#include "fs.h" +#include "subpage.h" + +/* workspace buffer size for s390 zlib hardware support */ +#define ZLIB_DFLTCC_BUF_SIZE (4 * PAGE_SIZE) + +struct workspace { + z_stream strm; + char *buf; + unsigned int buf_size; + struct list_head list; + int level; +}; + +struct list_head *zlib_get_workspace(struct btrfs_fs_info *fs_info, unsigned int level) +{ + struct list_head *ws = btrfs_get_workspace(fs_info, BTRFS_COMPRESS_ZLIB, level); + struct workspace *workspace = list_entry(ws, struct workspace, list); + + workspace->level = level; + + return ws; +} + +void zlib_free_workspace(struct list_head *ws) +{ + struct workspace *workspace = list_entry(ws, struct workspace, list); + + kvfree(workspace->strm.workspace); + kfree(workspace->buf); + kfree(workspace); +} + +/* + * For s390 hardware acceleration, the buffer size should be at least + * ZLIB_DFLTCC_BUF_SIZE to achieve the best performance. + * + * But if bs > ps we can have large enough folios that meet the s390 hardware + * handling. + */ +static bool need_special_buffer(struct btrfs_fs_info *fs_info) +{ + if (!zlib_deflate_dfltcc_enabled()) + return false; + if (btrfs_min_folio_size(fs_info) >= ZLIB_DFLTCC_BUF_SIZE) + return false; + return true; +} + +struct list_head *zlib_alloc_workspace(struct btrfs_fs_info *fs_info, unsigned int level) +{ + struct workspace *workspace; + int workspacesize; + + workspace = kzalloc_obj(*workspace); + if (!workspace) + return ERR_PTR(-ENOMEM); + + workspacesize = max(zlib_deflate_workspacesize(MAX_WBITS, MAX_MEM_LEVEL), + zlib_inflate_workspacesize()); + workspace->strm.workspace = kvzalloc(workspacesize, GFP_KERNEL | __GFP_NOWARN); + workspace->level = level; + workspace->buf = NULL; + if (need_special_buffer(fs_info)) { + workspace->buf = kmalloc(ZLIB_DFLTCC_BUF_SIZE, + __GFP_NOMEMALLOC | __GFP_NORETRY | + __GFP_NOWARN | GFP_NOIO); + workspace->buf_size = ZLIB_DFLTCC_BUF_SIZE; + } + if (!workspace->buf) { + workspace->buf = kmalloc(fs_info->sectorsize, GFP_KERNEL); + workspace->buf_size = fs_info->sectorsize; + } + if (!workspace->strm.workspace || !workspace->buf) + goto fail; + + INIT_LIST_HEAD(&workspace->list); + + return &workspace->list; +fail: + zlib_free_workspace(&workspace->list); + return ERR_PTR(-ENOMEM); +} + +/* + * Helper for S390x with hardware zlib compression support. + * + * That hardware acceleration requires a buffer size larger than a single page + * to get ideal performance, thus we need to do the memory copy rather than + * use the page cache directly as input buffer. + */ +static int copy_data_into_buffer(struct address_space *mapping, + struct workspace *workspace, u64 filepos, + unsigned long length) +{ + u64 cur = filepos; + + /* It's only for hardware accelerated zlib code. */ + ASSERT(zlib_deflate_dfltcc_enabled()); + + while (cur < filepos + length) { + struct folio *folio; + void *data_in; + unsigned int offset; + unsigned long copy_length; + int ret; + + ret = btrfs_compress_filemap_get_folio(mapping, cur, &folio); + if (ret < 0) + return ret; + + offset = offset_in_folio(folio, cur); + copy_length = min(folio_size(folio) - offset, + filepos + length - cur); + + data_in = kmap_local_folio(folio, offset); + memcpy(workspace->buf + cur - filepos, data_in, copy_length); + kunmap_local(data_in); + folio_put(folio); + cur += copy_length; + } + return 0; +} + +int zlib_compress_bio(struct list_head *ws, struct compressed_bio *cb) +{ + struct btrfs_inode *inode = cb->bbio.inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct workspace *workspace = list_entry(ws, struct workspace, list); + struct address_space *mapping = inode->vfs_inode.i_mapping; + struct bio *bio = &cb->bbio.bio; + u64 start = cb->start; + u32 len = cb->len; + const u32 min_folio_size = btrfs_min_folio_size(fs_info); + int ret; + char *data_in = NULL; + struct folio *in_folio = NULL; + struct folio *out_folio = NULL; + const u64 orig_end = start + len; + + ret = zlib_deflateInit(&workspace->strm, workspace->level); + if (unlikely(ret != Z_OK)) { + btrfs_err(fs_info, + "zlib compression init failed, error %d root %llu inode %llu offset %llu", + ret, btrfs_root_id(inode->root), btrfs_ino(inode), start); + ret = -EIO; + goto out; + } + + workspace->strm.total_in = 0; + workspace->strm.total_out = 0; + + out_folio = btrfs_alloc_compr_folio(fs_info, GFP_NOFS); + if (out_folio == NULL) { + ret = -ENOMEM; + goto out; + } + + workspace->strm.next_in = workspace->buf; + workspace->strm.avail_in = 0; + workspace->strm.next_out = folio_address(out_folio); + workspace->strm.avail_out = min_folio_size; + + while (workspace->strm.total_in < len) { + /* + * Get next input pages and copy the contents to the workspace + * buffer if required. + */ + if (workspace->strm.avail_in == 0) { + unsigned long bytes_left = len - workspace->strm.total_in; + unsigned int copy_length = min(bytes_left, workspace->buf_size); + + /* + * For s390 hardware accelerated zlib, and our folio is smaller + * than the copy_length, we need to fill the buffer so that + * we can take full advantage of hardware acceleration. + */ + if (need_special_buffer(fs_info)) { + ret = copy_data_into_buffer(mapping, workspace, + start, copy_length); + if (ret < 0) + goto out; + start += copy_length; + workspace->strm.next_in = workspace->buf; + workspace->strm.avail_in = copy_length; + } else { + unsigned int cur_len; + + if (data_in) { + kunmap_local(data_in); + folio_put(in_folio); + data_in = NULL; + } + ret = btrfs_compress_filemap_get_folio(mapping, + start, &in_folio); + if (ret < 0) + goto out; + cur_len = btrfs_calc_input_length(in_folio, orig_end, start); + data_in = kmap_local_folio(in_folio, + offset_in_folio(in_folio, start)); + start += cur_len; + workspace->strm.next_in = data_in; + workspace->strm.avail_in = cur_len; + } + } + + ret = zlib_deflate(&workspace->strm, Z_SYNC_FLUSH); + if (unlikely(ret != Z_OK)) { + btrfs_warn(fs_info, + "zlib compression failed, error %d root %llu inode %llu offset %llu", + ret, btrfs_root_id(inode->root), btrfs_ino(inode), + start); + zlib_deflateEnd(&workspace->strm); + ret = -EIO; + goto out; + } + + /* We're making it bigger, give up. */ + if (workspace->strm.total_in > fs_info->sectorsize * 2 && + workspace->strm.total_in < workspace->strm.total_out) { + ret = -E2BIG; + goto out; + } + if (workspace->strm.total_out >= len) { + ret = -E2BIG; + goto out; + } + /* Queue the full folio and allocate a new one. */ + if (workspace->strm.avail_out == 0) { + if (!bio_add_folio(bio, out_folio, folio_size(out_folio), 0)) { + ret = -E2BIG; + goto out; + } + + out_folio = btrfs_alloc_compr_folio(fs_info, GFP_NOFS); + if (out_folio == NULL) { + ret = -ENOMEM; + goto out; + } + workspace->strm.avail_out = min_folio_size; + workspace->strm.next_out = folio_address(out_folio); + } + /* We're all done. */ + if (workspace->strm.total_in >= len) + break; + } + + workspace->strm.avail_in = 0; + + /* + * Call deflate with Z_FINISH flush parameter providing more output + * space but no more input data, until it returns with Z_STREAM_END. + */ + while (ret != Z_STREAM_END) { + ret = zlib_deflate(&workspace->strm, Z_FINISH); + if (ret == Z_STREAM_END) + break; + if (unlikely(ret != Z_OK && ret != Z_BUF_ERROR)) { + zlib_deflateEnd(&workspace->strm); + ret = -EIO; + goto out; + } else if (workspace->strm.avail_out == 0) { + if (workspace->strm.total_out >= len) { + ret = -E2BIG; + goto out; + } + if (!bio_add_folio(bio, out_folio, folio_size(out_folio), 0)) { + ret = -E2BIG; + goto out; + } + /* Get another folio for the stream end. */ + out_folio = btrfs_alloc_compr_folio(fs_info, GFP_NOFS); + if (out_folio == NULL) { + ret = -ENOMEM; + goto out; + } + workspace->strm.avail_out = min_folio_size; + workspace->strm.next_out = folio_address(out_folio); + } + } + /* Queue the remaining part of the folio. */ + if (workspace->strm.total_out > bio->bi_iter.bi_size) { + const u32 cur_len = workspace->strm.total_out - bio->bi_iter.bi_size; + + ASSERT(cur_len <= folio_size(out_folio)); + + if (!bio_add_folio(bio, out_folio, cur_len, 0)) { + ret = -E2BIG; + goto out; + } + } else { + /* The last folio hasn't' been utilized. */ + btrfs_free_compr_folio(out_folio); + } + out_folio = NULL; + ASSERT(bio->bi_iter.bi_size == workspace->strm.total_out); + zlib_deflateEnd(&workspace->strm); + + if (workspace->strm.total_out >= workspace->strm.total_in) { + ret = -E2BIG; + goto out; + } + + ret = 0; +out: + if (out_folio) + btrfs_free_compr_folio(out_folio); + if (data_in) { + kunmap_local(data_in); + folio_put(in_folio); + } + + return ret; +} + +int zlib_decompress_bio(struct list_head *ws, struct compressed_bio *cb) +{ + struct btrfs_fs_info *fs_info = cb_to_fs_info(cb); + struct workspace *workspace = list_entry(ws, struct workspace, list); + struct folio_iter fi; + const u32 min_folio_size = btrfs_min_folio_size(fs_info); + int ret = 0, ret2; + int wbits = MAX_WBITS; + char *data_in; + size_t total_out = 0; + const size_t srclen = bio_get_size(&cb->bbio.bio); + unsigned long buf_start; + + bio_first_folio(&fi, &cb->bbio.bio, 0); + + /* We must have at least one folio here, that has the correct size. */ + if (unlikely(!fi.folio)) + return -EINVAL; + ASSERT(folio_size(fi.folio) == min_folio_size); + + data_in = kmap_local_folio(fi.folio, 0); + workspace->strm.next_in = data_in; + workspace->strm.avail_in = min_t(size_t, srclen, min_folio_size); + workspace->strm.total_in = 0; + + workspace->strm.total_out = 0; + workspace->strm.next_out = workspace->buf; + workspace->strm.avail_out = workspace->buf_size; + + /* If it's deflate, and it's got no preset dictionary, then + we can tell zlib to skip the adler32 check. */ + if (srclen > 2 && !(data_in[1] & PRESET_DICT) && + ((data_in[0] & 0x0f) == Z_DEFLATED) && + !(((data_in[0]<<8) + data_in[1]) % 31)) { + + wbits = -((data_in[0] >> 4) + 8); + workspace->strm.next_in += 2; + workspace->strm.avail_in -= 2; + } + + ret = zlib_inflateInit2(&workspace->strm, wbits); + if (unlikely(ret != Z_OK)) { + struct btrfs_inode *inode = cb->bbio.inode; + + kunmap_local(data_in); + btrfs_err(inode->root->fs_info, + "zlib decompression init failed, error %d root %llu inode %llu offset %llu", + ret, btrfs_root_id(inode->root), btrfs_ino(inode), cb->start); + return -EIO; + } + while (workspace->strm.total_in < srclen) { + ret = zlib_inflate(&workspace->strm, Z_NO_FLUSH); + if (ret != Z_OK && ret != Z_STREAM_END) + break; + + buf_start = total_out; + total_out = workspace->strm.total_out; + + /* we didn't make progress in this inflate call, we're done */ + if (buf_start == total_out) + break; + + ret2 = btrfs_decompress_buf2page(workspace->buf, + total_out - buf_start, cb, buf_start); + if (ret2 == 0) { + ret = 0; + goto done; + } + + workspace->strm.next_out = workspace->buf; + workspace->strm.avail_out = workspace->buf_size; + + if (workspace->strm.avail_in == 0) { + unsigned long tmp; + kunmap_local(data_in); + bio_next_folio(&fi, &cb->bbio.bio); + if (!fi.folio) { + data_in = NULL; + break; + } + ASSERT(folio_size(fi.folio) == min_folio_size); + data_in = kmap_local_folio(fi.folio, 0); + workspace->strm.next_in = data_in; + tmp = srclen - workspace->strm.total_in; + workspace->strm.avail_in = min(tmp, min_folio_size); + } + } + if (unlikely(ret != Z_STREAM_END)) { + btrfs_err(cb->bbio.inode->root->fs_info, + "zlib decompression failed, error %d root %llu inode %llu offset %llu", + ret, btrfs_root_id(cb->bbio.inode->root), + btrfs_ino(cb->bbio.inode), cb->start); + ret = -EIO; + } else { + ret = 0; + } +done: + zlib_inflateEnd(&workspace->strm); + if (data_in) + kunmap_local(data_in); + return ret; +} + +int zlib_decompress(struct list_head *ws, const u8 *data_in, + struct folio *dest_folio, unsigned long dest_pgoff, size_t srclen, + size_t destlen) +{ + struct workspace *workspace = list_entry(ws, struct workspace, list); + int ret = 0; + int wbits = MAX_WBITS; + unsigned long to_copy; + + workspace->strm.next_in = data_in; + workspace->strm.avail_in = srclen; + workspace->strm.total_in = 0; + + workspace->strm.next_out = workspace->buf; + workspace->strm.avail_out = workspace->buf_size; + workspace->strm.total_out = 0; + /* If it's deflate, and it's got no preset dictionary, then + we can tell zlib to skip the adler32 check. */ + if (srclen > 2 && !(data_in[1] & PRESET_DICT) && + ((data_in[0] & 0x0f) == Z_DEFLATED) && + !(((data_in[0]<<8) + data_in[1]) % 31)) { + + wbits = -((data_in[0] >> 4) + 8); + workspace->strm.next_in += 2; + workspace->strm.avail_in -= 2; + } + + ret = zlib_inflateInit2(&workspace->strm, wbits); + if (unlikely(ret != Z_OK)) { + struct btrfs_inode *inode = folio_to_inode(dest_folio); + + btrfs_err(inode->root->fs_info, + "zlib decompression init failed, error %d root %llu inode %llu offset %llu", + ret, btrfs_root_id(inode->root), btrfs_ino(inode), + folio_pos(dest_folio)); + return -EIO; + } + + /* + * Everything (in/out buf) should be at most one sector, there should + * be no need to switch any input/output buffer. + */ + ret = zlib_inflate(&workspace->strm, Z_FINISH); + to_copy = min(workspace->strm.total_out, destlen); + if (ret != Z_STREAM_END) + goto out; + + memcpy_to_folio(dest_folio, dest_pgoff, workspace->buf, to_copy); + +out: + if (unlikely(to_copy != destlen)) { + struct btrfs_inode *inode = folio_to_inode(dest_folio); + + btrfs_err(inode->root->fs_info, +"zlib decompression failed, error %d root %llu inode %llu offset %llu decompressed %lu expected %zu", + ret, btrfs_root_id(inode->root), btrfs_ino(inode), + folio_pos(dest_folio), to_copy, destlen); + ret = -EIO; + } else { + ret = 0; + } + + zlib_inflateEnd(&workspace->strm); + + if (unlikely(to_copy < destlen)) + folio_zero_range(dest_folio, dest_pgoff + to_copy, destlen - to_copy); + return ret; +} + +const struct btrfs_compress_levels btrfs_zlib_compress = { + .min_level = 1, + .max_level = 9, + .default_level = BTRFS_ZLIB_DEFAULT_LEVEL, +}; diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c new file mode 100644 index 000000000..a016cb471 --- /dev/null +++ b/fs/btrfs/zoned.c @@ -0,0 +1,3286 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include +#include +#include +#include "ctree.h" +#include "volumes.h" +#include "zoned.h" +#include "disk-io.h" +#include "block-group.h" +#include "dev-replace.h" +#include "space-info.h" +#include "fs.h" +#include "accessors.h" +#include "bio.h" +#include "transaction.h" +#include "sysfs.h" + +/* Maximum number of zones to report per blkdev_report_zones() call */ +#define BTRFS_REPORT_NR_ZONES 4096 +/* Invalid allocation pointer value for missing devices */ +#define WP_MISSING_DEV ((u64)-1) +/* Pseudo write pointer value for conventional zone */ +#define WP_CONVENTIONAL ((u64)-2) + +/* + * Location of the first zone of superblock logging zone pairs. + * + * - primary superblock: 0B (zone 0) + * - first copy: 512G (zone starting at that offset) + * - second copy: 4T (zone starting at that offset) + */ +#define BTRFS_SB_LOG_PRIMARY_OFFSET (0ULL) +#define BTRFS_SB_LOG_FIRST_OFFSET (512ULL * SZ_1G) +#define BTRFS_SB_LOG_SECOND_OFFSET (4096ULL * SZ_1G) + +#define BTRFS_SB_LOG_FIRST_SHIFT ilog2(BTRFS_SB_LOG_FIRST_OFFSET) +#define BTRFS_SB_LOG_SECOND_SHIFT ilog2(BTRFS_SB_LOG_SECOND_OFFSET) + +/* Number of superblock log zones */ +#define BTRFS_NR_SB_LOG_ZONES 2 + +/* Default number of max active zones when the device has no limits. */ +#define BTRFS_DEFAULT_MAX_ACTIVE_ZONES 128 + +/* + * Minimum of active zones we need: + * + * - BTRFS_SUPER_MIRROR_MAX zones for superblock mirrors + * - 3 zones to ensure at least one zone per SYSTEM, META and DATA block group + * - 1 zone for tree-log dedicated block group + * - 1 zone for relocation + */ +#define BTRFS_MIN_ACTIVE_ZONES (BTRFS_SUPER_MIRROR_MAX + 5) + +/* + * Minimum / maximum supported zone size. Currently, SMR disks have a zone + * size of 256MiB, and we are expecting ZNS drives to be in the 1-4GiB range. + * We do not expect the zone size to become larger than 8GiB or smaller than + * 4MiB in the near future. + */ +#define BTRFS_MAX_ZONE_SIZE SZ_8G +#define BTRFS_MIN_ZONE_SIZE SZ_4M + +#define SUPER_INFO_SECTORS ((u64)BTRFS_SUPER_INFO_SIZE >> SECTOR_SHIFT) + +static void wait_eb_writebacks(struct btrfs_block_group *block_group); +static int do_zone_finish(struct btrfs_block_group *block_group, bool fully_written); + +static inline bool sb_zone_is_full(const struct blk_zone *zone) +{ + return (zone->cond == BLK_ZONE_COND_FULL) || + (zone->wp + SUPER_INFO_SECTORS > zone->start + zone->capacity); +} + +static int copy_zone_info_cb(struct blk_zone *zone, unsigned int idx, void *data) +{ + struct blk_zone *zones = data; + + memcpy(&zones[idx], zone, sizeof(*zone)); + + return 0; +} + +static int sb_write_pointer(struct block_device *bdev, struct blk_zone *zones, + u64 *wp_ret) +{ + bool empty[BTRFS_NR_SB_LOG_ZONES]; + bool full[BTRFS_NR_SB_LOG_ZONES]; + sector_t sector; + + for (int i = 0; i < BTRFS_NR_SB_LOG_ZONES; i++) { + ASSERT(zones[i].type != BLK_ZONE_TYPE_CONVENTIONAL, + "zones[%d].type=%d", i, zones[i].type); + empty[i] = (zones[i].cond == BLK_ZONE_COND_EMPTY); + full[i] = sb_zone_is_full(&zones[i]); + } + + /* + * Possible states of log buffer zones + * + * Empty[0] In use[0] Full[0] + * Empty[1] * 0 1 + * In use[1] x x 1 + * Full[1] 0 0 C + * + * Log position: + * *: Special case, no superblock is written + * 0: Use write pointer of zones[0] + * 1: Use write pointer of zones[1] + * C: Compare super blocks from zones[0] and zones[1], use the latest + * one determined by generation + * x: Invalid state + */ + + if (empty[0] && empty[1]) { + /* Special case to distinguish no superblock to read */ + *wp_ret = zones[0].start << SECTOR_SHIFT; + return -ENOENT; + } else if (full[0] && full[1]) { + /* Compare two super blocks */ + struct address_space *mapping = bdev->bd_mapping; + struct page *page[BTRFS_NR_SB_LOG_ZONES]; + struct btrfs_super_block *super[BTRFS_NR_SB_LOG_ZONES]; + + for (int i = 0; i < BTRFS_NR_SB_LOG_ZONES; i++) { + u64 zone_end = (zones[i].start + zones[i].capacity) << SECTOR_SHIFT; + u64 bytenr = ALIGN_DOWN(zone_end, BTRFS_SUPER_INFO_SIZE) - + BTRFS_SUPER_INFO_SIZE; + + filemap_invalidate_lock_shared(mapping); + page[i] = read_cache_page_gfp(mapping, + bytenr >> PAGE_SHIFT, GFP_NOFS); + filemap_invalidate_unlock_shared(mapping); + if (IS_ERR(page[i])) { + if (i == 1) + btrfs_release_disk_super(super[0]); + return PTR_ERR(page[i]); + } + super[i] = page_address(page[i]); + } + + if (btrfs_super_generation(super[0]) > + btrfs_super_generation(super[1])) + sector = zones[1].start; + else + sector = zones[0].start; + + for (int i = 0; i < BTRFS_NR_SB_LOG_ZONES; i++) + btrfs_release_disk_super(super[i]); + } else if (!full[0] && (empty[1] || full[1])) { + sector = zones[0].wp; + } else if (full[0]) { + sector = zones[1].wp; + } else { + return -EUCLEAN; + } + *wp_ret = sector << SECTOR_SHIFT; + return 0; +} + +/* + * Get the first zone number of the superblock mirror + */ +static inline u32 sb_zone_number(int shift, int mirror) +{ + u64 zone = U64_MAX; + + ASSERT(mirror < BTRFS_SUPER_MIRROR_MAX, "mirror=%d", mirror); + switch (mirror) { + case 0: zone = 0; break; + case 1: zone = 1ULL << (BTRFS_SB_LOG_FIRST_SHIFT - shift); break; + case 2: zone = 1ULL << (BTRFS_SB_LOG_SECOND_SHIFT - shift); break; + } + + ASSERT(zone <= U32_MAX, "zone=%llu", zone); + + return (u32)zone; +} + +static inline sector_t zone_start_sector(u32 zone_number, + struct block_device *bdev) +{ + return (sector_t)zone_number << ilog2(bdev_zone_sectors(bdev)); +} + +static inline u64 zone_start_physical(u32 zone_number, + struct btrfs_zoned_device_info *zone_info) +{ + return (u64)zone_number << zone_info->zone_size_shift; +} + +/* + * Emulate blkdev_report_zones() for a non-zoned device. It slices up the block + * device into static sized chunks and fake a conventional zone on each of + * them. + */ +static int emulate_report_zones(struct btrfs_device *device, u64 pos, + struct blk_zone *zones, unsigned int nr_zones) +{ + const sector_t zone_sectors = device->fs_info->zone_size >> SECTOR_SHIFT; + sector_t bdev_size = bdev_nr_sectors(device->bdev); + unsigned int i; + + pos >>= SECTOR_SHIFT; + for (i = 0; i < nr_zones; i++) { + zones[i].start = i * zone_sectors + pos; + zones[i].len = zone_sectors; + zones[i].capacity = zone_sectors; + zones[i].wp = zones[i].start + zone_sectors; + zones[i].type = BLK_ZONE_TYPE_CONVENTIONAL; + zones[i].cond = BLK_ZONE_COND_NOT_WP; + + if (zones[i].wp >= bdev_size) { + i++; + break; + } + } + + return i; +} + +static int btrfs_get_dev_zones(struct btrfs_device *device, u64 pos, + struct blk_zone *zones, unsigned int *nr_zones) +{ + struct btrfs_zoned_device_info *zinfo = device->zone_info; + int ret; + + if (!*nr_zones) + return 0; + + if (!bdev_is_zoned(device->bdev)) { + ret = emulate_report_zones(device, pos, zones, *nr_zones); + *nr_zones = ret; + return 0; + } + + /* Check cache */ + if (zinfo->zone_cache) { + unsigned int i; + u32 zno; + + ASSERT(IS_ALIGNED(pos, zinfo->zone_size), + "pos=%llu zinfo->zone_size=%llu", pos, zinfo->zone_size); + zno = pos >> zinfo->zone_size_shift; + /* + * We cannot report zones beyond the zone end. So, it is OK to + * cap *nr_zones to at the end. + */ + *nr_zones = min_t(u32, *nr_zones, zinfo->nr_zones - zno); + + for (i = 0; i < *nr_zones; i++) { + struct blk_zone *zone_info; + + zone_info = &zinfo->zone_cache[zno + i]; + if (!zone_info->len) + break; + } + + if (i == *nr_zones) { + /* Cache hit on all the zones */ + memcpy(zones, zinfo->zone_cache + zno, + sizeof(*zinfo->zone_cache) * *nr_zones); + return 0; + } + } + + ret = blkdev_report_zones_cached(device->bdev, pos >> SECTOR_SHIFT, + *nr_zones, copy_zone_info_cb, zones); + if (ret < 0) { + btrfs_err(device->fs_info, + "zoned: failed to read zone %llu on %s (devid %llu)", + pos, rcu_dereference(device->name), + device->devid); + return ret; + } + *nr_zones = ret; + if (unlikely(!ret)) + return -EIO; + + /* Populate cache */ + if (zinfo->zone_cache) { + u32 zno = pos >> zinfo->zone_size_shift; + + memcpy(zinfo->zone_cache + zno, zones, + sizeof(*zinfo->zone_cache) * *nr_zones); + } + + return 0; +} + +/* The emulated zone size is determined from the size of device extent */ +static int calculate_emulated_zone_size(struct btrfs_fs_info *fs_info) +{ + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_root *root = fs_info->dev_root; + struct btrfs_key key; + struct extent_buffer *leaf; + struct btrfs_dev_extent *dext; + int ret = 0; + + key.objectid = 1; + key.type = BTRFS_DEV_EXTENT_KEY; + key.offset = 0; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + if (ret < 0) + return ret; + + if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) { + ret = btrfs_next_leaf(root, path); + if (ret < 0) + return ret; + /* No dev extents at all? Not good */ + if (unlikely(ret > 0)) + return -EUCLEAN; + } + + leaf = path->nodes[0]; + dext = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_extent); + fs_info->zone_size = btrfs_dev_extent_length(leaf, dext); + return 0; +} + +int btrfs_get_dev_zone_info_all_devices(struct btrfs_fs_info *fs_info) +{ + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; + struct btrfs_device *device; + int ret = 0; + + /* fs_info->zone_size might not set yet. Use the incomapt flag here. */ + if (!btrfs_fs_incompat(fs_info, ZONED)) + return 0; + + /* + * No need to take the device_list mutex here, we're still in the mount + * path and devices cannot be added to or removed from the list yet. + */ + list_for_each_entry(device, &fs_devices->devices, dev_list) { + /* We can skip reading of zone info for missing devices */ + if (!device->bdev) + continue; + + ret = btrfs_get_dev_zone_info(device, true); + if (ret) + break; + } + + return ret; +} + +static int btrfs_get_max_active_zones(struct btrfs_device *device, + struct btrfs_zoned_device_info *zone_info) +{ + struct block_device *bdev = device->bdev; + int max_active_zones; + + if (unlikely(zone_info->nr_zones < BTRFS_MIN_ACTIVE_ZONES)) { + btrfs_err(device->fs_info, "zoned: not enough zones to mount filesystem: %u < %d", + zone_info->nr_zones, BTRFS_MIN_ACTIVE_ZONES); + return -EINVAL; + } + + max_active_zones = min_not_zero(bdev_max_active_zones(bdev), + bdev_max_open_zones(bdev)); + if (max_active_zones == 0) + max_active_zones = min(zone_info->nr_zones / 4, + BTRFS_DEFAULT_MAX_ACTIVE_ZONES); + + zone_info->max_active_zones = max(max_active_zones, BTRFS_MIN_ACTIVE_ZONES); + return 0; +} + +int btrfs_get_dev_zone_info(struct btrfs_device *device, bool populate_cache) +{ + struct btrfs_fs_info *fs_info = device->fs_info; + struct btrfs_zoned_device_info *zone_info = NULL; + struct block_device *bdev = device->bdev; + unsigned int nactive; + sector_t nr_sectors; + sector_t sector = 0; + struct blk_zone *zones = NULL; + unsigned int i, nreported = 0, nr_zones; + sector_t zone_sectors; + char *model, *emulated; + int ret; + + /* + * Cannot use btrfs_is_zoned here, since fs_info::zone_size might not + * yet be set. + */ + if (!btrfs_fs_incompat(fs_info, ZONED)) + return 0; + + if (device->zone_info) + return 0; + + zone_info = kzalloc_obj(*zone_info); + if (!zone_info) + return -ENOMEM; + + device->zone_info = zone_info; + + if (!bdev_is_zoned(bdev)) { + if (!fs_info->zone_size) { + ret = calculate_emulated_zone_size(fs_info); + if (ret) + goto out; + } + + ASSERT(fs_info->zone_size); + zone_sectors = fs_info->zone_size >> SECTOR_SHIFT; + } else { + zone_sectors = bdev_zone_sectors(bdev); + } + + ASSERT(is_power_of_two_u64(zone_sectors)); + zone_info->zone_size = zone_sectors << SECTOR_SHIFT; + + /* We reject devices with a zone size larger than 8GB */ + if (zone_info->zone_size > BTRFS_MAX_ZONE_SIZE) { + btrfs_err(fs_info, + "zoned: %s: zone size %llu larger than supported maximum %llu", + rcu_dereference(device->name), + zone_info->zone_size, BTRFS_MAX_ZONE_SIZE); + ret = -EINVAL; + goto out; + } else if (zone_info->zone_size < BTRFS_MIN_ZONE_SIZE) { + btrfs_err(fs_info, + "zoned: %s: zone size %llu smaller than supported minimum %u", + rcu_dereference(device->name), + zone_info->zone_size, BTRFS_MIN_ZONE_SIZE); + ret = -EINVAL; + goto out; + } + + nr_sectors = bdev_nr_sectors(bdev); + zone_info->zone_size_shift = ilog2(zone_info->zone_size); + zone_info->nr_zones = nr_sectors >> ilog2(zone_sectors); + if (!IS_ALIGNED(nr_sectors, zone_sectors)) + zone_info->nr_zones++; + + ret = btrfs_get_max_active_zones(device, zone_info); + if (ret) + goto out; + + zone_info->seq_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL); + if (!zone_info->seq_zones) { + ret = -ENOMEM; + goto out; + } + + zone_info->empty_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL); + if (!zone_info->empty_zones) { + ret = -ENOMEM; + goto out; + } + + zone_info->active_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL); + if (!zone_info->active_zones) { + ret = -ENOMEM; + goto out; + } + + zones = kvzalloc_objs(struct blk_zone, BTRFS_REPORT_NR_ZONES); + if (!zones) { + ret = -ENOMEM; + goto out; + } + + /* + * Enable zone cache only for a zoned device. On a non-zoned device, we + * fill the zone info with emulated CONVENTIONAL zones, so no need to + * use the cache. + */ + if (populate_cache && bdev_is_zoned(device->bdev)) { + zone_info->zone_cache = vcalloc(zone_info->nr_zones, + sizeof(struct blk_zone)); + if (!zone_info->zone_cache) { + btrfs_err(device->fs_info, + "zoned: failed to allocate zone cache for %s", + rcu_dereference(device->name)); + ret = -ENOMEM; + goto out; + } + } + + /* Get zones type */ + nactive = 0; + while (sector < nr_sectors) { + nr_zones = BTRFS_REPORT_NR_ZONES; + ret = btrfs_get_dev_zones(device, sector << SECTOR_SHIFT, zones, + &nr_zones); + if (ret) + goto out; + + for (i = 0; i < nr_zones; i++) { + if (zones[i].type == BLK_ZONE_TYPE_SEQWRITE_REQ) + __set_bit(nreported, zone_info->seq_zones); + switch (zones[i].cond) { + case BLK_ZONE_COND_EMPTY: + __set_bit(nreported, zone_info->empty_zones); + break; + case BLK_ZONE_COND_IMP_OPEN: + case BLK_ZONE_COND_EXP_OPEN: + case BLK_ZONE_COND_CLOSED: + case BLK_ZONE_COND_ACTIVE: + __set_bit(nreported, zone_info->active_zones); + nactive++; + break; + } + nreported++; + } + sector = zones[nr_zones - 1].start + zones[nr_zones - 1].len; + } + + if (unlikely(nreported != zone_info->nr_zones)) { + btrfs_err(device->fs_info, + "inconsistent number of zones on %s (%u/%u)", + rcu_dereference(device->name), nreported, + zone_info->nr_zones); + ret = -EIO; + goto out; + } + + if (unlikely(nactive > zone_info->max_active_zones)) { + if (bdev_max_active_zones(bdev) > 0) { + btrfs_err(device->fs_info, + "zoned: %u active zones on %s exceeds max_active_zones %u", + nactive, rcu_dereference(device->name), + zone_info->max_active_zones); + ret = -EIO; + goto out; + } + + /* + * This is for backward compatibility with old filesystems that + * have a lot of active zones because the device doesn't report + * a maximum number of zones and we previously didn't care for + * the limit. + */ + zone_info->max_active_zones = 0; + } else { + atomic_set(&zone_info->active_zones_left, + zone_info->max_active_zones - nactive); + set_bit(BTRFS_FS_ACTIVE_ZONE_TRACKING, &fs_info->flags); + } + + /* Validate superblock log */ + nr_zones = BTRFS_NR_SB_LOG_ZONES; + for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) { + u32 sb_zone; + u64 sb_wp; + int sb_pos = BTRFS_NR_SB_LOG_ZONES * i; + + sb_zone = sb_zone_number(zone_info->zone_size_shift, i); + if (sb_zone + 1 >= zone_info->nr_zones) + continue; + + ret = btrfs_get_dev_zones(device, + zone_start_physical(sb_zone, zone_info), + &zone_info->sb_zones[sb_pos], + &nr_zones); + if (ret) + goto out; + + if (unlikely(nr_zones != BTRFS_NR_SB_LOG_ZONES)) { + btrfs_err(device->fs_info, + "zoned: failed to read super block log zone info at devid %llu zone %u", + device->devid, sb_zone); + ret = -EUCLEAN; + goto out; + } + + /* + * If zones[0] is conventional, always use the beginning of the + * zone to record superblock. No need to validate in that case. + */ + if (zone_info->sb_zones[BTRFS_NR_SB_LOG_ZONES * i].type == + BLK_ZONE_TYPE_CONVENTIONAL) + continue; + + ret = sb_write_pointer(device->bdev, + &zone_info->sb_zones[sb_pos], &sb_wp); + if (unlikely(ret != -ENOENT && ret)) { + btrfs_err(device->fs_info, + "zoned: super block log zone corrupted devid %llu zone %u", + device->devid, sb_zone); + ret = -EUCLEAN; + goto out; + } + } + + + kvfree(zones); + + if (bdev_is_zoned(bdev)) { + model = "host-managed zoned"; + emulated = ""; + } else { + model = "regular"; + emulated = "emulated "; + } + + btrfs_info(fs_info, + "%s block device %s, %u %szones of %llu bytes", + model, rcu_dereference(device->name), zone_info->nr_zones, + emulated, zone_info->zone_size); + + return 0; + +out: + kvfree(zones); + btrfs_destroy_dev_zone_info(device); + return ret; +} + +void btrfs_destroy_dev_zone_info(struct btrfs_device *device) +{ + struct btrfs_zoned_device_info *zone_info = device->zone_info; + + if (!zone_info) + return; + + bitmap_free(zone_info->active_zones); + bitmap_free(zone_info->seq_zones); + bitmap_free(zone_info->empty_zones); + vfree(zone_info->zone_cache); + kfree(zone_info); + device->zone_info = NULL; +} + +struct btrfs_zoned_device_info *btrfs_clone_dev_zone_info(struct btrfs_device *orig_dev) +{ + struct btrfs_zoned_device_info *zone_info; + + zone_info = kmemdup(orig_dev->zone_info, sizeof(*zone_info), GFP_KERNEL); + if (!zone_info) + return NULL; + + zone_info->seq_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL); + if (!zone_info->seq_zones) + goto out; + + bitmap_copy(zone_info->seq_zones, orig_dev->zone_info->seq_zones, + zone_info->nr_zones); + + zone_info->empty_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL); + if (!zone_info->empty_zones) + goto out; + + bitmap_copy(zone_info->empty_zones, orig_dev->zone_info->empty_zones, + zone_info->nr_zones); + + zone_info->active_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL); + if (!zone_info->active_zones) + goto out; + + bitmap_copy(zone_info->active_zones, orig_dev->zone_info->active_zones, + zone_info->nr_zones); + zone_info->zone_cache = NULL; + + return zone_info; + +out: + bitmap_free(zone_info->seq_zones); + bitmap_free(zone_info->empty_zones); + bitmap_free(zone_info->active_zones); + kfree(zone_info); + return NULL; +} + +static int btrfs_get_dev_zone(struct btrfs_device *device, u64 pos, struct blk_zone *zone) +{ + unsigned int nr_zones = 1; + int ret; + + ret = btrfs_get_dev_zones(device, pos, zone, &nr_zones); + if (ret != 0 || !nr_zones) + return ret ? ret : -EIO; + + return 0; +} + +static int btrfs_check_for_zoned_device(struct btrfs_fs_info *fs_info) +{ + struct btrfs_device *device; + + list_for_each_entry(device, &fs_info->fs_devices->devices, dev_list) { + if (device->bdev && bdev_is_zoned(device->bdev)) { + btrfs_err(fs_info, + "zoned: mode not enabled but zoned device found: %pg", + device->bdev); + return -EINVAL; + } + } + + return 0; +} + +int btrfs_check_zoned_mode(struct btrfs_fs_info *fs_info) +{ + struct queue_limits *lim = &fs_info->limits; + struct btrfs_device *device; + u64 zone_size = 0; + int ret; + + /* + * Host-Managed devices can't be used without the ZONED flag. With the + * ZONED all devices can be used, using zone emulation if required. + */ + if (!btrfs_fs_incompat(fs_info, ZONED)) + return btrfs_check_for_zoned_device(fs_info); + + blk_set_stacking_limits(lim); + + list_for_each_entry(device, &fs_info->fs_devices->devices, dev_list) { + struct btrfs_zoned_device_info *zone_info = device->zone_info; + + if (!device->bdev) + continue; + + if (!zone_size) { + zone_size = zone_info->zone_size; + } else if (zone_info->zone_size != zone_size) { + btrfs_err(fs_info, + "zoned: unequal block device zone sizes: have %llu found %llu", + zone_info->zone_size, zone_size); + return -EINVAL; + } + + /* + * With the zoned emulation, we can have non-zoned device on the + * zoned mode. In this case, we don't have a valid max zone + * append size. + */ + if (bdev_is_zoned(device->bdev)) + blk_stack_limits(lim, bdev_limits(device->bdev), 0); + } + + ret = blk_validate_limits(lim); + if (ret) { + btrfs_err(fs_info, "zoned: failed to validate queue limits"); + return ret; + } + + /* + * stripe_size is always aligned to BTRFS_STRIPE_LEN in + * btrfs_create_chunk(). Since we want stripe_len == zone_size, + * check the alignment here. + */ + if (!IS_ALIGNED(zone_size, BTRFS_STRIPE_LEN)) { + btrfs_err(fs_info, + "zoned: zone size %llu not aligned to stripe %u", + zone_size, BTRFS_STRIPE_LEN); + return -EINVAL; + } + + if (btrfs_fs_incompat(fs_info, MIXED_GROUPS)) { + btrfs_err(fs_info, "zoned: mixed block groups not supported"); + return -EINVAL; + } + + fs_info->zone_size = zone_size; + /* + * Also limit max_zone_append_size by max_segments * PAGE_SIZE. + * Technically, we can have multiple pages per segment. But, since + * we add the pages one by one to a bio, and cannot increase the + * metadata reservation even if it increases the number of extents, it + * is safe to stick with the limit. + */ + fs_info->max_zone_append_size = ALIGN_DOWN( + min3((u64)lim->max_zone_append_sectors << SECTOR_SHIFT, + (u64)lim->max_sectors << SECTOR_SHIFT, + (u64)lim->max_segments << PAGE_SHIFT), + fs_info->sectorsize); + fs_info->fs_devices->chunk_alloc_policy = BTRFS_CHUNK_ALLOC_ZONED; + + fs_info->max_extent_size = min_not_zero(fs_info->max_extent_size, + fs_info->max_zone_append_size); + + /* + * Check mount options here, because we might change fs_info->zoned + * from fs_info->zone_size. + */ + ret = btrfs_check_mountopts_zoned(fs_info, &fs_info->mount_opt); + if (ret) + return ret; + + btrfs_info(fs_info, "zoned mode enabled with zone size %llu", zone_size); + return 0; +} + +int btrfs_check_mountopts_zoned(const struct btrfs_fs_info *info, + unsigned long long *mount_opt) +{ + if (!btrfs_is_zoned(info)) + return 0; + + /* + * Space cache writing is not COWed. Disable that to avoid write errors + * in sequential zones. + */ + if (btrfs_raw_test_opt(*mount_opt, SPACE_CACHE)) { + btrfs_err(info, "zoned: space cache v1 is not supported"); + return -EINVAL; + } + + if (btrfs_raw_test_opt(*mount_opt, NODATACOW)) { + btrfs_err(info, "zoned: NODATACOW not supported"); + return -EINVAL; + } + + if (btrfs_raw_test_opt(*mount_opt, DISCARD_ASYNC)) { + btrfs_info(info, + "zoned: async discard ignored and disabled for zoned mode"); + btrfs_clear_opt(*mount_opt, DISCARD_ASYNC); + } + + return 0; +} + +static int sb_log_location(struct block_device *bdev, struct blk_zone *zones, + int rw, u64 *bytenr_ret) +{ + u64 wp; + int ret; + + if (zones[0].type == BLK_ZONE_TYPE_CONVENTIONAL) { + *bytenr_ret = zones[0].start << SECTOR_SHIFT; + return 0; + } + + ret = sb_write_pointer(bdev, zones, &wp); + if (ret != -ENOENT && ret < 0) + return ret; + + if (rw == WRITE) { + struct blk_zone *reset = NULL; + + if (wp == zones[0].start << SECTOR_SHIFT) + reset = &zones[0]; + else if (wp == zones[1].start << SECTOR_SHIFT) + reset = &zones[1]; + + if (reset && reset->cond != BLK_ZONE_COND_EMPTY) { + unsigned int nofs_flags; + + ASSERT(sb_zone_is_full(reset)); + + nofs_flags = memalloc_nofs_save(); + ret = blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET, + reset->start, reset->len); + memalloc_nofs_restore(nofs_flags); + if (ret) + return ret; + + reset->cond = BLK_ZONE_COND_EMPTY; + reset->wp = reset->start; + } + } else if (ret != -ENOENT) { + /* + * For READ, we want the previous one. Move write pointer to + * the end of a zone, if it is at the head of a zone. + */ + u64 zone_end = 0; + + if (wp == zones[0].start << SECTOR_SHIFT) + zone_end = zones[1].start + zones[1].capacity; + else if (wp == zones[1].start << SECTOR_SHIFT) + zone_end = zones[0].start + zones[0].capacity; + if (zone_end) + wp = ALIGN_DOWN(zone_end << SECTOR_SHIFT, + BTRFS_SUPER_INFO_SIZE); + + wp -= BTRFS_SUPER_INFO_SIZE; + } + + *bytenr_ret = wp; + return 0; + +} + +int btrfs_sb_log_location_bdev(struct block_device *bdev, int mirror, int rw, + u64 *bytenr_ret) +{ + struct blk_zone zones[BTRFS_NR_SB_LOG_ZONES]; + sector_t zone_sectors; + u32 sb_zone; + int ret; + u8 zone_sectors_shift; + sector_t nr_sectors; + u32 nr_zones; + + if (!bdev_is_zoned(bdev)) { + *bytenr_ret = btrfs_sb_offset(mirror); + return 0; + } + + ASSERT(rw == READ || rw == WRITE); + + zone_sectors = bdev_zone_sectors(bdev); + if (!is_power_of_2(zone_sectors)) + return -EINVAL; + zone_sectors_shift = ilog2(zone_sectors); + nr_sectors = bdev_nr_sectors(bdev); + nr_zones = nr_sectors >> zone_sectors_shift; + + sb_zone = sb_zone_number(zone_sectors_shift + SECTOR_SHIFT, mirror); + if (sb_zone + 1 >= nr_zones) + return -ENOENT; + + ret = blkdev_report_zones_cached(bdev, zone_start_sector(sb_zone, bdev), + BTRFS_NR_SB_LOG_ZONES, + copy_zone_info_cb, zones); + if (ret < 0) + return ret; + if (unlikely(ret != BTRFS_NR_SB_LOG_ZONES)) + return -EIO; + + return sb_log_location(bdev, zones, rw, bytenr_ret); +} + +int btrfs_sb_log_location(struct btrfs_device *device, int mirror, int rw, + u64 *bytenr_ret) +{ + struct btrfs_zoned_device_info *zinfo = device->zone_info; + u32 zone_num; + + /* + * For a zoned filesystem on a non-zoned block device, use the same + * super block locations as regular filesystem. Doing so, the super + * block can always be retrieved and the zoned flag of the volume + * detected from the super block information. + */ + if (!bdev_is_zoned(device->bdev)) { + *bytenr_ret = btrfs_sb_offset(mirror); + return 0; + } + + zone_num = sb_zone_number(zinfo->zone_size_shift, mirror); + if (zone_num + 1 >= zinfo->nr_zones) + return -ENOENT; + + return sb_log_location(device->bdev, + &zinfo->sb_zones[BTRFS_NR_SB_LOG_ZONES * mirror], + rw, bytenr_ret); +} + +static inline bool is_sb_log_zone(struct btrfs_zoned_device_info *zinfo, + int mirror) +{ + u32 zone_num; + + if (!zinfo) + return false; + + zone_num = sb_zone_number(zinfo->zone_size_shift, mirror); + if (zone_num + 1 >= zinfo->nr_zones) + return false; + + if (!test_bit(zone_num, zinfo->seq_zones)) + return false; + + return true; +} + +int btrfs_advance_sb_log(struct btrfs_device *device, int mirror) +{ + struct btrfs_zoned_device_info *zinfo = device->zone_info; + struct blk_zone *zone; + int i; + + if (!is_sb_log_zone(zinfo, mirror)) + return 0; + + zone = &zinfo->sb_zones[BTRFS_NR_SB_LOG_ZONES * mirror]; + for (i = 0; i < BTRFS_NR_SB_LOG_ZONES; i++) { + /* Advance the next zone */ + if (zone->cond == BLK_ZONE_COND_FULL) { + zone++; + continue; + } + + if (zone->cond == BLK_ZONE_COND_EMPTY) + zone->cond = BLK_ZONE_COND_IMP_OPEN; + + zone->wp += SUPER_INFO_SECTORS; + + if (sb_zone_is_full(zone)) { + /* + * No room left to write new superblock. Since + * superblock is written with REQ_SYNC, it is safe to + * finish the zone now. + * + * If the write pointer is exactly at the capacity, + * explicit ZONE_FINISH is not necessary. + */ + if (zone->wp != zone->start + zone->capacity) { + unsigned int nofs_flags; + int ret; + + nofs_flags = memalloc_nofs_save(); + ret = blkdev_zone_mgmt(device->bdev, + REQ_OP_ZONE_FINISH, zone->start, + zone->len); + memalloc_nofs_restore(nofs_flags); + if (ret) + return ret; + } + + zone->wp = zone->start + zone->len; + zone->cond = BLK_ZONE_COND_FULL; + } + return 0; + } + + /* All the zones are FULL. Should not reach here. */ + DEBUG_WARN("unexpected state, all zones full"); + return -EIO; +} + +int btrfs_reset_sb_log_zones(struct block_device *bdev, int mirror) +{ + unsigned int nofs_flags; + sector_t zone_sectors; + sector_t nr_sectors; + u8 zone_sectors_shift; + u32 sb_zone; + u32 nr_zones; + int ret; + + zone_sectors = bdev_zone_sectors(bdev); + zone_sectors_shift = ilog2(zone_sectors); + nr_sectors = bdev_nr_sectors(bdev); + nr_zones = nr_sectors >> zone_sectors_shift; + + sb_zone = sb_zone_number(zone_sectors_shift + SECTOR_SHIFT, mirror); + if (sb_zone + 1 >= nr_zones) + return -ENOENT; + + nofs_flags = memalloc_nofs_save(); + ret = blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET, + zone_start_sector(sb_zone, bdev), + zone_sectors * BTRFS_NR_SB_LOG_ZONES); + memalloc_nofs_restore(nofs_flags); + return ret; +} + +/* + * Find allocatable zones within a given region. + * + * @device: the device to allocate a region on + * @hole_start: the position of the hole to allocate the region + * @num_bytes: size of wanted region + * @hole_end: the end of the hole + * @return: position of allocatable zones + * + * Allocatable region should not contain any superblock locations. + */ +u64 btrfs_find_allocatable_zones(struct btrfs_device *device, u64 hole_start, + u64 hole_end, u64 num_bytes) +{ + struct btrfs_zoned_device_info *zinfo = device->zone_info; + const u8 shift = zinfo->zone_size_shift; + u64 nzones = num_bytes >> shift; + u64 pos = hole_start; + u64 begin, end; + bool have_sb; + int i; + + ASSERT(IS_ALIGNED(hole_start, zinfo->zone_size), + "hole_start=%llu zinfo->zone_size=%llu", hole_start, zinfo->zone_size); + ASSERT(IS_ALIGNED(num_bytes, zinfo->zone_size), + "num_bytes=%llu zinfo->zone_size=%llu", num_bytes, zinfo->zone_size); + + while (pos < hole_end) { + begin = pos >> shift; + end = begin + nzones; + + if (end > zinfo->nr_zones) + return hole_end; + + /* Check if zones in the region are all empty */ + if (btrfs_dev_is_sequential(device, pos) && + !bitmap_test_range_all_set(zinfo->empty_zones, begin, nzones)) { + pos += zinfo->zone_size; + continue; + } + + have_sb = false; + for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) { + u32 sb_zone; + u64 sb_pos; + + sb_zone = sb_zone_number(shift, i); + if (!(end <= sb_zone || + sb_zone + BTRFS_NR_SB_LOG_ZONES <= begin)) { + have_sb = true; + pos = zone_start_physical( + sb_zone + BTRFS_NR_SB_LOG_ZONES, zinfo); + break; + } + + /* We also need to exclude regular superblock positions */ + sb_pos = btrfs_sb_offset(i); + if (!(pos + num_bytes <= sb_pos || + sb_pos + BTRFS_SUPER_INFO_SIZE <= pos)) { + have_sb = true; + pos = ALIGN(sb_pos + BTRFS_SUPER_INFO_SIZE, + zinfo->zone_size); + break; + } + } + if (!have_sb) + break; + } + + return pos; +} + +static bool btrfs_dev_set_active_zone(struct btrfs_device *device, u64 pos) +{ + struct btrfs_zoned_device_info *zone_info = device->zone_info; + unsigned int zno = (pos >> zone_info->zone_size_shift); + + /* We can use any number of zones */ + if (zone_info->max_active_zones == 0) + return true; + + if (!test_bit(zno, zone_info->active_zones)) { + /* Active zone left? */ + if (atomic_dec_if_positive(&zone_info->active_zones_left) < 0) + return false; + if (test_and_set_bit(zno, zone_info->active_zones)) { + /* Someone already set the bit */ + atomic_inc(&zone_info->active_zones_left); + } + } + + return true; +} + +static void btrfs_dev_clear_active_zone(struct btrfs_device *device, u64 pos) +{ + struct btrfs_zoned_device_info *zone_info = device->zone_info; + unsigned int zno = (pos >> zone_info->zone_size_shift); + + /* We can use any number of zones */ + if (zone_info->max_active_zones == 0) + return; + + if (test_and_clear_bit(zno, zone_info->active_zones)) + atomic_inc(&zone_info->active_zones_left); +} + +int btrfs_reset_device_zone(struct btrfs_device *device, u64 physical, + u64 length, u64 *bytes) +{ + unsigned int nofs_flags; + int ret; + + *bytes = 0; + nofs_flags = memalloc_nofs_save(); + ret = blkdev_zone_mgmt(device->bdev, REQ_OP_ZONE_RESET, + physical >> SECTOR_SHIFT, length >> SECTOR_SHIFT); + memalloc_nofs_restore(nofs_flags); + if (ret) + return ret; + + *bytes = length; + while (length) { + btrfs_dev_set_zone_empty(device, physical); + btrfs_dev_clear_active_zone(device, physical); + physical += device->zone_info->zone_size; + length -= device->zone_info->zone_size; + } + + return 0; +} + +int btrfs_ensure_empty_zones(struct btrfs_device *device, u64 start, u64 size) +{ + struct btrfs_zoned_device_info *zinfo = device->zone_info; + const u8 shift = zinfo->zone_size_shift; + unsigned long begin = start >> shift; + unsigned long nbits = size >> shift; + u64 pos; + int ret; + + ASSERT(IS_ALIGNED(start, zinfo->zone_size), + "start=%llu, zinfo->zone_size=%llu", start, zinfo->zone_size); + ASSERT(IS_ALIGNED(size, zinfo->zone_size), + "size=%llu, zinfo->zone_size=%llu", size, zinfo->zone_size); + + if (begin + nbits > zinfo->nr_zones) + return -ERANGE; + + /* All the zones are conventional */ + if (bitmap_test_range_all_zero(zinfo->seq_zones, begin, nbits)) + return 0; + + /* All the zones are sequential and empty */ + if (bitmap_test_range_all_set(zinfo->seq_zones, begin, nbits) && + bitmap_test_range_all_set(zinfo->empty_zones, begin, nbits)) + return 0; + + for (pos = start; pos < start + size; pos += zinfo->zone_size) { + u64 reset_bytes; + + if (!btrfs_dev_is_sequential(device, pos) || + btrfs_dev_is_empty_zone(device, pos)) + continue; + + /* Free regions should be empty */ + btrfs_warn( + device->fs_info, + "zoned: resetting device %s (devid %llu) zone %llu for allocation", + rcu_dereference(device->name), device->devid, pos >> shift); + WARN_ON_ONCE(1); + + ret = btrfs_reset_device_zone(device, pos, zinfo->zone_size, + &reset_bytes); + if (ret) + return ret; + } + + return 0; +} + +/* + * Calculate an allocation pointer from the extent allocation information + * for a block group consist of conventional zones. It is pointed to the + * end of the highest addressed extent in the block group as an allocation + * offset. + */ +static int calculate_alloc_pointer(struct btrfs_block_group *cache, + u64 *offset_ret, bool new) +{ + struct btrfs_fs_info *fs_info = cache->fs_info; + struct btrfs_root *root; + BTRFS_PATH_AUTO_FREE(path); + struct btrfs_key key; + struct btrfs_key found_key; + const u64 bg_end = btrfs_block_group_end(cache); + int ret; + u64 length; + + /* + * Avoid tree lookups for a new block group, there's no use for it. + * It must always be 0. + * + * Also, we have a lock chain of extent buffer lock -> chunk mutex. + * For new a block group, this function is called from + * btrfs_make_block_group() which is already taking the chunk mutex. + * Thus, we cannot call calculate_alloc_pointer() which takes extent + * buffer locks to avoid deadlock. + */ + if (new) { + *offset_ret = 0; + return 0; + } + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = bg_end; + key.type = 0; + key.offset = 0; + + root = btrfs_extent_root(fs_info, key.objectid); + if (unlikely(!root)) { + btrfs_err(fs_info, + "missing extent root for extent at bytenr %llu", + key.objectid); + return -EUCLEAN; + } + + ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); + /* We should not find the exact match */ + if (unlikely(!ret)) + ret = -EUCLEAN; + if (ret < 0) + return ret; + + ret = btrfs_previous_extent_item(root, path, cache->start); + if (ret) { + if (ret == 1) { + ret = 0; + *offset_ret = 0; + } + return ret; + } + + btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]); + + if (found_key.type == BTRFS_EXTENT_ITEM_KEY) + length = found_key.offset; + else + length = fs_info->nodesize; + + if (unlikely(!(found_key.objectid >= cache->start && + found_key.objectid + length <= bg_end))) { + return -EUCLEAN; + } + *offset_ret = found_key.objectid + length - cache->start; + return 0; +} + +struct zone_info { + u64 physical; + u64 capacity; + u64 alloc_offset; +}; + +static int btrfs_load_zone_info(struct btrfs_fs_info *fs_info, int zone_idx, + struct zone_info *info, unsigned long *active, + struct btrfs_chunk_map *map, bool new) +{ + struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; + struct btrfs_device *device; + bool dev_replace_is_ongoing = false; + unsigned int nofs_flag; + struct blk_zone zone; + int ret; + + info->physical = map->stripes[zone_idx].physical; + + down_read(&dev_replace->rwsem); + device = map->stripes[zone_idx].dev; + + if (!device->bdev) { + up_read(&dev_replace->rwsem); + info->alloc_offset = WP_MISSING_DEV; + return 0; + } + + /* Consider a zone as active if we can allow any number of active zones. */ + if (!device->zone_info->max_active_zones) + __set_bit(zone_idx, active); + + if (!btrfs_dev_is_sequential(device, info->physical)) { + up_read(&dev_replace->rwsem); + info->alloc_offset = WP_CONVENTIONAL; + info->capacity = device->zone_info->zone_size; + return 0; + } + + ASSERT(!new || btrfs_dev_is_empty_zone(device, info->physical)); + + /* This zone will be used for allocation, so mark this zone non-empty. */ + btrfs_dev_clear_zone_empty(device, info->physical); + + dev_replace_is_ongoing = btrfs_dev_replace_is_ongoing(dev_replace); + if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL) + btrfs_dev_clear_zone_empty(dev_replace->tgtdev, info->physical); + + /* + * The group is mapped to a sequential zone. Get the zone write pointer + * to determine the allocation offset within the zone. + */ + WARN_ON(!IS_ALIGNED(info->physical, fs_info->zone_size)); + + if (new) { + sector_t capacity; + + capacity = bdev_zone_capacity(device->bdev, info->physical >> SECTOR_SHIFT); + up_read(&dev_replace->rwsem); + info->alloc_offset = 0; + info->capacity = capacity << SECTOR_SHIFT; + + return 0; + } + + nofs_flag = memalloc_nofs_save(); + ret = btrfs_get_dev_zone(device, info->physical, &zone); + memalloc_nofs_restore(nofs_flag); + if (ret) { + up_read(&dev_replace->rwsem); + if (ret != -EIO && ret != -EOPNOTSUPP) + return ret; + info->alloc_offset = WP_MISSING_DEV; + return 0; + } + + if (unlikely(zone.type == BLK_ZONE_TYPE_CONVENTIONAL)) { + btrfs_err(fs_info, + "zoned: unexpected conventional zone %llu on device %s (devid %llu)", + zone.start << SECTOR_SHIFT, rcu_dereference(device->name), + device->devid); + up_read(&dev_replace->rwsem); + return -EIO; + } + + info->capacity = (zone.capacity << SECTOR_SHIFT); + + switch (zone.cond) { + case BLK_ZONE_COND_OFFLINE: + case BLK_ZONE_COND_READONLY: + btrfs_err(fs_info, + "zoned: offline/readonly zone %llu on device %s (devid %llu)", + (info->physical >> device->zone_info->zone_size_shift), + rcu_dereference(device->name), device->devid); + info->alloc_offset = WP_MISSING_DEV; + break; + case BLK_ZONE_COND_EMPTY: + info->alloc_offset = 0; + break; + case BLK_ZONE_COND_FULL: + info->alloc_offset = info->capacity; + break; + default: + /* Partially used zone. */ + info->alloc_offset = ((zone.wp - zone.start) << SECTOR_SHIFT); + __set_bit(zone_idx, active); + break; + } + + up_read(&dev_replace->rwsem); + + return 0; +} + +static int btrfs_load_block_group_single(struct btrfs_block_group *bg, + struct zone_info *info, + unsigned long *active) +{ + if (unlikely(info->alloc_offset == WP_MISSING_DEV)) { + btrfs_err(bg->fs_info, + "zoned: cannot recover write pointer for zone %llu", + info->physical); + return -EIO; + } + + bg->alloc_offset = info->alloc_offset; + bg->zone_capacity = info->capacity; + if (test_bit(0, active)) + set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &bg->runtime_flags); + return 0; +} + +static int btrfs_load_block_group_dup(struct btrfs_block_group *bg, + struct btrfs_chunk_map *map, + struct zone_info *zone_info, + unsigned long *active, + u64 last_alloc) +{ + struct btrfs_fs_info *fs_info = bg->fs_info; + + if ((map->type & BTRFS_BLOCK_GROUP_DATA) && !fs_info->stripe_root) { + btrfs_err(fs_info, "zoned: data DUP profile needs raid-stripe-tree"); + return -EINVAL; + } + + bg->zone_capacity = min_not_zero(zone_info[0].capacity, zone_info[1].capacity); + + if (unlikely(zone_info[0].alloc_offset == WP_MISSING_DEV)) { + btrfs_err(fs_info, + "zoned: cannot recover write pointer for zone %llu", + zone_info[0].physical); + return -EIO; + } + if (unlikely(zone_info[1].alloc_offset == WP_MISSING_DEV)) { + btrfs_err(fs_info, + "zoned: cannot recover write pointer for zone %llu", + zone_info[1].physical); + return -EIO; + } + + /* + * When the last extent is removed, last_alloc can be smaller than the other write + * pointer. In that case, last_alloc should be moved to the corresponding write + * pointer position. + */ + for (int i = 0; i < map->num_stripes; i++) { + if (zone_info[i].alloc_offset == WP_CONVENTIONAL) + continue; + if (last_alloc <= zone_info[i].alloc_offset) { + last_alloc = zone_info[i].alloc_offset; + break; + } + } + + if (zone_info[0].alloc_offset == WP_CONVENTIONAL) + zone_info[0].alloc_offset = last_alloc; + + if (zone_info[1].alloc_offset == WP_CONVENTIONAL) + zone_info[1].alloc_offset = last_alloc; + + if (unlikely(zone_info[0].alloc_offset != zone_info[1].alloc_offset)) { + btrfs_err(fs_info, + "zoned: write pointer offset mismatch of zones in DUP profile"); + return -EIO; + } + + if (test_bit(0, active) != test_bit(1, active)) { + if (unlikely(!btrfs_zone_activate(bg))) + return -EIO; + } else if (test_bit(0, active)) { + set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &bg->runtime_flags); + } + + bg->alloc_offset = zone_info[0].alloc_offset; + return 0; +} + +static int btrfs_load_block_group_raid1(struct btrfs_block_group *bg, + struct btrfs_chunk_map *map, + struct zone_info *zone_info, + unsigned long *active, + u64 last_alloc) +{ + struct btrfs_fs_info *fs_info = bg->fs_info; + int i; + + if ((map->type & BTRFS_BLOCK_GROUP_DATA) && !fs_info->stripe_root) { + btrfs_err(fs_info, "zoned: data %s needs raid-stripe-tree", + btrfs_bg_type_to_raid_name(map->type)); + return -EINVAL; + } + + /* In case a device is missing we have a cap of 0, so don't use it. */ + bg->zone_capacity = min_not_zero(zone_info[0].capacity, zone_info[1].capacity); + + /* + * When the last extent is removed, last_alloc can be smaller than the other write + * pointer. In that case, last_alloc should be moved to the corresponding write + * pointer position. + */ + for (i = 0; i < map->num_stripes; i++) { + if (zone_info[i].alloc_offset == WP_MISSING_DEV || + zone_info[i].alloc_offset == WP_CONVENTIONAL) + continue; + if (last_alloc <= zone_info[i].alloc_offset) { + last_alloc = zone_info[i].alloc_offset; + break; + } + } + + for (i = 0; i < map->num_stripes; i++) { + if (zone_info[i].alloc_offset == WP_MISSING_DEV) + continue; + + if (zone_info[i].alloc_offset == WP_CONVENTIONAL) + zone_info[i].alloc_offset = last_alloc; + + if (unlikely((zone_info[0].alloc_offset != zone_info[i].alloc_offset) && + !btrfs_test_opt(fs_info, DEGRADED))) { + btrfs_err(fs_info, + "zoned: write pointer offset mismatch of zones in %s profile", + btrfs_bg_type_to_raid_name(map->type)); + return -EIO; + } + if (test_bit(0, active) != test_bit(i, active)) { + if (unlikely(!btrfs_test_opt(fs_info, DEGRADED) && + !btrfs_zone_activate(bg))) { + return -EIO; + } + } else { + if (test_bit(0, active)) + set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &bg->runtime_flags); + } + } + + if (zone_info[0].alloc_offset != WP_MISSING_DEV) + bg->alloc_offset = zone_info[0].alloc_offset; + else + bg->alloc_offset = zone_info[i - 1].alloc_offset; + + return 0; +} + +static int btrfs_load_block_group_raid0(struct btrfs_block_group *bg, + struct btrfs_chunk_map *map, + struct zone_info *zone_info, + unsigned long *active, + u64 last_alloc) +{ + struct btrfs_fs_info *fs_info = bg->fs_info; + u64 stripe_nr = 0, stripe_offset = 0; + u64 prev_offset = 0; + u32 stripe_index = 0; + bool has_partial = false, has_conventional = false; + + if ((map->type & BTRFS_BLOCK_GROUP_DATA) && !fs_info->stripe_root) { + btrfs_err(fs_info, "zoned: data %s needs raid-stripe-tree", + btrfs_bg_type_to_raid_name(map->type)); + return -EINVAL; + } + + /* + * When the last extent is removed, last_alloc can be smaller than the other write + * pointer. In that case, last_alloc should be moved to the corresponding write + * pointer position. + */ + for (int i = 0; i < map->num_stripes; i++) { + u64 alloc; + + if (zone_info[i].alloc_offset == WP_MISSING_DEV || + zone_info[i].alloc_offset == WP_CONVENTIONAL) + continue; + + stripe_nr = zone_info[i].alloc_offset >> BTRFS_STRIPE_LEN_SHIFT; + stripe_offset = zone_info[i].alloc_offset & BTRFS_STRIPE_LEN_MASK; + if (stripe_offset == 0 && stripe_nr > 0) { + stripe_nr--; + stripe_offset = BTRFS_STRIPE_LEN; + } + alloc = ((stripe_nr * map->num_stripes + i) << BTRFS_STRIPE_LEN_SHIFT) + + stripe_offset; + last_alloc = max(last_alloc, alloc); + + /* Partially written stripe found. It should be last. */ + if (zone_info[i].alloc_offset & BTRFS_STRIPE_LEN_MASK) + break; + } + stripe_nr = 0; + stripe_offset = 0; + + if (last_alloc) { + u32 factor = map->num_stripes; + + stripe_nr = last_alloc >> BTRFS_STRIPE_LEN_SHIFT; + stripe_offset = last_alloc & BTRFS_STRIPE_LEN_MASK; + stripe_nr = div_u64_rem(stripe_nr, factor, &stripe_index); + } + + for (int i = 0; i < map->num_stripes; i++) { + if (zone_info[i].alloc_offset == WP_MISSING_DEV) + continue; + + if (zone_info[i].alloc_offset == WP_CONVENTIONAL) { + has_conventional = true; + zone_info[i].alloc_offset = btrfs_stripe_nr_to_offset(stripe_nr); + + if (stripe_index > i) + zone_info[i].alloc_offset += BTRFS_STRIPE_LEN; + else if (stripe_index == i) + zone_info[i].alloc_offset += stripe_offset; + } + + /* Verification */ + if (i != 0) { + if (unlikely(prev_offset < zone_info[i].alloc_offset)) { + btrfs_err(fs_info, + "zoned: stripe position disorder found in block group %llu", + bg->start); + return -EIO; + } + + if (unlikely(has_partial && + (zone_info[i].alloc_offset & BTRFS_STRIPE_LEN_MASK))) { + btrfs_err(fs_info, + "zoned: multiple partial written stripe found in block group %llu", + bg->start); + return -EIO; + } + } + prev_offset = zone_info[i].alloc_offset; + + if ((zone_info[i].alloc_offset & BTRFS_STRIPE_LEN_MASK) != 0) + has_partial = true; + + if (test_bit(0, active) != test_bit(i, active)) { + if (unlikely(!btrfs_zone_activate(bg))) + return -EIO; + } else { + if (test_bit(0, active)) + set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &bg->runtime_flags); + } + bg->zone_capacity += zone_info[i].capacity; + bg->alloc_offset += zone_info[i].alloc_offset; + } + + /* Check if all devices stay in the same stripe row. */ + if (unlikely(zone_info[0].alloc_offset - + zone_info[map->num_stripes - 1].alloc_offset > BTRFS_STRIPE_LEN)) { + btrfs_err(fs_info, "zoned: stripe gap too large in block group %llu", bg->start); + return -EIO; + } + + if (unlikely(has_conventional && bg->alloc_offset < last_alloc)) { + btrfs_err(fs_info, "zoned: allocated extent stays beyond write pointers %llu %llu", + bg->alloc_offset, last_alloc); + return -EIO; + } + + return 0; +} + +static int btrfs_load_block_group_raid10(struct btrfs_block_group *bg, + struct btrfs_chunk_map *map, + struct zone_info *zone_info, + unsigned long *active, + u64 last_alloc) +{ + struct btrfs_fs_info *fs_info = bg->fs_info; + u64 AUTO_KFREE(raid0_allocs); + u64 stripe_nr = 0, stripe_offset = 0; + u32 stripe_index = 0; + bool has_partial = false, has_conventional = false; + u64 prev_offset = 0; + + if ((map->type & BTRFS_BLOCK_GROUP_DATA) && !fs_info->stripe_root) { + btrfs_err(fs_info, "zoned: data %s needs raid-stripe-tree", + btrfs_bg_type_to_raid_name(map->type)); + return -EINVAL; + } + + raid0_allocs = kzalloc_objs(*raid0_allocs, map->num_stripes / map->sub_stripes, GFP_NOFS); + if (!raid0_allocs) + return -ENOMEM; + + /* + * When the last extent is removed, last_alloc can be smaller than the other write + * pointer. In that case, last_alloc should be moved to the corresponding write + * pointer position. + */ + for (int i = 0; i < map->num_stripes; i += map->sub_stripes) { + u64 alloc = zone_info[i].alloc_offset; + + for (int j = 1; j < map->sub_stripes; j++) { + int idx = i + j; + + if (zone_info[idx].alloc_offset == WP_MISSING_DEV || + zone_info[idx].alloc_offset == WP_CONVENTIONAL) + continue; + if (alloc == WP_MISSING_DEV || alloc == WP_CONVENTIONAL) { + alloc = zone_info[idx].alloc_offset; + } else if (unlikely(zone_info[idx].alloc_offset != alloc)) { + btrfs_err(fs_info, + "zoned: write pointer mismatch found in block group %llu", + bg->start); + return -EIO; + } + } + + raid0_allocs[i / map->sub_stripes] = alloc; + if (alloc == WP_CONVENTIONAL) + continue; + if (unlikely(alloc == WP_MISSING_DEV)) { + btrfs_err(fs_info, + "zoned: cannot recover write pointer of block group %llu due to missing device", + bg->start); + return -EIO; + } + + stripe_nr = alloc >> BTRFS_STRIPE_LEN_SHIFT; + stripe_offset = alloc & BTRFS_STRIPE_LEN_MASK; + if (stripe_offset == 0 && stripe_nr > 0) { + stripe_nr--; + stripe_offset = BTRFS_STRIPE_LEN; + } + + alloc = ((stripe_nr * (map->num_stripes / map->sub_stripes) + + (i / map->sub_stripes)) << + BTRFS_STRIPE_LEN_SHIFT) + stripe_offset; + last_alloc = max(last_alloc, alloc); + } + stripe_nr = 0; + stripe_offset = 0; + + if (last_alloc) { + u32 factor = map->num_stripes / map->sub_stripes; + + stripe_nr = last_alloc >> BTRFS_STRIPE_LEN_SHIFT; + stripe_offset = last_alloc & BTRFS_STRIPE_LEN_MASK; + stripe_nr = div_u64_rem(stripe_nr, factor, &stripe_index); + } + + for (int i = 0; i < map->num_stripes; i++) { + int idx = i / map->sub_stripes; + + if (raid0_allocs[idx] == WP_CONVENTIONAL) { + has_conventional = true; + raid0_allocs[idx] = btrfs_stripe_nr_to_offset(stripe_nr); + + if (stripe_index > idx) + raid0_allocs[idx] += BTRFS_STRIPE_LEN; + else if (stripe_index == idx) + raid0_allocs[idx] += stripe_offset; + } + + if ((i % map->sub_stripes) == 0) { + /* Verification */ + if (i != 0) { + if (unlikely(prev_offset < raid0_allocs[idx])) { + btrfs_err(fs_info, + "zoned: stripe position disorder found in block group %llu", + bg->start); + return -EIO; + } + + if (unlikely(has_partial && + (raid0_allocs[idx] & BTRFS_STRIPE_LEN_MASK))) { + btrfs_err(fs_info, + "zoned: multiple partial written stripe found in block group %llu", + bg->start); + return -EIO; + } + } + prev_offset = raid0_allocs[idx]; + + if ((raid0_allocs[idx] & BTRFS_STRIPE_LEN_MASK) != 0) + has_partial = true; + } + + if (zone_info[i].alloc_offset == WP_MISSING_DEV || + zone_info[i].alloc_offset == WP_CONVENTIONAL) + zone_info[i].alloc_offset = raid0_allocs[idx]; + + if (test_bit(0, active) != test_bit(i, active)) { + if (unlikely(!btrfs_zone_activate(bg))) + return -EIO; + } else if (test_bit(0, active)) { + set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &bg->runtime_flags); + } + + if ((i % map->sub_stripes) == 0) { + bg->zone_capacity += zone_info[i].capacity; + bg->alloc_offset += zone_info[i].alloc_offset; + } + } + + /* Check if all devices stay in the same stripe row. */ + if (unlikely(zone_info[0].alloc_offset - + zone_info[map->num_stripes - 1].alloc_offset > BTRFS_STRIPE_LEN)) { + btrfs_err(fs_info, "zoned: stripe gap too large in block group %llu", + bg->start); + return -EIO; + } + + if (unlikely(has_conventional && bg->alloc_offset < last_alloc)) { + btrfs_err(fs_info, "zoned: allocated extent stays beyond write pointers %llu %llu", + bg->alloc_offset, last_alloc); + return -EIO; + } + + return 0; +} + +EXPORT_FOR_TESTS +int btrfs_load_block_group_by_raid_type(struct btrfs_block_group *bg, + struct btrfs_chunk_map *map, + struct zone_info *zone_info, + unsigned long *active, u64 last_alloc) +{ + struct btrfs_fs_info *fs_info = bg->fs_info; + u64 profile; + int ret; + + profile = map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK; + switch (profile) { + case 0: /* single */ + ret = btrfs_load_block_group_single(bg, &zone_info[0], active); + break; + case BTRFS_BLOCK_GROUP_DUP: + ret = btrfs_load_block_group_dup(bg, map, zone_info, active, last_alloc); + break; + case BTRFS_BLOCK_GROUP_RAID1: + case BTRFS_BLOCK_GROUP_RAID1C3: + case BTRFS_BLOCK_GROUP_RAID1C4: + ret = btrfs_load_block_group_raid1(bg, map, zone_info, active, last_alloc); + break; + case BTRFS_BLOCK_GROUP_RAID0: + ret = btrfs_load_block_group_raid0(bg, map, zone_info, active, last_alloc); + break; + case BTRFS_BLOCK_GROUP_RAID10: + ret = btrfs_load_block_group_raid10(bg, map, zone_info, active, last_alloc); + break; + case BTRFS_BLOCK_GROUP_RAID5: + case BTRFS_BLOCK_GROUP_RAID6: + default: + btrfs_err(fs_info, "zoned: profile %s not yet supported", + btrfs_bg_type_to_raid_name(map->type)); + return -EINVAL; + } + + if (ret == -EIO && profile != 0 && profile != BTRFS_BLOCK_GROUP_RAID0 && + profile != BTRFS_BLOCK_GROUP_RAID10) { + /* + * Detected broken write pointer. Make this block group + * unallocatable by setting the allocation pointer at the end of + * allocatable region. Relocating this block group will fix the + * mismatch. + * + * Currently, we cannot handle RAID0 or RAID10 case like this + * because we don't have a proper zone_capacity value. But, + * reading from this block group won't work anyway by a missing + * stripe. + */ + bg->alloc_offset = bg->zone_capacity; + } + + return ret; +} + +int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache, bool new) +{ + struct btrfs_fs_info *fs_info = cache->fs_info; + struct btrfs_chunk_map *map; + u64 logical = cache->start; + u64 length = cache->length; + struct zone_info AUTO_KFREE(zone_info); + int ret; + int i; + unsigned long *active = NULL; + u64 last_alloc = 0; + u32 num_sequential = 0, num_conventional = 0; + + if (!btrfs_is_zoned(fs_info)) + return 0; + + /* Sanity check */ + if (unlikely(!IS_ALIGNED(length, fs_info->zone_size))) { + btrfs_err(fs_info, + "zoned: block group %llu len %llu unaligned to zone size %llu", + logical, length, fs_info->zone_size); + return -EIO; + } + + map = btrfs_find_chunk_map(fs_info, logical, length); + if (!map) + return -EINVAL; + + cache->physical_map = map; + + zone_info = kzalloc_objs(*zone_info, map->num_stripes, GFP_NOFS); + if (!zone_info) { + ret = -ENOMEM; + goto out; + } + + active = bitmap_zalloc(map->num_stripes, GFP_NOFS); + if (!active) { + ret = -ENOMEM; + goto out; + } + + for (i = 0; i < map->num_stripes; i++) { + ret = btrfs_load_zone_info(fs_info, i, &zone_info[i], active, map, new); + if (ret) + goto out; + + if (zone_info[i].alloc_offset == WP_CONVENTIONAL) + num_conventional++; + else + num_sequential++; + } + + if (num_sequential > 0) + set_bit(BLOCK_GROUP_FLAG_SEQUENTIAL_ZONE, &cache->runtime_flags); + + if (num_conventional > 0) { + ret = calculate_alloc_pointer(cache, &last_alloc, new); + if (ret) { + btrfs_err(fs_info, + "zoned: failed to determine allocation offset of bg %llu", + cache->start); + goto out; + } else if (map->num_stripes == num_conventional) { + cache->alloc_offset = last_alloc; + cache->zone_capacity = cache->length; + set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &cache->runtime_flags); + goto out; + } + } + + ret = btrfs_load_block_group_by_raid_type(cache, map, zone_info, active, last_alloc); + +out: + /* Reject non SINGLE data profiles without RST */ + if ((map->type & BTRFS_BLOCK_GROUP_DATA) && + (map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) && + !fs_info->stripe_root) { + btrfs_err(fs_info, "zoned: data %s needs raid-stripe-tree", + btrfs_bg_type_to_raid_name(map->type)); + ret = -EINVAL; + } + + if (unlikely(cache->alloc_offset > cache->zone_capacity)) { + btrfs_err(fs_info, +"zoned: invalid write pointer %llu (larger than zone capacity %llu) in block group %llu", + cache->alloc_offset, cache->zone_capacity, + cache->start); + ret = -EIO; + } + + /* An extent is allocated after the write pointer */ + if (!ret && num_conventional && last_alloc > cache->alloc_offset) { + btrfs_err(fs_info, + "zoned: got wrong write pointer in BG %llu: %llu > %llu", + logical, last_alloc, cache->alloc_offset); + ret = -EIO; + } + + if (!ret) { + cache->meta_write_pointer = cache->alloc_offset + cache->start; + if (test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &cache->runtime_flags)) { + btrfs_get_block_group(cache); + spin_lock(&fs_info->zone_active_bgs_lock); + list_add_tail(&cache->active_bg_list, + &fs_info->zone_active_bgs); + spin_unlock(&fs_info->zone_active_bgs_lock); + } + } else { + btrfs_free_chunk_map(cache->physical_map); + cache->physical_map = NULL; + } + bitmap_free(active); + + return ret; +} + +void btrfs_calc_zone_unusable(struct btrfs_block_group *cache) +{ + u64 unusable, free; + + if (!btrfs_is_zoned(cache->fs_info)) + return; + + WARN_ON(cache->bytes_super != 0); + unusable = (cache->alloc_offset - cache->used) + + (cache->length - cache->zone_capacity); + free = cache->zone_capacity - cache->alloc_offset; + + /* We only need ->free_space in ALLOC_SEQ block groups */ + cache->cached = BTRFS_CACHE_FINISHED; + cache->free_space_ctl->free_space = free; + cache->zone_unusable = unusable; +} + +bool btrfs_use_zone_append(struct btrfs_bio *bbio) +{ + u64 start = (bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT); + struct btrfs_inode *inode = bbio->inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct btrfs_block_group *cache; + bool ret = false; + + if (!btrfs_is_zoned(fs_info)) + return false; + + if (!is_data_inode(inode)) + return false; + + if (btrfs_op(&bbio->bio) != BTRFS_MAP_WRITE) + return false; + + /* + * Using REQ_OP_ZONE_APPEND for relocation can break assumptions on the + * extent layout the relocation code has. + * Furthermore we have set aside own block-group from which only the + * relocation "process" can allocate and make sure only one process at a + * time can add pages to an extent that gets relocated, so it's safe to + * use regular REQ_OP_WRITE for this special case. + */ + if (btrfs_is_data_reloc_root(inode->root)) + return false; + + cache = btrfs_lookup_block_group(fs_info, start); + ASSERT(cache); + if (!cache) + return false; + + ret = !!test_bit(BLOCK_GROUP_FLAG_SEQUENTIAL_ZONE, &cache->runtime_flags); + btrfs_put_block_group(cache); + + return ret; +} + +void btrfs_record_physical_zoned(struct btrfs_bio *bbio) +{ + const u64 physical = bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT; + struct btrfs_ordered_sum *sum = bbio->sums; + + if (physical < bbio->orig_physical) + sum->logical -= bbio->orig_physical - physical; + else + sum->logical += physical - bbio->orig_physical; +} + +static void btrfs_rewrite_logical_zoned(struct btrfs_ordered_extent *ordered, + u64 logical) +{ + struct extent_map_tree *em_tree = &ordered->inode->extent_tree; + struct extent_map *em; + + ordered->disk_bytenr = logical; + + write_lock(&em_tree->lock); + em = btrfs_search_extent_mapping(em_tree, ordered->file_offset, + ordered->num_bytes); + /* The em should be a new COW extent, thus it should not have an offset. */ + ASSERT(em->offset == 0, "em->offset=%llu", em->offset); + em->disk_bytenr = logical; + btrfs_free_extent_map(em); + write_unlock(&em_tree->lock); +} + +static bool btrfs_zoned_split_ordered(struct btrfs_ordered_extent *ordered, + u64 logical, u64 len) +{ + struct btrfs_ordered_extent *new; + + if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered->flags) && + btrfs_split_extent_map(ordered->inode, ordered->file_offset, + ordered->num_bytes, len, logical)) + return false; + + new = btrfs_split_ordered_extent(ordered, len); + if (IS_ERR(new)) + return false; + new->disk_bytenr = logical; + btrfs_finish_one_ordered(new); + return true; +} + +void btrfs_finish_ordered_zoned(struct btrfs_ordered_extent *ordered) +{ + struct btrfs_inode *inode = ordered->inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct btrfs_ordered_sum *sum; + u64 logical, len; + + /* + * Write to pre-allocated region is for the data relocation, and so + * it should use WRITE operation. No split/rewrite are necessary. + */ + if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered->flags)) + return; + + /* + * A fully truncated ordered extent wrote no data and so has + * no zone append result to record. + */ + if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags) && + ordered->truncated_len == 0) { + ASSERT(list_empty(&ordered->csum_list)); + return; + } + + ASSERT(!list_empty(&ordered->csum_list)); + sum = list_first_entry(&ordered->csum_list, struct btrfs_ordered_sum, list); + logical = sum->logical; + len = sum->len; + + while (len < ordered->disk_num_bytes) { + sum = list_next_entry(sum, list); + if (sum->logical == logical + len) { + len += sum->len; + continue; + } + if (!btrfs_zoned_split_ordered(ordered, logical, len)) { + btrfs_mark_ordered_extent_error(ordered); + btrfs_err(fs_info, "failed to split ordered extent"); + goto out; + } + logical = sum->logical; + len = sum->len; + } + + if (ordered->disk_bytenr != logical) + btrfs_rewrite_logical_zoned(ordered, logical); + +out: + /* + * If we end up here for nodatasum I/O, the btrfs_ordered_sum structures + * were allocated by btrfs_alloc_dummy_sum only to record the logical + * addresses and don't contain actual checksums. We thus must free them + * here so that we don't attempt to log the csums later. + */ + if ((inode->flags & BTRFS_INODE_NODATASUM) || + test_bit(BTRFS_FS_STATE_NO_DATA_CSUMS, &fs_info->fs_state)) { + while ((sum = list_first_entry_or_null(&ordered->csum_list, + typeof(*sum), list))) { + list_del(&sum->list); + kfree(sum); + } + } +} + +static bool check_bg_is_active(struct btrfs_eb_write_context *ctx, + struct btrfs_block_group **active_bg) +{ + const struct writeback_control *wbc = ctx->wbc; + struct btrfs_block_group *block_group = ctx->zoned_bg; + struct btrfs_fs_info *fs_info = block_group->fs_info; + + if (test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &block_group->runtime_flags)) + return true; + + if (fs_info->treelog_bg == block_group->start) { + if (!btrfs_zone_activate(block_group)) { + int ret_fin; + + btrfs_zoned_meta_io_unlock(fs_info); + ret_fin = btrfs_zone_finish_one_bg(fs_info); + btrfs_zoned_meta_io_lock(fs_info); + + if (ret_fin != 1 || !btrfs_zone_activate(block_group)) + return false; + } + } else if (*active_bg != block_group) { + struct btrfs_block_group *tgt = *active_bg; + + /* zoned_meta_io_lock protects fs_info->active_{meta,system}_bg. */ + lockdep_assert_held(&fs_info->zoned_meta_io_lock); + + if (tgt) { + /* + * If there is an unsent IO left in the allocated area, + * we cannot wait for them as it may cause a deadlock. + */ + if (tgt->meta_write_pointer < tgt->start + tgt->alloc_offset) { + if (wbc->sync_mode == WB_SYNC_NONE || + (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync)) + return false; + } + + /* Pivot active metadata/system block group. */ + btrfs_zoned_meta_io_unlock(fs_info); + wait_eb_writebacks(tgt); + do_zone_finish(tgt, true); + btrfs_zoned_meta_io_lock(fs_info); + if (*active_bg == tgt) { + btrfs_put_block_group(tgt); + *active_bg = NULL; + } + } + if (!btrfs_zone_activate(block_group)) + return false; + if (*active_bg != block_group) { + ASSERT(*active_bg == NULL); + *active_bg = block_group; + btrfs_get_block_group(block_group); + } + } + + return true; +} + +/* + * Check if @ctx->eb is aligned to the write pointer. + * + * Return: + * 0: @ctx->eb is at the write pointer. You can write it. + * -EAGAIN: There is a hole. The caller should handle the case. + * -EBUSY: There is a hole, but the caller can just bail out. + */ +int btrfs_check_meta_write_pointer(struct btrfs_fs_info *fs_info, + struct btrfs_eb_write_context *ctx) +{ + const struct writeback_control *wbc = ctx->wbc; + const struct extent_buffer *eb = ctx->eb; + struct btrfs_block_group *block_group = ctx->zoned_bg; + + if (!btrfs_is_zoned(fs_info)) + return 0; + + if (block_group) { + if (block_group->start > eb->start || + btrfs_block_group_end(block_group) <= eb->start) { + btrfs_put_block_group(block_group); + block_group = NULL; + ctx->zoned_bg = NULL; + } + } + + if (!block_group) { + block_group = btrfs_lookup_block_group(fs_info, eb->start); + if (!block_group) + return 0; + ctx->zoned_bg = block_group; + } + + if (block_group->meta_write_pointer == eb->start) { + struct btrfs_block_group **tgt; + + if (!test_bit(BTRFS_FS_ACTIVE_ZONE_TRACKING, &fs_info->flags)) + return 0; + + if (block_group->flags & BTRFS_BLOCK_GROUP_SYSTEM) + tgt = &fs_info->active_system_bg; + else + tgt = &fs_info->active_meta_bg; + if (check_bg_is_active(ctx, tgt)) + return 0; + } + + /* + * Since we may release fs_info->zoned_meta_io_lock, someone can already + * start writing this eb. In that case, we can just bail out. + */ + if (block_group->meta_write_pointer > eb->start) + return -EBUSY; + + /* If for_sync, this hole will be filled with transaction commit. */ + if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync) + return -EAGAIN; + return -EBUSY; +} + +int btrfs_zoned_issue_zeroout(struct btrfs_device *device, u64 physical, u64 length) +{ + if (!btrfs_dev_is_sequential(device, physical)) + return -EOPNOTSUPP; + + return blkdev_issue_zeroout(device->bdev, physical >> SECTOR_SHIFT, + length >> SECTOR_SHIFT, GFP_NOFS, 0); +} + +static int read_zone_info(struct btrfs_fs_info *fs_info, u64 logical, + struct blk_zone *zone) +{ + struct btrfs_io_context *bioc = NULL; + u64 mapped_length = PAGE_SIZE; + unsigned int nofs_flag; + int nmirrors; + int i, ret; + + ret = btrfs_map_block(fs_info, BTRFS_MAP_GET_READ_MIRRORS, logical, + &mapped_length, &bioc, NULL, NULL); + if (unlikely(ret || !bioc || mapped_length < PAGE_SIZE)) { + ret = -EIO; + goto out_put_bioc; + } + + if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) { + ret = -EINVAL; + goto out_put_bioc; + } + + nofs_flag = memalloc_nofs_save(); + nmirrors = (int)bioc->num_stripes; + for (i = 0; i < nmirrors; i++) { + u64 physical = bioc->stripes[i].physical; + struct btrfs_device *dev = bioc->stripes[i].dev; + + /* Missing device */ + if (!dev->bdev) + continue; + + ret = btrfs_get_dev_zone(dev, physical, zone); + /* Failing device */ + if (ret == -EIO || ret == -EOPNOTSUPP) + continue; + break; + } + memalloc_nofs_restore(nofs_flag); +out_put_bioc: + btrfs_put_bioc(bioc); + return ret; +} + +/* + * Synchronize write pointer in a zone at @physical_start on @tgt_dev, by + * filling zeros between @physical_pos to a write pointer of dev-replace + * source device. + */ +int btrfs_sync_zone_write_pointer(struct btrfs_device *tgt_dev, u64 logical, + u64 physical_start, u64 physical_pos) +{ + struct btrfs_fs_info *fs_info = tgt_dev->fs_info; + struct blk_zone zone; + u64 length; + u64 wp; + int ret; + + if (!btrfs_dev_is_sequential(tgt_dev, physical_pos)) + return 0; + + ret = read_zone_info(fs_info, logical, &zone); + if (ret) + return ret; + + wp = physical_start + ((zone.wp - zone.start) << SECTOR_SHIFT); + + if (physical_pos == wp) + return 0; + + if (unlikely(physical_pos > wp)) + return -EUCLEAN; + + length = wp - physical_pos; + return btrfs_zoned_issue_zeroout(tgt_dev, physical_pos, length); +} + +/* + * Activate block group and underlying device zones + * + * @block_group: the block group to activate + * + * Return: true on success, false otherwise + */ +bool btrfs_zone_activate(struct btrfs_block_group *block_group) +{ + struct btrfs_fs_info *fs_info = block_group->fs_info; + struct btrfs_chunk_map *map; + struct btrfs_device *device; + u64 physical; + const bool is_data = (block_group->flags & BTRFS_BLOCK_GROUP_DATA); + bool ret; + int i; + + if (!btrfs_is_zoned(block_group->fs_info)) + return true; + + if (unlikely(btrfs_is_testing(fs_info))) + return true; + + map = block_group->physical_map; + + spin_lock(&fs_info->zone_active_bgs_lock); + spin_lock(&block_group->lock); + if (test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &block_group->runtime_flags)) { + ret = true; + goto out_unlock; + } + + if (block_group->flags & BTRFS_BLOCK_GROUP_DATA) { + /* The caller should check if the block group is full. */ + if (WARN_ON_ONCE(btrfs_zoned_bg_is_full(block_group))) { + ret = false; + goto out_unlock; + } + } else { + /* Since it is already written, it should have been active. */ + WARN_ON_ONCE(block_group->meta_write_pointer != block_group->start); + } + + for (i = 0; i < map->num_stripes; i++) { + struct btrfs_zoned_device_info *zinfo; + int reserved = 0; + + device = map->stripes[i].dev; + physical = map->stripes[i].physical; + zinfo = device->zone_info; + + if (!device->bdev) + continue; + + if (zinfo->max_active_zones == 0) + continue; + + if (is_data) + reserved = zinfo->reserved_active_zones; + /* + * For the data block group, leave active zones for one + * metadata block group and one system block group. + */ + if (atomic_read(&zinfo->active_zones_left) <= reserved) { + ret = false; + goto out_unlock; + } + + if (!btrfs_dev_set_active_zone(device, physical)) { + /* Cannot activate the zone */ + ret = false; + goto out_unlock; + } + if (!is_data) + zinfo->reserved_active_zones--; + } + + /* Successfully activated all the zones */ + set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &block_group->runtime_flags); + spin_unlock(&block_group->lock); + + /* For the active block group list */ + btrfs_get_block_group(block_group); + list_add_tail(&block_group->active_bg_list, &fs_info->zone_active_bgs); + spin_unlock(&fs_info->zone_active_bgs_lock); + + return true; + +out_unlock: + spin_unlock(&block_group->lock); + spin_unlock(&fs_info->zone_active_bgs_lock); + return ret; +} + +static void wait_eb_writebacks(struct btrfs_block_group *block_group) +{ + struct btrfs_fs_info *fs_info = block_group->fs_info; + const u64 end = btrfs_block_group_end(block_group); + struct extent_buffer *eb; + unsigned long index, start = (block_group->start >> fs_info->nodesize_bits); + + rcu_read_lock(); + xa_for_each_start(&fs_info->buffer_tree, index, eb, start) { + if (eb->start < block_group->start) + continue; + if (eb->start >= end) + break; + rcu_read_unlock(); + wait_on_extent_buffer_writeback(eb); + rcu_read_lock(); + } + rcu_read_unlock(); +} + +static int call_zone_finish(struct btrfs_block_group *block_group, + struct btrfs_io_stripe *stripe) +{ + struct btrfs_device *device = stripe->dev; + const u64 physical = stripe->physical; + struct btrfs_zoned_device_info *zinfo = device->zone_info; + int ret; + + if (!device->bdev) + return 0; + + if (zinfo->max_active_zones == 0) + return 0; + + if (btrfs_dev_is_sequential(device, physical)) { + unsigned int nofs_flags; + + nofs_flags = memalloc_nofs_save(); + ret = blkdev_zone_mgmt(device->bdev, REQ_OP_ZONE_FINISH, + physical >> SECTOR_SHIFT, + zinfo->zone_size >> SECTOR_SHIFT); + memalloc_nofs_restore(nofs_flags); + + if (ret) + return ret; + } + + if (!(block_group->flags & BTRFS_BLOCK_GROUP_DATA)) + zinfo->reserved_active_zones++; + btrfs_dev_clear_active_zone(device, physical); + + return 0; +} + +static int do_zone_finish(struct btrfs_block_group *block_group, bool fully_written) +{ + struct btrfs_fs_info *fs_info = block_group->fs_info; + struct btrfs_chunk_map *map; + const bool is_metadata = (block_group->flags & + (BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_SYSTEM)); + struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; + int ret = 0; + int i; + + spin_lock(&block_group->lock); + if (!test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &block_group->runtime_flags)) { + spin_unlock(&block_group->lock); + return 0; + } + + /* Check if we have unwritten allocated space */ + if (is_metadata && + block_group->start + block_group->alloc_offset > block_group->meta_write_pointer) { + spin_unlock(&block_group->lock); + return -EAGAIN; + } + + /* + * If we are sure that the block group is full (= no more room left for + * new allocation) and the IO for the last usable block is completed, we + * don't need to wait for the other IOs. This holds because we ensure + * the sequential IO submissions using the ZONE_APPEND command for data + * and block_group->meta_write_pointer for metadata. + */ + if (!fully_written) { + if (test_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, &block_group->runtime_flags)) { + spin_unlock(&block_group->lock); + return -EAGAIN; + } + spin_unlock(&block_group->lock); + + ret = btrfs_inc_block_group_ro(block_group, false); + if (ret) + return ret; + + /* Ensure all writes in this block group finish */ + btrfs_wait_block_group_reservations(block_group); + /* No need to wait for NOCOW writers. Zoned mode does not allow that */ + btrfs_wait_ordered_roots(fs_info, U64_MAX, block_group); + /* Wait for extent buffers to be written. */ + if (is_metadata) + wait_eb_writebacks(block_group); + + spin_lock(&block_group->lock); + + /* + * Bail out if someone already deactivated the block group, or + * allocated space is left in the block group. + */ + if (!test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, + &block_group->runtime_flags)) { + spin_unlock(&block_group->lock); + btrfs_dec_block_group_ro(block_group); + return 0; + } + + if (block_group->reserved || + test_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, + &block_group->runtime_flags)) { + spin_unlock(&block_group->lock); + btrfs_dec_block_group_ro(block_group); + return -EAGAIN; + } + } + + clear_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &block_group->runtime_flags); + block_group->alloc_offset = block_group->zone_capacity; + if (block_group->flags & (BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_SYSTEM)) + block_group->meta_write_pointer = block_group->start + + block_group->zone_capacity; + block_group->free_space_ctl->free_space = 0; + btrfs_clear_treelog_bg(block_group); + btrfs_clear_data_reloc_bg(block_group); + spin_unlock(&block_group->lock); + + down_read(&dev_replace->rwsem); + map = block_group->physical_map; + for (i = 0; i < map->num_stripes; i++) { + + ret = call_zone_finish(block_group, &map->stripes[i]); + if (ret) { + up_read(&dev_replace->rwsem); + return ret; + } + } + up_read(&dev_replace->rwsem); + + if (!fully_written) + btrfs_dec_block_group_ro(block_group); + + spin_lock(&fs_info->zone_active_bgs_lock); + ASSERT(!list_empty(&block_group->active_bg_list)); + list_del_init(&block_group->active_bg_list); + spin_unlock(&fs_info->zone_active_bgs_lock); + + /* For active_bg_list */ + btrfs_put_block_group(block_group); + + clear_and_wake_up_bit(BTRFS_FS_NEED_ZONE_FINISH, &fs_info->flags); + + return 0; +} + +int btrfs_zone_finish(struct btrfs_block_group *block_group) +{ + if (!btrfs_is_zoned(block_group->fs_info)) + return 0; + + return do_zone_finish(block_group, false); +} + +bool btrfs_can_activate_zone(struct btrfs_fs_devices *fs_devices, u64 flags) +{ + struct btrfs_fs_info *fs_info = fs_devices->fs_info; + struct btrfs_device *device; + bool ret = false; + + if (!btrfs_is_zoned(fs_info)) + return true; + + if (test_bit(BTRFS_FS_NEED_ZONE_FINISH, &fs_info->flags)) + return false; + + /* Check if there is a device with active zones left */ + mutex_lock(&fs_info->chunk_mutex); + spin_lock(&fs_info->zone_active_bgs_lock); + list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) { + struct btrfs_zoned_device_info *zinfo = device->zone_info; + int reserved = 0; + + if (!device->bdev) + continue; + + if (!zinfo->max_active_zones) { + ret = true; + break; + } + + if (flags & BTRFS_BLOCK_GROUP_DATA) + reserved = zinfo->reserved_active_zones; + + switch (flags & BTRFS_BLOCK_GROUP_PROFILE_MASK) { + case 0: /* single */ + ret = (atomic_read(&zinfo->active_zones_left) >= (1 + reserved)); + break; + case BTRFS_BLOCK_GROUP_DUP: + ret = (atomic_read(&zinfo->active_zones_left) >= (2 + reserved)); + break; + } + if (ret) + break; + } + spin_unlock(&fs_info->zone_active_bgs_lock); + mutex_unlock(&fs_info->chunk_mutex); + + if (!ret) + set_bit(BTRFS_FS_NEED_ZONE_FINISH, &fs_info->flags); + + return ret; +} + +int btrfs_zone_finish_endio(struct btrfs_fs_info *fs_info, u64 logical, u64 length) +{ + struct btrfs_block_group *block_group; + u64 min_alloc_bytes; + + if (!btrfs_is_zoned(fs_info)) + return 0; + + block_group = btrfs_lookup_block_group(fs_info, logical); + if (WARN_ON_ONCE(!block_group)) + return -ENOENT; + + /* No MIXED_BG on zoned btrfs. */ + if (block_group->flags & BTRFS_BLOCK_GROUP_DATA) + min_alloc_bytes = fs_info->sectorsize; + else + min_alloc_bytes = fs_info->nodesize; + + /* Bail out if we can allocate more data from this block group. */ + if (logical + length + min_alloc_bytes <= + block_group->start + block_group->zone_capacity) + goto out; + + do_zone_finish(block_group, true); + +out: + btrfs_put_block_group(block_group); + return 0; +} + +static void btrfs_zone_finish_endio_workfn(struct work_struct *work) +{ + int ret; + struct btrfs_block_group *bg = + container_of(work, struct btrfs_block_group, zone_finish_work); + + wait_on_extent_buffer_writeback(bg->last_eb); + free_extent_buffer(bg->last_eb); + ret = do_zone_finish(bg, true); + if (ret) + btrfs_handle_fs_error(bg->fs_info, ret, + "Failed to finish block-group's zone"); + btrfs_put_block_group(bg); +} + +void btrfs_schedule_zone_finish_bg(struct btrfs_block_group *bg, + struct extent_buffer *eb) +{ + if (!test_bit(BLOCK_GROUP_FLAG_SEQUENTIAL_ZONE, &bg->runtime_flags) || + eb->start + eb->len * 2 <= bg->start + bg->zone_capacity) + return; + + if (WARN_ON(bg->zone_finish_work.func == btrfs_zone_finish_endio_workfn)) { + btrfs_err(bg->fs_info, "double scheduling of bg %llu zone finishing", + bg->start); + return; + } + + /* For the work */ + btrfs_get_block_group(bg); + refcount_inc(&eb->refs); + bg->last_eb = eb; + INIT_WORK(&bg->zone_finish_work, btrfs_zone_finish_endio_workfn); + queue_work(system_dfl_wq, &bg->zone_finish_work); +} + +void btrfs_clear_data_reloc_bg(struct btrfs_block_group *bg) +{ + struct btrfs_fs_info *fs_info = bg->fs_info; + + spin_lock(&fs_info->relocation_bg_lock); + if (fs_info->data_reloc_bg == bg->start) + fs_info->data_reloc_bg = 0; + spin_unlock(&fs_info->relocation_bg_lock); +} + +void btrfs_zoned_reserve_data_reloc_bg(struct btrfs_fs_info *fs_info) +{ + struct btrfs_space_info *data_sinfo = fs_info->data_sinfo; + struct btrfs_space_info *space_info = data_sinfo; + struct btrfs_trans_handle *trans; + struct btrfs_block_group *bg; + struct list_head *bg_list; + u64 alloc_flags; + bool did_chunk_alloc = false; + int index; + int ret; + + if (!btrfs_is_zoned(fs_info)) + return; + + if (fs_info->data_reloc_bg) + return; + + if (sb_rdonly(fs_info->sb)) + return; + + alloc_flags = btrfs_get_alloc_profile(fs_info, space_info->flags); + index = btrfs_bg_flags_to_raid_index(alloc_flags); + +again: + bg_list = &space_info->block_groups[index]; + list_for_each_entry(bg, bg_list, list) { + + if (bg->alloc_offset != 0) + continue; + + if (space_info == data_sinfo) { + /* Migrate the block group to the data relocation space_info. */ + struct btrfs_space_info *reloc_sinfo = data_sinfo->sub_group[0]; + int factor; + + ASSERT(reloc_sinfo->subgroup_id == BTRFS_SUB_GROUP_DATA_RELOC, + "reloc_sinfo->subgroup_id=%d", reloc_sinfo->subgroup_id); + factor = btrfs_bg_type_to_factor(bg->flags); + + down_write(&space_info->groups_sem); + list_del_init(&bg->list); + up_write(&space_info->groups_sem); + + spin_lock(&space_info->lock); + space_info->total_bytes -= bg->length; + space_info->disk_total -= bg->length * factor; + space_info->disk_total -= bg->zone_unusable; + /* There is no allocation ever happened. */ + ASSERT(bg->used == 0, "bg->used=%llu", bg->used); + /* No super block in a block group on the zoned setup. */ + ASSERT(bg->bytes_super == 0, "bg->bytes_super=%llu", bg->bytes_super); + spin_unlock(&space_info->lock); + + bg->space_info = reloc_sinfo; + if (reloc_sinfo->block_group_kobjs[index] == NULL) + btrfs_sysfs_add_block_group_type(bg); + + btrfs_add_bg_to_space_info(fs_info, bg); + } + + fs_info->data_reloc_bg = bg->start; + set_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, &bg->runtime_flags); + btrfs_zone_activate(bg); + + return; + } + + if (did_chunk_alloc) + return; + + trans = btrfs_join_transaction(fs_info->tree_root); + if (IS_ERR(trans)) + return; + + /* Allocate new BG in the data relocation space_info. */ + space_info = data_sinfo->sub_group[0]; + ASSERT(space_info->subgroup_id == BTRFS_SUB_GROUP_DATA_RELOC, + "space_info->subgroup_id=%d", space_info->subgroup_id); + ret = btrfs_chunk_alloc(trans, space_info, alloc_flags, CHUNK_ALLOC_FORCE); + btrfs_end_transaction(trans); + if (ret == 1) { + /* + * We allocated a new block group in the data relocation space_info. We + * can take that one. + */ + did_chunk_alloc = true; + goto again; + } +} + +void btrfs_free_zone_cache(struct btrfs_fs_info *fs_info) +{ + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; + struct btrfs_device *device; + + if (!btrfs_is_zoned(fs_info)) + return; + + mutex_lock(&fs_devices->device_list_mutex); + list_for_each_entry(device, &fs_devices->devices, dev_list) { + if (device->zone_info) { + vfree(device->zone_info->zone_cache); + device->zone_info->zone_cache = NULL; + } + } + mutex_unlock(&fs_devices->device_list_mutex); +} + +bool btrfs_zoned_should_reclaim(const struct btrfs_fs_info *fs_info) +{ + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; + struct btrfs_device *device; + u64 total = btrfs_super_total_bytes(fs_info->super_copy); + u64 used = 0; + u64 factor; + + ASSERT(btrfs_is_zoned(fs_info)); + + if (fs_info->bg_reclaim_threshold == 0) + return false; + + mutex_lock(&fs_devices->device_list_mutex); + list_for_each_entry(device, &fs_devices->devices, dev_list) { + if (!device->bdev) + continue; + + used += device->bytes_used; + } + mutex_unlock(&fs_devices->device_list_mutex); + + factor = div64_u64(used * 100, total); + return factor >= fs_info->bg_reclaim_threshold; +} + +void btrfs_zoned_release_data_reloc_bg(struct btrfs_fs_info *fs_info, u64 logical, + u64 length) +{ + struct btrfs_block_group *block_group; + + if (!btrfs_is_zoned(fs_info)) + return; + + block_group = btrfs_lookup_block_group(fs_info, logical); + /* It should be called on a previous data relocation block group. */ + ASSERT(block_group && (block_group->flags & BTRFS_BLOCK_GROUP_DATA)); + + spin_lock(&block_group->lock); + if (!test_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, &block_group->runtime_flags)) + goto out; + + /* All relocation extents are written. */ + if (block_group->start + block_group->alloc_offset == logical + length) { + /* + * Now, release this block group for further allocations and + * zone finish. + */ + clear_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, + &block_group->runtime_flags); + } + +out: + spin_unlock(&block_group->lock); + btrfs_put_block_group(block_group); +} + +int btrfs_zone_finish_one_bg(struct btrfs_fs_info *fs_info) +{ + struct btrfs_block_group *block_group; + struct btrfs_block_group *min_bg = NULL; + u64 min_avail = U64_MAX; + int ret; + + spin_lock(&fs_info->zone_active_bgs_lock); + list_for_each_entry(block_group, &fs_info->zone_active_bgs, + active_bg_list) { + u64 avail; + + spin_lock(&block_group->lock); + if (block_group->reserved || block_group->alloc_offset == 0 || + !(block_group->flags & BTRFS_BLOCK_GROUP_DATA) || + test_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, &block_group->runtime_flags)) { + spin_unlock(&block_group->lock); + continue; + } + + avail = block_group->zone_capacity - block_group->alloc_offset; + if (min_avail > avail) { + if (min_bg) + btrfs_put_block_group(min_bg); + min_bg = block_group; + min_avail = avail; + btrfs_get_block_group(min_bg); + } + spin_unlock(&block_group->lock); + } + spin_unlock(&fs_info->zone_active_bgs_lock); + + if (!min_bg) + return 0; + + ret = btrfs_zone_finish(min_bg); + btrfs_put_block_group(min_bg); + + return ret < 0 ? ret : 1; +} + +int btrfs_zoned_activate_one_bg(struct btrfs_space_info *space_info, bool do_finish) +{ + struct btrfs_fs_info *fs_info = space_info->fs_info; + struct btrfs_block_group *bg; + int index; + + if (!btrfs_is_zoned(fs_info) || (space_info->flags & BTRFS_BLOCK_GROUP_DATA)) + return 0; + + for (;;) { + int ret; + bool need_finish = false; + + down_read(&space_info->groups_sem); + for (index = 0; index < BTRFS_NR_RAID_TYPES; index++) { + list_for_each_entry(bg, &space_info->block_groups[index], + list) { + if (!spin_trylock(&bg->lock)) + continue; + if (btrfs_zoned_bg_is_full(bg) || + test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, + &bg->runtime_flags)) { + spin_unlock(&bg->lock); + continue; + } + spin_unlock(&bg->lock); + + if (btrfs_zone_activate(bg)) { + up_read(&space_info->groups_sem); + return 1; + } + + need_finish = true; + } + } + up_read(&space_info->groups_sem); + + if (!do_finish || !need_finish) + break; + + ret = btrfs_zone_finish_one_bg(fs_info); + if (ret == 0) + break; + if (ret < 0) + return ret; + } + + return 0; +} + +/* + * Reserve zones for one metadata block group, one tree-log block group, and one + * system block group. + */ +void btrfs_check_active_zone_reservation(struct btrfs_fs_info *fs_info) +{ + struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; + struct btrfs_block_group *block_group; + struct btrfs_device *device; + /* Reserve zones for normal SINGLE metadata and tree-log block group. */ + unsigned int metadata_reserve = 2; + /* Reserve a zone for SINGLE system block group. */ + unsigned int system_reserve = 1; + + if (!test_bit(BTRFS_FS_ACTIVE_ZONE_TRACKING, &fs_info->flags)) + return; + + /* + * This function is called from the mount context. So, there is no + * parallel process touching the bits. No need for read_seqretry(). + */ + if (fs_info->avail_metadata_alloc_bits & BTRFS_BLOCK_GROUP_DUP) + metadata_reserve = 4; + if (fs_info->avail_system_alloc_bits & BTRFS_BLOCK_GROUP_DUP) + system_reserve = 2; + + /* Apply the reservation on all the devices. */ + mutex_lock(&fs_devices->device_list_mutex); + list_for_each_entry(device, &fs_devices->devices, dev_list) { + if (!device->bdev) + continue; + + device->zone_info->reserved_active_zones = + metadata_reserve + system_reserve; + } + mutex_unlock(&fs_devices->device_list_mutex); + + /* Release reservation for currently active block groups. */ + spin_lock(&fs_info->zone_active_bgs_lock); + list_for_each_entry(block_group, &fs_info->zone_active_bgs, active_bg_list) { + struct btrfs_chunk_map *map = block_group->physical_map; + + if (!(block_group->flags & + (BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_SYSTEM))) + continue; + + for (int i = 0; i < map->num_stripes; i++) + map->stripes[i].dev->zone_info->reserved_active_zones--; + } + spin_unlock(&fs_info->zone_active_bgs_lock); +} + +/* + * Reset the zones of unused block groups from @space_info->bytes_zone_unusable. + * + * @space_info: the space to work on + * @num_bytes: targeting reclaim bytes + * + * This one resets the zones of a block group, so we can reuse the region + * without removing the block group. On the other hand, btrfs_delete_unused_bgs() + * just removes a block group and frees up the underlying zones. So, we still + * need to allocate a new block group to reuse the zones. + * + * Resetting is faster than deleting/recreating a block group. It is similar + * to freeing the logical space on the regular mode. However, we cannot change + * the block group's profile with this operation. + */ +int btrfs_reset_unused_block_groups(struct btrfs_space_info *space_info, u64 num_bytes) +{ + struct btrfs_fs_info *fs_info = space_info->fs_info; + const sector_t zone_size_sectors = fs_info->zone_size >> SECTOR_SHIFT; + + if (!btrfs_is_zoned(fs_info)) + return 0; + + while (num_bytes > 0) { + struct btrfs_chunk_map *map; + struct btrfs_block_group *bg = NULL; + bool found = false; + u64 reclaimed = 0; + + /* + * Here, we choose a fully zone_unusable block group. It's + * technically possible to reset a partly zone_unusable block + * group, which still has some free space left. However, + * handling that needs to cope with the allocation side, which + * makes the logic more complex. So, let's handle the easy case + * for now. + */ + spin_lock(&fs_info->unused_bgs_lock); + list_for_each_entry(bg, &fs_info->unused_bgs, bg_list) { + if ((bg->flags & BTRFS_BLOCK_GROUP_TYPE_MASK) != space_info->flags) + continue; + + /* + * Use trylock to avoid locking order violation. In + * btrfs_reclaim_bgs_work(), the lock order is + * &bg->lock -> &fs_info->unused_bgs_lock. We skip a + * block group if we cannot take its lock. + */ + if (!spin_trylock(&bg->lock)) + continue; + if (btrfs_is_block_group_used(bg) || bg->zone_unusable < bg->length) { + spin_unlock(&bg->lock); + continue; + } + spin_unlock(&bg->lock); + found = true; + break; + } + if (!found) { + spin_unlock(&fs_info->unused_bgs_lock); + return 0; + } + + list_del_init(&bg->bg_list); + btrfs_put_block_group(bg); + spin_unlock(&fs_info->unused_bgs_lock); + + /* + * Since the block group is fully zone_unusable and we cannot + * allocate from this block group anymore, we don't need to set + * this block group read-only. + */ + + down_read(&fs_info->dev_replace.rwsem); + map = bg->physical_map; + for (int i = 0; i < map->num_stripes; i++) { + struct btrfs_io_stripe *stripe = &map->stripes[i]; + unsigned int nofs_flags; + int ret; + + nofs_flags = memalloc_nofs_save(); + ret = blkdev_zone_mgmt(stripe->dev->bdev, REQ_OP_ZONE_RESET, + stripe->physical >> SECTOR_SHIFT, + zone_size_sectors); + memalloc_nofs_restore(nofs_flags); + + if (ret) { + up_read(&fs_info->dev_replace.rwsem); + return ret; + } + } + up_read(&fs_info->dev_replace.rwsem); + + spin_lock(&space_info->lock); + spin_lock(&bg->lock); + ASSERT(!btrfs_is_block_group_used(bg)); + if (bg->ro) { + spin_unlock(&bg->lock); + spin_unlock(&space_info->lock); + continue; + } + + reclaimed = bg->alloc_offset; + bg->zone_unusable = bg->length - bg->zone_capacity; + bg->alloc_offset = 0; + /* + * The zone was just reset to empty, so alloc_offset went back to + * the start of the zone. For metadata/system block groups the + * write pointer must follow it back to the start of the zone; + * otherwise it stays stale at the previous (finished) zone end, + * and metadata written into the reused zone would sit behind the + * write pointer, could never be written out in sequential order, + * and would be stranded (pinning its folio) until unmount. + */ + if (bg->flags & (BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_SYSTEM)) + bg->meta_write_pointer = bg->start; + /* + * This holds because we currently reset fully used then freed + * block group. + */ + ASSERT(reclaimed == bg->zone_capacity, + "reclaimed=%llu bg->zone_capacity=%llu", reclaimed, bg->zone_capacity); + bg->free_space_ctl->free_space += reclaimed; + space_info->bytes_zone_unusable -= reclaimed; + spin_unlock(&bg->lock); + btrfs_return_free_space(space_info, reclaimed); + spin_unlock(&space_info->lock); + + if (num_bytes <= reclaimed) + break; + num_bytes -= reclaimed; + } + + return 0; +} + +void btrfs_show_zoned_stats(struct btrfs_fs_info *fs_info, struct seq_file *seq) +{ + struct btrfs_block_group *bg; + u64 data_reloc_bg; + u64 treelog_bg; + + seq_puts(seq, "\n zoned statistics:\n"); + + spin_lock(&fs_info->zone_active_bgs_lock); + seq_printf(seq, "\tactive block-groups: %zu\n", + list_count_nodes(&fs_info->zone_active_bgs)); + spin_unlock(&fs_info->zone_active_bgs_lock); + + spin_lock(&fs_info->unused_bgs_lock); + seq_printf(seq, "\t reclaimable: %zu\n", + list_count_nodes(&fs_info->reclaim_bgs)); + seq_printf(seq, "\t unused: %zu\n", list_count_nodes(&fs_info->unused_bgs)); + spin_unlock(&fs_info->unused_bgs_lock); + + seq_printf(seq,"\t need reclaim: %s\n", + str_true_false(btrfs_zoned_should_reclaim(fs_info))); + + data_reloc_bg = data_race(fs_info->data_reloc_bg); + if (data_reloc_bg) + seq_printf(seq, "\tdata relocation block-group: %llu\n", + data_reloc_bg); + treelog_bg = data_race(fs_info->treelog_bg); + if (treelog_bg) + seq_printf(seq, "\ttree-log block-group: %llu\n", treelog_bg); + + spin_lock(&fs_info->zone_active_bgs_lock); + seq_puts(seq, "\tactive zones:\n"); + list_for_each_entry(bg, &fs_info->zone_active_bgs, active_bg_list) { + u64 start; + u64 alloc_offset; + u64 used; + u64 reserved; + u64 zone_unusable; + const char *typestr = btrfs_space_info_type_str(bg->space_info); + + spin_lock(&bg->lock); + start = bg->start; + alloc_offset = bg->alloc_offset; + used = bg->used; + reserved = bg->reserved; + zone_unusable = bg->zone_unusable; + spin_unlock(&bg->lock); + + seq_printf(seq, + "\t start: %llu, wp: %llu used: %llu, reserved: %llu, unusable: %llu (%s)\n", + start, alloc_offset, used, reserved, zone_unusable, typestr); + } + spin_unlock(&fs_info->zone_active_bgs_lock); +} diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h new file mode 100644 index 000000000..8e21a836f --- /dev/null +++ b/fs/btrfs/zoned.h @@ -0,0 +1,435 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef BTRFS_ZONED_H +#define BTRFS_ZONED_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "messages.h" +#include "volumes.h" +#include "disk-io.h" +#include "block-group.h" +#include "btrfs_inode.h" + +struct block_device; +struct extent_buffer; +struct btrfs_bio; +struct btrfs_ordered_extent; +struct btrfs_fs_info; +struct btrfs_space_info; +struct btrfs_eb_write_context; +struct btrfs_fs_devices; + +#define BTRFS_DEFAULT_RECLAIM_THRESH (75) + +struct btrfs_zoned_device_info { + /* + * Number of zones, zone size and types of zones if bdev is a + * zoned block device. + */ + u64 zone_size; + u8 zone_size_shift; + u32 nr_zones; + unsigned int max_active_zones; + /* + * Reserved active zones for one metadata and one system block group. + * It can vary per-device depending on the allocation status. + */ + int reserved_active_zones; + atomic_t active_zones_left; + unsigned long *seq_zones; + unsigned long *empty_zones; + unsigned long *active_zones; + struct blk_zone *zone_cache; + struct blk_zone sb_zones[2 * BTRFS_SUPER_MIRROR_MAX]; +}; + +void btrfs_finish_ordered_zoned(struct btrfs_ordered_extent *ordered); + +#ifdef CONFIG_BLK_DEV_ZONED +int btrfs_get_dev_zone_info_all_devices(struct btrfs_fs_info *fs_info); +int btrfs_get_dev_zone_info(struct btrfs_device *device, bool populate_cache); +void btrfs_destroy_dev_zone_info(struct btrfs_device *device); +struct btrfs_zoned_device_info *btrfs_clone_dev_zone_info(struct btrfs_device *orig_dev); +int btrfs_check_zoned_mode(struct btrfs_fs_info *fs_info); +int btrfs_check_mountopts_zoned(const struct btrfs_fs_info *info, + unsigned long long *mount_opt); +int btrfs_sb_log_location_bdev(struct block_device *bdev, int mirror, int rw, + u64 *bytenr_ret); +int btrfs_sb_log_location(struct btrfs_device *device, int mirror, int rw, + u64 *bytenr_ret); +int btrfs_advance_sb_log(struct btrfs_device *device, int mirror); +int btrfs_reset_sb_log_zones(struct block_device *bdev, int mirror); +u64 btrfs_find_allocatable_zones(struct btrfs_device *device, u64 hole_start, + u64 hole_end, u64 num_bytes); +int btrfs_reset_device_zone(struct btrfs_device *device, u64 physical, + u64 length, u64 *bytes); +int btrfs_ensure_empty_zones(struct btrfs_device *device, u64 start, u64 size); +int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache, bool new); +void btrfs_calc_zone_unusable(struct btrfs_block_group *cache); +bool btrfs_use_zone_append(struct btrfs_bio *bbio); +void btrfs_record_physical_zoned(struct btrfs_bio *bbio); +int btrfs_check_meta_write_pointer(struct btrfs_fs_info *fs_info, + struct btrfs_eb_write_context *ctx); +int btrfs_zoned_issue_zeroout(struct btrfs_device *device, u64 physical, u64 length); +int btrfs_sync_zone_write_pointer(struct btrfs_device *tgt_dev, u64 logical, + u64 physical_start, u64 physical_pos); +bool btrfs_zone_activate(struct btrfs_block_group *block_group); +int btrfs_zone_finish(struct btrfs_block_group *block_group); +bool btrfs_can_activate_zone(struct btrfs_fs_devices *fs_devices, u64 flags); +int btrfs_zone_finish_endio(struct btrfs_fs_info *fs_info, u64 logical, + u64 length); +void btrfs_schedule_zone_finish_bg(struct btrfs_block_group *bg, + struct extent_buffer *eb); +void btrfs_clear_data_reloc_bg(struct btrfs_block_group *bg); +void btrfs_zoned_reserve_data_reloc_bg(struct btrfs_fs_info *fs_info); +void btrfs_free_zone_cache(struct btrfs_fs_info *fs_info); +bool btrfs_zoned_should_reclaim(const struct btrfs_fs_info *fs_info); +void btrfs_zoned_release_data_reloc_bg(struct btrfs_fs_info *fs_info, u64 logical, + u64 length); +int btrfs_zone_finish_one_bg(struct btrfs_fs_info *fs_info); +int btrfs_zoned_activate_one_bg(struct btrfs_space_info *space_info, bool do_finish); +void btrfs_check_active_zone_reservation(struct btrfs_fs_info *fs_info); +int btrfs_reset_unused_block_groups(struct btrfs_space_info *space_info, u64 num_bytes); +void btrfs_show_zoned_stats(struct btrfs_fs_info *fs_info, struct seq_file *seq); + +#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS +struct zone_info; + +int btrfs_load_block_group_by_raid_type(struct btrfs_block_group *bg, + struct btrfs_chunk_map *map, + struct zone_info *zone_info, + unsigned long *active, u64 last_alloc); +#endif + +#else /* CONFIG_BLK_DEV_ZONED */ + +static inline int btrfs_get_dev_zone_info_all_devices(struct btrfs_fs_info *fs_info) +{ + return 0; +} + +static inline int btrfs_get_dev_zone_info(struct btrfs_device *device, + bool populate_cache) +{ + return 0; +} + +static inline void btrfs_destroy_dev_zone_info(struct btrfs_device *device) { } + +/* + * In case the kernel is compiled without CONFIG_BLK_DEV_ZONED we'll never call + * into btrfs_clone_dev_zone_info() so it's safe to return NULL here. + */ +static inline struct btrfs_zoned_device_info *btrfs_clone_dev_zone_info( + struct btrfs_device *orig_dev) +{ + return NULL; +} + +static inline int btrfs_check_zoned_mode(const struct btrfs_fs_info *fs_info) +{ + if (!btrfs_is_zoned(fs_info)) + return 0; + + btrfs_err(fs_info, "zoned block devices support is not enabled"); + return -EOPNOTSUPP; +} + +static inline int btrfs_check_mountopts_zoned(const struct btrfs_fs_info *info, + unsigned long long *mount_opt) +{ + return 0; +} + +static inline int btrfs_sb_log_location_bdev(struct block_device *bdev, + int mirror, int rw, u64 *bytenr_ret) +{ + *bytenr_ret = btrfs_sb_offset(mirror); + return 0; +} + +static inline int btrfs_sb_log_location(struct btrfs_device *device, int mirror, + int rw, u64 *bytenr_ret) +{ + *bytenr_ret = btrfs_sb_offset(mirror); + return 0; +} + +static inline int btrfs_advance_sb_log(struct btrfs_device *device, int mirror) +{ + return 0; +} + +static inline int btrfs_reset_sb_log_zones(struct block_device *bdev, int mirror) +{ + return 0; +} + +static inline u64 btrfs_find_allocatable_zones(struct btrfs_device *device, + u64 hole_start, u64 hole_end, + u64 num_bytes) +{ + return hole_start; +} + +static inline int btrfs_reset_device_zone(struct btrfs_device *device, + u64 physical, u64 length, u64 *bytes) +{ + *bytes = 0; + return 0; +} + +static inline int btrfs_ensure_empty_zones(struct btrfs_device *device, + u64 start, u64 size) +{ + return 0; +} + +static inline int btrfs_load_block_group_zone_info( + struct btrfs_block_group *cache, bool new) +{ + return 0; +} + +static inline void btrfs_calc_zone_unusable(struct btrfs_block_group *cache) { } + +static inline bool btrfs_use_zone_append(struct btrfs_bio *bbio) +{ + return false; +} + +static inline void btrfs_record_physical_zoned(struct btrfs_bio *bbio) +{ +} + +static inline int btrfs_check_meta_write_pointer(struct btrfs_fs_info *fs_info, + struct btrfs_eb_write_context *ctx) +{ + return 0; +} + +static inline int btrfs_zoned_issue_zeroout(struct btrfs_device *device, + u64 physical, u64 length) +{ + return -EOPNOTSUPP; +} + +static inline int btrfs_sync_zone_write_pointer(struct btrfs_device *tgt_dev, + u64 logical, u64 physical_start, + u64 physical_pos) +{ + return -EOPNOTSUPP; +} + +static inline bool btrfs_zone_activate(struct btrfs_block_group *block_group) +{ + return true; +} + +static inline int btrfs_zone_finish(struct btrfs_block_group *block_group) +{ + return 0; +} + +static inline bool btrfs_can_activate_zone(struct btrfs_fs_devices *fs_devices, + u64 flags) +{ + return true; +} + +static inline int btrfs_zone_finish_endio(struct btrfs_fs_info *fs_info, + u64 logical, u64 length) +{ + return 0; +} + +static inline void btrfs_schedule_zone_finish_bg(struct btrfs_block_group *bg, + struct extent_buffer *eb) { } + +static inline void btrfs_clear_data_reloc_bg(struct btrfs_block_group *bg) { } + +static inline void btrfs_zoned_reserve_data_reloc_bg(struct btrfs_fs_info *fs_info) { } + +static inline void btrfs_free_zone_cache(struct btrfs_fs_info *fs_info) { } + +static inline bool btrfs_zoned_should_reclaim(const struct btrfs_fs_info *fs_info) +{ + return false; +} + +static inline void btrfs_zoned_release_data_reloc_bg(struct btrfs_fs_info *fs_info, + u64 logical, u64 length) { } + +static inline int btrfs_zone_finish_one_bg(struct btrfs_fs_info *fs_info) +{ + return 1; +} + +static inline int btrfs_zoned_activate_one_bg(struct btrfs_space_info *space_info, + bool do_finish) +{ + /* Consider all the block groups are active */ + return 0; +} + +static inline void btrfs_check_active_zone_reservation(struct btrfs_fs_info *fs_info) { } + +static inline int btrfs_reset_unused_block_groups(struct btrfs_space_info *space_info, + u64 num_bytes) +{ + return 0; +} + +static inline int btrfs_show_zoned_stats(struct btrfs_fs_info *fs_info, struct seq_file *seq) +{ + return 0; +} + +#endif + +static inline bool btrfs_dev_is_sequential(struct btrfs_device *device, u64 pos) +{ + struct btrfs_zoned_device_info *zone_info = device->zone_info; + + if (!zone_info) + return false; + + return test_bit(pos >> zone_info->zone_size_shift, zone_info->seq_zones); +} + +static inline bool btrfs_dev_is_empty_zone(struct btrfs_device *device, u64 pos) +{ + struct btrfs_zoned_device_info *zone_info = device->zone_info; + + if (!zone_info) + return true; + + return test_bit(pos >> zone_info->zone_size_shift, zone_info->empty_zones); +} + +static inline void btrfs_dev_set_empty_zone_bit(struct btrfs_device *device, + u64 pos, bool set) +{ + struct btrfs_zoned_device_info *zone_info = device->zone_info; + unsigned int zno; + + if (!zone_info) + return; + + zno = pos >> zone_info->zone_size_shift; + if (set) + set_bit(zno, zone_info->empty_zones); + else + clear_bit(zno, zone_info->empty_zones); +} + +static inline void btrfs_dev_set_zone_empty(struct btrfs_device *device, u64 pos) +{ + btrfs_dev_set_empty_zone_bit(device, pos, true); +} + +static inline void btrfs_dev_clear_zone_empty(struct btrfs_device *device, u64 pos) +{ + btrfs_dev_set_empty_zone_bit(device, pos, false); +} + +static inline bool btrfs_check_device_zone_type(const struct btrfs_fs_info *fs_info, + struct block_device *bdev) +{ + if (btrfs_is_zoned(fs_info)) { + /* + * We can allow a regular device on a zoned filesystem, because + * we will emulate the zoned capabilities. + */ + if (!bdev_is_zoned(bdev)) + return true; + + return fs_info->zone_size == + (bdev_zone_sectors(bdev) << SECTOR_SHIFT); + } + + /* Do not allow Host Managed zoned device. */ + return !bdev_is_zoned(bdev); +} + +static inline bool btrfs_check_super_location(struct btrfs_device *device, u64 pos) +{ + /* + * On a non-zoned device, any address is OK. On a zoned device, + * non-SEQUENTIAL WRITE REQUIRED zones are capable. + */ + return device->zone_info == NULL || !btrfs_dev_is_sequential(device, pos); +} + +static inline bool btrfs_can_zone_reset(struct btrfs_device *device, + u64 physical, u64 length) +{ + u64 zone_size; + + if (!btrfs_dev_is_sequential(device, physical)) + return false; + + zone_size = device->zone_info->zone_size; + if (!IS_ALIGNED(physical, zone_size) || !IS_ALIGNED(length, zone_size)) + return false; + + return true; +} + +static inline void btrfs_zoned_meta_io_lock(struct btrfs_fs_info *fs_info) +{ + if (!btrfs_is_zoned(fs_info)) + return; + mutex_lock(&fs_info->zoned_meta_io_lock); +} + +static inline void btrfs_zoned_meta_io_unlock(struct btrfs_fs_info *fs_info) +{ + if (!btrfs_is_zoned(fs_info)) + return; + mutex_unlock(&fs_info->zoned_meta_io_lock); +} + +static inline void btrfs_clear_treelog_bg(struct btrfs_block_group *bg) +{ + struct btrfs_fs_info *fs_info = bg->fs_info; + + if (!btrfs_is_zoned(fs_info)) + return; + + spin_lock(&fs_info->treelog_bg_lock); + if (fs_info->treelog_bg == bg->start) + fs_info->treelog_bg = 0; + spin_unlock(&fs_info->treelog_bg_lock); +} + +static inline void btrfs_zoned_data_reloc_lock(struct btrfs_inode *inode) +{ + struct btrfs_root *root = inode->root; + + if (btrfs_is_data_reloc_root(root) && btrfs_is_zoned(root->fs_info)) + mutex_lock(&root->fs_info->zoned_data_reloc_io_lock); +} + +static inline void btrfs_zoned_data_reloc_unlock(struct btrfs_inode *inode) +{ + struct btrfs_root *root = inode->root; + + if (btrfs_is_data_reloc_root(root) && btrfs_is_zoned(root->fs_info)) + mutex_unlock(&root->fs_info->zoned_data_reloc_io_lock); +} + +static inline bool btrfs_zoned_bg_is_full(const struct btrfs_block_group *bg) +{ + ASSERT(btrfs_is_zoned(bg->fs_info)); + return (bg->alloc_offset == bg->zone_capacity); +} + +#endif diff --git a/fs/btrfs/zstd.c b/fs/btrfs/zstd.c new file mode 100644 index 000000000..86919293f --- /dev/null +++ b/fs/btrfs/zstd.c @@ -0,0 +1,736 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "misc.h" +#include "fs.h" +#include "btrfs_inode.h" +#include "compression.h" +#include "super.h" + +#define ZSTD_BTRFS_MAX_WINDOWLOG 17 +#define ZSTD_BTRFS_MAX_INPUT (1U << ZSTD_BTRFS_MAX_WINDOWLOG) +#define ZSTD_BTRFS_DEFAULT_LEVEL 3 +#define ZSTD_BTRFS_MIN_LEVEL -15 +#define ZSTD_BTRFS_MAX_LEVEL 15 +/* 307s to avoid pathologically clashing with transaction commit */ +#define ZSTD_BTRFS_RECLAIM_JIFFIES (307 * HZ) + +static zstd_parameters zstd_get_btrfs_parameters(int level, + size_t src_len) +{ + zstd_parameters params = zstd_get_params(level, src_len); + + if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG) + params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG; + WARN_ON(src_len > ZSTD_BTRFS_MAX_INPUT); + return params; +} + +struct workspace { + void *mem; + size_t size; + char *buf; + int level; + int req_level; + unsigned long last_used; /* jiffies */ + struct list_head list; + struct list_head lru_list; + zstd_in_buffer in_buf; + zstd_out_buffer out_buf; + zstd_parameters params; +}; + +/* + * Zstd Workspace Management + * + * Zstd workspaces have different memory requirements depending on the level. + * The zstd workspaces are managed by having individual lists for each level + * and a global lru. Forward progress is maintained by protecting a max level + * workspace. + * + * Getting a workspace is done by using the bitmap to identify the levels that + * have available workspaces and scans up. This lets us recycle higher level + * workspaces because of the monotonic memory guarantee. A workspace's + * last_used is only updated if it is being used by the corresponding memory + * level. Putting a workspace involves adding it back to the appropriate places + * and adding it back to the lru if necessary. + * + * A timer is used to reclaim workspaces if they have not been used for + * ZSTD_BTRFS_RECLAIM_JIFFIES. This helps keep only active workspaces around. + * The upper bound is provided by the workqueue limit which is 2 (percpu limit). + */ + +struct zstd_workspace_manager { + spinlock_t lock; + struct list_head lru_list; + struct list_head idle_ws[ZSTD_BTRFS_MAX_LEVEL]; + unsigned long active_map; + wait_queue_head_t wait; + struct timer_list timer; +}; + +static size_t zstd_ws_mem_sizes[ZSTD_BTRFS_MAX_LEVEL]; + +static inline struct workspace *list_to_workspace(struct list_head *list) +{ + return container_of(list, struct workspace, list); +} + +static inline int clip_level(int level) +{ + return max(0, level - 1); +} + +/* + * Timer callback to free unused workspaces. + * + * @t: timer + * + * This scans the lru_list and attempts to reclaim any workspace that hasn't + * been used for ZSTD_BTRFS_RECLAIM_JIFFIES. + * + * The context is softirq and does not need the _bh locking primitives. + */ +static void zstd_reclaim_timer_fn(struct timer_list *timer) +{ + struct zstd_workspace_manager *zwsm = + container_of(timer, struct zstd_workspace_manager, timer); + unsigned long reclaim_threshold = jiffies - ZSTD_BTRFS_RECLAIM_JIFFIES; + struct list_head *pos, *next; + + spin_lock(&zwsm->lock); + + if (list_empty(&zwsm->lru_list)) { + spin_unlock(&zwsm->lock); + return; + } + + list_for_each_prev_safe(pos, next, &zwsm->lru_list) { + struct workspace *victim = container_of(pos, struct workspace, + lru_list); + int level; + + if (time_after(victim->last_used, reclaim_threshold)) + break; + + /* workspace is in use */ + if (victim->req_level) + continue; + + level = victim->level; + list_del(&victim->lru_list); + list_del(&victim->list); + zstd_free_workspace(&victim->list); + + if (list_empty(&zwsm->idle_ws[level])) + clear_bit(level, &zwsm->active_map); + + } + + if (!list_empty(&zwsm->lru_list)) + mod_timer(&zwsm->timer, jiffies + ZSTD_BTRFS_RECLAIM_JIFFIES); + + spin_unlock(&zwsm->lock); +} + +/* + * Calculate monotonic memory bounds. + * + * It is possible based on the level configurations that a higher level + * workspace uses less memory than a lower level workspace. In order to reuse + * workspaces, this must be made a monotonic relationship. This precomputes + * the required memory for each level and enforces the monotonicity between + * level and memory required. + */ +static void zstd_calc_ws_mem_sizes(void) +{ + size_t max_size = 0; + int level; + + for (level = ZSTD_BTRFS_MIN_LEVEL; level <= ZSTD_BTRFS_MAX_LEVEL; level++) { + if (level == 0) + continue; + zstd_parameters params = + zstd_get_btrfs_parameters(level, ZSTD_BTRFS_MAX_INPUT); + size_t level_size = + max_t(size_t, + zstd_cstream_workspace_bound(¶ms.cParams), + zstd_dstream_workspace_bound(ZSTD_BTRFS_MAX_INPUT)); + + max_size = max_t(size_t, max_size, level_size); + /* Use level 1 workspace size for all the fast mode negative levels. */ + zstd_ws_mem_sizes[clip_level(level)] = max_size; + } +} + +int zstd_alloc_workspace_manager(struct btrfs_fs_info *fs_info) +{ + struct zstd_workspace_manager *zwsm; + struct list_head *ws; + + ASSERT(fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD] == NULL); + zwsm = kzalloc_obj(*zwsm); + if (!zwsm) + return -ENOMEM; + zstd_calc_ws_mem_sizes(); + spin_lock_init(&zwsm->lock); + init_waitqueue_head(&zwsm->wait); + timer_setup(&zwsm->timer, zstd_reclaim_timer_fn, 0); + + INIT_LIST_HEAD(&zwsm->lru_list); + for (int i = 0; i < ZSTD_BTRFS_MAX_LEVEL; i++) + INIT_LIST_HEAD(&zwsm->idle_ws[i]); + fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD] = zwsm; + + ws = zstd_alloc_workspace(fs_info, ZSTD_BTRFS_MAX_LEVEL); + if (IS_ERR(ws)) { + btrfs_warn(NULL, "cannot preallocate zstd compression workspace"); + } else { + set_bit(ZSTD_BTRFS_MAX_LEVEL - 1, &zwsm->active_map); + list_add(ws, &zwsm->idle_ws[ZSTD_BTRFS_MAX_LEVEL - 1]); + } + return 0; +} + +void zstd_free_workspace_manager(struct btrfs_fs_info *fs_info) +{ + struct zstd_workspace_manager *zwsm = fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD]; + struct workspace *workspace; + + if (!zwsm) + return; + fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD] = NULL; + spin_lock_bh(&zwsm->lock); + for (int i = 0; i < ZSTD_BTRFS_MAX_LEVEL; i++) { + while (!list_empty(&zwsm->idle_ws[i])) { + workspace = container_of(zwsm->idle_ws[i].next, + struct workspace, list); + list_del(&workspace->list); + list_del(&workspace->lru_list); + zstd_free_workspace(&workspace->list); + } + } + spin_unlock_bh(&zwsm->lock); + timer_delete_sync(&zwsm->timer); + kfree(zwsm); +} + +/* + * Find workspace for given level. + * + * @level: compression level + * + * This iterates over the set bits in the active_map beginning at the requested + * compression level. This lets us utilize already allocated workspaces before + * allocating a new one. If the workspace is of a larger size, it is used, but + * the place in the lru_list and last_used times are not updated. This is to + * offer the opportunity to reclaim the workspace in favor of allocating an + * appropriately sized one in the future. + */ +static struct list_head *zstd_find_workspace(struct btrfs_fs_info *fs_info, int level) +{ + struct zstd_workspace_manager *zwsm = fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD]; + struct list_head *ws; + struct workspace *workspace; + int i = clip_level(level); + + ASSERT(zwsm); + spin_lock_bh(&zwsm->lock); + for_each_set_bit_from(i, &zwsm->active_map, ZSTD_BTRFS_MAX_LEVEL) { + if (!list_empty(&zwsm->idle_ws[i])) { + ws = zwsm->idle_ws[i].next; + workspace = list_to_workspace(ws); + list_del_init(ws); + /* keep its place if it's a lower level using this */ + workspace->req_level = level; + if (clip_level(level) == workspace->level) + list_del(&workspace->lru_list); + if (list_empty(&zwsm->idle_ws[i])) + clear_bit(i, &zwsm->active_map); + spin_unlock_bh(&zwsm->lock); + return ws; + } + } + spin_unlock_bh(&zwsm->lock); + + return NULL; +} + +/* + * Zstd get_workspace for level. + * + * @level: compression level + * + * If @level is 0, then any compression level can be used. Therefore, we begin + * scanning from 1. We first scan through possible workspaces and then after + * attempt to allocate a new workspace. If we fail to allocate one due to + * memory pressure, go to sleep waiting for the max level workspace to free up. + */ +struct list_head *zstd_get_workspace(struct btrfs_fs_info *fs_info, int level) +{ + struct zstd_workspace_manager *zwsm = fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD]; + struct list_head *ws; + unsigned int nofs_flag; + + ASSERT(zwsm); + + /* level == 0 means we can use any workspace */ + if (!level) + level = 1; + +again: + ws = zstd_find_workspace(fs_info, level); + if (ws) + return ws; + + nofs_flag = memalloc_nofs_save(); + ws = zstd_alloc_workspace(fs_info, level); + memalloc_nofs_restore(nofs_flag); + + if (IS_ERR(ws)) { + DEFINE_WAIT(wait); + + prepare_to_wait(&zwsm->wait, &wait, TASK_UNINTERRUPTIBLE); + schedule(); + finish_wait(&zwsm->wait, &wait); + + goto again; + } + + return ws; +} + +/* + * Zstd put_workspace. + * + * @ws: list_head for the workspace + * + * When putting back a workspace, we only need to update the LRU if we are of + * the requested compression level. Here is where we continue to protect the + * max level workspace or update last_used accordingly. If the reclaim timer + * isn't set, it is also set here. Only the max level workspace tries and wakes + * up waiting workspaces. + */ +void zstd_put_workspace(struct btrfs_fs_info *fs_info, struct list_head *ws) +{ + struct zstd_workspace_manager *zwsm = fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD]; + struct workspace *workspace = list_to_workspace(ws); + + ASSERT(zwsm); + spin_lock_bh(&zwsm->lock); + + /* A node is only taken off the lru if we are the corresponding level */ + if (clip_level(workspace->req_level) == workspace->level) { + /* Hide a max level workspace from reclaim */ + if (list_empty(&zwsm->idle_ws[ZSTD_BTRFS_MAX_LEVEL - 1])) { + INIT_LIST_HEAD(&workspace->lru_list); + } else { + workspace->last_used = jiffies; + list_add(&workspace->lru_list, &zwsm->lru_list); + if (!timer_pending(&zwsm->timer)) + mod_timer(&zwsm->timer, + jiffies + ZSTD_BTRFS_RECLAIM_JIFFIES); + } + } + + set_bit(workspace->level, &zwsm->active_map); + list_add(&workspace->list, &zwsm->idle_ws[workspace->level]); + workspace->req_level = 0; + + spin_unlock_bh(&zwsm->lock); + + if (workspace->level == clip_level(ZSTD_BTRFS_MAX_LEVEL)) + cond_wake_up(&zwsm->wait); +} + +void zstd_free_workspace(struct list_head *ws) +{ + struct workspace *workspace = list_entry(ws, struct workspace, list); + + kvfree(workspace->mem); + kfree(workspace->buf); + kfree(workspace); +} + +struct list_head *zstd_alloc_workspace(struct btrfs_fs_info *fs_info, int level) +{ + struct workspace *workspace; + + workspace = kzalloc_obj(*workspace); + if (!workspace) + return ERR_PTR(-ENOMEM); + + /* Use level 1 workspace size for all the fast mode negative levels. */ + workspace->size = zstd_ws_mem_sizes[clip_level(level)]; + workspace->level = clip_level(level); + workspace->req_level = level; + workspace->last_used = jiffies; + workspace->mem = kvmalloc(workspace->size, GFP_KERNEL | __GFP_NOWARN); + workspace->buf = kmalloc(fs_info->sectorsize, GFP_KERNEL); + if (!workspace->mem || !workspace->buf) + goto fail; + + INIT_LIST_HEAD(&workspace->list); + INIT_LIST_HEAD(&workspace->lru_list); + + return &workspace->list; +fail: + zstd_free_workspace(&workspace->list); + return ERR_PTR(-ENOMEM); +} + +int zstd_compress_bio(struct list_head *ws, struct compressed_bio *cb) +{ + struct btrfs_inode *inode = cb->bbio.inode; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct workspace *workspace = list_entry(ws, struct workspace, list); + struct address_space *mapping = inode->vfs_inode.i_mapping; + struct bio *bio = &cb->bbio.bio; + zstd_cstream *stream; + int ret = 0; + /* The current folio to read. */ + struct folio *in_folio = NULL; + /* The current folio to write to. */ + struct folio *out_folio = NULL; + unsigned long tot_in = 0; + unsigned long tot_out = 0; + const u64 start = cb->start; + const u32 len = cb->len; + const u64 end = start + len; + const u32 min_folio_size = btrfs_min_folio_size(fs_info); + + workspace->params = zstd_get_btrfs_parameters(workspace->req_level, len); + + /* Initialize the stream. */ + stream = zstd_init_cstream(&workspace->params, len, workspace->mem, workspace->size); + if (unlikely(!stream)) { + btrfs_err(fs_info, + "zstd compression init level %d failed, root %llu inode %llu offset %llu", + workspace->req_level, btrfs_root_id(inode->root), + btrfs_ino(inode), start); + ret = -EIO; + goto out; + } + + /* Map in the first page of input data. */ + ret = btrfs_compress_filemap_get_folio(mapping, start, &in_folio); + if (ret < 0) + goto out; + workspace->in_buf.src = kmap_local_folio(in_folio, offset_in_folio(in_folio, start)); + workspace->in_buf.pos = 0; + workspace->in_buf.size = btrfs_calc_input_length(in_folio, end, start); + + /* Allocate and map in the output buffer. */ + out_folio = btrfs_alloc_compr_folio(fs_info, GFP_NOFS); + if (out_folio == NULL) { + ret = -ENOMEM; + goto out; + } + workspace->out_buf.dst = folio_address(out_folio); + workspace->out_buf.pos = 0; + workspace->out_buf.size = min_folio_size; + + while (1) { + size_t ret2; + + ret2 = zstd_compress_stream(stream, &workspace->out_buf, &workspace->in_buf); + if (unlikely(zstd_is_error(ret2))) { + btrfs_warn(fs_info, +"zstd compression level %d failed, error %d root %llu inode %llu offset %llu", + workspace->req_level, zstd_get_error_code(ret2), + btrfs_root_id(inode->root), btrfs_ino(inode), + start + tot_in); + ret = -EIO; + goto out; + } + + /* Check to see if we are making it bigger. */ + if (tot_in + workspace->in_buf.pos > fs_info->sectorsize * 2 && + tot_in + workspace->in_buf.pos < tot_out + workspace->out_buf.pos) { + ret = -E2BIG; + goto out; + } + + /* Check if we need more output space. */ + if (workspace->out_buf.pos >= workspace->out_buf.size) { + tot_out += min_folio_size; + if (tot_out >= len) { + ret = -E2BIG; + goto out; + } + /* Queue the current foliot into the bio. */ + if (!bio_add_folio(bio, out_folio, folio_size(out_folio), 0)) { + ret = -E2BIG; + goto out; + } + + out_folio = btrfs_alloc_compr_folio(fs_info, GFP_NOFS); + if (out_folio == NULL) { + ret = -ENOMEM; + goto out; + } + workspace->out_buf.dst = folio_address(out_folio); + workspace->out_buf.pos = 0; + workspace->out_buf.size = min_folio_size; + } + + /* We've reached the end of the input. */ + if (tot_in + workspace->in_buf.pos >= len) { + tot_in += workspace->in_buf.pos; + break; + } + + /* Check if we need more input. */ + if (workspace->in_buf.pos >= workspace->in_buf.size) { + u64 cur; + + tot_in += workspace->in_buf.size; + cur = start + tot_in; + + kunmap_local(workspace->in_buf.src); + workspace->in_buf.src = NULL; + folio_put(in_folio); + + ret = btrfs_compress_filemap_get_folio(mapping, cur, &in_folio); + if (ret < 0) + goto out; + workspace->in_buf.src = kmap_local_folio(in_folio, + offset_in_folio(in_folio, cur)); + workspace->in_buf.pos = 0; + workspace->in_buf.size = btrfs_calc_input_length(in_folio, end, cur); + } + } + + while (1) { + size_t ret2; + + ret2 = zstd_end_stream(stream, &workspace->out_buf); + if (unlikely(zstd_is_error(ret2))) { + btrfs_err(fs_info, +"zstd compression end level %d failed, error %d root %llu inode %llu offset %llu", + workspace->req_level, zstd_get_error_code(ret2), + btrfs_root_id(inode->root), btrfs_ino(inode), + start + tot_in); + ret = -EIO; + goto out; + } + /* Queue the remaining part of the output folio into bio. */ + if (ret2 == 0) { + tot_out += workspace->out_buf.pos; + if (tot_out >= len) { + ret = -E2BIG; + goto out; + } + if (!bio_add_folio(bio, out_folio, workspace->out_buf.pos, 0)) { + ret = -E2BIG; + goto out; + } + out_folio = NULL; + break; + } + tot_out += min_folio_size; + if (tot_out >= len) { + ret = -E2BIG; + goto out; + } + if (!bio_add_folio(bio, out_folio, folio_size(out_folio), 0)) { + ret = -E2BIG; + goto out; + } + out_folio = btrfs_alloc_compr_folio(fs_info, GFP_NOFS); + if (out_folio == NULL) { + ret = -ENOMEM; + goto out; + } + workspace->out_buf.dst = folio_address(out_folio); + workspace->out_buf.pos = 0; + workspace->out_buf.size = min_folio_size; + } + + if (tot_out >= tot_in) { + ret = -E2BIG; + goto out; + } + + ret = 0; + ASSERT(tot_out == bio->bi_iter.bi_size); +out: + if (out_folio) + btrfs_free_compr_folio(out_folio); + if (workspace->in_buf.src) { + kunmap_local(workspace->in_buf.src); + folio_put(in_folio); + } + return ret; +} + +int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb) +{ + struct btrfs_fs_info *fs_info = cb_to_fs_info(cb); + struct workspace *workspace = list_entry(ws, struct workspace, list); + struct folio_iter fi; + size_t srclen = bio_get_size(&cb->bbio.bio); + zstd_dstream *stream; + int ret = 0; + const unsigned int min_folio_size = btrfs_min_folio_size(fs_info); + unsigned long folio_in_index = 0; + unsigned long total_folios_in = DIV_ROUND_UP(srclen, min_folio_size); + unsigned long buf_start; + unsigned long total_out = 0; + + bio_first_folio(&fi, &cb->bbio.bio, 0); + if (unlikely(!fi.folio)) + return -EINVAL; + ASSERT(folio_size(fi.folio) == min_folio_size); + + stream = zstd_init_dstream( + ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size); + if (unlikely(!stream)) { + struct btrfs_inode *inode = cb->bbio.inode; + + btrfs_err(inode->root->fs_info, + "zstd decompression init failed, root %llu inode %llu offset %llu", + btrfs_root_id(inode->root), btrfs_ino(inode), cb->start); + ret = -EIO; + goto done; + } + + workspace->in_buf.src = kmap_local_folio(fi.folio, 0); + workspace->in_buf.pos = 0; + workspace->in_buf.size = min_t(size_t, srclen, min_folio_size); + + workspace->out_buf.dst = workspace->buf; + workspace->out_buf.pos = 0; + workspace->out_buf.size = fs_info->sectorsize; + + while (1) { + size_t ret2; + + ret2 = zstd_decompress_stream(stream, &workspace->out_buf, + &workspace->in_buf); + if (unlikely(zstd_is_error(ret2))) { + struct btrfs_inode *inode = cb->bbio.inode; + + btrfs_err(inode->root->fs_info, + "zstd decompression failed, error %d root %llu inode %llu offset %llu", + zstd_get_error_code(ret2), btrfs_root_id(inode->root), + btrfs_ino(inode), cb->start); + ret = -EIO; + goto done; + } + buf_start = total_out; + total_out += workspace->out_buf.pos; + workspace->out_buf.pos = 0; + + ret = btrfs_decompress_buf2page(workspace->out_buf.dst, + total_out - buf_start, cb, buf_start); + if (ret == 0) + break; + + if (workspace->in_buf.pos >= srclen) + break; + + /* Check if we've hit the end of a frame */ + if (ret2 == 0) + break; + + if (workspace->in_buf.pos == workspace->in_buf.size) { + kunmap_local(workspace->in_buf.src); + folio_in_index++; + if (unlikely(folio_in_index >= total_folios_in)) { + workspace->in_buf.src = NULL; + ret = -EIO; + goto done; + } + srclen -= min_folio_size; + bio_next_folio(&fi, &cb->bbio.bio); + ASSERT(fi.folio); + workspace->in_buf.src = kmap_local_folio(fi.folio, 0); + workspace->in_buf.pos = 0; + workspace->in_buf.size = min_t(size_t, srclen, min_folio_size); + } + } + ret = 0; +done: + if (workspace->in_buf.src) + kunmap_local(workspace->in_buf.src); + return ret; +} + +int zstd_decompress(struct list_head *ws, const u8 *data_in, + struct folio *dest_folio, unsigned long dest_pgoff, size_t srclen, + size_t destlen) +{ + struct workspace *workspace = list_entry(ws, struct workspace, list); + struct btrfs_fs_info *fs_info = btrfs_sb(folio_inode(dest_folio)->i_sb); + zstd_dstream *stream; + int ret = 0; + unsigned long to_copy = 0; + + stream = zstd_init_dstream( + ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size); + if (unlikely(!stream)) { + struct btrfs_inode *inode = folio_to_inode(dest_folio); + + btrfs_err(inode->root->fs_info, + "zstd decompression init failed, root %llu inode %llu offset %llu", + btrfs_root_id(inode->root), btrfs_ino(inode), + folio_pos(dest_folio)); + ret = -EIO; + goto finish; + } + + workspace->in_buf.src = data_in; + workspace->in_buf.pos = 0; + workspace->in_buf.size = srclen; + + workspace->out_buf.dst = workspace->buf; + workspace->out_buf.pos = 0; + workspace->out_buf.size = fs_info->sectorsize; + + /* + * Since both input and output buffers should not exceed one sector, + * one call should end the decompression. + */ + ret = zstd_decompress_stream(stream, &workspace->out_buf, &workspace->in_buf); + if (unlikely(zstd_is_error(ret))) { + struct btrfs_inode *inode = folio_to_inode(dest_folio); + + btrfs_err(inode->root->fs_info, + "zstd decompression failed, error %d root %llu inode %llu offset %llu", + zstd_get_error_code(ret), btrfs_root_id(inode->root), + btrfs_ino(inode), folio_pos(dest_folio)); + goto finish; + } + to_copy = workspace->out_buf.pos; + memcpy_to_folio(dest_folio, dest_pgoff, workspace->out_buf.dst, to_copy); +finish: + /* Error or early end. */ + if (unlikely(to_copy < destlen)) { + ret = -EIO; + folio_zero_range(dest_folio, dest_pgoff + to_copy, destlen - to_copy); + } + return ret; +} + +const struct btrfs_compress_levels btrfs_zstd_compress = { + .min_level = ZSTD_BTRFS_MIN_LEVEL, + .max_level = ZSTD_BTRFS_MAX_LEVEL, + .default_level = ZSTD_BTRFS_DEFAULT_LEVEL, +}; diff --git a/fs/buffer.c b/fs/buffer.c new file mode 100644 index 000000000..ed966fa73 --- /dev/null +++ b/fs/buffer.c @@ -0,0 +1,3005 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/buffer.c + * + * Copyright (C) 1991, 1992, 2002 Linus Torvalds + */ + +/* + * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95 + * + * Removed a lot of unnecessary code and simplified things now that + * the buffer cache isn't our primary cache - Andrew Tridgell 12/96 + * + * Speed up hash, lru, and free list operations. Use gfp() for allocating + * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM + * + * Added 32k buffer block sizes - these are required older ARM systems. - RMK + * + * async buffer flushing, 1999 Andrea Arcangeli + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +#define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers) + +inline void touch_buffer(struct buffer_head *bh) +{ + trace_block_touch_buffer(bh); + folio_mark_accessed(bh->b_folio); +} +EXPORT_SYMBOL(touch_buffer); + +void __lock_buffer(struct buffer_head *bh) +{ + wait_on_bit_lock_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE); +} +EXPORT_SYMBOL(__lock_buffer); + +void unlock_buffer(struct buffer_head *bh) +{ + clear_and_wake_up_bit(BH_Lock, &bh->b_state); +} +EXPORT_SYMBOL(unlock_buffer); + +/* + * Returns if the folio has dirty or writeback buffers. If all the buffers + * are unlocked and clean then the folio_test_dirty information is stale. If + * any of the buffers are locked, it is assumed they are locked for IO. + */ +void buffer_check_dirty_writeback(struct folio *folio, + bool *dirty, bool *writeback) +{ + struct buffer_head *head, *bh; + *dirty = false; + *writeback = false; + + BUG_ON(!folio_test_locked(folio)); + + head = folio_buffers(folio); + if (!head) + return; + + if (folio_test_writeback(folio)) + *writeback = true; + + bh = head; + do { + if (buffer_locked(bh)) + *writeback = true; + + if (buffer_dirty(bh)) + *dirty = true; + + bh = bh->b_this_page; + } while (bh != head); +} + +/* + * Block until a buffer comes unlocked. This doesn't stop it + * from becoming locked again - you have to lock it yourself + * if you want to preserve its state. + */ +void __wait_on_buffer(struct buffer_head * bh) +{ + wait_on_bit_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE); +} +EXPORT_SYMBOL(__wait_on_buffer); + +static void buffer_io_error(struct buffer_head *bh, char *msg) +{ + if (!test_bit(BH_Quiet, &bh->b_state)) + printk_ratelimited(KERN_ERR + "Buffer I/O error on dev %pg, logical block %llu%s\n", + bh->b_bdev, (unsigned long long)bh->b_blocknr, msg); +} + +/** + * bio_endio_bh - Discard the bio used to submit a buffer. + * @bio: The bio. + * @bhp: Where to return the buffer_head. + * + * Call this in your bio_end_io handler to retrieve the buffer_head + * submitted in bh_submit(). If you did not call bh_submit(), do not + * call this function; it will return garbage. + * + * This function consumes the bio refcount which will probably free the + * bio. + * + * Return: True if the I/O succeeded. + */ +bool bio_endio_bh(struct bio *bio, struct buffer_head **bhp) +{ + bool success = bio->bi_status == BLK_STS_OK; + struct buffer_head *bh = bio->bi_private; + + if (unlikely(bio_flagged(bio, BIO_QUIET))) + set_bit(BH_Quiet, &bh->b_state); + bio_put(bio); + + *bhp = bh; + return success; +} +EXPORT_SYMBOL(bio_endio_bh); + +/** + * end_buffer_read_sync - Handle buffer reads finishing + * @bh: The buffer. + * @uptodate: True if the read was successful. + * + * If a buffer is read through a mechanism that isn't bh_submit(), you + * can call this function to finish the read. + */ +void end_buffer_read_sync(struct buffer_head *bh, int uptodate) +{ + if (uptodate) { + set_buffer_uptodate(bh); + } else { + /* This happens, due to failed read-ahead attempts. */ + clear_buffer_uptodate(bh); + } + unlock_buffer(bh); +} +EXPORT_SYMBOL(end_buffer_read_sync); + +/** + * bh_end_read - I/O end handler for reads + * @bio: The bio being completed. + * + * Pass this function to bh_submit() if you're reading into the buffer, + * unless you need your own special I/O end handler. + */ +void bh_end_read(struct bio *bio) +{ + struct buffer_head *bh; + bool uptodate = bio_endio_bh(bio, &bh); + end_buffer_read_sync(bh, uptodate); +} +EXPORT_SYMBOL(bh_end_read); + +/** + * bh_end_write - I/O end handler for writes + * @bio: The bio being completed. + * + * Pass this function to bh_submit() if you're writing from the buffer, + * unless you need your own special I/O end handler. + */ +void bh_end_write(struct bio *bio) +{ + struct buffer_head *bh; + bool success = bio_endio_bh(bio, &bh); + + if (success) { + set_buffer_uptodate(bh); + } else { + buffer_io_error(bh, ", lost sync page write"); + mark_buffer_write_io_error(bh); + clear_buffer_uptodate(bh); + } + unlock_buffer(bh); +} +EXPORT_SYMBOL(bh_end_write); + +static struct buffer_head * +__find_get_block_slow(struct block_device *bdev, sector_t block, bool atomic) +{ + struct address_space *bd_mapping = bdev->bd_mapping; + const int blkbits = bd_mapping->host->i_blkbits; + struct buffer_head *ret = NULL; + pgoff_t index; + struct buffer_head *bh; + struct buffer_head *head; + struct folio *folio; + int all_mapped = 1; + static DEFINE_RATELIMIT_STATE(last_warned, HZ, 1); + + index = ((loff_t)block << blkbits) / PAGE_SIZE; + folio = __filemap_get_folio(bd_mapping, index, FGP_ACCESSED, 0); + if (IS_ERR(folio)) + goto out; + + /* + * Folio lock protects the buffers. Callers that cannot block + * will fallback to serializing vs try_to_free_buffers() via + * the i_private_lock. + */ + if (atomic) + spin_lock(&bd_mapping->i_private_lock); + else + folio_lock(folio); + + head = folio_buffers(folio); + if (!head) + goto out_unlock; + /* + * Upon a noref migration, the folio lock serializes here; + * otherwise bail. + */ + if (test_bit_acquire(BH_Migrate, &head->b_state)) { + WARN_ON(!atomic); + goto out_unlock; + } + + bh = head; + do { + if (!buffer_mapped(bh)) + all_mapped = 0; + else if (bh->b_blocknr == block) { + ret = bh; + get_bh(bh); + goto out_unlock; + } + bh = bh->b_this_page; + } while (bh != head); + + /* we might be here because some of the buffers on this page are + * not mapped. This is due to various races between + * file io on the block device and getblk. It gets dealt with + * elsewhere, don't buffer_error if we had some unmapped buffers + */ + ratelimit_set_flags(&last_warned, RATELIMIT_MSG_ON_RELEASE); + if (all_mapped && __ratelimit(&last_warned)) { + printk("__find_get_block_slow() failed. block=%llu, " + "b_blocknr=%llu, b_state=0x%08lx, b_size=%zu, " + "device %pg blocksize: %d\n", + (unsigned long long)block, + (unsigned long long)bh->b_blocknr, + bh->b_state, bh->b_size, bdev, + 1 << blkbits); + } +out_unlock: + if (atomic) + spin_unlock(&bd_mapping->i_private_lock); + else + folio_unlock(folio); + folio_put(folio); +out: + return ret; +} + +static void end_buffer_async_read(struct buffer_head *bh, int uptodate) +{ + unsigned long flags; + struct buffer_head *first; + struct buffer_head *tmp; + struct folio *folio; + int folio_uptodate = 1; + + BUG_ON(!buffer_async_read(bh)); + + folio = bh->b_folio; + if (uptodate) { + set_buffer_uptodate(bh); + } else { + clear_buffer_uptodate(bh); + buffer_io_error(bh, ", async page read"); + } + + /* + * Be _very_ careful from here on. Bad things can happen if + * two buffer heads end IO at almost the same time and both + * decide that the page is now completely done. + */ + first = folio_buffers(folio); + spin_lock_irqsave(&first->b_uptodate_lock, flags); + clear_buffer_async_read(bh); + unlock_buffer(bh); + tmp = bh; + do { + if (!buffer_uptodate(tmp)) + folio_uptodate = 0; + if (buffer_async_read(tmp)) { + BUG_ON(!buffer_locked(tmp)); + goto still_busy; + } + tmp = tmp->b_this_page; + } while (tmp != bh); + spin_unlock_irqrestore(&first->b_uptodate_lock, flags); + + folio_end_read(folio, folio_uptodate); + return; + +still_busy: + spin_unlock_irqrestore(&first->b_uptodate_lock, flags); +} + +struct verify_bh_ctx { + struct work_struct work; + struct buffer_head *bh; + struct fsverity_info *vi; +}; + +static void verify_bh(struct work_struct *work) +{ + struct verify_bh_ctx *ctx = + container_of(work, struct verify_bh_ctx, work); + struct buffer_head *bh = ctx->bh; + bool valid; + + valid = fsverity_verify_blocks(ctx->vi, bh->b_folio, bh->b_size, + bh_offset(bh)); + end_buffer_async_read(bh, valid); + kfree(ctx); +} + +/* + * I/O completion handler for block_read_full_folio() - folios + * which come unlocked at the end of I/O. + */ +static void bh_end_async_read(struct bio *bio) +{ + struct buffer_head *bh; + bool uptodate = bio_endio_bh(bio, &bh); + struct inode *inode = bh->b_folio->mapping->host; + struct fsverity_info *vi = NULL; + + /* needed by ext4 */ + if (bh->b_folio->index < DIV_ROUND_UP(inode->i_size, PAGE_SIZE)) + vi = fsverity_get_info(inode); + + /* Verify (with fsverity) if needed. */ + if (vi && uptodate) { + struct verify_bh_ctx *ctx = kmalloc_obj(*ctx, GFP_ATOMIC); + + if (ctx) { + ctx->bh = bh; + ctx->vi = vi; + INIT_WORK(&ctx->work, verify_bh); + fsverity_enqueue_verify_work(&ctx->work); + return; + } + uptodate = false; + } + end_buffer_async_read(bh, uptodate); +} + +/** + * bh_end_async_write - I/O end handler for async folio writes + * @bio: The bio being completed. + * + * Pass this function to bh_submit() if you're doing the equivalent of + * block_write_full_folio(). That is, the folio is unlocked, and will + * have its writeback flag cleared once all async write buffers have + * completed. + */ +void bh_end_async_write(struct bio *bio) +{ + struct buffer_head *bh; + bool success = bio_endio_bh(bio, &bh); + unsigned long flags; + struct buffer_head *first; + struct buffer_head *tmp; + struct folio *folio; + + BUG_ON(!buffer_async_write(bh)); + + folio = bh->b_folio; + if (success) { + set_buffer_uptodate(bh); + } else { + buffer_io_error(bh, ", lost async page write"); + mark_buffer_write_io_error(bh); + clear_buffer_uptodate(bh); + } + + first = folio_buffers(folio); + spin_lock_irqsave(&first->b_uptodate_lock, flags); + + clear_buffer_async_write(bh); + unlock_buffer(bh); + tmp = bh->b_this_page; + while (tmp != bh) { + if (buffer_async_write(tmp)) { + BUG_ON(!buffer_locked(tmp)); + goto still_busy; + } + tmp = tmp->b_this_page; + } + spin_unlock_irqrestore(&first->b_uptodate_lock, flags); + folio_end_writeback(folio); + return; + +still_busy: + spin_unlock_irqrestore(&first->b_uptodate_lock, flags); +} +EXPORT_SYMBOL(bh_end_async_write); + + +/* + * fs/buffer.c contains helper functions for buffer-backed address space's + * fsync functions. A common requirement for buffer-based filesystems is + * that certain data from the backing blockdev needs to be written out for + * a successful fsync(). For example, ext2 indirect blocks need to be + * written back and waited upon before fsync() returns. + * + * The functions mmb_mark_buffer_dirty(), mmb_sync(), mmb_has_buffers() + * and mmb_invalidate() are provided for the management of a list of dependent + * buffers in mapping_metadata_bhs struct. + * + * The locking is a little subtle: The list of buffer heads is protected by + * the lock in mapping_metadata_bhs so functions coming from bdev mapping + * (such as try_to_free_buffers()) need to safely get to mapping_metadata_bhs + * using RCU, grab the lock, verify we didn't race with somebody detaching the + * bh / moving it to different inode and only then proceeding. + */ + +void mmb_init(struct mapping_metadata_bhs *mmb, struct address_space *mapping) +{ + spin_lock_init(&mmb->lock); + INIT_LIST_HEAD(&mmb->list); + mmb->mapping = mapping; +} +EXPORT_SYMBOL(mmb_init); + +static void __remove_assoc_queue(struct mapping_metadata_bhs *mmb, + struct buffer_head *bh) +{ + lockdep_assert_held(&mmb->lock); + list_del_init(&bh->b_assoc_buffers); + WARN_ON(!bh->b_mmb); + bh->b_mmb = NULL; +} + +static void remove_assoc_queue(struct buffer_head *bh) +{ + struct mapping_metadata_bhs *mmb; + + /* + * The locking dance is ugly here. We need to acquire the lock + * protecting the metadata bh list while possibly racing with bh + * being removed from the list or moved to a different one. We + * use RCU to pin mapping_metadata_bhs in memory to + * opportunistically acquire the lock and then recheck the bh + * didn't move under us. + */ + while (bh->b_mmb) { + rcu_read_lock(); + mmb = READ_ONCE(bh->b_mmb); + if (mmb) { + spin_lock(&mmb->lock); + if (bh->b_mmb == mmb) + __remove_assoc_queue(mmb, bh); + spin_unlock(&mmb->lock); + } + rcu_read_unlock(); + } +} + +bool mmb_has_buffers(struct mapping_metadata_bhs *mmb) +{ + return !list_empty(&mmb->list); +} +EXPORT_SYMBOL_GPL(mmb_has_buffers); + +/** + * mmb_sync - write out & wait upon all buffers in a list + * @mmb: the list of buffers to write + * + * Starts I/O against the buffers in the given list and waits upon + * that I/O. Basically, this is a convenience function for fsync(). @mmb is + * for a file or directory which needs those buffers to be written for a + * successful fsync(). + * + * We have conflicting pressures: we want to make sure that all + * initially dirty buffers get waited on, but that any subsequently + * dirtied buffers don't. After all, we don't want fsync to last + * forever if somebody is actively writing to the file. + * + * Do this in two main stages: first we copy dirty buffers to a + * temporary inode list, queueing the writes as we go. Then we clean + * up, waiting for those writes to complete. mark_buffer_dirty_inode() + * doesn't touch b_assoc_buffers list if b_mmb is not NULL so we are sure the + * buffer stays on our list until IO completes (at which point it can be + * reaped). + */ +int mmb_sync(struct mapping_metadata_bhs *mmb) +{ + struct buffer_head *bh; + int err = 0; + struct blk_plug plug; + LIST_HEAD(tmp); + + if (!mmb_has_buffers(mmb)) + return 0; + + blk_start_plug(&plug); + + spin_lock(&mmb->lock); + while (!list_empty(&mmb->list)) { + bh = BH_ENTRY(mmb->list.next); + WARN_ON_ONCE(bh->b_mmb != mmb); + __remove_assoc_queue(mmb, bh); + /* Avoid race with mark_buffer_dirty_inode() which does + * a lockless check and we rely on seeing the dirty bit */ + smp_mb(); + if (buffer_dirty(bh) || buffer_locked(bh)) { + list_add(&bh->b_assoc_buffers, &tmp); + bh->b_mmb = mmb; + if (buffer_dirty(bh)) { + get_bh(bh); + spin_unlock(&mmb->lock); + /* + * Ensure any pending I/O completes so that + * write_dirty_buffer() actually writes the + * current contents - it is a noop if I/O is + * still in flight on potentially older + * contents. + */ + write_dirty_buffer(bh, REQ_SYNC); + + /* + * Kick off IO for the previous mapping. Note + * that we will not run the very last mapping, + * wait_on_buffer() will do that for us + * through sync_buffer(). + */ + brelse(bh); + spin_lock(&mmb->lock); + } + } + } + + spin_unlock(&mmb->lock); + blk_finish_plug(&plug); + spin_lock(&mmb->lock); + + while (!list_empty(&tmp)) { + bh = BH_ENTRY(tmp.prev); + get_bh(bh); + __remove_assoc_queue(mmb, bh); + /* Avoid race with mark_buffer_dirty_inode() which does + * a lockless check and we rely on seeing the dirty bit */ + smp_mb(); + if (buffer_dirty(bh)) { + list_add(&bh->b_assoc_buffers, &mmb->list); + bh->b_mmb = mmb; + } + spin_unlock(&mmb->lock); + wait_on_buffer(bh); + if (!buffer_uptodate(bh)) + err = -EIO; + brelse(bh); + spin_lock(&mmb->lock); + } + spin_unlock(&mmb->lock); + return err; +} +EXPORT_SYMBOL(mmb_sync); + +/* + * Called when we've recently written block `bblock', and it is known that + * `bblock' was for a buffer_boundary() buffer. This means that the block at + * `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's + * dirty, schedule it for IO. So that indirects merge nicely with their data. + */ +void write_boundary_block(struct block_device *bdev, + sector_t bblock, unsigned blocksize) +{ + struct buffer_head *bh; + + bh = __find_get_block_nonatomic(bdev, bblock + 1, blocksize); + if (bh) { + if (buffer_dirty(bh)) + write_dirty_buffer(bh, 0); + put_bh(bh); + } +} + +void mmb_mark_buffer_dirty(struct buffer_head *bh, + struct mapping_metadata_bhs *mmb) +{ + mark_buffer_dirty(bh); + if (!bh->b_mmb) { + spin_lock(&mmb->lock); + /* + * For a corrupted filesystem with multiply claimed blocks this + * can fail. Avoid corrupting the linked list in that case. + */ + if (cmpxchg(&bh->b_mmb, NULL, mmb) != NULL) { + spin_unlock(&mmb->lock); + return; + } + list_move_tail(&bh->b_assoc_buffers, &mmb->list); + spin_unlock(&mmb->lock); + } +} +EXPORT_SYMBOL(mmb_mark_buffer_dirty); + +/** + * block_dirty_folio - Mark a folio as dirty. + * @mapping: The address space containing this folio. + * @folio: The folio to mark dirty. + * + * Filesystems which use buffer_heads can use this function as their + * ->dirty_folio implementation. Some filesystems need to do a little + * work before calling this function. Filesystems which do not use + * buffer_heads should call filemap_dirty_folio() instead. + * + * If the folio has buffers, the uptodate buffers are set dirty, to + * preserve dirty-state coherency between the folio and the buffers. + * Buffers added to a dirty folio are created dirty. + * + * The buffers are dirtied before the folio is dirtied. There's a small + * race window in which writeback may see the folio cleanness but not the + * buffer dirtiness. That's fine. If this code were to set the folio + * dirty before the buffers, writeback could clear the folio dirty flag, + * see a bunch of clean buffers and we'd end up with dirty buffers/clean + * folio on the dirty folio list. + * + * We use i_private_lock to lock against try_to_free_buffers() while + * using the folio's buffer list. This also prevents clean buffers + * being added to the folio after it was set dirty. + * + * Context: May only be called from process context. Does not sleep. + * Caller must ensure that @folio cannot be truncated during this call, + * typically by holding the folio lock or having a page in the folio + * mapped and holding the page table lock. + * + * Return: True if the folio was dirtied; false if it was already dirtied. + */ +bool block_dirty_folio(struct address_space *mapping, struct folio *folio) +{ + struct buffer_head *head; + bool newly_dirty; + + spin_lock(&mapping->i_private_lock); + head = folio_buffers(folio); + if (head) { + struct buffer_head *bh = head; + + do { + set_buffer_dirty(bh); + bh = bh->b_this_page; + } while (bh != head); + } + /* + * Lock out page's memcg migration to keep PageDirty + * synchronized with per-memcg dirty page counters. + */ + newly_dirty = !folio_test_set_dirty(folio); + spin_unlock(&mapping->i_private_lock); + + if (newly_dirty) + __folio_mark_dirty(folio, mapping, 1); + + if (newly_dirty) + __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); + + return newly_dirty; +} +EXPORT_SYMBOL(block_dirty_folio); + +/* + * Invalidate any and all dirty buffers on a given buffers list. We are + * probably unmounting the fs, but that doesn't mean we have already + * done a sync(). Just drop the buffers from the inode list. + */ +void mmb_invalidate(struct mapping_metadata_bhs *mmb) +{ + if (mmb_has_buffers(mmb)) { + spin_lock(&mmb->lock); + while (!list_empty(&mmb->list)) + __remove_assoc_queue(mmb, BH_ENTRY(mmb->list.next)); + spin_unlock(&mmb->lock); + } +} +EXPORT_SYMBOL(mmb_invalidate); + +/* + * Create the appropriate buffers when given a folio for data area and + * the size of each buffer.. Use the bh->b_this_page linked list to + * follow the buffers created. Return NULL if unable to create more + * buffers. + * + * The retry flag is used to differentiate async IO (paging, swapping) + * which may not fail from ordinary buffer allocations. + */ +struct buffer_head *folio_alloc_buffers(struct folio *folio, unsigned long size, + gfp_t gfp) +{ + struct buffer_head *bh, *head; + long offset; + struct mem_cgroup *memcg, *old_memcg; + + memcg = get_mem_cgroup_from_folio(folio); + old_memcg = set_active_memcg(memcg); + + head = NULL; + offset = folio_size(folio); + while ((offset -= size) >= 0) { + bh = alloc_buffer_head(gfp); + if (!bh) + goto no_grow; + + bh->b_this_page = head; + bh->b_blocknr = -1; + head = bh; + + bh->b_size = size; + + /* Link the buffer to its folio */ + folio_set_bh(bh, folio, offset); + } +out: + set_active_memcg(old_memcg); + mem_cgroup_put(memcg); + return head; +/* + * In case anything failed, we just free everything we got. + */ +no_grow: + if (head) { + do { + bh = head; + head = head->b_this_page; + free_buffer_head(bh); + } while (head); + } + + goto out; +} +EXPORT_SYMBOL_GPL(folio_alloc_buffers); + +struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size) +{ + gfp_t gfp = GFP_NOFS | __GFP_ACCOUNT; + + return folio_alloc_buffers(page_folio(page), size, gfp); +} +EXPORT_SYMBOL_GPL(alloc_page_buffers); + +static inline void link_dev_buffers(struct folio *folio, + struct buffer_head *head) +{ + struct buffer_head *bh, *tail; + + bh = head; + do { + tail = bh; + bh = bh->b_this_page; + } while (bh); + tail->b_this_page = head; + folio_attach_private(folio, head); +} + +static sector_t blkdev_max_block(struct block_device *bdev, unsigned int size) +{ + sector_t retval = ~((sector_t)0); + loff_t sz = bdev_nr_bytes(bdev); + + if (sz) { + unsigned int sizebits = blksize_bits(size); + retval = (sz >> sizebits); + } + return retval; +} + +/* + * Initialise the state of a blockdev folio's buffers. + */ +static sector_t folio_init_buffers(struct folio *folio, + struct block_device *bdev, unsigned size) +{ + struct buffer_head *head = folio_buffers(folio); + struct buffer_head *bh = head; + bool uptodate = folio_test_uptodate(folio); + sector_t block = div_u64(folio_pos(folio), size); + sector_t end_block = blkdev_max_block(bdev, size); + + do { + if (!buffer_mapped(bh)) { + bh->b_private = NULL; + bh->b_bdev = bdev; + bh->b_blocknr = block; + if (uptodate) + set_buffer_uptodate(bh); + if (block < end_block) + set_buffer_mapped(bh); + } + block++; + bh = bh->b_this_page; + } while (bh != head); + + /* + * Caller needs to validate requested block against end of device. + */ + return end_block; +} + +/* + * Create the page-cache folio that contains the requested block. + * + * This is used purely for blockdev mappings. + * + * Returns false if we have a failure which cannot be cured by retrying + * without sleeping. Returns true if we succeeded, or the caller should retry. + */ +static bool grow_dev_folio(struct block_device *bdev, sector_t block, + pgoff_t index, unsigned size, gfp_t gfp) +{ + struct address_space *mapping = bdev->bd_mapping; + struct folio *folio; + struct buffer_head *bh; + sector_t end_block = 0; + + folio = __filemap_get_folio(mapping, index, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp); + if (IS_ERR(folio)) + return false; + + bh = folio_buffers(folio); + if (bh) { + if (bh->b_size == size) { + end_block = folio_init_buffers(folio, bdev, size); + goto unlock; + } + + /* + * Retrying may succeed; for example the folio may finish + * writeback, or buffers may be cleaned. This should not + * happen very often; maybe we have old buffers attached to + * this blockdev's page cache and we're trying to change + * the block size? + */ + if (!try_to_free_buffers(folio)) { + end_block = ~0ULL; + goto unlock; + } + } + + bh = folio_alloc_buffers(folio, size, gfp | __GFP_ACCOUNT); + if (!bh) + goto unlock; + + /* + * Link the folio to the buffers and initialise them. Take the + * lock to be atomic wrt __find_get_block(), which does not + * run under the folio lock. + */ + spin_lock(&mapping->i_private_lock); + link_dev_buffers(folio, bh); + end_block = folio_init_buffers(folio, bdev, size); + spin_unlock(&mapping->i_private_lock); +unlock: + folio_unlock(folio); + folio_put(folio); + return block < end_block; +} + +/* + * Create buffers for the specified block device block's folio. If + * that folio was dirty, the buffers are set dirty also. Returns false + * if we've hit a permanent error. + */ +static bool grow_buffers(struct block_device *bdev, sector_t block, + unsigned size, gfp_t gfp) +{ + loff_t pos; + + /* + * Check for a block which lies outside our maximum possible + * pagecache index. + */ + if (check_mul_overflow(block, (sector_t)size, &pos) || pos > MAX_LFS_FILESIZE) { + printk(KERN_ERR "%s: requested out-of-range block %llu for device %pg\n", + __func__, (unsigned long long)block, + bdev); + return false; + } + + /* Create a folio with the proper size buffers */ + return grow_dev_folio(bdev, block, pos / PAGE_SIZE, size, gfp); +} + +static struct buffer_head * +__getblk_slow(struct block_device *bdev, sector_t block, + unsigned size, gfp_t gfp) +{ + bool blocking = gfpflags_allow_blocking(gfp); + + if (WARN_ON_ONCE(!IS_ALIGNED(size, bdev_logical_block_size(bdev)))) { + printk(KERN_ERR "getblk(): block size %d not aligned to logical block size %d\n", + size, bdev_logical_block_size(bdev)); + return NULL; + } + + for (;;) { + struct buffer_head *bh; + + if (!grow_buffers(bdev, block, size, gfp)) + return NULL; + + if (blocking) + bh = __find_get_block_nonatomic(bdev, block, size); + else + bh = __find_get_block(bdev, block, size); + if (bh) + return bh; + } +} + +/* + * The relationship between dirty buffers and dirty pages: + * + * Whenever a page has any dirty buffers, the page's dirty bit is set, and + * the page is tagged dirty in the page cache. + * + * At all times, the dirtiness of the buffers represents the dirtiness of + * subsections of the page. If the page has buffers, the page dirty bit is + * merely a hint about the true dirty state. + * + * When a page is set dirty in its entirety, all its buffers are marked dirty + * (if the page has buffers). + * + * When a buffer is marked dirty, its page is dirtied, but the page's other + * buffers are not. + * + * Also. When blockdev buffers are explicitly read with bread(), they + * individually become uptodate. But their backing page remains not + * uptodate - even if all of its buffers are uptodate. A subsequent + * block_read_full_folio() against that folio will discover all the uptodate + * buffers, will set the folio uptodate and will perform no I/O. + */ + +/** + * mark_buffer_dirty - mark a buffer_head as needing writeout + * @bh: the buffer_head to mark dirty + * + * mark_buffer_dirty() will set the dirty bit against the buffer, then set + * its backing page dirty, then tag the page as dirty in the page cache + * and then attach the address_space's inode to its superblock's dirty + * inode list. + * + * mark_buffer_dirty() is atomic. It takes bh->b_folio->mapping->i_private_lock, + * i_pages lock and mapping->host->i_lock. + */ +void mark_buffer_dirty(struct buffer_head *bh) +{ + WARN_ON_ONCE(!buffer_uptodate(bh)); + + trace_block_dirty_buffer(bh); + + /* + * Very *carefully* optimize the it-is-already-dirty case. + * + * Don't let the final "is it dirty" escape to before we + * perhaps modified the buffer. + */ + if (buffer_dirty(bh)) { + smp_mb(); + if (buffer_dirty(bh)) + return; + } + + if (!test_set_buffer_dirty(bh)) { + struct folio *folio = bh->b_folio; + struct address_space *mapping = NULL; + + if (!folio_test_set_dirty(folio)) { + mapping = folio->mapping; + if (mapping) + __folio_mark_dirty(folio, mapping, 0); + } + if (mapping) + __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); + } +} +EXPORT_SYMBOL(mark_buffer_dirty); + +void mark_buffer_write_io_error(struct buffer_head *bh) +{ + struct mapping_metadata_bhs *mmb; + + set_buffer_write_io_error(bh); + /* FIXME: do we need to set this in both places? */ + if (bh->b_folio && bh->b_folio->mapping) + mapping_set_error(bh->b_folio->mapping, -EIO); + /* Protect us from mmb & inode getting freed while we work on it */ + rcu_read_lock(); + mmb = READ_ONCE(bh->b_mmb); + if (mmb) + mapping_set_error(mmb->mapping, -EIO); + rcu_read_unlock(); +} +EXPORT_SYMBOL(mark_buffer_write_io_error); + +/** + * __brelse - Release a buffer. + * @bh: The buffer to release. + * + * This variant of brelse() can be called if @bh is guaranteed to not be NULL. + */ +void __brelse(struct buffer_head *bh) +{ + if (atomic_read(&bh->b_count)) { + put_bh(bh); + return; + } + WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n"); +} +EXPORT_SYMBOL(__brelse); + +/** + * __bforget - Discard any dirty data in a buffer. + * @bh: The buffer to forget. + * + * This variant of bforget() can be called if @bh is guaranteed to not + * be NULL. + */ +void __bforget(struct buffer_head *bh) +{ + clear_buffer_dirty(bh); + remove_assoc_queue(bh); + __brelse(bh); +} +EXPORT_SYMBOL(__bforget); + +static void buffer_set_crypto_ctx(struct bio *bio, const struct buffer_head *bh, + gfp_t gfp_mask) +{ + const struct address_space *mapping = folio_mapping(bh->b_folio); + + /* + * The ext4 journal (jbd2) can submit a buffer_head it directly created + * for a non-pagecache page. fscrypt doesn't care about these. + */ + if (!mapping) + return; + fscrypt_set_bio_crypt_ctx(bio, mapping->host, + folio_pos(bh->b_folio) + bh_offset(bh), gfp_mask); +} + +static void __bh_submit(struct buffer_head *bh, blk_opf_t opf, + enum rw_hint write_hint, struct writeback_control *wbc, + bio_end_io_t end_bio) +{ + const enum req_op op = opf & REQ_OP_MASK; + struct bio *bio; + + BUG_ON(!buffer_locked(bh)); + BUG_ON(!buffer_mapped(bh)); + BUG_ON(buffer_delay(bh)); + BUG_ON(buffer_unwritten(bh)); + + /* + * Only clear out a write error when rewriting + */ + if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE)) + clear_buffer_write_io_error(bh); + + if (buffer_meta(bh)) + opf |= REQ_META; + if (buffer_prio(bh)) + opf |= REQ_PRIO; + + bio = bio_alloc(bh->b_bdev, 1, opf, GFP_NOIO); + + if (folio_test_dropbehind(bh->b_folio) && op_is_write(opf)) + bio_set_flag(bio, BIO_COMPLETE_IN_TASK); + + if (IS_ENABLED(CONFIG_FS_ENCRYPTION)) + buffer_set_crypto_ctx(bio, bh, GFP_NOIO); + + bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); + bio->bi_write_hint = write_hint; + + bio_add_folio_nofail(bio, bh->b_folio, bh->b_size, bh_offset(bh)); + + bio->bi_end_io = end_bio; + bio->bi_private = bh; + + /* Take care of bh's that straddle the end of the device */ + guard_bio_eod(bio); + + if (wbc) { + wbc_init_bio(wbc, bio); + wbc_account_cgroup_owner(wbc, bh->b_folio, bh->b_size); + } + + blk_crypto_submit_bio(bio); +} + +/** + * bh_submit - Start I/O against a buffer head + * @bh: The buffer head to perform I/O on. + * @opf: Operation and flags for bio. + * @end_io: The routine to call when I/O has completed. + * + * If you need to do I/O on an individual bh (instead of allowing the + * page cache to do I/O on the folio that it is in), call this function. + */ +void bh_submit(struct buffer_head *bh, blk_opf_t opf, bio_end_io_t end_io) +{ + __bh_submit(bh, opf, WRITE_LIFE_NOT_SET, NULL, end_io); +} +EXPORT_SYMBOL(bh_submit); + +static struct buffer_head *__bread_slow(struct buffer_head *bh) +{ + lock_buffer(bh); + if (buffer_uptodate(bh)) { + unlock_buffer(bh); + return bh; + } else { + bh_submit(bh, REQ_OP_READ, bh_end_read); + wait_on_buffer(bh); + if (buffer_uptodate(bh)) + return bh; + } + brelse(bh); + return NULL; +} + +/* + * Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block(). + * The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their + * refcount elevated by one when they're in an LRU. A buffer can only appear + * once in a particular CPU's LRU. A single buffer can be present in multiple + * CPU's LRUs at the same time. + * + * This is a transparent caching front-end to sb_bread(), sb_getblk() and + * sb_find_get_block(). + * + * The LRUs themselves only need locking against invalidate_bh_lrus. We use + * a local interrupt disable for that. + */ + +#define BH_LRU_SIZE 16 + +struct bh_lru { + struct buffer_head *bhs[BH_LRU_SIZE]; +}; + +static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }}; + +#ifdef CONFIG_SMP +#define bh_lru_lock() local_irq_disable() +#define bh_lru_unlock() local_irq_enable() +#else +#define bh_lru_lock() preempt_disable() +#define bh_lru_unlock() preempt_enable() +#endif + +static inline void check_irqs_on(void) +{ +#ifdef irqs_disabled + BUG_ON(irqs_disabled()); +#endif +} + +/* + * Install a buffer_head into this cpu's LRU. If not already in the LRU, it is + * inserted at the front, and the buffer_head at the back if any is evicted. + * Or, if already in the LRU it is moved to the front. + */ +static void bh_lru_install(struct buffer_head *bh) +{ + struct buffer_head *evictee = bh; + struct bh_lru *b; + int i; + + check_irqs_on(); + bh_lru_lock(); + + /* + * the refcount of buffer_head in bh_lru prevents dropping the + * attached page(i.e., try_to_free_buffers) so it could cause + * failing page migration. + * Skip putting upcoming bh into bh_lru until migration is done. + */ + if (lru_cache_disabled() || cpu_is_isolated(smp_processor_id())) { + bh_lru_unlock(); + return; + } + + b = this_cpu_ptr(&bh_lrus); + for (i = 0; i < BH_LRU_SIZE; i++) { + swap(evictee, b->bhs[i]); + if (evictee == bh) { + bh_lru_unlock(); + return; + } + } + + get_bh(bh); + bh_lru_unlock(); + brelse(evictee); +} + +/* + * Look up the bh in this cpu's LRU. If it's there, move it to the head. + */ +static struct buffer_head * +lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size) +{ + struct buffer_head *ret = NULL; + unsigned int i; + + check_irqs_on(); + bh_lru_lock(); + if (cpu_is_isolated(smp_processor_id())) { + bh_lru_unlock(); + return NULL; + } + for (i = 0; i < BH_LRU_SIZE; i++) { + struct buffer_head *bh = __this_cpu_read(bh_lrus.bhs[i]); + + if (bh && bh->b_blocknr == block && bh->b_bdev == bdev && + bh->b_size == size) { + if (i) { + while (i) { + __this_cpu_write(bh_lrus.bhs[i], + __this_cpu_read(bh_lrus.bhs[i - 1])); + i--; + } + __this_cpu_write(bh_lrus.bhs[0], bh); + } + get_bh(bh); + ret = bh; + break; + } + } + bh_lru_unlock(); + return ret; +} + +/* + * Perform a pagecache lookup for the matching buffer. If it's there, refresh + * it in the LRU and mark it as accessed. If it is not present then return + * NULL. Atomic context callers may also return NULL if the buffer is being + * migrated; similarly the page is not marked accessed either. + */ +static struct buffer_head * +find_get_block_common(struct block_device *bdev, sector_t block, + unsigned size, bool atomic) +{ + struct buffer_head *bh = lookup_bh_lru(bdev, block, size); + + if (bh == NULL) { + /* __find_get_block_slow will mark the page accessed */ + bh = __find_get_block_slow(bdev, block, atomic); + if (bh) + bh_lru_install(bh); + } else + touch_buffer(bh); + + return bh; +} + +struct buffer_head * +__find_get_block(struct block_device *bdev, sector_t block, unsigned size) +{ + return find_get_block_common(bdev, block, size, true); +} +EXPORT_SYMBOL(__find_get_block); + +/* same as __find_get_block() but allows sleeping contexts */ +struct buffer_head * +__find_get_block_nonatomic(struct block_device *bdev, sector_t block, + unsigned size) +{ + return find_get_block_common(bdev, block, size, false); +} +EXPORT_SYMBOL(__find_get_block_nonatomic); + +/** + * bdev_getblk - Get a buffer_head in a block device's buffer cache. + * @bdev: The block device. + * @block: The block number. + * @size: The size of buffer_heads for this @bdev. + * @gfp: The memory allocation flags to use. + * + * The returned buffer head has its reference count incremented, but is + * not locked. The caller should call brelse() when it has finished + * with the buffer. The buffer may not be uptodate. If needed, the + * caller can bring it uptodate either by reading it or overwriting it. + * + * Return: The buffer head, or NULL if memory could not be allocated. + */ +struct buffer_head *bdev_getblk(struct block_device *bdev, sector_t block, + unsigned size, gfp_t gfp) +{ + struct buffer_head *bh; + + if (gfpflags_allow_blocking(gfp)) + bh = __find_get_block_nonatomic(bdev, block, size); + else + bh = __find_get_block(bdev, block, size); + + might_alloc(gfp); + if (bh) + return bh; + + return __getblk_slow(bdev, block, size, gfp); +} +EXPORT_SYMBOL(bdev_getblk); + +/* + * Do async read-ahead on a buffer.. + */ +void __breadahead(struct block_device *bdev, sector_t block, unsigned size) +{ + struct buffer_head *bh = bdev_getblk(bdev, block, size, + GFP_NOWAIT | __GFP_MOVABLE); + + if (likely(bh)) { + bh_readahead(bh, REQ_RAHEAD); + brelse(bh); + } +} +EXPORT_SYMBOL(__breadahead); + +/** + * __bread_gfp() - Read a block. + * @bdev: The block device to read from. + * @block: Block number in units of block size. + * @size: The block size of this device in bytes. + * @gfp: Not page allocation flags; see below. + * + * You are not expected to call this function. You should use one of + * sb_bread(), sb_bread_unmovable() or __bread(). + * + * Read a specified block, and return the buffer head that refers to it. + * If @gfp is 0, the memory will be allocated using the block device's + * default GFP flags. If @gfp is __GFP_MOVABLE, the memory may be + * allocated from a movable area. Do not pass in a complete set of + * GFP flags. + * + * The returned buffer head has its refcount increased. The caller should + * call brelse() when it has finished with the buffer. + * + * Context: May sleep waiting for I/O. + * Return: NULL if the block was unreadable. + */ +struct buffer_head *__bread_gfp(struct block_device *bdev, sector_t block, + unsigned size, gfp_t gfp) +{ + struct buffer_head *bh; + + gfp |= mapping_gfp_constraint(bdev->bd_mapping, ~__GFP_FS); + + /* + * Prefer looping in the allocator rather than here, at least that + * code knows what it's doing. + */ + gfp |= __GFP_NOFAIL; + + bh = bdev_getblk(bdev, block, size, gfp); + + if (likely(bh) && !buffer_uptodate(bh)) + bh = __bread_slow(bh); + return bh; +} +EXPORT_SYMBOL(__bread_gfp); + +static void __invalidate_bh_lrus(struct bh_lru *b) +{ + int i; + + for (i = 0; i < BH_LRU_SIZE; i++) { + brelse(b->bhs[i]); + b->bhs[i] = NULL; + } +} +/* + * invalidate_bh_lrus() is called rarely - but not only at unmount. + * This doesn't race because it runs in each cpu either in irq + * or with preempt disabled. + */ +static void invalidate_bh_lru(void *arg) +{ + struct bh_lru *b = &get_cpu_var(bh_lrus); + + __invalidate_bh_lrus(b); + put_cpu_var(bh_lrus); +} + +bool has_bh_in_lru(int cpu, void *dummy) +{ + struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu); + int i; + + for (i = 0; i < BH_LRU_SIZE; i++) { + if (b->bhs[i]) + return true; + } + + return false; +} + +void invalidate_bh_lrus(void) +{ + on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1); +} +EXPORT_SYMBOL_GPL(invalidate_bh_lrus); + +/* + * It's called from workqueue context so we need a bh_lru_lock to close + * the race with preemption/irq. + */ +void invalidate_bh_lrus_cpu(void) +{ + struct bh_lru *b; + + bh_lru_lock(); + b = this_cpu_ptr(&bh_lrus); + __invalidate_bh_lrus(b); + bh_lru_unlock(); +} + +void folio_set_bh(struct buffer_head *bh, struct folio *folio, + unsigned long offset) +{ + bh->b_folio = folio; + BUG_ON(offset >= folio_size(folio)); + if (folio_test_highmem(folio)) + /* + * This catches illegal uses and preserves the offset: + */ + bh->b_data = (char *)(0 + offset); + else + bh->b_data = folio_address(folio) + offset; +} +EXPORT_SYMBOL(folio_set_bh); + +/* + * Called when truncating a buffer on a page completely. + */ + +/* Bits that are cleared during an invalidate */ +#define BUFFER_FLAGS_DISCARD \ + (1 << BH_Mapped | 1 << BH_New | 1 << BH_Req | \ + 1 << BH_Delay | 1 << BH_Unwritten) + +static void discard_buffer(struct buffer_head * bh) +{ + unsigned long b_state; + + lock_buffer(bh); + clear_buffer_dirty(bh); + bh->b_bdev = NULL; + b_state = READ_ONCE(bh->b_state); + do { + } while (!try_cmpxchg_relaxed(&bh->b_state, &b_state, + b_state & ~BUFFER_FLAGS_DISCARD)); + unlock_buffer(bh); +} + +/** + * block_invalidate_folio - Invalidate part or all of a buffer-backed folio. + * @folio: The folio which is affected. + * @offset: start of the range to invalidate + * @length: length of the range to invalidate + * + * block_invalidate_folio() is called when all or part of the folio has been + * invalidated by a truncate operation. + * + * block_invalidate_folio() does not have to release all buffers, but it must + * ensure that no dirty buffer is left outside @offset and that no I/O + * is underway against any of the blocks which are outside the truncation + * point. Because the caller is about to free (and possibly reuse) those + * blocks on-disk. + */ +void block_invalidate_folio(struct folio *folio, size_t offset, size_t length) +{ + struct buffer_head *head, *bh, *next; + size_t curr_off = 0; + size_t stop = length + offset; + + BUG_ON(!folio_test_locked(folio)); + + /* + * Check for overflow + */ + BUG_ON(stop > folio_size(folio) || stop < length); + + head = folio_buffers(folio); + if (!head) + return; + + bh = head; + do { + size_t next_off = curr_off + bh->b_size; + next = bh->b_this_page; + + /* + * Are we still fully in range ? + */ + if (next_off > stop) + goto out; + + /* + * is this block fully invalidated? + */ + if (offset <= curr_off) + discard_buffer(bh); + curr_off = next_off; + bh = next; + } while (bh != head); + + /* + * We release buffers only if the entire folio is being invalidated. + * The get_block cached value has been unconditionally invalidated, + * so real IO is not possible anymore. + */ + if (length == folio_size(folio)) + filemap_release_folio(folio, 0); +out: + folio_clear_mappedtodisk(folio); +} +EXPORT_SYMBOL(block_invalidate_folio); + +/* + * We attach and possibly dirty the buffers atomically wrt + * block_dirty_folio() via i_private_lock. try_to_free_buffers + * is already excluded via the folio lock. + */ +struct buffer_head *create_empty_buffers(struct folio *folio, + unsigned long blocksize, unsigned long b_state) +{ + struct buffer_head *bh, *head, *tail; + gfp_t gfp = GFP_NOFS | __GFP_ACCOUNT | __GFP_NOFAIL; + + head = folio_alloc_buffers(folio, blocksize, gfp); + bh = head; + do { + bh->b_state |= b_state; + tail = bh; + bh = bh->b_this_page; + } while (bh); + tail->b_this_page = head; + + spin_lock(&folio->mapping->i_private_lock); + if (folio_test_uptodate(folio) || folio_test_dirty(folio)) { + bh = head; + do { + if (folio_test_dirty(folio)) + set_buffer_dirty(bh); + if (folio_test_uptodate(folio)) + set_buffer_uptodate(bh); + bh = bh->b_this_page; + } while (bh != head); + } + folio_attach_private(folio, head); + spin_unlock(&folio->mapping->i_private_lock); + + return head; +} +EXPORT_SYMBOL(create_empty_buffers); + +/** + * clean_bdev_aliases: clean a range of buffers in block device + * @bdev: Block device to clean buffers in + * @block: Start of a range of blocks to clean + * @len: Number of blocks to clean + * + * We are taking a range of blocks for data and we don't want writeback of any + * buffer-cache aliases starting from return from this function and until the + * moment when something will explicitly mark the buffer dirty (hopefully that + * will not happen until we will free that block ;-) We don't even need to mark + * it not-uptodate - nobody can expect anything from a newly allocated buffer + * anyway. We used to use unmap_buffer() for such invalidation, but that was + * wrong. We definitely don't want to mark the alias unmapped, for example - it + * would confuse anyone who might pick it with bread() afterwards... + * + * Also.. Note that bforget() doesn't lock the buffer. So there can be + * writeout I/O going on against recently-freed buffers. We don't wait on that + * I/O in bforget() - it's more efficient to wait on the I/O only if we really + * need to. That happens here. + */ +void clean_bdev_aliases(struct block_device *bdev, sector_t block, sector_t len) +{ + struct address_space *bd_mapping = bdev->bd_mapping; + const int blkbits = bd_mapping->host->i_blkbits; + struct folio_batch fbatch; + pgoff_t index = ((loff_t)block << blkbits) / PAGE_SIZE; + pgoff_t end; + int i, count; + struct buffer_head *bh; + struct buffer_head *head; + + end = ((loff_t)(block + len - 1) << blkbits) / PAGE_SIZE; + folio_batch_init(&fbatch); + while (filemap_get_folios(bd_mapping, &index, end, &fbatch)) { + count = folio_batch_count(&fbatch); + for (i = 0; i < count; i++) { + struct folio *folio = fbatch.folios[i]; + + if (!folio_buffers(folio)) + continue; + /* + * We use folio lock instead of bd_mapping->i_private_lock + * to pin buffers here since we can afford to sleep and + * it scales better than a global spinlock lock. + */ + folio_lock(folio); + /* Recheck when the folio is locked which pins bhs */ + head = folio_buffers(folio); + if (!head) + goto unlock_page; + bh = head; + do { + if (!buffer_mapped(bh) || (bh->b_blocknr < block)) + goto next; + if (bh->b_blocknr >= block + len) + break; + clear_buffer_dirty(bh); + wait_on_buffer(bh); + clear_buffer_req(bh); +next: + bh = bh->b_this_page; + } while (bh != head); +unlock_page: + folio_unlock(folio); + } + folio_batch_release(&fbatch); + cond_resched(); + /* End of range already reached? */ + if (index > end || !index) + break; + } +} +EXPORT_SYMBOL(clean_bdev_aliases); + +static struct buffer_head *folio_create_buffers(struct folio *folio, + struct inode *inode, + unsigned int b_state) +{ + struct buffer_head *bh; + + BUG_ON(!folio_test_locked(folio)); + + bh = folio_buffers(folio); + if (!bh) + bh = create_empty_buffers(folio, + 1 << READ_ONCE(inode->i_blkbits), b_state); + return bh; +} + +/* + * NOTE! All mapped/uptodate combinations are valid: + * + * Mapped Uptodate Meaning + * + * No No "unknown" - must do get_block() + * No Yes "hole" - zero-filled + * Yes No "allocated" - allocated on disk, not read in + * Yes Yes "valid" - allocated and up-to-date in memory. + * + * "Dirty" is valid only with the last case (mapped+uptodate). + */ + +/* + * While block_write_full_folio is writing back the dirty buffers under + * the folio lock, whoever dirtied the buffers may decide to clean them + * again at any time. We handle that by only looking at the buffer + * state inside lock_buffer(). + * + * If block_write_full_folio() is called for regular writeback + * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a folio which + * has a locked buffer. This only can happen if someone has written + * the buffer directly, with bh_submit(). At the address_space level + * the folio writeback flag prevents this contention from occurring. + * + * If block_write_full_folio() is called with wbc->sync_mode == + * WB_SYNC_ALL, the writes are posted using REQ_SYNC; this + * causes the writes to be flagged as synchronous writes. + */ +int __block_write_full_folio(struct inode *inode, struct folio *folio, + get_block_t *get_block, struct writeback_control *wbc) +{ + int err; + sector_t block; + sector_t last_block; + struct buffer_head *bh, *head; + size_t blocksize; + int nr_underway = 0; + blk_opf_t write_flags = wbc_to_write_flags(wbc); + + head = folio_create_buffers(folio, inode, + (1 << BH_Dirty) | (1 << BH_Uptodate)); + + /* + * Be very careful. We have no exclusion from block_dirty_folio + * here, and the (potentially unmapped) buffers may become dirty at + * any time. If a buffer becomes dirty here after we've inspected it + * then we just miss that fact, and the folio stays dirty. + * + * Buffers outside i_size may be dirtied by block_dirty_folio; + * handle that here by just cleaning them. + */ + + bh = head; + blocksize = bh->b_size; + + block = div_u64(folio_pos(folio), blocksize); + last_block = div_u64(i_size_read(inode) - 1, blocksize); + + /* + * Get all the dirty buffers mapped to disk addresses and + * handle any aliases from the underlying blockdev's mapping. + */ + do { + if (block > last_block) { + /* + * mapped buffers outside i_size will occur, because + * this folio can be outside i_size when there is a + * truncate in progress. + */ + /* + * The buffer was zeroed by block_write_full_folio() + */ + clear_buffer_dirty(bh); + set_buffer_uptodate(bh); + } else if ((!buffer_mapped(bh) || buffer_delay(bh)) && + buffer_dirty(bh)) { + WARN_ON(bh->b_size != blocksize); + err = get_block(inode, block, bh, 1); + if (err) + goto recover; + clear_buffer_delay(bh); + if (buffer_new(bh)) { + /* blockdev mappings never come here */ + clear_buffer_new(bh); + clean_bdev_bh_alias(bh); + } + } + bh = bh->b_this_page; + block++; + } while (bh != head); + + do { + if (!buffer_mapped(bh)) + continue; + /* + * If it's a fully non-blocking write attempt and we cannot + * lock the buffer then redirty the folio. Note that this can + * potentially cause a busy-wait loop from writeback threads + * and kswapd activity, but those code paths have their own + * higher-level throttling. + */ + if (wbc->sync_mode != WB_SYNC_NONE) { + lock_buffer(bh); + } else if (!trylock_buffer(bh)) { + folio_redirty_for_writepage(wbc, folio); + continue; + } + if (test_clear_buffer_dirty(bh)) { + set_buffer_async_write(bh); + } else { + unlock_buffer(bh); + } + } while ((bh = bh->b_this_page) != head); + + /* + * The folio and its buffers are protected by the writeback flag, + * so we can drop the bh refcounts early. + */ + BUG_ON(folio_test_writeback(folio)); + folio_start_writeback(folio); + + do { + struct buffer_head *next = bh->b_this_page; + if (buffer_async_write(bh)) { + __bh_submit(bh, REQ_OP_WRITE | write_flags, + inode->i_write_hint, wbc, + bh_end_async_write); + nr_underway++; + } + bh = next; + } while (bh != head); + folio_unlock(folio); + + err = 0; +done: + if (nr_underway == 0) { + /* + * The folio was marked dirty, but the buffers were + * clean. Someone wrote them back by hand with + * write_dirty_buffer/bh_submit. A rare case. + */ + folio_end_writeback(folio); + + /* + * The folio and buffer_heads can be released at any time from + * here on. + */ + } + return err; + +recover: + /* + * ENOSPC, or some other error. We may already have added some + * blocks to the file, so we need to write these out to avoid + * exposing stale data. + * The folio is currently locked and not marked for writeback + */ + bh = head; + /* Recovery: lock and submit the mapped buffers */ + do { + if (buffer_mapped(bh) && buffer_dirty(bh) && + !buffer_delay(bh)) { + lock_buffer(bh); + set_buffer_async_write(bh); + } else { + /* + * The buffer may have been set dirty during + * attachment to a dirty folio. + */ + clear_buffer_dirty(bh); + } + } while ((bh = bh->b_this_page) != head); + BUG_ON(folio_test_writeback(folio)); + mapping_set_error(folio->mapping, err); + folio_start_writeback(folio); + do { + struct buffer_head *next = bh->b_this_page; + if (buffer_async_write(bh)) { + clear_buffer_dirty(bh); + __bh_submit(bh, REQ_OP_WRITE | write_flags, + inode->i_write_hint, wbc, + bh_end_async_write); + nr_underway++; + } + bh = next; + } while (bh != head); + folio_unlock(folio); + goto done; +} +EXPORT_SYMBOL(__block_write_full_folio); + +/* + * If a folio has any new buffers, zero them out here, and mark them uptodate + * and dirty so they'll be written out (in order to prevent uninitialised + * block data from leaking). And clear the new bit. + */ +void folio_zero_new_buffers(struct folio *folio, size_t from, size_t to) +{ + size_t block_start, block_end; + struct buffer_head *head, *bh; + + BUG_ON(!folio_test_locked(folio)); + head = folio_buffers(folio); + if (!head) + return; + + bh = head; + block_start = 0; + do { + block_end = block_start + bh->b_size; + + if (buffer_new(bh)) { + if (block_end > from && block_start < to) { + if (!folio_test_uptodate(folio)) { + size_t start, xend; + + start = max(from, block_start); + xend = min(to, block_end); + + folio_zero_segment(folio, start, xend); + set_buffer_uptodate(bh); + } + + clear_buffer_new(bh); + mark_buffer_dirty(bh); + } + } + + block_start = block_end; + bh = bh->b_this_page; + } while (bh != head); +} +EXPORT_SYMBOL(folio_zero_new_buffers); + +static int +iomap_to_bh(struct inode *inode, sector_t block, struct buffer_head *bh, + const struct iomap *iomap) +{ + loff_t offset = (loff_t)block << inode->i_blkbits; + + bh->b_bdev = iomap->bdev; + + /* + * Block points to offset in file we need to map, iomap contains + * the offset at which the map starts. If the map ends before the + * current block, then do not map the buffer and let the caller + * handle it. + */ + if (offset >= iomap->offset + iomap->length) + return -EIO; + + switch (iomap->type) { + case IOMAP_HOLE: + /* + * If the buffer is not up to date or beyond the current EOF, + * we need to mark it as new to ensure sub-block zeroing is + * executed if necessary. + */ + if (!buffer_uptodate(bh) || + (offset >= i_size_read(inode))) + set_buffer_new(bh); + return 0; + case IOMAP_DELALLOC: + if (!buffer_uptodate(bh) || + (offset >= i_size_read(inode))) + set_buffer_new(bh); + set_buffer_uptodate(bh); + set_buffer_mapped(bh); + set_buffer_delay(bh); + return 0; + case IOMAP_UNWRITTEN: + /* + * For unwritten regions, we always need to ensure that regions + * in the block we are not writing to are zeroed. Mark the + * buffer as new to ensure this. + */ + set_buffer_new(bh); + set_buffer_unwritten(bh); + fallthrough; + case IOMAP_MAPPED: + if ((iomap->flags & IOMAP_F_NEW) || + offset >= i_size_read(inode)) { + /* + * This can happen if truncating the block device races + * with the check in the caller as i_size updates on + * block devices aren't synchronized by i_rwsem for + * block devices. + */ + if (S_ISBLK(inode->i_mode)) + return -EIO; + set_buffer_new(bh); + } + bh->b_blocknr = (iomap->addr + offset - iomap->offset) >> + inode->i_blkbits; + set_buffer_mapped(bh); + return 0; + default: + WARN_ON_ONCE(1); + return -EIO; + } +} + +int __block_write_begin_int(struct folio *folio, loff_t pos, unsigned len, + get_block_t *get_block, const struct iomap *iomap) +{ + size_t from = offset_in_folio(folio, pos); + size_t to = from + len; + struct inode *inode = folio->mapping->host; + size_t block_start, block_end; + sector_t block; + int err = 0; + size_t blocksize; + struct buffer_head *bh, *head, *wait[2], **wait_bh=wait; + + BUG_ON(!folio_test_locked(folio)); + BUG_ON(to > folio_size(folio)); + BUG_ON(from > to); + + head = folio_create_buffers(folio, inode, 0); + blocksize = head->b_size; + block = div_u64(folio_pos(folio), blocksize); + + for (bh = head, block_start = 0; bh != head || !block_start; + block++, block_start=block_end, bh = bh->b_this_page) { + block_end = block_start + blocksize; + if (block_end <= from || block_start >= to) { + if (folio_test_uptodate(folio)) { + if (!buffer_uptodate(bh)) + set_buffer_uptodate(bh); + } + continue; + } + if (buffer_new(bh)) + clear_buffer_new(bh); + if (!buffer_mapped(bh)) { + WARN_ON(bh->b_size != blocksize); + if (get_block) + err = get_block(inode, block, bh, 1); + else + err = iomap_to_bh(inode, block, bh, iomap); + if (err) + break; + + if (buffer_new(bh)) { + clean_bdev_bh_alias(bh); + if (folio_test_uptodate(folio)) { + clear_buffer_new(bh); + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + continue; + } + if (block_end > to || block_start < from) + folio_zero_segments(folio, + to, block_end, + block_start, from); + continue; + } + } + if (folio_test_uptodate(folio)) { + if (!buffer_uptodate(bh)) + set_buffer_uptodate(bh); + continue; + } + if (!buffer_uptodate(bh) && !buffer_delay(bh) && + !buffer_unwritten(bh) && + (block_start < from || block_end > to)) { + bh_read_nowait(bh, 0); + *wait_bh++=bh; + } + } + /* + * If we issued read requests - let them complete. + */ + while(wait_bh > wait) { + wait_on_buffer(*--wait_bh); + if (!buffer_uptodate(*wait_bh)) + err = -EIO; + } + if (unlikely(err)) + folio_zero_new_buffers(folio, from, to); + return err; +} + +int __block_write_begin(struct folio *folio, loff_t pos, unsigned len, + get_block_t *get_block) +{ + return __block_write_begin_int(folio, pos, len, get_block, NULL); +} +EXPORT_SYMBOL(__block_write_begin); + +void block_commit_write(struct folio *folio, size_t from, size_t to) +{ + size_t block_start, block_end; + bool partial = false; + bool uptodate = folio_test_uptodate(folio); + unsigned blocksize; + struct buffer_head *bh, *head; + + bh = head = folio_buffers(folio); + if (!bh) + return; + blocksize = bh->b_size; + + block_start = 0; + do { + block_end = block_start + blocksize; + if (block_end <= from || block_start >= to) { + if (!buffer_uptodate(bh)) + partial = true; + } else { + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + } + if (buffer_new(bh)) + clear_buffer_new(bh); + + block_start = block_end; + if (uptodate && block_start >= to) + break; + bh = bh->b_this_page; + } while (bh != head); + + /* + * If this is a partial write which happened to make all buffers + * uptodate then we can optimize away a bogus read_folio() for + * the next read(). Here we 'discover' whether the folio went + * uptodate as a result of this (potentially partial) write. + */ + if (!partial) + folio_mark_uptodate(folio); +} +EXPORT_SYMBOL(block_commit_write); + +/* + * block_write_begin takes care of the basic task of block allocation and + * bringing partial write blocks uptodate first. + * + * The filesystem needs to handle block truncation upon failure. + */ +int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len, + struct folio **foliop, get_block_t *get_block) +{ + pgoff_t index = pos >> PAGE_SHIFT; + struct folio *folio; + int status; + + folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN, + mapping_gfp_mask(mapping)); + if (IS_ERR(folio)) + return PTR_ERR(folio); + + status = __block_write_begin_int(folio, pos, len, get_block, NULL); + if (unlikely(status)) { + folio_unlock(folio); + folio_put(folio); + folio = NULL; + } + + *foliop = folio; + return status; +} +EXPORT_SYMBOL(block_write_begin); + +int block_write_end(loff_t pos, unsigned len, unsigned copied, + struct folio *folio) +{ + size_t start = pos - folio_pos(folio); + + if (unlikely(copied < len)) { + /* + * The buffers that were written will now be uptodate, so + * we don't have to worry about a read_folio reading them + * and overwriting a partial write. However if we have + * encountered a short write and only partially written + * into a buffer, it will not be marked uptodate, so a + * read_folio might come in and destroy our partial write. + * + * Do the simplest thing, and just treat any short write to a + * non uptodate folio as a zero-length write, and force the + * caller to redo the whole thing. + */ + if (!folio_test_uptodate(folio)) + copied = 0; + + folio_zero_new_buffers(folio, start+copied, start+len); + } + flush_dcache_folio(folio); + + /* This could be a short (even 0-length) commit */ + block_commit_write(folio, start, start + copied); + + return copied; +} +EXPORT_SYMBOL(block_write_end); + +int generic_write_end(const struct kiocb *iocb, struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct folio *folio, void *fsdata) +{ + struct inode *inode = mapping->host; + loff_t old_size = inode->i_size; + bool i_size_changed = false; + + copied = block_write_end(pos, len, copied, folio); + + /* + * No need to use i_size_read() here, the i_size cannot change under us + * because we hold i_rwsem. + * + * But it's important to update i_size while still holding folio lock: + * page writeout could otherwise come in and zero beyond i_size. + */ + if (pos + copied > inode->i_size) { + i_size_write(inode, pos + copied); + i_size_changed = true; + } + + folio_unlock(folio); + folio_put(folio); + + if (old_size < pos) + pagecache_isize_extended(inode, old_size, pos); + /* + * Don't mark the inode dirty under page lock. First, it unnecessarily + * makes the holding time of page lock longer. Second, it forces lock + * ordering of page lock and transaction start for journaling + * filesystems. + */ + if (i_size_changed) + mark_inode_dirty(inode); + return copied; +} +EXPORT_SYMBOL(generic_write_end); + +/* + * block_is_partially_uptodate checks whether buffers within a folio are + * uptodate or not. + * + * Returns true if all buffers which correspond to the specified part + * of the folio are uptodate. + */ +bool block_is_partially_uptodate(struct folio *folio, size_t from, size_t count) +{ + unsigned block_start, block_end, blocksize; + unsigned to; + struct buffer_head *bh, *head; + bool ret = true; + + head = folio_buffers(folio); + if (!head) + return false; + blocksize = head->b_size; + to = min(folio_size(folio) - from, count); + to = from + to; + if (from < blocksize && to > folio_size(folio) - blocksize) + return false; + + bh = head; + block_start = 0; + do { + block_end = block_start + blocksize; + if (block_end > from && block_start < to) { + if (!buffer_uptodate(bh)) { + ret = false; + break; + } + if (block_end >= to) + break; + } + block_start = block_end; + bh = bh->b_this_page; + } while (bh != head); + + return ret; +} +EXPORT_SYMBOL(block_is_partially_uptodate); + +/* + * Generic "read_folio" function for block devices that have the normal + * get_block functionality. This is most of the block device filesystems. + * Reads the folio asynchronously --- the unlock_buffer() and + * set/clear_buffer_uptodate() functions propagate buffer state into the + * folio once IO has completed. + */ +int block_read_full_folio(struct folio *folio, get_block_t *get_block) +{ + struct inode *inode = folio->mapping->host; + sector_t iblock, lblock; + struct buffer_head *bh, *head, *prev = NULL; + size_t blocksize; + int fully_mapped = 1; + bool page_error = false; + loff_t limit = i_size_read(inode); + + /* This is needed for ext4. */ + if (IS_ENABLED(CONFIG_FS_VERITY) && IS_VERITY(inode)) + limit = inode->i_sb->s_maxbytes; + + head = folio_create_buffers(folio, inode, 0); + blocksize = head->b_size; + + iblock = div_u64(folio_pos(folio), blocksize); + lblock = div_u64(limit + blocksize - 1, blocksize); + bh = head; + + do { + if (buffer_uptodate(bh)) + continue; + + if (!buffer_mapped(bh)) { + int err = 0; + + fully_mapped = 0; + if (iblock < lblock) { + WARN_ON(bh->b_size != blocksize); + err = get_block(inode, iblock, bh, 0); + if (err) + page_error = true; + } + if (!buffer_mapped(bh)) { + folio_zero_range(folio, bh_offset(bh), + blocksize); + if (!err) + set_buffer_uptodate(bh); + continue; + } + /* + * get_block() might have updated the buffer + * synchronously + */ + if (buffer_uptodate(bh)) + continue; + } + + lock_buffer(bh); + if (buffer_uptodate(bh)) { + unlock_buffer(bh); + continue; + } + + /* + * If a folio's buffers are under async readin + * (end_buffer_async_read completion) then there is a + * possibility that another thread of control could lock + * one of the buffers after it has completed but while + * some of the other buffers have not completed. This + * locked buffer would confuse end_buffer_async_read() + * into not unlocking the folio. So the absence of + * BH_Async_Read tells end_buffer_async_read() that this + * buffer is not under async I/O. + * + * The folio comes unlocked when it has no locked + * buffer_async buffers left. + * + * The folio lock prevents anyone starting new async + * I/O reads into any of the buffers. + * + * The writeback flag is used to prevent simultaneous + * writeout of the same folio. + * + * The folio lock prevents anyone from starting writeback + * of a folio which is under read I/O (the writeback + * flag is only ever set on a locked folio). + */ + set_buffer_async_read(bh); + if (prev) + bh_submit(prev, REQ_OP_READ, bh_end_async_read); + prev = bh; + } while (iblock++, (bh = bh->b_this_page) != head); + + if (fully_mapped) + folio_set_mappedtodisk(folio); + + /* + * All buffers are uptodate or get_block() returned an error + * when trying to map them - we must finish the read because + * end_buffer_async_read() will never be called on any buffer + * in this folio. + */ + if (prev) + bh_submit(prev, REQ_OP_READ, bh_end_async_read); + else + folio_end_read(folio, !page_error); + + return 0; +} +EXPORT_SYMBOL(block_read_full_folio); + +/* utility function for filesystems that need to do work on expanding + * truncates. Uses filesystem pagecache writes to allow the filesystem to + * deal with the hole. + */ +int generic_cont_expand_simple(struct inode *inode, loff_t size) +{ + struct address_space *mapping = inode->i_mapping; + const struct address_space_operations *aops = mapping->a_ops; + struct folio *folio; + void *fsdata = NULL; + int err; + + err = inode_newsize_ok(inode, size); + if (err) + goto out; + + err = aops->write_begin(NULL, mapping, size, 0, &folio, &fsdata); + if (err) + goto out; + + err = aops->write_end(NULL, mapping, size, 0, 0, folio, fsdata); + BUG_ON(err > 0); + +out: + return err; +} +EXPORT_SYMBOL(generic_cont_expand_simple); + +static int cont_expand_zero(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, loff_t *bytes) +{ + struct inode *inode = mapping->host; + const struct address_space_operations *aops = mapping->a_ops; + unsigned int blocksize = i_blocksize(inode); + struct folio *folio; + void *fsdata = NULL; + pgoff_t index, curidx; + loff_t curpos; + unsigned zerofrom, offset, len; + int err = 0; + + index = pos >> PAGE_SHIFT; + offset = pos & ~PAGE_MASK; + + while (index > (curidx = (curpos = *bytes)>>PAGE_SHIFT)) { + zerofrom = curpos & ~PAGE_MASK; + if (zerofrom & (blocksize-1)) { + *bytes |= (blocksize-1); + (*bytes)++; + } + len = PAGE_SIZE - zerofrom; + + err = aops->write_begin(iocb, mapping, curpos, len, + &folio, &fsdata); + if (err) + goto out; + folio_zero_range(folio, offset_in_folio(folio, curpos), len); + err = aops->write_end(iocb, mapping, curpos, len, len, + folio, fsdata); + if (err < 0) + goto out; + BUG_ON(err != len); + err = 0; + + balance_dirty_pages_ratelimited(mapping); + + if (fatal_signal_pending(current)) { + err = -EINTR; + goto out; + } + } + + /* page covers the boundary, find the boundary offset */ + if (index == curidx) { + zerofrom = curpos & ~PAGE_MASK; + /* if we will expand the thing last block will be filled */ + if (offset <= zerofrom) { + goto out; + } + if (zerofrom & (blocksize-1)) { + *bytes |= (blocksize-1); + (*bytes)++; + } + len = offset - zerofrom; + + err = aops->write_begin(iocb, mapping, curpos, len, + &folio, &fsdata); + if (err) + goto out; + folio_zero_range(folio, offset_in_folio(folio, curpos), len); + err = aops->write_end(iocb, mapping, curpos, len, len, + folio, fsdata); + if (err < 0) + goto out; + BUG_ON(err != len); + err = 0; + } +out: + return err; +} + +/* + * For moronic filesystems that do not allow holes in file. + * We may have to extend the file. + */ +int cont_write_begin(const struct kiocb *iocb, struct address_space *mapping, + loff_t pos, unsigned len, struct folio **foliop, + void **fsdata, get_block_t *get_block, loff_t *bytes) +{ + struct inode *inode = mapping->host; + unsigned int blocksize = i_blocksize(inode); + unsigned int zerofrom; + int err; + + err = cont_expand_zero(iocb, mapping, pos, bytes); + if (err) + return err; + + zerofrom = *bytes & ~PAGE_MASK; + if (pos+len > *bytes && zerofrom & (blocksize-1)) { + *bytes |= (blocksize-1); + (*bytes)++; + } + + return block_write_begin(mapping, pos, len, foliop, get_block); +} +EXPORT_SYMBOL(cont_write_begin); + +/* + * block_page_mkwrite() is not allowed to change the file size as it gets + * called from a page fault handler when a page is first dirtied. Hence we must + * be careful to check for EOF conditions here. We set the page up correctly + * for a written page which means we get ENOSPC checking when writing into + * holes and correct delalloc and unwritten extent mapping on filesystems that + * support these features. + * + * We are not allowed to take the i_rwsem here so we have to play games to + * protect against truncate races as the page could now be beyond EOF. Because + * truncate writes the inode size before removing pages, once we have the + * page lock we can determine safely if the page is beyond EOF. If it is not + * beyond EOF, then the page is guaranteed safe against truncation until we + * unlock the page. + * + * Direct callers of this function should protect against filesystem freezing + * using sb_start_pagefault() - sb_end_pagefault() functions. + */ +int block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf, + get_block_t get_block) +{ + struct folio *folio = page_folio(vmf->page); + struct inode *inode = file_inode(vma->vm_file); + unsigned long end; + loff_t size; + int ret; + + folio_lock(folio); + size = i_size_read(inode); + if ((folio->mapping != inode->i_mapping) || + (folio_pos(folio) >= size)) { + /* We overload EFAULT to mean page got truncated */ + ret = -EFAULT; + goto out_unlock; + } + + end = folio_size(folio); + /* folio is wholly or partially inside EOF */ + if (folio_pos(folio) + end > size) + end = size - folio_pos(folio); + + ret = __block_write_begin_int(folio, 0, end, get_block, NULL); + if (unlikely(ret)) + goto out_unlock; + + block_commit_write(folio, 0, end); + + folio_mark_dirty(folio); + folio_wait_stable(folio); + return 0; +out_unlock: + folio_unlock(folio); + return ret; +} +EXPORT_SYMBOL(block_page_mkwrite); + +int block_truncate_page(struct address_space *mapping, + loff_t from, get_block_t *get_block) +{ + pgoff_t index = from >> PAGE_SHIFT; + unsigned blocksize; + sector_t iblock; + size_t offset, length, pos; + struct inode *inode = mapping->host; + struct folio *folio; + struct buffer_head *bh; + int err = 0; + + blocksize = i_blocksize(inode); + length = from & (blocksize - 1); + + /* Block boundary? Nothing to do */ + if (!length) + return 0; + + length = blocksize - length; + iblock = ((loff_t)index * PAGE_SIZE) >> inode->i_blkbits; + + folio = filemap_grab_folio(mapping, index); + if (IS_ERR(folio)) + return PTR_ERR(folio); + + bh = folio_buffers(folio); + if (!bh) + bh = create_empty_buffers(folio, blocksize, 0); + + /* Find the buffer that contains "offset" */ + offset = offset_in_folio(folio, from); + pos = blocksize; + while (offset >= pos) { + bh = bh->b_this_page; + iblock++; + pos += blocksize; + } + + if (!buffer_mapped(bh)) { + WARN_ON(bh->b_size != blocksize); + err = get_block(inode, iblock, bh, 0); + if (err) + goto unlock; + /* unmapped? It's a hole - nothing to do */ + if (!buffer_mapped(bh)) + goto unlock; + } + + /* Ok, it's mapped. Make sure it's up-to-date */ + if (folio_test_uptodate(folio)) + set_buffer_uptodate(bh); + + if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) { + err = bh_read(bh, 0); + /* Uhhuh. Read error. Complain and punt. */ + if (err < 0) + goto unlock; + } + + folio_zero_range(folio, offset, length); + mark_buffer_dirty(bh); + +unlock: + folio_unlock(folio); + folio_put(folio); + + return err; +} +EXPORT_SYMBOL(block_truncate_page); + +/* + * The generic write folio function for buffer-backed address_spaces + */ +int block_write_full_folio(struct folio *folio, struct writeback_control *wbc, + void *get_block) +{ + struct inode * const inode = folio->mapping->host; + loff_t i_size = i_size_read(inode); + + /* Is the folio fully inside i_size? */ + if (folio_next_pos(folio) <= i_size) + return __block_write_full_folio(inode, folio, get_block, wbc); + + /* Is the folio fully outside i_size? (truncate in progress) */ + if (folio_pos(folio) >= i_size) { + folio_unlock(folio); + return 0; /* don't care */ + } + + /* + * The folio straddles i_size. It must be zeroed out on each and every + * writeback invocation because it may be mmapped. "A file is mapped + * in multiples of the page size. For a file that is not a multiple of + * the page size, the remaining memory is zeroed when mapped, and + * writes to that region are not written out to the file." + */ + folio_zero_segment(folio, offset_in_folio(folio, i_size), + folio_size(folio)); + return __block_write_full_folio(inode, folio, get_block, wbc); +} + +sector_t generic_block_bmap(struct address_space *mapping, sector_t block, + get_block_t *get_block) +{ + struct inode *inode = mapping->host; + struct buffer_head tmp = { + .b_size = i_blocksize(inode), + }; + + get_block(inode, block, &tmp, 0); + return tmp.b_blocknr; +} +EXPORT_SYMBOL(generic_block_bmap); + +void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags) +{ + lock_buffer(bh); + if (!test_clear_buffer_dirty(bh)) { + unlock_buffer(bh); + return; + } + bh_submit(bh, REQ_OP_WRITE | op_flags, bh_end_write); +} +EXPORT_SYMBOL(write_dirty_buffer); + +/* + * For a data-integrity writeout, we need to wait upon any in-progress I/O + * and then start new I/O and then wait upon it. The caller must have a ref on + * the buffer_head. + */ +int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags) +{ + WARN_ON(atomic_read(&bh->b_count) < 1); + lock_buffer(bh); + if (test_clear_buffer_dirty(bh)) { + /* + * The bh should be mapped, but it might not be if the + * device was hot-removed. Not much we can do but fail the I/O. + */ + if (!buffer_mapped(bh)) { + unlock_buffer(bh); + return -EIO; + } + + bh_submit(bh, REQ_OP_WRITE | op_flags, bh_end_write); + wait_on_buffer(bh); + if (!buffer_uptodate(bh)) + return -EIO; + } else { + unlock_buffer(bh); + } + return 0; +} +EXPORT_SYMBOL(__sync_dirty_buffer); + +int sync_dirty_buffer(struct buffer_head *bh) +{ + return __sync_dirty_buffer(bh, REQ_SYNC); +} +EXPORT_SYMBOL(sync_dirty_buffer); + +static inline int buffer_busy(struct buffer_head *bh) +{ + return atomic_read(&bh->b_count) | + (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock))); +} + +static bool +drop_buffers(struct folio *folio, struct buffer_head **buffers_to_free) +{ + struct buffer_head *head = folio_buffers(folio); + struct buffer_head *bh; + + bh = head; + do { + if (buffer_busy(bh)) + goto failed; + bh = bh->b_this_page; + } while (bh != head); + + do { + struct buffer_head *next = bh->b_this_page; + + remove_assoc_queue(bh); + bh = next; + } while (bh != head); + *buffers_to_free = head; + folio_detach_private(folio); + return true; +failed: + return false; +} + +/** + * try_to_free_buffers - Release buffers attached to this folio. + * @folio: The folio. + * + * If any buffers are in use (dirty, under writeback, elevated refcount), + * no buffers will be freed. + * + * If the folio is dirty but all the buffers are clean then we need to + * be sure to mark the folio clean as well. This is because the folio + * may be against a block device, and a later reattachment of buffers + * to a dirty folio will set *all* buffers dirty. Which would corrupt + * filesystem data on the same device. + * + * The same applies to regular filesystem folios: if all the buffers are + * clean then we set the folio clean and proceed. To do that, we require + * total exclusion from block_dirty_folio(). That is obtained with + * i_private_lock. + * + * Exclusion against try_to_free_buffers may be obtained by either + * locking the folio or by holding its mapping's i_private_lock. + * + * Context: Process context. @folio must be locked. Will not sleep. + * Return: true if all buffers attached to this folio were freed. + */ +bool try_to_free_buffers(struct folio *folio) +{ + struct address_space * const mapping = folio->mapping; + struct buffer_head *buffers_to_free = NULL; + bool ret = 0; + + BUG_ON(!folio_test_locked(folio)); + if (folio_test_writeback(folio)) + return false; + + /* Misconfigured folio check */ + if (WARN_ON_ONCE(!folio_buffers(folio))) + return true; + + if (mapping == NULL) { /* can this still happen? */ + ret = drop_buffers(folio, &buffers_to_free); + goto out; + } + + spin_lock(&mapping->i_private_lock); + ret = drop_buffers(folio, &buffers_to_free); + + /* + * If the filesystem writes its buffers by hand (eg ext3) + * then we can have clean buffers against a dirty folio. We + * clean the folio here; otherwise the VM will never notice + * that the filesystem did any IO at all. + * + * Also, during truncate, discard_buffer will have marked all + * the folio's buffers clean. We discover that here and clean + * the folio also. + * + * i_private_lock must be held over this entire operation in order + * to synchronise against block_dirty_folio and prevent the + * dirty bit from being lost. + */ + if (ret) + folio_cancel_dirty(folio); + spin_unlock(&mapping->i_private_lock); +out: + if (buffers_to_free) { + struct buffer_head *bh = buffers_to_free; + + do { + struct buffer_head *next = bh->b_this_page; + free_buffer_head(bh); + bh = next; + } while (bh != buffers_to_free); + } + return ret; +} +EXPORT_SYMBOL(try_to_free_buffers); + +/* + * Buffer-head allocation + */ +static struct kmem_cache *bh_cachep __ro_after_init; + +/* + * Once the number of bh's in the machine exceeds this level, we start + * stripping them in writeback. + */ +static unsigned long max_buffer_heads __ro_after_init; + +int buffer_heads_over_limit; + +struct bh_accounting { + int nr; /* Number of live bh's */ + int ratelimit; /* Limit cacheline bouncing */ +}; + +static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0}; + +static void recalc_bh_state(void) +{ + int i; + int tot = 0; + + if (__this_cpu_inc_return(bh_accounting.ratelimit) - 1 < 4096) + return; + __this_cpu_write(bh_accounting.ratelimit, 0); + for_each_online_cpu(i) + tot += per_cpu(bh_accounting, i).nr; + buffer_heads_over_limit = (tot > max_buffer_heads); +} + +struct buffer_head *alloc_buffer_head(gfp_t gfp_flags) +{ + struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags); + if (ret) { + INIT_LIST_HEAD(&ret->b_assoc_buffers); + spin_lock_init(&ret->b_uptodate_lock); + preempt_disable(); + __this_cpu_inc(bh_accounting.nr); + recalc_bh_state(); + preempt_enable(); + } + return ret; +} +EXPORT_SYMBOL(alloc_buffer_head); + +void free_buffer_head(struct buffer_head *bh) +{ + BUG_ON(!list_empty(&bh->b_assoc_buffers)); + kmem_cache_free(bh_cachep, bh); + preempt_disable(); + __this_cpu_dec(bh_accounting.nr); + recalc_bh_state(); + preempt_enable(); +} +EXPORT_SYMBOL(free_buffer_head); + +static int buffer_exit_cpu_dead(unsigned int cpu) +{ + int i; + struct bh_lru *b = &per_cpu(bh_lrus, cpu); + + for (i = 0; i < BH_LRU_SIZE; i++) { + brelse(b->bhs[i]); + b->bhs[i] = NULL; + } + this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr); + per_cpu(bh_accounting, cpu).nr = 0; + return 0; +} + +/** + * bh_uptodate_or_lock - Test whether the buffer is uptodate + * @bh: struct buffer_head + * + * Return true if the buffer is up-to-date and false, + * with the buffer locked, if not. + */ +int bh_uptodate_or_lock(struct buffer_head *bh) +{ + if (!buffer_uptodate(bh)) { + lock_buffer(bh); + if (!buffer_uptodate(bh)) + return 0; + unlock_buffer(bh); + } + return 1; +} +EXPORT_SYMBOL(bh_uptodate_or_lock); + +/** + * __bh_read - Submit read for a locked buffer + * @bh: struct buffer_head + * @op_flags: appending REQ_OP_* flags besides REQ_OP_READ + * @wait: wait until reading finish + * + * Returns zero on success or don't wait, and -EIO on error. + */ +int __bh_read(struct buffer_head *bh, blk_opf_t op_flags, bool wait) +{ + int ret = 0; + + BUG_ON(!buffer_locked(bh)); + + bh_submit(bh, REQ_OP_READ | op_flags, bh_end_read); + if (wait) { + wait_on_buffer(bh); + if (!buffer_uptodate(bh)) + ret = -EIO; + } + return ret; +} +EXPORT_SYMBOL(__bh_read); + +/** + * __bh_read_batch - Submit read for a batch of unlocked buffers + * @nr: entry number of the buffer batch + * @bhs: a batch of struct buffer_head + * @op_flags: appending REQ_OP_* flags besides REQ_OP_READ + * @force_lock: force to get a lock on the buffer if set, otherwise drops any + * buffer that cannot lock. + * + * Returns zero on success or don't wait, and -EIO on error. + */ +void __bh_read_batch(int nr, struct buffer_head *bhs[], + blk_opf_t op_flags, bool force_lock) +{ + int i; + + for (i = 0; i < nr; i++) { + struct buffer_head *bh = bhs[i]; + + if (buffer_uptodate(bh)) + continue; + + if (force_lock) + lock_buffer(bh); + else + if (!trylock_buffer(bh)) + continue; + + if (buffer_uptodate(bh)) { + unlock_buffer(bh); + continue; + } + + bh_submit(bh, REQ_OP_READ | op_flags, bh_end_read); + } +} +EXPORT_SYMBOL(__bh_read_batch); + +void __init buffer_init(void) +{ + unsigned long nrpages; + int ret; + + bh_cachep = KMEM_CACHE(buffer_head, + SLAB_RECLAIM_ACCOUNT|SLAB_PANIC); + /* + * Limit the bh occupancy to 10% of ZONE_NORMAL + */ + nrpages = (nr_free_buffer_pages() * 10) / 100; + max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head)); + ret = cpuhp_setup_state_nocalls(CPUHP_FS_BUFF_DEAD, "fs/buffer:dead", + NULL, buffer_exit_cpu_dead); + WARN_ON(ret < 0); +} diff --git a/fs/cachefiles/Kconfig b/fs/cachefiles/Kconfig new file mode 100644 index 000000000..afb25b6af --- /dev/null +++ b/fs/cachefiles/Kconfig @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config CACHEFILES + tristate "Filesystem caching on files" + depends on NETFS_SUPPORT && FSCACHE && BLOCK + help + This permits use of a mounted filesystem as a cache for other + filesystems - primarily networking filesystems - thus allowing fast + local disk to enhance the speed of slower devices. + + See Documentation/filesystems/caching/cachefiles.rst for more + information. + +config CACHEFILES_DEBUG + bool "Debug CacheFiles" + depends on CACHEFILES + help + This permits debugging to be dynamically enabled in the filesystem + caching on files module. If this is set, the debugging output may be + enabled by setting bits in /sys/modules/cachefiles/parameter/debug or + by including a debugging specifier in /etc/cachefilesd.conf. + +config CACHEFILES_ERROR_INJECTION + bool "Provide error injection for cachefiles" + depends on CACHEFILES && SYSCTL + help + This permits error injection to be enabled in cachefiles whilst a + cache is in service. diff --git a/fs/cachefiles/Makefile b/fs/cachefiles/Makefile new file mode 100644 index 000000000..16d811f1a --- /dev/null +++ b/fs/cachefiles/Makefile @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for caching in a mounted filesystem +# + +cachefiles-y := \ + cache.o \ + daemon.o \ + interface.o \ + io.o \ + key.o \ + main.o \ + namei.o \ + security.o \ + volume.o \ + xattr.o + +cachefiles-$(CONFIG_CACHEFILES_ERROR_INJECTION) += error_inject.o + +obj-$(CONFIG_CACHEFILES) := cachefiles.o diff --git a/fs/cachefiles/cache.c b/fs/cachefiles/cache.c new file mode 100644 index 000000000..9fb06dc16 --- /dev/null +++ b/fs/cachefiles/cache.c @@ -0,0 +1,428 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Manage high-level VFS aspects of a cache. + * + * Copyright (C) 2007, 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include "internal.h" + +/* + * Bring a cache online. + */ +int cachefiles_add_cache(struct cachefiles_cache *cache) +{ + struct fscache_cache *cache_cookie; + struct path path; + struct kstatfs stats; + struct dentry *graveyard, *cachedir, *root; + const struct cred *saved_cred; + int ret; + + _enter(""); + + cache_cookie = fscache_acquire_cache(cache->tag); + if (IS_ERR(cache_cookie)) + return PTR_ERR(cache_cookie); + + /* we want to work under the module's security ID */ + ret = cachefiles_get_security_ID(cache); + if (ret < 0) + goto error_getsec; + + cachefiles_begin_secure(cache, &saved_cred); + + /* look up the directory at the root of the cache */ + ret = kern_path(cache->rootdirname, LOOKUP_DIRECTORY, &path); + if (ret < 0) + goto error_open_root; + + cache->mnt = path.mnt; + root = path.dentry; + + ret = -EINVAL; + if (is_idmapped_mnt(path.mnt)) { + pr_warn("File cache on idmapped mounts not supported"); + goto error_unsupported; + } + + /* Check features of the backing filesystem: + * - Directories must support looking up and directory creation + * - We create tmpfiles to handle invalidation + * - We use xattrs to store metadata + * - We need to be able to query the amount of space available + * - We want to be able to sync the filesystem when stopping the cache + * - We use DIO to/from pages, so the blocksize mustn't be too big. + */ + ret = -EOPNOTSUPP; + if (d_is_negative(root) || + !d_backing_inode(root)->i_op->lookup || + !d_backing_inode(root)->i_op->mkdir || + !d_backing_inode(root)->i_op->tmpfile || + !(d_backing_inode(root)->i_opflags & IOP_XATTR) || + !root->d_sb->s_op->statfs || + !root->d_sb->s_op->sync_fs || + root->d_sb->s_blocksize > PAGE_SIZE) + goto error_unsupported; + + ret = -EROFS; + if (sb_rdonly(root->d_sb)) + goto error_unsupported; + + /* determine the security of the on-disk cache as this governs + * security ID of files we create */ + ret = cachefiles_determine_cache_security(cache, root, &saved_cred); + if (ret < 0) + goto error_unsupported; + + /* get the cache size and blocksize */ + ret = vfs_statfs(&path, &stats); + if (ret < 0) + goto error_unsupported; + + ret = -ERANGE; + if (stats.f_bsize <= 0) + goto error_unsupported; + + ret = -EOPNOTSUPP; + if (stats.f_bsize > PAGE_SIZE) + goto error_unsupported; + + cache->bsize = stats.f_bsize; + cache->bshift = ilog2(stats.f_bsize); + + _debug("blksize %u (shift %u)", + cache->bsize, cache->bshift); + + _debug("size %llu, avail %llu", + (unsigned long long) stats.f_blocks, + (unsigned long long) stats.f_bavail); + + /* set up caching limits */ + do_div(stats.f_files, 100); + cache->fstop = stats.f_files * cache->fstop_percent; + cache->fcull = stats.f_files * cache->fcull_percent; + cache->frun = stats.f_files * cache->frun_percent; + + _debug("limits {%llu,%llu,%llu} files", + (unsigned long long) cache->frun, + (unsigned long long) cache->fcull, + (unsigned long long) cache->fstop); + + do_div(stats.f_blocks, 100); + cache->bstop = stats.f_blocks * cache->bstop_percent; + cache->bcull = stats.f_blocks * cache->bcull_percent; + cache->brun = stats.f_blocks * cache->brun_percent; + + _debug("limits {%llu,%llu,%llu} blocks", + (unsigned long long) cache->brun, + (unsigned long long) cache->bcull, + (unsigned long long) cache->bstop); + + /* get the cache directory and check its type */ + cachedir = cachefiles_get_directory(cache, root, "cache", NULL); + if (IS_ERR(cachedir)) { + ret = PTR_ERR(cachedir); + goto error_unsupported; + } + + cache->store = cachedir; + + /* get the graveyard directory */ + graveyard = cachefiles_get_directory(cache, root, "graveyard", NULL); + if (IS_ERR(graveyard)) { + ret = PTR_ERR(graveyard); + goto error_unsupported; + } + + cache->graveyard = graveyard; + cache->cache = cache_cookie; + + ret = fscache_add_cache(cache_cookie, &cachefiles_cache_ops, cache); + if (ret < 0) + goto error_add_cache; + + /* done */ + set_bit(CACHEFILES_READY, &cache->flags); + dput(root); + + pr_info("File cache on %s registered\n", cache_cookie->name); + + /* check how much space the cache has */ + cachefiles_has_space(cache, 0, 0, cachefiles_has_space_check); + cachefiles_end_secure(cache, saved_cred); + _leave(" = 0 [%px]", cache->cache); + return 0; + +error_add_cache: + cachefiles_put_directory(cache->graveyard); + cache->graveyard = NULL; +error_unsupported: + cachefiles_put_directory(cache->store); + cache->store = NULL; + mntput(cache->mnt); + cache->mnt = NULL; + dput(root); +error_open_root: + cachefiles_end_secure(cache, saved_cred); + put_cred(cache->cache_cred); + cache->cache_cred = NULL; +error_getsec: + fscache_relinquish_cache(cache_cookie); + cache->cache = NULL; + pr_err("Failed to register: %d\n", ret); + return ret; +} + +/* + * See if we have space for a number of pages and/or a number of files in the + * cache + */ +int cachefiles_has_space(struct cachefiles_cache *cache, + unsigned fnr, unsigned bnr, + enum cachefiles_has_space_for reason) +{ + struct kstatfs stats; + u64 b_avail, b_writing; + int ret; + + struct path path = { + .mnt = cache->mnt, + .dentry = cache->mnt->mnt_root, + }; + + //_enter("{%llu,%llu,%llu,%llu,%llu,%llu},%u,%u", + // (unsigned long long) cache->frun, + // (unsigned long long) cache->fcull, + // (unsigned long long) cache->fstop, + // (unsigned long long) cache->brun, + // (unsigned long long) cache->bcull, + // (unsigned long long) cache->bstop, + // fnr, bnr); + + /* find out how many pages of blockdev are available */ + memset(&stats, 0, sizeof(stats)); + + ret = vfs_statfs(&path, &stats); + if (ret < 0) { + trace_cachefiles_vfs_error(NULL, d_inode(path.dentry), ret, + cachefiles_trace_statfs_error); + if (ret == -EIO) + cachefiles_io_error(cache, "statfs failed"); + _leave(" = %d", ret); + return ret; + } + + b_avail = stats.f_bavail; + b_writing = atomic_long_read(&cache->b_writing); + if (b_avail > b_writing) + b_avail -= b_writing; + else + b_avail = 0; + + //_debug("avail %llu,%llu", + // (unsigned long long)stats.f_ffree, + // (unsigned long long)b_avail); + + /* see if there is sufficient space */ + if (stats.f_ffree > fnr) + stats.f_ffree -= fnr; + else + stats.f_ffree = 0; + + if (b_avail > bnr) + b_avail -= bnr; + else + b_avail = 0; + + ret = -ENOBUFS; + if (stats.f_ffree < cache->fstop || + b_avail < cache->bstop) + goto stop_and_begin_cull; + + ret = 0; + if (stats.f_ffree < cache->fcull || + b_avail < cache->bcull) + goto begin_cull; + + if (test_bit(CACHEFILES_CULLING, &cache->flags) && + stats.f_ffree >= cache->frun && + b_avail >= cache->brun && + test_and_clear_bit(CACHEFILES_CULLING, &cache->flags) + ) { + _debug("cease culling"); + cachefiles_state_changed(cache); + } + + //_leave(" = 0"); + return 0; + +stop_and_begin_cull: + switch (reason) { + case cachefiles_has_space_for_write: + fscache_count_no_write_space(); + break; + case cachefiles_has_space_for_create: + fscache_count_no_create_space(); + break; + default: + break; + } +begin_cull: + if (!test_and_set_bit(CACHEFILES_CULLING, &cache->flags)) { + _debug("### CULL CACHE ###"); + cachefiles_state_changed(cache); + } + + _leave(" = %d", ret); + return ret; +} + +/* + * Mark all the objects as being out of service and queue them all for cleanup. + */ +static void cachefiles_withdraw_objects(struct cachefiles_cache *cache) +{ + struct cachefiles_object *object; + unsigned int count = 0; + + _enter(""); + + spin_lock(&cache->object_list_lock); + + while (!list_empty(&cache->object_list)) { + object = list_first_entry(&cache->object_list, + struct cachefiles_object, cache_link); + cachefiles_see_object(object, cachefiles_obj_see_withdrawal); + list_del_init(&object->cache_link); + fscache_withdraw_cookie(object->cookie); + count++; + if ((count & 63) == 0) { + spin_unlock(&cache->object_list_lock); + cond_resched(); + spin_lock(&cache->object_list_lock); + } + } + + spin_unlock(&cache->object_list_lock); + _leave(" [%u objs]", count); +} + +/* + * Withdraw fscache volumes. + */ +static void cachefiles_withdraw_fscache_volumes(struct cachefiles_cache *cache) +{ + struct list_head *cur; + struct cachefiles_volume *volume; + struct fscache_volume *vcookie; + + _enter(""); +retry: + spin_lock(&cache->object_list_lock); + list_for_each(cur, &cache->volumes) { + volume = list_entry(cur, struct cachefiles_volume, cache_link); + + if (atomic_read(&volume->vcookie->n_accesses) == 0) + continue; + + vcookie = fscache_try_get_volume(volume->vcookie, + fscache_volume_get_withdraw); + if (vcookie) { + spin_unlock(&cache->object_list_lock); + fscache_withdraw_volume(vcookie); + fscache_put_volume(vcookie, fscache_volume_put_withdraw); + goto retry; + } + } + spin_unlock(&cache->object_list_lock); + + _leave(""); +} + +/* + * Withdraw cachefiles volumes. + */ +static void cachefiles_withdraw_volumes(struct cachefiles_cache *cache) +{ + _enter(""); + + for (;;) { + struct fscache_volume *vcookie = NULL; + struct cachefiles_volume *volume = NULL; + + spin_lock(&cache->object_list_lock); + if (!list_empty(&cache->volumes)) { + volume = list_first_entry(&cache->volumes, + struct cachefiles_volume, cache_link); + vcookie = fscache_try_get_volume(volume->vcookie, + fscache_volume_get_withdraw); + if (!vcookie) { + spin_unlock(&cache->object_list_lock); + cpu_relax(); + continue; + } + list_del_init(&volume->cache_link); + } + spin_unlock(&cache->object_list_lock); + if (!volume) + break; + + cachefiles_withdraw_volume(volume); + fscache_put_volume(vcookie, fscache_volume_put_withdraw); + } + + _leave(""); +} + +/* + * Sync a cache to backing disk. + */ +static void cachefiles_sync_cache(struct cachefiles_cache *cache) +{ + const struct cred *saved_cred; + int ret; + + _enter("%s", cache->cache->name); + + /* make sure all pages pinned by operations on behalf of the netfs are + * written to disc */ + cachefiles_begin_secure(cache, &saved_cred); + down_read(&cache->mnt->mnt_sb->s_umount); + ret = sync_filesystem(cache->mnt->mnt_sb); + up_read(&cache->mnt->mnt_sb->s_umount); + cachefiles_end_secure(cache, saved_cred); + + if (ret == -EIO) + cachefiles_io_error(cache, + "Attempt to sync backing fs superblock returned error %d", + ret); +} + +/* + * Withdraw cache objects. + */ +void cachefiles_withdraw_cache(struct cachefiles_cache *cache) +{ + struct fscache_cache *fscache = cache->cache; + + pr_info("File cache on %s unregistering\n", fscache->name); + + fscache_withdraw_cache(fscache); + cachefiles_withdraw_fscache_volumes(cache); + + /* we now have to destroy all the active objects pertaining to this + * cache - which we do by passing them off to thread pool to be + * disposed of */ + cachefiles_withdraw_objects(cache); + fscache_wait_for_objects(fscache); + + cachefiles_withdraw_volumes(cache); + cachefiles_sync_cache(cache); + cache->cache = NULL; + fscache_relinquish_cache(fscache); +} diff --git a/fs/cachefiles/daemon.c b/fs/cachefiles/daemon.c new file mode 100644 index 000000000..1a66e0af2 --- /dev/null +++ b/fs/cachefiles/daemon.c @@ -0,0 +1,745 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Daemon interface + * + * Copyright (C) 2007, 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +static int cachefiles_daemon_open(struct inode *, struct file *); +static int cachefiles_daemon_release(struct inode *, struct file *); +static ssize_t cachefiles_daemon_read(struct file *, char __user *, size_t, + loff_t *); +static ssize_t cachefiles_daemon_write(struct file *, const char __user *, + size_t, loff_t *); +static __poll_t cachefiles_daemon_poll(struct file *, + struct poll_table_struct *); +static int cachefiles_daemon_frun(struct cachefiles_cache *, char *); +static int cachefiles_daemon_fcull(struct cachefiles_cache *, char *); +static int cachefiles_daemon_fstop(struct cachefiles_cache *, char *); +static int cachefiles_daemon_brun(struct cachefiles_cache *, char *); +static int cachefiles_daemon_bcull(struct cachefiles_cache *, char *); +static int cachefiles_daemon_bstop(struct cachefiles_cache *, char *); +static int cachefiles_daemon_cull(struct cachefiles_cache *, char *); +static int cachefiles_daemon_debug(struct cachefiles_cache *, char *); +static int cachefiles_daemon_dir(struct cachefiles_cache *, char *); +static int cachefiles_daemon_inuse(struct cachefiles_cache *, char *); +static int cachefiles_daemon_secctx(struct cachefiles_cache *, char *); +static int cachefiles_daemon_tag(struct cachefiles_cache *, char *); +static int cachefiles_daemon_bind(struct cachefiles_cache *, char *); +static void cachefiles_daemon_unbind(struct cachefiles_cache *); + +static unsigned long cachefiles_open; + +const struct file_operations cachefiles_daemon_fops = { + .owner = THIS_MODULE, + .open = cachefiles_daemon_open, + .release = cachefiles_daemon_release, + .read = cachefiles_daemon_read, + .write = cachefiles_daemon_write, + .poll = cachefiles_daemon_poll, + .llseek = noop_llseek, +}; + +struct cachefiles_daemon_cmd { + char name[8]; + int (*handler)(struct cachefiles_cache *cache, char *args); +}; + +static const struct cachefiles_daemon_cmd cachefiles_daemon_cmds[] = { + { "bind", cachefiles_daemon_bind }, + { "brun", cachefiles_daemon_brun }, + { "bcull", cachefiles_daemon_bcull }, + { "bstop", cachefiles_daemon_bstop }, + { "cull", cachefiles_daemon_cull }, + { "debug", cachefiles_daemon_debug }, + { "dir", cachefiles_daemon_dir }, + { "frun", cachefiles_daemon_frun }, + { "fcull", cachefiles_daemon_fcull }, + { "fstop", cachefiles_daemon_fstop }, + { "inuse", cachefiles_daemon_inuse }, + { "secctx", cachefiles_daemon_secctx }, + { "tag", cachefiles_daemon_tag }, + { "", NULL } +}; + + +/* + * Prepare a cache for caching. + */ +static int cachefiles_daemon_open(struct inode *inode, struct file *file) +{ + struct cachefiles_cache *cache; + + _enter(""); + + /* only the superuser may do this */ + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + /* the cachefiles device may only be open once at a time */ + if (xchg(&cachefiles_open, 1) == 1) + return -EBUSY; + + /* allocate a cache record */ + cache = kzalloc_obj(struct cachefiles_cache); + if (!cache) { + cachefiles_open = 0; + return -ENOMEM; + } + + mutex_init(&cache->daemon_mutex); + init_waitqueue_head(&cache->daemon_pollwq); + INIT_LIST_HEAD(&cache->volumes); + INIT_LIST_HEAD(&cache->object_list); + spin_lock_init(&cache->object_list_lock); + + /* set default caching limits + * - limit at 1% free space and/or free files + * - cull below 5% free space and/or free files + * - cease culling above 7% free space and/or free files + */ + cache->frun_percent = 7; + cache->fcull_percent = 5; + cache->fstop_percent = 1; + cache->brun_percent = 7; + cache->bcull_percent = 5; + cache->bstop_percent = 1; + + file->private_data = cache; + cache->cachefilesd = file; + return 0; +} + +/* + * Release a cache. + */ +static int cachefiles_daemon_release(struct inode *inode, struct file *file) +{ + struct cachefiles_cache *cache = file->private_data; + + _enter(""); + + ASSERT(cache); + + set_bit(CACHEFILES_DEAD, &cache->flags); + cachefiles_daemon_unbind(cache); + + /* clean up the control file interface */ + cache->cachefilesd = NULL; + file->private_data = NULL; + cachefiles_open = 0; + + kfree(cache); + _leave(""); + return 0; +} + +static ssize_t cachefiles_do_daemon_read(struct cachefiles_cache *cache, + char __user *_buffer, size_t buflen) +{ + unsigned long long b_released; + unsigned f_released; + char buffer[256]; + int n; + + /* check how much space the cache has */ + cachefiles_has_space(cache, 0, 0, cachefiles_has_space_check); + + /* summarise */ + f_released = atomic_xchg(&cache->f_released, 0); + b_released = atomic_long_xchg(&cache->b_released, 0); + clear_bit(CACHEFILES_STATE_CHANGED, &cache->flags); + + n = snprintf(buffer, sizeof(buffer), + "cull=%c" + " frun=%llx" + " fcull=%llx" + " fstop=%llx" + " brun=%llx" + " bcull=%llx" + " bstop=%llx" + " freleased=%x" + " breleased=%llx", + test_bit(CACHEFILES_CULLING, &cache->flags) ? '1' : '0', + (unsigned long long) cache->frun, + (unsigned long long) cache->fcull, + (unsigned long long) cache->fstop, + (unsigned long long) cache->brun, + (unsigned long long) cache->bcull, + (unsigned long long) cache->bstop, + f_released, + b_released); + + if (n > buflen) + return -EMSGSIZE; + + if (copy_to_user(_buffer, buffer, n) != 0) + return -EFAULT; + + return n; +} + +/* + * Read the cache state. + */ +static ssize_t cachefiles_daemon_read(struct file *file, char __user *_buffer, + size_t buflen, loff_t *pos) +{ + struct cachefiles_cache *cache = file->private_data; + + //_enter(",,%zu,", buflen); + + if (!test_bit(CACHEFILES_READY, &cache->flags)) + return 0; + + return cachefiles_do_daemon_read(cache, _buffer, buflen); +} + +/* + * Take a command from cachefilesd, parse it and act on it. + */ +static ssize_t cachefiles_daemon_write(struct file *file, + const char __user *_data, + size_t datalen, + loff_t *pos) +{ + const struct cachefiles_daemon_cmd *cmd; + struct cachefiles_cache *cache = file->private_data; + ssize_t ret; + char *data, *args, *cp; + + //_enter(",,%zu,", datalen); + + ASSERT(cache); + + if (test_bit(CACHEFILES_DEAD, &cache->flags)) + return -EIO; + + if (datalen > PAGE_SIZE - 1) + return -EOPNOTSUPP; + + /* drag the command string into the kernel so we can parse it */ + data = memdup_user_nul(_data, datalen); + if (IS_ERR(data)) + return PTR_ERR(data); + + ret = -EINVAL; + if (memchr(data, '\0', datalen)) + goto error; + + /* strip any newline */ + cp = memchr(data, '\n', datalen); + if (cp) { + if (cp == data) + goto error; + + *cp = '\0'; + } + + /* parse the command */ + ret = -EOPNOTSUPP; + + for (args = data; *args; args++) + if (isspace(*args)) + break; + if (*args) { + if (args == data) + goto error; + *args = '\0'; + args = skip_spaces(++args); + } + + /* run the appropriate command handler */ + for (cmd = cachefiles_daemon_cmds; cmd->name[0]; cmd++) + if (strcmp(cmd->name, data) == 0) + goto found_command; + +error: + kfree(data); + //_leave(" = %zd", ret); + return ret; + +found_command: + mutex_lock(&cache->daemon_mutex); + + ret = -EIO; + if (!test_bit(CACHEFILES_DEAD, &cache->flags)) + ret = cmd->handler(cache, args); + + mutex_unlock(&cache->daemon_mutex); + + if (ret == 0) + ret = datalen; + goto error; +} + +/* + * Poll for culling state + * - use EPOLLOUT to indicate culling state + */ +static __poll_t cachefiles_daemon_poll(struct file *file, + struct poll_table_struct *poll) +{ + struct cachefiles_cache *cache = file->private_data; + __poll_t mask; + + poll_wait(file, &cache->daemon_pollwq, poll); + mask = 0; + + if (test_bit(CACHEFILES_STATE_CHANGED, &cache->flags)) + mask |= EPOLLIN; + + if (test_bit(CACHEFILES_CULLING, &cache->flags)) + mask |= EPOLLOUT; + + return mask; +} + +/* + * Give a range error for cache space constraints + * - can be tail-called + */ +static int cachefiles_daemon_range_error(struct cachefiles_cache *cache, + char *args) +{ + pr_err("Free space limits must be in range 0%%<=stop%" + */ +static int cachefiles_daemon_frun(struct cachefiles_cache *cache, char *args) +{ + unsigned long frun; + + _enter(",%s", args); + + if (!*args) + return -EINVAL; + + frun = simple_strtoul(args, &args, 10); + if (args[0] != '%' || args[1] != '\0') + return -EINVAL; + + if (frun <= cache->fcull_percent || frun >= 100) + return cachefiles_daemon_range_error(cache, args); + + cache->frun_percent = frun; + return 0; +} + +/* + * Set the percentage of files at which to start culling + * - command: "fcull %" + */ +static int cachefiles_daemon_fcull(struct cachefiles_cache *cache, char *args) +{ + unsigned long fcull; + + _enter(",%s", args); + + if (!*args) + return -EINVAL; + + fcull = simple_strtoul(args, &args, 10); + if (args[0] != '%' || args[1] != '\0') + return -EINVAL; + + if (fcull <= cache->fstop_percent || fcull >= cache->frun_percent) + return cachefiles_daemon_range_error(cache, args); + + cache->fcull_percent = fcull; + return 0; +} + +/* + * Set the percentage of files at which to stop allocating + * - command: "fstop %" + */ +static int cachefiles_daemon_fstop(struct cachefiles_cache *cache, char *args) +{ + unsigned long fstop; + + _enter(",%s", args); + + if (!*args) + return -EINVAL; + + fstop = simple_strtoul(args, &args, 10); + if (args[0] != '%' || args[1] != '\0') + return -EINVAL; + + if (fstop >= cache->fcull_percent) + return cachefiles_daemon_range_error(cache, args); + + cache->fstop_percent = fstop; + return 0; +} + +/* + * Set the percentage of blocks at which to stop culling + * - command: "brun %" + */ +static int cachefiles_daemon_brun(struct cachefiles_cache *cache, char *args) +{ + unsigned long brun; + + _enter(",%s", args); + + if (!*args) + return -EINVAL; + + brun = simple_strtoul(args, &args, 10); + if (args[0] != '%' || args[1] != '\0') + return -EINVAL; + + if (brun <= cache->bcull_percent || brun >= 100) + return cachefiles_daemon_range_error(cache, args); + + cache->brun_percent = brun; + return 0; +} + +/* + * Set the percentage of blocks at which to start culling + * - command: "bcull %" + */ +static int cachefiles_daemon_bcull(struct cachefiles_cache *cache, char *args) +{ + unsigned long bcull; + + _enter(",%s", args); + + if (!*args) + return -EINVAL; + + bcull = simple_strtoul(args, &args, 10); + if (args[0] != '%' || args[1] != '\0') + return -EINVAL; + + if (bcull <= cache->bstop_percent || bcull >= cache->brun_percent) + return cachefiles_daemon_range_error(cache, args); + + cache->bcull_percent = bcull; + return 0; +} + +/* + * Set the percentage of blocks at which to stop allocating + * - command: "bstop %" + */ +static int cachefiles_daemon_bstop(struct cachefiles_cache *cache, char *args) +{ + unsigned long bstop; + + _enter(",%s", args); + + if (!*args) + return -EINVAL; + + bstop = simple_strtoul(args, &args, 10); + if (args[0] != '%' || args[1] != '\0') + return -EINVAL; + + if (bstop >= cache->bcull_percent) + return cachefiles_daemon_range_error(cache, args); + + cache->bstop_percent = bstop; + return 0; +} + +/* + * Set the cache directory + * - command: "dir " + */ +static int cachefiles_daemon_dir(struct cachefiles_cache *cache, char *args) +{ + char *dir; + + _enter(",%s", args); + + if (!*args) { + pr_err("Empty directory specified\n"); + return -EINVAL; + } + + if (cache->rootdirname) { + pr_err("Second cache directory specified\n"); + return -EEXIST; + } + + dir = kstrdup(args, GFP_KERNEL); + if (!dir) + return -ENOMEM; + + cache->rootdirname = dir; + return 0; +} + +/* + * Set the cache security context + * - command: "secctx " + */ +static int cachefiles_daemon_secctx(struct cachefiles_cache *cache, char *args) +{ + int err; + + _enter(",%s", args); + + if (!*args) { + pr_err("Empty security context specified\n"); + return -EINVAL; + } + + if (cache->have_secid) { + pr_err("Second security context specified\n"); + return -EINVAL; + } + + err = security_secctx_to_secid(args, strlen(args), &cache->secid); + if (err) + return err; + + cache->have_secid = true; + return 0; +} + +/* + * Set the cache tag + * - command: "tag " + */ +static int cachefiles_daemon_tag(struct cachefiles_cache *cache, char *args) +{ + char *tag; + + _enter(",%s", args); + + if (!*args) { + pr_err("Empty tag specified\n"); + return -EINVAL; + } + + if (cache->tag) + return -EEXIST; + + tag = kstrdup(args, GFP_KERNEL); + if (!tag) + return -ENOMEM; + + cache->tag = tag; + return 0; +} + +/* + * Request a node in the cache be culled from the current working directory + * - command: "cull " + */ +static int cachefiles_daemon_cull(struct cachefiles_cache *cache, char *args) +{ + struct path path; + const struct cred *saved_cred; + int ret; + + _enter(",%s", args); + + if (strchr(args, '/')) + goto inval; + + if (!test_bit(CACHEFILES_READY, &cache->flags)) { + pr_err("cull applied to unready cache\n"); + return -EIO; + } + + if (test_bit(CACHEFILES_DEAD, &cache->flags)) { + pr_err("cull applied to dead cache\n"); + return -EIO; + } + + get_fs_pwd(current->fs, &path); + + if (!d_can_lookup(path.dentry)) + goto notdir; + + cachefiles_begin_secure(cache, &saved_cred); + ret = cachefiles_cull(cache, path.dentry, args); + cachefiles_end_secure(cache, saved_cred); + + path_put(&path); + _leave(" = %d", ret); + return ret; + +notdir: + path_put(&path); + pr_err("cull command requires dirfd to be a directory\n"); + return -ENOTDIR; + +inval: + pr_err("cull command requires dirfd and filename\n"); + return -EINVAL; +} + +/* + * Set debugging mode + * - command: "debug " + */ +static int cachefiles_daemon_debug(struct cachefiles_cache *cache, char *args) +{ + unsigned long mask; + + _enter(",%s", args); + + mask = simple_strtoul(args, &args, 0); + if (args[0] != '\0') + goto inval; + + cachefiles_debug = mask; + _leave(" = 0"); + return 0; + +inval: + pr_err("debug command requires mask\n"); + return -EINVAL; +} + +/* + * Find out whether an object in the current working directory is in use or not + * - command: "inuse " + */ +static int cachefiles_daemon_inuse(struct cachefiles_cache *cache, char *args) +{ + struct path path; + const struct cred *saved_cred; + int ret; + + //_enter(",%s", args); + + if (strchr(args, '/')) + goto inval; + + if (!test_bit(CACHEFILES_READY, &cache->flags)) { + pr_err("inuse applied to unready cache\n"); + return -EIO; + } + + if (test_bit(CACHEFILES_DEAD, &cache->flags)) { + pr_err("inuse applied to dead cache\n"); + return -EIO; + } + + get_fs_pwd(current->fs, &path); + + if (!d_can_lookup(path.dentry)) + goto notdir; + + cachefiles_begin_secure(cache, &saved_cred); + ret = cachefiles_check_in_use(cache, path.dentry, args); + cachefiles_end_secure(cache, saved_cred); + + path_put(&path); + //_leave(" = %d", ret); + return ret; + +notdir: + path_put(&path); + pr_err("inuse command requires dirfd to be a directory\n"); + return -ENOTDIR; + +inval: + pr_err("inuse command requires dirfd and filename\n"); + return -EINVAL; +} + +/* + * Bind a directory as a cache + */ +static int cachefiles_daemon_bind(struct cachefiles_cache *cache, char *args) +{ + _enter("{%u,%u,%u,%u,%u,%u},%s", + cache->frun_percent, + cache->fcull_percent, + cache->fstop_percent, + cache->brun_percent, + cache->bcull_percent, + cache->bstop_percent, + args); + + if (cache->fstop_percent >= cache->fcull_percent || + cache->fcull_percent >= cache->frun_percent || + cache->frun_percent >= 100) + return -ERANGE; + + if (cache->bstop_percent >= cache->bcull_percent || + cache->bcull_percent >= cache->brun_percent || + cache->brun_percent >= 100) + return -ERANGE; + + if (!cache->rootdirname) { + pr_err("No cache directory specified\n"); + return -EINVAL; + } + + /* Don't permit already bound caches to be re-bound */ + if (test_bit(CACHEFILES_READY, &cache->flags)) { + pr_err("Cache already bound\n"); + return -EBUSY; + } + + if (*args) { + pr_err("'bind' command doesn't take an argument\n"); + return -EINVAL; + } + + /* Make sure we have copies of the tag string */ + if (!cache->tag) { + /* + * The tag string is released by the fops->release() + * function, so we don't release it on error here + */ + cache->tag = kstrdup("CacheFiles", GFP_KERNEL); + if (!cache->tag) + return -ENOMEM; + } + + return cachefiles_add_cache(cache); +} + +/* + * Unbind a cache. + */ +static void cachefiles_daemon_unbind(struct cachefiles_cache *cache) +{ + _enter(""); + + if (test_bit(CACHEFILES_READY, &cache->flags)) + cachefiles_withdraw_cache(cache); + + cachefiles_put_directory(cache->graveyard); + cachefiles_put_directory(cache->store); + mntput(cache->mnt); + put_cred(cache->cache_cred); + + kfree(cache->rootdirname); + kfree(cache->tag); + + _leave(""); +} diff --git a/fs/cachefiles/error_inject.c b/fs/cachefiles/error_inject.c new file mode 100644 index 000000000..e341ade47 --- /dev/null +++ b/fs/cachefiles/error_inject.c @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Error injection handling. + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include "internal.h" + +unsigned int cachefiles_error_injection_state; + +static struct ctl_table_header *cachefiles_sysctl; +static const struct ctl_table cachefiles_sysctls[] = { + { + .procname = "error_injection", + .data = &cachefiles_error_injection_state, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec, + }, +}; + +int __init cachefiles_register_error_injection(void) +{ + cachefiles_sysctl = register_sysctl("cachefiles", cachefiles_sysctls); + if (!cachefiles_sysctl) + return -ENOMEM; + return 0; + +} + +void cachefiles_unregister_error_injection(void) +{ + unregister_sysctl_table(cachefiles_sysctl); +} diff --git a/fs/cachefiles/interface.c b/fs/cachefiles/interface.c new file mode 100644 index 000000000..50a000310 --- /dev/null +++ b/fs/cachefiles/interface.c @@ -0,0 +1,453 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* FS-Cache interface to CacheFiles + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +static atomic_t cachefiles_object_debug_id; + +/* + * Allocate a cache object record. + */ +static +struct cachefiles_object *cachefiles_alloc_object(struct fscache_cookie *cookie) +{ + struct fscache_volume *vcookie = cookie->volume; + struct cachefiles_volume *volume = vcookie->cache_priv; + struct cachefiles_object *object; + + _enter("{%s},%x,", vcookie->key, cookie->debug_id); + + object = kmem_cache_zalloc(cachefiles_object_jar, GFP_KERNEL); + if (!object) + return NULL; + + refcount_set(&object->ref, 1); + + spin_lock_init(&object->lock); + INIT_LIST_HEAD(&object->cache_link); + object->volume = volume; + object->debug_id = atomic_inc_return(&cachefiles_object_debug_id); + object->cookie = fscache_get_cookie(cookie, fscache_cookie_get_attach_object); + + fscache_count_object(vcookie->cache); + trace_cachefiles_ref(object->debug_id, cookie->debug_id, 1, + cachefiles_obj_new); + return object; +} + +/* + * Note that an object has been seen. + */ +void cachefiles_see_object(struct cachefiles_object *object, + enum cachefiles_obj_ref_trace why) +{ + trace_cachefiles_ref(object->debug_id, object->cookie->debug_id, + refcount_read(&object->ref), why); +} + +/* + * Increment the usage count on an object; + */ +struct cachefiles_object *cachefiles_grab_object(struct cachefiles_object *object, + enum cachefiles_obj_ref_trace why) +{ + int r; + + __refcount_inc(&object->ref, &r); + trace_cachefiles_ref(object->debug_id, object->cookie->debug_id, r, why); + return object; +} + +/* + * dispose of a reference to an object + */ +void cachefiles_put_object(struct cachefiles_object *object, + enum cachefiles_obj_ref_trace why) +{ + unsigned int object_debug_id = object->debug_id; + unsigned int cookie_debug_id = object->cookie->debug_id; + struct fscache_cache *cache; + bool done; + int r; + + done = __refcount_dec_and_test(&object->ref, &r); + trace_cachefiles_ref(object_debug_id, cookie_debug_id, r, why); + if (done) { + _debug("- kill object OBJ%x", object_debug_id); + + ASSERTCMP(object->file, ==, NULL); + + kfree(object->d_name); + cache = object->volume->cache->cache; + fscache_put_cookie(object->cookie, fscache_cookie_put_object); + object->cookie = NULL; + kmem_cache_free(cachefiles_object_jar, object); + fscache_uncount_object(cache); + } + + _leave(""); +} + +/* + * Adjust the size of a cache file if necessary to match the DIO size. We keep + * the EOF marker a multiple of DIO blocks so that we don't fall back to doing + * non-DIO for a partial block straddling the EOF, but we also have to be + * careful of someone expanding the file and accidentally accreting the + * padding. + */ +static int cachefiles_adjust_size(struct cachefiles_object *object) +{ + struct iattr newattrs; + struct file *file = object->file; + uint64_t ni_size; + loff_t oi_size; + int ret; + + ni_size = object->cookie->object_size; + ni_size = round_up(ni_size, CACHEFILES_DIO_BLOCK_SIZE); + + _enter("{OBJ%x},[%llu]", + object->debug_id, (unsigned long long) ni_size); + + if (!file) + return -ENOBUFS; + + oi_size = i_size_read(file_inode(file)); + if (oi_size == ni_size) + return 0; + + inode_lock(file_inode(file)); + + /* if there's an extension to a partial page at the end of the backing + * file, we need to discard the partial page so that we pick up new + * data after it */ + if (oi_size & ~PAGE_MASK && ni_size > oi_size) { + _debug("discard tail %llx", oi_size); + newattrs.ia_valid = ATTR_SIZE; + newattrs.ia_size = oi_size & PAGE_MASK; + ret = cachefiles_inject_remove_error(); + if (ret == 0) + ret = notify_change(&nop_mnt_idmap, file->f_path.dentry, + &newattrs, NULL); + if (ret < 0) + goto truncate_failed; + } + + newattrs.ia_valid = ATTR_SIZE; + newattrs.ia_size = ni_size; + ret = cachefiles_inject_write_error(); + if (ret == 0) + ret = notify_change(&nop_mnt_idmap, file->f_path.dentry, + &newattrs, NULL); + +truncate_failed: + inode_unlock(file_inode(file)); + + if (ret < 0) + trace_cachefiles_io_error(NULL, file_inode(file), ret, + cachefiles_trace_notify_change_error); + if (ret == -EIO) { + cachefiles_io_error_obj(object, "Size set failed"); + ret = -ENOBUFS; + } + + _leave(" = %d", ret); + return ret; +} + +/* + * Attempt to look up the nominated node in this cache + */ +static bool cachefiles_lookup_cookie(struct fscache_cookie *cookie) +{ + struct cachefiles_object *object; + struct cachefiles_cache *cache = cookie->volume->cache->cache_priv; + const struct cred *saved_cred; + bool success; + + object = cachefiles_alloc_object(cookie); + if (!object) + goto fail; + + _enter("{OBJ%x}", object->debug_id); + + if (!cachefiles_cook_key(object)) + goto fail_put; + + cookie->cache_priv = object; + + cachefiles_begin_secure(cache, &saved_cred); + + success = cachefiles_look_up_object(object); + if (!success) + goto fail_withdraw; + + cachefiles_see_object(object, cachefiles_obj_see_lookup_cookie); + + spin_lock(&cache->object_list_lock); + list_add(&object->cache_link, &cache->object_list); + spin_unlock(&cache->object_list_lock); + cachefiles_adjust_size(object); + + cachefiles_end_secure(cache, saved_cred); + _leave(" = t"); + return true; + +fail_withdraw: + cachefiles_end_secure(cache, saved_cred); + cachefiles_see_object(object, cachefiles_obj_see_lookup_failed); + fscache_caching_failed(cookie); + _debug("failed c=%08x o=%08x", cookie->debug_id, object->debug_id); + /* The caller holds an access count on the cookie, so we need them to + * drop it before we can withdraw the object. + */ + return false; + +fail_put: + cachefiles_put_object(object, cachefiles_obj_put_alloc_fail); +fail: + return false; +} + +/* + * Shorten the backing object to discard any dirty data and free up + * any unused granules. + */ +static bool cachefiles_shorten_object(struct cachefiles_object *object, + struct file *file, loff_t new_size) +{ + struct cachefiles_cache *cache = object->volume->cache; + struct inode *inode = file_inode(file); + loff_t i_size, dio_size; + int ret; + + dio_size = round_up(new_size, CACHEFILES_DIO_BLOCK_SIZE); + i_size = i_size_read(inode); + + trace_cachefiles_trunc(object, inode, i_size, dio_size, + cachefiles_trunc_shrink); + ret = cachefiles_inject_remove_error(); + if (ret == 0) + ret = vfs_truncate(&file->f_path, dio_size); + if (ret < 0) { + trace_cachefiles_io_error(object, file_inode(file), ret, + cachefiles_trace_trunc_error); + cachefiles_io_error_obj(object, "Trunc-to-size failed %d", ret); + cachefiles_remove_object_xattr(cache, object, file->f_path.dentry); + return false; + } + + if (new_size < dio_size) { + trace_cachefiles_trunc(object, inode, dio_size, new_size, + cachefiles_trunc_dio_adjust); + ret = cachefiles_inject_write_error(); + if (ret == 0) + ret = vfs_fallocate(file, FALLOC_FL_ZERO_RANGE, + new_size, dio_size - new_size); + if (ret < 0) { + trace_cachefiles_io_error(object, file_inode(file), ret, + cachefiles_trace_fallocate_error); + cachefiles_io_error_obj(object, "Trunc-to-dio-size failed %d", ret); + cachefiles_remove_object_xattr(cache, object, file->f_path.dentry); + return false; + } + } + + return true; +} + +/* + * Resize the backing object. + */ +static void cachefiles_resize_cookie(struct netfs_cache_resources *cres, + loff_t new_size) +{ + struct cachefiles_object *object = cachefiles_cres_object(cres); + struct cachefiles_cache *cache = object->volume->cache; + struct fscache_cookie *cookie = object->cookie; + const struct cred *saved_cred; + struct file *file = cachefiles_cres_file(cres); + loff_t old_size = cookie->object_size; + + _enter("%llu->%llu", old_size, new_size); + + if (new_size < old_size) { + cachefiles_begin_secure(cache, &saved_cred); + cachefiles_shorten_object(object, file, new_size); + cachefiles_end_secure(cache, saved_cred); + object->cookie->object_size = new_size; + return; + } + + /* The file is being expanded. We don't need to do anything + * particularly. cookie->initial_size doesn't change and so the point + * at which we have to download before doesn't change. + */ + cookie->object_size = new_size; +} + +/* + * Commit changes to the object as we drop it. + */ +static void cachefiles_commit_object(struct cachefiles_object *object, + struct cachefiles_cache *cache) +{ + bool update = false; + + if (test_and_clear_bit(FSCACHE_COOKIE_LOCAL_WRITE, &object->cookie->flags)) + update = true; + if (test_and_clear_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &object->cookie->flags)) + update = true; + if (update) + cachefiles_set_object_xattr(object); + + if (test_bit(CACHEFILES_OBJECT_USING_TMPFILE, &object->flags)) + cachefiles_commit_tmpfile(cache, object); +} + +/* + * Finalise and object and close the VFS structs that we have. + */ +static void cachefiles_clean_up_object(struct cachefiles_object *object, + struct cachefiles_cache *cache) +{ + struct file *file; + + if (test_bit(FSCACHE_COOKIE_RETIRED, &object->cookie->flags)) { + if (!test_bit(CACHEFILES_OBJECT_USING_TMPFILE, &object->flags)) { + cachefiles_see_object(object, cachefiles_obj_see_clean_delete); + _debug("- inval object OBJ%x", object->debug_id); + cachefiles_delete_object(object, FSCACHE_OBJECT_WAS_RETIRED); + } else { + cachefiles_see_object(object, cachefiles_obj_see_clean_drop_tmp); + _debug("- inval object OBJ%x tmpfile", object->debug_id); + } + } else { + cachefiles_see_object(object, cachefiles_obj_see_clean_commit); + cachefiles_commit_object(object, cache); + } + + cachefiles_unmark_inode_in_use(object, object->file); + + spin_lock(&object->lock); + file = object->file; + object->file = NULL; + spin_unlock(&object->lock); + + if (file) + fput(file); +} + +/* + * Withdraw caching for a cookie. + */ +static void cachefiles_withdraw_cookie(struct fscache_cookie *cookie) +{ + struct cachefiles_object *object = cookie->cache_priv; + struct cachefiles_cache *cache = object->volume->cache; + const struct cred *saved_cred; + + _enter("o=%x", object->debug_id); + cachefiles_see_object(object, cachefiles_obj_see_withdraw_cookie); + + if (!list_empty(&object->cache_link)) { + spin_lock(&cache->object_list_lock); + cachefiles_see_object(object, cachefiles_obj_see_withdrawal); + list_del_init(&object->cache_link); + spin_unlock(&cache->object_list_lock); + } + + if (object->file) { + cachefiles_begin_secure(cache, &saved_cred); + cachefiles_clean_up_object(object, cache); + cachefiles_end_secure(cache, saved_cred); + } + + cookie->cache_priv = NULL; + cachefiles_put_object(object, cachefiles_obj_put_detach); +} + +/* + * Invalidate the storage associated with a cookie. + */ +static bool cachefiles_invalidate_cookie(struct fscache_cookie *cookie) +{ + struct cachefiles_object *object = cookie->cache_priv; + struct file *new_file, *old_file; + bool old_tmpfile; + + _enter("o=%x,[%llu]", object->debug_id, object->cookie->object_size); + + old_tmpfile = test_bit(CACHEFILES_OBJECT_USING_TMPFILE, &object->flags); + + if (!object->file) { + fscache_resume_after_invalidation(cookie); + _leave(" = t [light]"); + return true; + } + + new_file = cachefiles_create_tmpfile(object); + if (IS_ERR(new_file)) + goto failed; + + /* Substitute the VFS target */ + _debug("sub"); + spin_lock(&object->lock); + + old_file = object->file; + object->file = new_file; + object->content_info = CACHEFILES_CONTENT_NO_DATA; + set_bit(CACHEFILES_OBJECT_USING_TMPFILE, &object->flags); + set_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &object->cookie->flags); + + spin_unlock(&object->lock); + _debug("subbed"); + + /* Allow I/O to take place again */ + fscache_resume_after_invalidation(cookie); + + if (old_file) { + if (!old_tmpfile) { + struct cachefiles_volume *volume = object->volume; + struct dentry *fan = volume->fanout[(u8)cookie->key_hash]; + struct dentry *obj; + + obj = start_removing_dentry(fan, old_file->f_path.dentry); + if (!IS_ERR(obj)) + cachefiles_bury_object(volume->cache, object, + fan, obj, + FSCACHE_OBJECT_INVALIDATED); + } + fput(old_file); + } + + _leave(" = t"); + return true; + +failed: + _leave(" = f"); + return false; +} + +const struct fscache_cache_ops cachefiles_cache_ops = { + .name = "cachefiles", + .acquire_volume = cachefiles_acquire_volume, + .free_volume = cachefiles_free_volume, + .lookup_cookie = cachefiles_lookup_cookie, + .withdraw_cookie = cachefiles_withdraw_cookie, + .invalidate_cookie = cachefiles_invalidate_cookie, + .begin_operation = cachefiles_begin_operation, + .resize_cookie = cachefiles_resize_cookie, + .prepare_to_write = cachefiles_prepare_to_write, +}; diff --git a/fs/cachefiles/internal.h b/fs/cachefiles/internal.h new file mode 100644 index 000000000..c93324e0f --- /dev/null +++ b/fs/cachefiles/internal.h @@ -0,0 +1,408 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* General netfs cache on cache files internal defs + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#ifdef pr_fmt +#undef pr_fmt +#endif + +#define pr_fmt(fmt) "CacheFiles: " fmt + + +#include +#include +#include + +#define CACHEFILES_DIO_BLOCK_SIZE 4096 + +struct cachefiles_cache; +struct cachefiles_object; + +enum cachefiles_content { + /* These values are saved on disk */ + CACHEFILES_CONTENT_NO_DATA = 0, /* No content stored */ + CACHEFILES_CONTENT_SINGLE = 1, /* Content is monolithic, all is present */ + CACHEFILES_CONTENT_ALL = 2, /* Content is all present, no map */ + CACHEFILES_CONTENT_BACKFS_MAP = 3, /* Content is piecemeal, mapped through backing fs */ + CACHEFILES_CONTENT_DIRTY = 4, /* Content is dirty (only seen on disk) */ + nr__cachefiles_content +}; + +/* + * Cached volume representation. + */ +struct cachefiles_volume { + struct cachefiles_cache *cache; + struct list_head cache_link; /* Link in cache->volumes */ + struct fscache_volume *vcookie; /* The netfs's representation */ + struct dentry *dentry; /* The volume dentry */ + struct dentry *fanout[256]; /* Fanout subdirs */ +}; + +/* + * Backing file state. + */ +struct cachefiles_object { + struct fscache_cookie *cookie; /* Netfs data storage object cookie */ + struct cachefiles_volume *volume; /* Cache volume that holds this object */ + struct list_head cache_link; /* Link in cache->*_list */ + struct file *file; /* The file representing this object */ + char *d_name; /* Backing file name */ + int debug_id; + spinlock_t lock; + refcount_t ref; + enum cachefiles_content content_info:8; /* Info about content presence */ + unsigned long flags; +#define CACHEFILES_OBJECT_USING_TMPFILE 0 /* Have an unlinked tmpfile */ +}; + +/* + * Cache files cache definition + */ +struct cachefiles_cache { + struct fscache_cache *cache; /* Cache cookie */ + struct vfsmount *mnt; /* mountpoint holding the cache */ + struct dentry *store; /* Directory into which live objects go */ + struct dentry *graveyard; /* directory into which dead objects go */ + struct file *cachefilesd; /* manager daemon handle */ + struct list_head volumes; /* List of volume objects */ + struct list_head object_list; /* List of active objects */ + spinlock_t object_list_lock; /* Lock for volumes and object_list */ + const struct cred *cache_cred; /* security override for accessing cache */ + struct mutex daemon_mutex; /* command serialisation mutex */ + wait_queue_head_t daemon_pollwq; /* poll waitqueue for daemon */ + atomic_t gravecounter; /* graveyard uniquifier */ + atomic_t f_released; /* number of objects released lately */ + atomic_long_t b_released; /* number of blocks released lately */ + atomic_long_t b_writing; /* Number of blocks being written */ + unsigned frun_percent; /* when to stop culling (% files) */ + unsigned fcull_percent; /* when to start culling (% files) */ + unsigned fstop_percent; /* when to stop allocating (% files) */ + unsigned brun_percent; /* when to stop culling (% blocks) */ + unsigned bcull_percent; /* when to start culling (% blocks) */ + unsigned bstop_percent; /* when to stop allocating (% blocks) */ + unsigned bsize; /* cache's block size */ + unsigned bshift; /* ilog2(bsize) */ + uint64_t frun; /* when to stop culling */ + uint64_t fcull; /* when to start culling */ + uint64_t fstop; /* when to stop allocating */ + sector_t brun; /* when to stop culling */ + sector_t bcull; /* when to start culling */ + sector_t bstop; /* when to stop allocating */ + unsigned long flags; +#define CACHEFILES_READY 0 /* T if cache prepared */ +#define CACHEFILES_DEAD 1 /* T if cache dead */ +#define CACHEFILES_CULLING 2 /* T if cull engaged */ +#define CACHEFILES_STATE_CHANGED 3 /* T if state changed (poll trigger) */ + char *rootdirname; /* name of cache root directory */ + char *tag; /* cache binding tag */ + u32 secid; /* LSM security id */ + bool have_secid; /* whether "secid" was set */ +}; + +#include + +static inline +struct file *cachefiles_cres_file(struct netfs_cache_resources *cres) +{ + return cres->cache_priv2; +} + +static inline +struct cachefiles_object *cachefiles_cres_object(struct netfs_cache_resources *cres) +{ + return fscache_cres_cookie(cres)->cache_priv; +} + +/* + * note change of state for daemon + */ +static inline void cachefiles_state_changed(struct cachefiles_cache *cache) +{ + set_bit(CACHEFILES_STATE_CHANGED, &cache->flags); + wake_up_all(&cache->daemon_pollwq); +} + +/* + * cache.c + */ +extern int cachefiles_add_cache(struct cachefiles_cache *cache); +extern void cachefiles_withdraw_cache(struct cachefiles_cache *cache); + +enum cachefiles_has_space_for { + cachefiles_has_space_check, + cachefiles_has_space_for_write, + cachefiles_has_space_for_create, +}; +extern int cachefiles_has_space(struct cachefiles_cache *cache, + unsigned fnr, unsigned bnr, + enum cachefiles_has_space_for reason); + +/* + * daemon.c + */ +extern const struct file_operations cachefiles_daemon_fops; + +/* + * error_inject.c + */ +#ifdef CONFIG_CACHEFILES_ERROR_INJECTION +extern unsigned int cachefiles_error_injection_state; +extern int cachefiles_register_error_injection(void); +extern void cachefiles_unregister_error_injection(void); + +#else +#define cachefiles_error_injection_state 0 + +static inline int cachefiles_register_error_injection(void) +{ + return 0; +} + +static inline void cachefiles_unregister_error_injection(void) +{ +} +#endif + + +static inline int cachefiles_inject_read_error(void) +{ + return cachefiles_error_injection_state & 2 ? -EIO : 0; +} + +static inline int cachefiles_inject_write_error(void) +{ + return cachefiles_error_injection_state & 2 ? -EIO : + cachefiles_error_injection_state & 1 ? -ENOSPC : + 0; +} + +static inline int cachefiles_inject_remove_error(void) +{ + return cachefiles_error_injection_state & 2 ? -EIO : 0; +} + +/* + * interface.c + */ +extern const struct fscache_cache_ops cachefiles_cache_ops; +extern void cachefiles_see_object(struct cachefiles_object *object, + enum cachefiles_obj_ref_trace why); +extern struct cachefiles_object *cachefiles_grab_object(struct cachefiles_object *object, + enum cachefiles_obj_ref_trace why); +extern void cachefiles_put_object(struct cachefiles_object *object, + enum cachefiles_obj_ref_trace why); + +/* + * io.c + */ +extern bool cachefiles_begin_operation(struct netfs_cache_resources *cres, + enum fscache_want_state want_state); +extern int __cachefiles_prepare_write(struct cachefiles_object *object, + struct file *file, + loff_t *_start, size_t *_len, size_t upper_len, + bool no_space_allocated_yet); +extern int __cachefiles_write(struct cachefiles_object *object, + struct file *file, + loff_t start_pos, + struct iov_iter *iter, + netfs_io_terminated_t term_func, + void *term_func_priv); + +/* + * key.c + */ +extern bool cachefiles_cook_key(struct cachefiles_object *object); + +/* + * main.c + */ +extern struct kmem_cache *cachefiles_object_jar; + +/* + * namei.c + */ +extern void cachefiles_unmark_inode_in_use(struct cachefiles_object *object, + struct file *file); +extern int cachefiles_bury_object(struct cachefiles_cache *cache, + struct cachefiles_object *object, + struct dentry *dir, + struct dentry *rep, + enum fscache_why_object_killed why); +extern int cachefiles_delete_object(struct cachefiles_object *object, + enum fscache_why_object_killed why); +extern bool cachefiles_look_up_object(struct cachefiles_object *object); +extern struct dentry *cachefiles_get_directory(struct cachefiles_cache *cache, + struct dentry *dir, + const char *name, + bool *_is_new); +extern void cachefiles_put_directory(struct dentry *dir); + +extern int cachefiles_cull(struct cachefiles_cache *cache, struct dentry *dir, + char *filename); + +extern int cachefiles_check_in_use(struct cachefiles_cache *cache, + struct dentry *dir, char *filename); +extern struct file *cachefiles_create_tmpfile(struct cachefiles_object *object); +extern bool cachefiles_commit_tmpfile(struct cachefiles_cache *cache, + struct cachefiles_object *object); + +/* + * security.c + */ +extern int cachefiles_get_security_ID(struct cachefiles_cache *cache); +extern int cachefiles_determine_cache_security(struct cachefiles_cache *cache, + struct dentry *root, + const struct cred **_saved_cred); + +static inline void cachefiles_begin_secure(struct cachefiles_cache *cache, + const struct cred **_saved_cred) +{ + *_saved_cred = override_creds(cache->cache_cred); +} + +static inline void cachefiles_end_secure(struct cachefiles_cache *cache, + const struct cred *saved_cred) +{ + revert_creds(saved_cred); +} + +/* + * volume.c + */ +void cachefiles_acquire_volume(struct fscache_volume *volume); +void cachefiles_free_volume(struct fscache_volume *volume); +void cachefiles_withdraw_volume(struct cachefiles_volume *volume); + +/* + * xattr.c + */ +extern int cachefiles_set_object_xattr(struct cachefiles_object *object); +extern int cachefiles_check_auxdata(struct cachefiles_object *object, + struct file *file); +extern int cachefiles_remove_object_xattr(struct cachefiles_cache *cache, + struct cachefiles_object *object, + struct dentry *dentry); +extern void cachefiles_prepare_to_write(struct fscache_cookie *cookie); +extern bool cachefiles_set_volume_xattr(struct cachefiles_volume *volume); +extern int cachefiles_check_volume_xattr(struct cachefiles_volume *volume); + +/* + * Error handling + */ +#define cachefiles_io_error(___cache, FMT, ...) \ +do { \ + pr_err("I/O Error: " FMT"\n", ##__VA_ARGS__); \ + fscache_io_error((___cache)->cache); \ + set_bit(CACHEFILES_DEAD, &(___cache)->flags); \ +} while (0) + +#define cachefiles_io_error_obj(object, FMT, ...) \ +do { \ + struct cachefiles_cache *___cache; \ + \ + ___cache = (object)->volume->cache; \ + cachefiles_io_error(___cache, FMT " [o=%08x]", ##__VA_ARGS__, \ + (object)->debug_id); \ +} while (0) + + +/* + * Debug tracing + */ +extern unsigned cachefiles_debug; +#define CACHEFILES_DEBUG_KENTER 1 +#define CACHEFILES_DEBUG_KLEAVE 2 +#define CACHEFILES_DEBUG_KDEBUG 4 + +#define dbgprintk(FMT, ...) \ + printk(KERN_DEBUG "[%-6.6s] "FMT"\n", current->comm, ##__VA_ARGS__) + +#define kenter(FMT, ...) dbgprintk("==> %s("FMT")", __func__, ##__VA_ARGS__) +#define kleave(FMT, ...) dbgprintk("<== %s()"FMT"", __func__, ##__VA_ARGS__) +#define kdebug(FMT, ...) dbgprintk(FMT, ##__VA_ARGS__) + + +#if defined(__KDEBUG) +#define _enter(FMT, ...) kenter(FMT, ##__VA_ARGS__) +#define _leave(FMT, ...) kleave(FMT, ##__VA_ARGS__) +#define _debug(FMT, ...) kdebug(FMT, ##__VA_ARGS__) + +#elif defined(CONFIG_CACHEFILES_DEBUG) +#define _enter(FMT, ...) \ +do { \ + if (cachefiles_debug & CACHEFILES_DEBUG_KENTER) \ + kenter(FMT, ##__VA_ARGS__); \ +} while (0) + +#define _leave(FMT, ...) \ +do { \ + if (cachefiles_debug & CACHEFILES_DEBUG_KLEAVE) \ + kleave(FMT, ##__VA_ARGS__); \ +} while (0) + +#define _debug(FMT, ...) \ +do { \ + if (cachefiles_debug & CACHEFILES_DEBUG_KDEBUG) \ + kdebug(FMT, ##__VA_ARGS__); \ +} while (0) + +#else +#define _enter(FMT, ...) no_printk("==> %s("FMT")", __func__, ##__VA_ARGS__) +#define _leave(FMT, ...) no_printk("<== %s()"FMT"", __func__, ##__VA_ARGS__) +#define _debug(FMT, ...) no_printk(FMT, ##__VA_ARGS__) +#endif + +#if 1 /* defined(__KDEBUGALL) */ + +#define ASSERT(X) \ +do { \ + if (unlikely(!(X))) { \ + pr_err("\n"); \ + pr_err("Assertion failed\n"); \ + BUG(); \ + } \ +} while (0) + +#define ASSERTCMP(X, OP, Y) \ +do { \ + if (unlikely(!((X) OP (Y)))) { \ + pr_err("\n"); \ + pr_err("Assertion failed\n"); \ + pr_err("%lx " #OP " %lx is false\n", \ + (unsigned long)(X), (unsigned long)(Y)); \ + BUG(); \ + } \ +} while (0) + +#define ASSERTIF(C, X) \ +do { \ + if (unlikely((C) && !(X))) { \ + pr_err("\n"); \ + pr_err("Assertion failed\n"); \ + BUG(); \ + } \ +} while (0) + +#define ASSERTIFCMP(C, X, OP, Y) \ +do { \ + if (unlikely((C) && !((X) OP (Y)))) { \ + pr_err("\n"); \ + pr_err("Assertion failed\n"); \ + pr_err("%lx " #OP " %lx is false\n", \ + (unsigned long)(X), (unsigned long)(Y)); \ + BUG(); \ + } \ +} while (0) + +#else + +#define ASSERT(X) do {} while (0) +#define ASSERTCMP(X, OP, Y) do {} while (0) +#define ASSERTIF(C, X) do {} while (0) +#define ASSERTIFCMP(C, X, OP, Y) do {} while (0) + +#endif diff --git a/fs/cachefiles/io.c b/fs/cachefiles/io.c new file mode 100644 index 000000000..9540ec25b --- /dev/null +++ b/fs/cachefiles/io.c @@ -0,0 +1,738 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* kiocb-using read/write + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +struct cachefiles_kiocb { + struct kiocb iocb; + refcount_t ki_refcnt; + loff_t start; + union { + size_t skipped; + size_t len; + }; + struct cachefiles_object *object; + netfs_io_terminated_t term_func; + void *term_func_priv; + bool was_async; + unsigned int inval_counter; /* Copy of cookie->inval_counter */ + u64 b_writing; +}; + +static inline void cachefiles_put_kiocb(struct cachefiles_kiocb *ki) +{ + if (refcount_dec_and_test(&ki->ki_refcnt)) { + cachefiles_put_object(ki->object, cachefiles_obj_put_ioreq); + fput(ki->iocb.ki_filp); + kfree(ki); + } +} + +/* + * Handle completion of a read from the cache. + */ +static void cachefiles_read_complete(struct kiocb *iocb, long ret) +{ + struct cachefiles_kiocb *ki = container_of(iocb, struct cachefiles_kiocb, iocb); + struct inode *inode = file_inode(ki->iocb.ki_filp); + + _enter("%ld", ret); + + if (ret < 0) + trace_cachefiles_io_error(ki->object, inode, ret, + cachefiles_trace_read_error); + + if (ki->term_func) { + if (ret >= 0) { + if (ki->object->cookie->inval_counter == ki->inval_counter) + ki->skipped += ret; + else + ret = -ESTALE; + } + + ki->term_func(ki->term_func_priv, ret); + } + + cachefiles_put_kiocb(ki); +} + +/* + * Initiate a read from the cache. + */ +static int cachefiles_read(struct netfs_cache_resources *cres, + loff_t start_pos, + struct iov_iter *iter, + enum netfs_read_from_hole read_hole, + netfs_io_terminated_t term_func, + void *term_func_priv) +{ + struct cachefiles_object *object; + struct cachefiles_kiocb *ki; + struct file *file; + unsigned int old_nofs; + ssize_t ret = -ENOBUFS; + size_t len = iov_iter_count(iter), skipped = 0; + + if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ)) + goto presubmission_error; + + fscache_count_read(); + object = cachefiles_cres_object(cres); + file = cachefiles_cres_file(cres); + + _enter("%pD,%llu,%llx,%zx/%llx", + file, file_inode(file)->i_ino, start_pos, len, + i_size_read(file_inode(file))); + + /* If the caller asked us to seek for data before doing the read, then + * we should do that now. If we find a gap, we fill it with zeros. + */ + if (read_hole != NETFS_READ_HOLE_IGNORE) { + loff_t off = start_pos, off2; + + off2 = cachefiles_inject_read_error(); + if (off2 == 0) + off2 = vfs_llseek(file, off, SEEK_DATA); + if (off2 < 0 && off2 >= (loff_t)-MAX_ERRNO && off2 != -ENXIO) { + skipped = 0; + ret = off2; + goto presubmission_error; + } + + if (off2 == -ENXIO || off2 >= start_pos + len) { + /* The region is beyond the EOF or there's no more data + * in the region, so clear the rest of the buffer and + * return success. + */ + ret = -ENODATA; + if (read_hole == NETFS_READ_HOLE_FAIL) + goto presubmission_error; + + iov_iter_zero(len, iter); + skipped = len; + ret = 0; + goto presubmission_error; + } + + skipped = off2 - off; + iov_iter_zero(skipped, iter); + } + + ret = -ENOMEM; + ki = kzalloc_obj(struct cachefiles_kiocb); + if (!ki) + goto presubmission_error; + + refcount_set(&ki->ki_refcnt, 2); + ki->iocb.ki_filp = file; + ki->iocb.ki_pos = start_pos + skipped; + ki->iocb.ki_flags = IOCB_DIRECT; + ki->iocb.ki_ioprio = get_current_ioprio(); + ki->skipped = skipped; + ki->object = object; + ki->inval_counter = cres->inval_counter; + ki->term_func = term_func; + ki->term_func_priv = term_func_priv; + ki->was_async = true; + + if (ki->term_func) + ki->iocb.ki_complete = cachefiles_read_complete; + + get_file(ki->iocb.ki_filp); + cachefiles_grab_object(object, cachefiles_obj_get_ioreq); + + trace_cachefiles_read(object, file_inode(file), ki->iocb.ki_pos, len - skipped); + old_nofs = memalloc_nofs_save(); + ret = cachefiles_inject_read_error(); + if (ret == 0) + ret = vfs_iocb_iter_read(file, &ki->iocb, iter); + memalloc_nofs_restore(old_nofs); + switch (ret) { + case -EIOCBQUEUED: + goto in_progress; + + case -ERESTARTSYS: + case -ERESTARTNOINTR: + case -ERESTARTNOHAND: + case -ERESTART_RESTARTBLOCK: + /* There's no easy way to restart the syscall since other AIO's + * may be already running. Just fail this IO with EINTR. + */ + ret = -EINTR; + fallthrough; + default: + ki->was_async = false; + cachefiles_read_complete(&ki->iocb, ret); + if (ret > 0) + ret = 0; + break; + } + +in_progress: + cachefiles_put_kiocb(ki); + _leave(" = %zd", ret); + return ret; + +presubmission_error: + if (term_func) + term_func(term_func_priv, ret < 0 ? ret : skipped); + return ret; +} + +/* + * Query the occupancy of the cache in a region, returning where the next chunk + * of data starts and how long it is. + */ +static int cachefiles_query_occupancy(struct netfs_cache_resources *cres, + loff_t start, size_t len, size_t granularity, + loff_t *_data_start, size_t *_data_len) +{ + struct cachefiles_object *object; + struct file *file; + loff_t off, off2; + + *_data_start = -1; + *_data_len = 0; + + if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ)) + return -ENOBUFS; + + object = cachefiles_cres_object(cres); + file = cachefiles_cres_file(cres); + granularity = max_t(size_t, object->volume->cache->bsize, granularity); + + _enter("%pD,%llu,%llx,%zx/%llx", + file, file_inode(file)->i_ino, start, len, + i_size_read(file_inode(file))); + + off = cachefiles_inject_read_error(); + if (off == 0) + off = vfs_llseek(file, start, SEEK_DATA); + if (off == -ENXIO) + return -ENODATA; /* Beyond EOF */ + if (off < 0 && off >= (loff_t)-MAX_ERRNO) + return -ENOBUFS; /* Error. */ + if (round_up(off, granularity) >= start + len) + return -ENODATA; /* No data in range */ + + off2 = cachefiles_inject_read_error(); + if (off2 == 0) + off2 = vfs_llseek(file, off, SEEK_HOLE); + if (off2 == -ENXIO) + return -ENODATA; /* Beyond EOF */ + if (off2 < 0 && off2 >= (loff_t)-MAX_ERRNO) + return -ENOBUFS; /* Error. */ + + /* Round away partial blocks */ + off = round_up(off, granularity); + off2 = round_down(off2, granularity); + if (off2 <= off) + return -ENODATA; + + *_data_start = off; + if (off2 > start + len) + *_data_len = len; + else + *_data_len = off2 - off; + return 0; +} + +/* + * Handle completion of a write to the cache. + */ +static void cachefiles_write_complete(struct kiocb *iocb, long ret) +{ + struct cachefiles_kiocb *ki = container_of(iocb, struct cachefiles_kiocb, iocb); + struct cachefiles_object *object = ki->object; + struct inode *inode = file_inode(ki->iocb.ki_filp); + + _enter("%ld", ret); + + if (ki->was_async) + kiocb_end_write(iocb); + + if (ret < 0) + trace_cachefiles_io_error(object, inode, ret, + cachefiles_trace_write_error); + + atomic_long_sub(ki->b_writing, &object->volume->cache->b_writing); + set_bit(FSCACHE_COOKIE_HAVE_DATA, &object->cookie->flags); + if (ki->term_func) + ki->term_func(ki->term_func_priv, ret); + cachefiles_put_kiocb(ki); +} + +/* + * Initiate a write to the cache. + */ +int __cachefiles_write(struct cachefiles_object *object, + struct file *file, + loff_t start_pos, + struct iov_iter *iter, + netfs_io_terminated_t term_func, + void *term_func_priv) +{ + struct cachefiles_cache *cache; + struct cachefiles_kiocb *ki; + unsigned int old_nofs; + ssize_t ret; + size_t len = iov_iter_count(iter); + + fscache_count_write(); + cache = object->volume->cache; + + _enter("%pD,%llu,%llx,%zx/%llx", + file, file_inode(file)->i_ino, start_pos, len, + i_size_read(file_inode(file))); + + ki = kzalloc_obj(struct cachefiles_kiocb); + if (!ki) { + if (term_func) + term_func(term_func_priv, -ENOMEM); + return -ENOMEM; + } + + refcount_set(&ki->ki_refcnt, 2); + ki->iocb.ki_filp = file; + ki->iocb.ki_pos = start_pos; + ki->iocb.ki_flags = IOCB_DIRECT | IOCB_WRITE; + ki->iocb.ki_ioprio = get_current_ioprio(); + ki->object = object; + ki->start = start_pos; + ki->len = len; + ki->term_func = term_func; + ki->term_func_priv = term_func_priv; + ki->was_async = true; + ki->b_writing = (len + (1 << cache->bshift) - 1) >> cache->bshift; + + if (ki->term_func) + ki->iocb.ki_complete = cachefiles_write_complete; + atomic_long_add(ki->b_writing, &cache->b_writing); + + get_file(ki->iocb.ki_filp); + cachefiles_grab_object(object, cachefiles_obj_get_ioreq); + + trace_cachefiles_write(object, file_inode(file), ki->iocb.ki_pos, len); + old_nofs = memalloc_nofs_save(); + ret = cachefiles_inject_write_error(); + if (ret == 0) + ret = vfs_iocb_iter_write(file, &ki->iocb, iter); + memalloc_nofs_restore(old_nofs); + switch (ret) { + case -EIOCBQUEUED: + goto in_progress; + + case -ERESTARTSYS: + case -ERESTARTNOINTR: + case -ERESTARTNOHAND: + case -ERESTART_RESTARTBLOCK: + /* There's no easy way to restart the syscall since other AIO's + * may be already running. Just fail this IO with EINTR. + */ + ret = -EINTR; + fallthrough; + default: + ki->was_async = false; + cachefiles_write_complete(&ki->iocb, ret); + break; + } + +in_progress: + cachefiles_put_kiocb(ki); + _leave(" = %zd", ret); + return ret; +} + +static int cachefiles_write(struct netfs_cache_resources *cres, + loff_t start_pos, + struct iov_iter *iter, + netfs_io_terminated_t term_func, + void *term_func_priv) +{ + if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE)) { + if (term_func) + term_func(term_func_priv, -ENOBUFS); + trace_netfs_sreq(term_func_priv, netfs_sreq_trace_cache_nowrite); + return -ENOBUFS; + } + + return __cachefiles_write(cachefiles_cres_object(cres), + cachefiles_cres_file(cres), + start_pos, iter, + term_func, term_func_priv); +} + +static inline enum netfs_io_source +cachefiles_do_prepare_read(struct netfs_cache_resources *cres, + loff_t start, size_t *_len, loff_t i_size, + unsigned long *_flags, ino_t netfs_ino) +{ + enum cachefiles_prepare_read_trace why; + struct cachefiles_object *object = NULL; + struct cachefiles_cache *cache; + struct fscache_cookie *cookie = fscache_cres_cookie(cres); + const struct cred *saved_cred; + struct file *file = cachefiles_cres_file(cres); + enum netfs_io_source ret = NETFS_DOWNLOAD_FROM_SERVER; + size_t len = *_len; + loff_t off, to; + ino_t ino = file ? file_inode(file)->i_ino : 0; + + _enter("%zx @%llx/%llx", len, start, i_size); + + if (start >= i_size) { + ret = NETFS_FILL_WITH_ZEROES; + why = cachefiles_trace_read_after_eof; + goto out_no_object; + } + + if (test_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags)) { + __set_bit(NETFS_SREQ_COPY_TO_CACHE, _flags); + why = cachefiles_trace_read_no_data; + goto out_no_object; + } + + /* The object and the file may be being created in the background. */ + if (!file) { + why = cachefiles_trace_read_no_file; + if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ)) + goto out_no_object; + file = cachefiles_cres_file(cres); + if (!file) + goto out_no_object; + ino = file_inode(file)->i_ino; + } + + object = cachefiles_cres_object(cres); + cache = object->volume->cache; + cachefiles_begin_secure(cache, &saved_cred); + off = cachefiles_inject_read_error(); + if (off == 0) + off = vfs_llseek(file, start, SEEK_DATA); + if (off < 0 && off >= (loff_t)-MAX_ERRNO) { + if (off == (loff_t)-ENXIO) { + why = cachefiles_trace_read_seek_nxio; + goto download_and_store; + } + trace_cachefiles_io_error(object, file_inode(file), off, + cachefiles_trace_seek_error); + why = cachefiles_trace_read_seek_error; + goto out; + } + + if (off >= start + len) { + why = cachefiles_trace_read_found_hole; + goto download_and_store; + } + + if (off > start) { + off = round_up(off, cache->bsize); + len = off - start; + *_len = len; + why = cachefiles_trace_read_found_part; + goto download_and_store; + } + + to = cachefiles_inject_read_error(); + if (to == 0) + to = vfs_llseek(file, start, SEEK_HOLE); + if (to < 0 && to >= (loff_t)-MAX_ERRNO) { + trace_cachefiles_io_error(object, file_inode(file), to, + cachefiles_trace_seek_error); + why = cachefiles_trace_read_seek_error; + goto out; + } + + if (to < start + len) { + if (start + len >= i_size) + to = round_up(to, cache->bsize); + else + to = round_down(to, cache->bsize); + len = to - start; + *_len = len; + } + + why = cachefiles_trace_read_have_data; + ret = NETFS_READ_FROM_CACHE; + goto out; + +download_and_store: + __set_bit(NETFS_SREQ_COPY_TO_CACHE, _flags); +out: + cachefiles_end_secure(cache, saved_cred); +out_no_object: + trace_cachefiles_prep_read(object, start, len, *_flags, ret, why, ino, netfs_ino); + return ret; +} + +/* + * Prepare a read operation, shortening it to a cached/uncached + * boundary as appropriate. + */ +static enum netfs_io_source cachefiles_prepare_read(struct netfs_io_subrequest *subreq, + unsigned long long i_size) +{ + return cachefiles_do_prepare_read(&subreq->rreq->cache_resources, + subreq->start, &subreq->len, i_size, + &subreq->flags, subreq->rreq->inode->i_ino); +} + +/* + * Prepare for a write to occur. + */ +int __cachefiles_prepare_write(struct cachefiles_object *object, + struct file *file, + loff_t *_start, size_t *_len, size_t upper_len, + bool no_space_allocated_yet) +{ + struct cachefiles_cache *cache = object->volume->cache; + loff_t start = *_start, pos; + size_t len = *_len; + int ret; + + /* Round to DIO size */ + start = round_down(*_start, PAGE_SIZE); + if (start != *_start || *_len > upper_len) { + /* Probably asked to cache a streaming write written into the + * pagecache when the cookie was temporarily out of service to + * culling. + */ + fscache_count_dio_misfit(); + return -ENOBUFS; + } + + *_len = round_up(len, PAGE_SIZE); + + /* We need to work out whether there's sufficient disk space to perform + * the write - but we can skip that check if we have space already + * allocated. + */ + if (no_space_allocated_yet) + goto check_space; + + pos = cachefiles_inject_read_error(); + if (pos == 0) + pos = vfs_llseek(file, start, SEEK_DATA); + if (pos < 0 && pos >= (loff_t)-MAX_ERRNO) { + if (pos == -ENXIO) + goto check_space; /* Unallocated tail */ + trace_cachefiles_io_error(object, file_inode(file), pos, + cachefiles_trace_seek_error); + return pos; + } + if ((u64)pos >= (u64)start + *_len) + goto check_space; /* Unallocated region */ + + /* We have a block that's at least partially filled - if we're low on + * space, we need to see if it's fully allocated. If it's not, we may + * want to cull it. + */ + if (cachefiles_has_space(cache, 0, *_len / PAGE_SIZE, + cachefiles_has_space_check) == 0) + return 0; /* Enough space to simply overwrite the whole block */ + + pos = cachefiles_inject_read_error(); + if (pos == 0) + pos = vfs_llseek(file, start, SEEK_HOLE); + if (pos < 0 && pos >= (loff_t)-MAX_ERRNO) { + trace_cachefiles_io_error(object, file_inode(file), pos, + cachefiles_trace_seek_error); + return pos; + } + if ((u64)pos >= (u64)start + *_len) + return 0; /* Fully allocated */ + + /* Partially allocated, but insufficient space: cull. */ + fscache_count_no_write_space(); + ret = cachefiles_inject_remove_error(); + if (ret == 0) + ret = vfs_fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + start, *_len); + if (ret < 0) { + trace_cachefiles_io_error(object, file_inode(file), ret, + cachefiles_trace_fallocate_error); + cachefiles_io_error_obj(object, + "CacheFiles: fallocate failed (%d)\n", ret); + ret = -EIO; + } + + return ret; + +check_space: + return cachefiles_has_space(cache, 0, *_len / PAGE_SIZE, + cachefiles_has_space_for_write); +} + +static int cachefiles_prepare_write(struct netfs_cache_resources *cres, + loff_t *_start, size_t *_len, size_t upper_len, + loff_t i_size, bool no_space_allocated_yet) +{ + struct cachefiles_object *object = cachefiles_cres_object(cres); + struct cachefiles_cache *cache = object->volume->cache; + const struct cred *saved_cred; + int ret; + + if (!cachefiles_cres_file(cres)) { + if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE)) + return -ENOBUFS; + if (!cachefiles_cres_file(cres)) + return -ENOBUFS; + } + + cachefiles_begin_secure(cache, &saved_cred); + ret = __cachefiles_prepare_write(object, cachefiles_cres_file(cres), + _start, _len, upper_len, + no_space_allocated_yet); + cachefiles_end_secure(cache, saved_cred); + return ret; +} + +static void cachefiles_prepare_write_subreq(struct netfs_io_subrequest *subreq) +{ + struct netfs_io_request *wreq = subreq->rreq; + struct netfs_cache_resources *cres = &wreq->cache_resources; + struct netfs_io_stream *stream = &wreq->io_streams[subreq->stream_nr]; + + _enter("W=%x[%x] %llx", wreq->debug_id, subreq->debug_index, subreq->start); + + stream->sreq_max_len = MAX_RW_COUNT; + stream->sreq_max_segs = BIO_MAX_VECS; + + if (!cachefiles_cres_file(cres)) { + if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE)) + return netfs_prepare_write_failed(subreq); + if (!cachefiles_cres_file(cres)) + return netfs_prepare_write_failed(subreq); + } +} + +static void cachefiles_issue_write(struct netfs_io_subrequest *subreq) +{ + struct netfs_io_request *wreq = subreq->rreq; + struct netfs_cache_resources *cres = &wreq->cache_resources; + struct cachefiles_object *object = cachefiles_cres_object(cres); + struct cachefiles_cache *cache = object->volume->cache; + struct netfs_io_stream *stream = &wreq->io_streams[subreq->stream_nr]; + const struct cred *saved_cred; + size_t off, pre, post, len = subreq->len; + loff_t start = subreq->start; + int ret; + + _enter("W=%x[%x] %llx-%llx", + wreq->debug_id, subreq->debug_index, start, start + len - 1); + + /* We need to start on the cache granularity boundary */ + off = start & (CACHEFILES_DIO_BLOCK_SIZE - 1); + if (off) { + pre = CACHEFILES_DIO_BLOCK_SIZE - off; + if (pre >= len) { + fscache_count_dio_misfit(); + netfs_write_subrequest_terminated(subreq, len); + return; + } + subreq->transferred += pre; + start += pre; + len -= pre; + iov_iter_advance(&subreq->io_iter, pre); + } + + /* We also need to end on the cache granularity boundary */ + if (start + len == wreq->i_size) { + size_t part = len % CACHEFILES_DIO_BLOCK_SIZE; + size_t need = CACHEFILES_DIO_BLOCK_SIZE - part; + + if (part && stream->submit_extendable_to >= need) { + len += need; + subreq->len += need; + subreq->io_iter.count += need; + } + } + + post = len & (CACHEFILES_DIO_BLOCK_SIZE - 1); + if (post) { + len -= post; + if (len == 0) { + fscache_count_dio_misfit(); + netfs_write_subrequest_terminated(subreq, post); + return; + } + iov_iter_truncate(&subreq->io_iter, len); + } + + trace_netfs_sreq(subreq, netfs_sreq_trace_cache_prepare); + cachefiles_begin_secure(cache, &saved_cred); + ret = __cachefiles_prepare_write(object, cachefiles_cres_file(cres), + &start, &len, len, true); + cachefiles_end_secure(cache, saved_cred); + if (ret < 0) { + netfs_write_subrequest_terminated(subreq, ret); + return; + } + + trace_netfs_sreq(subreq, netfs_sreq_trace_cache_write); + cachefiles_write(&subreq->rreq->cache_resources, + subreq->start, &subreq->io_iter, + netfs_write_subrequest_terminated, subreq); +} + +/* + * Clean up an operation. + */ +static void cachefiles_end_operation(struct netfs_cache_resources *cres) +{ + struct file *file = cachefiles_cres_file(cres); + + if (file) + fput(file); + fscache_end_cookie_access(fscache_cres_cookie(cres), fscache_access_io_end); +} + +static const struct netfs_cache_ops cachefiles_netfs_cache_ops = { + .end_operation = cachefiles_end_operation, + .read = cachefiles_read, + .write = cachefiles_write, + .issue_write = cachefiles_issue_write, + .prepare_read = cachefiles_prepare_read, + .prepare_write = cachefiles_prepare_write, + .prepare_write_subreq = cachefiles_prepare_write_subreq, + .query_occupancy = cachefiles_query_occupancy, +}; + +/* + * Open the cache file when beginning a cache operation. + */ +bool cachefiles_begin_operation(struct netfs_cache_resources *cres, + enum fscache_want_state want_state) +{ + struct cachefiles_object *object = cachefiles_cres_object(cres); + + if (!cachefiles_cres_file(cres)) { + cres->ops = &cachefiles_netfs_cache_ops; + if (object->file) { + spin_lock(&object->lock); + if (!cres->cache_priv2 && object->file) + cres->cache_priv2 = get_file(object->file); + spin_unlock(&object->lock); + } + } + + if (!cachefiles_cres_file(cres) && want_state != FSCACHE_WANT_PARAMS) { + pr_err("failed to get cres->file\n"); + return false; + } + + return true; +} diff --git a/fs/cachefiles/key.c b/fs/cachefiles/key.c new file mode 100644 index 000000000..aae86af48 --- /dev/null +++ b/fs/cachefiles/key.c @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Key to pathname encoder + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include "internal.h" + +static const char cachefiles_charmap[64] __nonstring = + "0123456789" /* 0 - 9 */ + "abcdefghijklmnopqrstuvwxyz" /* 10 - 35 */ + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" /* 36 - 61 */ + "_-" /* 62 - 63 */ + ; + +static const char cachefiles_filecharmap[256] = { + /* we skip space and tab and control chars */ + [33 ... 46] = 1, /* '!' -> '.' */ + /* we skip '/' as it's significant to pathwalk */ + [48 ... 127] = 1, /* '0' -> '~' */ +}; + +static inline unsigned int how_many_hex_digits(unsigned int x) +{ + return x ? round_up(ilog2(x) + 1, 4) / 4 : 0; +} + +/* + * turn the raw key into something cooked + * - the key may be up to NAME_MAX in length (including the length word) + * - "base64" encode the strange keys, mapping 3 bytes of raw to four of + * cooked + * - need to cut the cooked key into 252 char lengths (189 raw bytes) + */ +bool cachefiles_cook_key(struct cachefiles_object *object) +{ + const u8 *key = fscache_get_key(object->cookie), *kend; + unsigned char ch; + unsigned int acc, i, n, nle, nbe, keylen = object->cookie->key_len; + unsigned int b64len, len, print, pad; + char *name, sep; + + _enter(",%u,%*phN", keylen, keylen, key); + + BUG_ON(keylen > NAME_MAX - 3); + + print = 1; + for (i = 0; i < keylen; i++) { + ch = key[i]; + print &= cachefiles_filecharmap[ch]; + } + + /* If the path is usable ASCII, then we render it directly */ + if (print) { + len = 1 + keylen; + name = kmalloc(len + 1, GFP_KERNEL); + if (!name) + return false; + + name[0] = 'D'; /* Data object type, string encoding */ + memcpy(name + 1, key, keylen); + goto success; + } + + /* See if it makes sense to encode it as "hex,hex,hex" for each 32-bit + * chunk. We rely on the key having been padded out to a whole number + * of 32-bit words. + */ + n = round_up(keylen, 4); + nbe = nle = 0; + for (i = 0; i < n; i += 4) { + u32 be = be32_to_cpu(*(__be32 *)(key + i)); + u32 le = le32_to_cpu(*(__le32 *)(key + i)); + + nbe += 1 + how_many_hex_digits(be); + nle += 1 + how_many_hex_digits(le); + } + + b64len = DIV_ROUND_UP(keylen, 3); + pad = b64len * 3 - keylen; + b64len = 2 + b64len * 4; /* Length if we base64-encode it */ + _debug("len=%u nbe=%u nle=%u b64=%u", keylen, nbe, nle, b64len); + if (nbe < b64len || nle < b64len) { + unsigned int nlen = min(nbe, nle) + 1; + name = kmalloc(nlen, GFP_KERNEL); + if (!name) + return false; + sep = (nbe <= nle) ? 'S' : 'T'; /* Encoding indicator */ + len = 0; + for (i = 0; i < n; i += 4) { + u32 x; + if (nbe <= nle) + x = be32_to_cpu(*(__be32 *)(key + i)); + else + x = le32_to_cpu(*(__le32 *)(key + i)); + name[len++] = sep; + if (x != 0) + len += snprintf(name + len, nlen - len, "%x", x); + sep = ','; + } + goto success; + } + + /* We need to base64-encode it */ + name = kmalloc(b64len + 1, GFP_KERNEL); + if (!name) + return false; + + name[0] = 'E'; + name[1] = '0' + pad; + len = 2; + kend = key + keylen; + do { + acc = *key++; + if (key < kend) { + acc |= *key++ << 8; + if (key < kend) + acc |= *key++ << 16; + } + + name[len++] = cachefiles_charmap[acc & 63]; + acc >>= 6; + name[len++] = cachefiles_charmap[acc & 63]; + acc >>= 6; + name[len++] = cachefiles_charmap[acc & 63]; + acc >>= 6; + name[len++] = cachefiles_charmap[acc & 63]; + } while (key < kend); + +success: + name[len] = 0; + object->d_name = name; + _leave(" = %s", object->d_name); + return true; +} diff --git a/fs/cachefiles/main.c b/fs/cachefiles/main.c new file mode 100644 index 000000000..3f369c6f8 --- /dev/null +++ b/fs/cachefiles/main.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Network filesystem caching backend to use cache files on a premounted + * filesystem + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define CREATE_TRACE_POINTS +#include "internal.h" + +unsigned cachefiles_debug; +module_param_named(debug, cachefiles_debug, uint, S_IWUSR | S_IRUGO); +MODULE_PARM_DESC(cachefiles_debug, "CacheFiles debugging mask"); + +MODULE_DESCRIPTION("Mounted-filesystem based cache"); +MODULE_AUTHOR("Red Hat, Inc."); +MODULE_LICENSE("GPL"); + +struct kmem_cache *cachefiles_object_jar; + +static struct miscdevice cachefiles_dev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "cachefiles", + .fops = &cachefiles_daemon_fops, +}; + +/* + * initialise the fs caching module + */ +static int __init cachefiles_init(void) +{ + int ret; + + ret = cachefiles_register_error_injection(); + if (ret < 0) + goto error_einj; + ret = misc_register(&cachefiles_dev); + if (ret < 0) + goto error_dev; + + /* create an object jar */ + ret = -ENOMEM; + cachefiles_object_jar = + kmem_cache_create("cachefiles_object_jar", + sizeof(struct cachefiles_object), + 0, SLAB_HWCACHE_ALIGN, NULL); + if (!cachefiles_object_jar) { + pr_notice("Failed to allocate an object jar\n"); + goto error_object_jar; + } + + pr_info("Loaded\n"); + return 0; + +error_object_jar: + misc_deregister(&cachefiles_dev); +error_dev: + cachefiles_unregister_error_injection(); +error_einj: + pr_err("failed to register: %d\n", ret); + return ret; +} + +fs_initcall(cachefiles_init); + +/* + * clean up on module removal + */ +static void __exit cachefiles_exit(void) +{ + pr_info("Unloading\n"); + + kmem_cache_destroy(cachefiles_object_jar); + misc_deregister(&cachefiles_dev); + cachefiles_unregister_error_injection(); +} + +module_exit(cachefiles_exit); diff --git a/fs/cachefiles/namei.c b/fs/cachefiles/namei.c new file mode 100644 index 000000000..88955249a --- /dev/null +++ b/fs/cachefiles/namei.c @@ -0,0 +1,824 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* CacheFiles path walking and related routines + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include "internal.h" + +/* + * Mark the backing file as being a cache file if it's not already in use. The + * mark tells the culling request command that it's not allowed to cull the + * file or directory. The caller must hold the inode lock. + */ +static bool __cachefiles_mark_inode_in_use(struct cachefiles_object *object, + struct inode *inode) +{ + bool can_use = false; + + if (!(inode->i_flags & S_KERNEL_FILE)) { + inode->i_flags |= S_KERNEL_FILE; + trace_cachefiles_mark_active(object, inode); + can_use = true; + } else { + trace_cachefiles_mark_failed(object, inode); + } + + return can_use; +} + +static bool cachefiles_mark_inode_in_use(struct cachefiles_object *object, + struct inode *inode) +{ + bool can_use; + + inode_lock(inode); + can_use = __cachefiles_mark_inode_in_use(object, inode); + inode_unlock(inode); + return can_use; +} + +/* + * Unmark a backing inode. The caller must hold the inode lock. + */ +static void __cachefiles_unmark_inode_in_use(struct cachefiles_object *object, + struct inode *inode) +{ + inode->i_flags &= ~S_KERNEL_FILE; + trace_cachefiles_mark_inactive(object, inode); +} + +static void cachefiles_do_unmark_inode_in_use(struct cachefiles_object *object, + struct inode *inode) +{ + inode_lock(inode); + __cachefiles_unmark_inode_in_use(object, inode); + inode_unlock(inode); +} + +/* + * Unmark a backing inode and tell cachefilesd that there's something that can + * be culled. + */ +void cachefiles_unmark_inode_in_use(struct cachefiles_object *object, + struct file *file) +{ + struct cachefiles_cache *cache = object->volume->cache; + struct inode *inode = file_inode(file); + + cachefiles_do_unmark_inode_in_use(object, inode); + + if (!test_bit(CACHEFILES_OBJECT_USING_TMPFILE, &object->flags)) { + atomic_long_add(inode->i_blocks, &cache->b_released); + if (atomic_inc_return(&cache->f_released)) + cachefiles_state_changed(cache); + } +} + +/* + * get a subdirectory + */ +struct dentry *cachefiles_get_directory(struct cachefiles_cache *cache, + struct dentry *dir, + const char *dirname, + bool *_is_new) +{ + struct dentry *subdir; + struct path path; + int ret; + + _enter(",,%s", dirname); + + /* search the current directory for the element name */ + +retry: + ret = cachefiles_inject_read_error(); + if (ret == 0) + subdir = start_creating(&nop_mnt_idmap, dir, &QSTR(dirname)); + else + subdir = ERR_PTR(ret); + trace_cachefiles_lookup(NULL, dir, subdir); + if (IS_ERR(subdir)) { + trace_cachefiles_vfs_error(NULL, d_backing_inode(dir), + PTR_ERR(subdir), + cachefiles_trace_lookup_error); + if (PTR_ERR(subdir) == -ENOMEM) + goto nomem_d_alloc; + goto lookup_error; + } + + _debug("subdir -> %pd %s", + subdir, d_backing_inode(subdir) ? "positive" : "negative"); + + /* we need to create the subdir if it doesn't exist yet */ + if (d_is_negative(subdir)) { + ret = cachefiles_has_space(cache, 1, 0, + cachefiles_has_space_for_create); + if (ret < 0) + goto mkdir_error; + + _debug("attempt mkdir"); + + path.mnt = cache->mnt; + path.dentry = dir; + ret = security_path_mkdir(&path, subdir, 0700); + if (ret < 0) + goto mkdir_error; + ret = cachefiles_inject_write_error(); + if (ret == 0) { + subdir = vfs_mkdir(&nop_mnt_idmap, d_inode(dir), subdir, 0700, NULL); + if (IS_ERR(subdir)) + ret = PTR_ERR(subdir); + } else { + end_creating(subdir); + subdir = ERR_PTR(ret); + } + if (IS_ERR(subdir)) { + trace_cachefiles_vfs_error(NULL, d_inode(dir), ret, + cachefiles_trace_mkdir_error); + goto mkdir_error; + } + trace_cachefiles_mkdir(dir, subdir); + + if (unlikely(d_unhashed(subdir) || d_is_negative(subdir))) { + end_creating(subdir); + goto retry; + } + ASSERT(d_backing_inode(subdir)); + + _debug("mkdir -> %pd{ino=%llu}", + subdir, d_backing_inode(subdir)->i_ino); + if (_is_new) + *_is_new = true; + } + + /* Tell rmdir() it's not allowed to delete the subdir */ + inode_lock(d_inode(subdir)); + end_creating_keep(subdir); + + if (!__cachefiles_mark_inode_in_use(NULL, d_inode(subdir))) { + pr_notice("cachefiles: Inode already in use: %pd (B=%llx)\n", + subdir, d_inode(subdir)->i_ino); + goto mark_error; + } + + inode_unlock(d_inode(subdir)); + + /* we need to make sure the subdir is a directory */ + ASSERT(d_backing_inode(subdir)); + + if (!d_can_lookup(subdir)) { + pr_err("%s is not a directory\n", dirname); + ret = -EIO; + goto check_error; + } + + ret = -EPERM; + if (!(d_backing_inode(subdir)->i_opflags & IOP_XATTR) || + !d_backing_inode(subdir)->i_op->lookup || + !d_backing_inode(subdir)->i_op->mkdir || + !d_backing_inode(subdir)->i_op->rename || + !d_backing_inode(subdir)->i_op->rmdir || + !d_backing_inode(subdir)->i_op->unlink) + goto check_error; + + _leave(" = [%llu]", d_backing_inode(subdir)->i_ino); + return subdir; + +check_error: + cachefiles_put_directory(subdir); + _leave(" = %d [check]", ret); + return ERR_PTR(ret); + +mark_error: + inode_unlock(d_inode(subdir)); + dput(subdir); + return ERR_PTR(-EBUSY); + +mkdir_error: + end_creating(subdir); + pr_err("mkdir %s failed with error %d\n", dirname, ret); + return ERR_PTR(ret); + +lookup_error: + ret = PTR_ERR(subdir); + pr_err("Lookup %s failed with error %d\n", dirname, ret); + return ERR_PTR(ret); + +nomem_d_alloc: + _leave(" = -ENOMEM"); + return ERR_PTR(-ENOMEM); +} + +/* + * Put a subdirectory. + */ +void cachefiles_put_directory(struct dentry *dir) +{ + if (dir) { + cachefiles_do_unmark_inode_in_use(NULL, d_inode(dir)); + dput(dir); + } +} + +/* + * Remove a regular file from the cache. + */ +static int cachefiles_unlink(struct cachefiles_cache *cache, + struct cachefiles_object *object, + struct dentry *dir, struct dentry *dentry, + enum fscache_why_object_killed why) +{ + struct path path = { + .mnt = cache->mnt, + .dentry = dir, + }; + int ret; + + trace_cachefiles_unlink(object, d_inode(dentry)->i_ino, why); + ret = security_path_unlink(&path, dentry); + if (ret < 0) { + cachefiles_io_error(cache, "Unlink security error"); + return ret; + } + + ret = cachefiles_inject_remove_error(); + if (ret == 0) { + ret = vfs_unlink(&nop_mnt_idmap, d_backing_inode(dir), dentry, NULL); + if (ret == -EIO) + cachefiles_io_error(cache, "Unlink failed"); + } + if (ret != 0) + trace_cachefiles_vfs_error(object, d_backing_inode(dir), ret, + cachefiles_trace_unlink_error); + return ret; +} + +/* + * Delete an object representation from the cache + * - File backed objects are unlinked + * - Directory backed objects are stuffed into the graveyard for userspace to + * delete + * On entry dir must be locked. It will be unlocked on exit. + * On entry there must be at least 2 refs on rep, one will be dropped on exit. + */ +int cachefiles_bury_object(struct cachefiles_cache *cache, + struct cachefiles_object *object, + struct dentry *dir, + struct dentry *rep, + enum fscache_why_object_killed why) +{ + struct dentry *grave; + struct renamedata rd = {}; + struct path path, path_to_graveyard; + char nbuffer[8 + 8 + 1]; + int ret; + + _enter(",'%pd','%pd'", dir, rep); + + if (rep->d_parent != dir) { + end_removing(rep); + _leave(" = -ESTALE"); + return -ESTALE; + } + + /* non-directories can just be unlinked */ + if (!d_is_dir(rep)) { + ret = cachefiles_unlink(cache, object, dir, rep, why); + end_removing(rep); + + _leave(" = %d", ret); + return ret; + } + + /* directories have to be moved to the graveyard */ + _debug("move stale object to graveyard"); + end_removing(rep); + +try_again: + /* first step is to make up a grave dentry in the graveyard */ + sprintf(nbuffer, "%08x%08x", + (uint32_t) ktime_get_real_seconds(), + (uint32_t) atomic_inc_return(&cache->gravecounter)); + + rd.mnt_idmap = &nop_mnt_idmap; + rd.old_parent = dir; + rd.new_parent = cache->graveyard; + rd.flags = 0; + ret = start_renaming_dentry(&rd, 0, rep, &QSTR(nbuffer)); + if (ret) { + /* Some errors aren't fatal */ + if (ret == -EXDEV) + /* double-lock failed */ + return ret; + if (d_unhashed(rep) || rep->d_parent != dir || IS_DEADDIR(d_inode(rep))) { + /* the entry was probably culled when we dropped the parent dir + * lock */ + _leave(" = 0 [culled?]"); + return 0; + } + if (ret == -EINVAL || ret == -ENOTEMPTY) { + cachefiles_io_error(cache, "May not make directory loop"); + return -EIO; + } + if (ret == -ENOMEM) { + _leave(" = -ENOMEM"); + return -ENOMEM; + } + + cachefiles_io_error(cache, "Lookup error %d", ret); + return -EIO; + } + + if (d_mountpoint(rep)) { + end_renaming(&rd); + cachefiles_io_error(cache, "Mountpoint in cache"); + return -EIO; + } + + grave = rd.new_dentry; + if (d_is_positive(grave)) { + end_renaming(&rd); + grave = NULL; + cond_resched(); + goto try_again; + } + + if (d_mountpoint(grave)) { + end_renaming(&rd); + cachefiles_io_error(cache, "Mountpoint in graveyard"); + return -EIO; + } + + /* attempt the rename */ + path.mnt = cache->mnt; + path.dentry = dir; + path_to_graveyard.mnt = cache->mnt; + path_to_graveyard.dentry = cache->graveyard; + ret = security_path_rename(&path, rep, &path_to_graveyard, grave, 0); + if (ret < 0) { + cachefiles_io_error(cache, "Rename security error %d", ret); + } else { + trace_cachefiles_rename(object, d_inode(rep)->i_ino, why); + ret = cachefiles_inject_read_error(); + if (ret == 0) + ret = vfs_rename(&rd); + if (ret != 0) + trace_cachefiles_vfs_error(object, d_inode(dir), ret, + cachefiles_trace_rename_error); + if (ret != 0 && ret != -ENOMEM) + cachefiles_io_error(cache, + "Rename failed with error %d", ret); + } + + cachefiles_do_unmark_inode_in_use(object, d_inode(rep)); + end_renaming(&rd); + _leave(" = 0"); + return 0; +} + +/* + * Delete a cache file. + */ +int cachefiles_delete_object(struct cachefiles_object *object, + enum fscache_why_object_killed why) +{ + struct cachefiles_volume *volume = object->volume; + struct dentry *dentry = object->file->f_path.dentry; + struct dentry *fan = volume->fanout[(u8)object->cookie->key_hash]; + int ret; + + _enter(",OBJ%x{%pD}", object->debug_id, object->file); + + dentry = start_removing_dentry(fan, dentry); + if (IS_ERR(dentry)) + ret = PTR_ERR(dentry); + else + ret = cachefiles_unlink(volume->cache, object, fan, dentry, why); + end_removing(dentry); + return ret; +} + +/* + * Create a temporary file and leave it unattached and un-xattr'd until the + * time comes to discard the object from memory. + */ +struct file *cachefiles_create_tmpfile(struct cachefiles_object *object) +{ + struct cachefiles_volume *volume = object->volume; + struct cachefiles_cache *cache = volume->cache; + const struct cred *saved_cred; + struct dentry *fan = volume->fanout[(u8)object->cookie->key_hash]; + struct file *file; + const struct path parentpath = { .mnt = cache->mnt, .dentry = fan }; + uint64_t ni_size; + long ret; + + + cachefiles_begin_secure(cache, &saved_cred); + + ret = cachefiles_inject_write_error(); + if (ret == 0) { + file = kernel_tmpfile_open(&nop_mnt_idmap, &parentpath, + S_IFREG | 0600, + O_RDWR | O_LARGEFILE | O_DIRECT, + cache->cache_cred); + ret = PTR_ERR_OR_ZERO(file); + } + if (ret) { + trace_cachefiles_vfs_error(object, d_inode(fan), ret, + cachefiles_trace_tmpfile_error); + if (ret == -EIO) + cachefiles_io_error_obj(object, "Failed to create tmpfile"); + goto err; + } + + trace_cachefiles_tmpfile(object, file_inode(file)); + + /* This is a newly created file with no other possible user */ + if (!cachefiles_mark_inode_in_use(object, file_inode(file))) + WARN_ON(1); + + ni_size = object->cookie->object_size; + ni_size = round_up(ni_size, CACHEFILES_DIO_BLOCK_SIZE); + + if (ni_size > 0) { + trace_cachefiles_trunc(object, file_inode(file), 0, ni_size, + cachefiles_trunc_expand_tmpfile); + ret = cachefiles_inject_write_error(); + if (ret == 0) + ret = vfs_truncate(&file->f_path, ni_size); + if (ret < 0) { + trace_cachefiles_vfs_error( + object, file_inode(file), ret, + cachefiles_trace_trunc_error); + goto err_unuse; + } + } + + ret = -EINVAL; + if (unlikely(!file->f_op->read_iter) || + unlikely(!file->f_op->write_iter)) { + pr_notice("Cache does not support read_iter and write_iter\n"); + goto err_unuse; + } +out: + cachefiles_end_secure(cache, saved_cred); + return file; + +err_unuse: + cachefiles_do_unmark_inode_in_use(object, file_inode(file)); + fput(file); +err: + file = ERR_PTR(ret); + goto out; +} + +/* + * Create a new file. + */ +static bool cachefiles_create_file(struct cachefiles_object *object) +{ + struct file *file; + int ret; + + ret = cachefiles_has_space(object->volume->cache, 1, 0, + cachefiles_has_space_for_create); + if (ret < 0) + return false; + + file = cachefiles_create_tmpfile(object); + if (IS_ERR(file)) + return false; + + set_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &object->cookie->flags); + set_bit(CACHEFILES_OBJECT_USING_TMPFILE, &object->flags); + _debug("create -> %pD{ino=%llu}", file, file_inode(file)->i_ino); + object->file = file; + return true; +} + +/* + * Open an existing file, checking its attributes and replacing it if it is + * stale. + */ +static bool cachefiles_open_file(struct cachefiles_object *object, + struct dentry *dentry) +{ + struct cachefiles_cache *cache = object->volume->cache; + struct file *file; + struct path path; + int ret; + + _enter("%pd", dentry); + + if (!cachefiles_mark_inode_in_use(object, d_inode(dentry))) { + pr_notice("cachefiles: Inode already in use: %pd (B=%llx)\n", + dentry, d_inode(dentry)->i_ino); + return false; + } + + /* We need to open a file interface onto a data file now as we can't do + * it on demand because writeback called from do_exit() sees + * current->fs == NULL - which breaks d_path() called from ext4 open. + */ + path.mnt = cache->mnt; + path.dentry = dentry; + file = kernel_file_open(&path, O_RDWR | O_LARGEFILE | O_DIRECT, cache->cache_cred); + if (IS_ERR(file)) { + trace_cachefiles_vfs_error(object, d_backing_inode(dentry), + PTR_ERR(file), + cachefiles_trace_open_error); + goto error; + } + + if (unlikely(!file->f_op->read_iter) || + unlikely(!file->f_op->write_iter)) { + pr_notice("Cache does not support read_iter and write_iter\n"); + goto error_fput; + } + _debug("file -> %pd positive", dentry); + + ret = cachefiles_check_auxdata(object, file); + if (ret < 0) + goto check_failed; + + clear_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &object->cookie->flags); + + object->file = file; + + /* Always update the atime on an object we've just looked up (this is + * used to keep track of culling, and atimes are only updated by read, + * write and readdir but not lookup or open). + */ + touch_atime(&file->f_path); + return true; + +check_failed: + fscache_cookie_lookup_negative(object->cookie); + cachefiles_unmark_inode_in_use(object, file); + fput(file); + if (ret == -ESTALE) + return cachefiles_create_file(object); + return false; + +error_fput: + fput(file); +error: + cachefiles_do_unmark_inode_in_use(object, d_inode(dentry)); + return false; +} + +/* + * walk from the parent object to the child object through the backing + * filesystem, creating directories as we go + */ +bool cachefiles_look_up_object(struct cachefiles_object *object) +{ + struct cachefiles_volume *volume = object->volume; + struct dentry *dentry, *fan = volume->fanout[(u8)object->cookie->key_hash]; + int ret; + + _enter("OBJ%x,%s,", object->debug_id, object->d_name); + + /* Look up path "cache/vol/fanout/file". */ + ret = cachefiles_inject_read_error(); + if (ret == 0) + dentry = lookup_one_positive_unlocked(&nop_mnt_idmap, + &QSTR(object->d_name), fan); + else + dentry = ERR_PTR(ret); + trace_cachefiles_lookup(object, fan, dentry); + if (IS_ERR(dentry)) { + if (dentry == ERR_PTR(-ENOENT)) + goto new_file; + if (dentry == ERR_PTR(-EIO)) + cachefiles_io_error_obj(object, "Lookup failed"); + return false; + } + + if (!d_is_reg(dentry)) { + pr_err("%pd is not a file\n", dentry); + struct dentry *de = start_removing_dentry(fan, dentry); + if (IS_ERR(de)) + ret = PTR_ERR(de); + else + ret = cachefiles_bury_object(volume->cache, object, + fan, de, + FSCACHE_OBJECT_IS_WEIRD); + dput(dentry); + if (ret < 0) + return false; + goto new_file; + } + + ret = cachefiles_open_file(object, dentry); + dput(dentry); + if (!ret) + return false; + + _leave(" = t [%llu]", file_inode(object->file)->i_ino); + return true; + +new_file: + fscache_cookie_lookup_negative(object->cookie); + return cachefiles_create_file(object); +} + +/* + * Attempt to link a temporary file into its rightful place in the cache. + */ +bool cachefiles_commit_tmpfile(struct cachefiles_cache *cache, + struct cachefiles_object *object) +{ + struct cachefiles_volume *volume = object->volume; + struct dentry *dentry, *fan = volume->fanout[(u8)object->cookie->key_hash]; + bool success = false; + int ret; + + _enter(",%pD", object->file); + + ret = cachefiles_inject_read_error(); + if (ret == 0) + dentry = start_creating(&nop_mnt_idmap, fan, &QSTR(object->d_name)); + else + dentry = ERR_PTR(ret); + if (IS_ERR(dentry)) { + trace_cachefiles_vfs_error(object, d_inode(fan), PTR_ERR(dentry), + cachefiles_trace_lookup_error); + _debug("lookup fail %ld", PTR_ERR(dentry)); + goto out; + } + + /* + * This loop will only execute more than once if some other thread + * races to create the object we are trying to create. + */ + while (!d_is_negative(dentry)) { + ret = cachefiles_unlink(volume->cache, object, fan, dentry, + FSCACHE_OBJECT_IS_STALE); + if (ret < 0) + goto out_end; + + end_creating(dentry); + + ret = cachefiles_inject_read_error(); + if (ret == 0) + dentry = start_creating(&nop_mnt_idmap, fan, + &QSTR(object->d_name)); + else + dentry = ERR_PTR(ret); + if (IS_ERR(dentry)) { + trace_cachefiles_vfs_error(object, d_inode(fan), PTR_ERR(dentry), + cachefiles_trace_lookup_error); + _debug("lookup fail %ld", PTR_ERR(dentry)); + goto out; + } + } + + ret = cachefiles_inject_read_error(); + if (ret == 0) + ret = vfs_link(object->file->f_path.dentry, &nop_mnt_idmap, + d_inode(fan), dentry, NULL); + if (ret < 0) { + trace_cachefiles_vfs_error(object, d_inode(fan), ret, + cachefiles_trace_link_error); + _debug("link fail %d", ret); + } else { + trace_cachefiles_link(object, file_inode(object->file)); + spin_lock(&object->lock); + /* TODO: Do we want to switch the file pointer to the new dentry? */ + clear_bit(CACHEFILES_OBJECT_USING_TMPFILE, &object->flags); + spin_unlock(&object->lock); + success = true; + } + +out_end: + end_creating(dentry); +out: + _leave(" = %u", success); + return success; +} + +/* + * Look up an inode to be checked or culled. Return -EBUSY if the inode is + * marked in use. + */ +static struct dentry *cachefiles_lookup_for_cull(struct cachefiles_cache *cache, + struct dentry *dir, + char *filename) +{ + struct dentry *victim; + int ret = -ENOENT; + + victim = start_removing(&nop_mnt_idmap, dir, &QSTR(filename)); + + if (IS_ERR(victim)) + goto lookup_error; + if (d_inode(victim)->i_flags & S_KERNEL_FILE) + goto lookup_busy; + return victim; + +lookup_busy: + ret = -EBUSY; + end_removing(victim); + return ERR_PTR(ret); + +lookup_error: + ret = PTR_ERR(victim); + if (ret == -ENOENT) + return ERR_PTR(-ESTALE); /* Probably got retired by the netfs */ + + if (ret == -EIO) { + cachefiles_io_error(cache, "Lookup failed"); + } else if (ret != -ENOMEM) { + pr_err("Internal error: %d\n", ret); + ret = -EIO; + } + + return ERR_PTR(ret); +} + +/* + * Cull an object if it's not in use + * - called only by cache manager daemon + */ +int cachefiles_cull(struct cachefiles_cache *cache, struct dentry *dir, + char *filename) +{ + struct dentry *victim; + struct inode *inode; + int ret; + + _enter(",%pd/,%s", dir, filename); + + victim = cachefiles_lookup_for_cull(cache, dir, filename); + if (IS_ERR(victim)) + return PTR_ERR(victim); + + /* check to see if someone is using this object */ + inode = d_inode(victim); + inode_lock(inode); + if (inode->i_flags & S_KERNEL_FILE) { + ret = -EBUSY; + } else { + /* Stop the cache from picking it back up */ + inode->i_flags |= S_KERNEL_FILE; + ret = 0; + } + inode_unlock(inode); + if (ret < 0) + goto error_unlock; + + /* + * cachefiles_bury_object() expects 2 references to 'victim', + * and drops one. + */ + dget(victim); + ret = cachefiles_bury_object(cache, NULL, dir, victim, + FSCACHE_OBJECT_WAS_CULLED); + dput(victim); + if (ret < 0) + goto error; + + fscache_count_culled(); + _leave(" = 0"); + return 0; + +error_unlock: + end_removing(victim); +error: + if (ret == -ENOENT) + return -ESTALE; /* Probably got retired by the netfs */ + + if (ret != -ENOMEM) { + pr_err("Internal error: %d\n", ret); + ret = -EIO; + } + + _leave(" = %d", ret); + return ret; +} + +/* + * Find out if an object is in use or not + * - called only by cache manager daemon + * - returns -EBUSY or 0 to indicate whether an object is in use or not + */ +int cachefiles_check_in_use(struct cachefiles_cache *cache, struct dentry *dir, + char *filename) +{ + struct dentry *victim; + int ret = 0; + + victim = cachefiles_lookup_for_cull(cache, dir, filename); + if (IS_ERR(victim)) + return PTR_ERR(victim); + + inode_unlock(d_inode(dir)); + dput(victim); + return ret; +} diff --git a/fs/cachefiles/security.c b/fs/cachefiles/security.c new file mode 100644 index 000000000..fc6611886 --- /dev/null +++ b/fs/cachefiles/security.c @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* CacheFiles security management + * + * Copyright (C) 2007, 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include "internal.h" + +/* + * determine the security context within which we access the cache from within + * the kernel + */ +int cachefiles_get_security_ID(struct cachefiles_cache *cache) +{ + struct cred *new; + int ret; + + _enter("{%u}", cache->have_secid ? cache->secid : 0); + + new = prepare_kernel_cred(current); + if (!new) { + ret = -ENOMEM; + goto error; + } + + if (cache->have_secid) { + ret = set_security_override(new, cache->secid); + if (ret < 0) { + put_cred(new); + pr_err("Security denies permission to nominate security context: error %d\n", + ret); + goto error; + } + } + + cache->cache_cred = new; + ret = 0; +error: + _leave(" = %d", ret); + return ret; +} + +/* + * see if mkdir and create can be performed in the root directory + */ +static int cachefiles_check_cache_dir(struct cachefiles_cache *cache, + struct dentry *root) +{ + int ret; + + ret = security_inode_mkdir(d_backing_inode(root), root, 0); + if (ret < 0) { + pr_err("Security denies permission to make dirs: error %d", + ret); + return ret; + } + + ret = security_inode_create(d_backing_inode(root), root, 0); + if (ret < 0) + pr_err("Security denies permission to create files: error %d", + ret); + + return ret; +} + +/* + * check the security details of the on-disk cache + * - must be called with security override in force + * - must return with a security override in force - even in the case of an + * error + */ +int cachefiles_determine_cache_security(struct cachefiles_cache *cache, + struct dentry *root, + const struct cred **_saved_cred) +{ + struct cred *new; + int ret; + + _enter(""); + + /* duplicate the cache creds for COW (the override is currently in + * force, so we can use prepare_creds() to do this) */ + new = prepare_creds(); + if (!new) + return -ENOMEM; + + cachefiles_end_secure(cache, *_saved_cred); + + /* use the cache root dir's security context as the basis with + * which create files */ + ret = set_create_files_as(new, d_backing_inode(root)); + if (ret < 0) { + abort_creds(new); + cachefiles_begin_secure(cache, _saved_cred); + _leave(" = %d [cfa]", ret); + return ret; + } + + put_cred(cache->cache_cred); + cache->cache_cred = new; + + cachefiles_begin_secure(cache, _saved_cred); + ret = cachefiles_check_cache_dir(cache, root); + + if (ret == -EOPNOTSUPP) + ret = 0; + _leave(" = %d", ret); + return ret; +} diff --git a/fs/cachefiles/volume.c b/fs/cachefiles/volume.c new file mode 100644 index 000000000..6c0da6136 --- /dev/null +++ b/fs/cachefiles/volume.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Volume handling. + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include "internal.h" +#include + +/* + * Allocate and set up a volume representation. We make sure all the fanout + * directories are created and pinned. + */ +void cachefiles_acquire_volume(struct fscache_volume *vcookie) +{ + struct cachefiles_volume *volume; + struct cachefiles_cache *cache = vcookie->cache->cache_priv; + const struct cred *saved_cred; + struct dentry *vdentry, *fan; + size_t len; + char *name; + bool is_new = false; + int ret, n_accesses, i; + + _enter(""); + + volume = kzalloc_obj(struct cachefiles_volume); + if (!volume) + return; + volume->vcookie = vcookie; + volume->cache = cache; + INIT_LIST_HEAD(&volume->cache_link); + + cachefiles_begin_secure(cache, &saved_cred); + + len = vcookie->key[0]; + name = kmalloc(len + 3, GFP_NOFS); + if (!name) + goto error_vol; + name[0] = 'I'; + memcpy(name + 1, vcookie->key + 1, len); + name[len + 1] = 0; + +retry: + vdentry = cachefiles_get_directory(cache, cache->store, name, &is_new); + if (IS_ERR(vdentry)) + goto error_name; + volume->dentry = vdentry; + + if (is_new) { + if (!cachefiles_set_volume_xattr(volume)) + goto error_dir; + } else { + ret = cachefiles_check_volume_xattr(volume); + if (ret < 0) { + if (ret != -ESTALE) + goto error_dir; + vdentry = start_removing_dentry(cache->store, vdentry); + if (!IS_ERR(vdentry)) + cachefiles_bury_object(cache, NULL, cache->store, + vdentry, + FSCACHE_VOLUME_IS_WEIRD); + cachefiles_put_directory(volume->dentry); + cond_resched(); + goto retry; + } + } + + for (i = 0; i < 256; i++) { + sprintf(name, "@%02x", i); + fan = cachefiles_get_directory(cache, vdentry, name, NULL); + if (IS_ERR(fan)) + goto error_fan; + volume->fanout[i] = fan; + } + + cachefiles_end_secure(cache, saved_cred); + + vcookie->cache_priv = volume; + n_accesses = atomic_inc_return(&vcookie->n_accesses); /* Stop wakeups on dec-to-0 */ + trace_fscache_access_volume(vcookie->debug_id, 0, + refcount_read(&vcookie->ref), + n_accesses, fscache_access_cache_pin); + + spin_lock(&cache->object_list_lock); + list_add(&volume->cache_link, &volume->cache->volumes); + spin_unlock(&cache->object_list_lock); + + kfree(name); + return; + +error_fan: + for (i = 0; i < 256; i++) + cachefiles_put_directory(volume->fanout[i]); +error_dir: + cachefiles_put_directory(volume->dentry); +error_name: + kfree(name); +error_vol: + kfree(volume); + cachefiles_end_secure(cache, saved_cred); +} + +/* + * Release a volume representation. + */ +static void __cachefiles_free_volume(struct cachefiles_volume *volume) +{ + int i; + + _enter(""); + + volume->vcookie->cache_priv = NULL; + + for (i = 0; i < 256; i++) + cachefiles_put_directory(volume->fanout[i]); + cachefiles_put_directory(volume->dentry); + kfree(volume); +} + +void cachefiles_free_volume(struct fscache_volume *vcookie) +{ + struct cachefiles_volume *volume = vcookie->cache_priv; + + if (volume) { + spin_lock(&volume->cache->object_list_lock); + list_del_init(&volume->cache_link); + spin_unlock(&volume->cache->object_list_lock); + __cachefiles_free_volume(volume); + } +} + +void cachefiles_withdraw_volume(struct cachefiles_volume *volume) +{ + cachefiles_set_volume_xattr(volume); + __cachefiles_free_volume(volume); +} diff --git a/fs/cachefiles/xattr.c b/fs/cachefiles/xattr.c new file mode 100644 index 000000000..f8ae78b3f --- /dev/null +++ b/fs/cachefiles/xattr.c @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* CacheFiles extended attribute management + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +#define CACHEFILES_COOKIE_TYPE_DATA 1 + +struct cachefiles_xattr { + __be64 object_size; /* Actual size of the object */ + __be64 zero_point; /* Size after which server has no data not written by us */ + __u8 type; /* Type of object */ + __u8 content; /* Content presence (enum cachefiles_content) */ + __u8 data[]; /* netfs coherency data */ +} __packed; + +static const char cachefiles_xattr_cache[] = + XATTR_USER_PREFIX "CacheFiles.cache"; + +struct cachefiles_vol_xattr { + __be32 reserved; /* Reserved, should be 0 */ + __u8 data[]; /* netfs volume coherency data */ +} __packed; + +/* + * set the state xattr on a cache file + */ +int cachefiles_set_object_xattr(struct cachefiles_object *object) +{ + struct cachefiles_xattr *buf; + struct dentry *dentry; + struct file *file = object->file; + unsigned int len = object->cookie->aux_len; + int ret; + + if (!file) + return -ESTALE; + dentry = file->f_path.dentry; + + _enter("%x,#%d", object->debug_id, len); + + buf = kmalloc(sizeof(struct cachefiles_xattr) + len, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + buf->object_size = cpu_to_be64(object->cookie->object_size); + buf->zero_point = 0; + buf->type = CACHEFILES_COOKIE_TYPE_DATA; + buf->content = object->content_info; + if (test_bit(FSCACHE_COOKIE_LOCAL_WRITE, &object->cookie->flags)) + buf->content = CACHEFILES_CONTENT_DIRTY; + if (len > 0) + memcpy(buf->data, fscache_get_aux(object->cookie), len); + + ret = cachefiles_inject_write_error(); + if (ret == 0) { + ret = mnt_want_write_file(file); + if (ret == 0) { + ret = vfs_setxattr(&nop_mnt_idmap, dentry, + cachefiles_xattr_cache, buf, + sizeof(struct cachefiles_xattr) + len, 0); + mnt_drop_write_file(file); + } + } + if (ret < 0) { + trace_cachefiles_vfs_error(object, file_inode(file), ret, + cachefiles_trace_setxattr_error); + trace_cachefiles_coherency(object, file_inode(file)->i_ino, + be64_to_cpup((__be64 *)buf->data), + buf->content, + cachefiles_coherency_set_fail); + if (ret != -ENOMEM) + cachefiles_io_error_obj( + object, + "Failed to set xattr with error %d", ret); + } else { + trace_cachefiles_coherency(object, file_inode(file)->i_ino, + be64_to_cpup((__be64 *)buf->data), + buf->content, + cachefiles_coherency_set_ok); + } + + kfree(buf); + _leave(" = %d", ret); + return ret; +} + +/* + * check the consistency between the backing cache and the FS-Cache cookie + */ +int cachefiles_check_auxdata(struct cachefiles_object *object, struct file *file) +{ + struct cachefiles_xattr *buf; + struct dentry *dentry = file->f_path.dentry; + unsigned int len = object->cookie->aux_len, tlen; + const void *p = fscache_get_aux(object->cookie); + enum cachefiles_coherency_trace why; + ssize_t xlen; + int ret = -ESTALE; + + tlen = sizeof(struct cachefiles_xattr) + len; + buf = kmalloc(tlen, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + xlen = cachefiles_inject_read_error(); + if (xlen == 0) + xlen = vfs_getxattr(&nop_mnt_idmap, dentry, cachefiles_xattr_cache, buf, tlen); + if (xlen != tlen) { + if (xlen < 0) { + ret = xlen; + trace_cachefiles_vfs_error(object, file_inode(file), xlen, + cachefiles_trace_getxattr_error); + } + if (xlen == -EIO) + cachefiles_io_error_obj( + object, + "Failed to read aux with error %zd", xlen); + why = cachefiles_coherency_check_xattr; + goto out; + } + + if (buf->type != CACHEFILES_COOKIE_TYPE_DATA) { + why = cachefiles_coherency_check_type; + } else if (memcmp(buf->data, p, len) != 0) { + why = cachefiles_coherency_check_aux; + } else if (be64_to_cpu(buf->object_size) != object->cookie->object_size) { + why = cachefiles_coherency_check_objsize; + } else if (buf->content == CACHEFILES_CONTENT_DIRTY) { + // TODO: Begin conflict resolution + pr_warn("Dirty object in cache\n"); + why = cachefiles_coherency_check_dirty; + } else { + why = cachefiles_coherency_check_ok; + ret = 0; + } + +out: + trace_cachefiles_coherency(object, file_inode(file)->i_ino, + be64_to_cpup((__be64 *)buf->data), + buf->content, why); + kfree(buf); + return ret; +} + +/* + * remove the object's xattr to mark it stale + */ +int cachefiles_remove_object_xattr(struct cachefiles_cache *cache, + struct cachefiles_object *object, + struct dentry *dentry) +{ + int ret; + + ret = cachefiles_inject_remove_error(); + if (ret == 0) { + ret = mnt_want_write(cache->mnt); + if (ret == 0) { + ret = vfs_removexattr(&nop_mnt_idmap, dentry, + cachefiles_xattr_cache); + mnt_drop_write(cache->mnt); + } + } + if (ret < 0) { + trace_cachefiles_vfs_error(object, d_inode(dentry), ret, + cachefiles_trace_remxattr_error); + if (ret == -ENOENT || ret == -ENODATA) + ret = 0; + else if (ret != -ENOMEM) + cachefiles_io_error(cache, + "Can't remove xattr from %llu" + " (error %d)", + d_backing_inode(dentry)->i_ino, -ret); + } + + _leave(" = %d", ret); + return ret; +} + +/* + * Stick a marker on the cache object to indicate that it's dirty. + */ +void cachefiles_prepare_to_write(struct fscache_cookie *cookie) +{ + const struct cred *saved_cred; + struct cachefiles_object *object = cookie->cache_priv; + struct cachefiles_cache *cache = object->volume->cache; + + _enter("c=%08x", object->cookie->debug_id); + + if (!test_bit(CACHEFILES_OBJECT_USING_TMPFILE, &object->flags)) { + cachefiles_begin_secure(cache, &saved_cred); + cachefiles_set_object_xattr(object); + cachefiles_end_secure(cache, saved_cred); + } +} + +/* + * Set the state xattr on a volume directory. + */ +bool cachefiles_set_volume_xattr(struct cachefiles_volume *volume) +{ + struct cachefiles_vol_xattr *buf; + unsigned int len = volume->vcookie->coherency_len; + const void *p = volume->vcookie->coherency; + struct dentry *dentry = volume->dentry; + int ret; + + _enter("%x,#%d", volume->vcookie->debug_id, len); + + len += sizeof(*buf); + buf = kmalloc(len, GFP_KERNEL); + if (!buf) + return false; + buf->reserved = cpu_to_be32(0); + memcpy(buf->data, p, volume->vcookie->coherency_len); + + ret = cachefiles_inject_write_error(); + if (ret == 0) { + ret = mnt_want_write(volume->cache->mnt); + if (ret == 0) { + ret = vfs_setxattr(&nop_mnt_idmap, dentry, + cachefiles_xattr_cache, + buf, len, 0); + mnt_drop_write(volume->cache->mnt); + } + } + if (ret < 0) { + trace_cachefiles_vfs_error(NULL, d_inode(dentry), ret, + cachefiles_trace_setxattr_error); + trace_cachefiles_vol_coherency(volume, d_inode(dentry)->i_ino, + cachefiles_coherency_vol_set_fail); + if (ret != -ENOMEM) + cachefiles_io_error( + volume->cache, "Failed to set xattr with error %d", ret); + } else { + trace_cachefiles_vol_coherency(volume, d_inode(dentry)->i_ino, + cachefiles_coherency_vol_set_ok); + } + + kfree(buf); + _leave(" = %d", ret); + return ret == 0; +} + +/* + * Check the consistency between the backing cache and the volume cookie. + */ +int cachefiles_check_volume_xattr(struct cachefiles_volume *volume) +{ + struct cachefiles_vol_xattr *buf; + struct dentry *dentry = volume->dentry; + unsigned int len = volume->vcookie->coherency_len; + const void *p = volume->vcookie->coherency; + enum cachefiles_coherency_trace why; + ssize_t xlen; + int ret = -ESTALE; + + _enter(""); + + len += sizeof(*buf); + buf = kmalloc(len, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + xlen = cachefiles_inject_read_error(); + if (xlen == 0) + xlen = vfs_getxattr(&nop_mnt_idmap, dentry, cachefiles_xattr_cache, buf, len); + if (xlen != len) { + if (xlen < 0) { + ret = xlen; + trace_cachefiles_vfs_error(NULL, d_inode(dentry), xlen, + cachefiles_trace_getxattr_error); + if (xlen == -EIO) + cachefiles_io_error( + volume->cache, + "Failed to read xattr with error %zd", xlen); + } + why = cachefiles_coherency_vol_check_xattr; + } else if (buf->reserved != cpu_to_be32(0)) { + why = cachefiles_coherency_vol_check_resv; + } else if (memcmp(buf->data, p, len - sizeof(*buf)) != 0) { + why = cachefiles_coherency_vol_check_cmp; + } else { + why = cachefiles_coherency_vol_check_ok; + ret = 0; + } + + trace_cachefiles_vol_coherency(volume, d_inode(dentry)->i_ino, why); + kfree(buf); + _leave(" = %d", ret); + return ret; +} diff --git a/fs/ceph/Kconfig b/fs/ceph/Kconfig new file mode 100644 index 000000000..3d64a316c --- /dev/null +++ b/fs/ceph/Kconfig @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: GPL-2.0-only +config CEPH_FS + tristate "Ceph distributed file system" + depends on INET + select CEPH_LIB + select NETFS_SUPPORT + select FS_ENCRYPTION_ALGS if FS_ENCRYPTION + default n + help + Choose Y or M here to include support for mounting the + experimental Ceph distributed file system. Ceph is an extremely + scalable file system designed to provide high performance, + reliable access to petabytes of storage. + + More information at https://ceph.io/. + + If unsure, say N. + +if CEPH_FS +config CEPH_FSCACHE + bool "Enable Ceph client caching support" + depends on CEPH_FS=m && FSCACHE || CEPH_FS=y && FSCACHE=y + help + Choose Y here to enable persistent, read-only local + caching support for Ceph clients using FS-Cache + +endif + +config CEPH_FS_POSIX_ACL + bool "Ceph POSIX Access Control Lists" + depends on CEPH_FS + select FS_POSIX_ACL + help + POSIX Access Control Lists (ACLs) support permissions for users and + groups beyond the owner/group/world scheme. + + If you don't know what Access Control Lists are, say N + +config CEPH_FS_SECURITY_LABEL + bool "CephFS Security Labels" + depends on CEPH_FS && SECURITY + help + Security labels support alternative access control models + implemented by security modules like SELinux. This option + enables an extended attribute handler for file security + labels in the Ceph filesystem. + + If you are not using a security module that requires using + extended attributes for file security labels, say N. diff --git a/fs/ceph/Makefile b/fs/ceph/Makefile new file mode 100644 index 000000000..ebb29d11a --- /dev/null +++ b/fs/ceph/Makefile @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for CEPH filesystem. +# + +obj-$(CONFIG_CEPH_FS) += ceph.o + +ceph-y := super.o inode.o dir.o file.o locks.o addr.o ioctl.o \ + export.o caps.o snap.o xattr.o quota.o io.o \ + mds_client.o mdsmap.o strings.o ceph_frag.o \ + debugfs.o util.o metric.o subvolume_metrics.o + +ceph-$(CONFIG_CEPH_FSCACHE) += cache.o +ceph-$(CONFIG_CEPH_FS_POSIX_ACL) += acl.o +ceph-$(CONFIG_FS_ENCRYPTION) += crypto.o diff --git a/fs/ceph/acl.c b/fs/ceph/acl.c new file mode 100644 index 000000000..85d3dd48b --- /dev/null +++ b/fs/ceph/acl.c @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/ceph/acl.c + * + * Copyright (C) 2013 Guangliang Zhao, + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "mds_client.h" + +static inline void ceph_set_cached_acl(struct inode *inode, + int type, struct posix_acl *acl) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + + spin_lock(&ci->i_ceph_lock); + if (__ceph_caps_issued_mask_metric(ci, CEPH_CAP_XATTR_SHARED, 0)) + set_cached_acl(inode, type, acl); + else + forget_cached_acl(inode, type); + spin_unlock(&ci->i_ceph_lock); +} + +struct posix_acl *ceph_get_acl(struct inode *inode, int type, bool rcu) +{ + struct ceph_client *cl = ceph_inode_to_client(inode); + int size; + unsigned int retry_cnt = 0; + const char *name; + char *value = NULL; + struct posix_acl *acl; + + if (rcu) + return ERR_PTR(-ECHILD); + + switch (type) { + case ACL_TYPE_ACCESS: + name = XATTR_NAME_POSIX_ACL_ACCESS; + break; + case ACL_TYPE_DEFAULT: + name = XATTR_NAME_POSIX_ACL_DEFAULT; + break; + default: + BUG(); + } + +retry: + size = __ceph_getxattr(inode, name, "", 0); + if (size > 0) { + value = kzalloc(size, GFP_NOFS); + if (!value) + return ERR_PTR(-ENOMEM); + size = __ceph_getxattr(inode, name, value, size); + } + + if (size == -ERANGE && retry_cnt < 10) { + retry_cnt++; + kfree(value); + value = NULL; + goto retry; + } + + if (size > 0) { + acl = posix_acl_from_xattr(&init_user_ns, value, size); + } else if (size == -ENODATA || size == 0) { + acl = NULL; + } else { + pr_err_ratelimited_client(cl, "%llx.%llx failed, err=%d\n", + ceph_vinop(inode), size); + acl = ERR_PTR(-EIO); + } + + kfree(value); + + if (!IS_ERR(acl)) + ceph_set_cached_acl(inode, type, acl); + + return acl; +} + +int ceph_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type) +{ + int ret = 0; + size_t size = 0; + const char *name = NULL; + char *value = NULL; + struct iattr newattrs; + struct inode *inode = d_inode(dentry); + struct timespec64 old_ctime = inode_get_ctime(inode); + umode_t new_mode = inode->i_mode, old_mode = inode->i_mode; + + if (ceph_snap(inode) != CEPH_NOSNAP) { + ret = -EROFS; + goto out; + } + + switch (type) { + case ACL_TYPE_ACCESS: + name = XATTR_NAME_POSIX_ACL_ACCESS; + if (acl) { + ret = posix_acl_update_mode(idmap, inode, + &new_mode, &acl); + if (ret) + goto out; + } + break; + case ACL_TYPE_DEFAULT: + if (!S_ISDIR(inode->i_mode)) { + ret = acl ? -EINVAL : 0; + goto out; + } + name = XATTR_NAME_POSIX_ACL_DEFAULT; + break; + default: + ret = -EINVAL; + goto out; + } + + if (acl) { + value = posix_acl_to_xattr(&init_user_ns, acl, &size, GFP_NOFS); + if (!value) { + ret = -ENOMEM; + goto out; + } + } + + if (new_mode != old_mode) { + newattrs.ia_ctime = current_time(inode); + newattrs.ia_mode = new_mode; + newattrs.ia_valid = ATTR_MODE | ATTR_CTIME; + ret = __ceph_setattr(idmap, inode, &newattrs, NULL); + if (ret) + goto out_free; + } + + ret = __ceph_setxattr(inode, name, value, size, 0); + if (ret) { + if (new_mode != old_mode) { + newattrs.ia_ctime = old_ctime; + newattrs.ia_mode = old_mode; + newattrs.ia_valid = ATTR_MODE | ATTR_CTIME; + __ceph_setattr(idmap, inode, &newattrs, NULL); + } + goto out_free; + } + + ceph_set_cached_acl(inode, type, acl); + +out_free: + kfree(value); +out: + return ret; +} + +int ceph_pre_init_acls(struct inode *dir, umode_t *mode, + struct ceph_acl_sec_ctx *as_ctx) +{ + struct posix_acl *acl, *default_acl; + size_t val_size1 = 0, val_size2 = 0; + struct ceph_pagelist *pagelist = NULL; + void *tmp_buf1 = NULL, *tmp_buf2 = NULL; + int err; + + err = posix_acl_create(dir, mode, &default_acl, &acl); + if (err) + return err; + + if (acl) { + err = posix_acl_equiv_mode(acl, mode); + if (err < 0) + goto out_err; + if (err == 0) { + posix_acl_release(acl); + acl = NULL; + } + } + + if (!default_acl && !acl) + return 0; + + err = -ENOMEM; + pagelist = ceph_pagelist_alloc(GFP_KERNEL); + if (!pagelist) + goto out_err; + + err = ceph_pagelist_reserve(pagelist, PAGE_SIZE); + if (err) + goto out_err; + + ceph_pagelist_encode_32(pagelist, acl && default_acl ? 2 : 1); + + if (acl) { + size_t len = strlen(XATTR_NAME_POSIX_ACL_ACCESS); + + err = -ENOMEM; + tmp_buf1 = posix_acl_to_xattr(&init_user_ns, acl, + &val_size1, GFP_KERNEL); + if (!tmp_buf1) + goto out_err; + err = ceph_pagelist_reserve(pagelist, len + val_size1 + 8); + if (err) + goto out_err; + ceph_pagelist_encode_string(pagelist, XATTR_NAME_POSIX_ACL_ACCESS, + len); + ceph_pagelist_encode_32(pagelist, val_size1); + ceph_pagelist_append(pagelist, tmp_buf1, val_size1); + } + if (default_acl) { + size_t len = strlen(XATTR_NAME_POSIX_ACL_DEFAULT); + + err = -ENOMEM; + tmp_buf2 = posix_acl_to_xattr(&init_user_ns, default_acl, + &val_size2, GFP_KERNEL); + if (!tmp_buf2) + goto out_err; + err = ceph_pagelist_reserve(pagelist, len + val_size2 + 8); + if (err) + goto out_err; + ceph_pagelist_encode_string(pagelist, + XATTR_NAME_POSIX_ACL_DEFAULT, len); + ceph_pagelist_encode_32(pagelist, val_size2); + ceph_pagelist_append(pagelist, tmp_buf2, val_size2); + } + + kfree(tmp_buf1); + kfree(tmp_buf2); + + as_ctx->acl = acl; + as_ctx->default_acl = default_acl; + as_ctx->pagelist = pagelist; + return 0; + +out_err: + posix_acl_release(acl); + posix_acl_release(default_acl); + kfree(tmp_buf1); + kfree(tmp_buf2); + if (pagelist) + ceph_pagelist_release(pagelist); + return err; +} + +void ceph_init_inode_acls(struct inode *inode, struct ceph_acl_sec_ctx *as_ctx) +{ + if (!inode) + return; + ceph_set_cached_acl(inode, ACL_TYPE_ACCESS, as_ctx->acl); + ceph_set_cached_acl(inode, ACL_TYPE_DEFAULT, as_ctx->default_acl); +} diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c new file mode 100644 index 000000000..ecf33b666 --- /dev/null +++ b/fs/ceph/addr.c @@ -0,0 +1,2644 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "mds_client.h" +#include "cache.h" +#include "metric.h" +#include "subvolume_metrics.h" +#include "crypto.h" +#include +#include + +/* + * Ceph address space ops. + * + * There are a few funny things going on here. + * + * The page->private field is used to reference a struct + * ceph_snap_context for _every_ dirty page. This indicates which + * snapshot the page was logically dirtied in, and thus which snap + * context needs to be associated with the osd write during writeback. + * + * Similarly, struct ceph_inode_info maintains a set of counters to + * count dirty pages on the inode. In the absence of snapshots, + * i_wrbuffer_ref == i_wrbuffer_ref_head == the dirty page count. + * + * When a snapshot is taken (that is, when the client receives + * notification that a snapshot was taken), each inode with caps and + * with dirty pages (dirty pages implies there is a cap) gets a new + * ceph_cap_snap in the i_cap_snaps list (which is sorted in ascending + * order, new snaps go to the tail). The i_wrbuffer_ref_head count is + * moved to capsnap->dirty. (Unless a sync write is currently in + * progress. In that case, the capsnap is said to be "pending", new + * writes cannot start, and the capsnap isn't "finalized" until the + * write completes (or fails) and a final size/mtime for the inode for + * that snap can be settled upon.) i_wrbuffer_ref_head is reset to 0. + * + * On writeback, we must submit writes to the osd IN SNAP ORDER. So, + * we look for the first capsnap in i_cap_snaps and write out pages in + * that snap context _only_. Then we move on to the next capsnap, + * eventually reaching the "live" or "head" context (i.e., pages that + * are not yet snapped) and are writing the most recently dirtied + * pages. + * + * Invalidate and so forth must take care to ensure the dirty page + * accounting is preserved. + */ + +#define CONGESTION_ON_THRESH(congestion_kb) (congestion_kb >> (PAGE_SHIFT-10)) +#define CONGESTION_OFF_THRESH(congestion_kb) \ + (CONGESTION_ON_THRESH(congestion_kb) - \ + (CONGESTION_ON_THRESH(congestion_kb) >> 2)) + +static int ceph_netfs_check_write_begin(struct file *file, loff_t pos, unsigned int len, + struct folio **foliop, void **_fsdata); + +static inline struct ceph_snap_context *page_snap_context(struct page *page) +{ + if (PagePrivate(page)) + return (void *)page->private; + return NULL; +} + +/* + * Dirty a page. Optimistically adjust accounting, on the assumption + * that we won't race with invalidate. If we do, readjust. + */ +static bool ceph_dirty_folio(struct address_space *mapping, struct folio *folio) +{ + struct inode *inode = mapping->host; + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb); + struct ceph_inode_info *ci; + struct ceph_snap_context *snapc; + + if (folio_test_dirty(folio)) { + doutc(cl, "%llx.%llx %p idx %lu -- already dirty\n", + ceph_vinop(inode), folio, folio->index); + VM_BUG_ON_FOLIO(!folio_test_private(folio), folio); + return false; + } + + atomic64_inc(&mdsc->dirty_folios); + + ci = ceph_inode(inode); + + /* dirty the head */ + spin_lock(&ci->i_ceph_lock); + if (__ceph_have_pending_cap_snap(ci)) { + struct ceph_cap_snap *capsnap = + list_last_entry(&ci->i_cap_snaps, + struct ceph_cap_snap, + ci_item); + snapc = ceph_get_snap_context(capsnap->context); + capsnap->dirty_pages++; + } else { + BUG_ON(!ci->i_head_snapc); + snapc = ceph_get_snap_context(ci->i_head_snapc); + ++ci->i_wrbuffer_ref_head; + } + if (ci->i_wrbuffer_ref == 0) + ihold(inode); + ++ci->i_wrbuffer_ref; + doutc(cl, "%llx.%llx %p idx %lu head %d/%d -> %d/%d " + "snapc %p seq %lld (%d snaps)\n", + ceph_vinop(inode), folio, folio->index, + ci->i_wrbuffer_ref-1, ci->i_wrbuffer_ref_head-1, + ci->i_wrbuffer_ref, ci->i_wrbuffer_ref_head, + snapc, snapc->seq, snapc->num_snaps); + spin_unlock(&ci->i_ceph_lock); + + /* + * Reference snap context in folio->private. Also set + * PagePrivate so that we get invalidate_folio callback. + */ + VM_WARN_ON_FOLIO(folio->private, folio); + folio_attach_private(folio, snapc); + + return ceph_fscache_dirty_folio(mapping, folio); +} + +/* + * If we are truncating the full folio (i.e. offset == 0), adjust the + * dirty folio counters appropriately. Only called if there is private + * data on the folio. + */ +static void ceph_invalidate_folio(struct folio *folio, size_t offset, + size_t length) +{ + struct inode *inode = folio->mapping->host; + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_snap_context *snapc; + + + if (offset != 0 || length != folio_size(folio)) { + doutc(cl, "%llx.%llx idx %lu partial dirty page %zu~%zu\n", + ceph_vinop(inode), folio->index, offset, length); + return; + } + + WARN_ON(!folio_test_locked(folio)); + if (folio_test_private(folio)) { + doutc(cl, "%llx.%llx idx %lu full dirty page\n", + ceph_vinop(inode), folio->index); + + snapc = folio_detach_private(folio); + ceph_put_wrbuffer_cap_refs(ci, 1, snapc); + ceph_put_snap_context(snapc); + } + + netfs_invalidate_folio(folio, offset, length); +} + +static void ceph_netfs_expand_readahead(struct netfs_io_request *rreq) +{ + struct inode *inode = rreq->inode; + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_file_layout *lo = &ci->i_layout; + unsigned long max_pages = inode->i_sb->s_bdi->ra_pages; + loff_t end = rreq->start + rreq->len, new_end; + struct ceph_netfs_request_data *priv = rreq->netfs_priv; + unsigned long max_len; + u32 blockoff; + + if (priv) { + /* Readahead is disabled by posix_fadvise POSIX_FADV_RANDOM */ + if (priv->file_ra_disabled) + max_pages = 0; + else + max_pages = priv->file_ra_pages; + + } + + /* Readahead is disabled */ + if (!max_pages) + return; + + max_len = max_pages << PAGE_SHIFT; + + /* + * Try to expand the length forward by rounding up it to the next + * block, but do not exceed the file size, unless the original + * request already exceeds it. + */ + new_end = umin(round_up(end, lo->stripe_unit), rreq->i_size); + if (new_end > end && new_end <= rreq->start + max_len) + rreq->len = new_end - rreq->start; + + /* Try to expand the start downward */ + div_u64_rem(rreq->start, lo->stripe_unit, &blockoff); + if (rreq->len + blockoff <= max_len) { + rreq->start -= blockoff; + rreq->len += blockoff; + } +} + +static void finish_netfs_read(struct ceph_osd_request *req) +{ + struct inode *inode = req->r_inode; + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + struct ceph_client *cl = fsc->client; + struct ceph_osd_data *osd_data = osd_req_op_extent_osd_data(req, 0); + struct netfs_io_subrequest *subreq = req->r_priv; + struct ceph_osd_req_op *op = &req->r_ops[0]; + int err = req->r_result; + bool sparse = (op->op == CEPH_OSD_OP_SPARSE_READ); + + ceph_update_read_metrics(&fsc->mdsc->metric, req->r_start_latency, + req->r_end_latency, osd_data->length, err); + + doutc(cl, "result %d subreq->len=%zu i_size=%lld\n", req->r_result, + subreq->len, i_size_read(req->r_inode)); + + /* no object means success but no data */ + if (err == -ENOENT) { + __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags); + __set_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); + err = 0; + } else if (err == -EBLOCKLISTED) { + fsc->blocklisted = true; + } + + if (err >= 0) { + if (sparse && err > 0) + err = ceph_sparse_ext_map_end(op); + if (err < subreq->len && + subreq->rreq->origin != NETFS_UNBUFFERED_READ && + subreq->rreq->origin != NETFS_DIO_READ) + __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags); + if (IS_ENCRYPTED(inode) && err > 0) { + err = ceph_fscrypt_decrypt_extents(inode, + osd_data->pages, subreq->start, + op->extent.sparse_ext, + op->extent.sparse_ext_cnt); + if (err > subreq->len) + err = subreq->len; + } + if (err > 0) + __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags); + } + + if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGES) { + ceph_put_page_vector(osd_data->pages, + calc_pages_for(osd_data->alignment, + osd_data->length), false); + } + if (err > 0) { + ceph_subvolume_metrics_record_io(fsc->mdsc, ceph_inode(inode), + false, err, + req->r_start_latency, + req->r_end_latency); + subreq->transferred = err; + err = 0; + } + subreq->error = err; + trace_netfs_sreq(subreq, netfs_sreq_trace_io_progress); + netfs_read_subreq_terminated(subreq); + iput(req->r_inode); + ceph_dec_osd_stopping_blocker(fsc->mdsc); +} + +static bool ceph_netfs_issue_op_inline(struct netfs_io_subrequest *subreq) +{ + struct netfs_io_request *rreq = subreq->rreq; + struct inode *inode = rreq->inode; + struct ceph_mds_reply_info_parsed *rinfo; + struct ceph_mds_reply_info_in *iinfo; + struct ceph_mds_request *req; + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb); + struct ceph_inode_info *ci = ceph_inode(inode); + ssize_t err = 0; + size_t len; + int mode; + + if (rreq->origin != NETFS_UNBUFFERED_READ && + rreq->origin != NETFS_DIO_READ) + __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags); + __clear_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags); + + if (subreq->start >= inode->i_size) + goto out; + + /* We need to fetch the inline data. */ + mode = ceph_try_to_choose_auth_mds(inode, CEPH_STAT_CAP_INLINE_DATA); + req = ceph_mdsc_create_request(mdsc, CEPH_MDS_OP_GETATTR, mode); + if (IS_ERR(req)) { + err = PTR_ERR(req); + goto out; + } + req->r_ino1 = ci->i_vino; + req->r_args.getattr.mask = cpu_to_le32(CEPH_STAT_CAP_INLINE_DATA); + req->r_num_caps = 2; + + trace_netfs_sreq(subreq, netfs_sreq_trace_submit); + err = ceph_mdsc_do_request(mdsc, NULL, req); + if (err < 0) + goto out; + + rinfo = &req->r_reply_info; + iinfo = &rinfo->targeti; + if (iinfo->inline_version == CEPH_INLINE_NONE) { + /* The data got uninlined */ + ceph_mdsc_put_request(req); + return false; + } + + len = min_t(size_t, iinfo->inline_len - subreq->start, subreq->len); + err = copy_to_iter(iinfo->inline_data + subreq->start, len, &subreq->io_iter); + if (err == 0) { + err = -EFAULT; + } else { + subreq->transferred += err; + err = 0; + } + + ceph_mdsc_put_request(req); +out: + subreq->error = err; + trace_netfs_sreq(subreq, netfs_sreq_trace_io_progress); + netfs_read_subreq_terminated(subreq); + return true; +} + +static int ceph_netfs_prepare_read(struct netfs_io_subrequest *subreq) +{ + struct netfs_io_request *rreq = subreq->rreq; + struct inode *inode = rreq->inode; + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + u64 objno, objoff; + u32 xlen; + + /* Truncate the extent at the end of the current block */ + ceph_calc_file_object_mapping(&ci->i_layout, subreq->start, subreq->len, + &objno, &objoff, &xlen); + rreq->io_streams[0].sreq_max_len = umin(xlen, fsc->mount_options->rsize); + return 0; +} + +static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq) +{ + struct netfs_io_request *rreq = subreq->rreq; + struct inode *inode = rreq->inode; + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + struct ceph_client *cl = fsc->client; + struct ceph_osd_request *req = NULL; + struct ceph_vino vino = ceph_vino(inode); + int err; + u64 len; + bool sparse = IS_ENCRYPTED(inode) || ceph_test_mount_opt(fsc, SPARSEREAD); + u64 off = subreq->start; + int extent_cnt; + + if (ceph_inode_is_shutdown(inode)) { + err = -EIO; + goto out; + } + + if (ceph_has_inline_data(ci) && ceph_netfs_issue_op_inline(subreq)) + return; + + // TODO: This rounding here is slightly dodgy. It *should* work, for + // now, as the cache only deals in blocks that are a multiple of + // PAGE_SIZE and fscrypt blocks are at most PAGE_SIZE. What needs to + // happen is for the fscrypt driving to be moved into netfslib and the + // data in the cache also to be stored encrypted. + len = subreq->len; + ceph_fscrypt_adjust_off_and_len(inode, &off, &len); + + req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, vino, + off, &len, 0, 1, sparse ? CEPH_OSD_OP_SPARSE_READ : CEPH_OSD_OP_READ, + CEPH_OSD_FLAG_READ, NULL, ci->i_truncate_seq, + ci->i_truncate_size, false); + if (IS_ERR(req)) { + err = PTR_ERR(req); + req = NULL; + goto out; + } + + if (sparse) { + extent_cnt = __ceph_sparse_read_ext_count(inode, len); + err = ceph_alloc_sparse_ext_map(&req->r_ops[0], extent_cnt); + if (err) + goto out; + } + + doutc(cl, "%llx.%llx pos=%llu orig_len=%zu len=%llu\n", + ceph_vinop(inode), subreq->start, subreq->len, len); + + /* + * FIXME: For now, use CEPH_OSD_DATA_TYPE_PAGES instead of _ITER for + * encrypted inodes. We'd need infrastructure that handles an iov_iter + * instead of page arrays, and we don't have that as of yet. Once the + * dust settles on the write helpers and encrypt/decrypt routines for + * netfs, we should be able to rework this. + */ + if (IS_ENCRYPTED(inode)) { + struct page **pages; + size_t page_off; + + /* + * FIXME: io_iter.count needs to be corrected to aligned + * length. Otherwise, iov_iter_get_pages_alloc2() operates + * with the initial unaligned length value. As a result, + * ceph_msg_data_cursor_init() triggers BUG_ON() in the case + * if msg->sparse_read_total > msg->data_length. + */ + subreq->io_iter.count = len; + + err = iov_iter_get_pages_alloc2(&subreq->io_iter, &pages, len, &page_off); + if (err < 0) { + doutc(cl, "%llx.%llx failed to allocate pages, %d\n", + ceph_vinop(inode), err); + goto out; + } + + /* should always give us a page-aligned read */ + WARN_ON_ONCE(page_off); + len = err; + err = 0; + + osd_req_op_extent_osd_data_pages(req, 0, pages, len, 0, false, + false); + } else { + osd_req_op_extent_osd_iter(req, 0, &subreq->io_iter); + } + if (!ceph_inc_osd_stopping_blocker(fsc->mdsc)) { + err = -EIO; + goto out; + } + req->r_callback = finish_netfs_read; + req->r_priv = subreq; + req->r_inode = inode; + ihold(inode); + + trace_netfs_sreq(subreq, netfs_sreq_trace_submit); + ceph_osdc_start_request(req->r_osdc, req); +out: + ceph_osdc_put_request(req); + if (err) { + subreq->error = err; + netfs_read_subreq_terminated(subreq); + } + doutc(cl, "%llx.%llx result %d\n", ceph_vinop(inode), err); +} + +static int ceph_init_request(struct netfs_io_request *rreq, struct file *file) +{ + struct inode *inode = rreq->inode; + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + struct ceph_client *cl = ceph_inode_to_client(inode); + int got = 0, want = CEPH_CAP_FILE_CACHE; + struct ceph_netfs_request_data *priv; + int ret = 0; + + /* [DEPRECATED] Use PG_private_2 to mark folio being written to the cache. */ + __set_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags); + + if (rreq->origin != NETFS_READAHEAD) + return 0; + + priv = kzalloc_obj(*priv, GFP_NOFS); + if (!priv) + return -ENOMEM; + + if (file) { + struct ceph_rw_context *rw_ctx; + struct ceph_file_info *fi = file->private_data; + + priv->file_ra_pages = file->f_ra.ra_pages; + priv->file_ra_disabled = file->f_mode & FMODE_RANDOM; + + rw_ctx = ceph_find_rw_context(fi); + if (rw_ctx) { + rreq->netfs_priv = priv; + return 0; + } + } + + /* + * readahead callers do not necessarily hold Fcb caps + * (e.g. fadvise, madvise). + */ + ret = ceph_try_get_caps(inode, CEPH_CAP_FILE_RD, want, true, &got); + if (ret < 0) { + doutc(cl, "%llx.%llx, error getting cap\n", ceph_vinop(inode)); + goto out; + } + + if (!(got & want)) { + doutc(cl, "%llx.%llx, no cache cap\n", ceph_vinop(inode)); + ret = -EACCES; + goto out; + } + if (ret == 0) { + ret = -EACCES; + goto out; + } + + priv->caps = got; + rreq->netfs_priv = priv; + rreq->io_streams[0].sreq_max_len = fsc->mount_options->rsize; + +out: + if (ret < 0) { + if (got) + ceph_put_cap_refs(ceph_inode(inode), got); + kfree(priv); + } + + return ret; +} + +static void ceph_netfs_free_request(struct netfs_io_request *rreq) +{ + struct ceph_netfs_request_data *priv = rreq->netfs_priv; + + if (!priv) + return; + + if (priv->caps) + ceph_put_cap_refs(ceph_inode(rreq->inode), priv->caps); + kfree(priv); + rreq->netfs_priv = NULL; +} + +const struct netfs_request_ops ceph_netfs_ops = { + .init_request = ceph_init_request, + .free_request = ceph_netfs_free_request, + .prepare_read = ceph_netfs_prepare_read, + .issue_read = ceph_netfs_issue_read, + .expand_readahead = ceph_netfs_expand_readahead, + .check_write_begin = ceph_netfs_check_write_begin, +}; + +#ifdef CONFIG_CEPH_FSCACHE +static void ceph_set_page_fscache(struct page *page) +{ + folio_start_private_2(page_folio(page)); /* [DEPRECATED] */ +} + +static void ceph_fscache_write_terminated(void *priv, ssize_t error) +{ + struct inode *inode = priv; + + if (IS_ERR_VALUE(error) && error != -ENOBUFS) + ceph_fscache_invalidate(inode, false); +} + +static void ceph_fscache_write_to_cache(struct inode *inode, u64 off, u64 len, bool caching) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct fscache_cookie *cookie = ceph_fscache_cookie(ci); + + fscache_write_to_cache(cookie, inode->i_mapping, off, len, i_size_read(inode), + ceph_fscache_write_terminated, inode, true, caching); +} +#else +static inline void ceph_set_page_fscache(struct page *page) +{ +} + +static inline void ceph_fscache_write_to_cache(struct inode *inode, u64 off, u64 len, bool caching) +{ +} +#endif /* CONFIG_CEPH_FSCACHE */ + +struct ceph_writeback_ctl +{ + loff_t i_size; + u64 truncate_size; + u32 truncate_seq; + bool size_stable; + + bool head_snapc; + struct ceph_snap_context *snapc; + struct ceph_snap_context *last_snapc; + + bool done; + bool should_loop; + bool range_whole; + pgoff_t start_index; + pgoff_t index; + pgoff_t end; + xa_mark_t tag; + + pgoff_t strip_unit_end; + unsigned int wsize; + unsigned int nr_folios; + unsigned int max_pages; + unsigned int locked_pages; + + int op_idx; + int num_ops; + u64 offset; + u64 len; + + struct folio_batch fbatch; + unsigned int processed_in_fbatch; + + bool from_pool; + struct page **pages; + struct page **data_pages; +}; + +/* + * Get ref for the oldest snapc for an inode with dirty data... that is, the + * only snap context we are allowed to write back. + */ +static struct ceph_snap_context * +get_oldest_context(struct inode *inode, struct ceph_writeback_ctl *ctl, + struct ceph_snap_context *page_snapc) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_snap_context *snapc = NULL; + struct ceph_cap_snap *capsnap = NULL; + + spin_lock(&ci->i_ceph_lock); + list_for_each_entry(capsnap, &ci->i_cap_snaps, ci_item) { + doutc(cl, " capsnap %p snapc %p has %d dirty pages\n", + capsnap, capsnap->context, capsnap->dirty_pages); + if (!capsnap->dirty_pages) + continue; + + /* get i_size, truncate_{seq,size} for page_snapc? */ + if (snapc && capsnap->context != page_snapc) + continue; + + if (ctl) { + if (capsnap->writing) { + ctl->i_size = i_size_read(inode); + ctl->size_stable = false; + } else { + ctl->i_size = capsnap->size; + ctl->size_stable = true; + } + ctl->truncate_size = capsnap->truncate_size; + ctl->truncate_seq = capsnap->truncate_seq; + ctl->head_snapc = false; + } + + if (snapc) + break; + + snapc = ceph_get_snap_context(capsnap->context); + if (!page_snapc || + page_snapc == snapc || + page_snapc->seq > snapc->seq) + break; + } + if (!snapc && ci->i_wrbuffer_ref_head) { + snapc = ceph_get_snap_context(ci->i_head_snapc); + doutc(cl, " head snapc %p has %d dirty pages\n", snapc, + ci->i_wrbuffer_ref_head); + if (ctl) { + ctl->i_size = i_size_read(inode); + ctl->truncate_size = ci->i_truncate_size; + ctl->truncate_seq = ci->i_truncate_seq; + ctl->size_stable = false; + ctl->head_snapc = true; + } + } + spin_unlock(&ci->i_ceph_lock); + return snapc; +} + +static u64 get_writepages_data_length(struct inode *inode, + struct page *page, u64 start) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_snap_context *snapc; + struct ceph_cap_snap *capsnap = NULL; + u64 end = i_size_read(inode); + u64 ret; + + snapc = page_snap_context(ceph_fscrypt_pagecache_page(page)); + if (snapc != ci->i_head_snapc) { + bool found = false; + spin_lock(&ci->i_ceph_lock); + list_for_each_entry(capsnap, &ci->i_cap_snaps, ci_item) { + if (capsnap->context == snapc) { + if (!capsnap->writing) + end = capsnap->size; + found = true; + break; + } + } + spin_unlock(&ci->i_ceph_lock); + WARN_ON(!found); + } + if (end > ceph_fscrypt_page_offset(page) + thp_size(page)) + end = ceph_fscrypt_page_offset(page) + thp_size(page); + ret = end > start ? end - start : 0; + if (ret && fscrypt_is_bounce_page(page)) + ret = round_up(ret, CEPH_FSCRYPT_BLOCK_SIZE); + return ret; +} + +/* + * Write a folio, but leave it locked. + * + * If we get a write error, mark the mapping for error, but still adjust the + * dirty page accounting (i.e., folio is no longer dirty). + */ +static int write_folio_nounlock(struct folio *folio, + struct writeback_control *wbc) +{ + struct page *page = &folio->page; + struct inode *inode = folio->mapping->host; + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + struct ceph_client *cl = fsc->client; + struct ceph_snap_context *snapc, *oldest; + loff_t page_off = folio_pos(folio); + int err; + loff_t len = folio_size(folio); + loff_t wlen; + struct ceph_writeback_ctl ceph_wbc; + struct ceph_osd_client *osdc = &fsc->client->osdc; + struct ceph_osd_request *req; + bool caching = ceph_is_cache_enabled(inode); + struct page *bounce_page = NULL; + + doutc(cl, "%llx.%llx folio %p idx %lu\n", ceph_vinop(inode), folio, + folio->index); + + if (ceph_inode_is_shutdown(inode)) + return -EIO; + + /* verify this is a writeable snap context */ + snapc = page_snap_context(&folio->page); + if (!snapc) { + doutc(cl, "%llx.%llx folio %p not dirty?\n", ceph_vinop(inode), + folio); + return 0; + } + oldest = get_oldest_context(inode, &ceph_wbc, snapc); + if (snapc->seq > oldest->seq) { + doutc(cl, "%llx.%llx folio %p snapc %p not writeable - noop\n", + ceph_vinop(inode), folio, snapc); + /* we should only noop if called by kswapd */ + WARN_ON(!(current->flags & PF_MEMALLOC)); + ceph_put_snap_context(oldest); + folio_redirty_for_writepage(wbc, folio); + return 0; + } + ceph_put_snap_context(oldest); + + /* is this a partial page at end of file? */ + if (page_off >= ceph_wbc.i_size) { + doutc(cl, "%llx.%llx folio at %lu beyond eof %llu\n", + ceph_vinop(inode), folio->index, ceph_wbc.i_size); + folio_invalidate(folio, 0, folio_size(folio)); + return 0; + } + + if (ceph_wbc.i_size < page_off + len) + len = ceph_wbc.i_size - page_off; + + wlen = IS_ENCRYPTED(inode) ? round_up(len, CEPH_FSCRYPT_BLOCK_SIZE) : len; + doutc(cl, "%llx.%llx folio %p index %lu on %llu~%llu snapc %p seq %lld\n", + ceph_vinop(inode), folio, folio->index, page_off, wlen, snapc, + snapc->seq); + + if (atomic_long_inc_return(&fsc->writeback_count) > + CONGESTION_ON_THRESH(fsc->mount_options->congestion_kb)) + fsc->write_congested = true; + + req = ceph_osdc_new_request(osdc, &ci->i_layout, ceph_vino(inode), + page_off, &wlen, 0, 1, CEPH_OSD_OP_WRITE, + CEPH_OSD_FLAG_WRITE, snapc, + ceph_wbc.truncate_seq, + ceph_wbc.truncate_size, true); + if (IS_ERR(req)) { + folio_redirty_for_writepage(wbc, folio); + if (atomic_long_dec_return(&fsc->writeback_count) < + CONGESTION_OFF_THRESH(fsc->mount_options->congestion_kb)) + fsc->write_congested = false; + return PTR_ERR(req); + } + + if (wlen < len) + len = wlen; + + folio_start_writeback(folio); + if (caching) + ceph_set_page_fscache(&folio->page); + ceph_fscache_write_to_cache(inode, page_off, len, caching); + + if (IS_ENCRYPTED(inode)) { + bounce_page = fscrypt_encrypt_pagecache_blocks(folio, + CEPH_FSCRYPT_BLOCK_SIZE, 0, + GFP_NOFS); + if (IS_ERR(bounce_page)) { + folio_redirty_for_writepage(wbc, folio); + folio_end_writeback(folio); + ceph_osdc_put_request(req); + if (atomic_long_dec_return(&fsc->writeback_count) < + CONGESTION_OFF_THRESH(fsc->mount_options->congestion_kb)) + fsc->write_congested = false; + return PTR_ERR(bounce_page); + } + } + + /* it may be a short write due to an object boundary */ + WARN_ON_ONCE(len > folio_size(folio)); + osd_req_op_extent_osd_data_pages(req, 0, + bounce_page ? &bounce_page : &page, wlen, 0, + false, false); + doutc(cl, "%llx.%llx %llu~%llu (%llu bytes, %sencrypted)\n", + ceph_vinop(inode), page_off, len, wlen, + IS_ENCRYPTED(inode) ? "" : "not "); + + req->r_mtime = inode_get_mtime(inode); + ceph_osdc_start_request(osdc, req); + err = ceph_osdc_wait_request(osdc, req); + + ceph_update_write_metrics(&fsc->mdsc->metric, req->r_start_latency, + req->r_end_latency, len, err); + if (err >= 0 && len > 0) + ceph_subvolume_metrics_record_io(fsc->mdsc, ci, true, len, + req->r_start_latency, + req->r_end_latency); + fscrypt_free_bounce_page(bounce_page); + ceph_osdc_put_request(req); + if (err == 0) + err = len; + + if (err < 0) { + struct writeback_control tmp_wbc; + if (!wbc) + wbc = &tmp_wbc; + if (err == -ERESTARTSYS) { + /* killed by SIGKILL */ + doutc(cl, "%llx.%llx interrupted page %p\n", + ceph_vinop(inode), folio); + folio_redirty_for_writepage(wbc, folio); + folio_end_writeback(folio); + if (atomic_long_dec_return(&fsc->writeback_count) < + CONGESTION_OFF_THRESH(fsc->mount_options->congestion_kb)) + fsc->write_congested = false; + return err; + } + if (err == -EBLOCKLISTED) + fsc->blocklisted = true; + doutc(cl, "%llx.%llx setting mapping error %d %p\n", + ceph_vinop(inode), err, folio); + mapping_set_error(&inode->i_data, err); + wbc->pages_skipped++; + } else { + doutc(cl, "%llx.%llx cleaned page %p\n", + ceph_vinop(inode), folio); + err = 0; /* vfs expects us to return 0 */ + } + oldest = folio_detach_private(folio); + WARN_ON_ONCE(oldest != snapc); + folio_end_writeback(folio); + ceph_put_wrbuffer_cap_refs(ci, 1, snapc); + ceph_put_snap_context(snapc); /* page's reference */ + + if (atomic_long_dec_return(&fsc->writeback_count) < + CONGESTION_OFF_THRESH(fsc->mount_options->congestion_kb)) + fsc->write_congested = false; + + return err; +} + +/* + * async writeback completion handler. + * + * If we get an error, set the mapping error bit, but not the individual + * page error bits. + */ +static void writepages_finish(struct ceph_osd_request *req) +{ + struct inode *inode = req->r_inode; + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_osd_data *osd_data; + struct page *page; + int num_pages, total_pages = 0; + int i, j; + int rc = req->r_result; + struct ceph_snap_context *snapc = req->r_snapc; + struct address_space *mapping = inode->i_mapping; + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb); + unsigned int len = 0; + bool remove_page; + + doutc(cl, "%llx.%llx rc %d\n", ceph_vinop(inode), rc); + if (rc < 0) { + mapping_set_error(mapping, rc); + ceph_set_error_write(ci); + if (rc == -EBLOCKLISTED) + fsc->blocklisted = true; + } else { + ceph_clear_error_write(ci); + } + + /* + * We lost the cache cap, need to truncate the page before + * it is unlocked, otherwise we'd truncate it later in the + * page truncation thread, possibly losing some data that + * raced its way in + */ + remove_page = !(ceph_caps_issued(ci) & + (CEPH_CAP_FILE_CACHE|CEPH_CAP_FILE_LAZYIO)); + + /* clean all pages */ + for (i = 0; i < req->r_num_ops; i++) { + if (req->r_ops[i].op != CEPH_OSD_OP_WRITE) { + pr_warn_client(cl, + "%llx.%llx incorrect op %d req %p index %d tid %llu\n", + ceph_vinop(inode), req->r_ops[i].op, req, i, + req->r_tid); + break; + } + + osd_data = osd_req_op_extent_osd_data(req, i); + BUG_ON(osd_data->type != CEPH_OSD_DATA_TYPE_PAGES); + len += osd_data->length; + num_pages = calc_pages_for((u64)osd_data->alignment, + (u64)osd_data->length); + total_pages += num_pages; + for (j = 0; j < num_pages; j++) { + page = osd_data->pages[j]; + if (fscrypt_is_bounce_page(page)) { + page = fscrypt_pagecache_page(page); + fscrypt_free_bounce_page(osd_data->pages[j]); + osd_data->pages[j] = page; + } + BUG_ON(!page); + WARN_ON(!PageUptodate(page)); + + if (atomic_long_dec_return(&fsc->writeback_count) < + CONGESTION_OFF_THRESH( + fsc->mount_options->congestion_kb)) + fsc->write_congested = false; + + ceph_put_snap_context(detach_page_private(page)); + end_page_writeback(page); + + if (atomic64_dec_return(&mdsc->dirty_folios) <= 0) { + wake_up_all(&mdsc->flush_end_wq); + WARN_ON(atomic64_read(&mdsc->dirty_folios) < 0); + } + + doutc(cl, "unlocking %p\n", page); + + if (remove_page) + generic_error_remove_folio(inode->i_mapping, + page_folio(page)); + + unlock_page(page); + } + doutc(cl, "%llx.%llx wrote %llu bytes cleaned %d pages\n", + ceph_vinop(inode), osd_data->length, + rc >= 0 ? num_pages : 0); + + release_pages(osd_data->pages, num_pages); + } + + ceph_update_write_metrics(&fsc->mdsc->metric, req->r_start_latency, + req->r_end_latency, len, rc); + + if (rc >= 0 && len > 0) + ceph_subvolume_metrics_record_io(mdsc, ci, true, len, + req->r_start_latency, + req->r_end_latency); + + ceph_put_wrbuffer_cap_refs(ci, total_pages, snapc); + + osd_data = osd_req_op_extent_osd_data(req, 0); + if (osd_data->pages_from_pool) + mempool_free(osd_data->pages, ceph_wb_pagevec_pool); + else + kfree(osd_data->pages); + ceph_osdc_put_request(req); + ceph_dec_osd_stopping_blocker(fsc->mdsc); +} + +static inline +bool is_forced_umount(struct address_space *mapping) +{ + struct inode *inode = mapping->host; + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + struct ceph_client *cl = fsc->client; + + if (ceph_inode_is_shutdown(inode)) { + if (ci->i_wrbuffer_ref > 0) { + pr_warn_ratelimited_client(cl, + "%llx.%llx %lld forced umount\n", + ceph_vinop(inode), ceph_ino(inode)); + } + mapping_set_error(mapping, -EIO); + return true; + } + + return false; +} + +static inline +unsigned int ceph_define_write_size(struct address_space *mapping) +{ + struct inode *inode = mapping->host; + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + unsigned int wsize = ci->i_layout.stripe_unit; + + if (fsc->mount_options->wsize < wsize) + wsize = fsc->mount_options->wsize; + + return wsize; +} + +static inline +void ceph_folio_batch_init(struct ceph_writeback_ctl *ceph_wbc) +{ + folio_batch_init(&ceph_wbc->fbatch); + ceph_wbc->processed_in_fbatch = 0; +} + +static inline +void ceph_folio_batch_reinit(struct ceph_writeback_ctl *ceph_wbc) +{ + folio_batch_release(&ceph_wbc->fbatch); + ceph_folio_batch_init(ceph_wbc); +} + +static inline +void ceph_init_writeback_ctl(struct address_space *mapping, + struct writeback_control *wbc, + struct ceph_writeback_ctl *ceph_wbc) +{ + ceph_wbc->snapc = NULL; + ceph_wbc->last_snapc = NULL; + + ceph_wbc->strip_unit_end = 0; + ceph_wbc->wsize = ceph_define_write_size(mapping); + + ceph_wbc->nr_folios = 0; + ceph_wbc->max_pages = 0; + ceph_wbc->locked_pages = 0; + + ceph_wbc->done = false; + ceph_wbc->should_loop = false; + ceph_wbc->range_whole = false; + + ceph_wbc->start_index = wbc->range_cyclic ? mapping->writeback_index : 0; + ceph_wbc->index = ceph_wbc->start_index; + ceph_wbc->end = -1; + + ceph_wbc->tag = wbc_to_tag(wbc); + + ceph_wbc->op_idx = -1; + ceph_wbc->num_ops = 0; + ceph_wbc->offset = 0; + ceph_wbc->len = 0; + ceph_wbc->from_pool = false; + + ceph_folio_batch_init(ceph_wbc); + + ceph_wbc->pages = NULL; + ceph_wbc->data_pages = NULL; +} + +static inline +int ceph_define_writeback_range(struct address_space *mapping, + struct writeback_control *wbc, + struct ceph_writeback_ctl *ceph_wbc) +{ + struct inode *inode = mapping->host; + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + struct ceph_client *cl = fsc->client; + + /* find oldest snap context with dirty data */ + ceph_wbc->snapc = get_oldest_context(inode, ceph_wbc, NULL); + if (!ceph_wbc->snapc) { + /* hmm, why does writepages get called when there + is no dirty data? */ + doutc(cl, " no snap context with dirty data?\n"); + return -ENODATA; + } + + doutc(cl, " oldest snapc is %p seq %lld (%d snaps)\n", + ceph_wbc->snapc, ceph_wbc->snapc->seq, + ceph_wbc->snapc->num_snaps); + + ceph_wbc->should_loop = false; + + if (ceph_wbc->head_snapc && ceph_wbc->snapc != ceph_wbc->last_snapc) { + /* where to start/end? */ + if (wbc->range_cyclic) { + ceph_wbc->index = ceph_wbc->start_index; + ceph_wbc->end = -1; + if (ceph_wbc->index > 0) + ceph_wbc->should_loop = true; + doutc(cl, " cyclic, start at %lu\n", ceph_wbc->index); + } else { + ceph_wbc->index = wbc->range_start >> PAGE_SHIFT; + ceph_wbc->end = wbc->range_end >> PAGE_SHIFT; + if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) + ceph_wbc->range_whole = true; + doutc(cl, " not cyclic, %lu to %lu\n", + ceph_wbc->index, ceph_wbc->end); + } + } else if (!ceph_wbc->head_snapc) { + /* Do not respect wbc->range_{start,end}. Dirty pages + * in that range can be associated with newer snapc. + * They are not writeable until we write all dirty pages + * associated with 'snapc' get written */ + if (ceph_wbc->index > 0) + ceph_wbc->should_loop = true; + doutc(cl, " non-head snapc, range whole\n"); + } + + ceph_put_snap_context(ceph_wbc->last_snapc); + ceph_wbc->last_snapc = ceph_wbc->snapc; + + return 0; +} + +static inline +bool has_writeback_done(struct ceph_writeback_ctl *ceph_wbc) +{ + return ceph_wbc->done && ceph_wbc->index > ceph_wbc->end; +} + +static inline +bool can_next_page_be_processed(struct ceph_writeback_ctl *ceph_wbc, + unsigned index) +{ + return index < ceph_wbc->nr_folios && + ceph_wbc->locked_pages < ceph_wbc->max_pages; +} + +static +int ceph_check_page_before_write(struct address_space *mapping, + struct writeback_control *wbc, + struct ceph_writeback_ctl *ceph_wbc, + struct folio *folio) +{ + struct inode *inode = mapping->host; + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + struct ceph_client *cl = fsc->client; + struct ceph_snap_context *pgsnapc; + + /* only dirty folios, or our accounting breaks */ + if (unlikely(!folio_test_dirty(folio) || folio->mapping != mapping)) { + doutc(cl, "!dirty or !mapping %p\n", folio); + return -ENODATA; + } + + /* only if matching snap context */ + pgsnapc = page_snap_context(&folio->page); + if (pgsnapc != ceph_wbc->snapc) { + doutc(cl, "folio snapc %p %lld != oldest %p %lld\n", + pgsnapc, pgsnapc->seq, + ceph_wbc->snapc, ceph_wbc->snapc->seq); + + if (!ceph_wbc->should_loop && !ceph_wbc->head_snapc && + wbc->sync_mode != WB_SYNC_NONE) + ceph_wbc->should_loop = true; + + return -ENODATA; + } + + if (folio_pos(folio) >= ceph_wbc->i_size) { + doutc(cl, "folio at %lu beyond eof %llu\n", + folio->index, ceph_wbc->i_size); + + if ((ceph_wbc->size_stable || + folio_pos(folio) >= i_size_read(inode)) && + folio_clear_dirty_for_io(folio)) + folio_invalidate(folio, 0, folio_size(folio)); + + return -ENODATA; + } + + if (ceph_wbc->strip_unit_end && + (folio->index > ceph_wbc->strip_unit_end)) { + doutc(cl, "end of strip unit %p\n", folio); + return -E2BIG; + } + + return 0; +} + +static inline +void __ceph_allocate_page_array(struct ceph_writeback_ctl *ceph_wbc, + unsigned int max_pages) +{ + ceph_wbc->pages = kmalloc_objs(*ceph_wbc->pages, max_pages, GFP_NOFS); + if (!ceph_wbc->pages) { + ceph_wbc->from_pool = true; + ceph_wbc->pages = mempool_alloc(ceph_wb_pagevec_pool, GFP_NOFS); + BUG_ON(!ceph_wbc->pages); + } +} + +static inline +void ceph_allocate_page_array(struct address_space *mapping, + struct ceph_writeback_ctl *ceph_wbc, + struct folio *folio) +{ + struct inode *inode = mapping->host; + struct ceph_inode_info *ci = ceph_inode(inode); + u64 objnum; + u64 objoff; + u32 xlen; + + /* prepare async write request */ + ceph_wbc->offset = (u64)folio_pos(folio); + ceph_calc_file_object_mapping(&ci->i_layout, + ceph_wbc->offset, ceph_wbc->wsize, + &objnum, &objoff, &xlen); + + ceph_wbc->num_ops = 1; + ceph_wbc->strip_unit_end = folio->index + ((xlen - 1) >> PAGE_SHIFT); + + BUG_ON(ceph_wbc->pages); + ceph_wbc->max_pages = calc_pages_for(0, (u64)xlen); + __ceph_allocate_page_array(ceph_wbc, ceph_wbc->max_pages); + + ceph_wbc->len = 0; +} + +static inline +bool is_folio_index_contiguous(const struct ceph_writeback_ctl *ceph_wbc, + const struct folio *folio) +{ + return folio->index == (ceph_wbc->offset + ceph_wbc->len) >> PAGE_SHIFT; +} + +static inline +bool is_num_ops_too_big(struct ceph_writeback_ctl *ceph_wbc) +{ + return ceph_wbc->num_ops >= + (ceph_wbc->from_pool ? CEPH_OSD_SLAB_OPS : CEPH_OSD_MAX_OPS); +} + +static inline +bool is_write_congestion_happened(struct ceph_fs_client *fsc) +{ + return atomic_long_inc_return(&fsc->writeback_count) > + CONGESTION_ON_THRESH(fsc->mount_options->congestion_kb); +} + +static inline int move_dirty_folio_in_page_array(struct address_space *mapping, + struct writeback_control *wbc, + struct ceph_writeback_ctl *ceph_wbc, struct folio *folio) +{ + struct inode *inode = mapping->host; + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + struct ceph_client *cl = fsc->client; + struct page **pages = ceph_wbc->pages; + unsigned int index = ceph_wbc->locked_pages; + gfp_t gfp_flags = ceph_wbc->locked_pages ? GFP_NOWAIT : GFP_NOFS; + + if (IS_ENCRYPTED(inode)) { + pages[index] = fscrypt_encrypt_pagecache_blocks(folio, + PAGE_SIZE, + 0, + gfp_flags); + if (IS_ERR(pages[index])) { + int err = PTR_ERR(pages[index]); + + if (err == -EINVAL) { + pr_err_client(cl, "inode->i_blkbits=%hhu\n", + inode->i_blkbits); + } + + /* better not fail on first page! */ + BUG_ON(ceph_wbc->locked_pages == 0); + + pages[index] = NULL; + return err; + } + } else { + pages[index] = &folio->page; + } + + ceph_wbc->locked_pages++; + + return 0; +} + +static +void ceph_process_folio_batch(struct address_space *mapping, + struct writeback_control *wbc, + struct ceph_writeback_ctl *ceph_wbc) +{ + struct inode *inode = mapping->host; + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + struct ceph_client *cl = fsc->client; + struct folio *folio = NULL; + unsigned i; + int rc; + + for (i = 0; can_next_page_be_processed(ceph_wbc, i); i++) { + folio = ceph_wbc->fbatch.folios[i]; + + if (!folio) + continue; + + doutc(cl, "? %p idx %lu, folio_test_writeback %#x, " + "folio_test_dirty %#x, folio_test_locked %#x\n", + folio, folio->index, folio_test_writeback(folio), + folio_test_dirty(folio), + folio_test_locked(folio)); + + if (folio_test_writeback(folio) || + folio_test_private_2(folio) /* [DEPRECATED] */) { + doutc(cl, "waiting on writeback %p\n", folio); + folio_wait_writeback(folio); + folio_wait_private_2(folio); /* [DEPRECATED] */ + continue; + } + + if (ceph_wbc->locked_pages == 0) + folio_lock(folio); + else if (!folio_trylock(folio)) + break; + + rc = ceph_check_page_before_write(mapping, wbc, + ceph_wbc, folio); + if (rc == -ENODATA) { + folio_unlock(folio); + folio_put(folio); + ceph_wbc->fbatch.folios[i] = NULL; + continue; + } else if (rc == -E2BIG) { + folio_unlock(folio); + break; + } + + if (!folio_clear_dirty_for_io(folio)) { + doutc(cl, "%p !folio_clear_dirty_for_io\n", folio); + folio_unlock(folio); + folio_put(folio); + ceph_wbc->fbatch.folios[i] = NULL; + continue; + } + + /* + * We have something to write. If this is + * the first locked page this time through, + * calculate max possible write size and + * allocate a page array + */ + if (ceph_wbc->locked_pages == 0) { + ceph_allocate_page_array(mapping, ceph_wbc, folio); + } else if (!is_folio_index_contiguous(ceph_wbc, folio)) { + if (is_num_ops_too_big(ceph_wbc)) { + folio_redirty_for_writepage(wbc, folio); + folio_unlock(folio); + break; + } + + ceph_wbc->num_ops++; + ceph_wbc->offset = (u64)folio_pos(folio); + ceph_wbc->len = 0; + } + + /* note position of first page in fbatch */ + doutc(cl, "%llx.%llx will write folio %p idx %lu\n", + ceph_vinop(inode), folio, folio->index); + + fsc->write_congested = is_write_congestion_happened(fsc); + + rc = move_dirty_folio_in_page_array(mapping, wbc, ceph_wbc, + folio); + if (rc) { + /* Did we just begin a new contiguous op? Nevermind! */ + if (ceph_wbc->len == 0) + ceph_wbc->num_ops--; + + folio_redirty_for_writepage(wbc, folio); + folio_unlock(folio); + break; + } + + ceph_wbc->fbatch.folios[i] = NULL; + ceph_wbc->len += folio_size(folio); + } + + ceph_wbc->processed_in_fbatch = i; +} + +static inline +void ceph_shift_unused_folios_left(struct folio_batch *fbatch) +{ + unsigned j, n = 0; + + /* shift unused page to beginning of fbatch */ + for (j = 0; j < folio_batch_count(fbatch); j++) { + if (!fbatch->folios[j]) + continue; + + if (n < j) { + fbatch->folios[n] = fbatch->folios[j]; + } + + n++; + } + + fbatch->nr = n; +} + +static +int ceph_submit_write(struct address_space *mapping, + struct writeback_control *wbc, + struct ceph_writeback_ctl *ceph_wbc) +{ + struct inode *inode = mapping->host; + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + struct ceph_client *cl = fsc->client; + struct ceph_vino vino = ceph_vino(inode); + struct ceph_osd_request *req = NULL; + struct page *page = NULL; + bool caching = ceph_is_cache_enabled(inode); + u64 offset; + u64 len; + unsigned i; + +new_request: + offset = ceph_fscrypt_page_offset(ceph_wbc->pages[0]); + len = ceph_wbc->wsize; + + req = ceph_osdc_new_request(&fsc->client->osdc, + &ci->i_layout, vino, + offset, &len, 0, ceph_wbc->num_ops, + CEPH_OSD_OP_WRITE, CEPH_OSD_FLAG_WRITE, + ceph_wbc->snapc, ceph_wbc->truncate_seq, + ceph_wbc->truncate_size, false); + if (IS_ERR(req)) { + req = ceph_osdc_new_request(&fsc->client->osdc, + &ci->i_layout, vino, + offset, &len, 0, + min(ceph_wbc->num_ops, + CEPH_OSD_SLAB_OPS), + CEPH_OSD_OP_WRITE, + CEPH_OSD_FLAG_WRITE, + ceph_wbc->snapc, + ceph_wbc->truncate_seq, + ceph_wbc->truncate_size, + true); + BUG_ON(IS_ERR(req)); + } + + page = ceph_wbc->pages[ceph_wbc->locked_pages - 1]; + BUG_ON(len < ceph_fscrypt_page_offset(page) + thp_size(page) - offset); + + if (!ceph_inc_osd_stopping_blocker(fsc->mdsc)) { + for (i = 0; i < folio_batch_count(&ceph_wbc->fbatch); i++) { + struct folio *folio = ceph_wbc->fbatch.folios[i]; + + if (!folio) + continue; + + page = &folio->page; + redirty_page_for_writepage(wbc, page); + unlock_page(page); + } + + for (i = 0; i < ceph_wbc->locked_pages; i++) { + page = ceph_fscrypt_pagecache_page(ceph_wbc->pages[i]); + + if (!page) + continue; + + redirty_page_for_writepage(wbc, page); + unlock_page(page); + } + + ceph_osdc_put_request(req); + return -EIO; + } + + req->r_callback = writepages_finish; + req->r_inode = inode; + + /* Format the osd request message and submit the write */ + len = 0; + ceph_wbc->data_pages = ceph_wbc->pages; + ceph_wbc->op_idx = 0; + for (i = 0; i < ceph_wbc->locked_pages; i++) { + u64 cur_offset; + + page = ceph_fscrypt_pagecache_page(ceph_wbc->pages[i]); + cur_offset = page_offset(page); + + /* + * Discontinuity in page range? Ceph can handle that by just passing + * multiple extents in the write op. + */ + if (offset + len != cur_offset) { + /* If it's full, stop here */ + if (ceph_wbc->op_idx + 1 == req->r_num_ops) + break; + + /* Kick off an fscache write with what we have so far. */ + ceph_fscache_write_to_cache(inode, offset, len, caching); + + /* Start a new extent */ + osd_req_op_extent_dup_last(req, ceph_wbc->op_idx, + cur_offset - offset); + + doutc(cl, "got pages at %llu~%llu\n", offset, len); + + osd_req_op_extent_osd_data_pages(req, ceph_wbc->op_idx, + ceph_wbc->data_pages, + len, 0, + ceph_wbc->from_pool, + false); + osd_req_op_extent_update(req, ceph_wbc->op_idx, len); + + len = 0; + offset = cur_offset; + ceph_wbc->data_pages = ceph_wbc->pages + i; + ceph_wbc->op_idx++; + } + + set_page_writeback(page); + + if (caching) + ceph_set_page_fscache(page); + + len += thp_size(page); + } + + ceph_fscache_write_to_cache(inode, offset, len, caching); + + if (ceph_wbc->size_stable) { + len = min(len, ceph_wbc->i_size - offset); + } else if (i == ceph_wbc->locked_pages) { + /* writepages_finish() clears writeback pages + * according to the data length, so make sure + * data length covers all locked pages */ + u64 min_len = len + 1 - thp_size(page); + len = get_writepages_data_length(inode, + ceph_wbc->pages[i - 1], + offset); + len = max(len, min_len); + } + + if (IS_ENCRYPTED(inode)) + len = round_up(len, CEPH_FSCRYPT_BLOCK_SIZE); + + doutc(cl, "got pages at %llu~%llu\n", offset, len); + + if (IS_ENCRYPTED(inode) && + ((offset | len) & ~CEPH_FSCRYPT_BLOCK_MASK)) { + pr_warn_client(cl, + "bad encrypted write offset=%lld len=%llu\n", + offset, len); + } + + osd_req_op_extent_osd_data_pages(req, ceph_wbc->op_idx, + ceph_wbc->data_pages, len, + 0, ceph_wbc->from_pool, false); + osd_req_op_extent_update(req, ceph_wbc->op_idx, len); + + BUG_ON(ceph_wbc->op_idx + 1 != req->r_num_ops); + + ceph_wbc->from_pool = false; + if (i < ceph_wbc->locked_pages) { + BUG_ON(ceph_wbc->num_ops <= req->r_num_ops); + ceph_wbc->num_ops -= req->r_num_ops; + ceph_wbc->locked_pages -= i; + + /* allocate new pages array for next request */ + ceph_wbc->data_pages = ceph_wbc->pages; + __ceph_allocate_page_array(ceph_wbc, ceph_wbc->locked_pages); + memcpy(ceph_wbc->pages, ceph_wbc->data_pages + i, + ceph_wbc->locked_pages * sizeof(*ceph_wbc->pages)); + memset(ceph_wbc->data_pages + i, 0, + ceph_wbc->locked_pages * sizeof(*ceph_wbc->pages)); + } else { + BUG_ON(ceph_wbc->num_ops != req->r_num_ops); + /* request message now owns the pages array */ + ceph_wbc->pages = NULL; + } + + req->r_mtime = inode_get_mtime(inode); + ceph_osdc_start_request(&fsc->client->osdc, req); + req = NULL; + + wbc->nr_to_write -= i; + if (ceph_wbc->pages) + goto new_request; + + return 0; +} + +static +void ceph_wait_until_current_writes_complete(struct address_space *mapping, + struct writeback_control *wbc, + struct ceph_writeback_ctl *ceph_wbc) +{ + struct page *page; + unsigned i, nr; + + if (wbc->sync_mode != WB_SYNC_NONE && + ceph_wbc->start_index == 0 && /* all dirty pages were checked */ + !ceph_wbc->head_snapc) { + ceph_wbc->index = 0; + + while ((ceph_wbc->index <= ceph_wbc->end) && + (nr = filemap_get_folios_tag(mapping, + &ceph_wbc->index, + (pgoff_t)-1, + PAGECACHE_TAG_WRITEBACK, + &ceph_wbc->fbatch))) { + for (i = 0; i < nr; i++) { + page = &ceph_wbc->fbatch.folios[i]->page; + if (page_snap_context(page) != ceph_wbc->snapc) + continue; + wait_on_page_writeback(page); + } + + folio_batch_release(&ceph_wbc->fbatch); + cond_resched(); + } + } +} + +/* + * initiate async writeback + */ +static int ceph_writepages_start(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct inode *inode = mapping->host; + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + struct ceph_client *cl = fsc->client; + struct ceph_writeback_ctl ceph_wbc; + int rc = 0; + + if (wbc->sync_mode == WB_SYNC_NONE && fsc->write_congested) + return 0; + + doutc(cl, "%llx.%llx (mode=%s)\n", ceph_vinop(inode), + wbc->sync_mode == WB_SYNC_NONE ? "NONE" : + (wbc->sync_mode == WB_SYNC_ALL ? "ALL" : "HOLD")); + + if (is_forced_umount(mapping)) { + /* we're in a forced umount, don't write! */ + return -EIO; + } + + ceph_init_writeback_ctl(mapping, wbc, &ceph_wbc); + + if (!ceph_inc_osd_stopping_blocker(fsc->mdsc)) { + rc = -EIO; + goto out; + } + +retry: + rc = ceph_define_writeback_range(mapping, wbc, &ceph_wbc); + if (rc == -ENODATA) { + /* hmm, why does writepages get called when there + is no dirty data? */ + rc = 0; + goto dec_osd_stopping_blocker; + } + + if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) + tag_pages_for_writeback(mapping, ceph_wbc.index, ceph_wbc.end); + + while (!has_writeback_done(&ceph_wbc)) { + BUG_ON(ceph_wbc.locked_pages); + BUG_ON(ceph_wbc.pages); + + ceph_wbc.max_pages = ceph_wbc.wsize >> PAGE_SHIFT; + +get_more_pages: + ceph_folio_batch_reinit(&ceph_wbc); + + ceph_wbc.nr_folios = filemap_get_folios_tag(mapping, + &ceph_wbc.index, + ceph_wbc.end, + ceph_wbc.tag, + &ceph_wbc.fbatch); + doutc(cl, "pagevec_lookup_range_tag for tag %#x got %d\n", + ceph_wbc.tag, ceph_wbc.nr_folios); + + if (!ceph_wbc.nr_folios && !ceph_wbc.locked_pages) + break; + +process_folio_batch: + ceph_process_folio_batch(mapping, wbc, &ceph_wbc); + ceph_shift_unused_folios_left(&ceph_wbc.fbatch); + + /* did we get anything? */ + if (!ceph_wbc.locked_pages) + goto release_folios; + + if (ceph_wbc.processed_in_fbatch) { + if (folio_batch_count(&ceph_wbc.fbatch) == 0 && + ceph_wbc.locked_pages < ceph_wbc.max_pages) { + doutc(cl, "reached end fbatch, trying for more\n"); + goto get_more_pages; + } + } + + rc = ceph_submit_write(mapping, wbc, &ceph_wbc); + if (rc) + goto release_folios; + + ceph_wbc.locked_pages = 0; + ceph_wbc.strip_unit_end = 0; + + if (folio_batch_count(&ceph_wbc.fbatch) > 0) { + ceph_wbc.nr_folios = + folio_batch_count(&ceph_wbc.fbatch); + goto process_folio_batch; + } + + /* + * We stop writing back only if we are not doing + * integrity sync. In case of integrity sync we have to + * keep going until we have written all the pages + * we tagged for writeback prior to entering this loop. + */ + if (wbc->nr_to_write <= 0 && wbc->sync_mode == WB_SYNC_NONE) + ceph_wbc.done = true; + +release_folios: + doutc(cl, "folio_batch release on %d folios (%p)\n", + (int)ceph_wbc.fbatch.nr, + ceph_wbc.fbatch.nr ? ceph_wbc.fbatch.folios[0] : NULL); + folio_batch_release(&ceph_wbc.fbatch); + } + + if (ceph_wbc.should_loop && !ceph_wbc.done) { + /* more to do; loop back to beginning of file */ + doutc(cl, "looping back to beginning of file\n"); + /* OK even when start_index == 0 */ + ceph_wbc.end = ceph_wbc.start_index - 1; + + /* to write dirty pages associated with next snapc, + * we need to wait until current writes complete */ + ceph_wait_until_current_writes_complete(mapping, wbc, &ceph_wbc); + + ceph_wbc.start_index = 0; + ceph_wbc.index = 0; + goto retry; + } + + if (wbc->range_cyclic || (ceph_wbc.range_whole && wbc->nr_to_write > 0)) + mapping->writeback_index = ceph_wbc.index; + +dec_osd_stopping_blocker: + ceph_dec_osd_stopping_blocker(fsc->mdsc); + +out: + ceph_put_snap_context(ceph_wbc.last_snapc); + doutc(cl, "%llx.%llx dend - startone, rc = %d\n", ceph_vinop(inode), + rc); + + return rc; +} + +/* + * See if a given @snapc is either writeable, or already written. + */ +static int context_is_writeable_or_written(struct inode *inode, + struct ceph_snap_context *snapc) +{ + struct ceph_snap_context *oldest = get_oldest_context(inode, NULL, NULL); + int ret = !oldest || snapc->seq <= oldest->seq; + + ceph_put_snap_context(oldest); + return ret; +} + +/** + * ceph_find_incompatible - find an incompatible context and return it + * @folio: folio being dirtied + * + * We are only allowed to write into/dirty a folio if the folio is + * clean, or already dirty within the same snap context. Returns a + * conflicting context if there is one, NULL if there isn't, or a + * negative error code on other errors. + * + * Must be called with folio lock held. + */ +static struct ceph_snap_context * +ceph_find_incompatible(struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + + if (ceph_inode_is_shutdown(inode)) { + doutc(cl, " %llx.%llx folio %p is shutdown\n", + ceph_vinop(inode), folio); + return ERR_PTR(-ESTALE); + } + + for (;;) { + struct ceph_snap_context *snapc, *oldest; + + folio_wait_writeback(folio); + + snapc = page_snap_context(&folio->page); + if (!snapc || snapc == ci->i_head_snapc) + break; + + /* + * this folio is already dirty in another (older) snap + * context! is it writeable now? + */ + oldest = get_oldest_context(inode, NULL, NULL); + if (snapc->seq > oldest->seq) { + /* not writeable -- return it for the caller to deal with */ + ceph_put_snap_context(oldest); + doutc(cl, " %llx.%llx folio %p snapc %p not current or oldest\n", + ceph_vinop(inode), folio, snapc); + return ceph_get_snap_context(snapc); + } + ceph_put_snap_context(oldest); + + /* yay, writeable, do it now (without dropping folio lock) */ + doutc(cl, " %llx.%llx folio %p snapc %p not current, but oldest\n", + ceph_vinop(inode), folio, snapc); + if (folio_clear_dirty_for_io(folio)) { + int r = write_folio_nounlock(folio, NULL); + if (r < 0) + return ERR_PTR(r); + } + } + return NULL; +} + +static int ceph_netfs_check_write_begin(struct file *file, loff_t pos, unsigned int len, + struct folio **foliop, void **_fsdata) +{ + struct inode *inode = file_inode(file); + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_snap_context *snapc; + + snapc = ceph_find_incompatible(*foliop); + if (snapc) { + int r; + + folio_unlock(*foliop); + folio_put(*foliop); + *foliop = NULL; + if (IS_ERR(snapc)) + return PTR_ERR(snapc); + + ceph_queue_writeback(inode); + r = wait_event_killable(ci->i_cap_wq, + context_is_writeable_or_written(inode, snapc)); + ceph_put_snap_context(snapc); + return r == 0 ? -EAGAIN : r; + } + return 0; +} + +/* + * We are only allowed to write into/dirty the page if the page is + * clean, or already dirty within the same snap context. + */ +static int ceph_write_begin(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, + struct folio **foliop, void **fsdata) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct ceph_inode_info *ci = ceph_inode(inode); + int r; + + r = netfs_write_begin(&ci->netfs, file, inode->i_mapping, pos, len, foliop, NULL); + if (r < 0) + return r; + + folio_wait_private_2(*foliop); /* [DEPRECATED] */ + WARN_ON_ONCE(!folio_test_locked(*foliop)); + return 0; +} + +/* + * we don't do anything in here that simple_write_end doesn't do + * except adjust dirty page accounting + */ +static int ceph_write_end(const struct kiocb *iocb, + struct address_space *mapping, loff_t pos, + unsigned len, unsigned copied, + struct folio *folio, void *fsdata) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct ceph_client *cl = ceph_inode_to_client(inode); + bool check_cap = false; + + doutc(cl, "%llx.%llx file %p folio %p %d~%d (%d)\n", ceph_vinop(inode), + file, folio, (int)pos, (int)copied, (int)len); + + if (!folio_test_uptodate(folio)) { + /* just return that nothing was copied on a short copy */ + if (copied < len) { + copied = 0; + goto out; + } + folio_mark_uptodate(folio); + } + + /* did file size increase? */ + if (pos+copied > i_size_read(inode)) + check_cap = ceph_inode_set_size(inode, pos+copied); + + folio_mark_dirty(folio); + +out: + folio_unlock(folio); + folio_put(folio); + + if (check_cap) + ceph_check_caps(ceph_inode(inode), CHECK_CAPS_AUTHONLY); + + return copied; +} + +const struct address_space_operations ceph_aops = { + .read_folio = netfs_read_folio, + .readahead = netfs_readahead, + .writepages = ceph_writepages_start, + .write_begin = ceph_write_begin, + .write_end = ceph_write_end, + .dirty_folio = ceph_dirty_folio, + .invalidate_folio = ceph_invalidate_folio, + .release_folio = netfs_release_folio, + .direct_IO = noop_direct_IO, + .migrate_folio = filemap_migrate_folio, +}; + +static void ceph_block_sigs(sigset_t *oldset) +{ + sigset_t mask; + siginitsetinv(&mask, sigmask(SIGKILL)); + sigprocmask(SIG_BLOCK, &mask, oldset); +} + +static void ceph_restore_sigs(sigset_t *oldset) +{ + sigprocmask(SIG_SETMASK, oldset, NULL); +} + +/* + * vm ops + */ +static vm_fault_t ceph_filemap_fault(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct inode *inode = file_inode(vma->vm_file); + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_file_info *fi = vma->vm_file->private_data; + loff_t off = (loff_t)vmf->pgoff << PAGE_SHIFT; + int want, got, err; + sigset_t oldset; + vm_fault_t ret = VM_FAULT_SIGBUS; + + if (ceph_inode_is_shutdown(inode)) + return ret; + + ceph_block_sigs(&oldset); + + doutc(cl, "%llx.%llx %llu trying to get caps\n", + ceph_vinop(inode), off); + if (fi->fmode & CEPH_FILE_MODE_LAZY) + want = CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_LAZYIO; + else + want = CEPH_CAP_FILE_CACHE; + + got = 0; + err = ceph_get_caps(vma->vm_file, CEPH_CAP_FILE_RD, want, -1, &got); + if (err < 0) + goto out_restore; + + doutc(cl, "%llx.%llx %llu got cap refs on %s\n", ceph_vinop(inode), + off, ceph_cap_string(got)); + + if ((got & (CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_LAZYIO)) || + !ceph_has_inline_data(ci)) { + CEPH_DEFINE_RW_CONTEXT(rw_ctx, got); + ceph_add_rw_context(fi, &rw_ctx); + ret = filemap_fault(vmf); + ceph_del_rw_context(fi, &rw_ctx); + doutc(cl, "%llx.%llx %llu drop cap refs %s ret %x\n", + ceph_vinop(inode), off, ceph_cap_string(got), ret); + } else + err = -EAGAIN; + + ceph_put_cap_refs(ci, got); + + if (err != -EAGAIN) + goto out_restore; + + /* read inline data */ + if (off >= PAGE_SIZE) { + /* does not support inline data > PAGE_SIZE */ + ret = VM_FAULT_SIGBUS; + } else { + struct address_space *mapping = inode->i_mapping; + struct page *page; + + filemap_invalidate_lock_shared(mapping); + page = find_or_create_page(mapping, 0, + mapping_gfp_constraint(mapping, ~__GFP_FS)); + if (!page) { + ret = VM_FAULT_OOM; + goto out_inline; + } + err = __ceph_do_getattr(inode, page, + CEPH_STAT_CAP_INLINE_DATA, true); + if (err < 0 || off >= i_size_read(inode)) { + unlock_page(page); + put_page(page); + ret = vmf_error(err); + goto out_inline; + } + if (err < PAGE_SIZE) + zero_user_segment(page, err, PAGE_SIZE); + else + flush_dcache_page(page); + SetPageUptodate(page); + vmf->page = page; + ret = VM_FAULT_MAJOR | VM_FAULT_LOCKED; +out_inline: + filemap_invalidate_unlock_shared(mapping); + doutc(cl, "%llx.%llx %llu read inline data ret %x\n", + ceph_vinop(inode), off, ret); + } +out_restore: + ceph_restore_sigs(&oldset); + if (err < 0) + ret = vmf_error(err); + + return ret; +} + +static vm_fault_t ceph_page_mkwrite(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct inode *inode = file_inode(vma->vm_file); + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_file_info *fi = vma->vm_file->private_data; + struct ceph_cap_flush *prealloc_cf; + struct folio *folio = page_folio(vmf->page); + loff_t off = folio_pos(folio); + loff_t size = i_size_read(inode); + size_t len; + int want, got, err; + sigset_t oldset; + vm_fault_t ret = VM_FAULT_SIGBUS; + + if (ceph_inode_is_shutdown(inode)) + return ret; + + prealloc_cf = ceph_alloc_cap_flush(); + if (!prealloc_cf) + return VM_FAULT_OOM; + + sb_start_pagefault(inode->i_sb); + ceph_block_sigs(&oldset); + + if (off + folio_size(folio) <= size) + len = folio_size(folio); + else + len = offset_in_folio(folio, size); + + doutc(cl, "%llx.%llx %llu~%zd getting caps i_size %llu\n", + ceph_vinop(inode), off, len, size); + if (fi->fmode & CEPH_FILE_MODE_LAZY) + want = CEPH_CAP_FILE_BUFFER | CEPH_CAP_FILE_LAZYIO; + else + want = CEPH_CAP_FILE_BUFFER; + + got = 0; + err = ceph_get_caps(vma->vm_file, CEPH_CAP_FILE_WR, want, off + len, &got); + if (err < 0) + goto out_free; + + doutc(cl, "%llx.%llx %llu~%zd got cap refs on %s\n", ceph_vinop(inode), + off, len, ceph_cap_string(got)); + + /* Update time before taking folio lock */ + file_update_time(vma->vm_file); + inode_inc_iversion_raw(inode); + + do { + struct ceph_snap_context *snapc; + + folio_lock(folio); + + if (folio_mkwrite_check_truncate(folio, inode) < 0) { + folio_unlock(folio); + ret = VM_FAULT_NOPAGE; + break; + } + + snapc = ceph_find_incompatible(folio); + if (!snapc) { + /* success. we'll keep the folio locked. */ + folio_mark_dirty(folio); + ret = VM_FAULT_LOCKED; + break; + } + + folio_unlock(folio); + + if (IS_ERR(snapc)) { + ret = VM_FAULT_SIGBUS; + break; + } + + ceph_queue_writeback(inode); + err = wait_event_killable(ci->i_cap_wq, + context_is_writeable_or_written(inode, snapc)); + ceph_put_snap_context(snapc); + } while (err == 0); + + if (ret == VM_FAULT_LOCKED) { + int dirty; + spin_lock(&ci->i_ceph_lock); + dirty = __ceph_mark_dirty_caps(ci, CEPH_CAP_FILE_WR, + &prealloc_cf); + spin_unlock(&ci->i_ceph_lock); + if (dirty) + __mark_inode_dirty(inode, dirty); + } + + doutc(cl, "%llx.%llx %llu~%zd dropping cap refs on %s ret %x\n", + ceph_vinop(inode), off, len, ceph_cap_string(got), ret); + ceph_put_cap_refs_async(ci, got); +out_free: + ceph_restore_sigs(&oldset); + sb_end_pagefault(inode->i_sb); + ceph_free_cap_flush(prealloc_cf); + if (err < 0) + ret = vmf_error(err); + return ret; +} + +void ceph_fill_inline_data(struct inode *inode, struct page *locked_page, + char *data, size_t len) +{ + struct ceph_client *cl = ceph_inode_to_client(inode); + struct address_space *mapping = inode->i_mapping; + struct page *page; + + if (locked_page) { + page = locked_page; + } else { + if (i_size_read(inode) == 0) + return; + page = find_or_create_page(mapping, 0, + mapping_gfp_constraint(mapping, + ~__GFP_FS)); + if (!page) + return; + if (PageUptodate(page)) { + unlock_page(page); + put_page(page); + return; + } + } + + doutc(cl, "%p %llx.%llx len %zu locked_page %p\n", inode, + ceph_vinop(inode), len, locked_page); + + if (len > 0) { + void *kaddr = kmap_atomic(page); + memcpy(kaddr, data, len); + kunmap_atomic(kaddr); + } + + if (page != locked_page) { + if (len < PAGE_SIZE) + zero_user_segment(page, len, PAGE_SIZE); + else + flush_dcache_page(page); + + SetPageUptodate(page); + unlock_page(page); + put_page(page); + } +} + +int ceph_uninline_data(struct file *file) +{ + struct inode *inode = file_inode(file); + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + struct ceph_client *cl = fsc->client; + struct ceph_osd_request *req = NULL; + struct ceph_cap_flush *prealloc_cf = NULL; + struct folio *folio = NULL; + struct ceph_snap_context *snapc = NULL; + u64 inline_version = CEPH_INLINE_NONE; + struct page *pages[1]; + int err = 0; + u64 len; + + spin_lock(&ci->i_ceph_lock); + inline_version = ci->i_inline_version; + spin_unlock(&ci->i_ceph_lock); + + doutc(cl, "%llx.%llx inline_version %llu\n", ceph_vinop(inode), + inline_version); + + if (ceph_inode_is_shutdown(inode)) { + err = -EIO; + goto out; + } + + if (inline_version == CEPH_INLINE_NONE) + return 0; + + prealloc_cf = ceph_alloc_cap_flush(); + if (!prealloc_cf) + return -ENOMEM; + + if (inline_version == 1) /* initial version, no data */ + goto out_uninline; + + down_read(&fsc->mdsc->snap_rwsem); + spin_lock(&ci->i_ceph_lock); + if (__ceph_have_pending_cap_snap(ci)) { + struct ceph_cap_snap *capsnap = + list_last_entry(&ci->i_cap_snaps, + struct ceph_cap_snap, + ci_item); + snapc = ceph_get_snap_context(capsnap->context); + } else { + if (!ci->i_head_snapc) { + ci->i_head_snapc = ceph_get_snap_context( + ci->i_snap_realm->cached_context); + } + snapc = ceph_get_snap_context(ci->i_head_snapc); + } + spin_unlock(&ci->i_ceph_lock); + up_read(&fsc->mdsc->snap_rwsem); + + folio = read_mapping_folio(inode->i_mapping, 0, file); + if (IS_ERR(folio)) { + err = PTR_ERR(folio); + goto out; + } + + folio_lock(folio); + + len = i_size_read(inode); + if (len > folio_size(folio)) + len = folio_size(folio); + + req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, + ceph_vino(inode), 0, &len, 0, 1, + CEPH_OSD_OP_CREATE, CEPH_OSD_FLAG_WRITE, + snapc, 0, 0, false); + if (IS_ERR(req)) { + err = PTR_ERR(req); + goto out_unlock; + } + + req->r_mtime = inode_get_mtime(inode); + ceph_osdc_start_request(&fsc->client->osdc, req); + err = ceph_osdc_wait_request(&fsc->client->osdc, req); + ceph_osdc_put_request(req); + if (err < 0) + goto out_unlock; + + req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, + ceph_vino(inode), 0, &len, 1, 3, + CEPH_OSD_OP_WRITE, CEPH_OSD_FLAG_WRITE, + snapc, ci->i_truncate_seq, + ci->i_truncate_size, false); + if (IS_ERR(req)) { + err = PTR_ERR(req); + goto out_unlock; + } + + pages[0] = folio_page(folio, 0); + osd_req_op_extent_osd_data_pages(req, 1, pages, len, 0, false, false); + + { + __le64 xattr_buf = cpu_to_le64(inline_version); + err = osd_req_op_xattr_init(req, 0, CEPH_OSD_OP_CMPXATTR, + "inline_version", &xattr_buf, + sizeof(xattr_buf), + CEPH_OSD_CMPXATTR_OP_GT, + CEPH_OSD_CMPXATTR_MODE_U64); + if (err) + goto out_put_req; + } + + { + char xattr_buf[32]; + int xattr_len = snprintf(xattr_buf, sizeof(xattr_buf), + "%llu", inline_version); + err = osd_req_op_xattr_init(req, 2, CEPH_OSD_OP_SETXATTR, + "inline_version", + xattr_buf, xattr_len, 0, 0); + if (err) + goto out_put_req; + } + + req->r_mtime = inode_get_mtime(inode); + ceph_osdc_start_request(&fsc->client->osdc, req); + err = ceph_osdc_wait_request(&fsc->client->osdc, req); + + ceph_update_write_metrics(&fsc->mdsc->metric, req->r_start_latency, + req->r_end_latency, len, err); + +out_uninline: + if (!err) { + int dirty; + + /* Set to CAP_INLINE_NONE and dirty the caps */ + down_read(&fsc->mdsc->snap_rwsem); + spin_lock(&ci->i_ceph_lock); + ci->i_inline_version = CEPH_INLINE_NONE; + dirty = __ceph_mark_dirty_caps(ci, CEPH_CAP_FILE_WR, &prealloc_cf); + spin_unlock(&ci->i_ceph_lock); + up_read(&fsc->mdsc->snap_rwsem); + if (dirty) + __mark_inode_dirty(inode, dirty); + } +out_put_req: + ceph_osdc_put_request(req); + if (err == -ECANCELED) + err = 0; +out_unlock: + if (folio) { + folio_unlock(folio); + folio_put(folio); + } +out: + ceph_put_snap_context(snapc); + ceph_free_cap_flush(prealloc_cf); + doutc(cl, "%llx.%llx inline_version %llu = %d\n", + ceph_vinop(inode), inline_version, err); + return err; +} + +static const struct vm_operations_struct ceph_vmops = { + .fault = ceph_filemap_fault, + .page_mkwrite = ceph_page_mkwrite, +}; + +int ceph_mmap_prepare(struct vm_area_desc *desc) +{ + struct address_space *mapping = desc->file->f_mapping; + + if (!mapping->a_ops->read_folio) + return -ENOEXEC; + desc->vm_ops = &ceph_vmops; + return 0; +} + +enum { + POOL_READ = 1, + POOL_WRITE = 2, +}; + +static int __ceph_pool_perm_get(struct ceph_inode_info *ci, + s64 pool, struct ceph_string *pool_ns) +{ + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(&ci->netfs.inode); + struct ceph_mds_client *mdsc = fsc->mdsc; + struct ceph_client *cl = fsc->client; + struct ceph_osd_request *rd_req = NULL, *wr_req = NULL; + struct rb_node **p, *parent; + struct ceph_pool_perm *perm; + struct page **pages; + size_t pool_ns_len; + int err = 0, err2 = 0, have = 0; + + down_read(&mdsc->pool_perm_rwsem); + p = &mdsc->pool_perm_tree.rb_node; + while (*p) { + perm = rb_entry(*p, struct ceph_pool_perm, node); + if (pool < perm->pool) + p = &(*p)->rb_left; + else if (pool > perm->pool) + p = &(*p)->rb_right; + else { + int ret = ceph_compare_string(pool_ns, + perm->pool_ns, + perm->pool_ns_len); + if (ret < 0) + p = &(*p)->rb_left; + else if (ret > 0) + p = &(*p)->rb_right; + else { + have = perm->perm; + break; + } + } + } + up_read(&mdsc->pool_perm_rwsem); + if (*p) + goto out; + + if (pool_ns) + doutc(cl, "pool %lld ns %.*s no perm cached\n", pool, + (int)pool_ns->len, pool_ns->str); + else + doutc(cl, "pool %lld no perm cached\n", pool); + + down_write(&mdsc->pool_perm_rwsem); + p = &mdsc->pool_perm_tree.rb_node; + parent = NULL; + while (*p) { + parent = *p; + perm = rb_entry(parent, struct ceph_pool_perm, node); + if (pool < perm->pool) + p = &(*p)->rb_left; + else if (pool > perm->pool) + p = &(*p)->rb_right; + else { + int ret = ceph_compare_string(pool_ns, + perm->pool_ns, + perm->pool_ns_len); + if (ret < 0) + p = &(*p)->rb_left; + else if (ret > 0) + p = &(*p)->rb_right; + else { + have = perm->perm; + break; + } + } + } + if (*p) { + up_write(&mdsc->pool_perm_rwsem); + goto out; + } + + rd_req = ceph_osdc_alloc_request(&fsc->client->osdc, NULL, + 1, false, GFP_NOFS); + if (!rd_req) { + err = -ENOMEM; + goto out_unlock; + } + + rd_req->r_flags = CEPH_OSD_FLAG_READ; + osd_req_op_init(rd_req, 0, CEPH_OSD_OP_STAT, 0); + rd_req->r_base_oloc.pool = pool; + if (pool_ns) + rd_req->r_base_oloc.pool_ns = ceph_get_string(pool_ns); + ceph_oid_printf(&rd_req->r_base_oid, "%llx.00000000", ci->i_vino.ino); + + err = ceph_osdc_alloc_messages(rd_req, GFP_NOFS); + if (err) + goto out_unlock; + + wr_req = ceph_osdc_alloc_request(&fsc->client->osdc, NULL, + 1, false, GFP_NOFS); + if (!wr_req) { + err = -ENOMEM; + goto out_unlock; + } + + wr_req->r_flags = CEPH_OSD_FLAG_WRITE; + osd_req_op_init(wr_req, 0, CEPH_OSD_OP_CREATE, CEPH_OSD_OP_FLAG_EXCL); + ceph_oloc_copy(&wr_req->r_base_oloc, &rd_req->r_base_oloc); + ceph_oid_copy(&wr_req->r_base_oid, &rd_req->r_base_oid); + + err = ceph_osdc_alloc_messages(wr_req, GFP_NOFS); + if (err) + goto out_unlock; + + /* one page should be large enough for STAT data */ + pages = ceph_alloc_page_vector(1, GFP_KERNEL); + if (IS_ERR(pages)) { + err = PTR_ERR(pages); + goto out_unlock; + } + + osd_req_op_raw_data_in_pages(rd_req, 0, pages, PAGE_SIZE, + 0, false, true); + ceph_osdc_start_request(&fsc->client->osdc, rd_req); + + wr_req->r_mtime = inode_get_mtime(&ci->netfs.inode); + ceph_osdc_start_request(&fsc->client->osdc, wr_req); + + err = ceph_osdc_wait_request(&fsc->client->osdc, rd_req); + err2 = ceph_osdc_wait_request(&fsc->client->osdc, wr_req); + + if (err >= 0 || err == -ENOENT) + have |= POOL_READ; + else if (err != -EPERM) { + if (err == -EBLOCKLISTED) + fsc->blocklisted = true; + goto out_unlock; + } + + if (err2 == 0 || err2 == -EEXIST) + have |= POOL_WRITE; + else if (err2 != -EPERM) { + if (err2 == -EBLOCKLISTED) + fsc->blocklisted = true; + err = err2; + goto out_unlock; + } + + pool_ns_len = pool_ns ? pool_ns->len : 0; + perm = kmalloc_flex(*perm, pool_ns, pool_ns_len + 1, GFP_NOFS); + if (!perm) { + err = -ENOMEM; + goto out_unlock; + } + + perm->pool = pool; + perm->perm = have; + perm->pool_ns_len = pool_ns_len; + if (pool_ns_len > 0) + memcpy(perm->pool_ns, pool_ns->str, pool_ns_len); + perm->pool_ns[pool_ns_len] = 0; + + rb_link_node(&perm->node, parent, p); + rb_insert_color(&perm->node, &mdsc->pool_perm_tree); + err = 0; +out_unlock: + up_write(&mdsc->pool_perm_rwsem); + + ceph_osdc_put_request(rd_req); + ceph_osdc_put_request(wr_req); +out: + if (!err) + err = have; + if (pool_ns) + doutc(cl, "pool %lld ns %.*s result = %d\n", pool, + (int)pool_ns->len, pool_ns->str, err); + else + doutc(cl, "pool %lld result = %d\n", pool, err); + return err; +} + +int ceph_pool_perm_check(struct inode *inode, int need) +{ + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_string *pool_ns; + s64 pool; + int ret; + unsigned long flags; + + /* Only need to do this for regular files */ + if (!S_ISREG(inode->i_mode)) + return 0; + + if (ci->i_vino.snap != CEPH_NOSNAP) { + /* + * Pool permission check needs to write to the first object. + * But for snapshot, head of the first object may have already + * been deleted. Skip check to avoid creating orphan object. + */ + return 0; + } + + if (ceph_test_mount_opt(ceph_inode_to_fs_client(inode), + NOPOOLPERM)) + return 0; + + spin_lock(&ci->i_ceph_lock); + flags = ci->i_ceph_flags; + pool = ci->i_layout.pool_id; + spin_unlock(&ci->i_ceph_lock); +check: + if (flags & CEPH_I_POOL_PERM) { + if ((need & CEPH_CAP_FILE_RD) && !(flags & CEPH_I_POOL_RD)) { + doutc(cl, "pool %lld no read perm\n", pool); + return -EPERM; + } + if ((need & CEPH_CAP_FILE_WR) && !(flags & CEPH_I_POOL_WR)) { + doutc(cl, "pool %lld no write perm\n", pool); + return -EPERM; + } + return 0; + } + + pool_ns = ceph_try_get_string(ci->i_layout.pool_ns); + ret = __ceph_pool_perm_get(ci, pool, pool_ns); + ceph_put_string(pool_ns); + if (ret < 0) + return ret; + + spin_lock(&ci->i_ceph_lock); + if (pool == ci->i_layout.pool_id && + pool_ns == rcu_dereference_raw(ci->i_layout.pool_ns)) { + set_bit(CEPH_I_POOL_PERM_BIT, &ci->i_ceph_flags); + if (ret & POOL_READ) + set_bit(CEPH_I_POOL_RD_BIT, &ci->i_ceph_flags); + if (ret & POOL_WRITE) + set_bit(CEPH_I_POOL_WR_BIT, &ci->i_ceph_flags); + } else { + pool = ci->i_layout.pool_id; + } + /* Re-read flags under the lock so check: sees the updated bits. */ + flags = ci->i_ceph_flags; + spin_unlock(&ci->i_ceph_lock); + goto check; +} + +void ceph_pool_perm_destroy(struct ceph_mds_client *mdsc) +{ + struct ceph_pool_perm *perm; + struct rb_node *n; + + while (!RB_EMPTY_ROOT(&mdsc->pool_perm_tree)) { + n = rb_first(&mdsc->pool_perm_tree); + perm = rb_entry(n, struct ceph_pool_perm, node); + rb_erase(n, &mdsc->pool_perm_tree); + kfree(perm); + } +} diff --git a/fs/ceph/cache.c b/fs/ceph/cache.c new file mode 100644 index 000000000..f678bab18 --- /dev/null +++ b/fs/ceph/cache.c @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Ceph cache definitions. + * + * Copyright (C) 2013 by Adfin Solutions, Inc. All Rights Reserved. + * Written by Milosz Tanski (milosz@adfin.com) + */ + +#include + +#include +#include "super.h" +#include "cache.h" + +void ceph_fscache_register_inode_cookie(struct inode *inode) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + + /* No caching for filesystem? */ + if (!fsc->fscache) + return; + + /* Regular files only */ + if (!S_ISREG(inode->i_mode)) + return; + + /* Only new inodes! */ + if (!(inode_state_read_once(inode) & I_NEW)) + return; + + WARN_ON_ONCE(ci->netfs.cache); + + ci->netfs.cache = + fscache_acquire_cookie(fsc->fscache, 0, + &ci->i_vino, sizeof(ci->i_vino), + &ci->i_version, sizeof(ci->i_version), + i_size_read(inode)); + if (ci->netfs.cache) + mapping_set_release_always(inode->i_mapping); +} + +void ceph_fscache_unregister_inode_cookie(struct ceph_inode_info *ci) +{ + fscache_relinquish_cookie(ceph_fscache_cookie(ci), false); +} + +void ceph_fscache_use_cookie(struct inode *inode, bool will_modify) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + + fscache_use_cookie(ceph_fscache_cookie(ci), will_modify); +} + +void ceph_fscache_unuse_cookie(struct inode *inode, bool update) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + + if (update) { + loff_t i_size = i_size_read(inode); + + fscache_unuse_cookie(ceph_fscache_cookie(ci), + &ci->i_version, &i_size); + } else { + fscache_unuse_cookie(ceph_fscache_cookie(ci), NULL, NULL); + } +} + +void ceph_fscache_update(struct inode *inode) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + loff_t i_size = i_size_read(inode); + + fscache_update_cookie(ceph_fscache_cookie(ci), &ci->i_version, &i_size); +} + +void ceph_fscache_invalidate(struct inode *inode, bool dio_write) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + + fscache_invalidate(ceph_fscache_cookie(ci), + &ci->i_version, i_size_read(inode), + dio_write ? FSCACHE_INVAL_DIO_WRITE : 0); +} + +int ceph_fscache_register_fs(struct ceph_fs_client* fsc, struct fs_context *fc) +{ + const struct ceph_fsid *fsid = &fsc->client->fsid; + const char *fscache_uniq = fsc->mount_options->fscache_uniq; + size_t uniq_len = fscache_uniq ? strlen(fscache_uniq) : 0; + char *name; + int err = 0; + + name = kasprintf(GFP_KERNEL, "ceph,%pU%s%s", fsid, uniq_len ? "," : "", + uniq_len ? fscache_uniq : ""); + if (!name) + return -ENOMEM; + + fsc->fscache = fscache_acquire_volume(name, NULL, NULL, 0); + if (IS_ERR_OR_NULL(fsc->fscache)) { + errorfc(fc, "Unable to register fscache cookie for %s", name); + err = fsc->fscache ? PTR_ERR(fsc->fscache) : -EOPNOTSUPP; + fsc->fscache = NULL; + } + kfree(name); + return err; +} + +void ceph_fscache_unregister_fs(struct ceph_fs_client* fsc) +{ + fscache_relinquish_volume(fsc->fscache, NULL, false); +} diff --git a/fs/ceph/cache.h b/fs/ceph/cache.h new file mode 100644 index 000000000..20efac020 --- /dev/null +++ b/fs/ceph/cache.h @@ -0,0 +1,117 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Ceph cache definitions. + * + * Copyright (C) 2013 by Adfin Solutions, Inc. All Rights Reserved. + * Written by Milosz Tanski (milosz@adfin.com) + */ + +#ifndef _CEPH_CACHE_H +#define _CEPH_CACHE_H + +#include + +#ifdef CONFIG_CEPH_FSCACHE +#include + +int ceph_fscache_register_fs(struct ceph_fs_client* fsc, struct fs_context *fc); +void ceph_fscache_unregister_fs(struct ceph_fs_client* fsc); + +void ceph_fscache_register_inode_cookie(struct inode *inode); +void ceph_fscache_unregister_inode_cookie(struct ceph_inode_info* ci); + +void ceph_fscache_use_cookie(struct inode *inode, bool will_modify); +void ceph_fscache_unuse_cookie(struct inode *inode, bool update); + +void ceph_fscache_update(struct inode *inode); +void ceph_fscache_invalidate(struct inode *inode, bool dio_write); + +static inline struct fscache_cookie *ceph_fscache_cookie(struct ceph_inode_info *ci) +{ + return netfs_i_cookie(&ci->netfs); +} + +static inline void ceph_fscache_resize(struct inode *inode, loff_t to) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct fscache_cookie *cookie = ceph_fscache_cookie(ci); + + if (cookie) { + ceph_fscache_use_cookie(inode, true); + fscache_resize_cookie(cookie, to); + ceph_fscache_unuse_cookie(inode, true); + } +} + +static inline int ceph_fscache_unpin_writeback(struct inode *inode, + struct writeback_control *wbc) +{ + return netfs_unpin_writeback(inode, wbc); +} + +#define ceph_fscache_dirty_folio netfs_dirty_folio + +static inline bool ceph_is_cache_enabled(struct inode *inode) +{ + return fscache_cookie_enabled(ceph_fscache_cookie(ceph_inode(inode))); +} + +#else /* CONFIG_CEPH_FSCACHE */ +static inline int ceph_fscache_register_fs(struct ceph_fs_client* fsc, + struct fs_context *fc) +{ + return 0; +} + +static inline void ceph_fscache_unregister_fs(struct ceph_fs_client* fsc) +{ +} + +static inline void ceph_fscache_register_inode_cookie(struct inode *inode) +{ +} + +static inline void ceph_fscache_unregister_inode_cookie(struct ceph_inode_info* ci) +{ +} + +static inline void ceph_fscache_use_cookie(struct inode *inode, bool will_modify) +{ +} + +static inline void ceph_fscache_unuse_cookie(struct inode *inode, bool update) +{ +} + +static inline void ceph_fscache_update(struct inode *inode) +{ +} + +static inline void ceph_fscache_invalidate(struct inode *inode, bool dio_write) +{ +} + +static inline struct fscache_cookie *ceph_fscache_cookie(struct ceph_inode_info *ci) +{ + return NULL; +} + +static inline void ceph_fscache_resize(struct inode *inode, loff_t to) +{ +} + +static inline int ceph_fscache_unpin_writeback(struct inode *inode, + struct writeback_control *wbc) +{ + return 0; +} + +#define ceph_fscache_dirty_folio filemap_dirty_folio + +static inline bool ceph_is_cache_enabled(struct inode *inode) +{ + return false; +} +#endif /* CONFIG_CEPH_FSCACHE */ + +#endif diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c new file mode 100644 index 000000000..d7283fb54 --- /dev/null +++ b/fs/ceph/caps.c @@ -0,0 +1,5126 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "mds_client.h" +#include "cache.h" +#include "crypto.h" +#include +#include +#include + +/* + * Capability management + * + * The Ceph metadata servers control client access to inode metadata + * and file data by issuing capabilities, granting clients permission + * to read and/or write both inode field and file data to OSDs + * (storage nodes). Each capability consists of a set of bits + * indicating which operations are allowed. + * + * If the client holds a *_SHARED cap, the client has a coherent value + * that can be safely read from the cached inode. + * + * In the case of a *_EXCL (exclusive) or FILE_WR capabilities, the + * client is allowed to change inode attributes (e.g., file size, + * mtime), note its dirty state in the ceph_cap, and asynchronously + * flush that metadata change to the MDS. + * + * In the event of a conflicting operation (perhaps by another + * client), the MDS will revoke the conflicting client capabilities. + * + * In order for a client to cache an inode, it must hold a capability + * with at least one MDS server. When inodes are released, release + * notifications are batched and periodically sent en masse to the MDS + * cluster to release server state. + */ + +static u64 __get_oldest_flush_tid(struct ceph_mds_client *mdsc); +static void __kick_flushing_caps(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session, + struct ceph_inode_info *ci, + u64 oldest_flush_tid); + +/* + * Generate readable cap strings for debugging output. + */ +#define MAX_CAP_STR 20 +static char cap_str[MAX_CAP_STR][40]; +static DEFINE_SPINLOCK(cap_str_lock); +static int last_cap_str; + +static char *gcap_string(char *s, int c) +{ + if (c & CEPH_CAP_GSHARED) + *s++ = 's'; + if (c & CEPH_CAP_GEXCL) + *s++ = 'x'; + if (c & CEPH_CAP_GCACHE) + *s++ = 'c'; + if (c & CEPH_CAP_GRD) + *s++ = 'r'; + if (c & CEPH_CAP_GWR) + *s++ = 'w'; + if (c & CEPH_CAP_GBUFFER) + *s++ = 'b'; + if (c & CEPH_CAP_GWREXTEND) + *s++ = 'a'; + if (c & CEPH_CAP_GLAZYIO) + *s++ = 'l'; + return s; +} + +const char *ceph_cap_string(int caps) +{ + int i; + char *s; + int c; + + spin_lock(&cap_str_lock); + i = last_cap_str++; + if (last_cap_str == MAX_CAP_STR) + last_cap_str = 0; + spin_unlock(&cap_str_lock); + + s = cap_str[i]; + + if (caps & CEPH_CAP_PIN) + *s++ = 'p'; + + c = (caps >> CEPH_CAP_SAUTH) & 3; + if (c) { + *s++ = 'A'; + s = gcap_string(s, c); + } + + c = (caps >> CEPH_CAP_SLINK) & 3; + if (c) { + *s++ = 'L'; + s = gcap_string(s, c); + } + + c = (caps >> CEPH_CAP_SXATTR) & 3; + if (c) { + *s++ = 'X'; + s = gcap_string(s, c); + } + + c = caps >> CEPH_CAP_SFILE; + if (c) { + *s++ = 'F'; + s = gcap_string(s, c); + } + + if (s == cap_str[i]) + *s++ = '-'; + *s = 0; + return cap_str[i]; +} + +void ceph_caps_init(struct ceph_mds_client *mdsc) +{ + INIT_LIST_HEAD(&mdsc->caps_list); + spin_lock_init(&mdsc->caps_list_lock); +} + +void ceph_caps_finalize(struct ceph_mds_client *mdsc) +{ + struct ceph_cap *cap; + + spin_lock(&mdsc->caps_list_lock); + while (!list_empty(&mdsc->caps_list)) { + cap = list_first_entry(&mdsc->caps_list, + struct ceph_cap, caps_item); + list_del(&cap->caps_item); + kmem_cache_free(ceph_cap_cachep, cap); + } + mdsc->caps_total_count = 0; + mdsc->caps_avail_count = 0; + mdsc->caps_use_count = 0; + mdsc->caps_reserve_count = 0; + mdsc->caps_min_count = 0; + spin_unlock(&mdsc->caps_list_lock); +} + +void ceph_adjust_caps_max_min(struct ceph_mds_client *mdsc, + struct ceph_mount_options *fsopt) +{ + spin_lock(&mdsc->caps_list_lock); + mdsc->caps_min_count = fsopt->max_readdir; + if (mdsc->caps_min_count < 1024) + mdsc->caps_min_count = 1024; + mdsc->caps_use_max = fsopt->caps_max; + if (mdsc->caps_use_max > 0 && + mdsc->caps_use_max < mdsc->caps_min_count) + mdsc->caps_use_max = mdsc->caps_min_count; + spin_unlock(&mdsc->caps_list_lock); +} + +static void __ceph_unreserve_caps(struct ceph_mds_client *mdsc, int nr_caps) +{ + struct ceph_cap *cap; + int i; + + if (nr_caps) { + BUG_ON(mdsc->caps_reserve_count < nr_caps); + mdsc->caps_reserve_count -= nr_caps; + if (mdsc->caps_avail_count >= + mdsc->caps_reserve_count + mdsc->caps_min_count) { + mdsc->caps_total_count -= nr_caps; + for (i = 0; i < nr_caps; i++) { + cap = list_first_entry(&mdsc->caps_list, + struct ceph_cap, caps_item); + list_del(&cap->caps_item); + kmem_cache_free(ceph_cap_cachep, cap); + } + } else { + mdsc->caps_avail_count += nr_caps; + } + + doutc(mdsc->fsc->client, + "caps %d = %d used + %d resv + %d avail\n", + mdsc->caps_total_count, mdsc->caps_use_count, + mdsc->caps_reserve_count, mdsc->caps_avail_count); + BUG_ON(mdsc->caps_total_count != mdsc->caps_use_count + + mdsc->caps_reserve_count + + mdsc->caps_avail_count); + } +} + +/* + * Called under mdsc->mutex. + */ +int ceph_reserve_caps(struct ceph_mds_client *mdsc, + struct ceph_cap_reservation *ctx, int need) +{ + struct ceph_client *cl = mdsc->fsc->client; + int i, j; + struct ceph_cap *cap; + int have; + int alloc = 0; + int max_caps; + int err = 0; + bool trimmed = false; + struct ceph_mds_session *s; + LIST_HEAD(newcaps); + + doutc(cl, "ctx=%p need=%d\n", ctx, need); + + /* first reserve any caps that are already allocated */ + spin_lock(&mdsc->caps_list_lock); + if (mdsc->caps_avail_count >= need) + have = need; + else + have = mdsc->caps_avail_count; + mdsc->caps_avail_count -= have; + mdsc->caps_reserve_count += have; + BUG_ON(mdsc->caps_total_count != mdsc->caps_use_count + + mdsc->caps_reserve_count + + mdsc->caps_avail_count); + spin_unlock(&mdsc->caps_list_lock); + + for (i = have; i < need; ) { + cap = kmem_cache_alloc(ceph_cap_cachep, GFP_NOFS); + if (cap) { + list_add(&cap->caps_item, &newcaps); + alloc++; + i++; + continue; + } + + if (!trimmed) { + for (j = 0; j < mdsc->max_sessions; j++) { + s = __ceph_lookup_mds_session(mdsc, j); + if (!s) + continue; + mutex_unlock(&mdsc->mutex); + + mutex_lock(&s->s_mutex); + max_caps = s->s_nr_caps - (need - i); + ceph_trim_caps(mdsc, s, max_caps); + mutex_unlock(&s->s_mutex); + + ceph_put_mds_session(s); + mutex_lock(&mdsc->mutex); + } + trimmed = true; + + spin_lock(&mdsc->caps_list_lock); + if (mdsc->caps_avail_count) { + int more_have; + if (mdsc->caps_avail_count >= need - i) + more_have = need - i; + else + more_have = mdsc->caps_avail_count; + + i += more_have; + have += more_have; + mdsc->caps_avail_count -= more_have; + mdsc->caps_reserve_count += more_have; + + } + spin_unlock(&mdsc->caps_list_lock); + + continue; + } + + pr_warn_client(cl, "ctx=%p ENOMEM need=%d got=%d\n", ctx, need, + have + alloc); + err = -ENOMEM; + break; + } + + if (!err) { + BUG_ON(have + alloc != need); + ctx->count = need; + ctx->used = 0; + } + + spin_lock(&mdsc->caps_list_lock); + mdsc->caps_total_count += alloc; + mdsc->caps_reserve_count += alloc; + list_splice(&newcaps, &mdsc->caps_list); + + BUG_ON(mdsc->caps_total_count != mdsc->caps_use_count + + mdsc->caps_reserve_count + + mdsc->caps_avail_count); + + if (err) + __ceph_unreserve_caps(mdsc, have + alloc); + + spin_unlock(&mdsc->caps_list_lock); + + doutc(cl, "ctx=%p %d = %d used + %d resv + %d avail\n", ctx, + mdsc->caps_total_count, mdsc->caps_use_count, + mdsc->caps_reserve_count, mdsc->caps_avail_count); + return err; +} + +void ceph_unreserve_caps(struct ceph_mds_client *mdsc, + struct ceph_cap_reservation *ctx) +{ + struct ceph_client *cl = mdsc->fsc->client; + bool reclaim = false; + if (!ctx->count) + return; + + doutc(cl, "ctx=%p count=%d\n", ctx, ctx->count); + spin_lock(&mdsc->caps_list_lock); + __ceph_unreserve_caps(mdsc, ctx->count); + ctx->count = 0; + + if (mdsc->caps_use_max > 0 && + mdsc->caps_use_count > mdsc->caps_use_max) + reclaim = true; + spin_unlock(&mdsc->caps_list_lock); + + if (reclaim) + ceph_reclaim_caps_nr(mdsc, ctx->used); +} + +struct ceph_cap *ceph_get_cap(struct ceph_mds_client *mdsc, + struct ceph_cap_reservation *ctx) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_cap *cap = NULL; + + /* temporary, until we do something about cap import/export */ + if (!ctx) { + cap = kmem_cache_alloc(ceph_cap_cachep, GFP_NOFS); + if (cap) { + spin_lock(&mdsc->caps_list_lock); + mdsc->caps_use_count++; + mdsc->caps_total_count++; + spin_unlock(&mdsc->caps_list_lock); + } else { + spin_lock(&mdsc->caps_list_lock); + if (mdsc->caps_avail_count) { + BUG_ON(list_empty(&mdsc->caps_list)); + + mdsc->caps_avail_count--; + mdsc->caps_use_count++; + cap = list_first_entry(&mdsc->caps_list, + struct ceph_cap, caps_item); + list_del(&cap->caps_item); + + BUG_ON(mdsc->caps_total_count != mdsc->caps_use_count + + mdsc->caps_reserve_count + mdsc->caps_avail_count); + } + spin_unlock(&mdsc->caps_list_lock); + } + + return cap; + } + + spin_lock(&mdsc->caps_list_lock); + doutc(cl, "ctx=%p (%d) %d = %d used + %d resv + %d avail\n", ctx, + ctx->count, mdsc->caps_total_count, mdsc->caps_use_count, + mdsc->caps_reserve_count, mdsc->caps_avail_count); + BUG_ON(!ctx->count); + BUG_ON(ctx->count > mdsc->caps_reserve_count); + BUG_ON(list_empty(&mdsc->caps_list)); + + ctx->count--; + ctx->used++; + mdsc->caps_reserve_count--; + mdsc->caps_use_count++; + + cap = list_first_entry(&mdsc->caps_list, struct ceph_cap, caps_item); + list_del(&cap->caps_item); + + BUG_ON(mdsc->caps_total_count != mdsc->caps_use_count + + mdsc->caps_reserve_count + mdsc->caps_avail_count); + spin_unlock(&mdsc->caps_list_lock); + return cap; +} + +void ceph_put_cap(struct ceph_mds_client *mdsc, struct ceph_cap *cap) +{ + struct ceph_client *cl = mdsc->fsc->client; + + spin_lock(&mdsc->caps_list_lock); + doutc(cl, "%p %d = %d used + %d resv + %d avail\n", cap, + mdsc->caps_total_count, mdsc->caps_use_count, + mdsc->caps_reserve_count, mdsc->caps_avail_count); + mdsc->caps_use_count--; + /* + * Keep some preallocated caps around (ceph_min_count), to + * avoid lots of free/alloc churn. + */ + if (mdsc->caps_avail_count >= mdsc->caps_reserve_count + + mdsc->caps_min_count) { + mdsc->caps_total_count--; + kmem_cache_free(ceph_cap_cachep, cap); + } else { + mdsc->caps_avail_count++; + list_add(&cap->caps_item, &mdsc->caps_list); + } + + BUG_ON(mdsc->caps_total_count != mdsc->caps_use_count + + mdsc->caps_reserve_count + mdsc->caps_avail_count); + spin_unlock(&mdsc->caps_list_lock); +} + +void ceph_reservation_status(struct ceph_fs_client *fsc, + int *total, int *avail, int *used, int *reserved, + int *min) +{ + struct ceph_mds_client *mdsc = fsc->mdsc; + + spin_lock(&mdsc->caps_list_lock); + + if (total) + *total = mdsc->caps_total_count; + if (avail) + *avail = mdsc->caps_avail_count; + if (used) + *used = mdsc->caps_use_count; + if (reserved) + *reserved = mdsc->caps_reserve_count; + if (min) + *min = mdsc->caps_min_count; + + spin_unlock(&mdsc->caps_list_lock); +} + +/* + * Find ceph_cap for given mds, if any. + * + * Called with i_ceph_lock held. + */ +struct ceph_cap *__get_cap_for_mds(struct ceph_inode_info *ci, int mds) +{ + struct ceph_cap *cap; + struct rb_node *n = ci->i_caps.rb_node; + + while (n) { + cap = rb_entry(n, struct ceph_cap, ci_node); + if (mds < cap->mds) + n = n->rb_left; + else if (mds > cap->mds) + n = n->rb_right; + else + return cap; + } + return NULL; +} + +struct ceph_cap *ceph_get_cap_for_mds(struct ceph_inode_info *ci, int mds) +{ + struct ceph_cap *cap; + + spin_lock(&ci->i_ceph_lock); + cap = __get_cap_for_mds(ci, mds); + spin_unlock(&ci->i_ceph_lock); + return cap; +} + +/* + * Called under i_ceph_lock. + */ +static void __insert_cap_node(struct ceph_inode_info *ci, + struct ceph_cap *new) +{ + struct rb_node **p = &ci->i_caps.rb_node; + struct rb_node *parent = NULL; + struct ceph_cap *cap = NULL; + + while (*p) { + parent = *p; + cap = rb_entry(parent, struct ceph_cap, ci_node); + if (new->mds < cap->mds) + p = &(*p)->rb_left; + else if (new->mds > cap->mds) + p = &(*p)->rb_right; + else + BUG(); + } + + rb_link_node(&new->ci_node, parent, p); + rb_insert_color(&new->ci_node, &ci->i_caps); +} + +/* + * (re)set cap hold timeouts, which control the delayed release + * of unused caps back to the MDS. Should be called on cap use. + */ +static void __cap_set_timeouts(struct ceph_mds_client *mdsc, + struct ceph_inode_info *ci) +{ + struct inode *inode = &ci->netfs.inode; + struct ceph_mount_options *opt = mdsc->fsc->mount_options; + + ci->i_hold_caps_max = round_jiffies(jiffies + + opt->caps_wanted_delay_max * HZ); + doutc(mdsc->fsc->client, "%p %llx.%llx %lu\n", inode, + ceph_vinop(inode), ci->i_hold_caps_max - jiffies); +} + +/* + * (Re)queue cap at the end of the delayed cap release list. + * + * If I_FLUSH is set, leave the inode at the front of the list. + * + * Caller holds i_ceph_lock + * -> we take mdsc->cap_delay_lock + */ +static void __cap_delay_requeue(struct ceph_mds_client *mdsc, + struct ceph_inode_info *ci) +{ + struct inode *inode = &ci->netfs.inode; + + doutc(mdsc->fsc->client, "%p %llx.%llx flags 0x%lx at %lu\n", + inode, ceph_vinop(inode), ci->i_ceph_flags, + ci->i_hold_caps_max); + if (!mdsc->stopping) { + spin_lock(&mdsc->cap_delay_lock); + if (!list_empty(&ci->i_cap_delay_list)) { + if (ci->i_ceph_flags & CEPH_I_FLUSH) + goto no_change; + list_del_init(&ci->i_cap_delay_list); + } + __cap_set_timeouts(mdsc, ci); + list_add_tail(&ci->i_cap_delay_list, &mdsc->cap_delay_list); +no_change: + spin_unlock(&mdsc->cap_delay_lock); + } +} + +/* + * Queue an inode for immediate writeback. Mark inode with I_FLUSH, + * indicating we should send a cap message to flush dirty metadata + * asap, and move to the front of the delayed cap list. + */ +static void __cap_delay_requeue_front(struct ceph_mds_client *mdsc, + struct ceph_inode_info *ci) +{ + struct inode *inode = &ci->netfs.inode; + + doutc(mdsc->fsc->client, "%p %llx.%llx\n", inode, ceph_vinop(inode)); + spin_lock(&mdsc->cap_delay_lock); + set_bit(CEPH_I_FLUSH_BIT, &ci->i_ceph_flags); + if (!list_empty(&ci->i_cap_delay_list)) + list_del_init(&ci->i_cap_delay_list); + list_add(&ci->i_cap_delay_list, &mdsc->cap_delay_list); + spin_unlock(&mdsc->cap_delay_lock); +} + +/* + * Cancel delayed work on cap. + * + * Caller must hold i_ceph_lock. + */ +static void __cap_delay_cancel(struct ceph_mds_client *mdsc, + struct ceph_inode_info *ci) +{ + struct inode *inode = &ci->netfs.inode; + + doutc(mdsc->fsc->client, "%p %llx.%llx\n", inode, ceph_vinop(inode)); + if (list_empty(&ci->i_cap_delay_list)) + return; + spin_lock(&mdsc->cap_delay_lock); + list_del_init(&ci->i_cap_delay_list); + spin_unlock(&mdsc->cap_delay_lock); +} + +/* Common issue checks for add_cap, handle_cap_grant. */ +static void __check_cap_issue(struct ceph_inode_info *ci, struct ceph_cap *cap, + unsigned issued) +{ + struct inode *inode = &ci->netfs.inode; + struct ceph_client *cl = ceph_inode_to_client(inode); + + unsigned had = __ceph_caps_issued(ci, NULL); + + lockdep_assert_held(&ci->i_ceph_lock); + + /* + * Each time we receive FILE_CACHE anew, we increment + * i_rdcache_gen. + */ + if (S_ISREG(ci->netfs.inode.i_mode) && + (issued & (CEPH_CAP_FILE_CACHE|CEPH_CAP_FILE_LAZYIO)) && + (had & (CEPH_CAP_FILE_CACHE|CEPH_CAP_FILE_LAZYIO)) == 0) { + ci->i_rdcache_gen++; + } + + /* + * If FILE_SHARED is newly issued, mark dir not complete. We don't + * know what happened to this directory while we didn't have the cap. + * If FILE_SHARED is being revoked, also mark dir not complete. It + * stops on-going cached readdir. + */ + if ((issued & CEPH_CAP_FILE_SHARED) != (had & CEPH_CAP_FILE_SHARED)) { + if (issued & CEPH_CAP_FILE_SHARED) + atomic_inc(&ci->i_shared_gen); + if (S_ISDIR(ci->netfs.inode.i_mode)) { + doutc(cl, " marking %p NOT complete\n", inode); + __ceph_dir_clear_complete(ci); + } + } + + /* Wipe saved layout if we're losing DIR_CREATE caps */ + if (S_ISDIR(ci->netfs.inode.i_mode) && (had & CEPH_CAP_DIR_CREATE) && + !(issued & CEPH_CAP_DIR_CREATE)) { + ceph_put_string(rcu_dereference_raw(ci->i_cached_layout.pool_ns)); + memset(&ci->i_cached_layout, 0, sizeof(ci->i_cached_layout)); + } +} + +/** + * change_auth_cap_ses - move inode to appropriate lists when auth caps change + * @ci: inode to be moved + * @session: new auth caps session + */ +void change_auth_cap_ses(struct ceph_inode_info *ci, + struct ceph_mds_session *session) +{ + lockdep_assert_held(&ci->i_ceph_lock); + + if (list_empty(&ci->i_dirty_item) && list_empty(&ci->i_flushing_item)) + return; + + spin_lock(&session->s_mdsc->cap_dirty_lock); + if (!list_empty(&ci->i_dirty_item)) + list_move(&ci->i_dirty_item, &session->s_cap_dirty); + if (!list_empty(&ci->i_flushing_item)) + list_move_tail(&ci->i_flushing_item, &session->s_cap_flushing); + spin_unlock(&session->s_mdsc->cap_dirty_lock); +} + +/* + * Add a capability under the given MDS session. + * + * Caller should hold session snap_rwsem (read) and ci->i_ceph_lock + * + * @fmode is the open file mode, if we are opening a file, otherwise + * it is < 0. (This is so we can atomically add the cap and add an + * open file reference to it.) + */ +void ceph_add_cap(struct inode *inode, + struct ceph_mds_session *session, u64 cap_id, + unsigned issued, unsigned wanted, + unsigned seq, unsigned mseq, u64 realmino, int flags, + struct ceph_cap **new_cap) +{ + struct ceph_mds_client *mdsc = ceph_inode_to_fs_client(inode)->mdsc; + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_cap *cap; + int mds = session->s_mds; + int actual_wanted; + u32 gen; + + lockdep_assert_held(&ci->i_ceph_lock); + + doutc(cl, "%p %llx.%llx mds%d cap %llx %s seq %d\n", inode, + ceph_vinop(inode), session->s_mds, cap_id, + ceph_cap_string(issued), seq); + + gen = atomic_read(&session->s_cap_gen); + + cap = __get_cap_for_mds(ci, mds); + if (!cap) { + cap = *new_cap; + *new_cap = NULL; + + cap->issued = 0; + cap->implemented = 0; + cap->mds = mds; + cap->mds_wanted = 0; + cap->mseq = 0; + + cap->ci = ci; + __insert_cap_node(ci, cap); + + /* add to session cap list */ + cap->session = session; + spin_lock(&session->s_cap_lock); + list_add_tail(&cap->session_caps, &session->s_caps); + session->s_nr_caps++; + atomic64_inc(&mdsc->metric.total_caps); + spin_unlock(&session->s_cap_lock); + } else { + spin_lock(&session->s_cap_lock); + list_move_tail(&cap->session_caps, &session->s_caps); + spin_unlock(&session->s_cap_lock); + + if (cap->cap_gen < gen) + cap->issued = cap->implemented = CEPH_CAP_PIN; + + /* + * auth mds of the inode changed. we received the cap export + * message, but still haven't received the cap import message. + * handle_cap_export() updated the new auth MDS' cap. + * + * "ceph_seq_cmp(seq, cap->seq) <= 0" means we are processing + * a message that was send before the cap import message. So + * don't remove caps. + */ + if (ceph_seq_cmp(seq, cap->seq) <= 0) { + WARN_ON(cap != ci->i_auth_cap); + WARN_ON(cap->cap_id != cap_id); + seq = cap->seq; + mseq = cap->mseq; + issued |= cap->issued; + flags |= CEPH_CAP_FLAG_AUTH; + } + } + + if (!ci->i_snap_realm || + ((flags & CEPH_CAP_FLAG_AUTH) && + realmino != (u64)-1 && ci->i_snap_realm->ino != realmino)) { + /* + * add this inode to the appropriate snap realm + */ + struct ceph_snap_realm *realm = ceph_lookup_snap_realm(mdsc, + realmino); + if (realm) + ceph_change_snap_realm(inode, realm); + else + WARN(1, "%s: couldn't find snap realm 0x%llx (ino 0x%llx oldrealm 0x%llx)\n", + __func__, realmino, ci->i_vino.ino, + ci->i_snap_realm ? ci->i_snap_realm->ino : 0); + } + + __check_cap_issue(ci, cap, issued); + + /* + * If we are issued caps we don't want, or the mds' wanted + * value appears to be off, queue a check so we'll release + * later and/or update the mds wanted value. + */ + actual_wanted = __ceph_caps_wanted(ci); + if ((wanted & ~actual_wanted) || + (issued & ~actual_wanted & CEPH_CAP_ANY_WR)) { + doutc(cl, "issued %s, mds wanted %s, actual %s, queueing\n", + ceph_cap_string(issued), ceph_cap_string(wanted), + ceph_cap_string(actual_wanted)); + __cap_delay_requeue(mdsc, ci); + } + + if (flags & CEPH_CAP_FLAG_AUTH) { + if (!ci->i_auth_cap || + ceph_seq_cmp(ci->i_auth_cap->mseq, mseq) < 0) { + if (ci->i_auth_cap && + ci->i_auth_cap->session != cap->session) + change_auth_cap_ses(ci, cap->session); + ci->i_auth_cap = cap; + cap->mds_wanted = wanted; + } + } else { + WARN_ON(ci->i_auth_cap == cap); + } + + doutc(cl, "inode %p %llx.%llx cap %p %s now %s seq %d mds%d\n", + inode, ceph_vinop(inode), cap, ceph_cap_string(issued), + ceph_cap_string(issued|cap->issued), seq, mds); + cap->cap_id = cap_id; + cap->issued = issued; + cap->implemented |= issued; + if (ceph_seq_cmp(mseq, cap->mseq) > 0) + cap->mds_wanted = wanted; + else + cap->mds_wanted |= wanted; + cap->seq = seq; + cap->issue_seq = seq; + cap->mseq = mseq; + cap->cap_gen = gen; + wake_up_all(&ci->i_cap_wq); +} + +/* + * Return true if cap has not timed out and belongs to the current + * generation of the MDS session (i.e. has not gone 'stale' due to + * us losing touch with the mds). + */ +static int __cap_is_valid(struct ceph_cap *cap) +{ + struct inode *inode = &cap->ci->netfs.inode; + struct ceph_client *cl = cap->session->s_mdsc->fsc->client; + unsigned long ttl; + u32 gen; + + gen = atomic_read(&cap->session->s_cap_gen); + ttl = cap->session->s_cap_ttl; + + if (cap->cap_gen < gen || time_after_eq(jiffies, ttl)) { + doutc(cl, "%p %llx.%llx cap %p issued %s but STALE (gen %u vs %u)\n", + inode, ceph_vinop(inode), cap, + ceph_cap_string(cap->issued), cap->cap_gen, gen); + return 0; + } + + return 1; +} + +/* + * Return set of valid cap bits issued to us. Note that caps time + * out, and may be invalidated in bulk if the client session times out + * and session->s_cap_gen is bumped. + */ +int __ceph_caps_issued(struct ceph_inode_info *ci, int *implemented) +{ + struct inode *inode = &ci->netfs.inode; + struct ceph_client *cl = ceph_inode_to_client(inode); + int have = ci->i_snap_caps; + struct ceph_cap *cap; + struct rb_node *p; + + if (implemented) + *implemented = 0; + for (p = rb_first(&ci->i_caps); p; p = rb_next(p)) { + cap = rb_entry(p, struct ceph_cap, ci_node); + if (!__cap_is_valid(cap)) + continue; + doutc(cl, "%p %llx.%llx cap %p issued %s\n", inode, + ceph_vinop(inode), cap, ceph_cap_string(cap->issued)); + have |= cap->issued; + if (implemented) + *implemented |= cap->implemented; + } + /* + * exclude caps issued by non-auth MDS, but are been revoking + * by the auth MDS. The non-auth MDS should be revoking/exporting + * these caps, but the message is delayed. + */ + if (ci->i_auth_cap) { + cap = ci->i_auth_cap; + have &= ~cap->implemented | cap->issued; + } + return have; +} + +/* + * Get cap bits issued by caps other than @ocap + */ +int __ceph_caps_issued_other(struct ceph_inode_info *ci, struct ceph_cap *ocap) +{ + int have = ci->i_snap_caps; + struct ceph_cap *cap; + struct rb_node *p; + + for (p = rb_first(&ci->i_caps); p; p = rb_next(p)) { + cap = rb_entry(p, struct ceph_cap, ci_node); + if (cap == ocap) + continue; + if (!__cap_is_valid(cap)) + continue; + have |= cap->issued; + } + return have; +} + +/* + * Move a cap to the end of the LRU (oldest caps at list head, newest + * at list tail). + */ +static void __touch_cap(struct ceph_cap *cap) +{ + struct inode *inode = &cap->ci->netfs.inode; + struct ceph_mds_session *s = cap->session; + struct ceph_client *cl = s->s_mdsc->fsc->client; + + spin_lock(&s->s_cap_lock); + if (!s->s_cap_iterator) { + doutc(cl, "%p %llx.%llx cap %p mds%d\n", inode, + ceph_vinop(inode), cap, s->s_mds); + list_move_tail(&cap->session_caps, &s->s_caps); + } else { + doutc(cl, "%p %llx.%llx cap %p mds%d NOP, iterating over caps\n", + inode, ceph_vinop(inode), cap, s->s_mds); + } + spin_unlock(&s->s_cap_lock); +} + +/* + * Check if we hold the given mask. If so, move the cap(s) to the + * front of their respective LRUs. (This is the preferred way for + * callers to check for caps they want.) + */ +int __ceph_caps_issued_mask(struct ceph_inode_info *ci, int mask, int touch) +{ + struct inode *inode = &ci->netfs.inode; + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_cap *cap; + struct rb_node *p; + int have = ci->i_snap_caps; + + if ((have & mask) == mask) { + doutc(cl, "mask %p %llx.%llx snap issued %s (mask %s)\n", + inode, ceph_vinop(inode), ceph_cap_string(have), + ceph_cap_string(mask)); + return 1; + } + + for (p = rb_first(&ci->i_caps); p; p = rb_next(p)) { + cap = rb_entry(p, struct ceph_cap, ci_node); + if (!__cap_is_valid(cap)) + continue; + if ((cap->issued & mask) == mask) { + doutc(cl, "mask %p %llx.%llx cap %p issued %s (mask %s)\n", + inode, ceph_vinop(inode), cap, + ceph_cap_string(cap->issued), + ceph_cap_string(mask)); + if (touch) + __touch_cap(cap); + return 1; + } + + /* does a combination of caps satisfy mask? */ + have |= cap->issued; + if ((have & mask) == mask) { + doutc(cl, "mask %p %llx.%llx combo issued %s (mask %s)\n", + inode, ceph_vinop(inode), + ceph_cap_string(cap->issued), + ceph_cap_string(mask)); + if (touch) { + struct rb_node *q; + + /* touch this + preceding caps */ + __touch_cap(cap); + for (q = rb_first(&ci->i_caps); q != p; + q = rb_next(q)) { + cap = rb_entry(q, struct ceph_cap, + ci_node); + if (!__cap_is_valid(cap)) + continue; + if (cap->issued & mask) + __touch_cap(cap); + } + } + return 1; + } + } + + return 0; +} + +int __ceph_caps_issued_mask_metric(struct ceph_inode_info *ci, int mask, + int touch) +{ + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(ci->netfs.inode.i_sb); + int r; + + r = __ceph_caps_issued_mask(ci, mask, touch); + if (r) + ceph_update_cap_hit(&fsc->mdsc->metric); + else + ceph_update_cap_mis(&fsc->mdsc->metric); + return r; +} + +/* + * Return true if mask caps are currently being revoked by an MDS. + */ +int __ceph_caps_revoking_other(struct ceph_inode_info *ci, + struct ceph_cap *ocap, int mask) +{ + struct ceph_cap *cap; + struct rb_node *p; + + for (p = rb_first(&ci->i_caps); p; p = rb_next(p)) { + cap = rb_entry(p, struct ceph_cap, ci_node); + if (cap != ocap && + (cap->implemented & ~cap->issued & mask)) + return 1; + } + return 0; +} + +int __ceph_caps_used(struct ceph_inode_info *ci) +{ + int used = 0; + if (ci->i_pin_ref) + used |= CEPH_CAP_PIN; + if (ci->i_rd_ref) + used |= CEPH_CAP_FILE_RD; + if (ci->i_rdcache_ref || + (S_ISREG(ci->netfs.inode.i_mode) && + ci->netfs.inode.i_data.nrpages)) + used |= CEPH_CAP_FILE_CACHE; + if (ci->i_wr_ref) + used |= CEPH_CAP_FILE_WR; + if (ci->i_wb_ref || ci->i_wrbuffer_ref) + used |= CEPH_CAP_FILE_BUFFER; + if (ci->i_fx_ref) + used |= CEPH_CAP_FILE_EXCL; + return used; +} + +#define FMODE_WAIT_BIAS 1000 + +/* + * wanted, by virtue of open file modes + */ +int __ceph_caps_file_wanted(struct ceph_inode_info *ci) +{ + const int PIN_SHIFT = ffs(CEPH_FILE_MODE_PIN); + const int RD_SHIFT = ffs(CEPH_FILE_MODE_RD); + const int WR_SHIFT = ffs(CEPH_FILE_MODE_WR); + const int LAZY_SHIFT = ffs(CEPH_FILE_MODE_LAZY); + struct ceph_mount_options *opt = + ceph_inode_to_fs_client(&ci->netfs.inode)->mount_options; + unsigned long used_cutoff = jiffies - opt->caps_wanted_delay_max * HZ; + unsigned long idle_cutoff = jiffies - opt->caps_wanted_delay_min * HZ; + + if (S_ISDIR(ci->netfs.inode.i_mode)) { + int want = 0; + + /* use used_cutoff here, to keep dir's wanted caps longer */ + if (ci->i_nr_by_mode[RD_SHIFT] > 0 || + time_after(ci->i_last_rd, used_cutoff)) + want |= CEPH_CAP_ANY_SHARED; + + if (ci->i_nr_by_mode[WR_SHIFT] > 0 || + time_after(ci->i_last_wr, used_cutoff)) { + want |= CEPH_CAP_ANY_SHARED | CEPH_CAP_FILE_EXCL; + if (opt->flags & CEPH_MOUNT_OPT_ASYNC_DIROPS) + want |= CEPH_CAP_ANY_DIR_OPS; + } + + if (want || ci->i_nr_by_mode[PIN_SHIFT] > 0) + want |= CEPH_CAP_PIN; + + return want; + } else { + int bits = 0; + + if (ci->i_nr_by_mode[RD_SHIFT] > 0) { + if (ci->i_nr_by_mode[RD_SHIFT] >= FMODE_WAIT_BIAS || + time_after(ci->i_last_rd, used_cutoff)) + bits |= 1 << RD_SHIFT; + } else if (time_after(ci->i_last_rd, idle_cutoff)) { + bits |= 1 << RD_SHIFT; + } + + if (ci->i_nr_by_mode[WR_SHIFT] > 0) { + if (ci->i_nr_by_mode[WR_SHIFT] >= FMODE_WAIT_BIAS || + time_after(ci->i_last_wr, used_cutoff)) + bits |= 1 << WR_SHIFT; + } else if (time_after(ci->i_last_wr, idle_cutoff)) { + bits |= 1 << WR_SHIFT; + } + + /* check lazyio only when read/write is wanted */ + if ((bits & (CEPH_FILE_MODE_RDWR << 1)) && + ci->i_nr_by_mode[LAZY_SHIFT] > 0) + bits |= 1 << LAZY_SHIFT; + + return bits ? ceph_caps_for_mode(bits >> 1) : 0; + } +} + +/* + * wanted, by virtue of open file modes AND cap refs (buffered/cached data) + */ +int __ceph_caps_wanted(struct ceph_inode_info *ci) +{ + int w = __ceph_caps_file_wanted(ci) | __ceph_caps_used(ci); + if (S_ISDIR(ci->netfs.inode.i_mode)) { + /* we want EXCL if holding caps of dir ops */ + if (w & CEPH_CAP_ANY_DIR_OPS) + w |= CEPH_CAP_FILE_EXCL; + } else { + /* we want EXCL if dirty data */ + if (w & CEPH_CAP_FILE_BUFFER) + w |= CEPH_CAP_FILE_EXCL; + } + return w; +} + +/* + * Return caps we have registered with the MDS(s) as 'wanted'. + */ +int __ceph_caps_mds_wanted(struct ceph_inode_info *ci, bool check) +{ + struct ceph_cap *cap; + struct rb_node *p; + int mds_wanted = 0; + + for (p = rb_first(&ci->i_caps); p; p = rb_next(p)) { + cap = rb_entry(p, struct ceph_cap, ci_node); + if (check && !__cap_is_valid(cap)) + continue; + if (cap == ci->i_auth_cap) + mds_wanted |= cap->mds_wanted; + else + mds_wanted |= (cap->mds_wanted & ~CEPH_CAP_ANY_FILE_WR); + } + return mds_wanted; +} + +int ceph_is_any_caps(struct inode *inode) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + int ret; + + spin_lock(&ci->i_ceph_lock); + ret = __ceph_is_any_real_caps(ci); + spin_unlock(&ci->i_ceph_lock); + + return ret; +} + +/* + * Remove a cap. Take steps to deal with a racing iterate_session_caps. + * + * caller should hold i_ceph_lock. + * caller will not hold session s_mutex if called from destroy_inode. + */ +void __ceph_remove_cap(struct ceph_cap *cap, bool queue_release) +{ + struct ceph_mds_session *session = cap->session; + struct ceph_client *cl = session->s_mdsc->fsc->client; + struct ceph_inode_info *ci = cap->ci; + struct inode *inode = &ci->netfs.inode; + struct ceph_mds_client *mdsc; + int removed = 0; + + /* 'ci' being NULL means the remove have already occurred */ + if (!ci) { + doutc(cl, "inode is NULL\n"); + return; + } + + lockdep_assert_held(&ci->i_ceph_lock); + + doutc(cl, "%p from %p %llx.%llx\n", cap, inode, ceph_vinop(inode)); + + mdsc = ceph_inode_to_fs_client(&ci->netfs.inode)->mdsc; + + /* remove from inode's cap rbtree, and clear auth cap */ + rb_erase(&cap->ci_node, &ci->i_caps); + if (ci->i_auth_cap == cap) + ci->i_auth_cap = NULL; + + /* remove from session list */ + spin_lock(&session->s_cap_lock); + if (session->s_cap_iterator == cap) { + /* not yet, we are iterating over this very cap */ + doutc(cl, "delaying %p removal from session %p\n", cap, + cap->session); + } else { + list_del_init(&cap->session_caps); + session->s_nr_caps--; + atomic64_dec(&mdsc->metric.total_caps); + cap->session = NULL; + removed = 1; + } + /* protect backpointer with s_cap_lock: see iterate_session_caps */ + cap->ci = NULL; + + /* + * s_cap_reconnect is protected by s_cap_lock. no one changes + * s_cap_gen while session is in the reconnect state. + */ + if (queue_release && + (!session->s_cap_reconnect || + cap->cap_gen == atomic_read(&session->s_cap_gen))) { + cap->queue_release = 1; + if (removed) { + __ceph_queue_cap_release(session, cap); + removed = 0; + } + } else { + cap->queue_release = 0; + } + cap->cap_ino = ci->i_vino.ino; + + spin_unlock(&session->s_cap_lock); + + if (removed) + ceph_put_cap(mdsc, cap); + + if (!__ceph_is_any_real_caps(ci)) { + /* when reconnect denied, we remove session caps forcibly, + * i_wr_ref can be non-zero. If there are ongoing write, + * keep i_snap_realm. + */ + if (ci->i_wr_ref == 0 && ci->i_snap_realm) + ceph_change_snap_realm(&ci->netfs.inode, NULL); + + __cap_delay_cancel(mdsc, ci); + } +} + +void ceph_remove_cap(struct ceph_mds_client *mdsc, struct ceph_cap *cap, + bool queue_release) +{ + struct ceph_inode_info *ci = cap->ci; + struct ceph_fs_client *fsc; + + /* 'ci' being NULL means the remove have already occurred */ + if (!ci) { + doutc(mdsc->fsc->client, "inode is NULL\n"); + return; + } + + lockdep_assert_held(&ci->i_ceph_lock); + + fsc = ceph_inode_to_fs_client(&ci->netfs.inode); + WARN_ON_ONCE(ci->i_auth_cap == cap && + !list_empty(&ci->i_dirty_item) && + !fsc->blocklisted && + !ceph_inode_is_shutdown(&ci->netfs.inode)); + + __ceph_remove_cap(cap, queue_release); +} + +struct cap_msg_args { + struct ceph_mds_session *session; + u64 ino, cid, follows; + u64 flush_tid, oldest_flush_tid, size, max_size; + u64 xattr_version; + u64 change_attr; + struct ceph_buffer *xattr_buf; + struct ceph_buffer *old_xattr_buf; + struct timespec64 atime, mtime, ctime, btime; + int op, caps, wanted, dirty; + u32 seq, issue_seq, mseq, time_warp_seq; + u32 flags; + kuid_t uid; + kgid_t gid; + umode_t mode; + bool inline_data; + bool wake; + bool encrypted; + u32 fscrypt_auth_len; + u8 fscrypt_auth[sizeof(struct ceph_fscrypt_auth)]; // for context +}; + +/* Marshal up the cap msg to the MDS */ +static void encode_cap_msg(struct ceph_msg *msg, struct cap_msg_args *arg) +{ + struct ceph_mds_caps *fc; + void *p; + struct ceph_mds_client *mdsc = arg->session->s_mdsc; + struct ceph_osd_client *osdc = &mdsc->fsc->client->osdc; + + doutc(mdsc->fsc->client, + "%s %llx %llx caps %s wanted %s dirty %s seq %u/%u" + " tid %llu/%llu mseq %u follows %lld size %llu/%llu" + " xattr_ver %llu xattr_len %d\n", + ceph_cap_op_name(arg->op), arg->cid, arg->ino, + ceph_cap_string(arg->caps), ceph_cap_string(arg->wanted), + ceph_cap_string(arg->dirty), arg->seq, arg->issue_seq, + arg->flush_tid, arg->oldest_flush_tid, arg->mseq, arg->follows, + arg->size, arg->max_size, arg->xattr_version, + arg->xattr_buf ? (int)arg->xattr_buf->vec.iov_len : 0); + + msg->hdr.version = cpu_to_le16(12); + msg->hdr.tid = cpu_to_le64(arg->flush_tid); + + fc = msg->front.iov_base; + memset(fc, 0, sizeof(*fc)); + + fc->cap_id = cpu_to_le64(arg->cid); + fc->op = cpu_to_le32(arg->op); + fc->seq = cpu_to_le32(arg->seq); + fc->issue_seq = cpu_to_le32(arg->issue_seq); + fc->migrate_seq = cpu_to_le32(arg->mseq); + fc->caps = cpu_to_le32(arg->caps); + fc->wanted = cpu_to_le32(arg->wanted); + fc->dirty = cpu_to_le32(arg->dirty); + fc->ino = cpu_to_le64(arg->ino); + fc->snap_follows = cpu_to_le64(arg->follows); + +#if IS_ENABLED(CONFIG_FS_ENCRYPTION) + if (arg->encrypted) + fc->size = cpu_to_le64(round_up(arg->size, + CEPH_FSCRYPT_BLOCK_SIZE)); + else +#endif + fc->size = cpu_to_le64(arg->size); + fc->max_size = cpu_to_le64(arg->max_size); + ceph_encode_timespec64(&fc->mtime, &arg->mtime); + ceph_encode_timespec64(&fc->atime, &arg->atime); + ceph_encode_timespec64(&fc->ctime, &arg->ctime); + fc->time_warp_seq = cpu_to_le32(arg->time_warp_seq); + + fc->uid = cpu_to_le32(from_kuid(&init_user_ns, arg->uid)); + fc->gid = cpu_to_le32(from_kgid(&init_user_ns, arg->gid)); + fc->mode = cpu_to_le32(arg->mode); + + fc->xattr_version = cpu_to_le64(arg->xattr_version); + if (arg->xattr_buf) { + msg->middle = ceph_buffer_get(arg->xattr_buf); + fc->xattr_len = cpu_to_le32(arg->xattr_buf->vec.iov_len); + msg->hdr.middle_len = cpu_to_le32(arg->xattr_buf->vec.iov_len); + } + + p = fc + 1; + /* flock buffer size (version 2) */ + ceph_encode_32(&p, 0); + /* inline version (version 4) */ + ceph_encode_64(&p, arg->inline_data ? 0 : CEPH_INLINE_NONE); + /* inline data size */ + ceph_encode_32(&p, 0); + /* + * osd_epoch_barrier (version 5) + * The epoch_barrier is protected osdc->lock, so READ_ONCE here in + * case it was recently changed + */ + ceph_encode_32(&p, READ_ONCE(osdc->epoch_barrier)); + /* oldest_flush_tid (version 6) */ + ceph_encode_64(&p, arg->oldest_flush_tid); + + /* + * caller_uid/caller_gid (version 7) + * + * Currently, we don't properly track which caller dirtied the caps + * last, and force a flush of them when there is a conflict. For now, + * just set this to 0:0, to emulate how the MDS has worked up to now. + */ + ceph_encode_32(&p, 0); + ceph_encode_32(&p, 0); + + /* pool namespace (version 8) (mds always ignores this) */ + ceph_encode_32(&p, 0); + + /* btime and change_attr (version 9) */ + ceph_encode_timespec64(p, &arg->btime); + p += sizeof(struct ceph_timespec); + ceph_encode_64(&p, arg->change_attr); + + /* Advisory flags (version 10) */ + ceph_encode_32(&p, arg->flags); + + /* dirstats (version 11) - these are r/o on the client */ + ceph_encode_64(&p, 0); + ceph_encode_64(&p, 0); + +#if IS_ENABLED(CONFIG_FS_ENCRYPTION) + /* + * fscrypt_auth and fscrypt_file (version 12) + * + * fscrypt_auth holds the crypto context (if any). fscrypt_file + * tracks the real i_size as an __le64 field (and we use a rounded-up + * i_size in the traditional size field). + */ + ceph_encode_32(&p, arg->fscrypt_auth_len); + ceph_encode_copy(&p, arg->fscrypt_auth, arg->fscrypt_auth_len); + ceph_encode_32(&p, sizeof(__le64)); + ceph_encode_64(&p, arg->size); +#else /* CONFIG_FS_ENCRYPTION */ + ceph_encode_32(&p, 0); + ceph_encode_32(&p, 0); +#endif /* CONFIG_FS_ENCRYPTION */ +} + +/* + * Queue cap releases when an inode is dropped from our cache. + */ +void __ceph_remove_caps(struct ceph_inode_info *ci) +{ + struct inode *inode = &ci->netfs.inode; + struct ceph_mds_client *mdsc = ceph_inode_to_fs_client(inode)->mdsc; + struct rb_node *p; + + /* lock i_ceph_lock, because ceph_d_revalidate(..., LOOKUP_RCU) + * may call __ceph_caps_issued_mask() on a freeing inode. */ + spin_lock(&ci->i_ceph_lock); + p = rb_first(&ci->i_caps); + while (p) { + struct ceph_cap *cap = rb_entry(p, struct ceph_cap, ci_node); + p = rb_next(p); + ceph_remove_cap(mdsc, cap, true); + } + spin_unlock(&ci->i_ceph_lock); +} + +/* + * Prepare to send a cap message to an MDS. Update the cap state, and populate + * the arg struct with the parameters that will need to be sent. This should + * be done under the i_ceph_lock to guard against changes to cap state. + * + * Make note of max_size reported/requested from mds, revoked caps + * that have now been implemented. + */ +static void __prep_cap(struct cap_msg_args *arg, struct ceph_cap *cap, + int op, int flags, int used, int want, int retain, + int flushing, u64 flush_tid, u64 oldest_flush_tid) +{ + struct ceph_inode_info *ci = cap->ci; + struct inode *inode = &ci->netfs.inode; + struct ceph_client *cl = ceph_inode_to_client(inode); + int held, revoking; + + lockdep_assert_held(&ci->i_ceph_lock); + + held = cap->issued | cap->implemented; + revoking = cap->implemented & ~cap->issued; + retain &= ~revoking; + + doutc(cl, "%p %llx.%llx cap %p session %p %s -> %s (revoking %s)\n", + inode, ceph_vinop(inode), cap, cap->session, + ceph_cap_string(held), ceph_cap_string(held & retain), + ceph_cap_string(revoking)); + BUG_ON((retain & CEPH_CAP_PIN) == 0); + + clear_bit(CEPH_I_FLUSH_BIT, &ci->i_ceph_flags); + + cap->issued &= retain; /* drop bits we don't want */ + /* + * Wake up any waiters on wanted -> needed transition. This is due to + * the weird transition from buffered to sync IO... we need to flush + * dirty pages _before_ allowing sync writes to avoid reordering. + */ + arg->wake = cap->implemented & ~cap->issued; + cap->implemented &= cap->issued | used; + cap->mds_wanted = want; + + arg->session = cap->session; + arg->ino = ceph_vino(inode).ino; + arg->cid = cap->cap_id; + arg->follows = flushing ? ci->i_head_snapc->seq : 0; + arg->flush_tid = flush_tid; + arg->oldest_flush_tid = oldest_flush_tid; + arg->size = i_size_read(inode); + ci->i_reported_size = arg->size; + arg->max_size = ci->i_wanted_max_size; + if (cap == ci->i_auth_cap) { + if (want & CEPH_CAP_ANY_FILE_WR) + ci->i_requested_max_size = arg->max_size; + else + ci->i_requested_max_size = 0; + } + + if (flushing & CEPH_CAP_XATTR_EXCL) { + arg->old_xattr_buf = __ceph_build_xattrs_blob(ci); + arg->xattr_version = ci->i_xattrs.version; + arg->xattr_buf = ceph_buffer_get(ci->i_xattrs.blob); + } else { + arg->xattr_buf = NULL; + arg->old_xattr_buf = NULL; + } + + arg->mtime = inode_get_mtime(inode); + arg->atime = inode_get_atime(inode); + arg->ctime = inode_get_ctime(inode); + arg->btime = ci->i_btime; + arg->change_attr = inode_peek_iversion_raw(inode); + + arg->op = op; + arg->caps = cap->implemented; + arg->wanted = want; + arg->dirty = flushing; + + arg->seq = cap->seq; + arg->issue_seq = cap->issue_seq; + arg->mseq = cap->mseq; + arg->time_warp_seq = ci->i_time_warp_seq; + + arg->uid = inode->i_uid; + arg->gid = inode->i_gid; + arg->mode = inode->i_mode; + + arg->inline_data = ci->i_inline_version != CEPH_INLINE_NONE; + if (!(flags & CEPH_CLIENT_CAPS_PENDING_CAPSNAP) && + !list_empty(&ci->i_cap_snaps)) { + struct ceph_cap_snap *capsnap; + list_for_each_entry_reverse(capsnap, &ci->i_cap_snaps, ci_item) { + if (capsnap->cap_flush.tid) + break; + if (capsnap->need_flush) { + flags |= CEPH_CLIENT_CAPS_PENDING_CAPSNAP; + break; + } + } + } + arg->flags = flags; + arg->encrypted = IS_ENCRYPTED(inode); +#if IS_ENABLED(CONFIG_FS_ENCRYPTION) + if (ci->fscrypt_auth_len && + WARN_ON_ONCE(ci->fscrypt_auth_len > sizeof(struct ceph_fscrypt_auth))) { + /* Don't set this if it's too big */ + arg->fscrypt_auth_len = 0; + } else { + arg->fscrypt_auth_len = ci->fscrypt_auth_len; + memcpy(arg->fscrypt_auth, ci->fscrypt_auth, + min_t(size_t, ci->fscrypt_auth_len, + sizeof(arg->fscrypt_auth))); + } +#endif /* CONFIG_FS_ENCRYPTION */ +} + +#if IS_ENABLED(CONFIG_FS_ENCRYPTION) +#define CAP_MSG_FIXED_FIELDS (sizeof(struct ceph_mds_caps) + \ + 4 + 8 + 4 + 4 + 8 + 4 + 4 + 4 + 8 + 8 + 4 + 8 + 8 + 4 + 4 + 8) + +static inline int cap_msg_size(struct cap_msg_args *arg) +{ + return CAP_MSG_FIXED_FIELDS + arg->fscrypt_auth_len; +} +#else +#define CAP_MSG_FIXED_FIELDS (sizeof(struct ceph_mds_caps) + \ + 4 + 8 + 4 + 4 + 8 + 4 + 4 + 4 + 8 + 8 + 4 + 8 + 8 + 4 + 4) + +static inline int cap_msg_size(struct cap_msg_args *arg) +{ + return CAP_MSG_FIXED_FIELDS; +} +#endif /* CONFIG_FS_ENCRYPTION */ + +/* + * Send a cap msg on the given inode. + * + * Caller should hold snap_rwsem (read), s_mutex. + */ +static void __send_cap(struct cap_msg_args *arg, struct ceph_inode_info *ci) +{ + struct ceph_msg *msg; + struct inode *inode = &ci->netfs.inode; + struct ceph_client *cl = ceph_inode_to_client(inode); + + msg = ceph_msg_new(CEPH_MSG_CLIENT_CAPS, cap_msg_size(arg), GFP_NOFS, + false); + if (!msg) { + pr_err_client(cl, + "error allocating cap msg: ino (%llx.%llx)" + " flushing %s tid %llu, requeuing cap.\n", + ceph_vinop(inode), ceph_cap_string(arg->dirty), + arg->flush_tid); + spin_lock(&ci->i_ceph_lock); + __cap_delay_requeue(arg->session->s_mdsc, ci); + spin_unlock(&ci->i_ceph_lock); + return; + } + + encode_cap_msg(msg, arg); + ceph_con_send(&arg->session->s_con, msg); + ceph_buffer_put(arg->old_xattr_buf); + ceph_buffer_put(arg->xattr_buf); + if (arg->wake) + wake_up_all(&ci->i_cap_wq); +} + +static inline int __send_flush_snap(struct inode *inode, + struct ceph_mds_session *session, + struct ceph_cap_snap *capsnap, + u32 mseq, u64 oldest_flush_tid) +{ + struct cap_msg_args arg; + struct ceph_msg *msg; + + arg.session = session; + arg.ino = ceph_vino(inode).ino; + arg.cid = 0; + arg.follows = capsnap->follows; + arg.flush_tid = capsnap->cap_flush.tid; + arg.oldest_flush_tid = oldest_flush_tid; + + arg.size = capsnap->size; + arg.max_size = 0; + arg.xattr_version = capsnap->xattr_version; + arg.xattr_buf = capsnap->xattr_blob; + arg.old_xattr_buf = NULL; + + arg.atime = capsnap->atime; + arg.mtime = capsnap->mtime; + arg.ctime = capsnap->ctime; + arg.btime = capsnap->btime; + arg.change_attr = capsnap->change_attr; + + arg.op = CEPH_CAP_OP_FLUSHSNAP; + arg.caps = capsnap->issued; + arg.wanted = 0; + arg.dirty = capsnap->dirty; + + arg.seq = 0; + arg.issue_seq = 0; + arg.mseq = mseq; + arg.time_warp_seq = capsnap->time_warp_seq; + + arg.uid = capsnap->uid; + arg.gid = capsnap->gid; + arg.mode = capsnap->mode; + + arg.inline_data = capsnap->inline_data; + arg.flags = 0; + arg.wake = false; + arg.encrypted = IS_ENCRYPTED(inode); + + /* No fscrypt_auth changes from a capsnap.*/ + arg.fscrypt_auth_len = 0; + + msg = ceph_msg_new(CEPH_MSG_CLIENT_CAPS, cap_msg_size(&arg), + GFP_NOFS, false); + if (!msg) + return -ENOMEM; + + encode_cap_msg(msg, &arg); + ceph_con_send(&arg.session->s_con, msg); + return 0; +} + +/* + * When a snapshot is taken, clients accumulate dirty metadata on + * inodes with capabilities in ceph_cap_snaps to describe the file + * state at the time the snapshot was taken. This must be flushed + * asynchronously back to the MDS once sync writes complete and dirty + * data is written out. + * + * Called under i_ceph_lock. + */ +static void __ceph_flush_snaps(struct ceph_inode_info *ci, + struct ceph_mds_session *session) + __releases(ci->i_ceph_lock) + __acquires(ci->i_ceph_lock) +{ + struct inode *inode = &ci->netfs.inode; + struct ceph_mds_client *mdsc = session->s_mdsc; + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_cap_snap *capsnap; + u64 oldest_flush_tid = 0; + u64 first_tid = 1, last_tid = 0; + + doutc(cl, "%p %llx.%llx session %p\n", inode, ceph_vinop(inode), + session); + + list_for_each_entry(capsnap, &ci->i_cap_snaps, ci_item) { + /* + * we need to wait for sync writes to complete and for dirty + * pages to be written out. + */ + if (capsnap->dirty_pages || capsnap->writing) + break; + + /* should be removed by ceph_try_drop_cap_snap() */ + BUG_ON(!capsnap->need_flush); + + /* only flush each capsnap once */ + if (capsnap->cap_flush.tid > 0) { + doutc(cl, "already flushed %p, skipping\n", capsnap); + continue; + } + + spin_lock(&mdsc->cap_dirty_lock); + capsnap->cap_flush.tid = ++mdsc->last_cap_flush_tid; + capsnap->cap_flush.ci = ci; + list_add_tail(&capsnap->cap_flush.g_list, + &mdsc->cap_flush_list); + if (oldest_flush_tid == 0) + oldest_flush_tid = __get_oldest_flush_tid(mdsc); + if (list_empty(&ci->i_flushing_item)) { + list_add_tail(&ci->i_flushing_item, + &session->s_cap_flushing); + } + spin_unlock(&mdsc->cap_dirty_lock); + + list_add_tail(&capsnap->cap_flush.i_list, + &ci->i_cap_flush_list); + + if (first_tid == 1) + first_tid = capsnap->cap_flush.tid; + last_tid = capsnap->cap_flush.tid; + } + + clear_bit(CEPH_I_FLUSH_SNAPS_BIT, &ci->i_ceph_flags); + + while (first_tid <= last_tid) { + struct ceph_cap *cap = ci->i_auth_cap; + struct ceph_cap_flush *cf = NULL, *iter; + int ret; + + if (!(cap && cap->session == session)) { + doutc(cl, "%p %llx.%llx auth cap %p not mds%d, stop\n", + inode, ceph_vinop(inode), cap, session->s_mds); + break; + } + + ret = -ENOENT; + list_for_each_entry(iter, &ci->i_cap_flush_list, i_list) { + if (iter->tid >= first_tid) { + cf = iter; + ret = 0; + break; + } + } + if (ret < 0) + break; + + first_tid = cf->tid + 1; + + capsnap = container_of(cf, struct ceph_cap_snap, cap_flush); + refcount_inc(&capsnap->nref); + spin_unlock(&ci->i_ceph_lock); + + doutc(cl, "%p %llx.%llx capsnap %p tid %llu %s\n", inode, + ceph_vinop(inode), capsnap, cf->tid, + ceph_cap_string(capsnap->dirty)); + + ret = __send_flush_snap(inode, session, capsnap, cap->mseq, + oldest_flush_tid); + if (ret < 0) { + pr_err_client(cl, "error sending cap flushsnap, " + "ino (%llx.%llx) tid %llu follows %llu\n", + ceph_vinop(inode), cf->tid, + capsnap->follows); + } + + ceph_put_cap_snap(capsnap); + spin_lock(&ci->i_ceph_lock); + } +} + +void ceph_flush_snaps(struct ceph_inode_info *ci, + struct ceph_mds_session **psession) +{ + struct inode *inode = &ci->netfs.inode; + struct ceph_mds_client *mdsc = ceph_inode_to_fs_client(inode)->mdsc; + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_mds_session *session = NULL; + bool need_put = false; + int mds; + + doutc(cl, "%p %llx.%llx\n", inode, ceph_vinop(inode)); + if (psession) + session = *psession; +retry: + spin_lock(&ci->i_ceph_lock); + if (!(ci->i_ceph_flags & CEPH_I_FLUSH_SNAPS)) { + doutc(cl, " no capsnap needs flush, doing nothing\n"); + goto out; + } + if (!ci->i_auth_cap) { + doutc(cl, " no auth cap (migrating?), doing nothing\n"); + goto out; + } + + mds = ci->i_auth_cap->session->s_mds; + if (session && session->s_mds != mds) { + doutc(cl, " oops, wrong session %p mutex\n", session); + ceph_put_mds_session(session); + session = NULL; + } + if (!session) { + spin_unlock(&ci->i_ceph_lock); + mutex_lock(&mdsc->mutex); + session = __ceph_lookup_mds_session(mdsc, mds); + mutex_unlock(&mdsc->mutex); + goto retry; + } + + // make sure flushsnap messages are sent in proper order. + if (ci->i_ceph_flags & CEPH_I_KICK_FLUSH) + __kick_flushing_caps(mdsc, session, ci, 0); + + __ceph_flush_snaps(ci, session); +out: + spin_unlock(&ci->i_ceph_lock); + + if (psession) + *psession = session; + else + ceph_put_mds_session(session); + /* we flushed them all; remove this inode from the queue */ + spin_lock(&mdsc->snap_flush_lock); + if (!list_empty(&ci->i_snap_flush_item)) + need_put = true; + list_del_init(&ci->i_snap_flush_item); + spin_unlock(&mdsc->snap_flush_lock); + + if (need_put) + iput(inode); +} + +/* + * Mark caps dirty. If inode is newly dirty, return the dirty flags. + * Caller is then responsible for calling __mark_inode_dirty with the + * returned flags value. + */ +int __ceph_mark_dirty_caps(struct ceph_inode_info *ci, int mask, + struct ceph_cap_flush **pcf) +{ + struct ceph_mds_client *mdsc = + ceph_sb_to_fs_client(ci->netfs.inode.i_sb)->mdsc; + struct inode *inode = &ci->netfs.inode; + struct ceph_client *cl = ceph_inode_to_client(inode); + int was = ci->i_dirty_caps; + int dirty = 0; + + lockdep_assert_held(&ci->i_ceph_lock); + + if (!ci->i_auth_cap) { + pr_warn_client(cl, "%p %llx.%llx mask %s, " + "but no auth cap (session was closed?)\n", + inode, ceph_vinop(inode), + ceph_cap_string(mask)); + return 0; + } + + doutc(cl, "%p %llx.%llx %s dirty %s -> %s\n", inode, + ceph_vinop(inode), ceph_cap_string(mask), + ceph_cap_string(was), ceph_cap_string(was | mask)); + ci->i_dirty_caps |= mask; + if (was == 0) { + struct ceph_mds_session *session = ci->i_auth_cap->session; + + WARN_ON_ONCE(ci->i_prealloc_cap_flush); + swap(ci->i_prealloc_cap_flush, *pcf); + + if (!ci->i_head_snapc) { + WARN_ON_ONCE(!rwsem_is_locked(&mdsc->snap_rwsem)); + ci->i_head_snapc = ceph_get_snap_context( + ci->i_snap_realm->cached_context); + } + doutc(cl, "%p %llx.%llx now dirty snapc %p auth cap %p\n", + inode, ceph_vinop(inode), ci->i_head_snapc, + ci->i_auth_cap); + BUG_ON(!list_empty(&ci->i_dirty_item)); + spin_lock(&mdsc->cap_dirty_lock); + list_add(&ci->i_dirty_item, &session->s_cap_dirty); + spin_unlock(&mdsc->cap_dirty_lock); + if (ci->i_flushing_caps == 0) { + ihold(inode); + dirty |= I_DIRTY_SYNC; + } + } else { + WARN_ON_ONCE(!ci->i_prealloc_cap_flush); + } + BUG_ON(list_empty(&ci->i_dirty_item)); + if (((was | ci->i_flushing_caps) & CEPH_CAP_FILE_BUFFER) && + (mask & CEPH_CAP_FILE_BUFFER)) + dirty |= I_DIRTY_DATASYNC; + __cap_delay_requeue(mdsc, ci); + return dirty; +} + +struct ceph_cap_flush *ceph_alloc_cap_flush(void) +{ + struct ceph_cap_flush *cf; + + cf = kmem_cache_alloc(ceph_cap_flush_cachep, GFP_KERNEL); + if (!cf) + return NULL; + + cf->is_capsnap = false; + cf->ci = NULL; + return cf; +} + +void ceph_free_cap_flush(struct ceph_cap_flush *cf) +{ + if (cf) + kmem_cache_free(ceph_cap_flush_cachep, cf); +} + +static u64 __get_oldest_flush_tid(struct ceph_mds_client *mdsc) +{ + if (!list_empty(&mdsc->cap_flush_list)) { + struct ceph_cap_flush *cf = + list_first_entry(&mdsc->cap_flush_list, + struct ceph_cap_flush, g_list); + return cf->tid; + } + return 0; +} + +/* + * Remove cap_flush from the mdsc's or inode's flushing cap list. + * Return true if caller needs to wake up flush waiters. + */ +static bool __detach_cap_flush_from_mdsc(struct ceph_mds_client *mdsc, + struct ceph_cap_flush *cf) +{ + struct ceph_cap_flush *prev; + bool wake = cf->wake; + + if (wake && cf->g_list.prev != &mdsc->cap_flush_list) { + prev = list_prev_entry(cf, g_list); + prev->wake = true; + wake = false; + } + list_del_init(&cf->g_list); + return wake; +} + +static bool __detach_cap_flush_from_ci(struct ceph_inode_info *ci, + struct ceph_cap_flush *cf) +{ + struct ceph_cap_flush *prev; + bool wake = cf->wake; + + if (wake && cf->i_list.prev != &ci->i_cap_flush_list) { + prev = list_prev_entry(cf, i_list); + prev->wake = true; + wake = false; + } + list_del_init(&cf->i_list); + return wake; +} + +/* + * Add dirty inode to the flushing list. Assigned a seq number so we + * can wait for caps to flush without starving. + * + * Called under i_ceph_lock. Returns the flush tid. + */ +static u64 __mark_caps_flushing(struct inode *inode, + struct ceph_mds_session *session, bool wake, + u64 *oldest_flush_tid) +{ + struct ceph_mds_client *mdsc = ceph_sb_to_fs_client(inode->i_sb)->mdsc; + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_cap_flush *cf = NULL; + int flushing; + + lockdep_assert_held(&ci->i_ceph_lock); + BUG_ON(ci->i_dirty_caps == 0); + BUG_ON(list_empty(&ci->i_dirty_item)); + BUG_ON(!ci->i_prealloc_cap_flush); + + flushing = ci->i_dirty_caps; + doutc(cl, "flushing %s, flushing_caps %s -> %s\n", + ceph_cap_string(flushing), + ceph_cap_string(ci->i_flushing_caps), + ceph_cap_string(ci->i_flushing_caps | flushing)); + ci->i_flushing_caps |= flushing; + ci->i_dirty_caps = 0; + doutc(cl, "%p %llx.%llx now !dirty\n", inode, ceph_vinop(inode)); + + swap(cf, ci->i_prealloc_cap_flush); + cf->ci = ci; + cf->caps = flushing; + cf->wake = wake; + + spin_lock(&mdsc->cap_dirty_lock); + list_del_init(&ci->i_dirty_item); + + cf->tid = ++mdsc->last_cap_flush_tid; + list_add_tail(&cf->g_list, &mdsc->cap_flush_list); + *oldest_flush_tid = __get_oldest_flush_tid(mdsc); + + if (list_empty(&ci->i_flushing_item)) { + list_add_tail(&ci->i_flushing_item, &session->s_cap_flushing); + mdsc->num_cap_flushing++; + } + spin_unlock(&mdsc->cap_dirty_lock); + + list_add_tail(&cf->i_list, &ci->i_cap_flush_list); + + return cf->tid; +} + +/* + * try to invalidate mapping pages without blocking. + */ +static int try_nonblocking_invalidate(struct inode *inode) + __releases(ci->i_ceph_lock) + __acquires(ci->i_ceph_lock) +{ + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + u32 invalidating_gen = ci->i_rdcache_gen; + + spin_unlock(&ci->i_ceph_lock); + ceph_fscache_invalidate(inode, false); + invalidate_mapping_pages(&inode->i_data, 0, -1); + spin_lock(&ci->i_ceph_lock); + + if (inode->i_data.nrpages == 0 && + invalidating_gen == ci->i_rdcache_gen) { + /* success. */ + doutc(cl, "%p %llx.%llx success\n", inode, + ceph_vinop(inode)); + /* save any racing async invalidate some trouble */ + ci->i_rdcache_revoking = ci->i_rdcache_gen - 1; + return 0; + } + doutc(cl, "%p %llx.%llx failed\n", inode, ceph_vinop(inode)); + return -1; +} + +bool __ceph_should_report_size(struct ceph_inode_info *ci) +{ + loff_t size = i_size_read(&ci->netfs.inode); + /* mds will adjust max size according to the reported size */ + if (ci->i_flushing_caps & CEPH_CAP_FILE_WR) + return false; + if (size >= ci->i_max_size) + return true; + /* half of previous max_size increment has been used */ + if (ci->i_max_size > ci->i_reported_size && + (size << 1) >= ci->i_max_size + ci->i_reported_size) + return true; + return false; +} + +/* + * Swiss army knife function to examine currently used and wanted + * versus held caps. Release, flush, ack revoked caps to mds as + * appropriate. + * + * CHECK_CAPS_AUTHONLY - we should only check the auth cap + * CHECK_CAPS_FLUSH - we should flush any dirty caps immediately, without + * further delay. + * CHECK_CAPS_FLUSH_FORCE - we should flush any caps immediately, without + * further delay. + */ +void ceph_check_caps(struct ceph_inode_info *ci, int flags) +{ + struct inode *inode = &ci->netfs.inode; + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb); + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_cap *cap; + u64 flush_tid, oldest_flush_tid; + int file_wanted, used, cap_used; + int issued, implemented, want, retain, revoking, flushing = 0; + int mds = -1; /* keep track of how far we've gone through i_caps list + to avoid an infinite loop on retry */ + struct rb_node *p; + bool queue_invalidate = false; + bool tried_invalidate = false; + bool queue_writeback = false; + struct ceph_mds_session *session = NULL; + + spin_lock(&ci->i_ceph_lock); + if (ci->i_ceph_flags & CEPH_I_ASYNC_CREATE) { + set_bit(CEPH_I_ASYNC_CHECK_CAPS_BIT, &ci->i_ceph_flags); + + /* Don't send messages until we get async create reply */ + spin_unlock(&ci->i_ceph_lock); + return; + } + + if (ci->i_ceph_flags & CEPH_I_FLUSH) + flags |= CHECK_CAPS_FLUSH; +retry: + /* Caps wanted by virtue of active open files. */ + file_wanted = __ceph_caps_file_wanted(ci); + + /* Caps which have active references against them */ + used = __ceph_caps_used(ci); + + /* + * "issued" represents the current caps that the MDS wants us to have. + * "implemented" is the set that we have been granted, and includes the + * ones that have not yet been returned to the MDS (the "revoking" set, + * usually because they have outstanding references). + */ + issued = __ceph_caps_issued(ci, &implemented); + revoking = implemented & ~issued; + + want = file_wanted; + + /* The ones we currently want to retain (may be adjusted below) */ + retain = file_wanted | used | CEPH_CAP_PIN; + if (!mdsc->stopping && inode->i_nlink > 0) { + if (file_wanted) { + retain |= CEPH_CAP_ANY; /* be greedy */ + } else if (S_ISDIR(inode->i_mode) && + (issued & CEPH_CAP_FILE_SHARED) && + __ceph_dir_is_complete(ci)) { + /* + * If a directory is complete, we want to keep + * the exclusive cap. So that MDS does not end up + * revoking the shared cap on every create/unlink + * operation. + */ + if (IS_RDONLY(inode)) { + want = CEPH_CAP_ANY_SHARED; + } else { + want |= CEPH_CAP_ANY_SHARED | CEPH_CAP_FILE_EXCL; + } + retain |= want; + } else { + + retain |= CEPH_CAP_ANY_SHARED; + /* + * keep RD only if we didn't have the file open RW, + * because then the mds would revoke it anyway to + * journal max_size=0. + */ + if (ci->i_max_size == 0) + retain |= CEPH_CAP_ANY_RD; + } + } + + doutc(cl, "%p %llx.%llx file_want %s used %s dirty %s " + "flushing %s issued %s revoking %s retain %s %s%s%s%s\n", + inode, ceph_vinop(inode), ceph_cap_string(file_wanted), + ceph_cap_string(used), ceph_cap_string(ci->i_dirty_caps), + ceph_cap_string(ci->i_flushing_caps), + ceph_cap_string(issued), ceph_cap_string(revoking), + ceph_cap_string(retain), + (flags & CHECK_CAPS_AUTHONLY) ? " AUTHONLY" : "", + (flags & CHECK_CAPS_FLUSH) ? " FLUSH" : "", + (flags & CHECK_CAPS_NOINVAL) ? " NOINVAL" : "", + (flags & CHECK_CAPS_FLUSH_FORCE) ? " FLUSH_FORCE" : ""); + + /* + * If we no longer need to hold onto old our caps, and we may + * have cached pages, but don't want them, then try to invalidate. + * If we fail, it's because pages are locked.... try again later. + */ + if ((!(flags & CHECK_CAPS_NOINVAL) || mdsc->stopping) && + S_ISREG(inode->i_mode) && + !(ci->i_wb_ref || ci->i_wrbuffer_ref) && /* no dirty pages... */ + inode->i_data.nrpages && /* have cached pages */ + (revoking & (CEPH_CAP_FILE_CACHE| + CEPH_CAP_FILE_LAZYIO)) && /* or revoking cache */ + !tried_invalidate) { + doutc(cl, "trying to invalidate on %p %llx.%llx\n", + inode, ceph_vinop(inode)); + if (try_nonblocking_invalidate(inode) < 0) { + doutc(cl, "queuing invalidate\n"); + queue_invalidate = true; + ci->i_rdcache_revoking = ci->i_rdcache_gen; + } + tried_invalidate = true; + goto retry; + } + + for (p = rb_first(&ci->i_caps); p; p = rb_next(p)) { + int mflags = 0; + struct cap_msg_args arg; + + cap = rb_entry(p, struct ceph_cap, ci_node); + + /* avoid looping forever */ + if (mds >= cap->mds || + ((flags & CHECK_CAPS_AUTHONLY) && cap != ci->i_auth_cap)) + continue; + + /* + * If we have an auth cap, we don't need to consider any + * overlapping caps as used. + */ + cap_used = used; + if (ci->i_auth_cap && cap != ci->i_auth_cap) + cap_used &= ~ci->i_auth_cap->issued; + + revoking = cap->implemented & ~cap->issued; + doutc(cl, " mds%d cap %p used %s issued %s implemented %s revoking %s\n", + cap->mds, cap, ceph_cap_string(cap_used), + ceph_cap_string(cap->issued), + ceph_cap_string(cap->implemented), + ceph_cap_string(revoking)); + + /* completed revocation? going down and there are no caps? */ + if (revoking) { + if ((revoking & cap_used) == 0) { + doutc(cl, "completed revocation of %s\n", + ceph_cap_string(cap->implemented & ~cap->issued)); + goto ack; + } + + /* + * If the "i_wrbuffer_ref" was increased by mmap or generic + * cache write just before the ceph_check_caps() is called, + * the Fb capability revoking will fail this time. Then we + * must wait for the BDI's delayed work to flush the dirty + * pages and to release the "i_wrbuffer_ref", which will cost + * at most 5 seconds. That means the MDS needs to wait at + * most 5 seconds to finished the Fb capability's revocation. + * + * Let's queue a writeback for it. + */ + if (S_ISREG(inode->i_mode) && ci->i_wrbuffer_ref && + (revoking & CEPH_CAP_FILE_BUFFER)) + queue_writeback = true; + } + + if (flags & CHECK_CAPS_FLUSH_FORCE) { + doutc(cl, "force to flush caps\n"); + goto ack; + } + + if (cap == ci->i_auth_cap && + (cap->issued & CEPH_CAP_FILE_WR)) { + /* request larger max_size from MDS? */ + if (ci->i_wanted_max_size > ci->i_max_size && + ci->i_wanted_max_size > ci->i_requested_max_size) { + doutc(cl, "requesting new max_size\n"); + goto ack; + } + + /* approaching file_max? */ + if (__ceph_should_report_size(ci)) { + doutc(cl, "i_size approaching max_size\n"); + goto ack; + } + } + /* flush anything dirty? */ + if (cap == ci->i_auth_cap) { + if ((flags & CHECK_CAPS_FLUSH) && ci->i_dirty_caps) { + doutc(cl, "flushing dirty caps\n"); + goto ack; + } + if (ci->i_ceph_flags & CEPH_I_FLUSH_SNAPS) { + doutc(cl, "flushing snap caps\n"); + goto ack; + } + } + + /* want more caps from mds? */ + if (want & ~cap->mds_wanted) { + if (want & ~(cap->mds_wanted | cap->issued)) + goto ack; + if (!__cap_is_valid(cap)) + goto ack; + } + + /* things we might delay */ + if ((cap->issued & ~retain) == 0) + continue; /* nope, all good */ + +ack: + ceph_put_mds_session(session); + session = ceph_get_mds_session(cap->session); + + /* kick flushing and flush snaps before sending normal + * cap message */ + if (cap == ci->i_auth_cap && + (ci->i_ceph_flags & + (CEPH_I_KICK_FLUSH | CEPH_I_FLUSH_SNAPS))) { + if (ci->i_ceph_flags & CEPH_I_KICK_FLUSH) + __kick_flushing_caps(mdsc, session, ci, 0); + if (ci->i_ceph_flags & CEPH_I_FLUSH_SNAPS) + __ceph_flush_snaps(ci, session); + + goto retry; + } + + if (cap == ci->i_auth_cap && ci->i_dirty_caps) { + flushing = ci->i_dirty_caps; + flush_tid = __mark_caps_flushing(inode, session, false, + &oldest_flush_tid); + if (flags & CHECK_CAPS_FLUSH && + list_empty(&session->s_cap_dirty)) + mflags |= CEPH_CLIENT_CAPS_SYNC; + } else { + flushing = 0; + flush_tid = 0; + spin_lock(&mdsc->cap_dirty_lock); + oldest_flush_tid = __get_oldest_flush_tid(mdsc); + spin_unlock(&mdsc->cap_dirty_lock); + } + + mds = cap->mds; /* remember mds, so we don't repeat */ + + __prep_cap(&arg, cap, CEPH_CAP_OP_UPDATE, mflags, cap_used, + want, retain, flushing, flush_tid, oldest_flush_tid); + + spin_unlock(&ci->i_ceph_lock); + __send_cap(&arg, ci); + spin_lock(&ci->i_ceph_lock); + + goto retry; /* retake i_ceph_lock and restart our cap scan. */ + } + + /* periodically re-calculate caps wanted by open files */ + if (__ceph_is_any_real_caps(ci) && + list_empty(&ci->i_cap_delay_list) && + (file_wanted & ~CEPH_CAP_PIN) && + !(used & (CEPH_CAP_FILE_RD | CEPH_CAP_ANY_FILE_WR))) { + __cap_delay_requeue(mdsc, ci); + } + + spin_unlock(&ci->i_ceph_lock); + + ceph_put_mds_session(session); + if (queue_writeback) + ceph_queue_writeback(inode); + if (queue_invalidate) + ceph_queue_invalidate(inode); +} + +/* + * Try to flush dirty caps back to the auth mds. + */ +static int try_flush_caps(struct inode *inode, u64 *ptid) +{ + struct ceph_mds_client *mdsc = ceph_sb_to_fs_client(inode->i_sb)->mdsc; + struct ceph_inode_info *ci = ceph_inode(inode); + int flushing = 0; + u64 flush_tid = 0, oldest_flush_tid = 0; + + spin_lock(&ci->i_ceph_lock); +retry_locked: + if (ci->i_dirty_caps && ci->i_auth_cap) { + struct ceph_cap *cap = ci->i_auth_cap; + struct cap_msg_args arg; + struct ceph_mds_session *session = cap->session; + + if (session->s_state < CEPH_MDS_SESSION_OPEN) { + spin_unlock(&ci->i_ceph_lock); + goto out; + } + + if (ci->i_ceph_flags & + (CEPH_I_KICK_FLUSH | CEPH_I_FLUSH_SNAPS)) { + if (ci->i_ceph_flags & CEPH_I_KICK_FLUSH) + __kick_flushing_caps(mdsc, session, ci, 0); + if (ci->i_ceph_flags & CEPH_I_FLUSH_SNAPS) + __ceph_flush_snaps(ci, session); + goto retry_locked; + } + + flushing = ci->i_dirty_caps; + flush_tid = __mark_caps_flushing(inode, session, true, + &oldest_flush_tid); + + __prep_cap(&arg, cap, CEPH_CAP_OP_FLUSH, CEPH_CLIENT_CAPS_SYNC, + __ceph_caps_used(ci), __ceph_caps_wanted(ci), + (cap->issued | cap->implemented), + flushing, flush_tid, oldest_flush_tid); + spin_unlock(&ci->i_ceph_lock); + + __send_cap(&arg, ci); + } else { + if (!list_empty(&ci->i_cap_flush_list)) { + struct ceph_cap_flush *cf = + list_last_entry(&ci->i_cap_flush_list, + struct ceph_cap_flush, i_list); + cf->wake = true; + flush_tid = cf->tid; + } + flushing = ci->i_flushing_caps; + spin_unlock(&ci->i_ceph_lock); + } +out: + *ptid = flush_tid; + return flushing; +} + +/* + * Return true if we've flushed caps through the given flush_tid. + */ +static int caps_are_flushed(struct inode *inode, u64 flush_tid) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + int ret = 1; + + spin_lock(&ci->i_ceph_lock); + if (!list_empty(&ci->i_cap_flush_list)) { + struct ceph_cap_flush * cf = + list_first_entry(&ci->i_cap_flush_list, + struct ceph_cap_flush, i_list); + if (cf->tid <= flush_tid) + ret = 0; + } + spin_unlock(&ci->i_ceph_lock); + return ret; +} + +/* + * flush the mdlog and wait for any unsafe requests to complete. + */ +static int flush_mdlog_and_wait_inode_unsafe_requests(struct inode *inode) +{ + struct ceph_mds_client *mdsc = ceph_sb_to_fs_client(inode->i_sb)->mdsc; + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_mds_request *req1 = NULL, *req2 = NULL; + int ret, err = 0; + + spin_lock(&ci->i_unsafe_lock); + if (S_ISDIR(inode->i_mode) && !list_empty(&ci->i_unsafe_dirops)) { + req1 = list_last_entry(&ci->i_unsafe_dirops, + struct ceph_mds_request, + r_unsafe_dir_item); + ceph_mdsc_get_request(req1); + } + if (!list_empty(&ci->i_unsafe_iops)) { + req2 = list_last_entry(&ci->i_unsafe_iops, + struct ceph_mds_request, + r_unsafe_target_item); + ceph_mdsc_get_request(req2); + } + spin_unlock(&ci->i_unsafe_lock); + + /* + * Trigger to flush the journal logs in all the relevant MDSes + * manually, or in the worst case we must wait at most 5 seconds + * to wait the journal logs to be flushed by the MDSes periodically. + */ + if (req1 || req2) { + struct ceph_mds_request *req; + struct ceph_mds_session **sessions; + struct ceph_mds_session *s; + unsigned int max_sessions; + int i; + + mutex_lock(&mdsc->mutex); + max_sessions = mdsc->max_sessions; + + sessions = kzalloc_objs(s, max_sessions); + if (!sessions) { + mutex_unlock(&mdsc->mutex); + err = -ENOMEM; + goto out; + } + + spin_lock(&ci->i_unsafe_lock); + if (req1) { + list_for_each_entry(req, &ci->i_unsafe_dirops, + r_unsafe_dir_item) { + s = req->r_session; + if (!s) + continue; + if (!sessions[s->s_mds]) { + s = ceph_get_mds_session(s); + sessions[s->s_mds] = s; + } + } + } + if (req2) { + list_for_each_entry(req, &ci->i_unsafe_iops, + r_unsafe_target_item) { + s = req->r_session; + if (!s) + continue; + if (!sessions[s->s_mds]) { + s = ceph_get_mds_session(s); + sessions[s->s_mds] = s; + } + } + } + spin_unlock(&ci->i_unsafe_lock); + + /* the auth MDS */ + spin_lock(&ci->i_ceph_lock); + if (ci->i_auth_cap) { + s = ci->i_auth_cap->session; + if (!sessions[s->s_mds]) + sessions[s->s_mds] = ceph_get_mds_session(s); + } + spin_unlock(&ci->i_ceph_lock); + mutex_unlock(&mdsc->mutex); + + /* send flush mdlog request to MDSes */ + for (i = 0; i < max_sessions; i++) { + s = sessions[i]; + if (s) { + send_flush_mdlog(s); + ceph_put_mds_session(s); + } + } + kfree(sessions); + } + + doutc(cl, "%p %llx.%llx wait on tid %llu %llu\n", inode, + ceph_vinop(inode), req1 ? req1->r_tid : 0ULL, + req2 ? req2->r_tid : 0ULL); + if (req1) { + ret = !wait_for_completion_timeout(&req1->r_safe_completion, + ceph_timeout_jiffies(req1->r_timeout)); + if (ret) + err = -EIO; + } + if (req2) { + ret = !wait_for_completion_timeout(&req2->r_safe_completion, + ceph_timeout_jiffies(req2->r_timeout)); + if (ret) + err = -EIO; + } + +out: + if (req1) + ceph_mdsc_put_request(req1); + if (req2) + ceph_mdsc_put_request(req2); + return err; +} + +int ceph_fsync(struct file *file, loff_t start, loff_t end, int datasync) +{ + struct inode *inode = file->f_mapping->host; + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_client *cl = ceph_inode_to_client(inode); + u64 flush_tid; + int ret, err; + int dirty; + + doutc(cl, "%p %llx.%llx%s\n", inode, ceph_vinop(inode), + datasync ? " datasync" : ""); + + ret = file_write_and_wait_range(file, start, end); + if (datasync) + goto out; + + ret = ceph_wait_on_async_create(inode); + if (ret) + goto out; + + dirty = try_flush_caps(inode, &flush_tid); + doutc(cl, "dirty caps are %s\n", ceph_cap_string(dirty)); + + err = flush_mdlog_and_wait_inode_unsafe_requests(inode); + + /* + * only wait on non-file metadata writeback (the mds + * can recover size and mtime, so we don't need to + * wait for that) + */ + if (!err && (dirty & ~CEPH_CAP_ANY_FILE_WR)) { + err = wait_event_interruptible(ci->i_cap_wq, + caps_are_flushed(inode, flush_tid)); + } + + if (err < 0) + ret = err; + + err = file_check_and_advance_wb_err(file); + if (err < 0) + ret = err; +out: + doutc(cl, "%p %llx.%llx%s result=%d\n", inode, ceph_vinop(inode), + datasync ? " datasync" : "", ret); + return ret; +} + +/* + * Flush any dirty caps back to the mds. If we aren't asked to wait, + * queue inode for flush but don't do so immediately, because we can + * get by with fewer MDS messages if we wait for data writeback to + * complete first. + */ +int ceph_write_inode(struct inode *inode, struct writeback_control *wbc) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_client *cl = ceph_inode_to_client(inode); + u64 flush_tid; + int err = 0; + int dirty; + int wait = (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync); + + doutc(cl, "%p %llx.%llx wait=%d\n", inode, ceph_vinop(inode), wait); + ceph_fscache_unpin_writeback(inode, wbc); + if (wait) { + err = ceph_wait_on_async_create(inode); + if (err) + return err; + dirty = try_flush_caps(inode, &flush_tid); + if (dirty) + err = wait_event_interruptible(ci->i_cap_wq, + caps_are_flushed(inode, flush_tid)); + } else { + struct ceph_mds_client *mdsc = + ceph_sb_to_fs_client(inode->i_sb)->mdsc; + + spin_lock(&ci->i_ceph_lock); + if (__ceph_caps_dirty(ci)) + __cap_delay_requeue_front(mdsc, ci); + spin_unlock(&ci->i_ceph_lock); + } + return err; +} + +static void __kick_flushing_caps(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session, + struct ceph_inode_info *ci, + u64 oldest_flush_tid) + __releases(ci->i_ceph_lock) + __acquires(ci->i_ceph_lock) +{ + struct inode *inode = &ci->netfs.inode; + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_cap *cap; + struct ceph_cap_flush *cf; + int ret; + u64 first_tid = 0; + u64 last_snap_flush = 0; + + /* Don't do anything until create reply comes in */ + if (ci->i_ceph_flags & CEPH_I_ASYNC_CREATE) + return; + + clear_bit(CEPH_I_KICK_FLUSH_BIT, &ci->i_ceph_flags); + + list_for_each_entry_reverse(cf, &ci->i_cap_flush_list, i_list) { + if (cf->is_capsnap) { + last_snap_flush = cf->tid; + break; + } + } + + list_for_each_entry(cf, &ci->i_cap_flush_list, i_list) { + if (cf->tid < first_tid) + continue; + + cap = ci->i_auth_cap; + if (!(cap && cap->session == session)) { + pr_err_client(cl, "%p auth cap %p not mds%d ???\n", + inode, cap, session->s_mds); + break; + } + + first_tid = cf->tid + 1; + + if (!cf->is_capsnap) { + struct cap_msg_args arg; + + doutc(cl, "%p %llx.%llx cap %p tid %llu %s\n", + inode, ceph_vinop(inode), cap, cf->tid, + ceph_cap_string(cf->caps)); + __prep_cap(&arg, cap, CEPH_CAP_OP_FLUSH, + (cf->tid < last_snap_flush ? + CEPH_CLIENT_CAPS_PENDING_CAPSNAP : 0), + __ceph_caps_used(ci), + __ceph_caps_wanted(ci), + (cap->issued | cap->implemented), + cf->caps, cf->tid, oldest_flush_tid); + spin_unlock(&ci->i_ceph_lock); + __send_cap(&arg, ci); + } else { + struct ceph_cap_snap *capsnap = + container_of(cf, struct ceph_cap_snap, + cap_flush); + doutc(cl, "%p %llx.%llx capsnap %p tid %llu %s\n", + inode, ceph_vinop(inode), capsnap, cf->tid, + ceph_cap_string(capsnap->dirty)); + + refcount_inc(&capsnap->nref); + spin_unlock(&ci->i_ceph_lock); + + ret = __send_flush_snap(inode, session, capsnap, cap->mseq, + oldest_flush_tid); + if (ret < 0) { + pr_err_client(cl, "error sending cap flushsnap," + " %p %llx.%llx tid %llu follows %llu\n", + inode, ceph_vinop(inode), cf->tid, + capsnap->follows); + } + + ceph_put_cap_snap(capsnap); + } + + spin_lock(&ci->i_ceph_lock); + } +} + +void ceph_early_kick_flushing_caps(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_inode_info *ci; + struct ceph_cap *cap; + u64 oldest_flush_tid; + + doutc(cl, "mds%d\n", session->s_mds); + + spin_lock(&mdsc->cap_dirty_lock); + oldest_flush_tid = __get_oldest_flush_tid(mdsc); + spin_unlock(&mdsc->cap_dirty_lock); + + list_for_each_entry(ci, &session->s_cap_flushing, i_flushing_item) { + struct inode *inode = &ci->netfs.inode; + + spin_lock(&ci->i_ceph_lock); + cap = ci->i_auth_cap; + if (!(cap && cap->session == session)) { + pr_err_client(cl, "%p %llx.%llx auth cap %p not mds%d ???\n", + inode, ceph_vinop(inode), cap, + session->s_mds); + spin_unlock(&ci->i_ceph_lock); + continue; + } + + + /* + * if flushing caps were revoked, we re-send the cap flush + * in client reconnect stage. This guarantees MDS * processes + * the cap flush message before issuing the flushing caps to + * other client. + */ + if ((cap->issued & ci->i_flushing_caps) != + ci->i_flushing_caps) { + /* encode_caps_cb() also will reset these sequence + * numbers. make sure sequence numbers in cap flush + * message match later reconnect message */ + cap->seq = 0; + cap->issue_seq = 0; + cap->mseq = 0; + __kick_flushing_caps(mdsc, session, ci, + oldest_flush_tid); + } else { + set_bit(CEPH_I_KICK_FLUSH_BIT, &ci->i_ceph_flags); + } + + spin_unlock(&ci->i_ceph_lock); + } +} + +void ceph_kick_flushing_caps(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_inode_info *ci; + struct ceph_cap *cap; + u64 oldest_flush_tid; + + lockdep_assert_held(&session->s_mutex); + + doutc(cl, "mds%d\n", session->s_mds); + + spin_lock(&mdsc->cap_dirty_lock); + oldest_flush_tid = __get_oldest_flush_tid(mdsc); + spin_unlock(&mdsc->cap_dirty_lock); + + list_for_each_entry(ci, &session->s_cap_flushing, i_flushing_item) { + struct inode *inode = &ci->netfs.inode; + + spin_lock(&ci->i_ceph_lock); + cap = ci->i_auth_cap; + if (!(cap && cap->session == session)) { + pr_err_client(cl, "%p %llx.%llx auth cap %p not mds%d ???\n", + inode, ceph_vinop(inode), cap, + session->s_mds); + spin_unlock(&ci->i_ceph_lock); + continue; + } + if (ci->i_ceph_flags & CEPH_I_KICK_FLUSH) { + __kick_flushing_caps(mdsc, session, ci, + oldest_flush_tid); + } + spin_unlock(&ci->i_ceph_lock); + } +} + +void ceph_kick_flushing_inode_caps(struct ceph_mds_session *session, + struct ceph_inode_info *ci) +{ + struct ceph_mds_client *mdsc = session->s_mdsc; + struct ceph_cap *cap = ci->i_auth_cap; + struct inode *inode = &ci->netfs.inode; + + lockdep_assert_held(&ci->i_ceph_lock); + + doutc(mdsc->fsc->client, "%p %llx.%llx flushing %s\n", + inode, ceph_vinop(inode), + ceph_cap_string(ci->i_flushing_caps)); + + if (!list_empty(&ci->i_cap_flush_list)) { + u64 oldest_flush_tid; + spin_lock(&mdsc->cap_dirty_lock); + list_move_tail(&ci->i_flushing_item, + &cap->session->s_cap_flushing); + oldest_flush_tid = __get_oldest_flush_tid(mdsc); + spin_unlock(&mdsc->cap_dirty_lock); + + __kick_flushing_caps(mdsc, session, ci, oldest_flush_tid); + } +} + + +/* + * Take references to capabilities we hold, so that we don't release + * them to the MDS prematurely. + */ +void ceph_take_cap_refs(struct ceph_inode_info *ci, int got, + bool snap_rwsem_locked) +{ + struct inode *inode = &ci->netfs.inode; + struct ceph_client *cl = ceph_inode_to_client(inode); + + lockdep_assert_held(&ci->i_ceph_lock); + + if (got & CEPH_CAP_PIN) + ci->i_pin_ref++; + if (got & CEPH_CAP_FILE_RD) + ci->i_rd_ref++; + if (got & CEPH_CAP_FILE_CACHE) + ci->i_rdcache_ref++; + if (got & CEPH_CAP_FILE_EXCL) + ci->i_fx_ref++; + if (got & CEPH_CAP_FILE_WR) { + if (ci->i_wr_ref == 0 && !ci->i_head_snapc) { + BUG_ON(!snap_rwsem_locked); + ci->i_head_snapc = ceph_get_snap_context( + ci->i_snap_realm->cached_context); + } + ci->i_wr_ref++; + } + if (got & CEPH_CAP_FILE_BUFFER) { + if (ci->i_wb_ref == 0) + ihold(inode); + ci->i_wb_ref++; + doutc(cl, "%p %llx.%llx wb %d -> %d (?)\n", inode, + ceph_vinop(inode), ci->i_wb_ref-1, ci->i_wb_ref); + } +} + +/* + * Try to grab cap references. Specify those refs we @want, and the + * minimal set we @need. Also include the larger offset we are writing + * to (when applicable), and check against max_size here as well. + * Note that caller is responsible for ensuring max_size increases are + * requested from the MDS. + * + * Returns 0 if caps were not able to be acquired (yet), 1 if succeed, + * or a negative error code. There are 3 special error codes: + * -EAGAIN: need to sleep but non-blocking is specified + * -EFBIG: ask caller to call check_max_size() and try again. + * -EUCLEAN: ask caller to call ceph_renew_caps() and try again. + */ +enum { + /* first 8 bits are reserved for CEPH_FILE_MODE_FOO */ + NON_BLOCKING = (1 << 8), + CHECK_FILELOCK = (1 << 9), +}; + +static int try_get_cap_refs(struct inode *inode, int need, int want, + loff_t endoff, int flags, int *got) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_mds_client *mdsc = ceph_inode_to_fs_client(inode)->mdsc; + struct ceph_client *cl = ceph_inode_to_client(inode); + int ret = 0; + int have, implemented; + bool snap_rwsem_locked = false; + + doutc(cl, "%p %llx.%llx need %s want %s\n", inode, + ceph_vinop(inode), ceph_cap_string(need), + ceph_cap_string(want)); + +again: + spin_lock(&ci->i_ceph_lock); + + if ((flags & CHECK_FILELOCK) && + test_bit(CEPH_I_ERROR_FILELOCK_BIT, &ci->i_ceph_flags)) { + doutc(cl, "%p %llx.%llx error filelock\n", inode, + ceph_vinop(inode)); + ret = -EIO; + goto out_unlock; + } + + /* finish pending truncate */ + while (ci->i_truncate_pending) { + spin_unlock(&ci->i_ceph_lock); + if (snap_rwsem_locked) { + up_read(&mdsc->snap_rwsem); + snap_rwsem_locked = false; + } + __ceph_do_pending_vmtruncate(inode); + spin_lock(&ci->i_ceph_lock); + } + + have = __ceph_caps_issued(ci, &implemented); + + if (have & need & CEPH_CAP_FILE_WR) { + if (endoff >= 0 && endoff > (loff_t)ci->i_max_size) { + doutc(cl, "%p %llx.%llx endoff %llu > maxsize %llu\n", + inode, ceph_vinop(inode), endoff, ci->i_max_size); + if (endoff > ci->i_requested_max_size) + ret = ci->i_auth_cap ? -EFBIG : -EUCLEAN; + goto out_unlock; + } + /* + * If a sync write is in progress, we must wait, so that we + * can get a final snapshot value for size+mtime. + */ + if (__ceph_have_pending_cap_snap(ci)) { + doutc(cl, "%p %llx.%llx cap_snap_pending\n", inode, + ceph_vinop(inode)); + goto out_unlock; + } + } + + if ((have & need) == need) { + /* + * Look at (implemented & ~have & not) so that we keep waiting + * on transition from wanted -> needed caps. This is needed + * for WRBUFFER|WR -> WR to avoid a new WR sync write from + * going before a prior buffered writeback happens. + * + * For RDCACHE|RD -> RD, there is not need to wait and we can + * just exclude the revoking caps and force to sync read. + */ + int not = want & ~(have & need); + int revoking = implemented & ~have; + int exclude = revoking & not; + doutc(cl, "%p %llx.%llx have %s but not %s (revoking %s)\n", + inode, ceph_vinop(inode), ceph_cap_string(have), + ceph_cap_string(not), ceph_cap_string(revoking)); + if (!exclude || !(exclude & CEPH_CAP_FILE_BUFFER)) { + if (!snap_rwsem_locked && + !ci->i_head_snapc && + (need & CEPH_CAP_FILE_WR)) { + if (!down_read_trylock(&mdsc->snap_rwsem)) { + /* + * we can not call down_read() when + * task isn't in TASK_RUNNING state + */ + if (flags & NON_BLOCKING) { + ret = -EAGAIN; + goto out_unlock; + } + + spin_unlock(&ci->i_ceph_lock); + down_read(&mdsc->snap_rwsem); + snap_rwsem_locked = true; + goto again; + } + snap_rwsem_locked = true; + } + if ((have & want) == want) + *got = need | (want & ~exclude); + else + *got = need; + ceph_take_cap_refs(ci, *got, true); + ret = 1; + } + } else { + int session_readonly = false; + int mds_wanted; + if (ci->i_auth_cap && + (need & (CEPH_CAP_FILE_WR | CEPH_CAP_FILE_EXCL))) { + struct ceph_mds_session *s = ci->i_auth_cap->session; + spin_lock(&s->s_cap_lock); + session_readonly = s->s_readonly; + spin_unlock(&s->s_cap_lock); + } + if (session_readonly) { + doutc(cl, "%p %llx.%llx need %s but mds%d readonly\n", + inode, ceph_vinop(inode), ceph_cap_string(need), + ci->i_auth_cap->mds); + ret = -EROFS; + goto out_unlock; + } + + if (ceph_inode_is_shutdown(inode)) { + doutc(cl, "%p %llx.%llx inode is shutdown\n", + inode, ceph_vinop(inode)); + ret = -ESTALE; + goto out_unlock; + } + mds_wanted = __ceph_caps_mds_wanted(ci, false); + if (need & ~mds_wanted) { + doutc(cl, "%p %llx.%llx need %s > mds_wanted %s\n", + inode, ceph_vinop(inode), ceph_cap_string(need), + ceph_cap_string(mds_wanted)); + ret = -EUCLEAN; + goto out_unlock; + } + + doutc(cl, "%p %llx.%llx have %s need %s\n", inode, + ceph_vinop(inode), ceph_cap_string(have), + ceph_cap_string(need)); + } +out_unlock: + + __ceph_touch_fmode(ci, mdsc, flags); + + spin_unlock(&ci->i_ceph_lock); + if (snap_rwsem_locked) + up_read(&mdsc->snap_rwsem); + + if (!ret) + ceph_update_cap_mis(&mdsc->metric); + else if (ret == 1) + ceph_update_cap_hit(&mdsc->metric); + + doutc(cl, "%p %llx.%llx ret %d got %s\n", inode, + ceph_vinop(inode), ret, ceph_cap_string(*got)); + return ret; +} + +/* + * Check the offset we are writing up to against our current + * max_size. If necessary, tell the MDS we want to write to + * a larger offset. + */ +static void check_max_size(struct inode *inode, loff_t endoff) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_client *cl = ceph_inode_to_client(inode); + int check = 0; + + /* do we need to explicitly request a larger max_size? */ + spin_lock(&ci->i_ceph_lock); + if (endoff >= ci->i_max_size && endoff > ci->i_wanted_max_size) { + doutc(cl, "write %p %llx.%llx at large endoff %llu, req max_size\n", + inode, ceph_vinop(inode), endoff); + ci->i_wanted_max_size = endoff; + } + /* duplicate ceph_check_caps()'s logic */ + if (ci->i_auth_cap && + (ci->i_auth_cap->issued & CEPH_CAP_FILE_WR) && + ci->i_wanted_max_size > ci->i_max_size && + ci->i_wanted_max_size > ci->i_requested_max_size) + check = 1; + spin_unlock(&ci->i_ceph_lock); + if (check) + ceph_check_caps(ci, CHECK_CAPS_AUTHONLY); +} + +static inline int get_used_fmode(int caps) +{ + int fmode = 0; + if (caps & CEPH_CAP_FILE_RD) + fmode |= CEPH_FILE_MODE_RD; + if (caps & CEPH_CAP_FILE_WR) + fmode |= CEPH_FILE_MODE_WR; + return fmode; +} + +int ceph_try_get_caps(struct inode *inode, int need, int want, + bool nonblock, int *got) +{ + int ret, flags; + + BUG_ON(need & ~CEPH_CAP_FILE_RD); + BUG_ON(want & ~(CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_LAZYIO | + CEPH_CAP_FILE_SHARED | CEPH_CAP_FILE_EXCL | + CEPH_CAP_ANY_DIR_OPS)); + if (need) { + ret = ceph_pool_perm_check(inode, need); + if (ret < 0) + return ret; + } + + flags = get_used_fmode(need | want); + if (nonblock) + flags |= NON_BLOCKING; + + ret = try_get_cap_refs(inode, need, want, 0, flags, got); + /* three special error codes */ + if (ret == -EAGAIN || ret == -EFBIG || ret == -EUCLEAN) + ret = 0; + return ret; +} + +/* + * Wait for caps, and take cap references. If we can't get a WR cap + * due to a small max_size, make sure we check_max_size (and possibly + * ask the mds) so we don't get hung up indefinitely. + */ +int __ceph_get_caps(struct inode *inode, struct ceph_file_info *fi, int need, + int want, loff_t endoff, int *got) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + int ret, _got, flags; + + ret = ceph_pool_perm_check(inode, need); + if (ret < 0) + return ret; + + if (fi && (fi->fmode & CEPH_FILE_MODE_WR) && + fi->filp_gen != READ_ONCE(fsc->filp_gen)) + return -EBADF; + + flags = get_used_fmode(need | want); + + while (true) { + flags &= CEPH_FILE_MODE_MASK; + if (vfs_inode_has_locks(inode)) + flags |= CHECK_FILELOCK; + _got = 0; + ret = try_get_cap_refs(inode, need, want, endoff, + flags, &_got); + WARN_ON_ONCE(ret == -EAGAIN); + if (!ret) { +#ifdef CONFIG_DEBUG_FS + struct ceph_mds_client *mdsc = fsc->mdsc; + struct cap_wait cw; +#endif + DEFINE_WAIT_FUNC(wait, woken_wake_function); + +#ifdef CONFIG_DEBUG_FS + cw.ino = ceph_ino(inode); + cw.tgid = current->tgid; + cw.need = need; + cw.want = want; + + spin_lock(&mdsc->caps_list_lock); + list_add(&cw.list, &mdsc->cap_wait_list); + spin_unlock(&mdsc->caps_list_lock); +#endif + + /* make sure used fmode not timeout */ + ceph_get_fmode(ci, flags, FMODE_WAIT_BIAS); + add_wait_queue(&ci->i_cap_wq, &wait); + + flags |= NON_BLOCKING; + while (!(ret = try_get_cap_refs(inode, need, want, + endoff, flags, &_got))) { + if (signal_pending(current)) { + ret = -ERESTARTSYS; + break; + } + + /* + * If a cap update is lost after + * mds_wanted was raised, waiting + * forever will never make progress. + * Retry the renew path periodically + * so we can resend synchronously. + */ + if (!wait_woken(&wait, TASK_INTERRUPTIBLE, + CEPH_GET_CAPS_WAIT_TIMEOUT)) { + ret = -EUCLEAN; + break; + } + } + + remove_wait_queue(&ci->i_cap_wq, &wait); + ceph_put_fmode(ci, flags, FMODE_WAIT_BIAS); + +#ifdef CONFIG_DEBUG_FS + spin_lock(&mdsc->caps_list_lock); + list_del(&cw.list); + spin_unlock(&mdsc->caps_list_lock); +#endif + + if (ret == -EAGAIN) + continue; + } + + if (fi && (fi->fmode & CEPH_FILE_MODE_WR) && + fi->filp_gen != READ_ONCE(fsc->filp_gen)) { + if (ret >= 0 && _got) + ceph_put_cap_refs(ci, _got); + return -EBADF; + } + + if (ret < 0) { + if (ret == -EFBIG || ret == -EUCLEAN) { + int ret2 = ceph_wait_on_async_create(inode); + if (ret2 < 0) + return ret2; + } + if (ret == -EFBIG) { + check_max_size(inode, endoff); + continue; + } + if (ret == -EUCLEAN) { + /* session was killed or a waited cap + * request needs a retry */ + ret = ceph_renew_caps(inode, flags); + if (ret == 0) + continue; + } + return ret; + } + + if (S_ISREG(ci->netfs.inode.i_mode) && + ceph_has_inline_data(ci) && + (_got & (CEPH_CAP_FILE_CACHE|CEPH_CAP_FILE_LAZYIO)) && + i_size_read(inode) > 0) { + struct page *page = + find_get_page(inode->i_mapping, 0); + if (page) { + bool uptodate = PageUptodate(page); + + put_page(page); + if (uptodate) + break; + } + /* + * drop cap refs first because getattr while + * holding * caps refs can cause deadlock. + */ + ceph_put_cap_refs(ci, _got); + _got = 0; + + /* + * getattr request will bring inline data into + * page cache + */ + ret = __ceph_do_getattr(inode, NULL, + CEPH_STAT_CAP_INLINE_DATA, + true); + if (ret < 0) + return ret; + continue; + } + break; + } + *got = _got; + return 0; +} + +int ceph_get_caps(struct file *filp, int need, int want, loff_t endoff, + int *got) +{ + struct ceph_file_info *fi = filp->private_data; + struct inode *inode = file_inode(filp); + + return __ceph_get_caps(inode, fi, need, want, endoff, got); +} + +/* + * Take cap refs. Caller must already know we hold at least one ref + * on the caps in question or we don't know this is safe. + */ +void ceph_get_cap_refs(struct ceph_inode_info *ci, int caps) +{ + spin_lock(&ci->i_ceph_lock); + ceph_take_cap_refs(ci, caps, false); + spin_unlock(&ci->i_ceph_lock); +} + + +/* + * drop cap_snap that is not associated with any snapshot. + * we don't need to send FLUSHSNAP message for it. + */ +static int ceph_try_drop_cap_snap(struct ceph_inode_info *ci, + struct ceph_cap_snap *capsnap) +{ + struct inode *inode = &ci->netfs.inode; + struct ceph_client *cl = ceph_inode_to_client(inode); + + if (!capsnap->need_flush && + !capsnap->writing && !capsnap->dirty_pages) { + doutc(cl, "%p follows %llu\n", capsnap, capsnap->follows); + BUG_ON(capsnap->cap_flush.tid > 0); + ceph_put_snap_context(capsnap->context); + if (!list_is_last(&capsnap->ci_item, &ci->i_cap_snaps)) + set_bit(CEPH_I_FLUSH_SNAPS_BIT, &ci->i_ceph_flags); + + list_del(&capsnap->ci_item); + ceph_put_cap_snap(capsnap); + return 1; + } + return 0; +} + +enum put_cap_refs_mode { + PUT_CAP_REFS_SYNC = 0, + PUT_CAP_REFS_ASYNC, +}; + +/* + * Release cap refs. + * + * If we released the last ref on any given cap, call ceph_check_caps + * to release (or schedule a release). + * + * If we are releasing a WR cap (from a sync write), finalize any affected + * cap_snap, and wake up any waiters. + */ +static void __ceph_put_cap_refs(struct ceph_inode_info *ci, int had, + enum put_cap_refs_mode mode) +{ + struct inode *inode = &ci->netfs.inode; + struct ceph_client *cl = ceph_inode_to_client(inode); + int last = 0, put = 0, flushsnaps = 0, wake = 0; + bool check_flushsnaps = false; + + spin_lock(&ci->i_ceph_lock); + if (had & CEPH_CAP_PIN) + --ci->i_pin_ref; + if (had & CEPH_CAP_FILE_RD) + if (--ci->i_rd_ref == 0) + last++; + if (had & CEPH_CAP_FILE_CACHE) + if (--ci->i_rdcache_ref == 0) + last++; + if (had & CEPH_CAP_FILE_EXCL) + if (--ci->i_fx_ref == 0) + last++; + if (had & CEPH_CAP_FILE_BUFFER) { + if (--ci->i_wb_ref == 0) { + last++; + /* put the ref held by ceph_take_cap_refs() */ + put++; + check_flushsnaps = true; + } + doutc(cl, "%p %llx.%llx wb %d -> %d (?)\n", inode, + ceph_vinop(inode), ci->i_wb_ref+1, ci->i_wb_ref); + } + if (had & CEPH_CAP_FILE_WR) { + if (--ci->i_wr_ref == 0) { + /* + * The Fb caps will always be took and released + * together with the Fw caps. + */ + WARN_ON_ONCE(ci->i_wb_ref); + + last++; + check_flushsnaps = true; + if (ci->i_wrbuffer_ref_head == 0 && + ci->i_dirty_caps == 0 && + ci->i_flushing_caps == 0) { + BUG_ON(!ci->i_head_snapc); + ceph_put_snap_context(ci->i_head_snapc); + ci->i_head_snapc = NULL; + } + /* see comment in __ceph_remove_cap() */ + if (!__ceph_is_any_real_caps(ci) && ci->i_snap_realm) + ceph_change_snap_realm(inode, NULL); + } + } + if (check_flushsnaps && __ceph_have_pending_cap_snap(ci)) { + struct ceph_cap_snap *capsnap = + list_last_entry(&ci->i_cap_snaps, + struct ceph_cap_snap, + ci_item); + + capsnap->writing = 0; + if (ceph_try_drop_cap_snap(ci, capsnap)) + /* put the ref held by ceph_queue_cap_snap() */ + put++; + else if (__ceph_finish_cap_snap(ci, capsnap)) + flushsnaps = 1; + wake = 1; + } + spin_unlock(&ci->i_ceph_lock); + + doutc(cl, "%p %llx.%llx had %s%s%s\n", inode, ceph_vinop(inode), + ceph_cap_string(had), last ? " last" : "", put ? " put" : ""); + + switch (mode) { + case PUT_CAP_REFS_SYNC: + if (last) + ceph_check_caps(ci, 0); + else if (flushsnaps) + ceph_flush_snaps(ci, NULL); + break; + case PUT_CAP_REFS_ASYNC: + if (last) + ceph_queue_check_caps(inode); + else if (flushsnaps) + ceph_queue_flush_snaps(inode); + break; + default: + break; + } + if (wake) + wake_up_all(&ci->i_cap_wq); + while (put-- > 0) + iput(inode); +} + +void ceph_put_cap_refs(struct ceph_inode_info *ci, int had) +{ + __ceph_put_cap_refs(ci, had, PUT_CAP_REFS_SYNC); +} + +void ceph_put_cap_refs_async(struct ceph_inode_info *ci, int had) +{ + __ceph_put_cap_refs(ci, had, PUT_CAP_REFS_ASYNC); +} + +/* + * Release @nr WRBUFFER refs on dirty pages for the given @snapc snap + * context. Adjust per-snap dirty page accounting as appropriate. + * Once all dirty data for a cap_snap is flushed, flush snapped file + * metadata back to the MDS. If we dropped the last ref, call + * ceph_check_caps. + */ +void ceph_put_wrbuffer_cap_refs(struct ceph_inode_info *ci, int nr, + struct ceph_snap_context *snapc) +{ + struct inode *inode = &ci->netfs.inode; + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_cap_snap *capsnap = NULL, *iter; + int put = 0; + bool last = false; + bool flush_snaps = false; + bool complete_capsnap = false; + + spin_lock(&ci->i_ceph_lock); + ci->i_wrbuffer_ref -= nr; + if (ci->i_wrbuffer_ref == 0) { + last = true; + put++; + } + + if (ci->i_head_snapc == snapc) { + ci->i_wrbuffer_ref_head -= nr; + if (ci->i_wrbuffer_ref_head == 0 && + ci->i_wr_ref == 0 && + ci->i_dirty_caps == 0 && + ci->i_flushing_caps == 0) { + BUG_ON(!ci->i_head_snapc); + ceph_put_snap_context(ci->i_head_snapc); + ci->i_head_snapc = NULL; + } + doutc(cl, "on %p %llx.%llx head %d/%d -> %d/%d %s\n", + inode, ceph_vinop(inode), ci->i_wrbuffer_ref+nr, + ci->i_wrbuffer_ref_head+nr, ci->i_wrbuffer_ref, + ci->i_wrbuffer_ref_head, last ? " LAST" : ""); + } else { + list_for_each_entry(iter, &ci->i_cap_snaps, ci_item) { + if (iter->context == snapc) { + capsnap = iter; + break; + } + } + + if (!capsnap) { + /* + * The capsnap should already be removed when removing + * auth cap in the case of a forced unmount. + */ + WARN_ON_ONCE(ci->i_auth_cap); + goto unlock; + } + + capsnap->dirty_pages -= nr; + if (capsnap->dirty_pages == 0) { + complete_capsnap = true; + if (!capsnap->writing) { + if (ceph_try_drop_cap_snap(ci, capsnap)) { + put++; + } else { + set_bit(CEPH_I_FLUSH_SNAPS_BIT, &ci->i_ceph_flags); + flush_snaps = true; + } + } + } + doutc(cl, "%p %llx.%llx cap_snap %p snap %lld %d/%d -> %d/%d %s%s\n", + inode, ceph_vinop(inode), capsnap, capsnap->context->seq, + ci->i_wrbuffer_ref+nr, capsnap->dirty_pages + nr, + ci->i_wrbuffer_ref, capsnap->dirty_pages, + last ? " (wrbuffer last)" : "", + complete_capsnap ? " (complete capsnap)" : ""); + } + +unlock: + spin_unlock(&ci->i_ceph_lock); + + if (last) { + ceph_check_caps(ci, 0); + } else if (flush_snaps) { + ceph_flush_snaps(ci, NULL); + } + if (complete_capsnap) + wake_up_all(&ci->i_cap_wq); + while (put-- > 0) { + iput(inode); + } +} + +/* + * Invalidate unlinked inode's aliases, so we can drop the inode ASAP. + */ +static void invalidate_aliases(struct inode *inode) +{ + struct ceph_client *cl = ceph_inode_to_client(inode); + struct dentry *dn, *prev = NULL; + + doutc(cl, "%p %llx.%llx\n", inode, ceph_vinop(inode)); + d_prune_aliases(inode); + /* + * For non-directory inode, d_find_alias() only returns + * hashed dentry. After calling d_invalidate(), the + * dentry becomes unhashed. + * + * For directory inode, d_find_alias() can return + * unhashed dentry. But directory inode should have + * one alias at most. + */ + while ((dn = d_find_alias(inode))) { + if (dn == prev) { + dput(dn); + break; + } + d_invalidate(dn); + if (prev) + dput(prev); + prev = dn; + } + if (prev) + dput(prev); +} + +struct cap_extra_info { + struct ceph_string *pool_ns; + /* inline data */ + u64 inline_version; + void *inline_data; + u32 inline_len; + /* dirstat */ + bool dirstat_valid; + u64 nfiles; + u64 nsubdirs; + u64 change_attr; + /* currently issued */ + int issued; + struct timespec64 btime; + u8 *fscrypt_auth; + u32 fscrypt_auth_len; + u64 fscrypt_file_size; +}; + +/* + * Handle a cap GRANT message from the MDS. (Note that a GRANT may + * actually be a revocation if it specifies a smaller cap set.) + * + * caller holds s_mutex and i_ceph_lock, we drop both. + */ +static void handle_cap_grant(struct inode *inode, + struct ceph_mds_session *session, + struct ceph_cap *cap, + struct ceph_mds_caps *grant, + struct ceph_buffer *xattr_buf, + struct cap_extra_info *extra_info) + __releases(ci->i_ceph_lock) + __releases(session->s_mdsc->snap_rwsem) +{ + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + int seq = le32_to_cpu(grant->seq); + int newcaps = le32_to_cpu(grant->caps); + int used, wanted, dirty; + u64 size = le64_to_cpu(grant->size); + u64 max_size = le64_to_cpu(grant->max_size); + unsigned char check_caps = 0; + bool was_stale = cap->cap_gen < atomic_read(&session->s_cap_gen); + bool wake = false; + bool writeback = false; + bool queue_trunc = false; + bool queue_invalidate = false; + bool deleted_inode = false; + bool fill_inline = false; + bool revoke_wait = false; + int flags = 0; + + /* + * If there is at least one crypto block then we'll trust + * fscrypt_file_size. If the real length of the file is 0, then + * ignore it (it has probably been truncated down to 0 by the MDS). + */ + if (IS_ENCRYPTED(inode) && size) + size = extra_info->fscrypt_file_size; + + doutc(cl, "%p %llx.%llx cap %p mds%d seq %d %s\n", inode, + ceph_vinop(inode), cap, session->s_mds, seq, + ceph_cap_string(newcaps)); + doutc(cl, " size %llu max_size %llu, i_size %llu\n", size, + max_size, i_size_read(inode)); + + + /* + * If CACHE is being revoked, and we have no dirty buffers, + * try to invalidate (once). (If there are dirty buffers, we + * will invalidate _after_ writeback.) + */ + if (S_ISREG(inode->i_mode) && /* don't invalidate readdir cache */ + ((cap->issued & ~newcaps) & CEPH_CAP_FILE_CACHE) && + (newcaps & CEPH_CAP_FILE_LAZYIO) == 0 && + !(ci->i_wrbuffer_ref || ci->i_wb_ref)) { + if (try_nonblocking_invalidate(inode)) { + /* there were locked pages.. invalidate later + in a separate thread. */ + if (ci->i_rdcache_revoking != ci->i_rdcache_gen) { + queue_invalidate = true; + ci->i_rdcache_revoking = ci->i_rdcache_gen; + } + } + } + + if (was_stale) + cap->issued = cap->implemented = CEPH_CAP_PIN; + + /* + * auth mds of the inode changed. we received the cap export message, + * but still haven't received the cap import message. handle_cap_export + * updated the new auth MDS' cap. + * + * "ceph_seq_cmp(seq, cap->seq) <= 0" means we are processing a message + * that was sent before the cap import message. So don't remove caps. + */ + if (ceph_seq_cmp(seq, cap->seq) <= 0) { + WARN_ON(cap != ci->i_auth_cap); + WARN_ON(cap->cap_id != le64_to_cpu(grant->cap_id)); + seq = cap->seq; + newcaps |= cap->issued; + } + + /* side effects now are allowed */ + cap->cap_gen = atomic_read(&session->s_cap_gen); + cap->seq = seq; + + __check_cap_issue(ci, cap, newcaps); + + inode_set_max_iversion_raw(inode, extra_info->change_attr); + + if ((newcaps & CEPH_CAP_AUTH_SHARED) && + (extra_info->issued & CEPH_CAP_AUTH_EXCL) == 0) { + umode_t mode = le32_to_cpu(grant->mode); + + if (inode_wrong_type(inode, mode)) + pr_warn_once("inode type changed! (ino %llx.%llx is 0%o, mds says 0%o)\n", + ceph_vinop(inode), inode->i_mode, mode); + else + inode->i_mode = mode; + inode->i_uid = make_kuid(&init_user_ns, le32_to_cpu(grant->uid)); + inode->i_gid = make_kgid(&init_user_ns, le32_to_cpu(grant->gid)); + ci->i_btime = extra_info->btime; + doutc(cl, "%p %llx.%llx mode 0%o uid.gid %d.%d\n", inode, + ceph_vinop(inode), inode->i_mode, + from_kuid(&init_user_ns, inode->i_uid), + from_kgid(&init_user_ns, inode->i_gid)); +#if IS_ENABLED(CONFIG_FS_ENCRYPTION) + if (ci->fscrypt_auth_len != extra_info->fscrypt_auth_len || + memcmp(ci->fscrypt_auth, extra_info->fscrypt_auth, + ci->fscrypt_auth_len)) + pr_warn_ratelimited_client(cl, + "cap grant attempt to change fscrypt_auth on non-I_NEW inode (old len %d new len %d)\n", + ci->fscrypt_auth_len, + extra_info->fscrypt_auth_len); +#endif + } + + if ((newcaps & CEPH_CAP_LINK_SHARED) && + (extra_info->issued & CEPH_CAP_LINK_EXCL) == 0) { + set_nlink(inode, le32_to_cpu(grant->nlink)); + if (inode->i_nlink == 0) + deleted_inode = true; + } + + if ((extra_info->issued & CEPH_CAP_XATTR_EXCL) == 0 && + grant->xattr_len) { + int len = le32_to_cpu(grant->xattr_len); + u64 version = le64_to_cpu(grant->xattr_version); + + if (version > ci->i_xattrs.version) { + doutc(cl, " got new xattrs v%llu on %p %llx.%llx len %d\n", + version, inode, ceph_vinop(inode), len); + if (ci->i_xattrs.blob) + ceph_buffer_put(ci->i_xattrs.blob); + ci->i_xattrs.blob = ceph_buffer_get(xattr_buf); + ci->i_xattrs.version = version; + ceph_forget_all_cached_acls(inode); + ceph_security_invalidate_secctx(inode); + } + } + + if (newcaps & CEPH_CAP_ANY_RD) { + struct timespec64 mtime, atime, ctime; + /* ctime/mtime/atime? */ + ceph_decode_timespec64(&mtime, &grant->mtime); + ceph_decode_timespec64(&atime, &grant->atime); + ceph_decode_timespec64(&ctime, &grant->ctime); + ceph_fill_file_time(inode, extra_info->issued, + le32_to_cpu(grant->time_warp_seq), + &ctime, &mtime, &atime); + } + + if ((newcaps & CEPH_CAP_FILE_SHARED) && extra_info->dirstat_valid) { + ci->i_files = extra_info->nfiles; + ci->i_subdirs = extra_info->nsubdirs; + } + + if (newcaps & (CEPH_CAP_ANY_FILE_RD | CEPH_CAP_ANY_FILE_WR)) { + /* file layout may have changed */ + s64 old_pool = ci->i_layout.pool_id; + struct ceph_string *old_ns; + + ceph_file_layout_from_legacy(&ci->i_layout, &grant->layout); + old_ns = rcu_dereference_protected(ci->i_layout.pool_ns, + lockdep_is_held(&ci->i_ceph_lock)); + rcu_assign_pointer(ci->i_layout.pool_ns, extra_info->pool_ns); + + if (ci->i_layout.pool_id != old_pool || + extra_info->pool_ns != old_ns) + clear_bit(CEPH_I_POOL_PERM_BIT, &ci->i_ceph_flags); + + extra_info->pool_ns = old_ns; + + /* size/truncate_seq? */ + queue_trunc = ceph_fill_file_size(inode, extra_info->issued, + le32_to_cpu(grant->truncate_seq), + le64_to_cpu(grant->truncate_size), + size); + } + + if (ci->i_auth_cap == cap && (newcaps & CEPH_CAP_ANY_FILE_WR)) { + if (max_size != ci->i_max_size) { + doutc(cl, "max_size %lld -> %llu\n", ci->i_max_size, + max_size); + ci->i_max_size = max_size; + if (max_size >= ci->i_wanted_max_size) { + ci->i_wanted_max_size = 0; /* reset */ + ci->i_requested_max_size = 0; + } + wake = true; + } + } + + /* check cap bits */ + wanted = __ceph_caps_wanted(ci); + used = __ceph_caps_used(ci); + dirty = __ceph_caps_dirty(ci); + doutc(cl, " my wanted = %s, used = %s, dirty %s\n", + ceph_cap_string(wanted), ceph_cap_string(used), + ceph_cap_string(dirty)); + + if ((was_stale || le32_to_cpu(grant->op) == CEPH_CAP_OP_IMPORT) && + (wanted & ~(cap->mds_wanted | newcaps))) { + /* + * If mds is importing cap, prior cap messages that update + * 'wanted' may get dropped by mds (migrate seq mismatch). + * + * We don't send cap message to update 'wanted' if what we + * want are already issued. If mds revokes caps, cap message + * that releases caps also tells mds what we want. But if + * caps got revoked by mds forcedly (session stale). We may + * haven't told mds what we want. + */ + check_caps = 1; + } + + /* revocation, grant, or no-op? */ + if (cap->issued & ~newcaps) { + int revoking = cap->issued & ~newcaps; + + doutc(cl, "revocation: %s -> %s (revoking %s)\n", + ceph_cap_string(cap->issued), ceph_cap_string(newcaps), + ceph_cap_string(revoking)); + if (S_ISREG(inode->i_mode) && + (revoking & used & CEPH_CAP_FILE_BUFFER)) { + writeback = true; /* initiate writeback; will delay ack */ + revoke_wait = true; + } else if (queue_invalidate && + revoking == CEPH_CAP_FILE_CACHE && + (newcaps & CEPH_CAP_FILE_LAZYIO) == 0) { + revoke_wait = true; /* do nothing yet, invalidation will be queued */ + } else if (cap == ci->i_auth_cap) { + check_caps = 1; /* check auth cap only */ + } else { + check_caps = 2; /* check all caps */ + } + /* If there is new caps, try to wake up the waiters */ + if (~cap->issued & newcaps) + wake = true; + cap->issued = newcaps; + cap->implemented |= newcaps; + } else if (cap->issued == newcaps) { + doutc(cl, "caps unchanged: %s -> %s\n", + ceph_cap_string(cap->issued), + ceph_cap_string(newcaps)); + } else { + doutc(cl, "grant: %s -> %s\n", ceph_cap_string(cap->issued), + ceph_cap_string(newcaps)); + /* non-auth MDS is revoking the newly grant caps ? */ + if (cap == ci->i_auth_cap && + __ceph_caps_revoking_other(ci, cap, newcaps)) + check_caps = 2; + + cap->issued = newcaps; + cap->implemented |= newcaps; /* add bits only, to + * avoid stepping on a + * pending revocation */ + wake = true; + } + BUG_ON(cap->issued & ~cap->implemented); + + /* don't let check_caps skip sending a response to MDS for revoke msgs */ + if (!revoke_wait && le32_to_cpu(grant->op) == CEPH_CAP_OP_REVOKE) { + cap->mds_wanted = 0; + flags |= CHECK_CAPS_FLUSH_FORCE; + if (cap == ci->i_auth_cap) + check_caps = 1; /* check auth cap only */ + else + check_caps = 2; /* check all caps */ + } + + if (extra_info->inline_version > 0 && + extra_info->inline_version >= ci->i_inline_version) { + ci->i_inline_version = extra_info->inline_version; + if (ci->i_inline_version != CEPH_INLINE_NONE && + (newcaps & (CEPH_CAP_FILE_CACHE|CEPH_CAP_FILE_LAZYIO))) + fill_inline = true; + } + + if (le32_to_cpu(grant->op) == CEPH_CAP_OP_IMPORT) { + if (ci->i_auth_cap == cap) { + if (newcaps & ~extra_info->issued) + wake = true; + + if (ci->i_requested_max_size > max_size || + !(le32_to_cpu(grant->wanted) & CEPH_CAP_ANY_FILE_WR)) { + /* re-request max_size if necessary */ + ci->i_requested_max_size = 0; + wake = true; + } + + ceph_kick_flushing_inode_caps(session, ci); + } + up_read(&session->s_mdsc->snap_rwsem); + } + spin_unlock(&ci->i_ceph_lock); + + if (fill_inline) + ceph_fill_inline_data(inode, NULL, extra_info->inline_data, + extra_info->inline_len); + + if (queue_trunc) + ceph_queue_vmtruncate(inode); + + if (writeback) + /* + * queue inode for writeback: we can't actually call + * filemap_write_and_wait, etc. from message handler + * context. + */ + ceph_queue_writeback(inode); + if (queue_invalidate) + ceph_queue_invalidate(inode); + if (deleted_inode) + invalidate_aliases(inode); + if (wake) + wake_up_all(&ci->i_cap_wq); + + mutex_unlock(&session->s_mutex); + if (check_caps == 1) + ceph_check_caps(ci, flags | CHECK_CAPS_AUTHONLY | CHECK_CAPS_NOINVAL); + else if (check_caps == 2) + ceph_check_caps(ci, flags | CHECK_CAPS_NOINVAL); +} + +/* + * Handle FLUSH_ACK from MDS, indicating that metadata we sent to the + * MDS has been safely committed. + */ +static void handle_cap_flush_ack(struct inode *inode, u64 flush_tid, + struct ceph_mds_caps *m, + struct ceph_mds_session *session, + struct ceph_cap *cap) + __releases(ci->i_ceph_lock) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_mds_client *mdsc = ceph_sb_to_fs_client(inode->i_sb)->mdsc; + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_cap_flush *cf, *tmp_cf; + LIST_HEAD(to_remove); + unsigned seq = le32_to_cpu(m->seq); + int dirty = le32_to_cpu(m->dirty); + int cleaned = 0; + bool drop = false; + bool wake_ci = false; + bool wake_mdsc = false; + + /* + * Flush tids are monotonically increasing and acks arrive in + * order under i_ceph_lock, so this is always the latest tid. + * Diagnostic readers use READ_ONCE() without holding the lock. + */ + WRITE_ONCE(ci->i_last_cap_flush_ack, flush_tid); + + list_for_each_entry_safe(cf, tmp_cf, &ci->i_cap_flush_list, i_list) { + /* Is this the one that was flushed? */ + if (cf->tid == flush_tid) + cleaned = cf->caps; + + /* Is this a capsnap? */ + if (cf->is_capsnap) + continue; + + if (cf->tid <= flush_tid) { + /* + * An earlier or current tid. The FLUSH_ACK should + * represent a superset of this flush's caps. + */ + wake_ci |= __detach_cap_flush_from_ci(ci, cf); + list_add_tail(&cf->i_list, &to_remove); + } else { + /* + * This is a later one. Any caps in it are still dirty + * so don't count them as cleaned. + */ + cleaned &= ~cf->caps; + if (!cleaned) + break; + } + } + + doutc(cl, "%p %llx.%llx mds%d seq %d on %s cleaned %s, flushing %s -> %s\n", + inode, ceph_vinop(inode), session->s_mds, seq, + ceph_cap_string(dirty), ceph_cap_string(cleaned), + ceph_cap_string(ci->i_flushing_caps), + ceph_cap_string(ci->i_flushing_caps & ~cleaned)); + + if (list_empty(&to_remove) && !cleaned) + goto out; + + ci->i_flushing_caps &= ~cleaned; + + spin_lock(&mdsc->cap_dirty_lock); + + list_for_each_entry(cf, &to_remove, i_list) + wake_mdsc |= __detach_cap_flush_from_mdsc(mdsc, cf); + + if (ci->i_flushing_caps == 0) { + if (list_empty(&ci->i_cap_flush_list)) { + list_del_init(&ci->i_flushing_item); + if (!list_empty(&session->s_cap_flushing)) { + struct inode *inode = + &list_first_entry(&session->s_cap_flushing, + struct ceph_inode_info, + i_flushing_item)->netfs.inode; + doutc(cl, " mds%d still flushing cap on %p %llx.%llx\n", + session->s_mds, inode, ceph_vinop(inode)); + } + } + mdsc->num_cap_flushing--; + doutc(cl, " %p %llx.%llx now !flushing\n", inode, + ceph_vinop(inode)); + + if (ci->i_dirty_caps == 0) { + doutc(cl, " %p %llx.%llx now clean\n", inode, + ceph_vinop(inode)); + BUG_ON(!list_empty(&ci->i_dirty_item)); + drop = true; + if (ci->i_wr_ref == 0 && + ci->i_wrbuffer_ref_head == 0) { + BUG_ON(!ci->i_head_snapc); + ceph_put_snap_context(ci->i_head_snapc); + ci->i_head_snapc = NULL; + } + } else { + BUG_ON(list_empty(&ci->i_dirty_item)); + } + } + spin_unlock(&mdsc->cap_dirty_lock); + +out: + spin_unlock(&ci->i_ceph_lock); + + while (!list_empty(&to_remove)) { + cf = list_first_entry(&to_remove, + struct ceph_cap_flush, i_list); + list_del_init(&cf->i_list); + if (!cf->is_capsnap) + ceph_free_cap_flush(cf); + } + + if (wake_ci) + wake_up_all(&ci->i_cap_wq); + if (wake_mdsc) + wake_up_all(&mdsc->cap_flushing_wq); + if (drop) + iput(inode); +} + +void __ceph_remove_capsnap(struct inode *inode, struct ceph_cap_snap *capsnap, + bool *wake_ci, bool *wake_mdsc) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_mds_client *mdsc = ceph_sb_to_fs_client(inode->i_sb)->mdsc; + struct ceph_client *cl = mdsc->fsc->client; + bool ret; + + lockdep_assert_held(&ci->i_ceph_lock); + + doutc(cl, "removing capsnap %p, %p %llx.%llx ci %p\n", capsnap, + inode, ceph_vinop(inode), ci); + + list_del_init(&capsnap->ci_item); + ret = __detach_cap_flush_from_ci(ci, &capsnap->cap_flush); + if (wake_ci) + *wake_ci = ret; + + spin_lock(&mdsc->cap_dirty_lock); + if (list_empty(&ci->i_cap_flush_list)) + list_del_init(&ci->i_flushing_item); + + ret = __detach_cap_flush_from_mdsc(mdsc, &capsnap->cap_flush); + if (wake_mdsc) + *wake_mdsc = ret; + spin_unlock(&mdsc->cap_dirty_lock); +} + +void ceph_remove_capsnap(struct inode *inode, struct ceph_cap_snap *capsnap, + bool *wake_ci, bool *wake_mdsc) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + + lockdep_assert_held(&ci->i_ceph_lock); + + WARN_ON_ONCE(capsnap->dirty_pages || capsnap->writing); + __ceph_remove_capsnap(inode, capsnap, wake_ci, wake_mdsc); +} + +/* + * Handle FLUSHSNAP_ACK. MDS has flushed snap data to disk and we can + * throw away our cap_snap. + * + * Caller hold s_mutex. + */ +static void handle_cap_flushsnap_ack(struct inode *inode, u64 flush_tid, + struct ceph_mds_caps *m, + struct ceph_mds_session *session) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_mds_client *mdsc = ceph_sb_to_fs_client(inode->i_sb)->mdsc; + struct ceph_client *cl = mdsc->fsc->client; + u64 follows = le64_to_cpu(m->snap_follows); + struct ceph_cap_snap *capsnap = NULL, *iter; + bool wake_ci = false; + bool wake_mdsc = false; + + doutc(cl, "%p %llx.%llx ci %p mds%d follows %lld\n", inode, + ceph_vinop(inode), ci, session->s_mds, follows); + + spin_lock(&ci->i_ceph_lock); + list_for_each_entry(iter, &ci->i_cap_snaps, ci_item) { + if (iter->follows == follows) { + if (iter->cap_flush.tid != flush_tid) { + doutc(cl, " cap_snap %p follows %lld " + "tid %lld != %lld\n", iter, + follows, flush_tid, + iter->cap_flush.tid); + break; + } + capsnap = iter; + break; + } else { + doutc(cl, " skipping cap_snap %p follows %lld\n", + iter, iter->follows); + } + } + if (capsnap) + ceph_remove_capsnap(inode, capsnap, &wake_ci, &wake_mdsc); + spin_unlock(&ci->i_ceph_lock); + + if (capsnap) { + ceph_put_snap_context(capsnap->context); + ceph_put_cap_snap(capsnap); + if (wake_ci) + wake_up_all(&ci->i_cap_wq); + if (wake_mdsc) + wake_up_all(&mdsc->cap_flushing_wq); + iput(inode); + } +} + +/* + * Handle TRUNC from MDS, indicating file truncation. + * + * caller hold s_mutex. + */ +static bool handle_cap_trunc(struct inode *inode, + struct ceph_mds_caps *trunc, + struct ceph_mds_session *session, + struct cap_extra_info *extra_info) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_client *cl = ceph_inode_to_client(inode); + int mds = session->s_mds; + int seq = le32_to_cpu(trunc->seq); + u32 truncate_seq = le32_to_cpu(trunc->truncate_seq); + u64 truncate_size = le64_to_cpu(trunc->truncate_size); + u64 size = le64_to_cpu(trunc->size); + int implemented = 0; + int dirty = __ceph_caps_dirty(ci); + int issued = __ceph_caps_issued(ceph_inode(inode), &implemented); + bool queue_trunc = false; + + lockdep_assert_held(&ci->i_ceph_lock); + + issued |= implemented | dirty; + + /* + * If there is at least one crypto block then we'll trust + * fscrypt_file_size. If the real length of the file is 0, then + * ignore it (it has probably been truncated down to 0 by the MDS). + */ + if (IS_ENCRYPTED(inode) && size) + size = extra_info->fscrypt_file_size; + + doutc(cl, "%p %llx.%llx mds%d seq %d to %lld truncate seq %d\n", + inode, ceph_vinop(inode), mds, seq, truncate_size, truncate_seq); + queue_trunc = ceph_fill_file_size(inode, issued, + truncate_seq, truncate_size, size); + return queue_trunc; +} + +/* + * Handle EXPORT from MDS. Cap is being migrated _from_ this mds to a + * different one. If we are the most recent migration we've seen (as + * indicated by mseq), make note of the migrating cap bits for the + * duration (until we see the corresponding IMPORT). + * + * caller holds s_mutex + */ +static void handle_cap_export(struct inode *inode, struct ceph_mds_caps *ex, + struct ceph_mds_cap_peer *ph, + struct ceph_mds_session *session) +{ + struct ceph_mds_client *mdsc = ceph_inode_to_fs_client(inode)->mdsc; + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_mds_session *tsession = NULL; + struct ceph_cap *cap, *tcap, *new_cap = NULL; + struct ceph_inode_info *ci = ceph_inode(inode); + u64 t_cap_id; + u32 t_issue_seq, t_mseq; + int target, issued; + int mds = session->s_mds; + + if (ph) { + t_cap_id = le64_to_cpu(ph->cap_id); + t_issue_seq = le32_to_cpu(ph->issue_seq); + t_mseq = le32_to_cpu(ph->mseq); + target = le32_to_cpu(ph->mds); + } else { + t_cap_id = t_issue_seq = t_mseq = 0; + target = -1; + } + + doutc(cl, " cap %llx.%llx export to peer %d piseq %u pmseq %u\n", + ceph_vinop(inode), target, t_issue_seq, t_mseq); +retry: + down_read(&mdsc->snap_rwsem); + spin_lock(&ci->i_ceph_lock); + cap = __get_cap_for_mds(ci, mds); + if (!cap || cap->cap_id != le64_to_cpu(ex->cap_id)) + goto out_unlock; + + if (target < 0) { + ceph_remove_cap(mdsc, cap, false); + goto out_unlock; + } + + /* + * now we know we haven't received the cap import message yet + * because the exported cap still exist. + */ + + issued = cap->issued; + if (issued != cap->implemented) + pr_err_ratelimited_client(cl, "issued != implemented: " + "%p %llx.%llx mds%d seq %d mseq %d" + " issued %s implemented %s\n", + inode, ceph_vinop(inode), mds, + cap->seq, cap->mseq, + ceph_cap_string(issued), + ceph_cap_string(cap->implemented)); + + + tcap = __get_cap_for_mds(ci, target); + if (tcap) { + /* already have caps from the target */ + if (tcap->cap_id == t_cap_id && + ceph_seq_cmp(tcap->seq, t_issue_seq) < 0) { + doutc(cl, " updating import cap %p mds%d\n", tcap, + target); + tcap->cap_id = t_cap_id; + tcap->seq = t_issue_seq - 1; + tcap->issue_seq = t_issue_seq - 1; + tcap->issued |= issued; + tcap->implemented |= issued; + if (cap == ci->i_auth_cap) { + ci->i_auth_cap = tcap; + change_auth_cap_ses(ci, tcap->session); + } + } + ceph_remove_cap(mdsc, cap, false); + goto out_unlock; + } else if (tsession) { + /* add placeholder for the export target */ + int flag = (cap == ci->i_auth_cap) ? CEPH_CAP_FLAG_AUTH : 0; + tcap = new_cap; + ceph_add_cap(inode, tsession, t_cap_id, issued, 0, + t_issue_seq - 1, t_mseq, (u64)-1, flag, &new_cap); + + if (!list_empty(&ci->i_cap_flush_list) && + ci->i_auth_cap == tcap) { + spin_lock(&mdsc->cap_dirty_lock); + list_move_tail(&ci->i_flushing_item, + &tcap->session->s_cap_flushing); + spin_unlock(&mdsc->cap_dirty_lock); + } + + ceph_remove_cap(mdsc, cap, false); + goto out_unlock; + } + + spin_unlock(&ci->i_ceph_lock); + up_read(&mdsc->snap_rwsem); + mutex_unlock(&session->s_mutex); + + /* open target session */ + tsession = ceph_mdsc_open_export_target_session(mdsc, target); + if (!IS_ERR(tsession)) { + if (mds > target) { + mutex_lock(&session->s_mutex); + mutex_lock_nested(&tsession->s_mutex, + SINGLE_DEPTH_NESTING); + } else { + mutex_lock(&tsession->s_mutex); + mutex_lock_nested(&session->s_mutex, + SINGLE_DEPTH_NESTING); + } + new_cap = ceph_get_cap(mdsc, NULL); + } else { + WARN_ON(1); + tsession = NULL; + target = -1; + mutex_lock(&session->s_mutex); + } + goto retry; + +out_unlock: + spin_unlock(&ci->i_ceph_lock); + up_read(&mdsc->snap_rwsem); + mutex_unlock(&session->s_mutex); + if (tsession) { + mutex_unlock(&tsession->s_mutex); + ceph_put_mds_session(tsession); + } + if (new_cap) + ceph_put_cap(mdsc, new_cap); +} + +/* + * Handle cap IMPORT. + * + * caller holds s_mutex. acquires i_ceph_lock + */ +static void handle_cap_import(struct ceph_mds_client *mdsc, + struct inode *inode, struct ceph_mds_caps *im, + struct ceph_mds_cap_peer *ph, + struct ceph_mds_session *session, + struct ceph_cap **target_cap, int *old_issued) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_cap *cap, *ocap, *new_cap = NULL; + int mds = session->s_mds; + int issued; + unsigned caps = le32_to_cpu(im->caps); + unsigned wanted = le32_to_cpu(im->wanted); + unsigned seq = le32_to_cpu(im->seq); + unsigned mseq = le32_to_cpu(im->migrate_seq); + u64 realmino = le64_to_cpu(im->realm); + u64 cap_id = le64_to_cpu(im->cap_id); + u64 p_cap_id; + u32 piseq = 0; + u32 pmseq = 0; + int peer; + + if (ph) { + p_cap_id = le64_to_cpu(ph->cap_id); + peer = le32_to_cpu(ph->mds); + piseq = le32_to_cpu(ph->issue_seq); + pmseq = le32_to_cpu(ph->mseq); + } else { + p_cap_id = 0; + peer = -1; + } + + doutc(cl, " cap %llx.%llx import from peer %d piseq %u pmseq %u\n", + ceph_vinop(inode), peer, piseq, pmseq); +retry: + cap = __get_cap_for_mds(ci, mds); + if (!cap) { + if (!new_cap) { + spin_unlock(&ci->i_ceph_lock); + new_cap = ceph_get_cap(mdsc, NULL); + spin_lock(&ci->i_ceph_lock); + goto retry; + } + cap = new_cap; + } else { + if (new_cap) { + ceph_put_cap(mdsc, new_cap); + new_cap = NULL; + } + } + + __ceph_caps_issued(ci, &issued); + issued |= __ceph_caps_dirty(ci); + + ceph_add_cap(inode, session, cap_id, caps, wanted, seq, mseq, + realmino, CEPH_CAP_FLAG_AUTH, &new_cap); + + ocap = peer >= 0 ? __get_cap_for_mds(ci, peer) : NULL; + if (ocap && ocap->cap_id == p_cap_id) { + doutc(cl, " remove export cap %p mds%d flags %d\n", + ocap, peer, ph->flags); + if ((ph->flags & CEPH_CAP_FLAG_AUTH) && + (ocap->seq != piseq || + ocap->mseq != pmseq)) { + pr_err_ratelimited_client(cl, "mismatched seq/mseq: " + "%p %llx.%llx mds%d seq %d mseq %d" + " importer mds%d has peer seq %d mseq %d\n", + inode, ceph_vinop(inode), peer, + ocap->seq, ocap->mseq, mds, piseq, pmseq); + } + ceph_remove_cap(mdsc, ocap, (ph->flags & CEPH_CAP_FLAG_RELEASE)); + } + + *old_issued = issued; + *target_cap = cap; +} + +#ifdef CONFIG_FS_ENCRYPTION +static int parse_fscrypt_fields(void **p, void *end, + struct cap_extra_info *extra) +{ + u32 len; + + ceph_decode_32_safe(p, end, extra->fscrypt_auth_len, bad); + if (extra->fscrypt_auth_len) { + ceph_decode_need(p, end, extra->fscrypt_auth_len, bad); + extra->fscrypt_auth = kmalloc(extra->fscrypt_auth_len, + GFP_KERNEL); + if (!extra->fscrypt_auth) + return -ENOMEM; + ceph_decode_copy_safe(p, end, extra->fscrypt_auth, + extra->fscrypt_auth_len, bad); + } + + ceph_decode_32_safe(p, end, len, bad); + if (len >= sizeof(u64)) { + ceph_decode_64_safe(p, end, extra->fscrypt_file_size, bad); + len -= sizeof(u64); + } + ceph_decode_skip_n(p, end, len, bad); + return 0; +bad: + return -EIO; +} +#else +static int parse_fscrypt_fields(void **p, void *end, + struct cap_extra_info *extra) +{ + u32 len; + + /* Don't care about these fields unless we're encryption-capable */ + ceph_decode_32_safe(p, end, len, bad); + if (len) + ceph_decode_skip_n(p, end, len, bad); + ceph_decode_32_safe(p, end, len, bad); + if (len) + ceph_decode_skip_n(p, end, len, bad); + return 0; +bad: + return -EIO; +} +#endif + +/* + * Handle a caps message from the MDS. + * + * Identify the appropriate session, inode, and call the right handler + * based on the cap op. + */ +void ceph_handle_caps(struct ceph_mds_session *session, + struct ceph_msg *msg) +{ + struct ceph_mds_client *mdsc = session->s_mdsc; + struct ceph_client *cl = mdsc->fsc->client; + struct inode *inode; + struct ceph_inode_info *ci; + struct ceph_cap *cap; + struct ceph_mds_caps *h; + struct ceph_mds_cap_peer *peer = NULL; + struct ceph_snap_realm *realm = NULL; + int op; + int msg_version = le16_to_cpu(msg->hdr.version); + u32 seq, mseq, issue_seq; + struct ceph_vino vino; + void *snaptrace; + size_t snaptrace_len; + void *p, *end; + struct cap_extra_info extra_info = {}; + bool queue_trunc; + bool close_sessions = false; + bool do_cap_release = false; + + if (!ceph_inc_mds_stopping_blocker(mdsc, session)) + return; + + /* decode */ + end = msg->front.iov_base + msg->front.iov_len; + if (msg->front.iov_len < sizeof(*h)) + goto bad; + h = msg->front.iov_base; + op = le32_to_cpu(h->op); + vino.ino = le64_to_cpu(h->ino); + vino.snap = CEPH_NOSNAP; + seq = le32_to_cpu(h->seq); + mseq = le32_to_cpu(h->migrate_seq); + issue_seq = le32_to_cpu(h->issue_seq); + + snaptrace = h + 1; + snaptrace_len = le32_to_cpu(h->snap_trace_len); + ceph_decode_need(&snaptrace, end, snaptrace_len, bad); + p = snaptrace + snaptrace_len; + + if (msg_version >= 2) { + u32 flock_len; + ceph_decode_32_safe(&p, end, flock_len, bad); + if (p + flock_len > end) + goto bad; + p += flock_len; + } + + if (msg_version >= 3) { + if (op == CEPH_CAP_OP_IMPORT) { + if (p + sizeof(*peer) > end) + goto bad; + peer = p; + p += sizeof(*peer); + } else if (op == CEPH_CAP_OP_EXPORT) { + /* recorded in unused fields */ + peer = (void *)&h->size; + } + } + + if (msg_version >= 4) { + ceph_decode_64_safe(&p, end, extra_info.inline_version, bad); + ceph_decode_32_safe(&p, end, extra_info.inline_len, bad); + if (p + extra_info.inline_len > end) + goto bad; + extra_info.inline_data = p; + p += extra_info.inline_len; + } + + if (msg_version >= 5) { + struct ceph_osd_client *osdc = &mdsc->fsc->client->osdc; + u32 epoch_barrier; + + ceph_decode_32_safe(&p, end, epoch_barrier, bad); + ceph_osdc_update_epoch_barrier(osdc, epoch_barrier); + } + + if (msg_version >= 8) { + u32 pool_ns_len; + + /* version >= 6 */ + ceph_decode_skip_64(&p, end, bad); // flush_tid + /* version >= 7 */ + ceph_decode_skip_32(&p, end, bad); // caller_uid + ceph_decode_skip_32(&p, end, bad); // caller_gid + /* version >= 8 */ + ceph_decode_32_safe(&p, end, pool_ns_len, bad); + if (pool_ns_len > 0) { + ceph_decode_need(&p, end, pool_ns_len, bad); + extra_info.pool_ns = + ceph_find_or_create_string(p, pool_ns_len); + p += pool_ns_len; + } + } + + if (msg_version >= 9) { + struct ceph_timespec *btime; + + if (p + sizeof(*btime) > end) + goto bad; + btime = p; + ceph_decode_timespec64(&extra_info.btime, btime); + p += sizeof(*btime); + ceph_decode_64_safe(&p, end, extra_info.change_attr, bad); + } + + if (msg_version >= 11) { + /* version >= 10 */ + ceph_decode_skip_32(&p, end, bad); // flags + /* version >= 11 */ + extra_info.dirstat_valid = true; + ceph_decode_64_safe(&p, end, extra_info.nfiles, bad); + ceph_decode_64_safe(&p, end, extra_info.nsubdirs, bad); + } + + if (msg_version >= 12) { + if (parse_fscrypt_fields(&p, end, &extra_info)) + goto bad; + } + + /* lookup ino */ + inode = ceph_find_inode(mdsc->fsc->sb, vino); + doutc(cl, " caps mds%d op %s ino %llx.%llx inode %p seq %u iseq %u mseq %u\n", + session->s_mds, ceph_cap_op_name(op), vino.ino, vino.snap, inode, + seq, issue_seq, mseq); + + trace_ceph_handle_caps(mdsc, session, op, &vino, ceph_inode(inode), + seq, issue_seq, mseq); + + mutex_lock(&session->s_mutex); + + if (!inode) { + doutc(cl, " i don't have ino %llx\n", vino.ino); + + switch (op) { + case CEPH_CAP_OP_IMPORT: + case CEPH_CAP_OP_REVOKE: + case CEPH_CAP_OP_GRANT: + do_cap_release = true; + break; + default: + break; + } + goto flush_cap_releases; + } + ci = ceph_inode(inode); + + /* these will work even if we don't have a cap yet */ + switch (op) { + case CEPH_CAP_OP_FLUSHSNAP_ACK: + handle_cap_flushsnap_ack(inode, le64_to_cpu(msg->hdr.tid), + h, session); + goto done; + + case CEPH_CAP_OP_EXPORT: + handle_cap_export(inode, h, peer, session); + goto done_unlocked; + + case CEPH_CAP_OP_IMPORT: + realm = NULL; + if (snaptrace_len) { + down_write(&mdsc->snap_rwsem); + if (ceph_update_snap_trace(mdsc, snaptrace, + snaptrace + snaptrace_len, + false, &realm)) { + up_write(&mdsc->snap_rwsem); + close_sessions = true; + goto done; + } + downgrade_write(&mdsc->snap_rwsem); + } else { + down_read(&mdsc->snap_rwsem); + } + spin_lock(&ci->i_ceph_lock); + handle_cap_import(mdsc, inode, h, peer, session, + &cap, &extra_info.issued); + handle_cap_grant(inode, session, cap, + h, msg->middle, &extra_info); + if (realm) + ceph_put_snap_realm(mdsc, realm); + goto done_unlocked; + } + + /* the rest require a cap */ + spin_lock(&ci->i_ceph_lock); + cap = __get_cap_for_mds(ceph_inode(inode), session->s_mds); + if (!cap) { + doutc(cl, " no cap on %p ino %llx.%llx from mds%d\n", + inode, ceph_ino(inode), ceph_snap(inode), + session->s_mds); + spin_unlock(&ci->i_ceph_lock); + switch (op) { + case CEPH_CAP_OP_REVOKE: + case CEPH_CAP_OP_GRANT: + do_cap_release = true; + break; + default: + break; + } + goto flush_cap_releases; + } + + /* note that each of these drops i_ceph_lock for us */ + switch (op) { + case CEPH_CAP_OP_REVOKE: + case CEPH_CAP_OP_GRANT: + __ceph_caps_issued(ci, &extra_info.issued); + extra_info.issued |= __ceph_caps_dirty(ci); + handle_cap_grant(inode, session, cap, + h, msg->middle, &extra_info); + goto done_unlocked; + + case CEPH_CAP_OP_FLUSH_ACK: + handle_cap_flush_ack(inode, le64_to_cpu(msg->hdr.tid), + h, session, cap); + break; + + case CEPH_CAP_OP_TRUNC: + queue_trunc = handle_cap_trunc(inode, h, session, + &extra_info); + spin_unlock(&ci->i_ceph_lock); + if (queue_trunc) + ceph_queue_vmtruncate(inode); + break; + + default: + spin_unlock(&ci->i_ceph_lock); + pr_err_client(cl, "unknown cap op %d %s\n", op, + ceph_cap_op_name(op)); + } + +done: + mutex_unlock(&session->s_mutex); +done_unlocked: + iput(inode); +out: + ceph_dec_mds_stopping_blocker(mdsc); + + ceph_put_string(extra_info.pool_ns); + + /* Defer closing the sessions after s_mutex lock being released */ + if (close_sessions) + ceph_mdsc_close_sessions(mdsc); + + kfree(extra_info.fscrypt_auth); + return; + +flush_cap_releases: + /* + * send any cap release message to try to move things + * along for the mds (who clearly thinks we still have this + * cap). + */ + if (do_cap_release) { + cap = ceph_get_cap(mdsc, NULL); + cap->cap_ino = vino.ino; + cap->queue_release = 1; + cap->cap_id = le64_to_cpu(h->cap_id); + cap->mseq = mseq; + cap->seq = seq; + cap->issue_seq = seq; + spin_lock(&session->s_cap_lock); + __ceph_queue_cap_release(session, cap); + spin_unlock(&session->s_cap_lock); + } + ceph_flush_session_cap_releases(mdsc, session); + goto done; + +bad: + pr_err_client(cl, "corrupt message\n"); + ceph_msg_dump(msg); + goto out; +} + +/* + * Delayed work handler to process end of delayed cap release LRU list. + * + * If new caps are added to the list while processing it, these won't get + * processed in this run. In this case, the ci->i_hold_caps_max will be + * returned so that the work can be scheduled accordingly. + */ +unsigned long ceph_check_delayed_caps(struct ceph_mds_client *mdsc) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct inode *inode; + struct ceph_inode_info *ci; + struct ceph_mount_options *opt = mdsc->fsc->mount_options; + unsigned long delay_max = opt->caps_wanted_delay_max * HZ; + unsigned long loop_start = jiffies; + unsigned long delay = 0; + + doutc(cl, "begin\n"); + spin_lock(&mdsc->cap_delay_lock); + while (!list_empty(&mdsc->cap_delay_list)) { + ci = list_first_entry(&mdsc->cap_delay_list, + struct ceph_inode_info, + i_cap_delay_list); + if (time_before(loop_start, ci->i_hold_caps_max - delay_max)) { + doutc(cl, "caps added recently. Exiting loop"); + delay = ci->i_hold_caps_max; + break; + } + if ((ci->i_ceph_flags & CEPH_I_FLUSH) == 0 && + time_before(jiffies, ci->i_hold_caps_max)) + break; + list_del_init(&ci->i_cap_delay_list); + + inode = igrab(&ci->netfs.inode); + if (inode) { + spin_unlock(&mdsc->cap_delay_lock); + doutc(cl, "on %p %llx.%llx\n", inode, + ceph_vinop(inode)); + ceph_check_caps(ci, 0); + iput(inode); + spin_lock(&mdsc->cap_delay_lock); + } + + /* + * Make sure too many dirty caps or general + * slowness doesn't block mdsc delayed work, + * preventing send_renew_caps() from running. + */ + if (time_after_eq(jiffies, loop_start + 5 * HZ)) + break; + } + spin_unlock(&mdsc->cap_delay_lock); + doutc(cl, "done\n"); + + return delay; +} + +/* + * Flush all dirty caps to the mds + */ +static void flush_dirty_session_caps(struct ceph_mds_session *s) +{ + struct ceph_mds_client *mdsc = s->s_mdsc; + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_inode_info *ci; + struct inode *inode; + + doutc(cl, "begin\n"); + spin_lock(&mdsc->cap_dirty_lock); + while (!list_empty(&s->s_cap_dirty)) { + ci = list_first_entry(&s->s_cap_dirty, struct ceph_inode_info, + i_dirty_item); + inode = &ci->netfs.inode; + ihold(inode); + doutc(cl, "%p %llx.%llx\n", inode, ceph_vinop(inode)); + spin_unlock(&mdsc->cap_dirty_lock); + ceph_wait_on_async_create(inode); + ceph_check_caps(ci, CHECK_CAPS_FLUSH); + iput(inode); + spin_lock(&mdsc->cap_dirty_lock); + } + spin_unlock(&mdsc->cap_dirty_lock); + doutc(cl, "done\n"); +} + +void ceph_flush_dirty_caps(struct ceph_mds_client *mdsc) +{ + ceph_mdsc_iterate_sessions(mdsc, flush_dirty_session_caps, true); +} + +/* + * Flush all cap releases to the mds + */ +static void flush_cap_releases(struct ceph_mds_session *s) +{ + struct ceph_mds_client *mdsc = s->s_mdsc; + struct ceph_client *cl = mdsc->fsc->client; + + doutc(cl, "begin\n"); + spin_lock(&s->s_cap_lock); + if (s->s_num_cap_releases) + ceph_flush_session_cap_releases(mdsc, s); + spin_unlock(&s->s_cap_lock); + doutc(cl, "done\n"); + +} + +void ceph_flush_cap_releases(struct ceph_mds_client *mdsc) +{ + ceph_mdsc_iterate_sessions(mdsc, flush_cap_releases, true); +} + +void __ceph_touch_fmode(struct ceph_inode_info *ci, + struct ceph_mds_client *mdsc, int fmode) +{ + unsigned long now = jiffies; + if (fmode & CEPH_FILE_MODE_RD) + ci->i_last_rd = now; + if (fmode & CEPH_FILE_MODE_WR) + ci->i_last_wr = now; + /* queue periodic check */ + if (fmode && + __ceph_is_any_real_caps(ci) && + list_empty(&ci->i_cap_delay_list)) + __cap_delay_requeue(mdsc, ci); +} + +void ceph_get_fmode(struct ceph_inode_info *ci, int fmode, int count) +{ + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(ci->netfs.inode.i_sb); + int bits = (fmode << 1) | 1; + bool already_opened = false; + int i; + + if (count == 1) + atomic64_inc(&mdsc->metric.opened_files); + + spin_lock(&ci->i_ceph_lock); + for (i = 0; i < CEPH_FILE_MODE_BITS; i++) { + /* + * If any of the mode ref is larger than 0, + * that means it has been already opened by + * others. Just skip checking the PIN ref. + */ + if (i && ci->i_nr_by_mode[i]) + already_opened = true; + + if (bits & (1 << i)) + ci->i_nr_by_mode[i] += count; + } + + if (!already_opened) + percpu_counter_inc(&mdsc->metric.opened_inodes); + spin_unlock(&ci->i_ceph_lock); +} + +/* + * Drop open file reference. If we were the last open file, + * we may need to release capabilities to the MDS (or schedule + * their delayed release). + */ +void ceph_put_fmode(struct ceph_inode_info *ci, int fmode, int count) +{ + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(ci->netfs.inode.i_sb); + int bits = (fmode << 1) | 1; + bool is_closed = true; + int i; + + if (count == 1) + atomic64_dec(&mdsc->metric.opened_files); + + spin_lock(&ci->i_ceph_lock); + for (i = 0; i < CEPH_FILE_MODE_BITS; i++) { + if (bits & (1 << i)) { + BUG_ON(ci->i_nr_by_mode[i] < count); + ci->i_nr_by_mode[i] -= count; + } + + /* + * If any of the mode ref is not 0 after + * decreased, that means it is still opened + * by others. Just skip checking the PIN ref. + */ + if (i && ci->i_nr_by_mode[i]) + is_closed = false; + } + + if (is_closed) + percpu_counter_dec(&mdsc->metric.opened_inodes); + spin_unlock(&ci->i_ceph_lock); +} + +/* + * For a soon-to-be unlinked file, drop the LINK caps. If it + * looks like the link count will hit 0, drop any other caps (other + * than PIN) we don't specifically want (due to the file still being + * open). + */ +int ceph_drop_caps_for_unlink(struct inode *inode) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + int drop = CEPH_CAP_LINK_SHARED | CEPH_CAP_LINK_EXCL; + + spin_lock(&ci->i_ceph_lock); + if (inode->i_nlink == 1) { + drop |= ~(__ceph_caps_wanted(ci) | CEPH_CAP_PIN); + + if (__ceph_caps_dirty(ci)) { + struct ceph_mds_client *mdsc = + ceph_inode_to_fs_client(inode)->mdsc; + + doutc(mdsc->fsc->client, "%p %llx.%llx\n", inode, + ceph_vinop(inode)); + spin_lock(&mdsc->cap_delay_lock); + set_bit(CEPH_I_FLUSH_BIT, &ci->i_ceph_flags); + if (!list_empty(&ci->i_cap_delay_list)) + list_del_init(&ci->i_cap_delay_list); + list_add_tail(&ci->i_cap_delay_list, + &mdsc->cap_unlink_delay_list); + spin_unlock(&mdsc->cap_delay_lock); + + /* + * Fire the work immediately, because the MDS maybe + * waiting for caps release. + */ + ceph_queue_cap_unlink_work(mdsc); + } + } + spin_unlock(&ci->i_ceph_lock); + return drop; +} + +/* + * Helpers for embedding cap and dentry lease releases into mds + * requests. + * + * @force is used by dentry_release (below) to force inclusion of a + * record for the directory inode, even when there aren't any caps to + * drop. + */ +int ceph_encode_inode_release(void **p, struct inode *inode, + int mds, int drop, int unless, int force) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_cap *cap; + struct ceph_mds_request_release *rel = *p; + int used, dirty; + int ret = 0; + + spin_lock(&ci->i_ceph_lock); + used = __ceph_caps_used(ci); + dirty = __ceph_caps_dirty(ci); + + doutc(cl, "%p %llx.%llx mds%d used|dirty %s drop %s unless %s\n", + inode, ceph_vinop(inode), mds, ceph_cap_string(used|dirty), + ceph_cap_string(drop), ceph_cap_string(unless)); + + /* only drop unused, clean caps */ + drop &= ~(used | dirty); + + cap = __get_cap_for_mds(ci, mds); + if (cap && __cap_is_valid(cap)) { + unless &= cap->issued; + if (unless) { + if (unless & CEPH_CAP_AUTH_EXCL) + drop &= ~CEPH_CAP_AUTH_SHARED; + if (unless & CEPH_CAP_LINK_EXCL) + drop &= ~CEPH_CAP_LINK_SHARED; + if (unless & CEPH_CAP_XATTR_EXCL) + drop &= ~CEPH_CAP_XATTR_SHARED; + if (unless & CEPH_CAP_FILE_EXCL) + drop &= ~CEPH_CAP_FILE_SHARED; + } + + if (force || (cap->issued & drop)) { + if (cap->issued & drop) { + int wanted = __ceph_caps_wanted(ci); + doutc(cl, "%p %llx.%llx cap %p %s -> %s, " + "wanted %s -> %s\n", inode, + ceph_vinop(inode), cap, + ceph_cap_string(cap->issued), + ceph_cap_string(cap->issued & ~drop), + ceph_cap_string(cap->mds_wanted), + ceph_cap_string(wanted)); + + cap->issued &= ~drop; + cap->implemented &= ~drop; + cap->mds_wanted = wanted; + if (cap == ci->i_auth_cap && + !(wanted & CEPH_CAP_ANY_FILE_WR)) + ci->i_requested_max_size = 0; + } else { + doutc(cl, "%p %llx.%llx cap %p %s (force)\n", + inode, ceph_vinop(inode), cap, + ceph_cap_string(cap->issued)); + } + + rel->ino = cpu_to_le64(ceph_ino(inode)); + rel->cap_id = cpu_to_le64(cap->cap_id); + rel->seq = cpu_to_le32(cap->seq); + rel->issue_seq = cpu_to_le32(cap->issue_seq); + rel->mseq = cpu_to_le32(cap->mseq); + rel->caps = cpu_to_le32(cap->implemented); + rel->wanted = cpu_to_le32(cap->mds_wanted); + rel->dname_len = 0; + rel->dname_seq = 0; + *p += sizeof(*rel); + ret = 1; + } else { + doutc(cl, "%p %llx.%llx cap %p %s (noop)\n", + inode, ceph_vinop(inode), cap, + ceph_cap_string(cap->issued)); + } + } + spin_unlock(&ci->i_ceph_lock); + return ret; +} + +/** + * ceph_encode_dentry_release - encode a dentry release into an outgoing request + * @p: outgoing request buffer + * @dentry: dentry to release + * @dir: dir to release it from + * @mds: mds that we're speaking to + * @drop: caps being dropped + * @unless: unless we have these caps + * + * Encode a dentry release into an outgoing request buffer. Returns 1 if the + * thing was released, or a negative error code otherwise. + */ +int ceph_encode_dentry_release(void **p, struct dentry *dentry, + struct inode *dir, + int mds, int drop, int unless) +{ + struct ceph_mds_request_release *rel = *p; + struct ceph_dentry_info *di = ceph_dentry(dentry); + struct ceph_client *cl; + int force = 0; + int ret; + + /* This shouldn't happen */ + BUG_ON(!dir); + + /* + * force an record for the directory caps if we have a dentry lease. + * this is racy (can't take i_ceph_lock and d_lock together), but it + * doesn't have to be perfect; the mds will revoke anything we don't + * release. + */ + spin_lock(&dentry->d_lock); + if (di->lease_session && di->lease_session->s_mds == mds) + force = 1; + spin_unlock(&dentry->d_lock); + + ret = ceph_encode_inode_release(p, dir, mds, drop, unless, force); + + cl = ceph_inode_to_client(dir); + spin_lock(&dentry->d_lock); + if (ret && di->lease_session && di->lease_session->s_mds == mds) { + int len = dentry->d_name.len; + doutc(cl, "%p mds%d seq %d\n", dentry, mds, + (int)di->lease_seq); + rel->dname_seq = cpu_to_le32(di->lease_seq); + __ceph_mdsc_drop_dentry_lease(dentry); + memcpy(*p, dentry->d_name.name, len); + spin_unlock(&dentry->d_lock); + if (IS_ENCRYPTED(dir) && fscrypt_has_encryption_key(dir)) { + len = ceph_encode_encrypted_dname(dir, *p, len); + if (len < 0) + return len; + } + rel->dname_len = cpu_to_le32(len); + *p += len; + } else { + spin_unlock(&dentry->d_lock); + } + return ret; +} + +static int remove_capsnaps(struct ceph_mds_client *mdsc, struct inode *inode) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_cap_snap *capsnap; + int capsnap_release = 0; + + lockdep_assert_held(&ci->i_ceph_lock); + + doutc(cl, "removing capsnaps, ci is %p, %p %llx.%llx\n", + ci, inode, ceph_vinop(inode)); + + while (!list_empty(&ci->i_cap_snaps)) { + capsnap = list_first_entry(&ci->i_cap_snaps, + struct ceph_cap_snap, ci_item); + __ceph_remove_capsnap(inode, capsnap, NULL, NULL); + ceph_put_snap_context(capsnap->context); + ceph_put_cap_snap(capsnap); + capsnap_release++; + } + wake_up_all(&ci->i_cap_wq); + wake_up_all(&mdsc->cap_flushing_wq); + return capsnap_release; +} + +int ceph_purge_inode_cap(struct inode *inode, struct ceph_cap *cap, bool *invalidate) +{ + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + struct ceph_mds_client *mdsc = fsc->mdsc; + struct ceph_client *cl = fsc->client; + struct ceph_inode_info *ci = ceph_inode(inode); + bool is_auth; + bool dirty_dropped = false; + int iputs = 0; + + lockdep_assert_held(&ci->i_ceph_lock); + + doutc(cl, "removing cap %p, ci is %p, %p %llx.%llx\n", + cap, ci, inode, ceph_vinop(inode)); + + is_auth = (cap == ci->i_auth_cap); + __ceph_remove_cap(cap, false); + if (is_auth) { + struct ceph_cap_flush *cf; + + if (ceph_inode_is_shutdown(inode)) { + if (inode->i_data.nrpages > 0) + *invalidate = true; + if (ci->i_wrbuffer_ref > 0) + mapping_set_error(&inode->i_data, -EIO); + } + + spin_lock(&mdsc->cap_dirty_lock); + + /* trash all of the cap flushes for this inode */ + while (!list_empty(&ci->i_cap_flush_list)) { + cf = list_first_entry(&ci->i_cap_flush_list, + struct ceph_cap_flush, i_list); + list_del_init(&cf->g_list); + list_del_init(&cf->i_list); + if (!cf->is_capsnap) + ceph_free_cap_flush(cf); + } + + if (!list_empty(&ci->i_dirty_item)) { + pr_warn_ratelimited_client(cl, + " dropping dirty %s state for %p %llx.%llx\n", + ceph_cap_string(ci->i_dirty_caps), + inode, ceph_vinop(inode)); + ci->i_dirty_caps = 0; + list_del_init(&ci->i_dirty_item); + dirty_dropped = true; + } + if (!list_empty(&ci->i_flushing_item)) { + pr_warn_ratelimited_client(cl, + " dropping dirty+flushing %s state for %p %llx.%llx\n", + ceph_cap_string(ci->i_flushing_caps), + inode, ceph_vinop(inode)); + ci->i_flushing_caps = 0; + list_del_init(&ci->i_flushing_item); + mdsc->num_cap_flushing--; + dirty_dropped = true; + } + spin_unlock(&mdsc->cap_dirty_lock); + + if (dirty_dropped) { + mapping_set_error(inode->i_mapping, -EIO); + + if (ci->i_wrbuffer_ref_head == 0 && + ci->i_wr_ref == 0 && + ci->i_dirty_caps == 0 && + ci->i_flushing_caps == 0) { + ceph_put_snap_context(ci->i_head_snapc); + ci->i_head_snapc = NULL; + } + } + + if (atomic_read(&ci->i_filelock_ref) > 0) { + /* make further file lock syscall return -EIO */ + set_bit(CEPH_I_ERROR_FILELOCK_BIT, &ci->i_ceph_flags); + pr_warn_ratelimited_client(cl, + " dropping file locks for %p %llx.%llx\n", + inode, ceph_vinop(inode)); + } + + if (!ci->i_dirty_caps && ci->i_prealloc_cap_flush) { + cf = ci->i_prealloc_cap_flush; + ci->i_prealloc_cap_flush = NULL; + if (!cf->is_capsnap) + ceph_free_cap_flush(cf); + } + + if (!list_empty(&ci->i_cap_snaps)) + iputs = remove_capsnaps(mdsc, inode); + } + if (dirty_dropped) + ++iputs; + return iputs; +} diff --git a/fs/ceph/ceph_frag.c b/fs/ceph/ceph_frag.c new file mode 100644 index 000000000..6f67d5b88 --- /dev/null +++ b/fs/ceph/ceph_frag.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Ceph 'frag' type + */ +#include +#include + +int ceph_frag_compare(__u32 a, __u32 b) +{ + unsigned va = ceph_frag_value(a); + unsigned vb = ceph_frag_value(b); + if (va < vb) + return -1; + if (va > vb) + return 1; + va = ceph_frag_bits(a); + vb = ceph_frag_bits(b); + if (va < vb) + return -1; + if (va > vb) + return 1; + return 0; +} diff --git a/fs/ceph/crypto.c b/fs/ceph/crypto.c new file mode 100644 index 000000000..64d240759 --- /dev/null +++ b/fs/ceph/crypto.c @@ -0,0 +1,605 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * The base64 encode/decode code was copied from fscrypt: + * Copyright (C) 2015, Google, Inc. + * Copyright (C) 2015, Motorola Mobility + * Written by Uday Savagaonkar, 2014. + * Modified by Jaegeuk Kim, 2015. + */ +#include +#include +#include +#include + +#include "super.h" +#include "mds_client.h" +#include "crypto.h" + +static int ceph_crypt_get_context(struct inode *inode, void *ctx, size_t len) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_fscrypt_auth *cfa = (struct ceph_fscrypt_auth *)ci->fscrypt_auth; + u32 ctxlen; + + /* Non existent or too short? */ + if (!cfa || (ci->fscrypt_auth_len < (offsetof(struct ceph_fscrypt_auth, cfa_blob) + 1))) + return -ENOBUFS; + + /* Some format we don't recognize? */ + if (le32_to_cpu(cfa->cfa_version) != CEPH_FSCRYPT_AUTH_VERSION) + return -ENOBUFS; + + ctxlen = le32_to_cpu(cfa->cfa_blob_len); + if (len < ctxlen) + return -ERANGE; + + memcpy(ctx, cfa->cfa_blob, ctxlen); + return ctxlen; +} + +static int ceph_crypt_set_context(struct inode *inode, const void *ctx, + size_t len, void *fs_data) +{ + int ret; + struct iattr attr = { }; + struct ceph_iattr cia = { }; + struct ceph_fscrypt_auth *cfa; + + WARN_ON_ONCE(fs_data); + + if (len > FSCRYPT_SET_CONTEXT_MAX_SIZE) + return -EINVAL; + + cfa = kzalloc_obj(*cfa); + if (!cfa) + return -ENOMEM; + + cfa->cfa_version = cpu_to_le32(CEPH_FSCRYPT_AUTH_VERSION); + cfa->cfa_blob_len = cpu_to_le32(len); + memcpy(cfa->cfa_blob, ctx, len); + + cia.fscrypt_auth = cfa; + + ret = __ceph_setattr(&nop_mnt_idmap, inode, &attr, &cia); + if (ret == 0) + inode_set_flags(inode, S_ENCRYPTED, S_ENCRYPTED); + kfree(cia.fscrypt_auth); + return ret; +} + +static bool ceph_crypt_empty_dir(struct inode *inode) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + + return ci->i_rsubdirs + ci->i_rfiles == 1; +} + +static const union fscrypt_policy *ceph_get_dummy_policy(struct super_block *sb) +{ + return ceph_sb_to_fs_client(sb)->fsc_dummy_enc_policy.policy; +} + +static struct fscrypt_operations ceph_fscrypt_ops = { + .inode_info_offs = (int)offsetof(struct ceph_inode_info, i_crypt_info) - + (int)offsetof(struct ceph_inode_info, netfs.inode), + .needs_bounce_pages = 1, + .get_context = ceph_crypt_get_context, + .set_context = ceph_crypt_set_context, + .get_dummy_policy = ceph_get_dummy_policy, + .empty_dir = ceph_crypt_empty_dir, +}; + +void ceph_fscrypt_set_ops(struct super_block *sb) +{ + fscrypt_set_ops(sb, &ceph_fscrypt_ops); +} + +void ceph_fscrypt_free_dummy_policy(struct ceph_fs_client *fsc) +{ + fscrypt_free_dummy_policy(&fsc->fsc_dummy_enc_policy); +} + +int ceph_fscrypt_prepare_context(struct inode *dir, struct inode *inode, + struct ceph_acl_sec_ctx *as) +{ + int ret, ctxsize; + bool encrypted = false; + struct ceph_inode_info *ci = ceph_inode(inode); + + ret = fscrypt_prepare_new_inode(dir, inode, &encrypted); + if (ret) + return ret; + if (!encrypted) + return 0; + + as->fscrypt_auth = kzalloc_obj(*as->fscrypt_auth); + if (!as->fscrypt_auth) + return -ENOMEM; + + ctxsize = fscrypt_context_for_new_inode(as->fscrypt_auth->cfa_blob, + inode); + if (ctxsize < 0) + return ctxsize; + + as->fscrypt_auth->cfa_version = cpu_to_le32(CEPH_FSCRYPT_AUTH_VERSION); + as->fscrypt_auth->cfa_blob_len = cpu_to_le32(ctxsize); + + WARN_ON_ONCE(ci->fscrypt_auth); + kfree(ci->fscrypt_auth); + ci->fscrypt_auth_len = ceph_fscrypt_auth_len(as->fscrypt_auth); + ci->fscrypt_auth = kmemdup(as->fscrypt_auth, ci->fscrypt_auth_len, + GFP_KERNEL); + if (!ci->fscrypt_auth) + return -ENOMEM; + + inode->i_flags |= S_ENCRYPTED; + + return 0; +} + +void ceph_fscrypt_as_ctx_to_req(struct ceph_mds_request *req, + struct ceph_acl_sec_ctx *as) +{ + swap(req->r_fscrypt_auth, as->fscrypt_auth); +} + +/* + * User-created snapshots can't start with '_'. Snapshots that start with this + * character are special (hint: there aren't real snapshots) and use the + * following format: + * + * __ + * + * where: + * - - the real snapshot name that may need to be decrypted, + * - - the inode number (in decimal) for the actual snapshot + * + * This function parses these snapshot names and returns the inode + * . 'name_len' will also bet set with the + * length. + */ +static struct inode *parse_longname(const struct inode *parent, + const char *name, int *name_len) +{ + struct ceph_client *cl = ceph_inode_to_client(parent); + struct inode *dir = NULL; + struct ceph_vino vino = { .snap = CEPH_NOSNAP }; + char *name_end, *inode_number; + int ret = -EIO; + /* Snapshot name must start with an underscore */ + if (*name_len <= 0 || name[0] != '_') + return ERR_PTR(-EIO); + /* Skip initial '_' and NUL-terminate */ + char *str __free(kfree) = kmemdup_nul(name + 1, *name_len - 1, GFP_KERNEL); + if (!str) + return ERR_PTR(-ENOMEM); + name_end = strrchr(str, '_'); + if (!name_end) { + doutc(cl, "failed to parse long snapshot name: %s\n", str); + return ERR_PTR(-EIO); + } + *name_len = (name_end - str); + if (*name_len <= 0) { + pr_err_client(cl, "failed to parse long snapshot name\n"); + return ERR_PTR(-EIO); + } + + /* Get the inode number */ + inode_number = name_end + 1; + ret = kstrtou64(inode_number, 10, &vino.ino); + if (ret) { + doutc(cl, "failed to parse inode number: %s\n", str); + return ERR_PTR(ret); + } + + /* And finally the inode */ + dir = ceph_find_inode(parent->i_sb, vino); + if (!dir) { + /* This can happen if we're not mounting cephfs on the root */ + dir = ceph_get_inode(parent->i_sb, vino, NULL); + if (IS_ERR(dir)) + doutc(cl, "can't find inode %s (%s)\n", inode_number, name); + } + return dir; +} + +int ceph_encode_encrypted_dname(struct inode *parent, char *buf, int elen) +{ + struct ceph_client *cl = ceph_inode_to_client(parent); + struct inode *dir = parent; + char *p = buf; + u32 len; + int name_len = elen; + int ret; + u8 *cryptbuf = NULL; + + /* Handle the special case of snapshot names that start with '_' */ + if (ceph_snap(dir) == CEPH_SNAPDIR && *p == '_') { + dir = parse_longname(parent, p, &name_len); + if (IS_ERR(dir)) + return PTR_ERR(dir); + p++; /* skip initial '_' */ + } + + if (!fscrypt_has_encryption_key(dir)) + goto out; + + /* + * Convert cleartext d_name to ciphertext. If result is longer than + * CEPH_NOHASH_NAME_MAX, sha256 the remaining bytes + * + * See: fscrypt_setup_filename + */ + if (!fscrypt_fname_encrypted_size(dir, name_len, NAME_MAX, &len)) { + elen = -ENAMETOOLONG; + goto out; + } + + /* Allocate a buffer appropriate to hold the result */ + cryptbuf = kmalloc(len > CEPH_NOHASH_NAME_MAX ? NAME_MAX : len, + GFP_KERNEL); + if (!cryptbuf) { + elen = -ENOMEM; + goto out; + } + + ret = fscrypt_fname_encrypt(dir, + &(struct qstr)QSTR_INIT(p, name_len), + cryptbuf, len); + if (ret) { + elen = ret; + goto out; + } + + /* hash the end if the name is long enough */ + if (len > CEPH_NOHASH_NAME_MAX) { + u8 hash[SHA256_DIGEST_SIZE]; + u8 *extra = cryptbuf + CEPH_NOHASH_NAME_MAX; + + /* + * hash the extra bytes and overwrite crypttext beyond that + * point with it + */ + sha256(extra, len - CEPH_NOHASH_NAME_MAX, hash); + memcpy(extra, hash, SHA256_DIGEST_SIZE); + len = CEPH_NOHASH_NAME_MAX + SHA256_DIGEST_SIZE; + } + + /* base64 encode the encrypted name */ + elen = base64_encode(cryptbuf, len, p, false, BASE64_IMAP); + doutc(cl, "base64-encoded ciphertext name = %.*s\n", elen, p); + + /* To understand the 240 limit, see CEPH_NOHASH_NAME_MAX comments */ + WARN_ON(elen > 240); + if (dir != parent) // leading _ is already there; append _ + elen += 1 + sprintf(p + elen, "_%llu", dir->i_ino); + +out: + kfree(cryptbuf); + if (dir != parent) { + if ((inode_state_read_once(dir) & I_NEW)) + discard_new_inode(dir); + else + iput(dir); + } + return elen; +} + +/** + * ceph_fname_to_usr - convert a filename for userland presentation + * @fname: ceph_fname to be converted + * @tname: temporary name buffer to use for conversion (may be NULL) + * @oname: where converted name should be placed + * @is_nokey: set to true if key wasn't available during conversion (may be NULL) + * + * Given a filename (usually from the MDS), format it for presentation to + * userland. If @parent is not encrypted, just pass it back as-is. + * + * Otherwise, base64 decode the string, and then ask fscrypt to format it + * for userland presentation. + * + * Returns 0 on success or negative error code on error. + */ +int ceph_fname_to_usr(const struct ceph_fname *fname, struct fscrypt_str *tname, + struct fscrypt_str *oname, bool *is_nokey) +{ + struct inode *dir = fname->dir; + struct fscrypt_str _tname = FSTR_INIT(NULL, 0); + struct fscrypt_str iname; + char *name = fname->name; + int name_len = fname->name_len; + int ret; + + /* Sanity check that the resulting name will fit in the buffer */ + if (fname->name_len > NAME_MAX || fname->ctext_len > NAME_MAX) + return -EIO; + + /* Handle the special case of snapshot names that start with '_' */ + if ((ceph_snap(dir) == CEPH_SNAPDIR) && (name_len > 0) && + (name[0] == '_')) { + dir = parse_longname(dir, name, &name_len); + if (IS_ERR(dir)) + return PTR_ERR(dir); + name++; /* skip initial '_' */ + } + + if (!IS_ENCRYPTED(dir)) { + oname->name = fname->name; + oname->len = fname->name_len; + ret = 0; + goto out_inode; + } + + ret = ceph_fscrypt_prepare_readdir(dir); + if (ret) + goto out_inode; + + /* + * Use the raw dentry name as sent by the MDS instead of + * generating a nokey name via fscrypt. + */ + if (!fscrypt_has_encryption_key(dir)) { + if (fname->no_copy) + oname->name = fname->name; + else + memcpy(oname->name, fname->name, fname->name_len); + oname->len = fname->name_len; + if (is_nokey) + *is_nokey = true; + ret = 0; + goto out_inode; + } + + if (fname->ctext_len == 0) { + int declen; + + if (!tname) { + ret = fscrypt_fname_alloc_buffer(NAME_MAX, &_tname); + if (ret) + goto out_inode; + tname = &_tname; + } + + declen = base64_decode(name, name_len, + tname->name, false, BASE64_IMAP); + if (declen <= 0) { + ret = -EIO; + goto out; + } + iname.name = tname->name; + iname.len = declen; + } else { + iname.name = fname->ctext; + iname.len = fname->ctext_len; + } + + ret = fscrypt_fname_disk_to_usr(dir, 0, 0, &iname, oname); + if (!ret && (dir != fname->dir)) { + char tmp_buf[BASE64_CHARS(NAME_MAX)]; + + name_len = snprintf(tmp_buf, sizeof(tmp_buf), "_%.*s_%llu", + oname->len, oname->name, dir->i_ino); + memcpy(oname->name, tmp_buf, name_len); + oname->len = name_len; + } + +out: + fscrypt_fname_free_buffer(&_tname); +out_inode: + if (dir != fname->dir) { + if ((inode_state_read_once(dir) & I_NEW)) + discard_new_inode(dir); + else + iput(dir); + } + return ret; +} + +/** + * ceph_fscrypt_prepare_readdir - simple __fscrypt_prepare_readdir() wrapper + * @dir: directory inode for readdir prep + * + * Simple wrapper around __fscrypt_prepare_readdir() that will mark directory as + * non-complete if this call results in having the directory unlocked. + * + * Returns: + * 1 - if directory was locked and key is now loaded (i.e. dir is unlocked) + * 0 - if directory is still locked + * < 0 - if __fscrypt_prepare_readdir() fails + */ +int ceph_fscrypt_prepare_readdir(struct inode *dir) +{ + bool had_key = fscrypt_has_encryption_key(dir); + int err; + + if (!IS_ENCRYPTED(dir)) + return 0; + + err = __fscrypt_prepare_readdir(dir); + if (err) + return err; + if (!had_key && fscrypt_has_encryption_key(dir)) { + /* directory just got unlocked, mark it as not complete */ + ceph_dir_clear_complete(dir); + return 1; + } + return 0; +} + +int ceph_fscrypt_decrypt_block_inplace(const struct inode *inode, + struct page *page, unsigned int len, + unsigned int offs, u64 lblk_num) +{ + struct ceph_client *cl = ceph_inode_to_client(inode); + + doutc(cl, "%p %llx.%llx len %u offs %u blk %llu\n", inode, + ceph_vinop(inode), len, offs, lblk_num); + return fscrypt_decrypt_block_inplace(inode, page, len, offs, lblk_num); +} + +int ceph_fscrypt_encrypt_block_inplace(const struct inode *inode, + struct page *page, unsigned int len, + unsigned int offs, u64 lblk_num) +{ + struct ceph_client *cl = ceph_inode_to_client(inode); + + doutc(cl, "%p %llx.%llx len %u offs %u blk %llu\n", inode, + ceph_vinop(inode), len, offs, lblk_num); + return fscrypt_encrypt_block_inplace(inode, page, len, offs, lblk_num); +} + +/** + * ceph_fscrypt_decrypt_pages - decrypt an array of pages + * @inode: pointer to inode associated with these pages + * @page: pointer to page array + * @off: offset into the file that the read data starts + * @len: max length to decrypt + * + * Decrypt an array of fscrypt'ed pages and return the amount of + * data decrypted. Any data in the page prior to the start of the + * first complete block in the read is ignored. Any incomplete + * crypto blocks at the end of the array are ignored (and should + * probably be zeroed by the caller). + * + * Returns the length of the decrypted data or a negative errno. + */ +int ceph_fscrypt_decrypt_pages(struct inode *inode, struct page **page, + u64 off, int len) +{ + int i, num_blocks; + u64 baseblk = off >> CEPH_FSCRYPT_BLOCK_SHIFT; + int ret = 0; + + /* + * We can't deal with partial blocks on an encrypted file, so mask off + * the last bit. + */ + num_blocks = ceph_fscrypt_blocks(off, len & CEPH_FSCRYPT_BLOCK_MASK); + + /* Decrypt each block */ + for (i = 0; i < num_blocks; ++i) { + int blkoff = i << CEPH_FSCRYPT_BLOCK_SHIFT; + int pgidx = blkoff >> PAGE_SHIFT; + unsigned int pgoffs = offset_in_page(blkoff); + int fret; + + fret = ceph_fscrypt_decrypt_block_inplace(inode, page[pgidx], + CEPH_FSCRYPT_BLOCK_SIZE, pgoffs, + baseblk + i); + if (fret < 0) { + if (ret == 0) + ret = fret; + break; + } + ret += CEPH_FSCRYPT_BLOCK_SIZE; + } + return ret; +} + +/** + * ceph_fscrypt_decrypt_extents: decrypt received extents in given buffer + * @inode: inode associated with pages being decrypted + * @page: pointer to page array + * @off: offset into the file that the data in page[0] starts + * @map: pointer to extent array + * @ext_cnt: length of extent array + * + * Given an extent map and a page array, decrypt the received data in-place, + * skipping holes. Returns the offset into buffer of end of last decrypted + * block. + */ +int ceph_fscrypt_decrypt_extents(struct inode *inode, struct page **page, + u64 off, struct ceph_sparse_extent *map, + u32 ext_cnt) +{ + struct ceph_client *cl = ceph_inode_to_client(inode); + int i, ret = 0; + struct ceph_inode_info *ci = ceph_inode(inode); + u64 objno, objoff; + u32 xlen; + + /* Nothing to do for empty array */ + if (ext_cnt == 0) { + doutc(cl, "%p %llx.%llx empty array, ret 0\n", inode, + ceph_vinop(inode)); + return 0; + } + + ceph_calc_file_object_mapping(&ci->i_layout, off, map[0].len, + &objno, &objoff, &xlen); + + for (i = 0; i < ext_cnt; ++i) { + struct ceph_sparse_extent *ext = &map[i]; + int pgsoff = ext->off - objoff; + int pgidx = pgsoff >> PAGE_SHIFT; + int fret; + + if ((ext->off | ext->len) & ~CEPH_FSCRYPT_BLOCK_MASK) { + pr_warn_client(cl, + "%p %llx.%llx bad encrypted sparse extent " + "idx %d off %llx len %llx\n", + inode, ceph_vinop(inode), i, ext->off, + ext->len); + return -EIO; + } + fret = ceph_fscrypt_decrypt_pages(inode, &page[pgidx], + off + pgsoff, ext->len); + doutc(cl, "%p %llx.%llx [%d] 0x%llx~0x%llx fret %d\n", inode, + ceph_vinop(inode), i, ext->off, ext->len, fret); + if (fret < 0) { + if (ret == 0) + ret = fret; + break; + } + ret = pgsoff + fret; + } + doutc(cl, "ret %d\n", ret); + return ret; +} + +/** + * ceph_fscrypt_encrypt_pages - encrypt an array of pages + * @inode: pointer to inode associated with these pages + * @page: pointer to page array + * @off: offset into the file that the data starts + * @len: max length to encrypt + * + * Encrypt an array of cleartext pages and return the amount of + * data encrypted. Any data in the page prior to the start of the + * first complete block in the read is ignored. Any incomplete + * crypto blocks at the end of the array are ignored. + * + * Returns the length of the encrypted data or a negative errno. + */ +int ceph_fscrypt_encrypt_pages(struct inode *inode, struct page **page, u64 off, + int len) +{ + int i, num_blocks; + u64 baseblk = off >> CEPH_FSCRYPT_BLOCK_SHIFT; + int ret = 0; + + /* + * We can't deal with partial blocks on an encrypted file, so mask off + * the last bit. + */ + num_blocks = ceph_fscrypt_blocks(off, len & CEPH_FSCRYPT_BLOCK_MASK); + + /* Encrypt each block */ + for (i = 0; i < num_blocks; ++i) { + int blkoff = i << CEPH_FSCRYPT_BLOCK_SHIFT; + int pgidx = blkoff >> PAGE_SHIFT; + unsigned int pgoffs = offset_in_page(blkoff); + int fret; + + fret = ceph_fscrypt_encrypt_block_inplace(inode, page[pgidx], + CEPH_FSCRYPT_BLOCK_SIZE, pgoffs, + baseblk + i); + if (fret < 0) { + if (ret == 0) + ret = fret; + break; + } + ret += CEPH_FSCRYPT_BLOCK_SIZE; + } + return ret; +} diff --git a/fs/ceph/crypto.h b/fs/ceph/crypto.h new file mode 100644 index 000000000..b748e2060 --- /dev/null +++ b/fs/ceph/crypto.h @@ -0,0 +1,272 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Ceph fscrypt functionality + */ + +#ifndef _CEPH_CRYPTO_H +#define _CEPH_CRYPTO_H + +#include +#include +#include + +#define CEPH_FSCRYPT_BLOCK_SHIFT 12 +#define CEPH_FSCRYPT_BLOCK_SIZE (_AC(1, UL) << CEPH_FSCRYPT_BLOCK_SHIFT) +#define CEPH_FSCRYPT_BLOCK_MASK (~(CEPH_FSCRYPT_BLOCK_SIZE-1)) + +struct ceph_fs_client; +struct ceph_acl_sec_ctx; +struct ceph_mds_request; + +struct ceph_fname { + struct inode *dir; + char *name; // b64 encoded, possibly hashed + unsigned char *ctext; // binary crypttext (if any) + u32 name_len; // length of name buffer + u32 ctext_len; // length of crypttext + bool no_copy; +}; + +/* + * Header for the encrypted file when truncating the size, this + * will be sent to MDS, and the MDS will update the encrypted + * last block and then truncate the size. + */ +struct ceph_fscrypt_truncate_size_header { + __u8 ver; + __u8 compat; + + /* + * It will be sizeof(assert_ver + file_offset + block_size) + * if the last block is empty when it's located in a file + * hole. Or the data_len will plus CEPH_FSCRYPT_BLOCK_SIZE. + */ + __le32 data_len; + + __le64 change_attr; + __le64 file_offset; + __le32 block_size; +} __packed; + +struct ceph_fscrypt_auth { + __le32 cfa_version; + __le32 cfa_blob_len; + u8 cfa_blob[FSCRYPT_SET_CONTEXT_MAX_SIZE]; +} __packed; + +#define CEPH_FSCRYPT_AUTH_VERSION 1 +static inline u32 ceph_fscrypt_auth_len(struct ceph_fscrypt_auth *fa) +{ + u32 ctxsize = le32_to_cpu(fa->cfa_blob_len); + + return offsetof(struct ceph_fscrypt_auth, cfa_blob) + ctxsize; +} + +#ifdef CONFIG_FS_ENCRYPTION +/* + * We want to encrypt filenames when creating them, but the encrypted + * versions of those names may have illegal characters in them. To mitigate + * that, we base64 encode them, but that gives us a result that can exceed + * NAME_MAX. + * + * Follow a similar scheme to fscrypt itself, and cap the filename to a + * smaller size. If the ciphertext name is longer than the value below, then + * sha256 hash the remaining bytes. + * + * For the fscrypt_nokey_name struct the dirhash[2] member is useless in ceph + * so the corresponding struct will be: + * + * struct fscrypt_ceph_nokey_name { + * u8 bytes[157]; + * u8 sha256[SHA256_DIGEST_SIZE]; + * }; // 180 bytes => 240 bytes base64-encoded, which is <= NAME_MAX (255) + * + * (240 bytes is the maximum size allowed for snapshot names to take into + * account the format: '__'.) + * + * Note that for long names that end up having their tail portion hashed, we + * must also store the full encrypted name (in the dentry's alternate_name + * field). + */ +#define CEPH_NOHASH_NAME_MAX (180 - SHA256_DIGEST_SIZE) + +void ceph_fscrypt_set_ops(struct super_block *sb); + +void ceph_fscrypt_free_dummy_policy(struct ceph_fs_client *fsc); + +int ceph_fscrypt_prepare_context(struct inode *dir, struct inode *inode, + struct ceph_acl_sec_ctx *as); +void ceph_fscrypt_as_ctx_to_req(struct ceph_mds_request *req, + struct ceph_acl_sec_ctx *as); +int ceph_encode_encrypted_dname(struct inode *parent, char *buf, int len); + +static inline int ceph_fname_alloc_buffer(struct inode *parent, + struct fscrypt_str *fname) +{ + if (!IS_ENCRYPTED(parent)) + return 0; + return fscrypt_fname_alloc_buffer(NAME_MAX, fname); +} + +static inline void ceph_fname_free_buffer(struct inode *parent, + struct fscrypt_str *fname) +{ + if (IS_ENCRYPTED(parent)) + fscrypt_fname_free_buffer(fname); +} + +int ceph_fname_to_usr(const struct ceph_fname *fname, struct fscrypt_str *tname, + struct fscrypt_str *oname, bool *is_nokey); +int ceph_fscrypt_prepare_readdir(struct inode *dir); + +static inline unsigned int ceph_fscrypt_blocks(u64 off, u64 len) +{ + /* crypto blocks cannot span more than one page */ + BUILD_BUG_ON(CEPH_FSCRYPT_BLOCK_SHIFT > PAGE_SHIFT); + + return ((off+len+CEPH_FSCRYPT_BLOCK_SIZE-1) >> CEPH_FSCRYPT_BLOCK_SHIFT) - + (off >> CEPH_FSCRYPT_BLOCK_SHIFT); +} + +/* + * If we have an encrypted inode then we must adjust the offset and + * range of the on-the-wire read to cover an entire encryption block. + * The copy will be done using the original offset and length, after + * we've decrypted the result. + */ +static inline void ceph_fscrypt_adjust_off_and_len(struct inode *inode, + u64 *off, u64 *len) +{ + if (IS_ENCRYPTED(inode)) { + *len = ceph_fscrypt_blocks(*off, *len) * CEPH_FSCRYPT_BLOCK_SIZE; + *off &= CEPH_FSCRYPT_BLOCK_MASK; + } +} + +int ceph_fscrypt_decrypt_block_inplace(const struct inode *inode, + struct page *page, unsigned int len, + unsigned int offs, u64 lblk_num); +int ceph_fscrypt_encrypt_block_inplace(const struct inode *inode, + struct page *page, unsigned int len, + unsigned int offs, u64 lblk_num); +int ceph_fscrypt_decrypt_pages(struct inode *inode, struct page **page, + u64 off, int len); +int ceph_fscrypt_decrypt_extents(struct inode *inode, struct page **page, + u64 off, struct ceph_sparse_extent *map, + u32 ext_cnt); +int ceph_fscrypt_encrypt_pages(struct inode *inode, struct page **page, u64 off, + int len); + +static inline struct page *ceph_fscrypt_pagecache_page(struct page *page) +{ + return fscrypt_is_bounce_page(page) ? fscrypt_pagecache_page(page) : page; +} + +#else /* CONFIG_FS_ENCRYPTION */ + +static inline void ceph_fscrypt_set_ops(struct super_block *sb) +{ +} + +static inline void ceph_fscrypt_free_dummy_policy(struct ceph_fs_client *fsc) +{ +} + +static inline int ceph_fscrypt_prepare_context(struct inode *dir, + struct inode *inode, + struct ceph_acl_sec_ctx *as) +{ + if (IS_ENCRYPTED(dir)) + return -EOPNOTSUPP; + return 0; +} + +static inline void ceph_fscrypt_as_ctx_to_req(struct ceph_mds_request *req, + struct ceph_acl_sec_ctx *as_ctx) +{ +} + +static inline int ceph_encode_encrypted_dname(struct inode *parent, char *buf, + int len) +{ + return len; +} + +static inline int ceph_fname_alloc_buffer(struct inode *parent, + struct fscrypt_str *fname) +{ + return 0; +} + +static inline void ceph_fname_free_buffer(struct inode *parent, + struct fscrypt_str *fname) +{ +} + +static inline int ceph_fname_to_usr(const struct ceph_fname *fname, + struct fscrypt_str *tname, + struct fscrypt_str *oname, bool *is_nokey) +{ + oname->name = fname->name; + oname->len = fname->name_len; + return 0; +} + +static inline int ceph_fscrypt_prepare_readdir(struct inode *dir) +{ + return 0; +} + +static inline void ceph_fscrypt_adjust_off_and_len(struct inode *inode, + u64 *off, u64 *len) +{ +} + +static inline int ceph_fscrypt_decrypt_block_inplace(const struct inode *inode, + struct page *page, unsigned int len, + unsigned int offs, u64 lblk_num) +{ + return 0; +} + +static inline int ceph_fscrypt_encrypt_block_inplace(const struct inode *inode, + struct page *page, unsigned int len, + unsigned int offs, u64 lblk_num) +{ + return 0; +} + +static inline int ceph_fscrypt_decrypt_pages(struct inode *inode, + struct page **page, u64 off, + int len) +{ + return 0; +} + +static inline int ceph_fscrypt_decrypt_extents(struct inode *inode, + struct page **page, u64 off, + struct ceph_sparse_extent *map, + u32 ext_cnt) +{ + return 0; +} + +static inline int ceph_fscrypt_encrypt_pages(struct inode *inode, + struct page **page, u64 off, + int len) +{ + return 0; +} + +static inline struct page *ceph_fscrypt_pagecache_page(struct page *page) +{ + return page; +} +#endif /* CONFIG_FS_ENCRYPTION */ + +static inline loff_t ceph_fscrypt_page_offset(struct page *page) +{ + return page_offset(ceph_fscrypt_pagecache_page(page)); +} + +#endif /* _CEPH_CRYPTO_H */ diff --git a/fs/ceph/debugfs.c b/fs/ceph/debugfs.c new file mode 100644 index 000000000..18eb5da03 --- /dev/null +++ b/fs/ceph/debugfs.c @@ -0,0 +1,745 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "super.h" + +#ifdef CONFIG_DEBUG_FS + +#include "mds_client.h" +#include "metric.h" +#include "subvolume_metrics.h" + +/** + * struct ceph_session_feature_desc - Maps feature bits to names for debugfs + * @bit: Feature bit number from enum ceph_feature_type (see mds_client.h) + * @name: Human-readable feature name for debugfs output + * + * Used by metric_features_show() to display negotiated session features. + */ +struct ceph_session_feature_desc { + unsigned int bit; + const char *name; +}; + +static const struct ceph_session_feature_desc ceph_session_feature_table[] = { + { CEPHFS_FEATURE_METRIC_COLLECT, "METRIC_COLLECT" }, + { CEPHFS_FEATURE_REPLY_ENCODING, "REPLY_ENCODING" }, + { CEPHFS_FEATURE_RECLAIM_CLIENT, "RECLAIM_CLIENT" }, + { CEPHFS_FEATURE_LAZY_CAP_WANTED, "LAZY_CAP_WANTED" }, + { CEPHFS_FEATURE_MULTI_RECONNECT, "MULTI_RECONNECT" }, + { CEPHFS_FEATURE_DELEG_INO, "DELEG_INO" }, + { CEPHFS_FEATURE_ALTERNATE_NAME, "ALTERNATE_NAME" }, + { CEPHFS_FEATURE_NOTIFY_SESSION_STATE, "NOTIFY_SESSION_STATE" }, + { CEPHFS_FEATURE_OP_GETVXATTR, "OP_GETVXATTR" }, + { CEPHFS_FEATURE_32BITS_RETRY_FWD, "32BITS_RETRY_FWD" }, + { CEPHFS_FEATURE_NEW_SNAPREALM_INFO, "NEW_SNAPREALM_INFO" }, + { CEPHFS_FEATURE_HAS_OWNER_UIDGID, "HAS_OWNER_UIDGID" }, + { CEPHFS_FEATURE_MDS_AUTH_CAPS_CHECK, "MDS_AUTH_CAPS_CHECK" }, + { CEPHFS_FEATURE_SUBVOLUME_METRICS, "SUBVOLUME_METRICS" }, +}; + +static int mdsmap_show(struct seq_file *s, void *p) +{ + int i; + struct ceph_fs_client *fsc = s->private; + struct ceph_mdsmap *mdsmap; + + if (!fsc->mdsc || !fsc->mdsc->mdsmap) + return 0; + mdsmap = fsc->mdsc->mdsmap; + seq_printf(s, "epoch %d\n", mdsmap->m_epoch); + seq_printf(s, "root %d\n", mdsmap->m_root); + seq_printf(s, "max_mds %d\n", mdsmap->m_max_mds); + seq_printf(s, "session_timeout %d\n", mdsmap->m_session_timeout); + seq_printf(s, "session_autoclose %d\n", mdsmap->m_session_autoclose); + for (i = 0; i < mdsmap->possible_max_rank; i++) { + struct ceph_entity_addr *addr = &mdsmap->m_info[i].addr; + int state = mdsmap->m_info[i].state; + seq_printf(s, "\tmds%d\t%s\t(%s)\n", i, + ceph_pr_addr(addr), + ceph_mds_state_name(state)); + } + return 0; +} + +/* + * mdsc debugfs + */ +static int mdsc_show(struct seq_file *s, void *p) +{ + struct ceph_fs_client *fsc = s->private; + struct ceph_mds_client *mdsc = fsc->mdsc; + struct ceph_mds_request *req; + struct rb_node *rp; + char *path; + + mutex_lock(&mdsc->mutex); + for (rp = rb_first(&mdsc->request_tree); rp; rp = rb_next(rp)) { + req = rb_entry(rp, struct ceph_mds_request, r_node); + + if (req->r_request && req->r_session) + seq_printf(s, "%lld\tmds%d\t", req->r_tid, + req->r_session->s_mds); + else if (!req->r_request) + seq_printf(s, "%lld\t(no request)\t", req->r_tid); + else + seq_printf(s, "%lld\t(no session)\t", req->r_tid); + + seq_printf(s, "%s", ceph_mds_op_name(req->r_op)); + + if (test_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags)) + seq_puts(s, "\t(unsafe)"); + else + seq_puts(s, "\t"); + + if (req->r_inode) { + seq_printf(s, " #%llx", ceph_ino(req->r_inode)); + } else if (req->r_dentry) { + struct ceph_path_info path_info = {0}; + path = ceph_mdsc_build_path(mdsc, req->r_dentry, &path_info, 0); + if (IS_ERR(path)) + path = NULL; + spin_lock(&req->r_dentry->d_lock); + seq_printf(s, " #%llx/%pd (%s)", + ceph_ino(d_inode(req->r_dentry->d_parent)), + req->r_dentry, + path ? path : ""); + spin_unlock(&req->r_dentry->d_lock); + ceph_mdsc_free_path_info(&path_info); + } else if (req->r_path1) { + seq_printf(s, " #%llx/%s", req->r_ino1.ino, + req->r_path1); + } else { + seq_printf(s, " #%llx", req->r_ino1.ino); + } + + if (req->r_old_dentry) { + struct ceph_path_info path_info = {0}; + path = ceph_mdsc_build_path(mdsc, req->r_old_dentry, &path_info, 0); + if (IS_ERR(path)) + path = NULL; + spin_lock(&req->r_old_dentry->d_lock); + seq_printf(s, " #%llx/%pd (%s)", + req->r_old_dentry_dir ? + ceph_ino(req->r_old_dentry_dir) : 0, + req->r_old_dentry, + path ? path : ""); + spin_unlock(&req->r_old_dentry->d_lock); + ceph_mdsc_free_path_info(&path_info); + } else if (req->r_path2 && req->r_op != CEPH_MDS_OP_SYMLINK) { + if (req->r_ino2.ino) + seq_printf(s, " #%llx/%s", req->r_ino2.ino, + req->r_path2); + else + seq_printf(s, " %s", req->r_path2); + } + + seq_puts(s, "\n"); + } + mutex_unlock(&mdsc->mutex); + + return 0; +} + +#define CEPH_LAT_METRIC_SHOW(name, total, avg, min, max, sq) { \ + s64 _total, _avg, _min, _max, _sq, _st; \ + _avg = ktime_to_us(avg); \ + _min = ktime_to_us(min == KTIME_MAX ? 0 : min); \ + _max = ktime_to_us(max); \ + _total = total - 1; \ + _sq = _total > 0 ? DIV64_U64_ROUND_CLOSEST(sq, _total) : 0; \ + _st = int_sqrt64(_sq); \ + _st = ktime_to_us(_st); \ + seq_printf(s, "%-14s%-12lld%-16lld%-16lld%-16lld%lld\n", \ + name, total, _avg, _min, _max, _st); \ +} + +#define CEPH_SZ_METRIC_SHOW(name, total, avg, min, max, sum) { \ + u64 _min = min == U64_MAX ? 0 : min; \ + seq_printf(s, "%-14s%-12lld%-16llu%-16llu%-16llu%llu\n", \ + name, total, avg, _min, max, sum); \ +} + +static int metrics_file_show(struct seq_file *s, void *p) +{ + struct ceph_fs_client *fsc = s->private; + struct ceph_client_metric *m = &fsc->mdsc->metric; + + seq_printf(s, "item total\n"); + seq_printf(s, "------------------------------------------\n"); + seq_printf(s, "%-35s%lld\n", "total inodes", + percpu_counter_sum(&m->total_inodes)); + seq_printf(s, "%-35s%lld\n", "opened files", + atomic64_read(&m->opened_files)); + seq_printf(s, "%-35s%lld\n", "pinned i_caps", + atomic64_read(&m->total_caps)); + seq_printf(s, "%-35s%lld\n", "opened inodes", + percpu_counter_sum(&m->opened_inodes)); + return 0; +} + +static const char * const metric_str[] = { + "read", + "write", + "metadata", + "copyfrom" +}; +static int metrics_latency_show(struct seq_file *s, void *p) +{ + struct ceph_fs_client *fsc = s->private; + struct ceph_client_metric *cm = &fsc->mdsc->metric; + struct ceph_metric *m; + s64 total, avg, min, max, sq; + int i; + + seq_printf(s, "item total avg_lat(us) min_lat(us) max_lat(us) stdev(us)\n"); + seq_printf(s, "-----------------------------------------------------------------------------------\n"); + + for (i = 0; i < METRIC_MAX; i++) { + m = &cm->metric[i]; + spin_lock(&m->lock); + total = m->total; + avg = m->latency_avg; + min = m->latency_min; + max = m->latency_max; + sq = m->latency_sq_sum; + spin_unlock(&m->lock); + CEPH_LAT_METRIC_SHOW(metric_str[i], total, avg, min, max, sq); + } + + return 0; +} + +static int metrics_size_show(struct seq_file *s, void *p) +{ + struct ceph_fs_client *fsc = s->private; + struct ceph_client_metric *cm = &fsc->mdsc->metric; + struct ceph_metric *m; + s64 total; + u64 sum, avg, min, max; + int i; + + seq_printf(s, "item total avg_sz(bytes) min_sz(bytes) max_sz(bytes) total_sz(bytes)\n"); + seq_printf(s, "----------------------------------------------------------------------------------------\n"); + + for (i = 0; i < METRIC_MAX; i++) { + /* skip 'metadata' as it doesn't use the size metric */ + if (i == METRIC_METADATA) + continue; + m = &cm->metric[i]; + spin_lock(&m->lock); + total = m->total; + sum = m->size_sum; + avg = total > 0 ? DIV64_U64_ROUND_CLOSEST(sum, total) : 0; + min = m->size_min; + max = m->size_max; + spin_unlock(&m->lock); + CEPH_SZ_METRIC_SHOW(metric_str[i], total, avg, min, max, sum); + } + + return 0; +} + +static int metrics_caps_show(struct seq_file *s, void *p) +{ + struct ceph_fs_client *fsc = s->private; + struct ceph_client_metric *m = &fsc->mdsc->metric; + int nr_caps = 0; + + seq_printf(s, "item total miss hit\n"); + seq_printf(s, "-------------------------------------------------\n"); + + seq_printf(s, "%-14s%-16lld%-16lld%lld\n", "d_lease", + atomic64_read(&m->total_dentries), + percpu_counter_sum(&m->d_lease_mis), + percpu_counter_sum(&m->d_lease_hit)); + + nr_caps = atomic64_read(&m->total_caps); + seq_printf(s, "%-14s%-16d%-16lld%lld\n", "caps", nr_caps, + percpu_counter_sum(&m->i_caps_mis), + percpu_counter_sum(&m->i_caps_hit)); + + return 0; +} + +static int caps_show_cb(struct inode *inode, int mds, void *p) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct seq_file *s = p; + struct ceph_cap *cap; + + spin_lock(&ci->i_ceph_lock); + cap = __get_cap_for_mds(ci, mds); + if (cap) + seq_printf(s, "0x%-17llx%-3d%-17s%-17s\n", ceph_ino(inode), + cap->session->s_mds, + ceph_cap_string(cap->issued), + ceph_cap_string(cap->implemented)); + spin_unlock(&ci->i_ceph_lock); + return 0; +} + +static int caps_show(struct seq_file *s, void *p) +{ + struct ceph_fs_client *fsc = s->private; + struct ceph_mds_client *mdsc = fsc->mdsc; + int total, avail, used, reserved, min, i; + struct cap_wait *cw; + + ceph_reservation_status(fsc, &total, &avail, &used, &reserved, &min); + seq_printf(s, "total\t\t%d\n" + "avail\t\t%d\n" + "used\t\t%d\n" + "reserved\t%d\n" + "min\t\t%d\n\n", + total, avail, used, reserved, min); + seq_printf(s, "ino mds issued implemented\n"); + seq_printf(s, "--------------------------------------------------\n"); + + mutex_lock(&mdsc->mutex); + for (i = 0; i < mdsc->max_sessions; i++) { + struct ceph_mds_session *session; + + session = __ceph_lookup_mds_session(mdsc, i); + if (!session) + continue; + mutex_unlock(&mdsc->mutex); + mutex_lock(&session->s_mutex); + ceph_iterate_session_caps(session, caps_show_cb, s); + mutex_unlock(&session->s_mutex); + ceph_put_mds_session(session); + mutex_lock(&mdsc->mutex); + } + mutex_unlock(&mdsc->mutex); + + seq_printf(s, "\n\nWaiters:\n--------\n"); + seq_printf(s, "tgid ino need want\n"); + seq_printf(s, "-----------------------------------------------------\n"); + + spin_lock(&mdsc->caps_list_lock); + list_for_each_entry(cw, &mdsc->cap_wait_list, list) { + seq_printf(s, "%-13d0x%-17llx%-17s%-17s\n", cw->tgid, cw->ino, + ceph_cap_string(cw->need), + ceph_cap_string(cw->want)); + } + spin_unlock(&mdsc->caps_list_lock); + + return 0; +} + +static int mds_sessions_show(struct seq_file *s, void *ptr) +{ + struct ceph_fs_client *fsc = s->private; + struct ceph_mds_client *mdsc = fsc->mdsc; + struct ceph_auth_client *ac = fsc->client->monc.auth; + struct ceph_options *opt = fsc->client->options; + int mds; + + mutex_lock(&mdsc->mutex); + + /* The 'num' portion of an 'entity name' */ + seq_printf(s, "global_id %llu\n", ac->global_id); + + /* The -o name mount argument */ + seq_printf(s, "name \"%s\"\n", opt->name ? opt->name : ""); + + /* The list of MDS session rank+state */ + for (mds = 0; mds < mdsc->max_sessions; mds++) { + struct ceph_mds_session *session = + __ceph_lookup_mds_session(mdsc, mds); + if (!session) { + continue; + } + mutex_unlock(&mdsc->mutex); + seq_printf(s, "mds.%d %s\n", + session->s_mds, + ceph_session_state_name(session->s_state)); + + ceph_put_mds_session(session); + mutex_lock(&mdsc->mutex); + } + mutex_unlock(&mdsc->mutex); + + return 0; +} + +static int status_show(struct seq_file *s, void *p) +{ + struct ceph_fs_client *fsc = s->private; + struct ceph_entity_inst *inst = &fsc->client->msgr.inst; + struct ceph_entity_addr *client_addr = ceph_client_addr(fsc->client); + + seq_printf(s, "instance: %s.%lld %s/%u\n", ENTITY_NAME(inst->name), + ceph_pr_addr(client_addr), le32_to_cpu(client_addr->nonce)); + seq_printf(s, "blocklisted: %s\n", str_true_false(fsc->blocklisted)); + + return 0; +} + +static int reset_status_show(struct seq_file *s, void *p) +{ + struct ceph_fs_client *fsc = s->private; + struct ceph_mds_client *mdsc = fsc->mdsc; + struct ceph_client_reset_state *st; + u64 trigger = 0, success = 0, failure = 0; + unsigned long last_start = 0, last_finish = 0; + int last_errno = 0; + enum ceph_client_reset_phase phase = CEPH_CLIENT_RESET_IDLE; + bool drain_timed_out = false; + int sessions_reset = 0; + int blocked_requests = 0; + char reason[CEPH_CLIENT_RESET_REASON_LEN]; + + if (!mdsc) + return 0; + + st = &mdsc->reset_state; + + spin_lock(&st->lock); + trigger = st->trigger_count; + success = st->success_count; + failure = st->failure_count; + last_start = st->last_start; + last_finish = st->last_finish; + last_errno = st->last_errno; + phase = st->phase; + drain_timed_out = st->drain_timed_out; + sessions_reset = st->sessions_reset; + strscpy(reason, st->last_reason, sizeof(reason)); + spin_unlock(&st->lock); + + blocked_requests = atomic_read(&st->blocked_requests); + + seq_printf(s, "phase: %s\n", ceph_reset_phase_name(phase)); + seq_printf(s, "trigger_count: %llu\n", trigger); + seq_printf(s, "success_count: %llu\n", success); + seq_printf(s, "failure_count: %llu\n", failure); + if (last_start) + seq_printf(s, "last_start_ms_ago: %u\n", + jiffies_to_msecs(jiffies - last_start)); + else + seq_puts(s, "last_start_ms_ago: (never)\n"); + if (last_finish) + seq_printf(s, "last_finish_ms_ago: %u\n", + jiffies_to_msecs(jiffies - last_finish)); + else + seq_puts(s, "last_finish_ms_ago: (never)\n"); + seq_printf(s, "last_errno: %d\n", last_errno); + seq_printf(s, "last_reason: %s\n", + reason[0] ? reason : "(none)"); + seq_printf(s, "drain_timed_out: %s\n", + drain_timed_out ? "yes" : "no"); + seq_printf(s, "sessions_reset: %d\n", sessions_reset); + seq_printf(s, "blocked_requests: %d\n", blocked_requests); + + return 0; +} + +static ssize_t reset_trigger_write(struct file *file, const char __user *buf, + size_t len, loff_t *ppos) +{ + struct ceph_fs_client *fsc = file->private_data; + struct ceph_mds_client *mdsc = fsc->mdsc; + char reason[CEPH_CLIENT_RESET_REASON_LEN]; + size_t copy; + int ret; + + if (!mdsc) + return -ENODEV; + + copy = min_t(size_t, len, sizeof(reason) - 1); + if (copy && copy_from_user(reason, buf, copy)) + return -EFAULT; + reason[copy] = '\0'; + strim(reason); + + ret = ceph_mdsc_schedule_reset(mdsc, reason); + if (ret) + return ret; + + return len; +} + +static int subvolume_metrics_show(struct seq_file *s, void *p) +{ + struct ceph_fs_client *fsc = s->private; + struct ceph_mds_client *mdsc = fsc->mdsc; + struct ceph_subvol_metric_snapshot *snapshot = NULL; + u32 nr = 0; + u64 total_sent = 0; + u64 nonzero_sends = 0; + u32 i; + + if (!mdsc) { + seq_puts(s, "mds client unavailable\n"); + return 0; + } + + mutex_lock(&mdsc->subvol_metrics_last_mutex); + if (mdsc->subvol_metrics_last && mdsc->subvol_metrics_last_nr) { + nr = mdsc->subvol_metrics_last_nr; + snapshot = kmemdup_array(mdsc->subvol_metrics_last, nr, + sizeof(*snapshot), GFP_KERNEL); + if (!snapshot) + nr = 0; + } + total_sent = mdsc->subvol_metrics_sent; + nonzero_sends = mdsc->subvol_metrics_nonzero_sends; + mutex_unlock(&mdsc->subvol_metrics_last_mutex); + + seq_puts(s, "Last sent subvolume metrics:\n"); + if (!nr) { + seq_puts(s, " (none)\n"); + } else { + seq_puts(s, " subvol_id rd_ops wr_ops rd_bytes wr_bytes rd_lat_us wr_lat_us\n"); + for (i = 0; i < nr; i++) { + const struct ceph_subvol_metric_snapshot *e = &snapshot[i]; + + seq_printf(s, " %-18llu %-9llu %-9llu %-14llu %-14llu %-14llu %-14llu\n", + e->subvolume_id, + e->read_ops, e->write_ops, + e->read_bytes, e->write_bytes, + e->read_latency_us, e->write_latency_us); + } + } + kfree(snapshot); + + seq_puts(s, "\nStatistics:\n"); + seq_printf(s, " entries_sent: %llu\n", total_sent); + seq_printf(s, " non_zero_sends: %llu\n", nonzero_sends); + + seq_puts(s, "\nPending (unsent) subvolume metrics:\n"); + ceph_subvolume_metrics_dump(&mdsc->subvol_metrics, s); + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(mdsmap); +DEFINE_SHOW_ATTRIBUTE(mdsc); +DEFINE_SHOW_ATTRIBUTE(caps); +DEFINE_SHOW_ATTRIBUTE(mds_sessions); +DEFINE_SHOW_ATTRIBUTE(status); +DEFINE_SHOW_ATTRIBUTE(reset_status); +DEFINE_SHOW_ATTRIBUTE(metrics_file); +DEFINE_SHOW_ATTRIBUTE(metrics_latency); +DEFINE_SHOW_ATTRIBUTE(metrics_size); +DEFINE_SHOW_ATTRIBUTE(metrics_caps); +DEFINE_SHOW_ATTRIBUTE(subvolume_metrics); + +static int metric_features_show(struct seq_file *s, void *p) +{ + struct ceph_fs_client *fsc = s->private; + struct ceph_mds_client *mdsc = fsc->mdsc; + unsigned long session_features = 0; + bool have_session = false; + bool metric_collect = false; + bool subvol_support = false; + bool metrics_enabled = false; + bool subvol_enabled = false; + int i; + + if (!mdsc) { + seq_puts(s, "mds client unavailable\n"); + return 0; + } + + mutex_lock(&mdsc->mutex); + if (mdsc->metric.session) { + have_session = true; + session_features = mdsc->metric.session->s_features; + } + mutex_unlock(&mdsc->mutex); + + if (have_session) { + metric_collect = + test_bit(CEPHFS_FEATURE_METRIC_COLLECT, + &session_features); + subvol_support = + test_bit(CEPHFS_FEATURE_SUBVOLUME_METRICS, + &session_features); + } + + metrics_enabled = !disable_send_metrics && have_session && metric_collect; + subvol_enabled = metrics_enabled && subvol_support; + + seq_printf(s, + "metrics_enabled: %s (disable_send_metrics=%d, session=%s, metric_collect=%s)\n", + metrics_enabled ? "yes" : "no", + disable_send_metrics ? 1 : 0, + have_session ? "yes" : "no", + metric_collect ? "yes" : "no"); + seq_printf(s, "subvolume_metrics_enabled: %s\n", + subvol_enabled ? "yes" : "no"); + seq_printf(s, "session_feature_bits: 0x%lx\n", session_features); + + if (!have_session) { + seq_puts(s, "(no active MDS session for metrics)\n"); + return 0; + } + + for (i = 0; i < ARRAY_SIZE(ceph_session_feature_table); i++) { + const struct ceph_session_feature_desc *desc = + &ceph_session_feature_table[i]; + bool set = test_bit(desc->bit, &session_features); + + seq_printf(s, " %-24s : %s\n", desc->name, + set ? "yes" : "no"); + } + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(metric_features); + +static const struct file_operations ceph_reset_trigger_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = reset_trigger_write, + .llseek = noop_llseek, +}; + +/* + * debugfs + */ +static int congestion_kb_set(void *data, u64 val) +{ + struct ceph_fs_client *fsc = (struct ceph_fs_client *)data; + + fsc->mount_options->congestion_kb = (int)val; + return 0; +} + +static int congestion_kb_get(void *data, u64 *val) +{ + struct ceph_fs_client *fsc = (struct ceph_fs_client *)data; + + *val = (u64)fsc->mount_options->congestion_kb; + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(congestion_kb_fops, congestion_kb_get, + congestion_kb_set, "%llu\n"); + + +void ceph_fs_debugfs_cleanup(struct ceph_fs_client *fsc) +{ + doutc(fsc->client, "begin\n"); + debugfs_remove(fsc->debugfs_bdi); + debugfs_remove(fsc->debugfs_congestion_kb); + debugfs_remove(fsc->debugfs_mdsmap); + debugfs_remove(fsc->debugfs_mds_sessions); + debugfs_remove(fsc->debugfs_caps); + debugfs_remove(fsc->debugfs_status); + debugfs_remove(fsc->debugfs_mdsc); + debugfs_remove_recursive(fsc->debugfs_reset_dir); + debugfs_remove(fsc->debugfs_subvolume_metrics); + debugfs_remove_recursive(fsc->debugfs_metrics_dir); + doutc(fsc->client, "done\n"); +} + +void ceph_fs_debugfs_init(struct ceph_fs_client *fsc) +{ + char name[NAME_MAX]; + + doutc(fsc->client, "begin\n"); + fsc->debugfs_congestion_kb = + debugfs_create_file("writeback_congestion_kb", + 0600, + fsc->client->debugfs_dir, + fsc, + &congestion_kb_fops); + + snprintf(name, sizeof(name), "../../bdi/%s", + bdi_dev_name(fsc->sb->s_bdi)); + fsc->debugfs_bdi = + debugfs_create_symlink("bdi", + fsc->client->debugfs_dir, + name); + + fsc->debugfs_mdsmap = debugfs_create_file("mdsmap", + 0400, + fsc->client->debugfs_dir, + fsc, + &mdsmap_fops); + + fsc->debugfs_mds_sessions = debugfs_create_file("mds_sessions", + 0400, + fsc->client->debugfs_dir, + fsc, + &mds_sessions_fops); + + fsc->debugfs_mdsc = debugfs_create_file("mdsc", + 0400, + fsc->client->debugfs_dir, + fsc, + &mdsc_fops); + + fsc->debugfs_caps = debugfs_create_file("caps", + 0400, + fsc->client->debugfs_dir, + fsc, + &caps_fops); + + fsc->debugfs_reset_dir = debugfs_create_dir("reset", + fsc->client->debugfs_dir); + debugfs_create_file("trigger", 0200, + fsc->debugfs_reset_dir, fsc, + &ceph_reset_trigger_fops); + debugfs_create_file("status", 0400, + fsc->debugfs_reset_dir, fsc, + &reset_status_fops); + + fsc->debugfs_status = debugfs_create_file("status", + 0400, + fsc->client->debugfs_dir, + fsc, + &status_fops); + + fsc->debugfs_metrics_dir = debugfs_create_dir("metrics", + fsc->client->debugfs_dir); + + debugfs_create_file("file", 0400, fsc->debugfs_metrics_dir, fsc, + &metrics_file_fops); + debugfs_create_file("latency", 0400, fsc->debugfs_metrics_dir, fsc, + &metrics_latency_fops); + debugfs_create_file("size", 0400, fsc->debugfs_metrics_dir, fsc, + &metrics_size_fops); + debugfs_create_file("caps", 0400, fsc->debugfs_metrics_dir, fsc, + &metrics_caps_fops); + debugfs_create_file("metric_features", 0400, fsc->debugfs_metrics_dir, + fsc, &metric_features_fops); + fsc->debugfs_subvolume_metrics = + debugfs_create_file("subvolumes", 0400, + fsc->debugfs_metrics_dir, fsc, + &subvolume_metrics_fops); + doutc(fsc->client, "done\n"); +} + + +#else /* CONFIG_DEBUG_FS */ + +void ceph_fs_debugfs_init(struct ceph_fs_client *fsc) +{ +} + +void ceph_fs_debugfs_cleanup(struct ceph_fs_client *fsc) +{ +} + +#endif /* CONFIG_DEBUG_FS */ diff --git a/fs/ceph/dir.c b/fs/ceph/dir.c new file mode 100644 index 000000000..4a5e0290f --- /dev/null +++ b/fs/ceph/dir.c @@ -0,0 +1,2279 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +#include +#include +#include +#include +#include + +#include "super.h" +#include "mds_client.h" +#include "crypto.h" + +/* + * Directory operations: readdir, lookup, create, link, unlink, + * rename, etc. + */ + +/* + * Ceph MDS operations are specified in terms of a base ino and + * relative path. Thus, the client can specify an operation on a + * specific inode (e.g., a getattr due to fstat(2)), or as a path + * relative to, say, the root directory. + * + * Normally, we limit ourselves to strict inode ops (no path component) + * or dentry operations (a single path component relative to an ino). The + * exception to this is open_root_dentry(), which will open the mount + * point by name. + */ + +const struct dentry_operations ceph_dentry_ops; + +static bool __dentry_lease_is_valid(struct ceph_dentry_info *di); +static int __dir_lease_try_check(const struct dentry *dentry); + +/* + * Initialize ceph dentry state. + */ +static int ceph_d_init(struct dentry *dentry) +{ + struct ceph_dentry_info *di; + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(dentry->d_sb); + + di = kmem_cache_zalloc(ceph_dentry_cachep, GFP_KERNEL); + if (!di) + return -ENOMEM; /* oh well */ + + di->dentry = dentry; + di->lease_session = NULL; + di->time = jiffies; + dentry->d_fsdata = di; + INIT_LIST_HEAD(&di->lease_list); + + atomic64_inc(&mdsc->metric.total_dentries); + + return 0; +} + +/* + * for f_pos for readdir: + * - hash order: + * (0xff << 52) | ((24 bits hash) << 28) | + * (the nth entry has hash collision); + * - frag+name order; + * ((frag value) << 28) | (the nth entry in frag); + */ +#define OFFSET_BITS 28 +#define OFFSET_MASK ((1 << OFFSET_BITS) - 1) +#define HASH_ORDER (0xffull << (OFFSET_BITS + 24)) +loff_t ceph_make_fpos(unsigned high, unsigned off, bool hash_order) +{ + loff_t fpos = ((loff_t)high << 28) | (loff_t)off; + if (hash_order) + fpos |= HASH_ORDER; + return fpos; +} + +static bool is_hash_order(loff_t p) +{ + return (p & HASH_ORDER) == HASH_ORDER; +} + +static unsigned fpos_frag(loff_t p) +{ + return p >> OFFSET_BITS; +} + +static unsigned fpos_hash(loff_t p) +{ + return ceph_frag_value(fpos_frag(p)); +} + +static unsigned fpos_off(loff_t p) +{ + return p & OFFSET_MASK; +} + +static int fpos_cmp(loff_t l, loff_t r) +{ + int v = ceph_frag_compare(fpos_frag(l), fpos_frag(r)); + if (v) + return v; + return (int)(fpos_off(l) - fpos_off(r)); +} + +/* + * make note of the last dentry we read, so we can + * continue at the same lexicographical point, + * regardless of what dir changes take place on the + * server. + */ +static int note_last_dentry(struct ceph_fs_client *fsc, + struct ceph_dir_file_info *dfi, + const char *name, + int len, unsigned next_offset) +{ + char *buf = kmalloc(len+1, GFP_KERNEL); + if (!buf) + return -ENOMEM; + kfree(dfi->last_name); + dfi->last_name = buf; + memcpy(dfi->last_name, name, len); + dfi->last_name[len] = 0; + dfi->next_offset = next_offset; + doutc(fsc->client, "'%s'\n", dfi->last_name); + return 0; +} + + +static struct dentry * +__dcache_find_get_entry(struct dentry *parent, u64 idx, + struct ceph_readdir_cache_control *cache_ctl) +{ + struct inode *dir = d_inode(parent); + struct ceph_client *cl = ceph_inode_to_client(dir); + struct dentry *dentry; + unsigned idx_mask = (PAGE_SIZE / sizeof(struct dentry *)) - 1; + loff_t ptr_pos = idx * sizeof(struct dentry *); + pgoff_t ptr_pgoff = ptr_pos >> PAGE_SHIFT; + + if (ptr_pos >= i_size_read(dir)) + return NULL; + + if (!cache_ctl->folio || ptr_pgoff != cache_ctl->folio->index) { + ceph_readdir_cache_release(cache_ctl); + cache_ctl->folio = filemap_lock_folio(&dir->i_data, ptr_pgoff); + if (IS_ERR(cache_ctl->folio)) { + cache_ctl->folio = NULL; + doutc(cl, " folio %lu not found\n", ptr_pgoff); + return ERR_PTR(-EAGAIN); + } + /* reading/filling the cache are serialized by + i_rwsem, no need to use folio lock */ + folio_unlock(cache_ctl->folio); + cache_ctl->dentries = kmap_local_folio(cache_ctl->folio, 0); + } + + cache_ctl->index = idx & idx_mask; + + rcu_read_lock(); + spin_lock(&parent->d_lock); + /* check i_size again here, because empty directory can be + * marked as complete while not holding the i_rwsem. */ + if (ceph_dir_is_complete_ordered(dir) && ptr_pos < i_size_read(dir)) + dentry = cache_ctl->dentries[cache_ctl->index]; + else + dentry = NULL; + spin_unlock(&parent->d_lock); + if (dentry && !lockref_get_not_dead(&dentry->d_lockref)) + dentry = NULL; + rcu_read_unlock(); + return dentry ? : ERR_PTR(-EAGAIN); +} + +/* + * When possible, we try to satisfy a readdir by peeking at the + * dcache. We make this work by carefully ordering dentries on + * d_children when we initially get results back from the MDS, and + * falling back to a "normal" sync readdir if any dentries in the dir + * are dropped. + * + * Complete dir indicates that we have all dentries in the dir. It is + * defined IFF we hold CEPH_CAP_FILE_SHARED (which will be revoked by + * the MDS if/when the directory is modified). + */ +static int __dcache_readdir(struct file *file, struct dir_context *ctx, + int shared_gen) +{ + struct ceph_dir_file_info *dfi = file->private_data; + struct dentry *parent = file->f_path.dentry; + struct inode *dir = d_inode(parent); + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(dir); + struct ceph_client *cl = ceph_inode_to_client(dir); + struct dentry *dentry, *last = NULL; + struct ceph_dentry_info *di; + struct ceph_readdir_cache_control cache_ctl = {}; + u64 idx = 0; + int err = 0; + + doutc(cl, "%p %llx.%llx v%u at %llx\n", dir, ceph_vinop(dir), + (unsigned)shared_gen, ctx->pos); + + /* search start position */ + if (ctx->pos > 2) { + u64 count = div_u64(i_size_read(dir), sizeof(struct dentry *)); + while (count > 0) { + u64 step = count >> 1; + dentry = __dcache_find_get_entry(parent, idx + step, + &cache_ctl); + if (!dentry) { + /* use linear search */ + idx = 0; + break; + } + if (IS_ERR(dentry)) { + err = PTR_ERR(dentry); + goto out; + } + di = ceph_dentry(dentry); + spin_lock(&dentry->d_lock); + if (fpos_cmp(di->offset, ctx->pos) < 0) { + idx += step + 1; + count -= step + 1; + } else { + count = step; + } + spin_unlock(&dentry->d_lock); + dput(dentry); + } + + doutc(cl, "%p %llx.%llx cache idx %llu\n", dir, + ceph_vinop(dir), idx); + } + + + for (;;) { + bool emit_dentry = false; + dentry = __dcache_find_get_entry(parent, idx++, &cache_ctl); + if (!dentry) { + dfi->file_info.flags |= CEPH_F_ATEND; + err = 0; + break; + } + if (IS_ERR(dentry)) { + err = PTR_ERR(dentry); + goto out; + } + + spin_lock(&dentry->d_lock); + di = ceph_dentry(dentry); + if (d_unhashed(dentry) || + d_really_is_negative(dentry) || + di->lease_shared_gen != shared_gen || + ((dentry->d_flags & DCACHE_NOKEY_NAME) && + fscrypt_has_encryption_key(dir))) { + spin_unlock(&dentry->d_lock); + dput(dentry); + err = -EAGAIN; + goto out; + } + if (fpos_cmp(ctx->pos, di->offset) <= 0) { + __ceph_dentry_dir_lease_touch(di); + emit_dentry = true; + } + spin_unlock(&dentry->d_lock); + + if (emit_dentry) { + doutc(cl, " %llx dentry %p %pd %p\n", di->offset, + dentry, dentry, d_inode(dentry)); + ctx->pos = di->offset; + if (!dir_emit(ctx, dentry->d_name.name, + dentry->d_name.len, ceph_present_inode(d_inode(dentry)), + d_inode(dentry)->i_mode >> 12)) { + dput(dentry); + err = 0; + break; + } + ctx->pos++; + + if (last) + dput(last); + last = dentry; + } else { + dput(dentry); + } + } +out: + ceph_readdir_cache_release(&cache_ctl); + if (last) { + int ret; + di = ceph_dentry(last); + ret = note_last_dentry(fsc, dfi, last->d_name.name, + last->d_name.len, + fpos_off(di->offset) + 1); + if (ret < 0) + err = ret; + dput(last); + /* last_name no longer match cache index */ + if (dfi->readdir_cache_idx >= 0) { + dfi->readdir_cache_idx = -1; + dfi->dir_release_count = 0; + } + } + return err; +} + +static bool need_send_readdir(struct ceph_dir_file_info *dfi, loff_t pos) +{ + if (!dfi->last_readdir) + return true; + if (is_hash_order(pos)) + return !ceph_frag_contains_value(dfi->frag, fpos_hash(pos)); + else + return dfi->frag != fpos_frag(pos); +} + +static int ceph_readdir(struct file *file, struct dir_context *ctx) +{ + struct ceph_dir_file_info *dfi = file->private_data; + struct inode *inode = file_inode(file); + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + struct ceph_mds_client *mdsc = fsc->mdsc; + struct ceph_client *cl = fsc->client; + int i; + int err; + unsigned frag = -1; + struct ceph_mds_reply_info_parsed *rinfo; + + doutc(cl, "%p %llx.%llx file %p pos %llx\n", inode, + ceph_vinop(inode), file, ctx->pos); + if (dfi->file_info.flags & CEPH_F_ATEND) + return 0; + + /* always start with . and .. */ + if (ctx->pos == 0) { + doutc(cl, "%p %llx.%llx off 0 -> '.'\n", inode, + ceph_vinop(inode)); + if (!dir_emit(ctx, ".", 1, ceph_present_inode(inode), + inode->i_mode >> 12)) + return 0; + ctx->pos = 1; + } + if (ctx->pos == 1) { + u64 ino; + struct dentry *dentry = file->f_path.dentry; + + spin_lock(&dentry->d_lock); + ino = ceph_present_inode(dentry->d_parent->d_inode); + spin_unlock(&dentry->d_lock); + + doutc(cl, "%p %llx.%llx off 1 -> '..'\n", inode, + ceph_vinop(inode)); + if (!dir_emit(ctx, "..", 2, ino, inode->i_mode >> 12)) + return 0; + ctx->pos = 2; + } + + err = ceph_fscrypt_prepare_readdir(inode); + if (err < 0) + return err; + + spin_lock(&ci->i_ceph_lock); + /* request Fx cap. if have Fx, we don't need to release Fs cap + * for later create/unlink. */ + __ceph_touch_fmode(ci, mdsc, CEPH_FILE_MODE_WR); + /* can we use the dcache? */ + if (ceph_test_mount_opt(fsc, DCACHE) && + !ceph_test_mount_opt(fsc, NOASYNCREADDIR) && + ceph_snap(inode) != CEPH_SNAPDIR && + __ceph_dir_is_complete_ordered(ci) && + __ceph_caps_issued_mask_metric(ci, CEPH_CAP_FILE_SHARED, 1)) { + int shared_gen = atomic_read(&ci->i_shared_gen); + + spin_unlock(&ci->i_ceph_lock); + err = __dcache_readdir(file, ctx, shared_gen); + if (err != -EAGAIN) + return err; + } else { + spin_unlock(&ci->i_ceph_lock); + } + + /* proceed with a normal readdir */ +more: + /* do we have the correct frag content buffered? */ + if (need_send_readdir(dfi, ctx->pos)) { + struct ceph_mds_request *req; + int op = ceph_snap(inode) == CEPH_SNAPDIR ? + CEPH_MDS_OP_LSSNAP : CEPH_MDS_OP_READDIR; + + /* discard old result, if any */ + if (dfi->last_readdir) { + ceph_mdsc_put_request(dfi->last_readdir); + dfi->last_readdir = NULL; + } + + if (is_hash_order(ctx->pos)) { + /* fragtree isn't always accurate. choose frag + * based on previous reply when possible. */ + if (frag == (unsigned)-1) + frag = ceph_choose_frag(ci, fpos_hash(ctx->pos), + NULL, NULL); + } else { + frag = fpos_frag(ctx->pos); + } + + doutc(cl, "fetching %p %llx.%llx frag %x offset '%s'\n", + inode, ceph_vinop(inode), frag, dfi->last_name); + req = ceph_mdsc_create_request(mdsc, op, USE_AUTH_MDS); + if (IS_ERR(req)) + return PTR_ERR(req); + + err = ceph_alloc_readdir_reply_buffer(req, inode); + if (err) { + ceph_mdsc_put_request(req); + return err; + } + /* hints to request -> mds selection code */ + req->r_direct_mode = USE_AUTH_MDS; + if (op == CEPH_MDS_OP_READDIR) { + req->r_direct_hash = ceph_frag_value(frag); + __set_bit(CEPH_MDS_R_DIRECT_IS_HASH, &req->r_req_flags); + req->r_inode_drop = CEPH_CAP_FILE_EXCL; + } + if (dfi->last_name) { + int len = strlen(dfi->last_name); + + req->r_path2 = kzalloc(NAME_MAX + 1, GFP_KERNEL); + if (!req->r_path2) { + ceph_mdsc_put_request(req); + return -ENOMEM; + } + memcpy(req->r_path2, dfi->last_name, len); + + err = ceph_encode_encrypted_dname(inode, req->r_path2, len); + if (err < 0) { + ceph_mdsc_put_request(req); + return err; + } + } else if (is_hash_order(ctx->pos)) { + req->r_args.readdir.offset_hash = + cpu_to_le32(fpos_hash(ctx->pos)); + } + + req->r_dir_release_cnt = dfi->dir_release_count; + req->r_dir_ordered_cnt = dfi->dir_ordered_count; + req->r_readdir_cache_idx = dfi->readdir_cache_idx; + req->r_readdir_offset = dfi->next_offset; + req->r_args.readdir.frag = cpu_to_le32(frag); + req->r_args.readdir.flags = + cpu_to_le16(CEPH_READDIR_REPLY_BITFLAGS); + + req->r_inode = inode; + ihold(inode); + req->r_dentry = dget(file->f_path.dentry); + err = ceph_mdsc_do_request(mdsc, NULL, req); + if (err < 0) { + ceph_mdsc_put_request(req); + return err; + } + doutc(cl, "%p %llx.%llx got and parsed readdir result=%d" + "on frag %x, end=%d, complete=%d, hash_order=%d\n", + inode, ceph_vinop(inode), err, frag, + (int)req->r_reply_info.dir_end, + (int)req->r_reply_info.dir_complete, + (int)req->r_reply_info.hash_order); + + rinfo = &req->r_reply_info; + if (le32_to_cpu(rinfo->dir_dir->frag) != frag) { + frag = le32_to_cpu(rinfo->dir_dir->frag); + if (!rinfo->hash_order) { + dfi->next_offset = req->r_readdir_offset; + /* adjust ctx->pos to beginning of frag */ + ctx->pos = ceph_make_fpos(frag, + dfi->next_offset, + false); + } + } + + dfi->frag = frag; + dfi->last_readdir = req; + + if (test_bit(CEPH_MDS_R_DID_PREPOPULATE, &req->r_req_flags)) { + dfi->readdir_cache_idx = req->r_readdir_cache_idx; + if (dfi->readdir_cache_idx < 0) { + /* preclude from marking dir ordered */ + dfi->dir_ordered_count = 0; + } else if (ceph_frag_is_leftmost(frag) && + dfi->next_offset == 2) { + /* note dir version at start of readdir so + * we can tell if any dentries get dropped */ + dfi->dir_release_count = req->r_dir_release_cnt; + dfi->dir_ordered_count = req->r_dir_ordered_cnt; + } + } else { + doutc(cl, "%p %llx.%llx !did_prepopulate\n", inode, + ceph_vinop(inode)); + /* disable readdir cache */ + dfi->readdir_cache_idx = -1; + /* preclude from marking dir complete */ + dfi->dir_release_count = 0; + } + + /* note next offset and last dentry name */ + if (rinfo->dir_nr > 0) { + struct ceph_mds_reply_dir_entry *rde = + rinfo->dir_entries + (rinfo->dir_nr-1); + unsigned next_offset = req->r_reply_info.dir_end ? + 2 : (fpos_off(rde->offset) + 1); + err = note_last_dentry(fsc, dfi, rde->name, + rde->name_len, next_offset); + if (err) { + ceph_mdsc_put_request(dfi->last_readdir); + dfi->last_readdir = NULL; + return err; + } + } else if (req->r_reply_info.dir_end) { + dfi->next_offset = 2; + /* keep last name */ + } + } + + rinfo = &dfi->last_readdir->r_reply_info; + doutc(cl, "%p %llx.%llx frag %x num %d pos %llx chunk first %llx\n", + inode, ceph_vinop(inode), dfi->frag, rinfo->dir_nr, ctx->pos, + rinfo->dir_nr ? rinfo->dir_entries[0].offset : 0LL); + + i = 0; + /* search start position */ + if (rinfo->dir_nr > 0) { + int step, nr = rinfo->dir_nr; + while (nr > 0) { + step = nr >> 1; + if (rinfo->dir_entries[i + step].offset < ctx->pos) { + i += step + 1; + nr -= step + 1; + } else { + nr = step; + } + } + } + for (; i < rinfo->dir_nr; i++) { + struct ceph_mds_reply_dir_entry *rde = rinfo->dir_entries + i; + + if (rde->offset < ctx->pos) { + pr_warn_client(cl, + "%p %llx.%llx rde->offset 0x%llx ctx->pos 0x%llx\n", + inode, ceph_vinop(inode), rde->offset, ctx->pos); + ceph_mdsc_put_request(dfi->last_readdir); + dfi->last_readdir = NULL; + return -EIO; + } + + if (WARN_ON_ONCE(!rde->inode.in)) { + ceph_mdsc_put_request(dfi->last_readdir); + dfi->last_readdir = NULL; + return -EIO; + } + + ctx->pos = rde->offset; + doutc(cl, "%p %llx.%llx (%d/%d) -> %llx '%.*s' %p\n", inode, + ceph_vinop(inode), i, rinfo->dir_nr, ctx->pos, + rde->name_len, rde->name, &rde->inode.in); + + if (!dir_emit(ctx, rde->name, rde->name_len, + ceph_present_ino(inode->i_sb, le64_to_cpu(rde->inode.in->ino)), + le32_to_cpu(rde->inode.in->mode) >> 12)) { + /* + * NOTE: Here no need to put the 'dfi->last_readdir', + * because when dir_emit stops us it's most likely + * doesn't have enough memory, etc. So for next readdir + * it will continue. + */ + doutc(cl, "filldir stopping us...\n"); + return 0; + } + + /* Reset the lengths to their original allocated vals */ + ctx->pos++; + } + + ceph_mdsc_put_request(dfi->last_readdir); + dfi->last_readdir = NULL; + + if (dfi->next_offset > 2) { + frag = dfi->frag; + goto more; + } + + /* more frags? */ + if (!ceph_frag_is_rightmost(dfi->frag)) { + frag = ceph_frag_next(dfi->frag); + if (is_hash_order(ctx->pos)) { + loff_t new_pos = ceph_make_fpos(ceph_frag_value(frag), + dfi->next_offset, true); + if (new_pos > ctx->pos) + ctx->pos = new_pos; + /* keep last_name */ + } else { + ctx->pos = ceph_make_fpos(frag, dfi->next_offset, + false); + kfree(dfi->last_name); + dfi->last_name = NULL; + } + doutc(cl, "%p %llx.%llx next frag is %x\n", inode, + ceph_vinop(inode), frag); + goto more; + } + dfi->file_info.flags |= CEPH_F_ATEND; + + /* + * if dir_release_count still matches the dir, no dentries + * were released during the whole readdir, and we should have + * the complete dir contents in our cache. + */ + if (atomic64_read(&ci->i_release_count) == + dfi->dir_release_count) { + spin_lock(&ci->i_ceph_lock); + if (dfi->dir_ordered_count == + atomic64_read(&ci->i_ordered_count)) { + doutc(cl, " marking %p %llx.%llx complete and ordered\n", + inode, ceph_vinop(inode)); + /* use i_size to track number of entries in + * readdir cache */ + BUG_ON(dfi->readdir_cache_idx < 0); + i_size_write(inode, dfi->readdir_cache_idx * + sizeof(struct dentry*)); + } else { + doutc(cl, " marking %llx.%llx complete\n", + ceph_vinop(inode)); + } + __ceph_dir_set_complete(ci, dfi->dir_release_count, + dfi->dir_ordered_count); + spin_unlock(&ci->i_ceph_lock); + } + doutc(cl, "%p %llx.%llx file %p done.\n", inode, ceph_vinop(inode), + file); + return 0; +} + +static void reset_readdir(struct ceph_dir_file_info *dfi) +{ + if (dfi->last_readdir) { + ceph_mdsc_put_request(dfi->last_readdir); + dfi->last_readdir = NULL; + } + kfree(dfi->last_name); + dfi->last_name = NULL; + dfi->dir_release_count = 0; + dfi->readdir_cache_idx = -1; + dfi->next_offset = 2; /* compensate for . and .. */ + dfi->file_info.flags &= ~CEPH_F_ATEND; +} + +/* + * discard buffered readdir content on seekdir(0), or seek to new frag, + * or seek prior to current chunk + */ +static bool need_reset_readdir(struct ceph_dir_file_info *dfi, loff_t new_pos) +{ + struct ceph_mds_reply_info_parsed *rinfo; + loff_t chunk_offset; + if (new_pos == 0) + return true; + if (is_hash_order(new_pos)) { + /* no need to reset last_name for a forward seek when + * dentries are sorted in hash order */ + } else if (dfi->frag != fpos_frag(new_pos)) { + return true; + } + rinfo = dfi->last_readdir ? &dfi->last_readdir->r_reply_info : NULL; + if (!rinfo || !rinfo->dir_nr) + return true; + chunk_offset = rinfo->dir_entries[0].offset; + return new_pos < chunk_offset || + is_hash_order(new_pos) != is_hash_order(chunk_offset); +} + +static loff_t ceph_dir_llseek(struct file *file, loff_t offset, int whence) +{ + struct ceph_dir_file_info *dfi = file->private_data; + struct inode *inode = file->f_mapping->host; + struct ceph_client *cl = ceph_inode_to_client(inode); + loff_t retval; + + inode_lock(inode); + retval = -EINVAL; + switch (whence) { + case SEEK_CUR: + offset += file->f_pos; + break; + case SEEK_SET: + break; + case SEEK_END: + retval = -EOPNOTSUPP; + goto out; + default: + goto out; + } + + if (offset >= 0) { + if (need_reset_readdir(dfi, offset)) { + doutc(cl, "%p %llx.%llx dropping %p content\n", + inode, ceph_vinop(inode), file); + reset_readdir(dfi); + } else if (is_hash_order(offset) && offset > file->f_pos) { + /* for hash offset, we don't know if a forward seek + * is within same frag */ + dfi->dir_release_count = 0; + dfi->readdir_cache_idx = -1; + } + + if (offset != file->f_pos) { + file->f_pos = offset; + dfi->file_info.flags &= ~CEPH_F_ATEND; + } + retval = offset; + } +out: + inode_unlock(inode); + return retval; +} + +/* + * Handle lookups for the hidden .snap directory. + */ +struct dentry *ceph_handle_snapdir(struct ceph_mds_request *req, + struct dentry *dentry) +{ + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(dentry->d_sb); + struct inode *parent = d_inode(dentry->d_parent); /* we hold i_rwsem */ + struct ceph_client *cl = ceph_inode_to_client(parent); + + /* .snap dir? */ + if (ceph_snap(parent) == CEPH_NOSNAP && + strcmp(dentry->d_name.name, fsc->mount_options->snapdir_name) == 0) { + struct dentry *res; + struct inode *inode = ceph_get_snapdir(parent); + + res = d_splice_alias(inode, dentry); + doutc(cl, "ENOENT on snapdir %p '%pd', linking to " + "snapdir %p %llx.%llx. Spliced dentry %p\n", + dentry, dentry, inode, ceph_vinop(inode), res); + if (res) + dentry = res; + } + return dentry; +} + +/* + * Figure out final result of a lookup/open request. + * + * Mainly, make sure we return the final req->r_dentry (if it already + * existed) in place of the original VFS-provided dentry when they + * differ. + * + * Gracefully handle the case where the MDS replies with -ENOENT and + * no trace (which it may do, at its discretion, e.g., if it doesn't + * care to issue a lease on the negative dentry). + */ +struct dentry *ceph_finish_lookup(struct ceph_mds_request *req, + struct dentry *dentry, int err) +{ + struct ceph_client *cl = req->r_mdsc->fsc->client; + + if (err == -ENOENT) { + /* no trace? */ + err = 0; + if (!req->r_reply_info.head->is_dentry) { + doutc(cl, + "ENOENT and no trace, dentry %p inode %llx.%llx\n", + dentry, ceph_vinop(d_inode(dentry))); + if (d_really_is_positive(dentry)) { + d_drop(dentry); + err = -ENOENT; + } else { + if (d_unhashed(dentry)) + d_add(dentry, NULL); + } + } + } + if (err) + dentry = ERR_PTR(err); + else if (dentry != req->r_dentry) + dentry = dget(req->r_dentry); /* we got spliced */ + else + dentry = NULL; + return dentry; +} + +static bool is_root_ceph_dentry(struct inode *inode, struct dentry *dentry) +{ + return ceph_ino(inode) == CEPH_INO_ROOT && + strncmp(dentry->d_name.name, ".ceph", 5) == 0; +} + +/* + * Look up a single dir entry. If there is a lookup intent, inform + * the MDS so that it gets our 'caps wanted' value in a single op. + */ +static struct dentry *ceph_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(dir->i_sb); + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(dir->i_sb); + struct ceph_client *cl = fsc->client; + struct ceph_mds_request *req; + int op; + int mask; + int err; + + doutc(cl, "%p %llx.%llx/'%pd' dentry %p\n", dir, ceph_vinop(dir), + dentry, dentry); + + if (dentry->d_name.len > NAME_MAX) + return ERR_PTR(-ENAMETOOLONG); + + if (IS_ENCRYPTED(dir)) { + bool had_key = fscrypt_has_encryption_key(dir); + + err = fscrypt_prepare_lookup_partial(dir, dentry); + if (err < 0) + return ERR_PTR(err); + + /* mark directory as incomplete if it has been unlocked */ + if (!had_key && fscrypt_has_encryption_key(dir)) + ceph_dir_clear_complete(dir); + } + + /* can we conclude ENOENT locally? */ + if (d_really_is_negative(dentry)) { + struct ceph_inode_info *ci = ceph_inode(dir); + struct ceph_dentry_info *di = ceph_dentry(dentry); + + spin_lock(&ci->i_ceph_lock); + doutc(cl, " dir %llx.%llx flags are 0x%lx\n", + ceph_vinop(dir), ci->i_ceph_flags); + if (strncmp(dentry->d_name.name, + fsc->mount_options->snapdir_name, + dentry->d_name.len) && + !is_root_ceph_dentry(dir, dentry) && + ceph_test_mount_opt(fsc, DCACHE) && + __ceph_dir_is_complete(ci) && + __ceph_caps_issued_mask_metric(ci, CEPH_CAP_FILE_SHARED, 1)) { + __ceph_touch_fmode(ci, mdsc, CEPH_FILE_MODE_RD); + spin_unlock(&ci->i_ceph_lock); + doutc(cl, " dir %llx.%llx complete, -ENOENT\n", + ceph_vinop(dir)); + if (d_unhashed(dentry)) + d_add(dentry, NULL); + di->lease_shared_gen = atomic_read(&ci->i_shared_gen); + return NULL; + } + spin_unlock(&ci->i_ceph_lock); + } + + op = ceph_snap(dir) == CEPH_SNAPDIR ? + CEPH_MDS_OP_LOOKUPSNAP : CEPH_MDS_OP_LOOKUP; + req = ceph_mdsc_create_request(mdsc, op, USE_ANY_MDS); + if (IS_ERR(req)) + return ERR_CAST(req); + req->r_dentry = dget(dentry); + req->r_num_caps = 2; + + mask = CEPH_STAT_CAP_INODE | CEPH_CAP_AUTH_SHARED; + if (ceph_security_xattr_wanted(dir)) + mask |= CEPH_CAP_XATTR_SHARED; + req->r_args.getattr.mask = cpu_to_le32(mask); + + ihold(dir); + req->r_parent = dir; + set_bit(CEPH_MDS_R_PARENT_LOCKED, &req->r_req_flags); + err = ceph_mdsc_do_request(mdsc, NULL, req); + if (err == -ENOENT) { + struct dentry *res; + + res = ceph_handle_snapdir(req, dentry); + if (IS_ERR(res)) { + err = PTR_ERR(res); + } else { + dentry = res; + err = 0; + } + } + dentry = ceph_finish_lookup(req, dentry, err); + ceph_mdsc_put_request(req); /* will dput(dentry) */ + doutc(cl, "result=%p\n", dentry); + return dentry; +} + +/* + * If we do a create but get no trace back from the MDS, follow up with + * a lookup (the VFS expects us to link up the provided dentry). + */ +int ceph_handle_notrace_create(struct inode *dir, struct dentry *dentry) +{ + struct dentry *result = ceph_lookup(dir, dentry, 0); + + if (result && !IS_ERR(result)) { + /* + * We created the item, then did a lookup, and found + * it was already linked to another inode we already + * had in our cache (and thus got spliced). To not + * confuse VFS (especially when inode is a directory), + * we don't link our dentry to that inode, return an + * error instead. + * + * This event should be rare and it happens only when + * we talk to old MDS. Recent MDS does not send traceless + * reply for request that creates new inode. + */ + d_drop(result); + return -ESTALE; + } + return PTR_ERR(result); +} + +static int ceph_mknod(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) +{ + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(dir->i_sb); + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_mds_request *req; + struct ceph_acl_sec_ctx as_ctx = {}; + int err; + + if (ceph_snap(dir) != CEPH_NOSNAP) + return -EROFS; + + err = ceph_wait_on_conflict_unlink(dentry); + if (err) + return err; + + if (ceph_quota_is_max_files_exceeded(dir)) { + err = -EDQUOT; + goto out; + } + + doutc(cl, "%p %llx.%llx/'%pd' dentry %p mode 0%ho rdev %d\n", + dir, ceph_vinop(dir), dentry, dentry, mode, rdev); + req = ceph_mdsc_create_request(mdsc, CEPH_MDS_OP_MKNOD, USE_AUTH_MDS); + if (IS_ERR(req)) { + err = PTR_ERR(req); + goto out; + } + + req->r_new_inode = ceph_new_inode(dir, dentry, &mode, &as_ctx); + if (IS_ERR(req->r_new_inode)) { + err = PTR_ERR(req->r_new_inode); + req->r_new_inode = NULL; + goto out_req; + } + + if (S_ISREG(mode) && IS_ENCRYPTED(dir)) + set_bit(CEPH_MDS_R_FSCRYPT_FILE, &req->r_req_flags); + + req->r_dentry = dget(dentry); + req->r_num_caps = 2; + req->r_parent = dir; + ihold(dir); + set_bit(CEPH_MDS_R_PARENT_LOCKED, &req->r_req_flags); + req->r_mnt_idmap = mnt_idmap_get(idmap); + req->r_args.mknod.mode = cpu_to_le32(mode); + req->r_args.mknod.rdev = cpu_to_le32(rdev); + req->r_dentry_drop = CEPH_CAP_FILE_SHARED | CEPH_CAP_AUTH_EXCL | + CEPH_CAP_XATTR_EXCL; + req->r_dentry_unless = CEPH_CAP_FILE_EXCL; + + ceph_as_ctx_to_req(req, &as_ctx); + + err = ceph_mdsc_do_request(mdsc, dir, req); + if (!err && !req->r_reply_info.head->is_dentry) + err = ceph_handle_notrace_create(dir, dentry); +out_req: + ceph_mdsc_put_request(req); +out: + if (!err) + ceph_init_inode_acls(d_inode(dentry), &as_ctx); + else + d_drop(dentry); + ceph_release_acl_sec_ctx(&as_ctx); + return err; +} + +static int ceph_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + return ceph_mknod(idmap, dir, dentry, mode, 0); +} + +#if IS_ENABLED(CONFIG_FS_ENCRYPTION) +static int prep_encrypted_symlink_target(struct ceph_mds_request *req, + const char *dest) +{ + int err; + int len = strlen(dest); + struct fscrypt_str osd_link = FSTR_INIT(NULL, 0); + + err = fscrypt_prepare_symlink(req->r_parent, dest, len, PATH_MAX, + &osd_link); + if (err) + goto out; + + err = fscrypt_encrypt_symlink(req->r_new_inode, dest, len, &osd_link); + if (err) + goto out; + + req->r_path2 = kmalloc(BASE64_CHARS(osd_link.len) + 1, GFP_KERNEL); + if (!req->r_path2) { + err = -ENOMEM; + goto out; + } + + len = base64_encode(osd_link.name, osd_link.len, + req->r_path2, false, BASE64_IMAP); + req->r_path2[len] = '\0'; +out: + fscrypt_fname_free_buffer(&osd_link); + return err; +} +#else +static int prep_encrypted_symlink_target(struct ceph_mds_request *req, + const char *dest) +{ + return -EOPNOTSUPP; +} +#endif + +static int ceph_symlink(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const char *dest) +{ + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(dir->i_sb); + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_mds_request *req; + struct ceph_acl_sec_ctx as_ctx = {}; + umode_t mode = S_IFLNK | 0777; + int err; + + if (ceph_snap(dir) != CEPH_NOSNAP) + return -EROFS; + + err = ceph_wait_on_conflict_unlink(dentry); + if (err) + return err; + + if (ceph_quota_is_max_files_exceeded(dir)) { + err = -EDQUOT; + goto out; + } + + doutc(cl, "%p %llx.%llx/'%pd' to '%s'\n", dir, ceph_vinop(dir), dentry, + dest); + req = ceph_mdsc_create_request(mdsc, CEPH_MDS_OP_SYMLINK, USE_AUTH_MDS); + if (IS_ERR(req)) { + err = PTR_ERR(req); + goto out; + } + + req->r_new_inode = ceph_new_inode(dir, dentry, &mode, &as_ctx); + if (IS_ERR(req->r_new_inode)) { + err = PTR_ERR(req->r_new_inode); + req->r_new_inode = NULL; + goto out_req; + } + + req->r_parent = dir; + ihold(dir); + + if (IS_ENCRYPTED(req->r_new_inode)) { + err = prep_encrypted_symlink_target(req, dest); + if (err) + goto out_req; + } else { + req->r_path2 = kstrdup(dest, GFP_KERNEL); + if (!req->r_path2) { + err = -ENOMEM; + goto out_req; + } + } + + set_bit(CEPH_MDS_R_PARENT_LOCKED, &req->r_req_flags); + req->r_mnt_idmap = mnt_idmap_get(idmap); + req->r_dentry = dget(dentry); + req->r_num_caps = 2; + req->r_dentry_drop = CEPH_CAP_FILE_SHARED | CEPH_CAP_AUTH_EXCL | + CEPH_CAP_XATTR_EXCL; + req->r_dentry_unless = CEPH_CAP_FILE_EXCL; + + ceph_as_ctx_to_req(req, &as_ctx); + + err = ceph_mdsc_do_request(mdsc, dir, req); + if (!err && !req->r_reply_info.head->is_dentry) + err = ceph_handle_notrace_create(dir, dentry); +out_req: + ceph_mdsc_put_request(req); +out: + if (err) + d_drop(dentry); + ceph_release_acl_sec_ctx(&as_ctx); + return err; +} + +static struct dentry *ceph_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(dir->i_sb); + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_mds_request *req; + struct ceph_acl_sec_ctx as_ctx = {}; + struct dentry *ret; + int err; + int op; + + err = ceph_wait_on_conflict_unlink(dentry); + if (err) + return ERR_PTR(err); + + if (ceph_snap(dir) == CEPH_SNAPDIR) { + /* mkdir .snap/foo is a MKSNAP */ + op = CEPH_MDS_OP_MKSNAP; + doutc(cl, "mksnap %llx.%llx/'%pd' dentry %p\n", + ceph_vinop(dir), dentry, dentry); + } else if (ceph_snap(dir) == CEPH_NOSNAP) { + doutc(cl, "mkdir %llx.%llx/'%pd' dentry %p mode 0%ho\n", + ceph_vinop(dir), dentry, dentry, mode); + op = CEPH_MDS_OP_MKDIR; + } else { + ret = ERR_PTR(-EROFS); + goto out; + } + + if (op == CEPH_MDS_OP_MKDIR && + ceph_quota_is_max_files_exceeded(dir)) { + ret = ERR_PTR(-EDQUOT); + goto out; + } + if ((op == CEPH_MDS_OP_MKSNAP) && IS_ENCRYPTED(dir) && + !fscrypt_has_encryption_key(dir)) { + ret = ERR_PTR(-ENOKEY); + goto out; + } + + + req = ceph_mdsc_create_request(mdsc, op, USE_AUTH_MDS); + if (IS_ERR(req)) { + ret = ERR_CAST(req); + goto out; + } + + req->r_new_inode = ceph_new_inode(dir, dentry, &mode, &as_ctx); + if (IS_ERR(req->r_new_inode)) { + ret = ERR_CAST(req->r_new_inode); + req->r_new_inode = NULL; + goto out_req; + } + + req->r_dentry = dget(dentry); + req->r_num_caps = 2; + req->r_parent = dir; + ihold(dir); + set_bit(CEPH_MDS_R_PARENT_LOCKED, &req->r_req_flags); + if (op == CEPH_MDS_OP_MKDIR) + req->r_mnt_idmap = mnt_idmap_get(idmap); + req->r_args.mkdir.mode = cpu_to_le32(mode); + req->r_dentry_drop = CEPH_CAP_FILE_SHARED | CEPH_CAP_AUTH_EXCL | + CEPH_CAP_XATTR_EXCL; + req->r_dentry_unless = CEPH_CAP_FILE_EXCL; + + ceph_as_ctx_to_req(req, &as_ctx); + + err = ceph_mdsc_do_request(mdsc, dir, req); + if (!err && + !req->r_reply_info.head->is_target && + !req->r_reply_info.head->is_dentry) + err = ceph_handle_notrace_create(dir, dentry); + ret = ERR_PTR(err); +out_req: + if (!IS_ERR(ret) && req->r_dentry != dentry) + /* Some other dentry was spliced in */ + ret = dget(req->r_dentry); + ceph_mdsc_put_request(req); +out: + if (!IS_ERR(ret)) { + if (ret) + dentry = ret; + ceph_init_inode_acls(d_inode(dentry), &as_ctx); + } else { + d_drop(dentry); + } + ceph_release_acl_sec_ctx(&as_ctx); + return ret; +} + +static int ceph_link(struct dentry *old_dentry, struct inode *dir, + struct dentry *dentry) +{ + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(dir->i_sb); + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_mds_request *req; + int err; + + if (dentry->d_flags & DCACHE_DISCONNECTED) + return -EINVAL; + + err = ceph_wait_on_conflict_unlink(dentry); + if (err) + return err; + + if (ceph_snap(dir) != CEPH_NOSNAP) + return -EROFS; + + err = fscrypt_prepare_link(old_dentry, dir, dentry); + if (err) + return err; + + doutc(cl, "%p %llx.%llx/'%pd' to '%pd'\n", dir, ceph_vinop(dir), + old_dentry, dentry); + req = ceph_mdsc_create_request(mdsc, CEPH_MDS_OP_LINK, USE_AUTH_MDS); + if (IS_ERR(req)) { + d_drop(dentry); + return PTR_ERR(req); + } + req->r_dentry = dget(dentry); + req->r_num_caps = 2; + req->r_old_dentry = dget(old_dentry); + /* + * The old_dentry maybe a DCACHE_DISCONNECTED dentry, then we + * will just pass the ino# to MDSs. + */ + if (old_dentry->d_flags & DCACHE_DISCONNECTED) + req->r_ino2 = ceph_vino(d_inode(old_dentry)); + req->r_parent = dir; + ihold(dir); + set_bit(CEPH_MDS_R_PARENT_LOCKED, &req->r_req_flags); + req->r_dentry_drop = CEPH_CAP_FILE_SHARED | CEPH_CAP_XATTR_EXCL; + req->r_dentry_unless = CEPH_CAP_FILE_EXCL; + /* release LINK_SHARED on source inode (mds will lock it) */ + req->r_old_inode_drop = CEPH_CAP_LINK_SHARED | CEPH_CAP_LINK_EXCL; + err = ceph_mdsc_do_request(mdsc, dir, req); + if (err) { + d_drop(dentry); + } else if (!req->r_reply_info.head->is_dentry) { + ihold(d_inode(old_dentry)); + d_instantiate(dentry, d_inode(old_dentry)); + } + ceph_mdsc_put_request(req); + return err; +} + +static void ceph_async_unlink_cb(struct ceph_mds_client *mdsc, + struct ceph_mds_request *req) +{ + struct dentry *dentry = req->r_dentry; + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(dentry->d_sb); + struct ceph_client *cl = fsc->client; + struct ceph_dentry_info *di = ceph_dentry(dentry); + int result = req->r_err ? req->r_err : + le32_to_cpu(req->r_reply_info.head->result); + + if (!test_bit(CEPH_DENTRY_ASYNC_UNLINK_BIT, &di->flags)) + pr_warn_client(cl, + "dentry %p:%pd async unlink bit is not set\n", + dentry, dentry); + + spin_lock(&fsc->async_unlink_conflict_lock); + hash_del_rcu(&di->hnode); + spin_unlock(&fsc->async_unlink_conflict_lock); + + spin_lock(&dentry->d_lock); + clear_and_wake_up_bit(CEPH_DENTRY_ASYNC_UNLINK_BIT, &di->flags); + spin_unlock(&dentry->d_lock); + + synchronize_rcu(); + + if (result == -EJUKEBOX) + goto out; + + /* If op failed, mark everyone involved for errors */ + if (result) { + struct ceph_path_info path_info = {0}; + char *path = ceph_mdsc_build_path(mdsc, dentry, &path_info, 0); + + /* mark error on parent + clear complete */ + mapping_set_error(req->r_parent->i_mapping, result); + ceph_dir_clear_complete(req->r_parent); + + /* drop the dentry -- we don't know its status */ + if (!d_unhashed(dentry)) + d_drop(dentry); + + /* mark inode itself for an error (since metadata is bogus) */ + mapping_set_error(req->r_old_inode->i_mapping, result); + + pr_warn_client(cl, "failure path=(%llx)%s result=%d!\n", + path_info.vino.ino, IS_ERR(path) ? "<>" : path, result); + ceph_mdsc_free_path_info(&path_info); + } +out: + iput(req->r_old_inode); + ceph_mdsc_release_dir_caps(req); +} + +static int get_caps_for_async_unlink(struct inode *dir, struct dentry *dentry) +{ + struct ceph_inode_info *ci = ceph_inode(dir); + struct ceph_dentry_info *di; + int got = 0, want = CEPH_CAP_FILE_EXCL | CEPH_CAP_DIR_UNLINK; + + spin_lock(&ci->i_ceph_lock); + if ((__ceph_caps_issued(ci, NULL) & want) == want) { + ceph_take_cap_refs(ci, want, false); + got = want; + } + spin_unlock(&ci->i_ceph_lock); + + /* If we didn't get anything, return 0 */ + if (!got) + return 0; + + spin_lock(&dentry->d_lock); + di = ceph_dentry(dentry); + /* + * - We are holding Fx, which implies Fs caps. + * - Only support async unlink for primary linkage + */ + if (atomic_read(&ci->i_shared_gen) != di->lease_shared_gen || + !(di->flags & CEPH_DENTRY_PRIMARY_LINK)) + want = 0; + spin_unlock(&dentry->d_lock); + + /* Do we still want what we've got? */ + if (want == got) + return got; + + ceph_put_cap_refs(ci, got); + return 0; +} + +/* + * rmdir and unlink are differ only by the metadata op code + */ +static int ceph_unlink(struct inode *dir, struct dentry *dentry) +{ + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(dir->i_sb); + struct ceph_client *cl = fsc->client; + struct ceph_mds_client *mdsc = fsc->mdsc; + struct inode *inode = d_inode(dentry); + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_mds_request *req; + bool try_async = ceph_test_mount_opt(fsc, ASYNC_DIROPS); + struct dentry *dn; + int err = -EROFS; + int op; + char *path; + + if (ceph_snap(dir) == CEPH_SNAPDIR) { + /* rmdir .snap/foo is RMSNAP */ + doutc(cl, "rmsnap %llx.%llx/'%pd' dn\n", ceph_vinop(dir), + dentry); + op = CEPH_MDS_OP_RMSNAP; + } else if (ceph_snap(dir) == CEPH_NOSNAP) { + doutc(cl, "unlink/rmdir %llx.%llx/'%pd' inode %llx.%llx\n", + ceph_vinop(dir), dentry, ceph_vinop(inode)); + op = d_is_dir(dentry) ? + CEPH_MDS_OP_RMDIR : CEPH_MDS_OP_UNLINK; + } else + goto out; + + dn = d_find_alias(dir); + if (!dn) { + try_async = false; + } else { + struct ceph_path_info path_info = {0}; + path = ceph_mdsc_build_path(mdsc, dn, &path_info, 0); + if (IS_ERR(path)) { + try_async = false; + err = 0; + } else { + err = ceph_mds_check_access(mdsc, path, MAY_WRITE); + } + ceph_mdsc_free_path_info(&path_info); + dput(dn); + + /* For none EACCES cases will let the MDS do the mds auth check */ + if (err == -EACCES) { + return err; + } else if (err < 0) { + try_async = false; + err = 0; + } + } + +retry: + req = ceph_mdsc_create_request(mdsc, op, USE_AUTH_MDS); + if (IS_ERR(req)) { + err = PTR_ERR(req); + goto out; + } + req->r_dentry = dget(dentry); + req->r_num_caps = 2; + req->r_parent = dir; + ihold(dir); + req->r_dentry_drop = CEPH_CAP_FILE_SHARED | CEPH_CAP_XATTR_EXCL; + req->r_dentry_unless = CEPH_CAP_FILE_EXCL; + req->r_inode_drop = ceph_drop_caps_for_unlink(inode); + + if (try_async && op == CEPH_MDS_OP_UNLINK && + (req->r_dir_caps = get_caps_for_async_unlink(dir, dentry))) { + struct ceph_dentry_info *di = ceph_dentry(dentry); + + doutc(cl, "async unlink on %llx.%llx/'%pd' caps=%s", + ceph_vinop(dir), dentry, + ceph_cap_string(req->r_dir_caps)); + set_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags); + req->r_callback = ceph_async_unlink_cb; + req->r_old_inode = d_inode(dentry); + ihold(req->r_old_inode); + + spin_lock(&dentry->d_lock); + di->flags |= CEPH_DENTRY_ASYNC_UNLINK; + spin_unlock(&dentry->d_lock); + + spin_lock(&fsc->async_unlink_conflict_lock); + hash_add_rcu(fsc->async_unlink_conflict, &di->hnode, + dentry->d_name.hash); + spin_unlock(&fsc->async_unlink_conflict_lock); + + err = ceph_mdsc_submit_request(mdsc, dir, req); + if (!err) { + /* + * We have enough caps, so we assume that the unlink + * will succeed. Fix up the target inode and dcache. + */ + + /* + * Protect the i_nlink update with i_ceph_lock + * to precent racing against ceph_fill_inode() + * handling our completion on a worker thread + * and don't decrement if i_nlink has already + * been updated to zero by this completion. + */ + spin_lock(&ci->i_ceph_lock); + if (inode->i_nlink > 0) + drop_nlink(inode); + spin_unlock(&ci->i_ceph_lock); + + d_delete(dentry); + } else { + spin_lock(&fsc->async_unlink_conflict_lock); + hash_del_rcu(&di->hnode); + spin_unlock(&fsc->async_unlink_conflict_lock); + + spin_lock(&dentry->d_lock); + di->flags &= ~CEPH_DENTRY_ASYNC_UNLINK; + spin_unlock(&dentry->d_lock); + + if (err == -EJUKEBOX) { + try_async = false; + ceph_mdsc_put_request(req); + goto retry; + } + } + } else { + set_bit(CEPH_MDS_R_PARENT_LOCKED, &req->r_req_flags); + err = ceph_mdsc_do_request(mdsc, dir, req); + if (!err && !req->r_reply_info.head->is_dentry) + d_delete(dentry); + } + + ceph_mdsc_put_request(req); +out: + return err; +} + +static int ceph_rename(struct mnt_idmap *idmap, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) +{ + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(old_dir->i_sb); + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_mds_request *req; + int op = CEPH_MDS_OP_RENAME; + int err; + + if (flags) + return -EINVAL; + + if (ceph_snap(old_dir) != ceph_snap(new_dir)) + return -EXDEV; + if (ceph_snap(old_dir) != CEPH_NOSNAP) { + if (old_dir == new_dir && ceph_snap(old_dir) == CEPH_SNAPDIR) + op = CEPH_MDS_OP_RENAMESNAP; + else + return -EROFS; + } + /* don't allow cross-quota renames */ + if ((old_dir != new_dir) && + (!ceph_quota_is_same_realm(old_dir, new_dir))) + return -EXDEV; + + err = ceph_wait_on_conflict_unlink(new_dentry); + if (err) + return err; + + err = fscrypt_prepare_rename(old_dir, old_dentry, new_dir, new_dentry, + flags); + if (err) + return err; + + doutc(cl, "%llx.%llx/'%pd' to %llx.%llx/'%pd'\n", + ceph_vinop(old_dir), old_dentry, ceph_vinop(new_dir), + new_dentry); + req = ceph_mdsc_create_request(mdsc, op, USE_AUTH_MDS); + if (IS_ERR(req)) + return PTR_ERR(req); + ihold(old_dir); + req->r_dentry = dget(new_dentry); + req->r_num_caps = 2; + req->r_old_dentry = dget(old_dentry); + req->r_old_dentry_dir = old_dir; + req->r_parent = new_dir; + ihold(new_dir); + set_bit(CEPH_MDS_R_PARENT_LOCKED, &req->r_req_flags); + req->r_old_dentry_drop = CEPH_CAP_FILE_SHARED | CEPH_CAP_XATTR_EXCL; + req->r_old_dentry_unless = CEPH_CAP_FILE_EXCL; + req->r_dentry_drop = CEPH_CAP_FILE_SHARED | CEPH_CAP_XATTR_EXCL; + req->r_dentry_unless = CEPH_CAP_FILE_EXCL; + /* release LINK_RDCACHE on source inode (mds will lock it) */ + req->r_old_inode_drop = CEPH_CAP_LINK_SHARED | CEPH_CAP_LINK_EXCL; + if (d_really_is_positive(new_dentry)) { + req->r_inode_drop = + ceph_drop_caps_for_unlink(d_inode(new_dentry)); + } + err = ceph_mdsc_do_request(mdsc, old_dir, req); + if (!err && !req->r_reply_info.head->is_dentry) { + /* + * Normally d_move() is done by fill_trace (called by + * do_request, above). If there is no trace, we need + * to do it here. + */ + d_move(old_dentry, new_dentry); + } + ceph_mdsc_put_request(req); + return err; +} + +/* + * Move dentry to tail of mdsc->dentry_leases list when lease is updated. + * Leases at front of the list will expire first. (Assume all leases have + * similar duration) + * + * Called under dentry->d_lock. + */ +void __ceph_dentry_lease_touch(struct ceph_dentry_info *di) +{ + struct dentry *dn = di->dentry; + struct ceph_mds_client *mdsc = ceph_sb_to_fs_client(dn->d_sb)->mdsc; + struct ceph_client *cl = mdsc->fsc->client; + + doutc(cl, "%p %p '%pd'\n", di, dn, dn); + + di->flags |= CEPH_DENTRY_LEASE_LIST; + if (di->flags & CEPH_DENTRY_SHRINK_LIST) { + di->flags |= CEPH_DENTRY_REFERENCED; + return; + } + + spin_lock(&mdsc->dentry_list_lock); + list_move_tail(&di->lease_list, &mdsc->dentry_leases); + spin_unlock(&mdsc->dentry_list_lock); +} + +static void __dentry_dir_lease_touch(struct ceph_mds_client* mdsc, + struct ceph_dentry_info *di) +{ + di->flags &= ~(CEPH_DENTRY_LEASE_LIST | CEPH_DENTRY_REFERENCED); + di->lease_gen = 0; + di->time = jiffies; + list_move_tail(&di->lease_list, &mdsc->dentry_dir_leases); +} + +/* + * When dir lease is used, add dentry to tail of mdsc->dentry_dir_leases + * list if it's not in the list, otherwise set 'referenced' flag. + * + * Called under dentry->d_lock. + */ +void __ceph_dentry_dir_lease_touch(struct ceph_dentry_info *di) +{ + struct dentry *dn = di->dentry; + struct ceph_mds_client *mdsc = ceph_sb_to_fs_client(dn->d_sb)->mdsc; + struct ceph_client *cl = mdsc->fsc->client; + + doutc(cl, "%p %p '%pd' (offset 0x%llx)\n", di, dn, dn, di->offset); + + if (!list_empty(&di->lease_list)) { + if (di->flags & CEPH_DENTRY_LEASE_LIST) { + /* don't remove dentry from dentry lease list + * if its lease is valid */ + if (__dentry_lease_is_valid(di)) + return; + } else { + di->flags |= CEPH_DENTRY_REFERENCED; + return; + } + } + + if (di->flags & CEPH_DENTRY_SHRINK_LIST) { + di->flags |= CEPH_DENTRY_REFERENCED; + di->flags &= ~CEPH_DENTRY_LEASE_LIST; + return; + } + + spin_lock(&mdsc->dentry_list_lock); + __dentry_dir_lease_touch(mdsc, di); + spin_unlock(&mdsc->dentry_list_lock); +} + +static void __dentry_lease_unlist(struct ceph_dentry_info *di) +{ + struct ceph_mds_client *mdsc; + if (di->flags & CEPH_DENTRY_SHRINK_LIST) + return; + if (list_empty(&di->lease_list)) + return; + + mdsc = ceph_sb_to_fs_client(di->dentry->d_sb)->mdsc; + spin_lock(&mdsc->dentry_list_lock); + list_del_init(&di->lease_list); + spin_unlock(&mdsc->dentry_list_lock); +} + +enum { + KEEP = 0, + DELETE = 1, + TOUCH = 2, + STOP = 4, +}; + +struct ceph_lease_walk_control { + bool dir_lease; + bool expire_dir_lease; + unsigned long nr_to_scan; + unsigned long dir_lease_ttl; +}; + +static int __dir_lease_check(const struct dentry *, struct ceph_lease_walk_control *); +static int __dentry_lease_check(const struct dentry *); + +static unsigned long +__dentry_leases_walk(struct ceph_mds_client *mdsc, + struct ceph_lease_walk_control *lwc) +{ + struct ceph_dentry_info *di, *tmp; + struct dentry *dentry, *last = NULL; + struct list_head* list; + LIST_HEAD(dispose); + unsigned long freed = 0; + int ret = 0; + + list = lwc->dir_lease ? &mdsc->dentry_dir_leases : &mdsc->dentry_leases; + spin_lock(&mdsc->dentry_list_lock); + list_for_each_entry_safe(di, tmp, list, lease_list) { + if (!lwc->nr_to_scan) + break; + --lwc->nr_to_scan; + + dentry = di->dentry; + if (last == dentry) + break; + + if (!spin_trylock(&dentry->d_lock)) + continue; + + if (lockref_is_dead(&dentry->d_lockref)) { + list_del_init(&di->lease_list); + goto next; + } + + if (lwc->dir_lease) + ret = __dir_lease_check(dentry, lwc); + else + ret = __dentry_lease_check(dentry); + if (ret & TOUCH) { + /* move it into tail of dir lease list */ + __dentry_dir_lease_touch(mdsc, di); + if (!last) + last = dentry; + } + if (ret & DELETE) { + /* stale lease */ + di->flags &= ~CEPH_DENTRY_REFERENCED; + if (dentry->d_lockref.count > 0) { + /* update_dentry_lease() will re-add + * it to lease list, or + * ceph_d_delete() will return 1 when + * last reference is dropped */ + list_del_init(&di->lease_list); + } else { + di->flags |= CEPH_DENTRY_SHRINK_LIST; + list_move_tail(&di->lease_list, &dispose); + dget_dlock(dentry); + } + } +next: + spin_unlock(&dentry->d_lock); + if (ret & STOP) + break; + } + spin_unlock(&mdsc->dentry_list_lock); + + while (!list_empty(&dispose)) { + di = list_first_entry(&dispose, struct ceph_dentry_info, + lease_list); + dentry = di->dentry; + spin_lock(&dentry->d_lock); + + list_del_init(&di->lease_list); + di->flags &= ~CEPH_DENTRY_SHRINK_LIST; + if (di->flags & CEPH_DENTRY_REFERENCED) { + spin_lock(&mdsc->dentry_list_lock); + if (di->flags & CEPH_DENTRY_LEASE_LIST) { + list_add_tail(&di->lease_list, + &mdsc->dentry_leases); + } else { + __dentry_dir_lease_touch(mdsc, di); + } + spin_unlock(&mdsc->dentry_list_lock); + } else { + freed++; + } + + spin_unlock(&dentry->d_lock); + /* ceph_d_delete() does the trick */ + dput(dentry); + } + return freed; +} + +static int __dentry_lease_check(const struct dentry *dentry) +{ + struct ceph_dentry_info *di = ceph_dentry(dentry); + int ret; + + if (__dentry_lease_is_valid(di)) + return STOP; + ret = __dir_lease_try_check(dentry); + if (ret == -EBUSY) + return KEEP; + if (ret > 0) + return TOUCH; + return DELETE; +} + +static int __dir_lease_check(const struct dentry *dentry, + struct ceph_lease_walk_control *lwc) +{ + struct ceph_dentry_info *di = ceph_dentry(dentry); + + int ret = __dir_lease_try_check(dentry); + if (ret == -EBUSY) + return KEEP; + if (ret > 0) { + if (time_before(jiffies, di->time + lwc->dir_lease_ttl)) + return STOP; + /* Move dentry to tail of dir lease list if we don't want + * to delete it. So dentries in the list are checked in a + * round robin manner */ + if (!lwc->expire_dir_lease) + return TOUCH; + if (dentry->d_lockref.count > 0 || + (di->flags & CEPH_DENTRY_REFERENCED)) + return TOUCH; + /* invalidate dir lease */ + di->lease_shared_gen = 0; + } + return DELETE; +} + +int ceph_trim_dentries(struct ceph_mds_client *mdsc) +{ + struct ceph_lease_walk_control lwc; + unsigned long count; + unsigned long freed; + + spin_lock(&mdsc->caps_list_lock); + if (mdsc->caps_use_max > 0 && + mdsc->caps_use_count > mdsc->caps_use_max) + count = mdsc->caps_use_count - mdsc->caps_use_max; + else + count = 0; + spin_unlock(&mdsc->caps_list_lock); + + lwc.dir_lease = false; + lwc.nr_to_scan = CEPH_CAPS_PER_RELEASE * 2; + freed = __dentry_leases_walk(mdsc, &lwc); + if (!lwc.nr_to_scan) /* more invalid leases */ + return -EAGAIN; + + if (lwc.nr_to_scan < CEPH_CAPS_PER_RELEASE) + lwc.nr_to_scan = CEPH_CAPS_PER_RELEASE; + + lwc.dir_lease = true; + lwc.expire_dir_lease = freed < count; + lwc.dir_lease_ttl = mdsc->fsc->mount_options->caps_wanted_delay_max * HZ; + freed +=__dentry_leases_walk(mdsc, &lwc); + if (!lwc.nr_to_scan) /* more to check */ + return -EAGAIN; + + return freed > 0 ? 1 : 0; +} + +/* + * Ensure a dentry lease will no longer revalidate. + */ +void ceph_invalidate_dentry_lease(struct dentry *dentry) +{ + struct ceph_dentry_info *di = ceph_dentry(dentry); + spin_lock(&dentry->d_lock); + di->time = jiffies; + di->lease_shared_gen = 0; + di->flags &= ~CEPH_DENTRY_PRIMARY_LINK; + __dentry_lease_unlist(di); + spin_unlock(&dentry->d_lock); +} + +/* + * Check if dentry lease is valid. If not, delete the lease. Try to + * renew if the least is more than half up. + */ +static bool __dentry_lease_is_valid(struct ceph_dentry_info *di) +{ + struct ceph_mds_session *session; + + if (!di->lease_gen) + return false; + + session = di->lease_session; + if (session) { + u32 gen; + unsigned long ttl; + + gen = atomic_read(&session->s_cap_gen); + ttl = session->s_cap_ttl; + + if (di->lease_gen == gen && + time_before(jiffies, ttl) && + time_before(jiffies, di->time)) + return true; + } + di->lease_gen = 0; + return false; +} + +static int dentry_lease_is_valid(struct dentry *dentry, unsigned int flags) +{ + struct ceph_dentry_info *di; + struct ceph_mds_session *session = NULL; + struct ceph_mds_client *mdsc = ceph_sb_to_fs_client(dentry->d_sb)->mdsc; + struct ceph_client *cl = mdsc->fsc->client; + u32 seq = 0; + int valid = 0; + + spin_lock(&dentry->d_lock); + di = ceph_dentry(dentry); + if (di && __dentry_lease_is_valid(di)) { + valid = 1; + + if (di->lease_renew_after && + time_after(jiffies, di->lease_renew_after)) { + /* + * We should renew. If we're in RCU walk mode + * though, we can't do that so just return + * -ECHILD. + */ + if (flags & LOOKUP_RCU) { + valid = -ECHILD; + } else { + session = ceph_get_mds_session(di->lease_session); + seq = di->lease_seq; + di->lease_renew_after = 0; + di->lease_renew_from = jiffies; + } + } + } + spin_unlock(&dentry->d_lock); + + if (session) { + ceph_mdsc_lease_send_msg(session, dentry, + CEPH_MDS_LEASE_RENEW, seq); + ceph_put_mds_session(session); + } + doutc(cl, "dentry %p = %d\n", dentry, valid); + return valid; +} + +/* + * Called under dentry->d_lock. + */ +static int __dir_lease_try_check(const struct dentry *dentry) +{ + struct ceph_dentry_info *di = ceph_dentry(dentry); + struct inode *dir; + struct ceph_inode_info *ci; + int valid = 0; + + if (!di->lease_shared_gen) + return 0; + if (IS_ROOT(dentry)) + return 0; + + dir = d_inode(dentry->d_parent); + ci = ceph_inode(dir); + + if (spin_trylock(&ci->i_ceph_lock)) { + if (atomic_read(&ci->i_shared_gen) == di->lease_shared_gen && + __ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 0)) + valid = 1; + spin_unlock(&ci->i_ceph_lock); + } else { + valid = -EBUSY; + } + + if (!valid) + di->lease_shared_gen = 0; + return valid; +} + +/* + * Check if directory-wide content lease/cap is valid. + */ +static int dir_lease_is_valid(struct inode *dir, struct dentry *dentry, + struct ceph_mds_client *mdsc) +{ + struct ceph_inode_info *ci = ceph_inode(dir); + struct ceph_client *cl = mdsc->fsc->client; + int valid; + int shared_gen; + + spin_lock(&ci->i_ceph_lock); + valid = __ceph_caps_issued_mask(ci, CEPH_CAP_FILE_SHARED, 1); + if (valid) { + __ceph_touch_fmode(ci, mdsc, CEPH_FILE_MODE_RD); + shared_gen = atomic_read(&ci->i_shared_gen); + } + spin_unlock(&ci->i_ceph_lock); + if (valid) { + struct ceph_dentry_info *di; + spin_lock(&dentry->d_lock); + di = ceph_dentry(dentry); + if (dir == d_inode(dentry->d_parent) && + di && di->lease_shared_gen == shared_gen) + __ceph_dentry_dir_lease_touch(di); + else + valid = 0; + spin_unlock(&dentry->d_lock); + } + doutc(cl, "dir %p %llx.%llx v%u dentry %p '%pd' = %d\n", dir, + ceph_vinop(dir), (unsigned)atomic_read(&ci->i_shared_gen), + dentry, dentry, valid); + return valid; +} + +/* + * Check if cached dentry can be trusted. + */ +static int ceph_d_revalidate(struct inode *dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + struct ceph_mds_client *mdsc = ceph_sb_to_fs_client(dentry->d_sb)->mdsc; + struct ceph_client *cl = mdsc->fsc->client; + int valid = 0; + struct inode *inode; + + valid = fscrypt_d_revalidate(dir, name, dentry, flags); + if (valid <= 0) + return valid; + + inode = d_inode_rcu(dentry); + + doutc(cl, "%p '%pd' inode %p offset 0x%llx nokey %d\n", + dentry, dentry, inode, ceph_dentry(dentry)->offset, + !!(dentry->d_flags & DCACHE_NOKEY_NAME)); + + mdsc = ceph_sb_to_fs_client(dir->i_sb)->mdsc; + + /* always trust cached snapped dentries, snapdir dentry */ + if (ceph_snap(dir) != CEPH_NOSNAP) { + doutc(cl, "%p '%pd' inode %p is SNAPPED\n", dentry, + dentry, inode); + valid = 1; + } else if (inode && ceph_snap(inode) == CEPH_SNAPDIR) { + valid = 1; + } else { + valid = dentry_lease_is_valid(dentry, flags); + if (valid == -ECHILD) + return valid; + if (valid || dir_lease_is_valid(dir, dentry, mdsc)) { + if (inode) + valid = ceph_is_any_caps(inode); + else + valid = 1; + } + } + + if (!valid) { + struct ceph_mds_request *req; + int op, err; + u32 mask; + + if (flags & LOOKUP_RCU) + return -ECHILD; + + percpu_counter_inc(&mdsc->metric.d_lease_mis); + + op = ceph_snap(dir) == CEPH_SNAPDIR ? + CEPH_MDS_OP_LOOKUPSNAP : CEPH_MDS_OP_LOOKUP; + req = ceph_mdsc_create_request(mdsc, op, USE_ANY_MDS); + if (!IS_ERR(req)) { + req->r_dentry = dget(dentry); + req->r_num_caps = 2; + req->r_parent = dir; + ihold(dir); + + req->r_dname = name; + + mask = CEPH_STAT_CAP_INODE | CEPH_CAP_AUTH_SHARED; + if (ceph_security_xattr_wanted(dir)) + mask |= CEPH_CAP_XATTR_SHARED; + req->r_args.getattr.mask = cpu_to_le32(mask); + + err = ceph_mdsc_do_request(mdsc, NULL, req); + switch (err) { + case 0: + if (d_really_is_positive(dentry) && + d_inode(dentry) == req->r_target_inode) + valid = 1; + break; + case -ENOENT: + if (d_really_is_negative(dentry)) + valid = 1; + fallthrough; + default: + break; + } + ceph_mdsc_put_request(req); + doutc(cl, "%p '%pd', lookup result=%d\n", dentry, + dentry, err); + } + } else { + percpu_counter_inc(&mdsc->metric.d_lease_hit); + } + + doutc(cl, "%p '%pd' %s\n", dentry, dentry, valid ? "valid" : "invalid"); + if (!valid) + ceph_dir_clear_complete(dir); + return valid; +} + +/* + * Delete unused dentry that doesn't have valid lease + * + * Called under dentry->d_lock. + */ +static int ceph_d_delete(const struct dentry *dentry) +{ + struct ceph_dentry_info *di; + + /* won't release caps */ + if (d_really_is_negative(dentry)) + return 0; + if (ceph_snap(d_inode(dentry)) != CEPH_NOSNAP) + return 0; + /* valid lease? */ + di = ceph_dentry(dentry); + if (di) { + if (__dentry_lease_is_valid(di)) + return 0; + if (__dir_lease_try_check(dentry)) + return 0; + } + return 1; +} + +/* + * Release our ceph_dentry_info. + */ +static void ceph_d_release(struct dentry *dentry) +{ + struct ceph_dentry_info *di = ceph_dentry(dentry); + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(dentry->d_sb); + + doutc(fsc->client, "dentry %p '%pd'\n", dentry, dentry); + + atomic64_dec(&fsc->mdsc->metric.total_dentries); + + spin_lock(&dentry->d_lock); + __dentry_lease_unlist(di); + dentry->d_fsdata = NULL; + spin_unlock(&dentry->d_lock); + + ceph_put_mds_session(di->lease_session); + kmem_cache_free(ceph_dentry_cachep, di); +} + +/* + * When the VFS prunes a dentry from the cache, we need to clear the + * complete flag on the parent directory. + * + * Called under dentry->d_lock. + */ +static void ceph_d_prune(struct dentry *dentry) +{ + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(dentry->d_sb); + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_inode_info *dir_ci; + struct ceph_dentry_info *di; + + doutc(cl, "dentry %p '%pd'\n", dentry, dentry); + + /* do we have a valid parent? */ + if (IS_ROOT(dentry)) + return; + + /* we hold d_lock, so d_parent is stable */ + dir_ci = ceph_inode(d_inode(dentry->d_parent)); + if (dir_ci->i_vino.snap == CEPH_SNAPDIR) + return; + + /* who calls d_delete() should also disable dcache readdir */ + if (d_really_is_negative(dentry)) + return; + + /* d_fsdata does not get cleared until d_release */ + if (!d_unhashed(dentry)) { + __ceph_dir_clear_complete(dir_ci); + return; + } + + /* Disable dcache readdir just in case that someone called d_drop() + * or d_invalidate(), but MDS didn't revoke CEPH_CAP_FILE_SHARED + * properly (dcache readdir is still enabled) */ + di = ceph_dentry(dentry); + if (di->offset > 0 && + di->lease_shared_gen == atomic_read(&dir_ci->i_shared_gen)) + __ceph_dir_clear_ordered(dir_ci); +} + +/* + * read() on a dir. This weird interface hack only works if mounted + * with '-o dirstat'. + */ +static ssize_t ceph_read_dir(struct file *file, char __user *buf, size_t size, + loff_t *ppos) +{ + struct ceph_dir_file_info *dfi = file->private_data; + struct inode *inode = file_inode(file); + struct ceph_inode_info *ci = ceph_inode(inode); + int left; + const int bufsize = 1024; + + if (!ceph_test_mount_opt(ceph_sb_to_fs_client(inode->i_sb), DIRSTAT)) + return -EISDIR; + + if (!dfi->dir_info) { + dfi->dir_info = kmalloc(bufsize, GFP_KERNEL); + if (!dfi->dir_info) + return -ENOMEM; + dfi->dir_info_len = + snprintf(dfi->dir_info, bufsize, + "entries: %20lld\n" + " files: %20lld\n" + " subdirs: %20lld\n" + "rentries: %20lld\n" + " rfiles: %20lld\n" + " rsubdirs: %20lld\n" + "rbytes: %20lld\n" + "rctime: %ptSp\n", + ci->i_files + ci->i_subdirs, + ci->i_files, + ci->i_subdirs, + ci->i_rfiles + ci->i_rsubdirs, + ci->i_rfiles, + ci->i_rsubdirs, + ci->i_rbytes, + &ci->i_rctime); + } + + if (*ppos >= dfi->dir_info_len) + return 0; + size = min_t(unsigned, size, dfi->dir_info_len-*ppos); + left = copy_to_user(buf, dfi->dir_info + *ppos, size); + if (left == size) + return -EFAULT; + *ppos += (size - left); + return size - left; +} + + + +/* + * Return name hash for a given dentry. This is dependent on + * the parent directory's hash function. + */ +unsigned ceph_dentry_hash(struct inode *dir, struct dentry *dn) +{ + struct ceph_inode_info *dci = ceph_inode(dir); + unsigned hash; + + switch (dci->i_dir_layout.dl_dir_hash) { + case 0: /* for backward compat */ + case CEPH_STR_HASH_LINUX: + return dn->d_name.hash; + + default: + spin_lock(&dn->d_lock); + hash = ceph_str_hash(dci->i_dir_layout.dl_dir_hash, + dn->d_name.name, dn->d_name.len); + spin_unlock(&dn->d_lock); + return hash; + } +} + +WRAP_DIR_ITER(ceph_readdir) // FIXME! +const struct file_operations ceph_dir_fops = { + .read = ceph_read_dir, + .iterate_shared = shared_ceph_readdir, + .llseek = ceph_dir_llseek, + .open = ceph_open, + .release = ceph_release, + .unlocked_ioctl = ceph_ioctl, + .compat_ioctl = compat_ptr_ioctl, + .fsync = ceph_fsync, + .lock = ceph_lock, + .flock = ceph_flock, +}; + +const struct file_operations ceph_snapdir_fops = { + .iterate_shared = shared_ceph_readdir, + .llseek = ceph_dir_llseek, + .open = ceph_open, + .release = ceph_release, +}; + +const struct inode_operations ceph_dir_iops = { + .lookup = ceph_lookup, + .permission = ceph_permission, + .getattr = ceph_getattr, + .setattr = ceph_setattr, + .listxattr = ceph_listxattr, + .get_inode_acl = ceph_get_acl, + .set_acl = ceph_set_acl, + .mknod = ceph_mknod, + .symlink = ceph_symlink, + .mkdir = ceph_mkdir, + .link = ceph_link, + .unlink = ceph_unlink, + .rmdir = ceph_unlink, + .rename = ceph_rename, + .create = ceph_create, + .atomic_open = ceph_atomic_open, +}; + +const struct inode_operations ceph_snapdir_iops = { + .lookup = ceph_lookup, + .permission = ceph_permission, + .getattr = ceph_getattr, + .mkdir = ceph_mkdir, + .rmdir = ceph_unlink, + .rename = ceph_rename, +}; + +const struct dentry_operations ceph_dentry_ops = { + .d_revalidate = ceph_d_revalidate, + .d_delete = ceph_d_delete, + .d_release = ceph_d_release, + .d_prune = ceph_d_prune, + .d_init = ceph_d_init, +}; diff --git a/fs/ceph/export.c b/fs/ceph/export.c new file mode 100644 index 000000000..b2f2af104 --- /dev/null +++ b/fs/ceph/export.c @@ -0,0 +1,618 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +#include +#include +#include + +#include "super.h" +#include "mds_client.h" +#include "crypto.h" + +/* + * Basic fh + */ +struct ceph_nfs_fh { + u64 ino; +} __attribute__ ((packed)); + +/* + * Larger fh that includes parent ino. + */ +struct ceph_nfs_confh { + u64 ino, parent_ino; +} __attribute__ ((packed)); + +/* + * fh for snapped inode + */ +struct ceph_nfs_snapfh { + u64 ino; + u64 snapid; + u64 parent_ino; + u32 hash; +} __attribute__ ((packed)); + +#define BYTES_PER_U32 (sizeof(u32)) +#define CEPH_FH_BASIC_SIZE \ + (sizeof(struct ceph_nfs_fh) / BYTES_PER_U32) +#define CEPH_FH_WITH_PARENT_SIZE \ + (sizeof(struct ceph_nfs_confh) / BYTES_PER_U32) +#define CEPH_FH_SNAPPED_INODE_SIZE \ + (sizeof(struct ceph_nfs_snapfh) / BYTES_PER_U32) + +static int ceph_encode_snapfh(struct inode *inode, u32 *rawfh, int *max_len, + struct inode *parent_inode) +{ + struct ceph_client *cl = ceph_inode_to_client(inode); + static const int snap_handle_length = CEPH_FH_SNAPPED_INODE_SIZE; + struct ceph_nfs_snapfh *sfh = (void *)rawfh; + u64 snapid = ceph_snap(inode); + int ret; + bool no_parent = true; + + if (*max_len < snap_handle_length) { + *max_len = snap_handle_length; + ret = FILEID_INVALID; + goto out; + } + + ret = -EINVAL; + if (snapid != CEPH_SNAPDIR) { + struct inode *dir; + struct dentry *dentry = d_find_alias(inode); + if (!dentry) + goto out; + + rcu_read_lock(); + dir = d_inode_rcu(dentry->d_parent); + if (ceph_snap(dir) != CEPH_SNAPDIR) { + sfh->parent_ino = ceph_ino(dir); + sfh->hash = ceph_dentry_hash(dir, dentry); + no_parent = false; + } + rcu_read_unlock(); + dput(dentry); + } + + if (no_parent) { + if (!S_ISDIR(inode->i_mode)) + goto out; + sfh->parent_ino = sfh->ino; + sfh->hash = 0; + } + sfh->ino = ceph_ino(inode); + sfh->snapid = snapid; + + *max_len = snap_handle_length; + ret = FILEID_BTRFS_WITH_PARENT; +out: + doutc(cl, "%p %llx.%llx ret=%d\n", inode, ceph_vinop(inode), ret); + return ret; +} + +static int ceph_encode_fh(struct inode *inode, u32 *rawfh, int *max_len, + struct inode *parent_inode) +{ + struct ceph_client *cl = ceph_inode_to_client(inode); + static const int handle_length = CEPH_FH_BASIC_SIZE; + static const int connected_handle_length = CEPH_FH_WITH_PARENT_SIZE; + int type; + + if (ceph_snap(inode) != CEPH_NOSNAP) + return ceph_encode_snapfh(inode, rawfh, max_len, parent_inode); + + if (parent_inode && (*max_len < connected_handle_length)) { + *max_len = connected_handle_length; + return FILEID_INVALID; + } else if (*max_len < handle_length) { + *max_len = handle_length; + return FILEID_INVALID; + } + + if (parent_inode) { + struct ceph_nfs_confh *cfh = (void *)rawfh; + doutc(cl, "%p %llx.%llx with parent %p %llx.%llx\n", inode, + ceph_vinop(inode), parent_inode, ceph_vinop(parent_inode)); + cfh->ino = ceph_ino(inode); + cfh->parent_ino = ceph_ino(parent_inode); + *max_len = connected_handle_length; + type = FILEID_INO32_GEN_PARENT; + } else { + struct ceph_nfs_fh *fh = (void *)rawfh; + doutc(cl, "%p %llx.%llx\n", inode, ceph_vinop(inode)); + fh->ino = ceph_ino(inode); + *max_len = handle_length; + type = FILEID_INO32_GEN; + } + return type; +} + +static struct inode *__lookup_inode(struct super_block *sb, u64 ino) +{ + struct ceph_mds_client *mdsc = ceph_sb_to_fs_client(sb)->mdsc; + struct inode *inode; + struct ceph_vino vino; + int err; + + vino.ino = ino; + vino.snap = CEPH_NOSNAP; + + if (ceph_vino_is_reserved(vino)) + return ERR_PTR(-ESTALE); + + inode = ceph_find_inode(sb, vino); + if (!inode) { + struct ceph_mds_request *req; + int mask; + + req = ceph_mdsc_create_request(mdsc, CEPH_MDS_OP_LOOKUPINO, + USE_ANY_MDS); + if (IS_ERR(req)) + return ERR_CAST(req); + + mask = CEPH_STAT_CAP_INODE; + if (ceph_security_xattr_wanted(d_inode(sb->s_root))) + mask |= CEPH_CAP_XATTR_SHARED; + req->r_args.lookupino.mask = cpu_to_le32(mask); + + req->r_ino1 = vino; + req->r_num_caps = 1; + err = ceph_mdsc_do_request(mdsc, NULL, req); + inode = req->r_target_inode; + if (inode) + ihold(inode); + ceph_mdsc_put_request(req); + if (!inode) + return err < 0 ? ERR_PTR(err) : ERR_PTR(-ESTALE); + } else { + if (ceph_inode_is_shutdown(inode)) { + iput(inode); + return ERR_PTR(-ESTALE); + } + } + return inode; +} + +struct inode *ceph_lookup_inode(struct super_block *sb, u64 ino) +{ + struct inode *inode = __lookup_inode(sb, ino); + if (IS_ERR(inode)) + return inode; + if (inode->i_nlink == 0) { + iput(inode); + return ERR_PTR(-ESTALE); + } + return inode; +} + +static struct dentry *__fh_to_dentry(struct super_block *sb, u64 ino) +{ + struct inode *inode = __lookup_inode(sb, ino); + struct ceph_inode_info *ci = ceph_inode(inode); + int err; + + if (IS_ERR(inode)) + return ERR_CAST(inode); + /* We need LINK caps to reliably check i_nlink */ + err = ceph_do_getattr(inode, CEPH_CAP_LINK_SHARED, false); + if (err) { + iput(inode); + return ERR_PTR(err); + } + /* -ESTALE if inode as been unlinked and no file is open */ + if ((inode->i_nlink == 0) && !__ceph_is_file_opened(ci)) { + iput(inode); + return ERR_PTR(-ESTALE); + } + return d_obtain_alias(inode); +} + +static struct dentry *__snapfh_to_dentry(struct super_block *sb, + struct ceph_nfs_snapfh *sfh, + bool want_parent) +{ + struct ceph_mds_client *mdsc = ceph_sb_to_fs_client(sb)->mdsc; + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_mds_request *req; + struct inode *inode; + struct ceph_vino vino; + int mask; + int err; + bool unlinked = false; + + if (want_parent) { + vino.ino = sfh->parent_ino; + if (sfh->snapid == CEPH_SNAPDIR) + vino.snap = CEPH_NOSNAP; + else if (sfh->ino == sfh->parent_ino) + vino.snap = CEPH_SNAPDIR; + else + vino.snap = sfh->snapid; + } else { + vino.ino = sfh->ino; + vino.snap = sfh->snapid; + } + + if (ceph_vino_is_reserved(vino)) + return ERR_PTR(-ESTALE); + + inode = ceph_find_inode(sb, vino); + if (inode) { + if (ceph_inode_is_shutdown(inode)) { + iput(inode); + return ERR_PTR(-ESTALE); + } + return d_obtain_alias(inode); + } + + req = ceph_mdsc_create_request(mdsc, CEPH_MDS_OP_LOOKUPINO, + USE_ANY_MDS); + if (IS_ERR(req)) + return ERR_CAST(req); + + mask = CEPH_STAT_CAP_INODE; + if (ceph_security_xattr_wanted(d_inode(sb->s_root))) + mask |= CEPH_CAP_XATTR_SHARED; + req->r_args.lookupino.mask = cpu_to_le32(mask); + if (vino.snap < CEPH_NOSNAP) { + req->r_args.lookupino.snapid = cpu_to_le64(vino.snap); + if (!want_parent && sfh->ino != sfh->parent_ino) { + req->r_args.lookupino.parent = + cpu_to_le64(sfh->parent_ino); + req->r_args.lookupino.hash = + cpu_to_le32(sfh->hash); + } + } + + req->r_ino1 = vino; + req->r_num_caps = 1; + err = ceph_mdsc_do_request(mdsc, NULL, req); + inode = req->r_target_inode; + if (inode) { + if (vino.snap == CEPH_SNAPDIR) { + if (inode->i_nlink == 0) + unlinked = true; + inode = ceph_get_snapdir(inode); + } else if (ceph_snap(inode) == vino.snap) { + ihold(inode); + } else { + /* mds does not support lookup snapped inode */ + inode = ERR_PTR(-EOPNOTSUPP); + } + } else { + inode = ERR_PTR(-ESTALE); + } + ceph_mdsc_put_request(req); + + if (want_parent) { + doutc(cl, "%llx.%llx\n err=%d\n", vino.ino, vino.snap, err); + } else { + doutc(cl, "%llx.%llx parent %llx hash %x err=%d", vino.ino, + vino.snap, sfh->parent_ino, sfh->hash, err); + } + /* see comments in ceph_get_parent() */ + return unlinked ? d_obtain_root(inode) : d_obtain_alias(inode); +} + +/* + * convert regular fh to dentry + */ +static struct dentry *ceph_fh_to_dentry(struct super_block *sb, + struct fid *fid, + int fh_len, int fh_type) +{ + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(sb); + struct ceph_nfs_fh *fh = (void *)fid->raw; + + if (fh_type == FILEID_BTRFS_WITH_PARENT) { + struct ceph_nfs_snapfh *sfh = (void *)fid->raw; + return __snapfh_to_dentry(sb, sfh, false); + } + + if (fh_type != FILEID_INO32_GEN && + fh_type != FILEID_INO32_GEN_PARENT) + return NULL; + if (fh_len < sizeof(*fh) / BYTES_PER_U32) + return NULL; + + doutc(fsc->client, "%llx\n", fh->ino); + return __fh_to_dentry(sb, fh->ino); +} + +static struct dentry *__get_parent(struct super_block *sb, + struct dentry *child, u64 ino) +{ + struct ceph_mds_client *mdsc = ceph_sb_to_fs_client(sb)->mdsc; + struct ceph_mds_request *req; + struct inode *inode; + int mask; + int err; + + req = ceph_mdsc_create_request(mdsc, CEPH_MDS_OP_LOOKUPPARENT, + USE_ANY_MDS); + if (IS_ERR(req)) + return ERR_CAST(req); + + if (child) { + req->r_inode = d_inode(child); + ihold(d_inode(child)); + } else { + req->r_ino1 = (struct ceph_vino) { + .ino = ino, + .snap = CEPH_NOSNAP, + }; + } + + mask = CEPH_STAT_CAP_INODE; + if (ceph_security_xattr_wanted(d_inode(sb->s_root))) + mask |= CEPH_CAP_XATTR_SHARED; + req->r_args.getattr.mask = cpu_to_le32(mask); + + req->r_num_caps = 1; + err = ceph_mdsc_do_request(mdsc, NULL, req); + if (err) { + ceph_mdsc_put_request(req); + return ERR_PTR(err); + } + + inode = req->r_target_inode; + if (inode) + ihold(inode); + ceph_mdsc_put_request(req); + if (!inode) + return ERR_PTR(-ENOENT); + + return d_obtain_alias(inode); +} + +static struct dentry *ceph_get_parent(struct dentry *child) +{ + struct inode *inode = d_inode(child); + struct ceph_client *cl = ceph_inode_to_client(inode); + struct dentry *dn; + + if (ceph_snap(inode) != CEPH_NOSNAP) { + struct inode* dir; + bool unlinked = false; + /* do not support non-directory */ + if (!d_is_dir(child)) { + dn = ERR_PTR(-EINVAL); + goto out; + } + dir = __lookup_inode(inode->i_sb, ceph_ino(inode)); + if (IS_ERR(dir)) { + dn = ERR_CAST(dir); + goto out; + } + /* There can be multiple paths to access snapped inode. + * For simplicity, treat snapdir of head inode as parent */ + if (ceph_snap(inode) != CEPH_SNAPDIR) { + struct inode *snapdir = ceph_get_snapdir(dir); + if (dir->i_nlink == 0) + unlinked = true; + iput(dir); + if (IS_ERR(snapdir)) { + dn = ERR_CAST(snapdir); + goto out; + } + dir = snapdir; + } + /* If directory has already been deleted, further get_parent + * will fail. Do not mark snapdir dentry as disconnected, + * this prevents exportfs from doing further get_parent. */ + if (unlinked) + dn = d_obtain_root(dir); + else + dn = d_obtain_alias(dir); + } else { + dn = __get_parent(child->d_sb, child, 0); + } +out: + doutc(cl, "child %p %p %llx.%llx err=%ld\n", child, inode, + ceph_vinop(inode), (long)PTR_ERR_OR_ZERO(dn)); + return dn; +} + +/* + * convert regular fh to parent + */ +static struct dentry *ceph_fh_to_parent(struct super_block *sb, + struct fid *fid, + int fh_len, int fh_type) +{ + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(sb); + struct ceph_nfs_confh *cfh = (void *)fid->raw; + struct dentry *dentry; + + if (fh_type == FILEID_BTRFS_WITH_PARENT) { + struct ceph_nfs_snapfh *sfh = (void *)fid->raw; + return __snapfh_to_dentry(sb, sfh, true); + } + + if (fh_type != FILEID_INO32_GEN_PARENT) + return NULL; + if (fh_len < sizeof(*cfh) / BYTES_PER_U32) + return NULL; + + doutc(fsc->client, "%llx\n", cfh->parent_ino); + dentry = __get_parent(sb, NULL, cfh->ino); + if (unlikely(dentry == ERR_PTR(-ENOENT))) + dentry = __fh_to_dentry(sb, cfh->parent_ino); + return dentry; +} + +static int __get_snap_name(struct dentry *parent, char *name, + struct dentry *child) +{ + struct inode *inode = d_inode(child); + struct inode *dir = d_inode(parent); + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + struct ceph_mds_request *req = NULL; + char *last_name = NULL; + unsigned next_offset = 2; + int err = -EINVAL; + + if (ceph_ino(inode) != ceph_ino(dir)) + goto out; + if (ceph_snap(inode) == CEPH_SNAPDIR) { + if (ceph_snap(dir) == CEPH_NOSNAP) { + /* + * .get_name() from struct export_operations + * assumes that its 'name' parameter is pointing + * to a NAME_MAX+1 sized buffer + */ + strscpy(name, fsc->mount_options->snapdir_name, + NAME_MAX + 1); + err = 0; + } + goto out; + } + if (ceph_snap(dir) != CEPH_SNAPDIR) + goto out; + + while (1) { + struct ceph_mds_reply_info_parsed *rinfo; + struct ceph_mds_reply_dir_entry *rde; + int i; + + req = ceph_mdsc_create_request(fsc->mdsc, CEPH_MDS_OP_LSSNAP, + USE_AUTH_MDS); + if (IS_ERR(req)) { + err = PTR_ERR(req); + req = NULL; + goto out; + } + err = ceph_alloc_readdir_reply_buffer(req, inode); + if (err) + goto out; + + req->r_direct_mode = USE_AUTH_MDS; + req->r_readdir_offset = next_offset; + req->r_args.readdir.flags = + cpu_to_le16(CEPH_READDIR_REPLY_BITFLAGS); + if (last_name) { + req->r_path2 = last_name; + last_name = NULL; + } + + req->r_inode = dir; + ihold(dir); + req->r_dentry = dget(parent); + + inode_lock(dir); + err = ceph_mdsc_do_request(fsc->mdsc, NULL, req); + inode_unlock(dir); + + if (err < 0) + goto out; + + rinfo = &req->r_reply_info; + for (i = 0; i < rinfo->dir_nr; i++) { + rde = rinfo->dir_entries + i; + BUG_ON(!rde->inode.in); + if (ceph_snap(inode) == + le64_to_cpu(rde->inode.in->snapid)) { + memcpy(name, rde->name, rde->name_len); + name[rde->name_len] = '\0'; + err = 0; + goto out; + } + } + + if (rinfo->dir_end) + break; + + BUG_ON(rinfo->dir_nr <= 0); + rde = rinfo->dir_entries + (rinfo->dir_nr - 1); + next_offset += rinfo->dir_nr; + last_name = kstrndup(rde->name, rde->name_len, GFP_KERNEL); + if (!last_name) { + err = -ENOMEM; + goto out; + } + + ceph_mdsc_put_request(req); + req = NULL; + } + err = -ENOENT; +out: + if (req) + ceph_mdsc_put_request(req); + kfree(last_name); + doutc(fsc->client, "child dentry %p %p %llx.%llx err=%d\n", child, + inode, ceph_vinop(inode), err); + return err; +} + +static int ceph_get_name(struct dentry *parent, char *name, + struct dentry *child) +{ + struct ceph_mds_client *mdsc; + struct ceph_mds_request *req; + struct inode *dir = d_inode(parent); + struct inode *inode = d_inode(child); + struct ceph_mds_reply_info_parsed *rinfo; + int err; + + if (ceph_snap(inode) != CEPH_NOSNAP) + return __get_snap_name(parent, name, child); + + mdsc = ceph_inode_to_fs_client(inode)->mdsc; + req = ceph_mdsc_create_request(mdsc, CEPH_MDS_OP_LOOKUPNAME, + USE_ANY_MDS); + if (IS_ERR(req)) + return PTR_ERR(req); + + inode_lock(dir); + req->r_inode = inode; + ihold(inode); + req->r_ino2 = ceph_vino(d_inode(parent)); + req->r_parent = dir; + ihold(dir); + set_bit(CEPH_MDS_R_PARENT_LOCKED, &req->r_req_flags); + req->r_num_caps = 2; + err = ceph_mdsc_do_request(mdsc, NULL, req); + inode_unlock(dir); + + if (err) + goto out; + + rinfo = &req->r_reply_info; + if (!IS_ENCRYPTED(dir)) { + memcpy(name, rinfo->dname, rinfo->dname_len); + name[rinfo->dname_len] = 0; + } else { + struct fscrypt_str oname = FSTR_INIT(NULL, 0); + struct ceph_fname fname = { .dir = dir, + .name = rinfo->dname, + .ctext = rinfo->altname, + .name_len = rinfo->dname_len, + .ctext_len = rinfo->altname_len }; + + err = ceph_fname_alloc_buffer(dir, &oname); + if (err < 0) + goto out; + + err = ceph_fname_to_usr(&fname, NULL, &oname, NULL); + if (!err) { + memcpy(name, oname.name, oname.len); + name[oname.len] = 0; + } + ceph_fname_free_buffer(dir, &oname); + } +out: + doutc(mdsc->fsc->client, "child dentry %p %p %llx.%llx err %d %s%s\n", + child, inode, ceph_vinop(inode), err, err ? "" : "name ", + err ? "" : name); + ceph_mdsc_put_request(req); + return err; +} + +const struct export_operations ceph_export_ops = { + .encode_fh = ceph_encode_fh, + .fh_to_dentry = ceph_fh_to_dentry, + .fh_to_parent = ceph_fh_to_parent, + .get_parent = ceph_get_parent, + .get_name = ceph_get_name, +}; diff --git a/fs/ceph/file.c b/fs/ceph/file.c new file mode 100644 index 000000000..a4a2a4b6a --- /dev/null +++ b/fs/ceph/file.c @@ -0,0 +1,3260 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "mds_client.h" +#include "cache.h" +#include "io.h" +#include "metric.h" +#include "subvolume_metrics.h" + +/* + * Record I/O for subvolume metrics tracking. + * + * Callers must ensure bytes > 0 for reads (ret > 0 check) to avoid counting + * EOF as an I/O operation. For writes, the condition is (ret >= 0 && len > 0). + */ +static inline void ceph_record_subvolume_io(struct inode *inode, bool is_write, + ktime_t start, ktime_t end, + size_t bytes) +{ + if (!bytes) + return; + + ceph_subvolume_metrics_record_io(ceph_sb_to_mdsc(inode->i_sb), + ceph_inode(inode), + is_write, bytes, start, end); +} + +static __le32 ceph_flags_sys2wire(struct ceph_mds_client *mdsc, u32 flags) +{ + struct ceph_client *cl = mdsc->fsc->client; + u32 wire_flags = 0; + + switch (flags & O_ACCMODE) { + case O_RDONLY: + wire_flags |= CEPH_O_RDONLY; + break; + case O_WRONLY: + wire_flags |= CEPH_O_WRONLY; + break; + case O_RDWR: + wire_flags |= CEPH_O_RDWR; + break; + } + + flags &= ~O_ACCMODE; + +#define ceph_sys2wire(a) if (flags & a) { wire_flags |= CEPH_##a; flags &= ~a; } + + ceph_sys2wire(O_CREAT); + ceph_sys2wire(O_EXCL); + ceph_sys2wire(O_TRUNC); + ceph_sys2wire(O_DIRECTORY); + ceph_sys2wire(O_NOFOLLOW); + +#undef ceph_sys2wire + + if (flags) + doutc(cl, "unused open flags: %x\n", flags); + + return cpu_to_le32(wire_flags); +} + +/* + * Ceph file operations + * + * Implement basic open/close functionality, and implement + * read/write. + * + * We implement three modes of file I/O: + * - buffered uses the generic_file_aio_{read,write} helpers + * + * - synchronous is used when there is multi-client read/write + * sharing, avoids the page cache, and synchronously waits for an + * ack from the OSD. + * + * - direct io takes the variant of the sync path that references + * user pages directly. + * + * fsync() flushes and waits on dirty pages, but just queues metadata + * for writeback: since the MDS can recover size and mtime there is no + * need to wait for MDS acknowledgement. + */ + +/* + * How many pages to get in one call to iov_iter_get_pages(). This + * determines the size of the on-stack array used as a buffer. + */ +#define ITER_GET_BVECS_PAGES 64 + +static ssize_t __iter_get_bvecs(struct iov_iter *iter, size_t maxsize, + struct bio_vec *bvecs) +{ + size_t size = 0; + int bvec_idx = 0; + + if (maxsize > iov_iter_count(iter)) + maxsize = iov_iter_count(iter); + + while (size < maxsize) { + struct page *pages[ITER_GET_BVECS_PAGES]; + ssize_t bytes; + size_t start; + int idx = 0; + + bytes = iov_iter_get_pages2(iter, pages, maxsize - size, + ITER_GET_BVECS_PAGES, &start); + if (bytes < 0) + return size ?: bytes; + + size += bytes; + + for ( ; bytes; idx++, bvec_idx++) { + int len = min_t(int, bytes, PAGE_SIZE - start); + + bvec_set_page(&bvecs[bvec_idx], pages[idx], len, start); + bytes -= len; + start = 0; + } + } + + return size; +} + +/* + * iov_iter_get_pages() only considers one iov_iter segment, no matter + * what maxsize or maxpages are given. For ITER_BVEC that is a single + * page. + * + * Attempt to get up to @maxsize bytes worth of pages from @iter. + * Return the number of bytes in the created bio_vec array, or an error. + */ +static ssize_t iter_get_bvecs_alloc(struct iov_iter *iter, size_t maxsize, + struct bio_vec **bvecs, int *num_bvecs) +{ + struct bio_vec *bv; + size_t orig_count = iov_iter_count(iter); + ssize_t bytes; + int npages; + + iov_iter_truncate(iter, maxsize); + npages = iov_iter_npages(iter, INT_MAX); + iov_iter_reexpand(iter, orig_count); + + /* + * __iter_get_bvecs() may populate only part of the array -- zero it + * out. + */ + bv = kvmalloc_objs(*bv, npages, GFP_KERNEL | __GFP_ZERO); + if (!bv) + return -ENOMEM; + + bytes = __iter_get_bvecs(iter, maxsize, bv); + if (bytes < 0) { + /* + * No pages were pinned -- just free the array. + */ + kvfree(bv); + return bytes; + } + + *bvecs = bv; + *num_bvecs = npages; + return bytes; +} + +static void put_bvecs(struct bio_vec *bvecs, int num_bvecs, bool should_dirty) +{ + int i; + + for (i = 0; i < num_bvecs; i++) { + if (bvecs[i].bv_page) { + if (should_dirty) + set_page_dirty_lock(bvecs[i].bv_page); + put_page(bvecs[i].bv_page); + } + } + kvfree(bvecs); +} + +/* + * Prepare an open request. Preallocate ceph_cap to avoid an + * inopportune ENOMEM later. + */ +static struct ceph_mds_request * +prepare_open_request(struct super_block *sb, int flags, int create_mode) +{ + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(sb); + struct ceph_mds_request *req; + int want_auth = USE_ANY_MDS; + int op = (flags & O_CREAT) ? CEPH_MDS_OP_CREATE : CEPH_MDS_OP_OPEN; + + if (flags & (O_WRONLY|O_RDWR|O_CREAT|O_TRUNC)) + want_auth = USE_AUTH_MDS; + + req = ceph_mdsc_create_request(mdsc, op, want_auth); + if (IS_ERR(req)) + goto out; + req->r_fmode = ceph_flags_to_mode(flags); + req->r_args.open.flags = ceph_flags_sys2wire(mdsc, flags); + req->r_args.open.mode = cpu_to_le32(create_mode); +out: + return req; +} + +static int ceph_init_file_info(struct inode *inode, struct file *file, + int fmode, bool isdir) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_mount_options *opt = + ceph_inode_to_fs_client(&ci->netfs.inode)->mount_options; + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_file_info *fi; + int ret; + + doutc(cl, "%p %llx.%llx %p 0%o (%s)\n", inode, ceph_vinop(inode), + file, inode->i_mode, isdir ? "dir" : "regular"); + BUG_ON(inode->i_fop->release != ceph_release); + + if (isdir) { + struct ceph_dir_file_info *dfi = + kmem_cache_zalloc(ceph_dir_file_cachep, GFP_KERNEL); + if (!dfi) + return -ENOMEM; + + file->private_data = dfi; + fi = &dfi->file_info; + dfi->next_offset = 2; + dfi->readdir_cache_idx = -1; + } else { + fi = kmem_cache_zalloc(ceph_file_cachep, GFP_KERNEL); + if (!fi) + return -ENOMEM; + + if (opt->flags & CEPH_MOUNT_OPT_NOPAGECACHE) + fi->flags |= CEPH_F_SYNC; + + file->private_data = fi; + } + + ceph_get_fmode(ci, fmode, 1); + fi->fmode = fmode; + + spin_lock_init(&fi->rw_contexts_lock); + INIT_LIST_HEAD(&fi->rw_contexts); + fi->filp_gen = READ_ONCE(ceph_inode_to_fs_client(inode)->filp_gen); + + if ((file->f_mode & FMODE_WRITE) && ceph_has_inline_data(ci)) { + ret = ceph_uninline_data(file); + if (ret < 0) + goto error; + } + + return 0; + +error: + ceph_fscache_unuse_cookie(inode, file->f_mode & FMODE_WRITE); + ceph_put_fmode(ci, fi->fmode, 1); + kmem_cache_free(ceph_file_cachep, fi); + /* wake up anyone waiting for caps on this inode */ + wake_up_all(&ci->i_cap_wq); + return ret; +} + +/* + * initialize private struct file data. + * if we fail, clean up by dropping fmode reference on the ceph_inode + */ +static int ceph_init_file(struct inode *inode, struct file *file, int fmode) +{ + struct ceph_client *cl = ceph_inode_to_client(inode); + int ret = 0; + + switch (inode->i_mode & S_IFMT) { + case S_IFREG: + ceph_fscache_use_cookie(inode, file->f_mode & FMODE_WRITE); + fallthrough; + case S_IFDIR: + ret = ceph_init_file_info(inode, file, fmode, + S_ISDIR(inode->i_mode)); + break; + + case S_IFLNK: + doutc(cl, "%p %llx.%llx %p 0%o (symlink)\n", inode, + ceph_vinop(inode), file, inode->i_mode); + break; + + default: + doutc(cl, "%p %llx.%llx %p 0%o (special)\n", inode, + ceph_vinop(inode), file, inode->i_mode); + /* + * we need to drop the open ref now, since we don't + * have .release set to ceph_release. + */ + BUG_ON(inode->i_fop->release == ceph_release); + + /* call the proper open fop */ + ret = inode->i_fop->open(inode, file); + } + return ret; +} + +/* + * Retry cap acquisition after a stale session or a lost cap update. + */ +int ceph_renew_caps(struct inode *inode, int fmode) +{ + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb); + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_mds_request *req; + int err, flags, wanted, issued; + + spin_lock(&ci->i_ceph_lock); + __ceph_touch_fmode(ci, mdsc, fmode); + wanted = __ceph_caps_file_wanted(ci); + issued = __ceph_caps_issued(ci, NULL); + if (__ceph_is_any_real_caps(ci) && + (!(wanted & CEPH_CAP_ANY_WR) || ci->i_auth_cap) && + (issued & wanted) == wanted) { + spin_unlock(&ci->i_ceph_lock); + doutc(cl, "%p %llx.%llx want %s issued %s updating mds_wanted\n", + inode, ceph_vinop(inode), ceph_cap_string(wanted), + ceph_cap_string(issued)); + ceph_check_caps(ci, 0); + return 0; + } + spin_unlock(&ci->i_ceph_lock); + + flags = 0; + if ((wanted & CEPH_CAP_FILE_RD) && (wanted & CEPH_CAP_FILE_WR)) + flags = O_RDWR; + else if (wanted & CEPH_CAP_FILE_RD) + flags = O_RDONLY; + else if (wanted & CEPH_CAP_FILE_WR) + flags = O_WRONLY; +#ifdef O_LAZY + if (wanted & CEPH_CAP_FILE_LAZYIO) + flags |= O_LAZY; +#endif + + req = prepare_open_request(inode->i_sb, flags, 0); + if (IS_ERR(req)) { + err = PTR_ERR(req); + goto out; + } + + req->r_inode = inode; + ihold(inode); + req->r_num_caps = 1; + + err = ceph_mdsc_do_request(mdsc, NULL, req); + ceph_mdsc_put_request(req); +out: + doutc(cl, "%p %llx.%llx open result=%d\n", inode, ceph_vinop(inode), + err); + return err < 0 ? err : 0; +} + +/* + * If we already have the requisite capabilities, we can satisfy + * the open request locally (no need to request new caps from the + * MDS). We do, however, need to inform the MDS (asynchronously) + * if our wanted caps set expands. + */ +int ceph_open(struct inode *inode, struct file *file) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(inode->i_sb); + struct ceph_client *cl = fsc->client; + struct ceph_mds_client *mdsc = fsc->mdsc; + struct ceph_mds_request *req; + struct ceph_file_info *fi = file->private_data; + int err; + int flags, fmode, wanted; + struct dentry *dentry; + char *path; + bool do_sync = false; + int mask = MAY_READ; + + if (fi) { + doutc(cl, "file %p is already opened\n", file); + return 0; + } + + /* filter out O_CREAT|O_EXCL; vfs did that already. yuck. */ + flags = file->f_flags & ~(O_CREAT|O_EXCL); + if (S_ISDIR(inode->i_mode)) { + flags = O_DIRECTORY; /* mds likes to know */ + } else if (S_ISREG(inode->i_mode)) { + err = fscrypt_file_open(inode, file); + if (err) + return err; + } + + doutc(cl, "%p %llx.%llx file %p flags %d (%d)\n", inode, + ceph_vinop(inode), file, flags, file->f_flags); + fmode = ceph_flags_to_mode(flags); + wanted = ceph_caps_for_mode(fmode); + + if (fmode & CEPH_FILE_MODE_WR) + mask |= MAY_WRITE; + dentry = d_find_alias(inode); + if (!dentry) { + do_sync = true; + } else { + struct ceph_path_info path_info = {0}; + path = ceph_mdsc_build_path(mdsc, dentry, &path_info, 0); + if (IS_ERR(path)) { + do_sync = true; + err = 0; + } else { + err = ceph_mds_check_access(mdsc, path, mask); + } + ceph_mdsc_free_path_info(&path_info); + dput(dentry); + + /* For none EACCES cases will let the MDS do the mds auth check */ + if (err == -EACCES) { + return err; + } else if (err < 0) { + do_sync = true; + err = 0; + } + } + + /* snapped files are read-only */ + if (ceph_snap(inode) != CEPH_NOSNAP && (file->f_mode & FMODE_WRITE)) + return -EROFS; + + /* trivially open snapdir */ + if (ceph_snap(inode) == CEPH_SNAPDIR) { + return ceph_init_file(inode, file, fmode); + } + + /* + * No need to block if we have caps on the auth MDS (for + * write) or any MDS (for read). Update wanted set + * asynchronously. + */ + spin_lock(&ci->i_ceph_lock); + if (!do_sync && __ceph_is_any_real_caps(ci) && + (((fmode & CEPH_FILE_MODE_WR) == 0) || ci->i_auth_cap)) { + int mds_wanted = __ceph_caps_mds_wanted(ci, true); + int issued = __ceph_caps_issued(ci, NULL); + + doutc(cl, "open %p fmode %d want %s issued %s using existing\n", + inode, fmode, ceph_cap_string(wanted), + ceph_cap_string(issued)); + __ceph_touch_fmode(ci, mdsc, fmode); + spin_unlock(&ci->i_ceph_lock); + + /* adjust wanted? */ + if ((issued & wanted) != wanted && + (mds_wanted & wanted) != wanted && + ceph_snap(inode) != CEPH_SNAPDIR) + ceph_check_caps(ci, 0); + + return ceph_init_file(inode, file, fmode); + } else if (!do_sync && ceph_snap(inode) != CEPH_NOSNAP && + (ci->i_snap_caps & wanted) == wanted) { + __ceph_touch_fmode(ci, mdsc, fmode); + spin_unlock(&ci->i_ceph_lock); + return ceph_init_file(inode, file, fmode); + } + + spin_unlock(&ci->i_ceph_lock); + + doutc(cl, "open fmode %d wants %s\n", fmode, ceph_cap_string(wanted)); + req = prepare_open_request(inode->i_sb, flags, 0); + if (IS_ERR(req)) { + err = PTR_ERR(req); + goto out; + } + req->r_inode = inode; + ihold(inode); + + req->r_num_caps = 1; + err = ceph_mdsc_do_request(mdsc, NULL, req); + if (!err) + err = ceph_init_file(inode, file, req->r_fmode); + ceph_mdsc_put_request(req); + doutc(cl, "open result=%d on %llx.%llx\n", err, ceph_vinop(inode)); +out: + return err; +} + +/* Clone the layout from a synchronous create, if the dir now has Dc caps */ +static void +cache_file_layout(struct inode *dst, struct inode *src) +{ + struct ceph_inode_info *cdst = ceph_inode(dst); + struct ceph_inode_info *csrc = ceph_inode(src); + + spin_lock(&cdst->i_ceph_lock); + if ((__ceph_caps_issued(cdst, NULL) & CEPH_CAP_DIR_CREATE) && + !ceph_file_layout_is_valid(&cdst->i_cached_layout)) { + memcpy(&cdst->i_cached_layout, &csrc->i_layout, + sizeof(cdst->i_cached_layout)); + rcu_assign_pointer(cdst->i_cached_layout.pool_ns, + ceph_try_get_string(csrc->i_layout.pool_ns)); + } + spin_unlock(&cdst->i_ceph_lock); +} + +/* + * Try to set up an async create. We need caps, a file layout, and inode number, + * and either a lease on the dentry or complete dir info. If any of those + * criteria are not satisfied, then return false and the caller can go + * synchronous. + */ +static int try_prep_async_create(struct inode *dir, struct dentry *dentry, + struct ceph_file_layout *lo, u64 *pino) +{ + struct ceph_inode_info *ci = ceph_inode(dir); + struct ceph_dentry_info *di = ceph_dentry(dentry); + int got = 0, want = CEPH_CAP_FILE_EXCL | CEPH_CAP_DIR_CREATE; + u64 ino; + + spin_lock(&ci->i_ceph_lock); + /* No auth cap means no chance for Dc caps */ + if (!ci->i_auth_cap) + goto no_async; + + /* Any delegated inos? */ + if (xa_empty(&ci->i_auth_cap->session->s_delegated_inos)) + goto no_async; + + if (!ceph_file_layout_is_valid(&ci->i_cached_layout)) + goto no_async; + + if ((__ceph_caps_issued(ci, NULL) & want) != want) + goto no_async; + + if (d_in_lookup(dentry)) { + if (!__ceph_dir_is_complete(ci)) + goto no_async; + spin_lock(&dentry->d_lock); + di->lease_shared_gen = atomic_read(&ci->i_shared_gen); + spin_unlock(&dentry->d_lock); + } else if (atomic_read(&ci->i_shared_gen) != + READ_ONCE(di->lease_shared_gen)) { + goto no_async; + } + + ino = ceph_get_deleg_ino(ci->i_auth_cap->session); + if (!ino) + goto no_async; + + *pino = ino; + ceph_take_cap_refs(ci, want, false); + memcpy(lo, &ci->i_cached_layout, sizeof(*lo)); + rcu_assign_pointer(lo->pool_ns, + ceph_try_get_string(ci->i_cached_layout.pool_ns)); + got = want; +no_async: + spin_unlock(&ci->i_ceph_lock); + return got; +} + +static void restore_deleg_ino(struct inode *dir, u64 ino) +{ + struct ceph_client *cl = ceph_inode_to_client(dir); + struct ceph_inode_info *ci = ceph_inode(dir); + struct ceph_mds_session *s = NULL; + + spin_lock(&ci->i_ceph_lock); + if (ci->i_auth_cap) + s = ceph_get_mds_session(ci->i_auth_cap->session); + spin_unlock(&ci->i_ceph_lock); + if (s) { + int err = ceph_restore_deleg_ino(s, ino); + if (err) + pr_warn_client(cl, + "unable to restore delegated ino 0x%llx to session: %d\n", + ino, err); + ceph_put_mds_session(s); + } +} + +static void wake_async_create_waiters(struct inode *inode, + struct ceph_mds_session *session) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + bool check_cap = false; + + spin_lock(&ci->i_ceph_lock); + if (ci->i_ceph_flags & CEPH_I_ASYNC_CREATE) { + /* Serialized by i_ceph_lock; the two ops touch different bits. */ + clear_and_wake_up_bit(CEPH_I_ASYNC_CREATE_BIT, &ci->i_ceph_flags); + + if (test_and_clear_bit(CEPH_I_ASYNC_CHECK_CAPS_BIT, + &ci->i_ceph_flags)) + check_cap = true; + } + ceph_kick_flushing_inode_caps(session, ci); + spin_unlock(&ci->i_ceph_lock); + + if (check_cap) + ceph_check_caps(ci, CHECK_CAPS_FLUSH); +} + +static void ceph_async_create_cb(struct ceph_mds_client *mdsc, + struct ceph_mds_request *req) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct dentry *dentry = req->r_dentry; + struct inode *dinode = d_inode(dentry); + struct inode *tinode = req->r_target_inode; + int result = req->r_err ? req->r_err : + le32_to_cpu(req->r_reply_info.head->result); + + WARN_ON_ONCE(dinode && tinode && dinode != tinode); + + /* MDS changed -- caller must resubmit */ + if (result == -EJUKEBOX) + goto out; + + mapping_set_error(req->r_parent->i_mapping, result); + + if (result) { + struct ceph_path_info path_info = {0}; + char *path = ceph_mdsc_build_path(mdsc, req->r_dentry, &path_info, 0); + + pr_warn_client(cl, + "async create failure path=(%llx)%s result=%d!\n", + path_info.vino.ino, IS_ERR(path) ? "<>" : path, result); + ceph_mdsc_free_path_info(&path_info); + + ceph_dir_clear_complete(req->r_parent); + if (!d_unhashed(dentry)) + d_drop(dentry); + + if (dinode) { + mapping_set_error(dinode->i_mapping, result); + ceph_inode_shutdown(dinode); + wake_async_create_waiters(dinode, req->r_session); + } + } + + if (tinode) { + u64 ino = ceph_vino(tinode).ino; + + if (req->r_deleg_ino != ino) + pr_warn_client(cl, + "inode number mismatch! err=%d deleg_ino=0x%llx target=0x%llx\n", + req->r_err, req->r_deleg_ino, ino); + + mapping_set_error(tinode->i_mapping, result); + wake_async_create_waiters(tinode, req->r_session); + } else if (!result) { + pr_warn_client(cl, "no req->r_target_inode for 0x%llx\n", + req->r_deleg_ino); + } +out: + ceph_mdsc_release_dir_caps(req); +} + +static int ceph_finish_async_create(struct inode *dir, struct inode *inode, + struct dentry *dentry, + struct file *file, umode_t mode, + struct ceph_mds_request *req, + struct ceph_acl_sec_ctx *as_ctx, + struct ceph_file_layout *lo) +{ + int ret; + char xattr_buf[4]; + struct ceph_mds_reply_inode in = { }; + struct ceph_mds_reply_info_in iinfo = { .in = &in }; + struct ceph_inode_info *ci = ceph_inode(dir); + struct ceph_dentry_info *di = ceph_dentry(dentry); + struct timespec64 now; + struct ceph_string *pool_ns; + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(dir->i_sb); + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_vino vino = { .ino = req->r_deleg_ino, + .snap = CEPH_NOSNAP }; + + ktime_get_real_ts64(&now); + + iinfo.inline_version = CEPH_INLINE_NONE; + iinfo.change_attr = 1; + ceph_encode_timespec64(&iinfo.btime, &now); + + if (req->r_pagelist) { + iinfo.xattr_len = req->r_pagelist->length; + iinfo.xattr_data = req->r_pagelist->mapped_tail; + } else { + /* fake it */ + iinfo.xattr_len = ARRAY_SIZE(xattr_buf); + iinfo.xattr_data = xattr_buf; + memset(iinfo.xattr_data, 0, iinfo.xattr_len); + } + + in.ino = cpu_to_le64(vino.ino); + in.snapid = cpu_to_le64(CEPH_NOSNAP); + in.version = cpu_to_le64(1); // ??? + in.cap.caps = in.cap.wanted = cpu_to_le32(CEPH_CAP_ALL_FILE); + in.cap.cap_id = cpu_to_le64(1); + in.cap.realm = cpu_to_le64(ci->i_snap_realm->ino); + in.cap.flags = CEPH_CAP_FLAG_AUTH; + in.ctime = in.mtime = in.atime = iinfo.btime; + in.truncate_seq = cpu_to_le32(1); + in.truncate_size = cpu_to_le64(-1ULL); + in.xattr_version = cpu_to_le64(1); + in.uid = cpu_to_le32(from_kuid(&init_user_ns, + mapped_fsuid(req->r_mnt_idmap, + &init_user_ns))); + if (dir->i_mode & S_ISGID) { + in.gid = cpu_to_le32(from_kgid(&init_user_ns, dir->i_gid)); + + /* Directories always inherit the setgid bit. */ + if (S_ISDIR(mode)) + mode |= S_ISGID; + } else { + in.gid = cpu_to_le32(from_kgid(&init_user_ns, + mapped_fsgid(req->r_mnt_idmap, + &init_user_ns))); + } + in.mode = cpu_to_le32((u32)mode); + + in.nlink = cpu_to_le32(1); + in.max_size = cpu_to_le64(lo->stripe_unit); + + ceph_file_layout_to_legacy(lo, &in.layout); + /* lo is private, so pool_ns can't change */ + pool_ns = rcu_dereference_raw(lo->pool_ns); + if (pool_ns) { + iinfo.pool_ns_len = pool_ns->len; + iinfo.pool_ns_data = pool_ns->str; + } + + down_read(&mdsc->snap_rwsem); + ret = ceph_fill_inode(inode, NULL, &iinfo, NULL, req->r_session, + req->r_fmode, NULL); + up_read(&mdsc->snap_rwsem); + if (ret) { + doutc(cl, "failed to fill inode: %d\n", ret); + ceph_dir_clear_complete(dir); + if (!d_unhashed(dentry)) + d_drop(dentry); + discard_new_inode(inode); + } else { + struct dentry *dn; + + doutc(cl, "d_adding new inode 0x%llx to 0x%llx/%s\n", + vino.ino, ceph_ino(dir), dentry->d_name.name); + ceph_dir_clear_ordered(dir); + ceph_init_inode_acls(inode, as_ctx); + if (inode_state_read_once(inode) & I_NEW) { + /* + * If it's not I_NEW, then someone created this before + * we got here. Assume the server is aware of it at + * that point and don't worry about setting + * CEPH_I_ASYNC_CREATE. + */ + set_bit(CEPH_I_ASYNC_CREATE_BIT, + &ceph_inode(inode)->i_ceph_flags); + unlock_new_inode(inode); + } + if (d_in_lookup(dentry) || d_really_is_negative(dentry)) { + if (!d_unhashed(dentry)) + d_drop(dentry); + dn = d_splice_alias(inode, dentry); + WARN_ON_ONCE(dn && dn != dentry); + } + file->f_mode |= FMODE_CREATED; + ret = finish_open(file, dentry, ceph_open); + } + + spin_lock(&dentry->d_lock); + clear_and_wake_up_bit(CEPH_DENTRY_ASYNC_CREATE_BIT, &di->flags); + spin_unlock(&dentry->d_lock); + + return ret; +} + +/* + * Do a lookup + open with a single request. If we get a non-existent + * file or symlink, return 1 so the VFS can retry. + */ +int ceph_atomic_open(struct inode *dir, struct dentry *dentry, + struct file *file, unsigned flags, umode_t mode) +{ + struct mnt_idmap *idmap = file_mnt_idmap(file); + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(dir->i_sb); + struct ceph_client *cl = fsc->client; + struct ceph_mds_client *mdsc = fsc->mdsc; + struct ceph_mds_request *req; + struct inode *new_inode = NULL; + struct dentry *dn; + struct ceph_acl_sec_ctx as_ctx = {}; + bool try_async = ceph_test_mount_opt(fsc, ASYNC_DIROPS); + int mask; + int err; + char *path; + + doutc(cl, "%p %llx.%llx dentry %p '%pd' %s flags %d mode 0%o\n", + dir, ceph_vinop(dir), dentry, dentry, + d_unhashed(dentry) ? "unhashed" : "hashed", flags, mode); + + if (dentry->d_name.len > NAME_MAX) + return -ENAMETOOLONG; + + err = ceph_wait_on_conflict_unlink(dentry); + if (err) + return err; + /* + * Do not truncate the file, since atomic_open is called before the + * permission check. The caller will do the truncation afterward. + */ + flags &= ~O_TRUNC; + + dn = d_find_alias(dir); + if (!dn) { + try_async = false; + } else { + struct ceph_path_info path_info = {0}; + path = ceph_mdsc_build_path(mdsc, dn, &path_info, 0); + if (IS_ERR(path)) { + try_async = false; + err = 0; + } else { + int fmode = ceph_flags_to_mode(flags); + + mask = MAY_READ; + if (fmode & CEPH_FILE_MODE_WR) + mask |= MAY_WRITE; + err = ceph_mds_check_access(mdsc, path, mask); + } + ceph_mdsc_free_path_info(&path_info); + dput(dn); + + /* For none EACCES cases will let the MDS do the mds auth check */ + if (err == -EACCES) { + return err; + } else if (err < 0) { + try_async = false; + err = 0; + } + } + +retry: + if (flags & O_CREAT) { + if (ceph_quota_is_max_files_exceeded(dir)) + return -EDQUOT; + + new_inode = ceph_new_inode(dir, dentry, &mode, &as_ctx); + if (IS_ERR(new_inode)) { + err = PTR_ERR(new_inode); + goto out_ctx; + } + /* Async create can't handle more than a page of xattrs */ + if (as_ctx.pagelist && + !list_is_singular(&as_ctx.pagelist->head)) + try_async = false; + } else if (!d_in_lookup(dentry)) { + /* If it's not being looked up, it's negative */ + return -ENOENT; + } + + /* do the open */ + req = prepare_open_request(dir->i_sb, flags, mode); + if (IS_ERR(req)) { + err = PTR_ERR(req); + goto out_ctx; + } + req->r_dentry = dget(dentry); + req->r_num_caps = 2; + mask = CEPH_STAT_CAP_INODE | CEPH_CAP_AUTH_SHARED; + if (ceph_security_xattr_wanted(dir)) + mask |= CEPH_CAP_XATTR_SHARED; + req->r_args.open.mask = cpu_to_le32(mask); + req->r_parent = dir; + if (req->r_op == CEPH_MDS_OP_CREATE) + req->r_mnt_idmap = mnt_idmap_get(idmap); + ihold(dir); + if (IS_ENCRYPTED(dir)) { + set_bit(CEPH_MDS_R_FSCRYPT_FILE, &req->r_req_flags); + err = fscrypt_prepare_lookup_partial(dir, dentry); + if (err < 0) + goto out_req; + } + + if (flags & O_CREAT) { + struct ceph_file_layout lo; + + req->r_dentry_drop = CEPH_CAP_FILE_SHARED | CEPH_CAP_AUTH_EXCL | + CEPH_CAP_XATTR_EXCL; + req->r_dentry_unless = CEPH_CAP_FILE_EXCL; + + ceph_as_ctx_to_req(req, &as_ctx); + + if (try_async && (req->r_dir_caps = + try_prep_async_create(dir, dentry, &lo, + &req->r_deleg_ino))) { + struct ceph_vino vino = { .ino = req->r_deleg_ino, + .snap = CEPH_NOSNAP }; + struct ceph_dentry_info *di = ceph_dentry(dentry); + + set_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags); + req->r_args.open.flags |= cpu_to_le32(CEPH_O_EXCL); + req->r_callback = ceph_async_create_cb; + + /* Hash inode before RPC */ + new_inode = ceph_get_inode(dir->i_sb, vino, new_inode); + if (IS_ERR(new_inode)) { + err = PTR_ERR(new_inode); + new_inode = NULL; + goto out_req; + } + WARN_ON_ONCE(!(inode_state_read_once(new_inode) & I_NEW)); + + spin_lock(&dentry->d_lock); + di->flags |= CEPH_DENTRY_ASYNC_CREATE; + spin_unlock(&dentry->d_lock); + + err = ceph_mdsc_submit_request(mdsc, dir, req); + if (!err) { + err = ceph_finish_async_create(dir, new_inode, + dentry, file, + mode, req, + &as_ctx, &lo); + new_inode = NULL; + } else if (err == -EJUKEBOX) { + restore_deleg_ino(dir, req->r_deleg_ino); + ceph_mdsc_put_request(req); + discard_new_inode(new_inode); + ceph_release_acl_sec_ctx(&as_ctx); + memset(&as_ctx, 0, sizeof(as_ctx)); + new_inode = NULL; + try_async = false; + ceph_put_string(rcu_dereference_raw(lo.pool_ns)); + goto retry; + } + ceph_put_string(rcu_dereference_raw(lo.pool_ns)); + goto out_req; + } + } + + set_bit(CEPH_MDS_R_PARENT_LOCKED, &req->r_req_flags); + req->r_new_inode = new_inode; + new_inode = NULL; + err = ceph_mdsc_do_request(mdsc, (flags & O_CREAT) ? dir : NULL, req); + if (err == -ENOENT) { + dentry = ceph_handle_snapdir(req, dentry); + if (IS_ERR(dentry)) { + err = PTR_ERR(dentry); + goto out_req; + } + err = 0; + } + + if (!err && (flags & O_CREAT) && !req->r_reply_info.head->is_dentry) + err = ceph_handle_notrace_create(dir, dentry); + + if (d_in_lookup(dentry)) { + dn = ceph_finish_lookup(req, dentry, err); + if (IS_ERR(dn)) + err = PTR_ERR(dn); + } else { + /* we were given a hashed negative dentry */ + dn = NULL; + } + if (err) + goto out_req; + if (dn || d_really_is_negative(dentry) || d_is_symlink(dentry)) { + /* make vfs retry on splice, ENOENT, or symlink */ + doutc(cl, "finish_no_open on dn %p\n", dn); + err = finish_no_open(file, dn); + } else { + if (IS_ENCRYPTED(dir) && + !fscrypt_has_permitted_context(dir, d_inode(dentry))) { + pr_warn_client(cl, + "Inconsistent encryption context (parent %llx:%llx child %llx:%llx)\n", + ceph_vinop(dir), ceph_vinop(d_inode(dentry))); + goto out_req; + } + + doutc(cl, "finish_open on dn %p\n", dn); + if (req->r_op == CEPH_MDS_OP_CREATE && req->r_reply_info.has_create_ino) { + struct inode *newino = d_inode(dentry); + + cache_file_layout(dir, newino); + ceph_init_inode_acls(newino, &as_ctx); + file->f_mode |= FMODE_CREATED; + } + if ((flags & __O_REGULAR) && !d_is_reg(dentry)) { + err = -EFTYPE; + goto out_req; + } + err = finish_open(file, dentry, ceph_open); + } +out_req: + ceph_mdsc_put_request(req); + iput(new_inode); +out_ctx: + ceph_release_acl_sec_ctx(&as_ctx); + doutc(cl, "result=%d\n", err); + return err; +} + +int ceph_release(struct inode *inode, struct file *file) +{ + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + + if (S_ISDIR(inode->i_mode)) { + struct ceph_dir_file_info *dfi = file->private_data; + doutc(cl, "%p %llx.%llx dir file %p\n", inode, + ceph_vinop(inode), file); + WARN_ON(!list_empty(&dfi->file_info.rw_contexts)); + + ceph_put_fmode(ci, dfi->file_info.fmode, 1); + + if (dfi->last_readdir) + ceph_mdsc_put_request(dfi->last_readdir); + kfree(dfi->last_name); + kfree(dfi->dir_info); + kmem_cache_free(ceph_dir_file_cachep, dfi); + } else { + struct ceph_file_info *fi = file->private_data; + doutc(cl, "%p %llx.%llx regular file %p\n", inode, + ceph_vinop(inode), file); + WARN_ON(!list_empty(&fi->rw_contexts)); + + ceph_fscache_unuse_cookie(inode, file->f_mode & FMODE_WRITE); + ceph_put_fmode(ci, fi->fmode, 1); + + kmem_cache_free(ceph_file_cachep, fi); + } + + /* wake up anyone waiting for caps on this inode */ + wake_up_all(&ci->i_cap_wq); + return 0; +} + +enum { + HAVE_RETRIED = 1, + CHECK_EOF = 2, + READ_INLINE = 3, +}; + +/* + * Completely synchronous read and write methods. Direct from __user + * buffer to osd, or directly to user pages (if O_DIRECT). + * + * If the read spans object boundary, just do multiple reads. (That's not + * atomic, but good enough for now.) + * + * If we get a short result from the OSD, check against i_size; we need to + * only return a short read to the caller if we hit EOF. + */ +ssize_t __ceph_sync_read(struct inode *inode, loff_t *ki_pos, + struct iov_iter *to, int *retry_op, + u64 *last_objver) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + struct ceph_client *cl = fsc->client; + struct ceph_osd_client *osdc = &fsc->client->osdc; + ssize_t ret; + u64 off = *ki_pos; + u64 len = iov_iter_count(to); + u64 i_size = i_size_read(inode); + bool sparse = IS_ENCRYPTED(inode) || ceph_test_mount_opt(fsc, SPARSEREAD); + u64 objver = 0; + + doutc(cl, "on inode %p %llx.%llx %llx~%llx\n", inode, + ceph_vinop(inode), *ki_pos, len); + + if (ceph_inode_is_shutdown(inode)) + return -EIO; + + if (!len || !i_size) + return 0; + /* + * flush any page cache pages in this range. this + * will make concurrent normal and sync io slow, + * but it will at least behave sensibly when they are + * in sequence. + */ + ret = filemap_write_and_wait_range(inode->i_mapping, + off, off + len - 1); + if (ret < 0) + return ret; + + ret = 0; + while ((len = iov_iter_count(to)) > 0) { + struct ceph_osd_request *req; + struct page **pages; + int num_pages; + size_t page_off; + bool more; + int idx = 0; + size_t left; + struct ceph_osd_req_op *op; + u64 read_off = off; + u64 read_len = len; + int extent_cnt; + + /* determine new offset/length if encrypted */ + ceph_fscrypt_adjust_off_and_len(inode, &read_off, &read_len); + + doutc(cl, "orig %llu~%llu reading %llu~%llu", off, len, + read_off, read_len); + + req = ceph_osdc_new_request(osdc, &ci->i_layout, + ci->i_vino, read_off, &read_len, 0, 1, + sparse ? CEPH_OSD_OP_SPARSE_READ : + CEPH_OSD_OP_READ, + CEPH_OSD_FLAG_READ, + NULL, ci->i_truncate_seq, + ci->i_truncate_size, false); + if (IS_ERR(req)) { + ret = PTR_ERR(req); + break; + } + + /* adjust len downward if the request truncated the len */ + if (off + len > read_off + read_len) + len = read_off + read_len - off; + more = len < iov_iter_count(to); + + op = &req->r_ops[0]; + if (sparse) { + extent_cnt = __ceph_sparse_read_ext_count(inode, read_len); + ret = ceph_alloc_sparse_ext_map(op, extent_cnt); + if (ret) { + ceph_osdc_put_request(req); + break; + } + } + + num_pages = calc_pages_for(read_off, read_len); + page_off = offset_in_page(off); + pages = ceph_alloc_page_vector(num_pages, GFP_KERNEL); + if (IS_ERR(pages)) { + ceph_osdc_put_request(req); + ret = PTR_ERR(pages); + break; + } + + osd_req_op_extent_osd_data_pages(req, 0, pages, read_len, + offset_in_page(read_off), + false, true); + + ceph_osdc_start_request(osdc, req); + ret = ceph_osdc_wait_request(osdc, req); + + ceph_update_read_metrics(&fsc->mdsc->metric, + req->r_start_latency, + req->r_end_latency, + read_len, ret); + /* + * Only record subvolume metrics for actual bytes read. + * ret == 0 means EOF (no data), not an I/O operation. + */ + if (ret > 0) + ceph_record_subvolume_io(inode, false, + req->r_start_latency, + req->r_end_latency, + ret); + + if (ret > 0) + objver = req->r_version; + + i_size = i_size_read(inode); + doutc(cl, "%llu~%llu got %zd i_size %llu%s\n", off, len, + ret, i_size, (more ? " MORE" : "")); + + /* Fix it to go to end of extent map */ + if (sparse && ret >= 0) + ret = ceph_sparse_ext_map_end(op); + else if (ret == -ENOENT) + ret = 0; + + if (ret < 0) { + ceph_osdc_put_request(req); + if (ret == -EBLOCKLISTED) + fsc->blocklisted = true; + break; + } + + if (IS_ENCRYPTED(inode)) { + int fret; + + fret = ceph_fscrypt_decrypt_extents(inode, pages, + read_off, op->extent.sparse_ext, + op->extent.sparse_ext_cnt); + if (fret < 0) { + ret = fret; + ceph_osdc_put_request(req); + break; + } + + /* account for any partial block at the beginning */ + fret -= (off - read_off); + + /* + * Short read after big offset adjustment? + * Nothing is usable, just call it a zero + * len read. + */ + fret = max(fret, 0); + + /* account for partial block at the end */ + ret = min_t(ssize_t, fret, len); + } + + /* Short read but not EOF? Zero out the remainder. */ + if (ret < len && (off + ret < i_size)) { + int zlen = min(len - ret, i_size - off - ret); + int zoff = page_off + ret; + + doutc(cl, "zero gap %llu~%llu\n", off + ret, + off + ret + zlen); + ceph_zero_page_vector_range(zoff, zlen, pages); + ret += zlen; + } + + if (off + ret > i_size) + left = (i_size > off) ? i_size - off : 0; + else + left = ret; + + while (left > 0) { + size_t plen, copied; + + plen = min_t(size_t, left, PAGE_SIZE - page_off); + SetPageUptodate(pages[idx]); + copied = copy_page_to_iter(pages[idx++], + page_off, plen, to); + off += copied; + left -= copied; + page_off = 0; + if (copied < plen) { + ret = -EFAULT; + break; + } + } + + ceph_osdc_put_request(req); + + if (off >= i_size || !more) + break; + } + + if (ret > 0) { + if (off >= i_size) { + *retry_op = CHECK_EOF; + ret = i_size - *ki_pos; + *ki_pos = i_size; + } else { + ret = off - *ki_pos; + *ki_pos = off; + } + + if (last_objver) + *last_objver = objver; + } + doutc(cl, "result %zd retry_op %d\n", ret, *retry_op); + return ret; +} + +static ssize_t ceph_sync_read(struct kiocb *iocb, struct iov_iter *to, + int *retry_op) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct ceph_client *cl = ceph_inode_to_client(inode); + + doutc(cl, "on file %p %llx~%zx %s\n", file, iocb->ki_pos, + iov_iter_count(to), + (file->f_flags & O_DIRECT) ? "O_DIRECT" : ""); + + return __ceph_sync_read(inode, &iocb->ki_pos, to, retry_op, NULL); +} + +struct ceph_aio_request { + struct kiocb *iocb; + size_t total_len; + bool write; + bool should_dirty; + int error; + struct list_head osd_reqs; + unsigned num_reqs; + atomic_t pending_reqs; + struct timespec64 mtime; + struct ceph_cap_flush *prealloc_cf; +}; + +struct ceph_aio_work { + struct work_struct work; + struct ceph_osd_request *req; +}; + +static void ceph_aio_retry_work(struct work_struct *work); + +static void ceph_aio_complete(struct inode *inode, + struct ceph_aio_request *aio_req) +{ + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + int ret; + + if (!atomic_dec_and_test(&aio_req->pending_reqs)) + return; + + if (aio_req->iocb->ki_flags & IOCB_DIRECT) + inode_dio_end(inode); + + ret = aio_req->error; + if (!ret) + ret = aio_req->total_len; + + doutc(cl, "%p %llx.%llx rc %d\n", inode, ceph_vinop(inode), ret); + + if (ret >= 0 && aio_req->write) { + int dirty; + + loff_t endoff = aio_req->iocb->ki_pos + aio_req->total_len; + if (endoff > i_size_read(inode)) { + if (ceph_inode_set_size(inode, endoff)) + ceph_check_caps(ci, CHECK_CAPS_AUTHONLY); + } + + spin_lock(&ci->i_ceph_lock); + dirty = __ceph_mark_dirty_caps(ci, CEPH_CAP_FILE_WR, + &aio_req->prealloc_cf); + spin_unlock(&ci->i_ceph_lock); + if (dirty) + __mark_inode_dirty(inode, dirty); + + } + + ceph_put_cap_refs(ci, (aio_req->write ? CEPH_CAP_FILE_WR : + CEPH_CAP_FILE_RD)); + + aio_req->iocb->ki_complete(aio_req->iocb, ret); + + ceph_free_cap_flush(aio_req->prealloc_cf); + kfree(aio_req); +} + +static void ceph_aio_complete_req(struct ceph_osd_request *req) +{ + int rc = req->r_result; + struct inode *inode = req->r_inode; + struct ceph_aio_request *aio_req = req->r_priv; + struct ceph_osd_data *osd_data = osd_req_op_extent_osd_data(req, 0); + struct ceph_osd_req_op *op = &req->r_ops[0]; + struct ceph_client_metric *metric = &ceph_sb_to_mdsc(inode->i_sb)->metric; + unsigned int len = osd_data->bvec_pos.iter.bi_size; + bool sparse = (op->op == CEPH_OSD_OP_SPARSE_READ); + struct ceph_client *cl = ceph_inode_to_client(inode); + + BUG_ON(osd_data->type != CEPH_OSD_DATA_TYPE_BVECS); + BUG_ON(!osd_data->num_bvecs); + + doutc(cl, "req %p inode %p %llx.%llx, rc %d bytes %u\n", req, + inode, ceph_vinop(inode), rc, len); + + if (rc == -EOLDSNAPC) { + struct ceph_aio_work *aio_work; + BUG_ON(!aio_req->write); + + aio_work = kmalloc_obj(*aio_work, GFP_NOFS); + if (aio_work) { + INIT_WORK(&aio_work->work, ceph_aio_retry_work); + aio_work->req = req; + queue_work(ceph_inode_to_fs_client(inode)->inode_wq, + &aio_work->work); + return; + } + rc = -ENOMEM; + } else if (!aio_req->write) { + if (sparse && rc >= 0) + rc = ceph_sparse_ext_map_end(op); + if (rc == -ENOENT) + rc = 0; + if (rc >= 0 && len > rc) { + struct iov_iter i; + int zlen = len - rc; + + /* + * If read is satisfied by single OSD request, + * it can pass EOF. Otherwise read is within + * i_size. + */ + if (aio_req->num_reqs == 1) { + loff_t i_size = i_size_read(inode); + loff_t endoff = aio_req->iocb->ki_pos + rc; + if (endoff < i_size) + zlen = min_t(size_t, zlen, + i_size - endoff); + aio_req->total_len = rc + zlen; + } + + iov_iter_bvec(&i, ITER_DEST, osd_data->bvec_pos.bvecs, + osd_data->num_bvecs, len); + iov_iter_advance(&i, rc); + iov_iter_zero(zlen, &i); + } + } + + /* r_start_latency == 0 means the request was not submitted */ + if (req->r_start_latency) { + if (aio_req->write) { + ceph_update_write_metrics(metric, req->r_start_latency, + req->r_end_latency, len, rc); + if (rc >= 0 && len) + ceph_record_subvolume_io(inode, true, + req->r_start_latency, + req->r_end_latency, + len); + } else { + ceph_update_read_metrics(metric, req->r_start_latency, + req->r_end_latency, len, rc); + if (rc > 0) + ceph_record_subvolume_io(inode, false, + req->r_start_latency, + req->r_end_latency, + rc); + } + } + + put_bvecs(osd_data->bvec_pos.bvecs, osd_data->num_bvecs, + aio_req->should_dirty); + ceph_osdc_put_request(req); + + if (rc < 0) + cmpxchg(&aio_req->error, 0, rc); + + ceph_aio_complete(inode, aio_req); + return; +} + +static void ceph_aio_retry_work(struct work_struct *work) +{ + struct ceph_aio_work *aio_work = + container_of(work, struct ceph_aio_work, work); + struct ceph_osd_request *orig_req = aio_work->req; + struct ceph_aio_request *aio_req = orig_req->r_priv; + struct inode *inode = orig_req->r_inode; + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_snap_context *snapc; + struct ceph_osd_request *req; + int ret; + + spin_lock(&ci->i_ceph_lock); + if (__ceph_have_pending_cap_snap(ci)) { + struct ceph_cap_snap *capsnap = + list_last_entry(&ci->i_cap_snaps, + struct ceph_cap_snap, + ci_item); + snapc = ceph_get_snap_context(capsnap->context); + } else { + BUG_ON(!ci->i_head_snapc); + snapc = ceph_get_snap_context(ci->i_head_snapc); + } + spin_unlock(&ci->i_ceph_lock); + + req = ceph_osdc_alloc_request(orig_req->r_osdc, snapc, 1, + false, GFP_NOFS); + if (!req) { + ret = -ENOMEM; + req = orig_req; + goto out; + } + + req->r_flags = /* CEPH_OSD_FLAG_ORDERSNAP | */ CEPH_OSD_FLAG_WRITE; + ceph_oloc_copy(&req->r_base_oloc, &orig_req->r_base_oloc); + ceph_oid_copy(&req->r_base_oid, &orig_req->r_base_oid); + + req->r_ops[0] = orig_req->r_ops[0]; + + req->r_mtime = aio_req->mtime; + req->r_data_offset = req->r_ops[0].extent.offset; + + ret = ceph_osdc_alloc_messages(req, GFP_NOFS); + if (ret) { + ceph_osdc_put_request(req); + req = orig_req; + goto out; + } + + ceph_osdc_put_request(orig_req); + + req->r_callback = ceph_aio_complete_req; + req->r_inode = inode; + req->r_priv = aio_req; + + ceph_osdc_start_request(req->r_osdc, req); +out: + if (ret < 0) { + req->r_result = ret; + ceph_aio_complete_req(req); + } + + ceph_put_snap_context(snapc); + kfree(aio_work); +} + +static ssize_t +ceph_direct_read_write(struct kiocb *iocb, struct iov_iter *iter, + struct ceph_snap_context *snapc, + struct ceph_cap_flush **pcf) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + struct ceph_client *cl = fsc->client; + struct ceph_client_metric *metric = &fsc->mdsc->metric; + struct ceph_vino vino; + struct ceph_osd_request *req; + struct bio_vec *bvecs; + struct ceph_aio_request *aio_req = NULL; + int num_pages = 0; + int flags; + int ret = 0; + struct timespec64 mtime = current_time(inode); + size_t count = iov_iter_count(iter); + loff_t pos = iocb->ki_pos; + bool write = iov_iter_rw(iter) == WRITE; + bool should_dirty = !write && user_backed_iter(iter); + bool sparse = ceph_test_mount_opt(fsc, SPARSEREAD); + + if (write && ceph_snap(file_inode(file)) != CEPH_NOSNAP) + return -EROFS; + + doutc(cl, "sync_direct_%s on file %p %lld~%u snapc %p seq %lld\n", + (write ? "write" : "read"), file, pos, (unsigned)count, + snapc, snapc ? snapc->seq : 0); + + if (write) { + int ret2; + + ceph_fscache_invalidate(inode, true); + + ret2 = invalidate_inode_pages2_range(inode->i_mapping, + pos >> PAGE_SHIFT, + (pos + count - 1) >> PAGE_SHIFT); + if (ret2 < 0) + doutc(cl, "invalidate_inode_pages2_range returned %d\n", + ret2); + + flags = /* CEPH_OSD_FLAG_ORDERSNAP | */ CEPH_OSD_FLAG_WRITE; + } else { + flags = CEPH_OSD_FLAG_READ; + } + + while (iov_iter_count(iter) > 0) { + u64 size = iov_iter_count(iter); + ssize_t len; + struct ceph_osd_req_op *op; + int readop = sparse ? CEPH_OSD_OP_SPARSE_READ : CEPH_OSD_OP_READ; + int extent_cnt; + + if (write) + size = min_t(u64, size, fsc->mount_options->wsize); + else + size = min_t(u64, size, fsc->mount_options->rsize); + + vino = ceph_vino(inode); + req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, + vino, pos, &size, 0, + 1, + write ? CEPH_OSD_OP_WRITE : readop, + flags, snapc, + ci->i_truncate_seq, + ci->i_truncate_size, + false); + if (IS_ERR(req)) { + ret = PTR_ERR(req); + break; + } + + op = &req->r_ops[0]; + if (!write && sparse) { + extent_cnt = __ceph_sparse_read_ext_count(inode, size); + ret = ceph_alloc_sparse_ext_map(op, extent_cnt); + if (ret) { + ceph_osdc_put_request(req); + break; + } + } + + len = iter_get_bvecs_alloc(iter, size, &bvecs, &num_pages); + if (len < 0) { + ceph_osdc_put_request(req); + ret = len; + break; + } + if (len != size) + osd_req_op_extent_update(req, 0, len); + + osd_req_op_extent_osd_data_bvecs(req, 0, bvecs, num_pages, len); + + /* + * To simplify error handling, allow AIO when IO within i_size + * or IO can be satisfied by single OSD request. + */ + if (pos == iocb->ki_pos && !is_sync_kiocb(iocb) && + (len == count || pos + count <= i_size_read(inode))) { + aio_req = kzalloc_obj(*aio_req); + if (aio_req) { + aio_req->iocb = iocb; + aio_req->write = write; + aio_req->should_dirty = should_dirty; + INIT_LIST_HEAD(&aio_req->osd_reqs); + if (write) { + aio_req->mtime = mtime; + swap(aio_req->prealloc_cf, *pcf); + } + } + /* ignore error */ + } + + if (write) { + /* + * throw out any page cache pages in this range. this + * may block. + */ + truncate_inode_pages_range(inode->i_mapping, pos, + PAGE_ALIGN(pos + len) - 1); + + req->r_mtime = mtime; + } + + if (aio_req) { + aio_req->total_len += len; + aio_req->num_reqs++; + atomic_inc(&aio_req->pending_reqs); + + req->r_callback = ceph_aio_complete_req; + req->r_inode = inode; + req->r_priv = aio_req; + list_add_tail(&req->r_private_item, &aio_req->osd_reqs); + + pos += len; + continue; + } + + ceph_osdc_start_request(req->r_osdc, req); + ret = ceph_osdc_wait_request(&fsc->client->osdc, req); + + if (write) { + ceph_update_write_metrics(metric, req->r_start_latency, + req->r_end_latency, len, ret); + if (ret >= 0 && len) + ceph_record_subvolume_io(inode, true, + req->r_start_latency, + req->r_end_latency, + len); + } else { + ceph_update_read_metrics(metric, req->r_start_latency, + req->r_end_latency, len, ret); + if (ret > 0) + ceph_record_subvolume_io(inode, false, + req->r_start_latency, + req->r_end_latency, + ret); + } + + size = i_size_read(inode); + if (!write) { + if (sparse && ret >= 0) + ret = ceph_sparse_ext_map_end(op); + else if (ret == -ENOENT) + ret = 0; + + if (ret >= 0 && ret < len && pos + ret < size) { + struct iov_iter i; + int zlen = min_t(size_t, len - ret, + size - pos - ret); + + iov_iter_bvec(&i, ITER_DEST, bvecs, num_pages, len); + iov_iter_advance(&i, ret); + iov_iter_zero(zlen, &i); + ret += zlen; + } + if (ret >= 0) + len = ret; + } + + put_bvecs(bvecs, num_pages, should_dirty); + ceph_osdc_put_request(req); + if (ret < 0) + break; + + pos += len; + if (!write && pos >= size) + break; + + if (write && pos > size) { + if (ceph_inode_set_size(inode, pos)) + ceph_check_caps(ceph_inode(inode), + CHECK_CAPS_AUTHONLY); + } + } + + if (aio_req) { + LIST_HEAD(osd_reqs); + + if (aio_req->num_reqs == 0) { + kfree(aio_req); + return ret; + } + + ceph_get_cap_refs(ci, write ? CEPH_CAP_FILE_WR : + CEPH_CAP_FILE_RD); + + list_splice(&aio_req->osd_reqs, &osd_reqs); + inode_dio_begin(inode); + while (!list_empty(&osd_reqs)) { + req = list_first_entry(&osd_reqs, + struct ceph_osd_request, + r_private_item); + list_del_init(&req->r_private_item); + if (ret >= 0) + ceph_osdc_start_request(req->r_osdc, req); + if (ret < 0) { + req->r_result = ret; + ceph_aio_complete_req(req); + } + } + return -EIOCBQUEUED; + } + + if (ret != -EOLDSNAPC && pos > iocb->ki_pos) { + ret = pos - iocb->ki_pos; + iocb->ki_pos = pos; + } + return ret; +} + +/* + * Synchronous write, straight from __user pointer or user pages. + * + * If write spans object boundary, just do multiple writes. (For a + * correct atomic write, we should e.g. take write locks on all + * objects, rollback on failure, etc.) + */ +static ssize_t +ceph_sync_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos, + struct ceph_snap_context *snapc) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + struct ceph_client *cl = fsc->client; + struct ceph_osd_client *osdc = &fsc->client->osdc; + struct ceph_osd_request *req; + struct page **pages; + u64 len; + int num_pages; + int written = 0; + int ret; + bool check_caps = false; + struct timespec64 mtime = current_time(inode); + size_t count = iov_iter_count(from); + + if (ceph_snap(file_inode(file)) != CEPH_NOSNAP) + return -EROFS; + + doutc(cl, "on file %p %lld~%u snapc %p seq %lld\n", file, pos, + (unsigned)count, snapc, snapc->seq); + + ret = filemap_write_and_wait_range(inode->i_mapping, + pos, pos + count - 1); + if (ret < 0) + return ret; + + ceph_fscache_invalidate(inode, false); + + while ((len = iov_iter_count(from)) > 0) { + size_t left; + int n; + u64 write_pos = pos; + u64 write_len = len; + u64 objnum, objoff; + u32 xlen; + u64 assert_ver = 0; + bool rmw; + bool first, last; + struct iov_iter saved_iter = *from; + size_t off; + + ceph_fscrypt_adjust_off_and_len(inode, &write_pos, &write_len); + + /* clamp the length to the end of first object */ + ceph_calc_file_object_mapping(&ci->i_layout, write_pos, + write_len, &objnum, &objoff, + &xlen); + write_len = xlen; + + /* adjust len downward if it goes beyond current object */ + if (pos + len > write_pos + write_len) + len = write_pos + write_len - pos; + + /* + * If we had to adjust the length or position to align with a + * crypto block, then we must do a read/modify/write cycle. We + * use a version assertion to redrive the thing if something + * changes in between. + */ + first = pos != write_pos; + last = (pos + len) != (write_pos + write_len); + rmw = first || last; + + doutc(cl, "ino %llx %lld~%llu adjusted %lld~%llu -- %srmw\n", + ci->i_vino.ino, pos, len, write_pos, write_len, + rmw ? "" : "no "); + + /* + * The data is emplaced into the page as it would be if it were + * in an array of pagecache pages. + */ + num_pages = calc_pages_for(write_pos, write_len); + pages = ceph_alloc_page_vector(num_pages, GFP_KERNEL); + if (IS_ERR(pages)) { + ret = PTR_ERR(pages); + break; + } + + /* Do we need to preload the pages? */ + if (rmw) { + u64 first_pos = write_pos; + u64 last_pos = (write_pos + write_len) - CEPH_FSCRYPT_BLOCK_SIZE; + u64 read_len = CEPH_FSCRYPT_BLOCK_SIZE; + struct ceph_osd_req_op *op; + + /* We should only need to do this for encrypted inodes */ + WARN_ON_ONCE(!IS_ENCRYPTED(inode)); + + /* No need to do two reads if first and last blocks are same */ + if (first && last_pos == first_pos) + last = false; + + /* + * Allocate a read request for one or two extents, + * depending on how the request was aligned. + */ + req = ceph_osdc_new_request(osdc, &ci->i_layout, + ci->i_vino, first ? first_pos : last_pos, + &read_len, 0, (first && last) ? 2 : 1, + CEPH_OSD_OP_SPARSE_READ, CEPH_OSD_FLAG_READ, + NULL, ci->i_truncate_seq, + ci->i_truncate_size, false); + if (IS_ERR(req)) { + ceph_release_page_vector(pages, num_pages); + ret = PTR_ERR(req); + break; + } + + /* Something is misaligned! */ + if (read_len != CEPH_FSCRYPT_BLOCK_SIZE) { + ceph_osdc_put_request(req); + ceph_release_page_vector(pages, num_pages); + ret = -EIO; + break; + } + + /* Add extent for first block? */ + op = &req->r_ops[0]; + + if (first) { + osd_req_op_extent_osd_data_pages(req, 0, pages, + CEPH_FSCRYPT_BLOCK_SIZE, + offset_in_page(first_pos), + false, false); + /* We only expect a single extent here */ + ret = __ceph_alloc_sparse_ext_map(op, 1); + if (ret) { + ceph_osdc_put_request(req); + ceph_release_page_vector(pages, num_pages); + break; + } + } + + /* Add extent for last block */ + if (last) { + /* Init the other extent if first extent has been used */ + if (first) { + op = &req->r_ops[1]; + osd_req_op_extent_init(req, 1, + CEPH_OSD_OP_SPARSE_READ, + last_pos, CEPH_FSCRYPT_BLOCK_SIZE, + ci->i_truncate_size, + ci->i_truncate_seq); + } + + ret = __ceph_alloc_sparse_ext_map(op, 1); + if (ret) { + ceph_osdc_put_request(req); + ceph_release_page_vector(pages, num_pages); + break; + } + + osd_req_op_extent_osd_data_pages(req, first ? 1 : 0, + &pages[num_pages - 1], + CEPH_FSCRYPT_BLOCK_SIZE, + offset_in_page(last_pos), + false, false); + } + + ceph_osdc_start_request(osdc, req); + ret = ceph_osdc_wait_request(osdc, req); + + /* FIXME: length field is wrong if there are 2 extents */ + ceph_update_read_metrics(&fsc->mdsc->metric, + req->r_start_latency, + req->r_end_latency, + read_len, ret); + if (ret > 0) + ceph_record_subvolume_io(inode, false, + req->r_start_latency, + req->r_end_latency, + ret); + + /* Ok if object is not already present */ + if (ret == -ENOENT) { + /* + * If there is no object, then we can't assert + * on its version. Set it to 0, and we'll use an + * exclusive create instead. + */ + ceph_osdc_put_request(req); + ret = 0; + + /* + * zero out the soon-to-be uncopied parts of the + * first and last pages. + */ + if (first) + zero_user_segment(pages[0], 0, + offset_in_page(first_pos)); + if (last) + zero_user_segment(pages[num_pages - 1], + offset_in_page(last_pos), + PAGE_SIZE); + } else { + if (ret < 0) { + ceph_osdc_put_request(req); + ceph_release_page_vector(pages, num_pages); + break; + } + + op = &req->r_ops[0]; + if (op->extent.sparse_ext_cnt == 0) { + if (first) + zero_user_segment(pages[0], 0, + offset_in_page(first_pos)); + else + zero_user_segment(pages[num_pages - 1], + offset_in_page(last_pos), + PAGE_SIZE); + } else if (op->extent.sparse_ext_cnt != 1 || + ceph_sparse_ext_map_end(op) != + CEPH_FSCRYPT_BLOCK_SIZE) { + ret = -EIO; + ceph_osdc_put_request(req); + ceph_release_page_vector(pages, num_pages); + break; + } + + if (first && last) { + op = &req->r_ops[1]; + if (op->extent.sparse_ext_cnt == 0) { + zero_user_segment(pages[num_pages - 1], + offset_in_page(last_pos), + PAGE_SIZE); + } else if (op->extent.sparse_ext_cnt != 1 || + ceph_sparse_ext_map_end(op) != + CEPH_FSCRYPT_BLOCK_SIZE) { + ret = -EIO; + ceph_osdc_put_request(req); + ceph_release_page_vector(pages, num_pages); + break; + } + } + + /* Grab assert version. It must be non-zero. */ + assert_ver = req->r_version; + WARN_ON_ONCE(ret > 0 && assert_ver == 0); + + ceph_osdc_put_request(req); + if (first) { + ret = ceph_fscrypt_decrypt_block_inplace(inode, + pages[0], CEPH_FSCRYPT_BLOCK_SIZE, + offset_in_page(first_pos), + first_pos >> CEPH_FSCRYPT_BLOCK_SHIFT); + if (ret < 0) { + ceph_release_page_vector(pages, num_pages); + break; + } + } + if (last) { + ret = ceph_fscrypt_decrypt_block_inplace(inode, + pages[num_pages - 1], + CEPH_FSCRYPT_BLOCK_SIZE, + offset_in_page(last_pos), + last_pos >> CEPH_FSCRYPT_BLOCK_SHIFT); + if (ret < 0) { + ceph_release_page_vector(pages, num_pages); + break; + } + } + } + } + + left = len; + off = offset_in_page(pos); + for (n = 0; n < num_pages; n++) { + size_t plen = min_t(size_t, left, PAGE_SIZE - off); + + /* copy the data */ + ret = copy_page_from_iter(pages[n], off, plen, from); + if (ret != plen) { + ret = -EFAULT; + break; + } + off = 0; + left -= ret; + } + if (ret < 0) { + doutc(cl, "write failed with %d\n", ret); + ceph_release_page_vector(pages, num_pages); + break; + } + + if (IS_ENCRYPTED(inode)) { + ret = ceph_fscrypt_encrypt_pages(inode, pages, + write_pos, write_len); + if (ret < 0) { + doutc(cl, "encryption failed with %d\n", ret); + ceph_release_page_vector(pages, num_pages); + break; + } + } + + req = ceph_osdc_new_request(osdc, &ci->i_layout, + ci->i_vino, write_pos, &write_len, + rmw ? 1 : 0, rmw ? 2 : 1, + CEPH_OSD_OP_WRITE, + CEPH_OSD_FLAG_WRITE, + snapc, ci->i_truncate_seq, + ci->i_truncate_size, false); + if (IS_ERR(req)) { + ret = PTR_ERR(req); + ceph_release_page_vector(pages, num_pages); + break; + } + + doutc(cl, "write op %lld~%llu\n", write_pos, write_len); + osd_req_op_extent_osd_data_pages(req, rmw ? 1 : 0, pages, write_len, + offset_in_page(write_pos), false, + true); + req->r_inode = inode; + req->r_mtime = mtime; + + /* Set up the assertion */ + if (rmw) { + /* + * Set up the assertion. If we don't have a version + * number, then the object doesn't exist yet. Use an + * exclusive create instead of a version assertion in + * that case. + */ + if (assert_ver) { + osd_req_op_init(req, 0, CEPH_OSD_OP_ASSERT_VER, 0); + req->r_ops[0].assert_ver.ver = assert_ver; + } else { + osd_req_op_init(req, 0, CEPH_OSD_OP_CREATE, + CEPH_OSD_OP_FLAG_EXCL); + } + } + + ceph_osdc_start_request(osdc, req); + ret = ceph_osdc_wait_request(osdc, req); + + ceph_update_write_metrics(&fsc->mdsc->metric, req->r_start_latency, + req->r_end_latency, len, ret); + if (ret >= 0 && write_len) + ceph_record_subvolume_io(inode, true, + req->r_start_latency, + req->r_end_latency, + write_len); + ceph_osdc_put_request(req); + if (ret != 0) { + doutc(cl, "osd write returned %d\n", ret); + /* Version changed! Must re-do the rmw cycle */ + if ((assert_ver && (ret == -ERANGE || ret == -EOVERFLOW)) || + (!assert_ver && ret == -EEXIST)) { + /* We should only ever see this on a rmw */ + WARN_ON_ONCE(!rmw); + + /* The version should never go backward */ + WARN_ON_ONCE(ret == -EOVERFLOW); + + *from = saved_iter; + + /* FIXME: limit number of times we loop? */ + continue; + } + ceph_set_error_write(ci); + break; + } + + ceph_clear_error_write(ci); + + /* + * We successfully wrote to a range of the file. Declare + * that region of the pagecache invalid. + */ + ret = invalidate_inode_pages2_range( + inode->i_mapping, + pos >> PAGE_SHIFT, + (pos + len - 1) >> PAGE_SHIFT); + if (ret < 0) { + doutc(cl, "invalidate_inode_pages2_range returned %d\n", + ret); + ret = 0; + } + pos += len; + written += len; + doutc(cl, "written %d\n", written); + if (pos > i_size_read(inode)) { + check_caps = ceph_inode_set_size(inode, pos); + if (check_caps) + ceph_check_caps(ceph_inode(inode), + CHECK_CAPS_AUTHONLY); + } + + } + + if (ret != -EOLDSNAPC && written > 0) { + ret = written; + iocb->ki_pos = pos; + } + doutc(cl, "returning %d\n", ret); + return ret; +} + +/* + * Wrap generic_file_aio_read with checks for cap bits on the inode. + * Atomically grab references, so that those bits are not released + * back to the MDS mid-read. + * + * Hmm, the sync read case isn't actually async... should it be? + */ +static ssize_t ceph_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct file *filp = iocb->ki_filp; + struct ceph_file_info *fi = filp->private_data; + size_t len = iov_iter_count(to); + struct inode *inode = file_inode(filp); + struct ceph_inode_info *ci = ceph_inode(inode); + bool direct_lock = iocb->ki_flags & IOCB_DIRECT; + struct ceph_client *cl = ceph_inode_to_client(inode); + ssize_t ret; + int want = 0, got = 0; + int retry_op = 0, read = 0; + +again: + doutc(cl, "%llu~%u trying to get caps on %p %llx.%llx\n", + iocb->ki_pos, (unsigned)len, inode, ceph_vinop(inode)); + + if (ceph_inode_is_shutdown(inode)) + return -ESTALE; + + ret = direct_lock ? ceph_start_io_direct(inode) : + ceph_start_io_read(inode); + if (ret) + return ret; + + if (!(fi->flags & CEPH_F_SYNC) && !direct_lock) + want |= CEPH_CAP_FILE_CACHE; + if (fi->fmode & CEPH_FILE_MODE_LAZY) + want |= CEPH_CAP_FILE_LAZYIO; + + ret = ceph_get_caps(filp, CEPH_CAP_FILE_RD, want, -1, &got); + if (ret < 0) { + if (direct_lock) + ceph_end_io_direct(inode); + else + ceph_end_io_read(inode); + return ret; + } + + if ((got & (CEPH_CAP_FILE_CACHE|CEPH_CAP_FILE_LAZYIO)) == 0 || + (iocb->ki_flags & IOCB_DIRECT) || + (fi->flags & CEPH_F_SYNC)) { + + doutc(cl, "sync %p %llx.%llx %llu~%u got cap refs on %s\n", + inode, ceph_vinop(inode), iocb->ki_pos, (unsigned)len, + ceph_cap_string(got)); + + if (!ceph_has_inline_data(ci)) { + if (!retry_op && + (iocb->ki_flags & IOCB_DIRECT) && + !IS_ENCRYPTED(inode)) { + ret = ceph_direct_read_write(iocb, to, + NULL, NULL); + if (ret >= 0 && ret < len) + retry_op = CHECK_EOF; + } else { + ret = ceph_sync_read(iocb, to, &retry_op); + } + } else { + retry_op = READ_INLINE; + } + } else { + CEPH_DEFINE_RW_CONTEXT(rw_ctx, got); + doutc(cl, "async %p %llx.%llx %llu~%u got cap refs on %s\n", + inode, ceph_vinop(inode), iocb->ki_pos, (unsigned)len, + ceph_cap_string(got)); + ceph_add_rw_context(fi, &rw_ctx); + ret = generic_file_read_iter(iocb, to); + ceph_del_rw_context(fi, &rw_ctx); + } + + doutc(cl, "%p %llx.%llx dropping cap refs on %s = %d\n", + inode, ceph_vinop(inode), ceph_cap_string(got), (int)ret); + ceph_put_cap_refs(ci, got); + + if (direct_lock) + ceph_end_io_direct(inode); + else + ceph_end_io_read(inode); + + if (retry_op > HAVE_RETRIED && ret >= 0) { + int statret; + struct page *page = NULL; + loff_t i_size; + int mask = CEPH_STAT_CAP_SIZE; + if (retry_op == READ_INLINE) { + page = __page_cache_alloc(GFP_KERNEL); + if (!page) + return -ENOMEM; + + mask = CEPH_STAT_CAP_INLINE_DATA; + } + + statret = __ceph_do_getattr(inode, page, mask, !!page); + if (statret < 0) { + if (page) + __free_page(page); + if (statret == -ENODATA) { + BUG_ON(retry_op != READ_INLINE); + goto again; + } + return statret; + } + + i_size = i_size_read(inode); + if (retry_op == READ_INLINE) { + BUG_ON(ret > 0 || read > 0); + if (iocb->ki_pos < i_size && + iocb->ki_pos < PAGE_SIZE) { + loff_t end = min_t(loff_t, i_size, + iocb->ki_pos + len); + end = min_t(loff_t, end, PAGE_SIZE); + if (statret < end) + zero_user_segment(page, statret, end); + ret = copy_page_to_iter(page, + iocb->ki_pos & ~PAGE_MASK, + end - iocb->ki_pos, to); + iocb->ki_pos += ret; + read += ret; + } + if (iocb->ki_pos < i_size && read < len) { + size_t zlen = min_t(size_t, len - read, + i_size - iocb->ki_pos); + ret = iov_iter_zero(zlen, to); + iocb->ki_pos += ret; + read += ret; + } + __free_pages(page, 0); + return read; + } + + /* hit EOF or hole? */ + if (retry_op == CHECK_EOF && iocb->ki_pos < i_size && + ret < len) { + doutc(cl, "may hit hole, ppos %lld < size %lld, reading more\n", + iocb->ki_pos, i_size); + + read += ret; + len -= ret; + retry_op = HAVE_RETRIED; + goto again; + } + } + + if (ret >= 0) + ret += read; + + return ret; +} + +/* + * Wrap filemap_splice_read with checks for cap bits on the inode. + * Atomically grab references, so that those bits are not released + * back to the MDS mid-read. + */ +static ssize_t ceph_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, + size_t len, unsigned int flags) +{ + struct ceph_file_info *fi = in->private_data; + struct inode *inode = file_inode(in); + struct ceph_inode_info *ci = ceph_inode(inode); + ssize_t ret; + int want = 0, got = 0; + CEPH_DEFINE_RW_CONTEXT(rw_ctx, 0); + + dout("splice_read %p %llx.%llx %llu~%zu trying to get caps on %p\n", + inode, ceph_vinop(inode), *ppos, len, inode); + + if (ceph_inode_is_shutdown(inode)) + return -ESTALE; + + if (ceph_has_inline_data(ci) || + (fi->flags & CEPH_F_SYNC)) + return copy_splice_read(in, ppos, pipe, len, flags); + + ret = ceph_start_io_read(inode); + if (ret) + return ret; + + want = CEPH_CAP_FILE_CACHE; + if (fi->fmode & CEPH_FILE_MODE_LAZY) + want |= CEPH_CAP_FILE_LAZYIO; + + ret = ceph_get_caps(in, CEPH_CAP_FILE_RD, want, -1, &got); + if (ret < 0) + goto out_end; + + if ((got & (CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_LAZYIO)) == 0) { + dout("splice_read/sync %p %llx.%llx %llu~%zu got cap refs on %s\n", + inode, ceph_vinop(inode), *ppos, len, + ceph_cap_string(got)); + + ceph_put_cap_refs(ci, got); + ceph_end_io_read(inode); + return copy_splice_read(in, ppos, pipe, len, flags); + } + + dout("splice_read %p %llx.%llx %llu~%zu got cap refs on %s\n", + inode, ceph_vinop(inode), *ppos, len, ceph_cap_string(got)); + + rw_ctx.caps = got; + ceph_add_rw_context(fi, &rw_ctx); + ret = filemap_splice_read(in, ppos, pipe, len, flags); + ceph_del_rw_context(fi, &rw_ctx); + + dout("splice_read %p %llx.%llx dropping cap refs on %s = %zd\n", + inode, ceph_vinop(inode), ceph_cap_string(got), ret); + + ceph_put_cap_refs(ci, got); +out_end: + ceph_end_io_read(inode); + return ret; +} + +/* + * Take cap references to avoid releasing caps to MDS mid-write. + * + * If we are synchronous, and write with an old snap context, the OSD + * may return EOLDSNAPC. In that case, retry the write.. _after_ + * dropping our cap refs and allowing the pending snap to logically + * complete _before_ this write occurs. + * + * If we are near ENOSPC, write synchronously. + */ +static ssize_t ceph_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct ceph_file_info *fi = file->private_data; + struct inode *inode = file_inode(file); + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + struct ceph_client *cl = fsc->client; + struct ceph_osd_client *osdc = &fsc->client->osdc; + struct ceph_cap_flush *prealloc_cf; + ssize_t count, written = 0; + int err, want = 0, got; + bool direct_lock = false; + u32 map_flags; + u64 pool_flags; + loff_t pos; + loff_t limit = max(i_size_read(inode), fsc->max_file_size); + + if (ceph_inode_is_shutdown(inode)) + return -ESTALE; + + if (ceph_snap(inode) != CEPH_NOSNAP) + return -EROFS; + + prealloc_cf = ceph_alloc_cap_flush(); + if (!prealloc_cf) + return -ENOMEM; + + if ((iocb->ki_flags & (IOCB_DIRECT | IOCB_APPEND)) == IOCB_DIRECT) + direct_lock = true; + +retry_snap: + err = direct_lock ? ceph_start_io_direct(inode) : + ceph_start_io_write(inode); + if (err) + goto out_unlocked; + + if (iocb->ki_flags & IOCB_APPEND) { + err = ceph_do_getattr(inode, CEPH_STAT_CAP_SIZE, false); + if (err < 0) + goto out; + } + + err = generic_write_checks(iocb, from); + if (err <= 0) + goto out; + + pos = iocb->ki_pos; + if (unlikely(pos >= limit)) { + err = -EFBIG; + goto out; + } else { + iov_iter_truncate(from, limit - pos); + } + + count = iov_iter_count(from); + if (ceph_quota_is_max_bytes_exceeded(inode, pos + count)) { + err = -EDQUOT; + goto out; + } + + down_read(&osdc->lock); + map_flags = osdc->osdmap->flags; + pool_flags = ceph_pg_pool_flags(osdc->osdmap, ci->i_layout.pool_id); + up_read(&osdc->lock); + if ((map_flags & CEPH_OSDMAP_FULL) || + (pool_flags & CEPH_POOL_FLAG_FULL)) { + err = -ENOSPC; + goto out; + } + + err = file_remove_privs(file); + if (err) + goto out; + + doutc(cl, "%p %llx.%llx %llu~%zd getting caps. i_size %llu\n", + inode, ceph_vinop(inode), pos, count, + i_size_read(inode)); + if (!(fi->flags & CEPH_F_SYNC) && !direct_lock) + want |= CEPH_CAP_FILE_BUFFER; + if (fi->fmode & CEPH_FILE_MODE_LAZY) + want |= CEPH_CAP_FILE_LAZYIO; + got = 0; + err = ceph_get_caps(file, CEPH_CAP_FILE_WR, want, pos + count, &got); + if (err < 0) + goto out; + + err = file_update_time(file); + if (err) + goto out_caps; + + inode_inc_iversion_raw(inode); + + doutc(cl, "%p %llx.%llx %llu~%zd got cap refs on %s\n", + inode, ceph_vinop(inode), pos, count, ceph_cap_string(got)); + + if ((got & (CEPH_CAP_FILE_BUFFER|CEPH_CAP_FILE_LAZYIO)) == 0 || + (iocb->ki_flags & IOCB_DIRECT) || (fi->flags & CEPH_F_SYNC) || + test_bit(CEPH_I_ERROR_WRITE_BIT, &ci->i_ceph_flags)) { + struct ceph_snap_context *snapc; + struct iov_iter data; + + spin_lock(&ci->i_ceph_lock); + if (__ceph_have_pending_cap_snap(ci)) { + struct ceph_cap_snap *capsnap = + list_last_entry(&ci->i_cap_snaps, + struct ceph_cap_snap, + ci_item); + snapc = ceph_get_snap_context(capsnap->context); + } else { + BUG_ON(!ci->i_head_snapc); + snapc = ceph_get_snap_context(ci->i_head_snapc); + } + spin_unlock(&ci->i_ceph_lock); + + /* we might need to revert back to that point */ + data = *from; + if ((iocb->ki_flags & IOCB_DIRECT) && !IS_ENCRYPTED(inode)) + written = ceph_direct_read_write(iocb, &data, snapc, + &prealloc_cf); + else + written = ceph_sync_write(iocb, &data, pos, snapc); + if (direct_lock) + ceph_end_io_direct(inode); + else + ceph_end_io_write(inode); + if (written > 0) + iov_iter_advance(from, written); + ceph_put_snap_context(snapc); + } else { + /* + * No need to acquire the i_truncate_mutex. Because + * the MDS revokes Fwb caps before sending truncate + * message to us. We can't get Fwb cap while there + * are pending vmtruncate. So write and vmtruncate + * can not run at the same time + */ + written = generic_perform_write(iocb, from); + ceph_end_io_write(inode); + } + + if (written >= 0) { + int dirty; + + spin_lock(&ci->i_ceph_lock); + dirty = __ceph_mark_dirty_caps(ci, CEPH_CAP_FILE_WR, + &prealloc_cf); + spin_unlock(&ci->i_ceph_lock); + if (dirty) + __mark_inode_dirty(inode, dirty); + if (ceph_quota_is_max_bytes_approaching(inode, iocb->ki_pos)) + ceph_check_caps(ci, CHECK_CAPS_FLUSH); + } + + doutc(cl, "%p %llx.%llx %llu~%u dropping cap refs on %s\n", + inode, ceph_vinop(inode), pos, (unsigned)count, + ceph_cap_string(got)); + ceph_put_cap_refs(ci, got); + + if (written == -EOLDSNAPC) { + doutc(cl, "%p %llx.%llx %llu~%u" "got EOLDSNAPC, retrying\n", + inode, ceph_vinop(inode), pos, (unsigned)count); + goto retry_snap; + } + + if (written >= 0) { + if ((map_flags & CEPH_OSDMAP_NEARFULL) || + (pool_flags & CEPH_POOL_FLAG_NEARFULL)) + iocb->ki_flags |= IOCB_DSYNC; + written = generic_write_sync(iocb, written); + } + + goto out_unlocked; +out_caps: + ceph_put_cap_refs(ci, got); +out: + if (direct_lock) + ceph_end_io_direct(inode); + else + ceph_end_io_write(inode); +out_unlocked: + ceph_free_cap_flush(prealloc_cf); + return written ? written : err; +} + +/* + * llseek. be sure to verify file size on SEEK_END. + */ +static loff_t ceph_llseek(struct file *file, loff_t offset, int whence) +{ + if (whence == SEEK_END || whence == SEEK_DATA || whence == SEEK_HOLE) { + struct inode *inode = file_inode(file); + int ret; + + ret = ceph_do_getattr(inode, CEPH_STAT_CAP_SIZE, false); + if (ret < 0) + return ret; + } + return generic_file_llseek(file, offset, whence); +} + +static inline void ceph_zero_partial_page(struct inode *inode, + loff_t offset, size_t size) +{ + struct folio *folio; + + folio = filemap_lock_folio(inode->i_mapping, offset >> PAGE_SHIFT); + if (IS_ERR(folio)) + return; + + folio_wait_writeback(folio); + folio_zero_range(folio, offset_in_folio(folio, offset), size); + folio_unlock(folio); + folio_put(folio); +} + +static void ceph_zero_pagecache_range(struct inode *inode, loff_t offset, + loff_t length) +{ + loff_t nearly = round_up(offset, PAGE_SIZE); + if (offset < nearly) { + loff_t size = nearly - offset; + if (length < size) + size = length; + ceph_zero_partial_page(inode, offset, size); + offset += size; + length -= size; + } + if (length >= PAGE_SIZE) { + loff_t size = round_down(length, PAGE_SIZE); + truncate_pagecache_range(inode, offset, offset + size - 1); + offset += size; + length -= size; + } + if (length) + ceph_zero_partial_page(inode, offset, length); +} + +static int ceph_zero_partial_object(struct inode *inode, + loff_t offset, loff_t *length) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + struct ceph_osd_request *req; + struct ceph_snap_context *snapc; + int ret = 0; + loff_t zero = 0; + int op; + + if (ceph_inode_is_shutdown(inode)) + return -EIO; + + if (!length) { + op = offset ? CEPH_OSD_OP_DELETE : CEPH_OSD_OP_TRUNCATE; + length = &zero; + } else { + op = CEPH_OSD_OP_ZERO; + } + + spin_lock(&ci->i_ceph_lock); + if (__ceph_have_pending_cap_snap(ci)) { + struct ceph_cap_snap *capsnap = + list_last_entry(&ci->i_cap_snaps, + struct ceph_cap_snap, + ci_item); + snapc = ceph_get_snap_context(capsnap->context); + } else { + BUG_ON(!ci->i_head_snapc); + snapc = ceph_get_snap_context(ci->i_head_snapc); + } + spin_unlock(&ci->i_ceph_lock); + + req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout, + ceph_vino(inode), + offset, length, + 0, 1, op, + CEPH_OSD_FLAG_WRITE, + snapc, 0, 0, false); + if (IS_ERR(req)) { + ret = PTR_ERR(req); + goto out; + } + + req->r_mtime = inode_get_mtime(inode); + ceph_osdc_start_request(&fsc->client->osdc, req); + ret = ceph_osdc_wait_request(&fsc->client->osdc, req); + if (ret == -ENOENT) + ret = 0; + ceph_osdc_put_request(req); + +out: + ceph_put_snap_context(snapc); + return ret; +} + +static int ceph_zero_objects(struct inode *inode, loff_t offset, loff_t length) +{ + int ret = 0; + struct ceph_inode_info *ci = ceph_inode(inode); + s32 stripe_unit = ci->i_layout.stripe_unit; + s32 stripe_count = ci->i_layout.stripe_count; + s32 object_size = ci->i_layout.object_size; + u64 object_set_size = (u64) object_size * stripe_count; + u64 nearly, t; + + /* round offset up to next period boundary */ + nearly = offset + object_set_size - 1; + t = nearly; + nearly -= do_div(t, object_set_size); + + while (length && offset < nearly) { + loff_t size = length; + ret = ceph_zero_partial_object(inode, offset, &size); + if (ret < 0) + return ret; + offset += size; + length -= size; + } + while (length >= object_set_size) { + int i; + loff_t pos = offset; + for (i = 0; i < stripe_count; ++i) { + ret = ceph_zero_partial_object(inode, pos, NULL); + if (ret < 0) + return ret; + pos += stripe_unit; + } + offset += object_set_size; + length -= object_set_size; + } + while (length) { + loff_t size = length; + ret = ceph_zero_partial_object(inode, offset, &size); + if (ret < 0) + return ret; + offset += size; + length -= size; + } + return ret; +} + +static long ceph_fallocate(struct file *file, int mode, + loff_t offset, loff_t length) +{ + struct ceph_file_info *fi = file->private_data; + struct inode *inode = file_inode(file); + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_cap_flush *prealloc_cf; + struct ceph_client *cl = ceph_inode_to_client(inode); + int want, got = 0; + int dirty; + int ret = 0; + loff_t endoff = 0; + loff_t size; + + doutc(cl, "%p %llx.%llx mode %x, offset %llu length %llu\n", + inode, ceph_vinop(inode), mode, offset, length); + + if (mode != (FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)) + return -EOPNOTSUPP; + + if (!S_ISREG(inode->i_mode)) + return -EOPNOTSUPP; + + if (IS_ENCRYPTED(inode)) + return -EOPNOTSUPP; + + prealloc_cf = ceph_alloc_cap_flush(); + if (!prealloc_cf) + return -ENOMEM; + + inode_lock(inode); + + if (ceph_snap(inode) != CEPH_NOSNAP) { + ret = -EROFS; + goto unlock; + } + + size = i_size_read(inode); + + /* Are we punching a hole beyond EOF? */ + if (offset >= size) + goto unlock; + if ((offset + length) > size) + length = size - offset; + + if (fi->fmode & CEPH_FILE_MODE_LAZY) + want = CEPH_CAP_FILE_BUFFER | CEPH_CAP_FILE_LAZYIO; + else + want = CEPH_CAP_FILE_BUFFER; + + ret = ceph_get_caps(file, CEPH_CAP_FILE_WR, want, endoff, &got); + if (ret < 0) + goto unlock; + + ret = file_modified(file); + if (ret) + goto put_caps; + + filemap_invalidate_lock(inode->i_mapping); + ceph_fscache_invalidate(inode, false); + ceph_zero_pagecache_range(inode, offset, length); + ret = ceph_zero_objects(inode, offset, length); + + if (!ret) { + spin_lock(&ci->i_ceph_lock); + dirty = __ceph_mark_dirty_caps(ci, CEPH_CAP_FILE_WR, + &prealloc_cf); + spin_unlock(&ci->i_ceph_lock); + if (dirty) + __mark_inode_dirty(inode, dirty); + } + filemap_invalidate_unlock(inode->i_mapping); + +put_caps: + ceph_put_cap_refs(ci, got); +unlock: + inode_unlock(inode); + ceph_free_cap_flush(prealloc_cf); + return ret; +} + +/* + * This function tries to get FILE_WR capabilities for dst_ci and FILE_RD for + * src_ci. Two attempts are made to obtain both caps, and an error is return if + * this fails; zero is returned on success. + */ +static int get_rd_wr_caps(struct file *src_filp, int *src_got, + struct file *dst_filp, + loff_t dst_endoff, int *dst_got) +{ + int ret = 0; + bool retrying = false; + +retry_caps: + ret = ceph_get_caps(dst_filp, CEPH_CAP_FILE_WR, CEPH_CAP_FILE_BUFFER, + dst_endoff, dst_got); + if (ret < 0) + return ret; + + /* + * Since we're already holding the FILE_WR capability for the dst file, + * we would risk a deadlock by using ceph_get_caps. Thus, we'll do some + * retry dance instead to try to get both capabilities. + */ + ret = ceph_try_get_caps(file_inode(src_filp), + CEPH_CAP_FILE_RD, CEPH_CAP_FILE_SHARED, + false, src_got); + if (ret <= 0) { + /* Start by dropping dst_ci caps and getting src_ci caps */ + ceph_put_cap_refs(ceph_inode(file_inode(dst_filp)), *dst_got); + if (retrying) { + if (!ret) + /* ceph_try_get_caps masks EAGAIN */ + ret = -EAGAIN; + return ret; + } + ret = ceph_get_caps(src_filp, CEPH_CAP_FILE_RD, + CEPH_CAP_FILE_SHARED, -1, src_got); + if (ret < 0) + return ret; + /*... drop src_ci caps too, and retry */ + ceph_put_cap_refs(ceph_inode(file_inode(src_filp)), *src_got); + retrying = true; + goto retry_caps; + } + return ret; +} + +static void put_rd_wr_caps(struct ceph_inode_info *src_ci, int src_got, + struct ceph_inode_info *dst_ci, int dst_got) +{ + ceph_put_cap_refs(src_ci, src_got); + ceph_put_cap_refs(dst_ci, dst_got); +} + +/* + * This function does several size-related checks, returning an error if: + * - source file is smaller than off+len + * - destination file size is not OK (inode_newsize_ok()) + * - max bytes quotas is exceeded + */ +static int is_file_size_ok(struct inode *src_inode, struct inode *dst_inode, + loff_t src_off, loff_t dst_off, size_t len) +{ + struct ceph_client *cl = ceph_inode_to_client(src_inode); + loff_t size, endoff; + + size = i_size_read(src_inode); + /* + * Don't copy beyond source file EOF. Instead of simply setting length + * to (size - src_off), just drop to VFS default implementation, as the + * local i_size may be stale due to other clients writing to the source + * inode. + */ + if (src_off + len > size) { + doutc(cl, "Copy beyond EOF (%llu + %zu > %llu)\n", src_off, + len, size); + return -EOPNOTSUPP; + } + size = i_size_read(dst_inode); + + endoff = dst_off + len; + if (inode_newsize_ok(dst_inode, endoff)) + return -EOPNOTSUPP; + + if (ceph_quota_is_max_bytes_exceeded(dst_inode, endoff)) + return -EDQUOT; + + return 0; +} + +static struct ceph_osd_request * +ceph_alloc_copyfrom_request(struct ceph_osd_client *osdc, + u64 src_snapid, + struct ceph_object_id *src_oid, + struct ceph_object_locator *src_oloc, + struct ceph_object_id *dst_oid, + struct ceph_object_locator *dst_oloc, + u32 truncate_seq, u64 truncate_size) +{ + struct ceph_osd_request *req; + int ret; + u32 src_fadvise_flags = + CEPH_OSD_OP_FLAG_FADVISE_SEQUENTIAL | + CEPH_OSD_OP_FLAG_FADVISE_NOCACHE; + u32 dst_fadvise_flags = + CEPH_OSD_OP_FLAG_FADVISE_SEQUENTIAL | + CEPH_OSD_OP_FLAG_FADVISE_DONTNEED; + + req = ceph_osdc_alloc_request(osdc, NULL, 1, false, GFP_KERNEL); + if (!req) + return ERR_PTR(-ENOMEM); + + req->r_flags = CEPH_OSD_FLAG_WRITE; + + ceph_oloc_copy(&req->r_t.base_oloc, dst_oloc); + ceph_oid_copy(&req->r_t.base_oid, dst_oid); + + ret = osd_req_op_copy_from_init(req, src_snapid, 0, + src_oid, src_oloc, + src_fadvise_flags, + dst_fadvise_flags, + truncate_seq, + truncate_size, + CEPH_OSD_COPY_FROM_FLAG_TRUNCATE_SEQ); + if (ret) + goto out; + + ret = ceph_osdc_alloc_messages(req, GFP_KERNEL); + if (ret) + goto out; + + return req; + +out: + ceph_osdc_put_request(req); + return ERR_PTR(ret); +} + +static ssize_t ceph_do_objects_copy(struct ceph_inode_info *src_ci, u64 *src_off, + struct ceph_inode_info *dst_ci, u64 *dst_off, + struct ceph_fs_client *fsc, + size_t len, unsigned int flags) +{ + struct ceph_object_locator src_oloc, dst_oloc; + struct ceph_object_id src_oid, dst_oid; + struct ceph_osd_client *osdc; + struct ceph_osd_request *req; + ssize_t bytes = 0; + u64 src_objnum, src_objoff, dst_objnum, dst_objoff; + u32 src_objlen, dst_objlen; + u32 object_size = src_ci->i_layout.object_size; + struct ceph_client *cl = fsc->client; + int ret; + + src_oloc.pool = src_ci->i_layout.pool_id; + src_oloc.pool_ns = ceph_try_get_string(src_ci->i_layout.pool_ns); + dst_oloc.pool = dst_ci->i_layout.pool_id; + dst_oloc.pool_ns = ceph_try_get_string(dst_ci->i_layout.pool_ns); + osdc = &fsc->client->osdc; + + while (len >= object_size) { + ceph_calc_file_object_mapping(&src_ci->i_layout, *src_off, + object_size, &src_objnum, + &src_objoff, &src_objlen); + ceph_calc_file_object_mapping(&dst_ci->i_layout, *dst_off, + object_size, &dst_objnum, + &dst_objoff, &dst_objlen); + ceph_oid_init(&src_oid); + ceph_oid_printf(&src_oid, "%llx.%08llx", + src_ci->i_vino.ino, src_objnum); + ceph_oid_init(&dst_oid); + ceph_oid_printf(&dst_oid, "%llx.%08llx", + dst_ci->i_vino.ino, dst_objnum); + /* Do an object remote copy */ + req = ceph_alloc_copyfrom_request(osdc, src_ci->i_vino.snap, + &src_oid, &src_oloc, + &dst_oid, &dst_oloc, + dst_ci->i_truncate_seq, + dst_ci->i_truncate_size); + if (IS_ERR(req)) + ret = PTR_ERR(req); + else { + ceph_osdc_start_request(osdc, req); + ret = ceph_osdc_wait_request(osdc, req); + ceph_update_copyfrom_metrics(&fsc->mdsc->metric, + req->r_start_latency, + req->r_end_latency, + object_size, ret); + ceph_osdc_put_request(req); + } + if (ret) { + if (ret == -EOPNOTSUPP) { + fsc->have_copy_from2 = false; + pr_notice_client(cl, + "OSDs don't support copy-from2; disabling copy offload\n"); + } + doutc(cl, "returned %d\n", ret); + if (bytes <= 0) + bytes = ret; + goto out; + } + len -= object_size; + bytes += object_size; + *src_off += object_size; + *dst_off += object_size; + } + +out: + ceph_oloc_destroy(&src_oloc); + ceph_oloc_destroy(&dst_oloc); + return bytes; +} + +static ssize_t __ceph_copy_file_range(struct file *src_file, loff_t src_off, + struct file *dst_file, loff_t dst_off, + size_t len, unsigned int flags) +{ + struct inode *src_inode = file_inode(src_file); + struct inode *dst_inode = file_inode(dst_file); + struct ceph_inode_info *src_ci = ceph_inode(src_inode); + struct ceph_inode_info *dst_ci = ceph_inode(dst_inode); + struct ceph_cap_flush *prealloc_cf; + struct ceph_fs_client *src_fsc = ceph_inode_to_fs_client(src_inode); + struct ceph_client *cl = src_fsc->client; + loff_t size; + ssize_t ret = -EIO, bytes; + u64 src_objnum, dst_objnum, src_objoff, dst_objoff; + u32 src_objlen, dst_objlen; + int src_got = 0, dst_got = 0, err, dirty; + + if (src_inode->i_sb != dst_inode->i_sb) { + struct ceph_fs_client *dst_fsc = ceph_inode_to_fs_client(dst_inode); + + if (ceph_fsid_compare(&src_fsc->client->fsid, + &dst_fsc->client->fsid)) { + dout("Copying files across clusters: src: %pU dst: %pU\n", + &src_fsc->client->fsid, &dst_fsc->client->fsid); + return -EXDEV; + } + } + if (ceph_snap(dst_inode) != CEPH_NOSNAP) + return -EROFS; + + /* + * Some of the checks below will return -EOPNOTSUPP, which will force a + * fallback to the default VFS copy_file_range implementation. This is + * desirable in several cases (for ex, the 'len' is smaller than the + * size of the objects, or in cases where that would be more + * efficient). + */ + + if (ceph_test_mount_opt(src_fsc, NOCOPYFROM)) + return -EOPNOTSUPP; + + if (!src_fsc->have_copy_from2) + return -EOPNOTSUPP; + + /* + * Striped file layouts require that we copy partial objects, but the + * OSD copy-from operation only supports full-object copies. Limit + * this to non-striped file layouts for now. + */ + if ((src_ci->i_layout.stripe_unit != dst_ci->i_layout.stripe_unit) || + (src_ci->i_layout.stripe_count != 1) || + (dst_ci->i_layout.stripe_count != 1) || + (src_ci->i_layout.object_size != dst_ci->i_layout.object_size)) { + doutc(cl, "Invalid src/dst files layout\n"); + return -EOPNOTSUPP; + } + + /* Every encrypted inode gets its own key, so we can't offload them */ + if (IS_ENCRYPTED(src_inode) || IS_ENCRYPTED(dst_inode)) + return -EOPNOTSUPP; + + if (len < src_ci->i_layout.object_size) + return -EOPNOTSUPP; /* no remote copy will be done */ + + prealloc_cf = ceph_alloc_cap_flush(); + if (!prealloc_cf) + return -ENOMEM; + + /* Start by sync'ing the source and destination files */ + ret = file_write_and_wait_range(src_file, src_off, (src_off + len)); + if (ret < 0) { + doutc(cl, "failed to write src file (%zd)\n", ret); + goto out; + } + ret = file_write_and_wait_range(dst_file, dst_off, (dst_off + len)); + if (ret < 0) { + doutc(cl, "failed to write dst file (%zd)\n", ret); + goto out; + } + + /* + * We need FILE_WR caps for dst_ci and FILE_RD for src_ci as other + * clients may have dirty data in their caches. And OSDs know nothing + * about caps, so they can't safely do the remote object copies. + */ + err = get_rd_wr_caps(src_file, &src_got, + dst_file, (dst_off + len), &dst_got); + if (err < 0) { + doutc(cl, "get_rd_wr_caps returned %d\n", err); + ret = -EOPNOTSUPP; + goto out; + } + + ret = is_file_size_ok(src_inode, dst_inode, src_off, dst_off, len); + if (ret < 0) + goto out_caps; + + /* Drop dst file cached pages */ + ceph_fscache_invalidate(dst_inode, false); + ret = invalidate_inode_pages2_range(dst_inode->i_mapping, + dst_off >> PAGE_SHIFT, + (dst_off + len) >> PAGE_SHIFT); + if (ret < 0) { + doutc(cl, "Failed to invalidate inode pages (%zd)\n", + ret); + ret = 0; /* XXX */ + } + ceph_calc_file_object_mapping(&src_ci->i_layout, src_off, + src_ci->i_layout.object_size, + &src_objnum, &src_objoff, &src_objlen); + ceph_calc_file_object_mapping(&dst_ci->i_layout, dst_off, + dst_ci->i_layout.object_size, + &dst_objnum, &dst_objoff, &dst_objlen); + /* object-level offsets need to the same */ + if (src_objoff != dst_objoff) { + ret = -EOPNOTSUPP; + goto out_caps; + } + + /* + * Do a manual copy if the object offset isn't object aligned. + * 'src_objlen' contains the bytes left until the end of the object, + * starting at the src_off + */ + if (src_objoff) { + doutc(cl, "Initial partial copy of %u bytes\n", src_objlen); + + /* + * we need to temporarily drop all caps as we'll be calling + * {read,write}_iter, which will get caps again. + */ + put_rd_wr_caps(src_ci, src_got, dst_ci, dst_got); + ret = splice_file_range(src_file, &src_off, dst_file, &dst_off, + src_objlen); + /* Abort on short copies or on error */ + if (ret < (long)src_objlen) { + doutc(cl, "Failed partial copy (%zd)\n", ret); + goto out; + } + len -= ret; + err = get_rd_wr_caps(src_file, &src_got, + dst_file, (dst_off + len), &dst_got); + if (err < 0) + goto out; + err = is_file_size_ok(src_inode, dst_inode, + src_off, dst_off, len); + if (err < 0) + goto out_caps; + } + + size = i_size_read(dst_inode); + bytes = ceph_do_objects_copy(src_ci, &src_off, dst_ci, &dst_off, + src_fsc, len, flags); + if (bytes <= 0) { + if (!ret) + ret = bytes; + goto out_caps; + } + doutc(cl, "Copied %zu bytes out of %zu\n", bytes, len); + len -= bytes; + ret += bytes; + + file_update_time(dst_file); + inode_inc_iversion_raw(dst_inode); + + if (dst_off > size) { + /* Let the MDS know about dst file size change */ + if (ceph_inode_set_size(dst_inode, dst_off) || + ceph_quota_is_max_bytes_approaching(dst_inode, dst_off)) + ceph_check_caps(dst_ci, CHECK_CAPS_AUTHONLY | CHECK_CAPS_FLUSH); + } + /* Mark Fw dirty */ + spin_lock(&dst_ci->i_ceph_lock); + dirty = __ceph_mark_dirty_caps(dst_ci, CEPH_CAP_FILE_WR, &prealloc_cf); + spin_unlock(&dst_ci->i_ceph_lock); + if (dirty) + __mark_inode_dirty(dst_inode, dirty); + +out_caps: + put_rd_wr_caps(src_ci, src_got, dst_ci, dst_got); + + /* + * Do the final manual copy if we still have some bytes left, unless + * there were errors in remote object copies (len >= object_size). + */ + if (len && (len < src_ci->i_layout.object_size)) { + doutc(cl, "Final partial copy of %zu bytes\n", len); + bytes = splice_file_range(src_file, &src_off, dst_file, + &dst_off, len); + if (bytes > 0) + ret += bytes; + else + doutc(cl, "Failed partial copy (%zd)\n", bytes); + } + +out: + ceph_free_cap_flush(prealloc_cf); + + return ret; +} + +static ssize_t ceph_copy_file_range(struct file *src_file, loff_t src_off, + struct file *dst_file, loff_t dst_off, + size_t len, unsigned int flags) +{ + ssize_t ret; + + ret = __ceph_copy_file_range(src_file, src_off, dst_file, dst_off, + len, flags); + + if (ret == -EOPNOTSUPP || ret == -EXDEV) + ret = splice_copy_file_range(src_file, src_off, dst_file, + dst_off, len); + return ret; +} + +const struct file_operations ceph_file_fops = { + .open = ceph_open, + .release = ceph_release, + .llseek = ceph_llseek, + .read_iter = ceph_read_iter, + .write_iter = ceph_write_iter, + .mmap_prepare = ceph_mmap_prepare, + .fsync = ceph_fsync, + .lock = ceph_lock, + .flock = ceph_flock, + .splice_read = ceph_splice_read, + .splice_write = iter_file_splice_write, + .unlocked_ioctl = ceph_ioctl, + .compat_ioctl = compat_ptr_ioctl, + .fallocate = ceph_fallocate, + .copy_file_range = ceph_copy_file_range, +}; diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c new file mode 100644 index 000000000..61d7c0b81 --- /dev/null +++ b/fs/ceph/inode.c @@ -0,0 +1,3258 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "mds_client.h" +#include "cache.h" +#include "crypto.h" +#include + +/* + * Ceph inode operations + * + * Implement basic inode helpers (get, alloc) and inode ops (getattr, + * setattr, etc.), xattr helpers, and helpers for assimilating + * metadata returned by the MDS into our cache. + * + * Also define helpers for doing asynchronous writeback, invalidation, + * and truncation for the benefit of those who can't afford to block + * (typically because they are in the message handler path). + */ + +static const struct inode_operations ceph_symlink_iops; +static const struct inode_operations ceph_encrypted_symlink_iops; + +static void ceph_inode_work(struct work_struct *work); + +/* + * find or create an inode, given the ceph ino number + */ +static int ceph_set_ino_cb(struct inode *inode, void *data) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb); + + ci->i_vino = *(struct ceph_vino *)data; + inode->i_ino = ceph_vino_to_ino_t(ci->i_vino); + inode_set_iversion_raw(inode, 0); + percpu_counter_inc(&mdsc->metric.total_inodes); + + return 0; +} + +/* + * Check if the parent inode matches the vino from directory reply info + */ +static inline bool ceph_vino_matches_parent(struct inode *parent, + struct ceph_vino vino) +{ + return ceph_ino(parent) == vino.ino && ceph_snap(parent) == vino.snap; +} + +/* + * Validate that the directory inode referenced by @req->r_parent matches the + * inode number and snapshot id contained in the reply's directory record. If + * they do not match – which can theoretically happen if the parent dentry was + * moved between the time the request was issued and the reply arrived – fall + * back to looking up the correct inode in the inode cache. + * + * A reference is *always* returned. Callers that receive a different inode + * than the original @parent are responsible for dropping the extra reference + * once the reply has been processed. + */ +static struct inode *ceph_get_reply_dir(struct super_block *sb, + struct inode *parent, + struct ceph_mds_reply_info_parsed *rinfo) +{ + struct ceph_vino vino; + + if (unlikely(!rinfo->diri.in)) + return parent; /* nothing to compare against */ + + /* If we didn't have a cached parent inode to begin with, just bail out. */ + if (!parent) + return NULL; + + vino.ino = le64_to_cpu(rinfo->diri.in->ino); + vino.snap = le64_to_cpu(rinfo->diri.in->snapid); + + if (likely(ceph_vino_matches_parent(parent, vino))) + return parent; /* matches – use the original reference */ + + /* Mismatch – this should be rare. Emit a WARN and obtain the correct inode. */ + WARN_ONCE(1, "ceph: reply dir mismatch (parent valid %llx.%llx reply %llx.%llx)\n", + ceph_ino(parent), ceph_snap(parent), vino.ino, vino.snap); + + return ceph_get_inode(sb, vino, NULL); +} + +/** + * ceph_new_inode - allocate a new inode in advance of an expected create + * @dir: parent directory for new inode + * @dentry: dentry that may eventually point to new inode + * @mode: mode of new inode + * @as_ctx: pointer to inherited security context + * + * Allocate a new inode in advance of an operation to create a new inode. + * This allocates the inode and sets up the acl_sec_ctx with appropriate + * info for the new inode. + * + * Returns a pointer to the new inode or an ERR_PTR. + */ +struct inode *ceph_new_inode(struct inode *dir, struct dentry *dentry, + umode_t *mode, struct ceph_acl_sec_ctx *as_ctx) +{ + int err; + struct inode *inode; + + inode = new_inode(dir->i_sb); + if (!inode) + return ERR_PTR(-ENOMEM); + + inode->i_blkbits = CEPH_FSCRYPT_BLOCK_SHIFT; + + if (!S_ISLNK(*mode)) { + err = ceph_pre_init_acls(dir, mode, as_ctx); + if (err < 0) + goto out_err; + } + + inode_state_assign_raw(inode, 0); + inode->i_mode = *mode; + + err = ceph_security_init_secctx(dentry, *mode, as_ctx); + if (err < 0) + goto out_err; + + /* + * We'll skip setting fscrypt context for snapshots, leaving that for + * the handle_reply(). + */ + if (ceph_snap(dir) != CEPH_SNAPDIR) { + err = ceph_fscrypt_prepare_context(dir, inode, as_ctx); + if (err) + goto out_err; + } + + return inode; +out_err: + iput(inode); + return ERR_PTR(err); +} + +void ceph_as_ctx_to_req(struct ceph_mds_request *req, + struct ceph_acl_sec_ctx *as_ctx) +{ + if (as_ctx->pagelist) { + req->r_pagelist = as_ctx->pagelist; + as_ctx->pagelist = NULL; + } + ceph_fscrypt_as_ctx_to_req(req, as_ctx); +} + +/** + * ceph_get_inode - find or create/hash a new inode + * @sb: superblock to search and allocate in + * @vino: vino to search for + * @newino: optional new inode to insert if one isn't found (may be NULL) + * + * Search for or insert a new inode into the hash for the given vino, and + * return a reference to it. If new is non-NULL, its reference is consumed. + */ +struct inode *ceph_get_inode(struct super_block *sb, struct ceph_vino vino, + struct inode *newino) +{ + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(sb); + struct ceph_client *cl = mdsc->fsc->client; + struct inode *inode; + + if (ceph_vino_is_reserved(vino)) + return ERR_PTR(-EREMOTEIO); + + if (newino) { + inode = inode_insert5(newino, (unsigned long)vino.ino, + ceph_ino_compare, ceph_set_ino_cb, &vino); + if (inode != newino) + iput(newino); + } else { + inode = iget5_locked(sb, (unsigned long)vino.ino, + ceph_ino_compare, ceph_set_ino_cb, &vino); + } + + if (!inode) { + doutc(cl, "no inode found for %llx.%llx\n", vino.ino, vino.snap); + return ERR_PTR(-ENOMEM); + } + + doutc(cl, "on %llx=%llx.%llx got %p new %d\n", + ceph_present_inode(inode), ceph_vinop(inode), inode, + !!(inode_state_read_once(inode) & I_NEW)); + return inode; +} + +/* + * get/construct snapdir inode for a given directory + */ +struct inode *ceph_get_snapdir(struct inode *parent) +{ + struct ceph_client *cl = ceph_inode_to_client(parent); + struct ceph_vino vino = { + .ino = ceph_ino(parent), + .snap = CEPH_SNAPDIR, + }; + struct inode *inode = ceph_get_inode(parent->i_sb, vino, NULL); + struct ceph_inode_info *ci = ceph_inode(inode); + int ret = -ENOTDIR; + + if (IS_ERR(inode)) + return inode; + + if (!S_ISDIR(parent->i_mode)) { + pr_warn_once_client(cl, "bad snapdir parent type (mode=0%o)\n", + parent->i_mode); + goto err; + } + + if (!(inode_state_read_once(inode) & I_NEW) && !S_ISDIR(inode->i_mode)) { + pr_warn_once_client(cl, "bad snapdir inode type (mode=0%o)\n", + inode->i_mode); + goto err; + } + + inode->i_mode = parent->i_mode; + inode->i_uid = parent->i_uid; + inode->i_gid = parent->i_gid; + inode_set_mtime_to_ts(inode, inode_get_mtime(parent)); + inode_set_ctime_to_ts(inode, inode_get_ctime(parent)); + inode_set_atime_to_ts(inode, inode_get_atime(parent)); + ci->i_rbytes = 0; + ci->i_btime = ceph_inode(parent)->i_btime; + +#ifdef CONFIG_FS_ENCRYPTION + /* if encrypted, just borrow fscrypt_auth from parent */ + if (IS_ENCRYPTED(parent)) { + struct ceph_inode_info *pci = ceph_inode(parent); + + ci->fscrypt_auth = kmemdup(pci->fscrypt_auth, + pci->fscrypt_auth_len, + GFP_KERNEL); + if (ci->fscrypt_auth) { + inode->i_flags |= S_ENCRYPTED; + ci->fscrypt_auth_len = pci->fscrypt_auth_len; + } else { + doutc(cl, "Failed to alloc snapdir fscrypt_auth\n"); + ret = -ENOMEM; + goto err; + } + } +#endif + if (inode_state_read_once(inode) & I_NEW) { + inode->i_op = &ceph_snapdir_iops; + inode->i_fop = &ceph_snapdir_fops; + ci->i_snap_caps = CEPH_CAP_PIN; /* so we can open */ + unlock_new_inode(inode); + } + + return inode; +err: + if ((inode_state_read_once(inode) & I_NEW)) + discard_new_inode(inode); + else + iput(inode); + return ERR_PTR(ret); +} + +const struct inode_operations ceph_file_iops = { + .permission = ceph_permission, + .setattr = ceph_setattr, + .getattr = ceph_getattr, + .listxattr = ceph_listxattr, + .get_inode_acl = ceph_get_acl, + .set_acl = ceph_set_acl, +}; + + +/* + * We use a 'frag tree' to keep track of the MDS's directory fragments + * for a given inode (usually there is just a single fragment). We + * need to know when a child frag is delegated to a new MDS, or when + * it is flagged as replicated, so we can direct our requests + * accordingly. + */ + +/* + * find/create a frag in the tree + */ +static struct ceph_inode_frag *__get_or_create_frag(struct ceph_inode_info *ci, + u32 f) +{ + struct inode *inode = &ci->netfs.inode; + struct ceph_client *cl = ceph_inode_to_client(inode); + struct rb_node **p; + struct rb_node *parent = NULL; + struct ceph_inode_frag *frag; + int c; + + p = &ci->i_fragtree.rb_node; + while (*p) { + parent = *p; + frag = rb_entry(parent, struct ceph_inode_frag, node); + c = ceph_frag_compare(f, frag->frag); + if (c < 0) + p = &(*p)->rb_left; + else if (c > 0) + p = &(*p)->rb_right; + else + return frag; + } + + frag = kmalloc_obj(*frag, GFP_NOFS); + if (!frag) + return ERR_PTR(-ENOMEM); + + frag->frag = f; + frag->split_by = 0; + frag->mds = -1; + frag->ndist = 0; + + rb_link_node(&frag->node, parent, p); + rb_insert_color(&frag->node, &ci->i_fragtree); + + doutc(cl, "added %p %llx.%llx frag %x\n", inode, ceph_vinop(inode), f); + return frag; +} + +/* + * find a specific frag @f + */ +struct ceph_inode_frag *__ceph_find_frag(struct ceph_inode_info *ci, u32 f) +{ + struct rb_node *n = ci->i_fragtree.rb_node; + + while (n) { + struct ceph_inode_frag *frag = + rb_entry(n, struct ceph_inode_frag, node); + int c = ceph_frag_compare(f, frag->frag); + if (c < 0) + n = n->rb_left; + else if (c > 0) + n = n->rb_right; + else + return frag; + } + return NULL; +} + +/* + * Choose frag containing the given value @v. If @pfrag is + * specified, copy the frag delegation info to the caller if + * it is present. + */ +static u32 __ceph_choose_frag(struct ceph_inode_info *ci, u32 v, + struct ceph_inode_frag *pfrag, int *found) +{ + struct ceph_client *cl = ceph_inode_to_client(&ci->netfs.inode); + u32 t = ceph_frag_make(0, 0); + struct ceph_inode_frag *frag; + unsigned nway, i; + u32 n; + + if (found) + *found = 0; + + while (1) { + WARN_ON(!ceph_frag_contains_value(t, v)); + frag = __ceph_find_frag(ci, t); + if (!frag) + break; /* t is a leaf */ + if (frag->split_by == 0) { + if (pfrag) + memcpy(pfrag, frag, sizeof(*pfrag)); + if (found) + *found = 1; + break; + } + + /* choose child */ + nway = 1 << frag->split_by; + doutc(cl, "frag(%x) %x splits by %d (%d ways)\n", v, t, + frag->split_by, nway); + for (i = 0; i < nway; i++) { + n = ceph_frag_make_child(t, frag->split_by, i); + if (ceph_frag_contains_value(n, v)) { + t = n; + break; + } + } + BUG_ON(i == nway); + } + doutc(cl, "frag(%x) = %x\n", v, t); + + return t; +} + +u32 ceph_choose_frag(struct ceph_inode_info *ci, u32 v, + struct ceph_inode_frag *pfrag, int *found) +{ + u32 ret; + mutex_lock(&ci->i_fragtree_mutex); + ret = __ceph_choose_frag(ci, v, pfrag, found); + mutex_unlock(&ci->i_fragtree_mutex); + return ret; +} + +/* + * Process dirfrag (delegation) info from the mds. Include leaf + * fragment in tree ONLY if ndist > 0. Otherwise, only + * branches/splits are included in i_fragtree) + */ +static int ceph_fill_dirfrag(struct inode *inode, + struct ceph_mds_reply_dirfrag *dirinfo) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_inode_frag *frag; + u32 id = le32_to_cpu(dirinfo->frag); + int mds = le32_to_cpu(dirinfo->auth); + int ndist = le32_to_cpu(dirinfo->ndist); + int diri_auth = -1; + int i; + int err = 0; + + spin_lock(&ci->i_ceph_lock); + if (ci->i_auth_cap) + diri_auth = ci->i_auth_cap->mds; + spin_unlock(&ci->i_ceph_lock); + + if (mds == -1) /* CDIR_AUTH_PARENT */ + mds = diri_auth; + + mutex_lock(&ci->i_fragtree_mutex); + if (ndist == 0 && mds == diri_auth) { + /* no delegation info needed. */ + frag = __ceph_find_frag(ci, id); + if (!frag) + goto out; + if (frag->split_by == 0) { + /* tree leaf, remove */ + doutc(cl, "removed %p %llx.%llx frag %x (no ref)\n", + inode, ceph_vinop(inode), id); + rb_erase(&frag->node, &ci->i_fragtree); + kfree(frag); + } else { + /* tree branch, keep and clear */ + doutc(cl, "cleared %p %llx.%llx frag %x referral\n", + inode, ceph_vinop(inode), id); + frag->mds = -1; + frag->ndist = 0; + } + goto out; + } + + + /* find/add this frag to store mds delegation info */ + frag = __get_or_create_frag(ci, id); + if (IS_ERR(frag)) { + /* this is not the end of the world; we can continue + with bad/inaccurate delegation info */ + pr_err_client(cl, "ENOMEM on mds ref %p %llx.%llx fg %x\n", + inode, ceph_vinop(inode), + le32_to_cpu(dirinfo->frag)); + err = -ENOMEM; + goto out; + } + + frag->mds = mds; + frag->ndist = min_t(u32, ndist, CEPH_MAX_DIRFRAG_REP); + for (i = 0; i < frag->ndist; i++) + frag->dist[i] = le32_to_cpu(dirinfo->dist[i]); + doutc(cl, "%p %llx.%llx frag %x ndist=%d\n", inode, + ceph_vinop(inode), frag->frag, frag->ndist); + +out: + mutex_unlock(&ci->i_fragtree_mutex); + return err; +} + +static int frag_tree_split_cmp(const void *l, const void *r) +{ + struct ceph_frag_tree_split *ls = (struct ceph_frag_tree_split*)l; + struct ceph_frag_tree_split *rs = (struct ceph_frag_tree_split*)r; + return ceph_frag_compare(le32_to_cpu(ls->frag), + le32_to_cpu(rs->frag)); +} + +static bool is_frag_child(u32 f, struct ceph_inode_frag *frag) +{ + if (!frag) + return f == ceph_frag_make(0, 0); + if (ceph_frag_bits(f) != ceph_frag_bits(frag->frag) + frag->split_by) + return false; + return ceph_frag_contains_value(frag->frag, ceph_frag_value(f)); +} + +static int ceph_fill_fragtree(struct inode *inode, + struct ceph_frag_tree_head *fragtree, + struct ceph_mds_reply_dirfrag *dirinfo) +{ + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_inode_frag *frag, *prev_frag = NULL; + struct rb_node *rb_node; + unsigned i, split_by, nsplits; + u32 id; + bool update = false; + + mutex_lock(&ci->i_fragtree_mutex); + nsplits = le32_to_cpu(fragtree->nsplits); + if (nsplits != ci->i_fragtree_nsplits) { + update = true; + } else if (nsplits) { + i = get_random_u32_below(nsplits); + id = le32_to_cpu(fragtree->splits[i].frag); + if (!__ceph_find_frag(ci, id)) + update = true; + } else if (!RB_EMPTY_ROOT(&ci->i_fragtree)) { + rb_node = rb_first(&ci->i_fragtree); + frag = rb_entry(rb_node, struct ceph_inode_frag, node); + if (frag->frag != ceph_frag_make(0, 0) || rb_next(rb_node)) + update = true; + } + if (!update && dirinfo) { + id = le32_to_cpu(dirinfo->frag); + if (id != __ceph_choose_frag(ci, id, NULL, NULL)) + update = true; + } + if (!update) + goto out_unlock; + + if (nsplits > 1) { + sort(fragtree->splits, nsplits, sizeof(fragtree->splits[0]), + frag_tree_split_cmp, NULL); + } + + doutc(cl, "%p %llx.%llx\n", inode, ceph_vinop(inode)); + rb_node = rb_first(&ci->i_fragtree); + for (i = 0; i < nsplits; i++) { + id = le32_to_cpu(fragtree->splits[i].frag); + split_by = le32_to_cpu(fragtree->splits[i].by); + if (split_by == 0 || ceph_frag_bits(id) + split_by > 24) { + pr_err_client(cl, "%p %llx.%llx invalid split %d/%u, " + "frag %x split by %d\n", inode, + ceph_vinop(inode), i, nsplits, id, split_by); + continue; + } + frag = NULL; + while (rb_node) { + frag = rb_entry(rb_node, struct ceph_inode_frag, node); + if (ceph_frag_compare(frag->frag, id) >= 0) { + if (frag->frag != id) + frag = NULL; + else + rb_node = rb_next(rb_node); + break; + } + rb_node = rb_next(rb_node); + /* delete stale split/leaf node */ + if (frag->split_by > 0 || + !is_frag_child(frag->frag, prev_frag)) { + rb_erase(&frag->node, &ci->i_fragtree); + if (frag->split_by > 0) + ci->i_fragtree_nsplits--; + kfree(frag); + } + frag = NULL; + } + if (!frag) { + frag = __get_or_create_frag(ci, id); + if (IS_ERR(frag)) + continue; + } + if (frag->split_by == 0) + ci->i_fragtree_nsplits++; + frag->split_by = split_by; + doutc(cl, " frag %x split by %d\n", frag->frag, frag->split_by); + prev_frag = frag; + } + while (rb_node) { + frag = rb_entry(rb_node, struct ceph_inode_frag, node); + rb_node = rb_next(rb_node); + /* delete stale split/leaf node */ + if (frag->split_by > 0 || + !is_frag_child(frag->frag, prev_frag)) { + rb_erase(&frag->node, &ci->i_fragtree); + if (frag->split_by > 0) + ci->i_fragtree_nsplits--; + kfree(frag); + } + } +out_unlock: + mutex_unlock(&ci->i_fragtree_mutex); + return 0; +} + +/* + * initialize a newly allocated inode. + */ +struct inode *ceph_alloc_inode(struct super_block *sb) +{ + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(sb); + struct ceph_inode_info *ci; + int i; + + ci = alloc_inode_sb(sb, ceph_inode_cachep, GFP_NOFS); + if (!ci) + return NULL; + + doutc(fsc->client, "%p\n", &ci->netfs.inode); + + /* Set parameters for the netfs library */ + netfs_inode_init(&ci->netfs, &ceph_netfs_ops, false); + + spin_lock_init(&ci->i_ceph_lock); + + ci->i_version = 0; + ci->i_inline_version = 0; + ci->i_time_warp_seq = 0; + ci->i_ceph_flags = 0; + atomic64_set(&ci->i_ordered_count, 1); + atomic64_set(&ci->i_release_count, 1); + atomic64_set(&ci->i_complete_seq[0], 0); + atomic64_set(&ci->i_complete_seq[1], 0); + ci->i_symlink = NULL; + + ci->i_max_bytes = 0; + ci->i_max_files = 0; + ci->i_subvolume_id = CEPH_SUBVOLUME_ID_NONE; + + memset(&ci->i_dir_layout, 0, sizeof(ci->i_dir_layout)); + memset(&ci->i_cached_layout, 0, sizeof(ci->i_cached_layout)); + RCU_INIT_POINTER(ci->i_layout.pool_ns, NULL); + + ci->i_fragtree = RB_ROOT; + mutex_init(&ci->i_fragtree_mutex); + + ci->i_xattrs.blob = NULL; + ci->i_xattrs.prealloc_blob = NULL; + ci->i_xattrs.dirty = false; + ci->i_xattrs.index = RB_ROOT; + ci->i_xattrs.count = 0; + ci->i_xattrs.names_size = 0; + ci->i_xattrs.vals_size = 0; + ci->i_xattrs.version = 0; + ci->i_xattrs.index_version = 0; + + ci->i_caps = RB_ROOT; + ci->i_auth_cap = NULL; + ci->i_dirty_caps = 0; + ci->i_flushing_caps = 0; + INIT_LIST_HEAD(&ci->i_dirty_item); + INIT_LIST_HEAD(&ci->i_flushing_item); + ci->i_prealloc_cap_flush = NULL; + INIT_LIST_HEAD(&ci->i_cap_flush_list); + init_waitqueue_head(&ci->i_cap_wq); + ci->i_hold_caps_max = 0; + INIT_LIST_HEAD(&ci->i_cap_delay_list); + INIT_LIST_HEAD(&ci->i_cap_snaps); + ci->i_head_snapc = NULL; + ci->i_snap_caps = 0; + ci->i_last_cap_flush_ack = 0; + + ci->i_last_rd = ci->i_last_wr = jiffies - 3600 * HZ; + for (i = 0; i < CEPH_FILE_MODE_BITS; i++) + ci->i_nr_by_mode[i] = 0; + + mutex_init(&ci->i_truncate_mutex); + ci->i_truncate_seq = 0; + ci->i_truncate_size = 0; + ci->i_truncate_pending = 0; + ci->i_truncate_pagecache_size = 0; + + ci->i_max_size = 0; + ci->i_reported_size = 0; + ci->i_wanted_max_size = 0; + ci->i_requested_max_size = 0; + + ci->i_pin_ref = 0; + ci->i_rd_ref = 0; + ci->i_rdcache_ref = 0; + ci->i_wr_ref = 0; + ci->i_wb_ref = 0; + ci->i_fx_ref = 0; + ci->i_wrbuffer_ref = 0; + ci->i_wrbuffer_ref_head = 0; + atomic_set(&ci->i_filelock_ref, 0); + atomic_set(&ci->i_shared_gen, 1); + ci->i_rdcache_gen = 0; + ci->i_rdcache_revoking = 0; + + INIT_LIST_HEAD(&ci->i_unsafe_dirops); + INIT_LIST_HEAD(&ci->i_unsafe_iops); + spin_lock_init(&ci->i_unsafe_lock); + + ci->i_snap_realm = NULL; + INIT_LIST_HEAD(&ci->i_snap_realm_item); + INIT_LIST_HEAD(&ci->i_snap_flush_item); + + INIT_WORK(&ci->i_work, ceph_inode_work); + ci->i_work_mask = 0; + memset(&ci->i_btime, '\0', sizeof(ci->i_btime)); +#ifdef CONFIG_FS_ENCRYPTION + ci->i_crypt_info = NULL; + ci->fscrypt_auth = NULL; + ci->fscrypt_auth_len = 0; +#endif + return &ci->netfs.inode; +} + +void ceph_free_inode(struct inode *inode) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + + kfree(ci->i_symlink); +#ifdef CONFIG_FS_ENCRYPTION + kfree(ci->fscrypt_auth); +#endif + fscrypt_free_inode(inode); + kmem_cache_free(ceph_inode_cachep, ci); +} + +void ceph_evict_inode(struct inode *inode) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb); + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_inode_frag *frag; + struct rb_node *n; + + doutc(cl, "%p ino %llx.%llx\n", inode, ceph_vinop(inode)); + + percpu_counter_dec(&mdsc->metric.total_inodes); + + ci->i_subvolume_id = CEPH_SUBVOLUME_ID_NONE; + + netfs_wait_for_outstanding_io(inode); + truncate_inode_pages_final(&inode->i_data); + if (inode_state_read_once(inode) & I_PINNING_NETFS_WB) + ceph_fscache_unuse_cookie(inode, true); + clear_inode(inode); + + ceph_fscache_unregister_inode_cookie(ci); + fscrypt_put_encryption_info(inode); + + __ceph_remove_caps(ci); + + if (__ceph_has_quota(ci, QUOTA_GET_ANY)) + ceph_adjust_quota_realms_count(inode, false); + + /* + * we may still have a snap_realm reference if there are stray + * caps in i_snap_caps. + */ + if (ci->i_snap_realm) { + if (ceph_snap(inode) == CEPH_NOSNAP) { + doutc(cl, " dropping residual ref to snap realm %p\n", + ci->i_snap_realm); + ceph_change_snap_realm(inode, NULL); + } else { + ceph_put_snapid_map(mdsc, ci->i_snapid_map); + ci->i_snap_realm = NULL; + } + } + + while ((n = rb_first(&ci->i_fragtree)) != NULL) { + frag = rb_entry(n, struct ceph_inode_frag, node); + rb_erase(n, &ci->i_fragtree); + kfree(frag); + } + ci->i_fragtree_nsplits = 0; + + __ceph_destroy_xattrs(ci); + if (ci->i_xattrs.blob) + ceph_buffer_put(ci->i_xattrs.blob); + if (ci->i_xattrs.prealloc_blob) + ceph_buffer_put(ci->i_xattrs.prealloc_blob); + + ceph_put_string(rcu_dereference_raw(ci->i_layout.pool_ns)); + ceph_put_string(rcu_dereference_raw(ci->i_cached_layout.pool_ns)); +} + +static inline blkcnt_t calc_inode_blocks(u64 size) +{ + return (size + (1<<9) - 1) >> 9; +} + +/* + * Helpers to fill in size, ctime, mtime, and atime. We have to be + * careful because either the client or MDS may have more up to date + * info, depending on which capabilities are held, and whether + * time_warp_seq or truncate_seq have increased. (Ordinarily, mtime + * and size are monotonically increasing, except when utimes() or + * truncate() increments the corresponding _seq values.) + */ +int ceph_fill_file_size(struct inode *inode, int issued, + u32 truncate_seq, u64 truncate_size, u64 size) +{ + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + int queue_trunc = 0; + loff_t isize = i_size_read(inode); + + if (ceph_seq_cmp(truncate_seq, ci->i_truncate_seq) > 0 || + (truncate_seq == ci->i_truncate_seq && size > isize)) { + doutc(cl, "size %lld -> %llu\n", isize, size); + if (size > 0 && S_ISDIR(inode->i_mode)) { + pr_err_client(cl, "non-zero size for directory\n"); + size = 0; + } + i_size_write(inode, size); + inode->i_blocks = calc_inode_blocks(size); + /* + * If we're expanding, then we should be able to just update + * the existing cookie. + */ + if (size > isize) + ceph_fscache_update(inode); + ci->i_reported_size = size; + if (truncate_seq != ci->i_truncate_seq) { + doutc(cl, "truncate_seq %u -> %u\n", + ci->i_truncate_seq, truncate_seq); + ci->i_truncate_seq = truncate_seq; + + /* the MDS should have revoked these caps */ + WARN_ON_ONCE(issued & (CEPH_CAP_FILE_RD | + CEPH_CAP_FILE_LAZYIO)); + /* + * If we hold relevant caps, or in the case where we're + * not the only client referencing this file and we + * don't hold those caps, then we need to check whether + * the file is either opened or mmaped + */ + if ((issued & (CEPH_CAP_FILE_CACHE| + CEPH_CAP_FILE_BUFFER)) || + mapping_mapped(inode->i_mapping) || + __ceph_is_file_opened(ci)) { + ci->i_truncate_pending++; + queue_trunc = 1; + } + } + } + + /* + * It's possible that the new sizes of the two consecutive + * size truncations will be in the same fscrypt last block, + * and we need to truncate the corresponding page caches + * anyway. + */ + if (ceph_seq_cmp(truncate_seq, ci->i_truncate_seq) >= 0) { + doutc(cl, "truncate_size %lld -> %llu, encrypted %d\n", + ci->i_truncate_size, truncate_size, + !!IS_ENCRYPTED(inode)); + + ci->i_truncate_size = truncate_size; + + if (IS_ENCRYPTED(inode)) { + doutc(cl, "truncate_pagecache_size %lld -> %llu\n", + ci->i_truncate_pagecache_size, size); + ci->i_truncate_pagecache_size = size; + } else { + ci->i_truncate_pagecache_size = truncate_size; + } + } + return queue_trunc; +} + +/* + * Set the subvolume ID for an inode. + * + * The subvolume_id identifies which CephFS subvolume this inode belongs to. + * CEPH_SUBVOLUME_ID_NONE (0) means unknown/unset - the MDS only sends + * non-zero IDs for inodes within subvolumes. + * + * An inode's subvolume membership is immutable - once an inode is created + * in a subvolume, it stays there. Therefore, if we already have a valid + * (non-zero) subvolume_id and receive a different one, that indicates a bug. + */ +void ceph_inode_set_subvolume(struct inode *inode, u64 subvolume_id) +{ + struct ceph_inode_info *ci; + u64 old; + + if (!inode || subvolume_id == CEPH_SUBVOLUME_ID_NONE) + return; + + ci = ceph_inode(inode); + old = READ_ONCE(ci->i_subvolume_id); + + if (old == subvolume_id) + return; + + if (old != CEPH_SUBVOLUME_ID_NONE) { + /* subvolume_id should not change once set */ + WARN_ON_ONCE(1); + return; + } + + WRITE_ONCE(ci->i_subvolume_id, subvolume_id); +} + +void ceph_fill_file_time(struct inode *inode, int issued, + u64 time_warp_seq, struct timespec64 *ctime, + struct timespec64 *mtime, struct timespec64 *atime) +{ + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + struct timespec64 iatime = inode_get_atime(inode); + struct timespec64 ictime = inode_get_ctime(inode); + struct timespec64 imtime = inode_get_mtime(inode); + int warn = 0; + + if (issued & (CEPH_CAP_FILE_EXCL| + CEPH_CAP_FILE_WR| + CEPH_CAP_FILE_BUFFER| + CEPH_CAP_AUTH_EXCL| + CEPH_CAP_XATTR_EXCL)) { + if (ci->i_version == 0 || + timespec64_compare(ctime, &ictime) > 0) { + doutc(cl, "ctime %ptSp -> %ptSp inc w/ cap\n", &ictime, ctime); + inode_set_ctime_to_ts(inode, *ctime); + } + if (ci->i_version == 0 || + ceph_seq_cmp(time_warp_seq, ci->i_time_warp_seq) > 0) { + /* the MDS did a utimes() */ + doutc(cl, "mtime %ptSp -> %ptSp tw %d -> %d\n", &imtime, mtime, + ci->i_time_warp_seq, (int)time_warp_seq); + + inode_set_mtime_to_ts(inode, *mtime); + inode_set_atime_to_ts(inode, *atime); + ci->i_time_warp_seq = time_warp_seq; + } else if (time_warp_seq == ci->i_time_warp_seq) { + /* nobody did utimes(); take the max */ + if (timespec64_compare(mtime, &imtime) > 0) { + doutc(cl, "mtime %ptSp -> %ptSp inc\n", &imtime, mtime); + inode_set_mtime_to_ts(inode, *mtime); + } + if (timespec64_compare(atime, &iatime) > 0) { + doutc(cl, "atime %ptSp -> %ptSp inc\n", &iatime, atime); + inode_set_atime_to_ts(inode, *atime); + } + } else if (issued & CEPH_CAP_FILE_EXCL) { + /* we did a utimes(); ignore mds values */ + } else { + warn = 1; + } + } else { + /* we have no write|excl caps; whatever the MDS says is true */ + if (ceph_seq_cmp(time_warp_seq, ci->i_time_warp_seq) >= 0) { + inode_set_ctime_to_ts(inode, *ctime); + inode_set_mtime_to_ts(inode, *mtime); + inode_set_atime_to_ts(inode, *atime); + ci->i_time_warp_seq = time_warp_seq; + } else { + warn = 1; + } + } + if (warn) /* time_warp_seq shouldn't go backwards */ + doutc(cl, "%p mds time_warp_seq %llu < %u\n", inode, + time_warp_seq, ci->i_time_warp_seq); +} + +#if IS_ENABLED(CONFIG_FS_ENCRYPTION) +static int decode_encrypted_symlink(struct ceph_mds_client *mdsc, + const char *encsym, + int enclen, u8 **decsym) +{ + struct ceph_client *cl = mdsc->fsc->client; + int declen; + u8 *sym; + + sym = kmalloc(enclen + 1, GFP_NOFS); + if (!sym) + return -ENOMEM; + + declen = base64_decode(encsym, enclen, sym, false, BASE64_IMAP); + if (declen < 0) { + pr_err_client(cl, + "can't decode symlink (%d). Content: %.*s\n", + declen, enclen, encsym); + kfree(sym); + return -EIO; + } + sym[declen + 1] = '\0'; + *decsym = sym; + return declen; +} +#else +static int decode_encrypted_symlink(struct ceph_mds_client *mdsc, + const char *encsym, + int symlen, u8 **decsym) +{ + return -EOPNOTSUPP; +} +#endif + +/* + * Populate an inode based on info from mds. May be called on new or + * existing inodes. + */ +int ceph_fill_inode(struct inode *inode, struct page *locked_page, + struct ceph_mds_reply_info_in *iinfo, + struct ceph_mds_reply_dirfrag *dirinfo, + struct ceph_mds_session *session, int cap_fmode, + struct ceph_cap_reservation *caps_reservation) +{ + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb); + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_mds_reply_inode *info = iinfo->in; + struct ceph_inode_info *ci = ceph_inode(inode); + int issued, new_issued, info_caps; + struct timespec64 mtime, atime, ctime; + struct ceph_buffer *xattr_blob = NULL; + struct ceph_buffer *old_blob = NULL; + struct ceph_string *pool_ns = NULL; + struct ceph_cap *new_cap = NULL; + int err = 0; + bool wake = false; + bool queue_trunc = false; + bool new_version = false; + bool fill_inline = false; + umode_t mode = le32_to_cpu(info->mode); + dev_t rdev = le32_to_cpu(info->rdev); + + lockdep_assert_held(&mdsc->snap_rwsem); + + doutc(cl, "%p ino %llx.%llx v %llu had %llu\n", inode, ceph_vinop(inode), + le64_to_cpu(info->version), ci->i_version); + + /* Once I_NEW is cleared, we can't change type or dev numbers */ + if (inode_state_read_once(inode) & I_NEW) { + inode->i_mode = mode; + } else { + if (inode_wrong_type(inode, mode)) { + pr_warn_once_client(cl, + "inode type changed! (ino %llx.%llx is 0%o, mds says 0%o)\n", + ceph_vinop(inode), inode->i_mode, mode); + return -ESTALE; + } + + if ((S_ISCHR(mode) || S_ISBLK(mode)) && inode->i_rdev != rdev) { + pr_warn_once_client(cl, + "dev inode rdev changed! (ino %llx.%llx is %u:%u, mds says %u:%u)\n", + ceph_vinop(inode), MAJOR(inode->i_rdev), + MINOR(inode->i_rdev), MAJOR(rdev), + MINOR(rdev)); + return -ESTALE; + } + } + + info_caps = le32_to_cpu(info->cap.caps); + + /* prealloc new cap struct */ + if (info_caps && ceph_snap(inode) == CEPH_NOSNAP) { + new_cap = ceph_get_cap(mdsc, caps_reservation); + if (!new_cap) + return -ENOMEM; + } + + /* + * prealloc xattr data, if it looks like we'll need it. only + * if len > 4 (meaning there are actually xattrs; the first 4 + * bytes are the xattr count). + */ + if (iinfo->xattr_len > 4) { + xattr_blob = ceph_buffer_new(iinfo->xattr_len, GFP_NOFS); + if (!xattr_blob) + pr_err_client(cl, "ENOMEM xattr blob %d bytes\n", + iinfo->xattr_len); + } + + if (iinfo->pool_ns_len > 0) + pool_ns = ceph_find_or_create_string(iinfo->pool_ns_data, + iinfo->pool_ns_len); + + if (ceph_snap(inode) != CEPH_NOSNAP && !ci->i_snapid_map) + ci->i_snapid_map = ceph_get_snapid_map(mdsc, ceph_snap(inode)); + + spin_lock(&ci->i_ceph_lock); + + /* + * provided version will be odd if inode value is projected, + * even if stable. skip the update if we have newer stable + * info (ours>=theirs, e.g. due to racing mds replies), unless + * we are getting projected (unstable) info (in which case the + * version is odd, and we want ours>theirs). + * us them + * 2 2 skip + * 3 2 skip + * 3 3 update + */ + if (ci->i_version == 0 || + ((info->cap.flags & CEPH_CAP_FLAG_AUTH) && + le64_to_cpu(info->version) > (ci->i_version & ~1))) + new_version = true; + + /* Update change_attribute */ + inode_set_max_iversion_raw(inode, iinfo->change_attr); + + __ceph_caps_issued(ci, &issued); + issued |= __ceph_caps_dirty(ci); + new_issued = ~issued & info_caps; + + __ceph_update_quota(ci, iinfo->max_bytes, iinfo->max_files); + ceph_inode_set_subvolume(inode, iinfo->subvolume_id); + +#ifdef CONFIG_FS_ENCRYPTION + if (iinfo->fscrypt_auth_len && + ((inode_state_read_once(inode) & I_NEW) || (ci->fscrypt_auth_len == 0))) { + kfree(ci->fscrypt_auth); + ci->fscrypt_auth_len = iinfo->fscrypt_auth_len; + ci->fscrypt_auth = iinfo->fscrypt_auth; + iinfo->fscrypt_auth = NULL; + iinfo->fscrypt_auth_len = 0; + inode_set_flags(inode, S_ENCRYPTED, S_ENCRYPTED); + } +#endif + + if ((new_version || (new_issued & CEPH_CAP_AUTH_SHARED)) && + (issued & CEPH_CAP_AUTH_EXCL) == 0) { + inode->i_mode = mode; + inode->i_uid = make_kuid(&init_user_ns, le32_to_cpu(info->uid)); + inode->i_gid = make_kgid(&init_user_ns, le32_to_cpu(info->gid)); + doutc(cl, "%p %llx.%llx mode 0%o uid.gid %d.%d\n", inode, + ceph_vinop(inode), inode->i_mode, + from_kuid(&init_user_ns, inode->i_uid), + from_kgid(&init_user_ns, inode->i_gid)); + ceph_decode_timespec64(&ci->i_btime, &iinfo->btime); + ceph_decode_timespec64(&ci->i_snap_btime, &iinfo->snap_btime); + } + + /* directories have fl_stripe_unit set to zero */ + if (IS_ENCRYPTED(inode)) + inode->i_blkbits = CEPH_FSCRYPT_BLOCK_SHIFT; + else if (le32_to_cpu(info->layout.fl_stripe_unit)) + inode->i_blkbits = + fls(le32_to_cpu(info->layout.fl_stripe_unit)) - 1; + else + inode->i_blkbits = CEPH_BLOCK_SHIFT; + + if ((new_version || (new_issued & CEPH_CAP_LINK_SHARED)) && + (issued & CEPH_CAP_LINK_EXCL) == 0) + set_nlink(inode, le32_to_cpu(info->nlink)); + + if (new_version || (new_issued & CEPH_CAP_ANY_RD)) { + /* be careful with mtime, atime, size */ + ceph_decode_timespec64(&atime, &info->atime); + ceph_decode_timespec64(&mtime, &info->mtime); + ceph_decode_timespec64(&ctime, &info->ctime); + ceph_fill_file_time(inode, issued, + le32_to_cpu(info->time_warp_seq), + &ctime, &mtime, &atime); + } + + if (new_version || (info_caps & CEPH_CAP_FILE_SHARED)) { + ci->i_files = le64_to_cpu(info->files); + ci->i_subdirs = le64_to_cpu(info->subdirs); + } + + if (new_version || + (new_issued & (CEPH_CAP_ANY_FILE_RD | CEPH_CAP_ANY_FILE_WR))) { + u64 size = le64_to_cpu(info->size); + s64 old_pool = ci->i_layout.pool_id; + struct ceph_string *old_ns; + + ceph_file_layout_from_legacy(&ci->i_layout, &info->layout); + old_ns = rcu_dereference_protected(ci->i_layout.pool_ns, + lockdep_is_held(&ci->i_ceph_lock)); + rcu_assign_pointer(ci->i_layout.pool_ns, pool_ns); + + if (ci->i_layout.pool_id != old_pool || pool_ns != old_ns) + clear_bit(CEPH_I_POOL_PERM_BIT, &ci->i_ceph_flags); + + pool_ns = old_ns; + + if (IS_ENCRYPTED(inode) && size && + iinfo->fscrypt_file_len == sizeof(__le64)) { + u64 fsize = __le64_to_cpu(*(__le64 *)iinfo->fscrypt_file); + + if (size == round_up(fsize, CEPH_FSCRYPT_BLOCK_SIZE)) { + size = fsize; + } else { + pr_warn_client(cl, + "fscrypt size mismatch: size=%llu fscrypt_file=%llu, discarding fscrypt_file size.\n", + info->size, size); + } + } + + queue_trunc = ceph_fill_file_size(inode, issued, + le32_to_cpu(info->truncate_seq), + le64_to_cpu(info->truncate_size), + size); + /* only update max_size on auth cap */ + if ((info->cap.flags & CEPH_CAP_FLAG_AUTH) && + ci->i_max_size != le64_to_cpu(info->max_size)) { + doutc(cl, "max_size %lld -> %llu\n", + ci->i_max_size, le64_to_cpu(info->max_size)); + ci->i_max_size = le64_to_cpu(info->max_size); + } + } + + /* layout and rstat are not tracked by capability, update them if + * the inode info is from auth mds */ + if (new_version || (info->cap.flags & CEPH_CAP_FLAG_AUTH)) { + if (S_ISDIR(inode->i_mode)) { + ci->i_dir_layout = iinfo->dir_layout; + ci->i_rbytes = le64_to_cpu(info->rbytes); + ci->i_rfiles = le64_to_cpu(info->rfiles); + ci->i_rsubdirs = le64_to_cpu(info->rsubdirs); + ci->i_dir_pin = iinfo->dir_pin; + ci->i_rsnaps = iinfo->rsnaps; + ceph_decode_timespec64(&ci->i_rctime, &info->rctime); + } + } + + /* xattrs */ + /* note that if i_xattrs.len <= 4, i_xattrs.data will still be NULL. */ + if ((ci->i_xattrs.version == 0 || !(issued & CEPH_CAP_XATTR_EXCL)) && + le64_to_cpu(info->xattr_version) > ci->i_xattrs.version) { + if (ci->i_xattrs.blob) + old_blob = ci->i_xattrs.blob; + ci->i_xattrs.blob = xattr_blob; + if (xattr_blob) + memcpy(ci->i_xattrs.blob->vec.iov_base, + iinfo->xattr_data, iinfo->xattr_len); + ci->i_xattrs.version = le64_to_cpu(info->xattr_version); + ceph_forget_all_cached_acls(inode); + ceph_security_invalidate_secctx(inode); + xattr_blob = NULL; + } + + /* finally update i_version */ + if (le64_to_cpu(info->version) > ci->i_version) + ci->i_version = le64_to_cpu(info->version); + + inode->i_mapping->a_ops = &ceph_aops; + + switch (inode->i_mode & S_IFMT) { + case S_IFIFO: + case S_IFBLK: + case S_IFCHR: + case S_IFSOCK: + inode->i_blkbits = PAGE_SHIFT; + init_special_inode(inode, inode->i_mode, rdev); + inode->i_op = &ceph_file_iops; + break; + case S_IFREG: + inode->i_op = &ceph_file_iops; + inode->i_fop = &ceph_file_fops; + break; + case S_IFLNK: + if (!ci->i_symlink) { + u32 symlen = iinfo->symlink_len; + char *sym; + + spin_unlock(&ci->i_ceph_lock); + + if (IS_ENCRYPTED(inode)) { + if (symlen != i_size_read(inode)) + pr_err_client(cl, + "%p %llx.%llx BAD symlink size %lld\n", + inode, ceph_vinop(inode), + i_size_read(inode)); + + err = decode_encrypted_symlink(mdsc, iinfo->symlink, + symlen, (u8 **)&sym); + if (err < 0) { + pr_err_client(cl, + "decoding encrypted symlink failed: %d\n", + err); + goto out; + } + symlen = err; + i_size_write(inode, symlen); + inode->i_blocks = calc_inode_blocks(symlen); + } else { + if (symlen != i_size_read(inode)) { + pr_err_client(cl, + "%p %llx.%llx BAD symlink size %lld\n", + inode, ceph_vinop(inode), + i_size_read(inode)); + i_size_write(inode, symlen); + inode->i_blocks = calc_inode_blocks(symlen); + } + + err = -ENOMEM; + sym = kstrndup(iinfo->symlink, symlen, GFP_NOFS); + if (!sym) + goto out; + } + + spin_lock(&ci->i_ceph_lock); + if (!ci->i_symlink) + ci->i_symlink = sym; + else + kfree(sym); /* lost a race */ + } + + if (IS_ENCRYPTED(inode)) { + /* + * Encrypted symlinks need to be decrypted before we can + * cache their targets in i_link. Don't touch it here. + */ + inode->i_op = &ceph_encrypted_symlink_iops; + } else { + inode->i_link = ci->i_symlink; + inode->i_op = &ceph_symlink_iops; + } + break; + case S_IFDIR: + inode->i_op = &ceph_dir_iops; + inode->i_fop = &ceph_dir_fops; + break; + default: + pr_err_client(cl, "%p %llx.%llx BAD mode 0%o\n", inode, + ceph_vinop(inode), inode->i_mode); + } + + /* were we issued a capability? */ + if (info_caps) { + if (ceph_snap(inode) == CEPH_NOSNAP) { + ceph_add_cap(inode, session, + le64_to_cpu(info->cap.cap_id), + info_caps, + le32_to_cpu(info->cap.wanted), + le32_to_cpu(info->cap.seq), + le32_to_cpu(info->cap.mseq), + le64_to_cpu(info->cap.realm), + info->cap.flags, &new_cap); + + /* set dir completion flag? */ + if (S_ISDIR(inode->i_mode) && + ci->i_files == 0 && ci->i_subdirs == 0 && + (info_caps & CEPH_CAP_FILE_SHARED) && + (issued & CEPH_CAP_FILE_EXCL) == 0 && + !__ceph_dir_is_complete(ci)) { + doutc(cl, " marking %p complete (empty)\n", + inode); + i_size_write(inode, 0); + __ceph_dir_set_complete(ci, + atomic64_read(&ci->i_release_count), + atomic64_read(&ci->i_ordered_count)); + } + + wake = true; + } else { + doutc(cl, " %p got snap_caps %s\n", inode, + ceph_cap_string(info_caps)); + ci->i_snap_caps |= info_caps; + } + } + + if (iinfo->inline_version > 0 && + iinfo->inline_version >= ci->i_inline_version) { + int cache_caps = CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_LAZYIO; + ci->i_inline_version = iinfo->inline_version; + if (ceph_has_inline_data(ci) && + (locked_page || (info_caps & cache_caps))) + fill_inline = true; + } + + if (cap_fmode >= 0) { + if (!info_caps) + pr_warn_client(cl, "mds issued no caps on %llx.%llx\n", + ceph_vinop(inode)); + __ceph_touch_fmode(ci, mdsc, cap_fmode); + } + + spin_unlock(&ci->i_ceph_lock); + + ceph_fscache_register_inode_cookie(inode); + + if (fill_inline) + ceph_fill_inline_data(inode, locked_page, + iinfo->inline_data, iinfo->inline_len); + + if (wake) + wake_up_all(&ci->i_cap_wq); + + /* queue truncate if we saw i_size decrease */ + if (queue_trunc) + ceph_queue_vmtruncate(inode); + + /* populate frag tree */ + if (S_ISDIR(inode->i_mode)) + ceph_fill_fragtree(inode, &info->fragtree, dirinfo); + + /* update delegation info? */ + if (dirinfo) + ceph_fill_dirfrag(inode, dirinfo); + + err = 0; +out: + if (new_cap) + ceph_put_cap(mdsc, new_cap); + ceph_buffer_put(old_blob); + ceph_buffer_put(xattr_blob); + ceph_put_string(pool_ns); + return err; +} + +/* + * caller should hold session s_mutex and dentry->d_lock. + */ +static void __update_dentry_lease(struct inode *dir, struct dentry *dentry, + struct ceph_mds_reply_lease *lease, + struct ceph_mds_session *session, + unsigned long from_time, + struct ceph_mds_session **old_lease_session) +{ + struct ceph_client *cl = ceph_inode_to_client(dir); + struct ceph_dentry_info *di = ceph_dentry(dentry); + unsigned mask = le16_to_cpu(lease->mask); + long unsigned duration = le32_to_cpu(lease->duration_ms); + long unsigned ttl = from_time + (duration * HZ) / 1000; + long unsigned half_ttl = from_time + (duration * HZ / 2) / 1000; + + doutc(cl, "%p duration %lu ms ttl %lu\n", dentry, duration, ttl); + + /* only track leases on regular dentries */ + if (ceph_snap(dir) != CEPH_NOSNAP) + return; + + if (mask & CEPH_LEASE_PRIMARY_LINK) + di->flags |= CEPH_DENTRY_PRIMARY_LINK; + else + di->flags &= ~CEPH_DENTRY_PRIMARY_LINK; + + di->lease_shared_gen = atomic_read(&ceph_inode(dir)->i_shared_gen); + if (!(mask & CEPH_LEASE_VALID)) { + __ceph_dentry_dir_lease_touch(di); + return; + } + + if (di->lease_gen == atomic_read(&session->s_cap_gen) && + time_before(ttl, di->time)) + return; /* we already have a newer lease. */ + + if (di->lease_session && di->lease_session != session) { + *old_lease_session = di->lease_session; + di->lease_session = NULL; + } + + if (!di->lease_session) + di->lease_session = ceph_get_mds_session(session); + di->lease_gen = atomic_read(&session->s_cap_gen); + di->lease_seq = le32_to_cpu(lease->seq); + di->lease_renew_after = half_ttl; + di->lease_renew_from = 0; + di->time = ttl; + + __ceph_dentry_lease_touch(di); +} + +static inline void update_dentry_lease(struct inode *dir, struct dentry *dentry, + struct ceph_mds_reply_lease *lease, + struct ceph_mds_session *session, + unsigned long from_time) +{ + struct ceph_mds_session *old_lease_session = NULL; + spin_lock(&dentry->d_lock); + __update_dentry_lease(dir, dentry, lease, session, from_time, + &old_lease_session); + spin_unlock(&dentry->d_lock); + ceph_put_mds_session(old_lease_session); +} + +/* + * update dentry lease without having parent inode locked + */ +static void update_dentry_lease_careful(struct dentry *dentry, + struct ceph_mds_reply_lease *lease, + struct ceph_mds_session *session, + unsigned long from_time, + char *dname, u32 dname_len, + struct ceph_vino *pdvino, + struct ceph_vino *ptvino) + +{ + struct inode *dir; + struct ceph_mds_session *old_lease_session = NULL; + + spin_lock(&dentry->d_lock); + /* make sure dentry's name matches target */ + if (dentry->d_name.len != dname_len || + memcmp(dentry->d_name.name, dname, dname_len)) + goto out_unlock; + + dir = d_inode(dentry->d_parent); + /* make sure parent matches dvino */ + if (!ceph_ino_compare(dir, pdvino)) + goto out_unlock; + + /* make sure dentry's inode matches target. NULL ptvino means that + * we expect a negative dentry */ + if (ptvino) { + if (d_really_is_negative(dentry)) + goto out_unlock; + if (!ceph_ino_compare(d_inode(dentry), ptvino)) + goto out_unlock; + } else { + if (d_really_is_positive(dentry)) + goto out_unlock; + } + + __update_dentry_lease(dir, dentry, lease, session, + from_time, &old_lease_session); +out_unlock: + spin_unlock(&dentry->d_lock); + ceph_put_mds_session(old_lease_session); +} + +/* + * splice a dentry to an inode. + * caller must hold directory i_rwsem for this to be safe. + */ +static int splice_dentry(struct dentry **pdn, struct inode *in) +{ + struct ceph_client *cl = ceph_inode_to_client(in); + struct dentry *dn = *pdn; + struct dentry *realdn; + + BUG_ON(d_inode(dn)); + + if (S_ISDIR(in->i_mode)) { + /* If inode is directory, d_splice_alias() below will remove + * 'realdn' from its origin parent. We need to ensure that + * origin parent's readdir cache will not reference 'realdn' + */ + realdn = d_find_any_alias(in); + if (realdn) { + struct ceph_dentry_info *di = ceph_dentry(realdn); + spin_lock(&realdn->d_lock); + + realdn->d_op->d_prune(realdn); + + di->time = jiffies; + di->lease_shared_gen = 0; + di->offset = 0; + + spin_unlock(&realdn->d_lock); + dput(realdn); + } + } + + /* dn must be unhashed */ + if (!d_unhashed(dn)) + d_drop(dn); + realdn = d_splice_alias(in, dn); + if (IS_ERR(realdn)) { + pr_err_client(cl, "error %ld %p inode %p ino %llx.%llx\n", + PTR_ERR(realdn), dn, in, ceph_vinop(in)); + return PTR_ERR(realdn); + } + + if (realdn) { + doutc(cl, "dn %p (%d) spliced with %p (%d) inode %p ino %llx.%llx\n", + dn, d_count(dn), realdn, d_count(realdn), + d_inode(realdn), ceph_vinop(d_inode(realdn))); + dput(dn); + *pdn = realdn; + } else { + BUG_ON(!ceph_dentry(dn)); + doutc(cl, "dn %p attached to %p ino %llx.%llx\n", dn, + d_inode(dn), ceph_vinop(d_inode(dn))); + } + return 0; +} + +/* + * Incorporate results into the local cache. This is either just + * one inode, or a directory, dentry, and possibly linked-to inode (e.g., + * after a lookup). + * + * A reply may contain + * a directory inode along with a dentry. + * and/or a target inode + * + * Called with snap_rwsem (read). + */ +int ceph_fill_trace(struct super_block *sb, struct ceph_mds_request *req) +{ + struct ceph_mds_session *session = req->r_session; + struct ceph_mds_reply_info_parsed *rinfo = &req->r_reply_info; + struct inode *in = NULL; + struct ceph_vino tvino, dvino; + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(sb); + struct ceph_client *cl = fsc->client; + struct inode *parent_dir = NULL; + int err = 0; + + doutc(cl, "%p is_dentry %d is_target %d\n", req, + rinfo->head->is_dentry, rinfo->head->is_target); + + if (!rinfo->head->is_target && !rinfo->head->is_dentry) { + doutc(cl, "reply is empty!\n"); + if (rinfo->head->result == 0 && req->r_parent) + ceph_invalidate_dir_request(req); + return 0; + } + + if (rinfo->head->is_dentry) { + /* + * r_parent may be stale, in cases when R_PARENT_LOCKED is not set, + * so we need to get the correct inode + */ + parent_dir = ceph_get_reply_dir(sb, req->r_parent, rinfo); + if (unlikely(IS_ERR(parent_dir))) { + err = PTR_ERR(parent_dir); + goto done; + } + if (parent_dir) { + ceph_inode_set_subvolume(parent_dir, + rinfo->diri.subvolume_id); + err = ceph_fill_inode(parent_dir, NULL, &rinfo->diri, + rinfo->dirfrag, session, -1, + &req->r_caps_reservation); + if (err < 0) + goto done; + } else { + WARN_ON_ONCE(1); + } + + if (parent_dir && req->r_op == CEPH_MDS_OP_LOOKUPNAME && + test_bit(CEPH_MDS_R_PARENT_LOCKED, &req->r_req_flags) && + !test_bit(CEPH_MDS_R_ABORTED, &req->r_req_flags)) { + bool is_nokey = false; + struct qstr dname; + struct dentry *dn, *parent; + struct fscrypt_str oname = FSTR_INIT(NULL, 0); + struct ceph_fname fname = { .dir = parent_dir, + .name = rinfo->dname, + .ctext = rinfo->altname, + .name_len = rinfo->dname_len, + .ctext_len = rinfo->altname_len }; + + BUG_ON(!rinfo->head->is_target); + BUG_ON(req->r_dentry); + + parent = d_find_any_alias(parent_dir); + BUG_ON(!parent); + + err = ceph_fname_alloc_buffer(parent_dir, &oname); + if (err < 0) { + dput(parent); + goto done; + } + + err = ceph_fname_to_usr(&fname, NULL, &oname, &is_nokey); + if (err < 0) { + dput(parent); + ceph_fname_free_buffer(parent_dir, &oname); + goto done; + } + dname.name = oname.name; + dname.len = oname.len; + dname.hash = full_name_hash(parent, dname.name, dname.len); + tvino.ino = le64_to_cpu(rinfo->targeti.in->ino); + tvino.snap = le64_to_cpu(rinfo->targeti.in->snapid); +retry_lookup: + dn = d_lookup(parent, &dname); + doutc(cl, "d_lookup on parent=%p name=%.*s got %p\n", + parent, dname.len, dname.name, dn); + + if (!dn) { + dn = d_alloc(parent, &dname); + doutc(cl, "d_alloc %p '%.*s' = %p\n", parent, + dname.len, dname.name, dn); + if (!dn) { + dput(parent); + ceph_fname_free_buffer(parent_dir, &oname); + err = -ENOMEM; + goto done; + } + if (is_nokey) { + spin_lock(&dn->d_lock); + dn->d_flags |= DCACHE_NOKEY_NAME; + spin_unlock(&dn->d_lock); + } + err = 0; + } else if (d_really_is_positive(dn) && + (ceph_ino(d_inode(dn)) != tvino.ino || + ceph_snap(d_inode(dn)) != tvino.snap)) { + doutc(cl, " dn %p points to wrong inode %p\n", + dn, d_inode(dn)); + ceph_dir_clear_ordered(parent_dir); + d_delete(dn); + dput(dn); + goto retry_lookup; + } + ceph_fname_free_buffer(parent_dir, &oname); + + req->r_dentry = dn; + dput(parent); + } + } + + if (rinfo->head->is_target) { + /* Should be filled in by handle_reply */ + BUG_ON(!req->r_target_inode); + + in = req->r_target_inode; + ceph_inode_set_subvolume(in, rinfo->targeti.subvolume_id); + err = ceph_fill_inode(in, req->r_locked_page, &rinfo->targeti, + NULL, session, + (!test_bit(CEPH_MDS_R_ABORTED, &req->r_req_flags) && + !test_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags) && + rinfo->head->result == 0) ? req->r_fmode : -1, + &req->r_caps_reservation); + if (err < 0) { + pr_err_client(cl, "badness %p %llx.%llx\n", in, + ceph_vinop(in)); + req->r_target_inode = NULL; + if (inode_state_read_once(in) & I_NEW) + discard_new_inode(in); + else + iput(in); + goto done; + } + if (inode_state_read_once(in) & I_NEW) + unlock_new_inode(in); + } + + /* + * ignore null lease/binding on snapdir ENOENT, or else we + * will have trouble splicing in the virtual snapdir later + */ + if (rinfo->head->is_dentry && + !test_bit(CEPH_MDS_R_ABORTED, &req->r_req_flags) && + test_bit(CEPH_MDS_R_PARENT_LOCKED, &req->r_req_flags) && + (rinfo->head->is_target || strncmp(req->r_dentry->d_name.name, + fsc->mount_options->snapdir_name, + req->r_dentry->d_name.len))) { + /* + * lookup link rename : null -> possibly existing inode + * mknod symlink mkdir : null -> new inode + * unlink : linked -> null + */ + struct inode *dir = req->r_parent; + struct dentry *dn = req->r_dentry; + bool have_dir_cap, have_lease; + + BUG_ON(!dn); + BUG_ON(!dir); + BUG_ON(d_inode(dn->d_parent) != dir); + + dvino.ino = le64_to_cpu(rinfo->diri.in->ino); + dvino.snap = le64_to_cpu(rinfo->diri.in->snapid); + + BUG_ON(ceph_ino(dir) != dvino.ino); + BUG_ON(ceph_snap(dir) != dvino.snap); + + /* do we have a lease on the whole dir? */ + have_dir_cap = + (le32_to_cpu(rinfo->diri.in->cap.caps) & + CEPH_CAP_FILE_SHARED); + + /* do we have a dn lease? */ + have_lease = have_dir_cap || + le32_to_cpu(rinfo->dlease->duration_ms); + if (!have_lease) + doutc(cl, "no dentry lease or dir cap\n"); + + /* rename? */ + if (req->r_old_dentry && req->r_op == CEPH_MDS_OP_RENAME) { + struct inode *olddir = req->r_old_dentry_dir; + BUG_ON(!olddir); + + doutc(cl, " src %p '%pd' dst %p '%pd'\n", + req->r_old_dentry, req->r_old_dentry, dn, dn); + doutc(cl, "doing d_move %p -> %p\n", req->r_old_dentry, dn); + + /* d_move screws up sibling dentries' offsets */ + ceph_dir_clear_ordered(dir); + ceph_dir_clear_ordered(olddir); + + d_move(req->r_old_dentry, dn); + doutc(cl, " src %p '%pd' dst %p '%pd'\n", + req->r_old_dentry, req->r_old_dentry, dn, dn); + + /* ensure target dentry is invalidated, despite + rehashing bug in vfs_rename_dir */ + ceph_invalidate_dentry_lease(dn); + + doutc(cl, "dn %p gets new offset %lld\n", + req->r_old_dentry, + ceph_dentry(req->r_old_dentry)->offset); + + /* swap r_dentry and r_old_dentry in case that + * splice_dentry() gets called later. This is safe + * because no other place will use them */ + req->r_dentry = req->r_old_dentry; + req->r_old_dentry = dn; + dn = req->r_dentry; + } + + /* null dentry? */ + if (!rinfo->head->is_target) { + doutc(cl, "null dentry\n"); + if (d_really_is_positive(dn)) { + doutc(cl, "d_delete %p\n", dn); + ceph_dir_clear_ordered(dir); + d_delete(dn); + } else if (have_lease) { + if (d_unhashed(dn)) + d_add(dn, NULL); + } + + if (!d_unhashed(dn) && have_lease) + update_dentry_lease(dir, dn, + rinfo->dlease, session, + req->r_request_started); + goto done; + } + + if (unlikely(!in)) { + err = -EINVAL; + goto done; + } + + /* attach proper inode */ + if (d_really_is_negative(dn)) { + ceph_dir_clear_ordered(dir); + ihold(in); + err = splice_dentry(&req->r_dentry, in); + if (err < 0) + goto done; + dn = req->r_dentry; /* may have spliced */ + } else if (d_really_is_positive(dn) && d_inode(dn) != in) { + doutc(cl, " %p links to %p %llx.%llx, not %llx.%llx\n", + dn, d_inode(dn), ceph_vinop(d_inode(dn)), + ceph_vinop(in)); + d_invalidate(dn); + have_lease = false; + } + + if (have_lease) { + update_dentry_lease(dir, dn, + rinfo->dlease, session, + req->r_request_started); + } + doutc(cl, " final dn %p\n", dn); + } else if ((req->r_op == CEPH_MDS_OP_LOOKUPSNAP || + req->r_op == CEPH_MDS_OP_MKSNAP) && + test_bit(CEPH_MDS_R_PARENT_LOCKED, &req->r_req_flags) && + !test_bit(CEPH_MDS_R_ABORTED, &req->r_req_flags)) { + struct inode *dir = req->r_parent; + + /* fill out a snapdir LOOKUPSNAP dentry */ + BUG_ON(!dir); + BUG_ON(ceph_snap(dir) != CEPH_SNAPDIR); + BUG_ON(!req->r_dentry); + doutc(cl, " linking snapped dir %p to dn %p\n", in, + req->r_dentry); + ceph_dir_clear_ordered(dir); + + if (unlikely(!in)) { + err = -EINVAL; + goto done; + } + + ihold(in); + err = splice_dentry(&req->r_dentry, in); + if (err < 0) + goto done; + } else if (rinfo->head->is_dentry && req->r_dentry) { + /* parent inode is not locked, be careful */ + struct ceph_vino *ptvino = NULL; + dvino.ino = le64_to_cpu(rinfo->diri.in->ino); + dvino.snap = le64_to_cpu(rinfo->diri.in->snapid); + if (rinfo->head->is_target) { + tvino.ino = le64_to_cpu(rinfo->targeti.in->ino); + tvino.snap = le64_to_cpu(rinfo->targeti.in->snapid); + ptvino = &tvino; + } + update_dentry_lease_careful(req->r_dentry, rinfo->dlease, + session, req->r_request_started, + rinfo->dname, rinfo->dname_len, + &dvino, ptvino); + } +done: + /* Drop extra ref from ceph_get_reply_dir() if it returned a new inode */ + if (unlikely(!IS_ERR_OR_NULL(parent_dir) && parent_dir != req->r_parent)) + iput(parent_dir); + doutc(cl, "done err=%d\n", err); + return err; +} + +/* + * Prepopulate our cache with readdir results, leases, etc. + */ +static int readdir_prepopulate_inodes_only(struct ceph_mds_request *req, + struct ceph_mds_session *session) +{ + struct ceph_mds_reply_info_parsed *rinfo = &req->r_reply_info; + struct ceph_client *cl = session->s_mdsc->fsc->client; + int i, err = 0; + + for (i = 0; i < rinfo->dir_nr; i++) { + struct ceph_mds_reply_dir_entry *rde = rinfo->dir_entries + i; + struct ceph_vino vino; + struct inode *in; + int rc; + + vino.ino = le64_to_cpu(rde->inode.in->ino); + vino.snap = le64_to_cpu(rde->inode.in->snapid); + + in = ceph_get_inode(req->r_dentry->d_sb, vino, NULL); + if (IS_ERR(in)) { + err = PTR_ERR(in); + doutc(cl, "badness got %d\n", err); + continue; + } + rc = ceph_fill_inode(in, NULL, &rde->inode, NULL, session, + -1, &req->r_caps_reservation); + if (rc < 0) { + pr_err_client(cl, "inode badness on %p got %d\n", in, + rc); + err = rc; + if (inode_state_read_once(in) & I_NEW) { + ihold(in); + discard_new_inode(in); + } + } else if (inode_state_read_once(in) & I_NEW) { + unlock_new_inode(in); + } + + iput(in); + } + + return err; +} + +void ceph_readdir_cache_release(struct ceph_readdir_cache_control *ctl) +{ + if (ctl->folio) { + folio_release_kmap(ctl->folio, ctl->dentries); + ctl->folio = NULL; + } +} + +static int fill_readdir_cache(struct inode *dir, struct dentry *dn, + struct ceph_readdir_cache_control *ctl, + struct ceph_mds_request *req) +{ + struct ceph_client *cl = ceph_inode_to_client(dir); + struct ceph_inode_info *ci = ceph_inode(dir); + unsigned nsize = PAGE_SIZE / sizeof(struct dentry*); + unsigned idx = ctl->index % nsize; + pgoff_t pgoff = ctl->index / nsize; + + if (!ctl->folio || pgoff != ctl->folio->index) { + ceph_readdir_cache_release(ctl); + fgf_t fgf = FGP_LOCK; + + if (idx == 0) + fgf |= FGP_ACCESSED | FGP_CREAT; + + ctl->folio = __filemap_get_folio(&dir->i_data, pgoff, + fgf, mapping_gfp_mask(&dir->i_data)); + if (IS_ERR(ctl->folio)) { + int err = PTR_ERR(ctl->folio); + + ctl->folio = NULL; + ctl->index = -1; + return idx == 0 ? err : 0; + } + /* reading/filling the cache are serialized by + * i_rwsem, no need to use folio lock */ + folio_unlock(ctl->folio); + ctl->dentries = kmap_local_folio(ctl->folio, 0); + if (idx == 0) + memset(ctl->dentries, 0, PAGE_SIZE); + } + + if (req->r_dir_release_cnt == atomic64_read(&ci->i_release_count) && + req->r_dir_ordered_cnt == atomic64_read(&ci->i_ordered_count)) { + doutc(cl, "dn %p idx %d\n", dn, ctl->index); + ctl->dentries[idx] = dn; + ctl->index++; + } else { + doutc(cl, "disable readdir cache\n"); + ctl->index = -1; + } + return 0; +} + +int ceph_readdir_prepopulate(struct ceph_mds_request *req, + struct ceph_mds_session *session) +{ + struct dentry *parent = req->r_dentry; + struct inode *inode = d_inode(parent); + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_mds_reply_info_parsed *rinfo = &req->r_reply_info; + struct ceph_client *cl = session->s_mdsc->fsc->client; + struct qstr dname; + struct dentry *dn; + struct inode *in; + int err = 0, skipped = 0, ret, i; + u32 frag = le32_to_cpu(req->r_args.readdir.frag); + u32 last_hash = 0; + u32 fpos_offset; + struct ceph_readdir_cache_control cache_ctl = {}; + + if (test_bit(CEPH_MDS_R_ABORTED, &req->r_req_flags)) + return readdir_prepopulate_inodes_only(req, session); + + if (rinfo->hash_order) { + if (req->r_path2) { + last_hash = ceph_str_hash(ci->i_dir_layout.dl_dir_hash, + req->r_path2, + strlen(req->r_path2)); + last_hash = ceph_frag_value(last_hash); + } else if (rinfo->offset_hash) { + /* mds understands offset_hash */ + WARN_ON_ONCE(req->r_readdir_offset != 2); + last_hash = le32_to_cpu(req->r_args.readdir.offset_hash); + } + } + + if (rinfo->dir_dir && + le32_to_cpu(rinfo->dir_dir->frag) != frag) { + doutc(cl, "got new frag %x -> %x\n", frag, + le32_to_cpu(rinfo->dir_dir->frag)); + frag = le32_to_cpu(rinfo->dir_dir->frag); + if (!rinfo->hash_order) + req->r_readdir_offset = 2; + } + + if (le32_to_cpu(rinfo->head->op) == CEPH_MDS_OP_LSSNAP) { + doutc(cl, "%d items under SNAPDIR dn %p\n", + rinfo->dir_nr, parent); + } else { + doutc(cl, "%d items under dn %p\n", rinfo->dir_nr, parent); + if (rinfo->dir_dir) + ceph_fill_dirfrag(d_inode(parent), rinfo->dir_dir); + + if (ceph_frag_is_leftmost(frag) && + req->r_readdir_offset == 2 && + !(rinfo->hash_order && last_hash)) { + /* note dir version at start of readdir so we can + * tell if any dentries get dropped */ + req->r_dir_release_cnt = + atomic64_read(&ci->i_release_count); + req->r_dir_ordered_cnt = + atomic64_read(&ci->i_ordered_count); + req->r_readdir_cache_idx = 0; + } + } + + cache_ctl.index = req->r_readdir_cache_idx; + fpos_offset = req->r_readdir_offset; + + /* FIXME: release caps/leases if error occurs */ + for (i = 0; i < rinfo->dir_nr; i++) { + struct ceph_mds_reply_dir_entry *rde = rinfo->dir_entries + i; + struct ceph_vino tvino; + + dname.name = rde->name; + dname.len = rde->name_len; + dname.hash = full_name_hash(parent, dname.name, dname.len); + + tvino.ino = le64_to_cpu(rde->inode.in->ino); + tvino.snap = le64_to_cpu(rde->inode.in->snapid); + + if (rinfo->hash_order) { + u32 hash = ceph_frag_value(rde->raw_hash); + if (hash != last_hash) + fpos_offset = 2; + last_hash = hash; + rde->offset = ceph_make_fpos(hash, fpos_offset++, true); + } else { + rde->offset = ceph_make_fpos(frag, fpos_offset++, false); + } + +retry_lookup: + dn = d_lookup(parent, &dname); + doutc(cl, "d_lookup on parent=%p name=%.*s got %p\n", + parent, dname.len, dname.name, dn); + + if (!dn) { + dn = d_alloc(parent, &dname); + doutc(cl, "d_alloc %p '%.*s' = %p\n", parent, + dname.len, dname.name, dn); + if (!dn) { + doutc(cl, "d_alloc badness\n"); + err = -ENOMEM; + goto out; + } + if (rde->is_nokey) { + spin_lock(&dn->d_lock); + dn->d_flags |= DCACHE_NOKEY_NAME; + spin_unlock(&dn->d_lock); + } + } else if (d_really_is_positive(dn) && + (ceph_ino(d_inode(dn)) != tvino.ino || + ceph_snap(d_inode(dn)) != tvino.snap)) { + struct ceph_dentry_info *di = ceph_dentry(dn); + doutc(cl, " dn %p points to wrong inode %p\n", + dn, d_inode(dn)); + + spin_lock(&dn->d_lock); + if (di->offset > 0 && + di->lease_shared_gen == + atomic_read(&ci->i_shared_gen)) { + __ceph_dir_clear_ordered(ci); + di->offset = 0; + } + spin_unlock(&dn->d_lock); + + d_delete(dn); + dput(dn); + goto retry_lookup; + } + + /* inode */ + if (d_really_is_positive(dn)) { + in = d_inode(dn); + } else { + in = ceph_get_inode(parent->d_sb, tvino, NULL); + if (IS_ERR(in)) { + doutc(cl, "new_inode badness\n"); + d_drop(dn); + dput(dn); + err = PTR_ERR(in); + goto out; + } + } + + ret = ceph_fill_inode(in, NULL, &rde->inode, NULL, session, + -1, &req->r_caps_reservation); + if (ret < 0) { + pr_err_client(cl, "badness on %p %llx.%llx\n", in, + ceph_vinop(in)); + if (d_really_is_negative(dn)) { + if (inode_state_read_once(in) & I_NEW) { + ihold(in); + discard_new_inode(in); + } + iput(in); + } + d_drop(dn); + err = ret; + goto next_item; + } + if (inode_state_read_once(in) & I_NEW) + unlock_new_inode(in); + + if (d_really_is_negative(dn)) { + if (ceph_security_xattr_deadlock(in)) { + doutc(cl, " skip splicing dn %p to inode %p" + " (security xattr deadlock)\n", dn, in); + iput(in); + skipped++; + goto next_item; + } + + err = splice_dentry(&dn, in); + if (err < 0) + goto next_item; + } + + ceph_dentry(dn)->offset = rde->offset; + + update_dentry_lease(d_inode(parent), dn, + rde->lease, req->r_session, + req->r_request_started); + + if (err == 0 && skipped == 0 && cache_ctl.index >= 0) { + ret = fill_readdir_cache(d_inode(parent), dn, + &cache_ctl, req); + if (ret < 0) + err = ret; + } +next_item: + dput(dn); + } +out: + if (err == 0 && skipped == 0) { + set_bit(CEPH_MDS_R_DID_PREPOPULATE, &req->r_req_flags); + req->r_readdir_cache_idx = cache_ctl.index; + } + ceph_readdir_cache_release(&cache_ctl); + doutc(cl, "done\n"); + return err; +} + +bool ceph_inode_set_size(struct inode *inode, loff_t size) +{ + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + bool ret; + + spin_lock(&ci->i_ceph_lock); + doutc(cl, "set_size %p %llu -> %llu\n", inode, i_size_read(inode), size); + i_size_write(inode, size); + ceph_fscache_update(inode); + inode->i_blocks = calc_inode_blocks(size); + + ret = __ceph_should_report_size(ci); + + spin_unlock(&ci->i_ceph_lock); + + return ret; +} + +void ceph_queue_inode_work(struct inode *inode, int work_bit) +{ + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + struct ceph_client *cl = fsc->client; + struct ceph_inode_info *ci = ceph_inode(inode); + set_bit(work_bit, &ci->i_work_mask); + + ihold(inode); + if (queue_work(fsc->inode_wq, &ci->i_work)) { + doutc(cl, "%p %llx.%llx mask=%lx\n", inode, + ceph_vinop(inode), ci->i_work_mask); + } else { + doutc(cl, "%p %llx.%llx already queued, mask=%lx\n", + inode, ceph_vinop(inode), ci->i_work_mask); + iput(inode); + } +} + +static void ceph_do_invalidate_pages(struct inode *inode) +{ + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + u32 orig_gen; + int check = 0; + + ceph_fscache_invalidate(inode, false); + + mutex_lock(&ci->i_truncate_mutex); + + if (ceph_inode_is_shutdown(inode)) { + pr_warn_ratelimited_client(cl, + "%p %llx.%llx is shut down\n", inode, + ceph_vinop(inode)); + mapping_set_error(inode->i_mapping, -EIO); + truncate_pagecache(inode, 0); + mutex_unlock(&ci->i_truncate_mutex); + goto out; + } + + spin_lock(&ci->i_ceph_lock); + doutc(cl, "%p %llx.%llx gen %d revoking %d\n", inode, + ceph_vinop(inode), ci->i_rdcache_gen, ci->i_rdcache_revoking); + if (ci->i_rdcache_revoking != ci->i_rdcache_gen) { + if (__ceph_caps_revoking_other(ci, NULL, CEPH_CAP_FILE_CACHE)) + check = 1; + spin_unlock(&ci->i_ceph_lock); + mutex_unlock(&ci->i_truncate_mutex); + goto out; + } + orig_gen = ci->i_rdcache_gen; + spin_unlock(&ci->i_ceph_lock); + + if (invalidate_inode_pages2(inode->i_mapping) < 0) { + pr_err_client(cl, "invalidate_inode_pages2 %llx.%llx failed\n", + ceph_vinop(inode)); + } + + spin_lock(&ci->i_ceph_lock); + if (orig_gen == ci->i_rdcache_gen && + orig_gen == ci->i_rdcache_revoking) { + doutc(cl, "%p %llx.%llx gen %d successful\n", inode, + ceph_vinop(inode), ci->i_rdcache_gen); + ci->i_rdcache_revoking--; + check = 1; + } else { + doutc(cl, "%p %llx.%llx gen %d raced, now %d revoking %d\n", + inode, ceph_vinop(inode), orig_gen, ci->i_rdcache_gen, + ci->i_rdcache_revoking); + if (__ceph_caps_revoking_other(ci, NULL, CEPH_CAP_FILE_CACHE)) + check = 1; + } + spin_unlock(&ci->i_ceph_lock); + mutex_unlock(&ci->i_truncate_mutex); +out: + if (check) + ceph_check_caps(ci, 0); +} + +/* + * Make sure any pending truncation is applied before doing anything + * that may depend on it. + */ +void __ceph_do_pending_vmtruncate(struct inode *inode) +{ + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + u64 to; + int wrbuffer_refs, finish = 0; + + mutex_lock(&ci->i_truncate_mutex); +retry: + spin_lock(&ci->i_ceph_lock); + if (ci->i_truncate_pending == 0) { + doutc(cl, "%p %llx.%llx none pending\n", inode, + ceph_vinop(inode)); + spin_unlock(&ci->i_ceph_lock); + mutex_unlock(&ci->i_truncate_mutex); + return; + } + + /* + * make sure any dirty snapped pages are flushed before we + * possibly truncate them.. so write AND block! + */ + if (ci->i_wrbuffer_ref_head < ci->i_wrbuffer_ref) { + spin_unlock(&ci->i_ceph_lock); + doutc(cl, "%p %llx.%llx flushing snaps first\n", inode, + ceph_vinop(inode)); + filemap_write_and_wait_range(&inode->i_data, 0, + inode->i_sb->s_maxbytes); + goto retry; + } + + /* there should be no reader or writer */ + WARN_ON_ONCE(ci->i_rd_ref || ci->i_wr_ref); + + to = ci->i_truncate_pagecache_size; + wrbuffer_refs = ci->i_wrbuffer_ref; + doutc(cl, "%p %llx.%llx (%d) to %lld\n", inode, ceph_vinop(inode), + ci->i_truncate_pending, to); + spin_unlock(&ci->i_ceph_lock); + + ceph_fscache_resize(inode, to); + truncate_pagecache(inode, to); + + spin_lock(&ci->i_ceph_lock); + if (to == ci->i_truncate_pagecache_size) { + ci->i_truncate_pending = 0; + finish = 1; + } + spin_unlock(&ci->i_ceph_lock); + if (!finish) + goto retry; + + mutex_unlock(&ci->i_truncate_mutex); + + if (wrbuffer_refs == 0) + ceph_check_caps(ci, 0); + + wake_up_all(&ci->i_cap_wq); +} + +static void ceph_inode_work(struct work_struct *work) +{ + struct ceph_inode_info *ci = container_of(work, struct ceph_inode_info, + i_work); + struct inode *inode = &ci->netfs.inode; + struct ceph_client *cl = ceph_inode_to_client(inode); + + if (test_and_clear_bit(CEPH_I_WORK_WRITEBACK, &ci->i_work_mask)) { + doutc(cl, "writeback %p %llx.%llx\n", inode, ceph_vinop(inode)); + filemap_fdatawrite(&inode->i_data); + } + if (test_and_clear_bit(CEPH_I_WORK_INVALIDATE_PAGES, &ci->i_work_mask)) + ceph_do_invalidate_pages(inode); + + if (test_and_clear_bit(CEPH_I_WORK_VMTRUNCATE, &ci->i_work_mask)) + __ceph_do_pending_vmtruncate(inode); + + if (test_and_clear_bit(CEPH_I_WORK_CHECK_CAPS, &ci->i_work_mask)) + ceph_check_caps(ci, 0); + + if (test_and_clear_bit(CEPH_I_WORK_FLUSH_SNAPS, &ci->i_work_mask)) + ceph_flush_snaps(ci, NULL); + + iput(inode); +} + +static const char *ceph_encrypted_get_link(struct dentry *dentry, + struct inode *inode, + struct delayed_call *done) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + + if (!dentry) + return ERR_PTR(-ECHILD); + + return fscrypt_get_symlink(inode, ci->i_symlink, i_size_read(inode), + done); +} + +static int ceph_encrypted_symlink_getattr(struct mnt_idmap *idmap, + const struct path *path, + struct kstat *stat, u32 request_mask, + unsigned int query_flags) +{ + int ret; + + ret = ceph_getattr(idmap, path, stat, request_mask, query_flags); + if (ret) + return ret; + return fscrypt_symlink_getattr(path, stat); +} + +/* + * symlinks + */ +static const struct inode_operations ceph_symlink_iops = { + .get_link = simple_get_link, + .setattr = ceph_setattr, + .getattr = ceph_getattr, + .listxattr = ceph_listxattr, +}; + +static const struct inode_operations ceph_encrypted_symlink_iops = { + .get_link = ceph_encrypted_get_link, + .setattr = ceph_setattr, + .getattr = ceph_encrypted_symlink_getattr, + .listxattr = ceph_listxattr, +}; + +/* + * Transfer the encrypted last block to the MDS and the MDS + * will help update it when truncating a smaller size. + * + * We don't support a PAGE_SIZE that is smaller than the + * CEPH_FSCRYPT_BLOCK_SIZE. + */ +static int fill_fscrypt_truncate(struct inode *inode, + struct ceph_mds_request *req, + struct iattr *attr) +{ + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + int boff = attr->ia_size % CEPH_FSCRYPT_BLOCK_SIZE; + loff_t pos, orig_pos = round_down(attr->ia_size, + CEPH_FSCRYPT_BLOCK_SIZE); + u64 block = orig_pos >> CEPH_FSCRYPT_BLOCK_SHIFT; + struct ceph_pagelist *pagelist = NULL; + struct kvec iov = {0}; + struct iov_iter iter; + struct page *page = NULL; + struct ceph_fscrypt_truncate_size_header header; + int retry_op = 0; + int len = CEPH_FSCRYPT_BLOCK_SIZE; + loff_t i_size = i_size_read(inode); + int got, ret, issued; + u64 objver; + + ret = __ceph_get_caps(inode, NULL, CEPH_CAP_FILE_RD, 0, -1, &got); + if (ret < 0) + return ret; + + issued = __ceph_caps_issued(ci, NULL); + + doutc(cl, "size %lld -> %lld got cap refs on %s, issued %s\n", + i_size, attr->ia_size, ceph_cap_string(got), + ceph_cap_string(issued)); + + /* Try to writeback the dirty pagecaches */ + if (issued & (CEPH_CAP_FILE_BUFFER)) { + loff_t lend = orig_pos + CEPH_FSCRYPT_BLOCK_SIZE - 1; + + ret = filemap_write_and_wait_range(inode->i_mapping, + orig_pos, lend); + if (ret < 0) + goto out; + } + + page = __page_cache_alloc(GFP_KERNEL); + if (page == NULL) { + ret = -ENOMEM; + goto out; + } + + pagelist = ceph_pagelist_alloc(GFP_KERNEL); + if (!pagelist) { + ret = -ENOMEM; + goto out; + } + + iov.iov_base = kmap_local_page(page); + iov.iov_len = len; + iov_iter_kvec(&iter, READ, &iov, 1, len); + + pos = orig_pos; + ret = __ceph_sync_read(inode, &pos, &iter, &retry_op, &objver); + if (ret < 0) + goto out; + + /* Insert the header first */ + header.ver = 1; + header.compat = 1; + header.change_attr = cpu_to_le64(inode_peek_iversion_raw(inode)); + + /* + * Always set the block_size to CEPH_FSCRYPT_BLOCK_SIZE, + * because in MDS it may need this to do the truncate. + */ + header.block_size = cpu_to_le32(CEPH_FSCRYPT_BLOCK_SIZE); + + /* + * If we hit a hole here, we should just skip filling + * the fscrypt for the request, because once the fscrypt + * is enabled, the file will be split into many blocks + * with the size of CEPH_FSCRYPT_BLOCK_SIZE, if there + * has a hole, the hole size should be multiple of block + * size. + * + * If the Rados object doesn't exist, it will be set to 0. + */ + if (!objver) { + doutc(cl, "hit hole, ppos %lld < size %lld\n", pos, i_size); + + header.data_len = cpu_to_le32(8 + 8 + 4); + header.file_offset = 0; + ret = 0; + } else { + header.data_len = cpu_to_le32(8 + 8 + 4 + CEPH_FSCRYPT_BLOCK_SIZE); + header.file_offset = cpu_to_le64(orig_pos); + + doutc(cl, "encrypt block boff/bsize %d/%lu\n", boff, + CEPH_FSCRYPT_BLOCK_SIZE); + + /* truncate and zero out the extra contents for the last block */ + memset(iov.iov_base + boff, 0, PAGE_SIZE - boff); + + /* encrypt the last block */ + ret = ceph_fscrypt_encrypt_block_inplace(inode, page, + CEPH_FSCRYPT_BLOCK_SIZE, + 0, block); + if (ret) + goto out; + } + + /* Insert the header */ + ret = ceph_pagelist_append(pagelist, &header, sizeof(header)); + if (ret) + goto out; + + if (header.block_size) { + /* Append the last block contents to pagelist */ + ret = ceph_pagelist_append(pagelist, iov.iov_base, + CEPH_FSCRYPT_BLOCK_SIZE); + if (ret) + goto out; + } + req->r_pagelist = pagelist; +out: + doutc(cl, "%p %llx.%llx size dropping cap refs on %s\n", inode, + ceph_vinop(inode), ceph_cap_string(got)); + ceph_put_cap_refs(ci, got); + if (iov.iov_base) + kunmap_local(iov.iov_base); + if (page) + __free_pages(page, 0); + if (ret && pagelist) + ceph_pagelist_release(pagelist); + return ret; +} + +int __ceph_setattr(struct mnt_idmap *idmap, struct inode *inode, + struct iattr *attr, struct ceph_iattr *cia) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + unsigned int ia_valid = attr->ia_valid; + struct ceph_mds_request *req; + struct ceph_mds_client *mdsc = ceph_sb_to_fs_client(inode->i_sb)->mdsc; + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_cap_flush *prealloc_cf; + loff_t isize = i_size_read(inode); + int issued; + int release = 0, dirtied = 0; + int mask = 0; + int err = 0; + int inode_dirty_flags = 0; + bool lock_snap_rwsem = false; + bool fill_fscrypt; + int truncate_retry = 20; /* The RMW will take around 50ms */ + struct dentry *dentry; + char *path; + bool do_sync = false; + + dentry = d_find_alias(inode); + if (!dentry) { + do_sync = true; + } else { + struct ceph_path_info path_info = {0}; + path = ceph_mdsc_build_path(mdsc, dentry, &path_info, 0); + if (IS_ERR(path)) { + do_sync = true; + err = 0; + } else { + err = ceph_mds_check_access(mdsc, path, MAY_WRITE); + } + ceph_mdsc_free_path_info(&path_info); + dput(dentry); + + /* For none EACCES cases will let the MDS do the mds auth check */ + if (err == -EACCES) { + return err; + } else if (err < 0) { + do_sync = true; + err = 0; + } + } + +retry: + prealloc_cf = ceph_alloc_cap_flush(); + if (!prealloc_cf) + return -ENOMEM; + + req = ceph_mdsc_create_request(mdsc, CEPH_MDS_OP_SETATTR, + USE_AUTH_MDS); + if (IS_ERR(req)) { + ceph_free_cap_flush(prealloc_cf); + return PTR_ERR(req); + } + + fill_fscrypt = false; + spin_lock(&ci->i_ceph_lock); + issued = __ceph_caps_issued(ci, NULL); + + if (!ci->i_head_snapc && + (issued & (CEPH_CAP_ANY_EXCL | CEPH_CAP_FILE_WR))) { + lock_snap_rwsem = true; + if (!down_read_trylock(&mdsc->snap_rwsem)) { + spin_unlock(&ci->i_ceph_lock); + down_read(&mdsc->snap_rwsem); + spin_lock(&ci->i_ceph_lock); + issued = __ceph_caps_issued(ci, NULL); + } + } + + doutc(cl, "%p %llx.%llx issued %s\n", inode, ceph_vinop(inode), + ceph_cap_string(issued)); +#if IS_ENABLED(CONFIG_FS_ENCRYPTION) + if (cia && cia->fscrypt_auth) { + u32 len = ceph_fscrypt_auth_len(cia->fscrypt_auth); + + if (len > sizeof(*cia->fscrypt_auth)) { + err = -EINVAL; + spin_unlock(&ci->i_ceph_lock); + goto out; + } + + doutc(cl, "%p %llx.%llx fscrypt_auth len %u to %u)\n", inode, + ceph_vinop(inode), ci->fscrypt_auth_len, len); + + /* It should never be re-set once set */ + WARN_ON_ONCE(ci->fscrypt_auth); + + if (!do_sync && (issued & CEPH_CAP_AUTH_EXCL)) { + dirtied |= CEPH_CAP_AUTH_EXCL; + kfree(ci->fscrypt_auth); + ci->fscrypt_auth = (u8 *)cia->fscrypt_auth; + ci->fscrypt_auth_len = len; + } else if ((issued & CEPH_CAP_AUTH_SHARED) == 0 || + ci->fscrypt_auth_len != len || + memcmp(ci->fscrypt_auth, cia->fscrypt_auth, len)) { + req->r_fscrypt_auth = cia->fscrypt_auth; + mask |= CEPH_SETATTR_FSCRYPT_AUTH; + release |= CEPH_CAP_AUTH_SHARED; + } + cia->fscrypt_auth = NULL; + } +#else + if (cia && cia->fscrypt_auth) { + err = -EINVAL; + spin_unlock(&ci->i_ceph_lock); + goto out; + } +#endif /* CONFIG_FS_ENCRYPTION */ + + if (ia_valid & ATTR_UID) { + kuid_t fsuid = from_vfsuid(idmap, i_user_ns(inode), attr->ia_vfsuid); + + doutc(cl, "%p %llx.%llx uid %d -> %d\n", inode, + ceph_vinop(inode), + from_kuid(&init_user_ns, inode->i_uid), + from_kuid(&init_user_ns, attr->ia_uid)); + if (!do_sync && (issued & CEPH_CAP_AUTH_EXCL)) { + inode->i_uid = fsuid; + dirtied |= CEPH_CAP_AUTH_EXCL; + } else if ((issued & CEPH_CAP_AUTH_SHARED) == 0 || + !uid_eq(fsuid, inode->i_uid)) { + req->r_args.setattr.uid = cpu_to_le32( + from_kuid(&init_user_ns, fsuid)); + mask |= CEPH_SETATTR_UID; + release |= CEPH_CAP_AUTH_SHARED; + } + } + if (ia_valid & ATTR_GID) { + kgid_t fsgid = from_vfsgid(idmap, i_user_ns(inode), attr->ia_vfsgid); + + doutc(cl, "%p %llx.%llx gid %d -> %d\n", inode, + ceph_vinop(inode), + from_kgid(&init_user_ns, inode->i_gid), + from_kgid(&init_user_ns, attr->ia_gid)); + if (!do_sync && (issued & CEPH_CAP_AUTH_EXCL)) { + inode->i_gid = fsgid; + dirtied |= CEPH_CAP_AUTH_EXCL; + } else if ((issued & CEPH_CAP_AUTH_SHARED) == 0 || + !gid_eq(fsgid, inode->i_gid)) { + req->r_args.setattr.gid = cpu_to_le32( + from_kgid(&init_user_ns, fsgid)); + mask |= CEPH_SETATTR_GID; + release |= CEPH_CAP_AUTH_SHARED; + } + } + if (ia_valid & ATTR_MODE) { + doutc(cl, "%p %llx.%llx mode 0%o -> 0%o\n", inode, + ceph_vinop(inode), inode->i_mode, attr->ia_mode); + if (!do_sync && (issued & CEPH_CAP_AUTH_EXCL)) { + inode->i_mode = attr->ia_mode; + dirtied |= CEPH_CAP_AUTH_EXCL; + } else if ((issued & CEPH_CAP_AUTH_SHARED) == 0 || + attr->ia_mode != inode->i_mode) { + inode->i_mode = attr->ia_mode; + req->r_args.setattr.mode = cpu_to_le32(attr->ia_mode); + mask |= CEPH_SETATTR_MODE; + release |= CEPH_CAP_AUTH_SHARED; + } + } + + if (ia_valid & ATTR_ATIME) { + struct timespec64 atime = inode_get_atime(inode); + + doutc(cl, "%p %llx.%llx atime %ptSp -> %ptSp\n", + inode, ceph_vinop(inode), &atime, &attr->ia_atime); + if (!do_sync && (issued & CEPH_CAP_FILE_EXCL)) { + ci->i_time_warp_seq++; + inode_set_atime_to_ts(inode, attr->ia_atime); + dirtied |= CEPH_CAP_FILE_EXCL; + } else if (!do_sync && (issued & CEPH_CAP_FILE_WR) && + timespec64_compare(&atime, + &attr->ia_atime) < 0) { + inode_set_atime_to_ts(inode, attr->ia_atime); + dirtied |= CEPH_CAP_FILE_WR; + } else if ((issued & CEPH_CAP_FILE_SHARED) == 0 || + !timespec64_equal(&atime, &attr->ia_atime)) { + ceph_encode_timespec64(&req->r_args.setattr.atime, + &attr->ia_atime); + mask |= CEPH_SETATTR_ATIME; + release |= CEPH_CAP_FILE_SHARED | + CEPH_CAP_FILE_RD | CEPH_CAP_FILE_WR; + } + } + if (ia_valid & ATTR_SIZE) { + doutc(cl, "%p %llx.%llx size %lld -> %lld\n", inode, + ceph_vinop(inode), isize, attr->ia_size); + /* + * Only when the new size is smaller and not aligned to + * CEPH_FSCRYPT_BLOCK_SIZE will the RMW is needed. + */ + if (IS_ENCRYPTED(inode) && attr->ia_size < isize && + (attr->ia_size % CEPH_FSCRYPT_BLOCK_SIZE)) { + mask |= CEPH_SETATTR_SIZE; + release |= CEPH_CAP_FILE_SHARED | CEPH_CAP_FILE_EXCL | + CEPH_CAP_FILE_RD | CEPH_CAP_FILE_WR; + set_bit(CEPH_MDS_R_FSCRYPT_FILE, &req->r_req_flags); + mask |= CEPH_SETATTR_FSCRYPT_FILE; + req->r_args.setattr.size = + cpu_to_le64(round_up(attr->ia_size, + CEPH_FSCRYPT_BLOCK_SIZE)); + req->r_args.setattr.old_size = + cpu_to_le64(round_up(isize, + CEPH_FSCRYPT_BLOCK_SIZE)); + req->r_fscrypt_file = attr->ia_size; + fill_fscrypt = true; + } else if (!do_sync && (issued & CEPH_CAP_FILE_EXCL) && attr->ia_size >= isize) { + if (attr->ia_size > isize) { + i_size_write(inode, attr->ia_size); + inode->i_blocks = calc_inode_blocks(attr->ia_size); + ci->i_reported_size = attr->ia_size; + dirtied |= CEPH_CAP_FILE_EXCL; + ia_valid |= ATTR_MTIME; + } + } else if ((issued & CEPH_CAP_FILE_SHARED) == 0 || + attr->ia_size != isize) { + mask |= CEPH_SETATTR_SIZE; + release |= CEPH_CAP_FILE_SHARED | CEPH_CAP_FILE_EXCL | + CEPH_CAP_FILE_RD | CEPH_CAP_FILE_WR; + if (IS_ENCRYPTED(inode) && attr->ia_size) { + set_bit(CEPH_MDS_R_FSCRYPT_FILE, &req->r_req_flags); + mask |= CEPH_SETATTR_FSCRYPT_FILE; + req->r_args.setattr.size = + cpu_to_le64(round_up(attr->ia_size, + CEPH_FSCRYPT_BLOCK_SIZE)); + req->r_args.setattr.old_size = + cpu_to_le64(round_up(isize, + CEPH_FSCRYPT_BLOCK_SIZE)); + req->r_fscrypt_file = attr->ia_size; + } else { + req->r_args.setattr.size = cpu_to_le64(attr->ia_size); + req->r_args.setattr.old_size = cpu_to_le64(isize); + req->r_fscrypt_file = 0; + } + } + } + if (ia_valid & ATTR_MTIME) { + struct timespec64 mtime = inode_get_mtime(inode); + + doutc(cl, "%p %llx.%llx mtime %ptSp -> %ptSp\n", + inode, ceph_vinop(inode), &mtime, &attr->ia_mtime); + if (!do_sync && (issued & CEPH_CAP_FILE_EXCL)) { + ci->i_time_warp_seq++; + inode_set_mtime_to_ts(inode, attr->ia_mtime); + dirtied |= CEPH_CAP_FILE_EXCL; + } else if (!do_sync && (issued & CEPH_CAP_FILE_WR) && + timespec64_compare(&mtime, &attr->ia_mtime) < 0) { + inode_set_mtime_to_ts(inode, attr->ia_mtime); + dirtied |= CEPH_CAP_FILE_WR; + } else if ((issued & CEPH_CAP_FILE_SHARED) == 0 || + !timespec64_equal(&mtime, &attr->ia_mtime)) { + ceph_encode_timespec64(&req->r_args.setattr.mtime, + &attr->ia_mtime); + mask |= CEPH_SETATTR_MTIME; + release |= CEPH_CAP_FILE_SHARED | + CEPH_CAP_FILE_RD | CEPH_CAP_FILE_WR; + } + } + + /* these do nothing */ + if (ia_valid & ATTR_CTIME) { + struct timespec64 ictime = inode_get_ctime(inode); + bool only = (ia_valid & (ATTR_SIZE|ATTR_MTIME|ATTR_ATIME| + ATTR_MODE|ATTR_UID|ATTR_GID)) == 0; + doutc(cl, "%p %llx.%llx ctime %ptSp -> %ptSp (%s)\n", + inode, ceph_vinop(inode), &ictime, &attr->ia_ctime, + only ? "ctime only" : "ignored"); + if (only) { + /* + * if kernel wants to dirty ctime but nothing else, + * we need to choose a cap to dirty under, or do + * a almost-no-op setattr + */ + if (issued & CEPH_CAP_AUTH_EXCL) + dirtied |= CEPH_CAP_AUTH_EXCL; + else if (issued & CEPH_CAP_FILE_EXCL) + dirtied |= CEPH_CAP_FILE_EXCL; + else if (issued & CEPH_CAP_XATTR_EXCL) + dirtied |= CEPH_CAP_XATTR_EXCL; + else + mask |= CEPH_SETATTR_CTIME; + } + } + if (ia_valid & ATTR_FILE) + doutc(cl, "%p %llx.%llx ATTR_FILE ... hrm!\n", inode, + ceph_vinop(inode)); + + if (dirtied) { + inode_dirty_flags = __ceph_mark_dirty_caps(ci, dirtied, + &prealloc_cf); + inode_set_ctime_to_ts(inode, attr->ia_ctime); + inode_inc_iversion_raw(inode); + } + + release &= issued; + spin_unlock(&ci->i_ceph_lock); + if (lock_snap_rwsem) { + up_read(&mdsc->snap_rwsem); + lock_snap_rwsem = false; + } + + if (inode_dirty_flags) + __mark_inode_dirty(inode, inode_dirty_flags); + + if (mask) { + req->r_inode = inode; + ihold(inode); + req->r_inode_drop = release; + req->r_args.setattr.mask = cpu_to_le32(mask); + req->r_num_caps = 1; + req->r_stamp = attr->ia_ctime; + if (fill_fscrypt) { + err = fill_fscrypt_truncate(inode, req, attr); + if (err) + goto out; + } + + /* + * The truncate request will return -EAGAIN when the + * last block has been updated just before the MDS + * successfully gets the xlock for the FILE lock. To + * avoid corrupting the file contents we need to retry + * it. + */ + err = ceph_mdsc_do_request(mdsc, NULL, req); + if (err == -EAGAIN && truncate_retry--) { + doutc(cl, "%p %llx.%llx result=%d (%s locally, %d remote), retry it!\n", + inode, ceph_vinop(inode), err, + ceph_cap_string(dirtied), mask); + ceph_mdsc_put_request(req); + ceph_free_cap_flush(prealloc_cf); + goto retry; + } + } +out: + doutc(cl, "%p %llx.%llx result=%d (%s locally, %d remote)\n", inode, + ceph_vinop(inode), err, ceph_cap_string(dirtied), mask); + + ceph_mdsc_put_request(req); + ceph_free_cap_flush(prealloc_cf); + + if (err >= 0 && (mask & CEPH_SETATTR_SIZE)) + __ceph_do_pending_vmtruncate(inode); + + return err; +} + +/* + * setattr + */ +int ceph_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +{ + struct inode *inode = d_inode(dentry); + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + int err; + + if (ceph_snap(inode) != CEPH_NOSNAP) + return -EROFS; + + if (ceph_inode_is_shutdown(inode)) + return -ESTALE; + + err = fscrypt_prepare_setattr(dentry, attr); + if (err) + return err; + + err = setattr_prepare(idmap, dentry, attr); + if (err != 0) + return err; + + if ((attr->ia_valid & ATTR_SIZE) && + attr->ia_size > max(i_size_read(inode), fsc->max_file_size)) + return -EFBIG; + + if ((attr->ia_valid & ATTR_SIZE) && + ceph_quota_is_max_bytes_exceeded(inode, attr->ia_size)) + return -EDQUOT; + + err = __ceph_setattr(idmap, inode, attr, NULL); + + if (err >= 0 && (attr->ia_valid & ATTR_MODE)) + err = posix_acl_chmod(idmap, dentry, attr->ia_mode); + + return err; +} + +int ceph_try_to_choose_auth_mds(struct inode *inode, int mask) +{ + int issued = ceph_caps_issued(ceph_inode(inode)); + + /* + * If any 'x' caps is issued we can just choose the auth MDS + * instead of the random replica MDSes. Because only when the + * Locker is in LOCK_EXEC state will the loner client could + * get the 'x' caps. And if we send the getattr requests to + * any replica MDS it must auth pin and tries to rdlock from + * the auth MDS, and then the auth MDS need to do the Locker + * state transition to LOCK_SYNC. And after that the lock state + * will change back. + * + * This cost much when doing the Locker state transition and + * usually will need to revoke caps from clients. + * + * And for the 'Xs' caps for getxattr we will also choose the + * auth MDS, because the MDS side code is buggy due to setxattr + * won't notify the replica MDSes when the values changed and + * the replica MDS will return the old values. Though we will + * fix it in MDS code, but this still makes sense for old ceph. + */ + if (((mask & CEPH_CAP_ANY_SHARED) && (issued & CEPH_CAP_ANY_EXCL)) + || (mask & (CEPH_STAT_RSTAT | CEPH_STAT_CAP_XATTR))) + return USE_AUTH_MDS; + else + return USE_ANY_MDS; +} + +/* + * Verify that we have a lease on the given mask. If not, + * do a getattr against an mds. + */ +int __ceph_do_getattr(struct inode *inode, struct page *locked_page, + int mask, bool force) +{ + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(inode->i_sb); + struct ceph_client *cl = fsc->client; + struct ceph_mds_client *mdsc = fsc->mdsc; + struct ceph_mds_request *req; + int mode; + int err; + + if (ceph_snap(inode) == CEPH_SNAPDIR) { + doutc(cl, "inode %p %llx.%llx SNAPDIR\n", inode, + ceph_vinop(inode)); + return 0; + } + + doutc(cl, "inode %p %llx.%llx mask %s mode 0%o\n", inode, + ceph_vinop(inode), ceph_cap_string(mask), inode->i_mode); + if (!force && ceph_caps_issued_mask_metric(ceph_inode(inode), mask, 1)) + return 0; + + mode = ceph_try_to_choose_auth_mds(inode, mask); + req = ceph_mdsc_create_request(mdsc, CEPH_MDS_OP_GETATTR, mode); + if (IS_ERR(req)) + return PTR_ERR(req); + req->r_inode = inode; + ihold(inode); + req->r_num_caps = 1; + req->r_args.getattr.mask = cpu_to_le32(mask); + req->r_locked_page = locked_page; + err = ceph_mdsc_do_request(mdsc, NULL, req); + if (locked_page && err == 0) { + u64 inline_version = req->r_reply_info.targeti.inline_version; + if (inline_version == 0) { + /* the reply is supposed to contain inline data */ + err = -EINVAL; + } else if (inline_version == CEPH_INLINE_NONE || + inline_version == 1) { + err = -ENODATA; + } else { + err = req->r_reply_info.targeti.inline_len; + } + } + ceph_mdsc_put_request(req); + doutc(cl, "result=%d\n", err); + return err; +} + +int ceph_do_getvxattr(struct inode *inode, const char *name, void *value, + size_t size) +{ + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(inode->i_sb); + struct ceph_client *cl = fsc->client; + struct ceph_mds_client *mdsc = fsc->mdsc; + struct ceph_mds_request *req; + int mode = USE_AUTH_MDS; + int err; + char *xattr_value; + size_t xattr_value_len; + + req = ceph_mdsc_create_request(mdsc, CEPH_MDS_OP_GETVXATTR, mode); + if (IS_ERR(req)) { + err = -ENOMEM; + goto out; + } + + req->r_feature_needed = CEPHFS_FEATURE_OP_GETVXATTR; + req->r_path2 = kstrdup(name, GFP_NOFS); + if (!req->r_path2) { + err = -ENOMEM; + goto put; + } + + ihold(inode); + req->r_inode = inode; + err = ceph_mdsc_do_request(mdsc, NULL, req); + if (err < 0) + goto put; + + xattr_value = req->r_reply_info.xattr_info.xattr_value; + xattr_value_len = req->r_reply_info.xattr_info.xattr_value_len; + + doutc(cl, "xattr_value_len:%zu, size:%zu\n", xattr_value_len, size); + + err = (int)xattr_value_len; + if (size == 0) + goto put; + + if (xattr_value_len > size) { + err = -ERANGE; + goto put; + } + + memcpy(value, xattr_value, xattr_value_len); +put: + ceph_mdsc_put_request(req); +out: + doutc(cl, "result=%d\n", err); + return err; +} + + +/* + * Check inode permissions. We verify we have a valid value for + * the AUTH cap, then call the generic handler. + */ +int ceph_permission(struct mnt_idmap *idmap, struct inode *inode, + int mask) +{ + int err; + + if (mask & MAY_NOT_BLOCK) + return -ECHILD; + + err = ceph_do_getattr(inode, CEPH_CAP_AUTH_SHARED, false); + + if (!err) + err = generic_permission(idmap, inode, mask); + return err; +} + +/* Craft a mask of needed caps given a set of requested statx attrs. */ +static int statx_to_caps(u32 want, umode_t mode) +{ + int mask = 0; + + if (want & (STATX_MODE|STATX_UID|STATX_GID|STATX_CTIME|STATX_BTIME|STATX_CHANGE_COOKIE)) + mask |= CEPH_CAP_AUTH_SHARED; + + if (want & (STATX_NLINK|STATX_CTIME|STATX_CHANGE_COOKIE)) { + /* + * The link count for directories depends on inode->i_subdirs, + * and that is only updated when Fs caps are held. + */ + if (S_ISDIR(mode)) + mask |= CEPH_CAP_FILE_SHARED; + else + mask |= CEPH_CAP_LINK_SHARED; + } + + if (want & (STATX_ATIME|STATX_MTIME|STATX_CTIME|STATX_SIZE|STATX_BLOCKS|STATX_CHANGE_COOKIE)) + mask |= CEPH_CAP_FILE_SHARED; + + if (want & (STATX_CTIME|STATX_CHANGE_COOKIE)) + mask |= CEPH_CAP_XATTR_SHARED; + + return mask; +} + +/* + * Get all the attributes. If we have sufficient caps for the requested attrs, + * then we can avoid talking to the MDS at all. + */ +int ceph_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags) +{ + struct inode *inode = d_inode(path->dentry); + struct super_block *sb = inode->i_sb; + struct ceph_inode_info *ci = ceph_inode(inode); + u32 valid_mask = STATX_BASIC_STATS; + int err = 0; + + if (ceph_inode_is_shutdown(inode)) + return -ESTALE; + + /* Skip the getattr altogether if we're asked not to sync */ + if ((flags & AT_STATX_SYNC_TYPE) != AT_STATX_DONT_SYNC) { + err = ceph_do_getattr(inode, + statx_to_caps(request_mask, inode->i_mode), + flags & AT_STATX_FORCE_SYNC); + if (err) + return err; + } + + generic_fillattr(idmap, request_mask, inode, stat); + stat->ino = ceph_present_inode(inode); + + /* + * btime on newly-allocated inodes is 0, so if this is still set to + * that, then assume that it's not valid. + */ + if (ci->i_btime.tv_sec || ci->i_btime.tv_nsec) { + stat->btime = ci->i_btime; + valid_mask |= STATX_BTIME; + } + + if (request_mask & STATX_CHANGE_COOKIE) { + stat->change_cookie = inode_peek_iversion_raw(inode); + valid_mask |= STATX_CHANGE_COOKIE; + } + + if (ceph_snap(inode) == CEPH_NOSNAP) + stat->dev = sb->s_dev; + else + stat->dev = ci->i_snapid_map ? ci->i_snapid_map->dev : 0; + + if (S_ISDIR(inode->i_mode)) { + if (ceph_test_mount_opt(ceph_sb_to_fs_client(sb), RBYTES)) { + stat->size = ci->i_rbytes; + } else if (ceph_snap(inode) == CEPH_SNAPDIR) { + struct ceph_inode_info *pci; + struct ceph_snap_realm *realm; + struct inode *parent; + + parent = ceph_lookup_inode(sb, ceph_ino(inode)); + if (IS_ERR(parent)) + return PTR_ERR(parent); + + pci = ceph_inode(parent); + spin_lock(&pci->i_ceph_lock); + realm = pci->i_snap_realm; + if (realm) + stat->size = realm->num_snaps; + else + stat->size = 0; + spin_unlock(&pci->i_ceph_lock); + iput(parent); + } else { + stat->size = ci->i_files + ci->i_subdirs; + } + stat->blocks = 0; + stat->blksize = 65536; + /* + * Some applications rely on the number of st_nlink + * value on directories to be either 0 (if unlinked) + * or 2 + number of subdirectories. + */ + if (stat->nlink == 1) + /* '.' + '..' + subdirs */ + stat->nlink = 1 + 1 + ci->i_subdirs; + } + + stat->attributes |= STATX_ATTR_CHANGE_MONOTONIC; + if (IS_ENCRYPTED(inode)) + stat->attributes |= STATX_ATTR_ENCRYPTED; + stat->attributes_mask |= (STATX_ATTR_CHANGE_MONOTONIC | + STATX_ATTR_ENCRYPTED); + + stat->result_mask = request_mask & valid_mask; + return err; +} + +void ceph_inode_shutdown(struct inode *inode) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct rb_node *p; + int iputs = 0; + bool invalidate = false; + + spin_lock(&ci->i_ceph_lock); + set_bit(CEPH_I_SHUTDOWN_BIT, &ci->i_ceph_flags); + p = rb_first(&ci->i_caps); + while (p) { + struct ceph_cap *cap = rb_entry(p, struct ceph_cap, ci_node); + + p = rb_next(p); + iputs += ceph_purge_inode_cap(inode, cap, &invalidate); + } + spin_unlock(&ci->i_ceph_lock); + + if (invalidate) + ceph_queue_invalidate(inode); + while (iputs--) + iput(inode); +} diff --git a/fs/ceph/io.c b/fs/ceph/io.c new file mode 100644 index 000000000..2d10f49c9 --- /dev/null +++ b/fs/ceph/io.c @@ -0,0 +1,213 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2016 Trond Myklebust + * Copyright (c) 2019 Jeff Layton + * + * I/O and data path helper functionality. + * + * Heavily borrowed from equivalent code in fs/nfs/io.c + */ + +#include + +#include +#include +#include +#include + +#include "super.h" +#include "io.h" + +/* Call with exclusively locked inode->i_rwsem */ +static void ceph_block_o_direct(struct ceph_inode_info *ci, struct inode *inode) +{ + bool is_odirect; + + lockdep_assert_held_write(&inode->i_rwsem); + + spin_lock(&ci->i_ceph_lock); + /* ensure that bit state is consistent */ + smp_mb__before_atomic(); + is_odirect = READ_ONCE(ci->i_ceph_flags) & CEPH_I_ODIRECT; + if (is_odirect) { + clear_bit(CEPH_I_ODIRECT_BIT, &ci->i_ceph_flags); + /* ensure modified bit is visible */ + smp_mb__after_atomic(); + } + spin_unlock(&ci->i_ceph_lock); + + if (is_odirect) + inode_dio_wait(inode); +} + +/** + * ceph_start_io_read - declare the file is being used for buffered reads + * @inode: file inode + * + * Declare that a buffered read operation is about to start, and ensure + * that we block all direct I/O. + * On exit, the function ensures that the CEPH_I_ODIRECT flag is unset, + * and holds a shared lock on inode->i_rwsem to ensure that the flag + * cannot be changed. + * In practice, this means that buffered read operations are allowed to + * execute in parallel, thanks to the shared lock, whereas direct I/O + * operations need to wait to grab an exclusive lock in order to set + * CEPH_I_ODIRECT. + * Note that buffered writes and truncates both take a write lock on + * inode->i_rwsem, meaning that those are serialised w.r.t. the reads. + */ +int ceph_start_io_read(struct inode *inode) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + bool is_odirect; + int err; + + /* Be an optimist! */ + err = down_read_killable(&inode->i_rwsem); + if (err) + return err; + + spin_lock(&ci->i_ceph_lock); + /* ensure that bit state is consistent */ + smp_mb__before_atomic(); + is_odirect = READ_ONCE(ci->i_ceph_flags) & CEPH_I_ODIRECT; + spin_unlock(&ci->i_ceph_lock); + if (!is_odirect) + return 0; + up_read(&inode->i_rwsem); + + /* Slow path.... */ + err = down_write_killable(&inode->i_rwsem); + if (err) + return err; + + ceph_block_o_direct(ci, inode); + downgrade_write(&inode->i_rwsem); + + return 0; +} + +/** + * ceph_end_io_read - declare that the buffered read operation is done + * @inode: file inode + * + * Declare that a buffered read operation is done, and release the shared + * lock on inode->i_rwsem. + */ +void +ceph_end_io_read(struct inode *inode) +{ + up_read(&inode->i_rwsem); +} + +/** + * ceph_start_io_write - declare the file is being used for buffered writes + * @inode: file inode + * + * Declare that a buffered write operation is about to start, and ensure + * that we block all direct I/O. + */ +int ceph_start_io_write(struct inode *inode) +{ + int err = down_write_killable(&inode->i_rwsem); + if (!err) + ceph_block_o_direct(ceph_inode(inode), inode); + return err; +} + +/** + * ceph_end_io_write - declare that the buffered write operation is done + * @inode: file inode + * + * Declare that a buffered write operation is done, and release the + * lock on inode->i_rwsem. + */ +void +ceph_end_io_write(struct inode *inode) +{ + up_write(&inode->i_rwsem); +} + +/* Call with exclusively locked inode->i_rwsem */ +static void ceph_block_buffered(struct ceph_inode_info *ci, struct inode *inode) +{ + bool is_odirect; + + lockdep_assert_held_write(&inode->i_rwsem); + + spin_lock(&ci->i_ceph_lock); + /* ensure that bit state is consistent */ + smp_mb__before_atomic(); + is_odirect = READ_ONCE(ci->i_ceph_flags) & CEPH_I_ODIRECT; + if (!is_odirect) { + set_bit(CEPH_I_ODIRECT_BIT, &ci->i_ceph_flags); + /* ensure modified bit is visible */ + smp_mb__after_atomic(); + } + spin_unlock(&ci->i_ceph_lock); + + if (!is_odirect) { + /* FIXME: unmap_mapping_range? */ + filemap_write_and_wait(inode->i_mapping); + } +} + +/** + * ceph_start_io_direct - declare the file is being used for direct i/o + * @inode: file inode + * + * Declare that a direct I/O operation is about to start, and ensure + * that we block all buffered I/O. + * On exit, the function ensures that the CEPH_I_ODIRECT flag is set, + * and holds a shared lock on inode->i_rwsem to ensure that the flag + * cannot be changed. + * In practice, this means that direct I/O operations are allowed to + * execute in parallel, thanks to the shared lock, whereas buffered I/O + * operations need to wait to grab an exclusive lock in order to clear + * CEPH_I_ODIRECT. + * Note that buffered writes and truncates both take a write lock on + * inode->i_rwsem, meaning that those are serialised w.r.t. O_DIRECT. + */ +int ceph_start_io_direct(struct inode *inode) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + bool is_odirect; + int err; + + /* Be an optimist! */ + err = down_read_killable(&inode->i_rwsem); + if (err) + return err; + + spin_lock(&ci->i_ceph_lock); + /* ensure that bit state is consistent */ + smp_mb__before_atomic(); + is_odirect = READ_ONCE(ci->i_ceph_flags) & CEPH_I_ODIRECT; + spin_unlock(&ci->i_ceph_lock); + if (is_odirect) + return 0; + up_read(&inode->i_rwsem); + + /* Slow path.... */ + err = down_write_killable(&inode->i_rwsem); + if (err) + return err; + + ceph_block_buffered(ci, inode); + downgrade_write(&inode->i_rwsem); + + return 0; +} + +/** + * ceph_end_io_direct - declare that the direct i/o operation is done + * @inode: file inode + * + * Declare that a direct I/O operation is done, and release the shared + * lock on inode->i_rwsem. + */ +void +ceph_end_io_direct(struct inode *inode) +{ + up_read(&inode->i_rwsem); +} diff --git a/fs/ceph/io.h b/fs/ceph/io.h new file mode 100644 index 000000000..79029825e --- /dev/null +++ b/fs/ceph/io.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _FS_CEPH_IO_H +#define _FS_CEPH_IO_H + +#include + +int __must_check ceph_start_io_read(struct inode *inode); +void ceph_end_io_read(struct inode *inode); +int __must_check ceph_start_io_write(struct inode *inode); +void ceph_end_io_write(struct inode *inode); +int __must_check ceph_start_io_direct(struct inode *inode); +void ceph_end_io_direct(struct inode *inode); + +#endif /* FS_CEPH_IO_H */ diff --git a/fs/ceph/ioctl.c b/fs/ceph/ioctl.c new file mode 100644 index 000000000..12cb50e0a --- /dev/null +++ b/fs/ceph/ioctl.c @@ -0,0 +1,438 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +#include "super.h" +#include "mds_client.h" +#include "ioctl.h" +#include +#include + +/* + * ioctls + */ + +/* + * get and set the file layout + */ +static long ceph_ioctl_get_layout(struct file *file, void __user *arg) +{ + struct ceph_inode_info *ci = ceph_inode(file_inode(file)); + struct ceph_ioctl_layout l; + int err; + + err = ceph_do_getattr(file_inode(file), CEPH_STAT_CAP_LAYOUT, false); + if (!err) { + l.stripe_unit = ci->i_layout.stripe_unit; + l.stripe_count = ci->i_layout.stripe_count; + l.object_size = ci->i_layout.object_size; + l.data_pool = ci->i_layout.pool_id; + l.preferred_osd = -1; + if (copy_to_user(arg, &l, sizeof(l))) + return -EFAULT; + } + + return err; +} + +static long __validate_layout(struct ceph_mds_client *mdsc, + struct ceph_ioctl_layout *l) +{ + int i, err; + + /* validate striping parameters */ + if ((l->object_size & ~PAGE_MASK) || + (l->stripe_unit & ~PAGE_MASK) || + ((unsigned)l->stripe_unit != 0 && + ((unsigned)l->object_size % (unsigned)l->stripe_unit))) + return -EINVAL; + + /* make sure it's a valid data pool */ + mutex_lock(&mdsc->mutex); + err = -EINVAL; + for (i = 0; i < mdsc->mdsmap->m_num_data_pg_pools; i++) + if (mdsc->mdsmap->m_data_pg_pools[i] == l->data_pool) { + err = 0; + break; + } + mutex_unlock(&mdsc->mutex); + if (err) + return err; + + return 0; +} + +static long ceph_ioctl_set_layout(struct file *file, void __user *arg) +{ + struct inode *inode = file_inode(file); + struct ceph_mds_client *mdsc = ceph_sb_to_fs_client(inode->i_sb)->mdsc; + struct ceph_mds_request *req; + struct ceph_ioctl_layout l; + struct ceph_inode_info *ci = ceph_inode(file_inode(file)); + struct ceph_ioctl_layout nl; + int err; + + if (!inode_owner_or_capable(file_mnt_idmap(file), inode)) + return -EACCES; + + if (copy_from_user(&l, arg, sizeof(l))) + return -EFAULT; + + /* validate changed params against current layout */ + err = ceph_do_getattr(file_inode(file), CEPH_STAT_CAP_LAYOUT, false); + if (err) + return err; + + memset(&nl, 0, sizeof(nl)); + if (l.stripe_count) + nl.stripe_count = l.stripe_count; + else + nl.stripe_count = ci->i_layout.stripe_count; + if (l.stripe_unit) + nl.stripe_unit = l.stripe_unit; + else + nl.stripe_unit = ci->i_layout.stripe_unit; + if (l.object_size) + nl.object_size = l.object_size; + else + nl.object_size = ci->i_layout.object_size; + if (l.data_pool) + nl.data_pool = l.data_pool; + else + nl.data_pool = ci->i_layout.pool_id; + + /* this is obsolete, and always -1 */ + nl.preferred_osd = -1; + + err = __validate_layout(mdsc, &nl); + if (err) + return err; + + req = ceph_mdsc_create_request(mdsc, CEPH_MDS_OP_SETLAYOUT, + USE_AUTH_MDS); + if (IS_ERR(req)) + return PTR_ERR(req); + req->r_inode = inode; + ihold(inode); + req->r_num_caps = 1; + + req->r_inode_drop = CEPH_CAP_FILE_SHARED | CEPH_CAP_FILE_EXCL; + + req->r_args.setlayout.layout.fl_stripe_unit = + cpu_to_le32(l.stripe_unit); + req->r_args.setlayout.layout.fl_stripe_count = + cpu_to_le32(l.stripe_count); + req->r_args.setlayout.layout.fl_object_size = + cpu_to_le32(l.object_size); + req->r_args.setlayout.layout.fl_pg_pool = cpu_to_le32(l.data_pool); + + err = ceph_mdsc_do_request(mdsc, NULL, req); + ceph_mdsc_put_request(req); + return err; +} + +/* + * Set a layout policy on a directory inode. All items in the tree + * rooted at this inode will inherit this layout on creation, + * (It doesn't apply retroactively ) + * unless a subdirectory has its own layout policy. + */ +static long ceph_ioctl_set_layout_policy (struct file *file, void __user *arg) +{ + struct inode *inode = file_inode(file); + struct ceph_mds_request *req; + struct ceph_ioctl_layout l; + int err; + struct ceph_mds_client *mdsc = ceph_sb_to_fs_client(inode->i_sb)->mdsc; + + if (!inode_owner_or_capable(file_mnt_idmap(file), inode)) + return -EACCES; + + /* copy and validate */ + if (copy_from_user(&l, arg, sizeof(l))) + return -EFAULT; + + err = __validate_layout(mdsc, &l); + if (err) + return err; + + req = ceph_mdsc_create_request(mdsc, CEPH_MDS_OP_SETDIRLAYOUT, + USE_AUTH_MDS); + + if (IS_ERR(req)) + return PTR_ERR(req); + req->r_inode = inode; + ihold(inode); + req->r_num_caps = 1; + + req->r_args.setlayout.layout.fl_stripe_unit = + cpu_to_le32(l.stripe_unit); + req->r_args.setlayout.layout.fl_stripe_count = + cpu_to_le32(l.stripe_count); + req->r_args.setlayout.layout.fl_object_size = + cpu_to_le32(l.object_size); + req->r_args.setlayout.layout.fl_pg_pool = + cpu_to_le32(l.data_pool); + + err = ceph_mdsc_do_request(mdsc, inode, req); + ceph_mdsc_put_request(req); + return err; +} + +/* + * Return object name, size/offset information, and location (OSD + * number, network address) for a given file offset. + */ +static long ceph_ioctl_get_dataloc(struct file *file, void __user *arg) +{ + struct ceph_ioctl_dataloc dl; + struct inode *inode = file_inode(file); + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_osd_client *osdc = + &ceph_sb_to_fs_client(inode->i_sb)->client->osdc; + struct ceph_object_locator oloc; + CEPH_DEFINE_OID_ONSTACK(oid); + u32 xlen; + u64 tmp; + struct ceph_pg pgid; + int r; + + /* copy and validate */ + if (copy_from_user(&dl, arg, sizeof(dl))) + return -EFAULT; + + down_read(&osdc->lock); + ceph_calc_file_object_mapping(&ci->i_layout, dl.file_offset, 1, + &dl.object_no, &dl.object_offset, &xlen); + dl.file_offset -= dl.object_offset; + dl.object_size = ci->i_layout.object_size; + dl.block_size = ci->i_layout.stripe_unit; + + /* block_offset = object_offset % block_size */ + tmp = dl.object_offset; + dl.block_offset = do_div(tmp, dl.block_size); + + snprintf(dl.object_name, sizeof(dl.object_name), "%llx.%08llx", + ceph_ino(inode), dl.object_no); + + oloc.pool = ci->i_layout.pool_id; + oloc.pool_ns = ceph_try_get_string(ci->i_layout.pool_ns); + ceph_oid_printf(&oid, "%s", dl.object_name); + + r = ceph_object_locator_to_pg(osdc->osdmap, &oid, &oloc, &pgid); + + ceph_oloc_destroy(&oloc); + if (r < 0) { + up_read(&osdc->lock); + return r; + } + + dl.osd = ceph_pg_to_acting_primary(osdc->osdmap, &pgid); + if (dl.osd >= 0) { + struct ceph_entity_addr *a = + ceph_osd_addr(osdc->osdmap, dl.osd); + if (a) + memcpy(&dl.osd_addr, &a->in_addr, sizeof(dl.osd_addr)); + } else { + memset(&dl.osd_addr, 0, sizeof(dl.osd_addr)); + } + up_read(&osdc->lock); + + /* send result back to user */ + if (copy_to_user(arg, &dl, sizeof(dl))) + return -EFAULT; + + return 0; +} + +static long ceph_ioctl_lazyio(struct file *file) +{ + struct ceph_file_info *fi = file->private_data; + struct inode *inode = file_inode(file); + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_mds_client *mdsc = ceph_inode_to_fs_client(inode)->mdsc; + struct ceph_client *cl = mdsc->fsc->client; + bool is_file_already_lazy = false; + + spin_lock(&ci->i_ceph_lock); + if ((fi->fmode & CEPH_FILE_MODE_LAZY) == 0) { + fi->fmode |= CEPH_FILE_MODE_LAZY; + ci->i_nr_by_mode[ffs(CEPH_FILE_MODE_LAZY)]++; + __ceph_touch_fmode(ci, mdsc, fi->fmode); + } else { + is_file_already_lazy = true; + } + spin_unlock(&ci->i_ceph_lock); + + if (is_file_already_lazy) { + doutc(cl, "file %p %p %llx.%llx already lazy\n", file, inode, + ceph_vinop(inode)); + } else { + doutc(cl, "file %p %p %llx.%llx marked lazy\n", file, inode, + ceph_vinop(inode)); + + ceph_check_caps(ci, 0); + } + + return 0; +} + +static long ceph_ioctl_syncio(struct file *file) +{ + struct ceph_file_info *fi = file->private_data; + + fi->flags |= CEPH_F_SYNC; + return 0; +} + +static int vet_mds_for_fscrypt(struct file *file) +{ + int i, ret = -EOPNOTSUPP; + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(file_inode(file)->i_sb); + + mutex_lock(&mdsc->mutex); + for (i = 0; i < mdsc->max_sessions; i++) { + struct ceph_mds_session *s = mdsc->sessions[i]; + + if (!s) + continue; + if (test_bit(CEPHFS_FEATURE_ALTERNATE_NAME, &s->s_features)) + ret = 0; + break; + } + mutex_unlock(&mdsc->mutex); + return ret; +} + +static long ceph_set_encryption_policy(struct file *file, unsigned long arg) +{ + int ret, got = 0; + struct inode *inode = file_inode(file); + struct ceph_inode_info *ci = ceph_inode(inode); + + /* encrypted directories can't have striped layout */ + if (ci->i_layout.stripe_count > 1) + return -EINVAL; + + ret = vet_mds_for_fscrypt(file); + if (ret) + return ret; + + /* + * Ensure we hold these caps so that we _know_ that the rstats check + * in the empty_dir check is reliable. + */ + ret = ceph_get_caps(file, CEPH_CAP_FILE_SHARED, 0, -1, &got); + if (ret) + return ret; + + ret = fscrypt_ioctl_set_policy(file, (const void __user *)arg); + if (got) + ceph_put_cap_refs(ci, got); + + return ret; +} + +static const char *ceph_ioctl_cmd_name(const unsigned int cmd) +{ + switch (cmd) { + case CEPH_IOC_GET_LAYOUT: + return "get_layout"; + case CEPH_IOC_SET_LAYOUT: + return "set_layout"; + case CEPH_IOC_SET_LAYOUT_POLICY: + return "set_layout_policy"; + case CEPH_IOC_GET_DATALOC: + return "get_dataloc"; + case CEPH_IOC_LAZYIO: + return "lazyio"; + case CEPH_IOC_SYNCIO: + return "syncio"; + case FS_IOC_SET_ENCRYPTION_POLICY: + return "set_encryption_policy"; + case FS_IOC_GET_ENCRYPTION_POLICY: + return "get_encryption_policy"; + case FS_IOC_GET_ENCRYPTION_POLICY_EX: + return "get_encryption_policy_ex"; + case FS_IOC_ADD_ENCRYPTION_KEY: + return "add_encryption_key"; + case FS_IOC_REMOVE_ENCRYPTION_KEY: + return "remove_encryption_key"; + case FS_IOC_REMOVE_ENCRYPTION_KEY_ALL_USERS: + return "remove_encryption_key_all_users"; + case FS_IOC_GET_ENCRYPTION_KEY_STATUS: + return "get_encryption_key_status"; + case FS_IOC_GET_ENCRYPTION_NONCE: + return "get_encryption_nonce"; + default: + return "unknown"; + } +} + +long ceph_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + struct inode *inode = file_inode(file); + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + int ret; + + doutc(fsc->client, "file %p %p %llx.%llx cmd %s arg %lu\n", file, + inode, ceph_vinop(inode), ceph_ioctl_cmd_name(cmd), arg); + switch (cmd) { + case CEPH_IOC_GET_LAYOUT: + return ceph_ioctl_get_layout(file, (void __user *)arg); + + case CEPH_IOC_SET_LAYOUT: + return ceph_ioctl_set_layout(file, (void __user *)arg); + + case CEPH_IOC_SET_LAYOUT_POLICY: + return ceph_ioctl_set_layout_policy(file, (void __user *)arg); + + case CEPH_IOC_GET_DATALOC: + return ceph_ioctl_get_dataloc(file, (void __user *)arg); + + case CEPH_IOC_LAZYIO: + return ceph_ioctl_lazyio(file); + + case CEPH_IOC_SYNCIO: + return ceph_ioctl_syncio(file); + + case FS_IOC_SET_ENCRYPTION_POLICY: + return ceph_set_encryption_policy(file, arg); + + case FS_IOC_GET_ENCRYPTION_POLICY: + ret = vet_mds_for_fscrypt(file); + if (ret) + return ret; + return fscrypt_ioctl_get_policy(file, (void __user *)arg); + + case FS_IOC_GET_ENCRYPTION_POLICY_EX: + ret = vet_mds_for_fscrypt(file); + if (ret) + return ret; + return fscrypt_ioctl_get_policy_ex(file, (void __user *)arg); + + case FS_IOC_ADD_ENCRYPTION_KEY: + ret = vet_mds_for_fscrypt(file); + if (ret) + return ret; + return fscrypt_ioctl_add_key(file, (void __user *)arg); + + case FS_IOC_REMOVE_ENCRYPTION_KEY: + return fscrypt_ioctl_remove_key(file, (void __user *)arg); + + case FS_IOC_REMOVE_ENCRYPTION_KEY_ALL_USERS: + return fscrypt_ioctl_remove_key_all_users(file, + (void __user *)arg); + + case FS_IOC_GET_ENCRYPTION_KEY_STATUS: + return fscrypt_ioctl_get_key_status(file, (void __user *)arg); + + case FS_IOC_GET_ENCRYPTION_NONCE: + ret = vet_mds_for_fscrypt(file); + if (ret) + return ret; + return fscrypt_ioctl_get_nonce(file, (void __user *)arg); + } + + return -ENOTTY; +} diff --git a/fs/ceph/ioctl.h b/fs/ceph/ioctl.h new file mode 100644 index 000000000..51f7f1d39 --- /dev/null +++ b/fs/ceph/ioctl.h @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef FS_CEPH_IOCTL_H +#define FS_CEPH_IOCTL_H + +#include +#include + +#define CEPH_IOCTL_MAGIC 0x97 + +/* + * CEPH_IOC_GET_LAYOUT - get file layout or dir layout policy + * CEPH_IOC_SET_LAYOUT - set file layout + * CEPH_IOC_SET_LAYOUT_POLICY - set dir layout policy + * + * The file layout specifies how file data is striped over objects in + * the distributed object store, which object pool they belong to (if + * it differs from the default), and an optional 'preferred osd' to + * store them on. + * + * Files get a new layout based on the policy set on the containing + * directory or one of its ancestors. The GET_LAYOUT ioctl will let + * you examine the layout for a file or the policy on a directory. + * + * SET_LAYOUT will let you set a layout on a newly created file. This + * only works immediately after the file is created and before any + * data is written to it. + * + * SET_LAYOUT_POLICY will let you set a layout policy (default layout) + * on a directory that will apply to any new files created in that + * directory (or any child directory that doesn't specify a layout of + * its own). + */ + +/* use u64 to align sanely on all archs */ +struct ceph_ioctl_layout { + __u64 stripe_unit, stripe_count, object_size; + __u64 data_pool; + + /* obsolete. new values ignored, always return -1 */ + __s64 preferred_osd; +}; + +#define CEPH_IOC_GET_LAYOUT _IOR(CEPH_IOCTL_MAGIC, 1, \ + struct ceph_ioctl_layout) +#define CEPH_IOC_SET_LAYOUT _IOW(CEPH_IOCTL_MAGIC, 2, \ + struct ceph_ioctl_layout) +#define CEPH_IOC_SET_LAYOUT_POLICY _IOW(CEPH_IOCTL_MAGIC, 5, \ + struct ceph_ioctl_layout) + +/* + * CEPH_IOC_GET_DATALOC - get location of file data in the cluster + * + * Extract identity, address of the OSD and object storing a given + * file offset. + */ +struct ceph_ioctl_dataloc { + __u64 file_offset; /* in+out: file offset */ + __u64 object_offset; /* out: offset in object */ + __u64 object_no; /* out: object # */ + __u64 object_size; /* out: object size */ + char object_name[64]; /* out: object name */ + __u64 block_offset; /* out: offset in block */ + __u64 block_size; /* out: block length */ + __s64 osd; /* out: osd # */ + struct sockaddr_storage osd_addr; /* out: osd address */ +}; + +#define CEPH_IOC_GET_DATALOC _IOWR(CEPH_IOCTL_MAGIC, 3, \ + struct ceph_ioctl_dataloc) + +/* + * CEPH_IOC_LAZYIO - relax consistency + * + * Normally Ceph switches to synchronous IO when multiple clients have + * the file open (and or more for write). Reads and writes bypass the + * page cache and go directly to the OSD. Setting this flag on a file + * descriptor will allow buffered IO for this file in cases where the + * application knows it won't interfere with other nodes (or doesn't + * care). + */ +#define CEPH_IOC_LAZYIO _IO(CEPH_IOCTL_MAGIC, 4) + +/* + * CEPH_IOC_SYNCIO - force synchronous IO + * + * This ioctl sets a file flag that forces the synchronous IO that + * bypasses the page cache, even if it is not necessary. This is + * essentially the opposite behavior of IOC_LAZYIO. This forces the + * same read/write path as a file opened by multiple clients when one + * or more of those clients is opened for write. + * + * Note that this type of sync IO takes a different path than a file + * opened with O_SYNC/D_SYNC (writes hit the page cache and are + * immediately flushed on page boundaries). It is very similar to + * O_DIRECT (writes bypass the page cache) excep that O_DIRECT writes + * are not copied (user page must remain stable) and O_DIRECT writes + * have alignment restrictions (on the buffer and file offset). + */ +#define CEPH_IOC_SYNCIO _IO(CEPH_IOCTL_MAGIC, 5) + +#endif diff --git a/fs/ceph/locks.c b/fs/ceph/locks.c new file mode 100644 index 000000000..677221bd6 --- /dev/null +++ b/fs/ceph/locks.c @@ -0,0 +1,528 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +#include +#include +#include + +#include "super.h" +#include "mds_client.h" +#include +#include + +static u64 lock_secret; +static int ceph_lock_wait_for_completion(struct ceph_mds_client *mdsc, + struct ceph_mds_request *req); + +static inline u64 secure_addr(void *addr) +{ + u64 v = lock_secret ^ (u64)(unsigned long)addr; + /* + * Set the most significant bit, so that MDS knows the 'owner' + * is sufficient to identify the owner of lock. (old code uses + * both 'owner' and 'pid') + */ + v |= (1ULL << 63); + return v; +} + +void __init ceph_flock_init(void) +{ + get_random_bytes(&lock_secret, sizeof(lock_secret)); +} + +static void ceph_fl_copy_lock(struct file_lock *dst, struct file_lock *src) +{ + struct inode *inode = file_inode(dst->c.flc_file); + atomic_inc(&ceph_inode(inode)->i_filelock_ref); + dst->fl_u.ceph.inode = igrab(inode); +} + +/* + * Do not use the 'fl->fl_file' in release function, which + * is possibly already released by another thread. + */ +static void ceph_fl_release_lock(struct file_lock *fl) +{ + struct inode *inode = fl->fl_u.ceph.inode; + struct ceph_inode_info *ci; + + /* + * If inode is NULL it should be a request file_lock, + * nothing we can do. + */ + if (!inode) + return; + + ci = ceph_inode(inode); + if (atomic_dec_and_test(&ci->i_filelock_ref)) { + /* clear error when all locks are released */ + clear_bit(CEPH_I_ERROR_FILELOCK_BIT, &ci->i_ceph_flags); + } + fl->fl_u.ceph.inode = NULL; + iput(inode); +} + +static const struct file_lock_operations ceph_fl_lock_ops = { + .fl_copy_lock = ceph_fl_copy_lock, + .fl_release_private = ceph_fl_release_lock, +}; + +/* + * Implement fcntl and flock locking functions. + */ +static int ceph_lock_message(u8 lock_type, u16 operation, struct inode *inode, + int cmd, u8 wait, struct file_lock *fl) +{ + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb); + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_mds_request *req; + int err; + u64 length = 0; + u64 owner; + + if (operation == CEPH_MDS_OP_SETFILELOCK) { + /* + * increasing i_filelock_ref closes race window between + * handling request reply and adding file_lock struct to + * inode. Otherwise, auth caps may get trimmed in the + * window. Caller function will decrease the counter. + */ + fl->fl_ops = &ceph_fl_lock_ops; + fl->fl_ops->fl_copy_lock(fl, NULL); + } + + if (operation != CEPH_MDS_OP_SETFILELOCK || cmd == CEPH_LOCK_UNLOCK) + wait = 0; + + req = ceph_mdsc_create_request(mdsc, operation, USE_AUTH_MDS); + if (IS_ERR(req)) + return PTR_ERR(req); + req->r_inode = inode; + ihold(inode); + req->r_num_caps = 1; + + /* mds requires start and length rather than start and end */ + if (LLONG_MAX == fl->fl_end) + length = 0; + else + length = fl->fl_end - fl->fl_start + 1; + + owner = secure_addr(fl->c.flc_owner); + + doutc(cl, "rule: %d, op: %d, owner: %llx, pid: %llu, " + "start: %llu, length: %llu, wait: %d, type: %d\n", + (int)lock_type, (int)operation, owner, + (u64) fl->c.flc_pid, + fl->fl_start, length, wait, fl->c.flc_type); + + req->r_args.filelock_change.rule = lock_type; + req->r_args.filelock_change.type = cmd; + req->r_args.filelock_change.owner = cpu_to_le64(owner); + req->r_args.filelock_change.pid = cpu_to_le64((u64) fl->c.flc_pid); + req->r_args.filelock_change.start = cpu_to_le64(fl->fl_start); + req->r_args.filelock_change.length = cpu_to_le64(length); + req->r_args.filelock_change.wait = wait; + + err = ceph_mdsc_submit_request(mdsc, inode, req); + if (!err) + err = ceph_mdsc_wait_request(mdsc, req, wait ? + ceph_lock_wait_for_completion : NULL); + if (!err && operation == CEPH_MDS_OP_GETFILELOCK) { + fl->c.flc_pid = -le64_to_cpu(req->r_reply_info.filelock_reply->pid); + if (CEPH_LOCK_SHARED == req->r_reply_info.filelock_reply->type) + fl->c.flc_type = F_RDLCK; + else if (CEPH_LOCK_EXCL == req->r_reply_info.filelock_reply->type) + fl->c.flc_type = F_WRLCK; + else + fl->c.flc_type = F_UNLCK; + + fl->fl_start = le64_to_cpu(req->r_reply_info.filelock_reply->start); + length = le64_to_cpu(req->r_reply_info.filelock_reply->start) + + le64_to_cpu(req->r_reply_info.filelock_reply->length); + if (length >= 1) + fl->fl_end = length -1; + else + fl->fl_end = 0; + + } + ceph_mdsc_put_request(req); + doutc(cl, "rule: %d, op: %d, pid: %llu, start: %llu, " + "length: %llu, wait: %d, type: %d, err code %d\n", + (int)lock_type, (int)operation, (u64) fl->c.flc_pid, + fl->fl_start, length, wait, fl->c.flc_type, err); + return err; +} + +static int ceph_lock_wait_for_completion(struct ceph_mds_client *mdsc, + struct ceph_mds_request *req) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_mds_request *intr_req; + struct inode *inode = req->r_inode; + int err, lock_type; + + BUG_ON(req->r_op != CEPH_MDS_OP_SETFILELOCK); + if (req->r_args.filelock_change.rule == CEPH_LOCK_FCNTL) + lock_type = CEPH_LOCK_FCNTL_INTR; + else if (req->r_args.filelock_change.rule == CEPH_LOCK_FLOCK) + lock_type = CEPH_LOCK_FLOCK_INTR; + else + BUG_ON(1); + BUG_ON(req->r_args.filelock_change.type == CEPH_LOCK_UNLOCK); + + err = wait_for_completion_interruptible(&req->r_completion); + if (!err) + return 0; + + doutc(cl, "request %llu was interrupted\n", req->r_tid); + + mutex_lock(&mdsc->mutex); + if (test_bit(CEPH_MDS_R_GOT_RESULT, &req->r_req_flags)) { + err = 0; + } else { + /* + * ensure we aren't running concurrently with + * ceph_fill_trace or ceph_readdir_prepopulate, which + * rely on locks (dir mutex) held by our caller. + */ + mutex_lock(&req->r_fill_mutex); + req->r_err = err; + set_bit(CEPH_MDS_R_ABORTED, &req->r_req_flags); + mutex_unlock(&req->r_fill_mutex); + + if (!req->r_session) { + // haven't sent the request + err = 0; + } + } + mutex_unlock(&mdsc->mutex); + if (!err) + return 0; + + intr_req = ceph_mdsc_create_request(mdsc, CEPH_MDS_OP_SETFILELOCK, + USE_AUTH_MDS); + if (IS_ERR(intr_req)) + return PTR_ERR(intr_req); + + intr_req->r_inode = inode; + ihold(inode); + intr_req->r_num_caps = 1; + + intr_req->r_args.filelock_change = req->r_args.filelock_change; + intr_req->r_args.filelock_change.rule = lock_type; + intr_req->r_args.filelock_change.type = CEPH_LOCK_UNLOCK; + + err = ceph_mdsc_do_request(mdsc, inode, intr_req); + ceph_mdsc_put_request(intr_req); + + if (err && err != -ERESTARTSYS) + return err; + + err = wait_for_completion_killable(&req->r_safe_completion); + if (err) + return err; + + return 0; +} + +static int try_unlock_file(struct file *file, struct file_lock *fl) +{ + int err; + unsigned int orig_flags = fl->c.flc_flags; + fl->c.flc_flags |= FL_EXISTS; + err = locks_lock_file_wait(file, fl); + fl->c.flc_flags = orig_flags; + if (err == -ENOENT) { + if (!(orig_flags & FL_EXISTS)) + err = 0; + return err; + } + return 1; +} + +/* + * Attempt to set an fcntl lock. + * For now, this just goes away to the server. Later it may be more awesome. + */ +int ceph_lock(struct file *file, int cmd, struct file_lock *fl) +{ + struct inode *inode = file_inode(file); + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb); + struct ceph_client *cl = ceph_inode_to_client(inode); + int err = 0; + u16 op = CEPH_MDS_OP_SETFILELOCK; + u8 wait = 0; + u8 lock_cmd; + + if (!(fl->c.flc_flags & FL_POSIX)) + return -ENOLCK; + + if (ceph_inode_is_shutdown(inode)) + return -ESTALE; + + doutc(cl, "fl_owner: %p\n", fl->c.flc_owner); + + /* set wait bit as appropriate, then make command as Ceph expects it*/ + if (IS_GETLK(cmd)) + op = CEPH_MDS_OP_GETFILELOCK; + else if (IS_SETLKW(cmd)) + wait = 1; + + if (test_bit(CEPH_I_ERROR_FILELOCK_BIT, &ci->i_ceph_flags)) { + if (op == CEPH_MDS_OP_SETFILELOCK && lock_is_unlock(fl)) + posix_lock_file(file, fl, NULL); + return -EIO; + } + + /* Wait for reset to complete before acquiring new locks */ + if (op == CEPH_MDS_OP_SETFILELOCK && !lock_is_unlock(fl)) { + err = ceph_mdsc_wait_for_reset(mdsc); + if (err) + return err; + } + + if (lock_is_read(fl)) + lock_cmd = CEPH_LOCK_SHARED; + else if (lock_is_write(fl)) + lock_cmd = CEPH_LOCK_EXCL; + else + lock_cmd = CEPH_LOCK_UNLOCK; + + if (op == CEPH_MDS_OP_SETFILELOCK && lock_is_unlock(fl)) { + err = try_unlock_file(file, fl); + if (err <= 0) + return err; + } + + err = ceph_lock_message(CEPH_LOCK_FCNTL, op, inode, lock_cmd, wait, fl); + if (!err) { + if (op == CEPH_MDS_OP_SETFILELOCK && F_UNLCK != fl->c.flc_type) { + doutc(cl, "locking locally\n"); + err = posix_lock_file(file, fl, NULL); + if (err) { + /* undo! This should only happen if + * the kernel detects local + * deadlock. */ + ceph_lock_message(CEPH_LOCK_FCNTL, op, inode, + CEPH_LOCK_UNLOCK, 0, fl); + doutc(cl, "got %d on posix_lock_file, undid lock\n", + err); + } + } + } + return err; +} + +int ceph_flock(struct file *file, int cmd, struct file_lock *fl) +{ + struct inode *inode = file_inode(file); + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb); + struct ceph_client *cl = ceph_inode_to_client(inode); + int err = 0; + u8 wait = 0; + u8 lock_cmd; + + if (!(fl->c.flc_flags & FL_FLOCK)) + return -ENOLCK; + + if (ceph_inode_is_shutdown(inode)) + return -ESTALE; + + doutc(cl, "fl_file: %p\n", fl->c.flc_file); + + if (test_bit(CEPH_I_ERROR_FILELOCK_BIT, &ci->i_ceph_flags)) { + if (lock_is_unlock(fl)) + locks_lock_file_wait(file, fl); + return -EIO; + } + + /* Wait for reset to complete before acquiring new locks */ + if (!lock_is_unlock(fl)) { + err = ceph_mdsc_wait_for_reset(mdsc); + if (err) + return err; + } + + if (IS_SETLKW(cmd)) + wait = 1; + + if (lock_is_read(fl)) + lock_cmd = CEPH_LOCK_SHARED; + else if (lock_is_write(fl)) + lock_cmd = CEPH_LOCK_EXCL; + else + lock_cmd = CEPH_LOCK_UNLOCK; + + if (lock_is_unlock(fl)) { + err = try_unlock_file(file, fl); + if (err <= 0) + return err; + } + + err = ceph_lock_message(CEPH_LOCK_FLOCK, CEPH_MDS_OP_SETFILELOCK, + inode, lock_cmd, wait, fl); + if (!err && F_UNLCK != fl->c.flc_type) { + err = locks_lock_file_wait(file, fl); + if (err) { + ceph_lock_message(CEPH_LOCK_FLOCK, + CEPH_MDS_OP_SETFILELOCK, + inode, CEPH_LOCK_UNLOCK, 0, fl); + doutc(cl, "got %d on locks_lock_file_wait, undid lock\n", + err); + } + } + return err; +} + +/* + * Fills in the passed counter variables, so you can prepare pagelist metadata + * before calling ceph_encode_locks. + */ +void ceph_count_locks(struct inode *inode, int *fcntl_count, int *flock_count) +{ + struct ceph_client *cl = ceph_inode_to_client(inode); + struct file_lock *lock; + struct file_lock_context *ctx; + + *fcntl_count = 0; + *flock_count = 0; + + ctx = locks_inode_context(inode); + if (ctx) { + spin_lock(&ctx->flc_lock); + for_each_file_lock(lock, &ctx->flc_posix) + ++(*fcntl_count); + for_each_file_lock(lock, &ctx->flc_flock) + ++(*flock_count); + spin_unlock(&ctx->flc_lock); + } + doutc(cl, "counted %d flock locks and %d fcntl locks\n", + *flock_count, *fcntl_count); +} + +/* + * Given a pointer to a lock, convert it to a ceph filelock + */ +static int lock_to_ceph_filelock(struct inode *inode, + struct file_lock *lock, + struct ceph_filelock *cephlock) +{ + struct ceph_client *cl = ceph_inode_to_client(inode); + int err = 0; + + cephlock->start = cpu_to_le64(lock->fl_start); + cephlock->length = cpu_to_le64(lock->fl_end - lock->fl_start + 1); + cephlock->client = cpu_to_le64(0); + cephlock->pid = cpu_to_le64((u64) lock->c.flc_pid); + cephlock->owner = cpu_to_le64(secure_addr(lock->c.flc_owner)); + + switch (lock->c.flc_type) { + case F_RDLCK: + cephlock->type = CEPH_LOCK_SHARED; + break; + case F_WRLCK: + cephlock->type = CEPH_LOCK_EXCL; + break; + case F_UNLCK: + cephlock->type = CEPH_LOCK_UNLOCK; + break; + default: + doutc(cl, "Have unknown lock type %d\n", + lock->c.flc_type); + err = -EINVAL; + } + + return err; +} + +/* + * Encode the flock and fcntl locks for the given inode into the ceph_filelock + * array. Must be called with inode->i_lock already held. + * If we encounter more of a specific lock type than expected, return -ENOSPC. + */ +int ceph_encode_locks_to_buffer(struct inode *inode, + struct ceph_filelock *flocks, + int num_fcntl_locks, int num_flock_locks) +{ + struct file_lock *lock; + struct file_lock_context *ctx = locks_inode_context(inode); + struct ceph_client *cl = ceph_inode_to_client(inode); + int err = 0; + int seen_fcntl = 0; + int seen_flock = 0; + int l = 0; + + doutc(cl, "encoding %d flock and %d fcntl locks\n", num_flock_locks, + num_fcntl_locks); + + if (!ctx) + return 0; + + spin_lock(&ctx->flc_lock); + for_each_file_lock(lock, &ctx->flc_posix) { + ++seen_fcntl; + if (seen_fcntl > num_fcntl_locks) { + err = -ENOSPC; + goto fail; + } + err = lock_to_ceph_filelock(inode, lock, &flocks[l]); + if (err) + goto fail; + ++l; + } + for_each_file_lock(lock, &ctx->flc_flock) { + ++seen_flock; + if (seen_flock > num_flock_locks) { + err = -ENOSPC; + goto fail; + } + err = lock_to_ceph_filelock(inode, lock, &flocks[l]); + if (err) + goto fail; + ++l; + } +fail: + spin_unlock(&ctx->flc_lock); + return err; +} + +/* + * Copy the encoded flock and fcntl locks into the pagelist. + * Format is: #fcntl locks, sequential fcntl locks, #flock locks, + * sequential flock locks. + * Returns zero on success. + */ +int ceph_locks_to_pagelist(struct ceph_filelock *flocks, + struct ceph_pagelist *pagelist, + int num_fcntl_locks, int num_flock_locks) +{ + int err = 0; + __le32 nlocks; + + nlocks = cpu_to_le32(num_fcntl_locks); + err = ceph_pagelist_append(pagelist, &nlocks, sizeof(nlocks)); + if (err) + goto out_fail; + + if (num_fcntl_locks > 0) { + err = ceph_pagelist_append(pagelist, flocks, + num_fcntl_locks * sizeof(*flocks)); + if (err) + goto out_fail; + } + + nlocks = cpu_to_le32(num_flock_locks); + err = ceph_pagelist_append(pagelist, &nlocks, sizeof(nlocks)); + if (err) + goto out_fail; + + if (num_flock_locks > 0) { + err = ceph_pagelist_append(pagelist, &flocks[num_fcntl_locks], + num_flock_locks * sizeof(*flocks)); + } +out_fail: + return err; +} diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c new file mode 100644 index 000000000..3c692ad02 --- /dev/null +++ b/fs/ceph/mds_client.c @@ -0,0 +1,7367 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "mds_client.h" +#include "crypto.h" + +#include +#include +#include +#include +#include +#include +#include + +#define RECONNECT_MAX_SIZE (INT_MAX - PAGE_SIZE) + +/* + * A cluster of MDS (metadata server) daemons is responsible for + * managing the file system namespace (the directory hierarchy and + * inodes) and for coordinating shared access to storage. Metadata is + * partitioning hierarchically across a number of servers, and that + * partition varies over time as the cluster adjusts the distribution + * in order to balance load. + * + * The MDS client is primarily responsible to managing synchronous + * metadata requests for operations like open, unlink, and so forth. + * If there is a MDS failure, we find out about it when we (possibly + * request and) receive a new MDS map, and can resubmit affected + * requests. + * + * For the most part, though, we take advantage of a lossless + * communications channel to the MDS, and do not need to worry about + * timing out or resubmitting requests. + * + * We maintain a stateful "session" with each MDS we interact with. + * Within each session, we sent periodic heartbeat messages to ensure + * any capabilities or leases we have been issues remain valid. If + * the session times out and goes stale, our leases and capabilities + * are no longer valid. + */ + +struct ceph_reconnect_state { + struct ceph_mds_session *session; + int nr_caps, nr_realms; + struct ceph_pagelist *pagelist; + unsigned msg_version; + bool allow_multi; +}; + +static void __wake_requests(struct ceph_mds_client *mdsc, + struct list_head *head); +static void ceph_cap_release_work(struct work_struct *work); +static void ceph_cap_reclaim_work(struct work_struct *work); +static void ceph_mdsc_reset_workfn(struct work_struct *work); + +static const struct ceph_connection_operations mds_con_ops; + +static void ceph_metric_bind_session(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session) +{ + struct ceph_mds_session *old; + + if (!mdsc || !session || disable_send_metrics) + return; + + old = mdsc->metric.session; + mdsc->metric.session = ceph_get_mds_session(session); + if (old) + ceph_put_mds_session(old); + + metric_schedule_delayed(&mdsc->metric); +} + +/* + * mds reply parsing + */ + +static int parse_reply_info_quota(void **p, void *end, + struct ceph_mds_reply_info_in *info) +{ + u8 struct_v, struct_compat; + u32 struct_len; + + ceph_decode_8_safe(p, end, struct_v, bad); + ceph_decode_8_safe(p, end, struct_compat, bad); + /* struct_v is expected to be >= 1. we only + * understand encoding with struct_compat == 1. */ + if (!struct_v || struct_compat != 1) + goto bad; + ceph_decode_32_safe(p, end, struct_len, bad); + ceph_decode_need(p, end, struct_len, bad); + end = *p + struct_len; + ceph_decode_64_safe(p, end, info->max_bytes, bad); + ceph_decode_64_safe(p, end, info->max_files, bad); + *p = end; + return 0; +bad: + return -EIO; +} + +static int parse_reply_info_in(void **p, void *end, + struct ceph_mds_reply_info_in *info, + u64 features, + struct ceph_mds_client *mdsc) +{ + int err = 0; + u8 struct_v = 0; + u8 struct_compat = 0; + u32 struct_len = 0; + + info->subvolume_id = CEPH_SUBVOLUME_ID_NONE; + + if (features == (u64)-1) { + ceph_decode_8_safe(p, end, struct_v, bad); + ceph_decode_8_safe(p, end, struct_compat, bad); + /* struct_v is expected to be >= 1. we only understand + * encoding with struct_compat == 1. */ + if (!struct_v || struct_compat != 1) + goto bad; + ceph_decode_32_safe(p, end, struct_len, bad); + ceph_decode_need(p, end, struct_len, bad); + end = *p + struct_len; + } + + ceph_decode_need(p, end, sizeof(struct ceph_mds_reply_inode), bad); + info->in = *p; + *p += sizeof(struct ceph_mds_reply_inode) + + sizeof(*info->in->fragtree.splits) * + le32_to_cpu(info->in->fragtree.nsplits); + + ceph_decode_32_safe(p, end, info->symlink_len, bad); + ceph_decode_need(p, end, info->symlink_len, bad); + info->symlink = *p; + *p += info->symlink_len; + + ceph_decode_copy_safe(p, end, &info->dir_layout, + sizeof(info->dir_layout), bad); + ceph_decode_32_safe(p, end, info->xattr_len, bad); + ceph_decode_need(p, end, info->xattr_len, bad); + info->xattr_data = *p; + *p += info->xattr_len; + + if (features == (u64)-1) { + /* inline data */ + ceph_decode_64_safe(p, end, info->inline_version, bad); + ceph_decode_32_safe(p, end, info->inline_len, bad); + ceph_decode_need(p, end, info->inline_len, bad); + info->inline_data = *p; + *p += info->inline_len; + /* quota */ + err = parse_reply_info_quota(p, end, info); + if (err < 0) + goto out_bad; + /* pool namespace */ + ceph_decode_32_safe(p, end, info->pool_ns_len, bad); + if (info->pool_ns_len > 0) { + ceph_decode_need(p, end, info->pool_ns_len, bad); + info->pool_ns_data = *p; + *p += info->pool_ns_len; + } + + /* btime */ + ceph_decode_need(p, end, sizeof(info->btime), bad); + ceph_decode_copy(p, &info->btime, sizeof(info->btime)); + + /* change attribute */ + ceph_decode_64_safe(p, end, info->change_attr, bad); + + /* dir pin */ + if (struct_v >= 2) { + ceph_decode_32_safe(p, end, info->dir_pin, bad); + } else { + info->dir_pin = -ENODATA; + } + + /* snapshot birth time, remains zero for v<=2 */ + if (struct_v >= 3) { + ceph_decode_need(p, end, sizeof(info->snap_btime), bad); + ceph_decode_copy(p, &info->snap_btime, + sizeof(info->snap_btime)); + } else { + memset(&info->snap_btime, 0, sizeof(info->snap_btime)); + } + + /* snapshot count, remains zero for v<=3 */ + if (struct_v >= 4) { + ceph_decode_64_safe(p, end, info->rsnaps, bad); + } else { + info->rsnaps = 0; + } + + if (struct_v >= 5) { + u32 alen; + + ceph_decode_32_safe(p, end, alen, bad); + + while (alen--) { + u32 len; + + /* key */ + ceph_decode_32_safe(p, end, len, bad); + ceph_decode_skip_n(p, end, len, bad); + /* value */ + ceph_decode_32_safe(p, end, len, bad); + ceph_decode_skip_n(p, end, len, bad); + } + } + + /* fscrypt flag -- ignore */ + if (struct_v >= 6) + ceph_decode_skip_8(p, end, bad); + + info->fscrypt_auth = NULL; + info->fscrypt_auth_len = 0; + info->fscrypt_file = NULL; + info->fscrypt_file_len = 0; + if (struct_v >= 7) { + ceph_decode_32_safe(p, end, info->fscrypt_auth_len, bad); + if (info->fscrypt_auth_len) { + info->fscrypt_auth = kmalloc(info->fscrypt_auth_len, + GFP_KERNEL); + if (!info->fscrypt_auth) + return -ENOMEM; + ceph_decode_copy_safe(p, end, info->fscrypt_auth, + info->fscrypt_auth_len, bad); + } + ceph_decode_32_safe(p, end, info->fscrypt_file_len, bad); + if (info->fscrypt_file_len) { + info->fscrypt_file = kmalloc(info->fscrypt_file_len, + GFP_KERNEL); + if (!info->fscrypt_file) + return -ENOMEM; + ceph_decode_copy_safe(p, end, info->fscrypt_file, + info->fscrypt_file_len, bad); + } + } + + /* + * InodeStat encoding versions: + * v1-v7: various fields added over time + * v8: added optmetadata (versioned sub-structure containing + * optional inode metadata like charmap for case-insensitive + * filesystems). The kernel client doesn't support + * case-insensitive lookups, so we skip this field. + * v9: added subvolume_id (parsed below) + */ + if (struct_v >= 8) { + u32 v8_struct_len; + + /* skip optmetadata versioned sub-structure */ + ceph_decode_skip_8(p, end, bad); /* struct_v */ + ceph_decode_skip_8(p, end, bad); /* struct_compat */ + ceph_decode_32_safe(p, end, v8_struct_len, bad); + ceph_decode_skip_n(p, end, v8_struct_len, bad); + } + + /* struct_v 9 added subvolume_id */ + if (struct_v >= 9) + ceph_decode_64_safe(p, end, info->subvolume_id, bad); + + *p = end; + } else { + /* legacy (unversioned) struct */ + if (features & CEPH_FEATURE_MDS_INLINE_DATA) { + ceph_decode_64_safe(p, end, info->inline_version, bad); + ceph_decode_32_safe(p, end, info->inline_len, bad); + ceph_decode_need(p, end, info->inline_len, bad); + info->inline_data = *p; + *p += info->inline_len; + } else + info->inline_version = CEPH_INLINE_NONE; + + if (features & CEPH_FEATURE_MDS_QUOTA) { + err = parse_reply_info_quota(p, end, info); + if (err < 0) + goto out_bad; + } else { + info->max_bytes = 0; + info->max_files = 0; + } + + info->pool_ns_len = 0; + info->pool_ns_data = NULL; + if (features & CEPH_FEATURE_FS_FILE_LAYOUT_V2) { + ceph_decode_32_safe(p, end, info->pool_ns_len, bad); + if (info->pool_ns_len > 0) { + ceph_decode_need(p, end, info->pool_ns_len, bad); + info->pool_ns_data = *p; + *p += info->pool_ns_len; + } + } + + if (features & CEPH_FEATURE_FS_BTIME) { + ceph_decode_need(p, end, sizeof(info->btime), bad); + ceph_decode_copy(p, &info->btime, sizeof(info->btime)); + ceph_decode_64_safe(p, end, info->change_attr, bad); + } + + info->dir_pin = -ENODATA; + /* info->snap_btime and info->rsnaps remain zero */ + } + return 0; +bad: + err = -EIO; +out_bad: + return err; +} + +static int parse_reply_info_dir(void **p, void *end, + struct ceph_mds_reply_dirfrag **dirfrag, + u64 features) +{ + if (features == (u64)-1) { + u8 struct_v, struct_compat; + u32 struct_len; + ceph_decode_8_safe(p, end, struct_v, bad); + ceph_decode_8_safe(p, end, struct_compat, bad); + /* struct_v is expected to be >= 1. we only understand + * encoding whose struct_compat == 1. */ + if (!struct_v || struct_compat != 1) + goto bad; + ceph_decode_32_safe(p, end, struct_len, bad); + ceph_decode_need(p, end, struct_len, bad); + end = *p + struct_len; + } + + ceph_decode_need(p, end, sizeof(**dirfrag), bad); + *dirfrag = *p; + *p += sizeof(**dirfrag) + sizeof(u32) * le32_to_cpu((*dirfrag)->ndist); + if (unlikely(*p > end)) + goto bad; + if (features == (u64)-1) + *p = end; + return 0; +bad: + return -EIO; +} + +static int parse_reply_info_lease(void **p, void *end, + struct ceph_mds_reply_lease **lease, + u64 features, u32 *altname_len, u8 **altname) +{ + u8 struct_v; + u32 struct_len; + void *lend; + + if (features == (u64)-1) { + u8 struct_compat; + + ceph_decode_8_safe(p, end, struct_v, bad); + ceph_decode_8_safe(p, end, struct_compat, bad); + + /* struct_v is expected to be >= 1. we only understand + * encoding whose struct_compat == 1. */ + if (!struct_v || struct_compat != 1) + goto bad; + + ceph_decode_32_safe(p, end, struct_len, bad); + } else { + struct_len = sizeof(**lease); + *altname_len = 0; + *altname = NULL; + } + + lend = *p + struct_len; + ceph_decode_need(p, end, struct_len, bad); + *lease = *p; + *p += sizeof(**lease); + + if (features == (u64)-1) { + if (struct_v >= 2) { + ceph_decode_32_safe(p, end, *altname_len, bad); + ceph_decode_need(p, end, *altname_len, bad); + *altname = *p; + *p += *altname_len; + } else { + *altname = NULL; + *altname_len = 0; + } + } + *p = lend; + return 0; +bad: + return -EIO; +} + +/* + * parse a normal reply, which may contain a (dir+)dentry and/or a + * target inode. + */ +static int parse_reply_info_trace(void **p, void *end, + struct ceph_mds_reply_info_parsed *info, + u64 features, + struct ceph_mds_client *mdsc) +{ + int err; + + if (info->head->is_dentry) { + err = parse_reply_info_in(p, end, &info->diri, features, mdsc); + if (err < 0) + goto out_bad; + + err = parse_reply_info_dir(p, end, &info->dirfrag, features); + if (err < 0) + goto out_bad; + + ceph_decode_32_safe(p, end, info->dname_len, bad); + ceph_decode_need(p, end, info->dname_len, bad); + info->dname = *p; + *p += info->dname_len; + + err = parse_reply_info_lease(p, end, &info->dlease, features, + &info->altname_len, &info->altname); + if (err < 0) + goto out_bad; + } + + if (info->head->is_target) { + err = parse_reply_info_in(p, end, &info->targeti, features, + mdsc); + if (err < 0) + goto out_bad; + } + + if (unlikely(*p != end)) + goto bad; + return 0; + +bad: + err = -EIO; +out_bad: + pr_err("problem parsing mds trace %d\n", err); + return err; +} + +/* + * parse readdir results + */ +static int parse_reply_info_readdir(void **p, void *end, + struct ceph_mds_request *req, + u64 features, + struct ceph_mds_client *mdsc) +{ + struct ceph_mds_reply_info_parsed *info = &req->r_reply_info; + struct ceph_client *cl = req->r_mdsc->fsc->client; + u32 num, i = 0; + int err; + + err = parse_reply_info_dir(p, end, &info->dir_dir, features); + if (err < 0) + goto out_bad; + + ceph_decode_need(p, end, sizeof(num) + 2, bad); + num = ceph_decode_32(p); + { + u16 flags = ceph_decode_16(p); + info->dir_end = !!(flags & CEPH_READDIR_FRAG_END); + info->dir_complete = !!(flags & CEPH_READDIR_FRAG_COMPLETE); + info->hash_order = !!(flags & CEPH_READDIR_HASH_ORDER); + info->offset_hash = !!(flags & CEPH_READDIR_OFFSET_HASH); + } + if (num == 0) + goto done; + + BUG_ON(!info->dir_entries); + if ((unsigned long)(info->dir_entries + num) > + (unsigned long)info->dir_entries + info->dir_buf_size) { + pr_err_client(cl, "dir contents are larger than expected\n"); + WARN_ON(1); + goto bad; + } + + info->dir_nr = num; + while (num) { + struct inode *inode = d_inode(req->r_dentry); + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_mds_reply_dir_entry *rde = info->dir_entries + i; + struct fscrypt_str tname = FSTR_INIT(NULL, 0); + struct fscrypt_str oname = FSTR_INIT(NULL, 0); + struct ceph_fname fname; + u32 altname_len, _name_len; + u8 *altname, *_name; + + /* dentry */ + ceph_decode_32_safe(p, end, _name_len, bad); + ceph_decode_need(p, end, _name_len, bad); + _name = *p; + *p += _name_len; + doutc(cl, "parsed dir dname '%.*s'\n", _name_len, _name); + + if (info->hash_order) + rde->raw_hash = ceph_str_hash(ci->i_dir_layout.dl_dir_hash, + _name, _name_len); + + /* dentry lease */ + err = parse_reply_info_lease(p, end, &rde->lease, features, + &altname_len, &altname); + if (err) + goto out_bad; + + /* + * Try to dencrypt the dentry names and update them + * in the ceph_mds_reply_dir_entry struct. + */ + fname.dir = inode; + fname.name = _name; + fname.name_len = _name_len; + fname.ctext = altname; + fname.ctext_len = altname_len; + /* + * The _name_len maybe larger than altname_len, such as + * when the human readable name length is in range of + * (CEPH_NOHASH_NAME_MAX, CEPH_NOHASH_NAME_MAX + SHA256_DIGEST_SIZE), + * then the copy in ceph_fname_to_usr will corrupt the + * data if there has no encryption key. + * + * Just set the no_copy flag and then if there has no + * encryption key the oname.name will be assigned to + * _name always. + */ + fname.no_copy = true; + if (altname_len == 0) { + /* + * Set tname to _name, and this will be used + * to do the base64_decode in-place. It's + * safe because the decoded string should + * always be shorter, which is 3/4 of origin + * string. + */ + tname.name = _name; + + /* + * Set oname to _name too, and this will be + * used to do the dencryption in-place. + */ + oname.name = _name; + oname.len = _name_len; + } else { + /* + * This will do the decryption only in-place + * from altname cryptext directly. + */ + oname.name = altname; + oname.len = altname_len; + } + rde->is_nokey = false; + err = ceph_fname_to_usr(&fname, &tname, &oname, &rde->is_nokey); + if (err) { + pr_err_client(cl, "unable to decode %.*s, got %d\n", + _name_len, _name, err); + goto out_bad; + } + rde->name = oname.name; + rde->name_len = oname.len; + + /* inode */ + err = parse_reply_info_in(p, end, &rde->inode, features, mdsc); + if (err < 0) + goto out_bad; + /* ceph_readdir_prepopulate() will update it */ + rde->offset = 0; + i++; + num--; + } + +done: + /* Skip over any unrecognized fields */ + *p = end; + return 0; + +bad: + err = -EIO; +out_bad: + pr_err_client(cl, "problem parsing dir contents %d\n", err); + return err; +} + +/* + * parse fcntl F_GETLK results + */ +static int parse_reply_info_filelock(void **p, void *end, + struct ceph_mds_reply_info_parsed *info, + u64 features) +{ + if (*p + sizeof(*info->filelock_reply) > end) + goto bad; + + info->filelock_reply = *p; + + /* Skip over any unrecognized fields */ + *p = end; + return 0; +bad: + return -EIO; +} + + +#if BITS_PER_LONG == 64 + +#define DELEGATED_INO_AVAILABLE xa_mk_value(1) + +static int ceph_parse_deleg_inos(void **p, void *end, + struct ceph_mds_session *s) +{ + struct ceph_client *cl = s->s_mdsc->fsc->client; + u32 sets; + + ceph_decode_32_safe(p, end, sets, bad); + doutc(cl, "got %u sets of delegated inodes\n", sets); + while (sets--) { + u64 start, len; + + ceph_decode_64_safe(p, end, start, bad); + ceph_decode_64_safe(p, end, len, bad); + + /* Don't accept a delegation of system inodes */ + if (start < CEPH_INO_SYSTEM_BASE) { + pr_warn_ratelimited_client(cl, + "ignoring reserved inode range delegation (start=0x%llx len=0x%llx)\n", + start, len); + continue; + } + while (len--) { + int err = xa_insert(&s->s_delegated_inos, start++, + DELEGATED_INO_AVAILABLE, + GFP_KERNEL); + if (!err) { + doutc(cl, "added delegated inode 0x%llx\n", start - 1); + } else if (err == -EBUSY) { + pr_warn_client(cl, + "MDS delegated inode 0x%llx more than once.\n", + start - 1); + } else { + return err; + } + } + } + return 0; +bad: + return -EIO; +} + +u64 ceph_get_deleg_ino(struct ceph_mds_session *s) +{ + unsigned long ino; + void *val; + + xa_for_each(&s->s_delegated_inos, ino, val) { + val = xa_erase(&s->s_delegated_inos, ino); + if (val == DELEGATED_INO_AVAILABLE) + return ino; + } + return 0; +} + +int ceph_restore_deleg_ino(struct ceph_mds_session *s, u64 ino) +{ + return xa_insert(&s->s_delegated_inos, ino, DELEGATED_INO_AVAILABLE, + GFP_KERNEL); +} +#else /* BITS_PER_LONG == 64 */ +/* + * FIXME: xarrays can't handle 64-bit indexes on a 32-bit arch. For now, just + * ignore delegated_inos on 32 bit arch. Maybe eventually add xarrays for top + * and bottom words? + */ +static int ceph_parse_deleg_inos(void **p, void *end, + struct ceph_mds_session *s) +{ + u32 sets; + + ceph_decode_32_safe(p, end, sets, bad); + if (sets) + ceph_decode_skip_n(p, end, sets * 2 * sizeof(__le64), bad); + return 0; +bad: + return -EIO; +} + +u64 ceph_get_deleg_ino(struct ceph_mds_session *s) +{ + return 0; +} + +int ceph_restore_deleg_ino(struct ceph_mds_session *s, u64 ino) +{ + return 0; +} +#endif /* BITS_PER_LONG == 64 */ + +/* + * parse create results + */ +static int parse_reply_info_create(void **p, void *end, + struct ceph_mds_reply_info_parsed *info, + u64 features, struct ceph_mds_session *s) +{ + int ret; + + if (features == (u64)-1 || + (features & CEPH_FEATURE_REPLY_CREATE_INODE)) { + if (*p == end) { + /* Malformed reply? */ + info->has_create_ino = false; + } else if (test_bit(CEPHFS_FEATURE_DELEG_INO, &s->s_features)) { + info->has_create_ino = true; + /* struct_v, struct_compat, and len */ + ceph_decode_skip_n(p, end, 2 + sizeof(u32), bad); + ceph_decode_64_safe(p, end, info->ino, bad); + ret = ceph_parse_deleg_inos(p, end, s); + if (ret) + return ret; + } else { + /* legacy */ + ceph_decode_64_safe(p, end, info->ino, bad); + info->has_create_ino = true; + } + } else { + if (*p != end) + goto bad; + } + + /* Skip over any unrecognized fields */ + *p = end; + return 0; +bad: + return -EIO; +} + +static int parse_reply_info_getvxattr(void **p, void *end, + struct ceph_mds_reply_info_parsed *info, + u64 features) +{ + u32 value_len; + + ceph_decode_skip_8(p, end, bad); /* skip current version: 1 */ + ceph_decode_skip_8(p, end, bad); /* skip first version: 1 */ + ceph_decode_skip_32(p, end, bad); /* skip payload length */ + + ceph_decode_32_safe(p, end, value_len, bad); + + if (value_len == end - *p) { + info->xattr_info.xattr_value = *p; + info->xattr_info.xattr_value_len = value_len; + *p = end; + return value_len; + } +bad: + return -EIO; +} + +/* + * parse extra results + */ +static int parse_reply_info_extra(void **p, void *end, + struct ceph_mds_request *req, + u64 features, struct ceph_mds_session *s) +{ + struct ceph_mds_reply_info_parsed *info = &req->r_reply_info; + u32 op = le32_to_cpu(info->head->op); + + if (op == CEPH_MDS_OP_GETFILELOCK) + return parse_reply_info_filelock(p, end, info, features); + else if (op == CEPH_MDS_OP_READDIR || op == CEPH_MDS_OP_LSSNAP) + return parse_reply_info_readdir(p, end, req, features, + req->r_mdsc); + else if (op == CEPH_MDS_OP_CREATE) + return parse_reply_info_create(p, end, info, features, s); + else if (op == CEPH_MDS_OP_GETVXATTR) + return parse_reply_info_getvxattr(p, end, info, features); + else + return -EIO; +} + +/* + * parse entire mds reply + */ +static int parse_reply_info(struct ceph_mds_session *s, struct ceph_msg *msg, + struct ceph_mds_request *req, u64 features) +{ + struct ceph_mds_reply_info_parsed *info = &req->r_reply_info; + struct ceph_client *cl = s->s_mdsc->fsc->client; + void *p, *end; + u32 len; + int err; + + info->head = msg->front.iov_base; + p = msg->front.iov_base + sizeof(struct ceph_mds_reply_head); + end = p + msg->front.iov_len - sizeof(struct ceph_mds_reply_head); + + /* trace */ + ceph_decode_32_safe(&p, end, len, bad); + if (len > 0) { + ceph_decode_need(&p, end, len, bad); + err = parse_reply_info_trace(&p, p + len, info, features, + s->s_mdsc); + if (err < 0) + goto out_bad; + } + + /* extra */ + ceph_decode_32_safe(&p, end, len, bad); + if (len > 0) { + ceph_decode_need(&p, end, len, bad); + err = parse_reply_info_extra(&p, p + len, req, features, s); + if (err < 0) + goto out_bad; + } + + /* snap blob */ + ceph_decode_32_safe(&p, end, len, bad); + info->snapblob_len = len; + info->snapblob = p; + p += len; + + if (p != end) + goto bad; + return 0; + +bad: + err = -EIO; +out_bad: + pr_err_client(cl, "mds parse_reply err %d\n", err); + ceph_msg_dump(msg); + return err; +} + +static void destroy_reply_info(struct ceph_mds_reply_info_parsed *info) +{ + int i; + + kfree(info->diri.fscrypt_auth); + kfree(info->diri.fscrypt_file); + kfree(info->targeti.fscrypt_auth); + kfree(info->targeti.fscrypt_file); + if (!info->dir_entries) + return; + + for (i = 0; i < info->dir_nr; i++) { + struct ceph_mds_reply_dir_entry *rde = info->dir_entries + i; + + kfree(rde->inode.fscrypt_auth); + kfree(rde->inode.fscrypt_file); + } + free_pages((unsigned long)info->dir_entries, get_order(info->dir_buf_size)); +} + +/* + * In async unlink case the kclient won't wait for the first reply + * from MDS and just drop all the links and unhash the dentry and then + * succeeds immediately. + * + * For any new create/link/rename,etc requests followed by using the + * same file names we must wait for the first reply of the inflight + * unlink request, or the MDS possibly will fail these following + * requests with -EEXIST if the inflight async unlink request was + * delayed for some reasons. + * + * And the worst case is that for the none async openc request it will + * successfully open the file if the CDentry hasn't been unlinked yet, + * but later the previous delayed async unlink request will remove the + * CDentry. That means the just created file is possibly deleted later + * by accident. + * + * We need to wait for the inflight async unlink requests to finish + * when creating new files/directories by using the same file names. + */ +int ceph_wait_on_conflict_unlink(struct dentry *dentry) +{ + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(dentry->d_sb); + struct ceph_client *cl = fsc->client; + struct dentry *pdentry = dentry->d_parent; + struct dentry *udentry, *found = NULL; + struct ceph_dentry_info *di; + struct qstr dname; + u32 hash = dentry->d_name.hash; + int err; + + dname.name = dentry->d_name.name; + dname.len = dentry->d_name.len; + + rcu_read_lock(); + hash_for_each_possible_rcu(fsc->async_unlink_conflict, di, + hnode, hash) { + udentry = di->dentry; + + spin_lock(&udentry->d_lock); + if (udentry->d_name.hash != hash) + goto next; + if (unlikely(udentry->d_parent != pdentry)) + goto next; + if (!hash_hashed(&di->hnode)) + goto next; + + if (!test_bit(CEPH_DENTRY_ASYNC_UNLINK_BIT, &di->flags)) + pr_warn_client(cl, "dentry %p:%pd async unlink bit is not set\n", + dentry, dentry); + + if (!d_same_name(udentry, pdentry, &dname)) + goto next; + + found = dget_dlock(udentry); + spin_unlock(&udentry->d_lock); + break; +next: + spin_unlock(&udentry->d_lock); + } + rcu_read_unlock(); + + if (likely(!found)) + return 0; + + doutc(cl, "dentry %p:%pd conflict with old %p:%pd\n", dentry, dentry, + found, found); + + err = wait_on_bit(&di->flags, CEPH_DENTRY_ASYNC_UNLINK_BIT, + TASK_KILLABLE); + dput(found); + return err; +} + + +/* + * sessions + */ +const char *ceph_session_state_name(int s) +{ + switch (s) { + case CEPH_MDS_SESSION_NEW: return "new"; + case CEPH_MDS_SESSION_OPENING: return "opening"; + case CEPH_MDS_SESSION_OPEN: return "open"; + case CEPH_MDS_SESSION_HUNG: return "hung"; + case CEPH_MDS_SESSION_CLOSING: return "closing"; + case CEPH_MDS_SESSION_CLOSED: return "closed"; + case CEPH_MDS_SESSION_RESTARTING: return "restarting"; + case CEPH_MDS_SESSION_RECONNECTING: return "reconnecting"; + case CEPH_MDS_SESSION_REJECTED: return "rejected"; + default: return "???"; + } +} + +struct ceph_mds_session *ceph_get_mds_session(struct ceph_mds_session *s) +{ + if (refcount_inc_not_zero(&s->s_ref)) + return s; + return NULL; +} + +void ceph_put_mds_session(struct ceph_mds_session *s) +{ + if (IS_ERR_OR_NULL(s)) + return; + + if (refcount_dec_and_test(&s->s_ref)) { + if (s->s_auth.authorizer) + ceph_auth_destroy_authorizer(s->s_auth.authorizer); + WARN_ON(mutex_is_locked(&s->s_mutex)); + xa_destroy(&s->s_delegated_inos); + kfree(s); + } +} + +/* + * called under mdsc->mutex + */ +struct ceph_mds_session *__ceph_lookup_mds_session(struct ceph_mds_client *mdsc, + int mds) +{ + if (mds >= mdsc->max_sessions || !mdsc->sessions[mds]) + return NULL; + return ceph_get_mds_session(mdsc->sessions[mds]); +} + +static bool __have_session(struct ceph_mds_client *mdsc, int mds) +{ + if (mds >= mdsc->max_sessions || !mdsc->sessions[mds]) + return false; + else + return true; +} + +static int __verify_registered_session(struct ceph_mds_client *mdsc, + struct ceph_mds_session *s) +{ + if (s->s_mds >= mdsc->max_sessions || + mdsc->sessions[s->s_mds] != s) + return -ENOENT; + return 0; +} + +/* + * create+register a new session for given mds. + * called under mdsc->mutex. + */ +static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, + int mds) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_mds_session *s; + + if (READ_ONCE(mdsc->fsc->mount_state) == CEPH_MOUNT_FENCE_IO) + return ERR_PTR(-EIO); + + if (mds >= mdsc->mdsmap->possible_max_rank) + return ERR_PTR(-EINVAL); + + s = kzalloc_obj(*s, GFP_NOFS); + if (!s) + return ERR_PTR(-ENOMEM); + + if (mds >= mdsc->max_sessions) { + int newmax = 1 << get_count_order(mds + 1); + struct ceph_mds_session **sa; + size_t ptr_size = sizeof(struct ceph_mds_session *); + + doutc(cl, "realloc to %d\n", newmax); + sa = kcalloc(newmax, ptr_size, GFP_NOFS); + if (!sa) + goto fail_realloc; + if (mdsc->sessions) { + memcpy(sa, mdsc->sessions, + mdsc->max_sessions * ptr_size); + kfree(mdsc->sessions); + } + mdsc->sessions = sa; + mdsc->max_sessions = newmax; + } + + doutc(cl, "mds%d\n", mds); + s->s_mdsc = mdsc; + s->s_mds = mds; + s->s_state = CEPH_MDS_SESSION_NEW; + mutex_init(&s->s_mutex); + + ceph_con_init(&s->s_con, s, &mds_con_ops, &mdsc->fsc->client->msgr); + + atomic_set(&s->s_cap_gen, 1); + s->s_cap_ttl = jiffies - 1; + + spin_lock_init(&s->s_cap_lock); + INIT_LIST_HEAD(&s->s_caps); + refcount_set(&s->s_ref, 1); + INIT_LIST_HEAD(&s->s_waiting); + INIT_LIST_HEAD(&s->s_unsafe); + xa_init(&s->s_delegated_inos); + INIT_LIST_HEAD(&s->s_cap_releases); + INIT_WORK(&s->s_cap_release_work, ceph_cap_release_work); + + INIT_LIST_HEAD(&s->s_cap_dirty); + INIT_LIST_HEAD(&s->s_cap_flushing); + + mdsc->sessions[mds] = s; + atomic_inc(&mdsc->num_sessions); + refcount_inc(&s->s_ref); /* one ref to sessions[], one to caller */ + + ceph_con_open(&s->s_con, CEPH_ENTITY_TYPE_MDS, mds, + ceph_mdsmap_get_addr(mdsc->mdsmap, mds)); + + return s; + +fail_realloc: + kfree(s); + return ERR_PTR(-ENOMEM); +} + +/* + * called under mdsc->mutex + */ +static void __unregister_session(struct ceph_mds_client *mdsc, + struct ceph_mds_session *s) +{ + doutc(mdsc->fsc->client, "mds%d %p\n", s->s_mds, s); + BUG_ON(mdsc->sessions[s->s_mds] != s); + mdsc->sessions[s->s_mds] = NULL; + ceph_con_close(&s->s_con); + ceph_put_mds_session(s); + atomic_dec(&mdsc->num_sessions); +} + +/* + * drop session refs in request. + * + * should be last request ref, or hold mdsc->mutex + */ +static void put_request_session(struct ceph_mds_request *req) +{ + if (req->r_session) { + ceph_put_mds_session(req->r_session); + req->r_session = NULL; + } +} + +void ceph_mdsc_iterate_sessions(struct ceph_mds_client *mdsc, + void (*cb)(struct ceph_mds_session *), + bool check_state) +{ + int mds; + + mutex_lock(&mdsc->mutex); + for (mds = 0; mds < mdsc->max_sessions; ++mds) { + struct ceph_mds_session *s; + + s = __ceph_lookup_mds_session(mdsc, mds); + if (!s) + continue; + + if (check_state && !check_session_state(s)) { + ceph_put_mds_session(s); + continue; + } + + mutex_unlock(&mdsc->mutex); + cb(s); + ceph_put_mds_session(s); + mutex_lock(&mdsc->mutex); + } + mutex_unlock(&mdsc->mutex); +} + +void ceph_mdsc_release_request(struct kref *kref) +{ + struct ceph_mds_request *req = container_of(kref, + struct ceph_mds_request, + r_kref); + ceph_mdsc_release_dir_caps_async(req); + destroy_reply_info(&req->r_reply_info); + if (req->r_request) + ceph_msg_put(req->r_request); + if (req->r_reply) + ceph_msg_put(req->r_reply); + if (req->r_inode) { + ceph_put_cap_refs(ceph_inode(req->r_inode), CEPH_CAP_PIN); + iput(req->r_inode); + } + if (req->r_parent) { + ceph_put_cap_refs(ceph_inode(req->r_parent), CEPH_CAP_PIN); + iput(req->r_parent); + } + iput(req->r_target_inode); + iput(req->r_new_inode); + if (req->r_dentry) + dput(req->r_dentry); + if (req->r_old_dentry) + dput(req->r_old_dentry); + if (req->r_old_dentry_dir) { + /* + * track (and drop pins for) r_old_dentry_dir + * separately, since r_old_dentry's d_parent may have + * changed between the dir mutex being dropped and + * this request being freed. + */ + ceph_put_cap_refs(ceph_inode(req->r_old_dentry_dir), + CEPH_CAP_PIN); + iput(req->r_old_dentry_dir); + } + kfree(req->r_path1); + kfree(req->r_path2); + put_cred(req->r_cred); + if (req->r_mnt_idmap) + mnt_idmap_put(req->r_mnt_idmap); + if (req->r_pagelist) + ceph_pagelist_release(req->r_pagelist); + kfree(req->r_fscrypt_auth); + kfree(req->r_altname); + put_request_session(req); + ceph_unreserve_caps(req->r_mdsc, &req->r_caps_reservation); + WARN_ON_ONCE(!list_empty(&req->r_wait)); + kmem_cache_free(ceph_mds_request_cachep, req); +} + +DEFINE_RB_FUNCS(request, struct ceph_mds_request, r_tid, r_node) + +/* + * lookup session, bump ref if found. + * + * called under mdsc->mutex. + */ +static struct ceph_mds_request * +lookup_get_request(struct ceph_mds_client *mdsc, u64 tid) +{ + struct ceph_mds_request *req; + + req = lookup_request(&mdsc->request_tree, tid); + if (req) + ceph_mdsc_get_request(req); + + return req; +} + +/* + * Register an in-flight request, and assign a tid. Link to directory + * are modifying (if any). + * + * Called under mdsc->mutex. + */ +static void __register_request(struct ceph_mds_client *mdsc, + struct ceph_mds_request *req, + struct inode *dir) +{ + struct ceph_client *cl = mdsc->fsc->client; + int ret = 0; + + req->r_tid = ++mdsc->last_tid; + if (req->r_num_caps) { + ret = ceph_reserve_caps(mdsc, &req->r_caps_reservation, + req->r_num_caps); + if (ret < 0) { + pr_err_client(cl, "%p failed to reserve caps: %d\n", + req, ret); + /* set req->r_err to fail early from __do_request */ + req->r_err = ret; + return; + } + } + doutc(cl, "%p tid %lld\n", req, req->r_tid); + ceph_mdsc_get_request(req); + insert_request(&mdsc->request_tree, req); + + req->r_cred = get_current_cred(); + if (!req->r_mnt_idmap) + req->r_mnt_idmap = &nop_mnt_idmap; + + if (mdsc->oldest_tid == 0 && req->r_op != CEPH_MDS_OP_SETFILELOCK) + mdsc->oldest_tid = req->r_tid; + + if (dir) { + struct ceph_inode_info *ci = ceph_inode(dir); + + ihold(dir); + req->r_unsafe_dir = dir; + spin_lock(&ci->i_unsafe_lock); + list_add_tail(&req->r_unsafe_dir_item, &ci->i_unsafe_dirops); + spin_unlock(&ci->i_unsafe_lock); + } +} + +static void __unregister_request(struct ceph_mds_client *mdsc, + struct ceph_mds_request *req) +{ + doutc(mdsc->fsc->client, "%p tid %lld\n", req, req->r_tid); + + /* Never leave an unregistered request on an unsafe list! */ + list_del_init(&req->r_unsafe_item); + + if (req->r_tid == mdsc->oldest_tid) { + struct rb_node *p = rb_next(&req->r_node); + mdsc->oldest_tid = 0; + while (p) { + struct ceph_mds_request *next_req = + rb_entry(p, struct ceph_mds_request, r_node); + if (next_req->r_op != CEPH_MDS_OP_SETFILELOCK) { + mdsc->oldest_tid = next_req->r_tid; + break; + } + p = rb_next(p); + } + } + + erase_request(&mdsc->request_tree, req); + + if (req->r_unsafe_dir) { + struct ceph_inode_info *ci = ceph_inode(req->r_unsafe_dir); + spin_lock(&ci->i_unsafe_lock); + list_del_init(&req->r_unsafe_dir_item); + spin_unlock(&ci->i_unsafe_lock); + } + if (req->r_target_inode && + test_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags)) { + struct ceph_inode_info *ci = ceph_inode(req->r_target_inode); + spin_lock(&ci->i_unsafe_lock); + list_del_init(&req->r_unsafe_target_item); + spin_unlock(&ci->i_unsafe_lock); + } + + if (req->r_unsafe_dir) { + iput(req->r_unsafe_dir); + req->r_unsafe_dir = NULL; + } + + complete_all(&req->r_safe_completion); + + ceph_mdsc_put_request(req); +} + +/* + * Walk back up the dentry tree until we hit a dentry representing a + * non-snapshot inode. We do this using the rcu_read_lock (which must be held + * when calling this) to ensure that the objects won't disappear while we're + * working with them. Once we hit a candidate dentry, we attempt to take a + * reference to it, and return that as the result. + */ +static struct inode *get_nonsnap_parent(struct dentry *dentry) +{ + struct inode *inode = NULL; + + while (dentry && !IS_ROOT(dentry)) { + inode = d_inode_rcu(dentry); + if (!inode || ceph_snap(inode) == CEPH_NOSNAP) + break; + dentry = dentry->d_parent; + } + if (inode) + inode = igrab(inode); + return inode; +} + +/* + * Choose mds to send request to next. If there is a hint set in the + * request (e.g., due to a prior forward hint from the mds), use that. + * Otherwise, consult frag tree and/or caps to identify the + * appropriate mds. If all else fails, choose randomly. + * + * Called under mdsc->mutex. + */ +static int __choose_mds(struct ceph_mds_client *mdsc, + struct ceph_mds_request *req, + bool *random) +{ + struct inode *inode; + struct ceph_inode_info *ci; + struct ceph_cap *cap; + int mode = req->r_direct_mode; + int mds = -1; + u32 hash = req->r_direct_hash; + bool is_hash = test_bit(CEPH_MDS_R_DIRECT_IS_HASH, &req->r_req_flags); + struct ceph_client *cl = mdsc->fsc->client; + + if (random) + *random = false; + + /* + * is there a specific mds we should try? ignore hint if we have + * no session and the mds is not up (active or recovering). + */ + if (req->r_resend_mds >= 0 && + (__have_session(mdsc, req->r_resend_mds) || + ceph_mdsmap_get_state(mdsc->mdsmap, req->r_resend_mds) > 0)) { + doutc(cl, "using resend_mds mds%d\n", req->r_resend_mds); + return req->r_resend_mds; + } + + if (mode == USE_RANDOM_MDS) + goto random; + + inode = NULL; + if (req->r_inode) { + if (ceph_snap(req->r_inode) != CEPH_SNAPDIR) { + inode = req->r_inode; + ihold(inode); + } else { + /* req->r_dentry is non-null for LSSNAP request */ + rcu_read_lock(); + inode = get_nonsnap_parent(req->r_dentry); + rcu_read_unlock(); + doutc(cl, "using snapdir's parent %p %llx.%llx\n", + inode, ceph_vinop(inode)); + } + } else if (req->r_dentry) { + /* ignore race with rename; old or new d_parent is okay */ + struct dentry *parent; + struct inode *dir; + + rcu_read_lock(); + parent = READ_ONCE(req->r_dentry->d_parent); + dir = req->r_parent ? : d_inode_rcu(parent); + + if (!dir || dir->i_sb != mdsc->fsc->sb) { + /* not this fs or parent went negative */ + inode = d_inode(req->r_dentry); + if (inode) + ihold(inode); + } else if (ceph_snap(dir) != CEPH_NOSNAP) { + /* direct snapped/virtual snapdir requests + * based on parent dir inode */ + inode = get_nonsnap_parent(parent); + doutc(cl, "using nonsnap parent %p %llx.%llx\n", + inode, ceph_vinop(inode)); + } else { + /* dentry target */ + inode = d_inode(req->r_dentry); + if (!inode || mode == USE_AUTH_MDS) { + /* dir + name */ + inode = igrab(dir); + hash = ceph_dentry_hash(dir, req->r_dentry); + is_hash = true; + } else { + ihold(inode); + } + } + rcu_read_unlock(); + } + + if (!inode) + goto random; + + doutc(cl, "%p %llx.%llx is_hash=%d (0x%x) mode %d\n", inode, + ceph_vinop(inode), (int)is_hash, hash, mode); + ci = ceph_inode(inode); + + if (is_hash && S_ISDIR(inode->i_mode)) { + struct ceph_inode_frag frag; + int found; + + ceph_choose_frag(ci, hash, &frag, &found); + if (found) { + if (mode == USE_ANY_MDS && frag.ndist > 0) { + u8 r; + + /* choose a random replica */ + get_random_bytes(&r, 1); + r %= frag.ndist; + mds = frag.dist[r]; + doutc(cl, "%p %llx.%llx frag %u mds%d (%d/%d)\n", + inode, ceph_vinop(inode), frag.frag, + mds, (int)r, frag.ndist); + if (ceph_mdsmap_get_state(mdsc->mdsmap, mds) >= + CEPH_MDS_STATE_ACTIVE && + !ceph_mdsmap_is_laggy(mdsc->mdsmap, mds)) + goto out; + } + + /* since this file/dir wasn't known to be + * replicated, then we want to look for the + * authoritative mds. */ + if (frag.mds >= 0) { + /* choose auth mds */ + mds = frag.mds; + doutc(cl, "%p %llx.%llx frag %u mds%d (auth)\n", + inode, ceph_vinop(inode), frag.frag, mds); + if (ceph_mdsmap_get_state(mdsc->mdsmap, mds) >= + CEPH_MDS_STATE_ACTIVE) { + if (!ceph_mdsmap_is_laggy(mdsc->mdsmap, + mds)) + goto out; + } + } + mode = USE_AUTH_MDS; + } + } + + spin_lock(&ci->i_ceph_lock); + cap = NULL; + if (mode == USE_AUTH_MDS) + cap = ci->i_auth_cap; + if (!cap && !RB_EMPTY_ROOT(&ci->i_caps)) + cap = rb_entry(rb_first(&ci->i_caps), struct ceph_cap, ci_node); + if (!cap) { + spin_unlock(&ci->i_ceph_lock); + iput(inode); + goto random; + } + mds = cap->session->s_mds; + doutc(cl, "%p %llx.%llx mds%d (%scap %p)\n", inode, + ceph_vinop(inode), mds, + cap == ci->i_auth_cap ? "auth " : "", cap); + spin_unlock(&ci->i_ceph_lock); +out: + iput(inode); + return mds; + +random: + if (random) + *random = true; + + mds = ceph_mdsmap_get_random_mds(mdsc->mdsmap); + doutc(cl, "chose random mds%d\n", mds); + return mds; +} + + +/* + * session messages + */ +struct ceph_msg *ceph_create_session_msg(u32 op, u64 seq) +{ + struct ceph_msg *msg; + struct ceph_mds_session_head *h; + + msg = ceph_msg_new(CEPH_MSG_CLIENT_SESSION, sizeof(*h), GFP_NOFS, + false); + if (!msg) { + pr_err("ENOMEM creating session %s msg\n", + ceph_session_op_name(op)); + return NULL; + } + h = msg->front.iov_base; + h->op = cpu_to_le32(op); + h->seq = cpu_to_le64(seq); + + return msg; +} + +static const unsigned char feature_bits[] = CEPHFS_FEATURES_CLIENT_SUPPORTED; +#define FEATURE_BYTES(c) (DIV_ROUND_UP((size_t)feature_bits[c - 1] + 1, 64) * 8) +static int encode_supported_features(void **p, void *end) +{ + static const size_t count = ARRAY_SIZE(feature_bits); + + if (count > 0) { + size_t i; + size_t size = FEATURE_BYTES(count); + unsigned long bit; + + if (WARN_ON_ONCE(*p + 4 + size > end)) + return -ERANGE; + + ceph_encode_32(p, size); + memset(*p, 0, size); + for (i = 0; i < count; i++) { + bit = feature_bits[i]; + ((unsigned char *)(*p))[bit / 8] |= BIT(bit % 8); + } + *p += size; + } else { + if (WARN_ON_ONCE(*p + 4 > end)) + return -ERANGE; + + ceph_encode_32(p, 0); + } + + return 0; +} + +static const unsigned char metric_bits[] = CEPHFS_METRIC_SPEC_CLIENT_SUPPORTED; +#define METRIC_BYTES(cnt) (DIV_ROUND_UP((size_t)metric_bits[cnt - 1] + 1, 64) * 8) +static int encode_metric_spec(void **p, void *end) +{ + static const size_t count = ARRAY_SIZE(metric_bits); + + /* header */ + if (WARN_ON_ONCE(*p + 2 > end)) + return -ERANGE; + + ceph_encode_8(p, 1); /* version */ + ceph_encode_8(p, 1); /* compat */ + + if (count > 0) { + size_t i; + size_t size = METRIC_BYTES(count); + + if (WARN_ON_ONCE(*p + 4 + 4 + size > end)) + return -ERANGE; + + /* metric spec info length */ + ceph_encode_32(p, 4 + size); + + /* metric spec */ + ceph_encode_32(p, size); + memset(*p, 0, size); + for (i = 0; i < count; i++) + ((unsigned char *)(*p))[i / 8] |= BIT(metric_bits[i] % 8); + *p += size; + } else { + if (WARN_ON_ONCE(*p + 4 + 4 > end)) + return -ERANGE; + + /* metric spec info length */ + ceph_encode_32(p, 4); + /* metric spec */ + ceph_encode_32(p, 0); + } + + return 0; +} + +/* + * session message, specialization for CEPH_SESSION_REQUEST_OPEN + * to include additional client metadata fields. + */ +static struct ceph_msg * +create_session_full_msg(struct ceph_mds_client *mdsc, int op, u64 seq) +{ + struct ceph_msg *msg; + struct ceph_mds_session_head *h; + int i; + int extra_bytes = 0; + int metadata_key_count = 0; + struct ceph_options *opt = mdsc->fsc->client->options; + struct ceph_mount_options *fsopt = mdsc->fsc->mount_options; + struct ceph_client *cl = mdsc->fsc->client; + size_t size, count; + void *p, *end; + int ret; + + const char* metadata[][2] = { + {"hostname", mdsc->nodename}, + {"kernel_version", init_utsname()->release}, + {"entity_id", opt->name ? : ""}, + {"root", fsopt->server_path ? : "/"}, + {NULL, NULL} + }; + + /* Calculate serialized length of metadata */ + extra_bytes = 4; /* map length */ + for (i = 0; metadata[i][0]; ++i) { + extra_bytes += 8 + strlen(metadata[i][0]) + + strlen(metadata[i][1]); + metadata_key_count++; + } + + /* supported feature */ + size = 0; + count = ARRAY_SIZE(feature_bits); + if (count > 0) + size = FEATURE_BYTES(count); + extra_bytes += 4 + size; + + /* metric spec */ + size = 0; + count = ARRAY_SIZE(metric_bits); + if (count > 0) + size = METRIC_BYTES(count); + extra_bytes += 2 + 4 + 4 + size; + + /* flags, mds auth caps and oldest_client_tid */ + extra_bytes += 4 + 4 + 8; + + /* Allocate the message */ + msg = ceph_msg_new(CEPH_MSG_CLIENT_SESSION, sizeof(*h) + extra_bytes, + GFP_NOFS, false); + if (!msg) { + pr_err_client(cl, "ENOMEM creating session open msg\n"); + return ERR_PTR(-ENOMEM); + } + p = msg->front.iov_base; + end = p + msg->front.iov_len; + + h = p; + h->op = cpu_to_le32(op); + h->seq = cpu_to_le64(seq); + + /* + * Serialize client metadata into waiting buffer space, using + * the format that userspace expects for map + * + * ClientSession messages with metadata are v7 + */ + msg->hdr.version = cpu_to_le16(7); + msg->hdr.compat_version = cpu_to_le16(1); + + /* The write pointer, following the session_head structure */ + p += sizeof(*h); + + /* Number of entries in the map */ + ceph_encode_32(&p, metadata_key_count); + + /* Two length-prefixed strings for each entry in the map */ + for (i = 0; metadata[i][0]; ++i) { + size_t const key_len = strlen(metadata[i][0]); + size_t const val_len = strlen(metadata[i][1]); + + ceph_encode_32(&p, key_len); + memcpy(p, metadata[i][0], key_len); + p += key_len; + ceph_encode_32(&p, val_len); + memcpy(p, metadata[i][1], val_len); + p += val_len; + } + + ret = encode_supported_features(&p, end); + if (ret) { + pr_err_client(cl, "encode_supported_features failed!\n"); + ceph_msg_put(msg); + return ERR_PTR(ret); + } + + ret = encode_metric_spec(&p, end); + if (ret) { + pr_err_client(cl, "encode_metric_spec failed!\n"); + ceph_msg_put(msg); + return ERR_PTR(ret); + } + + /* version == 5, flags */ + ceph_encode_32(&p, 0); + + /* version == 6, mds auth caps */ + ceph_encode_32(&p, 0); + + /* version == 7, oldest_client_tid */ + ceph_encode_64(&p, mdsc->oldest_tid); + + msg->front.iov_len = p - msg->front.iov_base; + msg->hdr.front_len = cpu_to_le32(msg->front.iov_len); + + return msg; +} + +/* + * send session open request. + * + * called under mdsc->mutex + */ +static int __open_session(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session) +{ + struct ceph_msg *msg; + int mstate; + int mds = session->s_mds; + + if (READ_ONCE(mdsc->fsc->mount_state) == CEPH_MOUNT_FENCE_IO) + return -EIO; + + /* wait for mds to go active? */ + mstate = ceph_mdsmap_get_state(mdsc->mdsmap, mds); + doutc(mdsc->fsc->client, "open_session to mds%d (%s)\n", mds, + ceph_mds_state_name(mstate)); + session->s_state = CEPH_MDS_SESSION_OPENING; + session->s_renew_requested = jiffies; + + /* send connect message */ + msg = create_session_full_msg(mdsc, CEPH_SESSION_REQUEST_OPEN, + session->s_seq); + if (IS_ERR(msg)) + return PTR_ERR(msg); + ceph_con_send(&session->s_con, msg); + return 0; +} + +/* + * open sessions for any export targets for the given mds + * + * called under mdsc->mutex + */ +static struct ceph_mds_session * +__open_export_target_session(struct ceph_mds_client *mdsc, int target) +{ + struct ceph_mds_session *session; + int ret; + + session = __ceph_lookup_mds_session(mdsc, target); + if (!session) { + session = register_session(mdsc, target); + if (IS_ERR(session)) + return session; + } + if (session->s_state == CEPH_MDS_SESSION_NEW || + session->s_state == CEPH_MDS_SESSION_CLOSING) { + ret = __open_session(mdsc, session); + if (ret) + return ERR_PTR(ret); + } + + return session; +} + +struct ceph_mds_session * +ceph_mdsc_open_export_target_session(struct ceph_mds_client *mdsc, int target) +{ + struct ceph_mds_session *session; + struct ceph_client *cl = mdsc->fsc->client; + + doutc(cl, "to mds%d\n", target); + + mutex_lock(&mdsc->mutex); + session = __open_export_target_session(mdsc, target); + mutex_unlock(&mdsc->mutex); + + return session; +} + +static void __open_export_target_sessions(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session) +{ + struct ceph_mds_info *mi; + struct ceph_mds_session *ts; + int i, mds = session->s_mds; + struct ceph_client *cl = mdsc->fsc->client; + + if (mds >= mdsc->mdsmap->possible_max_rank) + return; + + mi = &mdsc->mdsmap->m_info[mds]; + doutc(cl, "for mds%d (%d targets)\n", session->s_mds, + mi->num_export_targets); + + for (i = 0; i < mi->num_export_targets; i++) { + ts = __open_export_target_session(mdsc, mi->export_targets[i]); + ceph_put_mds_session(ts); + } +} + +/* + * session caps + */ + +static void detach_cap_releases(struct ceph_mds_session *session, + struct list_head *target) +{ + struct ceph_client *cl = session->s_mdsc->fsc->client; + + lockdep_assert_held(&session->s_cap_lock); + + list_splice_init(&session->s_cap_releases, target); + session->s_num_cap_releases = 0; + doutc(cl, "mds%d\n", session->s_mds); +} + +static void dispose_cap_releases(struct ceph_mds_client *mdsc, + struct list_head *dispose) +{ + while (!list_empty(dispose)) { + struct ceph_cap *cap; + /* zero out the in-progress message */ + cap = list_first_entry(dispose, struct ceph_cap, session_caps); + list_del(&cap->session_caps); + ceph_put_cap(mdsc, cap); + } +} + +static void cleanup_session_requests(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_mds_request *req; + struct rb_node *p; + + doutc(cl, "mds%d\n", session->s_mds); + mutex_lock(&mdsc->mutex); + while (!list_empty(&session->s_unsafe)) { + req = list_first_entry(&session->s_unsafe, + struct ceph_mds_request, r_unsafe_item); + pr_warn_ratelimited_client(cl, " dropping unsafe request %llu\n", + req->r_tid); + if (req->r_target_inode) + mapping_set_error(req->r_target_inode->i_mapping, -EIO); + if (req->r_unsafe_dir) + mapping_set_error(req->r_unsafe_dir->i_mapping, -EIO); + __unregister_request(mdsc, req); + } + /* zero r_attempts, so kick_requests() will re-send requests */ + p = rb_first(&mdsc->request_tree); + while (p) { + req = rb_entry(p, struct ceph_mds_request, r_node); + p = rb_next(p); + if (req->r_session && + req->r_session->s_mds == session->s_mds) + req->r_attempts = 0; + } + mutex_unlock(&mdsc->mutex); +} + +/* + * Helper to safely iterate over all caps associated with a session, with + * special care taken to handle a racing __ceph_remove_cap(). + * + * Caller must hold session s_mutex. + */ +int ceph_iterate_session_caps(struct ceph_mds_session *session, + int (*cb)(struct inode *, int mds, void *), + void *arg) +{ + struct ceph_client *cl = session->s_mdsc->fsc->client; + struct list_head *p; + struct ceph_cap *cap; + struct inode *inode, *last_inode = NULL; + struct ceph_cap *old_cap = NULL; + int ret; + + doutc(cl, "%p mds%d\n", session, session->s_mds); + spin_lock(&session->s_cap_lock); + p = session->s_caps.next; + while (p != &session->s_caps) { + int mds; + + cap = list_entry(p, struct ceph_cap, session_caps); + inode = igrab(&cap->ci->netfs.inode); + if (!inode) { + p = p->next; + continue; + } + session->s_cap_iterator = cap; + mds = cap->mds; + spin_unlock(&session->s_cap_lock); + + if (last_inode) { + iput(last_inode); + last_inode = NULL; + } + if (old_cap) { + ceph_put_cap(session->s_mdsc, old_cap); + old_cap = NULL; + } + + ret = cb(inode, mds, arg); + last_inode = inode; + + spin_lock(&session->s_cap_lock); + p = p->next; + if (!cap->ci) { + doutc(cl, "finishing cap %p removal\n", cap); + BUG_ON(cap->session != session); + cap->session = NULL; + list_del_init(&cap->session_caps); + session->s_nr_caps--; + atomic64_dec(&session->s_mdsc->metric.total_caps); + if (cap->queue_release) + __ceph_queue_cap_release(session, cap); + else + old_cap = cap; /* put_cap it w/o locks held */ + } + if (ret < 0) + goto out; + } + ret = 0; +out: + session->s_cap_iterator = NULL; + spin_unlock(&session->s_cap_lock); + + iput(last_inode); + if (old_cap) + ceph_put_cap(session->s_mdsc, old_cap); + + return ret; +} + +static int remove_session_caps_cb(struct inode *inode, int mds, void *arg) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_client *cl = ceph_inode_to_client(inode); + bool invalidate = false; + struct ceph_cap *cap; + int iputs = 0; + + spin_lock(&ci->i_ceph_lock); + cap = __get_cap_for_mds(ci, mds); + if (cap) { + doutc(cl, " removing cap %p, ci is %p, inode is %p\n", + cap, ci, &ci->netfs.inode); + + iputs = ceph_purge_inode_cap(inode, cap, &invalidate); + } + spin_unlock(&ci->i_ceph_lock); + + if (cap) + wake_up_all(&ci->i_cap_wq); + if (invalidate) + ceph_queue_invalidate(inode); + while (iputs--) + iput(inode); + return 0; +} + +/* + * caller must hold session s_mutex + */ +static void remove_session_caps(struct ceph_mds_session *session) +{ + struct ceph_fs_client *fsc = session->s_mdsc->fsc; + struct super_block *sb = fsc->sb; + LIST_HEAD(dispose); + + doutc(fsc->client, "on %p\n", session); + ceph_iterate_session_caps(session, remove_session_caps_cb, fsc); + + wake_up_all(&fsc->mdsc->cap_flushing_wq); + + spin_lock(&session->s_cap_lock); + if (session->s_nr_caps > 0) { + struct inode *inode; + struct ceph_cap *cap, *prev = NULL; + struct ceph_vino vino; + /* + * iterate_session_caps() skips inodes that are being + * deleted, we need to wait until deletions are complete. + * __wait_on_freeing_inode() is designed for the job, + * but it is not exported, so use lookup inode function + * to access it. + */ + while (!list_empty(&session->s_caps)) { + cap = list_entry(session->s_caps.next, + struct ceph_cap, session_caps); + if (cap == prev) + break; + prev = cap; + vino = cap->ci->i_vino; + spin_unlock(&session->s_cap_lock); + + inode = ceph_find_inode(sb, vino); + iput(inode); + + spin_lock(&session->s_cap_lock); + } + } + + // drop cap expires and unlock s_cap_lock + detach_cap_releases(session, &dispose); + + BUG_ON(session->s_nr_caps > 0); + BUG_ON(!list_empty(&session->s_cap_flushing)); + spin_unlock(&session->s_cap_lock); + dispose_cap_releases(session->s_mdsc, &dispose); +} + +enum { + RECONNECT, + RENEWCAPS, + FORCE_RO, +}; + +/* + * wake up any threads waiting on this session's caps. if the cap is + * old (didn't get renewed on the client reconnect), remove it now. + * + * caller must hold s_mutex. + */ +static int wake_up_session_cb(struct inode *inode, int mds, void *arg) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + unsigned long ev = (unsigned long)arg; + + if (ev == RECONNECT) { + spin_lock(&ci->i_ceph_lock); + ci->i_wanted_max_size = 0; + ci->i_requested_max_size = 0; + spin_unlock(&ci->i_ceph_lock); + } else if (ev == RENEWCAPS) { + struct ceph_cap *cap; + + spin_lock(&ci->i_ceph_lock); + cap = __get_cap_for_mds(ci, mds); + /* mds did not re-issue stale cap */ + if (cap && cap->cap_gen < atomic_read(&cap->session->s_cap_gen)) + cap->issued = cap->implemented = CEPH_CAP_PIN; + spin_unlock(&ci->i_ceph_lock); + } else if (ev == FORCE_RO) { + } + wake_up_all(&ci->i_cap_wq); + return 0; +} + +static void wake_up_session_caps(struct ceph_mds_session *session, int ev) +{ + struct ceph_client *cl = session->s_mdsc->fsc->client; + + doutc(cl, "session %p mds%d\n", session, session->s_mds); + ceph_iterate_session_caps(session, wake_up_session_cb, + (void *)(unsigned long)ev); +} + +/* + * Send periodic message to MDS renewing all currently held caps. The + * ack will reset the expiration for all caps from this session. + * + * caller holds s_mutex + */ +static int send_renew_caps(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_msg *msg; + int state; + + if (time_after_eq(jiffies, session->s_cap_ttl) && + time_after_eq(session->s_cap_ttl, session->s_renew_requested)) + pr_info_client(cl, "mds%d caps stale\n", session->s_mds); + session->s_renew_requested = jiffies; + + /* do not try to renew caps until a recovering mds has reconnected + * with its clients. */ + state = ceph_mdsmap_get_state(mdsc->mdsmap, session->s_mds); + if (state < CEPH_MDS_STATE_RECONNECT) { + doutc(cl, "ignoring mds%d (%s)\n", session->s_mds, + ceph_mds_state_name(state)); + return 0; + } + + doutc(cl, "to mds%d (%s)\n", session->s_mds, + ceph_mds_state_name(state)); + msg = create_session_full_msg(mdsc, CEPH_SESSION_REQUEST_RENEWCAPS, + ++session->s_renew_seq); + if (IS_ERR(msg)) + return PTR_ERR(msg); + ceph_con_send(&session->s_con, msg); + return 0; +} + +static int send_flushmsg_ack(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session, u64 seq) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_msg *msg; + + doutc(cl, "to mds%d (%s)s seq %lld\n", session->s_mds, + ceph_session_state_name(session->s_state), seq); + msg = ceph_create_session_msg(CEPH_SESSION_FLUSHMSG_ACK, seq); + if (!msg) + return -ENOMEM; + ceph_con_send(&session->s_con, msg); + return 0; +} + + +/* + * Note new cap ttl, and any transition from stale -> not stale (fresh?). + * + * Called under session->s_mutex + */ +static void renewed_caps(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session, int is_renew) +{ + struct ceph_client *cl = mdsc->fsc->client; + int was_stale; + int wake = 0; + + spin_lock(&session->s_cap_lock); + was_stale = is_renew && time_after_eq(jiffies, session->s_cap_ttl); + + session->s_cap_ttl = session->s_renew_requested + + mdsc->mdsmap->m_session_timeout*HZ; + + if (was_stale) { + if (time_before(jiffies, session->s_cap_ttl)) { + pr_info_client(cl, "mds%d caps renewed\n", + session->s_mds); + wake = 1; + } else { + pr_info_client(cl, "mds%d caps still stale\n", + session->s_mds); + } + } + doutc(cl, "mds%d ttl now %lu, was %s, now %s\n", session->s_mds, + session->s_cap_ttl, was_stale ? "stale" : "fresh", + time_before(jiffies, session->s_cap_ttl) ? "stale" : "fresh"); + spin_unlock(&session->s_cap_lock); + + if (wake) + wake_up_session_caps(session, RENEWCAPS); +} + +/* + * send a session close request + */ +static int request_close_session(struct ceph_mds_session *session) +{ + struct ceph_client *cl = session->s_mdsc->fsc->client; + struct ceph_msg *msg; + + doutc(cl, "mds%d state %s seq %lld\n", session->s_mds, + ceph_session_state_name(session->s_state), session->s_seq); + msg = ceph_create_session_msg(CEPH_SESSION_REQUEST_CLOSE, + session->s_seq); + if (!msg) + return -ENOMEM; + ceph_con_send(&session->s_con, msg); + return 1; +} + +/* + * Called with s_mutex held. + */ +static int __close_session(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session) +{ + if (session->s_state >= CEPH_MDS_SESSION_CLOSING) + return 0; + session->s_state = CEPH_MDS_SESSION_CLOSING; + return request_close_session(session); +} + +static bool drop_negative_children(struct dentry *dentry) +{ + struct dentry *child; + bool all_negative = true; + + if (!d_is_dir(dentry)) + goto out; + + spin_lock(&dentry->d_lock); + hlist_for_each_entry(child, &dentry->d_children, d_sib) { + if (d_really_is_positive(child)) { + all_negative = false; + break; + } + } + spin_unlock(&dentry->d_lock); + + if (all_negative) + shrink_dcache_parent(dentry); +out: + return all_negative; +} + +/* + * Trim old(er) caps. + * + * Because we can't cache an inode without one or more caps, we do + * this indirectly: if a cap is unused, we prune its aliases, at which + * point the inode will hopefully get dropped to. + * + * Yes, this is a bit sloppy. Our only real goal here is to respond to + * memory pressure from the MDS, though, so it needn't be perfect. + */ +static int trim_caps_cb(struct inode *inode, int mds, void *arg) +{ + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb); + struct ceph_client *cl = mdsc->fsc->client; + int *remaining = arg; + struct ceph_inode_info *ci = ceph_inode(inode); + int used, wanted, oissued, mine; + struct ceph_cap *cap; + + if (*remaining <= 0) + return -1; + + spin_lock(&ci->i_ceph_lock); + cap = __get_cap_for_mds(ci, mds); + if (!cap) { + spin_unlock(&ci->i_ceph_lock); + return 0; + } + mine = cap->issued | cap->implemented; + used = __ceph_caps_used(ci); + wanted = __ceph_caps_file_wanted(ci); + oissued = __ceph_caps_issued_other(ci, cap); + + doutc(cl, "%p %llx.%llx cap %p mine %s oissued %s used %s wanted %s\n", + inode, ceph_vinop(inode), cap, ceph_cap_string(mine), + ceph_cap_string(oissued), ceph_cap_string(used), + ceph_cap_string(wanted)); + if (cap == ci->i_auth_cap) { + if (ci->i_dirty_caps || ci->i_flushing_caps || + !list_empty(&ci->i_cap_snaps)) + goto out; + if ((used | wanted) & CEPH_CAP_ANY_WR) + goto out; + /* Note: it's possible that i_filelock_ref becomes non-zero + * after dropping auth caps. It doesn't hurt because reply + * of lock mds request will re-add auth caps. */ + if (atomic_read(&ci->i_filelock_ref) > 0) + goto out; + } + /* The inode has cached pages, but it's no longer used. + * we can safely drop it */ + if (S_ISREG(inode->i_mode) && + wanted == 0 && used == CEPH_CAP_FILE_CACHE && + !(oissued & CEPH_CAP_FILE_CACHE)) { + used = 0; + oissued = 0; + } + if ((used | wanted) & ~oissued & mine) + goto out; /* we need these caps */ + + if (oissued) { + /* we aren't the only cap.. just remove us */ + ceph_remove_cap(mdsc, cap, true); + (*remaining)--; + } else { + struct dentry *dentry; + /* try dropping referring dentries */ + spin_unlock(&ci->i_ceph_lock); + dentry = d_find_any_alias(inode); + if (dentry && drop_negative_children(dentry)) { + int count; + dput(dentry); + d_prune_aliases(inode); + count = icount_read_once(inode); + if (count == 1) + (*remaining)--; + doutc(cl, "%p %llx.%llx cap %p pruned, count now %d\n", + inode, ceph_vinop(inode), cap, count); + } else { + dput(dentry); + } + return 0; + } + +out: + spin_unlock(&ci->i_ceph_lock); + return 0; +} + +/* + * Trim session cap count down to some max number. + */ +int ceph_trim_caps(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session, + int max_caps) +{ + struct ceph_client *cl = mdsc->fsc->client; + int trim_caps = session->s_nr_caps - max_caps; + + doutc(cl, "mds%d start: %d / %d, trim %d\n", session->s_mds, + session->s_nr_caps, max_caps, trim_caps); + if (trim_caps > 0) { + int remaining = trim_caps; + + ceph_iterate_session_caps(session, trim_caps_cb, &remaining); + doutc(cl, "mds%d done: %d / %d, trimmed %d\n", + session->s_mds, session->s_nr_caps, max_caps, + trim_caps - remaining); + } + + ceph_flush_session_cap_releases(mdsc, session); + return 0; +} + +static int check_caps_flush(struct ceph_mds_client *mdsc, + u64 want_flush_tid) +{ + struct ceph_client *cl = mdsc->fsc->client; + int ret = 1; + + spin_lock(&mdsc->cap_dirty_lock); + if (!list_empty(&mdsc->cap_flush_list)) { + struct ceph_cap_flush *cf = + list_first_entry(&mdsc->cap_flush_list, + struct ceph_cap_flush, g_list); + if (cf->tid <= want_flush_tid) { + doutc(cl, "still flushing tid %llu <= %llu\n", + cf->tid, want_flush_tid); + ret = 0; + } + } + spin_unlock(&mdsc->cap_dirty_lock); + return ret; +} + +/* + * Snapshot of a single cap_flush entry for diagnostic dump. + * Collected under cap_dirty_lock, printed after releasing it. + */ +struct flush_dump_entry { + u64 ino; /* inode number */ + u64 snap; /* snap id */ + int caps; /* dirty cap bits */ + u64 tid; /* flush transaction id */ + u64 last_ack; /* most recent ack tid for this inode */ + bool wake; /* whether completion was requested */ + bool is_capsnap; /* true if this is a cap snap flush */ + bool ci_null; /* true if cf->ci was unexpectedly NULL */ +}; + +/* + * Dump pending cap flushes for diagnostic purposes. + * + * cf->ci is safe to dereference here: cap_flush entries hold a + * reference on the inode (via the cap), and entries are removed from + * cap_flush_list under cap_dirty_lock before the cap (and thus the + * inode reference) is released. Holding cap_dirty_lock therefore + * guarantees the inode remains valid for the lifetime of the scan. + */ + +static void dump_cap_flushes(struct ceph_mds_client *mdsc, u64 want_tid) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct flush_dump_entry entries[CEPH_CAP_FLUSH_MAX_DUMP_ENTRIES]; + struct ceph_cap_flush *cf; + int n = 0, remaining = 0; + int i; + + spin_lock(&mdsc->cap_dirty_lock); + list_for_each_entry(cf, &mdsc->cap_flush_list, g_list) { + if (cf->tid > want_tid) + break; + if (n < CEPH_CAP_FLUSH_MAX_DUMP_ENTRIES) { + struct flush_dump_entry *e = &entries[n++]; + + e->ci_null = WARN_ON_ONCE(!cf->ci); + if (!e->ci_null) { + e->ino = ceph_ino(&cf->ci->netfs.inode); + e->snap = ceph_snap(&cf->ci->netfs.inode); + e->last_ack = READ_ONCE(cf->ci->i_last_cap_flush_ack); + } + e->caps = cf->caps; + e->tid = cf->tid; + e->wake = cf->wake; + e->is_capsnap = cf->is_capsnap; + } else { + remaining++; + } + } + spin_unlock(&mdsc->cap_dirty_lock); + + pr_info_client(cl, "still waiting for cap flushes through %llu:\n", + want_tid); + for (i = 0; i < n; i++) { + struct flush_dump_entry *e = &entries[i]; + + if (e->ci_null) + pr_info_client(cl, + " (null ci) %s tid=%llu wake=%d%s\n", + ceph_cap_string(e->caps), e->tid, + e->wake, + e->is_capsnap ? " is_capsnap" : ""); + else + pr_info_client(cl, + " %llx.%llx %s tid=%llu last_ack=%llu wake=%d%s\n", + e->ino, e->snap, + ceph_cap_string(e->caps), e->tid, + e->last_ack, e->wake, + e->is_capsnap ? " is_capsnap" : ""); + } + if (remaining) + pr_info_client(cl, " ... and %d more pending flushes\n", + remaining); +} + +/* + * Wait for all cap flushes through @want_flush_tid to complete. + * Periodically dumps pending cap flush state for diagnostics. + */ +static void wait_caps_flush(struct ceph_mds_client *mdsc, + u64 want_flush_tid) +{ + struct ceph_client *cl = mdsc->fsc->client; + int i = 0; + long ret; + + doutc(cl, "want %llu\n", want_flush_tid); + + do { + /* 60 * HZ fits in a long on all supported architectures. */ + ret = wait_event_timeout(mdsc->cap_flushing_wq, + check_caps_flush(mdsc, want_flush_tid), + CEPH_CAP_FLUSH_WAIT_TIMEOUT_SEC * HZ); + if (ret == 0) { + if (i < CEPH_CAP_FLUSH_MAX_DUMP_ITERS) + dump_cap_flushes(mdsc, want_flush_tid); + else if (i == CEPH_CAP_FLUSH_MAX_DUMP_ITERS) + pr_info_client(cl, + "still waiting for cap flushes; suppressing further dumps\n"); + i++; + } + } while (ret == 0); + + doutc(cl, "ok, flushed thru %llu\n", want_flush_tid); +} + +/* + * called under s_mutex + */ +static void ceph_send_cap_releases(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_msg *msg = NULL; + struct ceph_mds_cap_release *head; + struct ceph_mds_cap_item *item; + struct ceph_osd_client *osdc = &mdsc->fsc->client->osdc; + struct ceph_cap *cap; + LIST_HEAD(tmp_list); + int num_cap_releases; + __le32 barrier, *cap_barrier; + + down_read(&osdc->lock); + barrier = cpu_to_le32(osdc->epoch_barrier); + up_read(&osdc->lock); + + spin_lock(&session->s_cap_lock); +again: + list_splice_init(&session->s_cap_releases, &tmp_list); + num_cap_releases = session->s_num_cap_releases; + session->s_num_cap_releases = 0; + spin_unlock(&session->s_cap_lock); + + while (!list_empty(&tmp_list)) { + if (!msg) { + msg = ceph_msg_new(CEPH_MSG_CLIENT_CAPRELEASE, + PAGE_SIZE, GFP_NOFS, false); + if (!msg) + goto out_err; + head = msg->front.iov_base; + head->num = cpu_to_le32(0); + msg->front.iov_len = sizeof(*head); + + msg->hdr.version = cpu_to_le16(2); + msg->hdr.compat_version = cpu_to_le16(1); + } + + cap = list_first_entry(&tmp_list, struct ceph_cap, + session_caps); + list_del(&cap->session_caps); + num_cap_releases--; + + head = msg->front.iov_base; + put_unaligned_le32(get_unaligned_le32(&head->num) + 1, + &head->num); + item = msg->front.iov_base + msg->front.iov_len; + item->ino = cpu_to_le64(cap->cap_ino); + item->cap_id = cpu_to_le64(cap->cap_id); + item->migrate_seq = cpu_to_le32(cap->mseq); + item->issue_seq = cpu_to_le32(cap->issue_seq); + msg->front.iov_len += sizeof(*item); + + ceph_put_cap(mdsc, cap); + + if (le32_to_cpu(head->num) == CEPH_CAPS_PER_RELEASE) { + // Append cap_barrier field + cap_barrier = msg->front.iov_base + msg->front.iov_len; + *cap_barrier = barrier; + msg->front.iov_len += sizeof(*cap_barrier); + + msg->hdr.front_len = cpu_to_le32(msg->front.iov_len); + doutc(cl, "mds%d %p\n", session->s_mds, msg); + ceph_con_send(&session->s_con, msg); + msg = NULL; + } + } + + BUG_ON(num_cap_releases != 0); + + spin_lock(&session->s_cap_lock); + if (!list_empty(&session->s_cap_releases)) + goto again; + spin_unlock(&session->s_cap_lock); + + if (msg) { + // Append cap_barrier field + cap_barrier = msg->front.iov_base + msg->front.iov_len; + *cap_barrier = barrier; + msg->front.iov_len += sizeof(*cap_barrier); + + msg->hdr.front_len = cpu_to_le32(msg->front.iov_len); + doutc(cl, "mds%d %p\n", session->s_mds, msg); + ceph_con_send(&session->s_con, msg); + } + return; +out_err: + pr_err_client(cl, "mds%d, failed to allocate message\n", + session->s_mds); + spin_lock(&session->s_cap_lock); + list_splice(&tmp_list, &session->s_cap_releases); + session->s_num_cap_releases += num_cap_releases; + spin_unlock(&session->s_cap_lock); +} + +static void ceph_cap_release_work(struct work_struct *work) +{ + struct ceph_mds_session *session = + container_of(work, struct ceph_mds_session, s_cap_release_work); + + mutex_lock(&session->s_mutex); + if (session->s_state == CEPH_MDS_SESSION_OPEN || + session->s_state == CEPH_MDS_SESSION_HUNG) + ceph_send_cap_releases(session->s_mdsc, session); + mutex_unlock(&session->s_mutex); + ceph_put_mds_session(session); +} + +void ceph_flush_session_cap_releases(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session) +{ + struct ceph_client *cl = mdsc->fsc->client; + if (mdsc->stopping) + return; + + ceph_get_mds_session(session); + if (queue_work(mdsc->fsc->cap_wq, + &session->s_cap_release_work)) { + doutc(cl, "cap release work queued\n"); + } else { + ceph_put_mds_session(session); + doutc(cl, "failed to queue cap release work\n"); + } +} + +/* + * caller holds session->s_cap_lock + */ +void __ceph_queue_cap_release(struct ceph_mds_session *session, + struct ceph_cap *cap) +{ + list_add_tail(&cap->session_caps, &session->s_cap_releases); + session->s_num_cap_releases++; + + if (!(session->s_num_cap_releases % CEPH_CAPS_PER_RELEASE)) + ceph_flush_session_cap_releases(session->s_mdsc, session); +} + +static void ceph_cap_reclaim_work(struct work_struct *work) +{ + struct ceph_mds_client *mdsc = + container_of(work, struct ceph_mds_client, cap_reclaim_work); + int ret = ceph_trim_dentries(mdsc); + if (ret == -EAGAIN) + ceph_queue_cap_reclaim_work(mdsc); +} + +void ceph_queue_cap_reclaim_work(struct ceph_mds_client *mdsc) +{ + struct ceph_client *cl = mdsc->fsc->client; + if (mdsc->stopping) + return; + + if (queue_work(mdsc->fsc->cap_wq, &mdsc->cap_reclaim_work)) { + doutc(cl, "caps reclaim work queued\n"); + } else { + doutc(cl, "failed to queue caps release work\n"); + } +} + +void ceph_reclaim_caps_nr(struct ceph_mds_client *mdsc, int nr) +{ + int val; + if (!nr) + return; + val = atomic_add_return(nr, &mdsc->cap_reclaim_pending); + if ((val % CEPH_CAPS_PER_RELEASE) < nr) { + atomic_set(&mdsc->cap_reclaim_pending, 0); + ceph_queue_cap_reclaim_work(mdsc); + } +} + +void ceph_queue_cap_unlink_work(struct ceph_mds_client *mdsc) +{ + struct ceph_client *cl = mdsc->fsc->client; + if (mdsc->stopping) + return; + + if (queue_work(mdsc->fsc->cap_wq, &mdsc->cap_unlink_work)) { + doutc(cl, "caps unlink work queued\n"); + } else { + doutc(cl, "failed to queue caps unlink work\n"); + } +} + +static void ceph_cap_unlink_work(struct work_struct *work) +{ + struct ceph_mds_client *mdsc = + container_of(work, struct ceph_mds_client, cap_unlink_work); + struct ceph_client *cl = mdsc->fsc->client; + + doutc(cl, "begin\n"); + spin_lock(&mdsc->cap_delay_lock); + while (!list_empty(&mdsc->cap_unlink_delay_list)) { + struct ceph_inode_info *ci; + struct inode *inode; + + ci = list_first_entry(&mdsc->cap_unlink_delay_list, + struct ceph_inode_info, + i_cap_delay_list); + list_del_init(&ci->i_cap_delay_list); + + inode = igrab(&ci->netfs.inode); + if (inode) { + spin_unlock(&mdsc->cap_delay_lock); + doutc(cl, "on %p %llx.%llx\n", inode, + ceph_vinop(inode)); + ceph_check_caps(ci, CHECK_CAPS_FLUSH); + iput(inode); + spin_lock(&mdsc->cap_delay_lock); + } + } + spin_unlock(&mdsc->cap_delay_lock); + doutc(cl, "done\n"); +} + +/* + * requests + */ + +int ceph_alloc_readdir_reply_buffer(struct ceph_mds_request *req, + struct inode *dir) +{ + struct ceph_inode_info *ci = ceph_inode(dir); + struct ceph_mds_reply_info_parsed *rinfo = &req->r_reply_info; + struct ceph_mount_options *opt = req->r_mdsc->fsc->mount_options; + size_t size = sizeof(struct ceph_mds_reply_dir_entry); + unsigned int num_entries; + u64 bytes_count; + int order; + + spin_lock(&ci->i_ceph_lock); + num_entries = ci->i_files + ci->i_subdirs; + spin_unlock(&ci->i_ceph_lock); + num_entries = max(num_entries, 1U); + num_entries = min(num_entries, opt->max_readdir); + + bytes_count = (u64)size * num_entries; + if (unlikely(bytes_count > ULONG_MAX)) + bytes_count = ULONG_MAX; + + order = get_order((unsigned long)bytes_count); + while (order >= 0) { + rinfo->dir_entries = (void*)__get_free_pages(GFP_KERNEL | + __GFP_NOWARN | + __GFP_ZERO, + order); + if (rinfo->dir_entries) + break; + order--; + } + if (!rinfo->dir_entries || unlikely(order < 0)) + return -ENOMEM; + + num_entries = (PAGE_SIZE << order) / size; + num_entries = min(num_entries, opt->max_readdir); + + rinfo->dir_buf_size = PAGE_SIZE << order; + req->r_num_caps = num_entries + 1; + req->r_args.readdir.max_entries = cpu_to_le32(num_entries); + req->r_args.readdir.max_bytes = cpu_to_le32(opt->max_readdir_bytes); + return 0; +} + +/* + * Create an mds request. + */ +struct ceph_mds_request * +ceph_mdsc_create_request(struct ceph_mds_client *mdsc, int op, int mode) +{ + struct ceph_mds_request *req; + + req = kmem_cache_zalloc(ceph_mds_request_cachep, GFP_NOFS); + if (!req) + return ERR_PTR(-ENOMEM); + + mutex_init(&req->r_fill_mutex); + req->r_mdsc = mdsc; + req->r_started = jiffies; + req->r_start_latency = ktime_get(); + req->r_resend_mds = -1; + INIT_LIST_HEAD(&req->r_unsafe_dir_item); + INIT_LIST_HEAD(&req->r_unsafe_target_item); + req->r_fmode = -1; + req->r_feature_needed = -1; + kref_init(&req->r_kref); + RB_CLEAR_NODE(&req->r_node); + INIT_LIST_HEAD(&req->r_wait); + init_completion(&req->r_completion); + init_completion(&req->r_safe_completion); + INIT_LIST_HEAD(&req->r_unsafe_item); + + ktime_get_coarse_real_ts64(&req->r_stamp); + + req->r_op = op; + req->r_direct_mode = mode; + return req; +} + +/* + * return oldest (lowest) request, tid in request tree, 0 if none. + * + * called under mdsc->mutex. + */ +static struct ceph_mds_request *__get_oldest_req(struct ceph_mds_client *mdsc) +{ + if (RB_EMPTY_ROOT(&mdsc->request_tree)) + return NULL; + return rb_entry(rb_first(&mdsc->request_tree), + struct ceph_mds_request, r_node); +} + +static inline u64 __get_oldest_tid(struct ceph_mds_client *mdsc) +{ + return mdsc->oldest_tid; +} + +#if IS_ENABLED(CONFIG_FS_ENCRYPTION) +static u8 *get_fscrypt_altname(const struct ceph_mds_request *req, u32 *plen) +{ + struct inode *dir = req->r_parent; + struct dentry *dentry = req->r_dentry; + const struct qstr *name = req->r_dname; + u8 *cryptbuf = NULL; + u32 len = 0; + int ret = 0; + + /* only encode if we have parent and dentry */ + if (!dir || !dentry) + goto success; + + /* No-op unless this is encrypted */ + if (!IS_ENCRYPTED(dir)) + goto success; + + ret = ceph_fscrypt_prepare_readdir(dir); + if (ret < 0) + return ERR_PTR(ret); + + /* No key? Just ignore it. */ + if (!fscrypt_has_encryption_key(dir)) + goto success; + + if (!name) + name = &dentry->d_name; + + if (!fscrypt_fname_encrypted_size(dir, name->len, NAME_MAX, &len)) { + WARN_ON_ONCE(1); + return ERR_PTR(-ENAMETOOLONG); + } + + /* No need to append altname if name is short enough */ + if (len <= CEPH_NOHASH_NAME_MAX) { + len = 0; + goto success; + } + + cryptbuf = kmalloc(len, GFP_KERNEL); + if (!cryptbuf) + return ERR_PTR(-ENOMEM); + + ret = fscrypt_fname_encrypt(dir, name, cryptbuf, len); + if (ret) { + kfree(cryptbuf); + return ERR_PTR(ret); + } +success: + *plen = len; + return cryptbuf; +} +#else +static u8 *get_fscrypt_altname(const struct ceph_mds_request *req, u32 *plen) +{ + *plen = 0; + return NULL; +} +#endif + +/** + * ceph_mdsc_build_path - build a path string to a given dentry + * @mdsc: mds client + * @dentry: dentry to which path should be built + * @path_info: output path, length, base ino+snap, and freepath ownership flag + * @for_wire: is this path going to be sent to the MDS? + * + * Build a string that represents the path to the dentry. This is mostly called + * for two different purposes: + * + * 1) we need to build a path string to send to the MDS (for_wire == true) + * 2) we need a path string for local presentation (e.g. debugfs) + * (for_wire == false) + * + * The path is built in reverse, starting with the dentry. Walk back up toward + * the root, building the path until the first non-snapped inode is reached + * (for_wire) or the root inode is reached (!for_wire). + * + * Encode hidden .snap dirs as a double /, i.e. + * foo/.snap/bar -> foo//bar + */ +char *ceph_mdsc_build_path(struct ceph_mds_client *mdsc, struct dentry *dentry, + struct ceph_path_info *path_info, int for_wire) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct dentry *cur; + struct inode *inode; + char *path; + int pos; + unsigned seq; + u64 base; + + if (!dentry) + return ERR_PTR(-EINVAL); + + path = __getname(); + if (!path) + return ERR_PTR(-ENOMEM); +retry: + pos = PATH_MAX - 1; + path[pos] = '\0'; + + seq = read_seqbegin(&rename_lock); + cur = dget(dentry); + for (;;) { + struct dentry *parent; + + spin_lock(&cur->d_lock); + inode = d_inode(cur); + if (inode && ceph_snap(inode) == CEPH_SNAPDIR) { + doutc(cl, "path+%d: %p SNAPDIR\n", pos, cur); + spin_unlock(&cur->d_lock); + parent = dget_parent(cur); + } else if (for_wire && inode && dentry != cur && + ceph_snap(inode) == CEPH_NOSNAP) { + spin_unlock(&cur->d_lock); + pos++; /* get rid of any prepended '/' */ + break; + } else if (!for_wire || !IS_ENCRYPTED(d_inode(cur->d_parent))) { + pos -= cur->d_name.len; + if (pos < 0) { + spin_unlock(&cur->d_lock); + break; + } + memcpy(path + pos, cur->d_name.name, cur->d_name.len); + spin_unlock(&cur->d_lock); + parent = dget_parent(cur); + } else { + int len, ret; + char buf[NAME_MAX]; + + /* + * Proactively copy name into buf, in case we need to + * present it as-is. + */ + memcpy(buf, cur->d_name.name, cur->d_name.len); + len = cur->d_name.len; + spin_unlock(&cur->d_lock); + parent = dget_parent(cur); + + ret = ceph_fscrypt_prepare_readdir(d_inode(parent)); + if (ret < 0) { + dput(parent); + dput(cur); + __putname(path); + return ERR_PTR(ret); + } + + if (fscrypt_has_encryption_key(d_inode(parent))) { + len = ceph_encode_encrypted_dname(d_inode(parent), + buf, len); + if (len < 0) { + dput(parent); + dput(cur); + __putname(path); + return ERR_PTR(len); + } + } + pos -= len; + if (pos < 0) { + dput(parent); + break; + } + memcpy(path + pos, buf, len); + } + dput(cur); + cur = parent; + + /* Are we at the root? */ + if (IS_ROOT(cur)) + break; + + /* Are we out of buffer? */ + if (--pos < 0) + break; + + path[pos] = '/'; + } + inode = d_inode(cur); + base = inode ? ceph_ino(inode) : 0; + dput(cur); + + if (read_seqretry(&rename_lock, seq)) + goto retry; + + if (pos < 0) { + /* + * The path is longer than PATH_MAX and this function + * cannot ever succeed. Creating paths that long is + * possible with Ceph, but Linux cannot use them. + */ + __putname(path); + return ERR_PTR(-ENAMETOOLONG); + } + + /* Initialize the output structure */ + memset(path_info, 0, sizeof(*path_info)); + + path_info->vino.ino = base; + path_info->pathlen = PATH_MAX - 1 - pos; + path_info->path = path + pos; + path_info->freepath = true; + + /* Set snap from dentry if available */ + if (d_inode(dentry)) + path_info->vino.snap = ceph_snap(d_inode(dentry)); + else + path_info->vino.snap = CEPH_NOSNAP; + + doutc(cl, "on %p %d built %llx '%.*s'\n", dentry, d_count(dentry), + base, PATH_MAX - 1 - pos, path + pos); + return path + pos; +} + +static int build_dentry_path(struct ceph_mds_client *mdsc, struct dentry *dentry, + struct inode *dir, struct ceph_path_info *path_info, + bool parent_locked) +{ + char *path; + + rcu_read_lock(); + if (!dir) + dir = d_inode_rcu(dentry->d_parent); + if (dir && parent_locked && ceph_snap(dir) == CEPH_NOSNAP && + !IS_ENCRYPTED(dir)) { + path_info->vino.ino = ceph_ino(dir); + path_info->vino.snap = ceph_snap(dir); + rcu_read_unlock(); + path_info->path = dentry->d_name.name; + path_info->pathlen = dentry->d_name.len; + path_info->freepath = false; + return 0; + } + rcu_read_unlock(); + path = ceph_mdsc_build_path(mdsc, dentry, path_info, 1); + if (IS_ERR(path)) + return PTR_ERR(path); + /* + * ceph_mdsc_build_path already fills path_info, including snap handling. + */ + return 0; +} + +static int build_inode_path(struct inode *inode, struct ceph_path_info *path_info) +{ + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb); + struct dentry *dentry; + char *path; + + if (ceph_snap(inode) == CEPH_NOSNAP) { + path_info->vino.ino = ceph_ino(inode); + path_info->vino.snap = ceph_snap(inode); + path_info->pathlen = 0; + path_info->freepath = false; + return 0; + } + dentry = d_find_alias(inode); + path = ceph_mdsc_build_path(mdsc, dentry, path_info, 1); + dput(dentry); + if (IS_ERR(path)) + return PTR_ERR(path); + /* + * ceph_mdsc_build_path already fills path_info, including snap from dentry. + * Override with inode's snap since that's what this function is for. + */ + path_info->vino.snap = ceph_snap(inode); + return 0; +} + +/* + * request arguments may be specified via an inode *, a dentry *, or + * an explicit ino+path. + */ +static int set_request_path_attr(struct ceph_mds_client *mdsc, struct inode *rinode, + struct dentry *rdentry, struct inode *rdiri, + const char *rpath, u64 rino, + struct ceph_path_info *path_info, + bool parent_locked) +{ + struct ceph_client *cl = mdsc->fsc->client; + int r = 0; + + /* Initialize the output structure */ + memset(path_info, 0, sizeof(*path_info)); + + if (rinode) { + r = build_inode_path(rinode, path_info); + doutc(cl, " inode %p %llx.%llx\n", rinode, ceph_ino(rinode), + ceph_snap(rinode)); + } else if (rdentry) { + r = build_dentry_path(mdsc, rdentry, rdiri, path_info, parent_locked); + doutc(cl, " dentry %p %llx/%.*s\n", rdentry, path_info->vino.ino, + path_info->pathlen, path_info->path); + } else if (rpath || rino) { + path_info->vino.ino = rino; + path_info->vino.snap = CEPH_NOSNAP; + path_info->path = rpath; + path_info->pathlen = rpath ? strlen(rpath) : 0; + path_info->freepath = false; + + doutc(cl, " path %.*s\n", path_info->pathlen, rpath); + } + + return r; +} + +static void encode_mclientrequest_tail(void **p, + const struct ceph_mds_request *req) +{ + struct ceph_timespec ts; + int i; + + ceph_encode_timespec64(&ts, &req->r_stamp); + ceph_encode_copy(p, &ts, sizeof(ts)); + + /* v4: gid_list */ + ceph_encode_32(p, req->r_cred->group_info->ngroups); + for (i = 0; i < req->r_cred->group_info->ngroups; i++) + ceph_encode_64(p, from_kgid(&init_user_ns, + req->r_cred->group_info->gid[i])); + + /* v5: altname */ + ceph_encode_32(p, req->r_altname_len); + ceph_encode_copy(p, req->r_altname, req->r_altname_len); + + /* v6: fscrypt_auth and fscrypt_file */ + if (req->r_fscrypt_auth) { + u32 authlen = ceph_fscrypt_auth_len(req->r_fscrypt_auth); + + ceph_encode_32(p, authlen); + ceph_encode_copy(p, req->r_fscrypt_auth, authlen); + } else { + ceph_encode_32(p, 0); + } + if (test_bit(CEPH_MDS_R_FSCRYPT_FILE, &req->r_req_flags)) { + ceph_encode_32(p, sizeof(__le64)); + ceph_encode_64(p, req->r_fscrypt_file); + } else { + ceph_encode_32(p, 0); + } +} + +static inline u16 mds_supported_head_version(struct ceph_mds_session *session) +{ + if (!test_bit(CEPHFS_FEATURE_32BITS_RETRY_FWD, &session->s_features)) + return 1; + + if (!test_bit(CEPHFS_FEATURE_HAS_OWNER_UIDGID, &session->s_features)) + return 2; + + return CEPH_MDS_REQUEST_HEAD_VERSION; +} + +static struct ceph_mds_request_head_legacy * +find_legacy_request_head(void *p, u64 features) +{ + bool legacy = !(features & CEPH_FEATURE_FS_BTIME); + struct ceph_mds_request_head *head; + + if (legacy) + return (struct ceph_mds_request_head_legacy *)p; + head = (struct ceph_mds_request_head *)p; + return (struct ceph_mds_request_head_legacy *)&head->oldest_client_tid; +} + +/* + * called under mdsc->mutex + */ +static struct ceph_msg *create_request_message(struct ceph_mds_session *session, + struct ceph_mds_request *req, + bool drop_cap_releases) +{ + int mds = session->s_mds; + struct ceph_mds_client *mdsc = session->s_mdsc; + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_msg *msg; + struct ceph_mds_request_head_legacy *lhead; + struct ceph_path_info path_info1 = {0}; + struct ceph_path_info path_info2 = {0}; + struct dentry *old_dentry = NULL; + int len; + u16 releases; + void *p, *end; + int ret; + bool legacy = !(session->s_con.peer_features & CEPH_FEATURE_FS_BTIME); + u16 request_head_version = mds_supported_head_version(session); + kuid_t caller_fsuid = req->r_cred->fsuid; + kgid_t caller_fsgid = req->r_cred->fsgid; + bool parent_locked = test_bit(CEPH_MDS_R_PARENT_LOCKED, &req->r_req_flags); + + ret = set_request_path_attr(mdsc, req->r_inode, req->r_dentry, + req->r_parent, req->r_path1, req->r_ino1.ino, + &path_info1, parent_locked); + if (ret < 0) { + msg = ERR_PTR(ret); + goto out; + } + + /* + * When the parent directory's i_rwsem is *not* locked, req->r_parent may + * have become stale (e.g. after a concurrent rename) between the time the + * dentry was looked up and now. If we detect that the stored r_parent + * does not match the inode number we just encoded for the request, switch + * to the correct inode so that the MDS receives a valid parent reference. + */ + if (!parent_locked && req->r_parent && path_info1.vino.ino && + ceph_ino(req->r_parent) != path_info1.vino.ino) { + struct inode *old_parent = req->r_parent; + struct inode *correct_dir = ceph_get_inode(mdsc->fsc->sb, path_info1.vino, NULL); + if (!IS_ERR(correct_dir)) { + WARN_ONCE(1, "ceph: r_parent mismatch (had %llx wanted %llx) - updating\n", + ceph_ino(old_parent), path_info1.vino.ino); + /* + * Transfer CEPH_CAP_PIN from the old parent to the new one. + * The pin was taken earlier in ceph_mdsc_submit_request(). + */ + ceph_put_cap_refs(ceph_inode(old_parent), CEPH_CAP_PIN); + iput(old_parent); + req->r_parent = correct_dir; + ceph_get_cap_refs(ceph_inode(req->r_parent), CEPH_CAP_PIN); + } + } + + /* If r_old_dentry is set, then assume that its parent is locked */ + if (req->r_old_dentry && + !(req->r_old_dentry->d_flags & DCACHE_DISCONNECTED)) + old_dentry = req->r_old_dentry; + ret = set_request_path_attr(mdsc, NULL, old_dentry, + req->r_old_dentry_dir, + req->r_path2, req->r_ino2.ino, + &path_info2, true); + if (ret < 0) { + msg = ERR_PTR(ret); + goto out_free1; + } + + req->r_altname = get_fscrypt_altname(req, &req->r_altname_len); + if (IS_ERR(req->r_altname)) { + msg = ERR_CAST(req->r_altname); + req->r_altname = NULL; + goto out_free2; + } + + /* + * For old cephs without supporting the 32bit retry/fwd feature + * it will copy the raw memories directly when decoding the + * requests. While new cephs will decode the head depending the + * version member, so we need to make sure it will be compatible + * with them both. + */ + if (legacy) + len = sizeof(struct ceph_mds_request_head_legacy); + else if (request_head_version == 1) + len = offsetofend(struct ceph_mds_request_head, args); + else if (request_head_version == 2) + len = offsetofend(struct ceph_mds_request_head, ext_num_fwd); + else + len = sizeof(struct ceph_mds_request_head); + + /* filepaths */ + len += 2 * (1 + sizeof(u32) + sizeof(u64)); + len += path_info1.pathlen + path_info2.pathlen; + + /* cap releases */ + len += sizeof(struct ceph_mds_request_release) * + (!!req->r_inode_drop + !!req->r_dentry_drop + + !!req->r_old_inode_drop + !!req->r_old_dentry_drop); + + if (req->r_dentry_drop) + len += path_info1.pathlen; + if (req->r_old_dentry_drop) + len += path_info2.pathlen; + + /* MClientRequest tail */ + + /* req->r_stamp */ + len += sizeof(struct ceph_timespec); + + /* gid list */ + len += sizeof(u32) + (sizeof(u64) * req->r_cred->group_info->ngroups); + + /* alternate name */ + len += sizeof(u32) + req->r_altname_len; + + /* fscrypt_auth */ + len += sizeof(u32); // fscrypt_auth + if (req->r_fscrypt_auth) + len += ceph_fscrypt_auth_len(req->r_fscrypt_auth); + + /* fscrypt_file */ + len += sizeof(u32); + if (test_bit(CEPH_MDS_R_FSCRYPT_FILE, &req->r_req_flags)) + len += sizeof(__le64); + + msg = ceph_msg_new2(CEPH_MSG_CLIENT_REQUEST, len, 1, GFP_NOFS, false); + if (!msg) { + msg = ERR_PTR(-ENOMEM); + goto out_free2; + } + + msg->hdr.tid = cpu_to_le64(req->r_tid); + + lhead = find_legacy_request_head(msg->front.iov_base, + session->s_con.peer_features); + + if ((req->r_mnt_idmap != &nop_mnt_idmap) && + !test_bit(CEPHFS_FEATURE_HAS_OWNER_UIDGID, &session->s_features)) { + WARN_ON_ONCE(!IS_CEPH_MDS_OP_NEWINODE(req->r_op)); + + if (enable_unsafe_idmap) { + pr_warn_once_client(cl, + "idmapped mount is used and CEPHFS_FEATURE_HAS_OWNER_UIDGID" + " is not supported by MDS. UID/GID-based restrictions may" + " not work properly.\n"); + + caller_fsuid = from_vfsuid(req->r_mnt_idmap, &init_user_ns, + VFSUIDT_INIT(req->r_cred->fsuid)); + caller_fsgid = from_vfsgid(req->r_mnt_idmap, &init_user_ns, + VFSGIDT_INIT(req->r_cred->fsgid)); + } else { + pr_err_ratelimited_client(cl, + "idmapped mount is used and CEPHFS_FEATURE_HAS_OWNER_UIDGID" + " is not supported by MDS. Fail request with -EIO.\n"); + + ret = -EIO; + goto out_err; + } + } + + /* + * The ceph_mds_request_head_legacy didn't contain a version field, and + * one was added when we moved the message version from 3->4. + */ + if (legacy) { + msg->hdr.version = cpu_to_le16(3); + p = msg->front.iov_base + sizeof(*lhead); + } else if (request_head_version == 1) { + struct ceph_mds_request_head *nhead = msg->front.iov_base; + + msg->hdr.version = cpu_to_le16(4); + nhead->version = cpu_to_le16(1); + p = msg->front.iov_base + offsetofend(struct ceph_mds_request_head, args); + } else if (request_head_version == 2) { + struct ceph_mds_request_head *nhead = msg->front.iov_base; + + msg->hdr.version = cpu_to_le16(6); + nhead->version = cpu_to_le16(2); + + p = msg->front.iov_base + offsetofend(struct ceph_mds_request_head, ext_num_fwd); + } else { + struct ceph_mds_request_head *nhead = msg->front.iov_base; + kuid_t owner_fsuid; + kgid_t owner_fsgid; + + msg->hdr.version = cpu_to_le16(6); + nhead->version = cpu_to_le16(CEPH_MDS_REQUEST_HEAD_VERSION); + nhead->struct_len = cpu_to_le32(sizeof(struct ceph_mds_request_head)); + + if (IS_CEPH_MDS_OP_NEWINODE(req->r_op)) { + owner_fsuid = from_vfsuid(req->r_mnt_idmap, &init_user_ns, + VFSUIDT_INIT(req->r_cred->fsuid)); + owner_fsgid = from_vfsgid(req->r_mnt_idmap, &init_user_ns, + VFSGIDT_INIT(req->r_cred->fsgid)); + nhead->owner_uid = cpu_to_le32(from_kuid(&init_user_ns, owner_fsuid)); + nhead->owner_gid = cpu_to_le32(from_kgid(&init_user_ns, owner_fsgid)); + } else { + nhead->owner_uid = cpu_to_le32(-1); + nhead->owner_gid = cpu_to_le32(-1); + } + + p = msg->front.iov_base + sizeof(*nhead); + } + + end = msg->front.iov_base + msg->front.iov_len; + + lhead->mdsmap_epoch = cpu_to_le32(mdsc->mdsmap->m_epoch); + lhead->op = cpu_to_le32(req->r_op); + lhead->caller_uid = cpu_to_le32(from_kuid(&init_user_ns, + caller_fsuid)); + lhead->caller_gid = cpu_to_le32(from_kgid(&init_user_ns, + caller_fsgid)); + lhead->ino = cpu_to_le64(req->r_deleg_ino); + lhead->args = req->r_args; + + ceph_encode_filepath(&p, end, path_info1.vino.ino, path_info1.path); + ceph_encode_filepath(&p, end, path_info2.vino.ino, path_info2.path); + + /* make note of release offset, in case we need to replay */ + req->r_request_release_offset = p - msg->front.iov_base; + + /* cap releases */ + releases = 0; + if (req->r_inode_drop) + releases += ceph_encode_inode_release(&p, + req->r_inode ? req->r_inode : d_inode(req->r_dentry), + mds, req->r_inode_drop, req->r_inode_unless, + req->r_op == CEPH_MDS_OP_READDIR); + if (req->r_dentry_drop) { + ret = ceph_encode_dentry_release(&p, req->r_dentry, + req->r_parent, mds, req->r_dentry_drop, + req->r_dentry_unless); + if (ret < 0) + goto out_err; + releases += ret; + } + if (req->r_old_dentry_drop) { + ret = ceph_encode_dentry_release(&p, req->r_old_dentry, + req->r_old_dentry_dir, mds, + req->r_old_dentry_drop, + req->r_old_dentry_unless); + if (ret < 0) + goto out_err; + releases += ret; + } + if (req->r_old_inode_drop) + releases += ceph_encode_inode_release(&p, + d_inode(req->r_old_dentry), + mds, req->r_old_inode_drop, req->r_old_inode_unless, 0); + + if (drop_cap_releases) { + releases = 0; + p = msg->front.iov_base + req->r_request_release_offset; + } + + lhead->num_releases = cpu_to_le16(releases); + + encode_mclientrequest_tail(&p, req); + + if (WARN_ON_ONCE(p > end)) { + ceph_msg_put(msg); + msg = ERR_PTR(-ERANGE); + goto out_free2; + } + + msg->front.iov_len = p - msg->front.iov_base; + msg->hdr.front_len = cpu_to_le32(msg->front.iov_len); + + if (req->r_pagelist) { + struct ceph_pagelist *pagelist = req->r_pagelist; + ceph_msg_data_add_pagelist(msg, pagelist); + msg->hdr.data_len = cpu_to_le32(pagelist->length); + } else { + msg->hdr.data_len = 0; + } + + msg->hdr.data_off = cpu_to_le16(0); + +out_free2: + ceph_mdsc_free_path_info(&path_info2); +out_free1: + ceph_mdsc_free_path_info(&path_info1); +out: + return msg; +out_err: + ceph_msg_put(msg); + msg = ERR_PTR(ret); + goto out_free2; +} + +/* + * called under mdsc->mutex if error, under no mutex if + * success. + */ +static void complete_request(struct ceph_mds_client *mdsc, + struct ceph_mds_request *req) +{ + req->r_end_latency = ktime_get(); + + trace_ceph_mdsc_complete_request(mdsc, req); + + if (req->r_callback) + req->r_callback(mdsc, req); + complete_all(&req->r_completion); +} + +/* + * called under mdsc->mutex + */ +static int __prepare_send_request(struct ceph_mds_session *session, + struct ceph_mds_request *req, + bool drop_cap_releases) +{ + int mds = session->s_mds; + struct ceph_mds_client *mdsc = session->s_mdsc; + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_mds_request_head_legacy *lhead; + struct ceph_mds_request_head *nhead; + struct ceph_msg *msg; + int flags = 0, old_max_retry; + bool old_version = !test_bit(CEPHFS_FEATURE_32BITS_RETRY_FWD, + &session->s_features); + + /* + * Avoid infinite retrying after overflow. The client will + * increase the retry count and if the MDS is old version, + * so we limit to retry at most 256 times. + */ + if (req->r_attempts) { + old_max_retry = sizeof_field(struct ceph_mds_request_head, + num_retry); + old_max_retry = 1 << (old_max_retry * BITS_PER_BYTE); + if ((old_version && req->r_attempts >= old_max_retry) || + ((uint32_t)req->r_attempts >= U32_MAX)) { + pr_warn_ratelimited_client(cl, "request tid %llu seq overflow\n", + req->r_tid); + return -EMULTIHOP; + } + } + + req->r_attempts++; + if (req->r_inode) { + struct ceph_cap *cap = + ceph_get_cap_for_mds(ceph_inode(req->r_inode), mds); + + if (cap) + req->r_sent_on_mseq = cap->mseq; + else + req->r_sent_on_mseq = -1; + } + doutc(cl, "%p tid %lld %s (attempt %d)\n", req, req->r_tid, + ceph_mds_op_name(req->r_op), req->r_attempts); + + if (test_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags)) { + void *p; + + /* + * Replay. Do not regenerate message (and rebuild + * paths, etc.); just use the original message. + * Rebuilding paths will break for renames because + * d_move mangles the src name. + */ + msg = req->r_request; + lhead = find_legacy_request_head(msg->front.iov_base, + session->s_con.peer_features); + + flags = le32_to_cpu(lhead->flags); + flags |= CEPH_MDS_FLAG_REPLAY; + lhead->flags = cpu_to_le32(flags); + + if (req->r_target_inode) + lhead->ino = cpu_to_le64(ceph_ino(req->r_target_inode)); + + lhead->num_retry = req->r_attempts - 1; + if (!old_version) { + nhead = (struct ceph_mds_request_head*)msg->front.iov_base; + nhead->ext_num_retry = cpu_to_le32(req->r_attempts - 1); + } + + /* remove cap/dentry releases from message */ + lhead->num_releases = 0; + + p = msg->front.iov_base + req->r_request_release_offset; + encode_mclientrequest_tail(&p, req); + + msg->front.iov_len = p - msg->front.iov_base; + msg->hdr.front_len = cpu_to_le32(msg->front.iov_len); + return 0; + } + + if (req->r_request) { + ceph_msg_put(req->r_request); + req->r_request = NULL; + } + msg = create_request_message(session, req, drop_cap_releases); + if (IS_ERR(msg)) { + req->r_err = PTR_ERR(msg); + return PTR_ERR(msg); + } + req->r_request = msg; + + lhead = find_legacy_request_head(msg->front.iov_base, + session->s_con.peer_features); + lhead->oldest_client_tid = cpu_to_le64(__get_oldest_tid(mdsc)); + if (test_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags)) + flags |= CEPH_MDS_FLAG_REPLAY; + if (test_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags)) + flags |= CEPH_MDS_FLAG_ASYNC; + if (req->r_parent) + flags |= CEPH_MDS_FLAG_WANT_DENTRY; + lhead->flags = cpu_to_le32(flags); + lhead->num_fwd = req->r_num_fwd; + lhead->num_retry = req->r_attempts - 1; + if (!old_version) { + nhead = (struct ceph_mds_request_head*)msg->front.iov_base; + nhead->ext_num_fwd = cpu_to_le32(req->r_num_fwd); + nhead->ext_num_retry = cpu_to_le32(req->r_attempts - 1); + } + + doutc(cl, " r_parent = %p\n", req->r_parent); + return 0; +} + +/* + * called under mdsc->mutex + */ +static int __send_request(struct ceph_mds_session *session, + struct ceph_mds_request *req, + bool drop_cap_releases) +{ + int err; + + trace_ceph_mdsc_send_request(session, req); + + err = __prepare_send_request(session, req, drop_cap_releases); + if (!err) { + ceph_msg_get(req->r_request); + ceph_con_send(&session->s_con, req->r_request); + } + + return err; +} + +/* + * send request, or put it on the appropriate wait list. + */ +static void __do_request(struct ceph_mds_client *mdsc, + struct ceph_mds_request *req) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_mds_session *session = NULL; + int mds = -1; + int err = 0; + bool random; + + if (req->r_err || test_bit(CEPH_MDS_R_GOT_RESULT, &req->r_req_flags)) { + if (test_bit(CEPH_MDS_R_ABORTED, &req->r_req_flags)) + __unregister_request(mdsc, req); + return; + } + + if (READ_ONCE(mdsc->fsc->mount_state) == CEPH_MOUNT_FENCE_IO) { + doutc(cl, "metadata corrupted\n"); + err = -EIO; + goto finish; + } + if (req->r_timeout && + time_after_eq(jiffies, req->r_started + req->r_timeout)) { + doutc(cl, "timed out\n"); + err = -ETIMEDOUT; + goto finish; + } + if (READ_ONCE(mdsc->fsc->mount_state) == CEPH_MOUNT_SHUTDOWN) { + doutc(cl, "forced umount\n"); + err = -EIO; + goto finish; + } + if (READ_ONCE(mdsc->fsc->mount_state) == CEPH_MOUNT_MOUNTING) { + if (mdsc->mdsmap_err) { + err = mdsc->mdsmap_err; + doutc(cl, "mdsmap err %d\n", err); + goto finish; + } + if (mdsc->mdsmap->m_epoch == 0) { + doutc(cl, "no mdsmap, waiting for map\n"); + trace_ceph_mdsc_suspend_request(mdsc, session, req, + ceph_mdsc_suspend_reason_no_mdsmap); + list_add(&req->r_wait, &mdsc->waiting_for_map); + return; + } + if (!(mdsc->fsc->mount_options->flags & + CEPH_MOUNT_OPT_MOUNTWAIT) && + !ceph_mdsmap_is_cluster_available(mdsc->mdsmap)) { + err = -EHOSTUNREACH; + goto finish; + } + } + + put_request_session(req); + + mds = __choose_mds(mdsc, req, &random); + if (mds < 0 || + ceph_mdsmap_get_state(mdsc->mdsmap, mds) < CEPH_MDS_STATE_ACTIVE) { + if (test_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags)) { + err = -EJUKEBOX; + goto finish; + } + doutc(cl, "no mds or not active, waiting for map\n"); + trace_ceph_mdsc_suspend_request(mdsc, session, req, + ceph_mdsc_suspend_reason_no_active_mds); + list_add(&req->r_wait, &mdsc->waiting_for_map); + return; + } + + /* get, open session */ + session = __ceph_lookup_mds_session(mdsc, mds); + if (!session) { + session = register_session(mdsc, mds); + if (IS_ERR(session)) { + err = PTR_ERR(session); + goto finish; + } + } + req->r_session = ceph_get_mds_session(session); + + doutc(cl, "mds%d session %p state %s\n", mds, session, + ceph_session_state_name(session->s_state)); + + /* + * The old ceph will crash the MDSs when see unknown OPs + */ + if (req->r_feature_needed > 0 && + !test_bit(req->r_feature_needed, &session->s_features)) { + err = -EOPNOTSUPP; + goto out_session; + } + + if (session->s_state != CEPH_MDS_SESSION_OPEN && + session->s_state != CEPH_MDS_SESSION_HUNG) { + /* + * We cannot queue async requests since the caps and delegated + * inodes are bound to the session. Just return -EJUKEBOX and + * let the caller retry a sync request in that case. + */ + if (test_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags)) { + err = -EJUKEBOX; + goto out_session; + } + + /* + * If the session has been REJECTED, then return a hard error, + * unless it's a CLEANRECOVER mount, in which case we'll queue + * it to the mdsc queue. + */ + if (session->s_state == CEPH_MDS_SESSION_REJECTED) { + if (ceph_test_mount_opt(mdsc->fsc, CLEANRECOVER)) { + trace_ceph_mdsc_suspend_request(mdsc, session, req, + ceph_mdsc_suspend_reason_rejected); + list_add(&req->r_wait, &mdsc->waiting_for_map); + } else + err = -EACCES; + goto out_session; + } + + if (session->s_state == CEPH_MDS_SESSION_NEW || + session->s_state == CEPH_MDS_SESSION_CLOSING) { + err = __open_session(mdsc, session); + if (err) + goto out_session; + /* retry the same mds later */ + if (random) + req->r_resend_mds = mds; + } + trace_ceph_mdsc_suspend_request(mdsc, session, req, + ceph_mdsc_suspend_reason_session); + list_add(&req->r_wait, &session->s_waiting); + goto out_session; + } + + /* send request */ + req->r_resend_mds = -1; /* forget any previous mds hint */ + + if (req->r_request_started == 0) /* note request start time */ + req->r_request_started = jiffies; + + /* + * For async create we will choose the auth MDS of frag in parent + * directory to send the request and usually this works fine, but + * if the migrated the dirtory to another MDS before it could handle + * it the request will be forwarded. + * + * And then the auth cap will be changed. + */ + if (test_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags) && req->r_num_fwd) { + struct ceph_dentry_info *di = ceph_dentry(req->r_dentry); + struct ceph_inode_info *ci; + struct ceph_cap *cap; + + /* + * The request maybe handled very fast and the new inode + * hasn't been linked to the dentry yet. We need to wait + * for the ceph_finish_async_create(), which shouldn't be + * stuck too long or fail in thoery, to finish when forwarding + * the request. + */ + if (!d_inode(req->r_dentry)) { + err = wait_on_bit(&di->flags, CEPH_DENTRY_ASYNC_CREATE_BIT, + TASK_KILLABLE); + if (err) { + mutex_lock(&req->r_fill_mutex); + set_bit(CEPH_MDS_R_ABORTED, &req->r_req_flags); + mutex_unlock(&req->r_fill_mutex); + goto out_session; + } + } + + ci = ceph_inode(d_inode(req->r_dentry)); + + spin_lock(&ci->i_ceph_lock); + cap = ci->i_auth_cap; + if (test_bit(CEPH_I_ASYNC_CREATE_BIT, &ci->i_ceph_flags) && + mds != cap->mds) { + doutc(cl, "session changed for auth cap %d -> %d\n", + cap->session->s_mds, session->s_mds); + + /* Remove the auth cap from old session */ + spin_lock(&cap->session->s_cap_lock); + cap->session->s_nr_caps--; + list_del_init(&cap->session_caps); + spin_unlock(&cap->session->s_cap_lock); + + /* Add the auth cap to the new session */ + cap->mds = mds; + cap->session = session; + spin_lock(&session->s_cap_lock); + session->s_nr_caps++; + list_add_tail(&cap->session_caps, &session->s_caps); + spin_unlock(&session->s_cap_lock); + + change_auth_cap_ses(ci, session); + } + spin_unlock(&ci->i_ceph_lock); + } + + err = __send_request(session, req, false); + +out_session: + ceph_put_mds_session(session); +finish: + if (err) { + doutc(cl, "early error %d\n", err); + req->r_err = err; + complete_request(mdsc, req); + __unregister_request(mdsc, req); + } + return; +} + +/* + * called under mdsc->mutex + */ +static void __wake_requests(struct ceph_mds_client *mdsc, + struct list_head *head) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_mds_request *req; + LIST_HEAD(tmp_list); + + list_splice_init(head, &tmp_list); + + while (!list_empty(&tmp_list)) { + req = list_entry(tmp_list.next, + struct ceph_mds_request, r_wait); + list_del_init(&req->r_wait); + doutc(cl, " wake request %p tid %llu\n", req, + req->r_tid); + trace_ceph_mdsc_resume_request(mdsc, req); + __do_request(mdsc, req); + } +} + +/* + * Wake up threads with requests pending for @mds, so that they can + * resubmit their requests to a possibly different mds. + */ +static void kick_requests(struct ceph_mds_client *mdsc, int mds) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_mds_request *req; + struct rb_node *p = rb_first(&mdsc->request_tree); + + doutc(cl, "kick_requests mds%d\n", mds); + while (p) { + req = rb_entry(p, struct ceph_mds_request, r_node); + p = rb_next(p); + if (test_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags)) + continue; + if (req->r_attempts > 0) + continue; /* only new requests */ + if (req->r_session && + req->r_session->s_mds == mds) { + doutc(cl, " kicking tid %llu\n", req->r_tid); + list_del_init(&req->r_wait); + trace_ceph_mdsc_resume_request(mdsc, req); + __do_request(mdsc, req); + } + } +} + +int ceph_mdsc_submit_request(struct ceph_mds_client *mdsc, struct inode *dir, + struct ceph_mds_request *req) +{ + struct ceph_client *cl = mdsc->fsc->client; + int err = 0; + + /* + * If a reset is in progress, wait for it to complete. + * + * This is best-effort: a request can pass this check just + * before the phase leaves IDLE and proceed concurrently with + * reset. That is acceptable because (a) such requests will + * either complete normally or fail and be retried by the + * caller, and (b) adding lock serialization here would + * penalize every request for a rare manual operation. + */ + err = ceph_mdsc_wait_for_reset(mdsc); + if (err) { + doutc(cl, "wait_for_reset failed: %d\n", err); + return err; + } + + /* take CAP_PIN refs for r_inode, r_parent, r_old_dentry */ + if (req->r_inode) + ceph_get_cap_refs(ceph_inode(req->r_inode), CEPH_CAP_PIN); + if (req->r_parent) { + struct ceph_inode_info *ci = ceph_inode(req->r_parent); + int fmode = (req->r_op & CEPH_MDS_OP_WRITE) ? + CEPH_FILE_MODE_WR : CEPH_FILE_MODE_RD; + spin_lock(&ci->i_ceph_lock); + ceph_take_cap_refs(ci, CEPH_CAP_PIN, false); + __ceph_touch_fmode(ci, mdsc, fmode); + spin_unlock(&ci->i_ceph_lock); + } + if (req->r_old_dentry_dir) + ceph_get_cap_refs(ceph_inode(req->r_old_dentry_dir), + CEPH_CAP_PIN); + + if (req->r_inode) { + err = ceph_wait_on_async_create(req->r_inode); + if (err) { + doutc(cl, "wait for async create returned: %d\n", err); + return err; + } + } + + if (!err && req->r_old_inode) { + err = ceph_wait_on_async_create(req->r_old_inode); + if (err) { + doutc(cl, "wait for async create returned: %d\n", err); + return err; + } + } + + doutc(cl, "submit_request on %p for inode %p\n", req, dir); + mutex_lock(&mdsc->mutex); + __register_request(mdsc, req, dir); + trace_ceph_mdsc_submit_request(mdsc, req); + __do_request(mdsc, req); + err = req->r_err; + mutex_unlock(&mdsc->mutex); + return err; +} + +int ceph_mdsc_wait_request(struct ceph_mds_client *mdsc, + struct ceph_mds_request *req, + ceph_mds_request_wait_callback_t wait_func) +{ + struct ceph_client *cl = mdsc->fsc->client; + int err; + + /* wait */ + doutc(cl, "do_request waiting\n"); + if (wait_func) { + err = wait_func(mdsc, req); + } else { + long timeleft = wait_for_completion_killable_timeout( + &req->r_completion, + ceph_timeout_jiffies(req->r_timeout)); + if (timeleft > 0) + err = 0; + else if (!timeleft) + err = -ETIMEDOUT; /* timed out */ + else + err = timeleft; /* killed */ + } + doutc(cl, "do_request waited, got %d\n", err); + mutex_lock(&mdsc->mutex); + + /* only abort if we didn't race with a real reply */ + if (test_bit(CEPH_MDS_R_GOT_RESULT, &req->r_req_flags)) { + err = le32_to_cpu(req->r_reply_info.head->result); + } else if (err < 0) { + doutc(cl, "aborted request %lld with %d\n", req->r_tid, err); + + /* + * ensure we aren't running concurrently with + * ceph_fill_trace or ceph_readdir_prepopulate, which + * rely on locks (dir mutex) held by our caller. + */ + mutex_lock(&req->r_fill_mutex); + req->r_err = err; + set_bit(CEPH_MDS_R_ABORTED, &req->r_req_flags); + mutex_unlock(&req->r_fill_mutex); + + if (req->r_parent && + (req->r_op & CEPH_MDS_OP_WRITE)) + ceph_invalidate_dir_request(req); + } else { + err = req->r_err; + } + + mutex_unlock(&mdsc->mutex); + return err; +} + +/* + * Synchrously perform an mds request. Take care of all of the + * session setup, forwarding, retry details. + */ +int ceph_mdsc_do_request(struct ceph_mds_client *mdsc, + struct inode *dir, + struct ceph_mds_request *req) +{ + struct ceph_client *cl = mdsc->fsc->client; + int err; + + doutc(cl, "do_request on %p\n", req); + + /* issue */ + err = ceph_mdsc_submit_request(mdsc, dir, req); + if (!err) + err = ceph_mdsc_wait_request(mdsc, req, NULL); + doutc(cl, "do_request %p done, result %d\n", req, err); + return err; +} + +/* + * Invalidate dir's completeness, dentry lease state on an aborted MDS + * namespace request. + */ +void ceph_invalidate_dir_request(struct ceph_mds_request *req) +{ + struct inode *dir = req->r_parent; + struct inode *old_dir = req->r_old_dentry_dir; + struct ceph_client *cl = req->r_mdsc->fsc->client; + + doutc(cl, "invalidate_dir_request %p %p (complete, lease(s))\n", + dir, old_dir); + + ceph_dir_clear_complete(dir); + if (old_dir) + ceph_dir_clear_complete(old_dir); + if (req->r_dentry) + ceph_invalidate_dentry_lease(req->r_dentry); + if (req->r_old_dentry) + ceph_invalidate_dentry_lease(req->r_old_dentry); +} + +/* + * Handle mds reply. + * + * We take the session mutex and parse and process the reply immediately. + * This preserves the logical ordering of replies, capabilities, etc., sent + * by the MDS as they are applied to our local cache. + */ +static void handle_reply(struct ceph_mds_session *session, struct ceph_msg *msg) +{ + struct ceph_mds_client *mdsc = session->s_mdsc; + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_mds_request *req; + struct ceph_mds_reply_head *head = msg->front.iov_base; + struct ceph_mds_reply_info_parsed *rinfo; /* parsed reply info */ + struct ceph_snap_realm *realm; + unsigned int nofs_flags; + u64 tid; + int err, result; + int mds = session->s_mds; + bool close_sessions = false; + + if (msg->front.iov_len < sizeof(*head)) { + pr_err_client(cl, "got corrupt (short) reply\n"); + ceph_msg_dump(msg); + return; + } + + /* get request, session */ + tid = le64_to_cpu(msg->hdr.tid); + mutex_lock(&mdsc->mutex); + req = lookup_get_request(mdsc, tid); + if (!req) { + doutc(cl, "on unknown tid %llu\n", tid); + mutex_unlock(&mdsc->mutex); + return; + } + doutc(cl, "handle_reply %p\n", req); + + /* correct session? */ + if (req->r_session != session) { + pr_err_client(cl, "got %llu on session mds%d not mds%d\n", + tid, session->s_mds, + req->r_session ? req->r_session->s_mds : -1); + mutex_unlock(&mdsc->mutex); + goto out; + } + + /* dup? */ + if ((test_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags) && !head->safe) || + (test_bit(CEPH_MDS_R_GOT_SAFE, &req->r_req_flags) && head->safe)) { + pr_warn_client(cl, "got a dup %s reply on %llu from mds%d\n", + head->safe ? "safe" : "unsafe", tid, mds); + mutex_unlock(&mdsc->mutex); + goto out; + } + if (test_bit(CEPH_MDS_R_GOT_SAFE, &req->r_req_flags)) { + pr_warn_client(cl, "got unsafe after safe on %llu from mds%d\n", + tid, mds); + mutex_unlock(&mdsc->mutex); + goto out; + } + + result = le32_to_cpu(head->result); + + if (head->safe) { + set_bit(CEPH_MDS_R_GOT_SAFE, &req->r_req_flags); + __unregister_request(mdsc, req); + + /* last request during umount? */ + if (mdsc->stopping && !__get_oldest_req(mdsc)) + complete_all(&mdsc->safe_umount_waiters); + + if (test_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags)) { + /* + * We already handled the unsafe response, now do the + * cleanup. No need to examine the response; the MDS + * doesn't include any result info in the safe + * response. And even if it did, there is nothing + * useful we could do with a revised return value. + */ + doutc(cl, "got safe reply %llu, mds%d\n", tid, mds); + + mutex_unlock(&mdsc->mutex); + goto out; + } + } else { + set_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags); + list_add_tail(&req->r_unsafe_item, &req->r_session->s_unsafe); + } + + doutc(cl, "tid %lld result %d\n", tid, result); + if (test_bit(CEPHFS_FEATURE_REPLY_ENCODING, &session->s_features)) + err = parse_reply_info(session, msg, req, (u64)-1); + else + err = parse_reply_info(session, msg, req, + session->s_con.peer_features); + mutex_unlock(&mdsc->mutex); + + /* Must find target inode outside of mutexes to avoid deadlocks */ + rinfo = &req->r_reply_info; + if ((err >= 0) && rinfo->head->is_target) { + struct inode *in = xchg(&req->r_new_inode, NULL); + struct ceph_vino tvino = { + .ino = le64_to_cpu(rinfo->targeti.in->ino), + .snap = le64_to_cpu(rinfo->targeti.in->snapid) + }; + + /* + * If we ended up opening an existing inode, discard + * r_new_inode + */ + if (req->r_op == CEPH_MDS_OP_CREATE && + !req->r_reply_info.has_create_ino) { + /* This should never happen on an async create */ + WARN_ON_ONCE(req->r_deleg_ino); + iput(in); + in = NULL; + } + + in = ceph_get_inode(mdsc->fsc->sb, tvino, in); + if (IS_ERR(in)) { + err = PTR_ERR(in); + mutex_lock(&session->s_mutex); + goto out_err; + } + req->r_target_inode = in; + ceph_inode_set_subvolume(in, rinfo->targeti.subvolume_id); + } + + mutex_lock(&session->s_mutex); + if (err < 0) { + pr_err_client(cl, "got corrupt reply mds%d(tid:%lld)\n", + mds, tid); + ceph_msg_dump(msg); + goto out_err; + } + + /* snap trace */ + realm = NULL; + if (rinfo->snapblob_len) { + down_write(&mdsc->snap_rwsem); + err = ceph_update_snap_trace(mdsc, rinfo->snapblob, + rinfo->snapblob + rinfo->snapblob_len, + le32_to_cpu(head->op) == CEPH_MDS_OP_RMSNAP, + &realm); + if (err) { + up_write(&mdsc->snap_rwsem); + close_sessions = true; + if (err == -EIO) + ceph_msg_dump(msg); + goto out_err; + } + downgrade_write(&mdsc->snap_rwsem); + } else { + down_read(&mdsc->snap_rwsem); + } + + /* insert trace into our cache */ + mutex_lock(&req->r_fill_mutex); + + /* disable fs reclaim while we are using current->journal_info + * for our own purposes, or else shrinkers of other + * filesystems might dereference this pointer as a different + * type + */ + nofs_flags = memalloc_nofs_save(); + + current->journal_info = req; + err = ceph_fill_trace(mdsc->fsc->sb, req); + if (err == 0) { + if (result == 0 && (req->r_op == CEPH_MDS_OP_READDIR || + req->r_op == CEPH_MDS_OP_LSSNAP)) + err = ceph_readdir_prepopulate(req, req->r_session); + } + current->journal_info = NULL; + memalloc_nofs_restore(nofs_flags); + mutex_unlock(&req->r_fill_mutex); + + up_read(&mdsc->snap_rwsem); + if (realm) + ceph_put_snap_realm(mdsc, realm); + + if (err == 0) { + if (req->r_target_inode && + test_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags)) { + struct ceph_inode_info *ci = + ceph_inode(req->r_target_inode); + spin_lock(&ci->i_unsafe_lock); + list_add_tail(&req->r_unsafe_target_item, + &ci->i_unsafe_iops); + spin_unlock(&ci->i_unsafe_lock); + } + + ceph_unreserve_caps(mdsc, &req->r_caps_reservation); + } +out_err: + mutex_lock(&mdsc->mutex); + if (!test_bit(CEPH_MDS_R_ABORTED, &req->r_req_flags)) { + if (err) { + req->r_err = err; + } else { + req->r_reply = ceph_msg_get(msg); + set_bit(CEPH_MDS_R_GOT_RESULT, &req->r_req_flags); + } + } else { + doutc(cl, "reply arrived after request %lld was aborted\n", tid); + } + mutex_unlock(&mdsc->mutex); + + mutex_unlock(&session->s_mutex); + + /* kick calling process */ + complete_request(mdsc, req); + + ceph_update_metadata_metrics(&mdsc->metric, req->r_start_latency, + req->r_end_latency, err); +out: + ceph_mdsc_put_request(req); + + /* Defer closing the sessions after s_mutex lock being released */ + if (close_sessions) + ceph_mdsc_close_sessions(mdsc); + return; +} + + + +/* + * handle mds notification that our request has been forwarded. + */ +static void handle_forward(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session, + struct ceph_msg *msg) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_mds_request *req; + u64 tid = le64_to_cpu(msg->hdr.tid); + u32 next_mds; + u32 fwd_seq; + int err = -EINVAL; + void *p = msg->front.iov_base; + void *end = p + msg->front.iov_len; + bool aborted = false; + + ceph_decode_need(&p, end, 2*sizeof(u32), bad); + next_mds = ceph_decode_32(&p); + fwd_seq = ceph_decode_32(&p); + + mutex_lock(&mdsc->mutex); + req = lookup_get_request(mdsc, tid); + if (!req) { + mutex_unlock(&mdsc->mutex); + doutc(cl, "forward tid %llu to mds%d - req dne\n", tid, next_mds); + return; /* dup reply? */ + } + + if (test_bit(CEPH_MDS_R_ABORTED, &req->r_req_flags)) { + doutc(cl, "forward tid %llu aborted, unregistering\n", tid); + __unregister_request(mdsc, req); + } else if (fwd_seq <= req->r_num_fwd || (uint32_t)fwd_seq >= U32_MAX) { + /* + * Avoid infinite retrying after overflow. + * + * The MDS will increase the fwd count and in client side + * if the num_fwd is less than the one saved in request + * that means the MDS is an old version and overflowed of + * 8 bits. + */ + mutex_lock(&req->r_fill_mutex); + req->r_err = -EMULTIHOP; + set_bit(CEPH_MDS_R_ABORTED, &req->r_req_flags); + mutex_unlock(&req->r_fill_mutex); + aborted = true; + pr_warn_ratelimited_client(cl, "forward tid %llu seq overflow\n", + tid); + } else { + /* resend. forward race not possible; mds would drop */ + doutc(cl, "forward tid %llu to mds%d (we resend)\n", tid, next_mds); + BUG_ON(req->r_err); + BUG_ON(test_bit(CEPH_MDS_R_GOT_RESULT, &req->r_req_flags)); + req->r_attempts = 0; + req->r_num_fwd = fwd_seq; + req->r_resend_mds = next_mds; + put_request_session(req); + __do_request(mdsc, req); + } + mutex_unlock(&mdsc->mutex); + + /* kick calling process */ + if (aborted) + complete_request(mdsc, req); + ceph_mdsc_put_request(req); + return; + +bad: + pr_err_client(cl, "decode error err=%d\n", err); + ceph_msg_dump(msg); +} + +static int __decode_session_metadata(void **p, void *end, + bool *blocklisted) +{ + /* map */ + u32 n; + bool err_str; + ceph_decode_32_safe(p, end, n, bad); + while (n-- > 0) { + u32 len; + ceph_decode_32_safe(p, end, len, bad); + ceph_decode_need(p, end, len, bad); + err_str = !strncmp(*p, "error_string", len); + *p += len; + ceph_decode_32_safe(p, end, len, bad); + ceph_decode_need(p, end, len, bad); + /* + * Match "blocklisted (blacklisted)" from newer MDSes, + * or "blacklisted" from older MDSes. + */ + if (err_str && strnstr(*p, "blacklisted", len)) + *blocklisted = true; + *p += len; + } + return 0; +bad: + return -1; +} + +/* + * handle a mds session control message + */ +static void handle_session(struct ceph_mds_session *session, + struct ceph_msg *msg) +{ + struct ceph_mds_client *mdsc = session->s_mdsc; + struct ceph_client *cl = mdsc->fsc->client; + int mds = session->s_mds; + int msg_version = le16_to_cpu(msg->hdr.version); + void *p = msg->front.iov_base; + void *end = p + msg->front.iov_len; + struct ceph_mds_session_head *h; + struct ceph_mds_cap_auth *cap_auths = NULL; + u32 op, cap_auths_num = 0; + u64 seq, features = 0; + int wake = 0; + bool blocklisted = false; + u32 i; + + + /* decode */ + ceph_decode_need(&p, end, sizeof(*h), bad); + h = p; + p += sizeof(*h); + + op = le32_to_cpu(h->op); + seq = le64_to_cpu(h->seq); + + if (msg_version >= 3) { + u32 len; + /* version >= 2 and < 5, decode metadata, skip otherwise + * as it's handled via flags. + */ + if (msg_version >= 5) + ceph_decode_skip_map(&p, end, string, string, bad); + else if (__decode_session_metadata(&p, end, &blocklisted) < 0) + goto bad; + + /* version >= 3, feature bits */ + ceph_decode_32_safe(&p, end, len, bad); + if (len) { + ceph_decode_64_safe(&p, end, features, bad); + p += len - sizeof(features); + } + } + + if (msg_version >= 5) { + u32 flags, len; + + /* version >= 4 */ + ceph_decode_skip_16(&p, end, bad); /* struct_v, struct_cv */ + ceph_decode_32_safe(&p, end, len, bad); /* len */ + ceph_decode_skip_n(&p, end, len, bad); /* metric_spec */ + + /* version >= 5, flags */ + ceph_decode_32_safe(&p, end, flags, bad); + if (flags & CEPH_SESSION_BLOCKLISTED) { + pr_warn_client(cl, "mds%d session blocklisted\n", + session->s_mds); + blocklisted = true; + } + } + + if (msg_version >= 6) { + ceph_decode_32_safe(&p, end, cap_auths_num, bad); + doutc(cl, "cap_auths_num %d\n", cap_auths_num); + + if (cap_auths_num && op != CEPH_SESSION_OPEN) { + WARN_ON_ONCE(op != CEPH_SESSION_OPEN); + goto skip_cap_auths; + } + + cap_auths = kzalloc_objs(struct ceph_mds_cap_auth, + cap_auths_num); + if (!cap_auths) { + pr_err_client(cl, "No memory for cap_auths\n"); + return; + } + + for (i = 0; i < cap_auths_num; i++) { + u32 _len, j; + + /* struct_v, struct_compat, and struct_len in MDSCapAuth */ + ceph_decode_skip_n(&p, end, 2 + sizeof(u32), bad); + + /* struct_v, struct_compat, and struct_len in MDSCapMatch */ + ceph_decode_skip_n(&p, end, 2 + sizeof(u32), bad); + ceph_decode_64_safe(&p, end, cap_auths[i].match.uid, bad); + ceph_decode_32_safe(&p, end, _len, bad); + if (_len) { + cap_auths[i].match.gids = kcalloc(_len, sizeof(u32), + GFP_KERNEL); + if (!cap_auths[i].match.gids) { + pr_err_client(cl, "No memory for gids\n"); + goto fail; + } + + cap_auths[i].match.num_gids = _len; + for (j = 0; j < _len; j++) + ceph_decode_32_safe(&p, end, + cap_auths[i].match.gids[j], + bad); + } + + ceph_decode_32_safe(&p, end, _len, bad); + if (_len) { + cap_auths[i].match.path = kcalloc(_len + 1, sizeof(char), + GFP_KERNEL); + if (!cap_auths[i].match.path) { + pr_err_client(cl, "No memory for path\n"); + goto fail; + } + ceph_decode_copy(&p, cap_auths[i].match.path, _len); + + /* Remove the tailing '/' */ + while (_len && cap_auths[i].match.path[_len - 1] == '/') { + cap_auths[i].match.path[_len - 1] = '\0'; + _len -= 1; + } + } + + ceph_decode_32_safe(&p, end, _len, bad); + if (_len) { + cap_auths[i].match.fs_name = kcalloc(_len + 1, sizeof(char), + GFP_KERNEL); + if (!cap_auths[i].match.fs_name) { + pr_err_client(cl, "No memory for fs_name\n"); + goto fail; + } + ceph_decode_copy(&p, cap_auths[i].match.fs_name, _len); + } + + ceph_decode_8_safe(&p, end, cap_auths[i].match.root_squash, bad); + ceph_decode_8_safe(&p, end, cap_auths[i].readable, bad); + ceph_decode_8_safe(&p, end, cap_auths[i].writeable, bad); + doutc(cl, "uid %lld, num_gids %u, path %s, fs_name %s, root_squash %d, readable %d, writeable %d\n", + cap_auths[i].match.uid, cap_auths[i].match.num_gids, + cap_auths[i].match.path, cap_auths[i].match.fs_name, + cap_auths[i].match.root_squash, + cap_auths[i].readable, cap_auths[i].writeable); + } + } + +skip_cap_auths: + mutex_lock(&mdsc->mutex); + if (op == CEPH_SESSION_OPEN) { + if (mdsc->s_cap_auths) { + for (i = 0; i < mdsc->s_cap_auths_num; i++) { + kfree(mdsc->s_cap_auths[i].match.gids); + kfree(mdsc->s_cap_auths[i].match.path); + kfree(mdsc->s_cap_auths[i].match.fs_name); + } + kfree(mdsc->s_cap_auths); + } + mdsc->s_cap_auths_num = cap_auths_num; + mdsc->s_cap_auths = cap_auths; + + session->s_features = features; + if (test_bit(CEPHFS_FEATURE_METRIC_COLLECT, + &session->s_features)) + ceph_metric_bind_session(mdsc, session); + } + if (op == CEPH_SESSION_CLOSE) { + ceph_get_mds_session(session); + __unregister_session(mdsc, session); + } + /* FIXME: this ttl calculation is generous */ + session->s_ttl = jiffies + HZ*mdsc->mdsmap->m_session_autoclose; + mutex_unlock(&mdsc->mutex); + + mutex_lock(&session->s_mutex); + + doutc(cl, "mds%d %s %p state %s seq %llu\n", mds, + ceph_session_op_name(op), session, + ceph_session_state_name(session->s_state), seq); + + if (session->s_state == CEPH_MDS_SESSION_HUNG) { + session->s_state = CEPH_MDS_SESSION_OPEN; + pr_info_client(cl, "mds%d came back\n", session->s_mds); + } + + switch (op) { + case CEPH_SESSION_OPEN: + if (session->s_state == CEPH_MDS_SESSION_RECONNECTING) + pr_info_client(cl, "mds%d reconnect success\n", + session->s_mds); + + if (test_bit(CEPHFS_FEATURE_SUBVOLUME_METRICS, + &session->s_features)) + ceph_subvolume_metrics_enable(&mdsc->subvol_metrics, true); + else + ceph_subvolume_metrics_enable(&mdsc->subvol_metrics, false); + if (session->s_state == CEPH_MDS_SESSION_OPEN) { + pr_notice_client(cl, "mds%d is already opened\n", + session->s_mds); + } else { + session->s_state = CEPH_MDS_SESSION_OPEN; + renewed_caps(mdsc, session, 0); + if (test_bit(CEPHFS_FEATURE_METRIC_COLLECT, + &session->s_features)) + metric_schedule_delayed(&mdsc->metric); + } + + /* + * The connection maybe broken and the session in client + * side has been reinitialized, need to update the seq + * anyway. + */ + if (!session->s_seq && seq) + session->s_seq = seq; + + wake = 1; + if (mdsc->stopping) + __close_session(mdsc, session); + break; + + case CEPH_SESSION_RENEWCAPS: + if (session->s_renew_seq == seq) + renewed_caps(mdsc, session, 1); + break; + + case CEPH_SESSION_CLOSE: + if (session->s_state == CEPH_MDS_SESSION_RECONNECTING) + pr_info_client(cl, "mds%d reconnect denied\n", + session->s_mds); + session->s_state = CEPH_MDS_SESSION_CLOSED; + cleanup_session_requests(mdsc, session); + remove_session_caps(session); + wake = 2; /* for good measure */ + wake_up_all(&mdsc->session_close_wq); + break; + + case CEPH_SESSION_STALE: + pr_info_client(cl, "mds%d caps went stale, renewing\n", + session->s_mds); + atomic_inc(&session->s_cap_gen); + session->s_cap_ttl = jiffies - 1; + send_renew_caps(mdsc, session); + break; + + case CEPH_SESSION_RECALL_STATE: + ceph_trim_caps(mdsc, session, le32_to_cpu(h->max_caps)); + break; + + case CEPH_SESSION_FLUSHMSG: + /* flush cap releases */ + spin_lock(&session->s_cap_lock); + if (session->s_num_cap_releases) + ceph_flush_session_cap_releases(mdsc, session); + spin_unlock(&session->s_cap_lock); + + send_flushmsg_ack(mdsc, session, seq); + break; + + case CEPH_SESSION_FORCE_RO: + doutc(cl, "force_session_readonly %p\n", session); + spin_lock(&session->s_cap_lock); + session->s_readonly = true; + spin_unlock(&session->s_cap_lock); + wake_up_session_caps(session, FORCE_RO); + break; + + case CEPH_SESSION_REJECT: + WARN_ON(session->s_state != CEPH_MDS_SESSION_OPENING && + session->s_state != CEPH_MDS_SESSION_RECONNECTING); + if (session->s_state == CEPH_MDS_SESSION_RECONNECTING) + pr_info_client(cl, "mds%d reconnect rejected\n", + session->s_mds); + else + pr_info_client(cl, "mds%d rejected session\n", + session->s_mds); + session->s_state = CEPH_MDS_SESSION_REJECTED; + cleanup_session_requests(mdsc, session); + remove_session_caps(session); + if (blocklisted) + mdsc->fsc->blocklisted = true; + wake = 2; /* for good measure */ + break; + + default: + pr_err_client(cl, "bad op %d mds%d\n", op, mds); + WARN_ON(1); + } + + mutex_unlock(&session->s_mutex); + if (wake) { + mutex_lock(&mdsc->mutex); + __wake_requests(mdsc, &session->s_waiting); + if (wake == 2) + kick_requests(mdsc, mds); + mutex_unlock(&mdsc->mutex); + } + if (op == CEPH_SESSION_CLOSE) + ceph_put_mds_session(session); + return; + +bad: + pr_err_client(cl, "corrupt message mds%d len %d\n", mds, + (int)msg->front.iov_len); + ceph_msg_dump(msg); +fail: + for (i = 0; i < cap_auths_num; i++) { + kfree(cap_auths[i].match.gids); + kfree(cap_auths[i].match.path); + kfree(cap_auths[i].match.fs_name); + } + kfree(cap_auths); + return; +} + +void ceph_mdsc_release_dir_caps(struct ceph_mds_request *req) +{ + struct ceph_client *cl = req->r_mdsc->fsc->client; + int dcaps; + + dcaps = xchg(&req->r_dir_caps, 0); + if (dcaps) { + doutc(cl, "releasing r_dir_caps=%s\n", ceph_cap_string(dcaps)); + ceph_put_cap_refs(ceph_inode(req->r_parent), dcaps); + } +} + +void ceph_mdsc_release_dir_caps_async(struct ceph_mds_request *req) +{ + struct ceph_client *cl = req->r_mdsc->fsc->client; + int dcaps; + + dcaps = xchg(&req->r_dir_caps, 0); + if (dcaps) { + doutc(cl, "releasing r_dir_caps=%s\n", ceph_cap_string(dcaps)); + ceph_put_cap_refs_async(ceph_inode(req->r_parent), dcaps); + } +} + +/* + * called under session->mutex. + */ +static void replay_unsafe_requests(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session) +{ + struct ceph_mds_request *req, *nreq; + struct rb_node *p; + + doutc(mdsc->fsc->client, "mds%d\n", session->s_mds); + + mutex_lock(&mdsc->mutex); + list_for_each_entry_safe(req, nreq, &session->s_unsafe, r_unsafe_item) + __send_request(session, req, true); + + /* + * also re-send old requests when MDS enters reconnect stage. So that MDS + * can process completed request in clientreplay stage. + */ + p = rb_first(&mdsc->request_tree); + while (p) { + req = rb_entry(p, struct ceph_mds_request, r_node); + p = rb_next(p); + if (test_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags)) + continue; + if (req->r_attempts == 0) + continue; /* only old requests */ + if (!req->r_session) + continue; + if (req->r_session->s_mds != session->s_mds) + continue; + + ceph_mdsc_release_dir_caps_async(req); + + __send_request(session, req, true); + } + mutex_unlock(&mdsc->mutex); +} + +static int send_reconnect_partial(struct ceph_reconnect_state *recon_state) +{ + struct ceph_msg *reply; + struct ceph_pagelist *_pagelist; + struct page *page; + __le32 *addr; + int err = -ENOMEM; + + if (!recon_state->allow_multi) + return -ENOSPC; + + /* can't handle message that contains both caps and realm */ + BUG_ON(!recon_state->nr_caps == !recon_state->nr_realms); + + /* pre-allocate new pagelist */ + _pagelist = ceph_pagelist_alloc(GFP_NOFS); + if (!_pagelist) + return -ENOMEM; + + reply = ceph_msg_new2(CEPH_MSG_CLIENT_RECONNECT, 0, 1, GFP_NOFS, false); + if (!reply) + goto fail_msg; + + /* placeholder for nr_caps */ + err = ceph_pagelist_encode_32(_pagelist, 0); + if (err < 0) + goto fail; + + if (recon_state->nr_caps) { + /* currently encoding caps */ + err = ceph_pagelist_encode_32(recon_state->pagelist, 0); + if (err) + goto fail; + } else { + /* placeholder for nr_realms (currently encoding relams) */ + err = ceph_pagelist_encode_32(_pagelist, 0); + if (err < 0) + goto fail; + } + + err = ceph_pagelist_encode_8(recon_state->pagelist, 1); + if (err) + goto fail; + + page = list_first_entry(&recon_state->pagelist->head, struct page, lru); + addr = kmap_atomic(page); + if (recon_state->nr_caps) { + /* currently encoding caps */ + *addr = cpu_to_le32(recon_state->nr_caps); + } else { + /* currently encoding relams */ + *(addr + 1) = cpu_to_le32(recon_state->nr_realms); + } + kunmap_atomic(addr); + + reply->hdr.version = cpu_to_le16(5); + reply->hdr.compat_version = cpu_to_le16(4); + + reply->hdr.data_len = cpu_to_le32(recon_state->pagelist->length); + ceph_msg_data_add_pagelist(reply, recon_state->pagelist); + + ceph_con_send(&recon_state->session->s_con, reply); + ceph_pagelist_release(recon_state->pagelist); + + recon_state->pagelist = _pagelist; + recon_state->nr_caps = 0; + recon_state->nr_realms = 0; + recon_state->msg_version = 5; + return 0; +fail: + ceph_msg_put(reply); +fail_msg: + ceph_pagelist_release(_pagelist); + return err; +} + +static struct dentry* d_find_primary(struct inode *inode) +{ + struct dentry *alias, *dn = NULL; + + if (hlist_empty(&inode->i_dentry)) + return NULL; + + spin_lock(&inode->i_lock); + if (hlist_empty(&inode->i_dentry)) + goto out_unlock; + + if (S_ISDIR(inode->i_mode)) { + alias = hlist_entry(inode->i_dentry.first, struct dentry, d_alias); + if (!IS_ROOT(alias)) + dn = dget(alias); + goto out_unlock; + } + + for_each_alias(alias, inode) { + spin_lock(&alias->d_lock); + if (!d_unhashed(alias) && + (ceph_dentry(alias)->flags & CEPH_DENTRY_PRIMARY_LINK)) { + dn = dget_dlock(alias); + } + spin_unlock(&alias->d_lock); + if (dn) + break; + } +out_unlock: + spin_unlock(&inode->i_lock); + return dn; +} + +/* + * Encode information about a cap for a reconnect with the MDS. + */ +static int reconnect_caps_cb(struct inode *inode, int mds, void *arg) +{ + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb); + struct ceph_client *cl = ceph_inode_to_client(inode); + union { + struct ceph_mds_cap_reconnect v2; + struct ceph_mds_cap_reconnect_v1 v1; + } rec; + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_reconnect_state *recon_state = arg; + struct ceph_pagelist *pagelist = recon_state->pagelist; + struct dentry *dentry; + struct ceph_cap *cap; + struct ceph_path_info path_info = {0}; + int err; + u64 snap_follows; + + dentry = d_find_primary(inode); + if (dentry) { + /* set pathbase to parent dir when msg_version >= 2 */ + char *path = ceph_mdsc_build_path(mdsc, dentry, &path_info, + recon_state->msg_version >= 2); + dput(dentry); + if (IS_ERR(path)) { + err = PTR_ERR(path); + goto out_err; + } + } + + spin_lock(&ci->i_ceph_lock); + cap = __get_cap_for_mds(ci, mds); + if (!cap) { + spin_unlock(&ci->i_ceph_lock); + err = 0; + goto out_err; + } + doutc(cl, " adding %p ino %llx.%llx cap %p %lld %s\n", inode, + ceph_vinop(inode), cap, cap->cap_id, + ceph_cap_string(cap->issued)); + + cap->seq = 0; /* reset cap seq */ + cap->issue_seq = 0; /* and issue_seq */ + cap->mseq = 0; /* and migrate_seq */ + cap->cap_gen = atomic_read(&cap->session->s_cap_gen); + + /* + * Note: CEPH_I_ERROR_FILELOCK is not set during reconnect. + * Instead, locks are submitted for best-effort MDS reclaim + * via the flock_len field below. If reclaim fails (e.g., + * another client grabbed a conflicting lock), future lock + * operations will fail and set the error flag at that point. + */ + + /* These are lost when the session goes away */ + if (S_ISDIR(inode->i_mode)) { + if (cap->issued & CEPH_CAP_DIR_CREATE) { + ceph_put_string(rcu_dereference_raw(ci->i_cached_layout.pool_ns)); + memset(&ci->i_cached_layout, 0, sizeof(ci->i_cached_layout)); + } + cap->issued &= ~CEPH_CAP_ANY_DIR_OPS; + } + + if (recon_state->msg_version >= 2) { + rec.v2.cap_id = cpu_to_le64(cap->cap_id); + rec.v2.wanted = cpu_to_le32(__ceph_caps_wanted(ci)); + rec.v2.issued = cpu_to_le32(cap->issued); + rec.v2.snaprealm = cpu_to_le64(ci->i_snap_realm->ino); + rec.v2.pathbase = cpu_to_le64(path_info.vino.ino); + rec.v2.flock_len = cpu_to_le32( + test_bit(CEPH_I_ERROR_FILELOCK_BIT, + &ci->i_ceph_flags) ? 0 : 1); + } else { + struct timespec64 ts; + + rec.v1.cap_id = cpu_to_le64(cap->cap_id); + rec.v1.wanted = cpu_to_le32(__ceph_caps_wanted(ci)); + rec.v1.issued = cpu_to_le32(cap->issued); + rec.v1.size = cpu_to_le64(i_size_read(inode)); + ts = inode_get_mtime(inode); + ceph_encode_timespec64(&rec.v1.mtime, &ts); + ts = inode_get_atime(inode); + ceph_encode_timespec64(&rec.v1.atime, &ts); + rec.v1.snaprealm = cpu_to_le64(ci->i_snap_realm->ino); + rec.v1.pathbase = cpu_to_le64(path_info.vino.ino); + } + + if (list_empty(&ci->i_cap_snaps)) { + snap_follows = ci->i_head_snapc ? ci->i_head_snapc->seq : 0; + } else { + struct ceph_cap_snap *capsnap = + list_first_entry(&ci->i_cap_snaps, + struct ceph_cap_snap, ci_item); + snap_follows = capsnap->follows; + } + spin_unlock(&ci->i_ceph_lock); + + if (recon_state->msg_version >= 2) { + int num_fcntl_locks, num_flock_locks; + struct ceph_filelock *flocks = NULL; + size_t struct_len, total_len = sizeof(u64); + u8 struct_v = 0; + +encode_again: + if (rec.v2.flock_len) { + ceph_count_locks(inode, &num_fcntl_locks, &num_flock_locks); + } else { + num_fcntl_locks = 0; + num_flock_locks = 0; + } + if (num_fcntl_locks + num_flock_locks > 0) { + flocks = kmalloc_objs(struct ceph_filelock, + num_fcntl_locks + num_flock_locks, + GFP_NOFS); + if (!flocks) { + err = -ENOMEM; + goto out_err; + } + err = ceph_encode_locks_to_buffer(inode, flocks, + num_fcntl_locks, + num_flock_locks); + if (err) { + kfree(flocks); + flocks = NULL; + if (err == -ENOSPC) + goto encode_again; + goto out_err; + } + } else { + kfree(flocks); + flocks = NULL; + } + + if (recon_state->msg_version >= 3) { + /* version, compat_version and struct_len */ + total_len += 2 * sizeof(u8) + sizeof(u32); + struct_v = 2; + } + /* + * number of encoded locks is stable, so copy to pagelist + */ + struct_len = 2 * sizeof(u32) + + (num_fcntl_locks + num_flock_locks) * + sizeof(struct ceph_filelock); + rec.v2.flock_len = cpu_to_le32(struct_len); + + struct_len += sizeof(u32) + path_info.pathlen + sizeof(rec.v2); + + if (struct_v >= 2) + struct_len += sizeof(u64); /* snap_follows */ + + total_len += struct_len; + + if (pagelist->length + total_len > RECONNECT_MAX_SIZE) { + err = send_reconnect_partial(recon_state); + if (err) + goto out_freeflocks; + pagelist = recon_state->pagelist; + } + + err = ceph_pagelist_reserve(pagelist, total_len); + if (err) + goto out_freeflocks; + + ceph_pagelist_encode_64(pagelist, ceph_ino(inode)); + if (recon_state->msg_version >= 3) { + ceph_pagelist_encode_8(pagelist, struct_v); + ceph_pagelist_encode_8(pagelist, 1); + ceph_pagelist_encode_32(pagelist, struct_len); + } + ceph_pagelist_encode_string(pagelist, (char *)path_info.path, path_info.pathlen); + ceph_pagelist_append(pagelist, &rec, sizeof(rec.v2)); + ceph_locks_to_pagelist(flocks, pagelist, + num_fcntl_locks, num_flock_locks); + if (struct_v >= 2) + ceph_pagelist_encode_64(pagelist, snap_follows); +out_freeflocks: + kfree(flocks); + } else { + err = ceph_pagelist_reserve(pagelist, + sizeof(u64) + sizeof(u32) + + path_info.pathlen + sizeof(rec.v1)); + if (err) + goto out_err; + + ceph_pagelist_encode_64(pagelist, ceph_ino(inode)); + ceph_pagelist_encode_string(pagelist, (char *)path_info.path, path_info.pathlen); + ceph_pagelist_append(pagelist, &rec, sizeof(rec.v1)); + } + +out_err: + ceph_mdsc_free_path_info(&path_info); + if (!err) + recon_state->nr_caps++; + return err; +} + +static int encode_snap_realms(struct ceph_mds_client *mdsc, + struct ceph_reconnect_state *recon_state) +{ + struct rb_node *p; + struct ceph_pagelist *pagelist = recon_state->pagelist; + struct ceph_client *cl = mdsc->fsc->client; + int err = 0; + + if (recon_state->msg_version >= 4) { + err = ceph_pagelist_encode_32(pagelist, mdsc->num_snap_realms); + if (err < 0) + goto fail; + } + + /* + * snaprealms. we provide mds with the ino, seq (version), and + * parent for all of our realms. If the mds has any newer info, + * it will tell us. + */ + for (p = rb_first(&mdsc->snap_realms); p; p = rb_next(p)) { + struct ceph_snap_realm *realm = + rb_entry(p, struct ceph_snap_realm, node); + struct ceph_mds_snaprealm_reconnect sr_rec; + + if (recon_state->msg_version >= 4) { + size_t need = sizeof(u8) * 2 + sizeof(u32) + + sizeof(sr_rec); + + if (pagelist->length + need > RECONNECT_MAX_SIZE) { + err = send_reconnect_partial(recon_state); + if (err) + goto fail; + pagelist = recon_state->pagelist; + } + + err = ceph_pagelist_reserve(pagelist, need); + if (err) + goto fail; + + ceph_pagelist_encode_8(pagelist, 1); + ceph_pagelist_encode_8(pagelist, 1); + ceph_pagelist_encode_32(pagelist, sizeof(sr_rec)); + } + + doutc(cl, " adding snap realm %llx seq %lld parent %llx\n", + realm->ino, realm->seq, realm->parent_ino); + sr_rec.ino = cpu_to_le64(realm->ino); + sr_rec.seq = cpu_to_le64(realm->seq); + sr_rec.parent = cpu_to_le64(realm->parent_ino); + + err = ceph_pagelist_append(pagelist, &sr_rec, sizeof(sr_rec)); + if (err) + goto fail; + + recon_state->nr_realms++; + } +fail: + return err; +} + + +/* + * If an MDS fails and recovers, clients need to reconnect in order to + * reestablish shared state. This includes all caps issued through + * this session _and_ the snap_realm hierarchy. Because it's not + * clear which snap realms the mds cares about, we send everything we + * know about.. that ensures we'll then get any new info the + * recovering MDS might have. + * + * This is a relatively heavyweight operation, but it's rare. + */ +static int send_mds_reconnect(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_msg *reply; + int mds = session->s_mds; + int err = -ENOMEM; + int old_state; + struct ceph_reconnect_state recon_state = { + .session = session, + }; + LIST_HEAD(dispose); + + recon_state.pagelist = ceph_pagelist_alloc(GFP_NOFS); + if (!recon_state.pagelist) + goto fail_nopagelist; + + reply = ceph_msg_new2(CEPH_MSG_CLIENT_RECONNECT, 0, 1, GFP_NOFS, false); + if (!reply) + goto fail_nomsg; + + mutex_lock(&session->s_mutex); + + /* Serialized by s_mutex against concurrent ceph_get_deleg_ino(). */ + xa_destroy(&session->s_delegated_inos); + if (session->s_state == CEPH_MDS_SESSION_CLOSED || + session->s_state == CEPH_MDS_SESSION_REJECTED) { + pr_info_client(cl, "mds%d skipping reconnect, session %s\n", + mds, + ceph_session_state_name(session->s_state)); + mutex_unlock(&session->s_mutex); + ceph_msg_put(reply); + err = -ESTALE; + goto fail_return; + } + + /* s_mutex -> mdsc->mutex matches cleanup_session_requests() order. */ + mutex_lock(&mdsc->mutex); + if (mds >= mdsc->max_sessions || mdsc->sessions[mds] != session) { + mutex_unlock(&mdsc->mutex); + pr_info_client(cl, + "mds%d skipping reconnect, session unregistered\n", + mds); + mutex_unlock(&session->s_mutex); + ceph_msg_put(reply); + err = -ENOENT; + goto fail_return; + } + mutex_unlock(&mdsc->mutex); + + pr_info_client(cl, "mds%d reconnect start\n", mds); + old_state = session->s_state; + session->s_state = CEPH_MDS_SESSION_RECONNECTING; + session->s_seq = 0; + + doutc(cl, "session %p state %s\n", session, + ceph_session_state_name(session->s_state)); + + atomic_inc(&session->s_cap_gen); + + spin_lock(&session->s_cap_lock); + /* don't know if session is readonly */ + session->s_readonly = 0; + /* + * notify __ceph_remove_cap() that we are composing cap reconnect. + * If a cap get released before being added to the cap reconnect, + * __ceph_remove_cap() should skip queuing cap release. + */ + session->s_cap_reconnect = 1; + /* drop old cap expires; we're about to reestablish that state */ + detach_cap_releases(session, &dispose); + spin_unlock(&session->s_cap_lock); + dispose_cap_releases(mdsc, &dispose); + + /* trim unused caps to reduce MDS's cache rejoin time */ + if (mdsc->fsc->sb->s_root) + shrink_dcache_parent(mdsc->fsc->sb->s_root); + + ceph_con_close(&session->s_con); + ceph_con_open(&session->s_con, + CEPH_ENTITY_TYPE_MDS, mds, + ceph_mdsmap_get_addr(mdsc->mdsmap, mds)); + + /* replay unsafe requests */ + replay_unsafe_requests(mdsc, session); + + ceph_early_kick_flushing_caps(mdsc, session); + + down_read(&mdsc->snap_rwsem); + + /* placeholder for nr_caps */ + err = ceph_pagelist_encode_32(recon_state.pagelist, 0); + if (err) + goto fail_clear_cap_reconnect; + + if (test_bit(CEPHFS_FEATURE_MULTI_RECONNECT, &session->s_features)) { + recon_state.msg_version = 3; + recon_state.allow_multi = true; + } else if (session->s_con.peer_features & CEPH_FEATURE_MDSENC) { + recon_state.msg_version = 3; + } else { + recon_state.msg_version = 2; + } + /* traverse this session's caps */ + err = ceph_iterate_session_caps(session, reconnect_caps_cb, &recon_state); + + spin_lock(&session->s_cap_lock); + session->s_cap_reconnect = 0; + spin_unlock(&session->s_cap_lock); + + if (err < 0) + goto fail; + + /* check if all realms can be encoded into current message */ + if (mdsc->num_snap_realms) { + size_t total_len = + recon_state.pagelist->length + + mdsc->num_snap_realms * + sizeof(struct ceph_mds_snaprealm_reconnect); + if (recon_state.msg_version >= 4) { + /* number of realms */ + total_len += sizeof(u32); + /* version, compat_version and struct_len */ + total_len += mdsc->num_snap_realms * + (2 * sizeof(u8) + sizeof(u32)); + } + if (total_len > RECONNECT_MAX_SIZE) { + if (!recon_state.allow_multi) { + err = -ENOSPC; + goto fail; + } + if (recon_state.nr_caps) { + err = send_reconnect_partial(&recon_state); + if (err) + goto fail; + } + recon_state.msg_version = 5; + } + } + + err = encode_snap_realms(mdsc, &recon_state); + if (err < 0) + goto fail; + + if (recon_state.msg_version >= 5) { + err = ceph_pagelist_encode_8(recon_state.pagelist, 0); + if (err < 0) + goto fail; + } + + if (recon_state.nr_caps || recon_state.nr_realms) { + struct page *page = + list_first_entry(&recon_state.pagelist->head, + struct page, lru); + __le32 *addr = kmap_atomic(page); + if (recon_state.nr_caps) { + WARN_ON(recon_state.nr_realms != mdsc->num_snap_realms); + *addr = cpu_to_le32(recon_state.nr_caps); + } else if (recon_state.msg_version >= 4) { + *(addr + 1) = cpu_to_le32(recon_state.nr_realms); + } + kunmap_atomic(addr); + } + + reply->hdr.version = cpu_to_le16(recon_state.msg_version); + if (recon_state.msg_version >= 4) + reply->hdr.compat_version = cpu_to_le16(4); + + reply->hdr.data_len = cpu_to_le32(recon_state.pagelist->length); + ceph_msg_data_add_pagelist(reply, recon_state.pagelist); + + ceph_con_send(&session->s_con, reply); + + mutex_unlock(&session->s_mutex); + + mutex_lock(&mdsc->mutex); + __wake_requests(mdsc, &session->s_waiting); + mutex_unlock(&mdsc->mutex); + + up_read(&mdsc->snap_rwsem); + ceph_pagelist_release(recon_state.pagelist); + return 0; + +fail_clear_cap_reconnect: + spin_lock(&session->s_cap_lock); + session->s_cap_reconnect = 0; + spin_unlock(&session->s_cap_lock); +fail: + ceph_msg_put(reply); + up_read(&mdsc->snap_rwsem); + /* + * Restore prior session state so map-driven reconnect logic + * (check_new_map) can retry. Without this, a transient build + * failure strands the session in RECONNECTING indefinitely. + */ + session->s_state = old_state; + mutex_unlock(&session->s_mutex); +fail_nomsg: + ceph_pagelist_release(recon_state.pagelist); +fail_nopagelist: + pr_err_client(cl, "error %d preparing reconnect for mds%d\n", + err, mds); + return err; + +fail_return: + /* + * Early-exit path for expected concurrent-teardown races + * (-ESTALE for closed/rejected sessions, -ENOENT for + * unregistered sessions). Skip the pr_err_client diagnostic + * since these are not genuine reconnect build failures. + */ + ceph_pagelist_release(recon_state.pagelist); + return err; +} + +const char *ceph_reset_phase_name(enum ceph_client_reset_phase phase) +{ + switch (phase) { + case CEPH_CLIENT_RESET_IDLE: return "idle"; + case CEPH_CLIENT_RESET_QUIESCING: return "quiescing"; + case CEPH_CLIENT_RESET_DRAINING: return "draining"; + case CEPH_CLIENT_RESET_TEARDOWN: return "teardown"; + default: return "unknown"; + } +} + +/** + * ceph_mdsc_wait_for_reset - wait for an active reset to complete + * @mdsc: MDS client + * + * Returns 0 if reset completed successfully or no reset was active. + * Returns -EAGAIN if reset completed with an error, signalling the + * caller to retry. The internal error (e.g. -ENOMEM) is not propagated + * because callers like open() or flock() have no way to act on + * work-function internals. The detailed error is available via debugfs + * reset/status and tracepoints. + * Returns -ETIMEDOUT if we timed out waiting. + * Returns -ERESTARTSYS if interrupted by signal. + */ +int ceph_mdsc_wait_for_reset(struct ceph_mds_client *mdsc) +{ + struct ceph_client_reset_state *st = &mdsc->reset_state; + struct ceph_client *cl = mdsc->fsc->client; + unsigned long deadline = jiffies + CEPH_CLIENT_RESET_WAIT_TIMEOUT_SEC * HZ; + int blocked_count; + long remaining; + long wait_ret; + int ret; + + if (ceph_reset_is_idle(st)) + return 0; + + blocked_count = atomic_inc_return(&st->blocked_requests); + doutc(cl, "request blocked during reset, %d total blocked\n", + blocked_count); + trace_ceph_client_reset_blocked(mdsc, blocked_count); + +retry: + remaining = max_t(long, deadline - jiffies, 1); + wait_ret = wait_event_interruptible_timeout(st->blocked_wq, + ceph_reset_is_idle(st), + remaining); + + if (wait_ret == 0) { + atomic_dec(&st->blocked_requests); + pr_warn_client(cl, "timed out waiting for reset to complete\n"); + trace_ceph_client_reset_unblocked(mdsc, -ETIMEDOUT); + return -ETIMEDOUT; + } + if (wait_ret < 0) { + atomic_dec(&st->blocked_requests); + trace_ceph_client_reset_unblocked(mdsc, (int)wait_ret); + return (int)wait_ret; /* -ERESTARTSYS */ + } + + /* + * Verify phase is still IDLE under the lock. If another reset + * was scheduled between the wake-up and this check, loop back + * and wait for it to finish rather than returning a stale result. + */ + spin_lock(&st->lock); + if (st->phase != CEPH_CLIENT_RESET_IDLE) { + spin_unlock(&st->lock); + if (time_before(jiffies, deadline)) + goto retry; + atomic_dec(&st->blocked_requests); + trace_ceph_client_reset_unblocked(mdsc, -ETIMEDOUT); + return -ETIMEDOUT; + } + ret = st->last_errno; + spin_unlock(&st->lock); + + atomic_dec(&st->blocked_requests); + trace_ceph_client_reset_unblocked(mdsc, ret); + return ret ? -EAGAIN : 0; +} + +static void ceph_mdsc_reset_complete(struct ceph_mds_client *mdsc, int ret) +{ + struct ceph_client_reset_state *st = &mdsc->reset_state; + + spin_lock(&st->lock); + /* + * If destroy already marked us as shut down, it owns the + * final bookkeeping and waiter wakeup. Just bail so we + * don't overwrite its state. + */ + if (st->shutdown) { + spin_unlock(&st->lock); + return; + } + st->last_finish = jiffies; + st->last_errno = ret; + st->phase = CEPH_CLIENT_RESET_IDLE; + if (ret) + st->failure_count++; + else + st->success_count++; + spin_unlock(&st->lock); + + /* Wake up all requests that were blocked waiting for reset */ + wake_up_all(&st->blocked_wq); + + trace_ceph_client_reset_complete(mdsc, ret); +} + +static void ceph_mdsc_reset_workfn(struct work_struct *work) +{ + struct ceph_mds_client *mdsc = + container_of(work, struct ceph_mds_client, reset_work); + struct ceph_client_reset_state *st = &mdsc->reset_state; + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_mds_session **sessions = NULL; + char reason[CEPH_CLIENT_RESET_REASON_LEN]; + unsigned long drain_deadline; + int max_sessions, i, n = 0, torn_down = 0; + int ret = 0; + + spin_lock(&st->lock); + strscpy(reason, st->last_reason, sizeof(reason)); + spin_unlock(&st->lock); + + mutex_lock(&mdsc->mutex); + max_sessions = mdsc->max_sessions; + if (max_sessions <= 0) { + mutex_unlock(&mdsc->mutex); + goto out_complete; + } + + sessions = kcalloc(max_sessions, sizeof(*sessions), GFP_KERNEL); + if (!sessions) { + mutex_unlock(&mdsc->mutex); + ret = -ENOMEM; + pr_err_client(cl, + "manual session reset failed to allocate session array\n"); + ceph_mdsc_reset_complete(mdsc, ret); + return; + } + + for (i = 0; i < max_sessions; i++) { + struct ceph_mds_session *session = mdsc->sessions[i]; + + if (!session) + continue; + + /* + * Read session state without s_mutex to avoid nesting + * mdsc->mutex -> s_mutex, which would invert the + * s_mutex -> mdsc->mutex order used by + * cleanup_session_requests(). s_state is an int + * so loads are atomic; the teardown loop below + * handles races with concurrent state transitions. + */ + switch (READ_ONCE(session->s_state)) { + case CEPH_MDS_SESSION_OPEN: + case CEPH_MDS_SESSION_HUNG: + case CEPH_MDS_SESSION_OPENING: + case CEPH_MDS_SESSION_RESTARTING: + case CEPH_MDS_SESSION_RECONNECTING: + case CEPH_MDS_SESSION_CLOSING: + sessions[n++] = ceph_get_mds_session(session); + break; + default: + pr_info_client(cl, + "mds%d in state %s, skipping reset\n", + session->s_mds, + ceph_session_state_name(session->s_state)); + break; + } + } + mutex_unlock(&mdsc->mutex); + + pr_info_client(cl, + "manual session reset executing (sessions=%d, reason=\"%s\")\n", + n, reason); + + if (n == 0) { + kfree(sessions); + goto out_complete; + } + + spin_lock(&st->lock); + if (st->shutdown) { + spin_unlock(&st->lock); + goto out_sessions; + } + st->phase = CEPH_CLIENT_RESET_DRAINING; + spin_unlock(&st->lock); + + /* + * Best-effort drain: flush dirty state while sessions are still + * alive. New requests are blocked while phase != IDLE. + * The sessions are functional, so non-stuck state drains normally. + * Stuck state (the cause of the stalemate the operator is trying + * to break) will not drain -- that is expected, and we proceed to + * forced teardown after the timeout. + * + * Four things are drained: + * 1. MDS journal -- send_flush_mdlog asks each MDS to journal + * pending unsafe operations (creates, renames, setattrs). + * 2. Unsafe requests -- bounded wait for each unsafe write + * request to reach safe status via r_safe_completion. + * 3. Dirty caps -- ceph_flush_dirty_caps triggers cap flush on + * all sessions. Non-stuck caps flush in milliseconds. + * 4. Cap releases -- push pending cap release messages. + * + * The unsafe-request wait and cap-flush wait below provide + * the bounded drain window during which all categories can + * make progress. + */ + for (i = 0; i < n; i++) + send_flush_mdlog(sessions[i]); + + /* + * Both drain legs (unsafe requests and cap flushes) share a + * single deadline so the total drain time is bounded at + * CEPH_CLIENT_RESET_DRAIN_SEC. + */ + drain_deadline = jiffies + CEPH_CLIENT_RESET_DRAIN_SEC * HZ; + + /* + * Wait for unsafe write requests (creates, renames, setattrs) + * to reach safe status. Uses the same pattern as + * flush_mdlog_and_wait_mdsc_unsafe_requests() but bounded by + * the shared drain deadline. Requests that do not complete within + * the window are force-dropped during teardown. + */ + { + struct ceph_mds_request *req; + struct rb_node *rn; + u64 last_tid; + + mutex_lock(&mdsc->mutex); + last_tid = mdsc->last_tid; + mutex_unlock(&mdsc->mutex); + + mutex_lock(&mdsc->mutex); + rn = rb_first(&mdsc->request_tree); + while (rn) { + req = rb_entry(rn, struct ceph_mds_request, r_node); + if (req->r_tid > last_tid) + break; + if (req->r_op == CEPH_MDS_OP_SETFILELOCK || + !(req->r_op & CEPH_MDS_OP_WRITE)) { + rn = rb_next(rn); + continue; + } + ceph_mdsc_get_request(req); + mutex_unlock(&mdsc->mutex); + + wait_for_completion_timeout(&req->r_safe_completion, + max_t(long, drain_deadline - jiffies, 1)); + + mutex_lock(&mdsc->mutex); + ceph_mdsc_put_request(req); + if (time_after(jiffies, drain_deadline)) + break; + rn = rb_first(&mdsc->request_tree); + } + mutex_unlock(&mdsc->mutex); + + if (time_after_eq(jiffies, drain_deadline)) + WRITE_ONCE(st->drain_timed_out, true); + } + + ceph_flush_dirty_caps(mdsc); + ceph_flush_cap_releases(mdsc); + + spin_lock(&mdsc->cap_dirty_lock); + if (!list_empty(&mdsc->cap_flush_list)) { + struct ceph_cap_flush *cf = + list_last_entry(&mdsc->cap_flush_list, + struct ceph_cap_flush, g_list); + u64 want_flush = mdsc->last_cap_flush_tid; + long drain_ret; + + /* + * Setting wake on the last entry is sufficient: flush + * entries complete in order, so when this entry finishes + * all earlier ones are already done. + */ + cf->wake = true; + spin_unlock(&mdsc->cap_dirty_lock); + pr_info_client(cl, + "draining (want_flush=%llu, %d sessions)\n", + want_flush, n); + drain_ret = wait_event_timeout(mdsc->cap_flushing_wq, + check_caps_flush(mdsc, + want_flush), + max_t(long, + drain_deadline - jiffies, + 1)); + if (drain_ret == 0) { + pr_info_client(cl, + "drain timed out, proceeding with forced teardown\n"); + WRITE_ONCE(st->drain_timed_out, true); + } else { + pr_info_client(cl, "drain completed successfully\n"); + } + } else { + spin_unlock(&mdsc->cap_dirty_lock); + } + + spin_lock(&st->lock); + if (st->shutdown) { + spin_unlock(&st->lock); + goto out_sessions; + } + st->phase = CEPH_CLIENT_RESET_TEARDOWN; + spin_unlock(&st->lock); + + /* + * Ask each MDS to close the session before we tear it down + * locally. Without this the MDS sees only a connection drop and + * waits for the client to reconnect (up to session_autoclose + * seconds) before evicting the session and releasing locks. + * + * Reuse the normal close machinery so the session state/sequence + * snapshot is serialized under s_mutex and a racing s_seq bump + * retransmits REQUEST_CLOSE while the session remains CLOSING. + * We send all close requests first, then yield briefly to let the + * network stack transmit them before __unregister_session() + * closes the connections. + */ + for (i = 0; i < n; i++) { + int err; + + mutex_lock(&sessions[i]->s_mutex); + err = __close_session(mdsc, sessions[i]); + mutex_unlock(&sessions[i]->s_mutex); + if (err < 0) + pr_warn_client(cl, + "mds%d failed to queue close request before reset: %d\n", + sessions[i]->s_mds, err); + } + /* + * Best-effort grace period: yield briefly so the network stack + * can transmit the queued REQUEST_CLOSE messages before we tear + * down connections. Not a correctness requirement -- the MDS + * will still evict via session_autoclose if it never receives + * the close request. + * + * Event-based waiting is not viable here: there is no completion + * event for "message left the NIC," and waiting for the MDS + * SESSION_CLOSE response would re-create the stalemate that the + * reset is meant to break. + */ + if (n > 0) + msleep(CEPH_CLIENT_RESET_CLOSE_GRACE_MS); + + /* + * Tear down each session: close the connection, remove all + * caps, clean up requests, then kick pending requests so they + * re-open a fresh session on the next attempt. + * + * This is modeled on the check_new_map() forced-close path + * for stopped MDS ranks - a proven pattern for hard session + * teardown. We do NOT attempt send_mds_reconnect() because + * the MDS only accepts reconnects during its own RECONNECT + * phase (after MDS restart), not from an active client. + * + * Any state that did not drain (caps that didn't flush, unsafe + * requests that the MDS didn't journal) is force-dropped here. + * This is intentional: that state is stuck and is the reason + * the operator triggered the reset. + */ + for (i = 0; i < n; i++) { + int mds = sessions[i]->s_mds; + + pr_info_client(cl, "mds%d resetting session\n", mds); + + mutex_lock(&mdsc->mutex); + if (mds >= mdsc->max_sessions || + mdsc->sessions[mds] != sessions[i]) { + pr_info_client(cl, + "mds%d session already torn down, skipping\n", + mds); + mutex_unlock(&mdsc->mutex); + ceph_put_mds_session(sessions[i]); + sessions[i] = NULL; + continue; + } + sessions[i]->s_state = CEPH_MDS_SESSION_CLOSED; + __unregister_session(mdsc, sessions[i]); + __wake_requests(mdsc, &sessions[i]->s_waiting); + mutex_unlock(&mdsc->mutex); + + mutex_lock(&sessions[i]->s_mutex); + cleanup_session_requests(mdsc, sessions[i]); + remove_session_caps(sessions[i]); + mutex_unlock(&sessions[i]->s_mutex); + + wake_up_all(&mdsc->session_close_wq); + + ceph_put_mds_session(sessions[i]); + + mutex_lock(&mdsc->mutex); + kick_requests(mdsc, mds); + mutex_unlock(&mdsc->mutex); + + torn_down++; + pr_info_client(cl, "mds%d session reset complete\n", mds); + } + + kfree(sessions); + + spin_lock(&st->lock); + st->sessions_reset = torn_down; + spin_unlock(&st->lock); + +out_complete: + ceph_mdsc_reset_complete(mdsc, ret); + return; + +out_sessions: + /* shutdown == true: ceph_mdsc_destroy() owns the final transition. */ + for (i = 0; i < n; i++) + ceph_put_mds_session(sessions[i]); + kfree(sessions); +} + +int ceph_mdsc_schedule_reset(struct ceph_mds_client *mdsc, + const char *reason) +{ + struct ceph_client_reset_state *st = &mdsc->reset_state; + struct ceph_fs_client *fsc = mdsc->fsc; + const char *msg = (reason && reason[0]) ? reason : "manual"; + int mount_state; + + mount_state = READ_ONCE(fsc->mount_state); + if (mount_state != CEPH_MOUNT_MOUNTED) { + pr_warn_client(fsc->client, + "reset rejected: mount_state=%d (not mounted)\n", + mount_state); + return -EINVAL; + } + + spin_lock(&st->lock); + if (st->phase != CEPH_CLIENT_RESET_IDLE) { + spin_unlock(&st->lock); + return -EBUSY; + } + + st->phase = CEPH_CLIENT_RESET_QUIESCING; + st->last_start = jiffies; + st->last_errno = 0; + st->drain_timed_out = false; + st->sessions_reset = 0; + st->trigger_count++; + strscpy(st->last_reason, msg, sizeof(st->last_reason)); + spin_unlock(&st->lock); + + if (WARN_ON_ONCE(!queue_work(system_unbound_wq, &mdsc->reset_work))) { + spin_lock(&st->lock); + st->phase = CEPH_CLIENT_RESET_IDLE; + st->last_errno = -EALREADY; + st->last_finish = jiffies; + st->failure_count++; + spin_unlock(&st->lock); + wake_up_all(&st->blocked_wq); + return -EALREADY; + } + + pr_info_client(mdsc->fsc->client, + "manual session reset scheduled (reason=\"%s\")\n", + msg); + trace_ceph_client_reset_schedule(mdsc, msg); + return 0; +} + + +/* + * compare old and new mdsmaps, kicking requests + * and closing out old connections as necessary + * + * called under mdsc->mutex. + */ +static void check_new_map(struct ceph_mds_client *mdsc, + struct ceph_mdsmap *newmap, + struct ceph_mdsmap *oldmap) +{ + int i, j, err; + int oldstate, newstate; + struct ceph_mds_session *s; + unsigned long targets[DIV_ROUND_UP(CEPH_MAX_MDS, sizeof(unsigned long))] = {0}; + struct ceph_client *cl = mdsc->fsc->client; + + doutc(cl, "new %u old %u\n", newmap->m_epoch, oldmap->m_epoch); + + if (newmap->m_info) { + for (i = 0; i < newmap->possible_max_rank; i++) { + for (j = 0; j < newmap->m_info[i].num_export_targets; j++) + set_bit(newmap->m_info[i].export_targets[j], targets); + } + } + + for (i = 0; i < oldmap->possible_max_rank && i < mdsc->max_sessions; i++) { + if (!mdsc->sessions[i]) + continue; + s = mdsc->sessions[i]; + oldstate = ceph_mdsmap_get_state(oldmap, i); + newstate = ceph_mdsmap_get_state(newmap, i); + + doutc(cl, "mds%d state %s%s -> %s%s (session %s)\n", + i, ceph_mds_state_name(oldstate), + ceph_mdsmap_is_laggy(oldmap, i) ? " (laggy)" : "", + ceph_mds_state_name(newstate), + ceph_mdsmap_is_laggy(newmap, i) ? " (laggy)" : "", + ceph_session_state_name(s->s_state)); + + if (i >= newmap->possible_max_rank) { + /* force close session for stopped mds */ + ceph_get_mds_session(s); + __unregister_session(mdsc, s); + __wake_requests(mdsc, &s->s_waiting); + mutex_unlock(&mdsc->mutex); + + mutex_lock(&s->s_mutex); + cleanup_session_requests(mdsc, s); + remove_session_caps(s); + mutex_unlock(&s->s_mutex); + + ceph_put_mds_session(s); + + mutex_lock(&mdsc->mutex); + kick_requests(mdsc, i); + continue; + } + + if (memcmp(ceph_mdsmap_get_addr(oldmap, i), + ceph_mdsmap_get_addr(newmap, i), + sizeof(struct ceph_entity_addr))) { + /* just close it */ + mutex_unlock(&mdsc->mutex); + mutex_lock(&s->s_mutex); + mutex_lock(&mdsc->mutex); + ceph_con_close(&s->s_con); + mutex_unlock(&s->s_mutex); + s->s_state = CEPH_MDS_SESSION_RESTARTING; + } else if (oldstate == newstate) { + continue; /* nothing new with this mds */ + } + + /* + * send reconnect? + */ + if (s->s_state == CEPH_MDS_SESSION_RESTARTING && + newstate >= CEPH_MDS_STATE_RECONNECT) { + int rc; + + mutex_unlock(&mdsc->mutex); + clear_bit(i, targets); + rc = send_mds_reconnect(mdsc, s); + if (rc) + pr_warn_client(cl, + "mds%d reconnect failed: %d\n", + i, rc); + mutex_lock(&mdsc->mutex); + } + + /* + * kick request on any mds that has gone active. + */ + if (oldstate < CEPH_MDS_STATE_ACTIVE && + newstate >= CEPH_MDS_STATE_ACTIVE) { + if (oldstate != CEPH_MDS_STATE_CREATING && + oldstate != CEPH_MDS_STATE_STARTING) + pr_info_client(cl, "mds%d recovery completed\n", + s->s_mds); + kick_requests(mdsc, i); + mutex_unlock(&mdsc->mutex); + mutex_lock(&s->s_mutex); + mutex_lock(&mdsc->mutex); + ceph_kick_flushing_caps(mdsc, s); + mutex_unlock(&s->s_mutex); + wake_up_session_caps(s, RECONNECT); + } + } + + /* + * Only open and reconnect sessions that don't exist yet. + */ + for (i = 0; i < newmap->possible_max_rank; i++) { + /* + * In case the import MDS is crashed just after + * the EImportStart journal is flushed, so when + * a standby MDS takes over it and is replaying + * the EImportStart journal the new MDS daemon + * will wait the client to reconnect it, but the + * client may never register/open the session yet. + * + * Will try to reconnect that MDS daemon if the + * rank number is in the export targets array and + * is the up:reconnect state. + */ + newstate = ceph_mdsmap_get_state(newmap, i); + if (!test_bit(i, targets) || newstate != CEPH_MDS_STATE_RECONNECT) + continue; + + /* + * The session maybe registered and opened by some + * requests which were choosing random MDSes during + * the mdsc->mutex's unlock/lock gap below in rare + * case. But the related MDS daemon will just queue + * that requests and be still waiting for the client's + * reconnection request in up:reconnect state. + */ + s = __ceph_lookup_mds_session(mdsc, i); + if (likely(!s)) { + s = __open_export_target_session(mdsc, i); + if (IS_ERR(s)) { + err = PTR_ERR(s); + pr_err_client(cl, + "failed to open export target session, err %d\n", + err); + continue; + } + } + doutc(cl, "send reconnect to export target mds.%d\n", i); + mutex_unlock(&mdsc->mutex); + err = send_mds_reconnect(mdsc, s); + if (err) + pr_warn_client(cl, + "mds%d export target reconnect failed: %d\n", + i, err); + ceph_put_mds_session(s); + mutex_lock(&mdsc->mutex); + } + + for (i = 0; i < newmap->possible_max_rank && i < mdsc->max_sessions; i++) { + s = mdsc->sessions[i]; + if (!s) + continue; + if (!ceph_mdsmap_is_laggy(newmap, i)) + continue; + if (s->s_state == CEPH_MDS_SESSION_OPEN || + s->s_state == CEPH_MDS_SESSION_HUNG || + s->s_state == CEPH_MDS_SESSION_CLOSING) { + doutc(cl, " connecting to export targets of laggy mds%d\n", i); + __open_export_target_sessions(mdsc, s); + } + } +} + + + +/* + * leases + */ + +/* + * caller must hold session s_mutex, dentry->d_lock + */ +void __ceph_mdsc_drop_dentry_lease(struct dentry *dentry) +{ + struct ceph_dentry_info *di = ceph_dentry(dentry); + + ceph_put_mds_session(di->lease_session); + di->lease_session = NULL; +} + +static void handle_lease(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session, + struct ceph_msg *msg) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct super_block *sb = mdsc->fsc->sb; + struct inode *inode; + struct dentry *parent, *dentry; + struct ceph_dentry_info *di; + int mds = session->s_mds; + struct ceph_mds_lease *h = msg->front.iov_base; + u32 seq; + struct ceph_vino vino; + struct qstr dname; + int release = 0; + + doutc(cl, "from mds%d\n", mds); + + if (!ceph_inc_mds_stopping_blocker(mdsc, session)) + return; + + /* decode */ + if (msg->front.iov_len < sizeof(*h) + sizeof(u32)) + goto bad; + vino.ino = le64_to_cpu(h->ino); + vino.snap = CEPH_NOSNAP; + seq = le32_to_cpu(h->seq); + dname.len = get_unaligned_le32(h + 1); + if (msg->front.iov_len < sizeof(*h) + sizeof(u32) + dname.len) + goto bad; + dname.name = (void *)(h + 1) + sizeof(u32); + + /* lookup inode */ + inode = ceph_find_inode(sb, vino); + doutc(cl, "%s, ino %llx %p %.*s\n", ceph_lease_op_name(h->action), + vino.ino, inode, dname.len, dname.name); + + mutex_lock(&session->s_mutex); + if (!inode) { + doutc(cl, "no inode %llx\n", vino.ino); + goto release; + } + + /* dentry */ + parent = d_find_alias(inode); + if (!parent) { + doutc(cl, "no parent dentry on inode %p\n", inode); + WARN_ON(1); + goto release; /* hrm... */ + } + dname.hash = full_name_hash(parent, dname.name, dname.len); + dentry = d_lookup(parent, &dname); + dput(parent); + if (!dentry) + goto release; + + spin_lock(&dentry->d_lock); + di = ceph_dentry(dentry); + switch (h->action) { + case CEPH_MDS_LEASE_REVOKE: + if (di->lease_session == session) { + if (ceph_seq_cmp(di->lease_seq, seq) > 0) + h->seq = cpu_to_le32(di->lease_seq); + __ceph_mdsc_drop_dentry_lease(dentry); + } + release = 1; + break; + + case CEPH_MDS_LEASE_RENEW: + if (di->lease_session == session && + di->lease_gen == atomic_read(&session->s_cap_gen) && + di->lease_renew_from && + di->lease_renew_after == 0) { + unsigned long duration = + msecs_to_jiffies(le32_to_cpu(h->duration_ms)); + + di->lease_seq = seq; + di->time = di->lease_renew_from + duration; + di->lease_renew_after = di->lease_renew_from + + (duration >> 1); + di->lease_renew_from = 0; + } + break; + } + spin_unlock(&dentry->d_lock); + dput(dentry); + + if (!release) + goto out; + +release: + /* let's just reuse the same message */ + h->action = CEPH_MDS_LEASE_REVOKE_ACK; + ceph_msg_get(msg); + ceph_con_send(&session->s_con, msg); + +out: + mutex_unlock(&session->s_mutex); + iput(inode); + + ceph_dec_mds_stopping_blocker(mdsc); + return; + +bad: + ceph_dec_mds_stopping_blocker(mdsc); + + pr_err_client(cl, "corrupt lease message\n"); + ceph_msg_dump(msg); +} + +void ceph_mdsc_lease_send_msg(struct ceph_mds_session *session, + struct dentry *dentry, char action, + u32 seq) +{ + struct ceph_client *cl = session->s_mdsc->fsc->client; + struct ceph_msg *msg; + struct ceph_mds_lease *lease; + struct inode *dir; + int len = sizeof(*lease) + sizeof(u32) + NAME_MAX; + + doutc(cl, "identry %p %s to mds%d\n", dentry, ceph_lease_op_name(action), + session->s_mds); + + msg = ceph_msg_new(CEPH_MSG_CLIENT_LEASE, len, GFP_NOFS, false); + if (!msg) + return; + lease = msg->front.iov_base; + lease->action = action; + lease->seq = cpu_to_le32(seq); + + spin_lock(&dentry->d_lock); + dir = d_inode(dentry->d_parent); + lease->ino = cpu_to_le64(ceph_ino(dir)); + lease->first = lease->last = cpu_to_le64(ceph_snap(dir)); + + put_unaligned_le32(dentry->d_name.len, lease + 1); + memcpy((void *)(lease + 1) + 4, + dentry->d_name.name, dentry->d_name.len); + spin_unlock(&dentry->d_lock); + + ceph_con_send(&session->s_con, msg); +} + +/* + * lock unlock the session, to wait ongoing session activities + */ +static void lock_unlock_session(struct ceph_mds_session *s) +{ + mutex_lock(&s->s_mutex); + mutex_unlock(&s->s_mutex); +} + +static void maybe_recover_session(struct ceph_mds_client *mdsc) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_fs_client *fsc = mdsc->fsc; + + if (!ceph_test_mount_opt(fsc, CLEANRECOVER)) + return; + + if (READ_ONCE(fsc->mount_state) != CEPH_MOUNT_MOUNTED) + return; + + if (!READ_ONCE(fsc->blocklisted)) + return; + + pr_info_client(cl, "auto reconnect after blocklisted\n"); + ceph_force_reconnect(fsc->sb); +} + +bool check_session_state(struct ceph_mds_session *s) +{ + struct ceph_client *cl = s->s_mdsc->fsc->client; + + switch (s->s_state) { + case CEPH_MDS_SESSION_OPEN: + if (s->s_ttl && time_after(jiffies, s->s_ttl)) { + s->s_state = CEPH_MDS_SESSION_HUNG; + pr_info_client(cl, "mds%d hung\n", s->s_mds); + } + break; + case CEPH_MDS_SESSION_CLOSING: + case CEPH_MDS_SESSION_NEW: + case CEPH_MDS_SESSION_RESTARTING: + case CEPH_MDS_SESSION_CLOSED: + case CEPH_MDS_SESSION_REJECTED: + return false; + } + + return true; +} + +/* + * If the sequence is incremented while we're waiting on a REQUEST_CLOSE reply, + * then we need to retransmit that request. + */ +void inc_session_sequence(struct ceph_mds_session *s) +{ + struct ceph_client *cl = s->s_mdsc->fsc->client; + + lockdep_assert_held(&s->s_mutex); + + s->s_seq++; + + if (s->s_state == CEPH_MDS_SESSION_CLOSING) { + int ret; + + doutc(cl, "resending session close request for mds%d\n", s->s_mds); + ret = request_close_session(s); + if (ret < 0) + pr_err_client(cl, "unable to close session to mds%d: %d\n", + s->s_mds, ret); + } +} + +/* + * delayed work -- periodically trim expired leases, renew caps with mds. If + * the @delay parameter is set to 0 or if it's more than 5 secs, the default + * workqueue delay value of 5 secs will be used. + */ +static void schedule_delayed(struct ceph_mds_client *mdsc, unsigned long delay) +{ + unsigned long max_delay = HZ * 5; + + /* 5 secs default delay */ + if (!delay || (delay > max_delay)) + delay = max_delay; + schedule_delayed_work(&mdsc->delayed_work, + round_jiffies_relative(delay)); +} + +static void delayed_work(struct work_struct *work) +{ + struct ceph_mds_client *mdsc = + container_of(work, struct ceph_mds_client, delayed_work.work); + unsigned long delay; + int renew_interval; + int renew_caps; + int i; + + doutc(mdsc->fsc->client, "mdsc delayed_work\n"); + + if (mdsc->stopping >= CEPH_MDSC_STOPPING_FLUSHED) + return; + + mutex_lock(&mdsc->mutex); + renew_interval = mdsc->mdsmap->m_session_timeout >> 2; + renew_caps = time_after_eq(jiffies, HZ*renew_interval + + mdsc->last_renew_caps); + if (renew_caps) + mdsc->last_renew_caps = jiffies; + + for (i = 0; i < mdsc->max_sessions; i++) { + struct ceph_mds_session *s = __ceph_lookup_mds_session(mdsc, i); + if (!s) + continue; + + if (!check_session_state(s)) { + ceph_put_mds_session(s); + continue; + } + mutex_unlock(&mdsc->mutex); + + ceph_flush_session_cap_releases(mdsc, s); + + mutex_lock(&s->s_mutex); + if (renew_caps) + send_renew_caps(mdsc, s); + else + ceph_con_keepalive(&s->s_con); + if (s->s_state == CEPH_MDS_SESSION_OPEN || + s->s_state == CEPH_MDS_SESSION_HUNG) + ceph_send_cap_releases(mdsc, s); + mutex_unlock(&s->s_mutex); + ceph_put_mds_session(s); + + mutex_lock(&mdsc->mutex); + } + mutex_unlock(&mdsc->mutex); + + delay = ceph_check_delayed_caps(mdsc); + + ceph_queue_cap_reclaim_work(mdsc); + + ceph_trim_snapid_map(mdsc); + + maybe_recover_session(mdsc); + + schedule_delayed(mdsc, delay); +} + +int ceph_mdsc_init(struct ceph_fs_client *fsc) + +{ + struct ceph_mds_client *mdsc; + int err; + + mdsc = kzalloc_obj(struct ceph_mds_client, GFP_NOFS); + if (!mdsc) + return -ENOMEM; + mdsc->fsc = fsc; + mutex_init(&mdsc->mutex); + mdsc->mdsmap = kzalloc_obj(*mdsc->mdsmap, GFP_NOFS); + if (!mdsc->mdsmap) { + err = -ENOMEM; + goto err_mdsc; + } + + init_completion(&mdsc->safe_umount_waiters); + spin_lock_init(&mdsc->stopping_lock); + atomic_set(&mdsc->stopping_blockers, 0); + init_completion(&mdsc->stopping_waiter); + atomic64_set(&mdsc->dirty_folios, 0); + init_waitqueue_head(&mdsc->flush_end_wq); + init_waitqueue_head(&mdsc->session_close_wq); + INIT_LIST_HEAD(&mdsc->waiting_for_map); + mdsc->quotarealms_inodes = RB_ROOT; + mutex_init(&mdsc->quotarealms_inodes_mutex); + init_rwsem(&mdsc->snap_rwsem); + mdsc->snap_realms = RB_ROOT; + INIT_LIST_HEAD(&mdsc->snap_empty); + spin_lock_init(&mdsc->snap_empty_lock); + mdsc->request_tree = RB_ROOT; + INIT_DELAYED_WORK(&mdsc->delayed_work, delayed_work); + mdsc->last_renew_caps = jiffies; + INIT_LIST_HEAD(&mdsc->cap_delay_list); +#ifdef CONFIG_DEBUG_FS + INIT_LIST_HEAD(&mdsc->cap_wait_list); +#endif + spin_lock_init(&mdsc->cap_delay_lock); + INIT_LIST_HEAD(&mdsc->cap_unlink_delay_list); + INIT_LIST_HEAD(&mdsc->snap_flush_list); + spin_lock_init(&mdsc->snap_flush_lock); + mdsc->last_cap_flush_tid = 1; + INIT_LIST_HEAD(&mdsc->cap_flush_list); + INIT_LIST_HEAD(&mdsc->cap_dirty_migrating); + spin_lock_init(&mdsc->cap_dirty_lock); + init_waitqueue_head(&mdsc->cap_flushing_wq); + INIT_WORK(&mdsc->cap_reclaim_work, ceph_cap_reclaim_work); + INIT_WORK(&mdsc->cap_unlink_work, ceph_cap_unlink_work); + err = ceph_metric_init(&mdsc->metric); + if (err) + goto err_mdsmap; + ceph_subvolume_metrics_init(&mdsc->subvol_metrics); + mutex_init(&mdsc->subvol_metrics_last_mutex); + mdsc->subvol_metrics_last = NULL; + mdsc->subvol_metrics_last_nr = 0; + mdsc->subvol_metrics_sent = 0; + mdsc->subvol_metrics_nonzero_sends = 0; + + spin_lock_init(&mdsc->dentry_list_lock); + INIT_LIST_HEAD(&mdsc->dentry_leases); + INIT_LIST_HEAD(&mdsc->dentry_dir_leases); + + spin_lock_init(&mdsc->reset_state.lock); + init_waitqueue_head(&mdsc->reset_state.blocked_wq); + atomic_set(&mdsc->reset_state.blocked_requests, 0); + INIT_WORK(&mdsc->reset_work, ceph_mdsc_reset_workfn); + + ceph_caps_init(mdsc); + ceph_adjust_caps_max_min(mdsc, fsc->mount_options); + + spin_lock_init(&mdsc->snapid_map_lock); + mdsc->snapid_map_tree = RB_ROOT; + INIT_LIST_HEAD(&mdsc->snapid_map_lru); + + init_rwsem(&mdsc->pool_perm_rwsem); + mdsc->pool_perm_tree = RB_ROOT; + + strscpy(mdsc->nodename, utsname()->nodename, + sizeof(mdsc->nodename)); + + fsc->mdsc = mdsc; + return 0; + +err_mdsmap: + kfree(mdsc->mdsmap); +err_mdsc: + kfree(mdsc); + return err; +} + +/* + * Wait for safe replies on open mds requests. If we time out, drop + * all requests from the tree to avoid dangling dentry refs. + */ +static void wait_requests(struct ceph_mds_client *mdsc) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_options *opts = mdsc->fsc->client->options; + struct ceph_mds_request *req; + + mutex_lock(&mdsc->mutex); + if (__get_oldest_req(mdsc)) { + mutex_unlock(&mdsc->mutex); + + doutc(cl, "waiting for requests\n"); + wait_for_completion_timeout(&mdsc->safe_umount_waiters, + ceph_timeout_jiffies(opts->mount_timeout)); + + /* tear down remaining requests */ + mutex_lock(&mdsc->mutex); + while ((req = __get_oldest_req(mdsc))) { + doutc(cl, "timed out on tid %llu\n", req->r_tid); + list_del_init(&req->r_wait); + __unregister_request(mdsc, req); + } + } + mutex_unlock(&mdsc->mutex); + doutc(cl, "done\n"); +} + +void send_flush_mdlog(struct ceph_mds_session *s) +{ + struct ceph_client *cl = s->s_mdsc->fsc->client; + struct ceph_msg *msg; + + /* + * Pre-luminous MDS crashes when it sees an unknown session request + */ + if (!CEPH_HAVE_FEATURE(s->s_con.peer_features, SERVER_LUMINOUS)) + return; + + mutex_lock(&s->s_mutex); + doutc(cl, "request mdlog flush to mds%d (%s)s seq %lld\n", + s->s_mds, ceph_session_state_name(s->s_state), s->s_seq); + msg = ceph_create_session_msg(CEPH_SESSION_REQUEST_FLUSH_MDLOG, + s->s_seq); + if (!msg) { + pr_err_client(cl, "failed to request mdlog flush to mds%d (%s) seq %lld\n", + s->s_mds, ceph_session_state_name(s->s_state), s->s_seq); + } else { + ceph_con_send(&s->s_con, msg); + } + mutex_unlock(&s->s_mutex); +} + +static int ceph_mds_auth_match(struct ceph_mds_client *mdsc, + struct ceph_mds_cap_auth *auth, + const struct cred *cred, + char *tpath) +{ + u32 caller_uid = from_kuid(&init_user_ns, cred->fsuid); + u32 caller_gid = from_kgid(&init_user_ns, cred->fsgid); + struct ceph_client *cl = mdsc->fsc->client; + const char *fs_name = mdsc->mdsmap->m_fs_name; + const char *spath = mdsc->fsc->mount_options->server_path; + bool gid_matched = false; + u32 gid, tlen, len; + int i, j; + + doutc(cl, "fsname check fs_name=%s match.fs_name=%s\n", + fs_name, auth->match.fs_name ? auth->match.fs_name : ""); + + if (!ceph_namespace_match(auth->match.fs_name, fs_name)) { + /* fsname mismatch, try next one */ + return 0; + } + + doutc(cl, "match.uid %lld\n", auth->match.uid); + if (auth->match.uid != MDS_AUTH_UID_ANY) { + if (auth->match.uid != caller_uid) + return 0; + if (auth->match.num_gids) { + for (i = 0; i < auth->match.num_gids; i++) { + if (caller_gid == auth->match.gids[i]) + gid_matched = true; + } + if (!gid_matched && cred->group_info->ngroups) { + for (i = 0; i < cred->group_info->ngroups; i++) { + gid = from_kgid(&init_user_ns, + cred->group_info->gid[i]); + for (j = 0; j < auth->match.num_gids; j++) { + if (gid == auth->match.gids[j]) { + gid_matched = true; + break; + } + } + if (gid_matched) + break; + } + } + if (!gid_matched) + return 0; + } + } + + /* path match */ + if (auth->match.path) { + if (!tpath) + return 0; + + tlen = strlen(tpath); + len = strlen(auth->match.path); + if (len) { + char *_tpath = tpath; + bool free_tpath = false; + int m, n; + + doutc(cl, "server path %s, tpath %s, match.path %s\n", + spath, tpath, auth->match.path); + if (spath && (m = strlen(spath)) != 1) { + /* mount path + '/' + tpath + an extra space */ + n = m + 1 + tlen + 1; + _tpath = kmalloc(n, GFP_NOFS); + if (!_tpath) + return -ENOMEM; + /* remove the leading '/' */ + snprintf(_tpath, n, "%s/%s", spath + 1, tpath); + free_tpath = true; + tlen = strlen(_tpath); + } + + /* + * Please note the tailing '/' for match.path has already + * been removed when parsing. + * + * Remove the tailing '/' for the target path. + */ + while (tlen && _tpath[tlen - 1] == '/') { + _tpath[tlen - 1] = '\0'; + tlen -= 1; + } + doutc(cl, "_tpath %s\n", _tpath); + + /* + * In case first == _tpath && tlen == len: + * match.path=/foo --> /foo _path=/foo --> match + * match.path=/foo/ --> /foo _path=/foo --> match + * + * In case first == _tmatch.path && tlen > len: + * match.path=/foo/ --> /foo _path=/foo/ --> match + * match.path=/foo --> /foo _path=/foo/ --> match + * match.path=/foo/ --> /foo _path=/foo/d --> match + * match.path=/foo --> /foo _path=/food --> mismatch + * + * All the other cases --> mismatch + */ + bool path_matched = true; + char *first = strstr(_tpath, auth->match.path); + if (first != _tpath || + (tlen > len && _tpath[len] != '/')) { + path_matched = false; + } + + if (free_tpath) + kfree(_tpath); + + if (!path_matched) + return 0; + } + } + + doutc(cl, "matched\n"); + return 1; +} + +int ceph_mds_check_access(struct ceph_mds_client *mdsc, char *tpath, int mask) +{ + const struct cred *cred = get_current_cred(); + u32 caller_uid = from_kuid(&init_user_ns, cred->fsuid); + u32 caller_gid = from_kgid(&init_user_ns, cred->fsgid); + struct ceph_mds_cap_auth *rw_perms_s = NULL; + struct ceph_client *cl = mdsc->fsc->client; + bool root_squash_perms = true; + int i, err; + + doutc(cl, "tpath '%s', mask %d, caller_uid %d, caller_gid %d\n", + tpath, mask, caller_uid, caller_gid); + + for (i = 0; i < mdsc->s_cap_auths_num; i++) { + struct ceph_mds_cap_auth *s = &mdsc->s_cap_auths[i]; + + err = ceph_mds_auth_match(mdsc, s, cred, tpath); + if (err < 0) { + put_cred(cred); + return err; + } else if (err > 0) { + /* always follow the last auth caps' permission */ + root_squash_perms = true; + rw_perms_s = NULL; + if ((mask & MAY_WRITE) && s->writeable && + s->match.root_squash && (!caller_uid || !caller_gid)) + root_squash_perms = false; + + if (((mask & MAY_WRITE) && !s->writeable) || + ((mask & MAY_READ) && !s->readable)) + rw_perms_s = s; + } + } + + put_cred(cred); + + doutc(cl, "root_squash_perms %d, rw_perms_s %p\n", root_squash_perms, + rw_perms_s); + if (root_squash_perms && rw_perms_s == NULL) { + doutc(cl, "access allowed\n"); + return 0; + } + + if (!root_squash_perms) { + doutc(cl, "root_squash is enabled and user(%d %d) isn't allowed to write", + caller_uid, caller_gid); + } + if (rw_perms_s) { + doutc(cl, "mds auth caps readable/writeable %d/%d while request r/w %d/%d", + rw_perms_s->readable, rw_perms_s->writeable, + !!(mask & MAY_READ), !!(mask & MAY_WRITE)); + } + doutc(cl, "access denied\n"); + return -EACCES; +} + +/* + * called before mount is ro, and before dentries are torn down. + * (hmm, does this still race with new lookups?) + */ +void ceph_mdsc_pre_umount(struct ceph_mds_client *mdsc) +{ + doutc(mdsc->fsc->client, "begin\n"); + mdsc->stopping = CEPH_MDSC_STOPPING_BEGIN; + + ceph_mdsc_iterate_sessions(mdsc, send_flush_mdlog, true); + ceph_mdsc_iterate_sessions(mdsc, lock_unlock_session, false); + ceph_flush_dirty_caps(mdsc); + wait_requests(mdsc); + + /* + * wait for reply handlers to drop their request refs and + * their inode/dcache refs + */ + ceph_msgr_flush(); + + ceph_cleanup_quotarealms_inodes(mdsc); + doutc(mdsc->fsc->client, "done\n"); +} + +/* + * flush the mdlog and wait for all write mds requests to flush. + */ +static void flush_mdlog_and_wait_mdsc_unsafe_requests(struct ceph_mds_client *mdsc, + u64 want_tid) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_mds_request *req = NULL, *nextreq; + struct ceph_mds_session *last_session = NULL; + struct rb_node *n; + + mutex_lock(&mdsc->mutex); + doutc(cl, "want %lld\n", want_tid); +restart: + req = __get_oldest_req(mdsc); + while (req && req->r_tid <= want_tid) { + /* find next request */ + n = rb_next(&req->r_node); + if (n) + nextreq = rb_entry(n, struct ceph_mds_request, r_node); + else + nextreq = NULL; + if (req->r_op != CEPH_MDS_OP_SETFILELOCK && + (req->r_op & CEPH_MDS_OP_WRITE)) { + struct ceph_mds_session *s = req->r_session; + + if (!s) { + req = nextreq; + continue; + } + + /* write op */ + ceph_mdsc_get_request(req); + if (nextreq) + ceph_mdsc_get_request(nextreq); + s = ceph_get_mds_session(s); + mutex_unlock(&mdsc->mutex); + + /* send flush mdlog request to MDS */ + if (last_session != s) { + send_flush_mdlog(s); + ceph_put_mds_session(last_session); + last_session = s; + } else { + ceph_put_mds_session(s); + } + doutc(cl, "wait on %llu (want %llu)\n", + req->r_tid, want_tid); + wait_for_completion(&req->r_safe_completion); + + mutex_lock(&mdsc->mutex); + ceph_mdsc_put_request(req); + if (!nextreq) + break; /* next dne before, so we're done! */ + if (RB_EMPTY_NODE(&nextreq->r_node)) { + /* next request was removed from tree */ + ceph_mdsc_put_request(nextreq); + goto restart; + } + ceph_mdsc_put_request(nextreq); /* won't go away */ + } + req = nextreq; + } + mutex_unlock(&mdsc->mutex); + ceph_put_mds_session(last_session); + doutc(cl, "done\n"); +} + +void ceph_mdsc_sync(struct ceph_mds_client *mdsc) +{ + struct ceph_client *cl = mdsc->fsc->client; + u64 want_tid, want_flush; + + if (READ_ONCE(mdsc->fsc->mount_state) >= CEPH_MOUNT_SHUTDOWN) + return; + + doutc(cl, "sync\n"); + mutex_lock(&mdsc->mutex); + want_tid = mdsc->last_tid; + mutex_unlock(&mdsc->mutex); + + ceph_flush_dirty_caps(mdsc); + ceph_flush_cap_releases(mdsc); + spin_lock(&mdsc->cap_dirty_lock); + want_flush = mdsc->last_cap_flush_tid; + if (!list_empty(&mdsc->cap_flush_list)) { + struct ceph_cap_flush *cf = + list_last_entry(&mdsc->cap_flush_list, + struct ceph_cap_flush, g_list); + cf->wake = true; + } + spin_unlock(&mdsc->cap_dirty_lock); + + doutc(cl, "sync want tid %lld flush_seq %lld\n", want_tid, want_flush); + + flush_mdlog_and_wait_mdsc_unsafe_requests(mdsc, want_tid); + wait_caps_flush(mdsc, want_flush); +} + +/* + * true if all sessions are closed, or we force unmount + */ +static bool done_closing_sessions(struct ceph_mds_client *mdsc, int skipped) +{ + if (READ_ONCE(mdsc->fsc->mount_state) == CEPH_MOUNT_SHUTDOWN) + return true; + return atomic_read(&mdsc->num_sessions) <= skipped; +} + +/* + * called after sb is ro or when metadata corrupted. + */ +void ceph_mdsc_close_sessions(struct ceph_mds_client *mdsc) +{ + struct ceph_options *opts = mdsc->fsc->client->options; + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_mds_session *session; + int i; + int skipped = 0; + + doutc(cl, "begin\n"); + + /* close sessions */ + mutex_lock(&mdsc->mutex); + for (i = 0; i < mdsc->max_sessions; i++) { + session = __ceph_lookup_mds_session(mdsc, i); + if (!session) + continue; + mutex_unlock(&mdsc->mutex); + mutex_lock(&session->s_mutex); + if (__close_session(mdsc, session) <= 0) + skipped++; + mutex_unlock(&session->s_mutex); + ceph_put_mds_session(session); + mutex_lock(&mdsc->mutex); + } + mutex_unlock(&mdsc->mutex); + + doutc(cl, "waiting for sessions to close\n"); + wait_event_timeout(mdsc->session_close_wq, + done_closing_sessions(mdsc, skipped), + ceph_timeout_jiffies(opts->mount_timeout)); + + /* tear down remaining sessions */ + mutex_lock(&mdsc->mutex); + for (i = 0; i < mdsc->max_sessions; i++) { + if (mdsc->sessions[i]) { + session = ceph_get_mds_session(mdsc->sessions[i]); + __unregister_session(mdsc, session); + mutex_unlock(&mdsc->mutex); + mutex_lock(&session->s_mutex); + remove_session_caps(session); + mutex_unlock(&session->s_mutex); + ceph_put_mds_session(session); + mutex_lock(&mdsc->mutex); + } + } + WARN_ON(!list_empty(&mdsc->cap_delay_list)); + mutex_unlock(&mdsc->mutex); + + ceph_cleanup_snapid_map(mdsc); + ceph_cleanup_global_and_empty_realms(mdsc); + + cancel_work_sync(&mdsc->cap_reclaim_work); + cancel_work_sync(&mdsc->cap_unlink_work); + cancel_delayed_work_sync(&mdsc->delayed_work); /* cancel timer */ + + doutc(cl, "done\n"); +} + +void ceph_mdsc_force_umount(struct ceph_mds_client *mdsc) +{ + struct ceph_mds_session *session; + int mds; + + doutc(mdsc->fsc->client, "force umount\n"); + + mutex_lock(&mdsc->mutex); + for (mds = 0; mds < mdsc->max_sessions; mds++) { + session = __ceph_lookup_mds_session(mdsc, mds); + if (!session) + continue; + + if (session->s_state == CEPH_MDS_SESSION_REJECTED) + __unregister_session(mdsc, session); + __wake_requests(mdsc, &session->s_waiting); + mutex_unlock(&mdsc->mutex); + + mutex_lock(&session->s_mutex); + __close_session(mdsc, session); + if (session->s_state == CEPH_MDS_SESSION_CLOSING) { + cleanup_session_requests(mdsc, session); + remove_session_caps(session); + } + mutex_unlock(&session->s_mutex); + ceph_put_mds_session(session); + + mutex_lock(&mdsc->mutex); + kick_requests(mdsc, mds); + } + __wake_requests(mdsc, &mdsc->waiting_for_map); + mutex_unlock(&mdsc->mutex); +} + +static void ceph_mdsc_stop(struct ceph_mds_client *mdsc) +{ + doutc(mdsc->fsc->client, "stop\n"); + /* + * Make sure the delayed work stopped before releasing + * the resources. + * + * Because the cancel_delayed_work_sync() will only + * guarantee that the work finishes executing. But the + * delayed work will re-arm itself again after that. + */ + flush_delayed_work(&mdsc->delayed_work); + + if (mdsc->mdsmap) + ceph_mdsmap_destroy(mdsc->mdsmap); + kfree(mdsc->sessions); + ceph_caps_finalize(mdsc); + + if (mdsc->s_cap_auths) { + int i; + + for (i = 0; i < mdsc->s_cap_auths_num; i++) { + kfree(mdsc->s_cap_auths[i].match.gids); + kfree(mdsc->s_cap_auths[i].match.path); + kfree(mdsc->s_cap_auths[i].match.fs_name); + } + kfree(mdsc->s_cap_auths); + } + + ceph_pool_perm_destroy(mdsc); +} + +void ceph_mdsc_destroy(struct ceph_fs_client *fsc) +{ + struct ceph_mds_client *mdsc = fsc->mdsc; + doutc(fsc->client, "%p\n", mdsc); + + if (!mdsc) + return; + + /* flush out any connection work with references to us */ + ceph_msgr_flush(); + + /* + * Mark reset as failed and wake any blocked waiters before + * cancelling, so unmount doesn't stall on blocked_wq timeout + * if cancel_work_sync() prevents the work from running. + */ + spin_lock(&mdsc->reset_state.lock); + mdsc->reset_state.shutdown = true; + if (mdsc->reset_state.phase != CEPH_CLIENT_RESET_IDLE) { + mdsc->reset_state.phase = CEPH_CLIENT_RESET_IDLE; + mdsc->reset_state.last_errno = -ESHUTDOWN; + mdsc->reset_state.last_finish = jiffies; + mdsc->reset_state.failure_count++; + } + spin_unlock(&mdsc->reset_state.lock); + wake_up_all(&mdsc->reset_state.blocked_wq); + + cancel_work_sync(&mdsc->reset_work); + ceph_mdsc_stop(mdsc); + + ceph_metric_destroy(&mdsc->metric); + ceph_subvolume_metrics_destroy(&mdsc->subvol_metrics); + kfree(mdsc->subvol_metrics_last); + + fsc->mdsc = NULL; + kfree(mdsc); + doutc(fsc->client, "%p done\n", mdsc); +} + +void ceph_mdsc_handle_fsmap(struct ceph_mds_client *mdsc, struct ceph_msg *msg) +{ + struct ceph_fs_client *fsc = mdsc->fsc; + struct ceph_client *cl = fsc->client; + const char *mds_namespace = fsc->mount_options->mds_namespace; + void *p = msg->front.iov_base; + void *end = p + msg->front.iov_len; + u32 epoch; + u32 num_fs; + u32 mount_fscid = (u32)-1; + int err = -EINVAL; + + ceph_decode_need(&p, end, sizeof(u32), bad); + epoch = ceph_decode_32(&p); + + doutc(cl, "epoch %u\n", epoch); + + /* struct_v, struct_cv, map_len, epoch, legacy_client_fscid */ + ceph_decode_skip_n(&p, end, 2 + sizeof(u32) * 3, bad); + + ceph_decode_32_safe(&p, end, num_fs, bad); + while (num_fs-- > 0) { + void *info_p, *info_end; + u32 info_len; + u32 fscid, namelen; + + ceph_decode_need(&p, end, 2 + sizeof(u32), bad); + p += 2; // info_v, info_cv + info_len = ceph_decode_32(&p); + ceph_decode_need(&p, end, info_len, bad); + info_p = p; + info_end = p + info_len; + p = info_end; + + ceph_decode_need(&info_p, info_end, sizeof(u32) * 2, bad); + fscid = ceph_decode_32(&info_p); + namelen = ceph_decode_32(&info_p); + ceph_decode_need(&info_p, info_end, namelen, bad); + + if (mds_namespace && + strlen(mds_namespace) == namelen && + !strncmp(mds_namespace, (char *)info_p, namelen)) { + mount_fscid = fscid; + break; + } + } + + ceph_monc_got_map(&fsc->client->monc, CEPH_SUB_FSMAP, epoch); + if (mount_fscid != (u32)-1) { + fsc->client->monc.fs_cluster_id = mount_fscid; + ceph_monc_want_map(&fsc->client->monc, CEPH_SUB_MDSMAP, + 0, true); + ceph_monc_renew_subs(&fsc->client->monc); + } else { + err = -ENOENT; + goto err_out; + } + return; + +bad: + pr_err_client(cl, "error decoding fsmap %d. Shutting down mount.\n", + err); + ceph_umount_begin(mdsc->fsc->sb); + ceph_msg_dump(msg); +err_out: + mutex_lock(&mdsc->mutex); + mdsc->mdsmap_err = err; + __wake_requests(mdsc, &mdsc->waiting_for_map); + mutex_unlock(&mdsc->mutex); +} + +/* + * handle mds map update. + */ +void ceph_mdsc_handle_mdsmap(struct ceph_mds_client *mdsc, struct ceph_msg *msg) +{ + struct ceph_client *cl = mdsc->fsc->client; + u32 epoch; + u32 maplen; + void *p = msg->front.iov_base; + void *end = p + msg->front.iov_len; + struct ceph_mdsmap *newmap, *oldmap; + struct ceph_fsid fsid; + int err = -EINVAL; + + ceph_decode_need(&p, end, sizeof(fsid)+2*sizeof(u32), bad); + ceph_decode_copy(&p, &fsid, sizeof(fsid)); + if (ceph_check_fsid(mdsc->fsc->client, &fsid) < 0) + return; + epoch = ceph_decode_32(&p); + maplen = ceph_decode_32(&p); + doutc(cl, "epoch %u len %d\n", epoch, (int)maplen); + + /* do we need it? */ + mutex_lock(&mdsc->mutex); + if (mdsc->mdsmap && epoch <= mdsc->mdsmap->m_epoch) { + doutc(cl, "epoch %u <= our %u\n", epoch, mdsc->mdsmap->m_epoch); + mutex_unlock(&mdsc->mutex); + return; + } + + newmap = ceph_mdsmap_decode(mdsc, &p, end, ceph_msgr2(mdsc->fsc->client)); + if (IS_ERR(newmap)) { + err = PTR_ERR(newmap); + goto bad_unlock; + } + + /* swap into place */ + if (mdsc->mdsmap) { + oldmap = mdsc->mdsmap; + mdsc->mdsmap = newmap; + check_new_map(mdsc, newmap, oldmap); + ceph_mdsmap_destroy(oldmap); + } else { + mdsc->mdsmap = newmap; /* first mds map */ + } + mdsc->fsc->max_file_size = min((loff_t)mdsc->mdsmap->m_max_file_size, + MAX_LFS_FILESIZE); + + __wake_requests(mdsc, &mdsc->waiting_for_map); + ceph_monc_got_map(&mdsc->fsc->client->monc, CEPH_SUB_MDSMAP, + mdsc->mdsmap->m_epoch); + + mutex_unlock(&mdsc->mutex); + schedule_delayed(mdsc, 0); + return; + +bad_unlock: + mutex_unlock(&mdsc->mutex); +bad: + pr_err_client(cl, "error decoding mdsmap %d. Shutting down mount.\n", + err); + ceph_umount_begin(mdsc->fsc->sb); + ceph_msg_dump(msg); + return; +} + +static struct ceph_connection *mds_get_con(struct ceph_connection *con) +{ + struct ceph_mds_session *s = con->private; + + if (ceph_get_mds_session(s)) + return con; + return NULL; +} + +static void mds_put_con(struct ceph_connection *con) +{ + struct ceph_mds_session *s = con->private; + + ceph_put_mds_session(s); +} + +/* + * if the client is unresponsive for long enough, the mds will kill + * the session entirely. + */ +static void mds_peer_reset(struct ceph_connection *con) +{ + struct ceph_mds_session *s = con->private; + struct ceph_mds_client *mdsc = s->s_mdsc; + int session_state; + + pr_warn_client(mdsc->fsc->client, "mds%d closed our session\n", + s->s_mds); + + if (READ_ONCE(mdsc->fsc->mount_state) == CEPH_MOUNT_FENCE_IO || + ceph_mdsmap_get_state(mdsc->mdsmap, s->s_mds) < CEPH_MDS_STATE_RECONNECT) + return; + + /* + * Only reconnect if MDS is in its RECONNECT phase. An MDS past + * RECONNECT (REJOIN, CLIENTREPLAY, ACTIVE) will reject reconnect + * attempts, so those states fall through to session teardown below. + */ + if (ceph_mdsmap_get_state(mdsc->mdsmap, s->s_mds) == CEPH_MDS_STATE_RECONNECT) { + int rc = send_mds_reconnect(mdsc, s); + + if (rc) + pr_warn_client(mdsc->fsc->client, + "mds%d reconnect failed: %d\n", + s->s_mds, rc); + return; + } + + /* + * MDS is active (past RECONNECT). It will not accept a + * CLIENT_RECONNECT from us, so tear the session down locally + * and let new requests re-open a fresh session. + * + * Snapshot session state with READ_ONCE, then revalidate under + * mdsc->mutex before acting. The subsequent mdsc->mutex + * section rechecks s_state to catch concurrent transitions, so + * the lockless snapshot here is safe. s->s_mutex is taken + * separately for cleanup after unregistration, which avoids + * introducing a new s->s_mutex + mdsc->mutex nesting. + */ + session_state = READ_ONCE(s->s_state); + + switch (session_state) { + case CEPH_MDS_SESSION_RESTARTING: + case CEPH_MDS_SESSION_RECONNECTING: + case CEPH_MDS_SESSION_CLOSING: + case CEPH_MDS_SESSION_OPEN: + case CEPH_MDS_SESSION_HUNG: + case CEPH_MDS_SESSION_OPENING: + mutex_lock(&mdsc->mutex); + if (s->s_mds >= mdsc->max_sessions || + mdsc->sessions[s->s_mds] != s || + s->s_state != session_state) { + pr_info_client(mdsc->fsc->client, + "mds%d state changed to %s during peer reset\n", + s->s_mds, + ceph_session_state_name(s->s_state)); + mutex_unlock(&mdsc->mutex); + return; + } + + ceph_get_mds_session(s); + s->s_state = CEPH_MDS_SESSION_CLOSED; + __unregister_session(mdsc, s); + __wake_requests(mdsc, &s->s_waiting); + mutex_unlock(&mdsc->mutex); + + mutex_lock(&s->s_mutex); + cleanup_session_requests(mdsc, s); + remove_session_caps(s); + mutex_unlock(&s->s_mutex); + + wake_up_all(&mdsc->session_close_wq); + + mutex_lock(&mdsc->mutex); + kick_requests(mdsc, s->s_mds); + mutex_unlock(&mdsc->mutex); + + ceph_put_mds_session(s); + break; + case CEPH_MDS_SESSION_CLOSED: + case CEPH_MDS_SESSION_REJECTED: + break; + default: + pr_warn_client(mdsc->fsc->client, + "mds%d peer reset in unexpected state %s\n", + s->s_mds, + ceph_session_state_name(session_state)); + break; + } +} + +static void mds_dispatch(struct ceph_connection *con, struct ceph_msg *msg) +{ + struct ceph_mds_session *s = con->private; + struct ceph_mds_client *mdsc = s->s_mdsc; + struct ceph_client *cl = mdsc->fsc->client; + int type = le16_to_cpu(msg->hdr.type); + + mutex_lock(&mdsc->mutex); + if (__verify_registered_session(mdsc, s) < 0) { + doutc(cl, "dropping tid %llu from unregistered session %d\n", + le64_to_cpu(msg->hdr.tid), s->s_mds); + mutex_unlock(&mdsc->mutex); + goto out; + } + mutex_unlock(&mdsc->mutex); + + switch (type) { + case CEPH_MSG_MDS_MAP: + ceph_mdsc_handle_mdsmap(mdsc, msg); + break; + case CEPH_MSG_FS_MAP_USER: + ceph_mdsc_handle_fsmap(mdsc, msg); + break; + case CEPH_MSG_CLIENT_SESSION: + handle_session(s, msg); + break; + case CEPH_MSG_CLIENT_REPLY: + handle_reply(s, msg); + break; + case CEPH_MSG_CLIENT_REQUEST_FORWARD: + handle_forward(mdsc, s, msg); + break; + case CEPH_MSG_CLIENT_CAPS: + ceph_handle_caps(s, msg); + break; + case CEPH_MSG_CLIENT_SNAP: + ceph_handle_snap(mdsc, s, msg); + break; + case CEPH_MSG_CLIENT_LEASE: + handle_lease(mdsc, s, msg); + break; + case CEPH_MSG_CLIENT_QUOTA: + ceph_handle_quota(mdsc, s, msg); + break; + + default: + pr_err_client(cl, "received unknown message type %d %s\n", + type, ceph_msg_type_name(type)); + } +out: + ceph_msg_put(msg); +} + +/* + * authentication + */ + +/* + * Note: returned pointer is the address of a structure that's + * managed separately. Caller must *not* attempt to free it. + */ +static struct ceph_auth_handshake * +mds_get_authorizer(struct ceph_connection *con, int *proto, int force_new) +{ + struct ceph_mds_session *s = con->private; + struct ceph_mds_client *mdsc = s->s_mdsc; + struct ceph_auth_client *ac = mdsc->fsc->client->monc.auth; + struct ceph_auth_handshake *auth = &s->s_auth; + int ret; + + ret = __ceph_auth_get_authorizer(ac, auth, CEPH_ENTITY_TYPE_MDS, + force_new, proto, NULL, NULL); + if (ret) + return ERR_PTR(ret); + + return auth; +} + +static int mds_add_authorizer_challenge(struct ceph_connection *con, + void *challenge_buf, int challenge_buf_len) +{ + struct ceph_mds_session *s = con->private; + struct ceph_mds_client *mdsc = s->s_mdsc; + struct ceph_auth_client *ac = mdsc->fsc->client->monc.auth; + + return ceph_auth_add_authorizer_challenge(ac, s->s_auth.authorizer, + challenge_buf, challenge_buf_len); +} + +static int mds_verify_authorizer_reply(struct ceph_connection *con) +{ + struct ceph_mds_session *s = con->private; + struct ceph_mds_client *mdsc = s->s_mdsc; + struct ceph_auth_client *ac = mdsc->fsc->client->monc.auth; + struct ceph_auth_handshake *auth = &s->s_auth; + + return ceph_auth_verify_authorizer_reply(ac, auth->authorizer, + auth->authorizer_reply_buf, auth->authorizer_reply_buf_len, + NULL, NULL, NULL, NULL); +} + +static int mds_invalidate_authorizer(struct ceph_connection *con) +{ + struct ceph_mds_session *s = con->private; + struct ceph_mds_client *mdsc = s->s_mdsc; + struct ceph_auth_client *ac = mdsc->fsc->client->monc.auth; + + ceph_auth_invalidate_authorizer(ac, CEPH_ENTITY_TYPE_MDS); + + return ceph_monc_validate_auth(&mdsc->fsc->client->monc); +} + +static int mds_get_auth_request(struct ceph_connection *con, + void *buf, int *buf_len, + void **authorizer, int *authorizer_len) +{ + struct ceph_mds_session *s = con->private; + struct ceph_auth_client *ac = s->s_mdsc->fsc->client->monc.auth; + struct ceph_auth_handshake *auth = &s->s_auth; + int ret; + + ret = ceph_auth_get_authorizer(ac, auth, CEPH_ENTITY_TYPE_MDS, + buf, buf_len); + if (ret) + return ret; + + *authorizer = auth->authorizer_buf; + *authorizer_len = auth->authorizer_buf_len; + return 0; +} + +static int mds_handle_auth_reply_more(struct ceph_connection *con, + void *reply, int reply_len, + void *buf, int *buf_len, + void **authorizer, int *authorizer_len) +{ + struct ceph_mds_session *s = con->private; + struct ceph_auth_client *ac = s->s_mdsc->fsc->client->monc.auth; + struct ceph_auth_handshake *auth = &s->s_auth; + int ret; + + ret = ceph_auth_handle_svc_reply_more(ac, auth, reply, reply_len, + buf, buf_len); + if (ret) + return ret; + + *authorizer = auth->authorizer_buf; + *authorizer_len = auth->authorizer_buf_len; + return 0; +} + +static int mds_handle_auth_done(struct ceph_connection *con, + u64 global_id, void *reply, int reply_len, + u8 *session_key, int *session_key_len, + u8 *con_secret, int *con_secret_len) +{ + struct ceph_mds_session *s = con->private; + struct ceph_auth_client *ac = s->s_mdsc->fsc->client->monc.auth; + struct ceph_auth_handshake *auth = &s->s_auth; + + return ceph_auth_handle_svc_reply_done(ac, auth, reply, reply_len, + session_key, session_key_len, + con_secret, con_secret_len); +} + +static int mds_handle_auth_bad_method(struct ceph_connection *con, + int used_proto, int result, + const int *allowed_protos, int proto_cnt, + const int *allowed_modes, int mode_cnt) +{ + struct ceph_mds_session *s = con->private; + struct ceph_mon_client *monc = &s->s_mdsc->fsc->client->monc; + int ret; + + if (ceph_auth_handle_bad_authorizer(monc->auth, CEPH_ENTITY_TYPE_MDS, + used_proto, result, + allowed_protos, proto_cnt, + allowed_modes, mode_cnt)) { + ret = ceph_monc_validate_auth(monc); + if (ret) + return ret; + } + + return -EACCES; +} + +static struct ceph_msg *mds_alloc_msg(struct ceph_connection *con, + struct ceph_msg_header *hdr, int *skip) +{ + struct ceph_msg *msg; + int type = (int) le16_to_cpu(hdr->type); + int front_len = (int) le32_to_cpu(hdr->front_len); + + if (con->in_msg) + return con->in_msg; + + *skip = 0; + msg = ceph_msg_new(type, front_len, GFP_NOFS, false); + if (!msg) { + pr_err("unable to allocate msg type %d len %d\n", + type, front_len); + return NULL; + } + + return msg; +} + +static int mds_sign_message(struct ceph_msg *msg) +{ + struct ceph_mds_session *s = msg->con->private; + struct ceph_auth_handshake *auth = &s->s_auth; + + return ceph_auth_sign_message(auth, msg); +} + +static int mds_check_message_signature(struct ceph_msg *msg) +{ + struct ceph_mds_session *s = msg->con->private; + struct ceph_auth_handshake *auth = &s->s_auth; + + return ceph_auth_check_message_signature(auth, msg); +} + +static const struct ceph_connection_operations mds_con_ops = { + .get = mds_get_con, + .put = mds_put_con, + .alloc_msg = mds_alloc_msg, + .dispatch = mds_dispatch, + .peer_reset = mds_peer_reset, + .get_authorizer = mds_get_authorizer, + .add_authorizer_challenge = mds_add_authorizer_challenge, + .verify_authorizer_reply = mds_verify_authorizer_reply, + .invalidate_authorizer = mds_invalidate_authorizer, + .sign_message = mds_sign_message, + .check_message_signature = mds_check_message_signature, + .get_auth_request = mds_get_auth_request, + .handle_auth_reply_more = mds_handle_auth_reply_more, + .handle_auth_done = mds_handle_auth_done, + .handle_auth_bad_method = mds_handle_auth_bad_method, +}; + +/* eof */ diff --git a/fs/ceph/mds_client.h b/fs/ceph/mds_client.h new file mode 100644 index 000000000..0ece4c9e3 --- /dev/null +++ b/fs/ceph/mds_client.h @@ -0,0 +1,733 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _FS_CEPH_MDS_CLIENT_H +#define _FS_CEPH_MDS_CLIENT_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "mdsmap.h" +#include "metric.h" +#include "subvolume_metrics.h" +#include "super.h" + +/* The first 8 bits are reserved for old ceph releases */ +enum ceph_feature_type { + CEPHFS_FEATURE_MIMIC = 8, + CEPHFS_FEATURE_REPLY_ENCODING, + CEPHFS_FEATURE_RECLAIM_CLIENT, + CEPHFS_FEATURE_LAZY_CAP_WANTED, + CEPHFS_FEATURE_MULTI_RECONNECT, + CEPHFS_FEATURE_DELEG_INO, + CEPHFS_FEATURE_METRIC_COLLECT, + CEPHFS_FEATURE_ALTERNATE_NAME, + CEPHFS_FEATURE_NOTIFY_SESSION_STATE, + CEPHFS_FEATURE_OP_GETVXATTR, + CEPHFS_FEATURE_32BITS_RETRY_FWD, + CEPHFS_FEATURE_NEW_SNAPREALM_INFO, + CEPHFS_FEATURE_HAS_OWNER_UIDGID, + CEPHFS_FEATURE_MDS_AUTH_CAPS_CHECK, + CEPHFS_FEATURE_SUBVOLUME_METRICS, + + CEPHFS_FEATURE_MAX = CEPHFS_FEATURE_SUBVOLUME_METRICS, +}; + +#define CEPHFS_FEATURES_CLIENT_SUPPORTED { \ + 0, 1, 2, 3, 4, 5, 6, 7, \ + CEPHFS_FEATURE_MIMIC, \ + CEPHFS_FEATURE_REPLY_ENCODING, \ + CEPHFS_FEATURE_LAZY_CAP_WANTED, \ + CEPHFS_FEATURE_MULTI_RECONNECT, \ + CEPHFS_FEATURE_DELEG_INO, \ + CEPHFS_FEATURE_METRIC_COLLECT, \ + CEPHFS_FEATURE_ALTERNATE_NAME, \ + CEPHFS_FEATURE_NOTIFY_SESSION_STATE, \ + CEPHFS_FEATURE_OP_GETVXATTR, \ + CEPHFS_FEATURE_32BITS_RETRY_FWD, \ + CEPHFS_FEATURE_HAS_OWNER_UIDGID, \ + CEPHFS_FEATURE_MDS_AUTH_CAPS_CHECK, \ + CEPHFS_FEATURE_SUBVOLUME_METRICS, \ +} + +/* + * Some lock dependencies: + * + * session->s_mutex + * mdsc->mutex + * + * mdsc->snap_rwsem + * + * ci->i_ceph_lock + * mdsc->snap_flush_lock + * mdsc->cap_delay_lock + * + */ + +struct ceph_fs_client; +struct ceph_cap; + +#define MDS_AUTH_UID_ANY -1 +#define CEPH_GET_CAPS_WAIT_TIMEOUT (5 * HZ) +#define CEPH_CAP_FLUSH_WAIT_TIMEOUT_SEC 60 +#define CEPH_CAP_FLUSH_MAX_DUMP_ENTRIES 5 +#define CEPH_CAP_FLUSH_MAX_DUMP_ITERS 5 +#define CEPH_CLIENT_RESET_REASON_LEN 64 +#define CEPH_CLIENT_RESET_DRAIN_SEC 30 +#define CEPH_CLIENT_RESET_CLOSE_GRACE_MS 100 +#define CEPH_CLIENT_RESET_WAIT_TIMEOUT_SEC 120 + +enum ceph_client_reset_phase { + CEPH_CLIENT_RESET_IDLE = 0, + /* + * QUIESCING is set synchronously by schedule_reset() before the + * workqueue item is dispatched. It gates new requests (any + * phase != IDLE blocks callers) during the window between + * scheduling and the work function's transition to DRAINING. + */ + CEPH_CLIENT_RESET_QUIESCING, + CEPH_CLIENT_RESET_DRAINING, + CEPH_CLIENT_RESET_TEARDOWN, +}; + +struct ceph_client_reset_state { + spinlock_t lock; /* protects all fields below */ + u64 trigger_count; /* number of resets triggered */ + u64 success_count; /* number of successful resets */ + u64 failure_count; /* number of failed resets */ + unsigned long last_start; /* jiffies when last reset started */ + unsigned long last_finish; /* jiffies when last reset finished */ + int last_errno; /* result of most recent reset */ + enum ceph_client_reset_phase phase; /* current reset phase */ + bool drain_timed_out; /* drain exceeded timeout */ + bool shutdown; /* destroy in progress */ + int sessions_reset; /* sessions torn down in last reset */ + char last_reason[CEPH_CLIENT_RESET_REASON_LEN]; /* operator-supplied reason */ + + /* Request blocking during reset */ + wait_queue_head_t blocked_wq; /* waitqueue for blocked callers */ + atomic_t blocked_requests; /* count of blocked callers */ +}; + +static inline bool ceph_reset_is_idle(struct ceph_client_reset_state *st) +{ + return READ_ONCE(st->phase) == CEPH_CLIENT_RESET_IDLE; +} + +struct ceph_mds_cap_match { + s64 uid; /* default to MDS_AUTH_UID_ANY */ + u32 num_gids; + u32 *gids; /* use these GIDs */ + char *path; /* require path to be child of this + (may be "" or "/" for any) */ + char *fs_name; + bool root_squash; /* default to false */ +}; + +struct ceph_mds_cap_auth { + struct ceph_mds_cap_match match; + bool readable; + bool writeable; +}; + +/* + * parsed info about a single inode. pointers are into the encoded + * on-wire structures within the mds reply message payload. + */ +struct ceph_mds_reply_info_in { + struct ceph_mds_reply_inode *in; + struct ceph_dir_layout dir_layout; + u32 symlink_len; + char *symlink; + u32 xattr_len; + char *xattr_data; + u64 inline_version; + u32 inline_len; + char *inline_data; + u32 pool_ns_len; + char *pool_ns_data; + u64 max_bytes; + u64 max_files; + s32 dir_pin; + struct ceph_timespec btime; + struct ceph_timespec snap_btime; + u8 *fscrypt_auth; + u8 *fscrypt_file; + u32 fscrypt_auth_len; + u32 fscrypt_file_len; + u64 rsnaps; + u64 change_attr; + u64 subvolume_id; +}; + +struct ceph_mds_reply_dir_entry { + bool is_nokey; + char *name; + u32 name_len; + u32 raw_hash; + struct ceph_mds_reply_lease *lease; + struct ceph_mds_reply_info_in inode; + loff_t offset; +}; + +struct ceph_mds_reply_xattr { + char *xattr_value; + size_t xattr_value_len; +}; + +/* + * parsed info about an mds reply, including information about + * either: 1) the target inode and/or its parent directory and dentry, + * and directory contents (for readdir results), or + * 2) the file range lock info (for fcntl F_GETLK results). + */ +struct ceph_mds_reply_info_parsed { + struct ceph_mds_reply_head *head; + + /* trace */ + struct ceph_mds_reply_info_in diri, targeti; + struct ceph_mds_reply_dirfrag *dirfrag; + char *dname; + u8 *altname; + u32 dname_len; + u32 altname_len; + struct ceph_mds_reply_lease *dlease; + struct ceph_mds_reply_xattr xattr_info; + + /* extra */ + union { + /* for fcntl F_GETLK results */ + struct ceph_filelock *filelock_reply; + + /* for readdir results */ + struct { + struct ceph_mds_reply_dirfrag *dir_dir; + size_t dir_buf_size; + int dir_nr; + bool dir_end; + bool dir_complete; + bool hash_order; + bool offset_hash; + struct ceph_mds_reply_dir_entry *dir_entries; + }; + + /* for create results */ + struct { + bool has_create_ino; + u64 ino; + }; + }; + + /* encoded blob describing snapshot contexts for certain + operations (e.g., open) */ + void *snapblob; + int snapblob_len; +}; + + +/* + * cap releases are batched and sent to the MDS en masse. + * + * Account for per-message overhead of mds_cap_release header + * and __le32 for osd epoch barrier trailing field. + */ +#define CEPH_CAPS_PER_RELEASE ((PAGE_SIZE - sizeof(u32) - \ + sizeof(struct ceph_mds_cap_release)) / \ + sizeof(struct ceph_mds_cap_item)) + + +/* + * state associated with each MDS<->client session + */ +enum { + CEPH_MDS_SESSION_NEW = 1, + CEPH_MDS_SESSION_OPENING = 2, + CEPH_MDS_SESSION_OPEN = 3, + CEPH_MDS_SESSION_HUNG = 4, + CEPH_MDS_SESSION_RESTARTING = 5, + CEPH_MDS_SESSION_RECONNECTING = 6, + CEPH_MDS_SESSION_CLOSING = 7, + CEPH_MDS_SESSION_CLOSED = 8, + CEPH_MDS_SESSION_REJECTED = 9, +}; + +struct ceph_mds_session { + struct ceph_mds_client *s_mdsc; + int s_mds; + int s_state; + unsigned long s_ttl; /* time until mds kills us */ + unsigned long s_features; + u64 s_seq; /* incoming msg seq # */ + struct mutex s_mutex; /* serialize session messages */ + + struct ceph_connection s_con; + + struct ceph_auth_handshake s_auth; + + atomic_t s_cap_gen; /* inc each time we get mds stale msg */ + unsigned long s_cap_ttl; /* when session caps expire. protected by s_mutex */ + + /* protected by s_cap_lock */ + spinlock_t s_cap_lock; + refcount_t s_ref; + struct list_head s_caps; /* all caps issued by this session */ + struct ceph_cap *s_cap_iterator; + int s_nr_caps; + int s_num_cap_releases; + int s_cap_reconnect; + int s_readonly; + struct list_head s_cap_releases; /* waiting cap_release messages */ + struct work_struct s_cap_release_work; + + /* See ceph_inode_info->i_dirty_item. */ + struct list_head s_cap_dirty; /* inodes w/ dirty caps */ + + /* See ceph_inode_info->i_flushing_item. */ + struct list_head s_cap_flushing; /* inodes w/ flushing caps */ + + unsigned long s_renew_requested; /* last time we sent a renew req */ + u64 s_renew_seq; + + struct list_head s_waiting; /* waiting requests */ + struct list_head s_unsafe; /* unsafe requests */ + struct xarray s_delegated_inos; +}; + +/* + * modes of choosing which MDS to send a request to + */ +enum { + USE_ANY_MDS, + USE_RANDOM_MDS, + USE_AUTH_MDS, /* prefer authoritative mds for this metadata item */ +}; + +struct ceph_mds_request; +struct ceph_mds_client; + +/* + * request completion callback + */ +typedef void (*ceph_mds_request_callback_t) (struct ceph_mds_client *mdsc, + struct ceph_mds_request *req); +/* + * wait for request completion callback + */ +typedef int (*ceph_mds_request_wait_callback_t) (struct ceph_mds_client *mdsc, + struct ceph_mds_request *req); + +/* + * an in-flight mds request + */ +struct ceph_mds_request { + u64 r_tid; /* transaction id */ + struct rb_node r_node; + struct ceph_mds_client *r_mdsc; + + struct kref r_kref; + int r_op; /* mds op code */ + + /* operation on what? */ + struct inode *r_inode; /* arg1 */ + struct dentry *r_dentry; /* arg1 */ + struct dentry *r_old_dentry; /* arg2: rename from or link from */ + struct inode *r_old_dentry_dir; /* arg2: old dentry's parent dir */ + char *r_path1, *r_path2; + struct ceph_vino r_ino1, r_ino2; + + struct inode *r_parent; /* parent dir inode */ + struct inode *r_target_inode; /* resulting inode */ + struct inode *r_new_inode; /* new inode (for creates) */ + + const struct qstr *r_dname; /* stable name (for ->d_revalidate) */ + +#define CEPH_MDS_R_DIRECT_IS_HASH (1) /* r_direct_hash is valid */ +#define CEPH_MDS_R_ABORTED (2) /* call was aborted */ +#define CEPH_MDS_R_GOT_UNSAFE (3) /* got an unsafe reply */ +#define CEPH_MDS_R_GOT_SAFE (4) /* got a safe reply */ +#define CEPH_MDS_R_GOT_RESULT (5) /* got a result */ +#define CEPH_MDS_R_DID_PREPOPULATE (6) /* prepopulated readdir */ +#define CEPH_MDS_R_PARENT_LOCKED (7) /* is r_parent->i_rwsem wlocked? */ +#define CEPH_MDS_R_ASYNC (8) /* async request */ +#define CEPH_MDS_R_FSCRYPT_FILE (9) /* must marshal fscrypt_file field */ + unsigned long r_req_flags; + + struct mutex r_fill_mutex; + + union ceph_mds_request_args r_args; + + struct ceph_fscrypt_auth *r_fscrypt_auth; + u64 r_fscrypt_file; + + u8 *r_altname; /* fscrypt binary crypttext for long filenames */ + u32 r_altname_len; /* length of r_altname */ + + int r_fmode; /* file mode, if expecting cap */ + int r_request_release_offset; + const struct cred *r_cred; + struct mnt_idmap *r_mnt_idmap; + struct timespec64 r_stamp; + + /* for choosing which mds to send this request to */ + int r_direct_mode; + u32 r_direct_hash; /* choose dir frag based on this dentry hash */ + + /* data payload is used for xattr ops */ + struct ceph_pagelist *r_pagelist; + + /* what caps shall we drop? */ + int r_inode_drop, r_inode_unless; + int r_dentry_drop, r_dentry_unless; + int r_old_dentry_drop, r_old_dentry_unless; + struct inode *r_old_inode; + int r_old_inode_drop, r_old_inode_unless; + + struct ceph_msg *r_request; /* original request */ + struct ceph_msg *r_reply; + struct ceph_mds_reply_info_parsed r_reply_info; + int r_err; + u32 r_readdir_offset; + + struct page *r_locked_page; + int r_dir_caps; + int r_num_caps; + + unsigned long r_timeout; /* optional. jiffies, 0 is "wait forever" */ + unsigned long r_started; /* start time to measure timeout against */ + unsigned long r_start_latency; /* start time to measure latency */ + unsigned long r_end_latency; /* finish time to measure latency */ + unsigned long r_request_started; /* start time for mds request only, + used to measure lease durations */ + + /* link unsafe requests to parent directory, for fsync */ + struct inode *r_unsafe_dir; + struct list_head r_unsafe_dir_item; + + /* unsafe requests that modify the target inode */ + struct list_head r_unsafe_target_item; + + struct ceph_mds_session *r_session; + + int r_attempts; /* resend attempts */ + int r_num_fwd; /* number of forward attempts */ + int r_resend_mds; /* mds to resend to next, if any*/ + u32 r_sent_on_mseq; /* cap mseq request was sent at*/ + u64 r_deleg_ino; + + struct list_head r_wait; + struct completion r_completion; + struct completion r_safe_completion; + ceph_mds_request_callback_t r_callback; + struct list_head r_unsafe_item; /* per-session unsafe list item */ + + long long r_dir_release_cnt; + long long r_dir_ordered_cnt; + int r_readdir_cache_idx; + + int r_feature_needed; + + struct ceph_cap_reservation r_caps_reservation; +}; + +struct ceph_pool_perm { + struct rb_node node; + int perm; + s64 pool; + size_t pool_ns_len; + char pool_ns[]; +}; + +struct ceph_snapid_map { + struct rb_node node; + struct list_head lru; + atomic_t ref; + dev_t dev; + u64 snap; + unsigned long last_used; +}; + +/* + * node for list of quotarealm inodes that are not visible from the filesystem + * mountpoint, but required to handle, e.g. quotas. + */ +struct ceph_quotarealm_inode { + struct rb_node node; + u64 ino; + unsigned long timeout; /* last time a lookup failed for this inode */ + struct mutex mutex; + struct inode *inode; +}; + +#ifdef CONFIG_DEBUG_FS + +struct cap_wait { + struct list_head list; + u64 ino; + pid_t tgid; + int need; + int want; +}; + +#endif + +enum { + CEPH_MDSC_STOPPING_BEGIN = 1, + CEPH_MDSC_STOPPING_FLUSHING = 2, + CEPH_MDSC_STOPPING_FLUSHED = 3, +}; + +/* + * mds client state + */ +struct ceph_mds_client { + struct ceph_fs_client *fsc; + struct mutex mutex; /* all nested structures */ + + struct ceph_mdsmap *mdsmap; + struct completion safe_umount_waiters; + wait_queue_head_t session_close_wq; + struct list_head waiting_for_map; + int mdsmap_err; + + struct ceph_mds_session **sessions; /* NULL for mds if no session */ + atomic_t num_sessions; + int max_sessions; /* len of sessions array */ + + spinlock_t stopping_lock; /* protect snap_empty */ + int stopping; /* the stage of shutting down */ + atomic_t stopping_blockers; + struct completion stopping_waiter; + + atomic64_t dirty_folios; + wait_queue_head_t flush_end_wq; + + atomic64_t quotarealms_count; /* # realms with quota */ + /* + * We keep a list of inodes we don't see in the mountpoint but that we + * need to track quota realms. + */ + struct rb_root quotarealms_inodes; + struct mutex quotarealms_inodes_mutex; + + /* + * snap_rwsem will cover cap linkage into snaprealms, and + * realm snap contexts. (later, we can do per-realm snap + * contexts locks..) the empty list contains realms with no + * references (implying they contain no inodes with caps) that + * should be destroyed. + */ + u64 last_snap_seq; + struct rw_semaphore snap_rwsem; + struct rb_root snap_realms; + struct list_head snap_empty; + int num_snap_realms; + spinlock_t snap_empty_lock; /* protect snap_empty */ + + u64 last_tid; /* most recent mds request */ + u64 oldest_tid; /* oldest incomplete mds request, + excluding setfilelock requests */ + struct rb_root request_tree; /* pending mds requests */ + struct delayed_work delayed_work; /* delayed work */ + unsigned long last_renew_caps; /* last time we renewed our caps */ + struct list_head cap_delay_list; /* caps with delayed release */ + struct list_head cap_unlink_delay_list; /* caps with delayed release for unlink */ + spinlock_t cap_delay_lock; /* protects cap_delay_list and cap_unlink_delay_list */ + struct list_head snap_flush_list; /* cap_snaps ready to flush */ + spinlock_t snap_flush_lock; + + u64 last_cap_flush_tid; + struct list_head cap_flush_list; + struct list_head cap_dirty_migrating; /* ...that are migration... */ + int num_cap_flushing; /* # caps we are flushing */ + spinlock_t cap_dirty_lock; /* protects above items */ + wait_queue_head_t cap_flushing_wq; + + struct work_struct cap_reclaim_work; + atomic_t cap_reclaim_pending; + + struct work_struct cap_unlink_work; + + /* + * Cap reservations + * + * Maintain a global pool of preallocated struct ceph_caps, referenced + * by struct ceph_caps_reservations. This ensures that we preallocate + * memory needed to successfully process an MDS response. (If an MDS + * sends us cap information and we fail to process it, we will have + * problems due to the client and MDS being out of sync.) + * + * Reservations are 'owned' by a ceph_cap_reservation context. + */ + spinlock_t caps_list_lock; + struct list_head caps_list; /* unused (reserved or + unreserved) */ +#ifdef CONFIG_DEBUG_FS + struct list_head cap_wait_list; +#endif + int caps_total_count; /* total caps allocated */ + int caps_use_count; /* in use */ + int caps_use_max; /* max used caps */ + int caps_reserve_count; /* unused, reserved */ + int caps_avail_count; /* unused, unreserved */ + int caps_min_count; /* keep at least this many + (unreserved) */ + spinlock_t dentry_list_lock; + struct list_head dentry_leases; /* fifo list */ + struct list_head dentry_dir_leases; /* lru list */ + + struct ceph_client_metric metric; + struct work_struct reset_work; + struct ceph_client_reset_state reset_state; + struct ceph_subvolume_metrics_tracker subvol_metrics; + + /* Subvolume metrics send tracking */ + struct mutex subvol_metrics_last_mutex; + struct ceph_subvol_metric_snapshot *subvol_metrics_last; + u32 subvol_metrics_last_nr; + u64 subvol_metrics_sent; + u64 subvol_metrics_nonzero_sends; + + spinlock_t snapid_map_lock; + struct rb_root snapid_map_tree; + struct list_head snapid_map_lru; + + struct rw_semaphore pool_perm_rwsem; + struct rb_root pool_perm_tree; + + u32 s_cap_auths_num; + struct ceph_mds_cap_auth *s_cap_auths; + + char nodename[__NEW_UTS_LEN + 1]; +}; + +extern const char *ceph_mds_op_name(int op); + +extern bool check_session_state(struct ceph_mds_session *s); +void inc_session_sequence(struct ceph_mds_session *s); + +extern struct ceph_mds_session * +__ceph_lookup_mds_session(struct ceph_mds_client *, int mds); + +extern const char *ceph_session_state_name(int s); +extern const char *ceph_reset_phase_name(enum ceph_client_reset_phase phase); + +extern struct ceph_mds_session * +ceph_get_mds_session(struct ceph_mds_session *s); +extern void ceph_put_mds_session(struct ceph_mds_session *s); +int ceph_mdsc_schedule_reset(struct ceph_mds_client *mdsc, + const char *reason); +int ceph_mdsc_wait_for_reset(struct ceph_mds_client *mdsc); + +extern int ceph_mdsc_init(struct ceph_fs_client *fsc); +extern void ceph_mdsc_close_sessions(struct ceph_mds_client *mdsc); +extern void ceph_mdsc_force_umount(struct ceph_mds_client *mdsc); +extern void ceph_mdsc_destroy(struct ceph_fs_client *fsc); + +extern void ceph_mdsc_sync(struct ceph_mds_client *mdsc); + +extern void ceph_invalidate_dir_request(struct ceph_mds_request *req); +extern int ceph_alloc_readdir_reply_buffer(struct ceph_mds_request *req, + struct inode *dir); +extern struct ceph_mds_request * +ceph_mdsc_create_request(struct ceph_mds_client *mdsc, int op, int mode); +extern int ceph_mdsc_submit_request(struct ceph_mds_client *mdsc, + struct inode *dir, + struct ceph_mds_request *req); +int ceph_mdsc_wait_request(struct ceph_mds_client *mdsc, + struct ceph_mds_request *req, + ceph_mds_request_wait_callback_t wait_func); +extern int ceph_mdsc_do_request(struct ceph_mds_client *mdsc, + struct inode *dir, + struct ceph_mds_request *req); +extern void ceph_mdsc_release_dir_caps(struct ceph_mds_request *req); +extern void ceph_mdsc_release_dir_caps_async(struct ceph_mds_request *req); +static inline void ceph_mdsc_get_request(struct ceph_mds_request *req) +{ + kref_get(&req->r_kref); +} +extern void ceph_mdsc_release_request(struct kref *kref); +static inline void ceph_mdsc_put_request(struct ceph_mds_request *req) +{ + kref_put(&req->r_kref, ceph_mdsc_release_request); +} + +extern void send_flush_mdlog(struct ceph_mds_session *s); +extern void ceph_mdsc_iterate_sessions(struct ceph_mds_client *mdsc, + void (*cb)(struct ceph_mds_session *), + bool check_state); +extern struct ceph_msg *ceph_create_session_msg(u32 op, u64 seq); +extern void __ceph_queue_cap_release(struct ceph_mds_session *session, + struct ceph_cap *cap); +extern void ceph_flush_session_cap_releases(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session); +extern void ceph_queue_cap_reclaim_work(struct ceph_mds_client *mdsc); +extern void ceph_reclaim_caps_nr(struct ceph_mds_client *mdsc, int nr); +extern void ceph_queue_cap_unlink_work(struct ceph_mds_client *mdsc); +extern int ceph_iterate_session_caps(struct ceph_mds_session *session, + int (*cb)(struct inode *, int mds, void *), + void *arg); +extern int ceph_mds_check_access(struct ceph_mds_client *mdsc, char *tpath, + int mask); + +extern void ceph_mdsc_pre_umount(struct ceph_mds_client *mdsc); + +/* + * Structure to group path-related output parameters for build_*_path functions + */ +struct ceph_path_info { + const char *path; + int pathlen; + struct ceph_vino vino; + bool freepath; +}; + +static inline void ceph_mdsc_free_path_info(const struct ceph_path_info *path_info) +{ + if (path_info && path_info->freepath && !IS_ERR_OR_NULL(path_info->path)) + __putname((char *)path_info->path - (PATH_MAX - 1 - path_info->pathlen)); +} + +extern char *ceph_mdsc_build_path(struct ceph_mds_client *mdsc, + struct dentry *dentry, struct ceph_path_info *path_info, + int for_wire); + +extern void __ceph_mdsc_drop_dentry_lease(struct dentry *dentry); +extern void ceph_mdsc_lease_send_msg(struct ceph_mds_session *session, + struct dentry *dentry, char action, + u32 seq); + +extern void ceph_mdsc_handle_mdsmap(struct ceph_mds_client *mdsc, + struct ceph_msg *msg); +extern void ceph_mdsc_handle_fsmap(struct ceph_mds_client *mdsc, + struct ceph_msg *msg); + +extern struct ceph_mds_session * +ceph_mdsc_open_export_target_session(struct ceph_mds_client *mdsc, int target); + +extern int ceph_trim_caps(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session, + int max_caps); + +static inline int ceph_wait_on_async_create(struct inode *inode) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + + return wait_on_bit(&ci->i_ceph_flags, CEPH_I_ASYNC_CREATE_BIT, + TASK_KILLABLE); +} + +extern int ceph_wait_on_conflict_unlink(struct dentry *dentry); +extern u64 ceph_get_deleg_ino(struct ceph_mds_session *session); +extern int ceph_restore_deleg_ino(struct ceph_mds_session *session, u64 ino); + +extern bool enable_unsafe_idmap; +#endif diff --git a/fs/ceph/mdsmap.c b/fs/ceph/mdsmap.c new file mode 100644 index 000000000..450a4dc96 --- /dev/null +++ b/fs/ceph/mdsmap.c @@ -0,0 +1,462 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include "mdsmap.h" +#include "mds_client.h" +#include "super.h" + +#define CEPH_MDS_IS_READY(i, ignore_laggy) \ + (m->m_info[i].state > 0 && (ignore_laggy ? true : !m->m_info[i].laggy)) + +static int __mdsmap_get_random_mds(struct ceph_mdsmap *m, bool ignore_laggy) +{ + int n = 0; + int i, j; + + /* count */ + for (i = 0; i < m->possible_max_rank; i++) + if (CEPH_MDS_IS_READY(i, ignore_laggy)) + n++; + if (n == 0) + return -1; + + /* pick */ + n = get_random_u32_below(n); + for (j = 0, i = 0; i < m->possible_max_rank; i++) { + if (CEPH_MDS_IS_READY(i, ignore_laggy)) + j++; + if (j > n) + break; + } + + return i; +} + +/* + * choose a random mds that is "up" (i.e. has a state > 0), or -1. + */ +int ceph_mdsmap_get_random_mds(struct ceph_mdsmap *m) +{ + int mds; + + mds = __mdsmap_get_random_mds(m, false); + if (mds == m->possible_max_rank || mds == -1) + mds = __mdsmap_get_random_mds(m, true); + + return mds == m->possible_max_rank ? -1 : mds; +} + +#define __decode_and_drop_type(p, end, type, bad) \ + do { \ + if (*p + sizeof(type) > end) \ + goto bad; \ + *p += sizeof(type); \ + } while (0) + +#define __decode_and_drop_set(p, end, type, bad) \ + do { \ + u32 n; \ + size_t need; \ + ceph_decode_32_safe(p, end, n, bad); \ + need = sizeof(type) * n; \ + ceph_decode_need(p, end, need, bad); \ + *p += need; \ + } while (0) + +#define __decode_and_drop_map(p, end, ktype, vtype, bad) \ + do { \ + u32 n; \ + size_t need; \ + ceph_decode_32_safe(p, end, n, bad); \ + need = (sizeof(ktype) + sizeof(vtype)) * n; \ + ceph_decode_need(p, end, need, bad); \ + *p += need; \ + } while (0) + + +static int __decode_and_drop_compat_set(void **p, void* end) +{ + int i; + /* compat, ro_compat, incompat*/ + for (i = 0; i < 3; i++) { + u32 n; + ceph_decode_need(p, end, sizeof(u64) + sizeof(u32), bad); + /* mask */ + *p += sizeof(u64); + /* names (map) */ + n = ceph_decode_32(p); + while (n-- > 0) { + u32 len; + ceph_decode_need(p, end, sizeof(u64) + sizeof(u32), + bad); + *p += sizeof(u64); + len = ceph_decode_32(p); + ceph_decode_need(p, end, len, bad); + *p += len; + } + } + return 0; +bad: + return -1; +} + +/* + * Decode an MDS map + * + * Ignore any fields we don't care about (there are quite a few of + * them). + */ +struct ceph_mdsmap *ceph_mdsmap_decode(struct ceph_mds_client *mdsc, void **p, + void *end, bool msgr2) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_mdsmap *m; + const void *start = *p; + int i, j, n; + int err; + u8 mdsmap_v; + u16 mdsmap_ev; + u32 target; + + m = kzalloc_obj(*m, GFP_NOFS); + if (!m) + return ERR_PTR(-ENOMEM); + + ceph_decode_need(p, end, 1 + 1, bad); + mdsmap_v = ceph_decode_8(p); + *p += sizeof(u8); /* mdsmap_cv */ + if (mdsmap_v >= 4) { + u32 mdsmap_len; + ceph_decode_32_safe(p, end, mdsmap_len, bad); + if (end < *p + mdsmap_len) + goto bad; + end = *p + mdsmap_len; + } + + ceph_decode_need(p, end, 8*sizeof(u32) + sizeof(u64), bad); + m->m_epoch = ceph_decode_32(p); + m->m_client_epoch = ceph_decode_32(p); + m->m_last_failure = ceph_decode_32(p); + m->m_root = ceph_decode_32(p); + m->m_session_timeout = ceph_decode_32(p); + m->m_session_autoclose = ceph_decode_32(p); + m->m_max_file_size = ceph_decode_64(p); + m->m_max_mds = ceph_decode_32(p); + + /* + * pick out the active nodes as the m_num_active_mds, the + * m_num_active_mds maybe larger than m_max_mds when decreasing + * the max_mds in cluster side, in other case it should less + * than or equal to m_max_mds. + */ + m->m_num_active_mds = n = ceph_decode_32(p); + + /* + * the possible max rank, it maybe larger than the m_num_active_mds, + * for example if the mds_max == 2 in the cluster, when the MDS(0) + * was laggy and being replaced by a new MDS, we will temporarily + * receive a new mds map with n_num_mds == 1 and the active MDS(1), + * and the mds rank >= m_num_active_mds. + */ + m->possible_max_rank = max(m->m_num_active_mds, m->m_max_mds); + + m->m_info = kzalloc_objs(*m->m_info, m->possible_max_rank, GFP_NOFS); + if (!m->m_info) + goto nomem; + + /* pick out active nodes from mds_info (state > 0) */ + for (i = 0; i < n; i++) { + u64 global_id; + u32 namelen; + s32 mds, inc, state; + u8 info_v; + void *info_end = NULL; + struct ceph_entity_addr addr; + u32 num_export_targets; + void *pexport_targets = NULL; + struct ceph_timespec laggy_since; + struct ceph_mds_info *info; + bool laggy; + + ceph_decode_need(p, end, sizeof(u64) + 1, bad); + global_id = ceph_decode_64(p); + info_v= ceph_decode_8(p); + if (info_v >= 4) { + u32 info_len; + ceph_decode_need(p, end, 1 + sizeof(u32), bad); + *p += sizeof(u8); /* info_cv */ + info_len = ceph_decode_32(p); + info_end = *p + info_len; + if (info_end > end) + goto bad; + } + + ceph_decode_need(p, end, sizeof(u64) + sizeof(u32), bad); + *p += sizeof(u64); + namelen = ceph_decode_32(p); /* skip mds name */ + *p += namelen; + + ceph_decode_32_safe(p, end, mds, bad); + ceph_decode_32_safe(p, end, inc, bad); + ceph_decode_32_safe(p, end, state, bad); + *p += sizeof(u64); /* state_seq */ + if (info_v >= 8) + err = ceph_decode_entity_addrvec(p, end, msgr2, &addr); + else + err = ceph_decode_entity_addr(p, end, &addr); + if (err) + goto corrupt; + + ceph_decode_copy_safe(p, end, &laggy_since, sizeof(laggy_since), + bad); + laggy = laggy_since.tv_sec != 0 || laggy_since.tv_nsec != 0; + *p += sizeof(u32); + ceph_decode_32_safe(p, end, namelen, bad); + *p += namelen; + if (info_v >= 2) { + ceph_decode_32_safe(p, end, num_export_targets, bad); + pexport_targets = *p; + *p += num_export_targets * sizeof(u32); + } else { + num_export_targets = 0; + } + + if (info_end && *p != info_end) { + if (*p > info_end) + goto bad; + *p = info_end; + } + + doutc(cl, "%d/%d %lld mds%d.%d %s %s%s\n", i+1, n, global_id, + mds, inc, ceph_pr_addr(&addr), + ceph_mds_state_name(state), laggy ? "(laggy)" : ""); + + if (mds < 0 || mds >= m->possible_max_rank) { + pr_warn_client(cl, "got incorrect mds(%d)\n", mds); + continue; + } + + if (state <= 0) { + doutc(cl, "got incorrect state(%s)\n", + ceph_mds_state_name(state)); + continue; + } + + info = &m->m_info[mds]; + info->global_id = global_id; + info->state = state; + info->addr = addr; + info->laggy = laggy; + info->num_export_targets = num_export_targets; + if (num_export_targets) { + info->export_targets = kcalloc(num_export_targets, + sizeof(u32), GFP_NOFS); + if (!info->export_targets) + goto nomem; + for (j = 0; j < num_export_targets; j++) { + target = ceph_decode_32(&pexport_targets); + info->export_targets[j] = target; + } + } else { + info->export_targets = NULL; + } + } + + /* pg_pools */ + ceph_decode_32_safe(p, end, n, bad); + m->m_num_data_pg_pools = n; + m->m_data_pg_pools = kcalloc(n, sizeof(u64), GFP_NOFS); + if (!m->m_data_pg_pools) + goto nomem; + ceph_decode_need(p, end, sizeof(u64)*(n+1), bad); + for (i = 0; i < n; i++) + m->m_data_pg_pools[i] = ceph_decode_64(p); + m->m_cas_pg_pool = ceph_decode_64(p); + m->m_enabled = m->m_epoch > 1; + + mdsmap_ev = 1; + if (mdsmap_v >= 2) { + ceph_decode_16_safe(p, end, mdsmap_ev, bad_ext); + } + if (mdsmap_ev >= 3) { + if (__decode_and_drop_compat_set(p, end) < 0) + goto bad_ext; + } + /* metadata_pool */ + if (mdsmap_ev < 5) { + __decode_and_drop_type(p, end, u32, bad_ext); + } else { + __decode_and_drop_type(p, end, u64, bad_ext); + } + + /* created + modified + tableserver */ + __decode_and_drop_type(p, end, struct ceph_timespec, bad_ext); + __decode_and_drop_type(p, end, struct ceph_timespec, bad_ext); + __decode_and_drop_type(p, end, u32, bad_ext); + + /* in */ + { + int num_laggy = 0; + ceph_decode_32_safe(p, end, n, bad_ext); + ceph_decode_need(p, end, sizeof(u32) * n, bad_ext); + + for (i = 0; i < n; i++) { + s32 mds = ceph_decode_32(p); + if (mds >= 0 && mds < m->possible_max_rank) { + if (m->m_info[mds].laggy) + num_laggy++; + } + } + m->m_num_laggy = num_laggy; + + if (n > m->possible_max_rank) { + void *new_m_info = krealloc(m->m_info, + n * sizeof(*m->m_info), + GFP_NOFS | __GFP_ZERO); + if (!new_m_info) + goto nomem; + m->m_info = new_m_info; + } + m->possible_max_rank = n; + } + + /* inc */ + __decode_and_drop_map(p, end, u32, u32, bad_ext); + /* up */ + __decode_and_drop_map(p, end, u32, u64, bad_ext); + /* failed */ + __decode_and_drop_set(p, end, u32, bad_ext); + /* stopped */ + __decode_and_drop_set(p, end, u32, bad_ext); + + if (mdsmap_ev >= 4) { + /* last_failure_osd_epoch */ + __decode_and_drop_type(p, end, u32, bad_ext); + } + if (mdsmap_ev >= 6) { + /* ever_allowed_snaps */ + __decode_and_drop_type(p, end, u8, bad_ext); + /* explicitly_allowed_snaps */ + __decode_and_drop_type(p, end, u8, bad_ext); + } + if (mdsmap_ev >= 7) { + /* inline_data_enabled */ + __decode_and_drop_type(p, end, u8, bad_ext); + } + if (mdsmap_ev >= 8) { + size_t fsname_len; + + /* enabled */ + ceph_decode_8_safe(p, end, m->m_enabled, bad_ext); + + /* fs_name */ + m->m_fs_name = ceph_extract_encoded_string(p, end, + &fsname_len, + GFP_NOFS); + if (IS_ERR(m->m_fs_name)) { + m->m_fs_name = NULL; + goto nomem; + } + + /* validate fsname against mds_namespace */ + if (!namespace_equals(mdsc->fsc->mount_options, m->m_fs_name, + fsname_len)) { + pr_warn_client(cl, "fsname %s doesn't match mds_namespace %s\n", + m->m_fs_name, + mdsc->fsc->mount_options->mds_namespace); + goto bad; + } + } else { + m->m_enabled = false; + m->m_fs_name = kstrdup(CEPH_OLD_FS_NAME, GFP_NOFS); + if (!m->m_fs_name) + goto nomem; + } + /* damaged */ + if (mdsmap_ev >= 9) { + size_t need; + ceph_decode_32_safe(p, end, n, bad_ext); + need = sizeof(u32) * n; + ceph_decode_need(p, end, need, bad_ext); + *p += need; + m->m_damaged = n > 0; + } else { + m->m_damaged = false; + } + if (mdsmap_ev >= 17) { + /* balancer */ + ceph_decode_skip_string(p, end, bad_ext); + /* standby_count_wanted */ + ceph_decode_skip_32(p, end, bad_ext); + /* old_max_mds */ + ceph_decode_skip_32(p, end, bad_ext); + /* min_compat_client */ + ceph_decode_skip_8(p, end, bad_ext); + /* required_client_features */ + ceph_decode_skip_set(p, end, 64, bad_ext); + /* bal_rank_mask */ + ceph_decode_skip_string(p, end, bad_ext); + } + if (mdsmap_ev >= 18) { + ceph_decode_64_safe(p, end, m->m_max_xattr_size, bad_ext); + } +bad_ext: + doutc(cl, "m_enabled: %d, m_damaged: %d, m_num_laggy: %d\n", + !!m->m_enabled, !!m->m_damaged, m->m_num_laggy); + *p = end; + doutc(cl, "success epoch %u\n", m->m_epoch); + return m; +nomem: + err = -ENOMEM; + goto out_err; +corrupt: + pr_err_client(cl, "corrupt mdsmap\n"); + print_hex_dump(KERN_DEBUG, "mdsmap: ", + DUMP_PREFIX_OFFSET, 16, 1, + start, end - start, true); +out_err: + ceph_mdsmap_destroy(m); + return ERR_PTR(err); +bad: + err = -EINVAL; + goto corrupt; +} + +void ceph_mdsmap_destroy(struct ceph_mdsmap *m) +{ + int i; + + if (m->m_info) { + for (i = 0; i < m->possible_max_rank; i++) + kfree(m->m_info[i].export_targets); + kfree(m->m_info); + } + kfree(m->m_data_pg_pools); + kfree(m->m_fs_name); + kfree(m); +} + +bool ceph_mdsmap_is_cluster_available(struct ceph_mdsmap *m) +{ + int i, nr_active = 0; + if (!m->m_enabled) + return false; + if (m->m_damaged) + return false; + if (m->m_num_laggy == m->m_num_active_mds) + return false; + for (i = 0; i < m->possible_max_rank; i++) { + if (m->m_info[i].state == CEPH_MDS_STATE_ACTIVE) + nr_active++; + } + return nr_active > 0; +} diff --git a/fs/ceph/mdsmap.h b/fs/ceph/mdsmap.h new file mode 100644 index 000000000..d48d07c35 --- /dev/null +++ b/fs/ceph/mdsmap.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _FS_CEPH_MDSMAP_H +#define _FS_CEPH_MDSMAP_H + +#include +#include + +struct ceph_mds_client; + +/* + * mds map - describe servers in the mds cluster. + * + * we limit fields to those the client actually xcares about + */ +struct ceph_mds_info { + u64 global_id; + struct ceph_entity_addr addr; + s32 state; + int num_export_targets; + bool laggy; + u32 *export_targets; +}; + +struct ceph_mdsmap { + u32 m_epoch, m_client_epoch, m_last_failure; + u32 m_root; + u32 m_session_timeout; /* seconds */ + u32 m_session_autoclose; /* seconds */ + u64 m_max_file_size; + /* + * maximum size for xattrs blob. + * Zeroed by default to force the usage of the (sync) SETXATTR Op. + */ + u64 m_max_xattr_size; + u32 m_max_mds; /* expected up:active mds number */ + u32 m_num_active_mds; /* actual up:active mds number */ + u32 possible_max_rank; /* possible max rank index */ + struct ceph_mds_info *m_info; + + /* which object pools file data can be stored in */ + int m_num_data_pg_pools; + u64 *m_data_pg_pools; + u64 m_cas_pg_pool; + + bool m_enabled; + bool m_damaged; + int m_num_laggy; + char *m_fs_name; +}; + +static inline struct ceph_entity_addr * +ceph_mdsmap_get_addr(struct ceph_mdsmap *m, int w) +{ + if (w >= m->possible_max_rank) + return NULL; + return &m->m_info[w].addr; +} + +static inline int ceph_mdsmap_get_state(struct ceph_mdsmap *m, int w) +{ + BUG_ON(w < 0); + if (w >= m->possible_max_rank) + return CEPH_MDS_STATE_DNE; + return m->m_info[w].state; +} + +static inline bool ceph_mdsmap_is_laggy(struct ceph_mdsmap *m, int w) +{ + if (w >= 0 && w < m->possible_max_rank) + return m->m_info[w].laggy; + return false; +} + +extern int ceph_mdsmap_get_random_mds(struct ceph_mdsmap *m); +struct ceph_mdsmap *ceph_mdsmap_decode(struct ceph_mds_client *mdsc, void **p, + void *end, bool msgr2); +extern void ceph_mdsmap_destroy(struct ceph_mdsmap *m); +extern bool ceph_mdsmap_is_cluster_available(struct ceph_mdsmap *m); + +#endif diff --git a/fs/ceph/metric.c b/fs/ceph/metric.c new file mode 100644 index 000000000..b6450fdac --- /dev/null +++ b/fs/ceph/metric.c @@ -0,0 +1,533 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include + +#include +#include +#include +#include + +#include + +#include "metric.h" +#include "mds_client.h" + +static bool metrics_disable_warned; + +static inline u32 ceph_subvolume_entry_payload_len(void) +{ + return sizeof(struct ceph_subvolume_metric_entry_wire); +} + +static inline u32 ceph_subvolume_entry_encoded_len(void) +{ + return CEPH_ENCODING_START_BLK_LEN + + ceph_subvolume_entry_payload_len(); +} + +static inline u32 ceph_subvolume_outer_payload_len(u32 nr_subvols) +{ + /* count is encoded as le64 (size_t on wire) to match FUSE client */ + return sizeof(__le64) + + nr_subvols * ceph_subvolume_entry_encoded_len(); +} + +static inline u32 ceph_subvolume_metric_data_len(u32 nr_subvols) +{ + return CEPH_ENCODING_START_BLK_LEN + + ceph_subvolume_outer_payload_len(nr_subvols); +} + +static inline u32 ceph_subvolume_clamp_u32(u64 val) +{ + return val > U32_MAX ? U32_MAX : (u32)val; +} + +static void ceph_init_subvolume_wire_entry( + struct ceph_subvolume_metric_entry_wire *dst, + const struct ceph_subvol_metric_snapshot *src) +{ + dst->subvolume_id = cpu_to_le64(src->subvolume_id); + dst->read_ops = cpu_to_le32(ceph_subvolume_clamp_u32(src->read_ops)); + dst->write_ops = cpu_to_le32(ceph_subvolume_clamp_u32(src->write_ops)); + dst->read_bytes = cpu_to_le64(src->read_bytes); + dst->write_bytes = cpu_to_le64(src->write_bytes); + dst->read_latency_us = cpu_to_le64(src->read_latency_us); + dst->write_latency_us = cpu_to_le64(src->write_latency_us); + dst->time_stamp = 0; +} + +static int ceph_encode_subvolume_metrics(void **p, void *end, + struct ceph_subvol_metric_snapshot *subvols, + u32 nr_subvols) +{ + u32 i; + + ceph_start_encoding(p, 1, 1, + ceph_subvolume_outer_payload_len(nr_subvols)); + /* count is encoded as le64 (size_t on wire) to match FUSE client */ + ceph_encode_64_safe(p, end, (u64)nr_subvols, enc_err); + + for (i = 0; i < nr_subvols; i++) { + struct ceph_subvolume_metric_entry_wire wire_entry; + + ceph_init_subvolume_wire_entry(&wire_entry, &subvols[i]); + ceph_start_encoding(p, 1, 1, + ceph_subvolume_entry_payload_len()); + ceph_encode_copy_safe(p, end, &wire_entry, + sizeof(wire_entry), enc_err); + } + + return 0; +enc_err: + return -ERANGE; +} + +static void ktime_to_ceph_timespec(struct ceph_timespec *ts, ktime_t val) +{ + struct timespec64 t = ktime_to_timespec64(val); + ceph_encode_timespec64(ts, &t); +} + +static bool ceph_mdsc_send_metrics(struct ceph_mds_client *mdsc, + struct ceph_mds_session *s) +{ + struct ceph_metric_head *head; + struct ceph_metric_cap *cap; + struct ceph_metric_read_latency *read; + struct ceph_metric_write_latency *write; + struct ceph_metric_metadata_latency *meta; + struct ceph_metric_dlease *dlease; + struct ceph_opened_files *files; + struct ceph_pinned_icaps *icaps; + struct ceph_opened_inodes *inodes; + struct ceph_read_io_size *rsize; + struct ceph_write_io_size *wsize; + struct ceph_client_metric *m = &mdsc->metric; + struct ceph_subvol_metric_snapshot *subvols = NULL; + u64 nr_caps = atomic64_read(&m->total_caps); + u32 header_len = sizeof(struct ceph_metric_header); + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_msg *msg; + u32 nr_subvols = 0; + size_t subvol_len = 0; + void *cursor; + s64 sum; + s32 items = 0; + s32 len; + + /* Do not send the metrics until the MDS rank is ready */ + mutex_lock(&mdsc->mutex); + if (ceph_mdsmap_get_state(mdsc->mdsmap, s->s_mds) != CEPH_MDS_STATE_ACTIVE) { + mutex_unlock(&mdsc->mutex); + return false; + } + mutex_unlock(&mdsc->mutex); + + if (ceph_subvolume_metrics_enabled(&mdsc->subvol_metrics) && + test_bit(CEPHFS_FEATURE_SUBVOLUME_METRICS, &s->s_features)) { + int ret; + + ret = ceph_subvolume_metrics_snapshot(&mdsc->subvol_metrics, + &subvols, &nr_subvols, + true); + if (ret) { + pr_warn_client(cl, "failed to snapshot subvolume metrics: %d\n", + ret); + /* + * On error, ceph_subvolume_metrics_snapshot() guarantees + * *out = NULL and *nr = 0 at function entry, so subvols + * is already NULL here - no cleanup needed. + */ + nr_subvols = 0; + subvols = NULL; + } + } + + if (nr_subvols) { + /* type (le32) + ENCODE_START payload - no metric header */ + subvol_len = sizeof(__le32) + + ceph_subvolume_metric_data_len(nr_subvols); + } + + len = sizeof(*head) + sizeof(*cap) + sizeof(*read) + sizeof(*write) + + sizeof(*meta) + sizeof(*dlease) + sizeof(*files) + + sizeof(*icaps) + sizeof(*inodes) + sizeof(*rsize) + + sizeof(*wsize) + subvol_len; + + msg = ceph_msg_new(CEPH_MSG_CLIENT_METRICS, len, GFP_NOFS, true); + if (!msg) { + pr_err_client(cl, "to mds%d, failed to allocate message\n", + s->s_mds); + kfree(subvols); + return false; + } + + head = msg->front.iov_base; + + /* encode the cap metric */ + cap = (struct ceph_metric_cap *)(head + 1); + cap->header.type = cpu_to_le32(CLIENT_METRIC_TYPE_CAP_INFO); + cap->header.ver = 1; + cap->header.compat = 1; + cap->header.data_len = cpu_to_le32(sizeof(*cap) - header_len); + cap->hit = cpu_to_le64(percpu_counter_sum(&m->i_caps_hit)); + cap->mis = cpu_to_le64(percpu_counter_sum(&m->i_caps_mis)); + cap->total = cpu_to_le64(nr_caps); + items++; + + /* encode the read latency metric */ + read = (struct ceph_metric_read_latency *)(cap + 1); + read->header.type = cpu_to_le32(CLIENT_METRIC_TYPE_READ_LATENCY); + read->header.ver = 2; + read->header.compat = 1; + read->header.data_len = cpu_to_le32(sizeof(*read) - header_len); + sum = m->metric[METRIC_READ].latency_sum; + ktime_to_ceph_timespec(&read->lat, sum); + ktime_to_ceph_timespec(&read->avg, m->metric[METRIC_READ].latency_avg); + read->sq_sum = cpu_to_le64(m->metric[METRIC_READ].latency_sq_sum); + read->count = cpu_to_le64(m->metric[METRIC_READ].total); + items++; + + /* encode the write latency metric */ + write = (struct ceph_metric_write_latency *)(read + 1); + write->header.type = cpu_to_le32(CLIENT_METRIC_TYPE_WRITE_LATENCY); + write->header.ver = 2; + write->header.compat = 1; + write->header.data_len = cpu_to_le32(sizeof(*write) - header_len); + sum = m->metric[METRIC_WRITE].latency_sum; + ktime_to_ceph_timespec(&write->lat, sum); + ktime_to_ceph_timespec(&write->avg, m->metric[METRIC_WRITE].latency_avg); + write->sq_sum = cpu_to_le64(m->metric[METRIC_WRITE].latency_sq_sum); + write->count = cpu_to_le64(m->metric[METRIC_WRITE].total); + items++; + + /* encode the metadata latency metric */ + meta = (struct ceph_metric_metadata_latency *)(write + 1); + meta->header.type = cpu_to_le32(CLIENT_METRIC_TYPE_METADATA_LATENCY); + meta->header.ver = 2; + meta->header.compat = 1; + meta->header.data_len = cpu_to_le32(sizeof(*meta) - header_len); + sum = m->metric[METRIC_METADATA].latency_sum; + ktime_to_ceph_timespec(&meta->lat, sum); + ktime_to_ceph_timespec(&meta->avg, m->metric[METRIC_METADATA].latency_avg); + meta->sq_sum = cpu_to_le64(m->metric[METRIC_METADATA].latency_sq_sum); + meta->count = cpu_to_le64(m->metric[METRIC_METADATA].total); + items++; + + /* encode the dentry lease metric */ + dlease = (struct ceph_metric_dlease *)(meta + 1); + dlease->header.type = cpu_to_le32(CLIENT_METRIC_TYPE_DENTRY_LEASE); + dlease->header.ver = 1; + dlease->header.compat = 1; + dlease->header.data_len = cpu_to_le32(sizeof(*dlease) - header_len); + dlease->hit = cpu_to_le64(percpu_counter_sum(&m->d_lease_hit)); + dlease->mis = cpu_to_le64(percpu_counter_sum(&m->d_lease_mis)); + dlease->total = cpu_to_le64(atomic64_read(&m->total_dentries)); + items++; + + sum = percpu_counter_sum(&m->total_inodes); + + /* encode the opened files metric */ + files = (struct ceph_opened_files *)(dlease + 1); + files->header.type = cpu_to_le32(CLIENT_METRIC_TYPE_OPENED_FILES); + files->header.ver = 1; + files->header.compat = 1; + files->header.data_len = cpu_to_le32(sizeof(*files) - header_len); + files->opened_files = cpu_to_le64(atomic64_read(&m->opened_files)); + files->total = cpu_to_le64(sum); + items++; + + /* encode the pinned icaps metric */ + icaps = (struct ceph_pinned_icaps *)(files + 1); + icaps->header.type = cpu_to_le32(CLIENT_METRIC_TYPE_PINNED_ICAPS); + icaps->header.ver = 1; + icaps->header.compat = 1; + icaps->header.data_len = cpu_to_le32(sizeof(*icaps) - header_len); + icaps->pinned_icaps = cpu_to_le64(nr_caps); + icaps->total = cpu_to_le64(sum); + items++; + + /* encode the opened inodes metric */ + inodes = (struct ceph_opened_inodes *)(icaps + 1); + inodes->header.type = cpu_to_le32(CLIENT_METRIC_TYPE_OPENED_INODES); + inodes->header.ver = 1; + inodes->header.compat = 1; + inodes->header.data_len = cpu_to_le32(sizeof(*inodes) - header_len); + inodes->opened_inodes = cpu_to_le64(percpu_counter_sum(&m->opened_inodes)); + inodes->total = cpu_to_le64(sum); + items++; + + /* encode the read io size metric */ + rsize = (struct ceph_read_io_size *)(inodes + 1); + rsize->header.type = cpu_to_le32(CLIENT_METRIC_TYPE_READ_IO_SIZES); + rsize->header.ver = 1; + rsize->header.compat = 1; + rsize->header.data_len = cpu_to_le32(sizeof(*rsize) - header_len); + rsize->total_ops = cpu_to_le64(m->metric[METRIC_READ].total); + rsize->total_size = cpu_to_le64(m->metric[METRIC_READ].size_sum); + items++; + + /* encode the write io size metric */ + wsize = (struct ceph_write_io_size *)(rsize + 1); + wsize->header.type = cpu_to_le32(CLIENT_METRIC_TYPE_WRITE_IO_SIZES); + wsize->header.ver = 1; + wsize->header.compat = 1; + wsize->header.data_len = cpu_to_le32(sizeof(*wsize) - header_len); + wsize->total_ops = cpu_to_le64(m->metric[METRIC_WRITE].total); + wsize->total_size = cpu_to_le64(m->metric[METRIC_WRITE].size_sum); + items++; + + cursor = wsize + 1; + + if (nr_subvols) { + void *payload; + void *payload_end; + int ret; + + /* Emit only the type (le32), no ver/compat/data_len */ + ceph_encode_32(&cursor, CLIENT_METRIC_TYPE_SUBVOLUME_METRICS); + items++; + + payload = cursor; + payload_end = (char *)payload + + ceph_subvolume_metric_data_len(nr_subvols); + + ret = ceph_encode_subvolume_metrics(&payload, payload_end, + subvols, nr_subvols); + if (ret) { + pr_warn_client(cl, + "failed to encode subvolume metrics\n"); + kfree(subvols); + ceph_msg_put(msg); + return false; + } + + WARN_ON(payload != payload_end); + cursor = payload; + } + + put_unaligned_le32(items, &head->num); + msg->front.iov_len = (char *)cursor - (char *)head; + msg->hdr.version = cpu_to_le16(1); + msg->hdr.compat_version = cpu_to_le16(1); + msg->hdr.front_len = cpu_to_le32(msg->front.iov_len); + + ceph_con_send(&s->s_con, msg); + + if (nr_subvols) { + mutex_lock(&mdsc->subvol_metrics_last_mutex); + kfree(mdsc->subvol_metrics_last); + mdsc->subvol_metrics_last = subvols; + mdsc->subvol_metrics_last_nr = nr_subvols; + mdsc->subvol_metrics_sent += nr_subvols; + mdsc->subvol_metrics_nonzero_sends++; + mutex_unlock(&mdsc->subvol_metrics_last_mutex); + + subvols = NULL; + } + kfree(subvols); + + return true; +} + + +static void metric_get_session(struct ceph_mds_client *mdsc) +{ + struct ceph_mds_session *s; + int i; + + mutex_lock(&mdsc->mutex); + for (i = 0; i < mdsc->max_sessions; i++) { + s = __ceph_lookup_mds_session(mdsc, i); + if (!s) + continue; + + /* + * Skip it if MDS doesn't support the metric collection, + * or the MDS will close the session's socket connection + * directly when it get this message. + * + * Also skip sessions that don't support SUBVOLUME_METRICS + * when subvolume metrics collection is enabled. This ensures + * we only send subvolume metrics to MDSs that understand them. + * If no session supports the feature, metrics won't be sent. + */ + if (check_session_state(s) && + test_bit(CEPHFS_FEATURE_METRIC_COLLECT, &s->s_features)) { + if (ceph_subvolume_metrics_enabled(&mdsc->subvol_metrics) && + !test_bit(CEPHFS_FEATURE_SUBVOLUME_METRICS, + &s->s_features)) { + ceph_put_mds_session(s); + continue; + } + mdsc->metric.session = s; + break; + } + + ceph_put_mds_session(s); + } + mutex_unlock(&mdsc->mutex); +} + +static void metric_delayed_work(struct work_struct *work) +{ + struct ceph_client_metric *m = + container_of(work, struct ceph_client_metric, delayed_work.work); + struct ceph_mds_client *mdsc = + container_of(m, struct ceph_mds_client, metric); + + if (mdsc->stopping) + return; + + if (disable_send_metrics) { + if (!metrics_disable_warned) { + pr_info("ceph: metrics sending disabled via module parameter\n"); + metrics_disable_warned = true; + } + return; + } + metrics_disable_warned = false; + + if (!m->session || !check_session_state(m->session)) { + if (m->session) { + ceph_put_mds_session(m->session); + m->session = NULL; + } + metric_get_session(mdsc); + } + + if (m->session) + ceph_mdsc_send_metrics(mdsc, m->session); + else + pr_warn_ratelimited("ceph: metrics worker has no MDS session\n"); + + metric_schedule_delayed(m); +} + +int ceph_metric_init(struct ceph_client_metric *m) +{ + struct ceph_metric *metric; + int ret, i; + + if (!m) + return -EINVAL; + + atomic64_set(&m->total_dentries, 0); + ret = percpu_counter_init(&m->d_lease_hit, 0, GFP_KERNEL); + if (ret) + return ret; + + ret = percpu_counter_init(&m->d_lease_mis, 0, GFP_KERNEL); + if (ret) + goto err_d_lease_mis; + + atomic64_set(&m->total_caps, 0); + ret = percpu_counter_init(&m->i_caps_hit, 0, GFP_KERNEL); + if (ret) + goto err_i_caps_hit; + + ret = percpu_counter_init(&m->i_caps_mis, 0, GFP_KERNEL); + if (ret) + goto err_i_caps_mis; + + for (i = 0; i < METRIC_MAX; i++) { + metric = &m->metric[i]; + spin_lock_init(&metric->lock); + metric->size_sum = 0; + metric->size_min = U64_MAX; + metric->size_max = 0; + metric->total = 0; + metric->latency_sum = 0; + metric->latency_avg = 0; + metric->latency_sq_sum = 0; + metric->latency_min = KTIME_MAX; + metric->latency_max = 0; + } + + atomic64_set(&m->opened_files, 0); + ret = percpu_counter_init(&m->opened_inodes, 0, GFP_KERNEL); + if (ret) + goto err_opened_inodes; + ret = percpu_counter_init(&m->total_inodes, 0, GFP_KERNEL); + if (ret) + goto err_total_inodes; + + m->session = NULL; + INIT_DELAYED_WORK(&m->delayed_work, metric_delayed_work); + + return 0; + +err_total_inodes: + percpu_counter_destroy(&m->opened_inodes); +err_opened_inodes: + percpu_counter_destroy(&m->i_caps_mis); +err_i_caps_mis: + percpu_counter_destroy(&m->i_caps_hit); +err_i_caps_hit: + percpu_counter_destroy(&m->d_lease_mis); +err_d_lease_mis: + percpu_counter_destroy(&m->d_lease_hit); + + return ret; +} + +void ceph_metric_destroy(struct ceph_client_metric *m) +{ + if (!m) + return; + + cancel_delayed_work_sync(&m->delayed_work); + + percpu_counter_destroy(&m->total_inodes); + percpu_counter_destroy(&m->opened_inodes); + percpu_counter_destroy(&m->i_caps_mis); + percpu_counter_destroy(&m->i_caps_hit); + percpu_counter_destroy(&m->d_lease_mis); + percpu_counter_destroy(&m->d_lease_hit); + + ceph_put_mds_session(m->session); +} + +#define METRIC_UPDATE_MIN_MAX(min, max, new) \ +{ \ + if (unlikely(new < min)) \ + min = new; \ + if (unlikely(new > max)) \ + max = new; \ +} + +static inline void __update_mean_and_stdev(ktime_t total, ktime_t *lavg, + ktime_t *sq_sump, ktime_t lat) +{ + ktime_t avg; + + if (unlikely(total == 1)) { + *lavg = lat; + } else { + /* the sq is (lat - old_avg) * (lat - new_avg) */ + avg = *lavg + div64_s64(lat - *lavg, total); + *sq_sump += (lat - *lavg)*(lat - avg); + *lavg = avg; + } +} + +void ceph_update_metrics(struct ceph_metric *m, + ktime_t r_start, ktime_t r_end, + unsigned int size, int rc) +{ + ktime_t lat = ktime_sub(r_end, r_start); + ktime_t total; + + if (unlikely(rc < 0 && rc != -ENOENT && rc != -ETIMEDOUT)) + return; + + spin_lock(&m->lock); + total = ++m->total; + m->size_sum += size; + METRIC_UPDATE_MIN_MAX(m->size_min, m->size_max, size); + m->latency_sum += lat; + METRIC_UPDATE_MIN_MAX(m->latency_min, m->latency_max, lat); + __update_mean_and_stdev(total, &m->latency_avg, &m->latency_sq_sum, + lat); + spin_unlock(&m->lock); +} diff --git a/fs/ceph/metric.h b/fs/ceph/metric.h new file mode 100644 index 000000000..519cd4d47 --- /dev/null +++ b/fs/ceph/metric.h @@ -0,0 +1,281 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _FS_CEPH_MDS_METRIC_H +#define _FS_CEPH_MDS_METRIC_H + +#include +#include +#include + +extern bool disable_send_metrics; + +enum ceph_metric_type { + CLIENT_METRIC_TYPE_CAP_INFO, + CLIENT_METRIC_TYPE_READ_LATENCY, + CLIENT_METRIC_TYPE_WRITE_LATENCY, + CLIENT_METRIC_TYPE_METADATA_LATENCY, + CLIENT_METRIC_TYPE_DENTRY_LEASE, + CLIENT_METRIC_TYPE_OPENED_FILES, + CLIENT_METRIC_TYPE_PINNED_ICAPS, + CLIENT_METRIC_TYPE_OPENED_INODES, + CLIENT_METRIC_TYPE_READ_IO_SIZES, + CLIENT_METRIC_TYPE_WRITE_IO_SIZES, + CLIENT_METRIC_TYPE_AVG_READ_LATENCY, + CLIENT_METRIC_TYPE_STDEV_READ_LATENCY, + CLIENT_METRIC_TYPE_AVG_WRITE_LATENCY, + CLIENT_METRIC_TYPE_STDEV_WRITE_LATENCY, + CLIENT_METRIC_TYPE_AVG_METADATA_LATENCY, + CLIENT_METRIC_TYPE_STDEV_METADATA_LATENCY, + CLIENT_METRIC_TYPE_SUBVOLUME_METRICS, + + CLIENT_METRIC_TYPE_MAX = CLIENT_METRIC_TYPE_SUBVOLUME_METRICS, +}; + +/* + * This will always have the highest metric bit value + * as the last element of the array. + */ +#define CEPHFS_METRIC_SPEC_CLIENT_SUPPORTED { \ + CLIENT_METRIC_TYPE_CAP_INFO, \ + CLIENT_METRIC_TYPE_READ_LATENCY, \ + CLIENT_METRIC_TYPE_WRITE_LATENCY, \ + CLIENT_METRIC_TYPE_METADATA_LATENCY, \ + CLIENT_METRIC_TYPE_DENTRY_LEASE, \ + CLIENT_METRIC_TYPE_OPENED_FILES, \ + CLIENT_METRIC_TYPE_PINNED_ICAPS, \ + CLIENT_METRIC_TYPE_OPENED_INODES, \ + CLIENT_METRIC_TYPE_READ_IO_SIZES, \ + CLIENT_METRIC_TYPE_WRITE_IO_SIZES, \ + CLIENT_METRIC_TYPE_AVG_READ_LATENCY, \ + CLIENT_METRIC_TYPE_STDEV_READ_LATENCY, \ + CLIENT_METRIC_TYPE_AVG_WRITE_LATENCY, \ + CLIENT_METRIC_TYPE_STDEV_WRITE_LATENCY, \ + CLIENT_METRIC_TYPE_AVG_METADATA_LATENCY, \ + CLIENT_METRIC_TYPE_STDEV_METADATA_LATENCY, \ + CLIENT_METRIC_TYPE_SUBVOLUME_METRICS, \ + \ + CLIENT_METRIC_TYPE_MAX, \ +} + +struct ceph_metric_header { + __le32 type; /* ceph metric type */ + __u8 ver; + __u8 compat; + __le32 data_len; /* length of sizeof(hit + mis + total) */ +} __packed; + +/* metric caps header */ +struct ceph_metric_cap { + struct ceph_metric_header header; + __le64 hit; + __le64 mis; + __le64 total; +} __packed; + +/* metric read latency header */ +struct ceph_metric_read_latency { + struct ceph_metric_header header; + struct ceph_timespec lat; + struct ceph_timespec avg; + __le64 sq_sum; + __le64 count; +} __packed; + +/* metric write latency header */ +struct ceph_metric_write_latency { + struct ceph_metric_header header; + struct ceph_timespec lat; + struct ceph_timespec avg; + __le64 sq_sum; + __le64 count; +} __packed; + +/* metric metadata latency header */ +struct ceph_metric_metadata_latency { + struct ceph_metric_header header; + struct ceph_timespec lat; + struct ceph_timespec avg; + __le64 sq_sum; + __le64 count; +} __packed; + +/* metric dentry lease header */ +struct ceph_metric_dlease { + struct ceph_metric_header header; + __le64 hit; + __le64 mis; + __le64 total; +} __packed; + +/* metric opened files header */ +struct ceph_opened_files { + struct ceph_metric_header header; + __le64 opened_files; + __le64 total; +} __packed; + +/* metric pinned i_caps header */ +struct ceph_pinned_icaps { + struct ceph_metric_header header; + __le64 pinned_icaps; + __le64 total; +} __packed; + +/* metric opened inodes header */ +struct ceph_opened_inodes { + struct ceph_metric_header header; + __le64 opened_inodes; + __le64 total; +} __packed; + +/* metric read io size header */ +struct ceph_read_io_size { + struct ceph_metric_header header; + __le64 total_ops; + __le64 total_size; +} __packed; + +/* metric write io size header */ +struct ceph_write_io_size { + struct ceph_metric_header header; + __le64 total_ops; + __le64 total_size; +} __packed; + +/** + * struct ceph_subvolume_metric_entry_wire - On-wire format sent to MDS + * @subvolume_id: Subvolume identifier + * @read_ops: Read operation count (32-bit, clamped from 64-bit internal) + * @write_ops: Write operation count (32-bit, clamped from 64-bit internal) + * @read_bytes: Total bytes read + * @write_bytes: Total bytes written + * @read_latency_us: Cumulative read latency in microseconds + * @write_latency_us: Cumulative write latency in microseconds + * @time_stamp: Collection timestamp (currently unused, set to 0) + * + * Wire format must match C++ AggregatedIOMetrics struct in MDS. + */ +struct ceph_subvolume_metric_entry_wire { + __le64 subvolume_id; + __le32 read_ops; + __le32 write_ops; + __le64 read_bytes; + __le64 write_bytes; + __le64 read_latency_us; + __le64 write_latency_us; + __le64 time_stamp; +} __packed; + +/* Old struct kept for internal tracking, not used on wire */ +struct ceph_subvolume_metric_entry { + __le64 subvolume_id; + __le64 read_ops; + __le64 write_ops; + __le64 read_bytes; + __le64 write_bytes; + __le64 read_latency_us; + __le64 write_latency_us; +} __packed; + +struct ceph_metric_head { + __le32 num; /* the number of metrics that will be sent */ +} __packed; + +enum metric_type { + METRIC_READ, + METRIC_WRITE, + METRIC_METADATA, + METRIC_COPYFROM, + METRIC_MAX +}; + +struct ceph_metric { + spinlock_t lock; + u64 total; + u64 size_sum; + u64 size_min; + u64 size_max; + ktime_t latency_sum; + ktime_t latency_avg; + ktime_t latency_sq_sum; + ktime_t latency_min; + ktime_t latency_max; +}; + +/* This is the global metrics */ +struct ceph_client_metric { + atomic64_t total_dentries; + struct percpu_counter d_lease_hit; + struct percpu_counter d_lease_mis; + + atomic64_t total_caps; + struct percpu_counter i_caps_hit; + struct percpu_counter i_caps_mis; + + struct ceph_metric metric[METRIC_MAX]; + + /* The total number of directories and files that are opened */ + atomic64_t opened_files; + + /* The total number of inodes that have opened files or directories */ + struct percpu_counter opened_inodes; + struct percpu_counter total_inodes; + + struct ceph_mds_session *session; + struct delayed_work delayed_work; /* delayed work */ +}; + +static inline void metric_schedule_delayed(struct ceph_client_metric *m) +{ + if (disable_send_metrics) + return; + + /* per second */ + schedule_delayed_work(&m->delayed_work, round_jiffies_relative(HZ)); +} + +extern int ceph_metric_init(struct ceph_client_metric *m); +extern void ceph_metric_destroy(struct ceph_client_metric *m); + +static inline void ceph_update_cap_hit(struct ceph_client_metric *m) +{ + percpu_counter_inc(&m->i_caps_hit); +} + +static inline void ceph_update_cap_mis(struct ceph_client_metric *m) +{ + percpu_counter_inc(&m->i_caps_mis); +} + +extern void ceph_update_metrics(struct ceph_metric *m, + ktime_t r_start, ktime_t r_end, + unsigned int size, int rc); + +static inline void ceph_update_read_metrics(struct ceph_client_metric *m, + ktime_t r_start, ktime_t r_end, + unsigned int size, int rc) +{ + ceph_update_metrics(&m->metric[METRIC_READ], + r_start, r_end, size, rc); +} +static inline void ceph_update_write_metrics(struct ceph_client_metric *m, + ktime_t r_start, ktime_t r_end, + unsigned int size, int rc) +{ + ceph_update_metrics(&m->metric[METRIC_WRITE], + r_start, r_end, size, rc); +} +static inline void ceph_update_metadata_metrics(struct ceph_client_metric *m, + ktime_t r_start, ktime_t r_end, + int rc) +{ + ceph_update_metrics(&m->metric[METRIC_METADATA], + r_start, r_end, 0, rc); +} +static inline void ceph_update_copyfrom_metrics(struct ceph_client_metric *m, + ktime_t r_start, ktime_t r_end, + unsigned int size, int rc) +{ + ceph_update_metrics(&m->metric[METRIC_COPYFROM], + r_start, r_end, size, rc); +} +#endif /* _FS_CEPH_MDS_METRIC_H */ diff --git a/fs/ceph/quota.c b/fs/ceph/quota.c new file mode 100644 index 000000000..053d5bf0c --- /dev/null +++ b/fs/ceph/quota.c @@ -0,0 +1,564 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * quota.c - CephFS quota + * + * Copyright (C) 2017-2018 SUSE + */ + +#include + +#include "super.h" +#include "mds_client.h" + +void ceph_adjust_quota_realms_count(struct inode *inode, bool inc) +{ + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb); + if (inc) + atomic64_inc(&mdsc->quotarealms_count); + else + atomic64_dec(&mdsc->quotarealms_count); +} + +static inline bool ceph_has_realms_with_quotas(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(sb); + struct inode *root = d_inode(sb->s_root); + + if (atomic64_read(&mdsc->quotarealms_count) > 0) + return true; + /* if root is the real CephFS root, we don't have quota realms */ + if (root && ceph_ino(root) == CEPH_INO_ROOT) + return false; + /* MDS stray dirs have no quota realms */ + if (ceph_vino_is_reserved(ceph_inode(inode)->i_vino)) + return false; + /* otherwise, we can't know for sure */ + return true; +} + +void ceph_handle_quota(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session, + struct ceph_msg *msg) +{ + struct super_block *sb = mdsc->fsc->sb; + struct ceph_mds_quota *h = msg->front.iov_base; + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_vino vino; + struct inode *inode; + struct ceph_inode_info *ci; + + if (!ceph_inc_mds_stopping_blocker(mdsc, session)) + return; + + if (msg->front.iov_len < sizeof(*h)) { + pr_err_client(cl, "corrupt message mds%d len %d\n", + session->s_mds, (int)msg->front.iov_len); + ceph_msg_dump(msg); + goto out; + } + + /* lookup inode */ + vino.ino = le64_to_cpu(h->ino); + vino.snap = CEPH_NOSNAP; + inode = ceph_find_inode(sb, vino); + if (!inode) { + pr_warn_client(cl, "failed to find inode %llx\n", vino.ino); + goto out; + } + ci = ceph_inode(inode); + + spin_lock(&ci->i_ceph_lock); + ci->i_rbytes = le64_to_cpu(h->rbytes); + ci->i_rfiles = le64_to_cpu(h->rfiles); + ci->i_rsubdirs = le64_to_cpu(h->rsubdirs); + __ceph_update_quota(ci, le64_to_cpu(h->max_bytes), + le64_to_cpu(h->max_files)); + spin_unlock(&ci->i_ceph_lock); + + iput(inode); +out: + ceph_dec_mds_stopping_blocker(mdsc); +} + +static struct ceph_quotarealm_inode * +find_quotarealm_inode(struct ceph_mds_client *mdsc, u64 ino) +{ + struct ceph_quotarealm_inode *qri = NULL; + struct rb_node **node, *parent = NULL; + struct ceph_client *cl = mdsc->fsc->client; + + mutex_lock(&mdsc->quotarealms_inodes_mutex); + node = &(mdsc->quotarealms_inodes.rb_node); + while (*node) { + parent = *node; + qri = container_of(*node, struct ceph_quotarealm_inode, node); + + if (ino < qri->ino) + node = &((*node)->rb_left); + else if (ino > qri->ino) + node = &((*node)->rb_right); + else + break; + } + if (!qri || (qri->ino != ino)) { + /* Not found, create a new one and insert it */ + qri = kmalloc_obj(*qri); + if (qri) { + qri->ino = ino; + qri->inode = NULL; + qri->timeout = 0; + mutex_init(&qri->mutex); + rb_link_node(&qri->node, parent, node); + rb_insert_color(&qri->node, &mdsc->quotarealms_inodes); + } else + pr_warn_client(cl, "Failed to alloc quotarealms_inode\n"); + } + mutex_unlock(&mdsc->quotarealms_inodes_mutex); + + return qri; +} + +/* + * This function will try to lookup a realm inode which isn't visible in the + * filesystem mountpoint. A list of these kind of inodes (not visible) is + * maintained in the mdsc and freed only when the filesystem is umounted. + * + * Note that these inodes are kept in this list even if the lookup fails, which + * allows to prevent useless lookup requests. + */ +static struct inode *lookup_quotarealm_inode(struct ceph_mds_client *mdsc, + struct super_block *sb, + struct ceph_snap_realm *realm) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_quotarealm_inode *qri; + struct inode *in; + + qri = find_quotarealm_inode(mdsc, realm->ino); + if (!qri) + return NULL; + + mutex_lock(&qri->mutex); + if (qri->inode && ceph_is_any_caps(qri->inode)) { + /* A request has already returned the inode */ + mutex_unlock(&qri->mutex); + return qri->inode; + } + /* Check if this inode lookup has failed recently */ + if (qri->timeout && + time_before_eq(jiffies, qri->timeout)) { + mutex_unlock(&qri->mutex); + return NULL; + } + if (qri->inode) { + /* get caps */ + int ret = __ceph_do_getattr(qri->inode, NULL, + CEPH_STAT_CAP_INODE, true); + if (ret >= 0) + in = qri->inode; + else + in = ERR_PTR(ret); + } else { + in = ceph_lookup_inode(sb, realm->ino); + } + + if (IS_ERR(in)) { + doutc(cl, "Can't lookup inode %llx (err: %ld)\n", realm->ino, + PTR_ERR(in)); + qri->timeout = jiffies + secs_to_jiffies(60); /* XXX */ + } else { + qri->timeout = 0; + qri->inode = in; + } + mutex_unlock(&qri->mutex); + + return in; +} + +void ceph_cleanup_quotarealms_inodes(struct ceph_mds_client *mdsc) +{ + struct ceph_quotarealm_inode *qri; + struct rb_node *node; + + /* + * It should now be safe to clean quotarealms_inode tree without holding + * mdsc->quotarealms_inodes_mutex... + */ + mutex_lock(&mdsc->quotarealms_inodes_mutex); + while (!RB_EMPTY_ROOT(&mdsc->quotarealms_inodes)) { + node = rb_first(&mdsc->quotarealms_inodes); + qri = rb_entry(node, struct ceph_quotarealm_inode, node); + rb_erase(node, &mdsc->quotarealms_inodes); + iput(qri->inode); + kfree(qri); + } + mutex_unlock(&mdsc->quotarealms_inodes_mutex); +} + +/* + * This function walks through the snaprealm for an inode and set the + * realmp with the first snaprealm that has quotas set (max_files, + * max_bytes, or any, depending on the 'which_quota' argument). If the root is + * reached, set the realmp with the root ceph_snap_realm instead. + * + * Note that the caller is responsible for calling ceph_put_snap_realm() on the + * returned realm. + * + * Callers of this function need to hold mdsc->snap_rwsem. However, if there's + * a need to do an inode lookup, this rwsem will be temporarily dropped. Hence + * the 'retry' argument: if rwsem needs to be dropped and 'retry' is 'false' + * this function will return -EAGAIN; otherwise, the snaprealms walk-through + * will be restarted. + */ +static int get_quota_realm(struct ceph_mds_client *mdsc, struct inode *inode, + enum quota_get_realm which_quota, + struct ceph_snap_realm **realmp, bool retry) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_inode_info *ci = NULL; + struct ceph_snap_realm *realm, *next; + struct inode *in; + bool has_quota; + + if (realmp) + *realmp = NULL; + if (ceph_snap(inode) != CEPH_NOSNAP) + return 0; + +restart: + realm = ceph_inode(inode)->i_snap_realm; + if (realm) { + ceph_get_snap_realm(mdsc, realm); + } else { + /* + * i_snap_realm is NULL when all caps have been released, e.g. + * after an MDS session rejection. This is a transient state; + * the realm will be restored once caps are re-granted. + * Treat it as "no quota realm found". + */ + doutc(cl, "%p %llx.%llx null i_snap_realm\n", + inode, ceph_vinop(inode)); + } + + while (realm) { + bool has_inode; + + spin_lock(&realm->inodes_with_caps_lock); + has_inode = realm->inode; + in = has_inode ? igrab(realm->inode) : NULL; + spin_unlock(&realm->inodes_with_caps_lock); + if (has_inode && !in) + break; + if (!in) { + up_read(&mdsc->snap_rwsem); + in = lookup_quotarealm_inode(mdsc, inode->i_sb, realm); + down_read(&mdsc->snap_rwsem); + if (IS_ERR_OR_NULL(in)) + break; + ceph_put_snap_realm(mdsc, realm); + if (!retry) + return -EAGAIN; + goto restart; + } + + ci = ceph_inode(in); + has_quota = __ceph_has_quota(ci, which_quota); + iput(in); + + next = realm->parent; + if (has_quota || !next) { + if (realmp) + *realmp = realm; + return 0; + } + + ceph_get_snap_realm(mdsc, next); + ceph_put_snap_realm(mdsc, realm); + realm = next; + } + if (realm) + ceph_put_snap_realm(mdsc, realm); + + return 0; +} + +bool ceph_quota_is_same_realm(struct inode *old, struct inode *new) +{ + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(old->i_sb); + struct ceph_snap_realm *old_realm, *new_realm; + bool is_same; + int ret; + +restart: + /* + * We need to lookup 2 quota realms atomically, i.e. with snap_rwsem. + * However, get_quota_realm may drop it temporarily. By setting the + * 'retry' parameter to 'false', we'll get -EAGAIN if the rwsem was + * dropped and we can then restart the whole operation. + */ + down_read(&mdsc->snap_rwsem); + get_quota_realm(mdsc, old, QUOTA_GET_ANY, &old_realm, true); + ret = get_quota_realm(mdsc, new, QUOTA_GET_ANY, &new_realm, false); + if (ret == -EAGAIN) { + up_read(&mdsc->snap_rwsem); + if (old_realm) + ceph_put_snap_realm(mdsc, old_realm); + goto restart; + } + is_same = (old_realm == new_realm); + up_read(&mdsc->snap_rwsem); + + if (old_realm) + ceph_put_snap_realm(mdsc, old_realm); + if (new_realm) + ceph_put_snap_realm(mdsc, new_realm); + + return is_same; +} + +enum quota_check_op { + QUOTA_CHECK_MAX_FILES_OP, /* check quota max_files limit */ + QUOTA_CHECK_MAX_BYTES_OP, /* check quota max_files limit */ + QUOTA_CHECK_MAX_BYTES_APPROACHING_OP /* check if quota max_files + limit is approaching */ +}; + +/* + * check_quota_exceeded() will walk up the snaprealm hierarchy and, for each + * realm, it will execute quota check operation defined by the 'op' parameter. + * The snaprealm walk is interrupted if the quota check detects that the quota + * is exceeded or if the root inode is reached. + */ +static bool check_quota_exceeded(struct inode *inode, enum quota_check_op op, + loff_t delta) +{ + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb); + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_inode_info *ci; + struct ceph_snap_realm *realm, *next; + struct inode *in; + u64 max, rvalue; + bool exceeded = false; + + if (ceph_snap(inode) != CEPH_NOSNAP) + return false; + + down_read(&mdsc->snap_rwsem); +restart: + realm = ceph_inode(inode)->i_snap_realm; + if (realm) { + ceph_get_snap_realm(mdsc, realm); + } else { + /* + * i_snap_realm is NULL when all caps have been released, e.g. + * after an MDS session rejection. This is a transient state; + * the realm will be restored once caps are re-granted. + * Treat it as "quota not exceeded". + */ + doutc(cl, "%p %llx.%llx null i_snap_realm\n", + inode, ceph_vinop(inode)); + } + + while (realm) { + bool has_inode; + + spin_lock(&realm->inodes_with_caps_lock); + has_inode = realm->inode; + in = has_inode ? igrab(realm->inode) : NULL; + spin_unlock(&realm->inodes_with_caps_lock); + if (has_inode && !in) + break; + if (!in) { + up_read(&mdsc->snap_rwsem); + in = lookup_quotarealm_inode(mdsc, inode->i_sb, realm); + down_read(&mdsc->snap_rwsem); + if (IS_ERR_OR_NULL(in)) + break; + ceph_put_snap_realm(mdsc, realm); + goto restart; + } + ci = ceph_inode(in); + spin_lock(&ci->i_ceph_lock); + if (op == QUOTA_CHECK_MAX_FILES_OP) { + max = ci->i_max_files; + rvalue = ci->i_rfiles + ci->i_rsubdirs; + } else { + max = ci->i_max_bytes; + rvalue = ci->i_rbytes; + } + spin_unlock(&ci->i_ceph_lock); + switch (op) { + case QUOTA_CHECK_MAX_FILES_OP: + case QUOTA_CHECK_MAX_BYTES_OP: + exceeded = (max && (rvalue + delta > max)); + break; + case QUOTA_CHECK_MAX_BYTES_APPROACHING_OP: + if (max) { + if (rvalue >= max) + exceeded = true; + else { + /* + * when we're writing more that 1/16th + * of the available space + */ + exceeded = + (((max - rvalue) >> 4) < delta); + } + } + break; + default: + /* Shouldn't happen */ + pr_warn_client(cl, "Invalid quota check op (%d)\n", op); + exceeded = true; /* Just break the loop */ + } + iput(in); + + next = realm->parent; + if (exceeded || !next) + break; + ceph_get_snap_realm(mdsc, next); + ceph_put_snap_realm(mdsc, realm); + realm = next; + } + if (realm) + ceph_put_snap_realm(mdsc, realm); + up_read(&mdsc->snap_rwsem); + + return exceeded; +} + +/* + * ceph_quota_is_max_files_exceeded - check if we can create a new file + * @inode: directory where a new file is being created + * + * This functions returns true is max_files quota allows a new file to be + * created. It is necessary to walk through the snaprealm hierarchy (until the + * FS root) to check all realms with quotas set. + */ +bool ceph_quota_is_max_files_exceeded(struct inode *inode) +{ + if (!ceph_has_realms_with_quotas(inode)) + return false; + + WARN_ON(!S_ISDIR(inode->i_mode)); + + return check_quota_exceeded(inode, QUOTA_CHECK_MAX_FILES_OP, 1); +} + +/* + * ceph_quota_is_max_bytes_exceeded - check if we can write to a file + * @inode: inode being written + * @newsize: new size if write succeeds + * + * This functions returns true is max_bytes quota allows a file size to reach + * @newsize; it returns false otherwise. + */ +bool ceph_quota_is_max_bytes_exceeded(struct inode *inode, loff_t newsize) +{ + loff_t size = i_size_read(inode); + + if (!ceph_has_realms_with_quotas(inode)) + return false; + + /* return immediately if we're decreasing file size */ + if (newsize <= size) + return false; + + return check_quota_exceeded(inode, QUOTA_CHECK_MAX_BYTES_OP, (newsize - size)); +} + +/* + * ceph_quota_is_max_bytes_approaching - check if we're reaching max_bytes + * @inode: inode being written + * @newsize: new size if write succeeds + * + * This function returns true if the new file size @newsize will be consuming + * more than 1/16th of the available quota space; it returns false otherwise. + */ +bool ceph_quota_is_max_bytes_approaching(struct inode *inode, loff_t newsize) +{ + loff_t size = ceph_inode(inode)->i_reported_size; + + if (!ceph_has_realms_with_quotas(inode)) + return false; + + /* return immediately if we're decreasing file size */ + if (newsize <= size) + return false; + + return check_quota_exceeded(inode, QUOTA_CHECK_MAX_BYTES_APPROACHING_OP, + (newsize - size)); +} + +/* + * ceph_quota_update_statfs - if root has quota update statfs with quota status + * @fsc: filesystem client instance + * @buf: statfs to update + * + * If the mounted filesystem root has max_bytes quota set, update the filesystem + * statistics with the quota status. + * + * This function returns true if the stats have been updated, false otherwise. + */ +bool ceph_quota_update_statfs(struct ceph_fs_client *fsc, struct kstatfs *buf) +{ + struct ceph_mds_client *mdsc = fsc->mdsc; + struct ceph_inode_info *ci; + struct ceph_snap_realm *realm; + struct inode *in; + u64 total = 0, used, free; + bool is_updated = false; + + if (!ceph_has_realms_with_quotas(d_inode(fsc->sb->s_root))) + return false; + + down_read(&mdsc->snap_rwsem); + get_quota_realm(mdsc, d_inode(fsc->sb->s_root), QUOTA_GET_MAX_BYTES, + &realm, true); + up_read(&mdsc->snap_rwsem); + if (!realm) + return false; + + spin_lock(&realm->inodes_with_caps_lock); + in = realm->inode ? igrab(realm->inode) : NULL; + spin_unlock(&realm->inodes_with_caps_lock); + if (in) { + ci = ceph_inode(in); + spin_lock(&ci->i_ceph_lock); + if (ci->i_max_bytes) { + total = ci->i_max_bytes >> CEPH_BLOCK_SHIFT; + used = ci->i_rbytes >> CEPH_BLOCK_SHIFT; + /* For quota size less than 4MB, use 4KB block size */ + if (!total) { + total = ci->i_max_bytes >> CEPH_4K_BLOCK_SHIFT; + used = ci->i_rbytes >> CEPH_4K_BLOCK_SHIFT; + buf->f_frsize = 1 << CEPH_4K_BLOCK_SHIFT; + } + /* It is possible for a quota to be exceeded. + * Report 'zero' in that case + */ + free = total > used ? total - used : 0; + /* For quota size less than 4KB, report the + * total=used=4KB,free=0 when quota is full + * and total=free=4KB, used=0 otherwise */ + if (!total) { + total = 1; + free = ci->i_max_bytes > ci->i_rbytes ? 1 : 0; + buf->f_frsize = 1 << CEPH_4K_BLOCK_SHIFT; + } + } + spin_unlock(&ci->i_ceph_lock); + if (total) { + buf->f_blocks = total; + buf->f_bfree = free; + buf->f_bavail = free; + is_updated = true; + } + iput(in); + } + ceph_put_snap_realm(mdsc, realm); + + return is_updated; +} + diff --git a/fs/ceph/snap.c b/fs/ceph/snap.c new file mode 100644 index 000000000..9b79a5eac --- /dev/null +++ b/fs/ceph/snap.c @@ -0,0 +1,1343 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +#include +#include +#include +#include +#include "super.h" +#include "mds_client.h" +#include + +/* unused map expires after 5 minutes */ +#define CEPH_SNAPID_MAP_TIMEOUT (5 * 60 * HZ) + +/* + * Snapshots in ceph are driven in large part by cooperation from the + * client. In contrast to local file systems or file servers that + * implement snapshots at a single point in the system, ceph's + * distributed access to storage requires clients to help decide + * whether a write logically occurs before or after a recently created + * snapshot. + * + * This provides a perfect instantanous client-wide snapshot. Between + * clients, however, snapshots may appear to be applied at slightly + * different points in time, depending on delays in delivering the + * snapshot notification. + * + * Snapshots are _not_ file system-wide. Instead, each snapshot + * applies to the subdirectory nested beneath some directory. This + * effectively divides the hierarchy into multiple "realms," where all + * of the files contained by each realm share the same set of + * snapshots. An individual realm's snap set contains snapshots + * explicitly created on that realm, as well as any snaps in its + * parent's snap set _after_ the point at which the parent became it's + * parent (due to, say, a rename). Similarly, snaps from prior parents + * during the time intervals during which they were the parent are included. + * + * The client is spared most of this detail, fortunately... it must only + * maintains a hierarchy of realms reflecting the current parent/child + * realm relationship, and for each realm has an explicit list of snaps + * inherited from prior parents. + * + * A snap_realm struct is maintained for realms containing every inode + * with an open cap in the system. (The needed snap realm information is + * provided by the MDS whenever a cap is issued, i.e., on open.) A 'seq' + * version number is used to ensure that as realm parameters change (new + * snapshot, new parent, etc.) the client's realm hierarchy is updated. + * + * The realm hierarchy drives the generation of a 'snap context' for each + * realm, which simply lists the resulting set of snaps for the realm. This + * is attached to any writes sent to OSDs. + */ +/* + * Unfortunately error handling is a bit mixed here. If we get a snap + * update, but don't have enough memory to update our realm hierarchy, + * it's not clear what we can do about it (besides complaining to the + * console). + */ + + +/* + * increase ref count for the realm + * + * caller must hold snap_rwsem. + */ +void ceph_get_snap_realm(struct ceph_mds_client *mdsc, + struct ceph_snap_realm *realm) +{ + lockdep_assert_held(&mdsc->snap_rwsem); + + /* + * The 0->1 and 1->0 transitions must take the snap_empty_lock + * atomically with the refcount change. Go ahead and bump the + * nref here, unless it's 0, in which case we take the spinlock + * and then do the increment and remove it from the list. + */ + if (atomic_inc_not_zero(&realm->nref)) + return; + + spin_lock(&mdsc->snap_empty_lock); + if (atomic_inc_return(&realm->nref) == 1) + list_del_init(&realm->empty_item); + spin_unlock(&mdsc->snap_empty_lock); +} + +static void __insert_snap_realm(struct rb_root *root, + struct ceph_snap_realm *new) +{ + struct rb_node **p = &root->rb_node; + struct rb_node *parent = NULL; + struct ceph_snap_realm *r = NULL; + + while (*p) { + parent = *p; + r = rb_entry(parent, struct ceph_snap_realm, node); + if (new->ino < r->ino) + p = &(*p)->rb_left; + else if (new->ino > r->ino) + p = &(*p)->rb_right; + else + BUG(); + } + + rb_link_node(&new->node, parent, p); + rb_insert_color(&new->node, root); +} + +/* + * create and get the realm rooted at @ino and bump its ref count. + * + * caller must hold snap_rwsem for write. + */ +static struct ceph_snap_realm *ceph_create_snap_realm( + struct ceph_mds_client *mdsc, + u64 ino) +{ + struct ceph_snap_realm *realm; + + lockdep_assert_held_write(&mdsc->snap_rwsem); + + realm = kzalloc_obj(*realm, GFP_NOFS); + if (!realm) + return ERR_PTR(-ENOMEM); + + /* Do not release the global dummy snaprealm until unmouting */ + if (ino == CEPH_INO_GLOBAL_SNAPREALM) + atomic_set(&realm->nref, 2); + else + atomic_set(&realm->nref, 1); + realm->ino = ino; + INIT_LIST_HEAD(&realm->children); + INIT_LIST_HEAD(&realm->child_item); + INIT_LIST_HEAD(&realm->empty_item); + INIT_LIST_HEAD(&realm->dirty_item); + INIT_LIST_HEAD(&realm->rebuild_item); + INIT_LIST_HEAD(&realm->inodes_with_caps); + spin_lock_init(&realm->inodes_with_caps_lock); + __insert_snap_realm(&mdsc->snap_realms, realm); + mdsc->num_snap_realms++; + + doutc(mdsc->fsc->client, "%llx %p\n", realm->ino, realm); + return realm; +} + +/* + * lookup the realm rooted at @ino. + * + * caller must hold snap_rwsem. + */ +static struct ceph_snap_realm *__lookup_snap_realm(struct ceph_mds_client *mdsc, + u64 ino) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct rb_node *n = mdsc->snap_realms.rb_node; + struct ceph_snap_realm *r; + + lockdep_assert_held(&mdsc->snap_rwsem); + + while (n) { + r = rb_entry(n, struct ceph_snap_realm, node); + if (ino < r->ino) + n = n->rb_left; + else if (ino > r->ino) + n = n->rb_right; + else { + doutc(cl, "%llx %p\n", r->ino, r); + return r; + } + } + return NULL; +} + +struct ceph_snap_realm *ceph_lookup_snap_realm(struct ceph_mds_client *mdsc, + u64 ino) +{ + struct ceph_snap_realm *r; + r = __lookup_snap_realm(mdsc, ino); + if (r) + ceph_get_snap_realm(mdsc, r); + return r; +} + +static void __put_snap_realm(struct ceph_mds_client *mdsc, + struct ceph_snap_realm *realm); + +/* + * called with snap_rwsem (write) + */ +static void __destroy_snap_realm(struct ceph_mds_client *mdsc, + struct ceph_snap_realm *realm) +{ + struct ceph_client *cl = mdsc->fsc->client; + lockdep_assert_held_write(&mdsc->snap_rwsem); + + doutc(cl, "%p %llx\n", realm, realm->ino); + + rb_erase(&realm->node, &mdsc->snap_realms); + mdsc->num_snap_realms--; + + if (realm->parent) { + list_del_init(&realm->child_item); + __put_snap_realm(mdsc, realm->parent); + } + + kfree(realm->prior_parent_snaps); + kfree(realm->snaps); + ceph_put_snap_context(realm->cached_context); + kfree(realm); +} + +/* + * caller holds snap_rwsem (write) + */ +static void __put_snap_realm(struct ceph_mds_client *mdsc, + struct ceph_snap_realm *realm) +{ + lockdep_assert_held_write(&mdsc->snap_rwsem); + + /* + * We do not require the snap_empty_lock here, as any caller that + * increments the value must hold the snap_rwsem. + */ + if (atomic_dec_and_test(&realm->nref)) + __destroy_snap_realm(mdsc, realm); +} + +/* + * See comments in ceph_get_snap_realm. Caller needn't hold any locks. + */ +void ceph_put_snap_realm(struct ceph_mds_client *mdsc, + struct ceph_snap_realm *realm) +{ + if (!atomic_dec_and_lock(&realm->nref, &mdsc->snap_empty_lock)) + return; + + if (down_write_trylock(&mdsc->snap_rwsem)) { + spin_unlock(&mdsc->snap_empty_lock); + __destroy_snap_realm(mdsc, realm); + up_write(&mdsc->snap_rwsem); + } else { + list_add(&realm->empty_item, &mdsc->snap_empty); + spin_unlock(&mdsc->snap_empty_lock); + } +} + +/* + * Clean up any realms whose ref counts have dropped to zero. Note + * that this does not include realms who were created but not yet + * used. + * + * Called under snap_rwsem (write) + */ +static void __cleanup_empty_realms(struct ceph_mds_client *mdsc) +{ + struct ceph_snap_realm *realm; + + lockdep_assert_held_write(&mdsc->snap_rwsem); + + spin_lock(&mdsc->snap_empty_lock); + while (!list_empty(&mdsc->snap_empty)) { + realm = list_first_entry(&mdsc->snap_empty, + struct ceph_snap_realm, empty_item); + list_del(&realm->empty_item); + spin_unlock(&mdsc->snap_empty_lock); + __destroy_snap_realm(mdsc, realm); + spin_lock(&mdsc->snap_empty_lock); + } + spin_unlock(&mdsc->snap_empty_lock); +} + +void ceph_cleanup_global_and_empty_realms(struct ceph_mds_client *mdsc) +{ + struct ceph_snap_realm *global_realm; + + down_write(&mdsc->snap_rwsem); + global_realm = __lookup_snap_realm(mdsc, CEPH_INO_GLOBAL_SNAPREALM); + if (global_realm) + ceph_put_snap_realm(mdsc, global_realm); + __cleanup_empty_realms(mdsc); + up_write(&mdsc->snap_rwsem); +} + +/* + * adjust the parent realm of a given @realm. adjust child list, and parent + * pointers, and ref counts appropriately. + * + * return true if parent was changed, 0 if unchanged, <0 on error. + * + * caller must hold snap_rwsem for write. + */ +static int adjust_snap_realm_parent(struct ceph_mds_client *mdsc, + struct ceph_snap_realm *realm, + u64 parentino) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_snap_realm *parent; + + lockdep_assert_held_write(&mdsc->snap_rwsem); + + if (realm->parent_ino == parentino) + return 0; + + parent = ceph_lookup_snap_realm(mdsc, parentino); + if (!parent) { + parent = ceph_create_snap_realm(mdsc, parentino); + if (IS_ERR(parent)) + return PTR_ERR(parent); + } + doutc(cl, "%llx %p: %llx %p -> %llx %p\n", realm->ino, realm, + realm->parent_ino, realm->parent, parentino, parent); + if (realm->parent) { + list_del_init(&realm->child_item); + ceph_put_snap_realm(mdsc, realm->parent); + } + realm->parent_ino = parentino; + realm->parent = parent; + list_add(&realm->child_item, &parent->children); + return 1; +} + + +static int cmpu64_rev(const void *a, const void *b) +{ + if (*(u64 *)a < *(u64 *)b) + return 1; + if (*(u64 *)a > *(u64 *)b) + return -1; + return 0; +} + + +/* + * build the snap context for a given realm. + */ +static int build_snap_context(struct ceph_mds_client *mdsc, + struct ceph_snap_realm *realm, + struct list_head *realm_queue, + struct list_head *dirty_realms) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_snap_realm *parent = realm->parent; + struct ceph_snap_context *snapc; + int err = 0; + u32 num = realm->num_prior_parent_snaps + realm->num_snaps; + + /* + * build parent context, if it hasn't been built. + * conservatively estimate that all parent snaps might be + * included by us. + */ + if (parent) { + if (!parent->cached_context) { + /* add to the queue head */ + list_add(&parent->rebuild_item, realm_queue); + return 1; + } + num += parent->cached_context->num_snaps; + } + + /* do i actually need to update? not if my context seq + matches realm seq, and my parents' does to. (this works + because we rebuild_snap_realms() works _downward_ in + hierarchy after each update.) */ + if (realm->cached_context && + realm->cached_context->seq == realm->seq && + (!parent || + realm->cached_context->seq >= parent->cached_context->seq)) { + doutc(cl, "%llx %p: %p seq %lld (%u snaps) (unchanged)\n", + realm->ino, realm, realm->cached_context, + realm->cached_context->seq, + (unsigned int)realm->cached_context->num_snaps); + return 0; + } + + /* alloc new snap context */ + err = -ENOMEM; + if ((size_t)num > (SIZE_MAX - sizeof(*snapc)) / sizeof(u64)) + goto fail; + snapc = ceph_create_snap_context(num, GFP_NOFS); + if (!snapc) + goto fail; + + /* build (reverse sorted) snap vector */ + num = 0; + snapc->seq = realm->seq; + if (parent) { + u32 i; + + /* include any of parent's snaps occurring _after_ my + parent became my parent */ + for (i = 0; i < parent->cached_context->num_snaps; i++) + if (parent->cached_context->snaps[i] >= + realm->parent_since) + snapc->snaps[num++] = + parent->cached_context->snaps[i]; + if (parent->cached_context->seq > snapc->seq) + snapc->seq = parent->cached_context->seq; + } + memcpy(snapc->snaps + num, realm->snaps, + sizeof(u64)*realm->num_snaps); + num += realm->num_snaps; + memcpy(snapc->snaps + num, realm->prior_parent_snaps, + sizeof(u64)*realm->num_prior_parent_snaps); + num += realm->num_prior_parent_snaps; + + sort(snapc->snaps, num, sizeof(u64), cmpu64_rev, NULL); + snapc->num_snaps = num; + doutc(cl, "%llx %p: %p seq %lld (%u snaps)\n", realm->ino, realm, + snapc, snapc->seq, (unsigned int) snapc->num_snaps); + + ceph_put_snap_context(realm->cached_context); + realm->cached_context = snapc; + /* queue realm for cap_snap creation */ + list_add_tail(&realm->dirty_item, dirty_realms); + return 0; + +fail: + /* + * if we fail, clear old (incorrect) cached_context... hopefully + * we'll have better luck building it later + */ + if (realm->cached_context) { + ceph_put_snap_context(realm->cached_context); + realm->cached_context = NULL; + } + pr_err_client(cl, "%llx %p fail %d\n", realm->ino, realm, err); + return err; +} + +/* + * rebuild snap context for the given realm and all of its children. + */ +static void rebuild_snap_realms(struct ceph_mds_client *mdsc, + struct ceph_snap_realm *realm, + struct list_head *dirty_realms) +{ + struct ceph_client *cl = mdsc->fsc->client; + LIST_HEAD(realm_queue); + int last = 0; + bool skip = false; + + list_add_tail(&realm->rebuild_item, &realm_queue); + + while (!list_empty(&realm_queue)) { + struct ceph_snap_realm *_realm, *child; + + _realm = list_first_entry(&realm_queue, + struct ceph_snap_realm, + rebuild_item); + + /* + * If the last building failed dues to memory + * issue, just empty the realm_queue and return + * to avoid infinite loop. + */ + if (last < 0) { + list_del_init(&_realm->rebuild_item); + continue; + } + + last = build_snap_context(mdsc, _realm, &realm_queue, + dirty_realms); + doutc(cl, "%llx %p, %s\n", realm->ino, realm, + last > 0 ? "is deferred" : !last ? "succeeded" : "failed"); + + /* is any child in the list ? */ + list_for_each_entry(child, &_realm->children, child_item) { + if (!list_empty(&child->rebuild_item)) { + skip = true; + break; + } + } + + if (!skip) { + list_for_each_entry(child, &_realm->children, child_item) + list_add_tail(&child->rebuild_item, &realm_queue); + } + + /* last == 1 means need to build parent first */ + if (last <= 0) + list_del_init(&_realm->rebuild_item); + } +} + + +/* + * helper to allocate and decode an array of snapids. free prior + * instance, if any. + */ +static int dup_array(u64 **dst, __le64 *src, u32 num) +{ + u32 i; + + kfree(*dst); + if (num) { + *dst = kcalloc(num, sizeof(u64), GFP_NOFS); + if (!*dst) + return -ENOMEM; + for (i = 0; i < num; i++) + (*dst)[i] = get_unaligned_le64(src + i); + } else { + *dst = NULL; + } + return 0; +} + +static bool has_new_snaps(struct ceph_snap_context *o, + struct ceph_snap_context *n) +{ + if (n->num_snaps == 0) + return false; + /* snaps are in descending order */ + return n->snaps[0] > o->seq; +} + +/* + * When a snapshot is applied, the size/mtime inode metadata is queued + * in a ceph_cap_snap (one for each snapshot) until writeback + * completes and the metadata can be flushed back to the MDS. + * + * However, if a (sync) write is currently in-progress when we apply + * the snapshot, we have to wait until the write succeeds or fails + * (and a final size/mtime is known). In this case the + * cap_snap->writing = 1, and is said to be "pending." When the write + * finishes, we __ceph_finish_cap_snap(). + * + * Caller must hold snap_rwsem for read (i.e., the realm topology won't + * change). + */ +static void ceph_queue_cap_snap(struct ceph_inode_info *ci, + struct ceph_cap_snap **pcapsnap) +{ + struct inode *inode = &ci->netfs.inode; + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_snap_context *old_snapc, *new_snapc; + struct ceph_cap_snap *capsnap = *pcapsnap; + struct ceph_buffer *old_blob = NULL; + int used, dirty; + + spin_lock(&ci->i_ceph_lock); + used = __ceph_caps_used(ci); + dirty = __ceph_caps_dirty(ci); + + old_snapc = ci->i_head_snapc; + new_snapc = ci->i_snap_realm->cached_context; + + /* + * If there is a write in progress, treat that as a dirty Fw, + * even though it hasn't completed yet; by the time we finish + * up this capsnap it will be. + */ + if (used & CEPH_CAP_FILE_WR) + dirty |= CEPH_CAP_FILE_WR; + + if (__ceph_have_pending_cap_snap(ci)) { + /* there is no point in queuing multiple "pending" cap_snaps, + as no new writes are allowed to start when pending, so any + writes in progress now were started before the previous + cap_snap. lucky us. */ + doutc(cl, "%p %llx.%llx already pending\n", inode, + ceph_vinop(inode)); + goto update_snapc; + } + if (ci->i_wrbuffer_ref_head == 0 && + !(dirty & (CEPH_CAP_ANY_EXCL|CEPH_CAP_FILE_WR))) { + doutc(cl, "%p %llx.%llx nothing dirty|writing\n", inode, + ceph_vinop(inode)); + goto update_snapc; + } + + BUG_ON(!old_snapc); + + /* + * There is no need to send FLUSHSNAP message to MDS if there is + * no new snapshot. But when there is dirty pages or on-going + * writes, we still need to create cap_snap. cap_snap is needed + * by the write path and page writeback path. + * + * also see ceph_try_drop_cap_snap() + */ + if (has_new_snaps(old_snapc, new_snapc)) { + if (dirty & (CEPH_CAP_ANY_EXCL|CEPH_CAP_FILE_WR)) + capsnap->need_flush = true; + } else { + if (!(used & CEPH_CAP_FILE_WR) && + ci->i_wrbuffer_ref_head == 0) { + doutc(cl, "%p %llx.%llx no new_snap|dirty_page|writing\n", + inode, ceph_vinop(inode)); + goto update_snapc; + } + } + + doutc(cl, "%p %llx.%llx cap_snap %p queuing under %p %s %s\n", + inode, ceph_vinop(inode), capsnap, old_snapc, + ceph_cap_string(dirty), capsnap->need_flush ? "" : "no_flush"); + ihold(inode); + + capsnap->follows = old_snapc->seq; + capsnap->issued = __ceph_caps_issued(ci, NULL); + capsnap->dirty = dirty; + + capsnap->mode = inode->i_mode; + capsnap->uid = inode->i_uid; + capsnap->gid = inode->i_gid; + + if (dirty & CEPH_CAP_XATTR_EXCL) { + old_blob = __ceph_build_xattrs_blob(ci); + capsnap->xattr_blob = + ceph_buffer_get(ci->i_xattrs.blob); + capsnap->xattr_version = ci->i_xattrs.version; + } else { + capsnap->xattr_blob = NULL; + capsnap->xattr_version = 0; + } + + capsnap->inline_data = ci->i_inline_version != CEPH_INLINE_NONE; + + /* dirty page count moved from _head to this cap_snap; + all subsequent writes page dirties occur _after_ this + snapshot. */ + capsnap->dirty_pages = ci->i_wrbuffer_ref_head; + ci->i_wrbuffer_ref_head = 0; + capsnap->context = old_snapc; + list_add_tail(&capsnap->ci_item, &ci->i_cap_snaps); + + if (used & CEPH_CAP_FILE_WR) { + doutc(cl, "%p %llx.%llx cap_snap %p snapc %p seq %llu used WR," + " now pending\n", inode, ceph_vinop(inode), capsnap, + old_snapc, old_snapc->seq); + capsnap->writing = 1; + } else { + /* note mtime, size NOW. */ + __ceph_finish_cap_snap(ci, capsnap); + } + *pcapsnap = NULL; + old_snapc = NULL; + +update_snapc: + if (ci->i_wrbuffer_ref_head == 0 && + ci->i_wr_ref == 0 && + ci->i_dirty_caps == 0 && + ci->i_flushing_caps == 0) { + ci->i_head_snapc = NULL; + } else { + ci->i_head_snapc = ceph_get_snap_context(new_snapc); + doutc(cl, " new snapc is %p\n", new_snapc); + } + spin_unlock(&ci->i_ceph_lock); + + ceph_buffer_put(old_blob); + ceph_put_snap_context(old_snapc); +} + +/* + * Finalize the size, mtime for a cap_snap.. that is, settle on final values + * to be used for the snapshot, to be flushed back to the mds. + * + * If capsnap can now be flushed, add to snap_flush list, and return 1. + * + * Caller must hold i_ceph_lock. + */ +int __ceph_finish_cap_snap(struct ceph_inode_info *ci, + struct ceph_cap_snap *capsnap) +{ + struct inode *inode = &ci->netfs.inode; + struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb); + struct ceph_client *cl = mdsc->fsc->client; + + BUG_ON(capsnap->writing); + capsnap->size = i_size_read(inode); + capsnap->mtime = inode_get_mtime(inode); + capsnap->atime = inode_get_atime(inode); + capsnap->ctime = inode_get_ctime(inode); + capsnap->btime = ci->i_btime; + capsnap->change_attr = inode_peek_iversion_raw(inode); + capsnap->time_warp_seq = ci->i_time_warp_seq; + capsnap->truncate_size = ci->i_truncate_size; + capsnap->truncate_seq = ci->i_truncate_seq; + if (capsnap->dirty_pages) { + doutc(cl, "%p %llx.%llx cap_snap %p snapc %p %llu %s " + "s=%llu still has %d dirty pages\n", inode, + ceph_vinop(inode), capsnap, capsnap->context, + capsnap->context->seq, + ceph_cap_string(capsnap->dirty), + capsnap->size, capsnap->dirty_pages); + return 0; + } + + /* + * Defer flushing the capsnap if the dirty buffer not flushed yet. + * And trigger to flush the buffer immediately. + */ + if (ci->i_wrbuffer_ref) { + doutc(cl, "%p %llx.%llx cap_snap %p snapc %p %llu %s " + "s=%llu used WRBUFFER, delaying\n", inode, + ceph_vinop(inode), capsnap, capsnap->context, + capsnap->context->seq, ceph_cap_string(capsnap->dirty), + capsnap->size); + ceph_queue_writeback(inode); + return 0; + } + + set_bit(CEPH_I_FLUSH_SNAPS_BIT, &ci->i_ceph_flags); + doutc(cl, "%p %llx.%llx cap_snap %p snapc %p %llu %s s=%llu\n", + inode, ceph_vinop(inode), capsnap, capsnap->context, + capsnap->context->seq, ceph_cap_string(capsnap->dirty), + capsnap->size); + + spin_lock(&mdsc->snap_flush_lock); + if (list_empty(&ci->i_snap_flush_item)) { + ihold(inode); + list_add_tail(&ci->i_snap_flush_item, &mdsc->snap_flush_list); + } + spin_unlock(&mdsc->snap_flush_lock); + return 1; /* caller may want to ceph_flush_snaps */ +} + +/* + * Queue cap_snaps for snap writeback for this realm and its children. + * Called under snap_rwsem, so realm topology won't change. + */ +static void queue_realm_cap_snaps(struct ceph_mds_client *mdsc, + struct ceph_snap_realm *realm) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_inode_info *ci; + struct inode *lastinode = NULL; + struct ceph_cap_snap *capsnap = NULL; + + doutc(cl, "%p %llx inode\n", realm, realm->ino); + + spin_lock(&realm->inodes_with_caps_lock); + list_for_each_entry(ci, &realm->inodes_with_caps, i_snap_realm_item) { + struct inode *inode = igrab(&ci->netfs.inode); + if (!inode) + continue; + spin_unlock(&realm->inodes_with_caps_lock); + iput(lastinode); + lastinode = inode; + + /* + * Allocate the capsnap memory outside of ceph_queue_cap_snap() + * to reduce very possible but unnecessary frequently memory + * allocate/free in this loop. + */ + if (!capsnap) { + capsnap = kmem_cache_zalloc(ceph_cap_snap_cachep, GFP_NOFS); + if (!capsnap) { + pr_err_client(cl, + "ENOMEM allocating ceph_cap_snap on %p\n", + inode); + return; + } + } + capsnap->cap_flush.is_capsnap = true; + refcount_set(&capsnap->nref, 1); + INIT_LIST_HEAD(&capsnap->cap_flush.i_list); + INIT_LIST_HEAD(&capsnap->cap_flush.g_list); + INIT_LIST_HEAD(&capsnap->ci_item); + + ceph_queue_cap_snap(ci, &capsnap); + spin_lock(&realm->inodes_with_caps_lock); + } + spin_unlock(&realm->inodes_with_caps_lock); + iput(lastinode); + + if (capsnap) + kmem_cache_free(ceph_cap_snap_cachep, capsnap); + doutc(cl, "%p %llx done\n", realm, realm->ino); +} + +/* + * Parse and apply a snapblob "snap trace" from the MDS. This specifies + * the snap realm parameters from a given realm and all of its ancestors, + * up to the root. + * + * Caller must hold snap_rwsem for write. + */ +int ceph_update_snap_trace(struct ceph_mds_client *mdsc, + void *p, void *e, bool deletion, + struct ceph_snap_realm **realm_ret) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_mds_snap_realm *ri; /* encoded */ + __le64 *snaps; /* encoded */ + __le64 *prior_parent_snaps; /* encoded */ + struct ceph_snap_realm *realm; + struct ceph_snap_realm *first_realm = NULL; + struct ceph_snap_realm *realm_to_rebuild = NULL; + struct ceph_client *client = mdsc->fsc->client; + int rebuild_snapcs; + int err = -ENOMEM; + int ret; + LIST_HEAD(dirty_realms); + + lockdep_assert_held_write(&mdsc->snap_rwsem); + + doutc(cl, "deletion=%d\n", deletion); +more: + realm = NULL; + rebuild_snapcs = 0; + ceph_decode_need(&p, e, sizeof(*ri), bad); + ri = p; + p += sizeof(*ri); + ceph_decode_need(&p, e, sizeof(u64)*(le32_to_cpu(ri->num_snaps) + + le32_to_cpu(ri->num_prior_parent_snaps)), bad); + snaps = p; + p += sizeof(u64) * le32_to_cpu(ri->num_snaps); + prior_parent_snaps = p; + p += sizeof(u64) * le32_to_cpu(ri->num_prior_parent_snaps); + + realm = ceph_lookup_snap_realm(mdsc, le64_to_cpu(ri->ino)); + if (!realm) { + realm = ceph_create_snap_realm(mdsc, le64_to_cpu(ri->ino)); + if (IS_ERR(realm)) { + err = PTR_ERR(realm); + goto fail; + } + } + + /* ensure the parent is correct */ + err = adjust_snap_realm_parent(mdsc, realm, le64_to_cpu(ri->parent)); + if (err < 0) + goto fail; + rebuild_snapcs += err; + + if (le64_to_cpu(ri->seq) > realm->seq) { + doutc(cl, "updating %llx %p %lld -> %lld\n", realm->ino, + realm, realm->seq, le64_to_cpu(ri->seq)); + /* update realm parameters, snap lists */ + realm->seq = le64_to_cpu(ri->seq); + realm->created = le64_to_cpu(ri->created); + realm->parent_since = le64_to_cpu(ri->parent_since); + + realm->num_snaps = le32_to_cpu(ri->num_snaps); + err = dup_array(&realm->snaps, snaps, realm->num_snaps); + if (err < 0) + goto fail; + + realm->num_prior_parent_snaps = + le32_to_cpu(ri->num_prior_parent_snaps); + err = dup_array(&realm->prior_parent_snaps, prior_parent_snaps, + realm->num_prior_parent_snaps); + if (err < 0) + goto fail; + + if (realm->seq > mdsc->last_snap_seq) + mdsc->last_snap_seq = realm->seq; + + rebuild_snapcs = 1; + } else if (!realm->cached_context) { + doutc(cl, "%llx %p seq %lld new\n", realm->ino, realm, + realm->seq); + rebuild_snapcs = 1; + } else { + doutc(cl, "%llx %p seq %lld unchanged\n", realm->ino, realm, + realm->seq); + } + + doutc(cl, "done with %llx %p, rebuild_snapcs=%d, %p %p\n", realm->ino, + realm, rebuild_snapcs, p, e); + + /* + * this will always track the uppest parent realm from which + * we need to rebuild the snapshot contexts _downward_ in + * hierarchy. + */ + if (rebuild_snapcs) + realm_to_rebuild = realm; + + /* rebuild_snapcs when we reach the _end_ (root) of the trace */ + if (realm_to_rebuild && p >= e) + rebuild_snap_realms(mdsc, realm_to_rebuild, &dirty_realms); + + if (!first_realm) + first_realm = realm; + else + ceph_put_snap_realm(mdsc, realm); + + if (p < e) + goto more; + + /* + * queue cap snaps _after_ we've built the new snap contexts, + * so that i_head_snapc can be set appropriately. + */ + while (!list_empty(&dirty_realms)) { + realm = list_first_entry(&dirty_realms, struct ceph_snap_realm, + dirty_item); + list_del_init(&realm->dirty_item); + queue_realm_cap_snaps(mdsc, realm); + } + + if (realm_ret) + *realm_ret = first_realm; + else + ceph_put_snap_realm(mdsc, first_realm); + + __cleanup_empty_realms(mdsc); + return 0; + +bad: + err = -EIO; +fail: + if (realm && !IS_ERR(realm)) + ceph_put_snap_realm(mdsc, realm); + if (first_realm) + ceph_put_snap_realm(mdsc, first_realm); + pr_err_client(cl, "error %d\n", err); + + /* + * When receiving a corrupted snap trace we don't know what + * exactly has happened in MDS side. And we shouldn't continue + * writing to OSD, which may corrupt the snapshot contents. + * + * Just try to blocklist this kclient and then this kclient + * must be remounted to continue after the corrupted metadata + * fixed in the MDS side. + */ + WRITE_ONCE(mdsc->fsc->mount_state, CEPH_MOUNT_FENCE_IO); + ret = ceph_monc_blocklist_add(&client->monc, &client->msgr.inst.addr); + if (ret) + pr_err_client(cl, "failed to blocklist %s: %d\n", + ceph_pr_addr(&client->msgr.inst.addr), ret); + + WARN(1, "[client.%lld] %s %s%sdo remount to continue%s", + client->monc.auth->global_id, __func__, + ret ? "" : ceph_pr_addr(&client->msgr.inst.addr), + ret ? "" : " was blocklisted, ", + err == -EIO ? " after corrupted snaptrace is fixed" : ""); + + return err; +} + + +/* + * Send any cap_snaps that are queued for flush. Try to carry + * s_mutex across multiple snap flushes to avoid locking overhead. + * + * Caller holds no locks. + */ +static void flush_snaps(struct ceph_mds_client *mdsc) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_inode_info *ci; + struct inode *inode; + struct ceph_mds_session *session = NULL; + + doutc(cl, "begin\n"); + spin_lock(&mdsc->snap_flush_lock); + while (!list_empty(&mdsc->snap_flush_list)) { + ci = list_first_entry(&mdsc->snap_flush_list, + struct ceph_inode_info, i_snap_flush_item); + inode = &ci->netfs.inode; + ihold(inode); + spin_unlock(&mdsc->snap_flush_lock); + ceph_flush_snaps(ci, &session); + iput(inode); + spin_lock(&mdsc->snap_flush_lock); + } + spin_unlock(&mdsc->snap_flush_lock); + + ceph_put_mds_session(session); + doutc(cl, "done\n"); +} + +/** + * ceph_change_snap_realm - change the snap_realm for an inode + * @inode: inode to move to new snap realm + * @realm: new realm to move inode into (may be NULL) + * + * Detach an inode from its old snaprealm (if any) and attach it to + * the new snaprealm (if any). The old snap realm reference held by + * the inode is put. If realm is non-NULL, then the caller's reference + * to it is taken over by the inode. + */ +void ceph_change_snap_realm(struct inode *inode, struct ceph_snap_realm *realm) +{ + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_mds_client *mdsc = ceph_inode_to_fs_client(inode)->mdsc; + struct ceph_snap_realm *oldrealm = ci->i_snap_realm; + + lockdep_assert_held(&ci->i_ceph_lock); + + if (oldrealm) { + spin_lock(&oldrealm->inodes_with_caps_lock); + list_del_init(&ci->i_snap_realm_item); + if (oldrealm->ino == ci->i_vino.ino) + oldrealm->inode = NULL; + spin_unlock(&oldrealm->inodes_with_caps_lock); + ceph_put_snap_realm(mdsc, oldrealm); + } + + ci->i_snap_realm = realm; + + if (realm) { + spin_lock(&realm->inodes_with_caps_lock); + list_add(&ci->i_snap_realm_item, &realm->inodes_with_caps); + if (realm->ino == ci->i_vino.ino) + realm->inode = inode; + spin_unlock(&realm->inodes_with_caps_lock); + } +} + +/* + * Handle a snap notification from the MDS. + * + * This can take two basic forms: the simplest is just a snap creation + * or deletion notification on an existing realm. This should update the + * realm and its children. + * + * The more difficult case is realm creation, due to snap creation at a + * new point in the file hierarchy, or due to a rename that moves a file or + * directory into another realm. + */ +void ceph_handle_snap(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session, + struct ceph_msg *msg) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct super_block *sb = mdsc->fsc->sb; + int mds = session->s_mds; + u64 split; + int op; + int trace_len; + struct ceph_snap_realm *realm = NULL; + void *p = msg->front.iov_base; + void *e = p + msg->front.iov_len; + struct ceph_mds_snap_head *h; + int num_split_inos, num_split_realms; + __le64 *split_inos = NULL, *split_realms = NULL; + int i; + int locked_rwsem = 0; + bool close_sessions = false; + + if (!ceph_inc_mds_stopping_blocker(mdsc, session)) + return; + + /* decode */ + if (msg->front.iov_len < sizeof(*h)) + goto bad; + h = p; + op = le32_to_cpu(h->op); + split = le64_to_cpu(h->split); /* non-zero if we are splitting an + * existing realm */ + num_split_inos = le32_to_cpu(h->num_split_inos); + num_split_realms = le32_to_cpu(h->num_split_realms); + trace_len = le32_to_cpu(h->trace_len); + p += sizeof(*h); + + doutc(cl, "from mds%d op %s split %llx tracelen %d\n", mds, + ceph_snap_op_name(op), split, trace_len); + + down_write(&mdsc->snap_rwsem); + locked_rwsem = 1; + + if (op == CEPH_SNAP_OP_SPLIT) { + struct ceph_mds_snap_realm *ri; + + /* + * A "split" breaks part of an existing realm off into + * a new realm. The MDS provides a list of inodes + * (with caps) and child realms that belong to the new + * child. + */ + split_inos = p; + p += sizeof(u64) * num_split_inos; + split_realms = p; + p += sizeof(u64) * num_split_realms; + ceph_decode_need(&p, e, sizeof(*ri), bad); + /* we will peek at realm info here, but will _not_ + * advance p, as the realm update will occur below in + * ceph_update_snap_trace. */ + ri = p; + + realm = ceph_lookup_snap_realm(mdsc, split); + if (!realm) { + realm = ceph_create_snap_realm(mdsc, split); + if (IS_ERR(realm)) + goto out; + } + + doutc(cl, "splitting snap_realm %llx %p\n", realm->ino, realm); + for (i = 0; i < num_split_inos; i++) { + struct ceph_vino vino = { + .ino = le64_to_cpu(split_inos[i]), + .snap = CEPH_NOSNAP, + }; + struct inode *inode = ceph_find_inode(sb, vino); + struct ceph_inode_info *ci; + + if (!inode) + continue; + ci = ceph_inode(inode); + + spin_lock(&ci->i_ceph_lock); + if (!ci->i_snap_realm) + goto skip_inode; + /* + * If this inode belongs to a realm that was + * created after our new realm, we experienced + * a race (due to another split notifications + * arriving from a different MDS). So skip + * this inode. + */ + if (ci->i_snap_realm->created > + le64_to_cpu(ri->created)) { + doutc(cl, " leaving %p %llx.%llx in newer realm %llx %p\n", + inode, ceph_vinop(inode), ci->i_snap_realm->ino, + ci->i_snap_realm); + goto skip_inode; + } + doutc(cl, " will move %p %llx.%llx to split realm %llx %p\n", + inode, ceph_vinop(inode), realm->ino, realm); + + ceph_get_snap_realm(mdsc, realm); + ceph_change_snap_realm(inode, realm); + spin_unlock(&ci->i_ceph_lock); + iput(inode); + continue; + +skip_inode: + spin_unlock(&ci->i_ceph_lock); + iput(inode); + } + + /* we may have taken some of the old realm's children. */ + for (i = 0; i < num_split_realms; i++) { + struct ceph_snap_realm *child = + __lookup_snap_realm(mdsc, + le64_to_cpu(split_realms[i])); + if (!child) + continue; + adjust_snap_realm_parent(mdsc, child, realm->ino); + } + } else { + /* + * In the non-split case both 'num_split_inos' and + * 'num_split_realms' should be 0, making this a no-op. + * However the MDS happens to populate 'split_realms' list + * in one of the UPDATE op cases by mistake. + * + * Skip both lists just in case to ensure that 'p' is + * positioned at the start of realm info, as expected by + * ceph_update_snap_trace(). + */ + p += sizeof(u64) * num_split_inos; + p += sizeof(u64) * num_split_realms; + } + + /* + * update using the provided snap trace. if we are deleting a + * snap, we can avoid queueing cap_snaps. + */ + if (ceph_update_snap_trace(mdsc, p, e, + op == CEPH_SNAP_OP_DESTROY, + NULL)) { + close_sessions = true; + goto bad; + } + + if (op == CEPH_SNAP_OP_SPLIT) + /* we took a reference when we created the realm, above */ + ceph_put_snap_realm(mdsc, realm); + + __cleanup_empty_realms(mdsc); + + up_write(&mdsc->snap_rwsem); + + flush_snaps(mdsc); + ceph_dec_mds_stopping_blocker(mdsc); + return; + +bad: + pr_err_client(cl, "corrupt snap message from mds%d\n", mds); + ceph_msg_dump(msg); +out: + if (locked_rwsem) + up_write(&mdsc->snap_rwsem); + + ceph_dec_mds_stopping_blocker(mdsc); + + if (close_sessions) + ceph_mdsc_close_sessions(mdsc); + return; +} + +struct ceph_snapid_map* ceph_get_snapid_map(struct ceph_mds_client *mdsc, + u64 snap) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_snapid_map *sm, *exist; + struct rb_node **p, *parent; + int ret; + + exist = NULL; + spin_lock(&mdsc->snapid_map_lock); + p = &mdsc->snapid_map_tree.rb_node; + while (*p) { + exist = rb_entry(*p, struct ceph_snapid_map, node); + if (snap > exist->snap) { + p = &(*p)->rb_left; + } else if (snap < exist->snap) { + p = &(*p)->rb_right; + } else { + if (atomic_inc_return(&exist->ref) == 1) + list_del_init(&exist->lru); + break; + } + exist = NULL; + } + spin_unlock(&mdsc->snapid_map_lock); + if (exist) { + doutc(cl, "found snapid map %llx -> %x\n", exist->snap, + exist->dev); + return exist; + } + + sm = kmalloc_obj(*sm, GFP_NOFS); + if (!sm) + return NULL; + + ret = get_anon_bdev(&sm->dev); + if (ret < 0) { + kfree(sm); + return NULL; + } + + INIT_LIST_HEAD(&sm->lru); + atomic_set(&sm->ref, 1); + sm->snap = snap; + + exist = NULL; + parent = NULL; + p = &mdsc->snapid_map_tree.rb_node; + spin_lock(&mdsc->snapid_map_lock); + while (*p) { + parent = *p; + exist = rb_entry(*p, struct ceph_snapid_map, node); + if (snap > exist->snap) + p = &(*p)->rb_left; + else if (snap < exist->snap) + p = &(*p)->rb_right; + else + break; + exist = NULL; + } + if (exist) { + if (atomic_inc_return(&exist->ref) == 1) + list_del_init(&exist->lru); + } else { + rb_link_node(&sm->node, parent, p); + rb_insert_color(&sm->node, &mdsc->snapid_map_tree); + } + spin_unlock(&mdsc->snapid_map_lock); + if (exist) { + free_anon_bdev(sm->dev); + kfree(sm); + doutc(cl, "found snapid map %llx -> %x\n", exist->snap, + exist->dev); + return exist; + } + + doutc(cl, "create snapid map %llx -> %x\n", sm->snap, sm->dev); + return sm; +} + +void ceph_put_snapid_map(struct ceph_mds_client* mdsc, + struct ceph_snapid_map *sm) +{ + if (!sm) + return; + if (atomic_dec_and_lock(&sm->ref, &mdsc->snapid_map_lock)) { + if (!RB_EMPTY_NODE(&sm->node)) { + sm->last_used = jiffies; + list_add_tail(&sm->lru, &mdsc->snapid_map_lru); + spin_unlock(&mdsc->snapid_map_lock); + } else { + /* already cleaned up by + * ceph_cleanup_snapid_map() */ + spin_unlock(&mdsc->snapid_map_lock); + kfree(sm); + } + } +} + +void ceph_trim_snapid_map(struct ceph_mds_client *mdsc) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_snapid_map *sm; + unsigned long now; + LIST_HEAD(to_free); + + spin_lock(&mdsc->snapid_map_lock); + now = jiffies; + + while (!list_empty(&mdsc->snapid_map_lru)) { + sm = list_first_entry(&mdsc->snapid_map_lru, + struct ceph_snapid_map, lru); + if (time_after(sm->last_used + CEPH_SNAPID_MAP_TIMEOUT, now)) + break; + + rb_erase(&sm->node, &mdsc->snapid_map_tree); + list_move(&sm->lru, &to_free); + } + spin_unlock(&mdsc->snapid_map_lock); + + while (!list_empty(&to_free)) { + sm = list_first_entry(&to_free, struct ceph_snapid_map, lru); + list_del(&sm->lru); + doutc(cl, "trim snapid map %llx -> %x\n", sm->snap, sm->dev); + free_anon_bdev(sm->dev); + kfree(sm); + } +} + +void ceph_cleanup_snapid_map(struct ceph_mds_client *mdsc) +{ + struct ceph_client *cl = mdsc->fsc->client; + struct ceph_snapid_map *sm; + struct rb_node *p; + LIST_HEAD(to_free); + + spin_lock(&mdsc->snapid_map_lock); + while ((p = rb_first(&mdsc->snapid_map_tree))) { + sm = rb_entry(p, struct ceph_snapid_map, node); + rb_erase(p, &mdsc->snapid_map_tree); + RB_CLEAR_NODE(p); + list_move(&sm->lru, &to_free); + } + spin_unlock(&mdsc->snapid_map_lock); + + while (!list_empty(&to_free)) { + sm = list_first_entry(&to_free, struct ceph_snapid_map, lru); + list_del(&sm->lru); + free_anon_bdev(sm->dev); + if (WARN_ON_ONCE(atomic_read(&sm->ref))) { + pr_err_client(cl, "snapid map %llx -> %x still in use\n", + sm->snap, sm->dev); + } + kfree(sm); + } +} diff --git a/fs/ceph/strings.c b/fs/ceph/strings.c new file mode 100644 index 000000000..e36e8948e --- /dev/null +++ b/fs/ceph/strings.c @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Ceph fs string constants + */ +#include +#include + + +const char *ceph_mds_state_name(int s) +{ + switch (s) { + /* down and out */ + case CEPH_MDS_STATE_DNE: return "down:dne"; + case CEPH_MDS_STATE_STOPPED: return "down:stopped"; + /* up and out */ + case CEPH_MDS_STATE_BOOT: return "up:boot"; + case CEPH_MDS_STATE_STANDBY: return "up:standby"; + case CEPH_MDS_STATE_STANDBY_REPLAY: return "up:standby-replay"; + case CEPH_MDS_STATE_REPLAYONCE: return "up:oneshot-replay"; + case CEPH_MDS_STATE_CREATING: return "up:creating"; + case CEPH_MDS_STATE_STARTING: return "up:starting"; + /* up and in */ + case CEPH_MDS_STATE_REPLAY: return "up:replay"; + case CEPH_MDS_STATE_RESOLVE: return "up:resolve"; + case CEPH_MDS_STATE_RECONNECT: return "up:reconnect"; + case CEPH_MDS_STATE_REJOIN: return "up:rejoin"; + case CEPH_MDS_STATE_CLIENTREPLAY: return "up:clientreplay"; + case CEPH_MDS_STATE_ACTIVE: return "up:active"; + case CEPH_MDS_STATE_STOPPING: return "up:stopping"; + } + return "???"; +} + +const char *ceph_session_op_name(int op) +{ + switch (op) { + case CEPH_SESSION_REQUEST_OPEN: return "request_open"; + case CEPH_SESSION_OPEN: return "open"; + case CEPH_SESSION_REQUEST_CLOSE: return "request_close"; + case CEPH_SESSION_CLOSE: return "close"; + case CEPH_SESSION_REQUEST_RENEWCAPS: return "request_renewcaps"; + case CEPH_SESSION_RENEWCAPS: return "renewcaps"; + case CEPH_SESSION_STALE: return "stale"; + case CEPH_SESSION_RECALL_STATE: return "recall_state"; + case CEPH_SESSION_FLUSHMSG: return "flushmsg"; + case CEPH_SESSION_FLUSHMSG_ACK: return "flushmsg_ack"; + case CEPH_SESSION_FORCE_RO: return "force_ro"; + case CEPH_SESSION_REJECT: return "reject"; + case CEPH_SESSION_REQUEST_FLUSH_MDLOG: return "flush_mdlog"; + } + return "???"; +} + +const char *ceph_mds_op_name(int op) +{ + switch (op) { + case CEPH_MDS_OP_LOOKUP: return "lookup"; + case CEPH_MDS_OP_LOOKUPHASH: return "lookuphash"; + case CEPH_MDS_OP_LOOKUPPARENT: return "lookupparent"; + case CEPH_MDS_OP_LOOKUPINO: return "lookupino"; + case CEPH_MDS_OP_LOOKUPNAME: return "lookupname"; + case CEPH_MDS_OP_GETATTR: return "getattr"; + case CEPH_MDS_OP_GETVXATTR: return "getvxattr"; + case CEPH_MDS_OP_SETXATTR: return "setxattr"; + case CEPH_MDS_OP_SETATTR: return "setattr"; + case CEPH_MDS_OP_RMXATTR: return "rmxattr"; + case CEPH_MDS_OP_SETLAYOUT: return "setlayou"; + case CEPH_MDS_OP_SETDIRLAYOUT: return "setdirlayout"; + case CEPH_MDS_OP_READDIR: return "readdir"; + case CEPH_MDS_OP_MKNOD: return "mknod"; + case CEPH_MDS_OP_LINK: return "link"; + case CEPH_MDS_OP_UNLINK: return "unlink"; + case CEPH_MDS_OP_RENAME: return "rename"; + case CEPH_MDS_OP_MKDIR: return "mkdir"; + case CEPH_MDS_OP_RMDIR: return "rmdir"; + case CEPH_MDS_OP_SYMLINK: return "symlink"; + case CEPH_MDS_OP_CREATE: return "create"; + case CEPH_MDS_OP_OPEN: return "open"; + case CEPH_MDS_OP_LOOKUPSNAP: return "lookupsnap"; + case CEPH_MDS_OP_LSSNAP: return "lssnap"; + case CEPH_MDS_OP_MKSNAP: return "mksnap"; + case CEPH_MDS_OP_RMSNAP: return "rmsnap"; + case CEPH_MDS_OP_RENAMESNAP: return "renamesnap"; + case CEPH_MDS_OP_SETFILELOCK: return "setfilelock"; + case CEPH_MDS_OP_GETFILELOCK: return "getfilelock"; + } + return "???"; +} + +const char *ceph_cap_op_name(int op) +{ + switch (op) { + case CEPH_CAP_OP_GRANT: return "grant"; + case CEPH_CAP_OP_REVOKE: return "revoke"; + case CEPH_CAP_OP_TRUNC: return "trunc"; + case CEPH_CAP_OP_EXPORT: return "export"; + case CEPH_CAP_OP_IMPORT: return "import"; + case CEPH_CAP_OP_UPDATE: return "update"; + case CEPH_CAP_OP_DROP: return "drop"; + case CEPH_CAP_OP_FLUSH: return "flush"; + case CEPH_CAP_OP_FLUSH_ACK: return "flush_ack"; + case CEPH_CAP_OP_FLUSHSNAP: return "flushsnap"; + case CEPH_CAP_OP_FLUSHSNAP_ACK: return "flushsnap_ack"; + case CEPH_CAP_OP_RELEASE: return "release"; + case CEPH_CAP_OP_RENEW: return "renew"; + } + return "???"; +} + +const char *ceph_lease_op_name(int o) +{ + switch (o) { + case CEPH_MDS_LEASE_REVOKE: return "revoke"; + case CEPH_MDS_LEASE_RELEASE: return "release"; + case CEPH_MDS_LEASE_RENEW: return "renew"; + case CEPH_MDS_LEASE_REVOKE_ACK: return "revoke_ack"; + } + return "???"; +} + +const char *ceph_snap_op_name(int o) +{ + switch (o) { + case CEPH_SNAP_OP_UPDATE: return "update"; + case CEPH_SNAP_OP_CREATE: return "create"; + case CEPH_SNAP_OP_DESTROY: return "destroy"; + case CEPH_SNAP_OP_SPLIT: return "split"; + } + return "???"; +} diff --git a/fs/ceph/subvolume_metrics.c b/fs/ceph/subvolume_metrics.c new file mode 100644 index 000000000..03fda1f92 --- /dev/null +++ b/fs/ceph/subvolume_metrics.c @@ -0,0 +1,416 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +#include +#include +#include + +#include "subvolume_metrics.h" +#include "mds_client.h" +#include "super.h" + +/** + * struct ceph_subvol_metric_rb_entry - Per-subvolume I/O metrics node + * @node: Red-black tree linkage for tracker->tree + * @subvolume_id: Subvolume identifier (key for rb-tree lookup) + * @read_ops: Accumulated read operation count since last snapshot + * @write_ops: Accumulated write operation count since last snapshot + * @read_bytes: Accumulated bytes read since last snapshot + * @write_bytes: Accumulated bytes written since last snapshot + * @read_latency_us: Sum of read latencies in microseconds + * @write_latency_us: Sum of write latencies in microseconds + */ +struct ceph_subvol_metric_rb_entry { + struct rb_node node; + u64 subvolume_id; + u64 read_ops; + u64 write_ops; + u64 read_bytes; + u64 write_bytes; + u64 read_latency_us; + u64 write_latency_us; +}; + +static struct kmem_cache *ceph_subvol_metric_entry_cachep; + +void ceph_subvolume_metrics_init(struct ceph_subvolume_metrics_tracker *tracker) +{ + spin_lock_init(&tracker->lock); + tracker->tree = RB_ROOT_CACHED; + tracker->nr_entries = 0; + tracker->enabled = false; + atomic64_set(&tracker->snapshot_attempts, 0); + atomic64_set(&tracker->snapshot_empty, 0); + atomic64_set(&tracker->snapshot_failures, 0); + atomic64_set(&tracker->record_calls, 0); + atomic64_set(&tracker->record_disabled, 0); + atomic64_set(&tracker->record_no_subvol, 0); + atomic64_set(&tracker->total_read_ops, 0); + atomic64_set(&tracker->total_read_bytes, 0); + atomic64_set(&tracker->total_write_ops, 0); + atomic64_set(&tracker->total_write_bytes, 0); +} + +static struct ceph_subvol_metric_rb_entry * +__lookup_entry(struct ceph_subvolume_metrics_tracker *tracker, u64 subvol_id) +{ + struct rb_node *node; + + node = tracker->tree.rb_root.rb_node; + while (node) { + struct ceph_subvol_metric_rb_entry *entry = + rb_entry(node, struct ceph_subvol_metric_rb_entry, node); + + if (subvol_id < entry->subvolume_id) + node = node->rb_left; + else if (subvol_id > entry->subvolume_id) + node = node->rb_right; + else + return entry; + } + + return NULL; +} + +static struct ceph_subvol_metric_rb_entry * +__insert_entry(struct ceph_subvolume_metrics_tracker *tracker, + struct ceph_subvol_metric_rb_entry *entry) +{ + struct rb_node **link = &tracker->tree.rb_root.rb_node; + struct rb_node *parent = NULL; + bool leftmost = true; + + while (*link) { + struct ceph_subvol_metric_rb_entry *cur = + rb_entry(*link, struct ceph_subvol_metric_rb_entry, node); + + parent = *link; + if (entry->subvolume_id < cur->subvolume_id) + link = &(*link)->rb_left; + else if (entry->subvolume_id > cur->subvolume_id) { + link = &(*link)->rb_right; + leftmost = false; + } else + return cur; + } + + rb_link_node(&entry->node, parent, link); + rb_insert_color_cached(&entry->node, &tracker->tree, leftmost); + tracker->nr_entries++; + return entry; +} + +static void ceph_subvolume_metrics_clear_locked( + struct ceph_subvolume_metrics_tracker *tracker) +{ + struct rb_node *node = rb_first_cached(&tracker->tree); + + while (node) { + struct ceph_subvol_metric_rb_entry *entry = + rb_entry(node, struct ceph_subvol_metric_rb_entry, node); + struct rb_node *next = rb_next(node); + + rb_erase_cached(&entry->node, &tracker->tree); + tracker->nr_entries--; + kmem_cache_free(ceph_subvol_metric_entry_cachep, entry); + node = next; + } + + tracker->tree = RB_ROOT_CACHED; +} + +void ceph_subvolume_metrics_destroy(struct ceph_subvolume_metrics_tracker *tracker) +{ + spin_lock(&tracker->lock); + ceph_subvolume_metrics_clear_locked(tracker); + tracker->enabled = false; + spin_unlock(&tracker->lock); +} + +void ceph_subvolume_metrics_enable(struct ceph_subvolume_metrics_tracker *tracker, + bool enable) +{ + spin_lock(&tracker->lock); + if (enable) { + tracker->enabled = true; + } else { + tracker->enabled = false; + ceph_subvolume_metrics_clear_locked(tracker); + } + spin_unlock(&tracker->lock); +} + +void ceph_subvolume_metrics_record(struct ceph_subvolume_metrics_tracker *tracker, + u64 subvol_id, bool is_write, + size_t size, u64 latency_us) +{ + struct ceph_subvol_metric_rb_entry *entry, *new_entry = NULL; + bool retry = false; + + /* CEPH_SUBVOLUME_ID_NONE (0) means unknown/unset subvolume */ + if (!READ_ONCE(tracker->enabled) || + subvol_id == CEPH_SUBVOLUME_ID_NONE || !size || !latency_us) + return; + + /* + * Retry loop for lock-free allocation pattern: + * 1. First iteration: lookup under lock, if miss -> drop lock, alloc, retry + * 2. Second iteration: lookup again (may have been inserted), insert if still missing + * 3. On race (another thread inserted same key): free our alloc, retry + * All successful paths exit via return, so retry flag doesn't need reset. + */ + do { + spin_lock(&tracker->lock); + if (!tracker->enabled) { + spin_unlock(&tracker->lock); + if (new_entry) + kmem_cache_free(ceph_subvol_metric_entry_cachep, new_entry); + return; + } + + entry = __lookup_entry(tracker, subvol_id); + if (!entry) { + if (!new_entry) { + spin_unlock(&tracker->lock); + new_entry = kmem_cache_zalloc(ceph_subvol_metric_entry_cachep, + GFP_NOFS); + if (!new_entry) + return; + new_entry->subvolume_id = subvol_id; + retry = true; + continue; + } + entry = __insert_entry(tracker, new_entry); + if (entry != new_entry) { + /* raced with another insert */ + spin_unlock(&tracker->lock); + kmem_cache_free(ceph_subvol_metric_entry_cachep, new_entry); + new_entry = NULL; + retry = true; + continue; + } + new_entry = NULL; + } + + if (is_write) { + entry->write_ops++; + entry->write_bytes += size; + entry->write_latency_us += latency_us; + atomic64_inc(&tracker->total_write_ops); + atomic64_add(size, &tracker->total_write_bytes); + } else { + entry->read_ops++; + entry->read_bytes += size; + entry->read_latency_us += latency_us; + atomic64_inc(&tracker->total_read_ops); + atomic64_add(size, &tracker->total_read_bytes); + } + spin_unlock(&tracker->lock); + if (new_entry) + kmem_cache_free(ceph_subvol_metric_entry_cachep, new_entry); + return; + } while (retry); +} + +int ceph_subvolume_metrics_snapshot(struct ceph_subvolume_metrics_tracker *tracker, + struct ceph_subvol_metric_snapshot **out, + u32 *nr, bool consume) +{ + struct ceph_subvol_metric_snapshot *snap = NULL; + struct rb_node *node; + u32 count = 0, idx = 0; + int ret = 0; + + *out = NULL; + *nr = 0; + + if (!READ_ONCE(tracker->enabled)) + return 0; + + atomic64_inc(&tracker->snapshot_attempts); + + spin_lock(&tracker->lock); + for (node = rb_first_cached(&tracker->tree); node; node = rb_next(node)) { + struct ceph_subvol_metric_rb_entry *entry = + rb_entry(node, struct ceph_subvol_metric_rb_entry, node); + + /* Include entries with ANY I/O activity (read OR write) */ + if (entry->read_ops || entry->write_ops) + count++; + } + spin_unlock(&tracker->lock); + + if (!count) { + atomic64_inc(&tracker->snapshot_empty); + return 0; + } + + snap = kcalloc(count, sizeof(*snap), GFP_NOFS); + if (!snap) { + atomic64_inc(&tracker->snapshot_failures); + return -ENOMEM; + } + + spin_lock(&tracker->lock); + node = rb_first_cached(&tracker->tree); + while (node) { + struct ceph_subvol_metric_rb_entry *entry = + rb_entry(node, struct ceph_subvol_metric_rb_entry, node); + struct rb_node *next = rb_next(node); + + /* Skip entries with NO I/O activity at all */ + if (!entry->read_ops && !entry->write_ops) { + rb_erase_cached(&entry->node, &tracker->tree); + tracker->nr_entries--; + kmem_cache_free(ceph_subvol_metric_entry_cachep, entry); + node = next; + continue; + } + + if (idx >= count) { + pr_warn("ceph: subvol metrics snapshot race (idx=%u count=%u)\n", + idx, count); + break; + } + + snap[idx].subvolume_id = entry->subvolume_id; + snap[idx].read_ops = entry->read_ops; + snap[idx].write_ops = entry->write_ops; + snap[idx].read_bytes = entry->read_bytes; + snap[idx].write_bytes = entry->write_bytes; + snap[idx].read_latency_us = entry->read_latency_us; + snap[idx].write_latency_us = entry->write_latency_us; + idx++; + + if (consume) { + entry->read_ops = 0; + entry->write_ops = 0; + entry->read_bytes = 0; + entry->write_bytes = 0; + entry->read_latency_us = 0; + entry->write_latency_us = 0; + rb_erase_cached(&entry->node, &tracker->tree); + tracker->nr_entries--; + kmem_cache_free(ceph_subvol_metric_entry_cachep, entry); + } + node = next; + } + spin_unlock(&tracker->lock); + + if (!idx) { + kfree(snap); + snap = NULL; + ret = 0; + } else { + *nr = idx; + *out = snap; + } + + return ret; +} + +void ceph_subvolume_metrics_free_snapshot(struct ceph_subvol_metric_snapshot *snapshot) +{ + kfree(snapshot); +} + +/* + * Dump subvolume metrics to a seq_file for debugfs. + * + * Iterates the rb-tree directly under spinlock to avoid allocation. + * The lock hold time is minimal since we're only doing seq_printf calls. + */ +void ceph_subvolume_metrics_dump(struct ceph_subvolume_metrics_tracker *tracker, + struct seq_file *s) +{ + struct rb_node *node; + bool found = false; + + spin_lock(&tracker->lock); + if (!tracker->enabled) { + spin_unlock(&tracker->lock); + seq_puts(s, "subvolume metrics disabled\n"); + return; + } + + for (node = rb_first_cached(&tracker->tree); node; node = rb_next(node)) { + struct ceph_subvol_metric_rb_entry *entry = + rb_entry(node, struct ceph_subvol_metric_rb_entry, node); + u64 avg_rd_lat, avg_wr_lat; + + if (!entry->read_ops && !entry->write_ops) + continue; + + if (!found) { + seq_puts(s, "subvol_id rd_ops rd_bytes rd_avg_lat_us wr_ops wr_bytes wr_avg_lat_us\n"); + seq_puts(s, "------------------------------------------------------------------------------------------------\n"); + found = true; + } + + avg_rd_lat = entry->read_ops ? + div64_u64(entry->read_latency_us, entry->read_ops) : 0; + avg_wr_lat = entry->write_ops ? + div64_u64(entry->write_latency_us, entry->write_ops) : 0; + + seq_printf(s, "%-15llu%-10llu%-12llu%-16llu%-10llu%-12llu%-16llu\n", + entry->subvolume_id, + entry->read_ops, + entry->read_bytes, + avg_rd_lat, + entry->write_ops, + entry->write_bytes, + avg_wr_lat); + } + spin_unlock(&tracker->lock); + + if (!found) + seq_puts(s, "(no subvolume metrics collected)\n"); +} + +void ceph_subvolume_metrics_record_io(struct ceph_mds_client *mdsc, + struct ceph_inode_info *ci, + bool is_write, size_t bytes, + ktime_t start, ktime_t end) +{ + struct ceph_subvolume_metrics_tracker *tracker; + u64 subvol_id; + s64 delta_us; + + if (!mdsc || !ci || !bytes) + return; + + tracker = &mdsc->subvol_metrics; + atomic64_inc(&tracker->record_calls); + + if (!ceph_subvolume_metrics_enabled(tracker)) { + atomic64_inc(&tracker->record_disabled); + return; + } + + subvol_id = READ_ONCE(ci->i_subvolume_id); + if (subvol_id == CEPH_SUBVOLUME_ID_NONE) { + atomic64_inc(&tracker->record_no_subvol); + return; + } + + delta_us = ktime_to_us(ktime_sub(end, start)); + if (delta_us <= 0) + delta_us = 1; + + ceph_subvolume_metrics_record(tracker, subvol_id, is_write, + bytes, (u64)delta_us); +} + +int __init ceph_subvolume_metrics_cache_init(void) +{ + ceph_subvol_metric_entry_cachep = KMEM_CACHE(ceph_subvol_metric_rb_entry, + SLAB_RECLAIM_ACCOUNT); + if (!ceph_subvol_metric_entry_cachep) + return -ENOMEM; + return 0; +} + +void ceph_subvolume_metrics_cache_destroy(void) +{ + kmem_cache_destroy(ceph_subvol_metric_entry_cachep); +} diff --git a/fs/ceph/subvolume_metrics.h b/fs/ceph/subvolume_metrics.h new file mode 100644 index 000000000..6f53ff726 --- /dev/null +++ b/fs/ceph/subvolume_metrics.h @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _FS_CEPH_SUBVOLUME_METRICS_H +#define _FS_CEPH_SUBVOLUME_METRICS_H + +#include +#include +#include +#include +#include + +struct seq_file; +struct ceph_mds_client; +struct ceph_inode_info; + +/** + * struct ceph_subvol_metric_snapshot - Point-in-time snapshot of subvolume metrics + * @subvolume_id: Subvolume identifier (inode number of subvolume root) + * @read_ops: Number of read operations since last snapshot + * @write_ops: Number of write operations since last snapshot + * @read_bytes: Total bytes read since last snapshot + * @write_bytes: Total bytes written since last snapshot + * @read_latency_us: Sum of read latencies in microseconds (for avg calculation) + * @write_latency_us: Sum of write latencies in microseconds (for avg calculation) + */ +struct ceph_subvol_metric_snapshot { + u64 subvolume_id; + u64 read_ops; + u64 write_ops; + u64 read_bytes; + u64 write_bytes; + u64 read_latency_us; + u64 write_latency_us; +}; + +/** + * struct ceph_subvolume_metrics_tracker - Tracks per-subvolume I/O metrics + * @lock: Protects @tree and @nr_entries during concurrent access + * @tree: Red-black tree of per-subvolume entries, keyed by subvolume_id + * @nr_entries: Number of entries currently in @tree + * @enabled: Whether collection is enabled (requires MDS feature support) + * @snapshot_attempts: Debug counter: total ceph_subvolume_metrics_snapshot() calls + * @snapshot_empty: Debug counter: snapshots that found no data to report + * @snapshot_failures: Debug counter: snapshots that failed to allocate memory + * @record_calls: Debug counter: total ceph_subvolume_metrics_record() calls + * @record_disabled: Debug counter: record calls skipped because disabled + * @record_no_subvol: Debug counter: record calls skipped (no subvolume_id) + * @total_read_ops: Cumulative read ops across all snapshots (never reset) + * @total_read_bytes: Cumulative bytes read across all snapshots (never reset) + * @total_write_ops: Cumulative write ops across all snapshots (never reset) + * @total_write_bytes: Cumulative bytes written across all snapshots (never reset) + */ +struct ceph_subvolume_metrics_tracker { + spinlock_t lock; + struct rb_root_cached tree; + u32 nr_entries; + bool enabled; + atomic64_t snapshot_attempts; + atomic64_t snapshot_empty; + atomic64_t snapshot_failures; + atomic64_t record_calls; + atomic64_t record_disabled; + atomic64_t record_no_subvol; + atomic64_t total_read_ops; + atomic64_t total_read_bytes; + atomic64_t total_write_ops; + atomic64_t total_write_bytes; +}; + +void ceph_subvolume_metrics_init(struct ceph_subvolume_metrics_tracker *tracker); +void ceph_subvolume_metrics_destroy(struct ceph_subvolume_metrics_tracker *tracker); +void ceph_subvolume_metrics_enable(struct ceph_subvolume_metrics_tracker *tracker, + bool enable); +void ceph_subvolume_metrics_record(struct ceph_subvolume_metrics_tracker *tracker, + u64 subvol_id, bool is_write, + size_t size, u64 latency_us); +int ceph_subvolume_metrics_snapshot(struct ceph_subvolume_metrics_tracker *tracker, + struct ceph_subvol_metric_snapshot **out, + u32 *nr, bool consume); +void ceph_subvolume_metrics_free_snapshot(struct ceph_subvol_metric_snapshot *snapshot); +void ceph_subvolume_metrics_dump(struct ceph_subvolume_metrics_tracker *tracker, + struct seq_file *s); + +void ceph_subvolume_metrics_record_io(struct ceph_mds_client *mdsc, + struct ceph_inode_info *ci, + bool is_write, size_t bytes, + ktime_t start, ktime_t end); + +static inline bool ceph_subvolume_metrics_enabled( + const struct ceph_subvolume_metrics_tracker *tracker) +{ + return READ_ONCE(tracker->enabled); +} + +int __init ceph_subvolume_metrics_cache_init(void); +void ceph_subvolume_metrics_cache_destroy(void); + +#endif /* _FS_CEPH_SUBVOLUME_METRICS_H */ diff --git a/fs/ceph/super.c b/fs/ceph/super.c new file mode 100644 index 000000000..c05fbd423 --- /dev/null +++ b/fs/ceph/super.c @@ -0,0 +1,1716 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "super.h" +#include "mds_client.h" +#include "cache.h" +#include "crypto.h" +#include "subvolume_metrics.h" + +#include +#include +#include +#include +#include + +#include + +#define CREATE_TRACE_POINTS +#include + +static DEFINE_SPINLOCK(ceph_fsc_lock); +static LIST_HEAD(ceph_fsc_list); + +/* + * Ceph superblock operations + * + * Handle the basics of mounting, unmounting. + */ + +/* + * super ops + */ +static void ceph_put_super(struct super_block *s) +{ + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(s); + + doutc(fsc->client, "begin\n"); + ceph_fscrypt_free_dummy_policy(fsc); + ceph_mdsc_close_sessions(fsc->mdsc); + doutc(fsc->client, "done\n"); +} + +static int ceph_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(d_inode(dentry)); + struct ceph_mon_client *monc = &fsc->client->monc; + struct ceph_statfs st; + int i, err; + u64 data_pool; + + doutc(fsc->client, "begin\n"); + if (fsc->mdsc->mdsmap->m_num_data_pg_pools == 1) { + data_pool = fsc->mdsc->mdsmap->m_data_pg_pools[0]; + } else { + data_pool = CEPH_NOPOOL; + } + + err = ceph_monc_do_statfs(monc, data_pool, &st); + if (err < 0) + return err; + + /* fill in kstatfs */ + buf->f_type = CEPH_SUPER_MAGIC; /* ?? */ + + /* + * Express utilization in terms of large blocks to avoid + * overflow on 32-bit machines. + */ + buf->f_frsize = 1 << CEPH_BLOCK_SHIFT; + + /* + * By default use root quota for stats; fallback to overall filesystem + * usage if using 'noquotadf' mount option or if the root dir doesn't + * have max_bytes quota set. + */ + if (ceph_test_mount_opt(fsc, NOQUOTADF) || + !ceph_quota_update_statfs(fsc, buf)) { + buf->f_blocks = le64_to_cpu(st.kb) >> (CEPH_BLOCK_SHIFT-10); + buf->f_bfree = le64_to_cpu(st.kb_avail) >> (CEPH_BLOCK_SHIFT-10); + buf->f_bavail = le64_to_cpu(st.kb_avail) >> (CEPH_BLOCK_SHIFT-10); + } + + /* + * NOTE: for the time being, we make bsize == frsize to humor + * not-yet-ancient versions of glibc that are broken. + * Someday, we will probably want to report a real block + * size... whatever that may mean for a network file system! + */ + buf->f_bsize = buf->f_frsize; + + buf->f_files = le64_to_cpu(st.num_objects); + buf->f_ffree = -1; + buf->f_namelen = NAME_MAX; + + /* Must convert the fsid, for consistent values across arches */ + buf->f_fsid.val[0] = 0; + mutex_lock(&monc->mutex); + for (i = 0 ; i < sizeof(monc->monmap->fsid) / sizeof(__le32) ; ++i) + buf->f_fsid.val[0] ^= le32_to_cpu(((__le32 *)&monc->monmap->fsid)[i]); + mutex_unlock(&monc->mutex); + + /* fold the fs_cluster_id into the upper bits */ + buf->f_fsid.val[1] = monc->fs_cluster_id; + + doutc(fsc->client, "done\n"); + return 0; +} + +static int ceph_sync_fs(struct super_block *sb, int wait) +{ + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(sb); + struct ceph_client *cl = fsc->client; + + if (!wait) { + doutc(cl, "(non-blocking)\n"); + ceph_flush_dirty_caps(fsc->mdsc); + ceph_flush_cap_releases(fsc->mdsc); + doutc(cl, "(non-blocking) done\n"); + return 0; + } + + doutc(cl, "(blocking)\n"); + ceph_osdc_sync(&fsc->client->osdc); + ceph_mdsc_sync(fsc->mdsc); + doutc(cl, "(blocking) done\n"); + return 0; +} + +/* + * mount options + */ +enum { + Opt_wsize, + Opt_rsize, + Opt_rasize, + Opt_caps_wanted_delay_min, + Opt_caps_wanted_delay_max, + Opt_caps_max, + Opt_readdir_max_entries, + Opt_readdir_max_bytes, + Opt_congestion_kb, + /* int args above */ + Opt_snapdirname, + Opt_mds_namespace, + Opt_recover_session, + Opt_source, + Opt_mon_addr, + Opt_test_dummy_encryption, + /* string args above */ + Opt_dirstat, + Opt_rbytes, + Opt_asyncreaddir, + Opt_dcache, + Opt_ino32, + Opt_fscache, + Opt_poolperm, + Opt_require_active_mds, + Opt_acl, + Opt_quotadf, + Opt_copyfrom, + Opt_wsync, + Opt_pagecache, + Opt_sparseread, +}; + +enum ceph_recover_session_mode { + ceph_recover_session_no, + ceph_recover_session_clean +}; + +static const struct constant_table ceph_param_recover[] = { + { "no", ceph_recover_session_no }, + { "clean", ceph_recover_session_clean }, + {} +}; + +static const struct fs_parameter_spec ceph_mount_parameters[] = { + fsparam_flag_no ("acl", Opt_acl), + fsparam_flag_no ("asyncreaddir", Opt_asyncreaddir), + fsparam_s32 ("caps_max", Opt_caps_max), + fsparam_u32 ("caps_wanted_delay_max", Opt_caps_wanted_delay_max), + fsparam_u32 ("caps_wanted_delay_min", Opt_caps_wanted_delay_min), + fsparam_u32 ("write_congestion_kb", Opt_congestion_kb), + fsparam_flag_no ("copyfrom", Opt_copyfrom), + fsparam_flag_no ("dcache", Opt_dcache), + fsparam_flag_no ("dirstat", Opt_dirstat), + fsparam_flag_no ("fsc", Opt_fscache), // fsc|nofsc + fsparam_string ("fsc", Opt_fscache), // fsc=... + fsparam_flag_no ("ino32", Opt_ino32), + fsparam_string ("mds_namespace", Opt_mds_namespace), + fsparam_string ("mon_addr", Opt_mon_addr), + fsparam_flag_no ("poolperm", Opt_poolperm), + fsparam_flag_no ("quotadf", Opt_quotadf), + fsparam_u32 ("rasize", Opt_rasize), + fsparam_flag_no ("rbytes", Opt_rbytes), + fsparam_u32 ("readdir_max_bytes", Opt_readdir_max_bytes), + fsparam_u32 ("readdir_max_entries", Opt_readdir_max_entries), + fsparam_enum ("recover_session", Opt_recover_session, ceph_param_recover), + fsparam_flag_no ("require_active_mds", Opt_require_active_mds), + fsparam_u32 ("rsize", Opt_rsize), + fsparam_string ("snapdirname", Opt_snapdirname), + fsparam_string ("source", Opt_source), + fsparam_flag ("test_dummy_encryption", Opt_test_dummy_encryption), + fsparam_string ("test_dummy_encryption", Opt_test_dummy_encryption), + fsparam_u32 ("wsize", Opt_wsize), + fsparam_flag_no ("wsync", Opt_wsync), + fsparam_flag_no ("pagecache", Opt_pagecache), + fsparam_flag_no ("sparseread", Opt_sparseread), + {} +}; + +struct ceph_parse_opts_ctx { + struct ceph_options *copts; + struct ceph_mount_options *opts; +}; + +/* + * Remove adjacent slashes and then the trailing slash, unless it is + * the only remaining character. + * + * E.g. "//dir1////dir2///" --> "/dir1/dir2", "///" --> "/". + */ +static void canonicalize_path(char *path) +{ + int i, j = 0; + + for (i = 0; path[i] != '\0'; i++) { + if (path[i] != '/' || j < 1 || path[j - 1] != '/') + path[j++] = path[i]; + } + + if (j > 1 && path[j - 1] == '/') + j--; + path[j] = '\0'; +} + +static int ceph_parse_old_source(const char *dev_name, const char *dev_name_end, + struct fs_context *fc) +{ + int r; + struct ceph_parse_opts_ctx *pctx = fc->fs_private; + struct ceph_mount_options *fsopt = pctx->opts; + + if (*dev_name_end != ':') + return invalfc(fc, "separator ':' missing in source"); + + r = ceph_parse_mon_ips(dev_name, dev_name_end - dev_name, + pctx->copts, fc->log.log, ','); + if (r) + return r; + + fsopt->new_dev_syntax = false; + return 0; +} + +static int ceph_parse_new_source(const char *dev_name, const char *dev_name_end, + struct fs_context *fc) +{ + size_t len; + struct ceph_fsid fsid; + struct ceph_parse_opts_ctx *pctx = fc->fs_private; + struct ceph_options *opts = pctx->copts; + struct ceph_mount_options *fsopt = pctx->opts; + const char *name_start = dev_name; + const char *fsid_start, *fs_name_start; + + if (*dev_name_end != '=') { + dout("separator '=' missing in source"); + return -EINVAL; + } + + fsid_start = strchr(dev_name, '@'); + if (!fsid_start) + return invalfc(fc, "missing cluster fsid"); + len = fsid_start - name_start; + kfree(opts->name); + opts->name = kstrndup(name_start, len, GFP_KERNEL); + if (!opts->name) + return -ENOMEM; + dout("using %s entity name", opts->name); + + ++fsid_start; /* start of cluster fsid */ + fs_name_start = strchr(fsid_start, '.'); + if (!fs_name_start) + return invalfc(fc, "missing file system name"); + + if (ceph_parse_fsid(fsid_start, &fsid)) + return invalfc(fc, "Invalid FSID"); + + ++fs_name_start; /* start of file system name */ + len = dev_name_end - fs_name_start; + + if (!namespace_equals(fsopt, fs_name_start, len)) + return invalfc(fc, "Mismatching mds_namespace"); + kfree(fsopt->mds_namespace); + fsopt->mds_namespace = kstrndup(fs_name_start, len, GFP_KERNEL); + if (!fsopt->mds_namespace) + return -ENOMEM; + dout("file system (mds namespace) '%s'\n", fsopt->mds_namespace); + + fsopt->new_dev_syntax = true; + return 0; +} + +/* + * Parse the source parameter for new device format. Distinguish the device + * spec from the path. Try parsing new device format and fallback to old + * format if needed. + * + * New device syntax will looks like: + * =/ + * where + * is name@fsid.fsname + * is optional, but if present must begin with '/' + * (monitor addresses are passed via mount option) + * + * Old device syntax is: + * [,...]:[] + * where + * is [:] + * is optional, but if present must begin with '/' + */ +static int ceph_parse_source(struct fs_parameter *param, struct fs_context *fc) +{ + struct ceph_parse_opts_ctx *pctx = fc->fs_private; + struct ceph_mount_options *fsopt = pctx->opts; + char *dev_name = param->string, *dev_name_end; + int ret; + + dout("'%s'\n", dev_name); + if (!dev_name || !*dev_name) + return invalfc(fc, "Empty source"); + + dev_name_end = strchr(dev_name, '/'); + if (dev_name_end) { + /* + * The server_path will include the whole chars from userland + * including the leading '/'. + */ + kfree(fsopt->server_path); + fsopt->server_path = kstrdup(dev_name_end, GFP_KERNEL); + if (!fsopt->server_path) + return -ENOMEM; + + canonicalize_path(fsopt->server_path); + } else { + dev_name_end = dev_name + strlen(dev_name); + } + + dev_name_end--; /* back up to separator */ + if (dev_name_end < dev_name) + return invalfc(fc, "Path missing in source"); + + dout("device name '%.*s'\n", (int)(dev_name_end - dev_name), dev_name); + if (fsopt->server_path) + dout("server path '%s'\n", fsopt->server_path); + + dout("trying new device syntax"); + ret = ceph_parse_new_source(dev_name, dev_name_end, fc); + if (ret) { + if (ret != -EINVAL) + return ret; + dout("trying old device syntax"); + ret = ceph_parse_old_source(dev_name, dev_name_end, fc); + if (ret) + return ret; + } + + fc->source = param->string; + param->string = NULL; + return 0; +} + +static int ceph_parse_mon_addr(struct fs_parameter *param, + struct fs_context *fc) +{ + struct ceph_parse_opts_ctx *pctx = fc->fs_private; + struct ceph_mount_options *fsopt = pctx->opts; + + kfree(fsopt->mon_addr); + fsopt->mon_addr = param->string; + param->string = NULL; + + return ceph_parse_mon_ips(fsopt->mon_addr, strlen(fsopt->mon_addr), + pctx->copts, fc->log.log, '/'); +} + +static int ceph_parse_mount_param(struct fs_context *fc, + struct fs_parameter *param) +{ + struct ceph_parse_opts_ctx *pctx = fc->fs_private; + struct ceph_mount_options *fsopt = pctx->opts; + struct fs_parse_result result; + unsigned int mode; + int token, ret; + + ret = ceph_parse_param(param, pctx->copts, fc->log.log); + if (ret != -ENOPARAM) + return ret; + + token = fs_parse(fc, ceph_mount_parameters, param, &result); + dout("%s: fs_parse '%s' token %d\n",__func__, param->key, token); + if (token < 0) + return token; + + switch (token) { + case Opt_snapdirname: + if (strlen(param->string) > NAME_MAX) + return invalfc(fc, "snapdirname too long"); + kfree(fsopt->snapdir_name); + fsopt->snapdir_name = param->string; + param->string = NULL; + break; + case Opt_mds_namespace: + if (!namespace_equals(fsopt, param->string, strlen(param->string))) + return invalfc(fc, "Mismatching mds_namespace"); + kfree(fsopt->mds_namespace); + fsopt->mds_namespace = param->string; + param->string = NULL; + break; + case Opt_recover_session: + mode = result.uint_32; + if (mode == ceph_recover_session_no) + fsopt->flags &= ~CEPH_MOUNT_OPT_CLEANRECOVER; + else if (mode == ceph_recover_session_clean) + fsopt->flags |= CEPH_MOUNT_OPT_CLEANRECOVER; + else + BUG(); + break; + case Opt_source: + if (fc->source) + return invalfc(fc, "Multiple sources specified"); + return ceph_parse_source(param, fc); + case Opt_mon_addr: + return ceph_parse_mon_addr(param, fc); + case Opt_wsize: + if (result.uint_32 < PAGE_SIZE || + result.uint_32 > CEPH_MAX_WRITE_SIZE) + goto out_of_range; + fsopt->wsize = ALIGN(result.uint_32, PAGE_SIZE); + break; + case Opt_rsize: + if (result.uint_32 < PAGE_SIZE || + result.uint_32 > CEPH_MAX_READ_SIZE) + goto out_of_range; + fsopt->rsize = ALIGN(result.uint_32, PAGE_SIZE); + break; + case Opt_rasize: + fsopt->rasize = ALIGN(result.uint_32, PAGE_SIZE); + break; + case Opt_caps_wanted_delay_min: + if (result.uint_32 < 1) + goto out_of_range; + fsopt->caps_wanted_delay_min = result.uint_32; + break; + case Opt_caps_wanted_delay_max: + if (result.uint_32 < 1) + goto out_of_range; + fsopt->caps_wanted_delay_max = result.uint_32; + break; + case Opt_caps_max: + if (result.int_32 < 0) + goto out_of_range; + fsopt->caps_max = result.int_32; + break; + case Opt_readdir_max_entries: + if (result.uint_32 < 1) + goto out_of_range; + fsopt->max_readdir = result.uint_32; + break; + case Opt_readdir_max_bytes: + if (result.uint_32 < PAGE_SIZE && result.uint_32 != 0) + goto out_of_range; + fsopt->max_readdir_bytes = result.uint_32; + break; + case Opt_congestion_kb: + if (result.uint_32 < 1024) /* at least 1M */ + goto out_of_range; + fsopt->congestion_kb = result.uint_32; + break; + case Opt_dirstat: + if (!result.negated) + fsopt->flags |= CEPH_MOUNT_OPT_DIRSTAT; + else + fsopt->flags &= ~CEPH_MOUNT_OPT_DIRSTAT; + break; + case Opt_rbytes: + if (!result.negated) + fsopt->flags |= CEPH_MOUNT_OPT_RBYTES; + else + fsopt->flags &= ~CEPH_MOUNT_OPT_RBYTES; + break; + case Opt_asyncreaddir: + if (!result.negated) + fsopt->flags &= ~CEPH_MOUNT_OPT_NOASYNCREADDIR; + else + fsopt->flags |= CEPH_MOUNT_OPT_NOASYNCREADDIR; + break; + case Opt_dcache: + if (!result.negated) + fsopt->flags |= CEPH_MOUNT_OPT_DCACHE; + else + fsopt->flags &= ~CEPH_MOUNT_OPT_DCACHE; + break; + case Opt_ino32: + if (!result.negated) + fsopt->flags |= CEPH_MOUNT_OPT_INO32; + else + fsopt->flags &= ~CEPH_MOUNT_OPT_INO32; + break; + + case Opt_fscache: +#ifdef CONFIG_CEPH_FSCACHE + kfree(fsopt->fscache_uniq); + fsopt->fscache_uniq = NULL; + if (result.negated) { + fsopt->flags &= ~CEPH_MOUNT_OPT_FSCACHE; + } else { + fsopt->flags |= CEPH_MOUNT_OPT_FSCACHE; + fsopt->fscache_uniq = param->string; + param->string = NULL; + } + break; +#else + return invalfc(fc, "fscache support is disabled"); +#endif + case Opt_poolperm: + if (!result.negated) + fsopt->flags &= ~CEPH_MOUNT_OPT_NOPOOLPERM; + else + fsopt->flags |= CEPH_MOUNT_OPT_NOPOOLPERM; + break; + case Opt_require_active_mds: + if (!result.negated) + fsopt->flags &= ~CEPH_MOUNT_OPT_MOUNTWAIT; + else + fsopt->flags |= CEPH_MOUNT_OPT_MOUNTWAIT; + break; + case Opt_quotadf: + if (!result.negated) + fsopt->flags &= ~CEPH_MOUNT_OPT_NOQUOTADF; + else + fsopt->flags |= CEPH_MOUNT_OPT_NOQUOTADF; + break; + case Opt_copyfrom: + if (!result.negated) + fsopt->flags &= ~CEPH_MOUNT_OPT_NOCOPYFROM; + else + fsopt->flags |= CEPH_MOUNT_OPT_NOCOPYFROM; + break; + case Opt_acl: + if (!result.negated) { +#ifdef CONFIG_CEPH_FS_POSIX_ACL + fc->sb_flags |= SB_POSIXACL; +#else + return invalfc(fc, "POSIX ACL support is disabled"); +#endif + } else { + fc->sb_flags &= ~SB_POSIXACL; + } + break; + case Opt_wsync: + if (!result.negated) + fsopt->flags &= ~CEPH_MOUNT_OPT_ASYNC_DIROPS; + else + fsopt->flags |= CEPH_MOUNT_OPT_ASYNC_DIROPS; + break; + case Opt_pagecache: + if (result.negated) + fsopt->flags |= CEPH_MOUNT_OPT_NOPAGECACHE; + else + fsopt->flags &= ~CEPH_MOUNT_OPT_NOPAGECACHE; + break; + case Opt_sparseread: + if (result.negated) + fsopt->flags &= ~CEPH_MOUNT_OPT_SPARSEREAD; + else + fsopt->flags |= CEPH_MOUNT_OPT_SPARSEREAD; + break; + case Opt_test_dummy_encryption: +#ifdef CONFIG_FS_ENCRYPTION + fscrypt_free_dummy_policy(&fsopt->dummy_enc_policy); + ret = fscrypt_parse_test_dummy_encryption(param, + &fsopt->dummy_enc_policy); + if (ret == -EINVAL) { + warnfc(fc, "Value of option \"%s\" is unrecognized", + param->key); + } else if (ret == -EEXIST) { + warnfc(fc, "Conflicting test_dummy_encryption options"); + ret = -EINVAL; + } +#else + warnfc(fc, + "FS encryption not supported: test_dummy_encryption mount option ignored"); +#endif + break; + default: + BUG(); + } + return 0; + +out_of_range: + return invalfc(fc, "%s out of range", param->key); +} + +static void destroy_mount_options(struct ceph_mount_options *args) +{ + dout("destroy_mount_options %p\n", args); + if (!args) + return; + + kfree(args->snapdir_name); + kfree(args->mds_namespace); + kfree(args->server_path); + kfree(args->fscache_uniq); + kfree(args->mon_addr); + fscrypt_free_dummy_policy(&args->dummy_enc_policy); + kfree(args); +} + +static int strcmp_null(const char *s1, const char *s2) +{ + if (!s1 && !s2) + return 0; + if (s1 && !s2) + return -1; + if (!s1 && s2) + return 1; + return strcmp(s1, s2); +} + +static int compare_mount_options(struct ceph_mount_options *new_fsopt, + struct ceph_options *new_opt, + struct ceph_fs_client *fsc) +{ + struct ceph_mount_options *fsopt1 = new_fsopt; + struct ceph_mount_options *fsopt2 = fsc->mount_options; + int ofs = offsetof(struct ceph_mount_options, snapdir_name); + int ret; + + ret = memcmp(fsopt1, fsopt2, ofs); + if (ret) + return ret; + + ret = strcmp_null(fsopt1->snapdir_name, fsopt2->snapdir_name); + if (ret) + return ret; + + ret = strcmp_null(fsopt1->mds_namespace, fsopt2->mds_namespace); + if (ret) + return ret; + + ret = strcmp_null(fsopt1->server_path, fsopt2->server_path); + if (ret) + return ret; + + ret = strcmp_null(fsopt1->fscache_uniq, fsopt2->fscache_uniq); + if (ret) + return ret; + + ret = strcmp_null(fsopt1->mon_addr, fsopt2->mon_addr); + if (ret) + return ret; + + return ceph_compare_options(new_opt, fsc->client); +} + +/** + * ceph_show_options - Show mount options in /proc/mounts + * @m: seq_file to write to + * @root: root of that (sub)tree + */ +static int ceph_show_options(struct seq_file *m, struct dentry *root) +{ + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(root->d_sb); + struct ceph_mount_options *fsopt = fsc->mount_options; + size_t pos; + int ret; + + /* a comma between MNT/MS and client options */ + seq_putc(m, ','); + pos = m->count; + + ret = ceph_print_client_options(m, fsc->client, false); + if (ret) + return ret; + + /* retract our comma if no client options */ + if (m->count == pos) + m->count--; + + if (fsopt->flags & CEPH_MOUNT_OPT_DIRSTAT) + seq_puts(m, ",dirstat"); + if ((fsopt->flags & CEPH_MOUNT_OPT_RBYTES)) + seq_puts(m, ",rbytes"); + if (fsopt->flags & CEPH_MOUNT_OPT_NOASYNCREADDIR) + seq_puts(m, ",noasyncreaddir"); + if ((fsopt->flags & CEPH_MOUNT_OPT_DCACHE) == 0) + seq_puts(m, ",nodcache"); + if (fsopt->flags & CEPH_MOUNT_OPT_INO32) + seq_puts(m, ",ino32"); + if (fsopt->flags & CEPH_MOUNT_OPT_FSCACHE) { + seq_show_option(m, "fsc", fsopt->fscache_uniq); + } + if (fsopt->flags & CEPH_MOUNT_OPT_NOPOOLPERM) + seq_puts(m, ",nopoolperm"); + if (fsopt->flags & CEPH_MOUNT_OPT_NOQUOTADF) + seq_puts(m, ",noquotadf"); + +#ifdef CONFIG_CEPH_FS_POSIX_ACL + if (root->d_sb->s_flags & SB_POSIXACL) + seq_puts(m, ",acl"); + else + seq_puts(m, ",noacl"); +#endif + + if ((fsopt->flags & CEPH_MOUNT_OPT_NOCOPYFROM) == 0) + seq_puts(m, ",copyfrom"); + + /* dump mds_namespace when old device syntax is in use */ + if (fsopt->mds_namespace && !fsopt->new_dev_syntax) + seq_show_option(m, "mds_namespace", fsopt->mds_namespace); + + if (fsopt->mon_addr) + seq_printf(m, ",mon_addr=%s", fsopt->mon_addr); + + if (fsopt->flags & CEPH_MOUNT_OPT_CLEANRECOVER) + seq_show_option(m, "recover_session", "clean"); + + if (!(fsopt->flags & CEPH_MOUNT_OPT_ASYNC_DIROPS)) + seq_puts(m, ",wsync"); + if (fsopt->flags & CEPH_MOUNT_OPT_NOPAGECACHE) + seq_puts(m, ",nopagecache"); + if (fsopt->flags & CEPH_MOUNT_OPT_SPARSEREAD) + seq_puts(m, ",sparseread"); + + fscrypt_show_test_dummy_encryption(m, ',', root->d_sb); + + if (fsopt->wsize != CEPH_MAX_WRITE_SIZE) + seq_printf(m, ",wsize=%u", fsopt->wsize); + if (fsopt->rsize != CEPH_MAX_READ_SIZE) + seq_printf(m, ",rsize=%u", fsopt->rsize); + if (fsopt->rasize != CEPH_RASIZE_DEFAULT) + seq_printf(m, ",rasize=%u", fsopt->rasize); + if (fsopt->congestion_kb != default_congestion_kb()) + seq_printf(m, ",write_congestion_kb=%u", fsopt->congestion_kb); + if (fsopt->caps_max) + seq_printf(m, ",caps_max=%d", fsopt->caps_max); + if (fsopt->caps_wanted_delay_min != CEPH_CAPS_WANTED_DELAY_MIN_DEFAULT) + seq_printf(m, ",caps_wanted_delay_min=%u", + fsopt->caps_wanted_delay_min); + if (fsopt->caps_wanted_delay_max != CEPH_CAPS_WANTED_DELAY_MAX_DEFAULT) + seq_printf(m, ",caps_wanted_delay_max=%u", + fsopt->caps_wanted_delay_max); + if (fsopt->max_readdir != CEPH_MAX_READDIR_DEFAULT) + seq_printf(m, ",readdir_max_entries=%u", fsopt->max_readdir); + if (fsopt->max_readdir_bytes != CEPH_MAX_READDIR_BYTES_DEFAULT) + seq_printf(m, ",readdir_max_bytes=%u", fsopt->max_readdir_bytes); + if (strcmp(fsopt->snapdir_name, CEPH_SNAPDIRNAME_DEFAULT)) + seq_show_option(m, "snapdirname", fsopt->snapdir_name); + + return 0; +} + +/* + * handle any mon messages the standard library doesn't understand. + * return error if we don't either. + */ +static int extra_mon_dispatch(struct ceph_client *client, struct ceph_msg *msg) +{ + struct ceph_fs_client *fsc = client->private; + int type = le16_to_cpu(msg->hdr.type); + + switch (type) { + case CEPH_MSG_MDS_MAP: + ceph_mdsc_handle_mdsmap(fsc->mdsc, msg); + return 0; + case CEPH_MSG_FS_MAP_USER: + ceph_mdsc_handle_fsmap(fsc->mdsc, msg); + return 0; + default: + return -1; + } +} + +/* + * create a new fs client + * + * Success or not, this function consumes @fsopt and @opt. + */ +static struct ceph_fs_client *create_fs_client(struct ceph_mount_options *fsopt, + struct ceph_options *opt) +{ + struct ceph_fs_client *fsc; + int err; + + fsc = kzalloc_obj(*fsc); + if (!fsc) { + err = -ENOMEM; + goto fail; + } + + fsc->client = ceph_create_client(opt, fsc); + if (IS_ERR(fsc->client)) { + err = PTR_ERR(fsc->client); + goto fail; + } + opt = NULL; /* fsc->client now owns this */ + + fsc->client->extra_mon_dispatch = extra_mon_dispatch; + ceph_set_opt(fsc->client, ABORT_ON_FULL); + + if (!fsopt->mds_namespace) { + ceph_monc_want_map(&fsc->client->monc, CEPH_SUB_MDSMAP, + 0, true); + } else { + ceph_monc_want_map(&fsc->client->monc, CEPH_SUB_FSMAP, + 0, false); + } + + fsc->mount_options = fsopt; + + fsc->sb = NULL; + fsc->mount_state = CEPH_MOUNT_MOUNTING; + fsc->filp_gen = 1; + fsc->have_copy_from2 = true; + + atomic_long_set(&fsc->writeback_count, 0); + fsc->write_congested = false; + + err = -ENOMEM; + /* + * The number of concurrent works can be high but they don't need + * to be processed in parallel, limit concurrency. + */ + fsc->inode_wq = alloc_workqueue("ceph-inode", WQ_UNBOUND, 0); + if (!fsc->inode_wq) + goto fail_client; + fsc->cap_wq = alloc_workqueue("ceph-cap", WQ_PERCPU, 1); + if (!fsc->cap_wq) + goto fail_inode_wq; + + hash_init(fsc->async_unlink_conflict); + spin_lock_init(&fsc->async_unlink_conflict_lock); + + spin_lock(&ceph_fsc_lock); + list_add_tail(&fsc->metric_wakeup, &ceph_fsc_list); + spin_unlock(&ceph_fsc_lock); + + return fsc; + +fail_inode_wq: + destroy_workqueue(fsc->inode_wq); +fail_client: + ceph_destroy_client(fsc->client); +fail: + kfree(fsc); + if (opt) + ceph_destroy_options(opt); + destroy_mount_options(fsopt); + return ERR_PTR(err); +} + +static void flush_fs_workqueues(struct ceph_fs_client *fsc) +{ + flush_workqueue(fsc->inode_wq); + flush_workqueue(fsc->cap_wq); +} + +static void destroy_fs_client(struct ceph_fs_client *fsc) +{ + doutc(fsc->client, "%p\n", fsc); + + spin_lock(&ceph_fsc_lock); + list_del(&fsc->metric_wakeup); + spin_unlock(&ceph_fsc_lock); + + ceph_mdsc_destroy(fsc); + destroy_workqueue(fsc->inode_wq); + destroy_workqueue(fsc->cap_wq); + + destroy_mount_options(fsc->mount_options); + + ceph_destroy_client(fsc->client); + + kfree(fsc); + dout("%s: %p done\n", __func__, fsc); +} + +/* + * caches + */ +struct kmem_cache *ceph_inode_cachep; +struct kmem_cache *ceph_cap_cachep; +struct kmem_cache *ceph_cap_snap_cachep; +struct kmem_cache *ceph_cap_flush_cachep; +struct kmem_cache *ceph_dentry_cachep; +struct kmem_cache *ceph_file_cachep; +struct kmem_cache *ceph_dir_file_cachep; +struct kmem_cache *ceph_mds_request_cachep; +mempool_t *ceph_wb_pagevec_pool; + +static void ceph_inode_init_once(void *foo) +{ + struct ceph_inode_info *ci = foo; + inode_init_once(&ci->netfs.inode); +} + +static int __init init_caches(void) +{ + int error = -ENOMEM; + + ceph_inode_cachep = kmem_cache_create("ceph_inode_info", + sizeof(struct ceph_inode_info), + __alignof__(struct ceph_inode_info), + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, + ceph_inode_init_once); + if (!ceph_inode_cachep) + return -ENOMEM; + + ceph_cap_cachep = KMEM_CACHE(ceph_cap, 0); + if (!ceph_cap_cachep) + goto bad_cap; + ceph_cap_snap_cachep = KMEM_CACHE(ceph_cap_snap, 0); + if (!ceph_cap_snap_cachep) + goto bad_cap_snap; + ceph_cap_flush_cachep = KMEM_CACHE(ceph_cap_flush, + SLAB_RECLAIM_ACCOUNT); + if (!ceph_cap_flush_cachep) + goto bad_cap_flush; + + ceph_dentry_cachep = KMEM_CACHE(ceph_dentry_info, + SLAB_RECLAIM_ACCOUNT); + if (!ceph_dentry_cachep) + goto bad_dentry; + + ceph_file_cachep = KMEM_CACHE(ceph_file_info, 0); + if (!ceph_file_cachep) + goto bad_file; + + ceph_dir_file_cachep = KMEM_CACHE(ceph_dir_file_info, 0); + if (!ceph_dir_file_cachep) + goto bad_dir_file; + + ceph_mds_request_cachep = KMEM_CACHE(ceph_mds_request, 0); + if (!ceph_mds_request_cachep) + goto bad_mds_req; + + ceph_wb_pagevec_pool = mempool_create_kmalloc_pool(10, + (CEPH_MAX_WRITE_SIZE >> PAGE_SHIFT) * sizeof(struct page *)); + if (!ceph_wb_pagevec_pool) + goto bad_pagevec_pool; + + error = ceph_subvolume_metrics_cache_init(); + if (error) + goto bad_subvol_metrics; + + return 0; + +bad_subvol_metrics: + mempool_destroy(ceph_wb_pagevec_pool); +bad_pagevec_pool: + kmem_cache_destroy(ceph_mds_request_cachep); +bad_mds_req: + kmem_cache_destroy(ceph_dir_file_cachep); +bad_dir_file: + kmem_cache_destroy(ceph_file_cachep); +bad_file: + kmem_cache_destroy(ceph_dentry_cachep); +bad_dentry: + kmem_cache_destroy(ceph_cap_flush_cachep); +bad_cap_flush: + kmem_cache_destroy(ceph_cap_snap_cachep); +bad_cap_snap: + kmem_cache_destroy(ceph_cap_cachep); +bad_cap: + kmem_cache_destroy(ceph_inode_cachep); + return error; +} + +static void destroy_caches(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + + kmem_cache_destroy(ceph_inode_cachep); + kmem_cache_destroy(ceph_cap_cachep); + kmem_cache_destroy(ceph_cap_snap_cachep); + kmem_cache_destroy(ceph_cap_flush_cachep); + kmem_cache_destroy(ceph_dentry_cachep); + kmem_cache_destroy(ceph_file_cachep); + kmem_cache_destroy(ceph_dir_file_cachep); + kmem_cache_destroy(ceph_mds_request_cachep); + mempool_destroy(ceph_wb_pagevec_pool); + ceph_subvolume_metrics_cache_destroy(); +} + +static void __ceph_umount_begin(struct ceph_fs_client *fsc) +{ + ceph_osdc_abort_requests(&fsc->client->osdc, -EIO); + ceph_mdsc_force_umount(fsc->mdsc); + fsc->filp_gen++; // invalidate open files +} + +/* + * ceph_umount_begin - initiate forced umount. Tear down the + * mount, skipping steps that may hang while waiting for server(s). + */ +void ceph_umount_begin(struct super_block *sb) +{ + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(sb); + + doutc(fsc->client, "starting forced umount\n"); + + fsc->mount_state = CEPH_MOUNT_SHUTDOWN; + __ceph_umount_begin(fsc); +} + +static const struct super_operations ceph_super_ops = { + .alloc_inode = ceph_alloc_inode, + .free_inode = ceph_free_inode, + .write_inode = ceph_write_inode, + .drop_inode = inode_just_drop, + .evict_inode = ceph_evict_inode, + .sync_fs = ceph_sync_fs, + .put_super = ceph_put_super, + .show_options = ceph_show_options, + .statfs = ceph_statfs, + .umount_begin = ceph_umount_begin, +}; + +/* + * Bootstrap mount by opening the root directory. Note the mount + * @started time from caller, and time out if this takes too long. + */ +static struct dentry *open_root_dentry(struct ceph_fs_client *fsc, + const char *path, + unsigned long started) +{ + struct ceph_client *cl = fsc->client; + struct ceph_mds_client *mdsc = fsc->mdsc; + struct ceph_mds_request *req = NULL; + int err; + struct dentry *root; + + /* open dir */ + doutc(cl, "opening '%s'\n", path); + req = ceph_mdsc_create_request(mdsc, CEPH_MDS_OP_GETATTR, USE_ANY_MDS); + if (IS_ERR(req)) + return ERR_CAST(req); + req->r_path1 = kstrdup(path, GFP_NOFS); + if (!req->r_path1) { + root = ERR_PTR(-ENOMEM); + goto out; + } + + req->r_ino1.ino = CEPH_INO_ROOT; + req->r_ino1.snap = CEPH_NOSNAP; + req->r_started = started; + req->r_timeout = fsc->client->options->mount_timeout; + req->r_args.getattr.mask = cpu_to_le32(CEPH_STAT_CAP_INODE); + req->r_num_caps = 2; + err = ceph_mdsc_do_request(mdsc, NULL, req); + if (err == 0) { + struct inode *inode = req->r_target_inode; + req->r_target_inode = NULL; + doutc(cl, "success\n"); + root = d_make_root(inode); + if (!root) { + root = ERR_PTR(-ENOMEM); + goto out; + } + doutc(cl, "success, root dentry is %p\n", root); + } else { + root = ERR_PTR(err); + } +out: + ceph_mdsc_put_request(req); + return root; +} + +#ifdef CONFIG_FS_ENCRYPTION +static int ceph_apply_test_dummy_encryption(struct super_block *sb, + struct fs_context *fc, + struct ceph_mount_options *fsopt) +{ + struct ceph_fs_client *fsc = sb->s_fs_info; + + if (!fscrypt_is_dummy_policy_set(&fsopt->dummy_enc_policy)) + return 0; + + /* No changing encryption context on remount. */ + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE && + !fscrypt_is_dummy_policy_set(&fsc->fsc_dummy_enc_policy)) { + if (fscrypt_dummy_policies_equal(&fsopt->dummy_enc_policy, + &fsc->fsc_dummy_enc_policy)) + return 0; + errorfc(fc, "Can't set test_dummy_encryption on remount"); + return -EINVAL; + } + + /* Also make sure fsopt doesn't contain a conflicting value. */ + if (fscrypt_is_dummy_policy_set(&fsc->fsc_dummy_enc_policy)) { + if (fscrypt_dummy_policies_equal(&fsopt->dummy_enc_policy, + &fsc->fsc_dummy_enc_policy)) + return 0; + errorfc(fc, "Conflicting test_dummy_encryption options"); + return -EINVAL; + } + + fsc->fsc_dummy_enc_policy = fsopt->dummy_enc_policy; + memset(&fsopt->dummy_enc_policy, 0, sizeof(fsopt->dummy_enc_policy)); + + warnfc(fc, "test_dummy_encryption mode enabled"); + return 0; +} +#else +static int ceph_apply_test_dummy_encryption(struct super_block *sb, + struct fs_context *fc, + struct ceph_mount_options *fsopt) +{ + return 0; +} +#endif + +/* + * mount: join the ceph cluster, and open root directory. + */ +static struct dentry *ceph_real_mount(struct ceph_fs_client *fsc, + struct fs_context *fc) +{ + struct ceph_client *cl = fsc->client; + int err; + unsigned long started = jiffies; /* note the start time */ + struct dentry *root; + + doutc(cl, "mount start %p\n", fsc); + mutex_lock(&fsc->client->mount_mutex); + + if (!fsc->sb->s_root) { + const char *path = fsc->mount_options->server_path ? + fsc->mount_options->server_path + 1 : ""; + + err = __ceph_open_session(fsc->client); + if (err < 0) + goto out; + + /* setup fscache */ + if (fsc->mount_options->flags & CEPH_MOUNT_OPT_FSCACHE) { + err = ceph_fscache_register_fs(fsc, fc); + if (err < 0) + goto out; + } + + err = ceph_apply_test_dummy_encryption(fsc->sb, fc, + fsc->mount_options); + if (err) + goto out; + + doutc(cl, "mount opening path '%s'\n", path); + + ceph_fs_debugfs_init(fsc); + + root = open_root_dentry(fsc, path, started); + if (IS_ERR(root)) { + err = PTR_ERR(root); + goto out; + } + fsc->sb->s_root = dget(root); + } else { + root = dget(fsc->sb->s_root); + } + + fsc->mount_state = CEPH_MOUNT_MOUNTED; + doutc(cl, "mount success\n"); + mutex_unlock(&fsc->client->mount_mutex); + return root; + +out: + mutex_unlock(&fsc->client->mount_mutex); + ceph_fscrypt_free_dummy_policy(fsc); + return ERR_PTR(err); +} + +static int ceph_set_super(struct super_block *s, struct fs_context *fc) +{ + struct ceph_fs_client *fsc = s->s_fs_info; + struct ceph_client *cl = fsc->client; + int ret; + + doutc(cl, "%p\n", s); + + s->s_maxbytes = MAX_LFS_FILESIZE; + + s->s_xattr = ceph_xattr_handlers; + fsc->sb = s; + fsc->max_file_size = 1ULL << 40; /* temp value until we get mdsmap */ + + s->s_op = &ceph_super_ops; + set_default_d_op(s, &ceph_dentry_ops); + s->s_export_op = &ceph_export_ops; + + s->s_time_gran = 1; + s->s_time_min = 0; + s->s_time_max = U32_MAX; + s->s_flags |= SB_NODIRATIME | SB_NOATIME; + s->s_magic = CEPH_SUPER_MAGIC; + + ceph_fscrypt_set_ops(s); + + ret = set_anon_super_fc(s, fc); + if (ret != 0) + fsc->sb = NULL; + return ret; +} + +/* + * share superblock if same fs AND options + */ +static int ceph_compare_super(struct super_block *sb, struct fs_context *fc) +{ + struct ceph_fs_client *new = fc->s_fs_info; + struct ceph_mount_options *fsopt = new->mount_options; + struct ceph_options *opt = new->client->options; + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(sb); + struct ceph_client *cl = fsc->client; + + doutc(cl, "%p\n", sb); + + if (compare_mount_options(fsopt, opt, fsc)) { + doutc(cl, "monitor(s)/mount options don't match\n"); + return 0; + } + if ((opt->flags & CEPH_OPT_FSID) && + ceph_fsid_compare(&opt->fsid, &fsc->client->fsid)) { + doutc(cl, "fsid doesn't match\n"); + return 0; + } + if (fc->sb_flags != (sb->s_flags & ~SB_BORN)) { + doutc(cl, "flags differ\n"); + return 0; + } + + if (fsc->blocklisted && !ceph_test_mount_opt(fsc, CLEANRECOVER)) { + doutc(cl, "client is blocklisted (and CLEANRECOVER is not set)\n"); + return 0; + } + + if (fsc->mount_state == CEPH_MOUNT_SHUTDOWN) { + doutc(cl, "client has been forcibly unmounted\n"); + return 0; + } + + return 1; +} + +/* + * construct our own bdi so we can control readahead, etc. + */ +static atomic_long_t bdi_seq = ATOMIC_LONG_INIT(0); + +static int ceph_setup_bdi(struct super_block *sb, struct ceph_fs_client *fsc) +{ + int err; + + err = super_setup_bdi_name(sb, "ceph-%ld", + atomic_long_inc_return(&bdi_seq)); + if (err) + return err; + + /* set ra_pages based on rasize mount option? */ + sb->s_bdi->ra_pages = fsc->mount_options->rasize >> PAGE_SHIFT; + + /* set io_pages based on max osd read size */ + sb->s_bdi->io_pages = fsc->mount_options->rsize >> PAGE_SHIFT; + + return 0; +} + +static int ceph_get_tree(struct fs_context *fc) +{ + struct ceph_parse_opts_ctx *pctx = fc->fs_private; + struct ceph_mount_options *fsopt = pctx->opts; + struct super_block *sb; + struct ceph_fs_client *fsc; + struct dentry *res; + int (*compare_super)(struct super_block *, struct fs_context *) = + ceph_compare_super; + int err; + + dout("ceph_get_tree\n"); + + if (!fc->source) + return invalfc(fc, "No source"); + if (fsopt->new_dev_syntax && !fsopt->mon_addr) + return invalfc(fc, "No monitor address"); + + /* create client (which we may/may not use) */ + fsc = create_fs_client(pctx->opts, pctx->copts); + pctx->opts = NULL; + pctx->copts = NULL; + if (IS_ERR(fsc)) { + err = PTR_ERR(fsc); + goto out_final; + } + + err = ceph_mdsc_init(fsc); + if (err < 0) + goto out; + + if (ceph_test_opt(fsc->client, NOSHARE)) + compare_super = NULL; + + fc->s_fs_info = fsc; + sb = sget_fc(fc, compare_super, ceph_set_super); + fc->s_fs_info = NULL; + if (IS_ERR(sb)) { + err = PTR_ERR(sb); + goto out; + } + + if (ceph_sb_to_fs_client(sb) != fsc) { + destroy_fs_client(fsc); + fsc = ceph_sb_to_fs_client(sb); + dout("get_sb got existing client %p\n", fsc); + } else { + dout("get_sb using new client %p\n", fsc); + err = ceph_setup_bdi(sb, fsc); + if (err < 0) + goto out_splat; + } + + res = ceph_real_mount(fsc, fc); + if (IS_ERR(res)) { + err = PTR_ERR(res); + goto out_splat; + } + + doutc(fsc->client, "root %p inode %p ino %llx.%llx\n", res, + d_inode(res), ceph_vinop(d_inode(res))); + fc->root = fsc->sb->s_root; + return 0; + +out_splat: + if (!ceph_mdsmap_is_cluster_available(fsc->mdsc->mdsmap)) { + pr_info("No mds server is up or the cluster is laggy\n"); + err = -EHOSTUNREACH; + } + + ceph_mdsc_close_sessions(fsc->mdsc); + deactivate_locked_super(sb); + goto out_final; + +out: + destroy_fs_client(fsc); +out_final: + dout("ceph_get_tree fail %d\n", err); + return err; +} + +static void ceph_free_fc(struct fs_context *fc) +{ + struct ceph_parse_opts_ctx *pctx = fc->fs_private; + + if (pctx) { + destroy_mount_options(pctx->opts); + ceph_destroy_options(pctx->copts); + kfree(pctx); + } +} + +static int ceph_reconfigure_fc(struct fs_context *fc) +{ + int err; + struct ceph_parse_opts_ctx *pctx = fc->fs_private; + struct ceph_mount_options *fsopt = pctx->opts; + struct super_block *sb = fc->root->d_sb; + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(sb); + + err = ceph_apply_test_dummy_encryption(sb, fc, fsopt); + if (err) + return err; + + if (fsopt->flags & CEPH_MOUNT_OPT_ASYNC_DIROPS) + ceph_set_mount_opt(fsc, ASYNC_DIROPS); + else + ceph_clear_mount_opt(fsc, ASYNC_DIROPS); + + if (fsopt->flags & CEPH_MOUNT_OPT_SPARSEREAD) + ceph_set_mount_opt(fsc, SPARSEREAD); + else + ceph_clear_mount_opt(fsc, SPARSEREAD); + + if (strcmp_null(fsc->mount_options->mon_addr, fsopt->mon_addr)) { + kfree(fsc->mount_options->mon_addr); + fsc->mount_options->mon_addr = fsopt->mon_addr; + fsopt->mon_addr = NULL; + pr_notice_client(fsc->client, + "monitor addresses recorded, but not used for reconnection"); + } + + sync_filesystem(sb); + return 0; +} + +static const struct fs_context_operations ceph_context_ops = { + .free = ceph_free_fc, + .parse_param = ceph_parse_mount_param, + .get_tree = ceph_get_tree, + .reconfigure = ceph_reconfigure_fc, +}; + +/* + * Set up the filesystem mount context. + */ +static int ceph_init_fs_context(struct fs_context *fc) +{ + struct ceph_parse_opts_ctx *pctx; + struct ceph_mount_options *fsopt; + + pctx = kzalloc_obj(*pctx); + if (!pctx) + return -ENOMEM; + + pctx->copts = ceph_alloc_options(); + if (!pctx->copts) + goto nomem; + + pctx->opts = kzalloc_obj(*pctx->opts); + if (!pctx->opts) + goto nomem; + + fsopt = pctx->opts; + fsopt->flags = CEPH_MOUNT_OPT_DEFAULT; + + fsopt->wsize = CEPH_MAX_WRITE_SIZE; + fsopt->rsize = CEPH_MAX_READ_SIZE; + fsopt->rasize = CEPH_RASIZE_DEFAULT; + fsopt->snapdir_name = kstrdup(CEPH_SNAPDIRNAME_DEFAULT, GFP_KERNEL); + if (!fsopt->snapdir_name) + goto nomem; + + fsopt->caps_wanted_delay_min = CEPH_CAPS_WANTED_DELAY_MIN_DEFAULT; + fsopt->caps_wanted_delay_max = CEPH_CAPS_WANTED_DELAY_MAX_DEFAULT; + fsopt->max_readdir = CEPH_MAX_READDIR_DEFAULT; + fsopt->max_readdir_bytes = CEPH_MAX_READDIR_BYTES_DEFAULT; + fsopt->congestion_kb = default_congestion_kb(); + +#ifdef CONFIG_CEPH_FS_POSIX_ACL + fc->sb_flags |= SB_POSIXACL; +#endif + + fc->fs_private = pctx; + fc->ops = &ceph_context_ops; + return 0; + +nomem: + destroy_mount_options(pctx->opts); + ceph_destroy_options(pctx->copts); + kfree(pctx); + return -ENOMEM; +} + +/* + * Return true if it successfully increases the blocker counter, + * or false if the mdsc is in stopping and flushed state. + */ +static bool __inc_stopping_blocker(struct ceph_mds_client *mdsc) +{ + spin_lock(&mdsc->stopping_lock); + if (mdsc->stopping >= CEPH_MDSC_STOPPING_FLUSHING) { + spin_unlock(&mdsc->stopping_lock); + return false; + } + atomic_inc(&mdsc->stopping_blockers); + spin_unlock(&mdsc->stopping_lock); + return true; +} + +static void __dec_stopping_blocker(struct ceph_mds_client *mdsc) +{ + spin_lock(&mdsc->stopping_lock); + if (!atomic_dec_return(&mdsc->stopping_blockers) && + mdsc->stopping >= CEPH_MDSC_STOPPING_FLUSHING) + complete_all(&mdsc->stopping_waiter); + spin_unlock(&mdsc->stopping_lock); +} + +/* For metadata IO requests */ +bool ceph_inc_mds_stopping_blocker(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session) +{ + mutex_lock(&session->s_mutex); + inc_session_sequence(session); + mutex_unlock(&session->s_mutex); + + return __inc_stopping_blocker(mdsc); +} + +void ceph_dec_mds_stopping_blocker(struct ceph_mds_client *mdsc) +{ + __dec_stopping_blocker(mdsc); +} + +/* For data IO requests */ +bool ceph_inc_osd_stopping_blocker(struct ceph_mds_client *mdsc) +{ + return __inc_stopping_blocker(mdsc); +} + +void ceph_dec_osd_stopping_blocker(struct ceph_mds_client *mdsc) +{ + __dec_stopping_blocker(mdsc); +} + +static void ceph_kill_sb(struct super_block *s) +{ + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(s); + struct ceph_client *cl = fsc->client; + struct ceph_mds_client *mdsc = fsc->mdsc; + bool wait; + + doutc(cl, "%p\n", s); + + ceph_mdsc_pre_umount(mdsc); + flush_fs_workqueues(fsc); + + /* + * Though the kill_anon_super() will finally trigger the + * sync_filesystem() anyway, we still need to do it here and + * then bump the stage of shutdown. This will allow us to + * drop any further message, which will increase the inodes' + * i_count reference counters but makes no sense any more, + * from MDSs. + * + * Without this when evicting the inodes it may fail in the + * kill_anon_super(), which will trigger a warning when + * destroying the fscrypt keyring and then possibly trigger + * a further crash in ceph module when the iput() tries to + * evict the inodes later. + */ + sync_filesystem(s); + + if (atomic64_read(&mdsc->dirty_folios) > 0) { + wait_queue_head_t *wq = &mdsc->flush_end_wq; + long timeleft = wait_event_killable_timeout(*wq, + atomic64_read(&mdsc->dirty_folios) <= 0, + fsc->client->options->mount_timeout); + if (!timeleft) /* timed out */ + pr_warn_client(cl, "umount timed out, %ld\n", timeleft); + else if (timeleft < 0) /* killed */ + pr_warn_client(cl, "umount was killed, %ld\n", timeleft); + } + + spin_lock(&mdsc->stopping_lock); + mdsc->stopping = CEPH_MDSC_STOPPING_FLUSHING; + wait = !!atomic_read(&mdsc->stopping_blockers); + spin_unlock(&mdsc->stopping_lock); + + if (wait && atomic_read(&mdsc->stopping_blockers)) { + long timeleft = wait_for_completion_killable_timeout( + &mdsc->stopping_waiter, + fsc->client->options->mount_timeout); + if (!timeleft) /* timed out */ + pr_warn_client(cl, "umount timed out, %ld\n", timeleft); + else if (timeleft < 0) /* killed */ + pr_warn_client(cl, "umount was killed, %ld\n", timeleft); + } + + mdsc->stopping = CEPH_MDSC_STOPPING_FLUSHED; + kill_anon_super(s); + + fsc->client->extra_mon_dispatch = NULL; + ceph_fs_debugfs_cleanup(fsc); + + ceph_fscache_unregister_fs(fsc); + + destroy_fs_client(fsc); +} + +static struct file_system_type ceph_fs_type = { + .owner = THIS_MODULE, + .name = "ceph", + .init_fs_context = ceph_init_fs_context, + .kill_sb = ceph_kill_sb, + .fs_flags = FS_RENAME_DOES_D_MOVE | FS_ALLOW_IDMAP, +}; +MODULE_ALIAS_FS("ceph"); + +int ceph_force_reconnect(struct super_block *sb) +{ + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(sb); + int err = 0; + + fsc->mount_state = CEPH_MOUNT_RECOVER; + __ceph_umount_begin(fsc); + + /* Make sure all page caches get invalidated. + * see remove_session_caps_cb() */ + flush_workqueue(fsc->inode_wq); + + /* In case that we were blocklisted. This also reset + * all mon/osd connections */ + ceph_reset_client_addr(fsc->client); + + ceph_osdc_clear_abort_err(&fsc->client->osdc); + + fsc->blocklisted = false; + fsc->mount_state = CEPH_MOUNT_MOUNTED; + + if (sb->s_root) { + err = __ceph_do_getattr(d_inode(sb->s_root), NULL, + CEPH_STAT_CAP_INODE, true); + } + return err; +} + +static int __init init_ceph(void) +{ + int ret = init_caches(); + if (ret) + goto out; + + ceph_flock_init(); + ret = register_filesystem(&ceph_fs_type); + if (ret) + goto out_caches; + + pr_info("loaded (mds proto %d)\n", CEPH_MDSC_PROTOCOL); + + return 0; + +out_caches: + destroy_caches(); +out: + return ret; +} + +static void __exit exit_ceph(void) +{ + dout("exit_ceph\n"); + unregister_filesystem(&ceph_fs_type); + destroy_caches(); +} + +static int param_set_metrics(const char *val, const struct kernel_param *kp) +{ + struct ceph_fs_client *fsc; + int ret; + + ret = param_set_bool(val, kp); + if (ret) { + pr_err("Failed to parse sending metrics switch value '%s'\n", + val); + return ret; + } else if (!disable_send_metrics) { + // wake up all the mds clients + spin_lock(&ceph_fsc_lock); + list_for_each_entry(fsc, &ceph_fsc_list, metric_wakeup) { + metric_schedule_delayed(&fsc->mdsc->metric); + } + spin_unlock(&ceph_fsc_lock); + } + + return 0; +} + +static const struct kernel_param_ops param_ops_metrics = { + .set = param_set_metrics, + .get = param_get_bool, +}; + +bool disable_send_metrics = false; +module_param_cb(disable_send_metrics, ¶m_ops_metrics, &disable_send_metrics, 0644); +MODULE_PARM_DESC(disable_send_metrics, "Enable sending perf metrics to ceph cluster (default: on)"); + +/* for both v1 and v2 syntax */ +static bool mount_support = true; +static const struct kernel_param_ops param_ops_mount_syntax = { + .get = param_get_bool, +}; +module_param_cb(mount_syntax_v1, ¶m_ops_mount_syntax, &mount_support, 0444); +module_param_cb(mount_syntax_v2, ¶m_ops_mount_syntax, &mount_support, 0444); + +bool enable_unsafe_idmap = false; +module_param(enable_unsafe_idmap, bool, 0644); +MODULE_PARM_DESC(enable_unsafe_idmap, + "Allow to use idmapped mounts with MDS without CEPHFS_FEATURE_HAS_OWNER_UIDGID"); + +module_init(init_ceph); +module_exit(exit_ceph); + +MODULE_AUTHOR("Sage Weil "); +MODULE_AUTHOR("Yehuda Sadeh "); +MODULE_AUTHOR("Patience Warnick "); +MODULE_DESCRIPTION("Ceph filesystem for Linux"); +MODULE_LICENSE("GPL"); diff --git a/fs/ceph/super.h b/fs/ceph/super.h new file mode 100644 index 000000000..1d6aab060 --- /dev/null +++ b/fs/ceph/super.h @@ -0,0 +1,1481 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _FS_CEPH_SUPER_H +#define _FS_CEPH_SUPER_H + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "crypto.h" + +/* large granularity for statfs utilization stats to facilitate + * large volume sizes on 32-bit machines. */ +#define CEPH_BLOCK_SHIFT 22 /* 4 MB */ +#define CEPH_BLOCK (1 << CEPH_BLOCK_SHIFT) +#define CEPH_4K_BLOCK_SHIFT 12 /* 4 KB */ + +#define CEPH_MOUNT_OPT_CLEANRECOVER (1<<1) /* auto reonnect (clean mode) after blocklisted */ +#define CEPH_MOUNT_OPT_DIRSTAT (1<<4) /* `cat dirname` for stats */ +#define CEPH_MOUNT_OPT_RBYTES (1<<5) /* dir st_bytes = rbytes */ +#define CEPH_MOUNT_OPT_NOASYNCREADDIR (1<<7) /* no dcache readdir */ +#define CEPH_MOUNT_OPT_INO32 (1<<8) /* 32 bit inos */ +#define CEPH_MOUNT_OPT_DCACHE (1<<9) /* use dcache for readdir etc */ +#define CEPH_MOUNT_OPT_FSCACHE (1<<10) /* use fscache */ +#define CEPH_MOUNT_OPT_NOPOOLPERM (1<<11) /* no pool permission check */ +#define CEPH_MOUNT_OPT_MOUNTWAIT (1<<12) /* mount waits if no mds is up */ +#define CEPH_MOUNT_OPT_NOQUOTADF (1<<13) /* no root dir quota in statfs */ +#define CEPH_MOUNT_OPT_NOCOPYFROM (1<<14) /* don't use RADOS 'copy-from' op */ +#define CEPH_MOUNT_OPT_ASYNC_DIROPS (1<<15) /* allow async directory ops */ +#define CEPH_MOUNT_OPT_NOPAGECACHE (1<<16) /* bypass pagecache altogether */ +#define CEPH_MOUNT_OPT_SPARSEREAD (1<<17) /* always do sparse reads */ + +#define CEPH_MOUNT_OPT_DEFAULT \ + (CEPH_MOUNT_OPT_DCACHE | \ + CEPH_MOUNT_OPT_NOCOPYFROM | \ + CEPH_MOUNT_OPT_ASYNC_DIROPS) + +#define ceph_set_mount_opt(fsc, opt) \ + (fsc)->mount_options->flags |= CEPH_MOUNT_OPT_##opt +#define ceph_clear_mount_opt(fsc, opt) \ + (fsc)->mount_options->flags &= ~CEPH_MOUNT_OPT_##opt +#define ceph_test_mount_opt(fsc, opt) \ + (!!((fsc)->mount_options->flags & CEPH_MOUNT_OPT_##opt)) + +/* max size of osd read request, limited by libceph */ +#define CEPH_MAX_READ_SIZE CEPH_MSG_MAX_DATA_LEN +/* osd has a configurable limitation of max write size. + * CEPH_MSG_MAX_DATA_LEN should be small enough. */ +#define CEPH_MAX_WRITE_SIZE CEPH_MSG_MAX_DATA_LEN +#define CEPH_RASIZE_DEFAULT (8192*1024) /* max readahead */ +#define CEPH_MAX_READDIR_DEFAULT 1024 +#define CEPH_MAX_READDIR_BYTES_DEFAULT (512*1024) +#define CEPH_SNAPDIRNAME_DEFAULT ".snap" + +/* + * Delay telling the MDS we no longer want caps, in case we reopen + * the file. Delay a minimum amount of time, even if we send a cap + * message for some other reason. Otherwise, take the oppotunity to + * update the mds to avoid sending another message later. + */ +#define CEPH_CAPS_WANTED_DELAY_MIN_DEFAULT 5 /* cap release delay */ +#define CEPH_CAPS_WANTED_DELAY_MAX_DEFAULT 60 /* cap release delay */ + +struct ceph_mount_options { + unsigned int flags; + + unsigned int wsize; /* max write size */ + unsigned int rsize; /* max read size */ + unsigned int rasize; /* max readahead */ + unsigned int congestion_kb; /* max writeback in flight */ + unsigned int caps_wanted_delay_min, caps_wanted_delay_max; + int caps_max; + unsigned int max_readdir; /* max readdir result (entries) */ + unsigned int max_readdir_bytes; /* max readdir result (bytes) */ + + bool new_dev_syntax; + + /* + * everything above this point can be memcmp'd; everything below + * is handled in compare_mount_options() + */ + + char *snapdir_name; /* default ".snap" */ + char *mds_namespace; /* default NULL */ + char *server_path; /* default NULL (means "/") */ + char *fscache_uniq; /* default NULL */ + char *mon_addr; + struct fscrypt_dummy_policy dummy_enc_policy; +}; + +#define CEPH_NAMESPACE_WILDCARD "*" + +static inline bool ceph_namespace_match(const char *pattern, + const char *target) +{ + if (!pattern || !pattern[0] || + !strcmp(pattern, CEPH_NAMESPACE_WILDCARD)) + return true; + + return !strcmp(pattern, target); +} + +/* + * Check if the mds namespace in ceph_mount_options matches + * the passed in namespace string. First time match (when + * ->mds_namespace is NULL) is treated specially, since + * ->mds_namespace needs to be initialized by the caller. + */ +static inline bool namespace_equals(struct ceph_mount_options *fsopt, + const char *namespace, size_t len) +{ + return !(fsopt->mds_namespace && + (strlen(fsopt->mds_namespace) != len || + strncmp(fsopt->mds_namespace, namespace, len))); +} + +/* mount state */ +enum { + CEPH_MOUNT_MOUNTING, + CEPH_MOUNT_MOUNTED, + CEPH_MOUNT_UNMOUNTING, + CEPH_MOUNT_UNMOUNTED, + CEPH_MOUNT_SHUTDOWN, + CEPH_MOUNT_RECOVER, + CEPH_MOUNT_FENCE_IO, +}; + +#define CEPH_ASYNC_CREATE_CONFLICT_BITS 8 + +struct ceph_fs_client { + struct super_block *sb; + + struct list_head metric_wakeup; + + struct ceph_mount_options *mount_options; + struct ceph_client *client; + + int mount_state; + + bool blocklisted; + + bool have_copy_from2; + + u32 filp_gen; + loff_t max_file_size; + + struct ceph_mds_client *mdsc; + + atomic_long_t writeback_count; + bool write_congested; + + struct workqueue_struct *inode_wq; + struct workqueue_struct *cap_wq; + + DECLARE_HASHTABLE(async_unlink_conflict, CEPH_ASYNC_CREATE_CONFLICT_BITS); + spinlock_t async_unlink_conflict_lock; + +#ifdef CONFIG_DEBUG_FS + struct dentry *debugfs_dentry_lru, *debugfs_caps; + struct dentry *debugfs_congestion_kb; + struct dentry *debugfs_bdi; + struct dentry *debugfs_mdsc, *debugfs_mdsmap; + struct dentry *debugfs_status; + struct dentry *debugfs_mds_sessions; + struct dentry *debugfs_metrics_dir; + struct dentry *debugfs_reset_dir; + struct dentry *debugfs_subvolume_metrics; +#endif + +#ifdef CONFIG_CEPH_FSCACHE + struct fscache_volume *fscache; +#endif +#ifdef CONFIG_FS_ENCRYPTION + struct fscrypt_dummy_policy fsc_dummy_enc_policy; +#endif +}; + +/* + * File i/o capability. This tracks shared state with the metadata + * server that allows us to cache or writeback attributes or to read + * and write data. For any given inode, we should have one or more + * capabilities, one issued by each metadata server, and our + * cumulative access is the OR of all issued capabilities. + * + * Each cap is referenced by the inode's i_caps rbtree and by per-mds + * session capability lists. + */ +struct ceph_cap { + struct ceph_inode_info *ci; + struct rb_node ci_node; /* per-ci cap tree */ + struct ceph_mds_session *session; + struct list_head session_caps; /* per-session caplist */ + u64 cap_id; /* unique cap id (mds provided) */ + union { + /* in-use caps */ + struct { + int issued; /* latest, from the mds */ + int implemented; /* implemented superset of + issued (for revocation) */ + int mds; /* mds index for this cap */ + int mds_wanted; /* caps wanted from this mds */ + }; + /* caps to release */ + struct { + u64 cap_ino; + int queue_release; + }; + }; + u32 seq, issue_seq, mseq; + u32 cap_gen; /* active/stale cycle */ + unsigned long last_used; + struct list_head caps_item; +}; + +#define CHECK_CAPS_AUTHONLY 1 /* only check auth cap */ +#define CHECK_CAPS_FLUSH 2 /* flush any dirty caps */ +#define CHECK_CAPS_NOINVAL 4 /* don't invalidate pagecache */ +#define CHECK_CAPS_FLUSH_FORCE 8 /* force flush any caps */ + +struct ceph_cap_flush { + u64 tid; + int caps; + bool wake; /* wake up flush waiters when finish ? */ + bool is_capsnap; /* true means capsnap */ + struct list_head g_list; // global + struct list_head i_list; // per inode + struct ceph_inode_info *ci; +}; + +/* + * Snapped cap state that is pending flush to mds. When a snapshot occurs, + * we first complete any in-process sync writes and writeback any dirty + * data before flushing the snapped state (tracked here) back to the MDS. + */ +struct ceph_cap_snap { + refcount_t nref; + struct list_head ci_item; + + struct ceph_cap_flush cap_flush; + + u64 follows; + int issued, dirty; + struct ceph_snap_context *context; + + umode_t mode; + kuid_t uid; + kgid_t gid; + + struct ceph_buffer *xattr_blob; + u64 xattr_version; + + u64 size; + u64 change_attr; + struct timespec64 mtime, atime, ctime, btime; + u64 time_warp_seq; + u64 truncate_size; + u32 truncate_seq; + int writing; /* a sync write is still in progress */ + int dirty_pages; /* dirty pages awaiting writeback */ + bool inline_data; + bool need_flush; +}; + +static inline void ceph_put_cap_snap(struct ceph_cap_snap *capsnap) +{ + if (refcount_dec_and_test(&capsnap->nref)) { + if (capsnap->xattr_blob) + ceph_buffer_put(capsnap->xattr_blob); + kmem_cache_free(ceph_cap_snap_cachep, capsnap); + } +} + +/* + * The frag tree describes how a directory is fragmented, potentially across + * multiple metadata servers. It is also used to indicate points where + * metadata authority is delegated, and whether/where metadata is replicated. + * + * A _leaf_ frag will be present in the i_fragtree IFF there is + * delegation info. That is, if mds >= 0 || ndist > 0. + */ +#define CEPH_MAX_DIRFRAG_REP 4 + +struct ceph_inode_frag { + struct rb_node node; + + /* fragtree state */ + u32 frag; + int split_by; /* i.e. 2^(split_by) children */ + + /* delegation and replication info */ + int mds; /* -1 if same authority as parent */ + int ndist; /* >0 if replicated */ + int dist[CEPH_MAX_DIRFRAG_REP]; +}; + +/* + * We cache inode xattrs as an encoded blob until they are first used, + * at which point we parse them into an rbtree. + */ +struct ceph_inode_xattr { + struct rb_node node; + + const char *name; + int name_len; + const char *val; + int val_len; + int dirty; + + int should_free_name; + int should_free_val; +}; + +/* + * Ceph dentry state + */ +struct ceph_dentry_info { + struct dentry *dentry; + struct ceph_mds_session *lease_session; + struct list_head lease_list; + struct hlist_node hnode; + unsigned long flags; + int lease_shared_gen; + u32 lease_gen; + u32 lease_seq; + unsigned long lease_renew_after, lease_renew_from; + unsigned long time; + u64 offset; +}; + +#define CEPH_DENTRY_REFERENCED (1 << 0) +#define CEPH_DENTRY_LEASE_LIST (1 << 1) +#define CEPH_DENTRY_SHRINK_LIST (1 << 2) +#define CEPH_DENTRY_PRIMARY_LINK (1 << 3) +#define CEPH_DENTRY_ASYNC_UNLINK_BIT (4) +#define CEPH_DENTRY_ASYNC_UNLINK (1 << CEPH_DENTRY_ASYNC_UNLINK_BIT) +#define CEPH_DENTRY_ASYNC_CREATE_BIT (5) +#define CEPH_DENTRY_ASYNC_CREATE (1 << CEPH_DENTRY_ASYNC_CREATE_BIT) + +struct ceph_inode_xattrs_info { + /* + * (still encoded) xattr blob. we avoid the overhead of parsing + * this until someone actually calls getxattr, etc. + * + * blob->vec.iov_len == 4 implies there are no xattrs; blob == + * NULL means we don't know. + */ + struct ceph_buffer *blob, *prealloc_blob; + + struct rb_root index; + bool dirty; + int count; + int names_size; + int vals_size; + u64 version, index_version; +}; + +/* + * Ceph inode. + */ +struct ceph_inode_info { + struct netfs_inode netfs; /* Netfslib context and vfs inode */ + struct ceph_vino i_vino; /* ceph ino + snap */ + + spinlock_t i_ceph_lock; + + u64 i_version; + u64 i_inline_version; + u32 i_time_warp_seq; + + unsigned long i_ceph_flags; + atomic64_t i_release_count; + atomic64_t i_ordered_count; + atomic64_t i_complete_seq[2]; + + struct ceph_dir_layout i_dir_layout; + struct ceph_file_layout i_layout; + struct ceph_file_layout i_cached_layout; // for async creates + char *i_symlink; + + /* for dirs */ + struct timespec64 i_rctime; + u64 i_rbytes, i_rfiles, i_rsubdirs, i_rsnaps; + u64 i_files, i_subdirs; + + /* quotas */ + u64 i_max_bytes, i_max_files; + + /* + * Subvolume ID this inode belongs to. CEPH_SUBVOLUME_ID_NONE (0) + * means unknown/unset, matching the FUSE client convention. + * Once set to a valid (non-zero) value, it should not change + * during the inode's lifetime. + */ +#define CEPH_SUBVOLUME_ID_NONE 0 + u64 i_subvolume_id; + + s32 i_dir_pin; + + struct rb_root i_fragtree; + int i_fragtree_nsplits; + struct mutex i_fragtree_mutex; + + struct ceph_inode_xattrs_info i_xattrs; + + /* capabilities. protected _both_ by i_ceph_lock and cap->session's + * s_mutex. */ + struct rb_root i_caps; /* cap list */ + struct ceph_cap *i_auth_cap; /* authoritative cap, if any */ + unsigned i_dirty_caps, i_flushing_caps; /* mask of dirtied fields */ + + /* + * Link to the auth cap's session's s_cap_dirty list. s_cap_dirty + * is protected by the mdsc->cap_dirty_lock, but each individual item + * is also protected by the inode's i_ceph_lock. Walking s_cap_dirty + * requires the mdsc->cap_dirty_lock. List presence for an item can + * be tested under the i_ceph_lock. Changing anything requires both. + */ + struct list_head i_dirty_item; + + /* + * Link to session's s_cap_flushing list. Protected in a similar + * fashion to i_dirty_item, but also by the s_mutex for changes. The + * s_cap_flushing list can be walked while holding either the s_mutex + * or msdc->cap_dirty_lock. List presence can also be checked while + * holding the i_ceph_lock for this inode. + */ + struct list_head i_flushing_item; + + /* we need to track cap writeback on a per-cap-bit basis, to allow + * overlapping, pipelined cap flushes to the mds. we can probably + * reduce the tid to 8 bits if we're concerned about inode size. */ + struct ceph_cap_flush *i_prealloc_cap_flush; + struct list_head i_cap_flush_list; + wait_queue_head_t i_cap_wq; /* threads waiting on a capability */ + unsigned long i_hold_caps_max; /* jiffies */ + struct list_head i_cap_delay_list; /* for delayed cap release to mds */ + struct ceph_cap_reservation i_cap_migration_resv; + struct list_head i_cap_snaps; /* snapped state pending flush to mds */ + struct ceph_snap_context *i_head_snapc; /* set if wr_buffer_head > 0 or + dirty|flushing caps */ + unsigned i_snap_caps; /* cap bits for snapped files */ + /* + * Written under i_ceph_lock, read via READ_ONCE() + * from diagnostic paths. + */ + u64 i_last_cap_flush_ack; + + unsigned long i_last_rd; + unsigned long i_last_wr; + int i_nr_by_mode[CEPH_FILE_MODE_BITS]; /* open file counts */ + + struct mutex i_truncate_mutex; + u32 i_truncate_seq; /* last truncate to smaller size */ + u64 i_truncate_size; /* and the size we last truncated down to */ + int i_truncate_pending; /* still need to call vmtruncate */ + /* + * For none fscrypt case it equals to i_truncate_size or it will + * equals to fscrypt_file_size + */ + u64 i_truncate_pagecache_size; + + u64 i_max_size; /* max file size authorized by mds */ + u64 i_reported_size; /* (max_)size reported to or requested of mds */ + u64 i_wanted_max_size; /* offset we'd like to write too */ + u64 i_requested_max_size; /* max_size we've requested */ + + /* held references to caps */ + int i_pin_ref; + int i_rd_ref, i_rdcache_ref, i_wr_ref, i_wb_ref, i_fx_ref; + int i_wrbuffer_ref, i_wrbuffer_ref_head; + atomic_t i_filelock_ref; + atomic_t i_shared_gen; /* increment each time we get FILE_SHARED */ + u32 i_rdcache_gen; /* incremented each time we get FILE_CACHE. */ + u32 i_rdcache_revoking; /* RDCACHE gen to async invalidate, if any */ + + struct list_head i_unsafe_dirops; /* uncommitted mds dir ops */ + struct list_head i_unsafe_iops; /* uncommitted mds inode ops */ + spinlock_t i_unsafe_lock; + + union { + struct ceph_snap_realm *i_snap_realm; /* snap realm (if caps) */ + struct ceph_snapid_map *i_snapid_map; /* snapid -> dev_t */ + }; + struct list_head i_snap_realm_item; + struct list_head i_snap_flush_item; + struct timespec64 i_btime; + struct timespec64 i_snap_btime; + + struct work_struct i_work; + unsigned long i_work_mask; + +#ifdef CONFIG_FS_ENCRYPTION + struct fscrypt_inode_info *i_crypt_info; + u32 fscrypt_auth_len; + u32 fscrypt_file_len; + u8 *fscrypt_auth; + u8 *fscrypt_file; +#endif +}; + +struct ceph_netfs_request_data { + int caps; + + /* + * Maximum size of a file readahead request. + * The fadvise could update the bdi's default ra_pages. + */ + unsigned int file_ra_pages; + + /* Set it if fadvise disables file readahead entirely */ + bool file_ra_disabled; +}; + +static inline struct ceph_inode_info * +ceph_inode(const struct inode *inode) +{ + return container_of(inode, struct ceph_inode_info, netfs.inode); +} + +static inline struct ceph_fs_client * +ceph_inode_to_fs_client(const struct inode *inode) +{ + return (struct ceph_fs_client *)inode->i_sb->s_fs_info; +} + +static inline struct ceph_fs_client * +ceph_sb_to_fs_client(const struct super_block *sb) +{ + return (struct ceph_fs_client *)sb->s_fs_info; +} + +static inline struct ceph_mds_client * +ceph_sb_to_mdsc(const struct super_block *sb) +{ + return (struct ceph_mds_client *)ceph_sb_to_fs_client(sb)->mdsc; +} + +static inline struct ceph_client * +ceph_inode_to_client(const struct inode *inode) +{ + return (struct ceph_client *)ceph_inode_to_fs_client(inode)->client; +} + +static inline struct ceph_vino +ceph_vino(const struct inode *inode) +{ + return ceph_inode(inode)->i_vino; +} + +static inline u32 ceph_ino_to_ino32(u64 vino) +{ + u32 ino = vino & 0xffffffff; + ino ^= vino >> 32; + if (!ino) + ino = 2; + return ino; +} + +/* + * Inode numbers in cephfs are 64 bits, but inode->i_ino is 32-bits on + * some arches. We generally do not use this value inside the ceph driver, but + * we do want to set it to something, so that generic vfs code has an + * appropriate value for tracepoints and the like. + */ +static inline ino_t ceph_vino_to_ino_t(struct ceph_vino vino) +{ + if (sizeof(ino_t) == sizeof(u32)) + return ceph_ino_to_ino32(vino.ino); + return (ino_t)vino.ino; +} + +/* for printf-style formatting */ +#define ceph_vinop(i) ceph_inode(i)->i_vino.ino, ceph_inode(i)->i_vino.snap + +static inline u64 ceph_ino(struct inode *inode) +{ + return ceph_inode(inode)->i_vino.ino; +} + +static inline u64 ceph_snap(struct inode *inode) +{ + return ceph_inode(inode)->i_vino.snap; +} + +/** + * ceph_present_ino - format an inode number for presentation to userland + * @sb: superblock where the inode lives + * @ino: inode number to (possibly) convert + * + * If the user mounted with the ino32 option, then the 64-bit value needs + * to be converted to something that can fit inside 32 bits. Note that + * internal kernel code never uses this value, so this is entirely for + * userland consumption. + */ +static inline u64 ceph_present_ino(struct super_block *sb, u64 ino) +{ + if (unlikely(ceph_test_mount_opt(ceph_sb_to_fs_client(sb), INO32))) + return ceph_ino_to_ino32(ino); + return ino; +} + +static inline u64 ceph_present_inode(struct inode *inode) +{ + return ceph_present_ino(inode->i_sb, ceph_ino(inode)); +} + +static inline int ceph_ino_compare(struct inode *inode, void *data) +{ + struct ceph_vino *pvino = (struct ceph_vino *)data; + struct ceph_inode_info *ci = ceph_inode(inode); + return ci->i_vino.ino == pvino->ino && + ci->i_vino.snap == pvino->snap; +} + +/* + * The MDS reserves a set of inodes for its own usage. These should never + * be accessible by clients, and so the MDS has no reason to ever hand these + * out. The range is CEPH_MDS_INO_MDSDIR_OFFSET..CEPH_INO_SYSTEM_BASE. + * + * These come from src/mds/mdstypes.h in the ceph sources. + */ +#define CEPH_MAX_MDS 0x100 +#define CEPH_NUM_STRAY 10 +#define CEPH_MDS_INO_MDSDIR_OFFSET (1 * CEPH_MAX_MDS) +#define CEPH_MDS_INO_LOG_OFFSET (2 * CEPH_MAX_MDS) +#define CEPH_INO_SYSTEM_BASE ((6*CEPH_MAX_MDS) + (CEPH_MAX_MDS * CEPH_NUM_STRAY)) + +static inline bool ceph_vino_is_reserved(const struct ceph_vino vino) +{ + if (vino.ino >= CEPH_INO_SYSTEM_BASE || + vino.ino < CEPH_MDS_INO_MDSDIR_OFFSET) + return false; + + /* Don't warn on mdsdirs */ + WARN_RATELIMIT(vino.ino >= CEPH_MDS_INO_LOG_OFFSET, + "Attempt to access reserved inode number 0x%llx", + vino.ino); + return true; +} + +static inline struct inode *ceph_find_inode(struct super_block *sb, + struct ceph_vino vino) +{ + if (ceph_vino_is_reserved(vino)) + return NULL; + + /* + * NB: The hashval will be run through the fs/inode.c hash function + * anyway, so there is no need to squash the inode number down to + * 32-bits first. Just use low-order bits on arches with 32-bit long. + */ + return ilookup5(sb, (unsigned long)vino.ino, ceph_ino_compare, &vino); +} + + +/* + * Ceph inode. + */ +#define CEPH_I_DIR_ORDERED_BIT (0) /* dentries in dir are ordered */ + /* bit 1 historically unused */ +#define CEPH_I_FLUSH_BIT (2) /* do not delay flush of dirty metadata */ +#define CEPH_I_POOL_PERM_BIT (3) /* pool rd/wr bits are valid */ +#define CEPH_I_POOL_RD_BIT (4) /* can read from pool */ +#define CEPH_I_POOL_WR_BIT (5) /* can write to pool */ +#define CEPH_I_SEC_INITED_BIT (6) /* security initialized */ +#define CEPH_I_KICK_FLUSH_BIT (7) /* kick flushing caps */ +#define CEPH_I_FLUSH_SNAPS_BIT (8) /* need flush snaps */ +#define CEPH_I_ERROR_WRITE_BIT (9) /* have seen write errors */ +#define CEPH_I_ERROR_FILELOCK_BIT (10) /* have seen file lock errors */ +#define CEPH_I_ODIRECT_BIT (11) /* inode in direct I/O mode */ +#define CEPH_I_ASYNC_CREATE_BIT (12) /* async create in flight for this */ +#define CEPH_I_SHUTDOWN_BIT (13) /* inode is no longer usable */ +#define CEPH_I_ASYNC_CHECK_CAPS_BIT (14) /* check caps after async creating finishes */ + +#define CEPH_I_DIR_ORDERED (1 << CEPH_I_DIR_ORDERED_BIT) +#define CEPH_I_FLUSH (1 << CEPH_I_FLUSH_BIT) +#define CEPH_I_POOL_PERM (1 << CEPH_I_POOL_PERM_BIT) +#define CEPH_I_POOL_RD (1 << CEPH_I_POOL_RD_BIT) +#define CEPH_I_POOL_WR (1 << CEPH_I_POOL_WR_BIT) +#define CEPH_I_SEC_INITED (1 << CEPH_I_SEC_INITED_BIT) +#define CEPH_I_KICK_FLUSH (1 << CEPH_I_KICK_FLUSH_BIT) +#define CEPH_I_FLUSH_SNAPS (1 << CEPH_I_FLUSH_SNAPS_BIT) +#define CEPH_I_ODIRECT (1 << CEPH_I_ODIRECT_BIT) +#define CEPH_I_ASYNC_CREATE (1 << CEPH_I_ASYNC_CREATE_BIT) +#define CEPH_I_SHUTDOWN (1 << CEPH_I_SHUTDOWN_BIT) + +/* + * Masks of ceph inode work. + */ +#define CEPH_I_WORK_WRITEBACK 0 +#define CEPH_I_WORK_INVALIDATE_PAGES 1 +#define CEPH_I_WORK_VMTRUNCATE 2 +#define CEPH_I_WORK_CHECK_CAPS 3 +#define CEPH_I_WORK_FLUSH_SNAPS 4 + +/* + * We set the ERROR_WRITE bit when we start seeing write errors on an inode + * and then clear it when they start succeeding. The write submission code + * just takes this as a hint, so we're not too worried if a few slip through + * in either direction. + */ +static inline void ceph_set_error_write(struct ceph_inode_info *ci) +{ + set_bit(CEPH_I_ERROR_WRITE_BIT, &ci->i_ceph_flags); +} + +static inline void ceph_clear_error_write(struct ceph_inode_info *ci) +{ + clear_bit(CEPH_I_ERROR_WRITE_BIT, &ci->i_ceph_flags); +} + +static inline void __ceph_dir_set_complete(struct ceph_inode_info *ci, + long long release_count, + long long ordered_count) +{ + /* + * Makes sure operations that setup readdir cache (update page + * cache and i_size) are strongly ordered w.r.t. the following + * atomic64_set() operations. + */ + smp_mb(); + atomic64_set(&ci->i_complete_seq[0], release_count); + atomic64_set(&ci->i_complete_seq[1], ordered_count); +} + +static inline void __ceph_dir_clear_complete(struct ceph_inode_info *ci) +{ + atomic64_inc(&ci->i_release_count); +} + +static inline void __ceph_dir_clear_ordered(struct ceph_inode_info *ci) +{ + atomic64_inc(&ci->i_ordered_count); +} + +static inline bool __ceph_dir_is_complete(struct ceph_inode_info *ci) +{ + return atomic64_read(&ci->i_complete_seq[0]) == + atomic64_read(&ci->i_release_count); +} + +static inline bool __ceph_dir_is_complete_ordered(struct ceph_inode_info *ci) +{ + return atomic64_read(&ci->i_complete_seq[0]) == + atomic64_read(&ci->i_release_count) && + atomic64_read(&ci->i_complete_seq[1]) == + atomic64_read(&ci->i_ordered_count); +} + +static inline void ceph_dir_clear_complete(struct inode *inode) +{ + __ceph_dir_clear_complete(ceph_inode(inode)); +} + +static inline void ceph_dir_clear_ordered(struct inode *inode) +{ + __ceph_dir_clear_ordered(ceph_inode(inode)); +} + +static inline bool ceph_dir_is_complete_ordered(struct inode *inode) +{ + bool ret = __ceph_dir_is_complete_ordered(ceph_inode(inode)); + smp_rmb(); + return ret; +} + +/* find a specific frag @f */ +extern struct ceph_inode_frag *__ceph_find_frag(struct ceph_inode_info *ci, + u32 f); + +/* + * choose fragment for value @v. copy frag content to pfrag, if leaf + * exists + */ +extern u32 ceph_choose_frag(struct ceph_inode_info *ci, u32 v, + struct ceph_inode_frag *pfrag, + int *found); + +static inline struct ceph_dentry_info *ceph_dentry(const struct dentry *dentry) +{ + return (struct ceph_dentry_info *)dentry->d_fsdata; +} + +/* + * caps helpers + */ +static inline bool __ceph_is_any_real_caps(struct ceph_inode_info *ci) +{ + return !RB_EMPTY_ROOT(&ci->i_caps); +} + +extern int __ceph_caps_issued(struct ceph_inode_info *ci, int *implemented); +extern int __ceph_caps_issued_mask(struct ceph_inode_info *ci, int mask, int t); +extern int __ceph_caps_issued_mask_metric(struct ceph_inode_info *ci, int mask, + int t); +extern int __ceph_caps_issued_other(struct ceph_inode_info *ci, + struct ceph_cap *cap); + +static inline int ceph_caps_issued(struct ceph_inode_info *ci) +{ + int issued; + spin_lock(&ci->i_ceph_lock); + issued = __ceph_caps_issued(ci, NULL); + spin_unlock(&ci->i_ceph_lock); + return issued; +} + +static inline int ceph_caps_issued_mask_metric(struct ceph_inode_info *ci, + int mask, int touch) +{ + int r; + spin_lock(&ci->i_ceph_lock); + r = __ceph_caps_issued_mask_metric(ci, mask, touch); + spin_unlock(&ci->i_ceph_lock); + return r; +} + +static inline int __ceph_caps_dirty(struct ceph_inode_info *ci) +{ + return ci->i_dirty_caps | ci->i_flushing_caps; +} +extern struct ceph_cap_flush *ceph_alloc_cap_flush(void); +extern void ceph_free_cap_flush(struct ceph_cap_flush *cf); +extern int __ceph_mark_dirty_caps(struct ceph_inode_info *ci, int mask, + struct ceph_cap_flush **pcf); + +extern int __ceph_caps_revoking_other(struct ceph_inode_info *ci, + struct ceph_cap *ocap, int mask); +extern int __ceph_caps_used(struct ceph_inode_info *ci); + +static inline bool __ceph_is_file_opened(struct ceph_inode_info *ci) +{ + return ci->i_nr_by_mode[0]; +} +extern int __ceph_caps_file_wanted(struct ceph_inode_info *ci); +extern int __ceph_caps_wanted(struct ceph_inode_info *ci); + +/* what the mds thinks we want */ +extern int __ceph_caps_mds_wanted(struct ceph_inode_info *ci, bool check); + +extern void ceph_caps_init(struct ceph_mds_client *mdsc); +extern void ceph_caps_finalize(struct ceph_mds_client *mdsc); +extern void ceph_adjust_caps_max_min(struct ceph_mds_client *mdsc, + struct ceph_mount_options *fsopt); +extern int ceph_reserve_caps(struct ceph_mds_client *mdsc, + struct ceph_cap_reservation *ctx, int need); +extern void ceph_unreserve_caps(struct ceph_mds_client *mdsc, + struct ceph_cap_reservation *ctx); +extern void ceph_reservation_status(struct ceph_fs_client *client, + int *total, int *avail, int *used, + int *reserved, int *min); +extern void change_auth_cap_ses(struct ceph_inode_info *ci, + struct ceph_mds_session *session); + + + +/* + * we keep buffered readdir results attached to file->private_data + */ +#define CEPH_F_SYNC 1 +#define CEPH_F_ATEND 2 + +struct ceph_file_info { + short fmode; /* initialized on open */ + short flags; /* CEPH_F_* */ + + spinlock_t rw_contexts_lock; + struct list_head rw_contexts; + + u32 filp_gen; +}; + +struct ceph_dir_file_info { + struct ceph_file_info file_info; + + /* readdir: position within the dir */ + u32 frag; + struct ceph_mds_request *last_readdir; + + /* readdir: position within a frag */ + unsigned next_offset; /* offset of next chunk (last_name's + 1) */ + char *last_name; /* last entry in previous chunk */ + long long dir_release_count; + long long dir_ordered_count; + int readdir_cache_idx; + + /* used for -o dirstat read() on directory thing */ + char *dir_info; + int dir_info_len; +}; + +struct ceph_rw_context { + struct list_head list; + struct task_struct *thread; + int caps; +}; + +#define CEPH_DEFINE_RW_CONTEXT(_name, _caps) \ + struct ceph_rw_context _name = { \ + .thread = current, \ + .caps = _caps, \ + } + +static inline void ceph_add_rw_context(struct ceph_file_info *cf, + struct ceph_rw_context *ctx) +{ + spin_lock(&cf->rw_contexts_lock); + list_add(&ctx->list, &cf->rw_contexts); + spin_unlock(&cf->rw_contexts_lock); +} + +static inline void ceph_del_rw_context(struct ceph_file_info *cf, + struct ceph_rw_context *ctx) +{ + spin_lock(&cf->rw_contexts_lock); + list_del(&ctx->list); + spin_unlock(&cf->rw_contexts_lock); +} + +static inline struct ceph_rw_context* +ceph_find_rw_context(struct ceph_file_info *cf) +{ + struct ceph_rw_context *ctx, *found = NULL; + spin_lock(&cf->rw_contexts_lock); + list_for_each_entry(ctx, &cf->rw_contexts, list) { + if (ctx->thread == current) { + found = ctx; + break; + } + } + spin_unlock(&cf->rw_contexts_lock); + return found; +} + +struct ceph_readdir_cache_control { + struct folio *folio; + struct dentry **dentries; + int index; +}; + +/* + * A "snap realm" describes a subset of the file hierarchy sharing + * the same set of snapshots that apply to it. The realms themselves + * are organized into a hierarchy, such that children inherit (some of) + * the snapshots of their parents. + * + * All inodes within the realm that have capabilities are linked into a + * per-realm list. + */ +struct ceph_snap_realm { + u64 ino; + struct inode *inode; + atomic_t nref; + struct rb_node node; + + u64 created, seq; + u64 parent_ino; + u64 parent_since; /* snapid when our current parent became so */ + + u64 *prior_parent_snaps; /* snaps inherited from any parents we */ + u32 num_prior_parent_snaps; /* had prior to parent_since */ + u64 *snaps; /* snaps specific to this realm */ + u32 num_snaps; + + struct ceph_snap_realm *parent; + struct list_head children; /* list of child realms */ + struct list_head child_item; + + struct list_head empty_item; /* if i have ref==0 */ + + struct list_head dirty_item; /* if realm needs new context */ + + struct list_head rebuild_item; /* rebuild snap realms _downward_ in hierarchy */ + + /* the current set of snaps for this realm */ + struct ceph_snap_context *cached_context; + + struct list_head inodes_with_caps; + spinlock_t inodes_with_caps_lock; +}; + +static inline int default_congestion_kb(void) +{ + int congestion_kb; + + /* + * Copied from NFS + * + * congestion size, scale with available memory. + * + * 64MB: 8192k + * 128MB: 11585k + * 256MB: 16384k + * 512MB: 23170k + * 1GB: 32768k + * 2GB: 46340k + * 4GB: 65536k + * 8GB: 92681k + * 16GB: 131072k + * + * This allows larger machines to have larger/more transfers. + * Limit the default to 256M + */ + congestion_kb = (16*int_sqrt(totalram_pages())) << (PAGE_SHIFT-10); + if (congestion_kb > 256*1024) + congestion_kb = 256*1024; + + return congestion_kb; +} + + +/* super.c */ +extern int ceph_force_reconnect(struct super_block *sb); +/* snap.c */ +struct ceph_snap_realm *ceph_lookup_snap_realm(struct ceph_mds_client *mdsc, + u64 ino); +extern void ceph_get_snap_realm(struct ceph_mds_client *mdsc, + struct ceph_snap_realm *realm); +extern void ceph_put_snap_realm(struct ceph_mds_client *mdsc, + struct ceph_snap_realm *realm); +extern int ceph_update_snap_trace(struct ceph_mds_client *m, + void *p, void *e, bool deletion, + struct ceph_snap_realm **realm_ret); +void ceph_change_snap_realm(struct inode *inode, struct ceph_snap_realm *realm); +extern void ceph_handle_snap(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session, + struct ceph_msg *msg); +extern int __ceph_finish_cap_snap(struct ceph_inode_info *ci, + struct ceph_cap_snap *capsnap); +extern void ceph_cleanup_global_and_empty_realms(struct ceph_mds_client *mdsc); + +extern struct ceph_snapid_map *ceph_get_snapid_map(struct ceph_mds_client *mdsc, + u64 snap); +extern void ceph_put_snapid_map(struct ceph_mds_client* mdsc, + struct ceph_snapid_map *sm); +extern void ceph_trim_snapid_map(struct ceph_mds_client *mdsc); +extern void ceph_cleanup_snapid_map(struct ceph_mds_client *mdsc); +void ceph_umount_begin(struct super_block *sb); + + +/* + * a cap_snap is "pending" if it is still awaiting an in-progress + * sync write (that may/may not still update size, mtime, etc.). + */ +static inline bool __ceph_have_pending_cap_snap(struct ceph_inode_info *ci) +{ + return !list_empty(&ci->i_cap_snaps) && + list_last_entry(&ci->i_cap_snaps, struct ceph_cap_snap, + ci_item)->writing; +} + +/* inode.c */ +struct ceph_mds_reply_info_in; +struct ceph_mds_reply_dirfrag; +struct ceph_acl_sec_ctx; + +extern const struct inode_operations ceph_file_iops; + +extern struct inode *ceph_alloc_inode(struct super_block *sb); +extern void ceph_evict_inode(struct inode *inode); +extern void ceph_free_inode(struct inode *inode); + +struct inode *ceph_new_inode(struct inode *dir, struct dentry *dentry, + umode_t *mode, struct ceph_acl_sec_ctx *as_ctx); +void ceph_as_ctx_to_req(struct ceph_mds_request *req, + struct ceph_acl_sec_ctx *as_ctx); + +extern struct inode *ceph_get_inode(struct super_block *sb, + struct ceph_vino vino, + struct inode *newino); +extern struct inode *ceph_get_snapdir(struct inode *parent); +extern int ceph_fill_file_size(struct inode *inode, int issued, + u32 truncate_seq, u64 truncate_size, u64 size); +extern void ceph_inode_set_subvolume(struct inode *inode, u64 subvolume_id); +extern void ceph_fill_file_time(struct inode *inode, int issued, + u64 time_warp_seq, struct timespec64 *ctime, + struct timespec64 *mtime, + struct timespec64 *atime); +extern int ceph_fill_inode(struct inode *inode, struct page *locked_page, + struct ceph_mds_reply_info_in *iinfo, + struct ceph_mds_reply_dirfrag *dirinfo, + struct ceph_mds_session *session, int cap_fmode, + struct ceph_cap_reservation *caps_reservation); +extern int ceph_fill_trace(struct super_block *sb, + struct ceph_mds_request *req); +extern int ceph_readdir_prepopulate(struct ceph_mds_request *req, + struct ceph_mds_session *session); + +extern bool ceph_inode_set_size(struct inode *inode, loff_t size); +extern void __ceph_do_pending_vmtruncate(struct inode *inode); + +void ceph_queue_inode_work(struct inode *inode, int work_bit); + +static inline void ceph_queue_vmtruncate(struct inode *inode) +{ + ceph_queue_inode_work(inode, CEPH_I_WORK_VMTRUNCATE); +} + +static inline void ceph_queue_invalidate(struct inode *inode) +{ + ceph_queue_inode_work(inode, CEPH_I_WORK_INVALIDATE_PAGES); +} + +static inline void ceph_queue_writeback(struct inode *inode) +{ + ceph_queue_inode_work(inode, CEPH_I_WORK_WRITEBACK); +} + +static inline void ceph_queue_check_caps(struct inode *inode) +{ + ceph_queue_inode_work(inode, CEPH_I_WORK_CHECK_CAPS); +} + +static inline void ceph_queue_flush_snaps(struct inode *inode) +{ + ceph_queue_inode_work(inode, CEPH_I_WORK_FLUSH_SNAPS); +} + +extern int ceph_try_to_choose_auth_mds(struct inode *inode, int mask); +extern int __ceph_do_getattr(struct inode *inode, struct page *locked_page, + int mask, bool force); +static inline int ceph_do_getattr(struct inode *inode, int mask, bool force) +{ + return __ceph_do_getattr(inode, NULL, mask, force); +} +extern int ceph_permission(struct mnt_idmap *idmap, + struct inode *inode, int mask); + +struct ceph_iattr { + struct ceph_fscrypt_auth *fscrypt_auth; +}; + +extern int __ceph_setattr(struct mnt_idmap *idmap, struct inode *inode, + struct iattr *attr, struct ceph_iattr *cia); +extern int ceph_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, struct iattr *attr); +extern int ceph_getattr(struct mnt_idmap *idmap, + const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int flags); +void ceph_inode_shutdown(struct inode *inode); + +static inline bool ceph_inode_is_shutdown(struct inode *inode) +{ + unsigned long flags = READ_ONCE(ceph_inode(inode)->i_ceph_flags); + struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode); + int state = READ_ONCE(fsc->mount_state); + + return (flags & CEPH_I_SHUTDOWN) || state >= CEPH_MOUNT_SHUTDOWN; +} + +/* xattr.c */ +int __ceph_setxattr(struct inode *, const char *, const void *, size_t, int); +int ceph_do_getvxattr(struct inode *inode, const char *name, void *value, size_t size); +ssize_t __ceph_getxattr(struct inode *, const char *, void *, size_t); +extern ssize_t ceph_listxattr(struct dentry *, char *, size_t); +extern struct ceph_buffer *__ceph_build_xattrs_blob(struct ceph_inode_info *ci); +extern void __ceph_destroy_xattrs(struct ceph_inode_info *ci); +extern const struct xattr_handler * const ceph_xattr_handlers[]; + +struct ceph_acl_sec_ctx { +#ifdef CONFIG_CEPH_FS_POSIX_ACL + void *default_acl; + void *acl; +#endif +#ifdef CONFIG_CEPH_FS_SECURITY_LABEL + struct lsm_context lsmctx; +#endif +#ifdef CONFIG_FS_ENCRYPTION + struct ceph_fscrypt_auth *fscrypt_auth; +#endif + struct ceph_pagelist *pagelist; +}; + +#ifdef CONFIG_SECURITY +extern bool ceph_security_xattr_deadlock(struct inode *in); +extern bool ceph_security_xattr_wanted(struct inode *in); +#else +static inline bool ceph_security_xattr_deadlock(struct inode *in) +{ + return false; +} +static inline bool ceph_security_xattr_wanted(struct inode *in) +{ + return false; +} +#endif + +#ifdef CONFIG_CEPH_FS_SECURITY_LABEL +extern int ceph_security_init_secctx(struct dentry *dentry, umode_t mode, + struct ceph_acl_sec_ctx *ctx); +static inline void ceph_security_invalidate_secctx(struct inode *inode) +{ + security_inode_invalidate_secctx(inode); +} +#else +static inline int ceph_security_init_secctx(struct dentry *dentry, umode_t mode, + struct ceph_acl_sec_ctx *ctx) +{ + return 0; +} +static inline void ceph_security_invalidate_secctx(struct inode *inode) +{ +} +#endif + +void ceph_release_acl_sec_ctx(struct ceph_acl_sec_ctx *as_ctx); + +/* acl.c */ +#ifdef CONFIG_CEPH_FS_POSIX_ACL + +struct posix_acl *ceph_get_acl(struct inode *, int, bool); +int ceph_set_acl(struct mnt_idmap *idmap, + struct dentry *dentry, struct posix_acl *acl, int type); +int ceph_pre_init_acls(struct inode *dir, umode_t *mode, + struct ceph_acl_sec_ctx *as_ctx); +void ceph_init_inode_acls(struct inode *inode, + struct ceph_acl_sec_ctx *as_ctx); + +static inline void ceph_forget_all_cached_acls(struct inode *inode) +{ + forget_all_cached_acls(inode); +} + +#else + +#define ceph_get_acl NULL +#define ceph_set_acl NULL + +static inline int ceph_pre_init_acls(struct inode *dir, umode_t *mode, + struct ceph_acl_sec_ctx *as_ctx) +{ + return 0; +} +static inline void ceph_init_inode_acls(struct inode *inode, + struct ceph_acl_sec_ctx *as_ctx) +{ +} + +static inline void ceph_forget_all_cached_acls(struct inode *inode) +{ +} + +#endif + +/* caps.c */ +extern const char *ceph_cap_string(int c); +extern void ceph_handle_caps(struct ceph_mds_session *session, + struct ceph_msg *msg); +extern struct ceph_cap *ceph_get_cap(struct ceph_mds_client *mdsc, + struct ceph_cap_reservation *ctx); +extern void ceph_add_cap(struct inode *inode, + struct ceph_mds_session *session, u64 cap_id, + unsigned issued, unsigned wanted, + unsigned cap, unsigned seq, u64 realmino, int flags, + struct ceph_cap **new_cap); +extern void __ceph_remove_cap(struct ceph_cap *cap, bool queue_release); +extern void ceph_remove_cap(struct ceph_mds_client *mdsc, struct ceph_cap *cap, + bool queue_release); +extern void __ceph_remove_caps(struct ceph_inode_info *ci); +extern void ceph_put_cap(struct ceph_mds_client *mdsc, + struct ceph_cap *cap); +extern int ceph_is_any_caps(struct inode *inode); + +extern int ceph_write_inode(struct inode *inode, struct writeback_control *wbc); +extern int ceph_fsync(struct file *file, loff_t start, loff_t end, + int datasync); +extern void ceph_early_kick_flushing_caps(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session); +extern void ceph_kick_flushing_caps(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session); +void ceph_kick_flushing_inode_caps(struct ceph_mds_session *session, + struct ceph_inode_info *ci); +extern struct ceph_cap *__get_cap_for_mds(struct ceph_inode_info *ci, + int mds); +extern struct ceph_cap *ceph_get_cap_for_mds(struct ceph_inode_info *ci, + int mds); +extern void ceph_take_cap_refs(struct ceph_inode_info *ci, int caps, + bool snap_rwsem_locked); +extern void ceph_get_cap_refs(struct ceph_inode_info *ci, int caps); +extern void ceph_put_cap_refs(struct ceph_inode_info *ci, int had); +extern void ceph_put_cap_refs_async(struct ceph_inode_info *ci, int had); +extern void ceph_put_wrbuffer_cap_refs(struct ceph_inode_info *ci, int nr, + struct ceph_snap_context *snapc); +extern void __ceph_remove_capsnap(struct inode *inode, + struct ceph_cap_snap *capsnap, + bool *wake_ci, bool *wake_mdsc); +extern void ceph_remove_capsnap(struct inode *inode, + struct ceph_cap_snap *capsnap, + bool *wake_ci, bool *wake_mdsc); +extern void ceph_flush_snaps(struct ceph_inode_info *ci, + struct ceph_mds_session **psession); +extern bool __ceph_should_report_size(struct ceph_inode_info *ci); +extern void ceph_check_caps(struct ceph_inode_info *ci, int flags); +extern unsigned long ceph_check_delayed_caps(struct ceph_mds_client *mdsc); +extern void ceph_flush_dirty_caps(struct ceph_mds_client *mdsc); +extern void ceph_flush_cap_releases(struct ceph_mds_client *mdsc); +extern int ceph_drop_caps_for_unlink(struct inode *inode); +extern int ceph_encode_inode_release(void **p, struct inode *inode, + int mds, int drop, int unless, int force); +extern int ceph_encode_dentry_release(void **p, struct dentry *dn, + struct inode *dir, + int mds, int drop, int unless); + +extern int __ceph_get_caps(struct inode *inode, struct ceph_file_info *fi, + int need, int want, loff_t endoff, int *got); +extern int ceph_get_caps(struct file *filp, int need, int want, + loff_t endoff, int *got); +extern int ceph_try_get_caps(struct inode *inode, + int need, int want, bool nonblock, int *got); + +/* for counting open files by mode */ +extern void ceph_get_fmode(struct ceph_inode_info *ci, int mode, int count); +extern void ceph_put_fmode(struct ceph_inode_info *ci, int mode, int count); +extern void __ceph_touch_fmode(struct ceph_inode_info *ci, + struct ceph_mds_client *mdsc, int fmode); + +/* addr.c */ +extern const struct address_space_operations ceph_aops; +extern const struct netfs_request_ops ceph_netfs_ops; +int ceph_mmap_prepare(struct vm_area_desc *desc); +extern int ceph_uninline_data(struct file *file); +extern int ceph_pool_perm_check(struct inode *inode, int need); +extern void ceph_pool_perm_destroy(struct ceph_mds_client* mdsc); +int ceph_purge_inode_cap(struct inode *inode, struct ceph_cap *cap, bool *invalidate); + +static inline bool ceph_has_inline_data(struct ceph_inode_info *ci) +{ + if (ci->i_inline_version == CEPH_INLINE_NONE || + ci->i_inline_version == 1) /* initial version, no data */ + return false; + return true; +} + +/* file.c */ +extern const struct file_operations ceph_file_fops; + +extern int ceph_renew_caps(struct inode *inode, int fmode); +extern int ceph_open(struct inode *inode, struct file *file); +extern int ceph_atomic_open(struct inode *dir, struct dentry *dentry, + struct file *file, unsigned flags, umode_t mode); +extern ssize_t __ceph_sync_read(struct inode *inode, loff_t *ki_pos, + struct iov_iter *to, int *retry_op, + u64 *last_objver); +extern int ceph_release(struct inode *inode, struct file *filp); +extern void ceph_fill_inline_data(struct inode *inode, struct page *locked_page, + char *data, size_t len); + +/* dir.c */ +extern const struct file_operations ceph_dir_fops; +extern const struct file_operations ceph_snapdir_fops; +extern const struct inode_operations ceph_dir_iops; +extern const struct inode_operations ceph_snapdir_iops; +extern const struct dentry_operations ceph_dentry_ops; + +extern loff_t ceph_make_fpos(unsigned high, unsigned off, bool hash_order); +extern int ceph_handle_notrace_create(struct inode *dir, struct dentry *dentry); +extern struct dentry *ceph_handle_snapdir(struct ceph_mds_request *req, + struct dentry *dentry); +extern struct dentry *ceph_finish_lookup(struct ceph_mds_request *req, + struct dentry *dentry, int err); + +extern void __ceph_dentry_lease_touch(struct ceph_dentry_info *di); +extern void __ceph_dentry_dir_lease_touch(struct ceph_dentry_info *di); +extern void ceph_invalidate_dentry_lease(struct dentry *dentry); +extern int ceph_trim_dentries(struct ceph_mds_client *mdsc); +extern unsigned ceph_dentry_hash(struct inode *dir, struct dentry *dn); +extern void ceph_readdir_cache_release(struct ceph_readdir_cache_control *ctl); + +/* ioctl.c */ +extern long ceph_ioctl(struct file *file, unsigned int cmd, unsigned long arg); + +/* export.c */ +extern const struct export_operations ceph_export_ops; +struct inode *ceph_lookup_inode(struct super_block *sb, u64 ino); + +/* locks.c */ +extern __init void ceph_flock_init(void); +extern int ceph_lock(struct file *file, int cmd, struct file_lock *fl); +extern int ceph_flock(struct file *file, int cmd, struct file_lock *fl); +extern void ceph_count_locks(struct inode *inode, int *p_num, int *f_num); +extern int ceph_encode_locks_to_buffer(struct inode *inode, + struct ceph_filelock *flocks, + int num_fcntl_locks, + int num_flock_locks); +extern int ceph_locks_to_pagelist(struct ceph_filelock *flocks, + struct ceph_pagelist *pagelist, + int num_fcntl_locks, int num_flock_locks); + +/* debugfs.c */ +extern void ceph_fs_debugfs_init(struct ceph_fs_client *client); +extern void ceph_fs_debugfs_cleanup(struct ceph_fs_client *client); + +/* quota.c */ + +enum quota_get_realm { + QUOTA_GET_MAX_FILES, + QUOTA_GET_MAX_BYTES, + QUOTA_GET_ANY +}; + +static inline bool __ceph_has_quota(struct ceph_inode_info *ci, + enum quota_get_realm which) +{ + bool has_quota = false; + + switch (which) { + case QUOTA_GET_MAX_BYTES: + has_quota = !!ci->i_max_bytes; + break; + case QUOTA_GET_MAX_FILES: + has_quota = !!ci->i_max_files; + break; + default: + has_quota = !!(ci->i_max_files || ci->i_max_bytes); + } + return has_quota; +} + +extern void ceph_adjust_quota_realms_count(struct inode *inode, bool inc); + +static inline void __ceph_update_quota(struct ceph_inode_info *ci, + u64 max_bytes, u64 max_files) +{ + bool had_quota, has_quota; + had_quota = __ceph_has_quota(ci, QUOTA_GET_ANY); + ci->i_max_bytes = max_bytes; + ci->i_max_files = max_files; + has_quota = __ceph_has_quota(ci, QUOTA_GET_ANY); + + if (had_quota != has_quota) + ceph_adjust_quota_realms_count(&ci->netfs.inode, has_quota); +} + +static inline int __ceph_sparse_read_ext_count(struct inode *inode, u64 len) +{ + int cnt = 0; + + if (IS_ENCRYPTED(inode)) { + cnt = len >> CEPH_FSCRYPT_BLOCK_SHIFT; + if (cnt > CEPH_SPARSE_EXT_ARRAY_INITIAL) + cnt = 0; + } + + return cnt; +} + +extern void ceph_handle_quota(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session, + struct ceph_msg *msg); +extern bool ceph_quota_is_max_files_exceeded(struct inode *inode); +extern bool ceph_quota_is_same_realm(struct inode *old, struct inode *new); +extern bool ceph_quota_is_max_bytes_exceeded(struct inode *inode, + loff_t newlen); +extern bool ceph_quota_is_max_bytes_approaching(struct inode *inode, + loff_t newlen); +extern bool ceph_quota_update_statfs(struct ceph_fs_client *fsc, + struct kstatfs *buf); +extern void ceph_cleanup_quotarealms_inodes(struct ceph_mds_client *mdsc); + +bool ceph_inc_mds_stopping_blocker(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session); +void ceph_dec_mds_stopping_blocker(struct ceph_mds_client *mdsc); +bool ceph_inc_osd_stopping_blocker(struct ceph_mds_client *mdsc); +void ceph_dec_osd_stopping_blocker(struct ceph_mds_client *mdsc); +#endif /* _FS_CEPH_SUPER_H */ diff --git a/fs/ceph/util.c b/fs/ceph/util.c new file mode 100644 index 000000000..2c3487567 --- /dev/null +++ b/fs/ceph/util.c @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Some non-inline ceph helpers + */ +#include +#include + +/* + * return true if @layout appears to be valid + */ +int ceph_file_layout_is_valid(const struct ceph_file_layout *layout) +{ + __u32 su = layout->stripe_unit; + __u32 sc = layout->stripe_count; + __u32 os = layout->object_size; + + /* stripe unit, object size must be non-zero, 64k increment */ + if (!su || (su & (CEPH_MIN_STRIPE_UNIT-1))) + return 0; + if (!os || (os & (CEPH_MIN_STRIPE_UNIT-1))) + return 0; + /* object size must be a multiple of stripe unit */ + if (os < su || os % su) + return 0; + /* stripe count must be non-zero */ + if (!sc) + return 0; + return 1; +} + +void ceph_file_layout_from_legacy(struct ceph_file_layout *fl, + struct ceph_file_layout_legacy *legacy) +{ + fl->stripe_unit = le32_to_cpu(legacy->fl_stripe_unit); + fl->stripe_count = le32_to_cpu(legacy->fl_stripe_count); + fl->object_size = le32_to_cpu(legacy->fl_object_size); + fl->pool_id = le32_to_cpu(legacy->fl_pg_pool); + if (fl->pool_id == 0 && fl->stripe_unit == 0 && + fl->stripe_count == 0 && fl->object_size == 0) + fl->pool_id = -1; +} + +void ceph_file_layout_to_legacy(struct ceph_file_layout *fl, + struct ceph_file_layout_legacy *legacy) +{ + legacy->fl_stripe_unit = cpu_to_le32(fl->stripe_unit); + legacy->fl_stripe_count = cpu_to_le32(fl->stripe_count); + legacy->fl_object_size = cpu_to_le32(fl->object_size); + if (fl->pool_id >= 0) + legacy->fl_pg_pool = cpu_to_le32(fl->pool_id); + else + legacy->fl_pg_pool = 0; +} + +int ceph_flags_to_mode(int flags) +{ + int mode; + +#ifdef O_DIRECTORY /* fixme */ + if ((flags & O_DIRECTORY) == O_DIRECTORY) + return CEPH_FILE_MODE_PIN; +#endif + + switch (flags & O_ACCMODE) { + case O_WRONLY: + mode = CEPH_FILE_MODE_WR; + break; + case O_RDONLY: + mode = CEPH_FILE_MODE_RD; + break; + case O_RDWR: + case O_ACCMODE: /* this is what the VFS does */ + mode = CEPH_FILE_MODE_RDWR; + break; + } +#ifdef O_LAZY + if (flags & O_LAZY) + mode |= CEPH_FILE_MODE_LAZY; +#endif + + return mode; +} + +int ceph_caps_for_mode(int mode) +{ + int caps = CEPH_CAP_PIN; + + if (mode & CEPH_FILE_MODE_RD) + caps |= CEPH_CAP_FILE_SHARED | + CEPH_CAP_FILE_RD | CEPH_CAP_FILE_CACHE; + if (mode & CEPH_FILE_MODE_WR) + caps |= CEPH_CAP_FILE_EXCL | + CEPH_CAP_FILE_WR | CEPH_CAP_FILE_BUFFER | + CEPH_CAP_AUTH_SHARED | CEPH_CAP_AUTH_EXCL | + CEPH_CAP_XATTR_SHARED | CEPH_CAP_XATTR_EXCL; + if (mode & CEPH_FILE_MODE_LAZY) + caps |= CEPH_CAP_FILE_LAZYIO; + + return caps; +} diff --git a/fs/ceph/xattr.c b/fs/ceph/xattr.c new file mode 100644 index 000000000..860fc8e18 --- /dev/null +++ b/fs/ceph/xattr.c @@ -0,0 +1,1485 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +#include "super.h" +#include "mds_client.h" + +#include + +#include +#include +#include +#include + +#define XATTR_CEPH_PREFIX "ceph." +#define XATTR_CEPH_PREFIX_LEN (sizeof (XATTR_CEPH_PREFIX) - 1) + +static int __remove_xattr(struct ceph_inode_info *ci, + struct ceph_inode_xattr *xattr); + +static bool ceph_is_valid_xattr(const char *name) +{ + return !strncmp(name, XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN) || + !strncmp(name, XATTR_CEPH_PREFIX, XATTR_CEPH_PREFIX_LEN) || + !strncmp(name, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN) || + !strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN); +} + +/* + * These define virtual xattrs exposing the recursive directory + * statistics and layout metadata. + */ +struct ceph_vxattr { + char *name; + size_t name_size; /* strlen(name) + 1 (for '\0') */ + ssize_t (*getxattr_cb)(struct ceph_inode_info *ci, char *val, + size_t size); + bool (*exists_cb)(struct ceph_inode_info *ci); + unsigned int flags; +}; + +#define VXATTR_FLAG_READONLY (1<<0) +#define VXATTR_FLAG_HIDDEN (1<<1) +#define VXATTR_FLAG_RSTAT (1<<2) +#define VXATTR_FLAG_DIRSTAT (1<<3) + +/* layouts */ + +static bool ceph_vxattrcb_layout_exists(struct ceph_inode_info *ci) +{ + struct ceph_file_layout *fl = &ci->i_layout; + return (fl->stripe_unit > 0 || fl->stripe_count > 0 || + fl->object_size > 0 || fl->pool_id >= 0 || + rcu_dereference_raw(fl->pool_ns) != NULL); +} + +static ssize_t ceph_vxattrcb_layout(struct ceph_inode_info *ci, char *val, + size_t size) +{ + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(ci->netfs.inode.i_sb); + struct ceph_client *cl = fsc->client; + struct ceph_osd_client *osdc = &fsc->client->osdc; + struct ceph_string *pool_ns; + s64 pool = ci->i_layout.pool_id; + const char *pool_name; + const char *ns_field = " pool_namespace="; + char buf[128]; + size_t len, total_len = 0; + ssize_t ret; + + pool_ns = ceph_try_get_string(ci->i_layout.pool_ns); + + doutc(cl, "%p\n", &ci->netfs.inode); + down_read(&osdc->lock); + pool_name = ceph_pg_pool_name_by_id(osdc->osdmap, pool); + if (pool_name) { + len = snprintf(buf, sizeof(buf), + "stripe_unit=%u stripe_count=%u object_size=%u pool=", + ci->i_layout.stripe_unit, ci->i_layout.stripe_count, + ci->i_layout.object_size); + total_len = len + strlen(pool_name); + } else { + len = snprintf(buf, sizeof(buf), + "stripe_unit=%u stripe_count=%u object_size=%u pool=%lld", + ci->i_layout.stripe_unit, ci->i_layout.stripe_count, + ci->i_layout.object_size, pool); + total_len = len; + } + + if (pool_ns) + total_len += strlen(ns_field) + pool_ns->len; + + ret = total_len; + if (size >= total_len) { + memcpy(val, buf, len); + ret = len; + if (pool_name) { + len = strlen(pool_name); + memcpy(val + ret, pool_name, len); + ret += len; + } + if (pool_ns) { + len = strlen(ns_field); + memcpy(val + ret, ns_field, len); + ret += len; + memcpy(val + ret, pool_ns->str, pool_ns->len); + ret += pool_ns->len; + } + } + up_read(&osdc->lock); + ceph_put_string(pool_ns); + return ret; +} + +/* + * The convention with strings in xattrs is that they should not be NULL + * terminated, since we're returning the length with them. snprintf always + * NULL terminates however, so call it on a temporary buffer and then memcpy + * the result into place. + */ +static __printf(3, 4) +int ceph_fmt_xattr(char *val, size_t size, const char *fmt, ...) +{ + int ret; + va_list args; + char buf[96]; /* NB: reevaluate size if new vxattrs are added */ + + va_start(args, fmt); + ret = vsnprintf(buf, size ? sizeof(buf) : 0, fmt, args); + va_end(args); + + /* Sanity check */ + if (size && ret + 1 > sizeof(buf)) { + WARN_ONCE(true, "Returned length too big (%d)", ret); + return -E2BIG; + } + + if (ret <= size) + memcpy(val, buf, ret); + return ret; +} + +static ssize_t ceph_vxattrcb_layout_stripe_unit(struct ceph_inode_info *ci, + char *val, size_t size) +{ + return ceph_fmt_xattr(val, size, "%u", ci->i_layout.stripe_unit); +} + +static ssize_t ceph_vxattrcb_layout_stripe_count(struct ceph_inode_info *ci, + char *val, size_t size) +{ + return ceph_fmt_xattr(val, size, "%u", ci->i_layout.stripe_count); +} + +static ssize_t ceph_vxattrcb_layout_object_size(struct ceph_inode_info *ci, + char *val, size_t size) +{ + return ceph_fmt_xattr(val, size, "%u", ci->i_layout.object_size); +} + +static ssize_t ceph_vxattrcb_layout_pool(struct ceph_inode_info *ci, + char *val, size_t size) +{ + ssize_t ret; + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(ci->netfs.inode.i_sb); + struct ceph_osd_client *osdc = &fsc->client->osdc; + s64 pool = ci->i_layout.pool_id; + const char *pool_name; + + down_read(&osdc->lock); + pool_name = ceph_pg_pool_name_by_id(osdc->osdmap, pool); + if (pool_name) { + ret = strlen(pool_name); + if (ret <= size) + memcpy(val, pool_name, ret); + } else { + ret = ceph_fmt_xattr(val, size, "%lld", pool); + } + up_read(&osdc->lock); + return ret; +} + +static ssize_t ceph_vxattrcb_layout_pool_namespace(struct ceph_inode_info *ci, + char *val, size_t size) +{ + ssize_t ret = 0; + struct ceph_string *ns = ceph_try_get_string(ci->i_layout.pool_ns); + + if (ns) { + ret = ns->len; + if (ret <= size) + memcpy(val, ns->str, ret); + ceph_put_string(ns); + } + return ret; +} + +/* directories */ + +static ssize_t ceph_vxattrcb_dir_entries(struct ceph_inode_info *ci, char *val, + size_t size) +{ + return ceph_fmt_xattr(val, size, "%lld", ci->i_files + ci->i_subdirs); +} + +static ssize_t ceph_vxattrcb_dir_files(struct ceph_inode_info *ci, char *val, + size_t size) +{ + return ceph_fmt_xattr(val, size, "%lld", ci->i_files); +} + +static ssize_t ceph_vxattrcb_dir_subdirs(struct ceph_inode_info *ci, char *val, + size_t size) +{ + return ceph_fmt_xattr(val, size, "%lld", ci->i_subdirs); +} + +static ssize_t ceph_vxattrcb_dir_rentries(struct ceph_inode_info *ci, char *val, + size_t size) +{ + return ceph_fmt_xattr(val, size, "%lld", + ci->i_rfiles + ci->i_rsubdirs); +} + +static ssize_t ceph_vxattrcb_dir_rfiles(struct ceph_inode_info *ci, char *val, + size_t size) +{ + return ceph_fmt_xattr(val, size, "%lld", ci->i_rfiles); +} + +static ssize_t ceph_vxattrcb_dir_rsubdirs(struct ceph_inode_info *ci, char *val, + size_t size) +{ + return ceph_fmt_xattr(val, size, "%lld", ci->i_rsubdirs); +} + +static ssize_t ceph_vxattrcb_dir_rsnaps(struct ceph_inode_info *ci, char *val, + size_t size) +{ + return ceph_fmt_xattr(val, size, "%lld", ci->i_rsnaps); +} + +static ssize_t ceph_vxattrcb_dir_rbytes(struct ceph_inode_info *ci, char *val, + size_t size) +{ + return ceph_fmt_xattr(val, size, "%lld", ci->i_rbytes); +} + +static ssize_t ceph_vxattrcb_dir_rctime(struct ceph_inode_info *ci, char *val, + size_t size) +{ + return ceph_fmt_xattr(val, size, "%ptSp", &ci->i_rctime); +} + +/* dir pin */ +static bool ceph_vxattrcb_dir_pin_exists(struct ceph_inode_info *ci) +{ + return ci->i_dir_pin != -ENODATA; +} + +static ssize_t ceph_vxattrcb_dir_pin(struct ceph_inode_info *ci, char *val, + size_t size) +{ + return ceph_fmt_xattr(val, size, "%d", (int)ci->i_dir_pin); +} + +/* quotas */ +static bool ceph_vxattrcb_quota_exists(struct ceph_inode_info *ci) +{ + bool ret = false; + spin_lock(&ci->i_ceph_lock); + if ((ci->i_max_files || ci->i_max_bytes) && + ci->i_vino.snap == CEPH_NOSNAP && + ci->i_snap_realm && + ci->i_snap_realm->ino == ci->i_vino.ino) + ret = true; + spin_unlock(&ci->i_ceph_lock); + return ret; +} + +static ssize_t ceph_vxattrcb_quota(struct ceph_inode_info *ci, char *val, + size_t size) +{ + return ceph_fmt_xattr(val, size, "max_bytes=%llu max_files=%llu", + ci->i_max_bytes, ci->i_max_files); +} + +static ssize_t ceph_vxattrcb_quota_max_bytes(struct ceph_inode_info *ci, + char *val, size_t size) +{ + return ceph_fmt_xattr(val, size, "%llu", ci->i_max_bytes); +} + +static ssize_t ceph_vxattrcb_quota_max_files(struct ceph_inode_info *ci, + char *val, size_t size) +{ + return ceph_fmt_xattr(val, size, "%llu", ci->i_max_files); +} + +/* snapshots */ +static bool ceph_vxattrcb_snap_btime_exists(struct ceph_inode_info *ci) +{ + return (ci->i_snap_btime.tv_sec != 0 || ci->i_snap_btime.tv_nsec != 0); +} + +static ssize_t ceph_vxattrcb_snap_btime(struct ceph_inode_info *ci, char *val, + size_t size) +{ + return ceph_fmt_xattr(val, size, "%ptSp", &ci->i_snap_btime); +} + +static ssize_t ceph_vxattrcb_cluster_fsid(struct ceph_inode_info *ci, + char *val, size_t size) +{ + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(ci->netfs.inode.i_sb); + + return ceph_fmt_xattr(val, size, "%pU", &fsc->client->fsid); +} + +static ssize_t ceph_vxattrcb_client_id(struct ceph_inode_info *ci, + char *val, size_t size) +{ + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(ci->netfs.inode.i_sb); + + return ceph_fmt_xattr(val, size, "client%lld", + ceph_client_gid(fsc->client)); +} + +static ssize_t ceph_vxattrcb_caps(struct ceph_inode_info *ci, char *val, + size_t size) +{ + int issued; + + spin_lock(&ci->i_ceph_lock); + issued = __ceph_caps_issued(ci, NULL); + spin_unlock(&ci->i_ceph_lock); + + return ceph_fmt_xattr(val, size, "%s/0x%x", + ceph_cap_string(issued), issued); +} + +static ssize_t ceph_vxattrcb_auth_mds(struct ceph_inode_info *ci, + char *val, size_t size) +{ + int ret; + + spin_lock(&ci->i_ceph_lock); + ret = ceph_fmt_xattr(val, size, "%d", + ci->i_auth_cap ? ci->i_auth_cap->session->s_mds : -1); + spin_unlock(&ci->i_ceph_lock); + return ret; +} + +#if IS_ENABLED(CONFIG_FS_ENCRYPTION) +static bool ceph_vxattrcb_fscrypt_auth_exists(struct ceph_inode_info *ci) +{ + return ci->fscrypt_auth_len; +} + +static ssize_t ceph_vxattrcb_fscrypt_auth(struct ceph_inode_info *ci, + char *val, size_t size) +{ + if (size) { + if (size < ci->fscrypt_auth_len) + return -ERANGE; + memcpy(val, ci->fscrypt_auth, ci->fscrypt_auth_len); + } + return ci->fscrypt_auth_len; +} +#endif /* CONFIG_FS_ENCRYPTION */ + +#define CEPH_XATTR_NAME(_type, _name) XATTR_CEPH_PREFIX #_type "." #_name +#define CEPH_XATTR_NAME2(_type, _name, _name2) \ + XATTR_CEPH_PREFIX #_type "." #_name "." #_name2 + +#define XATTR_NAME_CEPH(_type, _name, _flags) \ + { \ + .name = CEPH_XATTR_NAME(_type, _name), \ + .name_size = sizeof (CEPH_XATTR_NAME(_type, _name)), \ + .getxattr_cb = ceph_vxattrcb_ ## _type ## _ ## _name, \ + .exists_cb = NULL, \ + .flags = (VXATTR_FLAG_READONLY | _flags), \ + } +#define XATTR_RSTAT_FIELD(_type, _name) \ + XATTR_NAME_CEPH(_type, _name, VXATTR_FLAG_RSTAT) +#define XATTR_RSTAT_FIELD_UPDATABLE(_type, _name) \ + { \ + .name = CEPH_XATTR_NAME(_type, _name), \ + .name_size = sizeof (CEPH_XATTR_NAME(_type, _name)), \ + .getxattr_cb = ceph_vxattrcb_ ## _type ## _ ## _name, \ + .exists_cb = NULL, \ + .flags = VXATTR_FLAG_RSTAT, \ + } +#define XATTR_LAYOUT_FIELD(_type, _name, _field) \ + { \ + .name = CEPH_XATTR_NAME2(_type, _name, _field), \ + .name_size = sizeof (CEPH_XATTR_NAME2(_type, _name, _field)), \ + .getxattr_cb = ceph_vxattrcb_ ## _name ## _ ## _field, \ + .exists_cb = ceph_vxattrcb_layout_exists, \ + .flags = VXATTR_FLAG_HIDDEN, \ + } +#define XATTR_QUOTA_FIELD(_type, _name) \ + { \ + .name = CEPH_XATTR_NAME(_type, _name), \ + .name_size = sizeof(CEPH_XATTR_NAME(_type, _name)), \ + .getxattr_cb = ceph_vxattrcb_ ## _type ## _ ## _name, \ + .exists_cb = ceph_vxattrcb_quota_exists, \ + .flags = VXATTR_FLAG_HIDDEN, \ + } + +static struct ceph_vxattr ceph_dir_vxattrs[] = { + { + .name = "ceph.dir.layout", + .name_size = sizeof("ceph.dir.layout"), + .getxattr_cb = ceph_vxattrcb_layout, + .exists_cb = ceph_vxattrcb_layout_exists, + .flags = VXATTR_FLAG_HIDDEN, + }, + XATTR_LAYOUT_FIELD(dir, layout, stripe_unit), + XATTR_LAYOUT_FIELD(dir, layout, stripe_count), + XATTR_LAYOUT_FIELD(dir, layout, object_size), + XATTR_LAYOUT_FIELD(dir, layout, pool), + XATTR_LAYOUT_FIELD(dir, layout, pool_namespace), + XATTR_NAME_CEPH(dir, entries, VXATTR_FLAG_DIRSTAT), + XATTR_NAME_CEPH(dir, files, VXATTR_FLAG_DIRSTAT), + XATTR_NAME_CEPH(dir, subdirs, VXATTR_FLAG_DIRSTAT), + XATTR_RSTAT_FIELD(dir, rentries), + XATTR_RSTAT_FIELD(dir, rfiles), + XATTR_RSTAT_FIELD(dir, rsubdirs), + XATTR_RSTAT_FIELD(dir, rsnaps), + XATTR_RSTAT_FIELD(dir, rbytes), + XATTR_RSTAT_FIELD_UPDATABLE(dir, rctime), + { + .name = "ceph.dir.pin", + .name_size = sizeof("ceph.dir.pin"), + .getxattr_cb = ceph_vxattrcb_dir_pin, + .exists_cb = ceph_vxattrcb_dir_pin_exists, + .flags = VXATTR_FLAG_HIDDEN, + }, + { + .name = "ceph.quota", + .name_size = sizeof("ceph.quota"), + .getxattr_cb = ceph_vxattrcb_quota, + .exists_cb = ceph_vxattrcb_quota_exists, + .flags = VXATTR_FLAG_HIDDEN, + }, + XATTR_QUOTA_FIELD(quota, max_bytes), + XATTR_QUOTA_FIELD(quota, max_files), + { + .name = "ceph.snap.btime", + .name_size = sizeof("ceph.snap.btime"), + .getxattr_cb = ceph_vxattrcb_snap_btime, + .exists_cb = ceph_vxattrcb_snap_btime_exists, + .flags = VXATTR_FLAG_READONLY, + }, + { + .name = "ceph.caps", + .name_size = sizeof("ceph.caps"), + .getxattr_cb = ceph_vxattrcb_caps, + .exists_cb = NULL, + .flags = VXATTR_FLAG_HIDDEN, + }, + { .name = NULL, 0 } /* Required table terminator */ +}; + +/* files */ + +static struct ceph_vxattr ceph_file_vxattrs[] = { + { + .name = "ceph.file.layout", + .name_size = sizeof("ceph.file.layout"), + .getxattr_cb = ceph_vxattrcb_layout, + .exists_cb = ceph_vxattrcb_layout_exists, + .flags = VXATTR_FLAG_HIDDEN, + }, + XATTR_LAYOUT_FIELD(file, layout, stripe_unit), + XATTR_LAYOUT_FIELD(file, layout, stripe_count), + XATTR_LAYOUT_FIELD(file, layout, object_size), + XATTR_LAYOUT_FIELD(file, layout, pool), + XATTR_LAYOUT_FIELD(file, layout, pool_namespace), + { + .name = "ceph.snap.btime", + .name_size = sizeof("ceph.snap.btime"), + .getxattr_cb = ceph_vxattrcb_snap_btime, + .exists_cb = ceph_vxattrcb_snap_btime_exists, + .flags = VXATTR_FLAG_READONLY, + }, + { + .name = "ceph.caps", + .name_size = sizeof("ceph.caps"), + .getxattr_cb = ceph_vxattrcb_caps, + .exists_cb = NULL, + .flags = VXATTR_FLAG_HIDDEN, + }, + { .name = NULL, 0 } /* Required table terminator */ +}; + +static struct ceph_vxattr ceph_common_vxattrs[] = { + { + .name = "ceph.cluster_fsid", + .name_size = sizeof("ceph.cluster_fsid"), + .getxattr_cb = ceph_vxattrcb_cluster_fsid, + .exists_cb = NULL, + .flags = VXATTR_FLAG_READONLY, + }, + { + .name = "ceph.client_id", + .name_size = sizeof("ceph.client_id"), + .getxattr_cb = ceph_vxattrcb_client_id, + .exists_cb = NULL, + .flags = VXATTR_FLAG_READONLY, + }, + { + .name = "ceph.auth_mds", + .name_size = sizeof("ceph.auth_mds"), + .getxattr_cb = ceph_vxattrcb_auth_mds, + .exists_cb = NULL, + .flags = VXATTR_FLAG_READONLY, + }, +#if IS_ENABLED(CONFIG_FS_ENCRYPTION) + { + .name = "ceph.fscrypt.auth", + .name_size = sizeof("ceph.fscrypt.auth"), + .getxattr_cb = ceph_vxattrcb_fscrypt_auth, + .exists_cb = ceph_vxattrcb_fscrypt_auth_exists, + .flags = VXATTR_FLAG_READONLY, + }, +#endif /* CONFIG_FS_ENCRYPTION */ + { .name = NULL, 0 } /* Required table terminator */ +}; + +static struct ceph_vxattr *ceph_inode_vxattrs(struct inode *inode) +{ + if (S_ISDIR(inode->i_mode)) + return ceph_dir_vxattrs; + else if (S_ISREG(inode->i_mode)) + return ceph_file_vxattrs; + return NULL; +} + +static struct ceph_vxattr *ceph_match_vxattr(struct inode *inode, + const char *name) +{ + struct ceph_vxattr *vxattr = ceph_inode_vxattrs(inode); + + if (vxattr) { + while (vxattr->name) { + if (!strcmp(vxattr->name, name)) + return vxattr; + vxattr++; + } + } + + vxattr = ceph_common_vxattrs; + while (vxattr->name) { + if (!strcmp(vxattr->name, name)) + return vxattr; + vxattr++; + } + + return NULL; +} + +#define MAX_XATTR_VAL_PRINT_LEN 256 + +static int __set_xattr(struct ceph_inode_info *ci, + const char *name, int name_len, + const char *val, int val_len, + int flags, int update_xattr, + struct ceph_inode_xattr **newxattr) +{ + struct inode *inode = &ci->netfs.inode; + struct ceph_client *cl = ceph_inode_to_client(inode); + struct rb_node **p; + struct rb_node *parent = NULL; + struct ceph_inode_xattr *xattr = NULL; + int c; + int new = 0; + + p = &ci->i_xattrs.index.rb_node; + while (*p) { + parent = *p; + xattr = rb_entry(parent, struct ceph_inode_xattr, node); + c = strncmp(name, xattr->name, min(name_len, xattr->name_len)); + if (c < 0) + p = &(*p)->rb_left; + else if (c > 0) + p = &(*p)->rb_right; + else { + if (name_len == xattr->name_len) + break; + else if (name_len < xattr->name_len) + p = &(*p)->rb_left; + else + p = &(*p)->rb_right; + } + xattr = NULL; + } + + if (update_xattr) { + int err = 0; + + if (xattr && (flags & XATTR_CREATE)) + err = -EEXIST; + else if (!xattr && (flags & XATTR_REPLACE)) + err = -ENODATA; + if (err) { + kfree(name); + kfree(val); + kfree(*newxattr); + return err; + } + if (update_xattr < 0) { + if (xattr) + __remove_xattr(ci, xattr); + kfree(name); + kfree(*newxattr); + return 0; + } + } + + if (!xattr) { + new = 1; + xattr = *newxattr; + xattr->name = name; + xattr->name_len = name_len; + xattr->should_free_name = update_xattr; + + ci->i_xattrs.count++; + doutc(cl, "count=%d\n", ci->i_xattrs.count); + } else { + kfree(*newxattr); + *newxattr = NULL; + if (xattr->should_free_val) + kfree(xattr->val); + + if (update_xattr) { + kfree(name); + name = xattr->name; + } + ci->i_xattrs.names_size -= xattr->name_len; + ci->i_xattrs.vals_size -= xattr->val_len; + } + ci->i_xattrs.names_size += name_len; + ci->i_xattrs.vals_size += val_len; + if (val) + xattr->val = val; + else + xattr->val = ""; + + xattr->val_len = val_len; + xattr->dirty = update_xattr; + xattr->should_free_val = (val && update_xattr); + + if (new) { + rb_link_node(&xattr->node, parent, p); + rb_insert_color(&xattr->node, &ci->i_xattrs.index); + doutc(cl, "p=%p\n", p); + } + + doutc(cl, "added %p %llx.%llx xattr %p %.*s=%.*s%s\n", inode, + ceph_vinop(inode), xattr, name_len, name, min(val_len, + MAX_XATTR_VAL_PRINT_LEN), val, + val_len > MAX_XATTR_VAL_PRINT_LEN ? "..." : ""); + + return 0; +} + +static struct ceph_inode_xattr *__get_xattr(struct ceph_inode_info *ci, + const char *name) +{ + struct ceph_client *cl = ceph_inode_to_client(&ci->netfs.inode); + struct rb_node **p; + struct rb_node *parent = NULL; + struct ceph_inode_xattr *xattr = NULL; + int name_len = strlen(name); + int c; + + p = &ci->i_xattrs.index.rb_node; + while (*p) { + parent = *p; + xattr = rb_entry(parent, struct ceph_inode_xattr, node); + c = strncmp(name, xattr->name, xattr->name_len); + if (c == 0 && name_len > xattr->name_len) + c = 1; + if (c < 0) + p = &(*p)->rb_left; + else if (c > 0) + p = &(*p)->rb_right; + else { + int len = min(xattr->val_len, MAX_XATTR_VAL_PRINT_LEN); + + doutc(cl, "%s found %.*s%s\n", name, len, xattr->val, + xattr->val_len > len ? "..." : ""); + return xattr; + } + } + + doutc(cl, "%s not found\n", name); + + return NULL; +} + +static void __free_xattr(struct ceph_inode_xattr *xattr) +{ + BUG_ON(!xattr); + + if (xattr->should_free_name) + kfree(xattr->name); + if (xattr->should_free_val) + kfree(xattr->val); + + kfree(xattr); +} + +static int __remove_xattr(struct ceph_inode_info *ci, + struct ceph_inode_xattr *xattr) +{ + if (!xattr) + return -ENODATA; + + rb_erase(&xattr->node, &ci->i_xattrs.index); + + if (xattr->should_free_name) + kfree(xattr->name); + if (xattr->should_free_val) + kfree(xattr->val); + + ci->i_xattrs.names_size -= xattr->name_len; + ci->i_xattrs.vals_size -= xattr->val_len; + ci->i_xattrs.count--; + kfree(xattr); + + return 0; +} + +static char *__copy_xattr_names(struct ceph_inode_info *ci, + char *dest) +{ + struct ceph_client *cl = ceph_inode_to_client(&ci->netfs.inode); + struct rb_node *p; + struct ceph_inode_xattr *xattr = NULL; + + p = rb_first(&ci->i_xattrs.index); + doutc(cl, "count=%d\n", ci->i_xattrs.count); + + while (p) { + xattr = rb_entry(p, struct ceph_inode_xattr, node); + memcpy(dest, xattr->name, xattr->name_len); + dest[xattr->name_len] = '\0'; + + doutc(cl, "dest=%s %p (%s) (%d/%d)\n", dest, xattr, xattr->name, + xattr->name_len, ci->i_xattrs.names_size); + + dest += xattr->name_len + 1; + p = rb_next(p); + } + + return dest; +} + +void __ceph_destroy_xattrs(struct ceph_inode_info *ci) +{ + struct ceph_client *cl = ceph_inode_to_client(&ci->netfs.inode); + struct rb_node *p, *tmp; + struct ceph_inode_xattr *xattr = NULL; + + p = rb_first(&ci->i_xattrs.index); + + doutc(cl, "p=%p\n", p); + + while (p) { + xattr = rb_entry(p, struct ceph_inode_xattr, node); + tmp = p; + p = rb_next(tmp); + doutc(cl, "next p=%p (%.*s)\n", p, xattr->name_len, xattr->name); + rb_erase(tmp, &ci->i_xattrs.index); + + __free_xattr(xattr); + } + + ci->i_xattrs.names_size = 0; + ci->i_xattrs.vals_size = 0; + ci->i_xattrs.index_version = 0; + ci->i_xattrs.count = 0; + ci->i_xattrs.index = RB_ROOT; +} + +static int __build_xattrs(struct inode *inode) + __releases(ci->i_ceph_lock) + __acquires(ci->i_ceph_lock) +{ + struct ceph_client *cl = ceph_inode_to_client(inode); + u32 namelen; + u32 numattr = 0; + void *p, *end; + u32 len; + const char *name, *val; + struct ceph_inode_info *ci = ceph_inode(inode); + u64 xattr_version; + struct ceph_inode_xattr **xattrs = NULL; + int err = 0; + int i; + + doutc(cl, "len=%d\n", + ci->i_xattrs.blob ? (int)ci->i_xattrs.blob->vec.iov_len : 0); + + if (ci->i_xattrs.index_version >= ci->i_xattrs.version) + return 0; /* already built */ + + __ceph_destroy_xattrs(ci); + +start: + /* updated internal xattr rb tree */ + if (ci->i_xattrs.blob && ci->i_xattrs.blob->vec.iov_len > 4) { + p = ci->i_xattrs.blob->vec.iov_base; + end = p + ci->i_xattrs.blob->vec.iov_len; + ceph_decode_32_safe(&p, end, numattr, bad); + xattr_version = ci->i_xattrs.version; + spin_unlock(&ci->i_ceph_lock); + + xattrs = kzalloc_objs(struct ceph_inode_xattr *, numattr, + GFP_NOFS); + err = -ENOMEM; + if (!xattrs) + goto bad_lock; + + for (i = 0; i < numattr; i++) { + xattrs[i] = kmalloc_obj(struct ceph_inode_xattr, + GFP_NOFS); + if (!xattrs[i]) + goto bad_lock; + } + + spin_lock(&ci->i_ceph_lock); + if (ci->i_xattrs.version != xattr_version) { + /* lost a race, retry */ + for (i = 0; i < numattr; i++) + kfree(xattrs[i]); + kfree(xattrs); + xattrs = NULL; + goto start; + } + err = -EIO; + while (numattr--) { + ceph_decode_32_safe(&p, end, len, bad); + namelen = len; + name = p; + p += len; + ceph_decode_32_safe(&p, end, len, bad); + val = p; + p += len; + + err = __set_xattr(ci, name, namelen, val, len, + 0, 0, &xattrs[numattr]); + + if (err < 0) + goto bad; + } + kfree(xattrs); + } + ci->i_xattrs.index_version = ci->i_xattrs.version; + ci->i_xattrs.dirty = false; + + return err; +bad_lock: + spin_lock(&ci->i_ceph_lock); +bad: + if (xattrs) { + for (i = 0; i < numattr; i++) + kfree(xattrs[i]); + kfree(xattrs); + } + ci->i_xattrs.names_size = 0; + return err; +} + +static int __get_required_blob_size(struct ceph_inode_info *ci, int name_size, + int val_size) +{ + struct ceph_client *cl = ceph_inode_to_client(&ci->netfs.inode); + + /* + * 4 bytes for the length, and additional 4 bytes per each xattr name, + * 4 bytes per each value + */ + int size = 4 + ci->i_xattrs.count*(4 + 4) + + ci->i_xattrs.names_size + + ci->i_xattrs.vals_size; + doutc(cl, "c=%d names.size=%d vals.size=%d\n", ci->i_xattrs.count, + ci->i_xattrs.names_size, ci->i_xattrs.vals_size); + + if (name_size) + size += 4 + 4 + name_size + val_size; + + return size; +} + +/* + * If there are dirty xattrs, re-encode xattrs into the prealloc_blob + * and swap into place. It returns the old i_xattrs.blob (or NULL) so + * that it can be freed by the caller as the i_ceph_lock is likely to be + * held. + */ +struct ceph_buffer *__ceph_build_xattrs_blob(struct ceph_inode_info *ci) +{ + struct inode *inode = &ci->netfs.inode; + struct ceph_client *cl = ceph_inode_to_client(inode); + struct rb_node *p; + struct ceph_inode_xattr *xattr = NULL; + struct ceph_buffer *old_blob = NULL; + void *dest; + + doutc(cl, "%p %llx.%llx\n", inode, ceph_vinop(inode)); + if (ci->i_xattrs.dirty) { + int need = __get_required_blob_size(ci, 0, 0); + + BUG_ON(need > ci->i_xattrs.prealloc_blob->alloc_len); + + p = rb_first(&ci->i_xattrs.index); + dest = ci->i_xattrs.prealloc_blob->vec.iov_base; + + ceph_encode_32(&dest, ci->i_xattrs.count); + while (p) { + xattr = rb_entry(p, struct ceph_inode_xattr, node); + + ceph_encode_32(&dest, xattr->name_len); + memcpy(dest, xattr->name, xattr->name_len); + dest += xattr->name_len; + ceph_encode_32(&dest, xattr->val_len); + memcpy(dest, xattr->val, xattr->val_len); + dest += xattr->val_len; + + p = rb_next(p); + } + + /* adjust buffer len; it may be larger than we need */ + ci->i_xattrs.prealloc_blob->vec.iov_len = + dest - ci->i_xattrs.prealloc_blob->vec.iov_base; + + if (ci->i_xattrs.blob) + old_blob = ci->i_xattrs.blob; + ci->i_xattrs.blob = ci->i_xattrs.prealloc_blob; + ci->i_xattrs.prealloc_blob = NULL; + ci->i_xattrs.dirty = false; + ci->i_xattrs.version++; + } + + return old_blob; +} + +static inline int __get_request_mask(struct inode *in) { + struct ceph_mds_request *req = current->journal_info; + int mask = 0; + if (req && req->r_target_inode == in) { + if (req->r_op == CEPH_MDS_OP_LOOKUP || + req->r_op == CEPH_MDS_OP_LOOKUPINO || + req->r_op == CEPH_MDS_OP_LOOKUPPARENT || + req->r_op == CEPH_MDS_OP_GETATTR) { + mask = le32_to_cpu(req->r_args.getattr.mask); + } else if (req->r_op == CEPH_MDS_OP_OPEN || + req->r_op == CEPH_MDS_OP_CREATE) { + mask = le32_to_cpu(req->r_args.open.mask); + } + } + return mask; +} + +ssize_t __ceph_getxattr(struct inode *inode, const char *name, void *value, + size_t size) +{ + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_inode_xattr *xattr; + struct ceph_vxattr *vxattr; + int req_mask; + ssize_t err; + + if (strncmp(name, XATTR_CEPH_PREFIX, XATTR_CEPH_PREFIX_LEN)) + goto handle_non_vxattrs; + + /* let's see if a virtual xattr was requested */ + vxattr = ceph_match_vxattr(inode, name); + if (vxattr) { + int mask = 0; + if (vxattr->flags & VXATTR_FLAG_RSTAT) + mask |= CEPH_STAT_RSTAT; + if (vxattr->flags & VXATTR_FLAG_DIRSTAT) + mask |= CEPH_CAP_FILE_SHARED; + err = ceph_do_getattr(inode, mask, true); + if (err) + return err; + err = -ENODATA; + if (!(vxattr->exists_cb && !vxattr->exists_cb(ci))) { + err = vxattr->getxattr_cb(ci, value, size); + if (size && size < err) + err = -ERANGE; + } + return err; + } else { + err = ceph_do_getvxattr(inode, name, value, size); + /* this would happen with a new client and old server combo */ + if (err == -EOPNOTSUPP) + err = -ENODATA; + return err; + } +handle_non_vxattrs: + req_mask = __get_request_mask(inode); + + spin_lock(&ci->i_ceph_lock); + doutc(cl, "%p %llx.%llx name '%s' ver=%lld index_ver=%lld\n", inode, + ceph_vinop(inode), name, ci->i_xattrs.version, + ci->i_xattrs.index_version); + + if (ci->i_xattrs.version == 0 || + !((req_mask & CEPH_CAP_XATTR_SHARED) || + __ceph_caps_issued_mask_metric(ci, CEPH_CAP_XATTR_SHARED, 1))) { + spin_unlock(&ci->i_ceph_lock); + + /* security module gets xattr while filling trace */ + if (current->journal_info) { + pr_warn_ratelimited_client(cl, + "sync %p %llx.%llx during filling trace\n", + inode, ceph_vinop(inode)); + return -EBUSY; + } + + /* get xattrs from mds (if we don't already have them) */ + err = ceph_do_getattr(inode, CEPH_STAT_CAP_XATTR, true); + if (err) + return err; + spin_lock(&ci->i_ceph_lock); + } + + err = __build_xattrs(inode); + if (err < 0) + goto out; + + err = -ENODATA; /* == ENOATTR */ + xattr = __get_xattr(ci, name); + if (!xattr) + goto out; + + err = -ERANGE; + if (size && size < xattr->val_len) + goto out; + + err = xattr->val_len; + if (size == 0) + goto out; + + memcpy(value, xattr->val, xattr->val_len); + + if (current->journal_info && + !strncmp(name, XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN) && + security_ismaclabel(name + XATTR_SECURITY_PREFIX_LEN)) + set_bit(CEPH_I_SEC_INITED_BIT, &ci->i_ceph_flags); +out: + spin_unlock(&ci->i_ceph_lock); + return err; +} + +ssize_t ceph_listxattr(struct dentry *dentry, char *names, size_t size) +{ + struct inode *inode = d_inode(dentry); + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + bool len_only = (size == 0); + u32 namelen; + int err; + + spin_lock(&ci->i_ceph_lock); + doutc(cl, "%p %llx.%llx ver=%lld index_ver=%lld\n", inode, + ceph_vinop(inode), ci->i_xattrs.version, + ci->i_xattrs.index_version); + + if (ci->i_xattrs.version == 0 || + !__ceph_caps_issued_mask_metric(ci, CEPH_CAP_XATTR_SHARED, 1)) { + spin_unlock(&ci->i_ceph_lock); + err = ceph_do_getattr(inode, CEPH_STAT_CAP_XATTR, true); + if (err) + return err; + spin_lock(&ci->i_ceph_lock); + } + + err = __build_xattrs(inode); + if (err < 0) + goto out; + + /* add 1 byte for each xattr due to the null termination */ + namelen = ci->i_xattrs.names_size + ci->i_xattrs.count; + if (!len_only) { + if (namelen > size) { + err = -ERANGE; + goto out; + } + names = __copy_xattr_names(ci, names); + size -= namelen; + } + err = namelen; +out: + spin_unlock(&ci->i_ceph_lock); + return err; +} + +static int ceph_sync_setxattr(struct inode *inode, const char *name, + const char *value, size_t size, int flags) +{ + struct ceph_fs_client *fsc = ceph_sb_to_fs_client(inode->i_sb); + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_mds_request *req; + struct ceph_mds_client *mdsc = fsc->mdsc; + struct ceph_osd_client *osdc = &fsc->client->osdc; + struct ceph_pagelist *pagelist = NULL; + int op = CEPH_MDS_OP_SETXATTR; + int err; + + if (size > 0) { + /* copy value into pagelist */ + pagelist = ceph_pagelist_alloc(GFP_NOFS); + if (!pagelist) + return -ENOMEM; + + err = ceph_pagelist_append(pagelist, value, size); + if (err) + goto out; + } else if (!value) { + if (flags & CEPH_XATTR_REPLACE) + op = CEPH_MDS_OP_RMXATTR; + else + flags |= CEPH_XATTR_REMOVE; + } + + doutc(cl, "name %s value size %zu\n", name, size); + + /* do request */ + req = ceph_mdsc_create_request(mdsc, op, USE_AUTH_MDS); + if (IS_ERR(req)) { + err = PTR_ERR(req); + goto out; + } + + req->r_path2 = kstrdup(name, GFP_NOFS); + if (!req->r_path2) { + ceph_mdsc_put_request(req); + err = -ENOMEM; + goto out; + } + + if (op == CEPH_MDS_OP_SETXATTR) { + req->r_args.setxattr.flags = cpu_to_le32(flags); + req->r_args.setxattr.osdmap_epoch = + cpu_to_le32(osdc->osdmap->epoch); + req->r_pagelist = pagelist; + pagelist = NULL; + } + + req->r_inode = inode; + ihold(inode); + req->r_num_caps = 1; + req->r_inode_drop = CEPH_CAP_XATTR_SHARED; + + doutc(cl, "xattr.ver (before): %lld\n", ci->i_xattrs.version); + err = ceph_mdsc_do_request(mdsc, NULL, req); + ceph_mdsc_put_request(req); + doutc(cl, "xattr.ver (after): %lld\n", ci->i_xattrs.version); + +out: + if (pagelist) + ceph_pagelist_release(pagelist); + return err; +} + +int __ceph_setxattr(struct inode *inode, const char *name, + const void *value, size_t size, int flags) +{ + struct ceph_client *cl = ceph_inode_to_client(inode); + struct ceph_vxattr *vxattr; + struct ceph_inode_info *ci = ceph_inode(inode); + struct ceph_mds_client *mdsc = ceph_sb_to_fs_client(inode->i_sb)->mdsc; + struct ceph_cap_flush *prealloc_cf = NULL; + struct ceph_buffer *old_blob = NULL; + int issued; + int err; + int dirty = 0; + int name_len = strlen(name); + int val_len = size; + char *newname = NULL; + char *newval = NULL; + struct ceph_inode_xattr *xattr = NULL; + int required_blob_size; + bool check_realm = false; + bool lock_snap_rwsem = false; + + if (ceph_snap(inode) != CEPH_NOSNAP) + return -EROFS; + + vxattr = ceph_match_vxattr(inode, name); + if (vxattr) { + if (vxattr->flags & VXATTR_FLAG_READONLY) + return -EOPNOTSUPP; + if (value && !strncmp(vxattr->name, "ceph.quota", 10)) + check_realm = true; + } + + /* pass any unhandled ceph.* xattrs through to the MDS */ + if (!strncmp(name, XATTR_CEPH_PREFIX, XATTR_CEPH_PREFIX_LEN)) + goto do_sync_unlocked; + + /* preallocate memory for xattr name, value, index node */ + err = -ENOMEM; + newname = kmemdup(name, name_len + 1, GFP_NOFS); + if (!newname) + goto out; + + if (val_len) { + newval = kmemdup(value, val_len, GFP_NOFS); + if (!newval) + goto out; + } + + xattr = kmalloc_obj(struct ceph_inode_xattr, GFP_NOFS); + if (!xattr) + goto out; + + prealloc_cf = ceph_alloc_cap_flush(); + if (!prealloc_cf) + goto out; + + spin_lock(&ci->i_ceph_lock); +retry: + issued = __ceph_caps_issued(ci, NULL); + required_blob_size = __get_required_blob_size(ci, name_len, val_len); + if ((ci->i_xattrs.version == 0) || !(issued & CEPH_CAP_XATTR_EXCL) || + (required_blob_size > mdsc->mdsmap->m_max_xattr_size)) { + doutc(cl, "sync version: %llu size: %d max: %llu\n", + ci->i_xattrs.version, required_blob_size, + mdsc->mdsmap->m_max_xattr_size); + goto do_sync; + } + + if (!lock_snap_rwsem && !ci->i_head_snapc) { + lock_snap_rwsem = true; + if (!down_read_trylock(&mdsc->snap_rwsem)) { + spin_unlock(&ci->i_ceph_lock); + down_read(&mdsc->snap_rwsem); + spin_lock(&ci->i_ceph_lock); + goto retry; + } + } + + doutc(cl, "%p %llx.%llx name '%s' issued %s\n", inode, + ceph_vinop(inode), name, ceph_cap_string(issued)); + __build_xattrs(inode); + + /* + * __build_xattrs() may have released and reacquired i_ceph_lock, + * during which handle_cap_grant() could have replaced i_xattrs.blob + * with a newer MDS-provided blob and bumped i_xattrs.version. If that + * caused __build_xattrs() to rebuild the rb-tree from the new blob, + * count/names_size/vals_size may now be larger than when + * required_blob_size was computed above. Recompute it here so the + * prealloc_blob size check below reflects the current tree state. + */ + required_blob_size = __get_required_blob_size(ci, name_len, val_len); + if (required_blob_size > mdsc->mdsmap->m_max_xattr_size) { + doutc(cl, "sync (size too large): %d > %llu\n", + required_blob_size, mdsc->mdsmap->m_max_xattr_size); + goto do_sync; + } + + if (!ci->i_xattrs.prealloc_blob || + required_blob_size > ci->i_xattrs.prealloc_blob->alloc_len) { + struct ceph_buffer *blob; + + spin_unlock(&ci->i_ceph_lock); + ceph_buffer_put(old_blob); /* Shouldn't be required */ + doutc(cl, " pre-allocating new blob size=%d\n", + required_blob_size); + blob = ceph_buffer_new(required_blob_size, GFP_NOFS); + if (!blob) + goto do_sync_unlocked; + spin_lock(&ci->i_ceph_lock); + /* prealloc_blob can't be released while holding i_ceph_lock */ + if (ci->i_xattrs.prealloc_blob) + old_blob = ci->i_xattrs.prealloc_blob; + ci->i_xattrs.prealloc_blob = blob; + goto retry; + } + + err = __set_xattr(ci, newname, name_len, newval, val_len, + flags, value ? 1 : -1, &xattr); + + if (!err) { + dirty = __ceph_mark_dirty_caps(ci, CEPH_CAP_XATTR_EXCL, + &prealloc_cf); + ci->i_xattrs.dirty = true; + inode_set_ctime_current(inode); + } + + spin_unlock(&ci->i_ceph_lock); + ceph_buffer_put(old_blob); + if (lock_snap_rwsem) + up_read(&mdsc->snap_rwsem); + if (dirty) + __mark_inode_dirty(inode, dirty); + ceph_free_cap_flush(prealloc_cf); + return err; + +do_sync: + spin_unlock(&ci->i_ceph_lock); + ceph_buffer_put(old_blob); +do_sync_unlocked: + if (lock_snap_rwsem) + up_read(&mdsc->snap_rwsem); + + /* security module set xattr while filling trace */ + if (current->journal_info) { + pr_warn_ratelimited_client(cl, + "sync %p %llx.%llx during filling trace\n", + inode, ceph_vinop(inode)); + err = -EBUSY; + } else { + err = ceph_sync_setxattr(inode, name, value, size, flags); + if (err >= 0 && check_realm) { + /* check if snaprealm was created for quota inode */ + spin_lock(&ci->i_ceph_lock); + if ((ci->i_max_files || ci->i_max_bytes) && + !(ci->i_snap_realm && + ci->i_snap_realm->ino == ci->i_vino.ino)) + err = -EOPNOTSUPP; + spin_unlock(&ci->i_ceph_lock); + } + } +out: + ceph_free_cap_flush(prealloc_cf); + kfree(newname); + kfree(newval); + kfree(xattr); + return err; +} + +static int ceph_get_xattr_handler(const struct xattr_handler *handler, + struct dentry *dentry, struct inode *inode, + const char *name, void *value, size_t size) +{ + if (!ceph_is_valid_xattr(name)) + return -EOPNOTSUPP; + return __ceph_getxattr(inode, name, value, size); +} + +static int ceph_set_xattr_handler(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +{ + if (!ceph_is_valid_xattr(name)) + return -EOPNOTSUPP; + return __ceph_setxattr(inode, name, value, size, flags); +} + +static const struct xattr_handler ceph_other_xattr_handler = { + .prefix = "", /* match any name => handlers called with full name */ + .get = ceph_get_xattr_handler, + .set = ceph_set_xattr_handler, +}; + +#ifdef CONFIG_SECURITY +bool ceph_security_xattr_wanted(struct inode *in) +{ + return in->i_security != NULL; +} + +bool ceph_security_xattr_deadlock(struct inode *in) +{ + struct ceph_inode_info *ci; + bool ret; + if (!in->i_security) + return false; + ci = ceph_inode(in); + spin_lock(&ci->i_ceph_lock); + ret = !(ci->i_ceph_flags & CEPH_I_SEC_INITED) && + !(ci->i_xattrs.version > 0 && + __ceph_caps_issued_mask(ci, CEPH_CAP_XATTR_SHARED, 0)); + spin_unlock(&ci->i_ceph_lock); + return ret; +} + +#ifdef CONFIG_CEPH_FS_SECURITY_LABEL +int ceph_security_init_secctx(struct dentry *dentry, umode_t mode, + struct ceph_acl_sec_ctx *as_ctx) +{ + struct ceph_pagelist *pagelist = as_ctx->pagelist; + const char *name; + size_t name_len; + int err; + + err = security_dentry_init_security(dentry, mode, &dentry->d_name, + &name, &as_ctx->lsmctx); + if (err < 0) { + WARN_ON_ONCE(err != -EOPNOTSUPP); + err = 0; /* do nothing */ + goto out; + } + + err = -ENOMEM; + if (!pagelist) { + pagelist = ceph_pagelist_alloc(GFP_KERNEL); + if (!pagelist) + goto out; + err = ceph_pagelist_reserve(pagelist, PAGE_SIZE); + if (err) + goto out; + ceph_pagelist_encode_32(pagelist, 1); + } + + /* + * FIXME: Make security_dentry_init_security() generic. Currently + * It only supports single security module and only selinux has + * dentry_init_security hook. + */ + name_len = strlen(name); + err = ceph_pagelist_reserve(pagelist, + 4 * 2 + name_len + as_ctx->lsmctx.len); + if (err) + goto out; + + if (as_ctx->pagelist) { + /* update count of KV pairs */ + BUG_ON(pagelist->length <= sizeof(__le32)); + if (list_is_singular(&pagelist->head)) { + le32_add_cpu((__le32*)pagelist->mapped_tail, 1); + } else { + struct page *page = list_first_entry(&pagelist->head, + struct page, lru); + void *addr = kmap_atomic(page); + le32_add_cpu((__le32*)addr, 1); + kunmap_atomic(addr); + } + } else { + as_ctx->pagelist = pagelist; + } + + ceph_pagelist_encode_32(pagelist, name_len); + ceph_pagelist_append(pagelist, name, name_len); + + ceph_pagelist_encode_32(pagelist, as_ctx->lsmctx.len); + ceph_pagelist_append(pagelist, as_ctx->lsmctx.context, + as_ctx->lsmctx.len); + + err = 0; +out: + if (pagelist && !as_ctx->pagelist) + ceph_pagelist_release(pagelist); + return err; +} +#endif /* CONFIG_CEPH_FS_SECURITY_LABEL */ +#endif /* CONFIG_SECURITY */ + +void ceph_release_acl_sec_ctx(struct ceph_acl_sec_ctx *as_ctx) +{ +#ifdef CONFIG_CEPH_FS_POSIX_ACL + posix_acl_release(as_ctx->acl); + posix_acl_release(as_ctx->default_acl); +#endif +#ifdef CONFIG_CEPH_FS_SECURITY_LABEL + security_release_secctx(&as_ctx->lsmctx); +#endif +#ifdef CONFIG_FS_ENCRYPTION + kfree(as_ctx->fscrypt_auth); +#endif + if (as_ctx->pagelist) + ceph_pagelist_release(as_ctx->pagelist); +} + +/* + * List of handlers for synthetic system.* attributes. Other + * attributes are handled directly. + */ +const struct xattr_handler * const ceph_xattr_handlers[] = { + &ceph_other_xattr_handler, + NULL, +}; diff --git a/fs/char_dev.c b/fs/char_dev.c new file mode 100644 index 000000000..00229e25c --- /dev/null +++ b/fs/char_dev.c @@ -0,0 +1,689 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/char_dev.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +static struct kobj_map *cdev_map __ro_after_init; + +static DEFINE_MUTEX(chrdevs_lock); + +#define CHRDEV_MAJOR_HASH_SIZE 255 + +static struct char_device_struct { + struct char_device_struct *next; + unsigned int major; + unsigned int baseminor; + int minorct; + char name[64]; + struct cdev *cdev; /* will die */ +} *chrdevs[CHRDEV_MAJOR_HASH_SIZE]; + +/* index in the above */ +static inline int major_to_index(unsigned major) +{ + return major % CHRDEV_MAJOR_HASH_SIZE; +} + +#ifdef CONFIG_PROC_FS + +void chrdev_show(struct seq_file *f, off_t offset) +{ + struct char_device_struct *cd; + + mutex_lock(&chrdevs_lock); + for (cd = chrdevs[major_to_index(offset)]; cd; cd = cd->next) { + if (cd->major == offset) + seq_printf(f, "%3d %s\n", cd->major, cd->name); + } + mutex_unlock(&chrdevs_lock); +} + +#endif /* CONFIG_PROC_FS */ + +static int find_dynamic_major(void) +{ + int i; + struct char_device_struct *cd; + + for (i = ARRAY_SIZE(chrdevs)-1; i >= CHRDEV_MAJOR_DYN_END; i--) { + if (chrdevs[i] == NULL) + return i; + } + + for (i = CHRDEV_MAJOR_DYN_EXT_START; + i >= CHRDEV_MAJOR_DYN_EXT_END; i--) { + for (cd = chrdevs[major_to_index(i)]; cd; cd = cd->next) + if (cd->major == i) + break; + + if (cd == NULL) + return i; + } + + return -EBUSY; +} + +/* + * Register a single major with a specified minor range. + * + * If major == 0 this function will dynamically allocate an unused major. + * If major > 0 this function will attempt to reserve the range of minors + * with given major. + * + */ +static struct char_device_struct * +__register_chrdev_region(unsigned int major, unsigned int baseminor, + int minorct, const char *name) +{ + struct char_device_struct *cd __free(kfree) = NULL; + struct char_device_struct *curr, *prev = NULL; + int ret; + int i; + + if (major >= CHRDEV_MAJOR_MAX) { + pr_err("CHRDEV \"%s\" major requested (%u) is greater than the maximum (%u)\n", + name, major, CHRDEV_MAJOR_MAX-1); + return ERR_PTR(-EINVAL); + } + + if (minorct > MINORMASK + 1 - baseminor) { + pr_err("CHRDEV \"%s\" minor range requested (%u-%u) is out of range of maximum range (%u-%u) for a single major\n", + name, baseminor, baseminor + minorct - 1, 0, MINORMASK); + return ERR_PTR(-EINVAL); + } + + cd = kzalloc_obj(struct char_device_struct); + if (cd == NULL) + return ERR_PTR(-ENOMEM); + + guard(mutex)(&chrdevs_lock); + + if (major == 0) { + ret = find_dynamic_major(); + if (ret < 0) { + pr_err("CHRDEV \"%s\" dynamic allocation region is full\n", + name); + return ERR_PTR(ret); + } + major = ret; + } + + ret = -EBUSY; + i = major_to_index(major); + for (curr = chrdevs[i]; curr; prev = curr, curr = curr->next) { + if (curr->major < major) + continue; + + if (curr->major > major) + break; + + if (curr->baseminor + curr->minorct <= baseminor) + continue; + + if (curr->baseminor >= baseminor + minorct) + break; + + return ERR_PTR(ret); + } + + cd->major = major; + cd->baseminor = baseminor; + cd->minorct = minorct; + strscpy(cd->name, name, sizeof(cd->name)); + + if (!prev) { + cd->next = curr; + chrdevs[i] = cd; + } else { + cd->next = prev->next; + prev->next = cd; + } + + return_ptr(cd); +} + +static struct char_device_struct * +__unregister_chrdev_region(unsigned major, unsigned baseminor, int minorct) +{ + struct char_device_struct *cd = NULL, **cp; + int i = major_to_index(major); + + mutex_lock(&chrdevs_lock); + for (cp = &chrdevs[i]; *cp; cp = &(*cp)->next) + if ((*cp)->major == major && + (*cp)->baseminor == baseminor && + (*cp)->minorct == minorct) + break; + if (*cp) { + cd = *cp; + *cp = cd->next; + } + mutex_unlock(&chrdevs_lock); + return cd; +} + +/** + * register_chrdev_region() - register a range of device numbers + * @from: the first in the desired range of device numbers; must include + * the major number. + * @count: the number of consecutive device numbers required + * @name: the name of the device or driver. + * + * Return value is zero on success, a negative error code on failure. + */ +int register_chrdev_region(dev_t from, unsigned count, const char *name) +{ + struct char_device_struct *cd; + dev_t to = from + count; + dev_t n, next; + + for (n = from; n < to; n = next) { + next = MKDEV(MAJOR(n)+1, 0); + if (next > to) + next = to; + cd = __register_chrdev_region(MAJOR(n), MINOR(n), + next - n, name); + if (IS_ERR(cd)) + goto fail; + } + return 0; +fail: + to = n; + for (n = from; n < to; n = next) { + next = MKDEV(MAJOR(n)+1, 0); + kfree(__unregister_chrdev_region(MAJOR(n), MINOR(n), next - n)); + } + return PTR_ERR(cd); +} + +/** + * alloc_chrdev_region() - register a range of char device numbers + * @dev: output parameter for first assigned number + * @baseminor: first of the requested range of minor numbers + * @count: the number of minor numbers required + * @name: the name of the associated device or driver + * + * Allocates a range of char device numbers. The major number will be + * chosen dynamically, and returned (along with the first minor number) + * in @dev. Returns zero or a negative error code. + */ +int alloc_chrdev_region(dev_t *dev, unsigned baseminor, unsigned count, + const char *name) +{ + struct char_device_struct *cd; + cd = __register_chrdev_region(0, baseminor, count, name); + if (IS_ERR(cd)) + return PTR_ERR(cd); + *dev = MKDEV(cd->major, cd->baseminor); + return 0; +} + +/** + * __register_chrdev() - create and register a cdev occupying a range of minors + * @major: major device number or 0 for dynamic allocation + * @baseminor: first of the requested range of minor numbers + * @count: the number of minor numbers required + * @name: name of this range of devices + * @fops: file operations associated with this devices + * + * If @major == 0 this functions will dynamically allocate a major and return + * its number. + * + * If @major > 0 this function will attempt to reserve a device with the given + * major number and will return zero on success. + * + * Returns a -ve errno on failure. + * + * The name of this device has nothing to do with the name of the device in + * /dev. It only helps to keep track of the different owners of devices. If + * your module name has only one type of devices it's ok to use e.g. the name + * of the module here. + */ +int __register_chrdev(unsigned int major, unsigned int baseminor, + unsigned int count, const char *name, + const struct file_operations *fops) +{ + struct char_device_struct *cd; + struct cdev *cdev; + int err = -ENOMEM; + + cd = __register_chrdev_region(major, baseminor, count, name); + if (IS_ERR(cd)) + return PTR_ERR(cd); + + cdev = cdev_alloc(); + if (!cdev) + goto out2; + + cdev->owner = fops->owner; + cdev->ops = fops; + kobject_set_name(&cdev->kobj, "%s", name); + + err = cdev_add(cdev, MKDEV(cd->major, baseminor), count); + if (err) + goto out; + + cd->cdev = cdev; + + return major ? 0 : cd->major; +out: + kobject_put(&cdev->kobj); +out2: + kfree(__unregister_chrdev_region(cd->major, baseminor, count)); + return err; +} + +/** + * unregister_chrdev_region() - unregister a range of device numbers + * @from: the first in the range of numbers to unregister + * @count: the number of device numbers to unregister + * + * This function will unregister a range of @count device numbers, + * starting with @from. The caller should normally be the one who + * allocated those numbers in the first place... + */ +void unregister_chrdev_region(dev_t from, unsigned count) +{ + dev_t to = from + count; + dev_t n, next; + + for (n = from; n < to; n = next) { + next = MKDEV(MAJOR(n)+1, 0); + if (next > to) + next = to; + kfree(__unregister_chrdev_region(MAJOR(n), MINOR(n), next - n)); + } +} + +/** + * __unregister_chrdev - unregister and destroy a cdev + * @major: major device number + * @baseminor: first of the range of minor numbers + * @count: the number of minor numbers this cdev is occupying + * @name: name of this range of devices + * + * Unregister and destroy the cdev occupying the region described by + * @major, @baseminor and @count. This function undoes what + * __register_chrdev() did. + */ +void __unregister_chrdev(unsigned int major, unsigned int baseminor, + unsigned int count, const char *name) +{ + struct char_device_struct *cd; + + cd = __unregister_chrdev_region(major, baseminor, count); + if (cd && cd->cdev) + cdev_del(cd->cdev); + kfree(cd); +} + +static __cacheline_aligned_in_smp DEFINE_SPINLOCK(cdev_lock); + +static struct kobject *cdev_get(struct cdev *p) +{ + struct module *owner = p->owner; + struct kobject *kobj; + + if (!try_module_get(owner)) + return NULL; + kobj = kobject_get_unless_zero(&p->kobj); + if (!kobj) + module_put(owner); + return kobj; +} + +void cdev_put(struct cdev *p) +{ + if (p) { + struct module *owner = p->owner; + kobject_put(&p->kobj); + module_put(owner); + } +} + +/* + * Called every time a character special file is opened + */ +static int chrdev_open(struct inode *inode, struct file *filp) +{ + const struct file_operations *fops; + struct cdev *p; + struct cdev *new = NULL; + int ret = 0; + + spin_lock(&cdev_lock); + p = inode->i_cdev; + if (!p) { + struct kobject *kobj; + int idx; + spin_unlock(&cdev_lock); + kobj = kobj_lookup(cdev_map, inode->i_rdev, &idx); + if (!kobj) + return -ENXIO; + new = container_of(kobj, struct cdev, kobj); + spin_lock(&cdev_lock); + /* Check i_cdev again in case somebody beat us to it while + we dropped the lock. */ + p = inode->i_cdev; + if (!p) { + inode->i_cdev = p = new; + list_add(&inode->i_devices, &p->list); + new = NULL; + } else if (!cdev_get(p)) + ret = -ENXIO; + } else if (!cdev_get(p)) + ret = -ENXIO; + spin_unlock(&cdev_lock); + cdev_put(new); + if (ret) + return ret; + + ret = -ENXIO; + fops = fops_get(p->ops); + if (!fops) + goto out_cdev_put; + + replace_fops(filp, fops); + if (filp->f_op->open) { + ret = filp->f_op->open(inode, filp); + if (ret) + goto out_cdev_put; + } + + return 0; + + out_cdev_put: + cdev_put(p); + return ret; +} + +void cd_forget(struct inode *inode) +{ + spin_lock(&cdev_lock); + list_del_init(&inode->i_devices); + inode->i_cdev = NULL; + inode->i_mapping = &inode->i_data; + spin_unlock(&cdev_lock); +} + +static void cdev_purge(struct cdev *cdev) +{ + spin_lock(&cdev_lock); + while (!list_empty(&cdev->list)) { + struct inode *inode; + inode = container_of(cdev->list.next, struct inode, i_devices); + list_del_init(&inode->i_devices); + inode->i_cdev = NULL; + } + spin_unlock(&cdev_lock); +} + +/* + * Dummy default file-operations: the only thing this does + * is contain the open that then fills in the correct operations + * depending on the special file... + */ +const struct file_operations def_chr_fops = { + .open = chrdev_open, + .llseek = noop_llseek, +}; + +static struct kobject *exact_match(dev_t dev, int *part, void *data) +{ + struct cdev *p = data; + return &p->kobj; +} + +static int exact_lock(dev_t dev, void *data) +{ + struct cdev *p = data; + return cdev_get(p) ? 0 : -1; +} + +/** + * cdev_add() - add a char device to the system + * @p: the cdev structure for the device + * @dev: the first device number for which this device is responsible + * @count: the number of consecutive minor numbers corresponding to this + * device + * + * cdev_add() adds the device represented by @p to the system, making it + * live immediately. A negative error code is returned on failure. + */ +int cdev_add(struct cdev *p, dev_t dev, unsigned count) +{ + int error; + + p->dev = dev; + p->count = count; + + if (WARN_ON(dev == WHITEOUT_DEV)) { + error = -EBUSY; + goto err; + } + + error = kobj_map(cdev_map, dev, count, NULL, + exact_match, exact_lock, p); + if (error) + goto err; + + kobject_get(p->kobj.parent); + + return 0; + +err: + kfree_const(p->kobj.name); + p->kobj.name = NULL; + return error; +} + +/** + * cdev_set_parent() - set the parent kobject for a char device + * @p: the cdev structure + * @kobj: the kobject to take a reference to + * + * cdev_set_parent() sets a parent kobject which will be referenced + * appropriately so the parent is not freed before the cdev. This + * should be called before cdev_add. + */ +void cdev_set_parent(struct cdev *p, struct kobject *kobj) +{ + WARN_ON(!kobj->state_initialized); + p->kobj.parent = kobj; +} + +/** + * cdev_device_add() - add a char device and it's corresponding + * struct device, linkink + * @dev: the device structure + * @cdev: the cdev structure + * + * cdev_device_add() adds the char device represented by @cdev to the system, + * just as cdev_add does. It then adds @dev to the system using device_add + * The dev_t for the char device will be taken from the struct device which + * needs to be initialized first. This helper function correctly takes a + * reference to the parent device so the parent will not get released until + * all references to the cdev are released. + * + * This helper uses dev->devt for the device number. If it is not set + * it will not add the cdev and it will be equivalent to device_add. + * + * This function should be used whenever the struct cdev and the + * struct device are members of the same structure whose lifetime is + * managed by the struct device. + * + * NOTE: Callers must assume that userspace was able to open the cdev and + * can call cdev fops callbacks at any time, even if this function fails. + */ +int cdev_device_add(struct cdev *cdev, struct device *dev) +{ + int rc = 0; + + if (dev->devt) { + cdev_set_parent(cdev, &dev->kobj); + + rc = cdev_add(cdev, dev->devt, 1); + if (rc) + return rc; + } + + rc = device_add(dev); + if (rc && dev->devt) + cdev_del(cdev); + + return rc; +} + +/** + * cdev_device_del() - inverse of cdev_device_add + * @cdev: the cdev structure + * @dev: the device structure + * + * cdev_device_del() is a helper function to call cdev_del and device_del. + * It should be used whenever cdev_device_add is used. + * + * If dev->devt is not set it will not remove the cdev and will be equivalent + * to device_del. + * + * NOTE: This guarantees that associated sysfs callbacks are not running + * or runnable, however any cdevs already open will remain and their fops + * will still be callable even after this function returns. + */ +void cdev_device_del(struct cdev *cdev, struct device *dev) +{ + device_del(dev); + if (dev->devt) + cdev_del(cdev); +} + +static void cdev_unmap(dev_t dev, unsigned count) +{ + kobj_unmap(cdev_map, dev, count); +} + +/** + * cdev_del() - remove a cdev from the system + * @p: the cdev structure to be removed + * + * cdev_del() removes @p from the system, possibly freeing the structure + * itself. + * + * NOTE: This guarantees that cdev device will no longer be able to be + * opened, however any cdevs already open will remain and their fops will + * still be callable even after cdev_del returns. + */ +void cdev_del(struct cdev *p) +{ + cdev_unmap(p->dev, p->count); + kobject_put(&p->kobj); +} + + +static void cdev_default_release(struct kobject *kobj) +{ + struct cdev *p = container_of(kobj, struct cdev, kobj); + struct kobject *parent = kobj->parent; + + cdev_purge(p); + kobject_put(parent); +} + +static void cdev_dynamic_release(struct kobject *kobj) +{ + struct cdev *p = container_of(kobj, struct cdev, kobj); + struct kobject *parent = kobj->parent; + + cdev_purge(p); + kfree(p); + kobject_put(parent); +} + +static struct kobj_type ktype_cdev_default = { + .release = cdev_default_release, +}; + +static struct kobj_type ktype_cdev_dynamic = { + .release = cdev_dynamic_release, +}; + +/** + * cdev_alloc() - allocate a cdev structure + * + * Allocates and returns a cdev structure, or NULL on failure. + */ +struct cdev *cdev_alloc(void) +{ + struct cdev *p = kzalloc_obj(struct cdev); + if (p) { + INIT_LIST_HEAD(&p->list); + kobject_init(&p->kobj, &ktype_cdev_dynamic); + } + return p; +} + +/** + * cdev_init() - initialize a cdev structure + * @cdev: the structure to initialize + * @fops: the file_operations for this device + * + * Initializes @cdev, remembering @fops, making it ready to add to the + * system with cdev_add(). + */ +void cdev_init(struct cdev *cdev, const struct file_operations *fops) +{ + memset(cdev, 0, sizeof *cdev); + INIT_LIST_HEAD(&cdev->list); + kobject_init(&cdev->kobj, &ktype_cdev_default); + cdev->ops = fops; +} + +static struct kobject *base_probe(dev_t dev, int *part, void *data) +{ + if (request_module("char-major-%d-%d", MAJOR(dev), MINOR(dev)) > 0) + /* Make old-style 2.4 aliases work */ + request_module("char-major-%d", MAJOR(dev)); + return NULL; +} + +void __init chrdev_init(void) +{ + cdev_map = kobj_map_init(base_probe, &chrdevs_lock); +} + + +/* Let modules do char dev stuff */ +EXPORT_SYMBOL(register_chrdev_region); +EXPORT_SYMBOL(unregister_chrdev_region); +EXPORT_SYMBOL(alloc_chrdev_region); +EXPORT_SYMBOL(cdev_init); +EXPORT_SYMBOL(cdev_alloc); +EXPORT_SYMBOL(cdev_del); +EXPORT_SYMBOL(cdev_add); +EXPORT_SYMBOL(cdev_set_parent); +EXPORT_SYMBOL(cdev_device_add); +EXPORT_SYMBOL(cdev_device_del); +EXPORT_SYMBOL(__register_chrdev); +EXPORT_SYMBOL(__unregister_chrdev); diff --git a/fs/coda/Kconfig b/fs/coda/Kconfig new file mode 100644 index 000000000..c3477eeaf --- /dev/null +++ b/fs/coda/Kconfig @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: GPL-2.0-only +config CODA_FS + tristate "Coda file system support (advanced network fs)" + depends on INET + help + Coda is an advanced network file system, similar to NFS in that it + enables you to mount file systems of a remote server and access them + with regular Unix commands as if they were sitting on your hard + disk. Coda has several advantages over NFS: support for + disconnected operation (e.g. for laptops), read/write server + replication, security model for authentication and encryption, + persistent client caches and write back caching. + + If you say Y here, your Linux box will be able to act as a Coda + *client*. You will need user level code as well, both for the + client and server. Servers are currently user level, i.e. they need + no kernel support. Please read + and check out the Coda + home page . + + To compile the coda client support as a module, choose M here: the + module will be called coda. diff --git a/fs/coda/Makefile b/fs/coda/Makefile new file mode 100644 index 000000000..78befb836 --- /dev/null +++ b/fs/coda/Makefile @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the Linux Coda filesystem routines. +# + +obj-$(CONFIG_CODA_FS) += coda.o + +coda-objs := psdev.o cache.o cnode.o inode.o dir.o file.o upcall.o \ + coda_linux.o symlink.o pioctl.o +coda-$(CONFIG_SYSCTL) += sysctl.o + +# If you want debugging output, please uncomment the following line. + +# ccflags-y := -DDEBUG -DDEBUG_SMB_MALLOC=1 diff --git a/fs/coda/cache.c b/fs/coda/cache.c new file mode 100644 index 000000000..245131296 --- /dev/null +++ b/fs/coda/cache.c @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Cache operations for Coda. + * For Linux 2.1: (C) 1997 Carnegie Mellon University + * For Linux 2.3: (C) 2000 Carnegie Mellon University + * + * Carnegie Mellon encourages users of this code to contribute improvements + * to the Coda project http://www.coda.cs.cmu.edu/ . + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "coda_psdev.h" +#include "coda_linux.h" +#include "coda_cache.h" + +static atomic_t permission_epoch = ATOMIC_INIT(0); + +/* replace or extend an acl cache hit */ +void coda_cache_enter(struct inode *inode, int mask) +{ + struct coda_inode_info *cii = ITOC(inode); + + spin_lock(&cii->c_lock); + cii->c_cached_epoch = atomic_read(&permission_epoch); + if (!uid_eq(cii->c_uid, current_fsuid())) { + cii->c_uid = current_fsuid(); + cii->c_cached_perm = mask; + } else + cii->c_cached_perm |= mask; + spin_unlock(&cii->c_lock); +} + +/* remove cached acl from an inode */ +void coda_cache_clear_inode(struct inode *inode) +{ + struct coda_inode_info *cii = ITOC(inode); + spin_lock(&cii->c_lock); + cii->c_cached_epoch = atomic_read(&permission_epoch) - 1; + spin_unlock(&cii->c_lock); +} + +/* remove all acl caches */ +void coda_cache_clear_all(struct super_block *sb) +{ + atomic_inc(&permission_epoch); +} + + +/* check if the mask has been matched against the acl already */ +int coda_cache_check(struct inode *inode, int mask) +{ + struct coda_inode_info *cii = ITOC(inode); + int hit; + + spin_lock(&cii->c_lock); + hit = (mask & cii->c_cached_perm) == mask && + uid_eq(cii->c_uid, current_fsuid()) && + cii->c_cached_epoch == atomic_read(&permission_epoch); + spin_unlock(&cii->c_lock); + + return hit; +} + + +/* Purging dentries and children */ +/* The following routines drop dentries which are not + in use and flag dentries which are in use to be + zapped later. + + The flags are detected by: + - coda_dentry_revalidate (for lookups) if the flag is C_PURGE + - coda_dentry_delete: to remove dentry from the cache when d_count + falls to zero + - an inode method coda_revalidate (for attributes) if the + flag is C_VATTR +*/ + +/* this won't do any harm: just flag all children */ +static void coda_flag_children(struct dentry *parent, int flag) +{ + struct dentry *de; + + spin_lock(&parent->d_lock); + rcu_read_lock(); + hlist_for_each_entry(de, &parent->d_children, d_sib) { + struct inode *inode = d_inode_rcu(de); + /* don't know what to do with negative dentries */ + if (inode) + coda_flag_inode(inode, flag); + } + rcu_read_unlock(); + spin_unlock(&parent->d_lock); +} + +void coda_flag_inode_children(struct inode *inode, int flag) +{ + struct dentry *alias_de; + + if ( !inode || !S_ISDIR(inode->i_mode)) + return; + + alias_de = d_find_alias(inode); + if (!alias_de) + return; + coda_flag_children(alias_de, flag); + shrink_dcache_parent(alias_de); + dput(alias_de); +} + diff --git a/fs/coda/cnode.c b/fs/coda/cnode.c new file mode 100644 index 000000000..70bb0579b --- /dev/null +++ b/fs/coda/cnode.c @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0 +/* cnode related routines for the coda kernel code + (C) 1996 Peter Braam + */ + +#include +#include +#include + +#include +#include +#include "coda_psdev.h" +#include "coda_linux.h" + +static inline int coda_fideq(struct CodaFid *fid1, struct CodaFid *fid2) +{ + return memcmp(fid1, fid2, sizeof(*fid1)) == 0; +} + +static const struct inode_operations coda_symlink_inode_operations = { + .get_link = page_get_link, + .setattr = coda_setattr, +}; + +/* cnode.c */ +static void coda_fill_inode(struct inode *inode, struct coda_vattr *attr) +{ + coda_vattr_to_iattr(inode, attr); + + if (S_ISREG(inode->i_mode)) { + inode->i_op = &coda_file_inode_operations; + inode->i_fop = &coda_file_operations; + } else if (S_ISDIR(inode->i_mode)) { + inode->i_op = &coda_dir_inode_operations; + inode->i_fop = &coda_dir_operations; + } else if (S_ISLNK(inode->i_mode)) { + inode->i_op = &coda_symlink_inode_operations; + inode_nohighmem(inode); + inode->i_data.a_ops = &coda_symlink_aops; + inode->i_mapping = &inode->i_data; + } else + init_special_inode(inode, inode->i_mode, huge_decode_dev(attr->va_rdev)); +} + +static int coda_test_inode(struct inode *inode, void *data) +{ + struct CodaFid *fid = (struct CodaFid *)data; + struct coda_inode_info *cii = ITOC(inode); + return coda_fideq(&cii->c_fid, fid); +} + +static int coda_set_inode(struct inode *inode, void *data) +{ + struct CodaFid *fid = (struct CodaFid *)data; + struct coda_inode_info *cii = ITOC(inode); + cii->c_fid = *fid; + return 0; +} + +struct inode * coda_iget(struct super_block * sb, struct CodaFid * fid, + struct coda_vattr * attr) +{ + struct inode *inode; + struct coda_inode_info *cii; + unsigned long hash = coda_f2i(fid); + umode_t inode_type = coda_inode_type(attr); + +retry: + inode = iget5_locked(sb, hash, coda_test_inode, coda_set_inode, fid); + if (!inode) + return ERR_PTR(-ENOMEM); + + if (inode_state_read_once(inode) & I_NEW) { + cii = ITOC(inode); + /* we still need to set i_ino for things like stat(2) */ + inode->i_ino = hash; + /* inode is locked and unique, no need to grab cii->c_lock */ + cii->c_mapcount = 0; + coda_fill_inode(inode, attr); + unlock_new_inode(inode); + } else if ((inode->i_mode & S_IFMT) != inode_type) { + /* Inode has changed type, mark bad and grab a new one */ + remove_inode_hash(inode); + coda_flag_inode(inode, C_PURGE); + iput(inode); + goto retry; + } + return inode; +} + +/* this is effectively coda_iget: + - get attributes (might be cached) + - get the inode for the fid using vfs iget + - link the two up if this is needed + - fill in the attributes +*/ +struct inode *coda_cnode_make(struct CodaFid *fid, struct super_block *sb) +{ + struct coda_vattr attr; + struct inode *inode; + int error; + + /* We get inode numbers from Venus -- see venus source */ + error = venus_getattr(sb, fid, &attr); + if (error) + return ERR_PTR(error); + + inode = coda_iget(sb, fid, &attr); + if (IS_ERR(inode)) + pr_warn("%s: coda_iget failed\n", __func__); + return inode; +} + + +/* Although we treat Coda file identifiers as immutable, there is one + * special case for files created during a disconnection where they may + * not be globally unique. When an identifier collision is detected we + * first try to flush the cached inode from the kernel and finally + * resort to renaming/rehashing in-place. Userspace remembers both old + * and new values of the identifier to handle any in-flight upcalls. + * The real solution is to use globally unique UUIDs as identifiers, but + * retrofitting the existing userspace code for this is non-trivial. */ +void coda_replace_fid(struct inode *inode, struct CodaFid *oldfid, + struct CodaFid *newfid) +{ + struct coda_inode_info *cii = ITOC(inode); + unsigned long hash = coda_f2i(newfid); + + BUG_ON(!coda_fideq(&cii->c_fid, oldfid)); + + /* replace fid and rehash inode */ + /* XXX we probably need to hold some lock here! */ + remove_inode_hash(inode); + cii->c_fid = *newfid; + inode->i_ino = hash; + __insert_inode_hash(inode, hash); +} + +/* convert a fid to an inode. */ +struct inode *coda_fid_to_inode(struct CodaFid *fid, struct super_block *sb) +{ + struct inode *inode; + unsigned long hash = coda_f2i(fid); + + inode = ilookup5(sb, hash, coda_test_inode, fid); + if ( !inode ) + return NULL; + + /* we should never see newly created inodes because we intentionally + * fail in the initialization callback */ + BUG_ON(inode_state_read_once(inode) & I_NEW); + + return inode; +} + +struct coda_file_info *coda_ftoc(struct file *file) +{ + struct coda_file_info *cfi = file->private_data; + + BUG_ON(!cfi || cfi->cfi_magic != CODA_MAGIC); + + return cfi; + +} + +/* the CONTROL inode is made without asking attributes from Venus */ +struct inode *coda_cnode_makectl(struct super_block *sb) +{ + struct inode *inode = new_inode(sb); + if (inode) { + inode->i_ino = CTL_INO; + inode->i_op = &coda_ioctl_inode_operations; + inode->i_fop = &coda_ioctl_operations; + inode->i_mode = 0444; + return inode; + } + return ERR_PTR(-ENOMEM); +} + diff --git a/fs/coda/coda_cache.h b/fs/coda/coda_cache.h new file mode 100644 index 000000000..c9f7a77c0 --- /dev/null +++ b/fs/coda/coda_cache.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Coda filesystem -- Linux Minicache + * + * Copyright (C) 1989 - 1997 Carnegie Mellon University + * + * Carnegie Mellon University encourages users of this software to + * contribute improvements to the Coda project. Contact Peter Braam + * + */ + +#ifndef _CFSNC_HEADER_ +#define _CFSNC_HEADER_ + +/* credential cache */ +void coda_cache_enter(struct inode *inode, int mask); +void coda_cache_clear_inode(struct inode *); +void coda_cache_clear_all(struct super_block *sb); +int coda_cache_check(struct inode *inode, int mask); + +/* for downcalls and attributes and lookups */ +void coda_flag_inode_children(struct inode *inode, int flag); + +#endif /* _CFSNC_HEADER_ */ diff --git a/fs/coda/coda_fs_i.h b/fs/coda/coda_fs_i.h new file mode 100644 index 000000000..1763ff95d --- /dev/null +++ b/fs/coda/coda_fs_i.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * coda_fs_i.h + * + * Copyright (C) 1998 Carnegie Mellon University + * + */ + +#ifndef _LINUX_CODA_FS_I +#define _LINUX_CODA_FS_I + +#include +#include +#include +#include + +/* + * coda fs inode data + * c_lock protects accesses to c_flags, c_mapcount, c_cached_epoch, c_uid and + * c_cached_perm. + * vfs_inode is set only when the inode is created and never changes. + * c_fid is set when the inode is created and should be considered immutable. + */ +struct coda_inode_info { + struct CodaFid c_fid; /* Coda identifier */ + u_short c_flags; /* flags (see below) */ + unsigned int c_mapcount; /* nr of times this inode is mapped */ + unsigned int c_cached_epoch; /* epoch for cached permissions */ + kuid_t c_uid; /* fsuid for cached permissions */ + unsigned int c_cached_perm; /* cached access permissions */ + spinlock_t c_lock; + struct inode vfs_inode; +}; + +/* + * coda fs file private data + */ +#define CODA_MAGIC 0xC0DAC0DA +struct coda_file_info { + int cfi_magic; /* magic number */ + struct file *cfi_container; /* container file for this cnode */ + unsigned int cfi_mapcount; /* nr of times this file is mapped */ + bool cfi_access_intent; /* is access intent supported */ +}; + +/* flags */ +#define C_VATTR 0x1 /* Validity of vattr in inode */ +#define C_FLUSH 0x2 /* used after a flush */ +#define C_DYING 0x4 /* from venus (which died) */ +#define C_PURGE 0x8 + +struct inode *coda_cnode_make(struct CodaFid *, struct super_block *); +struct inode *coda_iget(struct super_block *sb, struct CodaFid *fid, struct coda_vattr *attr); +struct inode *coda_cnode_makectl(struct super_block *sb); +struct inode *coda_fid_to_inode(struct CodaFid *fid, struct super_block *sb); +struct coda_file_info *coda_ftoc(struct file *file); +void coda_replace_fid(struct inode *, struct CodaFid *, struct CodaFid *); + +#endif diff --git a/fs/coda/coda_int.h b/fs/coda/coda_int.h new file mode 100644 index 000000000..f82b59c9d --- /dev/null +++ b/fs/coda/coda_int.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _CODA_INT_ +#define _CODA_INT_ + +struct dentry; +struct file; + +extern struct file_system_type coda_fs_type; +extern unsigned long coda_timeout; +extern int coda_hard; +extern int coda_fake_statfs; + +void coda_destroy_inodecache(void); +int __init coda_init_inodecache(void); +int coda_fsync(struct file *coda_file, loff_t start, loff_t end, int datasync); + +#ifdef CONFIG_SYSCTL +void coda_sysctl_init(void); +void coda_sysctl_clean(void); +#else +static inline void coda_sysctl_init(void) +{ +} + +static inline void coda_sysctl_clean(void) +{ +} +#endif +#endif /* _CODA_INT_ */ + + diff --git a/fs/coda/coda_linux.c b/fs/coda/coda_linux.c new file mode 100644 index 000000000..1d2dac95f --- /dev/null +++ b/fs/coda/coda_linux.c @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Inode operations for Coda filesystem + * Original version: (C) 1996 P. Braam and M. Callahan + * Rewritten for Linux 2.1. (C) 1997 Carnegie Mellon University + * + * Carnegie Mellon encourages users to contribute improvements to + * the Coda project. Contact Peter Braam (coda@cs.cmu.edu). + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "coda_psdev.h" +#include "coda_linux.h" + +/* initialize the debugging variables */ +int coda_fake_statfs; + +/* print a fid */ +char * coda_f2s(struct CodaFid *f) +{ + static char s[60]; + + sprintf(s, "(%08x.%08x.%08x.%08x)", f->opaque[0], f->opaque[1], f->opaque[2], f->opaque[3]); + + return s; +} + +/* recognize special .CONTROL name */ +int coda_iscontrol(const char *name, size_t length) +{ + return ((CODA_CONTROLLEN == length) && + (strncmp(name, CODA_CONTROL, CODA_CONTROLLEN) == 0)); +} + +unsigned short coda_flags_to_cflags(unsigned short flags) +{ + unsigned short coda_flags = 0; + + if ((flags & O_ACCMODE) == O_RDONLY) + coda_flags |= C_O_READ; + + if ((flags & O_ACCMODE) == O_RDWR) + coda_flags |= C_O_READ | C_O_WRITE; + + if ((flags & O_ACCMODE) == O_WRONLY) + coda_flags |= C_O_WRITE; + + if (flags & O_TRUNC) + coda_flags |= C_O_TRUNC; + + if (flags & O_CREAT) + coda_flags |= C_O_CREAT; + + if (flags & O_EXCL) + coda_flags |= C_O_EXCL; + + return coda_flags; +} + +static struct timespec64 coda_to_timespec64(struct coda_timespec ts) +{ + struct timespec64 ts64 = { + .tv_sec = ts.tv_sec, + .tv_nsec = ts.tv_nsec, + }; + + return ts64; +} + +static struct coda_timespec timespec64_to_coda(struct timespec64 ts64) +{ + struct coda_timespec ts = { + .tv_sec = ts64.tv_sec, + .tv_nsec = ts64.tv_nsec, + }; + + return ts; +} + +/* utility functions below */ +umode_t coda_inode_type(struct coda_vattr *attr) +{ + switch (attr->va_type) { + case C_VREG: + return S_IFREG; + case C_VDIR: + return S_IFDIR; + case C_VLNK: + return S_IFLNK; + case C_VNON: + default: + return 0; + } +} + +void coda_vattr_to_iattr(struct inode *inode, struct coda_vattr *attr) +{ + /* inode's i_flags, i_ino are set by iget + * XXX: is this all we need ?? + */ + umode_t inode_type = coda_inode_type(attr); + inode->i_mode |= inode_type; + + if (attr->va_mode != (u_short) -1) + inode->i_mode = attr->va_mode | inode_type; + if (attr->va_uid != -1) + inode->i_uid = make_kuid(&init_user_ns, (uid_t) attr->va_uid); + if (attr->va_gid != -1) + inode->i_gid = make_kgid(&init_user_ns, (gid_t) attr->va_gid); + if (attr->va_nlink != -1) + set_nlink(inode, attr->va_nlink); + if (attr->va_size != -1) + inode->i_size = attr->va_size; + if (attr->va_size != -1) + inode->i_blocks = (attr->va_size + 511) >> 9; + if (attr->va_atime.tv_sec != -1) + inode_set_atime_to_ts(inode, + coda_to_timespec64(attr->va_atime)); + if (attr->va_mtime.tv_sec != -1) + inode_set_mtime_to_ts(inode, + coda_to_timespec64(attr->va_mtime)); + if (attr->va_ctime.tv_sec != -1) + inode_set_ctime_to_ts(inode, + coda_to_timespec64(attr->va_ctime)); +} + + +/* + * BSD sets attributes that need not be modified to -1. + * Linux uses the valid field to indicate what should be + * looked at. The BSD type field needs to be deduced from linux + * mode. + * So we have to do some translations here. + */ + +void coda_iattr_to_vattr(struct iattr *iattr, struct coda_vattr *vattr) +{ + unsigned int valid; + + /* clean out */ + vattr->va_mode = -1; + vattr->va_uid = (vuid_t) -1; + vattr->va_gid = (vgid_t) -1; + vattr->va_size = (off_t) -1; + vattr->va_atime.tv_sec = (int64_t) -1; + vattr->va_atime.tv_nsec = (long) -1; + vattr->va_mtime.tv_sec = (int64_t) -1; + vattr->va_mtime.tv_nsec = (long) -1; + vattr->va_ctime.tv_sec = (int64_t) -1; + vattr->va_ctime.tv_nsec = (long) -1; + vattr->va_type = C_VNON; + vattr->va_fileid = -1; + vattr->va_gen = -1; + vattr->va_bytes = -1; + vattr->va_nlink = -1; + vattr->va_blocksize = -1; + vattr->va_rdev = -1; + vattr->va_flags = 0; + + /* determine the type */ +#if 0 + mode = iattr->ia_mode; + if ( S_ISDIR(mode) ) { + vattr->va_type = C_VDIR; + } else if ( S_ISREG(mode) ) { + vattr->va_type = C_VREG; + } else if ( S_ISLNK(mode) ) { + vattr->va_type = C_VLNK; + } else { + /* don't do others */ + vattr->va_type = C_VNON; + } +#endif + + /* set those vattrs that need change */ + valid = iattr->ia_valid; + if ( valid & ATTR_MODE ) { + vattr->va_mode = iattr->ia_mode; + } + if ( valid & ATTR_UID ) { + vattr->va_uid = (vuid_t) from_kuid(&init_user_ns, iattr->ia_uid); + } + if ( valid & ATTR_GID ) { + vattr->va_gid = (vgid_t) from_kgid(&init_user_ns, iattr->ia_gid); + } + if ( valid & ATTR_SIZE ) { + vattr->va_size = iattr->ia_size; + } + if ( valid & ATTR_ATIME ) { + vattr->va_atime = timespec64_to_coda(iattr->ia_atime); + } + if ( valid & ATTR_MTIME ) { + vattr->va_mtime = timespec64_to_coda(iattr->ia_mtime); + } + if ( valid & ATTR_CTIME ) { + vattr->va_ctime = timespec64_to_coda(iattr->ia_ctime); + } +} + diff --git a/fs/coda/coda_linux.h b/fs/coda/coda_linux.h new file mode 100644 index 000000000..dd6277d87 --- /dev/null +++ b/fs/coda/coda_linux.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Coda File System, Linux Kernel module + * + * Original version, adapted from cfs_mach.c, (C) Carnegie Mellon University + * Linux modifications (C) 1996, Peter J. Braam + * Rewritten for Linux 2.1 (C) 1997 Carnegie Mellon University + * + * Carnegie Mellon University encourages users of this software to + * contribute improvements to the Coda project. + */ + +#ifndef _LINUX_CODA_FS +#define _LINUX_CODA_FS + +#ifdef pr_fmt +#undef pr_fmt +#endif + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include "coda_fs_i.h" + +/* operations */ +extern const struct inode_operations coda_dir_inode_operations; +extern const struct inode_operations coda_file_inode_operations; +extern const struct inode_operations coda_ioctl_inode_operations; + +extern const struct dentry_operations coda_dentry_operations; + +extern const struct address_space_operations coda_file_aops; +extern const struct address_space_operations coda_symlink_aops; + +extern const struct file_operations coda_dir_operations; +extern const struct file_operations coda_file_operations; +extern const struct file_operations coda_ioctl_operations; + +/* operations shared over more than one file */ +int coda_open(struct inode *i, struct file *f); +int coda_release(struct inode *i, struct file *f); +int coda_permission(struct mnt_idmap *idmap, struct inode *inode, + int mask); +int coda_revalidate_inode(struct inode *); +int coda_getattr(struct mnt_idmap *, const struct path *, struct kstat *, + u32, unsigned int); +int coda_setattr(struct mnt_idmap *, struct dentry *, struct iattr *); + +/* this file: helpers */ +char *coda_f2s(struct CodaFid *f); +int coda_iscontrol(const char *name, size_t length); + +umode_t coda_inode_type(struct coda_vattr *attr); +void coda_vattr_to_iattr(struct inode *, struct coda_vattr *); +void coda_iattr_to_vattr(struct iattr *, struct coda_vattr *); +unsigned short coda_flags_to_cflags(unsigned short); + +/* inode to cnode access functions */ + +static inline struct coda_inode_info *ITOC(struct inode *inode) +{ + return container_of(inode, struct coda_inode_info, vfs_inode); +} + +static __inline__ struct CodaFid *coda_i2f(struct inode *inode) +{ + return &(ITOC(inode)->c_fid); +} + +static __inline__ char *coda_i2s(struct inode *inode) +{ + return coda_f2s(&(ITOC(inode)->c_fid)); +} + +/* this will not zap the inode away */ +static __inline__ void coda_flag_inode(struct inode *inode, int flag) +{ + struct coda_inode_info *cii = ITOC(inode); + + if (!inode) + return; + + spin_lock(&cii->c_lock); + cii->c_flags |= flag; + spin_unlock(&cii->c_lock); +} + +#endif diff --git a/fs/coda/coda_psdev.h b/fs/coda/coda_psdev.h new file mode 100644 index 000000000..52da08c77 --- /dev/null +++ b/fs/coda/coda_psdev.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __CODA_PSDEV_H +#define __CODA_PSDEV_H + +#include +#include +#include + +#define CODA_PSDEV_MAJOR 67 +#define MAX_CODADEVS 5 /* how many do we allow */ + +struct kstatfs; + +/* messages between coda filesystem in kernel and Venus */ +struct upc_req { + struct list_head uc_chain; + caddr_t uc_data; + u_short uc_flags; + u_short uc_inSize; /* Size is at most 5000 bytes */ + u_short uc_outSize; + u_short uc_opcode; /* copied from data to save lookup */ + int uc_unique; + wait_queue_head_t uc_sleep; /* process' wait queue */ +}; + +#define CODA_REQ_ASYNC 0x1 +#define CODA_REQ_READ 0x2 +#define CODA_REQ_WRITE 0x4 +#define CODA_REQ_ABORT 0x8 + +/* communication pending/processing queues */ +struct venus_comm { + u_long vc_seq; + wait_queue_head_t vc_waitq; /* Venus wait queue */ + struct list_head vc_pending; + struct list_head vc_processing; + int vc_inuse; + struct super_block *vc_sb; + struct mutex vc_mutex; +}; + +static inline struct venus_comm *coda_vcp(struct super_block *sb) +{ + return (struct venus_comm *)((sb)->s_fs_info); +} + +/* upcalls */ +int venus_rootfid(struct super_block *sb, struct CodaFid *fidp); +int venus_getattr(struct super_block *sb, struct CodaFid *fid, + struct coda_vattr *attr); +int venus_setattr(struct super_block *, struct CodaFid *, struct coda_vattr *); +int venus_lookup(struct super_block *sb, struct CodaFid *fid, + const char *name, int length, int *type, + struct CodaFid *resfid); +int venus_close(struct super_block *sb, struct CodaFid *fid, int flags, + kuid_t uid); +int venus_open(struct super_block *sb, struct CodaFid *fid, int flags, + struct file **f); +int venus_mkdir(struct super_block *sb, struct CodaFid *dirfid, + const char *name, int length, + struct CodaFid *newfid, struct coda_vattr *attrs); +int venus_create(struct super_block *sb, struct CodaFid *dirfid, + const char *name, int length, int excl, int mode, + struct CodaFid *newfid, struct coda_vattr *attrs); +int venus_rmdir(struct super_block *sb, struct CodaFid *dirfid, + const char *name, int length); +int venus_remove(struct super_block *sb, struct CodaFid *dirfid, + const char *name, int length); +int venus_readlink(struct super_block *sb, struct CodaFid *fid, + char *buffer, int *length); +int venus_rename(struct super_block *sb, struct CodaFid *new_fid, + struct CodaFid *old_fid, size_t old_length, + size_t new_length, const char *old_name, + const char *new_name); +int venus_link(struct super_block *sb, struct CodaFid *fid, + struct CodaFid *dirfid, const char *name, int len ); +int venus_symlink(struct super_block *sb, struct CodaFid *fid, + const char *name, int len, const char *symname, int symlen); +int venus_access(struct super_block *sb, struct CodaFid *fid, int mask); +int venus_pioctl(struct super_block *sb, struct CodaFid *fid, + unsigned int cmd, struct PioctlData *data); +int coda_downcall(struct venus_comm *vcp, int opcode, union outputArgs *out, + size_t nbytes); +int venus_fsync(struct super_block *sb, struct CodaFid *fid); +int venus_statfs(struct dentry *dentry, struct kstatfs *sfs); +int venus_access_intent(struct super_block *sb, struct CodaFid *fid, + bool *access_intent_supported, + size_t count, loff_t ppos, int type); + +/* + * Statistics + */ + +extern struct venus_comm coda_comms[]; +#endif diff --git a/fs/coda/dir.c b/fs/coda/dir.c new file mode 100644 index 000000000..67148edfa --- /dev/null +++ b/fs/coda/dir.c @@ -0,0 +1,578 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Directory operations for Coda filesystem + * Original version: (C) 1996 P. Braam and M. Callahan + * Rewritten for Linux 2.1. (C) 1997 Carnegie Mellon University + * + * Carnegie Mellon encourages users to contribute improvements to + * the Coda project. Contact Peter Braam (coda@cs.cmu.edu). + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "coda_psdev.h" +#include "coda_linux.h" +#include "coda_cache.h" + +#include "coda_int.h" + +/* same as fs/bad_inode.c */ +static int coda_return_EIO(void) +{ + return -EIO; +} +#define CODA_EIO_ERROR ((void *) (coda_return_EIO)) + +/* inode operations for directories */ +/* access routines: lookup, readlink, permission */ +static struct dentry *coda_lookup(struct inode *dir, struct dentry *entry, unsigned int flags) +{ + struct super_block *sb = dir->i_sb; + const char *name = entry->d_name.name; + size_t length = entry->d_name.len; + struct inode *inode; + int type = 0; + + if (length > CODA_MAXNAMLEN) { + pr_err("name too long: lookup, %s %zu\n", + coda_i2s(dir), length); + return ERR_PTR(-ENAMETOOLONG); + } + + /* control object, create inode on the fly */ + if (is_root_inode(dir) && coda_iscontrol(name, length)) { + inode = coda_cnode_makectl(sb); + type = CODA_NOCACHE; + } else { + struct CodaFid fid = { { 0, } }; + int error = venus_lookup(sb, coda_i2f(dir), name, length, + &type, &fid); + inode = !error ? coda_cnode_make(&fid, sb) : ERR_PTR(error); + } + + if (!IS_ERR(inode) && (type & CODA_NOCACHE)) + coda_flag_inode(inode, C_VATTR | C_PURGE); + + if (inode == ERR_PTR(-ENOENT)) + inode = NULL; + + return d_splice_alias(inode, entry); +} + + +int coda_permission(struct mnt_idmap *idmap, struct inode *inode, + int mask) +{ + int error; + + if (mask & MAY_NOT_BLOCK) + return -ECHILD; + + mask &= MAY_READ | MAY_WRITE | MAY_EXEC; + + if (!mask) + return 0; + + if ((mask & MAY_EXEC) && !execute_ok(inode)) + return -EACCES; + + if (coda_cache_check(inode, mask)) + return 0; + + error = venus_access(inode->i_sb, coda_i2f(inode), mask); + + if (!error) + coda_cache_enter(inode, mask); + + return error; +} + + +static inline void coda_dir_update_mtime(struct inode *dir) +{ +#ifdef REQUERY_VENUS_FOR_MTIME + /* invalidate the directory cnode's attributes so we refetch the + * attributes from venus next time the inode is referenced */ + coda_flag_inode(dir, C_VATTR); +#else + /* optimistically we can also act as if our nose bleeds. The + * granularity of the mtime is coarse anyways so we might actually be + * right most of the time. Note: we only do this for directories. */ + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); +#endif +} + +/* we have to wrap inc_nlink/drop_nlink because sometimes userspace uses a + * trick to fool GNU find's optimizations. If we can't be sure of the link + * (because of volume mount points) we set i_nlink to 1 which forces find + * to consider every child as a possible directory. We should also never + * see an increment or decrement for deleted directories where i_nlink == 0 */ +static inline void coda_dir_inc_nlink(struct inode *dir) +{ + if (dir->i_nlink >= 2) + inc_nlink(dir); +} + +static inline void coda_dir_drop_nlink(struct inode *dir) +{ + if (dir->i_nlink > 2) + drop_nlink(dir); +} + +/* creation routines: create, mknod, mkdir, link, symlink */ +static int coda_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *de, umode_t mode) +{ + int error; + const char *name=de->d_name.name; + int length=de->d_name.len; + struct inode *inode; + struct CodaFid newfid; + struct coda_vattr attrs; + + if (is_root_inode(dir) && coda_iscontrol(name, length)) + return -EPERM; + + error = venus_create(dir->i_sb, coda_i2f(dir), name, length, + 0, mode, &newfid, &attrs); + if (error) + goto err_out; + + inode = coda_iget(dir->i_sb, &newfid, &attrs); + if (IS_ERR(inode)) { + error = PTR_ERR(inode); + goto err_out; + } + + /* invalidate the directory cnode's attributes */ + coda_dir_update_mtime(dir); + d_instantiate(de, inode); + return 0; +err_out: + d_drop(de); + return error; +} + +static struct dentry *coda_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *de, umode_t mode) +{ + struct inode *inode; + struct coda_vattr attrs; + const char *name = de->d_name.name; + int len = de->d_name.len; + int error; + struct CodaFid newfid; + + if (is_root_inode(dir) && coda_iscontrol(name, len)) + return ERR_PTR(-EPERM); + + /* + * vfs_mkdir() now passes S_IFDIR in @mode, but @mode is forwarded + * verbatim to userspace, which has only ever been given the permission + * bits. Strip the type bit until venus is known to cope with it. + */ + attrs.va_mode = mode & ~S_IFDIR; + error = venus_mkdir(dir->i_sb, coda_i2f(dir), + name, len, &newfid, &attrs); + if (error) + goto err_out; + + inode = coda_iget(dir->i_sb, &newfid, &attrs); + if (IS_ERR(inode)) { + error = PTR_ERR(inode); + goto err_out; + } + + /* invalidate the directory cnode's attributes */ + coda_dir_inc_nlink(dir); + coda_dir_update_mtime(dir); + d_instantiate(de, inode); + return NULL; +err_out: + d_drop(de); + return ERR_PTR(error); +} + +/* try to make de an entry in dir_inodde linked to source_de */ +static int coda_link(struct dentry *source_de, struct inode *dir_inode, + struct dentry *de) +{ + struct inode *inode = d_inode(source_de); + const char * name = de->d_name.name; + int len = de->d_name.len; + int error; + + if (is_root_inode(dir_inode) && coda_iscontrol(name, len)) + return -EPERM; + + error = venus_link(dir_inode->i_sb, coda_i2f(inode), + coda_i2f(dir_inode), (const char *)name, len); + if (error) { + d_drop(de); + return error; + } + + coda_dir_update_mtime(dir_inode); + ihold(inode); + d_instantiate(de, inode); + inc_nlink(inode); + return 0; +} + + +static int coda_symlink(struct mnt_idmap *idmap, + struct inode *dir_inode, struct dentry *de, + const char *symname) +{ + const char *name = de->d_name.name; + int len = de->d_name.len; + int symlen; + int error; + + if (is_root_inode(dir_inode) && coda_iscontrol(name, len)) + return -EPERM; + + symlen = strlen(symname); + if (symlen > CODA_MAXPATHLEN) + return -ENAMETOOLONG; + + /* + * This entry is now negative. Since we do not create + * an inode for the entry we have to drop it. + */ + d_drop(de); + error = venus_symlink(dir_inode->i_sb, coda_i2f(dir_inode), name, len, + symname, symlen); + + /* mtime is no good anymore */ + if (!error) + coda_dir_update_mtime(dir_inode); + + return error; +} + +/* destruction routines: unlink, rmdir */ +static int coda_unlink(struct inode *dir, struct dentry *de) +{ + int error; + const char *name = de->d_name.name; + int len = de->d_name.len; + + error = venus_remove(dir->i_sb, coda_i2f(dir), name, len); + if (error) + return error; + + coda_dir_update_mtime(dir); + drop_nlink(d_inode(de)); + return 0; +} + +static int coda_rmdir(struct inode *dir, struct dentry *de) +{ + const char *name = de->d_name.name; + int len = de->d_name.len; + int error; + + error = venus_rmdir(dir->i_sb, coda_i2f(dir), name, len); + if (!error) { + /* VFS may delete the child */ + if (d_really_is_positive(de)) + clear_nlink(d_inode(de)); + + /* fix the link count of the parent */ + coda_dir_drop_nlink(dir); + coda_dir_update_mtime(dir); + } + return error; +} + +/* rename */ +static int coda_rename(struct mnt_idmap *idmap, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) +{ + const char *old_name = old_dentry->d_name.name; + const char *new_name = new_dentry->d_name.name; + int old_length = old_dentry->d_name.len; + int new_length = new_dentry->d_name.len; + int error; + + if (flags) + return -EINVAL; + + error = venus_rename(old_dir->i_sb, coda_i2f(old_dir), + coda_i2f(new_dir), old_length, new_length, + (const char *) old_name, (const char *)new_name); + if (!error) { + if (d_really_is_positive(new_dentry)) { + if (d_is_dir(new_dentry)) { + coda_dir_drop_nlink(old_dir); + coda_dir_inc_nlink(new_dir); + } + coda_flag_inode(d_inode(new_dentry), C_VATTR); + } + coda_dir_update_mtime(old_dir); + coda_dir_update_mtime(new_dir); + } + return error; +} + +static inline unsigned int CDT2DT(unsigned char cdt) +{ + unsigned int dt; + + switch(cdt) { + case CDT_UNKNOWN: dt = DT_UNKNOWN; break; + case CDT_FIFO: dt = DT_FIFO; break; + case CDT_CHR: dt = DT_CHR; break; + case CDT_DIR: dt = DT_DIR; break; + case CDT_BLK: dt = DT_BLK; break; + case CDT_REG: dt = DT_REG; break; + case CDT_LNK: dt = DT_LNK; break; + case CDT_SOCK: dt = DT_SOCK; break; + case CDT_WHT: dt = DT_WHT; break; + default: dt = DT_UNKNOWN; break; + } + return dt; +} + +/* support routines */ +static int coda_venus_readdir(struct file *coda_file, struct dir_context *ctx) +{ + struct coda_file_info *cfi; + struct coda_inode_info *cii; + struct file *host_file; + struct venus_dirent *vdir; + unsigned long vdir_size = offsetof(struct venus_dirent, d_name); + unsigned int type; + struct qstr name; + ino_t ino; + int ret; + + cfi = coda_ftoc(coda_file); + host_file = cfi->cfi_container; + + cii = ITOC(file_inode(coda_file)); + + vdir = kmalloc_obj(*vdir); + if (!vdir) return -ENOMEM; + + if (!dir_emit_dots(coda_file, ctx)) + goto out; + + while (1) { + loff_t pos = ctx->pos - 2; + + /* read entries from the directory file */ + ret = kernel_read(host_file, vdir, sizeof(*vdir), &pos); + if (ret < 0) { + pr_err("%s: read dir %s failed %d\n", + __func__, coda_f2s(&cii->c_fid), ret); + break; + } + if (ret == 0) break; /* end of directory file reached */ + + /* catch truncated reads */ + if (ret < vdir_size || ret < vdir_size + vdir->d_namlen) { + pr_err("%s: short read on %s\n", + __func__, coda_f2s(&cii->c_fid)); + ret = -EBADF; + break; + } + /* validate whether the directory file actually makes sense */ + if (vdir->d_reclen < vdir_size + vdir->d_namlen) { + pr_err("%s: invalid dir %s\n", + __func__, coda_f2s(&cii->c_fid)); + ret = -EBADF; + break; + } + + name.len = vdir->d_namlen; + name.name = vdir->d_name; + + /* Make sure we skip '.' and '..', we already got those */ + if (name.name[0] == '.' && (name.len == 1 || + (name.name[1] == '.' && name.len == 2))) + vdir->d_fileno = name.len = 0; + + /* skip null entries */ + if (vdir->d_fileno && name.len) { + ino = vdir->d_fileno; + type = CDT2DT(vdir->d_type); + if (!dir_emit(ctx, name.name, name.len, ino, type)) + break; + } + /* we'll always have progress because d_reclen is unsigned and + * we've already established it is non-zero. */ + ctx->pos += vdir->d_reclen; + } +out: + kfree(vdir); + return 0; +} + +/* file operations for directories */ +static int coda_readdir(struct file *coda_file, struct dir_context *ctx) +{ + struct coda_file_info *cfi; + struct file *host_file; + int ret; + + cfi = coda_ftoc(coda_file); + host_file = cfi->cfi_container; + + ret = iterate_dir(host_file, ctx); + if (ret != -ENOTDIR) + return ret; + /* Venus: we must read Venus dirents from a file */ + return coda_venus_readdir(coda_file, ctx); +} + +/* called when a cache lookup succeeds */ +static int coda_dentry_revalidate(struct inode *dir, const struct qstr *name, + struct dentry *de, unsigned int flags) +{ + struct inode *inode; + struct coda_inode_info *cii; + + if (flags & LOOKUP_RCU) + return -ECHILD; + + inode = d_inode(de); + if (!inode || is_root_inode(inode)) + goto out; + + cii = ITOC(d_inode(de)); + if (!(cii->c_flags & (C_PURGE | C_FLUSH))) + goto out; + + shrink_dcache_parent(de); + + /* propagate for a flush */ + if (cii->c_flags & C_FLUSH) + coda_flag_inode_children(inode, C_FLUSH); + + if (d_count(de) > 1) + /* pretend it's valid, but don't change the flags */ + goto out; + + /* clear the flags. */ + spin_lock(&cii->c_lock); + cii->c_flags &= ~(C_VATTR | C_PURGE | C_FLUSH); + spin_unlock(&cii->c_lock); + return 0; +out: + return 1; +} + +/* + * This is the callback from dput() when d_count is going to 0. + * We use this to unhash dentries with bad inodes. + */ +static int coda_dentry_delete(const struct dentry * dentry) +{ + struct inode *inode = d_inode(dentry); + + if (!inode) + return 0; + + if (ITOC(inode)->c_flags & C_PURGE) + return 1; + + return 0; +} + + + +/* + * This is called when we want to check if the inode has + * changed on the server. Coda makes this easy since the + * cache manager Venus issues a downcall to the kernel when this + * happens + */ +int coda_revalidate_inode(struct inode *inode) +{ + struct coda_vattr attr; + int error; + int old_mode; + ino_t old_ino; + struct coda_inode_info *cii = ITOC(inode); + + if (!cii->c_flags) + return 0; + + if (cii->c_flags & (C_VATTR | C_PURGE | C_FLUSH)) { + error = venus_getattr(inode->i_sb, &(cii->c_fid), &attr); + if (error) + return -EIO; + + /* this inode may be lost if: + - it's ino changed + - type changes must be permitted for repair and + missing mount points. + */ + old_mode = inode->i_mode; + old_ino = inode->i_ino; + coda_vattr_to_iattr(inode, &attr); + + if ((old_mode & S_IFMT) != (inode->i_mode & S_IFMT)) { + pr_warn("inode %llu, fid %s changed type!\n", + inode->i_ino, coda_f2s(&(cii->c_fid))); + } + + /* the following can happen when a local fid is replaced + with a global one, here we lose and declare the inode bad */ + if (inode->i_ino != old_ino) + return -EIO; + + coda_flag_inode_children(inode, C_FLUSH); + + spin_lock(&cii->c_lock); + cii->c_flags &= ~(C_VATTR | C_PURGE | C_FLUSH); + spin_unlock(&cii->c_lock); + } + return 0; +} + +const struct dentry_operations coda_dentry_operations = { + .d_revalidate = coda_dentry_revalidate, + .d_delete = coda_dentry_delete, +}; + +const struct inode_operations coda_dir_inode_operations = { + .create = coda_create, + .lookup = coda_lookup, + .link = coda_link, + .unlink = coda_unlink, + .symlink = coda_symlink, + .mkdir = coda_mkdir, + .rmdir = coda_rmdir, + .mknod = CODA_EIO_ERROR, + .rename = coda_rename, + .permission = coda_permission, + .getattr = coda_getattr, + .setattr = coda_setattr, +}; + +WRAP_DIR_ITER(coda_readdir) // FIXME! +const struct file_operations coda_dir_operations = { + .llseek = generic_file_llseek, + .read = generic_read_dir, + .iterate_shared = shared_coda_readdir, + .open = coda_open, + .release = coda_release, + .fsync = coda_fsync, +}; diff --git a/fs/coda/file.c b/fs/coda/file.c new file mode 100644 index 000000000..6ca8bfcb5 --- /dev/null +++ b/fs/coda/file.c @@ -0,0 +1,331 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * File operations for Coda. + * Original version: (C) 1996 Peter Braam + * Rewritten for Linux 2.1: (C) 1997 Carnegie Mellon University + * + * Carnegie Mellon encourages users of this code to contribute improvements + * to the Coda project. Contact Peter Braam . + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "coda_psdev.h" +#include "coda_linux.h" +#include "coda_int.h" + +struct coda_vm_ops { + refcount_t refcnt; + struct file *coda_file; + const struct vm_operations_struct *host_vm_ops; + struct vm_operations_struct vm_ops; +}; + +static ssize_t +coda_file_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct file *coda_file = iocb->ki_filp; + struct inode *coda_inode = file_inode(coda_file); + struct coda_file_info *cfi = coda_ftoc(coda_file); + loff_t ki_pos = iocb->ki_pos; + size_t count = iov_iter_count(to); + ssize_t ret; + + ret = venus_access_intent(coda_inode->i_sb, coda_i2f(coda_inode), + &cfi->cfi_access_intent, + count, ki_pos, CODA_ACCESS_TYPE_READ); + if (ret) + goto finish_read; + + ret = vfs_iter_read(cfi->cfi_container, to, &iocb->ki_pos, 0); + +finish_read: + venus_access_intent(coda_inode->i_sb, coda_i2f(coda_inode), + &cfi->cfi_access_intent, + count, ki_pos, CODA_ACCESS_TYPE_READ_FINISH); + return ret; +} + +static ssize_t +coda_file_write_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct file *coda_file = iocb->ki_filp; + struct inode *coda_inode = file_inode(coda_file); + struct coda_file_info *cfi = coda_ftoc(coda_file); + struct file *host_file = cfi->cfi_container; + loff_t ki_pos = iocb->ki_pos; + size_t count = iov_iter_count(to); + ssize_t ret; + + ret = venus_access_intent(coda_inode->i_sb, coda_i2f(coda_inode), + &cfi->cfi_access_intent, + count, ki_pos, CODA_ACCESS_TYPE_WRITE); + if (ret) + goto finish_write; + + inode_lock(coda_inode); + ret = vfs_iter_write(cfi->cfi_container, to, &iocb->ki_pos, 0); + coda_inode->i_size = file_inode(host_file)->i_size; + coda_inode->i_blocks = (coda_inode->i_size + 511) >> 9; + inode_set_mtime_to_ts(coda_inode, inode_set_ctime_current(coda_inode)); + inode_unlock(coda_inode); + +finish_write: + venus_access_intent(coda_inode->i_sb, coda_i2f(coda_inode), + &cfi->cfi_access_intent, + count, ki_pos, CODA_ACCESS_TYPE_WRITE_FINISH); + return ret; +} + +static ssize_t +coda_file_splice_read(struct file *coda_file, loff_t *ppos, + struct pipe_inode_info *pipe, + size_t len, unsigned int flags) +{ + struct inode *coda_inode = file_inode(coda_file); + struct coda_file_info *cfi = coda_ftoc(coda_file); + struct file *in = cfi->cfi_container; + loff_t ki_pos = *ppos; + ssize_t ret; + + ret = venus_access_intent(coda_inode->i_sb, coda_i2f(coda_inode), + &cfi->cfi_access_intent, + len, ki_pos, CODA_ACCESS_TYPE_READ); + if (ret) + goto finish_read; + + ret = vfs_splice_read(in, ppos, pipe, len, flags); + +finish_read: + venus_access_intent(coda_inode->i_sb, coda_i2f(coda_inode), + &cfi->cfi_access_intent, + len, ki_pos, CODA_ACCESS_TYPE_READ_FINISH); + return ret; +} + +static void +coda_vm_open(struct vm_area_struct *vma) +{ + struct coda_vm_ops *cvm_ops = + container_of(vma->vm_ops, struct coda_vm_ops, vm_ops); + + refcount_inc(&cvm_ops->refcnt); + + if (cvm_ops->host_vm_ops && cvm_ops->host_vm_ops->open) + cvm_ops->host_vm_ops->open(vma); +} + +static void +coda_vm_close(struct vm_area_struct *vma) +{ + struct coda_vm_ops *cvm_ops = + container_of(vma->vm_ops, struct coda_vm_ops, vm_ops); + + if (cvm_ops->host_vm_ops && cvm_ops->host_vm_ops->close) + cvm_ops->host_vm_ops->close(vma); + + if (refcount_dec_and_test(&cvm_ops->refcnt)) { + vma->vm_ops = cvm_ops->host_vm_ops; + fput(cvm_ops->coda_file); + kfree(cvm_ops); + } +} + +static int +coda_file_mmap(struct file *coda_file, struct vm_area_struct *vma) +{ + struct inode *coda_inode = file_inode(coda_file); + struct coda_file_info *cfi = coda_ftoc(coda_file); + struct file *host_file = cfi->cfi_container; + struct inode *host_inode = file_inode(host_file); + struct coda_inode_info *cii; + struct coda_vm_ops *cvm_ops; + loff_t ppos; + size_t count; + int ret; + + if (!can_mmap_file(host_file)) + return -ENODEV; + + if (WARN_ON(coda_file != vma->vm_file)) + return -EIO; + + count = vma->vm_end - vma->vm_start; + ppos = vma->vm_pgoff * PAGE_SIZE; + + ret = venus_access_intent(coda_inode->i_sb, coda_i2f(coda_inode), + &cfi->cfi_access_intent, + count, ppos, CODA_ACCESS_TYPE_MMAP); + if (ret) + return ret; + + cvm_ops = kmalloc_obj(struct coda_vm_ops); + if (!cvm_ops) + return -ENOMEM; + + cii = ITOC(coda_inode); + spin_lock(&cii->c_lock); + coda_file->f_mapping = host_file->f_mapping; + if (coda_inode->i_mapping == &coda_inode->i_data) + coda_inode->i_mapping = host_inode->i_mapping; + + /* only allow additional mmaps as long as userspace isn't changing + * the container file on us! */ + else if (coda_inode->i_mapping != host_inode->i_mapping) { + spin_unlock(&cii->c_lock); + kfree(cvm_ops); + return -EBUSY; + } + + /* keep track of how often the coda_inode/host_file has been mmapped */ + cii->c_mapcount++; + cfi->cfi_mapcount++; + spin_unlock(&cii->c_lock); + + vma->vm_file = get_file(host_file); + ret = vfs_mmap(vma->vm_file, vma); + + if (ret) { + /* if vfs_mmap fails, our caller will put host_file so we + * should drop the reference to the coda_file that we got. + */ + fput(coda_file); + kfree(cvm_ops); + } else { + /* here we add redirects for the open/close vm_operations */ + cvm_ops->host_vm_ops = vma->vm_ops; + if (vma->vm_ops) + cvm_ops->vm_ops = *vma->vm_ops; + + cvm_ops->vm_ops.open = coda_vm_open; + cvm_ops->vm_ops.close = coda_vm_close; + cvm_ops->coda_file = coda_file; + refcount_set(&cvm_ops->refcnt, 1); + + vma->vm_ops = &cvm_ops->vm_ops; + } + return ret; +} + +int coda_open(struct inode *coda_inode, struct file *coda_file) +{ + struct file *host_file = NULL; + int error; + unsigned short flags = coda_file->f_flags & (~O_EXCL); + unsigned short coda_flags = coda_flags_to_cflags(flags); + struct coda_file_info *cfi; + + cfi = kmalloc_obj(struct coda_file_info); + if (!cfi) + return -ENOMEM; + + error = venus_open(coda_inode->i_sb, coda_i2f(coda_inode), coda_flags, + &host_file); + if (!host_file) + error = -EIO; + + if (error) { + kfree(cfi); + return error; + } + + host_file->f_flags |= coda_file->f_flags & (O_APPEND | O_SYNC); + + cfi->cfi_magic = CODA_MAGIC; + cfi->cfi_mapcount = 0; + cfi->cfi_container = host_file; + /* assume access intents are supported unless we hear otherwise */ + cfi->cfi_access_intent = true; + + BUG_ON(coda_file->private_data != NULL); + coda_file->private_data = cfi; + return 0; +} + +int coda_release(struct inode *coda_inode, struct file *coda_file) +{ + unsigned short flags = (coda_file->f_flags) & (~O_EXCL); + unsigned short coda_flags = coda_flags_to_cflags(flags); + struct coda_file_info *cfi; + struct coda_inode_info *cii; + struct inode *host_inode; + + cfi = coda_ftoc(coda_file); + + venus_close(coda_inode->i_sb, coda_i2f(coda_inode), + coda_flags, coda_file->f_cred->fsuid); + + host_inode = file_inode(cfi->cfi_container); + cii = ITOC(coda_inode); + + /* did we mmap this file? */ + spin_lock(&cii->c_lock); + if (coda_inode->i_mapping == &host_inode->i_data) { + cii->c_mapcount -= cfi->cfi_mapcount; + if (!cii->c_mapcount) + coda_inode->i_mapping = &coda_inode->i_data; + } + spin_unlock(&cii->c_lock); + + fput(cfi->cfi_container); + kfree(coda_file->private_data); + coda_file->private_data = NULL; + + /* VFS fput ignores the return value from file_operations->release, so + * there is no use returning an error here */ + return 0; +} + +int coda_fsync(struct file *coda_file, loff_t start, loff_t end, int datasync) +{ + struct file *host_file; + struct inode *coda_inode = file_inode(coda_file); + struct coda_file_info *cfi; + int err; + + if (!(S_ISREG(coda_inode->i_mode) || S_ISDIR(coda_inode->i_mode) || + S_ISLNK(coda_inode->i_mode))) + return -EINVAL; + + err = filemap_write_and_wait_range(coda_inode->i_mapping, start, end); + if (err) + return err; + inode_lock(coda_inode); + + cfi = coda_ftoc(coda_file); + host_file = cfi->cfi_container; + + err = vfs_fsync(host_file, datasync); + if (!err && !datasync) + err = venus_fsync(coda_inode->i_sb, coda_i2f(coda_inode)); + inode_unlock(coda_inode); + + return err; +} + +const struct file_operations coda_file_operations = { + .llseek = generic_file_llseek, + .read_iter = coda_file_read_iter, + .write_iter = coda_file_write_iter, + .mmap = coda_file_mmap, + .open = coda_open, + .release = coda_release, + .fsync = coda_fsync, + .splice_read = coda_file_splice_read, +}; diff --git a/fs/coda/inode.c b/fs/coda/inode.c new file mode 100644 index 000000000..40b43866e --- /dev/null +++ b/fs/coda/inode.c @@ -0,0 +1,402 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Super block/filesystem wide operations + * + * Copyright (C) 1996 Peter J. Braam and + * Michael Callahan + * + * Rewritten for Linux 2.1. Peter Braam + * Copyright (C) Carnegie Mellon University + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "coda_psdev.h" +#include "coda_linux.h" +#include "coda_cache.h" + +#include "coda_int.h" + +/* VFS super_block ops */ +static void coda_evict_inode(struct inode *); +static void coda_put_super(struct super_block *); +static int coda_statfs(struct dentry *dentry, struct kstatfs *buf); + +static struct kmem_cache * coda_inode_cachep; + +static struct inode *coda_alloc_inode(struct super_block *sb) +{ + struct coda_inode_info *ei; + ei = alloc_inode_sb(sb, coda_inode_cachep, GFP_KERNEL); + if (!ei) + return NULL; + memset(&ei->c_fid, 0, sizeof(struct CodaFid)); + ei->c_flags = 0; + ei->c_uid = GLOBAL_ROOT_UID; + ei->c_cached_perm = 0; + spin_lock_init(&ei->c_lock); + return &ei->vfs_inode; +} + +static void coda_free_inode(struct inode *inode) +{ + kmem_cache_free(coda_inode_cachep, ITOC(inode)); +} + +static void init_once(void *foo) +{ + struct coda_inode_info *ei = (struct coda_inode_info *) foo; + + inode_init_once(&ei->vfs_inode); +} + +int __init coda_init_inodecache(void) +{ + coda_inode_cachep = kmem_cache_create("coda_inode_cache", + sizeof(struct coda_inode_info), 0, + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, + init_once); + if (coda_inode_cachep == NULL) + return -ENOMEM; + return 0; +} + +void coda_destroy_inodecache(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(coda_inode_cachep); +} + +static int coda_reconfigure(struct fs_context *fc) +{ + sync_filesystem(fc->root->d_sb); + fc->sb_flags |= SB_NOATIME; + return 0; +} + +/* exported operations */ +static const struct super_operations coda_super_operations = +{ + .alloc_inode = coda_alloc_inode, + .free_inode = coda_free_inode, + .evict_inode = coda_evict_inode, + .put_super = coda_put_super, + .statfs = coda_statfs, +}; + +struct coda_fs_context { + int idx; +}; + +enum { + Opt_fd, +}; + +static const struct fs_parameter_spec coda_param_specs[] = { + fsparam_fd ("fd", Opt_fd), + {} +}; + +static int coda_set_idx(struct fs_context *fc, struct file *file) +{ + struct coda_fs_context *ctx = fc->fs_private; + struct inode *inode; + int idx; + + inode = file_inode(file); + if (!S_ISCHR(inode->i_mode) || imajor(inode) != CODA_PSDEV_MAJOR) { + return invalf(fc, "coda: Not coda psdev"); + } + idx = iminor(inode); + if (idx < 0 || idx >= MAX_CODADEVS) + return invalf(fc, "coda: Bad minor number"); + ctx->idx = idx; + return 0; +} + +static int coda_parse_fd(struct fs_context *fc, struct fs_parameter *param, + struct fs_parse_result *result) +{ + struct file *file; + int err; + + if (param->type == fs_value_is_file) { + file = param->file; + param->file = NULL; + } else { + file = fget(result->uint_32); + } + if (!file) + return -EBADF; + + err = coda_set_idx(fc, file); + fput(file); + return err; +} + +static int coda_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct fs_parse_result result; + int opt; + + opt = fs_parse(fc, coda_param_specs, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_fd: + return coda_parse_fd(fc, param, &result); + } + + return 0; +} + +/* + * Parse coda's binary mount data form. We ignore any errors and go with index + * 0 if we get one for backward compatibility. + */ +static int coda_parse_monolithic(struct fs_context *fc, void *_data) +{ + struct file *file; + struct coda_mount_data *data = _data; + + if (!data) + return invalf(fc, "coda: Bad mount data"); + + if (data->version != CODA_MOUNT_VERSION) + return invalf(fc, "coda: Bad mount version"); + + file = fget(data->fd); + if (file) { + coda_set_idx(fc, file); + fput(file); + } + return 0; +} + +static int coda_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct coda_fs_context *ctx = fc->fs_private; + struct inode *root = NULL; + struct venus_comm *vc; + struct CodaFid fid; + int error; + + infof(fc, "coda: device index: %i\n", ctx->idx); + + vc = &coda_comms[ctx->idx]; + mutex_lock(&vc->vc_mutex); + + if (!vc->vc_inuse) { + errorf(fc, "coda: No pseudo device"); + error = -EINVAL; + goto unlock_out; + } + + if (vc->vc_sb) { + errorf(fc, "coda: Device already mounted"); + error = -EBUSY; + goto unlock_out; + } + + vc->vc_sb = sb; + mutex_unlock(&vc->vc_mutex); + + sb->s_fs_info = vc; + sb->s_flags |= SB_NOATIME; + sb->s_blocksize = 4096; /* XXXXX what do we put here?? */ + sb->s_blocksize_bits = 12; + sb->s_magic = CODA_SUPER_MAGIC; + sb->s_op = &coda_super_operations; + set_default_d_op(sb, &coda_dentry_operations); + sb->s_time_gran = 1; + sb->s_time_min = S64_MIN; + sb->s_time_max = S64_MAX; + + error = super_setup_bdi(sb); + if (error) + goto error; + + /* get root fid from Venus: this needs the root inode */ + error = venus_rootfid(sb, &fid); + if ( error ) { + pr_warn("%s: coda_get_rootfid failed with %d\n", + __func__, error); + goto error; + } + pr_info("%s: rootfid is %s\n", __func__, coda_f2s(&fid)); + + /* make root inode */ + root = coda_cnode_make(&fid, sb); + if (IS_ERR(root)) { + error = PTR_ERR(root); + pr_warn("Failure of coda_cnode_make for root: error %d\n", + error); + goto error; + } + + pr_info("%s: rootinode is %llu dev %s\n", + __func__, root->i_ino, root->i_sb->s_id); + sb->s_root = d_make_root(root); + if (!sb->s_root) { + error = -EINVAL; + goto error; + } + return 0; + +error: + mutex_lock(&vc->vc_mutex); + vc->vc_sb = NULL; + sb->s_fs_info = NULL; +unlock_out: + mutex_unlock(&vc->vc_mutex); + return error; +} + +static void coda_put_super(struct super_block *sb) +{ + struct venus_comm *vcp = coda_vcp(sb); + mutex_lock(&vcp->vc_mutex); + vcp->vc_sb = NULL; + sb->s_fs_info = NULL; + mutex_unlock(&vcp->vc_mutex); + mutex_destroy(&vcp->vc_mutex); + + pr_info("Bye bye.\n"); +} + +static void coda_evict_inode(struct inode *inode) +{ + truncate_inode_pages_final(&inode->i_data); + clear_inode(inode); + coda_cache_clear_inode(inode); +} + +int coda_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags) +{ + int err = coda_revalidate_inode(d_inode(path->dentry)); + if (!err) + generic_fillattr(&nop_mnt_idmap, request_mask, + d_inode(path->dentry), stat); + return err; +} + +int coda_setattr(struct mnt_idmap *idmap, struct dentry *de, + struct iattr *iattr) +{ + struct inode *inode = d_inode(de); + struct coda_vattr vattr; + int error; + + memset(&vattr, 0, sizeof(vattr)); + + inode_set_ctime_current(inode); + coda_iattr_to_vattr(iattr, &vattr); + vattr.va_type = C_VNON; /* cannot set type */ + + /* Venus is responsible for truncating the container-file!!! */ + error = venus_setattr(inode->i_sb, coda_i2f(inode), &vattr); + + if (!error) { + coda_vattr_to_iattr(inode, &vattr); + coda_cache_clear_inode(inode); + } + return error; +} + +const struct inode_operations coda_file_inode_operations = { + .permission = coda_permission, + .getattr = coda_getattr, + .setattr = coda_setattr, +}; + +static int coda_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + int error; + + error = venus_statfs(dentry, buf); + + if (error) { + /* fake something like AFS does */ + buf->f_blocks = 9000000; + buf->f_bfree = 9000000; + buf->f_bavail = 9000000; + buf->f_files = 9000000; + buf->f_ffree = 9000000; + } + + /* and fill in the rest */ + buf->f_type = CODA_SUPER_MAGIC; + buf->f_bsize = 4096; + buf->f_namelen = CODA_MAXNAMLEN; + + return 0; +} + +static int coda_get_tree(struct fs_context *fc) +{ + if (task_active_pid_ns(current) != &init_pid_ns) + return -EINVAL; + + return get_tree_nodev(fc, coda_fill_super); +} + +static void coda_free_fc(struct fs_context *fc) +{ + kfree(fc->fs_private); +} + +static const struct fs_context_operations coda_context_ops = { + .free = coda_free_fc, + .parse_param = coda_parse_param, + .parse_monolithic = coda_parse_monolithic, + .get_tree = coda_get_tree, + .reconfigure = coda_reconfigure, +}; + +static int coda_init_fs_context(struct fs_context *fc) +{ + struct coda_fs_context *ctx; + + ctx = kzalloc_obj(struct coda_fs_context); + if (!ctx) + return -ENOMEM; + + fc->fs_private = ctx; + fc->ops = &coda_context_ops; + return 0; +} + +struct file_system_type coda_fs_type = { + .owner = THIS_MODULE, + .name = "coda", + .init_fs_context = coda_init_fs_context, + .parameters = coda_param_specs, + .kill_sb = kill_anon_super, + .fs_flags = FS_BINARY_MOUNTDATA, +}; +MODULE_ALIAS_FS("coda"); + diff --git a/fs/coda/pioctl.c b/fs/coda/pioctl.c new file mode 100644 index 000000000..36e35c155 --- /dev/null +++ b/fs/coda/pioctl.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Pioctl operations for Coda. + * Original version: (C) 1996 Peter Braam + * Rewritten for Linux 2.1: (C) 1997 Carnegie Mellon University + * + * Carnegie Mellon encourages users of this code to contribute improvements + * to the Coda project. Contact Peter Braam . + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "coda_psdev.h" +#include "coda_linux.h" + +/* pioctl ops */ +static int coda_ioctl_permission(struct mnt_idmap *idmap, + struct inode *inode, int mask); +static long coda_pioctl(struct file *filp, unsigned int cmd, + unsigned long user_data); + +/* exported from this file */ +const struct inode_operations coda_ioctl_inode_operations = { + .permission = coda_ioctl_permission, + .setattr = coda_setattr, +}; + +const struct file_operations coda_ioctl_operations = { + .unlocked_ioctl = coda_pioctl, + .llseek = noop_llseek, +}; + +/* the coda pioctl inode ops */ +static int coda_ioctl_permission(struct mnt_idmap *idmap, + struct inode *inode, int mask) +{ + return (mask & MAY_EXEC) ? -EACCES : 0; +} + +static long coda_pioctl(struct file *filp, unsigned int cmd, + unsigned long user_data) +{ + struct path path; + int error; + struct PioctlData data; + struct inode *inode = file_inode(filp); + struct inode *target_inode = NULL; + struct coda_inode_info *cnp; + + /* get the Pioctl data arguments from user space */ + if (copy_from_user(&data, (void __user *)user_data, sizeof(data))) + return -EINVAL; + + /* + * Look up the pathname. Note that the pathname is in + * user memory, and namei takes care of this + */ + error = user_path_at(AT_FDCWD, data.path, + data.follow ? LOOKUP_FOLLOW : 0, &path); + if (error) + return error; + + target_inode = d_inode(path.dentry); + + /* return if it is not a Coda inode */ + if (target_inode->i_sb != inode->i_sb) { + error = -EINVAL; + goto out; + } + + /* now proceed to make the upcall */ + cnp = ITOC(target_inode); + + error = venus_pioctl(inode->i_sb, &(cnp->c_fid), cmd, &data); +out: + path_put(&path); + return error; +} diff --git a/fs/coda/psdev.c b/fs/coda/psdev.c new file mode 100644 index 000000000..3c3148588 --- /dev/null +++ b/fs/coda/psdev.c @@ -0,0 +1,438 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * An implementation of a loadable kernel mode driver providing + * multiple kernel/user space bidirectional communications links. + * + * Author: Alan Cox + * + * Adapted to become the Linux 2.0 Coda pseudo device + * Peter Braam + * Michael Callahan + * + * Changes for Linux 2.1 + * Copyright (c) 1997 Carnegie-Mellon University + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "coda_psdev.h" +#include "coda_linux.h" + +#include "coda_int.h" + +/* statistics */ +int coda_hard; /* allows signals during upcalls */ +unsigned long coda_timeout = 30; /* .. secs, then signals will dequeue */ + + +struct venus_comm coda_comms[MAX_CODADEVS]; +static struct class *coda_psdev_class; + +/* + * Device operations + */ + +static __poll_t coda_psdev_poll(struct file *file, poll_table * wait) +{ + struct venus_comm *vcp = (struct venus_comm *) file->private_data; + __poll_t mask = EPOLLOUT | EPOLLWRNORM; + + poll_wait(file, &vcp->vc_waitq, wait); + mutex_lock(&vcp->vc_mutex); + if (!list_empty(&vcp->vc_pending)) + mask |= EPOLLIN | EPOLLRDNORM; + mutex_unlock(&vcp->vc_mutex); + + return mask; +} + +static long coda_psdev_ioctl(struct file * filp, unsigned int cmd, unsigned long arg) +{ + unsigned int data; + + switch(cmd) { + case CIOC_KERNEL_VERSION: + data = CODA_KERNEL_VERSION; + return put_user(data, (int __user *) arg); + default: + return -ENOTTY; + } + + return 0; +} + +/* + * Receive a message written by Venus to the psdev + */ + +static ssize_t coda_psdev_write(struct file *file, const char __user *buf, + size_t nbytes, loff_t *off) +{ + struct venus_comm *vcp = (struct venus_comm *) file->private_data; + struct upc_req *req = NULL; + struct upc_req *tmp; + struct list_head *lh; + struct coda_in_hdr hdr; + ssize_t retval = 0, count = 0; + int error; + + /* make sure there is enough to copy out the (opcode, unique) values */ + if (nbytes < (2 * sizeof(u_int32_t))) + return -EINVAL; + + /* Peek at the opcode, uniquefier */ + if (copy_from_user(&hdr, buf, 2 * sizeof(u_int32_t))) + return -EFAULT; + + if (DOWNCALL(hdr.opcode)) { + union outputArgs *dcbuf; + int size = sizeof(*dcbuf); + + if ( nbytes < sizeof(struct coda_out_hdr) ) { + pr_warn("coda_downcall opc %d uniq %d, not enough!\n", + hdr.opcode, hdr.unique); + count = nbytes; + goto out; + } + if ( nbytes > size ) { + pr_warn("downcall opc %d, uniq %d, too much!", + hdr.opcode, hdr.unique); + nbytes = size; + } + + dcbuf = vmemdup_user(buf, nbytes); + if (IS_ERR(dcbuf)) { + retval = PTR_ERR(dcbuf); + goto out; + } + + /* what downcall errors does Venus handle ? */ + error = coda_downcall(vcp, hdr.opcode, dcbuf, nbytes); + + kvfree(dcbuf); + if (error) { + pr_warn("%s: coda_downcall error: %d\n", + __func__, error); + retval = error; + goto out; + } + count = nbytes; + goto out; + } + + /* Look for the message on the processing queue. */ + mutex_lock(&vcp->vc_mutex); + list_for_each(lh, &vcp->vc_processing) { + tmp = list_entry(lh, struct upc_req , uc_chain); + if (tmp->uc_unique == hdr.unique) { + req = tmp; + list_del(&req->uc_chain); + break; + } + } + mutex_unlock(&vcp->vc_mutex); + + if (!req) { + pr_warn("%s: msg (%d, %d) not found\n", + __func__, hdr.opcode, hdr.unique); + retval = -ESRCH; + goto out; + } + + /* move data into response buffer. */ + if (req->uc_outSize < nbytes) { + pr_warn("%s: too much cnt: %d, cnt: %ld, opc: %d, uniq: %d.\n", + __func__, req->uc_outSize, (long)nbytes, + hdr.opcode, hdr.unique); + nbytes = req->uc_outSize; /* don't have more space! */ + } + if (copy_from_user(req->uc_data, buf, nbytes)) { + req->uc_flags |= CODA_REQ_ABORT; + wake_up(&req->uc_sleep); + retval = -EFAULT; + goto out; + } + + /* adjust outsize. is this useful ?? */ + req->uc_outSize = nbytes; + req->uc_flags |= CODA_REQ_WRITE; + count = nbytes; + + /* Convert filedescriptor into a file handle */ + if (req->uc_opcode == CODA_OPEN_BY_FD) { + struct coda_open_by_fd_out *outp = + (struct coda_open_by_fd_out *)req->uc_data; + if (!outp->oh.result) { + outp->fh = fget(outp->fd); + if (!outp->fh) + return -EBADF; + } + } + + wake_up(&req->uc_sleep); +out: + return(count ? count : retval); +} + +/* + * Read a message from the kernel to Venus + */ + +static ssize_t coda_psdev_read(struct file * file, char __user * buf, + size_t nbytes, loff_t *off) +{ + DECLARE_WAITQUEUE(wait, current); + struct venus_comm *vcp = (struct venus_comm *) file->private_data; + struct upc_req *req; + ssize_t retval = 0, count = 0; + + if (nbytes == 0) + return 0; + + mutex_lock(&vcp->vc_mutex); + + add_wait_queue(&vcp->vc_waitq, &wait); + set_current_state(TASK_INTERRUPTIBLE); + + while (list_empty(&vcp->vc_pending)) { + if (file->f_flags & O_NONBLOCK) { + retval = -EAGAIN; + break; + } + if (signal_pending(current)) { + retval = -ERESTARTSYS; + break; + } + mutex_unlock(&vcp->vc_mutex); + schedule(); + mutex_lock(&vcp->vc_mutex); + } + + set_current_state(TASK_RUNNING); + remove_wait_queue(&vcp->vc_waitq, &wait); + + if (retval) + goto out; + + req = list_entry(vcp->vc_pending.next, struct upc_req,uc_chain); + list_del(&req->uc_chain); + + /* Move the input args into userspace */ + count = req->uc_inSize; + if (nbytes < req->uc_inSize) { + pr_warn("%s: Venus read %ld bytes of %d in message\n", + __func__, (long)nbytes, req->uc_inSize); + count = nbytes; + } + + if (copy_to_user(buf, req->uc_data, count)) + retval = -EFAULT; + + /* If request was not a signal, enqueue and don't free */ + if (!(req->uc_flags & CODA_REQ_ASYNC)) { + req->uc_flags |= CODA_REQ_READ; + list_add_tail(&(req->uc_chain), &vcp->vc_processing); + goto out; + } + + kvfree(req->uc_data); + kfree(req); +out: + mutex_unlock(&vcp->vc_mutex); + return (count ? count : retval); +} + +static int coda_psdev_open(struct inode * inode, struct file * file) +{ + struct venus_comm *vcp; + int idx, err; + + if (task_active_pid_ns(current) != &init_pid_ns) + return -EINVAL; + + if (current_user_ns() != &init_user_ns) + return -EINVAL; + + idx = iminor(inode); + if (idx < 0 || idx >= MAX_CODADEVS) + return -ENODEV; + + err = -EBUSY; + vcp = &coda_comms[idx]; + mutex_lock(&vcp->vc_mutex); + + if (!vcp->vc_inuse) { + vcp->vc_inuse++; + + INIT_LIST_HEAD(&vcp->vc_pending); + INIT_LIST_HEAD(&vcp->vc_processing); + init_waitqueue_head(&vcp->vc_waitq); + vcp->vc_sb = NULL; + vcp->vc_seq = 0; + + file->private_data = vcp; + err = 0; + } + + mutex_unlock(&vcp->vc_mutex); + return err; +} + + +static int coda_psdev_release(struct inode * inode, struct file * file) +{ + struct venus_comm *vcp = (struct venus_comm *) file->private_data; + struct upc_req *req, *tmp; + + if (!vcp || !vcp->vc_inuse ) { + pr_warn("%s: Not open.\n", __func__); + return -1; + } + + mutex_lock(&vcp->vc_mutex); + + /* Wakeup clients so they can return. */ + list_for_each_entry_safe(req, tmp, &vcp->vc_pending, uc_chain) { + list_del(&req->uc_chain); + + /* Async requests need to be freed here */ + if (req->uc_flags & CODA_REQ_ASYNC) { + kvfree(req->uc_data); + kfree(req); + continue; + } + req->uc_flags |= CODA_REQ_ABORT; + wake_up(&req->uc_sleep); + } + + list_for_each_entry_safe(req, tmp, &vcp->vc_processing, uc_chain) { + list_del(&req->uc_chain); + + req->uc_flags |= CODA_REQ_ABORT; + wake_up(&req->uc_sleep); + } + + file->private_data = NULL; + vcp->vc_inuse--; + mutex_unlock(&vcp->vc_mutex); + return 0; +} + + +static const struct file_operations coda_psdev_fops = { + .owner = THIS_MODULE, + .read = coda_psdev_read, + .write = coda_psdev_write, + .poll = coda_psdev_poll, + .unlocked_ioctl = coda_psdev_ioctl, + .open = coda_psdev_open, + .release = coda_psdev_release, + .llseek = noop_llseek, +}; + +static int __init init_coda_psdev(void) +{ + int i, err = 0; + if (register_chrdev(CODA_PSDEV_MAJOR, "coda", &coda_psdev_fops)) { + pr_err("%s: unable to get major %d\n", + __func__, CODA_PSDEV_MAJOR); + return -EIO; + } + coda_psdev_class = class_create("coda"); + if (IS_ERR(coda_psdev_class)) { + err = PTR_ERR(coda_psdev_class); + goto out_chrdev; + } + for (i = 0; i < MAX_CODADEVS; i++) { + mutex_init(&(&coda_comms[i])->vc_mutex); + device_create(coda_psdev_class, NULL, + MKDEV(CODA_PSDEV_MAJOR, i), NULL, "cfs%d", i); + } + coda_sysctl_init(); + goto out; + +out_chrdev: + unregister_chrdev(CODA_PSDEV_MAJOR, "coda"); +out: + return err; +} + +MODULE_AUTHOR("Jan Harkes, Peter J. Braam"); +MODULE_DESCRIPTION("Coda Distributed File System VFS interface"); +MODULE_ALIAS_CHARDEV_MAJOR(CODA_PSDEV_MAJOR); +MODULE_LICENSE("GPL"); +MODULE_VERSION("7.2"); + +static int __init init_coda(void) +{ + int status; + int i; + + status = coda_init_inodecache(); + if (status) + goto out2; + status = init_coda_psdev(); + if ( status ) { + pr_warn("Problem (%d) in init_coda_psdev\n", status); + goto out1; + } + + status = register_filesystem(&coda_fs_type); + if (status) { + pr_warn("failed to register filesystem!\n"); + goto out; + } + return 0; +out: + for (i = 0; i < MAX_CODADEVS; i++) + device_destroy(coda_psdev_class, MKDEV(CODA_PSDEV_MAJOR, i)); + class_destroy(coda_psdev_class); + unregister_chrdev(CODA_PSDEV_MAJOR, "coda"); + coda_sysctl_clean(); +out1: + coda_destroy_inodecache(); +out2: + return status; +} + +static void __exit exit_coda(void) +{ + int err, i; + + err = unregister_filesystem(&coda_fs_type); + if (err != 0) + pr_warn("failed to unregister filesystem\n"); + for (i = 0; i < MAX_CODADEVS; i++) + device_destroy(coda_psdev_class, MKDEV(CODA_PSDEV_MAJOR, i)); + class_destroy(coda_psdev_class); + unregister_chrdev(CODA_PSDEV_MAJOR, "coda"); + coda_sysctl_clean(); + coda_destroy_inodecache(); +} + +module_init(init_coda); +module_exit(exit_coda); + diff --git a/fs/coda/symlink.c b/fs/coda/symlink.c new file mode 100644 index 000000000..40f84d014 --- /dev/null +++ b/fs/coda/symlink.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Symlink inode operations for Coda filesystem + * Original version: (C) 1996 P. Braam and M. Callahan + * Rewritten for Linux 2.1. (C) 1997 Carnegie Mellon University + * + * Carnegie Mellon encourages users to contribute improvements to + * the Coda project. Contact Peter Braam (coda@cs.cmu.edu). + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include "coda_psdev.h" +#include "coda_linux.h" + +static int coda_symlink_filler(struct file *file, struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + int error; + struct coda_inode_info *cii; + unsigned int len = PAGE_SIZE; + char *p = folio_address(folio); + + cii = ITOC(inode); + + error = venus_readlink(inode->i_sb, &cii->c_fid, p, &len); + folio_end_read(folio, error == 0); + return error; +} + +const struct address_space_operations coda_symlink_aops = { + .read_folio = coda_symlink_filler, +}; diff --git a/fs/coda/sysctl.c b/fs/coda/sysctl.c new file mode 100644 index 000000000..0df46f09b --- /dev/null +++ b/fs/coda/sysctl.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Sysctl operations for Coda filesystem + * Original version: (C) 1996 P. Braam and M. Callahan + * Rewritten for Linux 2.1. (C) 1997 Carnegie Mellon University + * + * Carnegie Mellon encourages users to contribute improvements to + * the Coda project. Contact Peter Braam (coda@cs.cmu.edu). + */ + +#include + +#include "coda_int.h" + +static struct ctl_table_header *fs_table_header; + +static const struct ctl_table coda_table[] = { + { + .procname = "timeout", + .data = &coda_timeout, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec + }, + { + .procname = "hard", + .data = &coda_hard, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec + }, + { + .procname = "fake_statfs", + .data = &coda_fake_statfs, + .maxlen = sizeof(int), + .mode = 0600, + .proc_handler = proc_dointvec + }, +}; + +void coda_sysctl_init(void) +{ + if ( !fs_table_header ) + fs_table_header = register_sysctl("coda", coda_table); +} + +void coda_sysctl_clean(void) +{ + if ( fs_table_header ) { + unregister_sysctl_table(fs_table_header); + fs_table_header = NULL; + } +} diff --git a/fs/coda/upcall.c b/fs/coda/upcall.c new file mode 100644 index 000000000..3438119b5 --- /dev/null +++ b/fs/coda/upcall.c @@ -0,0 +1,961 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Mostly platform independent upcall operations to Venus: + * -- upcalls + * -- upcall routines + * + * Linux 2.0 version + * Copyright (C) 1996 Peter J. Braam , + * Michael Callahan + * + * Redone for Linux 2.1 + * Copyright (C) 1997 Carnegie Mellon University + * + * Carnegie Mellon University encourages users of this code to contribute + * improvements to the Coda project. Contact Peter Braam . + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "coda_psdev.h" +#include "coda_linux.h" +#include "coda_cache.h" + +#include "coda_int.h" + +static int coda_upcall(struct venus_comm *vc, int inSize, int *outSize, + union inputArgs *buffer); + +static void *alloc_upcall(int opcode, int size) +{ + union inputArgs *inp; + + inp = kvzalloc(size, GFP_KERNEL); + if (!inp) + return ERR_PTR(-ENOMEM); + + inp->ih.opcode = opcode; + inp->ih.pid = task_pid_nr_ns(current, &init_pid_ns); + inp->ih.pgid = task_pgrp_nr_ns(current, &init_pid_ns); + inp->ih.uid = from_kuid(&init_user_ns, current_fsuid()); + + return (void*)inp; +} + +#define UPARG(op)\ +do {\ + inp = (union inputArgs *)alloc_upcall(op, insize); \ + if (IS_ERR(inp)) { return PTR_ERR(inp); }\ + outp = (union outputArgs *)(inp); \ + outsize = insize; \ +} while (0) + +#define INSIZE(tag) sizeof(struct coda_ ## tag ## _in) +#define OUTSIZE(tag) sizeof(struct coda_ ## tag ## _out) +#define SIZE(tag) max_t(unsigned int, INSIZE(tag), OUTSIZE(tag)) + + +/* the upcalls */ +int venus_rootfid(struct super_block *sb, struct CodaFid *fidp) +{ + union inputArgs *inp; + union outputArgs *outp; + int insize, outsize, error; + + insize = SIZE(root); + UPARG(CODA_ROOT); + + error = coda_upcall(coda_vcp(sb), insize, &outsize, inp); + if (!error) + *fidp = outp->coda_root.VFid; + + kvfree(inp); + return error; +} + +int venus_getattr(struct super_block *sb, struct CodaFid *fid, + struct coda_vattr *attr) +{ + union inputArgs *inp; + union outputArgs *outp; + int insize, outsize, error; + + insize = SIZE(getattr); + UPARG(CODA_GETATTR); + inp->coda_getattr.VFid = *fid; + + error = coda_upcall(coda_vcp(sb), insize, &outsize, inp); + if (!error) + *attr = outp->coda_getattr.attr; + + kvfree(inp); + return error; +} + +int venus_setattr(struct super_block *sb, struct CodaFid *fid, + struct coda_vattr *vattr) +{ + union inputArgs *inp; + union outputArgs *outp; + int insize, outsize, error; + + insize = SIZE(setattr); + UPARG(CODA_SETATTR); + + inp->coda_setattr.VFid = *fid; + inp->coda_setattr.attr = *vattr; + + error = coda_upcall(coda_vcp(sb), insize, &outsize, inp); + + kvfree(inp); + return error; +} + +int venus_lookup(struct super_block *sb, struct CodaFid *fid, + const char *name, int length, int * type, + struct CodaFid *resfid) +{ + union inputArgs *inp; + union outputArgs *outp; + int insize, outsize, error; + int offset; + + offset = INSIZE(lookup); + insize = max_t(unsigned int, offset + length +1, OUTSIZE(lookup)); + UPARG(CODA_LOOKUP); + + inp->coda_lookup.VFid = *fid; + inp->coda_lookup.name = offset; + inp->coda_lookup.flags = CLU_CASE_SENSITIVE; + /* send Venus a null terminated string */ + memcpy((char *)(inp) + offset, name, length); + *((char *)inp + offset + length) = '\0'; + + error = coda_upcall(coda_vcp(sb), insize, &outsize, inp); + if (!error) { + *resfid = outp->coda_lookup.VFid; + *type = outp->coda_lookup.vtype; + } + + kvfree(inp); + return error; +} + +int venus_close(struct super_block *sb, struct CodaFid *fid, int flags, + kuid_t uid) +{ + union inputArgs *inp; + union outputArgs *outp; + int insize, outsize, error; + + insize = SIZE(release); + UPARG(CODA_CLOSE); + + inp->ih.uid = from_kuid(&init_user_ns, uid); + inp->coda_close.VFid = *fid; + inp->coda_close.flags = flags; + + error = coda_upcall(coda_vcp(sb), insize, &outsize, inp); + + kvfree(inp); + return error; +} + +int venus_open(struct super_block *sb, struct CodaFid *fid, + int flags, struct file **fh) +{ + union inputArgs *inp; + union outputArgs *outp; + int insize, outsize, error; + + insize = SIZE(open_by_fd); + UPARG(CODA_OPEN_BY_FD); + + inp->coda_open_by_fd.VFid = *fid; + inp->coda_open_by_fd.flags = flags; + + error = coda_upcall(coda_vcp(sb), insize, &outsize, inp); + if (!error) + *fh = outp->coda_open_by_fd.fh; + + kvfree(inp); + return error; +} + +int venus_mkdir(struct super_block *sb, struct CodaFid *dirfid, + const char *name, int length, + struct CodaFid *newfid, struct coda_vattr *attrs) +{ + union inputArgs *inp; + union outputArgs *outp; + int insize, outsize, error; + int offset; + + offset = INSIZE(mkdir); + insize = max_t(unsigned int, offset + length + 1, OUTSIZE(mkdir)); + UPARG(CODA_MKDIR); + + inp->coda_mkdir.VFid = *dirfid; + inp->coda_mkdir.attr = *attrs; + inp->coda_mkdir.name = offset; + /* Venus must get null terminated string */ + memcpy((char *)(inp) + offset, name, length); + *((char *)inp + offset + length) = '\0'; + + error = coda_upcall(coda_vcp(sb), insize, &outsize, inp); + if (!error) { + *attrs = outp->coda_mkdir.attr; + *newfid = outp->coda_mkdir.VFid; + } + + kvfree(inp); + return error; +} + + +int venus_rename(struct super_block *sb, struct CodaFid *old_fid, + struct CodaFid *new_fid, size_t old_length, + size_t new_length, const char *old_name, + const char *new_name) +{ + union inputArgs *inp; + union outputArgs *outp; + int insize, outsize, error; + int offset, s; + + offset = INSIZE(rename); + insize = max_t(unsigned int, offset + new_length + old_length + 8, + OUTSIZE(rename)); + UPARG(CODA_RENAME); + + inp->coda_rename.sourceFid = *old_fid; + inp->coda_rename.destFid = *new_fid; + inp->coda_rename.srcname = offset; + + /* Venus must receive an null terminated string */ + s = ( old_length & ~0x3) +4; /* round up to word boundary */ + memcpy((char *)(inp) + offset, old_name, old_length); + *((char *)inp + offset + old_length) = '\0'; + + /* another null terminated string for Venus */ + offset += s; + inp->coda_rename.destname = offset; + s = ( new_length & ~0x3) +4; /* round up to word boundary */ + memcpy((char *)(inp) + offset, new_name, new_length); + *((char *)inp + offset + new_length) = '\0'; + + error = coda_upcall(coda_vcp(sb), insize, &outsize, inp); + + kvfree(inp); + return error; +} + +int venus_create(struct super_block *sb, struct CodaFid *dirfid, + const char *name, int length, int excl, int mode, + struct CodaFid *newfid, struct coda_vattr *attrs) +{ + union inputArgs *inp; + union outputArgs *outp; + int insize, outsize, error; + int offset; + + offset = INSIZE(create); + insize = max_t(unsigned int, offset + length + 1, OUTSIZE(create)); + UPARG(CODA_CREATE); + + inp->coda_create.VFid = *dirfid; + inp->coda_create.attr.va_mode = mode; + inp->coda_create.excl = excl; + inp->coda_create.mode = mode; + inp->coda_create.name = offset; + + /* Venus must get null terminated string */ + memcpy((char *)(inp) + offset, name, length); + *((char *)inp + offset + length) = '\0'; + + error = coda_upcall(coda_vcp(sb), insize, &outsize, inp); + if (!error) { + *attrs = outp->coda_create.attr; + *newfid = outp->coda_create.VFid; + } + + kvfree(inp); + return error; +} + +int venus_rmdir(struct super_block *sb, struct CodaFid *dirfid, + const char *name, int length) +{ + union inputArgs *inp; + union outputArgs *outp; + int insize, outsize, error; + int offset; + + offset = INSIZE(rmdir); + insize = max_t(unsigned int, offset + length + 1, OUTSIZE(rmdir)); + UPARG(CODA_RMDIR); + + inp->coda_rmdir.VFid = *dirfid; + inp->coda_rmdir.name = offset; + memcpy((char *)(inp) + offset, name, length); + *((char *)inp + offset + length) = '\0'; + + error = coda_upcall(coda_vcp(sb), insize, &outsize, inp); + + kvfree(inp); + return error; +} + +int venus_remove(struct super_block *sb, struct CodaFid *dirfid, + const char *name, int length) +{ + union inputArgs *inp; + union outputArgs *outp; + int error=0, insize, outsize, offset; + + offset = INSIZE(remove); + insize = max_t(unsigned int, offset + length + 1, OUTSIZE(remove)); + UPARG(CODA_REMOVE); + + inp->coda_remove.VFid = *dirfid; + inp->coda_remove.name = offset; + memcpy((char *)(inp) + offset, name, length); + *((char *)inp + offset + length) = '\0'; + + error = coda_upcall(coda_vcp(sb), insize, &outsize, inp); + + kvfree(inp); + return error; +} + +int venus_readlink(struct super_block *sb, struct CodaFid *fid, + char *buffer, int *length) +{ + union inputArgs *inp; + union outputArgs *outp; + int insize, outsize, error; + int retlen; + char *result; + + insize = max_t(unsigned int, + INSIZE(readlink), OUTSIZE(readlink)+ *length); + UPARG(CODA_READLINK); + + inp->coda_readlink.VFid = *fid; + + error = coda_upcall(coda_vcp(sb), insize, &outsize, inp); + if (!error) { + retlen = outp->coda_readlink.count; + if (retlen >= *length) + retlen = *length - 1; + *length = retlen; + result = (char *)outp + (long)outp->coda_readlink.data; + memcpy(buffer, result, retlen); + *(buffer + retlen) = '\0'; + } + + kvfree(inp); + return error; +} + + + +int venus_link(struct super_block *sb, struct CodaFid *fid, + struct CodaFid *dirfid, const char *name, int len ) +{ + union inputArgs *inp; + union outputArgs *outp; + int insize, outsize, error; + int offset; + + offset = INSIZE(link); + insize = max_t(unsigned int, offset + len + 1, OUTSIZE(link)); + UPARG(CODA_LINK); + + inp->coda_link.sourceFid = *fid; + inp->coda_link.destFid = *dirfid; + inp->coda_link.tname = offset; + + /* make sure strings are null terminated */ + memcpy((char *)(inp) + offset, name, len); + *((char *)inp + offset + len) = '\0'; + + error = coda_upcall(coda_vcp(sb), insize, &outsize, inp); + + kvfree(inp); + return error; +} + +int venus_symlink(struct super_block *sb, struct CodaFid *fid, + const char *name, int len, + const char *symname, int symlen) +{ + union inputArgs *inp; + union outputArgs *outp; + int insize, outsize, error; + int offset, s; + + offset = INSIZE(symlink); + insize = max_t(unsigned int, offset + len + symlen + 8, OUTSIZE(symlink)); + UPARG(CODA_SYMLINK); + + /* inp->coda_symlink.attr = *tva; XXXXXX */ + inp->coda_symlink.VFid = *fid; + + /* Round up to word boundary and null terminate */ + inp->coda_symlink.srcname = offset; + s = ( symlen & ~0x3 ) + 4; + memcpy((char *)(inp) + offset, symname, symlen); + *((char *)inp + offset + symlen) = '\0'; + + /* Round up to word boundary and null terminate */ + offset += s; + inp->coda_symlink.tname = offset; + s = (len & ~0x3) + 4; + memcpy((char *)(inp) + offset, name, len); + *((char *)inp + offset + len) = '\0'; + + error = coda_upcall(coda_vcp(sb), insize, &outsize, inp); + + kvfree(inp); + return error; +} + +int venus_fsync(struct super_block *sb, struct CodaFid *fid) +{ + union inputArgs *inp; + union outputArgs *outp; + int insize, outsize, error; + + insize=SIZE(fsync); + UPARG(CODA_FSYNC); + + inp->coda_fsync.VFid = *fid; + error = coda_upcall(coda_vcp(sb), insize, &outsize, inp); + + kvfree(inp); + return error; +} + +int venus_access(struct super_block *sb, struct CodaFid *fid, int mask) +{ + union inputArgs *inp; + union outputArgs *outp; + int insize, outsize, error; + + insize = SIZE(access); + UPARG(CODA_ACCESS); + + inp->coda_access.VFid = *fid; + inp->coda_access.flags = mask; + + error = coda_upcall(coda_vcp(sb), insize, &outsize, inp); + + kvfree(inp); + return error; +} + + +int venus_pioctl(struct super_block *sb, struct CodaFid *fid, + unsigned int cmd, struct PioctlData *data) +{ + union inputArgs *inp; + union outputArgs *outp; + int insize, outsize, error; + int iocsize; + + insize = VC_MAXMSGSIZE; + UPARG(CODA_IOCTL); + + /* build packet for Venus */ + if (data->vi.in_size > VC_MAXDATASIZE) { + error = -EINVAL; + goto exit; + } + + if (data->vi.out_size > VC_MAXDATASIZE) { + error = -EINVAL; + goto exit; + } + + inp->coda_ioctl.VFid = *fid; + + /* the cmd field was mutated by increasing its size field to + * reflect the path and follow args. We need to subtract that + * out before sending the command to Venus. */ + inp->coda_ioctl.cmd = (cmd & ~(PIOCPARM_MASK << 16)); + iocsize = ((cmd >> 16) & PIOCPARM_MASK) - sizeof(char *) - sizeof(int); + inp->coda_ioctl.cmd |= (iocsize & PIOCPARM_MASK) << 16; + + /* in->coda_ioctl.rwflag = flag; */ + inp->coda_ioctl.len = data->vi.in_size; + inp->coda_ioctl.data = (char *)(INSIZE(ioctl)); + + /* get the data out of user space */ + if (copy_from_user((char *)inp + (long)inp->coda_ioctl.data, + data->vi.in, data->vi.in_size)) { + error = -EINVAL; + goto exit; + } + + error = coda_upcall(coda_vcp(sb), SIZE(ioctl) + data->vi.in_size, + &outsize, inp); + + if (error) { + pr_warn("%s: Venus returns: %d for %s\n", + __func__, error, coda_f2s(fid)); + goto exit; + } + + if (outsize < (long)outp->coda_ioctl.data + outp->coda_ioctl.len) { + error = -EINVAL; + goto exit; + } + + /* Copy out the OUT buffer. */ + if (outp->coda_ioctl.len > data->vi.out_size) { + error = -EINVAL; + goto exit; + } + + /* Copy out the OUT buffer. */ + if (copy_to_user(data->vi.out, + (char *)outp + (long)outp->coda_ioctl.data, + outp->coda_ioctl.len)) { + error = -EFAULT; + goto exit; + } + + exit: + kvfree(inp); + return error; +} + +int venus_statfs(struct dentry *dentry, struct kstatfs *sfs) +{ + union inputArgs *inp; + union outputArgs *outp; + int insize, outsize, error; + + insize = SIZE(statfs); + UPARG(CODA_STATFS); + + error = coda_upcall(coda_vcp(dentry->d_sb), insize, &outsize, inp); + if (!error) { + sfs->f_blocks = outp->coda_statfs.stat.f_blocks; + sfs->f_bfree = outp->coda_statfs.stat.f_bfree; + sfs->f_bavail = outp->coda_statfs.stat.f_bavail; + sfs->f_files = outp->coda_statfs.stat.f_files; + sfs->f_ffree = outp->coda_statfs.stat.f_ffree; + } + + kvfree(inp); + return error; +} + +int venus_access_intent(struct super_block *sb, struct CodaFid *fid, + bool *access_intent_supported, + size_t count, loff_t ppos, int type) +{ + union inputArgs *inp; + union outputArgs *outp; + int insize, outsize, error; + bool finalizer = + type == CODA_ACCESS_TYPE_READ_FINISH || + type == CODA_ACCESS_TYPE_WRITE_FINISH; + + if (!*access_intent_supported && !finalizer) + return 0; + + insize = SIZE(access_intent); + UPARG(CODA_ACCESS_INTENT); + + inp->coda_access_intent.VFid = *fid; + inp->coda_access_intent.count = count; + inp->coda_access_intent.pos = ppos; + inp->coda_access_intent.type = type; + + error = coda_upcall(coda_vcp(sb), insize, + finalizer ? NULL : &outsize, inp); + + /* + * we have to free the request buffer for synchronous upcalls + * or when asynchronous upcalls fail, but not when asynchronous + * upcalls succeed + */ + if (!finalizer || error) + kvfree(inp); + + /* Chunked access is not supported or an old Coda client */ + if (error == -EOPNOTSUPP) { + *access_intent_supported = false; + error = 0; + } + return error; +} + +/* + * coda_upcall and coda_downcall routines. + */ +static void coda_block_signals(sigset_t *old) +{ + spin_lock_irq(¤t->sighand->siglock); + *old = current->blocked; + + sigfillset(¤t->blocked); + sigdelset(¤t->blocked, SIGKILL); + sigdelset(¤t->blocked, SIGSTOP); + sigdelset(¤t->blocked, SIGINT); + + recalc_sigpending(); + spin_unlock_irq(¤t->sighand->siglock); +} + +static void coda_unblock_signals(sigset_t *old) +{ + spin_lock_irq(¤t->sighand->siglock); + current->blocked = *old; + recalc_sigpending(); + spin_unlock_irq(¤t->sighand->siglock); +} + +/* Don't allow signals to interrupt the following upcalls before venus + * has seen them, + * - CODA_CLOSE or CODA_RELEASE upcall (to avoid reference count problems) + * - CODA_STORE (to avoid data loss) + * - CODA_ACCESS_INTENT (to avoid reference count problems) + */ +#define CODA_INTERRUPTIBLE(r) (!coda_hard && \ + (((r)->uc_opcode != CODA_CLOSE && \ + (r)->uc_opcode != CODA_STORE && \ + (r)->uc_opcode != CODA_ACCESS_INTENT && \ + (r)->uc_opcode != CODA_RELEASE) || \ + (r)->uc_flags & CODA_REQ_READ)) + +static inline void coda_waitfor_upcall(struct venus_comm *vcp, + struct upc_req *req) +{ + DECLARE_WAITQUEUE(wait, current); + unsigned long timeout = jiffies + coda_timeout * HZ; + sigset_t old; + int blocked; + + coda_block_signals(&old); + blocked = 1; + + add_wait_queue(&req->uc_sleep, &wait); + for (;;) { + if (CODA_INTERRUPTIBLE(req)) + set_current_state(TASK_INTERRUPTIBLE); + else + set_current_state(TASK_UNINTERRUPTIBLE); + + /* got a reply */ + if (req->uc_flags & (CODA_REQ_WRITE | CODA_REQ_ABORT)) + break; + + if (blocked && time_after(jiffies, timeout) && + CODA_INTERRUPTIBLE(req)) + { + coda_unblock_signals(&old); + blocked = 0; + } + + if (signal_pending(current)) { + list_del(&req->uc_chain); + break; + } + + mutex_unlock(&vcp->vc_mutex); + if (blocked) + schedule_timeout(HZ); + else + schedule(); + mutex_lock(&vcp->vc_mutex); + } + if (blocked) + coda_unblock_signals(&old); + + remove_wait_queue(&req->uc_sleep, &wait); + set_current_state(TASK_RUNNING); +} + + +/* + * coda_upcall will return an error in the case of + * failed communication with Venus _or_ will peek at Venus + * reply and return Venus' error. + * + * As venus has 2 types of errors, normal errors (positive) and internal + * errors (negative), normal errors are negated, while internal errors + * are all mapped to -EINTR, while showing a nice warning message. (jh) + */ +static int coda_upcall(struct venus_comm *vcp, + int inSize, int *outSize, + union inputArgs *buffer) +{ + union outputArgs *out; + union inputArgs *sig_inputArgs; + struct upc_req *req = NULL, *sig_req; + int error; + + mutex_lock(&vcp->vc_mutex); + + if (!vcp->vc_inuse) { + pr_notice("Venus dead, not sending upcall\n"); + error = -ENXIO; + goto exit; + } + + /* Format the request message. */ + req = kmalloc_obj(struct upc_req); + if (!req) { + error = -ENOMEM; + goto exit; + } + + buffer->ih.unique = ++vcp->vc_seq; + + req->uc_data = (void *)buffer; + req->uc_flags = outSize ? 0 : CODA_REQ_ASYNC; + req->uc_inSize = inSize; + req->uc_outSize = (outSize && *outSize) ? *outSize : inSize; + req->uc_opcode = buffer->ih.opcode; + req->uc_unique = buffer->ih.unique; + init_waitqueue_head(&req->uc_sleep); + + /* Append msg to pending queue and poke Venus. */ + list_add_tail(&req->uc_chain, &vcp->vc_pending); + wake_up_interruptible(&vcp->vc_waitq); + + /* We can return early on asynchronous requests */ + if (outSize == NULL) { + mutex_unlock(&vcp->vc_mutex); + return 0; + } + + /* We can be interrupted while we wait for Venus to process + * our request. If the interrupt occurs before Venus has read + * the request, we dequeue and return. If it occurs after the + * read but before the reply, we dequeue, send a signal + * message, and return. If it occurs after the reply we ignore + * it. In no case do we want to restart the syscall. If it + * was interrupted by a venus shutdown (psdev_close), return + * ENODEV. */ + + /* Go to sleep. Wake up on signals only after the timeout. */ + coda_waitfor_upcall(vcp, req); + + /* Op went through, interrupt or not... */ + if (req->uc_flags & CODA_REQ_WRITE) { + out = (union outputArgs *)req->uc_data; + /* here we map positive Venus errors to kernel errors */ + error = -out->oh.result; + *outSize = req->uc_outSize; + goto exit; + } + + error = -EINTR; + if ((req->uc_flags & CODA_REQ_ABORT) || !signal_pending(current)) { + pr_warn("Unexpected interruption.\n"); + goto exit; + } + + /* Interrupted before venus read it. */ + if (!(req->uc_flags & CODA_REQ_READ)) + goto exit; + + /* Venus saw the upcall, make sure we can send interrupt signal */ + if (!vcp->vc_inuse) { + pr_info("Venus dead, not sending signal.\n"); + goto exit; + } + + error = -ENOMEM; + sig_req = kmalloc_obj(struct upc_req); + if (!sig_req) goto exit; + + sig_inputArgs = kvzalloc_obj(*sig_inputArgs); + if (!sig_inputArgs) { + kfree(sig_req); + goto exit; + } + + error = -EINTR; + sig_inputArgs->ih.opcode = CODA_SIGNAL; + sig_inputArgs->ih.unique = req->uc_unique; + + sig_req->uc_flags = CODA_REQ_ASYNC; + sig_req->uc_opcode = sig_inputArgs->ih.opcode; + sig_req->uc_unique = sig_inputArgs->ih.unique; + sig_req->uc_data = (void *)sig_inputArgs; + sig_req->uc_inSize = sizeof(struct coda_in_hdr); + sig_req->uc_outSize = sizeof(struct coda_in_hdr); + + /* insert at head of queue! */ + list_add(&(sig_req->uc_chain), &vcp->vc_pending); + wake_up_interruptible(&vcp->vc_waitq); + +exit: + kfree(req); + mutex_unlock(&vcp->vc_mutex); + return error; +} + +/* + The statements below are part of the Coda opportunistic + programming -- taken from the Mach/BSD kernel code for Coda. + You don't get correct semantics by stating what needs to be + done without guaranteeing the invariants needed for it to happen. + When will be have time to find out what exactly is going on? (pjb) +*/ + + +/* + * There are 7 cases where cache invalidations occur. The semantics + * of each is listed here: + * + * CODA_FLUSH -- flush all entries from the name cache and the cnode cache. + * CODA_PURGEUSER -- flush all entries from the name cache for a specific user + * This call is a result of token expiration. + * + * The next arise as the result of callbacks on a file or directory. + * CODA_ZAPFILE -- flush the cached attributes for a file. + + * CODA_ZAPDIR -- flush the attributes for the dir and + * force a new lookup for all the children + of this dir. + + * + * The next is a result of Venus detecting an inconsistent file. + * CODA_PURGEFID -- flush the attribute for the file + * purge it and its children from the dcache + * + * The last allows Venus to replace local fids with global ones + * during reintegration. + * + * CODA_REPLACE -- replace one CodaFid with another throughout the name cache */ + +int coda_downcall(struct venus_comm *vcp, int opcode, union outputArgs *out, + size_t nbytes) +{ + struct inode *inode = NULL; + struct CodaFid *fid = NULL, *newfid; + struct super_block *sb; + + /* + * Make sure we have received enough data from the cache + * manager to populate the necessary fields in the buffer + */ + switch (opcode) { + case CODA_PURGEUSER: + if (nbytes < sizeof(struct coda_purgeuser_out)) + return -EINVAL; + break; + + case CODA_ZAPDIR: + if (nbytes < sizeof(struct coda_zapdir_out)) + return -EINVAL; + break; + + case CODA_ZAPFILE: + if (nbytes < sizeof(struct coda_zapfile_out)) + return -EINVAL; + break; + + case CODA_PURGEFID: + if (nbytes < sizeof(struct coda_purgefid_out)) + return -EINVAL; + break; + + case CODA_REPLACE: + if (nbytes < sizeof(struct coda_replace_out)) + return -EINVAL; + break; + } + + /* Handle invalidation requests. */ + mutex_lock(&vcp->vc_mutex); + sb = vcp->vc_sb; + if (!sb || !sb->s_root) + goto unlock_out; + + switch (opcode) { + case CODA_FLUSH: + coda_cache_clear_all(sb); + shrink_dcache_sb(sb); + if (d_really_is_positive(sb->s_root)) + coda_flag_inode(d_inode(sb->s_root), C_FLUSH); + break; + + case CODA_PURGEUSER: + coda_cache_clear_all(sb); + break; + + case CODA_ZAPDIR: + fid = &out->coda_zapdir.CodaFid; + break; + + case CODA_ZAPFILE: + fid = &out->coda_zapfile.CodaFid; + break; + + case CODA_PURGEFID: + fid = &out->coda_purgefid.CodaFid; + break; + + case CODA_REPLACE: + fid = &out->coda_replace.OldFid; + break; + } + if (fid) + inode = coda_fid_to_inode(fid, sb); + +unlock_out: + mutex_unlock(&vcp->vc_mutex); + + if (!inode) + return 0; + + switch (opcode) { + case CODA_ZAPDIR: + coda_flag_inode_children(inode, C_PURGE); + coda_flag_inode(inode, C_VATTR); + break; + + case CODA_ZAPFILE: + coda_flag_inode(inode, C_VATTR); + break; + + case CODA_PURGEFID: + coda_flag_inode_children(inode, C_PURGE); + + /* catch the dentries later if some are still busy */ + coda_flag_inode(inode, C_PURGE); + d_prune_aliases(inode); + break; + + case CODA_REPLACE: + newfid = &out->coda_replace.NewFid; + coda_replace_fid(inode, fid, newfid); + break; + } + iput(inode); + return 0; +} diff --git a/fs/compat_binfmt_elf.c b/fs/compat_binfmt_elf.c new file mode 100644 index 000000000..d5ef5469e --- /dev/null +++ b/fs/compat_binfmt_elf.c @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * 32-bit compatibility support for ELF format executables and core dumps. + * + * Copyright (C) 2007 Red Hat, Inc. All rights reserved. + * + * Red Hat Author: Roland McGrath. + * + * This file is used in a 64-bit kernel that wants to support 32-bit ELF. + * asm/elf.h is responsible for defining the compat_* and COMPAT_* macros + * used below, with definitions appropriate for 32-bit ABI compatibility. + * + * We use macros to rename the ABI types and machine-dependent + * functions used in binfmt_elf.c to compat versions. + */ + +#include +#include + +#define ELF_COMPAT 1 + +/* + * Rename the basic ELF layout types to refer to the 32-bit class of files. + */ +#undef ELF_CLASS +#define ELF_CLASS ELFCLASS32 + +#undef elfhdr +#undef elf_phdr +#undef elf_shdr +#undef elf_note +#undef elf_addr_t +#undef ELF_GNU_PROPERTY_ALIGN +#define elfhdr elf32_hdr +#define elf_phdr elf32_phdr +#define elf_shdr elf32_shdr +#define elf_note elf32_note +#define elf_addr_t Elf32_Addr +#define ELF_GNU_PROPERTY_ALIGN ELF32_GNU_PROPERTY_ALIGN + +/* + * Some data types as stored in coredump. + */ +#define user_long_t compat_long_t +#define user_siginfo_t compat_siginfo_t +#define copy_siginfo_to_external copy_siginfo_to_external32 + +/* + * The machine-dependent core note format types are defined in elfcore-compat.h, + * which requires asm/elf.h to define compat_elf_gregset_t et al. + */ +#define elf_prstatus compat_elf_prstatus +#define elf_prstatus_common compat_elf_prstatus_common +#define elf_prpsinfo compat_elf_prpsinfo + +#undef ns_to_kernel_old_timeval +#define ns_to_kernel_old_timeval ns_to_old_timeval32 + +/* + * To use this file, asm/elf.h must define compat_elf_check_arch. + * The other following macros can be defined if the compat versions + * differ from the native ones, or omitted when they match. + */ + +#undef elf_check_arch +#define elf_check_arch compat_elf_check_arch + +#ifdef COMPAT_ELF_PLATFORM +#undef ELF_PLATFORM +#define ELF_PLATFORM COMPAT_ELF_PLATFORM +#endif + +#ifdef COMPAT_ELF_HWCAP +#undef ELF_HWCAP +#define ELF_HWCAP COMPAT_ELF_HWCAP +#endif + +#ifdef COMPAT_ELF_HWCAP2 +#undef ELF_HWCAP2 +#define ELF_HWCAP2 COMPAT_ELF_HWCAP2 +#endif + +#ifdef COMPAT_ELF_HWCAP3 +#undef ELF_HWCAP3 +#define ELF_HWCAP3 COMPAT_ELF_HWCAP3 +#endif + +#ifdef COMPAT_ELF_HWCAP4 +#undef ELF_HWCAP4 +#define ELF_HWCAP4 COMPAT_ELF_HWCAP4 +#endif + +#ifdef COMPAT_ARCH_DLINFO +#undef ARCH_DLINFO +#define ARCH_DLINFO COMPAT_ARCH_DLINFO +#endif + +#ifdef COMPAT_ELF_ET_DYN_BASE +#undef ELF_ET_DYN_BASE +#define ELF_ET_DYN_BASE COMPAT_ELF_ET_DYN_BASE +#endif + +#ifdef COMPAT_ELF_PLAT_INIT +#undef ELF_PLAT_INIT +#define ELF_PLAT_INIT COMPAT_ELF_PLAT_INIT +#endif + +#ifdef COMPAT_SET_PERSONALITY +#undef SET_PERSONALITY +#define SET_PERSONALITY COMPAT_SET_PERSONALITY +#endif + +#ifdef compat_start_thread +#define COMPAT_START_THREAD(ex, regs, new_ip, new_sp) \ + compat_start_thread(regs, new_ip, new_sp) +#endif + +#ifdef COMPAT_START_THREAD +#undef START_THREAD +#define START_THREAD COMPAT_START_THREAD +#endif + +#ifdef compat_arch_setup_additional_pages +#define COMPAT_ARCH_SETUP_ADDITIONAL_PAGES(bprm, ex, interpreter) \ + compat_arch_setup_additional_pages(bprm, interpreter) +#endif + +#ifdef COMPAT_ARCH_SETUP_ADDITIONAL_PAGES +#undef ARCH_HAS_SETUP_ADDITIONAL_PAGES +#define ARCH_HAS_SETUP_ADDITIONAL_PAGES 1 +#undef ARCH_SETUP_ADDITIONAL_PAGES +#define ARCH_SETUP_ADDITIONAL_PAGES COMPAT_ARCH_SETUP_ADDITIONAL_PAGES +#endif + +#ifdef compat_elf_read_implies_exec +#undef elf_read_implies_exec +#define elf_read_implies_exec compat_elf_read_implies_exec +#endif + +/* + * Rename a few of the symbols that binfmt_elf.c will define. + * These are all local so the names don't really matter, but it + * might make some debugging less confusing not to duplicate them. + */ +#define elf_format compat_elf_format +#define init_elf_binfmt init_compat_elf_binfmt +#define exit_elf_binfmt exit_compat_elf_binfmt +#define binfmt_elf_test_cases compat_binfmt_elf_test_cases +#define binfmt_elf_test_suite compat_binfmt_elf_test_suite + +/* + * We share all the actual code with the native (64-bit) version. + */ +#include "binfmt_elf.c" diff --git a/fs/configfs/Kconfig b/fs/configfs/Kconfig new file mode 100644 index 000000000..1fcd761fe --- /dev/null +++ b/fs/configfs/Kconfig @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0-only +config CONFIGFS_FS + tristate "Userspace-driven configuration filesystem" + help + configfs is a RAM-based filesystem that provides the converse + of sysfs's functionality. Where sysfs is a filesystem-based + view of kernel objects, configfs is a filesystem-based manager + of kernel objects, or config_items. + + Both sysfs and configfs can and should exist together on the + same system. One is not a replacement for the other. diff --git a/fs/configfs/Makefile b/fs/configfs/Makefile new file mode 100644 index 000000000..0200498ed --- /dev/null +++ b/fs/configfs/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the configfs virtual filesystem +# + +obj-$(CONFIG_CONFIGFS_FS) += configfs.o + +configfs-objs := inode.o file.o dir.o symlink.o mount.o item.o diff --git a/fs/configfs/configfs_internal.h b/fs/configfs/configfs_internal.h new file mode 100644 index 000000000..acdeea8e2 --- /dev/null +++ b/fs/configfs/configfs_internal.h @@ -0,0 +1,158 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * configfs_internal.h - Internal stuff for configfs + * + * Based on sysfs: + * sysfs is Copyright (C) 2001, 2002, 2003 Patrick Mochel + * + * configfs Copyright (C) 2005 Oracle. All rights reserved. + */ + +#ifdef pr_fmt +#undef pr_fmt +#endif + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include + +struct configfs_fragment { + atomic_t frag_count; + struct rw_semaphore frag_sem; + bool frag_dead; +}; + +void put_fragment(struct configfs_fragment *); +struct configfs_fragment *get_fragment(struct configfs_fragment *); + +struct configfs_dirent { + atomic_t s_count; + int s_dependent_count; + struct list_head s_sibling; + struct list_head s_children; + int s_links; + void * s_element; + int s_type; + umode_t s_mode; + struct dentry * s_dentry; + struct iattr * s_iattr; +#ifdef CONFIG_LOCKDEP + int s_depth; +#endif + struct configfs_fragment *s_frag; +}; + +#define CONFIGFS_ROOT 0x0001 +#define CONFIGFS_DIR 0x0002 +#define CONFIGFS_ITEM_ATTR 0x0004 +#define CONFIGFS_ITEM_BIN_ATTR 0x0008 +#define CONFIGFS_ITEM_LINK 0x0020 +#define CONFIGFS_USET_DIR 0x0040 +#define CONFIGFS_USET_DEFAULT 0x0080 +#define CONFIGFS_USET_DROPPING 0x0100 +#define CONFIGFS_USET_IN_MKDIR 0x0200 +#define CONFIGFS_USET_CREATING 0x0400 +#define CONFIGFS_NOT_PINNED (CONFIGFS_ITEM_ATTR | CONFIGFS_ITEM_BIN_ATTR) +#define CONFIGFS_PINNED \ + (CONFIGFS_ROOT | CONFIGFS_DIR | CONFIGFS_ITEM_LINK) + +extern struct mutex configfs_symlink_mutex; +extern spinlock_t configfs_dirent_lock; + +extern struct kmem_cache *configfs_dir_cachep; + +extern int configfs_is_root(struct config_item *item); + +extern struct inode * configfs_new_inode(umode_t mode, struct configfs_dirent *, struct super_block *); +extern struct inode *configfs_create(struct dentry *, umode_t mode); + +extern int configfs_create_file(struct config_item *, const struct configfs_attribute *); +extern int configfs_create_bin_file(struct config_item *, + const struct configfs_bin_attribute *); +extern int configfs_make_dirent(struct configfs_dirent *, struct dentry *, + void *, umode_t, int, struct configfs_fragment *); +extern int configfs_dirent_is_ready(struct configfs_dirent *); + +extern const unsigned char * configfs_get_name(struct configfs_dirent *sd); +extern int configfs_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, struct iattr *iattr); + +extern struct dentry *configfs_pin_fs(void); +extern void configfs_release_fs(void); + +extern const struct file_operations configfs_dir_operations; +extern const struct file_operations configfs_file_operations; +extern const struct file_operations configfs_bin_file_operations; +extern const struct inode_operations configfs_dir_inode_operations; +extern const struct inode_operations configfs_root_inode_operations; +extern const struct inode_operations configfs_symlink_inode_operations; +extern const struct dentry_operations configfs_dentry_ops; + +extern int configfs_symlink(struct mnt_idmap *idmap, + struct inode *dir, struct dentry *dentry, + const char *symname); +extern int configfs_unlink(struct inode *dir, struct dentry *dentry); + +int configfs_create_link(struct configfs_dirent *target, struct dentry *parent, + struct dentry *dentry, char *body); + +static inline struct config_item * to_item(struct dentry * dentry) +{ + struct configfs_dirent * sd = dentry->d_fsdata; + return ((struct config_item *) sd->s_element); +} + +static inline struct configfs_attribute * to_attr(struct dentry * dentry) +{ + struct configfs_dirent * sd = dentry->d_fsdata; + return ((struct configfs_attribute *) sd->s_element); +} + +static inline struct configfs_bin_attribute *to_bin_attr(struct dentry *dentry) +{ + struct configfs_attribute *attr = to_attr(dentry); + + return container_of(attr, struct configfs_bin_attribute, cb_attr); +} + +static inline struct config_item *configfs_get_config_item(struct dentry *dentry) +{ + struct config_item * item = NULL; + + spin_lock(&dentry->d_lock); + if (!d_unhashed(dentry)) { + struct configfs_dirent * sd = dentry->d_fsdata; + item = config_item_get(sd->s_element); + } + spin_unlock(&dentry->d_lock); + + return item; +} + +static inline void release_configfs_dirent(struct configfs_dirent * sd) +{ + if (!(sd->s_type & CONFIGFS_ROOT)) { + kfree(sd->s_iattr); + put_fragment(sd->s_frag); + kmem_cache_free(configfs_dir_cachep, sd); + } +} + +static inline struct configfs_dirent * configfs_get(struct configfs_dirent * sd) +{ + if (sd) { + WARN_ON(!atomic_read(&sd->s_count)); + atomic_inc(&sd->s_count); + } + return sd; +} + +static inline void configfs_put(struct configfs_dirent * sd) +{ + WARN_ON(!atomic_read(&sd->s_count)); + if (atomic_dec_and_test(&sd->s_count)) + release_configfs_dirent(sd); +} + diff --git a/fs/configfs/dir.c b/fs/configfs/dir.c new file mode 100644 index 000000000..3c88f13f1 --- /dev/null +++ b/fs/configfs/dir.c @@ -0,0 +1,1980 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * dir.c - Operations for configfs directories. + * + * Based on sysfs: + * sysfs is Copyright (C) 2001, 2002, 2003 Patrick Mochel + * + * configfs Copyright (C) 2005 Oracle. All rights reserved. + */ + +#undef DEBUG + +#include +#include +#include +#include +#include +#include + +#include +#include "configfs_internal.h" + +/* + * Protects mutations of configfs_dirent linkage together with proper i_mutex + * Also protects mutations of symlinks linkage to target configfs_dirent + * Mutators of configfs_dirent linkage must *both* have the proper inode locked + * and configfs_dirent_lock locked, in that order. + * This allows one to safely traverse configfs_dirent trees and symlinks without + * having to lock inodes. + * + * Protects setting of CONFIGFS_USET_DROPPING: checking the flag + * unlocked is not reliable unless in detach_groups() called from + * rmdir()/unregister() and from configfs_attach_group() + */ +DEFINE_SPINLOCK(configfs_dirent_lock); + +/* + * All of link_obj/unlink_obj/link_group/unlink_group require that + * subsys->su_mutex is held. + * But parent configfs_subsystem is NULL when config_item is root. + * Use this mutex when config_item is root. + */ +static DEFINE_MUTEX(configfs_subsystem_mutex); + +static void configfs_d_iput(struct dentry * dentry, + struct inode * inode) +{ + struct configfs_dirent *sd = dentry->d_fsdata; + + if (sd) { + /* Coordinate with configfs_readdir */ + spin_lock(&configfs_dirent_lock); + /* + * Set sd->s_dentry to null only when this dentry is the one + * that is going to be killed. Otherwise configfs_d_iput may + * run just after configfs_lookup and set sd->s_dentry to + * NULL even it's still in use. + */ + if (sd->s_dentry == dentry) + sd->s_dentry = NULL; + + spin_unlock(&configfs_dirent_lock); + configfs_put(sd); + } + iput(inode); +} + +const struct dentry_operations configfs_dentry_ops = { + .d_iput = configfs_d_iput, +}; + +#ifdef CONFIG_LOCKDEP + +/* + * Helpers to make lockdep happy with our recursive locking of default groups' + * inodes (see configfs_attach_group() and configfs_detach_group()). + * We put default groups i_mutexes in separate classes according to their depth + * from the youngest non-default group ancestor. + * + * For a non-default group A having default groups A/B, A/C, and A/C/D, default + * groups A/B and A/C will have their inode's mutex in class + * default_group_class[0], and default group A/C/D will be in + * default_group_class[1]. + * + * The lock classes are declared and assigned in inode.c, according to the + * s_depth value. + * The s_depth value is initialized to -1, adjusted to >= 0 when attaching + * default groups, and reset to -1 when all default groups are attached. During + * attachment, if configfs_create() sees s_depth > 0, the lock class of the new + * inode's mutex is set to default_group_class[s_depth - 1]. + */ + +static void configfs_init_dirent_depth(struct configfs_dirent *sd) +{ + sd->s_depth = -1; +} + +static void configfs_set_dir_dirent_depth(struct configfs_dirent *parent_sd, + struct configfs_dirent *sd) +{ + int parent_depth = parent_sd->s_depth; + + if (parent_depth >= 0) + sd->s_depth = parent_depth + 1; +} + +static void +configfs_adjust_dir_dirent_depth_before_populate(struct configfs_dirent *sd) +{ + /* + * item's i_mutex class is already setup, so s_depth is now only + * used to set new sub-directories s_depth, which is always done + * with item's i_mutex locked. + */ + /* + * sd->s_depth == -1 iff we are a non default group. + * else (we are a default group) sd->s_depth > 0 (see + * create_dir()). + */ + if (sd->s_depth == -1) + /* + * We are a non default group and we are going to create + * default groups. + */ + sd->s_depth = 0; +} + +static void +configfs_adjust_dir_dirent_depth_after_populate(struct configfs_dirent *sd) +{ + /* We will not create default groups anymore. */ + sd->s_depth = -1; +} + +#else /* CONFIG_LOCKDEP */ + +static void configfs_init_dirent_depth(struct configfs_dirent *sd) +{ +} + +static void configfs_set_dir_dirent_depth(struct configfs_dirent *parent_sd, + struct configfs_dirent *sd) +{ +} + +static void +configfs_adjust_dir_dirent_depth_before_populate(struct configfs_dirent *sd) +{ +} + +static void +configfs_adjust_dir_dirent_depth_after_populate(struct configfs_dirent *sd) +{ +} + +#endif /* CONFIG_LOCKDEP */ + +static struct configfs_fragment *new_fragment(void) +{ + struct configfs_fragment *p; + + p = kmalloc_obj(struct configfs_fragment); + if (p) { + atomic_set(&p->frag_count, 1); + init_rwsem(&p->frag_sem); + p->frag_dead = false; + } + return p; +} + +void put_fragment(struct configfs_fragment *frag) +{ + if (frag && atomic_dec_and_test(&frag->frag_count)) + kfree(frag); +} + +struct configfs_fragment *get_fragment(struct configfs_fragment *frag) +{ + if (likely(frag)) + atomic_inc(&frag->frag_count); + return frag; +} + +/* + * Allocates a new configfs_dirent and links it to the parent configfs_dirent + */ +static struct configfs_dirent *configfs_new_dirent(struct configfs_dirent *parent_sd, + void *element, int type, + struct configfs_fragment *frag) +{ + struct configfs_dirent * sd; + + sd = kmem_cache_zalloc(configfs_dir_cachep, GFP_KERNEL); + if (!sd) + return ERR_PTR(-ENOMEM); + + atomic_set(&sd->s_count, 1); + INIT_LIST_HEAD(&sd->s_children); + sd->s_element = element; + sd->s_type = type; + configfs_init_dirent_depth(sd); + spin_lock(&configfs_dirent_lock); + if (parent_sd->s_type & CONFIGFS_USET_DROPPING) { + spin_unlock(&configfs_dirent_lock); + kmem_cache_free(configfs_dir_cachep, sd); + return ERR_PTR(-ENOENT); + } + sd->s_frag = get_fragment(frag); + + /* + * configfs_lookup scans only for unpinned items. s_children is + * partitioned so that configfs_lookup can bail out early. + * CONFIGFS_PINNED and CONFIGFS_NOT_PINNED are not symmetrical. readdir + * cursors still need to be inserted at the front of the list. + */ + if (sd->s_type & CONFIGFS_PINNED) + list_add_tail(&sd->s_sibling, &parent_sd->s_children); + else + list_add(&sd->s_sibling, &parent_sd->s_children); + spin_unlock(&configfs_dirent_lock); + + return sd; +} + +/* + * + * Return -EEXIST if there is already a configfs element with the same + * name for the same parent. + * + * called with parent inode's i_mutex held + */ +static int configfs_dirent_exists(struct dentry *dentry) +{ + struct configfs_dirent *parent_sd = dentry->d_parent->d_fsdata; + const unsigned char *new = dentry->d_name.name; + struct configfs_dirent *sd; + + spin_lock(&configfs_dirent_lock); + list_for_each_entry(sd, &parent_sd->s_children, s_sibling) { + if (sd->s_element) { + if (strcmp(configfs_get_name(sd), new) == 0) { + spin_unlock(&configfs_dirent_lock); + return -EEXIST; + } + } + } + spin_unlock(&configfs_dirent_lock); + + return 0; +} + + +int configfs_make_dirent(struct configfs_dirent * parent_sd, + struct dentry * dentry, void * element, + umode_t mode, int type, struct configfs_fragment *frag) +{ + struct configfs_dirent * sd; + + sd = configfs_new_dirent(parent_sd, element, type, frag); + if (IS_ERR(sd)) + return PTR_ERR(sd); + + sd->s_mode = mode; + sd->s_dentry = dentry; + if (dentry) + dentry->d_fsdata = configfs_get(sd); + + return 0; +} + +static void configfs_remove_dirent(struct dentry *dentry) +{ + struct configfs_dirent *sd = dentry->d_fsdata; + + if (!sd) + return; + spin_lock(&configfs_dirent_lock); + list_del_init(&sd->s_sibling); + spin_unlock(&configfs_dirent_lock); + configfs_put(sd); +} + +/** + * configfs_create_dir - create a directory for an config_item. + * @item: config_itemwe're creating directory for. + * @dentry: config_item's dentry. + * @frag: config_item's fragment. + * + * Note: user-created entries won't be allowed under this new directory + * until it is validated by configfs_dir_set_ready() + */ + +static int configfs_create_dir(struct config_item *item, struct dentry *dentry, + struct configfs_fragment *frag) +{ + int error; + umode_t mode = S_IFDIR| S_IRWXU | S_IRUGO | S_IXUGO; + struct dentry *p = dentry->d_parent; + struct inode *p_inode = d_inode(p); + struct inode *inode; + + BUG_ON(!item); + + error = configfs_make_dirent(p->d_fsdata, dentry, item, mode, + CONFIGFS_DIR | CONFIGFS_USET_CREATING, + frag); + if (unlikely(error)) + return error; + + configfs_set_dir_dirent_depth(p->d_fsdata, dentry->d_fsdata); + inode = configfs_create(dentry, mode); + if (IS_ERR(inode)) + goto out_remove; + + inode->i_op = &configfs_dir_inode_operations; + inode->i_fop = &configfs_dir_operations; + /* directory inodes start off with i_nlink == 2 (for "." entry) */ + inc_nlink(inode); + d_make_persistent(dentry, inode); + inc_nlink(p_inode); + inode_set_mtime_to_ts(p_inode, inode_set_ctime_current(p_inode)); + item->ci_dentry = dentry; + return 0; + +out_remove: + configfs_put(dentry->d_fsdata); + configfs_remove_dirent(dentry); + return PTR_ERR(inode); +} + +/* + * Allow userspace to create new entries under a new directory created with + * configfs_create_dir(), and under all of its chidlren directories recursively. + * @sd configfs_dirent of the new directory to validate + * + * Caller must hold configfs_dirent_lock. + */ +static void configfs_dir_set_ready(struct configfs_dirent *sd) +{ + struct configfs_dirent *child_sd; + + sd->s_type &= ~CONFIGFS_USET_CREATING; + list_for_each_entry(child_sd, &sd->s_children, s_sibling) + if (child_sd->s_type & CONFIGFS_USET_CREATING) + configfs_dir_set_ready(child_sd); +} + +/* + * Check that a directory does not belong to a directory hierarchy being + * attached and not validated yet. + * @sd configfs_dirent of the directory to check + * + * @return non-zero iff the directory was validated + * + * Note: takes configfs_dirent_lock, so the result may change from false to true + * in two consecutive calls, but never from true to false. + */ +int configfs_dirent_is_ready(struct configfs_dirent *sd) +{ + int ret; + + spin_lock(&configfs_dirent_lock); + ret = !(sd->s_type & CONFIGFS_USET_CREATING); + spin_unlock(&configfs_dirent_lock); + + return ret; +} + +int configfs_create_link(struct configfs_dirent *target, struct dentry *parent, + struct dentry *dentry, char *body) +{ + int err = 0; + umode_t mode = S_IFLNK | S_IRWXUGO; + struct configfs_dirent *p = parent->d_fsdata; + struct inode *p_inode = d_inode(parent); + struct inode *inode; + + err = configfs_make_dirent(p, dentry, target, mode, CONFIGFS_ITEM_LINK, + p->s_frag); + if (err) + return err; + + inode = configfs_create(dentry, mode); + if (IS_ERR(inode)) + goto out_remove; + + inode->i_link = body; + inode->i_op = &configfs_symlink_inode_operations; + d_make_persistent(dentry, inode); + inode_set_mtime_to_ts(p_inode, inode_set_ctime_current(p_inode)); + return 0; + +out_remove: + configfs_put(dentry->d_fsdata); + configfs_remove_dirent(dentry); + return PTR_ERR(inode); +} + +/** + * configfs_remove_dir - remove an config_item's directory. + * @d: dentry we're removing. + * + * The only thing special about this is that we remove any files in + * the directory before we remove the directory, and we've inlined + * what used to be configfs_rmdir() below, instead of calling separately. + * + * Caller holds the mutex of the item's inode + */ + +static void configfs_remove_dir(struct dentry *d) +{ + struct dentry * parent = dget(d->d_parent); + + configfs_remove_dirent(d); + + if (d_really_is_positive(d)) { + if (unlikely(simple_rmdir(d_inode(parent), d))) + pr_warn("remove_dir (%pd): attributes remain", d); + } + + pr_debug(" o %pd removing done (%d)\n", d, d_count(d)); + + dput(parent); +} + +static struct dentry * configfs_lookup(struct inode *dir, + struct dentry *dentry, + unsigned int flags) +{ + struct configfs_dirent * parent_sd = dentry->d_parent->d_fsdata; + struct configfs_dirent * sd; + struct inode *inode = NULL; + + if (dentry->d_name.len > NAME_MAX) + return ERR_PTR(-ENAMETOOLONG); + + /* + * Fake invisibility if dir belongs to a group/default groups hierarchy + * being attached + * + * This forbids userspace to read/write attributes of items which may + * not complete their initialization, since the dentries of the + * attributes won't be instantiated. + */ + if (!configfs_dirent_is_ready(parent_sd)) + return ERR_PTR(-ENOENT); + + spin_lock(&configfs_dirent_lock); + list_for_each_entry(sd, &parent_sd->s_children, s_sibling) { + + /* + * s_children is partitioned, see configfs_new_dirent. The first + * pinned item indicates we can stop scanning. + */ + if (sd->s_type & CONFIGFS_PINNED) + break; + + /* + * Note: CONFIGFS_PINNED and CONFIGFS_NOT_PINNED are asymmetric. + * there may be a readdir cursor in this list + */ + if ((sd->s_type & CONFIGFS_NOT_PINNED) && + !strcmp(configfs_get_name(sd), dentry->d_name.name)) { + struct configfs_attribute *attr = sd->s_element; + umode_t mode = (attr->ca_mode & S_IALLUGO) | S_IFREG; + + dentry->d_fsdata = configfs_get(sd); + sd->s_dentry = dentry; + spin_unlock(&configfs_dirent_lock); + + inode = configfs_create(dentry, mode); + if (IS_ERR(inode)) { + spin_lock(&configfs_dirent_lock); + sd->s_dentry = NULL; + spin_unlock(&configfs_dirent_lock); + configfs_put(sd); + return ERR_CAST(inode); + } + if (sd->s_type & CONFIGFS_ITEM_BIN_ATTR) { + inode->i_size = 0; + inode->i_fop = &configfs_bin_file_operations; + } else { + inode->i_size = PAGE_SIZE; + inode->i_fop = &configfs_file_operations; + } + goto done; + } + } + spin_unlock(&configfs_dirent_lock); +done: + return d_splice_alias(inode, dentry); +} + +/* + * Only subdirectories count here. Files (CONFIGFS_NOT_PINNED) are + * attributes and are removed by rmdir(). We recurse, setting + * CONFIGFS_USET_DROPPING on all children that are candidates for + * default detach. + * If there is an error, the caller will reset the flags via + * configfs_detach_rollback(). + */ +static int configfs_detach_prep(struct configfs_dirent *parent_sd, struct dentry **wait) +{ + struct configfs_dirent *sd; + int ret; + + /* Mark that we're trying to drop the group */ + parent_sd->s_type |= CONFIGFS_USET_DROPPING; + + ret = -EBUSY; + if (parent_sd->s_links) + goto out; + + ret = 0; + list_for_each_entry(sd, &parent_sd->s_children, s_sibling) { + if (!sd->s_element || + (sd->s_type & CONFIGFS_NOT_PINNED)) + continue; + if (sd->s_type & CONFIGFS_USET_DEFAULT) { + /* Abort if racing with mkdir() */ + if (sd->s_type & CONFIGFS_USET_IN_MKDIR) { + if (wait) + *wait= dget(sd->s_dentry); + return -EAGAIN; + } + + /* + * Yup, recursive. If there's a problem, blame + * deep nesting of default_groups + */ + ret = configfs_detach_prep(sd, wait); + if (!ret) + continue; + } else + ret = -ENOTEMPTY; + + break; + } + +out: + return ret; +} + +/* + * Walk the tree, resetting CONFIGFS_USET_DROPPING wherever it was + * set. + */ +static void configfs_detach_rollback(struct configfs_dirent *parent_sd) +{ + struct configfs_dirent *sd; + + parent_sd->s_type &= ~CONFIGFS_USET_DROPPING; + + list_for_each_entry(sd, &parent_sd->s_children, s_sibling) + if (sd->s_type & CONFIGFS_USET_DEFAULT) + configfs_detach_rollback(sd); +} + +/* + * Find the next non-cursor. configfs_dirent_lock held by caller. + */ +static struct configfs_dirent *next_dirent(struct configfs_dirent *parent, + struct configfs_dirent *last) +{ + struct configfs_dirent *s; + + s = list_prepare_entry(last, &parent->s_children, s_sibling); + + list_for_each_entry_continue(s, &parent->s_children, s_sibling) { + if (s->s_element) + return s; + } + return NULL; +} + +static void detach_attrs(struct dentry *dentry) +{ + struct configfs_dirent *parent_sd; + struct configfs_dirent *sd, *next; + + pr_debug("configfs %pd: dropping attrs for dir\n", dentry); + + parent_sd = dentry->d_fsdata; + + spin_lock(&configfs_dirent_lock); + for (sd = next_dirent(parent_sd, NULL); sd; sd = next) { + struct dentry *child; + + next = next_dirent(parent_sd, sd); + if (!(sd->s_type & CONFIGFS_NOT_PINNED)) + continue; + list_del_init(&sd->s_sibling); + child = sd->s_dentry; + if (child) { + spin_lock(&child->d_lock); + if (simple_positive(child)) { + dget_dlock(child); + __d_drop(child); + spin_unlock(&child->d_lock); + } else { + spin_unlock(&child->d_lock); + child = NULL; + } + } + spin_unlock(&configfs_dirent_lock); + if (child) { + struct inode *inode = child->d_inode; + + inode_lock_nested(inode, I_MUTEX_NONDIR2); + inode_set_ctime_current(inode); + drop_nlink(inode); + inode_unlock(inode); + dput(child); + } + configfs_put(sd); + spin_lock(&configfs_dirent_lock); + } + spin_unlock(&configfs_dirent_lock); +} + +static int populate_attrs(struct config_item *item) +{ + const struct config_item_type *t = item->ci_type; + const struct configfs_group_operations *ops; + struct configfs_attribute *attr; + struct configfs_bin_attribute *bin_attr; + int error = 0; + int i; + + if (!t) + return -EINVAL; + + ops = t->ct_group_ops; + + if (t->ct_attrs) { + for (i = 0; (attr = t->ct_attrs[i]) != NULL; i++) { + if (ops && ops->is_visible && !ops->is_visible(item, attr, i)) + continue; + + error = configfs_create_file(item, attr); + if (error) + return error; + } + } + if (t->ct_bin_attrs) { + for (i = 0; (bin_attr = t->ct_bin_attrs[i]) != NULL; i++) { + if (ops && ops->is_bin_visible && !ops->is_bin_visible(item, bin_attr, i)) + continue; + + error = configfs_create_bin_file(item, bin_attr); + if (error) + return error; + } + } + + return 0; +} + +static int configfs_attach_group(struct config_item *item, + struct dentry *dentry, + struct configfs_fragment *frag); +static void configfs_detach_group(struct dentry *dentry); + +static void detach_groups(struct dentry *dentry) +{ + struct dentry *child; + struct configfs_dirent *parent_sd; + struct configfs_dirent *sd, *next; + + parent_sd = dentry->d_fsdata; + spin_lock(&configfs_dirent_lock); + for (sd = next_dirent(parent_sd, NULL); sd; sd = next) { + next = next_dirent(parent_sd, sd); + if (!(sd->s_type & CONFIGFS_USET_DEFAULT)) + continue; + + child = dget(sd->s_dentry); + spin_unlock(&configfs_dirent_lock); + + inode_lock(d_inode(child)); + + configfs_detach_group(child); + d_inode(child)->i_flags |= S_DEAD; + dont_mount(child); + + inode_unlock(d_inode(child)); + + d_delete(child); + dput(child); + spin_lock(&configfs_dirent_lock); + } + spin_unlock(&configfs_dirent_lock); +} + +/* + * This fakes mkdir(2) on a default_groups[] entry. It + * creates a dentry, attachs it, and then does fixup + * on the sd->s_type. + * + * We could, perhaps, tweak our parent's ->mkdir for a minute and + * try using vfs_mkdir. Just a thought. + */ +static int create_default_group(struct dentry *parent, + struct config_group *group, + struct configfs_fragment *frag) +{ + int ret; + struct configfs_dirent *sd; + /* We trust the caller holds a reference to parent */ + struct dentry *child; + + if (!group->cg_item.ci_name) + group->cg_item.ci_name = group->cg_item.ci_namebuf; + + ret = -ENOMEM; + child = d_alloc_name(parent, group->cg_item.ci_name); + if (child) { + d_add(child, NULL); + + ret = configfs_attach_group(&group->cg_item, child, frag); + if (!ret) { + sd = child->d_fsdata; + sd->s_type |= CONFIGFS_USET_DEFAULT; + } else { + BUG_ON(d_inode(child)); + d_drop(child); + } + dput(child); + } + + return ret; +} + +static int populate_groups(struct config_group *group, + struct configfs_fragment *frag) +{ + struct dentry *parent = group->cg_item.ci_dentry; + struct config_group *new_group; + + list_for_each_entry(new_group, &group->default_groups, group_entry) { + int ret = create_default_group(parent, new_group, frag); + if (ret) + return ret; + } + return 0; +} + +void configfs_remove_default_groups(struct config_group *group) +{ + struct config_group *g, *n; + + list_for_each_entry_safe(g, n, &group->default_groups, group_entry) { + list_del(&g->group_entry); + config_item_put(&g->cg_item); + } +} +EXPORT_SYMBOL(configfs_remove_default_groups); + +/* + * All of link_obj/unlink_obj/link_group/unlink_group require that + * subsys->su_mutex is held. + */ + +static void unlink_obj(struct config_item *item) +{ + struct config_group *group; + + group = item->ci_group; + if (group) { + list_del_init(&item->ci_entry); + + item->ci_group = NULL; + item->ci_parent = NULL; + + /* Drop the reference for ci_entry */ + config_item_put(item); + + /* Drop the reference for ci_parent */ + config_group_put(group); + } +} + +static void link_obj(struct config_item *parent_item, struct config_item *item) +{ + /* + * Parent seems redundant with group, but it makes certain + * traversals much nicer. + */ + item->ci_parent = parent_item; + + /* + * We hold a reference on the parent for the child's ci_parent + * link. + */ + item->ci_group = config_group_get(to_config_group(parent_item)); + list_add_tail(&item->ci_entry, &item->ci_group->cg_children); + + /* + * We hold a reference on the child for ci_entry on the parent's + * cg_children + */ + config_item_get(item); +} + +static void unlink_group(struct config_group *group) +{ + struct config_group *new_group; + + list_for_each_entry(new_group, &group->default_groups, group_entry) + unlink_group(new_group); + + group->cg_subsys = NULL; + unlink_obj(&group->cg_item); +} + +static void link_group(struct config_group *parent_group, struct config_group *group) +{ + struct config_group *new_group; + struct configfs_subsystem *subsys = NULL; /* gcc is a turd */ + + link_obj(&parent_group->cg_item, &group->cg_item); + + if (parent_group->cg_subsys) + subsys = parent_group->cg_subsys; + else if (configfs_is_root(&parent_group->cg_item)) + subsys = to_configfs_subsystem(group); + else + BUG(); + group->cg_subsys = subsys; + + list_for_each_entry(new_group, &group->default_groups, group_entry) + link_group(group, new_group); +} + +/* Caller holds the mutex of the item's inode */ +static void configfs_detach_item(struct dentry *dentry) +{ + detach_attrs(dentry); + configfs_remove_dir(dentry); +} + +/* + * The goal is that configfs_attach_item() (and + * configfs_attach_group()) can be called from either the VFS or this + * module. That is, they assume that the items have been created, + * the dentry allocated, and the dcache is all ready to go. + * + * If they fail, they must clean up after themselves as if they + * had never been called. The caller (VFS or local function) will + * handle cleaning up the dcache bits. + * + * configfs_detach_group() and configfs_detach_item() behave similarly on + * the way out. They assume that the proper semaphores are held, they + * clean up the configfs items, and they expect their callers will + * handle the dcache bits. + */ +static int configfs_attach_item(struct config_item *item, + struct dentry *dentry, + struct configfs_fragment *frag) +{ + int ret; + + ret = configfs_create_dir(item, dentry, frag); + if (!ret) { + ret = populate_attrs(item); + if (ret) { + /* + * We are going to remove an inode and its dentry but + * the VFS may already have hit and used them. Thus, + * we must lock them as rmdir() would. + */ + inode_lock(d_inode(dentry)); + configfs_detach_item(dentry); + d_inode(dentry)->i_flags |= S_DEAD; + dont_mount(dentry); + inode_unlock(d_inode(dentry)); + d_delete(dentry); + } + } + + return ret; +} + +/* Caller holds the mutex of the group's inode */ +static void configfs_detach_group(struct dentry *dentry) +{ + detach_groups(dentry); + configfs_detach_item(dentry); +} + +static int configfs_attach_group(struct config_item *item, + struct dentry *dentry, + struct configfs_fragment *frag) +{ + int ret; + struct configfs_dirent *sd; + + ret = configfs_attach_item(item, dentry, frag); + if (!ret) { + sd = dentry->d_fsdata; + sd->s_type |= CONFIGFS_USET_DIR; + + /* + * FYI, we're faking mkdir in populate_groups() + * We must lock the group's inode to avoid races with the VFS + * which can already hit the inode and try to add/remove entries + * under it. + * + * We must also lock the inode to remove it safely in case of + * error, as rmdir() would. + */ + inode_lock_nested(d_inode(dentry), I_MUTEX_CHILD); + configfs_adjust_dir_dirent_depth_before_populate(sd); + ret = populate_groups(to_config_group(item), frag); + if (ret) { + configfs_detach_group(dentry); + d_inode(dentry)->i_flags |= S_DEAD; + dont_mount(dentry); + } + configfs_adjust_dir_dirent_depth_after_populate(sd); + inode_unlock(d_inode(dentry)); + if (ret) + d_delete(dentry); + } + + return ret; +} + +/* + * After the item has been detached from the filesystem view, we are + * ready to tear it out of the hierarchy. Notify the client before + * we do that so they can perform any cleanup that requires + * navigating the hierarchy. A client does not need to provide this + * callback. The subsystem semaphore MUST be held by the caller, and + * references must be valid for both items. It also assumes the + * caller has validated ci_type. + */ +static void client_disconnect_notify(struct config_item *parent_item, + struct config_item *item) +{ + const struct config_item_type *type; + + type = parent_item->ci_type; + BUG_ON(!type); + + if (type->ct_group_ops && type->ct_group_ops->disconnect_notify) + type->ct_group_ops->disconnect_notify(to_config_group(parent_item), + item); +} + +/* + * Drop the initial reference from make_item()/make_group() + * This function assumes that reference is held on item + * and that item holds a valid reference to the parent. Also, it + * assumes the caller has validated ci_type. + */ +static void client_drop_item(struct config_item *parent_item, + struct config_item *item) +{ + const struct config_item_type *type; + + type = parent_item->ci_type; + BUG_ON(!type); + + /* + * If ->drop_item() exists, it is responsible for the + * config_item_put(). + */ + if (type->ct_group_ops && type->ct_group_ops->drop_item) + type->ct_group_ops->drop_item(to_config_group(parent_item), + item); + else + config_item_put(item); +} + +#ifdef DEBUG +static void configfs_dump_one(struct configfs_dirent *sd, int level) +{ + pr_info("%*s\"%s\":\n", level, " ", configfs_get_name(sd)); + +#define type_print(_type) if (sd->s_type & _type) pr_info("%*s %s\n", level, " ", #_type) + type_print(CONFIGFS_ROOT); + type_print(CONFIGFS_DIR); + type_print(CONFIGFS_ITEM_ATTR); + type_print(CONFIGFS_ITEM_LINK); + type_print(CONFIGFS_USET_DIR); + type_print(CONFIGFS_USET_DEFAULT); + type_print(CONFIGFS_USET_DROPPING); +#undef type_print +} + +static int configfs_dump(struct configfs_dirent *sd, int level) +{ + struct configfs_dirent *child_sd; + int ret = 0; + + configfs_dump_one(sd, level); + + if (!(sd->s_type & (CONFIGFS_DIR|CONFIGFS_ROOT))) + return 0; + + list_for_each_entry(child_sd, &sd->s_children, s_sibling) { + ret = configfs_dump(child_sd, level + 2); + if (ret) + break; + } + + return ret; +} +#endif + + +/* + * configfs_depend_item() and configfs_undepend_item() + * + * WARNING: Do not call these from a configfs callback! + * + * This describes these functions and their helpers. + * + * Allow another kernel system to depend on a config_item. If this + * happens, the item cannot go away until the dependent can live without + * it. The idea is to give client modules as simple an interface as + * possible. When a system asks them to depend on an item, they just + * call configfs_depend_item(). If the item is live and the client + * driver is in good shape, we'll happily do the work for them. + * + * Why is the locking complex? Because configfs uses the VFS to handle + * all locking, but this function is called outside the normal + * VFS->configfs path. So it must take VFS locks to prevent the + * VFS->configfs stuff (configfs_mkdir(), configfs_rmdir(), etc). This is + * why you can't call these functions underneath configfs callbacks. + * + * Note, btw, that this can be called at *any* time, even when a configfs + * subsystem isn't registered, or when configfs is loading or unloading. + * Just like configfs_register_subsystem(). So we take the same + * precautions. We pin the filesystem. We lock configfs_dirent_lock. + * If we can find the target item in the + * configfs tree, it must be part of the subsystem tree as well, so we + * do not need the subsystem semaphore. Holding configfs_dirent_lock helps + * locking out mkdir() and rmdir(), who might be racing us. + */ + +/* + * configfs_depend_prep() + * + * Only subdirectories count here. Files (CONFIGFS_NOT_PINNED) are + * attributes. This is similar but not the same to configfs_detach_prep(). + * Note that configfs_detach_prep() expects the parent to be locked when it + * is called, but we lock the parent *inside* configfs_depend_prep(). We + * do that so we can unlock it if we find nothing. + * + * Here we do a depth-first search of the dentry hierarchy looking for + * our object. + * We deliberately ignore items tagged as dropping since they are virtually + * dead, as well as items in the middle of attachment since they virtually + * do not exist yet. This completes the locking out of racing mkdir() and + * rmdir(). + * Note: subdirectories in the middle of attachment start with s_type = + * CONFIGFS_DIR|CONFIGFS_USET_CREATING set by create_dir(). When + * CONFIGFS_USET_CREATING is set, we ignore the item. The actual set of + * s_type is in configfs_new_dirent(), which has configfs_dirent_lock. + * + * If the target is not found, -ENOENT is bubbled up. + * + * This adds a requirement that all config_items be unique! + * + * This is recursive. There isn't + * much on the stack, though, so folks that need this function - be careful + * about your stack! Patches will be accepted to make it iterative. + */ +static int configfs_depend_prep(struct configfs_dirent *sd, + struct config_item *target) +{ + struct configfs_dirent *child_sd; + int ret = 0; + + if (sd->s_element == target) /* Boo-yah */ + goto out; + + list_for_each_entry(child_sd, &sd->s_children, s_sibling) { + if ((child_sd->s_type & CONFIGFS_DIR) && + !(child_sd->s_type & CONFIGFS_USET_DROPPING) && + !(child_sd->s_type & CONFIGFS_USET_CREATING)) { + ret = configfs_depend_prep(child_sd, target); + if (!ret) + goto out; /* Child path boo-yah */ + } + } + + /* We looped all our children and didn't find target */ + ret = -ENOENT; + +out: + return ret; +} + +static int configfs_do_depend_item(struct configfs_dirent *subsys_sd, + struct config_item *target) +{ + struct configfs_dirent *p; + int ret; + + spin_lock(&configfs_dirent_lock); + /* Scan the tree, return 0 if found */ + ret = configfs_depend_prep(subsys_sd, target); + if (ret) + goto out_unlock_dirent_lock; + + /* + * We are sure that the item is not about to be removed by rmdir(), and + * not in the middle of attachment by mkdir(). + */ + p = target->ci_dentry->d_fsdata; + p->s_dependent_count += 1; + +out_unlock_dirent_lock: + spin_unlock(&configfs_dirent_lock); + + return ret; +} + +static inline struct configfs_dirent * +configfs_find_subsys_dentry(struct configfs_dirent *root_sd, + struct config_item *subsys_item) +{ + struct configfs_dirent *p; + struct configfs_dirent *ret = NULL; + + spin_lock(&configfs_dirent_lock); + list_for_each_entry(p, &root_sd->s_children, s_sibling) { + if (p->s_type & CONFIGFS_DIR && + p->s_element == subsys_item) { + ret = p; + break; + } + } + spin_unlock(&configfs_dirent_lock); + + return ret; +} + + +int configfs_depend_item(struct configfs_subsystem *subsys, + struct config_item *target) +{ + int ret; + struct configfs_dirent *subsys_sd; + struct config_item *s_item = &subsys->su_group.cg_item; + struct dentry *root; + + /* + * Pin the configfs filesystem. This means we can safely access + * the root of the configfs filesystem. + */ + root = configfs_pin_fs(); + if (IS_ERR(root)) + return PTR_ERR(root); + + /* + * Next, lock the root directory. We're going to check that the + * subsystem is really registered, and so we need to lock out + * configfs_[un]register_subsystem(). + */ + inode_lock(d_inode(root)); + + subsys_sd = configfs_find_subsys_dentry(root->d_fsdata, s_item); + if (!subsys_sd) { + ret = -ENOENT; + goto out_unlock_fs; + } + + /* Ok, now we can trust subsys/s_item */ + ret = configfs_do_depend_item(subsys_sd, target); + +out_unlock_fs: + inode_unlock(d_inode(root)); + + /* + * If we succeeded, the fs is pinned via other methods. If not, + * we're done with it anyway. So release_fs() is always right. + */ + configfs_release_fs(); + + return ret; +} +EXPORT_SYMBOL(configfs_depend_item); + +/* + * Release the dependent linkage. This is much simpler than + * configfs_depend_item() because we know that the client driver is + * pinned, thus the subsystem is pinned, and therefore configfs is pinned. + */ +void configfs_undepend_item(struct config_item *target) +{ + struct configfs_dirent *sd; + + /* + * Since we can trust everything is pinned, we just need + * configfs_dirent_lock. + */ + spin_lock(&configfs_dirent_lock); + + sd = target->ci_dentry->d_fsdata; + BUG_ON(sd->s_dependent_count < 1); + + sd->s_dependent_count -= 1; + + /* + * After this unlock, we cannot trust the item to stay alive! + * DO NOT REFERENCE item after this unlock. + */ + spin_unlock(&configfs_dirent_lock); +} +EXPORT_SYMBOL(configfs_undepend_item); + +/* + * caller_subsys is a caller's subsystem not target's. This is used to + * determine if we should lock root and check subsys or not. When we are + * in the same subsystem as our target there is no need to do locking as + * we know that subsys is valid and is not unregistered during this function + * as we are called from callback of one of his children and VFS holds a lock + * on some inode. Otherwise we have to lock our root to ensure that target's + * subsystem it is not unregistered during this function. + */ +int configfs_depend_item_unlocked(struct configfs_subsystem *caller_subsys, + struct config_item *target) +{ + struct configfs_subsystem *target_subsys; + struct config_group *root, *parent; + struct configfs_dirent *subsys_sd; + int ret = -ENOENT; + + /* Disallow this function for configfs root */ + if (configfs_is_root(target)) + return -EINVAL; + + parent = target->ci_group; + /* + * This may happen when someone is trying to depend root + * directory of some subsystem + */ + if (configfs_is_root(&parent->cg_item)) { + target_subsys = to_configfs_subsystem(to_config_group(target)); + root = parent; + } else { + target_subsys = parent->cg_subsys; + /* Find a cofnigfs root as we may need it for locking */ + for (root = parent; !configfs_is_root(&root->cg_item); + root = root->cg_item.ci_group) + ; + } + + if (target_subsys != caller_subsys) { + /* + * We are in other configfs subsystem, so we have to do + * additional locking to prevent other subsystem from being + * unregistered + */ + inode_lock(d_inode(root->cg_item.ci_dentry)); + + /* + * As we are trying to depend item from other subsystem + * we have to check if this subsystem is still registered + */ + subsys_sd = configfs_find_subsys_dentry( + root->cg_item.ci_dentry->d_fsdata, + &target_subsys->su_group.cg_item); + if (!subsys_sd) + goto out_root_unlock; + } else { + subsys_sd = target_subsys->su_group.cg_item.ci_dentry->d_fsdata; + } + + /* Now we can execute core of depend item */ + ret = configfs_do_depend_item(subsys_sd, target); + + if (target_subsys != caller_subsys) +out_root_unlock: + /* + * We were called from subsystem other than our target so we + * took some locks so now it's time to release them + */ + inode_unlock(d_inode(root->cg_item.ci_dentry)); + + return ret; +} +EXPORT_SYMBOL(configfs_depend_item_unlocked); + +static struct dentry *configfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + int ret = 0; + int module_got = 0; + struct config_group *group = NULL; + struct config_item *item = NULL; + struct config_item *parent_item; + struct configfs_subsystem *subsys; + struct configfs_dirent *sd; + const struct config_item_type *type; + struct module *subsys_owner = NULL, *new_item_owner = NULL; + struct configfs_fragment *frag; + struct name_snapshot n; + const char *name; + + take_dentry_name_snapshot(&n, dentry); + name = n.name.name; + + sd = dentry->d_parent->d_fsdata; + + /* + * Fake invisibility if dir belongs to a group/default groups hierarchy + * being attached + */ + if (!configfs_dirent_is_ready(sd)) { + ret = -ENOENT; + goto out; + } + + if (!(sd->s_type & CONFIGFS_USET_DIR)) { + ret = -EPERM; + goto out; + } + + frag = new_fragment(); + if (!frag) { + ret = -ENOMEM; + goto out; + } + + /* Get a working ref for the duration of this function */ + parent_item = configfs_get_config_item(dentry->d_parent); + type = parent_item->ci_type; + subsys = to_config_group(parent_item)->cg_subsys; + BUG_ON(!subsys); + + if (!type || !type->ct_group_ops || + (!type->ct_group_ops->make_group && + !type->ct_group_ops->make_item)) { + ret = -EPERM; /* Lack-of-mkdir returns -EPERM */ + goto out_put; + } + + /* + * The subsystem may belong to a different module than the item + * being created. We don't want to safely pin the new item but + * fail to pin the subsystem it sits under. + */ + if (!subsys->su_group.cg_item.ci_type) { + ret = -EINVAL; + goto out_put; + } + subsys_owner = subsys->su_group.cg_item.ci_type->ct_owner; + if (!try_module_get(subsys_owner)) { + ret = -EINVAL; + goto out_put; + } + + mutex_lock(&subsys->su_mutex); + if (type->ct_group_ops->make_group) { + group = type->ct_group_ops->make_group(to_config_group(parent_item), name); + if (!group) + group = ERR_PTR(-ENOMEM); + if (!IS_ERR(group)) { + link_group(to_config_group(parent_item), group); + item = &group->cg_item; + } else + ret = PTR_ERR(group); + } else { + item = type->ct_group_ops->make_item(to_config_group(parent_item), name); + if (!item) + item = ERR_PTR(-ENOMEM); + if (!IS_ERR(item)) + link_obj(parent_item, item); + else + ret = PTR_ERR(item); + } + mutex_unlock(&subsys->su_mutex); + + if (ret) { + /* + * If ret != 0, then link_obj() was never called. + * There are no extra references to clean up. + */ + goto out_subsys_put; + } + + /* + * link_obj() has been called (via link_group() for groups). + * From here on out, errors must clean that up. + */ + + type = item->ci_type; + if (!type) { + ret = -EINVAL; + goto out_unlink; + } + + new_item_owner = type->ct_owner; + if (!try_module_get(new_item_owner)) { + ret = -EINVAL; + goto out_unlink; + } + + /* + * I hate doing it this way, but if there is + * an error, module_put() probably should + * happen after any cleanup. + */ + module_got = 1; + + /* + * Make racing rmdir() fail if it did not tag parent with + * CONFIGFS_USET_DROPPING + * Note: if CONFIGFS_USET_DROPPING is already set, attach_group() will + * fail and let rmdir() terminate correctly + */ + spin_lock(&configfs_dirent_lock); + /* This will make configfs_detach_prep() fail */ + sd->s_type |= CONFIGFS_USET_IN_MKDIR; + spin_unlock(&configfs_dirent_lock); + + if (group) + ret = configfs_attach_group(item, dentry, frag); + else + ret = configfs_attach_item(item, dentry, frag); + + spin_lock(&configfs_dirent_lock); + sd->s_type &= ~CONFIGFS_USET_IN_MKDIR; + if (!ret) + configfs_dir_set_ready(dentry->d_fsdata); + spin_unlock(&configfs_dirent_lock); + +out_unlink: + if (ret) { + /* Tear down everything we built up */ + mutex_lock(&subsys->su_mutex); + + client_disconnect_notify(parent_item, item); + if (group) + unlink_group(group); + else + unlink_obj(item); + client_drop_item(parent_item, item); + + mutex_unlock(&subsys->su_mutex); + + if (module_got) + module_put(new_item_owner); + } + +out_subsys_put: + if (ret) + module_put(subsys_owner); + +out_put: + /* + * link_obj()/link_group() took a reference from child->parent, + * so the parent is safely pinned. We can drop our working + * reference. + */ + config_item_put(parent_item); + put_fragment(frag); + +out: + release_dentry_name_snapshot(&n); + return ERR_PTR(ret); +} + +static int configfs_rmdir(struct inode *dir, struct dentry *dentry) +{ + struct config_item *parent_item; + struct config_item *item; + struct configfs_subsystem *subsys; + struct configfs_dirent *sd; + struct configfs_fragment *frag; + struct module *subsys_owner = NULL, *dead_item_owner = NULL; + int ret; + + sd = dentry->d_fsdata; + if (sd->s_type & CONFIGFS_USET_DEFAULT) + return -EPERM; + + /* Get a working ref until we have the child */ + parent_item = configfs_get_config_item(dentry->d_parent); + subsys = to_config_group(parent_item)->cg_subsys; + BUG_ON(!subsys); + + if (!parent_item->ci_type) { + config_item_put(parent_item); + return -EINVAL; + } + + /* configfs_mkdir() shouldn't have allowed this */ + BUG_ON(!subsys->su_group.cg_item.ci_type); + subsys_owner = subsys->su_group.cg_item.ci_type->ct_owner; + + /* + * Ensure that no racing symlink() will make detach_prep() fail while + * the new link is temporarily attached + */ + do { + struct dentry *wait; + + mutex_lock(&configfs_symlink_mutex); + spin_lock(&configfs_dirent_lock); + /* + * Here's where we check for dependents. We're protected by + * configfs_dirent_lock. + * If no dependent, atomically tag the item as dropping. + */ + ret = sd->s_dependent_count ? -EBUSY : 0; + if (!ret) { + ret = configfs_detach_prep(sd, &wait); + if (ret) + configfs_detach_rollback(sd); + } + spin_unlock(&configfs_dirent_lock); + mutex_unlock(&configfs_symlink_mutex); + + if (ret) { + if (ret != -EAGAIN) { + config_item_put(parent_item); + return ret; + } + + /* Wait until the racing operation terminates */ + inode_lock(d_inode(wait)); + inode_unlock(d_inode(wait)); + dput(wait); + } + } while (ret == -EAGAIN); + + frag = sd->s_frag; + if (down_write_killable(&frag->frag_sem)) { + spin_lock(&configfs_dirent_lock); + configfs_detach_rollback(sd); + spin_unlock(&configfs_dirent_lock); + config_item_put(parent_item); + return -EINTR; + } + frag->frag_dead = true; + up_write(&frag->frag_sem); + + /* Get a working ref for the duration of this function */ + item = configfs_get_config_item(dentry); + + /* Drop reference from above, item already holds one. */ + config_item_put(parent_item); + + if (item->ci_type) + dead_item_owner = item->ci_type->ct_owner; + + if (sd->s_type & CONFIGFS_USET_DIR) { + configfs_detach_group(dentry); + + mutex_lock(&subsys->su_mutex); + client_disconnect_notify(parent_item, item); + unlink_group(to_config_group(item)); + } else { + configfs_detach_item(dentry); + + mutex_lock(&subsys->su_mutex); + client_disconnect_notify(parent_item, item); + unlink_obj(item); + } + + client_drop_item(parent_item, item); + mutex_unlock(&subsys->su_mutex); + + /* Drop our reference from above */ + config_item_put(item); + + module_put(dead_item_owner); + module_put(subsys_owner); + + return 0; +} + +const struct inode_operations configfs_dir_inode_operations = { + .mkdir = configfs_mkdir, + .rmdir = configfs_rmdir, + .symlink = configfs_symlink, + .unlink = configfs_unlink, + .lookup = configfs_lookup, + .setattr = configfs_setattr, +}; + +const struct inode_operations configfs_root_inode_operations = { + .lookup = configfs_lookup, + .setattr = configfs_setattr, +}; + +static int configfs_dir_open(struct inode *inode, struct file *file) +{ + struct dentry * dentry = file->f_path.dentry; + struct configfs_dirent * parent_sd = dentry->d_fsdata; + int err; + + inode_lock(d_inode(dentry)); + /* + * Fake invisibility if dir belongs to a group/default groups hierarchy + * being attached + */ + err = -ENOENT; + if (configfs_dirent_is_ready(parent_sd)) { + file->private_data = configfs_new_dirent(parent_sd, NULL, 0, NULL); + err = PTR_ERR_OR_ZERO(file->private_data); + } + inode_unlock(d_inode(dentry)); + + return err; +} + +static int configfs_dir_close(struct inode *inode, struct file *file) +{ + struct dentry * dentry = file->f_path.dentry; + struct configfs_dirent * cursor = file->private_data; + + inode_lock(d_inode(dentry)); + spin_lock(&configfs_dirent_lock); + list_del_init(&cursor->s_sibling); + spin_unlock(&configfs_dirent_lock); + inode_unlock(d_inode(dentry)); + + release_configfs_dirent(cursor); + + return 0; +} + +static int configfs_readdir(struct file *file, struct dir_context *ctx) +{ + struct dentry *dentry = file->f_path.dentry; + struct super_block *sb = dentry->d_sb; + struct configfs_dirent * parent_sd = dentry->d_fsdata; + struct configfs_dirent *cursor = file->private_data; + struct list_head *p, *q = &cursor->s_sibling; + ino_t ino = 0; + + if (!dir_emit_dots(file, ctx)) + return 0; + spin_lock(&configfs_dirent_lock); + if (ctx->pos == 2) + list_move(q, &parent_sd->s_children); + for (p = q->next; p != &parent_sd->s_children; p = p->next) { + struct configfs_dirent *next; + const char *name; + int len; + struct inode *inode = NULL; + + next = list_entry(p, struct configfs_dirent, s_sibling); + if (!next->s_element) + continue; + + /* + * We'll have a dentry and an inode for + * PINNED items and for open attribute + * files. We lock here to prevent a race + * with configfs_d_iput() clearing + * s_dentry before calling iput(). + * + * Why do we go to the trouble? If + * someone has an attribute file open, + * the inode number should match until + * they close it. Beyond that, we don't + * care. + */ + dentry = next->s_dentry; + if (dentry) + inode = d_inode(dentry); + if (inode) + ino = inode->i_ino; + spin_unlock(&configfs_dirent_lock); + if (!inode) + ino = iunique(sb, 2); + + name = configfs_get_name(next); + len = strlen(name); + + if (!dir_emit(ctx, name, len, ino, + fs_umode_to_dtype(next->s_mode))) + return 0; + + spin_lock(&configfs_dirent_lock); + list_move(q, p); + p = q; + ctx->pos++; + } + spin_unlock(&configfs_dirent_lock); + return 0; +} + +static loff_t configfs_dir_lseek(struct file *file, loff_t offset, int whence) +{ + struct dentry * dentry = file->f_path.dentry; + + switch (whence) { + case 1: + offset += file->f_pos; + fallthrough; + case 0: + if (offset >= 0) + break; + fallthrough; + default: + return -EINVAL; + } + if (offset != file->f_pos) { + file->f_pos = offset; + if (file->f_pos >= 2) { + struct configfs_dirent *sd = dentry->d_fsdata; + struct configfs_dirent *cursor = file->private_data; + struct list_head *p; + loff_t n = file->f_pos - 2; + + spin_lock(&configfs_dirent_lock); + list_del(&cursor->s_sibling); + p = sd->s_children.next; + while (n && p != &sd->s_children) { + struct configfs_dirent *next; + next = list_entry(p, struct configfs_dirent, + s_sibling); + if (next->s_element) + n--; + p = p->next; + } + list_add_tail(&cursor->s_sibling, p); + spin_unlock(&configfs_dirent_lock); + } + } + return offset; +} + +const struct file_operations configfs_dir_operations = { + .open = configfs_dir_open, + .release = configfs_dir_close, + .llseek = configfs_dir_lseek, + .read = generic_read_dir, + .iterate_shared = configfs_readdir, +}; + +/** + * configfs_register_group - creates a parent-child relation between two groups + * @parent_group: parent group + * @group: child group + * + * link groups, creates dentry for the child and attaches it to the + * parent dentry. + * + * Return: 0 on success, negative errno code on error + */ +int configfs_register_group(struct config_group *parent_group, + struct config_group *group) +{ + struct configfs_subsystem *subsys = parent_group->cg_subsys; + struct dentry *parent; + struct configfs_fragment *frag; + int ret; + + frag = new_fragment(); + if (!frag) + return -ENOMEM; + + mutex_lock(&subsys->su_mutex); + link_group(parent_group, group); + mutex_unlock(&subsys->su_mutex); + + parent = parent_group->cg_item.ci_dentry; + + inode_lock_nested(d_inode(parent), I_MUTEX_PARENT); + ret = create_default_group(parent, group, frag); + if (ret) + goto err_out; + + spin_lock(&configfs_dirent_lock); + configfs_dir_set_ready(group->cg_item.ci_dentry->d_fsdata); + spin_unlock(&configfs_dirent_lock); + inode_unlock(d_inode(parent)); + put_fragment(frag); + return 0; +err_out: + inode_unlock(d_inode(parent)); + mutex_lock(&subsys->su_mutex); + unlink_group(group); + mutex_unlock(&subsys->su_mutex); + put_fragment(frag); + return ret; +} +EXPORT_SYMBOL(configfs_register_group); + +/** + * configfs_unregister_group() - unregisters a child group from its parent + * @group: parent group to be unregistered + * + * Undoes configfs_register_group() + */ +void configfs_unregister_group(struct config_group *group) +{ + struct configfs_subsystem *subsys = group->cg_subsys; + struct dentry *dentry = dget(group->cg_item.ci_dentry); + struct dentry *parent = group->cg_item.ci_parent->ci_dentry; + struct configfs_dirent *sd = dentry->d_fsdata; + struct configfs_fragment *frag = sd->s_frag; + + down_write(&frag->frag_sem); + frag->frag_dead = true; + up_write(&frag->frag_sem); + + inode_lock_nested(d_inode(parent), I_MUTEX_PARENT); + spin_lock(&configfs_dirent_lock); + configfs_detach_prep(sd, NULL); + spin_unlock(&configfs_dirent_lock); + + configfs_detach_group(dentry); + d_inode(dentry)->i_flags |= S_DEAD; + dont_mount(dentry); + d_drop(dentry); + fsnotify_rmdir(d_inode(parent), dentry); + inode_unlock(d_inode(parent)); + + dput(dentry); + + mutex_lock(&subsys->su_mutex); + unlink_group(group); + mutex_unlock(&subsys->su_mutex); +} +EXPORT_SYMBOL(configfs_unregister_group); + +/** + * configfs_register_default_group() - allocates and registers a child group + * @parent_group: parent group + * @name: child group name + * @item_type: child item type description + * + * boilerplate to allocate and register a child group with its parent. We need + * kzalloc'ed memory because child's default_group is initially empty. + * + * Return: allocated config group or ERR_PTR() on error + */ +struct config_group * +configfs_register_default_group(struct config_group *parent_group, + const char *name, + const struct config_item_type *item_type) +{ + int ret; + struct config_group *group; + + group = kzalloc_obj(*group); + if (!group) + return ERR_PTR(-ENOMEM); + config_group_init_type_name(group, name, item_type); + + ret = configfs_register_group(parent_group, group); + if (ret) { + kfree(group); + return ERR_PTR(ret); + } + return group; +} +EXPORT_SYMBOL(configfs_register_default_group); + +/** + * configfs_unregister_default_group() - unregisters and frees a child group + * @group: the group to act on + */ +void configfs_unregister_default_group(struct config_group *group) +{ + configfs_unregister_group(group); + kfree(group); +} +EXPORT_SYMBOL(configfs_unregister_default_group); + +int configfs_register_subsystem(struct configfs_subsystem *subsys) +{ + int err; + struct config_group *group = &subsys->su_group; + struct dentry *dentry; + struct dentry *root; + struct configfs_dirent *sd; + struct configfs_fragment *frag; + + frag = new_fragment(); + if (!frag) + return -ENOMEM; + + root = configfs_pin_fs(); + if (IS_ERR(root)) { + put_fragment(frag); + return PTR_ERR(root); + } + + if (!group->cg_item.ci_name) + group->cg_item.ci_name = group->cg_item.ci_namebuf; + + sd = root->d_fsdata; + mutex_lock(&configfs_subsystem_mutex); + link_group(to_config_group(sd->s_element), group); + mutex_unlock(&configfs_subsystem_mutex); + + inode_lock_nested(d_inode(root), I_MUTEX_PARENT); + + err = -ENOMEM; + dentry = d_alloc_name(root, group->cg_item.ci_name); + if (dentry) { + d_add(dentry, NULL); + + err = configfs_dirent_exists(dentry); + if (!err) + err = configfs_attach_group(&group->cg_item, + dentry, frag); + if (err) { + BUG_ON(d_inode(dentry)); + d_drop(dentry); + } else { + spin_lock(&configfs_dirent_lock); + configfs_dir_set_ready(dentry->d_fsdata); + spin_unlock(&configfs_dirent_lock); + } + dput(dentry); + } + + inode_unlock(d_inode(root)); + + if (err) { + mutex_lock(&configfs_subsystem_mutex); + unlink_group(group); + mutex_unlock(&configfs_subsystem_mutex); + configfs_release_fs(); + } + put_fragment(frag); + + return err; +} + +void configfs_unregister_subsystem(struct configfs_subsystem *subsys) +{ + struct config_group *group = &subsys->su_group; + struct dentry *dentry = dget(group->cg_item.ci_dentry); + struct dentry *root = dentry->d_sb->s_root; + struct configfs_dirent *sd = dentry->d_fsdata; + struct configfs_fragment *frag = sd->s_frag; + + if (dentry->d_parent != root) { + pr_err("Tried to unregister non-subsystem!\n"); + return; + } + + down_write(&frag->frag_sem); + frag->frag_dead = true; + up_write(&frag->frag_sem); + + inode_lock_nested(d_inode(root), + I_MUTEX_PARENT); + inode_lock_nested(d_inode(dentry), I_MUTEX_CHILD); + mutex_lock(&configfs_symlink_mutex); + spin_lock(&configfs_dirent_lock); + if (configfs_detach_prep(sd, NULL)) { + pr_err("Tried to unregister non-empty subsystem!\n"); + } + spin_unlock(&configfs_dirent_lock); + mutex_unlock(&configfs_symlink_mutex); + configfs_detach_group(dentry); + d_inode(dentry)->i_flags |= S_DEAD; + dont_mount(dentry); + inode_unlock(d_inode(dentry)); + + d_drop(dentry); + fsnotify_rmdir(d_inode(root), dentry); + + inode_unlock(d_inode(root)); + + dput(dentry); + + mutex_lock(&configfs_subsystem_mutex); + unlink_group(group); + mutex_unlock(&configfs_subsystem_mutex); + configfs_release_fs(); +} + +EXPORT_SYMBOL(configfs_register_subsystem); +EXPORT_SYMBOL(configfs_unregister_subsystem); diff --git a/fs/configfs/file.c b/fs/configfs/file.c new file mode 100644 index 000000000..a48cece77 --- /dev/null +++ b/fs/configfs/file.c @@ -0,0 +1,481 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * file.c - operations for regular (text) files. + * + * Based on sysfs: + * sysfs is Copyright (C) 2001, 2002, 2003 Patrick Mochel + * + * configfs Copyright (C) 2005 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "configfs_internal.h" + +/* + * A simple attribute can only be 4096 characters. Why 4k? Because the + * original code limited it to PAGE_SIZE. That's a bad idea, though, + * because an attribute of 16k on ia64 won't work on x86. So we limit to + * 4k, our minimum common page size. + */ +#define SIMPLE_ATTR_SIZE 4096 + +struct configfs_buffer { + size_t count; + loff_t pos; + char * page; + const struct configfs_item_operations *ops; + struct mutex mutex; + int needs_read_fill; + bool read_in_progress; + bool write_in_progress; + char *bin_buffer; + int bin_buffer_size; + int cb_max_size; + struct config_item *item; + struct module *owner; + union { + struct configfs_attribute *attr; + struct configfs_bin_attribute *bin_attr; + }; +}; + +static inline struct configfs_fragment *to_frag(struct file *file) +{ + struct configfs_dirent *sd = file->f_path.dentry->d_fsdata; + + return sd->s_frag; +} + +static int fill_read_buffer(struct file *file, struct configfs_buffer *buffer) +{ + struct configfs_fragment *frag = to_frag(file); + ssize_t count = -ENOENT; + + if (!buffer->page) + buffer->page = kzalloc(PAGE_SIZE, GFP_KERNEL); + if (!buffer->page) + return -ENOMEM; + + down_read(&frag->frag_sem); + if (!frag->frag_dead) + count = buffer->attr->show(buffer->item, buffer->page); + up_read(&frag->frag_sem); + + if (count < 0) + return count; + if (WARN_ON_ONCE(count > (ssize_t)SIMPLE_ATTR_SIZE)) + return -EIO; + buffer->needs_read_fill = 0; + buffer->count = count; + return 0; +} + +static ssize_t configfs_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct file *file = iocb->ki_filp; + struct configfs_buffer *buffer = file->private_data; + ssize_t retval = 0; + + mutex_lock(&buffer->mutex); + if (buffer->needs_read_fill) { + retval = fill_read_buffer(file, buffer); + if (retval) + goto out; + } + pr_debug("%s: count = %zd, pos = %lld, buf = %s\n", + __func__, iov_iter_count(to), iocb->ki_pos, buffer->page); + if (iocb->ki_pos >= buffer->count) + goto out; + retval = copy_to_iter(buffer->page + iocb->ki_pos, + buffer->count - iocb->ki_pos, to); + iocb->ki_pos += retval; + if (retval == 0) + retval = -EFAULT; +out: + mutex_unlock(&buffer->mutex); + return retval; +} + +static ssize_t configfs_bin_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct file *file = iocb->ki_filp; + struct configfs_fragment *frag = to_frag(file); + struct configfs_buffer *buffer = file->private_data; + ssize_t retval = 0; + ssize_t len; + + mutex_lock(&buffer->mutex); + + /* we don't support switching read/write modes */ + if (buffer->write_in_progress) { + retval = -ETXTBSY; + goto out; + } + buffer->read_in_progress = true; + + if (buffer->needs_read_fill) { + /* perform first read with buf == NULL to get extent */ + down_read(&frag->frag_sem); + if (!frag->frag_dead) + len = buffer->bin_attr->read(buffer->item, NULL, 0); + else + len = -ENOENT; + up_read(&frag->frag_sem); + if (len <= 0) { + retval = len; + goto out; + } + + /* do not exceed the maximum value */ + if (buffer->cb_max_size && len > buffer->cb_max_size) { + retval = -EFBIG; + goto out; + } + + buffer->bin_buffer = vmalloc(len); + if (buffer->bin_buffer == NULL) { + retval = -ENOMEM; + goto out; + } + buffer->bin_buffer_size = len; + + /* perform second read to fill buffer */ + down_read(&frag->frag_sem); + if (!frag->frag_dead) + len = buffer->bin_attr->read(buffer->item, + buffer->bin_buffer, len); + else + len = -ENOENT; + up_read(&frag->frag_sem); + if (len < 0) { + retval = len; + vfree(buffer->bin_buffer); + buffer->bin_buffer_size = 0; + buffer->bin_buffer = NULL; + goto out; + } + + buffer->needs_read_fill = 0; + } + + if (iocb->ki_pos >= buffer->bin_buffer_size) + goto out; + retval = copy_to_iter(buffer->bin_buffer + iocb->ki_pos, + buffer->bin_buffer_size - iocb->ki_pos, to); + iocb->ki_pos += retval; + if (retval == 0) + retval = -EFAULT; +out: + mutex_unlock(&buffer->mutex); + return retval; +} + +/* Fill @buffer with data coming from @from. */ +static int fill_write_buffer(struct configfs_buffer *buffer, + struct iov_iter *from) +{ + int copied; + + if (!buffer->page) + buffer->page = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!buffer->page) + return -ENOMEM; + + copied = copy_from_iter(buffer->page, SIMPLE_ATTR_SIZE - 1, from); + buffer->needs_read_fill = 1; + /* if buf is assumed to contain a string, terminate it by \0, + * so e.g. sscanf() can scan the string easily */ + buffer->page[copied] = 0; + return copied ? : -EFAULT; +} + +static int +flush_write_buffer(struct file *file, struct configfs_buffer *buffer, size_t count) +{ + struct configfs_fragment *frag = to_frag(file); + int res = -ENOENT; + + down_read(&frag->frag_sem); + if (!frag->frag_dead) + res = buffer->attr->store(buffer->item, buffer->page, count); + up_read(&frag->frag_sem); + return res; +} + + +/* + * There is no easy way for us to know if userspace is only doing a partial + * write, so we don't support them. We expect the entire buffer to come on the + * first write. + * Hint: if you're writing a value, first read the file, modify only the value + * you're changing, then write entire buffer back. + */ +static ssize_t configfs_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct configfs_buffer *buffer = file->private_data; + int len; + + mutex_lock(&buffer->mutex); + len = fill_write_buffer(buffer, from); + if (len > 0) + len = flush_write_buffer(file, buffer, len); + if (len > 0) + iocb->ki_pos += len; + mutex_unlock(&buffer->mutex); + return len; +} + +static ssize_t configfs_bin_write_iter(struct kiocb *iocb, + struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct configfs_buffer *buffer = file->private_data; + void *tbuf = NULL; + size_t end_offset; + ssize_t len; + + mutex_lock(&buffer->mutex); + + /* we don't support switching read/write modes */ + if (buffer->read_in_progress) { + len = -ETXTBSY; + goto out; + } + buffer->write_in_progress = true; + + /* buffer grows? */ + end_offset = iocb->ki_pos + iov_iter_count(from); + if (end_offset > buffer->bin_buffer_size) { + if (buffer->cb_max_size && end_offset > buffer->cb_max_size) { + len = -EFBIG; + goto out; + } + + tbuf = vmalloc(end_offset); + if (tbuf == NULL) { + len = -ENOMEM; + goto out; + } + + /* copy old contents */ + if (buffer->bin_buffer) { + memcpy(tbuf, buffer->bin_buffer, + buffer->bin_buffer_size); + vfree(buffer->bin_buffer); + } + + /* clear the new area */ + memset(tbuf + buffer->bin_buffer_size, 0, + end_offset - buffer->bin_buffer_size); + buffer->bin_buffer = tbuf; + buffer->bin_buffer_size = end_offset; + } + + len = copy_from_iter(buffer->bin_buffer + iocb->ki_pos, + buffer->bin_buffer_size - iocb->ki_pos, from); + iocb->ki_pos += len; +out: + mutex_unlock(&buffer->mutex); + return len ? : -EFAULT; +} + +static int __configfs_open_file(struct inode *inode, struct file *file, int type) +{ + struct dentry *dentry = file->f_path.dentry; + struct configfs_fragment *frag = to_frag(file); + struct configfs_attribute *attr; + struct configfs_buffer *buffer; + int error; + + error = -ENOMEM; + buffer = kzalloc_obj(struct configfs_buffer); + if (!buffer) + goto out; + + error = -ENOENT; + down_read(&frag->frag_sem); + if (unlikely(frag->frag_dead)) + goto out_free_buffer; + + error = -EINVAL; + buffer->item = to_item(dentry->d_parent); + if (!buffer->item) + goto out_free_buffer; + + attr = to_attr(dentry); + if (!attr) + goto out_free_buffer; + + if (type & CONFIGFS_ITEM_BIN_ATTR) { + buffer->bin_attr = to_bin_attr(dentry); + buffer->cb_max_size = buffer->bin_attr->cb_max_size; + } else { + buffer->attr = attr; + } + + buffer->owner = attr->ca_owner; + /* Grab the module reference for this attribute if we have one */ + error = -ENODEV; + if (!try_module_get(buffer->owner)) + goto out_free_buffer; + + error = -EACCES; + if (!buffer->item->ci_type) + goto out_put_module; + + buffer->ops = buffer->item->ci_type->ct_item_ops; + + /* File needs write support. + * The inode's perms must say it's ok, + * and we must have a store method. + */ + if (file->f_mode & FMODE_WRITE) { + if (!(inode->i_mode & S_IWUGO)) + goto out_put_module; + if ((type & CONFIGFS_ITEM_ATTR) && !attr->store) + goto out_put_module; + if ((type & CONFIGFS_ITEM_BIN_ATTR) && !buffer->bin_attr->write) + goto out_put_module; + } + + /* File needs read support. + * The inode's perms must say it's ok, and we there + * must be a show method for it. + */ + if (file->f_mode & FMODE_READ) { + if (!(inode->i_mode & S_IRUGO)) + goto out_put_module; + if ((type & CONFIGFS_ITEM_ATTR) && !attr->show) + goto out_put_module; + if ((type & CONFIGFS_ITEM_BIN_ATTR) && !buffer->bin_attr->read) + goto out_put_module; + } + + mutex_init(&buffer->mutex); + buffer->needs_read_fill = 1; + buffer->read_in_progress = false; + buffer->write_in_progress = false; + file->private_data = buffer; + up_read(&frag->frag_sem); + return 0; + +out_put_module: + module_put(buffer->owner); +out_free_buffer: + up_read(&frag->frag_sem); + kfree(buffer); +out: + return error; +} + +static int configfs_release(struct inode *inode, struct file *filp) +{ + struct configfs_buffer *buffer = filp->private_data; + + module_put(buffer->owner); + kfree(buffer->page); + mutex_destroy(&buffer->mutex); + kfree(buffer); + return 0; +} + +static int configfs_open_file(struct inode *inode, struct file *filp) +{ + return __configfs_open_file(inode, filp, CONFIGFS_ITEM_ATTR); +} + +static int configfs_open_bin_file(struct inode *inode, struct file *filp) +{ + return __configfs_open_file(inode, filp, CONFIGFS_ITEM_BIN_ATTR); +} + +static int configfs_release_bin_file(struct inode *inode, struct file *file) +{ + struct configfs_buffer *buffer = file->private_data; + + if (buffer->write_in_progress) { + struct configfs_fragment *frag = to_frag(file); + + down_read(&frag->frag_sem); + if (!frag->frag_dead) { + /* result of ->release() is ignored */ + buffer->bin_attr->write(buffer->item, + buffer->bin_buffer, + buffer->bin_buffer_size); + } + up_read(&frag->frag_sem); + } + + vfree(buffer->bin_buffer); + + configfs_release(inode, file); + return 0; +} + + +const struct file_operations configfs_file_operations = { + .read_iter = configfs_read_iter, + .write_iter = configfs_write_iter, + .llseek = generic_file_llseek, + .open = configfs_open_file, + .release = configfs_release, +}; + +const struct file_operations configfs_bin_file_operations = { + .read_iter = configfs_bin_read_iter, + .write_iter = configfs_bin_write_iter, + .llseek = NULL, /* bin file is not seekable */ + .open = configfs_open_bin_file, + .release = configfs_release_bin_file, +}; + +/** + * configfs_create_file - create an attribute file for an item. + * @item: item we're creating for. + * @attr: atrribute descriptor. + */ + +int configfs_create_file(struct config_item * item, const struct configfs_attribute * attr) +{ + struct dentry *dir = item->ci_dentry; + struct configfs_dirent *parent_sd = dir->d_fsdata; + umode_t mode = (attr->ca_mode & S_IALLUGO) | S_IFREG; + int error = 0; + + inode_lock_nested(d_inode(dir), I_MUTEX_NORMAL); + error = configfs_make_dirent(parent_sd, NULL, (void *) attr, mode, + CONFIGFS_ITEM_ATTR, parent_sd->s_frag); + inode_unlock(d_inode(dir)); + + return error; +} + +/** + * configfs_create_bin_file - create a binary attribute file for an item. + * @item: item we're creating for. + * @bin_attr: atrribute descriptor. + */ + +int configfs_create_bin_file(struct config_item *item, + const struct configfs_bin_attribute *bin_attr) +{ + struct dentry *dir = item->ci_dentry; + struct configfs_dirent *parent_sd = dir->d_fsdata; + umode_t mode = (bin_attr->cb_attr.ca_mode & S_IALLUGO) | S_IFREG; + int error = 0; + + inode_lock_nested(dir->d_inode, I_MUTEX_NORMAL); + error = configfs_make_dirent(parent_sd, NULL, (void *) bin_attr, mode, + CONFIGFS_ITEM_BIN_ATTR, parent_sd->s_frag); + inode_unlock(dir->d_inode); + + return error; +} diff --git a/fs/configfs/inode.c b/fs/configfs/inode.c new file mode 100644 index 000000000..68290fe0e --- /dev/null +++ b/fs/configfs/inode.c @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * inode.c - basic inode and dentry operations. + * + * Based on sysfs: + * sysfs is Copyright (C) 2001, 2002, 2003 Patrick Mochel + * + * configfs Copyright (C) 2005 Oracle. All rights reserved. + * + * Please see Documentation/filesystems/configfs.rst for more + * information. + */ + +#undef DEBUG + +#include +#include +#include +#include +#include +#include +#include + +#include +#include "configfs_internal.h" + +#ifdef CONFIG_LOCKDEP +static struct lock_class_key default_group_class[MAX_LOCK_DEPTH]; +#endif + +static const struct inode_operations configfs_inode_operations ={ + .setattr = configfs_setattr, +}; + +int configfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *iattr) +{ + struct inode * inode = d_inode(dentry); + struct configfs_dirent * sd = dentry->d_fsdata; + struct iattr * sd_iattr; + unsigned int ia_valid = iattr->ia_valid; + int error; + + if (!sd) + return -EINVAL; + + sd_iattr = sd->s_iattr; + if (!sd_iattr) { + /* setting attributes for the first time, allocate now */ + sd_iattr = kzalloc_obj(struct iattr); + if (!sd_iattr) + return -ENOMEM; + /* assign default attributes */ + sd_iattr->ia_mode = sd->s_mode; + sd_iattr->ia_uid = GLOBAL_ROOT_UID; + sd_iattr->ia_gid = GLOBAL_ROOT_GID; + sd_iattr->ia_atime = sd_iattr->ia_mtime = + sd_iattr->ia_ctime = current_time(inode); + sd->s_iattr = sd_iattr; + } + /* attributes were changed atleast once in past */ + + error = simple_setattr(idmap, dentry, iattr); + if (error) + return error; + + if (ia_valid & ATTR_UID) + sd_iattr->ia_uid = iattr->ia_uid; + if (ia_valid & ATTR_GID) + sd_iattr->ia_gid = iattr->ia_gid; + if (ia_valid & ATTR_ATIME) + sd_iattr->ia_atime = iattr->ia_atime; + if (ia_valid & ATTR_MTIME) + sd_iattr->ia_mtime = iattr->ia_mtime; + if (ia_valid & ATTR_CTIME) + sd_iattr->ia_ctime = iattr->ia_ctime; + if (ia_valid & ATTR_MODE) { + umode_t mode = iattr->ia_mode; + + if (!in_group_p(inode->i_gid) && !capable(CAP_FSETID)) + mode &= ~S_ISGID; + sd_iattr->ia_mode = sd->s_mode = mode; + } + + return error; +} + +static inline void set_default_inode_attr(struct inode * inode, umode_t mode) +{ + inode->i_mode = mode; + simple_inode_init_ts(inode); +} + +static inline void set_inode_attr(struct inode * inode, struct iattr * iattr) +{ + inode->i_mode = iattr->ia_mode; + inode->i_uid = iattr->ia_uid; + inode->i_gid = iattr->ia_gid; + inode_set_atime_to_ts(inode, iattr->ia_atime); + inode_set_mtime_to_ts(inode, iattr->ia_mtime); + inode_set_ctime_to_ts(inode, iattr->ia_ctime); +} + +struct inode *configfs_new_inode(umode_t mode, struct configfs_dirent *sd, + struct super_block *s) +{ + struct inode * inode = new_inode(s); + if (inode) { + inode->i_ino = get_next_ino(); + inode->i_mapping->a_ops = &ram_aops; + inode->i_op = &configfs_inode_operations; + + if (sd->s_iattr) { + /* sysfs_dirent has non-default attributes + * get them for the new inode from persistent copy + * in sysfs_dirent + */ + set_inode_attr(inode, sd->s_iattr); + } else + set_default_inode_attr(inode, mode); + } + return inode; +} + +#ifdef CONFIG_LOCKDEP + +static void configfs_set_inode_lock_class(struct configfs_dirent *sd, + struct inode *inode) +{ + int depth = sd->s_depth; + + if (depth > 0) { + if (depth <= ARRAY_SIZE(default_group_class)) { + lockdep_set_class(&inode->i_rwsem, + &default_group_class[depth - 1]); + } else { + /* + * In practice the maximum level of locking depth is + * already reached. Just inform about possible reasons. + */ + pr_info("Too many levels of inodes for the locking correctness validator.\n"); + pr_info("Spurious warnings may appear.\n"); + } + } +} + +#else /* CONFIG_LOCKDEP */ + +static void configfs_set_inode_lock_class(struct configfs_dirent *sd, + struct inode *inode) +{ +} + +#endif /* CONFIG_LOCKDEP */ + +struct inode *configfs_create(struct dentry *dentry, umode_t mode) +{ + struct inode *inode = NULL; + struct configfs_dirent *sd; + + if (!dentry) + return ERR_PTR(-ENOENT); + + if (d_really_is_positive(dentry)) + return ERR_PTR(-EEXIST); + + sd = dentry->d_fsdata; + inode = configfs_new_inode(mode, sd, dentry->d_sb); + if (!inode) + return ERR_PTR(-ENOMEM); + + configfs_set_inode_lock_class(sd, inode); + return inode; +} + +/* + * Get the name for corresponding element represented by the given configfs_dirent + */ +const unsigned char * configfs_get_name(struct configfs_dirent *sd) +{ + struct configfs_attribute *attr; + + BUG_ON(!sd || !sd->s_element); + + /* These always have a dentry, so use that */ + if (sd->s_type & (CONFIGFS_DIR | CONFIGFS_ITEM_LINK)) + return sd->s_dentry->d_name.name; + + if (sd->s_type & (CONFIGFS_ITEM_ATTR | CONFIGFS_ITEM_BIN_ATTR)) { + attr = sd->s_element; + return attr->ca_name; + } + return NULL; +} diff --git a/fs/configfs/item.c b/fs/configfs/item.c new file mode 100644 index 000000000..c378b5cbf --- /dev/null +++ b/fs/configfs/item.c @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * item.c - library routines for handling generic config items + * + * Based on kobject: + * kobject is Copyright (c) 2002-2003 Patrick Mochel + * + * configfs Copyright (C) 2005 Oracle. All rights reserved. + * + * Please see the file Documentation/filesystems/configfs.rst for + * critical information about using the config_item interface. + */ + +#include +#include +#include +#include + +#include + + +static inline struct config_item *to_item(struct list_head *entry) +{ + return container_of(entry, struct config_item, ci_entry); +} + +/* Evil kernel */ +static void config_item_release(struct kref *kref); + +/** + * config_item_init - initialize item. + * @item: item in question. + */ +static void config_item_init(struct config_item *item) +{ + kref_init(&item->ci_kref); + INIT_LIST_HEAD(&item->ci_entry); +} + +/** + * config_item_set_name - Set the name of an item + * @item: item. + * @fmt: The vsnprintf()'s format string. + * + * If strlen(name) >= CONFIGFS_ITEM_NAME_LEN, then use a + * dynamically allocated string that @item->ci_name points to. + * Otherwise, use the static @item->ci_namebuf array. + */ +int config_item_set_name(struct config_item *item, const char *fmt, ...) +{ + int limit = CONFIGFS_ITEM_NAME_LEN; + int need; + va_list args; + char *name; + + /* + * First, try the static array + */ + va_start(args, fmt); + need = vsnprintf(item->ci_namebuf, limit, fmt, args); + va_end(args); + if (need < limit) + name = item->ci_namebuf; + else { + va_start(args, fmt); + name = kvasprintf(GFP_KERNEL, fmt, args); + va_end(args); + if (!name) + return -ENOMEM; + } + + /* Free the old name, if necessary. */ + if (item->ci_name && item->ci_name != item->ci_namebuf) + kfree(item->ci_name); + + /* Now, set the new name */ + item->ci_name = name; + return 0; +} +EXPORT_SYMBOL(config_item_set_name); + +void config_item_init_type_name(struct config_item *item, + const char *name, + const struct config_item_type *type) +{ + config_item_set_name(item, "%s", name); + item->ci_type = type; + config_item_init(item); +} +EXPORT_SYMBOL(config_item_init_type_name); + +void config_group_init_type_name(struct config_group *group, const char *name, + const struct config_item_type *type) +{ + config_item_set_name(&group->cg_item, "%s", name); + group->cg_item.ci_type = type; + config_group_init(group); +} +EXPORT_SYMBOL(config_group_init_type_name); + +struct config_item *config_item_get(struct config_item *item) +{ + if (item) + kref_get(&item->ci_kref); + return item; +} +EXPORT_SYMBOL(config_item_get); + +struct config_item *config_item_get_unless_zero(struct config_item *item) +{ + if (item && kref_get_unless_zero(&item->ci_kref)) + return item; + return NULL; +} +EXPORT_SYMBOL(config_item_get_unless_zero); + +static void config_item_cleanup(struct config_item *item) +{ + const struct config_item_type *t = item->ci_type; + struct config_group *s = item->ci_group; + struct config_item *parent = item->ci_parent; + + pr_debug("config_item %s: cleaning up\n", config_item_name(item)); + if (item->ci_name != item->ci_namebuf) + kfree(item->ci_name); + item->ci_name = NULL; + if (t && t->ct_item_ops && t->ct_item_ops->release) + t->ct_item_ops->release(item); + if (s) + config_group_put(s); + if (parent) + config_item_put(parent); +} + +static void config_item_release(struct kref *kref) +{ + config_item_cleanup(container_of(kref, struct config_item, ci_kref)); +} + +/** + * config_item_put - decrement refcount for item. + * @item: item. + * + * Decrement the refcount, and if 0, call config_item_cleanup(). + */ +void config_item_put(struct config_item *item) +{ + if (item) + kref_put(&item->ci_kref, config_item_release); +} +EXPORT_SYMBOL(config_item_put); + +/** + * config_group_init - initialize a group for use + * @group: config_group + */ +void config_group_init(struct config_group *group) +{ + config_item_init(&group->cg_item); + INIT_LIST_HEAD(&group->cg_children); + INIT_LIST_HEAD(&group->default_groups); +} +EXPORT_SYMBOL(config_group_init); + +/** + * config_group_find_item - search for item in group. + * @group: group we're looking in. + * @name: item's name. + * + * Iterate over @group->cg_list, looking for a matching config_item. + * If matching item is found take a reference and return the item. + * Caller must have locked group via @group->cg_subsys->su_mtx. + */ +struct config_item *config_group_find_item(struct config_group *group, + const char *name) +{ + struct list_head *entry; + struct config_item *ret = NULL; + + list_for_each(entry, &group->cg_children) { + struct config_item *item = to_item(entry); + if (config_item_name(item) && + !strcmp(config_item_name(item), name)) { + ret = config_item_get(item); + break; + } + } + return ret; +} +EXPORT_SYMBOL(config_group_find_item); diff --git a/fs/configfs/mount.c b/fs/configfs/mount.c new file mode 100644 index 000000000..4929f3431 --- /dev/null +++ b/fs/configfs/mount.c @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * mount.c - operations for initializing and mounting configfs. + * + * Based on sysfs: + * sysfs is Copyright (C) 2001, 2002, 2003 Patrick Mochel + * + * configfs Copyright (C) 2005 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include "configfs_internal.h" + +/* Random magic number */ +#define CONFIGFS_MAGIC 0x62656570 + +static struct vfsmount *configfs_mount = NULL; +struct kmem_cache *configfs_dir_cachep; +static int configfs_mnt_count = 0; + + +static void configfs_free_inode(struct inode *inode) +{ + if (S_ISLNK(inode->i_mode)) + kfree(inode->i_link); + free_inode_nonrcu(inode); +} + +static const struct super_operations configfs_ops = { + .statfs = simple_statfs, + .drop_inode = inode_just_drop, + .free_inode = configfs_free_inode, +}; + +static struct config_group configfs_root_group = { + .cg_item = { + .ci_namebuf = "root", + .ci_name = configfs_root_group.cg_item.ci_namebuf, + }, +}; + +int configfs_is_root(struct config_item *item) +{ + return item == &configfs_root_group.cg_item; +} + +static struct configfs_dirent configfs_root = { + .s_sibling = LIST_HEAD_INIT(configfs_root.s_sibling), + .s_children = LIST_HEAD_INIT(configfs_root.s_children), + .s_element = &configfs_root_group.cg_item, + .s_type = CONFIGFS_ROOT, + .s_iattr = NULL, +}; + +static int configfs_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct inode *inode; + struct dentry *root; + + sb->s_blocksize = PAGE_SIZE; + sb->s_blocksize_bits = PAGE_SHIFT; + sb->s_magic = CONFIGFS_MAGIC; + sb->s_op = &configfs_ops; + sb->s_time_gran = 1; + + inode = configfs_new_inode(S_IFDIR | S_IRWXU | S_IRUGO | S_IXUGO, + &configfs_root, sb); + if (inode) { + inode->i_op = &configfs_root_inode_operations; + inode->i_fop = &configfs_dir_operations; + /* directory inodes start off with i_nlink == 2 (for "." entry) */ + inc_nlink(inode); + } else { + pr_debug("could not get root inode\n"); + return -ENOMEM; + } + + root = d_make_root(inode); + if (!root) { + pr_debug("%s: could not get root dentry!\n",__func__); + return -ENOMEM; + } + config_group_init(&configfs_root_group); + configfs_root_group.cg_item.ci_dentry = root; + root->d_fsdata = &configfs_root; + sb->s_root = root; + set_default_d_op(sb, &configfs_dentry_ops); /* the rest get that */ + sb->s_d_flags |= DCACHE_DONTCACHE; + return 0; +} + +static int configfs_get_tree(struct fs_context *fc) +{ + return get_tree_single(fc, configfs_fill_super); +} + +static const struct fs_context_operations configfs_context_ops = { + .get_tree = configfs_get_tree, +}; + +static int configfs_init_fs_context(struct fs_context *fc) +{ + fc->ops = &configfs_context_ops; + return 0; +} + +static struct file_system_type configfs_fs_type = { + .owner = THIS_MODULE, + .name = "configfs", + .init_fs_context = configfs_init_fs_context, + .kill_sb = kill_anon_super, +}; +MODULE_ALIAS_FS("configfs"); + +struct dentry *configfs_pin_fs(void) +{ + int err = simple_pin_fs(&configfs_fs_type, &configfs_mount, + &configfs_mnt_count); + return err ? ERR_PTR(err) : configfs_mount->mnt_root; +} + +void configfs_release_fs(void) +{ + simple_release_fs(&configfs_mount, &configfs_mnt_count); +} + + +static int __init configfs_init(void) +{ + int err = -ENOMEM; + + configfs_dir_cachep = kmem_cache_create("configfs_dir_cache", + sizeof(struct configfs_dirent), + 0, 0, NULL); + if (!configfs_dir_cachep) + goto out; + + err = sysfs_create_mount_point(kernel_kobj, "config"); + if (err) + goto out2; + + err = register_filesystem(&configfs_fs_type); + if (err) + goto out3; + + return 0; +out3: + pr_err("Unable to register filesystem!\n"); + sysfs_remove_mount_point(kernel_kobj, "config"); +out2: + kmem_cache_destroy(configfs_dir_cachep); + configfs_dir_cachep = NULL; +out: + return err; +} + +static void __exit configfs_exit(void) +{ + unregister_filesystem(&configfs_fs_type); + sysfs_remove_mount_point(kernel_kobj, "config"); + kmem_cache_destroy(configfs_dir_cachep); + configfs_dir_cachep = NULL; +} + +MODULE_AUTHOR("Oracle"); +MODULE_LICENSE("GPL"); +MODULE_VERSION("0.0.2"); +MODULE_DESCRIPTION("Simple RAM filesystem for user driven kernel subsystem configuration."); + +core_initcall(configfs_init); +module_exit(configfs_exit); diff --git a/fs/configfs/symlink.c b/fs/configfs/symlink.c new file mode 100644 index 000000000..31eb28b27 --- /dev/null +++ b/fs/configfs/symlink.c @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * symlink.c - operations for configfs symlinks. + * + * Based on sysfs: + * sysfs is Copyright (C) 2001, 2002, 2003 Patrick Mochel + * + * configfs Copyright (C) 2005 Oracle. All rights reserved. + */ + +#include +#include +#include +#include + +#include +#include "configfs_internal.h" + +/* Protects attachments of new symlinks */ +DEFINE_MUTEX(configfs_symlink_mutex); + +static int item_depth(struct config_item * item) +{ + struct config_item * p = item; + int depth = 0; + do { depth++; } while ((p = p->ci_parent) && !configfs_is_root(p)); + return depth; +} + +static int item_path_length(struct config_item * item) +{ + struct config_item * p = item; + int length = 1; + do { + length += strlen(config_item_name(p)) + 1; + p = p->ci_parent; + } while (p && !configfs_is_root(p)); + return length; +} + +static void fill_item_path(struct config_item * item, char * buffer, int length) +{ + struct config_item * p; + + --length; + for (p = item; p && !configfs_is_root(p); p = p->ci_parent) { + int cur = strlen(config_item_name(p)); + + /* back up enough to print this bus id with '/' */ + length -= cur; + memcpy(buffer + length, config_item_name(p), cur); + *(buffer + --length) = '/'; + } +} + +static int configfs_get_target_path(struct config_item *item, + struct config_item *target, char *path) +{ + int depth, size; + char *s; + + depth = item_depth(item); + size = item_path_length(target) + depth * 3 - 1; + if (size > PATH_MAX) + return -ENAMETOOLONG; + + pr_debug("%s: depth = %d, size = %d\n", __func__, depth, size); + + for (s = path; depth--; s += 3) + strcpy(s,"../"); + + fill_item_path(target, path, size); + pr_debug("%s: path = '%s'\n", __func__, path); + return 0; +} + +static int create_link(struct config_item *parent_item, + struct config_item *item, + struct dentry *dentry) +{ + struct configfs_dirent *target_sd = item->ci_dentry->d_fsdata; + char *body; + int ret; + + if (!configfs_dirent_is_ready(target_sd)) + return -ENOENT; + + body = kzalloc(PAGE_SIZE, GFP_KERNEL); + if (!body) + return -ENOMEM; + + configfs_get(target_sd); + spin_lock(&configfs_dirent_lock); + if (target_sd->s_type & CONFIGFS_USET_DROPPING) { + spin_unlock(&configfs_dirent_lock); + configfs_put(target_sd); + kfree(body); + return -ENOENT; + } + target_sd->s_links++; + spin_unlock(&configfs_dirent_lock); + ret = configfs_get_target_path(parent_item, item, body); + if (!ret) + ret = configfs_create_link(target_sd, parent_item->ci_dentry, + dentry, body); + if (ret) { + spin_lock(&configfs_dirent_lock); + target_sd->s_links--; + spin_unlock(&configfs_dirent_lock); + configfs_put(target_sd); + kfree(body); + } + return ret; +} + + +static int get_target(const char *symname, struct config_item **target, + struct super_block *sb) +{ + struct path path __free(path_put) = {}; + int ret; + + ret = kern_path(symname, LOOKUP_FOLLOW|LOOKUP_DIRECTORY, &path); + if (ret) + return ret; + if (path.dentry->d_sb != sb) + return -EPERM; + *target = configfs_get_config_item(path.dentry); + if (!*target) + return -ENOENT; + return 0; +} + + +int configfs_symlink(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const char *symname) +{ + int ret; + struct configfs_dirent *sd; + struct config_item *parent_item; + struct config_item *target_item = NULL; + const struct config_item_type *type; + + sd = dentry->d_parent->d_fsdata; + /* + * Fake invisibility if dir belongs to a group/default groups hierarchy + * being attached + */ + if (!configfs_dirent_is_ready(sd)) + return -ENOENT; + + parent_item = configfs_get_config_item(dentry->d_parent); + type = parent_item->ci_type; + + ret = -EPERM; + if (!type || !type->ct_item_ops || + !type->ct_item_ops->allow_link) + goto out_put; + + /* + * This is really sick. What they wanted was a hybrid of + * link(2) and symlink(2) - they wanted the target resolved + * at syscall time (as link(2) would've done), be a directory + * (which link(2) would've refused to do) *AND* be a deep + * fucking magic, making the target busy from rmdir POV. + * symlink(2) is nothing of that sort, and the locking it + * gets matches the normal symlink(2) semantics. Without + * attempts to resolve the target (which might very well + * not even exist yet) done prior to locking the parent + * directory. This perversion, OTOH, needs to resolve + * the target, which would lead to obvious deadlocks if + * attempted with any directories locked. + * + * Unfortunately, that garbage is userland ABI and we should've + * said "no" back in 2005. Too late now, so we get to + * play very ugly games with locking. + * + * Try *ANYTHING* of that sort in new code, and you will + * really regret it. Just ask yourself - what could a BOFH + * do to me and do I want to find it out first-hand? + * + * AV, a thoroughly annoyed bastard. + */ + inode_unlock(dir); + ret = get_target(symname, &target_item, dentry->d_sb); + inode_lock(dir); + if (ret) + goto out_put; + + if (dentry->d_inode || d_unhashed(dentry)) + ret = -EEXIST; + else + ret = inode_permission(&nop_mnt_idmap, dir, + MAY_WRITE | MAY_EXEC); + if (!ret) + ret = type->ct_item_ops->allow_link(parent_item, target_item); + if (!ret) { + mutex_lock(&configfs_symlink_mutex); + ret = create_link(parent_item, target_item, dentry); + mutex_unlock(&configfs_symlink_mutex); + if (ret && type->ct_item_ops->drop_link) + type->ct_item_ops->drop_link(parent_item, + target_item); + } + + config_item_put(target_item); + +out_put: + config_item_put(parent_item); + return ret; +} + +int configfs_unlink(struct inode *dir, struct dentry *dentry) +{ + struct configfs_dirent *sd = dentry->d_fsdata, *target_sd; + struct config_item *parent_item; + const struct config_item_type *type; + int ret; + + ret = -EPERM; /* What lack-of-symlink returns */ + if (!(sd->s_type & CONFIGFS_ITEM_LINK)) + goto out; + + target_sd = sd->s_element; + + parent_item = configfs_get_config_item(dentry->d_parent); + type = parent_item->ci_type; + + spin_lock(&configfs_dirent_lock); + list_del_init(&sd->s_sibling); + spin_unlock(&configfs_dirent_lock); + configfs_put(sd); + simple_unlink(dir, dentry); + + /* + * drop_link() must be called before + * decrementing target's ->s_links, so that the order of + * drop_link(this, target) and drop_item(target) is preserved. + */ + if (type && type->ct_item_ops && + type->ct_item_ops->drop_link) + type->ct_item_ops->drop_link(parent_item, + target_sd->s_element); + + spin_lock(&configfs_dirent_lock); + target_sd->s_links--; + spin_unlock(&configfs_dirent_lock); + configfs_put(target_sd); + + config_item_put(parent_item); + + ret = 0; + +out: + return ret; +} + +const struct inode_operations configfs_symlink_inode_operations = { + .get_link = simple_get_link, + .setattr = configfs_setattr, +}; + diff --git a/fs/coredump.c b/fs/coredump.c new file mode 100644 index 000000000..ac3cd7480 --- /dev/null +++ b/fs/coredump.c @@ -0,0 +1,1778 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include "internal.h" + +#include + +#define CREATE_TRACE_POINTS +#include + +static bool dump_vma_snapshot(struct coredump_params *cprm); +static void free_vma_snapshot(struct coredump_params *cprm); + +#define CORE_FILE_NOTE_SIZE_DEFAULT (4*1024*1024) +/* Define a reasonable max cap */ +#define CORE_FILE_NOTE_SIZE_MAX (16*1024*1024) +/* + * File descriptor number for the pidfd for the thread-group leader of + * the coredumping task installed into the usermode helper's file + * descriptor table. + */ +#define COREDUMP_PIDFD_NUMBER 3 + +static int core_uses_pid; +static unsigned int core_pipe_limit; +static unsigned int core_sort_vma; +static char core_pattern[CORENAME_MAX_SIZE] = "core"; +static int core_name_size = CORENAME_MAX_SIZE; +unsigned int core_file_note_size_limit = CORE_FILE_NOTE_SIZE_DEFAULT; +static atomic_t core_pipe_count = ATOMIC_INIT(0); + +enum coredump_type_t { + COREDUMP_FILE = 1, + COREDUMP_PIPE = 2, + COREDUMP_SOCK = 3, + COREDUMP_SOCK_REQ = 4, +}; + +struct core_name { + char *corename __counted_by_ptr(size); + int used, size; + unsigned int core_pipe_limit; + bool core_dumped; + enum coredump_type_t core_type; + u64 mask; +}; + +static int expand_corename(struct core_name *cn, int size) +{ + char *corename; + + size = kmalloc_size_roundup(size); + corename = krealloc(cn->corename, size, GFP_KERNEL); + if (!corename) + return -ENOMEM; + + cn->corename = corename; + cn->size = size; + + if (size > core_name_size) /* racy but harmless */ + core_name_size = size; + + return 0; +} + +static __printf(2, 0) int cn_vprintf(struct core_name *cn, const char *fmt, + va_list arg) +{ + int free, need; + va_list arg_copy; + +again: + free = cn->size - cn->used; + + va_copy(arg_copy, arg); + need = vsnprintf(cn->corename + cn->used, free, fmt, arg_copy); + va_end(arg_copy); + + if (need < free) { + cn->used += need; + return 0; + } + + if (!expand_corename(cn, cn->size + need - free + 1)) + goto again; + + return -ENOMEM; +} + +static __printf(2, 3) int cn_printf(struct core_name *cn, const char *fmt, ...) +{ + va_list arg; + int ret; + + va_start(arg, fmt); + ret = cn_vprintf(cn, fmt, arg); + va_end(arg); + + return ret; +} + +static __printf(2, 3) +int cn_esc_printf(struct core_name *cn, const char *fmt, ...) +{ + int cur = cn->used; + va_list arg; + int ret; + + va_start(arg, fmt); + ret = cn_vprintf(cn, fmt, arg); + va_end(arg); + + if (ret == 0) { + /* + * Ensure that this coredump name component can't cause the + * resulting corefile path to consist of a ".." or ".". + */ + if ((cn->used - cur == 1 && cn->corename[cur] == '.') || + (cn->used - cur == 2 && cn->corename[cur] == '.' + && cn->corename[cur+1] == '.')) + cn->corename[cur] = '!'; + + /* + * Empty names are fishy and could be used to create a "//" in a + * corefile name, causing the coredump to happen one directory + * level too high. Enforce that all components of the core + * pattern are at least one character long. + */ + if (cn->used == cur) + ret = cn_printf(cn, "!"); + } + + for (; cur < cn->used; ++cur) { + if (cn->corename[cur] == '/') + cn->corename[cur] = '!'; + } + return ret; +} + +static int cn_print_exe_file(struct core_name *cn, bool name_only) +{ + struct file *exe_file; + char *pathbuf, *path, *ptr; + int ret; + + exe_file = get_mm_exe_file(current->mm); + if (!exe_file) + return cn_esc_printf(cn, "%s (path unknown)", current->comm); + + pathbuf = kmalloc(PATH_MAX, GFP_KERNEL); + if (!pathbuf) { + ret = -ENOMEM; + goto put_exe_file; + } + + path = file_path(exe_file, pathbuf, PATH_MAX); + if (IS_ERR(path)) { + ret = PTR_ERR(path); + goto free_buf; + } + + if (name_only) { + ptr = strrchr(path, '/'); + if (ptr) + path = ptr + 1; + } + ret = cn_esc_printf(cn, "%s", path); + +free_buf: + kfree(pathbuf); +put_exe_file: + fput(exe_file); + return ret; +} + +/* + * coredump_parse will inspect the pattern parameter, and output a name + * into corename, which must have space for at least CORENAME_MAX_SIZE + * bytes plus one byte for the zero terminator. + */ +static bool coredump_parse(struct core_name *cn, struct coredump_params *cprm, + size_t **argv, int *argc) +{ + const struct cred *cred = current_cred(); + const char *pat_ptr = core_pattern; + bool was_space = false; + int pid_in_pattern = 0; + int err = 0; + + cn->mask = COREDUMP_KERNEL; + if (core_pipe_limit) + cn->mask |= COREDUMP_WAIT; + cn->used = 0; + cn->corename = NULL; + cn->core_pipe_limit = 0; + cn->core_dumped = false; + if (*pat_ptr == '|') + cn->core_type = COREDUMP_PIPE; + else if (*pat_ptr == '@') + cn->core_type = COREDUMP_SOCK; + else + cn->core_type = COREDUMP_FILE; + if (expand_corename(cn, core_name_size)) + return false; + cn->corename[0] = '\0'; + + switch (cn->core_type) { + case COREDUMP_PIPE: { + int argvs = sizeof(core_pattern) / 2; + (*argv) = kmalloc_objs(**argv, argvs); + if (!(*argv)) + return false; + (*argv)[(*argc)++] = 0; + ++pat_ptr; + if (!(*pat_ptr)) + return false; + break; + } + case COREDUMP_SOCK: { + /* skip the @ */ + pat_ptr++; + if (!(*pat_ptr)) + return false; + if (*pat_ptr == '@') { + pat_ptr++; + if (!(*pat_ptr)) + return false; + + cn->core_type = COREDUMP_SOCK_REQ; + } + + err = cn_printf(cn, "%s", pat_ptr); + if (err) + return false; + + /* Require absolute paths. */ + if (cn->corename[0] != '/') + return false; + + /* + * Ensure we can uses spaces to indicate additional + * parameters in the future. + */ + if (strchr(cn->corename, ' ')) { + coredump_report_failure("Coredump socket may not %s contain spaces", cn->corename); + return false; + } + + /* Must not contain ".." in the path. */ + if (name_contains_dotdot(cn->corename)) { + coredump_report_failure("Coredump socket may not %s contain '..' spaces", cn->corename); + return false; + } + + if (strlen(cn->corename) >= UNIX_PATH_MAX) { + coredump_report_failure("Coredump socket path %s too long", cn->corename); + return false; + } + + /* + * Currently no need to parse any other options. + * Relevant information can be retrieved from the peer + * pidfd retrievable via SO_PEERPIDFD by the receiver or + * via /proc/, using the SO_PEERPIDFD to guard + * against pid recycling when opening /proc/. + */ + return true; + } + case COREDUMP_FILE: + break; + default: + WARN_ON_ONCE(true); + return false; + } + + /* Repeat as long as we have more pattern to process and more output + space */ + while (*pat_ptr) { + /* + * Split on spaces before doing template expansion so that + * %e and %E don't get split if they have spaces in them + */ + if (cn->core_type == COREDUMP_PIPE) { + if (isspace(*pat_ptr)) { + if (cn->used != 0) + was_space = true; + pat_ptr++; + continue; + } else if (was_space) { + was_space = false; + err = cn_printf(cn, "%c", '\0'); + if (err) + return false; + (*argv)[(*argc)++] = cn->used; + } + } + if (*pat_ptr != '%') { + err = cn_printf(cn, "%c", *pat_ptr++); + } else { + switch (*++pat_ptr) { + /* single % at the end, drop that */ + case 0: + goto out; + /* Double percent, output one percent */ + case '%': + err = cn_printf(cn, "%c", '%'); + break; + /* pid */ + case 'p': + pid_in_pattern = 1; + err = cn_printf(cn, "%d", + task_tgid_vnr(current)); + break; + /* global pid */ + case 'P': + err = cn_printf(cn, "%d", + task_tgid_nr(current)); + break; + case 'i': + err = cn_printf(cn, "%d", + task_pid_vnr(current)); + break; + case 'I': + err = cn_printf(cn, "%d", + task_pid_nr(current)); + break; + /* uid */ + case 'u': + err = cn_printf(cn, "%u", + from_kuid(&init_user_ns, + cred->uid)); + break; + /* gid */ + case 'g': + err = cn_printf(cn, "%u", + from_kgid(&init_user_ns, + cred->gid)); + break; + case 'd': + err = cn_printf(cn, "%d", cprm->dumpable); + break; + /* signal that caused the coredump */ + case 's': + err = cn_printf(cn, "%d", + cprm->siginfo->si_signo); + break; + /* UNIX time of coredump */ + case 't': { + time64_t time; + + time = ktime_get_real_seconds(); + err = cn_printf(cn, "%lld", time); + break; + } + /* hostname */ + case 'h': + down_read(&uts_sem); + err = cn_esc_printf(cn, "%s", + utsname()->nodename); + up_read(&uts_sem); + break; + /* executable, could be changed by prctl PR_SET_NAME etc */ + case 'e': + err = cn_esc_printf(cn, "%s", current->comm); + break; + /* file name of executable */ + case 'f': + err = cn_print_exe_file(cn, true); + break; + case 'E': + err = cn_print_exe_file(cn, false); + break; + /* core limit size */ + case 'c': + err = cn_printf(cn, "%lu", + rlimit(RLIMIT_CORE)); + break; + /* CPU the task ran on */ + case 'C': + err = cn_printf(cn, "%d", cprm->cpu); + break; + /* pidfd number */ + case 'F': { + /* + * Installing a pidfd only makes sense if + * we actually spawn a usermode helper. + */ + if (cn->core_type != COREDUMP_PIPE) + break; + + /* + * Note that we'll install a pidfd for the + * thread-group leader. We know that task + * linkage hasn't been removed yet and even if + * this @current isn't the actual thread-group + * leader we know that the thread-group leader + * cannot be reaped until @current has exited. + */ + cprm->pid = task_tgid(current); + err = cn_printf(cn, "%d", COREDUMP_PIDFD_NUMBER); + break; + } + default: + break; + } + ++pat_ptr; + } + + if (err) + return false; + } + +out: + /* Backward compatibility with core_uses_pid: + * + * If core_pattern does not include a %p (as is the default) + * and core_uses_pid is set, then .%pid will be appended to + * the filename. Do not do this for piped commands. */ + if (cn->core_type == COREDUMP_FILE && !pid_in_pattern && core_uses_pid) + return cn_printf(cn, ".%d", task_tgid_vnr(current)) == 0; + + return true; +} + +static int zap_process(struct signal_struct *signal, int exit_code) +{ + struct task_struct *t; + int nr = 0; + + signal->flags = SIGNAL_GROUP_EXIT; + signal->group_exit_code = exit_code; + signal->group_stop_count = 0; + + __for_each_thread(signal, t) { + task_clear_jobctl_pending(t, JOBCTL_PENDING_MASK); + if (t != current && !(t->flags & PF_POSTCOREDUMP)) { + sigaddset(&t->pending.signal, SIGKILL); + signal_wake_up(t, 1); + nr++; + } + } + + return nr; +} + +static int zap_threads(struct task_struct *tsk, + struct core_state *core_state, int exit_code) +{ + struct signal_struct *signal = tsk->signal; + int nr = -EAGAIN; + + spin_lock_irq(&tsk->sighand->siglock); + if (!(signal->flags & SIGNAL_GROUP_EXIT) && !signal->group_exec_task) { + /* Allow SIGKILL, see prepare_signal() */ + signal->core_state = core_state; + nr = zap_process(signal, exit_code); + clear_tsk_thread_flag(tsk, TIF_SIGPENDING); + tsk->flags |= PF_DUMPCORE; + atomic_set(&core_state->nr_threads, nr); + } + spin_unlock_irq(&tsk->sighand->siglock); + return nr; +} + +static int coredump_wait(int exit_code, struct core_state *core_state) +{ + struct task_struct *tsk = current; + int core_waiters = -EBUSY; + + init_completion(&core_state->startup); + core_state->dumper.task = tsk; + core_state->dumper.next = NULL; + + core_waiters = zap_threads(tsk, core_state, exit_code); + if (core_waiters > 0) { + struct core_thread *ptr; + + wait_for_completion_state(&core_state->startup, + TASK_UNINTERRUPTIBLE|TASK_FREEZABLE); + /* + * Wait for all the threads to become inactive, so that + * all the thread context (extended register state, like + * fpu etc) gets copied to the memory. + */ + ptr = core_state->dumper.next; + while (ptr != NULL) { + wait_task_inactive(ptr->task, TASK_ANY); + ptr = ptr->next; + } + } + + return core_waiters; +} + +static void coredump_finish(bool core_dumped) +{ + struct core_thread *curr, *next; + struct task_struct *task; + + spin_lock_irq(¤t->sighand->siglock); + if (core_dumped && !__fatal_signal_pending(current)) + current->signal->group_exit_code |= 0x80; + next = current->signal->core_state->dumper.next; + current->signal->core_state = NULL; + spin_unlock_irq(¤t->sighand->siglock); + + while ((curr = next) != NULL) { + next = curr->next; + task = curr->task; + /* + * see coredump_task_exit(), curr->task must not see + * ->task == NULL before we read ->next. + */ + smp_mb(); + curr->task = NULL; + wake_up_process(task); + } +} + +static bool dump_interrupted(void) +{ + /* + * SIGKILL or freezing() interrupt the coredumping. Perhaps we + * can do try_to_freeze() and check __fatal_signal_pending(), + * but then we need to teach dump_write() to restart and clear + * TIF_SIGPENDING. + */ + return fatal_signal_pending(current) || freezing(current); +} + +static void wait_for_dump_helpers(struct file *file) +{ + struct pipe_inode_info *pipe = file->private_data; + + pipe_lock(pipe); + pipe->readers++; + pipe->writers--; + wake_up_interruptible_sync(&pipe->rd_wait); + kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); + pipe_unlock(pipe); + + /* + * We actually want wait_event_freezable() but then we need + * to clear TIF_SIGPENDING and improve dump_interrupted(). + */ + wait_event_interruptible(pipe->rd_wait, pipe->readers == 1); + + pipe_lock(pipe); + pipe->readers--; + pipe->writers++; + pipe_unlock(pipe); +} + +/* + * umh_coredump_setup + * helper function to customize the process used + * to collect the core in userspace. Specifically + * it sets up a pipe and installs it as fd 0 (stdin) + * for the process. Returns 0 on success, or + * PTR_ERR on failure. + * Note that it also sets the core limit to 1. This + * is a special value that we use to trap recursive + * core dumps + */ +static int umh_coredump_setup(struct subprocess_info *info, struct cred *new) +{ + struct file *files[2]; + struct coredump_params *cp = (struct coredump_params *)info->data; + int err; + + if (cp->pid) { + struct file *pidfs_file __free(fput) = NULL; + + pidfs_file = pidfs_alloc_file(cp->pid, 0); + if (IS_ERR(pidfs_file)) + return PTR_ERR(pidfs_file); + + pidfs_coredump(cp); + + /* + * Usermode helpers are childen of either + * system_dfl_wq or of kthreadd. So we know that + * we're starting off with a clean file descriptor + * table. So we should always be able to use + * COREDUMP_PIDFD_NUMBER as our file descriptor value. + */ + err = replace_fd(COREDUMP_PIDFD_NUMBER, pidfs_file, 0); + if (err < 0) + return err; + } + + err = create_pipe_files(files, 0); + if (err) + return err; + + cp->file = files[1]; + + err = replace_fd(0, files[0], 0); + fput(files[0]); + if (err < 0) + return err; + + /* and disallow core files too */ + current->signal->rlim[RLIMIT_CORE] = (struct rlimit){1, 1}; + + return 0; +} + +#ifdef CONFIG_UNIX +static bool coredump_sock_connect(struct core_name *cn, struct coredump_params *cprm) +{ + struct file *file __free(fput) = NULL; + struct sockaddr_un addr = { + .sun_family = AF_UNIX, + }; + ssize_t addr_len; + int retval; + struct socket *socket; + + addr_len = strscpy(addr.sun_path, cn->corename); + if (addr_len < 0) + return false; + addr_len += offsetof(struct sockaddr_un, sun_path) + 1; + + /* + * It is possible that the userspace process which is supposed + * to handle the coredump and is listening on the AF_UNIX socket + * coredumps. Userspace should just mark itself non dumpable. + */ + + retval = sock_create_kern(&init_net, AF_UNIX, SOCK_STREAM, 0, &socket); + if (retval < 0) + return false; + + file = sock_alloc_file(socket, 0, NULL); + if (IS_ERR(file)) + return false; + + /* + * Set the thread-group leader pid which is used for the peer + * credentials during connect() below. Then immediately register + * it in pidfs... + */ + cprm->pid = task_tgid(current); + retval = pidfs_register_pid(cprm->pid); + if (retval) + return false; + + /* + * ... and set the coredump information so userspace has it + * available after connect()... + */ + pidfs_coredump(cprm); + + retval = kernel_connect(socket, (struct sockaddr_unsized *)(&addr), addr_len, + O_NONBLOCK | SOCK_COREDUMP); + + if (retval) { + if (retval == -EAGAIN) + coredump_report_failure("Coredump socket %s receive queue full", addr.sun_path); + else + coredump_report_failure("Coredump socket connection %s failed %d", addr.sun_path, retval); + return false; + } + + /* ... and validate that @sk_peer_pid matches @cprm.pid. */ + if (WARN_ON_ONCE(unix_peer(socket->sk)->sk_peer_pid != cprm->pid)) + return false; + + cprm->limit = RLIM_INFINITY; + cprm->file = no_free_ptr(file); + + return true; +} + +static inline bool coredump_sock_recv(struct file *file, struct coredump_ack *ack, size_t size, int flags) +{ + struct msghdr msg = {}; + struct kvec iov = { .iov_base = ack, .iov_len = size }; + ssize_t ret; + + memset(ack, 0, size); + ret = kernel_recvmsg(sock_from_file(file), &msg, &iov, 1, size, flags); + return ret == size; +} + +static inline bool coredump_sock_send(struct file *file, struct coredump_req *req) +{ + struct msghdr msg = { .msg_flags = MSG_NOSIGNAL }; + struct kvec iov = { .iov_base = req, .iov_len = sizeof(*req) }; + ssize_t ret; + + ret = kernel_sendmsg(sock_from_file(file), &msg, &iov, 1, sizeof(*req)); + return ret == sizeof(*req); +} + +static_assert(sizeof(enum coredump_mark) == sizeof(__u32)); + +static inline bool coredump_sock_mark(struct file *file, enum coredump_mark mark) +{ + struct msghdr msg = { .msg_flags = MSG_NOSIGNAL }; + struct kvec iov = { .iov_base = &mark, .iov_len = sizeof(mark) }; + ssize_t ret; + + ret = kernel_sendmsg(sock_from_file(file), &msg, &iov, 1, sizeof(mark)); + return ret == sizeof(mark); +} + +static inline void coredump_sock_wait(struct file *file) +{ + ssize_t n; + + /* + * We use a simple read to wait for the coredump processing to + * finish. Either the socket is closed or we get sent unexpected + * data. In both cases, we're done. + */ + n = __kernel_read(file, &(char){ 0 }, 1, NULL); + if (n > 0) + coredump_report_failure("Coredump socket had unexpected data"); + else if (n < 0) + coredump_report_failure("Coredump socket failed"); +} + +static inline void coredump_sock_shutdown(struct file *file) +{ + struct socket *socket; + + socket = sock_from_file(file); + if (!socket) + return; + + /* Let userspace know we're done processing the coredump. */ + kernel_sock_shutdown(socket, SHUT_WR); +} + +static bool coredump_sock_request(struct core_name *cn, struct coredump_params *cprm) +{ + struct coredump_req req = { + .size = sizeof(struct coredump_req), + .mask = COREDUMP_KERNEL | COREDUMP_USERSPACE | + COREDUMP_REJECT | COREDUMP_WAIT, + .size_ack = sizeof(struct coredump_ack), + }; + struct coredump_ack ack = {}; + ssize_t usize; + + if (cn->core_type != COREDUMP_SOCK_REQ) + return true; + + /* Let userspace know what we support. */ + if (!coredump_sock_send(cprm->file, &req)) + return false; + + /* Peek the size of the coredump_ack. */ + if (!coredump_sock_recv(cprm->file, &ack, sizeof(ack.size), + MSG_PEEK | MSG_WAITALL)) + return false; + + /* Refuse unknown coredump_ack sizes. */ + usize = ack.size; + if (usize < COREDUMP_ACK_SIZE_VER0) { + coredump_sock_mark(cprm->file, COREDUMP_MARK_MINSIZE); + return false; + } + + if (usize > sizeof(ack)) { + coredump_sock_mark(cprm->file, COREDUMP_MARK_MAXSIZE); + return false; + } + + /* Now retrieve the coredump_ack. */ + if (!coredump_sock_recv(cprm->file, &ack, usize, MSG_WAITALL)) + return false; + if (ack.size != usize) + return false; + + /* Refuse unknown coredump_ack flags. */ + if (ack.mask & ~req.mask) { + coredump_sock_mark(cprm->file, COREDUMP_MARK_UNSUPPORTED); + return false; + } + + /* Refuse mutually exclusive options. */ + if (hweight64(ack.mask & (COREDUMP_USERSPACE | COREDUMP_KERNEL | + COREDUMP_REJECT)) != 1) { + coredump_sock_mark(cprm->file, COREDUMP_MARK_CONFLICTING); + return false; + } + + if (ack.spare) { + coredump_sock_mark(cprm->file, COREDUMP_MARK_UNSUPPORTED); + return false; + } + + cn->mask = ack.mask; + return coredump_sock_mark(cprm->file, COREDUMP_MARK_REQACK); +} + +static bool coredump_socket(struct core_name *cn, struct coredump_params *cprm) +{ + if (!coredump_sock_connect(cn, cprm)) + return false; + + return coredump_sock_request(cn, cprm); +} +#else +static inline void coredump_sock_wait(struct file *file) { } +static inline void coredump_sock_shutdown(struct file *file) { } +static inline bool coredump_socket(struct core_name *cn, struct coredump_params *cprm) { return false; } +#endif + +/* cprm->dumpable is the snapshot of task dumpability at dump start. */ +static inline bool coredump_force_suid_safe(const struct coredump_params *cprm) +{ + /* Require nonrelative corefile path and be extra careful. */ + return cprm->dumpable == TASK_DUMPABLE_ROOT; +} + +static bool coredump_file(struct core_name *cn, struct coredump_params *cprm, + const struct linux_binfmt *binfmt) +{ + struct mnt_idmap *idmap; + struct inode *inode; + struct file *file __free(fput) = NULL; + int open_flags = O_CREAT | O_WRONLY | O_NOFOLLOW | O_LARGEFILE | O_EXCL; + + if (cprm->limit < binfmt->min_coredump) + return false; + + if (coredump_force_suid_safe(cprm) && cn->corename[0] != '/') { + coredump_report_failure("this process can only dump core to a fully qualified path, skipping core dump"); + return false; + } + + /* + * Unlink the file if it exists unless this is a SUID + * binary - in that case, we're running around with root + * privs and don't want to unlink another user's coredump. + */ + if (!coredump_force_suid_safe(cprm)) { + CLASS(filename_kernel, name)(cn->corename); + /* + * If it doesn't exist, that's fine. If there's some + * other problem, we'll catch it at the filp_open(). + */ + filename_unlinkat(AT_FDCWD, name); + } + + /* + * There is a race between unlinking and creating the + * file, but if that causes an EEXIST here, that's + * fine - another process raced with us while creating + * the corefile, and the other process won. To userspace, + * what matters is that at least one of the two processes + * writes its coredump successfully, not which one. + */ + if (coredump_force_suid_safe(cprm)) { + /* + * Using user namespaces, normal user tasks can change + * their current->fs->root to point to arbitrary + * directories. Since the intention of the "only dump + * with a fully qualified path" rule is to control where + * coredumps may be placed using root privileges, + * current->fs->root must not be used. Instead, use the + * root directory of PID 1. + */ + scoped_with_init_fs() + file = filp_open(cn->corename, open_flags, 0600); + } else { + file = filp_open(cn->corename, open_flags, 0600); + } + if (IS_ERR(file)) + return false; + + inode = file_inode(file); + if (inode->i_nlink > 1) + return false; + if (d_unhashed(file->f_path.dentry)) + return false; + /* + * AK: actually i see no reason to not allow this for named + * pipes etc, but keep the previous behaviour for now. + */ + if (!S_ISREG(inode->i_mode)) + return false; + /* + * Don't dump core if the filesystem changed owner or mode + * of the file during file creation. This is an issue when + * a process dumps core while its cwd is e.g. on a vfat + * filesystem. + */ + idmap = file_mnt_idmap(file); + if (!vfsuid_eq_kuid(i_uid_into_vfsuid(idmap, inode), current_fsuid())) { + coredump_report_failure("Core dump to %s aborted: cannot preserve file owner", cn->corename); + return false; + } + if ((inode->i_mode & 0677) != 0600) { + coredump_report_failure("Core dump to %s aborted: cannot preserve file permissions", cn->corename); + return false; + } + if (!(file->f_mode & FMODE_CAN_WRITE)) + return false; + if (do_truncate(idmap, file->f_path.dentry, 0, 0, file)) + return false; + + cprm->file = no_free_ptr(file); + return true; +} + +static bool coredump_pipe(struct core_name *cn, struct coredump_params *cprm, + size_t *argv, int argc) +{ + int argi; + char **helper_argv __free(kfree) = NULL; + struct subprocess_info *sub_info; + + if (cprm->limit == 1) { + /* See umh_coredump_setup() which sets RLIMIT_CORE = 1. + * + * Normally core limits are irrelevant to pipes, since + * we're not writing to the file system, but we use + * cprm.limit of 1 here as a special value, this is a + * consistent way to catch recursive crashes. + * We can still crash if the core_pattern binary sets + * RLIM_CORE = !1, but it runs as root, and can do + * lots of stupid things. + * + * Note that we use task_tgid_vnr here to grab the pid + * of the process group leader. That way we get the + * right pid if a thread in a multi-threaded + * core_pattern process dies. + */ + coredump_report_failure("RLIMIT_CORE is set to 1, aborting core"); + return false; + } + cprm->limit = RLIM_INFINITY; + + cn->core_pipe_limit = atomic_inc_return(&core_pipe_count); + if (core_pipe_limit && (core_pipe_limit < cn->core_pipe_limit)) { + coredump_report_failure("over core_pipe_limit, skipping core dump"); + return false; + } + + helper_argv = kmalloc_array(argc + 1, sizeof(*helper_argv), GFP_KERNEL); + if (!helper_argv) { + coredump_report_failure("%s failed to allocate memory", __func__); + return false; + } + for (argi = 0; argi < argc; argi++) + helper_argv[argi] = cn->corename + argv[argi]; + helper_argv[argi] = NULL; + + sub_info = call_usermodehelper_setup(helper_argv[0], helper_argv, NULL, + GFP_KERNEL, umh_coredump_setup, + NULL, cprm); + if (!sub_info) + return false; + + if (call_usermodehelper_exec(sub_info, UMH_WAIT_EXEC)) { + coredump_report_failure("|%s pipe failed", cn->corename); + return false; + } + + /* + * umh disabled with CONFIG_STATIC_USERMODEHELPER_PATH="" would + * have this set to NULL. + */ + if (!cprm->file) { + coredump_report_failure("Core dump to |%s disabled", cn->corename); + return false; + } + + return true; +} + +static bool coredump_write(struct core_name *cn, + struct coredump_params *cprm, + const struct linux_binfmt *binfmt) +{ + + if (dump_interrupted()) + return true; + + if (!dump_vma_snapshot(cprm)) + return false; + + file_start_write(cprm->file); + cn->core_dumped = binfmt->core_dump(cprm); + /* + * Ensures that file size is big enough to contain the current + * file postion. This prevents gdb from complaining about + * a truncated file if the last "write" to the file was + * dump_skip. + */ + if (cprm->to_skip) { + cprm->to_skip--; + dump_emit(cprm, "", 1); + } + file_end_write(cprm->file); + free_vma_snapshot(cprm); + return true; +} + +static void coredump_cleanup(struct core_name *cn, struct coredump_params *cprm) +{ + if (cprm->file) + filp_close(cprm->file, NULL); + if (cn->core_pipe_limit) { + VFS_WARN_ON_ONCE(cn->core_type != COREDUMP_PIPE); + atomic_dec(&core_pipe_count); + } + kfree(cn->corename); + coredump_finish(cn->core_dumped); +} + +static inline bool coredump_skip(const struct coredump_params *cprm, + const struct linux_binfmt *binfmt) +{ + if (!binfmt) + return true; + if (!binfmt->core_dump) + return true; + if (cprm->dumpable == TASK_DUMPABLE_OFF) + return true; + return false; +} + +static void do_coredump(struct core_name *cn, struct coredump_params *cprm, + size_t **argv, int *argc, const struct linux_binfmt *binfmt) +{ + trace_coredump(cprm->siginfo->si_signo); + + if (!coredump_parse(cn, cprm, argv, argc)) { + coredump_report_failure("format_corename failed, aborting core"); + return; + } + + switch (cn->core_type) { + case COREDUMP_FILE: + if (!coredump_file(cn, cprm, binfmt)) + return; + break; + case COREDUMP_PIPE: + if (!coredump_pipe(cn, cprm, *argv, *argc)) + return; + break; + case COREDUMP_SOCK_REQ: + fallthrough; + case COREDUMP_SOCK: + if (!coredump_socket(cn, cprm)) + return; + break; + default: + WARN_ON_ONCE(true); + return; + } + + /* Don't even generate the coredump. */ + if (cn->mask & COREDUMP_REJECT) + return; + + /* get us an unshared descriptor table; almost always a no-op */ + /* The cell spufs coredump code reads the file descriptor tables */ + if (unshare_files()) + return; + + if ((cn->mask & COREDUMP_KERNEL) && !coredump_write(cn, cprm, binfmt)) + return; + + coredump_sock_shutdown(cprm->file); + + /* Let the parent know that a coredump was generated. */ + if (cn->mask & COREDUMP_USERSPACE) + cn->core_dumped = true; + + /* + * When core_pipe_limit is set we wait for the coredump server + * or usermodehelper to finish before exiting so it can e.g., + * inspect /proc/. + */ + if (cn->mask & COREDUMP_WAIT) { + switch (cn->core_type) { + case COREDUMP_PIPE: + wait_for_dump_helpers(cprm->file); + break; + case COREDUMP_SOCK_REQ: + fallthrough; + case COREDUMP_SOCK: + coredump_sock_wait(cprm->file); + break; + default: + break; + } + } +} + +void vfs_coredump(const kernel_siginfo_t *siginfo) +{ + size_t *argv __free(kfree) = NULL; + struct core_state core_state; + struct core_name cn; + const struct mm_struct *mm = current->mm; + const struct linux_binfmt *binfmt = mm->binfmt; + int argc = 0; + struct coredump_params cprm = { + .siginfo = siginfo, + .limit = rlimit(RLIMIT_CORE), + /* Snapshot MMF_DUMP_FILTER_* (unlocked) and dumpable for the dump. */ + .mm_flags = __mm_flags_get_word(mm), + .dumpable = task_exec_state_get_dumpable(current), + .vma_meta = NULL, + .cpu = raw_smp_processor_id(), + }; + + audit_core_dumps(siginfo->si_signo); + + if (coredump_skip(&cprm, binfmt)) + return; + + CLASS(prepare_creds, cred)(); + if (!cred) + return; + /* + * We cannot trust fsuid as being the "true" uid of the process + * nor do we know its entire history. We only know it was tainted + * so we dump it as root in mode 2, and only into a controlled + * environment (pipe handler or fully qualified path). + */ + if (coredump_force_suid_safe(&cprm)) + cred->fsuid = GLOBAL_ROOT_UID; + + if (coredump_wait(siginfo->si_signo, &core_state) < 0) + return; + + scoped_with_creds(cred) + do_coredump(&cn, &cprm, &argv, &argc, binfmt); + coredump_cleanup(&cn, &cprm); + return; +} + +/* + * Core dumping helper functions. These are the only things you should + * do on a core-file: use only these functions to write out all the + * necessary info. + */ +static int __dump_emit(struct coredump_params *cprm, const void *addr, int nr) +{ + struct file *file = cprm->file; + loff_t pos = file->f_pos; + ssize_t n; + + if (cprm->written + nr > cprm->limit) + return 0; + if (dump_interrupted()) + return 0; + n = __kernel_write(file, addr, nr, &pos); + if (n != nr) + return 0; + file->f_pos = pos; + cprm->written += n; + cprm->pos += n; + + return 1; +} + +static int __dump_skip(struct coredump_params *cprm, size_t nr) +{ + static char zeroes[PAGE_SIZE]; + struct file *file = cprm->file; + + if (file->f_mode & FMODE_LSEEK) { + if (dump_interrupted() || vfs_llseek(file, nr, SEEK_CUR) < 0) + return 0; + cprm->pos += nr; + return 1; + } + + while (nr > PAGE_SIZE) { + if (!__dump_emit(cprm, zeroes, PAGE_SIZE)) + return 0; + nr -= PAGE_SIZE; + } + + return __dump_emit(cprm, zeroes, nr); +} + +int dump_emit(struct coredump_params *cprm, const void *addr, int nr) +{ + if (cprm->to_skip) { + if (!__dump_skip(cprm, cprm->to_skip)) + return 0; + cprm->to_skip = 0; + } + return __dump_emit(cprm, addr, nr); +} +EXPORT_SYMBOL(dump_emit); + +void dump_skip_to(struct coredump_params *cprm, unsigned long pos) +{ + cprm->to_skip = pos - cprm->pos; +} +EXPORT_SYMBOL(dump_skip_to); + +void dump_skip(struct coredump_params *cprm, size_t nr) +{ + cprm->to_skip += nr; +} +EXPORT_SYMBOL(dump_skip); + +#ifdef CONFIG_ELF_CORE +static int dump_emit_page(struct coredump_params *cprm, struct page *page) +{ + struct bio_vec bvec; + struct iov_iter iter; + struct file *file = cprm->file; + loff_t pos; + ssize_t n; + + if (!page) + return 0; + + if (cprm->to_skip) { + if (!__dump_skip(cprm, cprm->to_skip)) + return 0; + cprm->to_skip = 0; + } + if (cprm->written + PAGE_SIZE > cprm->limit) + return 0; + if (dump_interrupted()) + return 0; + pos = file->f_pos; + bvec_set_page(&bvec, page, PAGE_SIZE, 0); + iov_iter_bvec(&iter, ITER_SOURCE, &bvec, 1, PAGE_SIZE); + n = __kernel_write_iter(cprm->file, &iter, &pos); + if (n != PAGE_SIZE) + return 0; + file->f_pos = pos; + cprm->written += PAGE_SIZE; + cprm->pos += PAGE_SIZE; + + return 1; +} + +/* + * If we might get machine checks from kernel accesses during the + * core dump, let's get those errors early rather than during the + * IO. This is not performance-critical enough to warrant having + * all the machine check logic in the iovec paths. + */ +#ifdef copy_mc_to_kernel + +#define dump_page_alloc() alloc_page(GFP_KERNEL) +#define dump_page_free(x) __free_page(x) +static struct page *dump_page_copy(struct page *src, struct page *dst) +{ + void *buf = kmap_local_page(src); + size_t left = copy_mc_to_kernel(page_address(dst), buf, PAGE_SIZE); + kunmap_local(buf); + return left ? NULL : dst; +} + +#else + +/* We just want to return non-NULL; it's never used. */ +#define dump_page_alloc() ERR_PTR(-EINVAL) +#define dump_page_free(x) ((void)(x)) +static inline struct page *dump_page_copy(struct page *src, struct page *dst) +{ + return src; +} +#endif + +int dump_user_range(struct coredump_params *cprm, unsigned long start, + unsigned long len) +{ + unsigned long addr; + struct page *dump_page; + int locked, ret; + + dump_page = dump_page_alloc(); + if (!dump_page) + return 0; + + ret = 0; + locked = 0; + for (addr = start; addr < start + len; addr += PAGE_SIZE) { + struct page *page; + + if (!locked) { + if (mmap_read_lock_killable(current->mm)) + goto out; + locked = 1; + } + + /* + * To avoid having to allocate page tables for virtual address + * ranges that have never been used yet, and also to make it + * easy to generate sparse core files, use a helper that returns + * NULL when encountering an empty page table entry that would + * otherwise have been filled with the zero page. + */ + page = get_dump_page(addr, &locked); + if (page) { + if (locked) { + mmap_read_unlock(current->mm); + locked = 0; + } + int stop = !dump_emit_page(cprm, dump_page_copy(page, dump_page)); + put_page(page); + if (stop) + goto out; + } else { + dump_skip(cprm, PAGE_SIZE); + } + + if (dump_interrupted()) + goto out; + + if (!need_resched()) + continue; + if (locked) { + mmap_read_unlock(current->mm); + locked = 0; + } + cond_resched(); + } + ret = 1; +out: + if (locked) + mmap_read_unlock(current->mm); + + dump_page_free(dump_page); + return ret; +} +#endif + +int dump_align(struct coredump_params *cprm, int align) +{ + unsigned mod = (cprm->pos + cprm->to_skip) & (align - 1); + if (align & (align - 1)) + return 0; + if (mod) + cprm->to_skip += align - mod; + return 1; +} +EXPORT_SYMBOL(dump_align); + +#ifdef CONFIG_SYSCTL + +void validate_coredump_safety(void) +{ + if (suid_dumpable == TASK_DUMPABLE_ROOT && + core_pattern[0] != '/' && core_pattern[0] != '|' && core_pattern[0] != '@') { + + coredump_report_failure("Unsafe core_pattern used with fs.suid_dumpable=2: " + "pipe handler or fully qualified core dump path required. " + "Set kernel.core_pattern before fs.suid_dumpable."); + } +} + +static inline bool check_coredump_socket(void) +{ + const char *p; + + if (core_pattern[0] != '@') + return true; + + /* + * Coredump socket must be located in the initial mount + * namespace. Don't give the impression that anything else is + * supported right now. + */ + if (current->nsproxy->mnt_ns != init_task.nsproxy->mnt_ns) + return false; + + /* Must be an absolute path... */ + if (core_pattern[1] != '/') { + /* ... or the socket request protocol... */ + if (core_pattern[1] != '@') + return false; + /* ... and if so must be an absolute path. */ + if (core_pattern[2] != '/') + return false; + p = &core_pattern[2]; + } else { + p = &core_pattern[1]; + } + + /* The path obviously cannot exceed UNIX_PATH_MAX. */ + if (strlen(p) >= UNIX_PATH_MAX) + return false; + + /* Must not contain ".." in the path. */ + if (name_contains_dotdot(core_pattern)) + return false; + + return true; +} + +static int proc_dostring_coredump(const struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + int error; + ssize_t retval; + char old_core_pattern[CORENAME_MAX_SIZE]; + + if (!write) + return proc_dostring(table, write, buffer, lenp, ppos); + + retval = strscpy(old_core_pattern, core_pattern, CORENAME_MAX_SIZE); + + error = proc_dostring(table, write, buffer, lenp, ppos); + if (error) + return error; + + if (!check_coredump_socket()) { + strscpy(core_pattern, old_core_pattern, retval + 1); + return -EINVAL; + } + + if (strncmp(old_core_pattern, core_pattern, CORENAME_MAX_SIZE)) + validate_coredump_safety(); + return error; +} + +static const unsigned int core_file_note_size_min = CORE_FILE_NOTE_SIZE_DEFAULT; +static const unsigned int core_file_note_size_max = CORE_FILE_NOTE_SIZE_MAX; +static char core_modes[] = { + "file\npipe" +#ifdef CONFIG_UNIX + "\nsocket" +#endif +}; + +static const struct ctl_table coredump_sysctls[] = { + { + .procname = "core_uses_pid", + .data = &core_uses_pid, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "core_pattern", + .data = core_pattern, + .maxlen = CORENAME_MAX_SIZE, + .mode = 0644, + .proc_handler = proc_dostring_coredump, + }, + { + .procname = "core_pipe_limit", + .data = &core_pipe_limit, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_INT_MAX, + }, + { + .procname = "core_file_note_size_limit", + .data = &core_file_note_size_limit, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = (unsigned int *)&core_file_note_size_min, + .extra2 = (unsigned int *)&core_file_note_size_max, + }, + { + .procname = "core_sort_vma", + .data = &core_sort_vma, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "core_modes", + .data = core_modes, + .maxlen = sizeof(core_modes) - 1, + .mode = 0444, + .proc_handler = proc_dostring, + }, +}; + +static int __init init_fs_coredump_sysctls(void) +{ + register_sysctl_init("kernel", coredump_sysctls); + return 0; +} +fs_initcall(init_fs_coredump_sysctls); +#endif /* CONFIG_SYSCTL */ + +/* + * The purpose of always_dump_vma() is to make sure that special kernel mappings + * that are useful for post-mortem analysis are included in every core dump. + * In that way we ensure that the core dump is fully interpretable later + * without matching up the same kernel and hardware config to see what PC values + * meant. These special mappings include - vDSO, vsyscall, and other + * architecture specific mappings + */ +static bool always_dump_vma(struct vm_area_struct *vma) +{ + /* Any vsyscall mappings? */ + if (vma == get_gate_vma(vma->vm_mm)) + return true; + + /* + * Assume that all vmas with a .name op should always be dumped. + * If this changes, a new vm_ops field can easily be added. + */ + if (vma->vm_ops && vma->vm_ops->name && vma->vm_ops->name(vma)) + return true; + + /* + * arch_vma_name() returns non-NULL for special architecture mappings, + * such as vDSO sections. + */ + if (arch_vma_name(vma)) + return true; + + return false; +} + +#define DUMP_SIZE_MAYBE_ELFHDR_PLACEHOLDER 1 + +/* + * Decide how much of @vma's contents should be included in a core dump. + */ +static unsigned long vma_dump_size(struct vm_area_struct *vma, + unsigned long mm_flags) +{ +#define FILTER(type) (mm_flags & (1UL << MMF_DUMP_##type)) + + /* always dump the vdso and vsyscall sections */ + if (always_dump_vma(vma)) + goto whole; + + if (vma->vm_flags & VM_DONTDUMP) + return 0; + + /* support for DAX */ + if (vma_is_dax(vma)) { + if ((vma->vm_flags & VM_SHARED) && FILTER(DAX_SHARED)) + goto whole; + if (!(vma->vm_flags & VM_SHARED) && FILTER(DAX_PRIVATE)) + goto whole; + return 0; + } + + /* Hugetlb memory check */ + if (is_vm_hugetlb_page(vma)) { + if ((vma->vm_flags & VM_SHARED) && FILTER(HUGETLB_SHARED)) + goto whole; + if (!(vma->vm_flags & VM_SHARED) && FILTER(HUGETLB_PRIVATE)) + goto whole; + return 0; + } + + /* Do not dump I/O mapped devices or special mappings */ + if (vma->vm_flags & VM_IO) + return 0; + + /* By default, dump shared memory if mapped from an anonymous file. */ + if (vma->vm_flags & VM_SHARED) { + if (file_inode(vma->vm_file)->i_nlink == 0 ? + FILTER(ANON_SHARED) : FILTER(MAPPED_SHARED)) + goto whole; + return 0; + } + + /* Dump segments that have been written to. */ + if ((!IS_ENABLED(CONFIG_MMU) || vma->anon_vma) && FILTER(ANON_PRIVATE)) + goto whole; + if (vma->vm_file == NULL) + return 0; + + if (FILTER(MAPPED_PRIVATE)) + goto whole; + + /* + * If this is the beginning of an executable file mapping, + * dump the first page to aid in determining what was mapped here. + */ + if (FILTER(ELF_HEADERS) && + vma->vm_pgoff == 0 && (vma->vm_flags & VM_READ)) { + if ((READ_ONCE(file_inode(vma->vm_file)->i_mode) & 0111) != 0) + return PAGE_SIZE; + + /* + * ELF libraries aren't always executable. + * We'll want to check whether the mapping starts with the ELF + * magic, but not now - we're holding the mmap lock, + * so copy_from_user() doesn't work here. + * Use a placeholder instead, and fix it up later in + * dump_vma_snapshot(). + */ + return DUMP_SIZE_MAYBE_ELFHDR_PLACEHOLDER; + } + +#undef FILTER + + return 0; + +whole: + return vma->vm_end - vma->vm_start; +} + +/* + * Helper function for iterating across a vma list. It ensures that the caller + * will visit `gate_vma' prior to terminating the search. + */ +static struct vm_area_struct *coredump_next_vma(struct vma_iterator *vmi, + struct vm_area_struct *vma, + struct vm_area_struct *gate_vma) +{ + if (gate_vma && (vma == gate_vma)) + return NULL; + + vma = vma_next(vmi); + if (vma) + return vma; + return gate_vma; +} + +static void free_vma_snapshot(struct coredump_params *cprm) +{ + if (cprm->vma_meta) { + int i; + for (i = 0; i < cprm->vma_count; i++) { + struct file *file = cprm->vma_meta[i].file; + if (file) + fput(file); + } + kvfree(cprm->vma_meta); + cprm->vma_meta = NULL; + } +} + +static int cmp_vma_size(const void *vma_meta_lhs_ptr, const void *vma_meta_rhs_ptr) +{ + const struct core_vma_metadata *vma_meta_lhs = vma_meta_lhs_ptr; + const struct core_vma_metadata *vma_meta_rhs = vma_meta_rhs_ptr; + + if (vma_meta_lhs->dump_size < vma_meta_rhs->dump_size) + return -1; + if (vma_meta_lhs->dump_size > vma_meta_rhs->dump_size) + return 1; + return 0; +} + +/* + * Under the mmap_lock, take a snapshot of relevant information about the task's + * VMAs. + */ +static bool dump_vma_snapshot(struct coredump_params *cprm) +{ + struct vm_area_struct *gate_vma, *vma = NULL; + struct mm_struct *mm = current->mm; + VMA_ITERATOR(vmi, mm, 0); + int i = 0; + + /* + * Once the stack expansion code is fixed to not change VMA bounds + * under mmap_lock in read mode, this can be changed to take the + * mmap_lock in read mode. + */ + if (mmap_write_lock_killable(mm)) + return false; + + cprm->vma_data_size = 0; + gate_vma = get_gate_vma(mm); + cprm->vma_count = mm->map_count + (gate_vma ? 1 : 0); + + cprm->vma_meta = kvmalloc_objs(*cprm->vma_meta, cprm->vma_count); + if (!cprm->vma_meta) { + mmap_write_unlock(mm); + return false; + } + + while ((vma = coredump_next_vma(&vmi, vma, gate_vma)) != NULL) { + struct core_vma_metadata *m = cprm->vma_meta + i; + + m->start = vma->vm_start; + m->end = vma->vm_end; + m->flags = vma->vm_flags; + m->dump_size = vma_dump_size(vma, cprm->mm_flags); + m->pgoff = vma->vm_pgoff; + m->file = vma->vm_file; + if (m->file) + get_file(m->file); + i++; + } + + mmap_write_unlock(mm); + + for (i = 0; i < cprm->vma_count; i++) { + struct core_vma_metadata *m = cprm->vma_meta + i; + + if (m->dump_size == DUMP_SIZE_MAYBE_ELFHDR_PLACEHOLDER) { + char elfmag[SELFMAG]; + + if (copy_from_user(elfmag, (void __user *)m->start, SELFMAG) || + memcmp(elfmag, ELFMAG, SELFMAG) != 0) { + m->dump_size = 0; + } else { + m->dump_size = PAGE_SIZE; + } + } + + cprm->vma_data_size += m->dump_size; + } + + if (core_sort_vma) + sort(cprm->vma_meta, cprm->vma_count, sizeof(*cprm->vma_meta), + cmp_vma_size, NULL); + + return true; +} diff --git a/fs/cramfs/Kconfig b/fs/cramfs/Kconfig new file mode 100644 index 000000000..4612c9bbf --- /dev/null +++ b/fs/cramfs/Kconfig @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: GPL-2.0-only +config CRAMFS + tristate "Compressed ROM file system support (cramfs)" + select ZLIB_INFLATE + help + Saying Y here includes support for CramFs (Compressed ROM File + System). CramFs is designed to be a simple, small, and compressed + file system for ROM based embedded systems. CramFs is read-only, + limited to 256MB file systems (with 16MB files), and doesn't support + 16/32 bits uid/gid, hard links and timestamps. + + See and + for further information. + + To compile this as a module, choose M here: the module will be called + cramfs. Note that the root file system (the one containing the + directory /) cannot be compiled as a module. + + This filesystem is limited in capabilities and performance on + purpose to remain small and low on RAM usage. It is most suitable + for small embedded systems. If you have ample RAM to spare, you may + consider a more capable compressed filesystem such as SquashFS + which is much better in terms of performance and features. + + If unsure, say N. + +config CRAMFS_BLOCKDEV + bool "Support CramFs image over a regular block device" if EXPERT + depends on CRAMFS && BLOCK + default y + help + This option allows the CramFs driver to load data from a regular + block device such a disk partition or a ramdisk. + +config CRAMFS_MTD + bool "Support CramFs image directly mapped in physical memory" + depends on CRAMFS && CRAMFS <= MTD + default y if !CRAMFS_BLOCKDEV + help + This option allows the CramFs driver to load data directly from + a linear addressed memory range (usually non-volatile memory + like flash) instead of going through the block device layer. + This saves some memory since no intermediate buffering is + necessary. + + The location of the CramFs image is determined by a + MTD device capable of direct memory mapping e.g. from + the 'physmap' map driver or a resulting MTD partition. + For example, this would mount the cramfs image stored in + the MTD partition named "xip_fs" on the /mnt mountpoint: + + mount -t cramfs mtd:xip_fs /mnt + + If unsure, say N. diff --git a/fs/cramfs/Makefile b/fs/cramfs/Makefile new file mode 100644 index 000000000..8c3ed2982 --- /dev/null +++ b/fs/cramfs/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the linux cramfs routines. +# + +obj-$(CONFIG_CRAMFS) += cramfs.o + +cramfs-objs := inode.o uncompress.o diff --git a/fs/cramfs/README b/fs/cramfs/README new file mode 100644 index 000000000..c0052a11a --- /dev/null +++ b/fs/cramfs/README @@ -0,0 +1,107 @@ +Notes on Filesystem Layout +-------------------------- + +These notes describe what mkcramfs generates. Kernel requirements are +a bit looser, e.g. it doesn't care if the items are +swapped around (though it does care that directory entries (inodes) in +a given directory are contiguous, as this is used by readdir). + +All data is currently in host-endian format; neither mkcramfs nor the +kernel ever do swabbing. (See section `Block Size' below.) + +: + + + + +: struct cramfs_super (see cramfs_fs.h). + +: + For each file: + struct cramfs_inode (see cramfs_fs.h). + Filename. Not generally null-terminated, but it is + null-padded to a multiple of 4 bytes. + +The order of inode traversal is described as "width-first" (not to be +confused with breadth-first); i.e. like depth-first but listing all of +a directory's entries before recursing down its subdirectories: the +same order as `ls -AUR' (but without the /^\..*:$/ directory header +lines); put another way, the same order as `find -type d -exec +ls -AU1 {} \;'. + +Beginning in 2.4.7, directory entries are sorted. This optimization +allows cramfs_lookup to return more quickly when a filename does not +exist, speeds up user-space directory sorts, etc. + +: + One for each file that's either a symlink or a + regular file of non-zero st_size. + +: + nblocks * + (where nblocks = (st_size - 1) / blksize + 1) + nblocks * + padding to multiple of 4 bytes + +The i'th for a file stores the byte offset of the +*end* of the i'th (i.e. one past the last byte, which is the +same as the start of the (i+1)'th if there is one). The first + immediately follows the last for the file. +s are each 32 bits long. + +When the CRAMFS_FLAG_EXT_BLOCK_POINTERS capability bit is set, each +'s top bits may contain special flags as follows: + +CRAMFS_BLK_FLAG_UNCOMPRESSED (bit 31): + The block data is not compressed and should be copied verbatim. + +CRAMFS_BLK_FLAG_DIRECT_PTR (bit 30): + The stores the actual block start offset and not + its end, shifted right by 2 bits. The block must therefore be + aligned to a 4-byte boundary. The block size is either blksize + if CRAMFS_BLK_FLAG_UNCOMPRESSED is also specified, otherwise + the compressed data length is included in the first 2 bytes of + the block data. This is used to allow discontiguous data layout + and specific data block alignments e.g. for XIP applications. + + +The order of 's is a depth-first descent of the directory +tree, i.e. the same order as `find -size +0 \( -type f -o -type l \) +-print'. + + +: The i'th is the output of zlib's compress function +applied to the i'th blksize-sized chunk of the input data if the +corresponding CRAMFS_BLK_FLAG_UNCOMPRESSED bit is not set, +otherwise it is the input data directly. +(For the last of the file, the input may of course be smaller.) +Each may be a different size. (See above.) + +s are merely byte-aligned, not generally u32-aligned. + +When CRAMFS_BLK_FLAG_DIRECT_PTR is specified then the corresponding + may be located anywhere and not necessarily contiguous with +the previous/next blocks. In that case it is minimally u32-aligned. +If CRAMFS_BLK_FLAG_UNCOMPRESSED is also specified then the size is always +blksize except for the last block which is limited by the file length. +If CRAMFS_BLK_FLAG_DIRECT_PTR is set and CRAMFS_BLK_FLAG_UNCOMPRESSED +is not set then the first 2 bytes of the block contains the size of the +remaining block data as this cannot be determined from the placement of +logically adjacent blocks. + + +Holes +----- + +This kernel supports cramfs holes (i.e. [efficient representation of] +blocks in uncompressed data consisting entirely of NUL bytes), but by +default mkcramfs doesn't test for & create holes, since cramfs in +kernels up to at least 2.3.39 didn't support holes. Run mkcramfs +with -z if you want it to create files that can have holes in them. + + +Tools +----- + +The cramfs user-space tools, including mkcramfs and cramfsck, are +located at . diff --git a/fs/cramfs/inode.c b/fs/cramfs/inode.c new file mode 100644 index 000000000..d4cd03f4f --- /dev/null +++ b/fs/cramfs/inode.c @@ -0,0 +1,1012 @@ +/* + * Compressed rom filesystem for Linux. + * + * Copyright (C) 1999 Linus Torvalds. + * + * This file is released under the GPL. + */ + +/* + * These are the VFS interfaces to the compressed rom filesystem. + * The actual compression is based on zlib, see the other files. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +/* + * cramfs super-block data in memory + */ +struct cramfs_sb_info { + unsigned long magic; + unsigned long size; + unsigned long blocks; + unsigned long files; + unsigned long flags; + void *linear_virt_addr; + resource_size_t linear_phys_addr; + size_t mtd_point_size; +}; + +static inline struct cramfs_sb_info *CRAMFS_SB(struct super_block *sb) +{ + return sb->s_fs_info; +} + +static const struct super_operations cramfs_ops; +static const struct inode_operations cramfs_dir_inode_operations; +static const struct file_operations cramfs_directory_operations; +static const struct file_operations cramfs_physmem_fops; +static const struct address_space_operations cramfs_aops; + +static DEFINE_MUTEX(read_mutex); + + +/* These macros may change in future, to provide better st_ino semantics. */ +#define OFFSET(x) ((x)->i_ino) + +static unsigned long cramino(const struct cramfs_inode *cino, unsigned int offset) +{ + if (!cino->offset) + return offset + 1; + if (!cino->size) + return offset + 1; + + /* + * The file mode test fixes buggy mkcramfs implementations where + * cramfs_inode->offset is set to a non zero value for entries + * which did not contain data, like devices node and fifos. + */ + switch (cino->mode & S_IFMT) { + case S_IFREG: + case S_IFDIR: + case S_IFLNK: + return cino->offset << 2; + default: + break; + } + return offset + 1; +} + +static struct inode *get_cramfs_inode(struct super_block *sb, + const struct cramfs_inode *cramfs_inode, unsigned int offset) +{ + struct inode *inode; + static struct timespec64 zerotime; + + inode = iget_locked(sb, cramino(cramfs_inode, offset)); + if (!inode) + return ERR_PTR(-ENOMEM); + if (!(inode_state_read_once(inode) & I_NEW)) + return inode; + + switch (cramfs_inode->mode & S_IFMT) { + case S_IFREG: + inode->i_fop = &generic_ro_fops; + inode->i_data.a_ops = &cramfs_aops; + if (IS_ENABLED(CONFIG_CRAMFS_MTD) && + CRAMFS_SB(sb)->flags & CRAMFS_FLAG_EXT_BLOCK_POINTERS && + CRAMFS_SB(sb)->linear_phys_addr) + inode->i_fop = &cramfs_physmem_fops; + break; + case S_IFDIR: + inode->i_op = &cramfs_dir_inode_operations; + inode->i_fop = &cramfs_directory_operations; + break; + case S_IFLNK: + inode->i_op = &page_symlink_inode_operations; + inode_nohighmem(inode); + inode->i_data.a_ops = &cramfs_aops; + break; + case S_IFCHR: + case S_IFBLK: + case S_IFIFO: + case S_IFSOCK: + init_special_inode(inode, cramfs_inode->mode, + old_decode_dev(cramfs_inode->size)); + break; + default: + printk(KERN_DEBUG "CRAMFS: Invalid file type 0%04o for inode %llu.\n", + inode->i_mode, inode->i_ino); + iget_failed(inode); + return ERR_PTR(-EIO); + } + + inode->i_mode = cramfs_inode->mode; + i_uid_write(inode, cramfs_inode->uid); + i_gid_write(inode, cramfs_inode->gid); + + /* if the lower 2 bits are zero, the inode contains data */ + if (!(inode->i_ino & 3)) { + inode->i_size = cramfs_inode->size; + inode->i_blocks = (cramfs_inode->size - 1) / 512 + 1; + } + + /* Struct copy intentional */ + inode_set_mtime_to_ts(inode, + inode_set_atime_to_ts(inode, inode_set_ctime_to_ts(inode, zerotime))); + /* inode->i_nlink is left 1 - arguably wrong for directories, + but it's the best we can do without reading the directory + contents. 1 yields the right result in GNU find, even + without -noleaf option. */ + + unlock_new_inode(inode); + + return inode; +} + +/* + * We have our own block cache: don't fill up the buffer cache + * with the rom-image, because the way the filesystem is set + * up the accesses should be fairly regular and cached in the + * page cache and dentry tree anyway.. + * + * This also acts as a way to guarantee contiguous areas of up to + * BLKS_PER_BUF*PAGE_SIZE, so that the caller doesn't need to + * worry about end-of-buffer issues even when decompressing a full + * page cache. + * + * Note: This is all optimized away at compile time when + * CONFIG_CRAMFS_BLOCKDEV=n. + */ +#define READ_BUFFERS (2) +/* NEXT_BUFFER(): Loop over [0..(READ_BUFFERS-1)]. */ +#define NEXT_BUFFER(_ix) ((_ix) ^ 1) + +/* + * BLKS_PER_BUF_SHIFT should be at least 2 to allow for "compressed" + * data that takes up more space than the original and with unlucky + * alignment. + */ +#define BLKS_PER_BUF_SHIFT (2) +#define BLKS_PER_BUF (1 << BLKS_PER_BUF_SHIFT) +#define BUFFER_SIZE (BLKS_PER_BUF*PAGE_SIZE) + +static unsigned char read_buffers[READ_BUFFERS][BUFFER_SIZE]; +static unsigned buffer_blocknr[READ_BUFFERS]; +static struct super_block *buffer_dev[READ_BUFFERS]; +static int next_buffer; + +/* + * Populate our block cache and return a pointer to it. + */ +static void *cramfs_blkdev_read(struct super_block *sb, unsigned int offset, + unsigned int len) +{ + struct address_space *mapping = sb->s_bdev->bd_mapping; + struct file_ra_state ra = {}; + struct page *pages[BLKS_PER_BUF]; + unsigned i, blocknr, buffer; + unsigned long devsize; + char *data; + + if (!len) + return NULL; + blocknr = offset >> PAGE_SHIFT; + offset &= PAGE_SIZE - 1; + + /* Check if an existing buffer already has the data.. */ + for (i = 0; i < READ_BUFFERS; i++) { + unsigned int blk_offset; + + if (buffer_dev[i] != sb) + continue; + if (blocknr < buffer_blocknr[i]) + continue; + blk_offset = (blocknr - buffer_blocknr[i]) << PAGE_SHIFT; + blk_offset += offset; + if (blk_offset > BUFFER_SIZE || + blk_offset + len > BUFFER_SIZE) + continue; + return read_buffers[i] + blk_offset; + } + + devsize = bdev_nr_bytes(sb->s_bdev) >> PAGE_SHIFT; + + /* Ok, read in BLKS_PER_BUF pages completely first. */ + file_ra_state_init(&ra, mapping); + page_cache_sync_readahead(mapping, &ra, NULL, blocknr, BLKS_PER_BUF); + + for (i = 0; i < BLKS_PER_BUF; i++) { + struct page *page = NULL; + + if (blocknr + i < devsize) { + page = read_mapping_page(mapping, blocknr + i, NULL); + /* synchronous error? */ + if (IS_ERR(page)) + page = NULL; + } + pages[i] = page; + } + + buffer = next_buffer; + next_buffer = NEXT_BUFFER(buffer); + buffer_blocknr[buffer] = blocknr; + buffer_dev[buffer] = sb; + + data = read_buffers[buffer]; + for (i = 0; i < BLKS_PER_BUF; i++) { + struct page *page = pages[i]; + + if (page) { + memcpy_from_page(data, page, 0, PAGE_SIZE); + put_page(page); + } else + memset(data, 0, PAGE_SIZE); + data += PAGE_SIZE; + } + return read_buffers[buffer] + offset; +} + +/* + * Return a pointer to the linearly addressed cramfs image in memory. + */ +static void *cramfs_direct_read(struct super_block *sb, unsigned int offset, + unsigned int len) +{ + struct cramfs_sb_info *sbi = CRAMFS_SB(sb); + + if (!len) + return NULL; + if (len > sbi->size || offset > sbi->size - len) + return page_address(ZERO_PAGE(0)); + return sbi->linear_virt_addr + offset; +} + +/* + * Returns a pointer to a buffer containing at least LEN bytes of + * filesystem starting at byte offset OFFSET into the filesystem. + */ +static void *cramfs_read(struct super_block *sb, unsigned int offset, + unsigned int len) +{ + struct cramfs_sb_info *sbi = CRAMFS_SB(sb); + + if (IS_ENABLED(CONFIG_CRAMFS_MTD) && sbi->linear_virt_addr) + return cramfs_direct_read(sb, offset, len); + else if (IS_ENABLED(CONFIG_CRAMFS_BLOCKDEV)) + return cramfs_blkdev_read(sb, offset, len); + else + return NULL; +} + +/* + * For a mapping to be possible, we need a range of uncompressed and + * contiguous blocks. Return the offset for the first block and number of + * valid blocks for which that is true, or zero otherwise. + */ +static u32 cramfs_get_block_range(struct inode *inode, u32 pgoff, u32 *pages) +{ + struct cramfs_sb_info *sbi = CRAMFS_SB(inode->i_sb); + int i; + u32 *blockptrs, first_block_addr; + + /* + * We can dereference memory directly here as this code may be + * reached only when there is a direct filesystem image mapping + * available in memory. + */ + blockptrs = (u32 *)(sbi->linear_virt_addr + OFFSET(inode) + pgoff * 4); + first_block_addr = blockptrs[0] & ~CRAMFS_BLK_FLAGS; + i = 0; + do { + u32 block_off = i * (PAGE_SIZE >> CRAMFS_BLK_DIRECT_PTR_SHIFT); + u32 expect = (first_block_addr + block_off) | + CRAMFS_BLK_FLAG_DIRECT_PTR | + CRAMFS_BLK_FLAG_UNCOMPRESSED; + if (blockptrs[i] != expect) { + pr_debug("range: block %d/%d got %#x expects %#x\n", + pgoff+i, pgoff + *pages - 1, + blockptrs[i], expect); + if (i == 0) + return 0; + break; + } + } while (++i < *pages); + + *pages = i; + return first_block_addr << CRAMFS_BLK_DIRECT_PTR_SHIFT; +} + +#ifdef CONFIG_MMU + +/* + * Return true if the last page of a file in the filesystem image contains + * some other data that doesn't belong to that file. It is assumed that the + * last block is CRAMFS_BLK_FLAG_DIRECT_PTR | CRAMFS_BLK_FLAG_UNCOMPRESSED + * (verified by cramfs_get_block_range() and directly accessible in memory. + */ +static bool cramfs_last_page_is_shared(struct inode *inode) +{ + struct cramfs_sb_info *sbi = CRAMFS_SB(inode->i_sb); + u32 partial, last_page, blockaddr, *blockptrs; + char *tail_data; + + partial = offset_in_page(inode->i_size); + if (!partial) + return false; + last_page = inode->i_size >> PAGE_SHIFT; + blockptrs = (u32 *)(sbi->linear_virt_addr + OFFSET(inode)); + blockaddr = blockptrs[last_page] & ~CRAMFS_BLK_FLAGS; + blockaddr <<= CRAMFS_BLK_DIRECT_PTR_SHIFT; + tail_data = sbi->linear_virt_addr + blockaddr + partial; + return memchr_inv(tail_data, 0, PAGE_SIZE - partial) ? true : false; +} + +static int cramfs_physmem_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct inode *inode = file_inode(file); + struct cramfs_sb_info *sbi = CRAMFS_SB(inode->i_sb); + unsigned int pages, max_pages, offset; + unsigned long address, pgoff = vma->vm_pgoff; + char *bailout_reason; + int ret; + + ret = generic_file_readonly_mmap(file, vma); + if (ret) + return ret; + + /* + * Now try to pre-populate ptes for this vma with a direct + * mapping avoiding memory allocation when possible. + */ + + /* Could COW work here? */ + bailout_reason = "vma is writable"; + if (vma->vm_flags & VM_WRITE) + goto bailout; + + max_pages = (inode->i_size + PAGE_SIZE - 1) >> PAGE_SHIFT; + bailout_reason = "beyond file limit"; + if (pgoff >= max_pages) + goto bailout; + pages = min(vma_pages(vma), max_pages - pgoff); + + offset = cramfs_get_block_range(inode, pgoff, &pages); + bailout_reason = "unsuitable block layout"; + if (!offset) + goto bailout; + address = sbi->linear_phys_addr + offset; + bailout_reason = "data is not page aligned"; + if (!PAGE_ALIGNED(address)) + goto bailout; + + /* Don't map the last page if it contains some other data */ + if (pgoff + pages == max_pages && cramfs_last_page_is_shared(inode)) { + pr_debug("mmap: %pD: last page is shared\n", file); + pages--; + } + + if (!pages) { + bailout_reason = "no suitable block remaining"; + goto bailout; + } + + if (pages == vma_pages(vma)) { + /* + * The entire vma is mappable. remap_pfn_range() will + * make it distinguishable from a non-direct mapping + * in /proc//maps by substituting the file offset + * with the actual physical address. + */ + ret = remap_pfn_range(vma, vma->vm_start, address >> PAGE_SHIFT, + pages * PAGE_SIZE, vma->vm_page_prot); + } else { + /* + * Let's create a mixed map if we can't map it all. + * The normal paging machinery will take care of the + * unpopulated ptes via cramfs_read_folio(). + */ + int i; + vm_flags_set(vma, VM_MIXEDMAP); + for (i = 0; i < pages && !ret; i++) { + vm_fault_t vmf; + unsigned long off = i * PAGE_SIZE; + vmf = vmf_insert_mixed(vma, vma->vm_start + off, + PHYS_PFN(address + off)); + if (vmf & VM_FAULT_ERROR) + ret = vm_fault_to_errno(vmf, 0); + } + } + + if (!ret) + pr_debug("mapped %pD[%lu] at 0x%08lx (%u/%lu pages) " + "to vma 0x%08lx, page_prot 0x%llx\n", file, + pgoff, address, pages, vma_pages(vma), vma->vm_start, + (unsigned long long)pgprot_val(vma->vm_page_prot)); + return ret; + +bailout: + pr_debug("%pD[%lu]: direct mmap impossible: %s\n", + file, pgoff, bailout_reason); + /* Didn't manage any direct map, but normal paging is still possible */ + return 0; +} + +#else /* CONFIG_MMU */ + +static int cramfs_physmem_mmap(struct file *file, struct vm_area_struct *vma) +{ + return is_nommu_shared_mapping(vma->vm_flags) ? 0 : -ENOSYS; +} + +static unsigned long cramfs_physmem_get_unmapped_area(struct file *file, + unsigned long addr, unsigned long len, + unsigned long pgoff, unsigned long flags) +{ + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + struct cramfs_sb_info *sbi = CRAMFS_SB(sb); + unsigned int pages, block_pages, max_pages, offset; + + pages = (len + PAGE_SIZE - 1) >> PAGE_SHIFT; + max_pages = (inode->i_size + PAGE_SIZE - 1) >> PAGE_SHIFT; + if (pgoff >= max_pages || pages > max_pages - pgoff) + return -EINVAL; + block_pages = pages; + offset = cramfs_get_block_range(inode, pgoff, &block_pages); + if (!offset || block_pages != pages) + return -ENOSYS; + addr = sbi->linear_phys_addr + offset; + pr_debug("get_unmapped for %pD ofs %#lx siz %lu at 0x%08lx\n", + file, pgoff*PAGE_SIZE, len, addr); + return addr; +} + +static unsigned int cramfs_physmem_mmap_capabilities(struct file *file) +{ + return NOMMU_MAP_COPY | NOMMU_MAP_DIRECT | + NOMMU_MAP_READ | NOMMU_MAP_EXEC; +} + +#endif /* CONFIG_MMU */ + +static const struct file_operations cramfs_physmem_fops = { + .llseek = generic_file_llseek, + .read_iter = generic_file_read_iter, + .splice_read = filemap_splice_read, + .mmap = cramfs_physmem_mmap, +#ifndef CONFIG_MMU + .get_unmapped_area = cramfs_physmem_get_unmapped_area, + .mmap_capabilities = cramfs_physmem_mmap_capabilities, +#endif +}; + +static void cramfs_kill_sb(struct super_block *sb) +{ + struct cramfs_sb_info *sbi = CRAMFS_SB(sb); + + generic_shutdown_super(sb); + + if (IS_ENABLED(CONFIG_CRAMFS_MTD) && sb->s_mtd) { + if (sbi && sbi->mtd_point_size) + mtd_unpoint(sb->s_mtd, 0, sbi->mtd_point_size); + put_mtd_device(sb->s_mtd); + sb->s_mtd = NULL; + } else if (IS_ENABLED(CONFIG_CRAMFS_BLOCKDEV) && sb->s_bdev) { + sync_blockdev(sb->s_bdev); + fs_bdev_file_release(sb->s_bdev_file, sb); + } + kfree(sbi); +} + +static int cramfs_reconfigure(struct fs_context *fc) +{ + sync_filesystem(fc->root->d_sb); + fc->sb_flags |= SB_RDONLY; + return 0; +} + +static int cramfs_read_super(struct super_block *sb, struct fs_context *fc, + struct cramfs_super *super) +{ + struct cramfs_sb_info *sbi = CRAMFS_SB(sb); + unsigned long root_offset; + bool silent = fc->sb_flags & SB_SILENT; + + /* We don't know the real size yet */ + sbi->size = PAGE_SIZE; + + /* Read the first block and get the superblock from it */ + mutex_lock(&read_mutex); + memcpy(super, cramfs_read(sb, 0, sizeof(*super)), sizeof(*super)); + mutex_unlock(&read_mutex); + + /* Do sanity checks on the superblock */ + if (super->magic != CRAMFS_MAGIC) { + /* check for wrong endianness */ + if (super->magic == CRAMFS_MAGIC_WEND) { + if (!silent) + errorfc(fc, "wrong endianness"); + return -EINVAL; + } + + /* check at 512 byte offset */ + mutex_lock(&read_mutex); + memcpy(super, + cramfs_read(sb, 512, sizeof(*super)), + sizeof(*super)); + mutex_unlock(&read_mutex); + if (super->magic != CRAMFS_MAGIC) { + if (super->magic == CRAMFS_MAGIC_WEND && !silent) + errorfc(fc, "wrong endianness"); + else if (!silent) + errorfc(fc, "wrong magic"); + return -EINVAL; + } + } + + /* get feature flags first */ + if (super->flags & ~CRAMFS_SUPPORTED_FLAGS) { + errorfc(fc, "unsupported filesystem features"); + return -EINVAL; + } + + /* Check that the root inode is in a sane state */ + if (!S_ISDIR(super->root.mode)) { + errorfc(fc, "root is not a directory"); + return -EINVAL; + } + /* correct strange, hard-coded permissions of mkcramfs */ + super->root.mode |= 0555; + + root_offset = super->root.offset << 2; + if (super->flags & CRAMFS_FLAG_FSID_VERSION_2) { + sbi->size = super->size; + sbi->blocks = super->fsid.blocks; + sbi->files = super->fsid.files; + } else { + sbi->size = 1<<28; + sbi->blocks = 0; + sbi->files = 0; + } + sbi->magic = super->magic; + sbi->flags = super->flags; + if (root_offset == 0) + infofc(fc, "empty filesystem"); + else if (!(super->flags & CRAMFS_FLAG_SHIFTED_ROOT_OFFSET) && + ((root_offset != sizeof(struct cramfs_super)) && + (root_offset != 512 + sizeof(struct cramfs_super)))) + { + errorfc(fc, "bad root offset %lu", root_offset); + return -EINVAL; + } + + return 0; +} + +static int cramfs_finalize_super(struct super_block *sb, + struct cramfs_inode *cramfs_root) +{ + struct inode *root; + + /* Set it all up.. */ + sb->s_flags |= SB_RDONLY; + sb->s_time_min = 0; + sb->s_time_max = 0; + sb->s_op = &cramfs_ops; + root = get_cramfs_inode(sb, cramfs_root, 0); + if (IS_ERR(root)) + return PTR_ERR(root); + sb->s_root = d_make_root(root); + if (!sb->s_root) + return -ENOMEM; + return 0; +} + +static int cramfs_blkdev_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct cramfs_sb_info *sbi; + struct cramfs_super super; + int i, err; + + sbi = kzalloc_obj(struct cramfs_sb_info); + if (!sbi) + return -ENOMEM; + sb->s_fs_info = sbi; + + /* Invalidate the read buffers on mount: think disk change.. */ + for (i = 0; i < READ_BUFFERS; i++) + buffer_blocknr[i] = -1; + + err = cramfs_read_super(sb, fc, &super); + if (err) + return err; + return cramfs_finalize_super(sb, &super.root); +} + +static int cramfs_mtd_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct cramfs_sb_info *sbi; + struct cramfs_super super; + int err; + + sbi = kzalloc_obj(struct cramfs_sb_info); + if (!sbi) + return -ENOMEM; + sb->s_fs_info = sbi; + + /* Map only one page for now. Will remap it when fs size is known. */ + err = mtd_point(sb->s_mtd, 0, PAGE_SIZE, &sbi->mtd_point_size, + &sbi->linear_virt_addr, &sbi->linear_phys_addr); + if (err || sbi->mtd_point_size != PAGE_SIZE) { + pr_err("unable to get direct memory access to mtd:%s\n", + sb->s_mtd->name); + return err ? : -ENODATA; + } + + pr_info("checking physical address %pap for linear cramfs image\n", + &sbi->linear_phys_addr); + err = cramfs_read_super(sb, fc, &super); + if (err) + return err; + + /* Remap the whole filesystem now */ + pr_info("linear cramfs image on mtd:%s appears to be %lu KB in size\n", + sb->s_mtd->name, sbi->size/1024); + mtd_unpoint(sb->s_mtd, 0, PAGE_SIZE); + err = mtd_point(sb->s_mtd, 0, sbi->size, &sbi->mtd_point_size, + &sbi->linear_virt_addr, &sbi->linear_phys_addr); + if (err || sbi->mtd_point_size != sbi->size) { + pr_err("unable to get direct memory access to mtd:%s\n", + sb->s_mtd->name); + return err ? : -ENODATA; + } + + return cramfs_finalize_super(sb, &super.root); +} + +static int cramfs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + u64 id = 0; + + if (sb->s_bdev) + id = huge_encode_dev(sb->s_bdev->bd_dev); + else if (sb->s_dev) + id = huge_encode_dev(sb->s_dev); + + buf->f_type = CRAMFS_MAGIC; + buf->f_bsize = PAGE_SIZE; + buf->f_blocks = CRAMFS_SB(sb)->blocks; + buf->f_bfree = 0; + buf->f_bavail = 0; + buf->f_files = CRAMFS_SB(sb)->files; + buf->f_ffree = 0; + buf->f_fsid = u64_to_fsid(id); + buf->f_namelen = CRAMFS_MAXPATHLEN; + return 0; +} + +/* + * Read a cramfs directory entry. + */ +static int cramfs_readdir(struct file *file, struct dir_context *ctx) +{ + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + char *buf; + unsigned int offset; + + /* Offset within the thing. */ + if (ctx->pos >= inode->i_size) + return 0; + offset = ctx->pos; + /* Directory entries are always 4-byte aligned */ + if (offset & 3) + return -EINVAL; + + buf = kmalloc(CRAMFS_MAXPATHLEN, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + while (offset < inode->i_size) { + struct cramfs_inode *de; + unsigned long nextoffset; + char *name; + ino_t ino; + umode_t mode; + int namelen; + + mutex_lock(&read_mutex); + de = cramfs_read(sb, OFFSET(inode) + offset, sizeof(*de)+CRAMFS_MAXPATHLEN); + name = (char *)(de+1); + + /* + * Namelengths on disk are shifted by two + * and the name padded out to 4-byte boundaries + * with zeroes. + */ + namelen = de->namelen << 2; + memcpy(buf, name, namelen); + ino = cramino(de, OFFSET(inode) + offset); + mode = de->mode; + mutex_unlock(&read_mutex); + nextoffset = offset + sizeof(*de) + namelen; + for (;;) { + if (!namelen) { + kfree(buf); + return -EIO; + } + if (buf[namelen-1]) + break; + namelen--; + } + if (!dir_emit(ctx, buf, namelen, ino, mode >> 12)) + break; + + ctx->pos = offset = nextoffset; + } + kfree(buf); + return 0; +} + +/* + * Lookup and fill in the inode data.. + */ +static struct dentry *cramfs_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) +{ + unsigned int offset = 0; + struct inode *inode = NULL; + int sorted; + + mutex_lock(&read_mutex); + sorted = CRAMFS_SB(dir->i_sb)->flags & CRAMFS_FLAG_SORTED_DIRS; + while (offset < dir->i_size) { + struct cramfs_inode *de; + char *name; + int namelen, retval; + int dir_off = OFFSET(dir) + offset; + + de = cramfs_read(dir->i_sb, dir_off, sizeof(*de)+CRAMFS_MAXPATHLEN); + name = (char *)(de+1); + + /* Try to take advantage of sorted directories */ + if (sorted && (dentry->d_name.name[0] < name[0])) + break; + + namelen = de->namelen << 2; + offset += sizeof(*de) + namelen; + + /* Quick check that the name is roughly the right length */ + if (((dentry->d_name.len + 3) & ~3) != namelen) + continue; + + for (;;) { + if (!namelen) { + inode = ERR_PTR(-EIO); + goto out; + } + if (name[namelen-1]) + break; + namelen--; + } + if (namelen != dentry->d_name.len) + continue; + retval = memcmp(dentry->d_name.name, name, namelen); + if (retval > 0) + continue; + if (!retval) { + inode = get_cramfs_inode(dir->i_sb, de, dir_off); + break; + } + /* else (retval < 0) */ + if (sorted) + break; + } +out: + mutex_unlock(&read_mutex); + return d_splice_alias(inode, dentry); +} + +static int cramfs_read_folio(struct file *file, struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + u32 maxblock; + int bytes_filled; + void *pgdata; + bool success = false; + + maxblock = (inode->i_size + PAGE_SIZE - 1) >> PAGE_SHIFT; + bytes_filled = 0; + pgdata = kmap_local_folio(folio, 0); + + if (folio->index < maxblock) { + struct super_block *sb = inode->i_sb; + u32 blkptr_offset = OFFSET(inode) + folio->index * 4; + u32 block_ptr, block_start, block_len; + bool uncompressed, direct; + + mutex_lock(&read_mutex); + block_ptr = *(u32 *) cramfs_read(sb, blkptr_offset, 4); + uncompressed = (block_ptr & CRAMFS_BLK_FLAG_UNCOMPRESSED); + direct = (block_ptr & CRAMFS_BLK_FLAG_DIRECT_PTR); + block_ptr &= ~CRAMFS_BLK_FLAGS; + + if (direct) { + /* + * The block pointer is an absolute start pointer, + * shifted by 2 bits. The size is included in the + * first 2 bytes of the data block when compressed, + * or PAGE_SIZE otherwise. + */ + block_start = block_ptr << CRAMFS_BLK_DIRECT_PTR_SHIFT; + if (uncompressed) { + block_len = PAGE_SIZE; + /* if last block: cap to file length */ + if (folio->index == maxblock - 1) + block_len = + offset_in_page(inode->i_size); + } else { + block_len = *(u16 *) + cramfs_read(sb, block_start, 2); + block_start += 2; + } + } else { + /* + * The block pointer indicates one past the end of + * the current block (start of next block). If this + * is the first block then it starts where the block + * pointer table ends, otherwise its start comes + * from the previous block's pointer. + */ + block_start = OFFSET(inode) + maxblock * 4; + if (folio->index) + block_start = *(u32 *) + cramfs_read(sb, blkptr_offset - 4, 4); + /* Beware... previous ptr might be a direct ptr */ + if (unlikely(block_start & CRAMFS_BLK_FLAG_DIRECT_PTR)) { + /* See comments on earlier code. */ + u32 prev_start = block_start; + block_start = prev_start & ~CRAMFS_BLK_FLAGS; + block_start <<= CRAMFS_BLK_DIRECT_PTR_SHIFT; + if (prev_start & CRAMFS_BLK_FLAG_UNCOMPRESSED) { + block_start += PAGE_SIZE; + } else { + block_len = *(u16 *) + cramfs_read(sb, block_start, 2); + block_start += 2 + block_len; + } + } + block_start &= ~CRAMFS_BLK_FLAGS; + block_len = block_ptr - block_start; + } + + if (block_len == 0) + ; /* hole */ + else if (unlikely(block_len > 2*PAGE_SIZE || + (uncompressed && block_len > PAGE_SIZE))) { + mutex_unlock(&read_mutex); + pr_err("bad data blocksize %u\n", block_len); + goto err; + } else if (uncompressed) { + memcpy(pgdata, + cramfs_read(sb, block_start, block_len), + block_len); + bytes_filled = block_len; + } else { + bytes_filled = cramfs_uncompress_block(pgdata, + PAGE_SIZE, + cramfs_read(sb, block_start, block_len), + block_len); + } + mutex_unlock(&read_mutex); + if (unlikely(bytes_filled < 0)) + goto err; + } + + memset(pgdata + bytes_filled, 0, PAGE_SIZE - bytes_filled); + flush_dcache_folio(folio); + + success = true; +err: + kunmap_local(pgdata); + folio_end_read(folio, success); + return 0; +} + +static const struct address_space_operations cramfs_aops = { + .read_folio = cramfs_read_folio +}; + +/* + * Our operations: + */ + +/* + * A directory can only readdir + */ +static const struct file_operations cramfs_directory_operations = { + .llseek = generic_file_llseek, + .read = generic_read_dir, + .iterate_shared = cramfs_readdir, + .setlease = generic_setlease, +}; + +static const struct inode_operations cramfs_dir_inode_operations = { + .lookup = cramfs_lookup, +}; + +static const struct super_operations cramfs_ops = { + .statfs = cramfs_statfs, +}; + +static int cramfs_get_tree(struct fs_context *fc) +{ + int ret = -ENOPROTOOPT; + + if (IS_ENABLED(CONFIG_CRAMFS_MTD)) { + ret = get_tree_mtd(fc, cramfs_mtd_fill_super); + if (!ret) + return 0; + } + if (IS_ENABLED(CONFIG_CRAMFS_BLOCKDEV)) + ret = get_tree_bdev(fc, cramfs_blkdev_fill_super); + return ret; +} + +static const struct fs_context_operations cramfs_context_ops = { + .get_tree = cramfs_get_tree, + .reconfigure = cramfs_reconfigure, +}; + +/* + * Set up the filesystem mount context. + */ +static int cramfs_init_fs_context(struct fs_context *fc) +{ + fc->ops = &cramfs_context_ops; + return 0; +} + +static struct file_system_type cramfs_fs_type = { + .owner = THIS_MODULE, + .name = "cramfs", + .init_fs_context = cramfs_init_fs_context, + .kill_sb = cramfs_kill_sb, + .fs_flags = FS_REQUIRES_DEV, +}; +MODULE_ALIAS_FS("cramfs"); + +static int __init init_cramfs_fs(void) +{ + int rv; + + rv = cramfs_uncompress_init(); + if (rv < 0) + return rv; + rv = register_filesystem(&cramfs_fs_type); + if (rv < 0) + cramfs_uncompress_exit(); + return rv; +} + +static void __exit exit_cramfs_fs(void) +{ + cramfs_uncompress_exit(); + unregister_filesystem(&cramfs_fs_type); +} + +module_init(init_cramfs_fs) +module_exit(exit_cramfs_fs) +MODULE_DESCRIPTION("Compressed ROM file system support"); +MODULE_LICENSE("GPL"); diff --git a/fs/cramfs/internal.h b/fs/cramfs/internal.h new file mode 100644 index 000000000..349d71272 --- /dev/null +++ b/fs/cramfs/internal.h @@ -0,0 +1,4 @@ +/* Uncompression interfaces to the underlying zlib */ +int cramfs_uncompress_block(void *dst, int dstlen, void *src, int srclen); +int cramfs_uncompress_init(void); +void cramfs_uncompress_exit(void); diff --git a/fs/cramfs/uncompress.c b/fs/cramfs/uncompress.c new file mode 100644 index 000000000..975d98fc2 --- /dev/null +++ b/fs/cramfs/uncompress.c @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * uncompress.c + * + * (C) Copyright 1999 Linus Torvalds + * + * cramfs interfaces to the uncompression library. There's really just + * three entrypoints: + * + * - cramfs_uncompress_init() - called to initialize the thing. + * - cramfs_uncompress_exit() - tell me when you're done + * - cramfs_uncompress_block() - uncompress a block. + * + * NOTE NOTE NOTE! The uncompression is entirely single-threaded. We + * only have one stream, and we'll initialize it only once even if it + * then is used by multiple filesystems. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include "internal.h" + +static z_stream stream; +static int initialized; + +/* Returns length of decompressed data. */ +int cramfs_uncompress_block(void *dst, int dstlen, void *src, int srclen) +{ + int err; + + stream.next_in = src; + stream.avail_in = srclen; + + stream.next_out = dst; + stream.avail_out = dstlen; + + err = zlib_inflateReset(&stream); + if (err != Z_OK) { + pr_err("zlib_inflateReset error %d\n", err); + zlib_inflateEnd(&stream); + zlib_inflateInit(&stream); + } + + err = zlib_inflate(&stream, Z_FINISH); + if (err != Z_STREAM_END) + goto err; + return stream.total_out; + +err: + pr_err("Error %d while decompressing!\n", err); + pr_err("%p(%d)->%p(%d)\n", src, srclen, dst, dstlen); + return -EIO; +} + +int cramfs_uncompress_init(void) +{ + if (!initialized++) { + stream.workspace = vmalloc(zlib_inflate_workspacesize()); + if (!stream.workspace) { + initialized = 0; + return -ENOMEM; + } + stream.next_in = NULL; + stream.avail_in = 0; + zlib_inflateInit(&stream); + } + return 0; +} + +void cramfs_uncompress_exit(void) +{ + if (!--initialized) { + zlib_inflateEnd(&stream); + vfree(stream.workspace); + } +} diff --git a/fs/crypto/Kconfig b/fs/crypto/Kconfig new file mode 100644 index 000000000..cd934e31d --- /dev/null +++ b/fs/crypto/Kconfig @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: GPL-2.0-only +config FS_ENCRYPTION + bool "FS Encryption (Per-file encryption)" + select BLK_INLINE_ENCRYPTION if BLOCK + select BLK_INLINE_ENCRYPTION_FALLBACK if BLOCK + select CRYPTO + select CRYPTO_SKCIPHER + select CRYPTO_LIB_AES + select CRYPTO_LIB_SHA256 + select CRYPTO_LIB_SHA512 + select KEYS + help + Enable encryption of files and directories. This + feature is similar to ecryptfs, but it is more memory + efficient since it avoids caching the encrypted and + decrypted pages in the page cache. Currently Ext4, + F2FS, UBIFS, and CephFS make use of this feature. + +# Filesystems supporting encryption must select this if FS_ENCRYPTION. This +# allows the algorithms to be built as modules when all the filesystems are, +# whereas selecting them from FS_ENCRYPTION would force them to be built-in. +# +# Note: this option only pulls in the algorithms that filesystem encryption +# needs "by default". If userspace will use "non-default" encryption modes such +# as Adiantum encryption, then those other modes need to be explicitly enabled +# in the crypto API; see Documentation/filesystems/fscrypt.rst for details. +# +# Also note that this option only pulls in the generic implementations of the +# algorithms, not any per-architecture optimized implementations. It is +# strongly recommended to enable optimized implementations too. +config FS_ENCRYPTION_ALGS + tristate + select CRYPTO_AES + select CRYPTO_CBC + select CRYPTO_CTS + select CRYPTO_XTS + +config FS_ENCRYPTION_INLINE_CRYPT + bool + default y if FS_ENCRYPTION && BLOCK diff --git a/fs/crypto/Makefile b/fs/crypto/Makefile new file mode 100644 index 000000000..b03e02f0f --- /dev/null +++ b/fs/crypto/Makefile @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_FS_ENCRYPTION) += fscrypto.o + +fscrypto-y := crypto.o \ + fname.o \ + hkdf.o \ + hooks.o \ + keyring.o \ + keysetup.o \ + keysetup_v1.o \ + policy.o + +fscrypto-$(CONFIG_BLOCK) += block.o diff --git a/fs/crypto/block.c b/fs/crypto/block.c new file mode 100644 index 000000000..5193f8ba3 --- /dev/null +++ b/fs/crypto/block.c @@ -0,0 +1,415 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * File contents en/decryption on block-based filesystems + * + * Copyright 2019 Google LLC + */ + +/* + * This file implements fscrypt's file contents en/decryption using blk-crypto + * (Documentation/block/inline-encryption.rst). fscrypt assigns a bio_crypt_ctx + * with a key and IV to each bio, and the block layer does the en/decryption. + * + * This file's exported functions are called only by block-based filesystems. + */ + +#include +#include +#include +#include +#include +#include + +#include "fscrypt_private.h" + +static unsigned int +fscrypt_get_devices(struct super_block *sb, + struct block_device *devs[FSCRYPT_MAX_DEVICES]) +{ + if (sb->s_cop->get_devices) + return sb->s_cop->get_devices(sb, devs); + devs[0] = sb->s_bdev; + return 1; +} + +static unsigned int fscrypt_get_dun_bytes(const struct fscrypt_inode_info *ci) +{ + const struct super_block *sb = ci->ci_inode->i_sb; + unsigned int flags = fscrypt_policy_flags(&ci->ci_policy); + int dun_bits; + + if (flags & FSCRYPT_POLICY_FLAG_DIRECT_KEY) + return offsetofend(union fscrypt_iv, nonce); + + if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64) + return sizeof(__le64); + + if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32) + return sizeof(__le32); + + /* Default case: IVs are just the file data unit index */ + dun_bits = fscrypt_max_file_dun_bits(sb, ci->ci_data_unit_bits); + return DIV_ROUND_UP(dun_bits, 8); +} + +/* + * Log a message when starting to use blk-crypto (native) or blk-crypto-fallback + * for an encryption mode for the first time. This is the blk-crypto + * counterpart to the message logged when starting to use the crypto API for the + * first time. A limitation is that these messages don't convey which specific + * filesystems or files are using each implementation. However, *usually* + * systems use just one implementation per mode, which makes these messages + * helpful for debugging problems where the "wrong" implementation is used. + */ +static void fscrypt_log_blk_crypto_impl(struct fscrypt_mode *mode, + struct block_device *dev, + const struct blk_crypto_key *blk_key) +{ + if (blk_crypto_config_supported_natively(dev, &blk_key->crypto_cfg)) { + if (!xchg(&mode->logged_blk_crypto_native, 1)) + pr_info("fscrypt: %s using blk-crypto (native)\n", + mode->friendly_name); + } else if (!xchg(&mode->logged_blk_crypto_fallback, 1)) { + pr_info("fscrypt: %s using blk-crypto-fallback\n", + mode->friendly_name); + } +} + +int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key, + const u8 *key_bytes, size_t key_size, + bool is_hw_wrapped, + const struct fscrypt_inode_info *ci) +{ + const struct inode *inode = ci->ci_inode; + struct super_block *sb = inode->i_sb; + bool inlinecrypt = sb->s_flags & SB_INLINECRYPT; + struct fscrypt_mode *mode = ci->ci_mode; + enum blk_crypto_key_type key_type = is_hw_wrapped ? + BLK_CRYPTO_KEY_TYPE_HW_WRAPPED : BLK_CRYPTO_KEY_TYPE_RAW; + struct blk_crypto_key *blk_key; + struct block_device *devs[FSCRYPT_MAX_DEVICES]; + unsigned int num_devs; + unsigned int i; + int err; + + if (is_hw_wrapped && !inlinecrypt) { + /* + * blk_crypto_init_key() would catch this anyway, but this + * provides a clearer error message. + */ + fscrypt_err( + inode, + "Hardware-wrapped keys require inline encryption (-o inlinecrypt)"); + return -EINVAL; + } + + blk_key = kmalloc_obj(*blk_key); + if (!blk_key) + return -ENOMEM; + + err = blk_crypto_init_key(blk_key, key_bytes, key_size, key_type, + mode->blk_crypto_mode, + fscrypt_get_dun_bytes(ci), + 1U << ci->ci_data_unit_bits, + inlinecrypt ? BLK_CRYPTO_CFG_ALLOW_HW : 0); + if (err) { + fscrypt_err(inode, "Error %d initializing blk-crypto key", err); + goto fail; + } + + /* Start using blk-crypto on all the filesystem's block devices. */ + num_devs = fscrypt_get_devices(sb, devs); + for (i = 0; i < num_devs; i++) { + err = blk_crypto_start_using_key(devs[i], blk_key); + if (err) + break; + fscrypt_log_blk_crypto_impl(mode, devs[i], blk_key); + } + if (err) { + if (err == -EOPNOTSUPP && is_hw_wrapped) + fscrypt_err( + inode, + "Hardware-wrapped key required, but no suitable inline encryption capabilities are available"); + else + fscrypt_err(inode, + "Error %d starting to use blk-crypto", err); + goto fail; + } + + prep_key->blk_key = blk_key; + return 0; + +fail: + kfree_sensitive(blk_key); + return err; +} + +void fscrypt_destroy_inline_crypt_key(struct super_block *sb, + struct fscrypt_prepared_key *prep_key) +{ + struct blk_crypto_key *blk_key = prep_key->blk_key; + struct block_device *devs[FSCRYPT_MAX_DEVICES]; + unsigned int num_devs; + unsigned int i; + + if (!blk_key) + return; + + /* + * Evict the key from all the filesystem's block devices. + * This *must* be done before the key is freed. + */ + num_devs = fscrypt_get_devices(sb, devs); + for (i = 0; i < num_devs; i++) + blk_crypto_evict_key(devs[i], blk_key); + + kfree_sensitive(blk_key); +} + +/* + * Ask the inline encryption hardware to derive the software secret from a + * hardware-wrapped key. Returns -EOPNOTSUPP if hardware-wrapped keys aren't + * supported on this filesystem or hardware. + */ +int fscrypt_derive_sw_secret(struct super_block *sb, + const u8 *wrapped_key, size_t wrapped_key_size, + u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE]) +{ + int err; + + /* The filesystem must be mounted with -o inlinecrypt. */ + if (!(sb->s_flags & SB_INLINECRYPT)) { + fscrypt_warn(NULL, + "%s: filesystem not mounted with inlinecrypt\n", + sb->s_id); + return -EOPNOTSUPP; + } + + err = blk_crypto_derive_sw_secret(sb->s_bdev, wrapped_key, + wrapped_key_size, sw_secret); + if (err == -EOPNOTSUPP) + fscrypt_warn(NULL, + "%s: block device doesn't support hardware-wrapped keys\n", + sb->s_id); + return err; +} + +static void fscrypt_generate_dun(const struct fscrypt_inode_info *ci, + loff_t pos, u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE]) +{ + union fscrypt_iv iv; + int i; + + fscrypt_generate_iv(&iv, pos >> ci->ci_data_unit_bits, ci); + + BUILD_BUG_ON(FSCRYPT_MAX_IV_SIZE > BLK_CRYPTO_MAX_IV_SIZE); + memset(dun, 0, BLK_CRYPTO_MAX_IV_SIZE); + for (i = 0; i < ci->ci_mode->ivsize/sizeof(dun[0]); i++) + dun[i] = le64_to_cpu(iv.dun[i]); +} + +/** + * fscrypt_set_bio_crypt_ctx() - prepare a file contents bio for inline crypto + * @bio: a bio which will eventually be submitted to the file + * @inode: the file's inode + * @pos: the first file position (in bytes) in the I/O + * @gfp_mask: memory allocation flags - these must be a waiting mask so that + * bio_crypt_set_ctx can't fail. + * + * If the contents of the file should be encrypted (or decrypted), then assign + * the appropriate encryption context to the bio. + * + * Normally the bio should be newly allocated (i.e. no pages added yet), as + * otherwise fscrypt_mergeable_bio() won't work as intended. + * + * The encryption context will be freed automatically when the bio is freed. + */ +void fscrypt_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, + loff_t pos, gfp_t gfp_mask) +{ + const struct fscrypt_inode_info *ci; + u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; + + if (!fscrypt_needs_contents_encryption(inode)) + return; + ci = fscrypt_get_inode_info_raw(inode); + + fscrypt_generate_dun(ci, pos, dun); + bio_crypt_set_ctx(bio, ci->ci_enc_key.blk_key, dun, gfp_mask); +} +EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx); + +/** + * fscrypt_mergeable_bio() - test whether data can be added to a bio + * @bio: the bio being built up + * @inode: the inode for the next part of the I/O + * @pos: the next file position (in bytes) in the I/O + * + * When building a bio which may contain data which should undergo encryption + * (or decryption) via fscrypt, filesystems should call this function to ensure + * that the resulting bio contains only contiguous data unit numbers. This will + * return false if the next part of the I/O cannot be merged with the bio + * because either the encryption key would be different or the encryption data + * unit numbers would be discontiguous. + * + * fscrypt_set_bio_crypt_ctx() must have already been called on the bio. + * + * This function isn't required in cases where crypto-mergeability is ensured in + * another way, such as I/O targeting only a single file (and thus a single key) + * combined with fscrypt_limit_io_blocks() to ensure DUN contiguity. + * + * Return: true iff the I/O is mergeable + */ +bool fscrypt_mergeable_bio(struct bio *bio, const struct inode *inode, + loff_t pos) +{ + const struct bio_crypt_ctx *bc = bio->bi_crypt_context; + const struct fscrypt_inode_info *ci; + u64 next_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; + + if (!!bc != fscrypt_needs_contents_encryption(inode)) + return false; + if (!bc) + return true; + ci = fscrypt_get_inode_info_raw(inode); + + /* + * Comparing the key pointers is good enough, as all I/O for each key + * uses the same pointer. I.e., there's currently no need to support + * merging requests where the keys are the same but the pointers differ. + */ + if (bc->bc_key != ci->ci_enc_key.blk_key) + return false; + + fscrypt_generate_dun(ci, pos, next_dun); + return bio_crypt_dun_is_contiguous(bc, bio->bi_iter.bi_size, next_dun); +} +EXPORT_SYMBOL_GPL(fscrypt_mergeable_bio); + +/** + * fscrypt_limit_io_blocks() - limit I/O blocks to avoid discontiguous DUNs + * @inode: the file on which I/O is being done + * @lblk: the block at which the I/O is being started from + * @nr_blocks: the number of blocks we want to submit starting at @lblk + * + * Determine the limit to the number of blocks that can be submitted in a bio + * targeting @lblk without causing a data unit number (DUN) discontiguity. + * + * This is normally just @nr_blocks, as normally the DUNs just increment along + * with the logical blocks. (Or the file is not encrypted.) + * + * In rare cases, fscrypt can be using an IV generation method that allows the + * DUN to wrap around within logically contiguous blocks, and that wraparound + * will occur. If this happens, a value less than @nr_blocks will be returned + * so that the wraparound doesn't occur in the middle of a bio, which would + * cause encryption/decryption to produce wrong results. + * + * Return: the actual number of blocks that can be submitted + */ +u64 fscrypt_limit_io_blocks(const struct inode *inode, u64 lblk, u64 nr_blocks) +{ + const struct fscrypt_inode_info *ci; + u32 dun; + + if (!fscrypt_needs_contents_encryption(inode)) + return nr_blocks; + + if (nr_blocks <= 1) + return nr_blocks; + + ci = fscrypt_get_inode_info_raw(inode); + if (!(fscrypt_policy_flags(&ci->ci_policy) & + FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32)) + return nr_blocks; + + /* With IV_INO_LBLK_32, the DUN can wrap around from U32_MAX to 0. */ + + dun = ci->ci_hashed_ino + lblk; + + return min_t(u64, nr_blocks, (u64)U32_MAX + 1 - dun); +} +EXPORT_SYMBOL_GPL(fscrypt_limit_io_blocks); + +struct fscrypt_zero_done { + atomic_t pending; + blk_status_t status; + struct completion done; +}; + +static void fscrypt_zeroout_range_done(struct fscrypt_zero_done *done) +{ + if (atomic_dec_and_test(&done->pending)) + complete(&done->done); +} + +static void fscrypt_zeroout_range_end_io(struct bio *bio) +{ + struct fscrypt_zero_done *done = bio->bi_private; + + if (bio->bi_status) + cmpxchg(&done->status, 0, bio->bi_status); + fscrypt_zeroout_range_done(done); + bio_put(bio); +} + +/** + * fscrypt_zeroout_range() - zero out a range of blocks in an encrypted file + * @inode: the file's inode + * @pos: the first file position (in bytes) to zero out + * @sector: the first sector to zero out + * @len: bytes to zero out + * + * Zero out filesystem blocks in an encrypted regular file on-disk, i.e. write + * ciphertext blocks which decrypt to the all-zeroes block. The blocks must be + * both logically and physically contiguous. It's also assumed that the + * filesystem only uses a single block device, ->s_bdev. @len must be a + * multiple of the file system logical block size. + * + * Note that since each block uses a different IV, this involves writing a + * different ciphertext to each block; we can't simply reuse the same one. + * + * Return: 0 on success; -errno on failure. + */ +int fscrypt_zeroout_range(const struct inode *inode, loff_t pos, + sector_t sector, u64 len) +{ + struct fscrypt_zero_done done = { + .pending = ATOMIC_INIT(1), + .done = COMPLETION_INITIALIZER_ONSTACK(done.done), + }; + + if (len == 0) + return 0; + + do { + struct bio *bio; + unsigned int n; + + bio = bio_alloc(inode->i_sb->s_bdev, BIO_MAX_VECS, REQ_OP_WRITE, + GFP_NOFS); + bio->bi_iter.bi_sector = sector; + bio->bi_private = &done; + bio->bi_end_io = fscrypt_zeroout_range_end_io; + fscrypt_set_bio_crypt_ctx(bio, inode, pos, GFP_NOFS); + + for (n = 0; n < BIO_MAX_VECS; n++) { + unsigned int bytes_this_page = min(len, PAGE_SIZE); + + __bio_add_page(bio, ZERO_PAGE(0), bytes_this_page, 0); + len -= bytes_this_page; + pos += bytes_this_page; + sector += (bytes_this_page >> SECTOR_SHIFT); + if (!len || !fscrypt_mergeable_bio(bio, inode, pos)) + break; + } + + atomic_inc(&done.pending); + blk_crypto_submit_bio(bio); + } while (len); + + fscrypt_zeroout_range_done(&done); + + wait_for_completion(&done.done); + return blk_status_to_errno(done.status); +} +EXPORT_SYMBOL(fscrypt_zeroout_range); diff --git a/fs/crypto/crypto.c b/fs/crypto/crypto.c new file mode 100644 index 000000000..5286a124b --- /dev/null +++ b/fs/crypto/crypto.c @@ -0,0 +1,346 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This contains encryption functions for per-file encryption. + * + * Copyright (C) 2015, Google, Inc. + * Copyright (C) 2015, Motorola Mobility + * + * Written by Michael Halcrow, 2014. + * + * Filename encryption additions + * Uday Savagaonkar, 2014 + * Encryption policy handling additions + * Ildar Muslukhov, 2014 + * Add fscrypt_pullback_bio_page() + * Jaegeuk Kim, 2015. + * + * This has not yet undergone a rigorous security audit. + * + * The usage of AES-XTS should conform to recommendations in NIST + * Special Publication 800-38E and IEEE P1619/D16. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "fscrypt_private.h" + +static unsigned int num_prealloc_crypto_pages = 32; + +module_param(num_prealloc_crypto_pages, uint, 0444); +MODULE_PARM_DESC(num_prealloc_crypto_pages, + "Number of crypto pages to preallocate"); + +static mempool_t *fscrypt_bounce_page_pool = NULL; + +static DEFINE_MUTEX(fscrypt_init_mutex); + +struct kmem_cache *fscrypt_inode_info_cachep; + +static struct page *fscrypt_alloc_bounce_page(gfp_t gfp_flags) +{ + if (WARN_ON_ONCE(!fscrypt_bounce_page_pool)) { + /* + * Oops, the filesystem called a function that uses the bounce + * page pool, but it didn't set needs_bounce_pages. + */ + return NULL; + } + return mempool_alloc(fscrypt_bounce_page_pool, gfp_flags); +} + +/** + * fscrypt_free_bounce_page() - free a ciphertext bounce page + * @bounce_page: the bounce page to free, or NULL + * + * Free a bounce page that was allocated by fscrypt_encrypt_pagecache_blocks(). + */ +void fscrypt_free_bounce_page(struct page *bounce_page) +{ + if (!bounce_page) + return; + set_page_private(bounce_page, (unsigned long)NULL); + ClearPagePrivate(bounce_page); + mempool_free(bounce_page, fscrypt_bounce_page_pool); +} +EXPORT_SYMBOL(fscrypt_free_bounce_page); + +/* + * Generate the IV for the given data unit index within the given file. + * For filenames encryption, index == 0. + * + * Keep this in sync with fscrypt_limit_io_blocks(). fscrypt_limit_io_blocks() + * needs to know about any IV generation methods where the low bits of IV don't + * simply contain the data unit index (e.g., IV_INO_LBLK_32). + */ +void fscrypt_generate_iv(union fscrypt_iv *iv, u64 index, + const struct fscrypt_inode_info *ci) +{ + u8 flags = fscrypt_policy_flags(&ci->ci_policy); + + memset(iv, 0, sizeof(*iv)); + + if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64) { + WARN_ON_ONCE(index > U32_MAX); + WARN_ON_ONCE(ci->ci_inode->i_ino > U32_MAX); + index |= (u64)ci->ci_inode->i_ino << 32; + } else if (flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32) { + WARN_ON_ONCE(index > U32_MAX); + index = (u32)(ci->ci_hashed_ino + index); + } else if (flags & FSCRYPT_POLICY_FLAG_DIRECT_KEY) { + memcpy(iv->nonce, ci->ci_nonce, FSCRYPT_FILE_NONCE_SIZE); + } + iv->index = cpu_to_le64(index); +} + +/* Encrypt or decrypt a single "data unit" of file contents. */ +static int fscrypt_crypt_data_unit(const struct fscrypt_inode_info *ci, + fscrypt_direction_t rw, u64 index, + struct page *src_page, + struct page *dest_page, unsigned int len, + unsigned int offs) +{ + struct crypto_sync_skcipher *tfm; + union fscrypt_iv iv; + struct scatterlist dst, src; + int err; + + if (WARN_ON_ONCE(ci == NULL)) /* File hasn't been opened yet? */ + return -ENOKEY; + tfm = ci->ci_enc_key.tfm; + if (WARN_ON_ONCE(tfm == NULL)) /* Called on block-based filesystem? */ + return -ENOKEY; + + if (WARN_ON_ONCE(len <= 0)) + return -EINVAL; + if (WARN_ON_ONCE(len % FSCRYPT_CONTENTS_ALIGNMENT != 0)) + return -EINVAL; + + fscrypt_generate_iv(&iv, index, ci); + + { + SYNC_SKCIPHER_REQUEST_ON_STACK(req, tfm); + skcipher_request_set_callback(req, + CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP, + NULL, NULL); + sg_init_table(&dst, 1); + sg_set_page(&dst, dest_page, len, offs); + sg_init_table(&src, 1); + sg_set_page(&src, src_page, len, offs); + skcipher_request_set_crypt(req, &src, &dst, len, &iv); + if (rw == FS_DECRYPT) + err = crypto_skcipher_decrypt(req); + else + err = crypto_skcipher_encrypt(req); + } + if (err) + fscrypt_err(ci->ci_inode, + "%scryption failed for data unit %llu: %d", + (rw == FS_DECRYPT ? "De" : "En"), index, err); + return err; +} + +/** + * fscrypt_encrypt_pagecache_blocks() - Encrypt data from a pagecache folio + * @folio: the locked pagecache folio containing the data to encrypt + * @len: size of the data to encrypt, in bytes + * @offs: offset within @page of the data to encrypt, in bytes + * @gfp_flags: memory allocation flags; see details below + * + * This allocates a new bounce page and encrypts the given data into it. The + * length and offset of the data must be aligned to the file's crypto data unit + * size. Alignment to the filesystem block size fulfills this requirement, as + * the filesystem block size is always a multiple of the data unit size. + * + * In the bounce page, the ciphertext data will be located at the same offset at + * which the plaintext data was located in the source page. Any other parts of + * the bounce page will be left uninitialized. + * + * This is for use by the ->writepages() method of non-block-based filesystems. + * + * The bounce page allocation is mempool-backed, so it will always succeed when + * @gfp_flags includes __GFP_DIRECT_RECLAIM, e.g. when it's GFP_NOFS. However, + * only the first page of each bio can be allocated this way. To prevent + * deadlocks, for any additional pages a mask like GFP_NOWAIT must be used. + * + * Return: the new encrypted bounce page on success; an ERR_PTR() on failure + */ +struct page *fscrypt_encrypt_pagecache_blocks(struct folio *folio, + size_t len, size_t offs, gfp_t gfp_flags) +{ + const struct inode *inode = folio->mapping->host; + const struct fscrypt_inode_info *ci = fscrypt_get_inode_info_raw(inode); + unsigned int du_bits; + unsigned int du_size; + struct page *ciphertext_page; + u64 index; + unsigned int i; + int err; + + if (WARN_ON_ONCE(ci == NULL)) /* File hasn't been opened yet? */ + return ERR_PTR(-ENOKEY); + + du_bits = ci->ci_data_unit_bits; + du_size = 1U << du_bits; + index = (folio_pos(folio) + offs) >> du_bits; + + VM_BUG_ON_FOLIO(folio_test_large(folio), folio); + if (WARN_ON_ONCE(!folio_test_locked(folio))) + return ERR_PTR(-EINVAL); + + if (WARN_ON_ONCE(len <= 0 || !IS_ALIGNED(len | offs, du_size))) + return ERR_PTR(-EINVAL); + + ciphertext_page = fscrypt_alloc_bounce_page(gfp_flags); + if (!ciphertext_page) + return ERR_PTR(-ENOMEM); + + for (i = offs; i < offs + len; i += du_size, index++) { + err = fscrypt_crypt_data_unit(ci, FS_ENCRYPT, index, + &folio->page, ciphertext_page, + du_size, i); + if (err) { + fscrypt_free_bounce_page(ciphertext_page); + return ERR_PTR(err); + } + } + SetPagePrivate(ciphertext_page); + set_page_private(ciphertext_page, (unsigned long)folio); + return ciphertext_page; +} +EXPORT_SYMBOL(fscrypt_encrypt_pagecache_blocks); + +/** + * fscrypt_encrypt_block_inplace() - Encrypt a filesystem block in-place + * @inode: The inode to which this block belongs + * @page: The page containing the block to encrypt + * @len: Size of block to encrypt. This must be a multiple of + * FSCRYPT_CONTENTS_ALIGNMENT. + * @offs: Byte offset within @page at which the block to encrypt begins + * @lblk_num: Filesystem logical block number of the block, i.e. the 0-based + * number of the block within the file + * + * Encrypt a possibly-compressed filesystem block that is located in an + * arbitrary page, not necessarily in the original pagecache page. The @inode + * and @lblk_num must be specified, as they can't be determined from @page. + * + * This function only supports non-block-based filesystems that don't support + * sub-block data units (as indicated by the fscrypt_operations fields). + * + * Return: 0 on success; -errno on failure + */ +int fscrypt_encrypt_block_inplace(const struct inode *inode, struct page *page, + unsigned int len, unsigned int offs, + u64 lblk_num) +{ + if (WARN_ON_ONCE(inode->i_sb->s_cop->supports_subblock_data_units)) + return -EOPNOTSUPP; + return fscrypt_crypt_data_unit(fscrypt_get_inode_info_raw(inode), + FS_ENCRYPT, lblk_num, page, page, len, + offs); +} +EXPORT_SYMBOL(fscrypt_encrypt_block_inplace); + +/** + * fscrypt_decrypt_block_inplace() - Decrypt a filesystem block in-place + * @inode: The inode to which this block belongs + * @page: The page containing the block to decrypt + * @len: Size of block to decrypt. This must be a multiple of + * FSCRYPT_CONTENTS_ALIGNMENT. + * @offs: Byte offset within @page at which the block to decrypt begins + * @lblk_num: Filesystem logical block number of the block, i.e. the 0-based + * number of the block within the file + * + * Decrypt a possibly-compressed filesystem block that is located in an + * arbitrary page, not necessarily in the original pagecache page. The @inode + * and @lblk_num must be specified, as they can't be determined from @page. + * + * This function only supports non-block-based filesystems that don't support + * sub-block data units (as indicated by the fscrypt_operations fields). + * + * Return: 0 on success; -errno on failure + */ +int fscrypt_decrypt_block_inplace(const struct inode *inode, struct page *page, + unsigned int len, unsigned int offs, + u64 lblk_num) +{ + if (WARN_ON_ONCE(inode->i_sb->s_cop->supports_subblock_data_units)) + return -EOPNOTSUPP; + return fscrypt_crypt_data_unit(fscrypt_get_inode_info_raw(inode), + FS_DECRYPT, lblk_num, page, page, len, + offs); +} +EXPORT_SYMBOL(fscrypt_decrypt_block_inplace); + +/** + * fscrypt_initialize() - allocate major buffers for fs encryption. + * @sb: the filesystem superblock + * + * We only call this when we start accessing encrypted files, since it + * results in memory getting allocated that wouldn't otherwise be used. + * + * Return: 0 on success; -errno on failure + */ +int fscrypt_initialize(struct super_block *sb) +{ + mempool_t *pool; + + /* pairs with smp_store_release() below */ + if (smp_load_acquire(&fscrypt_bounce_page_pool)) + return 0; + + /* No need to allocate a bounce page pool if this FS won't use it. */ + if (!sb->s_cop->needs_bounce_pages) + return 0; + + guard(mutex)(&fscrypt_init_mutex); + if (fscrypt_bounce_page_pool) + return 0; + + pool = mempool_create_page_pool(num_prealloc_crypto_pages, 0); + if (!pool) + return -ENOMEM; + /* pairs with smp_load_acquire() above */ + smp_store_release(&fscrypt_bounce_page_pool, pool); + return 0; +} + +void fscrypt_msg(const struct inode *inode, const char *level, + const char *fmt, ...) +{ + static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + struct va_format vaf; + va_list args; + + if (!__ratelimit(&rs)) + return; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + if (inode && inode->i_ino) + printk("%sfscrypt (%s, inode %llu): %pV\n", + level, inode->i_sb->s_id, inode->i_ino, &vaf); + else if (inode) + printk("%sfscrypt (%s): %pV\n", level, inode->i_sb->s_id, &vaf); + else + printk("%sfscrypt: %pV\n", level, &vaf); + va_end(args); +} + +static int __init fscrypt_init(void) +{ + fscrypt_inode_info_cachep = KMEM_CACHE(fscrypt_inode_info, + SLAB_RECLAIM_ACCOUNT | + SLAB_PANIC); + fscrypt_init_keyring(); + return 0; +} +late_initcall(fscrypt_init) diff --git a/fs/crypto/fname.c b/fs/crypto/fname.c new file mode 100644 index 000000000..629eb0d72 --- /dev/null +++ b/fs/crypto/fname.c @@ -0,0 +1,514 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * This contains functions for filename crypto management + * + * Copyright (C) 2015, Google, Inc. + * Copyright (C) 2015, Motorola Mobility + * + * Written by Uday Savagaonkar, 2014. + * Modified by Jaegeuk Kim, 2015. + * + * This has not yet undergone a rigorous security audit. + */ + +#include +#include +#include +#include +#include +#include + +#include "fscrypt_private.h" + +/* + * The minimum message length (input and output length), in bytes, for all + * filenames encryption modes. Filenames shorter than this will be zero-padded + * before being encrypted. + */ +#define FSCRYPT_FNAME_MIN_MSG_LEN 16 + +/* + * struct fscrypt_nokey_name - identifier for directory entry when key is absent + * + * When userspace lists an encrypted directory without access to the key, the + * filesystem must present a unique "no-key name" for each filename that allows + * it to find the directory entry again if requested. Naively, that would just + * mean using the ciphertext filenames. However, since the ciphertext filenames + * can contain illegal characters ('\0' and '/'), they must be encoded in some + * way. We use base64url. But that can cause names to exceed NAME_MAX (255 + * bytes), so we also need to use a strong hash to abbreviate long names. + * + * The filesystem may also need another kind of hash, the "dirhash", to quickly + * find the directory entry. Since filesystems normally compute the dirhash + * over the on-disk filename (i.e. the ciphertext), it's not computable from + * no-key names that abbreviate the ciphertext using the strong hash to fit in + * NAME_MAX. It's also not computable if it's a keyed hash taken over the + * plaintext (but it may still be available in the on-disk directory entry); + * casefolded directories use this type of dirhash. At least in these cases, + * each no-key name must include the name's dirhash too. + * + * To meet all these requirements, we base64url-encode the following + * variable-length structure. It contains the dirhash, or 0's if the filesystem + * didn't provide one; up to 149 bytes of the ciphertext name; and for + * ciphertexts longer than 149 bytes, also the SHA-256 of the remaining bytes. + * + * This ensures that each no-key name contains everything needed to find the + * directory entry again, contains only legal characters, doesn't exceed + * NAME_MAX, is unambiguous unless there's a SHA-256 collision, and that we only + * take the performance hit of SHA-256 on very long filenames (which are rare). + */ +struct fscrypt_nokey_name { + u32 dirhash[2]; + u8 bytes[149]; + u8 sha256[SHA256_DIGEST_SIZE]; +}; /* 189 bytes => 252 bytes base64url-encoded, which is <= NAME_MAX (255) */ + +/* + * Decoded size of max-size no-key name, i.e. a name that was abbreviated using + * the strong hash and thus includes the 'sha256' field. This isn't simply + * sizeof(struct fscrypt_nokey_name), as the padding at the end isn't included. + */ +#define FSCRYPT_NOKEY_NAME_MAX offsetofend(struct fscrypt_nokey_name, sha256) + +/* Encoded size of max-size no-key name */ +#define FSCRYPT_NOKEY_NAME_MAX_ENCODED \ + BASE64_CHARS(FSCRYPT_NOKEY_NAME_MAX) + +static inline bool fscrypt_is_dot_dotdot(const struct qstr *str) +{ + return name_is_dot_dotdot(str->name, str->len); +} + +/** + * fscrypt_fname_encrypt() - encrypt a filename + * @inode: inode of the parent directory (for regular filenames) + * or of the symlink (for symlink targets). Key must already be + * set up. + * @iname: the filename to encrypt + * @out: (output) the encrypted filename + * @olen: size of the encrypted filename. It must be at least @iname->len. + * Any extra space is filled with NUL padding before encryption. + * + * Return: 0 on success, -errno on failure + */ +int fscrypt_fname_encrypt(const struct inode *inode, const struct qstr *iname, + u8 *out, unsigned int olen) +{ + const struct fscrypt_inode_info *ci = fscrypt_get_inode_info_raw(inode); + struct crypto_sync_skcipher *tfm = ci->ci_enc_key.tfm; + SYNC_SKCIPHER_REQUEST_ON_STACK(req, tfm); + union fscrypt_iv iv; + struct scatterlist sg; + int err; + + /* + * Copy the filename to the output buffer for encrypting in-place and + * pad it with the needed number of NUL bytes. + */ + if (WARN_ON_ONCE(olen < iname->len)) + return -ENOBUFS; + memcpy(out, iname->name, iname->len); + memset(out + iname->len, 0, olen - iname->len); + + fscrypt_generate_iv(&iv, 0, ci); + + skcipher_request_set_callback( + req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, + NULL, NULL); + sg_init_one(&sg, out, olen); + skcipher_request_set_crypt(req, &sg, &sg, olen, &iv); + err = crypto_skcipher_encrypt(req); + if (err) + fscrypt_err(inode, "Filename encryption failed: %d", err); + return err; +} +EXPORT_SYMBOL_GPL(fscrypt_fname_encrypt); + +/** + * fname_decrypt() - decrypt a filename + * @inode: inode of the parent directory (for regular filenames) + * or of the symlink (for symlink targets) + * @iname: the encrypted filename to decrypt + * @oname: (output) the decrypted filename. The caller must have allocated + * enough space for this, e.g. using fscrypt_fname_alloc_buffer(). + * + * Return: 0 on success, -errno on failure + */ +static int fname_decrypt(const struct inode *inode, + const struct fscrypt_str *iname, + struct fscrypt_str *oname) +{ + const struct fscrypt_inode_info *ci = fscrypt_get_inode_info_raw(inode); + struct crypto_sync_skcipher *tfm = ci->ci_enc_key.tfm; + SYNC_SKCIPHER_REQUEST_ON_STACK(req, tfm); + union fscrypt_iv iv; + struct scatterlist src_sg, dst_sg; + int err; + + fscrypt_generate_iv(&iv, 0, ci); + + skcipher_request_set_callback( + req, CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, + NULL, NULL); + sg_init_one(&src_sg, iname->name, iname->len); + sg_init_one(&dst_sg, oname->name, oname->len); + skcipher_request_set_crypt(req, &src_sg, &dst_sg, iname->len, &iv); + err = crypto_skcipher_decrypt(req); + if (err) { + fscrypt_err(inode, "Filename decryption failed: %d", err); + return err; + } + + oname->len = strnlen(oname->name, iname->len); + return 0; +} + +bool __fscrypt_fname_encrypted_size(const union fscrypt_policy *policy, + u32 orig_len, u32 max_len, + u32 *encrypted_len_ret) +{ + int padding = 4 << (fscrypt_policy_flags(policy) & + FSCRYPT_POLICY_FLAGS_PAD_MASK); + u32 encrypted_len; + + if (orig_len > max_len) + return false; + encrypted_len = max_t(u32, orig_len, FSCRYPT_FNAME_MIN_MSG_LEN); + encrypted_len = round_up(encrypted_len, padding); + *encrypted_len_ret = min(encrypted_len, max_len); + return true; +} + +/** + * fscrypt_fname_encrypted_size() - calculate length of encrypted filename + * @inode: parent inode of dentry name being encrypted. Key must + * already be set up. + * @orig_len: length of the original filename + * @max_len: maximum length to return + * @encrypted_len_ret: where calculated length should be returned (on success) + * + * Filenames that are shorter than the maximum length may have their lengths + * increased slightly by encryption, due to padding that is applied. + * + * Return: false if the orig_len is greater than max_len. Otherwise, true and + * fill out encrypted_len_ret with the length (up to max_len). + */ +bool fscrypt_fname_encrypted_size(const struct inode *inode, u32 orig_len, + u32 max_len, u32 *encrypted_len_ret) +{ + const struct fscrypt_inode_info *ci = fscrypt_get_inode_info_raw(inode); + + return __fscrypt_fname_encrypted_size(&ci->ci_policy, orig_len, max_len, + encrypted_len_ret); +} +EXPORT_SYMBOL_GPL(fscrypt_fname_encrypted_size); + +/** + * fscrypt_fname_alloc_buffer() - allocate a buffer for presented filenames + * @max_encrypted_len: maximum length of encrypted filenames the buffer will be + * used to present + * @crypto_str: (output) buffer to allocate + * + * Allocate a buffer that is large enough to hold any decrypted or encoded + * filename (null-terminated), for the given maximum encrypted filename length. + * + * Return: 0 on success, -errno on failure + */ +int fscrypt_fname_alloc_buffer(u32 max_encrypted_len, + struct fscrypt_str *crypto_str) +{ + u32 max_presented_len = max_t(u32, FSCRYPT_NOKEY_NAME_MAX_ENCODED, + max_encrypted_len); + + crypto_str->name = kmalloc(max_presented_len + 1, GFP_NOFS); + if (!crypto_str->name) + return -ENOMEM; + crypto_str->len = max_presented_len; + return 0; +} +EXPORT_SYMBOL(fscrypt_fname_alloc_buffer); + +/** + * fscrypt_fname_free_buffer() - free a buffer for presented filenames + * @crypto_str: the buffer to free + * + * Free a buffer that was allocated by fscrypt_fname_alloc_buffer(). + */ +void fscrypt_fname_free_buffer(struct fscrypt_str *crypto_str) +{ + if (!crypto_str) + return; + kfree(crypto_str->name); + crypto_str->name = NULL; +} +EXPORT_SYMBOL(fscrypt_fname_free_buffer); + +/** + * fscrypt_fname_disk_to_usr() - convert an encrypted filename to + * user-presentable form + * @inode: inode of the parent directory (for regular filenames) + * or of the symlink (for symlink targets) + * @hash: first part of the name's dirhash, if applicable. This only needs to + * be provided if the filename is located in an indexed directory whose + * encryption key may be unavailable. Not needed for symlink targets. + * @minor_hash: second part of the name's dirhash, if applicable + * @iname: encrypted filename to convert. May also be "." or "..", which + * aren't actually encrypted. + * @oname: output buffer for the user-presentable filename. The caller must + * have allocated enough space for this, e.g. using + * fscrypt_fname_alloc_buffer(). + * + * If the key is available, we'll decrypt the disk name. Otherwise, we'll + * encode it for presentation in fscrypt_nokey_name format. + * See struct fscrypt_nokey_name for details. + * + * Return: 0 on success, -errno on failure + */ +int fscrypt_fname_disk_to_usr(const struct inode *inode, + u32 hash, u32 minor_hash, + const struct fscrypt_str *iname, + struct fscrypt_str *oname) +{ + const struct qstr qname = FSTR_TO_QSTR(iname); + struct fscrypt_nokey_name nokey_name; + u32 size; /* size of the unencoded no-key name */ + + if (fscrypt_is_dot_dotdot(&qname)) { + oname->name[0] = '.'; + oname->name[iname->len - 1] = '.'; + oname->len = iname->len; + return 0; + } + + if (iname->len < FSCRYPT_FNAME_MIN_MSG_LEN) + return -EUCLEAN; + + if (fscrypt_has_encryption_key(inode)) + return fname_decrypt(inode, iname, oname); + + /* + * Sanity check that struct fscrypt_nokey_name doesn't have padding + * between fields and that its encoded size never exceeds NAME_MAX. + */ + BUILD_BUG_ON(offsetofend(struct fscrypt_nokey_name, dirhash) != + offsetof(struct fscrypt_nokey_name, bytes)); + BUILD_BUG_ON(offsetofend(struct fscrypt_nokey_name, bytes) != + offsetof(struct fscrypt_nokey_name, sha256)); + BUILD_BUG_ON(FSCRYPT_NOKEY_NAME_MAX_ENCODED > NAME_MAX); + + nokey_name.dirhash[0] = hash; + nokey_name.dirhash[1] = minor_hash; + + if (iname->len <= sizeof(nokey_name.bytes)) { + memcpy(nokey_name.bytes, iname->name, iname->len); + size = offsetof(struct fscrypt_nokey_name, bytes[iname->len]); + } else { + memcpy(nokey_name.bytes, iname->name, sizeof(nokey_name.bytes)); + /* Compute strong hash of remaining part of name. */ + sha256(&iname->name[sizeof(nokey_name.bytes)], + iname->len - sizeof(nokey_name.bytes), + nokey_name.sha256); + size = FSCRYPT_NOKEY_NAME_MAX; + } + oname->len = base64_encode((const u8 *)&nokey_name, size, + oname->name, false, BASE64_URLSAFE); + return 0; +} +EXPORT_SYMBOL(fscrypt_fname_disk_to_usr); + +/** + * fscrypt_setup_filename() - prepare to search a possibly encrypted directory + * @dir: the directory that will be searched + * @iname: the user-provided filename being searched for + * @lookup: 1 if we're allowed to proceed without the key because it's + * ->lookup() or we're finding the dir_entry for deletion; 0 if we cannot + * proceed without the key because we're going to create the dir_entry. + * @fname: the filename information to be filled in + * + * Given a user-provided filename @iname, this function sets @fname->disk_name + * to the name that would be stored in the on-disk directory entry, if possible. + * If the directory is unencrypted this is simply @iname. Else, if we have the + * directory's encryption key, then @iname is the plaintext, so we encrypt it to + * get the disk_name. + * + * Else, for keyless @lookup operations, @iname should be a no-key name, so we + * decode it to get the struct fscrypt_nokey_name. Non-@lookup operations will + * be impossible in this case, so we fail them with ENOKEY. + * + * If successful, fscrypt_free_filename() must be called later to clean up. + * + * Return: 0 on success, -errno on failure + */ +int fscrypt_setup_filename(struct inode *dir, const struct qstr *iname, + int lookup, struct fscrypt_name *fname) +{ + struct fscrypt_nokey_name *nokey_name; + int ret; + + memset(fname, 0, sizeof(struct fscrypt_name)); + fname->usr_fname = iname; + + if (!IS_ENCRYPTED(dir) || fscrypt_is_dot_dotdot(iname)) { + fname->disk_name.name = (unsigned char *)iname->name; + fname->disk_name.len = iname->len; + return 0; + } + ret = fscrypt_get_encryption_info(dir, lookup); + if (ret) + return ret; + + if (fscrypt_has_encryption_key(dir)) { + if (!fscrypt_fname_encrypted_size(dir, iname->len, NAME_MAX, + &fname->crypto_buf.len)) + return -ENAMETOOLONG; + fname->crypto_buf.name = kmalloc(fname->crypto_buf.len, + GFP_NOFS); + if (!fname->crypto_buf.name) + return -ENOMEM; + + ret = fscrypt_fname_encrypt(dir, iname, fname->crypto_buf.name, + fname->crypto_buf.len); + if (ret) + goto errout; + fname->disk_name.name = fname->crypto_buf.name; + fname->disk_name.len = fname->crypto_buf.len; + return 0; + } + if (!lookup) + return -ENOKEY; + fname->is_nokey_name = true; + + /* + * We don't have the key and we are doing a lookup; decode the + * user-supplied name + */ + + if (iname->len > FSCRYPT_NOKEY_NAME_MAX_ENCODED) + return -ENOENT; + + fname->crypto_buf.name = kmalloc(FSCRYPT_NOKEY_NAME_MAX, GFP_KERNEL); + if (fname->crypto_buf.name == NULL) + return -ENOMEM; + + ret = base64_decode(iname->name, iname->len, + fname->crypto_buf.name, false, BASE64_URLSAFE); + if (ret < (int)offsetof(struct fscrypt_nokey_name, bytes[1]) || + (ret > offsetof(struct fscrypt_nokey_name, sha256) && + ret != FSCRYPT_NOKEY_NAME_MAX)) { + ret = -ENOENT; + goto errout; + } + fname->crypto_buf.len = ret; + + nokey_name = (void *)fname->crypto_buf.name; + fname->hash = nokey_name->dirhash[0]; + fname->minor_hash = nokey_name->dirhash[1]; + if (ret != FSCRYPT_NOKEY_NAME_MAX) { + /* The full ciphertext filename is available. */ + fname->disk_name.name = nokey_name->bytes; + fname->disk_name.len = + ret - offsetof(struct fscrypt_nokey_name, bytes); + } + return 0; + +errout: + kfree(fname->crypto_buf.name); + return ret; +} +EXPORT_SYMBOL(fscrypt_setup_filename); + +/** + * fscrypt_match_name() - test whether the given name matches a directory entry + * @fname: the name being searched for + * @de_name: the name from the directory entry + * @de_name_len: the length of @de_name in bytes + * + * Normally @fname->disk_name will be set, and in that case we simply compare + * that to the name stored in the directory entry. The only exception is that + * if we don't have the key for an encrypted directory and the name we're + * looking for is very long, then we won't have the full disk_name and instead + * we'll need to match against a fscrypt_nokey_name that includes a strong hash. + * + * Return: %true if the name matches, otherwise %false. + */ +bool fscrypt_match_name(const struct fscrypt_name *fname, + const u8 *de_name, u32 de_name_len) +{ + const struct fscrypt_nokey_name *nokey_name = + (const void *)fname->crypto_buf.name; + u8 digest[SHA256_DIGEST_SIZE]; + + if (likely(fname->disk_name.name)) { + if (de_name_len != fname->disk_name.len) + return false; + return !memcmp(de_name, fname->disk_name.name, de_name_len); + } + if (de_name_len <= sizeof(nokey_name->bytes)) + return false; + if (memcmp(de_name, nokey_name->bytes, sizeof(nokey_name->bytes))) + return false; + sha256(&de_name[sizeof(nokey_name->bytes)], + de_name_len - sizeof(nokey_name->bytes), digest); + return !memcmp(digest, nokey_name->sha256, sizeof(digest)); +} +EXPORT_SYMBOL_GPL(fscrypt_match_name); + +/** + * fscrypt_fname_siphash() - calculate the SipHash of a filename + * @dir: the parent directory + * @name: the filename to calculate the SipHash of + * + * Given a plaintext filename @name and a directory @dir which uses SipHash as + * its dirhash method and has had its fscrypt key set up, this function + * calculates the SipHash of that name using the directory's secret dirhash key. + * + * Return: the SipHash of @name using the hash key of @dir + */ +u64 fscrypt_fname_siphash(const struct inode *dir, const struct qstr *name) +{ + const struct fscrypt_inode_info *ci = fscrypt_get_inode_info_raw(dir); + + WARN_ON_ONCE(!ci->ci_dirhash_key_initialized); + + return siphash(name->name, name->len, &ci->ci_dirhash_key); +} +EXPORT_SYMBOL_GPL(fscrypt_fname_siphash); + +/* + * Validate dentries in encrypted directories to make sure we aren't potentially + * caching stale dentries after a key has been added. + */ +int fscrypt_d_revalidate(struct inode *dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + int err; + + /* + * Plaintext names are always valid, since fscrypt doesn't support + * reverting to no-key names without evicting the directory's inode + * -- which implies eviction of the dentries in the directory. + */ + if (!(dentry->d_flags & DCACHE_NOKEY_NAME)) + return 1; + + /* + * No-key name; valid if the directory's key is still unavailable. + * + * Note in RCU mode we have to bail if we get here - + * fscrypt_get_encryption_info() may block. + */ + + if (flags & LOOKUP_RCU) + return -ECHILD; + + /* + * Pass allow_unsupported=true, so that files with an unsupported + * encryption policy can be deleted. + */ + err = fscrypt_get_encryption_info(dir, true); + if (err < 0) + return err; + + return !fscrypt_has_encryption_key(dir); +} +EXPORT_SYMBOL_GPL(fscrypt_d_revalidate); diff --git a/fs/crypto/fscrypt_private.h b/fs/crypto/fscrypt_private.h new file mode 100644 index 000000000..74329e095 --- /dev/null +++ b/fs/crypto/fscrypt_private.h @@ -0,0 +1,783 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * fscrypt_private.h + * + * Copyright (C) 2015, Google, Inc. + * + * Originally written by Michael Halcrow, Ildar Muslukhov, and Uday Savagaonkar. + * Heavily modified since then. + */ + +#ifndef _FSCRYPT_PRIVATE_H +#define _FSCRYPT_PRIVATE_H + +#include +#include +#include +#include +#include + +#define CONST_STRLEN(str) (sizeof(str) - 1) + +#define FSCRYPT_FILE_NONCE_SIZE 16 + +/* + * Minimum size of an fscrypt master key. Note: a longer key will be required + * if ciphers with a 256-bit security strength are used. This is just the + * absolute minimum, which applies when only 128-bit encryption is used. + */ +#define FSCRYPT_MIN_KEY_SIZE 16 + +/* Maximum size of a raw fscrypt master key */ +#define FSCRYPT_MAX_RAW_KEY_SIZE 64 + +/* Maximum size of a hardware-wrapped fscrypt master key */ +#define FSCRYPT_MAX_HW_WRAPPED_KEY_SIZE BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE + +/* Maximum size of an fscrypt master key across both key types */ +#define FSCRYPT_MAX_ANY_KEY_SIZE \ + MAX(FSCRYPT_MAX_RAW_KEY_SIZE, FSCRYPT_MAX_HW_WRAPPED_KEY_SIZE) + +/* + * FSCRYPT_MAX_KEY_SIZE is defined in the UAPI header, but the addition of + * hardware-wrapped keys has made it misleading as it's only for raw keys. + * Don't use it in kernel code; use one of the above constants instead. + */ +#undef FSCRYPT_MAX_KEY_SIZE + +/* + * This mask is passed as the third argument to the crypto_alloc_*() functions + * to prevent fscrypt from using the Crypto API drivers for non-inline crypto + * engines. Those drivers have been problematic for fscrypt. fscrypt users + * have reported hangs and even incorrect en/decryption with these drivers. + * Since going to the driver, off CPU, and back again is really slow, such + * drivers can be over 50 times slower than the CPU-based code for fscrypt's + * workload. Even on platforms that lack AES instructions on the CPU, using the + * offloads has been shown to be slower, even staying with AES. (Of course, + * Adiantum is faster still, and is the recommended option on such platforms...) + * + * Note that fscrypt also supports inline crypto engines. Those don't use the + * Crypto API and work much better than the old-style (non-inline) engines. + */ +#define FSCRYPT_CRYPTOAPI_MASK \ + (CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY | \ + CRYPTO_ALG_KERN_DRIVER_ONLY) + +#define FSCRYPT_CONTEXT_V1 1 +#define FSCRYPT_CONTEXT_V2 2 + +struct fscrypt_context_v1 { + u8 version; /* FSCRYPT_CONTEXT_V1 */ + u8 contents_encryption_mode; + u8 filenames_encryption_mode; + u8 flags; + u8 master_key_descriptor[FSCRYPT_KEY_DESCRIPTOR_SIZE]; + u8 nonce[FSCRYPT_FILE_NONCE_SIZE]; +}; + +struct fscrypt_context_v2 { + u8 version; /* FSCRYPT_CONTEXT_V2 */ + u8 contents_encryption_mode; + u8 filenames_encryption_mode; + u8 flags; + u8 log2_data_unit_size; + u8 __reserved[3]; + u8 master_key_identifier[FSCRYPT_KEY_IDENTIFIER_SIZE]; + u8 nonce[FSCRYPT_FILE_NONCE_SIZE]; +}; + +/* + * fscrypt_context - the encryption context of an inode + * + * This is the on-disk equivalent of an fscrypt_policy, stored alongside each + * encrypted file usually in a hidden extended attribute. It contains the + * fields from the fscrypt_policy, in order to identify the encryption algorithm + * and key with which the file is encrypted. It also contains a nonce that was + * randomly generated by fscrypt itself; this is used as KDF input or as a tweak + * to cause different files to be encrypted differently. + */ +union fscrypt_context { + u8 version; + struct fscrypt_context_v1 v1; + struct fscrypt_context_v2 v2; +}; + +/* + * Return the size expected for the given fscrypt_context based on its version + * number, or 0 if the context version is unrecognized. + */ +static inline int fscrypt_context_size(const union fscrypt_context *ctx) +{ + switch (ctx->version) { + case FSCRYPT_CONTEXT_V1: + BUILD_BUG_ON(sizeof(ctx->v1) != 28); + return sizeof(ctx->v1); + case FSCRYPT_CONTEXT_V2: + BUILD_BUG_ON(sizeof(ctx->v2) != 40); + return sizeof(ctx->v2); + } + return 0; +} + +/* Check whether an fscrypt_context has a recognized version number and size */ +static inline bool fscrypt_context_is_valid(const union fscrypt_context *ctx, + int ctx_size) +{ + return ctx_size >= 1 && ctx_size == fscrypt_context_size(ctx); +} + +/* Retrieve the context's nonce, assuming the context was already validated */ +static inline const u8 *fscrypt_context_nonce(const union fscrypt_context *ctx) +{ + switch (ctx->version) { + case FSCRYPT_CONTEXT_V1: + return ctx->v1.nonce; + case FSCRYPT_CONTEXT_V2: + return ctx->v2.nonce; + } + WARN_ON_ONCE(1); + return NULL; +} + +union fscrypt_policy { + u8 version; + struct fscrypt_policy_v1 v1; + struct fscrypt_policy_v2 v2; +}; + +/* + * Return the size expected for the given fscrypt_policy based on its version + * number, or 0 if the policy version is unrecognized. + */ +static inline int fscrypt_policy_size(const union fscrypt_policy *policy) +{ + switch (policy->version) { + case FSCRYPT_POLICY_V1: + return sizeof(policy->v1); + case FSCRYPT_POLICY_V2: + return sizeof(policy->v2); + } + return 0; +} + +/* Return the contents encryption mode of a valid encryption policy */ +static inline u8 +fscrypt_policy_contents_mode(const union fscrypt_policy *policy) +{ + switch (policy->version) { + case FSCRYPT_POLICY_V1: + return policy->v1.contents_encryption_mode; + case FSCRYPT_POLICY_V2: + return policy->v2.contents_encryption_mode; + } + BUG(); +} + +/* Return the filenames encryption mode of a valid encryption policy */ +static inline u8 +fscrypt_policy_fnames_mode(const union fscrypt_policy *policy) +{ + switch (policy->version) { + case FSCRYPT_POLICY_V1: + return policy->v1.filenames_encryption_mode; + case FSCRYPT_POLICY_V2: + return policy->v2.filenames_encryption_mode; + } + BUG(); +} + +/* Return the flags (FSCRYPT_POLICY_FLAG*) of a valid encryption policy */ +static inline u8 +fscrypt_policy_flags(const union fscrypt_policy *policy) +{ + switch (policy->version) { + case FSCRYPT_POLICY_V1: + return policy->v1.flags; + case FSCRYPT_POLICY_V2: + return policy->v2.flags; + } + BUG(); +} + +static inline int +fscrypt_policy_v2_du_bits(const struct fscrypt_policy_v2 *policy, + const struct inode *inode) +{ + return policy->log2_data_unit_size ?: inode->i_blkbits; +} + +static inline int +fscrypt_policy_du_bits(const union fscrypt_policy *policy, + const struct inode *inode) +{ + switch (policy->version) { + case FSCRYPT_POLICY_V1: + return inode->i_blkbits; + case FSCRYPT_POLICY_V2: + return fscrypt_policy_v2_du_bits(&policy->v2, inode); + } + BUG(); +} + +/* + * For encrypted symlinks, the ciphertext length is stored at the beginning + * of the string in little-endian format. + */ +struct fscrypt_symlink_data { + __le16 len; + char encrypted_path[]; +} __packed; + +/** + * struct fscrypt_prepared_key - a key prepared for actual encryption/decryption + * @tfm: crypto API transform object + * @blk_key: key for blk-crypto + * + * Only one of the fields is non-NULL. + */ +struct fscrypt_prepared_key { + struct crypto_sync_skcipher *tfm; +#ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT + struct blk_crypto_key *blk_key; +#endif +}; + +/* An entry in the linked list ->mk_mode_keys */ +struct fscrypt_mode_key { + struct fscrypt_prepared_key key; + struct list_head link; + u8 hkdf_context; + u8 mode_num; + u8 data_unit_bits; +}; + +/* + * fscrypt_inode_info - the "encryption key" for an inode + * + * When an encrypted file's key is made available, an instance of this struct is + * allocated and a pointer to it is stored in the file's in-memory inode. Once + * created, it remains until the inode is evicted. + */ +struct fscrypt_inode_info { + + /* The key in a form prepared for actual encryption/decryption */ + struct fscrypt_prepared_key ci_enc_key; + + /* True if ci_enc_key should be freed when this struct is freed */ + u8 ci_owns_key : 1; + + /* True if ci_dirhash_key is initialized */ + u8 ci_dirhash_key_initialized : 1; + + /* + * log2 of the data unit size (granularity of contents encryption) of + * this file. This is computable from ci_policy and ci_inode but is + * cached here for efficiency. Only used for regular files. + */ + u8 ci_data_unit_bits; + + /* Hashed inode number. Only set for IV_INO_LBLK_32 */ + u32 ci_hashed_ino; + + /* + * Encryption mode used for this inode. It corresponds to either the + * contents or filenames encryption mode, depending on the inode type. + */ + struct fscrypt_mode *ci_mode; + + /* Back-pointer to the inode */ + struct inode *ci_inode; + + /* + * The master key with which this inode was unlocked (decrypted). This + * will be NULL if the master key was found in a process-subscribed + * keyring rather than in the filesystem-level keyring. + */ + struct fscrypt_master_key *ci_master_key; + + /* + * Link in list of inodes that were unlocked with the master key. + * Only used when ->ci_master_key is set. + */ + struct list_head ci_master_key_link; + + /* + * If non-NULL, then encryption is done using the master key directly + * and ci_enc_key will equal ci_direct_key->dk_key. + */ + struct fscrypt_direct_key *ci_direct_key; + + /* + * This inode's hash key for filenames. This is a 128-bit SipHash-2-4 + * key. This is only set for directories that use a keyed dirhash over + * the plaintext filenames -- currently just casefolded directories. + */ + siphash_key_t ci_dirhash_key; + + /* The encryption policy used by this inode */ + union fscrypt_policy ci_policy; + + /* This inode's nonce, copied from the fscrypt_context */ + u8 ci_nonce[FSCRYPT_FILE_NONCE_SIZE]; +}; + +typedef enum { + FS_DECRYPT = 0, + FS_ENCRYPT, +} fscrypt_direction_t; + +/* crypto.c */ +extern struct kmem_cache *fscrypt_inode_info_cachep; +int fscrypt_initialize(struct super_block *sb); + +void __printf(3, 4) __cold +fscrypt_msg(const struct inode *inode, const char *level, const char *fmt, ...); + +#define fscrypt_warn(inode, fmt, ...) \ + fscrypt_msg((inode), KERN_WARNING, fmt, ##__VA_ARGS__) +#define fscrypt_err(inode, fmt, ...) \ + fscrypt_msg((inode), KERN_ERR, fmt, ##__VA_ARGS__) + +#define FSCRYPT_MAX_IV_SIZE 32 + +union fscrypt_iv { + struct { + /* zero-based index of data unit within the file */ + __le64 index; + + /* per-file nonce; only set in DIRECT_KEY mode */ + u8 nonce[FSCRYPT_FILE_NONCE_SIZE]; + }; + u8 raw[FSCRYPT_MAX_IV_SIZE]; + __le64 dun[FSCRYPT_MAX_IV_SIZE / sizeof(__le64)]; +}; + +void fscrypt_generate_iv(union fscrypt_iv *iv, u64 index, + const struct fscrypt_inode_info *ci); + +/* + * Return the number of bits used by the maximum file data unit index that is + * possible on the given filesystem, using the given log2 data unit size. + */ +static inline int +fscrypt_max_file_dun_bits(const struct super_block *sb, int du_bits) +{ + return fls64(sb->s_maxbytes - 1) - du_bits; +} + +/* fname.c */ +bool __fscrypt_fname_encrypted_size(const union fscrypt_policy *policy, + u32 orig_len, u32 max_len, + u32 *encrypted_len_ret); + +/* hkdf.c */ +void fscrypt_init_hkdf(struct hmac_sha512_key *hkdf, const u8 *master_key, + unsigned int master_key_size); + +/* + * The list of contexts in which fscrypt uses HKDF. These values are used as + * the first byte of the HKDF application-specific info string to guarantee that + * info strings are never repeated between contexts. This ensures that all HKDF + * outputs are unique and cryptographically isolated, i.e. knowledge of one + * output doesn't reveal another. + */ +#define HKDF_CONTEXT_KEY_IDENTIFIER_FOR_RAW_KEY 1 /* info= */ +#define HKDF_CONTEXT_PER_FILE_ENC_KEY 2 /* info=file_nonce */ +#define HKDF_CONTEXT_DIRECT_KEY 3 /* info=mode_num */ +#define HKDF_CONTEXT_IV_INO_LBLK_64_KEY 4 /* info=mode_num||fs_uuid */ +#define HKDF_CONTEXT_DIRHASH_KEY 5 /* info=file_nonce */ +#define HKDF_CONTEXT_IV_INO_LBLK_32_KEY 6 /* info=mode_num||fs_uuid */ +#define HKDF_CONTEXT_INODE_HASH_KEY 7 /* info= */ +#define HKDF_CONTEXT_KEY_IDENTIFIER_FOR_HW_WRAPPED_KEY \ + 8 /* info= */ + +void fscrypt_hkdf_expand(const struct hmac_sha512_key *hkdf, u8 context, + const u8 *info, unsigned int infolen, + u8 *okm, unsigned int okmlen); + +/* block.c */ +#ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT +static inline bool +fscrypt_using_inline_encryption(const struct fscrypt_inode_info *ci) +{ + const struct inode *inode = ci->ci_inode; + + return S_ISREG(inode->i_mode) && inode->i_sb->s_cop->is_block_based; +} + +int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key, + const u8 *key_bytes, size_t key_size, + bool is_hw_wrapped, + const struct fscrypt_inode_info *ci); + +void fscrypt_destroy_inline_crypt_key(struct super_block *sb, + struct fscrypt_prepared_key *prep_key); + +int fscrypt_derive_sw_secret(struct super_block *sb, + const u8 *wrapped_key, size_t wrapped_key_size, + u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE]); + +/* + * Check whether the crypto transform or blk-crypto key has been allocated in + * @prep_key, depending on which encryption implementation the file will use. + */ +static inline bool +fscrypt_is_key_prepared(const struct fscrypt_prepared_key *prep_key, + const struct fscrypt_inode_info *ci) +{ + if (fscrypt_using_inline_encryption(ci)) + return prep_key->blk_key != NULL; + return prep_key->tfm != NULL; +} + +#else /* CONFIG_FS_ENCRYPTION_INLINE_CRYPT */ + +static inline bool +fscrypt_using_inline_encryption(const struct fscrypt_inode_info *ci) +{ + return false; +} + +static inline int +fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key, + const u8 *key_bytes, size_t key_size, + bool is_hw_wrapped, + const struct fscrypt_inode_info *ci) +{ + WARN_ON_ONCE(1); + return -EOPNOTSUPP; +} + +static inline void +fscrypt_destroy_inline_crypt_key(struct super_block *sb, + struct fscrypt_prepared_key *prep_key) +{ +} + +static inline int +fscrypt_derive_sw_secret(struct super_block *sb, + const u8 *wrapped_key, size_t wrapped_key_size, + u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE]) +{ + fscrypt_warn(NULL, "kernel doesn't support hardware-wrapped keys"); + return -EOPNOTSUPP; +} + +static inline bool +fscrypt_is_key_prepared(const struct fscrypt_prepared_key *prep_key, + const struct fscrypt_inode_info *ci) +{ + return prep_key->tfm != NULL; +} +#endif /* !CONFIG_FS_ENCRYPTION_INLINE_CRYPT */ + +/* keyring.c */ + +/* + * fscrypt_master_key_user - a user's claim to a master key + */ +struct fscrypt_master_key_user { + struct list_head link; + kuid_t uid; + /* + * This 'struct key' contains no secret. It exists solely to charge the + * appropriate user's key quota. + */ + struct key *quota_key; +}; + +/* + * fscrypt_master_key_secret - secret key material of an in-use master key + */ +struct fscrypt_master_key_secret { + + /* + * The KDF with which subkeys of this key can be derived. + * + * For v1 policy keys, this isn't applicable and won't be set. + * Otherwise, this KDF will be keyed by this master key if + * ->is_hw_wrapped=false, or by the "software secret" that hardware + * derived from this master key if ->is_hw_wrapped=true. + */ + struct hmac_sha512_key hkdf; + + /* + * True if this key is a hardware-wrapped key; false if this key is a + * raw key (i.e. a "software key"). For v1 policy keys this will always + * be false, as v1 policy support is a legacy feature which doesn't + * support newer functionality such as hardware-wrapped keys. + */ + bool is_hw_wrapped; + + /* + * Size of the key in bytes. This remains set even if ->bytes was + * zeroized due to no longer being needed. I.e. we still remember the + * size of the key even if we don't need to remember the key itself. + */ + u32 size; + + /* + * The bytes of the key, when still needed. This can be either a raw + * key or a hardware-wrapped key, as indicated by ->is_hw_wrapped. In + * the case of a raw, v2 policy key, there is no need to remember the + * actual key separately from ->hkdf so this field will be zeroized as + * soon as ->hkdf is initialized. + */ + u8 bytes[FSCRYPT_MAX_ANY_KEY_SIZE]; + +} __randomize_layout; + +/* + * fscrypt_master_key - an in-use master key + * + * This represents a master encryption key which has been added to the + * filesystem. There are three high-level states that a key can be in: + * + * FSCRYPT_KEY_STATUS_PRESENT + * Key is fully usable; it can be used to unlock inodes that are encrypted + * with it (this includes being able to create new inodes). ->mk_present + * indicates whether the key is in this state. ->mk_secret exists, the key + * is in the keyring, and ->mk_active_refs > 0 due to ->mk_present. + * + * FSCRYPT_KEY_STATUS_INCOMPLETELY_REMOVED + * Removal of this key has been initiated, but some inodes that were + * unlocked with it are still in-use. Like ABSENT, ->mk_secret is wiped, + * and the key can no longer be used to unlock inodes. Unlike ABSENT, the + * key is still in the keyring; ->mk_decrypted_inodes is nonempty; and + * ->mk_active_refs > 0, being equal to the size of ->mk_decrypted_inodes. + * + * This state transitions to ABSENT if ->mk_decrypted_inodes becomes empty, + * or to PRESENT if FS_IOC_ADD_ENCRYPTION_KEY is called again for this key. + * + * FSCRYPT_KEY_STATUS_ABSENT + * Key is fully removed. The key is no longer in the keyring, + * ->mk_decrypted_inodes is empty, ->mk_active_refs == 0, ->mk_secret is + * wiped, and the key can no longer be used to unlock inodes. + */ +struct fscrypt_master_key { + + /* + * Link in ->s_master_keys->key_hashtable. + * Only valid if ->mk_active_refs > 0. + */ + struct hlist_node mk_node; + + /* Semaphore that protects ->mk_secret, ->mk_users, and ->mk_present */ + struct rw_semaphore mk_sem; + + /* + * Active and structural reference counts. An active ref guarantees + * that the struct continues to exist, continues to be in the keyring + * ->s_master_keys, and that any non-file-scoped subkeys (e.g. + * ->mk_mode_keys) that have been prepared continue to exist. + * A structural ref only guarantees that the struct continues to exist. + * + * There is one active ref associated with ->mk_present being true, and + * one active ref for each inode in ->mk_decrypted_inodes. + * + * There is one structural ref associated with the active refcount being + * nonzero. Finding a key in the keyring also takes a structural ref, + * which is then held temporarily while the key is operated on. + */ + refcount_t mk_active_refs; + refcount_t mk_struct_refs; + + struct rcu_head mk_rcu_head; + + /* + * The secret key material. Wiped as soon as it is no longer needed; + * for details, see the fscrypt_master_key struct comment. + * + * Locking: protected by ->mk_sem. + */ + struct fscrypt_master_key_secret mk_secret; + + /* + * For v1 policy keys: an arbitrary key descriptor which was assigned by + * userspace (->descriptor). + * + * For v2 policy keys: a cryptographic hash of this key (->identifier). + */ + struct fscrypt_key_specifier mk_spec; + + /* + * List of user claims to this key (struct fscrypt_master_key_user). + * Normally each key will be added by just one user, but it's possible + * that multiple users share a key, and in that case we need to keep + * track of those users so that one user can't remove the key before the + * others want it removed too. + * + * Used only for v2 policy keys. v1 policy keys can be added only by + * root, so user tracking doesn't apply to them. + * + * Locking: protected by ->mk_sem. + */ + struct list_head mk_users; + + /* + * List of inodes that were unlocked using this key. This allows the + * inodes to be evicted efficiently if the key is removed. + */ + struct list_head mk_decrypted_inodes; + spinlock_t mk_decrypted_inodes_lock; + + /* + * A list of 'struct fscrypt_mode_key' for the (hkdf_context, mode_num, + * data_unit_bits, inlinecrypt) combinations that are in use for this + * master key, for hkdf_context in [HKDF_CONTEXT_DIRECT_KEY, + * HKDF_CONTEXT_IV_INO_LBLK_32_KEY, HKDF_CONTEXT_IV_INO_LBLK_64_KEY]. + * + * This is a linked list and not a hash table because in practice + * there's just a single encryption policy per master key, using + * _at most_ 2 nodes in this list. Per-file keys don't use this at all. + * + * This list is append-only until the master key is fully removed, at + * which time the list is cleared. Before then, + * fscrypt_mode_key_setup_mutex synchronizes appends, and searches use + * the RCU read lock together with ->mk_sem held for read. + */ + struct list_head mk_mode_keys; + + /* Hash key for inode numbers. Initialized only when needed. */ + siphash_key_t mk_ino_hash_key; + bool mk_ino_hash_key_initialized; + + /* + * Whether this key is in the "present" state, i.e. fully usable. For + * details, see the fscrypt_master_key struct comment. + * + * Locking: protected by ->mk_sem, but can be read locklessly using + * READ_ONCE(). Writers must use WRITE_ONCE() when concurrent readers + * are possible. + */ + bool mk_present; + +} __randomize_layout; + +static inline const char *master_key_spec_type( + const struct fscrypt_key_specifier *spec) +{ + switch (spec->type) { + case FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR: + return "descriptor"; + case FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER: + return "identifier"; + } + return "[unknown]"; +} + +static inline int master_key_spec_len(const struct fscrypt_key_specifier *spec) +{ + switch (spec->type) { + case FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR: + return FSCRYPT_KEY_DESCRIPTOR_SIZE; + case FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER: + return FSCRYPT_KEY_IDENTIFIER_SIZE; + } + return 0; +} + +void fscrypt_put_master_key(struct fscrypt_master_key *mk); + +void fscrypt_put_master_key_activeref(struct super_block *sb, + struct fscrypt_master_key *mk); + +struct fscrypt_master_key * +fscrypt_find_master_key(struct super_block *sb, + const struct fscrypt_key_specifier *mk_spec); + +void fscrypt_get_test_dummy_key_identifier( + u8 key_identifier[FSCRYPT_KEY_IDENTIFIER_SIZE]); + +int fscrypt_add_test_dummy_key(struct super_block *sb, + struct fscrypt_key_specifier *key_spec); + +int fscrypt_verify_key_added(struct super_block *sb, + const u8 identifier[FSCRYPT_KEY_IDENTIFIER_SIZE]); + +void __init fscrypt_init_keyring(void); + +/* keysetup.c */ + +struct fscrypt_mode { + const char *friendly_name; + const char *cipher_str; + int keysize; /* key size in bytes */ + int security_strength; /* security strength in bytes */ + int ivsize; /* IV size in bytes */ + int logged_cryptoapi_impl; + int logged_blk_crypto_native; + int logged_blk_crypto_fallback; + enum blk_crypto_mode_num blk_crypto_mode; +}; + +extern struct fscrypt_mode fscrypt_modes[]; + +int fscrypt_prepare_key(struct fscrypt_prepared_key *prep_key, + const u8 *raw_key, const struct fscrypt_inode_info *ci); + +void fscrypt_destroy_prepared_key(struct super_block *sb, + struct fscrypt_prepared_key *prep_key); + +int fscrypt_set_per_file_enc_key(struct fscrypt_inode_info *ci, + const u8 *raw_key); + +void fscrypt_derive_dirhash_key(struct fscrypt_inode_info *ci, + const struct fscrypt_master_key *mk); + +void fscrypt_hash_inode_number(struct fscrypt_inode_info *ci, + const struct fscrypt_master_key *mk); + +int fscrypt_get_encryption_info(struct inode *inode, bool allow_unsupported); + +/** + * fscrypt_require_key() - require an inode's encryption key + * @inode: the inode we need the key for + * + * If the inode is encrypted, set up its encryption key if not already done. + * Then require that the key be present and return -ENOKEY otherwise. + * + * No locks are needed, and the key will live as long as the struct inode --- so + * it won't go away from under you. + * + * Return: 0 on success, -ENOKEY if the key is missing, or another -errno code + * if a problem occurred while setting up the encryption key. + */ +static inline int fscrypt_require_key(struct inode *inode) +{ + if (IS_ENCRYPTED(inode)) { + int err = fscrypt_get_encryption_info(inode, false); + + if (err) + return err; + if (!fscrypt_has_encryption_key(inode)) + return -ENOKEY; + } + return 0; +} + +/* keysetup_v1.c */ + +void fscrypt_put_direct_key(struct fscrypt_direct_key *dk); + +int fscrypt_setup_v1_file_key(struct fscrypt_inode_info *ci, + const u8 *raw_master_key); + +int fscrypt_setup_v1_file_key_via_subscribed_keyrings( + struct fscrypt_inode_info *ci); + +/* policy.c */ + +bool fscrypt_policies_equal(const union fscrypt_policy *policy1, + const union fscrypt_policy *policy2); +int fscrypt_policy_to_key_spec(const union fscrypt_policy *policy, + struct fscrypt_key_specifier *key_spec); +const union fscrypt_policy *fscrypt_get_dummy_policy(struct super_block *sb); +bool fscrypt_supported_policy(const union fscrypt_policy *policy_u, + const struct inode *inode); +int fscrypt_policy_from_context(union fscrypt_policy *policy_u, + const union fscrypt_context *ctx_u, + int ctx_size); +const union fscrypt_policy *fscrypt_policy_to_inherit(struct inode *dir); + +#endif /* _FSCRYPT_PRIVATE_H */ diff --git a/fs/crypto/hkdf.c b/fs/crypto/hkdf.c new file mode 100644 index 000000000..706f56d00 --- /dev/null +++ b/fs/crypto/hkdf.c @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Implementation of HKDF ("HMAC-based Extract-and-Expand Key Derivation + * Function"), aka RFC 5869. See also the original paper (Krawczyk 2010): + * "Cryptographic Extraction and Key Derivation: The HKDF Scheme". + * + * This is used to derive keys from the fscrypt master keys (or from the + * "software secrets" which hardware derives from the fscrypt master keys, in + * the case that the fscrypt master keys are hardware-wrapped keys). + * + * Copyright 2019 Google LLC + */ + +#include "fscrypt_private.h" + +/* + * HKDF supports any unkeyed cryptographic hash algorithm, but fscrypt uses + * SHA-512 because it is well-established, secure, and reasonably efficient. + * + * HKDF-SHA256 was also considered, as its 256-bit security strength would be + * sufficient here. A 512-bit security strength is "nice to have", though. + * Also, on 64-bit CPUs, SHA-512 is usually just as fast as SHA-256. In the + * common case of deriving an AES-256-XTS key (512 bits), that can result in + * HKDF-SHA512 being much faster than HKDF-SHA256, as the longer digest size of + * SHA-512 causes HKDF-Expand to only need to do one iteration rather than two. + */ +#define HKDF_HASHLEN SHA512_DIGEST_SIZE + +/* + * HKDF consists of two steps: + * + * 1. HKDF-Extract: extract a pseudorandom key of length HKDF_HASHLEN bytes from + * the input keying material and optional salt. + * 2. HKDF-Expand: expand the pseudorandom key into output keying material of + * any length, parameterized by an application-specific info string. + * + * HKDF-Extract can be skipped if the input is already a pseudorandom key of + * length HKDF_HASHLEN bytes. However, cipher modes other than AES-256-XTS take + * shorter keys, and we don't want to force users of those modes to provide + * unnecessarily long master keys. Thus fscrypt still does HKDF-Extract. No + * salt is used, since fscrypt master keys should already be pseudorandom and + * there's no way to persist a random salt per master key from kernel mode. + */ + +/* + * Compute HKDF-Extract using 'master_key' as the input keying material, and + * prepare the resulting HMAC key in 'hkdf'. Afterwards, 'hkdf' can be used for + * HKDF-Expand many times without having to recompute HKDF-Extract each time. + */ +void fscrypt_init_hkdf(struct hmac_sha512_key *hkdf, const u8 *master_key, + unsigned int master_key_size) +{ + static const u8 default_salt[HKDF_HASHLEN]; + u8 prk[HKDF_HASHLEN]; + + hmac_sha512_usingrawkey(default_salt, sizeof(default_salt), + master_key, master_key_size, prk); + hmac_sha512_preparekey(hkdf, prk, sizeof(prk)); + memzero_explicit(prk, sizeof(prk)); +} + +/* + * HKDF-Expand (RFC 5869 section 2.3). Expand the HMAC key 'hkdf' into 'okmlen' + * bytes of output keying material parameterized by the application-specific + * 'info' of length 'infolen' bytes, prefixed by "fscrypt\0" and the 'context' + * byte. This is thread-safe and may be called by multiple threads in parallel. + * + * ('context' isn't part of the HKDF specification; it's just a prefix fscrypt + * adds to its application-specific info strings to guarantee that it doesn't + * accidentally repeat an info string when using HKDF for different purposes.) + */ +void fscrypt_hkdf_expand(const struct hmac_sha512_key *hkdf, u8 context, + const u8 *info, unsigned int infolen, + u8 *okm, unsigned int okmlen) +{ + struct hmac_sha512_ctx ctx; + u8 counter = 1; + u8 tmp[HKDF_HASHLEN]; + + WARN_ON_ONCE(okmlen > 255 * HKDF_HASHLEN); + + for (unsigned int i = 0; i < okmlen; i += HKDF_HASHLEN) { + hmac_sha512_init(&ctx, hkdf); + if (i != 0) + hmac_sha512_update(&ctx, &okm[i - HKDF_HASHLEN], + HKDF_HASHLEN); + hmac_sha512_update(&ctx, "fscrypt\0", 8); + hmac_sha512_update(&ctx, &context, 1); + hmac_sha512_update(&ctx, info, infolen); + hmac_sha512_update(&ctx, &counter, 1); + if (okmlen - i < HKDF_HASHLEN) { + hmac_sha512_final(&ctx, tmp); + memcpy(&okm[i], tmp, okmlen - i); + memzero_explicit(tmp, sizeof(tmp)); + } else { + hmac_sha512_final(&ctx, &okm[i]); + } + counter++; + } +} diff --git a/fs/crypto/hooks.c b/fs/crypto/hooks.c new file mode 100644 index 000000000..a7a8a3f58 --- /dev/null +++ b/fs/crypto/hooks.c @@ -0,0 +1,473 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * fs/crypto/hooks.c + * + * Encryption hooks for higher-level filesystem operations. + */ + +#include + +#include "fscrypt_private.h" + +/** + * fscrypt_file_open() - prepare to open a possibly-encrypted regular file + * @inode: the inode being opened + * @filp: the struct file being set up + * + * Currently, an encrypted regular file can only be opened if its encryption key + * is available; access to the raw encrypted contents is not supported. + * Therefore, we first set up the inode's encryption key (if not already done) + * and return an error if it's unavailable. + * + * We also verify that if the parent directory (from the path via which the file + * is being opened) is encrypted, then the inode being opened uses the same + * encryption policy. This is needed as part of the enforcement that all files + * in an encrypted directory tree use the same encryption policy, as a + * protection against certain types of offline attacks. Note that this check is + * needed even when opening an *unencrypted* file, since it's forbidden to have + * an unencrypted file in an encrypted directory. + * + * Return: 0 on success, -ENOKEY if the key is missing, or another -errno code + */ +int fscrypt_file_open(struct inode *inode, struct file *filp) +{ + int err; + struct dentry *dentry, *dentry_parent; + struct inode *inode_parent; + + err = fscrypt_require_key(inode); + if (err) + return err; + + dentry = file_dentry(filp); + + /* + * Getting a reference to the parent dentry is needed for the actual + * encryption policy comparison, but it's expensive on multi-core + * systems. Since this function runs on unencrypted files too, start + * with a lightweight RCU-mode check for the parent directory being + * unencrypted (in which case it's fine for the child to be either + * unencrypted, or encrypted with any policy). Only continue on to the + * full policy check if the parent directory is actually encrypted. + */ + rcu_read_lock(); + dentry_parent = READ_ONCE(dentry->d_parent); + inode_parent = d_inode_rcu(dentry_parent); + if (inode_parent != NULL && !IS_ENCRYPTED(inode_parent)) { + rcu_read_unlock(); + return 0; + } + rcu_read_unlock(); + + dentry_parent = dget_parent(dentry); + if (!fscrypt_has_permitted_context(d_inode(dentry_parent), inode)) { + fscrypt_warn(inode, + "Inconsistent encryption context (parent directory: %llu)", + d_inode(dentry_parent)->i_ino); + err = -EPERM; + } + dput(dentry_parent); + return err; +} +EXPORT_SYMBOL_GPL(fscrypt_file_open); + +int __fscrypt_prepare_link(struct inode *inode, struct inode *dir, + struct dentry *dentry) +{ + if (fscrypt_is_nokey_name(dentry)) + return -ENOKEY; + /* + * We don't need to separately check that the directory inode's key is + * available, as it's implied by the dentry not being a no-key name. + */ + + if (!fscrypt_has_permitted_context(dir, inode)) + return -EXDEV; + + return 0; +} +EXPORT_SYMBOL_GPL(__fscrypt_prepare_link); + +int __fscrypt_prepare_rename(struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry, + unsigned int flags) +{ + if (fscrypt_is_nokey_name(old_dentry) || + fscrypt_is_nokey_name(new_dentry)) + return -ENOKEY; + /* + * We don't need to separately check that the directory inodes' keys are + * available, as it's implied by the dentries not being no-key names. + */ + + if (old_dir != new_dir) { + if (IS_ENCRYPTED(new_dir) && + !fscrypt_has_permitted_context(new_dir, + d_inode(old_dentry))) + return -EXDEV; + + if ((flags & RENAME_EXCHANGE) && + IS_ENCRYPTED(old_dir) && + !fscrypt_has_permitted_context(old_dir, + d_inode(new_dentry))) + return -EXDEV; + } + return 0; +} +EXPORT_SYMBOL_GPL(__fscrypt_prepare_rename); + +int __fscrypt_prepare_lookup(struct inode *dir, struct dentry *dentry, + struct fscrypt_name *fname) +{ + int err = fscrypt_setup_filename(dir, &dentry->d_name, 1, fname); + + if (err && err != -ENOENT) + return err; + + fscrypt_prepare_dentry(dentry, fname->is_nokey_name); + + return err; +} +EXPORT_SYMBOL_GPL(__fscrypt_prepare_lookup); + +/** + * fscrypt_prepare_lookup_partial() - prepare lookup without filename setup + * @dir: the encrypted directory being searched + * @dentry: the dentry being looked up in @dir + * + * This function should be used by the ->lookup and ->atomic_open methods of + * filesystems that handle filename encryption and no-key name encoding + * themselves and thus can't use fscrypt_prepare_lookup(). Like + * fscrypt_prepare_lookup(), this will try to set up the directory's encryption + * key and will set DCACHE_NOKEY_NAME on the dentry if the key is unavailable. + * However, this function doesn't set up a struct fscrypt_name for the filename. + * + * Return: 0 on success; -errno on error. Note that the encryption key being + * unavailable is not considered an error. It is also not an error if + * the encryption policy is unsupported by this kernel; that is treated + * like the key being unavailable, so that files can still be deleted. + */ +int fscrypt_prepare_lookup_partial(struct inode *dir, struct dentry *dentry) +{ + int err = fscrypt_get_encryption_info(dir, true); + bool is_nokey_name = (!err && !fscrypt_has_encryption_key(dir)); + + fscrypt_prepare_dentry(dentry, is_nokey_name); + + return err; +} +EXPORT_SYMBOL_GPL(fscrypt_prepare_lookup_partial); + +int __fscrypt_prepare_readdir(struct inode *dir) +{ + return fscrypt_get_encryption_info(dir, true); +} +EXPORT_SYMBOL_GPL(__fscrypt_prepare_readdir); + +int __fscrypt_prepare_setattr(struct dentry *dentry, struct iattr *attr) +{ + if (attr->ia_valid & ATTR_SIZE) + return fscrypt_require_key(d_inode(dentry)); + return 0; +} +EXPORT_SYMBOL_GPL(__fscrypt_prepare_setattr); + +/** + * fscrypt_prepare_setflags() - prepare to change flags with FS_IOC_SETFLAGS + * @inode: the inode on which flags are being changed + * @oldflags: the old flags + * @flags: the new flags + * + * The caller should be holding i_rwsem for write. + * + * Return: 0 on success; -errno if the flags change isn't allowed or if + * another error occurs. + */ +int fscrypt_prepare_setflags(struct inode *inode, + unsigned int oldflags, unsigned int flags) +{ + struct fscrypt_inode_info *ci; + struct fscrypt_master_key *mk; + int err; + + /* + * When the CASEFOLD flag is set on an encrypted directory, we must + * derive the secret key needed for the dirhash. This is only possible + * if the directory uses a v2 encryption policy. + */ + if (IS_ENCRYPTED(inode) && (flags & ~oldflags & FS_CASEFOLD_FL)) { + err = fscrypt_require_key(inode); + if (err) + return err; + ci = fscrypt_get_inode_info_raw(inode); + if (ci->ci_policy.version != FSCRYPT_POLICY_V2) + return -EINVAL; + mk = ci->ci_master_key; + down_read(&mk->mk_sem); + if (mk->mk_present) + fscrypt_derive_dirhash_key(ci, mk); + else + err = -ENOKEY; + up_read(&mk->mk_sem); + return err; + } + return 0; +} + +/** + * fscrypt_prepare_symlink() - prepare to create a possibly-encrypted symlink + * @dir: directory in which the symlink is being created + * @target: plaintext symlink target + * @len: length of @target excluding null terminator + * @max_len: space the filesystem has available to store the symlink target + * @disk_link: (out) the on-disk symlink target being prepared + * + * This function computes the size the symlink target will require on-disk, + * stores it in @disk_link->len, and validates it against @max_len. An + * encrypted symlink may be longer than the original. + * + * Additionally, @disk_link->name is set to @target if the symlink will be + * unencrypted, but left NULL if the symlink will be encrypted. For encrypted + * symlinks, the filesystem must call fscrypt_encrypt_symlink() to create the + * on-disk target later. (The reason for the two-step process is that some + * filesystems need to know the size of the symlink target before creating the + * inode, e.g. to determine whether it will be a "fast" or "slow" symlink.) + * + * Return: 0 on success, -ENAMETOOLONG if the symlink target is too long, + * -ENOKEY if the encryption key is missing, or another -errno code if a problem + * occurred while setting up the encryption key. + */ +int fscrypt_prepare_symlink(struct inode *dir, const char *target, + unsigned int len, unsigned int max_len, + struct fscrypt_str *disk_link) +{ + const union fscrypt_policy *policy; + + /* + * To calculate the size of the encrypted symlink target we need to know + * the amount of NUL padding, which is determined by the flags set in + * the encryption policy which will be inherited from the directory. + */ + policy = fscrypt_policy_to_inherit(dir); + if (policy == NULL) { + /* Not encrypted */ + disk_link->name = (unsigned char *)target; + disk_link->len = len + 1; + if (disk_link->len > max_len) + return -ENAMETOOLONG; + return 0; + } + if (IS_ERR(policy)) + return PTR_ERR(policy); + + /* + * Calculate the size of the encrypted symlink and verify it won't + * exceed max_len. Note that for historical reasons, encrypted symlink + * targets are prefixed with the ciphertext length, despite this + * actually being redundant with i_size. This decreases by 2 bytes the + * longest symlink target we can accept. + * + * We could recover 1 byte by not counting a null terminator, but + * counting it (even though it is meaningless for ciphertext) is simpler + * for now since filesystems will assume it is there and subtract it. + */ + if (!__fscrypt_fname_encrypted_size(policy, len, + max_len - sizeof(struct fscrypt_symlink_data) - 1, + &disk_link->len)) + return -ENAMETOOLONG; + disk_link->len += sizeof(struct fscrypt_symlink_data) + 1; + + disk_link->name = NULL; + return 0; +} +EXPORT_SYMBOL_GPL(fscrypt_prepare_symlink); + +int __fscrypt_encrypt_symlink(struct inode *inode, const char *target, + unsigned int len, struct fscrypt_str *disk_link) +{ + int err; + struct qstr iname = QSTR_INIT(target, len); + struct fscrypt_symlink_data *sd; + unsigned int ciphertext_len; + + /* + * fscrypt_prepare_new_inode() should have already set up the new + * symlink inode's encryption key. We don't wait until now to do it, + * since we may be in a filesystem transaction now. + */ + if (WARN_ON_ONCE(!fscrypt_has_encryption_key(inode))) + return -ENOKEY; + + if (disk_link->name) { + /* filesystem-provided buffer */ + sd = (struct fscrypt_symlink_data *)disk_link->name; + } else { + sd = kmalloc(disk_link->len, GFP_NOFS); + if (!sd) + return -ENOMEM; + } + ciphertext_len = disk_link->len - sizeof(*sd) - 1; + sd->len = cpu_to_le16(ciphertext_len); + + err = fscrypt_fname_encrypt(inode, &iname, sd->encrypted_path, + ciphertext_len); + if (err) + goto err_free_sd; + + /* + * Null-terminating the ciphertext doesn't make sense, but we still + * count the null terminator in the length, so we might as well + * initialize it just in case the filesystem writes it out. + */ + sd->encrypted_path[ciphertext_len] = '\0'; + + /* Cache the plaintext symlink target for later use by get_link() */ + err = -ENOMEM; + inode->i_link = kmemdup(target, len + 1, GFP_NOFS); + if (!inode->i_link) + goto err_free_sd; + + if (!disk_link->name) + disk_link->name = (unsigned char *)sd; + return 0; + +err_free_sd: + if (!disk_link->name) + kfree(sd); + return err; +} +EXPORT_SYMBOL_GPL(__fscrypt_encrypt_symlink); + +/** + * fscrypt_get_symlink() - get the target of an encrypted symlink + * @inode: the symlink inode + * @caddr: the on-disk contents of the symlink + * @max_size: size of @caddr buffer + * @done: if successful, will be set up to free the returned target if needed + * + * If the symlink's encryption key is available, we decrypt its target. + * Otherwise, we encode its target for presentation. + * + * This may sleep, so the filesystem must have dropped out of RCU mode already. + * + * Return: the presentable symlink target or an ERR_PTR() + */ +const char *fscrypt_get_symlink(struct inode *inode, const void *caddr, + unsigned int max_size, + struct delayed_call *done) +{ + const struct fscrypt_symlink_data *sd; + struct fscrypt_str cstr, pstr; + bool has_key; + int err; + + /* This is for encrypted symlinks only */ + if (WARN_ON_ONCE(!IS_ENCRYPTED(inode))) + return ERR_PTR(-EINVAL); + + /* If the decrypted target is already cached, just return it. */ + pstr.name = READ_ONCE(inode->i_link); + if (pstr.name) + return pstr.name; + + /* + * Try to set up the symlink's encryption key, but we can continue + * regardless of whether the key is available or not. + */ + err = fscrypt_get_encryption_info(inode, false); + if (err) + return ERR_PTR(err); + has_key = fscrypt_has_encryption_key(inode); + + /* + * For historical reasons, encrypted symlink targets are prefixed with + * the ciphertext length, even though this is redundant with i_size. + */ + + if (max_size < sizeof(*sd) + 1) + return ERR_PTR(-EUCLEAN); + sd = caddr; + cstr.name = (unsigned char *)sd->encrypted_path; + cstr.len = le16_to_cpu(sd->len); + + if (cstr.len == 0) + return ERR_PTR(-EUCLEAN); + + if (cstr.len + sizeof(*sd) > max_size) + return ERR_PTR(-EUCLEAN); + + err = fscrypt_fname_alloc_buffer(cstr.len, &pstr); + if (err) + return ERR_PTR(err); + + err = fscrypt_fname_disk_to_usr(inode, 0, 0, &cstr, &pstr); + if (err) + goto err_kfree; + + err = -EUCLEAN; + if (pstr.name[0] == '\0') + goto err_kfree; + + pstr.name[pstr.len] = '\0'; + + /* + * Cache decrypted symlink targets in i_link for later use. Don't cache + * symlink targets encoded without the key, since those become outdated + * once the key is added. This pairs with the READ_ONCE() above and in + * the VFS path lookup code. + */ + if (!has_key || + cmpxchg_release(&inode->i_link, NULL, pstr.name) != NULL) + set_delayed_call(done, kfree_link, pstr.name); + + return pstr.name; + +err_kfree: + kfree(pstr.name); + return ERR_PTR(err); +} +EXPORT_SYMBOL_GPL(fscrypt_get_symlink); + +/** + * fscrypt_symlink_getattr() - set the correct st_size for encrypted symlinks + * @path: the path for the encrypted symlink being queried + * @stat: the struct being filled with the symlink's attributes + * + * Override st_size of encrypted symlinks to be the length of the decrypted + * symlink target (or the no-key encoded symlink target, if the key is + * unavailable) rather than the length of the encrypted symlink target. This is + * necessary for st_size to match the symlink target that userspace actually + * sees. POSIX requires this, and some userspace programs depend on it. + * + * This requires reading the symlink target from disk if needed, setting up the + * inode's encryption key if possible, and then decrypting or encoding the + * symlink target. This makes lstat() more heavyweight than is normally the + * case. However, decrypted symlink targets will be cached in ->i_link, so + * usually the symlink won't have to be read and decrypted again later if/when + * it is actually followed, readlink() is called, or lstat() is called again. + * + * Return: 0 on success, -errno on failure + */ +int fscrypt_symlink_getattr(const struct path *path, struct kstat *stat) +{ + struct dentry *dentry = path->dentry; + struct inode *inode = d_inode(dentry); + const char *link; + DEFINE_DELAYED_CALL(done); + + /* + * To get the symlink target that userspace will see (whether it's the + * decrypted target or the no-key encoded target), we can just get it in + * the same way the VFS does during path resolution and readlink(). + */ + link = READ_ONCE(inode->i_link); + if (!link) { + link = inode->i_op->get_link(dentry, inode, &done); + if (IS_ERR(link)) + return PTR_ERR(link); + } + stat->size = strlen(link); + do_delayed_call(&done); + return 0; +} +EXPORT_SYMBOL_GPL(fscrypt_symlink_getattr); diff --git a/fs/crypto/keyring.c b/fs/crypto/keyring.c new file mode 100644 index 000000000..76e28d1e0 --- /dev/null +++ b/fs/crypto/keyring.c @@ -0,0 +1,1238 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Filesystem-level keyring for fscrypt + * + * Copyright 2019 Google LLC + */ + +/* + * This file implements management of fscrypt master keys in the + * filesystem-level keyring, including the ioctls: + * + * - FS_IOC_ADD_ENCRYPTION_KEY + * - FS_IOC_REMOVE_ENCRYPTION_KEY + * - FS_IOC_REMOVE_ENCRYPTION_KEY_ALL_USERS + * - FS_IOC_GET_ENCRYPTION_KEY_STATUS + * + * See the "User API" section of Documentation/filesystems/fscrypt.rst for more + * information about these ioctls. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "fscrypt_private.h" + +/* The master encryption keys for a filesystem (->s_master_keys) */ +struct fscrypt_keyring { + /* + * Lock that protects ->key_hashtable. It does *not* protect the + * fscrypt_master_key structs themselves. + */ + spinlock_t lock; + + /* Hash table that maps fscrypt_key_specifier to fscrypt_master_key */ + struct hlist_head key_hashtable[128]; +}; + +static void wipe_master_key_secret(struct fscrypt_master_key_secret *secret) +{ + memzero_explicit(secret, sizeof(*secret)); +} + +static void move_master_key_secret(struct fscrypt_master_key_secret *dst, + struct fscrypt_master_key_secret *src) +{ + memcpy(dst, src, sizeof(*dst)); + memzero_explicit(src, sizeof(*src)); +} + +static void fscrypt_free_master_key(struct rcu_head *head) +{ + struct fscrypt_master_key *mk = + container_of(head, struct fscrypt_master_key, mk_rcu_head); + /* + * The master key secret and any embedded subkeys should have already + * been wiped when the last active reference to the fscrypt_master_key + * struct was dropped; doing it here would be unnecessarily late. + * Nevertheless, use kfree_sensitive() in case anything was missed. + */ + kfree_sensitive(mk); +} + +static void clear_mk_users(struct fscrypt_master_key *mk); + +void fscrypt_put_master_key(struct fscrypt_master_key *mk) +{ + if (!refcount_dec_and_test(&mk->mk_struct_refs)) + return; + /* + * No structural references left, so clear ->mk_users, and also free the + * fscrypt_master_key struct itself after an RCU grace period ensures + * that concurrent keyring lookups can no longer find it. + */ + WARN_ON_ONCE(refcount_read(&mk->mk_active_refs) != 0); + clear_mk_users(mk); + call_rcu(&mk->mk_rcu_head, fscrypt_free_master_key); +} + +void fscrypt_put_master_key_activeref(struct super_block *sb, + struct fscrypt_master_key *mk) +{ + struct fscrypt_mode_key *node, *tmp; + + if (!refcount_dec_and_test(&mk->mk_active_refs)) + return; + /* + * No active references left, so complete the full removal of this + * fscrypt_master_key struct by removing it from the keyring and + * destroying any non-file-scoped subkeys. + */ + + if (WARN_ON_ONCE(!sb->s_master_keys)) + return; + spin_lock(&sb->s_master_keys->lock); + hlist_del_rcu(&mk->mk_node); + spin_unlock(&sb->s_master_keys->lock); + + /* + * ->mk_active_refs == 0 implies that ->mk_present is false and + * ->mk_decrypted_inodes is empty. + */ + WARN_ON_ONCE(mk->mk_present); + WARN_ON_ONCE(!list_empty(&mk->mk_decrypted_inodes)); + + /* + * Destroy any non-file-scoped subkeys. Since ->mk_active_refs == 0, + * they're no longer referenced by any inodes. Nor can key setup run + * and use them again. So they're no longer needed. (This implies no + * concurrent readers, so we don't need list_del_rcu() for example.) + */ + list_for_each_entry_safe(node, tmp, &mk->mk_mode_keys, link) { + fscrypt_destroy_prepared_key(sb, &node->key); + list_del(&node->link); + kfree(node); + } + memzero_explicit(&mk->mk_ino_hash_key, + sizeof(mk->mk_ino_hash_key)); + mk->mk_ino_hash_key_initialized = false; + + /* Drop the structural ref associated with the active refs. */ + fscrypt_put_master_key(mk); +} + +/* + * This transitions the key state from present to incompletely removed, and then + * potentially to absent (depending on whether inodes remain). + */ +static void fscrypt_initiate_key_removal(struct super_block *sb, + struct fscrypt_master_key *mk) +{ + WRITE_ONCE(mk->mk_present, false); + wipe_master_key_secret(&mk->mk_secret); + fscrypt_put_master_key_activeref(sb, mk); +} + +static inline bool valid_key_spec(const struct fscrypt_key_specifier *spec) +{ + if (spec->__reserved) + return false; + return master_key_spec_len(spec) != 0; +} + +static int fscrypt_user_key_instantiate(struct key *key, + struct key_preparsed_payload *prep) +{ + /* + * We just charge FSCRYPT_MAX_RAW_KEY_SIZE bytes to the user's key quota + * for each key, regardless of the exact key size. The amount of memory + * actually used is greater than the size of the raw key anyway. + */ + return key_payload_reserve(key, FSCRYPT_MAX_RAW_KEY_SIZE); +} + +static void fscrypt_user_key_describe(const struct key *key, struct seq_file *m) +{ + seq_puts(m, key->description); +} + +/* + * Type of fscrypt_master_key_user::quota_key. This contains no secret; it + * exists solely to charge a user's key quota. + * + * Note that the name of this key type really should be something like + * ".fscrypt-user" instead of simply ".fscrypt". But the shorter name is chosen + * mainly for simplicity of presentation in /proc/keys when read by a non-root + * user. And it is expected to be rare that a key is actually added by multiple + * users, since users should keep their encryption keys confidential. + */ +static struct key_type key_type_fscrypt_user = { + .name = ".fscrypt", + .instantiate = fscrypt_user_key_instantiate, + .describe = fscrypt_user_key_describe, +}; + +#define FSCRYPT_MK_USER_DESCRIPTION_SIZE \ + (2 * FSCRYPT_KEY_IDENTIFIER_SIZE + CONST_STRLEN(".uid.") + 10 + 1) + +/* Create ->s_master_keys if needed. Synchronized by fscrypt_add_key_mutex. */ +static int allocate_filesystem_keyring(struct super_block *sb) +{ + struct fscrypt_keyring *keyring; + + if (sb->s_master_keys) + return 0; + + keyring = kzalloc_obj(*keyring); + if (!keyring) + return -ENOMEM; + spin_lock_init(&keyring->lock); + /* + * Pairs with the smp_load_acquire() in fscrypt_find_master_key(). + * I.e., here we publish ->s_master_keys with a RELEASE barrier so that + * concurrent tasks can ACQUIRE it. + */ + smp_store_release(&sb->s_master_keys, keyring); + return 0; +} + +/* + * Release all encryption keys that have been added to the filesystem, along + * with the keyring that contains them. + * + * This is called at unmount time, after all potentially-encrypted inodes have + * been evicted. The filesystem's underlying block device(s) are still + * available at this time; this is important because after user file accesses + * have been allowed, this function may need to evict keys from the keyslots of + * an inline crypto engine, which requires the block device(s). + */ +void fscrypt_destroy_keyring(struct super_block *sb) +{ + struct fscrypt_keyring *keyring = sb->s_master_keys; + size_t i; + + if (!keyring) + return; + + for (i = 0; i < ARRAY_SIZE(keyring->key_hashtable); i++) { + struct hlist_head *bucket = &keyring->key_hashtable[i]; + struct fscrypt_master_key *mk; + struct hlist_node *tmp; + + hlist_for_each_entry_safe(mk, tmp, bucket, mk_node) { + /* + * Since all potentially-encrypted inodes were already + * evicted, every key remaining in the keyring should + * have an empty inode list, and should only still be in + * the keyring due to the single active ref associated + * with ->mk_present. There should be no structural + * refs beyond the one associated with the active ref. + */ + WARN_ON_ONCE(refcount_read(&mk->mk_active_refs) != 1); + WARN_ON_ONCE(refcount_read(&mk->mk_struct_refs) != 1); + WARN_ON_ONCE(!mk->mk_present); + fscrypt_initiate_key_removal(sb, mk); + } + } + kfree_sensitive(keyring); + sb->s_master_keys = NULL; +} + +static struct hlist_head * +fscrypt_mk_hash_bucket(struct fscrypt_keyring *keyring, + const struct fscrypt_key_specifier *mk_spec) +{ + /* + * Since key specifiers should be "random" values, it is sufficient to + * use a trivial hash function that just takes the first several bits of + * the key specifier. + */ + unsigned long i = get_unaligned((unsigned long *)&mk_spec->u); + + return &keyring->key_hashtable[i % ARRAY_SIZE(keyring->key_hashtable)]; +} + +/* + * Find the specified master key struct in ->s_master_keys and take a structural + * ref to it. The structural ref guarantees that the key struct continues to + * exist, but it does *not* guarantee that ->s_master_keys continues to contain + * the key struct. The structural ref needs to be dropped by + * fscrypt_put_master_key(). Returns NULL if the key struct is not found. + */ +struct fscrypt_master_key * +fscrypt_find_master_key(struct super_block *sb, + const struct fscrypt_key_specifier *mk_spec) +{ + struct fscrypt_keyring *keyring; + struct hlist_head *bucket; + struct fscrypt_master_key *mk; + + /* + * Pairs with the smp_store_release() in allocate_filesystem_keyring(). + * I.e., another task can publish ->s_master_keys concurrently, + * executing a RELEASE barrier. We need to use smp_load_acquire() here + * to safely ACQUIRE the memory the other task published. + */ + keyring = smp_load_acquire(&sb->s_master_keys); + if (keyring == NULL) + return NULL; /* No keyring yet, so no keys yet. */ + + bucket = fscrypt_mk_hash_bucket(keyring, mk_spec); + rcu_read_lock(); + switch (mk_spec->type) { + case FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR: + hlist_for_each_entry_rcu(mk, bucket, mk_node) { + if (mk->mk_spec.type == + FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR && + memcmp(mk->mk_spec.u.descriptor, + mk_spec->u.descriptor, + FSCRYPT_KEY_DESCRIPTOR_SIZE) == 0 && + refcount_inc_not_zero(&mk->mk_struct_refs)) + goto out; + } + break; + case FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER: + hlist_for_each_entry_rcu(mk, bucket, mk_node) { + if (mk->mk_spec.type == + FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER && + memcmp(mk->mk_spec.u.identifier, + mk_spec->u.identifier, + FSCRYPT_KEY_IDENTIFIER_SIZE) == 0 && + refcount_inc_not_zero(&mk->mk_struct_refs)) + goto out; + } + break; + } + mk = NULL; +out: + rcu_read_unlock(); + return mk; +} + +/* Find the current user's claim in ->mk_users. ->mk_sem must be held. */ +static struct fscrypt_master_key_user * +find_master_key_user(struct fscrypt_master_key *mk) +{ + struct fscrypt_master_key_user *mk_user; + kuid_t uid = current_fsuid(); + + list_for_each_entry(mk_user, &mk->mk_users, link) { + if (uid_eq(mk_user->uid, uid)) + return mk_user; + } + return NULL; +} + +/* + * Give the current user a claim in ->mk_users. This charges the user's quota + * and marks the master key as added by the current user, so that it cannot be + * removed by another user with the key. Either ->mk_sem must be held for + * write, or the master key must be still undergoing initialization. + */ +static int add_master_key_user(struct fscrypt_master_key *mk) +{ + kuid_t uid = current_fsuid(); + char description[FSCRYPT_MK_USER_DESCRIPTION_SIZE]; + struct key *quota_key; + struct fscrypt_master_key_user *mk_user; + int err; + + snprintf(description, sizeof(description), "%*phN.uid.%u", + FSCRYPT_KEY_IDENTIFIER_SIZE, mk->mk_spec.u.identifier, + __kuid_val(uid)); + quota_key = key_alloc(&key_type_fscrypt_user, description, uid, + current_gid(), current_cred(), + KEY_POS_SEARCH | KEY_USR_VIEW, 0, NULL); + if (IS_ERR(quota_key)) + return PTR_ERR(quota_key); + + err = key_instantiate_and_link(quota_key, NULL, 0, NULL, NULL); + if (err) { + key_put(quota_key); + return err; + } + + mk_user = kzalloc_obj(*mk_user); + if (!mk_user) { + key_put(quota_key); + return -ENOMEM; + } + mk_user->uid = uid; + mk_user->quota_key = quota_key; + list_add(&mk_user->link, &mk->mk_users); + return 0; +} + +static void unlink_and_free_mk_user(struct fscrypt_master_key_user *mk_user) +{ + list_del(&mk_user->link); + key_put(mk_user->quota_key); + kfree(mk_user); +} + +/* + * Remove the current user's claim from ->mk_users. + * ->mk_sem must be held for write. + * + * Returns 0 if removed or -ENOKEY if not found. + */ +static int remove_master_key_user(struct fscrypt_master_key *mk) +{ + struct fscrypt_master_key_user *mk_user; + + mk_user = find_master_key_user(mk); + if (!mk_user) + return -ENOKEY; + unlink_and_free_mk_user(mk_user); + return 0; +} + +/* + * Clear ->mk_users. Either ->mk_sem must be held for write, or 'mk' must have + * no structural references left. + */ +static void clear_mk_users(struct fscrypt_master_key *mk) +{ + struct fscrypt_master_key_user *mk_user, *tmp; + + list_for_each_entry_safe(mk_user, tmp, &mk->mk_users, link) + unlink_and_free_mk_user(mk_user); +} + +/* + * Allocate a new fscrypt_master_key, transfer the given secret over to it, and + * insert it into sb->s_master_keys. + */ +static int add_new_master_key(struct super_block *sb, + struct fscrypt_master_key_secret *secret, + const struct fscrypt_key_specifier *mk_spec) +{ + struct fscrypt_keyring *keyring = sb->s_master_keys; + struct fscrypt_master_key *mk; + int err; + + mk = kzalloc_obj(*mk); + if (!mk) + return -ENOMEM; + + init_rwsem(&mk->mk_sem); + refcount_set(&mk->mk_struct_refs, 1); + mk->mk_spec = *mk_spec; + + INIT_LIST_HEAD(&mk->mk_users); + + INIT_LIST_HEAD(&mk->mk_decrypted_inodes); + spin_lock_init(&mk->mk_decrypted_inodes_lock); + + INIT_LIST_HEAD(&mk->mk_mode_keys); + + if (mk_spec->type == FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER) { + err = add_master_key_user(mk); + if (err) + goto out_put; + } + + move_master_key_secret(&mk->mk_secret, secret); + mk->mk_present = true; + refcount_set(&mk->mk_active_refs, 1); /* ->mk_present is true */ + + spin_lock(&keyring->lock); + hlist_add_head_rcu(&mk->mk_node, + fscrypt_mk_hash_bucket(keyring, mk_spec)); + spin_unlock(&keyring->lock); + return 0; + +out_put: + fscrypt_put_master_key(mk); + return err; +} + +#define KEY_DEAD 1 + +static int add_existing_master_key(struct fscrypt_master_key *mk, + struct fscrypt_master_key_secret *secret) +{ + int err; + + /* + * For v2 policy keys (FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER): If the current + * user is already in ->mk_users, then there's nothing to do. + * Otherwise, add the user to ->mk_users. + */ + if (mk->mk_spec.type == FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER) { + if (find_master_key_user(mk) != NULL) + return 0; + err = add_master_key_user(mk); + if (err) + return err; + } + + /* If the key is incompletely removed, make it present again. */ + if (!mk->mk_present) { + if (!refcount_inc_not_zero(&mk->mk_active_refs)) { + /* + * Raced with the last active ref being dropped, so the + * key has become, or is about to become, "absent". + * Therefore, we need to allocate a new key struct. + */ + return KEY_DEAD; + } + move_master_key_secret(&mk->mk_secret, secret); + WRITE_ONCE(mk->mk_present, true); + } + + return 0; +} + +static int do_add_master_key(struct super_block *sb, + struct fscrypt_master_key_secret *secret, + const struct fscrypt_key_specifier *mk_spec) +{ + static DEFINE_MUTEX(fscrypt_add_key_mutex); + struct fscrypt_master_key *mk; + int err; + + guard(mutex)(&fscrypt_add_key_mutex); /* serialize find + link */ + + mk = fscrypt_find_master_key(sb, mk_spec); + if (!mk) { + /* Didn't find the key in ->s_master_keys. Add it. */ + err = allocate_filesystem_keyring(sb); + if (!err) + err = add_new_master_key(sb, secret, mk_spec); + } else { + /* + * Found the key in ->s_master_keys. Add the user to ->mk_users + * if needed, and make the key "present" again if possible. + */ + down_write(&mk->mk_sem); + err = add_existing_master_key(mk, secret); + up_write(&mk->mk_sem); + if (err == KEY_DEAD) { + /* + * We found a key struct, but it's already been fully + * removed. Ignore the old struct and add a new one. + * fscrypt_add_key_mutex means we don't need to worry + * about concurrent adds. + */ + err = add_new_master_key(sb, secret, mk_spec); + } + fscrypt_put_master_key(mk); + } + return err; +} + +static int add_master_key(struct super_block *sb, + struct fscrypt_master_key_secret *secret, + struct fscrypt_key_specifier *key_spec) +{ + int err; + + if (key_spec->type == FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER) { + u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE]; + u8 *kdf_key = secret->bytes; + unsigned int kdf_key_size = secret->size; + u8 keyid_kdf_ctx = HKDF_CONTEXT_KEY_IDENTIFIER_FOR_RAW_KEY; + + /* + * For raw keys, the fscrypt master key is used directly as the + * fscrypt KDF key. For hardware-wrapped keys, we have to pass + * the master key to the hardware to derive the KDF key, which + * is then only used to derive non-file-contents subkeys. + */ + if (secret->is_hw_wrapped) { + err = fscrypt_derive_sw_secret(sb, secret->bytes, + secret->size, sw_secret); + if (err) + return err; + kdf_key = sw_secret; + kdf_key_size = sizeof(sw_secret); + /* + * To avoid weird behavior if someone manages to + * determine sw_secret and add it as a raw key, ensure + * that hardware-wrapped keys and raw keys will have + * different key identifiers by deriving their key + * identifiers using different KDF contexts. + */ + keyid_kdf_ctx = + HKDF_CONTEXT_KEY_IDENTIFIER_FOR_HW_WRAPPED_KEY; + } + fscrypt_init_hkdf(&secret->hkdf, kdf_key, kdf_key_size); + /* + * Now that the KDF context is initialized, the raw KDF key is + * no longer needed. + */ + memzero_explicit(kdf_key, kdf_key_size); + + /* Calculate the key identifier */ + fscrypt_hkdf_expand(&secret->hkdf, keyid_kdf_ctx, NULL, 0, + key_spec->u.identifier, + FSCRYPT_KEY_IDENTIFIER_SIZE); + } + return do_add_master_key(sb, secret, key_spec); +} + +/* + * Validate the size of an fscrypt master key being added. Note that this is + * just an initial check, as we don't know which ciphers will be used yet. + * There is a stricter size check later when the key is actually used by a file. + */ +static inline bool fscrypt_valid_key_size(size_t size, u32 add_key_flags) +{ + u32 max_size = (add_key_flags & FSCRYPT_ADD_KEY_FLAG_HW_WRAPPED) ? + FSCRYPT_MAX_HW_WRAPPED_KEY_SIZE : + FSCRYPT_MAX_RAW_KEY_SIZE; + + return size >= FSCRYPT_MIN_KEY_SIZE && size <= max_size; +} + +static int fscrypt_provisioning_key_preparse(struct key_preparsed_payload *prep) +{ + const struct fscrypt_provisioning_key_payload *payload = prep->data; + + if (prep->datalen < sizeof(*payload)) + return -EINVAL; + + if (!fscrypt_valid_key_size(prep->datalen - sizeof(*payload), + payload->flags)) + return -EINVAL; + + if (payload->type != FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR && + payload->type != FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER) + return -EINVAL; + + if (payload->flags & ~FSCRYPT_ADD_KEY_FLAG_HW_WRAPPED) + return -EINVAL; + + prep->payload.data[0] = kmemdup(payload, prep->datalen, GFP_KERNEL); + if (!prep->payload.data[0]) + return -ENOMEM; + + prep->quotalen = prep->datalen; + return 0; +} + +static void fscrypt_provisioning_key_free_preparse( + struct key_preparsed_payload *prep) +{ + kfree_sensitive(prep->payload.data[0]); +} + +static void fscrypt_provisioning_key_describe(const struct key *key, + struct seq_file *m) +{ + seq_puts(m, key->description); + if (key_is_positive(key)) { + const struct fscrypt_provisioning_key_payload *payload = + key->payload.data[0]; + + seq_printf(m, ": %u [%u]", key->datalen, payload->type); + } +} + +static void fscrypt_provisioning_key_destroy(struct key *key) +{ + kfree_sensitive(key->payload.data[0]); +} + +static struct key_type key_type_fscrypt_provisioning = { + .name = "fscrypt-provisioning", + .preparse = fscrypt_provisioning_key_preparse, + .free_preparse = fscrypt_provisioning_key_free_preparse, + .instantiate = generic_key_instantiate, + .describe = fscrypt_provisioning_key_describe, + .destroy = fscrypt_provisioning_key_destroy, +}; + +/* + * Retrieve the key from the Linux keyring key specified by 'key_id', and store + * it into 'secret'. + * + * The key must be of type "fscrypt-provisioning" and must have the 'type' and + * 'flags' field of the payload set to the given values, indicating that the key + * is intended for use for the specified purpose. We don't use the "logon" key + * type because there's no way to completely restrict the use of such keys; they + * can be used by any kernel API that accepts "logon" keys and doesn't require a + * specific service prefix. + * + * The ability to specify the key via Linux keyring key is intended for cases + * where userspace needs to re-add keys after the filesystem is unmounted and + * re-mounted. Most users should just provide the key directly instead. + */ +static int get_keyring_key(u32 key_id, u32 type, u32 flags, + struct fscrypt_master_key_secret *secret) +{ + key_ref_t ref; + struct key *key; + const struct fscrypt_provisioning_key_payload *payload; + int err; + + ref = lookup_user_key(key_id, 0, KEY_NEED_SEARCH); + if (IS_ERR(ref)) + return PTR_ERR(ref); + key = key_ref_to_ptr(ref); + + if (key->type != &key_type_fscrypt_provisioning) + goto bad_key; + payload = key->payload.data[0]; + + /* + * Don't allow fscrypt v1 keys to be used as v2 keys and vice versa. + * Similarly, don't allow hardware-wrapped keys to be used as + * non-hardware-wrapped keys and vice versa. + */ + if (payload->type != type || payload->flags != flags) + goto bad_key; + + secret->size = key->datalen - sizeof(*payload); + memcpy(secret->bytes, payload->raw, secret->size); + err = 0; + goto out_put; + +bad_key: + err = -EKEYREJECTED; +out_put: + key_ref_put(ref); + return err; +} + +/* + * Add a master encryption key to the filesystem, causing all files which were + * encrypted with it to appear "unlocked" (decrypted) when accessed. + * + * When adding a key for use by v1 encryption policies, this ioctl is + * privileged, and userspace must provide the 'key_descriptor'. + * + * When adding a key for use by v2+ encryption policies, this ioctl is + * unprivileged. This is needed, in general, to allow non-root users to use + * encryption without encountering the visibility problems of process-subscribed + * keyrings and the inability to properly remove keys. This works by having + * each key identified by its cryptographically secure hash --- the + * 'key_identifier'. The cryptographic hash ensures that a malicious user + * cannot add the wrong key for a given identifier. Furthermore, each added key + * is charged to the appropriate user's quota for the keyrings service, which + * prevents a malicious user from adding too many keys. Finally, we forbid a + * user from removing a key while other users have added it too, which prevents + * a user who knows another user's key from causing a denial-of-service by + * removing it at an inopportune time. (We tolerate that a user who knows a key + * can prevent other users from removing it.) + * + * For more details, see the "FS_IOC_ADD_ENCRYPTION_KEY" section of + * Documentation/filesystems/fscrypt.rst. + */ +int fscrypt_ioctl_add_key(struct file *filp, void __user *_uarg) +{ + struct super_block *sb = file_inode(filp)->i_sb; + struct fscrypt_add_key_arg __user *uarg = _uarg; + struct fscrypt_add_key_arg arg; + struct fscrypt_master_key_secret secret; + int err; + + if (copy_from_user(&arg, uarg, sizeof(arg))) + return -EFAULT; + + if (!valid_key_spec(&arg.key_spec)) + return -EINVAL; + + if (memchr_inv(arg.__reserved, 0, sizeof(arg.__reserved))) + return -EINVAL; + + /* + * Only root can add keys that are identified by an arbitrary descriptor + * rather than by a cryptographic hash --- since otherwise a malicious + * user could add the wrong key. + */ + if (arg.key_spec.type == FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR && + !capable(CAP_SYS_ADMIN)) + return -EACCES; + + memset(&secret, 0, sizeof(secret)); + + if (arg.flags) { + if (arg.flags & ~FSCRYPT_ADD_KEY_FLAG_HW_WRAPPED) + return -EINVAL; + if (arg.key_spec.type != FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER) + return -EINVAL; + secret.is_hw_wrapped = true; + } + + if (arg.key_id) { + if (arg.raw_size != 0) + return -EINVAL; + err = get_keyring_key(arg.key_id, arg.key_spec.type, arg.flags, + &secret); + if (err) + goto out_wipe_secret; + } else { + if (!fscrypt_valid_key_size(arg.raw_size, arg.flags)) + return -EINVAL; + secret.size = arg.raw_size; + err = -EFAULT; + if (copy_from_user(secret.bytes, uarg->raw, secret.size)) + goto out_wipe_secret; + } + + err = add_master_key(sb, &secret, &arg.key_spec); + if (err) + goto out_wipe_secret; + + /* Return the key identifier to userspace, if applicable */ + err = -EFAULT; + if (arg.key_spec.type == FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER && + copy_to_user(uarg->key_spec.u.identifier, arg.key_spec.u.identifier, + FSCRYPT_KEY_IDENTIFIER_SIZE)) + goto out_wipe_secret; + err = 0; +out_wipe_secret: + wipe_master_key_secret(&secret); + return err; +} +EXPORT_SYMBOL_GPL(fscrypt_ioctl_add_key); + +static void +fscrypt_get_test_dummy_secret(struct fscrypt_master_key_secret *secret) +{ + static u8 test_key[FSCRYPT_MAX_RAW_KEY_SIZE]; + + get_random_once(test_key, sizeof(test_key)); + + memset(secret, 0, sizeof(*secret)); + secret->size = sizeof(test_key); + memcpy(secret->bytes, test_key, sizeof(test_key)); +} + +void fscrypt_get_test_dummy_key_identifier( + u8 key_identifier[FSCRYPT_KEY_IDENTIFIER_SIZE]) +{ + struct fscrypt_master_key_secret secret; + + fscrypt_get_test_dummy_secret(&secret); + fscrypt_init_hkdf(&secret.hkdf, secret.bytes, secret.size); + fscrypt_hkdf_expand(&secret.hkdf, + HKDF_CONTEXT_KEY_IDENTIFIER_FOR_RAW_KEY, NULL, 0, + key_identifier, FSCRYPT_KEY_IDENTIFIER_SIZE); + wipe_master_key_secret(&secret); +} + +/** + * fscrypt_add_test_dummy_key() - add the test dummy encryption key + * @sb: the filesystem instance to add the key to + * @key_spec: the key specifier of the test dummy encryption key + * + * Add the key for the test_dummy_encryption mount option to the filesystem. To + * prevent misuse of this mount option, a per-boot random key is used instead of + * a hardcoded one. This makes it so that any encrypted files created using + * this option won't be accessible after a reboot. + * + * Return: 0 on success, -errno on failure + */ +int fscrypt_add_test_dummy_key(struct super_block *sb, + struct fscrypt_key_specifier *key_spec) +{ + struct fscrypt_master_key_secret secret; + int err; + + fscrypt_get_test_dummy_secret(&secret); + err = add_master_key(sb, &secret, key_spec); + wipe_master_key_secret(&secret); + return err; +} + +/* + * Verify that the current user has added a master key with the given identifier + * (returns -ENOKEY if not). This is needed to prevent a user from encrypting + * their files using some other user's key which they don't actually know. + * Cryptographically this isn't much of a problem, but the semantics of this + * would be a bit weird, so it's best to just forbid it. + * + * The system administrator (CAP_FOWNER) can override this, which should be + * enough for any use cases where encryption policies are being set using keys + * that were chosen ahead of time but aren't available at the moment. + * + * Note that the key may have already removed by the time this returns, but + * that's okay; we just care whether the key was there at some point. + * + * Return: 0 if the key is added, -ENOKEY if it isn't, or another -errno code + */ +int fscrypt_verify_key_added(struct super_block *sb, + const u8 identifier[FSCRYPT_KEY_IDENTIFIER_SIZE]) +{ + struct fscrypt_key_specifier mk_spec; + struct fscrypt_master_key *mk; + int err; + + mk_spec.type = FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER; + memcpy(mk_spec.u.identifier, identifier, FSCRYPT_KEY_IDENTIFIER_SIZE); + + mk = fscrypt_find_master_key(sb, &mk_spec); + if (!mk) { + err = -ENOKEY; + goto out; + } + down_read(&mk->mk_sem); + if (find_master_key_user(mk) != NULL) + err = 0; + else + err = -ENOKEY; + up_read(&mk->mk_sem); + fscrypt_put_master_key(mk); +out: + if (err == -ENOKEY && capable(CAP_FOWNER)) + err = 0; + return err; +} + +/* + * Try to evict the inode's dentries from the dentry cache. If the inode is a + * directory, then it can have at most one dentry; however, that dentry may be + * pinned by child dentries, so first try to evict the children too. + */ +static void shrink_dcache_inode(struct inode *inode) +{ + struct dentry *dentry; + + if (S_ISDIR(inode->i_mode)) { + dentry = d_find_any_alias(inode); + if (dentry) { + shrink_dcache_parent(dentry); + dput(dentry); + } + } + d_prune_aliases(inode); +} + +static void evict_dentries_for_decrypted_inodes(struct fscrypt_master_key *mk) +{ + struct fscrypt_inode_info *ci; + struct inode *inode; + struct inode *toput_inode = NULL; + + spin_lock(&mk->mk_decrypted_inodes_lock); + + list_for_each_entry(ci, &mk->mk_decrypted_inodes, ci_master_key_link) { + inode = ci->ci_inode; + spin_lock(&inode->i_lock); + if (inode_state_read(inode) & (I_FREEING | I_WILL_FREE | I_NEW)) { + spin_unlock(&inode->i_lock); + continue; + } + __iget(inode); + spin_unlock(&inode->i_lock); + spin_unlock(&mk->mk_decrypted_inodes_lock); + + shrink_dcache_inode(inode); + iput(toput_inode); + toput_inode = inode; + + spin_lock(&mk->mk_decrypted_inodes_lock); + } + + spin_unlock(&mk->mk_decrypted_inodes_lock); + iput(toput_inode); +} + +static int check_for_busy_inodes(struct super_block *sb, + struct fscrypt_master_key *mk) +{ + struct list_head *pos; + size_t busy_count = 0; + char ino_str[50] = ""; + u64 ino; + + spin_lock(&mk->mk_decrypted_inodes_lock); + + list_for_each(pos, &mk->mk_decrypted_inodes) + busy_count++; + + if (busy_count == 0) { + spin_unlock(&mk->mk_decrypted_inodes_lock); + return 0; + } + + { + /* select an example file to show for debugging purposes */ + struct inode *inode = + list_first_entry(&mk->mk_decrypted_inodes, + struct fscrypt_inode_info, + ci_master_key_link)->ci_inode; + ino = inode->i_ino; + } + spin_unlock(&mk->mk_decrypted_inodes_lock); + + /* If the inode is currently being created, ino may still be 0. */ + if (ino) + snprintf(ino_str, sizeof(ino_str), ", including ino %llu", ino); + + fscrypt_warn(NULL, + "%s: %zu inode(s) still busy after removing key with %s %*phN%s", + sb->s_id, busy_count, master_key_spec_type(&mk->mk_spec), + master_key_spec_len(&mk->mk_spec), (u8 *)&mk->mk_spec.u, + ino_str); + return -EBUSY; +} + +static int try_to_lock_encrypted_files(struct super_block *sb, + struct fscrypt_master_key *mk) +{ + int err1; + int err2; + + /* + * An inode can't be evicted while it is dirty or has dirty pages. + * Thus, we first have to clean the inodes in ->mk_decrypted_inodes. + * + * Just do it the easy way: call sync_filesystem(). It's overkill, but + * it works, and it's more important to minimize the amount of caches we + * drop than the amount of data we sync. Also, unprivileged users can + * already call sync_filesystem() via sys_syncfs() or sys_sync(). + */ + down_read(&sb->s_umount); + err1 = sync_filesystem(sb); + up_read(&sb->s_umount); + /* If a sync error occurs, still try to evict as much as possible. */ + + /* + * Inodes are pinned by their dentries, so we have to evict their + * dentries. shrink_dcache_sb() would suffice, but would be overkill + * and inappropriate for use by unprivileged users. So instead go + * through the inodes' alias lists and try to evict each dentry. + */ + evict_dentries_for_decrypted_inodes(mk); + + /* + * evict_dentries_for_decrypted_inodes() already iput() each inode in + * the list; any inodes for which that dropped the last reference will + * have been evicted due to fscrypt_drop_inode() detecting the key + * removal and telling the VFS to evict the inode. So to finish, we + * just need to check whether any inodes couldn't be evicted. + */ + err2 = check_for_busy_inodes(sb, mk); + + return err1 ?: err2; +} + +/* + * Try to remove an fscrypt master encryption key. + * + * FS_IOC_REMOVE_ENCRYPTION_KEY (all_users=false) removes the current user's + * claim to the key, then removes the key itself if no other users have claims. + * FS_IOC_REMOVE_ENCRYPTION_KEY_ALL_USERS (all_users=true) always removes the + * key itself. + * + * To "remove the key itself", first we transition the key to the "incompletely + * removed" state, so that no more inodes can be unlocked with it. Then we try + * to evict all cached inodes that had been unlocked with the key. + * + * If all inodes were evicted, then we unlink the fscrypt_master_key from the + * keyring. Otherwise it remains in the keyring in the "incompletely removed" + * state where it tracks the list of remaining inodes. Userspace can execute + * the ioctl again later to retry eviction, or alternatively can re-add the key. + * + * For more details, see the "Removing keys" section of + * Documentation/filesystems/fscrypt.rst. + */ +static int do_remove_key(struct file *filp, void __user *_uarg, bool all_users) +{ + struct super_block *sb = file_inode(filp)->i_sb; + struct fscrypt_remove_key_arg __user *uarg = _uarg; + struct fscrypt_remove_key_arg arg; + struct fscrypt_master_key *mk; + u32 status_flags = 0; + int err; + bool inodes_remain; + + if (copy_from_user(&arg, uarg, sizeof(arg))) + return -EFAULT; + + if (!valid_key_spec(&arg.key_spec)) + return -EINVAL; + + if (memchr_inv(arg.__reserved, 0, sizeof(arg.__reserved))) + return -EINVAL; + + /* + * Only root can add and remove keys that are identified by an arbitrary + * descriptor rather than by a cryptographic hash. + */ + if (arg.key_spec.type == FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR && + !capable(CAP_SYS_ADMIN)) + return -EACCES; + + /* Find the key being removed. */ + mk = fscrypt_find_master_key(sb, &arg.key_spec); + if (!mk) + return -ENOKEY; + down_write(&mk->mk_sem); + + /* If relevant, remove current user's (or all users) claim to the key */ + if (!list_empty(&mk->mk_users)) { + if (all_users) { + clear_mk_users(mk); + err = 0; + } else { + err = remove_master_key_user(mk); + } + if (err) { + up_write(&mk->mk_sem); + goto out_put_key; + } + if (!list_empty(&mk->mk_users)) { + /* + * Other users have still added the key too. We removed + * the current user's claim to the key, but we still + * can't remove the key itself. + */ + status_flags |= + FSCRYPT_KEY_REMOVAL_STATUS_FLAG_OTHER_USERS; + err = 0; + up_write(&mk->mk_sem); + goto out_put_key; + } + } + + /* No user claims remaining. Initiate removal of the key. */ + err = -ENOKEY; + if (mk->mk_present) { + fscrypt_initiate_key_removal(sb, mk); + err = 0; + } + inodes_remain = refcount_read(&mk->mk_active_refs) > 0; + up_write(&mk->mk_sem); + + if (inodes_remain) { + /* Some inodes still reference this key; try to evict them. */ + err = try_to_lock_encrypted_files(sb, mk); + if (err == -EBUSY) { + status_flags |= + FSCRYPT_KEY_REMOVAL_STATUS_FLAG_FILES_BUSY; + err = 0; + } + } + /* + * We return 0 if we successfully did something: removed a claim to the + * key, initiated removal of the key, or tried locking the files again. + * Users need to check the informational status flags if they care + * whether the key has been fully removed including all files locked. + */ +out_put_key: + fscrypt_put_master_key(mk); + if (err == 0) + err = put_user(status_flags, &uarg->removal_status_flags); + return err; +} + +int fscrypt_ioctl_remove_key(struct file *filp, void __user *uarg) +{ + return do_remove_key(filp, uarg, false); +} +EXPORT_SYMBOL_GPL(fscrypt_ioctl_remove_key); + +int fscrypt_ioctl_remove_key_all_users(struct file *filp, void __user *uarg) +{ + if (!capable(CAP_SYS_ADMIN)) + return -EACCES; + return do_remove_key(filp, uarg, true); +} +EXPORT_SYMBOL_GPL(fscrypt_ioctl_remove_key_all_users); + +/* + * Retrieve the status of an fscrypt master encryption key. + * + * We set ->status to indicate whether the key is absent, present, or + * incompletely removed. (For an explanation of what these statuses mean and + * how they are represented internally, see struct fscrypt_master_key.) This + * field allows applications to easily determine the status of an encrypted + * directory without using a hack such as trying to open a regular file in it + * (which can confuse the "incompletely removed" status with absent or present). + * + * In addition, for v2 policy keys we allow applications to determine, via + * ->status_flags and ->user_count, whether the key has been added by the + * current user, by other users, or by both. Most applications should not need + * this, since ordinarily only one user should know a given key. However, if a + * secret key is shared by multiple users, applications may wish to add an + * already-present key to prevent other users from removing it. This ioctl can + * be used to check whether that really is the case before the work is done to + * add the key --- which might e.g. require prompting the user for a passphrase. + * + * For more details, see the "FS_IOC_GET_ENCRYPTION_KEY_STATUS" section of + * Documentation/filesystems/fscrypt.rst. + */ +int fscrypt_ioctl_get_key_status(struct file *filp, void __user *uarg) +{ + struct super_block *sb = file_inode(filp)->i_sb; + struct fscrypt_get_key_status_arg arg; + struct fscrypt_master_key *mk; + kuid_t uid; + const struct fscrypt_master_key_user *mk_user; + int err; + + if (copy_from_user(&arg, uarg, sizeof(arg))) + return -EFAULT; + + if (!valid_key_spec(&arg.key_spec)) + return -EINVAL; + + if (memchr_inv(arg.__reserved, 0, sizeof(arg.__reserved))) + return -EINVAL; + + arg.status_flags = 0; + arg.user_count = 0; + memset(arg.__out_reserved, 0, sizeof(arg.__out_reserved)); + + mk = fscrypt_find_master_key(sb, &arg.key_spec); + if (!mk) { + arg.status = FSCRYPT_KEY_STATUS_ABSENT; + err = 0; + goto out; + } + down_read(&mk->mk_sem); + + if (!mk->mk_present) { + arg.status = refcount_read(&mk->mk_active_refs) > 0 ? + FSCRYPT_KEY_STATUS_INCOMPLETELY_REMOVED : + FSCRYPT_KEY_STATUS_ABSENT /* raced with full removal */; + err = 0; + goto out_release_key; + } + + arg.status = FSCRYPT_KEY_STATUS_PRESENT; + + uid = current_fsuid(); + list_for_each_entry(mk_user, &mk->mk_users, link) { + arg.user_count++; + if (uid_eq(mk_user->uid, uid)) + arg.status_flags |= + FSCRYPT_KEY_STATUS_FLAG_ADDED_BY_SELF; + } + err = 0; +out_release_key: + up_read(&mk->mk_sem); + fscrypt_put_master_key(mk); +out: + if (!err && copy_to_user(uarg, &arg, sizeof(arg))) + err = -EFAULT; + return err; +} +EXPORT_SYMBOL_GPL(fscrypt_ioctl_get_key_status); + +void __init fscrypt_init_keyring(void) +{ + int err; + + /* + * Note that register_key_type() fails only if a key type with the same + * name already exists, which should never happen here. + */ + err = register_key_type(&key_type_fscrypt_user); + if (err) + panic("failed to register .fscrypt key type (%d)", err); + err = register_key_type(&key_type_fscrypt_provisioning); + if (err) + panic("failed to register fscrypt-provisioning key type (%d)", + err); +} diff --git a/fs/crypto/keysetup.c b/fs/crypto/keysetup.c new file mode 100644 index 000000000..892044ebc --- /dev/null +++ b/fs/crypto/keysetup.c @@ -0,0 +1,850 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Key setup facility for FS encryption support. + * + * Copyright (C) 2015, Google, Inc. + * + * Originally written by Michael Halcrow, Ildar Muslukhov, and Uday Savagaonkar. + * Heavily modified since then. + */ + +#include +#include +#include + +#include "fscrypt_private.h" + +struct fscrypt_mode fscrypt_modes[] = { + [FSCRYPT_MODE_AES_256_XTS] = { + .friendly_name = "AES-256-XTS", + .cipher_str = "xts(aes)", + .keysize = 64, + .security_strength = 32, + .ivsize = 16, + .blk_crypto_mode = BLK_ENCRYPTION_MODE_AES_256_XTS, + }, + [FSCRYPT_MODE_AES_256_CTS] = { + .friendly_name = "AES-256-CBC-CTS", + .cipher_str = "cts(cbc(aes))", + .keysize = 32, + .security_strength = 32, + .ivsize = 16, + }, + [FSCRYPT_MODE_AES_128_CBC] = { + .friendly_name = "AES-128-CBC-ESSIV", + .cipher_str = "essiv(cbc(aes),sha256)", + .keysize = 16, + .security_strength = 16, + .ivsize = 16, + .blk_crypto_mode = BLK_ENCRYPTION_MODE_AES_128_CBC_ESSIV, + }, + [FSCRYPT_MODE_AES_128_CTS] = { + .friendly_name = "AES-128-CBC-CTS", + .cipher_str = "cts(cbc(aes))", + .keysize = 16, + .security_strength = 16, + .ivsize = 16, + }, + [FSCRYPT_MODE_SM4_XTS] = { + .friendly_name = "SM4-XTS", + .cipher_str = "xts(sm4)", + .keysize = 32, + .security_strength = 16, + .ivsize = 16, + .blk_crypto_mode = BLK_ENCRYPTION_MODE_SM4_XTS, + }, + [FSCRYPT_MODE_SM4_CTS] = { + .friendly_name = "SM4-CBC-CTS", + .cipher_str = "cts(cbc(sm4))", + .keysize = 16, + .security_strength = 16, + .ivsize = 16, + }, + [FSCRYPT_MODE_ADIANTUM] = { + .friendly_name = "Adiantum", + .cipher_str = "adiantum(xchacha12,aes)", + .keysize = 32, + .security_strength = 32, + .ivsize = 32, + .blk_crypto_mode = BLK_ENCRYPTION_MODE_ADIANTUM, + }, + [FSCRYPT_MODE_AES_256_HCTR2] = { + .friendly_name = "AES-256-HCTR2", + .cipher_str = "hctr2(aes)", + .keysize = 32, + .security_strength = 32, + .ivsize = 32, + }, +}; + +static DEFINE_MUTEX(fscrypt_mode_key_setup_mutex); + +static struct fscrypt_mode * +select_encryption_mode(const union fscrypt_policy *policy, + const struct inode *inode) +{ + if (S_ISREG(inode->i_mode)) + return &fscrypt_modes[fscrypt_policy_contents_mode(policy)]; + + if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode)) + return &fscrypt_modes[fscrypt_policy_fnames_mode(policy)]; + + WARN_ONCE(1, "fscrypt: filesystem tried to load encryption info for inode %llu, which is not encryptable (file type %d)\n", + inode->i_ino, (inode->i_mode & S_IFMT)); + return ERR_PTR(-EINVAL); +} + +/* Create a symmetric cipher object for the given encryption mode and key */ +static struct crypto_sync_skcipher * +fscrypt_allocate_skcipher(struct fscrypt_mode *mode, const u8 *raw_key, + const struct inode *inode) +{ + struct crypto_sync_skcipher *tfm; + int err; + + tfm = crypto_alloc_sync_skcipher(mode->cipher_str, 0, + FSCRYPT_CRYPTOAPI_MASK); + if (IS_ERR(tfm)) { + if (PTR_ERR(tfm) == -ENOENT) { + fscrypt_warn(inode, + "Missing crypto API support for %s (API name: \"%s\")", + mode->friendly_name, mode->cipher_str); + return ERR_PTR(-ENOPKG); + } + fscrypt_err(inode, "Error allocating '%s' transform: %ld", + mode->cipher_str, PTR_ERR(tfm)); + return tfm; + } + if (!xchg(&mode->logged_cryptoapi_impl, 1)) { + /* + * fscrypt performance can vary greatly depending on which + * crypto algorithm implementation is used. Help people debug + * performance problems by logging the ->cra_driver_name the + * first time a mode is used. + */ + pr_info("fscrypt: %s using implementation \"%s\"\n", + mode->friendly_name, + crypto_skcipher_driver_name(&tfm->base)); + } + if (WARN_ON_ONCE(crypto_sync_skcipher_ivsize(tfm) != mode->ivsize)) { + err = -EINVAL; + goto err_free_tfm; + } + crypto_sync_skcipher_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); + err = crypto_sync_skcipher_setkey(tfm, raw_key, mode->keysize); + if (err) + goto err_free_tfm; + + return tfm; + +err_free_tfm: + crypto_free_sync_skcipher(tfm); + return ERR_PTR(err); +} + +/* + * Prepare the crypto transform object or blk-crypto key in @prep_key, given the + * raw key, encryption mode (@ci->ci_mode), predicate indicating which style of + * key is needed (fscrypt_using_inline_encryption(ci)), IV generation method + * (@ci->ci_policy.flags), and data unit size (@ci->ci_data_unit_bits). + */ +int fscrypt_prepare_key(struct fscrypt_prepared_key *prep_key, + const u8 *raw_key, const struct fscrypt_inode_info *ci) +{ + struct crypto_sync_skcipher *tfm; + + if (fscrypt_using_inline_encryption(ci)) + return fscrypt_prepare_inline_crypt_key(prep_key, raw_key, + ci->ci_mode->keysize, + false, ci); + + tfm = fscrypt_allocate_skcipher(ci->ci_mode, raw_key, ci->ci_inode); + if (IS_ERR(tfm)) + return PTR_ERR(tfm); + prep_key->tfm = tfm; + return 0; +} + +/* Destroy a crypto transform object and/or blk-crypto key. */ +void fscrypt_destroy_prepared_key(struct super_block *sb, + struct fscrypt_prepared_key *prep_key) +{ + crypto_free_sync_skcipher(prep_key->tfm); + fscrypt_destroy_inline_crypt_key(sb, prep_key); + memzero_explicit(prep_key, sizeof(*prep_key)); +} + +/* Given a per-file encryption key, set up the file's crypto transform object */ +int fscrypt_set_per_file_enc_key(struct fscrypt_inode_info *ci, + const u8 *raw_key) +{ + ci->ci_owns_key = true; + return fscrypt_prepare_key(&ci->ci_enc_key, raw_key, ci); +} + +/* + * Find the fscrypt_prepared_key (if any) for a particular (mk, hkdf_context, + * mode_num, data_unit_bits, inlinecrypt) combination. + * + * The caller must hold ->mk_sem for reading and ->mk_present must be true, + * ensuring that ->mk_mode_keys is still append-only. + */ +static struct fscrypt_prepared_key * +fscrypt_find_mode_key(struct fscrypt_master_key *mk, u8 hkdf_context, + u8 mode_num, const struct fscrypt_inode_info *ci) +{ + struct fscrypt_mode_key *node; + + /* + * The RCU read lock here is used only to synchronize with concurrent + * list_add_tail_rcu(). Concurrent deletions are impossible here, so + * returning a pointer to a node without taking any refcount is safe. + */ + guard(rcu)(); + list_for_each_entry_rcu(node, &mk->mk_mode_keys, link) { + if (node->hkdf_context == hkdf_context && + node->mode_num == mode_num && + node->data_unit_bits == ci->ci_data_unit_bits && + fscrypt_is_key_prepared(&node->key, ci)) + return &node->key; + } + return NULL; +} + +static int setup_per_mode_enc_key(struct fscrypt_inode_info *ci, + struct fscrypt_master_key *mk, + u8 hkdf_context, bool include_fs_uuid) +{ + const struct inode *inode = ci->ci_inode; + const struct super_block *sb = inode->i_sb; + struct fscrypt_mode *mode = ci->ci_mode; + const u8 mode_num = mode - fscrypt_modes; + struct fscrypt_prepared_key *prep_key; + struct fscrypt_mode_key *new_node; + u8 raw_mode_key[FSCRYPT_MAX_RAW_KEY_SIZE]; + u8 hkdf_info[sizeof(mode_num) + sizeof(sb->s_uuid)]; + unsigned int hkdf_infolen = 0; + int err; + + prep_key = fscrypt_find_mode_key(mk, hkdf_context, mode_num, ci); + if (prep_key) { + ci->ci_enc_key = *prep_key; + return 0; + } + + guard(mutex)(&fscrypt_mode_key_setup_mutex); + + prep_key = fscrypt_find_mode_key(mk, hkdf_context, mode_num, ci); + if (prep_key) { + ci->ci_enc_key = *prep_key; + return 0; + } + + new_node = kzalloc_obj(*new_node); + if (!new_node) + return -ENOMEM; + new_node->hkdf_context = hkdf_context; + new_node->mode_num = mode_num; + new_node->data_unit_bits = ci->ci_data_unit_bits; + prep_key = &new_node->key; + + if (mk->mk_secret.is_hw_wrapped && S_ISREG(inode->i_mode)) { + err = fscrypt_prepare_inline_crypt_key(prep_key, + mk->mk_secret.bytes, + mk->mk_secret.size, true, + ci); + } else { + static_assert(sizeof(mode_num) == 1); + static_assert(sizeof(sb->s_uuid) == 16); + static_assert(sizeof(hkdf_info) == 17); + hkdf_info[hkdf_infolen++] = mode_num; + if (include_fs_uuid) { + memcpy(&hkdf_info[hkdf_infolen], &sb->s_uuid, + sizeof(sb->s_uuid)); + hkdf_infolen += sizeof(sb->s_uuid); + } + fscrypt_hkdf_expand(&mk->mk_secret.hkdf, hkdf_context, + hkdf_info, hkdf_infolen, raw_mode_key, + mode->keysize); + err = fscrypt_prepare_key(prep_key, raw_mode_key, ci); + memzero_explicit(raw_mode_key, sizeof(raw_mode_key)); + } + if (err) { + kfree(new_node); + return err; + } + list_add_tail_rcu(&new_node->link, &mk->mk_mode_keys); + ci->ci_enc_key = *prep_key; + return 0; +} + +/* + * Derive a SipHash key from the given fscrypt master key and the given + * application-specific information string. + * + * Note that the KDF produces a byte array, but the SipHash APIs expect the key + * as a pair of 64-bit words. Therefore, on big endian CPUs we have to do an + * endianness swap in order to get the same results as on little endian CPUs. + */ +static void fscrypt_derive_siphash_key(const struct fscrypt_master_key *mk, + u8 context, const u8 *info, + unsigned int infolen, siphash_key_t *key) +{ + fscrypt_hkdf_expand(&mk->mk_secret.hkdf, context, info, infolen, + (u8 *)key, sizeof(*key)); + BUILD_BUG_ON(sizeof(*key) != 16); + BUILD_BUG_ON(ARRAY_SIZE(key->key) != 2); + le64_to_cpus(&key->key[0]); + le64_to_cpus(&key->key[1]); +} + +void fscrypt_derive_dirhash_key(struct fscrypt_inode_info *ci, + const struct fscrypt_master_key *mk) +{ + fscrypt_derive_siphash_key(mk, HKDF_CONTEXT_DIRHASH_KEY, + ci->ci_nonce, FSCRYPT_FILE_NONCE_SIZE, + &ci->ci_dirhash_key); + ci->ci_dirhash_key_initialized = true; +} + +void fscrypt_hash_inode_number(struct fscrypt_inode_info *ci, + const struct fscrypt_master_key *mk) +{ + WARN_ON_ONCE(ci->ci_inode->i_ino == 0); + WARN_ON_ONCE(!mk->mk_ino_hash_key_initialized); + + ci->ci_hashed_ino = (u32)siphash_1u64(ci->ci_inode->i_ino, + &mk->mk_ino_hash_key); +} + +static int fscrypt_setup_iv_ino_lblk_32_key(struct fscrypt_inode_info *ci, + struct fscrypt_master_key *mk) +{ + int err; + + err = setup_per_mode_enc_key(ci, mk, HKDF_CONTEXT_IV_INO_LBLK_32_KEY, + true); + if (err) + return err; + + /* pairs with smp_store_release() below */ + if (!smp_load_acquire(&mk->mk_ino_hash_key_initialized)) { + guard(mutex)(&fscrypt_mode_key_setup_mutex); + + if (!mk->mk_ino_hash_key_initialized) { + fscrypt_derive_siphash_key(mk, + HKDF_CONTEXT_INODE_HASH_KEY, + NULL, 0, + &mk->mk_ino_hash_key); + /* pairs with smp_load_acquire() above */ + smp_store_release(&mk->mk_ino_hash_key_initialized, + true); + } + } + + /* + * New inodes may not have an inode number assigned yet. + * Hashing their inode number is delayed until later. + */ + if (ci->ci_inode->i_ino) + fscrypt_hash_inode_number(ci, mk); + return 0; +} + +static int fscrypt_setup_v2_file_key(struct fscrypt_inode_info *ci, + struct fscrypt_master_key *mk, + bool need_dirhash_key) +{ + int err; + + if (mk->mk_secret.is_hw_wrapped && + !(ci->ci_policy.v2.flags & (FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64 | + FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32))) { + fscrypt_warn(ci->ci_inode, + "Hardware-wrapped keys are only supported with IV_INO_LBLK policies"); + return -EINVAL; + } + + if (ci->ci_policy.v2.flags & FSCRYPT_POLICY_FLAG_DIRECT_KEY) { + /* + * DIRECT_KEY: instead of deriving per-file encryption keys, the + * per-file nonce will be included in all the IVs. But unlike + * v1 policies, for v2 policies in this case we don't encrypt + * with the master key directly but rather derive a per-mode + * encryption key. This ensures that the master key is + * consistently used only for HKDF, avoiding key reuse issues. + */ + err = setup_per_mode_enc_key(ci, mk, HKDF_CONTEXT_DIRECT_KEY, + false); + } else if (ci->ci_policy.v2.flags & + FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64) { + /* + * IV_INO_LBLK_64: encryption keys are derived from (master_key, + * mode_num, filesystem_uuid), and inode number is included in + * the IVs. This format is optimized for use with inline + * encryption hardware compliant with the UFS standard. + */ + err = setup_per_mode_enc_key( + ci, mk, HKDF_CONTEXT_IV_INO_LBLK_64_KEY, true); + } else if (ci->ci_policy.v2.flags & + FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32) { + err = fscrypt_setup_iv_ino_lblk_32_key(ci, mk); + } else { + u8 derived_key[FSCRYPT_MAX_RAW_KEY_SIZE]; + + fscrypt_hkdf_expand(&mk->mk_secret.hkdf, + HKDF_CONTEXT_PER_FILE_ENC_KEY, + ci->ci_nonce, FSCRYPT_FILE_NONCE_SIZE, + derived_key, ci->ci_mode->keysize); + err = fscrypt_set_per_file_enc_key(ci, derived_key); + memzero_explicit(derived_key, sizeof(derived_key)); + } + if (err) + return err; + + /* Derive a secret dirhash key for directories that need it. */ + if (need_dirhash_key) + fscrypt_derive_dirhash_key(ci, mk); + + return 0; +} + +/* + * Check whether the size of the given master key (@mk) is appropriate for the + * encryption settings which a particular file will use (@ci). + * + * If the file uses a v1 encryption policy, then the master key must be at least + * as long as the derived key, as this is a requirement of the v1 KDF. + * + * Otherwise, the KDF can accept any size key, so we enforce a slightly looser + * requirement: we require that the size of the master key be at least the + * maximum security strength of any algorithm whose key will be derived from it + * (but in practice we only need to consider @ci->ci_mode, since any other + * possible subkeys such as DIRHASH and INODE_HASH will never increase the + * required key size over @ci->ci_mode). This allows AES-256-XTS keys to be + * derived from a 256-bit master key, which is cryptographically sufficient, + * rather than requiring a 512-bit master key which is unnecessarily long. (We + * still allow 512-bit master keys if the user chooses to use them, though.) + */ +static bool fscrypt_valid_master_key_size(const struct fscrypt_master_key *mk, + const struct fscrypt_inode_info *ci) +{ + unsigned int min_keysize; + + if (ci->ci_policy.version == FSCRYPT_POLICY_V1) + min_keysize = ci->ci_mode->keysize; + else + min_keysize = ci->ci_mode->security_strength; + + if (mk->mk_secret.size < min_keysize) { + fscrypt_warn(NULL, + "key with %s %*phN is too short (got %u bytes, need %u+ bytes)", + master_key_spec_type(&mk->mk_spec), + master_key_spec_len(&mk->mk_spec), + (u8 *)&mk->mk_spec.u, + mk->mk_secret.size, min_keysize); + return false; + } + return true; +} + +/* + * Find the master key, then set up the inode's actual encryption key. + * + * If the master key is found in the filesystem-level keyring, then it is + * returned in *mk_ret with its semaphore read-locked. This is needed to ensure + * that only one task links the fscrypt_inode_info into ->mk_decrypted_inodes + * (as multiple tasks may race to create an fscrypt_inode_info for the same + * inode), and to synchronize the master key being removed with a new inode + * starting to use it. + */ +static int setup_file_encryption_key(struct fscrypt_inode_info *ci, + bool need_dirhash_key, + struct fscrypt_master_key **mk_ret) +{ + struct super_block *sb = ci->ci_inode->i_sb; + struct fscrypt_key_specifier mk_spec; + struct fscrypt_master_key *mk; + int err; + + err = fscrypt_policy_to_key_spec(&ci->ci_policy, &mk_spec); + if (err) + return err; + + mk = fscrypt_find_master_key(sb, &mk_spec); + if (unlikely(!mk)) { + const union fscrypt_policy *dummy_policy = + fscrypt_get_dummy_policy(sb); + + /* + * Add the test_dummy_encryption key on-demand. In principle, + * it should be added at mount time. Do it here instead so that + * the individual filesystems don't need to worry about adding + * this key at mount time and cleaning up on mount failure. + */ + if (dummy_policy && + fscrypt_policies_equal(dummy_policy, &ci->ci_policy)) { + err = fscrypt_add_test_dummy_key(sb, &mk_spec); + if (err) + return err; + mk = fscrypt_find_master_key(sb, &mk_spec); + } + } + if (unlikely(!mk)) { + if (ci->ci_policy.version != FSCRYPT_POLICY_V1) + return -ENOKEY; + + /* + * As a legacy fallback for v1 policies, search for the key in + * the current task's subscribed keyrings too. Don't move this + * to before the search of ->s_master_keys, since users + * shouldn't be able to override filesystem-level keys. + */ + return fscrypt_setup_v1_file_key_via_subscribed_keyrings(ci); + } + down_read(&mk->mk_sem); + + if (!mk->mk_present) { + /* FS_IOC_REMOVE_ENCRYPTION_KEY has been executed on this key */ + err = -ENOKEY; + goto out_release_key; + } + + if (!fscrypt_valid_master_key_size(mk, ci)) { + err = -ENOKEY; + goto out_release_key; + } + + switch (ci->ci_policy.version) { + case FSCRYPT_POLICY_V1: + if (WARN_ON_ONCE(mk->mk_secret.is_hw_wrapped)) { + /* + * This should never happen, as adding a v1 policy key + * that is hardware-wrapped isn't allowed. + */ + err = -EINVAL; + goto out_release_key; + } + err = fscrypt_setup_v1_file_key(ci, mk->mk_secret.bytes); + break; + case FSCRYPT_POLICY_V2: + err = fscrypt_setup_v2_file_key(ci, mk, need_dirhash_key); + break; + default: + WARN_ON_ONCE(1); + err = -EINVAL; + break; + } + if (err) + goto out_release_key; + + *mk_ret = mk; + return 0; + +out_release_key: + up_read(&mk->mk_sem); + fscrypt_put_master_key(mk); + return err; +} + +static void put_crypt_info(struct fscrypt_inode_info *ci) +{ + struct fscrypt_master_key *mk; + + if (!ci) + return; + + if (ci->ci_direct_key) + fscrypt_put_direct_key(ci->ci_direct_key); + else if (ci->ci_owns_key) + fscrypt_destroy_prepared_key(ci->ci_inode->i_sb, + &ci->ci_enc_key); + + mk = ci->ci_master_key; + if (mk) { + /* + * Remove this inode from the list of inodes that were unlocked + * with the master key. In addition, if we're removing the last + * inode from an incompletely removed key, then complete the + * full removal of the key. + */ + spin_lock(&mk->mk_decrypted_inodes_lock); + list_del(&ci->ci_master_key_link); + spin_unlock(&mk->mk_decrypted_inodes_lock); + fscrypt_put_master_key_activeref(ci->ci_inode->i_sb, mk); + } + memzero_explicit(ci, sizeof(*ci)); + kmem_cache_free(fscrypt_inode_info_cachep, ci); +} + +static int +fscrypt_setup_encryption_info(struct inode *inode, + const union fscrypt_policy *policy, + const u8 nonce[FSCRYPT_FILE_NONCE_SIZE], + bool need_dirhash_key) +{ + struct fscrypt_inode_info *crypt_info; + struct fscrypt_mode *mode; + struct fscrypt_master_key *mk = NULL; + int res; + + res = fscrypt_initialize(inode->i_sb); + if (res) + return res; + + crypt_info = kmem_cache_zalloc(fscrypt_inode_info_cachep, GFP_KERNEL); + if (!crypt_info) + return -ENOMEM; + + crypt_info->ci_inode = inode; + crypt_info->ci_policy = *policy; + memcpy(crypt_info->ci_nonce, nonce, FSCRYPT_FILE_NONCE_SIZE); + + mode = select_encryption_mode(&crypt_info->ci_policy, inode); + if (IS_ERR(mode)) { + res = PTR_ERR(mode); + goto out; + } + WARN_ON_ONCE(mode->ivsize > FSCRYPT_MAX_IV_SIZE); + crypt_info->ci_mode = mode; + + crypt_info->ci_data_unit_bits = + fscrypt_policy_du_bits(&crypt_info->ci_policy, inode); + + res = setup_file_encryption_key(crypt_info, need_dirhash_key, &mk); + if (res) + goto out; + + /* + * For existing inodes, multiple tasks may race to set the inode's + * fscrypt info pointer. So use cmpxchg_release(). This pairs with the + * smp_load_acquire() in fscrypt_get_inode_info(). I.e., publish the + * pointer with a RELEASE barrier so that other tasks can ACQUIRE it. + */ + if (cmpxchg_release(fscrypt_inode_info_addr(inode), NULL, crypt_info) == + NULL) { + /* + * We won the race and set the inode's fscrypt info to our + * crypt_info. Now link it into the master key's inode list. + */ + if (mk) { + crypt_info->ci_master_key = mk; + refcount_inc(&mk->mk_active_refs); + spin_lock(&mk->mk_decrypted_inodes_lock); + list_add(&crypt_info->ci_master_key_link, + &mk->mk_decrypted_inodes); + spin_unlock(&mk->mk_decrypted_inodes_lock); + } + crypt_info = NULL; + } + res = 0; +out: + if (mk) { + up_read(&mk->mk_sem); + fscrypt_put_master_key(mk); + } + put_crypt_info(crypt_info); + return res; +} + +/** + * fscrypt_get_encryption_info() - set up an inode's encryption key + * @inode: the inode to set up the key for. Must be encrypted. + * @allow_unsupported: if %true, treat an unsupported encryption policy (or + * unrecognized encryption context) the same way as the key + * being unavailable, instead of returning an error. Use + * %false unless the operation being performed is needed in + * order for files (or directories) to be deleted. + * + * Set up the inode's encryption key, if it hasn't already been done. + * + * Note: unless the key setup was already done, this isn't %GFP_NOFS-safe. So + * generally this shouldn't be called from within a filesystem transaction. + * + * Return: 0 if the key is now set up, *or* if it couldn't be set up because the + * needed master key is absent. (Use fscrypt_has_encryption_key() to + * distinguish these cases.) Also can return another -errno code. + */ +int fscrypt_get_encryption_info(struct inode *inode, bool allow_unsupported) +{ + int res; + union fscrypt_context ctx; + union fscrypt_policy policy; + + if (fscrypt_has_encryption_key(inode)) + return 0; + + res = inode->i_sb->s_cop->get_context(inode, &ctx, sizeof(ctx)); + if (res < 0) { + if (res == -ERANGE && allow_unsupported) + return 0; + fscrypt_warn(inode, "Error %d getting encryption context", res); + return res; + } + + res = fscrypt_policy_from_context(&policy, &ctx, res); + if (res) { + if (allow_unsupported) + return 0; + fscrypt_warn(inode, + "Unrecognized or corrupt encryption context"); + return res; + } + + if (!fscrypt_supported_policy(&policy, inode)) { + if (allow_unsupported) + return 0; + return -EINVAL; + } + + res = fscrypt_setup_encryption_info(inode, &policy, + fscrypt_context_nonce(&ctx), + IS_CASEFOLDED(inode) && + S_ISDIR(inode->i_mode)); + + if (res == -ENOPKG && allow_unsupported) /* Algorithm unavailable? */ + res = 0; + if (res == -ENOKEY) + res = 0; + return res; +} + +/** + * fscrypt_prepare_new_inode() - prepare to create a new inode in a directory + * @dir: a possibly-encrypted directory + * @inode: the new inode. ->i_mode and ->i_blkbits must be set already. + * ->i_ino doesn't need to be set yet. + * @encrypt_ret: (output) set to %true if the new inode will be encrypted + * + * If the directory is encrypted, set up its encryption key in preparation for + * encrypting the name of the new file. Also, if the new inode will be + * encrypted, set up its encryption key too and set *encrypt_ret=true. + * + * This isn't %GFP_NOFS-safe, and therefore it should be called before starting + * any filesystem transaction to create the inode. For this reason, ->i_ino + * isn't required to be set yet, as the filesystem may not have set it yet. + * + * This doesn't persist the new inode's encryption context. That still needs to + * be done later by calling fscrypt_set_context(). + * + * Return: 0 on success, -ENOKEY if a key needs to be set up for @dir or @inode + * but the needed master key is absent, or another -errno code + */ +int fscrypt_prepare_new_inode(struct inode *dir, struct inode *inode, + bool *encrypt_ret) +{ + const union fscrypt_policy *policy; + u8 nonce[FSCRYPT_FILE_NONCE_SIZE]; + + policy = fscrypt_policy_to_inherit(dir); + if (policy == NULL) + return 0; + if (IS_ERR(policy)) + return PTR_ERR(policy); + + if (WARN_ON_ONCE(inode->i_blkbits == 0)) + return -EINVAL; + + if (WARN_ON_ONCE(inode->i_mode == 0)) + return -EINVAL; + + /* + * Only regular files, directories, and symlinks are encrypted. + * Special files like device nodes and named pipes aren't. + */ + if (!S_ISREG(inode->i_mode) && + !S_ISDIR(inode->i_mode) && + !S_ISLNK(inode->i_mode)) + return 0; + + *encrypt_ret = true; + + get_random_bytes(nonce, FSCRYPT_FILE_NONCE_SIZE); + return fscrypt_setup_encryption_info(inode, policy, nonce, + IS_CASEFOLDED(dir) && + S_ISDIR(inode->i_mode)); +} +EXPORT_SYMBOL_GPL(fscrypt_prepare_new_inode); + +/** + * fscrypt_put_encryption_info() - free most of an inode's fscrypt data + * @inode: an inode being evicted + * + * Free the inode's fscrypt_inode_info. Filesystems must call this when the + * inode is being evicted. An RCU grace period need not have elapsed yet. + */ +void fscrypt_put_encryption_info(struct inode *inode) +{ + /* + * Ideally we'd start with a lightweight IS_ENCRYPTED() check here + * before proceeding to retrieve and check the pointer. However, during + * inode creation, the fscrypt_inode_info is set before S_ENCRYPTED. If + * an error occurs, it needs to be cleaned up regardless. + */ + struct fscrypt_inode_info **ci_addr = fscrypt_inode_info_addr(inode); + + put_crypt_info(*ci_addr); + *ci_addr = NULL; +} +EXPORT_SYMBOL(fscrypt_put_encryption_info); + +/** + * fscrypt_free_inode() - free an inode's fscrypt data requiring RCU delay + * @inode: an inode being freed + * + * Free the inode's cached decrypted symlink target, if any. Filesystems must + * call this after an RCU grace period, just before they free the inode. + */ +void fscrypt_free_inode(struct inode *inode) +{ + if (IS_ENCRYPTED(inode) && S_ISLNK(inode->i_mode)) { + kfree(inode->i_link); + inode->i_link = NULL; + } +} +EXPORT_SYMBOL(fscrypt_free_inode); + +/** + * fscrypt_drop_inode() - check whether the inode's master key has been removed + * @inode: an inode being considered for eviction + * + * Filesystems supporting fscrypt must call this from their ->drop_inode() + * method so that encrypted inodes are evicted as soon as they're no longer in + * use and their master key has been removed. + * + * Return: 1 if fscrypt wants the inode to be evicted now, otherwise 0 + */ +int fscrypt_drop_inode(struct inode *inode) +{ + const struct fscrypt_inode_info *ci = fscrypt_get_inode_info(inode); + + /* + * If ci is NULL, then the inode doesn't have an encryption key set up + * so it's irrelevant. If ci_master_key is NULL, then the master key + * was provided via the legacy mechanism of the process-subscribed + * keyrings, so we don't know whether it's been removed or not. + */ + if (!ci || !ci->ci_master_key) + return 0; + + /* + * With proper, non-racy use of FS_IOC_REMOVE_ENCRYPTION_KEY, all inodes + * protected by the key were cleaned by sync_filesystem(). But if + * userspace is still using the files, inodes can be dirtied between + * then and now. We mustn't lose any writes, so skip dirty inodes here. + */ + if (inode_state_read(inode) & I_DIRTY_ALL) + return 0; + + /* + * We can't take ->mk_sem here, since this runs in atomic context. + * Therefore, ->mk_present can change concurrently, and our result may + * immediately become outdated. But there's no correctness problem with + * unnecessarily evicting. Nor is there a correctness problem with not + * evicting while iput() is racing with the key being removed, since + * then the thread removing the key will either evict the inode itself + * or will correctly detect that it wasn't evicted due to the race. + */ + return !READ_ONCE(ci->ci_master_key->mk_present); +} +EXPORT_SYMBOL_GPL(fscrypt_drop_inode); diff --git a/fs/crypto/keysetup_v1.c b/fs/crypto/keysetup_v1.c new file mode 100644 index 000000000..87fe13ccb --- /dev/null +++ b/fs/crypto/keysetup_v1.c @@ -0,0 +1,291 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Key setup for v1 encryption policies + * + * Copyright 2015, 2019 Google LLC + */ + +/* + * This file implements compatibility functions for the original encryption + * policy version ("v1"), including: + * + * - Deriving per-file encryption keys using the AES-128-ECB based KDF + * (rather than the new method of using HKDF-SHA512) + * + * - Retrieving fscrypt master keys from process-subscribed keyrings + * (rather than the new method of using a filesystem-level keyring) + * + * - Handling policies with the DIRECT_KEY flag set using a master key table + * (rather than the new method of implementing DIRECT_KEY with per-mode keys + * managed alongside the master keys in the filesystem-level keyring) + */ + +#include +#include +#include +#include + +#include "fscrypt_private.h" + +/* Table of keys referenced by DIRECT_KEY policies */ +static DEFINE_HASHTABLE(fscrypt_direct_keys, 6); /* 6 bits = 64 buckets */ +static DEFINE_SPINLOCK(fscrypt_direct_keys_lock); + +/* + * Search the current task's subscribed keyrings for a "logon" key with + * description prefix:descriptor, and if found acquire a read lock on it and + * return a pointer to its validated payload in *payload_ret. + */ +static struct key * +find_and_lock_process_key(const char *prefix, + const u8 descriptor[FSCRYPT_KEY_DESCRIPTOR_SIZE], + unsigned int min_keysize, + const struct fscrypt_key **payload_ret) +{ + char *description; + struct key *key; + const struct user_key_payload *ukp; + const struct fscrypt_key *payload; + + description = kasprintf(GFP_KERNEL, "%s%*phN", prefix, + FSCRYPT_KEY_DESCRIPTOR_SIZE, descriptor); + if (!description) + return ERR_PTR(-ENOMEM); + + key = request_key(&key_type_logon, description, NULL); + kfree(description); + if (IS_ERR(key)) + return key; + + down_read(&key->sem); + ukp = user_key_payload_locked(key); + + if (!ukp) /* was the key revoked before we acquired its semaphore? */ + goto invalid; + + payload = (const struct fscrypt_key *)ukp->data; + + if (ukp->datalen != sizeof(struct fscrypt_key) || + payload->size < 1 || payload->size > sizeof(payload->raw)) { + fscrypt_warn(NULL, + "key with description '%s' has invalid payload", + key->description); + goto invalid; + } + + if (payload->size < min_keysize) { + fscrypt_warn(NULL, + "key with description '%s' is too short (got %u bytes, need %u+ bytes)", + key->description, payload->size, min_keysize); + goto invalid; + } + + *payload_ret = payload; + return key; + +invalid: + up_read(&key->sem); + key_put(key); + return ERR_PTR(-ENOKEY); +} + +/* Master key referenced by DIRECT_KEY policy */ +struct fscrypt_direct_key { + struct super_block *dk_sb; + struct hlist_node dk_node; + refcount_t dk_refcount; + const struct fscrypt_mode *dk_mode; + struct fscrypt_prepared_key dk_key; + u8 dk_descriptor[FSCRYPT_KEY_DESCRIPTOR_SIZE]; + u8 dk_raw[FSCRYPT_MAX_RAW_KEY_SIZE]; +}; + +static void free_direct_key(struct fscrypt_direct_key *dk) +{ + if (dk) { + fscrypt_destroy_prepared_key(dk->dk_sb, &dk->dk_key); + kfree_sensitive(dk); + } +} + +void fscrypt_put_direct_key(struct fscrypt_direct_key *dk) +{ + if (!refcount_dec_and_lock(&dk->dk_refcount, &fscrypt_direct_keys_lock)) + return; + hash_del(&dk->dk_node); + spin_unlock(&fscrypt_direct_keys_lock); + + free_direct_key(dk); +} + +/* + * Find/insert the given key into the fscrypt_direct_keys table. If found, it + * is returned with elevated refcount, and 'to_insert' is freed if non-NULL. If + * not found, 'to_insert' is inserted and returned if it's non-NULL; otherwise + * NULL is returned. + */ +static struct fscrypt_direct_key * +find_or_insert_direct_key(struct fscrypt_direct_key *to_insert, + const u8 *raw_key, + const struct fscrypt_inode_info *ci) +{ + unsigned long hash_key; + struct fscrypt_direct_key *dk; + + /* + * Careful: to avoid potentially leaking secret key bytes via timing + * information, we must key the hash table by descriptor rather than by + * raw key, and use crypto_memneq() when comparing raw keys. + */ + + BUILD_BUG_ON(sizeof(hash_key) > FSCRYPT_KEY_DESCRIPTOR_SIZE); + memcpy(&hash_key, ci->ci_policy.v1.master_key_descriptor, + sizeof(hash_key)); + + spin_lock(&fscrypt_direct_keys_lock); + hash_for_each_possible(fscrypt_direct_keys, dk, dk_node, hash_key) { + if (memcmp(ci->ci_policy.v1.master_key_descriptor, + dk->dk_descriptor, FSCRYPT_KEY_DESCRIPTOR_SIZE) != 0) + continue; + /* The sb is used at eviction time, so it must be the same. */ + if (ci->ci_inode->i_sb != dk->dk_sb) + continue; + if (ci->ci_mode != dk->dk_mode) + continue; + if (!fscrypt_is_key_prepared(&dk->dk_key, ci)) + continue; + if (crypto_memneq(raw_key, dk->dk_raw, ci->ci_mode->keysize)) + continue; + /* + * Use an existing prepared key with the same (descriptor, sb, + * mode, inlinecrypt, raw_key) combination. + */ + refcount_inc(&dk->dk_refcount); + spin_unlock(&fscrypt_direct_keys_lock); + free_direct_key(to_insert); + return dk; + } + if (to_insert) + hash_add(fscrypt_direct_keys, &to_insert->dk_node, hash_key); + spin_unlock(&fscrypt_direct_keys_lock); + return to_insert; +} + +/* Prepare to encrypt directly using the master key in the given mode */ +static struct fscrypt_direct_key * +fscrypt_get_direct_key(const struct fscrypt_inode_info *ci, const u8 *raw_key) +{ + struct fscrypt_direct_key *dk; + int err; + + /* Is there already a tfm for this key? */ + dk = find_or_insert_direct_key(NULL, raw_key, ci); + if (dk) + return dk; + + /* Nope, allocate one. */ + dk = kzalloc_obj(*dk); + if (!dk) + return ERR_PTR(-ENOMEM); + dk->dk_sb = ci->ci_inode->i_sb; + refcount_set(&dk->dk_refcount, 1); + dk->dk_mode = ci->ci_mode; + err = fscrypt_prepare_key(&dk->dk_key, raw_key, ci); + if (err) + goto err_free_dk; + memcpy(dk->dk_descriptor, ci->ci_policy.v1.master_key_descriptor, + FSCRYPT_KEY_DESCRIPTOR_SIZE); + memcpy(dk->dk_raw, raw_key, ci->ci_mode->keysize); + + return find_or_insert_direct_key(dk, raw_key, ci); + +err_free_dk: + free_direct_key(dk); + return ERR_PTR(err); +} + +/* v1 policy, DIRECT_KEY: use the master key directly */ +static int setup_v1_file_key_direct(struct fscrypt_inode_info *ci, + const u8 *raw_master_key) +{ + struct fscrypt_direct_key *dk; + + dk = fscrypt_get_direct_key(ci, raw_master_key); + if (IS_ERR(dk)) + return PTR_ERR(dk); + ci->ci_direct_key = dk; + ci->ci_enc_key = dk->dk_key; + return 0; +} + +/* + * v1 policy, !DIRECT_KEY: derive the file's encryption key. + * + * The v1 key derivation function generates the derived key by encrypting the + * master key with AES-128-ECB using the file's nonce as the AES key. This + * provides a unique derived key with sufficient entropy for each inode. + * However, it's nonstandard, non-extensible, doesn't evenly distribute the + * entropy from the master key, and is trivially reversible: an attacker who + * compromises a derived key can "decrypt" it to get back to the master key, + * then derive any other key. For all new code, use HKDF instead. + * + * The master key must be at least as long as the derived key. If the master + * key is longer, then only the first ci->ci_mode->keysize bytes are used. + */ +static int setup_v1_file_key_derived(struct fscrypt_inode_info *ci, + const u8 *raw_master_key) +{ + const unsigned int derived_keysize = ci->ci_mode->keysize; + u8 derived_key[FSCRYPT_MAX_RAW_KEY_SIZE]; + struct aes_enckey aes; + int err; + + if (WARN_ON_ONCE(derived_keysize > FSCRYPT_MAX_RAW_KEY_SIZE || + derived_keysize % AES_BLOCK_SIZE != 0)) + return -EINVAL; + + static_assert(FSCRYPT_FILE_NONCE_SIZE == AES_KEYSIZE_128); + aes_prepareenckey(&aes, ci->ci_nonce, FSCRYPT_FILE_NONCE_SIZE); + for (unsigned int i = 0; i < derived_keysize; i += AES_BLOCK_SIZE) + aes_encrypt(&aes, &derived_key[i], &raw_master_key[i]); + + err = fscrypt_set_per_file_enc_key(ci, derived_key); + + memzero_explicit(derived_key, sizeof(derived_key)); + /* No need to zeroize 'aes', as its key is not secret. */ + return err; +} + +int fscrypt_setup_v1_file_key(struct fscrypt_inode_info *ci, + const u8 *raw_master_key) +{ + if (ci->ci_policy.v1.flags & FSCRYPT_POLICY_FLAG_DIRECT_KEY) + return setup_v1_file_key_direct(ci, raw_master_key); + else + return setup_v1_file_key_derived(ci, raw_master_key); +} + +int +fscrypt_setup_v1_file_key_via_subscribed_keyrings(struct fscrypt_inode_info *ci) +{ + const struct super_block *sb = ci->ci_inode->i_sb; + struct key *key; + const struct fscrypt_key *payload; + int err; + + key = find_and_lock_process_key(FSCRYPT_KEY_DESC_PREFIX, + ci->ci_policy.v1.master_key_descriptor, + ci->ci_mode->keysize, &payload); + if (key == ERR_PTR(-ENOKEY) && sb->s_cop->legacy_key_prefix) { + key = find_and_lock_process_key(sb->s_cop->legacy_key_prefix, + ci->ci_policy.v1.master_key_descriptor, + ci->ci_mode->keysize, &payload); + } + if (IS_ERR(key)) + return PTR_ERR(key); + + err = fscrypt_setup_v1_file_key(ci, payload->raw); + up_read(&key->sem); + key_put(key); + return err; +} diff --git a/fs/crypto/policy.c b/fs/crypto/policy.c new file mode 100644 index 000000000..6dd510f93 --- /dev/null +++ b/fs/crypto/policy.c @@ -0,0 +1,899 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Encryption policy functions for per-file encryption support. + * + * Copyright (C) 2015, Google, Inc. + * Copyright (C) 2015, Motorola Mobility. + * + * Originally written by Michael Halcrow, 2015. + * Modified by Jaegeuk Kim, 2015. + * Modified by Eric Biggers, 2019 for v2 policy support. + */ + +#include +#include +#include +#include +#include +#include + +#include "fscrypt_private.h" + +/** + * fscrypt_policies_equal() - check whether two encryption policies are the same + * @policy1: the first policy + * @policy2: the second policy + * + * Return: %true if equal, else %false + */ +bool fscrypt_policies_equal(const union fscrypt_policy *policy1, + const union fscrypt_policy *policy2) +{ + if (policy1->version != policy2->version) + return false; + + return !memcmp(policy1, policy2, fscrypt_policy_size(policy1)); +} + +int fscrypt_policy_to_key_spec(const union fscrypt_policy *policy, + struct fscrypt_key_specifier *key_spec) +{ + switch (policy->version) { + case FSCRYPT_POLICY_V1: + key_spec->type = FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR; + memcpy(key_spec->u.descriptor, policy->v1.master_key_descriptor, + FSCRYPT_KEY_DESCRIPTOR_SIZE); + return 0; + case FSCRYPT_POLICY_V2: + key_spec->type = FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER; + memcpy(key_spec->u.identifier, policy->v2.master_key_identifier, + FSCRYPT_KEY_IDENTIFIER_SIZE); + return 0; + default: + WARN_ON_ONCE(1); + return -EINVAL; + } +} + +const union fscrypt_policy *fscrypt_get_dummy_policy(struct super_block *sb) +{ + if (!sb->s_cop->get_dummy_policy) + return NULL; + return sb->s_cop->get_dummy_policy(sb); +} + +/* + * Return %true if the given combination of encryption modes is supported for v1 + * (and later) encryption policies. + * + * Do *not* add anything new here, since v1 encryption policies are deprecated. + * New combinations of modes should go in fscrypt_valid_enc_modes_v2() only. + */ +static bool fscrypt_valid_enc_modes_v1(u32 contents_mode, u32 filenames_mode) +{ + if (contents_mode == FSCRYPT_MODE_AES_256_XTS && + filenames_mode == FSCRYPT_MODE_AES_256_CTS) + return true; + + if (contents_mode == FSCRYPT_MODE_AES_128_CBC && + filenames_mode == FSCRYPT_MODE_AES_128_CTS) + return true; + + if (contents_mode == FSCRYPT_MODE_ADIANTUM && + filenames_mode == FSCRYPT_MODE_ADIANTUM) + return true; + + return false; +} + +static bool fscrypt_valid_enc_modes_v2(u32 contents_mode, u32 filenames_mode) +{ + if (contents_mode == FSCRYPT_MODE_AES_256_XTS && + filenames_mode == FSCRYPT_MODE_AES_256_HCTR2) + return true; + + if (contents_mode == FSCRYPT_MODE_SM4_XTS && + filenames_mode == FSCRYPT_MODE_SM4_CTS) + return true; + + return fscrypt_valid_enc_modes_v1(contents_mode, filenames_mode); +} + +static bool supported_direct_key_modes(const struct inode *inode, + u32 contents_mode, u32 filenames_mode) +{ + const struct fscrypt_mode *mode; + + if (contents_mode != filenames_mode) { + fscrypt_warn(inode, + "Direct key flag not allowed with different contents and filenames modes"); + return false; + } + mode = &fscrypt_modes[contents_mode]; + + if (mode->ivsize < offsetofend(union fscrypt_iv, nonce)) { + fscrypt_warn(inode, "Direct key flag not allowed with %s", + mode->friendly_name); + return false; + } + return true; +} + +static bool supported_iv_ino_lblk_policy(const struct fscrypt_policy_v2 *policy, + const struct inode *inode) +{ + const char *type = (policy->flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64) + ? "IV_INO_LBLK_64" : "IV_INO_LBLK_32"; + struct super_block *sb = inode->i_sb; + + /* + * IV_INO_LBLK_* exist only because of hardware limitations, and + * currently the only known use case for them involves AES-256-XTS. + * That's also all we test currently. For these reasons, for now only + * allow AES-256-XTS here. This can be relaxed later if a use case for + * IV_INO_LBLK_* with other encryption modes arises. + */ + if (policy->contents_encryption_mode != FSCRYPT_MODE_AES_256_XTS) { + fscrypt_warn(inode, + "Can't use %s policy with contents mode other than AES-256-XTS", + type); + return false; + } + + /* + * It's unsafe to include inode numbers in the IVs if the filesystem can + * potentially renumber inodes, e.g. via filesystem shrinking. + */ + if (!sb->s_cop->has_stable_inodes || + !sb->s_cop->has_stable_inodes(sb)) { + fscrypt_warn(inode, + "Can't use %s policy on filesystem '%s' because it doesn't have stable inode numbers", + type, sb->s_id); + return false; + } + + /* + * IV_INO_LBLK_64 and IV_INO_LBLK_32 both require that inode numbers fit + * in 32 bits. In principle, IV_INO_LBLK_32 could support longer inode + * numbers because it hashes the inode number; however, currently the + * inode number is gotten from inode::i_ino which is 'unsigned long'. + * So for now the implementation limit is 32 bits. + */ + if (!sb->s_cop->has_32bit_inodes) { + fscrypt_warn(inode, + "Can't use %s policy on filesystem '%s' because its inode numbers are too long", + type, sb->s_id); + return false; + } + + /* + * IV_INO_LBLK_64 and IV_INO_LBLK_32 both require that file data unit + * indices fit in 32 bits. + */ + if (fscrypt_max_file_dun_bits(sb, + fscrypt_policy_v2_du_bits(policy, inode)) > 32) { + fscrypt_warn(inode, + "Can't use %s policy on filesystem '%s' because its maximum file size is too large", + type, sb->s_id); + return false; + } + + /* + * IV_INO_LBLK_32 isn't compatible with inline encryption when + * s_blocksize != PAGE_SIZE. In that case the DUN can wrap around in + * the middle of a page, but sometimes fscrypt_mergeable_bio() is called + * only for the first block per page. Since IV_INO_LBLK_32 exists only + * to support inline encryption hardware that is limited to 32-bit DUNs, + * just disallow IV_INO_LBLK_32 with s_blocksize != PAGE_SIZE entirely. + */ + if ((policy->flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32) && + sb->s_blocksize != PAGE_SIZE) { + fscrypt_warn(inode, + "Can't use %s policy on filesystem '%s' with block size != PAGE_SIZE", + type, sb->s_id); + return false; + } + + return true; +} + +static bool fscrypt_supported_v1_policy(const struct fscrypt_policy_v1 *policy, + const struct inode *inode) +{ + if (!fscrypt_valid_enc_modes_v1(policy->contents_encryption_mode, + policy->filenames_encryption_mode)) { + fscrypt_warn(inode, + "Unsupported encryption modes (contents %d, filenames %d)", + policy->contents_encryption_mode, + policy->filenames_encryption_mode); + return false; + } + + if (policy->flags & ~(FSCRYPT_POLICY_FLAGS_PAD_MASK | + FSCRYPT_POLICY_FLAG_DIRECT_KEY)) { + fscrypt_warn(inode, "Unsupported encryption flags (0x%02x)", + policy->flags); + return false; + } + + if ((policy->flags & FSCRYPT_POLICY_FLAG_DIRECT_KEY) && + !supported_direct_key_modes(inode, policy->contents_encryption_mode, + policy->filenames_encryption_mode)) + return false; + + if (IS_CASEFOLDED(inode)) { + /* With v1, there's no way to derive dirhash keys. */ + fscrypt_warn(inode, + "v1 policies can't be used on casefolded directories"); + return false; + } + + return true; +} + +static bool fscrypt_supported_v2_policy(const struct fscrypt_policy_v2 *policy, + const struct inode *inode) +{ + int count = 0; + + if (!fscrypt_valid_enc_modes_v2(policy->contents_encryption_mode, + policy->filenames_encryption_mode)) { + fscrypt_warn(inode, + "Unsupported encryption modes (contents %d, filenames %d)", + policy->contents_encryption_mode, + policy->filenames_encryption_mode); + return false; + } + + if (policy->flags & ~(FSCRYPT_POLICY_FLAGS_PAD_MASK | + FSCRYPT_POLICY_FLAG_DIRECT_KEY | + FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64 | + FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32)) { + fscrypt_warn(inode, "Unsupported encryption flags (0x%02x)", + policy->flags); + return false; + } + + count += !!(policy->flags & FSCRYPT_POLICY_FLAG_DIRECT_KEY); + count += !!(policy->flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64); + count += !!(policy->flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32); + if (count > 1) { + fscrypt_warn(inode, "Mutually exclusive encryption flags (0x%02x)", + policy->flags); + return false; + } + + if (policy->log2_data_unit_size) { + if (!inode->i_sb->s_cop->supports_subblock_data_units) { + fscrypt_warn(inode, + "Filesystem does not support configuring crypto data unit size"); + return false; + } + if (policy->log2_data_unit_size > inode->i_blkbits || + policy->log2_data_unit_size < SECTOR_SHIFT /* 9 */) { + fscrypt_warn(inode, + "Unsupported log2_data_unit_size in encryption policy: %d", + policy->log2_data_unit_size); + return false; + } + if (policy->log2_data_unit_size != inode->i_blkbits && + (policy->flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32)) { + /* + * Not safe to enable yet, as we need to ensure that DUN + * wraparound can only occur on a FS block boundary. + */ + fscrypt_warn(inode, + "Sub-block data units not yet supported with IV_INO_LBLK_32"); + return false; + } + } + + if ((policy->flags & FSCRYPT_POLICY_FLAG_DIRECT_KEY) && + !supported_direct_key_modes(inode, policy->contents_encryption_mode, + policy->filenames_encryption_mode)) + return false; + + if ((policy->flags & (FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64 | + FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32)) && + !supported_iv_ino_lblk_policy(policy, inode)) + return false; + + if (memchr_inv(policy->__reserved, 0, sizeof(policy->__reserved))) { + fscrypt_warn(inode, "Reserved bits set in encryption policy"); + return false; + } + + return true; +} + +/** + * fscrypt_supported_policy() - check whether an encryption policy is supported + * @policy_u: the encryption policy + * @inode: the inode on which the policy will be used + * + * Given an encryption policy, check whether all its encryption modes and other + * settings are supported by this kernel on the given inode. (But we don't + * currently don't check for crypto API support here, so attempting to use an + * algorithm not configured into the crypto API will still fail later.) + * + * Return: %true if supported, else %false + */ +bool fscrypt_supported_policy(const union fscrypt_policy *policy_u, + const struct inode *inode) +{ + switch (policy_u->version) { + case FSCRYPT_POLICY_V1: + return fscrypt_supported_v1_policy(&policy_u->v1, inode); + case FSCRYPT_POLICY_V2: + return fscrypt_supported_v2_policy(&policy_u->v2, inode); + } + return false; +} + +/** + * fscrypt_new_context() - create a new fscrypt_context + * @ctx_u: output context + * @policy_u: input policy + * @nonce: nonce to use + * + * Create an fscrypt_context for an inode that is being assigned the given + * encryption policy. @nonce must be a new random nonce. + * + * Return: the size of the new context in bytes. + */ +static int fscrypt_new_context(union fscrypt_context *ctx_u, + const union fscrypt_policy *policy_u, + const u8 nonce[FSCRYPT_FILE_NONCE_SIZE]) +{ + memset(ctx_u, 0, sizeof(*ctx_u)); + + switch (policy_u->version) { + case FSCRYPT_POLICY_V1: { + const struct fscrypt_policy_v1 *policy = &policy_u->v1; + struct fscrypt_context_v1 *ctx = &ctx_u->v1; + + ctx->version = FSCRYPT_CONTEXT_V1; + ctx->contents_encryption_mode = + policy->contents_encryption_mode; + ctx->filenames_encryption_mode = + policy->filenames_encryption_mode; + ctx->flags = policy->flags; + memcpy(ctx->master_key_descriptor, + policy->master_key_descriptor, + sizeof(ctx->master_key_descriptor)); + memcpy(ctx->nonce, nonce, FSCRYPT_FILE_NONCE_SIZE); + return sizeof(*ctx); + } + case FSCRYPT_POLICY_V2: { + const struct fscrypt_policy_v2 *policy = &policy_u->v2; + struct fscrypt_context_v2 *ctx = &ctx_u->v2; + + ctx->version = FSCRYPT_CONTEXT_V2; + ctx->contents_encryption_mode = + policy->contents_encryption_mode; + ctx->filenames_encryption_mode = + policy->filenames_encryption_mode; + ctx->flags = policy->flags; + ctx->log2_data_unit_size = policy->log2_data_unit_size; + memcpy(ctx->master_key_identifier, + policy->master_key_identifier, + sizeof(ctx->master_key_identifier)); + memcpy(ctx->nonce, nonce, FSCRYPT_FILE_NONCE_SIZE); + return sizeof(*ctx); + } + } + BUG(); +} + +/** + * fscrypt_policy_from_context() - convert an fscrypt_context to + * an fscrypt_policy + * @policy_u: output policy + * @ctx_u: input context + * @ctx_size: size of input context in bytes + * + * Given an fscrypt_context, build the corresponding fscrypt_policy. + * + * Return: 0 on success, or -EINVAL if the fscrypt_context has an unrecognized + * version number or size. + * + * This does *not* validate the settings within the policy itself, e.g. the + * modes, flags, and reserved bits. Use fscrypt_supported_policy() for that. + */ +int fscrypt_policy_from_context(union fscrypt_policy *policy_u, + const union fscrypt_context *ctx_u, + int ctx_size) +{ + memset(policy_u, 0, sizeof(*policy_u)); + + if (!fscrypt_context_is_valid(ctx_u, ctx_size)) + return -EINVAL; + + switch (ctx_u->version) { + case FSCRYPT_CONTEXT_V1: { + const struct fscrypt_context_v1 *ctx = &ctx_u->v1; + struct fscrypt_policy_v1 *policy = &policy_u->v1; + + policy->version = FSCRYPT_POLICY_V1; + policy->contents_encryption_mode = + ctx->contents_encryption_mode; + policy->filenames_encryption_mode = + ctx->filenames_encryption_mode; + policy->flags = ctx->flags; + memcpy(policy->master_key_descriptor, + ctx->master_key_descriptor, + sizeof(policy->master_key_descriptor)); + return 0; + } + case FSCRYPT_CONTEXT_V2: { + const struct fscrypt_context_v2 *ctx = &ctx_u->v2; + struct fscrypt_policy_v2 *policy = &policy_u->v2; + + policy->version = FSCRYPT_POLICY_V2; + policy->contents_encryption_mode = + ctx->contents_encryption_mode; + policy->filenames_encryption_mode = + ctx->filenames_encryption_mode; + policy->flags = ctx->flags; + policy->log2_data_unit_size = ctx->log2_data_unit_size; + memcpy(policy->__reserved, ctx->__reserved, + sizeof(policy->__reserved)); + memcpy(policy->master_key_identifier, + ctx->master_key_identifier, + sizeof(policy->master_key_identifier)); + return 0; + } + } + /* unreachable */ + return -EINVAL; +} + +/* Retrieve an inode's encryption policy */ +static int fscrypt_get_policy(struct inode *inode, union fscrypt_policy *policy) +{ + const struct fscrypt_inode_info *ci; + union fscrypt_context ctx; + int ret; + + ci = fscrypt_get_inode_info(inode); + if (ci) { + /* key available, use the cached policy */ + *policy = ci->ci_policy; + return 0; + } + + if (!IS_ENCRYPTED(inode)) + return -ENODATA; + + ret = inode->i_sb->s_cop->get_context(inode, &ctx, sizeof(ctx)); + if (ret < 0) + return (ret == -ERANGE) ? -EINVAL : ret; + + return fscrypt_policy_from_context(policy, &ctx, ret); +} + +static int set_encryption_policy(struct inode *inode, + const union fscrypt_policy *policy) +{ + u8 nonce[FSCRYPT_FILE_NONCE_SIZE]; + union fscrypt_context ctx; + int ctxsize; + int err; + + if (!fscrypt_supported_policy(policy, inode)) + return -EINVAL; + + switch (policy->version) { + case FSCRYPT_POLICY_V1: + /* + * The original encryption policy version provided no way of + * verifying that the correct master key was supplied, which was + * insecure in scenarios where multiple users have access to the + * same encrypted files (even just read-only access). The new + * encryption policy version fixes this and also implies use of + * an improved key derivation function and allows non-root users + * to securely remove keys. So as long as compatibility with + * old kernels isn't required, it is recommended to use the new + * policy version for all new encrypted directories. + */ + pr_warn_once("%s (pid %d) is setting deprecated v1 encryption policy; recommend upgrading to v2.\n", + current->comm, current->pid); + break; + case FSCRYPT_POLICY_V2: + err = fscrypt_verify_key_added(inode->i_sb, + policy->v2.master_key_identifier); + if (err) + return err; + if (policy->v2.flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32) + pr_warn_once("%s (pid %d) is setting an IV_INO_LBLK_32 encryption policy. This should only be used if there are certain hardware limitations.\n", + current->comm, current->pid); + break; + default: + WARN_ON_ONCE(1); + return -EINVAL; + } + + get_random_bytes(nonce, FSCRYPT_FILE_NONCE_SIZE); + ctxsize = fscrypt_new_context(&ctx, policy, nonce); + + return inode->i_sb->s_cop->set_context(inode, &ctx, ctxsize, NULL); +} + +int fscrypt_ioctl_set_policy(struct file *filp, const void __user *arg) +{ + union fscrypt_policy policy; + union fscrypt_policy existing_policy; + struct inode *inode = file_inode(filp); + int size; + int ret; + + if (get_user(policy.version, (const u8 __user *)arg)) + return -EFAULT; + + size = fscrypt_policy_size(&policy); + if (size <= 0) + return -EINVAL; + + if (copy_from_user((u8 *)&policy + 1, (const u8 __user *)arg + 1, + size - 1)) + return -EFAULT; + + if (!inode_owner_or_capable(file_mnt_idmap(filp), inode)) + return -EACCES; + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + + inode_lock(inode); + + ret = fscrypt_get_policy(inode, &existing_policy); + if (ret == -ENODATA) { + if (!S_ISDIR(inode->i_mode)) + ret = -ENOTDIR; + else if (IS_DEADDIR(inode)) + ret = -ENOENT; + else if (!inode->i_sb->s_cop->empty_dir(inode)) + ret = -ENOTEMPTY; + else + ret = set_encryption_policy(inode, &policy); + } else if (ret == -EINVAL || + (ret == 0 && !fscrypt_policies_equal(&policy, + &existing_policy))) { + /* The file already uses a different encryption policy. */ + ret = -EEXIST; + } + + inode_unlock(inode); + + mnt_drop_write_file(filp); + return ret; +} +EXPORT_SYMBOL(fscrypt_ioctl_set_policy); + +/* Original ioctl version; can only get the original policy version */ +int fscrypt_ioctl_get_policy(struct file *filp, void __user *arg) +{ + union fscrypt_policy policy; + int err; + + err = fscrypt_get_policy(file_inode(filp), &policy); + if (err) + return err; + + if (policy.version != FSCRYPT_POLICY_V1) + return -EINVAL; + + if (copy_to_user(arg, &policy, sizeof(policy.v1))) + return -EFAULT; + return 0; +} +EXPORT_SYMBOL(fscrypt_ioctl_get_policy); + +/* Extended ioctl version; can get policies of any version */ +int fscrypt_ioctl_get_policy_ex(struct file *filp, void __user *uarg) +{ + struct fscrypt_get_policy_ex_arg arg; + union fscrypt_policy *policy = (union fscrypt_policy *)&arg.policy; + size_t policy_size; + int err; + + /* arg is policy_size, then policy */ + BUILD_BUG_ON(offsetof(typeof(arg), policy_size) != 0); + BUILD_BUG_ON(offsetofend(typeof(arg), policy_size) != + offsetof(typeof(arg), policy)); + BUILD_BUG_ON(sizeof(arg.policy) != sizeof(*policy)); + + err = fscrypt_get_policy(file_inode(filp), policy); + if (err) + return err; + policy_size = fscrypt_policy_size(policy); + + if (copy_from_user(&arg, uarg, sizeof(arg.policy_size))) + return -EFAULT; + + if (policy_size > arg.policy_size) + return -EOVERFLOW; + arg.policy_size = policy_size; + + if (copy_to_user(uarg, &arg, sizeof(arg.policy_size) + policy_size)) + return -EFAULT; + return 0; +} +EXPORT_SYMBOL_GPL(fscrypt_ioctl_get_policy_ex); + +/* FS_IOC_GET_ENCRYPTION_NONCE: retrieve file's encryption nonce for testing */ +int fscrypt_ioctl_get_nonce(struct file *filp, void __user *arg) +{ + struct inode *inode = file_inode(filp); + union fscrypt_context ctx; + int ret; + + ret = inode->i_sb->s_cop->get_context(inode, &ctx, sizeof(ctx)); + if (ret < 0) + return ret; + if (!fscrypt_context_is_valid(&ctx, ret)) + return -EINVAL; + if (copy_to_user(arg, fscrypt_context_nonce(&ctx), + FSCRYPT_FILE_NONCE_SIZE)) + return -EFAULT; + return 0; +} +EXPORT_SYMBOL_GPL(fscrypt_ioctl_get_nonce); + +/** + * fscrypt_has_permitted_context() - is a file's encryption policy permitted + * within its directory? + * + * @parent: inode for parent directory + * @child: inode for file being looked up, opened, or linked into @parent + * + * Filesystems must call this before permitting access to an inode in a + * situation where the parent directory is encrypted (either before allowing + * ->lookup() to succeed, or for a regular file before allowing it to be opened) + * and before any operation that involves linking an inode into an encrypted + * directory, including link, rename, and cross rename. It enforces the + * constraint that within a given encrypted directory tree, all files use the + * same encryption policy. The pre-access check is needed to detect potentially + * malicious offline violations of this constraint, while the link and rename + * checks are needed to prevent online violations of this constraint. + * + * Return: 1 if permitted, 0 if forbidden. + */ +int fscrypt_has_permitted_context(struct inode *parent, struct inode *child) +{ + union fscrypt_policy parent_policy, child_policy; + int err, err1, err2; + + /* No restrictions on file types which are never encrypted */ + if (!S_ISREG(child->i_mode) && !S_ISDIR(child->i_mode) && + !S_ISLNK(child->i_mode)) + return 1; + + /* No restrictions if the parent directory is unencrypted */ + if (!IS_ENCRYPTED(parent)) + return 1; + + /* Encrypted directories must not contain unencrypted files */ + if (!IS_ENCRYPTED(child)) + return 0; + + /* + * Both parent and child are encrypted, so verify they use the same + * encryption policy. Compare the cached policies if the keys are + * available, otherwise retrieve and compare the fscrypt_contexts. + * + * Note that the fscrypt_context retrieval will be required frequently + * when accessing an encrypted directory tree without the key. + * Performance-wise this is not a big deal because we already don't + * really optimize for file access without the key (to the extent that + * such access is even possible), given that any attempted access + * already causes a fscrypt_context retrieval and keyring search. + * + * In any case, if an unexpected error occurs, fall back to "forbidden". + */ + + err = fscrypt_get_encryption_info(parent, true); + if (err) + return 0; + err = fscrypt_get_encryption_info(child, true); + if (err) + return 0; + + err1 = fscrypt_get_policy(parent, &parent_policy); + err2 = fscrypt_get_policy(child, &child_policy); + + /* + * Allow the case where the parent and child both have an unrecognized + * encryption policy, so that files with an unrecognized encryption + * policy can be deleted. + */ + if (err1 == -EINVAL && err2 == -EINVAL) + return 1; + + if (err1 || err2) + return 0; + + return fscrypt_policies_equal(&parent_policy, &child_policy); +} +EXPORT_SYMBOL(fscrypt_has_permitted_context); + +/* + * Return the encryption policy that new files in the directory will inherit, or + * NULL if none, or an ERR_PTR() on error. If the directory is encrypted, also + * ensure that its key is set up, so that the new filename can be encrypted. + */ +const union fscrypt_policy *fscrypt_policy_to_inherit(struct inode *dir) +{ + int err; + + if (IS_ENCRYPTED(dir)) { + err = fscrypt_require_key(dir); + if (err) + return ERR_PTR(err); + return &fscrypt_get_inode_info_raw(dir)->ci_policy; + } + + return fscrypt_get_dummy_policy(dir->i_sb); +} + +/** + * fscrypt_context_for_new_inode() - create an encryption context for a new inode + * @ctx: where context should be written + * @inode: inode from which to fetch policy and nonce + * + * Given an in-core "prepared" (via fscrypt_prepare_new_inode) inode, + * generate a new context and write it to ctx. ctx _must_ be at least + * FSCRYPT_SET_CONTEXT_MAX_SIZE bytes. + * + * Return: size of the resulting context or a negative error code. + */ +int fscrypt_context_for_new_inode(void *ctx, struct inode *inode) +{ + struct fscrypt_inode_info *ci = fscrypt_get_inode_info_raw(inode); + + BUILD_BUG_ON(sizeof(union fscrypt_context) != + FSCRYPT_SET_CONTEXT_MAX_SIZE); + + /* fscrypt_prepare_new_inode() should have set up the key already. */ + if (WARN_ON_ONCE(!ci)) + return -ENOKEY; + + return fscrypt_new_context(ctx, &ci->ci_policy, ci->ci_nonce); +} +EXPORT_SYMBOL_GPL(fscrypt_context_for_new_inode); + +/** + * fscrypt_set_context() - Set the fscrypt context of a new inode + * @inode: a new inode + * @fs_data: private data given by FS and passed to ->set_context() + * + * This should be called after fscrypt_prepare_new_inode(), generally during a + * filesystem transaction. Everything here must be %GFP_NOFS-safe. + * + * Return: 0 on success, -errno on failure + */ +int fscrypt_set_context(struct inode *inode, void *fs_data) +{ + struct fscrypt_inode_info *ci; + union fscrypt_context ctx; + int ctxsize; + + ctxsize = fscrypt_context_for_new_inode(&ctx, inode); + if (ctxsize < 0) + return ctxsize; + + /* + * This may be the first time the inode number is available, so do any + * delayed key setup that requires the inode number. + */ + ci = fscrypt_get_inode_info_raw(inode); + if (ci->ci_policy.version == FSCRYPT_POLICY_V2 && + (ci->ci_policy.v2.flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32)) + fscrypt_hash_inode_number(ci, ci->ci_master_key); + + return inode->i_sb->s_cop->set_context(inode, &ctx, ctxsize, fs_data); +} +EXPORT_SYMBOL_GPL(fscrypt_set_context); + +/** + * fscrypt_parse_test_dummy_encryption() - parse the test_dummy_encryption mount option + * @param: the mount option + * @dummy_policy: (input/output) the place to write the dummy policy that will + * result from parsing the option. Zero-initialize this. If a policy is + * already set here (due to test_dummy_encryption being given multiple + * times), then this function will verify that the policies are the same. + * + * Return: 0 on success; -EINVAL if the argument is invalid; -EEXIST if the + * argument conflicts with one already specified; or -ENOMEM. + */ +int fscrypt_parse_test_dummy_encryption(const struct fs_parameter *param, + struct fscrypt_dummy_policy *dummy_policy) +{ + const char *arg = "v2"; + union fscrypt_policy *policy; + int err; + + if (param->type == fs_value_is_string && *param->string) + arg = param->string; + + policy = kzalloc_obj(*policy); + if (!policy) + return -ENOMEM; + + if (!strcmp(arg, "v1")) { + policy->version = FSCRYPT_POLICY_V1; + policy->v1.contents_encryption_mode = FSCRYPT_MODE_AES_256_XTS; + policy->v1.filenames_encryption_mode = FSCRYPT_MODE_AES_256_CTS; + memset(policy->v1.master_key_descriptor, 0x42, + FSCRYPT_KEY_DESCRIPTOR_SIZE); + } else if (!strcmp(arg, "v2")) { + policy->version = FSCRYPT_POLICY_V2; + policy->v2.contents_encryption_mode = FSCRYPT_MODE_AES_256_XTS; + policy->v2.filenames_encryption_mode = FSCRYPT_MODE_AES_256_CTS; + fscrypt_get_test_dummy_key_identifier( + policy->v2.master_key_identifier); + } else { + err = -EINVAL; + goto out; + } + + if (dummy_policy->policy) { + if (fscrypt_policies_equal(policy, dummy_policy->policy)) + err = 0; + else + err = -EEXIST; + goto out; + } + dummy_policy->policy = policy; + policy = NULL; + err = 0; +out: + kfree(policy); + return err; +} +EXPORT_SYMBOL_GPL(fscrypt_parse_test_dummy_encryption); + +/** + * fscrypt_dummy_policies_equal() - check whether two dummy policies are equal + * @p1: the first test dummy policy (may be unset) + * @p2: the second test dummy policy (may be unset) + * + * Return: %true if the dummy policies are both set and equal, or both unset. + */ +bool fscrypt_dummy_policies_equal(const struct fscrypt_dummy_policy *p1, + const struct fscrypt_dummy_policy *p2) +{ + if (!p1->policy && !p2->policy) + return true; + if (!p1->policy || !p2->policy) + return false; + return fscrypt_policies_equal(p1->policy, p2->policy); +} +EXPORT_SYMBOL_GPL(fscrypt_dummy_policies_equal); + +/** + * fscrypt_show_test_dummy_encryption() - show '-o test_dummy_encryption' + * @seq: the seq_file to print the option to + * @sep: the separator character to use + * @sb: the filesystem whose options are being shown + * + * Show the test_dummy_encryption mount option, if it was specified. + * This is mainly used for /proc/mounts. + */ +void fscrypt_show_test_dummy_encryption(struct seq_file *seq, char sep, + struct super_block *sb) +{ + const union fscrypt_policy *policy = fscrypt_get_dummy_policy(sb); + int vers; + + if (!policy) + return; + + vers = policy->version; + if (vers == FSCRYPT_POLICY_V1) /* Handle numbering quirk */ + vers = 1; + + seq_printf(seq, "%ctest_dummy_encryption=v%d", sep, vers); +} +EXPORT_SYMBOL_GPL(fscrypt_show_test_dummy_encryption); diff --git a/fs/d_path.c b/fs/d_path.c new file mode 100644 index 000000000..c25309006 --- /dev/null +++ b/fs/d_path.c @@ -0,0 +1,450 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include +#include +#include +#include +#include +#include +#include "mount.h" +#include "internal.h" + +struct prepend_buffer { + char *buf; + int len; +}; +#define DECLARE_BUFFER(__name, __buf, __len) \ + struct prepend_buffer __name = {.buf = __buf + __len, .len = __len} + +static char *extract_string(struct prepend_buffer *p) +{ + if (likely(p->len >= 0)) + return p->buf; + return ERR_PTR(-ENAMETOOLONG); +} + +static bool prepend_char(struct prepend_buffer *p, unsigned char c) +{ + if (likely(p->len > 0)) { + p->len--; + *--p->buf = c; + return true; + } + p->len = -1; + return false; +} + +/* + * The source of the prepend data can be an optimistic load + * of a dentry name and length. And because we don't hold any + * locks, the length and the pointer to the name may not be + * in sync if a concurrent rename happens, and the kernel + * copy might fault as a result. + * + * The end result will correct itself when we check the + * rename sequence count, but we need to be able to handle + * the fault gracefully. + */ +static bool prepend_copy(void *dst, const void *src, int len) +{ + if (unlikely(copy_from_kernel_nofault(dst, src, len))) { + memset(dst, 'x', len); + return false; + } + return true; +} + +static bool prepend(struct prepend_buffer *p, const char *str, int namelen) +{ + // Already overflowed? + if (p->len < 0) + return false; + + // Will overflow? + if (p->len < namelen) { + // Fill as much as possible from the end of the name + str += namelen - p->len; + p->buf -= p->len; + prepend_copy(p->buf, str, p->len); + p->len = -1; + return false; + } + + // Fits fully + p->len -= namelen; + p->buf -= namelen; + return prepend_copy(p->buf, str, namelen); +} + +/** + * prepend_name - prepend a pathname in front of current buffer pointer + * @p: prepend buffer which contains buffer pointer and allocated length + * @name: name string and length qstr structure + * + * With RCU path tracing, it may race with d_move(). Use READ_ONCE() to + * make sure that either the old or the new name pointer and length are + * fetched. However, there may be mismatch between length and pointer. + * But since the length cannot be trusted, we need to copy the name very + * carefully when doing the prepend_copy(). It also prepends "/" at + * the beginning of the name. The sequence number check at the caller will + * retry it again when a d_move() does happen. So any garbage in the buffer + * due to mismatched pointer and length will be discarded. + * + * Load acquire is needed to make sure that we see the new name data even + * if we might get the length wrong. + */ +static bool prepend_name(struct prepend_buffer *p, const struct qstr *name) +{ + const char *dname = smp_load_acquire(&name->name); /* ^^^ */ + u32 dlen = READ_ONCE(name->len); + + return prepend(p, dname, dlen) && prepend_char(p, '/'); +} + +static int __prepend_path(const struct dentry *dentry, const struct mount *mnt, + const struct path *root, struct prepend_buffer *p) +{ + while (dentry != root->dentry || &mnt->mnt != root->mnt) { + const struct dentry *parent = READ_ONCE(dentry->d_parent); + + if (dentry == mnt->mnt.mnt_root) { + struct mount *m = READ_ONCE(mnt->mnt_parent); + struct mnt_namespace *mnt_ns; + + if (likely(mnt != m)) { + dentry = READ_ONCE(mnt->mnt_mountpoint); + mnt = m; + continue; + } + /* Global root */ + mnt_ns = READ_ONCE(mnt->mnt_ns); + /* open-coded is_mounted() to use local mnt_ns */ + if (!IS_ERR_OR_NULL(mnt_ns) && !is_anon_ns(mnt_ns)) + return 1; // absolute root + else + return 2; // detached or not attached yet + } + + if (unlikely(dentry == parent)) + /* Escaped? */ + return 3; + + prefetch(parent); + if (!prepend_name(p, &dentry->d_name)) + break; + dentry = parent; + } + return 0; +} + +/** + * prepend_path - Prepend path string to a buffer + * @path: the dentry/vfsmount to report + * @root: root vfsmnt/dentry + * @p: prepend buffer which contains buffer pointer and allocated length + * + * The function will first try to write out the pathname without taking any + * lock other than the RCU read lock to make sure that dentries won't go away. + * It only checks the sequence number of the global rename_lock as any change + * in the dentry's d_seq will be preceded by changes in the rename_lock + * sequence number. If the sequence number had been changed, it will restart + * the whole pathname back-tracing sequence again by taking the rename_lock. + * In this case, there is no need to take the RCU read lock as the recursive + * parent pointer references will keep the dentry chain alive as long as no + * rename operation is performed. + */ +static int prepend_path(const struct path *path, + const struct path *root, + struct prepend_buffer *p) +{ + unsigned seq, m_seq = 0; + struct prepend_buffer b; + int error; + + rcu_read_lock(); +restart_mnt: + read_seqbegin_or_lock(&mount_lock, &m_seq); + seq = 0; + rcu_read_lock(); +restart: + b = *p; + read_seqbegin_or_lock(&rename_lock, &seq); + error = __prepend_path(path->dentry, real_mount(path->mnt), root, &b); + if (!(seq & 1)) + rcu_read_unlock(); + if (need_seqretry(&rename_lock, seq)) { + seq = 1; + goto restart; + } + done_seqretry(&rename_lock, seq); + + if (!(m_seq & 1)) + rcu_read_unlock(); + if (need_seqretry(&mount_lock, m_seq)) { + m_seq = 1; + goto restart_mnt; + } + done_seqretry(&mount_lock, m_seq); + + if (unlikely(error == 3)) + b = *p; + + if (b.len == p->len) + prepend_char(&b, '/'); + + *p = b; + return error; +} + +/** + * __d_path - return the path of a dentry + * @path: the dentry/vfsmount to report + * @root: root vfsmnt/dentry + * @buf: buffer to return value in + * @buflen: buffer length + * + * Convert a dentry into an ASCII path name. + * + * Returns a pointer into the buffer or an error code if the + * path was too long. + * + * "buflen" should be positive. + * + * If the path is not reachable from the supplied root, return %NULL. + */ +char *__d_path(const struct path *path, + const struct path *root, + char *buf, int buflen) +{ + DECLARE_BUFFER(b, buf, buflen); + + prepend_char(&b, 0); + if (unlikely(prepend_path(path, root, &b) > 0)) + return NULL; + return extract_string(&b); +} + +char *d_absolute_path(const struct path *path, + char *buf, int buflen) +{ + struct path root = {}; + DECLARE_BUFFER(b, buf, buflen); + + prepend_char(&b, 0); + if (unlikely(prepend_path(path, &root, &b) > 1)) + return ERR_PTR(-EINVAL); + return extract_string(&b); +} + +static void get_fs_root_rcu(struct fs_struct *fs, struct path *root) +{ + unsigned seq; + + do { + seq = read_seqbegin(&fs->seq); + *root = fs->root; + } while (read_seqretry(&fs->seq, seq)); +} + +/** + * d_path - return the path of a dentry + * @path: path to report + * @buf: buffer to return value in + * @buflen: buffer length + * + * Convert a dentry into an ASCII path name. If the entry has been deleted + * the string " (deleted)" is appended. Note that this is ambiguous. + * + * Returns a pointer into the buffer or an error code if the path was + * too long. Note: Callers should use the returned pointer, not the passed + * in buffer, to use the name! The implementation often starts at an offset + * into the buffer, and may leave 0 bytes at the start. + * + * "buflen" should be positive. + */ +char *d_path(const struct path *path, char *buf, int buflen) +{ + DECLARE_BUFFER(b, buf, buflen); + struct path root; + + /* + * We have various synthetic filesystems that never get mounted. On + * these filesystems dentries are never used for lookup purposes, and + * thus don't need to be hashed. They also don't need a name until a + * user wants to identify the object in /proc/pid/fd/. The little hack + * below allows us to generate a name for these objects on demand: + * + * Some pseudo inodes are mountable. When they are mounted + * path->dentry == path->mnt->mnt_root. In that case don't call d_dname + * and instead have d_path return the mounted path. + */ + if (path->dentry->d_op && path->dentry->d_op->d_dname && + (!IS_ROOT(path->dentry) || path->dentry != path->mnt->mnt_root || + failfs_mnt(path->mnt))) + return path->dentry->d_op->d_dname(path->dentry, buf, buflen); + + rcu_read_lock(); + get_fs_root_rcu(current->fs, &root); + if (unlikely(d_unlinked(path->dentry))) + prepend(&b, " (deleted)", 11); + else + prepend_char(&b, 0); + prepend_path(path, &root, &b); + rcu_read_unlock(); + + return extract_string(&b); +} +EXPORT_SYMBOL(d_path); + +/* + * Helper function for dentry_operations.d_dname() members + */ +char *dynamic_dname(char *buffer, int buflen, const char *fmt, ...) +{ + va_list args; + char *start; + int sz; + + va_start(args, fmt); + sz = vsnprintf(buffer, buflen, fmt, args) + 1; + va_end(args); + + if (sz > NAME_MAX || sz > buflen) + return ERR_PTR(-ENAMETOOLONG); + + /* Move the formatted d_name to the end of the buffer. */ + start = buffer + (buflen - sz); + return memmove(start, buffer, sz); +} + +char *simple_dname(struct dentry *dentry, char *buffer, int buflen) +{ + DECLARE_BUFFER(b, buffer, buflen); + /* these dentries are never renamed, so d_lock is not needed */ + prepend(&b, " (deleted)", 11); + prepend(&b, dentry->d_name.name, dentry->d_name.len); + prepend_char(&b, '/'); + return extract_string(&b); +} + +/* + * Write full pathname from the root of the filesystem into the buffer. + */ +static char *__dentry_path(const struct dentry *d, struct prepend_buffer *p) +{ + const struct dentry *dentry; + struct prepend_buffer b; + int seq = 0; + + rcu_read_lock(); +restart: + dentry = d; + b = *p; + read_seqbegin_or_lock(&rename_lock, &seq); + while (!IS_ROOT(dentry)) { + const struct dentry *parent = dentry->d_parent; + + prefetch(parent); + if (!prepend_name(&b, &dentry->d_name)) + break; + dentry = parent; + } + if (!(seq & 1)) + rcu_read_unlock(); + if (need_seqretry(&rename_lock, seq)) { + seq = 1; + goto restart; + } + done_seqretry(&rename_lock, seq); + if (b.len == p->len) + prepend_char(&b, '/'); + return extract_string(&b); +} + +char *dentry_path_raw(const struct dentry *dentry, char *buf, int buflen) +{ + DECLARE_BUFFER(b, buf, buflen); + + prepend_char(&b, 0); + return __dentry_path(dentry, &b); +} +EXPORT_SYMBOL(dentry_path_raw); + +char *dentry_path(const struct dentry *dentry, char *buf, int buflen) +{ + DECLARE_BUFFER(b, buf, buflen); + + if (unlikely(d_unlinked(dentry))) + prepend(&b, "//deleted", 10); + else + prepend_char(&b, 0); + return __dentry_path(dentry, &b); +} + +static void get_fs_root_and_pwd_rcu(struct fs_struct *fs, struct path *root, + struct path *pwd) +{ + unsigned seq; + + do { + seq = read_seqbegin(&fs->seq); + *root = fs->root; + *pwd = fs->pwd; + } while (read_seqretry(&fs->seq, seq)); +} + +/* + * NOTE! The user-level library version returns a + * character pointer. The kernel system call just + * returns the length of the buffer filled (which + * includes the ending '\0' character), or a negative + * error value. So libc would do something like + * + * char *getcwd(char * buf, size_t size) + * { + * int retval; + * + * retval = sys_getcwd(buf, size); + * if (retval >= 0) + * return buf; + * errno = -retval; + * return NULL; + * } + */ +SYSCALL_DEFINE2(getcwd, char __user *, buf, unsigned long, size) +{ + int error; + struct path pwd, root; + char *page = __getname(); + + if (!page) + return -ENOMEM; + + rcu_read_lock(); + get_fs_root_and_pwd_rcu(current->fs, &root, &pwd); + + if (unlikely(d_unlinked(pwd.dentry))) { + rcu_read_unlock(); + error = -ENOENT; + } else { + unsigned len; + DECLARE_BUFFER(b, page, PATH_MAX); + + prepend_char(&b, 0); + if (unlikely(prepend_path(&pwd, &root, &b) > 0)) + prepend(&b, "(unreachable)", 13); + rcu_read_unlock(); + + len = PATH_MAX - b.len; + if (unlikely(len > PATH_MAX)) + error = -ENAMETOOLONG; + else if (unlikely(len > size)) + error = -ERANGE; + else if (copy_to_user(buf, b.buf, len)) + error = -EFAULT; + else + error = len; + } + __putname(page); + return error; +} diff --git a/fs/dax.c b/fs/dax.c new file mode 100644 index 000000000..6ba50142e --- /dev/null +++ b/fs/dax.c @@ -0,0 +1,2278 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * fs/dax.c - Direct Access filesystem code + * Copyright (c) 2013-2014 Intel Corporation + * Author: Matthew Wilcox + * Author: Ross Zwisler + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CREATE_TRACE_POINTS +#include + +/* We choose 4096 entries - same as per-zone page wait tables */ +#define DAX_WAIT_TABLE_BITS 12 +#define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS) + +/* The 'colour' (ie low bits) within a PMD of a page offset. */ +#define PG_PMD_COLOUR ((PMD_SIZE >> PAGE_SHIFT) - 1) +#define PG_PMD_NR (PMD_SIZE >> PAGE_SHIFT) + +static wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES]; + +static int __init init_dax_wait_table(void) +{ + int i; + + for (i = 0; i < DAX_WAIT_TABLE_ENTRIES; i++) + init_waitqueue_head(wait_table + i); + return 0; +} +fs_initcall(init_dax_wait_table); + +/* + * DAX pagecache entries use XArray value entries so they can't be mistaken + * for pages. We use one bit for locking, one bit for the entry size (PMD) + * and two more to tell us if the entry is a zero page or an empty entry that + * is just used for locking. In total four special bits. + * + * If the PMD bit isn't set the entry has size PAGE_SIZE, and if the ZERO_PAGE + * and EMPTY bits aren't set the entry is a normal DAX entry with a filesystem + * block allocation. + */ +#define DAX_SHIFT (4) +#define DAX_LOCKED (1UL << 0) +#define DAX_PMD (1UL << 1) +#define DAX_ZERO_PAGE (1UL << 2) +#define DAX_EMPTY (1UL << 3) + +static unsigned long dax_to_pfn(void *entry) +{ + return xa_to_value(entry) >> DAX_SHIFT; +} + +static struct folio *dax_to_folio(void *entry) +{ + return page_folio(pfn_to_page(dax_to_pfn(entry))); +} + +static void *dax_make_entry(unsigned long pfn, unsigned long flags) +{ + return xa_mk_value(flags | (pfn << DAX_SHIFT)); +} + +static bool dax_is_locked(void *entry) +{ + return xa_to_value(entry) & DAX_LOCKED; +} + +static unsigned int dax_entry_order(void *entry) +{ + if (xa_to_value(entry) & DAX_PMD) + return PMD_ORDER; + return 0; +} + +static unsigned long dax_is_pmd_entry(void *entry) +{ + return xa_to_value(entry) & DAX_PMD; +} + +static bool dax_is_pte_entry(void *entry) +{ + return !(xa_to_value(entry) & DAX_PMD); +} + +static int dax_is_zero_entry(void *entry) +{ + return xa_to_value(entry) & DAX_ZERO_PAGE; +} + +static int dax_is_empty_entry(void *entry) +{ + return xa_to_value(entry) & DAX_EMPTY; +} + +/* + * true if the entry that was found is of a smaller order than the entry + * we were looking for + */ +static bool dax_is_conflict(void *entry) +{ + return entry == XA_RETRY_ENTRY; +} + +/* + * DAX page cache entry locking + */ +struct exceptional_entry_key { + struct xarray *xa; + pgoff_t entry_start; +}; + +struct wait_exceptional_entry_queue { + wait_queue_entry_t wait; + struct exceptional_entry_key key; +}; + +/** + * enum dax_wake_mode: waitqueue wakeup behaviour + * @WAKE_ALL: wake all waiters in the waitqueue + * @WAKE_NEXT: wake only the first waiter in the waitqueue + */ +enum dax_wake_mode { + WAKE_ALL, + WAKE_NEXT, +}; + +static wait_queue_head_t *dax_entry_waitqueue(struct xa_state *xas, + void *entry, struct exceptional_entry_key *key) +{ + unsigned long hash; + unsigned long index = xas->xa_index; + + /* + * If 'entry' is a PMD, align the 'index' that we use for the wait + * queue to the start of that PMD. This ensures that all offsets in + * the range covered by the PMD map to the same bit lock. + */ + if (dax_is_pmd_entry(entry)) + index &= ~PG_PMD_COLOUR; + key->xa = xas->xa; + key->entry_start = index; + + hash = hash_long((unsigned long)xas->xa ^ index, DAX_WAIT_TABLE_BITS); + return wait_table + hash; +} + +static int wake_exceptional_entry_func(wait_queue_entry_t *wait, + unsigned int mode, int sync, void *keyp) +{ + struct exceptional_entry_key *key = keyp; + struct wait_exceptional_entry_queue *ewait = + container_of(wait, struct wait_exceptional_entry_queue, wait); + + if (key->xa != ewait->key.xa || + key->entry_start != ewait->key.entry_start) + return 0; + return autoremove_wake_function(wait, mode, sync, NULL); +} + +/* + * @entry may no longer be the entry at the index in the mapping. + * The important information it's conveying is whether the entry at + * this index used to be a PMD entry. + */ +static void dax_wake_entry(struct xa_state *xas, void *entry, + enum dax_wake_mode mode) +{ + struct exceptional_entry_key key; + wait_queue_head_t *wq; + + wq = dax_entry_waitqueue(xas, entry, &key); + + /* + * Checking for locked entry and prepare_to_wait_exclusive() happens + * under the i_pages lock, ditto for entry handling in our callers. + * So at this point all tasks that could have seen our entry locked + * must be in the waitqueue and the following check will see them. + */ + if (waitqueue_active(wq)) + __wake_up(wq, TASK_NORMAL, mode == WAKE_ALL ? 0 : 1, &key); +} + +/* + * Look up entry in page cache, wait for it to become unlocked if it + * is a DAX entry and return it. The caller must subsequently call + * put_unlocked_entry() if it did not lock the entry or dax_unlock_entry() + * if it did. The entry returned may have a larger order than @order. + * If @order is larger than the order of the entry found in i_pages, this + * function returns a dax_is_conflict entry. + * + * Must be called with the i_pages lock held. + */ +static void *get_next_unlocked_entry(struct xa_state *xas, unsigned int order) +{ + void *entry; + struct wait_exceptional_entry_queue ewait; + wait_queue_head_t *wq; + + init_wait(&ewait.wait); + ewait.wait.func = wake_exceptional_entry_func; + + for (;;) { + entry = xas_find_conflict(xas); + if (!entry || WARN_ON_ONCE(!xa_is_value(entry))) + return entry; + if (dax_entry_order(entry) < order) + return XA_RETRY_ENTRY; + if (!dax_is_locked(entry)) + return entry; + + wq = dax_entry_waitqueue(xas, entry, &ewait.key); + prepare_to_wait_exclusive(wq, &ewait.wait, + TASK_UNINTERRUPTIBLE); + xas_unlock_irq(xas); + xas_reset(xas); + schedule(); + finish_wait(wq, &ewait.wait); + xas_lock_irq(xas); + } +} + +/* + * Wait for the given entry to become unlocked. Caller must hold the i_pages + * lock and call either put_unlocked_entry() if it did not lock the entry or + * dax_unlock_entry() if it did. Returns an unlocked entry if still present. + */ +static void *wait_entry_unlocked_exclusive(struct xa_state *xas, void *entry) +{ + struct wait_exceptional_entry_queue ewait; + wait_queue_head_t *wq; + + init_wait(&ewait.wait); + ewait.wait.func = wake_exceptional_entry_func; + + while (unlikely(dax_is_locked(entry))) { + wq = dax_entry_waitqueue(xas, entry, &ewait.key); + prepare_to_wait_exclusive(wq, &ewait.wait, + TASK_UNINTERRUPTIBLE); + xas_reset(xas); + xas_unlock_irq(xas); + schedule(); + finish_wait(wq, &ewait.wait); + xas_lock_irq(xas); + entry = xas_load(xas); + } + + if (xa_is_internal(entry)) + return NULL; + + return entry; +} + +/* + * The only thing keeping the address space around is the i_pages lock + * (it's cycled in clear_inode() after removing the entries from i_pages) + * After we call xas_unlock_irq(), we cannot touch xas->xa. + */ +static void wait_entry_unlocked(struct xa_state *xas, void *entry) +{ + struct wait_exceptional_entry_queue ewait; + wait_queue_head_t *wq; + + init_wait(&ewait.wait); + ewait.wait.func = wake_exceptional_entry_func; + + wq = dax_entry_waitqueue(xas, entry, &ewait.key); + /* + * Unlike get_next_unlocked_entry() there is no guarantee that this + * path ever successfully retrieves an unlocked entry before an + * inode dies. Perform a non-exclusive wait in case this path + * never successfully performs its own wake up. + */ + prepare_to_wait(wq, &ewait.wait, TASK_UNINTERRUPTIBLE); + xas_unlock_irq(xas); + schedule(); + finish_wait(wq, &ewait.wait); +} + +static void put_unlocked_entry(struct xa_state *xas, void *entry, + enum dax_wake_mode mode) +{ + if (entry && !dax_is_conflict(entry)) + dax_wake_entry(xas, entry, mode); +} + +/* + * We used the xa_state to get the entry, but then we locked the entry and + * dropped the xa_lock, so we know the xa_state is stale and must be reset + * before use. + */ +static void dax_unlock_entry(struct xa_state *xas, void *entry) +{ + void *old; + + BUG_ON(dax_is_locked(entry)); + xas_reset(xas); + xas_lock_irq(xas); + old = xas_store(xas, entry); + xas_unlock_irq(xas); + BUG_ON(!dax_is_locked(old)); + dax_wake_entry(xas, entry, WAKE_NEXT); +} + +/* + * Return: The entry stored at this location before it was locked. + */ +static void *dax_lock_entry(struct xa_state *xas, void *entry) +{ + unsigned long v = xa_to_value(entry); + return xas_store(xas, xa_mk_value(v | DAX_LOCKED)); +} + +static unsigned long dax_entry_size(void *entry) +{ + if (dax_is_zero_entry(entry)) + return 0; + else if (dax_is_empty_entry(entry)) + return 0; + else if (dax_is_pmd_entry(entry)) + return PMD_SIZE; + else + return PAGE_SIZE; +} + +/* + * A DAX folio is considered shared if it has no mapping set and ->share (which + * shares the ->index field) is non-zero. Note this may return false even if the + * page is shared between multiple files but has not yet actually been mapped + * into multiple address spaces. + */ +static inline bool dax_folio_is_shared(struct folio *folio) +{ + return !folio->mapping && folio->share; +} + +/* + * When it is called by dax_insert_entry(), the shared flag will indicate + * whether this entry is shared by multiple files. If the page has not + * previously been associated with any mappings the ->mapping and ->index + * fields will be set. If it has already been associated with a mapping + * the mapping will be cleared and the share count set. It's then up to + * reverse map users like memory_failure() to call back into the filesystem to + * recover ->mapping and ->index information. For example by implementing + * dax_holder_operations. + */ +static void dax_folio_make_shared(struct folio *folio) +{ + /* + * folio is not currently shared so mark it as shared by clearing + * folio->mapping. + */ + folio->mapping = NULL; + + /* + * folio has previously been mapped into one address space so set the + * share count. + */ + folio->share = 1; +} + +/** + * dax_folio_reset_order - Reset a compound DAX folio to order-0 pages + * @folio: The folio to reset + * + * Splits a compound folio back into individual order-0 pages, + * clearing compound state and restoring pgmap pointers. + * + * Returns: the original folio order (0 if already order-0) + */ +int dax_folio_reset_order(struct folio *folio) +{ + struct dev_pagemap *pgmap = page_pgmap(&folio->page); + int order = folio_order(folio); + + /* + * Clear the mapping and the index/share union word. folio->share + * and folio->index occupy the same union in struct folio. For + * non-shared folios (mapping != NULL), the union holds folio->index + * (file page offset); for shared folios (mapping == NULL), it holds + * folio->share (reference count). Either way, we are releasing the + * folio and both fields should be zeroed. + */ + folio->mapping = NULL; + folio->share = 0; + + if (!order) { + /* + * Restore pgmap explicitly even for order-0 folios. For the + * dax_folio_put() caller this is a no-op (same value), but + * fsdev_clear_folio_state() may call this on folios that + * were previously compound and need pgmap re-established. + */ + folio->pgmap = pgmap; + return 0; + } + + folio_reset_order(folio); + + for (int i = 0; i < (1UL << order); i++) { + struct page *page = folio_page(folio, i); + struct folio *f = (struct folio *)page; + + ClearPageHead(page); + clear_compound_head(page); + f->mapping = NULL; + f->share = 0; + f->pgmap = pgmap; + } + + return order; +} +EXPORT_SYMBOL_GPL(dax_folio_reset_order); + +static inline unsigned long dax_folio_put(struct folio *folio) +{ + unsigned long ref; + int order, i; + + if (!dax_folio_is_shared(folio)) + ref = 0; + else + ref = --folio->share; + + if (ref) + return ref; + + order = dax_folio_reset_order(folio); + + /* Debug check: verify refcounts are zero for all sub-folios */ + for (i = 0; i < (1UL << order); i++) { + struct page *page = folio_page(folio, i); + + WARN_ON_ONCE(folio_ref_count((struct folio *)page)); + } + + return ref; +} + +static void dax_folio_init(void *entry) +{ + struct folio *folio = dax_to_folio(entry); + int order = dax_entry_order(entry); + + /* + * Folio should have been split back to order-0 pages in + * dax_folio_put() when they were removed from their + * final mapping. + */ + WARN_ON_ONCE(folio_order(folio)); + + if (order > 0) { + prep_compound_page(&folio->page, order); + if (order > 1) + INIT_LIST_HEAD(&folio->_deferred_list); + WARN_ON_ONCE(folio_ref_count(folio)); + } +} + +static void dax_associate_entry(void *entry, struct address_space *mapping, + struct vm_area_struct *vma, + unsigned long address, bool shared) +{ + unsigned long size = dax_entry_size(entry), index; + struct folio *folio = dax_to_folio(entry); + + if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) + return; + + index = linear_page_index(vma, address & ~(size - 1)); + if (shared && (folio->mapping || dax_folio_is_shared(folio))) { + if (folio->mapping) + dax_folio_make_shared(folio); + + WARN_ON_ONCE(!folio->share); + WARN_ON_ONCE(dax_entry_order(entry) != folio_order(folio)); + folio->share++; + } else { + WARN_ON_ONCE(folio->mapping); + dax_folio_init(entry); + folio = dax_to_folio(entry); + folio->mapping = mapping; + folio->index = index; + } +} + +static void dax_disassociate_entry(void *entry, struct address_space *mapping, + bool trunc) +{ + struct folio *folio = dax_to_folio(entry); + + if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) + return; + + dax_folio_put(folio); +} + +static struct page *dax_busy_page(void *entry) +{ + struct folio *folio = dax_to_folio(entry); + + if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) + return NULL; + + if (folio_ref_count(folio) - folio_mapcount(folio)) + return &folio->page; + else + return NULL; +} + +/** + * dax_lock_folio - Lock the DAX entry corresponding to a folio + * @folio: The folio whose entry we want to lock + * + * Context: Process context. + * Return: A cookie to pass to dax_unlock_folio() or 0 if the entry could + * not be locked. + */ +dax_entry_t dax_lock_folio(struct folio *folio) +{ + XA_STATE(xas, NULL, 0); + void *entry; + + /* Ensure folio->mapping isn't freed while we look at it */ + rcu_read_lock(); + for (;;) { + struct address_space *mapping = READ_ONCE(folio->mapping); + + entry = NULL; + if (!mapping || !dax_mapping(mapping)) + break; + + /* + * In the device-dax case there's no need to lock, a + * struct dev_pagemap pin is sufficient to keep the + * inode alive, and we assume we have dev_pagemap pin + * otherwise we would not have a valid pfn_to_page() + * translation. + */ + entry = (void *)~0UL; + if (S_ISCHR(mapping->host->i_mode)) + break; + + xas.xa = &mapping->i_pages; + xas_lock_irq(&xas); + if (mapping != folio->mapping) { + xas_unlock_irq(&xas); + continue; + } + xas_set(&xas, folio->index); + entry = xas_load(&xas); + if (dax_is_locked(entry)) { + rcu_read_unlock(); + wait_entry_unlocked(&xas, entry); + rcu_read_lock(); + continue; + } + dax_lock_entry(&xas, entry); + xas_unlock_irq(&xas); + break; + } + rcu_read_unlock(); + return (dax_entry_t)entry; +} + +void dax_unlock_folio(struct folio *folio, dax_entry_t cookie) +{ + struct address_space *mapping = folio->mapping; + XA_STATE(xas, &mapping->i_pages, folio->index); + + if (S_ISCHR(mapping->host->i_mode)) + return; + + dax_unlock_entry(&xas, (void *)cookie); +} + +/* + * dax_lock_mapping_entry - Lock the DAX entry corresponding to a mapping + * @mapping: the file's mapping whose entry we want to lock + * @index: the offset within this file + * @page: output the dax page corresponding to this dax entry + * + * Return: A cookie to pass to dax_unlock_mapping_entry() or 0 if the entry + * could not be locked. + */ +dax_entry_t dax_lock_mapping_entry(struct address_space *mapping, pgoff_t index, + struct page **page) +{ + XA_STATE(xas, NULL, 0); + void *entry; + + rcu_read_lock(); + for (;;) { + entry = NULL; + if (!dax_mapping(mapping)) + break; + + xas.xa = &mapping->i_pages; + xas_lock_irq(&xas); + xas_set(&xas, index); + entry = xas_load(&xas); + if (dax_is_locked(entry)) { + rcu_read_unlock(); + wait_entry_unlocked(&xas, entry); + rcu_read_lock(); + continue; + } + if (!entry || + dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) { + /* + * Because we are looking for entry from file's mapping + * and index, so the entry may not be inserted for now, + * or even a zero/empty entry. We don't think this is + * an error case. So, return a special value and do + * not output @page. + */ + entry = (void *)~0UL; + } else { + *page = pfn_to_page(dax_to_pfn(entry)); + dax_lock_entry(&xas, entry); + } + xas_unlock_irq(&xas); + break; + } + rcu_read_unlock(); + return (dax_entry_t)entry; +} + +void dax_unlock_mapping_entry(struct address_space *mapping, pgoff_t index, + dax_entry_t cookie) +{ + XA_STATE(xas, &mapping->i_pages, index); + + if (cookie == ~0UL) + return; + + dax_unlock_entry(&xas, (void *)cookie); +} + +/* + * Find page cache entry at given index. If it is a DAX entry, return it + * with the entry locked. If the page cache doesn't contain an entry at + * that index, add a locked empty entry. + * + * When requesting an entry with size DAX_PMD, grab_mapping_entry() will + * either return that locked entry or will return VM_FAULT_FALLBACK. + * This will happen if there are any PTE entries within the PMD range + * that we are requesting. + * + * We always favor PTE entries over PMD entries. There isn't a flow where we + * evict PTE entries in order to 'upgrade' them to a PMD entry. A PMD + * insertion will fail if it finds any PTE entries already in the tree, and a + * PTE insertion will cause an existing PMD entry to be unmapped and + * downgraded to PTE entries. This happens for both PMD zero pages as + * well as PMD empty entries. + * + * The exception to this downgrade path is for PMD entries that have + * real storage backing them. We will leave these real PMD entries in + * the tree, and PTE writes will simply dirty the entire PMD entry. + * + * Note: Unlike filemap_fault() we don't honor FAULT_FLAG_RETRY flags. For + * persistent memory the benefit is doubtful. We can add that later if we can + * show it helps. + * + * On error, this function does not return an ERR_PTR. Instead it returns + * a VM_FAULT code, encoded as an xarray internal entry. The ERR_PTR values + * overlap with xarray value entries. + */ +static void *grab_mapping_entry(struct xa_state *xas, + struct address_space *mapping, unsigned int order) +{ + unsigned long index = xas->xa_index; + bool pmd_downgrade; /* splitting PMD entry into PTE entries? */ + void *entry; + +retry: + pmd_downgrade = false; + xas_lock_irq(xas); + entry = get_next_unlocked_entry(xas, order); + + if (entry) { + if (dax_is_conflict(entry)) + goto fallback; + if (!xa_is_value(entry)) { + xas_set_err(xas, -EIO); + goto out_unlock; + } + + if (order == 0) { + if (dax_is_pmd_entry(entry) && + (dax_is_zero_entry(entry) || + dax_is_empty_entry(entry))) { + pmd_downgrade = true; + } + } + } + + if (pmd_downgrade) { + /* + * Make sure 'entry' remains valid while we drop + * the i_pages lock. + */ + dax_lock_entry(xas, entry); + + /* + * Besides huge zero pages the only other thing that gets + * downgraded are empty entries which don't need to be + * unmapped. + */ + if (dax_is_zero_entry(entry)) { + xas_unlock_irq(xas); + unmap_mapping_pages(mapping, + xas->xa_index & ~PG_PMD_COLOUR, + PG_PMD_NR, false); + xas_reset(xas); + xas_lock_irq(xas); + } + + dax_disassociate_entry(entry, mapping, false); + xas_store(xas, NULL); /* undo the PMD join */ + dax_wake_entry(xas, entry, WAKE_ALL); + mapping->nrpages -= PG_PMD_NR; + entry = NULL; + xas_set(xas, index); + } + + if (entry) { + dax_lock_entry(xas, entry); + } else { + unsigned long flags = DAX_EMPTY; + + if (order > 0) + flags |= DAX_PMD; + entry = dax_make_entry(0, flags); + dax_lock_entry(xas, entry); + if (xas_error(xas)) + goto out_unlock; + mapping->nrpages += 1UL << order; + } + +out_unlock: + xas_unlock_irq(xas); + if (xas_nomem(xas, mapping_gfp_mask(mapping) & ~__GFP_HIGHMEM)) + goto retry; + if (xas->xa_node == XA_ERROR(-ENOMEM)) + return xa_mk_internal(VM_FAULT_OOM); + if (xas_error(xas)) + return xa_mk_internal(VM_FAULT_SIGBUS); + return entry; +fallback: + xas_unlock_irq(xas); + return xa_mk_internal(VM_FAULT_FALLBACK); +} + +/** + * dax_layout_busy_page_range - find first pinned page in @mapping + * @mapping: address space to scan for a page with ref count > 1 + * @start: Starting offset. Page containing 'start' is included. + * @end: End offset. Page containing 'end' is included. If 'end' is LLONG_MAX, + * pages from 'start' till the end of file are included. + * + * DAX requires ZONE_DEVICE mapped pages. These pages are never + * 'onlined' to the page allocator so they are considered idle when + * page->count == 1. A filesystem uses this interface to determine if + * any page in the mapping is busy, i.e. for DMA, or other + * get_user_pages() usages. + * + * It is expected that the filesystem is holding locks to block the + * establishment of new mappings in this address_space. I.e. it expects + * to be able to run unmap_mapping_range() and subsequently not race + * mapping_mapped() becoming true. + */ +struct page *dax_layout_busy_page_range(struct address_space *mapping, + loff_t start, loff_t end) +{ + void *entry; + unsigned int scanned = 0; + struct page *page = NULL; + pgoff_t start_idx = start >> PAGE_SHIFT; + pgoff_t end_idx; + XA_STATE(xas, &mapping->i_pages, start_idx); + + if (!dax_mapping(mapping)) + return NULL; + + /* If end == LLONG_MAX, all pages from start to till end of file */ + if (end == LLONG_MAX) + end_idx = ULONG_MAX; + else + end_idx = end >> PAGE_SHIFT; + /* + * If we race get_user_pages_fast() here either we'll see the + * elevated page count in the iteration and wait, or + * get_user_pages_fast() will see that the page it took a reference + * against is no longer mapped in the page tables and bail to the + * get_user_pages() slow path. The slow path is protected by + * pte_lock() and pmd_lock(). New references are not taken without + * holding those locks, and unmap_mapping_pages() will not zero the + * pte or pmd without holding the respective lock, so we are + * guaranteed to either see new references or prevent new + * references from being established. + */ + unmap_mapping_pages(mapping, start_idx, end_idx - start_idx + 1, 0); + + xas_lock_irq(&xas); + xas_for_each(&xas, entry, end_idx) { + if (WARN_ON_ONCE(!xa_is_value(entry))) + continue; + entry = wait_entry_unlocked_exclusive(&xas, entry); + if (entry) + page = dax_busy_page(entry); + put_unlocked_entry(&xas, entry, WAKE_NEXT); + if (page) + break; + if (++scanned % XA_CHECK_SCHED) + continue; + + xas_pause(&xas); + xas_unlock_irq(&xas); + cond_resched(); + xas_lock_irq(&xas); + } + xas_unlock_irq(&xas); + return page; +} +EXPORT_SYMBOL_GPL(dax_layout_busy_page_range); + +struct page *dax_layout_busy_page(struct address_space *mapping) +{ + return dax_layout_busy_page_range(mapping, 0, LLONG_MAX); +} +EXPORT_SYMBOL_GPL(dax_layout_busy_page); + +static int __dax_invalidate_entry(struct address_space *mapping, + pgoff_t index, bool trunc) +{ + XA_STATE(xas, &mapping->i_pages, index); + int ret = 0; + void *entry; + + xas_lock_irq(&xas); + entry = get_next_unlocked_entry(&xas, 0); + if (!entry || WARN_ON_ONCE(!xa_is_value(entry))) + goto out; + if (!trunc && + (xas_get_mark(&xas, PAGECACHE_TAG_DIRTY) || + xas_get_mark(&xas, PAGECACHE_TAG_TOWRITE))) + goto out; + dax_disassociate_entry(entry, mapping, trunc); + xas_store(&xas, NULL); + mapping->nrpages -= 1UL << dax_entry_order(entry); + ret = 1; +out: + put_unlocked_entry(&xas, entry, WAKE_ALL); + xas_unlock_irq(&xas); + return ret; +} + +static int __dax_clear_dirty_range(struct address_space *mapping, + pgoff_t start, pgoff_t end) +{ + XA_STATE(xas, &mapping->i_pages, start); + unsigned int scanned = 0; + void *entry; + + xas_lock_irq(&xas); + xas_for_each(&xas, entry, end) { + entry = wait_entry_unlocked_exclusive(&xas, entry); + if (!entry) + continue; + xas_clear_mark(&xas, PAGECACHE_TAG_DIRTY); + xas_clear_mark(&xas, PAGECACHE_TAG_TOWRITE); + put_unlocked_entry(&xas, entry, WAKE_NEXT); + + if (++scanned % XA_CHECK_SCHED) + continue; + + xas_pause(&xas); + xas_unlock_irq(&xas); + cond_resched(); + xas_lock_irq(&xas); + } + xas_unlock_irq(&xas); + + return 0; +} + +/* + * Delete DAX entry at @index from @mapping. Wait for it + * to be unlocked before deleting it. + */ +int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index) +{ + int ret = __dax_invalidate_entry(mapping, index, true); + + /* + * This gets called from truncate / punch_hole path. As such, the caller + * must hold locks protecting against concurrent modifications of the + * page cache (usually fs-private i_mmap_sem for writing). Since the + * caller has seen a DAX entry for this index, we better find it + * at that index as well... + */ + WARN_ON_ONCE(!ret); + return ret; +} + +void dax_delete_mapping_range(struct address_space *mapping, + loff_t start, loff_t end) +{ + void *entry; + pgoff_t start_idx = start >> PAGE_SHIFT; + pgoff_t end_idx; + XA_STATE(xas, &mapping->i_pages, start_idx); + + /* If end == LLONG_MAX, all pages from start to till end of file */ + if (end == LLONG_MAX) + end_idx = ULONG_MAX; + else + end_idx = end >> PAGE_SHIFT; + + xas_lock_irq(&xas); + xas_for_each(&xas, entry, end_idx) { + if (!xa_is_value(entry)) + continue; + entry = wait_entry_unlocked_exclusive(&xas, entry); + if (!entry) + continue; + dax_disassociate_entry(entry, mapping, true); + xas_store(&xas, NULL); + mapping->nrpages -= 1UL << dax_entry_order(entry); + put_unlocked_entry(&xas, entry, WAKE_ALL); + } + xas_unlock_irq(&xas); +} +EXPORT_SYMBOL_GPL(dax_delete_mapping_range); + +static int wait_page_idle(struct page *page, + void (cb)(struct inode *), + struct inode *inode) +{ + return ___wait_var_event(page, dax_page_is_idle(page), + TASK_INTERRUPTIBLE, 0, 0, cb(inode)); +} + +static void wait_page_idle_uninterruptible(struct page *page, + struct inode *inode) +{ + ___wait_var_event(page, dax_page_is_idle(page), + TASK_UNINTERRUPTIBLE, 0, 0, schedule()); +} + +/* + * Unmaps the inode and waits for any DMA to complete prior to deleting the + * DAX mapping entries for the range. + * + * For NOWAIT behavior, pass @cb as NULL to early-exit on first found + * busy page + */ +int dax_break_layout(struct inode *inode, loff_t start, loff_t end, + void (cb)(struct inode *)) +{ + struct page *page; + int error = 0; + + if (!dax_mapping(inode->i_mapping)) + return 0; + + do { + page = dax_layout_busy_page_range(inode->i_mapping, start, end); + if (!page) + break; + if (!cb) { + error = -ERESTARTSYS; + break; + } + + error = wait_page_idle(page, cb, inode); + } while (error == 0); + + if (!page) + dax_delete_mapping_range(inode->i_mapping, start, end); + + return error; +} +EXPORT_SYMBOL_GPL(dax_break_layout); + +void dax_break_layout_final(struct inode *inode) +{ + struct page *page; + + if (!dax_mapping(inode->i_mapping)) + return; + + do { + page = dax_layout_busy_page_range(inode->i_mapping, 0, + LLONG_MAX); + if (!page) + break; + + wait_page_idle_uninterruptible(page, inode); + } while (true); + + if (!page) + dax_delete_mapping_range(inode->i_mapping, 0, LLONG_MAX); +} +EXPORT_SYMBOL_GPL(dax_break_layout_final); + +/* + * Invalidate DAX entry if it is clean. + */ +int dax_invalidate_mapping_entry_sync(struct address_space *mapping, + pgoff_t index) +{ + return __dax_invalidate_entry(mapping, index, false); +} + +static pgoff_t dax_iomap_pgoff(const struct iomap *iomap, loff_t pos) +{ + return PHYS_PFN(iomap->addr + (pos & PAGE_MASK) - iomap->offset); +} + +static int copy_cow_page_dax(struct vm_fault *vmf, const struct iomap_iter *iter) +{ + pgoff_t pgoff = dax_iomap_pgoff(&iter->iomap, iter->pos); + void *vto, *kaddr; + long rc; + int id; + + id = dax_read_lock(); + rc = dax_direct_access(iter->iomap.dax_dev, pgoff, 1, DAX_ACCESS, + &kaddr, NULL); + if (rc < 0) { + dax_read_unlock(id); + return rc; + } + vto = kmap_atomic(vmf->cow_page); + copy_user_page(vto, kaddr, vmf->address, vmf->cow_page); + kunmap_atomic(vto); + dax_read_unlock(id); + return 0; +} + +/* + * MAP_SYNC on a dax mapping guarantees dirty metadata is + * flushed on write-faults (non-cow), but not read-faults. + */ +static bool dax_fault_is_synchronous(const struct iomap_iter *iter, + struct vm_area_struct *vma) +{ + return (iter->flags & IOMAP_WRITE) && (vma->vm_flags & VM_SYNC) && + (iter->iomap.flags & IOMAP_F_DIRTY); +} + +/* + * By this point grab_mapping_entry() has ensured that we have a locked entry + * of the appropriate size so we don't have to worry about downgrading PMDs to + * PTEs. If we happen to be trying to insert a PTE and there is a PMD + * already in the tree, we will skip the insertion and just dirty the PMD as + * appropriate. + */ +static void *dax_insert_entry(struct xa_state *xas, struct vm_fault *vmf, + const struct iomap_iter *iter, void *entry, unsigned long pfn, + unsigned long flags) +{ + struct address_space *mapping = vmf->vma->vm_file->f_mapping; + void *new_entry = dax_make_entry(pfn, flags); + bool write = iter->flags & IOMAP_WRITE; + bool dirty = write && !dax_fault_is_synchronous(iter, vmf->vma); + bool shared = iter->iomap.flags & IOMAP_F_SHARED; + + if (dirty) + __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); + + if (shared || (dax_is_zero_entry(entry) && !(flags & DAX_ZERO_PAGE))) { + unsigned long index = xas->xa_index; + /* we are replacing a zero page with block mapping */ + if (dax_is_pmd_entry(entry)) + unmap_mapping_pages(mapping, index & ~PG_PMD_COLOUR, + PG_PMD_NR, false); + else /* pte entry */ + unmap_mapping_pages(mapping, index, 1, false); + } + + xas_reset(xas); + xas_lock_irq(xas); + if (shared || dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) { + void *old; + + dax_disassociate_entry(entry, mapping, false); + dax_associate_entry(new_entry, mapping, vmf->vma, + vmf->address, shared); + + /* + * Only swap our new entry into the page cache if the current + * entry is a zero page or an empty entry. If a normal PTE or + * PMD entry is already in the cache, we leave it alone. This + * means that if we are trying to insert a PTE and the + * existing entry is a PMD, we will just leave the PMD in the + * tree and dirty it if necessary. + */ + old = dax_lock_entry(xas, new_entry); + WARN_ON_ONCE(old != xa_mk_value(xa_to_value(entry) | + DAX_LOCKED)); + entry = new_entry; + } else { + xas_load(xas); /* Walk the xa_state */ + } + + if (dirty) + xas_set_mark(xas, PAGECACHE_TAG_DIRTY); + + if (write && shared) + xas_set_mark(xas, PAGECACHE_TAG_TOWRITE); + + xas_unlock_irq(xas); + return entry; +} + +static int dax_writeback_one(struct xa_state *xas, struct dax_device *dax_dev, + struct address_space *mapping, void *entry) +{ + unsigned long pfn, index, count, end; + long ret = 0; + struct vm_area_struct *vma; + + /* + * A page got tagged dirty in DAX mapping? Something is seriously + * wrong. + */ + if (WARN_ON(!xa_is_value(entry))) + return -EIO; + + if (unlikely(dax_is_locked(entry))) { + void *old_entry = entry; + + entry = get_next_unlocked_entry(xas, 0); + + /* Entry got punched out / reallocated? */ + if (!entry || WARN_ON_ONCE(!xa_is_value(entry))) + goto put_unlocked; + /* + * Entry got reallocated elsewhere? No need to writeback. + * We have to compare pfns as we must not bail out due to + * difference in lockbit or entry type. + */ + if (dax_to_pfn(old_entry) != dax_to_pfn(entry)) + goto put_unlocked; + if (WARN_ON_ONCE(dax_is_empty_entry(entry) || + dax_is_zero_entry(entry))) { + ret = -EIO; + goto put_unlocked; + } + + /* Another fsync thread may have already done this entry */ + if (!xas_get_mark(xas, PAGECACHE_TAG_TOWRITE)) + goto put_unlocked; + } + + /* Lock the entry to serialize with page faults */ + dax_lock_entry(xas, entry); + + /* + * We can clear the tag now but we have to be careful so that concurrent + * dax_writeback_one() calls for the same index cannot finish before we + * actually flush the caches. This is achieved as the calls will look + * at the entry only under the i_pages lock and once they do that + * they will see the entry locked and wait for it to unlock. + */ + xas_clear_mark(xas, PAGECACHE_TAG_TOWRITE); + xas_unlock_irq(xas); + + /* + * If dax_writeback_mapping_range() was given a wbc->range_start + * in the middle of a PMD, the 'index' we use needs to be + * aligned to the start of the PMD. + * This allows us to flush for PMD_SIZE and not have to worry about + * partial PMD writebacks. + */ + pfn = dax_to_pfn(entry); + count = 1UL << dax_entry_order(entry); + index = xas->xa_index & ~(count - 1); + end = index + count - 1; + + /* Walk all mappings of a given index of a file and writeprotect them */ + i_mmap_lock_read(mapping); + mapping_rmap_tree_foreach(vma, mapping, index, end) { + pfn_mkclean_range(pfn, count, index, vma); + cond_resched(); + } + i_mmap_unlock_read(mapping); + + dax_flush(dax_dev, page_address(pfn_to_page(pfn)), count * PAGE_SIZE); + /* + * After we have flushed the cache, we can clear the dirty tag. There + * cannot be new dirty data in the pfn after the flush has completed as + * the pfn mappings are writeprotected and fault waits for mapping + * entry lock. + */ + xas_reset(xas); + xas_lock_irq(xas); + xas_store(xas, entry); + xas_clear_mark(xas, PAGECACHE_TAG_DIRTY); + dax_wake_entry(xas, entry, WAKE_NEXT); + + trace_dax_writeback_one(mapping->host, index, count); + return ret; + + put_unlocked: + put_unlocked_entry(xas, entry, WAKE_NEXT); + return ret; +} + +/* + * Flush the mapping to the persistent domain within the byte range of [start, + * end]. This is required by data integrity operations to ensure file data is + * on persistent storage prior to completion of the operation. + */ +int dax_writeback_mapping_range(struct address_space *mapping, + struct dax_device *dax_dev, struct writeback_control *wbc) +{ + XA_STATE(xas, &mapping->i_pages, wbc->range_start >> PAGE_SHIFT); + struct inode *inode = mapping->host; + pgoff_t end_index = wbc->range_end >> PAGE_SHIFT; + void *entry; + int ret = 0; + unsigned int scanned = 0; + + if (WARN_ON_ONCE(inode->i_blkbits != PAGE_SHIFT)) + return -EIO; + + if (mapping_empty(mapping) || wbc->sync_mode != WB_SYNC_ALL) + return 0; + + trace_dax_writeback_range(inode, xas.xa_index, end_index); + + tag_pages_for_writeback(mapping, xas.xa_index, end_index); + + xas_lock_irq(&xas); + xas_for_each_marked(&xas, entry, end_index, PAGECACHE_TAG_TOWRITE) { + ret = dax_writeback_one(&xas, dax_dev, mapping, entry); + if (ret < 0) { + mapping_set_error(mapping, ret); + break; + } + if (++scanned % XA_CHECK_SCHED) + continue; + + xas_pause(&xas); + xas_unlock_irq(&xas); + cond_resched(); + xas_lock_irq(&xas); + } + xas_unlock_irq(&xas); + trace_dax_writeback_range_done(inode, xas.xa_index, end_index); + return ret; +} +EXPORT_SYMBOL_GPL(dax_writeback_mapping_range); + +static int dax_iomap_direct_access(const struct iomap *iomap, loff_t pos, + size_t size, void **kaddr, unsigned long *pfnp) +{ + pgoff_t pgoff = dax_iomap_pgoff(iomap, pos); + int id, rc = 0; + long length; + + id = dax_read_lock(); + length = dax_direct_access(iomap->dax_dev, pgoff, PHYS_PFN(size), + DAX_ACCESS, kaddr, pfnp); + if (length < 0) { + rc = length; + goto out; + } + if (!pfnp) + goto out_check_addr; + rc = -EINVAL; + if (PFN_PHYS(length) < size) + goto out; + if (*pfnp & (PHYS_PFN(size)-1)) + goto out; + + rc = 0; + +out_check_addr: + if (!kaddr) + goto out; + if (!*kaddr) + rc = -EFAULT; +out: + dax_read_unlock(id); + return rc; +} + +/** + * dax_iomap_copy_around - Prepare for an unaligned write to a shared/cow page + * by copying the data before and after the range to be written. + * @pos: address to do copy from. + * @length: size of copy operation. + * @align_size: aligned w.r.t align_size (either PMD_SIZE or PAGE_SIZE) + * @srcmap: iomap srcmap + * @daddr: destination address to copy to. + * + * This can be called from two places. Either during DAX write fault (page + * aligned), to copy the length size data to daddr. Or, while doing normal DAX + * write operation, dax_iomap_iter() might call this to do the copy of either + * start or end unaligned address. In the latter case the rest of the copy of + * aligned ranges is taken care by dax_iomap_iter() itself. + * If the srcmap contains invalid data, such as HOLE and UNWRITTEN, zero the + * area to make sure no old data remains. + */ +static int dax_iomap_copy_around(loff_t pos, uint64_t length, size_t align_size, + const struct iomap *srcmap, void *daddr) +{ + loff_t head_off = pos & (align_size - 1); + size_t size = ALIGN(head_off + length, align_size); + loff_t end = pos + length; + loff_t pg_end = round_up(end, align_size); + /* copy_all is usually in page fault case */ + bool copy_all = head_off == 0 && end == pg_end; + /* zero the edges if srcmap is a HOLE or IOMAP_UNWRITTEN */ + bool zero_edge = srcmap->flags & IOMAP_F_SHARED || + srcmap->type == IOMAP_UNWRITTEN; + void *saddr = NULL; + int ret = 0; + + if (!zero_edge) { + ret = dax_iomap_direct_access(srcmap, pos, size, &saddr, NULL); + if (ret) + return dax_mem2blk_err(ret); + } + + if (copy_all) { + if (zero_edge) + memset(daddr, 0, size); + else + ret = copy_mc_to_kernel(daddr, saddr, length); + goto out; + } + + /* Copy the head part of the range */ + if (head_off) { + if (zero_edge) + memset(daddr, 0, head_off); + else { + ret = copy_mc_to_kernel(daddr, saddr, head_off); + if (ret) + return -EIO; + } + } + + /* Copy the tail part of the range */ + if (end < pg_end) { + loff_t tail_off = head_off + length; + loff_t tail_len = pg_end - end; + + if (zero_edge) + memset(daddr + tail_off, 0, tail_len); + else { + ret = copy_mc_to_kernel(daddr + tail_off, + saddr + tail_off, tail_len); + if (ret) + return -EIO; + } + } +out: + if (zero_edge) + dax_flush(srcmap->dax_dev, daddr, size); + return ret ? -EIO : 0; +} + +/* + * The user has performed a load from a hole in the file. Allocating a new + * page in the file would cause excessive storage usage for workloads with + * sparse files. Instead we insert a read-only mapping of the 4k zero page. + * If this page is ever written to we will re-fault and change the mapping to + * point to real DAX storage instead. + */ +static vm_fault_t dax_load_hole(struct xa_state *xas, struct vm_fault *vmf, + const struct iomap_iter *iter, void **entry) +{ + struct inode *inode = iter->inode; + unsigned long vaddr = vmf->address; + unsigned long pfn = zero_pfn(vaddr); + vm_fault_t ret; + + *entry = dax_insert_entry(xas, vmf, iter, *entry, pfn, DAX_ZERO_PAGE); + + ret = vmf_insert_page_mkwrite(vmf, pfn_to_page(pfn), false); + trace_dax_load_hole(inode, vmf, ret); + return ret; +} + +#ifdef CONFIG_FS_DAX_PMD +static vm_fault_t dax_pmd_load_hole(struct xa_state *xas, struct vm_fault *vmf, + const struct iomap_iter *iter, void **entry) +{ + struct address_space *mapping = vmf->vma->vm_file->f_mapping; + struct inode *inode = mapping->host; + struct folio *zero_folio; + vm_fault_t ret; + + zero_folio = mm_get_huge_zero_folio(vmf->vma->vm_mm); + + if (unlikely(!zero_folio)) { + trace_dax_pmd_load_hole_fallback(inode, vmf, zero_folio, *entry); + return VM_FAULT_FALLBACK; + } + + *entry = dax_insert_entry(xas, vmf, iter, *entry, folio_pfn(zero_folio), + DAX_PMD | DAX_ZERO_PAGE); + + ret = vmf_insert_folio_pmd(vmf, zero_folio, false); + if (ret == VM_FAULT_NOPAGE) + trace_dax_pmd_load_hole(inode, vmf, zero_folio, *entry); + return ret; +} +#else +static vm_fault_t dax_pmd_load_hole(struct xa_state *xas, struct vm_fault *vmf, + const struct iomap_iter *iter, void **entry) +{ + return VM_FAULT_FALLBACK; +} +#endif /* CONFIG_FS_DAX_PMD */ + +static int dax_unshare_iter(struct iomap_iter *iter) +{ + struct iomap *iomap = &iter->iomap; + const struct iomap *srcmap = iomap_iter_srcmap(iter); + loff_t copy_pos = iter->pos; + u64 copy_len = iomap_length(iter); + u32 mod; + int id = 0; + s64 ret; + void *daddr = NULL, *saddr = NULL; + + if (!iomap_want_unshare_iter(iter)) + return iomap_iter_advance_full(iter); + + /* + * Extend the file range to be aligned to fsblock/pagesize, because + * we need to copy entire blocks, not just the byte range specified. + * Invalidate the mapping because we're about to CoW. + */ + mod = offset_in_page(copy_pos); + if (mod) { + copy_len += mod; + copy_pos -= mod; + } + + mod = offset_in_page(copy_pos + copy_len); + if (mod) + copy_len += PAGE_SIZE - mod; + + invalidate_inode_pages2_range(iter->inode->i_mapping, + copy_pos >> PAGE_SHIFT, + (copy_pos + copy_len - 1) >> PAGE_SHIFT); + + id = dax_read_lock(); + ret = dax_iomap_direct_access(iomap, copy_pos, copy_len, &daddr, NULL); + if (ret < 0) + goto out_unlock; + + ret = dax_iomap_direct_access(srcmap, copy_pos, copy_len, &saddr, NULL); + if (ret < 0) + goto out_unlock; + + if (copy_mc_to_kernel(daddr, saddr, copy_len) != 0) + ret = -EIO; + +out_unlock: + dax_read_unlock(id); + if (ret < 0) + return dax_mem2blk_err(ret); + return iomap_iter_advance_full(iter); +} + +int dax_file_unshare(struct inode *inode, loff_t pos, loff_t len, + const struct iomap_ops *ops) +{ + struct iomap_iter iter = { + .inode = inode, + .pos = pos, + .flags = IOMAP_WRITE | IOMAP_UNSHARE | IOMAP_DAX, + }; + loff_t size = i_size_read(inode); + int ret; + + if (pos < 0 || pos >= size) + return 0; + + iter.len = min(len, size - pos); + while ((ret = iomap_iter(&iter, ops)) > 0) + iter.status = dax_unshare_iter(&iter); + return ret; +} +EXPORT_SYMBOL_GPL(dax_file_unshare); + +static int dax_memzero(struct iomap_iter *iter, loff_t pos, size_t size) +{ + const struct iomap *iomap = &iter->iomap; + const struct iomap *srcmap = iomap_iter_srcmap(iter); + unsigned offset = offset_in_page(pos); + pgoff_t pgoff = dax_iomap_pgoff(iomap, pos); + void *kaddr; + long ret; + + ret = dax_direct_access(iomap->dax_dev, pgoff, 1, DAX_ACCESS, &kaddr, + NULL); + if (ret < 0) + return dax_mem2blk_err(ret); + + memset(kaddr + offset, 0, size); + if (iomap->flags & IOMAP_F_SHARED) + ret = dax_iomap_copy_around(pos, size, PAGE_SIZE, srcmap, + kaddr); + else + dax_flush(iomap->dax_dev, kaddr + offset, size); + return ret; +} + +static int dax_zero_iter(struct iomap_iter *iter, bool *did_zero) +{ + const struct iomap *iomap = &iter->iomap; + const struct iomap *srcmap = iomap_iter_srcmap(iter); + u64 length = iomap_length(iter); + int ret; + + /* already zeroed? we're done. */ + if (srcmap->type == IOMAP_HOLE || srcmap->type == IOMAP_UNWRITTEN) + return iomap_iter_advance(iter, length); + + /* + * invalidate the pages whose sharing state is to be changed + * because of CoW. + */ + if (iomap->flags & IOMAP_F_SHARED) + invalidate_inode_pages2_range(iter->inode->i_mapping, + iter->pos >> PAGE_SHIFT, + (iter->pos + length - 1) >> PAGE_SHIFT); + + do { + loff_t pos = iter->pos; + unsigned offset = offset_in_page(pos); + pgoff_t pgoff = dax_iomap_pgoff(iomap, pos); + int id; + + length = min_t(u64, PAGE_SIZE - offset, length); + + id = dax_read_lock(); + if (IS_ALIGNED(pos, PAGE_SIZE) && length == PAGE_SIZE) + ret = dax_zero_page_range(iomap->dax_dev, pgoff, 1); + else + ret = dax_memzero(iter, pos, length); + dax_read_unlock(id); + + if (ret < 0) + return ret; + + ret = iomap_iter_advance(iter, length); + if (ret) + return ret; + } while ((length = iomap_length(iter)) > 0); + + if (did_zero) + *did_zero = true; + return ret; +} + +int dax_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero, + const struct iomap_ops *ops) +{ + struct iomap_iter iter = { + .inode = inode, + .pos = pos, + .len = len, + .flags = IOMAP_DAX | IOMAP_ZERO, + }; + int ret; + + while ((ret = iomap_iter(&iter, ops)) > 0) + iter.status = dax_zero_iter(&iter, did_zero); + return ret; +} +EXPORT_SYMBOL_GPL(dax_zero_range); + +int dax_truncate_page(struct inode *inode, loff_t pos, bool *did_zero, + const struct iomap_ops *ops) +{ + unsigned int blocksize = i_blocksize(inode); + unsigned int off = pos & (blocksize - 1); + + /* Block boundary? Nothing to do */ + if (!off) + return 0; + return dax_zero_range(inode, pos, blocksize - off, did_zero, ops); +} +EXPORT_SYMBOL_GPL(dax_truncate_page); + +static int dax_iomap_iter(struct iomap_iter *iomi, struct iov_iter *iter) +{ + const struct iomap *iomap = &iomi->iomap; + const struct iomap *srcmap = iomap_iter_srcmap(iomi); + loff_t length = iomap_length(iomi); + loff_t pos = iomi->pos; + struct dax_device *dax_dev = iomap->dax_dev; + loff_t end = pos + length, done = 0; + bool write = iov_iter_rw(iter) == WRITE; + bool cow = write && iomap->flags & IOMAP_F_SHARED; + ssize_t ret = 0; + size_t xfer; + int id; + + if (!write) { + end = min(end, i_size_read(iomi->inode)); + if (pos >= end) + return 0; + + if (iomap->type == IOMAP_HOLE || iomap->type == IOMAP_UNWRITTEN) { + done = iov_iter_zero(min(length, end - pos), iter); + return iomap_iter_advance(iomi, done); + } + } + + /* + * In DAX mode, enforce either pure overwrites of written extents, or + * writes to unwritten extents as part of a copy-on-write operation. + */ + if (WARN_ON_ONCE(iomap->type != IOMAP_MAPPED && + !(iomap->flags & IOMAP_F_SHARED))) + return -EIO; + + /* + * Write can allocate block for an area which has a hole page mapped + * into page tables. We have to tear down these mappings so that data + * written by write(2) is visible in mmap. + */ + if (iomap->flags & IOMAP_F_NEW || cow) { + /* + * Filesystem allows CoW on non-shared extents. The src extents + * may have been mmapped with dirty mark before. To be able to + * invalidate its dax entries, we need to clear the dirty mark + * in advance. + */ + if (cow) + __dax_clear_dirty_range(iomi->inode->i_mapping, + pos >> PAGE_SHIFT, + (end - 1) >> PAGE_SHIFT); + invalidate_inode_pages2_range(iomi->inode->i_mapping, + pos >> PAGE_SHIFT, + (end - 1) >> PAGE_SHIFT); + } + + id = dax_read_lock(); + while ((pos = iomi->pos) < end) { + unsigned offset = pos & (PAGE_SIZE - 1); + const size_t size = ALIGN(length + offset, PAGE_SIZE); + pgoff_t pgoff = dax_iomap_pgoff(iomap, pos); + ssize_t map_len; + bool recovery = false; + void *kaddr; + + if (fatal_signal_pending(current)) { + ret = -EINTR; + break; + } + + map_len = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size), + DAX_ACCESS, &kaddr, NULL); + if (map_len == -EHWPOISON && iov_iter_rw(iter) == WRITE) { + map_len = dax_direct_access(dax_dev, pgoff, + PHYS_PFN(size), DAX_RECOVERY_WRITE, + &kaddr, NULL); + if (map_len > 0) + recovery = true; + } + if (map_len < 0) { + ret = dax_mem2blk_err(map_len); + break; + } + + if (cow) { + ret = dax_iomap_copy_around(pos, length, PAGE_SIZE, + srcmap, kaddr); + if (ret) + break; + } + + map_len = PFN_PHYS(map_len); + kaddr += offset; + map_len -= offset; + if (map_len > end - pos) + map_len = end - pos; + + if (recovery) + xfer = dax_recovery_write(dax_dev, pgoff, kaddr, + map_len, iter); + else if (write) + xfer = dax_copy_from_iter(dax_dev, pgoff, kaddr, + map_len, iter); + else + xfer = dax_copy_to_iter(dax_dev, pgoff, kaddr, + map_len, iter); + + ret = iomap_iter_advance(iomi, xfer); + if (!ret && xfer == 0) + ret = -EFAULT; + if (xfer < map_len) + break; + length = iomap_length(iomi); + } + dax_read_unlock(id); + + return ret; +} + +/** + * dax_iomap_rw - Perform I/O to a DAX file + * @iocb: The control block for this I/O + * @iter: The addresses to do I/O from or to + * @ops: iomap ops passed from the file system + * + * This function performs read and write operations to directly mapped + * persistent memory. The callers needs to take care of read/write exclusion + * and evicting any page cache pages in the region under I/O. + */ +ssize_t +dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter, + const struct iomap_ops *ops) +{ + struct iomap_iter iomi = { + .inode = iocb->ki_filp->f_mapping->host, + .pos = iocb->ki_pos, + .len = iov_iter_count(iter), + .flags = IOMAP_DAX, + }; + loff_t done = 0; + int ret; + + if (WARN_ON_ONCE(iocb->ki_flags & IOCB_ATOMIC)) + return -EIO; + + if (!iomi.len) + return 0; + + if (iov_iter_rw(iter) == WRITE) { + lockdep_assert_held_write(&iomi.inode->i_rwsem); + iomi.flags |= IOMAP_WRITE; + } else if (!sb_rdonly(iomi.inode->i_sb)) { + lockdep_assert_held(&iomi.inode->i_rwsem); + } + + if (iocb->ki_flags & IOCB_NOWAIT) + iomi.flags |= IOMAP_NOWAIT; + + while ((ret = iomap_iter(&iomi, ops)) > 0) + iomi.status = dax_iomap_iter(&iomi, iter); + + done = iomi.pos - iocb->ki_pos; + iocb->ki_pos = iomi.pos; + return done ? done : ret; +} +EXPORT_SYMBOL_GPL(dax_iomap_rw); + +static vm_fault_t dax_fault_return(int error) +{ + if (error == 0) + return VM_FAULT_NOPAGE; + return vmf_error(error); +} + +/* + * When handling a synchronous page fault and the inode need a fsync, we can + * insert the PTE/PMD into page tables only after that fsync happened. Skip + * insertion for now and return the pfn so that caller can insert it after the + * fsync is done. + */ +static vm_fault_t dax_fault_synchronous_pfnp(unsigned long *pfnp, + unsigned long pfn) +{ + if (WARN_ON_ONCE(!pfnp)) + return VM_FAULT_SIGBUS; + *pfnp = pfn; + return VM_FAULT_NEEDDSYNC; +} + +static vm_fault_t dax_fault_cow_page(struct vm_fault *vmf, + const struct iomap_iter *iter) +{ + vm_fault_t ret; + int error = 0; + + switch (iter->iomap.type) { + case IOMAP_HOLE: + case IOMAP_UNWRITTEN: + clear_user_highpage(vmf->cow_page, vmf->address); + break; + case IOMAP_MAPPED: + error = copy_cow_page_dax(vmf, iter); + break; + default: + WARN_ON_ONCE(1); + error = -EIO; + break; + } + + if (error) + return dax_fault_return(error); + + __SetPageUptodate(vmf->cow_page); + ret = finish_fault(vmf); + if (!ret) + return VM_FAULT_DONE_COW; + return ret; +} + +/** + * dax_fault_iter - Common actor to handle pfn insertion in PTE/PMD fault. + * @vmf: vm fault instance + * @iter: iomap iter + * @pfnp: pfn to be returned + * @xas: the dax mapping tree of a file + * @entry: an unlocked dax entry to be inserted + * @pmd: distinguish whether it is a pmd fault + */ +static vm_fault_t dax_fault_iter(struct vm_fault *vmf, + const struct iomap_iter *iter, unsigned long *pfnp, + struct xa_state *xas, void **entry, bool pmd) +{ + const struct iomap *iomap = &iter->iomap; + const struct iomap *srcmap = iomap_iter_srcmap(iter); + size_t size = pmd ? PMD_SIZE : PAGE_SIZE; + loff_t pos = (loff_t)xas->xa_index << PAGE_SHIFT; + bool write = iter->flags & IOMAP_WRITE; + unsigned long entry_flags = pmd ? DAX_PMD : 0; + struct folio *folio; + int ret, err = 0; + unsigned long pfn; + void *kaddr; + + if (!pmd && vmf->cow_page) + return dax_fault_cow_page(vmf, iter); + + /* if we are reading UNWRITTEN and HOLE, return a hole. */ + if (!write && + (iomap->type == IOMAP_UNWRITTEN || iomap->type == IOMAP_HOLE)) { + if (!pmd) + return dax_load_hole(xas, vmf, iter, entry); + return dax_pmd_load_hole(xas, vmf, iter, entry); + } + + if (iomap->type != IOMAP_MAPPED && !(iomap->flags & IOMAP_F_SHARED)) { + WARN_ON_ONCE(1); + return pmd ? VM_FAULT_FALLBACK : VM_FAULT_SIGBUS; + } + + err = dax_iomap_direct_access(iomap, pos, size, &kaddr, &pfn); + if (err) + return pmd ? VM_FAULT_FALLBACK : dax_fault_return(err); + + *entry = dax_insert_entry(xas, vmf, iter, *entry, pfn, entry_flags); + + if (write && iomap->flags & IOMAP_F_SHARED) { + err = dax_iomap_copy_around(pos, size, size, srcmap, kaddr); + if (err) + return dax_fault_return(err); + } + + folio = dax_to_folio(*entry); + if (dax_fault_is_synchronous(iter, vmf->vma)) + return dax_fault_synchronous_pfnp(pfnp, pfn); + + folio_ref_inc(folio); + if (pmd) + ret = vmf_insert_folio_pmd(vmf, pfn_folio(pfn), write); + else + ret = vmf_insert_page_mkwrite(vmf, pfn_to_page(pfn), write); + folio_put(folio); + + return ret; +} + +static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, unsigned long *pfnp, + int *iomap_errp, const struct iomap_ops *ops) +{ + struct address_space *mapping = vmf->vma->vm_file->f_mapping; + XA_STATE(xas, &mapping->i_pages, vmf->pgoff); + struct iomap_iter iter = { + .inode = mapping->host, + .pos = (loff_t)vmf->pgoff << PAGE_SHIFT, + .len = PAGE_SIZE, + .flags = IOMAP_DAX | IOMAP_FAULT, + }; + vm_fault_t ret = 0; + void *entry; + int error; + + trace_dax_pte_fault(iter.inode, vmf, ret); + /* + * Check whether offset isn't beyond end of file now. Caller is supposed + * to hold locks serializing us with truncate / punch hole so this is + * a reliable test. + */ + if (iter.pos >= i_size_read(iter.inode)) { + ret = VM_FAULT_SIGBUS; + goto out; + } + + if ((vmf->flags & FAULT_FLAG_WRITE) && !vmf->cow_page) + iter.flags |= IOMAP_WRITE; + + entry = grab_mapping_entry(&xas, mapping, 0); + if (xa_is_internal(entry)) { + ret = xa_to_internal(entry); + goto out; + } + + /* + * It is possible, particularly with mixed reads & writes to private + * mappings, that we have raced with a PMD fault that overlaps with + * the PTE we need to set up. If so just return and the fault will be + * retried. + */ + if (pmd_trans_huge(*vmf->pmd)) { + ret = VM_FAULT_NOPAGE; + goto unlock_entry; + } + + while ((error = iomap_iter(&iter, ops)) > 0) { + if (WARN_ON_ONCE(iomap_length(&iter) < PAGE_SIZE)) { + iter.status = -EIO; /* fs corruption? */ + continue; + } + + ret = dax_fault_iter(vmf, &iter, pfnp, &xas, &entry, false); + if (ret != VM_FAULT_SIGBUS && + (iter.iomap.flags & IOMAP_F_NEW)) { + count_vm_event(PGMAJFAULT); + count_memcg_event_mm(vmf->vma->vm_mm, PGMAJFAULT); + ret |= VM_FAULT_MAJOR; + } + + if (!(ret & VM_FAULT_ERROR)) + iter.status = iomap_iter_advance(&iter, PAGE_SIZE); + } + + if (iomap_errp) + *iomap_errp = error; + if (!ret && error) + ret = dax_fault_return(error); + +unlock_entry: + dax_unlock_entry(&xas, entry); +out: + trace_dax_pte_fault_done(iter.inode, vmf, ret); + return ret; +} + +#ifdef CONFIG_FS_DAX_PMD +static bool dax_fault_check_fallback(struct vm_fault *vmf, struct xa_state *xas, + pgoff_t max_pgoff) +{ + unsigned long pmd_addr = vmf->address & PMD_MASK; + bool write = vmf->flags & FAULT_FLAG_WRITE; + + /* + * Make sure that the faulting address's PMD offset (color) matches + * the PMD offset from the start of the file. This is necessary so + * that a PMD range in the page table overlaps exactly with a PMD + * range in the page cache. + */ + if ((vmf->pgoff & PG_PMD_COLOUR) != + ((vmf->address >> PAGE_SHIFT) & PG_PMD_COLOUR)) + return true; + + /* Fall back to PTEs if we're going to COW */ + if (write && !(vmf->vma->vm_flags & VM_SHARED)) + return true; + + /* If the PMD would extend outside the VMA */ + if (pmd_addr < vmf->vma->vm_start) + return true; + if ((pmd_addr + PMD_SIZE) > vmf->vma->vm_end) + return true; + + /* If the PMD would extend beyond the file size */ + if ((xas->xa_index | PG_PMD_COLOUR) >= max_pgoff) + return true; + + return false; +} + +static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, unsigned long *pfnp, + const struct iomap_ops *ops) +{ + struct address_space *mapping = vmf->vma->vm_file->f_mapping; + XA_STATE_ORDER(xas, &mapping->i_pages, vmf->pgoff, PMD_ORDER); + struct iomap_iter iter = { + .inode = mapping->host, + .len = PMD_SIZE, + .flags = IOMAP_DAX | IOMAP_FAULT, + }; + vm_fault_t ret = VM_FAULT_FALLBACK; + pgoff_t max_pgoff; + void *entry; + + if (vmf->flags & FAULT_FLAG_WRITE) + iter.flags |= IOMAP_WRITE; + + /* + * Check whether offset isn't beyond end of file now. Caller is + * supposed to hold locks serializing us with truncate / punch hole so + * this is a reliable test. + */ + max_pgoff = DIV_ROUND_UP(i_size_read(iter.inode), PAGE_SIZE); + + trace_dax_pmd_fault(iter.inode, vmf, max_pgoff, 0); + + if (xas.xa_index >= max_pgoff) { + ret = VM_FAULT_SIGBUS; + goto out; + } + + if (dax_fault_check_fallback(vmf, &xas, max_pgoff)) + goto fallback; + + /* + * grab_mapping_entry() will make sure we get an empty PMD entry, + * a zero PMD entry or a DAX PMD. If it can't (because a PTE + * entry is already in the array, for instance), it will return + * VM_FAULT_FALLBACK. + */ + entry = grab_mapping_entry(&xas, mapping, PMD_ORDER); + if (xa_is_internal(entry)) { + ret = xa_to_internal(entry); + goto fallback; + } + + /* + * It is possible, particularly with mixed reads & writes to private + * mappings, that we have raced with a PTE fault that overlaps with + * the PMD we need to set up. If so just return and the fault will be + * retried. + */ + if (!pmd_none(*vmf->pmd) && !pmd_trans_huge(*vmf->pmd)) { + ret = 0; + goto unlock_entry; + } + + iter.pos = (loff_t)xas.xa_index << PAGE_SHIFT; + while (iomap_iter(&iter, ops) > 0) { + if (iomap_length(&iter) < PMD_SIZE) + continue; /* actually breaks out of the loop */ + + ret = dax_fault_iter(vmf, &iter, pfnp, &xas, &entry, true); + if (ret != VM_FAULT_FALLBACK) + iter.status = iomap_iter_advance(&iter, PMD_SIZE); + } + +unlock_entry: + dax_unlock_entry(&xas, entry); +fallback: + if (ret == VM_FAULT_FALLBACK) { + split_huge_pmd(vmf->vma, vmf->pmd, vmf->address); + count_vm_event(THP_FAULT_FALLBACK); + } +out: + trace_dax_pmd_fault_done(iter.inode, vmf, max_pgoff, ret); + return ret; +} +#else +static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, unsigned long *pfnp, + const struct iomap_ops *ops) +{ + return VM_FAULT_FALLBACK; +} +#endif /* CONFIG_FS_DAX_PMD */ + +/** + * dax_iomap_fault - handle a page fault on a DAX file + * @vmf: The description of the fault + * @order: Order of the page to fault in + * @pfnp: PFN to insert for synchronous faults if fsync is required + * @iomap_errp: Storage for detailed error code in case of error + * @ops: Iomap ops passed from the file system + * + * When a page fault occurs, filesystems may call this helper in + * their fault handler for DAX files. dax_iomap_fault() assumes the caller + * has done all the necessary locking for page fault to proceed + * successfully. + */ +vm_fault_t dax_iomap_fault(struct vm_fault *vmf, unsigned int order, + unsigned long *pfnp, int *iomap_errp, + const struct iomap_ops *ops) +{ + if (order == 0) + return dax_iomap_pte_fault(vmf, pfnp, iomap_errp, ops); + else if (order == PMD_ORDER) + return dax_iomap_pmd_fault(vmf, pfnp, ops); + else + return VM_FAULT_FALLBACK; +} +EXPORT_SYMBOL_GPL(dax_iomap_fault); + +/* + * dax_insert_pfn_mkwrite - insert PTE or PMD entry into page tables + * @vmf: The description of the fault + * @pfn: PFN to insert + * @order: Order of entry to insert. + * + * This function inserts a writeable PTE or PMD entry into the page tables + * for an mmaped DAX file. It also marks the page cache entry as dirty. + */ +static vm_fault_t dax_insert_pfn_mkwrite(struct vm_fault *vmf, + unsigned long pfn, unsigned int order) +{ + struct address_space *mapping = vmf->vma->vm_file->f_mapping; + XA_STATE_ORDER(xas, &mapping->i_pages, vmf->pgoff, order); + struct folio *folio; + void *entry; + vm_fault_t ret; + + xas_lock_irq(&xas); + entry = get_next_unlocked_entry(&xas, order); + /* Did we race with someone splitting entry or so? */ + if (!entry || dax_is_conflict(entry) || + (order == 0 && !dax_is_pte_entry(entry))) { + put_unlocked_entry(&xas, entry, WAKE_NEXT); + xas_unlock_irq(&xas); + trace_dax_insert_pfn_mkwrite_no_entry(mapping->host, vmf, + VM_FAULT_NOPAGE); + return VM_FAULT_NOPAGE; + } + xas_set_mark(&xas, PAGECACHE_TAG_DIRTY); + dax_lock_entry(&xas, entry); + xas_unlock_irq(&xas); + folio = pfn_folio(pfn); + folio_ref_inc(folio); + if (order == 0) + ret = vmf_insert_page_mkwrite(vmf, &folio->page, true); +#ifdef CONFIG_FS_DAX_PMD + else if (order == PMD_ORDER) + ret = vmf_insert_folio_pmd(vmf, folio, FAULT_FLAG_WRITE); +#endif + else + ret = VM_FAULT_FALLBACK; + folio_put(folio); + dax_unlock_entry(&xas, entry); + trace_dax_insert_pfn_mkwrite(mapping->host, vmf, ret); + return ret; +} + +/** + * dax_finish_sync_fault - finish synchronous page fault + * @vmf: The description of the fault + * @order: Order of entry to be inserted + * @pfn: PFN to insert + * + * This function ensures that the file range touched by the page fault is + * stored persistently on the media and handles inserting of appropriate page + * table entry. + */ +vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf, unsigned int order, + unsigned long pfn) +{ + int err; + loff_t start = ((loff_t)vmf->pgoff) << PAGE_SHIFT; + size_t len = PAGE_SIZE << order; + + err = vfs_fsync_range(vmf->vma->vm_file, start, start + len - 1, 1); + if (err) + return VM_FAULT_SIGBUS; + return dax_insert_pfn_mkwrite(vmf, pfn, order); +} +EXPORT_SYMBOL_GPL(dax_finish_sync_fault); + +static int dax_range_compare_iter(struct iomap_iter *it_src, + struct iomap_iter *it_dest, u64 len, bool *same) +{ + const struct iomap *smap = &it_src->iomap; + const struct iomap *dmap = &it_dest->iomap; + loff_t pos1 = it_src->pos, pos2 = it_dest->pos; + void *saddr, *daddr; + int id, ret; + + len = min(len, min(smap->length, dmap->length)); + + if (smap->type == IOMAP_HOLE && dmap->type == IOMAP_HOLE) { + *same = true; + goto advance; + } + + if (smap->type == IOMAP_HOLE || dmap->type == IOMAP_HOLE) { + *same = false; + return 0; + } + + id = dax_read_lock(); + ret = dax_iomap_direct_access(smap, pos1, ALIGN(pos1 + len, PAGE_SIZE), + &saddr, NULL); + if (ret < 0) + goto out_unlock; + + ret = dax_iomap_direct_access(dmap, pos2, ALIGN(pos2 + len, PAGE_SIZE), + &daddr, NULL); + if (ret < 0) + goto out_unlock; + + *same = !memcmp(saddr, daddr, len); + if (!*same) + len = 0; + dax_read_unlock(id); + +advance: + ret = iomap_iter_advance(it_src, len); + if (!ret) + ret = iomap_iter_advance(it_dest, len); + return ret; + +out_unlock: + dax_read_unlock(id); + return -EIO; +} + +int dax_dedupe_file_range_compare(struct inode *src, loff_t srcoff, + struct inode *dst, loff_t dstoff, loff_t len, bool *same, + const struct iomap_ops *ops) +{ + struct iomap_iter src_iter = { + .inode = src, + .pos = srcoff, + .len = len, + .flags = IOMAP_DAX, + }; + struct iomap_iter dst_iter = { + .inode = dst, + .pos = dstoff, + .len = len, + .flags = IOMAP_DAX, + }; + int ret, status; + + while ((ret = iomap_iter(&src_iter, ops)) > 0 && + (ret = iomap_iter(&dst_iter, ops)) > 0) { + status = dax_range_compare_iter(&src_iter, &dst_iter, + min(src_iter.len, dst_iter.len), same); + if (status < 0) + return ret; + src_iter.status = dst_iter.status = status; + } + return ret; +} + +int dax_remap_file_range_prep(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + loff_t *len, unsigned int remap_flags, + const struct iomap_ops *ops) +{ + return __generic_remap_file_range_prep(file_in, pos_in, file_out, + pos_out, len, remap_flags, ops); +} +EXPORT_SYMBOL_GPL(dax_remap_file_range_prep); diff --git a/fs/dcache.c b/fs/dcache.c new file mode 100644 index 000000000..1b1a81f10 --- /dev/null +++ b/fs/dcache.c @@ -0,0 +1,3525 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * fs/dcache.c + * + * Complete reimplementation + * (C) 1997 Thomas Schoebel-Theuer, + * with heavy changes by Linus Torvalds + */ + +/* + * Notes on the allocation strategy: + * + * The dcache is a master of the icache - whenever a dcache entry + * exists, the inode will always exist. "iput()" is done either when + * the dcache entry is deleted or garbage collected. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" +#include "mount.h" + +#include + +/* + * Usage: + * dcache->d_inode->i_lock protects: + * - i_dentry, d_alias, d_inode of aliases + * dcache_hash_bucket lock protects: + * - the dcache hash table + * s_roots_lock protects: + * - the s_roots list (see __d_move()/dentry_unlist()/d_obtain_root()) + * dentry->d_sb->s_dentry_lru_lock protects: + * - the dcache lru lists and counters + * d_lock protects: + * - d_flags + * - d_name + * - d_lru + * - d_count + * - d_unhashed() + * - d_parent and d_chilren + * - childrens' d_sib and d_parent + * - d_alias, d_inode + * + * Ordering: + * dentry->d_inode->i_lock + * dentry->d_lock + * dentry->d_sb->s_dentry_lru_lock + * dcache_hash_bucket lock + * s_roots lock + * + * If there is an ancestor relationship: + * dentry->d_parent->...->d_parent->d_lock + * ... + * dentry->d_parent->d_lock + * dentry->d_lock + * + * If no ancestor relationship: + * arbitrary, since it's serialized on rename_lock + */ +static int sysctl_vfs_cache_pressure __read_mostly = 100; +static int sysctl_vfs_cache_pressure_denom __read_mostly = 100; + +unsigned long vfs_pressure_ratio(unsigned long val) +{ + return mult_frac(val, sysctl_vfs_cache_pressure, sysctl_vfs_cache_pressure_denom); +} +EXPORT_SYMBOL_GPL(vfs_pressure_ratio); + +__cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock); + +EXPORT_SYMBOL(rename_lock); + +static struct kmem_cache *__dentry_cache __ro_after_init; +#define dentry_cache runtime_const_ptr(__dentry_cache) + +const struct qstr empty_name = QSTR_INIT("", 0); +EXPORT_SYMBOL(empty_name); +const struct qstr slash_name = QSTR_INIT("/", 1); +EXPORT_SYMBOL(slash_name); +const struct qstr dotdot_name = QSTR_INIT("..", 2); +EXPORT_SYMBOL(dotdot_name); + +/* + * This is the single most critical data structure when it comes + * to the dcache: the hashtable for lookups. Somebody should try + * to make this good - I've just made it work. + * + * This hash-function tries to avoid losing too many bits of hash + * information, yet avoid using a prime hash-size or similar. + * + * Marking the variables "used" ensures that the compiler doesn't + * optimize them away completely on architectures with runtime + * constant infrastructure, this allows debuggers to see their + * values. But updating these values has no effect on those arches. + */ + +static unsigned int d_hash_shift __ro_after_init __used; + +static struct hlist_bl_head *dentry_hashtable __ro_after_init __used; + +static inline struct hlist_bl_head *d_hash(unsigned long hashlen) +{ + return runtime_const_ptr(dentry_hashtable) + + runtime_const_shift_right_32(hashlen, d_hash_shift); +} + +#define IN_LOOKUP_SHIFT 10 +static struct hlist_bl_head in_lookup_hashtable[1 << IN_LOOKUP_SHIFT]; + +static inline struct hlist_bl_head *in_lookup_hash(const struct dentry *parent, + unsigned int hash) +{ + hash += (unsigned long) parent / L1_CACHE_BYTES; + return in_lookup_hashtable + hash_32(hash, IN_LOOKUP_SHIFT); +} + +struct dentry_stat_t { + long nr_dentry; + long nr_unused; + long age_limit; /* age in seconds */ + long want_pages; /* pages requested by system */ + long nr_negative; /* # of unused negative dentries */ + long dummy; /* Reserved for future use */ +}; + +static DEFINE_PER_CPU(long, nr_dentry); +static DEFINE_PER_CPU(long, nr_dentry_unused); +static DEFINE_PER_CPU(long, nr_dentry_negative); +static int dentry_negative_policy; + +#if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS) +/* Statistics gathering. */ +static struct dentry_stat_t dentry_stat = { + .age_limit = 45, +}; + +/* + * Here we resort to our own counters instead of using generic per-cpu counters + * for consistency with what the vfs inode code does. We are expected to harvest + * better code and performance by having our own specialized counters. + * + * Please note that the loop is done over all possible CPUs, not over all online + * CPUs. The reason for this is that we don't want to play games with CPUs going + * on and off. If one of them goes off, we will just keep their counters. + * + * glommer: See cffbc8a for details, and if you ever intend to change this, + * please update all vfs counters to match. + */ +static long get_nr_dentry(void) +{ + int i; + long sum = 0; + for_each_possible_cpu(i) + sum += per_cpu(nr_dentry, i); + return sum < 0 ? 0 : sum; +} + +static long get_nr_dentry_unused(void) +{ + int i; + long sum = 0; + for_each_possible_cpu(i) + sum += per_cpu(nr_dentry_unused, i); + return sum < 0 ? 0 : sum; +} + +static long get_nr_dentry_negative(void) +{ + int i; + long sum = 0; + + for_each_possible_cpu(i) + sum += per_cpu(nr_dentry_negative, i); + return sum < 0 ? 0 : sum; +} + +static int proc_nr_dentry(const struct ctl_table *table, int write, void *buffer, + size_t *lenp, loff_t *ppos) +{ + dentry_stat.nr_dentry = get_nr_dentry(); + dentry_stat.nr_unused = get_nr_dentry_unused(); + dentry_stat.nr_negative = get_nr_dentry_negative(); + return proc_doulongvec_minmax(table, write, buffer, lenp, ppos); +} + +static const struct ctl_table fs_dcache_sysctls[] = { + { + .procname = "dentry-state", + .data = &dentry_stat, + .maxlen = 6*sizeof(long), + .mode = 0444, + .proc_handler = proc_nr_dentry, + }, + { + .procname = "dentry-negative", + .data = &dentry_negative_policy, + .maxlen = sizeof(dentry_negative_policy), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, +}; + +static const struct ctl_table vm_dcache_sysctls[] = { + { + .procname = "vfs_cache_pressure", + .data = &sysctl_vfs_cache_pressure, + .maxlen = sizeof(sysctl_vfs_cache_pressure), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + }, + { + .procname = "vfs_cache_pressure_denom", + .data = &sysctl_vfs_cache_pressure_denom, + .maxlen = sizeof(sysctl_vfs_cache_pressure_denom), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ONE_HUNDRED, + }, +}; + +static int __init init_fs_dcache_sysctls(void) +{ + register_sysctl_init("vm", vm_dcache_sysctls); + register_sysctl_init("fs", fs_dcache_sysctls); + return 0; +} +fs_initcall(init_fs_dcache_sysctls); +#endif + +/* + * Compare 2 name strings, return 0 if they match, otherwise non-zero. + * The strings are both count bytes long, and count is non-zero. + */ +#ifdef CONFIG_DCACHE_WORD_ACCESS + +#include +/* + * NOTE! 'cs' and 'scount' come from a dentry, so it has a + * aligned allocation for this particular component. We don't + * strictly need the load_unaligned_zeropad() safety, but it + * doesn't hurt either. + * + * In contrast, 'ct' and 'tcount' can be from a pathname, and do + * need the careful unaligned handling. + */ +static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount) +{ + unsigned long a,b,mask; + + for (;;) { + a = read_word_at_a_time(cs); + b = load_unaligned_zeropad(ct); + if (tcount < sizeof(unsigned long)) + break; + if (unlikely(a != b)) + return 1; + cs += sizeof(unsigned long); + ct += sizeof(unsigned long); + tcount -= sizeof(unsigned long); + if (!tcount) + return 0; + } + mask = bytemask_from_count(tcount); + return unlikely(!!((a ^ b) & mask)); +} + +#else + +static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount) +{ + do { + if (*cs != *ct) + return 1; + cs++; + ct++; + tcount--; + } while (tcount); + return 0; +} + +#endif + +static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *ct, unsigned tcount) +{ + /* + * Be careful about RCU walk racing with rename: + * use 'READ_ONCE' to fetch the name pointer. + * + * NOTE! Even if a rename will mean that the length + * was not loaded atomically, we don't care. The + * RCU walk will check the sequence count eventually, + * and catch it. And we won't overrun the buffer, + * because we're reading the name pointer atomically, + * and a dentry name is guaranteed to be properly + * terminated with a NUL byte. + * + * End result: even if 'len' is wrong, we'll exit + * early because the data cannot match (there can + * be no NUL in the ct/tcount data) + */ + const unsigned char *cs = READ_ONCE(dentry->d_name.name); + + return dentry_string_cmp(cs, ct, tcount); +} + +/* + * long names are allocated separately from dentry and never modified. + * Refcounted, freeing is RCU-delayed. See take_dentry_name_snapshot() + * for the reason why ->count and ->head can't be combined into a union. + * dentry_string_cmp() relies upon ->name[] being word-aligned. + */ +struct external_name { + atomic_t count; + struct rcu_head head; + unsigned char name[] __aligned(sizeof(unsigned long)); +}; + +static inline struct external_name *external_name(struct dentry *dentry) +{ + return container_of(dentry->d_name.name, struct external_name, name[0]); +} + +static void __d_free(struct rcu_head *head) +{ + struct dentry *dentry = container_of(head, struct dentry, d_rcu); + + kmem_cache_free(dentry_cache, dentry); +} + +static void __d_free_external(struct rcu_head *head) +{ + struct dentry *dentry = container_of(head, struct dentry, d_rcu); + kfree(external_name(dentry)); + kmem_cache_free(dentry_cache, dentry); +} + +static inline int dname_external(const struct dentry *dentry) +{ + return dentry->d_name.name != dentry->d_shortname.string; +} + +void take_dentry_name_snapshot(struct name_snapshot *name, struct dentry *dentry) +{ + unsigned seq; + const unsigned char *s; + + rcu_read_lock(); +retry: + seq = read_seqcount_begin(&dentry->d_seq); + s = READ_ONCE(dentry->d_name.name); + name->name.hash_len = dentry->d_name.hash_len; + name->name.name = name->inline_name.string; + if (likely(s == dentry->d_shortname.string)) { + name->inline_name = dentry->d_shortname; + } else { + struct external_name *p; + p = container_of(s, struct external_name, name[0]); + // get a valid reference + if (unlikely(!atomic_inc_not_zero(&p->count))) + goto retry; + name->name.name = s; + } + if (read_seqcount_retry(&dentry->d_seq, seq)) { + release_dentry_name_snapshot(name); + goto retry; + } + rcu_read_unlock(); +} +EXPORT_SYMBOL(take_dentry_name_snapshot); + +void release_dentry_name_snapshot(struct name_snapshot *name) +{ + if (unlikely(name->name.name != name->inline_name.string)) { + struct external_name *p; + p = container_of(name->name.name, struct external_name, name[0]); + if (unlikely(atomic_dec_and_test(&p->count))) + kfree_rcu(p, head); + } +} +EXPORT_SYMBOL(release_dentry_name_snapshot); + +static inline void __d_set_inode_and_type(struct dentry *dentry, + struct inode *inode, + unsigned type_flags) +{ + unsigned flags; + + dentry->d_inode = inode; + flags = READ_ONCE(dentry->d_flags); + flags &= ~DCACHE_ENTRY_TYPE; + flags |= type_flags; + smp_store_release(&dentry->d_flags, flags); +} + +static inline void __d_clear_type_and_inode(struct dentry *dentry) +{ + unsigned flags = READ_ONCE(dentry->d_flags); + + flags &= ~DCACHE_ENTRY_TYPE; + WRITE_ONCE(dentry->d_flags, flags); + dentry->d_inode = NULL; + /* + * The negative counter only tracks dentries on the LRU. Don't inc if + * d_lru is on another list. + */ + if ((flags & (DCACHE_LRU_LIST|DCACHE_SHRINK_LIST)) == DCACHE_LRU_LIST) + this_cpu_inc(nr_dentry_negative); +} + +#define DENTRY_WARN_ONCE(condition, dentry) \ + WARN_ONCE((condition), "dentry=%p d_flags=0x%x\n", (dentry), (dentry)->d_flags) +#define D_FLAG_VERIFY(dentry, x) \ + DENTRY_WARN_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_LIST)) != (x), (dentry)) + +static void dentry_free(struct dentry *dentry) +{ + DENTRY_WARN_ONCE(d_really_is_positive(dentry), dentry); + DENTRY_WARN_ONCE(!lockref_is_dead(&dentry->d_lockref), dentry); + D_FLAG_VERIFY(dentry, 0); + if (unlikely(dname_external(dentry))) { + struct external_name *p = external_name(dentry); + if (likely(atomic_dec_and_test(&p->count))) { + call_rcu(&dentry->d_rcu, __d_free_external); + return; + } + } + /* if dentry was never visible to RCU, immediate free is OK */ + if (dentry->d_flags & DCACHE_NORCU) + __d_free(&dentry->d_rcu); + else + call_rcu(&dentry->d_rcu, __d_free); +} + +/* + * Release the dentry's inode, using the filesystem + * d_iput() operation if defined. + */ +static void dentry_unlink_inode(struct dentry * dentry) + __releases(dentry->d_lock) + __releases(dentry->d_inode->i_lock) +{ + struct inode *inode = dentry->d_inode; + + raw_write_seqcount_begin(&dentry->d_seq); + __d_clear_type_and_inode(dentry); + __hlist_del(&dentry->d_alias); + /* + * dentry becomes negative, so the space occupied by ->d_alias + * belongs to ->waiters now. + */ + dentry->waiters = NULL; + raw_write_seqcount_end(&dentry->d_seq); + spin_unlock(&dentry->d_lock); + spin_unlock(&inode->i_lock); + if (!inode->i_nlink) + fsnotify_inoderemove(inode); + if (dentry->d_op && dentry->d_op->d_iput) + dentry->d_op->d_iput(dentry, inode); + else + iput(inode); +} + +/* + * The DCACHE_LRU_LIST bit is set whenever the 'd_lru' entry + * is in use - which includes both the "real" per-superblock + * LRU list _and_ the DCACHE_SHRINK_LIST use. + * + * The DCACHE_SHRINK_LIST bit is set whenever the dentry is + * on the shrink list (ie not on the superblock LRU list). + * + * The per-cpu "nr_dentry_unused" counters are updated with + * the DCACHE_LRU_LIST bit. + * + * The per-cpu "nr_dentry_negative" counters are only updated + * when deleted from or added to the per-superblock LRU list, not + * from/to the shrink list. That is to avoid an unneeded dec/inc + * pair when moving from LRU to shrink list in select_collect(). + * + * These helper functions make sure we always follow the + * rules. d_lock must be held by the caller. + */ +static void d_lru_add(struct dentry *dentry) +{ + D_FLAG_VERIFY(dentry, 0); + dentry->d_flags |= DCACHE_LRU_LIST; + this_cpu_inc(nr_dentry_unused); + if (d_is_negative(dentry)) + this_cpu_inc(nr_dentry_negative); + WARN_ON_ONCE(!list_lru_add_obj( + &dentry->d_sb->s_dentry_lru, &dentry->d_lru)); +} + +static void d_lru_del(struct dentry *dentry) +{ + D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST); + dentry->d_flags &= ~DCACHE_LRU_LIST; + this_cpu_dec(nr_dentry_unused); + if (d_is_negative(dentry)) + this_cpu_dec(nr_dentry_negative); + WARN_ON_ONCE(!list_lru_del_obj( + &dentry->d_sb->s_dentry_lru, &dentry->d_lru)); +} + +static void d_shrink_del(struct dentry *dentry) +{ + D_FLAG_VERIFY(dentry, DCACHE_SHRINK_LIST | DCACHE_LRU_LIST); + list_del_init(&dentry->d_lru); + dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST); + this_cpu_dec(nr_dentry_unused); +} + +static void d_shrink_add(struct dentry *dentry, struct list_head *list) +{ + D_FLAG_VERIFY(dentry, 0); + list_add(&dentry->d_lru, list); + dentry->d_flags |= DCACHE_SHRINK_LIST | DCACHE_LRU_LIST; + this_cpu_inc(nr_dentry_unused); +} + +/* + * These can only be called under the global LRU lock, ie during the + * callback for freeing the LRU list. "isolate" removes it from the + * LRU lists entirely, while shrink_move moves it to the indicated + * private list. + */ +static void d_lru_isolate(struct list_lru_one *lru, struct dentry *dentry) +{ + D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST); + dentry->d_flags &= ~DCACHE_LRU_LIST; + this_cpu_dec(nr_dentry_unused); + if (d_is_negative(dentry)) + this_cpu_dec(nr_dentry_negative); + list_lru_isolate(lru, &dentry->d_lru); +} + +static void d_lru_shrink_move(struct list_lru_one *lru, struct dentry *dentry, + struct list_head *list) +{ + D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST); + dentry->d_flags |= DCACHE_SHRINK_LIST; + if (d_is_negative(dentry)) + this_cpu_dec(nr_dentry_negative); + list_lru_isolate_move(lru, &dentry->d_lru, list); +} + +static void ___d_drop(struct dentry *dentry) +{ + struct hlist_bl_head *b = d_hash(dentry->d_name.hash); + + hlist_bl_lock(b); + __hlist_bl_del(&dentry->d_hash); + hlist_bl_unlock(b); +} + +void __d_drop(struct dentry *dentry) +{ + if (!d_unhashed(dentry)) { + ___d_drop(dentry); + dentry->d_hash.pprev = NULL; + write_seqcount_invalidate(&dentry->d_seq); + } +} +EXPORT_SYMBOL(__d_drop); + +/** + * d_drop - drop a dentry + * @dentry: dentry to drop + * + * d_drop() unhashes the entry from the parent dentry hashes, so that it won't + * be found through a VFS lookup any more. Note that this is different from + * deleting the dentry - d_delete will try to mark the dentry negative if + * possible, giving a successful _negative_ lookup, while d_drop will + * just make the cache lookup fail. + * + * d_drop() is used mainly for stuff that wants to invalidate a dentry for some + * reason (NFS timeouts or autofs deletes). + * + * __d_drop requires dentry->d_lock + * + * ___d_drop doesn't mark dentry as "unhashed" + * (dentry->d_hash.pprev will be LIST_POISON2, not NULL). + */ +void d_drop(struct dentry *dentry) +{ + spin_lock(&dentry->d_lock); + __d_drop(dentry); + spin_unlock(&dentry->d_lock); +} +EXPORT_SYMBOL(d_drop); + +struct completion_list { + struct completion_list *next; + struct completion completion; +}; + +/* + * shrink_dcache_tree() needs to be notified when dentry in process of + * being evicted finally gets unlisted. Such dentries are + * already with negative ->d_count + * already negative + * already not in in-lookup hash + * reachable only via ->d_sib. + * + * Use ->waiters for a single-linked list of struct completion_list of + * waiters. + */ +static inline bool d_add_waiter(struct dentry *dentry, struct completion_list *p) +{ + if (unlikely(dentry->d_flags & DCACHE_DENTRY_KILLED)) + return false; + init_completion(&p->completion); + p->next = dentry->waiters; + dentry->waiters = p; + return true; +} + +static inline void d_complete_waiters(struct dentry *dentry) +{ + struct completion_list *v = dentry->waiters; + if (unlikely(v)) { + /* some shrink_dcache_tree() instances are waiting */ + dentry->waiters = NULL; + while (v) { + struct completion *r = &v->completion; + v = v->next; + complete(r); + } + } +} + +static void unlink_secondary_root(struct dentry *dentry) +{ + spin_lock(&dentry->d_sb->s_roots_lock); + hlist_del_init(&dentry->d_sib); + spin_unlock(&dentry->d_sb->s_roots_lock); +} + +static inline void dentry_unlist(struct dentry *dentry) +{ + struct dentry *next; + /* + * Inform d_walk() and shrink_dentry_list() that we are no longer + * attached to the dentry tree + */ + dentry->d_flags |= DCACHE_DENTRY_KILLED; + d_complete_waiters(dentry); + if (unlikely(hlist_unhashed(&dentry->d_sib))) + return; + if (unlikely(IS_ROOT(dentry))) { + unlink_secondary_root(dentry); // secondary root goes away + return; + } + __hlist_del(&dentry->d_sib); + /* + * Cursors can move around the list of children. While we'd been + * a normal list member, it didn't matter - ->d_sib.next would've + * been updated. However, from now on it won't be and for the + * things like d_walk() it might end up with a nasty surprise. + * Normally d_walk() doesn't care about cursors moving around - + * ->d_lock on parent prevents that and since a cursor has no children + * of its own, we get through it without ever unlocking the parent. + * There is one exception, though - if we ascend from a child that + * gets killed as soon as we unlock it, the next sibling is found + * using the value left in its ->d_sib.next. And if _that_ + * pointed to a cursor, and cursor got moved (e.g. by lseek()) + * before d_walk() regains parent->d_lock, we'll end up skipping + * everything the cursor had been moved past. + * + * Solution: make sure that the pointer left behind in ->d_sib.next + * points to something that won't be moving around. I.e. skip the + * cursors. + */ + while (dentry->d_sib.next) { + next = hlist_entry(dentry->d_sib.next, struct dentry, d_sib); + if (likely(!(next->d_flags & DCACHE_DENTRY_CURSOR))) + break; + dentry->d_sib.next = next->d_sib.next; + } +} + +/* + * Prepare locking environment for killing a dentry. + * Called under dentry->d_lock. To proceed with eviction of a positive dentry + * we need to get ->i_lock of the inode of that dentry as well. + * However, ->i_lock nests outside of ->d_lock, so if trylock fails we might + * have to drop and regain the latter. Dentry state can change while its + * ->d_lock is not held - it might end up getting killed, becoming busy, + * negative, etc., so we need to be careful. + * + * For NORCU dentries memory safety relies upon having only one call of + * lock_for_kill() in the entire lifetime of dentry and dentry_free() being + * called only by the caller of lock_for_kill(). That this is NORCU-specific; + * the crucial part is that refcounts of NORCU dentries never grow once having + * dropped to zero. + * + * For normal dentries we can not assume that there won't be concurrent calls + * of dentry_free() - dentry might end up being evicted by another thread + * while we are dropping/retaking locks on the slow path. Memory safety is + * provided by keeping the RCU read-side critical area contiguous with + * an explicit rcu_read_lock() scope bridging over the break in spinlock scopes. + * + * If dentry is busy (or busy dying, or already dead), unlock dentry + * and return false. Otherwise, return true and have that dentry's + * inode (if any) locked in addition to dentry itself. + */ +static bool lock_for_kill(struct dentry *dentry) +{ + struct inode *inode = dentry->d_inode; + + if (unlikely(dentry->d_lockref.count)) { + spin_unlock(&dentry->d_lock); + return false; + } + + if (!inode || likely(spin_trylock(&inode->i_lock))) + return true; + + // Too bad - we need to drop ->d_lock and take locks in correct order. + // To avoid breaking RCU read-side critical area when we drop ->d_lock, + // take an explicit rcu_read_lock() while we are switching locks. + rcu_read_lock(); + do { + spin_unlock(&dentry->d_lock); + spin_lock(&inode->i_lock); + spin_lock(&dentry->d_lock); + // make sure we'd locked the right inode - ->d_inode might've + // changed while we were not holding ->d_lock + if (likely(inode == dentry->d_inode)) + break; + spin_unlock(&inode->i_lock); + inode = dentry->d_inode; + } while (inode); + rcu_read_unlock(); + if (likely(!dentry->d_lockref.count)) + return true; + if (inode) + spin_unlock(&inode->i_lock); + spin_unlock(&dentry->d_lock); + return false; +} + +/** + * dentry_kill - evict a dentry + * @dentry: dentry to be evicted + * + * All dentry evictions are done by this function. The reference we are + * passed does not contribute to the refcount; the caller had either + * already decremented the refcount or it had never held one in the + * first place. @dentry->d_lock is held by the caller and dropped + * by dentry_kill(@dentry). + * + * We are guaranteed that nobody had called dentry_free(@dentry) + * prior to the beginning of RCU read-side critical area we are in. + * + * Caller must not access @dentry after the call. + * + * If eviction of @dentry drops the last reference to its parent, + * the reference to parent is returned to caller. In that case + * it is guaranteed to satisfy the requirements for dentry_kill() + * argument - its ->d_lock is held and we are guaranteed that nobody + * had passed it to dentry_free() prior to acquisition of its ->d_lock. + * Otherwise %NULL is returned. + * + * If @dentry is idle and remains such after we assemble the full + * locking environment for eviction (see lock_for_kill() for details) + * we mark it doomed (see lockref_mark_dead()) and proceed to detaching + * it from any filesystem objects. Otherwise we drop ->d_lock and + * return %NULL. + * + * Once @dentry is detached from the filesystem objects, we complete + * detaching it from dentry tree. The parent, if any, gets locked + * and its refcount is decremented; dentry is carefully removed from + * the tree (see dentry_unlist() for details) and marked killed + * (%DCACHE_DENTRY_KILLED set in ->d_flags). At that point it's just + * an inert chunk of memory, accessible only via RCU references + * and possibly via a shrink list. If it is not on any shrink lists, + * we call dentry_free(), which schedules actual freeing of memory. + * Othewise freeing is left to the owner of the shrink list in question. + */ +static struct dentry *dentry_kill(struct dentry *dentry) +{ + struct dentry *parent = NULL; + bool can_free = true; + + if (unlikely(!lock_for_kill(dentry))) + return NULL; + + /* + * The dentry is now unrecoverably dead to the world. + */ + lockref_mark_dead(&dentry->d_lockref); + + /* + * inform the fs via d_prune that this dentry is about to be + * unhashed and destroyed. + */ + if (dentry->d_flags & DCACHE_OP_PRUNE) + dentry->d_op->d_prune(dentry); + + if (dentry->d_flags & DCACHE_LRU_LIST) { + if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) + d_lru_del(dentry); + } + /* if it was on the hash then remove it */ + __d_drop(dentry); + if (dentry->d_inode) + dentry_unlink_inode(dentry); + else + spin_unlock(&dentry->d_lock); + this_cpu_dec(nr_dentry); + if (dentry->d_op && dentry->d_op->d_release) + dentry->d_op->d_release(dentry); + + cond_resched(); + /* now that it's negative, ->d_parent is stable */ + if (!IS_ROOT(dentry)) { + parent = dentry->d_parent; + spin_lock(&parent->d_lock); + } + spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); + dentry_unlist(dentry); + if (dentry->d_flags & DCACHE_SHRINK_LIST) + can_free = false; + spin_unlock(&dentry->d_lock); + if (likely(can_free)) + dentry_free(dentry); + if (parent && --parent->d_lockref.count) { + spin_unlock(&parent->d_lock); + return NULL; + } + return parent; +} + +/* + * Decide if dentry is worth retaining. Usually this is called with dentry + * locked; if not locked, we are more limited and might not be able to tell + * without a lock. False in this case means "punt to locked path and recheck". + * + * In case we aren't locked, these predicates are not "stable". However, it is + * sufficient that at some point after we dropped the reference the dentry was + * hashed and the flags had the proper value. Other dentry users may have + * re-gotten a reference to the dentry and change that, but our work is done - + * we can leave the dentry around with a zero refcount. + */ +static inline bool retain_dentry(struct dentry *dentry, bool locked) +{ + unsigned int d_flags; + + smp_rmb(); + d_flags = READ_ONCE(dentry->d_flags); + + // Unreachable? Nobody would be able to look it up, no point retaining + if (unlikely(d_unhashed(dentry))) + return false; + + // Same if it's disconnected + if (unlikely(d_flags & DCACHE_DISCONNECTED)) + return false; + + // ->d_delete() might tell us not to bother, but that requires + // ->d_lock; can't decide without it + if (unlikely(d_flags & DCACHE_OP_DELETE)) { + if (!locked || dentry->d_op->d_delete(dentry)) + return false; + } + + // Explicitly told not to bother + if (unlikely(d_flags & DCACHE_DONTCACHE)) + return false; + + // At this point it looks like we ought to keep it. We also might + // need to do something - put it on LRU if it wasn't there already + // and mark it referenced if it was on LRU, but not marked yet. + // Unfortunately, both actions require ->d_lock, so in lockless + // case we'd have to punt rather than doing those. + if (unlikely(!(d_flags & DCACHE_LRU_LIST))) { + if (!locked) + return false; + d_lru_add(dentry); + } else if (unlikely(!(d_flags & DCACHE_REFERENCED))) { + if (!locked) + return false; + dentry->d_flags |= DCACHE_REFERENCED; + } + return true; +} + +void d_mark_dontcache(struct inode *inode) +{ + struct dentry *de; + + spin_lock(&inode->i_lock); + for_each_alias(de, inode) { + spin_lock(&de->d_lock); + de->d_flags |= DCACHE_DONTCACHE; + spin_unlock(&de->d_lock); + } + inode_state_set(inode, I_DONTCACHE); + spin_unlock(&inode->i_lock); +} +EXPORT_SYMBOL(d_mark_dontcache); + +/* + * Try to do a lockless dput(), and return whether that was successful. + * + * If unsuccessful, we return false, having already taken the dentry lock. + * In that case refcount is guaranteed to be zero and we have already + * decided that it's not worth keeping around. + */ +static inline bool fast_dput(struct dentry *dentry) +{ + int ret; + + /* + * Try to decrement the lockref optimistically. + * RCU read lock held so that dentry is guaranteed to stay around + * even if the refcount goes down to zero. + */ + rcu_read_lock(); + ret = lockref_put_return(&dentry->d_lockref); + + /* + * If the lockref_put_return() failed due to the lock being held + * by somebody else, the fast path has failed. We will need to + * get the lock, and then check the count again. + */ + if (unlikely(ret < 0)) { + spin_lock(&dentry->d_lock); + rcu_read_unlock(); + if (WARN_ON_ONCE(lockref_is_dead_or_zero(&dentry->d_lockref))) { + spin_unlock(&dentry->d_lock); + return true; + } + dentry->d_lockref.count--; + goto locked; + } + + /* + * If we weren't the last ref, we're done. + */ + if (ret) { + rcu_read_unlock(); + return true; + } + + /* + * Can we decide that decrement of refcount is all we needed without + * taking the lock? There's a very common case when it's all we need - + * dentry looks like it ought to be retained and there's nothing else + * to do. + */ + if (retain_dentry(dentry, false)) { + rcu_read_unlock(); + return true; + } + + /* + * Either not worth retaining or we can't tell without the lock. + * Get the lock, then. We've already decremented the refcount to 0, + * but we'll need to re-check the situation after getting the lock. + */ + spin_lock(&dentry->d_lock); + rcu_read_unlock(); + + /* + * Did somebody else grab a reference to it in the meantime, and + * we're no longer the last user after all? Alternatively, somebody + * else could have killed it and marked it dead. Either way, we + * don't need to do anything else. + */ +locked: + if (dentry->d_lockref.count || retain_dentry(dentry, true)) { + spin_unlock(&dentry->d_lock); + return true; + } + return false; +} + +static void finish_dput(struct dentry *dentry) + __releases(dentry->d_lock) +{ + while ((dentry = dentry_kill(dentry)) != NULL) { + if (retain_dentry(dentry, true)) { + spin_unlock(&dentry->d_lock); + return; + } + } +} + +/* + * This is dput + * + * This is complicated by the fact that we do not want to put + * dentries that are no longer on any hash chain on the unused + * list: we'd much rather just get rid of them immediately. + * + * However, that implies that we have to traverse the dentry + * tree upwards to the parents which might _also_ now be + * scheduled for deletion (it may have been only waiting for + * its last child to go away). + * + * This tail recursion is done by hand as we don't want to depend + * on the compiler to always get this right (gcc generally doesn't). + * Real recursion would eat up our stack space. + */ + +/* + * dput - release a dentry + * @dentry: dentry to release + * + * Release a dentry. This will drop the usage count and if appropriate + * call the dentry unlink method as well as removing it from the queues and + * releasing its resources. If the parent dentries were scheduled for release + * they too may now get deleted. + */ +void dput(struct dentry *dentry) +{ + if (!dentry) + return; + might_sleep(); + if (likely(fast_dput(dentry))) + return; + finish_dput(dentry); +} +EXPORT_SYMBOL(dput); + +void d_make_discardable(struct dentry *dentry) +{ + spin_lock(&dentry->d_lock); + WARN_ON(!(dentry->d_flags & DCACHE_PERSISTENT)); + dentry->d_flags &= ~DCACHE_PERSISTENT; + dentry->d_lockref.count--; + finish_dput(dentry); +} +EXPORT_SYMBOL(d_make_discardable); + +/** + * __move_to_shrink_list - try to place a dentry into a shrink list + * @dentry: dentry to try putting into shrink list + * @list: the list to put @dentry into. + * Returns: true @dentry had been placed into @list, false otherwise + * + * If @dentry is idle and not already include into a shrink list, move + * it into @list and return %true; otherwise do nothing and return %false. + * + * Caller must be holding @dentry->d_lock. There must have been no calls of + * dentry_free(@dentry) prior to the beginning of the RCU read-side critical + * area in which __move_to_shrink_list(@dentry, @list) is called. + * + * @list should be thread-private and eventually emptied by passing it to + * shrink_dentry_list(). + */ + +bool __move_to_shrink_list(struct dentry *dentry, struct list_head *list) +__must_hold(&dentry->d_lock) +{ + if (likely(!dentry->d_lockref.count && + !(dentry->d_flags & DCACHE_SHRINK_LIST))) { + if (dentry->d_flags & DCACHE_LRU_LIST) + d_lru_del(dentry); + d_shrink_add(dentry, list); + return true; + } + return false; +} +EXPORT_SYMBOL(__move_to_shrink_list); + +void dput_to_list(struct dentry *dentry, struct list_head *list) +{ + if (likely(fast_dput(dentry))) + return; + __move_to_shrink_list(dentry, list); + spin_unlock(&dentry->d_lock); +} + +struct dentry *dget_parent(struct dentry *dentry) +{ + int gotref; + struct dentry *ret; + unsigned seq; + + /* + * Do optimistic parent lookup without any + * locking. + */ + rcu_read_lock(); + seq = raw_seqcount_begin(&dentry->d_seq); + ret = READ_ONCE(dentry->d_parent); + gotref = lockref_get_not_zero(&ret->d_lockref); + rcu_read_unlock(); + if (likely(gotref)) { + if (!read_seqcount_retry(&dentry->d_seq, seq)) + return ret; + dput(ret); + } + +repeat: + /* + * Don't need rcu_dereference because we re-check it was correct under + * the lock. + */ + rcu_read_lock(); + ret = dentry->d_parent; + spin_lock(&ret->d_lock); + if (unlikely(ret != dentry->d_parent)) { + spin_unlock(&ret->d_lock); + rcu_read_unlock(); + goto repeat; + } + rcu_read_unlock(); + BUG_ON(!ret->d_lockref.count); + ret->d_lockref.count++; + spin_unlock(&ret->d_lock); + return ret; +} +EXPORT_SYMBOL(dget_parent); + +/* + * inode is a directory, inode->i_lock is held by the caller + */ +static struct dentry * __d_find_dir_alias(struct inode *inode) +{ + struct dentry *alias; + + if (hlist_empty(&inode->i_dentry)) + return NULL; + alias = hlist_entry(inode->i_dentry.first, struct dentry, d_alias); + lockref_get(&alias->d_lockref); + return alias; +} + +static struct dentry * __d_find_any_alias(struct inode *inode) +{ + struct dentry *alias; + + if (hlist_empty(&inode->i_dentry)) + return NULL; + for_each_alias(alias, inode) + if (dget_alias_ilocked(alias)) + return alias; + return NULL; +} + +/** + * d_find_any_alias - find any alias for a given inode + * @inode: inode to find an alias for + * + * If any aliases exist for the given inode, take and return a + * reference for one of them. If no aliases exist, return %NULL. + */ +struct dentry *d_find_any_alias(struct inode *inode) +{ + struct dentry *de; + + spin_lock(&inode->i_lock); + de = __d_find_any_alias(inode); + spin_unlock(&inode->i_lock); + return de; +} +EXPORT_SYMBOL(d_find_any_alias); + +static struct dentry *__d_find_alias(struct inode *inode) +{ + struct dentry *alias; + + if (S_ISDIR(inode->i_mode)) + return __d_find_dir_alias(inode); + + for_each_alias(alias, inode) { + spin_lock(&alias->d_lock); + if (!d_unhashed(alias)) { + dget_dlock(alias); + spin_unlock(&alias->d_lock); + return alias; + } + spin_unlock(&alias->d_lock); + } + return NULL; +} + +/** + * d_find_alias - grab a hashed alias of inode + * @inode: inode in question + * + * If inode has a hashed alias, or is a directory and has any alias, + * acquire the reference to alias and return it. Otherwise return NULL. + * Notice that if inode is a directory there can be only one alias and + * it can be unhashed only if it has no children, or if it is the root + * of a filesystem, or if the directory was renamed and d_revalidate + * was the first vfs operation to notice. + * + * If the inode has an IS_ROOT, DCACHE_DISCONNECTED alias, then prefer + * any other hashed alias over that one. + */ +struct dentry *d_find_alias(struct inode *inode) +{ + struct dentry *de = NULL; + + if (!hlist_empty(&inode->i_dentry)) { + spin_lock(&inode->i_lock); + de = __d_find_alias(inode); + spin_unlock(&inode->i_lock); + } + return de; +} +EXPORT_SYMBOL(d_find_alias); + +/* + * Caller MUST be holding rcu_read_lock() and be guaranteed + * that inode won't get freed until rcu_read_unlock(). + */ +struct dentry *d_find_alias_rcu(struct inode *inode) +{ + struct hlist_head *l = &inode->i_dentry; + struct dentry *de = NULL; + + spin_lock(&inode->i_lock); + // ->i_dentry and ->i_rcu are colocated, but the latter won't be + // used without having I_FREEING set, which means no aliases left + if (likely(!(inode_state_read(inode) & I_FREEING) && !hlist_empty(l))) { + if (S_ISDIR(inode->i_mode)) { + de = hlist_entry(l->first, struct dentry, d_alias); + } else { + hlist_for_each_entry(de, l, d_alias) + if (!d_unhashed(de)) + break; + } + } + spin_unlock(&inode->i_lock); + return de; +} + +/* + * Try to kill dentries associated with this inode. + * WARNING: you must own a reference to inode. + */ +void d_prune_aliases(struct inode *inode) +{ + LIST_HEAD(dispose); + struct dentry *dentry; + + spin_lock(&inode->i_lock); + for_each_alias(dentry, inode) { + spin_lock(&dentry->d_lock); + if (likely(!(dentry->d_flags & DCACHE_NORCU))) + __move_to_shrink_list(dentry, &dispose); + spin_unlock(&dentry->d_lock); + } + spin_unlock(&inode->i_lock); + shrink_dentry_list(&dispose); +} +EXPORT_SYMBOL(d_prune_aliases); + +static inline void shrink_kill(struct dentry *victim) +{ + while ((victim = dentry_kill(victim)) != NULL) + ; +} + +void shrink_dentry_list(struct list_head *list) +{ + while (!list_empty(list)) { + struct dentry *dentry; + + dentry = list_entry(list->prev, struct dentry, d_lru); + spin_lock(&dentry->d_lock); + d_shrink_del(dentry); + if (unlikely(dentry->d_flags & DCACHE_DENTRY_KILLED)) { + spin_unlock(&dentry->d_lock); + dentry_free(dentry); + continue; + } + shrink_kill(dentry); + } +} +EXPORT_SYMBOL(shrink_dentry_list); + +static enum lru_status dentry_lru_isolate(struct list_head *item, + struct list_lru_one *lru, void *arg) +{ + struct list_head *freeable = arg; + struct dentry *dentry = container_of(item, struct dentry, d_lru); + + + /* + * we are inverting the lru lock/dentry->d_lock here, + * so use a trylock. If we fail to get the lock, just skip + * it + */ + if (!spin_trylock(&dentry->d_lock)) + return LRU_SKIP; + + /* + * Referenced dentries are still in use. If they have active + * counts, just remove them from the LRU. Otherwise give them + * another pass through the LRU. + */ + if (dentry->d_lockref.count) { + d_lru_isolate(lru, dentry); + spin_unlock(&dentry->d_lock); + return LRU_REMOVED; + } + + if (dentry->d_flags & DCACHE_REFERENCED) { + dentry->d_flags &= ~DCACHE_REFERENCED; + spin_unlock(&dentry->d_lock); + + /* + * The list move itself will be made by the common LRU code. At + * this point, we've dropped the dentry->d_lock but keep the + * lru lock. This is safe to do, since every list movement is + * protected by the lru lock even if both locks are held. + * + * This is guaranteed by the fact that all LRU management + * functions are intermediated by the LRU API calls like + * list_lru_add_obj and list_lru_del_obj. List movement in this file + * only ever occur through this functions or through callbacks + * like this one, that are called from the LRU API. + * + * The only exceptions to this are functions like + * shrink_dentry_list, and code that first checks for the + * DCACHE_SHRINK_LIST flag. Those are guaranteed to be + * operating only with stack provided lists after they are + * properly isolated from the main list. It is thus, always a + * local access. + */ + return LRU_ROTATE; + } + + d_lru_shrink_move(lru, dentry, freeable); + spin_unlock(&dentry->d_lock); + + return LRU_REMOVED; +} + +/** + * prune_dcache_sb - shrink the dcache + * @sb: superblock + * @sc: shrink control, passed to list_lru_shrink_walk() + * + * Attempt to shrink the superblock dcache LRU by @sc->nr_to_scan entries. This + * is done when we need more memory and called from the superblock shrinker + * function. + * + * This function may fail to free any resources if all the dentries are in + * use. + */ +long prune_dcache_sb(struct super_block *sb, struct shrink_control *sc) +{ + LIST_HEAD(dispose); + long freed; + + freed = list_lru_shrink_walk(&sb->s_dentry_lru, sc, + dentry_lru_isolate, &dispose); + shrink_dentry_list(&dispose); + return freed; +} + +static enum lru_status dentry_lru_isolate_shrink(struct list_head *item, + struct list_lru_one *lru, void *arg) +{ + struct list_head *freeable = arg; + struct dentry *dentry = container_of(item, struct dentry, d_lru); + + /* + * we are inverting the lru lock/dentry->d_lock here, + * so use a trylock. If we fail to get the lock, just skip + * it + */ + if (!spin_trylock(&dentry->d_lock)) + return LRU_SKIP; + + d_lru_shrink_move(lru, dentry, freeable); + spin_unlock(&dentry->d_lock); + + return LRU_REMOVED; +} + + +/** + * shrink_dcache_sb - shrink dcache for a superblock + * @sb: superblock + * + * Shrink the dcache for the specified super block. This is used to free + * the dcache before unmounting a file system. + */ +void shrink_dcache_sb(struct super_block *sb) +{ + do { + LIST_HEAD(dispose); + + list_lru_walk(&sb->s_dentry_lru, + dentry_lru_isolate_shrink, &dispose, 1024); + shrink_dentry_list(&dispose); + } while (list_lru_count(&sb->s_dentry_lru) > 0); +} +EXPORT_SYMBOL(shrink_dcache_sb); + +/** + * enum d_walk_ret - action to take during tree walk + * @D_WALK_CONTINUE: continue walk + * @D_WALK_QUIT: quit walk + * @D_WALK_NORETRY: quit when retry is needed + * @D_WALK_SKIP: skip this dentry and its children + */ +enum d_walk_ret { + D_WALK_CONTINUE, + D_WALK_QUIT, + D_WALK_NORETRY, + D_WALK_SKIP, +}; + +/** + * d_walk - walk the dentry tree + * @parent: start of walk + * @data: data passed to @enter() and @finish() + * @enter: callback when first entering the dentry + * + * The @enter() callbacks are called with d_lock held. + */ +static void d_walk(struct dentry *parent, void *data, + enum d_walk_ret (*enter)(void *, struct dentry *)) +{ + struct dentry *this_parent, *dentry; + unsigned seq = 0; + enum d_walk_ret ret; + bool retry = true; + +again: + read_seqbegin_or_lock(&rename_lock, &seq); + this_parent = parent; + spin_lock(&this_parent->d_lock); + if (unlikely(this_parent->d_flags & DCACHE_DENTRY_CURSOR)) + goto out_unlock; + + ret = enter(data, this_parent); + switch (ret) { + case D_WALK_CONTINUE: + break; + case D_WALK_QUIT: + case D_WALK_SKIP: + goto out_unlock; + case D_WALK_NORETRY: + retry = false; + break; + } +repeat: + dentry = d_first_child(this_parent); +resume: + hlist_for_each_entry_from(dentry, d_sib) { + if (unlikely(dentry->d_flags & DCACHE_DENTRY_CURSOR)) + continue; + + spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); + + ret = enter(data, dentry); + switch (ret) { + case D_WALK_CONTINUE: + break; + case D_WALK_QUIT: + spin_unlock(&dentry->d_lock); + goto out_unlock; + case D_WALK_NORETRY: + retry = false; + break; + case D_WALK_SKIP: + spin_unlock(&dentry->d_lock); + continue; + } + + if (!hlist_empty(&dentry->d_children)) { + spin_unlock(&this_parent->d_lock); + spin_release(&dentry->d_lock.dep_map, _RET_IP_); + this_parent = dentry; + spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_); + goto repeat; + } + spin_unlock(&dentry->d_lock); + } + /* + * All done at this level ... ascend and resume the search. + */ +ascend: + if (this_parent != parent) { + dentry = this_parent; + this_parent = dentry->d_parent; + + rcu_read_lock(); + spin_unlock(&dentry->d_lock); + spin_lock(&this_parent->d_lock); + rcu_read_unlock(); + + /* might go back up the wrong parent if we have had a rename. */ + if (need_seqretry(&rename_lock, seq)) + goto rename_retry; + /* go into the first sibling still alive */ + hlist_for_each_entry_continue(dentry, d_sib) { + if (likely(!(dentry->d_flags & DCACHE_DENTRY_KILLED))) { + goto resume; + } + } + goto ascend; + } + if (need_seqretry(&rename_lock, seq)) + goto rename_retry; + +out_unlock: + spin_unlock(&this_parent->d_lock); + done_seqretry(&rename_lock, seq); + return; + +rename_retry: + spin_unlock(&this_parent->d_lock); + BUG_ON(seq & 1); + if (!retry) + return; + seq = 1; + goto again; +} + +struct check_mount { + struct vfsmount *mnt; + unsigned int mounted; +}; + +/* locks: mount_locked_reader && dentry->d_lock */ +static enum d_walk_ret path_check_mount(void *data, struct dentry *dentry) +{ + struct check_mount *info = data; + struct path path = { .mnt = info->mnt, .dentry = dentry }; + + if (likely(!d_mountpoint(dentry))) + return D_WALK_CONTINUE; + if (__path_is_mountpoint(&path)) { + info->mounted = 1; + return D_WALK_QUIT; + } + return D_WALK_CONTINUE; +} + +/** + * path_has_submounts - check for mounts over a dentry in the + * current namespace. + * @parent: path to check. + * + * Return true if the parent or its subdirectories contain + * a mount point in the current namespace. + */ +int path_has_submounts(const struct path *parent) +{ + struct check_mount data = { .mnt = parent->mnt, .mounted = 0 }; + + guard(mount_locked_reader)(); + d_walk(parent->dentry, &data, path_check_mount); + + return data.mounted; +} +EXPORT_SYMBOL(path_has_submounts); + +/* + * Called by mount code to set a mountpoint and check if the mountpoint is + * reachable (e.g. NFS can unhash a directory dentry and then the complete + * subtree can become unreachable). + * + * Only one of d_invalidate() and d_set_mounted() must succeed. For + * this reason take rename_lock and d_lock on dentry and ancestors. + */ +int d_set_mounted(struct dentry *dentry) +{ + struct dentry *p; + int ret = -ENOENT; + read_seqlock_excl(&rename_lock); + for (p = dentry->d_parent; !IS_ROOT(p); p = p->d_parent) { + /* Need exclusion wrt. d_invalidate() */ + spin_lock(&p->d_lock); + if (unlikely(d_unhashed(p))) { + spin_unlock(&p->d_lock); + goto out; + } + spin_unlock(&p->d_lock); + } + spin_lock(&dentry->d_lock); + if (!d_unlinked(dentry)) { + ret = -EBUSY; + if (!d_mountpoint(dentry)) { + dentry->d_flags |= DCACHE_MOUNTED; + ret = 0; + } + } + spin_unlock(&dentry->d_lock); +out: + read_sequnlock_excl(&rename_lock); + return ret; +} + +/* + * Search the dentry child list of the specified parent, + * and move any unused dentries to the end of the unused + * list for prune_dcache(). We descend to the next level + * whenever the d_children list is non-empty and continue + * searching. + * + * It returns zero iff there are no unused children, + * otherwise it returns the number of children moved to + * the end of the unused list. This may not be the total + * number of unused children, because select_parent can + * drop the lock and return early due to latency + * constraints. + */ + +struct select_data { + struct dentry *start; + union { + long found; + struct dentry *victim; + }; + struct list_head dispose; +}; + +static enum d_walk_ret select_collect(void *_data, struct dentry *dentry) +{ + struct select_data *data = _data; + enum d_walk_ret ret = D_WALK_CONTINUE; + + if (data->start == dentry) + goto out; + + if (lockref_is_dead_or_zero(&dentry->d_lockref)) { + __move_to_shrink_list(dentry, &data->dispose); + data->found++; + } + /* + * We can return to the caller if we have found some (this + * ensures forward progress). We'll be coming back to find + * the rest. + */ + if (!list_empty(&data->dispose)) + ret = need_resched() ? D_WALK_QUIT : D_WALK_NORETRY; +out: + return ret; +} + +static enum d_walk_ret select_collect_umount(void *_data, struct dentry *dentry) +{ + if (dentry->d_flags & DCACHE_PERSISTENT) { + dentry->d_flags &= ~DCACHE_PERSISTENT; + dentry->d_lockref.count--; + } + return select_collect(_data, dentry); +} + +static enum d_walk_ret select_collect2(void *_data, struct dentry *dentry) +{ + struct select_data *data = _data; + enum d_walk_ret ret = D_WALK_CONTINUE; + + if (data->start == dentry) + goto out; + + if (lockref_is_dead_or_zero(&dentry->d_lockref)) { + if (!__move_to_shrink_list(dentry, &data->dispose)) { + /* + * We need an enter RCU read-side critical area that + * would extend past the return from d_walk() and + * we are in the scope of ->d_lock that will terminate + * before that, so we use rcu_read_lock() to bridge + * over to the scope of ->d_lock in d_walk() caller. + * The scope of rcu_read_lock() spans from here to + * paired rcu_read_unlock() in shrink_dcache_tree(). + */ + rcu_read_lock(); + data->victim = dentry; + return D_WALK_QUIT; + } + } + /* + * We can return to the caller if we have found some (this + * ensures forward progress). We'll be coming back to find + * the rest. + */ + if (!list_empty(&data->dispose)) + ret = need_resched() ? D_WALK_QUIT : D_WALK_NORETRY; +out: + return ret; +} + +/** + * shrink_dcache_tree - prune dcache + * @parent: parent of entries to prune + * @for_umount: true if we want to unpin the persistent ones + * + * Prune the dcache to remove unused children of the parent dentry. + */ +static void shrink_dcache_tree(struct dentry *parent, bool for_umount) +{ + for (;;) { + struct completion_list wait; + bool need_wait = false; + struct select_data data = { .start = parent }; + + INIT_LIST_HEAD(&data.dispose); + d_walk(parent, &data, + for_umount ? select_collect_umount : select_collect); + + if (!list_empty(&data.dispose)) { + shrink_dentry_list(&data.dispose); + continue; + } + + cond_resched(); + if (!data.found) + break; + data.victim = NULL; + d_walk(parent, &data, select_collect2); + if (data.victim) { + struct dentry *v = data.victim; + /* + * select_collect2() has picked a dentry that was + * either dying or on a shrink list and arranged + * for it to be returned to us. We are still in + * the RCU read-side critical area started there + * (rcu_read_lock() scope opened in select_collect2()), + * so dentry couldn't have been freed yet, but its + * state might've changed since we dropped ->d_lock + * on the way out. Switch over to ->d_lock scope + * and recheck the dentry state. + */ + spin_lock(&v->d_lock); + rcu_read_unlock(); + + if (unlikely(lockref_is_dead(&v->d_lockref))) { + // It's doomed; if it isn't dead yet, notify us + // once it becomes invisible to d_walk(). + need_wait = d_add_waiter(v, &wait); + spin_unlock(&v->d_lock); + } else { + shrink_kill(v); + } + } + shrink_dentry_list(&data.dispose); + if (unlikely(need_wait)) + wait_for_completion(&wait.completion); + } +} + +void shrink_dcache_parent(struct dentry *parent) +{ + shrink_dcache_tree(parent, false); +} +EXPORT_SYMBOL(shrink_dcache_parent); + +static enum d_walk_ret umount_check(void *_data, struct dentry *dentry) +{ + /* it has busy descendents; complain about those instead */ + if (!hlist_empty(&dentry->d_children)) + return D_WALK_CONTINUE; + + /* root with refcount 1 is fine */ + if (dentry == _data && dentry->d_lockref.count == 1) + return D_WALK_CONTINUE; + + WARN(1, "BUG: Dentry %p{i=%llx,n=%pd} " + " still in use (%d) [unmount of %s %s]\n", + dentry, + dentry->d_inode ? + dentry->d_inode->i_ino : (u64)0, + dentry, + dentry->d_lockref.count, + dentry->d_sb->s_type->name, + dentry->d_sb->s_id); + return D_WALK_CONTINUE; +} + +static void do_one_tree(struct dentry *dentry) +{ + shrink_dcache_tree(dentry, true); + d_walk(dentry, dentry, umount_check); + spin_lock(&dentry->d_lock); + __d_drop(dentry); + /* A busy root survives the dput() below so don't leave it on ->s_roots. */ + if (unlikely(!hlist_unhashed(&dentry->d_sib))) + unlink_secondary_root(dentry); + spin_unlock(&dentry->d_lock); + dput(dentry); +} + +/* + * destroy the dentries attached to a superblock on unmounting + */ +void shrink_dcache_for_umount(struct super_block *sb) +{ + struct dentry *dentry; + + rwsem_assert_held_write(&sb->s_umount); + + dentry = sb->s_root; + sb->s_root = NULL; + do_one_tree(dentry); + + for (;;) { + spin_lock(&sb->s_roots_lock); + dentry = hlist_entry_safe(sb->s_roots.first, + struct dentry, d_sib); + if (!dentry) { + spin_unlock(&sb->s_roots_lock); + break; + } + rcu_read_lock(); + spin_unlock(&sb->s_roots_lock); + spin_lock(&dentry->d_lock); + rcu_read_unlock(); + if (unlikely(lockref_is_dead(&dentry->d_lockref))) { + struct completion_list wait; + bool need_wait = d_add_waiter(dentry, &wait); + + spin_unlock(&dentry->d_lock); + if (need_wait) + wait_for_completion(&wait.completion); + } else { + dget_dlock(dentry); + spin_unlock(&dentry->d_lock); + do_one_tree(dentry); + } + } +} + +static enum d_walk_ret find_submount(void *_data, struct dentry *dentry) +{ + struct dentry **victim = _data; + if (d_mountpoint(dentry)) { + *victim = dget_dlock(dentry); + return D_WALK_QUIT; + } + return D_WALK_CONTINUE; +} + +/** + * d_invalidate - detach submounts, prune dcache, and drop + * @dentry: dentry to invalidate (aka detach, prune and drop) + */ +void d_invalidate(struct dentry *dentry) +{ + bool had_submounts = false; + spin_lock(&dentry->d_lock); + if (d_unhashed(dentry)) { + spin_unlock(&dentry->d_lock); + return; + } + __d_drop(dentry); + spin_unlock(&dentry->d_lock); + + /* Negative dentries can be dropped without further checks */ + if (!dentry->d_inode) + return; + + shrink_dcache_parent(dentry); + for (;;) { + struct dentry *victim = NULL; + d_walk(dentry, &victim, find_submount); + if (!victim) { + if (had_submounts) + shrink_dcache_parent(dentry); + return; + } + had_submounts = true; + detach_mounts(victim); + dput(victim); + } +} +EXPORT_SYMBOL(d_invalidate); + +/** + * __d_alloc - allocate a dcache entry + * @sb: filesystem it will belong to + * @name: qstr of the name + * + * Allocates a dentry. It returns %NULL if there is insufficient memory + * available. On a success the dentry is returned. The name passed in is + * copied and the copy passed in may be reused after this call. + */ + +static struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name) +{ + struct dentry *dentry; + char *dname; + int err; + + dentry = kmem_cache_alloc_lru(dentry_cache, &sb->s_dentry_lru, + GFP_KERNEL); + if (!dentry) + return NULL; + + /* + * We guarantee that the inline name is always NUL-terminated. + * This way the memcpy() done by the name switching in rename + * will still always have a NUL at the end, even if we might + * be overwriting an internal NUL character + */ + dentry->d_shortname.string[DNAME_INLINE_LEN-1] = 0; + if (unlikely(!name)) { + name = &slash_name; + dname = dentry->d_shortname.string; + } else if (name->len > DNAME_INLINE_LEN-1) { + struct external_name *p; + + p = kmalloc_flex(*p, name, name->len + 1, + GFP_KERNEL_ACCOUNT | __GFP_RECLAIMABLE); + if (!p) { + kmem_cache_free(dentry_cache, dentry); + return NULL; + } + atomic_set(&p->count, 1); + dname = p->name; + } else { + dname = dentry->d_shortname.string; + } + + dentry->__d_name.len = name->len; + dentry->__d_name.hash = name->hash; + memcpy(dname, name->name, name->len); + dname[name->len] = 0; + + /* Make sure we always see the terminating NUL character */ + smp_store_release(&dentry->__d_name.name, dname); /* ^^^ */ + + dentry->d_flags = 0; + lockref_init(&dentry->d_lockref); + seqcount_spinlock_init(&dentry->d_seq, &dentry->d_lock); + dentry->d_inode = NULL; + dentry->d_parent = dentry; + dentry->d_sb = sb; + dentry->d_op = sb->__s_d_op; + dentry->d_flags = sb->s_d_flags; + dentry->d_fsdata = NULL; + INIT_HLIST_BL_NODE(&dentry->d_hash); + INIT_LIST_HEAD(&dentry->d_lru); + INIT_HLIST_HEAD(&dentry->d_children); + dentry->waiters = NULL; + INIT_HLIST_NODE(&dentry->d_sib); + + if (dentry->d_op && dentry->d_op->d_init) { + err = dentry->d_op->d_init(dentry); + if (err) { + if (dname_external(dentry)) + kfree(external_name(dentry)); + kmem_cache_free(dentry_cache, dentry); + return NULL; + } + } + + this_cpu_inc(nr_dentry); + + return dentry; +} + +/** + * d_alloc - allocate a dcache entry + * @parent: parent of entry to allocate + * @name: qstr of the name + * + * Allocates a dentry. It returns %NULL if there is insufficient memory + * available. On a success the dentry is returned. The name passed in is + * copied and the copy passed in may be reused after this call. + */ +struct dentry *d_alloc(struct dentry * parent, const struct qstr *name) +{ + struct dentry *dentry = __d_alloc(parent->d_sb, name); + if (!dentry) + return NULL; + spin_lock(&parent->d_lock); + /* + * don't need child lock because it is not subject + * to concurrency here + */ + dentry->d_parent = dget_dlock(parent); + hlist_add_head(&dentry->d_sib, &parent->d_children); + spin_unlock(&parent->d_lock); + + return dentry; +} +EXPORT_SYMBOL(d_alloc); + +struct dentry *d_alloc_anon(struct super_block *sb) +{ + return __d_alloc(sb, NULL); +} +EXPORT_SYMBOL(d_alloc_anon); + +struct dentry *d_alloc_cursor(struct dentry * parent) +{ + struct dentry *dentry = d_alloc_anon(parent->d_sb); + if (dentry) { + dentry->d_flags |= DCACHE_DENTRY_CURSOR | DCACHE_NORCU; + dentry->d_parent = dget(parent); + } + return dentry; +} + +/** + * d_alloc_pseudo - allocate a dentry (for lookup-less filesystems) + * @sb: the superblock + * @name: qstr of the name + * + * For a filesystem that just pins its dentries in memory and never + * performs lookups at all, return an unhashed IS_ROOT dentry. + * This is used for pipes, sockets et.al. - the stuff that should + * never be anyone's children or parents. Unlike all other + * dentries, these will not have RCU delay between dropping the + * last reference and freeing them. + * + * The only user is alloc_file_pseudo() and that's what should + * be considered a public interface. Don't use directly. + */ +struct dentry *d_alloc_pseudo(struct super_block *sb, const struct qstr *name) +{ + static const struct dentry_operations anon_ops = { + .d_dname = simple_dname + }; + struct dentry *dentry = __d_alloc(sb, name); + if (likely(dentry)) { + dentry->d_flags |= DCACHE_NORCU; + /* d_op_flags(&anon_ops) is 0 */ + if (!dentry->d_op) + dentry->d_op = &anon_ops; + } + return dentry; +} + +struct dentry *d_alloc_name(struct dentry *parent, const char *name) +{ + struct qstr q; + + q.name = name; + q.hash_len = hashlen_string(parent, name); + return d_alloc(parent, &q); +} +EXPORT_SYMBOL(d_alloc_name); + +#define DCACHE_OP_FLAGS \ + (DCACHE_OP_HASH | DCACHE_OP_COMPARE | DCACHE_OP_REVALIDATE | \ + DCACHE_OP_WEAK_REVALIDATE | DCACHE_OP_DELETE | DCACHE_OP_PRUNE | \ + DCACHE_OP_REAL) + +static unsigned int d_op_flags(const struct dentry_operations *op) +{ + unsigned int flags = 0; + if (op) { + if (op->d_hash) + flags |= DCACHE_OP_HASH; + if (op->d_compare) + flags |= DCACHE_OP_COMPARE; + if (op->d_revalidate) + flags |= DCACHE_OP_REVALIDATE; + if (op->d_weak_revalidate) + flags |= DCACHE_OP_WEAK_REVALIDATE; + if (op->d_delete) + flags |= DCACHE_OP_DELETE; + if (op->d_prune) + flags |= DCACHE_OP_PRUNE; + if (op->d_real) + flags |= DCACHE_OP_REAL; + } + return flags; +} + +static void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op) +{ + unsigned int flags = d_op_flags(op); + WARN_ON_ONCE(dentry->d_op); + WARN_ON_ONCE(dentry->d_flags & DCACHE_OP_FLAGS); + dentry->d_op = op; + if (flags) + dentry->d_flags |= flags; +} + +void set_default_d_op(struct super_block *s, const struct dentry_operations *ops) +{ + unsigned int flags = d_op_flags(ops); + s->__s_d_op = ops; + s->s_d_flags = (s->s_d_flags & ~DCACHE_OP_FLAGS) | flags; +} +EXPORT_SYMBOL(set_default_d_op); + +static unsigned d_flags_for_inode(struct inode *inode) +{ + unsigned add_flags = DCACHE_REGULAR_TYPE; + + if (!inode) + return DCACHE_MISS_TYPE; + + if (S_ISDIR(inode->i_mode)) { + add_flags = DCACHE_DIRECTORY_TYPE; + if (unlikely(!(inode->i_opflags & IOP_LOOKUP))) { + if (unlikely(!inode->i_op->lookup)) + add_flags = DCACHE_AUTODIR_TYPE; + else + inode->i_opflags |= IOP_LOOKUP; + } + goto type_determined; + } + + if (unlikely(!(inode->i_opflags & IOP_NOFOLLOW))) { + if (unlikely(inode->i_op->get_link)) { + add_flags = DCACHE_SYMLINK_TYPE; + goto type_determined; + } + inode->i_opflags |= IOP_NOFOLLOW; + } + + if (unlikely(!S_ISREG(inode->i_mode))) + add_flags = DCACHE_SPECIAL_TYPE; + +type_determined: + if (unlikely(IS_AUTOMOUNT(inode))) + add_flags |= DCACHE_NEED_AUTOMOUNT; + return add_flags; +} + +static void __d_instantiate(struct dentry *dentry, struct inode *inode) +{ + unsigned add_flags = d_flags_for_inode(inode); + WARN_ON(d_in_lookup(dentry)); + + /* + * The negative counter only tracks dentries on the LRU. Don't dec if + * d_lru is on another list. + */ + if ((dentry->d_flags & + (DCACHE_LRU_LIST|DCACHE_SHRINK_LIST)) == DCACHE_LRU_LIST) + this_cpu_dec(nr_dentry_negative); + hlist_add_head(&dentry->d_alias, &inode->i_dentry); + raw_write_seqcount_begin(&dentry->d_seq); + __d_set_inode_and_type(dentry, inode, add_flags); + raw_write_seqcount_end(&dentry->d_seq); + fsnotify_update_flags(dentry); +} + +/** + * d_instantiate - fill in inode information for a dentry + * @entry: dentry to complete + * @inode: inode to attach to this dentry + * + * Fill in inode information in the entry. + * + * This turns negative dentries into productive full members + * of society. + * + * NOTE! This assumes that the inode count has been incremented + * (or otherwise set) by the caller to indicate that it is now + * in use by the dcache. + */ + +void d_instantiate(struct dentry *entry, struct inode * inode) +{ + BUG_ON(d_really_is_positive(entry)); + if (inode) { + security_d_instantiate(entry, inode); + spin_lock(&inode->i_lock); + spin_lock(&entry->d_lock); + __d_instantiate(entry, inode); + spin_unlock(&entry->d_lock); + spin_unlock(&inode->i_lock); + } +} +EXPORT_SYMBOL(d_instantiate); + +/* + * This should be equivalent to d_instantiate() + unlock_new_inode(), + * with lockdep-related part of unlock_new_inode() done before + * anything else. Use that instead of open-coding d_instantiate()/ + * unlock_new_inode() combinations. + */ +void d_instantiate_new(struct dentry *entry, struct inode *inode) +{ + BUG_ON(d_really_is_positive(entry)); + BUG_ON(!inode); + lockdep_annotate_inode_mutex_key(inode); + security_d_instantiate(entry, inode); + spin_lock(&inode->i_lock); + spin_lock(&entry->d_lock); + __d_instantiate(entry, inode); + spin_unlock(&entry->d_lock); + WARN_ON(!(inode_state_read(inode) & I_NEW)); + /* + * Paired with igrab_from_hash() + */ + smp_wmb(); + inode_state_clear(inode, I_NEW | I_CREATING); + inode_wake_up_bit(inode, __I_NEW); + spin_unlock(&inode->i_lock); +} +EXPORT_SYMBOL(d_instantiate_new); + +struct dentry *d_make_root(struct inode *root_inode) +{ + struct dentry *res = NULL; + + if (root_inode) { + res = d_alloc_anon(root_inode->i_sb); + if (res) + d_instantiate(res, root_inode); + else + iput(root_inode); + } + return res; +} +EXPORT_SYMBOL(d_make_root); + +static struct dentry *__d_obtain_alias(struct inode *inode, bool disconnected) +{ + struct super_block *sb; + struct dentry *new, *res; + + if (!inode) + return ERR_PTR(-ESTALE); + if (IS_ERR(inode)) + return ERR_CAST(inode); + + sb = inode->i_sb; + + res = d_find_any_alias(inode); /* existing alias? */ + if (res) + goto out; + + new = d_alloc_anon(sb); + if (!new) { + res = ERR_PTR(-ENOMEM); + goto out; + } + + security_d_instantiate(new, inode); + spin_lock(&inode->i_lock); + res = __d_find_any_alias(inode); /* recheck under lock */ + if (likely(!res)) { /* still no alias, attach a disconnected dentry */ + unsigned add_flags = d_flags_for_inode(inode); + + if (disconnected) + add_flags |= DCACHE_DISCONNECTED; + + spin_lock(&new->d_lock); + __d_set_inode_and_type(new, inode, add_flags); + hlist_add_head(&new->d_alias, &inode->i_dentry); + if (!disconnected) { + spin_lock(&sb->s_roots_lock); + hlist_add_head(&new->d_sib, &sb->s_roots); + spin_unlock(&sb->s_roots_lock); + } + spin_unlock(&new->d_lock); + spin_unlock(&inode->i_lock); + inode = NULL; /* consumed by new->d_inode */ + res = new; + } else { + spin_unlock(&inode->i_lock); + dput(new); + } + + out: + iput(inode); + return res; +} + +/** + * d_obtain_alias - find or allocate a DISCONNECTED dentry for a given inode + * @inode: inode to allocate the dentry for + * + * Obtain a dentry for an inode resulting from NFS filehandle conversion or + * similar open by handle operations. The returned dentry may be anonymous, + * or may have a full name (if the inode was already in the cache). + * + * When called on a directory inode, we must ensure that the inode only ever + * has one dentry. If a dentry is found, that is returned instead of + * allocating a new one. + * + * On successful return, the reference to the inode has been transferred + * to the dentry. In case of an error the reference on the inode is released. + * To make it easier to use in export operations a %NULL or IS_ERR inode may + * be passed in and the error will be propagated to the return value, + * with a %NULL @inode replaced by ERR_PTR(-ESTALE). + */ +struct dentry *d_obtain_alias(struct inode *inode) +{ + return __d_obtain_alias(inode, true); +} +EXPORT_SYMBOL(d_obtain_alias); + +/** + * d_obtain_root - find or allocate a dentry for a given inode + * @inode: inode to allocate the dentry for + * + * Obtain an IS_ROOT dentry for the root of a filesystem. + * + * We must ensure that directory inodes only ever have one dentry. If a + * dentry is found, that is returned instead of allocating a new one. + * + * On successful return, the reference to the inode has been transferred + * to the dentry. In case of an error the reference on the inode is + * released. A %NULL or IS_ERR inode may be passed in and will be the + * error will be propagate to the return value, with a %NULL @inode + * replaced by ERR_PTR(-ESTALE). + */ +struct dentry *d_obtain_root(struct inode *inode) +{ + return __d_obtain_alias(inode, false); +} +EXPORT_SYMBOL(d_obtain_root); + +/** + * d_add_ci - lookup or allocate new dentry with case-exact name + * @dentry: the negative dentry that was passed to the parent's lookup func + * @inode: the inode case-insensitive lookup has found + * @name: the case-exact name to be associated with the returned dentry + * + * This is to avoid filling the dcache with case-insensitive names to the + * same inode, only the actual correct case is stored in the dcache for + * case-insensitive filesystems. + * + * For a case-insensitive lookup match and if the case-exact dentry + * already exists in the dcache, use it and return it. + * + * If no entry exists with the exact case name, allocate new dentry with + * the exact case, and return the spliced entry. + */ +struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode, + struct qstr *name) +{ + struct dentry *found, *res; + + /* + * First check if a dentry matching the name already exists, + * if not go ahead and create it now. + */ + found = d_hash_and_lookup(dentry->d_parent, name); + if (found) { + iput(inode); + return found; + } + if (d_in_lookup(dentry)) { + found = d_alloc_parallel(dentry->d_parent, name); + if (IS_ERR(found) || !d_in_lookup(found)) { + iput(inode); + return found; + } + } else { + found = d_alloc(dentry->d_parent, name); + if (!found) { + iput(inode); + return ERR_PTR(-ENOMEM); + } + } + res = d_splice_alias(inode, found); + if (res) { + d_lookup_done(found); + dput(found); + return res; + } + return found; +} +EXPORT_SYMBOL(d_add_ci); + +/** + * d_same_name - compare dentry name with case-exact name + * @dentry: the negative dentry that was passed to the parent's lookup func + * @parent: parent dentry + * @name: the case-exact name to be associated with the returned dentry + * + * Return: true if names are same, or false + */ +bool d_same_name(const struct dentry *dentry, const struct dentry *parent, + const struct qstr *name) +{ + if (likely(!(parent->d_flags & DCACHE_OP_COMPARE))) { + if (dentry->d_name.len != name->len) + return false; + return dentry_cmp(dentry, name->name, name->len) == 0; + } + return parent->d_op->d_compare(dentry, + dentry->d_name.len, dentry->d_name.name, + name) == 0; +} +EXPORT_SYMBOL_GPL(d_same_name); + +/* + * This is __d_lookup_rcu() when the parent dentry has + * DCACHE_OP_COMPARE, which makes things much nastier. + */ +static noinline struct dentry *__d_lookup_rcu_op_compare( + const struct dentry *parent, + const struct qstr *name, + unsigned *seqp) +{ + u64 hashlen = name->hash_len; + struct hlist_bl_head *b = d_hash(hashlen); + struct hlist_bl_node *node; + struct dentry *dentry; + + hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { + int tlen; + const char *tname; + unsigned seq; + +seqretry: + seq = raw_seqcount_begin(&dentry->d_seq); + if (dentry->d_parent != parent) + continue; + if (d_unhashed(dentry)) + continue; + if (dentry->d_name.hash != hashlen_hash(hashlen)) + continue; + tlen = dentry->d_name.len; + tname = dentry->d_name.name; + /* we want a consistent (name,len) pair */ + if (read_seqcount_retry(&dentry->d_seq, seq)) { + cpu_relax(); + goto seqretry; + } + if (parent->d_op->d_compare(dentry, tlen, tname, name) != 0) + continue; + *seqp = seq; + return dentry; + } + return NULL; +} + +/** + * __d_lookup_rcu - search for a dentry (racy, store-free) + * @parent: parent dentry + * @name: qstr of name we wish to find + * @seqp: returns d_seq value at the point where the dentry was found + * Returns: dentry, or NULL + * + * __d_lookup_rcu is the dcache lookup function for rcu-walk name + * resolution (store-free path walking) design described in + * Documentation/filesystems/path-lookup.txt. + * + * This is not to be used outside core vfs. + * + * __d_lookup_rcu must only be used in rcu-walk mode, ie. with vfsmount lock + * held, and rcu_read_lock held. The returned dentry must not be stored into + * without taking d_lock and checking d_seq sequence count against @seq + * returned here. + * + * Alternatively, __d_lookup_rcu may be called again to look up the child of + * the returned dentry, so long as its parent's seqlock is checked after the + * child is looked up. Thus, an interlocking stepping of sequence lock checks + * is formed, giving integrity down the path walk. + * + * NOTE! The caller *has* to check the resulting dentry against the sequence + * number we've returned before using any of the resulting dentry state! + */ +struct dentry *__d_lookup_rcu(const struct dentry *parent, + const struct qstr *name, + unsigned *seqp) +{ + u64 hashlen = name->hash_len; + const unsigned char *str = name->name; + struct hlist_bl_head *b = d_hash(hashlen); + struct hlist_bl_node *node; + struct dentry *dentry; + + /* + * Note: There is significant duplication with __d_lookup_rcu which is + * required to prevent single threaded performance regressions + * especially on architectures where smp_rmb (in seqcounts) are costly. + * Keep the two functions in sync. + */ + + if (unlikely(parent->d_flags & DCACHE_OP_COMPARE)) + return __d_lookup_rcu_op_compare(parent, name, seqp); + + /* + * The hash list is protected using RCU. + * + * Carefully use d_seq when comparing a candidate dentry, to avoid + * races with d_move(). + * + * It is possible that concurrent renames can mess up our list + * walk here and result in missing our dentry, resulting in the + * false-negative result. d_lookup() protects against concurrent + * renames using rename_lock seqlock. + * + * See Documentation/filesystems/path-lookup.txt for more details. + */ + hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { + unsigned seq; + + /* + * The dentry sequence count protects us from concurrent + * renames, and thus protects parent and name fields. + * + * The caller must perform a seqcount check in order + * to do anything useful with the returned dentry. + * + * NOTE! We do a "raw" seqcount_begin here. That means that + * we don't wait for the sequence count to stabilize if it + * is in the middle of a sequence change. If we do the slow + * dentry compare, we will do seqretries until it is stable, + * and if we end up with a successful lookup, we actually + * want to exit RCU lookup anyway. + * + * Note that raw_seqcount_begin still *does* smp_rmb(), so + * we are still guaranteed NUL-termination of ->d_name.name. + */ + seq = raw_seqcount_begin(&dentry->d_seq); + if (dentry->d_parent != parent) + continue; + if (dentry->d_name.hash_len != hashlen) + continue; + if (unlikely(dentry_cmp(dentry, str, hashlen_len(hashlen)) != 0)) + continue; + /* + * Check for the dentry being unhashed. + * + * As tempting as it is, we *can't* skip it because of a race window + * between us finding the dentry before it gets unhashed and loading + * the sequence counter after unhashing is finished. + * + * We can at least predict on it. + */ + if (unlikely(d_unhashed(dentry))) + continue; + *seqp = seq; + return dentry; + } + return NULL; +} + +/** + * d_lookup - search for a dentry + * @parent: parent dentry + * @name: qstr of name we wish to find + * Returns: dentry, or NULL + * + * d_lookup searches the children of the parent dentry for the name in + * question. If the dentry is found its reference count is incremented and the + * dentry is returned. The caller must use dput to free the entry when it has + * finished using it. %NULL is returned if the dentry does not exist. + */ +struct dentry *d_lookup(const struct dentry *parent, const struct qstr *name) +{ + struct dentry *dentry; + unsigned seq; + + do { + seq = read_seqbegin(&rename_lock); + dentry = __d_lookup(parent, name); + if (dentry) + break; + } while (read_seqretry(&rename_lock, seq)); + return dentry; +} +EXPORT_SYMBOL(d_lookup); + +/** + * __d_lookup - search for a dentry (racy) + * @parent: parent dentry + * @name: qstr of name we wish to find + * Returns: dentry, or NULL + * + * __d_lookup is like d_lookup, however it may (rarely) return a + * false-negative result due to unrelated rename activity. + * + * __d_lookup is slightly faster by avoiding rename_lock read seqlock, + * however it must be used carefully, eg. with a following d_lookup in + * the case of failure. + * + * __d_lookup callers must be commented. + */ +struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name) +{ + unsigned int hash = name->hash; + struct hlist_bl_head *b = d_hash(hash); + struct hlist_bl_node *node; + struct dentry *found = NULL; + struct dentry *dentry; + + /* + * Note: There is significant duplication with __d_lookup_rcu which is + * required to prevent single threaded performance regressions + * especially on architectures where smp_rmb (in seqcounts) are costly. + * Keep the two functions in sync. + */ + + /* + * The hash list is protected using RCU. + * + * Take d_lock when comparing a candidate dentry, to avoid races + * with d_move(). + * + * It is possible that concurrent renames can mess up our list + * walk here and result in missing our dentry, resulting in the + * false-negative result. d_lookup() protects against concurrent + * renames using rename_lock seqlock. + * + * See Documentation/filesystems/path-lookup.txt for more details. + */ + rcu_read_lock(); + + hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { + + if (dentry->d_name.hash != hash) + continue; + + spin_lock(&dentry->d_lock); + if (dentry->d_parent != parent) + goto next; + if (d_unhashed(dentry)) + goto next; + + if (!d_same_name(dentry, parent, name)) + goto next; + + dentry->d_lockref.count++; + found = dentry; + spin_unlock(&dentry->d_lock); + break; +next: + spin_unlock(&dentry->d_lock); + } + rcu_read_unlock(); + + return found; +} + +/** + * d_hash_and_lookup - hash the qstr then search for a dentry + * @dir: Directory to search in + * @name: qstr of name we wish to find + * + * On lookup failure NULL is returned; on bad name - ERR_PTR(-error) + */ +struct dentry *d_hash_and_lookup(struct dentry *dir, struct qstr *name) +{ + /* + * Check for a fs-specific hash function. Note that we must + * calculate the standard hash first, as the d_op->d_hash() + * routine may choose to leave the hash value unchanged. + */ + name->hash = full_name_hash(dir, name->name, name->len); + if (dir->d_flags & DCACHE_OP_HASH) { + int err = dir->d_op->d_hash(dir, name); + if (unlikely(err < 0)) + return ERR_PTR(err); + } + return d_lookup(dir, name); +} + +/* + * When a file is deleted, we have two options: + * - turn this dentry into a negative dentry + * - unhash this dentry and free it. + * + * Usually, we want to just turn this into + * a negative dentry, but if anybody else is + * currently using the dentry or the inode + * we can't do that and we fall back on removing + * it from the hash queues and waiting for + * it to be deleted later when it has no users + */ + +/** + * d_delete - delete a dentry + * @dentry: The dentry to delete + * + * Turn the dentry into a negative dentry if possible, otherwise + * remove it from the hash queues so it can be deleted later + */ + +void d_delete(struct dentry * dentry) +{ + struct inode *inode = dentry->d_inode; + + spin_lock(&inode->i_lock); + spin_lock(&dentry->d_lock); + /* + * Are we the only user? + */ + if (dentry->d_lockref.count == 1) { + if (dentry_negative_policy) + __d_drop(dentry); + dentry->d_flags &= ~DCACHE_CANT_MOUNT; + dentry_unlink_inode(dentry); + } else { + __d_drop(dentry); + spin_unlock(&dentry->d_lock); + spin_unlock(&inode->i_lock); + } +} +EXPORT_SYMBOL(d_delete); + +static void __d_rehash(struct dentry *entry) +{ + struct hlist_bl_head *b = d_hash(entry->d_name.hash); + + hlist_bl_lock(b); + hlist_bl_add_head_rcu(&entry->d_hash, b); + hlist_bl_unlock(b); +} + +/** + * d_rehash - add an entry back to the hash + * @entry: dentry to add to the hash + * + * Adds a dentry to the hash according to its name. + */ + +void d_rehash(struct dentry * entry) +{ + spin_lock(&entry->d_lock); + __d_rehash(entry); + spin_unlock(&entry->d_lock); +} +EXPORT_SYMBOL(d_rehash); + +static inline unsigned start_dir_add(struct inode *dir) +{ + preempt_disable_nested(); + for (;;) { + unsigned n = READ_ONCE(dir->i_dir_seq); + if (!(n & 1) && try_cmpxchg(&dir->i_dir_seq, &n, n + 1)) + return n; + cpu_relax(); + } +} + +static inline void end_dir_add(struct inode *dir, unsigned int n) +{ + smp_store_release(&dir->i_dir_seq, n + 2); + preempt_enable_nested(); +} + +static void d_wait_lookup(struct dentry *dentry) +{ + if (likely(d_in_lookup(dentry))) { + dentry->d_flags |= DCACHE_LOOKUP_WAITERS; + wait_var_event_spinlock(&dentry->d_flags, + !d_in_lookup(dentry), + &dentry->d_lock); + } +} + +struct dentry *d_alloc_parallel(struct dentry *parent, + const struct qstr *name) +{ + unsigned int hash = name->hash; + struct hlist_bl_head *b = in_lookup_hash(parent, hash); + struct hlist_bl_node *node; + struct dentry *new = __d_alloc(parent->d_sb, name); + struct dentry *dentry; + unsigned seq, r_seq, d_seq; + + if (unlikely(!new)) + return ERR_PTR(-ENOMEM); + + new->d_flags |= DCACHE_PAR_LOOKUP; + spin_lock(&parent->d_lock); + new->d_parent = dget_dlock(parent); + hlist_add_head(&new->d_sib, &parent->d_children); + if (parent->d_flags & DCACHE_DISCONNECTED) + new->d_flags |= DCACHE_DISCONNECTED; + spin_unlock(&parent->d_lock); + +retry: + seq = smp_load_acquire(&parent->d_inode->i_dir_seq); + r_seq = read_seqbegin(&rename_lock); + rcu_read_lock(); + dentry = __d_lookup_rcu(parent, name, &d_seq); + if (unlikely(dentry)) { + if (!lockref_get_not_dead(&dentry->d_lockref)) { + rcu_read_unlock(); + goto retry; + } + rcu_read_unlock(); + if (read_seqcount_retry(&dentry->d_seq, d_seq)) { + dput(dentry); + goto retry; + } + dput(new); + return dentry; + } + rcu_read_unlock(); + if (unlikely(read_seqretry(&rename_lock, r_seq))) + goto retry; + + if (unlikely(seq & 1)) + goto retry; + + hlist_bl_lock(b); + if (unlikely(READ_ONCE(parent->d_inode->i_dir_seq) != seq)) { + hlist_bl_unlock(b); + goto retry; + } + /* + * No changes for the parent since the beginning of d_lookup(). + * Since all removals from the chain happen with hlist_bl_lock(), + * any potential in-lookup matches are going to stay here until + * we unlock the chain. All fields are stable in everything + * we encounter. + */ + hlist_bl_for_each_entry(dentry, node, b, d_in_lookup_hash) { + if (dentry->d_name.hash != hash) + continue; + if (dentry->d_parent != parent) + continue; + if (!d_same_name(dentry, parent, name)) + continue; + rcu_read_lock(); + hlist_bl_unlock(b); + spin_lock(&dentry->d_lock); + rcu_read_unlock(); + /* now we can try to grab a reference */ + if (unlikely(lockref_is_dead(&dentry->d_lockref))) { + spin_unlock(&dentry->d_lock); + goto retry; + } + /* + * somebody is likely to be still doing lookup for it; + * pin it and wait for them to finish + */ + dget_dlock(dentry); + d_wait_lookup(dentry); + /* + * it's not in-lookup anymore; in principle we should repeat + * everything from dcache lookup, but it's likely to be what + * d_lookup() would've found anyway. If it is, just return it; + * otherwise we really have to repeat the whole thing. + */ + if (unlikely(dentry->d_name.hash != hash)) + goto mismatch; + if (unlikely(dentry->d_parent != parent)) + goto mismatch; + if (unlikely(d_unhashed(dentry))) + goto mismatch; + if (unlikely(!d_same_name(dentry, parent, name))) + goto mismatch; + /* OK, it *is* a hashed match; return it */ + spin_unlock(&dentry->d_lock); + dput(new); + return dentry; + } + hlist_bl_add_head(&new->d_in_lookup_hash, b); + hlist_bl_unlock(b); + return new; +mismatch: + spin_unlock(&dentry->d_lock); + dput(dentry); + goto retry; +} +EXPORT_SYMBOL(d_alloc_parallel); + +/* + * Move dentry from in-lookup state to busy-negative one. + * + * From now on d_in_lookup(dentry) will return false and dentry is gone from + * in-lookup hash. + * + * Anyone who had been waiting on it in d_alloc_parallel() is free to + * proceed after that. Note that waking such waiters up is left to + * the callers; PREEMPT_RT kernels can't have that wakeup done while + * in write-side critical area for ->i_dir_seq, so it's done by calling + * __d_wake_in_lookup_waiters() once it's safe to do so. + * + * Both __d_lookup_unhash() and __d_wake_in_lookup_waiters() should + * be called within the same ->d_lock scope. PAR_LOOKUP is cleared + * here, while LOOKUP_WAITERS (set by somebody finding dentry in + * the in-lookup hash and setting down to wait) is checked and cleared + * in __d_wake_in_lookup_waiters(). Both are gone by the end of + * ->d_lock scope. + */ +static void __d_lookup_unhash(struct dentry *dentry) +{ + struct hlist_bl_head *b; + + lockdep_assert_held(&dentry->d_lock); + + b = in_lookup_hash(dentry->d_parent, dentry->d_name.hash); + hlist_bl_lock(b); + dentry->d_flags &= ~DCACHE_PAR_LOOKUP; + __hlist_bl_del(&dentry->d_in_lookup_hash); + hlist_bl_unlock(b); + dentry->waiters = NULL; +} + +static inline void __d_wake_in_lookup_waiters(struct dentry *dentry) +{ + if (dentry->d_flags & DCACHE_LOOKUP_WAITERS) { + wake_up_var_locked(&dentry->d_flags, &dentry->d_lock); + dentry->d_flags &= ~DCACHE_LOOKUP_WAITERS; + } +} + +void __d_lookup_unhash_wake(struct dentry *dentry) +{ + spin_lock(&dentry->d_lock); + __d_lookup_unhash(dentry); + __d_wake_in_lookup_waiters(dentry); + spin_unlock(&dentry->d_lock); +} +EXPORT_SYMBOL(__d_lookup_unhash_wake); + +/* inode->i_lock held if inode is non-NULL */ + +static inline void __d_add(struct dentry *dentry, struct inode *inode, + const struct dentry_operations *ops) +{ + struct inode *dir = NULL; + unsigned n; + spin_lock(&dentry->d_lock); + if (unlikely(d_in_lookup(dentry))) { + dir = dentry->d_parent->d_inode; + n = start_dir_add(dir); + __d_lookup_unhash(dentry); + } + if (unlikely(ops)) + d_set_d_op(dentry, ops); + if (inode) { + unsigned add_flags = d_flags_for_inode(inode); + hlist_add_head(&dentry->d_alias, &inode->i_dentry); + raw_write_seqcount_begin(&dentry->d_seq); + __d_set_inode_and_type(dentry, inode, add_flags); + raw_write_seqcount_end(&dentry->d_seq); + fsnotify_update_flags(dentry); + } + __d_rehash(dentry); + if (dir) { + end_dir_add(dir, n); + __d_wake_in_lookup_waiters(dentry); + } + spin_unlock(&dentry->d_lock); + if (inode) + spin_unlock(&inode->i_lock); +} + +/** + * d_add - add dentry to hash queues + * @entry: dentry to add + * @inode: The inode to attach to this dentry + * + * This adds the entry to the hash queues and initializes @inode. + * The entry was actually filled in earlier during d_alloc(). + */ + +void d_add(struct dentry *entry, struct inode *inode) +{ + if (inode) { + security_d_instantiate(entry, inode); + spin_lock(&inode->i_lock); + } + __d_add(entry, inode, NULL); +} +EXPORT_SYMBOL(d_add); + +struct dentry *d_make_persistent(struct dentry *dentry, struct inode *inode) +{ + WARN_ON(d_really_is_positive(dentry)); + WARN_ON(!inode); + security_d_instantiate(dentry, inode); + spin_lock(&inode->i_lock); + spin_lock(&dentry->d_lock); + __d_instantiate(dentry, inode); + dentry->d_flags |= DCACHE_PERSISTENT; + dget_dlock(dentry); + if (d_unhashed(dentry)) + __d_rehash(dentry); + spin_unlock(&dentry->d_lock); + spin_unlock(&inode->i_lock); + return dentry; +} +EXPORT_SYMBOL(d_make_persistent); + +static void swap_names(struct dentry *dentry, struct dentry *target) +{ + if (unlikely(dname_external(target))) { + if (unlikely(dname_external(dentry))) { + /* + * Both external: swap the pointers + */ + swap(target->__d_name.name, dentry->__d_name.name); + } else { + /* + * dentry:internal, target:external. Steal target's + * storage and make target internal. + */ + dentry->__d_name.name = target->__d_name.name; + target->d_shortname = dentry->d_shortname; + target->__d_name.name = target->d_shortname.string; + } + } else { + if (unlikely(dname_external(dentry))) { + /* + * dentry:external, target:internal. Give dentry's + * storage to target and make dentry internal + */ + target->__d_name.name = dentry->__d_name.name; + dentry->d_shortname = target->d_shortname; + dentry->__d_name.name = dentry->d_shortname.string; + } else { + /* + * Both are internal. + */ + for (int i = 0; i < DNAME_INLINE_WORDS; i++) + swap(dentry->d_shortname.words[i], + target->d_shortname.words[i]); + } + } + swap(dentry->__d_name.hash_len, target->__d_name.hash_len); +} + +static void copy_name(struct dentry *dentry, struct dentry *target) +{ + struct external_name *old_name = NULL; + if (unlikely(dname_external(dentry))) + old_name = external_name(dentry); + if (unlikely(dname_external(target))) { + atomic_inc(&external_name(target)->count); + dentry->__d_name = target->__d_name; + } else { + dentry->d_shortname = target->d_shortname; + dentry->__d_name.name = dentry->d_shortname.string; + dentry->__d_name.hash_len = target->__d_name.hash_len; + } + if (old_name && likely(atomic_dec_and_test(&old_name->count))) + kfree_rcu(old_name, head); +} + +/* + * __d_move - move a dentry + * @dentry: entry to move + * @target: new dentry + * @exchange: exchange the two dentries + * + * Update the dcache to reflect the move of a file name. Negative dcache + * entries should not be moved in this way. Caller must hold rename_lock, the + * i_rwsem of the source and target directories (exclusively), and the sb-> + * s_vfs_rename_mutex if they differ. See lock_rename(). + */ +static void __d_move(struct dentry *dentry, struct dentry *target, + bool exchange) +{ + struct dentry *old_parent, *p; + struct inode *dir = NULL; + unsigned n; + + WARN_ON(!dentry->d_inode); + if (WARN_ON(dentry == target)) + return; + + BUG_ON(d_ancestor(target, dentry)); + old_parent = dentry->d_parent; + p = d_ancestor(old_parent, target); + if (IS_ROOT(dentry)) { + BUG_ON(p); + spin_lock(&target->d_parent->d_lock); + } else if (!p) { + /* target is not a descendent of dentry->d_parent */ + spin_lock(&target->d_parent->d_lock); + spin_lock_nested(&old_parent->d_lock, DENTRY_D_LOCK_NESTED); + } else { + BUG_ON(p == dentry); + spin_lock(&old_parent->d_lock); + if (p != target) + spin_lock_nested(&target->d_parent->d_lock, + DENTRY_D_LOCK_NESTED); + } + spin_lock_nested(&dentry->d_lock, 2); + spin_lock_nested(&target->d_lock, 3); + + if (unlikely(d_in_lookup(target))) { + dir = target->d_parent->d_inode; + n = start_dir_add(dir); + __d_lookup_unhash(target); + } + + write_seqcount_begin(&dentry->d_seq); + write_seqcount_begin_nested(&target->d_seq, DENTRY_D_LOCK_NESTED); + + /* unhash both */ + if (!d_unhashed(dentry)) + ___d_drop(dentry); + if (!d_unhashed(target)) + ___d_drop(target); + + /* ... and switch them in the tree */ + dentry->d_parent = target->d_parent; + if (!exchange) { + copy_name(dentry, target); + target->d_hash.pprev = NULL; + dentry->d_parent->d_lockref.count++; + if (dentry != old_parent) /* wasn't IS_ROOT */ + WARN_ON(!--old_parent->d_lockref.count); + } else { + target->d_parent = old_parent; + swap_names(dentry, target); + if (!hlist_unhashed(&target->d_sib)) + __hlist_del(&target->d_sib); + hlist_add_head(&target->d_sib, &target->d_parent->d_children); + __d_rehash(target); + fsnotify_update_flags(target); + } + if (!hlist_unhashed(&dentry->d_sib)) + __hlist_del(&dentry->d_sib); + hlist_add_head(&dentry->d_sib, &dentry->d_parent->d_children); + __d_rehash(dentry); + fsnotify_update_flags(dentry); + fscrypt_handle_d_move(dentry); + + write_seqcount_end(&target->d_seq); + write_seqcount_end(&dentry->d_seq); + + if (dir) { + end_dir_add(dir, n); + __d_wake_in_lookup_waiters(target); + } + if (dentry->d_parent != old_parent) + spin_unlock(&dentry->d_parent->d_lock); + if (dentry != old_parent) + spin_unlock(&old_parent->d_lock); + spin_unlock(&target->d_lock); + spin_unlock(&dentry->d_lock); +} + +/* + * d_move - move a dentry + * @dentry: entry to move + * @target: new dentry + * + * Update the dcache to reflect the move of a file name. Negative + * dcache entries should not be moved in this way. See the locking + * requirements for __d_move. + */ +void d_move(struct dentry *dentry, struct dentry *target) +{ + write_seqlock(&rename_lock); + __d_move(dentry, target, false); + write_sequnlock(&rename_lock); +} +EXPORT_SYMBOL(d_move); + +/* + * d_exchange - exchange two dentries + * @dentry1: first dentry + * @dentry2: second dentry + */ +void d_exchange(struct dentry *dentry1, struct dentry *dentry2) +{ + write_seqlock(&rename_lock); + + WARN_ON(!dentry1->d_inode); + WARN_ON(!dentry2->d_inode); + WARN_ON(IS_ROOT(dentry1)); + WARN_ON(IS_ROOT(dentry2)); + + __d_move(dentry1, dentry2, true); + + write_sequnlock(&rename_lock); +} +EXPORT_SYMBOL(d_exchange); + +/** + * d_ancestor - search for an ancestor + * @p1: ancestor dentry + * @p2: child dentry + * + * Returns the ancestor dentry of p2 which is a child of p1, if p1 is + * an ancestor of p2, else NULL. + */ +struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2) +{ + struct dentry *p; + + for (p = p2; !IS_ROOT(p); p = p->d_parent) { + if (p->d_parent == p1) + return p; + } + return NULL; +} + +/* + * This helper attempts to cope with remotely renamed directories + * + * It assumes that the caller is already holding + * dentry->d_parent->d_inode->i_rwsem, and rename_lock + * + * Note: If ever the locking in lock_rename() changes, then please + * remember to update this too... + */ +static int __d_unalias(struct dentry *dentry, struct dentry *alias) +{ + struct mutex *m1 = NULL; + struct rw_semaphore *m2 = NULL; + int ret = -ESTALE; + + /* If alias and dentry share a parent, then no extra locks required */ + if (alias->d_parent == dentry->d_parent) + goto out_unalias; + + /* See lock_rename() */ + if (!mutex_trylock(&dentry->d_sb->s_vfs_rename_mutex)) + goto out_err; + m1 = &dentry->d_sb->s_vfs_rename_mutex; + if (!inode_trylock_shared(alias->d_parent->d_inode)) + goto out_err; + m2 = &alias->d_parent->d_inode->i_rwsem; +out_unalias: + if (alias->d_op && alias->d_op->d_unalias_trylock && + !alias->d_op->d_unalias_trylock(alias)) + goto out_err; + __d_move(alias, dentry, false); + if (alias->d_op && alias->d_op->d_unalias_unlock) + alias->d_op->d_unalias_unlock(alias); + ret = 0; +out_err: + if (m2) + up_read(m2); + if (m1) + mutex_unlock(m1); + return ret; +} + +struct dentry *d_splice_alias_ops(struct inode *inode, struct dentry *dentry, + const struct dentry_operations *ops) +{ + if (IS_ERR(inode)) + return ERR_CAST(inode); + + BUG_ON(!d_unhashed(dentry)); + + if (!inode) + goto out; + + security_d_instantiate(dentry, inode); + spin_lock(&inode->i_lock); + if (S_ISDIR(inode->i_mode)) { + struct dentry *new = __d_find_dir_alias(inode); + if (unlikely(new)) { + /* The reference to new ensures it remains an alias */ + spin_unlock(&inode->i_lock); + write_seqlock(&rename_lock); + if (unlikely(d_ancestor(new, dentry))) { + write_sequnlock(&rename_lock); + dput(new); + new = ERR_PTR(-ELOOP); + pr_warn_ratelimited( + "VFS: Lookup of '%s' in %s %s" + " would have caused loop\n", + dentry->d_name.name, + inode->i_sb->s_type->name, + inode->i_sb->s_id); + } else if (!IS_ROOT(new)) { + struct dentry *old_parent = dget(new->d_parent); + int err = __d_unalias(dentry, new); + write_sequnlock(&rename_lock); + if (err) { + dput(new); + new = ERR_PTR(err); + } + dput(old_parent); + } else { + if (unlikely(!hlist_unhashed(&new->d_sib))) { + // secondary root getting spliced + spin_lock(&new->d_lock); + unlink_secondary_root(new); + spin_unlock(&new->d_lock); + } + __d_move(new, dentry, false); + write_sequnlock(&rename_lock); + } + iput(inode); + return new; + } + } +out: + __d_add(dentry, inode, ops); + return NULL; +} + +/** + * d_splice_alias - splice a disconnected dentry into the tree if one exists + * @inode: the inode which may have a disconnected dentry + * @dentry: a negative dentry which we want to point to the inode. + * + * If inode is a directory and has an IS_ROOT alias, then d_move that in + * place of the given dentry and return it, else simply d_add the inode + * to the dentry and return NULL. + * + * If a non-IS_ROOT directory is found, the filesystem is corrupt, and + * we should error out: directories can't have multiple aliases. + * + * This is needed in the lookup routine of any filesystem that is exportable + * (via knfsd) so that we can build dcache paths to directories effectively. + * + * If a dentry was found and moved, then it is returned. Otherwise NULL + * is returned. This matches the expected return value of ->lookup. + * + * Cluster filesystems may call this function with a negative, hashed dentry. + * In that case, we know that the inode will be a regular file, and also this + * will only occur during atomic_open. So we need to check for the dentry + * being already hashed only in the final case. + */ +struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry) +{ + return d_splice_alias_ops(inode, dentry, NULL); +} +EXPORT_SYMBOL(d_splice_alias); + +/* + * Test whether new_dentry is a subdirectory of old_dentry. + * + * Trivially implemented using the dcache structure + */ + +/** + * is_subdir - is new dentry a subdirectory of old_dentry + * @new_dentry: new dentry + * @old_dentry: old dentry + * + * Returns true if new_dentry is a subdirectory of the parent (at any depth). + * Returns false otherwise. + * Caller must ensure that "new_dentry" is pinned before calling is_subdir() + */ + +bool is_subdir(struct dentry *new_dentry, struct dentry *old_dentry) +{ + bool subdir; + unsigned seq; + + if (new_dentry == old_dentry) + return true; + + /* Access d_parent under rcu as d_move() may change it. */ + rcu_read_lock(); + seq = read_seqbegin(&rename_lock); + subdir = d_ancestor(old_dentry, new_dentry); + /* Try lockless once... */ + if (read_seqretry(&rename_lock, seq)) { + /* ...else acquire lock for progress even on deep chains. */ + read_seqlock_excl(&rename_lock); + subdir = d_ancestor(old_dentry, new_dentry); + read_sequnlock_excl(&rename_lock); + } + rcu_read_unlock(); + return subdir; +} +EXPORT_SYMBOL(is_subdir); + +void d_mark_tmpfile(struct file *file, struct inode *inode) +{ + struct dentry *dentry = file->f_path.dentry; + + BUG_ON(dname_external(dentry) || + d_really_is_positive(dentry) || + !d_unlinked(dentry)); + spin_lock(&dentry->d_parent->d_lock); + spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); + dentry->__d_name.len = sprintf(dentry->d_shortname.string, "#%llu", + (unsigned long long)inode->i_ino); + spin_unlock(&dentry->d_lock); + spin_unlock(&dentry->d_parent->d_lock); +} +EXPORT_SYMBOL(d_mark_tmpfile); + +int d_mark_tmpfile_name(struct file *file, const struct qstr *name) +{ + struct dentry *dentry = file->f_path.dentry; + char *dname = dentry->d_shortname.string; + + if (unlikely(dname_external(dentry) || + d_really_is_positive(dentry) || + !d_unlinked(dentry))) + return -EINVAL; + if (unlikely(name->len > DNAME_INLINE_LEN - 1)) + return -ENAMETOOLONG; + + spin_lock(&dentry->d_parent->d_lock); + spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); + dentry->__d_name.len = name->len; + memcpy(dname, name->name, name->len); + dname[name->len] = '\0'; + spin_unlock(&dentry->d_lock); + spin_unlock(&dentry->d_parent->d_lock); + return 0; +} +EXPORT_SYMBOL(d_mark_tmpfile_name); + +void d_tmpfile(struct file *file, struct inode *inode) +{ + struct dentry *dentry = file->f_path.dentry; + + inode_dec_link_count(inode); + d_mark_tmpfile(file, inode); + d_instantiate(dentry, inode); +} +EXPORT_SYMBOL(d_tmpfile); + +/* + * Obtain inode number of the parent dentry. + */ +ino_t d_parent_ino(struct dentry *dentry) +{ + struct dentry *parent; + struct inode *iparent; + unsigned seq; + ino_t ret; + + scoped_guard(rcu) { + seq = raw_seqcount_begin(&dentry->d_seq); + parent = READ_ONCE(dentry->d_parent); + iparent = d_inode_rcu(parent); + if (likely(iparent)) { + ret = iparent->i_ino; + if (!read_seqcount_retry(&dentry->d_seq, seq)) + return ret; + } + } + + spin_lock(&dentry->d_lock); + ret = dentry->d_parent->d_inode->i_ino; + spin_unlock(&dentry->d_lock); + return ret; +} +EXPORT_SYMBOL(d_parent_ino); + +static __initdata unsigned long dhash_entries; +static int __init set_dhash_entries(char *str) +{ + return kstrtoul(str, 0, &dhash_entries) == 0; +} +__setup("dhash_entries=", set_dhash_entries); + +static void __init dcache_init_early(void) +{ + /* If hashes are distributed across NUMA nodes, defer + * hash allocation until vmalloc space is available. + */ + if (hashdist) + return; + + dentry_hashtable = + alloc_large_system_hash("Dentry cache", + sizeof(struct hlist_bl_head), + dhash_entries, + 13, + HASH_EARLY | HASH_ZERO, + &d_hash_shift, + NULL, + 2, + 0); + d_hash_shift = 32 - d_hash_shift; + + runtime_const_init(shift, d_hash_shift); + runtime_const_init(ptr, dentry_hashtable); +} + +static void __init dcache_init(void) +{ + /* + * A constructor could be added for stable state like the lists, + * but it is probably not worth it because of the cache nature + * of the dcache. + */ + __dentry_cache = KMEM_CACHE_USERCOPY(dentry, + SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_ACCOUNT, + d_shortname.string); + runtime_const_init(ptr, __dentry_cache); + + /* Hash may have been set up in dcache_init_early */ + if (!hashdist) + return; + + dentry_hashtable = + alloc_large_system_hash("Dentry cache", + sizeof(struct hlist_bl_head), + dhash_entries, + 13, + HASH_ZERO, + &d_hash_shift, + NULL, + 2, + 0); + d_hash_shift = 32 - d_hash_shift; + + runtime_const_init(shift, d_hash_shift); + runtime_const_init(ptr, dentry_hashtable); +} + +void __init vfs_caches_init_early(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(in_lookup_hashtable); i++) + INIT_HLIST_BL_HEAD(&in_lookup_hashtable[i]); + + dcache_init_early(); + inode_init_early(); +} + +void __init vfs_caches_init(void) +{ + filename_init(); + dcache_init(); + inode_init(); + files_init(); + files_maxfiles_init(); + mnt_init(); + bdev_cache_init(); + chrdev_init(); +} diff --git a/fs/debugfs/Makefile b/fs/debugfs/Makefile new file mode 100644 index 000000000..9c0fe38cf --- /dev/null +++ b/fs/debugfs/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0-only +debugfs-objs := inode.o file.o + +obj-$(CONFIG_DEBUG_FS) += debugfs.o + diff --git a/fs/debugfs/file.c b/fs/debugfs/file.c new file mode 100644 index 000000000..08de6652a --- /dev/null +++ b/fs/debugfs/file.c @@ -0,0 +1,1436 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * file.c - part of debugfs, a tiny little debug file system + * + * Copyright (C) 2004 Greg Kroah-Hartman + * Copyright (C) 2004 IBM Inc. + * + * debugfs is for people to use instead of /proc or /sys. + * See Documentation/filesystems/ for more details. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +struct poll_table_struct; + +static ssize_t default_read_file(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + return 0; +} + +static ssize_t default_write_file(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + return count; +} + +const struct file_operations debugfs_noop_file_operations = { + .read = default_read_file, + .write = default_write_file, + .open = simple_open, + .llseek = noop_llseek, +}; + +#define F_DENTRY(filp) ((filp)->f_path.dentry) + +void *debugfs_get_aux(const struct file *file) +{ + return DEBUGFS_I(file_inode(file))->aux; +} +EXPORT_SYMBOL_GPL(debugfs_get_aux); + +enum dbgfs_get_mode { + DBGFS_GET_ALREADY, + DBGFS_GET_REGULAR, + DBGFS_GET_SHORT, +}; + +static int __debugfs_file_get(struct dentry *dentry, enum dbgfs_get_mode mode) +{ + struct debugfs_fsdata *fsd; + void *d_fsd; + + /* + * This could only happen if some debugfs user erroneously calls + * debugfs_file_get() on a dentry that isn't even a file, let + * them know about it. + */ + if (WARN_ON(!d_is_reg(dentry))) + return -EINVAL; + + d_fsd = READ_ONCE(dentry->d_fsdata); + if (d_fsd) { + fsd = d_fsd; + } else { + struct inode *inode = dentry->d_inode; + unsigned int methods = 0; + + if (WARN_ON(mode == DBGFS_GET_ALREADY)) + return -EINVAL; + + fsd = kmalloc_obj(*fsd); + if (!fsd) + return -ENOMEM; + + if (mode == DBGFS_GET_SHORT) { + const struct debugfs_short_fops *ops; + ops = fsd->short_fops = DEBUGFS_I(inode)->short_fops; + if (ops->llseek) + methods |= HAS_LSEEK; + if (ops->read) + methods |= HAS_READ; + if (ops->write) + methods |= HAS_WRITE; + fsd->real_fops = NULL; + } else { + const struct file_operations *ops; + ops = fsd->real_fops = DEBUGFS_I(inode)->real_fops; + if (ops->llseek) + methods |= HAS_LSEEK; + if (ops->read) + methods |= HAS_READ; + if (ops->write) + methods |= HAS_WRITE; + if (ops->unlocked_ioctl) + methods |= HAS_IOCTL; + if (ops->poll) + methods |= HAS_POLL; + fsd->short_fops = NULL; + } + fsd->methods = methods; + refcount_set(&fsd->active_users, 1); + init_completion(&fsd->active_users_drained); + INIT_LIST_HEAD(&fsd->cancellations); + mutex_init(&fsd->cancellations_mtx); + + d_fsd = cmpxchg(&dentry->d_fsdata, NULL, fsd); + if (d_fsd) { + mutex_destroy(&fsd->cancellations_mtx); + kfree(fsd); + fsd = d_fsd; + } + } + + /* + * In case of a successful cmpxchg() above, this check is + * strictly necessary and must follow it, see the comment in + * __debugfs_remove_file(). + * OTOH, if the cmpxchg() hasn't been executed or wasn't + * successful, this serves the purpose of not starving + * removers. + */ + if (d_unlinked(dentry)) + return -EIO; + + if (!refcount_inc_not_zero(&fsd->active_users)) + return -EIO; + + return 0; +} + +/** + * debugfs_file_get - mark the beginning of file data access + * @dentry: the dentry object whose data is being accessed. + * + * Up to a matching call to debugfs_file_put(), any successive call + * into the file removing functions debugfs_remove() and + * debugfs_remove_recursive() will block. Since associated private + * file data may only get freed after a successful return of any of + * the removal functions, you may safely access it after a successful + * call to debugfs_file_get() without worrying about lifetime issues. + * + * If -%EIO is returned, the file has already been removed and thus, + * it is not safe to access any of its data. If, on the other hand, + * it is allowed to access the file data, zero is returned. + */ +int debugfs_file_get(struct dentry *dentry) +{ + return __debugfs_file_get(dentry, DBGFS_GET_ALREADY); +} +EXPORT_SYMBOL_GPL(debugfs_file_get); + +/** + * debugfs_file_put - mark the end of file data access + * @dentry: the dentry object formerly passed to + * debugfs_file_get(). + * + * Allow any ongoing concurrent call into debugfs_remove() or + * debugfs_remove_recursive() blocked by a former call to + * debugfs_file_get() to proceed and return to its caller. + */ +void debugfs_file_put(struct dentry *dentry) +{ + struct debugfs_fsdata *fsd = READ_ONCE(dentry->d_fsdata); + + if (refcount_dec_and_test(&fsd->active_users)) + complete(&fsd->active_users_drained); +} +EXPORT_SYMBOL_GPL(debugfs_file_put); + +/** + * debugfs_enter_cancellation - enter a debugfs cancellation + * @file: the file being accessed + * @cancellation: the cancellation object, the cancel callback + * inside of it must be initialized + * + * When a debugfs file is removed it needs to wait for all active + * operations to complete. However, the operation itself may need + * to wait for hardware or completion of some asynchronous process + * or similar. As such, it may need to be cancelled to avoid long + * waits or even deadlocks. + * + * This function can be used inside a debugfs handler that may + * need to be cancelled. As soon as this function is called, the + * cancellation's 'cancel' callback may be called, at which point + * the caller should proceed to call debugfs_leave_cancellation() + * and leave the debugfs handler function as soon as possible. + * Note that the 'cancel' callback is only ever called in the + * context of some kind of debugfs_remove(). + * + * This function must be paired with debugfs_leave_cancellation(). + */ +void debugfs_enter_cancellation(struct file *file, + struct debugfs_cancellation *cancellation) +{ + struct debugfs_fsdata *fsd; + struct dentry *dentry = F_DENTRY(file); + + INIT_LIST_HEAD(&cancellation->list); + + if (WARN_ON(!d_is_reg(dentry))) + return; + + if (WARN_ON(!cancellation->cancel)) + return; + + fsd = READ_ONCE(dentry->d_fsdata); + if (WARN_ON(!fsd)) + return; + + mutex_lock(&fsd->cancellations_mtx); + list_add(&cancellation->list, &fsd->cancellations); + mutex_unlock(&fsd->cancellations_mtx); + + /* if we're already removing wake it up to cancel */ + if (d_unlinked(dentry)) + complete(&fsd->active_users_drained); +} +EXPORT_SYMBOL_GPL(debugfs_enter_cancellation); + +/** + * debugfs_leave_cancellation - leave cancellation section + * @file: the file being accessed + * @cancellation: the cancellation previously registered with + * debugfs_enter_cancellation() + * + * See the documentation of debugfs_enter_cancellation(). + */ +void debugfs_leave_cancellation(struct file *file, + struct debugfs_cancellation *cancellation) +{ + struct debugfs_fsdata *fsd; + struct dentry *dentry = F_DENTRY(file); + + if (WARN_ON(!d_is_reg(dentry))) + return; + + fsd = READ_ONCE(dentry->d_fsdata); + if (WARN_ON(!fsd)) + return; + + mutex_lock(&fsd->cancellations_mtx); + if (!list_empty(&cancellation->list)) + list_del(&cancellation->list); + mutex_unlock(&fsd->cancellations_mtx); +} +EXPORT_SYMBOL_GPL(debugfs_leave_cancellation); + +/* + * Only permit access to world-readable files when the kernel is locked down. + * We also need to exclude any file that has ways to write or alter it as root + * can bypass the permissions check. + */ +static int debugfs_locked_down(struct inode *inode, + struct file *filp, + const struct file_operations *real_fops) +{ + if ((inode->i_mode & 07777 & ~0444) == 0 && + !(filp->f_mode & FMODE_WRITE) && + (!real_fops || + (!real_fops->unlocked_ioctl && + !real_fops->compat_ioctl && + !real_fops->mmap && + !real_fops->mmap_prepare))) + return 0; + + if (security_locked_down(LOCKDOWN_DEBUGFS)) + return -EPERM; + + return 0; +} + +static int open_proxy_open(struct inode *inode, struct file *filp) +{ + struct dentry *dentry = F_DENTRY(filp); + const struct file_operations *real_fops = DEBUGFS_I(inode)->real_fops; + int r; + + r = __debugfs_file_get(dentry, DBGFS_GET_REGULAR); + if (r) + return r == -EIO ? -ENOENT : r; + + r = debugfs_locked_down(inode, filp, real_fops); + if (r) + goto out; + + if (!fops_get(real_fops)) { +#ifdef CONFIG_MODULES + if (real_fops->owner && + real_fops->owner->state == MODULE_STATE_GOING) { + r = -ENXIO; + goto out; + } +#endif + + /* Huh? Module did not clean up after itself at exit? */ + WARN(1, "debugfs file owner did not clean up at exit: %pd", + dentry); + r = -ENXIO; + goto out; + } + replace_fops(filp, real_fops); + + if (real_fops->open) + r = real_fops->open(inode, filp); + +out: + debugfs_file_put(dentry); + return r; +} + +const struct file_operations debugfs_open_proxy_file_operations = { + .open = open_proxy_open, +}; + +#define PROTO(args...) args +#define ARGS(args...) args + +#define FULL_PROXY_FUNC(name, ret_type, filp, proto, args, bit, ret) \ +static ret_type full_proxy_ ## name(proto) \ +{ \ + struct dentry *dentry = F_DENTRY(filp); \ + struct debugfs_fsdata *fsd = dentry->d_fsdata; \ + ret_type r; \ + \ + if (!(fsd->methods & bit)) \ + return ret; \ + r = debugfs_file_get(dentry); \ + if (unlikely(r)) \ + return r; \ + r = fsd->real_fops->name(args); \ + debugfs_file_put(dentry); \ + return r; \ +} + +#define SHORT_PROXY_FUNC(name, ret_type, filp, proto, args, bit, ret) \ +static ret_type short_proxy_ ## name(proto) \ +{ \ + struct dentry *dentry = F_DENTRY(filp); \ + struct debugfs_fsdata *fsd = dentry->d_fsdata; \ + ret_type r; \ + \ + if (!(fsd->methods & bit)) \ + return ret; \ + r = debugfs_file_get(dentry); \ + if (unlikely(r)) \ + return r; \ + r = fsd->short_fops->name(args); \ + debugfs_file_put(dentry); \ + return r; \ +} + +SHORT_PROXY_FUNC(llseek, loff_t, filp, + PROTO(struct file *filp, loff_t offset, int whence), + ARGS(filp, offset, whence), HAS_LSEEK, -ESPIPE); + +FULL_PROXY_FUNC(llseek, loff_t, filp, + PROTO(struct file *filp, loff_t offset, int whence), + ARGS(filp, offset, whence), HAS_LSEEK, -ESPIPE); + +SHORT_PROXY_FUNC(read, ssize_t, filp, + PROTO(struct file *filp, char __user *buf, size_t size, + loff_t *ppos), + ARGS(filp, buf, size, ppos), HAS_READ, -EINVAL); + +FULL_PROXY_FUNC(read, ssize_t, filp, + PROTO(struct file *filp, char __user *buf, size_t size, + loff_t *ppos), + ARGS(filp, buf, size, ppos), HAS_READ, -EINVAL); + +SHORT_PROXY_FUNC(write, ssize_t, filp, + PROTO(struct file *filp, const char __user *buf, + size_t size, loff_t *ppos), + ARGS(filp, buf, size, ppos), HAS_WRITE, -EINVAL); + +FULL_PROXY_FUNC(write, ssize_t, filp, + PROTO(struct file *filp, const char __user *buf, + size_t size, loff_t *ppos), + ARGS(filp, buf, size, ppos), HAS_WRITE, -EINVAL); + +FULL_PROXY_FUNC(unlocked_ioctl, long, filp, + PROTO(struct file *filp, unsigned int cmd, unsigned long arg), + ARGS(filp, cmd, arg), HAS_IOCTL, -ENOTTY); + +static __poll_t full_proxy_poll(struct file *filp, + struct poll_table_struct *wait) +{ + struct dentry *dentry = F_DENTRY(filp); + struct debugfs_fsdata *fsd = dentry->d_fsdata; + __poll_t r = 0; + + if (!(fsd->methods & HAS_POLL)) + return DEFAULT_POLLMASK; + if (debugfs_file_get(dentry)) + return EPOLLHUP; + + r = fsd->real_fops->poll(filp, wait); + debugfs_file_put(dentry); + return r; +} + +static int full_proxy_release(struct inode *inode, struct file *file) +{ + struct debugfs_fsdata *fsd = F_DENTRY(file)->d_fsdata; + const struct file_operations *real_fops = fsd->real_fops; + int r = 0; + + /* + * We must not protect this against removal races here: the + * original releaser should be called unconditionally in order + * not to leak any resources. Releasers must not assume that + * ->i_private is still being meaningful here. + */ + if (real_fops->release) + r = real_fops->release(inode, file); + + fops_put(real_fops); + return r; +} + +static int full_proxy_open_regular(struct inode *inode, struct file *filp) +{ + struct dentry *dentry = F_DENTRY(filp); + const struct file_operations *real_fops; + struct debugfs_fsdata *fsd; + int r; + + r = __debugfs_file_get(dentry, DBGFS_GET_REGULAR); + if (r) + return r == -EIO ? -ENOENT : r; + + fsd = dentry->d_fsdata; + real_fops = fsd->real_fops; + r = debugfs_locked_down(inode, filp, real_fops); + if (r) + goto out; + + if (!fops_get(real_fops)) { +#ifdef CONFIG_MODULES + if (real_fops->owner && + real_fops->owner->state == MODULE_STATE_GOING) { + r = -ENXIO; + goto out; + } +#endif + + /* Huh? Module did not cleanup after itself at exit? */ + WARN(1, "debugfs file owner did not clean up at exit: %pd", + dentry); + r = -ENXIO; + goto out; + } + + if (real_fops->open) { + r = real_fops->open(inode, filp); + if (r) { + fops_put(real_fops); + } else if (filp->f_op != &debugfs_full_proxy_file_operations) { + /* No protection against file removal anymore. */ + WARN(1, "debugfs file owner replaced proxy fops: %pd", + dentry); + fops_put(real_fops); + } + } +out: + debugfs_file_put(dentry); + return r; +} + +const struct file_operations debugfs_full_proxy_file_operations = { + .open = full_proxy_open_regular, + .release = full_proxy_release, + .llseek = full_proxy_llseek, + .read = full_proxy_read, + .write = full_proxy_write, + .poll = full_proxy_poll, + .unlocked_ioctl = full_proxy_unlocked_ioctl +}; + +static int full_proxy_open_short(struct inode *inode, struct file *filp) +{ + struct dentry *dentry = F_DENTRY(filp); + int r; + + r = __debugfs_file_get(dentry, DBGFS_GET_SHORT); + if (r) + return r == -EIO ? -ENOENT : r; + r = debugfs_locked_down(inode, filp, NULL); + if (!r) + r = simple_open(inode, filp); + debugfs_file_put(dentry); + return r; +} + +const struct file_operations debugfs_full_short_proxy_file_operations = { + .open = full_proxy_open_short, + .llseek = short_proxy_llseek, + .read = short_proxy_read, + .write = short_proxy_write, +}; + +ssize_t debugfs_attr_read(struct file *file, char __user *buf, + size_t len, loff_t *ppos) +{ + struct dentry *dentry = F_DENTRY(file); + ssize_t ret; + + ret = debugfs_file_get(dentry); + if (unlikely(ret)) + return ret; + ret = simple_attr_read(file, buf, len, ppos); + debugfs_file_put(dentry); + return ret; +} +EXPORT_SYMBOL_GPL(debugfs_attr_read); + +static ssize_t debugfs_attr_write_xsigned(struct file *file, const char __user *buf, + size_t len, loff_t *ppos, bool is_signed) +{ + struct dentry *dentry = F_DENTRY(file); + ssize_t ret; + + ret = debugfs_file_get(dentry); + if (unlikely(ret)) + return ret; + if (is_signed) + ret = simple_attr_write_signed(file, buf, len, ppos); + else + ret = simple_attr_write(file, buf, len, ppos); + debugfs_file_put(dentry); + return ret; +} + +ssize_t debugfs_attr_write(struct file *file, const char __user *buf, + size_t len, loff_t *ppos) +{ + return debugfs_attr_write_xsigned(file, buf, len, ppos, false); +} +EXPORT_SYMBOL_GPL(debugfs_attr_write); + +ssize_t debugfs_attr_write_signed(struct file *file, const char __user *buf, + size_t len, loff_t *ppos) +{ + return debugfs_attr_write_xsigned(file, buf, len, ppos, true); +} +EXPORT_SYMBOL_GPL(debugfs_attr_write_signed); + +static struct dentry *debugfs_create_mode_unsafe(const char *name, umode_t mode, + struct dentry *parent, void *value, + const struct file_operations *fops, + const struct file_operations *fops_ro, + const struct file_operations *fops_wo) +{ + /* if there are no write bits set, make read only */ + if (!(mode & S_IWUGO)) + return debugfs_create_file_unsafe(name, mode, parent, value, + fops_ro); + /* if there are no read bits set, make write only */ + if (!(mode & S_IRUGO)) + return debugfs_create_file_unsafe(name, mode, parent, value, + fops_wo); + + return debugfs_create_file_unsafe(name, mode, parent, value, fops); +} + +static int debugfs_u8_set(void *data, u64 val) +{ + *(u8 *)data = val; + return 0; +} +static int debugfs_u8_get(void *data, u64 *val) +{ + *val = *(u8 *)data; + return 0; +} +DEFINE_DEBUGFS_ATTRIBUTE(fops_u8, debugfs_u8_get, debugfs_u8_set, "%llu\n"); +DEFINE_DEBUGFS_ATTRIBUTE(fops_u8_ro, debugfs_u8_get, NULL, "%llu\n"); +DEFINE_DEBUGFS_ATTRIBUTE(fops_u8_wo, NULL, debugfs_u8_set, "%llu\n"); + +/** + * debugfs_create_u8 - create a debugfs file that is used to read and write an unsigned 8-bit value + * @name: a pointer to a string containing the name of the file to create. + * @mode: the permission that the file should have + * @parent: a pointer to the parent dentry for this file. This should be a + * directory dentry if set. If this parameter is %NULL, then the + * file will be created in the root of the debugfs filesystem. + * @value: a pointer to the variable that the file should read to and write + * from. + * + * This function creates a file in debugfs with the given name that + * contains the value of the variable @value. If the @mode variable is so + * set, it can be read from, and written to. + */ +void debugfs_create_u8(const char *name, umode_t mode, struct dentry *parent, + u8 *value) +{ + debugfs_create_mode_unsafe(name, mode, parent, value, &fops_u8, + &fops_u8_ro, &fops_u8_wo); +} +EXPORT_SYMBOL_GPL(debugfs_create_u8); + +static int debugfs_u16_set(void *data, u64 val) +{ + *(u16 *)data = val; + return 0; +} +static int debugfs_u16_get(void *data, u64 *val) +{ + *val = *(u16 *)data; + return 0; +} +DEFINE_DEBUGFS_ATTRIBUTE(fops_u16, debugfs_u16_get, debugfs_u16_set, "%llu\n"); +DEFINE_DEBUGFS_ATTRIBUTE(fops_u16_ro, debugfs_u16_get, NULL, "%llu\n"); +DEFINE_DEBUGFS_ATTRIBUTE(fops_u16_wo, NULL, debugfs_u16_set, "%llu\n"); + +/** + * debugfs_create_u16 - create a debugfs file that is used to read and write an unsigned 16-bit value + * @name: a pointer to a string containing the name of the file to create. + * @mode: the permission that the file should have + * @parent: a pointer to the parent dentry for this file. This should be a + * directory dentry if set. If this parameter is %NULL, then the + * file will be created in the root of the debugfs filesystem. + * @value: a pointer to the variable that the file should read to and write + * from. + * + * This function creates a file in debugfs with the given name that + * contains the value of the variable @value. If the @mode variable is so + * set, it can be read from, and written to. + */ +void debugfs_create_u16(const char *name, umode_t mode, struct dentry *parent, + u16 *value) +{ + debugfs_create_mode_unsafe(name, mode, parent, value, &fops_u16, + &fops_u16_ro, &fops_u16_wo); +} +EXPORT_SYMBOL_GPL(debugfs_create_u16); + +static int debugfs_u32_set(void *data, u64 val) +{ + *(u32 *)data = val; + return 0; +} +static int debugfs_u32_get(void *data, u64 *val) +{ + *val = *(u32 *)data; + return 0; +} +DEFINE_DEBUGFS_ATTRIBUTE(fops_u32, debugfs_u32_get, debugfs_u32_set, "%llu\n"); +DEFINE_DEBUGFS_ATTRIBUTE(fops_u32_ro, debugfs_u32_get, NULL, "%llu\n"); +DEFINE_DEBUGFS_ATTRIBUTE(fops_u32_wo, NULL, debugfs_u32_set, "%llu\n"); + +/** + * debugfs_create_u32 - create a debugfs file that is used to read and write an unsigned 32-bit value + * @name: a pointer to a string containing the name of the file to create. + * @mode: the permission that the file should have + * @parent: a pointer to the parent dentry for this file. This should be a + * directory dentry if set. If this parameter is %NULL, then the + * file will be created in the root of the debugfs filesystem. + * @value: a pointer to the variable that the file should read to and write + * from. + * + * This function creates a file in debugfs with the given name that + * contains the value of the variable @value. If the @mode variable is so + * set, it can be read from, and written to. + */ +void debugfs_create_u32(const char *name, umode_t mode, struct dentry *parent, + u32 *value) +{ + debugfs_create_mode_unsafe(name, mode, parent, value, &fops_u32, + &fops_u32_ro, &fops_u32_wo); +} +EXPORT_SYMBOL_GPL(debugfs_create_u32); + +static int debugfs_u64_set(void *data, u64 val) +{ + *(u64 *)data = val; + return 0; +} + +static int debugfs_u64_get(void *data, u64 *val) +{ + *val = *(u64 *)data; + return 0; +} +DEFINE_DEBUGFS_ATTRIBUTE(fops_u64, debugfs_u64_get, debugfs_u64_set, "%llu\n"); +DEFINE_DEBUGFS_ATTRIBUTE(fops_u64_ro, debugfs_u64_get, NULL, "%llu\n"); +DEFINE_DEBUGFS_ATTRIBUTE(fops_u64_wo, NULL, debugfs_u64_set, "%llu\n"); + +/** + * debugfs_create_u64 - create a debugfs file that is used to read and write an unsigned 64-bit value + * @name: a pointer to a string containing the name of the file to create. + * @mode: the permission that the file should have + * @parent: a pointer to the parent dentry for this file. This should be a + * directory dentry if set. If this parameter is %NULL, then the + * file will be created in the root of the debugfs filesystem. + * @value: a pointer to the variable that the file should read to and write + * from. + * + * This function creates a file in debugfs with the given name that + * contains the value of the variable @value. If the @mode variable is so + * set, it can be read from, and written to. + */ +void debugfs_create_u64(const char *name, umode_t mode, struct dentry *parent, + u64 *value) +{ + debugfs_create_mode_unsafe(name, mode, parent, value, &fops_u64, + &fops_u64_ro, &fops_u64_wo); +} +EXPORT_SYMBOL_GPL(debugfs_create_u64); + +static int debugfs_ulong_set(void *data, u64 val) +{ + *(unsigned long *)data = val; + return 0; +} + +static int debugfs_ulong_get(void *data, u64 *val) +{ + *val = *(unsigned long *)data; + return 0; +} +DEFINE_DEBUGFS_ATTRIBUTE(fops_ulong, debugfs_ulong_get, debugfs_ulong_set, + "%llu\n"); +DEFINE_DEBUGFS_ATTRIBUTE(fops_ulong_ro, debugfs_ulong_get, NULL, "%llu\n"); +DEFINE_DEBUGFS_ATTRIBUTE(fops_ulong_wo, NULL, debugfs_ulong_set, "%llu\n"); + +/** + * debugfs_create_ulong - create a debugfs file that is used to read and write + * an unsigned long value. + * @name: a pointer to a string containing the name of the file to create. + * @mode: the permission that the file should have + * @parent: a pointer to the parent dentry for this file. This should be a + * directory dentry if set. If this parameter is %NULL, then the + * file will be created in the root of the debugfs filesystem. + * @value: a pointer to the variable that the file should read to and write + * from. + * + * This function creates a file in debugfs with the given name that + * contains the value of the variable @value. If the @mode variable is so + * set, it can be read from, and written to. + */ +void debugfs_create_ulong(const char *name, umode_t mode, struct dentry *parent, + unsigned long *value) +{ + debugfs_create_mode_unsafe(name, mode, parent, value, &fops_ulong, + &fops_ulong_ro, &fops_ulong_wo); +} +EXPORT_SYMBOL_GPL(debugfs_create_ulong); + +DEFINE_DEBUGFS_ATTRIBUTE(fops_x8, debugfs_u8_get, debugfs_u8_set, "0x%02llx\n"); +DEFINE_DEBUGFS_ATTRIBUTE(fops_x8_ro, debugfs_u8_get, NULL, "0x%02llx\n"); +DEFINE_DEBUGFS_ATTRIBUTE(fops_x8_wo, NULL, debugfs_u8_set, "0x%02llx\n"); + +DEFINE_DEBUGFS_ATTRIBUTE(fops_x16, debugfs_u16_get, debugfs_u16_set, + "0x%04llx\n"); +DEFINE_DEBUGFS_ATTRIBUTE(fops_x16_ro, debugfs_u16_get, NULL, "0x%04llx\n"); +DEFINE_DEBUGFS_ATTRIBUTE(fops_x16_wo, NULL, debugfs_u16_set, "0x%04llx\n"); + +DEFINE_DEBUGFS_ATTRIBUTE(fops_x32, debugfs_u32_get, debugfs_u32_set, + "0x%08llx\n"); +DEFINE_DEBUGFS_ATTRIBUTE(fops_x32_ro, debugfs_u32_get, NULL, "0x%08llx\n"); +DEFINE_DEBUGFS_ATTRIBUTE(fops_x32_wo, NULL, debugfs_u32_set, "0x%08llx\n"); + +DEFINE_DEBUGFS_ATTRIBUTE(fops_x64, debugfs_u64_get, debugfs_u64_set, + "0x%016llx\n"); +DEFINE_DEBUGFS_ATTRIBUTE(fops_x64_ro, debugfs_u64_get, NULL, "0x%016llx\n"); +DEFINE_DEBUGFS_ATTRIBUTE(fops_x64_wo, NULL, debugfs_u64_set, "0x%016llx\n"); + +/* + * debugfs_create_x{8,16,32,64} - create a debugfs file that is used to read and write an unsigned {8,16,32,64}-bit value + * + * These functions are exactly the same as the above functions (but use a hex + * output for the decimal challenged). For details look at the above unsigned + * decimal functions. + */ + +/** + * debugfs_create_x8 - create a debugfs file that is used to read and write an unsigned 8-bit value + * @name: a pointer to a string containing the name of the file to create. + * @mode: the permission that the file should have + * @parent: a pointer to the parent dentry for this file. This should be a + * directory dentry if set. If this parameter is %NULL, then the + * file will be created in the root of the debugfs filesystem. + * @value: a pointer to the variable that the file should read to and write + * from. + */ +void debugfs_create_x8(const char *name, umode_t mode, struct dentry *parent, + u8 *value) +{ + debugfs_create_mode_unsafe(name, mode, parent, value, &fops_x8, + &fops_x8_ro, &fops_x8_wo); +} +EXPORT_SYMBOL_GPL(debugfs_create_x8); + +/** + * debugfs_create_x16 - create a debugfs file that is used to read and write an unsigned 16-bit value + * @name: a pointer to a string containing the name of the file to create. + * @mode: the permission that the file should have + * @parent: a pointer to the parent dentry for this file. This should be a + * directory dentry if set. If this parameter is %NULL, then the + * file will be created in the root of the debugfs filesystem. + * @value: a pointer to the variable that the file should read to and write + * from. + */ +void debugfs_create_x16(const char *name, umode_t mode, struct dentry *parent, + u16 *value) +{ + debugfs_create_mode_unsafe(name, mode, parent, value, &fops_x16, + &fops_x16_ro, &fops_x16_wo); +} +EXPORT_SYMBOL_GPL(debugfs_create_x16); + +/** + * debugfs_create_x32 - create a debugfs file that is used to read and write an unsigned 32-bit value + * @name: a pointer to a string containing the name of the file to create. + * @mode: the permission that the file should have + * @parent: a pointer to the parent dentry for this file. This should be a + * directory dentry if set. If this parameter is %NULL, then the + * file will be created in the root of the debugfs filesystem. + * @value: a pointer to the variable that the file should read to and write + * from. + */ +void debugfs_create_x32(const char *name, umode_t mode, struct dentry *parent, + u32 *value) +{ + debugfs_create_mode_unsafe(name, mode, parent, value, &fops_x32, + &fops_x32_ro, &fops_x32_wo); +} +EXPORT_SYMBOL_GPL(debugfs_create_x32); + +/** + * debugfs_create_x64 - create a debugfs file that is used to read and write an unsigned 64-bit value + * @name: a pointer to a string containing the name of the file to create. + * @mode: the permission that the file should have + * @parent: a pointer to the parent dentry for this file. This should be a + * directory dentry if set. If this parameter is %NULL, then the + * file will be created in the root of the debugfs filesystem. + * @value: a pointer to the variable that the file should read to and write + * from. + */ +void debugfs_create_x64(const char *name, umode_t mode, struct dentry *parent, + u64 *value) +{ + debugfs_create_mode_unsafe(name, mode, parent, value, &fops_x64, + &fops_x64_ro, &fops_x64_wo); +} +EXPORT_SYMBOL_GPL(debugfs_create_x64); + + +static int debugfs_size_t_set(void *data, u64 val) +{ + *(size_t *)data = val; + return 0; +} +static int debugfs_size_t_get(void *data, u64 *val) +{ + *val = *(size_t *)data; + return 0; +} +DEFINE_DEBUGFS_ATTRIBUTE(fops_size_t, debugfs_size_t_get, debugfs_size_t_set, + "%llu\n"); /* %llu and %zu are more or less the same */ +DEFINE_DEBUGFS_ATTRIBUTE(fops_size_t_ro, debugfs_size_t_get, NULL, "%llu\n"); +DEFINE_DEBUGFS_ATTRIBUTE(fops_size_t_wo, NULL, debugfs_size_t_set, "%llu\n"); + +/** + * debugfs_create_size_t - create a debugfs file that is used to read and write an size_t value + * @name: a pointer to a string containing the name of the file to create. + * @mode: the permission that the file should have + * @parent: a pointer to the parent dentry for this file. This should be a + * directory dentry if set. If this parameter is %NULL, then the + * file will be created in the root of the debugfs filesystem. + * @value: a pointer to the variable that the file should read to and write + * from. + */ +void debugfs_create_size_t(const char *name, umode_t mode, + struct dentry *parent, size_t *value) +{ + debugfs_create_mode_unsafe(name, mode, parent, value, &fops_size_t, + &fops_size_t_ro, &fops_size_t_wo); +} +EXPORT_SYMBOL_GPL(debugfs_create_size_t); + +static int debugfs_atomic_t_set(void *data, u64 val) +{ + atomic_set((atomic_t *)data, val); + return 0; +} +static int debugfs_atomic_t_get(void *data, u64 *val) +{ + *val = atomic_read((atomic_t *)data); + return 0; +} +DEFINE_DEBUGFS_ATTRIBUTE_SIGNED(fops_atomic_t, debugfs_atomic_t_get, + debugfs_atomic_t_set, "%lld\n"); +DEFINE_DEBUGFS_ATTRIBUTE_SIGNED(fops_atomic_t_ro, debugfs_atomic_t_get, NULL, + "%lld\n"); +DEFINE_DEBUGFS_ATTRIBUTE_SIGNED(fops_atomic_t_wo, NULL, debugfs_atomic_t_set, + "%lld\n"); + +/** + * debugfs_create_atomic_t - create a debugfs file that is used to read and + * write an atomic_t value + * @name: a pointer to a string containing the name of the file to create. + * @mode: the permission that the file should have + * @parent: a pointer to the parent dentry for this file. This should be a + * directory dentry if set. If this parameter is %NULL, then the + * file will be created in the root of the debugfs filesystem. + * @value: a pointer to the variable that the file should read to and write + * from. + */ +void debugfs_create_atomic_t(const char *name, umode_t mode, + struct dentry *parent, atomic_t *value) +{ + debugfs_create_mode_unsafe(name, mode, parent, value, &fops_atomic_t, + &fops_atomic_t_ro, &fops_atomic_t_wo); +} +EXPORT_SYMBOL_GPL(debugfs_create_atomic_t); + +ssize_t debugfs_read_file_bool(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + char buf[2]; + bool val; + int r; + struct dentry *dentry = F_DENTRY(file); + + r = debugfs_file_get(dentry); + if (unlikely(r)) + return r; + val = *(bool *)file->private_data; + debugfs_file_put(dentry); + + if (val) + buf[0] = 'Y'; + else + buf[0] = 'N'; + buf[1] = '\n'; + return simple_read_from_buffer(user_buf, count, ppos, buf, 2); +} +EXPORT_SYMBOL_GPL(debugfs_read_file_bool); + +ssize_t debugfs_write_file_bool(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + bool bv; + int r; + bool *val = file->private_data; + struct dentry *dentry = F_DENTRY(file); + + r = kstrtobool_from_user(user_buf, count, &bv); + if (!r) { + r = debugfs_file_get(dentry); + if (unlikely(r)) + return r; + *val = bv; + debugfs_file_put(dentry); + } + + return count; +} +EXPORT_SYMBOL_GPL(debugfs_write_file_bool); + +static const struct file_operations fops_bool = { + .read = debugfs_read_file_bool, + .write = debugfs_write_file_bool, + .open = simple_open, + .llseek = default_llseek, +}; + +static const struct file_operations fops_bool_ro = { + .read = debugfs_read_file_bool, + .open = simple_open, + .llseek = default_llseek, +}; + +static const struct file_operations fops_bool_wo = { + .write = debugfs_write_file_bool, + .open = simple_open, + .llseek = default_llseek, +}; + +/** + * debugfs_create_bool - create a debugfs file that is used to read and write a boolean value + * @name: a pointer to a string containing the name of the file to create. + * @mode: the permission that the file should have + * @parent: a pointer to the parent dentry for this file. This should be a + * directory dentry if set. If this parameter is %NULL, then the + * file will be created in the root of the debugfs filesystem. + * @value: a pointer to the variable that the file should read to and write + * from. + * + * This function creates a file in debugfs with the given name that + * contains the value of the variable @value. If the @mode variable is so + * set, it can be read from, and written to. + */ +void debugfs_create_bool(const char *name, umode_t mode, struct dentry *parent, + bool *value) +{ + debugfs_create_mode_unsafe(name, mode, parent, value, &fops_bool, + &fops_bool_ro, &fops_bool_wo); +} +EXPORT_SYMBOL_GPL(debugfs_create_bool); + +ssize_t debugfs_read_file_str(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct dentry *dentry = F_DENTRY(file); + char *str, *copy = NULL; + int copy_len, len; + ssize_t ret; + + ret = debugfs_file_get(dentry); + if (unlikely(ret)) + return ret; + + str = *(char **)file->private_data; + len = strlen(str) + 1; + copy = kmalloc(len, GFP_KERNEL); + if (!copy) { + debugfs_file_put(dentry); + return -ENOMEM; + } + + copy_len = strscpy(copy, str, len); + debugfs_file_put(dentry); + if (copy_len < 0) { + kfree(copy); + return copy_len; + } + + copy[copy_len] = '\n'; + + ret = simple_read_from_buffer(user_buf, count, ppos, copy, len); + kfree(copy); + + return ret; +} + +static ssize_t debugfs_write_file_str(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct dentry *dentry = F_DENTRY(file); + char *old, *new = NULL; + int pos = *ppos; + int r; + + r = debugfs_file_get(dentry); + if (unlikely(r)) + return r; + + old = *(char **)file->private_data; + + /* only allow strict concatenation */ + r = -EINVAL; + if (pos && pos != strlen(old)) + goto error; + + r = -E2BIG; + if (pos + count + 1 > PAGE_SIZE) + goto error; + + r = -ENOMEM; + new = kmalloc(pos + count + 1, GFP_KERNEL); + if (!new) + goto error; + + if (pos) + memcpy(new, old, pos); + + r = -EFAULT; + if (copy_from_user(new + pos, user_buf, count)) + goto error; + + new[pos + count] = '\0'; + strim(new); + + rcu_assign_pointer(*(char __rcu **)file->private_data, new); + synchronize_rcu(); + kfree(old); + + debugfs_file_put(dentry); + return count; + +error: + kfree(new); + debugfs_file_put(dentry); + return r; +} + +static const struct file_operations fops_str = { + .read = debugfs_read_file_str, + .write = debugfs_write_file_str, + .open = simple_open, + .llseek = default_llseek, +}; + +static const struct file_operations fops_str_ro = { + .read = debugfs_read_file_str, + .open = simple_open, + .llseek = default_llseek, +}; + +static const struct file_operations fops_str_wo = { + .write = debugfs_write_file_str, + .open = simple_open, + .llseek = default_llseek, +}; + +/** + * debugfs_create_str - create a debugfs file that is used to read and write a string value + * @name: a pointer to a string containing the name of the file to create. + * @mode: the permission that the file should have + * @parent: a pointer to the parent dentry for this file. This should be a + * directory dentry if set. If this parameter is %NULL, then the + * file will be created in the root of the debugfs filesystem. + * @value: a pointer to the variable that the file should read to and write + * from. This pointer and the string it points to must not be %NULL. + * + * This function creates a file in debugfs with the given name that + * contains the value of the variable @value. If the @mode variable is so + * set, it can be read from, and written to. + */ +void debugfs_create_str(const char *name, umode_t mode, + struct dentry *parent, char **value) +{ + if (WARN_ON(!value || !*value)) + return; + + debugfs_create_mode_unsafe(name, mode, parent, value, &fops_str, + &fops_str_ro, &fops_str_wo); +} +EXPORT_SYMBOL_GPL(debugfs_create_str); + +static ssize_t read_file_blob(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct debugfs_blob_wrapper *blob = file->private_data; + struct dentry *dentry = F_DENTRY(file); + ssize_t r; + + r = debugfs_file_get(dentry); + if (unlikely(r)) + return r; + r = simple_read_from_buffer(user_buf, count, ppos, blob->data, + blob->size); + debugfs_file_put(dentry); + return r; +} + +static ssize_t write_file_blob(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct debugfs_blob_wrapper *blob = file->private_data; + struct dentry *dentry = F_DENTRY(file); + ssize_t r; + + r = debugfs_file_get(dentry); + if (unlikely(r)) + return r; + r = simple_write_to_buffer(blob->data, blob->size, ppos, user_buf, + count); + + debugfs_file_put(dentry); + return r; +} + +static const struct file_operations fops_blob = { + .read = read_file_blob, + .write = write_file_blob, + .open = simple_open, + .llseek = default_llseek, +}; + +/** + * debugfs_create_blob - create a debugfs file that is used to read and write + * a binary blob + * @name: a pointer to a string containing the name of the file to create. + * @mode: the permission that the file should have + * @parent: a pointer to the parent dentry for this file. This should be a + * directory dentry if set. If this parameter is %NULL, then the + * file will be created in the root of the debugfs filesystem. + * @blob: a pointer to a struct debugfs_blob_wrapper which contains a pointer + * to the blob data and the size of the data. + * + * This function creates a file in debugfs with the given name that exports + * @blob->data as a binary blob. If the @mode variable is so set it can be + * read from and written to. + * + * This function will return a pointer to a dentry if it succeeds. This + * pointer must be passed to the debugfs_remove() function when the file is + * to be removed (no automatic cleanup happens if your module is unloaded, + * you are responsible here.) If an error occurs, ERR_PTR(-ERROR) will be + * returned. + * + * If debugfs is not enabled in the kernel, the value ERR_PTR(-ENODEV) will + * be returned. + */ +struct dentry *debugfs_create_blob(const char *name, umode_t mode, + struct dentry *parent, + struct debugfs_blob_wrapper *blob) +{ + return debugfs_create_file_unsafe(name, mode & 0644, parent, blob, &fops_blob); +} +EXPORT_SYMBOL_GPL(debugfs_create_blob); + +static size_t u32_format_array(char *buf, size_t bufsize, + u32 *array, int array_size) +{ + size_t ret = 0; + + while (--array_size >= 0) { + size_t len; + char term = array_size ? ' ' : '\n'; + + len = snprintf(buf, bufsize, "%u%c", *array++, term); + ret += len; + + buf += len; + bufsize -= len; + } + return ret; +} + +static int u32_array_open(struct inode *inode, struct file *file) +{ + struct debugfs_u32_array *data = inode->i_private; + int size, elements = data->n_elements; + char *buf; + + /* + * Max size: + * - 10 digits + ' '/'\n' = 11 bytes per number + * - terminating NUL character + */ + size = elements*11; + buf = kmalloc(size+1, GFP_KERNEL); + if (!buf) + return -ENOMEM; + buf[size] = 0; + + file->private_data = buf; + u32_format_array(buf, size, data->array, data->n_elements); + + return nonseekable_open(inode, file); +} + +static ssize_t u32_array_read(struct file *file, char __user *buf, size_t len, + loff_t *ppos) +{ + size_t size = strlen(file->private_data); + + return simple_read_from_buffer(buf, len, ppos, + file->private_data, size); +} + +static int u32_array_release(struct inode *inode, struct file *file) +{ + kfree(file->private_data); + + return 0; +} + +static const struct file_operations u32_array_fops = { + .owner = THIS_MODULE, + .open = u32_array_open, + .release = u32_array_release, + .read = u32_array_read, +}; + +/** + * debugfs_create_u32_array - create a debugfs file that is used to read u32 + * array. + * @name: a pointer to a string containing the name of the file to create. + * @mode: the permission that the file should have. + * @parent: a pointer to the parent dentry for this file. This should be a + * directory dentry if set. If this parameter is %NULL, then the + * file will be created in the root of the debugfs filesystem. + * @array: wrapper struct containing data pointer and size of the array. + * + * This function creates a file in debugfs with the given name that exports + * @array as data. If the @mode variable is so set it can be read from. + * Writing is not supported. Seek within the file is also not supported. + * Once array is created its size can not be changed. + */ +void debugfs_create_u32_array(const char *name, umode_t mode, + struct dentry *parent, + struct debugfs_u32_array *array) +{ + debugfs_create_file_unsafe(name, mode, parent, array, &u32_array_fops); +} +EXPORT_SYMBOL_GPL(debugfs_create_u32_array); + +#ifdef CONFIG_HAS_IOMEM + +/* + * The regset32 stuff is used to print 32-bit registers using the + * seq_file utilities. We offer printing a register set in an already-opened + * sequential file or create a debugfs file that only prints a regset32. + */ + +/** + * debugfs_print_regs32 - use seq_print to describe a set of registers + * @s: the seq_file structure being used to generate output + * @regs: an array if struct debugfs_reg32 structures + * @nregs: the length of the above array + * @base: the base address to be used in reading the registers + * @prefix: a string to be prefixed to every output line + * + * This function outputs a text block describing the current values of + * some 32-bit hardware registers. It is meant to be used within debugfs + * files based on seq_file that need to show registers, intermixed with other + * information. The prefix argument may be used to specify a leading string, + * because some peripherals have several blocks of identical registers, + * for example configuration of dma channels + */ +void debugfs_print_regs32(struct seq_file *s, const struct debugfs_reg32 *regs, + int nregs, void __iomem *base, char *prefix) +{ + int i; + + for (i = 0; i < nregs; i++, regs++) { + if (prefix) + seq_printf(s, "%s", prefix); + seq_printf(s, "%s = 0x%08x\n", regs->name, + readl(base + regs->offset)); + if (seq_has_overflowed(s)) + break; + } +} +EXPORT_SYMBOL_GPL(debugfs_print_regs32); + +static int debugfs_regset32_show(struct seq_file *s, void *data) +{ + struct debugfs_regset32 *regset = s->private; + + if (regset->dev) + pm_runtime_get_sync(regset->dev); + + debugfs_print_regs32(s, regset->regs, regset->nregs, regset->base, ""); + + if (regset->dev) + pm_runtime_put(regset->dev); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(debugfs_regset32); + +/** + * debugfs_create_regset32 - create a debugfs file that returns register values + * @name: a pointer to a string containing the name of the file to create. + * @mode: the permission that the file should have + * @parent: a pointer to the parent dentry for this file. This should be a + * directory dentry if set. If this parameter is %NULL, then the + * file will be created in the root of the debugfs filesystem. + * @regset: a pointer to a struct debugfs_regset32, which contains a pointer + * to an array of register definitions, the array size and the base + * address where the register bank is to be found. + * + * This function creates a file in debugfs with the given name that reports + * the names and values of a set of 32-bit registers. If the @mode variable + * is so set it can be read from. Writing is not supported. + */ +void debugfs_create_regset32(const char *name, umode_t mode, + struct dentry *parent, + struct debugfs_regset32 *regset) +{ + debugfs_create_file(name, mode, parent, regset, &debugfs_regset32_fops); +} +EXPORT_SYMBOL_GPL(debugfs_create_regset32); + +#endif /* CONFIG_HAS_IOMEM */ + +struct debugfs_devm_entry { + int (*read)(struct seq_file *seq, void *data); + struct device *dev; +}; + +static int debugfs_devm_entry_open(struct inode *inode, struct file *f) +{ + struct debugfs_devm_entry *entry = inode->i_private; + + return single_open(f, entry->read, entry->dev); +} + +static const struct file_operations debugfs_devm_entry_ops = { + .owner = THIS_MODULE, + .open = debugfs_devm_entry_open, + .release = single_release, + .read = seq_read, + .llseek = seq_lseek +}; + +/** + * debugfs_create_devm_seqfile - create a debugfs file that is bound to device. + * + * @dev: device related to this debugfs file. + * @name: name of the debugfs file. + * @parent: a pointer to the parent dentry for this file. This should be a + * directory dentry if set. If this parameter is %NULL, then the + * file will be created in the root of the debugfs filesystem. + * @read_fn: function pointer called to print the seq_file content. + */ +void debugfs_create_devm_seqfile(struct device *dev, const char *name, + struct dentry *parent, + int (*read_fn)(struct seq_file *s, void *data)) +{ + struct debugfs_devm_entry *entry; + + if (IS_ERR(parent)) + return; + + entry = devm_kzalloc(dev, sizeof(*entry), GFP_KERNEL); + if (!entry) + return; + + entry->read = read_fn; + entry->dev = dev; + + debugfs_create_file(name, S_IRUGO, parent, entry, + &debugfs_devm_entry_ops); +} +EXPORT_SYMBOL_GPL(debugfs_create_devm_seqfile); diff --git a/fs/debugfs/inode.c b/fs/debugfs/inode.c new file mode 100644 index 000000000..e054e6291 --- /dev/null +++ b/fs/debugfs/inode.c @@ -0,0 +1,932 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * inode.c - part of debugfs, a tiny little debug file system + * + * Copyright (C) 2004,2019 Greg Kroah-Hartman + * Copyright (C) 2004 IBM Inc. + * Copyright (C) 2019 Linux Foundation + * + * debugfs is for people to use instead of /proc or /sys. + * See ./Documentation/core-api/kernel-api.rst for more details. + */ + +#define pr_fmt(fmt) "debugfs: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +#define DEBUGFS_DEFAULT_MODE 0700 + +static struct vfsmount *debugfs_mount; +static int debugfs_mount_count; +static bool debugfs_registered; +static bool debugfs_enabled __ro_after_init = IS_ENABLED(CONFIG_DEBUG_FS_ALLOW_ALL); + +/* + * Don't allow access attributes to be changed whilst the kernel is locked down + * so that we can use the file mode as part of a heuristic to determine whether + * to lock down individual files. + */ +static int debugfs_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, struct iattr *ia) +{ + int ret; + + if (ia->ia_valid & (ATTR_MODE | ATTR_UID | ATTR_GID)) { + ret = security_locked_down(LOCKDOWN_DEBUGFS); + if (ret) + return ret; + } + return simple_setattr(&nop_mnt_idmap, dentry, ia); +} + +static const struct inode_operations debugfs_file_inode_operations = { + .setattr = debugfs_setattr, +}; +static const struct inode_operations debugfs_dir_inode_operations = { + .lookup = simple_lookup, + .setattr = debugfs_setattr, +}; +static const struct inode_operations debugfs_symlink_inode_operations = { + .get_link = simple_get_link, + .setattr = debugfs_setattr, +}; + +static struct inode *debugfs_get_inode(struct super_block *sb) +{ + struct inode *inode = new_inode(sb); + if (inode) { + inode->i_ino = get_next_ino(); + simple_inode_init_ts(inode); + } + return inode; +} + +struct debugfs_fs_info { + kuid_t uid; + kgid_t gid; + umode_t mode; + /* Opt_* bitfield. */ + unsigned int opts; +}; + +enum { + Opt_uid, + Opt_gid, + Opt_mode, + Opt_source, +}; + +static const struct fs_parameter_spec debugfs_param_specs[] = { + fsparam_gid ("gid", Opt_gid), + fsparam_u32oct ("mode", Opt_mode), + fsparam_uid ("uid", Opt_uid), + fsparam_string ("source", Opt_source), + {} +}; + +static int debugfs_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct debugfs_fs_info *opts = fc->s_fs_info; + struct fs_parse_result result; + int opt; + + opt = fs_parse(fc, debugfs_param_specs, param, &result); + if (opt < 0) { + /* + * We might like to report bad mount options here; but + * traditionally debugfs has ignored all mount options + */ + if (opt == -ENOPARAM) + return 0; + + return opt; + } + + switch (opt) { + case Opt_uid: + opts->uid = result.uid; + break; + case Opt_gid: + opts->gid = result.gid; + break; + case Opt_mode: + opts->mode = result.uint_32 & S_IALLUGO; + break; + case Opt_source: + if (fc->source) + return invalfc(fc, "Multiple sources specified"); + fc->source = param->string; + param->string = NULL; + break; + /* + * We might like to report bad mount options here; + * but traditionally debugfs has ignored all mount options + */ + } + + opts->opts |= BIT(opt); + + return 0; +} + +static void _debugfs_apply_options(struct super_block *sb, bool remount) +{ + struct debugfs_fs_info *fsi = sb->s_fs_info; + struct inode *inode = d_inode(sb->s_root); + + /* + * On remount, only reset mode/uid/gid if they were provided as mount + * options. + */ + + if (!remount || fsi->opts & BIT(Opt_mode)) { + inode->i_mode &= ~S_IALLUGO; + inode->i_mode |= fsi->mode; + } + + if (!remount || fsi->opts & BIT(Opt_uid)) + inode->i_uid = fsi->uid; + + if (!remount || fsi->opts & BIT(Opt_gid)) + inode->i_gid = fsi->gid; +} + +static void debugfs_apply_options(struct super_block *sb) +{ + _debugfs_apply_options(sb, false); +} + +static void debugfs_apply_options_remount(struct super_block *sb) +{ + _debugfs_apply_options(sb, true); +} + +static int debugfs_reconfigure(struct fs_context *fc) +{ + struct super_block *sb = fc->root->d_sb; + struct debugfs_fs_info *sb_opts = sb->s_fs_info; + struct debugfs_fs_info *new_opts = fc->s_fs_info; + + if (!new_opts) + return 0; + + sync_filesystem(sb); + + /* structure copy of new mount options to sb */ + *sb_opts = *new_opts; + debugfs_apply_options_remount(sb); + + return 0; +} + +static int debugfs_show_options(struct seq_file *m, struct dentry *root) +{ + struct debugfs_fs_info *fsi = root->d_sb->s_fs_info; + + if (!uid_eq(fsi->uid, GLOBAL_ROOT_UID)) + seq_printf(m, ",uid=%u", + from_kuid_munged(&init_user_ns, fsi->uid)); + if (!gid_eq(fsi->gid, GLOBAL_ROOT_GID)) + seq_printf(m, ",gid=%u", + from_kgid_munged(&init_user_ns, fsi->gid)); + if (fsi->mode != DEBUGFS_DEFAULT_MODE) + seq_printf(m, ",mode=%o", fsi->mode); + + return 0; +} + +static struct kmem_cache *debugfs_inode_cachep __ro_after_init; + +static void init_once(void *foo) +{ + struct debugfs_inode_info *info = foo; + inode_init_once(&info->vfs_inode); +} + +static struct inode *debugfs_alloc_inode(struct super_block *sb) +{ + struct debugfs_inode_info *info; + info = alloc_inode_sb(sb, debugfs_inode_cachep, GFP_KERNEL); + if (!info) + return NULL; + return &info->vfs_inode; +} + +static void debugfs_free_inode(struct inode *inode) +{ + if (S_ISLNK(inode->i_mode)) + kfree(inode->i_link); + kmem_cache_free(debugfs_inode_cachep, DEBUGFS_I(inode)); +} + +static const struct super_operations debugfs_super_operations = { + .statfs = simple_statfs, + .show_options = debugfs_show_options, + .alloc_inode = debugfs_alloc_inode, + .free_inode = debugfs_free_inode, +}; + +static void debugfs_release_dentry(struct dentry *dentry) +{ + struct debugfs_fsdata *fsd = dentry->d_fsdata; + + if (fsd) { + WARN_ON(!list_empty(&fsd->cancellations)); + mutex_destroy(&fsd->cancellations_mtx); + } + kfree(fsd); +} + +static struct vfsmount *debugfs_automount(struct path *path) +{ + struct inode *inode = path->dentry->d_inode; + + return DEBUGFS_I(inode)->automount(path->dentry, inode->i_private); +} + +static const struct dentry_operations debugfs_dops = { + .d_release = debugfs_release_dentry, + .d_automount = debugfs_automount, +}; + +static int debugfs_fill_super(struct super_block *sb, struct fs_context *fc) +{ + static const struct tree_descr debug_files[] = {{""}}; + int err; + + err = simple_fill_super(sb, DEBUGFS_MAGIC, debug_files); + if (err) + return err; + + sb->s_op = &debugfs_super_operations; + set_default_d_op(sb, &debugfs_dops); + sb->s_d_flags |= DCACHE_DONTCACHE; + + debugfs_apply_options(sb); + + return 0; +} + +static int debugfs_get_tree(struct fs_context *fc) +{ + int err; + + err = get_tree_single(fc, debugfs_fill_super); + if (err) + return err; + + return debugfs_reconfigure(fc); +} + +static void debugfs_free_fc(struct fs_context *fc) +{ + kfree(fc->s_fs_info); +} + +static const struct fs_context_operations debugfs_context_ops = { + .free = debugfs_free_fc, + .parse_param = debugfs_parse_param, + .get_tree = debugfs_get_tree, + .reconfigure = debugfs_reconfigure, +}; + +static int debugfs_init_fs_context(struct fs_context *fc) +{ + struct debugfs_fs_info *fsi; + + fsi = kzalloc_obj(struct debugfs_fs_info); + if (!fsi) + return -ENOMEM; + + fsi->mode = DEBUGFS_DEFAULT_MODE; + + fc->s_fs_info = fsi; + fc->ops = &debugfs_context_ops; + return 0; +} + +static struct file_system_type debug_fs_type = { + .owner = THIS_MODULE, + .name = "debugfs", + .init_fs_context = debugfs_init_fs_context, + .parameters = debugfs_param_specs, + .kill_sb = kill_anon_super, +}; +MODULE_ALIAS_FS("debugfs"); + +/** + * debugfs_lookup() - look up an existing debugfs file + * @name: a pointer to a string containing the name of the file to look up. + * @parent: a pointer to the parent dentry of the file. + * + * This function will return a pointer to a dentry if it succeeds. If the file + * doesn't exist or an error occurs, %NULL will be returned. The returned + * dentry must be passed to dput() when it is no longer needed. + * + * If debugfs is not enabled in the kernel, the value -%ENODEV will be + * returned. + */ +struct dentry *debugfs_lookup(const char *name, struct dentry *parent) +{ + struct dentry *dentry; + + if (!debugfs_initialized() || IS_ERR_OR_NULL(name) || IS_ERR(parent)) + return NULL; + + if (!parent) + parent = debugfs_mount->mnt_root; + + dentry = lookup_noperm_positive_unlocked(&QSTR(name), parent); + if (IS_ERR(dentry)) + return NULL; + return dentry; +} +EXPORT_SYMBOL_GPL(debugfs_lookup); + +static struct dentry *debugfs_start_creating(const char *name, + struct dentry *parent) +{ + struct dentry *dentry; + int error; + + if (!debugfs_enabled) + return ERR_PTR(-EPERM); + + if (!debugfs_initialized()) { + pr_err("Unable to create file '%s', debugfs is not initialized yet\n", + name); + return ERR_PTR(-ENOENT); + } + + pr_debug("creating file '%s'\n", name); + + if (IS_ERR(parent)) + return parent; + + error = simple_pin_fs(&debug_fs_type, &debugfs_mount, + &debugfs_mount_count); + if (error) { + pr_err("Unable to pin filesystem for file '%s'\n", name); + return ERR_PTR(error); + } + + /* If the parent is not specified, we create it in the root. + * We need the root dentry to do this, which is in the super + * block. A pointer to that is in the struct vfsmount that we + * have around. + */ + if (!parent) + parent = debugfs_mount->mnt_root; + + dentry = simple_start_creating(parent, name); + if (IS_ERR(dentry)) { + if (dentry == ERR_PTR(-EEXIST)) + pr_err("'%s' already exists in '%pd'\n", name, parent); + simple_release_fs(&debugfs_mount, &debugfs_mount_count); + } + return dentry; +} + +static struct dentry *debugfs_failed_creating(struct dentry *dentry) +{ + simple_done_creating(dentry); + simple_release_fs(&debugfs_mount, &debugfs_mount_count); + return ERR_PTR(-ENOMEM); +} + +static struct dentry *debugfs_end_creating(struct dentry *dentry) +{ + simple_done_creating(dentry); + return dentry; // borrowed +} + +static struct dentry *__debugfs_create_file(const char *name, umode_t mode, + struct dentry *parent, void *data, + const void *aux, + const struct file_operations *proxy_fops, + const void *real_fops) +{ + struct dentry *dentry; + struct inode *inode; + + if (!(mode & S_IFMT)) + mode |= S_IFREG; + BUG_ON(!S_ISREG(mode)); + dentry = debugfs_start_creating(name, parent); + + if (IS_ERR(dentry)) + return dentry; + + inode = debugfs_get_inode(dentry->d_sb); + if (unlikely(!inode)) { + pr_err("out of free dentries, can not create file '%s'\n", + name); + return debugfs_failed_creating(dentry); + } + + inode->i_mode = mode; + inode->i_private = data; + + inode->i_op = &debugfs_file_inode_operations; + if (!real_fops) + proxy_fops = &debugfs_noop_file_operations; + inode->i_fop = proxy_fops; + DEBUGFS_I(inode)->raw = real_fops; + DEBUGFS_I(inode)->aux = (void *)aux; + + d_make_persistent(dentry, inode); + fsnotify_create(d_inode(dentry->d_parent), dentry); + return debugfs_end_creating(dentry); +} + +struct dentry *debugfs_create_file_full(const char *name, umode_t mode, + struct dentry *parent, void *data, + const void *aux, + const struct file_operations *fops) +{ + return __debugfs_create_file(name, mode, parent, data, aux, + &debugfs_full_proxy_file_operations, + fops); +} +EXPORT_SYMBOL_GPL(debugfs_create_file_full); + +struct dentry *debugfs_create_file_short(const char *name, umode_t mode, + struct dentry *parent, void *data, + const void *aux, + const struct debugfs_short_fops *fops) +{ + return __debugfs_create_file(name, mode, parent, data, aux, + &debugfs_full_short_proxy_file_operations, + fops); +} +EXPORT_SYMBOL_GPL(debugfs_create_file_short); + +/** + * debugfs_create_file_unsafe - create a file in the debugfs filesystem + * @name: a pointer to a string containing the name of the file to create. + * @mode: the permission that the file should have. + * @parent: a pointer to the parent dentry for this file. This should be a + * directory dentry if set. If this parameter is NULL, then the + * file will be created in the root of the debugfs filesystem. + * @data: a pointer to something that the caller will want to get to later + * on. The inode.i_private pointer will point to this value on + * the open() call. + * @fops: a pointer to a struct file_operations that should be used for + * this file. + * + * debugfs_create_file_unsafe() is completely analogous to + * debugfs_create_file(), the only difference being that the fops + * handed it will not get protected against file removals by the + * debugfs core. + * + * It is your responsibility to protect your struct file_operation + * methods against file removals by means of debugfs_file_get() + * and debugfs_file_put(). ->open() is still protected by + * debugfs though. + * + * Any struct file_operations defined by means of + * DEFINE_DEBUGFS_ATTRIBUTE() is protected against file removals and + * thus, may be used here. + */ +struct dentry *debugfs_create_file_unsafe(const char *name, umode_t mode, + struct dentry *parent, void *data, + const struct file_operations *fops) +{ + + return __debugfs_create_file(name, mode, parent, data, NULL, + &debugfs_open_proxy_file_operations, + fops); +} +EXPORT_SYMBOL_GPL(debugfs_create_file_unsafe); + +/** + * debugfs_create_file_size - create a file in the debugfs filesystem + * @name: a pointer to a string containing the name of the file to create. + * @mode: the permission that the file should have. + * @parent: a pointer to the parent dentry for this file. This should be a + * directory dentry if set. If this parameter is NULL, then the + * file will be created in the root of the debugfs filesystem. + * @data: a pointer to something that the caller will want to get to later + * on. The inode.i_private pointer will point to this value on + * the open() call. + * @fops: a pointer to a struct file_operations that should be used for + * this file. + * @file_size: initial file size + * + * This is the basic "create a file" function for debugfs. It allows for a + * wide range of flexibility in creating a file, or a directory (if you want + * to create a directory, the debugfs_create_dir() function is + * recommended to be used instead.) + */ +void debugfs_create_file_size(const char *name, umode_t mode, + struct dentry *parent, void *data, + const struct file_operations *fops, + loff_t file_size) +{ + struct dentry *de = debugfs_create_file(name, mode, parent, data, fops); + + if (!IS_ERR(de)) + d_inode(de)->i_size = file_size; +} +EXPORT_SYMBOL_GPL(debugfs_create_file_size); + +/** + * debugfs_create_dir - create a directory in the debugfs filesystem + * @name: a pointer to a string containing the name of the directory to + * create. + * @parent: a pointer to the parent dentry for this file. This should be a + * directory dentry if set. If this parameter is NULL, then the + * directory will be created in the root of the debugfs filesystem. + * + * This function creates a directory in debugfs with the given name. + * + * This function will return a pointer to a dentry if it succeeds. This + * pointer must be passed to the debugfs_remove() function when the file is + * to be removed (no automatic cleanup happens if your module is unloaded, + * you are responsible here.) If an error occurs, ERR_PTR(-ERROR) will be + * returned. + * + * If debugfs is not enabled in the kernel, the value -%ENODEV will be + * returned. + * + * NOTE: it's expected that most callers should _ignore_ the errors returned + * by this function. Other debugfs functions handle the fact that the "dentry" + * passed to them could be an error and they don't crash in that case. + * Drivers should generally work fine even if debugfs fails to init anyway. + */ +struct dentry *debugfs_create_dir(const char *name, struct dentry *parent) +{ + struct dentry *dentry = debugfs_start_creating(name, parent); + struct inode *inode; + + if (IS_ERR(dentry)) + return dentry; + + inode = debugfs_get_inode(dentry->d_sb); + if (unlikely(!inode)) { + pr_err("out of free dentries, can not create directory '%s'\n", + name); + return debugfs_failed_creating(dentry); + } + + inode->i_mode = S_IFDIR | S_IRWXU | S_IRUGO | S_IXUGO; + inode->i_op = &debugfs_dir_inode_operations; + inode->i_fop = &simple_dir_operations; + + /* directory inodes start off with i_nlink == 2 (for "." entry) */ + inc_nlink(inode); + d_make_persistent(dentry, inode); + inc_nlink(d_inode(dentry->d_parent)); + fsnotify_mkdir(d_inode(dentry->d_parent), dentry); + return debugfs_end_creating(dentry); +} +EXPORT_SYMBOL_GPL(debugfs_create_dir); + +/** + * debugfs_create_automount - create automount point in the debugfs filesystem + * @name: a pointer to a string containing the name of the file to create. + * @parent: a pointer to the parent dentry for this file. This should be a + * directory dentry if set. If this parameter is NULL, then the + * file will be created in the root of the debugfs filesystem. + * @f: function to be called when pathname resolution steps on that one. + * @data: opaque argument to pass to f(). + * + * @f should return what ->d_automount() would. + */ +struct dentry *debugfs_create_automount(const char *name, + struct dentry *parent, + debugfs_automount_t f, + void *data) +{ + struct dentry *dentry = debugfs_start_creating(name, parent); + struct inode *inode; + + if (IS_ERR(dentry)) + return dentry; + + inode = debugfs_get_inode(dentry->d_sb); + if (unlikely(!inode)) { + pr_err("out of free dentries, can not create automount '%s'\n", + name); + return debugfs_failed_creating(dentry); + } + + make_empty_dir_inode(inode); + inode->i_flags |= S_AUTOMOUNT; + inode->i_private = data; + DEBUGFS_I(inode)->automount = f; + /* directory inodes start off with i_nlink == 2 (for "." entry) */ + inc_nlink(inode); + d_make_persistent(dentry, inode); + inc_nlink(d_inode(dentry->d_parent)); + fsnotify_mkdir(d_inode(dentry->d_parent), dentry); + return debugfs_end_creating(dentry); +} +EXPORT_SYMBOL(debugfs_create_automount); + +/** + * debugfs_create_symlink- create a symbolic link in the debugfs filesystem + * @name: a pointer to a string containing the name of the symbolic link to + * create. + * @parent: a pointer to the parent dentry for this symbolic link. This + * should be a directory dentry if set. If this parameter is NULL, + * then the symbolic link will be created in the root of the debugfs + * filesystem. + * @target: a pointer to a string containing the path to the target of the + * symbolic link. + * + * This function creates a symbolic link with the given name in debugfs that + * links to the given target path. + * + * This function will return a pointer to a dentry if it succeeds. This + * pointer must be passed to the debugfs_remove() function when the symbolic + * link is to be removed (no automatic cleanup happens if your module is + * unloaded, you are responsible here.) If an error occurs, ERR_PTR(-ERROR) + * will be returned. + * + * If debugfs is not enabled in the kernel, the value -%ENODEV will be + * returned. + */ +struct dentry *debugfs_create_symlink(const char *name, struct dentry *parent, + const char *target) +{ + struct dentry *dentry; + struct inode *inode; + char *link = kstrdup(target, GFP_KERNEL); + if (!link) + return ERR_PTR(-ENOMEM); + + dentry = debugfs_start_creating(name, parent); + if (IS_ERR(dentry)) { + kfree(link); + return dentry; + } + + inode = debugfs_get_inode(dentry->d_sb); + if (unlikely(!inode)) { + pr_err("out of free dentries, can not create symlink '%s'\n", + name); + kfree(link); + return debugfs_failed_creating(dentry); + } + inode->i_mode = S_IFLNK | S_IRWXUGO; + inode->i_op = &debugfs_symlink_inode_operations; + inode->i_link = link; + d_make_persistent(dentry, inode); + return debugfs_end_creating(dentry); +} +EXPORT_SYMBOL_GPL(debugfs_create_symlink); + +static void __debugfs_file_removed(struct dentry *dentry) +{ + struct debugfs_fsdata *fsd; + + /* + * Paired with the closing smp_mb() implied by a successful + * cmpxchg() in debugfs_file_get(): either + * debugfs_file_get() must see a dead dentry or we must see a + * debugfs_fsdata instance at ->d_fsdata here (or both). + */ + smp_mb(); + fsd = READ_ONCE(dentry->d_fsdata); + if (!fsd) + return; + + /* if this was the last reference, we're done */ + if (refcount_dec_and_test(&fsd->active_users)) + return; + + /* + * If there's still a reference, the code that obtained it can + * be in different states: + * - The common case of not using cancellations, or already + * after debugfs_leave_cancellation(), where we just need + * to wait for debugfs_file_put() which signals the completion; + * - inside a cancellation section, i.e. between + * debugfs_enter_cancellation() and debugfs_leave_cancellation(), + * in which case we need to trigger the ->cancel() function, + * and then wait for debugfs_file_put() just like in the + * previous case; + * - before debugfs_enter_cancellation() (but obviously after + * debugfs_file_get()), in which case we may not see the + * cancellation in the list on the first round of the loop, + * but debugfs_enter_cancellation() signals the completion + * after adding it, so this code gets woken up to call the + * ->cancel() function. + */ + while (refcount_read(&fsd->active_users)) { + struct debugfs_cancellation *c; + + /* + * Lock the cancellations. Note that the cancellations + * structs are meant to be on the stack, so we need to + * ensure we either use them here or don't touch them, + * and debugfs_leave_cancellation() will wait for this + * to be finished processing before exiting one. It may + * of course win and remove the cancellation, but then + * chances are we never even got into this bit, we only + * do if the refcount isn't zero already. + */ + mutex_lock(&fsd->cancellations_mtx); + while ((c = list_first_entry_or_null(&fsd->cancellations, + typeof(*c), list))) { + list_del_init(&c->list); + c->cancel(dentry, c->cancel_data); + } + mutex_unlock(&fsd->cancellations_mtx); + + wait_for_completion(&fsd->active_users_drained); + } +} + +static void remove_one(struct dentry *victim) +{ + if (d_is_reg(victim)) + __debugfs_file_removed(victim); + simple_release_fs(&debugfs_mount, &debugfs_mount_count); +} + +/** + * debugfs_remove - recursively removes a directory + * @dentry: a pointer to a the dentry of the directory to be removed. If this + * parameter is NULL or an error value, nothing will be done. + * + * This function recursively removes a directory tree in debugfs that + * was previously created with a call to another debugfs function + * (like debugfs_create_file() or variants thereof.) + * + * This function is required to be called in order for the file to be + * removed, no automatic cleanup of files will happen when a module is + * removed, you are responsible here. + */ +void debugfs_remove(struct dentry *dentry) +{ + if (IS_ERR_OR_NULL(dentry)) + return; + + simple_pin_fs(&debug_fs_type, &debugfs_mount, &debugfs_mount_count); + simple_recursive_removal(dentry, remove_one); + simple_release_fs(&debugfs_mount, &debugfs_mount_count); +} +EXPORT_SYMBOL_GPL(debugfs_remove); + +/** + * debugfs_lookup_and_remove - lookup a directory or file and recursively remove it + * @name: a pointer to a string containing the name of the item to look up. + * @parent: a pointer to the parent dentry of the item. + * + * This is the equlivant of doing something like + * debugfs_remove(debugfs_lookup(..)) but with the proper reference counting + * handled for the directory being looked up. + */ +void debugfs_lookup_and_remove(const char *name, struct dentry *parent) +{ + struct dentry *dentry; + + dentry = debugfs_lookup(name, parent); + if (!dentry) + return; + + debugfs_remove(dentry); + dput(dentry); +} +EXPORT_SYMBOL_GPL(debugfs_lookup_and_remove); + +/** + * debugfs_change_name - rename a file/directory in the debugfs filesystem + * @dentry: dentry of an object to be renamed. + * @fmt: format for new name + * + * This function renames a file/directory in debugfs. The target must not + * exist for rename to succeed. + * + * This function will return 0 on success and -E... on failure. + * + * If debugfs is not enabled in the kernel, the value -%ENODEV will be + * returned. + */ +int __printf(2, 3) debugfs_change_name(struct dentry *dentry, const char *fmt, ...) +{ + int error = 0; + const char *new_name; + struct name_snapshot old_name; + struct dentry *target; + struct renamedata rd = {}; + struct inode *dir; + va_list ap; + + if (IS_ERR_OR_NULL(dentry)) + return 0; + + va_start(ap, fmt); + new_name = kvasprintf_const(GFP_KERNEL, fmt, ap); + va_end(ap); + if (!new_name) + return -ENOMEM; + + rd.old_parent = dget_parent(dentry); + rd.new_parent = rd.old_parent; + rd.flags = RENAME_NOREPLACE; + target = lookup_noperm_unlocked(&QSTR(new_name), rd.new_parent); + if (IS_ERR(target)) { + error = PTR_ERR(target); + goto out_free; + } + + error = start_renaming_two_dentries(&rd, dentry, target); + if (error) { + if (error == -EEXIST && target == dentry) + /* it isn't an error to rename a thing to itself */ + error = 0; + goto out; + } + + dir = d_inode(rd.old_parent); + take_dentry_name_snapshot(&old_name, dentry); + simple_rename_timestamp(dir, dentry, dir, rd.new_dentry); + d_move(dentry, rd.new_dentry); + fsnotify_move(dir, dir, &old_name.name, d_is_dir(dentry), NULL, dentry); + release_dentry_name_snapshot(&old_name); + end_renaming(&rd); +out: + dput(rd.old_parent); + dput(target); +out_free: + kfree_const(new_name); + return error; +} +EXPORT_SYMBOL_GPL(debugfs_change_name); + +/** + * debugfs_initialized - Tells whether debugfs has been registered + */ +bool debugfs_initialized(void) +{ + return debugfs_registered; +} +EXPORT_SYMBOL_GPL(debugfs_initialized); + +static int __init debugfs_kernel(char *str) +{ + if (str) { + if (!strcmp(str, "on")) + debugfs_enabled = true; + else if (!strcmp(str, "off")) + debugfs_enabled = false; + else if (!strcmp(str, "no-mount")) { + pr_notice("debugfs=no-mount is a deprecated alias " + "for debugfs=off\n"); + debugfs_enabled = false; + } + } + + return 0; +} +early_param("debugfs", debugfs_kernel); + +static int __init debugfs_init(void) +{ + int retval; + + if (!debugfs_enabled) + return -EPERM; + + retval = sysfs_create_mount_point(kernel_kobj, "debug"); + if (retval) + return retval; + + debugfs_inode_cachep = kmem_cache_create("debugfs_inode_cache", + sizeof(struct debugfs_inode_info), 0, + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, + init_once); + if (debugfs_inode_cachep == NULL) { + sysfs_remove_mount_point(kernel_kobj, "debug"); + return -ENOMEM; + } + + retval = register_filesystem(&debug_fs_type); + if (retval) { // Really not going to happen + sysfs_remove_mount_point(kernel_kobj, "debug"); + kmem_cache_destroy(debugfs_inode_cachep); + return retval; + } + debugfs_registered = true; + return 0; +} +core_initcall(debugfs_init); diff --git a/fs/debugfs/internal.h b/fs/debugfs/internal.h new file mode 100644 index 000000000..c95699b27 --- /dev/null +++ b/fs/debugfs/internal.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * internal.h - declarations internal to debugfs + * + * Copyright (C) 2016 Nicolai Stange + */ + +#ifndef _DEBUGFS_INTERNAL_H_ +#define _DEBUGFS_INTERNAL_H_ +#include + +struct file_operations; + +struct debugfs_inode_info { + struct inode vfs_inode; + union { + const void *raw; + const struct file_operations *real_fops; + const struct debugfs_short_fops *short_fops; + debugfs_automount_t automount; + }; + void *aux; +}; + +static inline struct debugfs_inode_info *DEBUGFS_I(struct inode *inode) +{ + return container_of(inode, struct debugfs_inode_info, vfs_inode); +} + +/* declared over in file.c */ +extern const struct file_operations debugfs_noop_file_operations; +extern const struct file_operations debugfs_open_proxy_file_operations; +extern const struct file_operations debugfs_full_proxy_file_operations; +extern const struct file_operations debugfs_full_short_proxy_file_operations; + +struct debugfs_fsdata { + const struct file_operations *real_fops; + const struct debugfs_short_fops *short_fops; + struct { + refcount_t active_users; + struct completion active_users_drained; + + /* protect cancellations */ + struct mutex cancellations_mtx; + struct list_head cancellations; + unsigned int methods; + }; +}; + +enum { + HAS_READ = 1, + HAS_WRITE = 2, + HAS_LSEEK = 4, + HAS_POLL = 8, + HAS_IOCTL = 16 +}; + +#endif /* _DEBUGFS_INTERNAL_H_ */ diff --git a/fs/devpts/Makefile b/fs/devpts/Makefile new file mode 100644 index 000000000..66064c8fc --- /dev/null +++ b/fs/devpts/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the Linux /dev/pts virtual filesystem. +# + +obj-$(CONFIG_UNIX98_PTYS) += devpts.o + +devpts-$(CONFIG_UNIX98_PTYS) := inode.o diff --git a/fs/devpts/inode.c b/fs/devpts/inode.c new file mode 100644 index 000000000..9844dcf35 --- /dev/null +++ b/fs/devpts/inode.c @@ -0,0 +1,572 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* -*- linux-c -*- --------------------------------------------------------- * + * + * linux/fs/devpts/inode.c + * + * Copyright 1998-2004 H. Peter Anvin -- All Rights Reserved + * + * ------------------------------------------------------------------------- */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DEVPTS_DEFAULT_MODE 0600 +/* + * ptmx is a new node in /dev/pts and will be unused in legacy (single- + * instance) mode. To prevent surprises in user space, set permissions of + * ptmx to 0. Use 'chmod' or remount with '-o ptmxmode' to set meaningful + * permissions. + */ +#define DEVPTS_DEFAULT_PTMX_MODE 0000 +#define PTMX_MINOR 2 + +/* + * sysctl support for setting limits on the number of Unix98 ptys allocated. + * Otherwise one can eat up all kernel memory by opening /dev/ptmx repeatedly. + */ +static int pty_limit = NR_UNIX98_PTY_DEFAULT; +static int pty_reserve = NR_UNIX98_PTY_RESERVE; +static int pty_limit_min; +static int pty_limit_max = INT_MAX; +static atomic_t pty_count = ATOMIC_INIT(0); + +static const struct ctl_table pty_table[] = { + { + .procname = "max", + .maxlen = sizeof(int), + .mode = 0644, + .data = &pty_limit, + .proc_handler = proc_dointvec_minmax, + .extra1 = &pty_limit_min, + .extra2 = &pty_limit_max, + }, { + .procname = "reserve", + .maxlen = sizeof(int), + .mode = 0644, + .data = &pty_reserve, + .proc_handler = proc_dointvec_minmax, + .extra1 = &pty_limit_min, + .extra2 = &pty_limit_max, + }, { + .procname = "nr", + .maxlen = sizeof(int), + .mode = 0444, + .data = &pty_count, + .proc_handler = proc_dointvec, + }, +}; + +struct pts_mount_opts { + int setuid; + int setgid; + kuid_t uid; + kgid_t gid; + umode_t mode; + umode_t ptmxmode; + int reserve; + int max; +}; + +enum { + Opt_uid, Opt_gid, Opt_mode, Opt_ptmxmode, Opt_newinstance, Opt_max, + Opt_err +}; + +static const struct fs_parameter_spec devpts_param_specs[] = { + fsparam_gid ("gid", Opt_gid), + fsparam_s32 ("max", Opt_max), + fsparam_u32oct ("mode", Opt_mode), + fsparam_flag ("newinstance", Opt_newinstance), + fsparam_u32oct ("ptmxmode", Opt_ptmxmode), + fsparam_uid ("uid", Opt_uid), + {} +}; + +struct pts_fs_info { + struct ida allocated_ptys; + struct pts_mount_opts mount_opts; + struct super_block *sb; + struct inode *ptmx_inode; // borrowed +}; + +static inline struct pts_fs_info *DEVPTS_SB(struct super_block *sb) +{ + return sb->s_fs_info; +} + +static int devpts_ptmx_path(struct path *path) +{ + struct super_block *sb; + int err; + + /* Is a devpts filesystem at "pts" in the same directory? */ + err = path_pts(path); + if (err) + return err; + + /* Is the path the root of a devpts filesystem? */ + sb = path->mnt->mnt_sb; + if ((sb->s_magic != DEVPTS_SUPER_MAGIC) || + (path->mnt->mnt_root != sb->s_root)) + return -ENODEV; + + return 0; +} + +/* + * Try to find a suitable devpts filesystem. We support the following + * scenarios: + * - The ptmx device node is located in the same directory as the devpts + * mount where the pts device nodes are located. + * This is e.g. the case when calling open on the /dev/pts/ptmx device + * node when the devpts filesystem is mounted at /dev/pts. + * - The ptmx device node is located outside the devpts filesystem mount + * where the pts device nodes are located. For example, the ptmx device + * is a symlink, separate device node, or bind-mount. + * A supported scenario is bind-mounting /dev/pts/ptmx to /dev/ptmx and + * then calling open on /dev/ptmx. In this case a suitable pts + * subdirectory can be found in the common parent directory /dev of the + * devpts mount and the ptmx bind-mount, after resolving the /dev/ptmx + * bind-mount. + * If no suitable pts subdirectory can be found this function will fail. + * This is e.g. the case when bind-mounting /dev/pts/ptmx to /ptmx. + */ +struct vfsmount *devpts_mntget(struct file *filp, struct pts_fs_info *fsi) +{ + struct path path; + int err = 0; + + path = filp->f_path; + path_get(&path); + + /* Walk upward while the start point is a bind mount of + * a single file. + */ + while (path.mnt->mnt_root == path.dentry) + if (follow_up(&path) == 0) + break; + + /* devpts_ptmx_path() finds a devpts fs or returns an error. */ + if ((path.mnt->mnt_sb->s_magic != DEVPTS_SUPER_MAGIC) || + (DEVPTS_SB(path.mnt->mnt_sb) != fsi)) + err = devpts_ptmx_path(&path); + dput(path.dentry); + if (!err) { + if (DEVPTS_SB(path.mnt->mnt_sb) == fsi) + return path.mnt; + + err = -ENODEV; + } + + mntput(path.mnt); + return ERR_PTR(err); +} + +struct pts_fs_info *devpts_acquire(struct file *filp) +{ + struct pts_fs_info *result; + struct path path; + struct super_block *sb; + + path = filp->f_path; + path_get(&path); + + /* Has the devpts filesystem already been found? */ + if (path.mnt->mnt_sb->s_magic != DEVPTS_SUPER_MAGIC) { + int err; + + err = devpts_ptmx_path(&path); + if (err) { + result = ERR_PTR(err); + goto out; + } + } + + /* + * pty code needs to hold extra references in case of last /dev/tty close + */ + sb = path.mnt->mnt_sb; + atomic_inc(&sb->s_active); + result = DEVPTS_SB(sb); + +out: + path_put(&path); + return result; +} + +void devpts_release(struct pts_fs_info *fsi) +{ + deactivate_super(fsi->sb); +} + +/* + * devpts_parse_param - Parse mount parameters + */ +static int devpts_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct pts_fs_info *fsi = fc->s_fs_info; + struct pts_mount_opts *opts = &fsi->mount_opts; + struct fs_parse_result result; + int opt; + + opt = fs_parse(fc, devpts_param_specs, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_uid: + opts->uid = result.uid; + opts->setuid = 1; + break; + case Opt_gid: + opts->gid = result.gid; + opts->setgid = 1; + break; + case Opt_mode: + opts->mode = result.uint_32 & S_IALLUGO; + break; + case Opt_ptmxmode: + opts->ptmxmode = result.uint_32 & S_IALLUGO; + break; + case Opt_newinstance: + break; + case Opt_max: + if (result.uint_32 > NR_UNIX98_PTY_MAX) + return invalf(fc, "max out of range"); + opts->max = result.uint_32; + break; + } + + return 0; +} + +static int mknod_ptmx(struct super_block *sb, struct fs_context *fc) +{ + int mode; + struct dentry *dentry; + struct inode *inode; + struct dentry *root = sb->s_root; + struct pts_fs_info *fsi = DEVPTS_SB(sb); + struct pts_mount_opts *opts = &fsi->mount_opts; + kuid_t ptmx_uid = current_fsuid(); + kgid_t ptmx_gid = current_fsgid(); + + dentry = simple_start_creating(root, "ptmx"); + if (IS_ERR(dentry)) { + pr_err("Unable to alloc dentry for ptmx node\n"); + return PTR_ERR(dentry); + } + + /* + * Create a new 'ptmx' node in this mount of devpts. + */ + inode = new_inode(sb); + if (!inode) { + simple_done_creating(dentry); + pr_err("Unable to alloc inode for ptmx node\n"); + return -ENOMEM; + } + + inode->i_ino = 2; + simple_inode_init_ts(inode); + + mode = S_IFCHR|opts->ptmxmode; + init_special_inode(inode, mode, MKDEV(TTYAUX_MAJOR, 2)); + inode->i_uid = ptmx_uid; + inode->i_gid = ptmx_gid; + fsi->ptmx_inode = inode; + + d_make_persistent(dentry, inode); + + simple_done_creating(dentry); + + return 0; +} + +static void update_ptmx_mode(struct pts_fs_info *fsi) +{ + fsi->ptmx_inode->i_mode = S_IFCHR|fsi->mount_opts.ptmxmode; +} + +static int devpts_reconfigure(struct fs_context *fc) +{ + struct pts_fs_info *fsi = DEVPTS_SB(fc->root->d_sb); + struct pts_fs_info *new = fc->s_fs_info; + + /* Apply the revised options. We don't want to change ->reserve. + * Ideally, we'd update each option conditionally on it having been + * explicitly changed, but the default is to reset everything so that + * would break UAPI... + */ + fsi->mount_opts.setuid = new->mount_opts.setuid; + fsi->mount_opts.setgid = new->mount_opts.setgid; + fsi->mount_opts.uid = new->mount_opts.uid; + fsi->mount_opts.gid = new->mount_opts.gid; + fsi->mount_opts.mode = new->mount_opts.mode; + fsi->mount_opts.ptmxmode = new->mount_opts.ptmxmode; + fsi->mount_opts.max = new->mount_opts.max; + + /* + * parse_mount_options() restores options to default values + * before parsing and may have changed ptmxmode. So, update the + * mode in the inode too. Bogus options don't fail the remount, + * so do this even on error return. + */ + update_ptmx_mode(fsi); + + return 0; +} + +static int devpts_show_options(struct seq_file *seq, struct dentry *root) +{ + struct pts_fs_info *fsi = DEVPTS_SB(root->d_sb); + struct pts_mount_opts *opts = &fsi->mount_opts; + + if (opts->setuid) + seq_printf(seq, ",uid=%u", + from_kuid_munged(&init_user_ns, opts->uid)); + if (opts->setgid) + seq_printf(seq, ",gid=%u", + from_kgid_munged(&init_user_ns, opts->gid)); + seq_printf(seq, ",mode=%03o", opts->mode); + seq_printf(seq, ",ptmxmode=%03o", opts->ptmxmode); + if (opts->max < NR_UNIX98_PTY_MAX) + seq_printf(seq, ",max=%d", opts->max); + + return 0; +} + +static const struct super_operations devpts_sops = { + .statfs = simple_statfs, + .show_options = devpts_show_options, +}; + +static int devpts_fill_super(struct super_block *s, struct fs_context *fc) +{ + struct pts_fs_info *fsi = DEVPTS_SB(s); + struct inode *inode; + + s->s_iflags &= ~SB_I_NODEV; + s->s_blocksize = 1024; + s->s_blocksize_bits = 10; + s->s_magic = DEVPTS_SUPER_MAGIC; + s->s_op = &devpts_sops; + s->s_d_flags = DCACHE_DONTCACHE; + s->s_time_gran = 1; + fsi->sb = s; + + inode = new_inode(s); + if (!inode) + return -ENOMEM; + inode->i_ino = 1; + simple_inode_init_ts(inode); + inode->i_mode = S_IFDIR | S_IRUGO | S_IXUGO | S_IWUSR; + inode->i_op = &simple_dir_inode_operations; + inode->i_fop = &simple_dir_operations; + set_nlink(inode, 2); + + s->s_root = d_make_root(inode); + if (!s->s_root) { + pr_err("get root dentry failed\n"); + return -ENOMEM; + } + + return mknod_ptmx(s, fc); +} + +/* + * devpts_get_tree() + * + * Mount a new (private) instance of devpts. PTYs created in this + * instance are independent of the PTYs in other devpts instances. + */ +static int devpts_get_tree(struct fs_context *fc) +{ + return get_tree_nodev(fc, devpts_fill_super); +} + +static void devpts_free_fc(struct fs_context *fc) +{ + kfree(fc->s_fs_info); +} + +static const struct fs_context_operations devpts_context_ops = { + .free = devpts_free_fc, + .parse_param = devpts_parse_param, + .get_tree = devpts_get_tree, + .reconfigure = devpts_reconfigure, +}; + +/* + * Set up the filesystem mount context. + */ +static int devpts_init_fs_context(struct fs_context *fc) +{ + struct pts_fs_info *fsi; + + fsi = kzalloc_obj(struct pts_fs_info); + if (!fsi) + return -ENOMEM; + + ida_init(&fsi->allocated_ptys); + fsi->mount_opts.uid = GLOBAL_ROOT_UID; + fsi->mount_opts.gid = GLOBAL_ROOT_GID; + fsi->mount_opts.mode = DEVPTS_DEFAULT_MODE; + fsi->mount_opts.ptmxmode = DEVPTS_DEFAULT_PTMX_MODE; + fsi->mount_opts.max = NR_UNIX98_PTY_MAX; + + if (fc->purpose == FS_CONTEXT_FOR_MOUNT && + current->nsproxy->mnt_ns == init_task.nsproxy->mnt_ns) + fsi->mount_opts.reserve = true; + + fc->s_fs_info = fsi; + fc->ops = &devpts_context_ops; + return 0; +} + +static void devpts_kill_sb(struct super_block *sb) +{ + struct pts_fs_info *fsi = DEVPTS_SB(sb); + + if (fsi) + ida_destroy(&fsi->allocated_ptys); + kfree(fsi); + kill_anon_super(sb); +} + +static struct file_system_type devpts_fs_type = { + .name = "devpts", + .init_fs_context = devpts_init_fs_context, + .parameters = devpts_param_specs, + .kill_sb = devpts_kill_sb, + .fs_flags = FS_USERNS_MOUNT, +}; + +/* + * The normal naming convention is simply /dev/pts/; this conforms + * to the System V naming convention + */ + +int devpts_new_index(struct pts_fs_info *fsi) +{ + int index = -ENOSPC; + + if (atomic_inc_return(&pty_count) >= (pty_limit - + (fsi->mount_opts.reserve ? 0 : pty_reserve))) + goto out; + + index = ida_alloc_max(&fsi->allocated_ptys, fsi->mount_opts.max - 1, + GFP_KERNEL); + +out: + if (index < 0) + atomic_dec(&pty_count); + return index; +} + +void devpts_kill_index(struct pts_fs_info *fsi, int idx) +{ + ida_free(&fsi->allocated_ptys, idx); + atomic_dec(&pty_count); +} + +/** + * devpts_pty_new -- create a new inode in /dev/pts/ + * @fsi: Filesystem info for this instance. + * @index: used as a name of the node + * @priv: what's given back by devpts_get_priv + * + * The dentry for the created inode is returned. + * Remove it from /dev/pts/ with devpts_pty_kill(). + */ +struct dentry *devpts_pty_new(struct pts_fs_info *fsi, int index, void *priv) +{ + struct dentry *dentry; + struct super_block *sb = fsi->sb; + struct inode *inode; + struct dentry *root; + struct pts_mount_opts *opts; + char s[12]; + + root = sb->s_root; + opts = &fsi->mount_opts; + + inode = new_inode(sb); + if (!inode) + return ERR_PTR(-ENOMEM); + + inode->i_ino = index + 3; + inode->i_uid = opts->setuid ? opts->uid : current_fsuid(); + inode->i_gid = opts->setgid ? opts->gid : current_fsgid(); + simple_inode_init_ts(inode); + init_special_inode(inode, S_IFCHR|opts->mode, MKDEV(UNIX98_PTY_SLAVE_MAJOR, index)); + + sprintf(s, "%d", index); + + dentry = d_alloc_name(root, s); + if (!dentry) { + iput(inode); + return ERR_PTR(-ENOMEM); + } + dentry->d_fsdata = priv; + d_make_persistent(dentry, inode); + fsnotify_create(d_inode(root), dentry); + dput(dentry); + return dentry; // borrowed +} + +/** + * devpts_get_priv -- get private data for a slave + * @dentry: dentry of the slave + * + * Returns whatever was passed as priv in devpts_pty_new for a given inode. + */ +void *devpts_get_priv(struct dentry *dentry) +{ + if (dentry->d_sb->s_magic != DEVPTS_SUPER_MAGIC) + return NULL; + return dentry->d_fsdata; +} + +/** + * devpts_pty_kill -- remove inode form /dev/pts/ + * @dentry: dentry of the slave to be removed + * + * This is an inverse operation of devpts_pty_new. + */ +void devpts_pty_kill(struct dentry *dentry) +{ + WARN_ON_ONCE(dentry->d_sb->s_magic != DEVPTS_SUPER_MAGIC); + + dentry->d_fsdata = NULL; + drop_nlink(dentry->d_inode); + d_drop(dentry); + fsnotify_unlink(d_inode(dentry->d_parent), dentry); + d_make_discardable(dentry); +} + +static int __init init_devpts_fs(void) +{ + int err = register_filesystem(&devpts_fs_type); + if (!err) { + register_sysctl("kernel/pty", pty_table); + } + return err; +} +module_init(init_devpts_fs) diff --git a/fs/direct-io.c b/fs/direct-io.c new file mode 100644 index 000000000..9e4976716 --- /dev/null +++ b/fs/direct-io.c @@ -0,0 +1,1316 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * fs/direct-io.c + * + * Copyright (C) 2002, Linus Torvalds. + * + * O_DIRECT + * + * 04Jul2002 Andrew Morton + * Initial version + * 11Sep2002 janetinc@us.ibm.com + * added readv/writev support. + * 29Oct2002 Andrew Morton + * rewrote bio_add_page() support. + * 30Oct2002 pbadari@us.ibm.com + * added support for non-aligned IO. + * 06Nov2002 pbadari@us.ibm.com + * added asynchronous IO support. + * 21Jul2003 nathans@sgi.com + * added IO completion notifier. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +/* + * How many user pages to map in one call to iov_iter_extract_pages(). This + * determines the size of a structure in the slab cache + */ +#define DIO_PAGES 64 + +/* + * Flags for dio_complete() + */ +#define DIO_COMPLETE_ASYNC 0x01 /* This is async IO */ +#define DIO_COMPLETE_INVALIDATE 0x02 /* Can invalidate pages */ + +/* + * This code generally works in units of "dio_blocks". A dio_block is + * somewhere between the hard sector size and the filesystem block size. it + * is determined on a per-invocation basis. When talking to the filesystem + * we need to convert dio_blocks to fs_blocks by scaling the dio_block quantity + * down by dio->blkfactor. Similarly, fs-blocksize quantities are converted + * to bio_block quantities by shifting left by blkfactor. + * + * If blkfactor is zero then the user's request was aligned to the filesystem's + * blocksize. + */ + +/* dio_state only used in the submission path */ + +struct dio_submit { + struct bio *bio; /* bio under assembly */ + unsigned blkbits; /* doesn't change */ + unsigned blkfactor; /* When we're using an alignment which + is finer than the filesystem's soft + blocksize, this specifies how much + finer. blkfactor=2 means 1/4-block + alignment. Does not change */ + unsigned start_zero_done; /* flag: sub-blocksize zeroing has + been performed at the start of a + write */ + int pages_in_io; /* approximate total IO pages */ + sector_t block_in_file; /* Current offset into the underlying + file in dio_block units. */ + unsigned blocks_available; /* At block_in_file. changes */ + int reap_counter; /* rate limit reaping */ + sector_t final_block_in_request;/* doesn't change */ + int boundary; /* prev block is at a boundary */ + get_block_t *get_block; /* block mapping function */ + + loff_t logical_offset_in_bio; /* current first logical block in bio */ + sector_t final_block_in_bio; /* current final block in bio + 1 */ + sector_t next_block_for_io; /* next block to be put under IO, + in dio_blocks units */ + + /* + * Deferred addition of a page to the dio. These variables are + * private to dio_send_cur_page(), submit_page_section() and + * dio_bio_add_page(). + */ + struct page *cur_page; /* The page */ + unsigned cur_page_offset; /* Offset into it, in bytes */ + unsigned cur_page_len; /* Nr of bytes at cur_page_offset */ + sector_t cur_page_block; /* Where it starts */ + loff_t cur_page_fs_offset; /* Offset in file */ + + struct iov_iter *iter; + /* + * Page queue. These variables belong to dio_refill_pages() and + * dio_get_page(). + */ + unsigned head; /* next page to process */ + unsigned tail; /* last valid page + 1 */ + size_t from, to; +}; + +/* dio_state communicated between submission path and end_io */ +struct dio { + int flags; /* doesn't change */ + blk_opf_t opf; /* request operation type and flags */ + struct gendisk *bio_disk; + struct inode *inode; + loff_t i_size; /* i_size when submitted */ + dio_iodone_t *end_io; /* IO completion function */ + bool is_pinned; /* T if we have pins on the pages */ + + void *private; /* copy from map_bh.b_private */ + + /* BIO completion state */ + spinlock_t bio_lock; /* protects BIO fields below */ + int page_errors; /* err from iov_iter_extract_pages() */ + int is_async; /* is IO async ? */ + bool defer_completion; /* defer AIO completion to workqueue? */ + bool should_dirty; /* if pages should be dirtied */ + int io_error; /* IO error in completion path */ + unsigned long refcount; /* direct_io_worker() and bios */ + struct bio *bio_list; /* singly linked via bi_private */ + struct task_struct *waiter; /* waiting task (NULL if none) */ + + /* AIO related stuff */ + struct kiocb *iocb; /* kiocb */ + ssize_t result; /* IO result */ + + /* + * pages[] (and any fields placed after it) are not zeroed out at + * allocation time. Don't add new fields after pages[] unless you + * wish that they not be zeroed. + */ + union { + struct page *pages[DIO_PAGES]; /* page buffer */ + struct work_struct complete_work;/* deferred AIO completion */ + }; +} ____cacheline_aligned_in_smp; + +static struct kmem_cache *dio_cache __ro_after_init; + +/* + * How many pages are in the queue? + */ +static inline unsigned dio_pages_present(struct dio_submit *sdio) +{ + return sdio->tail - sdio->head; +} + +/* + * Go grab and pin some userspace pages. Typically we'll get 64 at a time. + */ +static inline int dio_refill_pages(struct dio *dio, struct dio_submit *sdio) +{ + struct page **pages = dio->pages; + const enum req_op dio_op = dio->opf & REQ_OP_MASK; + ssize_t ret; + + ret = iov_iter_extract_pages(sdio->iter, &pages, LONG_MAX, + DIO_PAGES, 0, &sdio->from); + + if (ret < 0 && sdio->blocks_available && dio_op == REQ_OP_WRITE) { + /* + * A memory fault, but the filesystem has some outstanding + * mapped blocks. We need to use those blocks up to avoid + * leaking stale data in the file. + */ + if (dio->page_errors == 0) + dio->page_errors = ret; + dio->pages[0] = ZERO_PAGE(0); + sdio->head = 0; + sdio->tail = 1; + sdio->from = 0; + sdio->to = PAGE_SIZE; + return 0; + } + + if (ret >= 0) { + ret += sdio->from; + sdio->head = 0; + sdio->tail = (ret + PAGE_SIZE - 1) / PAGE_SIZE; + sdio->to = ((ret - 1) & (PAGE_SIZE - 1)) + 1; + return 0; + } + return ret; +} + +/* + * Get another userspace page. Returns an ERR_PTR on error. Pages are + * buffered inside the dio so that we can call iov_iter_extract_pages() + * against a decent number of pages, less frequently. To provide nicer use of + * the L1 cache. + */ +static inline struct page *dio_get_page(struct dio *dio, + struct dio_submit *sdio) +{ + if (dio_pages_present(sdio) == 0) { + int ret; + + ret = dio_refill_pages(dio, sdio); + if (ret) + return ERR_PTR(ret); + BUG_ON(dio_pages_present(sdio) == 0); + } + return dio->pages[sdio->head]; +} + +static void dio_pin_page(struct dio *dio, struct page *page) +{ + if (dio->is_pinned) + folio_add_pin(page_folio(page)); +} + +static void dio_unpin_page(struct dio *dio, struct page *page) +{ + if (dio->is_pinned) + unpin_user_page(page); +} + +/* + * dio_complete() - called when all DIO BIO I/O has been completed + * + * This drops i_dio_count, lets interested parties know that a DIO operation + * has completed, and calculates the resulting return code for the operation. + * + * It lets the filesystem know if it registered an interest earlier via + * get_block. Pass the private field of the map buffer_head so that + * filesystems can use it to hold additional state between get_block calls and + * dio_complete. + */ +static ssize_t dio_complete(struct dio *dio, ssize_t ret, unsigned int flags) +{ + const enum req_op dio_op = dio->opf & REQ_OP_MASK; + loff_t offset = dio->iocb->ki_pos; + ssize_t transferred = 0; + int err; + + /* + * AIO submission can race with bio completion to get here while + * expecting to have the last io completed by bio completion. + * In that case -EIOCBQUEUED is in fact not an error we want + * to preserve through this call. + */ + if (ret == -EIOCBQUEUED) + ret = 0; + + if (dio->result) { + transferred = dio->result; + + /* Check for short read case */ + if (dio_op == REQ_OP_READ && + ((offset + transferred) > dio->i_size)) + transferred = dio->i_size - offset; + /* ignore EFAULT if some IO has been done */ + if (unlikely(ret == -EFAULT) && transferred) + ret = 0; + } + + if (ret == 0) + ret = dio->page_errors; + if (ret == 0) + ret = dio->io_error; + if (ret == 0) + ret = transferred; + + if (dio->end_io) { + // XXX: ki_pos?? + err = dio->end_io(dio->iocb, offset, ret, dio->private); + if (err) + ret = err; + } + + /* + * Try again to invalidate clean pages which might have been cached by + * non-direct readahead, or faulted in by get_user_pages() if the source + * of the write was an mmap'ed region of the file we're writing. Either + * one is a pretty crazy thing to do, so we don't support it 100%. If + * this invalidation fails, tough, the write still worked... + * + * And this page cache invalidation has to be after dio->end_io(), as + * some filesystems convert unwritten extents to real allocations in + * end_io() when necessary, otherwise a racing buffer read would cache + * zeros from unwritten extents. + */ + if (flags & DIO_COMPLETE_INVALIDATE && + ret > 0 && dio_op == REQ_OP_WRITE) + kiocb_invalidate_post_direct_write(dio->iocb, ret); + + inode_dio_end(dio->inode); + + if (flags & DIO_COMPLETE_ASYNC) { + /* + * generic_write_sync expects ki_pos to have been updated + * already, but the submission path only does this for + * synchronous I/O. + */ + dio->iocb->ki_pos += transferred; + + if (ret > 0 && dio_op == REQ_OP_WRITE) + ret = generic_write_sync(dio->iocb, ret); + dio->iocb->ki_complete(dio->iocb, ret); + } + + kmem_cache_free(dio_cache, dio); + return ret; +} + +static void dio_aio_complete_work(struct work_struct *work) +{ + struct dio *dio = container_of(work, struct dio, complete_work); + + dio_complete(dio, 0, DIO_COMPLETE_ASYNC | DIO_COMPLETE_INVALIDATE); +} + +static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio); + +/* + * Asynchronous IO callback. + */ +static void dio_bio_end_aio(struct bio *bio) +{ + struct dio *dio = bio->bi_private; + const enum req_op dio_op = dio->opf & REQ_OP_MASK; + unsigned long remaining; + unsigned long flags; + bool defer_completion = false; + + /* cleanup the bio */ + dio_bio_complete(dio, bio); + + spin_lock_irqsave(&dio->bio_lock, flags); + remaining = --dio->refcount; + if (remaining == 1 && dio->waiter) + wake_up_process(dio->waiter); + spin_unlock_irqrestore(&dio->bio_lock, flags); + + if (remaining == 0) { + /* + * Defer completion when defer_completion is set or + * when the inode has pages mapped and this is AIO write. + * We need to invalidate those pages because there is a + * chance they contain stale data in the case buffered IO + * went in between AIO submission and completion into the + * same region. + */ + if (dio->result) + defer_completion = dio->defer_completion || + (dio_op == REQ_OP_WRITE && + dio->inode->i_mapping->nrpages); + if (defer_completion) { + INIT_WORK(&dio->complete_work, dio_aio_complete_work); + queue_work(dio->inode->i_sb->s_dio_done_wq, + &dio->complete_work); + } else { + dio_complete(dio, 0, DIO_COMPLETE_ASYNC); + } + } +} + +/* + * The BIO completion handler simply queues the BIO up for the process-context + * handler. + * + * During I/O bi_private points at the dio. After I/O, bi_private is used to + * implement a singly-linked list of completed BIOs, at dio->bio_list. + */ +static void dio_bio_end_io(struct bio *bio) +{ + struct dio *dio = bio->bi_private; + unsigned long flags; + + spin_lock_irqsave(&dio->bio_lock, flags); + bio->bi_private = dio->bio_list; + dio->bio_list = bio; + if (--dio->refcount == 1 && dio->waiter) + wake_up_process(dio->waiter); + spin_unlock_irqrestore(&dio->bio_lock, flags); +} + +static inline void +dio_bio_alloc(struct dio *dio, struct dio_submit *sdio, + struct block_device *bdev, + sector_t first_sector, int nr_vecs) +{ + struct bio *bio; + + /* + * bio_alloc() is guaranteed to return a bio when allowed to sleep and + * we request a valid number of vectors. + */ + bio = bio_alloc(bdev, nr_vecs, dio->opf, GFP_KERNEL); + bio->bi_iter.bi_sector = first_sector; + if (dio->is_async) + bio->bi_end_io = dio_bio_end_aio; + else + bio->bi_end_io = dio_bio_end_io; + if (dio->is_pinned) + bio_set_flag(bio, BIO_PAGE_PINNED); + bio->bi_write_hint = file_inode(dio->iocb->ki_filp)->i_write_hint; + + sdio->bio = bio; + sdio->logical_offset_in_bio = sdio->cur_page_fs_offset; +} + +/* + * In the AIO read case we speculatively dirty the pages before starting IO. + * During IO completion, any of these pages which happen to have been written + * back will be redirtied by bio_check_pages_dirty(). + * + * bios hold a dio reference between submit_bio and ->end_io. + */ +static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio) +{ + const enum req_op dio_op = dio->opf & REQ_OP_MASK; + struct bio *bio = sdio->bio; + unsigned long flags; + + bio->bi_private = dio; + + spin_lock_irqsave(&dio->bio_lock, flags); + dio->refcount++; + spin_unlock_irqrestore(&dio->bio_lock, flags); + + if (dio->is_async && dio_op == REQ_OP_READ && dio->should_dirty) + bio_set_pages_dirty(bio); + + dio->bio_disk = bio->bi_bdev->bd_disk; + + submit_bio(bio); + + sdio->bio = NULL; + sdio->boundary = 0; + sdio->logical_offset_in_bio = 0; +} + +/* + * Release any resources in case of a failure + */ +static inline void dio_cleanup(struct dio *dio, struct dio_submit *sdio) +{ + if (dio->is_pinned) + unpin_user_pages(dio->pages + sdio->head, + sdio->tail - sdio->head); + sdio->head = sdio->tail; +} + +/* + * Wait for the next BIO to complete. Remove it and return it. NULL is + * returned once all BIOs have been completed. This must only be called once + * all bios have been issued so that dio->refcount can only decrease. This + * requires that the caller hold a reference on the dio. + */ +static struct bio *dio_await_one(struct dio *dio) +{ + unsigned long flags; + struct bio *bio = NULL; + + spin_lock_irqsave(&dio->bio_lock, flags); + + /* + * Wait as long as the list is empty and there are bios in flight. bio + * completion drops the count, maybe adds to the list, and wakes while + * holding the bio_lock so we don't need set_current_state()'s barrier + * and can call it after testing our condition. + */ + while (dio->refcount > 1 && dio->bio_list == NULL) { + __set_current_state(TASK_UNINTERRUPTIBLE); + dio->waiter = current; + spin_unlock_irqrestore(&dio->bio_lock, flags); + blk_io_schedule(); + /* wake up sets us TASK_RUNNING */ + spin_lock_irqsave(&dio->bio_lock, flags); + dio->waiter = NULL; + } + if (dio->bio_list) { + bio = dio->bio_list; + dio->bio_list = bio->bi_private; + } + spin_unlock_irqrestore(&dio->bio_lock, flags); + return bio; +} + +/* + * Process one completed BIO. No locks are held. + */ +static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio) +{ + blk_status_t err = bio->bi_status; + const enum req_op dio_op = dio->opf & REQ_OP_MASK; + bool should_dirty = dio_op == REQ_OP_READ && dio->should_dirty; + + if (err) + dio->io_error = -EIO; + + if (dio->is_async && should_dirty) { + bio_check_pages_dirty(bio); /* transfers ownership */ + } else { + bio_release_pages(bio, should_dirty); + bio_put(bio); + } + return err; +} + +/* + * Wait on and process all in-flight BIOs. This must only be called once + * all bios have been issued so that the refcount can only decrease. + * This just waits for all bios to make it through dio_bio_complete. IO + * errors are propagated through dio->io_error and should be propagated via + * dio_complete(). + */ +static void dio_await_completion(struct dio *dio) +{ + struct bio *bio; + do { + bio = dio_await_one(dio); + if (bio) + dio_bio_complete(dio, bio); + } while (bio); +} + +/* + * A really large O_DIRECT read or write can generate a lot of BIOs. So + * to keep the memory consumption sane we periodically reap any completed BIOs + * during the BIO generation phase. + * + * This also helps to limit the peak amount of pinned userspace memory. + */ +static inline int dio_bio_reap(struct dio *dio, struct dio_submit *sdio) +{ + int ret = 0; + + if (sdio->reap_counter++ >= 64) { + while (dio->bio_list) { + unsigned long flags; + struct bio *bio; + int ret2; + + spin_lock_irqsave(&dio->bio_lock, flags); + bio = dio->bio_list; + dio->bio_list = bio->bi_private; + spin_unlock_irqrestore(&dio->bio_lock, flags); + ret2 = blk_status_to_errno(dio_bio_complete(dio, bio)); + if (ret == 0) + ret = ret2; + } + sdio->reap_counter = 0; + } + return ret; +} + +static int dio_set_defer_completion(struct dio *dio) +{ + struct super_block *sb = dio->inode->i_sb; + + if (dio->defer_completion) + return 0; + dio->defer_completion = true; + if (!sb->s_dio_done_wq) + return sb_init_dio_done_wq(sb); + return 0; +} + +/* + * Call into the fs to map some more disk blocks. We record the current number + * of available blocks at sdio->blocks_available. These are in units of the + * fs blocksize, i_blocksize(inode). + * + * The fs is allowed to map lots of blocks at once. If it wants to do that, + * it uses the passed inode-relative block number as the file offset, as usual. + * + * get_block() is passed the number of i_blkbits-sized blocks which direct_io + * has remaining to do. The fs should not map more than this number of blocks. + * + * If the fs has mapped a lot of blocks, it should populate bh->b_size to + * indicate how much contiguous disk space has been made available at + * bh->b_blocknr. + * + * If *any* of the mapped blocks are new, then the fs must set buffer_new(). + * This isn't very efficient... + * + * In the case of filesystem holes: the fs may return an arbitrarily-large + * hole by returning an appropriate value in b_size and by clearing + * buffer_mapped(). However the direct-io code will only process holes one + * block at a time - it will repeatedly call get_block() as it walks the hole. + */ +static int get_more_blocks(struct dio *dio, struct dio_submit *sdio, + struct buffer_head *map_bh) +{ + const enum req_op dio_op = dio->opf & REQ_OP_MASK; + int ret; + sector_t fs_startblk; /* Into file, in filesystem-sized blocks */ + sector_t fs_endblk; /* Into file, in filesystem-sized blocks */ + unsigned long fs_count; /* Number of filesystem-sized blocks */ + int create; + unsigned int i_blkbits = sdio->blkbits + sdio->blkfactor; + loff_t i_size; + + /* + * If there was a memory error and we've overwritten all the + * mapped blocks then we can now return that memory error + */ + ret = dio->page_errors; + if (ret == 0) { + BUG_ON(sdio->block_in_file >= sdio->final_block_in_request); + fs_startblk = sdio->block_in_file >> sdio->blkfactor; + fs_endblk = (sdio->final_block_in_request - 1) >> + sdio->blkfactor; + fs_count = fs_endblk - fs_startblk + 1; + + map_bh->b_state = 0; + map_bh->b_size = fs_count << i_blkbits; + + /* + * For writes that could fill holes inside i_size on a + * DIO_SKIP_HOLES filesystem we forbid block creations: only + * overwrites are permitted. We will return early to the caller + * once we see an unmapped buffer head returned, and the caller + * will fall back to buffered I/O. + * + * Otherwise the decision is left to the get_blocks method, + * which may decide to handle it or also return an unmapped + * buffer head. + */ + create = dio_op == REQ_OP_WRITE; + if (dio->flags & DIO_SKIP_HOLES) { + i_size = i_size_read(dio->inode); + if (i_size && fs_startblk <= (i_size - 1) >> i_blkbits) + create = 0; + } + + ret = (*sdio->get_block)(dio->inode, fs_startblk, + map_bh, create); + + /* Store for completion */ + dio->private = map_bh->b_private; + + if (ret == 0 && buffer_defer_completion(map_bh)) + ret = dio_set_defer_completion(dio); + } + return ret; +} + +/* + * There is no bio. Make one now. + */ +static inline int dio_new_bio(struct dio *dio, struct dio_submit *sdio, + sector_t start_sector, struct buffer_head *map_bh) +{ + sector_t sector; + int ret, nr_pages; + + ret = dio_bio_reap(dio, sdio); + if (ret) + goto out; + sector = start_sector << (sdio->blkbits - 9); + nr_pages = bio_max_segs(sdio->pages_in_io); + BUG_ON(nr_pages <= 0); + dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages); + sdio->boundary = 0; +out: + return ret; +} + +/* + * Attempt to put the current chunk of 'cur_page' into the current BIO. If + * that was successful then update final_block_in_bio and take a ref against + * the just-added page. + * + * Return zero on success. Non-zero means the caller needs to start a new BIO. + */ +static inline int dio_bio_add_page(struct dio *dio, struct dio_submit *sdio) +{ + int ret; + + ret = bio_add_page(sdio->bio, sdio->cur_page, + sdio->cur_page_len, sdio->cur_page_offset); + if (ret == sdio->cur_page_len) { + /* + * Decrement count only, if we are done with this page + */ + if ((sdio->cur_page_len + sdio->cur_page_offset) == PAGE_SIZE) + sdio->pages_in_io--; + dio_pin_page(dio, sdio->cur_page); + sdio->final_block_in_bio = sdio->cur_page_block + + (sdio->cur_page_len >> sdio->blkbits); + ret = 0; + } else { + ret = 1; + } + return ret; +} + +/* + * Put cur_page under IO. The section of cur_page which is described by + * cur_page_offset,cur_page_len is put into a BIO. The section of cur_page + * starts on-disk at cur_page_block. + * + * We take a ref against the page here (on behalf of its presence in the bio). + * + * The caller of this function is responsible for removing cur_page from the + * dio, and for dropping the refcount which came from that presence. + */ +static inline int dio_send_cur_page(struct dio *dio, struct dio_submit *sdio, + struct buffer_head *map_bh) +{ + int ret = 0; + + if (sdio->bio) { + loff_t cur_offset = sdio->cur_page_fs_offset; + loff_t bio_next_offset = sdio->logical_offset_in_bio + + sdio->bio->bi_iter.bi_size; + + /* + * See whether this new request is contiguous with the old. + * + * Btrfs cannot handle having logically non-contiguous requests + * submitted. For example if you have + * + * Logical: [0-4095][HOLE][8192-12287] + * Physical: [0-4095] [4096-8191] + * + * We cannot submit those pages together as one BIO. So if our + * current logical offset in the file does not equal what would + * be the next logical offset in the bio, submit the bio we + * have. + */ + if (sdio->final_block_in_bio != sdio->cur_page_block || + cur_offset != bio_next_offset) + dio_bio_submit(dio, sdio); + } + + if (sdio->bio == NULL) { + ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh); + if (ret) + goto out; + } + + if (dio_bio_add_page(dio, sdio) != 0) { + dio_bio_submit(dio, sdio); + ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh); + if (ret == 0) { + ret = dio_bio_add_page(dio, sdio); + BUG_ON(ret != 0); + } + } +out: + return ret; +} + +/* + * An autonomous function to put a chunk of a page under deferred IO. + * + * The caller doesn't actually know (or care) whether this piece of page is in + * a BIO, or is under IO or whatever. We just take care of all possible + * situations here. The separation between the logic of do_direct_IO() and + * that of submit_page_section() is important for clarity. Please don't break. + * + * The chunk of page starts on-disk at blocknr. + * + * We perform deferred IO, by recording the last-submitted page inside our + * private part of the dio structure. If possible, we just expand the IO + * across that page here. + * + * If that doesn't work out then we put the old page into the bio and add this + * page to the dio instead. + */ +static inline int +submit_page_section(struct dio *dio, struct dio_submit *sdio, struct page *page, + unsigned offset, unsigned len, sector_t blocknr, + struct buffer_head *map_bh) +{ + const enum req_op dio_op = dio->opf & REQ_OP_MASK; + int ret = 0; + int boundary = sdio->boundary; /* dio_send_cur_page may clear it */ + + if (dio_op == REQ_OP_WRITE) { + /* + * Read accounting is performed in submit_bio() + */ + task_io_account_write(len); + } + + /* + * Can we just grow the current page's presence in the dio? + */ + if (sdio->cur_page == page && + sdio->cur_page_offset + sdio->cur_page_len == offset && + sdio->cur_page_block + + (sdio->cur_page_len >> sdio->blkbits) == blocknr) { + sdio->cur_page_len += len; + goto out; + } + + /* + * If there's a deferred page already there then send it. + */ + if (sdio->cur_page) { + ret = dio_send_cur_page(dio, sdio, map_bh); + dio_unpin_page(dio, sdio->cur_page); + sdio->cur_page = NULL; + if (ret) + return ret; + } + + dio_pin_page(dio, page); /* It is in dio */ + sdio->cur_page = page; + sdio->cur_page_offset = offset; + sdio->cur_page_len = len; + sdio->cur_page_block = blocknr; + sdio->cur_page_fs_offset = sdio->block_in_file << sdio->blkbits; +out: + /* + * If boundary then we want to schedule the IO now to + * avoid metadata seeks. + */ + if (boundary) { + ret = dio_send_cur_page(dio, sdio, map_bh); + if (sdio->bio) + dio_bio_submit(dio, sdio); + dio_unpin_page(dio, sdio->cur_page); + sdio->cur_page = NULL; + } + return ret; +} + +/* + * If we are not writing the entire block and get_block() allocated + * the block for us, we need to fill-in the unused portion of the + * block with zeros. This happens only if user-buffer, fileoffset or + * io length is not filesystem block-size multiple. + * + * `end' is zero if we're doing the start of the IO, 1 at the end of the + * IO. + */ +static inline void dio_zero_block(struct dio *dio, struct dio_submit *sdio, + int end, struct buffer_head *map_bh) +{ + unsigned dio_blocks_per_fs_block; + unsigned this_chunk_blocks; /* In dio_blocks */ + unsigned this_chunk_bytes; + struct page *page; + + sdio->start_zero_done = 1; + if (!sdio->blkfactor || !buffer_new(map_bh)) + return; + + dio_blocks_per_fs_block = 1 << sdio->blkfactor; + this_chunk_blocks = sdio->block_in_file & (dio_blocks_per_fs_block - 1); + + if (!this_chunk_blocks) + return; + + /* + * We need to zero out part of an fs block. It is either at the + * beginning or the end of the fs block. + */ + if (end) + this_chunk_blocks = dio_blocks_per_fs_block - this_chunk_blocks; + + this_chunk_bytes = this_chunk_blocks << sdio->blkbits; + + page = ZERO_PAGE(0); + if (submit_page_section(dio, sdio, page, 0, this_chunk_bytes, + sdio->next_block_for_io, map_bh)) + return; + + sdio->next_block_for_io += this_chunk_blocks; +} + +/* + * Walk the user pages, and the file, mapping blocks to disk and generating + * a sequence of (page,offset,len,block) mappings. These mappings are injected + * into submit_page_section(), which takes care of the next stage of submission + * + * Direct IO against a blockdev is different from a file. Because we can + * happily perform page-sized but 512-byte aligned IOs. It is important that + * blockdev IO be able to have fine alignment and large sizes. + * + * So what we do is to permit the ->get_block function to populate bh.b_size + * with the size of IO which is permitted at this offset and this i_blkbits. + * + * For best results, the blockdev should be set up with 512-byte i_blkbits and + * it should set b_size to PAGE_SIZE or more inside get_block(). This gives + * fine alignment but still allows this function to work in PAGE_SIZE units. + */ +static int do_direct_IO(struct dio *dio, struct dio_submit *sdio, + struct buffer_head *map_bh) +{ + const enum req_op dio_op = dio->opf & REQ_OP_MASK; + const unsigned blkbits = sdio->blkbits; + const unsigned i_blkbits = blkbits + sdio->blkfactor; + int ret = 0; + + while (sdio->block_in_file < sdio->final_block_in_request) { + struct page *page; + size_t from, to; + + page = dio_get_page(dio, sdio); + if (IS_ERR(page)) { + ret = PTR_ERR(page); + goto out; + } + from = sdio->head ? 0 : sdio->from; + to = (sdio->head == sdio->tail - 1) ? sdio->to : PAGE_SIZE; + sdio->head++; + + while (from < to) { + unsigned this_chunk_bytes; /* # of bytes mapped */ + unsigned this_chunk_blocks; /* # of blocks */ + unsigned u; + + if (sdio->blocks_available == 0) { + /* + * Need to go and map some more disk + */ + unsigned long blkmask; + unsigned long dio_remainder; + + ret = get_more_blocks(dio, sdio, map_bh); + if (ret) { + dio_unpin_page(dio, page); + goto out; + } + if (!buffer_mapped(map_bh)) + goto do_holes; + + sdio->blocks_available = + map_bh->b_size >> blkbits; + sdio->next_block_for_io = + map_bh->b_blocknr << sdio->blkfactor; + if (buffer_new(map_bh)) { + clean_bdev_aliases( + map_bh->b_bdev, + map_bh->b_blocknr, + map_bh->b_size >> i_blkbits); + } + + if (!sdio->blkfactor) + goto do_holes; + + blkmask = (1 << sdio->blkfactor) - 1; + dio_remainder = (sdio->block_in_file & blkmask); + + /* + * If we are at the start of IO and that IO + * starts partway into a fs-block, + * dio_remainder will be non-zero. If the IO + * is a read then we can simply advance the IO + * cursor to the first block which is to be + * read. But if the IO is a write and the + * block was newly allocated we cannot do that; + * the start of the fs block must be zeroed out + * on-disk + */ + if (!buffer_new(map_bh)) + sdio->next_block_for_io += dio_remainder; + sdio->blocks_available -= dio_remainder; + } +do_holes: + /* Handle holes */ + if (!buffer_mapped(map_bh)) { + loff_t i_size_aligned; + + /* AKPM: eargh, -ENOTBLK is a hack */ + if (dio_op == REQ_OP_WRITE) { + dio_unpin_page(dio, page); + return -ENOTBLK; + } + + /* + * Be sure to account for a partial block as the + * last block in the file + */ + i_size_aligned = ALIGN(i_size_read(dio->inode), + 1 << blkbits); + if (sdio->block_in_file >= + i_size_aligned >> blkbits) { + /* We hit eof */ + dio_unpin_page(dio, page); + goto out; + } + memzero_page(page, from, 1 << blkbits); + sdio->block_in_file++; + from += 1 << blkbits; + dio->result += 1 << blkbits; + goto next_block; + } + + /* + * If we're performing IO which has an alignment which + * is finer than the underlying fs, go check to see if + * we must zero out the start of this block. + */ + if (unlikely(sdio->blkfactor && !sdio->start_zero_done)) + dio_zero_block(dio, sdio, 0, map_bh); + + /* + * Work out, in this_chunk_blocks, how much disk we + * can add to this page + */ + this_chunk_blocks = sdio->blocks_available; + u = (to - from) >> blkbits; + if (this_chunk_blocks > u) + this_chunk_blocks = u; + u = sdio->final_block_in_request - sdio->block_in_file; + if (this_chunk_blocks > u) + this_chunk_blocks = u; + this_chunk_bytes = this_chunk_blocks << blkbits; + BUG_ON(this_chunk_bytes == 0); + + if (this_chunk_blocks == sdio->blocks_available) + sdio->boundary = buffer_boundary(map_bh); + ret = submit_page_section(dio, sdio, page, + from, + this_chunk_bytes, + sdio->next_block_for_io, + map_bh); + if (ret) { + dio_unpin_page(dio, page); + goto out; + } + sdio->next_block_for_io += this_chunk_blocks; + + sdio->block_in_file += this_chunk_blocks; + from += this_chunk_bytes; + dio->result += this_chunk_bytes; + sdio->blocks_available -= this_chunk_blocks; +next_block: + BUG_ON(sdio->block_in_file > sdio->final_block_in_request); + if (sdio->block_in_file == sdio->final_block_in_request) + break; + } + + /* Drop the pin which was taken in get_user_pages() */ + dio_unpin_page(dio, page); + } +out: + return ret; +} + +static inline int drop_refcount(struct dio *dio) +{ + int ret2; + unsigned long flags; + + /* + * Sync will always be dropping the final ref and completing the + * operation. AIO can if it was a broken operation described above or + * in fact if all the bios race to complete before we get here. In + * that case dio_complete() translates the EIOCBQUEUED into the proper + * return code that the caller will hand to ->complete(). + * + * This is managed by the bio_lock instead of being an atomic_t so that + * completion paths can drop their ref and use the remaining count to + * decide to wake the submission path atomically. + */ + spin_lock_irqsave(&dio->bio_lock, flags); + ret2 = --dio->refcount; + spin_unlock_irqrestore(&dio->bio_lock, flags); + return ret2; +} + +/* + * This is a library function for use by filesystem drivers. + * + * The locking rules are governed by the flags parameter: + * - if the flags value contains DIO_LOCKING we use a fancy locking + * scheme for dumb filesystems. + * For writes this function is called under i_rwsem and returns with + * i_rwsem held, for reads, i_rwsem is not held on entry, but it is + * taken and dropped again before returning. + * - if the flags value does NOT contain DIO_LOCKING we don't use any + * internal locking but rather rely on the filesystem to synchronize + * direct I/O reads/writes versus each other and truncate. + * + * To help with locking against truncate we incremented the i_dio_count + * counter before starting direct I/O, and decrement it once we are done. + * Truncate can wait for it to reach zero to provide exclusion. It is + * expected that filesystem provide exclusion between new direct I/O + * and truncates. For DIO_LOCKING filesystems this is done by i_rwsem, + * but other filesystems need to take care of this on their own. + * + * NOTE: if you pass "sdio" to anything by pointer make sure that function + * is always inlined. Otherwise gcc is unable to split the structure into + * individual fields and will generate much worse code. This is important + * for the whole file. + */ +ssize_t __blockdev_direct_IO(struct kiocb *iocb, struct inode *inode, + struct block_device *bdev, struct iov_iter *iter, + get_block_t get_block, dio_iodone_t end_io, + int flags) +{ + unsigned i_blkbits = READ_ONCE(inode->i_blkbits); + unsigned blkbits = i_blkbits; + unsigned blocksize_mask = (1 << blkbits) - 1; + ssize_t retval = -EINVAL; + const size_t count = iov_iter_count(iter); + loff_t offset = iocb->ki_pos; + const loff_t end = offset + count; + struct dio *dio; + struct dio_submit sdio = { NULL, }; + struct buffer_head map_bh = { 0, }; + struct blk_plug plug; + unsigned long align = offset | iov_iter_alignment(iter); + + /* watch out for a 0 len io from a tricksy fs */ + if (iov_iter_rw(iter) == READ && !count) + return 0; + + dio = kmem_cache_alloc(dio_cache, GFP_KERNEL); + if (!dio) + return -ENOMEM; + /* + * Believe it or not, zeroing out the page array caused a .5% + * performance regression in a database benchmark. So, we take + * care to only zero out what's needed. + */ + memset(dio, 0, offsetof(struct dio, pages)); + + dio->flags = flags; + if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ) { + /* will be released by direct_io_worker */ + inode_lock(inode); + } + dio->is_pinned = iov_iter_extract_will_pin(iter); + + /* Once we sampled i_size check for reads beyond EOF */ + dio->i_size = i_size_read(inode); + if (iov_iter_rw(iter) == READ && offset >= dio->i_size) { + retval = 0; + goto fail_dio; + } + + if (align & blocksize_mask) { + if (bdev) + blkbits = blksize_bits(bdev_logical_block_size(bdev)); + blocksize_mask = (1 << blkbits) - 1; + if (align & blocksize_mask) + goto fail_dio; + } + + if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ) { + struct address_space *mapping = iocb->ki_filp->f_mapping; + + retval = filemap_write_and_wait_range(mapping, offset, end - 1); + if (retval) + goto fail_dio; + } + + /* + * For file extending writes updating i_size before data writeouts + * complete can expose uninitialized blocks in dumb filesystems. + * In that case we need to wait for I/O completion even if asked + * for an asynchronous write. + */ + if (is_sync_kiocb(iocb)) + dio->is_async = false; + else if (iov_iter_rw(iter) == WRITE && end > i_size_read(inode)) + dio->is_async = false; + else + dio->is_async = true; + + dio->inode = inode; + if (iov_iter_rw(iter) == WRITE) + dio->opf = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE; + else + dio->opf = REQ_OP_READ; + + /* + * For AIO O_(D)SYNC writes we need to defer completions to a workqueue + * so that we can call ->fsync. + */ + if (dio->is_async && iov_iter_rw(iter) == WRITE) { + retval = 0; + if (iocb_is_dsync(iocb)) + retval = dio_set_defer_completion(dio); + else if (!dio->inode->i_sb->s_dio_done_wq) { + /* + * In case of AIO write racing with buffered read we + * need to defer completion. We can't decide this now, + * however the workqueue needs to be initialized here. + */ + retval = sb_init_dio_done_wq(dio->inode->i_sb); + } + if (retval) + goto fail_dio; + } + + /* + * Will be decremented at I/O completion time. + */ + inode_dio_begin(inode); + + sdio.blkbits = blkbits; + sdio.blkfactor = i_blkbits - blkbits; + sdio.block_in_file = offset >> blkbits; + + sdio.get_block = get_block; + dio->end_io = end_io; + sdio.final_block_in_bio = -1; + sdio.next_block_for_io = -1; + + dio->iocb = iocb; + + spin_lock_init(&dio->bio_lock); + dio->refcount = 1; + + dio->should_dirty = user_backed_iter(iter) && iov_iter_rw(iter) == READ; + sdio.iter = iter; + sdio.final_block_in_request = end >> blkbits; + + /* + * In case of non-aligned buffers, we may need 2 more + * pages since we need to zero out first and last block. + */ + if (unlikely(sdio.blkfactor)) + sdio.pages_in_io = 2; + + sdio.pages_in_io += iov_iter_npages(iter, INT_MAX); + + blk_start_plug(&plug); + + retval = do_direct_IO(dio, &sdio, &map_bh); + if (retval) + dio_cleanup(dio, &sdio); + + if (retval == -ENOTBLK) { + /* + * The remaining part of the request will be + * handled by buffered I/O when we return + */ + retval = 0; + } + /* + * There may be some unwritten disk at the end of a part-written + * fs-block-sized block. Go zero that now. + */ + dio_zero_block(dio, &sdio, 1, &map_bh); + + if (sdio.cur_page) { + ssize_t ret2; + + ret2 = dio_send_cur_page(dio, &sdio, &map_bh); + if (retval == 0) + retval = ret2; + dio_unpin_page(dio, sdio.cur_page); + sdio.cur_page = NULL; + } + if (sdio.bio) + dio_bio_submit(dio, &sdio); + + blk_finish_plug(&plug); + + /* + * It is possible that, we return short IO due to end of file. + * In that case, we need to release all the pages we got hold on. + */ + dio_cleanup(dio, &sdio); + + /* + * All block lookups have been performed. For READ requests + * we can let i_rwsem go now that its achieved its purpose + * of protecting us from looking up uninitialized blocks. + */ + if (iov_iter_rw(iter) == READ && (dio->flags & DIO_LOCKING)) + inode_unlock(dio->inode); + + /* + * The only time we want to leave bios in flight is when a successful + * partial aio read or full aio write have been setup. In that case + * bio completion will call aio_complete. The only time it's safe to + * call aio_complete is when we return -EIOCBQUEUED, so we key on that. + * This had *better* be the only place that raises -EIOCBQUEUED. + */ + BUG_ON(retval == -EIOCBQUEUED); + if (dio->is_async && retval == 0 && dio->result && + (iov_iter_rw(iter) == READ || dio->result == count)) + retval = -EIOCBQUEUED; + else + dio_await_completion(dio); + + if (drop_refcount(dio) == 0) { + retval = dio_complete(dio, retval, DIO_COMPLETE_INVALIDATE); + } else + BUG_ON(retval != -EIOCBQUEUED); + + return retval; + +fail_dio: + if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ) + inode_unlock(inode); + + kmem_cache_free(dio_cache, dio); + return retval; +} +EXPORT_SYMBOL(__blockdev_direct_IO); + +static __init int dio_init(void) +{ + dio_cache = KMEM_CACHE(dio, SLAB_PANIC); + return 0; +} +module_init(dio_init) diff --git a/fs/dlm/Kconfig b/fs/dlm/Kconfig new file mode 100644 index 000000000..fb6ba9f5a --- /dev/null +++ b/fs/dlm/Kconfig @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0-only +menuconfig DLM + tristate "Distributed Lock Manager (DLM)" + depends on INET + depends on SYSFS && CONFIGFS_FS + help + A general purpose distributed lock manager for kernel or userspace + applications. + +config DLM_DEBUG + bool "DLM debugging" + depends on DLM + help + Under the debugfs mount point, the name of each lockspace will + appear as a file in the "dlm" directory. The output is the + list of resource and locks the local node knows about. diff --git a/fs/dlm/Makefile b/fs/dlm/Makefile new file mode 100644 index 000000000..5a471af1d --- /dev/null +++ b/fs/dlm/Makefile @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_DLM) += dlm.o +dlm-y := ast.o \ + config.o \ + dir.o \ + lock.o \ + lockspace.o \ + main.o \ + member.o \ + memory.o \ + midcomms.o \ + lowcomms.o \ + plock.o \ + rcom.o \ + recover.o \ + recoverd.o \ + requestqueue.o \ + user.o \ + util.o +dlm-$(CONFIG_DLM_DEBUG) += debug_fs.o + diff --git a/fs/dlm/ast.c b/fs/dlm/ast.c new file mode 100644 index 000000000..0fe8d80ce --- /dev/null +++ b/fs/dlm/ast.c @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0-only +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2010 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#include + +#include "dlm_internal.h" +#include "lvb_table.h" +#include "memory.h" +#include "lock.h" +#include "user.h" +#include "ast.h" + +static void dlm_run_callback(uint32_t ls_id, uint32_t lkb_id, int8_t mode, + uint32_t flags, uint8_t sb_flags, int sb_status, + struct dlm_lksb *lksb, + void (*astfn)(void *astparam), + void (*bastfn)(void *astparam, int mode), + void *astparam, const char *res_name, + size_t res_length) +{ + if (flags & DLM_CB_BAST) { + trace_dlm_bast(ls_id, lkb_id, mode, res_name, res_length); + bastfn(astparam, mode); + } else if (flags & DLM_CB_CAST) { + trace_dlm_ast(ls_id, lkb_id, sb_flags, sb_status, res_name, + res_length); + lksb->sb_status = sb_status; + lksb->sb_flags = sb_flags; + astfn(astparam); + } +} + +static void dlm_do_callback(struct dlm_callback *cb) +{ + dlm_run_callback(cb->ls_id, cb->lkb_id, cb->mode, cb->flags, + cb->sb_flags, cb->sb_status, cb->lkb_lksb, + cb->astfn, cb->bastfn, cb->astparam, + cb->res_name, cb->res_length); + dlm_free_cb(cb); +} + +static void dlm_callback_work(struct work_struct *work) +{ + struct dlm_callback *cb = container_of(work, struct dlm_callback, work); + + dlm_do_callback(cb); +} + +bool dlm_may_skip_callback(struct dlm_lkb *lkb, uint32_t flags, int mode, + int status, uint32_t sbflags, int *copy_lvb) +{ + struct dlm_rsb *rsb = lkb->lkb_resource; + struct dlm_ls *ls = rsb->res_ls; + int prev_mode; + + if (copy_lvb) + *copy_lvb = 0; + + if (flags & DLM_CB_BAST) { + /* if cb is a bast, it should be skipped if the blocking mode is + * compatible with the last granted mode + */ + if (lkb->lkb_last_cast_cb_mode != -1) { + if (dlm_modes_compat(mode, lkb->lkb_last_cast_cb_mode)) { + log_debug(ls, "skip %x bast mode %d for cast mode %d", + lkb->lkb_id, mode, + lkb->lkb_last_cast_cb_mode); + return true; + } + } + + /* + * Suppress some redundant basts here, do more on removal. + * Don't even add a bast if the callback just before it + * is a bast for the same mode or a more restrictive mode. + * (the addional > PR check is needed for PR/CW inversion) + */ + if (lkb->lkb_last_cb_mode != -1 && + lkb->lkb_last_cb_flags & DLM_CB_BAST) { + prev_mode = lkb->lkb_last_cb_mode; + + if ((prev_mode == mode) || + (prev_mode > mode && prev_mode > DLM_LOCK_PR)) { + log_debug(ls, "skip %x add bast mode %d for bast mode %d", + lkb->lkb_id, mode, prev_mode); + return true; + } + } + + lkb->lkb_last_bast_time = ktime_get(); + lkb->lkb_last_bast_cb_mode = mode; + } else if (flags & DLM_CB_CAST) { + if (test_bit(DLM_DFL_USER_BIT, &lkb->lkb_dflags)) { + prev_mode = lkb->lkb_last_cast_cb_mode; + + if (!status && lkb->lkb_lksb->sb_lvbptr && + dlm_lvb_operations[prev_mode + 1][mode + 1]) { + if (copy_lvb) + *copy_lvb = 1; + } + } + + lkb->lkb_last_cast_cb_mode = mode; + lkb->lkb_last_cast_time = ktime_get(); + } + + lkb->lkb_last_cb_mode = mode; + lkb->lkb_last_cb_flags = flags; + + return false; +} + +int dlm_get_cb(struct dlm_lkb *lkb, uint32_t flags, int mode, + int status, uint32_t sbflags, + struct dlm_callback **cb) +{ + struct dlm_rsb *rsb = lkb->lkb_resource; + struct dlm_ls *ls = rsb->res_ls; + + *cb = dlm_allocate_cb(); + if (WARN_ON_ONCE(!*cb)) + return -ENOMEM; + + /* for tracing */ + (*cb)->lkb_id = lkb->lkb_id; + (*cb)->ls_id = ls->ls_global_id; + memcpy((*cb)->res_name, rsb->res_name, rsb->res_length); + (*cb)->res_length = rsb->res_length; + + (*cb)->flags = flags; + (*cb)->mode = mode; + (*cb)->sb_status = status; + (*cb)->sb_flags = (sbflags & 0x000000FF); + (*cb)->lkb_lksb = lkb->lkb_lksb; + + return 0; +} + +static int dlm_get_queue_cb(struct dlm_lkb *lkb, uint32_t flags, int mode, + int status, uint32_t sbflags, + struct dlm_callback **cb) +{ + int rv; + + rv = dlm_get_cb(lkb, flags, mode, status, sbflags, cb); + if (rv) + return rv; + + (*cb)->astfn = lkb->lkb_astfn; + (*cb)->bastfn = lkb->lkb_bastfn; + (*cb)->astparam = lkb->lkb_astparam; + INIT_WORK(&(*cb)->work, dlm_callback_work); + + return 0; +} + +void dlm_add_cb(struct dlm_lkb *lkb, uint32_t flags, int mode, int status, + uint32_t sbflags) +{ + struct dlm_rsb *rsb = lkb->lkb_resource; + struct dlm_ls *ls = rsb->res_ls; + struct dlm_callback *cb; + int rv; + + if (test_bit(DLM_DFL_USER_BIT, &lkb->lkb_dflags)) { + dlm_user_add_ast(lkb, flags, mode, status, sbflags); + return; + } + + if (dlm_may_skip_callback(lkb, flags, mode, status, sbflags, NULL)) + return; + + spin_lock_bh(&ls->ls_cb_lock); + if (test_bit(LSFL_CB_DELAY, &ls->ls_flags)) { + rv = dlm_get_queue_cb(lkb, flags, mode, status, sbflags, &cb); + if (!rv) + list_add(&cb->list, &ls->ls_cb_delay); + } else { + if (test_bit(LSFL_SOFTIRQ, &ls->ls_flags)) { + dlm_run_callback(ls->ls_global_id, lkb->lkb_id, mode, flags, + sbflags, status, lkb->lkb_lksb, + lkb->lkb_astfn, lkb->lkb_bastfn, + lkb->lkb_astparam, rsb->res_name, + rsb->res_length); + } else { + rv = dlm_get_queue_cb(lkb, flags, mode, status, sbflags, &cb); + if (!rv) + queue_work(ls->ls_callback_wq, &cb->work); + } + } + spin_unlock_bh(&ls->ls_cb_lock); +} + +int dlm_callback_start(struct dlm_ls *ls) +{ + if (!test_bit(LSFL_FS, &ls->ls_flags) || + test_bit(LSFL_SOFTIRQ, &ls->ls_flags)) + return 0; + + ls->ls_callback_wq = alloc_ordered_workqueue("dlm_callback", + WQ_HIGHPRI | WQ_MEM_RECLAIM); + if (!ls->ls_callback_wq) { + log_print("can't start dlm_callback workqueue"); + return -ENOMEM; + } + return 0; +} + +void dlm_callback_stop(struct dlm_ls *ls) +{ + if (ls->ls_callback_wq) + destroy_workqueue(ls->ls_callback_wq); +} + +void dlm_callback_suspend(struct dlm_ls *ls) +{ + if (!test_bit(LSFL_FS, &ls->ls_flags)) + return; + + spin_lock_bh(&ls->ls_cb_lock); + set_bit(LSFL_CB_DELAY, &ls->ls_flags); + spin_unlock_bh(&ls->ls_cb_lock); + + if (ls->ls_callback_wq) + flush_workqueue(ls->ls_callback_wq); +} + +#define MAX_CB_QUEUE 25 + +void dlm_callback_resume(struct dlm_ls *ls) +{ + struct dlm_callback *cb, *safe; + int count = 0, sum = 0; + bool empty; + + if (!test_bit(LSFL_FS, &ls->ls_flags)) + return; + +more: + spin_lock_bh(&ls->ls_cb_lock); + list_for_each_entry_safe(cb, safe, &ls->ls_cb_delay, list) { + list_del(&cb->list); + if (test_bit(LSFL_SOFTIRQ, &ls->ls_flags)) + dlm_do_callback(cb); + else + queue_work(ls->ls_callback_wq, &cb->work); + + count++; + if (count == MAX_CB_QUEUE) + break; + } + empty = list_empty(&ls->ls_cb_delay); + if (empty) + clear_bit(LSFL_CB_DELAY, &ls->ls_flags); + spin_unlock_bh(&ls->ls_cb_lock); + + sum += count; + if (!empty) { + count = 0; + cond_resched(); + goto more; + } + + if (sum) + log_rinfo(ls, "%s %d", __func__, sum); +} + diff --git a/fs/dlm/ast.h b/fs/dlm/ast.h new file mode 100644 index 000000000..e2b86845d --- /dev/null +++ b/fs/dlm/ast.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) 2005-2010 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#ifndef __ASTD_DOT_H__ +#define __ASTD_DOT_H__ + +bool dlm_may_skip_callback(struct dlm_lkb *lkb, uint32_t flags, int mode, + int status, uint32_t sbflags, int *copy_lvb); +int dlm_get_cb(struct dlm_lkb *lkb, uint32_t flags, int mode, + int status, uint32_t sbflags, + struct dlm_callback **cb); +void dlm_add_cb(struct dlm_lkb *lkb, uint32_t flags, int mode, int status, + uint32_t sbflags); + +int dlm_callback_start(struct dlm_ls *ls); +void dlm_callback_stop(struct dlm_ls *ls); +void dlm_callback_suspend(struct dlm_ls *ls); +void dlm_callback_resume(struct dlm_ls *ls); + +#endif + + diff --git a/fs/dlm/config.c b/fs/dlm/config.c new file mode 100644 index 000000000..53cd33293 --- /dev/null +++ b/fs/dlm/config.c @@ -0,0 +1,1053 @@ +// SPDX-License-Identifier: GPL-2.0-only +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2011 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "config.h" +#include "midcomms.h" +#include "lowcomms.h" + +/* + * /config/dlm//spaces//nodes//nodeid (refers to ) + * /config/dlm//spaces//nodes//weight + * /config/dlm//spaces//nodes//release_recover + * /config/dlm//comms//nodeid (refers to ) + * /config/dlm//comms//local + * /config/dlm//comms//addr (write only) + * /config/dlm//comms//addr_list (read only) + * The level is useless, but I haven't figured out how to avoid it. + */ + +static struct config_group *space_list; +static struct config_group *comm_list; +static struct dlm_comm *local_comm; +static uint32_t dlm_comm_count; + +struct dlm_clusters; +struct dlm_cluster; +struct dlm_spaces; +struct dlm_space; +struct dlm_comms; +struct dlm_comm; +struct dlm_nodes; +struct dlm_node; + +static struct config_group *make_cluster(struct config_group *, const char *); +static void drop_cluster(struct config_group *, struct config_item *); +static void release_cluster(struct config_item *); +static struct config_group *make_space(struct config_group *, const char *); +static void drop_space(struct config_group *, struct config_item *); +static void release_space(struct config_item *); +static struct config_item *make_comm(struct config_group *, const char *); +static void drop_comm(struct config_group *, struct config_item *); +static void release_comm(struct config_item *); +static struct config_item *make_node(struct config_group *, const char *); +static void drop_node(struct config_group *, struct config_item *); +static void release_node(struct config_item *); + +static struct configfs_attribute *comm_attrs[]; +static struct configfs_attribute *node_attrs[]; + +const struct rhashtable_params dlm_rhash_rsb_params = { + .nelem_hint = 3, /* start small */ + .key_len = DLM_RESNAME_MAXLEN, + .key_offset = offsetof(struct dlm_rsb, res_name), + .head_offset = offsetof(struct dlm_rsb, res_node), + .automatic_shrinking = true, +}; + +struct dlm_cluster { + struct config_group group; + struct dlm_spaces *sps; + struct dlm_comms *cms; +}; + +static struct dlm_cluster *config_item_to_cluster(struct config_item *i) +{ + return i ? container_of(to_config_group(i), struct dlm_cluster, group) : + NULL; +} + +enum { + CLUSTER_ATTR_TCP_PORT = 0, + CLUSTER_ATTR_BUFFER_SIZE, + CLUSTER_ATTR_RSBTBL_SIZE, + CLUSTER_ATTR_RECOVER_TIMER, + CLUSTER_ATTR_TOSS_SECS, + CLUSTER_ATTR_SCAN_SECS, + CLUSTER_ATTR_LOG_DEBUG, + CLUSTER_ATTR_LOG_INFO, + CLUSTER_ATTR_PROTOCOL, + CLUSTER_ATTR_MARK, + CLUSTER_ATTR_NEW_RSB_COUNT, + CLUSTER_ATTR_RECOVER_CALLBACKS, + CLUSTER_ATTR_CLUSTER_NAME, +}; + +static ssize_t cluster_cluster_name_show(struct config_item *item, char *buf) +{ + return sprintf(buf, "%s\n", dlm_config.ci_cluster_name); +} + +static ssize_t cluster_cluster_name_store(struct config_item *item, + const char *buf, size_t len) +{ + strscpy(dlm_config.ci_cluster_name, buf, + sizeof(dlm_config.ci_cluster_name)); + return len; +} + +CONFIGFS_ATTR(cluster_, cluster_name); + +static ssize_t cluster_tcp_port_show(struct config_item *item, char *buf) +{ + return sprintf(buf, "%u\n", be16_to_cpu(dlm_config.ci_tcp_port)); +} + +static int dlm_check_zero_and_dlm_running(unsigned int x) +{ + if (!x) + return -EINVAL; + + if (dlm_lowcomms_is_running()) + return -EBUSY; + + return 0; +} + +static ssize_t cluster_tcp_port_store(struct config_item *item, + const char *buf, size_t len) +{ + int rc; + u16 x; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + rc = kstrtou16(buf, 0, &x); + if (rc) + return rc; + + rc = dlm_check_zero_and_dlm_running(x); + if (rc) + return rc; + + dlm_config.ci_tcp_port = cpu_to_be16(x); + return len; +} + +CONFIGFS_ATTR(cluster_, tcp_port); + +static ssize_t cluster_set(unsigned int *info_field, + int (*check_cb)(unsigned int x), + const char *buf, size_t len) +{ + unsigned int x; + int rc; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + rc = kstrtouint(buf, 0, &x); + if (rc) + return rc; + + if (check_cb) { + rc = check_cb(x); + if (rc) + return rc; + } + + *info_field = x; + + return len; +} + +#define CLUSTER_ATTR(name, check_cb) \ +static ssize_t cluster_##name##_store(struct config_item *item, \ + const char *buf, size_t len) \ +{ \ + return cluster_set(&dlm_config.ci_##name, check_cb, buf, len); \ +} \ +static ssize_t cluster_##name##_show(struct config_item *item, char *buf) \ +{ \ + return snprintf(buf, PAGE_SIZE, "%u\n", dlm_config.ci_##name); \ +} \ +CONFIGFS_ATTR(cluster_, name); + +static int dlm_check_protocol_and_dlm_running(unsigned int x) +{ + switch (x) { + case 0: + /* TCP */ + break; + case 1: + /* SCTP */ + if (!IS_ENABLED(CONFIG_IP_SCTP)) + return -EOPNOTSUPP; + + break; + default: + return -EINVAL; + } + + if (dlm_lowcomms_is_running()) + return -EBUSY; + + return 0; +} + +static int dlm_check_zero(unsigned int x) +{ + if (!x) + return -EINVAL; + + return 0; +} + +static int dlm_check_buffer_size(unsigned int x) +{ + if (x < DLM_MAX_SOCKET_BUFSIZE) + return -EINVAL; + + return 0; +} + +CLUSTER_ATTR(buffer_size, dlm_check_buffer_size); +CLUSTER_ATTR(rsbtbl_size, dlm_check_zero); +CLUSTER_ATTR(recover_timer, dlm_check_zero); +CLUSTER_ATTR(toss_secs, dlm_check_zero); +CLUSTER_ATTR(scan_secs, dlm_check_zero); +CLUSTER_ATTR(log_debug, NULL); +CLUSTER_ATTR(log_info, NULL); +CLUSTER_ATTR(protocol, dlm_check_protocol_and_dlm_running); +CLUSTER_ATTR(mark, NULL); +CLUSTER_ATTR(new_rsb_count, NULL); +CLUSTER_ATTR(recover_callbacks, NULL); + +static struct configfs_attribute *cluster_attrs[] = { + [CLUSTER_ATTR_TCP_PORT] = &cluster_attr_tcp_port, + [CLUSTER_ATTR_BUFFER_SIZE] = &cluster_attr_buffer_size, + [CLUSTER_ATTR_RSBTBL_SIZE] = &cluster_attr_rsbtbl_size, + [CLUSTER_ATTR_RECOVER_TIMER] = &cluster_attr_recover_timer, + [CLUSTER_ATTR_TOSS_SECS] = &cluster_attr_toss_secs, + [CLUSTER_ATTR_SCAN_SECS] = &cluster_attr_scan_secs, + [CLUSTER_ATTR_LOG_DEBUG] = &cluster_attr_log_debug, + [CLUSTER_ATTR_LOG_INFO] = &cluster_attr_log_info, + [CLUSTER_ATTR_PROTOCOL] = &cluster_attr_protocol, + [CLUSTER_ATTR_MARK] = &cluster_attr_mark, + [CLUSTER_ATTR_NEW_RSB_COUNT] = &cluster_attr_new_rsb_count, + [CLUSTER_ATTR_RECOVER_CALLBACKS] = &cluster_attr_recover_callbacks, + [CLUSTER_ATTR_CLUSTER_NAME] = &cluster_attr_cluster_name, + NULL, +}; + +enum { + COMM_ATTR_NODEID = 0, + COMM_ATTR_LOCAL, + COMM_ATTR_ADDR, + COMM_ATTR_ADDR_LIST, + COMM_ATTR_MARK, +}; + +enum { + NODE_ATTR_NODEID = 0, + NODE_ATTR_WEIGHT, + NODE_ATTR_RELEASE_RECOVER, +}; + +struct dlm_clusters { + struct configfs_subsystem subsys; +}; + +struct dlm_spaces { + struct config_group ss_group; +}; + +struct dlm_space { + struct config_group group; + struct list_head members; + struct list_head members_gone; + int members_gone_count; + struct mutex members_lock; + int members_count; + struct dlm_nodes *nds; +}; + +struct dlm_comms { + struct config_group cs_group; +}; + +struct dlm_comm { + struct config_item item; + int seq; + int nodeid; + int local; + int addr_count; + unsigned int mark; + struct sockaddr_storage *addr[DLM_MAX_ADDR_COUNT]; +}; + +struct dlm_nodes { + struct config_group ns_group; +}; + +struct dlm_node { + struct config_item item; + struct list_head list; /* space->members */ + int nodeid; + int weight; + int new; + int comm_seq; /* copy of cm->seq when nd->nodeid is set */ + unsigned int release_recover; +}; + +struct dlm_member_gone { + int nodeid; + unsigned int release_recover; + + struct list_head list; /* space->members_gone */ +}; + +static const struct configfs_group_operations clusters_ops = { + .make_group = make_cluster, + .drop_item = drop_cluster, +}; + +static const struct configfs_item_operations cluster_ops = { + .release = release_cluster, +}; + +static const struct configfs_group_operations spaces_ops = { + .make_group = make_space, + .drop_item = drop_space, +}; + +static const struct configfs_item_operations space_ops = { + .release = release_space, +}; + +static const struct configfs_group_operations comms_ops = { + .make_item = make_comm, + .drop_item = drop_comm, +}; + +static const struct configfs_item_operations comm_ops = { + .release = release_comm, +}; + +static const struct configfs_group_operations nodes_ops = { + .make_item = make_node, + .drop_item = drop_node, +}; + +static const struct configfs_item_operations node_ops = { + .release = release_node, +}; + +static const struct config_item_type clusters_type = { + .ct_group_ops = &clusters_ops, + .ct_owner = THIS_MODULE, +}; + +static const struct config_item_type cluster_type = { + .ct_item_ops = &cluster_ops, + .ct_attrs = cluster_attrs, + .ct_owner = THIS_MODULE, +}; + +static const struct config_item_type spaces_type = { + .ct_group_ops = &spaces_ops, + .ct_owner = THIS_MODULE, +}; + +static const struct config_item_type space_type = { + .ct_item_ops = &space_ops, + .ct_owner = THIS_MODULE, +}; + +static const struct config_item_type comms_type = { + .ct_group_ops = &comms_ops, + .ct_owner = THIS_MODULE, +}; + +static const struct config_item_type comm_type = { + .ct_item_ops = &comm_ops, + .ct_attrs = comm_attrs, + .ct_owner = THIS_MODULE, +}; + +static const struct config_item_type nodes_type = { + .ct_group_ops = &nodes_ops, + .ct_owner = THIS_MODULE, +}; + +static const struct config_item_type node_type = { + .ct_item_ops = &node_ops, + .ct_attrs = node_attrs, + .ct_owner = THIS_MODULE, +}; + +static struct dlm_space *config_item_to_space(struct config_item *i) +{ + return i ? container_of(to_config_group(i), struct dlm_space, group) : + NULL; +} + +static struct dlm_comm *config_item_to_comm(struct config_item *i) +{ + return i ? container_of(i, struct dlm_comm, item) : NULL; +} + +static struct dlm_node *config_item_to_node(struct config_item *i) +{ + return i ? container_of(i, struct dlm_node, item) : NULL; +} + +static struct config_group *make_cluster(struct config_group *g, + const char *name) +{ + struct dlm_cluster *cl = NULL; + struct dlm_spaces *sps = NULL; + struct dlm_comms *cms = NULL; + + cl = kzalloc_obj(struct dlm_cluster, GFP_NOFS); + sps = kzalloc_obj(struct dlm_spaces, GFP_NOFS); + cms = kzalloc_obj(struct dlm_comms, GFP_NOFS); + + if (!cl || !sps || !cms) + goto fail; + + cl->sps = sps; + cl->cms = cms; + + config_group_init_type_name(&cl->group, name, &cluster_type); + config_group_init_type_name(&sps->ss_group, "spaces", &spaces_type); + config_group_init_type_name(&cms->cs_group, "comms", &comms_type); + + configfs_add_default_group(&sps->ss_group, &cl->group); + configfs_add_default_group(&cms->cs_group, &cl->group); + + space_list = &sps->ss_group; + comm_list = &cms->cs_group; + return &cl->group; + + fail: + kfree(cl); + kfree(sps); + kfree(cms); + return ERR_PTR(-ENOMEM); +} + +static void drop_cluster(struct config_group *g, struct config_item *i) +{ + struct dlm_cluster *cl = config_item_to_cluster(i); + + configfs_remove_default_groups(&cl->group); + + space_list = NULL; + comm_list = NULL; + + config_item_put(i); +} + +static void release_cluster(struct config_item *i) +{ + struct dlm_cluster *cl = config_item_to_cluster(i); + + kfree(cl->sps); + kfree(cl->cms); + kfree(cl); +} + +static struct config_group *make_space(struct config_group *g, const char *name) +{ + struct dlm_space *sp = NULL; + struct dlm_nodes *nds = NULL; + + sp = kzalloc_obj(struct dlm_space, GFP_NOFS); + nds = kzalloc_obj(struct dlm_nodes, GFP_NOFS); + + if (!sp || !nds) + goto fail; + + config_group_init_type_name(&sp->group, name, &space_type); + + config_group_init_type_name(&nds->ns_group, "nodes", &nodes_type); + configfs_add_default_group(&nds->ns_group, &sp->group); + + INIT_LIST_HEAD(&sp->members); + INIT_LIST_HEAD(&sp->members_gone); + mutex_init(&sp->members_lock); + sp->members_count = 0; + sp->nds = nds; + return &sp->group; + + fail: + kfree(sp); + kfree(nds); + return ERR_PTR(-ENOMEM); +} + +static void drop_space(struct config_group *g, struct config_item *i) +{ + struct dlm_space *sp = config_item_to_space(i); + + /* assert list_empty(&sp->members) */ + + configfs_remove_default_groups(&sp->group); + config_item_put(i); +} + +static void release_space(struct config_item *i) +{ + struct dlm_space *sp = config_item_to_space(i); + kfree(sp->nds); + kfree(sp); +} + +static struct config_item *make_comm(struct config_group *g, const char *name) +{ + struct dlm_comm *cm; + unsigned int nodeid; + int rv; + + rv = kstrtouint(name, 0, &nodeid); + if (rv) + return ERR_PTR(rv); + + cm = kzalloc_obj(struct dlm_comm, GFP_NOFS); + if (!cm) + return ERR_PTR(-ENOMEM); + + config_item_init_type_name(&cm->item, name, &comm_type); + + cm->seq = dlm_comm_count++; + if (!cm->seq) + cm->seq = dlm_comm_count++; + + cm->nodeid = nodeid; + cm->local = 0; + cm->addr_count = 0; + cm->mark = 0; + return &cm->item; +} + +static void drop_comm(struct config_group *g, struct config_item *i) +{ + struct dlm_comm *cm = config_item_to_comm(i); + if (local_comm == cm) + local_comm = NULL; + dlm_midcomms_close(cm->nodeid); + while (cm->addr_count--) + kfree(cm->addr[cm->addr_count]); + config_item_put(i); +} + +static void release_comm(struct config_item *i) +{ + struct dlm_comm *cm = config_item_to_comm(i); + kfree(cm); +} + +static struct config_item *make_node(struct config_group *g, const char *name) +{ + struct dlm_space *sp = config_item_to_space(g->cg_item.ci_parent); + unsigned int nodeid; + struct dlm_node *nd; + uint32_t seq = 0; + int rv; + + rv = kstrtouint(name, 0, &nodeid); + if (rv) + return ERR_PTR(rv); + + nd = kzalloc_obj(struct dlm_node, GFP_NOFS); + if (!nd) + return ERR_PTR(-ENOMEM); + + config_item_init_type_name(&nd->item, name, &node_type); + nd->nodeid = nodeid; + nd->weight = 1; /* default weight of 1 if none is set */ + nd->new = 1; /* set to 0 once it's been read by dlm_nodeid_list() */ + dlm_comm_seq(nodeid, &seq, true); + nd->comm_seq = seq; + + mutex_lock(&sp->members_lock); + list_add(&nd->list, &sp->members); + sp->members_count++; + mutex_unlock(&sp->members_lock); + + return &nd->item; +} + +static void drop_node(struct config_group *g, struct config_item *i) +{ + struct dlm_space *sp = config_item_to_space(g->cg_item.ci_parent); + struct dlm_node *nd = config_item_to_node(i); + struct dlm_member_gone *mb_gone; + + mb_gone = kzalloc_obj(*mb_gone); + if (!mb_gone) + return; + + mutex_lock(&sp->members_lock); + list_del(&nd->list); + sp->members_count--; + + mb_gone->nodeid = nd->nodeid; + mb_gone->release_recover = nd->release_recover; + list_add(&mb_gone->list, &sp->members_gone); + sp->members_gone_count++; + mutex_unlock(&sp->members_lock); + + config_item_put(i); +} + +static void release_node(struct config_item *i) +{ + struct dlm_node *nd = config_item_to_node(i); + kfree(nd); +} + +static struct dlm_clusters clusters_root = { + .subsys = { + .su_group = { + .cg_item = { + .ci_namebuf = "dlm", + .ci_type = &clusters_type, + }, + }, + }, +}; + +int __init dlm_config_init(void) +{ + config_group_init(&clusters_root.subsys.su_group); + mutex_init(&clusters_root.subsys.su_mutex); + return configfs_register_subsystem(&clusters_root.subsys); +} + +void dlm_config_exit(void) +{ + configfs_unregister_subsystem(&clusters_root.subsys); +} + +/* + * Functions for user space to read/write attributes + */ + +static ssize_t comm_nodeid_show(struct config_item *item, char *buf) +{ + unsigned int nodeid; + int rv; + + rv = kstrtouint(config_item_name(item), 0, &nodeid); + if (WARN_ON(rv)) + return rv; + + return sprintf(buf, "%u\n", nodeid); +} + +static ssize_t comm_nodeid_store(struct config_item *item, const char *buf, + size_t len) +{ + return len; +} + +static ssize_t comm_local_show(struct config_item *item, char *buf) +{ + return sprintf(buf, "%d\n", config_item_to_comm(item)->local); +} + +static ssize_t comm_local_store(struct config_item *item, const char *buf, + size_t len) +{ + struct dlm_comm *cm = config_item_to_comm(item); + int rc = kstrtoint(buf, 0, &cm->local); + + if (rc) + return rc; + if (cm->local && !local_comm) + local_comm = cm; + return len; +} + +static ssize_t comm_addr_store(struct config_item *item, const char *buf, + size_t len) +{ + struct dlm_comm *cm = config_item_to_comm(item); + struct sockaddr_storage *addr; + int rv; + + if (len != sizeof(struct sockaddr_storage)) + return -EINVAL; + + if (cm->addr_count >= DLM_MAX_ADDR_COUNT) + return -ENOSPC; + + addr = kzalloc_obj(*addr, GFP_NOFS); + if (!addr) + return -ENOMEM; + + memcpy(addr, buf, len); + + rv = dlm_midcomms_addr(cm->nodeid, addr); + if (rv) { + kfree(addr); + return rv; + } + + cm->addr[cm->addr_count++] = addr; + return len; +} + +static ssize_t comm_addr_list_show(struct config_item *item, char *buf) +{ + struct dlm_comm *cm = config_item_to_comm(item); + ssize_t s; + ssize_t allowance; + int i; + struct sockaddr_storage *addr; + struct sockaddr_in *addr_in; + struct sockaddr_in6 *addr_in6; + + /* Taken from ip6_addr_string() defined in lib/vsprintf.c */ + char buf0[sizeof("AF_INET6 xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:255.255.255.255\n")]; + + + /* Derived from SIMPLE_ATTR_SIZE of fs/configfs/file.c */ + allowance = 4096; + buf[0] = '\0'; + + for (i = 0; i < cm->addr_count; i++) { + addr = cm->addr[i]; + + switch(addr->ss_family) { + case AF_INET: + addr_in = (struct sockaddr_in *)addr; + s = sprintf(buf0, "AF_INET %pI4\n", &addr_in->sin_addr.s_addr); + break; + case AF_INET6: + addr_in6 = (struct sockaddr_in6 *)addr; + s = sprintf(buf0, "AF_INET6 %pI6\n", &addr_in6->sin6_addr); + break; + default: + s = sprintf(buf0, "%s\n", ""); + break; + } + allowance -= s; + if (allowance >= 0) + strcat(buf, buf0); + else { + allowance += s; + break; + } + } + return 4096 - allowance; +} + +static ssize_t comm_mark_show(struct config_item *item, char *buf) +{ + return sprintf(buf, "%u\n", config_item_to_comm(item)->mark); +} + +static ssize_t comm_mark_store(struct config_item *item, const char *buf, + size_t len) +{ + struct dlm_comm *comm; + unsigned int mark; + int rc; + + rc = kstrtouint(buf, 0, &mark); + if (rc) + return rc; + + if (mark == 0) + mark = dlm_config.ci_mark; + + comm = config_item_to_comm(item); + rc = dlm_lowcomms_nodes_set_mark(comm->nodeid, mark); + if (rc) + return rc; + + comm->mark = mark; + return len; +} + +CONFIGFS_ATTR(comm_, nodeid); +CONFIGFS_ATTR(comm_, local); +CONFIGFS_ATTR(comm_, mark); +CONFIGFS_ATTR_WO(comm_, addr); +CONFIGFS_ATTR_RO(comm_, addr_list); + +static struct configfs_attribute *comm_attrs[] = { + [COMM_ATTR_NODEID] = &comm_attr_nodeid, + [COMM_ATTR_LOCAL] = &comm_attr_local, + [COMM_ATTR_ADDR] = &comm_attr_addr, + [COMM_ATTR_ADDR_LIST] = &comm_attr_addr_list, + [COMM_ATTR_MARK] = &comm_attr_mark, + NULL, +}; + +static ssize_t node_nodeid_show(struct config_item *item, char *buf) +{ + unsigned int nodeid; + int rv; + + rv = kstrtouint(config_item_name(item), 0, &nodeid); + if (WARN_ON(rv)) + return rv; + + return sprintf(buf, "%u\n", nodeid); +} + +static ssize_t node_nodeid_store(struct config_item *item, const char *buf, + size_t len) +{ + return len; +} + +static ssize_t node_weight_show(struct config_item *item, char *buf) +{ + return sprintf(buf, "%d\n", config_item_to_node(item)->weight); +} + +static ssize_t node_weight_store(struct config_item *item, const char *buf, + size_t len) +{ + int rc = kstrtoint(buf, 0, &config_item_to_node(item)->weight); + + if (rc) + return rc; + return len; +} + +static ssize_t node_release_recover_show(struct config_item *item, char *buf) +{ + struct dlm_node *n = config_item_to_node(item); + + return sprintf(buf, "%u\n", n->release_recover); +} + +static ssize_t node_release_recover_store(struct config_item *item, + const char *buf, size_t len) +{ + struct dlm_node *n = config_item_to_node(item); + int rc; + + rc = kstrtouint(buf, 0, &n->release_recover); + if (rc) + return rc; + + return len; +} + +CONFIGFS_ATTR(node_, nodeid); +CONFIGFS_ATTR(node_, weight); +CONFIGFS_ATTR(node_, release_recover); + +static struct configfs_attribute *node_attrs[] = { + [NODE_ATTR_NODEID] = &node_attr_nodeid, + [NODE_ATTR_WEIGHT] = &node_attr_weight, + [NODE_ATTR_RELEASE_RECOVER] = &node_attr_release_recover, + NULL, +}; + +/* + * Functions for the dlm to get the info that's been configured + */ + +static struct dlm_space *get_space(char *name) +{ + struct config_item *i; + + if (!space_list) + return NULL; + + mutex_lock(&space_list->cg_subsys->su_mutex); + i = config_group_find_item(space_list, name); + mutex_unlock(&space_list->cg_subsys->su_mutex); + + return config_item_to_space(i); +} + +static void put_space(struct dlm_space *sp) +{ + config_item_put(&sp->group.cg_item); +} + +static struct dlm_comm *get_comm(int nodeid) +{ + struct config_item *i; + struct dlm_comm *cm = NULL; + int found = 0; + + if (!comm_list) + return NULL; + + WARN_ON_ONCE(!mutex_is_locked(&clusters_root.subsys.su_mutex)); + + list_for_each_entry(i, &comm_list->cg_children, ci_entry) { + cm = config_item_to_comm(i); + + if (cm->nodeid != nodeid) + continue; + found = 1; + config_item_get(i); + break; + } + + if (!found) + cm = NULL; + return cm; +} + +static void put_comm(struct dlm_comm *cm) +{ + config_item_put(&cm->item); +} + +/* caller must free mem */ +int dlm_config_nodes(char *lsname, struct dlm_config_node **nodes_out, + int *count_out) +{ + struct dlm_member_gone *mb_gone, *mb_safe; + struct dlm_config_node *nodes, *node; + struct dlm_space *sp; + struct dlm_node *nd; + int rv, count; + + sp = get_space(lsname); + if (!sp) + return -EEXIST; + + mutex_lock(&sp->members_lock); + if (!sp->members_count) { + rv = -EINVAL; + printk(KERN_ERR "dlm: zero members_count\n"); + goto out; + } + + count = sp->members_count + sp->members_gone_count; + + nodes = kzalloc_objs(struct dlm_config_node, count, GFP_NOFS); + if (!nodes) { + rv = -ENOMEM; + goto out; + } + + node = nodes; + list_for_each_entry(nd, &sp->members, list) { + node->nodeid = nd->nodeid; + node->weight = nd->weight; + node->new = nd->new; + node->comm_seq = nd->comm_seq; + node++; + + nd->new = 0; + } + + /* we delay the remove on nodes until here as configfs does + * not support addtional attributes for rmdir(). + */ + list_for_each_entry_safe(mb_gone, mb_safe, &sp->members_gone, list) { + node->nodeid = mb_gone->nodeid; + node->release_recover = mb_gone->release_recover; + node->gone = true; + node++; + + list_del(&mb_gone->list); + sp->members_gone_count--; + kfree(mb_gone); + } + + *count_out = count; + *nodes_out = nodes; + rv = 0; + out: + mutex_unlock(&sp->members_lock); + put_space(sp); + return rv; +} + +int dlm_comm_seq(int nodeid, uint32_t *seq, bool locked) +{ + struct dlm_comm *cm; + + if (locked) { + cm = get_comm(nodeid); + } else { + mutex_lock(&clusters_root.subsys.su_mutex); + cm = get_comm(nodeid); + mutex_unlock(&clusters_root.subsys.su_mutex); + } + if (!cm) + return -ENOENT; + + *seq = cm->seq; + put_comm(cm); + return 0; +} + +int dlm_our_nodeid(void) +{ + return local_comm->nodeid; +} + +/* num 0 is first addr, num 1 is second addr */ +int dlm_our_addr(struct sockaddr_storage *addr, int num) +{ + if (!local_comm) + return -1; + if (num + 1 > local_comm->addr_count) + return -1; + memcpy(addr, local_comm->addr[num], sizeof(*addr)); + return 0; +} + +/* Config file defaults */ +#define DEFAULT_TCP_PORT 21064 +#define DEFAULT_RSBTBL_SIZE 1024 +#define DEFAULT_RECOVER_TIMER 5 +#define DEFAULT_TOSS_SECS 10 +#define DEFAULT_SCAN_SECS 5 +#define DEFAULT_LOG_DEBUG 0 +#define DEFAULT_LOG_INFO 1 +#define DEFAULT_PROTOCOL DLM_PROTO_TCP +#define DEFAULT_MARK 0 +#define DEFAULT_NEW_RSB_COUNT 128 +#define DEFAULT_RECOVER_CALLBACKS 0 +#define DEFAULT_CLUSTER_NAME "" + +struct dlm_config_info dlm_config = { + .ci_tcp_port = cpu_to_be16(DEFAULT_TCP_PORT), + .ci_buffer_size = DLM_MAX_SOCKET_BUFSIZE, + .ci_rsbtbl_size = DEFAULT_RSBTBL_SIZE, + .ci_recover_timer = DEFAULT_RECOVER_TIMER, + .ci_toss_secs = DEFAULT_TOSS_SECS, + .ci_scan_secs = DEFAULT_SCAN_SECS, + .ci_log_debug = DEFAULT_LOG_DEBUG, + .ci_log_info = DEFAULT_LOG_INFO, + .ci_protocol = DEFAULT_PROTOCOL, + .ci_mark = DEFAULT_MARK, + .ci_new_rsb_count = DEFAULT_NEW_RSB_COUNT, + .ci_recover_callbacks = DEFAULT_RECOVER_CALLBACKS, + .ci_cluster_name = DEFAULT_CLUSTER_NAME +}; + diff --git a/fs/dlm/config.h b/fs/dlm/config.h new file mode 100644 index 000000000..4ebd45f75 --- /dev/null +++ b/fs/dlm/config.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2011 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#ifndef __CONFIG_DOT_H__ +#define __CONFIG_DOT_H__ + +#define DLM_MAX_SOCKET_BUFSIZE 4096 + +struct dlm_config_node { + int nodeid; + int weight; + bool gone; + int new; + uint32_t comm_seq; + unsigned int release_recover; +}; + +extern const struct rhashtable_params dlm_rhash_rsb_params; + +#define DLM_MAX_ADDR_COUNT 8 + +#define DLM_PROTO_TCP 0 +#define DLM_PROTO_SCTP 1 + +struct dlm_config_info { + __be16 ci_tcp_port; + unsigned int ci_buffer_size; + unsigned int ci_rsbtbl_size; + unsigned int ci_recover_timer; + unsigned int ci_toss_secs; + unsigned int ci_scan_secs; + unsigned int ci_log_debug; + unsigned int ci_log_info; + unsigned int ci_protocol; + unsigned int ci_mark; + unsigned int ci_new_rsb_count; + unsigned int ci_recover_callbacks; + char ci_cluster_name[DLM_LOCKSPACE_LEN]; +}; + +extern struct dlm_config_info dlm_config; + +int dlm_config_init(void); +void dlm_config_exit(void); +int dlm_config_nodes(char *lsname, struct dlm_config_node **nodes_out, + int *count_out); +int dlm_comm_seq(int nodeid, uint32_t *seq, bool locked); +int dlm_our_nodeid(void); +int dlm_our_addr(struct sockaddr_storage *addr, int num); + +#endif /* __CONFIG_DOT_H__ */ + diff --git a/fs/dlm/debug_fs.c b/fs/dlm/debug_fs.c new file mode 100644 index 000000000..700a0cbb2 --- /dev/null +++ b/fs/dlm/debug_fs.c @@ -0,0 +1,825 @@ +// SPDX-License-Identifier: GPL-2.0-only +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) 2005-2009 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#include +#include +#include +#include +#include +#include + +#include "dlm_internal.h" +#include "midcomms.h" +#include "lock.h" +#include "ast.h" + +#define DLM_DEBUG_BUF_LEN 4096 +static char debug_buf[DLM_DEBUG_BUF_LEN]; +static struct mutex debug_buf_lock; + +static struct dentry *dlm_root; +static struct dentry *dlm_comms; + +static char *print_lockmode(int mode) +{ + switch (mode) { + case DLM_LOCK_IV: + return "--"; + case DLM_LOCK_NL: + return "NL"; + case DLM_LOCK_CR: + return "CR"; + case DLM_LOCK_CW: + return "CW"; + case DLM_LOCK_PR: + return "PR"; + case DLM_LOCK_PW: + return "PW"; + case DLM_LOCK_EX: + return "EX"; + default: + return "??"; + } +} + +static void print_format1_lock(struct seq_file *s, struct dlm_lkb *lkb, + struct dlm_rsb *res) +{ + seq_printf(s, "%08x %s", lkb->lkb_id, print_lockmode(lkb->lkb_grmode)); + + if (lkb->lkb_status == DLM_LKSTS_CONVERT || + lkb->lkb_status == DLM_LKSTS_WAITING) + seq_printf(s, " (%s)", print_lockmode(lkb->lkb_rqmode)); + + if (lkb->lkb_nodeid) { + if (lkb->lkb_nodeid != res->res_nodeid) + seq_printf(s, " Remote: %3d %08x", lkb->lkb_nodeid, + lkb->lkb_remid); + else + seq_printf(s, " Master: %08x", lkb->lkb_remid); + } + + if (lkb->lkb_wait_type) + seq_printf(s, " wait_type: %d", lkb->lkb_wait_type); + + seq_putc(s, '\n'); +} + +static void print_format1(struct dlm_rsb *res, struct seq_file *s) +{ + struct dlm_lkb *lkb; + int i, lvblen = res->res_ls->ls_lvblen, recover_list, root_list; + + lock_rsb(res); + + seq_printf(s, "\nResource %p Name (len=%d) \"", res, res->res_length); + + for (i = 0; i < res->res_length; i++) { + if (isprint(res->res_name[i])) + seq_printf(s, "%c", res->res_name[i]); + else + seq_printf(s, "%c", '.'); + } + + if (res->res_nodeid > 0) + seq_printf(s, "\"\nLocal Copy, Master is node %d\n", + res->res_nodeid); + else if (res->res_nodeid == 0) + seq_puts(s, "\"\nMaster Copy\n"); + else if (res->res_nodeid == -1) + seq_printf(s, "\"\nLooking up master (lkid %x)\n", + res->res_first_lkid); + else + seq_printf(s, "\"\nInvalid master %d\n", res->res_nodeid); + if (seq_has_overflowed(s)) + goto out; + + /* Print the LVB: */ + if (res->res_lvbptr) { + seq_puts(s, "LVB: "); + for (i = 0; i < lvblen; i++) { + if (i == lvblen / 2) + seq_puts(s, "\n "); + seq_printf(s, "%02x ", + (unsigned char) res->res_lvbptr[i]); + } + if (rsb_flag(res, RSB_VALNOTVALID)) + seq_puts(s, " (INVALID)"); + seq_putc(s, '\n'); + if (seq_has_overflowed(s)) + goto out; + } + + root_list = !list_empty(&res->res_root_list); + recover_list = !list_empty(&res->res_recover_list); + + if (root_list || recover_list) { + seq_printf(s, "Recovery: root %d recover %d flags %lx count %d\n", + root_list, recover_list, + res->res_flags, res->res_recover_locks_count); + } + + /* Print the locks attached to this resource */ + seq_puts(s, "Granted Queue\n"); + list_for_each_entry(lkb, &res->res_grantqueue, lkb_statequeue) { + print_format1_lock(s, lkb, res); + if (seq_has_overflowed(s)) + goto out; + } + + seq_puts(s, "Conversion Queue\n"); + list_for_each_entry(lkb, &res->res_convertqueue, lkb_statequeue) { + print_format1_lock(s, lkb, res); + if (seq_has_overflowed(s)) + goto out; + } + + seq_puts(s, "Waiting Queue\n"); + list_for_each_entry(lkb, &res->res_waitqueue, lkb_statequeue) { + print_format1_lock(s, lkb, res); + if (seq_has_overflowed(s)) + goto out; + } + + if (list_empty(&res->res_lookup)) + goto out; + + seq_puts(s, "Lookup Queue\n"); + list_for_each_entry(lkb, &res->res_lookup, lkb_rsb_lookup) { + seq_printf(s, "%08x %s", + lkb->lkb_id, print_lockmode(lkb->lkb_rqmode)); + if (lkb->lkb_wait_type) + seq_printf(s, " wait_type: %d", lkb->lkb_wait_type); + seq_putc(s, '\n'); + if (seq_has_overflowed(s)) + goto out; + } + out: + unlock_rsb(res); +} + +static void print_format2_lock(struct seq_file *s, struct dlm_lkb *lkb, + struct dlm_rsb *r) +{ + u64 xid = 0; + u64 us; + + if (test_bit(DLM_DFL_USER_BIT, &lkb->lkb_dflags)) { + if (lkb->lkb_ua) + xid = lkb->lkb_ua->xid; + } + + /* microseconds since lkb was added to current queue */ + us = ktime_to_us(ktime_sub(ktime_get(), lkb->lkb_timestamp)); + + /* id nodeid remid pid xid exflags flags sts grmode rqmode time_us + r_nodeid r_len r_name */ + + seq_printf(s, "%x %d %x %u %llu %x %x %d %d %d %llu %u %d \"%s\"\n", + lkb->lkb_id, + lkb->lkb_nodeid, + lkb->lkb_remid, + lkb->lkb_ownpid, + (unsigned long long)xid, + lkb->lkb_exflags, + dlm_iflags_val(lkb), + lkb->lkb_status, + lkb->lkb_grmode, + lkb->lkb_rqmode, + (unsigned long long)us, + r->res_nodeid, + r->res_length, + r->res_name); +} + +static void print_format2(struct dlm_rsb *r, struct seq_file *s) +{ + struct dlm_lkb *lkb; + + lock_rsb(r); + + list_for_each_entry(lkb, &r->res_grantqueue, lkb_statequeue) { + print_format2_lock(s, lkb, r); + if (seq_has_overflowed(s)) + goto out; + } + + list_for_each_entry(lkb, &r->res_convertqueue, lkb_statequeue) { + print_format2_lock(s, lkb, r); + if (seq_has_overflowed(s)) + goto out; + } + + list_for_each_entry(lkb, &r->res_waitqueue, lkb_statequeue) { + print_format2_lock(s, lkb, r); + if (seq_has_overflowed(s)) + goto out; + } + out: + unlock_rsb(r); +} + +static void print_format3_lock(struct seq_file *s, struct dlm_lkb *lkb, + int rsb_lookup) +{ + u64 xid = 0; + + if (test_bit(DLM_DFL_USER_BIT, &lkb->lkb_dflags)) { + if (lkb->lkb_ua) + xid = lkb->lkb_ua->xid; + } + + seq_printf(s, "lkb %x %d %x %u %llu %x %x %d %d %d %d %d %d %u %llu %llu\n", + lkb->lkb_id, + lkb->lkb_nodeid, + lkb->lkb_remid, + lkb->lkb_ownpid, + (unsigned long long)xid, + lkb->lkb_exflags, + dlm_iflags_val(lkb), + lkb->lkb_status, + lkb->lkb_grmode, + lkb->lkb_rqmode, + lkb->lkb_last_bast_cb_mode, + rsb_lookup, + lkb->lkb_wait_type, + lkb->lkb_lvbseq, + (unsigned long long)ktime_to_ns(lkb->lkb_timestamp), + (unsigned long long)ktime_to_ns(lkb->lkb_last_bast_time)); +} + +static void print_format3(struct dlm_rsb *r, struct seq_file *s) +{ + struct dlm_lkb *lkb; + int i, lvblen = r->res_ls->ls_lvblen; + int print_name = 1; + + lock_rsb(r); + + seq_printf(s, "rsb %p %d %x %lx %d %d %u %d ", + r, + r->res_nodeid, + r->res_first_lkid, + r->res_flags, + !list_empty(&r->res_root_list), + !list_empty(&r->res_recover_list), + r->res_recover_locks_count, + r->res_length); + if (seq_has_overflowed(s)) + goto out; + + for (i = 0; i < r->res_length; i++) { + if (!isascii(r->res_name[i]) || !isprint(r->res_name[i])) + print_name = 0; + } + + seq_puts(s, print_name ? "str " : "hex"); + + for (i = 0; i < r->res_length; i++) { + if (print_name) + seq_printf(s, "%c", r->res_name[i]); + else + seq_printf(s, " %02x", (unsigned char)r->res_name[i]); + } + seq_putc(s, '\n'); + if (seq_has_overflowed(s)) + goto out; + + if (!r->res_lvbptr) + goto do_locks; + + seq_printf(s, "lvb %u %d", r->res_lvbseq, lvblen); + + for (i = 0; i < lvblen; i++) + seq_printf(s, " %02x", (unsigned char)r->res_lvbptr[i]); + seq_putc(s, '\n'); + if (seq_has_overflowed(s)) + goto out; + + do_locks: + list_for_each_entry(lkb, &r->res_grantqueue, lkb_statequeue) { + print_format3_lock(s, lkb, 0); + if (seq_has_overflowed(s)) + goto out; + } + + list_for_each_entry(lkb, &r->res_convertqueue, lkb_statequeue) { + print_format3_lock(s, lkb, 0); + if (seq_has_overflowed(s)) + goto out; + } + + list_for_each_entry(lkb, &r->res_waitqueue, lkb_statequeue) { + print_format3_lock(s, lkb, 0); + if (seq_has_overflowed(s)) + goto out; + } + + list_for_each_entry(lkb, &r->res_lookup, lkb_rsb_lookup) { + print_format3_lock(s, lkb, 1); + if (seq_has_overflowed(s)) + goto out; + } + out: + unlock_rsb(r); +} + +static void print_format4(struct dlm_rsb *r, struct seq_file *s) +{ + int our_nodeid = dlm_our_nodeid(); + int print_name = 1; + int i; + + lock_rsb(r); + + seq_printf(s, "rsb %p %d %d %d %d %lu %lx %d ", + r, + r->res_nodeid, + r->res_master_nodeid, + r->res_dir_nodeid, + our_nodeid, + r->res_toss_time, + r->res_flags, + r->res_length); + + for (i = 0; i < r->res_length; i++) { + if (!isascii(r->res_name[i]) || !isprint(r->res_name[i])) + print_name = 0; + } + + seq_puts(s, print_name ? "str " : "hex"); + + for (i = 0; i < r->res_length; i++) { + if (print_name) + seq_printf(s, "%c", r->res_name[i]); + else + seq_printf(s, " %02x", (unsigned char)r->res_name[i]); + } + seq_putc(s, '\n'); + unlock_rsb(r); +} + +static const struct seq_operations format1_seq_ops; +static const struct seq_operations format2_seq_ops; +static const struct seq_operations format3_seq_ops; +static const struct seq_operations format4_seq_ops; + +/* + * If the buffer is full, seq_printf can be called again, but it + * does nothing. So, the these printing routines periodically check + * seq_has_overflowed to avoid wasting too much time trying to print to + * a full buffer. + */ + +static int table_seq_show(struct seq_file *seq, void *iter_ptr) +{ + struct dlm_rsb *rsb = list_entry(iter_ptr, struct dlm_rsb, res_slow_list); + + if (seq->op == &format1_seq_ops) + print_format1(rsb, seq); + else if (seq->op == &format2_seq_ops) + print_format2(rsb, seq); + else if (seq->op == &format3_seq_ops) + print_format3(rsb, seq); + else if (seq->op == &format4_seq_ops) + print_format4(rsb, seq); + + return 0; +} + +static void *table_seq_start(struct seq_file *seq, loff_t *pos) +{ + struct dlm_ls *ls = seq->private; + struct list_head *list; + + if (!*pos) { + if (seq->op == &format2_seq_ops) + seq_puts(seq, "id nodeid remid pid xid exflags flags sts grmode rqmode time_ms r_nodeid r_len r_name\n"); + else if (seq->op == &format3_seq_ops) + seq_puts(seq, "rsb ptr nodeid first_lkid flags !root_list_empty !recover_list_empty recover_locks_count len\n"); + else if (seq->op == &format4_seq_ops) + seq_puts(seq, "rsb ptr nodeid master_nodeid dir_nodeid our_nodeid toss_time flags len str|hex name\n"); + } + + if (seq->op == &format4_seq_ops) + list = &ls->ls_slow_inactive; + else + list = &ls->ls_slow_active; + + read_lock_bh(&ls->ls_rsbtbl_lock); + return seq_list_start(list, *pos); +} + +static void *table_seq_next(struct seq_file *seq, void *iter_ptr, loff_t *pos) +{ + struct dlm_ls *ls = seq->private; + struct list_head *list; + + if (seq->op == &format4_seq_ops) + list = &ls->ls_slow_inactive; + else + list = &ls->ls_slow_active; + + return seq_list_next(iter_ptr, list, pos); +} + +static void table_seq_stop(struct seq_file *seq, void *iter_ptr) +{ + struct dlm_ls *ls = seq->private; + + read_unlock_bh(&ls->ls_rsbtbl_lock); +} + +static const struct seq_operations format1_seq_ops = { + .start = table_seq_start, + .next = table_seq_next, + .stop = table_seq_stop, + .show = table_seq_show, +}; + +static const struct seq_operations format2_seq_ops = { + .start = table_seq_start, + .next = table_seq_next, + .stop = table_seq_stop, + .show = table_seq_show, +}; + +static const struct seq_operations format3_seq_ops = { + .start = table_seq_start, + .next = table_seq_next, + .stop = table_seq_stop, + .show = table_seq_show, +}; + +static const struct seq_operations format4_seq_ops = { + .start = table_seq_start, + .next = table_seq_next, + .stop = table_seq_stop, + .show = table_seq_show, +}; + +static const struct file_operations format1_fops; +static const struct file_operations format2_fops; +static const struct file_operations format3_fops; +static const struct file_operations format4_fops; + +static int table_open1(struct inode *inode, struct file *file) +{ + struct seq_file *seq; + int ret; + + ret = seq_open(file, &format1_seq_ops); + if (ret) + return ret; + + seq = file->private_data; + seq->private = inode->i_private; /* the dlm_ls */ + return 0; +} + +static int table_open2(struct inode *inode, struct file *file) +{ + struct seq_file *seq; + int ret; + + ret = seq_open(file, &format2_seq_ops); + if (ret) + return ret; + + seq = file->private_data; + seq->private = inode->i_private; /* the dlm_ls */ + return 0; +} + +static ssize_t table_write2(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct seq_file *seq = file->private_data; + int n, len, lkb_nodeid, lkb_status, error; + char name[DLM_RESNAME_MAXLEN + 1] = {}; + struct dlm_ls *ls = seq->private; + unsigned int lkb_flags; + char buf[256] = {}; + uint32_t lkb_id; + + if (copy_from_user(buf, user_buf, + min_t(size_t, sizeof(buf) - 1, count))) + return -EFAULT; + + n = sscanf(buf, "%x %" __stringify(DLM_RESNAME_MAXLEN) "s %x %d %d", + &lkb_id, name, &lkb_flags, &lkb_nodeid, &lkb_status); + if (n != 5) + return -EINVAL; + + len = strnlen(name, DLM_RESNAME_MAXLEN); + error = dlm_debug_add_lkb(ls, lkb_id, name, len, lkb_flags, + lkb_nodeid, lkb_status); + if (error) + return error; + + return count; +} + +static int table_open3(struct inode *inode, struct file *file) +{ + struct seq_file *seq; + int ret; + + ret = seq_open(file, &format3_seq_ops); + if (ret) + return ret; + + seq = file->private_data; + seq->private = inode->i_private; /* the dlm_ls */ + return 0; +} + +static int table_open4(struct inode *inode, struct file *file) +{ + struct seq_file *seq; + int ret; + + ret = seq_open(file, &format4_seq_ops); + if (ret) + return ret; + + seq = file->private_data; + seq->private = inode->i_private; /* the dlm_ls */ + return 0; +} + +static const struct file_operations format1_fops = { + .owner = THIS_MODULE, + .open = table_open1, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release +}; + +static const struct file_operations format2_fops = { + .owner = THIS_MODULE, + .open = table_open2, + .read = seq_read, + .write = table_write2, + .llseek = seq_lseek, + .release = seq_release +}; + +static const struct file_operations format3_fops = { + .owner = THIS_MODULE, + .open = table_open3, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release +}; + +static const struct file_operations format4_fops = { + .owner = THIS_MODULE, + .open = table_open4, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release +}; + +/* + * dump lkb's on the ls_waiters list + */ +static ssize_t waiters_read(struct file *file, char __user *userbuf, + size_t count, loff_t *ppos) +{ + struct dlm_ls *ls = file->private_data; + struct dlm_lkb *lkb; + size_t len = DLM_DEBUG_BUF_LEN, pos = 0, ret, rv; + + mutex_lock(&debug_buf_lock); + ret = dlm_lock_recovery_try(ls); + if (!ret) { + rv = -EAGAIN; + goto out; + } + + spin_lock_bh(&ls->ls_waiters_lock); + memset(debug_buf, 0, sizeof(debug_buf)); + + list_for_each_entry(lkb, &ls->ls_waiters, lkb_wait_reply) { + ret = snprintf(debug_buf + pos, len - pos, "%x %d %d %s\n", + lkb->lkb_id, lkb->lkb_wait_type, + lkb->lkb_nodeid, lkb->lkb_resource->res_name); + if (ret >= len - pos) + break; + pos += ret; + } + spin_unlock_bh(&ls->ls_waiters_lock); + dlm_unlock_recovery(ls); + + rv = simple_read_from_buffer(userbuf, count, ppos, debug_buf, pos); +out: + mutex_unlock(&debug_buf_lock); + return rv; +} + +static ssize_t waiters_write(struct file *file, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct dlm_ls *ls = file->private_data; + int mstype, to_nodeid; + char buf[128] = {}; + uint32_t lkb_id; + int n, error; + + if (copy_from_user(buf, user_buf, + min_t(size_t, sizeof(buf) - 1, count))) + return -EFAULT; + + n = sscanf(buf, "%x %d %d", &lkb_id, &mstype, &to_nodeid); + if (n != 3) + return -EINVAL; + + error = dlm_lock_recovery_try(ls); + if (!error) + return -EAGAIN; + + error = dlm_debug_add_lkb_to_waiters(ls, lkb_id, mstype, to_nodeid); + dlm_unlock_recovery(ls); + if (error) + return error; + + return count; +} + +static const struct file_operations waiters_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = waiters_read, + .write = waiters_write, + .llseek = default_llseek, +}; + +void dlm_delete_debug_file(struct dlm_ls *ls) +{ + debugfs_remove(ls->ls_debug_rsb_dentry); + debugfs_remove(ls->ls_debug_waiters_dentry); + debugfs_remove(ls->ls_debug_locks_dentry); + debugfs_remove(ls->ls_debug_all_dentry); + debugfs_remove(ls->ls_debug_toss_dentry); + debugfs_remove(ls->ls_debug_queued_asts_dentry); +} + +static int dlm_state_show(struct seq_file *file, void *offset) +{ + seq_printf(file, "%s\n", dlm_midcomms_state(file->private)); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(dlm_state); + +static int dlm_flags_show(struct seq_file *file, void *offset) +{ + seq_printf(file, "%lu\n", dlm_midcomms_flags(file->private)); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(dlm_flags); + +static int dlm_send_queue_cnt_show(struct seq_file *file, void *offset) +{ + seq_printf(file, "%d\n", dlm_midcomms_send_queue_cnt(file->private)); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(dlm_send_queue_cnt); + +static int dlm_version_show(struct seq_file *file, void *offset) +{ + seq_printf(file, "0x%08x\n", dlm_midcomms_version(file->private)); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(dlm_version); + +static ssize_t dlm_rawmsg_write(struct file *fp, const char __user *user_buf, + size_t count, loff_t *ppos) +{ + void *buf; + int ret; + + if (count > PAGE_SIZE || count < sizeof(struct dlm_header)) + return -EINVAL; + + buf = kmalloc(PAGE_SIZE, GFP_NOFS); + if (!buf) + return -ENOMEM; + + if (copy_from_user(buf, user_buf, count)) { + ret = -EFAULT; + goto out; + } + + ret = dlm_midcomms_rawmsg_send(fp->private_data, buf, count); + if (ret) + goto out; + + kfree(buf); + return count; + +out: + kfree(buf); + return ret; +} + +static const struct file_operations dlm_rawmsg_fops = { + .open = simple_open, + .write = dlm_rawmsg_write, +}; + +void *dlm_create_debug_comms_file(int nodeid, void *data) +{ + struct dentry *d_node; + char name[256]; + + memset(name, 0, sizeof(name)); + snprintf(name, 256, "%d", nodeid); + + d_node = debugfs_create_dir(name, dlm_comms); + debugfs_create_file("state", 0444, d_node, data, &dlm_state_fops); + debugfs_create_file("flags", 0444, d_node, data, &dlm_flags_fops); + debugfs_create_file("send_queue_count", 0444, d_node, data, + &dlm_send_queue_cnt_fops); + debugfs_create_file("version", 0444, d_node, data, &dlm_version_fops); + debugfs_create_file("rawmsg", 0200, d_node, data, &dlm_rawmsg_fops); + + return d_node; +} + +void dlm_delete_debug_comms_file(void *ctx) +{ + debugfs_remove(ctx); +} + +void dlm_create_debug_file(struct dlm_ls *ls) +{ + /* Reserve enough space for the longest file name */ + char name[DLM_LOCKSPACE_LEN + sizeof("_queued_asts")]; + + /* format 1 */ + + ls->ls_debug_rsb_dentry = debugfs_create_file(ls->ls_name, + S_IFREG | S_IRUGO, + dlm_root, + ls, + &format1_fops); + + /* format 2 */ + + snprintf(name, sizeof(name), "%s_locks", ls->ls_name); + + ls->ls_debug_locks_dentry = debugfs_create_file(name, + 0644, + dlm_root, + ls, + &format2_fops); + + /* format 3 */ + + snprintf(name, sizeof(name), "%s_all", ls->ls_name); + + ls->ls_debug_all_dentry = debugfs_create_file(name, + S_IFREG | S_IRUGO, + dlm_root, + ls, + &format3_fops); + + /* format 4 */ + + snprintf(name, sizeof(name), "%s_toss", ls->ls_name); + + ls->ls_debug_toss_dentry = debugfs_create_file(name, + S_IFREG | S_IRUGO, + dlm_root, + ls, + &format4_fops); + + snprintf(name, sizeof(name), "%s_waiters", ls->ls_name); + + ls->ls_debug_waiters_dentry = debugfs_create_file(name, + 0644, + dlm_root, + ls, + &waiters_fops); +} + +void __init dlm_register_debugfs(void) +{ + mutex_init(&debug_buf_lock); + dlm_root = debugfs_create_dir("dlm", NULL); + dlm_comms = debugfs_create_dir("comms", dlm_root); +} + +void dlm_unregister_debugfs(void) +{ + debugfs_remove(dlm_root); +} + diff --git a/fs/dlm/dir.c b/fs/dlm/dir.c new file mode 100644 index 000000000..aa08e8f44 --- /dev/null +++ b/fs/dlm/dir.c @@ -0,0 +1,400 @@ +// SPDX-License-Identifier: GPL-2.0-only +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#include "dlm_internal.h" +#include "lockspace.h" +#include "member.h" +#include "lowcomms.h" +#include "rcom.h" +#include "config.h" +#include "memory.h" +#include "recover.h" +#include "util.h" +#include "lock.h" +#include "dir.h" + +/* + * We use the upper 16 bits of the hash value to select the directory node. + * Low bits are used for distribution of rsb's among hash buckets on each node. + * + * To give the exact range wanted (0 to num_nodes-1), we apply a modulus of + * num_nodes to the hash value. This value in the desired range is used as an + * offset into the sorted list of nodeid's to give the particular nodeid. + */ + +int dlm_hash2nodeid(struct dlm_ls *ls, uint32_t hash) +{ + uint32_t node; + + if (ls->ls_num_nodes == 1) + return dlm_our_nodeid(); + else { + node = (hash >> 16) % ls->ls_total_weight; + return ls->ls_node_array[node]; + } +} + +int dlm_dir_nodeid(struct dlm_rsb *r) +{ + return r->res_dir_nodeid; +} + +void dlm_recover_dir_nodeid(struct dlm_ls *ls, const struct list_head *root_list) +{ + struct dlm_rsb *r; + + list_for_each_entry(r, root_list, res_root_list) { + r->res_dir_nodeid = dlm_hash2nodeid(ls, r->res_hash); + } +} + +int dlm_recover_directory(struct dlm_ls *ls, uint64_t seq) +{ + struct dlm_member *memb; + char *b, *last_name = NULL; + int error = -ENOMEM, last_len, nodeid, result; + uint16_t namelen; + unsigned int count = 0, count_bad = 0, count_add = 0; + + log_rinfo(ls, "dlm_recover_directory"); + + if (dlm_no_directory(ls)) + goto out_status; + + last_name = kmalloc(DLM_RESNAME_MAXLEN, GFP_NOFS); + if (!last_name) + goto out; + + list_for_each_entry(memb, &ls->ls_nodes, list) { + if (memb->nodeid == dlm_our_nodeid()) + continue; + + memset(last_name, 0, DLM_RESNAME_MAXLEN); + last_len = 0; + + for (;;) { + int left; + if (dlm_recovery_stopped(ls)) { + error = -EINTR; + goto out_free; + } + + error = dlm_rcom_names(ls, memb->nodeid, + last_name, last_len, seq); + if (error) + goto out_free; + + cond_resched(); + + /* + * pick namelen/name pairs out of received buffer + */ + + b = ls->ls_recover_buf->rc_buf; + left = le16_to_cpu(ls->ls_recover_buf->rc_header.h_length); + left -= sizeof(struct dlm_rcom); + + for (;;) { + __be16 v; + + error = -EINVAL; + if (left < sizeof(__be16)) + goto out_free; + + memcpy(&v, b, sizeof(__be16)); + namelen = be16_to_cpu(v); + b += sizeof(__be16); + left -= sizeof(__be16); + + /* namelen of 0xFFFFF marks end of names for + this node; namelen of 0 marks end of the + buffer */ + + if (namelen == 0xFFFF) + goto done; + if (!namelen) + break; + + if (namelen > left) + goto out_free; + + if (namelen > DLM_RESNAME_MAXLEN) + goto out_free; + + error = dlm_master_lookup(ls, memb->nodeid, + b, namelen, + DLM_LU_RECOVER_DIR, + &nodeid, &result); + if (error) { + log_error(ls, "recover_dir lookup %d", + error); + goto out_free; + } + + /* The name was found in rsbtbl, but the + * master nodeid is different from + * memb->nodeid which says it is the master. + * This should not happen. */ + + if (result == DLM_LU_MATCH && + nodeid != memb->nodeid) { + count_bad++; + log_error(ls, "recover_dir lookup %d " + "nodeid %d memb %d bad %u", + result, nodeid, memb->nodeid, + count_bad); + print_hex_dump_bytes("dlm_recover_dir ", + DUMP_PREFIX_NONE, + b, namelen); + } + + /* The name was found in rsbtbl, and the + * master nodeid matches memb->nodeid. + + if (result == DLM_LU_MATCH && + nodeid == memb->nodeid) { + count_match++; + }*/ + + /* The name was not found in rsbtbl and was + * added with memb->nodeid as the master. */ + + if (result == DLM_LU_ADD) { + count_add++; + } + + last_len = namelen; + memcpy(last_name, b, namelen); + b += namelen; + left -= namelen; + count++; + } + } + done: + ; + } + + out_status: + error = 0; + dlm_set_recover_status(ls, DLM_RS_DIR); + + log_rinfo(ls, "dlm_recover_directory %u in %u new", + count, count_add); + out_free: + kfree(last_name); + out: + return error; +} + +static struct dlm_rsb *find_rsb_root(struct dlm_ls *ls, const char *name, + int len) +{ + struct dlm_rsb *r; + int rv; + + read_lock_bh(&ls->ls_rsbtbl_lock); + rv = dlm_search_rsb_tree(&ls->ls_rsbtbl, name, len, &r); + read_unlock_bh(&ls->ls_rsbtbl_lock); + if (!rv) + return r; + + list_for_each_entry(r, &ls->ls_masters_list, res_masters_list) { + if (len == r->res_length && !memcmp(name, r->res_name, len)) { + log_debug(ls, "find_rsb_root revert to root_list %s", + r->res_name); + return r; + } + } + return NULL; +} + +struct dlm_dir_dump { + /* init values to match if whole + * dump fits to one seq. Sanity check only. + */ + uint64_t seq_init; + uint64_t nodeid_init; + /* compare local pointer with last lookup, + * just a sanity check. + */ + struct list_head *last; + + unsigned int sent_res; /* for log info */ + unsigned int sent_msg; /* for log info */ + + struct list_head list; +}; + +static void drop_dir_ctx(struct dlm_ls *ls, int nodeid) +{ + struct dlm_dir_dump *dd, *safe; + + write_lock_bh(&ls->ls_dir_dump_lock); + list_for_each_entry_safe(dd, safe, &ls->ls_dir_dump_list, list) { + if (dd->nodeid_init == nodeid) { + log_error(ls, "drop dump seq %llu", + (unsigned long long)dd->seq_init); + list_del(&dd->list); + kfree(dd); + } + } + write_unlock_bh(&ls->ls_dir_dump_lock); +} + +static struct dlm_dir_dump *lookup_dir_dump(struct dlm_ls *ls, int nodeid) +{ + struct dlm_dir_dump *iter, *dd = NULL; + + read_lock_bh(&ls->ls_dir_dump_lock); + list_for_each_entry(iter, &ls->ls_dir_dump_list, list) { + if (iter->nodeid_init == nodeid) { + dd = iter; + break; + } + } + read_unlock_bh(&ls->ls_dir_dump_lock); + + return dd; +} + +static struct dlm_dir_dump *init_dir_dump(struct dlm_ls *ls, int nodeid) +{ + struct dlm_dir_dump *dd; + + dd = lookup_dir_dump(ls, nodeid); + if (dd) { + log_error(ls, "found ongoing dir dump for node %d, will drop it", + nodeid); + drop_dir_ctx(ls, nodeid); + } + + dd = kzalloc_obj(*dd, GFP_ATOMIC); + if (!dd) + return NULL; + + dd->seq_init = ls->ls_recover_seq; + dd->nodeid_init = nodeid; + + write_lock_bh(&ls->ls_dir_dump_lock); + list_add(&dd->list, &ls->ls_dir_dump_list); + write_unlock_bh(&ls->ls_dir_dump_lock); + + return dd; +} + +/* Find the rsb where we left off (or start again), then send rsb names + for rsb's we're master of and whose directory node matches the requesting + node. inbuf is the rsb name last sent, inlen is the name's length */ + +void dlm_copy_master_names(struct dlm_ls *ls, const char *inbuf, int inlen, + char *outbuf, int outlen, int nodeid) +{ + struct list_head *list; + struct dlm_rsb *r; + int offset = 0, dir_nodeid; + struct dlm_dir_dump *dd; + __be16 be_namelen; + + read_lock_bh(&ls->ls_masters_lock); + + if (inlen > 1) { + dd = lookup_dir_dump(ls, nodeid); + if (!dd) { + log_error(ls, "failed to lookup dir dump context nodeid: %d", + nodeid); + goto out; + } + + /* next chunk in dump */ + r = find_rsb_root(ls, inbuf, inlen); + if (!r) { + log_error(ls, "copy_master_names from %d start %d %.*s", + nodeid, inlen, inlen, inbuf); + goto out; + } + list = r->res_masters_list.next; + + /* sanity checks */ + if (dd->last != &r->res_masters_list || + dd->seq_init != ls->ls_recover_seq) { + log_error(ls, "failed dir dump sanity check seq_init: %llu seq: %llu", + (unsigned long long)dd->seq_init, + (unsigned long long)ls->ls_recover_seq); + goto out; + } + } else { + dd = init_dir_dump(ls, nodeid); + if (!dd) { + log_error(ls, "failed to allocate dir dump context"); + goto out; + } + + /* start dump */ + list = ls->ls_masters_list.next; + dd->last = list; + } + + for (offset = 0; list != &ls->ls_masters_list; list = list->next) { + r = list_entry(list, struct dlm_rsb, res_masters_list); + dir_nodeid = dlm_dir_nodeid(r); + if (dir_nodeid != nodeid) + continue; + + /* + * The block ends when we can't fit the following in the + * remaining buffer space: + * namelen (uint16_t) + + * name (r->res_length) + + * end-of-block record 0x0000 (uint16_t) + */ + + if (offset + sizeof(uint16_t)*2 + r->res_length > outlen) { + /* Write end-of-block record */ + be_namelen = cpu_to_be16(0); + memcpy(outbuf + offset, &be_namelen, sizeof(__be16)); + offset += sizeof(__be16); + dd->sent_msg++; + goto out; + } + + be_namelen = cpu_to_be16(r->res_length); + memcpy(outbuf + offset, &be_namelen, sizeof(__be16)); + offset += sizeof(__be16); + memcpy(outbuf + offset, r->res_name, r->res_length); + offset += r->res_length; + dd->sent_res++; + dd->last = list; + } + + /* + * If we've reached the end of the list (and there's room) write a + * terminating record. + */ + + if ((list == &ls->ls_masters_list) && + (offset + sizeof(uint16_t) <= outlen)) { + /* end dump */ + be_namelen = cpu_to_be16(0xFFFF); + memcpy(outbuf + offset, &be_namelen, sizeof(__be16)); + offset += sizeof(__be16); + dd->sent_msg++; + log_rinfo(ls, "dlm_recover_directory nodeid %d sent %u res out %u messages", + nodeid, dd->sent_res, dd->sent_msg); + + write_lock_bh(&ls->ls_dir_dump_lock); + list_del_init(&dd->list); + write_unlock_bh(&ls->ls_dir_dump_lock); + kfree(dd); + } + out: + read_unlock_bh(&ls->ls_masters_lock); +} + diff --git a/fs/dlm/dir.h b/fs/dlm/dir.h new file mode 100644 index 000000000..5b2a7ee37 --- /dev/null +++ b/fs/dlm/dir.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#ifndef __DIR_DOT_H__ +#define __DIR_DOT_H__ + +int dlm_dir_nodeid(struct dlm_rsb *rsb); +int dlm_hash2nodeid(struct dlm_ls *ls, uint32_t hash); +void dlm_recover_dir_nodeid(struct dlm_ls *ls, + const struct list_head *root_list); +int dlm_recover_directory(struct dlm_ls *ls, uint64_t seq); +void dlm_copy_master_names(struct dlm_ls *ls, const char *inbuf, int inlen, + char *outbuf, int outlen, int nodeid); + +#endif /* __DIR_DOT_H__ */ + diff --git a/fs/dlm/dlm_internal.h b/fs/dlm/dlm_internal.h new file mode 100644 index 000000000..9df842421 --- /dev/null +++ b/fs/dlm/dlm_internal.h @@ -0,0 +1,833 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2011 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#ifndef __DLM_INTERNAL_DOT_H__ +#define __DLM_INTERNAL_DOT_H__ + +/* + * This is the main header file to be included in each DLM source file. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "config.h" + +struct dlm_ls; +struct dlm_lkb; +struct dlm_rsb; +struct dlm_member; +struct dlm_rsbtable; +struct dlm_recover; +struct dlm_header; +struct dlm_message; +struct dlm_rcom; +struct dlm_mhandle; +struct dlm_msg; + +#define log_print(fmt, args...) \ + printk(KERN_ERR "dlm: "fmt"\n" , ##args) +#define log_print_ratelimited(fmt, args...) \ + printk_ratelimited(KERN_ERR "dlm: "fmt"\n", ##args) +#define log_error(ls, fmt, args...) \ + printk(KERN_ERR "dlm: %s: " fmt "\n", (ls)->ls_name , ##args) + +#define log_rinfo(ls, fmt, args...) \ +do { \ + if (dlm_config.ci_log_info) \ + printk(KERN_INFO "dlm: %s: " fmt "\n", \ + (ls)->ls_name, ##args); \ + else if (dlm_config.ci_log_debug) \ + printk(KERN_DEBUG "dlm: %s: " fmt "\n", \ + (ls)->ls_name , ##args); \ +} while (0) + +#define log_debug(ls, fmt, args...) \ +do { \ + if (dlm_config.ci_log_debug) \ + printk(KERN_DEBUG "dlm: %s: " fmt "\n", \ + (ls)->ls_name , ##args); \ +} while (0) + +#define log_limit(ls, fmt, args...) \ +do { \ + if (dlm_config.ci_log_debug) \ + printk_ratelimited(KERN_DEBUG "dlm: %s: " fmt "\n", \ + (ls)->ls_name , ##args); \ +} while (0) + +#define DLM_ASSERT(x, do) \ +{ \ + if (!(x)) \ + { \ + printk(KERN_ERR "\nDLM: Assertion failed on line %d of file %s\n" \ + "DLM: assertion: \"%s\"\n" \ + "DLM: time = %lu\n", \ + __LINE__, __FILE__, #x, jiffies); \ + {do} \ + printk("\n"); \ + panic("DLM: Record message above and reboot.\n"); \ + } \ +} + +/* + * Lockspace member (per node in a ls) + */ + +struct dlm_member { + struct list_head list; + int nodeid; + int weight; + int slot; + int slot_prev; + int comm_seq; + uint32_t generation; +}; + +/* + * Save and manage recovery state for a lockspace. + */ + +struct dlm_recover { + struct list_head list; + struct dlm_config_node *nodes; + int nodes_count; + uint64_t seq; +}; + +/* + * Pass input args to second stage locking function. + */ + +struct dlm_args { + uint32_t flags; + void (*astfn) (void *astparam); + void *astparam; + void (*bastfn) (void *astparam, int mode); + int mode; + struct dlm_lksb *lksb; +}; + + +/* + * Lock block + * + * A lock can be one of three types: + * + * local copy lock is mastered locally + * (lkb_nodeid is zero and DLM_LKF_MSTCPY is not set) + * process copy lock is mastered on a remote node + * (lkb_nodeid is non-zero and DLM_LKF_MSTCPY is not set) + * master copy master node's copy of a lock owned by remote node + * (lkb_nodeid is non-zero and DLM_LKF_MSTCPY is set) + * + * lkb_exflags: a copy of the most recent flags arg provided to dlm_lock or + * dlm_unlock. The dlm does not modify these or use any private flags in + * this field; it only contains DLM_LKF_ flags from dlm.h. These flags + * are sent as-is to the remote master when the lock is remote. + * + * lkb_flags: internal dlm flags (DLM_IFL_ prefix) from dlm_internal.h. + * Some internal flags are shared between the master and process nodes; + * these shared flags are kept in the lower two bytes. One of these + * flags set on the master copy will be propagated to the process copy + * and v.v. Other internal flags are private to the master or process + * node (e.g. DLM_IFL_MSTCPY). These are kept in the high two bytes. + * + * lkb_sbflags: status block flags. These flags are copied directly into + * the caller's lksb.sb_flags prior to the dlm_lock/dlm_unlock completion + * ast. All defined in dlm.h with DLM_SBF_ prefix. + * + * lkb_status: the lock status indicates which rsb queue the lock is + * on, grant, convert, or wait. DLM_LKSTS_ WAITING/GRANTED/CONVERT + * + * lkb_wait_type: the dlm message type (DLM_MSG_ prefix) for which a + * reply is needed. Only set when the lkb is on the lockspace waiters + * list awaiting a reply from a remote node. + * + * lkb_nodeid: when the lkb is a local copy, nodeid is 0; when the lkb + * is a master copy, nodeid specifies the remote lock holder, when the + * lkb is a process copy, the nodeid specifies the lock master. + */ + +/* lkb_status */ + +#define DLM_LKSTS_WAITING 1 +#define DLM_LKSTS_GRANTED 2 +#define DLM_LKSTS_CONVERT 3 + +/* lkb_iflags */ + +#define DLM_IFL_MSTCPY_BIT 16 +#define __DLM_IFL_MIN_BIT DLM_IFL_MSTCPY_BIT +#define DLM_IFL_RESEND_BIT 17 +#define DLM_IFL_DEAD_BIT 18 +#define DLM_IFL_OVERLAP_UNLOCK_BIT 19 +#define DLM_IFL_OVERLAP_CANCEL_BIT 20 +#define DLM_IFL_ENDOFLIFE_BIT 21 +#define DLM_IFL_DEADLOCK_CANCEL_BIT 24 +#define __DLM_IFL_MAX_BIT DLM_IFL_DEADLOCK_CANCEL_BIT + +/* lkb_dflags */ + +#define DLM_DFL_USER_BIT 0 +#define __DLM_DFL_MIN_BIT DLM_DFL_USER_BIT +#define DLM_DFL_ORPHAN_BIT 1 +#define __DLM_DFL_MAX_BIT DLM_DFL_ORPHAN_BIT + +#define DLM_CB_CAST 0x00000001 +#define DLM_CB_BAST 0x00000002 + +/* much of this is just saving user space pointers associated with the + * lock that we pass back to the user lib with an ast + */ + +struct dlm_user_args { + struct dlm_user_proc *proc; /* each process that opens the lockspace + * device has private data + * (dlm_user_proc) on the struct file, + * the process's locks point back to it + */ + struct dlm_lksb lksb; + struct dlm_lksb __user *user_lksb; + void __user *castparam; + void __user *castaddr; + void __user *bastparam; + void __user *bastaddr; + uint64_t xid; +}; + +struct dlm_callback { + uint32_t flags; /* DLM_CBF_ */ + int sb_status; /* copy to lksb status */ + uint8_t sb_flags; /* copy to lksb flags */ + int8_t mode; /* rq mode of bast, gr mode of cast */ + bool copy_lvb; + struct dlm_lksb *lkb_lksb; + unsigned char lvbptr[DLM_USER_LVB_LEN]; + + union { + void *astparam; /* caller's ast arg */ + struct dlm_user_args ua; + }; + struct work_struct work; + void (*bastfn)(void *astparam, int mode); + void (*astfn)(void *astparam); + char res_name[DLM_RESNAME_MAXLEN]; + size_t res_length; + uint32_t ls_id; + uint32_t lkb_id; + + struct list_head list; +}; + +struct dlm_lkb { + struct dlm_rsb *lkb_resource; /* the rsb */ + struct kref lkb_ref; + int lkb_nodeid; /* copied from rsb */ + int lkb_ownpid; /* pid of lock owner */ + uint32_t lkb_id; /* our lock ID */ + uint32_t lkb_remid; /* lock ID on remote partner */ + uint32_t lkb_exflags; /* external flags from caller */ + unsigned long lkb_sbflags; /* lksb flags */ + unsigned long lkb_dflags; /* distributed flags */ + unsigned long lkb_iflags; /* internal flags */ + uint32_t lkb_lvbseq; /* lvb sequence number */ + + int8_t lkb_status; /* granted, waiting, convert */ + int8_t lkb_rqmode; /* requested lock mode */ + int8_t lkb_grmode; /* granted lock mode */ + int8_t lkb_highbast; /* highest mode bast sent for */ + + int8_t lkb_wait_type; /* type of reply waiting for */ + int8_t lkb_wait_count; + int lkb_wait_nodeid; /* for debugging */ + + struct list_head lkb_statequeue; /* rsb g/c/w list */ + struct list_head lkb_rsb_lookup; /* waiting for rsb lookup */ + struct list_head lkb_wait_reply; /* waiting for remote reply */ + struct list_head lkb_ownqueue; /* list of locks for a process */ + ktime_t lkb_timestamp; + + int8_t lkb_last_cast_cb_mode; + int8_t lkb_last_bast_cb_mode; + int8_t lkb_last_cb_mode; + uint8_t lkb_last_cb_flags; + ktime_t lkb_last_cast_time; /* for debugging */ + ktime_t lkb_last_bast_time; /* for debugging */ + + uint64_t lkb_recover_seq; /* from ls_recover_seq */ + + char *lkb_lvbptr; + struct dlm_lksb *lkb_lksb; /* caller's status block */ + void (*lkb_astfn) (void *astparam); + void (*lkb_bastfn) (void *astparam, int mode); + union { + void *lkb_astparam; /* caller's ast arg */ + struct dlm_user_args *lkb_ua; + }; + struct rcu_head rcu; +}; + +/* + * res_master_nodeid is "normal": 0 is unset/invalid, non-zero is the real + * nodeid, even when nodeid is our_nodeid. + * + * res_nodeid is "odd": -1 is unset/invalid, zero means our_nodeid, + * greater than zero when another nodeid. + * + * (TODO: remove res_nodeid and only use res_master_nodeid) + */ + +struct dlm_rsb { + struct dlm_ls *res_ls; /* the lockspace */ + struct kref res_ref; + spinlock_t res_lock; + unsigned long res_flags; + int res_length; /* length of rsb name */ + int res_nodeid; + int res_master_nodeid; + int res_dir_nodeid; + unsigned long res_id; /* for ls_recover_xa */ + uint32_t res_lvbseq; + uint32_t res_hash; + unsigned long res_toss_time; + uint32_t res_first_lkid; + struct list_head res_lookup; /* lkbs waiting on first */ + struct rhash_head res_node; /* rsbtbl */ + struct list_head res_grantqueue; + struct list_head res_convertqueue; + struct list_head res_waitqueue; + + struct list_head res_slow_list; /* ls_slow_* */ + struct list_head res_scan_list; + struct list_head res_root_list; /* used for recovery */ + struct list_head res_masters_list; /* used for recovery */ + struct list_head res_recover_list; /* used for recovery */ + int res_recover_locks_count; + struct rcu_head rcu; + + char *res_lvbptr; + char res_name[DLM_RESNAME_MAXLEN+1]; +}; + +/* dlm_master_lookup() flags */ + +#define DLM_LU_RECOVER_DIR 1 +#define DLM_LU_RECOVER_MASTER 2 + +/* dlm_master_lookup() results */ + +#define DLM_LU_MATCH 1 +#define DLM_LU_ADD 2 + +/* find_rsb() flags */ + +#define R_REQUEST 0x00000001 +#define R_RECEIVE_REQUEST 0x00000002 +#define R_RECEIVE_RECOVER 0x00000004 + +/* rsb_flags */ + +enum rsb_flags { + RSB_MASTER_UNCERTAIN, + RSB_VALNOTVALID, + RSB_VALNOTVALID_PREV, + RSB_NEW_MASTER, + RSB_NEW_MASTER2, + RSB_RECOVER_CONVERT, + RSB_RECOVER_GRANT, + RSB_RECOVER_LVB_INVAL, + RSB_INACTIVE, + RSB_HASHED, /* set while rsb is on ls_rsbtbl */ +}; + +static inline void rsb_set_flag(struct dlm_rsb *r, enum rsb_flags flag) +{ + __set_bit(flag, &r->res_flags); +} + +static inline void rsb_clear_flag(struct dlm_rsb *r, enum rsb_flags flag) +{ + __clear_bit(flag, &r->res_flags); +} + +static inline int rsb_flag(struct dlm_rsb *r, enum rsb_flags flag) +{ + return test_bit(flag, &r->res_flags); +} + + +/* dlm_header is first element of all structs sent between nodes */ + +#define DLM_HEADER_MAJOR 0x00030000 +#define DLM_HEADER_MINOR 0x00000002 + +#define DLM_VERSION_3_1 0x00030001 +#define DLM_VERSION_3_2 0x00030002 + +#define DLM_HEADER_SLOTS 0x00000001 + +#define DLM_MSG 1 +#define DLM_RCOM 2 +#define DLM_OPTS 3 +#define DLM_ACK 4 +#define DLM_FIN 5 + +struct dlm_header { + __le32 h_version; + union { + /* for DLM_MSG and DLM_RCOM */ + __le32 h_lockspace; + /* for DLM_ACK and DLM_OPTS */ + __le32 h_seq; + } u; + __le32 h_nodeid; /* nodeid of sender */ + __le16 h_length; + uint8_t h_cmd; /* DLM_MSG, DLM_RCOM */ + uint8_t h_pad; +}; + +#define DLM_MSG_REQUEST 1 +#define DLM_MSG_CONVERT 2 +#define DLM_MSG_UNLOCK 3 +#define DLM_MSG_CANCEL 4 +#define DLM_MSG_REQUEST_REPLY 5 +#define DLM_MSG_CONVERT_REPLY 6 +#define DLM_MSG_UNLOCK_REPLY 7 +#define DLM_MSG_CANCEL_REPLY 8 +#define DLM_MSG_GRANT 9 +#define DLM_MSG_BAST 10 +#define DLM_MSG_LOOKUP 11 +#define DLM_MSG_REMOVE 12 +#define DLM_MSG_LOOKUP_REPLY 13 +#define DLM_MSG_PURGE 14 + +struct dlm_message { + struct dlm_header m_header; + __le32 m_type; /* DLM_MSG_ */ + __le32 m_nodeid; + __le32 m_pid; + __le32 m_lkid; /* lkid on sender */ + __le32 m_remid; /* lkid on receiver */ + __le32 m_parent_lkid; + __le32 m_parent_remid; + __le32 m_exflags; + __le32 m_sbflags; + __le32 m_flags; + __le32 m_lvbseq; + __le32 m_hash; + __le32 m_status; + __le32 m_grmode; + __le32 m_rqmode; + __le32 m_bastmode; + __le32 m_asts; + __le32 m_result; /* 0 or -EXXX */ + char m_extra[]; /* name or lvb */ +}; + + +#define DLM_RS_NODES 0x00000001 +#define DLM_RS_NODES_ALL 0x00000002 +#define DLM_RS_DIR 0x00000004 +#define DLM_RS_DIR_ALL 0x00000008 +#define DLM_RS_LOCKS 0x00000010 +#define DLM_RS_LOCKS_ALL 0x00000020 +#define DLM_RS_DONE 0x00000040 +#define DLM_RS_DONE_ALL 0x00000080 + +#define DLM_RCOM_STATUS 1 +#define DLM_RCOM_NAMES 2 +#define DLM_RCOM_LOOKUP 3 +#define DLM_RCOM_LOCK 4 +#define DLM_RCOM_STATUS_REPLY 5 +#define DLM_RCOM_NAMES_REPLY 6 +#define DLM_RCOM_LOOKUP_REPLY 7 +#define DLM_RCOM_LOCK_REPLY 8 + +struct dlm_rcom { + struct dlm_header rc_header; + __le32 rc_type; /* DLM_RCOM_ */ + __le32 rc_result; /* multi-purpose */ + __le64 rc_id; /* match reply with request */ + __le64 rc_seq; /* sender's ls_recover_seq */ + __le64 rc_seq_reply; /* remote ls_recover_seq */ + char rc_buf[]; +}; + +struct dlm_opt_header { + __le16 t_type; + __le16 t_length; + __le32 t_pad; + /* need to be 8 byte aligned */ + char t_value[]; +}; + +/* encapsulation header */ +struct dlm_opts { + struct dlm_header o_header; + uint8_t o_nextcmd; + uint8_t o_pad; + __le16 o_optlen; + __le32 o_pad2; + char o_opts[]; +}; + +union dlm_packet { + struct dlm_header header; /* common to other two */ + struct dlm_message message; + struct dlm_rcom rcom; + struct dlm_opts opts; +}; + +#define DLM_RSF_NEED_SLOTS 0x00000001 + +/* RCOM_STATUS data */ +struct rcom_status { + __le32 rs_flags; + __le32 rs_unused1; + __le64 rs_unused2; +}; + +/* RCOM_STATUS_REPLY data */ +struct rcom_config { + __le32 rf_lvblen; + __le32 rf_lsflags; + + /* DLM_HEADER_SLOTS adds: */ + __le32 rf_flags; + __le16 rf_our_slot; + __le16 rf_num_slots; + __le32 rf_generation; + __le32 rf_unused1; + __le64 rf_unused2; +}; + +struct rcom_slot { + __le32 ro_nodeid; + __le16 ro_slot; + __le16 ro_unused1; + __le64 ro_unused2; +}; + +struct rcom_lock { + __le32 rl_ownpid; + __le32 rl_lkid; + __le32 rl_remid; + __le32 rl_parent_lkid; + __le32 rl_parent_remid; + __le32 rl_exflags; + __le32 rl_flags; + __le32 rl_lvbseq; + __le32 rl_result; + int8_t rl_rqmode; + int8_t rl_grmode; + int8_t rl_status; + int8_t rl_asts; + __le16 rl_wait_type; + __le16 rl_namelen; + char rl_name[DLM_RESNAME_MAXLEN]; + char rl_lvb[]; +}; + +struct dlm_ls { + struct list_head ls_list; /* list of lockspaces */ + uint32_t ls_global_id; /* global unique lockspace ID */ + uint32_t ls_generation; + uint32_t ls_exflags; + int ls_lvblen; + atomic_t ls_count; /* refcount of processes in + the dlm using this ls */ + wait_queue_head_t ls_count_wait; + int ls_create_count; /* create/release refcount */ + unsigned long ls_flags; /* LSFL_ */ + struct kobject ls_kobj; + + struct xarray ls_lkbxa; + rwlock_t ls_lkbxa_lock; + + /* an rsb is on rsbtl for primary locking functions, + and on a slow list for recovery/dump iteration */ + struct rhashtable ls_rsbtbl; + rwlock_t ls_rsbtbl_lock; /* for ls_rsbtbl and ls_slow */ + struct list_head ls_slow_inactive; /* to iterate rsbtbl */ + struct list_head ls_slow_active; /* to iterate rsbtbl */ + + struct timer_list ls_scan_timer; /* based on first scan_list rsb toss_time */ + struct list_head ls_scan_list; /* rsbs ordered by res_toss_time */ + spinlock_t ls_scan_lock; + + spinlock_t ls_waiters_lock; + struct list_head ls_waiters; /* lkbs needing a reply */ + + spinlock_t ls_orphans_lock; + struct list_head ls_orphans; + + struct list_head ls_nodes; /* current nodes in ls */ + struct list_head ls_nodes_gone; /* dead node list, recovery */ + int ls_num_nodes; /* number of nodes in ls */ + int ls_low_nodeid; + int ls_total_weight; + int *ls_node_array; + + int ls_slot; + int ls_num_slots; + int ls_slots_size; + struct dlm_slot *ls_slots; + + struct dlm_rsb ls_local_rsb; /* for returning errors */ + struct dlm_lkb ls_local_lkb; /* for returning errors */ + + struct dentry *ls_debug_rsb_dentry; /* debugfs */ + struct dentry *ls_debug_waiters_dentry; /* debugfs */ + struct dentry *ls_debug_locks_dentry; /* debugfs */ + struct dentry *ls_debug_all_dentry; /* debugfs */ + struct dentry *ls_debug_toss_dentry; /* debugfs */ + struct dentry *ls_debug_queued_asts_dentry; /* debugfs */ + + wait_queue_head_t ls_uevent_wait; /* user part of join/leave */ + int ls_uevent_result; + struct completion ls_recovery_done; + int ls_recovery_result; + + struct miscdevice ls_device; + + struct workqueue_struct *ls_callback_wq; + + /* recovery related */ + + spinlock_t ls_cb_lock; + struct list_head ls_cb_delay; /* save for queue_work later */ + struct task_struct *ls_recoverd_task; + struct mutex ls_recoverd_active; + spinlock_t ls_recover_lock; + unsigned long ls_recover_begin; /* jiffies timestamp */ + uint32_t ls_recover_status; /* DLM_RS_ */ + uint64_t ls_recover_seq; + struct dlm_recover *ls_recover_args; + struct rw_semaphore ls_in_recovery; /* block local requests */ + rwlock_t ls_recv_active; /* block dlm_recv */ + struct list_head ls_requestqueue;/* queue remote requests */ + rwlock_t ls_requestqueue_lock; + struct dlm_rcom *ls_recover_buf; + int ls_recover_nodeid; /* for debugging */ + unsigned int ls_recover_locks_in; /* for log info */ + uint64_t ls_rcom_seq; + spinlock_t ls_rcom_spin; + struct list_head ls_recover_list; + spinlock_t ls_recover_list_lock; + int ls_recover_list_count; + struct xarray ls_recover_xa; + spinlock_t ls_recover_xa_lock; + wait_queue_head_t ls_wait_general; + wait_queue_head_t ls_recover_lock_wait; + spinlock_t ls_clear_proc_locks; + + struct list_head ls_masters_list; /* root resources */ + rwlock_t ls_masters_lock; /* protect root_list */ + struct list_head ls_dir_dump_list; /* root resources */ + rwlock_t ls_dir_dump_lock; /* protect root_list */ + + const struct dlm_lockspace_ops *ls_ops; + void *ls_ops_arg; + + struct work_struct ls_free_work; + + int ls_namelen; + char ls_name[DLM_LOCKSPACE_LEN + 1]; + + /* Must be last --ends in a flexible-array member.*/ + struct dlm_message ls_local_ms; /* for faking a reply */ +}; + +/* + * LSFL_RECOVER_STOP - dlm_ls_stop() sets this to tell dlm recovery routines + * that they should abort what they're doing so new recovery can be started. + * + * LSFL_RECOVER_DOWN - dlm_ls_stop() sets this to tell dlm_recoverd that it + * should do down_write() on the in_recovery rw_semaphore. (doing down_write + * within dlm_ls_stop causes complaints about the lock acquired/released + * in different contexts.) + * + * LSFL_RECOVER_LOCK - dlm_recoverd holds the in_recovery rw_semaphore. + * It sets this after it is done with down_write() on the in_recovery + * rw_semaphore and clears it after it has released the rw_semaphore. + * + * LSFL_RECOVER_WORK - dlm_ls_start() sets this to tell dlm_recoverd that it + * should begin recovery of the lockspace. + * + * LSFL_RUNNING - set when normal locking activity is enabled. + * dlm_ls_stop() clears this to tell dlm locking routines that they should + * quit what they are doing so recovery can run. dlm_recoverd sets + * this after recovery is finished. + */ + +#define LSFL_RECOVER_STOP 0 +#define LSFL_RECOVER_DOWN 1 +#define LSFL_RECOVER_LOCK 2 +#define LSFL_RECOVER_WORK 3 +#define LSFL_RUNNING 4 + +#define LSFL_RCOM_READY 5 +#define LSFL_RCOM_WAIT 6 +#define LSFL_UEVENT_WAIT 7 +#define LSFL_CB_DELAY 9 +#define LSFL_NODIR 10 +#define LSFL_RECV_MSG_BLOCKED 11 +#define LSFL_FS 12 +#define LSFL_SOFTIRQ 13 + +#define DLM_PROC_FLAGS_CLOSING 1 +#define DLM_PROC_FLAGS_COMPAT 2 + +/* locks list is kept so we can remove all a process's locks when it + exits (or orphan those that are persistent) */ + +struct dlm_user_proc { + dlm_lockspace_t *lockspace; + unsigned long flags; /* DLM_PROC_FLAGS */ + struct list_head asts; + spinlock_t asts_spin; + struct list_head locks; + spinlock_t locks_spin; + struct list_head unlocking; + wait_queue_head_t wait; +}; + +static inline int dlm_locking_stopped(struct dlm_ls *ls) +{ + return !test_bit(LSFL_RUNNING, &ls->ls_flags); +} + +static inline int dlm_recovery_stopped(struct dlm_ls *ls) +{ + return test_bit(LSFL_RECOVER_STOP, &ls->ls_flags); +} + +static inline int dlm_no_directory(struct dlm_ls *ls) +{ + return test_bit(LSFL_NODIR, &ls->ls_flags); +} + +/* takes a snapshot from dlm atomic flags */ +static inline uint32_t dlm_flags_val(const unsigned long *addr, + uint32_t min, uint32_t max) +{ + uint32_t bit = min, val = 0; + + for_each_set_bit_from(bit, addr, max + 1) { + val |= BIT(bit); + } + + return val; +} + +static inline uint32_t dlm_iflags_val(const struct dlm_lkb *lkb) +{ + return dlm_flags_val(&lkb->lkb_iflags, __DLM_IFL_MIN_BIT, + __DLM_IFL_MAX_BIT); +} + +static inline uint32_t dlm_dflags_val(const struct dlm_lkb *lkb) +{ + return dlm_flags_val(&lkb->lkb_dflags, __DLM_DFL_MIN_BIT, + __DLM_DFL_MAX_BIT); +} + +/* coming from UAPI header + * + * TODO: + * Move this to UAPI header and let other values point to them and use BIT() + */ +#define DLM_SBF_DEMOTED_BIT 0 +#define __DLM_SBF_MIN_BIT DLM_SBF_DEMOTED_BIT +#define DLM_SBF_VALNOTVALID_BIT 1 +#define DLM_SBF_ALTMODE_BIT 2 +#define __DLM_SBF_MAX_BIT DLM_SBF_ALTMODE_BIT + +static inline uint32_t dlm_sbflags_val(const struct dlm_lkb *lkb) +{ + /* be sure the next person updates this */ + BUILD_BUG_ON(BIT(__DLM_SBF_MAX_BIT) != DLM_SBF_ALTMODE); + + return dlm_flags_val(&lkb->lkb_sbflags, __DLM_SBF_MIN_BIT, + __DLM_SBF_MAX_BIT); +} + +static inline void dlm_set_flags_val(unsigned long *addr, uint32_t val, + uint32_t min, uint32_t max) +{ + uint32_t bit; + + for (bit = min; bit < (max + 1); bit++) { + if (val & BIT(bit)) + set_bit(bit, addr); + else + clear_bit(bit, addr); + } +} + +static inline void dlm_set_dflags_val(struct dlm_lkb *lkb, uint32_t val) +{ + dlm_set_flags_val(&lkb->lkb_dflags, val, __DLM_DFL_MIN_BIT, + __DLM_DFL_MAX_BIT); +} + +static inline void dlm_set_sbflags_val(struct dlm_lkb *lkb, uint32_t val) +{ + dlm_set_flags_val(&lkb->lkb_sbflags, val, __DLM_SBF_MIN_BIT, + __DLM_SBF_MAX_BIT); +} + +extern struct workqueue_struct *dlm_wq; + +int dlm_plock_init(void); +void dlm_plock_exit(void); + +#ifdef CONFIG_DLM_DEBUG +void dlm_register_debugfs(void); +void dlm_unregister_debugfs(void); +void dlm_create_debug_file(struct dlm_ls *ls); +void dlm_delete_debug_file(struct dlm_ls *ls); +void *dlm_create_debug_comms_file(int nodeid, void *data); +void dlm_delete_debug_comms_file(void *ctx); +#else +static inline void dlm_register_debugfs(void) { } +static inline void dlm_unregister_debugfs(void) { } +static inline void dlm_create_debug_file(struct dlm_ls *ls) { } +static inline void dlm_delete_debug_file(struct dlm_ls *ls) { } +static inline void *dlm_create_debug_comms_file(int nodeid, void *data) { return NULL; } +static inline void dlm_delete_debug_comms_file(void *ctx) { } +#endif + +#endif /* __DLM_INTERNAL_DOT_H__ */ + diff --git a/fs/dlm/lock.c b/fs/dlm/lock.c new file mode 100644 index 000000000..c381e1028 --- /dev/null +++ b/fs/dlm/lock.c @@ -0,0 +1,6338 @@ +// SPDX-License-Identifier: GPL-2.0-only +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) 2005-2010 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +/* Central locking logic has four stages: + + dlm_lock() + dlm_unlock() + + request_lock(ls, lkb) + convert_lock(ls, lkb) + unlock_lock(ls, lkb) + cancel_lock(ls, lkb) + + _request_lock(r, lkb) + _convert_lock(r, lkb) + _unlock_lock(r, lkb) + _cancel_lock(r, lkb) + + do_request(r, lkb) + do_convert(r, lkb) + do_unlock(r, lkb) + do_cancel(r, lkb) + + Stage 1 (lock, unlock) is mainly about checking input args and + splitting into one of the four main operations: + + dlm_lock = request_lock + dlm_lock+CONVERT = convert_lock + dlm_unlock = unlock_lock + dlm_unlock+CANCEL = cancel_lock + + Stage 2, xxxx_lock(), just finds and locks the relevant rsb which is + provided to the next stage. + + Stage 3, _xxxx_lock(), determines if the operation is local or remote. + When remote, it calls send_xxxx(), when local it calls do_xxxx(). + + Stage 4, do_xxxx(), is the guts of the operation. It manipulates the + given rsb and lkb and queues callbacks. + + For remote operations, send_xxxx() results in the corresponding do_xxxx() + function being executed on the remote node. The connecting send/receive + calls on local (L) and remote (R) nodes: + + L: send_xxxx() -> R: receive_xxxx() + R: do_xxxx() + L: receive_xxxx_reply() <- R: send_xxxx_reply() +*/ +#include + +#include +#include +#include +#include "dlm_internal.h" +#include +#include "memory.h" +#include "midcomms.h" +#include "requestqueue.h" +#include "util.h" +#include "dir.h" +#include "member.h" +#include "lockspace.h" +#include "ast.h" +#include "lock.h" +#include "rcom.h" +#include "recover.h" +#include "lvb_table.h" +#include "user.h" +#include "config.h" + +static int send_request(struct dlm_rsb *r, struct dlm_lkb *lkb); +static int send_convert(struct dlm_rsb *r, struct dlm_lkb *lkb); +static int send_unlock(struct dlm_rsb *r, struct dlm_lkb *lkb); +static int send_cancel(struct dlm_rsb *r, struct dlm_lkb *lkb); +static int send_grant(struct dlm_rsb *r, struct dlm_lkb *lkb); +static int send_bast(struct dlm_rsb *r, struct dlm_lkb *lkb, int mode); +static int send_lookup(struct dlm_rsb *r, struct dlm_lkb *lkb); +static int send_remove(struct dlm_rsb *r); +static int _request_lock(struct dlm_rsb *r, struct dlm_lkb *lkb); +static int _cancel_lock(struct dlm_rsb *r, struct dlm_lkb *lkb); +static void __receive_convert_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, + const struct dlm_message *ms, bool local); +static int receive_extralen(const struct dlm_message *ms); +static void do_purge(struct dlm_ls *ls, int nodeid, int pid); +static void deactivate_rsb(struct kref *kref); + +/* + * Lock compatibilty matrix - thanks Steve + * UN = Unlocked state. Not really a state, used as a flag + * PD = Padding. Used to make the matrix a nice power of two in size + * Other states are the same as the VMS DLM. + * Usage: matrix[grmode+1][rqmode+1] (although m[rq+1][gr+1] is the same) + */ + +static const int __dlm_compat_matrix[8][8] = { + /* UN NL CR CW PR PW EX PD */ + {1, 1, 1, 1, 1, 1, 1, 0}, /* UN */ + {1, 1, 1, 1, 1, 1, 1, 0}, /* NL */ + {1, 1, 1, 1, 1, 1, 0, 0}, /* CR */ + {1, 1, 1, 1, 0, 0, 0, 0}, /* CW */ + {1, 1, 1, 0, 1, 0, 0, 0}, /* PR */ + {1, 1, 1, 0, 0, 0, 0, 0}, /* PW */ + {1, 1, 0, 0, 0, 0, 0, 0}, /* EX */ + {0, 0, 0, 0, 0, 0, 0, 0} /* PD */ +}; + +/* + * This defines the direction of transfer of LVB data. + * Granted mode is the row; requested mode is the column. + * Usage: matrix[grmode+1][rqmode+1] + * 1 = LVB is returned to the caller + * 0 = LVB is written to the resource + * -1 = nothing happens to the LVB + */ + +const int dlm_lvb_operations[8][8] = { + /* UN NL CR CW PR PW EX PD*/ + { -1, 1, 1, 1, 1, 1, 1, -1 }, /* UN */ + { -1, 1, 1, 1, 1, 1, 1, 0 }, /* NL */ + { -1, -1, 1, 1, 1, 1, 1, 0 }, /* CR */ + { -1, -1, -1, 1, 1, 1, 1, 0 }, /* CW */ + { -1, -1, -1, -1, 1, 1, 1, 0 }, /* PR */ + { -1, 0, 0, 0, 0, 0, 1, 0 }, /* PW */ + { -1, 0, 0, 0, 0, 0, 0, 0 }, /* EX */ + { -1, 0, 0, 0, 0, 0, 0, 0 } /* PD */ +}; + +#define modes_compat(gr, rq) \ + __dlm_compat_matrix[(gr)->lkb_grmode + 1][(rq)->lkb_rqmode + 1] + +int dlm_modes_compat(int mode1, int mode2) +{ + return __dlm_compat_matrix[mode1 + 1][mode2 + 1]; +} + +/* + * Compatibility matrix for conversions with QUECVT set. + * Granted mode is the row; requested mode is the column. + * Usage: matrix[grmode+1][rqmode+1] + */ + +static const int __quecvt_compat_matrix[8][8] = { + /* UN NL CR CW PR PW EX PD */ + {0, 0, 0, 0, 0, 0, 0, 0}, /* UN */ + {0, 0, 1, 1, 1, 1, 1, 0}, /* NL */ + {0, 0, 0, 1, 1, 1, 1, 0}, /* CR */ + {0, 0, 0, 0, 1, 1, 1, 0}, /* CW */ + {0, 0, 0, 1, 0, 1, 1, 0}, /* PR */ + {0, 0, 0, 0, 0, 0, 1, 0}, /* PW */ + {0, 0, 0, 0, 0, 0, 0, 0}, /* EX */ + {0, 0, 0, 0, 0, 0, 0, 0} /* PD */ +}; + +void dlm_print_lkb(struct dlm_lkb *lkb) +{ + printk(KERN_ERR "lkb: nodeid %d id %x remid %x exflags %x flags %x " + "sts %d rq %d gr %d wait_type %d wait_nodeid %d seq %llu\n", + lkb->lkb_nodeid, lkb->lkb_id, lkb->lkb_remid, lkb->lkb_exflags, + dlm_iflags_val(lkb), lkb->lkb_status, lkb->lkb_rqmode, + lkb->lkb_grmode, lkb->lkb_wait_type, lkb->lkb_wait_nodeid, + (unsigned long long)lkb->lkb_recover_seq); +} + +static void dlm_print_rsb(struct dlm_rsb *r) +{ + printk(KERN_ERR "rsb: nodeid %d master %d dir %d flags %lx first %x " + "rlc %d name %s\n", + r->res_nodeid, r->res_master_nodeid, r->res_dir_nodeid, + r->res_flags, r->res_first_lkid, r->res_recover_locks_count, + r->res_name); +} + +void dlm_dump_rsb(struct dlm_rsb *r) +{ + struct dlm_lkb *lkb; + + dlm_print_rsb(r); + + printk(KERN_ERR "rsb: root_list empty %d recover_list empty %d\n", + list_empty(&r->res_root_list), list_empty(&r->res_recover_list)); + printk(KERN_ERR "rsb lookup list\n"); + list_for_each_entry(lkb, &r->res_lookup, lkb_rsb_lookup) + dlm_print_lkb(lkb); + printk(KERN_ERR "rsb grant queue:\n"); + list_for_each_entry(lkb, &r->res_grantqueue, lkb_statequeue) + dlm_print_lkb(lkb); + printk(KERN_ERR "rsb convert queue:\n"); + list_for_each_entry(lkb, &r->res_convertqueue, lkb_statequeue) + dlm_print_lkb(lkb); + printk(KERN_ERR "rsb wait queue:\n"); + list_for_each_entry(lkb, &r->res_waitqueue, lkb_statequeue) + dlm_print_lkb(lkb); +} + +/* Threads cannot use the lockspace while it's being recovered */ + +void dlm_lock_recovery(struct dlm_ls *ls) +{ + down_read(&ls->ls_in_recovery); +} + +void dlm_unlock_recovery(struct dlm_ls *ls) +{ + up_read(&ls->ls_in_recovery); +} + +int dlm_lock_recovery_try(struct dlm_ls *ls) +{ + return down_read_trylock(&ls->ls_in_recovery); +} + +static inline int can_be_queued(struct dlm_lkb *lkb) +{ + return !(lkb->lkb_exflags & DLM_LKF_NOQUEUE); +} + +static inline int force_blocking_asts(struct dlm_lkb *lkb) +{ + return (lkb->lkb_exflags & DLM_LKF_NOQUEUEBAST); +} + +static inline int is_demoted(struct dlm_lkb *lkb) +{ + return test_bit(DLM_SBF_DEMOTED_BIT, &lkb->lkb_sbflags); +} + +static inline int is_altmode(struct dlm_lkb *lkb) +{ + return test_bit(DLM_SBF_ALTMODE_BIT, &lkb->lkb_sbflags); +} + +static inline int is_granted(struct dlm_lkb *lkb) +{ + return (lkb->lkb_status == DLM_LKSTS_GRANTED); +} + +static inline int is_remote(struct dlm_rsb *r) +{ + DLM_ASSERT(r->res_nodeid >= 0, dlm_print_rsb(r);); + return !!r->res_nodeid; +} + +static inline int is_process_copy(struct dlm_lkb *lkb) +{ + return lkb->lkb_nodeid && + !test_bit(DLM_IFL_MSTCPY_BIT, &lkb->lkb_iflags); +} + +static inline int is_master_copy(struct dlm_lkb *lkb) +{ + return test_bit(DLM_IFL_MSTCPY_BIT, &lkb->lkb_iflags); +} + +static inline int middle_conversion(struct dlm_lkb *lkb) +{ + if ((lkb->lkb_grmode==DLM_LOCK_PR && lkb->lkb_rqmode==DLM_LOCK_CW) || + (lkb->lkb_rqmode==DLM_LOCK_PR && lkb->lkb_grmode==DLM_LOCK_CW)) + return 1; + return 0; +} + +static inline int down_conversion(struct dlm_lkb *lkb) +{ + return (!middle_conversion(lkb) && lkb->lkb_rqmode < lkb->lkb_grmode); +} + +static inline int is_overlap_unlock(struct dlm_lkb *lkb) +{ + return test_bit(DLM_IFL_OVERLAP_UNLOCK_BIT, &lkb->lkb_iflags); +} + +static inline int is_overlap_cancel(struct dlm_lkb *lkb) +{ + return test_bit(DLM_IFL_OVERLAP_CANCEL_BIT, &lkb->lkb_iflags); +} + +static inline int is_overlap(struct dlm_lkb *lkb) +{ + return test_bit(DLM_IFL_OVERLAP_UNLOCK_BIT, &lkb->lkb_iflags) || + test_bit(DLM_IFL_OVERLAP_CANCEL_BIT, &lkb->lkb_iflags); +} + +static void queue_cast(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv) +{ + if (is_master_copy(lkb)) + return; + + DLM_ASSERT(lkb->lkb_lksb, dlm_print_lkb(lkb);); + + if (rv == -DLM_ECANCEL && + test_and_clear_bit(DLM_IFL_DEADLOCK_CANCEL_BIT, &lkb->lkb_iflags)) + rv = -EDEADLK; + + dlm_add_cb(lkb, DLM_CB_CAST, lkb->lkb_grmode, rv, dlm_sbflags_val(lkb)); +} + +static inline void queue_cast_overlap(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + queue_cast(r, lkb, + is_overlap_unlock(lkb) ? -DLM_EUNLOCK : -DLM_ECANCEL); +} + +static void queue_bast(struct dlm_rsb *r, struct dlm_lkb *lkb, int rqmode) +{ + if (is_master_copy(lkb)) { + send_bast(r, lkb, rqmode); + } else { + dlm_add_cb(lkb, DLM_CB_BAST, rqmode, 0, 0); + } +} + +/* + * Basic operations on rsb's and lkb's + */ + +static inline unsigned long rsb_toss_jiffies(void) +{ + return jiffies + (READ_ONCE(dlm_config.ci_toss_secs) * HZ); +} + +/* This is only called to add a reference when the code already holds + a valid reference to the rsb, so there's no need for locking. */ + +static inline void hold_rsb(struct dlm_rsb *r) +{ + /* inactive rsbs are not ref counted */ + WARN_ON(rsb_flag(r, RSB_INACTIVE)); + kref_get(&r->res_ref); +} + +void dlm_hold_rsb(struct dlm_rsb *r) +{ + hold_rsb(r); +} + +/* TODO move this to lib/refcount.c */ +static __must_check bool +dlm_refcount_dec_and_write_lock_bh(refcount_t *r, rwlock_t *lock) + __cond_acquires(true, lock) +{ + if (refcount_dec_not_one(r)) + return false; + + write_lock_bh(lock); + if (!refcount_dec_and_test(r)) { + write_unlock_bh(lock); + return false; + } + + return true; +} + +/* TODO move this to include/linux/kref.h */ +static inline int dlm_kref_put_write_lock_bh(struct kref *kref, + void (*release)(struct kref *kref), + rwlock_t *lock) +{ + if (dlm_refcount_dec_and_write_lock_bh(&kref->refcount, lock)) { + release(kref); + return 1; + } + + return 0; +} + +static void put_rsb(struct dlm_rsb *r) +{ + struct dlm_ls *ls = r->res_ls; + int rv; + + rv = dlm_kref_put_write_lock_bh(&r->res_ref, deactivate_rsb, + &ls->ls_rsbtbl_lock); + if (rv) + write_unlock_bh(&ls->ls_rsbtbl_lock); +} + +void dlm_put_rsb(struct dlm_rsb *r) +{ + put_rsb(r); +} + +/* connected with timer_delete_sync() in dlm_ls_stop() to stop + * new timers when recovery is triggered and don't run them + * again until a resume_scan_timer() tries it again. + */ +static void enable_scan_timer(struct dlm_ls *ls, unsigned long jiffies) +{ + if (!dlm_locking_stopped(ls)) + mod_timer(&ls->ls_scan_timer, jiffies); +} + +/* This function tries to resume the timer callback if a rsb + * is on the scan list and no timer is pending. It might that + * the first entry is on currently executed as timer callback + * but we don't care if a timer queued up again and does + * nothing. Should be a rare case. + */ +void resume_scan_timer(struct dlm_ls *ls) +{ + struct dlm_rsb *r; + + spin_lock_bh(&ls->ls_scan_lock); + r = list_first_entry_or_null(&ls->ls_scan_list, struct dlm_rsb, + res_scan_list); + if (r && !timer_pending(&ls->ls_scan_timer)) + enable_scan_timer(ls, r->res_toss_time); + spin_unlock_bh(&ls->ls_scan_lock); +} + +/* ls_rsbtbl_lock must be held */ + +static void del_scan(struct dlm_ls *ls, struct dlm_rsb *r) +{ + struct dlm_rsb *first; + + /* active rsbs should never be on the scan list */ + WARN_ON(!rsb_flag(r, RSB_INACTIVE)); + + spin_lock_bh(&ls->ls_scan_lock); + r->res_toss_time = 0; + + /* if the rsb is not queued do nothing */ + if (list_empty(&r->res_scan_list)) + goto out; + + /* get the first element before delete */ + first = list_first_entry(&ls->ls_scan_list, struct dlm_rsb, + res_scan_list); + list_del_init(&r->res_scan_list); + /* check if the first element was the rsb we deleted */ + if (first == r) { + /* try to get the new first element, if the list + * is empty now try to delete the timer, if we are + * too late we don't care. + * + * if the list isn't empty and a new first element got + * in place, set the new timer expire time. + */ + first = list_first_entry_or_null(&ls->ls_scan_list, struct dlm_rsb, + res_scan_list); + if (!first) + timer_delete(&ls->ls_scan_timer); + else + enable_scan_timer(ls, first->res_toss_time); + } + +out: + spin_unlock_bh(&ls->ls_scan_lock); +} + +static void add_scan(struct dlm_ls *ls, struct dlm_rsb *r) +{ + int our_nodeid = dlm_our_nodeid(); + struct dlm_rsb *first; + + /* A dir record for a remote master rsb should never be on the scan list. */ + WARN_ON(!dlm_no_directory(ls) && + (r->res_master_nodeid != our_nodeid) && + (dlm_dir_nodeid(r) == our_nodeid)); + + /* An active rsb should never be on the scan list. */ + WARN_ON(!rsb_flag(r, RSB_INACTIVE)); + + /* An rsb should not already be on the scan list. */ + WARN_ON(!list_empty(&r->res_scan_list)); + + spin_lock_bh(&ls->ls_scan_lock); + /* set the new rsb absolute expire time in the rsb */ + r->res_toss_time = rsb_toss_jiffies(); + if (list_empty(&ls->ls_scan_list)) { + /* if the queue is empty add the element and it's + * our new expire time + */ + list_add_tail(&r->res_scan_list, &ls->ls_scan_list); + enable_scan_timer(ls, r->res_toss_time); + } else { + /* try to get the maybe new first element and then add + * to this rsb with the oldest expire time to the end + * of the queue. If the list was empty before this + * rsb expire time is our next expiration if it wasn't + * the now new first elemet is our new expiration time + */ + first = list_first_entry_or_null(&ls->ls_scan_list, struct dlm_rsb, + res_scan_list); + list_add_tail(&r->res_scan_list, &ls->ls_scan_list); + if (!first) + enable_scan_timer(ls, r->res_toss_time); + else + enable_scan_timer(ls, first->res_toss_time); + } + spin_unlock_bh(&ls->ls_scan_lock); +} + +/* if we hit contention we do in 250 ms a retry to trylock. + * if there is any other mod_timer in between we don't care + * about that it expires earlier again this is only for the + * unlikely case nothing happened in this time. + */ +#define DLM_TOSS_TIMER_RETRY (jiffies + msecs_to_jiffies(250)) + +/* Called by lockspace scan_timer to free unused rsb's. */ + +void dlm_rsb_scan(struct timer_list *timer) +{ + struct dlm_ls *ls = timer_container_of(ls, timer, ls_scan_timer); + int our_nodeid = dlm_our_nodeid(); + struct dlm_rsb *r; + int rv; + + while (1) { + /* interrupting point to leave iteration when + * recovery waits for timer_delete_sync(), recovery + * will take care to delete everything in scan list. + */ + if (dlm_locking_stopped(ls)) + break; + + rv = spin_trylock(&ls->ls_scan_lock); + if (!rv) { + /* rearm again try timer */ + enable_scan_timer(ls, DLM_TOSS_TIMER_RETRY); + break; + } + + r = list_first_entry_or_null(&ls->ls_scan_list, struct dlm_rsb, + res_scan_list); + if (!r) { + /* the next add_scan will enable the timer again */ + spin_unlock(&ls->ls_scan_lock); + break; + } + + /* + * If the first rsb is not yet expired, then stop because the + * list is sorted with nearest expiration first. + */ + if (time_before(jiffies, r->res_toss_time)) { + /* rearm with the next rsb to expire in the future */ + enable_scan_timer(ls, r->res_toss_time); + spin_unlock(&ls->ls_scan_lock); + break; + } + + /* in find_rsb_dir/nodir there is a reverse order of this + * lock, however this is only a trylock if we hit some + * possible contention we try it again. + */ + rv = write_trylock(&ls->ls_rsbtbl_lock); + if (!rv) { + spin_unlock(&ls->ls_scan_lock); + /* rearm again try timer */ + enable_scan_timer(ls, DLM_TOSS_TIMER_RETRY); + break; + } + + list_del(&r->res_slow_list); + rhashtable_remove_fast(&ls->ls_rsbtbl, &r->res_node, + dlm_rhash_rsb_params); + rsb_clear_flag(r, RSB_HASHED); + + /* ls_rsbtbl_lock is not needed when calling send_remove() */ + write_unlock(&ls->ls_rsbtbl_lock); + + list_del_init(&r->res_scan_list); + spin_unlock(&ls->ls_scan_lock); + + /* An rsb that is a dir record for a remote master rsb + * cannot be removed, and should not have a timer enabled. + */ + WARN_ON(!dlm_no_directory(ls) && + (r->res_master_nodeid != our_nodeid) && + (dlm_dir_nodeid(r) == our_nodeid)); + + /* We're the master of this rsb but we're not + * the directory record, so we need to tell the + * dir node to remove the dir record + */ + if (!dlm_no_directory(ls) && + (r->res_master_nodeid == our_nodeid) && + (dlm_dir_nodeid(r) != our_nodeid)) + send_remove(r); + + free_inactive_rsb(r); + } +} + +/* If ls->ls_new_rsb is empty, return -EAGAIN, so the caller can + unlock any spinlocks, go back and call pre_rsb_struct again. + Otherwise, take an rsb off the list and return it. */ + +static int get_rsb_struct(struct dlm_ls *ls, const void *name, int len, + struct dlm_rsb **r_ret) +{ + struct dlm_rsb *r; + + r = dlm_allocate_rsb(); + if (!r) + return -ENOMEM; + + r->res_ls = ls; + r->res_length = len; + memcpy(r->res_name, name, len); + spin_lock_init(&r->res_lock); + + INIT_LIST_HEAD(&r->res_lookup); + INIT_LIST_HEAD(&r->res_grantqueue); + INIT_LIST_HEAD(&r->res_convertqueue); + INIT_LIST_HEAD(&r->res_waitqueue); + INIT_LIST_HEAD(&r->res_root_list); + INIT_LIST_HEAD(&r->res_scan_list); + INIT_LIST_HEAD(&r->res_recover_list); + INIT_LIST_HEAD(&r->res_masters_list); + + *r_ret = r; + return 0; +} + +int dlm_search_rsb_tree(struct rhashtable *rhash, const void *name, int len, + struct dlm_rsb **r_ret) +{ + char key[DLM_RESNAME_MAXLEN] = {}; + if (len > DLM_RESNAME_MAXLEN) + return -EINVAL; + memcpy(key, name, len); + *r_ret = rhashtable_lookup_fast(rhash, &key, dlm_rhash_rsb_params); + if (*r_ret) + return 0; + + return -EBADR; +} + +static int rsb_insert(struct dlm_rsb *rsb, struct rhashtable *rhash) +{ + int rv; + + rv = rhashtable_insert_fast(rhash, &rsb->res_node, + dlm_rhash_rsb_params); + if (!rv) + rsb_set_flag(rsb, RSB_HASHED); + + return rv; +} + +/* + * Find rsb in rsbtbl and potentially create/add one + * + * Delaying the release of rsb's has a similar benefit to applications keeping + * NL locks on an rsb, but without the guarantee that the cached master value + * will still be valid when the rsb is reused. Apps aren't always smart enough + * to keep NL locks on an rsb that they may lock again shortly; this can lead + * to excessive master lookups and removals if we don't delay the release. + * + * Searching for an rsb means looking through both the normal list and toss + * list. When found on the toss list the rsb is moved to the normal list with + * ref count of 1; when found on normal list the ref count is incremented. + * + * rsb's on the keep list are being used locally and refcounted. + * rsb's on the toss list are not being used locally, and are not refcounted. + * + * The toss list rsb's were either + * - previously used locally but not any more (were on keep list, then + * moved to toss list when last refcount dropped) + * - created and put on toss list as a directory record for a lookup + * (we are the dir node for the res, but are not using the res right now, + * but some other node is) + * + * The purpose of find_rsb() is to return a refcounted rsb for local use. + * So, if the given rsb is on the toss list, it is moved to the keep list + * before being returned. + * + * deactivate_rsb() happens when all local usage of the rsb is done, i.e. no + * more refcounts exist, so the rsb is moved from the keep list to the + * toss list. + * + * rsb's on both keep and toss lists are used for doing a name to master + * lookups. rsb's that are in use locally (and being refcounted) are on + * the keep list, rsb's that are not in use locally (not refcounted) and + * only exist for name/master lookups are on the toss list. + * + * rsb's on the toss list who's dir_nodeid is not local can have stale + * name/master mappings. So, remote requests on such rsb's can potentially + * return with an error, which means the mapping is stale and needs to + * be updated with a new lookup. (The idea behind MASTER UNCERTAIN and + * first_lkid is to keep only a single outstanding request on an rsb + * while that rsb has a potentially stale master.) + */ + +static int find_rsb_dir(struct dlm_ls *ls, const void *name, int len, + uint32_t hash, int dir_nodeid, int from_nodeid, + unsigned int flags, struct dlm_rsb **r_ret) +{ + struct dlm_rsb *r = NULL; + int our_nodeid = dlm_our_nodeid(); + int from_local = 0; + int from_other = 0; + int from_dir = 0; + int create = 0; + int error; + + if (flags & R_RECEIVE_REQUEST) { + if (from_nodeid == dir_nodeid) + from_dir = 1; + else + from_other = 1; + } else if (flags & R_REQUEST) { + from_local = 1; + } + + /* + * flags & R_RECEIVE_RECOVER is from dlm_recover_master_copy, so + * from_nodeid has sent us a lock in dlm_recover_locks, believing + * we're the new master. Our local recovery may not have set + * res_master_nodeid to our_nodeid yet, so allow either. Don't + * create the rsb; dlm_recover_process_copy() will handle EBADR + * by resending. + * + * If someone sends us a request, we are the dir node, and we do + * not find the rsb anywhere, then recreate it. This happens if + * someone sends us a request after we have removed/freed an rsb. + * (They sent a request instead of lookup because they are using + * an rsb taken from their scan list.) + */ + + if (from_local || from_dir || + (from_other && (dir_nodeid == our_nodeid))) { + create = 1; + } + + retry: + error = dlm_search_rsb_tree(&ls->ls_rsbtbl, name, len, &r); + if (error) + goto do_new; + + /* check if the rsb is active under read lock - likely path */ + read_lock_bh(&ls->ls_rsbtbl_lock); + if (!rsb_flag(r, RSB_HASHED)) { + read_unlock_bh(&ls->ls_rsbtbl_lock); + error = -EBADR; + goto do_new; + } + + /* + * rsb is active, so we can't check master_nodeid without lock_rsb. + */ + + if (rsb_flag(r, RSB_INACTIVE)) { + read_unlock_bh(&ls->ls_rsbtbl_lock); + goto do_inactive; + } + + kref_get(&r->res_ref); + read_unlock_bh(&ls->ls_rsbtbl_lock); + goto out; + + + do_inactive: + write_lock_bh(&ls->ls_rsbtbl_lock); + + /* + * The expectation here is that the rsb will have HASHED and + * INACTIVE flags set, and that the rsb can be moved from + * inactive back to active again. However, between releasing + * the read lock and acquiring the write lock, this rsb could + * have been removed from rsbtbl, and had HASHED cleared, to + * be freed. To deal with this case, we would normally need + * to repeat dlm_search_rsb_tree while holding the write lock, + * but rcu allows us to simply check the HASHED flag, because + * the rcu read lock means the rsb will not be freed yet. + * If the HASHED flag is not set, then the rsb is being freed, + * so we add a new rsb struct. If the HASHED flag is set, + * and INACTIVE is not set, it means another thread has + * made the rsb active, as we're expecting to do here, and + * we just repeat the lookup (this will be very unlikely.) + */ + if (rsb_flag(r, RSB_HASHED)) { + if (!rsb_flag(r, RSB_INACTIVE)) { + write_unlock_bh(&ls->ls_rsbtbl_lock); + goto retry; + } + } else { + write_unlock_bh(&ls->ls_rsbtbl_lock); + error = -EBADR; + goto do_new; + } + + /* + * rsb found inactive (master_nodeid may be out of date unless + * we are the dir_nodeid or were the master) No other thread + * is using this rsb because it's inactive, so we can + * look at or update res_master_nodeid without lock_rsb. + */ + + if ((r->res_master_nodeid != our_nodeid) && from_other) { + /* our rsb was not master, and another node (not the dir node) + has sent us a request */ + log_debug(ls, "find_rsb inactive from_other %d master %d dir %d %s", + from_nodeid, r->res_master_nodeid, dir_nodeid, + r->res_name); + write_unlock_bh(&ls->ls_rsbtbl_lock); + error = -ENOTBLK; + goto out; + } + + if ((r->res_master_nodeid != our_nodeid) && from_dir) { + /* don't think this should ever happen */ + log_error(ls, "find_rsb inactive from_dir %d master %d", + from_nodeid, r->res_master_nodeid); + dlm_print_rsb(r); + /* fix it and go on */ + r->res_master_nodeid = our_nodeid; + r->res_nodeid = 0; + rsb_clear_flag(r, RSB_MASTER_UNCERTAIN); + r->res_first_lkid = 0; + } + + if (from_local && (r->res_master_nodeid != our_nodeid)) { + /* Because we have held no locks on this rsb, + res_master_nodeid could have become stale. */ + rsb_set_flag(r, RSB_MASTER_UNCERTAIN); + r->res_first_lkid = 0; + } + + /* we always deactivate scan timer for the rsb, when + * we move it out of the inactive state as rsb state + * can be changed and scan timers are only for inactive + * rsbs. + */ + del_scan(ls, r); + list_move(&r->res_slow_list, &ls->ls_slow_active); + rsb_clear_flag(r, RSB_INACTIVE); + kref_init(&r->res_ref); /* ref is now used in active state */ + write_unlock_bh(&ls->ls_rsbtbl_lock); + + goto out; + + + do_new: + /* + * rsb not found + */ + + if (error == -EBADR && !create) + goto out; + + error = get_rsb_struct(ls, name, len, &r); + if (WARN_ON_ONCE(error)) + goto out; + + r->res_hash = hash; + r->res_dir_nodeid = dir_nodeid; + kref_init(&r->res_ref); + + if (from_dir) { + /* want to see how often this happens */ + log_debug(ls, "find_rsb new from_dir %d recreate %s", + from_nodeid, r->res_name); + r->res_master_nodeid = our_nodeid; + r->res_nodeid = 0; + goto out_add; + } + + if (from_other && (dir_nodeid != our_nodeid)) { + /* should never happen */ + log_error(ls, "find_rsb new from_other %d dir %d our %d %s", + from_nodeid, dir_nodeid, our_nodeid, r->res_name); + dlm_free_rsb(r); + r = NULL; + error = -ENOTBLK; + goto out; + } + + if (from_other) { + log_debug(ls, "find_rsb new from_other %d dir %d %s", + from_nodeid, dir_nodeid, r->res_name); + } + + if (dir_nodeid == our_nodeid) { + /* When we are the dir nodeid, we can set the master + node immediately */ + r->res_master_nodeid = our_nodeid; + r->res_nodeid = 0; + } else { + /* set_master will send_lookup to dir_nodeid */ + r->res_master_nodeid = 0; + r->res_nodeid = -1; + } + + out_add: + + write_lock_bh(&ls->ls_rsbtbl_lock); + error = rsb_insert(r, &ls->ls_rsbtbl); + if (error == -EEXIST) { + /* somebody else was faster and it seems the + * rsb exists now, we do a whole relookup + */ + write_unlock_bh(&ls->ls_rsbtbl_lock); + dlm_free_rsb(r); + goto retry; + } else if (!error) { + list_add(&r->res_slow_list, &ls->ls_slow_active); + } + write_unlock_bh(&ls->ls_rsbtbl_lock); + out: + *r_ret = r; + return error; +} + +/* During recovery, other nodes can send us new MSTCPY locks (from + dlm_recover_locks) before we've made ourself master (in + dlm_recover_masters). */ + +static int find_rsb_nodir(struct dlm_ls *ls, const void *name, int len, + uint32_t hash, int dir_nodeid, int from_nodeid, + unsigned int flags, struct dlm_rsb **r_ret) +{ + struct dlm_rsb *r = NULL; + int our_nodeid = dlm_our_nodeid(); + int recover = (flags & R_RECEIVE_RECOVER); + int error; + + retry: + error = dlm_search_rsb_tree(&ls->ls_rsbtbl, name, len, &r); + if (error) + goto do_new; + + /* check if the rsb is in active state under read lock - likely path */ + read_lock_bh(&ls->ls_rsbtbl_lock); + if (!rsb_flag(r, RSB_HASHED)) { + read_unlock_bh(&ls->ls_rsbtbl_lock); + goto do_new; + } + + if (rsb_flag(r, RSB_INACTIVE)) { + read_unlock_bh(&ls->ls_rsbtbl_lock); + goto do_inactive; + } + + /* + * rsb is active, so we can't check master_nodeid without lock_rsb. + */ + + kref_get(&r->res_ref); + read_unlock_bh(&ls->ls_rsbtbl_lock); + + goto out; + + + do_inactive: + write_lock_bh(&ls->ls_rsbtbl_lock); + + /* See comment in find_rsb_dir. */ + if (rsb_flag(r, RSB_HASHED)) { + if (!rsb_flag(r, RSB_INACTIVE)) { + write_unlock_bh(&ls->ls_rsbtbl_lock); + goto retry; + } + } else { + write_unlock_bh(&ls->ls_rsbtbl_lock); + goto do_new; + } + + + /* + * rsb found inactive. No other thread is using this rsb because + * it's inactive, so we can look at or update res_master_nodeid + * without lock_rsb. + */ + + if (!recover && (r->res_master_nodeid != our_nodeid) && from_nodeid) { + /* our rsb is not master, and another node has sent us a + request; this should never happen */ + log_error(ls, "find_rsb inactive from_nodeid %d master %d dir %d", + from_nodeid, r->res_master_nodeid, dir_nodeid); + dlm_print_rsb(r); + write_unlock_bh(&ls->ls_rsbtbl_lock); + error = -ENOTBLK; + goto out; + } + + if (!recover && (r->res_master_nodeid != our_nodeid) && + (dir_nodeid == our_nodeid)) { + /* our rsb is not master, and we are dir; may as well fix it; + this should never happen */ + log_error(ls, "find_rsb inactive our %d master %d dir %d", + our_nodeid, r->res_master_nodeid, dir_nodeid); + dlm_print_rsb(r); + r->res_master_nodeid = our_nodeid; + r->res_nodeid = 0; + } + + del_scan(ls, r); + list_move(&r->res_slow_list, &ls->ls_slow_active); + rsb_clear_flag(r, RSB_INACTIVE); + kref_init(&r->res_ref); + write_unlock_bh(&ls->ls_rsbtbl_lock); + + goto out; + + + do_new: + /* + * rsb not found + */ + + error = get_rsb_struct(ls, name, len, &r); + if (WARN_ON_ONCE(error)) + goto out; + + r->res_hash = hash; + r->res_dir_nodeid = dir_nodeid; + r->res_master_nodeid = dir_nodeid; + r->res_nodeid = (dir_nodeid == our_nodeid) ? 0 : dir_nodeid; + kref_init(&r->res_ref); + + write_lock_bh(&ls->ls_rsbtbl_lock); + error = rsb_insert(r, &ls->ls_rsbtbl); + if (error == -EEXIST) { + /* somebody else was faster and it seems the + * rsb exists now, we do a whole relookup + */ + write_unlock_bh(&ls->ls_rsbtbl_lock); + dlm_free_rsb(r); + goto retry; + } else if (!error) { + list_add(&r->res_slow_list, &ls->ls_slow_active); + } + write_unlock_bh(&ls->ls_rsbtbl_lock); + + out: + *r_ret = r; + return error; +} + +/* + * rsb rcu usage + * + * While rcu read lock is held, the rsb cannot be freed, + * which allows a lookup optimization. + * + * Two threads are accessing the same rsb concurrently, + * the first (A) is trying to use the rsb, the second (B) + * is trying to free the rsb. + * + * thread A thread B + * (trying to use rsb) (trying to free rsb) + * + * A1. rcu read lock + * A2. rsbtbl read lock + * A3. look up rsb in rsbtbl + * A4. rsbtbl read unlock + * B1. rsbtbl write lock + * B2. look up rsb in rsbtbl + * B3. remove rsb from rsbtbl + * B4. clear rsb HASHED flag + * B5. rsbtbl write unlock + * B6. begin freeing rsb using rcu... + * + * (rsb is inactive, so try to make it active again) + * A5. read rsb HASHED flag (safe because rsb is not freed yet) + * A6. the rsb HASHED flag is not set, which it means the rsb + * is being removed from rsbtbl and freed, so don't use it. + * A7. rcu read unlock + * + * B7. ...finish freeing rsb using rcu + * A8. create a new rsb + * + * Without the rcu optimization, steps A5-8 would need to do + * an extra rsbtbl lookup: + * A5. rsbtbl write lock + * A6. look up rsb in rsbtbl, not found + * A7. rsbtbl write unlock + * A8. create a new rsb + */ + +static int find_rsb(struct dlm_ls *ls, const void *name, int len, + int from_nodeid, unsigned int flags, + struct dlm_rsb **r_ret) +{ + int dir_nodeid; + uint32_t hash; + int rv; + + if (len > DLM_RESNAME_MAXLEN) + return -EINVAL; + + hash = jhash(name, len, 0); + dir_nodeid = dlm_hash2nodeid(ls, hash); + + rcu_read_lock(); + if (dlm_no_directory(ls)) + rv = find_rsb_nodir(ls, name, len, hash, dir_nodeid, + from_nodeid, flags, r_ret); + else + rv = find_rsb_dir(ls, name, len, hash, dir_nodeid, + from_nodeid, flags, r_ret); + rcu_read_unlock(); + return rv; +} + +/* we have received a request and found that res_master_nodeid != our_nodeid, + so we need to return an error or make ourself the master */ + +static int validate_master_nodeid(struct dlm_ls *ls, struct dlm_rsb *r, + int from_nodeid) +{ + if (dlm_no_directory(ls)) { + log_error(ls, "find_rsb keep from_nodeid %d master %d dir %d", + from_nodeid, r->res_master_nodeid, + r->res_dir_nodeid); + dlm_print_rsb(r); + return -ENOTBLK; + } + + if (from_nodeid != r->res_dir_nodeid) { + /* our rsb is not master, and another node (not the dir node) + has sent us a request. this is much more common when our + master_nodeid is zero, so limit debug to non-zero. */ + + if (r->res_master_nodeid) { + log_debug(ls, "validate master from_other %d master %d " + "dir %d first %x %s", from_nodeid, + r->res_master_nodeid, r->res_dir_nodeid, + r->res_first_lkid, r->res_name); + } + return -ENOTBLK; + } else { + /* our rsb is not master, but the dir nodeid has sent us a + request; this could happen with master 0 / res_nodeid -1 */ + + if (r->res_master_nodeid) { + log_error(ls, "validate master from_dir %d master %d " + "first %x %s", + from_nodeid, r->res_master_nodeid, + r->res_first_lkid, r->res_name); + } + + r->res_master_nodeid = dlm_our_nodeid(); + r->res_nodeid = 0; + return 0; + } +} + +static void __dlm_master_lookup(struct dlm_ls *ls, struct dlm_rsb *r, int our_nodeid, + int from_nodeid, bool is_inactive, unsigned int flags, + int *r_nodeid, int *result) +{ + int fix_master = (flags & DLM_LU_RECOVER_MASTER); + int from_master = (flags & DLM_LU_RECOVER_DIR); + + if (r->res_dir_nodeid != our_nodeid) { + /* should not happen, but may as well fix it and carry on */ + log_error(ls, "%s res_dir %d our %d %s", __func__, + r->res_dir_nodeid, our_nodeid, r->res_name); + r->res_dir_nodeid = our_nodeid; + } + + if (fix_master && r->res_master_nodeid && dlm_is_removed(ls, r->res_master_nodeid)) { + /* Recovery uses this function to set a new master when + * the previous master failed. Setting NEW_MASTER will + * force dlm_recover_masters to call recover_master on this + * rsb even though the res_nodeid is no longer removed. + */ + + r->res_master_nodeid = from_nodeid; + r->res_nodeid = from_nodeid; + rsb_set_flag(r, RSB_NEW_MASTER); + + if (is_inactive) { + /* I don't think we should ever find it inactive. */ + log_error(ls, "%s fix_master inactive", __func__); + dlm_dump_rsb(r); + } + } + + if (from_master && (r->res_master_nodeid != from_nodeid)) { + /* this will happen if from_nodeid became master during + * a previous recovery cycle, and we aborted the previous + * cycle before recovering this master value + */ + + log_limit(ls, "%s from_master %d master_nodeid %d res_nodeid %d first %x %s", + __func__, from_nodeid, r->res_master_nodeid, + r->res_nodeid, r->res_first_lkid, r->res_name); + + if (r->res_master_nodeid == our_nodeid) { + log_error(ls, "from_master %d our_master", from_nodeid); + dlm_dump_rsb(r); + goto ret_assign; + } + + r->res_master_nodeid = from_nodeid; + r->res_nodeid = from_nodeid; + rsb_set_flag(r, RSB_NEW_MASTER); + } + + if (!r->res_master_nodeid) { + /* this will happen if recovery happens while we're looking + * up the master for this rsb + */ + + log_debug(ls, "%s master 0 to %d first %x %s", __func__, + from_nodeid, r->res_first_lkid, r->res_name); + r->res_master_nodeid = from_nodeid; + r->res_nodeid = from_nodeid; + } + + if (!from_master && !fix_master && + (r->res_master_nodeid == from_nodeid)) { + /* this can happen when the master sends remove, the dir node + * finds the rsb on the active list and ignores the remove, + * and the former master sends a lookup + */ + + log_limit(ls, "%s from master %d flags %x first %x %s", + __func__, from_nodeid, flags, r->res_first_lkid, + r->res_name); + } + + ret_assign: + *r_nodeid = r->res_master_nodeid; + if (result) + *result = DLM_LU_MATCH; +} + +/* + * We're the dir node for this res and another node wants to know the + * master nodeid. During normal operation (non recovery) this is only + * called from receive_lookup(); master lookups when the local node is + * the dir node are done by find_rsb(). + * + * normal operation, we are the dir node for a resource + * . _request_lock + * . set_master + * . send_lookup + * . receive_lookup + * . dlm_master_lookup flags 0 + * + * recover directory, we are rebuilding dir for all resources + * . dlm_recover_directory + * . dlm_rcom_names + * remote node sends back the rsb names it is master of and we are dir of + * . dlm_master_lookup RECOVER_DIR (fix_master 0, from_master 1) + * we either create new rsb setting remote node as master, or find existing + * rsb and set master to be the remote node. + * + * recover masters, we are finding the new master for resources + * . dlm_recover_masters + * . recover_master + * . dlm_send_rcom_lookup + * . receive_rcom_lookup + * . dlm_master_lookup RECOVER_MASTER (fix_master 1, from_master 0) + */ + +static int _dlm_master_lookup(struct dlm_ls *ls, int from_nodeid, const char *name, + int len, unsigned int flags, int *r_nodeid, int *result) +{ + struct dlm_rsb *r = NULL; + uint32_t hash; + int our_nodeid = dlm_our_nodeid(); + int dir_nodeid, error; + + if (len > DLM_RESNAME_MAXLEN) + return -EINVAL; + + if (from_nodeid == our_nodeid) { + log_error(ls, "dlm_master_lookup from our_nodeid %d flags %x", + our_nodeid, flags); + return -EINVAL; + } + + hash = jhash(name, len, 0); + dir_nodeid = dlm_hash2nodeid(ls, hash); + if (dir_nodeid != our_nodeid) { + log_error(ls, "dlm_master_lookup from %d dir %d our %d h %x %d", + from_nodeid, dir_nodeid, our_nodeid, hash, + ls->ls_num_nodes); + *r_nodeid = -1; + return -EINVAL; + } + + retry: + error = dlm_search_rsb_tree(&ls->ls_rsbtbl, name, len, &r); + if (error) + goto not_found; + + /* check if the rsb is active under read lock - likely path */ + read_lock_bh(&ls->ls_rsbtbl_lock); + if (!rsb_flag(r, RSB_HASHED)) { + read_unlock_bh(&ls->ls_rsbtbl_lock); + goto not_found; + } + + if (rsb_flag(r, RSB_INACTIVE)) { + read_unlock_bh(&ls->ls_rsbtbl_lock); + goto do_inactive; + } + + /* because the rsb is active, we need to lock_rsb before + * checking/changing re_master_nodeid + */ + + hold_rsb(r); + read_unlock_bh(&ls->ls_rsbtbl_lock); + lock_rsb(r); + + __dlm_master_lookup(ls, r, our_nodeid, from_nodeid, false, + flags, r_nodeid, result); + + /* the rsb was active */ + unlock_rsb(r); + put_rsb(r); + + return 0; + + do_inactive: + /* unlikely path - check if still part of ls_rsbtbl */ + write_lock_bh(&ls->ls_rsbtbl_lock); + + /* see comment in find_rsb_dir */ + if (rsb_flag(r, RSB_HASHED)) { + if (!rsb_flag(r, RSB_INACTIVE)) { + write_unlock_bh(&ls->ls_rsbtbl_lock); + /* something as changed, very unlikely but + * try again + */ + goto retry; + } + } else { + write_unlock_bh(&ls->ls_rsbtbl_lock); + goto not_found; + } + + /* because the rsb is inactive, it's not refcounted and lock_rsb + is not used, but is protected by the rsbtbl lock */ + + __dlm_master_lookup(ls, r, our_nodeid, from_nodeid, true, flags, + r_nodeid, result); + + /* A dir record rsb should never be on scan list. + * Except when we are the dir and master node. + * This function should only be called by the dir + * node. + */ + WARN_ON(!list_empty(&r->res_scan_list) && + r->res_master_nodeid != our_nodeid); + + write_unlock_bh(&ls->ls_rsbtbl_lock); + + return 0; + + not_found: + error = get_rsb_struct(ls, name, len, &r); + if (WARN_ON_ONCE(error)) + goto out; + + r->res_hash = hash; + r->res_dir_nodeid = our_nodeid; + r->res_master_nodeid = from_nodeid; + r->res_nodeid = from_nodeid; + rsb_set_flag(r, RSB_INACTIVE); + + write_lock_bh(&ls->ls_rsbtbl_lock); + error = rsb_insert(r, &ls->ls_rsbtbl); + if (error == -EEXIST) { + /* somebody else was faster and it seems the + * rsb exists now, we do a whole relookup + */ + write_unlock_bh(&ls->ls_rsbtbl_lock); + dlm_free_rsb(r); + goto retry; + } else if (error) { + write_unlock_bh(&ls->ls_rsbtbl_lock); + /* should never happen */ + dlm_free_rsb(r); + goto retry; + } + + list_add(&r->res_slow_list, &ls->ls_slow_inactive); + write_unlock_bh(&ls->ls_rsbtbl_lock); + + if (result) + *result = DLM_LU_ADD; + *r_nodeid = from_nodeid; + out: + return error; +} + +int dlm_master_lookup(struct dlm_ls *ls, int from_nodeid, const char *name, + int len, unsigned int flags, int *r_nodeid, int *result) +{ + int rv; + rcu_read_lock(); + rv = _dlm_master_lookup(ls, from_nodeid, name, len, flags, r_nodeid, result); + rcu_read_unlock(); + return rv; +} + +static void dlm_dump_rsb_hash(struct dlm_ls *ls, uint32_t hash) +{ + struct dlm_rsb *r; + + read_lock_bh(&ls->ls_rsbtbl_lock); + list_for_each_entry(r, &ls->ls_slow_active, res_slow_list) { + if (r->res_hash == hash) + dlm_dump_rsb(r); + } + read_unlock_bh(&ls->ls_rsbtbl_lock); +} + +void dlm_dump_rsb_name(struct dlm_ls *ls, const char *name, int len) +{ + struct dlm_rsb *r = NULL; + int error; + + rcu_read_lock(); + error = dlm_search_rsb_tree(&ls->ls_rsbtbl, name, len, &r); + if (!error) + goto out; + + dlm_dump_rsb(r); + out: + rcu_read_unlock(); +} + +static void deactivate_rsb(struct kref *kref) +{ + struct dlm_rsb *r = container_of(kref, struct dlm_rsb, res_ref); + struct dlm_ls *ls = r->res_ls; + int our_nodeid = dlm_our_nodeid(); + + DLM_ASSERT(list_empty(&r->res_root_list), dlm_print_rsb(r);); + rsb_set_flag(r, RSB_INACTIVE); + list_move(&r->res_slow_list, &ls->ls_slow_inactive); + + /* + * When the rsb becomes unused, there are two possibilities: + * 1. Leave the inactive rsb in place (don't remove it). + * 2. Add it to the scan list to be removed. + * + * 1 is done when the rsb is acting as the dir record + * for a remotely mastered rsb. The rsb must be left + * in place as an inactive rsb to act as the dir record. + * + * 2 is done when a) the rsb is not the master and not the + * dir record, b) when the rsb is both the master and the + * dir record, c) when the rsb is master but not dir record. + * + * (If no directory is used, the rsb can always be removed.) + */ + if (dlm_no_directory(ls) || + (r->res_master_nodeid == our_nodeid || + dlm_dir_nodeid(r) != our_nodeid)) + add_scan(ls, r); + + if (r->res_lvbptr) { + dlm_free_lvb(r->res_lvbptr); + r->res_lvbptr = NULL; + } +} + +void free_inactive_rsb(struct dlm_rsb *r) +{ + WARN_ON_ONCE(!rsb_flag(r, RSB_INACTIVE)); + + DLM_ASSERT(list_empty(&r->res_lookup), dlm_dump_rsb(r);); + DLM_ASSERT(list_empty(&r->res_grantqueue), dlm_dump_rsb(r);); + DLM_ASSERT(list_empty(&r->res_convertqueue), dlm_dump_rsb(r);); + DLM_ASSERT(list_empty(&r->res_waitqueue), dlm_dump_rsb(r);); + DLM_ASSERT(list_empty(&r->res_root_list), dlm_dump_rsb(r);); + DLM_ASSERT(list_empty(&r->res_scan_list), dlm_dump_rsb(r);); + DLM_ASSERT(list_empty(&r->res_recover_list), dlm_dump_rsb(r);); + DLM_ASSERT(list_empty(&r->res_masters_list), dlm_dump_rsb(r);); + + dlm_free_rsb(r); +} + +/* Attaching/detaching lkb's from rsb's is for rsb reference counting. + The rsb must exist as long as any lkb's for it do. */ + +static void attach_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + hold_rsb(r); + lkb->lkb_resource = r; +} + +static void detach_lkb(struct dlm_lkb *lkb) +{ + if (lkb->lkb_resource) { + put_rsb(lkb->lkb_resource); + lkb->lkb_resource = NULL; + } +} + +static int _create_lkb(struct dlm_ls *ls, struct dlm_lkb **lkb_ret, + unsigned long start, unsigned long end) +{ + struct xa_limit limit; + struct dlm_lkb *lkb; + int rv; + + limit.max = end; + limit.min = start; + + lkb = dlm_allocate_lkb(); + if (!lkb) + return -ENOMEM; + + lkb->lkb_last_bast_cb_mode = DLM_LOCK_IV; + lkb->lkb_last_cast_cb_mode = DLM_LOCK_IV; + lkb->lkb_last_cb_mode = DLM_LOCK_IV; + lkb->lkb_nodeid = -1; + lkb->lkb_grmode = DLM_LOCK_IV; + kref_init(&lkb->lkb_ref); + INIT_LIST_HEAD(&lkb->lkb_ownqueue); + INIT_LIST_HEAD(&lkb->lkb_rsb_lookup); + + write_lock_bh(&ls->ls_lkbxa_lock); + rv = xa_alloc(&ls->ls_lkbxa, &lkb->lkb_id, lkb, limit, GFP_ATOMIC); + write_unlock_bh(&ls->ls_lkbxa_lock); + + if (rv < 0) { + log_error(ls, "create_lkb xa error %d", rv); + dlm_free_lkb(lkb); + return rv; + } + + *lkb_ret = lkb; + return 0; +} + +static int create_lkb(struct dlm_ls *ls, struct dlm_lkb **lkb_ret) +{ + return _create_lkb(ls, lkb_ret, 1, ULONG_MAX); +} + +static int find_lkb(struct dlm_ls *ls, uint32_t lkid, struct dlm_lkb **lkb_ret) +{ + struct dlm_lkb *lkb; + + rcu_read_lock(); + lkb = xa_load(&ls->ls_lkbxa, lkid); + if (lkb) { + /* check if lkb is still part of lkbxa under lkbxa_lock as + * the lkb_ref is tight to the lkbxa data structure, see + * __put_lkb(). + */ + read_lock_bh(&ls->ls_lkbxa_lock); + if (kref_read(&lkb->lkb_ref)) + kref_get(&lkb->lkb_ref); + else + lkb = NULL; + read_unlock_bh(&ls->ls_lkbxa_lock); + } + rcu_read_unlock(); + + *lkb_ret = lkb; + return lkb ? 0 : -ENOENT; +} + +static void kill_lkb(struct kref *kref) +{ + struct dlm_lkb *lkb = container_of(kref, struct dlm_lkb, lkb_ref); + + /* All work is done after the return from kref_put() so we + can release the write_lock before the detach_lkb */ + + DLM_ASSERT(!lkb->lkb_status, dlm_print_lkb(lkb);); +} + +/* __put_lkb() is used when an lkb may not have an rsb attached to + it so we need to provide the lockspace explicitly */ + +static int __put_lkb(struct dlm_ls *ls, struct dlm_lkb *lkb) +{ + uint32_t lkid = lkb->lkb_id; + int rv; + + rv = dlm_kref_put_write_lock_bh(&lkb->lkb_ref, kill_lkb, + &ls->ls_lkbxa_lock); + if (rv) { + xa_erase(&ls->ls_lkbxa, lkid); + write_unlock_bh(&ls->ls_lkbxa_lock); + + detach_lkb(lkb); + + /* for local/process lkbs, lvbptr points to caller's lksb */ + if (lkb->lkb_lvbptr && is_master_copy(lkb)) + dlm_free_lvb(lkb->lkb_lvbptr); + dlm_free_lkb(lkb); + } + + return rv; +} + +int dlm_put_lkb(struct dlm_lkb *lkb) +{ + struct dlm_ls *ls; + + DLM_ASSERT(lkb->lkb_resource, dlm_print_lkb(lkb);); + DLM_ASSERT(lkb->lkb_resource->res_ls, dlm_print_lkb(lkb);); + + ls = lkb->lkb_resource->res_ls; + return __put_lkb(ls, lkb); +} + +/* This is only called to add a reference when the code already holds + a valid reference to the lkb, so there's no need for locking. */ + +static inline void hold_lkb(struct dlm_lkb *lkb) +{ + kref_get(&lkb->lkb_ref); +} + +static void unhold_lkb_assert(struct kref *kref) +{ + struct dlm_lkb *lkb = container_of(kref, struct dlm_lkb, lkb_ref); + + DLM_ASSERT(false, dlm_print_lkb(lkb);); +} + +/* This is called when we need to remove a reference and are certain + it's not the last ref. e.g. del_lkb is always called between a + find_lkb/put_lkb and is always the inverse of a previous add_lkb. + put_lkb would work fine, but would involve unnecessary locking */ + +static inline void unhold_lkb(struct dlm_lkb *lkb) +{ + kref_put(&lkb->lkb_ref, unhold_lkb_assert); +} + +static void lkb_add_ordered(struct list_head *new, struct list_head *head, + int mode) +{ + struct dlm_lkb *lkb = NULL, *iter; + + list_for_each_entry(iter, head, lkb_statequeue) + if (iter->lkb_rqmode < mode) { + lkb = iter; + list_add_tail(new, &iter->lkb_statequeue); + break; + } + + if (!lkb) + list_add_tail(new, head); +} + +/* add/remove lkb to rsb's grant/convert/wait queue */ + +static void add_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb, int status) +{ + kref_get(&lkb->lkb_ref); + + DLM_ASSERT(!lkb->lkb_status, dlm_print_lkb(lkb);); + + lkb->lkb_timestamp = ktime_get(); + + lkb->lkb_status = status; + + switch (status) { + case DLM_LKSTS_WAITING: + if (lkb->lkb_exflags & DLM_LKF_HEADQUE) + list_add(&lkb->lkb_statequeue, &r->res_waitqueue); + else + list_add_tail(&lkb->lkb_statequeue, &r->res_waitqueue); + break; + case DLM_LKSTS_GRANTED: + /* convention says granted locks kept in order of grmode */ + lkb_add_ordered(&lkb->lkb_statequeue, &r->res_grantqueue, + lkb->lkb_grmode); + break; + case DLM_LKSTS_CONVERT: + if (lkb->lkb_exflags & DLM_LKF_HEADQUE) + list_add(&lkb->lkb_statequeue, &r->res_convertqueue); + else + list_add_tail(&lkb->lkb_statequeue, + &r->res_convertqueue); + break; + default: + DLM_ASSERT(0, dlm_print_lkb(lkb); printk("sts=%d\n", status);); + } +} + +static void del_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + lkb->lkb_status = 0; + list_del(&lkb->lkb_statequeue); + unhold_lkb(lkb); +} + +static void move_lkb(struct dlm_rsb *r, struct dlm_lkb *lkb, int sts) +{ + del_lkb(r, lkb); + add_lkb(r, lkb, sts); +} + +static int msg_reply_type(int mstype) +{ + switch (mstype) { + case DLM_MSG_REQUEST: + return DLM_MSG_REQUEST_REPLY; + case DLM_MSG_CONVERT: + return DLM_MSG_CONVERT_REPLY; + case DLM_MSG_UNLOCK: + return DLM_MSG_UNLOCK_REPLY; + case DLM_MSG_CANCEL: + return DLM_MSG_CANCEL_REPLY; + case DLM_MSG_LOOKUP: + return DLM_MSG_LOOKUP_REPLY; + } + return -1; +} + +/* add/remove lkb from global waiters list of lkb's waiting for + a reply from a remote node */ + +static void add_to_waiters(struct dlm_lkb *lkb, int mstype, int to_nodeid) +{ + struct dlm_ls *ls = lkb->lkb_resource->res_ls; + + spin_lock_bh(&ls->ls_waiters_lock); + if (lkb->lkb_wait_type || is_overlap_cancel(lkb)) { + switch (mstype) { + case DLM_MSG_UNLOCK: + set_bit(DLM_IFL_OVERLAP_UNLOCK_BIT, &lkb->lkb_iflags); + break; + case DLM_MSG_CANCEL: + set_bit(DLM_IFL_OVERLAP_CANCEL_BIT, &lkb->lkb_iflags); + break; + default: + /* should never happen as validate_lock_args() checks + * on lkb_wait_type and validate_unlock_args() only + * creates UNLOCK or CANCEL messages. + */ + WARN_ON_ONCE(1); + goto out; + } + lkb->lkb_wait_count++; + hold_lkb(lkb); + + log_debug(ls, "addwait %x cur %d overlap %d count %d f %x", + lkb->lkb_id, lkb->lkb_wait_type, mstype, + lkb->lkb_wait_count, dlm_iflags_val(lkb)); + goto out; + } + + DLM_ASSERT(!lkb->lkb_wait_count, + dlm_print_lkb(lkb); + printk("wait_count %d\n", lkb->lkb_wait_count);); + + lkb->lkb_wait_count++; + lkb->lkb_wait_type = mstype; + lkb->lkb_wait_nodeid = to_nodeid; /* for debugging */ + hold_lkb(lkb); + list_add(&lkb->lkb_wait_reply, &ls->ls_waiters); + out: + spin_unlock_bh(&ls->ls_waiters_lock); +} + +/* We clear the RESEND flag because we might be taking an lkb off the waiters + list as part of process_requestqueue (e.g. a lookup that has an optimized + request reply on the requestqueue) between dlm_recover_waiters_pre() which + set RESEND and dlm_recover_waiters_post() */ + +static int _remove_from_waiters(struct dlm_lkb *lkb, int mstype, + const struct dlm_message *ms) +{ + struct dlm_ls *ls = lkb->lkb_resource->res_ls; + int overlap_done = 0; + + if (mstype == DLM_MSG_UNLOCK_REPLY && + test_and_clear_bit(DLM_IFL_OVERLAP_UNLOCK_BIT, &lkb->lkb_iflags)) { + log_debug(ls, "remwait %x unlock_reply overlap", lkb->lkb_id); + overlap_done = 1; + goto out_del; + } + + if (mstype == DLM_MSG_CANCEL_REPLY && + test_and_clear_bit(DLM_IFL_OVERLAP_CANCEL_BIT, &lkb->lkb_iflags)) { + log_debug(ls, "remwait %x cancel_reply overlap", lkb->lkb_id); + overlap_done = 1; + goto out_del; + } + + /* Cancel state was preemptively cleared by a successful convert, + see next comment, nothing to do. */ + + if ((mstype == DLM_MSG_CANCEL_REPLY) && + (lkb->lkb_wait_type != DLM_MSG_CANCEL)) { + log_debug(ls, "remwait %x cancel_reply wait_type %d", + lkb->lkb_id, lkb->lkb_wait_type); + return -1; + } + + /* Remove for the convert reply, and premptively remove for the + cancel reply. A convert has been granted while there's still + an outstanding cancel on it (the cancel is moot and the result + in the cancel reply should be 0). We preempt the cancel reply + because the app gets the convert result and then can follow up + with another op, like convert. This subsequent op would see the + lingering state of the cancel and fail with -EBUSY. */ + + if ((mstype == DLM_MSG_CONVERT_REPLY) && + (lkb->lkb_wait_type == DLM_MSG_CONVERT) && ms && !ms->m_result && + test_and_clear_bit(DLM_IFL_OVERLAP_CANCEL_BIT, &lkb->lkb_iflags)) { + log_debug(ls, "remwait %x convert_reply zap overlap_cancel", + lkb->lkb_id); + lkb->lkb_wait_type = 0; + lkb->lkb_wait_count--; + unhold_lkb(lkb); + goto out_del; + } + + /* N.B. type of reply may not always correspond to type of original + msg due to lookup->request optimization, verify others? */ + + if (lkb->lkb_wait_type) { + lkb->lkb_wait_type = 0; + goto out_del; + } + + log_error(ls, "remwait error %x remote %d %x msg %d flags %x no wait", + lkb->lkb_id, ms ? le32_to_cpu(ms->m_header.h_nodeid) : 0, + lkb->lkb_remid, mstype, dlm_iflags_val(lkb)); + return -1; + + out_del: + /* the force-unlock/cancel has completed and we haven't recvd a reply + to the op that was in progress prior to the unlock/cancel; we + give up on any reply to the earlier op. FIXME: not sure when/how + this would happen */ + + if (overlap_done && lkb->lkb_wait_type) { + log_error(ls, "remwait error %x reply %d wait_type %d overlap", + lkb->lkb_id, mstype, lkb->lkb_wait_type); + lkb->lkb_wait_count--; + unhold_lkb(lkb); + lkb->lkb_wait_type = 0; + } + + DLM_ASSERT(lkb->lkb_wait_count, dlm_print_lkb(lkb);); + + clear_bit(DLM_IFL_RESEND_BIT, &lkb->lkb_iflags); + lkb->lkb_wait_count--; + if (!lkb->lkb_wait_count) + list_del_init(&lkb->lkb_wait_reply); + unhold_lkb(lkb); + return 0; +} + +static int remove_from_waiters(struct dlm_lkb *lkb, int mstype) +{ + struct dlm_ls *ls = lkb->lkb_resource->res_ls; + int error; + + spin_lock_bh(&ls->ls_waiters_lock); + error = _remove_from_waiters(lkb, mstype, NULL); + spin_unlock_bh(&ls->ls_waiters_lock); + return error; +} + +/* Handles situations where we might be processing a "fake" or "local" reply in + * the recovery context which stops any locking activity. Only debugfs might + * change the lockspace waiters but they will held the recovery lock to ensure + * remove_from_waiters_ms() in local case will be the only user manipulating the + * lockspace waiters in recovery context. + */ + +static int remove_from_waiters_ms(struct dlm_lkb *lkb, + const struct dlm_message *ms, bool local) +{ + struct dlm_ls *ls = lkb->lkb_resource->res_ls; + int error; + + if (!local) + spin_lock_bh(&ls->ls_waiters_lock); + else + WARN_ON_ONCE(!rwsem_is_locked(&ls->ls_in_recovery) || + !dlm_locking_stopped(ls)); + error = _remove_from_waiters(lkb, le32_to_cpu(ms->m_type), ms); + if (!local) + spin_unlock_bh(&ls->ls_waiters_lock); + return error; +} + +/* lkb is master or local copy */ + +static void set_lvb_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + int b, len = r->res_ls->ls_lvblen; + + /* b=1 lvb returned to caller + b=0 lvb written to rsb or invalidated + b=-1 do nothing */ + + b = dlm_lvb_operations[lkb->lkb_grmode + 1][lkb->lkb_rqmode + 1]; + + if (b == 1) { + if (!lkb->lkb_lvbptr) + return; + + if (!(lkb->lkb_exflags & DLM_LKF_VALBLK)) + return; + + if (!r->res_lvbptr) + return; + + memcpy(lkb->lkb_lvbptr, r->res_lvbptr, len); + lkb->lkb_lvbseq = r->res_lvbseq; + + } else if (b == 0) { + if (lkb->lkb_exflags & DLM_LKF_IVVALBLK) { + rsb_set_flag(r, RSB_VALNOTVALID); + return; + } + + if (!lkb->lkb_lvbptr) + return; + + if (!(lkb->lkb_exflags & DLM_LKF_VALBLK)) + return; + + if (!r->res_lvbptr) + r->res_lvbptr = dlm_allocate_lvb(r->res_ls); + + if (!r->res_lvbptr) + return; + + memcpy(r->res_lvbptr, lkb->lkb_lvbptr, len); + r->res_lvbseq++; + lkb->lkb_lvbseq = r->res_lvbseq; + rsb_clear_flag(r, RSB_VALNOTVALID); + } + + if (rsb_flag(r, RSB_VALNOTVALID)) + set_bit(DLM_SBF_VALNOTVALID_BIT, &lkb->lkb_sbflags); +} + +static void set_lvb_unlock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + if (lkb->lkb_grmode < DLM_LOCK_PW) + return; + + if (lkb->lkb_exflags & DLM_LKF_IVVALBLK) { + rsb_set_flag(r, RSB_VALNOTVALID); + return; + } + + if (!lkb->lkb_lvbptr) + return; + + if (!(lkb->lkb_exflags & DLM_LKF_VALBLK)) + return; + + if (!r->res_lvbptr) + r->res_lvbptr = dlm_allocate_lvb(r->res_ls); + + if (!r->res_lvbptr) + return; + + memcpy(r->res_lvbptr, lkb->lkb_lvbptr, r->res_ls->ls_lvblen); + r->res_lvbseq++; + rsb_clear_flag(r, RSB_VALNOTVALID); +} + +/* lkb is process copy (pc) */ + +static void set_lvb_lock_pc(struct dlm_rsb *r, struct dlm_lkb *lkb, + const struct dlm_message *ms) +{ + int b; + + if (!lkb->lkb_lvbptr) + return; + + if (!(lkb->lkb_exflags & DLM_LKF_VALBLK)) + return; + + b = dlm_lvb_operations[lkb->lkb_grmode + 1][lkb->lkb_rqmode + 1]; + if (b == 1) { + int len = receive_extralen(ms); + if (len > r->res_ls->ls_lvblen) + len = r->res_ls->ls_lvblen; + memcpy(lkb->lkb_lvbptr, ms->m_extra, len); + lkb->lkb_lvbseq = le32_to_cpu(ms->m_lvbseq); + } +} + +/* Manipulate lkb's on rsb's convert/granted/waiting queues + remove_lock -- used for unlock, removes lkb from granted + revert_lock -- used for cancel, moves lkb from convert to granted + grant_lock -- used for request and convert, adds lkb to granted or + moves lkb from convert or waiting to granted + + Each of these is used for master or local copy lkb's. There is + also a _pc() variation used to make the corresponding change on + a process copy (pc) lkb. */ + +static void _remove_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + del_lkb(r, lkb); + lkb->lkb_grmode = DLM_LOCK_IV; + /* this unhold undoes the original ref from create_lkb() + so this leads to the lkb being freed */ + unhold_lkb(lkb); +} + +static void remove_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + set_lvb_unlock(r, lkb); + _remove_lock(r, lkb); +} + +static void remove_lock_pc(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + _remove_lock(r, lkb); +} + +/* returns: 0 did nothing + 1 moved lock to granted + -1 removed lock */ + +static int revert_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + int rv = 0; + + lkb->lkb_rqmode = DLM_LOCK_IV; + + switch (lkb->lkb_status) { + case DLM_LKSTS_GRANTED: + break; + case DLM_LKSTS_CONVERT: + move_lkb(r, lkb, DLM_LKSTS_GRANTED); + rv = 1; + break; + case DLM_LKSTS_WAITING: + del_lkb(r, lkb); + lkb->lkb_grmode = DLM_LOCK_IV; + /* this unhold undoes the original ref from create_lkb() + so this leads to the lkb being freed */ + unhold_lkb(lkb); + rv = -1; + break; + default: + log_print("invalid status for revert %d", lkb->lkb_status); + } + return rv; +} + +static int revert_lock_pc(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + return revert_lock(r, lkb); +} + +static void _grant_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + if (lkb->lkb_grmode != lkb->lkb_rqmode) { + lkb->lkb_grmode = lkb->lkb_rqmode; + if (lkb->lkb_status) + move_lkb(r, lkb, DLM_LKSTS_GRANTED); + else + add_lkb(r, lkb, DLM_LKSTS_GRANTED); + } + + lkb->lkb_rqmode = DLM_LOCK_IV; + lkb->lkb_highbast = 0; +} + +static void grant_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + set_lvb_lock(r, lkb); + _grant_lock(r, lkb); +} + +static void grant_lock_pc(struct dlm_rsb *r, struct dlm_lkb *lkb, + const struct dlm_message *ms) +{ + set_lvb_lock_pc(r, lkb, ms); + _grant_lock(r, lkb); +} + +/* called by grant_pending_locks() which means an async grant message must + be sent to the requesting node in addition to granting the lock if the + lkb belongs to a remote node. */ + +static void grant_lock_pending(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + grant_lock(r, lkb); + if (is_master_copy(lkb)) + send_grant(r, lkb); + else + queue_cast(r, lkb, 0); +} + +/* The special CONVDEADLK, ALTPR and ALTCW flags allow the master to + change the granted/requested modes. We're munging things accordingly in + the process copy. + CONVDEADLK: our grmode may have been forced down to NL to resolve a + conversion deadlock + ALTPR/ALTCW: our rqmode may have been changed to PR or CW to become + compatible with other granted locks */ + +static void munge_demoted(struct dlm_lkb *lkb) +{ + if (lkb->lkb_rqmode == DLM_LOCK_IV || lkb->lkb_grmode == DLM_LOCK_IV) { + log_print("munge_demoted %x invalid modes gr %d rq %d", + lkb->lkb_id, lkb->lkb_grmode, lkb->lkb_rqmode); + return; + } + + lkb->lkb_grmode = DLM_LOCK_NL; +} + +static void munge_altmode(struct dlm_lkb *lkb, const struct dlm_message *ms) +{ + if (ms->m_type != cpu_to_le32(DLM_MSG_REQUEST_REPLY) && + ms->m_type != cpu_to_le32(DLM_MSG_GRANT)) { + log_print("munge_altmode %x invalid reply type %d", + lkb->lkb_id, le32_to_cpu(ms->m_type)); + return; + } + + if (lkb->lkb_exflags & DLM_LKF_ALTPR) + lkb->lkb_rqmode = DLM_LOCK_PR; + else if (lkb->lkb_exflags & DLM_LKF_ALTCW) + lkb->lkb_rqmode = DLM_LOCK_CW; + else { + log_print("munge_altmode invalid exflags %x", lkb->lkb_exflags); + dlm_print_lkb(lkb); + } +} + +static inline int first_in_list(struct dlm_lkb *lkb, struct list_head *head) +{ + struct dlm_lkb *first = list_entry(head->next, struct dlm_lkb, + lkb_statequeue); + if (lkb->lkb_id == first->lkb_id) + return 1; + + return 0; +} + +/* Check if the given lkb conflicts with another lkb on the queue. */ + +static int queue_conflict(struct list_head *head, struct dlm_lkb *lkb) +{ + struct dlm_lkb *this; + + list_for_each_entry(this, head, lkb_statequeue) { + if (this == lkb) + continue; + if (!modes_compat(this, lkb)) + return 1; + } + return 0; +} + +/* + * "A conversion deadlock arises with a pair of lock requests in the converting + * queue for one resource. The granted mode of each lock blocks the requested + * mode of the other lock." + * + * Part 2: if the granted mode of lkb is preventing an earlier lkb in the + * convert queue from being granted, then deadlk/demote lkb. + * + * Example: + * Granted Queue: empty + * Convert Queue: NL->EX (first lock) + * PR->EX (second lock) + * + * The first lock can't be granted because of the granted mode of the second + * lock and the second lock can't be granted because it's not first in the + * list. We either cancel lkb's conversion (PR->EX) and return EDEADLK, or we + * demote the granted mode of lkb (from PR to NL) if it has the CONVDEADLK + * flag set and return DEMOTED in the lksb flags. + * + * Originally, this function detected conv-deadlk in a more limited scope: + * - if !modes_compat(lkb1, lkb2) && !modes_compat(lkb2, lkb1), or + * - if lkb1 was the first entry in the queue (not just earlier), and was + * blocked by the granted mode of lkb2, and there was nothing on the + * granted queue preventing lkb1 from being granted immediately, i.e. + * lkb2 was the only thing preventing lkb1 from being granted. + * + * That second condition meant we'd only say there was conv-deadlk if + * resolving it (by demotion) would lead to the first lock on the convert + * queue being granted right away. It allowed conversion deadlocks to exist + * between locks on the convert queue while they couldn't be granted anyway. + * + * Now, we detect and take action on conversion deadlocks immediately when + * they're created, even if they may not be immediately consequential. If + * lkb1 exists anywhere in the convert queue and lkb2 comes in with a granted + * mode that would prevent lkb1's conversion from being granted, we do a + * deadlk/demote on lkb2 right away and don't let it onto the convert queue. + * I think this means that the lkb_is_ahead condition below should always + * be zero, i.e. there will never be conv-deadlk between two locks that are + * both already on the convert queue. + */ + +static int conversion_deadlock_detect(struct dlm_rsb *r, struct dlm_lkb *lkb2) +{ + struct dlm_lkb *lkb1; + int lkb_is_ahead = 0; + + list_for_each_entry(lkb1, &r->res_convertqueue, lkb_statequeue) { + if (lkb1 == lkb2) { + lkb_is_ahead = 1; + continue; + } + + if (!lkb_is_ahead) { + if (!modes_compat(lkb2, lkb1)) + return 1; + } else { + if (!modes_compat(lkb2, lkb1) && + !modes_compat(lkb1, lkb2)) + return 1; + } + } + return 0; +} + +/* + * Return 1 if the lock can be granted, 0 otherwise. + * Also detect and resolve conversion deadlocks. + * + * lkb is the lock to be granted + * + * now is 1 if the function is being called in the context of the + * immediate request, it is 0 if called later, after the lock has been + * queued. + * + * recover is 1 if dlm_recover_grant() is trying to grant conversions + * after recovery. + * + * References are from chapter 6 of "VAXcluster Principles" by Roy Davis + */ + +static int _can_be_granted(struct dlm_rsb *r, struct dlm_lkb *lkb, int now, + int recover) +{ + int8_t conv = (lkb->lkb_grmode != DLM_LOCK_IV); + + /* + * 6-10: Version 5.4 introduced an option to address the phenomenon of + * a new request for a NL mode lock being blocked. + * + * 6-11: If the optional EXPEDITE flag is used with the new NL mode + * request, then it would be granted. In essence, the use of this flag + * tells the Lock Manager to expedite theis request by not considering + * what may be in the CONVERTING or WAITING queues... As of this + * writing, the EXPEDITE flag can be used only with new requests for NL + * mode locks. This flag is not valid for conversion requests. + * + * A shortcut. Earlier checks return an error if EXPEDITE is used in a + * conversion or used with a non-NL requested mode. We also know an + * EXPEDITE request is always granted immediately, so now must always + * be 1. The full condition to grant an expedite request: (now && + * !conv && lkb->rqmode == DLM_LOCK_NL && (flags & EXPEDITE)) can + * therefore be shortened to just checking the flag. + */ + + if (lkb->lkb_exflags & DLM_LKF_EXPEDITE) + return 1; + + /* + * A shortcut. Without this, !queue_conflict(grantqueue, lkb) would be + * added to the remaining conditions. + */ + + if (queue_conflict(&r->res_grantqueue, lkb)) + return 0; + + /* + * 6-3: By default, a conversion request is immediately granted if the + * requested mode is compatible with the modes of all other granted + * locks + */ + + if (queue_conflict(&r->res_convertqueue, lkb)) + return 0; + + /* + * The RECOVER_GRANT flag means dlm_recover_grant() is granting + * locks for a recovered rsb, on which lkb's have been rebuilt. + * The lkb's may have been rebuilt on the queues in a different + * order than they were in on the previous master. So, granting + * queued conversions in order after recovery doesn't make sense + * since the order hasn't been preserved anyway. The new order + * could also have created a new "in place" conversion deadlock. + * (e.g. old, failed master held granted EX, with PR->EX, NL->EX. + * After recovery, there would be no granted locks, and possibly + * NL->EX, PR->EX, an in-place conversion deadlock.) So, after + * recovery, grant conversions without considering order. + */ + + if (conv && recover) + return 1; + + /* + * 6-5: But the default algorithm for deciding whether to grant or + * queue conversion requests does not by itself guarantee that such + * requests are serviced on a "first come first serve" basis. This, in + * turn, can lead to a phenomenon known as "indefinate postponement". + * + * 6-7: This issue is dealt with by using the optional QUECVT flag with + * the system service employed to request a lock conversion. This flag + * forces certain conversion requests to be queued, even if they are + * compatible with the granted modes of other locks on the same + * resource. Thus, the use of this flag results in conversion requests + * being ordered on a "first come first servce" basis. + * + * DCT: This condition is all about new conversions being able to occur + * "in place" while the lock remains on the granted queue (assuming + * nothing else conflicts.) IOW if QUECVT isn't set, a conversion + * doesn't _have_ to go onto the convert queue where it's processed in + * order. The "now" variable is necessary to distinguish converts + * being received and processed for the first time now, because once a + * convert is moved to the conversion queue the condition below applies + * requiring fifo granting. + */ + + if (now && conv && !(lkb->lkb_exflags & DLM_LKF_QUECVT)) + return 1; + + /* + * Even if the convert is compat with all granted locks, + * QUECVT forces it behind other locks on the convert queue. + */ + + if (now && conv && (lkb->lkb_exflags & DLM_LKF_QUECVT)) { + if (list_empty(&r->res_convertqueue)) + return 1; + else + return 0; + } + + /* + * The NOORDER flag is set to avoid the standard vms rules on grant + * order. + */ + + if (lkb->lkb_exflags & DLM_LKF_NOORDER) + return 1; + + /* + * 6-3: Once in that queue [CONVERTING], a conversion request cannot be + * granted until all other conversion requests ahead of it are granted + * and/or canceled. + */ + + if (!now && conv && first_in_list(lkb, &r->res_convertqueue)) + return 1; + + /* + * 6-4: By default, a new request is immediately granted only if all + * three of the following conditions are satisfied when the request is + * issued: + * - The queue of ungranted conversion requests for the resource is + * empty. + * - The queue of ungranted new requests for the resource is empty. + * - The mode of the new request is compatible with the most + * restrictive mode of all granted locks on the resource. + */ + + if (now && !conv && list_empty(&r->res_convertqueue) && + list_empty(&r->res_waitqueue)) + return 1; + + /* + * 6-4: Once a lock request is in the queue of ungranted new requests, + * it cannot be granted until the queue of ungranted conversion + * requests is empty, all ungranted new requests ahead of it are + * granted and/or canceled, and it is compatible with the granted mode + * of the most restrictive lock granted on the resource. + */ + + if (!now && !conv && list_empty(&r->res_convertqueue) && + first_in_list(lkb, &r->res_waitqueue)) + return 1; + + return 0; +} + +static int can_be_granted(struct dlm_rsb *r, struct dlm_lkb *lkb, int now, + int recover, int *err) +{ + int rv; + int8_t alt = 0, rqmode = lkb->lkb_rqmode; + int8_t is_convert = (lkb->lkb_grmode != DLM_LOCK_IV); + + if (err) + *err = 0; + + rv = _can_be_granted(r, lkb, now, recover); + if (rv) + goto out; + + /* + * The CONVDEADLK flag is non-standard and tells the dlm to resolve + * conversion deadlocks by demoting grmode to NL, otherwise the dlm + * cancels one of the locks. + */ + + if (is_convert && can_be_queued(lkb) && + conversion_deadlock_detect(r, lkb)) { + if (lkb->lkb_exflags & DLM_LKF_CONVDEADLK) { + lkb->lkb_grmode = DLM_LOCK_NL; + set_bit(DLM_SBF_DEMOTED_BIT, &lkb->lkb_sbflags); + } else if (err) { + *err = -EDEADLK; + } else { + log_print("can_be_granted deadlock %x now %d", + lkb->lkb_id, now); + dlm_dump_rsb(r); + } + goto out; + } + + /* + * The ALTPR and ALTCW flags are non-standard and tell the dlm to try + * to grant a request in a mode other than the normal rqmode. It's a + * simple way to provide a big optimization to applications that can + * use them. + */ + + if (rqmode != DLM_LOCK_PR && (lkb->lkb_exflags & DLM_LKF_ALTPR)) + alt = DLM_LOCK_PR; + else if (rqmode != DLM_LOCK_CW && (lkb->lkb_exflags & DLM_LKF_ALTCW)) + alt = DLM_LOCK_CW; + + if (alt) { + lkb->lkb_rqmode = alt; + rv = _can_be_granted(r, lkb, now, 0); + if (rv) + set_bit(DLM_SBF_ALTMODE_BIT, &lkb->lkb_sbflags); + else + lkb->lkb_rqmode = rqmode; + } + out: + return rv; +} + +/* Returns the highest requested mode of all blocked conversions; sets + cw if there's a blocked conversion to DLM_LOCK_CW. */ + +static int grant_pending_convert(struct dlm_rsb *r, int high, int *cw, + unsigned int *count) +{ + struct dlm_lkb *lkb, *s; + int recover = rsb_flag(r, RSB_RECOVER_GRANT); + int hi, demoted, quit, grant_restart, demote_restart; + int deadlk; + + quit = 0; + restart: + grant_restart = 0; + demote_restart = 0; + hi = DLM_LOCK_IV; + + list_for_each_entry_safe(lkb, s, &r->res_convertqueue, lkb_statequeue) { + demoted = is_demoted(lkb); + deadlk = 0; + + if (can_be_granted(r, lkb, 0, recover, &deadlk)) { + grant_lock_pending(r, lkb); + grant_restart = 1; + if (count) + (*count)++; + continue; + } + + if (!demoted && is_demoted(lkb)) { + log_print("WARN: pending demoted %x node %d %s", + lkb->lkb_id, lkb->lkb_nodeid, r->res_name); + demote_restart = 1; + continue; + } + + if (deadlk) { + /* + * If DLM_LKB_NODLKWT flag is set and conversion + * deadlock is detected, we request blocking AST and + * down (or cancel) conversion. + */ + if (lkb->lkb_exflags & DLM_LKF_NODLCKWT) { + if (lkb->lkb_highbast < lkb->lkb_rqmode) { + queue_bast(r, lkb, lkb->lkb_rqmode); + lkb->lkb_highbast = lkb->lkb_rqmode; + } + } else { + log_print("WARN: pending deadlock %x node %d %s", + lkb->lkb_id, lkb->lkb_nodeid, + r->res_name); + dlm_dump_rsb(r); + } + continue; + } + + hi = max_t(int, lkb->lkb_rqmode, hi); + + if (cw && lkb->lkb_rqmode == DLM_LOCK_CW) + *cw = 1; + } + + if (grant_restart) + goto restart; + if (demote_restart && !quit) { + quit = 1; + goto restart; + } + + return max_t(int, high, hi); +} + +static int grant_pending_wait(struct dlm_rsb *r, int high, int *cw, + unsigned int *count) +{ + struct dlm_lkb *lkb, *s; + + list_for_each_entry_safe(lkb, s, &r->res_waitqueue, lkb_statequeue) { + if (can_be_granted(r, lkb, 0, 0, NULL)) { + grant_lock_pending(r, lkb); + if (count) + (*count)++; + } else { + high = max_t(int, lkb->lkb_rqmode, high); + if (lkb->lkb_rqmode == DLM_LOCK_CW) + *cw = 1; + } + } + + return high; +} + +/* cw of 1 means there's a lock with a rqmode of DLM_LOCK_CW that's blocked + on either the convert or waiting queue. + high is the largest rqmode of all locks blocked on the convert or + waiting queue. */ + +static int lock_requires_bast(struct dlm_lkb *gr, int high, int cw) +{ + if (gr->lkb_grmode == DLM_LOCK_PR && cw) { + if (gr->lkb_highbast < DLM_LOCK_EX) + return 1; + return 0; + } + + if (gr->lkb_highbast < high && + !__dlm_compat_matrix[gr->lkb_grmode+1][high+1]) + return 1; + return 0; +} + +static void grant_pending_locks(struct dlm_rsb *r, unsigned int *count) +{ + struct dlm_lkb *lkb, *s; + int high = DLM_LOCK_IV; + int cw = 0; + + if (!is_master(r)) { + log_print("grant_pending_locks r nodeid %d", r->res_nodeid); + dlm_dump_rsb(r); + return; + } + + high = grant_pending_convert(r, high, &cw, count); + high = grant_pending_wait(r, high, &cw, count); + + if (high == DLM_LOCK_IV) + return; + + /* + * If there are locks left on the wait/convert queue then send blocking + * ASTs to granted locks based on the largest requested mode (high) + * found above. + */ + + list_for_each_entry_safe(lkb, s, &r->res_grantqueue, lkb_statequeue) { + if (lkb->lkb_bastfn && lock_requires_bast(lkb, high, cw)) { + if (cw && high == DLM_LOCK_PR && + lkb->lkb_grmode == DLM_LOCK_PR) + queue_bast(r, lkb, DLM_LOCK_CW); + else + queue_bast(r, lkb, high); + lkb->lkb_highbast = high; + } + } +} + +static int modes_require_bast(struct dlm_lkb *gr, struct dlm_lkb *rq) +{ + if ((gr->lkb_grmode == DLM_LOCK_PR && rq->lkb_rqmode == DLM_LOCK_CW) || + (gr->lkb_grmode == DLM_LOCK_CW && rq->lkb_rqmode == DLM_LOCK_PR)) { + if (gr->lkb_highbast < DLM_LOCK_EX) + return 1; + return 0; + } + + if (gr->lkb_highbast < rq->lkb_rqmode && !modes_compat(gr, rq)) + return 1; + return 0; +} + +static void send_bast_queue(struct dlm_rsb *r, struct list_head *head, + struct dlm_lkb *lkb) +{ + struct dlm_lkb *gr; + + list_for_each_entry(gr, head, lkb_statequeue) { + /* skip self when sending basts to convertqueue */ + if (gr == lkb) + continue; + if (gr->lkb_bastfn && modes_require_bast(gr, lkb)) { + queue_bast(r, gr, lkb->lkb_rqmode); + gr->lkb_highbast = lkb->lkb_rqmode; + } + } +} + +static void send_blocking_asts(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + send_bast_queue(r, &r->res_grantqueue, lkb); +} + +static void send_blocking_asts_all(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + send_bast_queue(r, &r->res_grantqueue, lkb); + send_bast_queue(r, &r->res_convertqueue, lkb); +} + +/* set_master(r, lkb) -- set the master nodeid of a resource + + The purpose of this function is to set the nodeid field in the given + lkb using the nodeid field in the given rsb. If the rsb's nodeid is + known, it can just be copied to the lkb and the function will return + 0. If the rsb's nodeid is _not_ known, it needs to be looked up + before it can be copied to the lkb. + + When the rsb nodeid is being looked up remotely, the initial lkb + causing the lookup is kept on the ls_waiters list waiting for the + lookup reply. Other lkb's waiting for the same rsb lookup are kept + on the rsb's res_lookup list until the master is verified. + + Return values: + 0: nodeid is set in rsb/lkb and the caller should go ahead and use it + 1: the rsb master is not available and the lkb has been placed on + a wait queue +*/ + +static int set_master(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + int our_nodeid = dlm_our_nodeid(); + + if (rsb_flag(r, RSB_MASTER_UNCERTAIN)) { + rsb_clear_flag(r, RSB_MASTER_UNCERTAIN); + r->res_first_lkid = lkb->lkb_id; + lkb->lkb_nodeid = r->res_nodeid; + return 0; + } + + if (r->res_first_lkid && r->res_first_lkid != lkb->lkb_id) { + list_add_tail(&lkb->lkb_rsb_lookup, &r->res_lookup); + return 1; + } + + if (r->res_master_nodeid == our_nodeid) { + lkb->lkb_nodeid = 0; + return 0; + } + + if (r->res_master_nodeid) { + lkb->lkb_nodeid = r->res_master_nodeid; + return 0; + } + + if (dlm_dir_nodeid(r) == our_nodeid) { + /* This is a somewhat unusual case; find_rsb will usually + have set res_master_nodeid when dir nodeid is local, but + there are cases where we become the dir node after we've + past find_rsb and go through _request_lock again. + confirm_master() or process_lookup_list() needs to be + called after this. */ + log_debug(r->res_ls, "set_master %x self master %d dir %d %s", + lkb->lkb_id, r->res_master_nodeid, r->res_dir_nodeid, + r->res_name); + r->res_master_nodeid = our_nodeid; + r->res_nodeid = 0; + lkb->lkb_nodeid = 0; + return 0; + } + + r->res_first_lkid = lkb->lkb_id; + send_lookup(r, lkb); + return 1; +} + +static void process_lookup_list(struct dlm_rsb *r) +{ + struct dlm_lkb *lkb, *safe; + + list_for_each_entry_safe(lkb, safe, &r->res_lookup, lkb_rsb_lookup) { + list_del_init(&lkb->lkb_rsb_lookup); + _request_lock(r, lkb); + } +} + +/* confirm_master -- confirm (or deny) an rsb's master nodeid */ + +static void confirm_master(struct dlm_rsb *r, int error) +{ + struct dlm_lkb *lkb; + + if (!r->res_first_lkid) + return; + + switch (error) { + case 0: + case -EINPROGRESS: + r->res_first_lkid = 0; + process_lookup_list(r); + break; + + case -EAGAIN: + case -EBADR: + case -ENOTBLK: + /* the remote request failed and won't be retried (it was + a NOQUEUE, or has been canceled/unlocked); make a waiting + lkb the first_lkid */ + + r->res_first_lkid = 0; + + if (!list_empty(&r->res_lookup)) { + lkb = list_entry(r->res_lookup.next, struct dlm_lkb, + lkb_rsb_lookup); + list_del_init(&lkb->lkb_rsb_lookup); + r->res_first_lkid = lkb->lkb_id; + _request_lock(r, lkb); + } + break; + + default: + log_error(r->res_ls, "confirm_master unknown error %d", error); + } +} + +static int set_lock_args(int mode, struct dlm_lksb *lksb, uint32_t flags, + int namelen, void (*ast)(void *astparam), + void *astparam, + void (*bast)(void *astparam, int mode), + struct dlm_args *args) +{ + int rv = -EINVAL; + + /* check for invalid arg usage */ + + if (mode < 0 || mode > DLM_LOCK_EX) + goto out; + + if (!(flags & DLM_LKF_CONVERT) && (namelen > DLM_RESNAME_MAXLEN)) + goto out; + + if (flags & DLM_LKF_CANCEL) + goto out; + + if (flags & DLM_LKF_QUECVT && !(flags & DLM_LKF_CONVERT)) + goto out; + + if (flags & DLM_LKF_CONVDEADLK && !(flags & DLM_LKF_CONVERT)) + goto out; + + if (flags & DLM_LKF_CONVDEADLK && flags & DLM_LKF_NOQUEUE) + goto out; + + if (flags & DLM_LKF_EXPEDITE && flags & DLM_LKF_CONVERT) + goto out; + + if (flags & DLM_LKF_EXPEDITE && flags & DLM_LKF_QUECVT) + goto out; + + if (flags & DLM_LKF_EXPEDITE && flags & DLM_LKF_NOQUEUE) + goto out; + + if (flags & DLM_LKF_EXPEDITE && mode != DLM_LOCK_NL) + goto out; + + if (!ast || !lksb) + goto out; + + if (flags & DLM_LKF_VALBLK && !lksb->sb_lvbptr) + goto out; + + if (flags & DLM_LKF_CONVERT && !lksb->sb_lkid) + goto out; + + /* these args will be copied to the lkb in validate_lock_args, + it cannot be done now because when converting locks, fields in + an active lkb cannot be modified before locking the rsb */ + + args->flags = flags; + args->astfn = ast; + args->astparam = astparam; + args->bastfn = bast; + args->mode = mode; + args->lksb = lksb; + rv = 0; + out: + return rv; +} + +static int set_unlock_args(uint32_t flags, void *astarg, struct dlm_args *args) +{ + if (flags & ~(DLM_LKF_CANCEL | DLM_LKF_VALBLK | DLM_LKF_IVVALBLK | + DLM_LKF_FORCEUNLOCK)) + return -EINVAL; + + if (flags & DLM_LKF_CANCEL && flags & DLM_LKF_FORCEUNLOCK) + return -EINVAL; + + args->flags = flags; + args->astparam = astarg; + return 0; +} + +static int validate_lock_args(struct dlm_ls *ls, struct dlm_lkb *lkb, + struct dlm_args *args) +{ + int rv = -EBUSY; + + if (args->flags & DLM_LKF_CONVERT) { + if (lkb->lkb_status != DLM_LKSTS_GRANTED) + goto out; + + /* lock not allowed if there's any op in progress */ + if (lkb->lkb_wait_type || lkb->lkb_wait_count) + goto out; + + if (is_overlap(lkb)) + goto out; + + rv = -EINVAL; + if (test_bit(DLM_IFL_MSTCPY_BIT, &lkb->lkb_iflags)) + goto out; + + if (args->flags & DLM_LKF_QUECVT && + !__quecvt_compat_matrix[lkb->lkb_grmode+1][args->mode+1]) + goto out; + } + + lkb->lkb_exflags = args->flags; + dlm_set_sbflags_val(lkb, 0); + lkb->lkb_astfn = args->astfn; + lkb->lkb_astparam = args->astparam; + lkb->lkb_bastfn = args->bastfn; + lkb->lkb_rqmode = args->mode; + lkb->lkb_lksb = args->lksb; + lkb->lkb_lvbptr = args->lksb->sb_lvbptr; + lkb->lkb_ownpid = (int) current->pid; + rv = 0; + out: + switch (rv) { + case 0: + break; + case -EINVAL: + /* annoy the user because dlm usage is wrong */ + WARN_ON(1); + log_error(ls, "%s %d %x %x %x %d %d", __func__, + rv, lkb->lkb_id, dlm_iflags_val(lkb), args->flags, + lkb->lkb_status, lkb->lkb_wait_type); + break; + default: + log_debug(ls, "%s %d %x %x %x %d %d", __func__, + rv, lkb->lkb_id, dlm_iflags_val(lkb), args->flags, + lkb->lkb_status, lkb->lkb_wait_type); + break; + } + + return rv; +} + +/* when dlm_unlock() sees -EBUSY with CANCEL/FORCEUNLOCK it returns 0 + for success */ + +/* note: it's valid for lkb_nodeid/res_nodeid to be -1 when we get here + because there may be a lookup in progress and it's valid to do + cancel/unlockf on it */ + +static int validate_unlock_args(struct dlm_lkb *lkb, struct dlm_args *args) +{ + struct dlm_ls *ls = lkb->lkb_resource->res_ls; + int rv = -EBUSY; + + /* normal unlock not allowed if there's any op in progress */ + if (!(args->flags & (DLM_LKF_CANCEL | DLM_LKF_FORCEUNLOCK)) && + (lkb->lkb_wait_type || lkb->lkb_wait_count)) + goto out; + + /* an lkb may be waiting for an rsb lookup to complete where the + lookup was initiated by another lock */ + + if (!list_empty(&lkb->lkb_rsb_lookup)) { + if (args->flags & (DLM_LKF_CANCEL | DLM_LKF_FORCEUNLOCK)) { + log_debug(ls, "unlock on rsb_lookup %x", lkb->lkb_id); + list_del_init(&lkb->lkb_rsb_lookup); + queue_cast(lkb->lkb_resource, lkb, + args->flags & DLM_LKF_CANCEL ? + -DLM_ECANCEL : -DLM_EUNLOCK); + unhold_lkb(lkb); /* undoes create_lkb() */ + } + /* caller changes -EBUSY to 0 for CANCEL and FORCEUNLOCK */ + goto out; + } + + rv = -EINVAL; + if (test_bit(DLM_IFL_MSTCPY_BIT, &lkb->lkb_iflags)) { + log_error(ls, "unlock on MSTCPY %x", lkb->lkb_id); + dlm_print_lkb(lkb); + goto out; + } + + /* an lkb may still exist even though the lock is EOL'ed due to a + * cancel, unlock or failed noqueue request; an app can't use these + * locks; return same error as if the lkid had not been found at all + */ + + if (test_bit(DLM_IFL_ENDOFLIFE_BIT, &lkb->lkb_iflags)) { + log_debug(ls, "unlock on ENDOFLIFE %x", lkb->lkb_id); + rv = -ENOENT; + goto out; + } + + if (is_overlap_unlock(lkb)) + goto out; + + /* cancel not allowed with another cancel/unlock in progress */ + + if (args->flags & DLM_LKF_CANCEL) { + if (lkb->lkb_exflags & DLM_LKF_CANCEL) + goto out; + + if (is_overlap_cancel(lkb)) + goto out; + + if (test_bit(DLM_IFL_RESEND_BIT, &lkb->lkb_iflags)) { + set_bit(DLM_IFL_OVERLAP_CANCEL_BIT, &lkb->lkb_iflags); + rv = -EBUSY; + goto out; + } + + /* there's nothing to cancel */ + if (lkb->lkb_status == DLM_LKSTS_GRANTED && + !lkb->lkb_wait_type) { + rv = -EBUSY; + goto out; + } + + switch (lkb->lkb_wait_type) { + case DLM_MSG_LOOKUP: + case DLM_MSG_REQUEST: + set_bit(DLM_IFL_OVERLAP_CANCEL_BIT, &lkb->lkb_iflags); + rv = -EBUSY; + goto out; + case DLM_MSG_UNLOCK: + case DLM_MSG_CANCEL: + goto out; + } + /* add_to_waiters() will set OVERLAP_CANCEL */ + goto out_ok; + } + + /* do we need to allow a force-unlock if there's a normal unlock + already in progress? in what conditions could the normal unlock + fail such that we'd want to send a force-unlock to be sure? */ + + if (args->flags & DLM_LKF_FORCEUNLOCK) { + if (lkb->lkb_exflags & DLM_LKF_FORCEUNLOCK) + goto out; + + if (test_bit(DLM_IFL_RESEND_BIT, &lkb->lkb_iflags)) { + set_bit(DLM_IFL_OVERLAP_UNLOCK_BIT, &lkb->lkb_iflags); + rv = -EBUSY; + goto out; + } + + switch (lkb->lkb_wait_type) { + case DLM_MSG_LOOKUP: + case DLM_MSG_REQUEST: + set_bit(DLM_IFL_OVERLAP_UNLOCK_BIT, &lkb->lkb_iflags); + rv = -EBUSY; + goto out; + case DLM_MSG_UNLOCK: + goto out; + } + /* add_to_waiters() will set OVERLAP_UNLOCK */ + } + + out_ok: + /* an overlapping op shouldn't blow away exflags from other op */ + lkb->lkb_exflags |= args->flags; + dlm_set_sbflags_val(lkb, 0); + lkb->lkb_astparam = args->astparam; + rv = 0; + out: + switch (rv) { + case 0: + break; + case -EINVAL: + /* annoy the user because dlm usage is wrong */ + WARN_ON(1); + log_error(ls, "%s %d %x %x %x %x %d %s", __func__, rv, + lkb->lkb_id, dlm_iflags_val(lkb), lkb->lkb_exflags, + args->flags, lkb->lkb_wait_type, + lkb->lkb_resource->res_name); + break; + default: + log_debug(ls, "%s %d %x %x %x %x %d %s", __func__, rv, + lkb->lkb_id, dlm_iflags_val(lkb), lkb->lkb_exflags, + args->flags, lkb->lkb_wait_type, + lkb->lkb_resource->res_name); + break; + } + + return rv; +} + +/* + * Four stage 4 varieties: + * do_request(), do_convert(), do_unlock(), do_cancel() + * These are called on the master node for the given lock and + * from the central locking logic. + */ + +static int do_request(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + int error = 0; + + if (can_be_granted(r, lkb, 1, 0, NULL)) { + grant_lock(r, lkb); + queue_cast(r, lkb, 0); + goto out; + } + + if (can_be_queued(lkb)) { + error = -EINPROGRESS; + add_lkb(r, lkb, DLM_LKSTS_WAITING); + goto out; + } + + error = -EAGAIN; + queue_cast(r, lkb, -EAGAIN); + out: + return error; +} + +static void do_request_effects(struct dlm_rsb *r, struct dlm_lkb *lkb, + int error) +{ + switch (error) { + case -EAGAIN: + if (force_blocking_asts(lkb)) + send_blocking_asts_all(r, lkb); + break; + case -EINPROGRESS: + send_blocking_asts(r, lkb); + break; + } +} + +static int do_convert(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + int error = 0; + int deadlk = 0; + + /* changing an existing lock may allow others to be granted */ + + if (can_be_granted(r, lkb, 1, 0, &deadlk)) { + grant_lock(r, lkb); + queue_cast(r, lkb, 0); + goto out; + } + + /* can_be_granted() detected that this lock would block in a conversion + deadlock, so we leave it on the granted queue and return EDEADLK in + the ast for the convert. */ + + if (deadlk && !(lkb->lkb_exflags & DLM_LKF_NODLCKWT)) { + /* it's left on the granted queue */ + revert_lock(r, lkb); + queue_cast(r, lkb, -EDEADLK); + error = -EDEADLK; + goto out; + } + + /* is_demoted() means the can_be_granted() above set the grmode + to NL, and left us on the granted queue. This auto-demotion + (due to CONVDEADLK) might mean other locks, and/or this lock, are + now grantable. We have to try to grant other converting locks + before we try again to grant this one. */ + + if (is_demoted(lkb)) { + grant_pending_convert(r, DLM_LOCK_IV, NULL, NULL); + if (_can_be_granted(r, lkb, 1, 0)) { + grant_lock(r, lkb); + queue_cast(r, lkb, 0); + goto out; + } + /* else fall through and move to convert queue */ + } + + if (can_be_queued(lkb)) { + error = -EINPROGRESS; + del_lkb(r, lkb); + add_lkb(r, lkb, DLM_LKSTS_CONVERT); + goto out; + } + + error = -EAGAIN; + queue_cast(r, lkb, -EAGAIN); + out: + return error; +} + +static void do_convert_effects(struct dlm_rsb *r, struct dlm_lkb *lkb, + int error) +{ + switch (error) { + case 0: + grant_pending_locks(r, NULL); + /* grant_pending_locks also sends basts */ + break; + case -EAGAIN: + if (force_blocking_asts(lkb)) + send_blocking_asts_all(r, lkb); + break; + case -EINPROGRESS: + send_blocking_asts(r, lkb); + break; + } +} + +static int do_unlock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + remove_lock(r, lkb); + queue_cast(r, lkb, -DLM_EUNLOCK); + return -DLM_EUNLOCK; +} + +static void do_unlock_effects(struct dlm_rsb *r, struct dlm_lkb *lkb, + int error) +{ + grant_pending_locks(r, NULL); +} + +/* returns: 0 did nothing, -DLM_ECANCEL canceled lock */ + +static int do_cancel(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + int error; + + error = revert_lock(r, lkb); + if (error) { + queue_cast(r, lkb, -DLM_ECANCEL); + return -DLM_ECANCEL; + } + return 0; +} + +static void do_cancel_effects(struct dlm_rsb *r, struct dlm_lkb *lkb, + int error) +{ + if (error) + grant_pending_locks(r, NULL); +} + +/* + * Four stage 3 varieties: + * _request_lock(), _convert_lock(), _unlock_lock(), _cancel_lock() + */ + +/* add a new lkb to a possibly new rsb, called by requesting process */ + +static int _request_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + int error; + + /* set_master: sets lkb nodeid from r */ + + error = set_master(r, lkb); + if (error < 0) + goto out; + if (error) { + error = 0; + goto out; + } + + if (is_remote(r)) { + /* receive_request() calls do_request() on remote node */ + error = send_request(r, lkb); + } else { + error = do_request(r, lkb); + /* for remote locks the request_reply is sent + between do_request and do_request_effects */ + do_request_effects(r, lkb, error); + } + out: + return error; +} + +/* change some property of an existing lkb, e.g. mode */ + +static int _convert_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + int error; + + if (is_remote(r)) { + /* receive_convert() calls do_convert() on remote node */ + error = send_convert(r, lkb); + } else { + error = do_convert(r, lkb); + /* for remote locks the convert_reply is sent + between do_convert and do_convert_effects */ + do_convert_effects(r, lkb, error); + } + + return error; +} + +/* remove an existing lkb from the granted queue */ + +static int _unlock_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + int error; + + if (is_remote(r)) { + /* receive_unlock() calls do_unlock() on remote node */ + error = send_unlock(r, lkb); + } else { + error = do_unlock(r, lkb); + /* for remote locks the unlock_reply is sent + between do_unlock and do_unlock_effects */ + do_unlock_effects(r, lkb, error); + } + + return error; +} + +/* remove an existing lkb from the convert or wait queue */ + +static int _cancel_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + int error; + + if (is_remote(r)) { + /* receive_cancel() calls do_cancel() on remote node */ + error = send_cancel(r, lkb); + } else { + error = do_cancel(r, lkb); + /* for remote locks the cancel_reply is sent + between do_cancel and do_cancel_effects */ + do_cancel_effects(r, lkb, error); + } + + return error; +} + +/* + * Four stage 2 varieties: + * request_lock(), convert_lock(), unlock_lock(), cancel_lock() + */ + +static int request_lock(struct dlm_ls *ls, struct dlm_lkb *lkb, + const void *name, int len, + struct dlm_args *args) +{ + struct dlm_rsb *r; + int error; + + error = validate_lock_args(ls, lkb, args); + if (error) + return error; + + error = find_rsb(ls, name, len, 0, R_REQUEST, &r); + if (error) + return error; + + lock_rsb(r); + + attach_lkb(r, lkb); + lkb->lkb_lksb->sb_lkid = lkb->lkb_id; + + error = _request_lock(r, lkb); + + unlock_rsb(r); + put_rsb(r); + return error; +} + +static int convert_lock(struct dlm_ls *ls, struct dlm_lkb *lkb, + struct dlm_args *args) +{ + struct dlm_rsb *r; + int error; + + r = lkb->lkb_resource; + + hold_rsb(r); + lock_rsb(r); + + error = validate_lock_args(ls, lkb, args); + if (error) + goto out; + + error = _convert_lock(r, lkb); + out: + unlock_rsb(r); + put_rsb(r); + return error; +} + +static int unlock_lock(struct dlm_ls *ls, struct dlm_lkb *lkb, + struct dlm_args *args) +{ + struct dlm_rsb *r; + int error; + + r = lkb->lkb_resource; + + hold_rsb(r); + lock_rsb(r); + + error = validate_unlock_args(lkb, args); + if (error) + goto out; + + error = _unlock_lock(r, lkb); + out: + unlock_rsb(r); + put_rsb(r); + return error; +} + +static int cancel_lock(struct dlm_ls *ls, struct dlm_lkb *lkb, + struct dlm_args *args) +{ + struct dlm_rsb *r; + int error; + + r = lkb->lkb_resource; + + hold_rsb(r); + lock_rsb(r); + + error = validate_unlock_args(lkb, args); + if (error) + goto out; + + error = _cancel_lock(r, lkb); + out: + unlock_rsb(r); + put_rsb(r); + return error; +} + +/* + * Two stage 1 varieties: dlm_lock() and dlm_unlock() + */ + +int dlm_lock(dlm_lockspace_t *lockspace, + int mode, + struct dlm_lksb *lksb, + uint32_t flags, + const void *name, + unsigned int namelen, + uint32_t parent_lkid, + void (*ast) (void *astarg), + void *astarg, + void (*bast) (void *astarg, int mode)) +{ + struct dlm_ls *ls; + struct dlm_lkb *lkb; + struct dlm_args args; + int error, convert = flags & DLM_LKF_CONVERT; + + ls = dlm_find_lockspace_local(lockspace); + if (!ls) + return -EINVAL; + + dlm_lock_recovery(ls); + + if (convert) + error = find_lkb(ls, lksb->sb_lkid, &lkb); + else + error = create_lkb(ls, &lkb); + + if (error) + goto out; + + trace_dlm_lock_start(ls, lkb, name, namelen, mode, flags); + + error = set_lock_args(mode, lksb, flags, namelen, ast, astarg, bast, + &args); + if (error) + goto out_put; + + if (convert) + error = convert_lock(ls, lkb, &args); + else + error = request_lock(ls, lkb, name, namelen, &args); + + if (error == -EINPROGRESS) + error = 0; + out_put: + trace_dlm_lock_end(ls, lkb, name, namelen, mode, flags, error, true); + + if (convert || error) + __put_lkb(ls, lkb); + if (error == -EAGAIN || error == -EDEADLK) + error = 0; + out: + dlm_unlock_recovery(ls); + dlm_put_lockspace(ls); + return error; +} + +int dlm_unlock(dlm_lockspace_t *lockspace, + uint32_t lkid, + uint32_t flags, + struct dlm_lksb *lksb, + void *astarg) +{ + struct dlm_ls *ls; + struct dlm_lkb *lkb; + struct dlm_args args; + int error; + + ls = dlm_find_lockspace_local(lockspace); + if (!ls) + return -EINVAL; + + dlm_lock_recovery(ls); + + error = find_lkb(ls, lkid, &lkb); + if (error) + goto out; + + trace_dlm_unlock_start(ls, lkb, flags); + + error = set_unlock_args(flags, astarg, &args); + if (error) + goto out_put; + + if (flags & DLM_LKF_CANCEL) + error = cancel_lock(ls, lkb, &args); + else + error = unlock_lock(ls, lkb, &args); + + if (error == -DLM_EUNLOCK || error == -DLM_ECANCEL) + error = 0; + if (error == -EBUSY && (flags & (DLM_LKF_CANCEL | DLM_LKF_FORCEUNLOCK))) + error = 0; + out_put: + trace_dlm_unlock_end(ls, lkb, flags, error); + + dlm_put_lkb(lkb); + out: + dlm_unlock_recovery(ls); + dlm_put_lockspace(ls); + return error; +} + +/* + * send/receive routines for remote operations and replies + * + * send_args + * send_common + * send_request receive_request + * send_convert receive_convert + * send_unlock receive_unlock + * send_cancel receive_cancel + * send_grant receive_grant + * send_bast receive_bast + * send_lookup receive_lookup + * send_remove receive_remove + * + * send_common_reply + * receive_request_reply send_request_reply + * receive_convert_reply send_convert_reply + * receive_unlock_reply send_unlock_reply + * receive_cancel_reply send_cancel_reply + * receive_lookup_reply send_lookup_reply + */ + +static int _create_message(struct dlm_ls *ls, int mb_len, + int to_nodeid, int mstype, + struct dlm_message **ms_ret, + struct dlm_mhandle **mh_ret) +{ + struct dlm_message *ms; + struct dlm_mhandle *mh; + char *mb; + + /* get_buffer gives us a message handle (mh) that we need to + pass into midcomms_commit and a message buffer (mb) that we + write our data into */ + + mh = dlm_midcomms_get_mhandle(to_nodeid, mb_len, &mb); + if (!mh) + return -ENOBUFS; + + ms = (struct dlm_message *) mb; + + ms->m_header.h_version = cpu_to_le32(DLM_HEADER_MAJOR | DLM_HEADER_MINOR); + ms->m_header.u.h_lockspace = cpu_to_le32(ls->ls_global_id); + ms->m_header.h_nodeid = cpu_to_le32(dlm_our_nodeid()); + ms->m_header.h_length = cpu_to_le16(mb_len); + ms->m_header.h_cmd = DLM_MSG; + + ms->m_type = cpu_to_le32(mstype); + + *mh_ret = mh; + *ms_ret = ms; + return 0; +} + +static int create_message(struct dlm_rsb *r, struct dlm_lkb *lkb, + int to_nodeid, int mstype, + struct dlm_message **ms_ret, + struct dlm_mhandle **mh_ret) +{ + int mb_len = sizeof(struct dlm_message); + + switch (mstype) { + case DLM_MSG_REQUEST: + case DLM_MSG_LOOKUP: + case DLM_MSG_REMOVE: + mb_len += r->res_length; + break; + case DLM_MSG_CONVERT: + case DLM_MSG_UNLOCK: + case DLM_MSG_REQUEST_REPLY: + case DLM_MSG_CONVERT_REPLY: + case DLM_MSG_GRANT: + if (lkb && lkb->lkb_lvbptr && (lkb->lkb_exflags & DLM_LKF_VALBLK)) + mb_len += r->res_ls->ls_lvblen; + break; + } + + return _create_message(r->res_ls, mb_len, to_nodeid, mstype, + ms_ret, mh_ret); +} + +/* further lowcomms enhancements or alternate implementations may make + the return value from this function useful at some point */ + +static int send_message(struct dlm_mhandle *mh, struct dlm_message *ms, + const void *name, int namelen) +{ + dlm_midcomms_commit_mhandle(mh, name, namelen); + return 0; +} + +static void send_args(struct dlm_rsb *r, struct dlm_lkb *lkb, + struct dlm_message *ms) +{ + ms->m_nodeid = cpu_to_le32(lkb->lkb_nodeid); + ms->m_pid = cpu_to_le32(lkb->lkb_ownpid); + ms->m_lkid = cpu_to_le32(lkb->lkb_id); + ms->m_remid = cpu_to_le32(lkb->lkb_remid); + ms->m_exflags = cpu_to_le32(lkb->lkb_exflags); + ms->m_sbflags = cpu_to_le32(dlm_sbflags_val(lkb)); + ms->m_flags = cpu_to_le32(dlm_dflags_val(lkb)); + ms->m_lvbseq = cpu_to_le32(lkb->lkb_lvbseq); + ms->m_status = cpu_to_le32(lkb->lkb_status); + ms->m_grmode = cpu_to_le32(lkb->lkb_grmode); + ms->m_rqmode = cpu_to_le32(lkb->lkb_rqmode); + ms->m_hash = cpu_to_le32(r->res_hash); + + /* m_result and m_bastmode are set from function args, + not from lkb fields */ + + if (lkb->lkb_bastfn) + ms->m_asts |= cpu_to_le32(DLM_CB_BAST); + if (lkb->lkb_astfn) + ms->m_asts |= cpu_to_le32(DLM_CB_CAST); + + /* compare with switch in create_message; send_remove() doesn't + use send_args() */ + + switch (ms->m_type) { + case cpu_to_le32(DLM_MSG_REQUEST): + case cpu_to_le32(DLM_MSG_LOOKUP): + memcpy(ms->m_extra, r->res_name, r->res_length); + break; + case cpu_to_le32(DLM_MSG_CONVERT): + case cpu_to_le32(DLM_MSG_UNLOCK): + case cpu_to_le32(DLM_MSG_REQUEST_REPLY): + case cpu_to_le32(DLM_MSG_CONVERT_REPLY): + case cpu_to_le32(DLM_MSG_GRANT): + if (!lkb->lkb_lvbptr || !(lkb->lkb_exflags & DLM_LKF_VALBLK)) + break; + memcpy(ms->m_extra, lkb->lkb_lvbptr, r->res_ls->ls_lvblen); + break; + } +} + +static int send_common(struct dlm_rsb *r, struct dlm_lkb *lkb, int mstype) +{ + struct dlm_message *ms; + struct dlm_mhandle *mh; + int to_nodeid, error; + + to_nodeid = r->res_nodeid; + + add_to_waiters(lkb, mstype, to_nodeid); + error = create_message(r, lkb, to_nodeid, mstype, &ms, &mh); + if (error) + goto fail; + + send_args(r, lkb, ms); + + error = send_message(mh, ms, r->res_name, r->res_length); + if (error) + goto fail; + return 0; + + fail: + remove_from_waiters(lkb, msg_reply_type(mstype)); + return error; +} + +static int send_request(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + return send_common(r, lkb, DLM_MSG_REQUEST); +} + +static int send_convert(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + int error; + + error = send_common(r, lkb, DLM_MSG_CONVERT); + + /* down conversions go without a reply from the master */ + if (!error && down_conversion(lkb)) { + remove_from_waiters(lkb, DLM_MSG_CONVERT_REPLY); + r->res_ls->ls_local_ms.m_type = cpu_to_le32(DLM_MSG_CONVERT_REPLY); + r->res_ls->ls_local_ms.m_result = 0; + __receive_convert_reply(r, lkb, &r->res_ls->ls_local_ms, true); + } + + return error; +} + +/* FIXME: if this lkb is the only lock we hold on the rsb, then set + MASTER_UNCERTAIN to force the next request on the rsb to confirm + that the master is still correct. */ + +static int send_unlock(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + return send_common(r, lkb, DLM_MSG_UNLOCK); +} + +static int send_cancel(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + return send_common(r, lkb, DLM_MSG_CANCEL); +} + +static int send_grant(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + struct dlm_message *ms; + struct dlm_mhandle *mh; + int to_nodeid, error; + + to_nodeid = lkb->lkb_nodeid; + + error = create_message(r, lkb, to_nodeid, DLM_MSG_GRANT, &ms, &mh); + if (error) + goto out; + + send_args(r, lkb, ms); + + ms->m_result = 0; + + error = send_message(mh, ms, r->res_name, r->res_length); + out: + return error; +} + +static int send_bast(struct dlm_rsb *r, struct dlm_lkb *lkb, int mode) +{ + struct dlm_message *ms; + struct dlm_mhandle *mh; + int to_nodeid, error; + + to_nodeid = lkb->lkb_nodeid; + + error = create_message(r, NULL, to_nodeid, DLM_MSG_BAST, &ms, &mh); + if (error) + goto out; + + send_args(r, lkb, ms); + + ms->m_bastmode = cpu_to_le32(mode); + + error = send_message(mh, ms, r->res_name, r->res_length); + out: + return error; +} + +static int send_lookup(struct dlm_rsb *r, struct dlm_lkb *lkb) +{ + struct dlm_message *ms; + struct dlm_mhandle *mh; + int to_nodeid, error; + + to_nodeid = dlm_dir_nodeid(r); + + add_to_waiters(lkb, DLM_MSG_LOOKUP, to_nodeid); + error = create_message(r, NULL, to_nodeid, DLM_MSG_LOOKUP, &ms, &mh); + if (error) + goto fail; + + send_args(r, lkb, ms); + + error = send_message(mh, ms, r->res_name, r->res_length); + if (error) + goto fail; + return 0; + + fail: + remove_from_waiters(lkb, DLM_MSG_LOOKUP_REPLY); + return error; +} + +static int send_remove(struct dlm_rsb *r) +{ + struct dlm_message *ms; + struct dlm_mhandle *mh; + int to_nodeid, error; + + to_nodeid = dlm_dir_nodeid(r); + + error = create_message(r, NULL, to_nodeid, DLM_MSG_REMOVE, &ms, &mh); + if (error) + goto out; + + memcpy(ms->m_extra, r->res_name, r->res_length); + ms->m_hash = cpu_to_le32(r->res_hash); + + error = send_message(mh, ms, r->res_name, r->res_length); + out: + return error; +} + +static int send_common_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, + int mstype, int rv) +{ + struct dlm_message *ms; + struct dlm_mhandle *mh; + int to_nodeid, error; + + to_nodeid = lkb->lkb_nodeid; + + error = create_message(r, lkb, to_nodeid, mstype, &ms, &mh); + if (error) + goto out; + + send_args(r, lkb, ms); + + ms->m_result = cpu_to_le32(to_dlm_errno(rv)); + + error = send_message(mh, ms, r->res_name, r->res_length); + out: + return error; +} + +static int send_request_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv) +{ + return send_common_reply(r, lkb, DLM_MSG_REQUEST_REPLY, rv); +} + +static int send_convert_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv) +{ + return send_common_reply(r, lkb, DLM_MSG_CONVERT_REPLY, rv); +} + +static int send_unlock_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv) +{ + return send_common_reply(r, lkb, DLM_MSG_UNLOCK_REPLY, rv); +} + +static int send_cancel_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv) +{ + return send_common_reply(r, lkb, DLM_MSG_CANCEL_REPLY, rv); +} + +static int send_lookup_reply(struct dlm_ls *ls, + const struct dlm_message *ms_in, int ret_nodeid, + int rv) +{ + struct dlm_rsb *r = &ls->ls_local_rsb; + struct dlm_message *ms; + struct dlm_mhandle *mh; + int error, nodeid = le32_to_cpu(ms_in->m_header.h_nodeid); + + error = create_message(r, NULL, nodeid, DLM_MSG_LOOKUP_REPLY, &ms, &mh); + if (error) + goto out; + + ms->m_lkid = ms_in->m_lkid; + ms->m_result = cpu_to_le32(to_dlm_errno(rv)); + ms->m_nodeid = cpu_to_le32(ret_nodeid); + + error = send_message(mh, ms, ms_in->m_extra, receive_extralen(ms_in)); + out: + return error; +} + +/* which args we save from a received message depends heavily on the type + of message, unlike the send side where we can safely send everything about + the lkb for any type of message */ + +static void receive_flags(struct dlm_lkb *lkb, const struct dlm_message *ms) +{ + lkb->lkb_exflags = le32_to_cpu(ms->m_exflags); + dlm_set_sbflags_val(lkb, le32_to_cpu(ms->m_sbflags)); + dlm_set_dflags_val(lkb, le32_to_cpu(ms->m_flags)); +} + +static void receive_flags_reply(struct dlm_lkb *lkb, + const struct dlm_message *ms, + bool local) +{ + if (local) + return; + + dlm_set_sbflags_val(lkb, le32_to_cpu(ms->m_sbflags)); + dlm_set_dflags_val(lkb, le32_to_cpu(ms->m_flags)); +} + +static int receive_extralen(const struct dlm_message *ms) +{ + return (le16_to_cpu(ms->m_header.h_length) - + sizeof(struct dlm_message)); +} + +static int receive_lvb(struct dlm_ls *ls, struct dlm_lkb *lkb, + const struct dlm_message *ms) +{ + int len; + + if (lkb->lkb_exflags & DLM_LKF_VALBLK) { + if (!lkb->lkb_lvbptr) + lkb->lkb_lvbptr = dlm_allocate_lvb(ls); + if (!lkb->lkb_lvbptr) + return -ENOMEM; + len = receive_extralen(ms); + if (len > ls->ls_lvblen) + len = ls->ls_lvblen; + memcpy(lkb->lkb_lvbptr, ms->m_extra, len); + } + return 0; +} + +static void fake_bastfn(void *astparam, int mode) +{ + log_print("fake_bastfn should not be called"); +} + +static void fake_astfn(void *astparam) +{ + log_print("fake_astfn should not be called"); +} + +static int receive_request_args(struct dlm_ls *ls, struct dlm_lkb *lkb, + const struct dlm_message *ms) +{ + lkb->lkb_nodeid = le32_to_cpu(ms->m_header.h_nodeid); + lkb->lkb_ownpid = le32_to_cpu(ms->m_pid); + lkb->lkb_remid = le32_to_cpu(ms->m_lkid); + lkb->lkb_grmode = DLM_LOCK_IV; + lkb->lkb_rqmode = le32_to_cpu(ms->m_rqmode); + + lkb->lkb_bastfn = (ms->m_asts & cpu_to_le32(DLM_CB_BAST)) ? &fake_bastfn : NULL; + lkb->lkb_astfn = (ms->m_asts & cpu_to_le32(DLM_CB_CAST)) ? &fake_astfn : NULL; + + if (lkb->lkb_exflags & DLM_LKF_VALBLK) { + /* lkb was just created so there won't be an lvb yet */ + lkb->lkb_lvbptr = dlm_allocate_lvb(ls); + if (!lkb->lkb_lvbptr) + return -ENOMEM; + } + + return 0; +} + +static int receive_convert_args(struct dlm_ls *ls, struct dlm_lkb *lkb, + const struct dlm_message *ms) +{ + if (lkb->lkb_status != DLM_LKSTS_GRANTED) + return -EBUSY; + + if (receive_lvb(ls, lkb, ms)) + return -ENOMEM; + + lkb->lkb_rqmode = le32_to_cpu(ms->m_rqmode); + lkb->lkb_lvbseq = le32_to_cpu(ms->m_lvbseq); + + return 0; +} + +static int receive_unlock_args(struct dlm_ls *ls, struct dlm_lkb *lkb, + const struct dlm_message *ms) +{ + if (receive_lvb(ls, lkb, ms)) + return -ENOMEM; + return 0; +} + +/* We fill in the local-lkb fields with the info that send_xxxx_reply() + uses to send a reply and that the remote end uses to process the reply. */ + +static void setup_local_lkb(struct dlm_ls *ls, const struct dlm_message *ms) +{ + struct dlm_lkb *lkb = &ls->ls_local_lkb; + lkb->lkb_nodeid = le32_to_cpu(ms->m_header.h_nodeid); + lkb->lkb_remid = le32_to_cpu(ms->m_lkid); +} + +/* This is called after the rsb is locked so that we can safely inspect + fields in the lkb. */ + +static int validate_message(struct dlm_lkb *lkb, const struct dlm_message *ms) +{ + int from = le32_to_cpu(ms->m_header.h_nodeid); + int error = 0; + + /* currently mixing of user/kernel locks are not supported */ + if (ms->m_flags & cpu_to_le32(BIT(DLM_DFL_USER_BIT)) && + !test_bit(DLM_DFL_USER_BIT, &lkb->lkb_dflags)) { + log_error(lkb->lkb_resource->res_ls, + "got user dlm message for a kernel lock"); + error = -EINVAL; + goto out; + } + + switch (ms->m_type) { + case cpu_to_le32(DLM_MSG_CONVERT): + case cpu_to_le32(DLM_MSG_UNLOCK): + case cpu_to_le32(DLM_MSG_CANCEL): + if (!is_master_copy(lkb) || lkb->lkb_nodeid != from) + error = -EINVAL; + break; + + case cpu_to_le32(DLM_MSG_CONVERT_REPLY): + case cpu_to_le32(DLM_MSG_UNLOCK_REPLY): + case cpu_to_le32(DLM_MSG_CANCEL_REPLY): + case cpu_to_le32(DLM_MSG_GRANT): + case cpu_to_le32(DLM_MSG_BAST): + if (!is_process_copy(lkb) || lkb->lkb_nodeid != from) + error = -EINVAL; + break; + + case cpu_to_le32(DLM_MSG_REQUEST_REPLY): + if (!is_process_copy(lkb)) + error = -EINVAL; + else if (lkb->lkb_nodeid != -1 && lkb->lkb_nodeid != from) + error = -EINVAL; + break; + + default: + error = -EINVAL; + } + +out: + if (error) + log_error(lkb->lkb_resource->res_ls, + "ignore invalid message %d from %d %x %x %x %d", + le32_to_cpu(ms->m_type), from, lkb->lkb_id, + lkb->lkb_remid, dlm_iflags_val(lkb), + lkb->lkb_nodeid); + return error; +} + +static int receive_request(struct dlm_ls *ls, const struct dlm_message *ms) +{ + struct dlm_lkb *lkb; + struct dlm_rsb *r; + int from_nodeid; + int error, namelen = 0; + + from_nodeid = le32_to_cpu(ms->m_header.h_nodeid); + + error = create_lkb(ls, &lkb); + if (error) + goto fail; + + receive_flags(lkb, ms); + set_bit(DLM_IFL_MSTCPY_BIT, &lkb->lkb_iflags); + error = receive_request_args(ls, lkb, ms); + if (error) { + __put_lkb(ls, lkb); + goto fail; + } + + /* The dir node is the authority on whether we are the master + for this rsb or not, so if the master sends us a request, we should + recreate the rsb if we've destroyed it. This race happens when we + send a remove message to the dir node at the same time that the dir + node sends us a request for the rsb. */ + + namelen = receive_extralen(ms); + + error = find_rsb(ls, ms->m_extra, namelen, from_nodeid, + R_RECEIVE_REQUEST, &r); + if (error) { + __put_lkb(ls, lkb); + goto fail; + } + + lock_rsb(r); + + if (r->res_master_nodeid != dlm_our_nodeid()) { + error = validate_master_nodeid(ls, r, from_nodeid); + if (error) { + unlock_rsb(r); + put_rsb(r); + __put_lkb(ls, lkb); + goto fail; + } + } + + attach_lkb(r, lkb); + error = do_request(r, lkb); + send_request_reply(r, lkb, error); + do_request_effects(r, lkb, error); + + unlock_rsb(r); + put_rsb(r); + + if (error == -EINPROGRESS) + error = 0; + if (error) + dlm_put_lkb(lkb); + return 0; + + fail: + /* TODO: instead of returning ENOTBLK, add the lkb to res_lookup + and do this receive_request again from process_lookup_list once + we get the lookup reply. This would avoid a many repeated + ENOTBLK request failures when the lookup reply designating us + as master is delayed. */ + + if (error != -ENOTBLK) { + log_limit(ls, "receive_request %x from %d %d", + le32_to_cpu(ms->m_lkid), from_nodeid, error); + } + + setup_local_lkb(ls, ms); + send_request_reply(&ls->ls_local_rsb, &ls->ls_local_lkb, error); + return error; +} + +static int receive_convert(struct dlm_ls *ls, const struct dlm_message *ms) +{ + struct dlm_lkb *lkb; + struct dlm_rsb *r; + int error, reply = 1; + + error = find_lkb(ls, le32_to_cpu(ms->m_remid), &lkb); + if (error) + goto fail; + + if (lkb->lkb_remid != le32_to_cpu(ms->m_lkid)) { + log_error(ls, "receive_convert %x remid %x recover_seq %llu " + "remote %d %x", lkb->lkb_id, lkb->lkb_remid, + (unsigned long long)lkb->lkb_recover_seq, + le32_to_cpu(ms->m_header.h_nodeid), + le32_to_cpu(ms->m_lkid)); + error = -ENOENT; + dlm_put_lkb(lkb); + goto fail; + } + + r = lkb->lkb_resource; + + hold_rsb(r); + lock_rsb(r); + + error = validate_message(lkb, ms); + if (error) + goto out; + + receive_flags(lkb, ms); + + error = receive_convert_args(ls, lkb, ms); + if (error) { + send_convert_reply(r, lkb, error); + goto out; + } + + reply = !down_conversion(lkb); + + error = do_convert(r, lkb); + if (reply) + send_convert_reply(r, lkb, error); + do_convert_effects(r, lkb, error); + out: + unlock_rsb(r); + put_rsb(r); + dlm_put_lkb(lkb); + return 0; + + fail: + setup_local_lkb(ls, ms); + send_convert_reply(&ls->ls_local_rsb, &ls->ls_local_lkb, error); + return error; +} + +static int receive_unlock(struct dlm_ls *ls, const struct dlm_message *ms) +{ + struct dlm_lkb *lkb; + struct dlm_rsb *r; + int error; + + error = find_lkb(ls, le32_to_cpu(ms->m_remid), &lkb); + if (error) + goto fail; + + if (lkb->lkb_remid != le32_to_cpu(ms->m_lkid)) { + log_error(ls, "receive_unlock %x remid %x remote %d %x", + lkb->lkb_id, lkb->lkb_remid, + le32_to_cpu(ms->m_header.h_nodeid), + le32_to_cpu(ms->m_lkid)); + error = -ENOENT; + dlm_put_lkb(lkb); + goto fail; + } + + r = lkb->lkb_resource; + + hold_rsb(r); + lock_rsb(r); + + error = validate_message(lkb, ms); + if (error) + goto out; + + receive_flags(lkb, ms); + + error = receive_unlock_args(ls, lkb, ms); + if (error) { + send_unlock_reply(r, lkb, error); + goto out; + } + + error = do_unlock(r, lkb); + send_unlock_reply(r, lkb, error); + do_unlock_effects(r, lkb, error); + out: + unlock_rsb(r); + put_rsb(r); + dlm_put_lkb(lkb); + return 0; + + fail: + setup_local_lkb(ls, ms); + send_unlock_reply(&ls->ls_local_rsb, &ls->ls_local_lkb, error); + return error; +} + +static int receive_cancel(struct dlm_ls *ls, const struct dlm_message *ms) +{ + struct dlm_lkb *lkb; + struct dlm_rsb *r; + int error; + + error = find_lkb(ls, le32_to_cpu(ms->m_remid), &lkb); + if (error) + goto fail; + + receive_flags(lkb, ms); + + r = lkb->lkb_resource; + + hold_rsb(r); + lock_rsb(r); + + error = validate_message(lkb, ms); + if (error) + goto out; + + error = do_cancel(r, lkb); + send_cancel_reply(r, lkb, error); + do_cancel_effects(r, lkb, error); + out: + unlock_rsb(r); + put_rsb(r); + dlm_put_lkb(lkb); + return 0; + + fail: + setup_local_lkb(ls, ms); + send_cancel_reply(&ls->ls_local_rsb, &ls->ls_local_lkb, error); + return error; +} + +static int receive_grant(struct dlm_ls *ls, const struct dlm_message *ms) +{ + struct dlm_lkb *lkb; + struct dlm_rsb *r; + int error; + + error = find_lkb(ls, le32_to_cpu(ms->m_remid), &lkb); + if (error) + return error; + + r = lkb->lkb_resource; + + hold_rsb(r); + lock_rsb(r); + + error = validate_message(lkb, ms); + if (error) + goto out; + + receive_flags_reply(lkb, ms, false); + if (is_altmode(lkb)) + munge_altmode(lkb, ms); + grant_lock_pc(r, lkb, ms); + queue_cast(r, lkb, 0); + out: + unlock_rsb(r); + put_rsb(r); + dlm_put_lkb(lkb); + return 0; +} + +static int receive_bast(struct dlm_ls *ls, const struct dlm_message *ms) +{ + struct dlm_lkb *lkb; + struct dlm_rsb *r; + int error; + + error = find_lkb(ls, le32_to_cpu(ms->m_remid), &lkb); + if (error) + return error; + + r = lkb->lkb_resource; + + hold_rsb(r); + lock_rsb(r); + + error = validate_message(lkb, ms); + if (error) + goto out; + + queue_bast(r, lkb, le32_to_cpu(ms->m_bastmode)); + lkb->lkb_highbast = le32_to_cpu(ms->m_bastmode); + out: + unlock_rsb(r); + put_rsb(r); + dlm_put_lkb(lkb); + return 0; +} + +static void receive_lookup(struct dlm_ls *ls, const struct dlm_message *ms) +{ + int len, error, ret_nodeid, from_nodeid, our_nodeid; + + from_nodeid = le32_to_cpu(ms->m_header.h_nodeid); + our_nodeid = dlm_our_nodeid(); + + len = receive_extralen(ms); + + error = dlm_master_lookup(ls, from_nodeid, ms->m_extra, len, 0, + &ret_nodeid, NULL); + + /* Optimization: we're master so treat lookup as a request */ + if (!error && ret_nodeid == our_nodeid) { + receive_request(ls, ms); + return; + } + send_lookup_reply(ls, ms, ret_nodeid, error); +} + +static void receive_remove(struct dlm_ls *ls, const struct dlm_message *ms) +{ + char name[DLM_RESNAME_MAXLEN+1]; + struct dlm_rsb *r; + int rv, len, dir_nodeid, from_nodeid; + + from_nodeid = le32_to_cpu(ms->m_header.h_nodeid); + + len = receive_extralen(ms); + + if (len > DLM_RESNAME_MAXLEN) { + log_error(ls, "receive_remove from %d bad len %d", + from_nodeid, len); + return; + } + + dir_nodeid = dlm_hash2nodeid(ls, le32_to_cpu(ms->m_hash)); + if (dir_nodeid != dlm_our_nodeid()) { + log_error(ls, "receive_remove from %d bad nodeid %d", + from_nodeid, dir_nodeid); + return; + } + + /* + * Look for inactive rsb, if it's there, free it. + * If the rsb is active, it's being used, and we should ignore this + * message. This is an expected race between the dir node sending a + * request to the master node at the same time as the master node sends + * a remove to the dir node. The resolution to that race is for the + * dir node to ignore the remove message, and the master node to + * recreate the master rsb when it gets a request from the dir node for + * an rsb it doesn't have. + */ + + memset(name, 0, sizeof(name)); + memcpy(name, ms->m_extra, len); + + rcu_read_lock(); + rv = dlm_search_rsb_tree(&ls->ls_rsbtbl, name, len, &r); + if (rv) { + rcu_read_unlock(); + /* should not happen */ + log_error(ls, "%s from %d not found %s", __func__, + from_nodeid, name); + return; + } + + write_lock_bh(&ls->ls_rsbtbl_lock); + if (!rsb_flag(r, RSB_HASHED)) { + rcu_read_unlock(); + write_unlock_bh(&ls->ls_rsbtbl_lock); + /* should not happen */ + log_error(ls, "%s from %d got removed during removal %s", + __func__, from_nodeid, name); + return; + } + /* at this stage the rsb can only being freed here */ + rcu_read_unlock(); + + if (!rsb_flag(r, RSB_INACTIVE)) { + if (r->res_master_nodeid != from_nodeid) { + /* should not happen */ + log_error(ls, "receive_remove on active rsb from %d master %d", + from_nodeid, r->res_master_nodeid); + dlm_print_rsb(r); + write_unlock_bh(&ls->ls_rsbtbl_lock); + return; + } + + /* Ignore the remove message, see race comment above. */ + + log_debug(ls, "receive_remove from %d master %d first %x %s", + from_nodeid, r->res_master_nodeid, r->res_first_lkid, + name); + write_unlock_bh(&ls->ls_rsbtbl_lock); + return; + } + + if (r->res_master_nodeid != from_nodeid) { + log_error(ls, "receive_remove inactive from %d master %d", + from_nodeid, r->res_master_nodeid); + dlm_print_rsb(r); + write_unlock_bh(&ls->ls_rsbtbl_lock); + return; + } + + list_del(&r->res_slow_list); + rhashtable_remove_fast(&ls->ls_rsbtbl, &r->res_node, + dlm_rhash_rsb_params); + rsb_clear_flag(r, RSB_HASHED); + write_unlock_bh(&ls->ls_rsbtbl_lock); + + free_inactive_rsb(r); +} + +static void receive_purge(struct dlm_ls *ls, const struct dlm_message *ms) +{ + do_purge(ls, le32_to_cpu(ms->m_nodeid), le32_to_cpu(ms->m_pid)); +} + +static int receive_request_reply(struct dlm_ls *ls, + const struct dlm_message *ms) +{ + struct dlm_lkb *lkb; + struct dlm_rsb *r; + int error, mstype, result; + int from_nodeid = le32_to_cpu(ms->m_header.h_nodeid); + + error = find_lkb(ls, le32_to_cpu(ms->m_remid), &lkb); + if (error) + return error; + + r = lkb->lkb_resource; + hold_rsb(r); + lock_rsb(r); + + error = validate_message(lkb, ms); + if (error) + goto out; + + mstype = lkb->lkb_wait_type; + error = remove_from_waiters(lkb, DLM_MSG_REQUEST_REPLY); + if (error) { + log_error(ls, "receive_request_reply %x remote %d %x result %d", + lkb->lkb_id, from_nodeid, le32_to_cpu(ms->m_lkid), + from_dlm_errno(le32_to_cpu(ms->m_result))); + dlm_dump_rsb(r); + goto out; + } + + /* Optimization: the dir node was also the master, so it took our + lookup as a request and sent request reply instead of lookup reply */ + if (mstype == DLM_MSG_LOOKUP) { + r->res_master_nodeid = from_nodeid; + r->res_nodeid = from_nodeid; + lkb->lkb_nodeid = from_nodeid; + } + + /* this is the value returned from do_request() on the master */ + result = from_dlm_errno(le32_to_cpu(ms->m_result)); + + switch (result) { + case -EAGAIN: + /* request would block (be queued) on remote master */ + queue_cast(r, lkb, -EAGAIN); + confirm_master(r, -EAGAIN); + unhold_lkb(lkb); /* undoes create_lkb() */ + break; + + case -EINPROGRESS: + case 0: + /* request was queued or granted on remote master */ + receive_flags_reply(lkb, ms, false); + lkb->lkb_remid = le32_to_cpu(ms->m_lkid); + if (is_altmode(lkb)) + munge_altmode(lkb, ms); + if (result) { + add_lkb(r, lkb, DLM_LKSTS_WAITING); + } else { + grant_lock_pc(r, lkb, ms); + queue_cast(r, lkb, 0); + } + confirm_master(r, result); + break; + + case -EBADR: + case -ENOTBLK: + /* find_rsb failed to find rsb or rsb wasn't master */ + log_limit(ls, "receive_request_reply %x from %d %d " + "master %d dir %d first %x %s", lkb->lkb_id, + from_nodeid, result, r->res_master_nodeid, + r->res_dir_nodeid, r->res_first_lkid, r->res_name); + + if (r->res_dir_nodeid != dlm_our_nodeid() && + r->res_master_nodeid != dlm_our_nodeid()) { + /* cause _request_lock->set_master->send_lookup */ + r->res_master_nodeid = 0; + r->res_nodeid = -1; + lkb->lkb_nodeid = -1; + } + + if (is_overlap(lkb)) { + /* we'll ignore error in cancel/unlock reply */ + queue_cast_overlap(r, lkb); + confirm_master(r, result); + unhold_lkb(lkb); /* undoes create_lkb() */ + } else { + _request_lock(r, lkb); + + if (r->res_master_nodeid == dlm_our_nodeid()) + confirm_master(r, 0); + } + break; + + default: + log_error(ls, "receive_request_reply %x error %d", + lkb->lkb_id, result); + } + + if ((result == 0 || result == -EINPROGRESS) && + test_and_clear_bit(DLM_IFL_OVERLAP_UNLOCK_BIT, &lkb->lkb_iflags)) { + log_debug(ls, "receive_request_reply %x result %d unlock", + lkb->lkb_id, result); + clear_bit(DLM_IFL_OVERLAP_CANCEL_BIT, &lkb->lkb_iflags); + send_unlock(r, lkb); + } else if ((result == -EINPROGRESS) && + test_and_clear_bit(DLM_IFL_OVERLAP_CANCEL_BIT, + &lkb->lkb_iflags)) { + log_debug(ls, "receive_request_reply %x cancel", lkb->lkb_id); + clear_bit(DLM_IFL_OVERLAP_UNLOCK_BIT, &lkb->lkb_iflags); + send_cancel(r, lkb); + } else { + clear_bit(DLM_IFL_OVERLAP_CANCEL_BIT, &lkb->lkb_iflags); + clear_bit(DLM_IFL_OVERLAP_UNLOCK_BIT, &lkb->lkb_iflags); + } + out: + unlock_rsb(r); + put_rsb(r); + dlm_put_lkb(lkb); + return 0; +} + +static void __receive_convert_reply(struct dlm_rsb *r, struct dlm_lkb *lkb, + const struct dlm_message *ms, bool local) +{ + /* this is the value returned from do_convert() on the master */ + switch (from_dlm_errno(le32_to_cpu(ms->m_result))) { + case -EAGAIN: + /* convert would block (be queued) on remote master */ + queue_cast(r, lkb, -EAGAIN); + break; + + case -EDEADLK: + receive_flags_reply(lkb, ms, local); + revert_lock_pc(r, lkb); + queue_cast(r, lkb, -EDEADLK); + break; + + case -EINPROGRESS: + /* convert was queued on remote master */ + receive_flags_reply(lkb, ms, local); + if (is_demoted(lkb)) + munge_demoted(lkb); + del_lkb(r, lkb); + add_lkb(r, lkb, DLM_LKSTS_CONVERT); + break; + + case 0: + /* convert was granted on remote master */ + receive_flags_reply(lkb, ms, local); + if (is_demoted(lkb)) + munge_demoted(lkb); + grant_lock_pc(r, lkb, ms); + queue_cast(r, lkb, 0); + break; + + default: + log_error(r->res_ls, "receive_convert_reply %x remote %d %x %d", + lkb->lkb_id, le32_to_cpu(ms->m_header.h_nodeid), + le32_to_cpu(ms->m_lkid), + from_dlm_errno(le32_to_cpu(ms->m_result))); + dlm_print_rsb(r); + dlm_print_lkb(lkb); + } +} + +static void _receive_convert_reply(struct dlm_lkb *lkb, + const struct dlm_message *ms, bool local) +{ + struct dlm_rsb *r = lkb->lkb_resource; + int error; + + hold_rsb(r); + lock_rsb(r); + + error = validate_message(lkb, ms); + if (error) + goto out; + + error = remove_from_waiters_ms(lkb, ms, local); + if (error) + goto out; + + __receive_convert_reply(r, lkb, ms, local); + out: + unlock_rsb(r); + put_rsb(r); +} + +static int receive_convert_reply(struct dlm_ls *ls, + const struct dlm_message *ms) +{ + struct dlm_lkb *lkb; + int error; + + error = find_lkb(ls, le32_to_cpu(ms->m_remid), &lkb); + if (error) + return error; + + _receive_convert_reply(lkb, ms, false); + dlm_put_lkb(lkb); + return 0; +} + +static void _receive_unlock_reply(struct dlm_lkb *lkb, + const struct dlm_message *ms, bool local) +{ + struct dlm_rsb *r = lkb->lkb_resource; + int error; + + hold_rsb(r); + lock_rsb(r); + + error = validate_message(lkb, ms); + if (error) + goto out; + + error = remove_from_waiters_ms(lkb, ms, local); + if (error) + goto out; + + /* this is the value returned from do_unlock() on the master */ + + switch (from_dlm_errno(le32_to_cpu(ms->m_result))) { + case -DLM_EUNLOCK: + receive_flags_reply(lkb, ms, local); + remove_lock_pc(r, lkb); + queue_cast(r, lkb, -DLM_EUNLOCK); + break; + case -ENOENT: + break; + default: + log_error(r->res_ls, "receive_unlock_reply %x error %d", + lkb->lkb_id, from_dlm_errno(le32_to_cpu(ms->m_result))); + } + out: + unlock_rsb(r); + put_rsb(r); +} + +static int receive_unlock_reply(struct dlm_ls *ls, + const struct dlm_message *ms) +{ + struct dlm_lkb *lkb; + int error; + + error = find_lkb(ls, le32_to_cpu(ms->m_remid), &lkb); + if (error) + return error; + + _receive_unlock_reply(lkb, ms, false); + dlm_put_lkb(lkb); + return 0; +} + +static void _receive_cancel_reply(struct dlm_lkb *lkb, + const struct dlm_message *ms, bool local) +{ + struct dlm_rsb *r = lkb->lkb_resource; + int error; + + hold_rsb(r); + lock_rsb(r); + + error = validate_message(lkb, ms); + if (error) + goto out; + + error = remove_from_waiters_ms(lkb, ms, local); + if (error) + goto out; + + /* this is the value returned from do_cancel() on the master */ + + switch (from_dlm_errno(le32_to_cpu(ms->m_result))) { + case -DLM_ECANCEL: + receive_flags_reply(lkb, ms, local); + revert_lock_pc(r, lkb); + queue_cast(r, lkb, -DLM_ECANCEL); + break; + case 0: + break; + default: + log_error(r->res_ls, "receive_cancel_reply %x error %d", + lkb->lkb_id, + from_dlm_errno(le32_to_cpu(ms->m_result))); + } + out: + unlock_rsb(r); + put_rsb(r); +} + +static int receive_cancel_reply(struct dlm_ls *ls, + const struct dlm_message *ms) +{ + struct dlm_lkb *lkb; + int error; + + error = find_lkb(ls, le32_to_cpu(ms->m_remid), &lkb); + if (error) + return error; + + _receive_cancel_reply(lkb, ms, false); + dlm_put_lkb(lkb); + return 0; +} + +static void receive_lookup_reply(struct dlm_ls *ls, + const struct dlm_message *ms) +{ + struct dlm_lkb *lkb; + struct dlm_rsb *r; + int error, ret_nodeid; + int do_lookup_list = 0; + + error = find_lkb(ls, le32_to_cpu(ms->m_lkid), &lkb); + if (error) { + log_error(ls, "%s no lkid %x", __func__, + le32_to_cpu(ms->m_lkid)); + return; + } + + /* ms->m_result is the value returned by dlm_master_lookup on dir node + FIXME: will a non-zero error ever be returned? */ + + r = lkb->lkb_resource; + hold_rsb(r); + lock_rsb(r); + + error = remove_from_waiters(lkb, DLM_MSG_LOOKUP_REPLY); + if (error) + goto out; + + ret_nodeid = le32_to_cpu(ms->m_nodeid); + + /* We sometimes receive a request from the dir node for this + rsb before we've received the dir node's loookup_reply for it. + The request from the dir node implies we're the master, so we set + ourself as master in receive_request_reply, and verify here that + we are indeed the master. */ + + if (r->res_master_nodeid && (r->res_master_nodeid != ret_nodeid)) { + /* This should never happen */ + log_error(ls, "receive_lookup_reply %x from %d ret %d " + "master %d dir %d our %d first %x %s", + lkb->lkb_id, le32_to_cpu(ms->m_header.h_nodeid), + ret_nodeid, r->res_master_nodeid, r->res_dir_nodeid, + dlm_our_nodeid(), r->res_first_lkid, r->res_name); + } + + if (ret_nodeid == dlm_our_nodeid()) { + r->res_master_nodeid = ret_nodeid; + r->res_nodeid = 0; + do_lookup_list = 1; + r->res_first_lkid = 0; + } else if (ret_nodeid == -1) { + /* the remote node doesn't believe it's the dir node */ + log_error(ls, "receive_lookup_reply %x from %d bad ret_nodeid", + lkb->lkb_id, le32_to_cpu(ms->m_header.h_nodeid)); + r->res_master_nodeid = 0; + r->res_nodeid = -1; + lkb->lkb_nodeid = -1; + } else { + /* set_master() will set lkb_nodeid from r */ + r->res_master_nodeid = ret_nodeid; + r->res_nodeid = ret_nodeid; + } + + if (is_overlap(lkb)) { + log_debug(ls, "receive_lookup_reply %x unlock %x", + lkb->lkb_id, dlm_iflags_val(lkb)); + queue_cast_overlap(r, lkb); + unhold_lkb(lkb); /* undoes create_lkb() */ + goto out_list; + } + + _request_lock(r, lkb); + + out_list: + if (do_lookup_list) + process_lookup_list(r); + out: + unlock_rsb(r); + put_rsb(r); + dlm_put_lkb(lkb); +} + +static void _receive_message(struct dlm_ls *ls, const struct dlm_message *ms, + uint32_t saved_seq) +{ + int error = 0, noent = 0; + + if (WARN_ON_ONCE(!dlm_is_member(ls, le32_to_cpu(ms->m_header.h_nodeid)))) { + log_limit(ls, "receive %d from non-member %d %x %x %d", + le32_to_cpu(ms->m_type), + le32_to_cpu(ms->m_header.h_nodeid), + le32_to_cpu(ms->m_lkid), le32_to_cpu(ms->m_remid), + from_dlm_errno(le32_to_cpu(ms->m_result))); + return; + } + + switch (ms->m_type) { + + /* messages sent to a master node */ + + case cpu_to_le32(DLM_MSG_REQUEST): + error = receive_request(ls, ms); + break; + + case cpu_to_le32(DLM_MSG_CONVERT): + error = receive_convert(ls, ms); + break; + + case cpu_to_le32(DLM_MSG_UNLOCK): + error = receive_unlock(ls, ms); + break; + + case cpu_to_le32(DLM_MSG_CANCEL): + noent = 1; + error = receive_cancel(ls, ms); + break; + + /* messages sent from a master node (replies to above) */ + + case cpu_to_le32(DLM_MSG_REQUEST_REPLY): + error = receive_request_reply(ls, ms); + break; + + case cpu_to_le32(DLM_MSG_CONVERT_REPLY): + error = receive_convert_reply(ls, ms); + break; + + case cpu_to_le32(DLM_MSG_UNLOCK_REPLY): + error = receive_unlock_reply(ls, ms); + break; + + case cpu_to_le32(DLM_MSG_CANCEL_REPLY): + error = receive_cancel_reply(ls, ms); + break; + + /* messages sent from a master node (only two types of async msg) */ + + case cpu_to_le32(DLM_MSG_GRANT): + noent = 1; + error = receive_grant(ls, ms); + break; + + case cpu_to_le32(DLM_MSG_BAST): + noent = 1; + error = receive_bast(ls, ms); + break; + + /* messages sent to a dir node */ + + case cpu_to_le32(DLM_MSG_LOOKUP): + receive_lookup(ls, ms); + break; + + case cpu_to_le32(DLM_MSG_REMOVE): + receive_remove(ls, ms); + break; + + /* messages sent from a dir node (remove has no reply) */ + + case cpu_to_le32(DLM_MSG_LOOKUP_REPLY): + receive_lookup_reply(ls, ms); + break; + + /* other messages */ + + case cpu_to_le32(DLM_MSG_PURGE): + receive_purge(ls, ms); + break; + + default: + log_error(ls, "unknown message type %d", + le32_to_cpu(ms->m_type)); + } + + /* + * When checking for ENOENT, we're checking the result of + * find_lkb(m_remid): + * + * The lock id referenced in the message wasn't found. This may + * happen in normal usage for the async messages and cancel, so + * only use log_debug for them. + * + * Some errors are expected and normal. + */ + + if (error == -ENOENT && noent) { + log_debug(ls, "receive %d no %x remote %d %x saved_seq %u", + le32_to_cpu(ms->m_type), le32_to_cpu(ms->m_remid), + le32_to_cpu(ms->m_header.h_nodeid), + le32_to_cpu(ms->m_lkid), saved_seq); + } else if (error == -ENOENT) { + log_error(ls, "receive %d no %x remote %d %x saved_seq %u", + le32_to_cpu(ms->m_type), le32_to_cpu(ms->m_remid), + le32_to_cpu(ms->m_header.h_nodeid), + le32_to_cpu(ms->m_lkid), saved_seq); + + if (ms->m_type == cpu_to_le32(DLM_MSG_CONVERT)) + dlm_dump_rsb_hash(ls, le32_to_cpu(ms->m_hash)); + } + + if (error == -EINVAL) { + log_error(ls, "receive %d inval from %d lkid %x remid %x " + "saved_seq %u", + le32_to_cpu(ms->m_type), + le32_to_cpu(ms->m_header.h_nodeid), + le32_to_cpu(ms->m_lkid), le32_to_cpu(ms->m_remid), + saved_seq); + } +} + +/* If the lockspace is in recovery mode (locking stopped), then normal + messages are saved on the requestqueue for processing after recovery is + done. When not in recovery mode, we wait for dlm_recoverd to drain saved + messages off the requestqueue before we process new ones. This occurs right + after recovery completes when we transition from saving all messages on + requestqueue, to processing all the saved messages, to processing new + messages as they arrive. */ + +static void dlm_receive_message(struct dlm_ls *ls, const struct dlm_message *ms, + int nodeid) +{ +try_again: + read_lock_bh(&ls->ls_requestqueue_lock); + if (test_bit(LSFL_RECV_MSG_BLOCKED, &ls->ls_flags)) { + /* If we were a member of this lockspace, left, and rejoined, + other nodes may still be sending us messages from the + lockspace generation before we left. */ + if (WARN_ON_ONCE(!ls->ls_generation)) { + read_unlock_bh(&ls->ls_requestqueue_lock); + log_limit(ls, "receive %d from %d ignore old gen", + le32_to_cpu(ms->m_type), nodeid); + return; + } + + read_unlock_bh(&ls->ls_requestqueue_lock); + write_lock_bh(&ls->ls_requestqueue_lock); + /* recheck because we hold writelock now */ + if (!test_bit(LSFL_RECV_MSG_BLOCKED, &ls->ls_flags)) { + write_unlock_bh(&ls->ls_requestqueue_lock); + goto try_again; + } + + dlm_add_requestqueue(ls, nodeid, ms); + write_unlock_bh(&ls->ls_requestqueue_lock); + } else { + _receive_message(ls, ms, 0); + read_unlock_bh(&ls->ls_requestqueue_lock); + } +} + +/* This is called by dlm_recoverd to process messages that were saved on + the requestqueue. */ + +void dlm_receive_message_saved(struct dlm_ls *ls, const struct dlm_message *ms, + uint32_t saved_seq) +{ + _receive_message(ls, ms, saved_seq); +} + +/* This is called by the midcomms layer when something is received for + the lockspace. It could be either a MSG (normal message sent as part of + standard locking activity) or an RCOM (recovery message sent as part of + lockspace recovery). */ + +void dlm_receive_buffer(const union dlm_packet *p, int nodeid) +{ + const struct dlm_header *hd = &p->header; + struct dlm_ls *ls; + int type = 0; + + switch (hd->h_cmd) { + case DLM_MSG: + type = le32_to_cpu(p->message.m_type); + break; + case DLM_RCOM: + type = le32_to_cpu(p->rcom.rc_type); + break; + default: + log_print("invalid h_cmd %d from %u", hd->h_cmd, nodeid); + return; + } + + if (le32_to_cpu(hd->h_nodeid) != nodeid) { + log_print("invalid h_nodeid %d from %d lockspace %x", + le32_to_cpu(hd->h_nodeid), nodeid, + le32_to_cpu(hd->u.h_lockspace)); + return; + } + + ls = dlm_find_lockspace_global(le32_to_cpu(hd->u.h_lockspace)); + if (!ls) { + if (dlm_config.ci_log_debug) { + printk_ratelimited(KERN_DEBUG "dlm: invalid lockspace " + "%u from %d cmd %d type %d\n", + le32_to_cpu(hd->u.h_lockspace), nodeid, + hd->h_cmd, type); + } + + if (hd->h_cmd == DLM_RCOM && type == DLM_RCOM_STATUS) + dlm_send_ls_not_ready(nodeid, &p->rcom); + return; + } + + /* this rwsem allows dlm_ls_stop() to wait for all dlm_recv threads to + be inactive (in this ls) before transitioning to recovery mode */ + + read_lock_bh(&ls->ls_recv_active); + if (hd->h_cmd == DLM_MSG) + dlm_receive_message(ls, &p->message, nodeid); + else if (hd->h_cmd == DLM_RCOM) + dlm_receive_rcom(ls, &p->rcom, nodeid); + else + log_error(ls, "invalid h_cmd %d from %d lockspace %x", + hd->h_cmd, nodeid, le32_to_cpu(hd->u.h_lockspace)); + read_unlock_bh(&ls->ls_recv_active); + + dlm_put_lockspace(ls); +} + +static void recover_convert_waiter(struct dlm_ls *ls, struct dlm_lkb *lkb, + struct dlm_message *ms_local) +{ + if (middle_conversion(lkb) || lkb->lkb_rqmode >= lkb->lkb_grmode) + set_bit(DLM_IFL_RESEND_BIT, &lkb->lkb_iflags); + + /* lkb->lkb_rqmode < lkb->lkb_grmode shouldn't happen since down + conversions are async; there's no reply from the remote master */ +} + +/* A waiting lkb needs recovery if the master node has failed, or + the master node is changing (only when no directory is used) */ + +static int waiter_needs_recovery(struct dlm_ls *ls, struct dlm_lkb *lkb, + int dir_nodeid) +{ + if (dlm_no_directory(ls)) + return 1; + + if (dlm_is_removed(ls, lkb->lkb_wait_nodeid)) + return 1; + + return 0; +} + +/* Recovery for locks that are waiting for replies from nodes that are now + gone. We can just complete unlocks and cancels by faking a reply from the + dead node. Requests and up-conversions we flag to be resent after + recovery. Down-conversions can just be completed with a fake reply like + unlocks. Conversions between PR and CW need special attention. */ + +void dlm_recover_waiters_pre(struct dlm_ls *ls) +{ + struct dlm_lkb *lkb, *safe; + struct dlm_message *ms_local; + int wait_type, local_unlock_result, local_cancel_result; + int dir_nodeid; + + ms_local = kmalloc_obj(*ms_local); + if (!ms_local) + return; + + list_for_each_entry_safe(lkb, safe, &ls->ls_waiters, lkb_wait_reply) { + + dir_nodeid = dlm_dir_nodeid(lkb->lkb_resource); + + /* exclude debug messages about unlocks because there can be so + many and they aren't very interesting */ + + if (lkb->lkb_wait_type != DLM_MSG_UNLOCK) { + log_debug(ls, "waiter %x remote %x msg %d r_nodeid %d " + "lkb_nodeid %d wait_nodeid %d dir_nodeid %d", + lkb->lkb_id, + lkb->lkb_remid, + lkb->lkb_wait_type, + lkb->lkb_resource->res_nodeid, + lkb->lkb_nodeid, + lkb->lkb_wait_nodeid, + dir_nodeid); + } + + /* all outstanding lookups, regardless of destination will be + resent after recovery is done */ + + if (lkb->lkb_wait_type == DLM_MSG_LOOKUP) { + set_bit(DLM_IFL_RESEND_BIT, &lkb->lkb_iflags); + continue; + } + + if (!waiter_needs_recovery(ls, lkb, dir_nodeid)) + continue; + + wait_type = lkb->lkb_wait_type; + local_unlock_result = -DLM_EUNLOCK; + local_cancel_result = -DLM_ECANCEL; + + /* Main reply may have been received leaving a zero wait_type, + but a reply for the overlapping op may not have been + received. In that case we need to fake the appropriate + reply for the overlap op. */ + + if (!wait_type) { + if (is_overlap_cancel(lkb)) { + wait_type = DLM_MSG_CANCEL; + if (lkb->lkb_grmode == DLM_LOCK_IV) + local_cancel_result = 0; + } + if (is_overlap_unlock(lkb)) { + wait_type = DLM_MSG_UNLOCK; + if (lkb->lkb_grmode == DLM_LOCK_IV) + local_unlock_result = -ENOENT; + } + + log_debug(ls, "rwpre overlap %x %x %d %d %d", + lkb->lkb_id, dlm_iflags_val(lkb), wait_type, + local_cancel_result, local_unlock_result); + } + + switch (wait_type) { + + case DLM_MSG_REQUEST: + set_bit(DLM_IFL_RESEND_BIT, &lkb->lkb_iflags); + break; + + case DLM_MSG_CONVERT: + recover_convert_waiter(ls, lkb, ms_local); + break; + + case DLM_MSG_UNLOCK: + hold_lkb(lkb); + memset(ms_local, 0, sizeof(struct dlm_message)); + ms_local->m_type = cpu_to_le32(DLM_MSG_UNLOCK_REPLY); + ms_local->m_result = cpu_to_le32(to_dlm_errno(local_unlock_result)); + ms_local->m_header.h_nodeid = cpu_to_le32(lkb->lkb_nodeid); + _receive_unlock_reply(lkb, ms_local, true); + dlm_put_lkb(lkb); + break; + + case DLM_MSG_CANCEL: + hold_lkb(lkb); + memset(ms_local, 0, sizeof(struct dlm_message)); + ms_local->m_type = cpu_to_le32(DLM_MSG_CANCEL_REPLY); + ms_local->m_result = cpu_to_le32(to_dlm_errno(local_cancel_result)); + ms_local->m_header.h_nodeid = cpu_to_le32(lkb->lkb_nodeid); + _receive_cancel_reply(lkb, ms_local, true); + dlm_put_lkb(lkb); + break; + + default: + log_error(ls, "invalid lkb wait_type %d %d", + lkb->lkb_wait_type, wait_type); + } + schedule(); + } + kfree(ms_local); +} + +static struct dlm_lkb *find_resend_waiter(struct dlm_ls *ls) +{ + struct dlm_lkb *lkb = NULL, *iter; + + spin_lock_bh(&ls->ls_waiters_lock); + list_for_each_entry(iter, &ls->ls_waiters, lkb_wait_reply) { + if (test_bit(DLM_IFL_RESEND_BIT, &iter->lkb_iflags)) { + hold_lkb(iter); + lkb = iter; + break; + } + } + spin_unlock_bh(&ls->ls_waiters_lock); + + return lkb; +} + +/* + * Forced state reset for locks that were in the middle of remote operations + * when recovery happened (i.e. lkbs that were on the waiters list, waiting + * for a reply from a remote operation.) The lkbs remaining on the waiters + * list need to be reevaluated; some may need resending to a different node + * than previously, and some may now need local handling rather than remote. + * + * First, the lkb state for the voided remote operation is forcibly reset, + * equivalent to what remove_from_waiters() would normally do: + * . lkb removed from ls_waiters list + * . lkb wait_type cleared + * . lkb waiters_count cleared + * . lkb ref count decremented for each waiters_count (almost always 1, + * but possibly 2 in case of cancel/unlock overlapping, which means + * two remote replies were being expected for the lkb.) + * + * Second, the lkb is reprocessed like an original operation would be, + * by passing it to _request_lock or _convert_lock, which will either + * process the lkb operation locally, or send it to a remote node again + * and put the lkb back onto the waiters list. + * + * When reprocessing the lkb, we may find that it's flagged for an overlapping + * force-unlock or cancel, either from before recovery began, or after recovery + * finished. If this is the case, the unlock/cancel is done directly, and the + * original operation is not initiated again (no _request_lock/_convert_lock.) + */ + +int dlm_recover_waiters_post(struct dlm_ls *ls) +{ + struct dlm_lkb *lkb; + struct dlm_rsb *r; + int error = 0, mstype, err, oc, ou; + + while (1) { + if (dlm_locking_stopped(ls)) { + log_debug(ls, "recover_waiters_post aborted"); + error = -EINTR; + break; + } + + /* + * Find an lkb from the waiters list that's been affected by + * recovery node changes, and needs to be reprocessed. Does + * hold_lkb(), adding a refcount. + */ + lkb = find_resend_waiter(ls); + if (!lkb) + break; + + r = lkb->lkb_resource; + hold_rsb(r); + lock_rsb(r); + + /* + * If the lkb has been flagged for a force unlock or cancel, + * then the reprocessing below will be replaced by just doing + * the unlock/cancel directly. + */ + mstype = lkb->lkb_wait_type; + oc = test_and_clear_bit(DLM_IFL_OVERLAP_CANCEL_BIT, + &lkb->lkb_iflags); + ou = test_and_clear_bit(DLM_IFL_OVERLAP_UNLOCK_BIT, + &lkb->lkb_iflags); + err = 0; + + log_debug(ls, "waiter %x remote %x msg %d r_nodeid %d " + "lkb_nodeid %d wait_nodeid %d dir_nodeid %d " + "overlap %d %d", lkb->lkb_id, lkb->lkb_remid, mstype, + r->res_nodeid, lkb->lkb_nodeid, lkb->lkb_wait_nodeid, + dlm_dir_nodeid(r), oc, ou); + + /* + * No reply to the pre-recovery operation will now be received, + * so a forced equivalent of remove_from_waiters() is needed to + * reset the waiters state that was in place before recovery. + */ + + clear_bit(DLM_IFL_RESEND_BIT, &lkb->lkb_iflags); + + /* Forcibly clear wait_type */ + lkb->lkb_wait_type = 0; + + /* + * Forcibly reset wait_count and associated refcount. The + * wait_count will almost always be 1, but in case of an + * overlapping unlock/cancel it could be 2: see where + * add_to_waiters() finds the lkb is already on the waiters + * list and does lkb_wait_count++; hold_lkb(). + */ + while (lkb->lkb_wait_count) { + lkb->lkb_wait_count--; + unhold_lkb(lkb); + } + + /* Forcibly remove from waiters list */ + spin_lock_bh(&ls->ls_waiters_lock); + list_del_init(&lkb->lkb_wait_reply); + spin_unlock_bh(&ls->ls_waiters_lock); + + /* + * The lkb is now clear of all prior waiters state and can be + * processed locally, or sent to remote node again, or directly + * cancelled/unlocked. + */ + + if (oc || ou) { + /* do an unlock or cancel instead of resending */ + switch (mstype) { + case DLM_MSG_LOOKUP: + case DLM_MSG_REQUEST: + queue_cast(r, lkb, ou ? -DLM_EUNLOCK : + -DLM_ECANCEL); + unhold_lkb(lkb); /* undoes create_lkb() */ + break; + case DLM_MSG_CONVERT: + if (oc) { + queue_cast(r, lkb, -DLM_ECANCEL); + } else { + lkb->lkb_exflags |= DLM_LKF_FORCEUNLOCK; + _unlock_lock(r, lkb); + } + break; + default: + err = 1; + } + } else { + switch (mstype) { + case DLM_MSG_LOOKUP: + case DLM_MSG_REQUEST: + _request_lock(r, lkb); + if (r->res_nodeid != -1 && is_master(r)) + confirm_master(r, 0); + break; + case DLM_MSG_CONVERT: + _convert_lock(r, lkb); + break; + default: + err = 1; + } + } + + if (err) { + log_error(ls, "waiter %x msg %d r_nodeid %d " + "dir_nodeid %d overlap %d %d", + lkb->lkb_id, mstype, r->res_nodeid, + dlm_dir_nodeid(r), oc, ou); + } + unlock_rsb(r); + put_rsb(r); + dlm_put_lkb(lkb); + } + + return error; +} + +static void purge_mstcpy_list(struct dlm_ls *ls, struct dlm_rsb *r, + struct list_head *list) +{ + struct dlm_lkb *lkb, *safe; + + list_for_each_entry_safe(lkb, safe, list, lkb_statequeue) { + if (!is_master_copy(lkb)) + continue; + + /* don't purge lkbs we've added in recover_master_copy for + the current recovery seq */ + + if (lkb->lkb_recover_seq == ls->ls_recover_seq) + continue; + + del_lkb(r, lkb); + + /* this put should free the lkb */ + if (!dlm_put_lkb(lkb)) + log_error(ls, "purged mstcpy lkb not released"); + } +} + +void dlm_purge_mstcpy_locks(struct dlm_rsb *r) +{ + struct dlm_ls *ls = r->res_ls; + + purge_mstcpy_list(ls, r, &r->res_grantqueue); + purge_mstcpy_list(ls, r, &r->res_convertqueue); + purge_mstcpy_list(ls, r, &r->res_waitqueue); +} + +static void purge_dead_list(struct dlm_ls *ls, struct dlm_rsb *r, + struct list_head *list, + int nodeid_gone, unsigned int *count) +{ + struct dlm_lkb *lkb, *safe; + + list_for_each_entry_safe(lkb, safe, list, lkb_statequeue) { + if (!is_master_copy(lkb)) + continue; + + if ((lkb->lkb_nodeid == nodeid_gone) || + dlm_is_removed(ls, lkb->lkb_nodeid)) { + + /* tell recover_lvb to invalidate the lvb + because a node holding EX/PW failed */ + if ((lkb->lkb_exflags & DLM_LKF_VALBLK) && + (lkb->lkb_grmode >= DLM_LOCK_PW)) { + rsb_set_flag(r, RSB_RECOVER_LVB_INVAL); + } + + del_lkb(r, lkb); + + /* this put should free the lkb */ + if (!dlm_put_lkb(lkb)) + log_error(ls, "purged dead lkb not released"); + + rsb_set_flag(r, RSB_RECOVER_GRANT); + + (*count)++; + } + } +} + +/* Get rid of locks held by nodes that are gone. */ + +void dlm_recover_purge(struct dlm_ls *ls, const struct list_head *root_list) +{ + struct dlm_rsb *r; + struct dlm_member *memb; + int nodes_count = 0; + int nodeid_gone = 0; + unsigned int lkb_count = 0; + + /* cache one removed nodeid to optimize the common + case of a single node removed */ + + list_for_each_entry(memb, &ls->ls_nodes_gone, list) { + nodes_count++; + nodeid_gone = memb->nodeid; + } + + if (!nodes_count) + return; + + list_for_each_entry(r, root_list, res_root_list) { + lock_rsb(r); + if (r->res_nodeid != -1 && is_master(r)) { + purge_dead_list(ls, r, &r->res_grantqueue, + nodeid_gone, &lkb_count); + purge_dead_list(ls, r, &r->res_convertqueue, + nodeid_gone, &lkb_count); + purge_dead_list(ls, r, &r->res_waitqueue, + nodeid_gone, &lkb_count); + } + unlock_rsb(r); + + cond_resched(); + } + + if (lkb_count) + log_rinfo(ls, "dlm_recover_purge %u locks for %u nodes", + lkb_count, nodes_count); +} + +static struct dlm_rsb *find_grant_rsb(struct dlm_ls *ls) +{ + struct dlm_rsb *r; + + read_lock_bh(&ls->ls_rsbtbl_lock); + list_for_each_entry(r, &ls->ls_slow_active, res_slow_list) { + if (!rsb_flag(r, RSB_RECOVER_GRANT)) + continue; + if (!is_master(r)) { + rsb_clear_flag(r, RSB_RECOVER_GRANT); + continue; + } + hold_rsb(r); + read_unlock_bh(&ls->ls_rsbtbl_lock); + return r; + } + read_unlock_bh(&ls->ls_rsbtbl_lock); + return NULL; +} + +/* + * Attempt to grant locks on resources that we are the master of. + * Locks may have become grantable during recovery because locks + * from departed nodes have been purged (or not rebuilt), allowing + * previously blocked locks to now be granted. The subset of rsb's + * we are interested in are those with lkb's on either the convert or + * waiting queues. + * + * Simplest would be to go through each master rsb and check for non-empty + * convert or waiting queues, and attempt to grant on those rsbs. + * Checking the queues requires lock_rsb, though, for which we'd need + * to release the rsbtbl lock. This would make iterating through all + * rsb's very inefficient. So, we rely on earlier recovery routines + * to set RECOVER_GRANT on any rsb's that we should attempt to grant + * locks for. + */ + +void dlm_recover_grant(struct dlm_ls *ls) +{ + struct dlm_rsb *r; + unsigned int count = 0; + unsigned int rsb_count = 0; + unsigned int lkb_count = 0; + + while (1) { + r = find_grant_rsb(ls); + if (!r) + break; + + rsb_count++; + count = 0; + lock_rsb(r); + /* the RECOVER_GRANT flag is checked in the grant path */ + grant_pending_locks(r, &count); + rsb_clear_flag(r, RSB_RECOVER_GRANT); + lkb_count += count; + confirm_master(r, 0); + unlock_rsb(r); + put_rsb(r); + cond_resched(); + } + + if (lkb_count) + log_rinfo(ls, "dlm_recover_grant %u locks on %u resources", + lkb_count, rsb_count); +} + +static struct dlm_lkb *search_remid_list(struct list_head *head, int nodeid, + uint32_t remid) +{ + struct dlm_lkb *lkb; + + list_for_each_entry(lkb, head, lkb_statequeue) { + if (lkb->lkb_nodeid == nodeid && lkb->lkb_remid == remid) + return lkb; + } + return NULL; +} + +static struct dlm_lkb *search_remid(struct dlm_rsb *r, int nodeid, + uint32_t remid) +{ + struct dlm_lkb *lkb; + + lkb = search_remid_list(&r->res_grantqueue, nodeid, remid); + if (lkb) + return lkb; + lkb = search_remid_list(&r->res_convertqueue, nodeid, remid); + if (lkb) + return lkb; + lkb = search_remid_list(&r->res_waitqueue, nodeid, remid); + if (lkb) + return lkb; + return NULL; +} + +/* needs at least dlm_rcom + rcom_lock */ +static int receive_rcom_lock_args(struct dlm_ls *ls, struct dlm_lkb *lkb, + struct dlm_rsb *r, const struct dlm_rcom *rc) +{ + struct rcom_lock *rl = (struct rcom_lock *) rc->rc_buf; + + lkb->lkb_nodeid = le32_to_cpu(rc->rc_header.h_nodeid); + lkb->lkb_ownpid = le32_to_cpu(rl->rl_ownpid); + lkb->lkb_remid = le32_to_cpu(rl->rl_lkid); + lkb->lkb_exflags = le32_to_cpu(rl->rl_exflags); + dlm_set_dflags_val(lkb, le32_to_cpu(rl->rl_flags)); + set_bit(DLM_IFL_MSTCPY_BIT, &lkb->lkb_iflags); + lkb->lkb_lvbseq = le32_to_cpu(rl->rl_lvbseq); + lkb->lkb_rqmode = rl->rl_rqmode; + lkb->lkb_grmode = rl->rl_grmode; + /* don't set lkb_status because add_lkb wants to itself */ + + lkb->lkb_bastfn = (rl->rl_asts & DLM_CB_BAST) ? &fake_bastfn : NULL; + lkb->lkb_astfn = (rl->rl_asts & DLM_CB_CAST) ? &fake_astfn : NULL; + + if (lkb->lkb_exflags & DLM_LKF_VALBLK) { + int lvblen = le16_to_cpu(rc->rc_header.h_length) - + sizeof(struct dlm_rcom) - sizeof(struct rcom_lock); + if (lvblen > ls->ls_lvblen) + return -EINVAL; + lkb->lkb_lvbptr = dlm_allocate_lvb(ls); + if (!lkb->lkb_lvbptr) + return -ENOMEM; + memcpy(lkb->lkb_lvbptr, rl->rl_lvb, lvblen); + } + + /* Conversions between PR and CW (middle modes) need special handling. + The real granted mode of these converting locks cannot be determined + until all locks have been rebuilt on the rsb (recover_conversion) */ + + if (rl->rl_status == DLM_LKSTS_CONVERT && middle_conversion(lkb)) { + /* We may need to adjust grmode depending on other granted locks. */ + log_rinfo(ls, "%s %x middle convert gr %d rq %d remote %d %x", + __func__, lkb->lkb_id, lkb->lkb_grmode, + lkb->lkb_rqmode, lkb->lkb_nodeid, lkb->lkb_remid); + rsb_set_flag(r, RSB_RECOVER_CONVERT); + } + + return 0; +} + +/* This lkb may have been recovered in a previous aborted recovery so we need + to check if the rsb already has an lkb with the given remote nodeid/lkid. + If so we just send back a standard reply. If not, we create a new lkb with + the given values and send back our lkid. We send back our lkid by sending + back the rcom_lock struct we got but with the remid field filled in. */ + +/* needs at least dlm_rcom + rcom_lock */ +int dlm_recover_master_copy(struct dlm_ls *ls, const struct dlm_rcom *rc, + __le32 *rl_remid, __le32 *rl_result) +{ + struct rcom_lock *rl = (struct rcom_lock *) rc->rc_buf; + struct dlm_rsb *r; + struct dlm_lkb *lkb; + uint32_t remid = 0; + int from_nodeid = le32_to_cpu(rc->rc_header.h_nodeid); + int error; + + /* init rl_remid with rcom lock rl_remid */ + *rl_remid = rl->rl_remid; + + if (rl->rl_parent_lkid) { + error = -EOPNOTSUPP; + goto out; + } + + remid = le32_to_cpu(rl->rl_lkid); + + /* In general we expect the rsb returned to be R_MASTER, but we don't + have to require it. Recovery of masters on one node can overlap + recovery of locks on another node, so one node can send us MSTCPY + locks before we've made ourselves master of this rsb. We can still + add new MSTCPY locks that we receive here without any harm; when + we make ourselves master, dlm_recover_masters() won't touch the + MSTCPY locks we've received early. */ + + error = find_rsb(ls, rl->rl_name, le16_to_cpu(rl->rl_namelen), + from_nodeid, R_RECEIVE_RECOVER, &r); + if (error) + goto out; + + lock_rsb(r); + + if (dlm_no_directory(ls) && (dlm_dir_nodeid(r) != dlm_our_nodeid())) { + log_error(ls, "dlm_recover_master_copy remote %d %x not dir", + from_nodeid, remid); + error = -EBADR; + goto out_unlock; + } + + lkb = search_remid(r, from_nodeid, remid); + if (lkb) { + error = -EEXIST; + goto out_remid; + } + + error = create_lkb(ls, &lkb); + if (error) + goto out_unlock; + + error = receive_rcom_lock_args(ls, lkb, r, rc); + if (error) { + __put_lkb(ls, lkb); + goto out_unlock; + } + + attach_lkb(r, lkb); + add_lkb(r, lkb, rl->rl_status); + ls->ls_recover_locks_in++; + + if (!list_empty(&r->res_waitqueue) || !list_empty(&r->res_convertqueue)) + rsb_set_flag(r, RSB_RECOVER_GRANT); + + out_remid: + /* this is the new value returned to the lock holder for + saving in its process-copy lkb */ + *rl_remid = cpu_to_le32(lkb->lkb_id); + + lkb->lkb_recover_seq = ls->ls_recover_seq; + + out_unlock: + unlock_rsb(r); + put_rsb(r); + out: + if (error && error != -EEXIST) + log_rinfo(ls, "dlm_recover_master_copy remote %d %x error %d", + from_nodeid, remid, error); + *rl_result = cpu_to_le32(error); + return error; +} + +/* needs at least dlm_rcom + rcom_lock */ +int dlm_recover_process_copy(struct dlm_ls *ls, const struct dlm_rcom *rc, + uint64_t seq) +{ + struct rcom_lock *rl = (struct rcom_lock *) rc->rc_buf; + struct dlm_rsb *r; + struct dlm_lkb *lkb; + uint32_t lkid, remid; + int error, result; + + lkid = le32_to_cpu(rl->rl_lkid); + remid = le32_to_cpu(rl->rl_remid); + result = le32_to_cpu(rl->rl_result); + + error = find_lkb(ls, lkid, &lkb); + if (error) { + log_error(ls, "dlm_recover_process_copy no %x remote %d %x %d", + lkid, le32_to_cpu(rc->rc_header.h_nodeid), remid, + result); + return error; + } + + r = lkb->lkb_resource; + hold_rsb(r); + lock_rsb(r); + + if (!is_process_copy(lkb)) { + log_error(ls, "dlm_recover_process_copy bad %x remote %d %x %d", + lkid, le32_to_cpu(rc->rc_header.h_nodeid), remid, + result); + dlm_dump_rsb(r); + unlock_rsb(r); + put_rsb(r); + dlm_put_lkb(lkb); + return -EINVAL; + } + + switch (result) { + case -EBADR: + /* There's a chance the new master received our lock before + dlm_recover_master_reply(), this wouldn't happen if we did + a barrier between recover_masters and recover_locks. */ + + log_debug(ls, "dlm_recover_process_copy %x remote %d %x %d", + lkid, le32_to_cpu(rc->rc_header.h_nodeid), remid, + result); + + dlm_send_rcom_lock(r, lkb, seq); + goto out; + case -EEXIST: + case 0: + lkb->lkb_remid = remid; + break; + default: + log_error(ls, "dlm_recover_process_copy %x remote %d %x %d unk", + lkid, le32_to_cpu(rc->rc_header.h_nodeid), remid, + result); + } + + /* an ack for dlm_recover_locks() which waits for replies from + all the locks it sends to new masters */ + dlm_recovered_lock(r); + out: + unlock_rsb(r); + put_rsb(r); + dlm_put_lkb(lkb); + + return 0; +} + +int dlm_user_request(struct dlm_ls *ls, struct dlm_user_args *ua, + int mode, uint32_t flags, void *name, unsigned int namelen) +{ + struct dlm_lkb *lkb; + struct dlm_args args; + bool do_put = true; + int error; + + dlm_lock_recovery(ls); + + error = create_lkb(ls, &lkb); + if (error) { + kfree(ua); + goto out; + } + + trace_dlm_lock_start(ls, lkb, name, namelen, mode, flags); + + if (flags & DLM_LKF_VALBLK) { + ua->lksb.sb_lvbptr = kzalloc(DLM_USER_LVB_LEN, GFP_NOFS); + if (!ua->lksb.sb_lvbptr) { + kfree(ua); + error = -ENOMEM; + goto out_put; + } + } + error = set_lock_args(mode, &ua->lksb, flags, namelen, fake_astfn, ua, + fake_bastfn, &args); + if (error) { + kfree(ua->lksb.sb_lvbptr); + ua->lksb.sb_lvbptr = NULL; + kfree(ua); + goto out_put; + } + + /* After ua is attached to lkb it will be freed by dlm_free_lkb(). + When DLM_DFL_USER_BIT is set, the dlm knows that this is a userspace + lock and that lkb_astparam is the dlm_user_args structure. */ + set_bit(DLM_DFL_USER_BIT, &lkb->lkb_dflags); + error = request_lock(ls, lkb, name, namelen, &args); + + switch (error) { + case 0: + break; + case -EINPROGRESS: + error = 0; + break; + case -EAGAIN: + error = 0; + fallthrough; + default: + goto out_put; + } + + /* add this new lkb to the per-process list of locks */ + spin_lock_bh(&ua->proc->locks_spin); + hold_lkb(lkb); + list_add_tail(&lkb->lkb_ownqueue, &ua->proc->locks); + spin_unlock_bh(&ua->proc->locks_spin); + do_put = false; + out_put: + trace_dlm_lock_end(ls, lkb, name, namelen, mode, flags, error, false); + if (do_put) + __put_lkb(ls, lkb); + out: + dlm_unlock_recovery(ls); + return error; +} + +int dlm_user_convert(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, + int mode, uint32_t flags, uint32_t lkid, char *lvb_in) +{ + struct dlm_lkb *lkb; + struct dlm_args args; + struct dlm_user_args *ua; + int error; + + dlm_lock_recovery(ls); + + error = find_lkb(ls, lkid, &lkb); + if (error) + goto out; + + trace_dlm_lock_start(ls, lkb, NULL, 0, mode, flags); + + /* user can change the params on its lock when it converts it, or + add an lvb that didn't exist before */ + + ua = lkb->lkb_ua; + + if (flags & DLM_LKF_VALBLK && !ua->lksb.sb_lvbptr) { + ua->lksb.sb_lvbptr = kzalloc(DLM_USER_LVB_LEN, GFP_NOFS); + if (!ua->lksb.sb_lvbptr) { + error = -ENOMEM; + goto out_put; + } + } + if (lvb_in && ua->lksb.sb_lvbptr) + memcpy(ua->lksb.sb_lvbptr, lvb_in, DLM_USER_LVB_LEN); + + ua->xid = ua_tmp->xid; + ua->castparam = ua_tmp->castparam; + ua->castaddr = ua_tmp->castaddr; + ua->bastparam = ua_tmp->bastparam; + ua->bastaddr = ua_tmp->bastaddr; + ua->user_lksb = ua_tmp->user_lksb; + + error = set_lock_args(mode, &ua->lksb, flags, 0, fake_astfn, ua, + fake_bastfn, &args); + if (error) + goto out_put; + + error = convert_lock(ls, lkb, &args); + + if (error == -EINPROGRESS || error == -EAGAIN || error == -EDEADLK) + error = 0; + out_put: + trace_dlm_lock_end(ls, lkb, NULL, 0, mode, flags, error, false); + dlm_put_lkb(lkb); + out: + dlm_unlock_recovery(ls); + kfree(ua_tmp); + return error; +} + +/* + * The caller asks for an orphan lock on a given resource with a given mode. + * If a matching lock exists, it's moved to the owner's list of locks and + * the lkid is returned. + */ + +int dlm_user_adopt_orphan(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, + int mode, uint32_t flags, void *name, unsigned int namelen, + uint32_t *lkid) +{ + struct dlm_lkb *lkb = NULL, *iter; + struct dlm_user_args *ua; + int found_other_mode = 0; + int rv = 0; + + spin_lock_bh(&ls->ls_orphans_lock); + list_for_each_entry(iter, &ls->ls_orphans, lkb_ownqueue) { + if (iter->lkb_resource->res_length != namelen) + continue; + if (memcmp(iter->lkb_resource->res_name, name, namelen)) + continue; + if (iter->lkb_grmode != mode) { + found_other_mode = 1; + continue; + } + + lkb = iter; + list_del_init(&iter->lkb_ownqueue); + clear_bit(DLM_DFL_ORPHAN_BIT, &iter->lkb_dflags); + *lkid = iter->lkb_id; + break; + } + spin_unlock_bh(&ls->ls_orphans_lock); + + if (!lkb && found_other_mode) { + rv = -EAGAIN; + goto out; + } + + if (!lkb) { + rv = -ENOENT; + goto out; + } + + lkb->lkb_exflags = flags; + lkb->lkb_ownpid = (int) current->pid; + + ua = lkb->lkb_ua; + + ua->proc = ua_tmp->proc; + ua->xid = ua_tmp->xid; + ua->castparam = ua_tmp->castparam; + ua->castaddr = ua_tmp->castaddr; + ua->bastparam = ua_tmp->bastparam; + ua->bastaddr = ua_tmp->bastaddr; + ua->user_lksb = ua_tmp->user_lksb; + + /* + * The lkb reference from the ls_orphans list was not + * removed above, and is now considered the reference + * for the proc locks list. + */ + + spin_lock_bh(&ua->proc->locks_spin); + list_add_tail(&lkb->lkb_ownqueue, &ua->proc->locks); + spin_unlock_bh(&ua->proc->locks_spin); + out: + kfree(ua_tmp); + return rv; +} + +int dlm_user_unlock(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, + uint32_t flags, uint32_t lkid, char *lvb_in) +{ + struct dlm_lkb *lkb; + struct dlm_args args; + struct dlm_user_args *ua; + int error; + + dlm_lock_recovery(ls); + + error = find_lkb(ls, lkid, &lkb); + if (error) + goto out; + + trace_dlm_unlock_start(ls, lkb, flags); + + ua = lkb->lkb_ua; + + if (lvb_in && ua->lksb.sb_lvbptr) + memcpy(ua->lksb.sb_lvbptr, lvb_in, DLM_USER_LVB_LEN); + if (ua_tmp->castparam) + ua->castparam = ua_tmp->castparam; + ua->user_lksb = ua_tmp->user_lksb; + + error = set_unlock_args(flags, ua, &args); + if (error) + goto out_put; + + error = unlock_lock(ls, lkb, &args); + + if (error == -DLM_EUNLOCK) + error = 0; + /* from validate_unlock_args() */ + if (error == -EBUSY && (flags & DLM_LKF_FORCEUNLOCK)) + error = 0; + if (error) + goto out_put; + + spin_lock_bh(&ua->proc->locks_spin); + /* dlm_user_add_cb() may have already taken lkb off the proc list */ + if (!list_empty(&lkb->lkb_ownqueue)) + list_move(&lkb->lkb_ownqueue, &ua->proc->unlocking); + spin_unlock_bh(&ua->proc->locks_spin); + out_put: + trace_dlm_unlock_end(ls, lkb, flags, error); + dlm_put_lkb(lkb); + out: + dlm_unlock_recovery(ls); + kfree(ua_tmp); + return error; +} + +int dlm_user_cancel(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, + uint32_t flags, uint32_t lkid) +{ + struct dlm_lkb *lkb; + struct dlm_args args; + struct dlm_user_args *ua; + int error; + + dlm_lock_recovery(ls); + + error = find_lkb(ls, lkid, &lkb); + if (error) + goto out; + + trace_dlm_unlock_start(ls, lkb, flags); + + ua = lkb->lkb_ua; + if (ua_tmp->castparam) + ua->castparam = ua_tmp->castparam; + ua->user_lksb = ua_tmp->user_lksb; + + error = set_unlock_args(flags, ua, &args); + if (error) + goto out_put; + + error = cancel_lock(ls, lkb, &args); + + if (error == -DLM_ECANCEL) + error = 0; + /* from validate_unlock_args() */ + if (error == -EBUSY) + error = 0; + out_put: + trace_dlm_unlock_end(ls, lkb, flags, error); + dlm_put_lkb(lkb); + out: + dlm_unlock_recovery(ls); + kfree(ua_tmp); + return error; +} + +int dlm_user_deadlock(struct dlm_ls *ls, uint32_t flags, uint32_t lkid) +{ + struct dlm_lkb *lkb; + struct dlm_args args; + struct dlm_user_args *ua; + struct dlm_rsb *r; + int error; + + dlm_lock_recovery(ls); + + error = find_lkb(ls, lkid, &lkb); + if (error) + goto out; + + trace_dlm_unlock_start(ls, lkb, flags); + + ua = lkb->lkb_ua; + + error = set_unlock_args(flags, ua, &args); + if (error) + goto out_put; + + /* same as cancel_lock(), but set DEADLOCK_CANCEL after lock_rsb */ + + r = lkb->lkb_resource; + hold_rsb(r); + lock_rsb(r); + + error = validate_unlock_args(lkb, &args); + if (error) + goto out_r; + set_bit(DLM_IFL_DEADLOCK_CANCEL_BIT, &lkb->lkb_iflags); + + error = _cancel_lock(r, lkb); + out_r: + unlock_rsb(r); + put_rsb(r); + + if (error == -DLM_ECANCEL) + error = 0; + /* from validate_unlock_args() */ + if (error == -EBUSY) + error = 0; + out_put: + trace_dlm_unlock_end(ls, lkb, flags, error); + dlm_put_lkb(lkb); + out: + dlm_unlock_recovery(ls); + return error; +} + +/* lkb's that are removed from the waiters list by revert are just left on the + orphans list with the granted orphan locks, to be freed by purge */ + +static int orphan_proc_lock(struct dlm_ls *ls, struct dlm_lkb *lkb) +{ + struct dlm_args args; + int error; + + hold_lkb(lkb); /* reference for the ls_orphans list */ + spin_lock_bh(&ls->ls_orphans_lock); + list_add_tail(&lkb->lkb_ownqueue, &ls->ls_orphans); + spin_unlock_bh(&ls->ls_orphans_lock); + + set_unlock_args(0, lkb->lkb_ua, &args); + + error = cancel_lock(ls, lkb, &args); + if (error == -DLM_ECANCEL) + error = 0; + return error; +} + +/* The FORCEUNLOCK flag allows the unlock to go ahead even if the lkb isn't + granted. Regardless of what rsb queue the lock is on, it's removed and + freed. The IVVALBLK flag causes the lvb on the resource to be invalidated + if our lock is PW/EX (it's ignored if our granted mode is smaller.) */ + +static int unlock_proc_lock(struct dlm_ls *ls, struct dlm_lkb *lkb) +{ + struct dlm_args args; + int error; + + set_unlock_args(DLM_LKF_FORCEUNLOCK | DLM_LKF_IVVALBLK, + lkb->lkb_ua, &args); + + error = unlock_lock(ls, lkb, &args); + if (error == -DLM_EUNLOCK) + error = 0; + return error; +} + +/* We have to release clear_proc_locks mutex before calling unlock_proc_lock() + (which does lock_rsb) due to deadlock with receiving a message that does + lock_rsb followed by dlm_user_add_cb() */ + +static struct dlm_lkb *del_proc_lock(struct dlm_ls *ls, + struct dlm_user_proc *proc) +{ + struct dlm_lkb *lkb = NULL; + + spin_lock_bh(&ls->ls_clear_proc_locks); + if (list_empty(&proc->locks)) + goto out; + + lkb = list_entry(proc->locks.next, struct dlm_lkb, lkb_ownqueue); + list_del_init(&lkb->lkb_ownqueue); + + if (lkb->lkb_exflags & DLM_LKF_PERSISTENT) + set_bit(DLM_DFL_ORPHAN_BIT, &lkb->lkb_dflags); + else + set_bit(DLM_IFL_DEAD_BIT, &lkb->lkb_iflags); + out: + spin_unlock_bh(&ls->ls_clear_proc_locks); + return lkb; +} + +/* The ls_clear_proc_locks mutex protects against dlm_user_add_cb() which + 1) references lkb->ua which we free here and 2) adds lkbs to proc->asts, + which we clear here. */ + +/* proc CLOSING flag is set so no more device_reads should look at proc->asts + list, and no more device_writes should add lkb's to proc->locks list; so we + shouldn't need to take asts_spin or locks_spin here. this assumes that + device reads/writes/closes are serialized -- FIXME: we may need to serialize + them ourself. */ + +void dlm_clear_proc_locks(struct dlm_ls *ls, struct dlm_user_proc *proc) +{ + struct dlm_callback *cb, *cb_safe; + struct dlm_lkb *lkb, *safe; + + dlm_lock_recovery(ls); + + while (1) { + lkb = del_proc_lock(ls, proc); + if (!lkb) + break; + if (lkb->lkb_exflags & DLM_LKF_PERSISTENT) + orphan_proc_lock(ls, lkb); + else + unlock_proc_lock(ls, lkb); + + /* this removes the reference for the proc->locks list + added by dlm_user_request, it may result in the lkb + being freed */ + + dlm_put_lkb(lkb); + } + + spin_lock_bh(&ls->ls_clear_proc_locks); + + /* in-progress unlocks */ + list_for_each_entry_safe(lkb, safe, &proc->unlocking, lkb_ownqueue) { + list_del_init(&lkb->lkb_ownqueue); + set_bit(DLM_IFL_DEAD_BIT, &lkb->lkb_iflags); + dlm_put_lkb(lkb); + } + + list_for_each_entry_safe(cb, cb_safe, &proc->asts, list) { + list_del(&cb->list); + dlm_free_cb(cb); + } + + spin_unlock_bh(&ls->ls_clear_proc_locks); + dlm_unlock_recovery(ls); +} + +static void purge_proc_locks(struct dlm_ls *ls, struct dlm_user_proc *proc) +{ + struct dlm_callback *cb, *cb_safe; + struct dlm_lkb *lkb, *safe; + + while (1) { + lkb = NULL; + spin_lock_bh(&proc->locks_spin); + if (!list_empty(&proc->locks)) { + lkb = list_entry(proc->locks.next, struct dlm_lkb, + lkb_ownqueue); + list_del_init(&lkb->lkb_ownqueue); + } + spin_unlock_bh(&proc->locks_spin); + + if (!lkb) + break; + + set_bit(DLM_IFL_DEAD_BIT, &lkb->lkb_iflags); + unlock_proc_lock(ls, lkb); + dlm_put_lkb(lkb); /* ref from proc->locks list */ + } + + spin_lock_bh(&proc->locks_spin); + list_for_each_entry_safe(lkb, safe, &proc->unlocking, lkb_ownqueue) { + list_del_init(&lkb->lkb_ownqueue); + set_bit(DLM_IFL_DEAD_BIT, &lkb->lkb_iflags); + dlm_put_lkb(lkb); + } + spin_unlock_bh(&proc->locks_spin); + + spin_lock_bh(&proc->asts_spin); + list_for_each_entry_safe(cb, cb_safe, &proc->asts, list) { + list_del(&cb->list); + dlm_free_cb(cb); + } + spin_unlock_bh(&proc->asts_spin); +} + +/* pid of 0 means purge all orphans */ + +static void do_purge(struct dlm_ls *ls, int nodeid, int pid) +{ + struct dlm_lkb *lkb, *safe; + + spin_lock_bh(&ls->ls_orphans_lock); + list_for_each_entry_safe(lkb, safe, &ls->ls_orphans, lkb_ownqueue) { + if (pid && lkb->lkb_ownpid != pid) + continue; + unlock_proc_lock(ls, lkb); + list_del_init(&lkb->lkb_ownqueue); + dlm_put_lkb(lkb); + } + spin_unlock_bh(&ls->ls_orphans_lock); +} + +static int send_purge(struct dlm_ls *ls, int nodeid, int pid) +{ + struct dlm_message *ms; + struct dlm_mhandle *mh; + int error; + + error = _create_message(ls, sizeof(struct dlm_message), nodeid, + DLM_MSG_PURGE, &ms, &mh); + if (error) + return error; + ms->m_nodeid = cpu_to_le32(nodeid); + ms->m_pid = cpu_to_le32(pid); + + return send_message(mh, ms, NULL, 0); +} + +int dlm_user_purge(struct dlm_ls *ls, struct dlm_user_proc *proc, + int nodeid, int pid) +{ + int error = 0; + + if (nodeid && (nodeid != dlm_our_nodeid())) { + error = send_purge(ls, nodeid, pid); + } else { + dlm_lock_recovery(ls); + if (pid == current->pid) + purge_proc_locks(ls, proc); + else + do_purge(ls, nodeid, pid); + dlm_unlock_recovery(ls); + } + return error; +} + +/* debug functionality */ +int dlm_debug_add_lkb(struct dlm_ls *ls, uint32_t lkb_id, char *name, int len, + int lkb_nodeid, unsigned int lkb_dflags, int lkb_status) +{ + struct dlm_lksb *lksb; + struct dlm_lkb *lkb; + struct dlm_rsb *r; + int error; + + /* we currently can't set a valid user lock */ + if (lkb_dflags & BIT(DLM_DFL_USER_BIT)) + return -EOPNOTSUPP; + + lksb = kzalloc_obj(*lksb, GFP_NOFS); + if (!lksb) + return -ENOMEM; + + error = _create_lkb(ls, &lkb, lkb_id, lkb_id + 1); + if (error) { + kfree(lksb); + return error; + } + + dlm_set_dflags_val(lkb, lkb_dflags); + lkb->lkb_nodeid = lkb_nodeid; + lkb->lkb_lksb = lksb; + /* user specific pointer, just don't have it NULL for kernel locks */ + if (~lkb_dflags & BIT(DLM_DFL_USER_BIT)) + lkb->lkb_astparam = (void *)0xDEADBEEF; + + error = find_rsb(ls, name, len, 0, R_REQUEST, &r); + if (error) { + kfree(lksb); + __put_lkb(ls, lkb); + return error; + } + + lock_rsb(r); + attach_lkb(r, lkb); + add_lkb(r, lkb, lkb_status); + unlock_rsb(r); + put_rsb(r); + + return 0; +} + +int dlm_debug_add_lkb_to_waiters(struct dlm_ls *ls, uint32_t lkb_id, + int mstype, int to_nodeid) +{ + struct dlm_lkb *lkb; + int error; + + error = find_lkb(ls, lkb_id, &lkb); + if (error) + return error; + + add_to_waiters(lkb, mstype, to_nodeid); + dlm_put_lkb(lkb); + return 0; +} + diff --git a/fs/dlm/lock.h b/fs/dlm/lock.h new file mode 100644 index 000000000..b23d7b854 --- /dev/null +++ b/fs/dlm/lock.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) 2005-2007 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#ifndef __LOCK_DOT_H__ +#define __LOCK_DOT_H__ + +void dlm_dump_rsb(struct dlm_rsb *r); +void dlm_dump_rsb_name(struct dlm_ls *ls, const char *name, int len); +void dlm_print_lkb(struct dlm_lkb *lkb); +void dlm_receive_message_saved(struct dlm_ls *ls, const struct dlm_message *ms, + uint32_t saved_seq); +void dlm_receive_buffer(const union dlm_packet *p, int nodeid); +int dlm_modes_compat(int mode1, int mode2); +void free_inactive_rsb(struct dlm_rsb *r); +void dlm_put_rsb(struct dlm_rsb *r); +void dlm_hold_rsb(struct dlm_rsb *r); +int dlm_put_lkb(struct dlm_lkb *lkb); +int dlm_lock_recovery_try(struct dlm_ls *ls); +void dlm_lock_recovery(struct dlm_ls *ls); +void dlm_unlock_recovery(struct dlm_ls *ls); +void dlm_rsb_scan(struct timer_list *timer); +void resume_scan_timer(struct dlm_ls *ls); + +int dlm_master_lookup(struct dlm_ls *ls, int from_nodeid, const char *name, + int len, unsigned int flags, int *r_nodeid, int *result); + +int dlm_search_rsb_tree(struct rhashtable *rhash, const void *name, int len, + struct dlm_rsb **r_ret); + +void dlm_recover_purge(struct dlm_ls *ls, const struct list_head *root_list); +void dlm_purge_mstcpy_locks(struct dlm_rsb *r); +void dlm_recover_grant(struct dlm_ls *ls); +int dlm_recover_waiters_post(struct dlm_ls *ls); +void dlm_recover_waiters_pre(struct dlm_ls *ls); +int dlm_recover_master_copy(struct dlm_ls *ls, const struct dlm_rcom *rc, + __le32 *rl_remid, __le32 *rl_result); +int dlm_recover_process_copy(struct dlm_ls *ls, const struct dlm_rcom *rc, + uint64_t seq); + +int dlm_user_request(struct dlm_ls *ls, struct dlm_user_args *ua, int mode, + uint32_t flags, void *name, unsigned int namelen); +int dlm_user_convert(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, + int mode, uint32_t flags, uint32_t lkid, char *lvb_in); +int dlm_user_adopt_orphan(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, + int mode, uint32_t flags, void *name, unsigned int namelen, + uint32_t *lkid); +int dlm_user_unlock(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, + uint32_t flags, uint32_t lkid, char *lvb_in); +int dlm_user_cancel(struct dlm_ls *ls, struct dlm_user_args *ua_tmp, + uint32_t flags, uint32_t lkid); +int dlm_user_purge(struct dlm_ls *ls, struct dlm_user_proc *proc, + int nodeid, int pid); +int dlm_user_deadlock(struct dlm_ls *ls, uint32_t flags, uint32_t lkid); +void dlm_clear_proc_locks(struct dlm_ls *ls, struct dlm_user_proc *proc); +int dlm_debug_add_lkb(struct dlm_ls *ls, uint32_t lkb_id, char *name, int len, + int lkb_nodeid, unsigned int lkb_flags, int lkb_status); +int dlm_debug_add_lkb_to_waiters(struct dlm_ls *ls, uint32_t lkb_id, + int mstype, int to_nodeid); + +static inline int is_master(struct dlm_rsb *r) +{ + WARN_ON_ONCE(r->res_nodeid == -1); + + return !r->res_nodeid; +} + +static inline void lock_rsb(struct dlm_rsb *r) +{ + spin_lock_bh(&r->res_lock); +} + +static inline void unlock_rsb(struct dlm_rsb *r) +{ + spin_unlock_bh(&r->res_lock); +} + +#endif + diff --git a/fs/dlm/lockspace.c b/fs/dlm/lockspace.c new file mode 100644 index 000000000..a9c98b4f3 --- /dev/null +++ b/fs/dlm/lockspace.c @@ -0,0 +1,841 @@ +// SPDX-License-Identifier: GPL-2.0-only +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2011 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#include + +#include "dlm_internal.h" +#include "lockspace.h" +#include "member.h" +#include "recoverd.h" +#include "dir.h" +#include "midcomms.h" +#include "config.h" +#include "memory.h" +#include "lock.h" +#include "recover.h" +#include "requestqueue.h" +#include "user.h" +#include "ast.h" + +static int ls_count; +static struct mutex ls_lock; +static struct list_head lslist; +static spinlock_t lslist_lock; + +static ssize_t dlm_control_store(struct dlm_ls *ls, const char *buf, size_t len) +{ + ssize_t ret = len; + int n; + int rc = kstrtoint(buf, 0, &n); + + if (rc) + return rc; + ls = dlm_find_lockspace_local(ls); + if (!ls) + return -EINVAL; + + switch (n) { + case 0: + dlm_ls_stop(ls); + break; + case 1: + dlm_ls_start(ls); + break; + default: + ret = -EINVAL; + } + dlm_put_lockspace(ls); + return ret; +} + +static ssize_t dlm_event_store(struct dlm_ls *ls, const char *buf, size_t len) +{ + int rc = kstrtoint(buf, 0, &ls->ls_uevent_result); + + if (rc) + return rc; + set_bit(LSFL_UEVENT_WAIT, &ls->ls_flags); + wake_up(&ls->ls_uevent_wait); + return len; +} + +static ssize_t dlm_id_show(struct dlm_ls *ls, char *buf) +{ + return snprintf(buf, PAGE_SIZE, "%u\n", ls->ls_global_id); +} + +static ssize_t dlm_id_store(struct dlm_ls *ls, const char *buf, size_t len) +{ + int rc = kstrtouint(buf, 0, &ls->ls_global_id); + + if (rc) + return rc; + return len; +} + +static ssize_t dlm_nodir_show(struct dlm_ls *ls, char *buf) +{ + return snprintf(buf, PAGE_SIZE, "%u\n", dlm_no_directory(ls)); +} + +static ssize_t dlm_nodir_store(struct dlm_ls *ls, const char *buf, size_t len) +{ + int val; + int rc = kstrtoint(buf, 0, &val); + + if (rc) + return rc; + if (val == 1) + set_bit(LSFL_NODIR, &ls->ls_flags); + return len; +} + +static ssize_t dlm_recover_status_show(struct dlm_ls *ls, char *buf) +{ + uint32_t status = dlm_recover_status(ls); + return snprintf(buf, PAGE_SIZE, "%x\n", status); +} + +static ssize_t dlm_recover_nodeid_show(struct dlm_ls *ls, char *buf) +{ + return snprintf(buf, PAGE_SIZE, "%d\n", ls->ls_recover_nodeid); +} + +struct dlm_attr { + struct attribute attr; + ssize_t (*show)(struct dlm_ls *, char *); + ssize_t (*store)(struct dlm_ls *, const char *, size_t); +}; + +static struct dlm_attr dlm_attr_control = { + .attr = {.name = "control", .mode = S_IWUSR}, + .store = dlm_control_store +}; + +static struct dlm_attr dlm_attr_event = { + .attr = {.name = "event_done", .mode = S_IWUSR}, + .store = dlm_event_store +}; + +static struct dlm_attr dlm_attr_id = { + .attr = {.name = "id", .mode = S_IRUGO | S_IWUSR}, + .show = dlm_id_show, + .store = dlm_id_store +}; + +static struct dlm_attr dlm_attr_nodir = { + .attr = {.name = "nodir", .mode = S_IRUGO | S_IWUSR}, + .show = dlm_nodir_show, + .store = dlm_nodir_store +}; + +static struct dlm_attr dlm_attr_recover_status = { + .attr = {.name = "recover_status", .mode = S_IRUGO}, + .show = dlm_recover_status_show +}; + +static struct dlm_attr dlm_attr_recover_nodeid = { + .attr = {.name = "recover_nodeid", .mode = S_IRUGO}, + .show = dlm_recover_nodeid_show +}; + +static struct attribute *dlm_attrs[] = { + &dlm_attr_control.attr, + &dlm_attr_event.attr, + &dlm_attr_id.attr, + &dlm_attr_nodir.attr, + &dlm_attr_recover_status.attr, + &dlm_attr_recover_nodeid.attr, + NULL, +}; +ATTRIBUTE_GROUPS(dlm); + +static ssize_t dlm_attr_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + struct dlm_ls *ls = container_of(kobj, struct dlm_ls, ls_kobj); + struct dlm_attr *a = container_of(attr, struct dlm_attr, attr); + return a->show ? a->show(ls, buf) : 0; +} + +static ssize_t dlm_attr_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t len) +{ + struct dlm_ls *ls = container_of(kobj, struct dlm_ls, ls_kobj); + struct dlm_attr *a = container_of(attr, struct dlm_attr, attr); + return a->store ? a->store(ls, buf, len) : len; +} + +static const struct sysfs_ops dlm_attr_ops = { + .show = dlm_attr_show, + .store = dlm_attr_store, +}; + +static struct kobj_type dlm_ktype = { + .default_groups = dlm_groups, + .sysfs_ops = &dlm_attr_ops, +}; + +static struct kset *dlm_kset; + +static int do_uevent(struct dlm_ls *ls, int in, unsigned int release_recover) +{ + char message[512] = {}; + char *envp[] = { message, NULL }; + + if (in) { + kobject_uevent(&ls->ls_kobj, KOBJ_ONLINE); + } else { + snprintf(message, 511, "RELEASE_RECOVER=%u", release_recover); + kobject_uevent_env(&ls->ls_kobj, KOBJ_OFFLINE, envp); + } + + log_rinfo(ls, "%s the lockspace group...", in ? "joining" : "leaving"); + + /* dlm_controld will see the uevent, do the necessary group management + and then write to sysfs to wake us */ + + wait_event(ls->ls_uevent_wait, + test_and_clear_bit(LSFL_UEVENT_WAIT, &ls->ls_flags)); + + log_rinfo(ls, "group event done %d", ls->ls_uevent_result); + + return ls->ls_uevent_result; +} + +static int dlm_uevent(const struct kobject *kobj, struct kobj_uevent_env *env) +{ + const struct dlm_ls *ls = container_of(kobj, struct dlm_ls, ls_kobj); + + add_uevent_var(env, "LOCKSPACE=%s", ls->ls_name); + return 0; +} + +static const struct kset_uevent_ops dlm_uevent_ops = { + .uevent = dlm_uevent, +}; + +int __init dlm_lockspace_init(void) +{ + ls_count = 0; + mutex_init(&ls_lock); + INIT_LIST_HEAD(&lslist); + spin_lock_init(&lslist_lock); + + dlm_kset = kset_create_and_add("dlm", &dlm_uevent_ops, kernel_kobj); + if (!dlm_kset) { + printk(KERN_WARNING "%s: can not create kset\n", __func__); + return -ENOMEM; + } + return 0; +} + +void dlm_lockspace_exit(void) +{ + kset_unregister(dlm_kset); +} + +struct dlm_ls *dlm_find_lockspace_global(uint32_t id) +{ + struct dlm_ls *ls; + + spin_lock_bh(&lslist_lock); + + list_for_each_entry(ls, &lslist, ls_list) { + if (ls->ls_global_id == id) { + atomic_inc(&ls->ls_count); + goto out; + } + } + ls = NULL; + out: + spin_unlock_bh(&lslist_lock); + return ls; +} + +struct dlm_ls *dlm_find_lockspace_local(dlm_lockspace_t *lockspace) +{ + struct dlm_ls *ls = lockspace; + + atomic_inc(&ls->ls_count); + return ls; +} + +struct dlm_ls *dlm_find_lockspace_device(int minor) +{ + struct dlm_ls *ls; + + spin_lock_bh(&lslist_lock); + list_for_each_entry(ls, &lslist, ls_list) { + if (ls->ls_device.minor == minor) { + atomic_inc(&ls->ls_count); + goto out; + } + } + ls = NULL; + out: + spin_unlock_bh(&lslist_lock); + return ls; +} + +void dlm_put_lockspace(struct dlm_ls *ls) +{ + if (atomic_dec_and_test(&ls->ls_count)) + wake_up(&ls->ls_count_wait); +} + +static void remove_lockspace(struct dlm_ls *ls) +{ +retry: + wait_event(ls->ls_count_wait, atomic_read(&ls->ls_count) == 0); + + spin_lock_bh(&lslist_lock); + if (atomic_read(&ls->ls_count) != 0) { + spin_unlock_bh(&lslist_lock); + goto retry; + } + + WARN_ON(ls->ls_create_count != 0); + list_del(&ls->ls_list); + spin_unlock_bh(&lslist_lock); +} + +static int threads_start(void) +{ + int error; + + /* Thread for sending/receiving messages for all lockspace's */ + error = dlm_midcomms_start(); + if (error) + log_print("cannot start dlm midcomms %d", error); + + return error; +} + +static int lkb_idr_free(struct dlm_lkb *lkb) +{ + if (lkb->lkb_lvbptr && test_bit(DLM_IFL_MSTCPY_BIT, &lkb->lkb_iflags)) + dlm_free_lvb(lkb->lkb_lvbptr); + + dlm_free_lkb(lkb); + return 0; +} + +static void rhash_free_rsb(void *ptr, void *arg) +{ + struct dlm_rsb *rsb = ptr; + + dlm_free_rsb(rsb); +} + +static void free_lockspace(struct work_struct *work) +{ + struct dlm_ls *ls = container_of(work, struct dlm_ls, ls_free_work); + struct dlm_lkb *lkb; + unsigned long id; + + /* + * Free all lkb's in xa + */ + xa_for_each(&ls->ls_lkbxa, id, lkb) { + lkb_idr_free(lkb); + } + xa_destroy(&ls->ls_lkbxa); + + /* + * Free all rsb's on rsbtbl + */ + rhashtable_free_and_destroy(&ls->ls_rsbtbl, rhash_free_rsb, NULL); + + kfree(ls); +} + +static int new_lockspace(const char *name, const char *cluster, + uint32_t flags, int lvblen, + const struct dlm_lockspace_ops *ops, void *ops_arg, + int *ops_result, dlm_lockspace_t **lockspace) +{ + struct dlm_ls *ls; + int namelen = strlen(name); + int error; + + if (namelen > DLM_LOCKSPACE_LEN || namelen == 0) + return -EINVAL; + + if (lvblen % 8) + return -EINVAL; + + if (!try_module_get(THIS_MODULE)) + return -EINVAL; + + if (!dlm_user_daemon_available()) { + log_print("dlm user daemon not available"); + error = -EUNATCH; + goto out; + } + + if (ops && ops_result) { + if (!dlm_config.ci_recover_callbacks) + *ops_result = -EOPNOTSUPP; + else + *ops_result = 0; + } + + if (!cluster) + log_print("dlm cluster name '%s' is being used without an application provided cluster name", + dlm_config.ci_cluster_name); + + if (dlm_config.ci_recover_callbacks && cluster && + strncmp(cluster, dlm_config.ci_cluster_name, DLM_LOCKSPACE_LEN)) { + log_print("dlm cluster name '%s' does not match " + "the application cluster name '%s'", + dlm_config.ci_cluster_name, cluster); + error = -EBADR; + goto out; + } + + error = 0; + + spin_lock_bh(&lslist_lock); + list_for_each_entry(ls, &lslist, ls_list) { + WARN_ON(ls->ls_create_count <= 0); + if (ls->ls_namelen != namelen) + continue; + if (memcmp(ls->ls_name, name, namelen)) + continue; + if (flags & DLM_LSFL_NEWEXCL) { + error = -EEXIST; + break; + } + ls->ls_create_count++; + *lockspace = ls; + error = 1; + break; + } + spin_unlock_bh(&lslist_lock); + + if (error) + goto out; + + error = -ENOMEM; + + ls = kzalloc_obj(*ls, GFP_NOFS); + if (!ls) + goto out; + memcpy(ls->ls_name, name, namelen); + ls->ls_namelen = namelen; + ls->ls_lvblen = lvblen; + atomic_set(&ls->ls_count, 0); + init_waitqueue_head(&ls->ls_count_wait); + ls->ls_flags = 0; + + if (ops && dlm_config.ci_recover_callbacks) { + ls->ls_ops = ops; + ls->ls_ops_arg = ops_arg; + } + + if (flags & DLM_LSFL_SOFTIRQ) + set_bit(LSFL_SOFTIRQ, &ls->ls_flags); + + /* ls_exflags are forced to match among nodes, and we don't + * need to require all nodes to have some flags set + */ + ls->ls_exflags = (flags & ~(DLM_LSFL_FS | DLM_LSFL_NEWEXCL | + DLM_LSFL_SOFTIRQ)); + + INIT_LIST_HEAD(&ls->ls_slow_inactive); + INIT_LIST_HEAD(&ls->ls_slow_active); + rwlock_init(&ls->ls_rsbtbl_lock); + + error = rhashtable_init(&ls->ls_rsbtbl, &dlm_rhash_rsb_params); + if (error) + goto out_lsfree; + + xa_init_flags(&ls->ls_lkbxa, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_BH); + rwlock_init(&ls->ls_lkbxa_lock); + + INIT_LIST_HEAD(&ls->ls_waiters); + spin_lock_init(&ls->ls_waiters_lock); + INIT_LIST_HEAD(&ls->ls_orphans); + spin_lock_init(&ls->ls_orphans_lock); + + INIT_LIST_HEAD(&ls->ls_nodes); + INIT_LIST_HEAD(&ls->ls_nodes_gone); + ls->ls_num_nodes = 0; + ls->ls_low_nodeid = 0; + ls->ls_total_weight = 0; + ls->ls_node_array = NULL; + + memset(&ls->ls_local_rsb, 0, sizeof(struct dlm_rsb)); + ls->ls_local_rsb.res_ls = ls; + + ls->ls_debug_rsb_dentry = NULL; + ls->ls_debug_waiters_dentry = NULL; + + init_waitqueue_head(&ls->ls_uevent_wait); + ls->ls_uevent_result = 0; + init_completion(&ls->ls_recovery_done); + ls->ls_recovery_result = -1; + + spin_lock_init(&ls->ls_cb_lock); + INIT_LIST_HEAD(&ls->ls_cb_delay); + + INIT_WORK(&ls->ls_free_work, free_lockspace); + + ls->ls_recoverd_task = NULL; + mutex_init(&ls->ls_recoverd_active); + spin_lock_init(&ls->ls_recover_lock); + spin_lock_init(&ls->ls_rcom_spin); + get_random_bytes(&ls->ls_rcom_seq, sizeof(uint64_t)); + ls->ls_recover_status = 0; + ls->ls_recover_seq = get_random_u64(); + ls->ls_recover_args = NULL; + init_rwsem(&ls->ls_in_recovery); + rwlock_init(&ls->ls_recv_active); + INIT_LIST_HEAD(&ls->ls_requestqueue); + rwlock_init(&ls->ls_requestqueue_lock); + spin_lock_init(&ls->ls_clear_proc_locks); + + /* Due backwards compatibility with 3.1 we need to use maximum + * possible dlm message size to be sure the message will fit and + * not having out of bounds issues. However on sending side 3.2 + * might send less. + */ + ls->ls_recover_buf = kmalloc(DLM_MAX_SOCKET_BUFSIZE, GFP_NOFS); + if (!ls->ls_recover_buf) { + error = -ENOMEM; + goto out_lkbxa; + } + + ls->ls_slot = 0; + ls->ls_num_slots = 0; + ls->ls_slots_size = 0; + ls->ls_slots = NULL; + + INIT_LIST_HEAD(&ls->ls_recover_list); + spin_lock_init(&ls->ls_recover_list_lock); + xa_init_flags(&ls->ls_recover_xa, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_BH); + spin_lock_init(&ls->ls_recover_xa_lock); + ls->ls_recover_list_count = 0; + init_waitqueue_head(&ls->ls_wait_general); + INIT_LIST_HEAD(&ls->ls_masters_list); + rwlock_init(&ls->ls_masters_lock); + INIT_LIST_HEAD(&ls->ls_dir_dump_list); + rwlock_init(&ls->ls_dir_dump_lock); + + INIT_LIST_HEAD(&ls->ls_scan_list); + spin_lock_init(&ls->ls_scan_lock); + timer_setup(&ls->ls_scan_timer, dlm_rsb_scan, TIMER_DEFERRABLE); + + spin_lock_bh(&lslist_lock); + ls->ls_create_count = 1; + list_add(&ls->ls_list, &lslist); + spin_unlock_bh(&lslist_lock); + + if (flags & DLM_LSFL_FS) + set_bit(LSFL_FS, &ls->ls_flags); + + error = dlm_callback_start(ls); + if (error) { + log_error(ls, "can't start dlm_callback %d", error); + goto out_delist; + } + + init_waitqueue_head(&ls->ls_recover_lock_wait); + + /* + * Once started, dlm_recoverd first looks for ls in lslist, then + * initializes ls_in_recovery as locked in "down" mode. We need + * to wait for the wakeup from dlm_recoverd because in_recovery + * has to start out in down mode. + */ + + error = dlm_recoverd_start(ls); + if (error) { + log_error(ls, "can't start dlm_recoverd %d", error); + goto out_callback; + } + + wait_event(ls->ls_recover_lock_wait, + test_bit(LSFL_RECOVER_LOCK, &ls->ls_flags)); + + ls->ls_kobj.kset = dlm_kset; + error = kobject_init_and_add(&ls->ls_kobj, &dlm_ktype, NULL, + "%s", ls->ls_name); + if (error) + goto out_recoverd; + kobject_uevent(&ls->ls_kobj, KOBJ_ADD); + + /* This uevent triggers dlm_controld in userspace to add us to the + group of nodes that are members of this lockspace (managed by the + cluster infrastructure.) Once it's done that, it tells us who the + current lockspace members are (via configfs) and then tells the + lockspace to start running (via sysfs) in dlm_ls_start(). */ + + error = do_uevent(ls, 1, 0); + if (error < 0) + goto out_recoverd; + + /* wait until recovery is successful or failed */ + wait_for_completion(&ls->ls_recovery_done); + error = ls->ls_recovery_result; + if (error) + goto out_members; + + dlm_create_debug_file(ls); + + log_rinfo(ls, "join complete"); + *lockspace = ls; + return 0; + + out_members: + do_uevent(ls, 0, 0); + dlm_clear_members(ls); + kfree(ls->ls_node_array); + out_recoverd: + dlm_recoverd_stop(ls); + out_callback: + dlm_callback_stop(ls); + out_delist: + spin_lock_bh(&lslist_lock); + list_del(&ls->ls_list); + spin_unlock_bh(&lslist_lock); + xa_destroy(&ls->ls_recover_xa); + kfree(ls->ls_recover_buf); + out_lkbxa: + xa_destroy(&ls->ls_lkbxa); + rhashtable_destroy(&ls->ls_rsbtbl); + out_lsfree: + kobject_put(&ls->ls_kobj); + kfree(ls); + out: + module_put(THIS_MODULE); + return error; +} + +static int __dlm_new_lockspace(const char *name, const char *cluster, + uint32_t flags, int lvblen, + const struct dlm_lockspace_ops *ops, + void *ops_arg, int *ops_result, + dlm_lockspace_t **lockspace) +{ + int error = 0; + + mutex_lock(&ls_lock); + if (!ls_count) + error = threads_start(); + if (error) + goto out; + + error = new_lockspace(name, cluster, flags, lvblen, ops, ops_arg, + ops_result, lockspace); + if (!error) + ls_count++; + if (error > 0) + error = 0; + if (!ls_count) { + dlm_midcomms_shutdown(); + dlm_midcomms_stop(); + } + out: + mutex_unlock(&ls_lock); + return error; +} + +int dlm_new_lockspace(const char *name, const char *cluster, uint32_t flags, + int lvblen, const struct dlm_lockspace_ops *ops, + void *ops_arg, int *ops_result, + dlm_lockspace_t **lockspace) +{ + return __dlm_new_lockspace(name, cluster, flags | DLM_LSFL_FS, lvblen, + ops, ops_arg, ops_result, lockspace); +} + +int dlm_new_user_lockspace(const char *name, const char *cluster, + uint32_t flags, int lvblen, + const struct dlm_lockspace_ops *ops, + void *ops_arg, int *ops_result, + dlm_lockspace_t **lockspace) +{ + if (flags & DLM_LSFL_SOFTIRQ) + return -EINVAL; + + return __dlm_new_lockspace(name, cluster, flags, lvblen, ops, + ops_arg, ops_result, lockspace); +} + +/* NOTE: We check the lkbxa here rather than the resource table. + This is because there may be LKBs queued as ASTs that have been unlinked + from their RSBs and are pending deletion once the AST has been delivered */ + +static int lockspace_busy(struct dlm_ls *ls, unsigned int release_option) +{ + struct dlm_lkb *lkb; + unsigned long id; + int rv = 0; + + read_lock_bh(&ls->ls_lkbxa_lock); + if (release_option == DLM_RELEASE_NO_LOCKS) { + xa_for_each(&ls->ls_lkbxa, id, lkb) { + rv = 1; + break; + } + } else if (release_option == DLM_RELEASE_UNUSED) { + /* TODO: handle this UNUSED option as NO_LOCKS in later patch */ + xa_for_each(&ls->ls_lkbxa, id, lkb) { + if (lkb->lkb_nodeid == 0 && + lkb->lkb_grmode != DLM_LOCK_IV) { + rv = 1; + break; + } + } + } else { + rv = 0; + } + read_unlock_bh(&ls->ls_lkbxa_lock); + return rv; +} + +static int release_lockspace(struct dlm_ls *ls, unsigned int release_option) +{ + int busy, rv; + + busy = lockspace_busy(ls, release_option); + + spin_lock_bh(&lslist_lock); + if (ls->ls_create_count == 1) { + if (busy) { + rv = -EBUSY; + } else { + /* remove_lockspace takes ls off lslist */ + ls->ls_create_count = 0; + rv = 0; + } + } else if (ls->ls_create_count > 1) { + rv = --ls->ls_create_count; + } else { + rv = -EINVAL; + } + spin_unlock_bh(&lslist_lock); + + if (rv) { + log_debug(ls, "release_lockspace no remove %d", rv); + return rv; + } + + if (ls_count == 1) + dlm_midcomms_version_wait(); + + dlm_device_deregister(ls); + + if (release_option != DLM_RELEASE_NO_EVENT && + dlm_user_daemon_available()) + do_uevent(ls, 0, (release_option == DLM_RELEASE_RECOVER)); + + dlm_recoverd_stop(ls); + + /* clear the LSFL_RUNNING flag to fast up + * time_shutdown_sync(), we don't care anymore + */ + clear_bit(LSFL_RUNNING, &ls->ls_flags); + timer_shutdown_sync(&ls->ls_scan_timer); + + if (ls_count == 1) { + dlm_clear_members(ls); + dlm_midcomms_shutdown(); + } + + dlm_callback_stop(ls); + + remove_lockspace(ls); + + dlm_delete_debug_file(ls); + + kobject_put(&ls->ls_kobj); + + xa_destroy(&ls->ls_recover_xa); + kfree(ls->ls_recover_buf); + + /* + * Free structures on any other lists + */ + + dlm_purge_requestqueue(ls); + kfree(ls->ls_recover_args); + dlm_clear_members(ls); + dlm_clear_members_gone(ls); + kfree(ls->ls_node_array); + + log_rinfo(ls, "%s final free", __func__); + + /* delayed free of data structures see free_lockspace() */ + queue_work(dlm_wq, &ls->ls_free_work); + module_put(THIS_MODULE); + return 0; +} + +/* + * Called when a system has released all its locks and is not going to use the + * lockspace any longer. We free everything we're managing for this lockspace. + * Remaining nodes will go through the recovery process as if we'd died. The + * lockspace must continue to function as usual, participating in recoveries, + * until this returns. + * + * See DLM_RELEASE defines for release_option values and their meaning. + */ + +int dlm_release_lockspace(void *lockspace, unsigned int release_option) +{ + struct dlm_ls *ls; + int error; + + if (release_option > __DLM_RELEASE_MAX) + return -EINVAL; + + ls = dlm_find_lockspace_local(lockspace); + if (!ls) + return -EINVAL; + dlm_put_lockspace(ls); + + mutex_lock(&ls_lock); + error = release_lockspace(ls, release_option); + if (!error) + ls_count--; + if (!ls_count) + dlm_midcomms_stop(); + mutex_unlock(&ls_lock); + + return error; +} + +void dlm_stop_lockspaces(void) +{ + struct dlm_ls *ls; + int count; + + restart: + count = 0; + spin_lock_bh(&lslist_lock); + list_for_each_entry(ls, &lslist, ls_list) { + if (!test_bit(LSFL_RUNNING, &ls->ls_flags)) { + count++; + continue; + } + spin_unlock_bh(&lslist_lock); + log_error(ls, "no userland control daemon, stopping lockspace"); + dlm_ls_stop(ls); + goto restart; + } + spin_unlock_bh(&lslist_lock); + + if (count) + log_print("dlm user daemon left %d lockspaces", count); +} diff --git a/fs/dlm/lockspace.h b/fs/dlm/lockspace.h new file mode 100644 index 000000000..47ebd4411 --- /dev/null +++ b/fs/dlm/lockspace.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#ifndef __LOCKSPACE_DOT_H__ +#define __LOCKSPACE_DOT_H__ + +/* DLM_LSFL_FS + * The lockspace user is in the kernel (i.e. filesystem). Enables + * direct bast/cast callbacks. + * + * internal lockspace flag - will be removed in future + */ +#define DLM_LSFL_FS 0x00000004 + +int dlm_lockspace_init(void); +void dlm_lockspace_exit(void); +struct dlm_ls *dlm_find_lockspace_global(uint32_t id); +struct dlm_ls *dlm_find_lockspace_local(void *id); +struct dlm_ls *dlm_find_lockspace_device(int minor); +void dlm_put_lockspace(struct dlm_ls *ls); +void dlm_stop_lockspaces(void); +int dlm_new_user_lockspace(const char *name, const char *cluster, + uint32_t flags, int lvblen, + const struct dlm_lockspace_ops *ops, + void *ops_arg, int *ops_result, + dlm_lockspace_t **lockspace); + +#endif /* __LOCKSPACE_DOT_H__ */ + diff --git a/fs/dlm/lowcomms.c b/fs/dlm/lowcomms.c new file mode 100644 index 000000000..2aff1c7c1 --- /dev/null +++ b/fs/dlm/lowcomms.c @@ -0,0 +1,1987 @@ +// SPDX-License-Identifier: GPL-2.0-only +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2009 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +/* + * lowcomms.c + * + * This is the "low-level" comms layer. + * + * It is responsible for sending/receiving messages + * from other nodes in the cluster. + * + * Cluster nodes are referred to by their nodeids. nodeids are + * simply 32 bit numbers to the locking module - if they need to + * be expanded for the cluster infrastructure then that is its + * responsibility. It is this layer's + * responsibility to resolve these into IP address or + * whatever it needs for inter-node communication. + * + * The comms level is two kernel threads that deal mainly with + * the receiving of messages from other nodes and passing them + * up to the mid-level comms layer (which understands the + * message format) for execution by the locking core, and + * a send thread which does all the setting up of connections + * to remote nodes and the sending of data. Threads are not allowed + * to send their own data because it may cause them to wait in times + * of high load. Also, this way, the sending thread can collect together + * messages bound for one node and send them in one block. + * + * lowcomms will choose to use either TCP or SCTP as its transport layer + * depending on the configuration variable 'protocol'. This should be set + * to 0 (default) for TCP or 1 for SCTP. It should be configured using a + * cluster-wide mechanism as it must be the same on all nodes of the cluster + * for the DLM to function. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "dlm_internal.h" +#include "lowcomms.h" +#include "midcomms.h" +#include "memory.h" +#include "config.h" + +#define DLM_SHUTDOWN_WAIT_TIMEOUT msecs_to_jiffies(5000) +#define DLM_MAX_PROCESS_BUFFERS 24 +#define NEEDED_RMEM (4*1024*1024) + +struct connection { + struct socket *sock; /* NULL if not connected */ + uint32_t nodeid; /* So we know who we are in the list */ + /* this semaphore is used to allow parallel recv/send in read + * lock mode. When we release a sock we need to held the write lock. + * + * However this is locking code and not nice. When we remove the + * othercon handling we can look into other mechanism to synchronize + * io handling to call sock_release() at the right time. + */ + struct rw_semaphore sock_lock; + unsigned long flags; +#define CF_APP_LIMITED 0 +#define CF_RECV_PENDING 1 +#define CF_SEND_PENDING 2 +#define CF_RECV_INTR 3 +#define CF_IO_STOP 4 +#define CF_IS_OTHERCON 5 + struct list_head writequeue; /* List of outgoing writequeue_entries */ + spinlock_t writequeue_lock; + int retries; + struct hlist_node list; + /* due some connect()/accept() races we currently have this cross over + * connection attempt second connection for one node. + * + * There is a solution to avoid the race by introducing a connect + * rule as e.g. our_nodeid > nodeid_to_connect who is allowed to + * connect. Otherside can connect but will only be considered that + * the other side wants to have a reconnect. + * + * However changing to this behaviour will break backwards compatible. + * In a DLM protocol major version upgrade we should remove this! + */ + struct connection *othercon; + struct work_struct rwork; /* receive worker */ + struct work_struct swork; /* send worker */ + wait_queue_head_t shutdown_wait; + unsigned char rx_leftover_buf[DLM_MAX_SOCKET_BUFSIZE]; + int rx_leftover; + int mark; + int addr_count; + int curr_addr_index; + struct sockaddr_storage addr[DLM_MAX_ADDR_COUNT]; + spinlock_t addrs_lock; + struct rcu_head rcu; +}; +#define sock2con(x) ((struct connection *)(x)->sk_user_data) + +struct listen_connection { + struct socket *sock; + struct work_struct rwork; +}; + +#define DLM_WQ_REMAIN_BYTES(e) (PAGE_SIZE - e->end) +#define DLM_WQ_LENGTH_BYTES(e) (e->end - e->offset) + +/* An entry waiting to be sent */ +struct writequeue_entry { + struct list_head list; + struct page *page; + int offset; + int len; + int end; + int users; + bool dirty; + struct connection *con; + struct list_head msgs; + struct kref ref; +}; + +struct dlm_msg { + struct writequeue_entry *entry; + struct dlm_msg *orig_msg; + bool retransmit; + void *ppc; + int len; + int idx; /* new()/commit() idx exchange */ + + struct list_head list; + struct kref ref; +}; + +struct processqueue_entry { + unsigned char *buf; + int nodeid; + int buflen; + + struct list_head list; +}; + +struct dlm_proto_ops { + bool try_new_addr; + const char *name; + int proto; + int how; + + void (*sockopts)(struct socket *sock); + int (*bind)(struct socket *sock); + int (*listen_validate)(void); + void (*listen_sockopts)(struct socket *sock); + int (*listen_bind)(struct socket *sock); +}; + +static struct listen_sock_callbacks { + void (*sk_error_report)(struct sock *); + void (*sk_data_ready)(struct sock *); + void (*sk_state_change)(struct sock *); + void (*sk_write_space)(struct sock *); +} listen_sock; + +static struct listen_connection listen_con; +static struct sockaddr_storage dlm_local_addr[DLM_MAX_ADDR_COUNT]; +static int dlm_local_count; + +/* Work queues */ +static struct workqueue_struct *io_workqueue; +static struct workqueue_struct *process_workqueue; + +static struct hlist_head connection_hash[CONN_HASH_SIZE]; +static DEFINE_SPINLOCK(connections_lock); +DEFINE_STATIC_SRCU(connections_srcu); + +static const struct dlm_proto_ops *dlm_proto_ops; + +#define DLM_IO_SUCCESS 0 +#define DLM_IO_END 1 +#define DLM_IO_EOF 2 +#define DLM_IO_RESCHED 3 +#define DLM_IO_FLUSH 4 + +static void process_recv_sockets(struct work_struct *work); +static void process_send_sockets(struct work_struct *work); +static void process_dlm_messages(struct work_struct *work); + +static DECLARE_WORK(process_work, process_dlm_messages); +static DEFINE_SPINLOCK(processqueue_lock); +static bool process_dlm_messages_pending; +static DECLARE_WAIT_QUEUE_HEAD(processqueue_wq); +static atomic_t processqueue_count; +static LIST_HEAD(processqueue); + +bool dlm_lowcomms_is_running(void) +{ + return !!listen_con.sock; +} + +static void lowcomms_queue_swork(struct connection *con) +{ + assert_spin_locked(&con->writequeue_lock); + + if (!test_bit(CF_IO_STOP, &con->flags) && + !test_bit(CF_APP_LIMITED, &con->flags) && + !test_and_set_bit(CF_SEND_PENDING, &con->flags)) + queue_work(io_workqueue, &con->swork); +} + +static void lowcomms_queue_rwork(struct connection *con) +{ +#ifdef CONFIG_LOCKDEP + WARN_ON_ONCE(!lockdep_sock_is_held(con->sock->sk)); +#endif + + if (!test_bit(CF_IO_STOP, &con->flags) && + !test_and_set_bit(CF_RECV_PENDING, &con->flags)) + queue_work(io_workqueue, &con->rwork); +} + +static void writequeue_entry_ctor(void *data) +{ + struct writequeue_entry *entry = data; + + INIT_LIST_HEAD(&entry->msgs); +} + +struct kmem_cache *dlm_lowcomms_writequeue_cache_create(void) +{ + return kmem_cache_create("dlm_writequeue", sizeof(struct writequeue_entry), + 0, 0, writequeue_entry_ctor); +} + +struct kmem_cache *dlm_lowcomms_msg_cache_create(void) +{ + return KMEM_CACHE(dlm_msg, 0); +} + +/* need to held writequeue_lock */ +static struct writequeue_entry *con_next_wq(struct connection *con) +{ + struct writequeue_entry *e; + + e = list_first_entry_or_null(&con->writequeue, struct writequeue_entry, + list); + /* if len is zero nothing is to send, if there are users filling + * buffers we wait until the users are done so we can send more. + */ + if (!e || e->users || e->len == 0) + return NULL; + + return e; +} + +static struct connection *__find_con(int nodeid, int r) +{ + struct connection *con; + + hlist_for_each_entry_srcu(con, &connection_hash[r], list, + srcu_read_lock_held(&connections_srcu)) { + if (con->nodeid == nodeid) + return con; + } + + return NULL; +} + +static void dlm_con_init(struct connection *con, int nodeid) +{ + con->nodeid = nodeid; + init_rwsem(&con->sock_lock); + INIT_LIST_HEAD(&con->writequeue); + spin_lock_init(&con->writequeue_lock); + INIT_WORK(&con->swork, process_send_sockets); + INIT_WORK(&con->rwork, process_recv_sockets); + spin_lock_init(&con->addrs_lock); + init_waitqueue_head(&con->shutdown_wait); +} + +/* + * If 'allocation' is zero then we don't attempt to create a new + * connection structure for this node. + */ +static struct connection *nodeid2con(int nodeid, gfp_t alloc) +{ + struct connection *con, *tmp; + int r; + + r = nodeid_hash(nodeid); + con = __find_con(nodeid, r); + if (con || !alloc) + return con; + + con = kzalloc_obj(*con, alloc); + if (!con) + return NULL; + + dlm_con_init(con, nodeid); + + spin_lock(&connections_lock); + /* Because multiple workqueues/threads calls this function it can + * race on multiple cpu's. Instead of locking hot path __find_con() + * we just check in rare cases of recently added nodes again + * under protection of connections_lock. If this is the case we + * abort our connection creation and return the existing connection. + */ + tmp = __find_con(nodeid, r); + if (tmp) { + spin_unlock(&connections_lock); + kfree(con); + return tmp; + } + + hlist_add_head_rcu(&con->list, &connection_hash[r]); + spin_unlock(&connections_lock); + + return con; +} + +static int addr_compare(const struct sockaddr_storage *x, + const struct sockaddr_storage *y) +{ + switch (x->ss_family) { + case AF_INET: { + struct sockaddr_in *sinx = (struct sockaddr_in *)x; + struct sockaddr_in *siny = (struct sockaddr_in *)y; + if (sinx->sin_addr.s_addr != siny->sin_addr.s_addr) + return 0; + if (sinx->sin_port != siny->sin_port) + return 0; + break; + } + case AF_INET6: { + struct sockaddr_in6 *sinx = (struct sockaddr_in6 *)x; + struct sockaddr_in6 *siny = (struct sockaddr_in6 *)y; + if (!ipv6_addr_equal(&sinx->sin6_addr, &siny->sin6_addr)) + return 0; + if (sinx->sin6_port != siny->sin6_port) + return 0; + break; + } + default: + return 0; + } + return 1; +} + +static int nodeid_to_addr(int nodeid, struct sockaddr_storage *sas_out, + struct sockaddr *sa_out, bool try_new_addr, + unsigned int *mark) +{ + struct sockaddr_storage sas; + struct connection *con; + int idx; + + if (!dlm_local_count) + return -1; + + idx = srcu_read_lock(&connections_srcu); + con = nodeid2con(nodeid, 0); + if (!con) { + srcu_read_unlock(&connections_srcu, idx); + return -ENOENT; + } + + spin_lock(&con->addrs_lock); + if (!con->addr_count) { + spin_unlock(&con->addrs_lock); + srcu_read_unlock(&connections_srcu, idx); + return -ENOENT; + } + + memcpy(&sas, &con->addr[con->curr_addr_index], + sizeof(struct sockaddr_storage)); + + if (try_new_addr) { + con->curr_addr_index++; + if (con->curr_addr_index == con->addr_count) + con->curr_addr_index = 0; + } + + *mark = con->mark; + spin_unlock(&con->addrs_lock); + + if (sas_out) + memcpy(sas_out, &sas, sizeof(struct sockaddr_storage)); + + if (!sa_out) { + srcu_read_unlock(&connections_srcu, idx); + return 0; + } + + if (dlm_local_addr[0].ss_family == AF_INET) { + struct sockaddr_in *in4 = (struct sockaddr_in *) &sas; + struct sockaddr_in *ret4 = (struct sockaddr_in *) sa_out; + ret4->sin_addr.s_addr = in4->sin_addr.s_addr; + } else { + struct sockaddr_in6 *in6 = (struct sockaddr_in6 *) &sas; + struct sockaddr_in6 *ret6 = (struct sockaddr_in6 *) sa_out; + ret6->sin6_addr = in6->sin6_addr; + } + + srcu_read_unlock(&connections_srcu, idx); + return 0; +} + +static int addr_to_nodeid(struct sockaddr_storage *addr, int *nodeid, + unsigned int *mark) +{ + struct connection *con; + int i, idx, addr_i; + + idx = srcu_read_lock(&connections_srcu); + for (i = 0; i < CONN_HASH_SIZE; i++) { + hlist_for_each_entry_srcu(con, &connection_hash[i], list, + srcu_read_lock_held(&connections_srcu)) { + WARN_ON_ONCE(!con->addr_count); + + spin_lock(&con->addrs_lock); + for (addr_i = 0; addr_i < con->addr_count; addr_i++) { + if (addr_compare(&con->addr[addr_i], addr)) { + *nodeid = con->nodeid; + *mark = con->mark; + spin_unlock(&con->addrs_lock); + srcu_read_unlock(&connections_srcu, idx); + return 0; + } + } + spin_unlock(&con->addrs_lock); + } + } + srcu_read_unlock(&connections_srcu, idx); + + return -ENOENT; +} + +static bool dlm_lowcomms_con_has_addr(const struct connection *con, + const struct sockaddr_storage *addr) +{ + int i; + + for (i = 0; i < con->addr_count; i++) { + if (addr_compare(&con->addr[i], addr)) + return true; + } + + return false; +} + +int dlm_lowcomms_addr(int nodeid, struct sockaddr_storage *addr) +{ + struct connection *con; + bool ret; + int idx; + + idx = srcu_read_lock(&connections_srcu); + con = nodeid2con(nodeid, GFP_NOFS); + if (!con) { + srcu_read_unlock(&connections_srcu, idx); + return -ENOMEM; + } + + spin_lock(&con->addrs_lock); + if (!con->addr_count) { + memcpy(&con->addr[0], addr, sizeof(*addr)); + con->addr_count = 1; + con->mark = dlm_config.ci_mark; + spin_unlock(&con->addrs_lock); + srcu_read_unlock(&connections_srcu, idx); + return 0; + } + + ret = dlm_lowcomms_con_has_addr(con, addr); + if (ret) { + spin_unlock(&con->addrs_lock); + srcu_read_unlock(&connections_srcu, idx); + return -EEXIST; + } + + if (con->addr_count >= DLM_MAX_ADDR_COUNT) { + spin_unlock(&con->addrs_lock); + srcu_read_unlock(&connections_srcu, idx); + return -ENOSPC; + } + + memcpy(&con->addr[con->addr_count++], addr, sizeof(*addr)); + srcu_read_unlock(&connections_srcu, idx); + spin_unlock(&con->addrs_lock); + return 0; +} + +/* Data available on socket or listen socket received a connect */ +static void lowcomms_data_ready(struct sock *sk) +{ + struct connection *con = sock2con(sk); + + trace_sk_data_ready(sk); + + set_bit(CF_RECV_INTR, &con->flags); + lowcomms_queue_rwork(con); +} + +static void lowcomms_write_space(struct sock *sk) +{ + struct connection *con = sock2con(sk); + + clear_bit(SOCK_NOSPACE, &con->sock->flags); + + spin_lock_bh(&con->writequeue_lock); + if (test_and_clear_bit(CF_APP_LIMITED, &con->flags)) { + con->sock->sk->sk_write_pending--; + clear_bit(SOCKWQ_ASYNC_NOSPACE, &con->sock->flags); + } + + lowcomms_queue_swork(con); + spin_unlock_bh(&con->writequeue_lock); +} + +static void lowcomms_state_change(struct sock *sk) +{ + /* SCTP layer is not calling sk_data_ready when the connection + * is done, so we catch the signal through here. + */ + if (sk->sk_shutdown & RCV_SHUTDOWN) + lowcomms_data_ready(sk); +} + +static void lowcomms_listen_data_ready(struct sock *sk) +{ + trace_sk_data_ready(sk); + + queue_work(io_workqueue, &listen_con.rwork); +} + +int dlm_lowcomms_connect_node(int nodeid) +{ + struct connection *con; + int idx; + + idx = srcu_read_lock(&connections_srcu); + con = nodeid2con(nodeid, 0); + if (WARN_ON_ONCE(!con)) { + srcu_read_unlock(&connections_srcu, idx); + return -ENOENT; + } + + down_read(&con->sock_lock); + if (!con->sock) { + spin_lock_bh(&con->writequeue_lock); + lowcomms_queue_swork(con); + spin_unlock_bh(&con->writequeue_lock); + } + up_read(&con->sock_lock); + srcu_read_unlock(&connections_srcu, idx); + + cond_resched(); + return 0; +} + +int dlm_lowcomms_nodes_set_mark(int nodeid, unsigned int mark) +{ + struct connection *con; + int idx; + + idx = srcu_read_lock(&connections_srcu); + con = nodeid2con(nodeid, 0); + if (!con) { + srcu_read_unlock(&connections_srcu, idx); + return -ENOENT; + } + + spin_lock(&con->addrs_lock); + con->mark = mark; + spin_unlock(&con->addrs_lock); + srcu_read_unlock(&connections_srcu, idx); + return 0; +} + +static void lowcomms_error_report(struct sock *sk) +{ + struct connection *con = sock2con(sk); + struct inet_sock *inet; + + inet = inet_sk(sk); + switch (sk->sk_family) { + case AF_INET: + printk_ratelimited(KERN_ERR "dlm: node %d: socket error " + "sending to node %d at %pI4, dport %d, " + "sk_err=%d/%d\n", dlm_our_nodeid(), + con->nodeid, &inet->inet_daddr, + ntohs(inet->inet_dport), sk->sk_err, + READ_ONCE(sk->sk_err_soft)); + break; +#if IS_ENABLED(CONFIG_IPV6) + case AF_INET6: + printk_ratelimited(KERN_ERR "dlm: node %d: socket error " + "sending to node %d at %pI6c, " + "dport %d, sk_err=%d/%d\n", dlm_our_nodeid(), + con->nodeid, &sk->sk_v6_daddr, + ntohs(inet->inet_dport), sk->sk_err, + READ_ONCE(sk->sk_err_soft)); + break; +#endif + default: + printk_ratelimited(KERN_ERR "dlm: node %d: socket error " + "invalid socket family %d set, " + "sk_err=%d/%d\n", dlm_our_nodeid(), + sk->sk_family, sk->sk_err, + READ_ONCE(sk->sk_err_soft)); + break; + } + + dlm_midcomms_unack_msg_resend(con->nodeid); + + listen_sock.sk_error_report(sk); +} + +static void restore_callbacks(struct sock *sk) +{ +#ifdef CONFIG_LOCKDEP + WARN_ON_ONCE(!lockdep_sock_is_held(sk)); +#endif + + sk->sk_user_data = NULL; + sk->sk_data_ready = listen_sock.sk_data_ready; + sk->sk_state_change = listen_sock.sk_state_change; + sk->sk_write_space = listen_sock.sk_write_space; + sk->sk_error_report = listen_sock.sk_error_report; +} + +/* Make a socket active */ +static void add_sock(struct socket *sock, struct connection *con) +{ + struct sock *sk = sock->sk; + + lock_sock(sk); + con->sock = sock; + + sk->sk_user_data = con; + sk->sk_data_ready = lowcomms_data_ready; + sk->sk_write_space = lowcomms_write_space; + if (dlm_config.ci_protocol == DLM_PROTO_SCTP) + sk->sk_state_change = lowcomms_state_change; + sk->sk_allocation = GFP_NOFS; + sk->sk_use_task_frag = false; + sk->sk_error_report = lowcomms_error_report; + release_sock(sk); +} + +/* Add the port number to an IPv6 or 4 sockaddr and return the address + length */ +static void make_sockaddr(struct sockaddr_storage *saddr, __be16 port, + int *addr_len) +{ + saddr->ss_family = dlm_local_addr[0].ss_family; + if (saddr->ss_family == AF_INET) { + struct sockaddr_in *in4_addr = (struct sockaddr_in *)saddr; + in4_addr->sin_port = port; + *addr_len = sizeof(struct sockaddr_in); + memset(&in4_addr->sin_zero, 0, sizeof(in4_addr->sin_zero)); + } else { + struct sockaddr_in6 *in6_addr = (struct sockaddr_in6 *)saddr; + in6_addr->sin6_port = port; + *addr_len = sizeof(struct sockaddr_in6); + } + memset((char *)saddr + *addr_len, 0, sizeof(struct sockaddr_storage) - *addr_len); +} + +static void dlm_page_release(struct kref *kref) +{ + struct writequeue_entry *e = container_of(kref, struct writequeue_entry, + ref); + + __free_page(e->page); + dlm_free_writequeue(e); +} + +static void dlm_msg_release(struct kref *kref) +{ + struct dlm_msg *msg = container_of(kref, struct dlm_msg, ref); + + kref_put(&msg->entry->ref, dlm_page_release); + dlm_free_msg(msg); +} + +static void free_entry(struct writequeue_entry *e) +{ + struct dlm_msg *msg, *tmp; + + list_for_each_entry_safe(msg, tmp, &e->msgs, list) { + if (msg->orig_msg) { + msg->orig_msg->retransmit = false; + kref_put(&msg->orig_msg->ref, dlm_msg_release); + } + + list_del(&msg->list); + kref_put(&msg->ref, dlm_msg_release); + } + + list_del(&e->list); + kref_put(&e->ref, dlm_page_release); +} + +static void dlm_close_sock(struct socket **sock) +{ + lock_sock((*sock)->sk); + restore_callbacks((*sock)->sk); + release_sock((*sock)->sk); + + sock_release(*sock); + *sock = NULL; +} + +static void allow_connection_io(struct connection *con) +{ + if (con->othercon) + clear_bit(CF_IO_STOP, &con->othercon->flags); + clear_bit(CF_IO_STOP, &con->flags); +} + +static void stop_connection_io(struct connection *con) +{ + if (con->othercon) + stop_connection_io(con->othercon); + + spin_lock_bh(&con->writequeue_lock); + set_bit(CF_IO_STOP, &con->flags); + spin_unlock_bh(&con->writequeue_lock); + + down_write(&con->sock_lock); + if (con->sock) { + lock_sock(con->sock->sk); + restore_callbacks(con->sock->sk); + release_sock(con->sock->sk); + } + up_write(&con->sock_lock); + + cancel_work_sync(&con->swork); + cancel_work_sync(&con->rwork); +} + +/* Close a remote connection and tidy up */ +static void close_connection(struct connection *con, bool and_other) +{ + struct writequeue_entry *e; + + if (con->othercon && and_other) + close_connection(con->othercon, false); + + down_write(&con->sock_lock); + if (!con->sock) { + up_write(&con->sock_lock); + return; + } + + dlm_close_sock(&con->sock); + + /* if we send a writequeue entry only a half way, we drop the + * whole entry because reconnection and that we not start of the + * middle of a msg which will confuse the other end. + * + * we can always drop messages because retransmits, but what we + * cannot allow is to transmit half messages which may be processed + * at the other side. + * + * our policy is to start on a clean state when disconnects, we don't + * know what's send/received on transport layer in this case. + */ + spin_lock_bh(&con->writequeue_lock); + if (!list_empty(&con->writequeue)) { + e = list_first_entry(&con->writequeue, struct writequeue_entry, + list); + if (e->dirty) + free_entry(e); + } + spin_unlock_bh(&con->writequeue_lock); + + con->rx_leftover = 0; + con->retries = 0; + clear_bit(CF_APP_LIMITED, &con->flags); + clear_bit(CF_RECV_PENDING, &con->flags); + clear_bit(CF_SEND_PENDING, &con->flags); + up_write(&con->sock_lock); +} + +static void shutdown_connection(struct connection *con, bool and_other) +{ + int ret; + + if (con->othercon && and_other) + shutdown_connection(con->othercon, false); + + flush_workqueue(io_workqueue); + down_read(&con->sock_lock); + /* nothing to shutdown */ + if (!con->sock) { + up_read(&con->sock_lock); + return; + } + + ret = kernel_sock_shutdown(con->sock, dlm_proto_ops->how); + up_read(&con->sock_lock); + if (ret) { + log_print("Connection %p failed to shutdown: %d will force close", + con, ret); + goto force_close; + } else { + ret = wait_event_timeout(con->shutdown_wait, !con->sock, + DLM_SHUTDOWN_WAIT_TIMEOUT); + if (ret == 0) { + log_print("Connection %p shutdown timed out, will force close", + con); + goto force_close; + } + } + + return; + +force_close: + close_connection(con, false); +} + +static struct processqueue_entry *new_processqueue_entry(int nodeid, + int buflen) +{ + struct processqueue_entry *pentry; + + pentry = kmalloc_obj(*pentry, GFP_NOFS); + if (!pentry) + return NULL; + + pentry->buf = kmalloc(buflen, GFP_NOFS); + if (!pentry->buf) { + kfree(pentry); + return NULL; + } + + pentry->nodeid = nodeid; + return pentry; +} + +static void free_processqueue_entry(struct processqueue_entry *pentry) +{ + kfree(pentry->buf); + kfree(pentry); +} + +static void process_dlm_messages(struct work_struct *work) +{ + struct processqueue_entry *pentry; + + spin_lock_bh(&processqueue_lock); + pentry = list_first_entry_or_null(&processqueue, + struct processqueue_entry, list); + if (WARN_ON_ONCE(!pentry)) { + process_dlm_messages_pending = false; + spin_unlock_bh(&processqueue_lock); + return; + } + + list_del(&pentry->list); + if (atomic_dec_and_test(&processqueue_count)) + wake_up(&processqueue_wq); + spin_unlock_bh(&processqueue_lock); + + for (;;) { + dlm_process_incoming_buffer(pentry->nodeid, pentry->buf, + pentry->buflen); + free_processqueue_entry(pentry); + + spin_lock_bh(&processqueue_lock); + pentry = list_first_entry_or_null(&processqueue, + struct processqueue_entry, list); + if (!pentry) { + process_dlm_messages_pending = false; + spin_unlock_bh(&processqueue_lock); + break; + } + + list_del(&pentry->list); + if (atomic_dec_and_test(&processqueue_count)) + wake_up(&processqueue_wq); + spin_unlock_bh(&processqueue_lock); + } +} + +/* Data received from remote end */ +static int receive_from_sock(struct connection *con, int buflen) +{ + struct processqueue_entry *pentry; + int ret, buflen_real; + struct msghdr msg; + struct kvec iov; + + pentry = new_processqueue_entry(con->nodeid, buflen); + if (!pentry) + return DLM_IO_RESCHED; + + memcpy(pentry->buf, con->rx_leftover_buf, con->rx_leftover); + + /* calculate new buffer parameter regarding last receive and + * possible leftover bytes + */ + iov.iov_base = pentry->buf + con->rx_leftover; + iov.iov_len = buflen - con->rx_leftover; + + memset(&msg, 0, sizeof(msg)); + msg.msg_flags = MSG_DONTWAIT | MSG_NOSIGNAL; + clear_bit(CF_RECV_INTR, &con->flags); +again: + ret = kernel_recvmsg(con->sock, &msg, &iov, 1, iov.iov_len, + msg.msg_flags); + trace_dlm_recv(con->nodeid, ret); + if (ret == -EAGAIN) { + lock_sock(con->sock->sk); + if (test_and_clear_bit(CF_RECV_INTR, &con->flags)) { + release_sock(con->sock->sk); + goto again; + } + + clear_bit(CF_RECV_PENDING, &con->flags); + release_sock(con->sock->sk); + free_processqueue_entry(pentry); + return DLM_IO_END; + } else if (ret == 0) { + /* close will clear CF_RECV_PENDING */ + free_processqueue_entry(pentry); + return DLM_IO_EOF; + } else if (ret < 0) { + free_processqueue_entry(pentry); + return ret; + } + + /* new buflen according readed bytes and leftover from last receive */ + buflen_real = ret + con->rx_leftover; + ret = dlm_validate_incoming_buffer(con->nodeid, pentry->buf, + buflen_real); + if (ret < 0) { + free_processqueue_entry(pentry); + return ret; + } + + pentry->buflen = ret; + + /* calculate leftover bytes from process and put it into begin of + * the receive buffer, so next receive we have the full message + * at the start address of the receive buffer. + */ + con->rx_leftover = buflen_real - ret; + memmove(con->rx_leftover_buf, pentry->buf + ret, + con->rx_leftover); + + spin_lock_bh(&processqueue_lock); + ret = atomic_inc_return(&processqueue_count); + list_add_tail(&pentry->list, &processqueue); + if (!process_dlm_messages_pending) { + process_dlm_messages_pending = true; + queue_work(process_workqueue, &process_work); + } + spin_unlock_bh(&processqueue_lock); + + if (ret > DLM_MAX_PROCESS_BUFFERS) + return DLM_IO_FLUSH; + + return DLM_IO_SUCCESS; +} + +/* Listening socket is busy, accept a connection */ +static int accept_from_sock(void) +{ + struct sockaddr_storage peeraddr; + int len, idx, result, nodeid; + struct connection *newcon; + struct socket *newsock; + unsigned int mark; + + result = kernel_accept(listen_con.sock, &newsock, O_NONBLOCK); + if (result == -EAGAIN) + return DLM_IO_END; + else if (result < 0) + goto accept_err; + + /* Get the connected socket's peer */ + memset(&peeraddr, 0, sizeof(peeraddr)); + len = newsock->ops->getname(newsock, (struct sockaddr *)&peeraddr, 2); + if (len < 0) { + result = -ECONNABORTED; + goto accept_err; + } + + /* Get the new node's NODEID */ + make_sockaddr(&peeraddr, 0, &len); + if (addr_to_nodeid(&peeraddr, &nodeid, &mark)) { + switch (peeraddr.ss_family) { + case AF_INET: { + struct sockaddr_in *sin = (struct sockaddr_in *)&peeraddr; + + log_print("connect from non cluster IPv4 node %pI4", + &sin->sin_addr); + break; + } +#if IS_ENABLED(CONFIG_IPV6) + case AF_INET6: { + struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&peeraddr; + + log_print("connect from non cluster IPv6 node %pI6c", + &sin6->sin6_addr); + break; + } +#endif + default: + log_print("invalid family from non cluster node"); + break; + } + + sock_release(newsock); + return -1; + } + + log_print("got connection from %d", nodeid); + + /* Check to see if we already have a connection to this node. This + * could happen if the two nodes initiate a connection at roughly + * the same time and the connections cross on the wire. + * In this case we store the incoming one in "othercon" + */ + idx = srcu_read_lock(&connections_srcu); + newcon = nodeid2con(nodeid, 0); + if (WARN_ON_ONCE(!newcon)) { + srcu_read_unlock(&connections_srcu, idx); + result = -ENOENT; + goto accept_err; + } + + sock_set_mark(newsock->sk, mark); + + down_write(&newcon->sock_lock); + if (newcon->sock) { + struct connection *othercon = newcon->othercon; + + if (!othercon) { + othercon = kzalloc_obj(*othercon, GFP_NOFS); + if (!othercon) { + log_print("failed to allocate incoming socket"); + up_write(&newcon->sock_lock); + srcu_read_unlock(&connections_srcu, idx); + result = -ENOMEM; + goto accept_err; + } + + dlm_con_init(othercon, nodeid); + lockdep_set_subclass(&othercon->sock_lock, 1); + newcon->othercon = othercon; + set_bit(CF_IS_OTHERCON, &othercon->flags); + } else { + /* close other sock con if we have something new */ + close_connection(othercon, false); + } + + down_write(&othercon->sock_lock); + add_sock(newsock, othercon); + + /* check if we receved something while adding */ + lock_sock(othercon->sock->sk); + lowcomms_queue_rwork(othercon); + release_sock(othercon->sock->sk); + up_write(&othercon->sock_lock); + } + else { + /* accept copies the sk after we've saved the callbacks, so we + don't want to save them a second time or comm errors will + result in calling sk_error_report recursively. */ + add_sock(newsock, newcon); + + /* check if we receved something while adding */ + lock_sock(newcon->sock->sk); + lowcomms_queue_rwork(newcon); + release_sock(newcon->sock->sk); + } + up_write(&newcon->sock_lock); + srcu_read_unlock(&connections_srcu, idx); + + return DLM_IO_SUCCESS; + +accept_err: + if (newsock) + sock_release(newsock); + + return result; +} + +/* + * writequeue_entry_complete - try to delete and free write queue entry + * @e: write queue entry to try to delete + * @completed: bytes completed + * + * writequeue_lock must be held. + */ +static void writequeue_entry_complete(struct writequeue_entry *e, int completed) +{ + e->offset += completed; + e->len -= completed; + /* signal that page was half way transmitted */ + e->dirty = true; + + if (e->len == 0 && e->users == 0) + free_entry(e); +} + +/* + * sctp_bind_addrs - bind a SCTP socket to all our addresses + */ +static int sctp_bind_addrs(struct socket *sock, __be16 port) +{ + struct sockaddr_storage localaddr; + struct sockaddr_unsized *addr = (struct sockaddr_unsized *)&localaddr; + int i, addr_len, result = 0; + + for (i = 0; i < dlm_local_count; i++) { + memcpy(&localaddr, &dlm_local_addr[i], sizeof(localaddr)); + make_sockaddr(&localaddr, port, &addr_len); + + if (!i) + result = kernel_bind(sock, addr, addr_len); + else + result = sock_bind_add(sock->sk, addr, addr_len); + + if (result < 0) { + log_print("Can't bind to %d addr number %d, %d.\n", + port, i + 1, result); + break; + } + } + return result; +} + +/* Get local addresses */ +static void init_local(void) +{ + struct sockaddr_storage sas; + int i; + + dlm_local_count = 0; + for (i = 0; i < DLM_MAX_ADDR_COUNT; i++) { + if (dlm_our_addr(&sas, i)) + break; + + memcpy(&dlm_local_addr[dlm_local_count++], &sas, sizeof(sas)); + } +} + +static struct writequeue_entry *new_writequeue_entry(struct connection *con) +{ + struct writequeue_entry *entry; + + entry = dlm_allocate_writequeue(); + if (!entry) + return NULL; + + entry->page = alloc_page(GFP_ATOMIC | __GFP_ZERO); + if (!entry->page) { + dlm_free_writequeue(entry); + return NULL; + } + + entry->offset = 0; + entry->len = 0; + entry->end = 0; + entry->dirty = false; + entry->con = con; + entry->users = 1; + kref_init(&entry->ref); + return entry; +} + +static struct writequeue_entry *new_wq_entry(struct connection *con, int len, + char **ppc, void (*cb)(void *data), + void *data) +{ + struct writequeue_entry *e; + + spin_lock_bh(&con->writequeue_lock); + if (!list_empty(&con->writequeue)) { + e = list_last_entry(&con->writequeue, struct writequeue_entry, list); + if (DLM_WQ_REMAIN_BYTES(e) >= len) { + kref_get(&e->ref); + + *ppc = page_address(e->page) + e->end; + if (cb) + cb(data); + + e->end += len; + e->users++; + goto out; + } + } + + e = new_writequeue_entry(con); + if (!e) + goto out; + + kref_get(&e->ref); + *ppc = page_address(e->page); + e->end += len; + if (cb) + cb(data); + + list_add_tail(&e->list, &con->writequeue); + +out: + spin_unlock_bh(&con->writequeue_lock); + return e; +}; + +static struct dlm_msg *dlm_lowcomms_new_msg_con(struct connection *con, int len, + char **ppc, void (*cb)(void *data), + void *data) +{ + struct writequeue_entry *e; + struct dlm_msg *msg; + + msg = dlm_allocate_msg(); + if (!msg) + return NULL; + + kref_init(&msg->ref); + + e = new_wq_entry(con, len, ppc, cb, data); + if (!e) { + dlm_free_msg(msg); + return NULL; + } + + msg->retransmit = false; + msg->orig_msg = NULL; + msg->ppc = *ppc; + msg->len = len; + msg->entry = e; + + return msg; +} + +/* avoid false positive for nodes_srcu, unlock happens in + * dlm_lowcomms_commit_msg which is a must call if success + */ +#ifndef __CHECKER__ +struct dlm_msg *dlm_lowcomms_new_msg(int nodeid, int len, char **ppc, + void (*cb)(void *data), void *data) +{ + struct connection *con; + struct dlm_msg *msg; + int idx; + + if (len > DLM_MAX_SOCKET_BUFSIZE || + len < sizeof(struct dlm_header)) { + BUILD_BUG_ON(PAGE_SIZE < DLM_MAX_SOCKET_BUFSIZE); + log_print("failed to allocate a buffer of size %d", len); + WARN_ON_ONCE(1); + return NULL; + } + + idx = srcu_read_lock(&connections_srcu); + con = nodeid2con(nodeid, 0); + if (WARN_ON_ONCE(!con)) { + srcu_read_unlock(&connections_srcu, idx); + return NULL; + } + + msg = dlm_lowcomms_new_msg_con(con, len, ppc, cb, data); + if (!msg) { + srcu_read_unlock(&connections_srcu, idx); + return NULL; + } + + /* for dlm_lowcomms_commit_msg() */ + kref_get(&msg->ref); + /* we assume if successful commit must called */ + msg->idx = idx; + return msg; +} +#endif + +static void _dlm_lowcomms_commit_msg(struct dlm_msg *msg) +{ + struct writequeue_entry *e = msg->entry; + struct connection *con = e->con; + int users; + + spin_lock_bh(&con->writequeue_lock); + kref_get(&msg->ref); + list_add(&msg->list, &e->msgs); + + users = --e->users; + if (users) + goto out; + + e->len = DLM_WQ_LENGTH_BYTES(e); + + lowcomms_queue_swork(con); + +out: + spin_unlock_bh(&con->writequeue_lock); + return; +} + +/* avoid false positive for nodes_srcu, lock was happen in + * dlm_lowcomms_new_msg + */ +#ifndef __CHECKER__ +void dlm_lowcomms_commit_msg(struct dlm_msg *msg) +{ + _dlm_lowcomms_commit_msg(msg); + srcu_read_unlock(&connections_srcu, msg->idx); + /* because dlm_lowcomms_new_msg() */ + kref_put(&msg->ref, dlm_msg_release); +} +#endif + +void dlm_lowcomms_put_msg(struct dlm_msg *msg) +{ + kref_put(&msg->ref, dlm_msg_release); +} + +/* does not held connections_srcu, usage lowcomms_error_report only */ +int dlm_lowcomms_resend_msg(struct dlm_msg *msg) +{ + struct dlm_msg *msg_resend; + char *ppc; + + if (msg->retransmit) + return 1; + + msg_resend = dlm_lowcomms_new_msg_con(msg->entry->con, msg->len, &ppc, + NULL, NULL); + if (!msg_resend) + return -ENOMEM; + + msg->retransmit = true; + kref_get(&msg->ref); + msg_resend->orig_msg = msg; + + memcpy(ppc, msg->ppc, msg->len); + _dlm_lowcomms_commit_msg(msg_resend); + dlm_lowcomms_put_msg(msg_resend); + + return 0; +} + +/* Send a message */ +static int send_to_sock(struct connection *con) +{ + struct writequeue_entry *e; + struct bio_vec bvec; + struct msghdr msg = { + .msg_flags = MSG_SPLICE_PAGES | MSG_DONTWAIT | MSG_NOSIGNAL, + }; + int len, offset, ret; + + spin_lock_bh(&con->writequeue_lock); + e = con_next_wq(con); + if (!e) { + clear_bit(CF_SEND_PENDING, &con->flags); + spin_unlock_bh(&con->writequeue_lock); + return DLM_IO_END; + } + + len = e->len; + offset = e->offset; + WARN_ON_ONCE(len == 0 && e->users == 0); + spin_unlock_bh(&con->writequeue_lock); + + bvec_set_page(&bvec, e->page, len, offset); + iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, len); + ret = sock_sendmsg(con->sock, &msg); + trace_dlm_send(con->nodeid, ret); + if (ret == -EAGAIN || ret == 0) { + lock_sock(con->sock->sk); + spin_lock_bh(&con->writequeue_lock); + if (test_bit(SOCKWQ_ASYNC_NOSPACE, &con->sock->flags) && + !test_and_set_bit(CF_APP_LIMITED, &con->flags)) { + /* Notify TCP that we're limited by the + * application window size. + */ + set_bit(SOCK_NOSPACE, &con->sock->sk->sk_socket->flags); + con->sock->sk->sk_write_pending++; + + clear_bit(CF_SEND_PENDING, &con->flags); + spin_unlock_bh(&con->writequeue_lock); + release_sock(con->sock->sk); + + /* wait for write_space() event */ + return DLM_IO_END; + } + spin_unlock_bh(&con->writequeue_lock); + release_sock(con->sock->sk); + + return DLM_IO_RESCHED; + } else if (ret < 0) { + return ret; + } + + spin_lock_bh(&con->writequeue_lock); + writequeue_entry_complete(e, ret); + spin_unlock_bh(&con->writequeue_lock); + + return DLM_IO_SUCCESS; +} + +static void clean_one_writequeue(struct connection *con) +{ + struct writequeue_entry *e, *safe; + + spin_lock_bh(&con->writequeue_lock); + list_for_each_entry_safe(e, safe, &con->writequeue, list) { + free_entry(e); + } + spin_unlock_bh(&con->writequeue_lock); +} + +static void connection_release(struct rcu_head *rcu) +{ + struct connection *con = container_of(rcu, struct connection, rcu); + + WARN_ON_ONCE(!list_empty(&con->writequeue)); + WARN_ON_ONCE(con->sock); + kfree(con); +} + +/* Called from recovery when it knows that a node has + left the cluster */ +int dlm_lowcomms_close(int nodeid) +{ + struct connection *con; + int idx; + + log_print("closing connection to node %d", nodeid); + + idx = srcu_read_lock(&connections_srcu); + con = nodeid2con(nodeid, 0); + if (WARN_ON_ONCE(!con)) { + srcu_read_unlock(&connections_srcu, idx); + return -ENOENT; + } + + stop_connection_io(con); + log_print("io handling for node: %d stopped", nodeid); + close_connection(con, true); + + spin_lock(&connections_lock); + hlist_del_rcu(&con->list); + spin_unlock(&connections_lock); + + clean_one_writequeue(con); + call_srcu(&connections_srcu, &con->rcu, connection_release); + if (con->othercon) { + clean_one_writequeue(con->othercon); + call_srcu(&connections_srcu, &con->othercon->rcu, connection_release); + } + srcu_read_unlock(&connections_srcu, idx); + + /* for debugging we print when we are done to compare with other + * messages in between. This function need to be correctly synchronized + * with io handling + */ + log_print("closing connection to node %d done", nodeid); + + return 0; +} + +/* Receive worker function */ +static void process_recv_sockets(struct work_struct *work) +{ + struct connection *con = container_of(work, struct connection, rwork); + int ret, buflen; + + down_read(&con->sock_lock); + if (!con->sock) { + up_read(&con->sock_lock); + return; + } + + buflen = READ_ONCE(dlm_config.ci_buffer_size); + do { + ret = receive_from_sock(con, buflen); + } while (ret == DLM_IO_SUCCESS); + up_read(&con->sock_lock); + + switch (ret) { + case DLM_IO_END: + /* CF_RECV_PENDING cleared */ + break; + case DLM_IO_EOF: + close_connection(con, false); + wake_up(&con->shutdown_wait); + /* CF_RECV_PENDING cleared */ + break; + case DLM_IO_FLUSH: + /* we can't flush the process_workqueue here because a + * WQ_MEM_RECLAIM workequeue can occurr a deadlock for a non + * WQ_MEM_RECLAIM workqueue such as process_workqueue. Instead + * we have a waitqueue to wait until all messages are + * processed. + * + * This handling is only necessary to backoff the sender and + * not queue all messages from the socket layer into DLM + * processqueue. When DLM is capable to parse multiple messages + * on an e.g. per socket basis this handling can might be + * removed. Especially in a message burst we are too slow to + * process messages and the queue will fill up memory. + */ + wait_event(processqueue_wq, !atomic_read(&processqueue_count)); + fallthrough; + case DLM_IO_RESCHED: + cond_resched(); + queue_work(io_workqueue, &con->rwork); + /* CF_RECV_PENDING not cleared */ + break; + default: + if (ret < 0) { + if (test_bit(CF_IS_OTHERCON, &con->flags)) { + close_connection(con, false); + } else { + spin_lock_bh(&con->writequeue_lock); + lowcomms_queue_swork(con); + spin_unlock_bh(&con->writequeue_lock); + } + + /* CF_RECV_PENDING cleared for othercon + * we trigger send queue if not already done + * and process_send_sockets will handle it + */ + break; + } + + WARN_ON_ONCE(1); + break; + } +} + +static void process_listen_recv_socket(struct work_struct *work) +{ + int ret; + + if (WARN_ON_ONCE(!listen_con.sock)) + return; + + do { + ret = accept_from_sock(); + } while (ret == DLM_IO_SUCCESS); + + if (ret < 0) + log_print("critical error accepting connection: %d", ret); +} + +static int dlm_connect(struct connection *con) +{ + struct sockaddr_storage addr; + int result, addr_len; + struct socket *sock; + unsigned int mark; + + memset(&addr, 0, sizeof(addr)); + result = nodeid_to_addr(con->nodeid, &addr, NULL, + dlm_proto_ops->try_new_addr, &mark); + if (result < 0) { + log_print("no address for nodeid %d", con->nodeid); + return result; + } + + /* Create a socket to communicate with */ + result = sock_create_kern(&init_net, dlm_local_addr[0].ss_family, + SOCK_STREAM, dlm_proto_ops->proto, &sock); + if (result < 0) + return result; + + sock_set_mark(sock->sk, mark); + dlm_proto_ops->sockopts(sock); + + result = dlm_proto_ops->bind(sock); + if (result < 0) { + sock_release(sock); + return result; + } + + add_sock(sock, con); + + log_print_ratelimited("connecting to %d", con->nodeid); + make_sockaddr(&addr, dlm_config.ci_tcp_port, &addr_len); + result = kernel_connect(sock, (struct sockaddr_unsized *)&addr, addr_len, 0); + switch (result) { + case -EINPROGRESS: + /* not an error */ + fallthrough; + case 0: + break; + default: + if (result < 0) + dlm_close_sock(&con->sock); + + break; + } + + return result; +} + +/* Send worker function */ +static void process_send_sockets(struct work_struct *work) +{ + struct connection *con = container_of(work, struct connection, swork); + int ret; + + WARN_ON_ONCE(test_bit(CF_IS_OTHERCON, &con->flags)); + + down_read(&con->sock_lock); + if (!con->sock) { + up_read(&con->sock_lock); + down_write(&con->sock_lock); + if (!con->sock) { + ret = dlm_connect(con); + switch (ret) { + case 0: + break; + default: + /* CF_SEND_PENDING not cleared */ + up_write(&con->sock_lock); + log_print("connect to node %d try %d error %d", + con->nodeid, con->retries++, ret); + msleep(1000); + /* For now we try forever to reconnect. In + * future we should send a event to cluster + * manager to fence itself after certain amount + * of retries. + */ + queue_work(io_workqueue, &con->swork); + return; + } + } + downgrade_write(&con->sock_lock); + } + + do { + ret = send_to_sock(con); + } while (ret == DLM_IO_SUCCESS); + up_read(&con->sock_lock); + + switch (ret) { + case DLM_IO_END: + /* CF_SEND_PENDING cleared */ + break; + case DLM_IO_RESCHED: + /* CF_SEND_PENDING not cleared */ + cond_resched(); + queue_work(io_workqueue, &con->swork); + break; + default: + if (ret < 0) { + close_connection(con, false); + + /* CF_SEND_PENDING cleared */ + spin_lock_bh(&con->writequeue_lock); + lowcomms_queue_swork(con); + spin_unlock_bh(&con->writequeue_lock); + break; + } + + WARN_ON_ONCE(1); + break; + } +} + +static void work_stop(void) +{ + if (io_workqueue) { + destroy_workqueue(io_workqueue); + io_workqueue = NULL; + } + + if (process_workqueue) { + destroy_workqueue(process_workqueue); + process_workqueue = NULL; + } +} + +static int work_start(void) +{ + io_workqueue = alloc_workqueue("dlm_io", WQ_HIGHPRI | WQ_MEM_RECLAIM | + WQ_UNBOUND, 0); + if (!io_workqueue) { + log_print("can't start dlm_io"); + return -ENOMEM; + } + + process_workqueue = alloc_workqueue("dlm_process", WQ_HIGHPRI | WQ_BH | WQ_PERCPU, 0); + if (!process_workqueue) { + log_print("can't start dlm_process"); + destroy_workqueue(io_workqueue); + io_workqueue = NULL; + return -ENOMEM; + } + + return 0; +} + +void dlm_lowcomms_shutdown(void) +{ + struct connection *con; + int i, idx; + + /* stop lowcomms_listen_data_ready calls */ + lock_sock(listen_con.sock->sk); + listen_con.sock->sk->sk_data_ready = listen_sock.sk_data_ready; + release_sock(listen_con.sock->sk); + + cancel_work_sync(&listen_con.rwork); + dlm_close_sock(&listen_con.sock); + + idx = srcu_read_lock(&connections_srcu); + for (i = 0; i < CONN_HASH_SIZE; i++) { + hlist_for_each_entry_srcu(con, &connection_hash[i], list, + srcu_read_lock_held(&connections_srcu)) { + shutdown_connection(con, true); + stop_connection_io(con); + flush_workqueue(process_workqueue); + close_connection(con, true); + + clean_one_writequeue(con); + if (con->othercon) + clean_one_writequeue(con->othercon); + allow_connection_io(con); + } + } + srcu_read_unlock(&connections_srcu, idx); +} + +void dlm_lowcomms_stop(void) +{ + work_stop(); + dlm_proto_ops = NULL; +} + +static int dlm_listen_for_all(void) +{ + struct socket *sock; + int result; + + log_print("Using %s for communications", + dlm_proto_ops->name); + + result = dlm_proto_ops->listen_validate(); + if (result < 0) + return result; + + result = sock_create_kern(&init_net, dlm_local_addr[0].ss_family, + SOCK_STREAM, dlm_proto_ops->proto, &sock); + if (result < 0) { + log_print("Can't create comms socket: %d", result); + return result; + } + + sock_set_mark(sock->sk, dlm_config.ci_mark); + dlm_proto_ops->listen_sockopts(sock); + + result = dlm_proto_ops->listen_bind(sock); + if (result < 0) + goto out; + + lock_sock(sock->sk); + listen_sock.sk_data_ready = sock->sk->sk_data_ready; + listen_sock.sk_write_space = sock->sk->sk_write_space; + listen_sock.sk_error_report = sock->sk->sk_error_report; + listen_sock.sk_state_change = sock->sk->sk_state_change; + + listen_con.sock = sock; + + sock->sk->sk_allocation = GFP_NOFS; + sock->sk->sk_use_task_frag = false; + sock->sk->sk_data_ready = lowcomms_listen_data_ready; + release_sock(sock->sk); + + result = sock->ops->listen(sock, 128); + if (result < 0) { + dlm_close_sock(&listen_con.sock); + return result; + } + + return 0; + +out: + sock_release(sock); + return result; +} + +static int dlm_tcp_bind(struct socket *sock) +{ + struct sockaddr_storage src_addr; + int result, addr_len; + + /* Bind to our cluster-known address connecting to avoid + * routing problems. + */ + memcpy(&src_addr, &dlm_local_addr[0], sizeof(src_addr)); + make_sockaddr(&src_addr, 0, &addr_len); + + result = kernel_bind(sock, (struct sockaddr_unsized *)&src_addr, + addr_len); + if (result < 0) { + /* This *may* not indicate a critical error */ + log_print("could not bind for connect: %d", result); + } + + return 0; +} + +static int dlm_tcp_listen_validate(void) +{ + /* We don't support multi-homed hosts */ + if (dlm_local_count > 1) { + log_print("Detect multi-homed hosts but use only the first IP address."); + log_print("Try SCTP, if you want to enable multi-link."); + } + + return 0; +} + +static void dlm_tcp_sockopts(struct socket *sock) +{ + /* Turn off Nagle's algorithm */ + tcp_sock_set_nodelay(sock->sk); +} + +static void dlm_tcp_listen_sockopts(struct socket *sock) +{ + dlm_tcp_sockopts(sock); + sock_set_reuseaddr(sock->sk); +} + +static int dlm_tcp_listen_bind(struct socket *sock) +{ + int addr_len; + + /* Bind to our port */ + make_sockaddr(&dlm_local_addr[0], dlm_config.ci_tcp_port, &addr_len); + return kernel_bind(sock, (struct sockaddr_unsized *)&dlm_local_addr[0], + addr_len); +} + +static const struct dlm_proto_ops dlm_tcp_ops = { + .name = "TCP", + .proto = IPPROTO_TCP, + .how = SHUT_WR, + .sockopts = dlm_tcp_sockopts, + .bind = dlm_tcp_bind, + .listen_validate = dlm_tcp_listen_validate, + .listen_sockopts = dlm_tcp_listen_sockopts, + .listen_bind = dlm_tcp_listen_bind, +}; + +static int dlm_sctp_bind(struct socket *sock) +{ + return sctp_bind_addrs(sock, 0); +} + +static int dlm_sctp_listen_validate(void) +{ + if (!IS_ENABLED(CONFIG_IP_SCTP)) { + log_print("SCTP is not enabled by this kernel"); + return -EOPNOTSUPP; + } + + request_module("sctp"); + return 0; +} + +static int dlm_sctp_bind_listen(struct socket *sock) +{ + return sctp_bind_addrs(sock, dlm_config.ci_tcp_port); +} + +static void dlm_sctp_sockopts(struct socket *sock) +{ + /* Turn off Nagle's algorithm */ + sctp_sock_set_nodelay(sock->sk); + sock_set_rcvbuf(sock->sk, NEEDED_RMEM); +} + +static const struct dlm_proto_ops dlm_sctp_ops = { + .name = "SCTP", + .proto = IPPROTO_SCTP, + .how = SHUT_RDWR, + .try_new_addr = true, + .sockopts = dlm_sctp_sockopts, + .bind = dlm_sctp_bind, + .listen_validate = dlm_sctp_listen_validate, + .listen_sockopts = dlm_sctp_sockopts, + .listen_bind = dlm_sctp_bind_listen, +}; + +int dlm_lowcomms_start(void) +{ + int error; + + init_local(); + if (!dlm_local_count) { + error = -ENOTCONN; + log_print("no local IP address has been set"); + goto fail; + } + + error = work_start(); + if (error) + goto fail; + + /* Start listening */ + switch (dlm_config.ci_protocol) { + case DLM_PROTO_TCP: + dlm_proto_ops = &dlm_tcp_ops; + break; + case DLM_PROTO_SCTP: + dlm_proto_ops = &dlm_sctp_ops; + break; + default: + log_print("Invalid protocol identifier %d set", + dlm_config.ci_protocol); + error = -EINVAL; + goto fail_proto_ops; + } + + error = dlm_listen_for_all(); + if (error) + goto fail_listen; + + return 0; + +fail_listen: + dlm_proto_ops = NULL; +fail_proto_ops: + work_stop(); +fail: + return error; +} + +void dlm_lowcomms_init(void) +{ + int i; + + for (i = 0; i < CONN_HASH_SIZE; i++) + INIT_HLIST_HEAD(&connection_hash[i]); + + INIT_WORK(&listen_con.rwork, process_listen_recv_socket); +} + +void dlm_lowcomms_exit(void) +{ + struct connection *con; + int i, idx; + + idx = srcu_read_lock(&connections_srcu); + for (i = 0; i < CONN_HASH_SIZE; i++) { + hlist_for_each_entry_srcu(con, &connection_hash[i], list, + srcu_read_lock_held(&connections_srcu)) { + spin_lock(&connections_lock); + hlist_del_rcu(&con->list); + spin_unlock(&connections_lock); + + if (con->othercon) + call_srcu(&connections_srcu, &con->othercon->rcu, + connection_release); + call_srcu(&connections_srcu, &con->rcu, connection_release); + } + } + srcu_read_unlock(&connections_srcu, idx); +} diff --git a/fs/dlm/lowcomms.h b/fs/dlm/lowcomms.h new file mode 100644 index 000000000..fd0df604e --- /dev/null +++ b/fs/dlm/lowcomms.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2009 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#ifndef __LOWCOMMS_DOT_H__ +#define __LOWCOMMS_DOT_H__ + +#include "dlm_internal.h" + +#define DLM_MIDCOMMS_OPT_LEN sizeof(struct dlm_opts) +#define DLM_MAX_APP_BUFSIZE (DLM_MAX_SOCKET_BUFSIZE - \ + DLM_MIDCOMMS_OPT_LEN) + +#define CONN_HASH_SIZE 32 + +/* This is deliberately very simple because most clusters have simple + * sequential nodeids, so we should be able to go straight to a connection + * struct in the array + */ +static inline int nodeid_hash(int nodeid) +{ + return nodeid & (CONN_HASH_SIZE-1); +} + +/* check if dlm is running */ +bool dlm_lowcomms_is_running(void); + +int dlm_lowcomms_start(void); +void dlm_lowcomms_shutdown(void); +void dlm_lowcomms_shutdown_node(int nodeid, bool force); +void dlm_lowcomms_stop(void); +void dlm_lowcomms_init(void); +void dlm_lowcomms_exit(void); +int dlm_lowcomms_close(int nodeid); +struct dlm_msg *dlm_lowcomms_new_msg(int nodeid, int len, char **ppc, + void (*cb)(void *data), void *data); +void dlm_lowcomms_commit_msg(struct dlm_msg *msg); +void dlm_lowcomms_put_msg(struct dlm_msg *msg); +int dlm_lowcomms_resend_msg(struct dlm_msg *msg); +int dlm_lowcomms_connect_node(int nodeid); +int dlm_lowcomms_nodes_set_mark(int nodeid, unsigned int mark); +int dlm_lowcomms_addr(int nodeid, struct sockaddr_storage *addr); +void dlm_midcomms_receive_done(int nodeid); +struct kmem_cache *dlm_lowcomms_writequeue_cache_create(void); +struct kmem_cache *dlm_lowcomms_msg_cache_create(void); + +#endif /* __LOWCOMMS_DOT_H__ */ + diff --git a/fs/dlm/lvb_table.h b/fs/dlm/lvb_table.h new file mode 100644 index 000000000..09052d967 --- /dev/null +++ b/fs/dlm/lvb_table.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) 2005 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#ifndef __LVB_TABLE_DOT_H__ +#define __LVB_TABLE_DOT_H__ + +extern const int dlm_lvb_operations[8][8]; + +#endif diff --git a/fs/dlm/main.c b/fs/dlm/main.c new file mode 100644 index 000000000..a44d16da7 --- /dev/null +++ b/fs/dlm/main.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0-only +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2007 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#include + +#include "dlm_internal.h" +#include "lockspace.h" +#include "lock.h" +#include "user.h" +#include "memory.h" +#include "config.h" +#include "midcomms.h" + +#define CREATE_TRACE_POINTS +#include + +struct workqueue_struct *dlm_wq; + +static int __init init_dlm(void) +{ + int error; + + error = dlm_memory_init(); + if (error) + goto out; + + dlm_midcomms_init(); + + error = dlm_lockspace_init(); + if (error) + goto out_mem; + + error = dlm_config_init(); + if (error) + goto out_lockspace; + + dlm_register_debugfs(); + + error = dlm_user_init(); + if (error) + goto out_debug; + + error = dlm_plock_init(); + if (error) + goto out_user; + + dlm_wq = alloc_workqueue("dlm_wq", WQ_PERCPU, 0); + if (!dlm_wq) { + error = -ENOMEM; + goto out_plock; + } + + printk("DLM installed\n"); + + return 0; + + out_plock: + dlm_plock_exit(); + out_user: + dlm_user_exit(); + out_debug: + dlm_unregister_debugfs(); + dlm_config_exit(); + out_lockspace: + dlm_lockspace_exit(); + out_mem: + dlm_midcomms_exit(); + dlm_memory_exit(); + out: + return error; +} + +static void __exit exit_dlm(void) +{ + /* be sure every pending work e.g. freeing is done */ + destroy_workqueue(dlm_wq); + dlm_plock_exit(); + dlm_user_exit(); + dlm_config_exit(); + dlm_lockspace_exit(); + dlm_midcomms_exit(); + dlm_unregister_debugfs(); + dlm_memory_exit(); +} + +module_init(init_dlm); +module_exit(exit_dlm); + +MODULE_DESCRIPTION("Distributed Lock Manager"); +MODULE_AUTHOR("Red Hat, Inc."); +MODULE_LICENSE("GPL"); + +EXPORT_SYMBOL_GPL(dlm_new_lockspace); +EXPORT_SYMBOL_GPL(dlm_release_lockspace); +EXPORT_SYMBOL_GPL(dlm_lock); +EXPORT_SYMBOL_GPL(dlm_unlock); + diff --git a/fs/dlm/member.c b/fs/dlm/member.c new file mode 100644 index 000000000..f84233a0f --- /dev/null +++ b/fs/dlm/member.c @@ -0,0 +1,772 @@ +// SPDX-License-Identifier: GPL-2.0-only +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) 2005-2011 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#include "dlm_internal.h" +#include "lockspace.h" +#include "member.h" +#include "recoverd.h" +#include "recover.h" +#include "rcom.h" +#include "config.h" +#include "midcomms.h" +#include "lowcomms.h" + +int dlm_slots_version(const struct dlm_header *h) +{ + if ((le32_to_cpu(h->h_version) & 0x0000FFFF) < DLM_HEADER_SLOTS) + return 0; + return 1; +} + +void dlm_slot_save(struct dlm_ls *ls, struct dlm_rcom *rc, + struct dlm_member *memb) +{ + struct rcom_config *rf = (struct rcom_config *)rc->rc_buf; + + if (!dlm_slots_version(&rc->rc_header)) + return; + + memb->slot = le16_to_cpu(rf->rf_our_slot); + memb->generation = le32_to_cpu(rf->rf_generation); +} + +void dlm_slots_copy_out(struct dlm_ls *ls, struct dlm_rcom *rc) +{ + struct dlm_slot *slot; + struct rcom_slot *ro; + int i; + + ro = (struct rcom_slot *)(rc->rc_buf + sizeof(struct rcom_config)); + + /* ls_slots array is sparse, but not rcom_slots */ + + for (i = 0; i < ls->ls_slots_size; i++) { + slot = &ls->ls_slots[i]; + if (!slot->nodeid) + continue; + ro->ro_nodeid = cpu_to_le32(slot->nodeid); + ro->ro_slot = cpu_to_le16(slot->slot); + ro++; + } +} + +#define SLOT_DEBUG_LINE 128 + +static void log_slots(struct dlm_ls *ls, uint32_t gen, int num_slots, + struct rcom_slot *ro0, struct dlm_slot *array, + int array_size) +{ + char line[SLOT_DEBUG_LINE]; + int len = SLOT_DEBUG_LINE - 1; + int pos = 0; + int ret, i; + + memset(line, 0, sizeof(line)); + + if (array) { + for (i = 0; i < array_size; i++) { + if (!array[i].nodeid) + continue; + + ret = snprintf(line + pos, len - pos, " %d:%d", + array[i].slot, array[i].nodeid); + if (ret >= len - pos) + break; + pos += ret; + } + } else if (ro0) { + for (i = 0; i < num_slots; i++) { + ret = snprintf(line + pos, len - pos, " %d:%d", + ro0[i].ro_slot, ro0[i].ro_nodeid); + if (ret >= len - pos) + break; + pos += ret; + } + } + + log_rinfo(ls, "generation %u slots %d%s", gen, num_slots, line); +} + +int dlm_slots_copy_in(struct dlm_ls *ls) +{ + struct dlm_member *memb; + struct dlm_rcom *rc = ls->ls_recover_buf; + struct rcom_config *rf = (struct rcom_config *)rc->rc_buf; + struct rcom_slot *ro0, *ro; + int our_nodeid = dlm_our_nodeid(); + int i, num_slots; + uint32_t gen; + + if (!dlm_slots_version(&rc->rc_header)) + return -1; + + gen = le32_to_cpu(rf->rf_generation); + if (gen <= ls->ls_generation) { + log_error(ls, "dlm_slots_copy_in gen %u old %u", + gen, ls->ls_generation); + } + ls->ls_generation = gen; + + num_slots = le16_to_cpu(rf->rf_num_slots); + if (!num_slots) + return -1; + + ro0 = (struct rcom_slot *)(rc->rc_buf + sizeof(struct rcom_config)); + + log_slots(ls, gen, num_slots, ro0, NULL, 0); + + list_for_each_entry(memb, &ls->ls_nodes, list) { + for (i = 0, ro = ro0; i < num_slots; i++, ro++) { + if (le32_to_cpu(ro->ro_nodeid) != memb->nodeid) + continue; + memb->slot = le16_to_cpu(ro->ro_slot); + memb->slot_prev = memb->slot; + break; + } + + if (memb->nodeid == our_nodeid) { + if (ls->ls_slot && ls->ls_slot != memb->slot) { + log_error(ls, "dlm_slots_copy_in our slot " + "changed %d %d", ls->ls_slot, + memb->slot); + return -1; + } + + if (!ls->ls_slot) + ls->ls_slot = memb->slot; + } + + if (!memb->slot) { + log_error(ls, "dlm_slots_copy_in nodeid %d no slot", + memb->nodeid); + return -1; + } + } + + return 0; +} + +/* for any nodes that do not support slots, we will not have set memb->slot + in wait_status_all(), so memb->slot will remain -1, and we will not + assign slots or set ls_num_slots here */ + +int dlm_slots_assign(struct dlm_ls *ls, int *num_slots, int *slots_size, + struct dlm_slot **slots_out, uint32_t *gen_out) +{ + struct dlm_member *memb; + struct dlm_slot *array; + int our_nodeid = dlm_our_nodeid(); + int array_size, max_slots, i; + int need = 0; + int max = 0; + int num = 0; + uint32_t gen = 0; + + /* our own memb struct will have slot -1 gen 0 */ + + list_for_each_entry(memb, &ls->ls_nodes, list) { + if (memb->nodeid == our_nodeid) { + memb->slot = ls->ls_slot; + memb->generation = ls->ls_generation; + break; + } + } + + list_for_each_entry(memb, &ls->ls_nodes, list) { + if (memb->generation > gen) + gen = memb->generation; + + /* node doesn't support slots */ + + if (memb->slot == -1) + return -1; + + /* node needs a slot assigned */ + + if (!memb->slot) + need++; + + /* node has a slot assigned */ + + num++; + + if (!max || max < memb->slot) + max = memb->slot; + + /* sanity check, once slot is assigned it shouldn't change */ + + if (memb->slot_prev && memb->slot && memb->slot_prev != memb->slot) { + log_error(ls, "nodeid %d slot changed %d %d", + memb->nodeid, memb->slot_prev, memb->slot); + return -1; + } + memb->slot_prev = memb->slot; + } + + array_size = max + need; + array = kzalloc_objs(*array, array_size, GFP_NOFS); + if (!array) + return -ENOMEM; + + num = 0; + + /* fill in slots (offsets) that are used */ + + list_for_each_entry(memb, &ls->ls_nodes, list) { + if (!memb->slot) + continue; + + if (memb->slot > array_size) { + log_error(ls, "invalid slot number %d", memb->slot); + kfree(array); + return -1; + } + + array[memb->slot - 1].nodeid = memb->nodeid; + array[memb->slot - 1].slot = memb->slot; + num++; + } + + /* assign new slots from unused offsets */ + + list_for_each_entry(memb, &ls->ls_nodes, list) { + if (memb->slot) + continue; + + for (i = 0; i < array_size; i++) { + if (array[i].nodeid) + continue; + + memb->slot = i + 1; + memb->slot_prev = memb->slot; + array[i].nodeid = memb->nodeid; + array[i].slot = memb->slot; + num++; + + if (!ls->ls_slot && memb->nodeid == our_nodeid) + ls->ls_slot = memb->slot; + break; + } + + if (!memb->slot) { + log_error(ls, "no free slot found"); + kfree(array); + return -1; + } + } + + gen++; + + log_slots(ls, gen, num, NULL, array, array_size); + + max_slots = (DLM_MAX_APP_BUFSIZE - sizeof(struct dlm_rcom) - + sizeof(struct rcom_config)) / sizeof(struct rcom_slot); + + if (num > max_slots) { + log_error(ls, "num_slots %d exceeds max_slots %d", + num, max_slots); + kfree(array); + return -1; + } + + *gen_out = gen; + *slots_out = array; + *slots_size = array_size; + *num_slots = num; + return 0; +} + +static void add_ordered_member(struct dlm_ls *ls, struct dlm_member *new) +{ + struct dlm_member *memb = NULL; + struct list_head *tmp; + struct list_head *newlist = &new->list; + struct list_head *head = &ls->ls_nodes; + + list_for_each(tmp, head) { + memb = list_entry(tmp, struct dlm_member, list); + if (new->nodeid < memb->nodeid) + break; + } + + if (!memb) + list_add_tail(newlist, head); + else { + list_add_tail(newlist, tmp); + } +} + +static int add_remote_member(int nodeid) +{ + int error; + + if (nodeid == dlm_our_nodeid()) + return 0; + + error = dlm_lowcomms_connect_node(nodeid); + if (error < 0) + return error; + + dlm_midcomms_add_member(nodeid); + return 0; +} + +static int dlm_add_member(struct dlm_ls *ls, struct dlm_config_node *node) +{ + struct dlm_member *memb; + int error; + + memb = kzalloc_obj(*memb, GFP_NOFS); + if (!memb) + return -ENOMEM; + + memb->nodeid = node->nodeid; + memb->weight = node->weight; + memb->comm_seq = node->comm_seq; + + error = add_remote_member(node->nodeid); + if (error < 0) { + kfree(memb); + return error; + } + + add_ordered_member(ls, memb); + ls->ls_num_nodes++; + return 0; +} + +static struct dlm_member *find_memb(struct list_head *head, int nodeid) +{ + struct dlm_member *memb; + + list_for_each_entry(memb, head, list) { + if (memb->nodeid == nodeid) + return memb; + } + return NULL; +} + +int dlm_is_member(struct dlm_ls *ls, int nodeid) +{ + if (find_memb(&ls->ls_nodes, nodeid)) + return 1; + return 0; +} + +int dlm_is_removed(struct dlm_ls *ls, int nodeid) +{ + WARN_ON_ONCE(!nodeid || nodeid == -1); + + if (find_memb(&ls->ls_nodes_gone, nodeid)) + return 1; + return 0; +} + +static void clear_memb_list(struct list_head *head, + void (*after_del)(int nodeid)) +{ + struct dlm_member *memb; + + while (!list_empty(head)) { + memb = list_entry(head->next, struct dlm_member, list); + list_del(&memb->list); + if (after_del) + after_del(memb->nodeid); + kfree(memb); + } +} + +static void remove_remote_member(int nodeid) +{ + if (nodeid == dlm_our_nodeid()) + return; + + dlm_midcomms_remove_member(nodeid); +} + +void dlm_clear_members(struct dlm_ls *ls) +{ + clear_memb_list(&ls->ls_nodes, remove_remote_member); + ls->ls_num_nodes = 0; +} + +void dlm_clear_members_gone(struct dlm_ls *ls) +{ + clear_memb_list(&ls->ls_nodes_gone, NULL); +} + +static void make_member_array(struct dlm_ls *ls) +{ + struct dlm_member *memb; + int i, w, x = 0, total = 0, all_zero = 0, *array; + + kfree(ls->ls_node_array); + ls->ls_node_array = NULL; + + list_for_each_entry(memb, &ls->ls_nodes, list) { + if (memb->weight) + total += memb->weight; + } + + /* all nodes revert to weight of 1 if all have weight 0 */ + + if (!total) { + total = ls->ls_num_nodes; + all_zero = 1; + } + + ls->ls_total_weight = total; + array = kmalloc_objs(*array, total, GFP_NOFS); + if (!array) + return; + + list_for_each_entry(memb, &ls->ls_nodes, list) { + if (!all_zero && !memb->weight) + continue; + + if (all_zero) + w = 1; + else + w = memb->weight; + + DLM_ASSERT(x < total, printk("total %d x %d\n", total, x);); + + for (i = 0; i < w; i++) + array[x++] = memb->nodeid; + } + + ls->ls_node_array = array; +} + +/* send a status request to all members just to establish comms connections */ + +static int ping_members(struct dlm_ls *ls, uint64_t seq) +{ + struct dlm_member *memb; + int error = 0; + + list_for_each_entry(memb, &ls->ls_nodes, list) { + if (dlm_recovery_stopped(ls)) { + error = -EINTR; + break; + } + error = dlm_rcom_status(ls, memb->nodeid, 0, seq); + if (error) + break; + } + if (error) + log_rinfo(ls, "ping_members aborted %d last nodeid %d", + error, ls->ls_recover_nodeid); + return error; +} + +static void dlm_lsop_recover_prep(struct dlm_ls *ls) +{ + if (!ls->ls_ops || !ls->ls_ops->recover_prep) + return; + ls->ls_ops->recover_prep(ls->ls_ops_arg); +} + +static void dlm_lsop_recover_slot(struct dlm_ls *ls, struct dlm_member *memb, + unsigned int release_recover) +{ + struct dlm_slot slot; + uint32_t seq; + int error; + + if (!ls->ls_ops || !ls->ls_ops->recover_slot) + return; + + /* if there is no comms connection with this node + or the present comms connection is newer + than the one when this member was added, then + we consider the node to have failed (versus + being removed due to dlm_release_lockspace) */ + + error = dlm_comm_seq(memb->nodeid, &seq, false); + + if (!release_recover && !error && seq == memb->comm_seq) + return; + + slot.nodeid = memb->nodeid; + slot.slot = memb->slot; + + ls->ls_ops->recover_slot(ls->ls_ops_arg, &slot); +} + +void dlm_lsop_recover_done(struct dlm_ls *ls) +{ + struct dlm_member *memb; + struct dlm_slot *slots; + int i, num; + + if (!ls->ls_ops || !ls->ls_ops->recover_done) + return; + + num = ls->ls_num_nodes; + slots = kzalloc_objs(*slots, num); + if (!slots) + return; + + i = 0; + list_for_each_entry(memb, &ls->ls_nodes, list) { + if (i == num) { + log_error(ls, "dlm_lsop_recover_done bad num %d", num); + goto out; + } + slots[i].nodeid = memb->nodeid; + slots[i].slot = memb->slot; + i++; + } + + ls->ls_ops->recover_done(ls->ls_ops_arg, slots, num, + ls->ls_slot, ls->ls_generation); + out: + kfree(slots); +} + +static struct dlm_config_node *find_config_node(struct dlm_recover *rv, + int nodeid) +{ + int i; + + for (i = 0; i < rv->nodes_count; i++) { + if (rv->nodes[i].nodeid == nodeid) + return &rv->nodes[i]; + } + return NULL; +} + +int dlm_recover_members(struct dlm_ls *ls, struct dlm_recover *rv, int *neg_out) +{ + struct dlm_member *memb, *safe; + struct dlm_config_node *node; + int i, error, neg = 0, low = -1; + unsigned int release_recover; + + /* previously removed members that we've not finished removing need to + * count as a negative change so the "neg" recovery steps will happen + * + * This functionality must report all member changes to lsops or + * midcomms layer and must never return before. + */ + + list_for_each_entry(memb, &ls->ls_nodes_gone, list) { + log_rinfo(ls, "prev removed member %d", memb->nodeid); + neg++; + } + + /* move departed members from ls_nodes to ls_nodes_gone */ + + list_for_each_entry_safe(memb, safe, &ls->ls_nodes, list) { + node = find_config_node(rv, memb->nodeid); + if (!node) { + log_error(ls, "remove member %d invalid", + memb->nodeid); + return -EFAULT; + } + + if (!node->new && !node->gone) + continue; + + release_recover = 0; + + if (node->gone) { + release_recover = node->release_recover; + log_rinfo(ls, "remove member %d%s", memb->nodeid, + release_recover ? " (release_recover)" : ""); + } else { + /* removed and re-added */ + log_rinfo(ls, "remove member %d comm_seq %u %u", + memb->nodeid, memb->comm_seq, node->comm_seq); + } + + neg++; + list_move(&memb->list, &ls->ls_nodes_gone); + remove_remote_member(memb->nodeid); + ls->ls_num_nodes--; + dlm_lsop_recover_slot(ls, memb, release_recover); + } + + /* add new members to ls_nodes */ + + for (i = 0; i < rv->nodes_count; i++) { + node = &rv->nodes[i]; + if (node->gone) + continue; + + if (dlm_is_member(ls, node->nodeid)) + continue; + error = dlm_add_member(ls, node); + if (error) + return error; + + log_rinfo(ls, "add member %d", node->nodeid); + } + + list_for_each_entry(memb, &ls->ls_nodes, list) { + if (low == -1 || memb->nodeid < low) + low = memb->nodeid; + } + ls->ls_low_nodeid = low; + + make_member_array(ls); + *neg_out = neg; + + error = ping_members(ls, rv->seq); + log_rinfo(ls, "dlm_recover_members %d nodes", ls->ls_num_nodes); + return error; +} + +/* Userspace guarantees that dlm_ls_stop() has completed on all nodes before + dlm_ls_start() is called on any of them to start the new recovery. */ + +int dlm_ls_stop(struct dlm_ls *ls) +{ + int new; + + /* + * Prevent dlm_recv from being in the middle of something when we do + * the stop. This includes ensuring dlm_recv isn't processing a + * recovery message (rcom), while dlm_recoverd is aborting and + * resetting things from an in-progress recovery. i.e. we want + * dlm_recoverd to abort its recovery without worrying about dlm_recv + * processing an rcom at the same time. Stopping dlm_recv also makes + * it easy for dlm_receive_message() to check locking stopped and add a + * message to the requestqueue without races. + */ + + write_lock_bh(&ls->ls_recv_active); + + /* + * Abort any recovery that's in progress (see RECOVER_STOP, + * dlm_recovery_stopped()) and tell any other threads running in the + * dlm to quit any processing (see RUNNING, dlm_locking_stopped()). + */ + + spin_lock_bh(&ls->ls_recover_lock); + set_bit(LSFL_RECOVER_STOP, &ls->ls_flags); + new = test_and_clear_bit(LSFL_RUNNING, &ls->ls_flags); + if (new) + timer_delete_sync(&ls->ls_scan_timer); + ls->ls_recover_seq++; + + /* activate requestqueue and stop processing */ + write_lock_bh(&ls->ls_requestqueue_lock); + set_bit(LSFL_RECV_MSG_BLOCKED, &ls->ls_flags); + write_unlock_bh(&ls->ls_requestqueue_lock); + spin_unlock_bh(&ls->ls_recover_lock); + + /* + * Let dlm_recv run again, now any normal messages will be saved on the + * requestqueue for later. + */ + + write_unlock_bh(&ls->ls_recv_active); + + /* + * This in_recovery lock does two things: + * 1) Keeps this function from returning until all threads are out + * of locking routines and locking is truly stopped. + * 2) Keeps any new requests from being processed until it's unlocked + * when recovery is complete. + */ + + if (new) { + set_bit(LSFL_RECOVER_DOWN, &ls->ls_flags); + wake_up_process(ls->ls_recoverd_task); + wait_event(ls->ls_recover_lock_wait, + test_bit(LSFL_RECOVER_LOCK, &ls->ls_flags)); + } + + /* + * The recoverd suspend/resume makes sure that dlm_recoverd (if + * running) has noticed RECOVER_STOP above and quit processing the + * previous recovery. + */ + + dlm_recoverd_suspend(ls); + + spin_lock_bh(&ls->ls_recover_lock); + kfree(ls->ls_slots); + ls->ls_slots = NULL; + ls->ls_num_slots = 0; + ls->ls_slots_size = 0; + ls->ls_recover_status = 0; + spin_unlock_bh(&ls->ls_recover_lock); + + dlm_recoverd_resume(ls); + + if (!ls->ls_recover_begin) + ls->ls_recover_begin = jiffies; + + /* call recover_prep ops only once and not multiple times + * for each possible dlm_ls_stop() when recovery is already + * stopped. + * + * If we successful was able to clear LSFL_RUNNING bit and + * it was set we know it is the first dlm_ls_stop() call. + */ + if (new) + dlm_lsop_recover_prep(ls); + + return 0; +} + +int dlm_ls_start(struct dlm_ls *ls) +{ + struct dlm_recover *rv, *rv_old; + struct dlm_config_node *nodes = NULL; + int error, count; + + rv = kzalloc_obj(*rv, GFP_NOFS); + if (!rv) + return -ENOMEM; + + error = dlm_config_nodes(ls->ls_name, &nodes, &count); + if (error < 0) + goto fail_rv; + + spin_lock_bh(&ls->ls_recover_lock); + + /* the lockspace needs to be stopped before it can be started */ + + if (!dlm_locking_stopped(ls)) { + spin_unlock_bh(&ls->ls_recover_lock); + log_error(ls, "start ignored: lockspace running"); + error = -EINVAL; + goto fail; + } + + rv->nodes = nodes; + rv->nodes_count = count; + rv->seq = ++ls->ls_recover_seq; + rv_old = ls->ls_recover_args; + ls->ls_recover_args = rv; + spin_unlock_bh(&ls->ls_recover_lock); + + if (rv_old) { + log_error(ls, "unused recovery %llx %d", + (unsigned long long)rv_old->seq, rv_old->nodes_count); + kfree(rv_old->nodes); + kfree(rv_old); + } + + set_bit(LSFL_RECOVER_WORK, &ls->ls_flags); + wake_up_process(ls->ls_recoverd_task); + return 0; + + fail: + kfree(nodes); + fail_rv: + kfree(rv); + return error; +} + diff --git a/fs/dlm/member.h b/fs/dlm/member.h new file mode 100644 index 000000000..f61cfde46 --- /dev/null +++ b/fs/dlm/member.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) 2005-2011 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#ifndef __MEMBER_DOT_H__ +#define __MEMBER_DOT_H__ + +int dlm_ls_stop(struct dlm_ls *ls); +int dlm_ls_start(struct dlm_ls *ls); +void dlm_clear_members(struct dlm_ls *ls); +void dlm_clear_members_gone(struct dlm_ls *ls); +int dlm_recover_members(struct dlm_ls *ls, struct dlm_recover *rv,int *neg_out); +int dlm_is_removed(struct dlm_ls *ls, int nodeid); +int dlm_is_member(struct dlm_ls *ls, int nodeid); +int dlm_slots_version(const struct dlm_header *h); +void dlm_slot_save(struct dlm_ls *ls, struct dlm_rcom *rc, + struct dlm_member *memb); +void dlm_slots_copy_out(struct dlm_ls *ls, struct dlm_rcom *rc); +int dlm_slots_copy_in(struct dlm_ls *ls); +int dlm_slots_assign(struct dlm_ls *ls, int *num_slots, int *slots_size, + struct dlm_slot **slots_out, uint32_t *gen_out); +void dlm_lsop_recover_done(struct dlm_ls *ls); + +#endif /* __MEMBER_DOT_H__ */ + diff --git a/fs/dlm/memory.c b/fs/dlm/memory.c new file mode 100644 index 000000000..ee55994ce --- /dev/null +++ b/fs/dlm/memory.c @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0-only +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2007 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#include "dlm_internal.h" +#include "midcomms.h" +#include "lowcomms.h" +#include "config.h" +#include "memory.h" +#include "ast.h" + +static struct kmem_cache *writequeue_cache; +static struct kmem_cache *mhandle_cache; +static struct kmem_cache *msg_cache; +static struct kmem_cache *lkb_cache; +static struct kmem_cache *rsb_cache; +static struct kmem_cache *cb_cache; + + +int __init dlm_memory_init(void) +{ + writequeue_cache = dlm_lowcomms_writequeue_cache_create(); + if (!writequeue_cache) + goto out; + + mhandle_cache = dlm_midcomms_cache_create(); + if (!mhandle_cache) + goto mhandle; + + lkb_cache = kmem_cache_create("dlm_lkb", sizeof(struct dlm_lkb), + __alignof__(struct dlm_lkb), 0, NULL); + if (!lkb_cache) + goto lkb; + + msg_cache = dlm_lowcomms_msg_cache_create(); + if (!msg_cache) + goto msg; + + rsb_cache = kmem_cache_create("dlm_rsb", sizeof(struct dlm_rsb), + __alignof__(struct dlm_rsb), 0, NULL); + if (!rsb_cache) + goto rsb; + + cb_cache = kmem_cache_create_usercopy("dlm_cb", sizeof(struct dlm_callback), + __alignof__(struct dlm_callback), 0, + offsetof(struct dlm_callback, lvbptr), + sizeof_field(struct dlm_callback, lvbptr), + NULL); + if (!cb_cache) + goto cb; + + return 0; + +cb: + kmem_cache_destroy(rsb_cache); +rsb: + kmem_cache_destroy(msg_cache); +msg: + kmem_cache_destroy(lkb_cache); +lkb: + kmem_cache_destroy(mhandle_cache); +mhandle: + kmem_cache_destroy(writequeue_cache); +out: + return -ENOMEM; +} + +void dlm_memory_exit(void) +{ + rcu_barrier(); + + kmem_cache_destroy(writequeue_cache); + kmem_cache_destroy(mhandle_cache); + kmem_cache_destroy(msg_cache); + kmem_cache_destroy(lkb_cache); + kmem_cache_destroy(rsb_cache); + kmem_cache_destroy(cb_cache); +} + +char *dlm_allocate_lvb(struct dlm_ls *ls) +{ + return kzalloc(ls->ls_lvblen, GFP_ATOMIC); +} + +void dlm_free_lvb(char *p) +{ + kfree(p); +} + +struct dlm_rsb *dlm_allocate_rsb(void) +{ + return kmem_cache_zalloc(rsb_cache, GFP_ATOMIC); +} + +static void __free_rsb_rcu(struct rcu_head *rcu) +{ + struct dlm_rsb *r = container_of(rcu, struct dlm_rsb, rcu); + if (r->res_lvbptr) + dlm_free_lvb(r->res_lvbptr); + kmem_cache_free(rsb_cache, r); +} + +void dlm_free_rsb(struct dlm_rsb *r) +{ + call_rcu(&r->rcu, __free_rsb_rcu); +} + +struct dlm_lkb *dlm_allocate_lkb(void) +{ + return kmem_cache_zalloc(lkb_cache, GFP_ATOMIC); +} + +static void __free_lkb_rcu(struct rcu_head *rcu) +{ + struct dlm_lkb *lkb = container_of(rcu, struct dlm_lkb, rcu); + + if (test_bit(DLM_DFL_USER_BIT, &lkb->lkb_dflags)) { + struct dlm_user_args *ua; + ua = lkb->lkb_ua; + if (ua) { + kfree(ua->lksb.sb_lvbptr); + kfree(ua); + } + } + + kmem_cache_free(lkb_cache, lkb); +} + +void dlm_free_lkb(struct dlm_lkb *lkb) +{ + call_rcu(&lkb->rcu, __free_lkb_rcu); +} + +struct dlm_mhandle *dlm_allocate_mhandle(void) +{ + return kmem_cache_alloc(mhandle_cache, GFP_ATOMIC); +} + +void dlm_free_mhandle(struct dlm_mhandle *mhandle) +{ + kmem_cache_free(mhandle_cache, mhandle); +} + +struct writequeue_entry *dlm_allocate_writequeue(void) +{ + return kmem_cache_alloc(writequeue_cache, GFP_ATOMIC); +} + +void dlm_free_writequeue(struct writequeue_entry *writequeue) +{ + kmem_cache_free(writequeue_cache, writequeue); +} + +struct dlm_msg *dlm_allocate_msg(void) +{ + return kmem_cache_alloc(msg_cache, GFP_ATOMIC); +} + +void dlm_free_msg(struct dlm_msg *msg) +{ + kmem_cache_free(msg_cache, msg); +} + +struct dlm_callback *dlm_allocate_cb(void) +{ + return kmem_cache_alloc(cb_cache, GFP_ATOMIC); +} + +void dlm_free_cb(struct dlm_callback *cb) +{ + kmem_cache_free(cb_cache, cb); +} diff --git a/fs/dlm/memory.h b/fs/dlm/memory.h new file mode 100644 index 000000000..551b6b788 --- /dev/null +++ b/fs/dlm/memory.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2007 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#ifndef __MEMORY_DOT_H__ +#define __MEMORY_DOT_H__ + +int dlm_memory_init(void); +void dlm_memory_exit(void); +struct dlm_rsb *dlm_allocate_rsb(void); +void dlm_free_rsb(struct dlm_rsb *r); +struct dlm_lkb *dlm_allocate_lkb(void); +void dlm_free_lkb(struct dlm_lkb *l); +char *dlm_allocate_lvb(struct dlm_ls *ls); +void dlm_free_lvb(char *l); +struct dlm_mhandle *dlm_allocate_mhandle(void); +void dlm_free_mhandle(struct dlm_mhandle *mhandle); +struct writequeue_entry *dlm_allocate_writequeue(void); +void dlm_free_writequeue(struct writequeue_entry *writequeue); +struct dlm_msg *dlm_allocate_msg(void); +void dlm_free_msg(struct dlm_msg *msg); +struct dlm_callback *dlm_allocate_cb(void); +void dlm_free_cb(struct dlm_callback *cb); + +#endif /* __MEMORY_DOT_H__ */ + diff --git a/fs/dlm/midcomms.c b/fs/dlm/midcomms.c new file mode 100644 index 000000000..896416460 --- /dev/null +++ b/fs/dlm/midcomms.c @@ -0,0 +1,1514 @@ +// SPDX-License-Identifier: GPL-2.0-only +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2021 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +/* + * midcomms.c + * + * This is the appallingly named "mid-level" comms layer. It takes care about + * deliver an on application layer "reliable" communication above the used + * lowcomms transport layer. + * + * How it works: + * + * Each nodes keeps track of all send DLM messages in send_queue with a sequence + * number. The receive will send an DLM_ACK message back for every DLM message + * received at the other side. If a reconnect happens in lowcomms we will send + * all unacknowledged dlm messages again. The receiving side might drop any already + * received message by comparing sequence numbers. + * + * How version detection works: + * + * Due the fact that dlm has pre-configured node addresses on every side + * it is in it's nature that every side connects at starts to transmit + * dlm messages which ends in a race. However DLM_RCOM_NAMES, DLM_RCOM_STATUS + * and their replies are the first messages which are exchanges. Due backwards + * compatibility these messages are not covered by the midcomms re-transmission + * layer. These messages have their own re-transmission handling in the dlm + * application layer. The version field of every node will be set on these RCOM + * messages as soon as they arrived and the node isn't yet part of the nodes + * hash. There exists also logic to detect version mismatched if something weird + * going on or the first messages isn't an expected one. + * + * Termination: + * + * The midcomms layer does a 4 way handshake for termination on DLM protocol + * like TCP supports it with half-closed socket support. SCTP doesn't support + * half-closed socket, so we do it on DLM layer. Also socket shutdown() can be + * interrupted by .e.g. tcp reset itself. Additional there exists the othercon + * paradigm in lowcomms which cannot be easily without breaking backwards + * compatibility. A node cannot send anything to another node when a DLM_FIN + * message was send. There exists additional logic to print a warning if + * DLM wants to do it. There exists a state handling like RFC 793 but reduced + * to termination only. The event "member removal event" describes the cluster + * manager removed the node from internal lists, at this point DLM does not + * send any message to the other node. There exists two cases: + * + * 1. The cluster member was removed and we received a FIN + * OR + * 2. We received a FIN but the member was not removed yet + * + * One of these cases will do the CLOSE_WAIT to LAST_ACK change. + * + * + * +---------+ + * | CLOSED | + * +---------+ + * | add member/receive RCOM version + * | detection msg + * V + * +---------+ + * | ESTAB | + * +---------+ + * CLOSE | | rcv FIN + * ------- | | ------- + * +---------+ snd FIN / \ snd ACK +---------+ + * | FIN |<----------------- ------------------>| CLOSE | + * | WAIT-1 |------------------ | WAIT | + * +---------+ rcv FIN \ +---------+ + * | rcv ACK of FIN ------- | CLOSE | member + * | -------------- snd ACK | ------- | removal + * V x V snd FIN V event + * +---------+ +---------+ +---------+ + * |FINWAIT-2| | CLOSING | | LAST-ACK| + * +---------+ +---------+ +---------+ + * | rcv ACK of FIN | rcv ACK of FIN | + * | rcv FIN -------------- | -------------- | + * | ------- x V x V + * \ snd ACK +---------+ +---------+ + * ------------------------>| CLOSED | | CLOSED | + * +---------+ +---------+ + * + * NOTE: any state can interrupted by midcomms_close() and state will be + * switched to CLOSED in case of fencing. There exists also some timeout + * handling when we receive the version detection RCOM messages which is + * made by observation. + * + * Future improvements: + * + * There exists some known issues/improvements of the dlm handling. Some + * of them should be done in a next major dlm version bump which makes + * it incompatible with previous versions. + * + * Unaligned memory access: + * + * There exists cases when the dlm message buffer length is not aligned + * to 8 byte. However seems nobody detected any problem with it. This + * can be fixed in the next major version bump of dlm. + * + * Version detection: + * + * The version detection and how it's done is related to backwards + * compatibility. There exists better ways to make a better handling. + * However this should be changed in the next major version bump of dlm. + * + * Tail Size checking: + * + * There exists a message tail payload in e.g. DLM_MSG however we don't + * check it against the message length yet regarding to the receive buffer + * length. That need to be validated. + * + * Fencing bad nodes: + * + * At timeout places or weird sequence number behaviours we should send + * a fencing request to the cluster manager. + */ + +/* Debug switch to enable a 5 seconds sleep waiting of a termination. + * This can be useful to test fencing while termination is running. + * This requires a setup with only gfs2 as dlm user, so that the + * last umount will terminate the connection. + * + * However it became useful to test, while the 5 seconds block in umount + * just press the reset button. In a lot of dropping the termination + * process can could take several seconds. + */ +#define DLM_DEBUG_FENCE_TERMINATION 0 + +#include +#include + +#include "dlm_internal.h" +#include "lowcomms.h" +#include "config.h" +#include "memory.h" +#include "lock.h" +#include "util.h" +#include "midcomms.h" + +/* init value for sequence numbers for testing purpose only e.g. overflows */ +#define DLM_SEQ_INIT 0 +/* 5 seconds wait to sync ending of dlm */ +#define DLM_SHUTDOWN_TIMEOUT msecs_to_jiffies(5000) +#define DLM_VERSION_NOT_SET 0 +#define DLM_SEND_ACK_BACK_MSG_THRESHOLD 32 +#define DLM_RECV_ACK_BACK_MSG_THRESHOLD (DLM_SEND_ACK_BACK_MSG_THRESHOLD * 8) + +struct midcomms_node { + int nodeid; + uint32_t version; + atomic_t seq_send; + atomic_t seq_next; + /* These queues are unbound because we cannot drop any message in dlm. + * We could send a fence signal for a specific node to the cluster + * manager if queues hits some maximum value, however this handling + * not supported yet. + */ + struct list_head send_queue; + spinlock_t send_queue_lock; + atomic_t send_queue_cnt; +#define DLM_NODE_FLAG_CLOSE 1 +#define DLM_NODE_FLAG_STOP_TX 2 +#define DLM_NODE_FLAG_STOP_RX 3 + atomic_t ulp_delivered; + unsigned long flags; + wait_queue_head_t shutdown_wait; + + /* dlm tcp termination state */ +#define DLM_CLOSED 1 +#define DLM_ESTABLISHED 2 +#define DLM_FIN_WAIT1 3 +#define DLM_FIN_WAIT2 4 +#define DLM_CLOSE_WAIT 5 +#define DLM_LAST_ACK 6 +#define DLM_CLOSING 7 + int state; + spinlock_t state_lock; + + /* counts how many lockspaces are using this node + * this refcount is necessary to determine if the + * node wants to disconnect. + */ + int users; + + /* not protected by srcu, node_hash lifetime */ + void *debugfs; + + struct hlist_node hlist; + struct rcu_head rcu; +}; + +struct dlm_mhandle { + const union dlm_packet *inner_p; + struct midcomms_node *node; + struct dlm_opts *opts; + struct dlm_msg *msg; + bool committed; + uint32_t seq; + + void (*ack_rcv)(struct midcomms_node *node); + + /* get_mhandle/commit srcu idx exchange */ + int idx; + + struct list_head list; + struct rcu_head rcu; +}; + +static struct hlist_head node_hash[CONN_HASH_SIZE]; +static DEFINE_SPINLOCK(nodes_lock); +DEFINE_STATIC_SRCU(nodes_srcu); + +/* This mutex prevents that midcomms_close() is running while + * stop() or remove(). As I experienced invalid memory access + * behaviours when DLM_DEBUG_FENCE_TERMINATION is enabled and + * resetting machines. I will end in some double deletion in nodes + * datastructure. + */ +static DEFINE_MUTEX(close_lock); + +struct kmem_cache *dlm_midcomms_cache_create(void) +{ + return KMEM_CACHE(dlm_mhandle, 0); +} + +static inline const char *dlm_state_str(int state) +{ + switch (state) { + case DLM_CLOSED: + return "CLOSED"; + case DLM_ESTABLISHED: + return "ESTABLISHED"; + case DLM_FIN_WAIT1: + return "FIN_WAIT1"; + case DLM_FIN_WAIT2: + return "FIN_WAIT2"; + case DLM_CLOSE_WAIT: + return "CLOSE_WAIT"; + case DLM_LAST_ACK: + return "LAST_ACK"; + case DLM_CLOSING: + return "CLOSING"; + default: + return "UNKNOWN"; + } +} + +const char *dlm_midcomms_state(struct midcomms_node *node) +{ + return dlm_state_str(node->state); +} + +unsigned long dlm_midcomms_flags(struct midcomms_node *node) +{ + return node->flags; +} + +int dlm_midcomms_send_queue_cnt(struct midcomms_node *node) +{ + return atomic_read(&node->send_queue_cnt); +} + +uint32_t dlm_midcomms_version(struct midcomms_node *node) +{ + return node->version; +} + +static struct midcomms_node *__find_node(int nodeid, int r) +{ + struct midcomms_node *node; + + hlist_for_each_entry_srcu(node, &node_hash[r], hlist, + srcu_read_lock_held(&nodes_srcu)) { + if (node->nodeid == nodeid) + return node; + } + + return NULL; +} + +static void dlm_mhandle_release(struct rcu_head *rcu) +{ + struct dlm_mhandle *mh = container_of(rcu, struct dlm_mhandle, rcu); + + dlm_lowcomms_put_msg(mh->msg); + dlm_free_mhandle(mh); +} + +static void dlm_mhandle_delete(struct midcomms_node *node, + struct dlm_mhandle *mh) +{ + list_del_rcu(&mh->list); + atomic_dec(&node->send_queue_cnt); + call_rcu(&mh->rcu, dlm_mhandle_release); +} + +static void dlm_send_queue_flush(struct midcomms_node *node) +{ + struct dlm_mhandle *mh; + + pr_debug("flush midcomms send queue of node %d\n", node->nodeid); + + rcu_read_lock(); + spin_lock_bh(&node->send_queue_lock); + list_for_each_entry_rcu(mh, &node->send_queue, list) { + dlm_mhandle_delete(node, mh); + } + spin_unlock_bh(&node->send_queue_lock); + rcu_read_unlock(); +} + +static void midcomms_node_reset(struct midcomms_node *node) +{ + pr_debug("reset node %d\n", node->nodeid); + + atomic_set(&node->seq_next, DLM_SEQ_INIT); + atomic_set(&node->seq_send, DLM_SEQ_INIT); + atomic_set(&node->ulp_delivered, 0); + node->version = DLM_VERSION_NOT_SET; + node->flags = 0; + + dlm_send_queue_flush(node); + node->state = DLM_CLOSED; + wake_up(&node->shutdown_wait); +} + +static struct midcomms_node *nodeid2node(int nodeid) +{ + return __find_node(nodeid, nodeid_hash(nodeid)); +} + +int dlm_midcomms_addr(int nodeid, struct sockaddr_storage *addr) +{ + int ret, idx, r = nodeid_hash(nodeid); + struct midcomms_node *node; + + ret = dlm_lowcomms_addr(nodeid, addr); + if (ret) + return ret; + + idx = srcu_read_lock(&nodes_srcu); + node = __find_node(nodeid, r); + if (node) { + srcu_read_unlock(&nodes_srcu, idx); + return 0; + } + srcu_read_unlock(&nodes_srcu, idx); + + node = kmalloc_obj(*node, GFP_NOFS); + if (!node) + return -ENOMEM; + + node->debugfs = dlm_create_debug_comms_file(nodeid, node); + node->nodeid = nodeid; + spin_lock_init(&node->state_lock); + spin_lock_init(&node->send_queue_lock); + atomic_set(&node->send_queue_cnt, 0); + INIT_LIST_HEAD(&node->send_queue); + init_waitqueue_head(&node->shutdown_wait); + node->users = 0; + midcomms_node_reset(node); + + spin_lock_bh(&nodes_lock); + hlist_add_head_rcu(&node->hlist, &node_hash[r]); + spin_unlock_bh(&nodes_lock); + + return 0; +} + +static int dlm_send_ack(int nodeid, uint32_t seq) +{ + int mb_len = sizeof(struct dlm_header); + struct dlm_header *m_header; + struct dlm_msg *msg; + char *ppc; + + msg = dlm_lowcomms_new_msg(nodeid, mb_len, &ppc, NULL, NULL); + if (!msg) + return -ENOMEM; + + m_header = (struct dlm_header *)ppc; + + m_header->h_version = cpu_to_le32(DLM_HEADER_MAJOR | DLM_HEADER_MINOR); + m_header->h_nodeid = cpu_to_le32(dlm_our_nodeid()); + m_header->h_length = cpu_to_le16(mb_len); + m_header->h_cmd = DLM_ACK; + m_header->u.h_seq = cpu_to_le32(seq); + + dlm_lowcomms_commit_msg(msg); + dlm_lowcomms_put_msg(msg); + + return 0; +} + +static void dlm_send_ack_threshold(struct midcomms_node *node, + uint32_t threshold) +{ + uint32_t oval, nval; + bool send_ack; + + /* let only send one user trigger threshold to send ack back */ + do { + oval = atomic_read(&node->ulp_delivered); + send_ack = (oval > threshold); + /* abort if threshold is not reached */ + if (!send_ack) + break; + + nval = 0; + /* try to reset ulp_delivered counter */ + } while (atomic_cmpxchg(&node->ulp_delivered, oval, nval) != oval); + + if (send_ack) + dlm_send_ack(node->nodeid, atomic_read(&node->seq_next)); +} + +static int dlm_send_fin(struct midcomms_node *node, + void (*ack_rcv)(struct midcomms_node *node)) +{ + int mb_len = sizeof(struct dlm_header); + struct dlm_header *m_header; + struct dlm_mhandle *mh; + char *ppc; + + mh = dlm_midcomms_get_mhandle(node->nodeid, mb_len, &ppc); + if (!mh) + return -ENOMEM; + + set_bit(DLM_NODE_FLAG_STOP_TX, &node->flags); + mh->ack_rcv = ack_rcv; + + m_header = (struct dlm_header *)ppc; + + m_header->h_version = cpu_to_le32(DLM_HEADER_MAJOR | DLM_HEADER_MINOR); + m_header->h_nodeid = cpu_to_le32(dlm_our_nodeid()); + m_header->h_length = cpu_to_le16(mb_len); + m_header->h_cmd = DLM_FIN; + + pr_debug("sending fin msg to node %d\n", node->nodeid); + dlm_midcomms_commit_mhandle(mh, NULL, 0); + + return 0; +} + +static void dlm_receive_ack(struct midcomms_node *node, uint32_t seq) +{ + struct dlm_mhandle *mh; + + rcu_read_lock(); + list_for_each_entry_rcu(mh, &node->send_queue, list) { + if (before(mh->seq, seq)) { + if (mh->ack_rcv) + mh->ack_rcv(node); + } else { + /* send queue should be ordered */ + break; + } + } + + spin_lock_bh(&node->send_queue_lock); + list_for_each_entry_rcu(mh, &node->send_queue, list) { + if (before(mh->seq, seq)) { + dlm_mhandle_delete(node, mh); + } else { + /* send queue should be ordered */ + break; + } + } + spin_unlock_bh(&node->send_queue_lock); + rcu_read_unlock(); +} + +static void dlm_pas_fin_ack_rcv(struct midcomms_node *node) +{ + spin_lock_bh(&node->state_lock); + pr_debug("receive passive fin ack from node %d with state %s\n", + node->nodeid, dlm_state_str(node->state)); + + switch (node->state) { + case DLM_LAST_ACK: + /* DLM_CLOSED */ + midcomms_node_reset(node); + break; + case DLM_CLOSED: + /* not valid but somehow we got what we want */ + wake_up(&node->shutdown_wait); + break; + default: + spin_unlock_bh(&node->state_lock); + log_print("%s: unexpected state: %d", + __func__, node->state); + WARN_ON_ONCE(1); + return; + } + spin_unlock_bh(&node->state_lock); +} + +static void dlm_receive_buffer_3_2_trace(uint32_t seq, + const union dlm_packet *p) +{ + switch (p->header.h_cmd) { + case DLM_MSG: + trace_dlm_recv_message(dlm_our_nodeid(), seq, &p->message); + break; + case DLM_RCOM: + trace_dlm_recv_rcom(dlm_our_nodeid(), seq, &p->rcom); + break; + default: + break; + } +} + +static void dlm_midcomms_receive_buffer(const union dlm_packet *p, + struct midcomms_node *node, + uint32_t seq) +{ + bool is_expected_seq; + uint32_t oval, nval; + + do { + oval = atomic_read(&node->seq_next); + is_expected_seq = (oval == seq); + if (!is_expected_seq) + break; + + nval = oval + 1; + } while (atomic_cmpxchg(&node->seq_next, oval, nval) != oval); + + if (is_expected_seq) { + switch (p->header.h_cmd) { + case DLM_FIN: + spin_lock_bh(&node->state_lock); + pr_debug("receive fin msg from node %d with state %s\n", + node->nodeid, dlm_state_str(node->state)); + + switch (node->state) { + case DLM_ESTABLISHED: + dlm_send_ack(node->nodeid, nval); + + /* passive shutdown DLM_LAST_ACK case 1 + * additional we check if the node is used by + * cluster manager events at all. + */ + if (node->users == 0) { + node->state = DLM_LAST_ACK; + pr_debug("switch node %d to state %s case 1\n", + node->nodeid, dlm_state_str(node->state)); + set_bit(DLM_NODE_FLAG_STOP_RX, &node->flags); + dlm_send_fin(node, dlm_pas_fin_ack_rcv); + } else { + node->state = DLM_CLOSE_WAIT; + pr_debug("switch node %d to state %s\n", + node->nodeid, dlm_state_str(node->state)); + } + break; + case DLM_FIN_WAIT1: + dlm_send_ack(node->nodeid, nval); + node->state = DLM_CLOSING; + set_bit(DLM_NODE_FLAG_STOP_RX, &node->flags); + pr_debug("switch node %d to state %s\n", + node->nodeid, dlm_state_str(node->state)); + break; + case DLM_FIN_WAIT2: + dlm_send_ack(node->nodeid, nval); + midcomms_node_reset(node); + pr_debug("switch node %d to state %s\n", + node->nodeid, dlm_state_str(node->state)); + break; + case DLM_LAST_ACK: + /* probably remove_member caught it, do nothing */ + break; + default: + spin_unlock_bh(&node->state_lock); + log_print("%s: unexpected state: %d", + __func__, node->state); + WARN_ON_ONCE(1); + return; + } + spin_unlock_bh(&node->state_lock); + break; + default: + WARN_ON_ONCE(test_bit(DLM_NODE_FLAG_STOP_RX, &node->flags)); + dlm_receive_buffer_3_2_trace(seq, p); + dlm_receive_buffer(p, node->nodeid); + atomic_inc(&node->ulp_delivered); + /* unlikely case to send ack back when we don't transmit */ + dlm_send_ack_threshold(node, DLM_RECV_ACK_BACK_MSG_THRESHOLD); + break; + } + } else { + /* retry to ack message which we already have by sending back + * current node->seq_next number as ack. + */ + if (seq < oval) + dlm_send_ack(node->nodeid, oval); + + log_print_ratelimited("ignore dlm msg because seq mismatch, seq: %u, expected: %u, nodeid: %d", + seq, oval, node->nodeid); + } +} + +static int dlm_opts_check_msglen(const union dlm_packet *p, uint16_t msglen, + int nodeid) +{ + int len = msglen; + + /* we only trust outer header msglen because + * it's checked against receive buffer length. + */ + if (len < sizeof(struct dlm_opts)) + return -1; + len -= sizeof(struct dlm_opts); + + if (len < le16_to_cpu(p->opts.o_optlen)) + return -1; + len -= le16_to_cpu(p->opts.o_optlen); + + switch (p->opts.o_nextcmd) { + case DLM_FIN: + if (len < sizeof(struct dlm_header)) { + log_print("fin too small: %d, will skip this message from node %d", + len, nodeid); + return -1; + } + + break; + case DLM_MSG: + if (len < sizeof(struct dlm_message)) { + log_print("msg too small: %d, will skip this message from node %d", + msglen, nodeid); + return -1; + } + + break; + case DLM_RCOM: + if (len < sizeof(struct dlm_rcom)) { + log_print("rcom msg too small: %d, will skip this message from node %d", + len, nodeid); + return -1; + } + + break; + default: + log_print("unsupported o_nextcmd received: %u, will skip this message from node %d", + p->opts.o_nextcmd, nodeid); + return -1; + } + + return 0; +} + +static void dlm_midcomms_receive_buffer_3_2(const union dlm_packet *p, int nodeid) +{ + uint16_t msglen = le16_to_cpu(p->header.h_length); + struct midcomms_node *node; + uint32_t seq; + int ret, idx; + + idx = srcu_read_lock(&nodes_srcu); + node = nodeid2node(nodeid); + if (WARN_ON_ONCE(!node)) + goto out; + + switch (node->version) { + case DLM_VERSION_NOT_SET: + node->version = DLM_VERSION_3_2; + wake_up(&node->shutdown_wait); + log_print("version 0x%08x for node %d detected", DLM_VERSION_3_2, + node->nodeid); + + spin_lock(&node->state_lock); + switch (node->state) { + case DLM_CLOSED: + node->state = DLM_ESTABLISHED; + pr_debug("switch node %d to state %s\n", + node->nodeid, dlm_state_str(node->state)); + break; + default: + break; + } + spin_unlock(&node->state_lock); + + break; + case DLM_VERSION_3_2: + break; + default: + log_print_ratelimited("version mismatch detected, assumed 0x%08x but node %d has 0x%08x", + DLM_VERSION_3_2, node->nodeid, node->version); + goto out; + } + + switch (p->header.h_cmd) { + case DLM_RCOM: + /* these rcom message we use to determine version. + * they have their own retransmission handling and + * are the first messages of dlm. + * + * length already checked. + */ + switch (p->rcom.rc_type) { + case cpu_to_le32(DLM_RCOM_NAMES): + fallthrough; + case cpu_to_le32(DLM_RCOM_NAMES_REPLY): + fallthrough; + case cpu_to_le32(DLM_RCOM_STATUS): + fallthrough; + case cpu_to_le32(DLM_RCOM_STATUS_REPLY): + break; + default: + log_print("unsupported rcom type received: %u, will skip this message from node %d", + le32_to_cpu(p->rcom.rc_type), nodeid); + goto out; + } + + WARN_ON_ONCE(test_bit(DLM_NODE_FLAG_STOP_RX, &node->flags)); + dlm_receive_buffer(p, nodeid); + break; + case DLM_OPTS: + seq = le32_to_cpu(p->header.u.h_seq); + + ret = dlm_opts_check_msglen(p, msglen, nodeid); + if (ret < 0) { + log_print("opts msg too small: %u, will skip this message from node %d", + msglen, nodeid); + goto out; + } + + p = (union dlm_packet *)((unsigned char *)p->opts.o_opts + + le16_to_cpu(p->opts.o_optlen)); + + /* recheck inner msglen just if it's not garbage */ + msglen = le16_to_cpu(p->header.h_length); + switch (p->header.h_cmd) { + case DLM_RCOM: + if (msglen < sizeof(struct dlm_rcom)) { + log_print("inner rcom msg too small: %u, will skip this message from node %d", + msglen, nodeid); + goto out; + } + + break; + case DLM_MSG: + if (msglen < sizeof(struct dlm_message)) { + log_print("inner msg too small: %u, will skip this message from node %d", + msglen, nodeid); + goto out; + } + + break; + case DLM_FIN: + if (msglen < sizeof(struct dlm_header)) { + log_print("inner fin too small: %u, will skip this message from node %d", + msglen, nodeid); + goto out; + } + + break; + default: + log_print("unsupported inner h_cmd received: %u, will skip this message from node %d", + msglen, nodeid); + goto out; + } + + dlm_midcomms_receive_buffer(p, node, seq); + break; + case DLM_ACK: + seq = le32_to_cpu(p->header.u.h_seq); + dlm_receive_ack(node, seq); + break; + default: + log_print("unsupported h_cmd received: %u, will skip this message from node %d", + p->header.h_cmd, nodeid); + break; + } + +out: + srcu_read_unlock(&nodes_srcu, idx); +} + +static void dlm_midcomms_receive_buffer_3_1(const union dlm_packet *p, int nodeid) +{ + uint16_t msglen = le16_to_cpu(p->header.h_length); + struct midcomms_node *node; + int idx; + + idx = srcu_read_lock(&nodes_srcu); + node = nodeid2node(nodeid); + if (WARN_ON_ONCE(!node)) { + srcu_read_unlock(&nodes_srcu, idx); + return; + } + + switch (node->version) { + case DLM_VERSION_NOT_SET: + node->version = DLM_VERSION_3_1; + wake_up(&node->shutdown_wait); + log_print("version 0x%08x for node %d detected", DLM_VERSION_3_1, + node->nodeid); + break; + case DLM_VERSION_3_1: + break; + default: + log_print_ratelimited("version mismatch detected, assumed 0x%08x but node %d has 0x%08x", + DLM_VERSION_3_1, node->nodeid, node->version); + srcu_read_unlock(&nodes_srcu, idx); + return; + } + srcu_read_unlock(&nodes_srcu, idx); + + switch (p->header.h_cmd) { + case DLM_RCOM: + /* length already checked */ + break; + case DLM_MSG: + if (msglen < sizeof(struct dlm_message)) { + log_print("msg too small: %u, will skip this message from node %d", + msglen, nodeid); + return; + } + + break; + default: + log_print("unsupported h_cmd received: %u, will skip this message from node %d", + p->header.h_cmd, nodeid); + return; + } + + dlm_receive_buffer(p, nodeid); +} + +int dlm_validate_incoming_buffer(int nodeid, unsigned char *buf, int len) +{ + const unsigned char *ptr = buf; + const struct dlm_header *hd; + uint16_t msglen; + int ret = 0; + + while (len >= sizeof(struct dlm_header)) { + hd = (struct dlm_header *)ptr; + + /* no message should be more than DLM_MAX_SOCKET_BUFSIZE or + * less than dlm_header size. + * + * Some messages does not have a 8 byte length boundary yet + * which can occur in a unaligned memory access of some dlm + * messages. However this problem need to be fixed at the + * sending side, for now it seems nobody run into architecture + * related issues yet but it slows down some processing. + * Fixing this issue should be scheduled in future by doing + * the next major version bump. + */ + msglen = le16_to_cpu(hd->h_length); + if (msglen > DLM_MAX_SOCKET_BUFSIZE || + msglen < sizeof(struct dlm_header)) { + log_print("received invalid length header: %u from node %d, will abort message parsing", + msglen, nodeid); + return -EBADMSG; + } + + /* caller will take care that leftover + * will be parsed next call with more data + */ + if (msglen > len) + break; + + ret += msglen; + len -= msglen; + ptr += msglen; + } + + return ret; +} + +/* + * Called from the low-level comms layer to process a buffer of + * commands. + */ +int dlm_process_incoming_buffer(int nodeid, unsigned char *buf, int len) +{ + const unsigned char *ptr = buf; + const struct dlm_header *hd; + uint16_t msglen; + int ret = 0; + + while (len >= sizeof(struct dlm_header)) { + hd = (struct dlm_header *)ptr; + + msglen = le16_to_cpu(hd->h_length); + if (msglen > len) + break; + + switch (hd->h_version) { + case cpu_to_le32(DLM_VERSION_3_1): + dlm_midcomms_receive_buffer_3_1((const union dlm_packet *)ptr, nodeid); + break; + case cpu_to_le32(DLM_VERSION_3_2): + dlm_midcomms_receive_buffer_3_2((const union dlm_packet *)ptr, nodeid); + break; + default: + log_print("received invalid version header: %u from node %d, will skip this message", + le32_to_cpu(hd->h_version), nodeid); + break; + } + + ret += msglen; + len -= msglen; + ptr += msglen; + } + + return ret; +} + +void dlm_midcomms_unack_msg_resend(int nodeid) +{ + struct midcomms_node *node; + struct dlm_mhandle *mh; + int idx, ret; + + idx = srcu_read_lock(&nodes_srcu); + node = nodeid2node(nodeid); + if (WARN_ON_ONCE(!node)) { + srcu_read_unlock(&nodes_srcu, idx); + return; + } + + /* old protocol, we don't support to retransmit on failure */ + switch (node->version) { + case DLM_VERSION_3_2: + break; + default: + srcu_read_unlock(&nodes_srcu, idx); + return; + } + + rcu_read_lock(); + list_for_each_entry_rcu(mh, &node->send_queue, list) { + if (!mh->committed) + continue; + + ret = dlm_lowcomms_resend_msg(mh->msg); + if (!ret) + log_print_ratelimited("retransmit dlm msg, seq %u, nodeid %d", + mh->seq, node->nodeid); + } + rcu_read_unlock(); + srcu_read_unlock(&nodes_srcu, idx); +} + +static void dlm_fill_opts_header(struct dlm_opts *opts, uint16_t inner_len, + uint32_t seq) +{ + opts->o_header.h_cmd = DLM_OPTS; + opts->o_header.h_version = cpu_to_le32(DLM_HEADER_MAJOR | DLM_HEADER_MINOR); + opts->o_header.h_nodeid = cpu_to_le32(dlm_our_nodeid()); + opts->o_header.h_length = cpu_to_le16(DLM_MIDCOMMS_OPT_LEN + inner_len); + opts->o_header.u.h_seq = cpu_to_le32(seq); +} + +static void midcomms_new_msg_cb(void *data) +{ + struct dlm_mhandle *mh = data; + + atomic_inc(&mh->node->send_queue_cnt); + + spin_lock_bh(&mh->node->send_queue_lock); + /* need to be locked with list_add_tail_rcu() because list is ordered */ + mh->seq = atomic_fetch_inc(&mh->node->seq_send); + list_add_tail_rcu(&mh->list, &mh->node->send_queue); + spin_unlock_bh(&mh->node->send_queue_lock); +} + +static struct dlm_msg *dlm_midcomms_get_msg_3_2(struct dlm_mhandle *mh, int nodeid, + int len, char **ppc) +{ + struct dlm_opts *opts; + struct dlm_msg *msg; + + msg = dlm_lowcomms_new_msg(nodeid, len + DLM_MIDCOMMS_OPT_LEN, + ppc, midcomms_new_msg_cb, mh); + if (!msg) + return NULL; + + opts = (struct dlm_opts *)*ppc; + mh->opts = opts; + + /* add possible options here */ + dlm_fill_opts_header(opts, len, mh->seq); + + *ppc += sizeof(*opts); + mh->inner_p = (const union dlm_packet *)*ppc; + return msg; +} + +/* avoid false positive for nodes_srcu, unlock happens in + * dlm_midcomms_commit_mhandle which is a must call if success + */ +#ifndef __CHECKER__ +struct dlm_mhandle *dlm_midcomms_get_mhandle(int nodeid, int len, char **ppc) +{ + struct midcomms_node *node; + struct dlm_mhandle *mh; + struct dlm_msg *msg; + int idx; + + idx = srcu_read_lock(&nodes_srcu); + node = nodeid2node(nodeid); + if (WARN_ON_ONCE(!node)) + goto err; + + /* this is a bug, however we going on and hope it will be resolved */ + WARN_ON_ONCE(test_bit(DLM_NODE_FLAG_STOP_TX, &node->flags)); + + mh = dlm_allocate_mhandle(); + if (!mh) + goto err; + + mh->committed = false; + mh->ack_rcv = NULL; + mh->idx = idx; + mh->node = node; + + switch (node->version) { + case DLM_VERSION_3_1: + msg = dlm_lowcomms_new_msg(nodeid, len, ppc, NULL, NULL); + if (!msg) { + dlm_free_mhandle(mh); + goto err; + } + + break; + case DLM_VERSION_3_2: + /* send ack back if necessary */ + dlm_send_ack_threshold(node, DLM_SEND_ACK_BACK_MSG_THRESHOLD); + + msg = dlm_midcomms_get_msg_3_2(mh, nodeid, len, ppc); + if (!msg) { + dlm_free_mhandle(mh); + goto err; + } + break; + default: + dlm_free_mhandle(mh); + WARN_ON_ONCE(1); + goto err; + } + + mh->msg = msg; + + /* keep in mind that is a must to call + * dlm_midcomms_commit_msg() which releases + * nodes_srcu using mh->idx which is assumed + * here that the application will call it. + */ + return mh; + +err: + srcu_read_unlock(&nodes_srcu, idx); + return NULL; +} +#endif + +static void dlm_midcomms_commit_msg_3_2_trace(const struct dlm_mhandle *mh, + const void *name, int namelen) +{ + switch (mh->inner_p->header.h_cmd) { + case DLM_MSG: + trace_dlm_send_message(mh->node->nodeid, mh->seq, + &mh->inner_p->message, + name, namelen); + break; + case DLM_RCOM: + trace_dlm_send_rcom(mh->node->nodeid, mh->seq, + &mh->inner_p->rcom); + break; + default: + /* nothing to trace */ + break; + } +} + +static void dlm_midcomms_commit_msg_3_2(struct dlm_mhandle *mh, + const void *name, int namelen) +{ + /* nexthdr chain for fast lookup */ + mh->opts->o_nextcmd = mh->inner_p->header.h_cmd; + mh->committed = true; + dlm_midcomms_commit_msg_3_2_trace(mh, name, namelen); + dlm_lowcomms_commit_msg(mh->msg); +} + +/* avoid false positive for nodes_srcu, lock was happen in + * dlm_midcomms_get_mhandle + */ +#ifndef __CHECKER__ +void dlm_midcomms_commit_mhandle(struct dlm_mhandle *mh, + const void *name, int namelen) +{ + + switch (mh->node->version) { + case DLM_VERSION_3_1: + srcu_read_unlock(&nodes_srcu, mh->idx); + + dlm_lowcomms_commit_msg(mh->msg); + dlm_lowcomms_put_msg(mh->msg); + /* mh is not part of rcu list in this case */ + dlm_free_mhandle(mh); + break; + case DLM_VERSION_3_2: + /* held rcu read lock here, because we sending the + * dlm message out, when we do that we could receive + * an ack back which releases the mhandle and we + * get a use after free. + */ + rcu_read_lock(); + dlm_midcomms_commit_msg_3_2(mh, name, namelen); + srcu_read_unlock(&nodes_srcu, mh->idx); + rcu_read_unlock(); + break; + default: + srcu_read_unlock(&nodes_srcu, mh->idx); + WARN_ON_ONCE(1); + break; + } +} +#endif + +int dlm_midcomms_start(void) +{ + return dlm_lowcomms_start(); +} + +void dlm_midcomms_stop(void) +{ + dlm_lowcomms_stop(); +} + +void dlm_midcomms_init(void) +{ + int i; + + for (i = 0; i < CONN_HASH_SIZE; i++) + INIT_HLIST_HEAD(&node_hash[i]); + + dlm_lowcomms_init(); +} + +static void midcomms_node_release(struct rcu_head *rcu) +{ + struct midcomms_node *node = container_of(rcu, struct midcomms_node, rcu); + + WARN_ON_ONCE(atomic_read(&node->send_queue_cnt)); + dlm_send_queue_flush(node); + kfree(node); +} + +void dlm_midcomms_exit(void) +{ + struct midcomms_node *node; + int i, idx; + + idx = srcu_read_lock(&nodes_srcu); + for (i = 0; i < CONN_HASH_SIZE; i++) { + hlist_for_each_entry_srcu(node, &node_hash[i], hlist, + srcu_read_lock_held(&nodes_srcu)) { + dlm_delete_debug_comms_file(node->debugfs); + + spin_lock(&nodes_lock); + hlist_del_rcu(&node->hlist); + spin_unlock(&nodes_lock); + + call_srcu(&nodes_srcu, &node->rcu, midcomms_node_release); + } + } + srcu_read_unlock(&nodes_srcu, idx); + + dlm_lowcomms_exit(); +} + +static void dlm_act_fin_ack_rcv(struct midcomms_node *node) +{ + spin_lock_bh(&node->state_lock); + pr_debug("receive active fin ack from node %d with state %s\n", + node->nodeid, dlm_state_str(node->state)); + + switch (node->state) { + case DLM_FIN_WAIT1: + node->state = DLM_FIN_WAIT2; + pr_debug("switch node %d to state %s\n", + node->nodeid, dlm_state_str(node->state)); + break; + case DLM_CLOSING: + midcomms_node_reset(node); + pr_debug("switch node %d to state %s\n", + node->nodeid, dlm_state_str(node->state)); + break; + case DLM_CLOSED: + /* not valid but somehow we got what we want */ + wake_up(&node->shutdown_wait); + break; + default: + spin_unlock_bh(&node->state_lock); + log_print("%s: unexpected state: %d", + __func__, node->state); + WARN_ON_ONCE(1); + return; + } + spin_unlock_bh(&node->state_lock); +} + +void dlm_midcomms_add_member(int nodeid) +{ + struct midcomms_node *node; + int idx; + + idx = srcu_read_lock(&nodes_srcu); + node = nodeid2node(nodeid); + if (WARN_ON_ONCE(!node)) { + srcu_read_unlock(&nodes_srcu, idx); + return; + } + + spin_lock_bh(&node->state_lock); + if (!node->users) { + pr_debug("receive add member from node %d with state %s\n", + node->nodeid, dlm_state_str(node->state)); + switch (node->state) { + case DLM_ESTABLISHED: + break; + case DLM_CLOSED: + node->state = DLM_ESTABLISHED; + pr_debug("switch node %d to state %s\n", + node->nodeid, dlm_state_str(node->state)); + break; + default: + /* some invalid state passive shutdown + * was failed, we try to reset and + * hope it will go on. + */ + log_print("reset node %d because shutdown stuck", + node->nodeid); + + midcomms_node_reset(node); + node->state = DLM_ESTABLISHED; + break; + } + } + + node->users++; + pr_debug("node %d users inc count %d\n", nodeid, node->users); + spin_unlock_bh(&node->state_lock); + + srcu_read_unlock(&nodes_srcu, idx); +} + +void dlm_midcomms_remove_member(int nodeid) +{ + struct midcomms_node *node; + int idx; + + idx = srcu_read_lock(&nodes_srcu); + node = nodeid2node(nodeid); + /* in case of dlm_midcomms_close() removes node */ + if (!node) { + srcu_read_unlock(&nodes_srcu, idx); + return; + } + + spin_lock_bh(&node->state_lock); + /* case of dlm_midcomms_addr() created node but + * was not added before because dlm_midcomms_close() + * removed the node + */ + if (!node->users) { + spin_unlock_bh(&node->state_lock); + srcu_read_unlock(&nodes_srcu, idx); + return; + } + + node->users--; + pr_debug("node %d users dec count %d\n", nodeid, node->users); + + /* hitting users count to zero means the + * other side is running dlm_midcomms_stop() + * we meet us to have a clean disconnect. + */ + if (node->users == 0) { + pr_debug("receive remove member from node %d with state %s\n", + node->nodeid, dlm_state_str(node->state)); + switch (node->state) { + case DLM_ESTABLISHED: + break; + case DLM_CLOSE_WAIT: + /* passive shutdown DLM_LAST_ACK case 2 */ + node->state = DLM_LAST_ACK; + pr_debug("switch node %d to state %s case 2\n", + node->nodeid, dlm_state_str(node->state)); + set_bit(DLM_NODE_FLAG_STOP_RX, &node->flags); + dlm_send_fin(node, dlm_pas_fin_ack_rcv); + break; + case DLM_LAST_ACK: + /* probably receive fin caught it, do nothing */ + break; + case DLM_CLOSED: + /* already gone, do nothing */ + break; + default: + log_print("%s: unexpected state: %d", + __func__, node->state); + break; + } + } + spin_unlock_bh(&node->state_lock); + + srcu_read_unlock(&nodes_srcu, idx); +} + +void dlm_midcomms_version_wait(void) +{ + struct midcomms_node *node; + int i, idx, ret; + + idx = srcu_read_lock(&nodes_srcu); + for (i = 0; i < CONN_HASH_SIZE; i++) { + hlist_for_each_entry_srcu(node, &node_hash[i], hlist, + srcu_read_lock_held(&nodes_srcu)) { + ret = wait_event_timeout(node->shutdown_wait, + node->version != DLM_VERSION_NOT_SET || + node->state == DLM_CLOSED || + test_bit(DLM_NODE_FLAG_CLOSE, &node->flags), + DLM_SHUTDOWN_TIMEOUT); + if (!ret || test_bit(DLM_NODE_FLAG_CLOSE, &node->flags)) + pr_debug("version wait timed out for node %d with state %s\n", + node->nodeid, dlm_state_str(node->state)); + } + } + srcu_read_unlock(&nodes_srcu, idx); +} + +static void midcomms_shutdown(struct midcomms_node *node) +{ + int ret; + + /* old protocol, we don't wait for pending operations */ + switch (node->version) { + case DLM_VERSION_3_2: + break; + default: + return; + } + + spin_lock_bh(&node->state_lock); + pr_debug("receive active shutdown for node %d with state %s\n", + node->nodeid, dlm_state_str(node->state)); + switch (node->state) { + case DLM_ESTABLISHED: + node->state = DLM_FIN_WAIT1; + pr_debug("switch node %d to state %s case 2\n", + node->nodeid, dlm_state_str(node->state)); + dlm_send_fin(node, dlm_act_fin_ack_rcv); + break; + case DLM_CLOSED: + /* we have what we want */ + break; + default: + /* busy to enter DLM_FIN_WAIT1, wait until passive + * done in shutdown_wait to enter DLM_CLOSED. + */ + break; + } + spin_unlock_bh(&node->state_lock); + + if (DLM_DEBUG_FENCE_TERMINATION) + msleep(5000); + + /* wait for other side dlm + fin */ + ret = wait_event_timeout(node->shutdown_wait, + node->state == DLM_CLOSED || + test_bit(DLM_NODE_FLAG_CLOSE, &node->flags), + DLM_SHUTDOWN_TIMEOUT); + if (!ret) + pr_debug("active shutdown timed out for node %d with state %s\n", + node->nodeid, dlm_state_str(node->state)); + else + pr_debug("active shutdown done for node %d with state %s\n", + node->nodeid, dlm_state_str(node->state)); +} + +void dlm_midcomms_shutdown(void) +{ + struct midcomms_node *node; + int i, idx; + + mutex_lock(&close_lock); + idx = srcu_read_lock(&nodes_srcu); + for (i = 0; i < CONN_HASH_SIZE; i++) { + hlist_for_each_entry_srcu(node, &node_hash[i], hlist, + srcu_read_lock_held(&nodes_srcu)) { + midcomms_shutdown(node); + } + } + + dlm_lowcomms_shutdown(); + + for (i = 0; i < CONN_HASH_SIZE; i++) { + hlist_for_each_entry_srcu(node, &node_hash[i], hlist, + srcu_read_lock_held(&nodes_srcu)) { + midcomms_node_reset(node); + } + } + srcu_read_unlock(&nodes_srcu, idx); + mutex_unlock(&close_lock); +} + +int dlm_midcomms_close(int nodeid) +{ + struct midcomms_node *node; + int idx, ret; + + idx = srcu_read_lock(&nodes_srcu); + /* Abort pending close/remove operation */ + node = nodeid2node(nodeid); + if (node) { + /* let shutdown waiters leave */ + set_bit(DLM_NODE_FLAG_CLOSE, &node->flags); + wake_up(&node->shutdown_wait); + } + srcu_read_unlock(&nodes_srcu, idx); + + synchronize_srcu(&nodes_srcu); + + mutex_lock(&close_lock); + idx = srcu_read_lock(&nodes_srcu); + node = nodeid2node(nodeid); + if (!node) { + srcu_read_unlock(&nodes_srcu, idx); + mutex_unlock(&close_lock); + return dlm_lowcomms_close(nodeid); + } + + ret = dlm_lowcomms_close(nodeid); + dlm_delete_debug_comms_file(node->debugfs); + + spin_lock_bh(&nodes_lock); + hlist_del_rcu(&node->hlist); + spin_unlock_bh(&nodes_lock); + srcu_read_unlock(&nodes_srcu, idx); + + /* wait that all readers left until flush send queue */ + synchronize_srcu(&nodes_srcu); + + /* drop all pending dlm messages, this is fine as + * this function get called when the node is fenced + */ + dlm_send_queue_flush(node); + + call_srcu(&nodes_srcu, &node->rcu, midcomms_node_release); + mutex_unlock(&close_lock); + + return ret; +} + +/* debug functionality to send raw dlm msg from user space */ +struct dlm_rawmsg_data { + struct midcomms_node *node; + void *buf; +}; + +static void midcomms_new_rawmsg_cb(void *data) +{ + struct dlm_rawmsg_data *rd = data; + struct dlm_header *h = rd->buf; + + switch (h->h_version) { + case cpu_to_le32(DLM_VERSION_3_1): + break; + default: + switch (h->h_cmd) { + case DLM_OPTS: + if (!h->u.h_seq) + h->u.h_seq = cpu_to_le32(atomic_fetch_inc(&rd->node->seq_send)); + break; + default: + break; + } + break; + } +} + +int dlm_midcomms_rawmsg_send(struct midcomms_node *node, void *buf, + int buflen) +{ + struct dlm_rawmsg_data rd; + struct dlm_msg *msg; + char *msgbuf; + + rd.node = node; + rd.buf = buf; + + msg = dlm_lowcomms_new_msg(node->nodeid, buflen, &msgbuf, + midcomms_new_rawmsg_cb, &rd); + if (!msg) + return -ENOMEM; + + memcpy(msgbuf, buf, buflen); + dlm_lowcomms_commit_msg(msg); + return 0; +} + diff --git a/fs/dlm/midcomms.h b/fs/dlm/midcomms.h new file mode 100644 index 000000000..7fad1d170 --- /dev/null +++ b/fs/dlm/midcomms.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#ifndef __MIDCOMMS_DOT_H__ +#define __MIDCOMMS_DOT_H__ + +struct midcomms_node; + +int dlm_validate_incoming_buffer(int nodeid, unsigned char *buf, int len); +int dlm_process_incoming_buffer(int nodeid, unsigned char *buf, int buflen); +struct dlm_mhandle *dlm_midcomms_get_mhandle(int nodeid, int len, char **ppc); +void dlm_midcomms_commit_mhandle(struct dlm_mhandle *mh, const void *name, + int namelen); +int dlm_midcomms_addr(int nodeid, struct sockaddr_storage *addr); +void dlm_midcomms_version_wait(void); +int dlm_midcomms_close(int nodeid); +int dlm_midcomms_start(void); +void dlm_midcomms_stop(void); +void dlm_midcomms_init(void); +void dlm_midcomms_exit(void); +void dlm_midcomms_shutdown(void); +void dlm_midcomms_add_member(int nodeid); +void dlm_midcomms_remove_member(int nodeid); +void dlm_midcomms_unack_msg_resend(int nodeid); +const char *dlm_midcomms_state(struct midcomms_node *node); +unsigned long dlm_midcomms_flags(struct midcomms_node *node); +int dlm_midcomms_send_queue_cnt(struct midcomms_node *node); +uint32_t dlm_midcomms_version(struct midcomms_node *node); +int dlm_midcomms_rawmsg_send(struct midcomms_node *node, void *buf, + int buflen); +struct kmem_cache *dlm_midcomms_cache_create(void); + +#endif /* __MIDCOMMS_DOT_H__ */ + diff --git a/fs/dlm/plock.c b/fs/dlm/plock.c new file mode 100644 index 000000000..e9598b3fe --- /dev/null +++ b/fs/dlm/plock.c @@ -0,0 +1,630 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2005-2008 Red Hat, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "dlm_internal.h" +#include "lockspace.h" + +static DEFINE_SPINLOCK(ops_lock); +static LIST_HEAD(send_list); +static LIST_HEAD(recv_list); +static DECLARE_WAIT_QUEUE_HEAD(send_wq); +static DECLARE_WAIT_QUEUE_HEAD(recv_wq); + +struct plock_async_data { + void *fl; + void *file; + struct file_lock flc; + int (*callback)(struct file_lock *fl, int result); +}; + +struct plock_op { + struct list_head list; + int done; + struct dlm_plock_info info; + /* if set indicates async handling */ + struct plock_async_data *data; +}; + +static inline void set_version(struct dlm_plock_info *info) +{ + info->version[0] = DLM_PLOCK_VERSION_MAJOR; + info->version[1] = DLM_PLOCK_VERSION_MINOR; + info->version[2] = DLM_PLOCK_VERSION_PATCH; +} + +static struct plock_op *plock_lookup_waiter(const struct dlm_plock_info *info) +{ + struct plock_op *op = NULL, *iter; + + list_for_each_entry(iter, &recv_list, list) { + if (iter->info.fsid == info->fsid && + iter->info.number == info->number && + iter->info.owner == info->owner && + iter->info.pid == info->pid && + iter->info.start == info->start && + iter->info.end == info->end && + iter->info.ex == info->ex && + iter->info.wait) { + op = iter; + break; + } + } + + return op; +} + +static int check_version(struct dlm_plock_info *info) +{ + if ((DLM_PLOCK_VERSION_MAJOR != info->version[0]) || + (DLM_PLOCK_VERSION_MINOR < info->version[1])) { + log_print("plock device version mismatch: " + "kernel (%u.%u.%u), user (%u.%u.%u)", + DLM_PLOCK_VERSION_MAJOR, + DLM_PLOCK_VERSION_MINOR, + DLM_PLOCK_VERSION_PATCH, + info->version[0], + info->version[1], + info->version[2]); + return -EINVAL; + } + return 0; +} + +static void dlm_release_plock_op(struct plock_op *op) +{ + kfree(op->data); + kfree(op); +} + +static void send_op(struct plock_op *op) +{ + set_version(&op->info); + spin_lock(&ops_lock); + list_add_tail(&op->list, &send_list); + spin_unlock(&ops_lock); + wake_up(&send_wq); +} + +static int do_lock_cancel(const struct dlm_plock_info *orig_info) +{ + struct plock_op *op; + int rv; + + op = kzalloc_obj(*op, GFP_NOFS); + if (!op) + return -ENOMEM; + + op->info = *orig_info; + op->info.optype = DLM_PLOCK_OP_CANCEL; + op->info.wait = 0; + + send_op(op); + wait_event(recv_wq, (op->done != 0)); + + rv = op->info.rv; + + dlm_release_plock_op(op); + return rv; +} + +int dlm_posix_lock(dlm_lockspace_t *lockspace, u64 number, struct file *file, + int cmd, struct file_lock *fl) +{ + struct plock_async_data *op_data; + struct dlm_ls *ls; + struct plock_op *op; + int rv; + + ls = dlm_find_lockspace_local(lockspace); + if (!ls) + return -EINVAL; + + op = kzalloc_obj(*op, GFP_NOFS); + if (!op) { + rv = -ENOMEM; + goto out; + } + + op->info.optype = DLM_PLOCK_OP_LOCK; + op->info.pid = fl->c.flc_pid; + op->info.ex = lock_is_write(fl); + op->info.wait = !!(fl->c.flc_flags & FL_SLEEP); + op->info.fsid = ls->ls_global_id; + op->info.number = number; + op->info.start = fl->fl_start; + op->info.end = fl->fl_end; + op->info.owner = (__u64)(long) fl->c.flc_owner; + /* async handling */ + if (fl->fl_lmops && fl->fl_lmops->lm_grant) { + op_data = kzalloc_obj(*op_data, GFP_NOFS); + if (!op_data) { + dlm_release_plock_op(op); + rv = -ENOMEM; + goto out; + } + + op_data->callback = fl->fl_lmops->lm_grant; + locks_init_lock(&op_data->flc); + locks_copy_lock(&op_data->flc, fl); + op_data->fl = fl; + op_data->file = file; + + op->data = op_data; + + send_op(op); + rv = FILE_LOCK_DEFERRED; + goto out; + } + + send_op(op); + + if (op->info.wait) { + rv = wait_event_interruptible(recv_wq, (op->done != 0)); + if (rv == -ERESTARTSYS) { + spin_lock(&ops_lock); + /* recheck under ops_lock if we got a done != 0, + * if so this interrupt case should be ignored + */ + if (op->done != 0) { + spin_unlock(&ops_lock); + goto do_lock_wait; + } + spin_unlock(&ops_lock); + + rv = do_lock_cancel(&op->info); + switch (rv) { + case 0: + /* waiter was deleted in user space, answer will never come + * remove original request. The original request must be + * on recv_list because the answer of do_lock_cancel() + * synchronized it. + */ + spin_lock(&ops_lock); + list_del(&op->list); + spin_unlock(&ops_lock); + rv = -EINTR; + break; + case -ENOENT: + /* cancellation wasn't successful but op should be done */ + fallthrough; + default: + /* internal error doing cancel we need to wait */ + goto wait; + } + + log_debug(ls, "%s: wait interrupted %x %llx pid %d", + __func__, ls->ls_global_id, + (unsigned long long)number, op->info.pid); + dlm_release_plock_op(op); + goto out; + } + } else { +wait: + wait_event(recv_wq, (op->done != 0)); + } + +do_lock_wait: + + WARN_ON(!list_empty(&op->list)); + + rv = op->info.rv; + + if (!rv) { + if (locks_lock_file_wait(file, fl) < 0) + log_error(ls, "dlm_posix_lock: vfs lock error %llx", + (unsigned long long)number); + } + + dlm_release_plock_op(op); +out: + dlm_put_lockspace(ls); + return rv; +} +EXPORT_SYMBOL_GPL(dlm_posix_lock); + +/* Returns failure iff a successful lock operation should be canceled */ +static int dlm_plock_callback(struct plock_op *op) +{ + struct plock_async_data *op_data = op->data; + struct file *file; + struct file_lock *fl; + struct file_lock *flc; + int (*notify)(struct file_lock *fl, int result) = NULL; + int rv = 0; + + WARN_ON(!list_empty(&op->list)); + + /* check if the following 2 are still valid or make a copy */ + file = op_data->file; + flc = &op_data->flc; + fl = op_data->fl; + notify = op_data->callback; + + if (op->info.rv) { + notify(fl, op->info.rv); + goto out; + } + + /* got fs lock; bookkeep locally as well: */ + flc->c.flc_flags &= ~FL_SLEEP; + if (posix_lock_file(file, flc, NULL)) { + /* + * This can only happen in the case of kmalloc() failure. + * The filesystem's own lock is the authoritative lock, + * so a failure to get the lock locally is not a disaster. + * As long as the fs cannot reliably cancel locks (especially + * in a low-memory situation), we're better off ignoring + * this failure than trying to recover. + */ + log_print("dlm_plock_callback: vfs lock error %llx file %p fl %p", + (unsigned long long)op->info.number, file, fl); + } + + rv = notify(fl, 0); + if (rv) { + /* XXX: We need to cancel the fs lock here: */ + log_print("%s: lock granted after lock request failed; dangling lock!", + __func__); + goto out; + } + +out: + dlm_release_plock_op(op); + return rv; +} + +int dlm_posix_unlock(dlm_lockspace_t *lockspace, u64 number, struct file *file, + struct file_lock *fl) +{ + struct dlm_ls *ls; + struct plock_op *op; + int rv; + unsigned char saved_flags = fl->c.flc_flags; + + ls = dlm_find_lockspace_local(lockspace); + if (!ls) + return -EINVAL; + + op = kzalloc_obj(*op, GFP_NOFS); + if (!op) { + rv = -ENOMEM; + goto out; + } + + /* cause the vfs unlock to return ENOENT if lock is not found */ + fl->c.flc_flags |= FL_EXISTS; + + rv = locks_lock_file_wait(file, fl); + if (rv == -ENOENT) { + rv = 0; + goto out_free; + } + if (rv < 0) { + log_error(ls, "dlm_posix_unlock: vfs unlock error %d %llx", + rv, (unsigned long long)number); + } + + op->info.optype = DLM_PLOCK_OP_UNLOCK; + op->info.pid = fl->c.flc_pid; + op->info.fsid = ls->ls_global_id; + op->info.number = number; + op->info.start = fl->fl_start; + op->info.end = fl->fl_end; + op->info.owner = (__u64)(long) fl->c.flc_owner; + + if (fl->c.flc_flags & FL_CLOSE) { + op->info.flags |= DLM_PLOCK_FL_CLOSE; + send_op(op); + rv = 0; + goto out; + } + + send_op(op); + wait_event(recv_wq, (op->done != 0)); + + WARN_ON(!list_empty(&op->list)); + + rv = op->info.rv; + + if (rv == -ENOENT) + rv = 0; + +out_free: + dlm_release_plock_op(op); +out: + dlm_put_lockspace(ls); + fl->c.flc_flags = saved_flags; + return rv; +} +EXPORT_SYMBOL_GPL(dlm_posix_unlock); + +/* + * NOTE: This implementation can only handle async lock requests as nfs + * do it. It cannot handle cancellation of a pending lock request sitting + * in wait_event(), but for now only nfs is the only user local kernel + * user. + */ +int dlm_posix_cancel(dlm_lockspace_t *lockspace, u64 number, struct file *file, + struct file_lock *fl) +{ + struct dlm_plock_info info; + struct plock_op *op; + struct dlm_ls *ls; + int rv; + + /* this only works for async request for now and nfs is the only + * kernel user right now. + */ + if (WARN_ON_ONCE(!fl->fl_lmops || !fl->fl_lmops->lm_grant)) + return -EOPNOTSUPP; + + ls = dlm_find_lockspace_local(lockspace); + if (!ls) + return -EINVAL; + + memset(&info, 0, sizeof(info)); + info.pid = fl->c.flc_pid; + info.ex = lock_is_write(fl); + info.fsid = ls->ls_global_id; + dlm_put_lockspace(ls); + info.number = number; + info.start = fl->fl_start; + info.end = fl->fl_end; + info.owner = (__u64)(long) fl->c.flc_owner; + + rv = do_lock_cancel(&info); + switch (rv) { + case 0: + spin_lock(&ops_lock); + /* lock request to cancel must be on recv_list because + * do_lock_cancel() synchronizes it. + */ + op = plock_lookup_waiter(&info); + if (WARN_ON_ONCE(!op)) { + spin_unlock(&ops_lock); + rv = -ENOLCK; + break; + } + + list_del(&op->list); + spin_unlock(&ops_lock); + WARN_ON(op->info.optype != DLM_PLOCK_OP_LOCK); + op->data->callback(op->data->fl, -EINTR); + dlm_release_plock_op(op); + rv = -EINTR; + break; + case -ENOENT: + /* if cancel wasn't successful we probably were to late + * or it was a non-blocking lock request, so just unlock it. + */ + rv = dlm_posix_unlock(lockspace, number, file, fl); + break; + default: + break; + } + + return rv; +} +EXPORT_SYMBOL_GPL(dlm_posix_cancel); + +int dlm_posix_get(dlm_lockspace_t *lockspace, u64 number, struct file *file, + struct file_lock *fl) +{ + struct dlm_ls *ls; + struct plock_op *op; + int rv; + + ls = dlm_find_lockspace_local(lockspace); + if (!ls) + return -EINVAL; + + op = kzalloc_obj(*op, GFP_NOFS); + if (!op) { + rv = -ENOMEM; + goto out; + } + + op->info.optype = DLM_PLOCK_OP_GET; + op->info.pid = fl->c.flc_pid; + op->info.ex = lock_is_write(fl); + op->info.fsid = ls->ls_global_id; + op->info.number = number; + op->info.start = fl->fl_start; + op->info.end = fl->fl_end; + op->info.owner = (__u64)(long) fl->c.flc_owner; + + send_op(op); + wait_event(recv_wq, (op->done != 0)); + + WARN_ON(!list_empty(&op->list)); + + /* info.rv from userspace is 1 for conflict, 0 for no-conflict, + -ENOENT if there are no locks on the file */ + + rv = op->info.rv; + + fl->c.flc_type = F_UNLCK; + if (rv == -ENOENT) + rv = 0; + else if (rv > 0) { + locks_init_lock(fl); + fl->c.flc_type = (op->info.ex) ? F_WRLCK : F_RDLCK; + fl->c.flc_flags = FL_POSIX; + fl->c.flc_pid = op->info.pid; + if (op->info.nodeid != dlm_our_nodeid()) + fl->c.flc_pid = -fl->c.flc_pid; + fl->fl_start = op->info.start; + fl->fl_end = op->info.end; + rv = 0; + } + + dlm_release_plock_op(op); +out: + dlm_put_lockspace(ls); + return rv; +} +EXPORT_SYMBOL_GPL(dlm_posix_get); + +/* a read copies out one plock request from the send list */ +static ssize_t dev_read(struct file *file, char __user *u, size_t count, + loff_t *ppos) +{ + struct dlm_plock_info info; + struct plock_op *op = NULL; + + if (count < sizeof(info)) + return -EINVAL; + + spin_lock(&ops_lock); + if (!list_empty(&send_list)) { + op = list_first_entry(&send_list, struct plock_op, list); + if (op->info.flags & DLM_PLOCK_FL_CLOSE) + list_del(&op->list); + else + list_move_tail(&op->list, &recv_list); + memcpy(&info, &op->info, sizeof(info)); + } + spin_unlock(&ops_lock); + + if (!op) + return -EAGAIN; + + trace_dlm_plock_read(&info); + + /* there is no need to get a reply from userspace for unlocks + that were generated by the vfs cleaning up for a close + (the process did not make an unlock call). */ + + if (op->info.flags & DLM_PLOCK_FL_CLOSE) + dlm_release_plock_op(op); + + if (copy_to_user(u, &info, sizeof(info))) + return -EFAULT; + return sizeof(info); +} + +/* a write copies in one plock result that should match a plock_op + on the recv list */ +static ssize_t dev_write(struct file *file, const char __user *u, size_t count, + loff_t *ppos) +{ + struct plock_op *op = NULL, *iter; + struct dlm_plock_info info; + int do_callback = 0; + + if (count != sizeof(info)) + return -EINVAL; + + if (copy_from_user(&info, u, sizeof(info))) + return -EFAULT; + + trace_dlm_plock_write(&info); + + if (check_version(&info)) + return -EINVAL; + + /* + * The results for waiting ops (SETLKW) can be returned in any + * order, so match all fields to find the op. The results for + * non-waiting ops are returned in the order that they were sent + * to userspace, so match the result with the first non-waiting op. + */ + spin_lock(&ops_lock); + if (info.wait) { + op = plock_lookup_waiter(&info); + } else { + list_for_each_entry(iter, &recv_list, list) { + if (!iter->info.wait && + iter->info.fsid == info.fsid) { + op = iter; + break; + } + } + } + + if (op) { + /* Sanity check that op and info match. */ + if (info.wait) + WARN_ON(op->info.optype != DLM_PLOCK_OP_LOCK); + else + WARN_ON(op->info.number != info.number || + op->info.owner != info.owner || + op->info.optype != info.optype); + + list_del_init(&op->list); + memcpy(&op->info, &info, sizeof(info)); + if (op->data) + do_callback = 1; + else + op->done = 1; + } + spin_unlock(&ops_lock); + + if (op) { + if (do_callback) + dlm_plock_callback(op); + else + wake_up(&recv_wq); + } else + pr_debug("%s: no op %x %llx", __func__, + info.fsid, (unsigned long long)info.number); + return count; +} + +static __poll_t dev_poll(struct file *file, poll_table *wait) +{ + __poll_t mask = 0; + + poll_wait(file, &send_wq, wait); + + spin_lock(&ops_lock); + if (!list_empty(&send_list)) + mask = EPOLLIN | EPOLLRDNORM; + spin_unlock(&ops_lock); + + return mask; +} + +static const struct file_operations dev_fops = { + .read = dev_read, + .write = dev_write, + .poll = dev_poll, + .owner = THIS_MODULE, + .llseek = noop_llseek, +}; + +static struct miscdevice plock_dev_misc = { + .minor = MISC_DYNAMIC_MINOR, + .name = DLM_PLOCK_MISC_NAME, + .fops = &dev_fops +}; + +int dlm_plock_init(void) +{ + int rv; + + rv = misc_register(&plock_dev_misc); + if (rv) + log_print("dlm_plock_init: misc_register failed %d", rv); + return rv; +} + +void dlm_plock_exit(void) +{ + misc_deregister(&plock_dev_misc); + WARN_ON(!list_empty(&send_list)); + WARN_ON(!list_empty(&recv_list)); +} + diff --git a/fs/dlm/rcom.c b/fs/dlm/rcom.c new file mode 100644 index 000000000..be1a71a63 --- /dev/null +++ b/fs/dlm/rcom.c @@ -0,0 +1,691 @@ +// SPDX-License-Identifier: GPL-2.0-only +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2005-2008 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#include "dlm_internal.h" +#include "lockspace.h" +#include "member.h" +#include "lowcomms.h" +#include "midcomms.h" +#include "rcom.h" +#include "recover.h" +#include "dir.h" +#include "config.h" +#include "memory.h" +#include "lock.h" +#include "util.h" + +static int rcom_response(struct dlm_ls *ls) +{ + return test_bit(LSFL_RCOM_READY, &ls->ls_flags); +} + +static void _create_rcom(struct dlm_ls *ls, int to_nodeid, int type, int len, + struct dlm_rcom **rc_ret, char *mb, int mb_len, + uint64_t seq) +{ + struct dlm_rcom *rc; + + rc = (struct dlm_rcom *) mb; + + rc->rc_header.h_version = cpu_to_le32(DLM_HEADER_MAJOR | DLM_HEADER_MINOR); + rc->rc_header.u.h_lockspace = cpu_to_le32(ls->ls_global_id); + rc->rc_header.h_nodeid = cpu_to_le32(dlm_our_nodeid()); + rc->rc_header.h_length = cpu_to_le16(mb_len); + rc->rc_header.h_cmd = DLM_RCOM; + + rc->rc_type = cpu_to_le32(type); + rc->rc_seq = cpu_to_le64(seq); + + *rc_ret = rc; +} + +static int create_rcom(struct dlm_ls *ls, int to_nodeid, int type, int len, + struct dlm_rcom **rc_ret, struct dlm_mhandle **mh_ret, + uint64_t seq) +{ + int mb_len = sizeof(struct dlm_rcom) + len; + struct dlm_mhandle *mh; + char *mb; + + mh = dlm_midcomms_get_mhandle(to_nodeid, mb_len, &mb); + if (!mh) { + log_print("%s to %d type %d len %d ENOBUFS", + __func__, to_nodeid, type, len); + return -ENOBUFS; + } + + _create_rcom(ls, to_nodeid, type, len, rc_ret, mb, mb_len, seq); + *mh_ret = mh; + return 0; +} + +static int create_rcom_stateless(struct dlm_ls *ls, int to_nodeid, int type, + int len, struct dlm_rcom **rc_ret, + struct dlm_msg **msg_ret, uint64_t seq) +{ + int mb_len = sizeof(struct dlm_rcom) + len; + struct dlm_msg *msg; + char *mb; + + msg = dlm_lowcomms_new_msg(to_nodeid, mb_len, &mb, NULL, NULL); + if (!msg) { + log_print("create_rcom to %d type %d len %d ENOBUFS", + to_nodeid, type, len); + return -ENOBUFS; + } + + _create_rcom(ls, to_nodeid, type, len, rc_ret, mb, mb_len, seq); + *msg_ret = msg; + return 0; +} + +static void send_rcom(struct dlm_mhandle *mh, struct dlm_rcom *rc) +{ + dlm_midcomms_commit_mhandle(mh, NULL, 0); +} + +static void send_rcom_stateless(struct dlm_msg *msg, struct dlm_rcom *rc) +{ + dlm_lowcomms_commit_msg(msg); + dlm_lowcomms_put_msg(msg); +} + +static void set_rcom_status(struct dlm_ls *ls, struct rcom_status *rs, + uint32_t flags) +{ + rs->rs_flags = cpu_to_le32(flags); +} + +/* When replying to a status request, a node also sends back its + configuration values. The requesting node then checks that the remote + node is configured the same way as itself. */ + +static void set_rcom_config(struct dlm_ls *ls, struct rcom_config *rf, + uint32_t num_slots) +{ + rf->rf_lvblen = cpu_to_le32(ls->ls_lvblen); + rf->rf_lsflags = cpu_to_le32(ls->ls_exflags); + + rf->rf_our_slot = cpu_to_le16(ls->ls_slot); + rf->rf_num_slots = cpu_to_le16(num_slots); + rf->rf_generation = cpu_to_le32(ls->ls_generation); +} + +static int check_rcom_config(struct dlm_ls *ls, struct dlm_rcom *rc, int nodeid) +{ + struct rcom_config *rf = (struct rcom_config *) rc->rc_buf; + + if ((le32_to_cpu(rc->rc_header.h_version) & 0xFFFF0000) != DLM_HEADER_MAJOR) { + log_error(ls, "version mismatch: %x nodeid %d: %x", + DLM_HEADER_MAJOR | DLM_HEADER_MINOR, nodeid, + le32_to_cpu(rc->rc_header.h_version)); + return -EPROTO; + } + + if (le32_to_cpu(rf->rf_lvblen) != ls->ls_lvblen || + le32_to_cpu(rf->rf_lsflags) != ls->ls_exflags) { + log_error(ls, "config mismatch: %d,%x nodeid %d: %d,%x", + ls->ls_lvblen, ls->ls_exflags, nodeid, + le32_to_cpu(rf->rf_lvblen), + le32_to_cpu(rf->rf_lsflags)); + return -EPROTO; + } + return 0; +} + +static void allow_sync_reply(struct dlm_ls *ls, __le64 *new_seq) +{ + spin_lock_bh(&ls->ls_rcom_spin); + *new_seq = cpu_to_le64(++ls->ls_rcom_seq); + set_bit(LSFL_RCOM_WAIT, &ls->ls_flags); + spin_unlock_bh(&ls->ls_rcom_spin); +} + +static void disallow_sync_reply(struct dlm_ls *ls) +{ + spin_lock_bh(&ls->ls_rcom_spin); + clear_bit(LSFL_RCOM_WAIT, &ls->ls_flags); + clear_bit(LSFL_RCOM_READY, &ls->ls_flags); + spin_unlock_bh(&ls->ls_rcom_spin); +} + +/* + * low nodeid gathers one slot value at a time from each node. + * it sets need_slots=0, and saves rf_our_slot returned from each + * rcom_config. + * + * other nodes gather all slot values at once from the low nodeid. + * they set need_slots=1, and ignore the rf_our_slot returned from each + * rcom_config. they use the rf_num_slots returned from the low + * node's rcom_config. + */ + +int dlm_rcom_status(struct dlm_ls *ls, int nodeid, uint32_t status_flags, + uint64_t seq) +{ + struct dlm_rcom *rc; + struct dlm_msg *msg; + int error = 0; + + ls->ls_recover_nodeid = nodeid; + + if (nodeid == dlm_our_nodeid()) { + rc = ls->ls_recover_buf; + rc->rc_result = cpu_to_le32(dlm_recover_status(ls)); + goto out; + } + +retry: + error = create_rcom_stateless(ls, nodeid, DLM_RCOM_STATUS, + sizeof(struct rcom_status), &rc, &msg, + seq); + if (error) + goto out; + + set_rcom_status(ls, (struct rcom_status *)rc->rc_buf, status_flags); + + allow_sync_reply(ls, &rc->rc_id); + memset(ls->ls_recover_buf, 0, DLM_MAX_SOCKET_BUFSIZE); + + send_rcom_stateless(msg, rc); + + error = dlm_wait_function(ls, &rcom_response); + disallow_sync_reply(ls); + if (error == -ETIMEDOUT) + goto retry; + if (error) + goto out; + + rc = ls->ls_recover_buf; + + if (rc->rc_result == cpu_to_le32(-ESRCH)) { + /* we pretend the remote lockspace exists with 0 status */ + log_debug(ls, "remote node %d not ready", nodeid); + rc->rc_result = 0; + error = 0; + } else { + error = check_rcom_config(ls, rc, nodeid); + } + + /* the caller looks at rc_result for the remote recovery status */ + out: + return error; +} + +static void receive_rcom_status(struct dlm_ls *ls, + const struct dlm_rcom *rc_in, + uint64_t seq) +{ + struct dlm_rcom *rc; + struct rcom_status *rs; + uint32_t status; + int nodeid = le32_to_cpu(rc_in->rc_header.h_nodeid); + int len = sizeof(struct rcom_config); + struct dlm_msg *msg; + int num_slots = 0; + int error; + + if (!dlm_slots_version(&rc_in->rc_header)) { + status = dlm_recover_status(ls); + goto do_create; + } + + rs = (struct rcom_status *)rc_in->rc_buf; + + if (!(le32_to_cpu(rs->rs_flags) & DLM_RSF_NEED_SLOTS)) { + status = dlm_recover_status(ls); + goto do_create; + } + + spin_lock_bh(&ls->ls_recover_lock); + status = ls->ls_recover_status; + num_slots = ls->ls_num_slots; + spin_unlock_bh(&ls->ls_recover_lock); + len += num_slots * sizeof(struct rcom_slot); + + do_create: + error = create_rcom_stateless(ls, nodeid, DLM_RCOM_STATUS_REPLY, + len, &rc, &msg, seq); + if (error) + return; + + rc->rc_id = rc_in->rc_id; + rc->rc_seq_reply = rc_in->rc_seq; + rc->rc_result = cpu_to_le32(status); + + set_rcom_config(ls, (struct rcom_config *)rc->rc_buf, num_slots); + + if (!num_slots) + goto do_send; + + spin_lock_bh(&ls->ls_recover_lock); + if (ls->ls_num_slots != num_slots) { + spin_unlock_bh(&ls->ls_recover_lock); + log_debug(ls, "receive_rcom_status num_slots %d to %d", + num_slots, ls->ls_num_slots); + rc->rc_result = 0; + set_rcom_config(ls, (struct rcom_config *)rc->rc_buf, 0); + goto do_send; + } + + dlm_slots_copy_out(ls, rc); + spin_unlock_bh(&ls->ls_recover_lock); + + do_send: + send_rcom_stateless(msg, rc); +} + +static void receive_sync_reply(struct dlm_ls *ls, const struct dlm_rcom *rc_in) +{ + spin_lock_bh(&ls->ls_rcom_spin); + if (!test_bit(LSFL_RCOM_WAIT, &ls->ls_flags) || + le64_to_cpu(rc_in->rc_id) != ls->ls_rcom_seq) { + log_debug(ls, "reject reply %d from %d seq %llx expect %llx", + le32_to_cpu(rc_in->rc_type), + le32_to_cpu(rc_in->rc_header.h_nodeid), + (unsigned long long)le64_to_cpu(rc_in->rc_id), + (unsigned long long)ls->ls_rcom_seq); + goto out; + } + memcpy(ls->ls_recover_buf, rc_in, + le16_to_cpu(rc_in->rc_header.h_length)); + set_bit(LSFL_RCOM_READY, &ls->ls_flags); + clear_bit(LSFL_RCOM_WAIT, &ls->ls_flags); + wake_up(&ls->ls_wait_general); + out: + spin_unlock_bh(&ls->ls_rcom_spin); +} + +int dlm_rcom_names(struct dlm_ls *ls, int nodeid, char *last_name, + int last_len, uint64_t seq) +{ + struct dlm_mhandle *mh; + struct dlm_rcom *rc; + int error = 0; + + ls->ls_recover_nodeid = nodeid; + +retry: + error = create_rcom(ls, nodeid, DLM_RCOM_NAMES, last_len, + &rc, &mh, seq); + if (error) + goto out; + memcpy(rc->rc_buf, last_name, last_len); + + allow_sync_reply(ls, &rc->rc_id); + memset(ls->ls_recover_buf, 0, DLM_MAX_SOCKET_BUFSIZE); + + send_rcom(mh, rc); + + error = dlm_wait_function(ls, &rcom_response); + disallow_sync_reply(ls); + if (error == -ETIMEDOUT) + goto retry; + out: + return error; +} + +static void receive_rcom_names(struct dlm_ls *ls, const struct dlm_rcom *rc_in, + uint64_t seq) +{ + struct dlm_mhandle *mh; + struct dlm_rcom *rc; + int error, inlen, outlen, nodeid; + + nodeid = le32_to_cpu(rc_in->rc_header.h_nodeid); + inlen = le16_to_cpu(rc_in->rc_header.h_length) - + sizeof(struct dlm_rcom); + outlen = DLM_MAX_APP_BUFSIZE - sizeof(struct dlm_rcom); + + error = create_rcom(ls, nodeid, DLM_RCOM_NAMES_REPLY, outlen, + &rc, &mh, seq); + if (error) + return; + rc->rc_id = rc_in->rc_id; + rc->rc_seq_reply = rc_in->rc_seq; + + dlm_copy_master_names(ls, rc_in->rc_buf, inlen, rc->rc_buf, outlen, + nodeid); + send_rcom(mh, rc); +} + +int dlm_send_rcom_lookup(struct dlm_rsb *r, int dir_nodeid, uint64_t seq) +{ + struct dlm_rcom *rc; + struct dlm_mhandle *mh; + struct dlm_ls *ls = r->res_ls; + int error; + + error = create_rcom(ls, dir_nodeid, DLM_RCOM_LOOKUP, r->res_length, + &rc, &mh, seq); + if (error) + goto out; + memcpy(rc->rc_buf, r->res_name, r->res_length); + rc->rc_id = cpu_to_le64(r->res_id); + + send_rcom(mh, rc); + out: + return error; +} + +static void receive_rcom_lookup(struct dlm_ls *ls, + const struct dlm_rcom *rc_in, uint64_t seq) +{ + struct dlm_rcom *rc; + struct dlm_mhandle *mh; + int error, ret_nodeid, nodeid = le32_to_cpu(rc_in->rc_header.h_nodeid); + int len = le16_to_cpu(rc_in->rc_header.h_length) - + sizeof(struct dlm_rcom); + + /* Old code would send this special id to trigger a debug dump. */ + if (rc_in->rc_id == cpu_to_le64(0xFFFFFFFF)) { + log_error(ls, "receive_rcom_lookup dump from %d", nodeid); + dlm_dump_rsb_name(ls, rc_in->rc_buf, len); + return; + } + + error = create_rcom(ls, nodeid, DLM_RCOM_LOOKUP_REPLY, 0, &rc, &mh, + seq); + if (error) + return; + + error = dlm_master_lookup(ls, nodeid, rc_in->rc_buf, len, + DLM_LU_RECOVER_MASTER, &ret_nodeid, NULL); + if (error) + ret_nodeid = error; + rc->rc_result = cpu_to_le32(ret_nodeid); + rc->rc_id = rc_in->rc_id; + rc->rc_seq_reply = rc_in->rc_seq; + + send_rcom(mh, rc); +} + +static void receive_rcom_lookup_reply(struct dlm_ls *ls, + const struct dlm_rcom *rc_in) +{ + dlm_recover_master_reply(ls, rc_in); +} + +static void pack_rcom_lock(struct dlm_rsb *r, struct dlm_lkb *lkb, + struct rcom_lock *rl) +{ + memset(rl, 0, sizeof(*rl)); + + rl->rl_ownpid = cpu_to_le32(lkb->lkb_ownpid); + rl->rl_lkid = cpu_to_le32(lkb->lkb_id); + rl->rl_exflags = cpu_to_le32(lkb->lkb_exflags); + rl->rl_flags = cpu_to_le32(dlm_dflags_val(lkb)); + rl->rl_lvbseq = cpu_to_le32(lkb->lkb_lvbseq); + rl->rl_rqmode = lkb->lkb_rqmode; + rl->rl_grmode = lkb->lkb_grmode; + rl->rl_status = lkb->lkb_status; + rl->rl_wait_type = cpu_to_le16(lkb->lkb_wait_type); + + if (lkb->lkb_bastfn) + rl->rl_asts |= DLM_CB_BAST; + if (lkb->lkb_astfn) + rl->rl_asts |= DLM_CB_CAST; + + rl->rl_namelen = cpu_to_le16(r->res_length); + memcpy(rl->rl_name, r->res_name, r->res_length); + + /* FIXME: might we have an lvb without DLM_LKF_VALBLK set ? + If so, receive_rcom_lock_args() won't take this copy. */ + + if (lkb->lkb_lvbptr) + memcpy(rl->rl_lvb, lkb->lkb_lvbptr, r->res_ls->ls_lvblen); +} + +int dlm_send_rcom_lock(struct dlm_rsb *r, struct dlm_lkb *lkb, uint64_t seq) +{ + struct dlm_ls *ls = r->res_ls; + struct dlm_rcom *rc; + struct dlm_mhandle *mh; + struct rcom_lock *rl; + int error, len = sizeof(struct rcom_lock); + + if (lkb->lkb_lvbptr) + len += ls->ls_lvblen; + + error = create_rcom(ls, r->res_nodeid, DLM_RCOM_LOCK, len, &rc, &mh, + seq); + if (error) + goto out; + + rl = (struct rcom_lock *) rc->rc_buf; + pack_rcom_lock(r, lkb, rl); + rc->rc_id = cpu_to_le64((uintptr_t)r); + + send_rcom(mh, rc); + out: + return error; +} + +/* needs at least dlm_rcom + rcom_lock */ +static void receive_rcom_lock(struct dlm_ls *ls, const struct dlm_rcom *rc_in, + uint64_t seq) +{ + __le32 rl_remid, rl_result; + struct rcom_lock *rl; + struct dlm_rcom *rc; + struct dlm_mhandle *mh; + int error, nodeid = le32_to_cpu(rc_in->rc_header.h_nodeid); + + dlm_recover_master_copy(ls, rc_in, &rl_remid, &rl_result); + + error = create_rcom(ls, nodeid, DLM_RCOM_LOCK_REPLY, + sizeof(struct rcom_lock), &rc, &mh, seq); + if (error) + return; + + memcpy(rc->rc_buf, rc_in->rc_buf, sizeof(struct rcom_lock)); + rl = (struct rcom_lock *)rc->rc_buf; + /* set rl_remid and rl_result from dlm_recover_master_copy() */ + rl->rl_remid = rl_remid; + rl->rl_result = rl_result; + + rc->rc_id = rc_in->rc_id; + rc->rc_seq_reply = rc_in->rc_seq; + + send_rcom(mh, rc); +} + +/* If the lockspace doesn't exist then still send a status message + back; it's possible that it just doesn't have its global_id yet. */ + +int dlm_send_ls_not_ready(int nodeid, const struct dlm_rcom *rc_in) +{ + struct dlm_rcom *rc; + struct rcom_config *rf; + struct dlm_mhandle *mh; + char *mb; + int mb_len = sizeof(struct dlm_rcom) + sizeof(struct rcom_config); + + mh = dlm_midcomms_get_mhandle(nodeid, mb_len, &mb); + if (!mh) + return -ENOBUFS; + + rc = (struct dlm_rcom *) mb; + + rc->rc_header.h_version = cpu_to_le32(DLM_HEADER_MAJOR | DLM_HEADER_MINOR); + rc->rc_header.u.h_lockspace = rc_in->rc_header.u.h_lockspace; + rc->rc_header.h_nodeid = cpu_to_le32(dlm_our_nodeid()); + rc->rc_header.h_length = cpu_to_le16(mb_len); + rc->rc_header.h_cmd = DLM_RCOM; + + rc->rc_type = cpu_to_le32(DLM_RCOM_STATUS_REPLY); + rc->rc_id = rc_in->rc_id; + rc->rc_seq_reply = rc_in->rc_seq; + rc->rc_result = cpu_to_le32(-ESRCH); + + rf = (struct rcom_config *) rc->rc_buf; + rf->rf_lvblen = cpu_to_le32(~0U); + + dlm_midcomms_commit_mhandle(mh, NULL, 0); + + return 0; +} + +/* + * Ignore messages for stage Y before we set + * recover_status bit for stage X: + * + * recover_status = 0 + * + * dlm_recover_members() + * - send nothing + * - recv nothing + * - ignore NAMES, NAMES_REPLY + * - ignore LOOKUP, LOOKUP_REPLY + * - ignore LOCK, LOCK_REPLY + * + * recover_status |= NODES + * + * dlm_recover_members_wait() + * + * dlm_recover_directory() + * - send NAMES + * - recv NAMES_REPLY + * - ignore LOOKUP, LOOKUP_REPLY + * - ignore LOCK, LOCK_REPLY + * + * recover_status |= DIR + * + * dlm_recover_directory_wait() + * + * dlm_recover_masters() + * - send LOOKUP + * - recv LOOKUP_REPLY + * + * dlm_recover_locks() + * - send LOCKS + * - recv LOCKS_REPLY + * + * recover_status |= LOCKS + * + * dlm_recover_locks_wait() + * + * recover_status |= DONE + */ + +/* Called by dlm_recv; corresponds to dlm_receive_message() but special + recovery-only comms are sent through here. */ + +void dlm_receive_rcom(struct dlm_ls *ls, const struct dlm_rcom *rc, int nodeid) +{ + int lock_size = sizeof(struct dlm_rcom) + sizeof(struct rcom_lock); + int stop, reply = 0, names = 0, lookup = 0, lock = 0; + uint32_t status; + uint64_t seq; + + switch (rc->rc_type) { + case cpu_to_le32(DLM_RCOM_STATUS_REPLY): + reply = 1; + break; + case cpu_to_le32(DLM_RCOM_NAMES): + names = 1; + break; + case cpu_to_le32(DLM_RCOM_NAMES_REPLY): + names = 1; + reply = 1; + break; + case cpu_to_le32(DLM_RCOM_LOOKUP): + lookup = 1; + break; + case cpu_to_le32(DLM_RCOM_LOOKUP_REPLY): + lookup = 1; + reply = 1; + break; + case cpu_to_le32(DLM_RCOM_LOCK): + lock = 1; + break; + case cpu_to_le32(DLM_RCOM_LOCK_REPLY): + lock = 1; + reply = 1; + break; + } + + spin_lock_bh(&ls->ls_recover_lock); + status = ls->ls_recover_status; + stop = dlm_recovery_stopped(ls); + seq = ls->ls_recover_seq; + spin_unlock_bh(&ls->ls_recover_lock); + + if (stop && (rc->rc_type != cpu_to_le32(DLM_RCOM_STATUS))) + goto ignore; + + if (reply && (le64_to_cpu(rc->rc_seq_reply) != seq)) + goto ignore; + + if (!(status & DLM_RS_NODES) && (names || lookup || lock)) + goto ignore; + + if (!(status & DLM_RS_DIR) && (lookup || lock)) + goto ignore; + + switch (rc->rc_type) { + case cpu_to_le32(DLM_RCOM_STATUS): + receive_rcom_status(ls, rc, seq); + break; + + case cpu_to_le32(DLM_RCOM_NAMES): + receive_rcom_names(ls, rc, seq); + break; + + case cpu_to_le32(DLM_RCOM_LOOKUP): + receive_rcom_lookup(ls, rc, seq); + break; + + case cpu_to_le32(DLM_RCOM_LOCK): + if (le16_to_cpu(rc->rc_header.h_length) < lock_size) + goto Eshort; + receive_rcom_lock(ls, rc, seq); + break; + + case cpu_to_le32(DLM_RCOM_STATUS_REPLY): + receive_sync_reply(ls, rc); + break; + + case cpu_to_le32(DLM_RCOM_NAMES_REPLY): + receive_sync_reply(ls, rc); + break; + + case cpu_to_le32(DLM_RCOM_LOOKUP_REPLY): + receive_rcom_lookup_reply(ls, rc); + break; + + case cpu_to_le32(DLM_RCOM_LOCK_REPLY): + if (le16_to_cpu(rc->rc_header.h_length) < lock_size) + goto Eshort; + dlm_recover_process_copy(ls, rc, seq); + break; + + default: + log_error(ls, "receive_rcom bad type %d", + le32_to_cpu(rc->rc_type)); + } + return; + +ignore: + log_limit(ls, "dlm_receive_rcom ignore msg %d " + "from %d %llu %llu recover seq %llu sts %x gen %u", + le32_to_cpu(rc->rc_type), + nodeid, + (unsigned long long)le64_to_cpu(rc->rc_seq), + (unsigned long long)le64_to_cpu(rc->rc_seq_reply), + (unsigned long long)seq, + status, ls->ls_generation); + return; +Eshort: + log_error(ls, "recovery message %d from %d is too short", + le32_to_cpu(rc->rc_type), nodeid); +} + diff --git a/fs/dlm/rcom.h b/fs/dlm/rcom.h new file mode 100644 index 000000000..765926ae0 --- /dev/null +++ b/fs/dlm/rcom.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2005-2007 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#ifndef __RCOM_DOT_H__ +#define __RCOM_DOT_H__ + +int dlm_rcom_status(struct dlm_ls *ls, int nodeid, uint32_t status_flags, + uint64_t seq); +int dlm_rcom_names(struct dlm_ls *ls, int nodeid, char *last_name, + int last_len, uint64_t seq); +int dlm_send_rcom_lookup(struct dlm_rsb *r, int dir_nodeid, uint64_t seq); +int dlm_send_rcom_lock(struct dlm_rsb *r, struct dlm_lkb *lkb, uint64_t seq); +void dlm_receive_rcom(struct dlm_ls *ls, const struct dlm_rcom *rc, + int nodeid); +int dlm_send_ls_not_ready(int nodeid, const struct dlm_rcom *rc_in); + +#endif + diff --git a/fs/dlm/recover.c b/fs/dlm/recover.c new file mode 100644 index 000000000..3ac020fb8 --- /dev/null +++ b/fs/dlm/recover.c @@ -0,0 +1,917 @@ +// SPDX-License-Identifier: GPL-2.0-only +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#include "dlm_internal.h" +#include "lockspace.h" +#include "dir.h" +#include "config.h" +#include "ast.h" +#include "memory.h" +#include "rcom.h" +#include "lock.h" +#include "lowcomms.h" +#include "member.h" +#include "recover.h" + + +/* + * Recovery waiting routines: these functions wait for a particular reply from + * a remote node, or for the remote node to report a certain status. They need + * to abort if the lockspace is stopped indicating a node has failed (perhaps + * the one being waited for). + */ + +/* + * Wait until given function returns non-zero or lockspace is stopped + * (LS_RECOVERY_STOP set due to failure of a node in ls_nodes). When another + * function thinks it could have completed the waited-on task, they should wake + * up ls_wait_general to get an immediate response rather than waiting for the + * timeout. This uses a timeout so it can check periodically if the wait + * should abort due to node failure (which doesn't cause a wake_up). + * This should only be called by the dlm_recoverd thread. + */ + +int dlm_wait_function(struct dlm_ls *ls, int (*testfn) (struct dlm_ls *ls)) +{ + int error = 0; + int rv; + + while (1) { + rv = wait_event_timeout(ls->ls_wait_general, + testfn(ls) || dlm_recovery_stopped(ls), + dlm_config.ci_recover_timer * HZ); + if (rv) + break; + if (test_bit(LSFL_RCOM_WAIT, &ls->ls_flags)) { + log_debug(ls, "dlm_wait_function timed out"); + return -ETIMEDOUT; + } + } + + if (dlm_recovery_stopped(ls)) { + log_debug(ls, "dlm_wait_function aborted"); + error = -EINTR; + } + return error; +} + +/* + * An efficient way for all nodes to wait for all others to have a certain + * status. The node with the lowest nodeid polls all the others for their + * status (wait_status_all) and all the others poll the node with the low id + * for its accumulated result (wait_status_low). When all nodes have set + * status flag X, then status flag X_ALL will be set on the low nodeid. + */ + +uint32_t dlm_recover_status(struct dlm_ls *ls) +{ + uint32_t status; + spin_lock_bh(&ls->ls_recover_lock); + status = ls->ls_recover_status; + spin_unlock_bh(&ls->ls_recover_lock); + return status; +} + +static void _set_recover_status(struct dlm_ls *ls, uint32_t status) +{ + ls->ls_recover_status |= status; +} + +void dlm_set_recover_status(struct dlm_ls *ls, uint32_t status) +{ + spin_lock_bh(&ls->ls_recover_lock); + _set_recover_status(ls, status); + spin_unlock_bh(&ls->ls_recover_lock); +} + +static int wait_status_all(struct dlm_ls *ls, uint32_t wait_status, + int save_slots, uint64_t seq) +{ + struct dlm_rcom *rc = ls->ls_recover_buf; + struct dlm_member *memb; + int error = 0, delay; + + list_for_each_entry(memb, &ls->ls_nodes, list) { + delay = 0; + for (;;) { + if (dlm_recovery_stopped(ls)) { + error = -EINTR; + goto out; + } + + error = dlm_rcom_status(ls, memb->nodeid, 0, seq); + if (error) + goto out; + + if (save_slots) + dlm_slot_save(ls, rc, memb); + + if (le32_to_cpu(rc->rc_result) & wait_status) + break; + if (delay < 1000) + delay += 20; + msleep(delay); + } + } + out: + return error; +} + +static int wait_status_low(struct dlm_ls *ls, uint32_t wait_status, + uint32_t status_flags, uint64_t seq) +{ + struct dlm_rcom *rc = ls->ls_recover_buf; + int error = 0, delay = 0, nodeid = ls->ls_low_nodeid; + + for (;;) { + if (dlm_recovery_stopped(ls)) { + error = -EINTR; + goto out; + } + + error = dlm_rcom_status(ls, nodeid, status_flags, seq); + if (error) + break; + + if (le32_to_cpu(rc->rc_result) & wait_status) + break; + if (delay < 1000) + delay += 20; + msleep(delay); + } + out: + return error; +} + +static int wait_status(struct dlm_ls *ls, uint32_t status, uint64_t seq) +{ + uint32_t status_all = status << 1; + int error; + + if (ls->ls_low_nodeid == dlm_our_nodeid()) { + error = wait_status_all(ls, status, 0, seq); + if (!error) + dlm_set_recover_status(ls, status_all); + } else + error = wait_status_low(ls, status_all, 0, seq); + + return error; +} + +int dlm_recover_members_wait(struct dlm_ls *ls, uint64_t seq) +{ + struct dlm_member *memb; + struct dlm_slot *slots; + int num_slots, slots_size; + int error, rv; + uint32_t gen; + + list_for_each_entry(memb, &ls->ls_nodes, list) { + memb->slot = -1; + memb->generation = 0; + } + + if (ls->ls_low_nodeid == dlm_our_nodeid()) { + error = wait_status_all(ls, DLM_RS_NODES, 1, seq); + if (error) + goto out; + + /* slots array is sparse, slots_size may be > num_slots */ + + rv = dlm_slots_assign(ls, &num_slots, &slots_size, &slots, &gen); + if (!rv) { + spin_lock_bh(&ls->ls_recover_lock); + _set_recover_status(ls, DLM_RS_NODES_ALL); + ls->ls_num_slots = num_slots; + ls->ls_slots_size = slots_size; + ls->ls_slots = slots; + ls->ls_generation = gen; + spin_unlock_bh(&ls->ls_recover_lock); + } else { + dlm_set_recover_status(ls, DLM_RS_NODES_ALL); + } + } else { + error = wait_status_low(ls, DLM_RS_NODES_ALL, + DLM_RSF_NEED_SLOTS, seq); + if (error) + goto out; + + dlm_slots_copy_in(ls); + } + out: + return error; +} + +int dlm_recover_directory_wait(struct dlm_ls *ls, uint64_t seq) +{ + return wait_status(ls, DLM_RS_DIR, seq); +} + +int dlm_recover_locks_wait(struct dlm_ls *ls, uint64_t seq) +{ + return wait_status(ls, DLM_RS_LOCKS, seq); +} + +int dlm_recover_done_wait(struct dlm_ls *ls, uint64_t seq) +{ + return wait_status(ls, DLM_RS_DONE, seq); +} + +/* + * The recover_list contains all the rsb's for which we've requested the new + * master nodeid. As replies are returned from the resource directories the + * rsb's are removed from the list. When the list is empty we're done. + * + * The recover_list is later similarly used for all rsb's for which we've sent + * new lkb's and need to receive new corresponding lkid's. + * + * We use the address of the rsb struct as a simple local identifier for the + * rsb so we can match an rcom reply with the rsb it was sent for. + */ + +static int recover_list_empty(struct dlm_ls *ls) +{ + int empty; + + spin_lock_bh(&ls->ls_recover_list_lock); + empty = list_empty(&ls->ls_recover_list); + spin_unlock_bh(&ls->ls_recover_list_lock); + + return empty; +} + +static void recover_list_add(struct dlm_rsb *r) +{ + struct dlm_ls *ls = r->res_ls; + + spin_lock_bh(&ls->ls_recover_list_lock); + if (list_empty(&r->res_recover_list)) { + list_add_tail(&r->res_recover_list, &ls->ls_recover_list); + ls->ls_recover_list_count++; + dlm_hold_rsb(r); + } + spin_unlock_bh(&ls->ls_recover_list_lock); +} + +static void recover_list_del(struct dlm_rsb *r) +{ + struct dlm_ls *ls = r->res_ls; + + spin_lock_bh(&ls->ls_recover_list_lock); + list_del_init(&r->res_recover_list); + ls->ls_recover_list_count--; + spin_unlock_bh(&ls->ls_recover_list_lock); + + dlm_put_rsb(r); +} + +static void recover_list_clear(struct dlm_ls *ls) +{ + struct dlm_rsb *r, *s; + + spin_lock_bh(&ls->ls_recover_list_lock); + list_for_each_entry_safe(r, s, &ls->ls_recover_list, res_recover_list) { + list_del_init(&r->res_recover_list); + r->res_recover_locks_count = 0; + dlm_put_rsb(r); + ls->ls_recover_list_count--; + } + + if (ls->ls_recover_list_count != 0) { + log_error(ls, "warning: recover_list_count %d", + ls->ls_recover_list_count); + ls->ls_recover_list_count = 0; + } + spin_unlock_bh(&ls->ls_recover_list_lock); +} + +static int recover_xa_empty(struct dlm_ls *ls) +{ + int empty = 1; + + spin_lock_bh(&ls->ls_recover_xa_lock); + if (ls->ls_recover_list_count) + empty = 0; + spin_unlock_bh(&ls->ls_recover_xa_lock); + + return empty; +} + +static int recover_xa_add(struct dlm_rsb *r) +{ + struct dlm_ls *ls = r->res_ls; + struct xa_limit limit = { + .min = 1, + .max = UINT_MAX, + }; + uint32_t id; + int rv; + + spin_lock_bh(&ls->ls_recover_xa_lock); + if (r->res_id) { + rv = -1; + goto out_unlock; + } + rv = xa_alloc(&ls->ls_recover_xa, &id, r, limit, GFP_ATOMIC); + if (rv < 0) + goto out_unlock; + + r->res_id = id; + ls->ls_recover_list_count++; + dlm_hold_rsb(r); + rv = 0; +out_unlock: + spin_unlock_bh(&ls->ls_recover_xa_lock); + return rv; +} + +static void recover_xa_del(struct dlm_rsb *r) +{ + struct dlm_ls *ls = r->res_ls; + + spin_lock_bh(&ls->ls_recover_xa_lock); + xa_erase_bh(&ls->ls_recover_xa, r->res_id); + r->res_id = 0; + ls->ls_recover_list_count--; + spin_unlock_bh(&ls->ls_recover_xa_lock); + + dlm_put_rsb(r); +} + +static struct dlm_rsb *recover_xa_find(struct dlm_ls *ls, uint64_t id) +{ + struct dlm_rsb *r; + + spin_lock_bh(&ls->ls_recover_xa_lock); + r = xa_load(&ls->ls_recover_xa, (int)id); + spin_unlock_bh(&ls->ls_recover_xa_lock); + return r; +} + +static void recover_xa_clear(struct dlm_ls *ls) +{ + struct dlm_rsb *r; + unsigned long id; + + spin_lock_bh(&ls->ls_recover_xa_lock); + + xa_for_each(&ls->ls_recover_xa, id, r) { + xa_erase_bh(&ls->ls_recover_xa, id); + r->res_id = 0; + r->res_recover_locks_count = 0; + ls->ls_recover_list_count--; + + dlm_put_rsb(r); + } + + if (ls->ls_recover_list_count != 0) { + log_error(ls, "warning: recover_list_count %d", + ls->ls_recover_list_count); + ls->ls_recover_list_count = 0; + } + spin_unlock_bh(&ls->ls_recover_xa_lock); +} + + +/* Master recovery: find new master node for rsb's that were + mastered on nodes that have been removed. + + dlm_recover_masters + recover_master + dlm_send_rcom_lookup -> receive_rcom_lookup + dlm_dir_lookup + receive_rcom_lookup_reply <- + dlm_recover_master_reply + set_new_master + set_master_lkbs + set_lock_master +*/ + +/* + * Set the lock master for all LKBs in a lock queue + * If we are the new master of the rsb, we may have received new + * MSTCPY locks from other nodes already which we need to ignore + * when setting the new nodeid. + */ + +static void set_lock_master(struct list_head *queue, int nodeid) +{ + struct dlm_lkb *lkb; + + list_for_each_entry(lkb, queue, lkb_statequeue) { + if (!test_bit(DLM_IFL_MSTCPY_BIT, &lkb->lkb_iflags)) { + lkb->lkb_nodeid = nodeid; + lkb->lkb_remid = 0; + } + } +} + +static void set_master_lkbs(struct dlm_rsb *r) +{ + set_lock_master(&r->res_grantqueue, r->res_nodeid); + set_lock_master(&r->res_convertqueue, r->res_nodeid); + set_lock_master(&r->res_waitqueue, r->res_nodeid); +} + +/* + * Propagate the new master nodeid to locks + * The NEW_MASTER flag tells dlm_recover_locks() which rsb's to consider. + * The NEW_MASTER2 flag tells recover_lvb() and recover_grant() which + * rsb's to consider. + */ + +static void set_new_master(struct dlm_rsb *r) +{ + set_master_lkbs(r); + rsb_set_flag(r, RSB_NEW_MASTER); + rsb_set_flag(r, RSB_NEW_MASTER2); +} + +/* + * We do async lookups on rsb's that need new masters. The rsb's + * waiting for a lookup reply are kept on the recover_list. + * + * Another node recovering the master may have sent us a rcom lookup, + * and our dlm_master_lookup() set it as the new master, along with + * NEW_MASTER so that we'll recover it here (this implies dir_nodeid + * equals our_nodeid below). + */ + +static int recover_master(struct dlm_rsb *r, unsigned int *count, uint64_t seq) +{ + struct dlm_ls *ls = r->res_ls; + int our_nodeid, dir_nodeid; + int is_removed = 0; + int error; + + if (r->res_nodeid != -1 && is_master(r)) + return 0; + + if (r->res_nodeid != -1) + is_removed = dlm_is_removed(ls, r->res_nodeid); + + if (!is_removed && !rsb_flag(r, RSB_NEW_MASTER)) + return 0; + + our_nodeid = dlm_our_nodeid(); + dir_nodeid = dlm_dir_nodeid(r); + + if (dir_nodeid == our_nodeid) { + if (is_removed) { + r->res_master_nodeid = our_nodeid; + r->res_nodeid = 0; + } + + /* set master of lkbs to ourself when is_removed, or to + another new master which we set along with NEW_MASTER + in dlm_master_lookup */ + set_new_master(r); + error = 0; + } else { + recover_xa_add(r); + error = dlm_send_rcom_lookup(r, dir_nodeid, seq); + } + + (*count)++; + return error; +} + +/* + * All MSTCPY locks are purged and rebuilt, even if the master stayed the same. + * This is necessary because recovery can be started, aborted and restarted, + * causing the master nodeid to briefly change during the aborted recovery, and + * change back to the original value in the second recovery. The MSTCPY locks + * may or may not have been purged during the aborted recovery. Another node + * with an outstanding request in waiters list and a request reply saved in the + * requestqueue, cannot know whether it should ignore the reply and resend the + * request, or accept the reply and complete the request. It must do the + * former if the remote node purged MSTCPY locks, and it must do the later if + * the remote node did not. This is solved by always purging MSTCPY locks, in + * which case, the request reply would always be ignored and the request + * resent. + */ + +static int recover_master_static(struct dlm_rsb *r, unsigned int *count) +{ + int dir_nodeid = dlm_dir_nodeid(r); + int new_master = dir_nodeid; + + if (dir_nodeid == dlm_our_nodeid()) + new_master = 0; + + dlm_purge_mstcpy_locks(r); + r->res_master_nodeid = dir_nodeid; + r->res_nodeid = new_master; + set_new_master(r); + (*count)++; + return 0; +} + +/* + * Go through local root resources and for each rsb which has a master which + * has departed, get the new master nodeid from the directory. The dir will + * assign mastery to the first node to look up the new master. That means + * we'll discover in this lookup if we're the new master of any rsb's. + * + * We fire off all the dir lookup requests individually and asynchronously to + * the correct dir node. + */ + +int dlm_recover_masters(struct dlm_ls *ls, uint64_t seq, + const struct list_head *root_list) +{ + struct dlm_rsb *r; + unsigned int total = 0; + unsigned int count = 0; + int nodir = dlm_no_directory(ls); + int error; + + log_rinfo(ls, "dlm_recover_masters"); + + list_for_each_entry(r, root_list, res_root_list) { + if (dlm_recovery_stopped(ls)) { + error = -EINTR; + goto out; + } + + lock_rsb(r); + if (nodir) + error = recover_master_static(r, &count); + else + error = recover_master(r, &count, seq); + unlock_rsb(r); + cond_resched(); + total++; + + if (error) + goto out; + } + + log_rinfo(ls, "dlm_recover_masters %u of %u", count, total); + + error = dlm_wait_function(ls, &recover_xa_empty); + out: + if (error) + recover_xa_clear(ls); + return error; +} + +int dlm_recover_master_reply(struct dlm_ls *ls, const struct dlm_rcom *rc) +{ + struct dlm_rsb *r; + int ret_nodeid, new_master; + + r = recover_xa_find(ls, le64_to_cpu(rc->rc_id)); + if (!r) { + log_error(ls, "dlm_recover_master_reply no id %llx", + (unsigned long long)le64_to_cpu(rc->rc_id)); + goto out; + } + + ret_nodeid = le32_to_cpu(rc->rc_result); + + if (ret_nodeid == dlm_our_nodeid()) + new_master = 0; + else + new_master = ret_nodeid; + + lock_rsb(r); + r->res_master_nodeid = ret_nodeid; + r->res_nodeid = new_master; + set_new_master(r); + unlock_rsb(r); + recover_xa_del(r); + + if (recover_xa_empty(ls)) + wake_up(&ls->ls_wait_general); + out: + return 0; +} + + +/* Lock recovery: rebuild the process-copy locks we hold on a + remastered rsb on the new rsb master. + + dlm_recover_locks + recover_locks + recover_locks_queue + dlm_send_rcom_lock -> receive_rcom_lock + dlm_recover_master_copy + receive_rcom_lock_reply <- + dlm_recover_process_copy +*/ + + +/* + * keep a count of the number of lkb's we send to the new master; when we get + * an equal number of replies then recovery for the rsb is done + */ + +static int recover_locks_queue(struct dlm_rsb *r, struct list_head *head, + uint64_t seq) +{ + struct dlm_lkb *lkb; + int error = 0; + + list_for_each_entry(lkb, head, lkb_statequeue) { + error = dlm_send_rcom_lock(r, lkb, seq); + if (error) + break; + r->res_recover_locks_count++; + } + + return error; +} + +static int recover_locks(struct dlm_rsb *r, uint64_t seq) +{ + int error = 0; + + lock_rsb(r); + + DLM_ASSERT(!r->res_recover_locks_count, dlm_dump_rsb(r);); + + error = recover_locks_queue(r, &r->res_grantqueue, seq); + if (error) + goto out; + error = recover_locks_queue(r, &r->res_convertqueue, seq); + if (error) + goto out; + error = recover_locks_queue(r, &r->res_waitqueue, seq); + if (error) + goto out; + + if (r->res_recover_locks_count) + recover_list_add(r); + else + rsb_clear_flag(r, RSB_NEW_MASTER); + out: + unlock_rsb(r); + return error; +} + +int dlm_recover_locks(struct dlm_ls *ls, uint64_t seq, + const struct list_head *root_list) +{ + struct dlm_rsb *r; + int error, count = 0; + + list_for_each_entry(r, root_list, res_root_list) { + if (r->res_nodeid != -1 && is_master(r)) { + rsb_clear_flag(r, RSB_NEW_MASTER); + continue; + } + + if (!rsb_flag(r, RSB_NEW_MASTER)) + continue; + + if (dlm_recovery_stopped(ls)) { + error = -EINTR; + goto out; + } + + error = recover_locks(r, seq); + if (error) + goto out; + + count += r->res_recover_locks_count; + } + + log_rinfo(ls, "dlm_recover_locks %d out", count); + + error = dlm_wait_function(ls, &recover_list_empty); + out: + if (error) + recover_list_clear(ls); + return error; +} + +void dlm_recovered_lock(struct dlm_rsb *r) +{ + DLM_ASSERT(rsb_flag(r, RSB_NEW_MASTER), dlm_dump_rsb(r);); + + r->res_recover_locks_count--; + if (!r->res_recover_locks_count) { + rsb_clear_flag(r, RSB_NEW_MASTER); + recover_list_del(r); + } + + if (recover_list_empty(r->res_ls)) + wake_up(&r->res_ls->ls_wait_general); +} + +/* + * The lvb needs to be recovered on all master rsb's. This includes setting + * the VALNOTVALID flag if necessary, and determining the correct lvb contents + * based on the lvb's of the locks held on the rsb. + * + * RSB_VALNOTVALID is set in two cases: + * + * 1. we are master, but not new, and we purged an EX/PW lock held by a + * failed node (in dlm_recover_purge which set RSB_RECOVER_LVB_INVAL) + * + * 2. we are a new master, and there are only NL/CR locks left. + * (We could probably improve this by only invaliding in this way when + * the previous master left uncleanly. VMS docs mention that.) + * + * The LVB contents are only considered for changing when this is a new master + * of the rsb (NEW_MASTER2). Then, the rsb's lvb is taken from any lkb with + * mode > CR. If no lkb's exist with mode above CR, the lvb contents are taken + * from the lkb with the largest lvb sequence number. + */ + +static void recover_lvb(struct dlm_rsb *r) +{ + struct dlm_lkb *big_lkb = NULL, *iter, *high_lkb = NULL; + uint32_t high_seq = 0; + int lock_lvb_exists = 0; + int lvblen = r->res_ls->ls_lvblen; + + if (!rsb_flag(r, RSB_NEW_MASTER2) && + rsb_flag(r, RSB_RECOVER_LVB_INVAL)) { + /* case 1 above */ + rsb_set_flag(r, RSB_VALNOTVALID); + return; + } + + if (!rsb_flag(r, RSB_NEW_MASTER2)) + return; + + /* we are the new master, so figure out if VALNOTVALID should + be set, and set the rsb lvb from the best lkb available. */ + + list_for_each_entry(iter, &r->res_grantqueue, lkb_statequeue) { + if (!(iter->lkb_exflags & DLM_LKF_VALBLK)) + continue; + + lock_lvb_exists = 1; + + if (iter->lkb_grmode > DLM_LOCK_CR) { + big_lkb = iter; + goto setflag; + } + + if (((int)iter->lkb_lvbseq - (int)high_seq) >= 0) { + high_lkb = iter; + high_seq = iter->lkb_lvbseq; + } + } + + list_for_each_entry(iter, &r->res_convertqueue, lkb_statequeue) { + if (!(iter->lkb_exflags & DLM_LKF_VALBLK)) + continue; + + lock_lvb_exists = 1; + + if (iter->lkb_grmode > DLM_LOCK_CR) { + big_lkb = iter; + goto setflag; + } + + if (((int)iter->lkb_lvbseq - (int)high_seq) >= 0) { + high_lkb = iter; + high_seq = iter->lkb_lvbseq; + } + } + + setflag: + if (!lock_lvb_exists) + goto out; + + /* lvb is invalidated if only NL/CR locks remain */ + if (!big_lkb) + rsb_set_flag(r, RSB_VALNOTVALID); + + if (!r->res_lvbptr) { + r->res_lvbptr = dlm_allocate_lvb(r->res_ls); + if (!r->res_lvbptr) + goto out; + } + + if (big_lkb) { + r->res_lvbseq = big_lkb->lkb_lvbseq; + memcpy(r->res_lvbptr, big_lkb->lkb_lvbptr, lvblen); + } else if (high_lkb) { + r->res_lvbseq = high_lkb->lkb_lvbseq; + memcpy(r->res_lvbptr, high_lkb->lkb_lvbptr, lvblen); + } else { + r->res_lvbseq = 0; + memset(r->res_lvbptr, 0, lvblen); + } + out: + return; +} + +/* All master rsb's flagged RECOVER_CONVERT need to be looked at. The locks + * converting PR->CW or CW->PR may need to have their lkb_grmode changed. + */ + +static void recover_conversion(struct dlm_rsb *r) +{ + struct dlm_ls *ls = r->res_ls; + uint32_t other_lkid = 0; + int other_grmode = -1; + struct dlm_lkb *lkb; + + list_for_each_entry(lkb, &r->res_grantqueue, lkb_statequeue) { + if (lkb->lkb_grmode == DLM_LOCK_PR || + lkb->lkb_grmode == DLM_LOCK_CW) { + other_grmode = lkb->lkb_grmode; + other_lkid = lkb->lkb_id; + break; + } + } + + if (other_grmode == -1) + return; + + list_for_each_entry(lkb, &r->res_convertqueue, lkb_statequeue) { + /* Lock recovery created incompatible granted modes, so + * change the granted mode of the converting lock to + * NL. The rqmode of the converting lock should be CW, + * which means the converting lock should be granted at + * the end of recovery. + */ + if (((lkb->lkb_grmode == DLM_LOCK_PR) && (other_grmode == DLM_LOCK_CW)) || + ((lkb->lkb_grmode == DLM_LOCK_CW) && (other_grmode == DLM_LOCK_PR))) { + log_rinfo(ls, "%s %x gr %d rq %d, remote %d %x, other_lkid %u, other gr %d, set gr=NL", + __func__, lkb->lkb_id, lkb->lkb_grmode, + lkb->lkb_rqmode, lkb->lkb_nodeid, + lkb->lkb_remid, other_lkid, other_grmode); + lkb->lkb_grmode = DLM_LOCK_NL; + } + } +} + +/* We've become the new master for this rsb and waiting/converting locks may + need to be granted in dlm_recover_grant() due to locks that may have + existed from a removed node. */ + +static void recover_grant(struct dlm_rsb *r) +{ + if (!list_empty(&r->res_waitqueue) || !list_empty(&r->res_convertqueue)) + rsb_set_flag(r, RSB_RECOVER_GRANT); +} + +void dlm_recover_rsbs(struct dlm_ls *ls, const struct list_head *root_list) +{ + struct dlm_rsb *r; + unsigned int count = 0; + + list_for_each_entry(r, root_list, res_root_list) { + lock_rsb(r); + if (r->res_nodeid != -1 && is_master(r)) { + if (rsb_flag(r, RSB_RECOVER_CONVERT)) + recover_conversion(r); + + /* recover lvb before granting locks so the updated + lvb/VALNOTVALID is presented in the completion */ + recover_lvb(r); + + if (rsb_flag(r, RSB_NEW_MASTER2)) + recover_grant(r); + count++; + } else { + rsb_clear_flag(r, RSB_VALNOTVALID); + } + rsb_clear_flag(r, RSB_RECOVER_CONVERT); + rsb_clear_flag(r, RSB_RECOVER_LVB_INVAL); + rsb_clear_flag(r, RSB_NEW_MASTER2); + unlock_rsb(r); + } + + if (count) + log_rinfo(ls, "dlm_recover_rsbs %d done", count); +} + +void dlm_clear_inactive(struct dlm_ls *ls) +{ + struct dlm_rsb *r, *safe; + unsigned int count = 0; + + write_lock_bh(&ls->ls_rsbtbl_lock); + list_for_each_entry_safe(r, safe, &ls->ls_slow_inactive, res_slow_list) { + list_del(&r->res_slow_list); + rhashtable_remove_fast(&ls->ls_rsbtbl, &r->res_node, + dlm_rhash_rsb_params); + + if (!list_empty(&r->res_scan_list)) + list_del_init(&r->res_scan_list); + + free_inactive_rsb(r); + count++; + } + write_unlock_bh(&ls->ls_rsbtbl_lock); + + if (count) + log_rinfo(ls, "dlm_clear_inactive %u done", count); +} + diff --git a/fs/dlm/recover.h b/fs/dlm/recover.h new file mode 100644 index 000000000..ec6989646 --- /dev/null +++ b/fs/dlm/recover.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#ifndef __RECOVER_DOT_H__ +#define __RECOVER_DOT_H__ + +int dlm_wait_function(struct dlm_ls *ls, int (*testfn) (struct dlm_ls *ls)); +uint32_t dlm_recover_status(struct dlm_ls *ls); +void dlm_set_recover_status(struct dlm_ls *ls, uint32_t status); +int dlm_recover_members_wait(struct dlm_ls *ls, uint64_t seq); +int dlm_recover_directory_wait(struct dlm_ls *ls, uint64_t seq); +int dlm_recover_locks_wait(struct dlm_ls *ls, uint64_t seq); +int dlm_recover_done_wait(struct dlm_ls *ls, uint64_t seq); +int dlm_recover_masters(struct dlm_ls *ls, uint64_t seq, + const struct list_head *root_list); +int dlm_recover_master_reply(struct dlm_ls *ls, const struct dlm_rcom *rc); +int dlm_recover_locks(struct dlm_ls *ls, uint64_t seq, + const struct list_head *root_list); +void dlm_recovered_lock(struct dlm_rsb *r); +void dlm_clear_inactive(struct dlm_ls *ls); +void dlm_recover_rsbs(struct dlm_ls *ls, const struct list_head *root_list); + +#endif /* __RECOVER_DOT_H__ */ + diff --git a/fs/dlm/recoverd.c b/fs/dlm/recoverd.c new file mode 100644 index 000000000..12272a8f6 --- /dev/null +++ b/fs/dlm/recoverd.c @@ -0,0 +1,468 @@ +// SPDX-License-Identifier: GPL-2.0-only +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2011 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#include "dlm_internal.h" +#include "lockspace.h" +#include "member.h" +#include "dir.h" +#include "ast.h" +#include "recover.h" +#include "lowcomms.h" +#include "lock.h" +#include "requestqueue.h" +#include "recoverd.h" + +static int dlm_create_masters_list(struct dlm_ls *ls) +{ + struct dlm_rsb *r; + int error = 0; + + write_lock_bh(&ls->ls_masters_lock); + if (!list_empty(&ls->ls_masters_list)) { + log_error(ls, "root list not empty"); + error = -EINVAL; + goto out; + } + + read_lock_bh(&ls->ls_rsbtbl_lock); + list_for_each_entry(r, &ls->ls_slow_active, res_slow_list) { + if (r->res_nodeid) + continue; + + list_add(&r->res_masters_list, &ls->ls_masters_list); + dlm_hold_rsb(r); + } + read_unlock_bh(&ls->ls_rsbtbl_lock); + out: + write_unlock_bh(&ls->ls_masters_lock); + return error; +} + +static void dlm_release_masters_list(struct dlm_ls *ls) +{ + struct dlm_rsb *r, *safe; + + write_lock_bh(&ls->ls_masters_lock); + list_for_each_entry_safe(r, safe, &ls->ls_masters_list, res_masters_list) { + list_del_init(&r->res_masters_list); + dlm_put_rsb(r); + } + write_unlock_bh(&ls->ls_masters_lock); +} + +static void dlm_create_root_list(struct dlm_ls *ls, struct list_head *root_list) +{ + struct dlm_rsb *r; + + read_lock_bh(&ls->ls_rsbtbl_lock); + list_for_each_entry(r, &ls->ls_slow_active, res_slow_list) { + list_add(&r->res_root_list, root_list); + dlm_hold_rsb(r); + } + + WARN_ON_ONCE(!list_empty(&ls->ls_slow_inactive)); + read_unlock_bh(&ls->ls_rsbtbl_lock); +} + +static void dlm_release_root_list(struct list_head *root_list) +{ + struct dlm_rsb *r, *safe; + + list_for_each_entry_safe(r, safe, root_list, res_root_list) { + list_del_init(&r->res_root_list); + dlm_put_rsb(r); + } +} + +/* If the start for which we're re-enabling locking (seq) has been superseded + by a newer stop (ls_recover_seq), we need to leave locking disabled. + + We suspend dlm_recv threads here to avoid the race where dlm_recv a) sees + locking stopped and b) adds a message to the requestqueue, but dlm_recoverd + enables locking and clears the requestqueue between a and b. */ + +static int enable_locking(struct dlm_ls *ls, uint64_t seq) +{ + int error = -EINTR; + + write_lock_bh(&ls->ls_recv_active); + + spin_lock_bh(&ls->ls_recover_lock); + if (ls->ls_recover_seq == seq) { + set_bit(LSFL_RUNNING, &ls->ls_flags); + /* Schedule next timer if recovery put something on inactive. + * + * The rsbs that was queued while recovery on toss hasn't + * started yet because LSFL_RUNNING was set everything + * else recovery hasn't started as well because ls_in_recovery + * is still hold. So we should not run into the case that + * resume_scan_timer() queues a timer that can occur in + * a no op. + */ + resume_scan_timer(ls); + /* unblocks processes waiting to enter the dlm */ + up_write(&ls->ls_in_recovery); + clear_bit(LSFL_RECOVER_LOCK, &ls->ls_flags); + error = 0; + } + spin_unlock_bh(&ls->ls_recover_lock); + + write_unlock_bh(&ls->ls_recv_active); + return error; +} + +static int ls_recover(struct dlm_ls *ls, struct dlm_recover *rv) +{ + LIST_HEAD(root_list); + unsigned long start; + int error, neg = 0; + + log_rinfo(ls, "dlm_recover %llu", (unsigned long long)rv->seq); + + mutex_lock(&ls->ls_recoverd_active); + + dlm_callback_suspend(ls); + + dlm_clear_inactive(ls); + + /* + * This list of root rsb's will be the basis of most of the recovery + * routines. + */ + + dlm_create_root_list(ls, &root_list); + + /* + * Add or remove nodes from the lockspace's ls_nodes list. + * + * Due to the fact that we must report all membership changes to lsops + * or midcomms layer, it is not permitted to abort ls_recover() until + * this is done. + */ + + error = dlm_recover_members(ls, rv, &neg); + if (error) { + log_rinfo(ls, "dlm_recover_members error %d", error); + goto fail_root_list; + } + + dlm_recover_dir_nodeid(ls, &root_list); + + /* Create a snapshot of all active rsbs were we are the master of. + * During the barrier between dlm_recover_members_wait() and + * dlm_recover_directory() other nodes can dump their necessary + * directory dlm_rsb (r->res_dir_nodeid == nodeid) in rcom + * communication dlm_copy_master_names() handling. + * + * TODO We should create a per lockspace list that contains rsbs + * that we are the master of. Instead of creating this list while + * recovery we keep track of those rsbs while locking handling and + * recovery can use it when necessary. + */ + error = dlm_create_masters_list(ls); + if (error) { + log_rinfo(ls, "dlm_create_masters_list error %d", error); + goto fail_root_list; + } + + ls->ls_recover_locks_in = 0; + + dlm_set_recover_status(ls, DLM_RS_NODES); + + error = dlm_recover_members_wait(ls, rv->seq); + if (error) { + log_rinfo(ls, "dlm_recover_members_wait error %d", error); + dlm_release_masters_list(ls); + goto fail_root_list; + } + + start = jiffies; + + /* + * Rebuild our own share of the directory by collecting from all other + * nodes their master rsb names that hash to us. + */ + + error = dlm_recover_directory(ls, rv->seq); + if (error) { + log_rinfo(ls, "dlm_recover_directory error %d", error); + dlm_release_masters_list(ls); + goto fail_root_list; + } + + dlm_set_recover_status(ls, DLM_RS_DIR); + + error = dlm_recover_directory_wait(ls, rv->seq); + if (error) { + log_rinfo(ls, "dlm_recover_directory_wait error %d", error); + dlm_release_masters_list(ls); + goto fail_root_list; + } + + dlm_release_masters_list(ls); + + /* + * We may have outstanding operations that are waiting for a reply from + * a failed node. Mark these to be resent after recovery. Unlock and + * cancel ops can just be completed. + */ + + dlm_recover_waiters_pre(ls); + + if (dlm_recovery_stopped(ls)) { + error = -EINTR; + goto fail_root_list; + } + + if (neg || dlm_no_directory(ls)) { + /* + * Clear lkb's for departed nodes. + */ + + dlm_recover_purge(ls, &root_list); + + /* + * Get new master nodeid's for rsb's that were mastered on + * departed nodes. + */ + + error = dlm_recover_masters(ls, rv->seq, &root_list); + if (error) { + log_rinfo(ls, "dlm_recover_masters error %d", error); + goto fail_root_list; + } + + /* + * Send our locks on remastered rsb's to the new masters. + */ + + error = dlm_recover_locks(ls, rv->seq, &root_list); + if (error) { + log_rinfo(ls, "dlm_recover_locks error %d", error); + goto fail_root_list; + } + + dlm_set_recover_status(ls, DLM_RS_LOCKS); + + error = dlm_recover_locks_wait(ls, rv->seq); + if (error) { + log_rinfo(ls, "dlm_recover_locks_wait error %d", error); + goto fail_root_list; + } + + log_rinfo(ls, "dlm_recover_locks %u in", + ls->ls_recover_locks_in); + + /* + * Finalize state in master rsb's now that all locks can be + * checked. This includes conversion resolution and lvb + * settings. + */ + + dlm_recover_rsbs(ls, &root_list); + } else { + /* + * Other lockspace members may be going through the "neg" steps + * while also adding us to the lockspace, in which case they'll + * be doing the recover_locks (RS_LOCKS) barrier. + */ + dlm_set_recover_status(ls, DLM_RS_LOCKS); + + error = dlm_recover_locks_wait(ls, rv->seq); + if (error) { + log_rinfo(ls, "dlm_recover_locks_wait error %d", error); + goto fail_root_list; + } + } + + dlm_release_root_list(&root_list); + + /* + * Purge directory-related requests that are saved in requestqueue. + * All dir requests from before recovery are invalid now due to the dir + * rebuild and will be resent by the requesting nodes. + */ + + dlm_purge_requestqueue(ls); + + dlm_set_recover_status(ls, DLM_RS_DONE); + + error = dlm_recover_done_wait(ls, rv->seq); + if (error) { + log_rinfo(ls, "dlm_recover_done_wait error %d", error); + goto fail; + } + + dlm_clear_members_gone(ls); + + dlm_callback_resume(ls); + + error = enable_locking(ls, rv->seq); + if (error) { + log_rinfo(ls, "enable_locking error %d", error); + goto fail; + } + + error = dlm_process_requestqueue(ls); + if (error) { + log_rinfo(ls, "dlm_process_requestqueue error %d", error); + goto fail; + } + + error = dlm_recover_waiters_post(ls); + if (error) { + log_rinfo(ls, "dlm_recover_waiters_post error %d", error); + goto fail; + } + + dlm_recover_grant(ls); + + log_rinfo(ls, "dlm_recover %llu generation %u done: %u ms", + (unsigned long long)rv->seq, ls->ls_generation, + jiffies_to_msecs(jiffies - start)); + mutex_unlock(&ls->ls_recoverd_active); + + return 0; + + fail_root_list: + dlm_release_root_list(&root_list); + fail: + mutex_unlock(&ls->ls_recoverd_active); + + return error; +} + +/* The dlm_ls_start() that created the rv we take here may already have been + stopped via dlm_ls_stop(); in that case we need to leave the RECOVERY_STOP + flag set. */ + +static void do_ls_recovery(struct dlm_ls *ls) +{ + struct dlm_recover *rv = NULL; + int error; + + spin_lock_bh(&ls->ls_recover_lock); + rv = ls->ls_recover_args; + ls->ls_recover_args = NULL; + if (rv && ls->ls_recover_seq == rv->seq) + clear_bit(LSFL_RECOVER_STOP, &ls->ls_flags); + spin_unlock_bh(&ls->ls_recover_lock); + + if (rv) { + error = ls_recover(ls, rv); + switch (error) { + case 0: + ls->ls_recovery_result = 0; + complete(&ls->ls_recovery_done); + + dlm_lsop_recover_done(ls); + break; + case -EINTR: + /* if recovery was interrupted -EINTR we wait for the next + * ls_recover() iteration until it hopefully succeeds. + */ + log_rinfo(ls, "%s %llu interrupted and should be queued to run again", + __func__, (unsigned long long)rv->seq); + break; + default: + log_rinfo(ls, "%s %llu error %d", __func__, + (unsigned long long)rv->seq, error); + + /* let new_lockspace() get aware of critical error */ + ls->ls_recovery_result = error; + complete(&ls->ls_recovery_done); + break; + } + + kfree(rv->nodes); + kfree(rv); + } +} + +static int dlm_recoverd(void *arg) +{ + struct dlm_ls *ls; + + ls = dlm_find_lockspace_local(arg); + if (!ls) { + log_print("dlm_recoverd: no lockspace %p", arg); + return -1; + } + + down_write(&ls->ls_in_recovery); + set_bit(LSFL_RECOVER_LOCK, &ls->ls_flags); + wake_up(&ls->ls_recover_lock_wait); + + while (1) { + /* + * We call kthread_should_stop() after set_current_state(). + * This is because it works correctly if kthread_stop() is + * called just before set_current_state(). + */ + set_current_state(TASK_INTERRUPTIBLE); + if (kthread_should_stop()) { + set_current_state(TASK_RUNNING); + break; + } + if (!test_bit(LSFL_RECOVER_WORK, &ls->ls_flags) && + !test_bit(LSFL_RECOVER_DOWN, &ls->ls_flags)) { + if (kthread_should_stop()) + break; + schedule(); + } + set_current_state(TASK_RUNNING); + + if (test_and_clear_bit(LSFL_RECOVER_DOWN, &ls->ls_flags)) { + down_write(&ls->ls_in_recovery); + set_bit(LSFL_RECOVER_LOCK, &ls->ls_flags); + wake_up(&ls->ls_recover_lock_wait); + } + + if (test_and_clear_bit(LSFL_RECOVER_WORK, &ls->ls_flags)) + do_ls_recovery(ls); + } + + if (test_bit(LSFL_RECOVER_LOCK, &ls->ls_flags)) + up_write(&ls->ls_in_recovery); + + dlm_put_lockspace(ls); + return 0; +} + +int dlm_recoverd_start(struct dlm_ls *ls) +{ + struct task_struct *p; + int error = 0; + + p = kthread_run(dlm_recoverd, ls, "dlm_recoverd"); + if (IS_ERR(p)) + error = PTR_ERR(p); + else + ls->ls_recoverd_task = p; + return error; +} + +void dlm_recoverd_stop(struct dlm_ls *ls) +{ + kthread_stop(ls->ls_recoverd_task); +} + +void dlm_recoverd_suspend(struct dlm_ls *ls) +{ + wake_up(&ls->ls_wait_general); + mutex_lock(&ls->ls_recoverd_active); +} + +void dlm_recoverd_resume(struct dlm_ls *ls) +{ + mutex_unlock(&ls->ls_recoverd_active); +} + diff --git a/fs/dlm/recoverd.h b/fs/dlm/recoverd.h new file mode 100644 index 000000000..d1944dc5f --- /dev/null +++ b/fs/dlm/recoverd.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. +** Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#ifndef __RECOVERD_DOT_H__ +#define __RECOVERD_DOT_H__ + +void dlm_recoverd_stop(struct dlm_ls *ls); +int dlm_recoverd_start(struct dlm_ls *ls); +void dlm_recoverd_suspend(struct dlm_ls *ls); +void dlm_recoverd_resume(struct dlm_ls *ls); + +#endif /* __RECOVERD_DOT_H__ */ + diff --git a/fs/dlm/requestqueue.c b/fs/dlm/requestqueue.c new file mode 100644 index 000000000..719a5243a --- /dev/null +++ b/fs/dlm/requestqueue.c @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: GPL-2.0-only +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) 2005-2007 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#include "dlm_internal.h" +#include "member.h" +#include "lock.h" +#include "dir.h" +#include "config.h" +#include "requestqueue.h" +#include "util.h" + +struct rq_entry { + struct list_head list; + uint32_t recover_seq; + int nodeid; + struct dlm_message request; +}; + +/* + * Requests received while the lockspace is in recovery get added to the + * request queue and processed when recovery is complete. This happens when + * the lockspace is suspended on some nodes before it is on others, or the + * lockspace is enabled on some while still suspended on others. + */ + +void dlm_add_requestqueue(struct dlm_ls *ls, int nodeid, + const struct dlm_message *ms) +{ + struct rq_entry *e; + int length = le16_to_cpu(ms->m_header.h_length) - + sizeof(struct dlm_message); + + e = kmalloc(sizeof(struct rq_entry) + length, GFP_ATOMIC); + if (!e) { + log_print("dlm_add_requestqueue: out of memory len %d", length); + return; + } + + e->recover_seq = ls->ls_recover_seq & 0xFFFFFFFF; + e->nodeid = nodeid; + memcpy(&e->request, ms, sizeof(*ms)); + memcpy(&e->request.m_extra, ms->m_extra, length); + + list_add_tail(&e->list, &ls->ls_requestqueue); +} + +/* + * Called by dlm_recoverd to process normal messages saved while recovery was + * happening. Normal locking has been enabled before this is called. dlm_recv + * upon receiving a message, will wait for all saved messages to be drained + * here before processing the message it got. If a new dlm_ls_stop() arrives + * while we're processing these saved messages, it may block trying to suspend + * dlm_recv if dlm_recv is waiting for us in dlm_wait_requestqueue. In that + * case, we don't abort since locking_stopped is still 0. If dlm_recv is not + * waiting for us, then this processing may be aborted due to locking_stopped. + */ + +int dlm_process_requestqueue(struct dlm_ls *ls) +{ + struct rq_entry *e; + struct dlm_message *ms; + int error = 0; + + write_lock_bh(&ls->ls_requestqueue_lock); + for (;;) { + if (list_empty(&ls->ls_requestqueue)) { + clear_bit(LSFL_RECV_MSG_BLOCKED, &ls->ls_flags); + error = 0; + break; + } + e = list_first_entry(&ls->ls_requestqueue, struct rq_entry, list); + + ms = &e->request; + + log_limit(ls, "dlm_process_requestqueue msg %d from %d " + "lkid %x remid %x result %d seq %u", + le32_to_cpu(ms->m_type), + le32_to_cpu(ms->m_header.h_nodeid), + le32_to_cpu(ms->m_lkid), le32_to_cpu(ms->m_remid), + from_dlm_errno(le32_to_cpu(ms->m_result)), + e->recover_seq); + + dlm_receive_message_saved(ls, &e->request, e->recover_seq); + list_del(&e->list); + kfree(e); + + if (dlm_locking_stopped(ls)) { + log_debug(ls, "process_requestqueue abort running"); + error = -EINTR; + break; + } + write_unlock_bh(&ls->ls_requestqueue_lock); + schedule(); + write_lock_bh(&ls->ls_requestqueue_lock); + } + write_unlock_bh(&ls->ls_requestqueue_lock); + + return error; +} + +static int purge_request(struct dlm_ls *ls, struct dlm_message *ms, int nodeid) +{ + __le32 type = ms->m_type; + + /* the ls is being cleaned up and freed by release_lockspace */ + if (!atomic_read(&ls->ls_count)) + return 1; + + if (dlm_is_removed(ls, nodeid)) + return 1; + + /* directory operations are always purged because the directory is + always rebuilt during recovery and the lookups resent */ + + if (type == cpu_to_le32(DLM_MSG_REMOVE) || + type == cpu_to_le32(DLM_MSG_LOOKUP) || + type == cpu_to_le32(DLM_MSG_LOOKUP_REPLY)) + return 1; + + if (!dlm_no_directory(ls)) + return 0; + + return 1; +} + +void dlm_purge_requestqueue(struct dlm_ls *ls) +{ + struct dlm_message *ms; + struct rq_entry *e, *safe; + + write_lock_bh(&ls->ls_requestqueue_lock); + list_for_each_entry_safe(e, safe, &ls->ls_requestqueue, list) { + ms = &e->request; + + if (purge_request(ls, ms, e->nodeid)) { + list_del(&e->list); + kfree(e); + } + } + write_unlock_bh(&ls->ls_requestqueue_lock); +} + diff --git a/fs/dlm/requestqueue.h b/fs/dlm/requestqueue.h new file mode 100644 index 000000000..42bfe23ce --- /dev/null +++ b/fs/dlm/requestqueue.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) 2005-2007 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#ifndef __REQUESTQUEUE_DOT_H__ +#define __REQUESTQUEUE_DOT_H__ + +void dlm_add_requestqueue(struct dlm_ls *ls, int nodeid, + const struct dlm_message *ms); +int dlm_process_requestqueue(struct dlm_ls *ls); +void dlm_wait_requestqueue(struct dlm_ls *ls); +void dlm_purge_requestqueue(struct dlm_ls *ls); + +#endif + diff --git a/fs/dlm/user.c b/fs/dlm/user.c new file mode 100644 index 000000000..a8ed4c8fd --- /dev/null +++ b/fs/dlm/user.c @@ -0,0 +1,989 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2006-2010 Red Hat, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "dlm_internal.h" +#include "lockspace.h" +#include "lock.h" +#include "lvb_table.h" +#include "user.h" +#include "ast.h" +#include "config.h" +#include "memory.h" + +static const char name_prefix[] = "dlm"; +static const struct file_operations device_fops; +static atomic_t dlm_monitor_opened; +static int dlm_monitor_unused = 1; + +#ifdef CONFIG_COMPAT + +struct dlm_lock_params32 { + __u8 mode; + __u8 namelen; + __u16 unused; + __u32 flags; + __u32 lkid; + __u32 parent; + __u64 xid; + __u64 timeout; + __u32 castparam; + __u32 castaddr; + __u32 bastparam; + __u32 bastaddr; + __u32 lksb; + char lvb[DLM_USER_LVB_LEN]; + char name[]; +}; + +struct dlm_write_request32 { + __u32 version[3]; + __u8 cmd; + __u8 is64bit; + __u8 unused[2]; + + union { + struct dlm_lock_params32 lock; + struct dlm_lspace_params lspace; + struct dlm_purge_params purge; + } i; +}; + +struct dlm_lksb32 { + __u32 sb_status; + __u32 sb_lkid; + __u8 sb_flags; + __u32 sb_lvbptr; +}; + +struct dlm_lock_result32 { + __u32 version[3]; + __u32 length; + __u32 user_astaddr; + __u32 user_astparam; + __u32 user_lksb; + struct dlm_lksb32 lksb; + __u8 bast_mode; + __u8 unused[3]; + /* Offsets may be zero if no data is present */ + __u32 lvb_offset; +}; + +static void compat_input(struct dlm_write_request *kb, + struct dlm_write_request32 *kb32, + int namelen) +{ + kb->version[0] = kb32->version[0]; + kb->version[1] = kb32->version[1]; + kb->version[2] = kb32->version[2]; + + kb->cmd = kb32->cmd; + kb->is64bit = kb32->is64bit; + if (kb->cmd == DLM_USER_CREATE_LOCKSPACE || + kb->cmd == DLM_USER_REMOVE_LOCKSPACE) { + kb->i.lspace.flags = kb32->i.lspace.flags; + kb->i.lspace.minor = kb32->i.lspace.minor; + memcpy(kb->i.lspace.name, kb32->i.lspace.name, namelen); + } else if (kb->cmd == DLM_USER_PURGE) { + kb->i.purge.nodeid = kb32->i.purge.nodeid; + kb->i.purge.pid = kb32->i.purge.pid; + } else { + kb->i.lock.mode = kb32->i.lock.mode; + kb->i.lock.namelen = kb32->i.lock.namelen; + kb->i.lock.flags = kb32->i.lock.flags; + kb->i.lock.lkid = kb32->i.lock.lkid; + kb->i.lock.parent = kb32->i.lock.parent; + kb->i.lock.xid = kb32->i.lock.xid; + kb->i.lock.timeout = kb32->i.lock.timeout; + kb->i.lock.castparam = (__user void *)(long)kb32->i.lock.castparam; + kb->i.lock.castaddr = (__user void *)(long)kb32->i.lock.castaddr; + kb->i.lock.bastparam = (__user void *)(long)kb32->i.lock.bastparam; + kb->i.lock.bastaddr = (__user void *)(long)kb32->i.lock.bastaddr; + kb->i.lock.lksb = (__user void *)(long)kb32->i.lock.lksb; + memcpy(kb->i.lock.lvb, kb32->i.lock.lvb, DLM_USER_LVB_LEN); + memcpy(kb->i.lock.name, kb32->i.lock.name, namelen); + } +} + +static void compat_output(struct dlm_lock_result *res, + struct dlm_lock_result32 *res32) +{ + memset(res32, 0, sizeof(*res32)); + + res32->version[0] = res->version[0]; + res32->version[1] = res->version[1]; + res32->version[2] = res->version[2]; + + res32->user_astaddr = (__u32)(__force long)res->user_astaddr; + res32->user_astparam = (__u32)(__force long)res->user_astparam; + res32->user_lksb = (__u32)(__force long)res->user_lksb; + res32->bast_mode = res->bast_mode; + + res32->lvb_offset = res->lvb_offset; + res32->length = res->length; + + res32->lksb.sb_status = res->lksb.sb_status; + res32->lksb.sb_flags = res->lksb.sb_flags; + res32->lksb.sb_lkid = res->lksb.sb_lkid; + res32->lksb.sb_lvbptr = (__u32)(long)res->lksb.sb_lvbptr; +} +#endif + +/* Figure out if this lock is at the end of its life and no longer + available for the application to use. The lkb still exists until + the final ast is read. A lock becomes EOL in three situations: + 1. a noqueue request fails with EAGAIN + 2. an unlock completes with EUNLOCK + 3. a cancel of a waiting request completes with ECANCEL/EDEADLK + An EOL lock needs to be removed from the process's list of locks. + And we can't allow any new operation on an EOL lock. This is + not related to the lifetime of the lkb struct which is managed + entirely by refcount. */ + +static int lkb_is_endoflife(int mode, int status) +{ + switch (status) { + case -DLM_EUNLOCK: + return 1; + case -DLM_ECANCEL: + case -ETIMEDOUT: + case -EDEADLK: + case -EAGAIN: + if (mode == DLM_LOCK_IV) + return 1; + break; + } + return 0; +} + +/* we could possibly check if the cancel of an orphan has resulted in the lkb + being removed and then remove that lkb from the orphans list and free it */ + +void dlm_user_add_ast(struct dlm_lkb *lkb, uint32_t flags, int mode, + int status, uint32_t sbflags) +{ + struct dlm_ls *ls; + struct dlm_user_args *ua; + struct dlm_user_proc *proc; + struct dlm_callback *cb; + int rv, copy_lvb; + + if (test_bit(DLM_DFL_ORPHAN_BIT, &lkb->lkb_dflags) || + test_bit(DLM_IFL_DEAD_BIT, &lkb->lkb_iflags)) + return; + + ls = lkb->lkb_resource->res_ls; + spin_lock_bh(&ls->ls_clear_proc_locks); + + /* If ORPHAN/DEAD flag is set, it means the process is dead so an ast + can't be delivered. For ORPHAN's, dlm_clear_proc_locks() freed + lkb->ua so we can't try to use it. This second check is necessary + for cases where a completion ast is received for an operation that + began before clear_proc_locks did its cancel/unlock. */ + + if (test_bit(DLM_DFL_ORPHAN_BIT, &lkb->lkb_dflags) || + test_bit(DLM_IFL_DEAD_BIT, &lkb->lkb_iflags)) + goto out; + + DLM_ASSERT(lkb->lkb_ua, dlm_print_lkb(lkb);); + ua = lkb->lkb_ua; + proc = ua->proc; + + if ((flags & DLM_CB_BAST) && ua->bastaddr == NULL) + goto out; + + if ((flags & DLM_CB_CAST) && lkb_is_endoflife(mode, status)) + set_bit(DLM_IFL_ENDOFLIFE_BIT, &lkb->lkb_iflags); + + spin_lock_bh(&proc->asts_spin); + + if (!dlm_may_skip_callback(lkb, flags, mode, status, sbflags, + ©_lvb)) { + rv = dlm_get_cb(lkb, flags, mode, status, sbflags, &cb); + if (!rv) { + cb->copy_lvb = copy_lvb; + cb->ua = *ua; + cb->lkb_lksb = &cb->ua.lksb; + if (copy_lvb) { + memcpy(cb->lvbptr, ua->lksb.sb_lvbptr, + DLM_USER_LVB_LEN); + cb->lkb_lksb->sb_lvbptr = cb->lvbptr; + } + + list_add_tail(&cb->list, &proc->asts); + wake_up_interruptible(&proc->wait); + } + } + spin_unlock_bh(&proc->asts_spin); + + if (test_bit(DLM_IFL_ENDOFLIFE_BIT, &lkb->lkb_iflags)) { + /* N.B. spin_lock locks_spin, not asts_spin */ + spin_lock_bh(&proc->locks_spin); + if (!list_empty(&lkb->lkb_ownqueue)) { + list_del_init(&lkb->lkb_ownqueue); + dlm_put_lkb(lkb); + } + spin_unlock_bh(&proc->locks_spin); + } + out: + spin_unlock_bh(&ls->ls_clear_proc_locks); +} + +static int device_user_lock(struct dlm_user_proc *proc, + struct dlm_lock_params *params) +{ + struct dlm_ls *ls; + struct dlm_user_args *ua; + uint32_t lkid; + int error = -ENOMEM; + + ls = dlm_find_lockspace_local(proc->lockspace); + if (!ls) + return -ENOENT; + + if (!params->castaddr || !params->lksb) { + error = -EINVAL; + goto out; + } + + ua = kzalloc_obj(struct dlm_user_args, GFP_NOFS); + if (!ua) + goto out; + ua->proc = proc; + ua->user_lksb = params->lksb; + ua->castparam = params->castparam; + ua->castaddr = params->castaddr; + ua->bastparam = params->bastparam; + ua->bastaddr = params->bastaddr; + ua->xid = params->xid; + + if (params->flags & DLM_LKF_CONVERT) { + error = dlm_user_convert(ls, ua, + params->mode, params->flags, + params->lkid, params->lvb); + } else if (params->flags & DLM_LKF_ORPHAN) { + error = dlm_user_adopt_orphan(ls, ua, + params->mode, params->flags, + params->name, params->namelen, + &lkid); + if (!error) + error = lkid; + } else { + error = dlm_user_request(ls, ua, + params->mode, params->flags, + params->name, params->namelen); + if (!error) + error = ua->lksb.sb_lkid; + } + out: + dlm_put_lockspace(ls); + return error; +} + +static int device_user_unlock(struct dlm_user_proc *proc, + struct dlm_lock_params *params) +{ + struct dlm_ls *ls; + struct dlm_user_args *ua; + int error = -ENOMEM; + + ls = dlm_find_lockspace_local(proc->lockspace); + if (!ls) + return -ENOENT; + + ua = kzalloc_obj(struct dlm_user_args, GFP_NOFS); + if (!ua) + goto out; + ua->proc = proc; + ua->user_lksb = params->lksb; + ua->castparam = params->castparam; + ua->castaddr = params->castaddr; + + if (params->flags & DLM_LKF_CANCEL) + error = dlm_user_cancel(ls, ua, params->flags, params->lkid); + else + error = dlm_user_unlock(ls, ua, params->flags, params->lkid, + params->lvb); + out: + dlm_put_lockspace(ls); + return error; +} + +static int device_user_deadlock(struct dlm_user_proc *proc, + struct dlm_lock_params *params) +{ + struct dlm_ls *ls; + int error; + + ls = dlm_find_lockspace_local(proc->lockspace); + if (!ls) + return -ENOENT; + + error = dlm_user_deadlock(ls, params->flags, params->lkid); + + dlm_put_lockspace(ls); + return error; +} + +static int dlm_device_register(struct dlm_ls *ls, char *name) +{ + int error, len; + + /* The device is already registered. This happens when the + lockspace is created multiple times from userspace. */ + if (ls->ls_device.name) + return 0; + + error = -ENOMEM; + len = strlen(name) + strlen(name_prefix) + 2; + ls->ls_device.name = kzalloc(len, GFP_NOFS); + if (!ls->ls_device.name) + goto fail; + + snprintf((char *)ls->ls_device.name, len, "%s_%s", name_prefix, + name); + ls->ls_device.fops = &device_fops; + ls->ls_device.minor = MISC_DYNAMIC_MINOR; + + error = misc_register(&ls->ls_device); + if (error) { + kfree(ls->ls_device.name); + /* this has to be set to NULL + * to avoid a double-free in dlm_device_deregister + */ + ls->ls_device.name = NULL; + } +fail: + return error; +} + +int dlm_device_deregister(struct dlm_ls *ls) +{ + /* The device is not registered. This happens when the lockspace + was never used from userspace, or when device_create_lockspace() + calls dlm_release_lockspace() after the register fails. */ + if (!ls->ls_device.name) + return 0; + + misc_deregister(&ls->ls_device); + kfree(ls->ls_device.name); + return 0; +} + +static int device_user_purge(struct dlm_user_proc *proc, + struct dlm_purge_params *params) +{ + struct dlm_ls *ls; + int error; + + ls = dlm_find_lockspace_local(proc->lockspace); + if (!ls) + return -ENOENT; + + error = dlm_user_purge(ls, proc, params->nodeid, params->pid); + + dlm_put_lockspace(ls); + return error; +} + +static int device_create_lockspace(struct dlm_lspace_params *params) +{ + dlm_lockspace_t *lockspace; + struct dlm_ls *ls; + int error; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + error = dlm_new_user_lockspace(params->name, dlm_config.ci_cluster_name, + params->flags, DLM_USER_LVB_LEN, NULL, + NULL, NULL, &lockspace); + if (error) + return error; + + ls = dlm_find_lockspace_local(lockspace); + if (!ls) + return -ENOENT; + + error = dlm_device_register(ls, params->name); + dlm_put_lockspace(ls); + + if (error) + dlm_release_lockspace(lockspace, DLM_RELEASE_NO_LOCKS); + else + error = ls->ls_device.minor; + + return error; +} + +static int device_remove_lockspace(struct dlm_lspace_params *params) +{ + dlm_lockspace_t *lockspace; + struct dlm_ls *ls; + int error, force = DLM_RELEASE_NO_LOCKS; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + ls = dlm_find_lockspace_device(params->minor); + if (!ls) + return -ENOENT; + + if (params->flags & DLM_USER_LSFLG_FORCEFREE) + force = DLM_RELEASE_NORMAL; + + lockspace = ls; + dlm_put_lockspace(ls); + + /* The final dlm_release_lockspace waits for references to go to + zero, so all processes will need to close their device for the + ls before the release will proceed. release also calls the + device_deregister above. Converting a positive return value + from release to zero means that userspace won't know when its + release was the final one, but it shouldn't need to know. */ + + error = dlm_release_lockspace(lockspace, force); + if (error > 0) + error = 0; + return error; +} + +/* Check the user's version matches ours */ +static int check_version(struct dlm_write_request *req) +{ + if (req->version[0] != DLM_DEVICE_VERSION_MAJOR || + (req->version[0] == DLM_DEVICE_VERSION_MAJOR && + req->version[1] > DLM_DEVICE_VERSION_MINOR)) { + + printk(KERN_DEBUG "dlm: process %s (%d) version mismatch " + "user (%d.%d.%d) kernel (%d.%d.%d)\n", + current->comm, + task_pid_nr(current), + req->version[0], + req->version[1], + req->version[2], + DLM_DEVICE_VERSION_MAJOR, + DLM_DEVICE_VERSION_MINOR, + DLM_DEVICE_VERSION_PATCH); + return -EINVAL; + } + return 0; +} + +/* + * device_write + * + * device_user_lock + * dlm_user_request -> request_lock + * dlm_user_convert -> convert_lock + * + * device_user_unlock + * dlm_user_unlock -> unlock_lock + * dlm_user_cancel -> cancel_lock + * + * device_create_lockspace + * dlm_new_lockspace + * + * device_remove_lockspace + * dlm_release_lockspace + */ + +/* a write to a lockspace device is a lock or unlock request, a write + to the control device is to create/remove a lockspace */ + +static ssize_t device_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct dlm_user_proc *proc = file->private_data; + struct dlm_write_request *kbuf; + int error; + +#ifdef CONFIG_COMPAT + if (count < sizeof(struct dlm_write_request32)) +#else + if (count < sizeof(struct dlm_write_request)) +#endif + return -EINVAL; + + /* + * can't compare against COMPAT/dlm_write_request32 because + * we don't yet know if is64bit is zero + */ + if (count > sizeof(struct dlm_write_request) + DLM_RESNAME_MAXLEN) + return -EINVAL; + + kbuf = memdup_user_nul(buf, count); + if (IS_ERR(kbuf)) + return PTR_ERR(kbuf); + + if (check_version(kbuf)) { + error = -EBADE; + goto out_free; + } + +#ifdef CONFIG_COMPAT + if (!kbuf->is64bit) { + struct dlm_write_request32 *k32buf; + int namelen = 0; + + if (count > sizeof(struct dlm_write_request32)) + namelen = count - sizeof(struct dlm_write_request32); + + k32buf = (struct dlm_write_request32 *)kbuf; + + /* add 1 after namelen so that the name string is terminated */ + kbuf = kzalloc(sizeof(struct dlm_write_request) + namelen + 1, + GFP_NOFS); + if (!kbuf) { + kfree(k32buf); + return -ENOMEM; + } + + if (proc) + set_bit(DLM_PROC_FLAGS_COMPAT, &proc->flags); + + compat_input(kbuf, k32buf, namelen); + kfree(k32buf); + } +#endif + + /* do we really need this? can a write happen after a close? */ + if ((kbuf->cmd == DLM_USER_LOCK || kbuf->cmd == DLM_USER_UNLOCK) && + (proc && test_bit(DLM_PROC_FLAGS_CLOSING, &proc->flags))) { + error = -EINVAL; + goto out_free; + } + + error = -EINVAL; + + switch (kbuf->cmd) + { + case DLM_USER_LOCK: + if (!proc) { + log_print("no locking on control device"); + goto out_free; + } + error = device_user_lock(proc, &kbuf->i.lock); + break; + + case DLM_USER_UNLOCK: + if (!proc) { + log_print("no locking on control device"); + goto out_free; + } + error = device_user_unlock(proc, &kbuf->i.lock); + break; + + case DLM_USER_DEADLOCK: + if (!proc) { + log_print("no locking on control device"); + goto out_free; + } + error = device_user_deadlock(proc, &kbuf->i.lock); + break; + + case DLM_USER_CREATE_LOCKSPACE: + if (proc) { + log_print("create/remove only on control device"); + goto out_free; + } + error = device_create_lockspace(&kbuf->i.lspace); + break; + + case DLM_USER_REMOVE_LOCKSPACE: + if (proc) { + log_print("create/remove only on control device"); + goto out_free; + } + error = device_remove_lockspace(&kbuf->i.lspace); + break; + + case DLM_USER_PURGE: + if (!proc) { + log_print("no locking on control device"); + goto out_free; + } + error = device_user_purge(proc, &kbuf->i.purge); + break; + + default: + log_print("Unknown command passed to DLM device : %d\n", + kbuf->cmd); + } + + out_free: + kfree(kbuf); + return error; +} + +/* Every process that opens the lockspace device has its own "proc" structure + hanging off the open file that's used to keep track of locks owned by the + process and asts that need to be delivered to the process. */ + +static int device_open(struct inode *inode, struct file *file) +{ + struct dlm_user_proc *proc; + struct dlm_ls *ls; + + ls = dlm_find_lockspace_device(iminor(inode)); + if (!ls) + return -ENOENT; + + proc = kzalloc_obj(struct dlm_user_proc, GFP_NOFS); + if (!proc) { + dlm_put_lockspace(ls); + return -ENOMEM; + } + + proc->lockspace = ls; + INIT_LIST_HEAD(&proc->asts); + INIT_LIST_HEAD(&proc->locks); + INIT_LIST_HEAD(&proc->unlocking); + spin_lock_init(&proc->asts_spin); + spin_lock_init(&proc->locks_spin); + init_waitqueue_head(&proc->wait); + file->private_data = proc; + + return 0; +} + +static int device_close(struct inode *inode, struct file *file) +{ + struct dlm_user_proc *proc = file->private_data; + struct dlm_ls *ls; + + ls = dlm_find_lockspace_local(proc->lockspace); + if (!ls) + return -ENOENT; + + set_bit(DLM_PROC_FLAGS_CLOSING, &proc->flags); + + dlm_clear_proc_locks(ls, proc); + + /* at this point no more lkb's should exist for this lockspace, + so there's no chance of dlm_user_add_ast() being called and + looking for lkb->ua->proc */ + + kfree(proc); + file->private_data = NULL; + + dlm_put_lockspace(ls); + dlm_put_lockspace(ls); /* for the find in device_open() */ + + /* FIXME: AUTOFREE: if this ls is no longer used do + device_remove_lockspace() */ + + return 0; +} + +static int copy_result_to_user(struct dlm_user_args *ua, int compat, + uint32_t flags, int mode, int copy_lvb, + char __user *buf, size_t count) +{ +#ifdef CONFIG_COMPAT + struct dlm_lock_result32 result32; +#endif + struct dlm_lock_result result; + void *resultptr; + int error=0; + int len; + int struct_len; + + memset(&result, 0, sizeof(struct dlm_lock_result)); + result.version[0] = DLM_DEVICE_VERSION_MAJOR; + result.version[1] = DLM_DEVICE_VERSION_MINOR; + result.version[2] = DLM_DEVICE_VERSION_PATCH; + memcpy(&result.lksb, &ua->lksb, offsetof(struct dlm_lksb, sb_lvbptr)); + result.user_lksb = ua->user_lksb; + + /* FIXME: dlm1 provides for the user's bastparam/addr to not be updated + in a conversion unless the conversion is successful. See code + in dlm_user_convert() for updating ua from ua_tmp. OpenVMS, though, + notes that a new blocking AST address and parameter are set even if + the conversion fails, so maybe we should just do that. */ + + if (flags & DLM_CB_BAST) { + result.user_astaddr = ua->bastaddr; + result.user_astparam = ua->bastparam; + result.bast_mode = mode; + } else { + result.user_astaddr = ua->castaddr; + result.user_astparam = ua->castparam; + } + +#ifdef CONFIG_COMPAT + if (compat) + len = sizeof(struct dlm_lock_result32); + else +#endif + len = sizeof(struct dlm_lock_result); + struct_len = len; + + /* copy lvb to userspace if there is one, it's been updated, and + the user buffer has space for it */ + + if (copy_lvb && ua->lksb.sb_lvbptr && count >= len + DLM_USER_LVB_LEN) { + if (copy_to_user(buf+len, ua->lksb.sb_lvbptr, + DLM_USER_LVB_LEN)) { + error = -EFAULT; + goto out; + } + + result.lvb_offset = len; + len += DLM_USER_LVB_LEN; + } + + result.length = len; + resultptr = &result; +#ifdef CONFIG_COMPAT + if (compat) { + compat_output(&result, &result32); + resultptr = &result32; + } +#endif + + if (copy_to_user(buf, resultptr, struct_len)) + error = -EFAULT; + else + error = len; + out: + return error; +} + +static int copy_version_to_user(char __user *buf, size_t count) +{ + struct dlm_device_version ver; + + memset(&ver, 0, sizeof(struct dlm_device_version)); + ver.version[0] = DLM_DEVICE_VERSION_MAJOR; + ver.version[1] = DLM_DEVICE_VERSION_MINOR; + ver.version[2] = DLM_DEVICE_VERSION_PATCH; + + if (copy_to_user(buf, &ver, sizeof(struct dlm_device_version))) + return -EFAULT; + return sizeof(struct dlm_device_version); +} + +/* a read returns a single ast described in a struct dlm_lock_result */ + +static ssize_t device_read(struct file *file, char __user *buf, size_t count, + loff_t *ppos) +{ + struct dlm_user_proc *proc = file->private_data; + DECLARE_WAITQUEUE(wait, current); + struct dlm_callback *cb; + int rv, ret; + + if (count == sizeof(struct dlm_device_version)) { + rv = copy_version_to_user(buf, count); + return rv; + } + + if (!proc) { + log_print("non-version read from control device %zu", count); + return -EINVAL; + } + +#ifdef CONFIG_COMPAT + if (count < sizeof(struct dlm_lock_result32)) +#else + if (count < sizeof(struct dlm_lock_result)) +#endif + return -EINVAL; + + /* do we really need this? can a read happen after a close? */ + if (test_bit(DLM_PROC_FLAGS_CLOSING, &proc->flags)) + return -EINVAL; + + spin_lock_bh(&proc->asts_spin); + if (list_empty(&proc->asts)) { + if (file->f_flags & O_NONBLOCK) { + spin_unlock_bh(&proc->asts_spin); + return -EAGAIN; + } + + add_wait_queue(&proc->wait, &wait); + + repeat: + set_current_state(TASK_INTERRUPTIBLE); + if (list_empty(&proc->asts) && !signal_pending(current)) { + spin_unlock_bh(&proc->asts_spin); + schedule(); + spin_lock_bh(&proc->asts_spin); + goto repeat; + } + set_current_state(TASK_RUNNING); + remove_wait_queue(&proc->wait, &wait); + + if (signal_pending(current)) { + spin_unlock_bh(&proc->asts_spin); + return -ERESTARTSYS; + } + } + + /* if we empty lkb_callbacks, we don't want to unlock the spinlock + without removing lkb_cb_list; so empty lkb_cb_list is always + consistent with empty lkb_callbacks */ + + cb = list_first_entry(&proc->asts, struct dlm_callback, list); + list_del(&cb->list); + spin_unlock_bh(&proc->asts_spin); + + if (cb->flags & DLM_CB_BAST) { + trace_dlm_bast(cb->ls_id, cb->lkb_id, cb->mode, cb->res_name, + cb->res_length); + } else if (cb->flags & DLM_CB_CAST) { + cb->lkb_lksb->sb_status = cb->sb_status; + cb->lkb_lksb->sb_flags = cb->sb_flags; + trace_dlm_ast(cb->ls_id, cb->lkb_id, cb->sb_status, + cb->sb_flags, cb->res_name, cb->res_length); + } + + ret = copy_result_to_user(&cb->ua, + test_bit(DLM_PROC_FLAGS_COMPAT, &proc->flags), + cb->flags, cb->mode, cb->copy_lvb, buf, count); + dlm_free_cb(cb); + return ret; +} + +static __poll_t device_poll(struct file *file, poll_table *wait) +{ + struct dlm_user_proc *proc = file->private_data; + + poll_wait(file, &proc->wait, wait); + + spin_lock_bh(&proc->asts_spin); + if (!list_empty(&proc->asts)) { + spin_unlock_bh(&proc->asts_spin); + return EPOLLIN | EPOLLRDNORM; + } + spin_unlock_bh(&proc->asts_spin); + return 0; +} + +int dlm_user_daemon_available(void) +{ + /* dlm_controld hasn't started (or, has started, but not + properly populated configfs) */ + + if (!dlm_our_nodeid()) + return 0; + + /* This is to deal with versions of dlm_controld that don't + know about the monitor device. We assume that if the + dlm_controld was started (above), but the monitor device + was never opened, that it's an old version. dlm_controld + should open the monitor device before populating configfs. */ + + if (dlm_monitor_unused) + return 1; + + return atomic_read(&dlm_monitor_opened) ? 1 : 0; +} + +static int ctl_device_open(struct inode *inode, struct file *file) +{ + file->private_data = NULL; + return 0; +} + +static int ctl_device_close(struct inode *inode, struct file *file) +{ + return 0; +} + +static int monitor_device_open(struct inode *inode, struct file *file) +{ + atomic_inc(&dlm_monitor_opened); + dlm_monitor_unused = 0; + return 0; +} + +static int monitor_device_close(struct inode *inode, struct file *file) +{ + if (atomic_dec_and_test(&dlm_monitor_opened)) + dlm_stop_lockspaces(); + return 0; +} + +static const struct file_operations device_fops = { + .open = device_open, + .release = device_close, + .read = device_read, + .write = device_write, + .poll = device_poll, + .owner = THIS_MODULE, + .llseek = noop_llseek, +}; + +static const struct file_operations ctl_device_fops = { + .open = ctl_device_open, + .release = ctl_device_close, + .read = device_read, + .write = device_write, + .owner = THIS_MODULE, + .llseek = noop_llseek, +}; + +static struct miscdevice ctl_device = { + .name = "dlm-control", + .fops = &ctl_device_fops, + .minor = MISC_DYNAMIC_MINOR, +}; + +static const struct file_operations monitor_device_fops = { + .open = monitor_device_open, + .release = monitor_device_close, + .owner = THIS_MODULE, + .llseek = noop_llseek, +}; + +static struct miscdevice monitor_device = { + .name = "dlm-monitor", + .fops = &monitor_device_fops, + .minor = MISC_DYNAMIC_MINOR, +}; + +int __init dlm_user_init(void) +{ + int error; + + atomic_set(&dlm_monitor_opened, 0); + + error = misc_register(&ctl_device); + if (error) { + log_print("misc_register failed for control device"); + goto out; + } + + error = misc_register(&monitor_device); + if (error) { + log_print("misc_register failed for monitor device"); + misc_deregister(&ctl_device); + } + out: + return error; +} + +void dlm_user_exit(void) +{ + misc_deregister(&ctl_device); + misc_deregister(&monitor_device); +} + diff --git a/fs/dlm/user.h b/fs/dlm/user.h new file mode 100644 index 000000000..2caf8e6e2 --- /dev/null +++ b/fs/dlm/user.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2006-2010 Red Hat, Inc. All rights reserved. + */ + +#ifndef __USER_DOT_H__ +#define __USER_DOT_H__ + +void dlm_purge_lkb_callbacks(struct dlm_lkb *lkb); +void dlm_user_add_ast(struct dlm_lkb *lkb, uint32_t flags, int mode, + int status, uint32_t sbflags); +int dlm_user_init(void); +void dlm_user_exit(void); +int dlm_device_deregister(struct dlm_ls *ls); +int dlm_user_daemon_available(void); + +#endif diff --git a/fs/dlm/util.c b/fs/dlm/util.c new file mode 100644 index 000000000..f2bc401f3 --- /dev/null +++ b/fs/dlm/util.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0-only +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) 2005-2008 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#include "dlm_internal.h" +#include "rcom.h" +#include "util.h" + +#define DLM_ERRNO_EDEADLK 35 +#define DLM_ERRNO_EBADR 53 +#define DLM_ERRNO_EBADSLT 57 +#define DLM_ERRNO_EPROTO 71 +#define DLM_ERRNO_EOPNOTSUPP 95 +#define DLM_ERRNO_ETIMEDOUT 110 +#define DLM_ERRNO_EINPROGRESS 115 + +/* higher errno values are inconsistent across architectures, so select + one set of values for on the wire */ + +int to_dlm_errno(int err) +{ + switch (err) { + case -EDEADLK: + return -DLM_ERRNO_EDEADLK; + case -EBADR: + return -DLM_ERRNO_EBADR; + case -EBADSLT: + return -DLM_ERRNO_EBADSLT; + case -EPROTO: + return -DLM_ERRNO_EPROTO; + case -EOPNOTSUPP: + return -DLM_ERRNO_EOPNOTSUPP; + case -ETIMEDOUT: + return -DLM_ERRNO_ETIMEDOUT; + case -EINPROGRESS: + return -DLM_ERRNO_EINPROGRESS; + } + return err; +} + +int from_dlm_errno(int err) +{ + switch (err) { + case -DLM_ERRNO_EDEADLK: + return -EDEADLK; + case -DLM_ERRNO_EBADR: + return -EBADR; + case -DLM_ERRNO_EBADSLT: + return -EBADSLT; + case -DLM_ERRNO_EPROTO: + return -EPROTO; + case -DLM_ERRNO_EOPNOTSUPP: + return -EOPNOTSUPP; + case -DLM_ERRNO_ETIMEDOUT: + return -ETIMEDOUT; + case -DLM_ERRNO_EINPROGRESS: + return -EINPROGRESS; + } + return err; +} diff --git a/fs/dlm/util.h b/fs/dlm/util.h new file mode 100644 index 000000000..b6a4b8adc --- /dev/null +++ b/fs/dlm/util.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/****************************************************************************** +******************************************************************************* +** +** Copyright (C) 2005 Red Hat, Inc. All rights reserved. +** +** +******************************************************************************* +******************************************************************************/ + +#ifndef __UTIL_DOT_H__ +#define __UTIL_DOT_H__ + +int to_dlm_errno(int err); +int from_dlm_errno(int err); + +#endif + diff --git a/fs/drop_caches.c b/fs/drop_caches.c new file mode 100644 index 000000000..49f56a598 --- /dev/null +++ b/fs/drop_caches.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Implement the manual drop-all-pagecache function + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +/* A global variable is a bit ugly, but it keeps the code simple */ +static int sysctl_drop_caches; + +static void drop_pagecache_sb(struct super_block *sb, void *unused) +{ + struct inode *inode, *toput_inode = NULL; + + spin_lock(&sb->s_inode_list_lock); + list_for_each_entry(inode, &sb->s_inodes, i_sb_list) { + spin_lock(&inode->i_lock); + /* + * We must skip inodes in unusual state. We may also skip + * inodes without pages but we deliberately won't in case + * we need to reschedule to avoid softlockups. + */ + if ((inode_state_read(inode) & (I_FREEING | I_WILL_FREE | I_NEW)) || + (mapping_empty(inode->i_mapping) && !need_resched())) { + spin_unlock(&inode->i_lock); + continue; + } + __iget(inode); + spin_unlock(&inode->i_lock); + spin_unlock(&sb->s_inode_list_lock); + + invalidate_mapping_pages(inode->i_mapping, 0, -1); + iput(toput_inode); + toput_inode = inode; + + cond_resched(); + spin_lock(&sb->s_inode_list_lock); + } + spin_unlock(&sb->s_inode_list_lock); + iput(toput_inode); +} + +static int drop_caches_sysctl_handler(const struct ctl_table *table, int write, + void *buffer, size_t *length, loff_t *ppos) +{ + int ret; + + ret = proc_dointvec_minmax(table, write, buffer, length, ppos); + if (ret) + return ret; + if (write) { + static int stfu; + + if (sysctl_drop_caches & 1) { + lru_add_drain_all(); + iterate_supers(drop_pagecache_sb, NULL); + count_vm_event(DROP_PAGECACHE); + } + if (sysctl_drop_caches & 2) { + drop_slab(); + count_vm_event(DROP_SLAB); + } + if (!stfu) { + pr_info("%s (%d): drop_caches: %d\n", + current->comm, task_pid_nr(current), + sysctl_drop_caches); + } + stfu |= sysctl_drop_caches & 4; + } + return 0; +} + +static const struct ctl_table drop_caches_table[] = { + { + .procname = "drop_caches", + .data = &sysctl_drop_caches, + .maxlen = sizeof(int), + .mode = 0200, + .proc_handler = drop_caches_sysctl_handler, + .extra1 = SYSCTL_ONE, + .extra2 = SYSCTL_FOUR, + }, +}; + +static int __init init_vm_drop_caches_sysctls(void) +{ + register_sysctl_init("vm", drop_caches_table); + return 0; +} +fs_initcall(init_vm_drop_caches_sysctls); diff --git a/fs/ecryptfs/Kconfig b/fs/ecryptfs/Kconfig new file mode 100644 index 000000000..c2f4fb41b --- /dev/null +++ b/fs/ecryptfs/Kconfig @@ -0,0 +1,23 @@ +# SPDX-License-Identifier: GPL-2.0-only +config ECRYPT_FS + tristate "eCrypt filesystem layer support" + depends on KEYS && CRYPTO && (ENCRYPTED_KEYS || ENCRYPTED_KEYS=n) + select CRYPTO_ECB + select CRYPTO_CBC + select CRYPTO_LIB_MD5 + help + Encrypted filesystem that operates on the VFS layer. See + to learn more about + eCryptfs. Userspace components are required and can be + obtained from . + + To compile this file system support as a module, choose M here: the + module will be called ecryptfs. + +config ECRYPT_FS_MESSAGING + bool "Enable notifications for userspace key wrap/unwrap" + depends on ECRYPT_FS + help + Enables the /dev/ecryptfs entry for use by ecryptfsd. This allows + for userspace to wrap/unwrap file encryption keys by other + backends, like OpenSSL. diff --git a/fs/ecryptfs/Makefile b/fs/ecryptfs/Makefile new file mode 100644 index 000000000..4f2cc5b25 --- /dev/null +++ b/fs/ecryptfs/Makefile @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the Linux eCryptfs +# + +obj-$(CONFIG_ECRYPT_FS) += ecryptfs.o + +ecryptfs-y := dentry.o file.o inode.o main.o super.o mmap.o read_write.o \ + crypto.o keystore.o kthread.o debug.o + +ecryptfs-$(CONFIG_ECRYPT_FS_MESSAGING) += messaging.o miscdev.o diff --git a/fs/ecryptfs/crypto.c b/fs/ecryptfs/crypto.c new file mode 100644 index 000000000..e67119b60 --- /dev/null +++ b/fs/ecryptfs/crypto.c @@ -0,0 +1,1975 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * eCryptfs: Linux filesystem encryption layer + * + * Copyright (C) 1997-2004 Erez Zadok + * Copyright (C) 2001-2004 Stony Brook University + * Copyright (C) 2004-2007 International Business Machines Corp. + * Author(s): Michael A. Halcrow + * Michael C. Thompson + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ecryptfs_kernel.h" + +#define DECRYPT 0 +#define ENCRYPT 1 + +/** + * ecryptfs_from_hex + * @dst: Buffer to take the bytes from src hex; must be at least of + * size (src_size / 2) + * @src: Buffer to be converted from a hex string representation to raw value + * @dst_size: size of dst buffer, or number of hex characters pairs to convert + */ +void ecryptfs_from_hex(char *dst, char *src, int dst_size) +{ + int x; + char tmp[3] = { 0, }; + + for (x = 0; x < dst_size; x++) { + tmp[0] = src[x * 2]; + tmp[1] = src[x * 2 + 1]; + dst[x] = (unsigned char)simple_strtol(tmp, NULL, 16); + } +} + +static int ecryptfs_crypto_api_algify_cipher_name(char **algified_name, + const char *cipher_name, + const char *chaining_modifier) +{ + (*algified_name) = kasprintf(GFP_KERNEL, "%s(%s)", chaining_modifier, + cipher_name); + if (!(*algified_name)) + return -ENOMEM; + + return 0; +} + +/** + * ecryptfs_derive_iv + * @iv: destination for the derived iv value + * @crypt_stat: Pointer to crypt_stat struct for the current inode + * @offset: Offset of the extent whose IV we are to derive + * + * Generate the initialization vector from the given root IV and page + * offset. + */ +void ecryptfs_derive_iv(char *iv, struct ecryptfs_crypt_stat *crypt_stat, + loff_t offset) +{ + char dst[MD5_DIGEST_SIZE]; + char src[ECRYPTFS_MAX_IV_BYTES + 16]; + + if (unlikely(ecryptfs_verbosity > 0)) { + ecryptfs_printk(KERN_DEBUG, "root iv:\n"); + ecryptfs_dump_hex(crypt_stat->root_iv, crypt_stat->iv_bytes); + } + memcpy(src, crypt_stat->root_iv, crypt_stat->iv_bytes); + memset((src + crypt_stat->iv_bytes), 0, 16); + snprintf((src + crypt_stat->iv_bytes), 16, "%lld", offset); + if (unlikely(ecryptfs_verbosity > 0)) { + ecryptfs_printk(KERN_DEBUG, "source:\n"); + ecryptfs_dump_hex(src, (crypt_stat->iv_bytes + 16)); + } + md5(src, crypt_stat->iv_bytes + 16, dst); + memcpy(iv, dst, crypt_stat->iv_bytes); + if (unlikely(ecryptfs_verbosity > 0)) { + ecryptfs_printk(KERN_DEBUG, "derived iv:\n"); + ecryptfs_dump_hex(iv, crypt_stat->iv_bytes); + } +} + +/** + * ecryptfs_init_crypt_stat + * @crypt_stat: Pointer to the crypt_stat struct to initialize. + * + * Initialize the crypt_stat structure. + */ +void ecryptfs_init_crypt_stat(struct ecryptfs_crypt_stat *crypt_stat) +{ + memset((void *)crypt_stat, 0, sizeof(struct ecryptfs_crypt_stat)); + INIT_LIST_HEAD(&crypt_stat->keysig_list); + mutex_init(&crypt_stat->keysig_list_mutex); + mutex_init(&crypt_stat->cs_mutex); + mutex_init(&crypt_stat->cs_tfm_mutex); + crypt_stat->flags |= ECRYPTFS_STRUCT_INITIALIZED; +} + +/** + * ecryptfs_destroy_crypt_stat + * @crypt_stat: Pointer to the crypt_stat struct to initialize. + * + * Releases all memory associated with a crypt_stat struct. + */ +void ecryptfs_destroy_crypt_stat(struct ecryptfs_crypt_stat *crypt_stat) +{ + struct ecryptfs_key_sig *key_sig, *key_sig_tmp; + + crypto_free_skcipher(crypt_stat->tfm); + list_for_each_entry_safe(key_sig, key_sig_tmp, + &crypt_stat->keysig_list, crypt_stat_list) { + list_del(&key_sig->crypt_stat_list); + kmem_cache_free(ecryptfs_key_sig_cache, key_sig); + } + memset(crypt_stat, 0, sizeof(struct ecryptfs_crypt_stat)); +} + +void ecryptfs_destroy_mount_crypt_stat( + struct ecryptfs_mount_crypt_stat *mount_crypt_stat) +{ + struct ecryptfs_global_auth_tok *auth_tok, *auth_tok_tmp; + + if (!(mount_crypt_stat->flags & ECRYPTFS_MOUNT_CRYPT_STAT_INITIALIZED)) + return; + mutex_lock(&mount_crypt_stat->global_auth_tok_list_mutex); + list_for_each_entry_safe(auth_tok, auth_tok_tmp, + &mount_crypt_stat->global_auth_tok_list, + mount_crypt_stat_list) { + list_del(&auth_tok->mount_crypt_stat_list); + if (!(auth_tok->flags & ECRYPTFS_AUTH_TOK_INVALID)) + key_put(auth_tok->global_auth_tok_key); + kmem_cache_free(ecryptfs_global_auth_tok_cache, auth_tok); + } + mutex_unlock(&mount_crypt_stat->global_auth_tok_list_mutex); + memset(mount_crypt_stat, 0, sizeof(struct ecryptfs_mount_crypt_stat)); +} + +/** + * virt_to_scatterlist + * @addr: Virtual address + * @size: Size of data; should be an even multiple of the block size + * @sg: Pointer to scatterlist array; set to NULL to obtain only + * the number of scatterlist structs required in array + * @sg_size: Max array size + * + * Fills in a scatterlist array with page references for a passed + * virtual address. + * + * Returns the number of scatterlist structs in array used + */ +int virt_to_scatterlist(const void *addr, int size, struct scatterlist *sg, + int sg_size) +{ + int i = 0; + struct page *pg; + int offset; + int remainder_of_page; + + sg_init_table(sg, sg_size); + + while (size > 0 && i < sg_size) { + pg = virt_to_page(addr); + offset = offset_in_page(addr); + sg_set_page(&sg[i], pg, 0, offset); + remainder_of_page = PAGE_SIZE - offset; + if (size >= remainder_of_page) { + sg[i].length = remainder_of_page; + addr += remainder_of_page; + size -= remainder_of_page; + } else { + sg[i].length = size; + addr += size; + size = 0; + } + i++; + } + if (size > 0) + return -ENOMEM; + return i; +} + +/** + * crypt_scatterlist + * @crypt_stat: Pointer to the crypt_stat struct to initialize. + * @dst_sg: Destination of the data after performing the crypto operation + * @src_sg: Data to be encrypted or decrypted + * @size: Length of data + * @iv: IV to use + * @op: ENCRYPT or DECRYPT to indicate the desired operation + * + * Returns the number of bytes encrypted or decrypted; negative value on error + */ +static int crypt_scatterlist(struct ecryptfs_crypt_stat *crypt_stat, + struct scatterlist *dst_sg, + struct scatterlist *src_sg, int size, + unsigned char *iv, int op) +{ + struct skcipher_request *req = NULL; + DECLARE_CRYPTO_WAIT(ecr); + int rc = 0; + + if (unlikely(ecryptfs_verbosity > 0)) { + ecryptfs_printk(KERN_DEBUG, "Key size [%zd]; key:\n", + crypt_stat->key_size); + ecryptfs_dump_hex(crypt_stat->key, + crypt_stat->key_size); + } + + mutex_lock(&crypt_stat->cs_tfm_mutex); + req = skcipher_request_alloc(crypt_stat->tfm, GFP_NOFS); + if (!req) { + mutex_unlock(&crypt_stat->cs_tfm_mutex); + rc = -ENOMEM; + goto out; + } + + skcipher_request_set_callback(req, + CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, + crypto_req_done, &ecr); + /* Consider doing this once, when the file is opened */ + if (!(crypt_stat->flags & ECRYPTFS_KEY_SET)) { + rc = crypto_skcipher_setkey(crypt_stat->tfm, crypt_stat->key, + crypt_stat->key_size); + if (rc) { + ecryptfs_printk(KERN_ERR, + "Error setting key; rc = [%d]\n", + rc); + mutex_unlock(&crypt_stat->cs_tfm_mutex); + rc = -EINVAL; + goto out; + } + crypt_stat->flags |= ECRYPTFS_KEY_SET; + } + mutex_unlock(&crypt_stat->cs_tfm_mutex); + skcipher_request_set_crypt(req, src_sg, dst_sg, size, iv); + rc = op == ENCRYPT ? crypto_skcipher_encrypt(req) : + crypto_skcipher_decrypt(req); + rc = crypto_wait_req(rc, &ecr); +out: + skcipher_request_free(req); + return rc; +} + +/* + * lower_offset_for_page + * + * Convert an eCryptfs page index into a lower byte offset + */ +static loff_t lower_offset_for_page(struct ecryptfs_crypt_stat *crypt_stat, + struct folio *folio) +{ + return ecryptfs_lower_header_size(crypt_stat) + + (loff_t)folio->index * PAGE_SIZE; +} + +/** + * crypt_extent + * @crypt_stat: crypt_stat containing cryptographic context for the + * encryption operation + * @dst_page: The page to write the result into + * @src_page: The page to read from + * @page_index: The offset in the file (in units of PAGE_SIZE) + * @extent_offset: Page extent offset for use in generating IV + * @op: ENCRYPT or DECRYPT to indicate the desired operation + * + * Encrypts or decrypts one extent of data. + * + * Return zero on success; non-zero otherwise + */ +static int crypt_extent(struct ecryptfs_crypt_stat *crypt_stat, + struct page *dst_page, + struct page *src_page, + pgoff_t page_index, + unsigned long extent_offset, int op) +{ + loff_t extent_base; + char extent_iv[ECRYPTFS_MAX_IV_BYTES]; + struct scatterlist src_sg, dst_sg; + size_t extent_size = crypt_stat->extent_size; + int rc; + + extent_base = (((loff_t)page_index) * (PAGE_SIZE / extent_size)); + ecryptfs_derive_iv(extent_iv, crypt_stat, extent_base + extent_offset); + + sg_init_table(&src_sg, 1); + sg_init_table(&dst_sg, 1); + + sg_set_page(&src_sg, src_page, extent_size, + extent_offset * extent_size); + sg_set_page(&dst_sg, dst_page, extent_size, + extent_offset * extent_size); + + rc = crypt_scatterlist(crypt_stat, &dst_sg, &src_sg, extent_size, + extent_iv, op); + if (rc < 0) { + printk(KERN_ERR "%s: Error attempting to crypt page with " + "page_index = [%ld], extent_offset = [%ld]; " + "rc = [%d]\n", __func__, page_index, extent_offset, rc); + goto out; + } + rc = 0; +out: + return rc; +} + +/** + * ecryptfs_encrypt_page + * @folio: Folio mapped from the eCryptfs inode for the file; contains + * decrypted content that needs to be encrypted (to a temporary + * page; not in place) and written out to the lower file + * + * Encrypt an eCryptfs page. This is done on a per-extent basis. Note + * that eCryptfs pages may straddle the lower pages -- for instance, + * if the file was created on a machine with an 8K page size + * (resulting in an 8K header), and then the file is copied onto a + * host with a 32K page size, then when reading page 0 of the eCryptfs + * file, 24K of page 0 of the lower file will be read and decrypted, + * and then 8K of page 1 of the lower file will be read and decrypted. + * + * Returns zero on success; negative on error + */ +int ecryptfs_encrypt_page(struct folio *folio) +{ + struct inode *ecryptfs_inode; + struct ecryptfs_crypt_stat *crypt_stat; + char *enc_extent_virt; + struct page *enc_extent_page = NULL; + loff_t extent_offset; + loff_t lower_offset; + int rc = 0; + + ecryptfs_inode = folio->mapping->host; + crypt_stat = + &(ecryptfs_inode_to_private(ecryptfs_inode)->crypt_stat); + BUG_ON(!(crypt_stat->flags & ECRYPTFS_ENCRYPTED)); + enc_extent_page = alloc_page(GFP_USER); + if (!enc_extent_page) { + rc = -ENOMEM; + ecryptfs_printk(KERN_ERR, "Error allocating memory for " + "encrypted extent\n"); + goto out; + } + + for (extent_offset = 0; + extent_offset < (PAGE_SIZE / crypt_stat->extent_size); + extent_offset++) { + rc = crypt_extent(crypt_stat, enc_extent_page, + folio_page(folio, 0), folio->index, + extent_offset, ENCRYPT); + if (rc) { + printk(KERN_ERR "%s: Error encrypting extent; " + "rc = [%d]\n", __func__, rc); + goto out; + } + } + + lower_offset = lower_offset_for_page(crypt_stat, folio); + enc_extent_virt = kmap_local_page(enc_extent_page); + rc = ecryptfs_write_lower(ecryptfs_inode, enc_extent_virt, lower_offset, + PAGE_SIZE); + kunmap_local(enc_extent_virt); + if (rc < 0) { + ecryptfs_printk(KERN_ERR, + "Error attempting to write lower page; rc = [%d]\n", + rc); + goto out; + } + rc = 0; +out: + if (enc_extent_page) { + __free_page(enc_extent_page); + } + return rc; +} + +/** + * ecryptfs_decrypt_page + * @folio: Folio mapped from the eCryptfs inode for the file; data read + * and decrypted from the lower file will be written into this + * page + * + * Decrypt an eCryptfs page. This is done on a per-extent basis. Note + * that eCryptfs pages may straddle the lower pages -- for instance, + * if the file was created on a machine with an 8K page size + * (resulting in an 8K header), and then the file is copied onto a + * host with a 32K page size, then when reading page 0 of the eCryptfs + * file, 24K of page 0 of the lower file will be read and decrypted, + * and then 8K of page 1 of the lower file will be read and decrypted. + * + * Returns zero on success; negative on error + */ +int ecryptfs_decrypt_page(struct folio *folio) +{ + struct inode *ecryptfs_inode; + struct ecryptfs_crypt_stat *crypt_stat; + char *page_virt; + unsigned long extent_offset; + loff_t lower_offset; + int rc = 0; + + ecryptfs_inode = folio->mapping->host; + crypt_stat = + &(ecryptfs_inode_to_private(ecryptfs_inode)->crypt_stat); + BUG_ON(!(crypt_stat->flags & ECRYPTFS_ENCRYPTED)); + + lower_offset = lower_offset_for_page(crypt_stat, folio); + page_virt = kmap_local_folio(folio, 0); + rc = ecryptfs_read_lower(page_virt, lower_offset, PAGE_SIZE, + ecryptfs_inode); + kunmap_local(page_virt); + if (rc < 0) { + ecryptfs_printk(KERN_ERR, + "Error attempting to read lower page; rc = [%d]\n", + rc); + goto out; + } + + for (extent_offset = 0; + extent_offset < (PAGE_SIZE / crypt_stat->extent_size); + extent_offset++) { + struct page *page = folio_page(folio, 0); + rc = crypt_extent(crypt_stat, page, page, folio->index, + extent_offset, DECRYPT); + if (rc) { + printk(KERN_ERR "%s: Error decrypting extent; " + "rc = [%d]\n", __func__, rc); + goto out; + } + } +out: + return rc; +} + +#define ECRYPTFS_MAX_SCATTERLIST_LEN 4 + +/** + * ecryptfs_init_crypt_ctx + * @crypt_stat: Uninitialized crypt stats structure + * + * Initialize the crypto context. + * + * TODO: Performance: Keep a cache of initialized cipher contexts; + * only init if needed + */ +int ecryptfs_init_crypt_ctx(struct ecryptfs_crypt_stat *crypt_stat) +{ + char *full_alg_name; + int rc = -EINVAL; + + ecryptfs_printk(KERN_DEBUG, + "Initializing cipher [%s]; strlen = [%d]; " + "key_size_bits = [%zd]\n", + crypt_stat->cipher, (int)strlen(crypt_stat->cipher), + crypt_stat->key_size << 3); + mutex_lock(&crypt_stat->cs_tfm_mutex); + if (crypt_stat->tfm) { + rc = 0; + goto out_unlock; + } + rc = ecryptfs_crypto_api_algify_cipher_name(&full_alg_name, + crypt_stat->cipher, "cbc"); + if (rc) + goto out_unlock; + crypt_stat->tfm = crypto_alloc_skcipher(full_alg_name, 0, 0); + if (IS_ERR(crypt_stat->tfm)) { + rc = PTR_ERR(crypt_stat->tfm); + crypt_stat->tfm = NULL; + ecryptfs_printk(KERN_ERR, "cryptfs: init_crypt_ctx(): " + "Error initializing cipher [%s]\n", + full_alg_name); + goto out_free; + } + crypto_skcipher_set_flags(crypt_stat->tfm, + CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); + rc = 0; +out_free: + kfree(full_alg_name); +out_unlock: + mutex_unlock(&crypt_stat->cs_tfm_mutex); + return rc; +} + +static void set_extent_mask_and_shift(struct ecryptfs_crypt_stat *crypt_stat) +{ + int extent_size_tmp; + + crypt_stat->extent_mask = 0xFFFFFFFF; + crypt_stat->extent_shift = 0; + if (crypt_stat->extent_size == 0) + return; + extent_size_tmp = crypt_stat->extent_size; + while ((extent_size_tmp & 0x01) == 0) { + extent_size_tmp >>= 1; + crypt_stat->extent_mask <<= 1; + crypt_stat->extent_shift++; + } +} + +void ecryptfs_set_default_sizes(struct ecryptfs_crypt_stat *crypt_stat) +{ + /* Default values; may be overwritten as we are parsing the + * packets. */ + crypt_stat->extent_size = ECRYPTFS_DEFAULT_EXTENT_SIZE; + set_extent_mask_and_shift(crypt_stat); + crypt_stat->iv_bytes = ECRYPTFS_DEFAULT_IV_BYTES; + if (crypt_stat->flags & ECRYPTFS_METADATA_IN_XATTR) + crypt_stat->metadata_size = ECRYPTFS_MINIMUM_HEADER_EXTENT_SIZE; + else { + if (PAGE_SIZE <= ECRYPTFS_MINIMUM_HEADER_EXTENT_SIZE) + crypt_stat->metadata_size = + ECRYPTFS_MINIMUM_HEADER_EXTENT_SIZE; + else + crypt_stat->metadata_size = PAGE_SIZE; + } +} + +/* + * ecryptfs_compute_root_iv + * + * On error, sets the root IV to all 0's. + */ +int ecryptfs_compute_root_iv(struct ecryptfs_crypt_stat *crypt_stat) +{ + char dst[MD5_DIGEST_SIZE]; + + BUG_ON(crypt_stat->iv_bytes > MD5_DIGEST_SIZE); + BUG_ON(crypt_stat->iv_bytes <= 0); + if (!(crypt_stat->flags & ECRYPTFS_KEY_VALID)) { + ecryptfs_printk(KERN_WARNING, "Session key not valid; " + "cannot generate root IV\n"); + memset(crypt_stat->root_iv, 0, crypt_stat->iv_bytes); + crypt_stat->flags |= ECRYPTFS_SECURITY_WARNING; + return -EINVAL; + } + md5(crypt_stat->key, crypt_stat->key_size, dst); + memcpy(crypt_stat->root_iv, dst, crypt_stat->iv_bytes); + return 0; +} + +static void ecryptfs_generate_new_key(struct ecryptfs_crypt_stat *crypt_stat) +{ + get_random_bytes(crypt_stat->key, crypt_stat->key_size); + crypt_stat->flags |= ECRYPTFS_KEY_VALID; + ecryptfs_compute_root_iv(crypt_stat); + if (unlikely(ecryptfs_verbosity > 0)) { + ecryptfs_printk(KERN_DEBUG, "Generated new session key:\n"); + ecryptfs_dump_hex(crypt_stat->key, + crypt_stat->key_size); + } +} + +/** + * ecryptfs_copy_mount_wide_flags_to_inode_flags + * @crypt_stat: The inode's cryptographic context + * @mount_crypt_stat: The mount point's cryptographic context + * + * This function propagates the mount-wide flags to individual inode + * flags. + */ +static void ecryptfs_copy_mount_wide_flags_to_inode_flags( + struct ecryptfs_crypt_stat *crypt_stat, + struct ecryptfs_mount_crypt_stat *mount_crypt_stat) +{ + if (mount_crypt_stat->flags & ECRYPTFS_XATTR_METADATA_ENABLED) + crypt_stat->flags |= ECRYPTFS_METADATA_IN_XATTR; + if (mount_crypt_stat->flags & ECRYPTFS_ENCRYPTED_VIEW_ENABLED) + crypt_stat->flags |= ECRYPTFS_VIEW_AS_ENCRYPTED; + if (mount_crypt_stat->flags & ECRYPTFS_GLOBAL_ENCRYPT_FILENAMES) { + crypt_stat->flags |= ECRYPTFS_ENCRYPT_FILENAMES; + if (mount_crypt_stat->flags + & ECRYPTFS_GLOBAL_ENCFN_USE_MOUNT_FNEK) + crypt_stat->flags |= ECRYPTFS_ENCFN_USE_MOUNT_FNEK; + else if (mount_crypt_stat->flags + & ECRYPTFS_GLOBAL_ENCFN_USE_FEK) + crypt_stat->flags |= ECRYPTFS_ENCFN_USE_FEK; + } +} + +static int ecryptfs_copy_mount_wide_sigs_to_inode_sigs( + struct ecryptfs_crypt_stat *crypt_stat, + struct ecryptfs_mount_crypt_stat *mount_crypt_stat) +{ + struct ecryptfs_global_auth_tok *global_auth_tok; + int rc = 0; + + mutex_lock(&crypt_stat->keysig_list_mutex); + mutex_lock(&mount_crypt_stat->global_auth_tok_list_mutex); + + list_for_each_entry(global_auth_tok, + &mount_crypt_stat->global_auth_tok_list, + mount_crypt_stat_list) { + if (global_auth_tok->flags & ECRYPTFS_AUTH_TOK_FNEK) + continue; + rc = ecryptfs_add_keysig(crypt_stat, global_auth_tok->sig); + if (rc) { + printk(KERN_ERR "Error adding keysig; rc = [%d]\n", rc); + goto out; + } + } + +out: + mutex_unlock(&mount_crypt_stat->global_auth_tok_list_mutex); + mutex_unlock(&crypt_stat->keysig_list_mutex); + return rc; +} + +/** + * ecryptfs_set_default_crypt_stat_vals + * @crypt_stat: The inode's cryptographic context + * @mount_crypt_stat: The mount point's cryptographic context + * + * Default values in the event that policy does not override them. + */ +static void ecryptfs_set_default_crypt_stat_vals( + struct ecryptfs_crypt_stat *crypt_stat, + struct ecryptfs_mount_crypt_stat *mount_crypt_stat) +{ + ecryptfs_copy_mount_wide_flags_to_inode_flags(crypt_stat, + mount_crypt_stat); + ecryptfs_set_default_sizes(crypt_stat); + strscpy(crypt_stat->cipher, ECRYPTFS_DEFAULT_CIPHER); + crypt_stat->key_size = ECRYPTFS_DEFAULT_KEY_BYTES; + crypt_stat->flags &= ~(ECRYPTFS_KEY_VALID); + crypt_stat->file_version = ECRYPTFS_FILE_VERSION; + crypt_stat->mount_crypt_stat = mount_crypt_stat; +} + +/** + * ecryptfs_new_file_context + * @ecryptfs_inode: The eCryptfs inode + * + * If the crypto context for the file has not yet been established, + * this is where we do that. Establishing a new crypto context + * involves the following decisions: + * - What cipher to use? + * - What set of authentication tokens to use? + * Here we just worry about getting enough information into the + * authentication tokens so that we know that they are available. + * We associate the available authentication tokens with the new file + * via the set of signatures in the crypt_stat struct. Later, when + * the headers are actually written out, we may again defer to + * userspace to perform the encryption of the session key; for the + * foreseeable future, this will be the case with public key packets. + * + * Returns zero on success; non-zero otherwise + */ +int ecryptfs_new_file_context(struct inode *ecryptfs_inode) +{ + struct ecryptfs_crypt_stat *crypt_stat = + &ecryptfs_inode_to_private(ecryptfs_inode)->crypt_stat; + struct ecryptfs_mount_crypt_stat *mount_crypt_stat = + &ecryptfs_superblock_to_private( + ecryptfs_inode->i_sb)->mount_crypt_stat; + int rc = 0; + + ecryptfs_set_default_crypt_stat_vals(crypt_stat, mount_crypt_stat); + crypt_stat->flags |= (ECRYPTFS_ENCRYPTED | ECRYPTFS_KEY_VALID); + ecryptfs_copy_mount_wide_flags_to_inode_flags(crypt_stat, + mount_crypt_stat); + rc = ecryptfs_copy_mount_wide_sigs_to_inode_sigs(crypt_stat, + mount_crypt_stat); + if (rc) { + printk(KERN_ERR "Error attempting to copy mount-wide key sigs " + "to the inode key sigs; rc = [%d]\n", rc); + goto out; + } + strscpy(crypt_stat->cipher, + mount_crypt_stat->global_default_cipher_name); + crypt_stat->key_size = + mount_crypt_stat->global_default_cipher_key_size; + ecryptfs_generate_new_key(crypt_stat); + rc = ecryptfs_init_crypt_ctx(crypt_stat); + if (rc) + ecryptfs_printk(KERN_ERR, "Error initializing cryptographic " + "context for cipher [%s]: rc = [%d]\n", + crypt_stat->cipher, rc); +out: + return rc; +} + +/** + * ecryptfs_validate_marker - check for the ecryptfs marker + * @data: The data block in which to check + * + * Returns zero if marker found; -EINVAL if not found + */ +static int ecryptfs_validate_marker(char *data) +{ + u32 m_1, m_2; + + m_1 = get_unaligned_be32(data); + m_2 = get_unaligned_be32(data + 4); + if ((m_1 ^ MAGIC_ECRYPTFS_MARKER) == m_2) + return 0; + ecryptfs_printk(KERN_DEBUG, "m_1 = [0x%.8x]; m_2 = [0x%.8x]; " + "MAGIC_ECRYPTFS_MARKER = [0x%.8x]\n", m_1, m_2, + MAGIC_ECRYPTFS_MARKER); + ecryptfs_printk(KERN_DEBUG, "(m_1 ^ MAGIC_ECRYPTFS_MARKER) = " + "[0x%.8x]\n", (m_1 ^ MAGIC_ECRYPTFS_MARKER)); + return -EINVAL; +} + +struct ecryptfs_flag_map_elem { + u32 file_flag; + u32 local_flag; +}; + +/* Add support for additional flags by adding elements here. */ +static struct ecryptfs_flag_map_elem ecryptfs_flag_map[] = { + {0x00000001, ECRYPTFS_ENABLE_HMAC}, + {0x00000002, ECRYPTFS_ENCRYPTED}, + {0x00000004, ECRYPTFS_METADATA_IN_XATTR}, + {0x00000008, ECRYPTFS_ENCRYPT_FILENAMES} +}; + +/** + * ecryptfs_process_flags + * @crypt_stat: The cryptographic context + * @page_virt: Source data to be parsed + * @bytes_read: Updated with the number of bytes read + */ +static void ecryptfs_process_flags(struct ecryptfs_crypt_stat *crypt_stat, + char *page_virt, int *bytes_read) +{ + int i; + u32 flags; + + flags = get_unaligned_be32(page_virt); + for (i = 0; i < ARRAY_SIZE(ecryptfs_flag_map); i++) + if (flags & ecryptfs_flag_map[i].file_flag) { + crypt_stat->flags |= ecryptfs_flag_map[i].local_flag; + } else + crypt_stat->flags &= ~(ecryptfs_flag_map[i].local_flag); + /* Version is in top 8 bits of the 32-bit flag vector */ + crypt_stat->file_version = ((flags >> 24) & 0xFF); + (*bytes_read) = 4; +} + +/** + * write_ecryptfs_marker + * @page_virt: The pointer to in a page to begin writing the marker + * @written: Number of bytes written + * + * Marker = 0x3c81b7f5 + */ +static void write_ecryptfs_marker(char *page_virt, size_t *written) +{ + u32 m_1, m_2; + + get_random_bytes(&m_1, (MAGIC_ECRYPTFS_MARKER_SIZE_BYTES / 2)); + m_2 = (m_1 ^ MAGIC_ECRYPTFS_MARKER); + put_unaligned_be32(m_1, page_virt); + page_virt += (MAGIC_ECRYPTFS_MARKER_SIZE_BYTES / 2); + put_unaligned_be32(m_2, page_virt); + (*written) = MAGIC_ECRYPTFS_MARKER_SIZE_BYTES; +} + +void ecryptfs_write_crypt_stat_flags(char *page_virt, + struct ecryptfs_crypt_stat *crypt_stat, + size_t *written) +{ + u32 flags = 0; + int i; + + for (i = 0; i < ARRAY_SIZE(ecryptfs_flag_map); i++) + if (crypt_stat->flags & ecryptfs_flag_map[i].local_flag) + flags |= ecryptfs_flag_map[i].file_flag; + /* Version is in top 8 bits of the 32-bit flag vector */ + flags |= ((((u8)crypt_stat->file_version) << 24) & 0xFF000000); + put_unaligned_be32(flags, page_virt); + (*written) = 4; +} + +struct ecryptfs_cipher_code_str_map_elem { + char cipher_str[16]; + u8 cipher_code; +}; + +/* Add support for additional ciphers by adding elements here. The + * cipher_code is whatever OpenPGP applications use to identify the + * ciphers. List in order of probability. */ +static struct ecryptfs_cipher_code_str_map_elem +ecryptfs_cipher_code_str_map[] = { + {"aes",RFC2440_CIPHER_AES_128 }, + {"blowfish", RFC2440_CIPHER_BLOWFISH}, + {"des3_ede", RFC2440_CIPHER_DES3_EDE}, + {"cast5", RFC2440_CIPHER_CAST_5}, + {"twofish", RFC2440_CIPHER_TWOFISH}, + {"cast6", RFC2440_CIPHER_CAST_6}, + {"aes", RFC2440_CIPHER_AES_192}, + {"aes", RFC2440_CIPHER_AES_256} +}; + +/** + * ecryptfs_code_for_cipher_string + * @cipher_name: The string alias for the cipher + * @key_bytes: Length of key in bytes; used for AES code selection + * + * Returns zero on no match, or the cipher code on match + */ +u8 ecryptfs_code_for_cipher_string(char *cipher_name, size_t key_bytes) +{ + int i; + u8 code = 0; + struct ecryptfs_cipher_code_str_map_elem *map = + ecryptfs_cipher_code_str_map; + + if (strcmp(cipher_name, "aes") == 0) { + switch (key_bytes) { + case 16: + code = RFC2440_CIPHER_AES_128; + break; + case 24: + code = RFC2440_CIPHER_AES_192; + break; + case 32: + code = RFC2440_CIPHER_AES_256; + } + } else { + for (i = 0; i < ARRAY_SIZE(ecryptfs_cipher_code_str_map); i++) + if (strcmp(cipher_name, map[i].cipher_str) == 0) { + code = map[i].cipher_code; + break; + } + } + return code; +} + +/** + * ecryptfs_cipher_code_to_string + * @str: Destination to write out the cipher name + * @size: Destination buffer size + * @cipher_code: The code to convert to cipher name string + * + * Returns zero on success + */ +int ecryptfs_cipher_code_to_string(char *str, size_t size, u8 cipher_code) +{ + int rc = 0; + int i; + + str[0] = '\0'; + for (i = 0; i < ARRAY_SIZE(ecryptfs_cipher_code_str_map); i++) + if (cipher_code == ecryptfs_cipher_code_str_map[i].cipher_code) + strscpy(str, ecryptfs_cipher_code_str_map[i].cipher_str, + size); + if (str[0] == '\0') { + ecryptfs_printk(KERN_WARNING, "Cipher code not recognized: " + "[%d]\n", cipher_code); + rc = -EINVAL; + } + return rc; +} + +int ecryptfs_read_and_validate_header_region(struct inode *inode) +{ + u8 file_size[ECRYPTFS_SIZE_AND_MARKER_BYTES]; + u8 *marker = file_size + ECRYPTFS_FILE_SIZE_BYTES; + int rc; + + rc = ecryptfs_read_lower(file_size, 0, ECRYPTFS_SIZE_AND_MARKER_BYTES, + inode); + if (rc < 0) + return rc; + else if (rc < ECRYPTFS_SIZE_AND_MARKER_BYTES) + return -EINVAL; + rc = ecryptfs_validate_marker(marker); + if (!rc) + ecryptfs_i_size_init(file_size, inode); + return rc; +} + +void +ecryptfs_write_header_metadata(char *virt, + struct ecryptfs_crypt_stat *crypt_stat, + size_t *written) +{ + u32 header_extent_size; + u16 num_header_extents_at_front; + + header_extent_size = (u32)crypt_stat->extent_size; + num_header_extents_at_front = + (u16)(crypt_stat->metadata_size / crypt_stat->extent_size); + put_unaligned_be32(header_extent_size, virt); + virt += 4; + put_unaligned_be16(num_header_extents_at_front, virt); + (*written) = 6; +} + +struct kmem_cache *ecryptfs_header_cache; + +/** + * ecryptfs_write_headers_virt + * @page_virt: The virtual address to write the headers to + * @max: The size of memory allocated at page_virt + * @size: Set to the number of bytes written by this function + * @crypt_stat: The cryptographic context + * @ecryptfs_dentry: The eCryptfs dentry + * + * Format version: 1 + * + * Header Extent: + * Octets 0-7: Unencrypted file size (big-endian) + * Octets 8-15: eCryptfs special marker + * Octets 16-19: Flags + * Octet 16: File format version number (between 0 and 255) + * Octets 17-18: Reserved + * Octet 19: Bit 1 (lsb): Reserved + * Bit 2: Encrypted? + * Bits 3-8: Reserved + * Octets 20-23: Header extent size (big-endian) + * Octets 24-25: Number of header extents at front of file + * (big-endian) + * Octet 26: Begin RFC 2440 authentication token packet set + * Data Extent 0: + * Lower data (CBC encrypted) + * Data Extent 1: + * Lower data (CBC encrypted) + * ... + * + * Returns zero on success + */ +static int ecryptfs_write_headers_virt(char *page_virt, size_t max, + size_t *size, + struct ecryptfs_crypt_stat *crypt_stat, + struct dentry *ecryptfs_dentry) +{ + int rc; + size_t written; + size_t offset; + + offset = ECRYPTFS_FILE_SIZE_BYTES; + write_ecryptfs_marker((page_virt + offset), &written); + offset += written; + ecryptfs_write_crypt_stat_flags((page_virt + offset), crypt_stat, + &written); + offset += written; + ecryptfs_write_header_metadata((page_virt + offset), crypt_stat, + &written); + offset += written; + rc = ecryptfs_generate_key_packet_set((page_virt + offset), crypt_stat, + ecryptfs_dentry, &written, + max - offset); + if (rc) + ecryptfs_printk(KERN_WARNING, "Error generating key packet " + "set; rc = [%d]\n", rc); + if (size) { + offset += written; + *size = offset; + } + return rc; +} + +static int +ecryptfs_write_metadata_to_contents(struct inode *ecryptfs_inode, + char *virt, size_t virt_len) +{ + int rc; + + rc = ecryptfs_write_lower(ecryptfs_inode, virt, + 0, virt_len); + if (rc < 0) + printk(KERN_ERR "%s: Error attempting to write header " + "information to lower file; rc = [%d]\n", __func__, rc); + else + rc = 0; + return rc; +} + +static int +ecryptfs_write_metadata_to_xattr(struct dentry *ecryptfs_dentry, + struct inode *ecryptfs_inode, + char *page_virt, size_t size) +{ + int rc; + struct dentry *lower_dentry = ecryptfs_dentry_to_lower(ecryptfs_dentry); + struct inode *lower_inode = d_inode(lower_dentry); + + if (!(lower_inode->i_opflags & IOP_XATTR)) { + rc = -EOPNOTSUPP; + goto out; + } + + inode_lock(lower_inode); + rc = __vfs_setxattr(&nop_mnt_idmap, lower_dentry, lower_inode, + ECRYPTFS_XATTR_NAME, page_virt, size, 0); + if (!rc && ecryptfs_inode) + fsstack_copy_attr_all(ecryptfs_inode, lower_inode); + inode_unlock(lower_inode); +out: + return rc; +} + +static unsigned long ecryptfs_get_zeroed_pages(gfp_t gfp_mask, + unsigned int order) +{ + struct page *page; + + page = alloc_pages(gfp_mask | __GFP_ZERO, order); + if (page) + return (unsigned long) page_address(page); + return 0; +} + +/** + * ecryptfs_write_metadata + * @ecryptfs_dentry: The eCryptfs dentry, which should be negative + * @ecryptfs_inode: The newly created eCryptfs inode + * + * Write the file headers out. This will likely involve a userspace + * callout, in which the session key is encrypted with one or more + * public keys and/or the passphrase necessary to do the encryption is + * retrieved via a prompt. Exactly what happens at this point should + * be policy-dependent. + * + * Returns zero on success; non-zero on error + */ +int ecryptfs_write_metadata(struct dentry *ecryptfs_dentry, + struct inode *ecryptfs_inode) +{ + struct ecryptfs_crypt_stat *crypt_stat = + &ecryptfs_inode_to_private(ecryptfs_inode)->crypt_stat; + unsigned int order; + char *virt; + size_t virt_len; + size_t size = 0; + int rc = 0; + + if (likely(crypt_stat->flags & ECRYPTFS_ENCRYPTED)) { + if (!(crypt_stat->flags & ECRYPTFS_KEY_VALID)) { + printk(KERN_ERR "Key is invalid; bailing out\n"); + rc = -EINVAL; + goto out; + } + } else { + printk(KERN_WARNING "%s: Encrypted flag not set\n", + __func__); + rc = -EINVAL; + goto out; + } + virt_len = crypt_stat->metadata_size; + order = get_order(virt_len); + /* Released in this function */ + virt = (char *)ecryptfs_get_zeroed_pages(GFP_KERNEL, order); + if (!virt) { + printk(KERN_ERR "%s: Out of memory\n", __func__); + rc = -ENOMEM; + goto out; + } + /* Zeroed page ensures the in-header unencrypted i_size is set to 0 */ + rc = ecryptfs_write_headers_virt(virt, virt_len, &size, crypt_stat, + ecryptfs_dentry); + if (unlikely(rc)) { + printk(KERN_ERR "%s: Error whilst writing headers; rc = [%d]\n", + __func__, rc); + goto out_free; + } + if (crypt_stat->flags & ECRYPTFS_METADATA_IN_XATTR) + rc = ecryptfs_write_metadata_to_xattr(ecryptfs_dentry, ecryptfs_inode, + virt, size); + else + rc = ecryptfs_write_metadata_to_contents(ecryptfs_inode, virt, + virt_len); + if (rc) { + printk(KERN_ERR "%s: Error writing metadata out to lower file; " + "rc = [%d]\n", __func__, rc); + goto out_free; + } +out_free: + free_pages((unsigned long)virt, order); +out: + return rc; +} + +#define ECRYPTFS_DONT_VALIDATE_HEADER_SIZE 0 +#define ECRYPTFS_VALIDATE_HEADER_SIZE 1 +static int parse_header_metadata(struct ecryptfs_crypt_stat *crypt_stat, + char *virt, int *bytes_read, + int validate_header_size) +{ + int rc = 0; + u32 header_extent_size; + u16 num_header_extents_at_front; + + header_extent_size = get_unaligned_be32(virt); + virt += sizeof(__be32); + num_header_extents_at_front = get_unaligned_be16(virt); + crypt_stat->metadata_size = (((size_t)num_header_extents_at_front + * (size_t)header_extent_size)); + (*bytes_read) = (sizeof(__be32) + sizeof(__be16)); + if ((validate_header_size == ECRYPTFS_VALIDATE_HEADER_SIZE) + && (crypt_stat->metadata_size + < ECRYPTFS_MINIMUM_HEADER_EXTENT_SIZE)) { + rc = -EINVAL; + printk(KERN_WARNING "Invalid header size: [%zd]\n", + crypt_stat->metadata_size); + } + return rc; +} + +/** + * set_default_header_data + * @crypt_stat: The cryptographic context + * + * For version 0 file format; this function is only for backwards + * compatibility for files created with the prior versions of + * eCryptfs. + */ +static void set_default_header_data(struct ecryptfs_crypt_stat *crypt_stat) +{ + crypt_stat->metadata_size = ECRYPTFS_MINIMUM_HEADER_EXTENT_SIZE; +} + +void ecryptfs_i_size_init(const char *page_virt, struct inode *inode) +{ + struct ecryptfs_mount_crypt_stat *mount_crypt_stat; + struct ecryptfs_crypt_stat *crypt_stat; + u64 file_size; + + crypt_stat = &ecryptfs_inode_to_private(inode)->crypt_stat; + mount_crypt_stat = + &ecryptfs_superblock_to_private(inode->i_sb)->mount_crypt_stat; + if (mount_crypt_stat->flags & ECRYPTFS_ENCRYPTED_VIEW_ENABLED) { + file_size = i_size_read(ecryptfs_inode_to_lower(inode)); + if (crypt_stat->flags & ECRYPTFS_METADATA_IN_XATTR) + file_size += crypt_stat->metadata_size; + } else + file_size = get_unaligned_be64(page_virt); + i_size_write(inode, (loff_t)file_size); + crypt_stat->flags |= ECRYPTFS_I_SIZE_INITIALIZED; +} + +/** + * ecryptfs_read_headers_virt + * @page_virt: The virtual address into which to read the headers + * @crypt_stat: The cryptographic context + * @ecryptfs_dentry: The eCryptfs dentry + * @validate_header_size: Whether to validate the header size while reading + * + * Read/parse the header data. The header format is detailed in the + * comment block for the ecryptfs_write_headers_virt() function. + * + * Returns zero on success + */ +static int ecryptfs_read_headers_virt(char *page_virt, + struct ecryptfs_crypt_stat *crypt_stat, + struct dentry *ecryptfs_dentry, + int validate_header_size) +{ + int rc = 0; + int offset; + int bytes_read; + + ecryptfs_set_default_sizes(crypt_stat); + crypt_stat->mount_crypt_stat = &ecryptfs_superblock_to_private( + ecryptfs_dentry->d_sb)->mount_crypt_stat; + offset = ECRYPTFS_FILE_SIZE_BYTES; + rc = ecryptfs_validate_marker(page_virt + offset); + if (rc) + goto out; + if (!(crypt_stat->flags & ECRYPTFS_I_SIZE_INITIALIZED)) + ecryptfs_i_size_init(page_virt, d_inode(ecryptfs_dentry)); + offset += MAGIC_ECRYPTFS_MARKER_SIZE_BYTES; + ecryptfs_process_flags(crypt_stat, (page_virt + offset), &bytes_read); + if (crypt_stat->file_version > ECRYPTFS_SUPPORTED_FILE_VERSION) { + ecryptfs_printk(KERN_WARNING, "File version is [%d]; only " + "file version [%d] is supported by this " + "version of eCryptfs\n", + crypt_stat->file_version, + ECRYPTFS_SUPPORTED_FILE_VERSION); + rc = -EINVAL; + goto out; + } + offset += bytes_read; + if (crypt_stat->file_version >= 1) { + rc = parse_header_metadata(crypt_stat, (page_virt + offset), + &bytes_read, validate_header_size); + if (rc) { + ecryptfs_printk(KERN_WARNING, "Error reading header " + "metadata; rc = [%d]\n", rc); + } + offset += bytes_read; + } else + set_default_header_data(crypt_stat); + rc = ecryptfs_parse_packet_set(crypt_stat, (page_virt + offset), + PAGE_SIZE - offset, ecryptfs_dentry); +out: + return rc; +} + +/** + * ecryptfs_read_xattr_region + * @page_virt: The virtual address into which to read the xattr data + * @ecryptfs_inode: The eCryptfs inode + * + * Attempts to read the crypto metadata from the extended attribute + * region of the lower file. + * + * Returns zero on success; non-zero on error + */ +int ecryptfs_read_xattr_region(char *page_virt, struct inode *ecryptfs_inode) +{ + struct dentry *lower_dentry = + ecryptfs_inode_to_private(ecryptfs_inode)->lower_file->f_path.dentry; + ssize_t size; + int rc = 0; + + size = ecryptfs_getxattr_lower(lower_dentry, + ecryptfs_inode_to_lower(ecryptfs_inode), + ECRYPTFS_XATTR_NAME, + page_virt, ECRYPTFS_DEFAULT_EXTENT_SIZE); + if (size < 0) { + if (unlikely(ecryptfs_verbosity > 0)) + printk(KERN_INFO "Error attempting to read the [%s] " + "xattr from the lower file; return value = " + "[%zd]\n", ECRYPTFS_XATTR_NAME, size); + rc = -EINVAL; + goto out; + } +out: + return rc; +} + +int ecryptfs_read_and_validate_xattr_region(struct dentry *dentry, + struct inode *inode) +{ + u8 file_size[ECRYPTFS_SIZE_AND_MARKER_BYTES]; + u8 *marker = file_size + ECRYPTFS_FILE_SIZE_BYTES; + int rc; + + rc = ecryptfs_getxattr_lower(ecryptfs_dentry_to_lower(dentry), + ecryptfs_inode_to_lower(inode), + ECRYPTFS_XATTR_NAME, file_size, + ECRYPTFS_SIZE_AND_MARKER_BYTES); + if (rc < 0) + return rc; + else if (rc < ECRYPTFS_SIZE_AND_MARKER_BYTES) + return -EINVAL; + rc = ecryptfs_validate_marker(marker); + if (!rc) + ecryptfs_i_size_init(file_size, inode); + return rc; +} + +/* + * ecryptfs_read_metadata + * + * Common entry point for reading file metadata. From here, we could + * retrieve the header information from the header region of the file, + * the xattr region of the file, or some other repository that is + * stored separately from the file itself. The current implementation + * supports retrieving the metadata information from the file contents + * and from the xattr region. + * + * Returns zero if valid headers found and parsed; non-zero otherwise + */ +int ecryptfs_read_metadata(struct dentry *ecryptfs_dentry) +{ + int rc; + char *page_virt; + struct inode *ecryptfs_inode = d_inode(ecryptfs_dentry); + struct ecryptfs_crypt_stat *crypt_stat = + &ecryptfs_inode_to_private(ecryptfs_inode)->crypt_stat; + struct ecryptfs_mount_crypt_stat *mount_crypt_stat = + &ecryptfs_superblock_to_private( + ecryptfs_dentry->d_sb)->mount_crypt_stat; + + ecryptfs_copy_mount_wide_flags_to_inode_flags(crypt_stat, + mount_crypt_stat); + /* Read the first page from the underlying file */ + page_virt = kmem_cache_alloc(ecryptfs_header_cache, GFP_USER); + if (!page_virt) { + rc = -ENOMEM; + goto out; + } + rc = ecryptfs_read_lower(page_virt, 0, crypt_stat->extent_size, + ecryptfs_inode); + if (rc >= 0) + rc = ecryptfs_read_headers_virt(page_virt, crypt_stat, + ecryptfs_dentry, + ECRYPTFS_VALIDATE_HEADER_SIZE); + if (rc) { + /* metadata is not in the file header, so try xattrs */ + memset(page_virt, 0, PAGE_SIZE); + rc = ecryptfs_read_xattr_region(page_virt, ecryptfs_inode); + if (rc) { + printk(KERN_DEBUG "Valid eCryptfs headers not found in " + "file header region or xattr region, inode %llu\n", + ecryptfs_inode->i_ino); + rc = -EINVAL; + goto out; + } + rc = ecryptfs_read_headers_virt(page_virt, crypt_stat, + ecryptfs_dentry, + ECRYPTFS_DONT_VALIDATE_HEADER_SIZE); + if (rc) { + printk(KERN_DEBUG "Valid eCryptfs headers not found in " + "file xattr region either, inode %llu\n", + ecryptfs_inode->i_ino); + rc = -EINVAL; + } + if (crypt_stat->mount_crypt_stat->flags + & ECRYPTFS_XATTR_METADATA_ENABLED) { + crypt_stat->flags |= ECRYPTFS_METADATA_IN_XATTR; + } else { + printk(KERN_WARNING "Attempt to access file with " + "crypto metadata only in the extended attribute " + "region, but eCryptfs was mounted without " + "xattr support enabled. eCryptfs will not treat " + "this like an encrypted file, inode %llu\n", + ecryptfs_inode->i_ino); + rc = -EINVAL; + } + } +out: + if (page_virt) { + memset(page_virt, 0, PAGE_SIZE); + kmem_cache_free(ecryptfs_header_cache, page_virt); + } + return rc; +} + +/* + * ecryptfs_encrypt_filename - encrypt filename + * + * CBC-encrypts the filename. We do not want to encrypt the same + * filename with the same key and IV, which may happen with hard + * links, so we prepend random bits to each filename. + * + * Returns zero on success; non-zero otherwise + */ +static int +ecryptfs_encrypt_filename(struct ecryptfs_filename *filename, + struct ecryptfs_mount_crypt_stat *mount_crypt_stat) +{ + int rc = 0; + + filename->encrypted_filename = NULL; + filename->encrypted_filename_size = 0; + if (mount_crypt_stat && (mount_crypt_stat->flags + & ECRYPTFS_GLOBAL_ENCFN_USE_MOUNT_FNEK)) { + size_t packet_size; + size_t remaining_bytes; + + rc = ecryptfs_write_tag_70_packet( + NULL, NULL, + &filename->encrypted_filename_size, + mount_crypt_stat, NULL, + filename->filename_size); + if (rc) { + ecryptfs_printk(KERN_ERR, + "Error attempting to get packet size for tag 70; rc = [%d]\n", + rc); + filename->encrypted_filename_size = 0; + goto out; + } + filename->encrypted_filename = + kmalloc(filename->encrypted_filename_size, GFP_KERNEL); + if (!filename->encrypted_filename) { + rc = -ENOMEM; + goto out; + } + remaining_bytes = filename->encrypted_filename_size; + rc = ecryptfs_write_tag_70_packet(filename->encrypted_filename, + &remaining_bytes, + &packet_size, + mount_crypt_stat, + filename->filename, + filename->filename_size); + if (rc) { + printk(KERN_ERR "%s: Error attempting to generate " + "tag 70 packet; rc = [%d]\n", __func__, + rc); + kfree(filename->encrypted_filename); + filename->encrypted_filename = NULL; + filename->encrypted_filename_size = 0; + goto out; + } + filename->encrypted_filename_size = packet_size; + } else { + printk(KERN_ERR "%s: No support for requested filename " + "encryption method in this release\n", __func__); + rc = -EOPNOTSUPP; + goto out; + } +out: + return rc; +} + +static int ecryptfs_copy_filename(char **copied_name, size_t *copied_name_size, + const char *name, size_t name_size) +{ + (*copied_name) = kmemdup_nul(name, name_size, GFP_KERNEL); + if (!(*copied_name)) + return -ENOMEM; + (*copied_name_size) = name_size; + return 0; +} + +/** + * ecryptfs_process_key_cipher - Perform key cipher initialization. + * @key_tfm: Crypto context for key material, set by this function + * @cipher_name: Name of the cipher + * @key_size: Size of the key in bytes + * + * Returns zero on success. Any crypto_tfm structs allocated here + * should be released by other functions, such as on a superblock put + * event, regardless of whether this function succeeds for fails. + */ +static int +ecryptfs_process_key_cipher(struct crypto_skcipher **key_tfm, + char *cipher_name, size_t *key_size) +{ + char dummy_key[ECRYPTFS_MAX_KEY_BYTES]; + char *full_alg_name = NULL; + int rc; + + *key_tfm = NULL; + if (*key_size > ECRYPTFS_MAX_KEY_BYTES) { + rc = -EINVAL; + printk(KERN_ERR "Requested key size is [%zd] bytes; maximum " + "allowable is [%d]\n", *key_size, ECRYPTFS_MAX_KEY_BYTES); + goto out; + } + rc = ecryptfs_crypto_api_algify_cipher_name(&full_alg_name, cipher_name, + "ecb"); + if (rc) + goto out; + *key_tfm = crypto_alloc_skcipher(full_alg_name, 0, CRYPTO_ALG_ASYNC); + if (IS_ERR(*key_tfm)) { + rc = PTR_ERR(*key_tfm); + printk(KERN_ERR "Unable to allocate crypto cipher with name " + "[%s]; rc = [%d]\n", full_alg_name, rc); + goto out; + } + crypto_skcipher_set_flags(*key_tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); + if (*key_size == 0) + *key_size = crypto_skcipher_max_keysize(*key_tfm); + get_random_bytes(dummy_key, *key_size); + rc = crypto_skcipher_setkey(*key_tfm, dummy_key, *key_size); + if (rc) { + printk(KERN_ERR "Error attempting to set key of size [%zd] for " + "cipher [%s]; rc = [%d]\n", *key_size, full_alg_name, + rc); + rc = -EINVAL; + goto out; + } +out: + kfree(full_alg_name); + return rc; +} + +struct kmem_cache *ecryptfs_key_tfm_cache; +static struct list_head key_tfm_list; +DEFINE_MUTEX(key_tfm_list_mutex); + +int __init ecryptfs_init_crypto(void) +{ + INIT_LIST_HEAD(&key_tfm_list); + return 0; +} + +/** + * ecryptfs_destroy_crypto - free all cached key_tfms on key_tfm_list + * + * Called only at module unload time + */ +int ecryptfs_destroy_crypto(void) +{ + struct ecryptfs_key_tfm *key_tfm, *key_tfm_tmp; + + mutex_lock(&key_tfm_list_mutex); + list_for_each_entry_safe(key_tfm, key_tfm_tmp, &key_tfm_list, + key_tfm_list) { + list_del(&key_tfm->key_tfm_list); + crypto_free_skcipher(key_tfm->key_tfm); + kmem_cache_free(ecryptfs_key_tfm_cache, key_tfm); + } + mutex_unlock(&key_tfm_list_mutex); + return 0; +} + +int +ecryptfs_add_new_key_tfm(struct ecryptfs_key_tfm **key_tfm, char *cipher_name, + size_t key_size) +{ + struct ecryptfs_key_tfm *tmp_tfm; + int rc = 0; + + BUG_ON(!mutex_is_locked(&key_tfm_list_mutex)); + + tmp_tfm = kmem_cache_alloc(ecryptfs_key_tfm_cache, GFP_KERNEL); + if (key_tfm) + (*key_tfm) = tmp_tfm; + if (!tmp_tfm) { + rc = -ENOMEM; + goto out; + } + mutex_init(&tmp_tfm->key_tfm_mutex); + strscpy(tmp_tfm->cipher_name, cipher_name); + tmp_tfm->key_size = key_size; + rc = ecryptfs_process_key_cipher(&tmp_tfm->key_tfm, + tmp_tfm->cipher_name, + &tmp_tfm->key_size); + if (rc) { + printk(KERN_ERR "Error attempting to initialize key TFM " + "cipher with name = [%s]; rc = [%d]\n", + tmp_tfm->cipher_name, rc); + kmem_cache_free(ecryptfs_key_tfm_cache, tmp_tfm); + if (key_tfm) + (*key_tfm) = NULL; + goto out; + } + list_add(&tmp_tfm->key_tfm_list, &key_tfm_list); +out: + return rc; +} + +/** + * ecryptfs_tfm_exists - Search for existing tfm for cipher_name. + * @cipher_name: the name of the cipher to search for + * @key_tfm: set to corresponding tfm if found + * + * Searches for cached key_tfm matching @cipher_name + * Must be called with &key_tfm_list_mutex held + * Returns 1 if found, with @key_tfm set + * Returns 0 if not found, with @key_tfm set to NULL + */ +int ecryptfs_tfm_exists(char *cipher_name, struct ecryptfs_key_tfm **key_tfm) +{ + struct ecryptfs_key_tfm *tmp_key_tfm; + + BUG_ON(!mutex_is_locked(&key_tfm_list_mutex)); + + list_for_each_entry(tmp_key_tfm, &key_tfm_list, key_tfm_list) { + if (strcmp(tmp_key_tfm->cipher_name, cipher_name) == 0) { + if (key_tfm) + (*key_tfm) = tmp_key_tfm; + return 1; + } + } + if (key_tfm) + (*key_tfm) = NULL; + return 0; +} + +/** + * ecryptfs_get_tfm_and_mutex_for_cipher_name + * + * @tfm: set to cached tfm found, or new tfm created + * @tfm_mutex: set to mutex for cached tfm found, or new tfm created + * @cipher_name: the name of the cipher to search for and/or add + * + * Sets pointers to @tfm & @tfm_mutex matching @cipher_name. + * Searches for cached item first, and creates new if not found. + * Returns 0 on success, non-zero if adding new cipher failed + */ +int ecryptfs_get_tfm_and_mutex_for_cipher_name(struct crypto_skcipher **tfm, + struct mutex **tfm_mutex, + char *cipher_name) +{ + struct ecryptfs_key_tfm *key_tfm; + int rc = 0; + + (*tfm) = NULL; + (*tfm_mutex) = NULL; + + mutex_lock(&key_tfm_list_mutex); + if (!ecryptfs_tfm_exists(cipher_name, &key_tfm)) { + rc = ecryptfs_add_new_key_tfm(&key_tfm, cipher_name, 0); + if (rc) { + printk(KERN_ERR "Error adding new key_tfm to list; " + "rc = [%d]\n", rc); + goto out; + } + } + (*tfm) = key_tfm->key_tfm; + (*tfm_mutex) = &key_tfm->key_tfm_mutex; +out: + mutex_unlock(&key_tfm_list_mutex); + return rc; +} + +/* 64 characters forming a 6-bit target field */ +static unsigned char *portable_filename_chars = ("-.0123456789ABCD" + "EFGHIJKLMNOPQRST" + "UVWXYZabcdefghij" + "klmnopqrstuvwxyz"); + +/* We could either offset on every reverse map or just pad some 0x00's + * at the front here */ +static const unsigned char filename_rev_map[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 15 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 31 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 39 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, /* 47 */ + 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, /* 55 */ + 0x0A, 0x0B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 63 */ + 0x00, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, 0x12, /* 71 */ + 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, /* 79 */ + 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, 0x21, 0x22, /* 87 */ + 0x23, 0x24, 0x25, 0x00, 0x00, 0x00, 0x00, 0x00, /* 95 */ + 0x00, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, /* 103 */ + 0x2D, 0x2E, 0x2F, 0x30, 0x31, 0x32, 0x33, 0x34, /* 111 */ + 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, /* 119 */ + 0x3D, 0x3E, 0x3F /* 123 - 255 initialized to 0x00 */ +}; + +/** + * ecryptfs_encode_for_filename + * @dst: Destination location for encoded filename + * @dst_size: Size of the encoded filename in bytes + * @src: Source location for the filename to encode + * @src_size: Size of the source in bytes + */ +static void ecryptfs_encode_for_filename(unsigned char *dst, size_t *dst_size, + unsigned char *src, size_t src_size) +{ + size_t num_blocks; + size_t block_num = 0; + size_t dst_offset = 0; + unsigned char last_block[3]; + + if (src_size == 0) { + (*dst_size) = 0; + goto out; + } + num_blocks = (src_size / 3); + if ((src_size % 3) == 0) { + memcpy(last_block, (&src[src_size - 3]), 3); + } else { + num_blocks++; + last_block[2] = 0x00; + switch (src_size % 3) { + case 1: + last_block[0] = src[src_size - 1]; + last_block[1] = 0x00; + break; + case 2: + last_block[0] = src[src_size - 2]; + last_block[1] = src[src_size - 1]; + } + } + (*dst_size) = (num_blocks * 4); + if (!dst) + goto out; + while (block_num < num_blocks) { + unsigned char *src_block; + unsigned char dst_block[4]; + + if (block_num == (num_blocks - 1)) + src_block = last_block; + else + src_block = &src[block_num * 3]; + dst_block[0] = ((src_block[0] >> 2) & 0x3F); + dst_block[1] = (((src_block[0] << 4) & 0x30) + | ((src_block[1] >> 4) & 0x0F)); + dst_block[2] = (((src_block[1] << 2) & 0x3C) + | ((src_block[2] >> 6) & 0x03)); + dst_block[3] = (src_block[2] & 0x3F); + dst[dst_offset++] = portable_filename_chars[dst_block[0]]; + dst[dst_offset++] = portable_filename_chars[dst_block[1]]; + dst[dst_offset++] = portable_filename_chars[dst_block[2]]; + dst[dst_offset++] = portable_filename_chars[dst_block[3]]; + block_num++; + } +out: + return; +} + +static size_t ecryptfs_max_decoded_size(size_t encoded_size) +{ + /* Not exact; conservatively long. Every block of 4 + * encoded characters decodes into a block of 3 + * decoded characters. This segment of code provides + * the caller with the maximum amount of allocated + * space that @dst will need to point to in a + * subsequent call. */ + return ((encoded_size + 1) * 3) / 4; +} + +/** + * ecryptfs_decode_from_filename + * @dst: If NULL, this function only sets @dst_size and returns. If + * non-NULL, this function decodes the encoded octets in @src + * into the memory that @dst points to. + * @dst_size: Set to the size of the decoded string. + * @src: The encoded set of octets to decode. + * @src_size: The size of the encoded set of octets to decode. + */ +static void +ecryptfs_decode_from_filename(unsigned char *dst, size_t *dst_size, + const unsigned char *src, size_t src_size) +{ + u8 current_bit_offset = 0; + size_t src_byte_offset = 0; + size_t dst_byte_offset = 0; + + if (!dst) { + (*dst_size) = ecryptfs_max_decoded_size(src_size); + goto out; + } + while (src_byte_offset < src_size) { + unsigned char src_byte = + filename_rev_map[(int)src[src_byte_offset]]; + + switch (current_bit_offset) { + case 0: + dst[dst_byte_offset] = (src_byte << 2); + current_bit_offset = 6; + break; + case 6: + dst[dst_byte_offset++] |= (src_byte >> 4); + dst[dst_byte_offset] = ((src_byte & 0xF) + << 4); + current_bit_offset = 4; + break; + case 4: + dst[dst_byte_offset++] |= (src_byte >> 2); + dst[dst_byte_offset] = (src_byte << 6); + current_bit_offset = 2; + break; + case 2: + dst[dst_byte_offset++] |= (src_byte); + current_bit_offset = 0; + break; + } + src_byte_offset++; + } + (*dst_size) = dst_byte_offset; +out: + return; +} + +/** + * ecryptfs_encrypt_and_encode_filename - converts a plaintext file name to cipher text + * @encoded_name: The encrypted name + * @encoded_name_size: Length of the encrypted name + * @mount_crypt_stat: The crypt_stat struct associated with the file name to encode + * @name: The plaintext name + * @name_size: The length of the plaintext name + * + * Encrypts and encodes a filename into something that constitutes a + * valid filename for a filesystem, with printable characters. + * + * We assume that we have a properly initialized crypto context, + * pointed to by crypt_stat->tfm. + * + * Returns zero on success; non-zero on otherwise + */ +int ecryptfs_encrypt_and_encode_filename( + char **encoded_name, + size_t *encoded_name_size, + struct ecryptfs_mount_crypt_stat *mount_crypt_stat, + const char *name, size_t name_size) +{ + size_t encoded_name_no_prefix_size; + int rc = 0; + + (*encoded_name) = NULL; + (*encoded_name_size) = 0; + if (mount_crypt_stat && (mount_crypt_stat->flags + & ECRYPTFS_GLOBAL_ENCRYPT_FILENAMES)) { + struct ecryptfs_filename *filename; + + filename = kzalloc_obj(*filename); + if (!filename) { + rc = -ENOMEM; + goto out; + } + filename->filename = (char *)name; + filename->filename_size = name_size; + rc = ecryptfs_encrypt_filename(filename, mount_crypt_stat); + if (rc) { + ecryptfs_printk(KERN_ERR, + "Error attempting to encrypt filename; rc = [%d]\n", + rc); + kfree(filename); + goto out; + } + ecryptfs_encode_for_filename( + NULL, &encoded_name_no_prefix_size, + filename->encrypted_filename, + filename->encrypted_filename_size); + if (mount_crypt_stat->flags + & ECRYPTFS_GLOBAL_ENCFN_USE_MOUNT_FNEK) + (*encoded_name_size) = + (ECRYPTFS_FNEK_ENCRYPTED_FILENAME_PREFIX_SIZE + + encoded_name_no_prefix_size); + else + (*encoded_name_size) = + (ECRYPTFS_FEK_ENCRYPTED_FILENAME_PREFIX_SIZE + + encoded_name_no_prefix_size); + (*encoded_name) = kmalloc((*encoded_name_size) + 1, GFP_KERNEL); + if (!(*encoded_name)) { + rc = -ENOMEM; + kfree(filename->encrypted_filename); + kfree(filename); + goto out; + } + if (mount_crypt_stat->flags + & ECRYPTFS_GLOBAL_ENCFN_USE_MOUNT_FNEK) { + memcpy((*encoded_name), + ECRYPTFS_FNEK_ENCRYPTED_FILENAME_PREFIX, + ECRYPTFS_FNEK_ENCRYPTED_FILENAME_PREFIX_SIZE); + ecryptfs_encode_for_filename( + ((*encoded_name) + + ECRYPTFS_FNEK_ENCRYPTED_FILENAME_PREFIX_SIZE), + &encoded_name_no_prefix_size, + filename->encrypted_filename, + filename->encrypted_filename_size); + (*encoded_name_size) = + (ECRYPTFS_FNEK_ENCRYPTED_FILENAME_PREFIX_SIZE + + encoded_name_no_prefix_size); + (*encoded_name)[(*encoded_name_size)] = '\0'; + } else { + rc = -EOPNOTSUPP; + } + if (rc) { + ecryptfs_printk(KERN_ERR, + "Error attempting to encode encrypted filename; rc = [%d]\n", + rc); + kfree((*encoded_name)); + (*encoded_name) = NULL; + (*encoded_name_size) = 0; + } + kfree(filename->encrypted_filename); + kfree(filename); + } else { + rc = ecryptfs_copy_filename(encoded_name, + encoded_name_size, + name, name_size); + } +out: + return rc; +} + +/** + * ecryptfs_decode_and_decrypt_filename - converts the encoded cipher text name to decoded plaintext + * @plaintext_name: The plaintext name + * @plaintext_name_size: The plaintext name size + * @sb: Ecryptfs's super_block + * @name: The filename in cipher text + * @name_size: The cipher text name size + * + * Decrypts and decodes the filename. + * + * Returns zero on error; non-zero otherwise + */ +int ecryptfs_decode_and_decrypt_filename(char **plaintext_name, + size_t *plaintext_name_size, + struct super_block *sb, + const char *name, size_t name_size) +{ + struct ecryptfs_mount_crypt_stat *mount_crypt_stat = + &ecryptfs_superblock_to_private(sb)->mount_crypt_stat; + char *decoded_name; + size_t decoded_name_size; + size_t packet_size; + int rc = 0; + + if ((mount_crypt_stat->flags & ECRYPTFS_GLOBAL_ENCRYPT_FILENAMES) && + !(mount_crypt_stat->flags & ECRYPTFS_ENCRYPTED_VIEW_ENABLED)) { + if (name_is_dot_dotdot(name, name_size)) { + rc = ecryptfs_copy_filename(plaintext_name, + plaintext_name_size, + name, name_size); + goto out; + } + + if (name_size <= ECRYPTFS_FNEK_ENCRYPTED_FILENAME_PREFIX_SIZE || + strncmp(name, ECRYPTFS_FNEK_ENCRYPTED_FILENAME_PREFIX, + ECRYPTFS_FNEK_ENCRYPTED_FILENAME_PREFIX_SIZE)) { + rc = -EINVAL; + goto out; + } + + name += ECRYPTFS_FNEK_ENCRYPTED_FILENAME_PREFIX_SIZE; + name_size -= ECRYPTFS_FNEK_ENCRYPTED_FILENAME_PREFIX_SIZE; + ecryptfs_decode_from_filename(NULL, &decoded_name_size, + name, name_size); + decoded_name = kmalloc(decoded_name_size, GFP_KERNEL); + if (!decoded_name) { + rc = -ENOMEM; + goto out; + } + ecryptfs_decode_from_filename(decoded_name, &decoded_name_size, + name, name_size); + rc = ecryptfs_parse_tag_70_packet(plaintext_name, + plaintext_name_size, + &packet_size, + mount_crypt_stat, + decoded_name, + decoded_name_size); + if (rc) { + ecryptfs_printk(KERN_DEBUG, + "Could not parse tag 70 packet from filename\n"); + goto out_free; + } + } else { + rc = ecryptfs_copy_filename(plaintext_name, + plaintext_name_size, + name, name_size); + goto out; + } +out_free: + kfree(decoded_name); +out: + return rc; +} + +#define ENC_NAME_MAX_BLOCKLEN_8_OR_16 143 + +int ecryptfs_set_f_namelen(long *namelen, long lower_namelen, + struct ecryptfs_mount_crypt_stat *mount_crypt_stat) +{ + struct crypto_skcipher *tfm; + struct mutex *tfm_mutex; + size_t cipher_blocksize; + int rc; + + if (!(mount_crypt_stat->flags & ECRYPTFS_GLOBAL_ENCRYPT_FILENAMES)) { + (*namelen) = lower_namelen; + return 0; + } + + rc = ecryptfs_get_tfm_and_mutex_for_cipher_name(&tfm, &tfm_mutex, + mount_crypt_stat->global_default_fn_cipher_name); + if (unlikely(rc)) { + (*namelen) = 0; + return rc; + } + + mutex_lock(tfm_mutex); + cipher_blocksize = crypto_skcipher_blocksize(tfm); + mutex_unlock(tfm_mutex); + + /* Return an exact amount for the common cases */ + if (lower_namelen == NAME_MAX + && (cipher_blocksize == 8 || cipher_blocksize == 16)) { + (*namelen) = ENC_NAME_MAX_BLOCKLEN_8_OR_16; + return 0; + } + + /* Return a safe estimate for the uncommon cases */ + (*namelen) = lower_namelen; + (*namelen) -= ECRYPTFS_FNEK_ENCRYPTED_FILENAME_PREFIX_SIZE; + /* Since this is the max decoded size, subtract 1 "decoded block" len */ + (*namelen) = ecryptfs_max_decoded_size(*namelen) - 3; + (*namelen) -= ECRYPTFS_TAG_70_MAX_METADATA_SIZE; + (*namelen) -= ECRYPTFS_FILENAME_MIN_RANDOM_PREPEND_BYTES; + /* Worst case is that the filename is padded nearly a full block size */ + (*namelen) -= cipher_blocksize - 1; + + if ((*namelen) < 0) + (*namelen) = 0; + + return 0; +} diff --git a/fs/ecryptfs/debug.c b/fs/ecryptfs/debug.c new file mode 100644 index 000000000..426437024 --- /dev/null +++ b/fs/ecryptfs/debug.c @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * eCryptfs: Linux filesystem encryption layer + * Functions only useful for debugging. + * + * Copyright (C) 2006 International Business Machines Corp. + * Author(s): Michael A. Halcrow + */ + +#include +#include "ecryptfs_kernel.h" + +/* + * ecryptfs_dump_auth_tok - debug function to print auth toks + * + * This function will print the contents of an ecryptfs authentication + * token. + */ +void ecryptfs_dump_auth_tok(struct ecryptfs_auth_tok *auth_tok) +{ + char salt[ECRYPTFS_SALT_SIZE * 2 + 1]; + char sig[ECRYPTFS_SIG_SIZE_HEX + 1]; + + ecryptfs_printk(KERN_DEBUG, "Auth tok at mem loc [%p]:\n", + auth_tok); + if (auth_tok->flags & ECRYPTFS_PRIVATE_KEY) { + ecryptfs_printk(KERN_DEBUG, " * private key type\n"); + } else { + ecryptfs_printk(KERN_DEBUG, " * passphrase type\n"); + ecryptfs_to_hex(salt, auth_tok->token.password.salt, + ECRYPTFS_SALT_SIZE); + ecryptfs_printk(KERN_DEBUG, " * salt = [%s]\n", salt); + if (auth_tok->token.password.flags & + ECRYPTFS_PERSISTENT_PASSWORD) { + ecryptfs_printk(KERN_DEBUG, " * persistent\n"); + } + strscpy(sig, auth_tok->token.password.signature); + ecryptfs_printk(KERN_DEBUG, " * signature = [%s]\n", sig); + } + ecryptfs_printk(KERN_DEBUG, " * session_key.flags = [0x%x]\n", + auth_tok->session_key.flags); + if (auth_tok->session_key.flags + & ECRYPTFS_USERSPACE_SHOULD_TRY_TO_DECRYPT) + ecryptfs_printk(KERN_DEBUG, + " * Userspace decrypt request set\n"); + if (auth_tok->session_key.flags + & ECRYPTFS_USERSPACE_SHOULD_TRY_TO_ENCRYPT) + ecryptfs_printk(KERN_DEBUG, + " * Userspace encrypt request set\n"); + if (auth_tok->session_key.flags & ECRYPTFS_CONTAINS_DECRYPTED_KEY) { + ecryptfs_printk(KERN_DEBUG, " * Contains decrypted key\n"); + ecryptfs_printk(KERN_DEBUG, + " * session_key.decrypted_key_size = [0x%x]\n", + auth_tok->session_key.decrypted_key_size); + ecryptfs_printk(KERN_DEBUG, " * Decrypted session key " + "dump:\n"); + if (ecryptfs_verbosity > 0) + ecryptfs_dump_hex(auth_tok->session_key.decrypted_key, + ECRYPTFS_DEFAULT_KEY_BYTES); + } + if (auth_tok->session_key.flags & ECRYPTFS_CONTAINS_ENCRYPTED_KEY) { + ecryptfs_printk(KERN_DEBUG, " * Contains encrypted key\n"); + ecryptfs_printk(KERN_DEBUG, + " * session_key.encrypted_key_size = [0x%x]\n", + auth_tok->session_key.encrypted_key_size); + ecryptfs_printk(KERN_DEBUG, " * Encrypted session key " + "dump:\n"); + if (ecryptfs_verbosity > 0) + ecryptfs_dump_hex(auth_tok->session_key.encrypted_key, + auth_tok->session_key. + encrypted_key_size); + } +} + +/** + * ecryptfs_dump_hex - debug hex printer + * @data: string of bytes to be printed + * @bytes: number of bytes to print + * + * Dump hexadecimal representation of char array + */ +void ecryptfs_dump_hex(char *data, int bytes) +{ + if (ecryptfs_verbosity < 1) + return; + + print_hex_dump(KERN_DEBUG, "ecryptfs: ", DUMP_PREFIX_OFFSET, 16, 1, + data, bytes, false); +} diff --git a/fs/ecryptfs/dentry.c b/fs/ecryptfs/dentry.c new file mode 100644 index 000000000..6648a924e --- /dev/null +++ b/fs/ecryptfs/dentry.c @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * eCryptfs: Linux filesystem encryption layer + * + * Copyright (C) 1997-2003 Erez Zadok + * Copyright (C) 2001-2003 Stony Brook University + * Copyright (C) 2004-2006 International Business Machines Corp. + * Author(s): Michael A. Halcrow + */ + +#include +#include +#include +#include +#include +#include "ecryptfs_kernel.h" + +/** + * ecryptfs_d_revalidate - revalidate an ecryptfs dentry + * @dir: inode of expected parent + * @name: expected name + * @dentry: dentry to revalidate + * @flags: lookup flags + * + * Called when the VFS needs to revalidate a dentry. This + * is called whenever a name lookup finds a dentry in the + * dcache. Most filesystems leave this as NULL, because all their + * dentries in the dcache are valid. + * + * Returns 1 if valid, 0 otherwise. + * + */ +static int ecryptfs_d_revalidate(struct inode *dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + struct dentry *lower_dentry = ecryptfs_dentry_to_lower(dentry); + int rc = 1; + + if (flags & LOOKUP_RCU) + return -ECHILD; + + if (lower_dentry->d_flags & DCACHE_OP_REVALIDATE) { + struct inode *lower_dir = ecryptfs_inode_to_lower(dir); + struct name_snapshot n; + + take_dentry_name_snapshot(&n, lower_dentry); + rc = lower_dentry->d_op->d_revalidate(lower_dir, &n.name, + lower_dentry, flags); + release_dentry_name_snapshot(&n); + } + + if (d_really_is_positive(dentry)) { + struct inode *inode = d_inode(dentry); + + fsstack_copy_attr_all(inode, ecryptfs_inode_to_lower(inode)); + if (!inode->i_nlink) + return 0; + } + return rc; +} + +/** + * ecryptfs_d_release + * @dentry: The ecryptfs dentry + * + * Called when a dentry is really deallocated. + */ +static void ecryptfs_d_release(struct dentry *dentry) +{ + dput(dentry->d_fsdata); +} + +const struct dentry_operations ecryptfs_dops = { + .d_revalidate = ecryptfs_d_revalidate, + .d_release = ecryptfs_d_release, +}; diff --git a/fs/ecryptfs/ecryptfs_kernel.h b/fs/ecryptfs/ecryptfs_kernel.h new file mode 100644 index 000000000..58165928e --- /dev/null +++ b/fs/ecryptfs/ecryptfs_kernel.h @@ -0,0 +1,697 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * eCryptfs: Linux filesystem encryption layer + * Kernel declarations. + * + * Copyright (C) 1997-2003 Erez Zadok + * Copyright (C) 2001-2003 Stony Brook University + * Copyright (C) 2004-2008 International Business Machines Corp. + * Author(s): Michael A. Halcrow + * Trevor S. Highland + * Tyler Hicks + */ + +#ifndef ECRYPTFS_KERNEL_H +#define ECRYPTFS_KERNEL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define ECRYPTFS_DEFAULT_IV_BYTES 16 +#define ECRYPTFS_DEFAULT_EXTENT_SIZE 4096 +#define ECRYPTFS_MINIMUM_HEADER_EXTENT_SIZE 8192 +#define ECRYPTFS_DEFAULT_MSG_CTX_ELEMS 32 +#define ECRYPTFS_DEFAULT_SEND_TIMEOUT HZ +#define ECRYPTFS_MAX_MSG_CTX_TTL (HZ*3) +#define ECRYPTFS_DEFAULT_NUM_USERS 4 +#define ECRYPTFS_MAX_NUM_USERS 32768 +#define ECRYPTFS_XATTR_NAME "user.ecryptfs" + +void ecryptfs_dump_auth_tok(struct ecryptfs_auth_tok *auth_tok); +static inline void +ecryptfs_to_hex(char *dst, char *src, size_t src_size) +{ + char *end = bin2hex(dst, src, src_size); + *end = '\0'; +} + +extern void ecryptfs_from_hex(char *dst, char *src, int dst_size); + +struct ecryptfs_key_record { + unsigned char type; + size_t enc_key_size; + unsigned char sig[ECRYPTFS_SIG_SIZE]; + unsigned char enc_key[ECRYPTFS_MAX_ENCRYPTED_KEY_BYTES]; +}; + +struct ecryptfs_auth_tok_list { + struct ecryptfs_auth_tok *auth_tok; + struct list_head list; +}; + +struct ecryptfs_crypt_stat; +struct ecryptfs_mount_crypt_stat; + +struct ecryptfs_page_crypt_context { + struct page *page; +#define ECRYPTFS_PREPARE_COMMIT_MODE 0 +#define ECRYPTFS_WRITEPAGE_MODE 1 + unsigned int mode; + union { + struct file *lower_file; + struct writeback_control *wbc; + } param; +}; + +#if defined(CONFIG_ENCRYPTED_KEYS) || defined(CONFIG_ENCRYPTED_KEYS_MODULE) +static inline struct ecryptfs_auth_tok * +ecryptfs_get_encrypted_key_payload_data(struct key *key) +{ + struct encrypted_key_payload *payload; + + if (key->type != &key_type_encrypted) + return NULL; + + payload = key->payload.data[0]; + if (!payload) + return ERR_PTR(-EKEYREVOKED); + + return (struct ecryptfs_auth_tok *)payload->payload_data; +} + +static inline struct key *ecryptfs_get_encrypted_key(char *sig) +{ + return request_key(&key_type_encrypted, sig, NULL); +} + +#else +static inline struct ecryptfs_auth_tok * +ecryptfs_get_encrypted_key_payload_data(struct key *key) +{ + return NULL; +} + +static inline struct key *ecryptfs_get_encrypted_key(char *sig) +{ + return ERR_PTR(-ENOKEY); +} + +#endif /* CONFIG_ENCRYPTED_KEYS */ + +static inline struct ecryptfs_auth_tok * +ecryptfs_get_key_payload_data(struct key *key) +{ + struct ecryptfs_auth_tok *auth_tok; + struct user_key_payload *ukp; + + auth_tok = ecryptfs_get_encrypted_key_payload_data(key); + if (auth_tok) + return auth_tok; + + ukp = user_key_payload_locked(key); + if (!ukp) + return ERR_PTR(-EKEYREVOKED); + + return (struct ecryptfs_auth_tok *)ukp->data; +} + +#define ECRYPTFS_MAX_KEYSET_SIZE 1024 +#define ECRYPTFS_MAX_CIPHER_NAME_SIZE 31 +#define ECRYPTFS_MAX_NUM_ENC_KEYS 64 +#define ECRYPTFS_MAX_IV_BYTES 16 /* 128 bits */ +#define ECRYPTFS_SALT_BYTES 2 +#define MAGIC_ECRYPTFS_MARKER 0x3c81b7f5 +#define MAGIC_ECRYPTFS_MARKER_SIZE_BYTES 8 /* 4*2 */ +#define ECRYPTFS_FILE_SIZE_BYTES (sizeof(u64)) +#define ECRYPTFS_SIZE_AND_MARKER_BYTES (ECRYPTFS_FILE_SIZE_BYTES \ + + MAGIC_ECRYPTFS_MARKER_SIZE_BYTES) +#define ECRYPTFS_DEFAULT_CIPHER "aes" +#define ECRYPTFS_DEFAULT_KEY_BYTES 16 +#define ECRYPTFS_TAG_1_PACKET_TYPE 0x01 +#define ECRYPTFS_TAG_3_PACKET_TYPE 0x8C +#define ECRYPTFS_TAG_11_PACKET_TYPE 0xED +#define ECRYPTFS_TAG_64_PACKET_TYPE 0x40 +#define ECRYPTFS_TAG_65_PACKET_TYPE 0x41 +#define ECRYPTFS_TAG_66_PACKET_TYPE 0x42 +#define ECRYPTFS_TAG_67_PACKET_TYPE 0x43 +#define ECRYPTFS_TAG_70_PACKET_TYPE 0x46 /* FNEK-encrypted filename + * as dentry name */ +#define ECRYPTFS_TAG_71_PACKET_TYPE 0x47 /* FNEK-encrypted filename in + * metadata */ +#define ECRYPTFS_TAG_72_PACKET_TYPE 0x48 /* FEK-encrypted filename as + * dentry name */ +#define ECRYPTFS_TAG_73_PACKET_TYPE 0x49 /* FEK-encrypted filename as + * metadata */ +#define ECRYPTFS_MIN_PKT_LEN_SIZE 1 /* Min size to specify packet length */ +#define ECRYPTFS_MAX_PKT_LEN_SIZE 2 /* Pass at least this many bytes to + * ecryptfs_parse_packet_length() and + * ecryptfs_write_packet_length() + */ +/* Constraint: ECRYPTFS_FILENAME_MIN_RANDOM_PREPEND_BYTES >= + * ECRYPTFS_MAX_IV_BYTES */ +#define ECRYPTFS_FILENAME_MIN_RANDOM_PREPEND_BYTES 16 +#define ECRYPTFS_NON_NULL 0x42 /* A reasonable substitute for NULL */ +#define ECRYPTFS_TAG_70_MIN_METADATA_SIZE (1 + ECRYPTFS_MIN_PKT_LEN_SIZE \ + + ECRYPTFS_SIG_SIZE + 1 + 1) +#define ECRYPTFS_TAG_70_MAX_METADATA_SIZE (1 + ECRYPTFS_MAX_PKT_LEN_SIZE \ + + ECRYPTFS_SIG_SIZE + 1 + 1) +#define ECRYPTFS_FEK_ENCRYPTED_FILENAME_PREFIX "ECRYPTFS_FEK_ENCRYPTED." +#define ECRYPTFS_FEK_ENCRYPTED_FILENAME_PREFIX_SIZE 23 +#define ECRYPTFS_FNEK_ENCRYPTED_FILENAME_PREFIX "ECRYPTFS_FNEK_ENCRYPTED." +#define ECRYPTFS_FNEK_ENCRYPTED_FILENAME_PREFIX_SIZE 24 +#define ECRYPTFS_ENCRYPTED_DENTRY_NAME_LEN (18 + 1 + 4 + 1 + 32) + +#ifdef CONFIG_ECRYPT_FS_MESSAGING +# define ECRYPTFS_VERSIONING_MASK_MESSAGING (ECRYPTFS_VERSIONING_DEVMISC \ + | ECRYPTFS_VERSIONING_PUBKEY) +#else +# define ECRYPTFS_VERSIONING_MASK_MESSAGING 0 +#endif + +#define ECRYPTFS_VERSIONING_MASK (ECRYPTFS_VERSIONING_PASSPHRASE \ + | ECRYPTFS_VERSIONING_PLAINTEXT_PASSTHROUGH \ + | ECRYPTFS_VERSIONING_XATTR \ + | ECRYPTFS_VERSIONING_MULTKEY \ + | ECRYPTFS_VERSIONING_MASK_MESSAGING \ + | ECRYPTFS_VERSIONING_FILENAME_ENCRYPTION) +struct ecryptfs_key_sig { + struct list_head crypt_stat_list; + char keysig[ECRYPTFS_SIG_SIZE_HEX + 1]; +}; + +struct ecryptfs_filename { + struct list_head crypt_stat_list; +#define ECRYPTFS_FILENAME_CONTAINS_DECRYPTED 0x00000001 + u32 flags; + u32 seq_no; + char *filename; + char *encrypted_filename; + size_t filename_size; + size_t encrypted_filename_size; + char fnek_sig[ECRYPTFS_SIG_SIZE_HEX]; + char dentry_name[ECRYPTFS_ENCRYPTED_DENTRY_NAME_LEN + 1]; +}; + +/* + * This is the primary struct associated with each encrypted file. + * + * TODO: cache align/pack? + */ +struct ecryptfs_crypt_stat { +#define ECRYPTFS_STRUCT_INITIALIZED 0x00000001 +#define ECRYPTFS_POLICY_APPLIED 0x00000002 +#define ECRYPTFS_ENCRYPTED 0x00000004 +#define ECRYPTFS_SECURITY_WARNING 0x00000008 +#define ECRYPTFS_ENABLE_HMAC 0x00000010 +#define ECRYPTFS_ENCRYPT_IV_PAGES 0x00000020 +#define ECRYPTFS_KEY_VALID 0x00000040 +#define ECRYPTFS_METADATA_IN_XATTR 0x00000080 +#define ECRYPTFS_VIEW_AS_ENCRYPTED 0x00000100 +#define ECRYPTFS_KEY_SET 0x00000200 +#define ECRYPTFS_ENCRYPT_FILENAMES 0x00000400 +#define ECRYPTFS_ENCFN_USE_MOUNT_FNEK 0x00000800 +#define ECRYPTFS_ENCFN_USE_FEK 0x00001000 +#define ECRYPTFS_UNLINK_SIGS 0x00002000 +#define ECRYPTFS_I_SIZE_INITIALIZED 0x00004000 + u32 flags; + unsigned int file_version; + size_t iv_bytes; + size_t metadata_size; + size_t extent_size; /* Data extent size; default is 4096 */ + size_t key_size; + size_t extent_shift; + unsigned int extent_mask; + struct ecryptfs_mount_crypt_stat *mount_crypt_stat; + struct crypto_skcipher *tfm; + unsigned char cipher[ECRYPTFS_MAX_CIPHER_NAME_SIZE + 1]; + unsigned char key[ECRYPTFS_MAX_KEY_BYTES]; + unsigned char root_iv[ECRYPTFS_MAX_IV_BYTES]; + struct list_head keysig_list; + struct mutex keysig_list_mutex; + struct mutex cs_tfm_mutex; + struct mutex cs_mutex; +}; + +/* inode private data. */ +struct ecryptfs_inode_info { + struct inode vfs_inode; + struct inode *wii_inode; + struct mutex lower_file_mutex; + atomic_t lower_file_count; + struct file *lower_file; + struct ecryptfs_crypt_stat crypt_stat; +}; + +/** + * struct ecryptfs_global_auth_tok - A key used to encrypt all new files + * under the mountpoint + * @flags: Status flags + * @mount_crypt_stat_list: These auth_toks hang off the mount-wide + * cryptographic context. Every time a new + * inode comes into existence, eCryptfs copies + * the auth_toks on that list to the set of + * auth_toks on the inode's crypt_stat + * @global_auth_tok_key: The key from the user's keyring for the sig + * @sig: The key identifier + * + * ecryptfs_global_auth_tok structs refer to authentication token keys + * in the user keyring that apply to newly created files. A list of + * these objects hangs off of the mount_crypt_stat struct for any + * given eCryptfs mount. This struct maintains a reference to both the + * key contents and the key itself so that the key can be put on + * unmount. + */ +struct ecryptfs_global_auth_tok { +#define ECRYPTFS_AUTH_TOK_INVALID 0x00000001 +#define ECRYPTFS_AUTH_TOK_FNEK 0x00000002 + u32 flags; + struct list_head mount_crypt_stat_list; + struct key *global_auth_tok_key; + unsigned char sig[ECRYPTFS_SIG_SIZE_HEX + 1]; +}; + +/** + * struct ecryptfs_key_tfm - Persistent key tfm + * @key_tfm: crypto API handle to the key + * @key_size: Key size in bytes + * @key_tfm_mutex: Mutex to ensure only one operation in eCryptfs is + * using the persistent TFM at any point in time + * @key_tfm_list: Handle to hang this off the module-wide TFM list + * @cipher_name: String name for the cipher for this TFM + * + * Typically, eCryptfs will use the same ciphers repeatedly throughout + * the course of its operations. In order to avoid unnecessarily + * destroying and initializing the same cipher repeatedly, eCryptfs + * keeps a list of crypto API contexts around to use when needed. + */ +struct ecryptfs_key_tfm { + struct crypto_skcipher *key_tfm; + size_t key_size; + struct mutex key_tfm_mutex; + struct list_head key_tfm_list; + unsigned char cipher_name[ECRYPTFS_MAX_CIPHER_NAME_SIZE + 1]; +}; + +extern struct mutex key_tfm_list_mutex; + +/* + * This struct is to enable a mount-wide passphrase/salt combo. This + * is more or less a stopgap to provide similar functionality to other + * crypto filesystems like EncFS or CFS until full policy support is + * implemented in eCryptfs. + */ +struct ecryptfs_mount_crypt_stat { + /* Pointers to memory we do not own, do not free these */ +#define ECRYPTFS_PLAINTEXT_PASSTHROUGH_ENABLED 0x00000001 +#define ECRYPTFS_XATTR_METADATA_ENABLED 0x00000002 +#define ECRYPTFS_ENCRYPTED_VIEW_ENABLED 0x00000004 +#define ECRYPTFS_MOUNT_CRYPT_STAT_INITIALIZED 0x00000008 +#define ECRYPTFS_GLOBAL_ENCRYPT_FILENAMES 0x00000010 +#define ECRYPTFS_GLOBAL_ENCFN_USE_MOUNT_FNEK 0x00000020 +#define ECRYPTFS_GLOBAL_ENCFN_USE_FEK 0x00000040 +#define ECRYPTFS_GLOBAL_MOUNT_AUTH_TOK_ONLY 0x00000080 + u32 flags; + struct list_head global_auth_tok_list; + struct mutex global_auth_tok_list_mutex; + size_t global_default_cipher_key_size; + size_t global_default_fn_cipher_key_bytes; + unsigned char global_default_cipher_name[ECRYPTFS_MAX_CIPHER_NAME_SIZE + + 1]; + unsigned char global_default_fn_cipher_name[ + ECRYPTFS_MAX_CIPHER_NAME_SIZE + 1]; + char global_default_fnek_sig[ECRYPTFS_SIG_SIZE_HEX + 1]; +}; + +/* superblock private data. */ +struct ecryptfs_sb_info { + struct super_block *wsi_sb; + struct vfsmount *lower_mnt; + struct ecryptfs_mount_crypt_stat mount_crypt_stat; +}; + +/* file private data. */ +struct ecryptfs_file_info { + struct file *wfi_file; + struct ecryptfs_crypt_stat *crypt_stat; +}; + +/* auth_tok <=> encrypted_session_key mappings */ +struct ecryptfs_auth_tok_list_item { + unsigned char encrypted_session_key[ECRYPTFS_MAX_KEY_BYTES]; + struct list_head list; + struct ecryptfs_auth_tok auth_tok; +}; + +struct ecryptfs_message { + /* Can never be greater than ecryptfs_message_buf_len */ + /* Used to find the parent msg_ctx */ + /* Inherits from msg_ctx->index */ + u32 index; + u32 data_len; + u8 data[] __counted_by(data_len); +}; + +struct ecryptfs_msg_ctx { +#define ECRYPTFS_MSG_CTX_STATE_FREE 0x01 +#define ECRYPTFS_MSG_CTX_STATE_PENDING 0x02 +#define ECRYPTFS_MSG_CTX_STATE_DONE 0x03 +#define ECRYPTFS_MSG_CTX_STATE_NO_REPLY 0x04 + u8 state; +#define ECRYPTFS_MSG_HELO 100 +#define ECRYPTFS_MSG_QUIT 101 +#define ECRYPTFS_MSG_REQUEST 102 +#define ECRYPTFS_MSG_RESPONSE 103 + u8 type; + u32 index; + /* Counter converts to a sequence number. Each message sent + * out for which we expect a response has an associated + * sequence number. The response must have the same sequence + * number as the counter for the msg_stc for the message to be + * valid. */ + u32 counter; + size_t msg_size; + struct ecryptfs_message *msg; + struct task_struct *task; + struct list_head node; + struct list_head daemon_out_list; + struct mutex mux; +}; + +struct ecryptfs_daemon { +#define ECRYPTFS_DAEMON_IN_READ 0x00000001 +#define ECRYPTFS_DAEMON_IN_POLL 0x00000002 +#define ECRYPTFS_DAEMON_ZOMBIE 0x00000004 +#define ECRYPTFS_DAEMON_MISCDEV_OPEN 0x00000008 + u32 flags; + u32 num_queued_msg_ctx; + struct file *file; + struct mutex mux; + struct list_head msg_ctx_out_queue; + wait_queue_head_t wait; + struct hlist_node euid_chain; +}; + +#ifdef CONFIG_ECRYPT_FS_MESSAGING +extern struct mutex ecryptfs_daemon_hash_mux; +#endif + +static inline size_t +ecryptfs_lower_header_size(struct ecryptfs_crypt_stat *crypt_stat) +{ + if (crypt_stat->flags & ECRYPTFS_METADATA_IN_XATTR) + return 0; + return crypt_stat->metadata_size; +} + +static inline struct ecryptfs_file_info * +ecryptfs_file_to_private(struct file *file) +{ + return file->private_data; +} + +static inline void +ecryptfs_set_file_private(struct file *file, + struct ecryptfs_file_info *file_info) +{ + file->private_data = file_info; +} + +static inline struct file *ecryptfs_file_to_lower(struct file *file) +{ + return ((struct ecryptfs_file_info *)file->private_data)->wfi_file; +} + +static inline void +ecryptfs_set_file_lower(struct file *file, struct file *lower_file) +{ + ((struct ecryptfs_file_info *)file->private_data)->wfi_file = + lower_file; +} + +static inline struct ecryptfs_inode_info * +ecryptfs_inode_to_private(struct inode *inode) +{ + return container_of(inode, struct ecryptfs_inode_info, vfs_inode); +} + +static inline struct inode *ecryptfs_inode_to_lower(struct inode *inode) +{ + return ecryptfs_inode_to_private(inode)->wii_inode; +} + +static inline void +ecryptfs_set_inode_lower(struct inode *inode, struct inode *lower_inode) +{ + ecryptfs_inode_to_private(inode)->wii_inode = lower_inode; +} + +static inline struct ecryptfs_sb_info * +ecryptfs_superblock_to_private(struct super_block *sb) +{ + return (struct ecryptfs_sb_info *)sb->s_fs_info; +} + +static inline void +ecryptfs_set_superblock_private(struct super_block *sb, + struct ecryptfs_sb_info *sb_info) +{ + sb->s_fs_info = sb_info; +} + +static inline struct super_block * +ecryptfs_superblock_to_lower(struct super_block *sb) +{ + return ((struct ecryptfs_sb_info *)sb->s_fs_info)->wsi_sb; +} + +static inline void +ecryptfs_set_superblock_lower(struct super_block *sb, + struct super_block *lower_sb) +{ + ((struct ecryptfs_sb_info *)sb->s_fs_info)->wsi_sb = lower_sb; +} + +static inline void +ecryptfs_set_dentry_lower(struct dentry *dentry, + struct dentry *lower_dentry) +{ + dentry->d_fsdata = lower_dentry; +} + +static inline struct dentry * +ecryptfs_dentry_to_lower(struct dentry *dentry) +{ + return dentry->d_fsdata; +} + +static inline struct path +ecryptfs_lower_path(struct dentry *dentry) +{ + return (struct path){ + .mnt = ecryptfs_superblock_to_private(dentry->d_sb)->lower_mnt, + .dentry = ecryptfs_dentry_to_lower(dentry) + }; +} + +#define ecryptfs_printk(type, fmt, arg...) \ + __ecryptfs_printk(type "%s: " fmt, __func__, ## arg) +__printf(1, 2) +void __ecryptfs_printk(const char *fmt, ...); + +extern const struct file_operations ecryptfs_main_fops; +extern const struct file_operations ecryptfs_dir_fops; +extern const struct inode_operations ecryptfs_main_iops; +extern const struct inode_operations ecryptfs_dir_iops; +extern const struct inode_operations ecryptfs_symlink_iops; +extern const struct super_operations ecryptfs_sops; +extern const struct dentry_operations ecryptfs_dops; +extern const struct address_space_operations ecryptfs_aops; +extern int ecryptfs_verbosity; +extern unsigned int ecryptfs_message_buf_len; +extern signed long ecryptfs_message_wait_timeout; +extern unsigned int ecryptfs_number_of_users; + +extern struct kmem_cache *ecryptfs_auth_tok_list_item_cache; +extern struct kmem_cache *ecryptfs_file_info_cache; +extern struct kmem_cache *ecryptfs_inode_info_cache; +extern struct kmem_cache *ecryptfs_sb_info_cache; +extern struct kmem_cache *ecryptfs_header_cache; +extern struct kmem_cache *ecryptfs_xattr_cache; +extern struct kmem_cache *ecryptfs_key_record_cache; +extern struct kmem_cache *ecryptfs_key_sig_cache; +extern struct kmem_cache *ecryptfs_global_auth_tok_cache; +extern struct kmem_cache *ecryptfs_key_tfm_cache; + +struct inode *ecryptfs_get_inode(struct inode *lower_inode, + struct super_block *sb); +void ecryptfs_i_size_init(const char *page_virt, struct inode *inode); +int ecryptfs_initialize_file(struct dentry *ecryptfs_dentry, + struct inode *ecryptfs_inode); +int ecryptfs_decode_and_decrypt_filename(char **decrypted_name, + size_t *decrypted_name_size, + struct super_block *sb, + const char *name, size_t name_size); +int ecryptfs_encrypt_and_encode_filename( + char **encoded_name, + size_t *encoded_name_size, + struct ecryptfs_mount_crypt_stat *mount_crypt_stat, + const char *name, size_t name_size); +void ecryptfs_dump_hex(char *data, int bytes); +int virt_to_scatterlist(const void *addr, int size, struct scatterlist *sg, + int sg_size); +int ecryptfs_compute_root_iv(struct ecryptfs_crypt_stat *crypt_stat); +void ecryptfs_rotate_iv(unsigned char *iv); +void ecryptfs_init_crypt_stat(struct ecryptfs_crypt_stat *crypt_stat); +void ecryptfs_destroy_crypt_stat(struct ecryptfs_crypt_stat *crypt_stat); +void ecryptfs_destroy_mount_crypt_stat( + struct ecryptfs_mount_crypt_stat *mount_crypt_stat); +int ecryptfs_init_crypt_ctx(struct ecryptfs_crypt_stat *crypt_stat); +int ecryptfs_write_inode_size_to_metadata(struct inode *ecryptfs_inode); +int ecryptfs_encrypt_page(struct folio *folio); +int ecryptfs_decrypt_page(struct folio *folio); +int ecryptfs_write_metadata(struct dentry *ecryptfs_dentry, + struct inode *ecryptfs_inode); +int ecryptfs_read_metadata(struct dentry *ecryptfs_dentry); +int ecryptfs_new_file_context(struct inode *ecryptfs_inode); +void ecryptfs_write_crypt_stat_flags(char *page_virt, + struct ecryptfs_crypt_stat *crypt_stat, + size_t *written); +int ecryptfs_read_and_validate_header_region(struct inode *inode); +int ecryptfs_read_and_validate_xattr_region(struct dentry *dentry, + struct inode *inode); +u8 ecryptfs_code_for_cipher_string(char *cipher_name, size_t key_bytes); +int ecryptfs_cipher_code_to_string(char *str, size_t size, u8 cipher_code); +void ecryptfs_set_default_sizes(struct ecryptfs_crypt_stat *crypt_stat); +int ecryptfs_generate_key_packet_set(char *dest_base, + struct ecryptfs_crypt_stat *crypt_stat, + struct dentry *ecryptfs_dentry, + size_t *len, size_t max); +int +ecryptfs_parse_packet_set(struct ecryptfs_crypt_stat *crypt_stat, + unsigned char *src, size_t src_size, + struct dentry *ecryptfs_dentry); +int ecryptfs_truncate(struct dentry *dentry, loff_t new_length); +ssize_t +ecryptfs_getxattr_lower(struct dentry *lower_dentry, struct inode *lower_inode, + const char *name, void *value, size_t size); +int +ecryptfs_setxattr(struct dentry *dentry, struct inode *inode, const char *name, + const void *value, size_t size, int flags); +int ecryptfs_read_xattr_region(char *page_virt, struct inode *ecryptfs_inode); +#ifdef CONFIG_ECRYPT_FS_MESSAGING +int ecryptfs_process_response(struct ecryptfs_daemon *daemon, + struct ecryptfs_message *msg, u32 seq); +int ecryptfs_send_message(char *data, int data_len, + struct ecryptfs_msg_ctx **msg_ctx); +int ecryptfs_wait_for_response(struct ecryptfs_msg_ctx *msg_ctx, + struct ecryptfs_message **emsg); +int ecryptfs_init_messaging(void); +void ecryptfs_release_messaging(void); +#else +static inline int ecryptfs_init_messaging(void) +{ + return 0; +} +static inline void ecryptfs_release_messaging(void) +{ } +static inline int ecryptfs_send_message(char *data, int data_len, + struct ecryptfs_msg_ctx **msg_ctx) +{ + return -ENOTCONN; +} +static inline int ecryptfs_wait_for_response(struct ecryptfs_msg_ctx *msg_ctx, + struct ecryptfs_message **emsg) +{ + return -ENOMSG; +} +#endif + +void +ecryptfs_write_header_metadata(char *virt, + struct ecryptfs_crypt_stat *crypt_stat, + size_t *written); +int ecryptfs_add_keysig(struct ecryptfs_crypt_stat *crypt_stat, char *sig); +int +ecryptfs_add_global_auth_tok(struct ecryptfs_mount_crypt_stat *mount_crypt_stat, + char *sig, u32 global_auth_tok_flags); +int ecryptfs_get_global_auth_tok_for_sig( + struct ecryptfs_global_auth_tok **global_auth_tok, + struct ecryptfs_mount_crypt_stat *mount_crypt_stat, char *sig); +int +ecryptfs_add_new_key_tfm(struct ecryptfs_key_tfm **key_tfm, char *cipher_name, + size_t key_size); +int ecryptfs_init_crypto(void); +int ecryptfs_destroy_crypto(void); +int ecryptfs_tfm_exists(char *cipher_name, struct ecryptfs_key_tfm **key_tfm); +int ecryptfs_get_tfm_and_mutex_for_cipher_name(struct crypto_skcipher **tfm, + struct mutex **tfm_mutex, + char *cipher_name); +int ecryptfs_keyring_auth_tok_for_sig(struct key **auth_tok_key, + struct ecryptfs_auth_tok **auth_tok, + char *sig); +int ecryptfs_write_lower(struct inode *ecryptfs_inode, char *data, + loff_t offset, size_t size); +int ecryptfs_write_lower_page_segment(struct inode *ecryptfs_inode, + struct folio *folio_for_lower, + size_t offset_in_page, size_t size); +int ecryptfs_write(struct inode *inode, char *data, loff_t offset, size_t size); +int ecryptfs_read_lower(char *data, loff_t offset, size_t size, + struct inode *ecryptfs_inode); +int ecryptfs_read_lower_page_segment(struct folio *folio_for_ecryptfs, + pgoff_t page_index, + size_t offset_in_page, size_t size, + struct inode *ecryptfs_inode); +int ecryptfs_parse_packet_length(unsigned char *data, size_t *size, + size_t *length_size); +int ecryptfs_write_packet_length(char *dest, size_t size, + size_t *packet_size_length); +#ifdef CONFIG_ECRYPT_FS_MESSAGING +int ecryptfs_init_ecryptfs_miscdev(void); +void ecryptfs_destroy_ecryptfs_miscdev(void); +int ecryptfs_send_miscdev(char *data, size_t data_size, + struct ecryptfs_msg_ctx *msg_ctx, u8 msg_type, + u16 msg_flags, struct ecryptfs_daemon *daemon); +void ecryptfs_msg_ctx_alloc_to_free(struct ecryptfs_msg_ctx *msg_ctx); +int +ecryptfs_spawn_daemon(struct ecryptfs_daemon **daemon, struct file *file); +int ecryptfs_exorcise_daemon(struct ecryptfs_daemon *daemon); +int ecryptfs_find_daemon_by_euid(struct ecryptfs_daemon **daemon); +#endif +int ecryptfs_init_kthread(void); +void ecryptfs_destroy_kthread(void); +int ecryptfs_privileged_open(struct file **lower_file, + struct dentry *lower_dentry, + struct vfsmount *lower_mnt, + const struct cred *cred); +int ecryptfs_get_lower_file(struct dentry *dentry, struct inode *inode); +void ecryptfs_put_lower_file(struct inode *inode); +int +ecryptfs_write_tag_70_packet(char *dest, size_t *remaining_bytes, + size_t *packet_size, + struct ecryptfs_mount_crypt_stat *mount_crypt_stat, + char *filename, size_t filename_size); +int +ecryptfs_parse_tag_70_packet(char **filename, size_t *filename_size, + size_t *packet_size, + struct ecryptfs_mount_crypt_stat *mount_crypt_stat, + char *data, size_t max_packet_size); +int ecryptfs_set_f_namelen(long *namelen, long lower_namelen, + struct ecryptfs_mount_crypt_stat *mount_crypt_stat); +void ecryptfs_derive_iv(char *iv, struct ecryptfs_crypt_stat *crypt_stat, + loff_t offset); + +extern const struct xattr_handler * const ecryptfs_xattr_handlers[]; + +#endif /* #ifndef ECRYPTFS_KERNEL_H */ diff --git a/fs/ecryptfs/file.c b/fs/ecryptfs/file.c new file mode 100644 index 000000000..49b0fbe04 --- /dev/null +++ b/fs/ecryptfs/file.c @@ -0,0 +1,442 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * eCryptfs: Linux filesystem encryption layer + * + * Copyright (C) 1997-2004 Erez Zadok + * Copyright (C) 2001-2004 Stony Brook University + * Copyright (C) 2004-2007 International Business Machines Corp. + * Author(s): Michael A. Halcrow + * Michael C. Thompson + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "ecryptfs_kernel.h" + +/* + * ecryptfs_read_update_atime + * + * generic_file_read updates the atime of upper layer inode. But, it + * doesn't give us a chance to update the atime of the lower layer + * inode. This function is a wrapper to generic_file_read. It + * updates the atime of the lower level inode if generic_file_read + * returns without any errors. This is to be used only for file reads. + * The function to be used for directory reads is ecryptfs_read. + */ +static ssize_t ecryptfs_read_update_atime(struct kiocb *iocb, + struct iov_iter *to) +{ + ssize_t rc; + struct file *file = iocb->ki_filp; + + rc = generic_file_read_iter(iocb, to); + if (rc >= 0) { + struct path path = ecryptfs_lower_path(file->f_path.dentry); + touch_atime(&path); + } + return rc; +} + +/* + * ecryptfs_splice_read_update_atime + * + * filemap_splice_read updates the atime of upper layer inode. But, it + * doesn't give us a chance to update the atime of the lower layer inode. This + * function is a wrapper to generic_file_read. It updates the atime of the + * lower level inode if generic_file_read returns without any errors. This is + * to be used only for file reads. The function to be used for directory reads + * is ecryptfs_read. + */ +static ssize_t ecryptfs_splice_read_update_atime(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, + size_t len, unsigned int flags) +{ + ssize_t rc; + + rc = filemap_splice_read(in, ppos, pipe, len, flags); + if (rc >= 0) { + struct path path = ecryptfs_lower_path(in->f_path.dentry); + touch_atime(&path); + } + return rc; +} + +struct ecryptfs_getdents_callback { + struct dir_context ctx; + struct dir_context *caller; + struct super_block *sb; + int filldir_called; + int entries_written; +}; + +/* Inspired by generic filldir in fs/readdir.c */ +static bool +ecryptfs_filldir(struct dir_context *ctx, const char *lower_name, + int lower_namelen, loff_t offset, u64 ino, unsigned int d_type) +{ + struct ecryptfs_getdents_callback *buf = + container_of(ctx, struct ecryptfs_getdents_callback, ctx); + size_t name_size; + char *name; + int err; + bool res; + + buf->filldir_called++; + err = ecryptfs_decode_and_decrypt_filename(&name, &name_size, + buf->sb, lower_name, + lower_namelen); + if (err) { + if (err != -EINVAL) { + ecryptfs_printk(KERN_DEBUG, + "%s: Error attempting to decode and decrypt filename [%s]; rc = [%d]\n", + __func__, lower_name, err); + return false; + } + + /* Mask -EINVAL errors as these are most likely due a plaintext + * filename present in the lower filesystem despite filename + * encryption being enabled. One unavoidable example would be + * the "lost+found" dentry in the root directory of an Ext4 + * filesystem. + */ + return true; + } + + buf->caller->pos = buf->ctx.pos; + res = dir_emit(buf->caller, name, name_size, ino, d_type); + kfree(name); + if (res) + buf->entries_written++; + return res; +} + +/** + * ecryptfs_readdir + * @file: The eCryptfs directory file + * @ctx: The actor to feed the entries to + */ +static int ecryptfs_readdir(struct file *file, struct dir_context *ctx) +{ + int rc; + struct file *lower_file; + struct inode *inode = file_inode(file); + struct ecryptfs_getdents_callback buf = { + .ctx.actor = ecryptfs_filldir, + .caller = ctx, + .sb = inode->i_sb, + }; + lower_file = ecryptfs_file_to_lower(file); + rc = iterate_dir(lower_file, &buf.ctx); + ctx->pos = buf.ctx.pos; + if (rc >= 0 && (buf.entries_written || !buf.filldir_called)) + fsstack_copy_attr_atime(inode, file_inode(lower_file)); + return rc; +} + +struct kmem_cache *ecryptfs_file_info_cache; + +static int read_or_initialize_metadata(struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + struct ecryptfs_mount_crypt_stat *mount_crypt_stat; + struct ecryptfs_crypt_stat *crypt_stat; + int rc; + + crypt_stat = &ecryptfs_inode_to_private(inode)->crypt_stat; + mount_crypt_stat = &ecryptfs_superblock_to_private( + inode->i_sb)->mount_crypt_stat; + mutex_lock(&crypt_stat->cs_mutex); + + if (crypt_stat->flags & ECRYPTFS_POLICY_APPLIED && + crypt_stat->flags & ECRYPTFS_KEY_VALID) { + rc = 0; + goto out; + } + + rc = ecryptfs_read_metadata(dentry); + if (!rc) + goto out; + + if (mount_crypt_stat->flags & ECRYPTFS_PLAINTEXT_PASSTHROUGH_ENABLED) { + crypt_stat->flags &= ~(ECRYPTFS_I_SIZE_INITIALIZED + | ECRYPTFS_ENCRYPTED); + rc = 0; + goto out; + } + + if (!(mount_crypt_stat->flags & ECRYPTFS_XATTR_METADATA_ENABLED) && + !i_size_read(ecryptfs_inode_to_lower(inode))) { + rc = ecryptfs_initialize_file(dentry, inode); + if (!rc) + goto out; + } + + rc = -EIO; +out: + mutex_unlock(&crypt_stat->cs_mutex); + return rc; +} + +static int ecryptfs_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct file *lower_file = ecryptfs_file_to_lower(file); + /* + * Don't allow mmap on top of file systems that don't support it + * natively. If FILESYSTEM_MAX_STACK_DEPTH > 2 or ecryptfs + * allows recursive mounting, this will need to be extended. + */ + if (!can_mmap_file(lower_file)) + return -ENODEV; + return generic_file_mmap(file, vma); +} + +/** + * ecryptfs_open + * @inode: inode specifying file to open + * @file: Structure to return filled in + * + * Opens the file specified by inode. + * + * Returns zero on success; non-zero otherwise + */ +static int ecryptfs_open(struct inode *inode, struct file *file) +{ + int rc = 0; + struct ecryptfs_crypt_stat *crypt_stat = NULL; + struct dentry *ecryptfs_dentry = file->f_path.dentry; + /* Private value of ecryptfs_dentry allocated in + * ecryptfs_lookup() */ + struct ecryptfs_file_info *file_info; + + /* Released in ecryptfs_release or end of function if failure */ + file_info = kmem_cache_zalloc(ecryptfs_file_info_cache, GFP_KERNEL); + ecryptfs_set_file_private(file, file_info); + if (!file_info) { + ecryptfs_printk(KERN_ERR, + "Error attempting to allocate memory\n"); + rc = -ENOMEM; + goto out; + } + crypt_stat = &ecryptfs_inode_to_private(inode)->crypt_stat; + mutex_lock(&crypt_stat->cs_mutex); + if (!(crypt_stat->flags & ECRYPTFS_POLICY_APPLIED)) { + ecryptfs_printk(KERN_DEBUG, "Setting flags for stat...\n"); + /* Policy code enabled in future release */ + crypt_stat->flags |= (ECRYPTFS_POLICY_APPLIED + | ECRYPTFS_ENCRYPTED); + } + mutex_unlock(&crypt_stat->cs_mutex); + rc = ecryptfs_get_lower_file(ecryptfs_dentry, inode); + if (rc) { + printk(KERN_ERR "%s: Error attempting to initialize " + "the lower file for the dentry with name " + "[%pd]; rc = [%d]\n", __func__, + ecryptfs_dentry, rc); + goto out_free; + } + if ((ecryptfs_inode_to_private(inode)->lower_file->f_flags & O_ACCMODE) + == O_RDONLY && (file->f_flags & O_ACCMODE) != O_RDONLY) { + rc = -EPERM; + printk(KERN_WARNING "%s: Lower file is RO; eCryptfs " + "file must hence be opened RO\n", __func__); + goto out_put; + } + ecryptfs_set_file_lower( + file, ecryptfs_inode_to_private(inode)->lower_file); + rc = read_or_initialize_metadata(ecryptfs_dentry); + if (rc) + goto out_put; + ecryptfs_printk(KERN_DEBUG, "inode w/ addr = [0x%p], i_ino = " + "[0x%.16llx] size: [0x%.16llx]\n", inode, inode->i_ino, + (unsigned long long)i_size_read(inode)); + goto out; +out_put: + ecryptfs_put_lower_file(inode); +out_free: + kmem_cache_free(ecryptfs_file_info_cache, + ecryptfs_file_to_private(file)); +out: + return rc; +} + +/** + * ecryptfs_dir_open + * @inode: inode specifying file to open + * @file: Structure to return filled in + * + * Opens the file specified by inode. + * + * Returns zero on success; non-zero otherwise + */ +static int ecryptfs_dir_open(struct inode *inode, struct file *file) +{ + struct dentry *ecryptfs_dentry = file->f_path.dentry; + /* Private value of ecryptfs_dentry allocated in + * ecryptfs_lookup() */ + struct ecryptfs_file_info *file_info; + struct file *lower_file; + struct path path; + + /* Released in ecryptfs_release or end of function if failure */ + file_info = kmem_cache_zalloc(ecryptfs_file_info_cache, GFP_KERNEL); + ecryptfs_set_file_private(file, file_info); + if (unlikely(!file_info)) { + ecryptfs_printk(KERN_ERR, + "Error attempting to allocate memory\n"); + return -ENOMEM; + } + path = ecryptfs_lower_path(ecryptfs_dentry); + lower_file = dentry_open(&path, file->f_flags, current_cred()); + if (IS_ERR(lower_file)) { + printk(KERN_ERR "%s: Error attempting to initialize " + "the lower file for the dentry with name " + "[%pd]; rc = [%ld]\n", __func__, + ecryptfs_dentry, PTR_ERR(lower_file)); + kmem_cache_free(ecryptfs_file_info_cache, file_info); + return PTR_ERR(lower_file); + } + ecryptfs_set_file_lower(file, lower_file); + return 0; +} + +static int ecryptfs_flush(struct file *file, fl_owner_t td) +{ + struct file *lower_file = ecryptfs_file_to_lower(file); + + if (lower_file->f_op->flush) { + filemap_write_and_wait(file->f_mapping); + return lower_file->f_op->flush(lower_file, td); + } + + return 0; +} + +static int ecryptfs_release(struct inode *inode, struct file *file) +{ + ecryptfs_put_lower_file(inode); + kmem_cache_free(ecryptfs_file_info_cache, + ecryptfs_file_to_private(file)); + return 0; +} + +static int ecryptfs_dir_release(struct inode *inode, struct file *file) +{ + fput(ecryptfs_file_to_lower(file)); + kmem_cache_free(ecryptfs_file_info_cache, + ecryptfs_file_to_private(file)); + return 0; +} + +static loff_t ecryptfs_dir_llseek(struct file *file, loff_t offset, int whence) +{ + return vfs_llseek(ecryptfs_file_to_lower(file), offset, whence); +} + +static int +ecryptfs_fsync(struct file *file, loff_t start, loff_t end, int datasync) +{ + int rc; + + rc = file_write_and_wait(file); + if (rc) + return rc; + + return vfs_fsync(ecryptfs_file_to_lower(file), datasync); +} + +static int ecryptfs_fasync(int fd, struct file *file, int flag) +{ + int rc = 0; + struct file *lower_file = NULL; + + lower_file = ecryptfs_file_to_lower(file); + if (lower_file->f_op->fasync) + rc = lower_file->f_op->fasync(fd, lower_file, flag); + return rc; +} + +static long +ecryptfs_unlocked_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + struct file *lower_file = ecryptfs_file_to_lower(file); + long rc = -ENOTTY; + + if (!lower_file->f_op->unlocked_ioctl) + return rc; + + switch (cmd) { + case FITRIM: + case FS_IOC_GETFLAGS: + case FS_IOC_SETFLAGS: + case FS_IOC_GETVERSION: + case FS_IOC_SETVERSION: + rc = lower_file->f_op->unlocked_ioctl(lower_file, cmd, arg); + fsstack_copy_attr_all(file_inode(file), file_inode(lower_file)); + + return rc; + default: + return rc; + } +} + +#ifdef CONFIG_COMPAT +static long +ecryptfs_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + struct file *lower_file = ecryptfs_file_to_lower(file); + long rc = -ENOIOCTLCMD; + + if (!lower_file->f_op->compat_ioctl) + return rc; + + switch (cmd) { + case FITRIM: + case FS_IOC32_GETFLAGS: + case FS_IOC32_SETFLAGS: + case FS_IOC32_GETVERSION: + case FS_IOC32_SETVERSION: + rc = lower_file->f_op->compat_ioctl(lower_file, cmd, arg); + fsstack_copy_attr_all(file_inode(file), file_inode(lower_file)); + + return rc; + default: + return rc; + } +} +#endif + +const struct file_operations ecryptfs_dir_fops = { + .iterate_shared = ecryptfs_readdir, + .read = generic_read_dir, + .unlocked_ioctl = ecryptfs_unlocked_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = ecryptfs_compat_ioctl, +#endif + .open = ecryptfs_dir_open, + .release = ecryptfs_dir_release, + .fsync = ecryptfs_fsync, + .llseek = ecryptfs_dir_llseek, +}; + +const struct file_operations ecryptfs_main_fops = { + .llseek = generic_file_llseek, + .read_iter = ecryptfs_read_update_atime, + .write_iter = generic_file_write_iter, + .unlocked_ioctl = ecryptfs_unlocked_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = ecryptfs_compat_ioctl, +#endif + .mmap = ecryptfs_mmap, + .open = ecryptfs_open, + .flush = ecryptfs_flush, + .release = ecryptfs_release, + .fsync = ecryptfs_fsync, + .fasync = ecryptfs_fasync, + .splice_read = ecryptfs_splice_read_update_atime, +}; diff --git a/fs/ecryptfs/inode.c b/fs/ecryptfs/inode.c new file mode 100644 index 000000000..525297c7e --- /dev/null +++ b/fs/ecryptfs/inode.c @@ -0,0 +1,1183 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * eCryptfs: Linux filesystem encryption layer + * + * Copyright (C) 1997-2004 Erez Zadok + * Copyright (C) 2001-2004 Stony Brook University + * Copyright (C) 2004-2007 International Business Machines Corp. + * Author(s): Michael A. Halcrow + * Michael C. Thompsion + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ecryptfs_kernel.h" + +static struct dentry *ecryptfs_start_creating_dentry(struct dentry *dentry) +{ + struct dentry *parent = dget_parent(dentry); + struct dentry *ret; + + ret = start_creating_dentry(ecryptfs_dentry_to_lower(parent), + ecryptfs_dentry_to_lower(dentry)); + dput(parent); + return ret; +} + +static struct dentry *ecryptfs_start_removing_dentry(struct dentry *dentry) +{ + struct dentry *parent = dget_parent(dentry); + struct dentry *ret; + + ret = start_removing_dentry(ecryptfs_dentry_to_lower(parent), + ecryptfs_dentry_to_lower(dentry)); + dput(parent); + return ret; +} + +static int ecryptfs_inode_test(struct inode *inode, void *lower_inode) +{ + return ecryptfs_inode_to_lower(inode) == lower_inode; +} + +static int ecryptfs_inode_set(struct inode *inode, void *opaque) +{ + struct inode *lower_inode = opaque; + + ecryptfs_set_inode_lower(inode, lower_inode); + fsstack_copy_attr_all(inode, lower_inode); + /* i_size will be overwritten for encrypted regular files */ + fsstack_copy_inode_size(inode, lower_inode); + inode->i_ino = lower_inode->i_ino; + inode->i_mapping->a_ops = &ecryptfs_aops; + + if (S_ISLNK(inode->i_mode)) + inode->i_op = &ecryptfs_symlink_iops; + else if (S_ISDIR(inode->i_mode)) + inode->i_op = &ecryptfs_dir_iops; + else + inode->i_op = &ecryptfs_main_iops; + + if (S_ISDIR(inode->i_mode)) + inode->i_fop = &ecryptfs_dir_fops; + else if (special_file(inode->i_mode)) + init_special_inode(inode, inode->i_mode, inode->i_rdev); + else + inode->i_fop = &ecryptfs_main_fops; + + return 0; +} + +static struct inode *__ecryptfs_get_inode(struct inode *lower_inode, + struct super_block *sb) +{ + struct inode *inode; + + if (lower_inode->i_sb != ecryptfs_superblock_to_lower(sb)) + return ERR_PTR(-EXDEV); + + /* Reject dealing with casefold directories. */ + if (IS_CASEFOLDED(lower_inode)) { + pr_err_ratelimited("%s: Can't handle casefolded directory.\n", + __func__); + return ERR_PTR(-EREMOTE); + } + + if (!igrab(lower_inode)) + return ERR_PTR(-ESTALE); + inode = iget5_locked(sb, (unsigned long)lower_inode, + ecryptfs_inode_test, ecryptfs_inode_set, + lower_inode); + if (!inode) { + iput(lower_inode); + return ERR_PTR(-EACCES); + } + if (!(inode_state_read_once(inode) & I_NEW)) + iput(lower_inode); + + return inode; +} + +struct inode *ecryptfs_get_inode(struct inode *lower_inode, + struct super_block *sb) +{ + struct inode *inode = __ecryptfs_get_inode(lower_inode, sb); + + if (!IS_ERR(inode) && (inode_state_read_once(inode) & I_NEW)) + unlock_new_inode(inode); + + return inode; +} + +/** + * ecryptfs_interpose + * @lower_dentry: Existing dentry in the lower filesystem + * @dentry: ecryptfs' dentry + * @sb: ecryptfs's super_block + * + * Interposes upper and lower dentries. + * + * Returns zero on success; non-zero otherwise + */ +static int ecryptfs_interpose(struct dentry *lower_dentry, + struct dentry *dentry, struct super_block *sb) +{ + struct inode *inode = ecryptfs_get_inode(d_inode(lower_dentry), sb); + + if (IS_ERR(inode)) + return PTR_ERR(inode); + d_instantiate(dentry, inode); + + return 0; +} + +static int ecryptfs_do_unlink(struct inode *dir, struct dentry *dentry, + struct inode *inode) +{ + struct dentry *lower_dentry; + struct inode *lower_dir; + int rc; + + lower_dentry = ecryptfs_start_removing_dentry(dentry); + if (IS_ERR(lower_dentry)) + return PTR_ERR(lower_dentry); + + lower_dir = lower_dentry->d_parent->d_inode; + rc = vfs_unlink(&nop_mnt_idmap, lower_dir, lower_dentry, NULL); + if (rc) { + printk(KERN_ERR "Error in vfs_unlink; rc = [%d]\n", rc); + goto out_unlock; + } + fsstack_copy_attr_times(dir, lower_dir); + set_nlink(inode, ecryptfs_inode_to_lower(inode)->i_nlink); + inode_set_ctime_to_ts(inode, inode_get_ctime(dir)); +out_unlock: + end_removing(lower_dentry); + if (!rc) + d_drop(dentry); + return rc; +} + +/** + * ecryptfs_do_create + * @directory_inode: inode of the new file's dentry's parent in ecryptfs + * @ecryptfs_dentry: New file's dentry in ecryptfs + * @mode: The mode of the new file + * + * Creates the underlying file and the eCryptfs inode which will link to + * it. It will also update the eCryptfs directory inode to mimic the + * stat of the lower directory inode. + * + * Returns the new eCryptfs inode on success; an ERR_PTR on error condition + */ +static struct inode * +ecryptfs_do_create(struct inode *directory_inode, + struct dentry *ecryptfs_dentry, umode_t mode) +{ + int rc; + struct dentry *lower_dentry; + struct inode *lower_dir; + struct inode *inode; + + lower_dentry = ecryptfs_start_creating_dentry(ecryptfs_dentry); + if (IS_ERR(lower_dentry)) + return ERR_CAST(lower_dentry); + lower_dir = lower_dentry->d_parent->d_inode; + rc = vfs_create(&nop_mnt_idmap, lower_dentry, mode, NULL); + if (rc) { + printk(KERN_ERR "%s: Failure to create dentry in lower fs; " + "rc = [%d]\n", __func__, rc); + inode = ERR_PTR(rc); + goto out_lock; + } + inode = __ecryptfs_get_inode(d_inode(lower_dentry), + directory_inode->i_sb); + if (IS_ERR(inode)) { + vfs_unlink(&nop_mnt_idmap, lower_dir, lower_dentry, NULL); + goto out_lock; + } + fsstack_copy_attr_times(directory_inode, lower_dir); + fsstack_copy_inode_size(directory_inode, lower_dir); +out_lock: + end_creating(lower_dentry); + return inode; +} + +/* + * ecryptfs_initialize_file + * + * Cause the file to be changed from a basic empty file to an ecryptfs + * file with a header and first data page. + * + * Returns zero on success + */ +int ecryptfs_initialize_file(struct dentry *ecryptfs_dentry, + struct inode *ecryptfs_inode) +{ + struct ecryptfs_crypt_stat *crypt_stat = + &ecryptfs_inode_to_private(ecryptfs_inode)->crypt_stat; + int rc = 0; + + if (S_ISDIR(ecryptfs_inode->i_mode)) { + ecryptfs_printk(KERN_DEBUG, "This is a directory\n"); + crypt_stat->flags &= ~(ECRYPTFS_ENCRYPTED); + goto out; + } + ecryptfs_printk(KERN_DEBUG, "Initializing crypto context\n"); + rc = ecryptfs_new_file_context(ecryptfs_inode); + if (rc) { + ecryptfs_printk(KERN_ERR, "Error creating new file " + "context; rc = [%d]\n", rc); + goto out; + } + rc = ecryptfs_get_lower_file(ecryptfs_dentry, ecryptfs_inode); + if (rc) { + printk(KERN_ERR "%s: Error attempting to initialize " + "the lower file for the dentry with name " + "[%pd]; rc = [%d]\n", __func__, + ecryptfs_dentry, rc); + goto out; + } + rc = ecryptfs_write_metadata(ecryptfs_dentry, ecryptfs_inode); + if (rc) + printk(KERN_ERR "Error writing headers; rc = [%d]\n", rc); + ecryptfs_put_lower_file(ecryptfs_inode); +out: + return rc; +} + +/* + * ecryptfs_create + * @mode: The mode of the new file. + * + * Creates a new file. + * + * Returns zero on success; non-zero on error condition + */ +static int +ecryptfs_create(struct mnt_idmap *idmap, + struct inode *directory_inode, struct dentry *ecryptfs_dentry, + umode_t mode) +{ + struct inode *ecryptfs_inode; + int rc; + + ecryptfs_inode = ecryptfs_do_create(directory_inode, ecryptfs_dentry, + mode); + if (IS_ERR(ecryptfs_inode)) { + ecryptfs_printk(KERN_WARNING, "Failed to create file in" + "lower filesystem\n"); + rc = PTR_ERR(ecryptfs_inode); + goto out; + } + /* At this point, a file exists on "disk"; we need to make sure + * that this on disk file is prepared to be an ecryptfs file */ + rc = ecryptfs_initialize_file(ecryptfs_dentry, ecryptfs_inode); + if (rc) { + ecryptfs_do_unlink(directory_inode, ecryptfs_dentry, + ecryptfs_inode); + iget_failed(ecryptfs_inode); + goto out; + } + d_instantiate_new(ecryptfs_dentry, ecryptfs_inode); +out: + return rc; +} + +static int ecryptfs_i_size_read(struct dentry *dentry, struct inode *inode) +{ + struct ecryptfs_crypt_stat *crypt_stat; + int rc; + + rc = ecryptfs_get_lower_file(dentry, inode); + if (rc) { + printk(KERN_ERR "%s: Error attempting to initialize " + "the lower file for the dentry with name " + "[%pd]; rc = [%d]\n", __func__, + dentry, rc); + return rc; + } + + crypt_stat = &ecryptfs_inode_to_private(inode)->crypt_stat; + /* TODO: lock for crypt_stat comparison */ + if (!(crypt_stat->flags & ECRYPTFS_POLICY_APPLIED)) + ecryptfs_set_default_sizes(crypt_stat); + + rc = ecryptfs_read_and_validate_header_region(inode); + ecryptfs_put_lower_file(inode); + if (rc) { + rc = ecryptfs_read_and_validate_xattr_region(dentry, inode); + if (!rc) + crypt_stat->flags |= ECRYPTFS_METADATA_IN_XATTR; + } + + /* Must return 0 to allow non-eCryptfs files to be looked up, too */ + return 0; +} + +/* + * ecryptfs_lookup_interpose - Dentry interposition for a lookup + */ +static struct dentry *ecryptfs_lookup_interpose(struct dentry *dentry, + struct dentry *lower_dentry) +{ + struct dentry *lower_parent = ecryptfs_dentry_to_lower(dentry->d_parent); + struct inode *inode, *lower_inode; + int rc = 0; + + fsstack_copy_attr_atime(d_inode(dentry->d_parent), + d_inode(lower_parent)); + BUG_ON(!d_count(lower_dentry)); + + ecryptfs_set_dentry_lower(dentry, lower_dentry); + + /* + * negative dentry can go positive under us here - its parent is not + * locked. That's OK and that could happen just as we return from + * ecryptfs_lookup() anyway. Just need to be careful and fetch + * ->d_inode only once - it's not stable here. + */ + lower_inode = READ_ONCE(lower_dentry->d_inode); + + if (!lower_inode) /* We want to add because we couldn't find in lower */ + return d_splice_alias(NULL, dentry); + + inode = __ecryptfs_get_inode(lower_inode, dentry->d_sb); + if (IS_ERR(inode)) { + printk(KERN_ERR "%s: Error interposing; rc = [%ld]\n", + __func__, PTR_ERR(inode)); + return ERR_CAST(inode); + } + if (S_ISREG(inode->i_mode)) { + rc = ecryptfs_i_size_read(dentry, inode); + if (rc) { + make_bad_inode(inode); + return ERR_PTR(rc); + } + } + + if (inode_state_read_once(inode) & I_NEW) + unlock_new_inode(inode); + return d_splice_alias(inode, dentry); +} + +/** + * ecryptfs_lookup + * @ecryptfs_dir_inode: The eCryptfs directory inode + * @ecryptfs_dentry: The eCryptfs dentry that we are looking up + * @flags: lookup flags + * + * Find a file on disk. If the file does not exist, then we'll add it to the + * dentry cache and continue on to read it from the disk. + */ +static struct dentry *ecryptfs_lookup(struct inode *ecryptfs_dir_inode, + struct dentry *ecryptfs_dentry, + unsigned int flags) +{ + char *encrypted_and_encoded_name = NULL; + struct ecryptfs_mount_crypt_stat *mount_crypt_stat; + struct dentry *lower_dir_dentry, *lower_dentry; + struct qstr qname = QSTR_INIT(ecryptfs_dentry->d_name.name, + ecryptfs_dentry->d_name.len); + struct dentry *res; + int rc = 0; + + lower_dir_dentry = ecryptfs_dentry_to_lower(ecryptfs_dentry->d_parent); + + mount_crypt_stat = &ecryptfs_superblock_to_private( + ecryptfs_dentry->d_sb)->mount_crypt_stat; + if (mount_crypt_stat->flags & ECRYPTFS_GLOBAL_ENCRYPT_FILENAMES) { + size_t len = qname.len; + rc = ecryptfs_encrypt_and_encode_filename( + &encrypted_and_encoded_name, &len, + mount_crypt_stat, qname.name, len); + if (rc) { + printk(KERN_ERR "%s: Error attempting to encrypt and encode " + "filename; rc = [%d]\n", __func__, rc); + return ERR_PTR(rc); + } + qname.name = encrypted_and_encoded_name; + qname.len = len; + } + + lower_dentry = lookup_noperm_unlocked(&qname, lower_dir_dentry); + if (IS_ERR(lower_dentry)) { + ecryptfs_printk(KERN_DEBUG, "%s: lookup_noperm() returned " + "[%ld] on lower_dentry = [%s]\n", __func__, + PTR_ERR(lower_dentry), + qname.name); + res = ERR_CAST(lower_dentry); + } else { + res = ecryptfs_lookup_interpose(ecryptfs_dentry, lower_dentry); + } + kfree(encrypted_and_encoded_name); + return res; +} + +static int ecryptfs_link(struct dentry *old_dentry, struct inode *dir, + struct dentry *new_dentry) +{ + struct dentry *lower_old_dentry; + struct dentry *lower_new_dentry; + struct inode *lower_dir; + u64 file_size_save; + int rc; + + file_size_save = i_size_read(d_inode(old_dentry)); + lower_old_dentry = ecryptfs_dentry_to_lower(old_dentry); + lower_new_dentry = ecryptfs_start_creating_dentry(new_dentry); + if (IS_ERR(lower_new_dentry)) + return PTR_ERR(lower_new_dentry); + lower_dir = lower_new_dentry->d_parent->d_inode; + rc = vfs_link(lower_old_dentry, &nop_mnt_idmap, lower_dir, + lower_new_dentry, NULL); + if (rc || d_really_is_negative(lower_new_dentry)) + goto out_lock; + rc = ecryptfs_interpose(lower_new_dentry, new_dentry, dir->i_sb); + if (rc) + goto out_lock; + fsstack_copy_attr_times(dir, lower_dir); + fsstack_copy_inode_size(dir, lower_dir); + set_nlink(d_inode(old_dentry), + ecryptfs_inode_to_lower(d_inode(old_dentry))->i_nlink); + i_size_write(d_inode(new_dentry), file_size_save); +out_lock: + end_creating(lower_new_dentry); + return rc; +} + +static int ecryptfs_unlink(struct inode *dir, struct dentry *dentry) +{ + return ecryptfs_do_unlink(dir, dentry, d_inode(dentry)); +} + +static int ecryptfs_symlink(struct mnt_idmap *idmap, + struct inode *dir, struct dentry *dentry, + const char *symname) +{ + int rc; + struct dentry *lower_dentry; + struct inode *lower_dir; + char *encoded_symname; + size_t encoded_symlen; + struct ecryptfs_mount_crypt_stat *mount_crypt_stat = NULL; + + lower_dentry = ecryptfs_start_creating_dentry(dentry); + if (IS_ERR(lower_dentry)) + return PTR_ERR(lower_dentry); + lower_dir = lower_dentry->d_parent->d_inode; + + mount_crypt_stat = &ecryptfs_superblock_to_private( + dir->i_sb)->mount_crypt_stat; + rc = ecryptfs_encrypt_and_encode_filename(&encoded_symname, + &encoded_symlen, + mount_crypt_stat, symname, + strlen(symname)); + if (rc) + goto out_lock; + rc = vfs_symlink(&nop_mnt_idmap, lower_dir, lower_dentry, + encoded_symname, NULL); + kfree(encoded_symname); + if (rc || d_really_is_negative(lower_dentry)) + goto out_lock; + rc = ecryptfs_interpose(lower_dentry, dentry, dir->i_sb); + if (rc) + goto out_lock; + fsstack_copy_attr_times(dir, lower_dir); + fsstack_copy_inode_size(dir, lower_dir); +out_lock: + end_creating(lower_dentry); + if (d_really_is_negative(dentry)) + d_drop(dentry); + return rc; +} + +static struct dentry *ecryptfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + int rc; + struct dentry *lower_dentry; + struct dentry *lower_dir_dentry; + struct inode *lower_dir; + + lower_dentry = ecryptfs_start_creating_dentry(dentry); + if (IS_ERR(lower_dentry)) + return lower_dentry; + lower_dir_dentry = dget(lower_dentry->d_parent); + lower_dir = lower_dir_dentry->d_inode; + lower_dentry = vfs_mkdir(&nop_mnt_idmap, lower_dir, + lower_dentry, mode, NULL); + rc = PTR_ERR(lower_dentry); + if (IS_ERR(lower_dentry)) + goto out; + rc = 0; + if (d_unhashed(lower_dentry)) + goto out; + rc = ecryptfs_interpose(lower_dentry, dentry, dir->i_sb); + if (rc) + goto out; + fsstack_copy_attr_times(dir, lower_dir); + fsstack_copy_inode_size(dir, lower_dir); + set_nlink(dir, lower_dir->i_nlink); +out: + dput(lower_dir_dentry); + end_creating(lower_dentry); + if (d_really_is_negative(dentry)) + d_drop(dentry); + return ERR_PTR(rc); +} + +static int ecryptfs_rmdir(struct inode *dir, struct dentry *dentry) +{ + struct dentry *lower_dentry; + struct inode *lower_dir; + int rc; + + lower_dentry = ecryptfs_start_removing_dentry(dentry); + if (IS_ERR(lower_dentry)) + return PTR_ERR(lower_dentry); + lower_dir = lower_dentry->d_parent->d_inode; + + rc = vfs_rmdir(&nop_mnt_idmap, lower_dir, lower_dentry, NULL); + if (!rc) { + clear_nlink(d_inode(dentry)); + fsstack_copy_attr_times(dir, lower_dir); + set_nlink(dir, lower_dir->i_nlink); + } + end_removing(lower_dentry); + if (!rc) + d_drop(dentry); + return rc; +} + +static int +ecryptfs_mknod(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t dev) +{ + int rc; + struct dentry *lower_dentry; + struct inode *lower_dir; + + lower_dentry = ecryptfs_start_creating_dentry(dentry); + if (IS_ERR(lower_dentry)) + return PTR_ERR(lower_dentry); + lower_dir = lower_dentry->d_parent->d_inode; + + rc = vfs_mknod(&nop_mnt_idmap, lower_dir, lower_dentry, mode, dev, NULL); + if (rc || d_really_is_negative(lower_dentry)) + goto out; + rc = ecryptfs_interpose(lower_dentry, dentry, dir->i_sb); + if (rc) + goto out; + fsstack_copy_attr_times(dir, lower_dir); + fsstack_copy_inode_size(dir, lower_dir); +out: + end_creating(lower_dentry); + if (d_really_is_negative(dentry)) + d_drop(dentry); + return rc; +} + +static int +ecryptfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) +{ + int rc; + struct dentry *lower_old_dentry; + struct dentry *lower_new_dentry; + struct dentry *lower_old_dir_dentry; + struct dentry *lower_new_dir_dentry; + struct inode *target_inode; + struct renamedata rd = {}; + + if (flags) + return -EINVAL; + + lower_old_dir_dentry = ecryptfs_dentry_to_lower(old_dentry->d_parent); + lower_new_dir_dentry = ecryptfs_dentry_to_lower(new_dentry->d_parent); + + lower_old_dentry = ecryptfs_dentry_to_lower(old_dentry); + lower_new_dentry = ecryptfs_dentry_to_lower(new_dentry); + + target_inode = d_inode(new_dentry); + + rd.mnt_idmap = &nop_mnt_idmap; + rd.old_parent = lower_old_dir_dentry; + rd.new_parent = lower_new_dir_dentry; + rc = start_renaming_two_dentries(&rd, lower_old_dentry, lower_new_dentry); + if (rc) + return rc; + + rc = vfs_rename(&rd); + if (rc) + goto out_lock; + if (target_inode) + fsstack_copy_attr_all(target_inode, + ecryptfs_inode_to_lower(target_inode)); + fsstack_copy_attr_all(new_dir, d_inode(lower_new_dir_dentry)); + if (new_dir != old_dir) + fsstack_copy_attr_all(old_dir, d_inode(lower_old_dir_dentry)); +out_lock: + end_renaming(&rd); + return rc; +} + +static char *ecryptfs_readlink_lower(struct dentry *dentry, size_t *bufsiz) +{ + DEFINE_DELAYED_CALL(done); + struct dentry *lower_dentry = ecryptfs_dentry_to_lower(dentry); + const char *link; + char *buf; + int rc; + + link = vfs_get_link(lower_dentry, &done); + if (IS_ERR(link)) + return ERR_CAST(link); + + rc = ecryptfs_decode_and_decrypt_filename(&buf, bufsiz, dentry->d_sb, + link, strlen(link)); + do_delayed_call(&done); + if (rc) + return ERR_PTR(rc); + + return buf; +} + +static const char *ecryptfs_get_link(struct dentry *dentry, + struct inode *inode, + struct delayed_call *done) +{ + size_t len; + char *buf; + + if (!dentry) + return ERR_PTR(-ECHILD); + + buf = ecryptfs_readlink_lower(dentry, &len); + if (IS_ERR(buf)) + return buf; + fsstack_copy_attr_atime(d_inode(dentry), + d_inode(ecryptfs_dentry_to_lower(dentry))); + buf[len] = '\0'; + set_delayed_call(done, kfree_link, buf); + return buf; +} + +static void ecryptfs_iattr_to_lower(struct iattr *lower_ia, + const struct iattr *ia) +{ + memcpy(lower_ia, ia, sizeof(*lower_ia)); + if (ia->ia_valid & ATTR_FILE) + lower_ia->ia_file = ecryptfs_file_to_lower(ia->ia_file); + /* + * If the mode change is for clearing setuid/setgid bits, allow the lower + * file system to interpret this in its own way. + */ + if (lower_ia->ia_valid & (ATTR_KILL_SUID | ATTR_KILL_SGID)) + lower_ia->ia_valid &= ~ATTR_MODE; +} + +/** + * upper_size_to_lower_size + * @crypt_stat: Crypt_stat associated with file + * @upper_size: Size of the upper file + * + * Calculate the required size of the lower file based on the + * specified size of the upper file. This calculation is based on the + * number of headers in the underlying file and the extent size. + * + * Returns Calculated size of the lower file. + */ +static loff_t +upper_size_to_lower_size(struct ecryptfs_crypt_stat *crypt_stat, + loff_t upper_size) +{ + loff_t lower_size; + + lower_size = ecryptfs_lower_header_size(crypt_stat); + if (upper_size != 0) { + loff_t num_extents; + + num_extents = upper_size >> crypt_stat->extent_shift; + if (upper_size & ~crypt_stat->extent_mask) + num_extents++; + lower_size += (num_extents * crypt_stat->extent_size); + } + return lower_size; +} + +/** + * __ecryptfs_truncate + * @dentry: The ecryptfs layer dentry + * @ia: Address of the ecryptfs inode's attributes + * + * Handle truncations modifying the size of the file. Note that the file sizes + * are interpolated. When expanding, we are simply writing strings of 0's out. + * When truncating, we truncate the upper inode and update the lower_ia + * according to the page index interpolations. + * + * Returns zero on success; non-zero otherwise + */ +static int __ecryptfs_truncate(struct dentry *dentry, const struct iattr *ia) +{ + struct dentry *lower_dentry = ecryptfs_dentry_to_lower(dentry); + struct inode *inode = d_inode(dentry); + struct ecryptfs_crypt_stat *crypt_stat; + loff_t i_size = i_size_read(inode); + loff_t lower_size_before_truncate; + loff_t lower_size_after_truncate; + struct iattr lower_ia; + size_t num_zeros; + int rc; + + ecryptfs_iattr_to_lower(&lower_ia, ia); + + if (unlikely((ia->ia_size == i_size))) + return 0; + + crypt_stat = &ecryptfs_inode_to_private(inode)->crypt_stat; + lower_size_before_truncate = + upper_size_to_lower_size(crypt_stat, i_size); + lower_size_after_truncate = + upper_size_to_lower_size(crypt_stat, ia->ia_size); + if (lower_size_after_truncate > lower_size_before_truncate) { + /* + * The eCryptfs inode and the new *lower* size are mixed here + * because we may not have the lower i_mutex held and/or it may + * not be appropriate to call inode_newsize_ok() with inodes + * from other filesystems. + */ + rc = inode_newsize_ok(inode, lower_size_after_truncate); + if (rc) + return rc; + } + + rc = ecryptfs_get_lower_file(dentry, inode); + if (rc) + return rc; + + if (ia->ia_size > i_size) { + char zero[] = { 0x00 }; + + /* + * Write a single 0 at the last position of the file; this + * triggers code that will fill in 0's throughout the + * intermediate portion of the previous end of the file and the + * new end of the file. + */ + rc = ecryptfs_write(inode, zero, ia->ia_size - 1, 1); + goto out; + } + + if (!(crypt_stat->flags & ECRYPTFS_ENCRYPTED)) { + truncate_setsize(inode, ia->ia_size); + lower_ia.ia_size = ia->ia_size; + goto set_size; + } + + /* + * We're chopping off all the pages down to the page in which + * ia->ia_size is located. Fill in the end of that page from + * (ia->ia_size & ~PAGE_MASK) to PAGE_SIZE with zeros. + */ + num_zeros = PAGE_SIZE - (ia->ia_size & ~PAGE_MASK); + if (num_zeros) { + rc = ecryptfs_write(inode, page_address(ZERO_PAGE(0)), + ia->ia_size, num_zeros); + if (rc) { + pr_err("Error attempting to zero out the remainder of the end page on reducing truncate; rc = [%d]\n", + rc); + goto out; + } + } + truncate_setsize(inode, ia->ia_size); + rc = ecryptfs_write_inode_size_to_metadata(inode); + if (rc) { + pr_err("Problem with ecryptfs_write_inode_size_to_metadata; rc = [%d]\n", + rc); + goto out; + } + + /* + * We are reducing the size of the ecryptfs file, and need to know if we + * need to reduce the size of the lower file. + */ + if (lower_size_after_truncate >= lower_size_before_truncate) + goto out; + + lower_ia.ia_size = lower_size_after_truncate; +set_size: + lower_ia.ia_valid |= ATTR_SIZE; + inode_lock(d_inode(lower_dentry)); + rc = notify_change(&nop_mnt_idmap, lower_dentry, &lower_ia, NULL); + inode_unlock(d_inode(lower_dentry)); +out: + ecryptfs_put_lower_file(inode); + return rc; +} + +/** + * ecryptfs_truncate + * @dentry: The ecryptfs layer dentry + * @new_length: The length to expand the file to + * + * Simple function that handles the truncation of an eCryptfs inode and + * its corresponding lower inode. + * + * Returns zero on success; non-zero otherwise + */ +int ecryptfs_truncate(struct dentry *dentry, loff_t new_length) +{ + const struct iattr ia = { + .ia_valid = ATTR_SIZE, + .ia_size = new_length, + }; + + return __ecryptfs_truncate(dentry, &ia); +} + +static int +ecryptfs_permission(struct mnt_idmap *idmap, struct inode *inode, + int mask) +{ + return inode_permission(&nop_mnt_idmap, + ecryptfs_inode_to_lower(inode), mask); +} + +/** + * ecryptfs_setattr + * @idmap: idmap of the target mount + * @dentry: dentry handle to the inode to modify + * @ia: Structure with flags of what to change and values + * + * Updates the metadata of an inode. If the update is to the size + * i.e. truncation, then ecryptfs_truncate will handle the size modification + * of both the ecryptfs inode and the lower inode. + * + * All other metadata changes will be passed right to the lower filesystem, + * and we will just update our inode to look like the lower. + */ +static int ecryptfs_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, struct iattr *ia) +{ + struct inode *inode = d_inode(dentry); + struct dentry *lower_dentry = ecryptfs_dentry_to_lower(dentry); + struct inode *lower_inode = ecryptfs_inode_to_lower(inode); + struct ecryptfs_crypt_stat *crypt_stat; + int rc; + + crypt_stat = &ecryptfs_inode_to_private(d_inode(dentry))->crypt_stat; + if (!(crypt_stat->flags & ECRYPTFS_STRUCT_INITIALIZED)) + ecryptfs_init_crypt_stat(crypt_stat); + + mutex_lock(&crypt_stat->cs_mutex); + if (d_is_dir(dentry)) + crypt_stat->flags &= ~(ECRYPTFS_ENCRYPTED); + else if (d_is_reg(dentry) && + (!(crypt_stat->flags & ECRYPTFS_POLICY_APPLIED) || + !(crypt_stat->flags & ECRYPTFS_KEY_VALID))) { + struct ecryptfs_mount_crypt_stat *mount_crypt_stat; + + mount_crypt_stat = &ecryptfs_superblock_to_private( + dentry->d_sb)->mount_crypt_stat; + rc = ecryptfs_get_lower_file(dentry, inode); + if (rc) { + mutex_unlock(&crypt_stat->cs_mutex); + goto out; + } + rc = ecryptfs_read_metadata(dentry); + ecryptfs_put_lower_file(inode); + if (rc) { + if (!(mount_crypt_stat->flags & + ECRYPTFS_PLAINTEXT_PASSTHROUGH_ENABLED)) { + rc = -EIO; + printk(KERN_WARNING "Either the lower file " + "is not in a valid eCryptfs format, " + "or the key could not be retrieved. " + "Plaintext passthrough mode is not " + "enabled; returning -EIO\n"); + mutex_unlock(&crypt_stat->cs_mutex); + goto out; + } + rc = 0; + crypt_stat->flags &= ~(ECRYPTFS_I_SIZE_INITIALIZED + | ECRYPTFS_ENCRYPTED); + } + } + mutex_unlock(&crypt_stat->cs_mutex); + + rc = setattr_prepare(&nop_mnt_idmap, dentry, ia); + if (rc) + goto out; + + if (ia->ia_valid & ATTR_SIZE) { + rc = __ecryptfs_truncate(dentry, ia); + } else { + struct iattr lower_ia; + + ecryptfs_iattr_to_lower(&lower_ia, ia); + + inode_lock(d_inode(lower_dentry)); + rc = notify_change(&nop_mnt_idmap, lower_dentry, &lower_ia, + NULL); + inode_unlock(d_inode(lower_dentry)); + } +out: + fsstack_copy_attr_all(inode, lower_inode); + return rc; +} + +static int ecryptfs_getattr_link(struct mnt_idmap *idmap, + const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int flags) +{ + struct dentry *dentry = path->dentry; + struct ecryptfs_mount_crypt_stat *mount_crypt_stat; + int rc = 0; + + mount_crypt_stat = &ecryptfs_superblock_to_private( + dentry->d_sb)->mount_crypt_stat; + generic_fillattr(&nop_mnt_idmap, request_mask, d_inode(dentry), stat); + if (mount_crypt_stat->flags & ECRYPTFS_GLOBAL_ENCRYPT_FILENAMES) { + char *target; + size_t targetsiz; + + target = ecryptfs_readlink_lower(dentry, &targetsiz); + if (!IS_ERR(target)) { + kfree(target); + stat->size = targetsiz; + } else { + rc = PTR_ERR(target); + } + } + return rc; +} + +static int ecryptfs_getattr(struct mnt_idmap *idmap, + const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int flags) +{ + struct dentry *dentry = path->dentry; + struct kstat lower_stat; + struct path lower_path = ecryptfs_lower_path(dentry); + int rc; + + rc = vfs_getattr_nosec(&lower_path, &lower_stat, request_mask, flags); + if (!rc) { + fsstack_copy_attr_all(d_inode(dentry), + ecryptfs_inode_to_lower(d_inode(dentry))); + generic_fillattr(&nop_mnt_idmap, request_mask, + d_inode(dentry), stat); + stat->blocks = lower_stat.blocks; + } + return rc; +} + +int +ecryptfs_setxattr(struct dentry *dentry, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +{ + int rc; + struct dentry *lower_dentry; + struct inode *lower_inode; + + lower_dentry = ecryptfs_dentry_to_lower(dentry); + lower_inode = d_inode(lower_dentry); + if (!(lower_inode->i_opflags & IOP_XATTR)) { + rc = -EOPNOTSUPP; + goto out; + } + inode_lock(lower_inode); + rc = __vfs_setxattr_locked(&nop_mnt_idmap, lower_dentry, name, value, size, flags, NULL); + inode_unlock(lower_inode); + if (!rc && inode) + fsstack_copy_attr_all(inode, lower_inode); +out: + return rc; +} + +ssize_t +ecryptfs_getxattr_lower(struct dentry *lower_dentry, struct inode *lower_inode, + const char *name, void *value, size_t size) +{ + int rc; + + if (!(lower_inode->i_opflags & IOP_XATTR)) { + rc = -EOPNOTSUPP; + goto out; + } + inode_lock(lower_inode); + rc = __vfs_getxattr(lower_dentry, lower_inode, name, value, size); + inode_unlock(lower_inode); +out: + return rc; +} + +static ssize_t +ecryptfs_getxattr(struct dentry *dentry, struct inode *inode, + const char *name, void *value, size_t size) +{ + return ecryptfs_getxattr_lower(ecryptfs_dentry_to_lower(dentry), + ecryptfs_inode_to_lower(inode), + name, value, size); +} + +static ssize_t +ecryptfs_listxattr(struct dentry *dentry, char *list, size_t size) +{ + int rc = 0; + struct dentry *lower_dentry; + + lower_dentry = ecryptfs_dentry_to_lower(dentry); + if (!d_inode(lower_dentry)->i_op->listxattr) { + rc = -EOPNOTSUPP; + goto out; + } + inode_lock(d_inode(lower_dentry)); + rc = d_inode(lower_dentry)->i_op->listxattr(lower_dentry, list, size); + inode_unlock(d_inode(lower_dentry)); +out: + return rc; +} + +static int ecryptfs_removexattr(struct dentry *dentry, struct inode *inode, + const char *name) +{ + int rc; + struct dentry *lower_dentry; + struct inode *lower_inode; + + lower_dentry = ecryptfs_dentry_to_lower(dentry); + lower_inode = ecryptfs_inode_to_lower(inode); + if (!(lower_inode->i_opflags & IOP_XATTR)) { + rc = -EOPNOTSUPP; + goto out; + } + inode_lock(lower_inode); + rc = __vfs_removexattr(&nop_mnt_idmap, lower_dentry, name); + inode_unlock(lower_inode); +out: + return rc; +} + +static int ecryptfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + return vfs_fileattr_get(ecryptfs_dentry_to_lower(dentry), fa); +} + +static int ecryptfs_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa) +{ + struct dentry *lower_dentry = ecryptfs_dentry_to_lower(dentry); + int rc; + + rc = vfs_fileattr_set(&nop_mnt_idmap, lower_dentry, fa); + fsstack_copy_attr_all(d_inode(dentry), d_inode(lower_dentry)); + + return rc; +} + +static struct posix_acl *ecryptfs_get_acl(struct mnt_idmap *idmap, + struct dentry *dentry, int type) +{ + return vfs_get_acl(idmap, ecryptfs_dentry_to_lower(dentry), + posix_acl_xattr_name(type)); +} + +static int ecryptfs_set_acl(struct mnt_idmap *idmap, + struct dentry *dentry, struct posix_acl *acl, + int type) +{ + int rc; + struct dentry *lower_dentry = ecryptfs_dentry_to_lower(dentry); + struct inode *lower_inode = d_inode(lower_dentry); + + rc = vfs_set_acl(&nop_mnt_idmap, lower_dentry, + posix_acl_xattr_name(type), acl); + if (!rc) + fsstack_copy_attr_all(d_inode(dentry), lower_inode); + return rc; +} + +const struct inode_operations ecryptfs_symlink_iops = { + .get_link = ecryptfs_get_link, + .permission = ecryptfs_permission, + .setattr = ecryptfs_setattr, + .getattr = ecryptfs_getattr_link, + .listxattr = ecryptfs_listxattr, +}; + +const struct inode_operations ecryptfs_dir_iops = { + .create = ecryptfs_create, + .lookup = ecryptfs_lookup, + .link = ecryptfs_link, + .unlink = ecryptfs_unlink, + .symlink = ecryptfs_symlink, + .mkdir = ecryptfs_mkdir, + .rmdir = ecryptfs_rmdir, + .mknod = ecryptfs_mknod, + .rename = ecryptfs_rename, + .permission = ecryptfs_permission, + .setattr = ecryptfs_setattr, + .listxattr = ecryptfs_listxattr, + .fileattr_get = ecryptfs_fileattr_get, + .fileattr_set = ecryptfs_fileattr_set, + .get_acl = ecryptfs_get_acl, + .set_acl = ecryptfs_set_acl, +}; + +const struct inode_operations ecryptfs_main_iops = { + .permission = ecryptfs_permission, + .setattr = ecryptfs_setattr, + .getattr = ecryptfs_getattr, + .listxattr = ecryptfs_listxattr, + .fileattr_get = ecryptfs_fileattr_get, + .fileattr_set = ecryptfs_fileattr_set, + .get_acl = ecryptfs_get_acl, + .set_acl = ecryptfs_set_acl, +}; + +static int ecryptfs_xattr_get(const struct xattr_handler *handler, + struct dentry *dentry, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + return ecryptfs_getxattr(dentry, inode, name, buffer, size); +} + +static int ecryptfs_xattr_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *dentry, struct inode *inode, + const char *name, const void *value, size_t size, + int flags) +{ + if (value) + return ecryptfs_setxattr(dentry, inode, name, value, size, flags); + else { + BUG_ON(flags != XATTR_REPLACE); + return ecryptfs_removexattr(dentry, inode, name); + } +} + +static const struct xattr_handler ecryptfs_xattr_handler = { + .prefix = "", /* match anything */ + .get = ecryptfs_xattr_get, + .set = ecryptfs_xattr_set, +}; + +const struct xattr_handler * const ecryptfs_xattr_handlers[] = { + &ecryptfs_xattr_handler, + NULL +}; diff --git a/fs/ecryptfs/keystore.c b/fs/ecryptfs/keystore.c new file mode 100644 index 000000000..51651314b --- /dev/null +++ b/fs/ecryptfs/keystore.c @@ -0,0 +1,2516 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * eCryptfs: Linux filesystem encryption layer + * In-kernel key management code. Includes functions to parse and + * write authentication token-related packets with the underlying + * file. + * + * Copyright (C) 2004-2006 International Business Machines Corp. + * Author(s): Michael A. Halcrow + * Michael C. Thompson + * Trevor S. Highland + */ + +#include +#include +#include +#include +#include +#include +#include +#include "ecryptfs_kernel.h" + +/* + * request_key returned an error instead of a valid key address; + * determine the type of error, make appropriate log entries, and + * return an error code. + */ +static int process_request_key_err(long err_code) +{ + int rc = 0; + + switch (err_code) { + case -ENOKEY: + ecryptfs_printk(KERN_WARNING, "No key\n"); + rc = -ENOENT; + break; + case -EKEYEXPIRED: + ecryptfs_printk(KERN_WARNING, "Key expired\n"); + rc = -ETIME; + break; + case -EKEYREVOKED: + ecryptfs_printk(KERN_WARNING, "Key revoked\n"); + rc = -EINVAL; + break; + default: + ecryptfs_printk(KERN_WARNING, "Unknown error code: " + "[0x%.16lx]\n", err_code); + rc = -EINVAL; + } + return rc; +} + +static int process_find_global_auth_tok_for_sig_err(int err_code) +{ + int rc = err_code; + + switch (err_code) { + case -ENOENT: + ecryptfs_printk(KERN_WARNING, "Missing auth tok\n"); + break; + case -EINVAL: + ecryptfs_printk(KERN_WARNING, "Invalid auth tok\n"); + break; + default: + rc = process_request_key_err(err_code); + break; + } + return rc; +} + +/** + * ecryptfs_parse_packet_length + * @data: Pointer to memory containing length at offset + * @size: This function writes the decoded size to this memory + * address; zero on error + * @length_size: The number of bytes occupied by the encoded length + * + * Returns zero on success; non-zero on error + */ +int ecryptfs_parse_packet_length(unsigned char *data, size_t *size, + size_t *length_size) +{ + int rc = 0; + + (*length_size) = 0; + (*size) = 0; + if (data[0] < 192) { + /* One-byte length */ + (*size) = data[0]; + (*length_size) = 1; + } else if (data[0] < 224) { + /* Two-byte length */ + (*size) = (data[0] - 192) * 256; + (*size) += data[1] + 192; + (*length_size) = 2; + } else if (data[0] == 255) { + /* If support is added, adjust ECRYPTFS_MAX_PKT_LEN_SIZE */ + ecryptfs_printk(KERN_ERR, "Five-byte packet length not " + "supported\n"); + rc = -EINVAL; + goto out; + } else { + ecryptfs_printk(KERN_ERR, "Error parsing packet length\n"); + rc = -EINVAL; + goto out; + } +out: + return rc; +} + +/** + * ecryptfs_write_packet_length + * @dest: The byte array target into which to write the length. Must + * have at least ECRYPTFS_MAX_PKT_LEN_SIZE bytes allocated. + * @size: The length to write. + * @packet_size_length: The number of bytes used to encode the packet + * length is written to this address. + * + * Returns zero on success; non-zero on error. + */ +int ecryptfs_write_packet_length(char *dest, size_t size, + size_t *packet_size_length) +{ + int rc = 0; + + if (size < 192) { + dest[0] = size; + (*packet_size_length) = 1; + } else if (size < 65536) { + dest[0] = (((size - 192) / 256) + 192); + dest[1] = ((size - 192) % 256); + (*packet_size_length) = 2; + } else { + /* If support is added, adjust ECRYPTFS_MAX_PKT_LEN_SIZE */ + rc = -EINVAL; + ecryptfs_printk(KERN_WARNING, + "Unsupported packet size: [%zd]\n", size); + } + return rc; +} + +static int +write_tag_64_packet(char *signature, struct ecryptfs_session_key *session_key, + char **packet, size_t *packet_len) +{ + size_t i = 0; + size_t data_len; + size_t packet_size_len; + char *message; + int rc; + + /* + * ***** TAG 64 Packet Format ***** + * | Content Type | 1 byte | + * | Key Identifier Size | 1 or 2 bytes | + * | Key Identifier | arbitrary | + * | Encrypted File Encryption Key Size | 1 or 2 bytes | + * | Encrypted File Encryption Key | arbitrary | + */ + data_len = (5 + ECRYPTFS_SIG_SIZE_HEX + + session_key->encrypted_key_size); + *packet = kmalloc(data_len, GFP_KERNEL); + message = *packet; + if (!message) { + ecryptfs_printk(KERN_ERR, "Unable to allocate memory\n"); + rc = -ENOMEM; + goto out; + } + message[i++] = ECRYPTFS_TAG_64_PACKET_TYPE; + rc = ecryptfs_write_packet_length(&message[i], ECRYPTFS_SIG_SIZE_HEX, + &packet_size_len); + if (rc) { + ecryptfs_printk(KERN_ERR, "Error generating tag 64 packet " + "header; cannot generate packet length\n"); + goto out; + } + i += packet_size_len; + memcpy(&message[i], signature, ECRYPTFS_SIG_SIZE_HEX); + i += ECRYPTFS_SIG_SIZE_HEX; + rc = ecryptfs_write_packet_length(&message[i], + session_key->encrypted_key_size, + &packet_size_len); + if (rc) { + ecryptfs_printk(KERN_ERR, "Error generating tag 64 packet " + "header; cannot generate packet length\n"); + goto out; + } + i += packet_size_len; + memcpy(&message[i], session_key->encrypted_key, + session_key->encrypted_key_size); + i += session_key->encrypted_key_size; + *packet_len = i; +out: + return rc; +} + +static int +parse_tag_65_packet(struct ecryptfs_session_key *session_key, u8 *cipher_code, + struct ecryptfs_message *msg) +{ + size_t i = 0; + char *data; + size_t data_len; + size_t m_size; + size_t message_len; + u16 checksum = 0; + u16 expected_checksum = 0; + int rc; + + /* + * ***** TAG 65 Packet Format ***** + * | Content Type | 1 byte | + * | Status Indicator | 1 byte | + * | File Encryption Key Size | 1 or 2 bytes | + * | File Encryption Key | arbitrary | + */ + message_len = msg->data_len; + data = msg->data; + if (message_len < 4) { + rc = -EIO; + goto out; + } + if (data[i++] != ECRYPTFS_TAG_65_PACKET_TYPE) { + ecryptfs_printk(KERN_ERR, "Type should be ECRYPTFS_TAG_65\n"); + rc = -EIO; + goto out; + } + if (data[i++]) { + ecryptfs_printk(KERN_ERR, "Status indicator has non-zero value " + "[%d]\n", data[i-1]); + rc = -EIO; + goto out; + } + rc = ecryptfs_parse_packet_length(&data[i], &m_size, &data_len); + if (rc) { + ecryptfs_printk(KERN_WARNING, "Error parsing packet length; " + "rc = [%d]\n", rc); + goto out; + } + i += data_len; + if (message_len < (i + m_size)) { + ecryptfs_printk(KERN_ERR, "The message received from ecryptfsd " + "is shorter than expected\n"); + rc = -EIO; + goto out; + } + if (m_size < 3) { + ecryptfs_printk(KERN_ERR, + "The decrypted key is not long enough to " + "include a cipher code and checksum\n"); + rc = -EIO; + goto out; + } + *cipher_code = data[i++]; + /* The decrypted key includes 1 byte cipher code and 2 byte checksum */ + session_key->decrypted_key_size = m_size - 3; + if (session_key->decrypted_key_size > ECRYPTFS_MAX_KEY_BYTES) { + ecryptfs_printk(KERN_ERR, "key_size [%d] larger than " + "the maximum key size [%d]\n", + session_key->decrypted_key_size, + ECRYPTFS_MAX_ENCRYPTED_KEY_BYTES); + rc = -EIO; + goto out; + } + memcpy(session_key->decrypted_key, &data[i], + session_key->decrypted_key_size); + i += session_key->decrypted_key_size; + expected_checksum += (unsigned char)(data[i++]) << 8; + expected_checksum += (unsigned char)(data[i++]); + for (i = 0; i < session_key->decrypted_key_size; i++) + checksum += session_key->decrypted_key[i]; + if (expected_checksum != checksum) { + ecryptfs_printk(KERN_ERR, "Invalid checksum for file " + "encryption key; expected [%x]; calculated " + "[%x]\n", expected_checksum, checksum); + rc = -EIO; + } +out: + return rc; +} + + +static int +write_tag_66_packet(char *signature, u8 cipher_code, + struct ecryptfs_crypt_stat *crypt_stat, char **packet, + size_t *packet_len) +{ + size_t i = 0; + size_t j; + size_t data_len; + size_t checksum = 0; + size_t packet_size_len; + char *message; + int rc; + + /* + * ***** TAG 66 Packet Format ***** + * | Content Type | 1 byte | + * | Key Identifier Size | 1 or 2 bytes | + * | Key Identifier | arbitrary | + * | File Encryption Key Size | 1 or 2 bytes | + * | Cipher Code | 1 byte | + * | File Encryption Key | arbitrary | + * | Checksum | 2 bytes | + */ + data_len = (8 + ECRYPTFS_SIG_SIZE_HEX + crypt_stat->key_size); + *packet = kmalloc(data_len, GFP_KERNEL); + message = *packet; + if (!message) { + ecryptfs_printk(KERN_ERR, "Unable to allocate memory\n"); + rc = -ENOMEM; + goto out; + } + message[i++] = ECRYPTFS_TAG_66_PACKET_TYPE; + rc = ecryptfs_write_packet_length(&message[i], ECRYPTFS_SIG_SIZE_HEX, + &packet_size_len); + if (rc) { + ecryptfs_printk(KERN_ERR, "Error generating tag 66 packet " + "header; cannot generate packet length\n"); + goto out; + } + i += packet_size_len; + memcpy(&message[i], signature, ECRYPTFS_SIG_SIZE_HEX); + i += ECRYPTFS_SIG_SIZE_HEX; + /* The encrypted key includes 1 byte cipher code and 2 byte checksum */ + rc = ecryptfs_write_packet_length(&message[i], crypt_stat->key_size + 3, + &packet_size_len); + if (rc) { + ecryptfs_printk(KERN_ERR, "Error generating tag 66 packet " + "header; cannot generate packet length\n"); + goto out; + } + i += packet_size_len; + message[i++] = cipher_code; + memcpy(&message[i], crypt_stat->key, crypt_stat->key_size); + i += crypt_stat->key_size; + for (j = 0; j < crypt_stat->key_size; j++) + checksum += crypt_stat->key[j]; + message[i++] = (checksum / 256) % 256; + message[i++] = (checksum % 256); + *packet_len = i; +out: + return rc; +} + +static int +parse_tag_67_packet(struct ecryptfs_key_record *key_rec, + struct ecryptfs_message *msg) +{ + size_t i = 0; + char *data; + size_t data_len; + size_t message_len; + int rc; + + /* + * ***** TAG 67 Packet Format ***** + * | Content Type | 1 byte | + * | Status Indicator | 1 byte | + * | Encrypted File Encryption Key Size | 1 or 2 bytes | + * | Encrypted File Encryption Key | arbitrary | + */ + message_len = msg->data_len; + data = msg->data; + /* verify that everything through the encrypted FEK size is present */ + if (message_len < 4) { + rc = -EIO; + printk(KERN_ERR "%s: message_len is [%zd]; minimum acceptable " + "message length is [%d]\n", __func__, message_len, 4); + goto out; + } + if (data[i++] != ECRYPTFS_TAG_67_PACKET_TYPE) { + rc = -EIO; + printk(KERN_ERR "%s: Type should be ECRYPTFS_TAG_67\n", + __func__); + goto out; + } + if (data[i++]) { + rc = -EIO; + printk(KERN_ERR "%s: Status indicator has non zero " + "value [%d]\n", __func__, data[i-1]); + + goto out; + } + rc = ecryptfs_parse_packet_length(&data[i], &key_rec->enc_key_size, + &data_len); + if (rc) { + ecryptfs_printk(KERN_WARNING, "Error parsing packet length; " + "rc = [%d]\n", rc); + goto out; + } + i += data_len; + if (message_len < (i + key_rec->enc_key_size)) { + rc = -EIO; + printk(KERN_ERR "%s: message_len [%zd]; max len is [%zd]\n", + __func__, message_len, (i + key_rec->enc_key_size)); + goto out; + } + if (key_rec->enc_key_size > ECRYPTFS_MAX_ENCRYPTED_KEY_BYTES) { + rc = -EIO; + printk(KERN_ERR "%s: Encrypted key_size [%zd] larger than " + "the maximum key size [%d]\n", __func__, + key_rec->enc_key_size, + ECRYPTFS_MAX_ENCRYPTED_KEY_BYTES); + goto out; + } + memcpy(key_rec->enc_key, &data[i], key_rec->enc_key_size); +out: + return rc; +} + +/** + * ecryptfs_verify_version + * @version: The version number to confirm + * + * Returns zero on good version; non-zero otherwise + */ +static int ecryptfs_verify_version(u16 version) +{ + int rc = 0; + unsigned char major; + unsigned char minor; + + major = ((version >> 8) & 0xFF); + minor = (version & 0xFF); + if (major != ECRYPTFS_VERSION_MAJOR) { + ecryptfs_printk(KERN_ERR, "Major version number mismatch. " + "Expected [%d]; got [%d]\n", + ECRYPTFS_VERSION_MAJOR, major); + rc = -EINVAL; + goto out; + } + if (minor != ECRYPTFS_VERSION_MINOR) { + ecryptfs_printk(KERN_ERR, "Minor version number mismatch. " + "Expected [%d]; got [%d]\n", + ECRYPTFS_VERSION_MINOR, minor); + rc = -EINVAL; + goto out; + } +out: + return rc; +} + +/** + * ecryptfs_verify_auth_tok_from_key + * @auth_tok_key: key containing the authentication token + * @auth_tok: authentication token + * + * Returns zero on valid auth tok; -EINVAL if the payload is invalid; or + * -EKEYREVOKED if the key was revoked before we acquired its semaphore. + */ +static int +ecryptfs_verify_auth_tok_from_key(struct key *auth_tok_key, + struct ecryptfs_auth_tok **auth_tok) +{ + int rc = 0; + + (*auth_tok) = ecryptfs_get_key_payload_data(auth_tok_key); + if (IS_ERR(*auth_tok)) { + rc = PTR_ERR(*auth_tok); + *auth_tok = NULL; + goto out; + } + + if (ecryptfs_verify_version((*auth_tok)->version)) { + printk(KERN_ERR "Data structure version mismatch. Userspace " + "tools must match eCryptfs kernel module with major " + "version [%d] and minor version [%d]\n", + ECRYPTFS_VERSION_MAJOR, ECRYPTFS_VERSION_MINOR); + rc = -EINVAL; + goto out; + } + if ((*auth_tok)->token_type != ECRYPTFS_PASSWORD + && (*auth_tok)->token_type != ECRYPTFS_PRIVATE_KEY) { + printk(KERN_ERR "Invalid auth_tok structure " + "returned from key query\n"); + rc = -EINVAL; + goto out; + } +out: + return rc; +} + +static int +ecryptfs_find_global_auth_tok_for_sig( + struct key **auth_tok_key, + struct ecryptfs_auth_tok **auth_tok, + struct ecryptfs_mount_crypt_stat *mount_crypt_stat, char *sig) +{ + struct ecryptfs_global_auth_tok *walker; + int rc = 0; + + (*auth_tok_key) = NULL; + (*auth_tok) = NULL; + mutex_lock(&mount_crypt_stat->global_auth_tok_list_mutex); + list_for_each_entry(walker, + &mount_crypt_stat->global_auth_tok_list, + mount_crypt_stat_list) { + if (memcmp(walker->sig, sig, ECRYPTFS_SIG_SIZE_HEX)) + continue; + + if (walker->flags & ECRYPTFS_AUTH_TOK_INVALID) { + rc = -EINVAL; + goto out; + } + + rc = key_validate(walker->global_auth_tok_key); + if (rc) { + if (rc == -EKEYEXPIRED) + goto out; + goto out_invalid_auth_tok; + } + + down_write(&(walker->global_auth_tok_key->sem)); + rc = ecryptfs_verify_auth_tok_from_key( + walker->global_auth_tok_key, auth_tok); + if (rc) + goto out_invalid_auth_tok_unlock; + + (*auth_tok_key) = walker->global_auth_tok_key; + key_get(*auth_tok_key); + goto out; + } + rc = -ENOENT; + goto out; +out_invalid_auth_tok_unlock: + up_write(&(walker->global_auth_tok_key->sem)); +out_invalid_auth_tok: + printk(KERN_WARNING "Invalidating auth tok with sig = [%s]\n", sig); + walker->flags |= ECRYPTFS_AUTH_TOK_INVALID; + key_put(walker->global_auth_tok_key); + walker->global_auth_tok_key = NULL; +out: + mutex_unlock(&mount_crypt_stat->global_auth_tok_list_mutex); + return rc; +} + +/** + * ecryptfs_find_auth_tok_for_sig + * @auth_tok_key: key containing the authentication token + * @auth_tok: Set to the matching auth_tok; NULL if not found + * @mount_crypt_stat: inode crypt_stat crypto context + * @sig: Sig of auth_tok to find + * + * For now, this function simply looks at the registered auth_tok's + * linked off the mount_crypt_stat, so all the auth_toks that can be + * used must be registered at mount time. This function could + * potentially try a lot harder to find auth_tok's (e.g., by calling + * out to ecryptfsd to dynamically retrieve an auth_tok object) so + * that static registration of auth_tok's will no longer be necessary. + * + * Returns zero on no error; non-zero on error + */ +static int +ecryptfs_find_auth_tok_for_sig( + struct key **auth_tok_key, + struct ecryptfs_auth_tok **auth_tok, + struct ecryptfs_mount_crypt_stat *mount_crypt_stat, + char *sig) +{ + int rc = 0; + + rc = ecryptfs_find_global_auth_tok_for_sig(auth_tok_key, auth_tok, + mount_crypt_stat, sig); + if (rc == -ENOENT) { + /* if the flag ECRYPTFS_GLOBAL_MOUNT_AUTH_TOK_ONLY is set in the + * mount_crypt_stat structure, we prevent to use auth toks that + * are not inserted through the ecryptfs_add_global_auth_tok + * function. + */ + if (mount_crypt_stat->flags + & ECRYPTFS_GLOBAL_MOUNT_AUTH_TOK_ONLY) + return -EINVAL; + + rc = ecryptfs_keyring_auth_tok_for_sig(auth_tok_key, auth_tok, + sig); + } + return rc; +} + +/* + * write_tag_70_packet can gobble a lot of stack space. We stuff most + * of the function's parameters in a kmalloc'd struct to help reduce + * eCryptfs' overall stack usage. + */ +struct ecryptfs_write_tag_70_packet_silly_stack { + u8 cipher_code; + size_t max_packet_size; + size_t packet_size_len; + size_t block_aligned_filename_size; + size_t block_size; + size_t i; + size_t j; + size_t num_rand_bytes; + struct mutex *tfm_mutex; + char *block_aligned_filename; + struct ecryptfs_auth_tok *auth_tok; + struct scatterlist src_sg[2]; + struct scatterlist dst_sg[2]; + struct crypto_skcipher *skcipher_tfm; + struct skcipher_request *skcipher_req; + char iv[ECRYPTFS_MAX_IV_BYTES]; + char hash[MD5_DIGEST_SIZE]; +}; + +/* + * write_tag_70_packet - Write encrypted filename (EFN) packet against FNEK + * @filename: NULL-terminated filename string + * + * This is the simplest mechanism for achieving filename encryption in + * eCryptfs. It encrypts the given filename with the mount-wide + * filename encryption key (FNEK) and stores it in a packet to @dest, + * which the callee will encode and write directly into the dentry + * name. + */ +int +ecryptfs_write_tag_70_packet(char *dest, size_t *remaining_bytes, + size_t *packet_size, + struct ecryptfs_mount_crypt_stat *mount_crypt_stat, + char *filename, size_t filename_size) +{ + struct ecryptfs_write_tag_70_packet_silly_stack *s; + struct key *auth_tok_key = NULL; + int rc = 0; + + s = kzalloc_obj(*s); + if (!s) + return -ENOMEM; + + (*packet_size) = 0; + rc = ecryptfs_find_auth_tok_for_sig( + &auth_tok_key, + &s->auth_tok, mount_crypt_stat, + mount_crypt_stat->global_default_fnek_sig); + if (rc) { + printk(KERN_ERR "%s: Error attempting to find auth tok for " + "fnek sig [%s]; rc = [%d]\n", __func__, + mount_crypt_stat->global_default_fnek_sig, rc); + goto out; + } + rc = ecryptfs_get_tfm_and_mutex_for_cipher_name( + &s->skcipher_tfm, + &s->tfm_mutex, mount_crypt_stat->global_default_fn_cipher_name); + if (unlikely(rc)) { + printk(KERN_ERR "Internal error whilst attempting to get " + "tfm and mutex for cipher name [%s]; rc = [%d]\n", + mount_crypt_stat->global_default_fn_cipher_name, rc); + goto out; + } + mutex_lock(s->tfm_mutex); + s->block_size = crypto_skcipher_blocksize(s->skcipher_tfm); + /* Plus one for the \0 separator between the random prefix + * and the plaintext filename */ + s->num_rand_bytes = (ECRYPTFS_FILENAME_MIN_RANDOM_PREPEND_BYTES + 1); + s->block_aligned_filename_size = (s->num_rand_bytes + filename_size); + if ((s->block_aligned_filename_size % s->block_size) != 0) { + s->num_rand_bytes += (s->block_size + - (s->block_aligned_filename_size + % s->block_size)); + s->block_aligned_filename_size = (s->num_rand_bytes + + filename_size); + } + /* Octet 0: Tag 70 identifier + * Octets 1-N1: Tag 70 packet size (includes cipher identifier + * and block-aligned encrypted filename size) + * Octets N1-N2: FNEK sig (ECRYPTFS_SIG_SIZE) + * Octet N2-N3: Cipher identifier (1 octet) + * Octets N3-N4: Block-aligned encrypted filename + * - Consists of a minimum number of random characters, a \0 + * separator, and then the filename */ + s->max_packet_size = (ECRYPTFS_TAG_70_MAX_METADATA_SIZE + + s->block_aligned_filename_size); + if (!dest) { + (*packet_size) = s->max_packet_size; + goto out_unlock; + } + if (s->max_packet_size > (*remaining_bytes)) { + printk(KERN_WARNING "%s: Require [%zd] bytes to write; only " + "[%zd] available\n", __func__, s->max_packet_size, + (*remaining_bytes)); + rc = -EINVAL; + goto out_unlock; + } + + s->skcipher_req = skcipher_request_alloc(s->skcipher_tfm, GFP_KERNEL); + if (!s->skcipher_req) { + printk(KERN_ERR "%s: Out of kernel memory whilst attempting to " + "skcipher_request_alloc for %s\n", __func__, + crypto_skcipher_driver_name(s->skcipher_tfm)); + rc = -ENOMEM; + goto out_unlock; + } + + skcipher_request_set_callback(s->skcipher_req, + CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + + s->block_aligned_filename = kzalloc(s->block_aligned_filename_size, + GFP_KERNEL); + if (!s->block_aligned_filename) { + rc = -ENOMEM; + goto out_unlock; + } + dest[s->i++] = ECRYPTFS_TAG_70_PACKET_TYPE; + rc = ecryptfs_write_packet_length(&dest[s->i], + (ECRYPTFS_SIG_SIZE + + 1 /* Cipher code */ + + s->block_aligned_filename_size), + &s->packet_size_len); + if (rc) { + printk(KERN_ERR "%s: Error generating tag 70 packet " + "header; cannot generate packet length; rc = [%d]\n", + __func__, rc); + goto out_free_unlock; + } + s->i += s->packet_size_len; + ecryptfs_from_hex(&dest[s->i], + mount_crypt_stat->global_default_fnek_sig, + ECRYPTFS_SIG_SIZE); + s->i += ECRYPTFS_SIG_SIZE; + s->cipher_code = ecryptfs_code_for_cipher_string( + mount_crypt_stat->global_default_fn_cipher_name, + mount_crypt_stat->global_default_fn_cipher_key_bytes); + if (s->cipher_code == 0) { + printk(KERN_WARNING "%s: Unable to generate code for " + "cipher [%s] with key bytes [%zd]\n", __func__, + mount_crypt_stat->global_default_fn_cipher_name, + mount_crypt_stat->global_default_fn_cipher_key_bytes); + rc = -EINVAL; + goto out_free_unlock; + } + dest[s->i++] = s->cipher_code; + /* TODO: Support other key modules than passphrase for + * filename encryption */ + if (s->auth_tok->token_type != ECRYPTFS_PASSWORD) { + rc = -EOPNOTSUPP; + printk(KERN_INFO "%s: Filename encryption only supports " + "password tokens\n", __func__); + goto out_free_unlock; + } + + md5(s->auth_tok->token.password.session_key_encryption_key, + s->auth_tok->token.password.session_key_encryption_key_bytes, + s->hash); + for (s->j = 0; s->j < (s->num_rand_bytes - 1); s->j++) { + s->block_aligned_filename[s->j] = + s->hash[s->j % MD5_DIGEST_SIZE]; + if ((s->j % MD5_DIGEST_SIZE) == (MD5_DIGEST_SIZE - 1)) + md5(s->hash, MD5_DIGEST_SIZE, s->hash); + if (s->block_aligned_filename[s->j] == '\0') + s->block_aligned_filename[s->j] = ECRYPTFS_NON_NULL; + } + memcpy(&s->block_aligned_filename[s->num_rand_bytes], filename, + filename_size); + rc = virt_to_scatterlist(s->block_aligned_filename, + s->block_aligned_filename_size, s->src_sg, 2); + if (rc < 1) { + printk(KERN_ERR "%s: Internal error whilst attempting to " + "convert filename memory to scatterlist; rc = [%d]. " + "block_aligned_filename_size = [%zd]\n", __func__, rc, + s->block_aligned_filename_size); + goto out_free_unlock; + } + rc = virt_to_scatterlist(&dest[s->i], s->block_aligned_filename_size, + s->dst_sg, 2); + if (rc < 1) { + printk(KERN_ERR "%s: Internal error whilst attempting to " + "convert encrypted filename memory to scatterlist; " + "rc = [%d]. block_aligned_filename_size = [%zd]\n", + __func__, rc, s->block_aligned_filename_size); + goto out_free_unlock; + } + /* The characters in the first block effectively do the job + * of the IV here, so we just use 0's for the IV. Note the + * constraint that ECRYPTFS_FILENAME_MIN_RANDOM_PREPEND_BYTES + * >= ECRYPTFS_MAX_IV_BYTES. */ + rc = crypto_skcipher_setkey( + s->skcipher_tfm, + s->auth_tok->token.password.session_key_encryption_key, + mount_crypt_stat->global_default_fn_cipher_key_bytes); + if (rc < 0) { + printk(KERN_ERR "%s: Error setting key for crypto context; " + "rc = [%d]. s->auth_tok->token.password.session_key_" + "encryption_key = [0x%p]; mount_crypt_stat->" + "global_default_fn_cipher_key_bytes = [%zd]\n", __func__, + rc, + s->auth_tok->token.password.session_key_encryption_key, + mount_crypt_stat->global_default_fn_cipher_key_bytes); + goto out_free_unlock; + } + skcipher_request_set_crypt(s->skcipher_req, s->src_sg, s->dst_sg, + s->block_aligned_filename_size, s->iv); + rc = crypto_skcipher_encrypt(s->skcipher_req); + if (rc) { + printk(KERN_ERR "%s: Error attempting to encrypt filename; " + "rc = [%d]\n", __func__, rc); + goto out_free_unlock; + } + s->i += s->block_aligned_filename_size; + (*packet_size) = s->i; + (*remaining_bytes) -= (*packet_size); +out_free_unlock: + kfree_sensitive(s->block_aligned_filename); +out_unlock: + mutex_unlock(s->tfm_mutex); +out: + if (auth_tok_key) { + up_write(&(auth_tok_key->sem)); + key_put(auth_tok_key); + } + skcipher_request_free(s->skcipher_req); + kfree(s); + return rc; +} + +struct ecryptfs_parse_tag_70_packet_silly_stack { + u8 cipher_code; + size_t max_packet_size; + size_t packet_size_len; + size_t parsed_tag_70_packet_size; + size_t block_aligned_filename_size; + size_t block_size; + size_t i; + struct mutex *tfm_mutex; + char *decrypted_filename; + struct ecryptfs_auth_tok *auth_tok; + struct scatterlist src_sg[2]; + struct scatterlist dst_sg[2]; + struct crypto_skcipher *skcipher_tfm; + struct skcipher_request *skcipher_req; + char fnek_sig_hex[ECRYPTFS_SIG_SIZE_HEX + 1]; + char iv[ECRYPTFS_MAX_IV_BYTES]; + char cipher_string[ECRYPTFS_MAX_CIPHER_NAME_SIZE + 1]; +}; + +/** + * ecryptfs_parse_tag_70_packet - Parse and process FNEK-encrypted passphrase packet + * @filename: This function kmalloc's the memory for the filename + * @filename_size: This function sets this to the amount of memory + * kmalloc'd for the filename + * @packet_size: This function sets this to the number of octets + * in the packet parsed + * @mount_crypt_stat: The mount-wide cryptographic context + * @data: The memory location containing the start of the tag 70 + * packet + * @max_packet_size: The maximum legal size of the packet to be parsed + * from @data + * + * Returns zero on success; non-zero otherwise + */ +int +ecryptfs_parse_tag_70_packet(char **filename, size_t *filename_size, + size_t *packet_size, + struct ecryptfs_mount_crypt_stat *mount_crypt_stat, + char *data, size_t max_packet_size) +{ + struct ecryptfs_parse_tag_70_packet_silly_stack *s; + struct key *auth_tok_key = NULL; + int rc = 0; + + (*packet_size) = 0; + (*filename_size) = 0; + (*filename) = NULL; + s = kzalloc_obj(*s); + if (!s) + return -ENOMEM; + + if (max_packet_size < ECRYPTFS_TAG_70_MIN_METADATA_SIZE) { + printk(KERN_WARNING "%s: max_packet_size is [%zd]; it must be " + "at least [%d]\n", __func__, max_packet_size, + ECRYPTFS_TAG_70_MIN_METADATA_SIZE); + rc = -EINVAL; + goto out; + } + /* Octet 0: Tag 70 identifier + * Octets 1-N1: Tag 70 packet size (includes cipher identifier + * and block-aligned encrypted filename size) + * Octets N1-N2: FNEK sig (ECRYPTFS_SIG_SIZE) + * Octet N2-N3: Cipher identifier (1 octet) + * Octets N3-N4: Block-aligned encrypted filename + * - Consists of a minimum number of random numbers, a \0 + * separator, and then the filename */ + if (data[(*packet_size)++] != ECRYPTFS_TAG_70_PACKET_TYPE) { + printk(KERN_WARNING "%s: Invalid packet tag [0x%.2x]; must be " + "tag [0x%.2x]\n", __func__, + data[((*packet_size) - 1)], ECRYPTFS_TAG_70_PACKET_TYPE); + rc = -EINVAL; + goto out; + } + rc = ecryptfs_parse_packet_length(&data[(*packet_size)], + &s->parsed_tag_70_packet_size, + &s->packet_size_len); + if (rc) { + printk(KERN_WARNING "%s: Error parsing packet length; " + "rc = [%d]\n", __func__, rc); + goto out; + } + if (s->parsed_tag_70_packet_size < (ECRYPTFS_SIG_SIZE + 2)) { + ecryptfs_printk(KERN_WARNING, "Invalid packet size [%zd]\n", + s->parsed_tag_70_packet_size); + rc = -EINVAL; + goto out; + } + s->block_aligned_filename_size = (s->parsed_tag_70_packet_size + - ECRYPTFS_SIG_SIZE - 1); + if ((1 + s->packet_size_len + s->parsed_tag_70_packet_size) + > max_packet_size) { + printk(KERN_WARNING "%s: max_packet_size is [%zd]; real packet " + "size is [%zd]\n", __func__, max_packet_size, + (1 + s->packet_size_len + 1 + + s->block_aligned_filename_size)); + rc = -EINVAL; + goto out; + } + (*packet_size) += s->packet_size_len; + ecryptfs_to_hex(s->fnek_sig_hex, &data[(*packet_size)], + ECRYPTFS_SIG_SIZE); + (*packet_size) += ECRYPTFS_SIG_SIZE; + s->cipher_code = data[(*packet_size)++]; + rc = ecryptfs_cipher_code_to_string(s->cipher_string, + sizeof(s->cipher_string), + s->cipher_code); + if (rc) { + printk(KERN_WARNING "%s: Cipher code [%d] is invalid\n", + __func__, s->cipher_code); + goto out; + } + rc = ecryptfs_find_auth_tok_for_sig(&auth_tok_key, + &s->auth_tok, mount_crypt_stat, + s->fnek_sig_hex); + if (rc) { + printk(KERN_ERR "%s: Error attempting to find auth tok for " + "fnek sig [%s]; rc = [%d]\n", __func__, s->fnek_sig_hex, + rc); + goto out; + } + rc = ecryptfs_get_tfm_and_mutex_for_cipher_name(&s->skcipher_tfm, + &s->tfm_mutex, + s->cipher_string); + if (unlikely(rc)) { + printk(KERN_ERR "Internal error whilst attempting to get " + "tfm and mutex for cipher name [%s]; rc = [%d]\n", + s->cipher_string, rc); + goto out; + } + mutex_lock(s->tfm_mutex); + rc = virt_to_scatterlist(&data[(*packet_size)], + s->block_aligned_filename_size, s->src_sg, 2); + if (rc < 1) { + printk(KERN_ERR "%s: Internal error whilst attempting to " + "convert encrypted filename memory to scatterlist; " + "rc = [%d]. block_aligned_filename_size = [%zd]\n", + __func__, rc, s->block_aligned_filename_size); + goto out_unlock; + } + (*packet_size) += s->block_aligned_filename_size; + s->decrypted_filename = kmalloc(s->block_aligned_filename_size, + GFP_KERNEL); + if (!s->decrypted_filename) { + rc = -ENOMEM; + goto out_unlock; + } + rc = virt_to_scatterlist(s->decrypted_filename, + s->block_aligned_filename_size, s->dst_sg, 2); + if (rc < 1) { + printk(KERN_ERR "%s: Internal error whilst attempting to " + "convert decrypted filename memory to scatterlist; " + "rc = [%d]. block_aligned_filename_size = [%zd]\n", + __func__, rc, s->block_aligned_filename_size); + goto out_free_unlock; + } + + s->skcipher_req = skcipher_request_alloc(s->skcipher_tfm, GFP_KERNEL); + if (!s->skcipher_req) { + printk(KERN_ERR "%s: Out of kernel memory whilst attempting to " + "skcipher_request_alloc for %s\n", __func__, + crypto_skcipher_driver_name(s->skcipher_tfm)); + rc = -ENOMEM; + goto out_free_unlock; + } + + skcipher_request_set_callback(s->skcipher_req, + CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); + + /* The characters in the first block effectively do the job of + * the IV here, so we just use 0's for the IV. Note the + * constraint that ECRYPTFS_FILENAME_MIN_RANDOM_PREPEND_BYTES + * >= ECRYPTFS_MAX_IV_BYTES. */ + /* TODO: Support other key modules than passphrase for + * filename encryption */ + if (s->auth_tok->token_type != ECRYPTFS_PASSWORD) { + rc = -EOPNOTSUPP; + printk(KERN_INFO "%s: Filename encryption only supports " + "password tokens\n", __func__); + goto out_free_unlock; + } + rc = crypto_skcipher_setkey( + s->skcipher_tfm, + s->auth_tok->token.password.session_key_encryption_key, + mount_crypt_stat->global_default_fn_cipher_key_bytes); + if (rc < 0) { + printk(KERN_ERR "%s: Error setting key for crypto context; " + "rc = [%d]. s->auth_tok->token.password.session_key_" + "encryption_key = [0x%p]; mount_crypt_stat->" + "global_default_fn_cipher_key_bytes = [%zd]\n", __func__, + rc, + s->auth_tok->token.password.session_key_encryption_key, + mount_crypt_stat->global_default_fn_cipher_key_bytes); + goto out_free_unlock; + } + skcipher_request_set_crypt(s->skcipher_req, s->src_sg, s->dst_sg, + s->block_aligned_filename_size, s->iv); + rc = crypto_skcipher_decrypt(s->skcipher_req); + if (rc) { + printk(KERN_ERR "%s: Error attempting to decrypt filename; " + "rc = [%d]\n", __func__, rc); + goto out_free_unlock; + } + + while (s->i < s->block_aligned_filename_size && + s->decrypted_filename[s->i] != '\0') + s->i++; + if (s->i == s->block_aligned_filename_size) { + printk(KERN_WARNING "%s: Invalid tag 70 packet; could not " + "find valid separator between random characters and " + "the filename\n", __func__); + rc = -EINVAL; + goto out_free_unlock; + } + s->i++; + (*filename_size) = (s->block_aligned_filename_size - s->i); + if (!((*filename_size) > 0 && (*filename_size < PATH_MAX))) { + printk(KERN_WARNING "%s: Filename size is [%zd], which is " + "invalid\n", __func__, (*filename_size)); + rc = -EINVAL; + goto out_free_unlock; + } + (*filename) = kmalloc(((*filename_size) + 1), GFP_KERNEL); + if (!(*filename)) { + rc = -ENOMEM; + goto out_free_unlock; + } + memcpy((*filename), &s->decrypted_filename[s->i], (*filename_size)); + (*filename)[(*filename_size)] = '\0'; +out_free_unlock: + kfree(s->decrypted_filename); +out_unlock: + mutex_unlock(s->tfm_mutex); +out: + if (rc) { + (*packet_size) = 0; + (*filename_size) = 0; + (*filename) = NULL; + } + if (auth_tok_key) { + up_write(&(auth_tok_key->sem)); + key_put(auth_tok_key); + } + skcipher_request_free(s->skcipher_req); + kfree(s); + return rc; +} + +static int +ecryptfs_get_auth_tok_sig(char **sig, struct ecryptfs_auth_tok *auth_tok) +{ + int rc = 0; + + (*sig) = NULL; + switch (auth_tok->token_type) { + case ECRYPTFS_PASSWORD: + (*sig) = auth_tok->token.password.signature; + break; + case ECRYPTFS_PRIVATE_KEY: + (*sig) = auth_tok->token.private_key.signature; + break; + default: + printk(KERN_ERR "Cannot get sig for auth_tok of type [%d]\n", + auth_tok->token_type); + rc = -EINVAL; + } + return rc; +} + +/** + * decrypt_pki_encrypted_session_key - Decrypt the session key with the given auth_tok. + * @auth_tok: The key authentication token used to decrypt the session key + * @crypt_stat: The cryptographic context + * + * Returns zero on success; non-zero error otherwise. + */ +static int +decrypt_pki_encrypted_session_key(struct ecryptfs_auth_tok *auth_tok, + struct ecryptfs_crypt_stat *crypt_stat) +{ + u8 cipher_code = 0; + struct ecryptfs_msg_ctx *msg_ctx; + struct ecryptfs_message *msg = NULL; + char *auth_tok_sig; + char *payload = NULL; + size_t payload_len = 0; + int rc; + + rc = ecryptfs_get_auth_tok_sig(&auth_tok_sig, auth_tok); + if (rc) { + printk(KERN_ERR "Unrecognized auth tok type: [%d]\n", + auth_tok->token_type); + goto out; + } + rc = write_tag_64_packet(auth_tok_sig, &(auth_tok->session_key), + &payload, &payload_len); + if (rc) { + ecryptfs_printk(KERN_ERR, "Failed to write tag 64 packet\n"); + goto out; + } + rc = ecryptfs_send_message(payload, payload_len, &msg_ctx); + if (rc) { + ecryptfs_printk(KERN_ERR, "Error sending message to " + "ecryptfsd: %d\n", rc); + goto out; + } + rc = ecryptfs_wait_for_response(msg_ctx, &msg); + if (rc) { + ecryptfs_printk(KERN_ERR, "Failed to receive tag 65 packet " + "from the user space daemon\n"); + rc = -EIO; + goto out; + } + rc = parse_tag_65_packet(&(auth_tok->session_key), + &cipher_code, msg); + if (rc) { + printk(KERN_ERR "Failed to parse tag 65 packet; rc = [%d]\n", + rc); + goto out; + } + auth_tok->session_key.flags |= ECRYPTFS_CONTAINS_DECRYPTED_KEY; + memcpy(crypt_stat->key, auth_tok->session_key.decrypted_key, + auth_tok->session_key.decrypted_key_size); + crypt_stat->key_size = auth_tok->session_key.decrypted_key_size; + rc = ecryptfs_cipher_code_to_string(crypt_stat->cipher, + sizeof(crypt_stat->cipher), + cipher_code); + if (rc) { + ecryptfs_printk(KERN_ERR, "Cipher code [%d] is invalid\n", + cipher_code); + goto out; + } + crypt_stat->flags |= ECRYPTFS_KEY_VALID; + if (ecryptfs_verbosity > 0) { + ecryptfs_printk(KERN_DEBUG, "Decrypted session key:\n"); + ecryptfs_dump_hex(crypt_stat->key, + crypt_stat->key_size); + } +out: + kfree(msg); + kfree(payload); + return rc; +} + +static void wipe_auth_tok_list(struct list_head *auth_tok_list_head) +{ + struct ecryptfs_auth_tok_list_item *auth_tok_list_item; + struct ecryptfs_auth_tok_list_item *auth_tok_list_item_tmp; + + list_for_each_entry_safe(auth_tok_list_item, auth_tok_list_item_tmp, + auth_tok_list_head, list) { + list_del(&auth_tok_list_item->list); + kmem_cache_free(ecryptfs_auth_tok_list_item_cache, + auth_tok_list_item); + } +} + +struct kmem_cache *ecryptfs_auth_tok_list_item_cache; + +/** + * parse_tag_1_packet + * @crypt_stat: The cryptographic context to modify based on packet contents + * @data: The raw bytes of the packet. + * @auth_tok_list: eCryptfs parses packets into authentication tokens; + * a new authentication token will be placed at the + * end of this list for this packet. + * @new_auth_tok: Pointer to a pointer to memory that this function + * allocates; sets the memory address of the pointer to + * NULL on error. This object is added to the + * auth_tok_list. + * @packet_size: This function writes the size of the parsed packet + * into this memory location; zero on error. + * @max_packet_size: The maximum allowable packet size + * + * Returns zero on success; non-zero on error. + */ +static int +parse_tag_1_packet(struct ecryptfs_crypt_stat *crypt_stat, + unsigned char *data, struct list_head *auth_tok_list, + struct ecryptfs_auth_tok **new_auth_tok, + size_t *packet_size, size_t max_packet_size) +{ + size_t body_size; + struct ecryptfs_auth_tok_list_item *auth_tok_list_item; + size_t length_size; + int rc = 0; + + (*packet_size) = 0; + (*new_auth_tok) = NULL; + /** + * This format is inspired by OpenPGP; see RFC 2440 + * packet tag 1 + * + * Tag 1 identifier (1 byte) + * Max Tag 1 packet size (max 3 bytes) + * Version (1 byte) + * Key identifier (8 bytes; ECRYPTFS_SIG_SIZE) + * Cipher identifier (1 byte) + * Encrypted key size (arbitrary) + * + * 12 bytes minimum packet size + */ + if (unlikely(max_packet_size < 12)) { + printk(KERN_ERR "Invalid max packet size; must be >=12\n"); + rc = -EINVAL; + goto out; + } + if (data[(*packet_size)++] != ECRYPTFS_TAG_1_PACKET_TYPE) { + printk(KERN_ERR "Enter w/ first byte != 0x%.2x\n", + ECRYPTFS_TAG_1_PACKET_TYPE); + rc = -EINVAL; + goto out; + } + /* Released: wipe_auth_tok_list called in ecryptfs_parse_packet_set or + * at end of function upon failure */ + auth_tok_list_item = + kmem_cache_zalloc(ecryptfs_auth_tok_list_item_cache, + GFP_KERNEL); + if (!auth_tok_list_item) { + printk(KERN_ERR "Unable to allocate memory\n"); + rc = -ENOMEM; + goto out; + } + (*new_auth_tok) = &auth_tok_list_item->auth_tok; + rc = ecryptfs_parse_packet_length(&data[(*packet_size)], &body_size, + &length_size); + if (rc) { + printk(KERN_WARNING "Error parsing packet length; " + "rc = [%d]\n", rc); + goto out_free; + } + if (unlikely(body_size < (ECRYPTFS_SIG_SIZE + 2))) { + printk(KERN_WARNING "Invalid body size ([%td])\n", body_size); + rc = -EINVAL; + goto out_free; + } + (*packet_size) += length_size; + if (unlikely((*packet_size) + body_size > max_packet_size)) { + printk(KERN_WARNING "Packet size exceeds max\n"); + rc = -EINVAL; + goto out_free; + } + if (unlikely(data[(*packet_size)++] != 0x03)) { + printk(KERN_WARNING "Unknown version number [%d]\n", + data[(*packet_size) - 1]); + rc = -EINVAL; + goto out_free; + } + ecryptfs_to_hex((*new_auth_tok)->token.private_key.signature, + &data[(*packet_size)], ECRYPTFS_SIG_SIZE); + *packet_size += ECRYPTFS_SIG_SIZE; + /* This byte is skipped because the kernel does not need to + * know which public key encryption algorithm was used */ + (*packet_size)++; + (*new_auth_tok)->session_key.encrypted_key_size = + body_size - (ECRYPTFS_SIG_SIZE + 2); + if ((*new_auth_tok)->session_key.encrypted_key_size + > ECRYPTFS_MAX_ENCRYPTED_KEY_BYTES) { + printk(KERN_WARNING "Tag 1 packet contains key larger " + "than ECRYPTFS_MAX_ENCRYPTED_KEY_BYTES\n"); + rc = -EINVAL; + goto out_free; + } + memcpy((*new_auth_tok)->session_key.encrypted_key, + &data[(*packet_size)], (body_size - (ECRYPTFS_SIG_SIZE + 2))); + (*packet_size) += (*new_auth_tok)->session_key.encrypted_key_size; + (*new_auth_tok)->session_key.flags &= + ~ECRYPTFS_CONTAINS_DECRYPTED_KEY; + (*new_auth_tok)->session_key.flags |= + ECRYPTFS_CONTAINS_ENCRYPTED_KEY; + (*new_auth_tok)->token_type = ECRYPTFS_PRIVATE_KEY; + (*new_auth_tok)->flags = 0; + (*new_auth_tok)->session_key.flags &= + ~(ECRYPTFS_USERSPACE_SHOULD_TRY_TO_DECRYPT); + (*new_auth_tok)->session_key.flags &= + ~(ECRYPTFS_USERSPACE_SHOULD_TRY_TO_ENCRYPT); + list_add(&auth_tok_list_item->list, auth_tok_list); + goto out; +out_free: + (*new_auth_tok) = NULL; + memset(auth_tok_list_item, 0, + sizeof(struct ecryptfs_auth_tok_list_item)); + kmem_cache_free(ecryptfs_auth_tok_list_item_cache, + auth_tok_list_item); +out: + if (rc) + (*packet_size) = 0; + return rc; +} + +/** + * parse_tag_3_packet + * @crypt_stat: The cryptographic context to modify based on packet + * contents. + * @data: The raw bytes of the packet. + * @auth_tok_list: eCryptfs parses packets into authentication tokens; + * a new authentication token will be placed at the end + * of this list for this packet. + * @new_auth_tok: Pointer to a pointer to memory that this function + * allocates; sets the memory address of the pointer to + * NULL on error. This object is added to the + * auth_tok_list. + * @packet_size: This function writes the size of the parsed packet + * into this memory location; zero on error. + * @max_packet_size: maximum number of bytes to parse + * + * Returns zero on success; non-zero on error. + */ +static int +parse_tag_3_packet(struct ecryptfs_crypt_stat *crypt_stat, + unsigned char *data, struct list_head *auth_tok_list, + struct ecryptfs_auth_tok **new_auth_tok, + size_t *packet_size, size_t max_packet_size) +{ + size_t body_size; + struct ecryptfs_auth_tok_list_item *auth_tok_list_item; + size_t length_size; + int rc = 0; + + (*packet_size) = 0; + (*new_auth_tok) = NULL; + /** + *This format is inspired by OpenPGP; see RFC 2440 + * packet tag 3 + * + * Tag 3 identifier (1 byte) + * Max Tag 3 packet size (max 3 bytes) + * Version (1 byte) + * Cipher code (1 byte) + * S2K specifier (1 byte) + * Hash identifier (1 byte) + * Salt (ECRYPTFS_SALT_SIZE) + * Hash iterations (1 byte) + * Encrypted key (arbitrary) + * + * (ECRYPTFS_SALT_SIZE + 7) minimum packet size + */ + if (max_packet_size < (ECRYPTFS_SALT_SIZE + 7)) { + printk(KERN_ERR "Max packet size too large\n"); + rc = -EINVAL; + goto out; + } + if (data[(*packet_size)++] != ECRYPTFS_TAG_3_PACKET_TYPE) { + printk(KERN_ERR "First byte != 0x%.2x; invalid packet\n", + ECRYPTFS_TAG_3_PACKET_TYPE); + rc = -EINVAL; + goto out; + } + /* Released: wipe_auth_tok_list called in ecryptfs_parse_packet_set or + * at end of function upon failure */ + auth_tok_list_item = + kmem_cache_zalloc(ecryptfs_auth_tok_list_item_cache, GFP_KERNEL); + if (!auth_tok_list_item) { + printk(KERN_ERR "Unable to allocate memory\n"); + rc = -ENOMEM; + goto out; + } + (*new_auth_tok) = &auth_tok_list_item->auth_tok; + rc = ecryptfs_parse_packet_length(&data[(*packet_size)], &body_size, + &length_size); + if (rc) { + printk(KERN_WARNING "Error parsing packet length; rc = [%d]\n", + rc); + goto out_free; + } + if (unlikely(body_size < (ECRYPTFS_SALT_SIZE + 5))) { + printk(KERN_WARNING "Invalid body size ([%td])\n", body_size); + rc = -EINVAL; + goto out_free; + } + (*packet_size) += length_size; + if (unlikely((*packet_size) + body_size > max_packet_size)) { + printk(KERN_ERR "Packet size exceeds max\n"); + rc = -EINVAL; + goto out_free; + } + (*new_auth_tok)->session_key.encrypted_key_size = + (body_size - (ECRYPTFS_SALT_SIZE + 5)); + /* + * Although encrypted_key_size is copied into the + * encrypted_key[ECRYPTFS_MAX_ENCRYPTED_KEY_BYTES] buffer here, + * it later bounds operations on a smaller buffer: + * decrypt_passphrase_encrypted_session_key() sets decrypted_key_size = + * encrypted_key_size and decrypts into + * decrypted_key[ECRYPTFS_MAX_KEY_BYTES], then memcpy's into + * crypt_stat->key[ECRYPTFS_MAX_KEY_BYTES]. Limit to + * ECRYPTFS_MAX_KEY_BYTES to protect those smaller buffers. + */ + if ((*new_auth_tok)->session_key.encrypted_key_size + > ECRYPTFS_MAX_KEY_BYTES) { + printk(KERN_WARNING "Tag 3 packet contains key larger " + "than ECRYPTFS_MAX_KEY_BYTES\n"); + rc = -EINVAL; + goto out_free; + } + if (unlikely(data[(*packet_size)++] != 0x04)) { + printk(KERN_WARNING "Unknown version number [%d]\n", + data[(*packet_size) - 1]); + rc = -EINVAL; + goto out_free; + } + rc = ecryptfs_cipher_code_to_string(crypt_stat->cipher, + sizeof(crypt_stat->cipher), + (u16)data[(*packet_size)]); + if (rc) + goto out_free; + /* A little extra work to differentiate among the AES key + * sizes; see RFC2440 */ + switch(data[(*packet_size)++]) { + case RFC2440_CIPHER_AES_192: + crypt_stat->key_size = 24; + break; + default: + crypt_stat->key_size = + (*new_auth_tok)->session_key.encrypted_key_size; + } + rc = ecryptfs_init_crypt_ctx(crypt_stat); + if (rc) + goto out_free; + if (unlikely(data[(*packet_size)++] != 0x03)) { + printk(KERN_WARNING "Only S2K ID 3 is currently supported\n"); + rc = -ENOSYS; + goto out_free; + } + /* TODO: finish the hash mapping */ + switch (data[(*packet_size)++]) { + case 0x01: /* See RFC2440 for these numbers and their mappings */ + /* Choose MD5 */ + memcpy((*new_auth_tok)->token.password.salt, + &data[(*packet_size)], ECRYPTFS_SALT_SIZE); + (*packet_size) += ECRYPTFS_SALT_SIZE; + /* This conversion was taken straight from RFC2440 */ + (*new_auth_tok)->token.password.hash_iterations = + ((u32) 16 + (data[(*packet_size)] & 15)) + << ((data[(*packet_size)] >> 4) + 6); + (*packet_size)++; + /* Friendly reminder: + * (*new_auth_tok)->session_key.encrypted_key_size = + * (body_size - (ECRYPTFS_SALT_SIZE + 5)); */ + memcpy((*new_auth_tok)->session_key.encrypted_key, + &data[(*packet_size)], + (*new_auth_tok)->session_key.encrypted_key_size); + (*packet_size) += + (*new_auth_tok)->session_key.encrypted_key_size; + (*new_auth_tok)->session_key.flags &= + ~ECRYPTFS_CONTAINS_DECRYPTED_KEY; + (*new_auth_tok)->session_key.flags |= + ECRYPTFS_CONTAINS_ENCRYPTED_KEY; + (*new_auth_tok)->token.password.hash_algo = 0x01; /* MD5 */ + break; + default: + ecryptfs_printk(KERN_ERR, "Unsupported hash algorithm: " + "[%d]\n", data[(*packet_size) - 1]); + rc = -ENOSYS; + goto out_free; + } + (*new_auth_tok)->token_type = ECRYPTFS_PASSWORD; + /* TODO: Parametarize; we might actually want userspace to + * decrypt the session key. */ + (*new_auth_tok)->session_key.flags &= + ~(ECRYPTFS_USERSPACE_SHOULD_TRY_TO_DECRYPT); + (*new_auth_tok)->session_key.flags &= + ~(ECRYPTFS_USERSPACE_SHOULD_TRY_TO_ENCRYPT); + list_add(&auth_tok_list_item->list, auth_tok_list); + goto out; +out_free: + (*new_auth_tok) = NULL; + memset(auth_tok_list_item, 0, + sizeof(struct ecryptfs_auth_tok_list_item)); + kmem_cache_free(ecryptfs_auth_tok_list_item_cache, + auth_tok_list_item); +out: + if (rc) + (*packet_size) = 0; + return rc; +} + +/** + * parse_tag_11_packet + * @data: The raw bytes of the packet + * @contents: This function writes the data contents of the literal + * packet into this memory location + * @max_contents_bytes: The maximum number of bytes that this function + * is allowed to write into contents + * @tag_11_contents_size: This function writes the size of the parsed + * contents into this memory location; zero on + * error + * @packet_size: This function writes the size of the parsed packet + * into this memory location; zero on error + * @max_packet_size: maximum number of bytes to parse + * + * Returns zero on success; non-zero on error. + */ +static int +parse_tag_11_packet(unsigned char *data, unsigned char *contents, + size_t max_contents_bytes, size_t *tag_11_contents_size, + size_t *packet_size, size_t max_packet_size) +{ + size_t body_size; + size_t length_size; + int rc = 0; + + (*packet_size) = 0; + (*tag_11_contents_size) = 0; + /* This format is inspired by OpenPGP; see RFC 2440 + * packet tag 11 + * + * Tag 11 identifier (1 byte) + * Max Tag 11 packet size (max 3 bytes) + * Binary format specifier (1 byte) + * Filename length (1 byte) + * Filename ("_CONSOLE") (8 bytes) + * Modification date (4 bytes) + * Literal data (arbitrary) + * + * We need at least 16 bytes of data for the packet to even be + * valid. + */ + if (max_packet_size < 16) { + printk(KERN_ERR "Maximum packet size too small\n"); + rc = -EINVAL; + goto out; + } + if (data[(*packet_size)++] != ECRYPTFS_TAG_11_PACKET_TYPE) { + printk(KERN_WARNING "Invalid tag 11 packet format\n"); + rc = -EINVAL; + goto out; + } + rc = ecryptfs_parse_packet_length(&data[(*packet_size)], &body_size, + &length_size); + if (rc) { + printk(KERN_WARNING "Invalid tag 11 packet format\n"); + goto out; + } + if (body_size < 14) { + printk(KERN_WARNING "Invalid body size ([%td])\n", body_size); + rc = -EINVAL; + goto out; + } + (*packet_size) += length_size; + (*tag_11_contents_size) = (body_size - 14); + if (unlikely((*packet_size) + body_size > max_packet_size)) { + printk(KERN_ERR "Packet size exceeds max\n"); + rc = -EINVAL; + goto out; + } + if (unlikely((*tag_11_contents_size) > max_contents_bytes)) { + printk(KERN_ERR "Literal data section in tag 11 packet exceeds " + "expected size\n"); + rc = -EINVAL; + goto out; + } + if (data[(*packet_size)++] != 0x62) { + printk(KERN_WARNING "Unrecognizable packet\n"); + rc = -EINVAL; + goto out; + } + if (data[(*packet_size)++] != 0x08) { + printk(KERN_WARNING "Unrecognizable packet\n"); + rc = -EINVAL; + goto out; + } + (*packet_size) += 12; /* Ignore filename and modification date */ + memcpy(contents, &data[(*packet_size)], (*tag_11_contents_size)); + (*packet_size) += (*tag_11_contents_size); +out: + if (rc) { + (*packet_size) = 0; + (*tag_11_contents_size) = 0; + } + return rc; +} + +int ecryptfs_keyring_auth_tok_for_sig(struct key **auth_tok_key, + struct ecryptfs_auth_tok **auth_tok, + char *sig) +{ + int rc = 0; + + (*auth_tok_key) = request_key(&key_type_user, sig, NULL); + if (IS_ERR(*auth_tok_key)) { + (*auth_tok_key) = ecryptfs_get_encrypted_key(sig); + if (IS_ERR(*auth_tok_key)) { + printk(KERN_ERR "Could not find key with description: [%s]\n", + sig); + rc = process_request_key_err(PTR_ERR(*auth_tok_key)); + (*auth_tok_key) = NULL; + goto out; + } + } + down_write(&(*auth_tok_key)->sem); + rc = ecryptfs_verify_auth_tok_from_key(*auth_tok_key, auth_tok); + if (rc) { + up_write(&(*auth_tok_key)->sem); + key_put(*auth_tok_key); + (*auth_tok_key) = NULL; + goto out; + } +out: + return rc; +} + +/** + * decrypt_passphrase_encrypted_session_key - Decrypt the session key with the given auth_tok. + * @auth_tok: The passphrase authentication token to use to encrypt the FEK + * @crypt_stat: The cryptographic context + * + * Returns zero on success; non-zero error otherwise + */ +static int +decrypt_passphrase_encrypted_session_key(struct ecryptfs_auth_tok *auth_tok, + struct ecryptfs_crypt_stat *crypt_stat) +{ + struct scatterlist dst_sg[2]; + struct scatterlist src_sg[2]; + struct mutex *tfm_mutex; + struct crypto_skcipher *tfm; + struct skcipher_request *req = NULL; + int rc = 0; + + if (unlikely(ecryptfs_verbosity > 0)) { + ecryptfs_printk( + KERN_DEBUG, "Session key encryption key (size [%d]):\n", + auth_tok->token.password.session_key_encryption_key_bytes); + ecryptfs_dump_hex( + auth_tok->token.password.session_key_encryption_key, + auth_tok->token.password.session_key_encryption_key_bytes); + } + rc = ecryptfs_get_tfm_and_mutex_for_cipher_name(&tfm, &tfm_mutex, + crypt_stat->cipher); + if (unlikely(rc)) { + printk(KERN_ERR "Internal error whilst attempting to get " + "tfm and mutex for cipher name [%s]; rc = [%d]\n", + crypt_stat->cipher, rc); + goto out; + } + rc = virt_to_scatterlist(auth_tok->session_key.encrypted_key, + auth_tok->session_key.encrypted_key_size, + src_sg, 2); + if (rc < 1 || rc > 2) { + printk(KERN_ERR "Internal error whilst attempting to convert " + "auth_tok->session_key.encrypted_key to scatterlist; " + "expected rc = 1; got rc = [%d]. " + "auth_tok->session_key.encrypted_key_size = [%d]\n", rc, + auth_tok->session_key.encrypted_key_size); + goto out; + } + auth_tok->session_key.decrypted_key_size = + auth_tok->session_key.encrypted_key_size; + rc = virt_to_scatterlist(auth_tok->session_key.decrypted_key, + auth_tok->session_key.decrypted_key_size, + dst_sg, 2); + if (rc < 1 || rc > 2) { + printk(KERN_ERR "Internal error whilst attempting to convert " + "auth_tok->session_key.decrypted_key to scatterlist; " + "expected rc = 1; got rc = [%d]\n", rc); + goto out; + } + mutex_lock(tfm_mutex); + req = skcipher_request_alloc(tfm, GFP_KERNEL); + if (!req) { + mutex_unlock(tfm_mutex); + printk(KERN_ERR "%s: Out of kernel memory whilst attempting to " + "skcipher_request_alloc for %s\n", __func__, + crypto_skcipher_driver_name(tfm)); + rc = -ENOMEM; + goto out; + } + + skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, + NULL, NULL); + rc = crypto_skcipher_setkey( + tfm, auth_tok->token.password.session_key_encryption_key, + crypt_stat->key_size); + if (unlikely(rc < 0)) { + mutex_unlock(tfm_mutex); + printk(KERN_ERR "Error setting key for crypto context\n"); + rc = -EINVAL; + goto out; + } + skcipher_request_set_crypt(req, src_sg, dst_sg, + auth_tok->session_key.encrypted_key_size, + NULL); + rc = crypto_skcipher_decrypt(req); + mutex_unlock(tfm_mutex); + if (unlikely(rc)) { + printk(KERN_ERR "Error decrypting; rc = [%d]\n", rc); + goto out; + } + auth_tok->session_key.flags |= ECRYPTFS_CONTAINS_DECRYPTED_KEY; + memcpy(crypt_stat->key, auth_tok->session_key.decrypted_key, + auth_tok->session_key.decrypted_key_size); + crypt_stat->flags |= ECRYPTFS_KEY_VALID; + if (unlikely(ecryptfs_verbosity > 0)) { + ecryptfs_printk(KERN_DEBUG, "FEK of size [%zd]:\n", + crypt_stat->key_size); + ecryptfs_dump_hex(crypt_stat->key, + crypt_stat->key_size); + } +out: + skcipher_request_free(req); + return rc; +} + +/** + * ecryptfs_parse_packet_set + * @crypt_stat: The cryptographic context + * @src: Virtual address of region of memory containing the packets + * @src_size: Size of the packet set buffer + * @ecryptfs_dentry: The eCryptfs dentry associated with the packet set + * + * Get crypt_stat to have the file's session key if the requisite key + * is available to decrypt the session key. + * + * Returns Zero if a valid authentication token was retrieved and + * processed; negative value for file not encrypted or for error + * conditions. + */ +int ecryptfs_parse_packet_set(struct ecryptfs_crypt_stat *crypt_stat, + unsigned char *src, size_t src_size, + struct dentry *ecryptfs_dentry) +{ + size_t i = 0; + size_t next_packet_is_auth_tok_packet; + LIST_HEAD(auth_tok_list); + struct ecryptfs_auth_tok *matching_auth_tok; + struct ecryptfs_auth_tok *candidate_auth_tok; + char *candidate_auth_tok_sig; + size_t packet_size; + struct ecryptfs_auth_tok *new_auth_tok; + unsigned char sig_tmp_space[ECRYPTFS_SIG_SIZE]; + struct ecryptfs_auth_tok_list_item *auth_tok_list_item; + size_t tag_11_contents_size; + size_t tag_11_packet_size; + struct key *auth_tok_key = NULL; + int rc = 0; + + /* Parse the header to find as many packets as we can; these will be + * added the our &auth_tok_list */ + next_packet_is_auth_tok_packet = 1; + while (next_packet_is_auth_tok_packet) { + size_t max_packet_size; + + if (i >= src_size) + break; + max_packet_size = src_size - i; + + switch (src[i]) { + case ECRYPTFS_TAG_3_PACKET_TYPE: + rc = parse_tag_3_packet(crypt_stat, + (unsigned char *)&src[i], + &auth_tok_list, &new_auth_tok, + &packet_size, max_packet_size); + if (rc) { + ecryptfs_printk(KERN_ERR, "Error parsing " + "tag 3 packet\n"); + rc = -EIO; + goto out_wipe_list; + } + i += packet_size; + if (i > src_size) { + rc = -EIO; + goto out_wipe_list; + } + rc = parse_tag_11_packet((unsigned char *)&src[i], + sig_tmp_space, + ECRYPTFS_SIG_SIZE, + &tag_11_contents_size, + &tag_11_packet_size, + src_size - i); + if (rc) { + ecryptfs_printk(KERN_ERR, "No valid " + "(ecryptfs-specific) literal " + "packet containing " + "authentication token " + "signature found after " + "tag 3 packet\n"); + rc = -EIO; + goto out_wipe_list; + } + i += tag_11_packet_size; + if (i > src_size) { + rc = -EIO; + goto out_wipe_list; + } + if (ECRYPTFS_SIG_SIZE != tag_11_contents_size) { + ecryptfs_printk(KERN_ERR, "Expected " + "signature of size [%d]; " + "read size [%zd]\n", + ECRYPTFS_SIG_SIZE, + tag_11_contents_size); + rc = -EIO; + goto out_wipe_list; + } + ecryptfs_to_hex(new_auth_tok->token.password.signature, + sig_tmp_space, tag_11_contents_size); + crypt_stat->flags |= ECRYPTFS_ENCRYPTED; + break; + case ECRYPTFS_TAG_1_PACKET_TYPE: + rc = parse_tag_1_packet(crypt_stat, + (unsigned char *)&src[i], + &auth_tok_list, &new_auth_tok, + &packet_size, max_packet_size); + if (rc) { + ecryptfs_printk(KERN_ERR, "Error parsing " + "tag 1 packet\n"); + rc = -EIO; + goto out_wipe_list; + } + i += packet_size; + if (i > src_size) { + rc = -EIO; + goto out_wipe_list; + } + crypt_stat->flags |= ECRYPTFS_ENCRYPTED; + break; + case ECRYPTFS_TAG_11_PACKET_TYPE: + ecryptfs_printk(KERN_WARNING, "Invalid packet set " + "(Tag 11 not allowed by itself)\n"); + rc = -EIO; + goto out_wipe_list; + default: + ecryptfs_printk(KERN_DEBUG, "No packet at offset [%zd] " + "of the file header; hex value of " + "character is [0x%.2x]\n", i, src[i]); + next_packet_is_auth_tok_packet = 0; + } + } + if (list_empty(&auth_tok_list)) { + printk(KERN_ERR "The lower file appears to be a non-encrypted " + "eCryptfs file; this is not supported in this version " + "of the eCryptfs kernel module\n"); + rc = -EINVAL; + goto out; + } + /* auth_tok_list contains the set of authentication tokens + * parsed from the metadata. We need to find a matching + * authentication token that has the secret component(s) + * necessary to decrypt the EFEK in the auth_tok parsed from + * the metadata. There may be several potential matches, but + * just one will be sufficient to decrypt to get the FEK. */ +find_next_matching_auth_tok: + list_for_each_entry(auth_tok_list_item, &auth_tok_list, list) { + candidate_auth_tok = &auth_tok_list_item->auth_tok; + if (unlikely(ecryptfs_verbosity > 0)) { + ecryptfs_printk(KERN_DEBUG, + "Considering candidate auth tok:\n"); + ecryptfs_dump_auth_tok(candidate_auth_tok); + } + rc = ecryptfs_get_auth_tok_sig(&candidate_auth_tok_sig, + candidate_auth_tok); + if (rc) { + printk(KERN_ERR + "Unrecognized candidate auth tok type: [%d]\n", + candidate_auth_tok->token_type); + rc = -EINVAL; + goto out_wipe_list; + } + rc = ecryptfs_find_auth_tok_for_sig(&auth_tok_key, + &matching_auth_tok, + crypt_stat->mount_crypt_stat, + candidate_auth_tok_sig); + if (!rc) + goto found_matching_auth_tok; + } + ecryptfs_printk(KERN_ERR, + "Could not find a usable authentication token\n"); + rc = -EIO; + goto out_wipe_list; +found_matching_auth_tok: + if (candidate_auth_tok->token_type == ECRYPTFS_PRIVATE_KEY) { + memcpy(&(candidate_auth_tok->token.private_key), + &(matching_auth_tok->token.private_key), + sizeof(struct ecryptfs_private_key)); + up_write(&(auth_tok_key->sem)); + key_put(auth_tok_key); + rc = decrypt_pki_encrypted_session_key(candidate_auth_tok, + crypt_stat); + } else if (candidate_auth_tok->token_type == ECRYPTFS_PASSWORD) { + memcpy(&(candidate_auth_tok->token.password), + &(matching_auth_tok->token.password), + sizeof(struct ecryptfs_password)); + up_write(&(auth_tok_key->sem)); + key_put(auth_tok_key); + rc = decrypt_passphrase_encrypted_session_key( + candidate_auth_tok, crypt_stat); + } else { + up_write(&(auth_tok_key->sem)); + key_put(auth_tok_key); + rc = -EINVAL; + } + if (rc) { + struct ecryptfs_auth_tok_list_item *auth_tok_list_item_tmp; + + ecryptfs_printk(KERN_WARNING, "Error decrypting the " + "session key for authentication token with sig " + "[%.*s]; rc = [%d]. Removing auth tok " + "candidate from the list and searching for " + "the next match.\n", ECRYPTFS_SIG_SIZE_HEX, + candidate_auth_tok_sig, rc); + list_for_each_entry_safe(auth_tok_list_item, + auth_tok_list_item_tmp, + &auth_tok_list, list) { + if (candidate_auth_tok + == &auth_tok_list_item->auth_tok) { + list_del(&auth_tok_list_item->list); + kmem_cache_free( + ecryptfs_auth_tok_list_item_cache, + auth_tok_list_item); + goto find_next_matching_auth_tok; + } + } + BUG(); + } + rc = ecryptfs_compute_root_iv(crypt_stat); + if (rc) { + ecryptfs_printk(KERN_ERR, "Error computing " + "the root IV\n"); + goto out_wipe_list; + } + rc = ecryptfs_init_crypt_ctx(crypt_stat); + if (rc) { + ecryptfs_printk(KERN_ERR, "Error initializing crypto " + "context for cipher [%s]; rc = [%d]\n", + crypt_stat->cipher, rc); + } +out_wipe_list: + wipe_auth_tok_list(&auth_tok_list); +out: + return rc; +} + +static int +pki_encrypt_session_key(struct key *auth_tok_key, + struct ecryptfs_auth_tok *auth_tok, + struct ecryptfs_crypt_stat *crypt_stat, + struct ecryptfs_key_record *key_rec) +{ + struct ecryptfs_msg_ctx *msg_ctx = NULL; + char *payload = NULL; + size_t payload_len = 0; + struct ecryptfs_message *msg; + int rc; + + rc = write_tag_66_packet(auth_tok->token.private_key.signature, + ecryptfs_code_for_cipher_string( + crypt_stat->cipher, + crypt_stat->key_size), + crypt_stat, &payload, &payload_len); + up_write(&(auth_tok_key->sem)); + key_put(auth_tok_key); + if (rc) { + ecryptfs_printk(KERN_ERR, "Error generating tag 66 packet\n"); + goto out; + } + rc = ecryptfs_send_message(payload, payload_len, &msg_ctx); + if (rc) { + ecryptfs_printk(KERN_ERR, "Error sending message to " + "ecryptfsd: %d\n", rc); + goto out; + } + rc = ecryptfs_wait_for_response(msg_ctx, &msg); + if (rc) { + ecryptfs_printk(KERN_ERR, "Failed to receive tag 67 packet " + "from the user space daemon\n"); + rc = -EIO; + goto out; + } + rc = parse_tag_67_packet(key_rec, msg); + if (rc) + ecryptfs_printk(KERN_ERR, "Error parsing tag 67 packet\n"); + kfree(msg); +out: + kfree(payload); + return rc; +} +/** + * write_tag_1_packet - Write an RFC2440-compatible tag 1 (public key) packet + * @dest: Buffer into which to write the packet + * @remaining_bytes: Maximum number of bytes that can be writtn + * @auth_tok_key: The authentication token key to unlock and put when done with + * @auth_tok + * @auth_tok: The authentication token used for generating the tag 1 packet + * @crypt_stat: The cryptographic context + * @key_rec: The key record struct for the tag 1 packet + * @packet_size: This function will write the number of bytes that end + * up constituting the packet; set to zero on error + * + * Returns zero on success; non-zero on error. + */ +static int +write_tag_1_packet(char *dest, size_t *remaining_bytes, + struct key *auth_tok_key, struct ecryptfs_auth_tok *auth_tok, + struct ecryptfs_crypt_stat *crypt_stat, + struct ecryptfs_key_record *key_rec, size_t *packet_size) +{ + size_t i; + size_t encrypted_session_key_valid = 0; + size_t packet_size_length; + size_t max_packet_size; + int rc = 0; + + (*packet_size) = 0; + ecryptfs_from_hex(key_rec->sig, auth_tok->token.private_key.signature, + ECRYPTFS_SIG_SIZE); + encrypted_session_key_valid = 0; + for (i = 0; i < crypt_stat->key_size; i++) + encrypted_session_key_valid |= + auth_tok->session_key.encrypted_key[i]; + if (encrypted_session_key_valid) { + memcpy(key_rec->enc_key, + auth_tok->session_key.encrypted_key, + auth_tok->session_key.encrypted_key_size); + up_write(&(auth_tok_key->sem)); + key_put(auth_tok_key); + goto encrypted_session_key_set; + } + if (auth_tok->session_key.encrypted_key_size == 0) + auth_tok->session_key.encrypted_key_size = + auth_tok->token.private_key.key_size; + rc = pki_encrypt_session_key(auth_tok_key, auth_tok, crypt_stat, + key_rec); + if (rc) { + printk(KERN_ERR "Failed to encrypt session key via a key " + "module; rc = [%d]\n", rc); + goto out; + } + if (ecryptfs_verbosity > 0) { + ecryptfs_printk(KERN_DEBUG, "Encrypted key:\n"); + ecryptfs_dump_hex(key_rec->enc_key, key_rec->enc_key_size); + } +encrypted_session_key_set: + /* This format is inspired by OpenPGP; see RFC 2440 + * packet tag 1 */ + max_packet_size = (1 /* Tag 1 identifier */ + + 3 /* Max Tag 1 packet size */ + + 1 /* Version */ + + ECRYPTFS_SIG_SIZE /* Key identifier */ + + 1 /* Cipher identifier */ + + key_rec->enc_key_size); /* Encrypted key size */ + if (max_packet_size > (*remaining_bytes)) { + printk(KERN_ERR "Packet length larger than maximum allowable; " + "need up to [%td] bytes, but there are only [%td] " + "available\n", max_packet_size, (*remaining_bytes)); + rc = -EINVAL; + goto out; + } + dest[(*packet_size)++] = ECRYPTFS_TAG_1_PACKET_TYPE; + rc = ecryptfs_write_packet_length(&dest[(*packet_size)], + (max_packet_size - 4), + &packet_size_length); + if (rc) { + ecryptfs_printk(KERN_ERR, "Error generating tag 1 packet " + "header; cannot generate packet length\n"); + goto out; + } + (*packet_size) += packet_size_length; + dest[(*packet_size)++] = 0x03; /* version 3 */ + memcpy(&dest[(*packet_size)], key_rec->sig, ECRYPTFS_SIG_SIZE); + (*packet_size) += ECRYPTFS_SIG_SIZE; + dest[(*packet_size)++] = RFC2440_CIPHER_RSA; + memcpy(&dest[(*packet_size)], key_rec->enc_key, + key_rec->enc_key_size); + (*packet_size) += key_rec->enc_key_size; +out: + if (rc) + (*packet_size) = 0; + else + (*remaining_bytes) -= (*packet_size); + return rc; +} + +/** + * write_tag_11_packet + * @dest: Target into which Tag 11 packet is to be written + * @remaining_bytes: Maximum packet length + * @contents: Byte array of contents to copy in + * @contents_length: Number of bytes in contents + * @packet_length: Length of the Tag 11 packet written; zero on error + * + * Returns zero on success; non-zero on error. + */ +static int +write_tag_11_packet(char *dest, size_t *remaining_bytes, char *contents, + size_t contents_length, size_t *packet_length) +{ + size_t packet_size_length; + size_t max_packet_size; + int rc = 0; + + (*packet_length) = 0; + /* This format is inspired by OpenPGP; see RFC 2440 + * packet tag 11 */ + max_packet_size = (1 /* Tag 11 identifier */ + + 3 /* Max Tag 11 packet size */ + + 1 /* Binary format specifier */ + + 1 /* Filename length */ + + 8 /* Filename ("_CONSOLE") */ + + 4 /* Modification date */ + + contents_length); /* Literal data */ + if (max_packet_size > (*remaining_bytes)) { + printk(KERN_ERR "Packet length larger than maximum allowable; " + "need up to [%td] bytes, but there are only [%td] " + "available\n", max_packet_size, (*remaining_bytes)); + rc = -EINVAL; + goto out; + } + dest[(*packet_length)++] = ECRYPTFS_TAG_11_PACKET_TYPE; + rc = ecryptfs_write_packet_length(&dest[(*packet_length)], + (max_packet_size - 4), + &packet_size_length); + if (rc) { + printk(KERN_ERR "Error generating tag 11 packet header; cannot " + "generate packet length. rc = [%d]\n", rc); + goto out; + } + (*packet_length) += packet_size_length; + dest[(*packet_length)++] = 0x62; /* binary data format specifier */ + dest[(*packet_length)++] = 8; + memcpy(&dest[(*packet_length)], "_CONSOLE", 8); + (*packet_length) += 8; + memset(&dest[(*packet_length)], 0x00, 4); + (*packet_length) += 4; + memcpy(&dest[(*packet_length)], contents, contents_length); + (*packet_length) += contents_length; + out: + if (rc) + (*packet_length) = 0; + else + (*remaining_bytes) -= (*packet_length); + return rc; +} + +/** + * write_tag_3_packet + * @dest: Buffer into which to write the packet + * @remaining_bytes: Maximum number of bytes that can be written + * @auth_tok: Authentication token + * @crypt_stat: The cryptographic context + * @key_rec: encrypted key + * @packet_size: This function will write the number of bytes that end + * up constituting the packet; set to zero on error + * + * Returns zero on success; non-zero on error. + */ +static int +write_tag_3_packet(char *dest, size_t *remaining_bytes, + struct ecryptfs_auth_tok *auth_tok, + struct ecryptfs_crypt_stat *crypt_stat, + struct ecryptfs_key_record *key_rec, size_t *packet_size) +{ + size_t i; + size_t encrypted_session_key_valid = 0; + char session_key_encryption_key[ECRYPTFS_MAX_KEY_BYTES]; + struct scatterlist dst_sg[2]; + struct scatterlist src_sg[2]; + struct mutex *tfm_mutex = NULL; + u8 cipher_code; + size_t packet_size_length; + size_t max_packet_size; + struct ecryptfs_mount_crypt_stat *mount_crypt_stat = + crypt_stat->mount_crypt_stat; + struct crypto_skcipher *tfm; + struct skcipher_request *req; + int rc = 0; + + (*packet_size) = 0; + ecryptfs_from_hex(key_rec->sig, auth_tok->token.password.signature, + ECRYPTFS_SIG_SIZE); + rc = ecryptfs_get_tfm_and_mutex_for_cipher_name(&tfm, &tfm_mutex, + crypt_stat->cipher); + if (unlikely(rc)) { + printk(KERN_ERR "Internal error whilst attempting to get " + "tfm and mutex for cipher name [%s]; rc = [%d]\n", + crypt_stat->cipher, rc); + goto out; + } + if (mount_crypt_stat->global_default_cipher_key_size == 0) { + printk(KERN_WARNING "No key size specified at mount; " + "defaulting to [%d]\n", + crypto_skcipher_max_keysize(tfm)); + mount_crypt_stat->global_default_cipher_key_size = + crypto_skcipher_max_keysize(tfm); + } + if (crypt_stat->key_size == 0) + crypt_stat->key_size = + mount_crypt_stat->global_default_cipher_key_size; + if (auth_tok->session_key.encrypted_key_size == 0) + auth_tok->session_key.encrypted_key_size = + crypt_stat->key_size; + if (crypt_stat->key_size == 24 + && strcmp("aes", crypt_stat->cipher) == 0) { + memset((crypt_stat->key + 24), 0, 8); + auth_tok->session_key.encrypted_key_size = 32; + } else + auth_tok->session_key.encrypted_key_size = crypt_stat->key_size; + key_rec->enc_key_size = + auth_tok->session_key.encrypted_key_size; + encrypted_session_key_valid = 0; + for (i = 0; i < auth_tok->session_key.encrypted_key_size; i++) + encrypted_session_key_valid |= + auth_tok->session_key.encrypted_key[i]; + if (encrypted_session_key_valid) { + ecryptfs_printk(KERN_DEBUG, "encrypted_session_key_valid != 0; " + "using auth_tok->session_key.encrypted_key, " + "where key_rec->enc_key_size = [%zd]\n", + key_rec->enc_key_size); + memcpy(key_rec->enc_key, + auth_tok->session_key.encrypted_key, + key_rec->enc_key_size); + goto encrypted_session_key_set; + } + if (auth_tok->token.password.flags & + ECRYPTFS_SESSION_KEY_ENCRYPTION_KEY_SET) { + ecryptfs_printk(KERN_DEBUG, "Using previously generated " + "session key encryption key of size [%d]\n", + auth_tok->token.password. + session_key_encryption_key_bytes); + memcpy(session_key_encryption_key, + auth_tok->token.password.session_key_encryption_key, + crypt_stat->key_size); + ecryptfs_printk(KERN_DEBUG, + "Cached session key encryption key:\n"); + if (ecryptfs_verbosity > 0) + ecryptfs_dump_hex(session_key_encryption_key, 16); + } + if (unlikely(ecryptfs_verbosity > 0)) { + ecryptfs_printk(KERN_DEBUG, "Session key encryption key:\n"); + ecryptfs_dump_hex(session_key_encryption_key, 16); + } + rc = virt_to_scatterlist(crypt_stat->key, key_rec->enc_key_size, + src_sg, 2); + if (rc < 1 || rc > 2) { + ecryptfs_printk(KERN_ERR, "Error generating scatterlist " + "for crypt_stat session key; expected rc = 1; " + "got rc = [%d]. key_rec->enc_key_size = [%zd]\n", + rc, key_rec->enc_key_size); + rc = -ENOMEM; + goto out; + } + rc = virt_to_scatterlist(key_rec->enc_key, key_rec->enc_key_size, + dst_sg, 2); + if (rc < 1 || rc > 2) { + ecryptfs_printk(KERN_ERR, "Error generating scatterlist " + "for crypt_stat encrypted session key; " + "expected rc = 1; got rc = [%d]. " + "key_rec->enc_key_size = [%zd]\n", rc, + key_rec->enc_key_size); + rc = -ENOMEM; + goto out; + } + mutex_lock(tfm_mutex); + rc = crypto_skcipher_setkey(tfm, session_key_encryption_key, + crypt_stat->key_size); + if (rc < 0) { + mutex_unlock(tfm_mutex); + ecryptfs_printk(KERN_ERR, "Error setting key for crypto " + "context; rc = [%d]\n", rc); + goto out; + } + + req = skcipher_request_alloc(tfm, GFP_KERNEL); + if (!req) { + mutex_unlock(tfm_mutex); + ecryptfs_printk(KERN_ERR, "Out of kernel memory whilst " + "attempting to skcipher_request_alloc for " + "%s\n", crypto_skcipher_driver_name(tfm)); + rc = -ENOMEM; + goto out; + } + + skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, + NULL, NULL); + + rc = 0; + ecryptfs_printk(KERN_DEBUG, "Encrypting [%zd] bytes of the key\n", + crypt_stat->key_size); + skcipher_request_set_crypt(req, src_sg, dst_sg, + (*key_rec).enc_key_size, NULL); + rc = crypto_skcipher_encrypt(req); + mutex_unlock(tfm_mutex); + skcipher_request_free(req); + if (rc) { + printk(KERN_ERR "Error encrypting; rc = [%d]\n", rc); + goto out; + } + ecryptfs_printk(KERN_DEBUG, "This should be the encrypted key:\n"); + if (ecryptfs_verbosity > 0) { + ecryptfs_printk(KERN_DEBUG, "EFEK of size [%zd]:\n", + key_rec->enc_key_size); + ecryptfs_dump_hex(key_rec->enc_key, + key_rec->enc_key_size); + } +encrypted_session_key_set: + /* This format is inspired by OpenPGP; see RFC 2440 + * packet tag 3 */ + max_packet_size = (1 /* Tag 3 identifier */ + + 3 /* Max Tag 3 packet size */ + + 1 /* Version */ + + 1 /* Cipher code */ + + 1 /* S2K specifier */ + + 1 /* Hash identifier */ + + ECRYPTFS_SALT_SIZE /* Salt */ + + 1 /* Hash iterations */ + + key_rec->enc_key_size); /* Encrypted key size */ + if (max_packet_size > (*remaining_bytes)) { + printk(KERN_ERR "Packet too large; need up to [%td] bytes, but " + "there are only [%td] available\n", max_packet_size, + (*remaining_bytes)); + rc = -EINVAL; + goto out; + } + dest[(*packet_size)++] = ECRYPTFS_TAG_3_PACKET_TYPE; + /* Chop off the Tag 3 identifier(1) and Tag 3 packet size(3) + * to get the number of octets in the actual Tag 3 packet */ + rc = ecryptfs_write_packet_length(&dest[(*packet_size)], + (max_packet_size - 4), + &packet_size_length); + if (rc) { + printk(KERN_ERR "Error generating tag 3 packet header; cannot " + "generate packet length. rc = [%d]\n", rc); + goto out; + } + (*packet_size) += packet_size_length; + dest[(*packet_size)++] = 0x04; /* version 4 */ + /* TODO: Break from RFC2440 so that arbitrary ciphers can be + * specified with strings */ + cipher_code = ecryptfs_code_for_cipher_string(crypt_stat->cipher, + crypt_stat->key_size); + if (cipher_code == 0) { + ecryptfs_printk(KERN_WARNING, "Unable to generate code for " + "cipher [%s]\n", crypt_stat->cipher); + rc = -EINVAL; + goto out; + } + dest[(*packet_size)++] = cipher_code; + dest[(*packet_size)++] = 0x03; /* S2K */ + dest[(*packet_size)++] = 0x01; /* MD5 (TODO: parameterize) */ + memcpy(&dest[(*packet_size)], auth_tok->token.password.salt, + ECRYPTFS_SALT_SIZE); + (*packet_size) += ECRYPTFS_SALT_SIZE; /* salt */ + dest[(*packet_size)++] = 0x60; /* hash iterations (65536) */ + memcpy(&dest[(*packet_size)], key_rec->enc_key, + key_rec->enc_key_size); + (*packet_size) += key_rec->enc_key_size; +out: + if (rc) + (*packet_size) = 0; + else + (*remaining_bytes) -= (*packet_size); + return rc; +} + +struct kmem_cache *ecryptfs_key_record_cache; + +/** + * ecryptfs_generate_key_packet_set + * @dest_base: Virtual address from which to write the key record set + * @crypt_stat: The cryptographic context from which the + * authentication tokens will be retrieved + * @ecryptfs_dentry: The dentry, used to retrieve the mount crypt stat + * for the global parameters + * @len: The amount written + * @max: The maximum amount of data allowed to be written + * + * Generates a key packet set and writes it to the virtual address + * passed in. + * + * Returns zero on success; non-zero on error. + */ +int +ecryptfs_generate_key_packet_set(char *dest_base, + struct ecryptfs_crypt_stat *crypt_stat, + struct dentry *ecryptfs_dentry, size_t *len, + size_t max) +{ + struct ecryptfs_auth_tok *auth_tok; + struct key *auth_tok_key = NULL; + struct ecryptfs_mount_crypt_stat *mount_crypt_stat = + &ecryptfs_superblock_to_private( + ecryptfs_dentry->d_sb)->mount_crypt_stat; + size_t written; + struct ecryptfs_key_record *key_rec; + struct ecryptfs_key_sig *key_sig; + int rc = 0; + + (*len) = 0; + mutex_lock(&crypt_stat->keysig_list_mutex); + key_rec = kmem_cache_alloc(ecryptfs_key_record_cache, GFP_KERNEL); + if (!key_rec) { + rc = -ENOMEM; + goto out; + } + list_for_each_entry(key_sig, &crypt_stat->keysig_list, + crypt_stat_list) { + memset(key_rec, 0, sizeof(*key_rec)); + rc = ecryptfs_find_global_auth_tok_for_sig(&auth_tok_key, + &auth_tok, + mount_crypt_stat, + key_sig->keysig); + if (rc) { + printk(KERN_WARNING "Unable to retrieve auth tok with " + "sig = [%s]\n", key_sig->keysig); + rc = process_find_global_auth_tok_for_sig_err(rc); + goto out_free; + } + if (auth_tok->token_type == ECRYPTFS_PASSWORD) { + rc = write_tag_3_packet((dest_base + (*len)), + &max, auth_tok, + crypt_stat, key_rec, + &written); + up_write(&(auth_tok_key->sem)); + key_put(auth_tok_key); + if (rc) { + ecryptfs_printk(KERN_WARNING, "Error " + "writing tag 3 packet\n"); + goto out_free; + } + (*len) += written; + /* Write auth tok signature packet */ + rc = write_tag_11_packet((dest_base + (*len)), &max, + key_rec->sig, + ECRYPTFS_SIG_SIZE, &written); + if (rc) { + ecryptfs_printk(KERN_ERR, "Error writing " + "auth tok signature packet\n"); + goto out_free; + } + (*len) += written; + } else if (auth_tok->token_type == ECRYPTFS_PRIVATE_KEY) { + rc = write_tag_1_packet(dest_base + (*len), &max, + auth_tok_key, auth_tok, + crypt_stat, key_rec, &written); + if (rc) { + ecryptfs_printk(KERN_WARNING, "Error " + "writing tag 1 packet\n"); + goto out_free; + } + (*len) += written; + } else { + up_write(&(auth_tok_key->sem)); + key_put(auth_tok_key); + ecryptfs_printk(KERN_WARNING, "Unsupported " + "authentication token type\n"); + rc = -EINVAL; + goto out_free; + } + } + if (likely(max > 0)) { + dest_base[(*len)] = 0x00; + } else { + ecryptfs_printk(KERN_ERR, "Error writing boundary byte\n"); + rc = -EIO; + } +out_free: + kmem_cache_free(ecryptfs_key_record_cache, key_rec); +out: + if (rc) + (*len) = 0; + mutex_unlock(&crypt_stat->keysig_list_mutex); + return rc; +} + +struct kmem_cache *ecryptfs_key_sig_cache; + +int ecryptfs_add_keysig(struct ecryptfs_crypt_stat *crypt_stat, char *sig) +{ + struct ecryptfs_key_sig *new_key_sig; + + new_key_sig = kmem_cache_alloc(ecryptfs_key_sig_cache, GFP_KERNEL); + if (!new_key_sig) + return -ENOMEM; + + strscpy(new_key_sig->keysig, sig); + /* Caller must hold keysig_list_mutex */ + list_add(&new_key_sig->crypt_stat_list, &crypt_stat->keysig_list); + + return 0; +} + +struct kmem_cache *ecryptfs_global_auth_tok_cache; + +int +ecryptfs_add_global_auth_tok(struct ecryptfs_mount_crypt_stat *mount_crypt_stat, + char *sig, u32 global_auth_tok_flags) +{ + struct ecryptfs_global_auth_tok *new_auth_tok; + + new_auth_tok = kmem_cache_zalloc(ecryptfs_global_auth_tok_cache, + GFP_KERNEL); + if (!new_auth_tok) + return -ENOMEM; + + strscpy(new_auth_tok->sig, sig); + new_auth_tok->flags = global_auth_tok_flags; + mutex_lock(&mount_crypt_stat->global_auth_tok_list_mutex); + list_add(&new_auth_tok->mount_crypt_stat_list, + &mount_crypt_stat->global_auth_tok_list); + mutex_unlock(&mount_crypt_stat->global_auth_tok_list_mutex); + return 0; +} + diff --git a/fs/ecryptfs/kthread.c b/fs/ecryptfs/kthread.c new file mode 100644 index 000000000..ae4cb4e2e --- /dev/null +++ b/fs/ecryptfs/kthread.c @@ -0,0 +1,159 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * eCryptfs: Linux filesystem encryption layer + * + * Copyright (C) 2008 International Business Machines Corp. + * Author(s): Michael A. Halcrow + */ + +#include +#include +#include +#include +#include +#include "ecryptfs_kernel.h" + +struct ecryptfs_open_req { + struct file **lower_file; + struct path path; + struct completion done; + struct list_head kthread_ctl_list; +}; + +static struct ecryptfs_kthread_ctl { +#define ECRYPTFS_KTHREAD_ZOMBIE 0x00000001 + u32 flags; + struct mutex mux; + struct list_head req_list; + wait_queue_head_t wait; +} ecryptfs_kthread_ctl; + +static struct task_struct *ecryptfs_kthread; + +/** + * ecryptfs_threadfn + * @ignored: ignored + * + * The eCryptfs kernel thread that has the responsibility of getting + * the lower file with RW permissions. + * + * Returns zero on success; non-zero otherwise + */ +static int ecryptfs_threadfn(void *ignored) +{ + set_freezable(); + while (1) { + struct ecryptfs_open_req *req; + + wait_event_freezable( + ecryptfs_kthread_ctl.wait, + (!list_empty(&ecryptfs_kthread_ctl.req_list) + || kthread_should_stop())); + mutex_lock(&ecryptfs_kthread_ctl.mux); + if (ecryptfs_kthread_ctl.flags & ECRYPTFS_KTHREAD_ZOMBIE) { + mutex_unlock(&ecryptfs_kthread_ctl.mux); + goto out; + } + while (!list_empty(&ecryptfs_kthread_ctl.req_list)) { + req = list_first_entry(&ecryptfs_kthread_ctl.req_list, + struct ecryptfs_open_req, + kthread_ctl_list); + list_del(&req->kthread_ctl_list); + *req->lower_file = dentry_open(&req->path, + (O_RDWR | O_LARGEFILE), current_cred()); + complete(&req->done); + } + mutex_unlock(&ecryptfs_kthread_ctl.mux); + } +out: + return 0; +} + +int __init ecryptfs_init_kthread(void) +{ + int rc = 0; + + mutex_init(&ecryptfs_kthread_ctl.mux); + init_waitqueue_head(&ecryptfs_kthread_ctl.wait); + INIT_LIST_HEAD(&ecryptfs_kthread_ctl.req_list); + ecryptfs_kthread = kthread_run(&ecryptfs_threadfn, NULL, + "ecryptfs-kthread"); + if (IS_ERR(ecryptfs_kthread)) { + rc = PTR_ERR(ecryptfs_kthread); + printk(KERN_ERR "%s: Failed to create kernel thread; rc = [%d]" + "\n", __func__, rc); + } + return rc; +} + +void ecryptfs_destroy_kthread(void) +{ + struct ecryptfs_open_req *req, *tmp; + + mutex_lock(&ecryptfs_kthread_ctl.mux); + ecryptfs_kthread_ctl.flags |= ECRYPTFS_KTHREAD_ZOMBIE; + list_for_each_entry_safe(req, tmp, &ecryptfs_kthread_ctl.req_list, + kthread_ctl_list) { + list_del(&req->kthread_ctl_list); + *req->lower_file = ERR_PTR(-EIO); + complete(&req->done); + } + mutex_unlock(&ecryptfs_kthread_ctl.mux); + kthread_stop(ecryptfs_kthread); + wake_up(&ecryptfs_kthread_ctl.wait); +} + +/** + * ecryptfs_privileged_open + * @lower_file: Result of dentry_open by root on lower dentry + * @lower_dentry: Lower dentry for file to open + * @lower_mnt: Lower vfsmount for file to open + * @cred: credential to use for this call + * + * This function gets a r/w file opened against the lower dentry. + * + * Returns zero on success; non-zero otherwise + */ +int ecryptfs_privileged_open(struct file **lower_file, + struct dentry *lower_dentry, + struct vfsmount *lower_mnt, + const struct cred *cred) +{ + struct ecryptfs_open_req req; + int flags = O_LARGEFILE; + int rc = 0; + + init_completion(&req.done); + req.lower_file = lower_file; + req.path.dentry = lower_dentry; + req.path.mnt = lower_mnt; + + /* Corresponding dput() and mntput() are done when the + * lower file is fput() when all eCryptfs files for the inode are + * released. */ + flags |= IS_RDONLY(d_inode(lower_dentry)) ? O_RDONLY : O_RDWR; + (*lower_file) = dentry_open(&req.path, flags, cred); + if (!IS_ERR(*lower_file)) + goto out; + if ((flags & O_ACCMODE) == O_RDONLY) { + rc = PTR_ERR((*lower_file)); + goto out; + } + mutex_lock(&ecryptfs_kthread_ctl.mux); + if (ecryptfs_kthread_ctl.flags & ECRYPTFS_KTHREAD_ZOMBIE) { + rc = -EIO; + mutex_unlock(&ecryptfs_kthread_ctl.mux); + printk(KERN_ERR "%s: We are in the middle of shutting down; " + "aborting privileged request to open lower file\n", + __func__); + goto out; + } + list_add_tail(&req.kthread_ctl_list, &ecryptfs_kthread_ctl.req_list); + mutex_unlock(&ecryptfs_kthread_ctl.mux); + wake_up(&ecryptfs_kthread_ctl.wait); + wait_for_completion(&req.done); + if (IS_ERR(*lower_file)) + rc = PTR_ERR(*lower_file); +out: + return rc; +} diff --git a/fs/ecryptfs/main.c b/fs/ecryptfs/main.c new file mode 100644 index 000000000..5f37cddb9 --- /dev/null +++ b/fs/ecryptfs/main.c @@ -0,0 +1,895 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * eCryptfs: Linux filesystem encryption layer + * + * Copyright (C) 1997-2003 Erez Zadok + * Copyright (C) 2001-2003 Stony Brook University + * Copyright (C) 2004-2007 International Business Machines Corp. + * Author(s): Michael A. Halcrow + * Michael C. Thompson + * Tyler Hicks + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ecryptfs_kernel.h" + +/* + * Module parameter that defines the ecryptfs_verbosity level. + */ +int ecryptfs_verbosity = 0; + +module_param(ecryptfs_verbosity, int, 0); +MODULE_PARM_DESC(ecryptfs_verbosity, + "Initial verbosity level (0 or 1; defaults to " + "0, which is Quiet)"); + +/* + * Module parameter that defines the number of message buffer elements + */ +unsigned int ecryptfs_message_buf_len = ECRYPTFS_DEFAULT_MSG_CTX_ELEMS; + +module_param(ecryptfs_message_buf_len, uint, 0); +MODULE_PARM_DESC(ecryptfs_message_buf_len, + "Number of message buffer elements"); + +/* + * Module parameter that defines the maximum guaranteed amount of time to wait + * for a response from ecryptfsd. The actual sleep time will be, more than + * likely, a small amount greater than this specified value, but only less if + * the message successfully arrives. + */ +signed long ecryptfs_message_wait_timeout = ECRYPTFS_MAX_MSG_CTX_TTL / HZ; + +module_param(ecryptfs_message_wait_timeout, long, 0); +MODULE_PARM_DESC(ecryptfs_message_wait_timeout, + "Maximum number of seconds that an operation will " + "sleep while waiting for a message response from " + "userspace"); + +/* + * Module parameter that is an estimate of the maximum number of users + * that will be concurrently using eCryptfs. Set this to the right + * value to balance performance and memory use. + */ +unsigned int ecryptfs_number_of_users = ECRYPTFS_DEFAULT_NUM_USERS; + +module_param(ecryptfs_number_of_users, uint, 0); +MODULE_PARM_DESC(ecryptfs_number_of_users, "An estimate of the number of " + "concurrent users of eCryptfs"); + +void __ecryptfs_printk(const char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + if (fmt[1] == '7') { /* KERN_DEBUG */ + if (ecryptfs_verbosity >= 1) + vprintk(fmt, args); + } else + vprintk(fmt, args); + va_end(args); +} + +/* + * ecryptfs_init_lower_file + * @ecryptfs_dentry: Fully initialized eCryptfs dentry object, with + * the lower dentry and the lower mount set + * + * eCryptfs only ever keeps a single open file for every lower + * inode. All I/O operations to the lower inode occur through that + * file. When the first eCryptfs dentry that interposes with the first + * lower dentry for that inode is created, this function creates the + * lower file struct and associates it with the eCryptfs + * inode. When all eCryptfs files associated with the inode are released, the + * file is closed. + * + * The lower file will be opened with read/write permissions, if + * possible. Otherwise, it is opened read-only. + * + * This function does nothing if a lower file is already + * associated with the eCryptfs inode. + * + * Returns zero on success; non-zero otherwise + */ +static int ecryptfs_init_lower_file(struct dentry *dentry, + struct file **lower_file) +{ + const struct cred *cred = current_cred(); + struct path path = ecryptfs_lower_path(dentry); + int rc; + + rc = ecryptfs_privileged_open(lower_file, path.dentry, path.mnt, cred); + if (rc) { + printk(KERN_ERR "Error opening lower file " + "for lower_dentry [0x%p] and lower_mnt [0x%p]; " + "rc = [%d]\n", path.dentry, path.mnt, rc); + (*lower_file) = NULL; + } + return rc; +} + +int ecryptfs_get_lower_file(struct dentry *dentry, struct inode *inode) +{ + struct ecryptfs_inode_info *inode_info; + int count, rc = 0; + + inode_info = ecryptfs_inode_to_private(inode); + mutex_lock(&inode_info->lower_file_mutex); + count = atomic_inc_return(&inode_info->lower_file_count); + if (WARN_ON_ONCE(count < 1)) + rc = -EINVAL; + else if (count == 1) { + rc = ecryptfs_init_lower_file(dentry, + &inode_info->lower_file); + if (rc) + atomic_set(&inode_info->lower_file_count, 0); + } + mutex_unlock(&inode_info->lower_file_mutex); + return rc; +} + +void ecryptfs_put_lower_file(struct inode *inode) +{ + struct ecryptfs_inode_info *inode_info; + + inode_info = ecryptfs_inode_to_private(inode); + if (atomic_dec_and_mutex_lock(&inode_info->lower_file_count, + &inode_info->lower_file_mutex)) { + filemap_write_and_wait(inode->i_mapping); + fput(inode_info->lower_file); + inode_info->lower_file = NULL; + mutex_unlock(&inode_info->lower_file_mutex); + } +} + +enum { + Opt_sig, Opt_ecryptfs_sig, Opt_cipher, Opt_ecryptfs_cipher, + Opt_ecryptfs_key_bytes, Opt_passthrough, Opt_xattr_metadata, + Opt_encrypted_view, Opt_fnek_sig, Opt_fn_cipher, + Opt_fn_cipher_key_bytes, Opt_unlink_sigs, Opt_mount_auth_tok_only, + Opt_check_dev_ruid +}; + +static const struct fs_parameter_spec ecryptfs_fs_param_spec[] = { + fsparam_string ("sig", Opt_sig), + fsparam_string ("ecryptfs_sig", Opt_ecryptfs_sig), + fsparam_string ("cipher", Opt_cipher), + fsparam_string ("ecryptfs_cipher", Opt_ecryptfs_cipher), + fsparam_u32 ("ecryptfs_key_bytes", Opt_ecryptfs_key_bytes), + fsparam_flag ("ecryptfs_passthrough", Opt_passthrough), + fsparam_flag ("ecryptfs_xattr_metadata", Opt_xattr_metadata), + fsparam_flag ("ecryptfs_encrypted_view", Opt_encrypted_view), + fsparam_string ("ecryptfs_fnek_sig", Opt_fnek_sig), + fsparam_string ("ecryptfs_fn_cipher", Opt_fn_cipher), + fsparam_u32 ("ecryptfs_fn_key_bytes", Opt_fn_cipher_key_bytes), + fsparam_flag ("ecryptfs_unlink_sigs", Opt_unlink_sigs), + fsparam_flag ("ecryptfs_mount_auth_tok_only", Opt_mount_auth_tok_only), + fsparam_flag ("ecryptfs_check_dev_ruid", Opt_check_dev_ruid), + {} +}; + +static int ecryptfs_init_global_auth_toks( + struct ecryptfs_mount_crypt_stat *mount_crypt_stat) +{ + struct ecryptfs_global_auth_tok *global_auth_tok; + struct ecryptfs_auth_tok *auth_tok; + int rc = 0; + + list_for_each_entry(global_auth_tok, + &mount_crypt_stat->global_auth_tok_list, + mount_crypt_stat_list) { + rc = ecryptfs_keyring_auth_tok_for_sig( + &global_auth_tok->global_auth_tok_key, &auth_tok, + global_auth_tok->sig); + if (rc) { + printk(KERN_ERR "Could not find valid key in user " + "session keyring for sig specified in mount " + "option: [%s]\n", global_auth_tok->sig); + global_auth_tok->flags |= ECRYPTFS_AUTH_TOK_INVALID; + goto out; + } else { + global_auth_tok->flags &= ~ECRYPTFS_AUTH_TOK_INVALID; + up_write(&(global_auth_tok->global_auth_tok_key)->sem); + } + } +out: + return rc; +} + +static void ecryptfs_init_mount_crypt_stat( + struct ecryptfs_mount_crypt_stat *mount_crypt_stat) +{ + memset((void *)mount_crypt_stat, 0, + sizeof(struct ecryptfs_mount_crypt_stat)); + INIT_LIST_HEAD(&mount_crypt_stat->global_auth_tok_list); + mutex_init(&mount_crypt_stat->global_auth_tok_list_mutex); + mount_crypt_stat->flags |= ECRYPTFS_MOUNT_CRYPT_STAT_INITIALIZED; +} + +struct ecryptfs_fs_context { + /* Mount option status trackers */ + bool check_ruid; + bool sig_set; + bool cipher_name_set; + bool cipher_key_bytes_set; + bool fn_cipher_name_set; + bool fn_cipher_key_bytes_set; +}; + +/** + * ecryptfs_parse_param + * @fc: The ecryptfs filesystem context + * @param: The mount parameter to parse + * + * The signature of the key to use must be the description of a key + * already in the keyring. Mounting will fail if the key can not be + * found. + * + * Returns zero on success; non-zero on error + */ +static int ecryptfs_parse_param( + struct fs_context *fc, + struct fs_parameter *param) +{ + int rc; + int opt; + struct fs_parse_result result; + struct ecryptfs_fs_context *ctx = fc->fs_private; + struct ecryptfs_sb_info *sbi = fc->s_fs_info; + struct ecryptfs_mount_crypt_stat *mount_crypt_stat = + &sbi->mount_crypt_stat; + + opt = fs_parse(fc, ecryptfs_fs_param_spec, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_sig: + case Opt_ecryptfs_sig: + rc = ecryptfs_add_global_auth_tok(mount_crypt_stat, + param->string, 0); + if (rc) { + printk(KERN_ERR "Error attempting to register " + "global sig; rc = [%d]\n", rc); + return rc; + } + ctx->sig_set = 1; + break; + case Opt_cipher: + case Opt_ecryptfs_cipher: + strscpy(mount_crypt_stat->global_default_cipher_name, + param->string); + ctx->cipher_name_set = 1; + break; + case Opt_ecryptfs_key_bytes: + mount_crypt_stat->global_default_cipher_key_size = + result.uint_32; + ctx->cipher_key_bytes_set = 1; + break; + case Opt_passthrough: + mount_crypt_stat->flags |= + ECRYPTFS_PLAINTEXT_PASSTHROUGH_ENABLED; + break; + case Opt_xattr_metadata: + mount_crypt_stat->flags |= ECRYPTFS_XATTR_METADATA_ENABLED; + break; + case Opt_encrypted_view: + mount_crypt_stat->flags |= ECRYPTFS_XATTR_METADATA_ENABLED; + mount_crypt_stat->flags |= ECRYPTFS_ENCRYPTED_VIEW_ENABLED; + break; + case Opt_fnek_sig: + strscpy(mount_crypt_stat->global_default_fnek_sig, + param->string); + rc = ecryptfs_add_global_auth_tok( + mount_crypt_stat, + mount_crypt_stat->global_default_fnek_sig, + ECRYPTFS_AUTH_TOK_FNEK); + if (rc) { + printk(KERN_ERR "Error attempting to register " + "global fnek sig [%s]; rc = [%d]\n", + mount_crypt_stat->global_default_fnek_sig, rc); + return rc; + } + mount_crypt_stat->flags |= + (ECRYPTFS_GLOBAL_ENCRYPT_FILENAMES + | ECRYPTFS_GLOBAL_ENCFN_USE_MOUNT_FNEK); + break; + case Opt_fn_cipher: + strscpy(mount_crypt_stat->global_default_fn_cipher_name, + param->string); + ctx->fn_cipher_name_set = 1; + break; + case Opt_fn_cipher_key_bytes: + mount_crypt_stat->global_default_fn_cipher_key_bytes = + result.uint_32; + ctx->fn_cipher_key_bytes_set = 1; + break; + case Opt_unlink_sigs: + mount_crypt_stat->flags |= ECRYPTFS_UNLINK_SIGS; + break; + case Opt_mount_auth_tok_only: + mount_crypt_stat->flags |= ECRYPTFS_GLOBAL_MOUNT_AUTH_TOK_ONLY; + break; + case Opt_check_dev_ruid: + ctx->check_ruid = 1; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int ecryptfs_validate_options(struct fs_context *fc) +{ + int rc = 0; + u8 cipher_code; + struct ecryptfs_fs_context *ctx = fc->fs_private; + struct ecryptfs_sb_info *sbi = fc->s_fs_info; + struct ecryptfs_mount_crypt_stat *mount_crypt_stat; + + + mount_crypt_stat = &sbi->mount_crypt_stat; + + if (!ctx->sig_set) { + rc = -EINVAL; + ecryptfs_printk(KERN_ERR, "You must supply at least one valid " + "auth tok signature as a mount " + "parameter; see the eCryptfs README\n"); + goto out; + } + if (!ctx->cipher_name_set) { + int cipher_name_len = strlen(ECRYPTFS_DEFAULT_CIPHER); + + BUG_ON(cipher_name_len > ECRYPTFS_MAX_CIPHER_NAME_SIZE); + strscpy(mount_crypt_stat->global_default_cipher_name, + ECRYPTFS_DEFAULT_CIPHER); + } + if ((mount_crypt_stat->flags & ECRYPTFS_GLOBAL_ENCRYPT_FILENAMES) + && !ctx->fn_cipher_name_set) + strscpy(mount_crypt_stat->global_default_fn_cipher_name, + mount_crypt_stat->global_default_cipher_name); + if (!ctx->cipher_key_bytes_set) + mount_crypt_stat->global_default_cipher_key_size = 0; + if ((mount_crypt_stat->flags & ECRYPTFS_GLOBAL_ENCRYPT_FILENAMES) + && !ctx->fn_cipher_key_bytes_set) + mount_crypt_stat->global_default_fn_cipher_key_bytes = + mount_crypt_stat->global_default_cipher_key_size; + + cipher_code = ecryptfs_code_for_cipher_string( + mount_crypt_stat->global_default_cipher_name, + mount_crypt_stat->global_default_cipher_key_size); + if (!cipher_code) { + ecryptfs_printk(KERN_ERR, + "eCryptfs doesn't support cipher: %s\n", + mount_crypt_stat->global_default_cipher_name); + rc = -EINVAL; + goto out; + } + + mutex_lock(&key_tfm_list_mutex); + if (!ecryptfs_tfm_exists(mount_crypt_stat->global_default_cipher_name, + NULL)) { + rc = ecryptfs_add_new_key_tfm( + NULL, mount_crypt_stat->global_default_cipher_name, + mount_crypt_stat->global_default_cipher_key_size); + if (rc) { + printk(KERN_ERR "Error attempting to initialize " + "cipher with name = [%s] and key size = [%td]; " + "rc = [%d]\n", + mount_crypt_stat->global_default_cipher_name, + mount_crypt_stat->global_default_cipher_key_size, + rc); + rc = -EINVAL; + mutex_unlock(&key_tfm_list_mutex); + goto out; + } + } + if ((mount_crypt_stat->flags & ECRYPTFS_GLOBAL_ENCRYPT_FILENAMES) + && !ecryptfs_tfm_exists( + mount_crypt_stat->global_default_fn_cipher_name, NULL)) { + rc = ecryptfs_add_new_key_tfm( + NULL, mount_crypt_stat->global_default_fn_cipher_name, + mount_crypt_stat->global_default_fn_cipher_key_bytes); + if (rc) { + printk(KERN_ERR "Error attempting to initialize " + "cipher with name = [%s] and key size = [%td]; " + "rc = [%d]\n", + mount_crypt_stat->global_default_fn_cipher_name, + mount_crypt_stat->global_default_fn_cipher_key_bytes, + rc); + rc = -EINVAL; + mutex_unlock(&key_tfm_list_mutex); + goto out; + } + } + mutex_unlock(&key_tfm_list_mutex); + rc = ecryptfs_init_global_auth_toks(mount_crypt_stat); + if (rc) + printk(KERN_WARNING "One or more global auth toks could not " + "properly register; rc = [%d]\n", rc); +out: + return rc; +} + +struct kmem_cache *ecryptfs_sb_info_cache; +static struct file_system_type ecryptfs_fs_type; + +/* + * ecryptfs_get_tree + * @fc: The filesystem context + */ +static int ecryptfs_get_tree(struct fs_context *fc) +{ + struct super_block *s; + struct ecryptfs_fs_context *ctx = fc->fs_private; + struct ecryptfs_sb_info *sbi = fc->s_fs_info; + struct ecryptfs_mount_crypt_stat *mount_crypt_stat; + const char *err = "Getting sb failed"; + struct inode *inode; + struct path path; + int rc; + + if (!fc->source) { + rc = -EINVAL; + err = "Device name cannot be null"; + goto out; + } + + mount_crypt_stat = &sbi->mount_crypt_stat; + rc = ecryptfs_validate_options(fc); + if (rc) { + err = "Error validating options"; + goto out; + } + + if (fips_enabled) { + rc = -EINVAL; + err = "eCryptfs support is disabled due to FIPS"; + goto out; + } + + s = sget_fc(fc, NULL, set_anon_super_fc); + if (IS_ERR(s)) { + rc = PTR_ERR(s); + goto out; + } + + rc = super_setup_bdi(s); + if (rc) + goto out1; + + ecryptfs_set_superblock_private(s, sbi); + + /* ->kill_sb() will take care of sbi after that point */ + sbi = NULL; + s->s_op = &ecryptfs_sops; + s->s_xattr = ecryptfs_xattr_handlers; + set_default_d_op(s, &ecryptfs_dops); + + err = "Reading sb failed"; + rc = kern_path(fc->source, LOOKUP_FOLLOW | LOOKUP_DIRECTORY, &path); + if (rc) { + ecryptfs_printk(KERN_WARNING, "kern_path() failed\n"); + goto out1; + } + if (path.dentry->d_sb->s_type == &ecryptfs_fs_type) { + rc = -EINVAL; + printk(KERN_ERR "Mount on filesystem of type " + "eCryptfs explicitly disallowed due to " + "known incompatibilities\n"); + goto out_free; + } + + if (is_idmapped_mnt(path.mnt)) { + rc = -EINVAL; + printk(KERN_ERR "Mounting on idmapped mounts currently disallowed\n"); + goto out_free; + } + + if (ctx->check_ruid && + !uid_eq(d_inode(path.dentry)->i_uid, current_uid())) { + rc = -EPERM; + printk(KERN_ERR "Mount of device (uid: %d) not owned by " + "requested user (uid: %d)\n", + i_uid_read(d_inode(path.dentry)), + from_kuid(&init_user_ns, current_uid())); + goto out_free; + } + + ecryptfs_set_superblock_lower(s, path.dentry->d_sb); + + /** + * Set the POSIX ACL flag based on whether they're enabled in the lower + * mount. + */ + s->s_flags = fc->sb_flags & ~SB_POSIXACL; + s->s_flags |= path.dentry->d_sb->s_flags & SB_POSIXACL; + + /** + * Force a read-only eCryptfs mount when: + * 1) The lower mount is ro + * 2) The ecryptfs_encrypted_view mount option is specified + */ + if (sb_rdonly(path.dentry->d_sb) || mount_crypt_stat->flags & ECRYPTFS_ENCRYPTED_VIEW_ENABLED) + s->s_flags |= SB_RDONLY; + + s->s_maxbytes = path.dentry->d_sb->s_maxbytes; + s->s_blocksize = path.dentry->d_sb->s_blocksize; + s->s_magic = ECRYPTFS_SUPER_MAGIC; + s->s_stack_depth = path.dentry->d_sb->s_stack_depth + 1; + s->s_time_gran = path.dentry->d_sb->s_time_gran; + + rc = -EINVAL; + if (s->s_stack_depth > FILESYSTEM_MAX_STACK_DEPTH) { + pr_err("eCryptfs: maximum fs stacking depth exceeded\n"); + goto out_free; + } + + inode = ecryptfs_get_inode(d_inode(path.dentry), s); + rc = PTR_ERR(inode); + if (IS_ERR(inode)) + goto out_free; + + s->s_root = d_make_root(inode); + if (!s->s_root) { + rc = -ENOMEM; + goto out_free; + } + + ecryptfs_set_dentry_lower(s->s_root, path.dentry); + ecryptfs_superblock_to_private(s)->lower_mnt = path.mnt; + + s->s_flags |= SB_ACTIVE; + fc->root = dget(s->s_root); + return 0; + +out_free: + path_put(&path); +out1: + deactivate_locked_super(s); +out: + if (sbi) + ecryptfs_destroy_mount_crypt_stat(&sbi->mount_crypt_stat); + + printk(KERN_ERR "%s; rc = [%d]\n", err, rc); + return rc; +} + +/** + * ecryptfs_kill_block_super + * @sb: The ecryptfs super block + * + * Used to bring the superblock down and free the private data. + */ +static void ecryptfs_kill_block_super(struct super_block *sb) +{ + struct ecryptfs_sb_info *sb_info = ecryptfs_superblock_to_private(sb); + kill_anon_super(sb); + if (!sb_info) + return; + mntput(sb_info->lower_mnt); + ecryptfs_destroy_mount_crypt_stat(&sb_info->mount_crypt_stat); + kmem_cache_free(ecryptfs_sb_info_cache, sb_info); +} + +static void ecryptfs_free_fc(struct fs_context *fc) +{ + struct ecryptfs_fs_context *ctx = fc->fs_private; + struct ecryptfs_sb_info *sbi = fc->s_fs_info; + + kfree(ctx); + + if (sbi) { + ecryptfs_destroy_mount_crypt_stat(&sbi->mount_crypt_stat); + kmem_cache_free(ecryptfs_sb_info_cache, sbi); + } +} + +static const struct fs_context_operations ecryptfs_context_ops = { + .free = ecryptfs_free_fc, + .parse_param = ecryptfs_parse_param, + .get_tree = ecryptfs_get_tree, + .reconfigure = NULL, +}; + +static int ecryptfs_init_fs_context(struct fs_context *fc) +{ + struct ecryptfs_fs_context *ctx; + struct ecryptfs_sb_info *sbi = NULL; + + ctx = kzalloc_obj(struct ecryptfs_fs_context); + if (!ctx) + return -ENOMEM; + sbi = kmem_cache_zalloc(ecryptfs_sb_info_cache, GFP_KERNEL); + if (!sbi) { + kfree(ctx); + ctx = NULL; + return -ENOMEM; + } + + ecryptfs_init_mount_crypt_stat(&sbi->mount_crypt_stat); + + fc->fs_private = ctx; + fc->s_fs_info = sbi; + fc->ops = &ecryptfs_context_ops; + return 0; +} + +static struct file_system_type ecryptfs_fs_type = { + .owner = THIS_MODULE, + .name = "ecryptfs", + .init_fs_context = ecryptfs_init_fs_context, + .parameters = ecryptfs_fs_param_spec, + .kill_sb = ecryptfs_kill_block_super, + .fs_flags = 0 +}; +MODULE_ALIAS_FS("ecryptfs"); + +/* + * inode_info_init_once + * + * Initializes the ecryptfs_inode_info_cache when it is created + */ +static void +inode_info_init_once(void *vptr) +{ + struct ecryptfs_inode_info *ei = (struct ecryptfs_inode_info *)vptr; + + inode_init_once(&ei->vfs_inode); +} + +static struct ecryptfs_cache_info { + struct kmem_cache **cache; + const char *name; + size_t size; + slab_flags_t flags; + void (*ctor)(void *obj); +} ecryptfs_cache_infos[] = { + { + .cache = &ecryptfs_auth_tok_list_item_cache, + .name = "ecryptfs_auth_tok_list_item", + .size = sizeof(struct ecryptfs_auth_tok_list_item), + }, + { + .cache = &ecryptfs_file_info_cache, + .name = "ecryptfs_file_cache", + .size = sizeof(struct ecryptfs_file_info), + }, + { + .cache = &ecryptfs_inode_info_cache, + .name = "ecryptfs_inode_cache", + .size = sizeof(struct ecryptfs_inode_info), + .flags = SLAB_ACCOUNT, + .ctor = inode_info_init_once, + }, + { + .cache = &ecryptfs_sb_info_cache, + .name = "ecryptfs_sb_cache", + .size = sizeof(struct ecryptfs_sb_info), + }, + { + .cache = &ecryptfs_header_cache, + .name = "ecryptfs_headers", + .size = PAGE_SIZE, + }, + { + .cache = &ecryptfs_xattr_cache, + .name = "ecryptfs_xattr_cache", + .size = PAGE_SIZE, + }, + { + .cache = &ecryptfs_key_record_cache, + .name = "ecryptfs_key_record_cache", + .size = sizeof(struct ecryptfs_key_record), + }, + { + .cache = &ecryptfs_key_sig_cache, + .name = "ecryptfs_key_sig_cache", + .size = sizeof(struct ecryptfs_key_sig), + }, + { + .cache = &ecryptfs_global_auth_tok_cache, + .name = "ecryptfs_global_auth_tok_cache", + .size = sizeof(struct ecryptfs_global_auth_tok), + }, + { + .cache = &ecryptfs_key_tfm_cache, + .name = "ecryptfs_key_tfm_cache", + .size = sizeof(struct ecryptfs_key_tfm), + }, +}; + +static void ecryptfs_free_kmem_caches(void) +{ + int i; + + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + + for (i = 0; i < ARRAY_SIZE(ecryptfs_cache_infos); i++) { + struct ecryptfs_cache_info *info; + + info = &ecryptfs_cache_infos[i]; + kmem_cache_destroy(*(info->cache)); + } +} + +/** + * ecryptfs_init_kmem_caches + * + * Returns zero on success; non-zero otherwise + */ +static int ecryptfs_init_kmem_caches(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(ecryptfs_cache_infos); i++) { + struct ecryptfs_cache_info *info; + + info = &ecryptfs_cache_infos[i]; + *(info->cache) = kmem_cache_create(info->name, info->size, 0, + SLAB_HWCACHE_ALIGN | info->flags, info->ctor); + if (!*(info->cache)) { + ecryptfs_free_kmem_caches(); + ecryptfs_printk(KERN_WARNING, "%s: " + "kmem_cache_create failed\n", + info->name); + return -ENOMEM; + } + } + return 0; +} + +static struct kobject *ecryptfs_kobj; + +static ssize_t version_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buff) +{ + return sysfs_emit(buff, "%d\n", ECRYPTFS_VERSIONING_MASK); +} + +static struct kobj_attribute version_attr = __ATTR_RO(version); + +static struct attribute *attributes[] = { + &version_attr.attr, + NULL, +}; + +static const struct attribute_group attr_group = { + .attrs = attributes, +}; + +static int do_sysfs_registration(void) +{ + int rc; + + ecryptfs_kobj = kobject_create_and_add("ecryptfs", fs_kobj); + if (!ecryptfs_kobj) { + printk(KERN_ERR "Unable to create ecryptfs kset\n"); + rc = -ENOMEM; + goto out; + } + rc = sysfs_create_group(ecryptfs_kobj, &attr_group); + if (rc) { + printk(KERN_ERR + "Unable to create ecryptfs version attributes\n"); + kobject_put(ecryptfs_kobj); + } +out: + return rc; +} + +static void do_sysfs_unregistration(void) +{ + sysfs_remove_group(ecryptfs_kobj, &attr_group); + kobject_put(ecryptfs_kobj); +} + +static int __init ecryptfs_init(void) +{ + int rc; + + if (ECRYPTFS_DEFAULT_EXTENT_SIZE > PAGE_SIZE) { + rc = -EINVAL; + ecryptfs_printk(KERN_ERR, "The eCryptfs extent size is " + "larger than the host's page size, and so " + "eCryptfs cannot run on this system. The " + "default eCryptfs extent size is [%u] bytes; " + "the page size is [%lu] bytes.\n", + ECRYPTFS_DEFAULT_EXTENT_SIZE, + (unsigned long)PAGE_SIZE); + goto out; + } + rc = ecryptfs_init_kmem_caches(); + if (rc) { + printk(KERN_ERR + "Failed to allocate one or more kmem_cache objects\n"); + goto out; + } + rc = do_sysfs_registration(); + if (rc) { + printk(KERN_ERR "sysfs registration failed\n"); + goto out_free_kmem_caches; + } + rc = ecryptfs_init_kthread(); + if (rc) { + printk(KERN_ERR "%s: kthread initialization failed; " + "rc = [%d]\n", __func__, rc); + goto out_do_sysfs_unregistration; + } + rc = ecryptfs_init_messaging(); + if (rc) { + printk(KERN_ERR "Failure occurred while attempting to " + "initialize the communications channel to " + "ecryptfsd\n"); + goto out_destroy_kthread; + } + rc = ecryptfs_init_crypto(); + if (rc) { + printk(KERN_ERR "Failure whilst attempting to init crypto; " + "rc = [%d]\n", rc); + goto out_release_messaging; + } + rc = register_filesystem(&ecryptfs_fs_type); + if (rc) { + printk(KERN_ERR "Failed to register filesystem\n"); + goto out_destroy_crypto; + } + if (ecryptfs_verbosity > 0) + printk(KERN_CRIT "eCryptfs verbosity set to %d. Secret values " + "will be written to the syslog!\n", ecryptfs_verbosity); + + goto out; +out_destroy_crypto: + ecryptfs_destroy_crypto(); +out_release_messaging: + ecryptfs_release_messaging(); +out_destroy_kthread: + ecryptfs_destroy_kthread(); +out_do_sysfs_unregistration: + do_sysfs_unregistration(); +out_free_kmem_caches: + ecryptfs_free_kmem_caches(); +out: + return rc; +} + +static void __exit ecryptfs_exit(void) +{ + int rc; + + rc = ecryptfs_destroy_crypto(); + if (rc) + printk(KERN_ERR "Failure whilst attempting to destroy crypto; " + "rc = [%d]\n", rc); + ecryptfs_release_messaging(); + ecryptfs_destroy_kthread(); + do_sysfs_unregistration(); + unregister_filesystem(&ecryptfs_fs_type); + ecryptfs_free_kmem_caches(); +} + +MODULE_AUTHOR("Michael A. Halcrow "); +MODULE_DESCRIPTION("eCryptfs"); + +MODULE_LICENSE("GPL"); + +module_init(ecryptfs_init) +module_exit(ecryptfs_exit) diff --git a/fs/ecryptfs/messaging.c b/fs/ecryptfs/messaging.c new file mode 100644 index 000000000..73c7b8215 --- /dev/null +++ b/fs/ecryptfs/messaging.c @@ -0,0 +1,461 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * eCryptfs: Linux filesystem encryption layer + * + * Copyright (C) 2004-2008 International Business Machines Corp. + * Author(s): Michael A. Halcrow + * Tyler Hicks + */ +#include +#include +#include +#include +#include +#include "ecryptfs_kernel.h" + +static LIST_HEAD(ecryptfs_msg_ctx_free_list); +static LIST_HEAD(ecryptfs_msg_ctx_alloc_list); +static DEFINE_MUTEX(ecryptfs_msg_ctx_lists_mux); + +static struct hlist_head *ecryptfs_daemon_hash; +DEFINE_MUTEX(ecryptfs_daemon_hash_mux); +static int ecryptfs_hash_bits; +#define ecryptfs_current_euid_hash(uid) \ + hash_long((unsigned long)from_kuid(&init_user_ns, current_euid()), ecryptfs_hash_bits) + +static u32 ecryptfs_msg_counter; +static struct ecryptfs_msg_ctx *ecryptfs_msg_ctx_arr; + +/** + * ecryptfs_acquire_free_msg_ctx + * @msg_ctx: The context that was acquired from the free list + * + * Acquires a context element from the free list and locks the mutex + * on the context. Sets the msg_ctx task to current. Returns zero on + * success; non-zero on error or upon failure to acquire a free + * context element. Must be called with ecryptfs_msg_ctx_lists_mux + * held. + */ +static int ecryptfs_acquire_free_msg_ctx(struct ecryptfs_msg_ctx **msg_ctx) +{ + struct list_head *p; + int rc; + + if (list_empty(&ecryptfs_msg_ctx_free_list)) { + printk(KERN_WARNING "%s: The eCryptfs free " + "context list is empty. It may be helpful to " + "specify the ecryptfs_message_buf_len " + "parameter to be greater than the current " + "value of [%d]\n", __func__, ecryptfs_message_buf_len); + rc = -ENOMEM; + goto out; + } + list_for_each(p, &ecryptfs_msg_ctx_free_list) { + *msg_ctx = list_entry(p, struct ecryptfs_msg_ctx, node); + if (mutex_trylock(&(*msg_ctx)->mux)) { + (*msg_ctx)->task = current; + rc = 0; + goto out; + } + } + rc = -ENOMEM; +out: + return rc; +} + +/** + * ecryptfs_msg_ctx_free_to_alloc + * @msg_ctx: The context to move from the free list to the alloc list + * + * Must be called with ecryptfs_msg_ctx_lists_mux held. + */ +static void ecryptfs_msg_ctx_free_to_alloc(struct ecryptfs_msg_ctx *msg_ctx) +{ + list_move(&msg_ctx->node, &ecryptfs_msg_ctx_alloc_list); + msg_ctx->state = ECRYPTFS_MSG_CTX_STATE_PENDING; + msg_ctx->counter = ++ecryptfs_msg_counter; +} + +/** + * ecryptfs_msg_ctx_alloc_to_free + * @msg_ctx: The context to move from the alloc list to the free list + * + * Must be called with ecryptfs_msg_ctx_lists_mux held. + */ +void ecryptfs_msg_ctx_alloc_to_free(struct ecryptfs_msg_ctx *msg_ctx) +{ + list_move(&(msg_ctx->node), &ecryptfs_msg_ctx_free_list); + kfree(msg_ctx->msg); + msg_ctx->msg = NULL; + msg_ctx->state = ECRYPTFS_MSG_CTX_STATE_FREE; +} + +/** + * ecryptfs_find_daemon_by_euid + * @daemon: If return value is zero, points to the desired daemon pointer + * + * Must be called with ecryptfs_daemon_hash_mux held. + * + * Search the hash list for the current effective user id. + * + * Returns zero if the user id exists in the list; non-zero otherwise. + */ +int ecryptfs_find_daemon_by_euid(struct ecryptfs_daemon **daemon) +{ + int rc; + + hlist_for_each_entry(*daemon, + &ecryptfs_daemon_hash[ecryptfs_current_euid_hash()], + euid_chain) { + if (uid_eq((*daemon)->file->f_cred->euid, current_euid())) { + rc = 0; + goto out; + } + } + rc = -EINVAL; +out: + return rc; +} + +/** + * ecryptfs_spawn_daemon - Create and initialize a new daemon struct + * @daemon: Pointer to set to newly allocated daemon struct + * @file: File used when opening /dev/ecryptfs + * + * Must be called ceremoniously while in possession of + * ecryptfs_sacred_daemon_hash_mux + * + * Returns zero on success; non-zero otherwise + */ +int +ecryptfs_spawn_daemon(struct ecryptfs_daemon **daemon, struct file *file) +{ + int rc = 0; + + (*daemon) = kzalloc_obj(**daemon); + if (!(*daemon)) { + rc = -ENOMEM; + goto out; + } + (*daemon)->file = file; + mutex_init(&(*daemon)->mux); + INIT_LIST_HEAD(&(*daemon)->msg_ctx_out_queue); + init_waitqueue_head(&(*daemon)->wait); + (*daemon)->num_queued_msg_ctx = 0; + hlist_add_head(&(*daemon)->euid_chain, + &ecryptfs_daemon_hash[ecryptfs_current_euid_hash()]); +out: + return rc; +} + +/* + * ecryptfs_exorcise_daemon - Destroy the daemon struct + * + * Must be called ceremoniously while in possession of + * ecryptfs_daemon_hash_mux and the daemon's own mux. + */ +int ecryptfs_exorcise_daemon(struct ecryptfs_daemon *daemon) +{ + struct ecryptfs_msg_ctx *msg_ctx, *msg_ctx_tmp; + int rc = 0; + + mutex_lock(&daemon->mux); + if ((daemon->flags & ECRYPTFS_DAEMON_IN_READ) + || (daemon->flags & ECRYPTFS_DAEMON_IN_POLL)) { + rc = -EBUSY; + mutex_unlock(&daemon->mux); + goto out; + } + mutex_lock(&ecryptfs_msg_ctx_lists_mux); + list_for_each_entry_safe(msg_ctx, msg_ctx_tmp, + &daemon->msg_ctx_out_queue, daemon_out_list) { + list_del(&msg_ctx->daemon_out_list); + daemon->num_queued_msg_ctx--; + printk(KERN_WARNING "%s: Warning: dropping message that is in " + "the out queue of a dying daemon\n", __func__); + ecryptfs_msg_ctx_alloc_to_free(msg_ctx); + } + mutex_unlock(&ecryptfs_msg_ctx_lists_mux); + hlist_del(&daemon->euid_chain); + mutex_unlock(&daemon->mux); + kfree_sensitive(daemon); +out: + return rc; +} + +/** + * ecryptfs_process_response + * @daemon: eCryptfs daemon object + * @msg: The ecryptfs message received; the caller should sanity check + * msg->data_len and free the memory + * @seq: The sequence number of the message; must match the sequence + * number for the existing message context waiting for this + * response + * + * Processes a response message after sending an operation request to + * userspace. Some other process is awaiting this response. Before + * sending out its first communications, the other process allocated a + * msg_ctx from the ecryptfs_msg_ctx_arr at a particular index. The + * response message contains this index so that we can copy over the + * response message into the msg_ctx that the process holds a + * reference to. The other process is going to wake up, check to see + * that msg_ctx->state == ECRYPTFS_MSG_CTX_STATE_DONE, and then + * proceed to read off and process the response message. Returns zero + * upon delivery to desired context element; non-zero upon delivery + * failure or error. + * + * Returns zero on success; non-zero otherwise + */ +int ecryptfs_process_response(struct ecryptfs_daemon *daemon, + struct ecryptfs_message *msg, u32 seq) +{ + struct ecryptfs_msg_ctx *msg_ctx; + size_t msg_size; + int rc; + + if (msg->index >= ecryptfs_message_buf_len) { + rc = -EINVAL; + printk(KERN_ERR "%s: Attempt to reference " + "context buffer at index [%d]; maximum " + "allowable is [%d]\n", __func__, msg->index, + (ecryptfs_message_buf_len - 1)); + goto out; + } + msg_ctx = &ecryptfs_msg_ctx_arr[msg->index]; + mutex_lock(&msg_ctx->mux); + if (msg_ctx->state != ECRYPTFS_MSG_CTX_STATE_PENDING) { + rc = -EINVAL; + printk(KERN_WARNING "%s: Desired context element is not " + "pending a response\n", __func__); + goto unlock; + } else if (msg_ctx->counter != seq) { + rc = -EINVAL; + printk(KERN_WARNING "%s: Invalid message sequence; " + "expected [%d]; received [%d]\n", __func__, + msg_ctx->counter, seq); + goto unlock; + } + msg_size = struct_size(msg, data, msg->data_len); + msg_ctx->msg = kmemdup(msg, msg_size, GFP_KERNEL); + if (!msg_ctx->msg) { + rc = -ENOMEM; + goto unlock; + } + msg_ctx->state = ECRYPTFS_MSG_CTX_STATE_DONE; + wake_up_process(msg_ctx->task); + rc = 0; +unlock: + mutex_unlock(&msg_ctx->mux); +out: + return rc; +} + +/** + * ecryptfs_send_message_locked + * @data: The data to send + * @data_len: The length of data + * @msg_type: Type of message + * @msg_ctx: The message context allocated for the send + * + * Must be called with ecryptfs_daemon_hash_mux held. + * + * Returns zero on success; non-zero otherwise + */ +static int +ecryptfs_send_message_locked(char *data, int data_len, u8 msg_type, + struct ecryptfs_msg_ctx **msg_ctx) +{ + struct ecryptfs_daemon *daemon; + int rc; + + rc = ecryptfs_find_daemon_by_euid(&daemon); + if (rc) { + rc = -ENOTCONN; + goto out; + } + mutex_lock(&ecryptfs_msg_ctx_lists_mux); + rc = ecryptfs_acquire_free_msg_ctx(msg_ctx); + if (rc) { + mutex_unlock(&ecryptfs_msg_ctx_lists_mux); + printk(KERN_WARNING "%s: Could not claim a free " + "context element\n", __func__); + goto out; + } + ecryptfs_msg_ctx_free_to_alloc(*msg_ctx); + mutex_unlock(&(*msg_ctx)->mux); + mutex_unlock(&ecryptfs_msg_ctx_lists_mux); + rc = ecryptfs_send_miscdev(data, data_len, *msg_ctx, msg_type, 0, + daemon); + if (rc) { + printk(KERN_ERR "%s: Error attempting to send message to " + "userspace daemon; rc = [%d]\n", __func__, rc); + mutex_lock(&ecryptfs_msg_ctx_lists_mux); + mutex_lock(&(*msg_ctx)->mux); + ecryptfs_msg_ctx_alloc_to_free(*msg_ctx); + mutex_unlock(&(*msg_ctx)->mux); + mutex_unlock(&ecryptfs_msg_ctx_lists_mux); + *msg_ctx = NULL; + } +out: + return rc; +} + +/** + * ecryptfs_send_message + * @data: The data to send + * @data_len: The length of data + * @msg_ctx: The message context allocated for the send + * + * Grabs ecryptfs_daemon_hash_mux. + * + * Returns zero on success; non-zero otherwise + */ +int ecryptfs_send_message(char *data, int data_len, + struct ecryptfs_msg_ctx **msg_ctx) +{ + int rc; + + mutex_lock(&ecryptfs_daemon_hash_mux); + rc = ecryptfs_send_message_locked(data, data_len, ECRYPTFS_MSG_REQUEST, + msg_ctx); + mutex_unlock(&ecryptfs_daemon_hash_mux); + return rc; +} + +/** + * ecryptfs_wait_for_response + * @msg_ctx: The context that was assigned when sending a message + * @msg: The incoming message from userspace; not set if rc != 0 + * + * Sleeps until awaken by ecryptfs_receive_message or until the amount + * of time exceeds ecryptfs_message_wait_timeout. If zero is + * returned, msg will point to a valid message from userspace; a + * non-zero value is returned upon failure to receive a message or an + * error occurs. Callee must free @msg on success. + */ +int ecryptfs_wait_for_response(struct ecryptfs_msg_ctx *msg_ctx, + struct ecryptfs_message **msg) +{ + signed long timeout = ecryptfs_message_wait_timeout * HZ; + int rc = 0; + +sleep: + timeout = schedule_timeout_interruptible(timeout); + mutex_lock(&ecryptfs_msg_ctx_lists_mux); + mutex_lock(&msg_ctx->mux); + if (msg_ctx->state != ECRYPTFS_MSG_CTX_STATE_DONE) { + if (timeout) { + mutex_unlock(&msg_ctx->mux); + mutex_unlock(&ecryptfs_msg_ctx_lists_mux); + goto sleep; + } + rc = -ENOMSG; + } else { + *msg = msg_ctx->msg; + msg_ctx->msg = NULL; + } + ecryptfs_msg_ctx_alloc_to_free(msg_ctx); + mutex_unlock(&msg_ctx->mux); + mutex_unlock(&ecryptfs_msg_ctx_lists_mux); + return rc; +} + +int __init ecryptfs_init_messaging(void) +{ + int i; + int rc = 0; + + if (ecryptfs_number_of_users > ECRYPTFS_MAX_NUM_USERS) { + ecryptfs_number_of_users = ECRYPTFS_MAX_NUM_USERS; + printk(KERN_WARNING "%s: Specified number of users is " + "too large, defaulting to [%d] users\n", __func__, + ecryptfs_number_of_users); + } + mutex_lock(&ecryptfs_daemon_hash_mux); + ecryptfs_hash_bits = 1; + while (ecryptfs_number_of_users >> ecryptfs_hash_bits) + ecryptfs_hash_bits++; + ecryptfs_daemon_hash = kmalloc((sizeof(struct hlist_head) + * (1 << ecryptfs_hash_bits)), + GFP_KERNEL); + if (!ecryptfs_daemon_hash) { + rc = -ENOMEM; + mutex_unlock(&ecryptfs_daemon_hash_mux); + goto out; + } + for (i = 0; i < (1 << ecryptfs_hash_bits); i++) + INIT_HLIST_HEAD(&ecryptfs_daemon_hash[i]); + mutex_unlock(&ecryptfs_daemon_hash_mux); + ecryptfs_msg_ctx_arr = kmalloc((sizeof(struct ecryptfs_msg_ctx) + * ecryptfs_message_buf_len), + GFP_KERNEL); + if (!ecryptfs_msg_ctx_arr) { + kfree(ecryptfs_daemon_hash); + rc = -ENOMEM; + goto out; + } + mutex_lock(&ecryptfs_msg_ctx_lists_mux); + ecryptfs_msg_counter = 0; + for (i = 0; i < ecryptfs_message_buf_len; i++) { + INIT_LIST_HEAD(&ecryptfs_msg_ctx_arr[i].node); + INIT_LIST_HEAD(&ecryptfs_msg_ctx_arr[i].daemon_out_list); + mutex_init(&ecryptfs_msg_ctx_arr[i].mux); + mutex_lock(&ecryptfs_msg_ctx_arr[i].mux); + ecryptfs_msg_ctx_arr[i].index = i; + ecryptfs_msg_ctx_arr[i].state = ECRYPTFS_MSG_CTX_STATE_FREE; + ecryptfs_msg_ctx_arr[i].counter = 0; + ecryptfs_msg_ctx_arr[i].task = NULL; + ecryptfs_msg_ctx_arr[i].msg = NULL; + list_add_tail(&ecryptfs_msg_ctx_arr[i].node, + &ecryptfs_msg_ctx_free_list); + mutex_unlock(&ecryptfs_msg_ctx_arr[i].mux); + } + mutex_unlock(&ecryptfs_msg_ctx_lists_mux); + rc = ecryptfs_init_ecryptfs_miscdev(); + if (rc) + ecryptfs_release_messaging(); +out: + return rc; +} + +void ecryptfs_release_messaging(void) +{ + if (ecryptfs_msg_ctx_arr) { + int i; + + mutex_lock(&ecryptfs_msg_ctx_lists_mux); + for (i = 0; i < ecryptfs_message_buf_len; i++) { + mutex_lock(&ecryptfs_msg_ctx_arr[i].mux); + kfree(ecryptfs_msg_ctx_arr[i].msg); + mutex_unlock(&ecryptfs_msg_ctx_arr[i].mux); + } + kfree(ecryptfs_msg_ctx_arr); + mutex_unlock(&ecryptfs_msg_ctx_lists_mux); + } + if (ecryptfs_daemon_hash) { + struct ecryptfs_daemon *daemon; + struct hlist_node *n; + int i; + + mutex_lock(&ecryptfs_daemon_hash_mux); + for (i = 0; i < (1 << ecryptfs_hash_bits); i++) { + int rc; + + hlist_for_each_entry_safe(daemon, n, + &ecryptfs_daemon_hash[i], + euid_chain) { + rc = ecryptfs_exorcise_daemon(daemon); + if (rc) + printk(KERN_ERR "%s: Error whilst " + "attempting to destroy daemon; " + "rc = [%d]. Dazed and confused, " + "but trying to continue.\n", + __func__, rc); + } + } + kfree(ecryptfs_daemon_hash); + mutex_unlock(&ecryptfs_daemon_hash_mux); + } + ecryptfs_destroy_ecryptfs_miscdev(); + return; +} diff --git a/fs/ecryptfs/miscdev.c b/fs/ecryptfs/miscdev.c new file mode 100644 index 000000000..68804399a --- /dev/null +++ b/fs/ecryptfs/miscdev.c @@ -0,0 +1,499 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * eCryptfs: Linux filesystem encryption layer + * + * Copyright (C) 2008 International Business Machines Corp. + * Author(s): Michael A. Halcrow + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ecryptfs_kernel.h" + +static atomic_t ecryptfs_num_miscdev_opens; + +/** + * ecryptfs_miscdev_poll + * @file: dev file + * @pt: dev poll table (ignored) + * + * Returns the poll mask + */ +static __poll_t +ecryptfs_miscdev_poll(struct file *file, poll_table *pt) +{ + struct ecryptfs_daemon *daemon = file->private_data; + __poll_t mask = 0; + + mutex_lock(&daemon->mux); + if (daemon->flags & ECRYPTFS_DAEMON_ZOMBIE) { + printk(KERN_WARNING "%s: Attempt to poll on zombified " + "daemon\n", __func__); + goto out_unlock_daemon; + } + if (daemon->flags & ECRYPTFS_DAEMON_IN_READ) + goto out_unlock_daemon; + if (daemon->flags & ECRYPTFS_DAEMON_IN_POLL) + goto out_unlock_daemon; + daemon->flags |= ECRYPTFS_DAEMON_IN_POLL; + mutex_unlock(&daemon->mux); + poll_wait(file, &daemon->wait, pt); + mutex_lock(&daemon->mux); + if (!list_empty(&daemon->msg_ctx_out_queue)) + mask |= EPOLLIN | EPOLLRDNORM; +out_unlock_daemon: + daemon->flags &= ~ECRYPTFS_DAEMON_IN_POLL; + mutex_unlock(&daemon->mux); + return mask; +} + +/** + * ecryptfs_miscdev_open + * @inode: inode of miscdev handle (ignored) + * @file: file for miscdev handle + * + * Returns zero on success; non-zero otherwise + */ +static int +ecryptfs_miscdev_open(struct inode *inode, struct file *file) +{ + struct ecryptfs_daemon *daemon = NULL; + int rc; + + mutex_lock(&ecryptfs_daemon_hash_mux); + rc = ecryptfs_find_daemon_by_euid(&daemon); + if (!rc) { + rc = -EINVAL; + goto out_unlock_daemon_list; + } + rc = ecryptfs_spawn_daemon(&daemon, file); + if (rc) { + printk(KERN_ERR "%s: Error attempting to spawn daemon; " + "rc = [%d]\n", __func__, rc); + goto out_unlock_daemon_list; + } + mutex_lock(&daemon->mux); + if (daemon->flags & ECRYPTFS_DAEMON_MISCDEV_OPEN) { + rc = -EBUSY; + goto out_unlock_daemon; + } + daemon->flags |= ECRYPTFS_DAEMON_MISCDEV_OPEN; + file->private_data = daemon; + atomic_inc(&ecryptfs_num_miscdev_opens); +out_unlock_daemon: + mutex_unlock(&daemon->mux); +out_unlock_daemon_list: + mutex_unlock(&ecryptfs_daemon_hash_mux); + return rc; +} + +/** + * ecryptfs_miscdev_release + * @inode: inode of fs/ecryptfs/euid handle (ignored) + * @file: file for fs/ecryptfs/euid handle + * + * This keeps the daemon registered until the daemon sends another + * ioctl to fs/ecryptfs/ctl or until the kernel module unregisters. + * + * Returns zero on success; non-zero otherwise + */ +static int +ecryptfs_miscdev_release(struct inode *inode, struct file *file) +{ + struct ecryptfs_daemon *daemon = file->private_data; + int rc; + + mutex_lock(&daemon->mux); + BUG_ON(!(daemon->flags & ECRYPTFS_DAEMON_MISCDEV_OPEN)); + daemon->flags &= ~ECRYPTFS_DAEMON_MISCDEV_OPEN; + atomic_dec(&ecryptfs_num_miscdev_opens); + mutex_unlock(&daemon->mux); + + mutex_lock(&ecryptfs_daemon_hash_mux); + rc = ecryptfs_exorcise_daemon(daemon); + mutex_unlock(&ecryptfs_daemon_hash_mux); + if (rc) { + printk(KERN_CRIT "%s: Fatal error whilst attempting to " + "shut down daemon; rc = [%d]. Please report this " + "bug.\n", __func__, rc); + BUG(); + } + return rc; +} + +/** + * ecryptfs_send_miscdev + * @data: Data to send to daemon; may be NULL + * @data_size: Amount of data to send to daemon + * @msg_ctx: Message context, which is used to handle the reply. If + * this is NULL, then we do not expect a reply. + * @msg_type: Type of message + * @msg_flags: Flags for message + * @daemon: eCryptfs daemon object + * + * Add msg_ctx to queue and then, if it exists, notify the blocked + * miscdevess about the data being available. Must be called with + * ecryptfs_daemon_hash_mux held. + * + * Returns zero on success; non-zero otherwise + */ +int ecryptfs_send_miscdev(char *data, size_t data_size, + struct ecryptfs_msg_ctx *msg_ctx, u8 msg_type, + u16 msg_flags, struct ecryptfs_daemon *daemon) +{ + struct ecryptfs_message *msg; + size_t msg_size; + + msg_size = struct_size(msg, data, data_size); + msg = kmalloc(msg_size, GFP_KERNEL); + if (!msg) + return -ENOMEM; + + mutex_lock(&msg_ctx->mux); + msg_ctx->msg = msg; + msg_ctx->msg->index = msg_ctx->index; + msg_ctx->msg->data_len = data_size; + msg_ctx->type = msg_type; + memcpy(msg_ctx->msg->data, data, data_size); + msg_ctx->msg_size = msg_size; + list_add_tail(&msg_ctx->daemon_out_list, &daemon->msg_ctx_out_queue); + mutex_unlock(&msg_ctx->mux); + + mutex_lock(&daemon->mux); + daemon->num_queued_msg_ctx++; + wake_up_interruptible(&daemon->wait); + mutex_unlock(&daemon->mux); + + return 0; +} + +/* + * miscdevfs packet format: + * Octet 0: Type + * Octets 1-4: network byte order msg_ctx->counter + * Octets 5-N0: Size of struct ecryptfs_message to follow + * Octets N0-N1: struct ecryptfs_message (including data) + * + * Octets 5-N1 not written if the packet type does not include a message + */ +#define PKT_TYPE_SIZE 1 +#define PKT_CTR_SIZE 4 +#define MIN_NON_MSG_PKT_SIZE (PKT_TYPE_SIZE + PKT_CTR_SIZE) +#define MIN_MSG_PKT_SIZE (PKT_TYPE_SIZE + PKT_CTR_SIZE \ + + ECRYPTFS_MIN_PKT_LEN_SIZE) +/* 4 + ECRYPTFS_MAX_ENCRYPTED_KEY_BYTES comes from tag 65 packet format */ +#define MAX_MSG_PKT_SIZE (PKT_TYPE_SIZE + PKT_CTR_SIZE \ + + ECRYPTFS_MAX_PKT_LEN_SIZE \ + + sizeof(struct ecryptfs_message) \ + + 4 + ECRYPTFS_MAX_ENCRYPTED_KEY_BYTES) +#define PKT_TYPE_OFFSET 0 +#define PKT_CTR_OFFSET PKT_TYPE_SIZE +#define PKT_LEN_OFFSET (PKT_TYPE_SIZE + PKT_CTR_SIZE) + +/** + * ecryptfs_miscdev_read - format and send message from queue + * @file: miscdevfs handle + * @buf: User buffer into which to copy the next message on the daemon queue + * @count: Amount of space available in @buf + * @ppos: Offset in file (ignored) + * + * Pulls the most recent message from the daemon queue, formats it for + * being sent via a miscdevfs handle, and copies it into @buf + * + * Returns the number of bytes copied into the user buffer + */ +static ssize_t +ecryptfs_miscdev_read(struct file *file, char __user *buf, size_t count, + loff_t *ppos) +{ + struct ecryptfs_daemon *daemon = file->private_data; + struct ecryptfs_msg_ctx *msg_ctx; + size_t packet_length_size; + char packet_length[ECRYPTFS_MAX_PKT_LEN_SIZE]; + size_t i; + size_t total_length; + int rc; + + mutex_lock(&daemon->mux); + if (daemon->flags & ECRYPTFS_DAEMON_ZOMBIE) { + rc = 0; + printk(KERN_WARNING "%s: Attempt to read from zombified " + "daemon\n", __func__); + goto out_unlock_daemon; + } + if (daemon->flags & ECRYPTFS_DAEMON_IN_READ) { + rc = 0; + goto out_unlock_daemon; + } + /* This daemon will not go away so long as this flag is set */ + daemon->flags |= ECRYPTFS_DAEMON_IN_READ; +check_list: + if (list_empty(&daemon->msg_ctx_out_queue)) { + mutex_unlock(&daemon->mux); + rc = wait_event_interruptible( + daemon->wait, !list_empty(&daemon->msg_ctx_out_queue)); + mutex_lock(&daemon->mux); + if (rc < 0) { + rc = 0; + goto out_unlock_daemon; + } + } + if (daemon->flags & ECRYPTFS_DAEMON_ZOMBIE) { + rc = 0; + goto out_unlock_daemon; + } + if (list_empty(&daemon->msg_ctx_out_queue)) { + /* Something else jumped in since the + * wait_event_interruptable() and removed the + * message from the queue; try again */ + goto check_list; + } + msg_ctx = list_first_entry(&daemon->msg_ctx_out_queue, + struct ecryptfs_msg_ctx, daemon_out_list); + BUG_ON(!msg_ctx); + mutex_lock(&msg_ctx->mux); + if (msg_ctx->msg) { + rc = ecryptfs_write_packet_length(packet_length, + msg_ctx->msg_size, + &packet_length_size); + if (rc) { + rc = 0; + printk(KERN_WARNING "%s: Error writing packet length; " + "rc = [%d]\n", __func__, rc); + goto out_unlock_msg_ctx; + } + } else { + packet_length_size = 0; + msg_ctx->msg_size = 0; + } + total_length = (PKT_TYPE_SIZE + PKT_CTR_SIZE + packet_length_size + + msg_ctx->msg_size); + if (count < total_length) { + rc = 0; + printk(KERN_WARNING "%s: Only given user buffer of " + "size [%zd], but we need [%zd] to read the " + "pending message\n", __func__, count, total_length); + goto out_unlock_msg_ctx; + } + rc = -EFAULT; + if (put_user(msg_ctx->type, buf)) + goto out_unlock_msg_ctx; + if (put_user(cpu_to_be32(msg_ctx->counter), + (__be32 __user *)(&buf[PKT_CTR_OFFSET]))) + goto out_unlock_msg_ctx; + i = PKT_TYPE_SIZE + PKT_CTR_SIZE; + if (msg_ctx->msg) { + if (copy_to_user(&buf[i], packet_length, packet_length_size)) + goto out_unlock_msg_ctx; + i += packet_length_size; + if (copy_to_user(&buf[i], msg_ctx->msg, msg_ctx->msg_size)) + goto out_unlock_msg_ctx; + i += msg_ctx->msg_size; + } + rc = i; + list_del(&msg_ctx->daemon_out_list); + kfree(msg_ctx->msg); + msg_ctx->msg = NULL; + /* We do not expect a reply from the userspace daemon for any + * message type other than ECRYPTFS_MSG_REQUEST */ + if (msg_ctx->type != ECRYPTFS_MSG_REQUEST) + ecryptfs_msg_ctx_alloc_to_free(msg_ctx); +out_unlock_msg_ctx: + mutex_unlock(&msg_ctx->mux); +out_unlock_daemon: + daemon->flags &= ~ECRYPTFS_DAEMON_IN_READ; + mutex_unlock(&daemon->mux); + return rc; +} + +/** + * ecryptfs_miscdev_response - miscdevess response to message previously sent to daemon + * @daemon: eCryptfs daemon object + * @data: Bytes comprising struct ecryptfs_message + * @data_size: sizeof(struct ecryptfs_message) + data len + * @seq: Sequence number for miscdev response packet + * + * Returns zero on success; non-zero otherwise + */ +static int ecryptfs_miscdev_response(struct ecryptfs_daemon *daemon, char *data, + size_t data_size, u32 seq) +{ + struct ecryptfs_message *msg = (struct ecryptfs_message *)data; + int rc; + + if ((sizeof(*msg) + msg->data_len) != data_size) { + printk(KERN_WARNING "%s: (sizeof(*msg) + msg->data_len) = " + "[%zd]; data_size = [%zd]. Invalid packet.\n", __func__, + (sizeof(*msg) + msg->data_len), data_size); + rc = -EINVAL; + goto out; + } + rc = ecryptfs_process_response(daemon, msg, seq); + if (rc) + printk(KERN_ERR + "Error processing response message; rc = [%d]\n", rc); +out: + return rc; +} + +/** + * ecryptfs_miscdev_write - handle write to daemon miscdev handle + * @file: File for misc dev handle + * @buf: Buffer containing user data + * @count: Amount of data in @buf + * @ppos: Pointer to offset in file (ignored) + * + * Returns the number of bytes read from @buf + */ +static ssize_t +ecryptfs_miscdev_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + __be32 counter_nbo; + u32 seq; + size_t packet_size, packet_size_length; + char *data; + unsigned char packet_size_peek[ECRYPTFS_MAX_PKT_LEN_SIZE] = { }; + ssize_t rc; + + if (count == 0) { + return 0; + } else if (count == MIN_NON_MSG_PKT_SIZE) { + /* Likely a harmless MSG_HELO or MSG_QUIT - no packet length */ + goto memdup; + } else if (count < MIN_MSG_PKT_SIZE || count > MAX_MSG_PKT_SIZE) { + printk(KERN_WARNING "%s: Acceptable packet size range is " + "[%d-%zu], but amount of data written is [%zu].\n", + __func__, MIN_MSG_PKT_SIZE, MAX_MSG_PKT_SIZE, count); + return -EINVAL; + } + + if (copy_from_user(packet_size_peek, &buf[PKT_LEN_OFFSET], + min_t(size_t, count - PKT_LEN_OFFSET, + sizeof(packet_size_peek)))) { + printk(KERN_WARNING "%s: Error while inspecting packet size\n", + __func__); + return -EFAULT; + } + + rc = ecryptfs_parse_packet_length(packet_size_peek, &packet_size, + &packet_size_length); + if (rc) { + printk(KERN_WARNING "%s: Error parsing packet length; " + "rc = [%zd]\n", __func__, rc); + return rc; + } + + if ((PKT_TYPE_SIZE + PKT_CTR_SIZE + packet_size_length + packet_size) + != count) { + printk(KERN_WARNING "%s: Invalid packet size [%zu]\n", __func__, + packet_size); + return -EINVAL; + } + +memdup: + data = memdup_user(buf, count); + if (IS_ERR(data)) { + printk(KERN_ERR "%s: memdup_user returned error [%ld]\n", + __func__, PTR_ERR(data)); + return PTR_ERR(data); + } + switch (data[PKT_TYPE_OFFSET]) { + case ECRYPTFS_MSG_RESPONSE: + if (count < (MIN_MSG_PKT_SIZE + + sizeof(struct ecryptfs_message))) { + printk(KERN_WARNING "%s: Minimum acceptable packet " + "size is [%zd], but amount of data written is " + "only [%zd]. Discarding response packet.\n", + __func__, + (MIN_MSG_PKT_SIZE + + sizeof(struct ecryptfs_message)), count); + rc = -EINVAL; + goto out_free; + } + memcpy(&counter_nbo, &data[PKT_CTR_OFFSET], PKT_CTR_SIZE); + seq = be32_to_cpu(counter_nbo); + rc = ecryptfs_miscdev_response(file->private_data, + &data[PKT_LEN_OFFSET + packet_size_length], + packet_size, seq); + if (rc) { + printk(KERN_WARNING "%s: Failed to deliver miscdev " + "response to requesting operation; rc = [%zd]\n", + __func__, rc); + goto out_free; + } + break; + case ECRYPTFS_MSG_HELO: + case ECRYPTFS_MSG_QUIT: + break; + default: + ecryptfs_printk(KERN_WARNING, "Dropping miscdev " + "message of unrecognized type [%d]\n", + data[0]); + rc = -EINVAL; + goto out_free; + } + rc = count; +out_free: + kfree(data); + return rc; +} + + +static const struct file_operations ecryptfs_miscdev_fops = { + .owner = THIS_MODULE, + .open = ecryptfs_miscdev_open, + .poll = ecryptfs_miscdev_poll, + .read = ecryptfs_miscdev_read, + .write = ecryptfs_miscdev_write, + .release = ecryptfs_miscdev_release, + .llseek = noop_llseek, +}; + +static struct miscdevice ecryptfs_miscdev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "ecryptfs", + .fops = &ecryptfs_miscdev_fops +}; + +/** + * ecryptfs_init_ecryptfs_miscdev + * + * Messages sent to the userspace daemon from the kernel are placed on + * a queue associated with the daemon. The next read against the + * miscdev handle by that daemon will return the oldest message placed + * on the message queue for the daemon. + * + * Returns zero on success; non-zero otherwise + */ +int __init ecryptfs_init_ecryptfs_miscdev(void) +{ + int rc; + + atomic_set(&ecryptfs_num_miscdev_opens, 0); + rc = misc_register(&ecryptfs_miscdev); + if (rc) + printk(KERN_ERR "%s: Failed to register miscellaneous device " + "for communications with userspace daemons; rc = [%d]\n", + __func__, rc); + return rc; +} + +/** + * ecryptfs_destroy_ecryptfs_miscdev + * + * All of the daemons must be exorcised prior to calling this + * function. + */ +void ecryptfs_destroy_ecryptfs_miscdev(void) +{ + BUG_ON(atomic_read(&ecryptfs_num_miscdev_opens) != 0); + misc_deregister(&ecryptfs_miscdev); +} diff --git a/fs/ecryptfs/mmap.c b/fs/ecryptfs/mmap.c new file mode 100644 index 000000000..07a6c8448 --- /dev/null +++ b/fs/ecryptfs/mmap.c @@ -0,0 +1,514 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * eCryptfs: Linux filesystem encryption layer + * This is where eCryptfs coordinates the symmetric encryption and + * decryption of the file data as it passes between the lower + * encrypted file and the upper decrypted file. + * + * Copyright (C) 1997-2003 Erez Zadok + * Copyright (C) 2001-2003 Stony Brook University + * Copyright (C) 2004-2007 International Business Machines Corp. + * Author(s): Michael A. Halcrow + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ecryptfs_kernel.h" + +/* + * This is where we encrypt the data and pass the encrypted data to + * the lower filesystem. In OpenPGP-compatible mode, we operate on + * entire underlying packets. + */ +static int ecryptfs_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct folio *folio = NULL; + int error; + + while ((folio = writeback_iter(mapping, wbc, folio, &error))) { + error = ecryptfs_encrypt_page(folio); + if (error) { + ecryptfs_printk(KERN_WARNING, + "Error encrypting folio (index [0x%.16lx])\n", + folio->index); + folio_clear_uptodate(folio); + mapping_set_error(mapping, error); + } + folio_unlock(folio); + } + + return error; +} + +static void strip_xattr_flag(char *page_virt, + struct ecryptfs_crypt_stat *crypt_stat) +{ + if (crypt_stat->flags & ECRYPTFS_METADATA_IN_XATTR) { + size_t written; + + crypt_stat->flags &= ~ECRYPTFS_METADATA_IN_XATTR; + ecryptfs_write_crypt_stat_flags(page_virt, crypt_stat, + &written); + crypt_stat->flags |= ECRYPTFS_METADATA_IN_XATTR; + } +} + +/* + * Header Extent: + * Octets 0-7: Unencrypted file size (big-endian) + * Octets 8-15: eCryptfs special marker + * Octets 16-19: Flags + * Octet 16: File format version number (between 0 and 255) + * Octets 17-18: Reserved + * Octet 19: Bit 1 (lsb): Reserved + * Bit 2: Encrypted? + * Bits 3-8: Reserved + * Octets 20-23: Header extent size (big-endian) + * Octets 24-25: Number of header extents at front of file + * (big-endian) + * Octet 26: Begin RFC 2440 authentication token packet set + */ + +/** + * ecryptfs_copy_up_encrypted_with_header + * @folio: Sort of a ``virtual'' representation of the encrypted lower + * file. The actual lower file does not have the metadata in + * the header. This is locked. + * @crypt_stat: The eCryptfs inode's cryptographic context + * + * The ``view'' is the version of the file that userspace winds up + * seeing, with the header information inserted. + */ +static int +ecryptfs_copy_up_encrypted_with_header(struct folio *folio, + struct ecryptfs_crypt_stat *crypt_stat) +{ + loff_t extent_num_in_page = 0; + loff_t num_extents_per_page = (PAGE_SIZE + / crypt_stat->extent_size); + int rc = 0; + + while (extent_num_in_page < num_extents_per_page) { + loff_t view_extent_num = ((loff_t)folio->index + * num_extents_per_page) + + extent_num_in_page; + size_t num_header_extents_at_front = + (crypt_stat->metadata_size / crypt_stat->extent_size); + + if (view_extent_num < num_header_extents_at_front) { + /* This is a header extent */ + char *page_virt; + + page_virt = kmap_local_folio(folio, 0); + memset(page_virt, 0, PAGE_SIZE); + /* TODO: Support more than one header extent */ + if (view_extent_num == 0) { + size_t written; + + rc = ecryptfs_read_xattr_region( + page_virt, folio->mapping->host); + strip_xattr_flag(page_virt + 16, crypt_stat); + ecryptfs_write_header_metadata(page_virt + 20, + crypt_stat, + &written); + } + kunmap_local(page_virt); + flush_dcache_folio(folio); + if (rc) { + printk(KERN_ERR "%s: Error reading xattr " + "region; rc = [%d]\n", __func__, rc); + goto out; + } + } else { + /* This is an encrypted data extent */ + loff_t lower_offset = + ((view_extent_num * crypt_stat->extent_size) + - crypt_stat->metadata_size); + + rc = ecryptfs_read_lower_page_segment( + folio, (lower_offset >> PAGE_SHIFT), + (lower_offset & ~PAGE_MASK), + crypt_stat->extent_size, folio->mapping->host); + if (rc) { + printk(KERN_ERR "%s: Error attempting to read " + "extent at offset [%lld] in the lower " + "file; rc = [%d]\n", __func__, + lower_offset, rc); + goto out; + } + } + extent_num_in_page++; + } +out: + return rc; +} + +/** + * ecryptfs_read_folio + * @file: An eCryptfs file + * @folio: Folio from eCryptfs inode mapping into which to stick the read data + * + * Read in a folio, decrypting if necessary. + * + * Returns zero on success; non-zero on error. + */ +static int ecryptfs_read_folio(struct file *file, struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + struct ecryptfs_crypt_stat *crypt_stat = + &ecryptfs_inode_to_private(inode)->crypt_stat; + int err = 0; + + if (!crypt_stat || !(crypt_stat->flags & ECRYPTFS_ENCRYPTED)) { + err = ecryptfs_read_lower_page_segment(folio, folio->index, 0, + folio_size(folio), inode); + } else if (crypt_stat->flags & ECRYPTFS_VIEW_AS_ENCRYPTED) { + if (crypt_stat->flags & ECRYPTFS_METADATA_IN_XATTR) { + err = ecryptfs_copy_up_encrypted_with_header(folio, + crypt_stat); + if (err) { + printk(KERN_ERR "%s: Error attempting to copy " + "the encrypted content from the lower " + "file whilst inserting the metadata " + "from the xattr into the header; err = " + "[%d]\n", __func__, err); + goto out; + } + + } else { + err = ecryptfs_read_lower_page_segment(folio, + folio->index, 0, folio_size(folio), + inode); + if (err) { + printk(KERN_ERR "Error reading page; err = " + "[%d]\n", err); + goto out; + } + } + } else { + err = ecryptfs_decrypt_page(folio); + if (err) { + ecryptfs_printk(KERN_ERR, "Error decrypting page; " + "err = [%d]\n", err); + goto out; + } + } +out: + ecryptfs_printk(KERN_DEBUG, "Unlocking folio with index = [0x%.16lx]\n", + folio->index); + folio_end_read(folio, err == 0); + return err; +} + +/* + * Called with lower inode mutex held. + */ +static int fill_zeros_to_end_of_page(struct folio *folio, unsigned int to) +{ + struct inode *inode = folio->mapping->host; + int end_byte_in_page; + + if ((i_size_read(inode) / PAGE_SIZE) != folio->index) + goto out; + end_byte_in_page = i_size_read(inode) % PAGE_SIZE; + if (to > end_byte_in_page) + end_byte_in_page = to; + folio_zero_segment(folio, end_byte_in_page, PAGE_SIZE); +out: + return 0; +} + +/** + * ecryptfs_write_begin + * @iocb: I/O control block for the eCryptfs file + * @mapping: The eCryptfs object + * @pos: The file offset at which to start writing + * @len: Length of the write + * @foliop: Pointer to return the folio + * @fsdata: Pointer to return fs data (unused) + * + * This function must zero any hole we create + * + * Returns zero on success; non-zero otherwise + */ +static int ecryptfs_write_begin(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, + struct folio **foliop, void **fsdata) +{ + pgoff_t index = pos >> PAGE_SHIFT; + struct folio *folio; + loff_t prev_page_end_size; + int rc = 0; + + folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN, + mapping_gfp_mask(mapping)); + if (IS_ERR(folio)) + return PTR_ERR(folio); + *foliop = folio; + + prev_page_end_size = ((loff_t)index << PAGE_SHIFT); + if (!folio_test_uptodate(folio)) { + struct ecryptfs_crypt_stat *crypt_stat = + &ecryptfs_inode_to_private(mapping->host)->crypt_stat; + + if (!(crypt_stat->flags & ECRYPTFS_ENCRYPTED)) { + rc = ecryptfs_read_lower_page_segment( + folio, index, 0, PAGE_SIZE, mapping->host); + if (rc) { + printk(KERN_ERR "%s: Error attempting to read " + "lower page segment; rc = [%d]\n", + __func__, rc); + folio_clear_uptodate(folio); + goto out; + } else + folio_mark_uptodate(folio); + } else if (crypt_stat->flags & ECRYPTFS_VIEW_AS_ENCRYPTED) { + if (crypt_stat->flags & ECRYPTFS_METADATA_IN_XATTR) { + rc = ecryptfs_copy_up_encrypted_with_header( + folio, crypt_stat); + if (rc) { + printk(KERN_ERR "%s: Error attempting " + "to copy the encrypted content " + "from the lower file whilst " + "inserting the metadata from " + "the xattr into the header; rc " + "= [%d]\n", __func__, rc); + folio_clear_uptodate(folio); + goto out; + } + folio_mark_uptodate(folio); + } else { + rc = ecryptfs_read_lower_page_segment( + folio, index, 0, PAGE_SIZE, + mapping->host); + if (rc) { + printk(KERN_ERR "%s: Error reading " + "page; rc = [%d]\n", + __func__, rc); + folio_clear_uptodate(folio); + goto out; + } + folio_mark_uptodate(folio); + } + } else { + if (prev_page_end_size + >= i_size_read(mapping->host)) { + folio_zero_range(folio, 0, PAGE_SIZE); + folio_mark_uptodate(folio); + } else if (len < PAGE_SIZE) { + rc = ecryptfs_decrypt_page(folio); + if (rc) { + printk(KERN_ERR "%s: Error decrypting " + "page at index [%ld]; " + "rc = [%d]\n", + __func__, folio->index, rc); + folio_clear_uptodate(folio); + goto out; + } + folio_mark_uptodate(folio); + } + } + } + /* If creating a page or more of holes, zero them out via truncate. + * Note, this will increase i_size. */ + if (index != 0) { + if (prev_page_end_size > i_size_read(mapping->host)) { + rc = ecryptfs_truncate(iocb->ki_filp->f_path.dentry, + prev_page_end_size); + if (rc) { + printk(KERN_ERR "%s: Error on attempt to " + "truncate to (higher) offset [%lld];" + " rc = [%d]\n", __func__, + prev_page_end_size, rc); + goto out; + } + } + } + /* Writing to a new page, and creating a small hole from start + * of page? Zero it out. */ + if ((i_size_read(mapping->host) == prev_page_end_size) + && (pos != 0)) + folio_zero_range(folio, 0, PAGE_SIZE); +out: + if (unlikely(rc)) { + folio_unlock(folio); + folio_put(folio); + } + return rc; +} + +/* + * ecryptfs_write_inode_size_to_header + * + * Writes the lower file size to the first 8 bytes of the header. + * + * Returns zero on success; non-zero on error. + */ +static int ecryptfs_write_inode_size_to_header(struct inode *ecryptfs_inode) +{ + __be64 file_size; + int rc; + + file_size = cpu_to_be64(i_size_read(ecryptfs_inode)); + rc = ecryptfs_write_lower(ecryptfs_inode, (char *)&file_size, 0, + sizeof(file_size)); + if (rc < 0) + printk(KERN_ERR "%s: Error writing file size to header; " + "rc = [%d]\n", __func__, rc); + else + rc = 0; + return rc; +} + +struct kmem_cache *ecryptfs_xattr_cache; + +static int ecryptfs_write_inode_size_to_xattr(struct inode *ecryptfs_inode) +{ + ssize_t size; + void *xattr_virt; + struct dentry *lower_dentry = + ecryptfs_inode_to_private(ecryptfs_inode)->lower_file->f_path.dentry; + struct inode *lower_inode = d_inode(lower_dentry); + int rc; + + if (!(lower_inode->i_opflags & IOP_XATTR)) { + printk(KERN_WARNING + "No support for setting xattr in lower filesystem\n"); + rc = -ENOSYS; + goto out; + } + xattr_virt = kmem_cache_alloc(ecryptfs_xattr_cache, GFP_KERNEL); + if (!xattr_virt) { + rc = -ENOMEM; + goto out; + } + inode_lock(lower_inode); + size = __vfs_getxattr(lower_dentry, lower_inode, ECRYPTFS_XATTR_NAME, + xattr_virt, PAGE_SIZE); + if (size < 0) + size = 8; + put_unaligned_be64(i_size_read(ecryptfs_inode), xattr_virt); + rc = __vfs_setxattr(&nop_mnt_idmap, lower_dentry, lower_inode, + ECRYPTFS_XATTR_NAME, xattr_virt, size, 0); + inode_unlock(lower_inode); + if (rc) + printk(KERN_ERR "Error whilst attempting to write inode size " + "to lower file xattr; rc = [%d]\n", rc); + kmem_cache_free(ecryptfs_xattr_cache, xattr_virt); +out: + return rc; +} + +int ecryptfs_write_inode_size_to_metadata(struct inode *ecryptfs_inode) +{ + struct ecryptfs_crypt_stat *crypt_stat; + + crypt_stat = &ecryptfs_inode_to_private(ecryptfs_inode)->crypt_stat; + BUG_ON(!(crypt_stat->flags & ECRYPTFS_ENCRYPTED)); + if (crypt_stat->flags & ECRYPTFS_METADATA_IN_XATTR) + return ecryptfs_write_inode_size_to_xattr(ecryptfs_inode); + else + return ecryptfs_write_inode_size_to_header(ecryptfs_inode); +} + +/** + * ecryptfs_write_end + * @iocb: I/O control block for the eCryptfs file + * @mapping: The eCryptfs object + * @pos: The file position + * @len: The length of the data (unused) + * @copied: The amount of data copied + * @folio: The eCryptfs folio + * @fsdata: The fsdata (unused) + */ +static int ecryptfs_write_end(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct folio *folio, void *fsdata) +{ + pgoff_t index = pos >> PAGE_SHIFT; + unsigned from = pos & (PAGE_SIZE - 1); + unsigned to = from + copied; + struct inode *ecryptfs_inode = mapping->host; + struct ecryptfs_crypt_stat *crypt_stat = + &ecryptfs_inode_to_private(ecryptfs_inode)->crypt_stat; + int rc; + + ecryptfs_printk(KERN_DEBUG, "Calling fill_zeros_to_end_of_page" + "(page w/ index = [0x%.16lx], to = [%d])\n", index, to); + if (!(crypt_stat->flags & ECRYPTFS_ENCRYPTED)) { + rc = ecryptfs_write_lower_page_segment(ecryptfs_inode, + folio, 0, to); + if (!rc) { + rc = copied; + fsstack_copy_inode_size(ecryptfs_inode, + ecryptfs_inode_to_lower(ecryptfs_inode)); + } + goto out; + } + if (!folio_test_uptodate(folio)) { + if (copied < PAGE_SIZE) { + rc = 0; + goto out; + } + folio_mark_uptodate(folio); + } + /* Fills in zeros if 'to' goes beyond inode size */ + rc = fill_zeros_to_end_of_page(folio, to); + if (rc) { + ecryptfs_printk(KERN_WARNING, "Error attempting to fill " + "zeros in page with index = [0x%.16lx]\n", index); + goto out; + } + rc = ecryptfs_encrypt_page(folio); + if (rc) { + ecryptfs_printk(KERN_WARNING, "Error encrypting page (upper " + "index [0x%.16lx])\n", index); + goto out; + } + if (pos + copied > i_size_read(ecryptfs_inode)) { + i_size_write(ecryptfs_inode, pos + copied); + ecryptfs_printk(KERN_DEBUG, "Expanded file size to " + "[0x%.16llx]\n", + (unsigned long long)i_size_read(ecryptfs_inode)); + } + rc = ecryptfs_write_inode_size_to_metadata(ecryptfs_inode); + if (rc) + printk(KERN_ERR "Error writing inode size to metadata; " + "rc = [%d]\n", rc); + else + rc = copied; +out: + folio_unlock(folio); + folio_put(folio); + return rc; +} + +static sector_t ecryptfs_bmap(struct address_space *mapping, sector_t block) +{ + struct inode *lower_inode = ecryptfs_inode_to_lower(mapping->host); + int ret = bmap(lower_inode, &block); + + if (ret) + return 0; + return block; +} + +const struct address_space_operations ecryptfs_aops = { + .dirty_folio = filemap_dirty_folio, + .writepages = ecryptfs_writepages, + .read_folio = ecryptfs_read_folio, + .write_begin = ecryptfs_write_begin, + .write_end = ecryptfs_write_end, + .migrate_folio = filemap_migrate_folio, + .bmap = ecryptfs_bmap, +}; diff --git a/fs/ecryptfs/read_write.c b/fs/ecryptfs/read_write.c new file mode 100644 index 000000000..b3b451c2b --- /dev/null +++ b/fs/ecryptfs/read_write.c @@ -0,0 +1,263 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * eCryptfs: Linux filesystem encryption layer + * + * Copyright (C) 2007 International Business Machines Corp. + * Author(s): Michael A. Halcrow + */ + +#include +#include +#include + +#include "ecryptfs_kernel.h" + +/** + * ecryptfs_write_lower + * @ecryptfs_inode: The eCryptfs inode + * @data: Data to write + * @offset: Byte offset in the lower file to which to write the data + * @size: Number of bytes from @data to write at @offset in the lower + * file + * + * Write data to the lower file. + * + * Returns bytes written on success; less than zero on error + */ +int ecryptfs_write_lower(struct inode *ecryptfs_inode, char *data, + loff_t offset, size_t size) +{ + struct file *lower_file; + ssize_t rc; + + lower_file = ecryptfs_inode_to_private(ecryptfs_inode)->lower_file; + if (!lower_file) + return -EIO; + rc = kernel_write(lower_file, data, size, &offset); + mark_inode_dirty_sync(ecryptfs_inode); + return rc; +} + +/** + * ecryptfs_write_lower_page_segment + * @ecryptfs_inode: The eCryptfs inode + * @folio_for_lower: The folio containing the data to be written to the + * lower file + * @offset_in_page: The offset in the @folio_for_lower from which to + * start writing the data + * @size: The amount of data from @folio_for_lower to write to the + * lower file + * + * Determines the byte offset in the file for the given page and + * offset within the page, maps the page, and makes the call to write + * the contents of @folio_for_lower to the lower inode. + * + * Returns zero on success; non-zero otherwise + */ +int ecryptfs_write_lower_page_segment(struct inode *ecryptfs_inode, + struct folio *folio_for_lower, + size_t offset_in_page, size_t size) +{ + char *virt; + loff_t offset; + int rc; + + offset = (loff_t)folio_for_lower->index * PAGE_SIZE + offset_in_page; + virt = kmap_local_folio(folio_for_lower, 0); + rc = ecryptfs_write_lower(ecryptfs_inode, virt, offset, size); + if (rc > 0) + rc = 0; + kunmap_local(virt); + return rc; +} + +/** + * ecryptfs_write + * @ecryptfs_inode: The eCryptfs file into which to write + * @data: Virtual address where data to write is located + * @offset: Offset in the eCryptfs file at which to begin writing the + * data from @data + * @size: The number of bytes to write from @data + * + * Write an arbitrary amount of data to an arbitrary location in the + * eCryptfs inode page cache. This is done on a page-by-page, and then + * by an extent-by-extent, basis; individual extents are encrypted and + * written to the lower page cache (via VFS writes). This function + * takes care of all the address translation to locations in the lower + * filesystem; it also handles truncate events, writing out zeros + * where necessary. + * + * Returns zero on success; non-zero otherwise + */ +int ecryptfs_write(struct inode *ecryptfs_inode, char *data, loff_t offset, + size_t size) +{ + struct ecryptfs_crypt_stat *crypt_stat; + char *ecryptfs_page_virt; + loff_t ecryptfs_file_size = i_size_read(ecryptfs_inode); + loff_t data_offset = 0; + loff_t pos; + int rc = 0; + + crypt_stat = &ecryptfs_inode_to_private(ecryptfs_inode)->crypt_stat; + /* + * if we are writing beyond current size, then start pos + * at the current size - we'll fill in zeros from there. + */ + if (offset > ecryptfs_file_size) + pos = ecryptfs_file_size; + else + pos = offset; + while (pos < (offset + size)) { + struct folio *ecryptfs_folio; + pgoff_t ecryptfs_page_idx = (pos >> PAGE_SHIFT); + size_t start_offset_in_page = (pos & ~PAGE_MASK); + size_t num_bytes = (PAGE_SIZE - start_offset_in_page); + loff_t total_remaining_bytes = ((offset + size) - pos); + + if (fatal_signal_pending(current)) { + rc = -EINTR; + break; + } + + if (num_bytes > total_remaining_bytes) + num_bytes = total_remaining_bytes; + if (pos < offset) { + /* remaining zeros to write, up to destination offset */ + loff_t total_remaining_zeros = (offset - pos); + + if (num_bytes > total_remaining_zeros) + num_bytes = total_remaining_zeros; + } + ecryptfs_folio = read_mapping_folio(ecryptfs_inode->i_mapping, + ecryptfs_page_idx, NULL); + if (IS_ERR(ecryptfs_folio)) { + rc = PTR_ERR(ecryptfs_folio); + printk(KERN_ERR "%s: Error getting page at " + "index [%ld] from eCryptfs inode " + "mapping; rc = [%d]\n", __func__, + ecryptfs_page_idx, rc); + goto out; + } + folio_lock(ecryptfs_folio); + ecryptfs_page_virt = kmap_local_folio(ecryptfs_folio, 0); + + /* + * pos: where we're now writing, offset: where the request was + * If current pos is before request, we are filling zeros + * If we are at or beyond request, we are writing the *data* + * If we're in a fresh page beyond eof, zero it in either case + */ + if (pos < offset || !start_offset_in_page) { + /* We are extending past the previous end of the file. + * Fill in zero values to the end of the page */ + memset(((char *)ecryptfs_page_virt + + start_offset_in_page), 0, + PAGE_SIZE - start_offset_in_page); + } + + /* pos >= offset, we are now writing the data request */ + if (pos >= offset) { + memcpy(((char *)ecryptfs_page_virt + + start_offset_in_page), + (data + data_offset), num_bytes); + data_offset += num_bytes; + } + kunmap_local(ecryptfs_page_virt); + flush_dcache_folio(ecryptfs_folio); + folio_mark_uptodate(ecryptfs_folio); + folio_unlock(ecryptfs_folio); + if (crypt_stat->flags & ECRYPTFS_ENCRYPTED) + rc = ecryptfs_encrypt_page(ecryptfs_folio); + else + rc = ecryptfs_write_lower_page_segment(ecryptfs_inode, + ecryptfs_folio, + start_offset_in_page, + data_offset); + folio_put(ecryptfs_folio); + if (rc) { + printk(KERN_ERR "%s: Error encrypting " + "page; rc = [%d]\n", __func__, rc); + goto out; + } + pos += num_bytes; + } + if (pos > ecryptfs_file_size) { + i_size_write(ecryptfs_inode, pos); + if (crypt_stat->flags & ECRYPTFS_ENCRYPTED) { + int rc2; + + rc2 = ecryptfs_write_inode_size_to_metadata( + ecryptfs_inode); + if (rc2) { + printk(KERN_ERR "Problem with " + "ecryptfs_write_inode_size_to_metadata; " + "rc = [%d]\n", rc2); + if (!rc) + rc = rc2; + goto out; + } + } + } +out: + return rc; +} + +/** + * ecryptfs_read_lower + * @data: The read data is stored here by this function + * @offset: Byte offset in the lower file from which to read the data + * @size: Number of bytes to read from @offset of the lower file and + * store into @data + * @ecryptfs_inode: The eCryptfs inode + * + * Read @size bytes of data at byte offset @offset from the lower + * inode into memory location @data. + * + * Returns bytes read on success; 0 on EOF; less than zero on error + */ +int ecryptfs_read_lower(char *data, loff_t offset, size_t size, + struct inode *ecryptfs_inode) +{ + struct file *lower_file; + lower_file = ecryptfs_inode_to_private(ecryptfs_inode)->lower_file; + if (!lower_file) + return -EIO; + return kernel_read(lower_file, data, size, &offset); +} + +/** + * ecryptfs_read_lower_page_segment + * @folio_for_ecryptfs: The folio into which data for eCryptfs will be + * written + * @page_index: Page index in @page_for_ecryptfs from which to start + * writing + * @offset_in_page: Offset in @page_for_ecryptfs from which to start + * writing + * @size: The number of bytes to write into @page_for_ecryptfs + * @ecryptfs_inode: The eCryptfs inode + * + * Determines the byte offset in the file for the given page and + * offset within the page, maps the page, and makes the call to read + * the contents of @page_for_ecryptfs from the lower inode. + * + * Returns zero on success; non-zero otherwise + */ +int ecryptfs_read_lower_page_segment(struct folio *folio_for_ecryptfs, + pgoff_t page_index, + size_t offset_in_page, size_t size, + struct inode *ecryptfs_inode) +{ + char *virt; + loff_t offset; + int rc; + + offset = (loff_t)page_index * PAGE_SIZE + offset_in_page; + virt = kmap_local_folio(folio_for_ecryptfs, 0); + rc = ecryptfs_read_lower(virt, offset, size, ecryptfs_inode); + if (rc > 0) + rc = 0; + kunmap_local(virt); + flush_dcache_folio(folio_for_ecryptfs); + return rc; +} diff --git a/fs/ecryptfs/super.c b/fs/ecryptfs/super.c new file mode 100644 index 000000000..686b2b4a9 --- /dev/null +++ b/fs/ecryptfs/super.c @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * eCryptfs: Linux filesystem encryption layer + * + * Copyright (C) 1997-2003 Erez Zadok + * Copyright (C) 2001-2003 Stony Brook University + * Copyright (C) 2004-2006 International Business Machines Corp. + * Author(s): Michael A. Halcrow + * Michael C. Thompson + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "ecryptfs_kernel.h" + +struct kmem_cache *ecryptfs_inode_info_cache; + +/** + * ecryptfs_alloc_inode - allocate an ecryptfs inode + * @sb: Pointer to the ecryptfs super block + * + * Called to bring an inode into existence. + * + * Only handle allocation, setting up structures should be done in + * ecryptfs_read_inode. This is because the kernel, between now and + * then, will 0 out the private data pointer. + * + * Returns a pointer to a newly allocated inode, NULL otherwise + */ +static struct inode *ecryptfs_alloc_inode(struct super_block *sb) +{ + struct ecryptfs_inode_info *inode_info; + struct inode *inode = NULL; + + inode_info = alloc_inode_sb(sb, ecryptfs_inode_info_cache, GFP_KERNEL); + if (unlikely(!inode_info)) + goto out; + ecryptfs_init_crypt_stat(&inode_info->crypt_stat); + mutex_init(&inode_info->lower_file_mutex); + atomic_set(&inode_info->lower_file_count, 0); + inode_info->lower_file = NULL; + inode = &inode_info->vfs_inode; +out: + return inode; +} + +static void ecryptfs_free_inode(struct inode *inode) +{ + struct ecryptfs_inode_info *inode_info; + inode_info = ecryptfs_inode_to_private(inode); + + kmem_cache_free(ecryptfs_inode_info_cache, inode_info); +} + +/** + * ecryptfs_destroy_inode + * @inode: The ecryptfs inode + * + * This is used during the final destruction of the inode. All + * allocation of memory related to the inode, including allocated + * memory in the crypt_stat struct, will be released here. + * There should be no chance that this deallocation will be missed. + */ +static void ecryptfs_destroy_inode(struct inode *inode) +{ + struct ecryptfs_inode_info *inode_info; + + inode_info = ecryptfs_inode_to_private(inode); + BUG_ON(inode_info->lower_file); + ecryptfs_destroy_crypt_stat(&inode_info->crypt_stat); +} + +/** + * ecryptfs_statfs + * @dentry: The ecryptfs dentry + * @buf: The struct kstatfs to fill in with stats + * + * Get the filesystem statistics. Currently, we let this pass right through + * to the lower filesystem and take no action ourselves. + */ +static int ecryptfs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct dentry *lower_dentry = ecryptfs_dentry_to_lower(dentry); + int rc; + + if (!lower_dentry->d_sb->s_op->statfs) + return -ENOSYS; + + rc = lower_dentry->d_sb->s_op->statfs(lower_dentry, buf); + if (rc) + return rc; + + buf->f_type = ECRYPTFS_SUPER_MAGIC; + rc = ecryptfs_set_f_namelen(&buf->f_namelen, buf->f_namelen, + &ecryptfs_superblock_to_private(dentry->d_sb)->mount_crypt_stat); + + return rc; +} + +/** + * ecryptfs_evict_inode + * @inode: The ecryptfs inode + * + * Called by iput() when the inode reference count reached zero + * and the inode is not hashed anywhere. Used to clear anything + * that needs to be, before the inode is completely destroyed and put + * on the inode free list. We use this to drop out reference to the + * lower inode. + */ +static void ecryptfs_evict_inode(struct inode *inode) +{ + truncate_inode_pages_final(&inode->i_data); + clear_inode(inode); + iput(ecryptfs_inode_to_lower(inode)); +} + +/* + * ecryptfs_show_options + * + * Prints the mount options for a given superblock. + * Returns zero; does not fail. + */ +static int ecryptfs_show_options(struct seq_file *m, struct dentry *root) +{ + struct super_block *sb = root->d_sb; + struct ecryptfs_mount_crypt_stat *mount_crypt_stat = + &ecryptfs_superblock_to_private(sb)->mount_crypt_stat; + struct ecryptfs_global_auth_tok *walker; + + mutex_lock(&mount_crypt_stat->global_auth_tok_list_mutex); + list_for_each_entry(walker, + &mount_crypt_stat->global_auth_tok_list, + mount_crypt_stat_list) { + if (walker->flags & ECRYPTFS_AUTH_TOK_FNEK) + seq_printf(m, ",ecryptfs_fnek_sig=%s", walker->sig); + else + seq_printf(m, ",ecryptfs_sig=%s", walker->sig); + } + mutex_unlock(&mount_crypt_stat->global_auth_tok_list_mutex); + + seq_printf(m, ",ecryptfs_cipher=%s", + mount_crypt_stat->global_default_cipher_name); + + if (mount_crypt_stat->global_default_cipher_key_size) + seq_printf(m, ",ecryptfs_key_bytes=%zd", + mount_crypt_stat->global_default_cipher_key_size); + if (mount_crypt_stat->flags & ECRYPTFS_GLOBAL_ENCRYPT_FILENAMES) { + seq_printf(m, ",ecryptfs_fn_cipher=%s", + mount_crypt_stat->global_default_fn_cipher_name); + if (mount_crypt_stat->global_default_fn_cipher_key_bytes) + seq_printf(m, ",ecryptfs_fn_key_bytes=%zd", + mount_crypt_stat->global_default_fn_cipher_key_bytes); + } + if (mount_crypt_stat->flags & ECRYPTFS_PLAINTEXT_PASSTHROUGH_ENABLED) + seq_printf(m, ",ecryptfs_passthrough"); + if (mount_crypt_stat->flags & ECRYPTFS_XATTR_METADATA_ENABLED) + seq_printf(m, ",ecryptfs_xattr_metadata"); + if (mount_crypt_stat->flags & ECRYPTFS_ENCRYPTED_VIEW_ENABLED) + seq_printf(m, ",ecryptfs_encrypted_view"); + if (mount_crypt_stat->flags & ECRYPTFS_UNLINK_SIGS) + seq_printf(m, ",ecryptfs_unlink_sigs"); + if (mount_crypt_stat->flags & ECRYPTFS_GLOBAL_MOUNT_AUTH_TOK_ONLY) + seq_printf(m, ",ecryptfs_mount_auth_tok_only"); + + return 0; +} + +const struct super_operations ecryptfs_sops = { + .alloc_inode = ecryptfs_alloc_inode, + .destroy_inode = ecryptfs_destroy_inode, + .free_inode = ecryptfs_free_inode, + .statfs = ecryptfs_statfs, + .evict_inode = ecryptfs_evict_inode, + .show_options = ecryptfs_show_options +}; diff --git a/fs/efivarfs/Kconfig b/fs/efivarfs/Kconfig new file mode 100644 index 000000000..edec8a19c --- /dev/null +++ b/fs/efivarfs/Kconfig @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0-only +config EFIVAR_FS + tristate "EFI Variable filesystem" + depends on EFI + default m + help + efivarfs is a replacement filesystem for the old EFI + variable support via sysfs, as it doesn't suffer from the + same 1024-byte variable size limit. + + To compile this file system support as a module, choose M + here. The module will be called efivarfs. + + If unsure, say N. diff --git a/fs/efivarfs/Makefile b/fs/efivarfs/Makefile new file mode 100644 index 000000000..7bfc2f975 --- /dev/null +++ b/fs/efivarfs/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the efivarfs filesystem +# + +obj-$(CONFIG_EFIVAR_FS) += efivarfs.o + +efivarfs-objs := inode.o file.o super.o vars.o diff --git a/fs/efivarfs/file.c b/fs/efivarfs/file.c new file mode 100644 index 000000000..cb1b6d0c3 --- /dev/null +++ b/fs/efivarfs/file.c @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Red Hat, Inc. + * Copyright (C) 2012 Jeremy Kerr + */ + +#include +#include +#include +#include +#include + +#include "internal.h" + +static ssize_t efivarfs_file_write(struct file *file, + const char __user *userbuf, size_t count, loff_t *ppos) +{ + struct efivar_entry *var = file->private_data; + void *data; + u32 attributes; + struct inode *inode = file->f_mapping->host; + unsigned long datasize = count - sizeof(attributes); + ssize_t bytes; + bool set = false; + + if (count < sizeof(attributes)) + return -EINVAL; + + if (copy_from_user(&attributes, userbuf, sizeof(attributes))) + return -EFAULT; + + if (attributes & ~(EFI_VARIABLE_MASK)) + return -EINVAL; + + data = memdup_user(userbuf + sizeof(attributes), datasize); + if (IS_ERR(data)) + return PTR_ERR(data); + + inode_lock(inode); + if (var->removed) { + /* + * file got removed; don't allow a set. Caused by an + * unsuccessful create or successful delete write + * racing with us. + */ + bytes = -EIO; + goto out; + } + + bytes = efivar_entry_set_get_size(var, attributes, &datasize, + data, &set); + if (!set) { + if (bytes == -ENOENT) + bytes = -EIO; + goto out; + } + + if (bytes == -ENOENT) { + /* + * zero size signals to release that the write deleted + * the variable + */ + i_size_write(inode, 0); + } else { + i_size_write(inode, datasize + sizeof(attributes)); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + } + + bytes = count; + +out: + inode_unlock(inode); + + kfree(data); + + return bytes; +} + +static ssize_t efivarfs_file_read(struct file *file, char __user *userbuf, + size_t count, loff_t *ppos) +{ + struct efivar_entry *var = file->private_data; + unsigned long datasize = 0; + u32 attributes; + void *data; + ssize_t size = 0; + int err; + + while (!__ratelimit(&file->f_cred->user->ratelimit)) + msleep(50); + + err = efivar_entry_size(var, &datasize); + + /* + * efivarfs represents uncommitted variables with + * zero-length files. Reading them should return EOF. + */ + if (err == -ENOENT) + return 0; + else if (err) + return err; + + data = kmalloc(datasize + sizeof(attributes), GFP_KERNEL); + + if (!data) + return -ENOMEM; + + size = efivar_entry_get(var, &attributes, &datasize, + data + sizeof(attributes)); + if (size) + goto out_free; + + memcpy(data, &attributes, sizeof(attributes)); + size = simple_read_from_buffer(userbuf, count, ppos, + data, datasize + sizeof(attributes)); +out_free: + kfree(data); + + return size; +} + +static int efivarfs_file_release(struct inode *inode, struct file *file) +{ + struct efivar_entry *var = inode->i_private; + + inode_lock(inode); + var->removed = (--var->open_count == 0 && i_size_read(inode) == 0); + inode_unlock(inode); + + if (var->removed) + simple_recursive_removal(file->f_path.dentry, NULL); + + return 0; +} + +static int efivarfs_file_open(struct inode *inode, struct file *file) +{ + struct efivar_entry *entry = inode->i_private; + + file->private_data = entry; + + inode_lock(inode); + entry->open_count++; + inode_unlock(inode); + + return 0; +} + +const struct file_operations efivarfs_file_operations = { + .open = efivarfs_file_open, + .read = efivarfs_file_read, + .write = efivarfs_file_write, + .release = efivarfs_file_release, +}; diff --git a/fs/efivarfs/inode.c b/fs/efivarfs/inode.c new file mode 100644 index 000000000..f0d009555 --- /dev/null +++ b/fs/efivarfs/inode.c @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Red Hat, Inc. + * Copyright (C) 2012 Jeremy Kerr + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +static const struct inode_operations efivarfs_file_inode_operations; + +struct inode *efivarfs_get_inode(struct super_block *sb, + const struct inode *dir, int mode, + dev_t dev, bool is_removable) +{ + struct inode *inode = new_inode(sb); + struct efivarfs_fs_info *fsi = sb->s_fs_info; + struct efivarfs_mount_opts *opts = &fsi->mount_opts; + + if (inode) { + inode->i_uid = opts->uid; + inode->i_gid = opts->gid; + inode->i_ino = get_next_ino(); + inode->i_mode = mode; + simple_inode_init_ts(inode); + inode->i_flags = is_removable ? 0 : S_IMMUTABLE; + switch (mode & S_IFMT) { + case S_IFREG: + inode->i_op = &efivarfs_file_inode_operations; + inode->i_fop = &efivarfs_file_operations; + break; + case S_IFDIR: + inode->i_op = &efivarfs_dir_inode_operations; + inode->i_fop = &simple_dir_operations; + inc_nlink(inode); + break; + } + } + return inode; +} + +/* + * Return true if 'str' is a valid efivarfs filename of the form, + * + * VariableName-12345678-1234-1234-1234-1234567891bc + */ +static bool efivarfs_valid_name(const char *str, int len) +{ + const char *s = str + len - EFI_VARIABLE_GUID_LEN; + + /* + * We need a GUID, plus at least one letter for the variable name, + * plus the '-' separator + */ + if (len < EFI_VARIABLE_GUID_LEN + 2) + return false; + + /* GUID must be preceded by a '-' */ + if (*(s - 1) != '-') + return false; + + /* + * Validate that 's' is of the correct format, e.g. + * + * 12345678-1234-1234-1234-123456789abc + */ + return uuid_is_valid(s); +} + +static int efivarfs_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct inode *inode = NULL; + struct efivar_entry *var; + int namelen, i = 0, err = 0; + bool is_removable = false; + efi_guid_t vendor; + + if (!efivarfs_valid_name(dentry->d_name.name, dentry->d_name.len)) + return -EINVAL; + + /* length of the variable name itself: remove GUID and separator */ + namelen = dentry->d_name.len - EFI_VARIABLE_GUID_LEN - 1; + + err = guid_parse(dentry->d_name.name + namelen + 1, &vendor); + if (err) + return err; + if (guid_equal(&vendor, &LINUX_EFI_RANDOM_SEED_TABLE_GUID)) + return -EPERM; + + if (efivar_variable_is_removable(vendor, + dentry->d_name.name, namelen)) + is_removable = true; + + inode = efivarfs_get_inode(dir->i_sb, dir, mode, 0, is_removable); + if (!inode) + return -ENOMEM; + var = efivar_entry(inode); + + var->var.VendorGuid = vendor; + + for (i = 0; i < namelen; i++) + var->var.VariableName[i] = dentry->d_name.name[i]; + + var->var.VariableName[i] = '\0'; + + inode->i_private = var; + + d_make_persistent(dentry, inode); + + return 0; +} + +static int efivarfs_unlink(struct inode *dir, struct dentry *dentry) +{ + struct efivar_entry *var = d_inode(dentry)->i_private; + + if (efivar_entry_delete(var)) + return -EINVAL; + + return simple_unlink(dir, dentry); +}; + +const struct inode_operations efivarfs_dir_inode_operations = { + .lookup = simple_lookup, + .unlink = efivarfs_unlink, + .create = efivarfs_create, +}; + +static int +efivarfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + unsigned int i_flags; + unsigned int flags = 0; + + i_flags = d_inode(dentry)->i_flags; + if (i_flags & S_IMMUTABLE) + flags |= FS_IMMUTABLE_FL; + + fileattr_fill_flags(fa, flags); + + return 0; +} + +static int +efivarfs_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa) +{ + unsigned int i_flags = 0; + + if (fileattr_has_fsx(fa)) + return -EOPNOTSUPP; + + if (fa->flags & ~FS_IMMUTABLE_FL) + return -EOPNOTSUPP; + + if (fa->flags & FS_IMMUTABLE_FL) + i_flags |= S_IMMUTABLE; + + inode_set_flags(d_inode(dentry), i_flags, S_IMMUTABLE); + + return 0; +} + +/* copy of simple_setattr except that it doesn't do i_size updates */ +static int efivarfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *iattr) +{ + struct inode *inode = d_inode(dentry); + int error; + + error = setattr_prepare(idmap, dentry, iattr); + if (error) + return error; + + setattr_copy(idmap, inode, iattr); + mark_inode_dirty(inode); + return 0; +} + +static const struct inode_operations efivarfs_file_inode_operations = { + .fileattr_get = efivarfs_fileattr_get, + .fileattr_set = efivarfs_fileattr_set, + .setattr = efivarfs_setattr, +}; diff --git a/fs/efivarfs/internal.h b/fs/efivarfs/internal.h new file mode 100644 index 000000000..f913b6824 --- /dev/null +++ b/fs/efivarfs/internal.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012 Red Hat, Inc. + * Copyright (C) 2012 Jeremy Kerr + */ +#ifndef EFIVAR_FS_INTERNAL_H +#define EFIVAR_FS_INTERNAL_H + +#include + +struct efivarfs_mount_opts { + kuid_t uid; + kgid_t gid; +}; + +struct efivarfs_fs_info { + struct efivarfs_mount_opts mount_opts; + struct super_block *sb; + struct notifier_block nb; +}; + +struct efi_variable { + efi_char16_t VariableName[EFI_VAR_NAME_LEN/sizeof(efi_char16_t)]; + efi_guid_t VendorGuid; +}; + +struct efivar_entry { + struct efi_variable var; + struct inode vfs_inode; + unsigned long open_count; + bool removed; +}; + +static inline struct efivar_entry *efivar_entry(struct inode *inode) +{ + return container_of(inode, struct efivar_entry, vfs_inode); +} + +int efivar_init(int (*func)(efi_char16_t *, efi_guid_t, unsigned long, void *), + void *data, bool duplicate_check); + +int efivar_entry_delete(struct efivar_entry *entry); + +int efivar_entry_size(struct efivar_entry *entry, unsigned long *size); +int __efivar_entry_get(struct efivar_entry *entry, u32 *attributes, + unsigned long *size, void *data); +int efivar_entry_get(struct efivar_entry *entry, u32 *attributes, + unsigned long *size, void *data); +int efivar_entry_set_get_size(struct efivar_entry *entry, u32 attributes, + unsigned long *size, void *data, bool *set); + + +bool efivar_validate(efi_guid_t vendor, efi_char16_t *var_name, u8 *data, + unsigned long data_size); +bool efivar_variable_is_removable(efi_guid_t vendor, const char *name, + size_t len); +char *efivar_get_utf8name(const efi_char16_t *name16, efi_guid_t *vendor); +bool efivarfs_variable_is_present(efi_char16_t *variable_name, + efi_guid_t *vendor, void *data); + +extern const struct file_operations efivarfs_file_operations; +extern const struct inode_operations efivarfs_dir_inode_operations; +extern struct inode *efivarfs_get_inode(struct super_block *sb, + const struct inode *dir, int mode, dev_t dev, + bool is_removable); + +#endif /* EFIVAR_FS_INTERNAL_H */ diff --git a/fs/efivarfs/super.c b/fs/efivarfs/super.c new file mode 100644 index 000000000..8d33f11db --- /dev/null +++ b/fs/efivarfs/super.c @@ -0,0 +1,571 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Red Hat, Inc. + * Copyright (C) 2012 Jeremy Kerr + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" +#include "../internal.h" + +static int efivarfs_ops_notifier(struct notifier_block *nb, unsigned long event, + void *data) +{ + struct efivarfs_fs_info *sfi = container_of(nb, struct efivarfs_fs_info, nb); + + switch (event) { + case EFIVAR_OPS_RDONLY: + sfi->sb->s_flags |= SB_RDONLY; + break; + case EFIVAR_OPS_RDWR: + sfi->sb->s_flags &= ~SB_RDONLY; + break; + default: + return NOTIFY_DONE; + } + + return NOTIFY_OK; +} + +static struct inode *efivarfs_alloc_inode(struct super_block *sb) +{ + struct efivar_entry *entry = kzalloc_obj(*entry); + + if (!entry) + return NULL; + + inode_init_once(&entry->vfs_inode); + entry->removed = false; + + return &entry->vfs_inode; +} + +static void efivarfs_free_inode(struct inode *inode) +{ + struct efivar_entry *entry = efivar_entry(inode); + + kfree(entry); +} + +static int efivarfs_show_options(struct seq_file *m, struct dentry *root) +{ + struct super_block *sb = root->d_sb; + struct efivarfs_fs_info *sbi = sb->s_fs_info; + struct efivarfs_mount_opts *opts = &sbi->mount_opts; + + if (!uid_eq(opts->uid, GLOBAL_ROOT_UID)) + seq_printf(m, ",uid=%u", + from_kuid_munged(&init_user_ns, opts->uid)); + if (!gid_eq(opts->gid, GLOBAL_ROOT_GID)) + seq_printf(m, ",gid=%u", + from_kgid_munged(&init_user_ns, opts->gid)); + return 0; +} + +static int efivarfs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + const u32 attr = EFI_VARIABLE_NON_VOLATILE | + EFI_VARIABLE_BOOTSERVICE_ACCESS | + EFI_VARIABLE_RUNTIME_ACCESS; + u64 storage_space, remaining_space, max_variable_size; + u64 id = huge_encode_dev(dentry->d_sb->s_dev); + efi_status_t status; + + /* Some UEFI firmware does not implement QueryVariableInfo() */ + storage_space = remaining_space = 0; + if (efi_rt_services_supported(EFI_RT_SUPPORTED_QUERY_VARIABLE_INFO)) { + static DEFINE_RATELIMIT_STATE(_rs, 2 * HZ, 5); + static u64 storage, remaining; + static DEFINE_SPINLOCK(lock); + + if (!__ratelimit(&_rs)) { + ratelimit_set_flags(&_rs, RATELIMIT_MSG_ON_RELEASE); + + spin_lock(&lock); + storage_space = storage; + remaining_space = remaining; + spin_unlock(&lock); + } else { + status = efivar_query_variable_info(attr, &storage_space, + &remaining_space, + &max_variable_size); + if (status != EFI_SUCCESS && status != EFI_UNSUPPORTED) + pr_warn("query_variable_info() failed: 0x%lx\n", + status); + + spin_lock(&lock); + storage = storage_space; + remaining = remaining_space; + spin_unlock(&lock); + } + } + + /* + * This is not a normal filesystem, so no point in pretending it has a block + * size; we declare f_bsize to 1, so that we can then report the exact value + * sent by EFI QueryVariableInfo in f_blocks and f_bfree + */ + buf->f_bsize = 1; + buf->f_namelen = NAME_MAX; + buf->f_blocks = storage_space; + buf->f_bfree = remaining_space; + buf->f_type = dentry->d_sb->s_magic; + buf->f_fsid = u64_to_fsid(id); + + /* + * In f_bavail we declare the free space that the kernel will allow writing + * when the storage_paranoia x86 quirk is active. To use more, users + * should boot the kernel with efi_no_storage_paranoia. + */ + if (remaining_space > efivar_reserved_space()) + buf->f_bavail = remaining_space - efivar_reserved_space(); + else + buf->f_bavail = 0; + + return 0; +} + +static int efivarfs_freeze_fs(struct super_block *sb); +static int efivarfs_unfreeze_fs(struct super_block *sb); + +static const struct super_operations efivarfs_ops = { + .statfs = efivarfs_statfs, + .drop_inode = inode_just_drop, + .alloc_inode = efivarfs_alloc_inode, + .free_inode = efivarfs_free_inode, + .show_options = efivarfs_show_options, + .freeze_fs = efivarfs_freeze_fs, + .unfreeze_fs = efivarfs_unfreeze_fs, +}; + +/* + * Compare two efivarfs file names. + * + * An efivarfs filename is composed of two parts, + * + * 1. A case-sensitive variable name + * 2. A case-insensitive GUID + * + * So we need to perform a case-sensitive match on part 1 and a + * case-insensitive match on part 2. + */ +static int efivarfs_d_compare(const struct dentry *dentry, + unsigned int len, const char *str, + const struct qstr *name) +{ + int guid = len - EFI_VARIABLE_GUID_LEN; + + /* Parallel lookups may produce a temporary invalid filename */ + if (guid <= 0) + return 1; + + if (name->len != len) + return 1; + + /* Case-sensitive compare for the variable name */ + if (memcmp(str, name->name, guid)) + return 1; + + /* Case-insensitive compare for the GUID */ + return strncasecmp(name->name + guid, str + guid, EFI_VARIABLE_GUID_LEN); +} + +static int efivarfs_d_hash(const struct dentry *dentry, struct qstr *qstr) +{ + unsigned long hash = init_name_hash(dentry); + const unsigned char *s = qstr->name; + unsigned int len = qstr->len; + + while (len-- > EFI_VARIABLE_GUID_LEN) + hash = partial_name_hash(*s++, hash); + + /* GUID is case-insensitive. */ + while (len--) + hash = partial_name_hash(tolower(*s++), hash); + + qstr->hash = end_name_hash(hash); + return 0; +} + +static const struct dentry_operations efivarfs_d_ops = { + .d_compare = efivarfs_d_compare, + .d_hash = efivarfs_d_hash, +}; + +static struct dentry *efivarfs_alloc_dentry(struct dentry *parent, char *name) +{ + struct qstr q = QSTR(name); + struct dentry *d; + int err; + + err = efivarfs_d_hash(parent, &q); + if (err) + return ERR_PTR(err); + + d = d_alloc(parent, &q); + if (d) + return d; + + return ERR_PTR(-ENOMEM); +} + +bool efivarfs_variable_is_present(efi_char16_t *variable_name, + efi_guid_t *vendor, void *data) +{ + char *name = efivar_get_utf8name(variable_name, vendor); + struct super_block *sb = data; + struct dentry *dentry; + + if (!name) + /* + * If the allocation failed there'll already be an + * error in the log (and likely a huge and growing + * number of them since they system will be under + * extreme memory pressure), so simply assume + * collision for safety but don't add to the log + * flood. + */ + return true; + + dentry = try_lookup_noperm(&QSTR(name), sb->s_root); + kfree(name); + if (!IS_ERR_OR_NULL(dentry)) + dput(dentry); + + return dentry != NULL; +} + +static int efivarfs_create_dentry(struct super_block *sb, efi_char16_t *name16, + unsigned long name_size, efi_guid_t vendor, + char *name) +{ + struct efivar_entry *entry; + struct inode *inode; + struct dentry *dentry, *root = sb->s_root; + unsigned long size = 0; + int len; + int err = -ENOMEM; + bool is_removable = false; + + /* length of the variable name itself: remove GUID and separator */ + len = strlen(name) - EFI_VARIABLE_GUID_LEN - 1; + + if (efivar_variable_is_removable(vendor, name, len)) + is_removable = true; + + inode = efivarfs_get_inode(sb, d_inode(root), S_IFREG | 0644, 0, + is_removable); + if (!inode) + goto fail_name; + + entry = efivar_entry(inode); + + memcpy(entry->var.VariableName, name16, name_size); + memcpy(&(entry->var.VendorGuid), &vendor, sizeof(efi_guid_t)); + + dentry = efivarfs_alloc_dentry(root, name); + if (IS_ERR(dentry)) { + err = PTR_ERR(dentry); + goto fail_inode; + } + + __efivar_entry_get(entry, NULL, &size, NULL); + + /* copied by the above to local storage in the dentry. */ + kfree(name); + + inode_lock(inode); + inode->i_private = entry; + i_size_write(inode, size + sizeof(__u32)); /* attributes + data */ + inode_unlock(inode); + d_make_persistent(dentry, inode); + dput(dentry); + + return 0; + +fail_inode: + iput(inode); +fail_name: + kfree(name); + + return err; +} + +static int efivarfs_callback(efi_char16_t *name16, efi_guid_t vendor, + unsigned long name_size, void *data) +{ + struct super_block *sb = (struct super_block *)data; + char *name; + + if (guid_equal(&vendor, &LINUX_EFI_RANDOM_SEED_TABLE_GUID)) + return 0; + + name = efivar_get_utf8name(name16, &vendor); + if (!name) + return -ENOMEM; + + return efivarfs_create_dentry(sb, name16, name_size, vendor, name); +} + +enum { + Opt_uid, Opt_gid, +}; + +static const struct fs_parameter_spec efivarfs_parameters[] = { + fsparam_uid("uid", Opt_uid), + fsparam_gid("gid", Opt_gid), + {}, +}; + +static int efivarfs_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct efivarfs_fs_info *sbi = fc->s_fs_info; + struct efivarfs_mount_opts *opts = &sbi->mount_opts; + struct fs_parse_result result; + int opt; + + opt = fs_parse(fc, efivarfs_parameters, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_uid: + opts->uid = result.uid; + break; + case Opt_gid: + opts->gid = result.gid; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int efivarfs_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct efivarfs_fs_info *sfi = sb->s_fs_info; + struct inode *inode = NULL; + struct dentry *root; + int err; + + sb->s_maxbytes = MAX_LFS_FILESIZE; + sb->s_blocksize = PAGE_SIZE; + sb->s_blocksize_bits = PAGE_SHIFT; + sb->s_magic = EFIVARFS_MAGIC; + sb->s_op = &efivarfs_ops; + set_default_d_op(sb, &efivarfs_d_ops); + sb->s_d_flags |= DCACHE_DONTCACHE; + sb->s_time_gran = 1; + + if (!efivar_supports_writes()) + sb->s_flags |= SB_RDONLY; + + inode = efivarfs_get_inode(sb, NULL, S_IFDIR | 0755, 0, true); + if (!inode) + return -ENOMEM; + inode->i_op = &efivarfs_dir_inode_operations; + + root = d_make_root(inode); + sb->s_root = root; + if (!root) + return -ENOMEM; + + sfi->sb = sb; + sfi->nb.notifier_call = efivarfs_ops_notifier; + err = blocking_notifier_chain_register(&efivar_ops_nh, &sfi->nb); + if (err) + return err; + + return efivar_init(efivarfs_callback, sb, true); +} + +static int efivarfs_get_tree(struct fs_context *fc) +{ + return get_tree_single(fc, efivarfs_fill_super); +} + +static int efivarfs_reconfigure(struct fs_context *fc) +{ + if (!efivar_supports_writes() && !(fc->sb_flags & SB_RDONLY)) { + pr_err("Firmware does not support SetVariableRT. Can not remount with rw\n"); + return -EINVAL; + } + + return 0; +} + +static void efivarfs_free(struct fs_context *fc) +{ + kfree(fc->s_fs_info); +} + +static const struct fs_context_operations efivarfs_context_ops = { + .get_tree = efivarfs_get_tree, + .parse_param = efivarfs_parse_param, + .reconfigure = efivarfs_reconfigure, + .free = efivarfs_free, +}; + +static int efivarfs_check_missing(efi_char16_t *name16, efi_guid_t vendor, + unsigned long name_size, void *data) +{ + char *name; + struct super_block *sb = data; + struct dentry *dentry; + int err; + + if (guid_equal(&vendor, &LINUX_EFI_RANDOM_SEED_TABLE_GUID)) + return 0; + + name = efivar_get_utf8name(name16, &vendor); + if (!name) + return -ENOMEM; + + dentry = try_lookup_noperm(&QSTR(name), sb->s_root); + if (IS_ERR(dentry)) { + err = PTR_ERR(dentry); + goto out; + } + + if (!dentry) { + /* found missing entry */ + pr_info("efivarfs: creating variable %s\n", name); + return efivarfs_create_dentry(sb, name16, name_size, vendor, name); + } + + dput(dentry); + err = 0; + + out: + kfree(name); + + return err; +} + +static struct file_system_type efivarfs_type; + +static int efivarfs_freeze_fs(struct super_block *sb) +{ + /* Nothing for us to do. */ + return 0; +} + +static int efivarfs_unfreeze_fs(struct super_block *sb) +{ + struct dentry *child = NULL; + + /* + * Unconditionally resync the variable state on a thaw request. + * Given the size of efivarfs it really doesn't matter to simply + * iterate through all of the entries and resync. Freeze/thaw + * requests are rare enough for that to not matter and the + * number of entries is pretty low too. So we really don't care. + */ + pr_info("efivarfs: resyncing variable state\n"); + for (;;) { + int err; + unsigned long size = 0; + struct inode *inode; + struct efivar_entry *entry; + + child = find_next_child(sb->s_root, child); + if (!child) + break; + + inode = d_inode(child); + entry = efivar_entry(inode); + + err = efivar_entry_size(entry, &size); + if (err) + size = 0; + else + size += sizeof(__u32); + + inode_lock(inode); + i_size_write(inode, size); + inode_unlock(inode); + + /* The variable doesn't exist anymore, delete it. */ + if (!size) { + pr_info("efivarfs: removing variable %pd\n", child); + simple_recursive_removal(child, NULL); + } + } + + efivar_init(efivarfs_check_missing, sb, false); + pr_info("efivarfs: finished resyncing variable state\n"); + return 0; +} + +static int efivarfs_init_fs_context(struct fs_context *fc) +{ + struct efivarfs_fs_info *sfi; + + if (!efivar_is_available()) + return -EOPNOTSUPP; + + sfi = kzalloc_obj(*sfi); + if (!sfi) + return -ENOMEM; + + sfi->mount_opts.uid = GLOBAL_ROOT_UID; + sfi->mount_opts.gid = GLOBAL_ROOT_GID; + + fc->s_fs_info = sfi; + fc->ops = &efivarfs_context_ops; + + return 0; +} + +static void efivarfs_kill_sb(struct super_block *sb) +{ + struct efivarfs_fs_info *sfi = sb->s_fs_info; + + blocking_notifier_chain_unregister(&efivar_ops_nh, &sfi->nb); + kill_anon_super(sb); + + kfree(sfi); +} + +static struct file_system_type efivarfs_type = { + .owner = THIS_MODULE, + .name = "efivarfs", + .init_fs_context = efivarfs_init_fs_context, + .kill_sb = efivarfs_kill_sb, + .parameters = efivarfs_parameters, + .fs_flags = FS_POWER_FREEZE, +}; + +static __init int efivarfs_init(void) +{ + return register_filesystem(&efivarfs_type); +} + +static __exit void efivarfs_exit(void) +{ + unregister_filesystem(&efivarfs_type); +} + +MODULE_AUTHOR("Matthew Garrett, Jeremy Kerr"); +MODULE_DESCRIPTION("EFI Variable Filesystem"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS_FS("efivarfs"); + +module_init(efivarfs_init); +module_exit(efivarfs_exit); diff --git a/fs/efivarfs/vars.c b/fs/efivarfs/vars.c new file mode 100644 index 000000000..6833c3d24 --- /dev/null +++ b/fs/efivarfs/vars.c @@ -0,0 +1,633 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Originally from efivars.c + * + * Copyright (C) 2001,2003,2004 Dell + * Copyright (C) 2004 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +MODULE_IMPORT_NS("EFIVAR"); + +static bool +validate_device_path(efi_char16_t *var_name, int match, u8 *buffer, + unsigned long len) +{ + struct efi_generic_dev_path *node; + int offset = 0; + + node = (struct efi_generic_dev_path *)buffer; + + if (len < sizeof(*node)) + return false; + + while (offset <= len - sizeof(*node) && + node->length >= sizeof(*node) && + node->length <= len - offset) { + offset += node->length; + + if ((node->type == EFI_DEV_END_PATH || + node->type == EFI_DEV_END_PATH2) && + node->sub_type == EFI_DEV_END_ENTIRE) + return true; + + node = (struct efi_generic_dev_path *)(buffer + offset); + } + + /* + * If we're here then either node->length pointed past the end + * of the buffer or we reached the end of the buffer without + * finding a device path end node. + */ + return false; +} + +static bool +validate_boot_order(efi_char16_t *var_name, int match, u8 *buffer, + unsigned long len) +{ + /* An array of 16-bit integers */ + if ((len % 2) != 0) + return false; + + return true; +} + +static bool +validate_load_option(efi_char16_t *var_name, int match, u8 *buffer, + unsigned long len) +{ + u16 filepathlength; + int i, desclength = 0, namelen; + + namelen = ucs2_strnlen(var_name, EFI_VAR_NAME_LEN); + + /* Either "Boot" or "Driver" followed by four digits of hex */ + for (i = match; i < match+4; i++) { + if (var_name[i] > 127 || + hex_to_bin(var_name[i] & 0xff) < 0) + return true; + } + + /* Reject it if there's 4 digits of hex and then further content */ + if (namelen > match + 4) + return false; + + /* A valid entry must be at least 8 bytes */ + if (len < 8) + return false; + + filepathlength = buffer[4] | buffer[5] << 8; + + /* + * There's no stored length for the description, so it has to be + * found by hand + */ + desclength = ucs2_strsize((efi_char16_t *)(buffer + 6), len - 6) + 2; + + /* Each boot entry must have a descriptor */ + if (!desclength) + return false; + + /* + * If the sum of the length of the description, the claimed filepath + * length and the original header are greater than the length of the + * variable, it's malformed + */ + if ((desclength + filepathlength + 6) > len) + return false; + + /* + * And, finally, check the filepath + */ + return validate_device_path(var_name, match, buffer + desclength + 6, + filepathlength); +} + +static bool +validate_uint16(efi_char16_t *var_name, int match, u8 *buffer, + unsigned long len) +{ + /* A single 16-bit integer */ + if (len != 2) + return false; + + return true; +} + +static bool +validate_ascii_string(efi_char16_t *var_name, int match, u8 *buffer, + unsigned long len) +{ + int i; + + for (i = 0; i < len; i++) { + if (buffer[i] > 127) + return false; + + if (buffer[i] == 0) + return true; + } + + return false; +} + +struct variable_validate { + efi_guid_t vendor; + char *name; + bool (*validate)(efi_char16_t *var_name, int match, u8 *data, + unsigned long len); +}; + +/* + * This is the list of variables we need to validate, as well as the + * whitelist for what we think is safe not to default to immutable. + * + * If it has a validate() method that's not NULL, it'll go into the + * validation routine. If not, it is assumed valid, but still used for + * whitelisting. + * + * Note that it's sorted by {vendor,name}, but globbed names must come after + * any other name with the same prefix. + */ +static const struct variable_validate variable_validate[] = { + { EFI_GLOBAL_VARIABLE_GUID, "BootNext", validate_uint16 }, + { EFI_GLOBAL_VARIABLE_GUID, "BootOrder", validate_boot_order }, + { EFI_GLOBAL_VARIABLE_GUID, "Boot*", validate_load_option }, + { EFI_GLOBAL_VARIABLE_GUID, "DriverOrder", validate_boot_order }, + { EFI_GLOBAL_VARIABLE_GUID, "Driver*", validate_load_option }, + { EFI_GLOBAL_VARIABLE_GUID, "ConIn", validate_device_path }, + { EFI_GLOBAL_VARIABLE_GUID, "ConInDev", validate_device_path }, + { EFI_GLOBAL_VARIABLE_GUID, "ConOut", validate_device_path }, + { EFI_GLOBAL_VARIABLE_GUID, "ConOutDev", validate_device_path }, + { EFI_GLOBAL_VARIABLE_GUID, "ErrOut", validate_device_path }, + { EFI_GLOBAL_VARIABLE_GUID, "ErrOutDev", validate_device_path }, + { EFI_GLOBAL_VARIABLE_GUID, "Lang", validate_ascii_string }, + { EFI_GLOBAL_VARIABLE_GUID, "OsIndications", NULL }, + { EFI_GLOBAL_VARIABLE_GUID, "PlatformLang", validate_ascii_string }, + { EFI_GLOBAL_VARIABLE_GUID, "Timeout", validate_uint16 }, + { LINUX_EFI_CRASH_GUID, "*", NULL }, + { NULL_GUID, "", NULL }, +}; + +/* + * Check if @var_name matches the pattern given in @match_name. + * + * @var_name: an array of @len non-NUL characters. + * @match_name: a NUL-terminated pattern string, optionally ending in "*". A + * final "*" character matches any trailing characters @var_name, + * including the case when there are none left in @var_name. + * @match: on output, the number of non-wildcard characters in @match_name + * that @var_name matches, regardless of the return value. + * @return: whether @var_name fully matches @match_name. + */ +static bool +variable_matches(const char *var_name, size_t len, const char *match_name, + int *match) +{ + for (*match = 0; ; (*match)++) { + char c = match_name[*match]; + + switch (c) { + case '*': + /* Wildcard in @match_name means we've matched. */ + return true; + + case '\0': + /* @match_name has ended. Has @var_name too? */ + return (*match == len); + + default: + /* + * We've reached a non-wildcard char in @match_name. + * Continue only if there's an identical character in + * @var_name. + */ + if (*match < len && c == var_name[*match]) + continue; + return false; + } + } +} + +char * +efivar_get_utf8name(const efi_char16_t *name16, efi_guid_t *vendor) +{ + int len = ucs2_utf8size(name16); + char *name; + + /* name, plus '-', plus GUID, plus NUL*/ + name = kmalloc(len + 1 + EFI_VARIABLE_GUID_LEN + 1, GFP_KERNEL); + if (!name) + return NULL; + + ucs2_as_utf8(name, name16, len); + + name[len] = '-'; + + efi_guid_to_str(vendor, name + len + 1); + + name[len + EFI_VARIABLE_GUID_LEN+1] = '\0'; + + /* replace invalid slashes like kobject_set_name_vargs does for /sys/firmware/efi/vars. */ + strreplace(name, '/', '!'); + + return name; +} + +bool +efivar_validate(efi_guid_t vendor, efi_char16_t *var_name, u8 *data, + unsigned long data_size) +{ + int i; + unsigned long utf8_size; + u8 *utf8_name; + + utf8_size = ucs2_utf8size(var_name); + utf8_name = kmalloc(utf8_size + 1, GFP_KERNEL); + if (!utf8_name) + return false; + + ucs2_as_utf8(utf8_name, var_name, utf8_size); + utf8_name[utf8_size] = '\0'; + + for (i = 0; variable_validate[i].name[0] != '\0'; i++) { + const char *name = variable_validate[i].name; + int match = 0; + + if (efi_guidcmp(vendor, variable_validate[i].vendor)) + continue; + + if (variable_matches(utf8_name, utf8_size+1, name, &match)) { + if (variable_validate[i].validate == NULL) + break; + kfree(utf8_name); + return variable_validate[i].validate(var_name, match, + data, data_size); + } + } + kfree(utf8_name); + return true; +} + +bool +efivar_variable_is_removable(efi_guid_t vendor, const char *var_name, + size_t len) +{ + int i; + bool found = false; + int match = 0; + + /* + * Check if our variable is in the validated variables list + */ + for (i = 0; variable_validate[i].name[0] != '\0'; i++) { + if (efi_guidcmp(variable_validate[i].vendor, vendor)) + continue; + + if (variable_matches(var_name, len, + variable_validate[i].name, &match)) { + found = true; + break; + } + } + + /* + * If it's in our list, it is removable. + */ + return found; +} + +/* + * Returns the size of variable_name, in bytes, including the + * terminating NULL character, or variable_name_size if no NULL + * character is found among the first variable_name_size bytes. + */ +static unsigned long var_name_strnsize(efi_char16_t *variable_name, + unsigned long variable_name_size) +{ + unsigned long len; + efi_char16_t c; + + /* + * The variable name is, by definition, a NULL-terminated + * string, so make absolutely sure that variable_name_size is + * the value we expect it to be. If not, return the real size. + */ + for (len = 2; len <= variable_name_size; len += sizeof(c)) { + c = variable_name[(len / sizeof(c)) - 1]; + if (!c) + break; + } + + return min(len, variable_name_size); +} + +/* + * Print a warning when duplicate EFI variables are encountered and + * disable the sysfs workqueue since the firmware is buggy. + */ +static void dup_variable_bug(efi_char16_t *str16, efi_guid_t *vendor_guid, + unsigned long len16) +{ + size_t i, len8 = len16 / sizeof(efi_char16_t); + char *str8; + + str8 = kzalloc(len8, GFP_KERNEL); + if (!str8) + return; + + for (i = 0; i < len8; i++) + str8[i] = str16[i]; + + printk(KERN_WARNING "efivars: duplicate variable: %s-%pUl\n", + str8, vendor_guid); + kfree(str8); +} + +/** + * efivar_init - build the initial list of EFI variables + * @func: callback function to invoke for every variable + * @data: function-specific data to pass to @func + * @duplicate_check: fail if a duplicate variable is found + * + * Get every EFI variable from the firmware and invoke @func. @func + * should populate the initial dentry and inode tree. + * + * Returns 0 on success, or a kernel error code on failure. + */ +int efivar_init(int (*func)(efi_char16_t *, efi_guid_t, unsigned long, void *), + void *data, bool duplicate_check) +{ + unsigned long variable_name_size = 512; + efi_char16_t *variable_name; + efi_status_t status; + efi_guid_t vendor_guid; + int err = 0; + + variable_name = kzalloc(variable_name_size, GFP_KERNEL); + if (!variable_name) { + printk(KERN_ERR "efivars: Memory allocation failed.\n"); + return -ENOMEM; + } + + err = efivar_lock(); + if (err) + goto free; + + /* + * A small set of old UEFI implementations reject sizes + * above a certain threshold, the lowest seen in the wild + * is 512. + */ + + do { + variable_name_size = 512; + BUILD_BUG_ON(EFI_VAR_NAME_LEN < 512); + + status = efivar_get_next_variable(&variable_name_size, + variable_name, + &vendor_guid); + switch (status) { + case EFI_SUCCESS: + variable_name_size = var_name_strnsize(variable_name, + variable_name_size); + + /* + * Some firmware implementations return the + * same variable name on multiple calls to + * get_next_variable(). Terminate the loop + * immediately as there is no guarantee that + * we'll ever see a different variable name, + * and may end up looping here forever. + */ + if (duplicate_check && + efivarfs_variable_is_present(variable_name, + &vendor_guid, data)) { + dup_variable_bug(variable_name, &vendor_guid, + variable_name_size); + status = EFI_NOT_FOUND; + } else { + err = func(variable_name, vendor_guid, + variable_name_size, data); + if (err) + status = EFI_NOT_FOUND; + } + break; + case EFI_UNSUPPORTED: + err = -EOPNOTSUPP; + status = EFI_NOT_FOUND; + break; + case EFI_NOT_FOUND: + break; + case EFI_BUFFER_TOO_SMALL: + pr_warn("efivars: Variable name size exceeds maximum (%lu > 512)\n", + variable_name_size); + status = EFI_NOT_FOUND; + break; + default: + pr_warn("efivars: get_next_variable: status=%lx\n", status); + status = EFI_NOT_FOUND; + break; + } + + } while (status != EFI_NOT_FOUND); + + efivar_unlock(); +free: + kfree(variable_name); + + return err; +} + +/** + * efivar_entry_delete - delete variable + * @entry: entry containing variable to delete + * + * Delete the variable from the firmware. It is the caller's + * responsibility to free @entry (by deleting the dentry/inode) once + * we return. + * + * Returns 0 on success, -EINTR if we can't grab the semaphore, + * converted EFI status code if set_variable() fails. + */ +int efivar_entry_delete(struct efivar_entry *entry) +{ + efi_status_t status; + int err; + + err = efivar_lock(); + if (err) + return err; + + status = efivar_set_variable_locked(entry->var.VariableName, + &entry->var.VendorGuid, + 0, 0, NULL, false); + efivar_unlock(); + if (!(status == EFI_SUCCESS || status == EFI_NOT_FOUND)) + return efi_status_to_err(status); + + return 0; +} + +/** + * efivar_entry_size - obtain the size of a variable + * @entry: entry for this variable + * @size: location to store the variable's size + */ +int efivar_entry_size(struct efivar_entry *entry, unsigned long *size) +{ + efi_status_t status; + int err; + + *size = 0; + + err = efivar_lock(); + if (err) + return err; + + status = efivar_get_variable(entry->var.VariableName, + &entry->var.VendorGuid, NULL, size, NULL); + efivar_unlock(); + + if (status != EFI_BUFFER_TOO_SMALL) + return efi_status_to_err(status); + + return 0; +} + +/** + * __efivar_entry_get - call get_variable() + * @entry: read data for this variable + * @attributes: variable attributes + * @size: size of @data buffer + * @data: buffer to store variable data + * + * The caller MUST call efivar_entry_iter_begin() and + * efivar_entry_iter_end() before and after the invocation of this + * function, respectively. + */ +int __efivar_entry_get(struct efivar_entry *entry, u32 *attributes, + unsigned long *size, void *data) +{ + efi_status_t status; + + status = efivar_get_variable(entry->var.VariableName, + &entry->var.VendorGuid, + attributes, size, data); + + return efi_status_to_err(status); +} + +/** + * efivar_entry_get - call get_variable() + * @entry: read data for this variable + * @attributes: variable attributes + * @size: size of @data buffer + * @data: buffer to store variable data + */ +int efivar_entry_get(struct efivar_entry *entry, u32 *attributes, + unsigned long *size, void *data) +{ + int err; + + err = efivar_lock(); + if (err) + return err; + err = __efivar_entry_get(entry, attributes, size, data); + efivar_unlock(); + + return err; +} + +/** + * efivar_entry_set_get_size - call set_variable() and get new size (atomic) + * @entry: entry containing variable to set and get + * @attributes: attributes of variable to be written + * @size: size of data buffer + * @data: buffer containing data to write + * @set: did the set_variable() call succeed? + * + * This is a pretty special (complex) function. See efivarfs_file_write(). + * + * Atomically call set_variable() for @entry and if the call is + * successful, return the new size of the variable from get_variable() + * in @size. The success of set_variable() is indicated by @set. + * + * Returns 0 on success, -EINVAL if the variable data is invalid, + * -ENOSPC if the firmware does not have enough available space, or a + * converted EFI status code if either of set_variable() or + * get_variable() fail. + * + * If the EFI variable does not exist when calling set_variable() + * (EFI_NOT_FOUND). + */ +int efivar_entry_set_get_size(struct efivar_entry *entry, u32 attributes, + unsigned long *size, void *data, bool *set) +{ + efi_char16_t *name = entry->var.VariableName; + efi_guid_t *vendor = &entry->var.VendorGuid; + efi_status_t status; + int err; + + *set = false; + + if (efivar_validate(*vendor, name, data, *size) == false) + return -EINVAL; + + /* + * The lock here protects the get_variable call and the + * conditional set_variable call + */ + err = efivar_lock(); + if (err) + return err; + + status = efivar_set_variable_locked(name, vendor, attributes, *size, + data, false); + if (status != EFI_SUCCESS) { + err = efi_status_to_err(status); + goto out; + } + + *set = true; + + /* + * Writing to the variable may have caused a change in size (which + * could either be an append or an overwrite), or the variable to be + * deleted. Perform a GetVariable() so we can tell what actually + * happened. + */ + *size = 0; + status = efivar_get_variable(entry->var.VariableName, + &entry->var.VendorGuid, + NULL, size, NULL); + + efivar_unlock(); + + if (status && status != EFI_BUFFER_TOO_SMALL) + return efi_status_to_err(status); + + return 0; + +out: + efivar_unlock(); + return err; + +} diff --git a/fs/erofs/Kconfig b/fs/erofs/Kconfig new file mode 100644 index 000000000..2dfc31358 --- /dev/null +++ b/fs/erofs/Kconfig @@ -0,0 +1,212 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config EROFS_FS + tristate "EROFS filesystem support" + depends on BLOCK + select CRC32 + select CRYPTO if EROFS_FS_ZIP_ACCEL + select CRYPTO_DEFLATE if EROFS_FS_ZIP_ACCEL + select FS_IOMAP + select LZ4_DECOMPRESS if EROFS_FS_ZIP + select XXHASH if EROFS_FS_XATTR + select XZ_DEC if EROFS_FS_ZIP_LZMA + select XZ_DEC_MICROLZMA if EROFS_FS_ZIP_LZMA + select ZLIB_INFLATE if EROFS_FS_ZIP_DEFLATE + select ZSTD_DECOMPRESS if EROFS_FS_ZIP_ZSTD + help + EROFS (Enhanced Read-Only File System) is a modern, lightweight, + secure read-only filesystem for various use cases, such as immutable + system images, container images, application sandboxes, and datasets. + + EROFS uses a flexible, hierarchical on-disk design so that features + can be enabled on demand: the core on-disk format is block-aligned in + order to perform optimally on all kinds of devices, including block + and memory-backed devices; the format is easy to parse and has zero + metadata redundancy, unlike generic filesystems, making it ideal for + filesystem auditing and remote access; inline data, random-access + friendly directory data, inline/shared extended attributes and + chunk-based deduplication ensure space efficiency while maintaining + high performance. + + Optionally, it supports multiple devices to reference external data, + enabling data sharing for container images. + + It also has advanced encoded on-disk layouts, particularly for data + compression and fine-grained deduplication. It utilizes fixed-size + output compression to improve storage density while keeping relatively + high compression ratios. Furthermore, it implements in-place + decompression to reuse file pages to keep compressed data temporarily + with proper strategies, which ensures guaranteed end-to-end runtime + performance under extreme memory pressure without extra cost. + + For more details, see the web pages at + and the documentation at . + + To compile EROFS filesystem support as a module, choose M here. The + module will be called erofs. + + If unsure, say N. + +config EROFS_FS_DEBUG + bool "EROFS debugging feature" + depends on EROFS_FS + help + Print debugging messages and enable more BUG_ONs which check + filesystem consistency and find potential issues aggressively, + which can be used for Android eng build, for example. + + For daily use, say N. + +config EROFS_FS_XATTR + bool "EROFS extended attributes" + depends on EROFS_FS + default y + help + Extended attributes are name:value pairs associated with inodes by + the kernel or by users (see the attr(5) manual page, or visit + for details). + + If unsure, say N. + +config EROFS_FS_POSIX_ACL + bool "EROFS Access Control Lists" + depends on EROFS_FS_XATTR + select FS_POSIX_ACL + default y + help + Posix Access Control Lists (ACLs) support permissions for users and + groups beyond the owner/group/world scheme. + + To learn more about Access Control Lists, visit the POSIX ACLs for + Linux website . + + If you don't know what Access Control Lists are, say N. + +config EROFS_FS_SECURITY + bool "EROFS Security Labels" + depends on EROFS_FS_XATTR + default y + help + Security labels provide an access control facility to support Linux + Security Models (LSMs) accepted by AppArmor, SELinux, Smack and TOMOYO + Linux. This option enables an extended attribute handler for file + security labels in the erofs filesystem, so that it requires enabling + the extended attribute support in advance. + + If you are not using a security module, say N. + +config EROFS_FS_BACKED_BY_FILE + bool "File-backed EROFS filesystem support" + depends on EROFS_FS + default y + help + This allows EROFS to use filesystem image files directly, without + the intercession of loopback block devices or likewise. It is + particularly useful for container images with numerous blobs and + other sandboxes, where loop devices behave intricately. It can also + be used to simplify error-prone lifetime management of unnecessary + virtual block devices. + + If you don't want to enable this feature, say N. + +config EROFS_FS_ZIP + bool "EROFS Data Compression Support" + depends on EROFS_FS + default y + help + Enable EROFS compression layouts so that filesystems containing + compressed files can be parsed by the kernel. + + If you don't want to enable compression feature, say N. + +config EROFS_FS_ZIP_LZMA + bool "EROFS LZMA compressed data support" + depends on EROFS_FS_ZIP + default y + help + Saying Y here includes support for reading EROFS file systems + containing LZMA compressed data, specifically called microLZMA. It + gives better compression ratios than the default LZ4 format, at the + expense of more CPU overhead. + + Say N if you want to disable LZMA compression support. + +config EROFS_FS_ZIP_LZMA_DEFAULT_MAX_STREAMS + int "EROFS LZMA default maximum decompression streams" + depends on EROFS_FS_ZIP_LZMA + range 1 NR_CPUS if SMP + range 1 1 if !SMP + default 16 + help + By default EROFS allocates one LZMA decompression stream per CPU. + Each stream can hold a dictionary of up to 8 MiB taken from the + mounted image, so on systems with many CPUs this can reserve a lot + of memory. This caps the default; the lzma_streams module parameter + still overrides it. + + If unsure, keep the default of 16. + +config EROFS_FS_ZIP_DEFLATE + bool "EROFS DEFLATE compressed data support" + depends on EROFS_FS_ZIP + help + Saying Y here includes support for reading EROFS file systems + containing DEFLATE compressed data. It gives better compression + ratios than the default LZ4 format, while it costs more CPU + overhead. + + If unsure, say N. + +config EROFS_FS_ZIP_ZSTD + bool "EROFS Zstandard compressed data support" + depends on EROFS_FS_ZIP + help + Saying Y here includes support for reading EROFS file systems + containing Zstandard compressed data. It gives better compression + ratios than the default LZ4 format, while it costs more CPU + overhead and memory footprint. + + If unsure, say N. + +config EROFS_FS_ZIP_ACCEL + bool "EROFS hardware decompression support" + depends on EROFS_FS_ZIP + help + Saying Y here includes hardware accelerator support for reading + EROFS file systems containing compressed data. It gives better + decompression speed than the software-implemented decompression, and + it costs lower CPU overhead. + + Hardware accelerator support is an experimental feature for now and + file systems are still readable without selecting this option. + + If unsure, say N. + +config EROFS_FS_PCPU_KTHREAD + bool "EROFS per-cpu decompression kthread workers" + depends on EROFS_FS_ZIP + help + Saying Y here enables per-CPU kthread workers pool to carry out + async decompression for low latencies on some architectures. + + If unsure, say N. + +config EROFS_FS_PCPU_KTHREAD_HIPRI + bool "EROFS high priority per-CPU kthread workers" + depends on EROFS_FS_ZIP && EROFS_FS_PCPU_KTHREAD + default y + help + This permits EROFS to configure per-CPU kthread workers to run + at higher priority. + + If unsure, say N. + +config EROFS_FS_PAGE_CACHE_SHARE + bool "EROFS page cache share support (experimental)" + depends on EROFS_FS && EROFS_FS_XATTR + select FS_STACK + help + This enables page cache sharing among inodes with identical + content fingerprints on the same machine. + + If unsure, say N. diff --git a/fs/erofs/Makefile b/fs/erofs/Makefile new file mode 100644 index 000000000..304234967 --- /dev/null +++ b/fs/erofs/Makefile @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-$(CONFIG_EROFS_FS) += erofs.o +erofs-objs := super.o inode.o data.o namei.o dir.o sysfs.o +erofs-$(CONFIG_EROFS_FS_XATTR) += xattr.o +erofs-$(CONFIG_EROFS_FS_ZIP) += decompressor.o zmap.o zdata.o zutil.o +erofs-$(CONFIG_EROFS_FS_ZIP_LZMA) += decompressor_lzma.o +erofs-$(CONFIG_EROFS_FS_ZIP_DEFLATE) += decompressor_deflate.o +erofs-$(CONFIG_EROFS_FS_ZIP_ZSTD) += decompressor_zstd.o +erofs-$(CONFIG_EROFS_FS_ZIP_ACCEL) += decompressor_crypto.o +erofs-$(CONFIG_EROFS_FS_BACKED_BY_FILE) += fileio.o +erofs-$(CONFIG_EROFS_FS_PAGE_CACHE_SHARE) += ishare.o diff --git a/fs/erofs/compress.h b/fs/erofs/compress.h new file mode 100644 index 000000000..84c8e5258 --- /dev/null +++ b/fs/erofs/compress.h @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2019 HUAWEI, Inc. + * https://www.huawei.com/ + */ +#ifndef __EROFS_FS_COMPRESS_H +#define __EROFS_FS_COMPRESS_H + +#include "internal.h" + +struct z_erofs_decompress_req { + struct super_block *sb; + struct page **in, **out; + unsigned int inpages, outpages; + unsigned short pageofs_in, pageofs_out; + unsigned int inputsize, outputsize; + + unsigned int alg; /* the algorithm for decompression */ + bool inplace_io, partial_decoding, fillgaps; + gfp_t gfp; /* allocation flags for extra temporary buffers */ +}; + +struct z_erofs_decompressor { + int (*config)(struct super_block *sb, struct erofs_super_block *dsb, + void *data, int size); + const char *(*decompress)(struct z_erofs_decompress_req *rq, + struct page **pagepool); + int (*init)(void); + void (*exit)(void); + char *name; +}; + +#define Z_EROFS_SHORTLIVED_PAGE (-1UL << 2) +#define Z_EROFS_PREALLOCATED_FOLIO ((void *)(-2UL << 2)) + +/* + * Currently, short-lived pages are pages directly from buddy system + * with specific page->private (Z_EROFS_SHORTLIVED_PAGE). + * In the future world of Memdescs, it should be type 0 (Misc) memory + * which type can be checked with a new helper. + */ +static inline bool z_erofs_is_shortlived_page(struct page *page) +{ + return page->private == Z_EROFS_SHORTLIVED_PAGE; +} + +static inline bool z_erofs_put_shortlivedpage(struct page **pagepool, + struct page *page) +{ + if (!z_erofs_is_shortlived_page(page)) + return false; + erofs_pagepool_add(pagepool, page); + return true; +} + +extern const struct z_erofs_decompressor z_erofs_lzma_decomp; +extern const struct z_erofs_decompressor z_erofs_deflate_decomp; +extern const struct z_erofs_decompressor z_erofs_zstd_decomp; +extern const struct z_erofs_decompressor *z_erofs_decomp[]; + +struct z_erofs_stream_dctx { + struct z_erofs_decompress_req *rq; + int no, ni; /* the current {en,de}coded page # */ + + unsigned int avail_out; /* remaining bytes in the decoded buffer */ + unsigned int inbuf_pos, inbuf_sz; + /* current status of the encoded buffer */ + u8 *kin, *kout; /* buffer mapped pointers */ + void *bounce; /* bounce buffer for inplace I/Os */ + bool bounced; /* is the bounce buffer used now? */ +}; + +const char *z_erofs_stream_switch_bufs(struct z_erofs_stream_dctx *dctx, + void **dst, void **src, struct page **pgpl); +const char *z_erofs_fixup_insize(struct z_erofs_decompress_req *rq, + const char *padbuf, unsigned int padbufsize); +int __init z_erofs_init_decompressor(void); +void z_erofs_exit_decompressor(void); +int z_erofs_crypto_decompress(struct z_erofs_decompress_req *rq, + struct page **pgpl); +int z_erofs_crypto_enable_engine(const char *name, int len); +#ifdef CONFIG_EROFS_FS_ZIP_ACCEL +void z_erofs_crypto_disable_all_engines(void); +int z_erofs_crypto_show_engines(char *buf, int size, char sep); +#else +static inline void z_erofs_crypto_disable_all_engines(void) {} +static inline int z_erofs_crypto_show_engines(char *buf, int size, char sep) { return 0; } +#endif +#endif diff --git a/fs/erofs/data.c b/fs/erofs/data.c new file mode 100644 index 000000000..20962f8b6 --- /dev/null +++ b/fs/erofs/data.c @@ -0,0 +1,534 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021, Alibaba Cloud + */ +#include "internal.h" +#include +#include +#include + +void erofs_unmap_metabuf(struct erofs_buf *buf) +{ + if (!buf->base) + return; + kunmap_local(buf->base); + buf->base = NULL; +} + +void erofs_put_metabuf(struct erofs_buf *buf) +{ + if (!buf->page) + return; + erofs_unmap_metabuf(buf); + folio_put(page_folio(buf->page)); + buf->page = NULL; +} + +void *erofs_bread(struct erofs_buf *buf, erofs_off_t offset, bool need_kmap) +{ + pgoff_t index = (buf->off + offset) >> PAGE_SHIFT; + struct folio *folio = NULL; + + if (buf->page) { + folio = page_folio(buf->page); + if (folio_file_page(folio, index) != buf->page) + erofs_unmap_metabuf(buf); + } + if (!folio || !folio_contains(folio, index)) { + erofs_put_metabuf(buf); + folio = read_cache_folio(buf->mapping, index, + buf->mc ? erofs_read_meta_folio : NULL, NULL); + if (IS_ERR(folio)) + return folio; + } + buf->page = folio_file_page(folio, index); + if (!need_kmap) + return NULL; + if (!buf->base) + buf->base = kmap_local_page(buf->page); + return buf->base + (offset & ~PAGE_MASK); +} + +int erofs_init_metabuf(struct erofs_buf *buf, struct super_block *sb, + bool in_metabox) +{ + struct erofs_sb_info *sbi = EROFS_SB(sb); + + buf->mc = false; + if (in_metabox) { + if (unlikely(!sbi->metabox_inode)) + return -EFSCORRUPTED; + buf->mapping = sbi->metabox_inode->i_mapping; + return 0; + } + if (erofs_is_fileio_mode(sbi)) { + buf->mapping = sbi->managed_cache->i_mapping; + buf->mc = true; + } else { + buf->off = sbi->dif0.fsoff; + buf->mapping = sb->s_bdev->bd_mapping; + } + return 0; +} + +void *erofs_read_metabuf(struct erofs_buf *buf, struct super_block *sb, + erofs_off_t offset, bool in_metabox) +{ + int err; + + err = erofs_init_metabuf(buf, sb, in_metabox); + if (err) + return ERR_PTR(err); + return erofs_bread(buf, offset, true); +} + +static int erofs_map_chunks(struct inode *inode, struct erofs_map_blocks *map) +{ + struct erofs_buf buf = __EROFS_BUF_INITIALIZER; + struct super_block *sb = inode->i_sb; + struct erofs_inode *vi = EROFS_I(inode); + struct erofs_inode_chunk_index *idx; + unsigned int unit = vi->chunkformat & EROFS_CHUNK_FORMAT_INDEXES ? + sizeof(*idx) : EROFS_BLOCK_MAP_ENTRY_SIZE; + erofs_blk_t addrmask = (vi->chunkformat & EROFS_CHUNK_FORMAT_48BIT) ? + BIT_ULL(48) - 1 : BIT_ULL(32) - 1; + u64 nr = map->m_la >> vi->chunkbits, chunksize = 1ULL << vi->chunkbits; + erofs_off_t pos = ALIGN(erofs_iloc(inode) + vi->inode_isize + + vi->xattr_isize, unit) + unit * nr; + /* m_llen will be clamped to EOF in the end */ + erofs_off_t endpos = round_up(pos + 1, sb->s_blocksize); + u64 last, addr; + + idx = erofs_read_metabuf(&buf, sb, pos, erofs_inode_in_metabox(inode)); + if (IS_ERR(idx)) + return PTR_ERR(idx); + + map->m_la = nr << vi->chunkbits; + map->m_llen = 0; + nr = 0; + do { + if (unit == EROFS_BLOCK_MAP_ENTRY_SIZE) { + addr = le32_to_cpu(((__le32 *)idx)[nr]); + if (addr == (u32)EROFS_NULL_ADDR) + addr = EROFS_NULL_ADDR; + } else { + addr = (((u64)le16_to_cpu(idx[nr].startblk_hi) << 32) | + le32_to_cpu(idx[nr].startblk_lo)) & addrmask; + if (addr ^ (EROFS_NULL_ADDR & addrmask)) + addr |= (u64)(le16_to_cpu(idx[nr].device_id) & + EROFS_SB(sb)->device_id_mask) << 48; + else + addr = EROFS_NULL_ADDR; + } + if (!nr) { + last = addr; + continue; + } + /* expand and account the prior chunk here */ + map->m_llen += chunksize; + if (last != EROFS_NULL_ADDR) + last += erofs_blknr(sb, chunksize); + } while (addr == last && pos + (++nr) * unit < endpos); + + if (last != EROFS_NULL_ADDR) { + map->m_pa = erofs_pos(sb, last & addrmask) - map->m_llen; + map->m_deviceid = last >> 48; + map->m_flags = EROFS_MAP_MAPPED; + } + if (addr == last) + map->m_llen += chunksize; + map->m_llen = min_t(erofs_off_t, map->m_llen, + round_up(inode->i_size - map->m_la, sb->s_blocksize)); + erofs_put_metabuf(&buf); + return 0; +} + +int erofs_map_blocks(struct inode *inode, struct erofs_map_blocks *map) +{ + struct super_block *sb = inode->i_sb; + struct erofs_inode *vi = EROFS_I(inode); + bool tailinline = (vi->datalayout == EROFS_INODE_FLAT_INLINE); + erofs_off_t pos; + int err = 0; + + trace_erofs_map_blocks_enter(inode, map, 0); + map->m_deviceid = 0; + map->m_flags = 0; + if (map->m_la >= inode->i_size) + goto out; + if (vi->datalayout == EROFS_INODE_CHUNK_BASED) { + err = erofs_map_chunks(inode, map); + } else if (tailinline || vi->startblk != EROFS_NULL_ADDR) { + pos = erofs_pos(sb, erofs_iblks(inode) - tailinline); + map->m_flags = EROFS_MAP_MAPPED; + if (map->m_la < pos) { + map->m_pa = erofs_pos(sb, vi->startblk) + map->m_la; + map->m_llen = pos - map->m_la; + } else { + map->m_pa = erofs_iloc(inode) + vi->inode_isize + + vi->xattr_isize + erofs_blkoff(sb, map->m_la); + map->m_llen = inode->i_size - map->m_la; + map->m_flags |= EROFS_MAP_META; + if (erofs_blkoff(sb, map->m_pa) + map->m_llen > + sb->s_blocksize) { + erofs_err(sb, "inline data across blocks @ nid %llu", vi->nid); + return -EFSCORRUPTED; + } + } + } +out: + map->m_plen = err ? 0 : map->m_llen; + trace_erofs_map_blocks_exit(inode, map, 0, err); + return err; +} + +static void erofs_fill_from_devinfo(struct erofs_map_dev *map, + struct super_block *sb, struct erofs_device_info *dif) +{ + map->m_sb = sb; + map->m_dif = dif; + map->m_bdev = NULL; + if (dif->file && S_ISBLK(file_inode(dif->file)->i_mode)) + map->m_bdev = file_bdev(dif->file); +} + +int erofs_map_dev(struct super_block *sb, struct erofs_map_dev *map) +{ + struct erofs_dev_context *devs = EROFS_SB(sb)->devs; + struct erofs_device_info *dif; + erofs_off_t startoff; + int id; + + erofs_fill_from_devinfo(map, sb, &EROFS_SB(sb)->dif0); + map->m_bdev = sb->s_bdev; /* use s_bdev for the primary device */ + if (map->m_deviceid) { + down_read(&devs->rwsem); + dif = idr_find(&devs->tree, map->m_deviceid - 1); + if (!dif) { + up_read(&devs->rwsem); + return -ENODEV; + } + if (devs->flatdev) { + map->m_pa += erofs_pos(sb, dif->uniaddr); + up_read(&devs->rwsem); + return 0; + } + erofs_fill_from_devinfo(map, sb, dif); + up_read(&devs->rwsem); + } else if (devs->extra_devices && !devs->flatdev) { + down_read(&devs->rwsem); + idr_for_each_entry(&devs->tree, dif, id) { + if (!dif->uniaddr) + continue; + + startoff = erofs_pos(sb, dif->uniaddr); + if (map->m_pa >= startoff && + map->m_pa < startoff + erofs_pos(sb, dif->blocks)) { + map->m_pa -= startoff; + erofs_fill_from_devinfo(map, sb, dif); + break; + } + } + up_read(&devs->rwsem); + } + return 0; +} + +/* + * bit 30: I/O error occurred on this folio + * bit 29: CPU has dirty data in D-cache (needs aliasing handling); + * bit 0 - 29: remaining parts to complete this folio + */ +#define EROFS_ONLINEFOLIO_EIO 30 +#define EROFS_ONLINEFOLIO_DIRTY 29 + +void erofs_onlinefolio_init(struct folio *folio) +{ + union { + atomic_t o; + void *v; + } u = { .o = ATOMIC_INIT(1) }; + + folio->private = u.v; /* valid only if file-backed folio is locked */ +} + +void erofs_onlinefolio_split(struct folio *folio) +{ + atomic_inc((atomic_t *)&folio->private); +} + +void erofs_onlinefolio_end(struct folio *folio, int err, bool dirty) +{ + int orig, v; + + do { + orig = atomic_read((atomic_t *)&folio->private); + DBG_BUGON(orig <= 0); + v = dirty << EROFS_ONLINEFOLIO_DIRTY; + v |= (orig - 1) | (!!err << EROFS_ONLINEFOLIO_EIO); + } while (atomic_cmpxchg((atomic_t *)&folio->private, orig, v) != orig); + + if (v & (BIT(EROFS_ONLINEFOLIO_DIRTY) - 1)) + return; + folio->private = 0; + if (v & BIT(EROFS_ONLINEFOLIO_DIRTY)) + flush_dcache_folio(folio); + folio_end_read(folio, !(v & BIT(EROFS_ONLINEFOLIO_EIO))); +} + +struct erofs_iomap_iter_ctx { + struct page *page; + void *base; + struct inode *realinode; +}; + +static int erofs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, + unsigned int flags, struct iomap *iomap, struct iomap *srcmap) +{ + struct iomap_iter *iter = container_of(iomap, struct iomap_iter, iomap); + struct erofs_iomap_iter_ctx *ctx = iter->private; + struct inode *realinode = ctx ? ctx->realinode : inode; + struct super_block *sb = realinode->i_sb; + struct erofs_map_blocks map; + struct erofs_map_dev mdev; + int ret; + + map.m_la = offset; + map.m_llen = length; + ret = erofs_map_blocks(realinode, &map); + if (ret < 0) + return ret; + + iomap->offset = map.m_la; + iomap->length = map.m_llen; + iomap->flags = 0; + iomap->addr = IOMAP_NULL_ADDR; + if (!(map.m_flags & EROFS_MAP_MAPPED)) { + iomap->type = IOMAP_HOLE; + return 0; + } + + if (!(map.m_flags & EROFS_MAP_META) || !erofs_inode_in_metabox(realinode)) { + mdev = (struct erofs_map_dev) { + .m_deviceid = map.m_deviceid, + .m_pa = map.m_pa, + }; + ret = erofs_map_dev(sb, &mdev); + if (ret) + return ret; + + if (flags & IOMAP_DAX) + iomap->dax_dev = mdev.m_dif->dax_dev; + else + iomap->bdev = mdev.m_bdev; + iomap->addr = mdev.m_dif->fsoff + mdev.m_pa; + if (flags & IOMAP_DAX) + iomap->addr += mdev.m_dif->dax_part_off; + } + + if (map.m_flags & EROFS_MAP_META) { + iomap->type = IOMAP_INLINE; + /* read context should read the inlined data */ + if (ctx) { + struct erofs_buf buf = __EROFS_BUF_INITIALIZER; + void *ptr; + + ptr = erofs_read_metabuf(&buf, sb, map.m_pa, + erofs_inode_in_metabox(realinode)); + if (IS_ERR(ptr)) + return PTR_ERR(ptr); + iomap->inline_data = ptr; + ctx->page = buf.page; + ctx->base = buf.base; + } + } else { + iomap->type = IOMAP_MAPPED; + } + return 0; +} + +static int erofs_iomap_end(struct inode *inode, loff_t pos, loff_t length, + ssize_t written, unsigned int flags, struct iomap *iomap) +{ + struct iomap_iter *iter = container_of(iomap, struct iomap_iter, iomap); + struct erofs_iomap_iter_ctx *ctx = iter->private; + + if (ctx && ctx->base) { + struct erofs_buf buf = { + .page = ctx->page, + .base = ctx->base, + }; + + DBG_BUGON(iomap->type != IOMAP_INLINE); + erofs_put_metabuf(&buf); + ctx->base = NULL; + } + return written; +} + +static DEFINE_IOMAP_ITER_NEXT_END(erofs_iomap_next, erofs_iomap_begin, + erofs_iomap_end); + +static const struct iomap_ops erofs_iomap_ops = { + .iomap_next = erofs_iomap_next, +}; + +int erofs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + u64 start, u64 len) +{ + if (erofs_inode_is_data_compressed(EROFS_I(inode)->datalayout)) { + if (!IS_ENABLED(CONFIG_EROFS_FS_ZIP)) + return -EOPNOTSUPP; + return iomap_fiemap(inode, fieinfo, start, len, + &z_erofs_iomap_report_ops); + } + return iomap_fiemap(inode, fieinfo, start, len, &erofs_iomap_ops); +} + +/* + * since we dont have write or truncate flows, so no inode + * locking needs to be held at the moment. + */ +static int erofs_read_folio(struct file *file, struct folio *folio) +{ + struct iomap_read_folio_ctx read_ctx = { + .ops = &iomap_bio_read_ops, + .cur_folio = folio, + }; + bool need_iput; + struct erofs_iomap_iter_ctx iter_ctx = { + .realinode = erofs_real_inode(folio_inode(folio), &need_iput), + }; + + trace_erofs_read_folio(iter_ctx.realinode, folio, true); + iomap_read_folio(&erofs_iomap_ops, &read_ctx, &iter_ctx); + if (need_iput) + iput(iter_ctx.realinode); + return 0; +} + +static void erofs_readahead(struct readahead_control *rac) +{ + struct iomap_read_folio_ctx read_ctx = { + .ops = &iomap_bio_read_ops, + .rac = rac, + }; + bool need_iput; + struct erofs_iomap_iter_ctx iter_ctx = { + .realinode = erofs_real_inode(rac->mapping->host, &need_iput), + }; + + trace_erofs_readahead(iter_ctx.realinode, readahead_index(rac), + readahead_count(rac), true); + iomap_readahead(&erofs_iomap_ops, &read_ctx, &iter_ctx); + if (need_iput) + iput(iter_ctx.realinode); +} + +static sector_t erofs_bmap(struct address_space *mapping, sector_t block) +{ + return iomap_bmap(mapping, block, &erofs_iomap_ops); +} + +static ssize_t erofs_file_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct inode *inode = file_inode(iocb->ki_filp); + + /* no need taking (shared) inode lock since it's a ro filesystem */ + if (!iov_iter_count(to)) + return 0; + + if (IS_ENABLED(CONFIG_FS_DAX) && IS_DAX(inode)) + return dax_iomap_rw(iocb, to, &erofs_iomap_ops); + + if ((iocb->ki_flags & IOCB_DIRECT) && inode->i_sb->s_bdev) { + struct erofs_iomap_iter_ctx iter_ctx = { + .realinode = inode, + }; + + return iomap_dio_rw(iocb, to, &erofs_iomap_ops, + NULL, 0, &iter_ctx, 0); + } + return filemap_read(iocb, to, 0); +} + +/* for uncompressed (aligned) files and raw access for other files */ +const struct address_space_operations erofs_aops = { + .read_folio = erofs_read_folio, + .readahead = erofs_readahead, + .bmap = erofs_bmap, + .direct_IO = noop_direct_IO, + .release_folio = iomap_release_folio, + .invalidate_folio = iomap_invalidate_folio, +}; + +#ifdef CONFIG_FS_DAX +static vm_fault_t erofs_dax_huge_fault(struct vm_fault *vmf, + unsigned int order) +{ + return dax_iomap_fault(vmf, order, NULL, NULL, &erofs_iomap_ops); +} + +static vm_fault_t erofs_dax_fault(struct vm_fault *vmf) +{ + return erofs_dax_huge_fault(vmf, 0); +} + +static const struct vm_operations_struct erofs_dax_vm_ops = { + .fault = erofs_dax_fault, + .huge_fault = erofs_dax_huge_fault, +}; + +static int erofs_file_mmap_prepare(struct vm_area_desc *desc) +{ + if (!IS_DAX(file_inode(desc->file))) + return generic_file_readonly_mmap_prepare(desc); + + if (vma_desc_test_all(desc, VMA_SHARED_BIT, VMA_MAYWRITE_BIT)) + return -EINVAL; + + desc->vm_ops = &erofs_dax_vm_ops; + vma_desc_set_flags(desc, VMA_HUGEPAGE_BIT); + return 0; +} +#else +#define erofs_file_mmap_prepare generic_file_readonly_mmap_prepare +#endif + +static loff_t erofs_file_llseek(struct file *file, loff_t offset, int whence) +{ + struct inode *inode = file->f_mapping->host; + const struct iomap_ops *ops = &erofs_iomap_ops; + + if (erofs_inode_is_data_compressed(EROFS_I(inode)->datalayout)) { + if (!IS_ENABLED(CONFIG_EROFS_FS_ZIP)) + return generic_file_llseek(file, offset, whence); + ops = &z_erofs_iomap_report_ops; + } + + if (whence == SEEK_HOLE) + offset = iomap_seek_hole(inode, offset, ops); + else if (whence == SEEK_DATA) + offset = iomap_seek_data(inode, offset, ops); + else + return generic_file_llseek(file, offset, whence); + + if (offset < 0) + return offset; + return vfs_setpos(file, offset, inode->i_sb->s_maxbytes); +} + +const struct file_operations erofs_file_fops = { + .llseek = erofs_file_llseek, + .read_iter = erofs_file_read_iter, + .unlocked_ioctl = erofs_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = erofs_compat_ioctl, +#endif + .mmap_prepare = erofs_file_mmap_prepare, + .get_unmapped_area = thp_get_unmapped_area, + .splice_read = filemap_splice_read, + .setlease = generic_setlease, +}; diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c new file mode 100644 index 000000000..27caf4beb --- /dev/null +++ b/fs/erofs/decompressor.c @@ -0,0 +1,517 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2019 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2024 Alibaba Cloud + */ +#include "compress.h" +#include + +#define LZ4_MAX_DISTANCE_PAGES (DIV_ROUND_UP(LZ4_DISTANCE_MAX, PAGE_SIZE) + 1) + +static int z_erofs_load_lz4_config(struct super_block *sb, + struct erofs_super_block *dsb, void *data, int size) +{ + struct erofs_sb_info *sbi = EROFS_SB(sb); + struct z_erofs_lz4_cfgs *lz4 = data; + u16 distance; + + if (lz4) { + if (size < sizeof(struct z_erofs_lz4_cfgs)) { + erofs_err(sb, "invalid lz4 cfgs, size=%u", size); + return -EINVAL; + } + distance = le16_to_cpu(lz4->max_distance); + + sbi->lz4.max_pclusterblks = le16_to_cpu(lz4->max_pclusterblks); + if (!sbi->lz4.max_pclusterblks) { + sbi->lz4.max_pclusterblks = 1; /* reserved case */ + } else if (sbi->lz4.max_pclusterblks > + erofs_blknr(sb, Z_EROFS_PCLUSTER_MAX_SIZE)) { + erofs_err(sb, "too large lz4 pclusterblks %u", + sbi->lz4.max_pclusterblks); + return -EINVAL; + } + } else { + distance = le16_to_cpu(dsb->u1.lz4_max_distance); + if (!distance && !erofs_sb_has_lz4_0padding(sbi)) + return 0; + sbi->lz4.max_pclusterblks = 1; + sbi->available_compr_algs = 1 << Z_EROFS_COMPRESSION_LZ4; + } + + sbi->lz4.max_distance_pages = distance ? + DIV_ROUND_UP(distance, PAGE_SIZE) + 1 : + LZ4_MAX_DISTANCE_PAGES; + return z_erofs_gbuf_growsize(sbi->lz4.max_pclusterblks); +} + +/* + * Fill all gaps with bounce pages if it's a sparse page list. Also check if + * all physical pages are consecutive, which can be seen for moderate CR. + */ +static int z_erofs_lz4_prepare_dstpages(struct z_erofs_decompress_req *rq, + struct page **pagepool) +{ + struct page *availables[LZ4_MAX_DISTANCE_PAGES] = { NULL }; + unsigned long bounced[DIV_ROUND_UP(LZ4_MAX_DISTANCE_PAGES, + BITS_PER_LONG)] = { 0 }; + unsigned int lz4_max_distance_pages = + EROFS_SB(rq->sb)->lz4.max_distance_pages; + void *kaddr = NULL; + unsigned int i, j, top; + + top = 0; + for (i = j = 0; i < rq->outpages; ++i, ++j) { + struct page *const page = rq->out[i]; + struct page *victim; + + if (j >= lz4_max_distance_pages) + j = 0; + + /* 'valid' bounced can only be tested after a complete round */ + if (!rq->fillgaps && test_bit(j, bounced)) { + DBG_BUGON(i < lz4_max_distance_pages); + DBG_BUGON(top >= lz4_max_distance_pages); + availables[top++] = rq->out[i - lz4_max_distance_pages]; + } + + if (page) { + __clear_bit(j, bounced); + if (!PageHighMem(page)) { + if (!i) { + kaddr = page_address(page); + continue; + } + if (kaddr && + kaddr + PAGE_SIZE == page_address(page)) { + kaddr += PAGE_SIZE; + continue; + } + } + kaddr = NULL; + continue; + } + kaddr = NULL; + __set_bit(j, bounced); + + if (top) { + victim = availables[--top]; + } else { + victim = __erofs_allocpage(pagepool, rq->gfp, true); + if (!victim) + return -ENOMEM; + set_page_private(victim, Z_EROFS_SHORTLIVED_PAGE); + } + rq->out[i] = victim; + } + return kaddr ? 1 : 0; +} + +static void *z_erofs_lz4_handle_overlap(const struct z_erofs_decompress_req *rq, + void *inpage, void *out, unsigned int *inputmargin, + int *maptype, bool may_inplace) +{ + unsigned int oend, omargin, cnt, i; + struct page **in; + void *src; + + /* + * If in-place I/O isn't used, for example, the bounce compressed cache + * can hold data for incomplete read requests. Just map the compressed + * buffer as well and decompress directly. + */ + if (!rq->inplace_io) { + if (rq->inpages <= 1) { + *maptype = 0; + return inpage; + } + kunmap_local(inpage); + src = erofs_vm_map_ram(rq->in, rq->inpages); + if (!src) + return ERR_PTR(-ENOMEM); + *maptype = 1; + return src; + } + /* + * Then, deal with in-place I/Os. The reasons why in-place I/O is useful + * are: (1) It minimizes memory footprint during the I/O submission, + * which is useful for slow storage (including network devices and + * low-end HDDs/eMMCs) but with a lot inflight I/Os; (2) If in-place + * decompression can also be applied, it will reuse the unique buffer so + * that no extra CPU D-cache is polluted with temporary compressed data + * for extreme performance. + */ + oend = rq->pageofs_out + rq->outputsize; + omargin = PAGE_ALIGN(oend) - oend; + if (!rq->partial_decoding && may_inplace && + omargin >= LZ4_DECOMPRESS_INPLACE_MARGIN(rq->inputsize)) { + for (i = 0; i < rq->inpages; ++i) + if (rq->out[rq->outpages - rq->inpages + i] != + rq->in[i]) + break; + if (i >= rq->inpages) { + kunmap_local(inpage); + *maptype = 3; + return out + ((rq->outpages - rq->inpages) << PAGE_SHIFT); + } + } + /* + * If in-place decompression can't be applied, copy compressed data that + * may potentially overlap during decompression to a per-CPU buffer. + */ + src = z_erofs_get_gbuf(rq->inpages); + if (!src) { + DBG_BUGON(1); + kunmap_local(inpage); + return ERR_PTR(-EFAULT); + } + + for (i = 0, in = rq->in; i < rq->inputsize; i += cnt, ++in) { + cnt = min_t(u32, rq->inputsize - i, PAGE_SIZE - *inputmargin); + if (!inpage) + inpage = kmap_local_page(*in); + memcpy(src + i, inpage + *inputmargin, cnt); + kunmap_local(inpage); + inpage = NULL; + *inputmargin = 0; + } + *maptype = 2; + return src; +} + +/* + * Get the exact on-disk size of the compressed data: + * - For LZ4, it should apply if the zero_padding feature is on (5.3+); + * - For others, zero_padding is enabled all the time. + */ +const char *z_erofs_fixup_insize(struct z_erofs_decompress_req *rq, + const char *padbuf, unsigned int padbufsize) +{ + const char *padend; + + padend = memchr_inv(padbuf, 0, padbufsize); + if (!padend) + return "compressed data start not found"; + rq->inputsize -= padend - padbuf; + rq->pageofs_in += padend - padbuf; + return NULL; +} + +static const char *__z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq, + u8 *dst) +{ + bool may_inplace = false; + unsigned int inputmargin; + u8 *out, *headpage, *src; + const char *reason; + int ret, maptype; + + headpage = kmap_local_page(*rq->in); + reason = z_erofs_fixup_insize(rq, headpage + rq->pageofs_in, + min_t(unsigned int, rq->inputsize, + rq->sb->s_blocksize - rq->pageofs_in)); + if (reason) { + kunmap_local(headpage); + return reason; + } + may_inplace = !((rq->pageofs_in + rq->inputsize) & + (rq->sb->s_blocksize - 1)); + + inputmargin = rq->pageofs_in; + src = z_erofs_lz4_handle_overlap(rq, headpage, dst, &inputmargin, + &maptype, may_inplace); + if (IS_ERR(src)) + return ERR_CAST(src); + + out = dst + rq->pageofs_out; + if (rq->partial_decoding) + ret = LZ4_decompress_safe_partial(src + inputmargin, out, + rq->inputsize, rq->outputsize, rq->outputsize); + else + ret = LZ4_decompress_safe(src + inputmargin, out, + rq->inputsize, rq->outputsize); + if (ret == rq->outputsize) + reason = NULL; + else if (ret < 0) + reason = "corrupted compressed data"; + else + reason = "unexpected end of stream"; + + if (!maptype) { + kunmap_local(headpage); + } else if (maptype == 1) { + vm_unmap_ram(src, rq->inpages); + } else if (maptype == 2) { + z_erofs_put_gbuf(src); + } else if (maptype != 3) { + DBG_BUGON(1); + return ERR_PTR(-EFAULT); + } + return reason; +} + +static const char *z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq, + struct page **pagepool) +{ + unsigned int dst_maptype; + const char *reason; + void *dst; + int ret; + + /* one optimized fast path only for non bigpcluster cases yet */ + if (rq->inpages == 1 && rq->outpages == 1 && !rq->inplace_io) { + DBG_BUGON(!*rq->out); + dst = kmap_local_page(*rq->out); + dst_maptype = 0; + } else { + /* general decoding path which can be used for all cases */ + ret = z_erofs_lz4_prepare_dstpages(rq, pagepool); + if (ret < 0) + return ERR_PTR(ret); + if (ret > 0) { + dst = page_address(*rq->out); + dst_maptype = 1; + } else { + dst = erofs_vm_map_ram(rq->out, rq->outpages); + if (!dst) + return ERR_PTR(-ENOMEM); + dst_maptype = 2; + } + } + reason = __z_erofs_lz4_decompress(rq, dst); + if (!dst_maptype) + kunmap_local(dst); + else if (dst_maptype == 2) + vm_unmap_ram(dst, rq->outpages); + return reason; +} + +static const char *z_erofs_transform_plain(struct z_erofs_decompress_req *rq, + struct page **pagepool) +{ + const unsigned int nrpages_in = rq->inpages, nrpages_out = rq->outpages; + const unsigned int bs = rq->sb->s_blocksize; + unsigned int cur = 0, ni = 0, no, pi, po, insz, cnt; + u8 *kin; + + if (rq->outputsize > rq->inputsize) + return ERR_PTR(-EOPNOTSUPP); + if (rq->alg == Z_EROFS_COMPRESSION_INTERLACED) { + cur = bs - (rq->pageofs_out & (bs - 1)); + DBG_BUGON(rq->pageofs_in & (bs - 1)); + pi = (rq->pageofs_in + rq->inputsize - cur) & ~PAGE_MASK; + cur = min(cur, rq->outputsize); + if (cur && rq->out[0]) { + kin = kmap_local_page(rq->in[nrpages_in - 1]); + if (rq->out[0] == rq->in[nrpages_in - 1]) + memmove(kin + rq->pageofs_out, kin + pi, cur); + else + memcpy_to_page(rq->out[0], rq->pageofs_out, + kin + pi, cur); + kunmap_local(kin); + } + rq->outputsize -= cur; + } + + for (; rq->outputsize; rq->pageofs_in = 0, cur += insz, ni++) { + insz = min(PAGE_SIZE - rq->pageofs_in, rq->outputsize); + rq->outputsize -= insz; + if (!rq->in[ni]) + continue; + kin = kmap_local_page(rq->in[ni]); + pi = 0; + do { + no = (rq->pageofs_out + cur + pi) >> PAGE_SHIFT; + po = (rq->pageofs_out + cur + pi) & ~PAGE_MASK; + DBG_BUGON(no >= nrpages_out); + cnt = min(insz - pi, PAGE_SIZE - po); + if (rq->out[no] == rq->in[ni]) + memmove(kin + po, + kin + rq->pageofs_in + pi, cnt); + else if (rq->out[no]) + memcpy_to_page(rq->out[no], po, + kin + rq->pageofs_in + pi, cnt); + pi += cnt; + } while (pi < insz); + kunmap_local(kin); + } + DBG_BUGON(ni > nrpages_in); + return NULL; +} + +const char *z_erofs_stream_switch_bufs(struct z_erofs_stream_dctx *dctx, + void **dst, void **src, struct page **pgpl) +{ + struct z_erofs_decompress_req *rq = dctx->rq; + struct page **pgo, *tmppage; + unsigned int j; + + if (!dctx->avail_out) { + if (++dctx->no >= rq->outpages || !rq->outputsize) + return "insufficient space for decompressed data"; + + if (dctx->kout) + kunmap_local(dctx->kout); + dctx->avail_out = min(rq->outputsize, PAGE_SIZE - rq->pageofs_out); + rq->outputsize -= dctx->avail_out; + pgo = &rq->out[dctx->no]; + if (!*pgo && rq->fillgaps) { /* deduped */ + *pgo = erofs_allocpage(pgpl, rq->gfp); + if (!*pgo) { + dctx->kout = NULL; + return ERR_PTR(-ENOMEM); + } + set_page_private(*pgo, Z_EROFS_SHORTLIVED_PAGE); + } + if (*pgo) { + dctx->kout = kmap_local_page(*pgo); + *dst = dctx->kout + rq->pageofs_out; + } else { + *dst = dctx->kout = NULL; + } + rq->pageofs_out = 0; + } + + if (dctx->inbuf_pos == dctx->inbuf_sz && rq->inputsize) { + if (++dctx->ni >= rq->inpages) + return "invalid compressed data"; + if (dctx->kout) /* unlike kmap(), take care of the orders */ + kunmap_local(dctx->kout); + kunmap_local(dctx->kin); + + dctx->inbuf_sz = min_t(u32, rq->inputsize, PAGE_SIZE); + rq->inputsize -= dctx->inbuf_sz; + dctx->kin = kmap_local_page(rq->in[dctx->ni]); + *src = dctx->kin; + dctx->bounced = false; + if (dctx->kout) { + j = (u8 *)*dst - dctx->kout; + dctx->kout = kmap_local_page(rq->out[dctx->no]); + *dst = dctx->kout + j; + } + dctx->inbuf_pos = 0; + } + + /* + * Handle overlapping: Use the given bounce buffer if the input data is + * under processing; Or utilize short-lived pages from the on-stack page + * pool, where pages are shared among the same request. Note that only + * a few inplace I/O pages need to be doubled. + */ + if (!dctx->bounced && rq->out[dctx->no] == rq->in[dctx->ni]) { + memcpy(dctx->bounce, *src, dctx->inbuf_sz); + *src = dctx->bounce; + dctx->bounced = true; + } + + for (j = dctx->ni + 1; j < rq->inpages; ++j) { + if (rq->out[dctx->no] != rq->in[j]) + continue; + tmppage = erofs_allocpage(pgpl, rq->gfp); + if (!tmppage) + return ERR_PTR(-ENOMEM); + set_page_private(tmppage, Z_EROFS_SHORTLIVED_PAGE); + copy_highpage(tmppage, rq->in[j]); + rq->in[j] = tmppage; + } + return NULL; +} + +const struct z_erofs_decompressor *z_erofs_decomp[] = { + [Z_EROFS_COMPRESSION_SHIFTED] = &(const struct z_erofs_decompressor) { + .decompress = z_erofs_transform_plain, + .name = "shifted" + }, + [Z_EROFS_COMPRESSION_INTERLACED] = &(const struct z_erofs_decompressor) { + .decompress = z_erofs_transform_plain, + .name = "interlaced" + }, + [Z_EROFS_COMPRESSION_LZ4] = &(const struct z_erofs_decompressor) { + .config = z_erofs_load_lz4_config, + .decompress = z_erofs_lz4_decompress, + .init = z_erofs_gbuf_init, + .exit = z_erofs_gbuf_exit, + .name = "lz4" + }, +#ifdef CONFIG_EROFS_FS_ZIP_LZMA + [Z_EROFS_COMPRESSION_LZMA] = &z_erofs_lzma_decomp, +#endif +#ifdef CONFIG_EROFS_FS_ZIP_DEFLATE + [Z_EROFS_COMPRESSION_DEFLATE] = &z_erofs_deflate_decomp, +#endif +#ifdef CONFIG_EROFS_FS_ZIP_ZSTD + [Z_EROFS_COMPRESSION_ZSTD] = &z_erofs_zstd_decomp, +#endif +}; + +int z_erofs_parse_cfgs(struct super_block *sb, struct erofs_super_block *dsb) +{ + struct erofs_sb_info *sbi = EROFS_SB(sb); + struct erofs_buf buf = __EROFS_BUF_INITIALIZER; + unsigned long algs, alg; + erofs_off_t offset; + int size, ret = 0; + + if (!erofs_sb_has_compr_cfgs(sbi)) + return z_erofs_load_lz4_config(sb, dsb, NULL, 0); + + algs = le16_to_cpu(dsb->u1.available_compr_algs); + sbi->available_compr_algs = algs; + if (algs & ~Z_EROFS_ALL_COMPR_ALGS) { + erofs_err(sb, "unidentified algorithms %lx, please upgrade kernel", + algs & ~Z_EROFS_ALL_COMPR_ALGS); + return -EOPNOTSUPP; + } + + (void)erofs_init_metabuf(&buf, sb, false); + offset = EROFS_SUPER_OFFSET + sbi->sb_size; + for_each_set_bit(alg, &algs, Z_EROFS_COMPRESSION_MAX) { + const struct z_erofs_decompressor *dec = z_erofs_decomp[alg]; + void *data; + + data = erofs_read_metadata(sb, &buf, &offset, &size); + if (IS_ERR(data)) { + ret = PTR_ERR(data); + break; + } + + if (dec && dec->config) { + ret = dec->config(sb, dsb, data, size); + } else { + erofs_err(sb, "algorithm %ld isn't enabled on this kernel", + alg); + ret = -EOPNOTSUPP; + } + kfree(data); + if (ret) + break; + } + erofs_put_metabuf(&buf); + return ret; +} + +int __init z_erofs_init_decompressor(void) +{ + int i, err; + + for (i = 0; i < Z_EROFS_COMPRESSION_MAX; ++i) { + err = z_erofs_decomp[i] ? z_erofs_decomp[i]->init() : 0; + if (err) { + while (i--) + if (z_erofs_decomp[i]) + z_erofs_decomp[i]->exit(); + return err; + } + } + return 0; +} + +void z_erofs_exit_decompressor(void) +{ + int i; + + for (i = 0; i < Z_EROFS_COMPRESSION_MAX; ++i) + if (z_erofs_decomp[i]) + z_erofs_decomp[i]->exit(); +} diff --git a/fs/erofs/decompressor_crypto.c b/fs/erofs/decompressor_crypto.c new file mode 100644 index 000000000..77c6bd535 --- /dev/null +++ b/fs/erofs/decompressor_crypto.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#include +#include +#include "compress.h" + +static int __z_erofs_crypto_decompress(struct z_erofs_decompress_req *rq, + struct crypto_acomp *tfm) +{ + struct sg_table st_src, st_dst; + struct acomp_req *req; + struct crypto_wait wait; + const char *reason; + u8 *headpage; + int ret; + + headpage = kmap_local_page(*rq->in); + reason = z_erofs_fixup_insize(rq, headpage + rq->pageofs_in, + min_t(unsigned int, rq->inputsize, + rq->sb->s_blocksize - rq->pageofs_in)); + kunmap_local(headpage); + if (reason) + return IS_ERR(reason) ? PTR_ERR(reason) : -EFSCORRUPTED; + + req = acomp_request_alloc(tfm); + if (!req) + return -ENOMEM; + + ret = sg_alloc_table_from_pages_segment(&st_src, rq->in, rq->inpages, + rq->pageofs_in, rq->inputsize, UINT_MAX, GFP_KERNEL); + if (ret < 0) + goto failed_src_alloc; + + ret = sg_alloc_table_from_pages_segment(&st_dst, rq->out, rq->outpages, + rq->pageofs_out, rq->outputsize, UINT_MAX, GFP_KERNEL); + if (ret < 0) + goto failed_dst_alloc; + + acomp_request_set_params(req, st_src.sgl, + st_dst.sgl, rq->inputsize, rq->outputsize); + + crypto_init_wait(&wait); + acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + + ret = crypto_wait_req(crypto_acomp_decompress(req), &wait); + if (ret) { + erofs_err(rq->sb, "failed to decompress %d in[%u, %u] out[%u]", + ret, rq->inputsize, rq->pageofs_in, rq->outputsize); + ret = -EIO; + } + + sg_free_table(&st_dst); +failed_dst_alloc: + sg_free_table(&st_src); +failed_src_alloc: + acomp_request_free(req); + return ret; +} + +struct z_erofs_crypto_engine { + char *crypto_name; + struct crypto_acomp *tfm; +}; + +static struct z_erofs_crypto_engine *z_erofs_crypto[Z_EROFS_COMPRESSION_MAX] = { + [Z_EROFS_COMPRESSION_LZ4] = (struct z_erofs_crypto_engine[]) { + {}, + }, + [Z_EROFS_COMPRESSION_LZMA] = (struct z_erofs_crypto_engine[]) { + {}, + }, + [Z_EROFS_COMPRESSION_DEFLATE] = (struct z_erofs_crypto_engine[]) { + { .crypto_name = "qat_deflate", }, + {}, + }, + [Z_EROFS_COMPRESSION_ZSTD] = (struct z_erofs_crypto_engine[]) { + {}, + }, +}; +static DECLARE_RWSEM(z_erofs_crypto_rwsem); + +static struct crypto_acomp *z_erofs_crypto_get_engine(int alg) +{ + struct z_erofs_crypto_engine *e; + + for (e = z_erofs_crypto[alg]; e->crypto_name; ++e) + if (e->tfm) + return e->tfm; + return NULL; +} + +int z_erofs_crypto_decompress(struct z_erofs_decompress_req *rq, + struct page **pgpl) +{ + struct crypto_acomp *tfm; + int i, err; + + down_read(&z_erofs_crypto_rwsem); + tfm = z_erofs_crypto_get_engine(rq->alg); + if (!tfm) { + err = -EOPNOTSUPP; + goto out; + } + + for (i = 0; i < rq->outpages; i++) { + struct page *const page = rq->out[i]; + struct page *victim; + + if (!page) { + victim = __erofs_allocpage(pgpl, rq->gfp, true); + if (!victim) { + err = -ENOMEM; + goto out; + } + set_page_private(victim, Z_EROFS_SHORTLIVED_PAGE); + rq->out[i] = victim; + } + } + err = __z_erofs_crypto_decompress(rq, tfm); +out: + up_read(&z_erofs_crypto_rwsem); + return err; +} + +int z_erofs_crypto_enable_engine(const char *name, int len) +{ + struct z_erofs_crypto_engine *e; + struct crypto_acomp *tfm; + int alg; + + down_write(&z_erofs_crypto_rwsem); + for (alg = 0; alg < Z_EROFS_COMPRESSION_MAX; ++alg) { + for (e = z_erofs_crypto[alg]; e->crypto_name; ++e) { + if (!strncmp(name, e->crypto_name, len)) { + if (e->tfm) + break; + tfm = crypto_alloc_acomp(e->crypto_name, 0, 0); + if (IS_ERR(tfm)) { + up_write(&z_erofs_crypto_rwsem); + return -EOPNOTSUPP; + } + e->tfm = tfm; + break; + } + } + } + up_write(&z_erofs_crypto_rwsem); + return 0; +} + +void z_erofs_crypto_disable_all_engines(void) +{ + struct z_erofs_crypto_engine *e; + int alg; + + down_write(&z_erofs_crypto_rwsem); + for (alg = 0; alg < Z_EROFS_COMPRESSION_MAX; ++alg) { + for (e = z_erofs_crypto[alg]; e->crypto_name; ++e) { + if (!e->tfm) + continue; + crypto_free_acomp(e->tfm); + e->tfm = NULL; + } + } + up_write(&z_erofs_crypto_rwsem); +} + +int z_erofs_crypto_show_engines(char *buf, int size, char sep) +{ + struct z_erofs_crypto_engine *e; + int alg, len = 0; + + for (alg = 0; alg < Z_EROFS_COMPRESSION_MAX; ++alg) { + for (e = z_erofs_crypto[alg]; e->crypto_name; ++e) { + if (!e->tfm) + continue; + len += scnprintf(buf + len, size - len, "%s%c", + e->crypto_name, sep); + } + } + return len; +} diff --git a/fs/erofs/decompressor_deflate.c b/fs/erofs/decompressor_deflate.c new file mode 100644 index 000000000..6167af1ff --- /dev/null +++ b/fs/erofs/decompressor_deflate.c @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#include +#include "compress.h" + +struct z_erofs_deflate { + struct z_erofs_deflate *next; + struct z_stream_s z; + u8 bounce[PAGE_SIZE]; +}; + +static DEFINE_SPINLOCK(z_erofs_deflate_lock); +static unsigned int z_erofs_deflate_nstrms, z_erofs_deflate_avail_strms; +static struct z_erofs_deflate *z_erofs_deflate_head; +static DECLARE_WAIT_QUEUE_HEAD(z_erofs_deflate_wq); + +module_param_named(deflate_streams, z_erofs_deflate_nstrms, uint, 0444); + +static void z_erofs_deflate_exit(void) +{ + /* there should be no running fs instance */ + while (z_erofs_deflate_avail_strms) { + struct z_erofs_deflate *strm; + + spin_lock(&z_erofs_deflate_lock); + strm = z_erofs_deflate_head; + if (!strm) { + spin_unlock(&z_erofs_deflate_lock); + continue; + } + z_erofs_deflate_head = NULL; + spin_unlock(&z_erofs_deflate_lock); + + while (strm) { + struct z_erofs_deflate *n = strm->next; + + vfree(strm->z.workspace); + kfree(strm); + --z_erofs_deflate_avail_strms; + strm = n; + } + } +} + +static int __init z_erofs_deflate_init(void) +{ + /* by default, use # of possible CPUs instead */ + if (!z_erofs_deflate_nstrms) + z_erofs_deflate_nstrms = num_possible_cpus(); + return 0; +} + +static int z_erofs_load_deflate_config(struct super_block *sb, + struct erofs_super_block *dsb, void *data, int size) +{ + struct z_erofs_deflate_cfgs *dfl = data; + static DEFINE_MUTEX(deflate_resize_mutex); + static bool inited; + + if (!dfl || size < sizeof(struct z_erofs_deflate_cfgs)) { + erofs_err(sb, "invalid deflate cfgs, size=%u", size); + return -EINVAL; + } + + if (dfl->windowbits > MAX_WBITS) { + erofs_err(sb, "unsupported windowbits %u", dfl->windowbits); + return -EOPNOTSUPP; + } + mutex_lock(&deflate_resize_mutex); + if (!inited) { + for (; z_erofs_deflate_avail_strms < z_erofs_deflate_nstrms; + ++z_erofs_deflate_avail_strms) { + struct z_erofs_deflate *strm; + + strm = kzalloc_obj(*strm); + if (!strm) + goto failed; + /* XXX: in-kernel zlib cannot customize windowbits */ + strm->z.workspace = vmalloc(zlib_inflate_workspacesize()); + if (!strm->z.workspace) { + kfree(strm); + goto failed; + } + + spin_lock(&z_erofs_deflate_lock); + strm->next = z_erofs_deflate_head; + z_erofs_deflate_head = strm; + spin_unlock(&z_erofs_deflate_lock); + } + inited = true; + } + mutex_unlock(&deflate_resize_mutex); + return 0; +failed: + mutex_unlock(&deflate_resize_mutex); + z_erofs_deflate_exit(); + return -ENOMEM; +} + +static const char *__z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq, + struct page **pgpl) +{ + struct super_block *sb = rq->sb; + struct z_erofs_stream_dctx dctx = { .rq = rq, .no = -1, .ni = 0 }; + struct z_erofs_deflate *strm; + const char *reason; + int zerr; + + /* 1. get the exact DEFLATE compressed size */ + dctx.kin = kmap_local_page(*rq->in); + reason = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in, + min(rq->inputsize, sb->s_blocksize - rq->pageofs_in)); + if (reason) { + kunmap_local(dctx.kin); + return reason; + } + + /* 2. get an available DEFLATE context */ +again: + spin_lock(&z_erofs_deflate_lock); + strm = z_erofs_deflate_head; + if (!strm) { + spin_unlock(&z_erofs_deflate_lock); + wait_event(z_erofs_deflate_wq, READ_ONCE(z_erofs_deflate_head)); + goto again; + } + z_erofs_deflate_head = strm->next; + spin_unlock(&z_erofs_deflate_lock); + + /* 3. multi-call decompress */ + zerr = zlib_inflateInit2(&strm->z, -MAX_WBITS); + if (zerr != Z_OK) { + reason = ERR_PTR(-EINVAL); + goto failed_zinit; + } + + rq->fillgaps = true; /* DEFLATE doesn't support NULL output buffer */ + strm->z.avail_in = min(rq->inputsize, PAGE_SIZE - rq->pageofs_in); + rq->inputsize -= strm->z.avail_in; + strm->z.next_in = dctx.kin + rq->pageofs_in; + strm->z.avail_out = 0; + dctx.bounce = strm->bounce; + + while (1) { + dctx.avail_out = strm->z.avail_out; + dctx.inbuf_sz = strm->z.avail_in; + reason = z_erofs_stream_switch_bufs(&dctx, + (void **)&strm->z.next_out, + (void **)&strm->z.next_in, pgpl); + if (reason) + break; + strm->z.avail_out = dctx.avail_out; + strm->z.avail_in = dctx.inbuf_sz; + + zerr = zlib_inflate(&strm->z, Z_SYNC_FLUSH); + if (zerr != Z_OK || !(rq->outputsize + strm->z.avail_out)) { + if (zerr == Z_OK && rq->partial_decoding) + break; + if (zerr == Z_STREAM_END && !rq->outputsize) + break; + reason = (zerr == Z_DATA_ERROR ? + "corrupted compressed data" : + "unexpected end of stream"); + break; + } + } + if (zlib_inflateEnd(&strm->z) != Z_OK && !reason) + reason = ERR_PTR(-EIO); + if (dctx.kout) + kunmap_local(dctx.kout); +failed_zinit: + kunmap_local(dctx.kin); + /* 4. push back DEFLATE stream context to the global list */ + spin_lock(&z_erofs_deflate_lock); + strm->next = z_erofs_deflate_head; + z_erofs_deflate_head = strm; + spin_unlock(&z_erofs_deflate_lock); + wake_up(&z_erofs_deflate_wq); + return reason; +} + +static const char *z_erofs_deflate_decompress(struct z_erofs_decompress_req *rq, + struct page **pgpl) +{ +#ifdef CONFIG_EROFS_FS_ZIP_ACCEL + int err; + + if (!rq->partial_decoding) { + err = z_erofs_crypto_decompress(rq, pgpl); + if (err != -EOPNOTSUPP) + return ERR_PTR(err); + + } +#endif + return __z_erofs_deflate_decompress(rq, pgpl); +} + +const struct z_erofs_decompressor z_erofs_deflate_decomp = { + .config = z_erofs_load_deflate_config, + .decompress = z_erofs_deflate_decompress, + .init = z_erofs_deflate_init, + .exit = z_erofs_deflate_exit, + .name = "deflate", +}; diff --git a/fs/erofs/decompressor_lzma.c b/fs/erofs/decompressor_lzma.c new file mode 100644 index 000000000..6b0cdb446 --- /dev/null +++ b/fs/erofs/decompressor_lzma.c @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#include +#include "compress.h" + +struct z_erofs_lzma { + struct z_erofs_lzma *next; + struct xz_dec_microlzma *state; + u8 bounce[PAGE_SIZE]; +}; + +/* considering the LZMA performance, no need to use a lockless list for now */ +static DEFINE_SPINLOCK(z_erofs_lzma_lock); +static unsigned int z_erofs_lzma_max_dictsize; +static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms; +static struct z_erofs_lzma *z_erofs_lzma_head; +static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq); + +module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444); + +static void z_erofs_lzma_exit(void) +{ + /* there should be no running fs instance */ + while (z_erofs_lzma_avail_strms) { + struct z_erofs_lzma *strm; + + spin_lock(&z_erofs_lzma_lock); + strm = z_erofs_lzma_head; + if (!strm) { + spin_unlock(&z_erofs_lzma_lock); + DBG_BUGON(1); + return; + } + z_erofs_lzma_head = NULL; + spin_unlock(&z_erofs_lzma_lock); + + while (strm) { + struct z_erofs_lzma *n = strm->next; + + if (strm->state) + xz_dec_microlzma_end(strm->state); + kfree(strm); + --z_erofs_lzma_avail_strms; + strm = n; + } + } +} + +static int __init z_erofs_lzma_init(void) +{ + unsigned int i; + + /* by default, use # of possible CPUs instead */ + if (!z_erofs_lzma_nstrms) + z_erofs_lzma_nstrms = min_t(unsigned int, num_possible_cpus(), + CONFIG_EROFS_FS_ZIP_LZMA_DEFAULT_MAX_STREAMS); + + for (i = 0; i < z_erofs_lzma_nstrms; ++i) { + struct z_erofs_lzma *strm = kzalloc_obj(*strm); + + if (!strm) { + z_erofs_lzma_exit(); + return -ENOMEM; + } + spin_lock(&z_erofs_lzma_lock); + strm->next = z_erofs_lzma_head; + z_erofs_lzma_head = strm; + spin_unlock(&z_erofs_lzma_lock); + ++z_erofs_lzma_avail_strms; + } + return 0; +} + +static int z_erofs_load_lzma_config(struct super_block *sb, + struct erofs_super_block *dsb, void *data, int size) +{ + static DEFINE_MUTEX(lzma_resize_mutex); + struct z_erofs_lzma_cfgs *lzma = data; + unsigned int dict_size, i; + struct z_erofs_lzma *strm, *head = NULL; + int err; + + if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) { + erofs_err(sb, "invalid lzma cfgs, size=%u", size); + return -EINVAL; + } + if (lzma->format) { + erofs_err(sb, "unidentified lzma format %x, please check kernel version", + le16_to_cpu(lzma->format)); + return -EINVAL; + } + dict_size = le32_to_cpu(lzma->dict_size); + if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) { + erofs_err(sb, "unsupported lzma dictionary size %u", + dict_size); + return -EINVAL; + } + + /* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */ + mutex_lock(&lzma_resize_mutex); + + if (z_erofs_lzma_max_dictsize >= dict_size) { + mutex_unlock(&lzma_resize_mutex); + return 0; + } + + /* 1. collect/isolate all streams for the following check */ + for (i = 0; i < z_erofs_lzma_avail_strms; ++i) { + struct z_erofs_lzma *last; + +again: + spin_lock(&z_erofs_lzma_lock); + strm = z_erofs_lzma_head; + if (!strm) { + spin_unlock(&z_erofs_lzma_lock); + wait_event(z_erofs_lzma_wq, + READ_ONCE(z_erofs_lzma_head)); + goto again; + } + z_erofs_lzma_head = NULL; + spin_unlock(&z_erofs_lzma_lock); + + for (last = strm; last->next; last = last->next) + ++i; + last->next = head; + head = strm; + } + + err = 0; + /* 2. walk each isolated stream and grow max dict_size if needed */ + for (strm = head; strm; strm = strm->next) { + if (strm->state) + xz_dec_microlzma_end(strm->state); + strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size); + if (!strm->state) + err = -ENOMEM; + } + + /* 3. push back all to the global list and update max dict_size */ + spin_lock(&z_erofs_lzma_lock); + DBG_BUGON(z_erofs_lzma_head); + z_erofs_lzma_head = head; + spin_unlock(&z_erofs_lzma_lock); + wake_up_all(&z_erofs_lzma_wq); + + z_erofs_lzma_max_dictsize = dict_size; + mutex_unlock(&lzma_resize_mutex); + return err; +} + +static const char *z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq, + struct page **pgpl) +{ + struct super_block *sb = rq->sb; + struct z_erofs_stream_dctx dctx = { .rq = rq, .no = -1, .ni = 0 }; + struct xz_buf buf = {}; + struct z_erofs_lzma *strm; + enum xz_ret xz_err; + const char *reason; + + /* 1. get the exact LZMA compressed size */ + dctx.kin = kmap_local_page(*rq->in); + reason = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in, + min(rq->inputsize, sb->s_blocksize - rq->pageofs_in)); + if (reason) { + kunmap_local(dctx.kin); + return reason; + } + + /* 2. get an available lzma context */ +again: + spin_lock(&z_erofs_lzma_lock); + strm = z_erofs_lzma_head; + if (!strm) { + spin_unlock(&z_erofs_lzma_lock); + wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head)); + goto again; + } + z_erofs_lzma_head = strm->next; + spin_unlock(&z_erofs_lzma_lock); + + /* 3. multi-call decompress */ + xz_dec_microlzma_reset(strm->state, rq->inputsize, rq->outputsize, + !rq->partial_decoding); + buf.in_size = min(rq->inputsize, PAGE_SIZE - rq->pageofs_in); + rq->inputsize -= buf.in_size; + buf.in = dctx.kin + rq->pageofs_in; + dctx.bounce = strm->bounce; + do { + dctx.avail_out = buf.out_size - buf.out_pos; + dctx.inbuf_sz = buf.in_size; + dctx.inbuf_pos = buf.in_pos; + reason = z_erofs_stream_switch_bufs(&dctx, (void **)&buf.out, + (void **)&buf.in, pgpl); + if (reason) + break; + + if (buf.out_size == buf.out_pos) { + buf.out_size = dctx.avail_out; + buf.out_pos = 0; + } + buf.in_size = dctx.inbuf_sz; + buf.in_pos = dctx.inbuf_pos; + + xz_err = xz_dec_microlzma_run(strm->state, &buf); + DBG_BUGON(buf.out_pos > buf.out_size); + DBG_BUGON(buf.in_pos > buf.in_size); + + if (xz_err != XZ_OK) { + if (xz_err == XZ_STREAM_END && !rq->outputsize) + break; + reason = (xz_err == XZ_DATA_ERROR ? + "corrupted compressed data" : + "unexpected end of stream"); + break; + } + } while (1); + + if (dctx.kout) + kunmap_local(dctx.kout); + kunmap_local(dctx.kin); + /* 4. push back LZMA stream context to the global list */ + spin_lock(&z_erofs_lzma_lock); + strm->next = z_erofs_lzma_head; + z_erofs_lzma_head = strm; + spin_unlock(&z_erofs_lzma_lock); + wake_up(&z_erofs_lzma_wq); + return reason; +} + +const struct z_erofs_decompressor z_erofs_lzma_decomp = { + .config = z_erofs_load_lzma_config, + .decompress = z_erofs_lzma_decompress, + .init = z_erofs_lzma_init, + .exit = z_erofs_lzma_exit, + .name = "lzma" +}; diff --git a/fs/erofs/decompressor_zstd.c b/fs/erofs/decompressor_zstd.c new file mode 100644 index 000000000..ab318a240 --- /dev/null +++ b/fs/erofs/decompressor_zstd.c @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#include +#include "compress.h" + +struct z_erofs_zstd { + struct z_erofs_zstd *next; + u8 bounce[PAGE_SIZE]; + void *wksp; + unsigned int wkspsz; +}; + +static DEFINE_SPINLOCK(z_erofs_zstd_lock); +static unsigned int z_erofs_zstd_max_dictsize; +static unsigned int z_erofs_zstd_nstrms, z_erofs_zstd_avail_strms; +static struct z_erofs_zstd *z_erofs_zstd_head; +static DECLARE_WAIT_QUEUE_HEAD(z_erofs_zstd_wq); + +module_param_named(zstd_streams, z_erofs_zstd_nstrms, uint, 0444); + +static struct z_erofs_zstd *z_erofs_isolate_strms(bool all) +{ + struct z_erofs_zstd *strm; + +again: + spin_lock(&z_erofs_zstd_lock); + strm = z_erofs_zstd_head; + if (!strm) { + spin_unlock(&z_erofs_zstd_lock); + wait_event(z_erofs_zstd_wq, READ_ONCE(z_erofs_zstd_head)); + goto again; + } + z_erofs_zstd_head = all ? NULL : strm->next; + spin_unlock(&z_erofs_zstd_lock); + return strm; +} + +static void z_erofs_zstd_exit(void) +{ + while (z_erofs_zstd_avail_strms) { + struct z_erofs_zstd *strm, *n; + + for (strm = z_erofs_isolate_strms(true); strm; strm = n) { + n = strm->next; + + kvfree(strm->wksp); + kfree(strm); + --z_erofs_zstd_avail_strms; + } + } +} + +static int __init z_erofs_zstd_init(void) +{ + /* by default, use # of possible CPUs instead */ + if (!z_erofs_zstd_nstrms) + z_erofs_zstd_nstrms = num_possible_cpus(); + + for (; z_erofs_zstd_avail_strms < z_erofs_zstd_nstrms; + ++z_erofs_zstd_avail_strms) { + struct z_erofs_zstd *strm; + + strm = kzalloc_obj(*strm); + if (!strm) { + z_erofs_zstd_exit(); + return -ENOMEM; + } + spin_lock(&z_erofs_zstd_lock); + strm->next = z_erofs_zstd_head; + z_erofs_zstd_head = strm; + spin_unlock(&z_erofs_zstd_lock); + } + return 0; +} + +static int z_erofs_load_zstd_config(struct super_block *sb, + struct erofs_super_block *dsb, void *data, int size) +{ + static DEFINE_MUTEX(zstd_resize_mutex); + struct z_erofs_zstd_cfgs *zstd = data; + unsigned int dict_size, wkspsz; + struct z_erofs_zstd *strm, *head = NULL; + void *wksp; + + if (!zstd || size < sizeof(struct z_erofs_zstd_cfgs) || zstd->format) { + erofs_err(sb, "unsupported zstd format, size=%u", size); + return -EINVAL; + } + + if (zstd->windowlog > ilog2(Z_EROFS_ZSTD_MAX_DICT_SIZE) - 10) { + erofs_err(sb, "unsupported zstd window log %u", zstd->windowlog); + return -EINVAL; + } + dict_size = 1U << (zstd->windowlog + 10); + + /* in case 2 z_erofs_load_zstd_config() race to avoid deadlock */ + mutex_lock(&zstd_resize_mutex); + if (z_erofs_zstd_max_dictsize >= dict_size) { + mutex_unlock(&zstd_resize_mutex); + return 0; + } + + /* 1. collect/isolate all streams for the following check */ + while (z_erofs_zstd_avail_strms) { + struct z_erofs_zstd *n; + + for (strm = z_erofs_isolate_strms(true); strm; strm = n) { + n = strm->next; + strm->next = head; + head = strm; + --z_erofs_zstd_avail_strms; + } + } + + /* 2. walk each isolated stream and grow max dict_size if needed */ + wkspsz = zstd_dstream_workspace_bound(dict_size); + for (strm = head; strm; strm = strm->next) { + wksp = kvmalloc(wkspsz, GFP_KERNEL); + if (!wksp) + break; + kvfree(strm->wksp); + strm->wksp = wksp; + strm->wkspsz = wkspsz; + } + + /* 3. push back all to the global list and update max dict_size */ + spin_lock(&z_erofs_zstd_lock); + DBG_BUGON(z_erofs_zstd_head); + z_erofs_zstd_head = head; + spin_unlock(&z_erofs_zstd_lock); + z_erofs_zstd_avail_strms = z_erofs_zstd_nstrms; + wake_up_all(&z_erofs_zstd_wq); + if (!strm) + z_erofs_zstd_max_dictsize = dict_size; + mutex_unlock(&zstd_resize_mutex); + return strm ? -ENOMEM : 0; +} + +static const char *z_erofs_zstd_decompress(struct z_erofs_decompress_req *rq, + struct page **pgpl) +{ + struct super_block *sb = rq->sb; + struct z_erofs_stream_dctx dctx = { .rq = rq, .no = -1, .ni = 0 }; + zstd_in_buffer in_buf = { NULL, 0, 0 }; + zstd_out_buffer out_buf = { NULL, 0, 0 }; + struct z_erofs_zstd *strm; + zstd_dstream *stream; + const char *reason; + int zerr; + + /* 1. get the exact compressed size */ + dctx.kin = kmap_local_page(*rq->in); + reason = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in, + min(rq->inputsize, sb->s_blocksize - rq->pageofs_in)); + if (reason) { + kunmap_local(dctx.kin); + return reason; + } + + /* 2. get an available ZSTD context */ + strm = z_erofs_isolate_strms(false); + + /* 3. multi-call decompress */ + stream = zstd_init_dstream(z_erofs_zstd_max_dictsize, strm->wksp, strm->wkspsz); + if (!stream) { + reason = ERR_PTR(-ENOMEM); + goto failed_zinit; + } + + rq->fillgaps = true; /* ZSTD doesn't support NULL output buffer */ + in_buf.size = min_t(u32, rq->inputsize, PAGE_SIZE - rq->pageofs_in); + rq->inputsize -= in_buf.size; + in_buf.src = dctx.kin + rq->pageofs_in; + dctx.bounce = strm->bounce; + + do { + dctx.inbuf_sz = in_buf.size; + dctx.inbuf_pos = in_buf.pos; + reason = z_erofs_stream_switch_bufs(&dctx, &out_buf.dst, + (void **)&in_buf.src, pgpl); + if (reason) + break; + + if (out_buf.size == out_buf.pos) { + out_buf.size = dctx.avail_out; + out_buf.pos = 0; + } + in_buf.size = dctx.inbuf_sz; + in_buf.pos = dctx.inbuf_pos; + + zerr = zstd_decompress_stream(stream, &out_buf, &in_buf); + dctx.avail_out = out_buf.size - out_buf.pos; + if (zstd_is_error(zerr) || + ((rq->outputsize + dctx.avail_out) && (!zerr || (zerr > 0 && + !(rq->inputsize + in_buf.size - in_buf.pos))))) { + reason = zstd_is_error(zerr) ? zstd_get_error_name(zerr) : + "unexpected end of stream"; + break; + } + } while (rq->outputsize + dctx.avail_out); + + if (dctx.kout) + kunmap_local(dctx.kout); +failed_zinit: + kunmap_local(dctx.kin); + /* 4. push back ZSTD stream context to the global list */ + spin_lock(&z_erofs_zstd_lock); + strm->next = z_erofs_zstd_head; + z_erofs_zstd_head = strm; + spin_unlock(&z_erofs_zstd_lock); + wake_up(&z_erofs_zstd_wq); + return reason; +} + +const struct z_erofs_decompressor z_erofs_zstd_decomp = { + .config = z_erofs_load_zstd_config, + .decompress = z_erofs_zstd_decompress, + .init = z_erofs_zstd_init, + .exit = z_erofs_zstd_exit, + .name = "zstd", +}; diff --git a/fs/erofs/dir.c b/fs/erofs/dir.c new file mode 100644 index 000000000..4aa52a5f2 --- /dev/null +++ b/fs/erofs/dir.c @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2022, Alibaba Cloud + */ +#include "internal.h" +#include + +static int erofs_fill_dentries(struct inode *dir, struct dir_context *ctx, + void *dentry_blk, struct erofs_dirent *de, + unsigned int nameoff0, unsigned int maxsize) +{ + const struct erofs_dirent *end = dentry_blk + nameoff0; + + while (de < end) { + unsigned char d_type = fs_ftype_to_dtype(de->file_type); + unsigned int nameoff = le16_to_cpu(de->nameoff); + const char *de_name = (char *)dentry_blk + nameoff; + unsigned int de_namelen; + + /* non-trailing dirent in the directory block? */ + if (de + 1 < end) + de_namelen = le16_to_cpu(de[1].nameoff) - nameoff; + else if (maxsize <= nameoff) + goto err_bogus; + else + de_namelen = strnlen(de_name, maxsize - nameoff); + + /* a corrupted entry is found (including negative namelen) */ + if (!in_range32(de_namelen, 1, EROFS_NAME_LEN) || + nameoff + de_namelen > maxsize) + goto err_bogus; + + if (!dir_emit(ctx, de_name, de_namelen, + erofs_nid_to_ino64(EROFS_SB(dir->i_sb), + le64_to_cpu(de->nid)), d_type)) + return 1; + ++de; + ctx->pos += sizeof(struct erofs_dirent); + } + return 0; +err_bogus: + erofs_err(dir->i_sb, "bogus dirent @ nid %llu", EROFS_I(dir)->nid); + DBG_BUGON(1); + return -EFSCORRUPTED; +} + +static int erofs_readdir(struct file *f, struct dir_context *ctx) +{ + struct inode *dir = file_inode(f); + struct erofs_buf buf = __EROFS_BUF_INITIALIZER; + struct super_block *sb = dir->i_sb; + struct file_ra_state *ra = &f->f_ra; + unsigned long bsz = sb->s_blocksize; + unsigned int ofs = erofs_blkoff(sb, ctx->pos); + pgoff_t ra_pages = DIV_ROUND_UP_POW2( + EROFS_I_SB(dir)->dir_ra_bytes, PAGE_SIZE); + pgoff_t nr_pages = DIV_ROUND_UP_POW2(dir->i_size, PAGE_SIZE); + int err = 0; + bool initial = true; + + buf.mapping = dir->i_mapping; + while (ctx->pos < dir->i_size) { + erofs_off_t dbstart = ctx->pos - ofs; + struct erofs_dirent *de; + unsigned int nameoff, maxsize; + + if (fatal_signal_pending(current)) { + err = -ERESTARTSYS; + break; + } + + /* readahead blocks to enhance performance for large directories */ + if (ra_pages) { + pgoff_t idx = DIV_ROUND_UP_POW2(ctx->pos, PAGE_SIZE); + pgoff_t pages = min(nr_pages - idx, ra_pages); + + if (pages > 1 && !ra_has_index(ra, idx)) + page_cache_sync_readahead(dir->i_mapping, ra, + f, idx, pages); + } + + de = erofs_bread(&buf, dbstart, true); + if (IS_ERR(de)) { + erofs_err(sb, "failed to readdir of logical block %llu of nid %llu", + erofs_blknr(sb, dbstart), EROFS_I(dir)->nid); + err = PTR_ERR(de); + break; + } + + nameoff = le16_to_cpu(de->nameoff); + if (!nameoff || nameoff >= bsz || (nameoff % sizeof(*de))) { + erofs_err(sb, "invalid de[0].nameoff %u @ nid %llu", + nameoff, EROFS_I(dir)->nid); + err = -EFSCORRUPTED; + break; + } + + maxsize = min_t(unsigned int, dir->i_size - dbstart, bsz); + /* search dirents at the arbitrary position */ + if (initial) { + initial = false; + ofs = roundup(ofs, sizeof(struct erofs_dirent)); + ctx->pos = dbstart + ofs; + } + + err = erofs_fill_dentries(dir, ctx, de, (void *)de + ofs, + nameoff, maxsize); + if (err) + break; + ctx->pos = dbstart + maxsize; + ofs = 0; + cond_resched(); + } + erofs_put_metabuf(&buf); + if (EROFS_I(dir)->dot_omitted && ctx->pos == dir->i_size) { + if (!dir_emit_dot(f, ctx)) + return 0; + ++ctx->pos; + } + return err < 0 ? err : 0; +} + +const struct file_operations erofs_dir_fops = { + .llseek = generic_file_llseek, + .read = generic_read_dir, + .iterate_shared = erofs_readdir, + .unlocked_ioctl = erofs_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = erofs_compat_ioctl, +#endif + .setlease = generic_setlease, +}; diff --git a/fs/erofs/erofs_fs.h b/fs/erofs/erofs_fs.h new file mode 100644 index 000000000..16ec4fd33 --- /dev/null +++ b/fs/erofs/erofs_fs.h @@ -0,0 +1,469 @@ +/* SPDX-License-Identifier: MIT */ +/* + * EROFS (Enhanced ROM File System) on-disk format definition + * + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021, Alibaba Cloud + */ +#ifndef __EROFS_FS_H +#define __EROFS_FS_H + +/* to allow for x86 boot sectors and other oddities. */ +#define EROFS_SUPER_OFFSET 1024 + +#define EROFS_FEATURE_COMPAT_SB_CHKSUM 0x00000001 +#define EROFS_FEATURE_COMPAT_MTIME 0x00000002 +#define EROFS_FEATURE_COMPAT_XATTR_FILTER 0x00000004 +#define EROFS_FEATURE_COMPAT_SHARED_EA_IN_METABOX 0x00000008 +#define EROFS_FEATURE_COMPAT_PLAIN_XATTR_PFX 0x00000010 +#define EROFS_FEATURE_COMPAT_ISHARE_XATTRS 0x00000020 + +/* + * Any bits that aren't in EROFS_ALL_FEATURE_INCOMPAT should + * be incompatible with this kernel version. + */ +#define EROFS_FEATURE_INCOMPAT_LZ4_0PADDING 0x00000001 +#define EROFS_FEATURE_INCOMPAT_COMPR_CFGS 0x00000002 +#define EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER 0x00000002 +#define EROFS_FEATURE_INCOMPAT_CHUNKED_FILE 0x00000004 +#define EROFS_FEATURE_INCOMPAT_DEVICE_TABLE 0x00000008 +#define EROFS_FEATURE_INCOMPAT_COMPR_HEAD2 0x00000008 +#define EROFS_FEATURE_INCOMPAT_ZTAILPACKING 0x00000010 +#define EROFS_FEATURE_INCOMPAT_FRAGMENTS 0x00000020 +#define EROFS_FEATURE_INCOMPAT_DEDUPE 0x00000020 +#define EROFS_FEATURE_INCOMPAT_XATTR_PREFIXES 0x00000040 +#define EROFS_FEATURE_INCOMPAT_48BIT 0x00000080 +#define EROFS_FEATURE_INCOMPAT_METABOX 0x00000100 +#define EROFS_ALL_FEATURE_INCOMPAT \ + ((EROFS_FEATURE_INCOMPAT_METABOX << 1) - 1) + +#define EROFS_SB_EXTSLOT_SIZE 16 + +struct erofs_deviceslot { + u8 tag[64]; /* digest(sha256), etc. */ + __le32 blocks_lo; /* total blocks count of this device */ + __le32 uniaddr_lo; /* unified starting block of this device */ + __le16 blocks_hi; /* total blocks count MSB */ + __le16 uniaddr_hi; /* unified starting block MSB */ + u8 reserved[52]; +}; +#define EROFS_DEVT_SLOT_SIZE sizeof(struct erofs_deviceslot) + +/* erofs on-disk super block (currently 144 bytes at maximum) */ +struct erofs_super_block { + __le32 magic; /* file system magic number */ + __le32 checksum; /* crc32c to avoid unexpected on-disk overlap */ + __le32 feature_compat; + __u8 blkszbits; /* filesystem block size in bit shift */ + __u8 sb_extslots; /* superblock size = 128 + sb_extslots * 16 */ + union { + __le16 rootnid_2b; /* nid of root directory */ + __le16 blocks_hi; /* (48BIT on) blocks count MSB */ + } __packed rb; + __le64 inos; /* total valid ino # (== f_files - f_favail) */ + __le64 epoch; /* base seconds used for compact inodes */ + __le32 fixed_nsec; /* fixed nanoseconds for compact inodes */ + __le32 blocks_lo; /* blocks count LSB */ + __le32 meta_blkaddr; /* start block address of metadata area */ + __le32 xattr_blkaddr; /* start block address of shared xattr area */ + __u8 uuid[16]; /* 128-bit uuid for volume */ + __u8 volume_name[16]; /* volume name */ + __le32 feature_incompat; + union { + /* bitmap for available compression algorithms */ + __le16 available_compr_algs; + /* customized sliding window size instead of 64k by default */ + __le16 lz4_max_distance; + } __packed u1; + __le16 extra_devices; /* # of devices besides the primary device */ + __le16 devt_slotoff; /* startoff = devt_slotoff * devt_slotsize */ + __u8 dirblkbits; /* directory block size in bit shift */ + __u8 xattr_prefix_count; /* # of long xattr name prefixes */ + __le32 xattr_prefix_start; /* start of long xattr prefixes */ + __le64 packed_nid; /* nid of the special packed inode */ + __u8 xattr_filter_reserved; /* reserved for xattr name filter */ + __u8 ishare_xattr_prefix_id; + __u8 reserved[2]; + __le32 build_time; /* seconds added to epoch for mkfs time */ + __le64 rootnid_8b; /* (48BIT on) nid of root directory */ + __le64 reserved2; + __le64 metabox_nid; /* (METABOX on) nid of the metabox inode */ + __le64 reserved3; /* [align to extslot 1] */ +}; + +/* + * EROFS inode datalayout (i_format in on-disk inode): + * 0 - uncompressed flat inode without tail-packing inline data: + * 1 - compressed inode with non-compact indexes: + * 2 - uncompressed flat inode with tail-packing inline data: + * 3 - compressed inode with compact indexes: + * 4 - chunk-based inode with (optional) multi-device support: + * 5~7 - reserved + */ +enum { + EROFS_INODE_FLAT_PLAIN = 0, + EROFS_INODE_COMPRESSED_FULL = 1, + EROFS_INODE_FLAT_INLINE = 2, + EROFS_INODE_COMPRESSED_COMPACT = 3, + EROFS_INODE_CHUNK_BASED = 4, + EROFS_INODE_DATALAYOUT_MAX +}; + +static inline bool erofs_inode_is_data_compressed(unsigned int datamode) +{ + return datamode == EROFS_INODE_COMPRESSED_COMPACT || + datamode == EROFS_INODE_COMPRESSED_FULL; +} + +/* bit definitions of inode i_format */ +#define EROFS_I_VERSION_MASK 0x01 +#define EROFS_I_DATALAYOUT_MASK 0x07 + +#define EROFS_I_VERSION_BIT 0 +#define EROFS_I_DATALAYOUT_BIT 1 +#define EROFS_I_NLINK_1_BIT 4 /* non-directory compact inodes only */ +#define EROFS_I_DOT_OMITTED_BIT 4 /* (directories) omit the `.` dirent */ +#define EROFS_I_ALL ((1 << (EROFS_I_NLINK_1_BIT + 1)) - 1) + +/* indicate chunk blkbits, thus 'chunksize = blocksize << chunk blkbits' */ +#define EROFS_CHUNK_FORMAT_BLKBITS_MASK 0x001F +/* with chunk indexes or just a 4-byte block array */ +#define EROFS_CHUNK_FORMAT_INDEXES 0x0020 +#define EROFS_CHUNK_FORMAT_48BIT 0x0040 + +#define EROFS_CHUNK_FORMAT_ALL ((EROFS_CHUNK_FORMAT_48BIT << 1) - 1) + +/* 32-byte on-disk inode */ +#define EROFS_INODE_LAYOUT_COMPACT 0 +/* 64-byte on-disk inode */ +#define EROFS_INODE_LAYOUT_EXTENDED 1 + +struct erofs_inode_chunk_info { + __le16 format; /* chunk blkbits, etc. */ + __le16 reserved; +}; + +union erofs_inode_i_u { + __le32 blocks_lo; /* total blocks count (if compressed inodes) */ + __le32 startblk_lo; /* starting block number (if flat inodes) */ + __le32 rdev; /* device ID (if special inodes) */ + struct erofs_inode_chunk_info c; +}; + +union erofs_inode_i_nb { + __le16 nlink; /* if EROFS_I_NLINK_1_BIT is unset */ + __le16 blocks_hi; /* total blocks count MSB */ + __le16 startblk_hi; /* starting block number MSB */ +} __packed; + +/* 32-byte reduced form of an ondisk inode */ +struct erofs_inode_compact { + __le16 i_format; /* inode format hints */ + __le16 i_xattr_icount; + __le16 i_mode; + union erofs_inode_i_nb i_nb; + __le32 i_size; + __le32 i_mtime; + union erofs_inode_i_u i_u; + + __le32 i_ino; /* only used for 32-bit stat compatibility */ + __le16 i_uid; + __le16 i_gid; + __le32 i_reserved; +}; + +/* 64-byte complete form of an ondisk inode */ +struct erofs_inode_extended { + __le16 i_format; /* inode format hints */ + __le16 i_xattr_icount; + __le16 i_mode; + union erofs_inode_i_nb i_nb; + __le64 i_size; + union erofs_inode_i_u i_u; + + __le32 i_ino; /* only used for 32-bit stat compatibility */ + __le32 i_uid; + __le32 i_gid; + __le64 i_mtime; + __le32 i_mtime_nsec; + __le32 i_nlink; + __u8 i_reserved2[16]; +}; + +/* + * inline xattrs (n == i_xattr_icount): + * erofs_xattr_ibody_header(1) + (n - 1) * 4 bytes + * 12 bytes / \ + * / \ + * /-----------------------\ + * | erofs_xattr_entries+ | + * +-----------------------+ + * inline xattrs must starts in erofs_xattr_ibody_header, + * for read-only fs, no need to introduce h_refcount + */ +struct erofs_xattr_ibody_header { + __le32 h_name_filter; /* bit value 1 indicates not-present */ + __u8 h_shared_count; + __u8 h_reserved2[7]; + __le32 h_shared_xattrs[]; /* shared xattr id array */ +}; + +/* Name indexes */ +#define EROFS_XATTR_INDEX_USER 1 +#define EROFS_XATTR_INDEX_POSIX_ACL_ACCESS 2 +#define EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT 3 +#define EROFS_XATTR_INDEX_TRUSTED 4 +#define EROFS_XATTR_INDEX_LUSTRE 5 +#define EROFS_XATTR_INDEX_SECURITY 6 + +/* + * bit 7 of e_name_index is set when it refers to a long xattr name prefix, + * while the remained lower bits represent the index of the prefix. + */ +#define EROFS_XATTR_LONG_PREFIX 0x80 +#define EROFS_XATTR_LONG_PREFIX_MASK 0x7f + +#define EROFS_XATTR_FILTER_BITS 32 +#define EROFS_XATTR_FILTER_DEFAULT UINT32_MAX +#define EROFS_XATTR_FILTER_SEED 0x25BBE08F + +/* xattr entry (for both inline & shared xattrs) */ +struct erofs_xattr_entry { + __u8 e_name_len; /* length of name */ + __u8 e_name_index; /* attribute name index */ + __le16 e_value_size; /* size of attribute value */ + /* followed by e_name and e_value */ + char e_name[]; /* attribute name */ +}; + +/* long xattr name prefix */ +struct erofs_xattr_long_prefix { + __u8 base_index; /* short xattr name prefix index */ + char infix[]; /* infix apart from short prefix */ +}; + +static inline unsigned int erofs_xattr_ibody_size(__le16 i_xattr_icount) +{ + if (!i_xattr_icount) + return 0; + + /* 1 header + n-1 * 4 bytes inline xattr to keep continuity */ + return sizeof(struct erofs_xattr_ibody_header) + + sizeof(__u32) * (le16_to_cpu(i_xattr_icount) - 1); +} + +#define EROFS_XATTR_ALIGN(size) round_up(size, sizeof(struct erofs_xattr_entry)) + +static inline unsigned int erofs_xattr_entry_size(struct erofs_xattr_entry *e) +{ + return EROFS_XATTR_ALIGN(sizeof(struct erofs_xattr_entry) + + e->e_name_len + le16_to_cpu(e->e_value_size)); +} + +/* represent a zeroed chunk (hole) */ +#define EROFS_NULL_ADDR -1 + +/* 4-byte block address array */ +#define EROFS_BLOCK_MAP_ENTRY_SIZE sizeof(__le32) + +/* 8-byte inode chunk index */ +struct erofs_inode_chunk_index { + __le16 startblk_hi; /* starting block number MSB */ + __le16 device_id; /* back-end storage id (with bits masked) */ + __le32 startblk_lo; /* starting block number of this chunk */ +}; + +#define EROFS_DIRENT_NID_METABOX_BIT 63 +#define EROFS_DIRENT_NID_MASK (BIT_ULL(EROFS_DIRENT_NID_METABOX_BIT) - 1) + +/* dirent sorts in alphabet order, thus we can do binary search */ +struct erofs_dirent { + __le64 nid; /* node number */ + __le16 nameoff; /* start offset of file name */ + __u8 file_type; /* file type */ + __u8 reserved; /* reserved */ +} __packed; + +/* + * EROFS file types should match generic FT_* types and + * it seems no need to add BUILD_BUG_ONs since potential + * unmatchness will break other fses as well... + */ + +#define EROFS_NAME_LEN 255 + +/* maximum supported encoded size of a physical compressed cluster */ +#define Z_EROFS_PCLUSTER_MAX_SIZE (1024 * 1024) + +/* maximum supported decoded size of a physical compressed cluster */ +#define Z_EROFS_PCLUSTER_MAX_DSIZE (12 * 1024 * 1024) + +/* available compression algorithm types (for h_algorithmtype) */ +enum { + Z_EROFS_COMPRESSION_LZ4 = 0, + Z_EROFS_COMPRESSION_LZMA = 1, + Z_EROFS_COMPRESSION_DEFLATE = 2, + Z_EROFS_COMPRESSION_ZSTD = 3, + Z_EROFS_COMPRESSION_MAX +}; +#define Z_EROFS_ALL_COMPR_ALGS ((1 << Z_EROFS_COMPRESSION_MAX) - 1) + +/* 14 bytes (+ length field = 16 bytes) */ +struct z_erofs_lz4_cfgs { + __le16 max_distance; + __le16 max_pclusterblks; + u8 reserved[10]; +} __packed; + +/* 14 bytes (+ length field = 16 bytes) */ +struct z_erofs_lzma_cfgs { + __le32 dict_size; + __le16 format; + u8 reserved[8]; +} __packed; + +#define Z_EROFS_LZMA_MAX_DICT_SIZE (8 * Z_EROFS_PCLUSTER_MAX_SIZE) + +/* 6 bytes (+ length field = 8 bytes) */ +struct z_erofs_deflate_cfgs { + u8 windowbits; /* 8..15 for DEFLATE */ + u8 reserved[5]; +} __packed; + +/* 6 bytes (+ length field = 8 bytes) */ +struct z_erofs_zstd_cfgs { + u8 format; + u8 windowlog; /* windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN(10) */ + u8 reserved[4]; +} __packed; + +#define Z_EROFS_ZSTD_MAX_DICT_SIZE Z_EROFS_PCLUSTER_MAX_SIZE + +/* + * Enable COMPACTED_2B for EROFS_INODE_COMPRESSED_COMPACT inodes: + * 4B (disabled) vs 4B+2B+4B (enabled) + */ +#define Z_EROFS_ADVISE_COMPACTED_2B 0x0001 +/* Enable extent metadata for EROFS_INODE_COMPRESSED_FULL inodes */ +#define Z_EROFS_ADVISE_EXTENTS 0x0001 +#define Z_EROFS_ADVISE_BIG_PCLUSTER_1 0x0002 +#define Z_EROFS_ADVISE_BIG_PCLUSTER_2 0x0004 +#define Z_EROFS_ADVISE_INLINE_PCLUSTER 0x0008 +#define Z_EROFS_ADVISE_INTERLACED_PCLUSTER 0x0010 +#define Z_EROFS_ADVISE_FRAGMENT_PCLUSTER 0x0020 +/* Indicate the record size for each extent if extent metadata is used */ +#define Z_EROFS_ADVISE_EXTRECSZ_BIT 1 +#define Z_EROFS_ADVISE_EXTRECSZ_MASK 0x3 + +#define Z_EROFS_FRAGMENT_INODE_BIT 7 +struct z_erofs_map_header { + union { + /* fragment data offset in the packed inode */ + __le32 h_fragmentoff; + struct { + __le16 h_reserved1; + /* indicates the encoded size of tailpacking data */ + __le16 h_idata_size; + }; + __le32 h_extents_lo; /* extent count LSB */ + }; + __le16 h_advise; + union { + struct { + /* algorithm type (bit 0-3: HEAD1; bit 4-7: HEAD2) */ + __u8 h_algorithmtype; + /* + * bit 0-3 : logical cluster bits - blkszbits + * bit 4-6 : reserved + * bit 7 : pack the whole file into packed inode + */ + __u8 h_clusterbits; + } __packed; + __le16 h_extents_hi; /* extent count MSB */ + } __packed; +}; + +enum { + Z_EROFS_LCLUSTER_TYPE_PLAIN = 0, + Z_EROFS_LCLUSTER_TYPE_HEAD1 = 1, + Z_EROFS_LCLUSTER_TYPE_NONHEAD = 2, + Z_EROFS_LCLUSTER_TYPE_HEAD2 = 3, + Z_EROFS_LCLUSTER_TYPE_MAX +}; + +#define Z_EROFS_LI_LCLUSTER_TYPE_MASK (Z_EROFS_LCLUSTER_TYPE_MAX - 1) + +/* (noncompact only, HEAD) This pcluster refers to partial decompressed data */ +#define Z_EROFS_LI_PARTIAL_REF (1 << 15) +/* (noncompact only, HEAD) This pcluster can also be regarded as a HOLE */ +#define Z_EROFS_LI_HOLE (1 << 14) + +/* Set on 1st non-head lcluster to store compressed block counti (in blocks) */ +#define Z_EROFS_LI_D0_CBLKCNT (1 << 11) + +struct z_erofs_lcluster_index { + __le16 di_advise; + /* where to decompress in the head lcluster */ + __le16 di_clusterofs; + + union { + __le32 blkaddr; /* for the HEAD lclusters */ + /* + * [0] - distance to its HEAD lcluster + * [1] - distance to the next HEAD lcluster + */ + __le16 delta[2]; /* for the NONHEAD lclusters */ + } di_u; +}; + +#define Z_EROFS_MAP_HEADER_END(end) \ + (ALIGN(end, 8) + sizeof(struct z_erofs_map_header)) +#define Z_EROFS_FULL_INDEX_START(end) (Z_EROFS_MAP_HEADER_END(end) + 8) + +#define Z_EROFS_EXTENT_PLEN_PARTIAL BIT(27) +#define Z_EROFS_EXTENT_PLEN_FMT_BIT 28 +#define Z_EROFS_EXTENT_PLEN_MASK ((Z_EROFS_PCLUSTER_MAX_SIZE << 1) - 1) +struct z_erofs_extent { + __le32 plen; /* encoded length */ + __le32 pstart_lo; /* physical offset */ + __le32 pstart_hi; /* physical offset MSB */ + __le32 lstart_lo; /* logical offset */ + __le32 lstart_hi; /* logical offset MSB (>= 4GiB inodes) */ + __u8 reserved[12]; /* for future use */ +}; + +static inline int z_erofs_extent_recsize(unsigned int advise) +{ + return 4 << ((advise >> Z_EROFS_ADVISE_EXTRECSZ_BIT) & + Z_EROFS_ADVISE_EXTRECSZ_MASK); +} + +/* check the EROFS on-disk layout strictly at compile time */ +static inline void erofs_check_ondisk_layout_definitions(void) +{ + const __le64 fmh = *(__le64 *)&(struct z_erofs_map_header) { + .h_clusterbits = 1 << Z_EROFS_FRAGMENT_INODE_BIT + }; + + BUILD_BUG_ON(sizeof(struct erofs_super_block) != 144); + BUILD_BUG_ON(sizeof(struct erofs_inode_compact) != 32); + BUILD_BUG_ON(sizeof(struct erofs_inode_extended) != 64); + BUILD_BUG_ON(sizeof(struct erofs_xattr_ibody_header) != 12); + BUILD_BUG_ON(sizeof(struct erofs_xattr_entry) != 4); + BUILD_BUG_ON(sizeof(struct erofs_inode_chunk_info) != 4); + BUILD_BUG_ON(sizeof(struct erofs_inode_chunk_index) != 8); + BUILD_BUG_ON(sizeof(struct z_erofs_map_header) != 8); + BUILD_BUG_ON(sizeof(struct z_erofs_lcluster_index) != 8); + BUILD_BUG_ON(sizeof(struct erofs_dirent) != 12); + /* keep in sync between 2 index structures for better extendibility */ + BUILD_BUG_ON(sizeof(struct erofs_inode_chunk_index) != + sizeof(struct z_erofs_lcluster_index)); + BUILD_BUG_ON(sizeof(struct erofs_deviceslot) != 128); + + /* exclude old compiler versions like gcc 7.5.0 */ + BUILD_BUG_ON(__builtin_constant_p(fmh) ? + fmh != cpu_to_le64(1ULL << 63) : 0); +} + +#endif diff --git a/fs/erofs/fileio.c b/fs/erofs/fileio.c new file mode 100644 index 000000000..ebb81a7ff --- /dev/null +++ b/fs/erofs/fileio.c @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2024, Alibaba Cloud + */ +#include "internal.h" +#include + +struct erofs_fileio_rq { + struct bio_vec bvecs[16]; + struct bio bio; + struct kiocb iocb; + struct super_block *sb; + refcount_t ref; +}; + +struct erofs_fileio { + struct erofs_map_blocks map; + struct erofs_map_dev dev; + struct erofs_fileio_rq *rq; +}; + +static void erofs_fileio_ki_complete(struct kiocb *iocb, long ret) +{ + struct erofs_fileio_rq *rq = + container_of(iocb, struct erofs_fileio_rq, iocb); + struct folio_iter fi; + + if (ret >= 0 && ret != rq->bio.bi_iter.bi_size) + ret = -EIO; + if (!rq->bio.bi_end_io) { + bio_for_each_folio_all(fi, &rq->bio) { + DBG_BUGON(folio_test_uptodate(fi.folio)); + erofs_onlinefolio_end(fi.folio, ret < 0, false); + } + } else if (ret < 0 && !rq->bio.bi_status) { + rq->bio.bi_status = errno_to_blk_status(ret); + } + bio_endio(&rq->bio); + bio_uninit(&rq->bio); + if (refcount_dec_and_test(&rq->ref)) + kfree(rq); +} + +static void erofs_fileio_rq_submit(struct erofs_fileio_rq *rq) +{ + struct iov_iter iter; + ssize_t ret; + + if (!rq) + return; + rq->iocb.ki_pos = rq->bio.bi_iter.bi_sector << SECTOR_SHIFT; + rq->iocb.ki_ioprio = get_current_ioprio(); + rq->iocb.ki_complete = erofs_fileio_ki_complete; + if (test_opt(&EROFS_SB(rq->sb)->opt, DIRECT_IO) && + rq->iocb.ki_filp->f_mode & FMODE_CAN_ODIRECT) + rq->iocb.ki_flags = IOCB_DIRECT; + iov_iter_bvec(&iter, ITER_DEST, rq->bvecs, rq->bio.bi_vcnt, + rq->bio.bi_iter.bi_size); + scoped_with_creds(rq->iocb.ki_filp->f_cred) + ret = vfs_iocb_iter_read(rq->iocb.ki_filp, &rq->iocb, &iter); + if (ret != -EIOCBQUEUED) + erofs_fileio_ki_complete(&rq->iocb, ret); + if (refcount_dec_and_test(&rq->ref)) + kfree(rq); +} + +static struct erofs_fileio_rq *erofs_fileio_rq_alloc(struct erofs_map_dev *mdev) +{ + struct erofs_fileio_rq *rq = kzalloc_obj(*rq, GFP_KERNEL | __GFP_NOFAIL); + + bio_init(&rq->bio, NULL, rq->bvecs, ARRAY_SIZE(rq->bvecs), REQ_OP_READ); + rq->iocb.ki_filp = mdev->m_dif->file; + rq->sb = mdev->m_sb; + refcount_set(&rq->ref, 2); + return rq; +} + +struct bio *erofs_fileio_bio_alloc(struct erofs_map_dev *mdev) +{ + return &erofs_fileio_rq_alloc(mdev)->bio; +} + +void erofs_fileio_submit_bio(struct bio *bio) +{ + return erofs_fileio_rq_submit(container_of(bio, struct erofs_fileio_rq, + bio)); +} + +static int erofs_fileio_scan_folio(struct erofs_fileio *io, + struct inode *inode, struct folio *folio) +{ + struct erofs_map_blocks *map = &io->map; + unsigned int cur = 0, end = folio_size(folio), len, attached = 0; + loff_t pos = folio_pos(folio), ofs; + int err = 0; + + erofs_onlinefolio_init(folio); + while (cur < end) { + if (!in_range(pos + cur, map->m_la, map->m_llen)) { + map->m_la = pos + cur; + map->m_llen = end - cur; + err = erofs_map_blocks(inode, map); + if (err) + break; + } + + ofs = folio_pos(folio) + cur - map->m_la; + len = min_t(loff_t, map->m_llen - ofs, end - cur); + if (map->m_flags & EROFS_MAP_META) { + struct erofs_buf buf = __EROFS_BUF_INITIALIZER; + void *src; + + src = erofs_read_metabuf(&buf, inode->i_sb, + map->m_pa + ofs, erofs_inode_in_metabox(inode)); + if (IS_ERR(src)) { + err = PTR_ERR(src); + break; + } + memcpy_to_folio(folio, cur, src, len); + erofs_put_metabuf(&buf); + } else if (!(map->m_flags & EROFS_MAP_MAPPED)) { + folio_zero_segment(folio, cur, cur + len); + attached = 0; + } else { + if (io->rq && (map->m_pa + ofs != io->dev.m_pa || + map->m_deviceid != io->dev.m_deviceid)) { +io_retry: + erofs_fileio_rq_submit(io->rq); + io->rq = NULL; + } + + if (!io->rq) { + io->dev = (struct erofs_map_dev) { + .m_pa = io->map.m_pa + ofs, + .m_deviceid = io->map.m_deviceid, + }; + err = erofs_map_dev(inode->i_sb, &io->dev); + if (err) + break; + io->rq = erofs_fileio_rq_alloc(&io->dev); + io->rq->bio.bi_iter.bi_sector = + (io->dev.m_dif->fsoff + io->dev.m_pa) >> 9; + attached = 0; + } + if (!bio_add_folio(&io->rq->bio, folio, len, cur)) + goto io_retry; + if (!attached++) + erofs_onlinefolio_split(folio); + io->dev.m_pa += len; + } + cur += len; + } + erofs_onlinefolio_end(folio, err, false); + return err; +} + +static int erofs_fileio_read_folio(struct file *file, struct folio *folio) +{ + bool need_iput; + struct inode *realinode = erofs_real_inode(folio_inode(folio), &need_iput); + struct erofs_fileio io = {}; + int err; + + trace_erofs_read_folio(realinode, folio, true); + err = erofs_fileio_scan_folio(&io, realinode, folio); + erofs_fileio_rq_submit(io.rq); + if (need_iput) + iput(realinode); + return err; +} + +static void erofs_fileio_readahead(struct readahead_control *rac) +{ + bool need_iput; + struct inode *realinode = erofs_real_inode(rac->mapping->host, &need_iput); + struct erofs_fileio io = {}; + struct folio *folio; + int err; + + trace_erofs_readahead(realinode, readahead_index(rac), + readahead_count(rac), true); + while ((folio = readahead_folio(rac))) { + err = erofs_fileio_scan_folio(&io, realinode, folio); + if (err && err != -EINTR) + erofs_err(realinode->i_sb, "readahead error at folio %lu @ nid %llu", + folio->index, EROFS_I(realinode)->nid); + } + erofs_fileio_rq_submit(io.rq); + if (need_iput) + iput(realinode); +} + +const struct address_space_operations erofs_fileio_aops = { + .read_folio = erofs_fileio_read_folio, + .readahead = erofs_fileio_readahead, +}; + +int erofs_read_meta_folio(struct file *file, struct folio *folio) +{ + struct erofs_fileio io = { + .dev = { .m_pa = folio_pos(folio), }, + }; + struct inode *inode = folio_inode(folio); + int err; + + err = erofs_map_dev(inode->i_sb, &io.dev); + if (err) + return err; + + io.rq = erofs_fileio_rq_alloc(&io.dev); + io.rq->bio.bi_iter.bi_sector = + (io.dev.m_dif->fsoff + io.dev.m_pa) >> 9; + erofs_onlinefolio_init(folio); + bio_add_folio_nofail(&io.rq->bio, folio, folio_size(folio), 0); + erofs_fileio_rq_submit(io.rq); + return 0; +} diff --git a/fs/erofs/inode.c b/fs/erofs/inode.c new file mode 100644 index 000000000..45afe5c50 --- /dev/null +++ b/fs/erofs/inode.c @@ -0,0 +1,399 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021, Alibaba Cloud + */ +#include "xattr.h" +#include +#include + +static int erofs_fill_symlink(struct inode *inode, void *bptr, unsigned int ofs) +{ + struct erofs_inode *vi = EROFS_I(inode); + char *link; + loff_t end; + + ofs += vi->xattr_isize; + /* check whether the symlink data is small enough to be inlined */ + if (vi->datalayout == EROFS_INODE_FLAT_INLINE && + !check_add_overflow(ofs, inode->i_size, &end) && + end <= i_blocksize(inode)) { + link = kmemdup_nul(bptr + ofs, inode->i_size, GFP_KERNEL); + if (!link) + return -ENOMEM; + if (unlikely(!inode->i_size || strlen(link) != inode->i_size)) { + erofs_err(inode->i_sb, "invalid fast symlink size %llu @ nid %llu", + inode->i_size | 0ULL, vi->nid); + kfree(link); + return -EFSCORRUPTED; + } + inode_set_cached_link(inode, link, inode->i_size); + } + return 0; +} + +static int erofs_read_inode(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + erofs_blk_t blkaddr = erofs_blknr(sb, erofs_iloc(inode)); + unsigned int ofs = erofs_blkoff(sb, erofs_iloc(inode)); + bool in_mbox = erofs_inode_in_metabox(inode); + struct erofs_buf buf = __EROFS_BUF_INITIALIZER; + struct erofs_sb_info *sbi = EROFS_SB(sb); + erofs_blk_t addrmask = BIT_ULL(48) - 1; + struct erofs_inode *vi = EROFS_I(inode); + struct erofs_inode_extended *die, copied; + struct erofs_inode_compact *dic; + unsigned int ifmt; + void *ptr; + int err = 0; + + ptr = erofs_read_metabuf(&buf, sb, erofs_pos(sb, blkaddr), in_mbox); + if (IS_ERR(ptr)) { + err = PTR_ERR(ptr); + erofs_err(sb, "failed to read inode meta block (nid: %llu): %d", + vi->nid, err); + goto err_out; + } + + dic = ptr + ofs; + ifmt = le16_to_cpu(dic->i_format); + if (ifmt & ~EROFS_I_ALL) { + erofs_err(sb, "unsupported i_format %u of nid %llu", + ifmt, vi->nid); + err = -EOPNOTSUPP; + goto err_out; + } + + vi->datalayout = erofs_inode_datalayout(ifmt); + if (vi->datalayout >= EROFS_INODE_DATALAYOUT_MAX) { + erofs_err(sb, "unsupported datalayout %u of nid %llu", + vi->datalayout, vi->nid); + err = -EOPNOTSUPP; + goto err_out; + } + + switch (erofs_inode_version(ifmt)) { + case EROFS_INODE_LAYOUT_EXTENDED: + vi->inode_isize = sizeof(struct erofs_inode_extended); + /* check if the extended inode acrosses block boundary */ + if (ofs + vi->inode_isize <= sb->s_blocksize) { + ofs += vi->inode_isize; + die = (struct erofs_inode_extended *)dic; + copied.i_u = die->i_u; + copied.i_nb = die->i_nb; + } else { + const unsigned int gotten = sb->s_blocksize - ofs; + + memcpy(&copied, dic, gotten); + ptr = erofs_read_metabuf(&buf, sb, + erofs_pos(sb, blkaddr + 1), in_mbox); + if (IS_ERR(ptr)) { + err = PTR_ERR(ptr); + erofs_err(sb, "failed to read inode payload block (nid: %llu): %d", + vi->nid, err); + goto err_out; + } + ofs = vi->inode_isize - gotten; + memcpy((u8 *)&copied + gotten, ptr, ofs); + die = &copied; + } + vi->xattr_isize = erofs_xattr_ibody_size(die->i_xattr_icount); + + inode->i_mode = le16_to_cpu(die->i_mode); + i_uid_write(inode, le32_to_cpu(die->i_uid)); + i_gid_write(inode, le32_to_cpu(die->i_gid)); + set_nlink(inode, le32_to_cpu(die->i_nlink)); + inode_set_mtime(inode, le64_to_cpu(die->i_mtime), + le32_to_cpu(die->i_mtime_nsec)); + + inode->i_size = le64_to_cpu(die->i_size); + break; + case EROFS_INODE_LAYOUT_COMPACT: + vi->inode_isize = sizeof(struct erofs_inode_compact); + ofs += vi->inode_isize; + vi->xattr_isize = erofs_xattr_ibody_size(dic->i_xattr_icount); + + inode->i_mode = le16_to_cpu(dic->i_mode); + copied.i_u = dic->i_u; + i_uid_write(inode, le16_to_cpu(dic->i_uid)); + i_gid_write(inode, le16_to_cpu(dic->i_gid)); + if (!S_ISDIR(inode->i_mode) && + ((ifmt >> EROFS_I_NLINK_1_BIT) & 1)) { + set_nlink(inode, 1); + copied.i_nb = dic->i_nb; + } else { + set_nlink(inode, le16_to_cpu(dic->i_nb.nlink)); + copied.i_nb.startblk_hi = 0; + addrmask = BIT_ULL(32) - 1; + } + inode_set_mtime(inode, sbi->epoch + le32_to_cpu(dic->i_mtime), + sbi->fixed_nsec); + + inode->i_size = le32_to_cpu(dic->i_size); + break; + default: + erofs_err(sb, "unsupported on-disk inode version %u of nid %llu", + erofs_inode_version(ifmt), vi->nid); + err = -EOPNOTSUPP; + goto err_out; + } + + if (unlikely(inode->i_size < 0)) { + erofs_err(sb, "negative i_size @ nid %llu", vi->nid); + err = -EFSCORRUPTED; + goto err_out; + } + + if (IS_ENABLED(CONFIG_EROFS_FS_POSIX_ACL) && + erofs_inode_has_noacl(inode, ptr, ofs)) + cache_no_acl(inode); + + switch (inode->i_mode & S_IFMT) { + case S_IFDIR: + vi->dot_omitted = (ifmt >> EROFS_I_DOT_OMITTED_BIT) & 1; + fallthrough; + case S_IFREG: + case S_IFLNK: + vi->startblk = le32_to_cpu(copied.i_u.startblk_lo) | + ((u64)le16_to_cpu(copied.i_nb.startblk_hi) << 32); + if (vi->datalayout == EROFS_INODE_FLAT_PLAIN && + !((vi->startblk ^ EROFS_NULL_ADDR) & addrmask)) + vi->startblk = EROFS_NULL_ADDR; + + if(S_ISLNK(inode->i_mode)) { + err = erofs_fill_symlink(inode, ptr, ofs); + if (err) + goto err_out; + } + break; + case S_IFCHR: + case S_IFBLK: + inode->i_rdev = new_decode_dev(le32_to_cpu(copied.i_u.rdev)); + break; + case S_IFIFO: + case S_IFSOCK: + inode->i_rdev = 0; + break; + default: + erofs_err(sb, "bogus i_mode (%o) @ nid %llu", inode->i_mode, + vi->nid); + err = -EFSCORRUPTED; + goto err_out; + } + + if (!erofs_inode_is_data_compressed(vi->datalayout)) { + inode->i_blocks = round_up(inode->i_size, sb->s_blocksize) >> 9; + } else if (!IS_ENABLED(CONFIG_EROFS_FS_ZIP) || !sbi->available_compr_algs) { + erofs_err(sb, "compressed inode (nid %llu) is invalid in a plain filesystem", + vi->nid); + err = -EFSCORRUPTED; + goto err_out; + } else { + inode->i_blocks = (le32_to_cpu(copied.i_u.blocks_lo) | + ((u64)le16_to_cpu(copied.i_nb.blocks_hi) << 32)) << + (sb->s_blocksize_bits - 9); + } + + if (vi->datalayout == EROFS_INODE_CHUNK_BASED) { + /* fill chunked inode summary info */ + vi->chunkformat = le16_to_cpu(copied.i_u.c.format); + if (vi->chunkformat & ~EROFS_CHUNK_FORMAT_ALL) { + erofs_err(sb, "unsupported chunk format %x of nid %llu", + vi->chunkformat, vi->nid); + err = -EOPNOTSUPP; + goto err_out; + } + vi->chunkbits = sb->s_blocksize_bits + + (vi->chunkformat & EROFS_CHUNK_FORMAT_BLKBITS_MASK); + } + inode_set_atime_to_ts(inode, + inode_set_ctime_to_ts(inode, inode_get_mtime(inode))); + + inode->i_flags &= ~S_DAX; + if (test_opt(&sbi->opt, DAX_ALWAYS) && S_ISREG(inode->i_mode) && + (vi->datalayout == EROFS_INODE_FLAT_PLAIN || + vi->datalayout == EROFS_INODE_CHUNK_BASED)) + inode->i_flags |= S_DAX; +err_out: + erofs_put_metabuf(&buf); + return err; +} + +static int erofs_fill_inode(struct inode *inode) +{ + const struct address_space_operations *aops; + int err; + + trace_erofs_fill_inode(inode); + err = erofs_read_inode(inode); + if (err) + return err; + + switch (inode->i_mode & S_IFMT) { + case S_IFREG: + inode->i_op = &erofs_generic_iops; + inode->i_fop = erofs_ishare_fill_inode(inode) ? + &erofs_ishare_fops : &erofs_file_fops; + break; + case S_IFDIR: + inode->i_op = &erofs_dir_iops; + inode->i_fop = &erofs_dir_fops; + inode_nohighmem(inode); + break; + case S_IFLNK: + if (inode->i_link) + inode->i_op = &erofs_fast_symlink_iops; + else + inode->i_op = &erofs_symlink_iops; + inode_nohighmem(inode); + break; + default: + inode->i_op = &erofs_generic_iops; + init_special_inode(inode, inode->i_mode, inode->i_rdev); + return 0; + } + + mapping_set_large_folios(inode->i_mapping); + aops = erofs_get_aops(inode); + if (IS_ERR(aops)) + return PTR_ERR(aops); + inode->i_mapping->a_ops = aops; + return 0; +} + +/* + * ino_t is 32-bits on 32-bit arch. We have to squash the 64-bit value down + * so that it will fit. + */ +static ino_t erofs_squash_ino(struct super_block *sb, erofs_nid_t nid) +{ + u64 ino64 = erofs_nid_to_ino64(EROFS_SB(sb), nid); + + if (sizeof(ino_t) < sizeof(erofs_nid_t)) + ino64 ^= ino64 >> (sizeof(erofs_nid_t) - sizeof(ino_t)) * 8; + return (ino_t)ino64; +} + +static int erofs_iget5_eq(struct inode *inode, void *opaque) +{ + return EROFS_I(inode)->nid == *(erofs_nid_t *)opaque; +} + +static int erofs_iget5_set(struct inode *inode, void *opaque) +{ + const erofs_nid_t nid = *(erofs_nid_t *)opaque; + + inode->i_ino = erofs_squash_ino(inode->i_sb, nid); + EROFS_I(inode)->nid = nid; + return 0; +} + +struct inode *erofs_iget(struct super_block *sb, erofs_nid_t nid) +{ + struct inode *inode; + + inode = iget5_locked(sb, erofs_squash_ino(sb, nid), erofs_iget5_eq, + erofs_iget5_set, &nid); + if (!inode) + return ERR_PTR(-ENOMEM); + + if (inode_state_read_once(inode) & I_NEW) { + int err = erofs_fill_inode(inode); + + if (err) { + iget_failed(inode); + return ERR_PTR(err); + } + unlock_new_inode(inode); + } + return inode; +} + +int erofs_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, + unsigned int query_flags) +{ + struct inode *const inode = d_inode(path->dentry); + struct block_device *bdev = inode->i_sb->s_bdev; + bool compressed = + erofs_inode_is_data_compressed(EROFS_I(inode)->datalayout); + + if (compressed) + stat->attributes |= STATX_ATTR_COMPRESSED; + stat->attributes |= STATX_ATTR_IMMUTABLE; + stat->attributes_mask |= (STATX_ATTR_COMPRESSED | + STATX_ATTR_IMMUTABLE); + + /* + * Return the DIO alignment restrictions if requested. + * + * In EROFS, STATX_DIOALIGN is only supported in bdev-based mode + * and uncompressed inodes, otherwise we report no DIO support. + */ + if ((request_mask & STATX_DIOALIGN) && S_ISREG(inode->i_mode)) { + stat->result_mask |= STATX_DIOALIGN; + if (bdev && !compressed) { + stat->dio_mem_align = bdev_dma_alignment(bdev) + 1; + stat->dio_offset_align = bdev_logical_block_size(bdev); + } + } + generic_fillattr(idmap, request_mask, inode, stat); + return 0; +} + +static int erofs_ioctl_get_volume_label(struct inode *inode, void __user *arg) +{ + struct erofs_sb_info *sbi = EROFS_I_SB(inode); + int ret; + + if (!sbi->volume_name) + ret = clear_user(arg, 1); + else + ret = copy_to_user(arg, sbi->volume_name, + strlen(sbi->volume_name) + 1); + return ret ? -EFAULT : 0; +} + +long erofs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + void __user *argp = (void __user *)arg; + + switch (cmd) { + case FS_IOC_GETFSLABEL: + return erofs_ioctl_get_volume_label(inode, argp); + default: + return -ENOTTY; + } +} + +#ifdef CONFIG_COMPAT +long erofs_compat_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg) +{ + return erofs_ioctl(filp, cmd, (unsigned long)compat_ptr(arg)); +} +#endif + +const struct inode_operations erofs_generic_iops = { + .getattr = erofs_getattr, + .listxattr = erofs_listxattr, + .get_inode_acl = erofs_get_acl, + .fiemap = erofs_fiemap, +}; + +const struct inode_operations erofs_symlink_iops = { + .get_link = page_get_link, + .getattr = erofs_getattr, + .listxattr = erofs_listxattr, + .get_inode_acl = erofs_get_acl, +}; + +const struct inode_operations erofs_fast_symlink_iops = { + .get_link = simple_get_link, + .getattr = erofs_getattr, + .listxattr = erofs_listxattr, + .get_inode_acl = erofs_get_acl, +}; diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h new file mode 100644 index 000000000..9b7370f0f --- /dev/null +++ b/fs/erofs/internal.h @@ -0,0 +1,539 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021, Alibaba Cloud + */ +#ifndef __EROFS_INTERNAL_H +#define __EROFS_INTERNAL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "erofs_fs.h" + +__printf(2, 3) void _erofs_printk(struct super_block *sb, const char *fmt, ...); +#define erofs_err(sb, fmt, ...) \ + _erofs_printk(sb, KERN_ERR fmt "\n", ##__VA_ARGS__) +#define erofs_warn(sb, fmt, ...) \ + _erofs_printk(sb, KERN_WARNING fmt "\n", ##__VA_ARGS__) +#define erofs_info(sb, fmt, ...) \ + _erofs_printk(sb, KERN_INFO fmt "\n", ##__VA_ARGS__) + +#ifdef CONFIG_EROFS_FS_DEBUG +#define DBG_BUGON BUG_ON +#else +#define DBG_BUGON(x) ((void)(x)) +#endif /* !CONFIG_EROFS_FS_DEBUG */ + +/* EROFS_SUPER_MAGIC_V1 to represent the whole file system */ +#define EROFS_SUPER_MAGIC EROFS_SUPER_MAGIC_V1 + +typedef u64 erofs_nid_t; +typedef u64 erofs_off_t; +typedef u64 erofs_blk_t; + +struct erofs_device_info { + char *path; + struct file *file; + struct dax_device *dax_dev; + u64 fsoff, dax_part_off; + + erofs_blk_t blocks; + erofs_blk_t uniaddr; +}; + +enum { + EROFS_SYNC_DECOMPRESS_AUTO, + EROFS_SYNC_DECOMPRESS_FORCE_ON, + EROFS_SYNC_DECOMPRESS_FORCE_OFF +}; + +struct erofs_mount_opts { + /* current strategy of how to use managed cache */ + unsigned char cache_strategy; + unsigned int mount_opt; +}; + +struct erofs_dev_context { + struct idr tree; + struct rw_semaphore rwsem; + + unsigned int extra_devices; + bool flatdev; +}; + +/* all filesystem-wide lz4 configurations */ +struct erofs_sb_lz4_info { + /* # of pages needed for EROFS lz4 rolling decompression */ + u16 max_distance_pages; + /* maximum possible blocks for pclusters in the filesystem */ + u16 max_pclusterblks; +}; + +struct erofs_xattr_prefix_item { + struct erofs_xattr_long_prefix *prefix; + u8 infix_len; +}; + +struct erofs_sb_info { + struct erofs_device_info dif0; + struct erofs_mount_opts opt; /* options */ +#ifdef CONFIG_EROFS_FS_ZIP + /* list for all registered superblocks, mainly for shrinker */ + struct list_head list; + struct mutex umount_mutex; + + /* managed XArray arranged in physical block number */ + struct xarray managed_pslots; + + unsigned int sync_decompress; /* strategy for sync decompression */ + unsigned int shrinker_run_no; + + struct erofs_sb_lz4_info lz4; +#endif /* CONFIG_EROFS_FS_ZIP */ + struct inode *managed_cache; /* pseudo inode to cache physical data */ + struct inode *packed_inode; + struct inode *metabox_inode; + struct erofs_dev_context *devs; + u64 total_blocks; + + u32 meta_blkaddr; +#ifdef CONFIG_EROFS_FS_XATTR + u32 xattr_blkaddr; + u32 xattr_prefix_start; + u8 xattr_prefix_count; + u8 ishare_xattr_prefix_id; + struct erofs_xattr_prefix_item *xattr_prefixes; + unsigned int xattr_filter_reserved; +#endif + u16 device_id_mask; /* valid bits of device id to be used */ + + unsigned char islotbits; /* inode slot unit size in bit shift */ + unsigned char blkszbits; /* filesystem block size in bit shift */ + + u32 sb_size; /* total superblock size */ + u32 fixed_nsec; + s64 epoch; + + /* what we really care is nid, rather than ino.. */ + erofs_nid_t root_nid; + erofs_nid_t packed_nid; + erofs_nid_t metabox_nid; + /* used for statfs, f_files - f_favail */ + u64 inos; + + char *volume_name; + u32 feature_compat; + u32 feature_incompat; + u16 available_compr_algs; + + /* sysfs support */ + struct kobject s_kobj; /* /sys/fs/erofs/ */ + struct completion s_kobj_unregister; + erofs_off_t dir_ra_bytes; + + char *domain_id; +}; + +#define EROFS_SB(sb) ((struct erofs_sb_info *)(sb)->s_fs_info) +#define EROFS_I_SB(inode) ((struct erofs_sb_info *)(inode)->i_sb->s_fs_info) + +/* Mount flags set via mount options or defaults */ +#define EROFS_MOUNT_XATTR_USER 0x00000010 +#define EROFS_MOUNT_POSIX_ACL 0x00000020 +#define EROFS_MOUNT_DAX_ALWAYS 0x00000040 +#define EROFS_MOUNT_DAX_NEVER 0x00000080 +#define EROFS_MOUNT_DIRECT_IO 0x00000100 +#define EROFS_MOUNT_INODE_SHARE 0x00000200 + +#define clear_opt(opt, option) ((opt)->mount_opt &= ~EROFS_MOUNT_##option) +#define set_opt(opt, option) ((opt)->mount_opt |= EROFS_MOUNT_##option) +#define test_opt(opt, option) ((opt)->mount_opt & EROFS_MOUNT_##option) + +static inline bool erofs_is_fileio_mode(struct erofs_sb_info *sbi) +{ + return IS_ENABLED(CONFIG_EROFS_FS_BACKED_BY_FILE) && sbi->dif0.file; +} + +extern struct file_system_type erofs_anon_fs_type; + +enum { + EROFS_ZIP_CACHE_DISABLED, + EROFS_ZIP_CACHE_READAHEAD, + EROFS_ZIP_CACHE_READAROUND +}; + +struct erofs_buf { + struct address_space *mapping; + u64 off; + struct page *page; + void *base; + bool mc; +}; +#define __EROFS_BUF_INITIALIZER ((struct erofs_buf){ .page = NULL }) + +#define erofs_blknr(sb, pos) ((erofs_blk_t)((pos) >> (sb)->s_blocksize_bits)) +#define erofs_blkoff(sb, pos) ((pos) & ((sb)->s_blocksize - 1)) +#define erofs_pos(sb, blk) ((erofs_off_t)(blk) << (sb)->s_blocksize_bits) +#define erofs_iblks(i) (round_up((i)->i_size, i_blocksize(i)) >> (i)->i_blkbits) + +#define EROFS_FEATURE_FUNCS(name, compat, feature) \ +static inline bool erofs_sb_has_##name(struct erofs_sb_info *sbi) \ +{ \ + return sbi->feature_##compat & EROFS_FEATURE_##feature; \ +} + +EROFS_FEATURE_FUNCS(lz4_0padding, incompat, INCOMPAT_LZ4_0PADDING) +EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS) +EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER) +EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE) +EROFS_FEATURE_FUNCS(device_table, incompat, INCOMPAT_DEVICE_TABLE) +EROFS_FEATURE_FUNCS(compr_head2, incompat, INCOMPAT_COMPR_HEAD2) +EROFS_FEATURE_FUNCS(ztailpacking, incompat, INCOMPAT_ZTAILPACKING) +EROFS_FEATURE_FUNCS(fragments, incompat, INCOMPAT_FRAGMENTS) +EROFS_FEATURE_FUNCS(dedupe, incompat, INCOMPAT_DEDUPE) +EROFS_FEATURE_FUNCS(xattr_prefixes, incompat, INCOMPAT_XATTR_PREFIXES) +EROFS_FEATURE_FUNCS(48bit, incompat, INCOMPAT_48BIT) +EROFS_FEATURE_FUNCS(metabox, incompat, INCOMPAT_METABOX) +EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM) +EROFS_FEATURE_FUNCS(xattr_filter, compat, COMPAT_XATTR_FILTER) +EROFS_FEATURE_FUNCS(shared_ea_in_metabox, compat, COMPAT_SHARED_EA_IN_METABOX) +EROFS_FEATURE_FUNCS(plain_xattr_pfx, compat, COMPAT_PLAIN_XATTR_PFX) +EROFS_FEATURE_FUNCS(ishare_xattrs, compat, COMPAT_ISHARE_XATTRS) + +static inline u64 erofs_nid_to_ino64(struct erofs_sb_info *sbi, erofs_nid_t nid) +{ + if (!erofs_sb_has_metabox(sbi)) + return nid; + + /* + * When metadata compression is enabled, avoid generating excessively + * large inode numbers for metadata-compressed inodes. Shift NIDs in + * the 31-62 bit range left by one and move the metabox flag to bit 31. + * + * Note: on-disk NIDs remain unchanged as they are primarily used for + * compatibility with non-LFS 32-bit applications. + */ + return ((nid << 1) & GENMASK_ULL(63, 32)) | (nid & GENMASK(30, 0)) | + ((nid >> EROFS_DIRENT_NID_METABOX_BIT) << 31); +} + +/* atomic flag definitions */ +#define EROFS_I_EA_INITED_BIT 0 +#define EROFS_I_Z_INITED_BIT 1 + +/* bitlock definitions (arranged in reverse order) */ +#define EROFS_I_BL_XATTR_BIT (BITS_PER_LONG - 1) +#define EROFS_I_BL_Z_BIT (BITS_PER_LONG - 2) + +/* default readahead size of directories */ +#define EROFS_DIR_RA_BYTES 16384 + +struct erofs_inode_fingerprint { + u8 *opaque; + int size; +}; + +struct erofs_inode { + erofs_nid_t nid; + + /* atomic flags (including bitlocks) */ + unsigned long flags; + + unsigned char datalayout; + unsigned char inode_isize; + bool dot_omitted; + unsigned int xattr_isize; + + unsigned int xattr_name_filter; + unsigned int xattr_shared_count; + unsigned int *xattr_shared_xattrs; + + union { + erofs_blk_t startblk; + struct { + unsigned short chunkformat; + unsigned char chunkbits; + }; +#ifdef CONFIG_EROFS_FS_ZIP + struct { + unsigned short z_advise; + unsigned char z_algofmt[2]; + unsigned char z_lclusterbits; + union { + u64 z_tailextent_headlcn; + u64 z_extents; + }; + erofs_off_t z_fragmentoff; + unsigned short z_idata_size; + }; +#endif /* CONFIG_EROFS_FS_ZIP */ + }; +#ifdef CONFIG_EROFS_FS_PAGE_CACHE_SHARE + struct list_head ishare_list; + union { + /* for each anon shared inode */ + struct { + struct erofs_inode_fingerprint fingerprint; + spinlock_t ishare_lock; + }; + /* for each real filesystem inode */ + struct dentry *sharedentry; + }; +#endif + /* the corresponding vfs inode */ + struct inode vfs_inode; +}; + +#define EROFS_I(ptr) container_of(ptr, struct erofs_inode, vfs_inode) + +static inline bool erofs_inode_in_metabox(struct inode *inode) +{ + return EROFS_I(inode)->nid & BIT_ULL(EROFS_DIRENT_NID_METABOX_BIT); +} + +static inline erofs_off_t erofs_iloc(struct inode *inode) +{ + struct erofs_sb_info *sbi = EROFS_I_SB(inode); + erofs_nid_t nid_lo = EROFS_I(inode)->nid & EROFS_DIRENT_NID_MASK; + + if (erofs_inode_in_metabox(inode)) + return nid_lo << sbi->islotbits; + return erofs_pos(inode->i_sb, sbi->meta_blkaddr) + + (nid_lo << sbi->islotbits); +} + +static inline unsigned int erofs_inode_version(unsigned int ifmt) +{ + return (ifmt >> EROFS_I_VERSION_BIT) & EROFS_I_VERSION_MASK; +} + +static inline unsigned int erofs_inode_datalayout(unsigned int ifmt) +{ + return (ifmt >> EROFS_I_DATALAYOUT_BIT) & EROFS_I_DATALAYOUT_MASK; +} + +/* reclaiming is never triggered when allocating new folios. */ +static inline struct folio *erofs_grab_folio_nowait(struct address_space *as, + pgoff_t index) +{ + return __filemap_get_folio(as, index, + FGP_LOCK|FGP_CREAT|FGP_NOFS|FGP_NOWAIT, + readahead_gfp_mask(as) & ~__GFP_RECLAIM); +} + +/* Allocated on disk at @m_pa (e.g. NOT a fragment extent) */ +#define EROFS_MAP_MAPPED 0x0001 +/* Located in metadata (could be copied from bd_inode) */ +#define EROFS_MAP_META 0x0002 +/* @m_llen may be truncated by the runtime compared to the on-disk record */ +#define EROFS_MAP_PARTIAL_MAPPED 0x0004 +/* The on-disk @m_llen may cover only part of the encoded data */ +#define EROFS_MAP_PARTIAL_REF 0x0008 +/* Located in the special packed inode */ +#define EROFS_MAP_FRAGMENT 0x0010 +/* The encoded on-disk data will be fully handled (decompressed) */ +#define EROFS_MAP_FULL(f) (!((f) & (EROFS_MAP_PARTIAL_MAPPED | \ + EROFS_MAP_PARTIAL_REF))) + +struct erofs_map_blocks { + struct erofs_buf buf; + + erofs_off_t m_pa, m_la; + u64 m_plen, m_llen; + + unsigned short m_deviceid; + char m_algorithmformat; + unsigned int m_flags; +}; + +/* + * Used to get the exact decompressed length, e.g. fiemap (consider lookback + * approach instead if possible since it's more metadata lightweight.) + */ +#define EROFS_GET_BLOCKS_FIEMAP 0x0001 +/* Used to map the whole extent if non-negligible data is requested for LZMA */ +#define EROFS_GET_BLOCKS_READMORE 0x0002 +/* Used to map tail extent for tailpacking inline or fragment pcluster */ +#define EROFS_GET_BLOCKS_FINDTAIL 0x0004 + +enum { + Z_EROFS_COMPRESSION_SHIFTED = Z_EROFS_COMPRESSION_MAX, + Z_EROFS_COMPRESSION_INTERLACED, + Z_EROFS_COMPRESSION_RUNTIME_MAX +}; + +struct erofs_map_dev { + struct super_block *m_sb; + struct erofs_device_info *m_dif; + struct block_device *m_bdev; + + erofs_off_t m_pa; + unsigned int m_deviceid; +}; + +extern const struct super_operations erofs_sops; +extern const struct address_space_operations erofs_aops; +extern const struct address_space_operations erofs_fileio_aops; +extern const struct address_space_operations z_erofs_aops; + +extern const struct inode_operations erofs_generic_iops; +extern const struct inode_operations erofs_symlink_iops; +extern const struct inode_operations erofs_fast_symlink_iops; +extern const struct inode_operations erofs_dir_iops; + +extern const struct file_operations erofs_file_fops; +extern const struct file_operations erofs_dir_fops; +extern const struct file_operations erofs_ishare_fops; + +extern const struct iomap_ops z_erofs_iomap_report_ops; + +int erofs_setup_managed_cache(struct super_block *sb); +#ifdef CONFIG_EROFS_FS_BACKED_BY_FILE +int erofs_read_meta_folio(struct file *file, struct folio *folio); +#else +#define erofs_read_meta_folio NULL +#endif +void *erofs_read_metadata(struct super_block *sb, struct erofs_buf *buf, + erofs_off_t *offset, int *lengthp); +void erofs_unmap_metabuf(struct erofs_buf *buf); +void erofs_put_metabuf(struct erofs_buf *buf); +void *erofs_bread(struct erofs_buf *buf, erofs_off_t offset, bool need_kmap); +int erofs_init_metabuf(struct erofs_buf *buf, struct super_block *sb, + bool in_metabox); +void *erofs_read_metabuf(struct erofs_buf *buf, struct super_block *sb, + erofs_off_t offset, bool in_metabox); +int erofs_map_dev(struct super_block *sb, struct erofs_map_dev *dev); +int erofs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + u64 start, u64 len); +int erofs_map_blocks(struct inode *inode, struct erofs_map_blocks *map); +void erofs_onlinefolio_init(struct folio *folio); +void erofs_onlinefolio_split(struct folio *folio); +void erofs_onlinefolio_end(struct folio *folio, int err, bool dirty); +struct inode *erofs_iget(struct super_block *sb, erofs_nid_t nid); +int erofs_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, + unsigned int query_flags); +int erofs_namei(struct inode *dir, const struct qstr *name, + erofs_nid_t *nid, unsigned int *d_type); + +static inline void *erofs_vm_map_ram(struct page **pages, unsigned int count) +{ + int retried = 0; + + while (1) { + void *p = vm_map_ram(pages, count, -1); + + /* retry two more times (totally 3 times) */ + if (p || ++retried >= 3) + return p; + vm_unmap_aliases(); + } + return NULL; +} + +static inline const struct address_space_operations * +erofs_get_aops(struct inode *realinode) +{ + if (erofs_inode_is_data_compressed(EROFS_I(realinode)->datalayout)) { + if (!IS_ENABLED(CONFIG_EROFS_FS_ZIP)) + return ERR_PTR(-EOPNOTSUPP); + DO_ONCE_LITE_IF(realinode->i_blkbits != PAGE_SHIFT, + erofs_info, realinode->i_sb, + "EXPERIMENTAL EROFS subpage compressed block support in use. Use at your own risk!"); + return &z_erofs_aops; + } + if (IS_ENABLED(CONFIG_EROFS_FS_BACKED_BY_FILE) && + erofs_is_fileio_mode(EROFS_SB(realinode->i_sb))) + return &erofs_fileio_aops; + return &erofs_aops; +} + +int erofs_register_sysfs(struct super_block *sb); +void erofs_unregister_sysfs(struct super_block *sb); +int __init erofs_init_sysfs(void); +void erofs_exit_sysfs(void); + +struct page *__erofs_allocpage(struct page **pagepool, gfp_t gfp, bool tryrsv); +static inline struct page *erofs_allocpage(struct page **pagepool, gfp_t gfp) +{ + return __erofs_allocpage(pagepool, gfp, false); +} +static inline void erofs_pagepool_add(struct page **pagepool, struct page *page) +{ + set_page_private(page, (unsigned long)*pagepool); + *pagepool = page; +} +void erofs_release_pages(struct page **pagepool); + +#ifdef CONFIG_EROFS_FS_ZIP +#define MNGD_MAPPING(sbi) ((sbi)->managed_cache->i_mapping) + +extern atomic_long_t erofs_global_shrink_cnt; +void erofs_shrinker_register(struct super_block *sb); +void erofs_shrinker_unregister(struct super_block *sb); +int __init erofs_init_shrinker(void); +void erofs_exit_shrinker(void); +int __init z_erofs_init_subsystem(void); +void z_erofs_exit_subsystem(void); +int z_erofs_init_super(struct super_block *sb); +unsigned long z_erofs_shrink_scan(struct erofs_sb_info *sbi, + unsigned long nr_shrink); +int z_erofs_map_blocks_iter(struct inode *inode, struct erofs_map_blocks *map, + int flags); +void *z_erofs_get_gbuf(unsigned int requiredpages); +void z_erofs_put_gbuf(void *ptr); +int z_erofs_gbuf_growsize(unsigned int nrpages); +int __init z_erofs_gbuf_init(void); +void z_erofs_gbuf_exit(void); +#else +static inline void erofs_shrinker_register(struct super_block *sb) {} +static inline void erofs_shrinker_unregister(struct super_block *sb) {} +static inline int erofs_init_shrinker(void) { return 0; } +static inline void erofs_exit_shrinker(void) {} +static inline int z_erofs_init_subsystem(void) { return 0; } +static inline void z_erofs_exit_subsystem(void) {} +static inline int z_erofs_init_super(struct super_block *sb) { return 0; } +#endif /* !CONFIG_EROFS_FS_ZIP */ +int z_erofs_parse_cfgs(struct super_block *sb, struct erofs_super_block *dsb); + +#ifdef CONFIG_EROFS_FS_BACKED_BY_FILE +struct bio *erofs_fileio_bio_alloc(struct erofs_map_dev *mdev); +void erofs_fileio_submit_bio(struct bio *bio); +#else +static inline struct bio *erofs_fileio_bio_alloc(struct erofs_map_dev *mdev) { return NULL; } +static inline void erofs_fileio_submit_bio(struct bio *bio) {} +#endif + +#ifdef CONFIG_EROFS_FS_PAGE_CACHE_SHARE +int __init erofs_init_ishare(void); +void erofs_exit_ishare(void); +bool erofs_ishare_fill_inode(struct inode *inode); +void erofs_ishare_free_inode(struct inode *inode); +struct inode *erofs_real_inode(struct inode *inode, bool *need_iput); +#else +static inline int erofs_init_ishare(void) { return 0; } +static inline void erofs_exit_ishare(void) {} +static inline bool erofs_ishare_fill_inode(struct inode *inode) { return false; } +static inline void erofs_ishare_free_inode(struct inode *inode) {} +static inline struct inode *erofs_real_inode(struct inode *inode, bool *need_iput) +{ + *need_iput = false; + return inode; +} +#endif + +long erofs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); +long erofs_compat_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg); + +#endif /* __EROFS_INTERNAL_H */ diff --git a/fs/erofs/ishare.c b/fs/erofs/ishare.c new file mode 100644 index 000000000..fa7d4112d --- /dev/null +++ b/fs/erofs/ishare.c @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2024, Alibaba Cloud + */ +#include +#include +#include +#include +#include "internal.h" +#include "xattr.h" + +static struct vfsmount *erofs_ishare_mnt; + +static int erofs_ishare_iget5_eq(struct inode *inode, void *data) +{ + struct erofs_inode_fingerprint *fp1 = &EROFS_I(inode)->fingerprint; + struct erofs_inode_fingerprint *fp2 = data; + + return fp1->size == fp2->size && + !memcmp(fp1->opaque, fp2->opaque, fp2->size); +} + +static int erofs_ishare_iget5_set(struct inode *inode, void *data) +{ + struct erofs_inode *vi = EROFS_I(inode); + + vi->fingerprint = *(struct erofs_inode_fingerprint *)data; + INIT_LIST_HEAD(&vi->ishare_list); + spin_lock_init(&vi->ishare_lock); + return 0; +} + +bool erofs_ishare_fill_inode(struct inode *inode) +{ + static const struct file_operations empty_fops = {}; + struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb); + const struct address_space_operations *aops; + struct erofs_inode *vi = EROFS_I(inode); + struct erofs_inode_fingerprint fp; + struct dentry *sd; + struct inode *si; + + aops = erofs_get_aops(inode); + if (IS_ERR(aops)) + return false; + if (erofs_xattr_fill_inode_fingerprint(&fp, inode, sbi->domain_id)) + return false; + + si = iget5_locked(erofs_ishare_mnt->mnt_sb, + xxh32(fp.opaque, fp.size, 0), + erofs_ishare_iget5_eq, erofs_ishare_iget5_set, &fp); + if (si && (inode_state_read_once(si) & I_NEW)) { + si->i_fop = &empty_fops; + si->i_mapping->a_ops = aops; + si->i_mode = 0444 | S_IFREG; + si->i_size = inode->i_size; + unlock_new_inode(si); + } else { + kfree(fp.opaque); + if (!si || aops != si->i_mapping->a_ops) { + iput(si); + return false; + } + if (si->i_size != inode->i_size) { + erofs_warn(inode->i_sb, "i_size mismatch (%lld != %lld) for the same fingerprint", + inode->i_size, si->i_size); + iput(si); + return false; + } + } + sd = d_obtain_alias(si); /* disconnected denties for sharedinodes */ + if (IS_ERR(sd)) + return false; + vi->sharedentry = sd; + INIT_LIST_HEAD(&vi->ishare_list); + spin_lock(&EROFS_I(si)->ishare_lock); + list_add(&vi->ishare_list, &EROFS_I(si)->ishare_list); + spin_unlock(&EROFS_I(si)->ishare_lock); + return true; +} + +void erofs_ishare_free_inode(struct inode *inode) +{ + struct erofs_inode *vi = EROFS_I(inode), *svi; + + if (!vi->sharedentry) + return; + svi = EROFS_I(d_inode(vi->sharedentry)); + spin_lock(&svi->ishare_lock); + list_del(&vi->ishare_list); + spin_unlock(&svi->ishare_lock); + dput(vi->sharedentry); + vi->sharedentry = NULL; +} + +static int erofs_ishare_file_open(struct inode *inode, struct file *file) +{ + struct path sharedpath = { + .mnt = erofs_ishare_mnt, + .dentry = EROFS_I(inode)->sharedentry, + }; + struct file *rf; + + if (file->f_flags & O_DIRECT) + return -EINVAL; + + rf = backing_file_open(file, file->f_flags | O_NOATIME, + &sharedpath, current_cred()); + if (IS_ERR(rf)) + return PTR_ERR(rf); + file->private_data = rf; + return 0; +} + +static int erofs_ishare_file_release(struct inode *inode, struct file *file) +{ + fput(file->private_data); + file->private_data = NULL; + return 0; +} + +static ssize_t erofs_ishare_file_read_iter(struct kiocb *iocb, + struct iov_iter *to) +{ + struct file *realfile = iocb->ki_filp->private_data; + struct kiocb dedup_iocb; + ssize_t nread; + + if (!iov_iter_count(to)) + return 0; + kiocb_clone(&dedup_iocb, iocb, realfile); + nread = filemap_read(&dedup_iocb, to, 0); + iocb->ki_pos = dedup_iocb.ki_pos; + return nread; +} + +static int erofs_ishare_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct file *realfile = file->private_data; + int err; + + vma_set_file(vma, realfile); + + err = security_mmap_backing_file(vma, realfile, file); + if (err) + return err; + + return generic_file_readonly_mmap(file, vma); +} + +static int erofs_ishare_fadvise(struct file *file, loff_t offset, + loff_t len, int advice) +{ + return vfs_fadvise(file->private_data, offset, len, advice); +} + +const struct file_operations erofs_ishare_fops = { + .open = erofs_ishare_file_open, + .llseek = generic_file_llseek, + .read_iter = erofs_ishare_file_read_iter, + .mmap = erofs_ishare_mmap, + .release = erofs_ishare_file_release, + .get_unmapped_area = thp_get_unmapped_area, + .splice_read = filemap_splice_read, + .fadvise = erofs_ishare_fadvise, +}; + +struct inode *erofs_real_inode(struct inode *inode, bool *need_iput) +{ + struct erofs_inode *vi, *vi_share; + struct inode *realinode; + + *need_iput = false; + if (inode->i_sb != erofs_ishare_mnt->mnt_sb) + return inode; + + vi_share = EROFS_I(inode); + spin_lock(&vi_share->ishare_lock); + /* fetch any one as real inode */ + DBG_BUGON(list_empty(&vi_share->ishare_list)); + list_for_each_entry(vi, &vi_share->ishare_list, ishare_list) { + realinode = igrab(&vi->vfs_inode); + if (realinode) { + *need_iput = true; + break; + } + } + spin_unlock(&vi_share->ishare_lock); + + DBG_BUGON(!realinode); + return realinode; +} + +int __init erofs_init_ishare(void) +{ + struct vfsmount *mnt; + int ret; + + mnt = kern_mount(&erofs_anon_fs_type); + if (IS_ERR(mnt)) + return PTR_ERR(mnt); + /* generic_fadvise() doesn't work if s_bdi == &noop_backing_dev_info */ + ret = super_setup_bdi(mnt->mnt_sb); + if (ret) + kern_unmount(mnt); + else + erofs_ishare_mnt = mnt; + return ret; +} + +void erofs_exit_ishare(void) +{ + kern_unmount(erofs_ishare_mnt); +} diff --git a/fs/erofs/namei.c b/fs/erofs/namei.c new file mode 100644 index 000000000..f7cf4f41a --- /dev/null +++ b/fs/erofs/namei.c @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2022, Alibaba Cloud + */ +#include "xattr.h" +#include + +struct erofs_qstr { + const unsigned char *name; + const unsigned char *end; +}; + +/* based on the end of qn is accurate and it must have the trailing '\0' */ +static inline int erofs_dirnamecmp(const struct erofs_qstr *qn, + const struct erofs_qstr *qd, + unsigned int *matched) +{ + unsigned int i = *matched; + + /* + * on-disk error, let's only BUG_ON in the debugging mode. + * otherwise, it will return 1 to just skip the invalid name + * and go on (in consideration of the lookup performance). + */ + DBG_BUGON(qd->name > qd->end); + + /* qd could not have trailing '\0' */ + /* However it is absolutely safe if < qd->end */ + while (qd->name + i < qd->end && qd->name[i] != '\0') { + if (qn->name[i] != qd->name[i]) { + *matched = i; + return qn->name[i] > qd->name[i] ? 1 : -1; + } + ++i; + } + *matched = i; + /* See comments in __d_alloc on the terminating NUL character */ + return qn->name[i] == '\0' ? 0 : 1; +} + +#define nameoff_from_disk(off, sz) (le16_to_cpu(off) & ((sz) - 1)) + +static struct erofs_dirent *find_target_dirent(struct erofs_qstr *name, + u8 *data, + unsigned int dirblksize, + const int ndirents) +{ + int head, back; + unsigned int startprfx, endprfx; + struct erofs_dirent *const de = (struct erofs_dirent *)data; + + /* since the 1st dirent has been evaluated previously */ + head = 1; + back = ndirents - 1; + startprfx = endprfx = 0; + + while (head <= back) { + const int mid = head + (back - head) / 2; + const int nameoff = nameoff_from_disk(de[mid].nameoff, + dirblksize); + unsigned int matched = min(startprfx, endprfx); + struct erofs_qstr dname = { + .name = data + nameoff, + .end = mid >= ndirents - 1 ? + data + dirblksize : + data + nameoff_from_disk(de[mid + 1].nameoff, + dirblksize) + }; + + /* string comparison without already matched prefix */ + int ret = erofs_dirnamecmp(name, &dname, &matched); + + if (!ret) { + return de + mid; + } else if (ret > 0) { + head = mid + 1; + startprfx = matched; + } else { + back = mid - 1; + endprfx = matched; + } + } + + return ERR_PTR(-ENOENT); +} + +static void *erofs_find_target_block(struct erofs_buf *target, + struct inode *dir, struct erofs_qstr *name, int *_ndirents) +{ + unsigned int bsz = i_blocksize(dir); + int head = 0, back = erofs_iblks(dir) - 1; + unsigned int startprfx = 0, endprfx = 0; + void *candidate = ERR_PTR(-ENOENT); + + while (head <= back) { + const int mid = head + (back - head) / 2; + struct erofs_buf buf = __EROFS_BUF_INITIALIZER; + struct erofs_dirent *de; + + buf.mapping = dir->i_mapping; + de = erofs_bread(&buf, erofs_pos(dir->i_sb, mid), true); + if (!IS_ERR(de)) { + const int nameoff = nameoff_from_disk(de->nameoff, bsz); + const int ndirents = nameoff / sizeof(*de); + int diff; + unsigned int matched; + struct erofs_qstr dname; + + if (!ndirents) { + erofs_put_metabuf(&buf); + erofs_err(dir->i_sb, + "corrupted dir block %d @ nid %llu", + mid, EROFS_I(dir)->nid); + DBG_BUGON(1); + de = ERR_PTR(-EFSCORRUPTED); + goto out; + } + + matched = min(startprfx, endprfx); + + dname.name = (u8 *)de + nameoff; + if (ndirents == 1) + dname.end = (u8 *)de + bsz; + else + dname.end = (u8 *)de + + nameoff_from_disk(de[1].nameoff, bsz); + + /* string comparison without already matched prefix */ + diff = erofs_dirnamecmp(name, &dname, &matched); + + if (diff < 0) { + erofs_put_metabuf(&buf); + back = mid - 1; + endprfx = matched; + continue; + } + + if (!IS_ERR(candidate)) + erofs_put_metabuf(target); + *target = buf; + if (!diff) { + *_ndirents = 0; + return de; + } + head = mid + 1; + startprfx = matched; + candidate = de; + *_ndirents = ndirents; + continue; + } +out: /* free if the candidate is valid */ + if (!IS_ERR(candidate)) + erofs_put_metabuf(target); + return de; + } + return candidate; +} + +int erofs_namei(struct inode *dir, const struct qstr *name, erofs_nid_t *nid, + unsigned int *d_type) +{ + int ndirents; + struct erofs_buf buf = __EROFS_BUF_INITIALIZER; + struct erofs_dirent *de; + struct erofs_qstr qn; + + if (!dir->i_size) + return -ENOENT; + + qn.name = name->name; + qn.end = name->name + name->len; + buf.mapping = dir->i_mapping; + + ndirents = 0; + de = erofs_find_target_block(&buf, dir, &qn, &ndirents); + if (IS_ERR(de)) + return PTR_ERR(de); + + if (ndirents) + de = find_target_dirent(&qn, (u8 *)de, i_blocksize(dir), + ndirents); + + if (!IS_ERR(de)) { + *nid = le64_to_cpu(de->nid); + *d_type = de->file_type; + } + erofs_put_metabuf(&buf); + return PTR_ERR_OR_ZERO(de); +} + +static struct dentry *erofs_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + int err; + erofs_nid_t nid; + unsigned int d_type; + struct inode *inode; + + trace_erofs_lookup(dir, dentry, flags); + + if (dentry->d_name.len > EROFS_NAME_LEN) + return ERR_PTR(-ENAMETOOLONG); + + err = erofs_namei(dir, &dentry->d_name, &nid, &d_type); + + if (err == -ENOENT) + /* negative dentry */ + inode = NULL; + else if (err) + inode = ERR_PTR(err); + else + inode = erofs_iget(dir->i_sb, nid); + return d_splice_alias(inode, dentry); +} + +const struct inode_operations erofs_dir_iops = { + .lookup = erofs_lookup, + .getattr = erofs_getattr, + .listxattr = erofs_listxattr, + .get_inode_acl = erofs_get_acl, + .fiemap = erofs_fiemap, +}; diff --git a/fs/erofs/super.c b/fs/erofs/super.c new file mode 100644 index 000000000..0967aace9 --- /dev/null +++ b/fs/erofs/super.c @@ -0,0 +1,1146 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021, Alibaba Cloud + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "xattr.h" + +#define CREATE_TRACE_POINTS +#include + +static struct kmem_cache *erofs_inode_cachep __read_mostly; + +void _erofs_printk(struct super_block *sb, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + int level; + + va_start(args, fmt); + + level = printk_get_level(fmt); + vaf.fmt = printk_skip_level(fmt); + vaf.va = &args; + if (sb) + printk("%c%cerofs (device %s): %pV", + KERN_SOH_ASCII, level, sb->s_id, &vaf); + else + printk("%c%cerofs: %pV", KERN_SOH_ASCII, level, &vaf); + va_end(args); +} + +static int erofs_superblock_csum_verify(struct super_block *sb, void *sbdata) +{ + struct erofs_super_block *dsb = sbdata + EROFS_SUPER_OFFSET; + u32 len = 1 << EROFS_SB(sb)->blkszbits, crc; + + if (len > EROFS_SUPER_OFFSET) + len -= EROFS_SUPER_OFFSET; + len -= offsetof(struct erofs_super_block, checksum) + + sizeof(dsb->checksum); + + /* skip .magic(pre-verified) and .checksum(0) fields */ + crc = crc32c(0x5045B54A, (&dsb->checksum) + 1, len); + if (crc == le32_to_cpu(dsb->checksum)) + return 0; + erofs_err(sb, "invalid checksum 0x%08x, 0x%08x expected", + crc, le32_to_cpu(dsb->checksum)); + return -EBADMSG; +} + +static void erofs_inode_init_once(void *ptr) +{ + struct erofs_inode *vi = ptr; + + inode_init_once(&vi->vfs_inode); +} + +static struct inode *erofs_alloc_inode(struct super_block *sb) +{ + struct erofs_inode *vi = + alloc_inode_sb(sb, erofs_inode_cachep, GFP_KERNEL); + + if (!vi) + return NULL; + + /* zero out everything except vfs_inode */ + memset(vi, 0, offsetof(struct erofs_inode, vfs_inode)); + return &vi->vfs_inode; +} + +static void erofs_free_inode(struct inode *inode) +{ + struct erofs_inode *vi = EROFS_I(inode); + + if (inode->i_op == &erofs_fast_symlink_iops) + kfree(inode->i_link); + kfree(vi->xattr_shared_xattrs); + kmem_cache_free(erofs_inode_cachep, vi); +} + +/* read variable-sized metadata, offset will be aligned by 4-byte */ +void *erofs_read_metadata(struct super_block *sb, struct erofs_buf *buf, + erofs_off_t *offset, int *lengthp) +{ + u8 *buffer, *ptr; + int len, i, cnt; + + *offset = round_up(*offset, 4); + ptr = erofs_bread(buf, *offset, true); + if (IS_ERR(ptr)) + return ptr; + + len = le16_to_cpu(*(__le16 *)ptr); + if (!len) + len = U16_MAX + 1; + buffer = kmalloc(len, GFP_KERNEL); + if (!buffer) + return ERR_PTR(-ENOMEM); + *offset += sizeof(__le16); + *lengthp = len; + + for (i = 0; i < len; i += cnt) { + cnt = min_t(int, sb->s_blocksize - erofs_blkoff(sb, *offset), + len - i); + ptr = erofs_bread(buf, *offset, true); + if (IS_ERR(ptr)) { + kfree(buffer); + return ptr; + } + memcpy(buffer + i, ptr, cnt); + *offset += cnt; + } + return buffer; +} + +static int erofs_init_device(struct erofs_buf *buf, struct super_block *sb, + struct erofs_device_info *dif, erofs_off_t *pos) +{ + struct erofs_sb_info *sbi = EROFS_SB(sb); + struct erofs_deviceslot *dis; + struct file *file; + bool _48bit; + + dis = erofs_read_metabuf(buf, sb, *pos, false); + if (IS_ERR(dis)) + return PTR_ERR(dis); + + if (!sbi->devs->flatdev && !dif->path) { + if (!dis->tag[0]) { + erofs_err(sb, "empty device tag @ pos %llu", *pos); + return -EINVAL; + } + dif->path = kmemdup_nul(dis->tag, sizeof(dis->tag), GFP_KERNEL); + if (!dif->path) + return -ENOMEM; + } + + if (!sbi->devs->flatdev) { + file = erofs_is_fileio_mode(sbi) ? + filp_open(dif->path, O_RDONLY | O_LARGEFILE, 0) : + fs_bdev_file_open_by_path(dif->path, + BLK_OPEN_READ, sb->s_type, sb); + if (IS_ERR(file)) { + if (file == ERR_PTR(-ENOTBLK)) + return -EINVAL; + return PTR_ERR(file); + } + + if (!erofs_is_fileio_mode(sbi)) { + dif->dax_dev = fs_dax_get_by_bdev(file_bdev(file), + &dif->dax_part_off, NULL, NULL); + } else if (!S_ISREG(file_inode(file)->i_mode)) { + fput(file); + return -EINVAL; + } + if (!dif->dax_dev && test_opt(&sbi->opt, DAX_ALWAYS)) { + erofs_info(sb, "DAX unsupported by %s. Turning off DAX.", + dif->path); + clear_opt(&sbi->opt, DAX_ALWAYS); + } + dif->file = file; + } + + _48bit = erofs_sb_has_48bit(sbi); + dif->blocks = le32_to_cpu(dis->blocks_lo) | + (_48bit ? (u64)le16_to_cpu(dis->blocks_hi) << 32 : 0); + dif->uniaddr = le32_to_cpu(dis->uniaddr_lo) | + (_48bit ? (u64)le16_to_cpu(dis->uniaddr_hi) << 32 : 0); + sbi->total_blocks += dif->blocks; + *pos += EROFS_DEVT_SLOT_SIZE; + return 0; +} + +static int erofs_scan_devices(struct super_block *sb, + struct erofs_super_block *dsb) +{ + struct erofs_sb_info *sbi = EROFS_SB(sb); + unsigned int ondisk_extradevs; + erofs_off_t pos; + struct erofs_buf buf = __EROFS_BUF_INITIALIZER; + struct erofs_device_info *dif; + int id, err = 0; + + sbi->total_blocks = sbi->dif0.blocks; + if (!erofs_sb_has_device_table(sbi)) + ondisk_extradevs = 0; + else + ondisk_extradevs = le16_to_cpu(dsb->extra_devices); + + if (sbi->devs->extra_devices && + ondisk_extradevs != sbi->devs->extra_devices) { + erofs_err(sb, "extra devices don't match (ondisk %u, given %u)", + ondisk_extradevs, sbi->devs->extra_devices); + return -EINVAL; + } + + if (test_opt(&sbi->opt, DAX_ALWAYS) && !sbi->dif0.dax_dev) { + erofs_info(sb, "DAX unsupported by block device. Turning off DAX."); + clear_opt(&sbi->opt, DAX_ALWAYS); + } + if (!ondisk_extradevs) + return 0; + + if (!sbi->devs->extra_devices) + sbi->devs->flatdev = true; + + sbi->device_id_mask = roundup_pow_of_two(ondisk_extradevs + 1) - 1; + pos = le16_to_cpu(dsb->devt_slotoff) * EROFS_DEVT_SLOT_SIZE; + down_read(&sbi->devs->rwsem); + if (sbi->devs->extra_devices) { + idr_for_each_entry(&sbi->devs->tree, dif, id) { + err = erofs_init_device(&buf, sb, dif, &pos); + if (err) + break; + } + } else { + for (id = 0; id < ondisk_extradevs; id++) { + dif = kzalloc_obj(*dif); + if (!dif) { + err = -ENOMEM; + break; + } + + err = idr_alloc(&sbi->devs->tree, dif, 0, 0, GFP_KERNEL); + if (err < 0) { + kfree(dif); + break; + } + ++sbi->devs->extra_devices; + + err = erofs_init_device(&buf, sb, dif, &pos); + if (err) + break; + } + } + up_read(&sbi->devs->rwsem); + erofs_put_metabuf(&buf); + return err; +} + +static int erofs_read_superblock(struct super_block *sb) +{ + struct erofs_sb_info *sbi = EROFS_SB(sb); + struct erofs_buf buf = __EROFS_BUF_INITIALIZER; + struct erofs_super_block *dsb; + void *data; + int ret; + + data = erofs_read_metabuf(&buf, sb, 0, false); + if (IS_ERR(data)) { + erofs_err(sb, "cannot read erofs superblock"); + return PTR_ERR(data); + } + + dsb = (struct erofs_super_block *)(data + EROFS_SUPER_OFFSET); + ret = -EINVAL; + if (le32_to_cpu(dsb->magic) != EROFS_SUPER_MAGIC_V1) { + erofs_err(sb, "cannot find valid erofs superblock"); + goto out; + } + + sbi->blkszbits = dsb->blkszbits; + if (sbi->blkszbits < 9 || sbi->blkszbits > PAGE_SHIFT) { + erofs_err(sb, "blkszbits %u isn't supported", sbi->blkszbits); + goto out; + } + if (dsb->dirblkbits) { + erofs_err(sb, "dirblkbits %u isn't supported", dsb->dirblkbits); + goto out; + } + + sbi->feature_compat = le32_to_cpu(dsb->feature_compat); + if (erofs_sb_has_sb_chksum(sbi)) { + ret = erofs_superblock_csum_verify(sb, data); + if (ret) + goto out; + } + + ret = -EINVAL; + sbi->feature_incompat = le32_to_cpu(dsb->feature_incompat); + if (sbi->feature_incompat & ~EROFS_ALL_FEATURE_INCOMPAT) { + erofs_err(sb, "unidentified incompatible feature %x, please upgrade kernel", + sbi->feature_incompat & ~EROFS_ALL_FEATURE_INCOMPAT); + goto out; + } + + sbi->sb_size = 128 + dsb->sb_extslots * EROFS_SB_EXTSLOT_SIZE; + if (sbi->sb_size > PAGE_SIZE - EROFS_SUPER_OFFSET) { + erofs_err(sb, "invalid sb_extslots %u (more than a fs block)", + sbi->sb_size); + goto out; + } + sbi->dif0.blocks = le32_to_cpu(dsb->blocks_lo); + sbi->meta_blkaddr = le32_to_cpu(dsb->meta_blkaddr); +#ifdef CONFIG_EROFS_FS_XATTR + sbi->xattr_blkaddr = le32_to_cpu(dsb->xattr_blkaddr); + sbi->xattr_prefix_start = le32_to_cpu(dsb->xattr_prefix_start); + sbi->xattr_prefix_count = dsb->xattr_prefix_count; + sbi->xattr_filter_reserved = dsb->xattr_filter_reserved; + if (erofs_sb_has_ishare_xattrs(sbi)) { + if (dsb->ishare_xattr_prefix_id >= sbi->xattr_prefix_count) { + erofs_err(sb, "invalid ishare xattr prefix id %u", + dsb->ishare_xattr_prefix_id); + ret = -EFSCORRUPTED; + goto out; + } + sbi->ishare_xattr_prefix_id = dsb->ishare_xattr_prefix_id; + } +#endif + sbi->islotbits = ilog2(sizeof(struct erofs_inode_compact)); + if (erofs_sb_has_48bit(sbi) && dsb->rootnid_8b) { + sbi->root_nid = le64_to_cpu(dsb->rootnid_8b); + sbi->dif0.blocks = sbi->dif0.blocks | + ((u64)le16_to_cpu(dsb->rb.blocks_hi) << 32); + } else { + sbi->root_nid = le16_to_cpu(dsb->rb.rootnid_2b); + } + sbi->packed_nid = le64_to_cpu(dsb->packed_nid); + if (erofs_sb_has_metabox(sbi)) { + ret = -EFSCORRUPTED; + if (sbi->sb_size <= offsetof(struct erofs_super_block, + metabox_nid)) + goto out; + sbi->metabox_nid = le64_to_cpu(dsb->metabox_nid); + if (sbi->metabox_nid & BIT_ULL(EROFS_DIRENT_NID_METABOX_BIT)) + goto out; /* self-loop detection */ + } + sbi->inos = le64_to_cpu(dsb->inos); + + sbi->epoch = (s64)le64_to_cpu(dsb->epoch); + sbi->fixed_nsec = le32_to_cpu(dsb->fixed_nsec); + super_set_uuid(sb, (void *)dsb->uuid, sizeof(dsb->uuid)); + + if (dsb->volume_name[0]) { + sbi->volume_name = kstrndup(dsb->volume_name, + sizeof(dsb->volume_name), GFP_KERNEL); + if (!sbi->volume_name) { + ret = -ENOMEM; + goto out; + } + } + + if (IS_ENABLED(CONFIG_EROFS_FS_ZIP)) { + ret = z_erofs_parse_cfgs(sb, dsb); + if (ret < 0) + goto out; + } else if (dsb->u1.available_compr_algs || + erofs_sb_has_lz4_0padding(sbi)) { + erofs_err(sb, "compression disabled, unable to mount compressed EROFS"); + ret = -EOPNOTSUPP; + goto out; + } + + ret = erofs_scan_devices(sb, dsb); + + if (erofs_sb_has_48bit(sbi)) + erofs_info(sb, "EXPERIMENTAL 48-bit layout support in use. Use at your own risk!"); + if (erofs_sb_has_metabox(sbi)) + erofs_info(sb, "EXPERIMENTAL metadata compression support in use. Use at your own risk!"); +out: + erofs_put_metabuf(&buf); + return ret; +} + +static void erofs_default_options(struct erofs_sb_info *sbi) +{ +#ifdef CONFIG_EROFS_FS_ZIP + sbi->opt.cache_strategy = EROFS_ZIP_CACHE_READAROUND; + sbi->sync_decompress = EROFS_SYNC_DECOMPRESS_AUTO; +#endif + if (IS_ENABLED(CONFIG_EROFS_FS_XATTR)) + set_opt(&sbi->opt, XATTR_USER); + if (IS_ENABLED(CONFIG_EROFS_FS_POSIX_ACL)) + set_opt(&sbi->opt, POSIX_ACL); +} + +enum { + Opt_user_xattr, Opt_acl, Opt_cache_strategy, Opt_dax, Opt_dax_enum, + Opt_device, Opt_domain_id, Opt_directio, Opt_fsoffset, Opt_inode_share, + Opt_source, +}; + +static const struct constant_table erofs_param_cache_strategy[] = { + {"disabled", EROFS_ZIP_CACHE_DISABLED}, + {"readahead", EROFS_ZIP_CACHE_READAHEAD}, + {"readaround", EROFS_ZIP_CACHE_READAROUND}, + {} +}; + +static const struct constant_table erofs_dax_param_enums[] = { + {"always", EROFS_MOUNT_DAX_ALWAYS}, + {"never", EROFS_MOUNT_DAX_NEVER}, + {} +}; + +static const struct fs_parameter_spec erofs_fs_parameters[] = { + fsparam_flag_no("user_xattr", Opt_user_xattr), + fsparam_flag_no("acl", Opt_acl), + fsparam_enum("cache_strategy", Opt_cache_strategy, + erofs_param_cache_strategy), + fsparam_flag("dax", Opt_dax), + fsparam_enum("dax", Opt_dax_enum, erofs_dax_param_enums), + fsparam_string("device", Opt_device), + fsparam_string("domain_id", Opt_domain_id), + fsparam_flag_no("directio", Opt_directio), + fsparam_u64("fsoffset", Opt_fsoffset), + fsparam_flag("inode_share", Opt_inode_share), + fsparam_file_or_string("source", Opt_source), + {} +}; + +static bool erofs_fc_set_dax_mode(struct fs_context *fc, unsigned int mode) +{ + if (IS_ENABLED(CONFIG_FS_DAX)) { + struct erofs_sb_info *sbi = fc->s_fs_info; + + if (mode == EROFS_MOUNT_DAX_ALWAYS) { + set_opt(&sbi->opt, DAX_ALWAYS); + clear_opt(&sbi->opt, DAX_NEVER); + return true; + } else if (mode == EROFS_MOUNT_DAX_NEVER) { + set_opt(&sbi->opt, DAX_NEVER); + clear_opt(&sbi->opt, DAX_ALWAYS); + return true; + } + DBG_BUGON(1); + return false; + } + errorfc(fc, "dax options not supported"); + return false; +} + +static int erofs_fc_parse_source(struct fs_context *fc, + struct fs_parameter *param) +{ + struct erofs_sb_info *sbi = fc->s_fs_info; + + if (fc->source || sbi->dif0.file) + return invalf(fc, "Multiple sources"); + + switch (param->type) { + case fs_value_is_string: + fc->source = param->string; + param->string = NULL; + return 0; + case fs_value_is_file: { + char *buf __free(kfree) = kmalloc(PATH_MAX, GFP_KERNEL); + char *p; + + if (!buf) + return -ENOMEM; + p = file_path(param->file, buf, PATH_MAX); + if (IS_ERR(p)) + return PTR_ERR(p); + fc->source = kstrdup(p, GFP_KERNEL); + if (!fc->source) + return -ENOMEM; + sbi->dif0.file = no_free_ptr(param->file); + return 0; + } + default: + WARN_ON_ONCE(true); + return -EINVAL; + } +} + +static int erofs_fc_parse_param(struct fs_context *fc, + struct fs_parameter *param) +{ + struct erofs_sb_info *sbi = fc->s_fs_info; + struct fs_parse_result result; + struct erofs_device_info *dif; + int opt, ret; + + opt = fs_parse(fc, erofs_fs_parameters, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_user_xattr: + if (!IS_ENABLED(CONFIG_EROFS_FS_XATTR)) + errorfc(fc, "{,no}user_xattr options not supported"); + else if (result.boolean) + set_opt(&sbi->opt, XATTR_USER); + else + clear_opt(&sbi->opt, XATTR_USER); + break; + case Opt_acl: + if (!IS_ENABLED(CONFIG_EROFS_FS_POSIX_ACL)) + errorfc(fc, "{,no}acl options not supported"); + else if (result.boolean) + set_opt(&sbi->opt, POSIX_ACL); + else + clear_opt(&sbi->opt, POSIX_ACL); + break; + case Opt_cache_strategy: + if (!IS_ENABLED(CONFIG_EROFS_FS_ZIP)) + errorfc(fc, "compression not supported, cache_strategy ignored"); + else + sbi->opt.cache_strategy = result.uint_32; + break; + case Opt_dax: + if (!erofs_fc_set_dax_mode(fc, EROFS_MOUNT_DAX_ALWAYS)) + return -EINVAL; + break; + case Opt_dax_enum: + if (!erofs_fc_set_dax_mode(fc, result.uint_32)) + return -EINVAL; + break; + case Opt_device: + dif = kzalloc_obj(*dif); + if (!dif) + return -ENOMEM; + dif->path = kstrdup(param->string, GFP_KERNEL); + if (!dif->path) { + kfree(dif); + return -ENOMEM; + } + down_write(&sbi->devs->rwsem); + ret = idr_alloc(&sbi->devs->tree, dif, 0, 0, GFP_KERNEL); + up_write(&sbi->devs->rwsem); + if (ret < 0) { + kfree(dif->path); + kfree(dif); + return ret; + } + ++sbi->devs->extra_devices; + break; + case Opt_domain_id: + if (!IS_ENABLED(CONFIG_EROFS_FS_PAGE_CACHE_SHARE)) { + errorfc(fc, "%s option not supported", erofs_fs_parameters[opt].name); + } else { + kfree_sensitive(sbi->domain_id); + sbi->domain_id = no_free_ptr(param->string); + } + break; + case Opt_directio: + if (!IS_ENABLED(CONFIG_EROFS_FS_BACKED_BY_FILE)) + errorfc(fc, "%s option not supported", erofs_fs_parameters[opt].name); + else if (result.boolean) + set_opt(&sbi->opt, DIRECT_IO); + else + clear_opt(&sbi->opt, DIRECT_IO); + break; + case Opt_fsoffset: + sbi->dif0.fsoff = result.uint_64; + break; + case Opt_inode_share: + if (!IS_ENABLED(CONFIG_EROFS_FS_PAGE_CACHE_SHARE)) + errorfc(fc, "%s option not supported", erofs_fs_parameters[opt].name); + else + set_opt(&sbi->opt, INODE_SHARE); + break; + case Opt_source: + return erofs_fc_parse_source(fc, param); + } + return 0; +} + +static int erofs_encode_fh(struct inode *inode, u32 *fh, int *max_len, + struct inode *parent) +{ + erofs_nid_t nid = EROFS_I(inode)->nid; + int len = parent ? 6 : 3; + + if (*max_len < len) { + *max_len = len; + return FILEID_INVALID; + } + + fh[0] = (u32)(nid >> 32); + fh[1] = (u32)(nid & 0xffffffff); + fh[2] = inode->i_generation; + + if (parent) { + nid = EROFS_I(parent)->nid; + + fh[3] = (u32)(nid >> 32); + fh[4] = (u32)(nid & 0xffffffff); + fh[5] = parent->i_generation; + } + + *max_len = len; + return parent ? FILEID_INO64_GEN_PARENT : FILEID_INO64_GEN; +} + +static struct dentry *erofs_fh_to_dentry(struct super_block *sb, + struct fid *fid, int fh_len, int fh_type) +{ + if ((fh_type != FILEID_INO64_GEN && + fh_type != FILEID_INO64_GEN_PARENT) || fh_len < 3) + return NULL; + + return d_obtain_alias(erofs_iget(sb, + ((u64)fid->raw[0] << 32) | fid->raw[1])); +} + +static struct dentry *erofs_fh_to_parent(struct super_block *sb, + struct fid *fid, int fh_len, int fh_type) +{ + if (fh_type != FILEID_INO64_GEN_PARENT || fh_len < 6) + return NULL; + + return d_obtain_alias(erofs_iget(sb, + ((u64)fid->raw[3] << 32) | fid->raw[4])); +} + +static struct dentry *erofs_get_parent(struct dentry *child) +{ + erofs_nid_t nid; + unsigned int d_type; + int err; + + err = erofs_namei(d_inode(child), &dotdot_name, &nid, &d_type); + if (err) + return ERR_PTR(err); + return d_obtain_alias(erofs_iget(child->d_sb, nid)); +} + +static const struct export_operations erofs_export_ops = { + .encode_fh = erofs_encode_fh, + .fh_to_dentry = erofs_fh_to_dentry, + .fh_to_parent = erofs_fh_to_parent, + .get_parent = erofs_get_parent, +}; + +int erofs_setup_managed_cache(struct super_block *sb) +{ + if (!EROFS_SB(sb)->managed_cache) { + struct inode *inode = new_inode(sb); + + if (!inode) + return -ENOMEM; + set_nlink(inode, 1); + inode->i_size = OFFSET_MAX; + mapping_set_gfp_mask(inode->i_mapping, GFP_KERNEL); + EROFS_SB(sb)->managed_cache = inode; + } + return 0; +} + +static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct inode *inode; + struct erofs_sb_info *sbi = EROFS_SB(sb); + int err; + + sb->s_magic = EROFS_SUPER_MAGIC; + sb->s_flags |= SB_RDONLY | SB_NOATIME; + sb->s_maxbytes = MAX_LFS_FILESIZE; + sb->s_op = &erofs_sops; + + if (!sbi->domain_id && test_opt(&sbi->opt, INODE_SHARE)) { + errorfc(fc, "domain_id is needed when inode_share is on"); + return -EINVAL; + } + if (test_opt(&sbi->opt, DAX_ALWAYS) && test_opt(&sbi->opt, INODE_SHARE)) { + errorfc(fc, "FSDAX is not allowed when inode_share is on"); + return -EINVAL; + } + + sbi->blkszbits = PAGE_SHIFT; + if (erofs_is_fileio_mode(sbi)) { + /* + * (File-backed mounts) EROFS claims it's safe to nest other + * fs contexts (including its own) due to self-controlled RO + * accesses/contexts and no side-effect changes that need to + * context save & restore so it can reuse the current thread + * context. + * However, we still need to prevent kernel stack overflow due + * to filesystem nesting: just ensure that s_stack_depth is 0 + * to disallow mounting EROFS on stacked filesystems. + * Note: s_stack_depth is not incremented here for now, since + * EROFS is the only fs supporting file-backed mounts for now. + * It MUST change if another fs plans to support them, which + * may also require adjusting FILESYSTEM_MAX_STACK_DEPTH. + */ + inode = file_inode(sbi->dif0.file); + if ((inode->i_sb->s_op == &erofs_sops && + !inode->i_sb->s_bdev) || inode->i_sb->s_stack_depth) { + erofs_err(sb, "file-backed mounts cannot be applied to stacked fses"); + return -ENOTBLK; + } + sb->s_blocksize = PAGE_SIZE; + sb->s_blocksize_bits = PAGE_SHIFT; + + err = super_setup_bdi(sb); + if (err) + return err; + err = erofs_setup_managed_cache(sb); + if (err) + return err; + + snprintf(sb->s_id, sizeof(sb->s_id), + "%u:%u", MAJOR(sb->s_dev), MINOR(sb->s_dev)); + } else { + if (!sb_set_blocksize(sb, PAGE_SIZE)) { + errorfc(fc, "failed to set initial blksize"); + return -EINVAL; + } + sbi->dif0.dax_dev = fs_dax_get_by_bdev(sb->s_bdev, + &sbi->dif0.dax_part_off, NULL, NULL); + } + + err = erofs_read_superblock(sb); + if (err) + return err; + + if (sb->s_blocksize_bits != sbi->blkszbits) { + if (erofs_is_fileio_mode(sbi)) { + sb->s_blocksize = 1 << sbi->blkszbits; + sb->s_blocksize_bits = sbi->blkszbits; + } else if (!sb_set_blocksize(sb, 1 << sbi->blkszbits)) { + errorfc(fc, "failed to set erofs blksize"); + return -EINVAL; + } + } + if (sbi->dif0.fsoff & (sb->s_blocksize - 1)) + return invalfc(fc, "fsoffset %llu is not aligned to block size %lu", + sbi->dif0.fsoff, sb->s_blocksize); + + if (test_opt(&sbi->opt, DAX_ALWAYS) && sbi->blkszbits != PAGE_SHIFT) { + erofs_info(sb, "unsupported blocksize for DAX"); + clear_opt(&sbi->opt, DAX_ALWAYS); + } + if (test_opt(&sbi->opt, INODE_SHARE) && !erofs_sb_has_ishare_xattrs(sbi)) { + erofs_info(sb, "on-disk ishare xattrs not found. Turning off inode_share."); + clear_opt(&sbi->opt, INODE_SHARE); + } + if (test_opt(&sbi->opt, INODE_SHARE)) + erofs_info(sb, "EXPERIMENTAL EROFS page cache share support in use. Use at your own risk!"); + + sb->s_time_gran = 1; + sb->s_xattr = erofs_xattr_handlers; + sb->s_export_op = &erofs_export_ops; + + if (test_opt(&sbi->opt, POSIX_ACL)) + sb->s_flags |= SB_POSIXACL; + else + sb->s_flags &= ~SB_POSIXACL; + + err = z_erofs_init_super(sb); + if (err) + return err; + + if (erofs_sb_has_fragments(sbi) && sbi->packed_nid) { + inode = erofs_iget(sb, sbi->packed_nid); + if (IS_ERR(inode)) + return PTR_ERR(inode); + sbi->packed_inode = inode; + } + if (erofs_sb_has_metabox(sbi)) { + inode = erofs_iget(sb, sbi->metabox_nid); + if (IS_ERR(inode)) + return PTR_ERR(inode); + sbi->metabox_inode = inode; + } + + inode = erofs_iget(sb, sbi->root_nid); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + if (!S_ISDIR(inode->i_mode)) { + erofs_err(sb, "rootino(nid %llu) is not a directory(i_mode %o)", + sbi->root_nid, inode->i_mode); + iput(inode); + return -EINVAL; + } + sb->s_root = d_make_root(inode); + if (!sb->s_root) + return -ENOMEM; + + erofs_shrinker_register(sb); + err = erofs_xattr_prefixes_init(sb); + if (err) + return err; + + super_set_sysfs_name_id(sb); + err = erofs_register_sysfs(sb); + if (err) + return err; + + sbi->dir_ra_bytes = EROFS_DIR_RA_BYTES; + erofs_info(sb, "mounted with root inode @ nid %llu.", sbi->root_nid); + return 0; +} + +static int erofs_fc_get_tree(struct fs_context *fc) +{ + struct erofs_sb_info *sbi = fc->s_fs_info; + int ret; + + if (sbi->dif0.file) { + if (!IS_ENABLED(CONFIG_EROFS_FS_BACKED_BY_FILE)) { + errorfc(fc, "source fd option not supported"); + return -EINVAL; + } + if (!S_ISREG(file_inode(sbi->dif0.file)->i_mode) || + !sbi->dif0.file->f_mapping->a_ops->read_folio) { + errorfc(fc, "source is unsupported"); + return -EINVAL; + } + return get_tree_nodev(fc, erofs_fc_fill_super); + } + + ret = get_tree_bdev_flags(fc, erofs_fc_fill_super, + IS_ENABLED(CONFIG_EROFS_FS_BACKED_BY_FILE) ? + GET_TREE_BDEV_QUIET_LOOKUP : 0); + if (IS_ENABLED(CONFIG_EROFS_FS_BACKED_BY_FILE) && ret == -ENOTBLK) { + struct file *file; + + if (!fc->source) + return invalf(fc, "No source specified"); + file = filp_open(fc->source, O_RDONLY | O_LARGEFILE, 0); + if (IS_ERR(file)) + return PTR_ERR(file); + sbi->dif0.file = file; + + if (S_ISREG(file_inode(sbi->dif0.file)->i_mode) && + sbi->dif0.file->f_mapping->a_ops->read_folio) + return get_tree_nodev(fc, erofs_fc_fill_super); + } + return ret; +} + +static int erofs_fc_reconfigure(struct fs_context *fc) +{ + struct super_block *sb = fc->root->d_sb; + struct erofs_sb_info *sbi = EROFS_SB(sb); + struct erofs_sb_info *new_sbi = fc->s_fs_info; + + DBG_BUGON(!sb_rdonly(sb)); + + if (new_sbi->domain_id) + erofs_info(sb, "ignoring reconfiguration for domain_id."); + + if (test_opt(&new_sbi->opt, POSIX_ACL)) + fc->sb_flags |= SB_POSIXACL; + else + fc->sb_flags &= ~SB_POSIXACL; + + sbi->opt = new_sbi->opt; + + fc->sb_flags |= SB_RDONLY; + return 0; +} + +static int erofs_release_device_info(int id, void *ptr, void *data) +{ + struct super_block *sb = data; + struct erofs_device_info *dif = ptr; + + fs_put_dax(dif->dax_dev, NULL); + if (dif->file) { + if (S_ISBLK(file_inode(dif->file)->i_mode)) + fs_bdev_file_release(dif->file, sb); + else + fput(dif->file); + } + kfree(dif->path); + kfree(dif); + return 0; +} + +static void erofs_free_dev_context(struct erofs_dev_context *devs, + struct super_block *sb) +{ + if (!devs) + return; + idr_for_each(&devs->tree, &erofs_release_device_info, sb); + idr_destroy(&devs->tree); + kfree(devs); +} + +static void erofs_sb_free(struct erofs_sb_info *sbi, struct super_block *sb) +{ + erofs_free_dev_context(sbi->devs, sb); + kfree_sensitive(sbi->domain_id); + if (sbi->dif0.file) + fput(sbi->dif0.file); + kfree(sbi->volume_name); + kfree(sbi); +} + +static void erofs_fc_free(struct fs_context *fc) +{ + struct erofs_sb_info *sbi = fc->s_fs_info; + + /* + * Freed here only if an error occurs before the sb is set up; at that + * point no block-backed device has been claimed (that happens in + * fill_super), so the NULL sb never reaches fs_bdev_file_release(). + */ + if (sbi) + erofs_sb_free(sbi, NULL); +} + +static const struct fs_context_operations erofs_context_ops = { + .parse_param = erofs_fc_parse_param, + .get_tree = erofs_fc_get_tree, + .reconfigure = erofs_fc_reconfigure, + .free = erofs_fc_free, +}; + +static int erofs_init_fs_context(struct fs_context *fc) +{ + struct erofs_sb_info *sbi; + + sbi = kzalloc_obj(*sbi); + if (!sbi) + return -ENOMEM; + + sbi->devs = kzalloc_obj(struct erofs_dev_context); + if (!sbi->devs) { + kfree(sbi); + return -ENOMEM; + } + fc->s_fs_info = sbi; + + idr_init(&sbi->devs->tree); + init_rwsem(&sbi->devs->rwsem); + erofs_default_options(sbi); + fc->ops = &erofs_context_ops; + return 0; +} + +static void erofs_drop_internal_inodes(struct erofs_sb_info *sbi) +{ + iput(sbi->packed_inode); + sbi->packed_inode = NULL; + iput(sbi->metabox_inode); + sbi->metabox_inode = NULL; + iput(sbi->managed_cache); + sbi->managed_cache = NULL; +} + +static void erofs_kill_sb(struct super_block *sb) +{ + struct erofs_sb_info *sbi = EROFS_SB(sb); + + if (sbi->dif0.file) + kill_anon_super(sb); + else + kill_block_super(sb); + erofs_drop_internal_inodes(sbi); + fs_put_dax(sbi->dif0.dax_dev, NULL); + erofs_sb_free(sbi, sb); + sb->s_fs_info = NULL; +} + +static void erofs_put_super(struct super_block *sb) +{ + struct erofs_sb_info *const sbi = EROFS_SB(sb); + + erofs_unregister_sysfs(sb); + erofs_shrinker_unregister(sb); + erofs_xattr_prefixes_cleanup(sb); + erofs_drop_internal_inodes(sbi); + erofs_free_dev_context(sbi->devs, sb); + sbi->devs = NULL; +} + +static struct file_system_type erofs_fs_type = { + .owner = THIS_MODULE, + .name = "erofs", + .init_fs_context = erofs_init_fs_context, + .kill_sb = erofs_kill_sb, + .fs_flags = FS_REQUIRES_DEV | FS_ALLOW_IDMAP, +}; +MODULE_ALIAS_FS("erofs"); + +#ifdef CONFIG_EROFS_FS_PAGE_CACHE_SHARE +static void erofs_free_anon_inode(struct inode *inode) +{ + struct erofs_inode *vi = EROFS_I(inode); + + kfree(vi->fingerprint.opaque); + kmem_cache_free(erofs_inode_cachep, vi); +} + +static const struct super_operations erofs_anon_sops = { + .alloc_inode = erofs_alloc_inode, + .drop_inode = inode_just_drop, + .free_inode = erofs_free_anon_inode, +}; + +static int erofs_anon_init_fs_context(struct fs_context *fc) +{ + struct pseudo_fs_context *ctx; + + ctx = init_pseudo(fc, EROFS_SUPER_MAGIC); + if (!ctx) + return -ENOMEM; + ctx->ops = &erofs_anon_sops; + return 0; +} + +struct file_system_type erofs_anon_fs_type = { + .name = "pseudo_erofs", + .init_fs_context = erofs_anon_init_fs_context, + .kill_sb = kill_anon_super, +}; +#endif + +static int __init erofs_module_init(void) +{ + int err; + + erofs_check_ondisk_layout_definitions(); + + erofs_inode_cachep = kmem_cache_create("erofs_inode", + sizeof(struct erofs_inode), 0, + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, + erofs_inode_init_once); + if (!erofs_inode_cachep) + return -ENOMEM; + + err = erofs_init_shrinker(); + if (err) + goto shrinker_err; + + err = z_erofs_init_subsystem(); + if (err) + goto zip_err; + + err = erofs_init_sysfs(); + if (err) + goto sysfs_err; + + err = erofs_init_ishare(); + if (err) + goto ishare_err; + + err = register_filesystem(&erofs_fs_type); + if (err) + goto fs_err; + + return 0; + +fs_err: + erofs_exit_ishare(); +ishare_err: + erofs_exit_sysfs(); +sysfs_err: + z_erofs_exit_subsystem(); +zip_err: + erofs_exit_shrinker(); +shrinker_err: + kmem_cache_destroy(erofs_inode_cachep); + return err; +} + +static void __exit erofs_module_exit(void) +{ + unregister_filesystem(&erofs_fs_type); + erofs_exit_ishare(); + + /* ensure all delayed rcu free inodes & pclusters are flushed */ + rcu_barrier(); + + erofs_exit_sysfs(); + z_erofs_exit_subsystem(); + erofs_exit_shrinker(); + kmem_cache_destroy(erofs_inode_cachep); +} + +static int erofs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + struct erofs_sb_info *sbi = EROFS_SB(sb); + + buf->f_type = sb->s_magic; + buf->f_bsize = sb->s_blocksize; + buf->f_blocks = sbi->total_blocks; + buf->f_bfree = buf->f_bavail = 0; + buf->f_files = ULLONG_MAX; + buf->f_ffree = ULLONG_MAX - sbi->inos; + buf->f_namelen = EROFS_NAME_LEN; + + if (uuid_is_null(&sb->s_uuid)) + buf->f_fsid = u64_to_fsid(!sb->s_bdev ? 0 : + huge_encode_dev(sb->s_bdev->bd_dev)); + else + buf->f_fsid = uuid_to_fsid(sb->s_uuid.b); + return 0; +} + +static int erofs_show_options(struct seq_file *seq, struct dentry *root) +{ + struct erofs_sb_info *sbi = EROFS_SB(root->d_sb); + struct erofs_mount_opts *opt = &sbi->opt; + + if (IS_ENABLED(CONFIG_EROFS_FS_XATTR)) + seq_puts(seq, test_opt(opt, XATTR_USER) ? + ",user_xattr" : ",nouser_xattr"); + if (IS_ENABLED(CONFIG_EROFS_FS_POSIX_ACL)) + seq_puts(seq, test_opt(opt, POSIX_ACL) ? ",acl" : ",noacl"); + if (IS_ENABLED(CONFIG_EROFS_FS_ZIP) && sbi->available_compr_algs) + seq_printf(seq, ",cache_strategy=%s", + erofs_param_cache_strategy[opt->cache_strategy].name); + if (test_opt(opt, DAX_ALWAYS)) + seq_puts(seq, ",dax=always"); + if (test_opt(opt, DAX_NEVER)) + seq_puts(seq, ",dax=never"); + if (erofs_is_fileio_mode(sbi) && test_opt(opt, DIRECT_IO)) + seq_puts(seq, ",directio"); + if (sbi->dif0.fsoff) + seq_printf(seq, ",fsoffset=%llu", sbi->dif0.fsoff); + if (test_opt(opt, INODE_SHARE)) + seq_puts(seq, ",inode_share"); + return 0; +} + +static void erofs_evict_inode(struct inode *inode) +{ + if (IS_DAX(inode)) + dax_break_layout_final(inode); + erofs_ishare_free_inode(inode); + truncate_inode_pages_final(&inode->i_data); + clear_inode(inode); +} + +const struct super_operations erofs_sops = { + .put_super = erofs_put_super, + .alloc_inode = erofs_alloc_inode, + .free_inode = erofs_free_inode, + .evict_inode = erofs_evict_inode, + .statfs = erofs_statfs, + .show_options = erofs_show_options, +}; + +module_init(erofs_module_init); +module_exit(erofs_module_exit); + +MODULE_DESCRIPTION("Enhanced ROM File System"); +MODULE_AUTHOR("Gao Xiang, Chao Yu, Miao Xie, CONSUMER BG, HUAWEI Inc."); +MODULE_LICENSE("GPL"); diff --git a/fs/erofs/sysfs.c b/fs/erofs/sysfs.c new file mode 100644 index 000000000..6734483a4 --- /dev/null +++ b/fs/erofs/sysfs.c @@ -0,0 +1,302 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C), 2008-2021, OPPO Mobile Comm Corp., Ltd. + * https://www.oppo.com/ + */ +#include +#include + +#include "internal.h" +#include "compress.h" + +enum { + attr_feature, + attr_drop_caches, + attr_pointer_ui, + attr_pointer_bool, + attr_accel, +}; + +enum { + struct_erofs_sb_info, + struct_erofs_mount_opts, +}; + +struct erofs_attr { + struct attribute attr; + short attr_id; + int struct_type, offset; +}; + +#define EROFS_ATTR(_name, _mode, _id) \ +static struct erofs_attr erofs_attr_##_name = { \ + .attr = {.name = __stringify(_name), .mode = _mode }, \ + .attr_id = attr_##_id, \ +} +#define EROFS_ATTR_FUNC(_name, _mode) EROFS_ATTR(_name, _mode, _name) +#define EROFS_ATTR_FEATURE(_name) EROFS_ATTR(_name, 0444, feature) + +#define EROFS_ATTR_OFFSET(_name, _mode, _id, _struct) \ +static struct erofs_attr erofs_attr_##_name = { \ + .attr = {.name = __stringify(_name), .mode = _mode }, \ + .attr_id = attr_##_id, \ + .struct_type = struct_##_struct, \ + .offset = offsetof(struct _struct, _name),\ +} + +#define EROFS_ATTR_RW(_name, _id, _struct) \ + EROFS_ATTR_OFFSET(_name, 0644, _id, _struct) + +#define EROFS_RO_ATTR(_name, _id, _struct) \ + EROFS_ATTR_OFFSET(_name, 0444, _id, _struct) + +#define EROFS_ATTR_RW_UI(_name, _struct) \ + EROFS_ATTR_RW(_name, pointer_ui, _struct) + +#define EROFS_ATTR_RW_BOOL(_name, _struct) \ + EROFS_ATTR_RW(_name, pointer_bool, _struct) + +#define ATTR_LIST(name) (&erofs_attr_##name.attr) + +#ifdef CONFIG_EROFS_FS_ZIP +EROFS_ATTR_RW_UI(sync_decompress, erofs_sb_info); +EROFS_ATTR_FUNC(drop_caches, 0200); +#endif +#ifdef CONFIG_EROFS_FS_ZIP_ACCEL +EROFS_ATTR_FUNC(accel, 0644); +#endif +EROFS_ATTR_RW_UI(dir_ra_bytes, erofs_sb_info); + +static struct attribute *erofs_sb_attrs[] = { +#ifdef CONFIG_EROFS_FS_ZIP + ATTR_LIST(sync_decompress), + ATTR_LIST(drop_caches), +#endif + ATTR_LIST(dir_ra_bytes), + NULL, +}; +ATTRIBUTE_GROUPS(erofs_sb); + +static struct attribute *erofs_attrs[] = { +#ifdef CONFIG_EROFS_FS_ZIP_ACCEL + ATTR_LIST(accel), +#endif + NULL, +}; +ATTRIBUTE_GROUPS(erofs); + +/* Features this copy of erofs supports */ +EROFS_ATTR_FEATURE(compr_cfgs); +EROFS_ATTR_FEATURE(big_pcluster); +EROFS_ATTR_FEATURE(chunked_file); +EROFS_ATTR_FEATURE(device_table); +EROFS_ATTR_FEATURE(compr_head2); +EROFS_ATTR_FEATURE(sb_chksum); +EROFS_ATTR_FEATURE(ztailpacking); +EROFS_ATTR_FEATURE(fragments); +EROFS_ATTR_FEATURE(dedupe); +EROFS_ATTR_FEATURE(48bit); +EROFS_ATTR_FEATURE(metabox); + +static struct attribute *erofs_feat_attrs[] = { + ATTR_LIST(compr_cfgs), + ATTR_LIST(big_pcluster), + ATTR_LIST(chunked_file), + ATTR_LIST(device_table), + ATTR_LIST(compr_head2), + ATTR_LIST(sb_chksum), + ATTR_LIST(ztailpacking), + ATTR_LIST(fragments), + ATTR_LIST(dedupe), + ATTR_LIST(48bit), + ATTR_LIST(metabox), + NULL, +}; +ATTRIBUTE_GROUPS(erofs_feat); + +static unsigned char *__struct_ptr(struct erofs_sb_info *sbi, + int struct_type, int offset) +{ + if (struct_type == struct_erofs_sb_info) + return (unsigned char *)sbi + offset; + if (struct_type == struct_erofs_mount_opts) + return (unsigned char *)&sbi->opt + offset; + return NULL; +} + +static ssize_t erofs_attr_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct erofs_sb_info *sbi = container_of(kobj, struct erofs_sb_info, + s_kobj); + struct erofs_attr *a = container_of(attr, struct erofs_attr, attr); + unsigned char *ptr = __struct_ptr(sbi, a->struct_type, a->offset); + + switch (a->attr_id) { + case attr_feature: + return sysfs_emit(buf, "supported\n"); + case attr_pointer_ui: + if (!ptr) + return 0; + return sysfs_emit(buf, "%u\n", *(unsigned int *)ptr); + case attr_pointer_bool: + if (!ptr) + return 0; + return sysfs_emit(buf, "%d\n", *(bool *)ptr); + case attr_accel: + return z_erofs_crypto_show_engines(buf, PAGE_SIZE, '\n'); + } + return 0; +} + +static ssize_t erofs_attr_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t len) +{ + struct erofs_sb_info *sbi = container_of(kobj, struct erofs_sb_info, + s_kobj); + struct erofs_attr *a = container_of(attr, struct erofs_attr, attr); + unsigned char *ptr = __struct_ptr(sbi, a->struct_type, a->offset); + unsigned long t; + int ret; + + switch (a->attr_id) { + case attr_pointer_ui: + if (!ptr) + return 0; + ret = kstrtoul(skip_spaces(buf), 0, &t); + if (ret) + return ret; + if (t != (unsigned int)t) + return -ERANGE; + if (IS_ENABLED(CONFIG_EROFS_FS_ZIP) && + !strcmp(a->attr.name, "sync_decompress") && + (t > EROFS_SYNC_DECOMPRESS_FORCE_OFF)) + return -EINVAL; + *(unsigned int *)ptr = t; + return len; + case attr_pointer_bool: + if (!ptr) + return 0; + ret = kstrtoul(skip_spaces(buf), 0, &t); + if (ret) + return ret; + if (t != 0 && t != 1) + return -EINVAL; + *(bool *)ptr = !!t; + return len; +#ifdef CONFIG_EROFS_FS_ZIP + case attr_drop_caches: + ret = kstrtoul(skip_spaces(buf), 0, &t); + if (ret) + return ret; + if (t < 1 || t > 3) + return -EINVAL; + + if (t & 2) + z_erofs_shrink_scan(sbi, ~0UL); + if (t & 1) + invalidate_mapping_pages(MNGD_MAPPING(sbi), 0, -1); + return len; +#endif +#ifdef CONFIG_EROFS_FS_ZIP_ACCEL + case attr_accel: + buf = skip_spaces(buf); + z_erofs_crypto_disable_all_engines(); + while (*buf) { + t = strcspn(buf, "\n"); + ret = z_erofs_crypto_enable_engine(buf, t); + if (ret < 0) + return ret; + buf += buf[t] != '\0' ? t + 1 : t; + } + return len; +#endif + } + return 0; +} + +static void erofs_sb_release(struct kobject *kobj) +{ + struct erofs_sb_info *sbi = container_of(kobj, struct erofs_sb_info, + s_kobj); + complete(&sbi->s_kobj_unregister); +} + +static const struct sysfs_ops erofs_attr_ops = { + .show = erofs_attr_show, + .store = erofs_attr_store, +}; + +static const struct kobj_type erofs_sb_ktype = { + .default_groups = erofs_sb_groups, + .sysfs_ops = &erofs_attr_ops, + .release = erofs_sb_release, +}; + +static const struct kobj_type erofs_ktype = { + .default_groups = erofs_groups, + .sysfs_ops = &erofs_attr_ops, +}; + +static struct kset erofs_root = { + .kobj = {.ktype = &erofs_ktype}, +}; + +static const struct kobj_type erofs_feat_ktype = { + .default_groups = erofs_feat_groups, + .sysfs_ops = &erofs_attr_ops, +}; + +static struct kobject erofs_feat = { + .kset = &erofs_root, +}; + +int erofs_register_sysfs(struct super_block *sb) +{ + struct erofs_sb_info *sbi = EROFS_SB(sb); + int err; + + sbi->s_kobj.kset = &erofs_root; + init_completion(&sbi->s_kobj_unregister); + err = kobject_init_and_add(&sbi->s_kobj, &erofs_sb_ktype, NULL, "%s", + sb->s_sysfs_name); + if (err) { + kobject_put(&sbi->s_kobj); + wait_for_completion(&sbi->s_kobj_unregister); + } + return err; +} + +void erofs_unregister_sysfs(struct super_block *sb) +{ + struct erofs_sb_info *sbi = EROFS_SB(sb); + + if (sbi->s_kobj.state_in_sysfs) { + kobject_del(&sbi->s_kobj); + kobject_put(&sbi->s_kobj); + wait_for_completion(&sbi->s_kobj_unregister); + } +} + +void erofs_exit_sysfs(void) +{ + kobject_put(&erofs_feat); + kset_unregister(&erofs_root); +} + +int __init erofs_init_sysfs(void) +{ + int ret; + + kobject_set_name(&erofs_root.kobj, "erofs"); + erofs_root.kobj.parent = fs_kobj; + ret = kset_register(&erofs_root); + if (!ret) { + ret = kobject_init_and_add(&erofs_feat, &erofs_feat_ktype, + NULL, "features"); + if (!ret) + return 0; + erofs_exit_sysfs(); + } + return ret; +} diff --git a/fs/erofs/xattr.c b/fs/erofs/xattr.c new file mode 100644 index 000000000..df7ea0195 --- /dev/null +++ b/fs/erofs/xattr.c @@ -0,0 +1,649 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2021-2022, Alibaba Cloud + */ +#include +#include +#include "xattr.h" + +struct erofs_xattr_iter { + struct super_block *sb; + struct erofs_buf buf; + erofs_off_t pos; + void *kaddr; + + char *buffer; + int buffer_size, buffer_ofs; + + /* getxattr */ + int index, infix_len; + struct qstr name; + + /* listxattr */ + struct dentry *dentry; +}; + +static const char *erofs_xattr_prefix(unsigned int idx, struct dentry *dentry); + +static int erofs_init_inode_xattrs(struct inode *inode) +{ + struct erofs_buf buf = __EROFS_BUF_INITIALIZER; + struct erofs_inode *vi = EROFS_I(inode); + struct super_block *sb = inode->i_sb; + const struct erofs_xattr_ibody_header *ih; + __le32 *xattr_id; + erofs_off_t pos; + unsigned int i; + int ret = 0; + + if (!vi->xattr_isize) + return -ENODATA; + + /* the most case is that xattrs of this inode are initialized. */ + if (test_bit(EROFS_I_EA_INITED_BIT, &vi->flags)) { + /* + * paired with smp_mb() at the end of the function to ensure + * fields will only be observed after the bit is set. + */ + smp_mb(); + return 0; + } + if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_XATTR_BIT, TASK_KILLABLE)) + return -ERESTARTSYS; + + /* someone has initialized xattrs for us? */ + if (test_bit(EROFS_I_EA_INITED_BIT, &vi->flags)) + goto out_unlock; + + /* + * bypass all xattr operations if ->xattr_isize is not greater than + * sizeof(struct erofs_xattr_ibody_header), in detail: + * 1) it is not enough to contain erofs_xattr_ibody_header then + * ->xattr_isize should be 0 (it means no xattr); + * 2) it is just to contain erofs_xattr_ibody_header, which is on-disk + * undefined right now (maybe use later with some new sb feature). + */ + if (vi->xattr_isize == sizeof(struct erofs_xattr_ibody_header)) { + erofs_err(sb, "xattr_isize %d of nid %llu is not supported yet", + vi->xattr_isize, vi->nid); + ret = -EOPNOTSUPP; + goto out_unlock; + } else if (vi->xattr_isize < sizeof(struct erofs_xattr_ibody_header)) { + erofs_err(sb, "bogus xattr ibody @ nid %llu", vi->nid); + DBG_BUGON(1); + ret = -EFSCORRUPTED; + goto out_unlock; + } + + pos = erofs_iloc(inode) + vi->inode_isize; + ih = erofs_read_metabuf(&buf, sb, pos, erofs_inode_in_metabox(inode)); + if (IS_ERR(ih)) { + ret = PTR_ERR(ih); + goto out_unlock; + } + vi->xattr_name_filter = le32_to_cpu(ih->h_name_filter); + vi->xattr_shared_count = ih->h_shared_count; + if ((u32)vi->xattr_shared_count * sizeof(__le32) > + vi->xattr_isize - sizeof(struct erofs_xattr_ibody_header)) { + erofs_err(sb, "invalid h_shared_count %u @ nid %llu", + vi->xattr_shared_count, vi->nid); + ret = -EFSCORRUPTED; + goto out_unlock; + } + vi->xattr_shared_xattrs = kmalloc_objs(uint, vi->xattr_shared_count); + if (!vi->xattr_shared_xattrs) { + ret = -ENOMEM; + goto out_unlock; + } + + /* skip the ibody header and read the shared xattr array */ + pos += sizeof(struct erofs_xattr_ibody_header); + for (i = 0; i < vi->xattr_shared_count; ++i) { + xattr_id = erofs_bread(&buf, pos + i * sizeof(__le32), true); + if (IS_ERR(xattr_id)) { + kfree(vi->xattr_shared_xattrs); + vi->xattr_shared_xattrs = NULL; + ret = PTR_ERR(xattr_id); + goto out_unlock; + } + vi->xattr_shared_xattrs[i] = le32_to_cpu(*xattr_id); + } + + /* paired with smp_mb() at the beginning of the function. */ + smp_mb(); + set_bit(EROFS_I_EA_INITED_BIT, &vi->flags); +out_unlock: + erofs_put_metabuf(&buf); + clear_and_wake_up_bit(EROFS_I_BL_XATTR_BIT, &vi->flags); + return ret; +} + +static int erofs_xattr_copy_to_buffer(struct erofs_xattr_iter *it, + unsigned int len) +{ + unsigned int slice, processed; + struct super_block *sb = it->sb; + + for (processed = 0; processed < len; processed += slice) { + it->kaddr = erofs_bread(&it->buf, it->pos, true); + if (IS_ERR(it->kaddr)) + return PTR_ERR(it->kaddr); + + slice = min_t(unsigned int, sb->s_blocksize - + erofs_blkoff(sb, it->pos), len - processed); + memcpy(it->buffer + it->buffer_ofs, it->kaddr, slice); + it->buffer_ofs += slice; + it->pos += slice; + } + return 0; +} + +static int erofs_listxattr_foreach(struct erofs_xattr_iter *it) +{ + struct erofs_xattr_entry entry; + unsigned int base_index, name_total, prefix_len, infix_len = 0; + const char *prefix, *infix = NULL; + int err; + + /* 1. handle xattr entry */ + entry = *(struct erofs_xattr_entry *)it->kaddr; + it->pos += sizeof(struct erofs_xattr_entry); + + base_index = entry.e_name_index; + if (entry.e_name_index & EROFS_XATTR_LONG_PREFIX) { + struct erofs_sb_info *sbi = EROFS_SB(it->sb); + struct erofs_xattr_prefix_item *pf = sbi->xattr_prefixes + + (entry.e_name_index & EROFS_XATTR_LONG_PREFIX_MASK); + + if (pf >= sbi->xattr_prefixes + sbi->xattr_prefix_count) + return 0; + infix = pf->prefix->infix; + infix_len = pf->infix_len; + base_index = pf->prefix->base_index; + } + + prefix = erofs_xattr_prefix(base_index, it->dentry); + if (!prefix) + return 0; + prefix_len = strlen(prefix); + name_total = prefix_len + infix_len + entry.e_name_len + 1; + + if (!it->buffer) { + it->buffer_ofs += name_total; + return 0; + } + + if (it->buffer_ofs + name_total > it->buffer_size) + return -ERANGE; + + memcpy(it->buffer + it->buffer_ofs, prefix, prefix_len); + memcpy(it->buffer + it->buffer_ofs + prefix_len, infix, infix_len); + it->buffer_ofs += prefix_len + infix_len; + + /* 2. handle xattr name */ + err = erofs_xattr_copy_to_buffer(it, entry.e_name_len); + if (err) + return err; + + it->buffer[it->buffer_ofs++] = '\0'; + return 0; +} + +static int erofs_getxattr_foreach(struct erofs_xattr_iter *it) +{ + struct super_block *sb = it->sb; + struct erofs_xattr_entry entry; + unsigned int slice, processed, value_sz; + + /* 1. handle xattr entry */ + entry = *(struct erofs_xattr_entry *)it->kaddr; + it->pos += sizeof(struct erofs_xattr_entry); + value_sz = le16_to_cpu(entry.e_value_size); + + /* should also match the infix for long name prefixes */ + if (entry.e_name_index & EROFS_XATTR_LONG_PREFIX) { + struct erofs_sb_info *sbi = EROFS_SB(sb); + struct erofs_xattr_prefix_item *pf = sbi->xattr_prefixes + + (entry.e_name_index & EROFS_XATTR_LONG_PREFIX_MASK); + + if (pf >= sbi->xattr_prefixes + sbi->xattr_prefix_count) + return -ENODATA; + + if (it->index != pf->prefix->base_index || + it->name.len != entry.e_name_len + pf->infix_len) + return -ENODATA; + + if (memcmp(it->name.name, pf->prefix->infix, pf->infix_len)) + return -ENODATA; + + it->infix_len = pf->infix_len; + } else { + if (it->index != entry.e_name_index || + it->name.len != entry.e_name_len) + return -ENODATA; + + it->infix_len = 0; + } + + /* 2. handle xattr name */ + for (processed = 0; processed < entry.e_name_len; processed += slice) { + it->kaddr = erofs_bread(&it->buf, it->pos, true); + if (IS_ERR(it->kaddr)) + return PTR_ERR(it->kaddr); + + slice = min_t(unsigned int, + sb->s_blocksize - erofs_blkoff(sb, it->pos), + entry.e_name_len - processed); + if (memcmp(it->name.name + it->infix_len + processed, + it->kaddr, slice)) + return -ENODATA; + it->pos += slice; + } + + /* 3. handle xattr value */ + if (!it->buffer) { + it->buffer_ofs = value_sz; + return 0; + } + + if (it->buffer_size < value_sz) + return -ERANGE; + + return erofs_xattr_copy_to_buffer(it, value_sz); +} + +static int erofs_xattr_iter_inline(struct erofs_xattr_iter *it, + struct inode *inode, bool getxattr) +{ + struct erofs_inode *const vi = EROFS_I(inode); + unsigned int xattr_header_sz, remaining, entry_sz; + erofs_off_t next_pos; + int ret; + + xattr_header_sz = sizeof(struct erofs_xattr_ibody_header) + + sizeof(u32) * vi->xattr_shared_count; + if (xattr_header_sz >= vi->xattr_isize) { + DBG_BUGON(xattr_header_sz > vi->xattr_isize); + return -ENODATA; + } + + ret = erofs_init_metabuf(&it->buf, it->sb, erofs_inode_in_metabox(inode)); + if (ret) + return ret; + remaining = vi->xattr_isize - xattr_header_sz; + it->pos = erofs_iloc(inode) + vi->inode_isize + xattr_header_sz; + + while (remaining) { + it->kaddr = erofs_bread(&it->buf, it->pos, true); + if (IS_ERR(it->kaddr)) + return PTR_ERR(it->kaddr); + + entry_sz = erofs_xattr_entry_size(it->kaddr); + /* xattr on-disk corruption: xattr entry beyond xattr_isize */ + if (remaining < entry_sz) { + DBG_BUGON(1); + return -EFSCORRUPTED; + } + remaining -= entry_sz; + next_pos = it->pos + entry_sz; + + if (getxattr) + ret = erofs_getxattr_foreach(it); + else + ret = erofs_listxattr_foreach(it); + if ((getxattr && ret != -ENODATA) || (!getxattr && ret)) + break; + + it->pos = next_pos; + } + return ret; +} + +static int erofs_xattr_iter_shared(struct erofs_xattr_iter *it, + struct inode *inode, bool getxattr) +{ + struct erofs_inode *const vi = EROFS_I(inode); + struct super_block *const sb = it->sb; + struct erofs_sb_info *sbi = EROFS_SB(sb); + unsigned int i = 0; + int ret; + + ret = erofs_init_metabuf(&it->buf, sb, + erofs_sb_has_shared_ea_in_metabox(sbi)); + if (ret) + return ret; + + while (i < vi->xattr_shared_count) { + it->pos = erofs_pos(sb, sbi->xattr_blkaddr) + + vi->xattr_shared_xattrs[i++] * sizeof(__le32); + it->kaddr = erofs_bread(&it->buf, it->pos, true); + if (IS_ERR(it->kaddr)) + return PTR_ERR(it->kaddr); + + if (getxattr) + ret = erofs_getxattr_foreach(it); + else + ret = erofs_listxattr_foreach(it); + if ((getxattr && ret != -ENODATA) || (!getxattr && ret)) + break; + } + return i ? ret : -ENODATA; +} + +static int erofs_getxattr(struct inode *inode, int index, const char *name, + void *buffer, size_t buffer_size) +{ + int ret; + unsigned int hashbit; + struct erofs_xattr_iter it; + struct erofs_inode *vi = EROFS_I(inode); + struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb); + + if (!name) + return -EINVAL; + + ret = erofs_init_inode_xattrs(inode); + if (ret) + return ret; + + /* reserved flag is non-zero if there's any change of on-disk format */ + if (erofs_sb_has_xattr_filter(sbi) && !sbi->xattr_filter_reserved) { + hashbit = xxh32(name, strlen(name), + EROFS_XATTR_FILTER_SEED + index); + hashbit &= EROFS_XATTR_FILTER_BITS - 1; + if (vi->xattr_name_filter & (1U << hashbit)) + return -ENODATA; + } + + it.index = index; + it.name = QSTR(name); + if (it.name.len > EROFS_NAME_LEN) + return -ERANGE; + + it.sb = inode->i_sb; + it.buf = __EROFS_BUF_INITIALIZER; + it.buffer = buffer; + it.buffer_size = buffer_size; + it.buffer_ofs = 0; + + ret = erofs_xattr_iter_inline(&it, inode, true); + if (ret == -ENODATA) + ret = erofs_xattr_iter_shared(&it, inode, true); + erofs_put_metabuf(&it.buf); + return ret ? ret : it.buffer_ofs; +} + +ssize_t erofs_listxattr(struct dentry *dentry, char *buffer, size_t buffer_size) +{ + int ret; + struct erofs_xattr_iter it; + struct inode *inode = d_inode(dentry); + + ret = erofs_init_inode_xattrs(inode); + if (ret == -ENODATA) + return 0; + if (ret) + return ret; + + it.sb = dentry->d_sb; + it.buf = __EROFS_BUF_INITIALIZER; + it.dentry = dentry; + it.buffer = buffer; + it.buffer_size = buffer_size; + it.buffer_ofs = 0; + + ret = erofs_xattr_iter_inline(&it, inode, false); + if (!ret || ret == -ENODATA) + ret = erofs_xattr_iter_shared(&it, inode, false); + if (ret == -ENODATA) + ret = 0; + erofs_put_metabuf(&it.buf); + return ret ? ret : it.buffer_ofs; +} + +static bool erofs_xattr_user_list(struct dentry *dentry) +{ + return test_opt(&EROFS_SB(dentry->d_sb)->opt, XATTR_USER); +} + +static bool erofs_xattr_trusted_list(struct dentry *dentry) +{ + return capable(CAP_SYS_ADMIN); +} + +static int erofs_xattr_generic_get(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + if (handler->flags == EROFS_XATTR_INDEX_USER && + !test_opt(&EROFS_I_SB(inode)->opt, XATTR_USER)) + return -EOPNOTSUPP; + + return erofs_getxattr(inode, handler->flags, name, buffer, size); +} + +static const struct xattr_handler erofs_xattr_user_handler = { + .prefix = XATTR_USER_PREFIX, + .flags = EROFS_XATTR_INDEX_USER, + .list = erofs_xattr_user_list, + .get = erofs_xattr_generic_get, +}; + +static const struct xattr_handler erofs_xattr_trusted_handler = { + .prefix = XATTR_TRUSTED_PREFIX, + .flags = EROFS_XATTR_INDEX_TRUSTED, + .list = erofs_xattr_trusted_list, + .get = erofs_xattr_generic_get, +}; + +#ifdef CONFIG_EROFS_FS_SECURITY +static const struct xattr_handler erofs_xattr_security_handler = { + .prefix = XATTR_SECURITY_PREFIX, + .flags = EROFS_XATTR_INDEX_SECURITY, + .get = erofs_xattr_generic_get, +}; +#endif + +const struct xattr_handler * const erofs_xattr_handlers[] = { + &erofs_xattr_user_handler, + &erofs_xattr_trusted_handler, +#ifdef CONFIG_EROFS_FS_SECURITY + &erofs_xattr_security_handler, +#endif + NULL, +}; + +static const char *erofs_xattr_prefix(unsigned int idx, struct dentry *dentry) +{ + static const struct xattr_handler * const xattr_handler_map[] = { + [EROFS_XATTR_INDEX_USER] = &erofs_xattr_user_handler, +#ifdef CONFIG_EROFS_FS_POSIX_ACL + [EROFS_XATTR_INDEX_POSIX_ACL_ACCESS] = &nop_posix_acl_access, + [EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT] = &nop_posix_acl_default, +#endif + [EROFS_XATTR_INDEX_TRUSTED] = &erofs_xattr_trusted_handler, +#ifdef CONFIG_EROFS_FS_SECURITY + [EROFS_XATTR_INDEX_SECURITY] = &erofs_xattr_security_handler, +#endif + }; + const struct xattr_handler *handler = NULL; + + if (idx && idx < ARRAY_SIZE(xattr_handler_map)) { + handler = xattr_handler_map[idx]; + if (xattr_handler_can_list(handler, dentry)) + return xattr_prefix(handler); + } + return NULL; +} + +void erofs_xattr_prefixes_cleanup(struct super_block *sb) +{ + struct erofs_sb_info *sbi = EROFS_SB(sb); + int i; + + if (sbi->xattr_prefixes) { + for (i = 0; i < sbi->xattr_prefix_count; i++) + kfree(sbi->xattr_prefixes[i].prefix); + kfree(sbi->xattr_prefixes); + sbi->xattr_prefixes = NULL; + } +} + +int erofs_xattr_prefixes_init(struct super_block *sb) +{ + struct erofs_sb_info *sbi = EROFS_SB(sb); + struct erofs_buf buf = __EROFS_BUF_INITIALIZER; + erofs_off_t pos = (erofs_off_t)sbi->xattr_prefix_start << 2; + struct erofs_xattr_prefix_item *pfs; + int ret = 0, i, len; + bool plain = erofs_sb_has_plain_xattr_pfx(sbi); + + if (!sbi->xattr_prefix_count) + return 0; + + pfs = kzalloc_objs(*pfs, sbi->xattr_prefix_count); + if (!pfs) + return -ENOMEM; + + if (!plain) { + if (erofs_sb_has_metabox(sbi)) + (void)erofs_init_metabuf(&buf, sb, true); + else if (sbi->packed_inode) + buf.mapping = sbi->packed_inode->i_mapping; + else + plain = true; + } + if (plain) + (void)erofs_init_metabuf(&buf, sb, false); + + for (i = 0; i < sbi->xattr_prefix_count; i++) { + void *ptr = erofs_read_metadata(sb, &buf, &pos, &len); + + if (IS_ERR(ptr)) { + ret = PTR_ERR(ptr); + break; + } else if (len < sizeof(*pfs->prefix) || + len > EROFS_NAME_LEN + sizeof(*pfs->prefix)) { + kfree(ptr); + ret = -EFSCORRUPTED; + break; + } + pfs[i].prefix = ptr; + pfs[i].infix_len = len - sizeof(struct erofs_xattr_long_prefix); + } + + erofs_put_metabuf(&buf); + if (!ret && erofs_sb_has_ishare_xattrs(sbi)) { + struct erofs_xattr_prefix_item *pf = pfs + sbi->ishare_xattr_prefix_id; + struct erofs_xattr_long_prefix *newpfx; + + newpfx = krealloc(pf->prefix, + sizeof(*newpfx) + pf->infix_len + 1, GFP_KERNEL); + if (newpfx) { + newpfx->infix[pf->infix_len] = '\0'; + pf->prefix = newpfx; + } else { + ret = -ENOMEM; + } + } + sbi->xattr_prefixes = pfs; + if (ret) + erofs_xattr_prefixes_cleanup(sb); + return ret; +} + +#ifdef CONFIG_EROFS_FS_POSIX_ACL +struct posix_acl *erofs_get_acl(struct inode *inode, int type, bool rcu) +{ + struct posix_acl *acl; + int prefix, rc; + char *value = NULL; + + if (rcu) + return ERR_PTR(-ECHILD); + + switch (type) { + case ACL_TYPE_ACCESS: + prefix = EROFS_XATTR_INDEX_POSIX_ACL_ACCESS; + break; + case ACL_TYPE_DEFAULT: + prefix = EROFS_XATTR_INDEX_POSIX_ACL_DEFAULT; + break; + default: + return ERR_PTR(-EINVAL); + } + + rc = erofs_getxattr(inode, prefix, "", NULL, 0); + if (rc > 0) { + value = kmalloc(rc, GFP_KERNEL); + if (!value) + return ERR_PTR(-ENOMEM); + rc = erofs_getxattr(inode, prefix, "", value, rc); + } + + if (rc == -ENODATA) + acl = NULL; + else if (rc < 0) + acl = ERR_PTR(rc); + else + acl = posix_acl_from_xattr(&init_user_ns, value, rc); + kfree(value); + return acl; +} + +bool erofs_inode_has_noacl(struct inode *inode, void *kaddr, unsigned int ofs) +{ + static const unsigned int bitmask = + BIT(21) | /* system.posix_acl_default */ + BIT(30); /* system.posix_acl_access */ + struct erofs_sb_info *sbi = EROFS_I_SB(inode); + const struct erofs_xattr_ibody_header *ih = kaddr + ofs; + + if (EROFS_I(inode)->xattr_isize < sizeof(*ih)) + return true; + + if (erofs_sb_has_xattr_filter(sbi) && !sbi->xattr_filter_reserved && + !check_add_overflow(ofs, sizeof(*ih), &ofs) && + ofs <= i_blocksize(inode)) { + if ((le32_to_cpu(ih->h_name_filter) & bitmask) == bitmask) + return true; + } + return false; +} +#endif + +#ifdef CONFIG_EROFS_FS_PAGE_CACHE_SHARE +int erofs_xattr_fill_inode_fingerprint(struct erofs_inode_fingerprint *fp, + struct inode *inode, const char *domain_id) +{ + struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb); + struct erofs_xattr_prefix_item *prefix; + const char *infix; + int valuelen, base_index; + + if (!test_opt(&sbi->opt, INODE_SHARE)) + return -EOPNOTSUPP; + if (!sbi->xattr_prefixes) + return -EINVAL; + prefix = sbi->xattr_prefixes + sbi->ishare_xattr_prefix_id; + infix = prefix->prefix->infix; + base_index = prefix->prefix->base_index; + valuelen = erofs_getxattr(inode, base_index, infix, NULL, 0); + if (valuelen <= 0 || valuelen > (1 << sbi->blkszbits)) + return -EFSCORRUPTED; + fp->size = valuelen + (domain_id ? strlen(domain_id) : 0); + fp->opaque = kmalloc(fp->size, GFP_KERNEL); + if (!fp->opaque) + return -ENOMEM; + if (valuelen != erofs_getxattr(inode, base_index, infix, + fp->opaque, valuelen)) { + kfree(fp->opaque); + fp->opaque = NULL; + return -EFSCORRUPTED; + } + memcpy(fp->opaque + valuelen, domain_id, fp->size - valuelen); + return 0; +} +#endif diff --git a/fs/erofs/xattr.h b/fs/erofs/xattr.h new file mode 100644 index 000000000..4465f7018 --- /dev/null +++ b/fs/erofs/xattr.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2018 HUAWEI, Inc. + * https://www.huawei.com/ + */ +#ifndef __EROFS_XATTR_H +#define __EROFS_XATTR_H + +#include "internal.h" +#include +#include + +#ifdef CONFIG_EROFS_FS_XATTR +extern const struct xattr_handler * const erofs_xattr_handlers[]; + +int erofs_xattr_prefixes_init(struct super_block *sb); +void erofs_xattr_prefixes_cleanup(struct super_block *sb); +ssize_t erofs_listxattr(struct dentry *, char *, size_t); +#else +static inline int erofs_xattr_prefixes_init(struct super_block *sb) { return 0; } +static inline void erofs_xattr_prefixes_cleanup(struct super_block *sb) {} + +#define erofs_listxattr (NULL) +#define erofs_xattr_handlers (NULL) +#endif /* !CONFIG_EROFS_FS_XATTR */ + +#ifdef CONFIG_EROFS_FS_POSIX_ACL +struct posix_acl *erofs_get_acl(struct inode *inode, int type, bool rcu); +#else +#define erofs_get_acl (NULL) +#endif + +int erofs_xattr_fill_inode_fingerprint(struct erofs_inode_fingerprint *fp, + struct inode *inode, const char *domain_id); +bool erofs_inode_has_noacl(struct inode *inode, void *kaddr, unsigned int ofs); +#endif diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c new file mode 100644 index 000000000..e1e25ca0d --- /dev/null +++ b/fs/erofs/zdata.c @@ -0,0 +1,1935 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2022 Alibaba Cloud + */ +#include "compress.h" +#include +#include +#include + +#define Z_EROFS_MAX_SYNC_DECOMPRESS_BYTES 12288 +#define Z_EROFS_PCLUSTER_MAX_PAGES (Z_EROFS_PCLUSTER_MAX_SIZE / PAGE_SIZE) +#define Z_EROFS_INLINE_BVECS 2 + +struct z_erofs_bvec { + struct page *page; + int offset; + unsigned int end; +}; + +#define __Z_EROFS_BVSET(name, total) \ +struct name { \ + /* point to the next page which contains the following bvecs */ \ + struct page *nextpage; \ + struct z_erofs_bvec bvec[total]; \ +} +__Z_EROFS_BVSET(z_erofs_bvset,); +__Z_EROFS_BVSET(z_erofs_bvset_inline, Z_EROFS_INLINE_BVECS); + +/* + * Structure fields follow one of the following exclusion rules. + * + * I: Modifiable by initialization/destruction paths and read-only + * for everyone else; + * + * L: Field should be protected by the pcluster lock; + * + * A: Field should be accessed / updated in atomic for parallelized code. + */ +struct z_erofs_pcluster { + struct mutex lock; + struct lockref lockref; + + /* A: point to next chained pcluster or TAILs */ + struct z_erofs_pcluster *next; + + /* I: start physical position of this pcluster */ + erofs_off_t pos; + + /* L: the maximum decompression size of this round */ + unsigned int length; + + /* L: total number of bvecs */ + unsigned int vcnt; + + /* I: pcluster size (compressed size) in bytes */ + unsigned int pclustersize; + + /* I: page offset of start position of decompression */ + unsigned short pageofs_out; + + /* I: page offset of inline compressed data */ + unsigned short pageofs_in; + + union { + /* L: inline a certain number of bvec for bootstrap */ + struct z_erofs_bvset_inline bvset; + + /* I: can be used to free the pcluster by RCU. */ + struct rcu_head rcu; + }; + + /* I: compression algorithm format */ + unsigned char algorithmformat; + + /* I: whether compressed data is in-lined or not */ + bool from_meta; + + /* L: whether partial decompression or not */ + bool partial; + + /* L: whether extra buffer allocations are best-effort */ + bool besteffort; + + /* A: compressed bvecs (can be cached or inplaced pages) */ + struct z_erofs_bvec compressed_bvecs[]; +}; + +/* the end of a chain of pclusters */ +#define Z_EROFS_PCLUSTER_TAIL ((void *) 0x700 + POISON_POINTER_DELTA) + +struct z_erofs_decompressqueue { + struct super_block *sb; + struct z_erofs_pcluster *head; + atomic_t pending_bios; + + union { + struct completion done; + struct work_struct work; + struct kthread_work kthread_work; + } u; + bool eio, sync; +}; + +static inline unsigned int z_erofs_pclusterpages(struct z_erofs_pcluster *pcl) +{ + return PAGE_ALIGN(pcl->pageofs_in + pcl->pclustersize) >> PAGE_SHIFT; +} + +static bool erofs_folio_is_managed(struct erofs_sb_info *sbi, struct folio *fo) +{ + return fo->mapping == MNGD_MAPPING(sbi); +} + +#define Z_EROFS_ONSTACK_PAGES 32 + +/* + * since pclustersize is variable for big pcluster feature, introduce slab + * pools implementation for different pcluster sizes. + */ +struct z_erofs_pcluster_slab { + struct kmem_cache *slab; + unsigned int maxpages; + char name[48]; +}; + +#define _PCLP(n) { .maxpages = n } + +static struct z_erofs_pcluster_slab pcluster_pool[] __read_mostly = { + _PCLP(1), + _PCLP(4), +#if Z_EROFS_PCLUSTER_MAX_PAGES > 16 + _PCLP(16), +#endif +#if Z_EROFS_PCLUSTER_MAX_PAGES > 64 + _PCLP(64), +#endif +#if Z_EROFS_PCLUSTER_MAX_PAGES > 128 + _PCLP(128), +#endif + _PCLP(Z_EROFS_PCLUSTER_MAX_PAGES + 1) +}; + +struct z_erofs_bvec_iter { + struct page *bvpage; + struct z_erofs_bvset *bvset; + unsigned int nr, cur; +}; + +static struct page *z_erofs_bvec_iter_end(struct z_erofs_bvec_iter *iter) +{ + if (iter->bvpage) + kunmap_local(iter->bvset); + return iter->bvpage; +} + +static struct page *z_erofs_bvset_flip(struct z_erofs_bvec_iter *iter) +{ + unsigned long base = (unsigned long)((struct z_erofs_bvset *)0)->bvec; + /* have to access nextpage in advance, otherwise it will be unmapped */ + struct page *nextpage = iter->bvset->nextpage; + struct page *oldpage; + + DBG_BUGON(!nextpage); + oldpage = z_erofs_bvec_iter_end(iter); + iter->bvpage = nextpage; + iter->bvset = kmap_local_page(nextpage); + iter->nr = (PAGE_SIZE - base) / sizeof(struct z_erofs_bvec); + iter->cur = 0; + return oldpage; +} + +static void z_erofs_bvec_iter_begin(struct z_erofs_bvec_iter *iter, + struct z_erofs_bvset_inline *bvset, + unsigned int bootstrap_nr, + unsigned int cur) +{ + *iter = (struct z_erofs_bvec_iter) { + .nr = bootstrap_nr, + .bvset = (struct z_erofs_bvset *)bvset, + }; + + while (cur > iter->nr) { + cur -= iter->nr; + z_erofs_bvset_flip(iter); + } + iter->cur = cur; +} + +static int z_erofs_bvec_enqueue(struct z_erofs_bvec_iter *iter, + struct z_erofs_bvec *bvec, + struct page **candidate_bvpage, + struct page **pagepool) +{ + if (iter->cur >= iter->nr) { + struct page *nextpage = *candidate_bvpage; + + if (!nextpage) { + nextpage = __erofs_allocpage(pagepool, GFP_KERNEL, + true); + if (!nextpage) + return -ENOMEM; + set_page_private(nextpage, Z_EROFS_SHORTLIVED_PAGE); + } + DBG_BUGON(iter->bvset->nextpage); + iter->bvset->nextpage = nextpage; + z_erofs_bvset_flip(iter); + + iter->bvset->nextpage = NULL; + *candidate_bvpage = NULL; + } + iter->bvset->bvec[iter->cur++] = *bvec; + return 0; +} + +static void z_erofs_bvec_dequeue(struct z_erofs_bvec_iter *iter, + struct z_erofs_bvec *bvec, + struct page **old_bvpage) +{ + if (iter->cur == iter->nr) + *old_bvpage = z_erofs_bvset_flip(iter); + else + *old_bvpage = NULL; + *bvec = iter->bvset->bvec[iter->cur++]; +} + +static void z_erofs_destroy_pcluster_pool(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) { + if (!pcluster_pool[i].slab) + continue; + kmem_cache_destroy(pcluster_pool[i].slab); + pcluster_pool[i].slab = NULL; + } +} + +static int z_erofs_create_pcluster_pool(void) +{ + struct z_erofs_pcluster_slab *pcs; + struct z_erofs_pcluster *a; + unsigned int size; + + for (pcs = pcluster_pool; + pcs < pcluster_pool + ARRAY_SIZE(pcluster_pool); ++pcs) { + size = struct_size(a, compressed_bvecs, pcs->maxpages); + + sprintf(pcs->name, "erofs_pcluster-%u", pcs->maxpages); + pcs->slab = kmem_cache_create(pcs->name, size, 0, + SLAB_RECLAIM_ACCOUNT, NULL); + if (pcs->slab) + continue; + + z_erofs_destroy_pcluster_pool(); + return -ENOMEM; + } + return 0; +} + +static struct z_erofs_pcluster *z_erofs_alloc_pcluster(unsigned int size) +{ + unsigned int nrpages = PAGE_ALIGN(size) >> PAGE_SHIFT; + struct z_erofs_pcluster_slab *pcs = pcluster_pool; + + for (; pcs < pcluster_pool + ARRAY_SIZE(pcluster_pool); ++pcs) { + struct z_erofs_pcluster *pcl; + + if (nrpages > pcs->maxpages) + continue; + + pcl = kmem_cache_zalloc(pcs->slab, GFP_KERNEL); + if (!pcl) + return ERR_PTR(-ENOMEM); + return pcl; + } + return ERR_PTR(-EINVAL); +} + +static void z_erofs_free_pcluster(struct z_erofs_pcluster *pcl) +{ + unsigned int pclusterpages = z_erofs_pclusterpages(pcl); + int i; + + for (i = 0; i < ARRAY_SIZE(pcluster_pool); ++i) { + struct z_erofs_pcluster_slab *pcs = pcluster_pool + i; + + if (pclusterpages > pcs->maxpages) + continue; + + kmem_cache_free(pcs->slab, pcl); + return; + } + DBG_BUGON(1); +} + +static struct workqueue_struct *z_erofs_workqueue __read_mostly; + +#ifdef CONFIG_EROFS_FS_PCPU_KTHREAD +static struct kthread_worker __rcu **z_erofs_pcpu_workers; +static atomic_t erofs_percpu_workers_initialized = ATOMIC_INIT(0); + +static void erofs_destroy_percpu_workers(void) +{ + struct kthread_worker *worker; + unsigned int cpu; + + for_each_possible_cpu(cpu) { + worker = rcu_dereference_protected( + z_erofs_pcpu_workers[cpu], 1); + rcu_assign_pointer(z_erofs_pcpu_workers[cpu], NULL); + if (worker) + kthread_destroy_worker(worker); + } + kfree(z_erofs_pcpu_workers); +} + +static struct kthread_worker *erofs_init_percpu_worker(int cpu) +{ + struct kthread_worker *worker = + kthread_run_worker_on_cpu(cpu, 0, "erofs_worker/%u"); + + if (IS_ERR(worker)) + return worker; + if (IS_ENABLED(CONFIG_EROFS_FS_PCPU_KTHREAD_HIPRI)) + sched_set_fifo_low(worker->task); + return worker; +} + +static int erofs_init_percpu_workers(void) +{ + struct kthread_worker *worker; + unsigned int cpu; + + z_erofs_pcpu_workers = kzalloc_objs(struct kthread_worker *, + num_possible_cpus(), GFP_ATOMIC); + if (!z_erofs_pcpu_workers) + return -ENOMEM; + + for_each_online_cpu(cpu) { /* could miss cpu{off,on}line? */ + worker = erofs_init_percpu_worker(cpu); + if (!IS_ERR(worker)) + rcu_assign_pointer(z_erofs_pcpu_workers[cpu], worker); + } + return 0; +} + +#ifdef CONFIG_HOTPLUG_CPU +static DEFINE_SPINLOCK(z_erofs_pcpu_worker_lock); +static enum cpuhp_state erofs_cpuhp_state; + +static int erofs_cpu_online(unsigned int cpu) +{ + struct kthread_worker *worker, *old; + + worker = erofs_init_percpu_worker(cpu); + if (IS_ERR(worker)) + return PTR_ERR(worker); + + spin_lock(&z_erofs_pcpu_worker_lock); + old = rcu_dereference_protected(z_erofs_pcpu_workers[cpu], + lockdep_is_held(&z_erofs_pcpu_worker_lock)); + if (!old) + rcu_assign_pointer(z_erofs_pcpu_workers[cpu], worker); + spin_unlock(&z_erofs_pcpu_worker_lock); + if (old) + kthread_destroy_worker(worker); + return 0; +} + +static int erofs_cpu_offline(unsigned int cpu) +{ + struct kthread_worker *worker; + + spin_lock(&z_erofs_pcpu_worker_lock); + worker = rcu_dereference_protected(z_erofs_pcpu_workers[cpu], + lockdep_is_held(&z_erofs_pcpu_worker_lock)); + rcu_assign_pointer(z_erofs_pcpu_workers[cpu], NULL); + spin_unlock(&z_erofs_pcpu_worker_lock); + + synchronize_rcu(); + if (worker) + kthread_destroy_worker(worker); + return 0; +} + +static int erofs_cpu_hotplug_init(void) +{ + int state; + + state = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, + "fs/erofs:online", erofs_cpu_online, erofs_cpu_offline); + if (state < 0) + return state; + + erofs_cpuhp_state = state; + return 0; +} + +static void erofs_cpu_hotplug_destroy(void) +{ + if (erofs_cpuhp_state) + cpuhp_remove_state_nocalls(erofs_cpuhp_state); +} +#else /* !CONFIG_HOTPLUG_CPU */ +static inline int erofs_cpu_hotplug_init(void) { return 0; } +static inline void erofs_cpu_hotplug_destroy(void) {} +#endif/* CONFIG_HOTPLUG_CPU */ +static int z_erofs_init_pcpu_workers(struct super_block *sb) +{ + int err; + + if (atomic_xchg(&erofs_percpu_workers_initialized, 1)) + return 0; + + err = erofs_init_percpu_workers(); + if (err) { + erofs_err(sb, "per-cpu workers: failed to allocate."); + goto err_init_percpu_workers; + } + + err = erofs_cpu_hotplug_init(); + if (err < 0) { + erofs_err(sb, "per-cpu workers: failed CPU hotplug init."); + goto err_cpuhp_init; + } + erofs_info(sb, "initialized per-cpu workers successfully."); + return err; + +err_cpuhp_init: + erofs_destroy_percpu_workers(); +err_init_percpu_workers: + atomic_set(&erofs_percpu_workers_initialized, 0); + return err; +} + +static void z_erofs_destroy_pcpu_workers(void) +{ + if (!atomic_xchg(&erofs_percpu_workers_initialized, 0)) + return; + erofs_cpu_hotplug_destroy(); + erofs_destroy_percpu_workers(); +} +#else /* !CONFIG_EROFS_FS_PCPU_KTHREAD */ +static inline int z_erofs_init_pcpu_workers(struct super_block *sb) { return 0; } +static inline void z_erofs_destroy_pcpu_workers(void) {} +#endif/* CONFIG_EROFS_FS_PCPU_KTHREAD */ + +void z_erofs_exit_subsystem(void) +{ + z_erofs_destroy_pcpu_workers(); + destroy_workqueue(z_erofs_workqueue); + z_erofs_destroy_pcluster_pool(); + z_erofs_crypto_disable_all_engines(); + z_erofs_exit_decompressor(); +} + +int __init z_erofs_init_subsystem(void) +{ + int err = z_erofs_init_decompressor(); + + if (err) + goto err_decompressor; + + err = z_erofs_create_pcluster_pool(); + if (err) + goto err_pcluster_pool; + + z_erofs_workqueue = alloc_workqueue("erofs_worker", + WQ_UNBOUND | WQ_HIGHPRI, num_possible_cpus()); + if (!z_erofs_workqueue) { + err = -ENOMEM; + goto err_workqueue_init; + } + + return err; + +err_workqueue_init: + z_erofs_destroy_pcluster_pool(); +err_pcluster_pool: + z_erofs_exit_decompressor(); +err_decompressor: + return err; +} + +enum z_erofs_pclustermode { + /* It has previously been linked into another processing chain */ + Z_EROFS_PCLUSTER_INFLIGHT, + /* + * A weaker form of Z_EROFS_PCLUSTER_FOLLOWED; the difference is that it + * may be dispatched to the bypass queue later due to uptodated managed + * folios. All file-backed folios related to this pcluster cannot be + * reused for in-place I/O (or bvpage) since the pcluster may be decoded + * in a separate queue (and thus out of order). + */ + Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE, + /* + * The pcluster has just been linked to our processing chain. + * File-backed folios (except for the head page) related to it can be + * used for in-place I/O (or bvpage). + */ + Z_EROFS_PCLUSTER_FOLLOWED, +}; + +struct z_erofs_frontend { + struct inode *inode, *sharedinode; + struct erofs_map_blocks map; + struct z_erofs_bvec_iter biter; + + struct page *pagepool; + struct page *candidate_bvpage; + struct z_erofs_pcluster *pcl, *head; + enum z_erofs_pclustermode mode; + + erofs_off_t headoffset; + + /* a pointer used to pick up inplace I/O pages */ + unsigned int icur; +}; + +#define Z_EROFS_DEFINE_FRONTEND(fe, i, si, ho) struct z_erofs_frontend fe = { \ + .inode = i, .sharedinode = si, .head = Z_EROFS_PCLUSTER_TAIL, \ + .mode = Z_EROFS_PCLUSTER_FOLLOWED, .headoffset = ho } + +static bool z_erofs_should_alloc_cache(struct z_erofs_frontend *fe) +{ + unsigned int cachestrategy = EROFS_I_SB(fe->inode)->opt.cache_strategy; + + if (cachestrategy <= EROFS_ZIP_CACHE_DISABLED) + return false; + + if (fe->map.m_flags & EROFS_MAP_PARTIAL_MAPPED) + return true; + + if (cachestrategy >= EROFS_ZIP_CACHE_READAROUND && + fe->map.m_la < fe->headoffset) + return true; + + return false; +} + +static void z_erofs_bind_cache(struct z_erofs_frontend *fe) +{ + struct address_space *mc = MNGD_MAPPING(EROFS_I_SB(fe->inode)); + struct z_erofs_pcluster *pcl = fe->pcl; + unsigned int pclusterpages = z_erofs_pclusterpages(pcl); + bool shouldalloc = z_erofs_should_alloc_cache(fe); + pgoff_t poff = pcl->pos >> PAGE_SHIFT; + bool may_bypass = true; + /* Optimistic allocation, as in-place I/O can be used as a fallback */ + gfp_t gfp = (mapping_gfp_mask(mc) & ~__GFP_DIRECT_RECLAIM) | + __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN; + struct folio *folio, *newfolio; + unsigned int i; + + if (i_blocksize(fe->inode) != PAGE_SIZE || + fe->mode < Z_EROFS_PCLUSTER_FOLLOWED) + return; + + for (i = 0; i < pclusterpages; ++i) { + /* Inaccurate check w/o locking to avoid unneeded lookups */ + if (READ_ONCE(pcl->compressed_bvecs[i].page)) + continue; + + folio = filemap_get_folio(mc, poff + i); + if (IS_ERR(folio)) { + may_bypass = false; + if (!shouldalloc) + continue; + + /* + * Allocate a managed folio for cached I/O, or it may be + * then filled with a file-backed folio for in-place I/O + */ + newfolio = filemap_alloc_folio(gfp, 0, NULL); + if (!newfolio) + continue; + newfolio->private = Z_EROFS_PREALLOCATED_FOLIO; + folio = NULL; + } + spin_lock(&pcl->lockref.lock); + if (!pcl->compressed_bvecs[i].page) { + pcl->compressed_bvecs[i].page = + folio_page(folio ?: newfolio, 0); + spin_unlock(&pcl->lockref.lock); + continue; + } + spin_unlock(&pcl->lockref.lock); + folio_put(folio ?: newfolio); + } + + /* + * Don't perform in-place I/O if all compressed pages are available in + * the managed cache, as the pcluster can be moved to the bypass queue. + */ + if (may_bypass) + fe->mode = Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE; +} + +/* (erofs_shrinker) disconnect cached encoded data with pclusters */ +static int erofs_try_to_free_all_cached_folios(struct erofs_sb_info *sbi, + struct z_erofs_pcluster *pcl) +{ + unsigned int pclusterpages = z_erofs_pclusterpages(pcl); + struct folio *folio; + int i; + + DBG_BUGON(pcl->from_meta); + /* Each cached folio contains one page unless bs > ps is supported */ + for (i = 0; i < pclusterpages; ++i) { + if (pcl->compressed_bvecs[i].page) { + folio = page_folio(pcl->compressed_bvecs[i].page); + /* Avoid reclaiming or migrating this folio */ + if (!folio_trylock(folio)) + return -EBUSY; + + DBG_BUGON(!erofs_folio_is_managed(sbi, folio)); + pcl->compressed_bvecs[i].page = NULL; + folio_detach_private(folio); + folio_unlock(folio); + } + } + return 0; +} + +static bool z_erofs_cache_release_folio(struct folio *folio, gfp_t gfp) +{ + struct z_erofs_pcluster *pcl = folio_get_private(folio); + struct z_erofs_bvec *bvec = pcl->compressed_bvecs; + struct z_erofs_bvec *end = bvec + z_erofs_pclusterpages(pcl); + bool ret; + + if (!folio_test_private(folio)) + return true; + + ret = false; + spin_lock(&pcl->lockref.lock); + if (pcl->lockref.count <= 0) { + DBG_BUGON(pcl->from_meta); + for (; bvec < end; ++bvec) { + if (bvec->page && page_folio(bvec->page) == folio) { + bvec->page = NULL; + folio_detach_private(folio); + ret = true; + break; + } + } + } + spin_unlock(&pcl->lockref.lock); + return ret; +} + +/* + * It will be called only on inode eviction. In case that there are still some + * decompression requests in progress, wait with rescheduling for a bit here. + * An extra lock could be introduced instead but it seems unnecessary. + */ +static void z_erofs_cache_invalidate_folio(struct folio *folio, + size_t offset, size_t length) +{ + const size_t stop = length + offset; + + /* Check for potential overflow in debug mode */ + DBG_BUGON(stop > folio_size(folio) || stop < length); + + if (offset == 0 && stop == folio_size(folio)) + while (!z_erofs_cache_release_folio(folio, 0)) + cond_resched(); +} + +static const struct address_space_operations z_erofs_cache_aops = { + .release_folio = z_erofs_cache_release_folio, + .invalidate_folio = z_erofs_cache_invalidate_folio, +}; + +int z_erofs_init_super(struct super_block *sb) +{ + int err; + + err = z_erofs_init_pcpu_workers(sb); + if (err) + return err; + + err = erofs_setup_managed_cache(sb); + if (err) + return err; + EROFS_SB(sb)->managed_cache->i_mapping->a_ops = &z_erofs_cache_aops; + xa_init(&EROFS_SB(sb)->managed_pslots); + return 0; +} + +/* callers must be with pcluster lock held */ +static int z_erofs_attach_page(struct z_erofs_frontend *fe, + struct z_erofs_bvec *bvec, bool exclusive) +{ + struct z_erofs_pcluster *pcl = fe->pcl; + int ret; + + if (exclusive) { + /* Inplace I/O is limited to one page for uncompressed data */ + if (pcl->algorithmformat < Z_EROFS_COMPRESSION_MAX || + fe->icur <= 1) { + /* Try to prioritize inplace I/O here */ + spin_lock(&pcl->lockref.lock); + while (fe->icur > 0) { + if (pcl->compressed_bvecs[--fe->icur].page) + continue; + pcl->compressed_bvecs[fe->icur] = *bvec; + spin_unlock(&pcl->lockref.lock); + return 0; + } + spin_unlock(&pcl->lockref.lock); + } + + /* otherwise, check if it can be used as a bvpage */ + if (fe->mode >= Z_EROFS_PCLUSTER_FOLLOWED && + !fe->candidate_bvpage) + fe->candidate_bvpage = bvec->page; + } + ret = z_erofs_bvec_enqueue(&fe->biter, bvec, &fe->candidate_bvpage, + &fe->pagepool); + fe->pcl->vcnt += (ret >= 0); + return ret; +} + +static bool z_erofs_get_pcluster(struct z_erofs_pcluster *pcl) +{ + if (lockref_get_not_zero(&pcl->lockref)) + return true; + + spin_lock(&pcl->lockref.lock); + if (lockref_is_dead(&pcl->lockref)) { + spin_unlock(&pcl->lockref.lock); + return false; + } + + if (!pcl->lockref.count++) + atomic_long_dec(&erofs_global_shrink_cnt); + spin_unlock(&pcl->lockref.lock); + return true; +} + +static int z_erofs_register_pcluster(struct z_erofs_frontend *fe) +{ + struct erofs_map_blocks *map = &fe->map; + struct super_block *sb = fe->inode->i_sb; + struct erofs_sb_info *sbi = EROFS_SB(sb); + struct z_erofs_pcluster *pcl, *pre; + unsigned int pageofs_in; + int err; + + pageofs_in = erofs_blkoff(sb, map->m_pa); + pcl = z_erofs_alloc_pcluster(pageofs_in + map->m_plen); + if (IS_ERR(pcl)) + return PTR_ERR(pcl); + + lockref_init(&pcl->lockref); /* one ref for this request */ + pcl->algorithmformat = map->m_algorithmformat; + pcl->pclustersize = map->m_plen; + pcl->length = 0; + pcl->partial = true; + pcl->next = fe->head; + pcl->pos = map->m_pa; + pcl->pageofs_in = pageofs_in; + pcl->pageofs_out = map->m_la & ~PAGE_MASK; + pcl->from_meta = map->m_flags & EROFS_MAP_META; + fe->mode = Z_EROFS_PCLUSTER_FOLLOWED; + + /* + * lock all primary followed works before visible to others + * and mutex_trylock *never* fails for a new pcluster. + */ + mutex_init(&pcl->lock); + DBG_BUGON(!mutex_trylock(&pcl->lock)); + + if (!pcl->from_meta) { + while (1) { + xa_lock(&sbi->managed_pslots); + pre = __xa_cmpxchg(&sbi->managed_pslots, pcl->pos, + NULL, pcl, GFP_KERNEL); + if (!pre || xa_is_err(pre) || z_erofs_get_pcluster(pre)) { + xa_unlock(&sbi->managed_pslots); + break; + } + /* try to legitimize the current in-tree one */ + xa_unlock(&sbi->managed_pslots); + cond_resched(); + } + if (xa_is_err(pre)) { + err = xa_err(pre); + goto err_out; + } else if (pre) { + fe->pcl = pre; + err = -EEXIST; + goto err_out; + } + } + fe->head = fe->pcl = pcl; + return 0; + +err_out: + mutex_unlock(&pcl->lock); + z_erofs_free_pcluster(pcl); + return err; +} + +static int z_erofs_pcluster_begin(struct z_erofs_frontend *fe) +{ + struct erofs_map_blocks *map = &fe->map; + struct super_block *sb = fe->inode->i_sb; + struct z_erofs_pcluster *pcl = NULL; + void *ptr = NULL; + bool needretry; + int ret; + + DBG_BUGON(fe->pcl); + /* must be Z_EROFS_PCLUSTER_TAIL or pointed to previous pcluster */ + DBG_BUGON(!fe->head); + + if (map->m_flags & EROFS_MAP_META) { + ret = erofs_init_metabuf(&map->buf, sb, + erofs_inode_in_metabox(fe->inode)); + if (ret) + return ret; + ptr = erofs_bread(&map->buf, map->m_pa, false); + if (IS_ERR(ptr)) { + erofs_err(sb, "failed to read inline data %pe @ pa %llu of nid %llu", + ptr, map->m_pa, EROFS_I(fe->inode)->nid); + return PTR_ERR(ptr); + } + ptr = map->buf.page; + } else { + do { + rcu_read_lock(); + pcl = xa_load(&EROFS_SB(sb)->managed_pslots, map->m_pa); + needretry = pcl && !z_erofs_get_pcluster(pcl); + rcu_read_unlock(); + } while (needretry); + } + + if (pcl) { + DBG_BUGON(map->m_pa != pcl->pos); + fe->pcl = pcl; + ret = -EEXIST; + } else { + ret = z_erofs_register_pcluster(fe); + } + + if (ret == -EEXIST) { + mutex_lock(&fe->pcl->lock); + /* check if this pcluster hasn't been linked into any chain. */ + if (!cmpxchg(&fe->pcl->next, NULL, fe->head)) { + /* .. so it can be attached to our submission chain */ + fe->head = fe->pcl; + fe->mode = Z_EROFS_PCLUSTER_FOLLOWED; + } else { /* otherwise, it belongs to an inflight chain */ + fe->mode = Z_EROFS_PCLUSTER_INFLIGHT; + } + } else if (ret) { + return ret; + } + + z_erofs_bvec_iter_begin(&fe->biter, &fe->pcl->bvset, + Z_EROFS_INLINE_BVECS, fe->pcl->vcnt); + if (!fe->pcl->from_meta) { + /* bind cache first when cached decompression is preferred */ + z_erofs_bind_cache(fe); + } else { + folio_get(page_folio((struct page *)ptr)); + WRITE_ONCE(fe->pcl->compressed_bvecs[0].page, ptr); + fe->pcl->pageofs_in = map->m_pa & ~PAGE_MASK; + fe->mode = Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE; + } + /* file-backed inplace I/O pages are traversed in reverse order */ + fe->icur = z_erofs_pclusterpages(fe->pcl); + return 0; +} + +static void z_erofs_rcu_callback(struct rcu_head *head) +{ + z_erofs_free_pcluster(container_of(head, struct z_erofs_pcluster, rcu)); +} + +static bool __erofs_try_to_release_pcluster(struct erofs_sb_info *sbi, + struct z_erofs_pcluster *pcl) +{ + if (pcl->lockref.count) + return false; + + /* + * Note that all cached folios should be detached before deleted from + * the XArray. Otherwise some folios could be still attached to the + * orphan old pcluster when the new one is available in the tree. + */ + if (erofs_try_to_free_all_cached_folios(sbi, pcl)) + return false; + + /* + * It's impossible to fail after the pcluster is freezed, but in order + * to avoid some race conditions, add a DBG_BUGON to observe this. + */ + DBG_BUGON(__xa_erase(&sbi->managed_pslots, pcl->pos) != pcl); + + lockref_mark_dead(&pcl->lockref); + return true; +} + +static bool erofs_try_to_release_pcluster(struct erofs_sb_info *sbi, + struct z_erofs_pcluster *pcl) +{ + bool free; + + spin_lock(&pcl->lockref.lock); + free = __erofs_try_to_release_pcluster(sbi, pcl); + spin_unlock(&pcl->lockref.lock); + if (free) { + atomic_long_dec(&erofs_global_shrink_cnt); + call_rcu(&pcl->rcu, z_erofs_rcu_callback); + } + return free; +} + +unsigned long z_erofs_shrink_scan(struct erofs_sb_info *sbi, unsigned long nr) +{ + struct z_erofs_pcluster *pcl; + unsigned long index, freed = 0; + + xa_lock(&sbi->managed_pslots); + xa_for_each(&sbi->managed_pslots, index, pcl) { + /* try to shrink each valid pcluster */ + if (!erofs_try_to_release_pcluster(sbi, pcl)) + continue; + xa_unlock(&sbi->managed_pslots); + + ++freed; + if (!--nr) + return freed; + xa_lock(&sbi->managed_pslots); + } + xa_unlock(&sbi->managed_pslots); + return freed; +} + +static void z_erofs_put_pcluster(struct erofs_sb_info *sbi, + struct z_erofs_pcluster *pcl, bool try_free) +{ + bool free = false; + + if (lockref_put_or_lock(&pcl->lockref)) + return; + + DBG_BUGON(lockref_is_dead(&pcl->lockref)); + if (!--pcl->lockref.count) { + if (try_free && xa_trylock(&sbi->managed_pslots)) { + free = __erofs_try_to_release_pcluster(sbi, pcl); + xa_unlock(&sbi->managed_pslots); + } + atomic_long_add(!free, &erofs_global_shrink_cnt); + } + spin_unlock(&pcl->lockref.lock); + if (free) + call_rcu(&pcl->rcu, z_erofs_rcu_callback); +} + +static void z_erofs_pcluster_end(struct z_erofs_frontend *fe) +{ + struct z_erofs_pcluster *pcl = fe->pcl; + + if (!pcl) + return; + + z_erofs_bvec_iter_end(&fe->biter); + mutex_unlock(&pcl->lock); + + if (fe->candidate_bvpage) + fe->candidate_bvpage = NULL; + + /* Drop refcount if it doesn't belong to our processing chain */ + if (fe->mode < Z_EROFS_PCLUSTER_FOLLOWED_NOINPLACE) + z_erofs_put_pcluster(EROFS_I_SB(fe->inode), pcl, false); + fe->pcl = NULL; +} + +static int z_erofs_read_fragment(struct super_block *sb, struct folio *folio, + unsigned int cur, unsigned int end, erofs_off_t pos) +{ + struct inode *packed_inode = EROFS_SB(sb)->packed_inode; + struct erofs_buf buf = __EROFS_BUF_INITIALIZER; + unsigned int cnt; + u8 *src; + + if (!packed_inode) + return -EFSCORRUPTED; + + buf.mapping = packed_inode->i_mapping; + for (; cur < end; cur += cnt, pos += cnt) { + cnt = min(end - cur, sb->s_blocksize - erofs_blkoff(sb, pos)); + src = erofs_bread(&buf, pos, true); + if (IS_ERR(src)) { + erofs_put_metabuf(&buf); + return PTR_ERR(src); + } + memcpy_to_folio(folio, cur, src, cnt); + } + erofs_put_metabuf(&buf); + return 0; +} + +static int z_erofs_scan_folio(struct z_erofs_frontend *f, + struct folio *folio, bool ra) +{ + struct inode *const inode = f->inode; + struct erofs_map_blocks *const map = &f->map; + const loff_t offset = folio_pos(folio); + const unsigned int bs = i_blocksize(inode); + unsigned int end = folio_size(folio), split = 0, cur, pgs; + bool tight, excl; + int err = 0; + + tight = (bs == PAGE_SIZE); + erofs_onlinefolio_init(folio); + do { + if (offset + end - 1 < map->m_la || + offset + end - 1 >= map->m_la + map->m_llen) { + z_erofs_pcluster_end(f); + map->m_la = offset + end - 1; + map->m_llen = 0; + err = z_erofs_map_blocks_iter(inode, map, 0); + if (err) + break; + } + + cur = offset > map->m_la ? 0 : map->m_la - offset; + pgs = round_down(cur, PAGE_SIZE); + /* bump split parts first to avoid several separate cases */ + ++split; + + if (map->m_flags & EROFS_MAP_FRAGMENT) { + erofs_off_t fpos = offset + cur - map->m_la; + + err = z_erofs_read_fragment(inode->i_sb, folio, cur, + cur + min(map->m_llen - fpos, end - cur), + EROFS_I(inode)->z_fragmentoff + fpos); + if (err) + break; + tight = false; + } else if (!(map->m_flags & EROFS_MAP_MAPPED)) { + folio_zero_segment(folio, cur, end); + tight = false; + } else { + if (!f->pcl) { + err = z_erofs_pcluster_begin(f); + if (err) + break; + f->pcl->besteffort |= !ra; + } + + pgs = round_down(end - 1, PAGE_SIZE); + /* + * Ensure this partial page belongs to this submit chain + * rather than other concurrent submit chains or + * noio(bypass) chains since those chains are handled + * asynchronously thus it cannot be used for inplace I/O + * or bvpage (should be processed in the strict order.) + */ + tight &= (f->mode >= Z_EROFS_PCLUSTER_FOLLOWED); + excl = false; + if (cur <= pgs) { + excl = (split <= 1) || tight; + cur = pgs; + } + + err = z_erofs_attach_page(f, &((struct z_erofs_bvec) { + .page = folio_page(folio, pgs >> PAGE_SHIFT), + .offset = offset + pgs - map->m_la, + .end = end - pgs, }), excl); + if (err) + break; + + erofs_onlinefolio_split(folio); + if (f->pcl->length < offset + end - map->m_la) { + f->pcl->length = offset + end - map->m_la; + f->pcl->pageofs_out = map->m_la & ~PAGE_MASK; + } + if (EROFS_MAP_FULL(map->m_flags) && + f->pcl->length == map->m_llen) + f->pcl->partial = false; + } + /* shorten the remaining extent to update progress */ + map->m_llen = offset + cur - map->m_la; + map->m_flags |= EROFS_MAP_PARTIAL_MAPPED; + if (cur <= pgs) { + split = cur < pgs; + tight = (bs == PAGE_SIZE); + } + } while ((end = cur) > 0); + erofs_onlinefolio_end(folio, err, false); + return err; +} + +static bool z_erofs_page_is_invalidated(struct page *page) +{ + return !page_folio(page)->mapping && !z_erofs_is_shortlived_page(page); +} + +struct z_erofs_backend { + struct page *onstack_pages[Z_EROFS_ONSTACK_PAGES]; + struct super_block *sb; + struct z_erofs_pcluster *pcl; + /* pages with the longest decompressed length for deduplication */ + struct page **decompressed_pages; + /* pages to keep the compressed data */ + struct page **compressed_pages; + + struct list_head decompressed_secondary_bvecs; + struct page **pagepool; + unsigned int onstack_used, nr_pages; + /* indicate if temporary copies should be preserved for later use */ + bool keepxcpy; +}; + +struct z_erofs_bvec_item { + struct z_erofs_bvec bvec; + struct list_head list; +}; + +static void z_erofs_do_decompressed_bvec(struct z_erofs_backend *be, + struct z_erofs_bvec *bvec) +{ + int poff = bvec->offset + be->pcl->pageofs_out; + struct z_erofs_bvec_item *item; + struct page **page; + + if (!(poff & ~PAGE_MASK) && (bvec->end == PAGE_SIZE || + bvec->offset + bvec->end == be->pcl->length)) { + DBG_BUGON((poff >> PAGE_SHIFT) >= be->nr_pages); + page = be->decompressed_pages + (poff >> PAGE_SHIFT); + if (!*page) { + *page = bvec->page; + return; + } + } else { + be->keepxcpy = true; + } + + /* (cold path) one pcluster is requested multiple times */ + item = kmalloc_obj(*item, GFP_KERNEL | __GFP_NOFAIL); + item->bvec = *bvec; + list_add(&item->list, &be->decompressed_secondary_bvecs); +} + +static void z_erofs_fill_other_copies(struct z_erofs_backend *be, int err) +{ + unsigned int off0 = be->pcl->pageofs_out; + struct list_head *p, *n; + + list_for_each_safe(p, n, &be->decompressed_secondary_bvecs) { + struct z_erofs_bvec_item *bvi; + unsigned int end, cur; + void *dst, *src; + + bvi = container_of(p, struct z_erofs_bvec_item, list); + cur = bvi->bvec.offset < 0 ? -bvi->bvec.offset : 0; + end = min_t(unsigned int, be->pcl->length - bvi->bvec.offset, + bvi->bvec.end); + dst = kmap_local_page(bvi->bvec.page); + while (cur < end) { + unsigned int pgnr, scur, len; + + pgnr = (bvi->bvec.offset + cur + off0) >> PAGE_SHIFT; + DBG_BUGON(pgnr >= be->nr_pages); + + scur = bvi->bvec.offset + cur - + ((pgnr << PAGE_SHIFT) - off0); + len = min_t(unsigned int, end - cur, PAGE_SIZE - scur); + if (!be->decompressed_pages[pgnr]) { + err = -EFSCORRUPTED; + cur += len; + continue; + } + src = kmap_local_page(be->decompressed_pages[pgnr]); + memcpy(dst + cur, src + scur, len); + kunmap_local(src); + cur += len; + } + kunmap_local(dst); + erofs_onlinefolio_end(page_folio(bvi->bvec.page), err, true); + list_del(p); + kfree(bvi); + } +} + +static void z_erofs_parse_out_bvecs(struct z_erofs_backend *be) +{ + struct z_erofs_pcluster *pcl = be->pcl; + struct z_erofs_bvec_iter biter; + struct page *old_bvpage; + int i; + + z_erofs_bvec_iter_begin(&biter, &pcl->bvset, Z_EROFS_INLINE_BVECS, 0); + for (i = 0; i < pcl->vcnt; ++i) { + struct z_erofs_bvec bvec; + + z_erofs_bvec_dequeue(&biter, &bvec, &old_bvpage); + + if (old_bvpage) + z_erofs_put_shortlivedpage(be->pagepool, old_bvpage); + + DBG_BUGON(z_erofs_page_is_invalidated(bvec.page)); + z_erofs_do_decompressed_bvec(be, &bvec); + } + + old_bvpage = z_erofs_bvec_iter_end(&biter); + if (old_bvpage) + z_erofs_put_shortlivedpage(be->pagepool, old_bvpage); +} + +static int z_erofs_parse_in_bvecs(struct z_erofs_backend *be, bool *overlapped) +{ + struct z_erofs_pcluster *pcl = be->pcl; + unsigned int pclusterpages = z_erofs_pclusterpages(pcl); + int i, err = 0; + + *overlapped = false; + for (i = 0; i < pclusterpages; ++i) { + struct z_erofs_bvec *bvec = &pcl->compressed_bvecs[i]; + struct page *page = bvec->page; + + /* compressed data ought to be valid when decompressing */ + if (IS_ERR(page) || !page) { + bvec->page = NULL; /* clear the failure reason */ + err = page ? PTR_ERR(page) : -EIO; + continue; + } + be->compressed_pages[i] = page; + + if (pcl->from_meta || + erofs_folio_is_managed(EROFS_SB(be->sb), page_folio(page))) { + if (!PageUptodate(page)) + err = -EIO; + continue; + } + + DBG_BUGON(z_erofs_page_is_invalidated(page)); + if (z_erofs_is_shortlived_page(page)) + continue; + z_erofs_do_decompressed_bvec(be, bvec); + *overlapped = true; + } + return err; +} + +static int z_erofs_decompress_pcluster(struct z_erofs_backend *be, bool eio) +{ + struct erofs_sb_info *const sbi = EROFS_SB(be->sb); + struct z_erofs_pcluster *pcl = be->pcl; + unsigned int pclusterpages = z_erofs_pclusterpages(pcl); + const struct z_erofs_decompressor *alg = + z_erofs_decomp[pcl->algorithmformat]; + bool try_free = true; + int i, j, jtop, err2, err = eio ? -EIO : 0; + struct page *page; + bool overlapped; + const char *reason; + + mutex_lock(&pcl->lock); + be->nr_pages = PAGE_ALIGN(pcl->length + pcl->pageofs_out) >> PAGE_SHIFT; + + /* allocate (de)compressed page arrays if cannot be kept on stack */ + be->decompressed_pages = NULL; + be->compressed_pages = NULL; + be->onstack_used = 0; + if (be->nr_pages <= Z_EROFS_ONSTACK_PAGES) { + be->decompressed_pages = be->onstack_pages; + be->onstack_used = be->nr_pages; + memset(be->decompressed_pages, 0, + sizeof(struct page *) * be->nr_pages); + } + + if (pclusterpages + be->onstack_used <= Z_EROFS_ONSTACK_PAGES) + be->compressed_pages = be->onstack_pages + be->onstack_used; + + if (!be->decompressed_pages) + be->decompressed_pages = + kvzalloc_objs(struct page *, be->nr_pages, + GFP_KERNEL | __GFP_NOFAIL); + if (!be->compressed_pages) + be->compressed_pages = + kvzalloc_objs(struct page *, pclusterpages, + GFP_KERNEL | __GFP_NOFAIL); + + z_erofs_parse_out_bvecs(be); + err2 = z_erofs_parse_in_bvecs(be, &overlapped); + if (err2) + err = err2; + if (!err) { + reason = alg->decompress(&(struct z_erofs_decompress_req) { + .sb = be->sb, + .in = be->compressed_pages, + .out = be->decompressed_pages, + .inpages = pclusterpages, + .outpages = be->nr_pages, + .pageofs_in = pcl->pageofs_in, + .pageofs_out = pcl->pageofs_out, + .inputsize = pcl->pclustersize, + .outputsize = pcl->length, + .alg = pcl->algorithmformat, + .inplace_io = overlapped, + .partial_decoding = pcl->partial, + .fillgaps = be->keepxcpy, + .gfp = pcl->besteffort ? GFP_KERNEL : + GFP_NOWAIT | __GFP_NORETRY + }, be->pagepool); + if (IS_ERR(reason)) { + if (pcl->besteffort || reason != ERR_PTR(-ENOMEM)) + erofs_err(be->sb, "failed to decompress (%s) %pe @ pa %llu size %u => %u", + alg->name, reason, pcl->pos, + pcl->pclustersize, pcl->length); + err = PTR_ERR(reason); + } else if (unlikely(reason)) { + erofs_err(be->sb, "failed to decompress (%s) %s @ pa %llu size %u => %u", + alg->name, reason, pcl->pos, + pcl->pclustersize, pcl->length); + err = -EFSCORRUPTED; + } + } + + /* must handle all compressed pages before actual file pages */ + if (pcl->from_meta) { + folio_put(page_folio(pcl->compressed_bvecs[0].page)); + WRITE_ONCE(pcl->compressed_bvecs[0].page, NULL); + } else { + /* managed folios are still left in compressed_bvecs[] */ + for (i = 0; i < pclusterpages; ++i) { + page = be->compressed_pages[i]; + if (!page) + continue; + if (erofs_folio_is_managed(sbi, page_folio(page))) { + try_free = false; + continue; + } + (void)z_erofs_put_shortlivedpage(be->pagepool, page); + WRITE_ONCE(pcl->compressed_bvecs[i].page, NULL); + } + } + if (be->compressed_pages < be->onstack_pages || + be->compressed_pages >= be->onstack_pages + Z_EROFS_ONSTACK_PAGES) + kvfree(be->compressed_pages); + + jtop = 0; + z_erofs_fill_other_copies(be, err); + for (i = 0; i < be->nr_pages; ++i) { + page = be->decompressed_pages[i]; + if (!page) + continue; + + DBG_BUGON(z_erofs_page_is_invalidated(page)); + if (!z_erofs_is_shortlived_page(page)) { + erofs_onlinefolio_end(page_folio(page), err, true); + continue; + } + if (pcl->algorithmformat != Z_EROFS_COMPRESSION_LZ4) { + erofs_pagepool_add(be->pagepool, page); + continue; + } + for (j = 0; j < jtop && be->decompressed_pages[j] != page; ++j) + ; + if (j >= jtop) /* this bounce page is newly detected */ + be->decompressed_pages[jtop++] = page; + } + while (jtop) + erofs_pagepool_add(be->pagepool, + be->decompressed_pages[--jtop]); + if (be->decompressed_pages != be->onstack_pages) + kvfree(be->decompressed_pages); + + pcl->length = 0; + pcl->partial = true; + pcl->besteffort = false; + pcl->bvset.nextpage = NULL; + pcl->vcnt = 0; + + /* pcluster lock MUST be taken before the following line */ + WRITE_ONCE(pcl->next, NULL); + mutex_unlock(&pcl->lock); + + if (pcl->from_meta) + z_erofs_free_pcluster(pcl); + else + z_erofs_put_pcluster(sbi, pcl, try_free); + return err; +} + +static int z_erofs_decompress_queue(const struct z_erofs_decompressqueue *io, + struct page **pagepool) +{ + struct z_erofs_backend be = { + .sb = io->sb, + .pagepool = pagepool, + .decompressed_secondary_bvecs = + LIST_HEAD_INIT(be.decompressed_secondary_bvecs), + .pcl = io->head, + }; + struct z_erofs_pcluster *next; + int err = 0; + + for (; be.pcl != Z_EROFS_PCLUSTER_TAIL; be.pcl = next) { + DBG_BUGON(!be.pcl); + next = READ_ONCE(be.pcl->next); + err = z_erofs_decompress_pcluster(&be, io->eio) ?: err; + } + return err; +} + +static void z_erofs_decompressqueue_work(struct work_struct *work) +{ + struct z_erofs_decompressqueue *bgq = + container_of(work, struct z_erofs_decompressqueue, u.work); + struct page *pagepool = NULL; + + DBG_BUGON(bgq->head == Z_EROFS_PCLUSTER_TAIL); + z_erofs_decompress_queue(bgq, &pagepool); + erofs_release_pages(&pagepool); + kvfree(bgq); +} + +#ifdef CONFIG_EROFS_FS_PCPU_KTHREAD +static void z_erofs_decompressqueue_kthread_work(struct kthread_work *work) +{ + z_erofs_decompressqueue_work((struct work_struct *)work); +} +#endif + + +static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io, + int bios) +{ + struct erofs_sb_info *const sbi = EROFS_SB(io->sb); + int gfp_flag; + + /* wake up the caller thread for sync decompression */ + if (io->sync) { + if (!atomic_add_return(bios, &io->pending_bios)) + complete(&io->u.done); + return; + } + + if (atomic_add_return(bios, &io->pending_bios)) + return; + if (bio_in_atomic()) { + /* See `sync_decompress` in sysfs-fs-erofs for more details */ + if (sbi->sync_decompress == EROFS_SYNC_DECOMPRESS_AUTO) + sbi->sync_decompress = EROFS_SYNC_DECOMPRESS_FORCE_ON; +#ifdef CONFIG_EROFS_FS_PCPU_KTHREAD + scoped_guard(rcu) { + struct kthread_worker *worker; + + worker = rcu_dereference( + z_erofs_pcpu_workers[raw_smp_processor_id()]); + if (worker) { + kthread_queue_work(worker, &io->u.kthread_work); + return; + } + } + INIT_WORK(&io->u.work, z_erofs_decompressqueue_work); +#endif + queue_work(z_erofs_workqueue, &io->u.work); + return; + } + gfp_flag = memalloc_noio_save(); + z_erofs_decompressqueue_work(&io->u.work); + memalloc_noio_restore(gfp_flag); +} + +static void z_erofs_fill_bio_vec(struct bio_vec *bvec, + struct z_erofs_frontend *f, + struct z_erofs_pcluster *pcl, + unsigned int nr, + struct address_space *mc) +{ + gfp_t gfp = mapping_gfp_mask(mc); + bool tocache = false; + struct z_erofs_bvec zbv; + struct address_space *mapping; + struct folio *folio; + struct page *page; + int bs = i_blocksize(f->inode); + + /* Except for inplace folios, the entire folio can be used for I/Os */ + bvec->bv_offset = 0; + bvec->bv_len = PAGE_SIZE; +repeat: + spin_lock(&pcl->lockref.lock); + zbv = pcl->compressed_bvecs[nr]; + spin_unlock(&pcl->lockref.lock); + if (!zbv.page) + goto out_allocfolio; + + bvec->bv_page = zbv.page; + DBG_BUGON(z_erofs_is_shortlived_page(bvec->bv_page)); + + folio = page_folio(zbv.page); + /* + * Preallocated folios are added to the managed cache here rather than + * in z_erofs_bind_cache() in order to keep these folios locked in + * increasing (physical) address order. + * Clear folio->private before these folios become visible to others in + * the managed cache to avoid duplicate additions for unaligned extents. + */ + if (folio->private == Z_EROFS_PREALLOCATED_FOLIO) { + folio->private = NULL; + tocache = true; + goto out_tocache; + } + + mapping = READ_ONCE(folio->mapping); + /* + * File-backed folios for inplace I/Os are all locked steady, + * therefore it is impossible for `mapping` to be NULL. + */ + if (mapping && mapping != mc) { + if (zbv.offset < 0) + bvec->bv_offset = round_up(-zbv.offset, bs); + bvec->bv_len = round_up(zbv.end, bs) - bvec->bv_offset; + return; + } + + folio_lock(folio); + if (likely(folio->mapping == mc)) { + /* + * The cached folio is still in managed cache but without + * a valid `->private` pcluster hint. Let's reconnect them. + */ + if (!folio_test_private(folio)) { + folio_attach_private(folio, pcl); + /* compressed_bvecs[] already takes a ref before */ + folio_put(folio); + } + if (likely(folio->private == pcl)) { + /* don't submit cache I/Os again if already uptodate */ + if (folio_test_uptodate(folio)) { + folio_unlock(folio); + bvec->bv_page = NULL; + } + return; + } + } else { + DBG_BUGON(1); /* referenced managed folios can't be truncated */ + } + folio_unlock(folio); + folio_put(folio); +out_allocfolio: + page = __erofs_allocpage(&f->pagepool, gfp, true); + spin_lock(&pcl->lockref.lock); + if (unlikely(pcl->compressed_bvecs[nr].page != zbv.page)) { + if (page) + erofs_pagepool_add(&f->pagepool, page); + spin_unlock(&pcl->lockref.lock); + cond_resched(); + goto repeat; + } + pcl->compressed_bvecs[nr].page = page ? page : ERR_PTR(-ENOMEM); + spin_unlock(&pcl->lockref.lock); + bvec->bv_page = page; + if (!page) + return; + folio = page_folio(page); +out_tocache: + if (!tocache || bs != PAGE_SIZE || + filemap_add_folio(mc, folio, (pcl->pos >> PAGE_SHIFT) + nr, gfp)) { + /* turn into a temporary shortlived folio (1 ref) */ + folio->private = (void *)Z_EROFS_SHORTLIVED_PAGE; + return; + } + folio_attach_private(folio, pcl); + /* drop a refcount added by allocpage (then 2 refs in total here) */ + folio_put(folio); +} + +static struct z_erofs_decompressqueue *jobqueue_init(struct super_block *sb, + struct z_erofs_decompressqueue *fgq, bool *fg) +{ + struct z_erofs_decompressqueue *q; + + if (fg && !*fg) { + q = kvzalloc_obj(*q, GFP_KERNEL | __GFP_NOWARN); + if (!q) { + *fg = true; + goto fg_out; + } +#ifdef CONFIG_EROFS_FS_PCPU_KTHREAD + kthread_init_work(&q->u.kthread_work, + z_erofs_decompressqueue_kthread_work); +#else + INIT_WORK(&q->u.work, z_erofs_decompressqueue_work); +#endif + } else { +fg_out: + q = fgq; + init_completion(&fgq->u.done); + atomic_set(&fgq->pending_bios, 0); + q->eio = false; + q->sync = true; + } + q->sb = sb; + q->head = Z_EROFS_PCLUSTER_TAIL; + return q; +} + +/* define decompression jobqueue types */ +enum { + JQ_BYPASS, + JQ_SUBMIT, + NR_JOBQUEUES, +}; + +static void z_erofs_move_to_bypass_queue(struct z_erofs_pcluster *pcl, + struct z_erofs_pcluster *next, + struct z_erofs_pcluster **qtail[]) +{ + WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_TAIL); + WRITE_ONCE(*qtail[JQ_SUBMIT], next); + WRITE_ONCE(*qtail[JQ_BYPASS], pcl); + qtail[JQ_BYPASS] = &pcl->next; +} + +static void z_erofs_endio(struct bio *bio) +{ + struct z_erofs_decompressqueue *q = bio->bi_private; + blk_status_t err = bio->bi_status; + struct folio_iter fi; + + bio_for_each_folio_all(fi, bio) { + struct folio *folio = fi.folio; + + DBG_BUGON(folio_test_uptodate(folio)); + DBG_BUGON(z_erofs_page_is_invalidated(&folio->page)); + if (!erofs_folio_is_managed(EROFS_SB(q->sb), folio)) + continue; + + if (!err) + folio_mark_uptodate(folio); + folio_unlock(folio); + } + if (err) + q->eio = true; + z_erofs_decompress_kickoff(q, -1); + if (bio->bi_bdev) + bio_put(bio); +} + +static void z_erofs_submit_queue(struct z_erofs_frontend *f, + struct z_erofs_decompressqueue *fgq, + bool *force_fg, bool readahead) +{ + struct super_block *sb = f->inode->i_sb; + struct address_space *mc = MNGD_MAPPING(EROFS_SB(sb)); + struct z_erofs_pcluster **qtail[NR_JOBQUEUES]; + struct z_erofs_decompressqueue *q[NR_JOBQUEUES]; + struct z_erofs_pcluster *pcl, *next; + /* bio is NULL initially, so no need to initialize last_{index,bdev} */ + erofs_off_t last_pa; + unsigned int nr_bios = 0; + struct bio *bio = NULL; + unsigned long pflags; + int memstall = 0; + + /* No need to read from device for pclusters in the bypass queue. */ + q[JQ_BYPASS] = jobqueue_init(sb, fgq + JQ_BYPASS, NULL); + q[JQ_SUBMIT] = jobqueue_init(sb, fgq + JQ_SUBMIT, force_fg); + + qtail[JQ_BYPASS] = &q[JQ_BYPASS]->head; + qtail[JQ_SUBMIT] = &q[JQ_SUBMIT]->head; + + /* by default, all need io submission */ + q[JQ_SUBMIT]->head = next = f->head; + + do { + struct erofs_map_dev mdev; + erofs_off_t cur, end; + struct bio_vec bvec; + unsigned int i = 0; + bool bypass = true; + + pcl = next; + next = READ_ONCE(pcl->next); + if (pcl->from_meta) { + z_erofs_move_to_bypass_queue(pcl, next, qtail); + continue; + } + + /* no device id here, thus it will always succeed */ + mdev = (struct erofs_map_dev) { + .m_pa = round_down(pcl->pos, sb->s_blocksize), + }; + (void)erofs_map_dev(sb, &mdev); + + cur = mdev.m_pa; + end = round_up(cur + pcl->pageofs_in + pcl->pclustersize, + sb->s_blocksize); + do { + bvec.bv_page = NULL; + if (bio && (cur != last_pa || + bio->bi_bdev != mdev.m_bdev)) { +drain_io: + if (erofs_is_fileio_mode(EROFS_SB(sb))) + erofs_fileio_submit_bio(bio); + else + submit_bio(bio); + + if (memstall) { + psi_memstall_leave(&pflags); + memstall = 0; + } + bio = NULL; + } + + if (!bvec.bv_page) { + z_erofs_fill_bio_vec(&bvec, f, pcl, i++, mc); + if (!bvec.bv_page) + continue; + if (cur + bvec.bv_len > end) + bvec.bv_len = end - cur; + DBG_BUGON(bvec.bv_len < sb->s_blocksize); + } + + if (unlikely(PageWorkingset(bvec.bv_page)) && + !memstall) { + psi_memstall_enter(&pflags); + memstall = 1; + } + + if (!bio) { + if (erofs_is_fileio_mode(EROFS_SB(sb))) + bio = erofs_fileio_bio_alloc(&mdev); + else + bio = bio_alloc(mdev.m_bdev, BIO_MAX_VECS, + REQ_OP_READ, GFP_NOIO); + bio->bi_end_io = z_erofs_endio; + bio->bi_iter.bi_sector = + (mdev.m_dif->fsoff + cur) >> 9; + bio->bi_private = q[JQ_SUBMIT]; + if (readahead) + bio->bi_opf |= REQ_RAHEAD; + ++nr_bios; + } + + if (!bio_add_page(bio, bvec.bv_page, bvec.bv_len, + bvec.bv_offset)) + goto drain_io; + last_pa = cur + bvec.bv_len; + bypass = false; + } while ((cur += bvec.bv_len) < end); + + if (!bypass) + qtail[JQ_SUBMIT] = &pcl->next; + else + z_erofs_move_to_bypass_queue(pcl, next, qtail); + } while (next != Z_EROFS_PCLUSTER_TAIL); + + if (bio) { + if (erofs_is_fileio_mode(EROFS_SB(sb))) + erofs_fileio_submit_bio(bio); + else + submit_bio(bio); + } + if (memstall) + psi_memstall_leave(&pflags); + + /* + * although background is preferred, no one is pending for submission. + * don't issue decompression but drop it directly instead. + */ + if (!*force_fg && !nr_bios) { + kvfree(q[JQ_SUBMIT]); + return; + } + z_erofs_decompress_kickoff(q[JQ_SUBMIT], nr_bios); +} + +static int z_erofs_runqueue(struct z_erofs_frontend *f, unsigned int rabytes) +{ + struct z_erofs_decompressqueue io[NR_JOBQUEUES]; + struct erofs_sb_info *sbi = EROFS_I_SB(f->inode); + int syncmode = sbi->sync_decompress; + bool force_fg; + int err; + + force_fg = (syncmode == EROFS_SYNC_DECOMPRESS_AUTO && !rabytes) || + (syncmode == EROFS_SYNC_DECOMPRESS_FORCE_ON && + (rabytes <= Z_EROFS_MAX_SYNC_DECOMPRESS_BYTES)); + + if (f->head == Z_EROFS_PCLUSTER_TAIL) + return 0; + z_erofs_submit_queue(f, io, &force_fg, !!rabytes); + + /* handle bypass queue (no i/o pclusters) immediately */ + err = z_erofs_decompress_queue(&io[JQ_BYPASS], &f->pagepool); + if (!force_fg) + return err; + + /* wait until all bios are completed */ + wait_for_completion_io(&io[JQ_SUBMIT].u.done); + + /* handle synchronous decompress queue in the caller context */ + return z_erofs_decompress_queue(&io[JQ_SUBMIT], &f->pagepool) ?: err; +} + +/* + * Since partial uptodate is still unimplemented for now, we have to use + * approximate readmore strategies as a start. + */ +static void z_erofs_pcluster_readmore(struct z_erofs_frontend *f, + struct readahead_control *rac, bool backmost) +{ + struct inode *inode = f->inode; + struct erofs_map_blocks *map = &f->map; + erofs_off_t cur, end, headoffset = f->headoffset; + int err; + + if (backmost) { + if (rac) + end = headoffset + readahead_length(rac) - 1; + else + end = headoffset + PAGE_SIZE - 1; + map->m_la = end; + err = z_erofs_map_blocks_iter(inode, map, + EROFS_GET_BLOCKS_READMORE); + if (err || !(map->m_flags & EROFS_MAP_MAPPED)) + return; + + /* expand ra for the trailing edge if readahead */ + if (rac) { + cur = round_up(map->m_la + map->m_llen, PAGE_SIZE); + readahead_expand(rac, headoffset, cur - headoffset); + return; + } + end = round_up(end, PAGE_SIZE); + } else { + end = round_up(map->m_la, PAGE_SIZE); + if (!(map->m_flags & EROFS_MAP_MAPPED) || !map->m_llen) + return; + } + + cur = map->m_la + map->m_llen - 1; + while ((cur >= end) && (cur < i_size_read(inode))) { + pgoff_t index = cur >> PAGE_SHIFT; + struct folio *folio; + + folio = erofs_grab_folio_nowait(f->sharedinode->i_mapping, index); + if (!IS_ERR_OR_NULL(folio)) { + if (folio_test_uptodate(folio)) + folio_unlock(folio); + else + z_erofs_scan_folio(f, folio, !!rac); + folio_put(folio); + } + + if (cur < PAGE_SIZE) + break; + cur = ((loff_t)index << PAGE_SHIFT) - 1; + } +} + +static int z_erofs_read_folio(struct file *file, struct folio *folio) +{ + struct inode *sharedinode = folio->mapping->host; + bool need_iput; + struct inode *realinode = erofs_real_inode(sharedinode, &need_iput); + Z_EROFS_DEFINE_FRONTEND(f, realinode, sharedinode, folio_pos(folio)); + int err; + + trace_erofs_read_folio(realinode, folio, false); + z_erofs_pcluster_readmore(&f, NULL, true); + err = z_erofs_scan_folio(&f, folio, false); + z_erofs_pcluster_readmore(&f, NULL, false); + z_erofs_pcluster_end(&f); + + /* if some pclusters are ready, need submit them anyway */ + err = z_erofs_runqueue(&f, 0) ?: err; + if (err && err != -EINTR) + erofs_err(realinode->i_sb, "read error %d @ %lu of nid %llu", + err, folio->index, EROFS_I(realinode)->nid); + + erofs_put_metabuf(&f.map.buf); + erofs_release_pages(&f.pagepool); + + if (need_iput) + iput(realinode); + return err; +} + +static void z_erofs_readahead(struct readahead_control *rac) +{ + struct inode *sharedinode = rac->mapping->host; + bool need_iput; + struct inode *realinode = erofs_real_inode(sharedinode, &need_iput); + Z_EROFS_DEFINE_FRONTEND(f, realinode, sharedinode, readahead_pos(rac)); + unsigned int nrpages = readahead_count(rac); + struct folio *head = NULL, *folio; + int err; + + trace_erofs_readahead(realinode, readahead_index(rac), nrpages, false); + z_erofs_pcluster_readmore(&f, rac, true); + while ((folio = readahead_folio(rac))) { + folio->private = head; + head = folio; + } + + /* traverse in reverse order for best metadata I/O performance */ + while (head) { + folio = head; + head = folio_get_private(folio); + + err = z_erofs_scan_folio(&f, folio, true); + if (err && err != -EINTR) + erofs_err(realinode->i_sb, "readahead error at folio %lu @ nid %llu", + folio->index, EROFS_I(realinode)->nid); + } + z_erofs_pcluster_readmore(&f, rac, false); + z_erofs_pcluster_end(&f); + + (void)z_erofs_runqueue(&f, nrpages << PAGE_SHIFT); + erofs_put_metabuf(&f.map.buf); + erofs_release_pages(&f.pagepool); + + if (need_iput) + iput(realinode); +} + +const struct address_space_operations z_erofs_aops = { + .read_folio = z_erofs_read_folio, + .readahead = z_erofs_readahead, +}; diff --git a/fs/erofs/zmap.c b/fs/erofs/zmap.c new file mode 100644 index 000000000..f8981fc74 --- /dev/null +++ b/fs/erofs/zmap.c @@ -0,0 +1,838 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2018-2019 HUAWEI, Inc. + * https://www.huawei.com/ + */ +#include "internal.h" +#include +#include + +struct z_erofs_maprecorder { + struct inode *inode; + struct erofs_map_blocks *map; + u64 lcn; + /* compression extent information gathered */ + u8 type, headtype; + u16 clusterofs; + u16 delta[2]; + erofs_blk_t pblk; + erofs_off_t nextpackoff; + int compressedblks; + bool partialref, in_mbox; +}; + +static int z_erofs_load_full_lcluster(struct z_erofs_maprecorder *m, u64 lcn) +{ + struct inode *const inode = m->inode; + struct erofs_inode *const vi = EROFS_I(inode); + const erofs_off_t pos = Z_EROFS_FULL_INDEX_START(erofs_iloc(inode) + + vi->inode_isize + vi->xattr_isize) + + lcn * sizeof(struct z_erofs_lcluster_index); + struct z_erofs_lcluster_index *di; + unsigned int advise; + + di = erofs_read_metabuf(&m->map->buf, inode->i_sb, pos, m->in_mbox); + if (IS_ERR(di)) + return PTR_ERR(di); + m->lcn = lcn; + m->nextpackoff = pos + sizeof(struct z_erofs_lcluster_index); + + advise = le16_to_cpu(di->di_advise); + m->type = advise & Z_EROFS_LI_LCLUSTER_TYPE_MASK; + if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + m->clusterofs = 1 << vi->z_lclusterbits; + m->delta[0] = le16_to_cpu(di->di_u.delta[0]); + if (m->delta[0] & Z_EROFS_LI_D0_CBLKCNT) { + if (!(vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | + Z_EROFS_ADVISE_BIG_PCLUSTER_2))) { + DBG_BUGON(1); + return -EFSCORRUPTED; + } + m->compressedblks = m->delta[0] & ~Z_EROFS_LI_D0_CBLKCNT; + m->delta[0] = 1; + } + m->delta[1] = le16_to_cpu(di->di_u.delta[1]); + } else { + m->partialref = !!(advise & Z_EROFS_LI_PARTIAL_REF); + m->clusterofs = le16_to_cpu(di->di_clusterofs); + if (advise & Z_EROFS_LI_HOLE) { + m->compressedblks = 0; + m->pblk = EROFS_NULL_ADDR; + } else { + m->pblk = le32_to_cpu(di->di_u.blkaddr); + } + } + return 0; +} + +static unsigned int decode_compactedbits(unsigned int lobits, + u8 *in, unsigned int pos, u8 *type) +{ + const unsigned int v = get_unaligned_le32(in + pos / 8) >> (pos & 7); + const unsigned int lo = v & ((1 << lobits) - 1); + + *type = (v >> lobits) & 3; + return lo; +} + +static int get_compacted_la_distance(unsigned int lobits, + unsigned int encodebits, + unsigned int vcnt, u8 *in, int i) +{ + unsigned int lo, d1 = 0; + u8 type; + + DBG_BUGON(i >= vcnt); + + do { + lo = decode_compactedbits(lobits, in, encodebits * i, &type); + + if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) + return d1; + ++d1; + } while (++i < vcnt); + + /* vcnt - 1 (Z_EROFS_LCLUSTER_TYPE_NONHEAD) item */ + if (!(lo & Z_EROFS_LI_D0_CBLKCNT)) + d1 += lo - 1; + return d1; +} + +static int z_erofs_load_compact_lcluster(struct z_erofs_maprecorder *m, + u64 lcn, bool lookahead) +{ + struct inode *const inode = m->inode; + struct erofs_inode *const vi = EROFS_I(inode); + const erofs_off_t ebase = Z_EROFS_MAP_HEADER_END(erofs_iloc(inode) + + vi->inode_isize + vi->xattr_isize); + const unsigned int lclusterbits = vi->z_lclusterbits; + const unsigned int totalidx = erofs_iblks(inode); + unsigned int compacted_4b_initial, compacted_2b, amortizedshift; + unsigned int vcnt, lo, lobits, encodebits, nblk, bytes; + bool big_pcluster = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1; + erofs_off_t pos; + u8 *in, type; + int i; + + if (lcn >= totalidx || lclusterbits > 14) + return -EINVAL; + + m->lcn = lcn; + /* used to align to 32-byte (compacted_2b) alignment */ + compacted_4b_initial = ((32 - ebase % 32) / 4) & 7; + compacted_2b = 0; + if ((vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) && + compacted_4b_initial < totalidx) + compacted_2b = rounddown(totalidx - compacted_4b_initial, 16); + + pos = ebase; + amortizedshift = 2; /* compact_4b */ + if (lcn >= compacted_4b_initial) { + pos += compacted_4b_initial * 4; + lcn -= compacted_4b_initial; + if (lcn < compacted_2b) { + amortizedshift = 1; + } else { + pos += compacted_2b * 2; + lcn -= compacted_2b; + } + } + pos += lcn * (1 << amortizedshift); + + /* figure out the lcluster count in this pack */ + if (1 << amortizedshift == 4 && lclusterbits <= 14) + vcnt = 2; + else if (1 << amortizedshift == 2 && lclusterbits <= 12) + vcnt = 16; + else + return -EOPNOTSUPP; + + in = erofs_read_metabuf(&m->map->buf, inode->i_sb, pos, m->in_mbox); + if (IS_ERR(in)) + return PTR_ERR(in); + + /* it doesn't equal to round_up(..) */ + m->nextpackoff = round_down(pos, vcnt << amortizedshift) + + (vcnt << amortizedshift); + lobits = max(lclusterbits, ilog2(Z_EROFS_LI_D0_CBLKCNT) + 1U); + encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt; + bytes = pos & ((vcnt << amortizedshift) - 1); + in -= bytes; + i = bytes >> amortizedshift; + + lo = decode_compactedbits(lobits, in, encodebits * i, &type); + m->type = type; + if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + m->clusterofs = 1 << lclusterbits; + + /* figure out lookahead_distance: delta[1] if needed */ + if (lookahead) + m->delta[1] = get_compacted_la_distance(lobits, + encodebits, vcnt, in, i); + if (lo & Z_EROFS_LI_D0_CBLKCNT) { + if (!big_pcluster) { + DBG_BUGON(1); + return -EFSCORRUPTED; + } + m->compressedblks = lo & ~Z_EROFS_LI_D0_CBLKCNT; + m->delta[0] = 1; + return 0; + } else if (i + 1 != (int)vcnt) { + m->delta[0] = lo; + return 0; + } + /* + * since the last lcluster in the pack is special, + * of which lo saves delta[1] rather than delta[0]. + * Hence, get delta[0] by the previous lcluster indirectly. + */ + lo = decode_compactedbits(lobits, in, + encodebits * (i - 1), &type); + if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) + lo = 0; + else if (lo & Z_EROFS_LI_D0_CBLKCNT) + lo = 1; + m->delta[0] = lo + 1; + return 0; + } + m->clusterofs = lo; + m->delta[0] = 0; + /* figout out blkaddr (pblk) for HEAD lclusters */ + if (!big_pcluster) { + nblk = 1; + while (i > 0) { + --i; + lo = decode_compactedbits(lobits, in, + encodebits * i, &type); + if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) + i -= lo; + + if (i >= 0) + ++nblk; + } + } else { + nblk = 0; + while (i > 0) { + --i; + lo = decode_compactedbits(lobits, in, + encodebits * i, &type); + if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + if (lo & Z_EROFS_LI_D0_CBLKCNT) { + --i; + nblk += lo & ~Z_EROFS_LI_D0_CBLKCNT; + continue; + } + /* bigpcluster shouldn't have plain d0 == 1 */ + if (lo <= 1) { + DBG_BUGON(1); + return -EFSCORRUPTED; + } + i -= lo - 2; + continue; + } + ++nblk; + } + } + in += (vcnt << amortizedshift) - sizeof(__le32); + m->pblk = le32_to_cpu(*(__le32 *)in) + nblk; + return 0; +} + +static int z_erofs_load_lcluster_from_disk(struct z_erofs_maprecorder *m, + u64 lcn, bool lookahead) +{ + struct erofs_inode *vi = EROFS_I(m->inode); + int err; + + if (vi->datalayout == EROFS_INODE_COMPRESSED_COMPACT) { + err = z_erofs_load_compact_lcluster(m, lcn, lookahead); + } else { + DBG_BUGON(vi->datalayout != EROFS_INODE_COMPRESSED_FULL); + err = z_erofs_load_full_lcluster(m, lcn); + } + if (err) + return err; + + if (m->type >= Z_EROFS_LCLUSTER_TYPE_MAX) { + erofs_err(m->inode->i_sb, "unknown type %u @ lcn %llu of nid %llu", + m->type, lcn, EROFS_I(m->inode)->nid); + DBG_BUGON(1); + return -EOPNOTSUPP; + } else if (m->type != Z_EROFS_LCLUSTER_TYPE_NONHEAD && + m->clusterofs >= (1 << vi->z_lclusterbits)) { + DBG_BUGON(1); + return -EFSCORRUPTED; + } + return 0; +} + +static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m, + unsigned int lookback_distance) +{ + struct super_block *sb = m->inode->i_sb; + struct erofs_inode *const vi = EROFS_I(m->inode); + const unsigned int lclusterbits = vi->z_lclusterbits; + + while (m->lcn >= lookback_distance) { + u64 lcn = m->lcn - lookback_distance; + int err; + + if (!lookback_distance) + break; + + err = z_erofs_load_lcluster_from_disk(m, lcn, false); + if (err) + return err; + if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + lookback_distance = m->delta[0]; + continue; + } + m->headtype = m->type; + m->map->m_la = (lcn << lclusterbits) | m->clusterofs; + return 0; + } + erofs_err(sb, "bogus lookback distance %u @ lcn %llu of nid %llu", + lookback_distance, m->lcn, vi->nid); + DBG_BUGON(1); + return -EFSCORRUPTED; +} + +static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, + unsigned int initial_lcn) +{ + struct inode *inode = m->inode; + struct super_block *sb = inode->i_sb; + struct erofs_inode *vi = EROFS_I(inode); + bool bigpcl1 = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1; + bool bigpcl2 = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2; + u64 lcn = m->lcn + 1; + int err; + + DBG_BUGON(m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD); + DBG_BUGON(m->type != m->headtype); + + if ((m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD1 && !bigpcl1) || + ((m->headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN || + m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) && !bigpcl2) || + (lcn << vi->z_lclusterbits) >= inode->i_size) + if (m->compressedblks < 0) + m->compressedblks = 1; + + if (m->compressedblks >= 0) + goto out; + + err = z_erofs_load_lcluster_from_disk(m, lcn, false); + if (err) + return err; + + /* + * If the 1st NONHEAD lcluster has already been handled initially w/o + * valid compressedblks, which means at least it mustn't be CBLKCNT, or + * an internal implemenatation error is detected. + * + * The following code can also handle it properly anyway, but let's + * BUG_ON in the debugging mode only for developers to notice that. + */ + DBG_BUGON(lcn == initial_lcn && + m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD); + + if (m->type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + /* + * if the 1st NONHEAD lcluster is actually PLAIN or HEAD type + * rather than CBLKCNT, it's a 1 block-sized pcluster. + */ + if (m->compressedblks < 0) + m->compressedblks = 1; + } else if (m->delta[0] != 1 || m->compressedblks < 0) { + erofs_err(sb, "bogus CBLKCNT @ lcn %llu of nid %llu", lcn, vi->nid); + DBG_BUGON(1); + return -EFSCORRUPTED; + } + +out: + if (!m->compressedblks) + m->map->m_flags &= ~EROFS_MAP_MAPPED; + m->map->m_plen = erofs_pos(sb, m->compressedblks); + return 0; +} + +static int z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder *m) +{ + struct inode *inode = m->inode; + struct erofs_inode *vi = EROFS_I(inode); + struct erofs_map_blocks *map = m->map; + unsigned int lclusterbits = vi->z_lclusterbits; + u64 lcn = m->lcn, headlcn = map->m_la >> lclusterbits; + int err; + + while (1) { + /* handle the last EOF pcluster (no next HEAD lcluster) */ + if ((lcn << lclusterbits) >= inode->i_size) { + map->m_llen = inode->i_size - map->m_la; + return 0; + } + + err = z_erofs_load_lcluster_from_disk(m, lcn, true); + if (err) + return err; + + if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + /* work around invalid d1 generated by pre-1.0 mkfs */ + if (unlikely(!m->delta[1])) { + m->delta[1] = 1; + DBG_BUGON(1); + } + } else if (m->type < Z_EROFS_LCLUSTER_TYPE_MAX) { + if (lcn != headlcn) + break; /* ends at the next HEAD lcluster */ + m->delta[1] = 1; + } + lcn += m->delta[1]; + } + map->m_llen = (lcn << lclusterbits) + m->clusterofs - map->m_la; + return 0; +} + +static int z_erofs_map_blocks_fo(struct inode *inode, + struct erofs_map_blocks *map, int flags) +{ + struct erofs_inode *vi = EROFS_I(inode); + struct super_block *sb = inode->i_sb; + bool fragment = vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER; + bool ztailpacking = vi->z_idata_size; + unsigned int lclusterbits = vi->z_lclusterbits; + struct z_erofs_maprecorder m = { + .inode = inode, + .map = map, + .in_mbox = erofs_inode_in_metabox(inode), + .compressedblks = -1, + }; + unsigned int endoff; + unsigned long initial_lcn; + unsigned long long ofs, end; + int err; + + ofs = flags & EROFS_GET_BLOCKS_FINDTAIL ? inode->i_size - 1 : map->m_la; + if (fragment && !(flags & EROFS_GET_BLOCKS_FINDTAIL) && + !vi->z_tailextent_headlcn) { + map->m_la = 0; + map->m_llen = inode->i_size; + map->m_flags = EROFS_MAP_FRAGMENT; + return 0; + } + initial_lcn = ofs >> lclusterbits; + endoff = ofs & ((1 << lclusterbits) - 1); + + err = z_erofs_load_lcluster_from_disk(&m, initial_lcn, false); + if (err) + goto unmap_out; + + if ((flags & EROFS_GET_BLOCKS_FINDTAIL) && ztailpacking) + vi->z_fragmentoff = m.nextpackoff; + map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_PARTIAL_MAPPED; + end = (m.lcn + 1ULL) << lclusterbits; + + if (m.type != Z_EROFS_LCLUSTER_TYPE_NONHEAD && endoff >= m.clusterofs) { + m.headtype = m.type; + map->m_la = (m.lcn << lclusterbits) | m.clusterofs; + /* + * For ztailpacking files, in order to inline data more + * effectively, special EOF lclusters are now supported + * which can have three parts at most. + */ + if (ztailpacking && end > inode->i_size) + end = inode->i_size; + } else { + if (m.type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) { + end = (m.lcn << lclusterbits) | m.clusterofs; + map->m_flags &= ~EROFS_MAP_PARTIAL_MAPPED; + m.delta[0] = 1; + } + /* get the corresponding first chunk */ + err = z_erofs_extent_lookback(&m, m.delta[0]); + if (err) + goto unmap_out; + } + if (m.partialref) + map->m_flags |= EROFS_MAP_PARTIAL_REF; + map->m_llen = end - map->m_la; + + if (flags & EROFS_GET_BLOCKS_FINDTAIL) { + vi->z_tailextent_headlcn = m.lcn; + /* for non-compact indexes, fragmentoff is 64 bits */ + if (fragment && vi->datalayout == EROFS_INODE_COMPRESSED_FULL) + vi->z_fragmentoff |= (u64)m.pblk << 32; + } + if (ztailpacking && m.lcn == vi->z_tailextent_headlcn) { + map->m_flags |= EROFS_MAP_META; + map->m_pa = vi->z_fragmentoff; + map->m_plen = vi->z_idata_size; + if (erofs_blkoff(sb, map->m_pa) + map->m_plen > sb->s_blocksize) { + erofs_err(sb, "ztailpacking inline data across blocks @ nid %llu", + vi->nid); + err = -EFSCORRUPTED; + goto unmap_out; + } + } else if (fragment && m.lcn == vi->z_tailextent_headlcn) { + map->m_flags = EROFS_MAP_FRAGMENT; + } else { + map->m_pa = erofs_pos(sb, m.pblk); + err = z_erofs_get_extent_compressedlen(&m, initial_lcn); + if (err) + goto unmap_out; + } + + if (m.headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN) { + if ((vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) && + !(map->m_flags & EROFS_MAP_META)) + map->m_algorithmformat = Z_EROFS_COMPRESSION_INTERLACED; + else + map->m_algorithmformat = Z_EROFS_COMPRESSION_SHIFTED; + } else { + map->m_algorithmformat = + vi->z_algofmt[m.headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2]; + } + + if ((flags & EROFS_GET_BLOCKS_FIEMAP) || + ((flags & EROFS_GET_BLOCKS_READMORE) && + (map->m_algorithmformat == Z_EROFS_COMPRESSION_LZMA || + map->m_algorithmformat == Z_EROFS_COMPRESSION_DEFLATE || + map->m_algorithmformat == Z_EROFS_COMPRESSION_ZSTD) && + map->m_llen >= i_blocksize(inode))) { + err = z_erofs_get_extent_decompressedlen(&m); + if (!err) + map->m_flags &= ~EROFS_MAP_PARTIAL_MAPPED; + } + +unmap_out: + erofs_unmap_metabuf(&m.map->buf); + return err; +} + +static int z_erofs_map_blocks_ext(struct inode *inode, + struct erofs_map_blocks *map, int flags) +{ + struct erofs_inode *vi = EROFS_I(inode); + struct super_block *sb = inode->i_sb; + bool interlaced = vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER; + unsigned int recsz = z_erofs_extent_recsize(vi->z_advise); + erofs_off_t pos = round_up(Z_EROFS_MAP_HEADER_END(erofs_iloc(inode) + + vi->inode_isize + vi->xattr_isize), recsz); + unsigned int bmask = sb->s_blocksize - 1; + bool in_mbox = erofs_inode_in_metabox(inode); + erofs_off_t lend = inode->i_size; + erofs_off_t l, r, mid, pa, la, lstart; + struct z_erofs_extent *ext; + unsigned int fmt; + bool last; + + map->m_flags = 0; + if (recsz <= offsetof(struct z_erofs_extent, pstart_hi)) { + if (recsz <= offsetof(struct z_erofs_extent, pstart_lo)) { + ext = erofs_read_metabuf(&map->buf, sb, pos, in_mbox); + if (IS_ERR(ext)) + return PTR_ERR(ext); + pa = le64_to_cpu(*(__le64 *)ext); + pos += sizeof(__le64); + lstart = 0; + } else { + lstart = round_down(map->m_la, 1 << vi->z_lclusterbits); + pos += (lstart >> vi->z_lclusterbits) * recsz; + pa = EROFS_NULL_ADDR; + } + + for (; lstart <= map->m_la; lstart += 1 << vi->z_lclusterbits) { + ext = erofs_read_metabuf(&map->buf, sb, pos, in_mbox); + if (IS_ERR(ext)) + return PTR_ERR(ext); + map->m_plen = le32_to_cpu(ext->plen); + if (pa != EROFS_NULL_ADDR) { + map->m_pa = pa; + pa += map->m_plen & Z_EROFS_EXTENT_PLEN_MASK; + } else { + map->m_pa = le32_to_cpu(ext->pstart_lo); + } + pos += recsz; + } + last = (lstart >= round_up(lend, 1 << vi->z_lclusterbits)); + lend = min(lstart, lend); + lstart -= 1 << vi->z_lclusterbits; + } else { + lstart = lend; + for (l = 0, r = vi->z_extents; l < r; ) { + mid = l + (r - l) / 2; + ext = erofs_read_metabuf(&map->buf, sb, + pos + mid * recsz, in_mbox); + if (IS_ERR(ext)) + return PTR_ERR(ext); + + la = le32_to_cpu(ext->lstart_lo); + pa = le32_to_cpu(ext->pstart_lo) | + (u64)le32_to_cpu(ext->pstart_hi) << 32; + if (recsz > offsetof(struct z_erofs_extent, lstart_hi)) + la |= (u64)le32_to_cpu(ext->lstart_hi) << 32; + + if (la > map->m_la) { + r = mid; + if (la > lend) { + DBG_BUGON(1); + return -EFSCORRUPTED; + } + lend = la; + } else { + l = mid + 1; + if (map->m_la == la) + r = min(l + 1, r); + lstart = la; + map->m_plen = le32_to_cpu(ext->plen); + map->m_pa = pa; + } + } + last = (l >= vi->z_extents); + } + + if (lstart < lend) { + map->m_la = lstart; + if (last && (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER)) { + map->m_flags = EROFS_MAP_FRAGMENT; + vi->z_fragmentoff = map->m_plen; + if (recsz > offsetof(struct z_erofs_extent, pstart_lo)) + vi->z_fragmentoff |= map->m_pa << 32; + } else if (map->m_plen & Z_EROFS_EXTENT_PLEN_MASK) { + map->m_flags |= EROFS_MAP_MAPPED; + fmt = map->m_plen >> Z_EROFS_EXTENT_PLEN_FMT_BIT; + if (map->m_plen & Z_EROFS_EXTENT_PLEN_PARTIAL) + map->m_flags |= EROFS_MAP_PARTIAL_REF; + map->m_plen &= Z_EROFS_EXTENT_PLEN_MASK; + if (fmt) { + map->m_algorithmformat = --fmt; + if (fmt >= Z_EROFS_COMPRESSION_MAX) { + erofs_err(sb, "unknown algorithm %d @ pos %llu for nid %llu, please upgrade kernel", + fmt, map->m_la, vi->nid); + return -EOPNOTSUPP; + } + } else if (interlaced && !((map->m_pa | map->m_plen) & bmask)) + map->m_algorithmformat = + Z_EROFS_COMPRESSION_INTERLACED; + else + map->m_algorithmformat = + Z_EROFS_COMPRESSION_SHIFTED; + } + } + map->m_llen = lend - map->m_la; + return 0; +} + +static int z_erofs_fill_inode(struct inode *inode, struct erofs_map_blocks *map) +{ + struct erofs_inode *const vi = EROFS_I(inode); + struct super_block *const sb = inode->i_sb; + struct z_erofs_map_header *h; + erofs_off_t pos; + int err = 0, nr; + + if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) { + /* + * paired with smp_mb() at the end of the function to ensure + * fields will only be observed after the bit is set. + */ + smp_mb(); + return 0; + } + + if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_Z_BIT, TASK_KILLABLE)) + return -ERESTARTSYS; + + if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) + goto out_unlock; + + pos = ALIGN(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize, 8); + h = erofs_read_metabuf(&map->buf, sb, pos, erofs_inode_in_metabox(inode)); + if (IS_ERR(h)) { + err = PTR_ERR(h); + goto out_unlock; + } + + /* + * if the highest bit of the 8-byte map header is set, the whole file + * is stored in the packed inode. The rest bits keeps z_fragmentoff. + */ + if (h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT) { + vi->z_advise = Z_EROFS_ADVISE_FRAGMENT_PCLUSTER; + vi->z_fragmentoff = le64_to_cpu(*(__le64 *)h) ^ (1ULL << 63); + vi->z_tailextent_headlcn = 0; + goto done; + } + vi->z_advise = le16_to_cpu(h->h_advise); + vi->z_lclusterbits = sb->s_blocksize_bits + (h->h_clusterbits & 15); + if (vi->datalayout == EROFS_INODE_COMPRESSED_FULL && + (vi->z_advise & Z_EROFS_ADVISE_EXTENTS)) { + vi->z_extents = le32_to_cpu(h->h_extents_lo) | + ((u64)le16_to_cpu(h->h_extents_hi) << 32); + goto done; + } + + if (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) + vi->z_fragmentoff = le32_to_cpu(h->h_fragmentoff); + else if (vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) + vi->z_idata_size = le16_to_cpu(h->h_idata_size); + for (nr = 0; nr < 2; ++nr) { + vi->z_algofmt[nr] = (h->h_algorithmtype >> (4 * nr)) & 15; + if (vi->z_algofmt[nr] >= Z_EROFS_COMPRESSION_MAX) { + erofs_err(sb, "unknown HEAD%u format %u for nid %llu, please upgrade kernel", + nr + 1, vi->z_algofmt[nr], vi->nid); + err = -EOPNOTSUPP; + goto out_unlock; + } + } + + if (!erofs_sb_has_big_pcluster(EROFS_SB(sb)) && + vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | + Z_EROFS_ADVISE_BIG_PCLUSTER_2)) { + erofs_err(sb, "per-inode big pcluster without sb feature for nid %llu", + vi->nid); + err = -EFSCORRUPTED; + goto out_unlock; + } + if (vi->datalayout == EROFS_INODE_COMPRESSED_COMPACT && + !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) ^ + !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2)) { + erofs_err(sb, "big pcluster head1/2 of compact indexes should be consistent for nid %llu", + vi->nid); + err = -EFSCORRUPTED; + goto out_unlock; + } + + if (vi->z_idata_size || + (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER)) { + struct erofs_map_blocks tm = { + .buf = __EROFS_BUF_INITIALIZER + }; + + err = z_erofs_map_blocks_fo(inode, &tm, + EROFS_GET_BLOCKS_FINDTAIL); + erofs_put_metabuf(&tm.buf); + if (err < 0) + goto out_unlock; + } +done: + /* paired with smp_mb() at the beginning of the function */ + smp_mb(); + set_bit(EROFS_I_Z_INITED_BIT, &vi->flags); +out_unlock: + clear_and_wake_up_bit(EROFS_I_BL_Z_BIT, &vi->flags); + return err; +} + +static int z_erofs_map_sanity_check(struct inode *inode, + struct erofs_map_blocks *map) +{ + struct erofs_sb_info *sbi = EROFS_I_SB(inode); + u64 pend; + + if (!(map->m_flags & EROFS_MAP_MAPPED)) + return 0; + + DBG_BUGON(map->m_algorithmformat >= Z_EROFS_COMPRESSION_RUNTIME_MAX); + if (map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX) { + if (!(sbi->available_compr_algs & BIT(map->m_algorithmformat))) { + erofs_err(inode->i_sb, "inconsistent algorithmtype %u for nid %llu", + map->m_algorithmformat, EROFS_I(inode)->nid); + return -EFSCORRUPTED; + } + if (EROFS_MAP_FULL(map->m_flags) && map->m_llen < map->m_plen && + map->m_la + map->m_llen < inode->i_size) { + erofs_err(inode->i_sb, "too much compressed data @ la %llu of nid %llu", + map->m_la, EROFS_I(inode)->nid); + return -EFSCORRUPTED; + } + } else if (map->m_llen > map->m_plen) { + erofs_err(inode->i_sb, "not enough plain data on disk @ la %llu of nid %llu", + map->m_la, EROFS_I(inode)->nid); + return -EFSCORRUPTED; + } + if (unlikely(map->m_plen > Z_EROFS_PCLUSTER_MAX_SIZE || + map->m_llen > Z_EROFS_PCLUSTER_MAX_DSIZE)) + return -EOPNOTSUPP; + /* Filesystems beyond 48-bit physical block addresses are invalid */ + if (unlikely(check_add_overflow(map->m_pa, map->m_plen, &pend) || + (pend >> sbi->blkszbits) >= BIT_ULL(48))) + return -EFSCORRUPTED; + return 0; +} + +int z_erofs_map_blocks_iter(struct inode *inode, struct erofs_map_blocks *map, + int flags) +{ + struct erofs_inode *const vi = EROFS_I(inode); + int err = 0; + + trace_erofs_map_blocks_enter(inode, map, flags); + if (map->m_la >= inode->i_size) { /* post-EOF unmapped extent */ + map->m_llen = map->m_la + 1 - inode->i_size; + map->m_la = inode->i_size; + map->m_flags = 0; + } else { + err = z_erofs_fill_inode(inode, map); + if (!err) { + if (vi->datalayout == EROFS_INODE_COMPRESSED_FULL && + (vi->z_advise & Z_EROFS_ADVISE_EXTENTS)) + err = z_erofs_map_blocks_ext(inode, map, flags); + else + err = z_erofs_map_blocks_fo(inode, map, flags); + } + if (!err) + err = z_erofs_map_sanity_check(inode, map); + if (err) + map->m_llen = 0; + } + trace_erofs_map_blocks_exit(inode, map, flags, err); + return err; +} + +static int z_erofs_iomap_begin_report(struct inode *inode, loff_t offset, + loff_t length, unsigned int flags, + struct iomap *iomap, struct iomap *srcmap) +{ + int ret; + struct erofs_map_blocks map = { .m_la = offset }; + + ret = z_erofs_map_blocks_iter(inode, &map, EROFS_GET_BLOCKS_FIEMAP); + erofs_put_metabuf(&map.buf); + if (ret < 0) + return ret; + + iomap->bdev = inode->i_sb->s_bdev; + iomap->offset = map.m_la; + iomap->length = map.m_llen; + if (map.m_flags & EROFS_MAP_FRAGMENT) { + iomap->type = IOMAP_MAPPED; + iomap->addr = IOMAP_NULL_ADDR; + } else if (map.m_flags & EROFS_MAP_MAPPED) { + iomap->type = IOMAP_MAPPED; + iomap->addr = map.m_pa; + } else { + iomap->type = IOMAP_HOLE; + iomap->addr = IOMAP_NULL_ADDR; + /* + * No strict rule on how to describe extents for post EOF, yet + * we need to do like below. Otherwise, iomap itself will get + * into an endless loop on post EOF. + * + * Calculate the effective offset by subtracting extent start + * (map.m_la) from the requested offset, and add it to length. + * (NB: offset >= map.m_la always) + */ + if (iomap->offset >= inode->i_size) + iomap->length = length + offset - map.m_la; + } + iomap->flags = 0; + return 0; +} + +static DEFINE_IOMAP_ITER_NEXT(z_erofs_iomap_next_report, + z_erofs_iomap_begin_report); + +const struct iomap_ops z_erofs_iomap_report_ops = { + .iomap_next = z_erofs_iomap_next_report, +}; diff --git a/fs/erofs/zutil.c b/fs/erofs/zutil.c new file mode 100644 index 000000000..a0ce9226b --- /dev/null +++ b/fs/erofs/zutil.c @@ -0,0 +1,316 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2018 HUAWEI, Inc. + * https://www.huawei.com/ + * Copyright (C) 2024 Alibaba Cloud + */ +#include "internal.h" + +struct z_erofs_gbuf { + spinlock_t lock; + void *ptr; + struct page **pages; + unsigned int nrpages; +}; + +static struct z_erofs_gbuf *z_erofs_gbufpool, *z_erofs_rsvbuf; +static unsigned int z_erofs_gbuf_count, z_erofs_gbuf_nrpages, + z_erofs_rsv_nrpages; + +module_param_named(global_buffers, z_erofs_gbuf_count, uint, 0444); +module_param_named(reserved_pages, z_erofs_rsv_nrpages, uint, 0444); + +atomic_long_t erofs_global_shrink_cnt; /* for all mounted instances */ + +/* protects `erofs_sb_list_lock` and the mounted `erofs_sb_list` */ +static DEFINE_SPINLOCK(erofs_sb_list_lock); +static LIST_HEAD(erofs_sb_list); +static unsigned int shrinker_run_no; +static struct shrinker *erofs_shrinker_info; + +static unsigned int z_erofs_gbuf_id(void) +{ + return raw_smp_processor_id() % z_erofs_gbuf_count; +} + +void *z_erofs_get_gbuf(unsigned int requiredpages) + __acquires(gbuf->lock) +{ + struct z_erofs_gbuf *gbuf; + + migrate_disable(); + gbuf = &z_erofs_gbufpool[z_erofs_gbuf_id()]; + spin_lock(&gbuf->lock); + /* check if the buffer is too small */ + if (requiredpages > gbuf->nrpages) { + spin_unlock(&gbuf->lock); + migrate_enable(); + /* (for sparse checker) pretend gbuf->lock is still taken */ + __acquire(gbuf->lock); + return NULL; + } + return gbuf->ptr; +} + +void z_erofs_put_gbuf(void *ptr) __releases(gbuf->lock) +{ + struct z_erofs_gbuf *gbuf; + + gbuf = &z_erofs_gbufpool[z_erofs_gbuf_id()]; + DBG_BUGON(gbuf->ptr != ptr); + spin_unlock(&gbuf->lock); + migrate_enable(); +} + +int z_erofs_gbuf_growsize(unsigned int nrpages) +{ + static DEFINE_MUTEX(gbuf_resize_mutex); + struct page **tmp_pages = NULL; + struct z_erofs_gbuf *gbuf; + void *ptr, *old_ptr; + int last, i, j; + + mutex_lock(&gbuf_resize_mutex); + /* avoid shrinking gbufs, since no idea how many fses rely on */ + if (nrpages <= z_erofs_gbuf_nrpages) { + mutex_unlock(&gbuf_resize_mutex); + return 0; + } + + for (i = 0; i < z_erofs_gbuf_count; ++i) { + gbuf = &z_erofs_gbufpool[i]; + tmp_pages = kzalloc_objs(*tmp_pages, nrpages); + if (!tmp_pages) + goto out; + + for (j = 0; j < gbuf->nrpages; ++j) + tmp_pages[j] = gbuf->pages[j]; + do { + last = j; + j = alloc_pages_bulk(GFP_KERNEL, nrpages, + tmp_pages); + if (last == j) + goto out; + } while (j != nrpages); + + ptr = vmap(tmp_pages, nrpages, VM_MAP, PAGE_KERNEL); + if (!ptr) + goto out; + + spin_lock(&gbuf->lock); + kfree(gbuf->pages); + gbuf->pages = tmp_pages; + old_ptr = gbuf->ptr; + gbuf->ptr = ptr; + gbuf->nrpages = nrpages; + spin_unlock(&gbuf->lock); + if (old_ptr) + vunmap(old_ptr); + } + z_erofs_gbuf_nrpages = nrpages; +out: + if (i < z_erofs_gbuf_count && tmp_pages) { + for (j = 0; j < nrpages; ++j) + if (tmp_pages[j] && (j >= gbuf->nrpages || + tmp_pages[j] != gbuf->pages[j])) + __free_page(tmp_pages[j]); + kfree(tmp_pages); + } + mutex_unlock(&gbuf_resize_mutex); + return i < z_erofs_gbuf_count ? -ENOMEM : 0; +} + +int __init z_erofs_gbuf_init(void) +{ + unsigned int i, total = num_possible_cpus(); + + if (z_erofs_gbuf_count) + total = min(z_erofs_gbuf_count, total); + z_erofs_gbuf_count = total; + + /* The last (special) global buffer is the reserved buffer */ + total += !!z_erofs_rsv_nrpages; + + z_erofs_gbufpool = kzalloc_objs(*z_erofs_gbufpool, total); + if (!z_erofs_gbufpool) + return -ENOMEM; + + if (z_erofs_rsv_nrpages) { + z_erofs_rsvbuf = &z_erofs_gbufpool[total - 1]; + z_erofs_rsvbuf->pages = kzalloc_objs(*z_erofs_rsvbuf->pages, + z_erofs_rsv_nrpages); + if (!z_erofs_rsvbuf->pages) { + z_erofs_rsvbuf = NULL; + z_erofs_rsv_nrpages = 0; + } + } + for (i = 0; i < total; ++i) + spin_lock_init(&z_erofs_gbufpool[i].lock); + return 0; +} + +void z_erofs_gbuf_exit(void) +{ + int i, j; + + for (i = 0; i < z_erofs_gbuf_count + (!!z_erofs_rsvbuf); ++i) { + struct z_erofs_gbuf *gbuf = &z_erofs_gbufpool[i]; + + if (gbuf->ptr) { + vunmap(gbuf->ptr); + gbuf->ptr = NULL; + } + + if (!gbuf->pages) + continue; + + for (j = 0; j < gbuf->nrpages; ++j) + if (gbuf->pages[j]) + put_page(gbuf->pages[j]); + kfree(gbuf->pages); + gbuf->pages = NULL; + } + kfree(z_erofs_gbufpool); +} + +struct page *__erofs_allocpage(struct page **pagepool, gfp_t gfp, bool tryrsv) +{ + struct page *page = *pagepool; + + if (page) { + *pagepool = (struct page *)page_private(page); + } else if (tryrsv && z_erofs_rsvbuf && z_erofs_rsvbuf->nrpages) { + spin_lock(&z_erofs_rsvbuf->lock); + if (z_erofs_rsvbuf->nrpages) + page = z_erofs_rsvbuf->pages[--z_erofs_rsvbuf->nrpages]; + spin_unlock(&z_erofs_rsvbuf->lock); + } + if (!page) + page = alloc_page(gfp); + DBG_BUGON(page && page_ref_count(page) != 1); + return page; +} + +void erofs_release_pages(struct page **pagepool) +{ + while (*pagepool) { + struct page *page = *pagepool; + + *pagepool = (struct page *)page_private(page); + /* try to fill reserved global pool first */ + if (z_erofs_rsvbuf && z_erofs_rsvbuf->nrpages < + z_erofs_rsv_nrpages) { + spin_lock(&z_erofs_rsvbuf->lock); + if (z_erofs_rsvbuf->nrpages < z_erofs_rsv_nrpages) { + z_erofs_rsvbuf->pages[z_erofs_rsvbuf->nrpages++] + = page; + spin_unlock(&z_erofs_rsvbuf->lock); + continue; + } + spin_unlock(&z_erofs_rsvbuf->lock); + } + put_page(page); + } +} + +void erofs_shrinker_register(struct super_block *sb) +{ + struct erofs_sb_info *sbi = EROFS_SB(sb); + + mutex_init(&sbi->umount_mutex); + + spin_lock(&erofs_sb_list_lock); + list_add(&sbi->list, &erofs_sb_list); + spin_unlock(&erofs_sb_list_lock); +} + +void erofs_shrinker_unregister(struct super_block *sb) +{ + struct erofs_sb_info *const sbi = EROFS_SB(sb); + + mutex_lock(&sbi->umount_mutex); + while (!xa_empty(&sbi->managed_pslots)) { + z_erofs_shrink_scan(sbi, ~0UL); + cond_resched(); + } + spin_lock(&erofs_sb_list_lock); + list_del(&sbi->list); + spin_unlock(&erofs_sb_list_lock); + mutex_unlock(&sbi->umount_mutex); +} + +static unsigned long erofs_shrink_count(struct shrinker *shrink, + struct shrink_control *sc) +{ + return atomic_long_read(&erofs_global_shrink_cnt) ?: SHRINK_EMPTY; +} + +static unsigned long erofs_shrink_scan(struct shrinker *shrink, + struct shrink_control *sc) +{ + struct erofs_sb_info *sbi; + struct list_head *p; + + unsigned long nr = sc->nr_to_scan; + unsigned int run_no; + unsigned long freed = 0; + + spin_lock(&erofs_sb_list_lock); + do { + run_no = ++shrinker_run_no; + } while (run_no == 0); + + /* Iterate over all mounted superblocks and try to shrink them */ + p = erofs_sb_list.next; + while (p != &erofs_sb_list) { + sbi = list_entry(p, struct erofs_sb_info, list); + + /* + * We move the ones we do to the end of the list, so we stop + * when we see one we have already done. + */ + if (sbi->shrinker_run_no == run_no) + break; + + if (!mutex_trylock(&sbi->umount_mutex)) { + p = p->next; + continue; + } + + spin_unlock(&erofs_sb_list_lock); + sbi->shrinker_run_no = run_no; + freed += z_erofs_shrink_scan(sbi, nr - freed); + spin_lock(&erofs_sb_list_lock); + /* Get the next list element before we move this one */ + p = p->next; + + /* + * Move this one to the end of the list to provide some + * fairness. + */ + list_move_tail(&sbi->list, &erofs_sb_list); + mutex_unlock(&sbi->umount_mutex); + + if (freed >= nr) + break; + } + spin_unlock(&erofs_sb_list_lock); + return freed; +} + +int __init erofs_init_shrinker(void) +{ + erofs_shrinker_info = shrinker_alloc(0, "erofs-shrinker"); + if (!erofs_shrinker_info) + return -ENOMEM; + + erofs_shrinker_info->count_objects = erofs_shrink_count; + erofs_shrinker_info->scan_objects = erofs_shrink_scan; + shrinker_register(erofs_shrinker_info); + return 0; +} + +void erofs_exit_shrinker(void) +{ + shrinker_free(erofs_shrinker_info); +} diff --git a/fs/eventfd.c b/fs/eventfd.c new file mode 100644 index 000000000..9d33a0275 --- /dev/null +++ b/fs/eventfd.c @@ -0,0 +1,423 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * fs/eventfd.c + * + * Copyright (C) 2007 Davide Libenzi + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static DEFINE_IDA(eventfd_ida); + +struct eventfd_ctx { + struct kref kref; + wait_queue_head_t wqh; + /* + * Every time that a write(2) is performed on an eventfd, the + * value of the __u64 being written is added to "count" and a + * wakeup is performed on "wqh". If EFD_SEMAPHORE flag was not + * specified, a read(2) will return the "count" value to userspace, + * and will reset "count" to zero. The kernel side eventfd_signal() + * also, adds to the "count" counter and issue a wakeup. + */ + __u64 count; + unsigned int flags; + int id; +}; + +/** + * eventfd_signal_mask - Increment the event counter + * @ctx: [in] Pointer to the eventfd context. + * @mask: [in] poll mask + * + * This function is supposed to be called by the kernel in paths that do not + * allow sleeping. In this function we allow the counter to reach the ULLONG_MAX + * value, and we signal this as overflow condition by returning a EPOLLERR + * to poll(2). + */ +void eventfd_signal_mask(struct eventfd_ctx *ctx, __poll_t mask) +{ + unsigned long flags; + + /* + * Deadlock or stack overflow issues can happen if we recurse here + * through waitqueue wakeup handlers. If the caller users potentially + * nested waitqueues with custom wakeup handlers, then it should + * check eventfd_signal_allowed() before calling this function. If + * it returns false, the eventfd_signal() call should be deferred to a + * safe context. + */ + if (WARN_ON_ONCE(current->in_eventfd)) + return; + + spin_lock_irqsave(&ctx->wqh.lock, flags); + current->in_eventfd = 1; + if (ctx->count < ULLONG_MAX) + ctx->count++; + if (waitqueue_active(&ctx->wqh)) + wake_up_locked_poll(&ctx->wqh, EPOLLIN | mask); + current->in_eventfd = 0; + spin_unlock_irqrestore(&ctx->wqh.lock, flags); +} +EXPORT_SYMBOL_GPL(eventfd_signal_mask); + +static void eventfd_free_ctx(struct eventfd_ctx *ctx) +{ + if (ctx->id >= 0) + ida_free(&eventfd_ida, ctx->id); + kfree(ctx); +} + +static void eventfd_free(struct kref *kref) +{ + struct eventfd_ctx *ctx = container_of(kref, struct eventfd_ctx, kref); + + eventfd_free_ctx(ctx); +} + +/** + * eventfd_ctx_put - Releases a reference to the internal eventfd context. + * @ctx: [in] Pointer to eventfd context. + * + * The eventfd context reference must have been previously acquired either + * with eventfd_ctx_fdget() or eventfd_ctx_fileget(). + */ +void eventfd_ctx_put(struct eventfd_ctx *ctx) +{ + kref_put(&ctx->kref, eventfd_free); +} +EXPORT_SYMBOL_GPL(eventfd_ctx_put); + +static int eventfd_release(struct inode *inode, struct file *file) +{ + struct eventfd_ctx *ctx = file->private_data; + + wake_up_poll(&ctx->wqh, EPOLLHUP); + eventfd_ctx_put(ctx); + return 0; +} + +static __poll_t eventfd_poll(struct file *file, poll_table *wait) +{ + struct eventfd_ctx *ctx = file->private_data; + __poll_t events = 0; + u64 count; + + poll_wait(file, &ctx->wqh, wait); + + /* + * All writes to ctx->count occur within ctx->wqh.lock. This read + * can be done outside ctx->wqh.lock because we know that poll_wait + * takes that lock (through add_wait_queue) if our caller will sleep. + * + * The read _can_ therefore seep into add_wait_queue's critical + * section, but cannot move above it! add_wait_queue's spin_lock acts + * as an acquire barrier and ensures that the read be ordered properly + * against the writes. The following CAN happen and is safe: + * + * poll write + * ----------------- ------------ + * lock ctx->wqh.lock (in poll_wait) + * count = ctx->count + * __add_wait_queue + * unlock ctx->wqh.lock + * lock ctx->qwh.lock + * ctx->count += n + * if (waitqueue_active) + * wake_up_locked_poll + * unlock ctx->qwh.lock + * eventfd_poll returns 0 + * + * but the following, which would miss a wakeup, cannot happen: + * + * poll write + * ----------------- ------------ + * count = ctx->count (INVALID!) + * lock ctx->qwh.lock + * ctx->count += n + * **waitqueue_active is false** + * **no wake_up_locked_poll!** + * unlock ctx->qwh.lock + * lock ctx->wqh.lock (in poll_wait) + * __add_wait_queue + * unlock ctx->wqh.lock + * eventfd_poll returns 0 + */ + count = READ_ONCE(ctx->count); + + if (count > 0) + events |= EPOLLIN; + if (count == ULLONG_MAX) + events |= EPOLLERR; + if (ULLONG_MAX - 1 > count) + events |= EPOLLOUT; + + return events; +} + +void eventfd_ctx_do_read(struct eventfd_ctx *ctx, __u64 *cnt) +{ + lockdep_assert_held(&ctx->wqh.lock); + + *cnt = ((ctx->flags & EFD_SEMAPHORE) && ctx->count) ? 1 : ctx->count; + ctx->count -= *cnt; +} +EXPORT_SYMBOL_GPL(eventfd_ctx_do_read); + +/** + * eventfd_ctx_remove_wait_queue - Read the current counter and removes wait queue. + * @ctx: [in] Pointer to eventfd context. + * @wait: [in] Wait queue to be removed. + * @cnt: [out] Pointer to the 64-bit counter value. + * + * Returns %0 if successful, or the following error codes: + * + * -EAGAIN : The operation would have blocked. + * + * This is used to atomically remove a wait queue entry from the eventfd wait + * queue head, and read/reset the counter value. + */ +int eventfd_ctx_remove_wait_queue(struct eventfd_ctx *ctx, wait_queue_entry_t *wait, + __u64 *cnt) +{ + unsigned long flags; + + spin_lock_irqsave(&ctx->wqh.lock, flags); + eventfd_ctx_do_read(ctx, cnt); + __remove_wait_queue(&ctx->wqh, wait); + if (*cnt != 0 && waitqueue_active(&ctx->wqh)) + wake_up_locked_poll(&ctx->wqh, EPOLLOUT); + spin_unlock_irqrestore(&ctx->wqh.lock, flags); + + return *cnt != 0 ? 0 : -EAGAIN; +} +EXPORT_SYMBOL_GPL(eventfd_ctx_remove_wait_queue); + +static ssize_t eventfd_read(struct kiocb *iocb, struct iov_iter *to) +{ + struct file *file = iocb->ki_filp; + struct eventfd_ctx *ctx = file->private_data; + __u64 ucnt = 0; + + if (iov_iter_count(to) < sizeof(ucnt)) + return -EINVAL; + spin_lock_irq(&ctx->wqh.lock); + if (!ctx->count) { + if ((file->f_flags & O_NONBLOCK) || + (iocb->ki_flags & IOCB_NOWAIT)) { + spin_unlock_irq(&ctx->wqh.lock); + return -EAGAIN; + } + + if (wait_event_interruptible_locked_irq(ctx->wqh, ctx->count)) { + spin_unlock_irq(&ctx->wqh.lock); + return -ERESTARTSYS; + } + } + eventfd_ctx_do_read(ctx, &ucnt); + current->in_eventfd = 1; + if (waitqueue_active(&ctx->wqh)) + wake_up_locked_poll(&ctx->wqh, EPOLLOUT); + current->in_eventfd = 0; + spin_unlock_irq(&ctx->wqh.lock); + if (unlikely(copy_to_iter(&ucnt, sizeof(ucnt), to) != sizeof(ucnt))) + return -EFAULT; + + return sizeof(ucnt); +} + +static ssize_t eventfd_write(struct file *file, const char __user *buf, size_t count, + loff_t *ppos) +{ + struct eventfd_ctx *ctx = file->private_data; + ssize_t res; + __u64 ucnt; + + if (count != sizeof(ucnt)) + return -EINVAL; + if (copy_from_user(&ucnt, buf, sizeof(ucnt))) + return -EFAULT; + if (ucnt == ULLONG_MAX) + return -EINVAL; + spin_lock_irq(&ctx->wqh.lock); + res = -EAGAIN; + if (ULLONG_MAX - ctx->count > ucnt) + res = sizeof(ucnt); + else if (!(file->f_flags & O_NONBLOCK)) { + res = wait_event_interruptible_locked_irq(ctx->wqh, + ULLONG_MAX - ctx->count > ucnt); + if (!res) + res = sizeof(ucnt); + } + if (likely(res > 0)) { + ctx->count += ucnt; + current->in_eventfd = 1; + if (waitqueue_active(&ctx->wqh)) + wake_up_locked_poll(&ctx->wqh, EPOLLIN); + current->in_eventfd = 0; + } + spin_unlock_irq(&ctx->wqh.lock); + + return res; +} + +#ifdef CONFIG_PROC_FS +static void eventfd_show_fdinfo(struct seq_file *m, struct file *f) +{ + struct eventfd_ctx *ctx = f->private_data; + __u64 cnt; + + spin_lock_irq(&ctx->wqh.lock); + cnt = ctx->count; + spin_unlock_irq(&ctx->wqh.lock); + + seq_printf(m, + "eventfd-count: %16llx\n" + "eventfd-id: %d\n" + "eventfd-semaphore: %d\n", + cnt, + ctx->id, + !!(ctx->flags & EFD_SEMAPHORE)); +} +#endif + +static const struct file_operations eventfd_fops = { +#ifdef CONFIG_PROC_FS + .show_fdinfo = eventfd_show_fdinfo, +#endif + .release = eventfd_release, + .poll = eventfd_poll, + .read_iter = eventfd_read, + .write = eventfd_write, + .llseek = noop_llseek, +}; + +/** + * eventfd_fget - Acquire a reference of an eventfd file descriptor. + * @fd: [in] Eventfd file descriptor. + * + * Returns a pointer to the eventfd file structure in case of success, or the + * following error pointer: + * + * -EBADF : Invalid @fd file descriptor. + * -EINVAL : The @fd file descriptor is not an eventfd file. + */ +struct file *eventfd_fget(int fd) +{ + struct file *file; + + file = fget(fd); + if (!file) + return ERR_PTR(-EBADF); + if (file->f_op != &eventfd_fops) { + fput(file); + return ERR_PTR(-EINVAL); + } + + return file; +} +EXPORT_SYMBOL_GPL(eventfd_fget); + +/** + * eventfd_ctx_fdget - Acquires a reference to the internal eventfd context. + * @fd: [in] Eventfd file descriptor. + * + * Returns a pointer to the internal eventfd context, otherwise the error + * pointers returned by the following functions: + * + * eventfd_fget + */ +struct eventfd_ctx *eventfd_ctx_fdget(int fd) +{ + CLASS(fd, f)(fd); + if (fd_empty(f)) + return ERR_PTR(-EBADF); + return eventfd_ctx_fileget(fd_file(f)); +} +EXPORT_SYMBOL_GPL(eventfd_ctx_fdget); + +/** + * eventfd_ctx_fileget - Acquires a reference to the internal eventfd context. + * @file: [in] Eventfd file pointer. + * + * Returns a pointer to the internal eventfd context, otherwise the error + * pointer: + * + * -EINVAL : The @fd file descriptor is not an eventfd file. + */ +struct eventfd_ctx *eventfd_ctx_fileget(struct file *file) +{ + struct eventfd_ctx *ctx; + + if (file->f_op != &eventfd_fops) + return ERR_PTR(-EINVAL); + + ctx = file->private_data; + kref_get(&ctx->kref); + return ctx; +} +EXPORT_SYMBOL_GPL(eventfd_ctx_fileget); + +static int do_eventfd(unsigned int count, int flags) +{ + struct eventfd_ctx *ctx __free(kfree) = NULL; + + /* Check the EFD_* constants for consistency. */ + BUILD_BUG_ON(EFD_CLOEXEC != O_CLOEXEC); + BUILD_BUG_ON(EFD_NONBLOCK != O_NONBLOCK); + BUILD_BUG_ON(EFD_SEMAPHORE != (1 << 0)); + + if (flags & ~EFD_FLAGS_SET) + return -EINVAL; + + ctx = kmalloc_obj(*ctx); + if (!ctx) + return -ENOMEM; + + kref_init(&ctx->kref); + init_waitqueue_head(&ctx->wqh); + ctx->count = count; + ctx->flags = flags; + + flags &= EFD_SHARED_FCNTL_FLAGS; + flags |= O_RDWR; + + FD_PREPARE(fdf, flags, + anon_inode_getfile_fmode("[eventfd]", &eventfd_fops, ctx, + flags, FMODE_NOWAIT)); + if (fdf.err) + return fdf.err; + + ctx->id = ida_alloc(&eventfd_ida, GFP_KERNEL); + retain_and_null_ptr(ctx); + return fd_publish(fdf); +} + +SYSCALL_DEFINE2(eventfd2, unsigned int, count, int, flags) +{ + return do_eventfd(count, flags); +} + +SYSCALL_DEFINE1(eventfd, unsigned int, count) +{ + return do_eventfd(count, 0); +} + diff --git a/fs/eventpoll.c b/fs/eventpoll.c new file mode 100644 index 000000000..e0c4bf88a --- /dev/null +++ b/fs/eventpoll.c @@ -0,0 +1,3030 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * fs/eventpoll.c (Efficient event retrieval implementation) + * Copyright (C) 2001,...,2009 Davide Libenzi + * + * Davide Libenzi + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * fs/eventpoll.c - Efficient event polling ("epoll") kernel implementation. + * + * + * Overview + * -------- + * + * Each epoll_create(2) returns an anonymous [eventpoll] file whose + * ->private_data is a struct eventpoll. Each EPOLL_CTL_ADD installs + * a struct epitem linking one (watched file, fd) pair back to that + * eventpoll via the watched file's f_op->poll() wait queue(s). When + * the watched file signals readiness, ep_poll_callback() fires and + * marks the epitem ready. epoll_wait(2) drains the ready list under + * ep->mtx, re-queueing items in level-triggered mode. + * + * epoll instances can watch other epoll instances up to EP_MAX_NESTS + * deep; cycles are forbidden and detected at EPOLL_CTL_ADD time. + * + * + * Locking + * ------- + * + * Three levels, acquired from outer to inner: + * + * epnested_mutex (global; rare; taken only for EPOLL_CTL_ADD + * loop / path checks) + * > ep->mtx (per-eventpoll; sleepable; serializes most ops) + * > ep->lock (per-eventpoll; IRQ-safe spinlock) + * + * file->f_lock (per-file; NOT IRQ-safe; guards f_ep hlist ops; + * nested inside ep->mtx, outside ep->lock) + * + * Rationale: + * - ep->lock is a spinlock because ep_poll_callback() is called from + * wake_up() which may run in hard-IRQ context. All ep->lock + * critical sections use spin_lock_irqsave(). + * - ep->mtx is a sleepable mutex because the event delivery loop + * calls copy_to_user(), and ep_insert() may sleep in + * kmem_cache_alloc() and f_op->poll(). + * - epnested_mutex is global because cycle detection needs a global + * view of the epoll topology; a per-object scheme would let two + * concurrent inserts (A into B, B into A) construct a cycle + * without either observer seeing it. + * - Per-ep ep->mtx is preferred for scalability elsewhere. Events + * that require epnested_mutex are rare. + * + * When EPOLL_CTL_ADD nests one eventpoll inside another we acquire + * ep->mtx on both: outer first, target second. Since cycles are + * forbidden the set of live ep->mtx holds is always a strict chain, + * communicated to lockdep via mutex_lock_nested() subclasses derived + * from the current recursion depth. + * + * + * Field protection + * ---------------- + * + * struct eventpoll: + * mtx - self + * rbr - ep->mtx + * ovflist, rdllist - ep->lock (IRQ-safe) + * wq - ep->lock for queue mutation + * poll_wait - internal waitqueue spinlock + * refs - file->f_lock for adds; ep->mtx for removes; + * RCU for readers (hlist_del_rcu + kfree_rcu(ep)) + * ws - ep->mtx + * gen, loop_check_depth - epnested_mutex + * file, user - immutable after setup + * refcount - atomic (refcount_t) + * napi_* - READ_ONCE / WRITE_ONCE + * + * struct epitem: + * rbn / rcu union - rbn: ep->mtx (while epi is linked in ep->rbr). + * rcu: written only by kfree_rcu(epi) on the free + * path; otherwise untouched by epoll code. + * rdllink, next - ep->lock + * ffd, ep - immutable after ep_insert() + * pwqlist - ep->mtx for writes; POLLFREE clears pwq->whead + * via smp_store_release(), see below + * fllink - file->f_lock for mutation; hlist_del_rcu + + * kfree_rcu(epi) for safe RCU readers + * ws - RCU (rcu_assign_pointer / + * rcu_dereference_check(mtx)) + * event - ep->mtx for writes; lockless read in + * ep_poll_callback pairs with smp_mb() in + * ep_modify() + * + * + * Ready-list state machine + * ------------------------ + * + * Readiness is tracked in two lists under ep->lock: + * + * rdllist - doubly-linked FIFO; the "current" ready list. + * ovflist - singly-linked LIFO; used during a scan to catch + * events that arrive while rdllist is being iterated + * without ep->lock. + * + * Encoded in ep->ovflist: + * EP_UNACTIVE_PTR - no scan active; callback appends to rdllist. + * NULL - scan active, no spill yet. + * pointer to epi - scan active with spilled items (LIFO). + * + * Encoded in epi->ovflist_next: + * EP_UNACTIVE_PTR - epi is not on ovflist. + * otherwise - next epi on ovflist (NULL at tail). + * + * ep_start_scan() flips "not scanning" to "scanning" and splices + * rdllist into a caller-local scan_batch. ep_done_scan() drains ovflist + * back to rdllist (list_add head-insert reverses LIFO to FIFO), + * flips back to "not scanning", and re-splices any items the caller + * left in scan_batch (e.g., level-triggered re-queues). + * + * + * Removal paths + * ------------- + * + * Three paths dispose of epitems and/or eventpolls: + * + * A. ep_remove() - EPOLL_CTL_DEL and ep_insert() + * rollback. Caller holds ep->mtx. + * B. ep_clear_and_put() - close of the epoll fd itself + * (ep_eventpoll_release). + * C. eventpoll_release_file() - close of a watched file, invoked + * from __fput(). + * + * Coordination: + * A and C exclude each other via the watched file's refcount. + * A pins the file with epi_fget() before touching file->f_ep or + * file->f_lock; if the pin fails, __fput() is in flight and C + * will clean this epi up. See the epi_fget() block comment. + * A and B both hold ep->mtx serially. B walks the rbtree with + * rb_next() captured before ep_remove() erases the current node. + * B and C both take ep->mtx; the loser sees fewer entries or an + * empty file->f_ep. + * + * Within every path the internal order is strict: + * ep_unregister_pollwait() - drain pwqlist; synchronizes with any + * in-flight ep_poll_callback via the + * watched wait-queue head's lock. + * ep_remove_file() - hlist_del_rcu of epi->fllink and, + * if last watcher, clear file->f_ep, + * under file->f_lock. + * ep_remove_epi() - rb_erase, rdllist unlink (ep->lock), + * wakeup_source_unregister, + * kfree_rcu(epi). + * + * kfree_rcu(epi) defers the free past RCU readers in + * reverse_path_check_proc(); kfree_rcu(ep) defers past readers in + * ep_get_upwards_depth_proc(). + * + * + * POLLFREE handshake + * ------------------ + * + * When a subsystem tears down a wait-queue head that an epitem is + * registered on (binder, signalfd, ...), it wakes the callback with + * POLLFREE and must RCU-defer the head's free. The store/load pair: + * + * ep_poll_callback() POLLFREE branch: + * smp_store_release(&pwq->whead, NULL) + * + * ep_remove_wait_queue(): + * smp_load_acquire(&pwq->whead) + * + * See those sites for the full argument. + */ + +/* Epoll private bits inside the event mask */ +#define EP_PRIVATE_BITS (EPOLLWAKEUP | EPOLLONESHOT | EPOLLET | EPOLLEXCLUSIVE) + +#define EPOLLINOUT_BITS (EPOLLIN | EPOLLOUT) + +#define EPOLLEXCLUSIVE_OK_BITS (EPOLLINOUT_BITS | EPOLLERR | EPOLLHUP | \ + EPOLLWAKEUP | EPOLLET | EPOLLEXCLUSIVE) + +/* Maximum number of nesting allowed inside epoll sets */ +#define EP_MAX_NESTS 4 + +#define EP_MAX_EVENTS (INT_MAX / sizeof(struct epoll_event)) + +#define EP_UNACTIVE_PTR ((void *) -1L) + +#define EP_ITEM_COST (sizeof(struct epitem) + sizeof(struct eppoll_entry)) + +/* Wait structure used by the poll hooks */ +struct eppoll_entry { + /* List header used to link this structure to the "struct epitem" */ + struct eppoll_entry *next; + + /* The "base" pointer is set to the container "struct epitem" */ + struct epitem *base; + + /* + * Wait queue item that will be linked to the target file wait + * queue head. + */ + wait_queue_entry_t wait; + + /* The wait queue head that linked the "wait" wait queue item */ + wait_queue_head_t *whead; +}; + +/* + * Each file descriptor added to the eventpoll interface will + * have an entry of this type linked to the "rbr" RB tree. + * Avoid increasing the size of this struct, there can be many thousands + * of these on a server and we do not want this to take another cache line. + */ +struct epitem { + union { + /* RB tree node links this structure to the eventpoll RB tree */ + struct rb_node rbn; + /* Used to free the struct epitem */ + struct rcu_head rcu; + }; + + /* Link on the owning eventpoll's ready list (ep->rdllist). */ + struct list_head rdllink; + + /* + * Link on the owning eventpoll's scan-overflow list (ep->ovflist), + * EP_UNACTIVE_PTR when not linked. See epi_on_ovflist() / + * epi_clear_ovflist() and the "Ready-list state machine" section + * in the top-of-file banner. + */ + struct epitem *ovflist_next; + + /* The file descriptor information this item refers to */ + struct epoll_key ffd; + + /* List containing poll wait queues */ + struct eppoll_entry *pwqlist; + + /* The "container" of this item */ + struct eventpoll *ep; + + /* List header used to link this item to the "struct file" items list */ + struct hlist_node fllink; + + /* wakeup_source used when EPOLLWAKEUP is set */ + struct wakeup_source __rcu *ws; + + /* The structure that describe the interested events and the source fd */ + struct epoll_event event; +}; + +/* + * This structure is stored inside the "private_data" member of the file + * structure and represents the main data structure for the eventpoll + * interface. + */ +struct eventpoll { + /* + * This mutex is used to ensure that files are not removed + * while epoll is using them. This is held during the event + * collection loop, the file cleanup path, the epoll file exit + * code and the ctl operations. + */ + struct mutex mtx; + + /* Wait queue used by sys_epoll_wait() */ + wait_queue_head_t wq; + + /* Wait queue used by file->poll() */ + wait_queue_head_t poll_wait; + + /* List of ready file descriptors */ + struct list_head rdllist; + + /* Lock which protects rdllist and ovflist */ + spinlock_t lock; + + /* Protect switching between rdllist and ovflist */ + seqcount_spinlock_t seq; + + /* RB tree root used to store monitored fd structs */ + struct rb_root_cached rbr; + + /* + * This is a single linked list that chains all the "struct epitem" that + * happened while transferring ready events to userspace w/out + * holding ->lock. + */ + struct epitem *ovflist; + + /* wakeup_source used when ep_send_events or __ep_eventpoll_poll is running */ + struct wakeup_source *ws; + + /* The user that created the eventpoll descriptor */ + struct user_struct *user; + + struct file *file; + + /* used to optimize loop detection check */ + u64 gen; + struct hlist_head refs; + u8 loop_check_depth; + + /* usage count, orchestrates "struct eventpoll" disposal */ + refcount_t refcount; + + /* used to defer freeing past ep_get_upwards_depth_proc() RCU walk */ + struct rcu_head rcu; + +#ifdef CONFIG_NET_RX_BUSY_POLL + /* used to track busy poll napi_id */ + unsigned int napi_id; + /* busy poll timeout */ + u32 busy_poll_usecs; + /* busy poll packet budget */ + u16 busy_poll_budget; + bool prefer_busy_poll; +#endif + +#ifdef CONFIG_DEBUG_LOCK_ALLOC + /* tracks wakeup nests for lockdep validation */ + u8 nests; +#endif +}; + +/* Wrapper struct used by poll queueing */ +struct ep_pqueue { + poll_table pt; + struct epitem *epi; +}; + +/* + * Configuration options available inside /proc/sys/fs/epoll/ + */ +/* Maximum number of epoll watched descriptors, per user */ +static long max_user_watches __read_mostly; + +/* + * Cycle and path-length checks at EPOLL_CTL_ADD + * --------------------------------------------- + * + * When EPOLL_CTL_ADD creates a link that either targets an eventpoll + * file or extends an existing chain of eventpolls, two checks run: + * + * 1. no cycle is being formed -- ep_loop_check() walks downward + * from the candidate target, and ep_get_upwards_depth_proc() + * walks upward from the outer ep, both bounded by EP_MAX_NESTS. + * 2. no file accumulates more than path_limits[depth] wakeup paths + * of a given length -- reverse_path_check(). + * + * Both need a global view of the epoll topology and must be atomic + * with the insertion, so the check is serialized by epnested_mutex + * and carries its scratch state on a stack-allocated struct + * ep_ctl_ctx scoped to one do_epoll_ctl() call. Non-nested inserts + * skip this machinery entirely and take only ep->mtx. + * + * epnested_mutex Serializes the whole check. + * loop_check_gen Global monotonic stamp, bumped at the start of + * a check and again at the end. ep->gen caches + * the value under which ep was last visited by + * ep_loop_check_proc() or + * ep_get_upwards_depth_proc(); the post-check + * bump ensures those cached stamps can no longer + * equal loop_check_gen, so the + * "ep->gen == loop_check_gen" trigger in + * ep_ctl_lock() only fires while another check + * is in flight. + * + * struct ep_ctl_ctx carries the rest (inserting_into, tfile_check_list, + * path_count[]) through the walk; see its declaration below. + * + * Commits fdcfce93073d ("eventpoll: Fix integer overflow in + * ep_loop_check_proc()") and f2e467a48287 ("eventpoll: Fix + * semi-unbounded recursion") hardened the walk; any refactor must + * preserve both bail-outs. + */ +static DEFINE_MUTEX(epnested_mutex); +static u64 loop_check_gen = 0; + +#define PATH_ARR_SIZE 5 + +/* + * Per-do_epoll_ctl() scratch for the loop / path checks. Allocated on + * the caller's stack; populated by ep_ctl_lock() and the downward + * walk; consumed by reverse_path_check(); released by ep_ctl_unlock(). + * Only valid while the caller holds epnested_mutex. + */ +struct ep_ctl_ctx { + /* + * Outer eventpoll for one ep_loop_check(); if the downward walk + * reaches it the insert would form a cycle. + */ + struct eventpoll *inserting_into; + + /* + * Singly-linked list of epitems_head objects collected during + * ep_loop_check_proc(), then walked by reverse_path_check(). + * Terminated by EP_UNACTIVE_PTR, not NULL: epitems_head->next + * doubles as a membership flag (a NULL ->next means "not on this + * list", see ep_remove_file()), so the list uses a non-NULL + * sentinel to keep the tail head distinguishable from an unlisted + * one. + */ + struct epitems_head *tfile_check_list; + + /* + * Per-depth wakeup-path tally used by reverse_path_check_proc(); + * reinitialized to zero at the start of each reverse_path_check() + * iteration. + */ + int path_count[PATH_ARR_SIZE]; +}; + +/* Slab cache used to allocate "struct epitem" */ +static struct kmem_cache *epi_cache __ro_after_init; + +/* Slab cache used to allocate "struct eppoll_entry" */ +static struct kmem_cache *pwq_cache __ro_after_init; + +/* + * Wrapper anchor for file->f_ep when the watched file is not itself an + * eventpoll; for the epoll-watches-epoll case, file->f_ep points at + * &watched_ep->refs directly. The ->next field threads + * ctx->tfile_check_list during one EPOLL_CTL_ADD path check. The ->file + * field holds a reference to the associated file while the head is on + * the list. + */ +struct epitems_head { + struct hlist_head epitems; + struct epitems_head *next; + struct file *file; +}; + +static struct kmem_cache *ephead_cache __ro_after_init; + +static inline void free_ephead(struct epitems_head *head) +{ + if (head) + kmem_cache_free(ephead_cache, head); +} + +static void list_file(struct file *file, struct ep_ctl_ctx *ctx) +{ + struct epitems_head *head; + + head = container_of(file->f_ep, struct epitems_head, epitems); + if (!head->next) { + /* + * The caller owns a reference to @file or holds the ep->mtx for the + * epitem that led here. The latter blocks eventpoll_release_file() + * before the file allocation can be freed and reused. A dying leaf + * can be skipped since removing links cannot increase the reverse + * path count. + */ + if (!file_ref_get(&file->f_ref)) + return; + head->file = file; + head->next = ctx->tfile_check_list; + ctx->tfile_check_list = head; + } +} + +static void unlist_file(struct epitems_head *head) +{ + struct epitems_head *to_free = head; + struct hlist_node *p = rcu_dereference(hlist_first_rcu(&head->epitems)); + struct file *file = head->file; + if (p) { + struct epitem *epi= container_of(p, struct epitem, fllink); + spin_lock(&epi->ffd.file->f_lock); + if (!hlist_empty(&head->epitems)) + to_free = NULL; + head->next = NULL; + head->file = NULL; + spin_unlock(&epi->ffd.file->f_lock); + } + free_ephead(to_free); + fput(file); +} + +#ifdef CONFIG_SYSCTL + +#include + +static long long_zero; +static long long_max = LONG_MAX; + +static const struct ctl_table epoll_table[] = { + { + .procname = "max_user_watches", + .data = &max_user_watches, + .maxlen = sizeof(max_user_watches), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + .extra1 = &long_zero, + .extra2 = &long_max, + }, +}; + +static void __init epoll_sysctls_init(void) +{ + register_sysctl("fs/epoll", epoll_table); +} +#else +#define epoll_sysctls_init() do { } while (0) +#endif /* CONFIG_SYSCTL */ + +static const struct file_operations eventpoll_fops; + +bool is_file_epoll(struct file *f) +{ + return f->f_op == &eventpoll_fops; +} + +/* Compare RB tree keys */ +static inline int ep_cmp_ffd(struct epoll_key *p1, struct epoll_key *p2) +{ + return (p1->file > p2->file ? +1: + (p1->file < p2->file ? -1 : p1->fd - p2->fd)); +} + +/* True iff @epi is on its owning ep's ready list. */ +static inline bool ep_is_linked(struct epitem *epi) +{ + return !list_empty(&epi->rdllink); +} + +static inline struct eppoll_entry *ep_pwq_from_wait(wait_queue_entry_t *p) +{ + return container_of(p, struct eppoll_entry, wait); +} + +/* Get the "struct epitem" from a wait queue pointer */ +static inline struct epitem *ep_item_from_wait(wait_queue_entry_t *p) +{ + return container_of(p, struct eppoll_entry, wait)->base; +} + +/* + * Ready-list / ovflist state (see "Ready-list state machine" in the + * top-of-file banner for the full state machine). EP_UNACTIVE_PTR is + * the sentinel; these wrappers name each transition and each test so + * call sites do not need to know the sentinel's value. + */ + +/* True iff @ep is between ep_enter_scan() and ep_exit_scan(). */ +static inline bool ep_is_scanning(struct eventpoll *ep) +{ + return READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR; +} + +/* Called by ep_start_scan(): divert ep_poll_callback() to ovflist. */ +static inline void ep_enter_scan(struct eventpoll *ep) +{ + WRITE_ONCE(ep->ovflist, NULL); +} + +/* Called by ep_done_scan(): redirect ep_poll_callback() back to rdllist. */ +static inline void ep_exit_scan(struct eventpoll *ep) +{ + WRITE_ONCE(ep->ovflist, EP_UNACTIVE_PTR); +} + +/* True iff @epi is currently linked on its ep's ovflist. */ +static inline bool epi_on_ovflist(const struct epitem *epi) +{ + return epi->ovflist_next != EP_UNACTIVE_PTR; +} + +/* Mark @epi as not on any ovflist (init and post-drain). */ +static inline void epi_clear_ovflist(struct epitem *epi) +{ + epi->ovflist_next = EP_UNACTIVE_PTR; +} + +/* True iff @ep has ready events that epoll_wait() might harvest. */ +static inline bool ep_events_available(struct eventpoll *ep) +{ + unsigned int seq = read_seqcount_begin(&ep->seq); + + return !list_empty_careful(&ep->rdllist) || ep_is_scanning(ep) || + read_seqcount_retry(&ep->seq, seq); +} + +#ifdef CONFIG_NET_RX_BUSY_POLL +/** + * busy_loop_ep_timeout - check if busy poll has timed out. The timeout value + * from the epoll instance ep is preferred, but if it is not set fallback to + * the system-wide global via busy_loop_timeout. + * + * @start_time: The start time used to compute the remaining time until timeout. + * @ep: Pointer to the eventpoll context. + * + * Return: true if the timeout has expired, false otherwise. + */ +static bool busy_loop_ep_timeout(unsigned long start_time, + struct eventpoll *ep) +{ + unsigned long bp_usec = READ_ONCE(ep->busy_poll_usecs); + + if (bp_usec) { + unsigned long end_time = start_time + bp_usec; + unsigned long now = busy_loop_current_time(); + + return time_after(now, end_time); + } else { + return busy_loop_timeout(start_time); + } +} + +static bool ep_busy_loop_on(struct eventpoll *ep) +{ + return !!READ_ONCE(ep->busy_poll_usecs) || + READ_ONCE(ep->prefer_busy_poll) || + net_busy_loop_on(); +} + +static bool ep_busy_loop_end(void *p, unsigned long start_time) +{ + struct eventpoll *ep = p; + + return ep_events_available(ep) || busy_loop_ep_timeout(start_time, ep); +} + +/* + * Busy poll if globally on and supporting sockets found && no events, + * busy loop will return if need_resched or ep_events_available. + * + * we must do our busy polling with irqs enabled + */ +static bool ep_busy_loop(struct eventpoll *ep) +{ + unsigned int napi_id = READ_ONCE(ep->napi_id); + u16 budget = READ_ONCE(ep->busy_poll_budget); + bool prefer_busy_poll = READ_ONCE(ep->prefer_busy_poll); + + if (!budget) + budget = BUSY_POLL_BUDGET; + + if (napi_id_valid(napi_id) && ep_busy_loop_on(ep)) { + napi_busy_loop(napi_id, ep_busy_loop_end, + ep, prefer_busy_poll, budget); + if (ep_events_available(ep)) + return true; + /* + * Busy poll timed out. Drop NAPI ID for now, we can add + * it back in when we have moved a socket with a valid NAPI + * ID onto the ready list. + */ + if (prefer_busy_poll) + napi_resume_irqs(napi_id); + ep->napi_id = 0; + return false; + } + return false; +} + +/* + * Set epoll busy poll NAPI ID from sk. + */ +static inline void ep_set_busy_poll_napi_id(struct epitem *epi) +{ + struct eventpoll *ep = epi->ep; + unsigned int napi_id; + struct socket *sock; + struct sock *sk; + + if (!ep_busy_loop_on(ep)) + return; + + sock = sock_from_file(epi->ffd.file); + if (!sock) + return; + + sk = sock->sk; + if (!sk) + return; + + napi_id = READ_ONCE(sk->sk_napi_id); + + /* Non-NAPI IDs can be rejected + * or + * Nothing to do if we already have this ID + */ + if (!napi_id_valid(napi_id) || napi_id == ep->napi_id) + return; + + /* record NAPI ID for use in next busy poll */ + ep->napi_id = napi_id; +} + +static long ep_eventpoll_bp_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct eventpoll *ep = file->private_data; + void __user *uarg = (void __user *)arg; + struct epoll_params epoll_params; + + switch (cmd) { + case EPIOCSPARAMS: + if (copy_from_user(&epoll_params, uarg, sizeof(epoll_params))) + return -EFAULT; + + /* pad byte must be zero */ + if (epoll_params.__pad) + return -EINVAL; + + if (epoll_params.busy_poll_usecs > S32_MAX) + return -EINVAL; + + if (epoll_params.prefer_busy_poll > 1) + return -EINVAL; + + if (epoll_params.busy_poll_budget > NAPI_POLL_WEIGHT && + !capable(CAP_NET_ADMIN)) + return -EPERM; + + WRITE_ONCE(ep->busy_poll_usecs, epoll_params.busy_poll_usecs); + WRITE_ONCE(ep->busy_poll_budget, epoll_params.busy_poll_budget); + WRITE_ONCE(ep->prefer_busy_poll, epoll_params.prefer_busy_poll); + return 0; + case EPIOCGPARAMS: + memset(&epoll_params, 0, sizeof(epoll_params)); + epoll_params.busy_poll_usecs = READ_ONCE(ep->busy_poll_usecs); + epoll_params.busy_poll_budget = READ_ONCE(ep->busy_poll_budget); + epoll_params.prefer_busy_poll = READ_ONCE(ep->prefer_busy_poll); + if (copy_to_user(uarg, &epoll_params, sizeof(epoll_params))) + return -EFAULT; + return 0; + default: + return -ENOIOCTLCMD; + } +} + +static void ep_suspend_napi_irqs(struct eventpoll *ep) +{ + unsigned int napi_id = READ_ONCE(ep->napi_id); + + if (napi_id_valid(napi_id) && READ_ONCE(ep->prefer_busy_poll)) + napi_suspend_irqs(napi_id); +} + +static void ep_resume_napi_irqs(struct eventpoll *ep) +{ + unsigned int napi_id = READ_ONCE(ep->napi_id); + + if (napi_id_valid(napi_id) && READ_ONCE(ep->prefer_busy_poll)) + napi_resume_irqs(napi_id); +} + +#else + +static inline bool ep_busy_loop(struct eventpoll *ep) +{ + return false; +} + +static inline void ep_set_busy_poll_napi_id(struct epitem *epi) +{ +} + +static long ep_eventpoll_bp_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + return -EOPNOTSUPP; +} + +static void ep_suspend_napi_irqs(struct eventpoll *ep) +{ +} + +static void ep_resume_napi_irqs(struct eventpoll *ep) +{ +} + +#endif /* CONFIG_NET_RX_BUSY_POLL */ + +/* + * As described in commit 0ccf831cb lockdep: annotate epoll + * the use of wait queues used by epoll is done in a very controlled + * manner. Wake ups can nest inside each other, but are never done + * with the same locking. For example: + * + * dfd = socket(...); + * efd1 = epoll_create(); + * efd2 = epoll_create(); + * epoll_ctl(efd1, EPOLL_CTL_ADD, dfd, ...); + * epoll_ctl(efd2, EPOLL_CTL_ADD, efd1, ...); + * + * When a packet arrives to the device underneath "dfd", the net code will + * issue a wake_up() on its poll wake list. Epoll (efd1) has installed a + * callback wakeup entry on that queue, and the wake_up() performed by the + * "dfd" net code will end up in ep_poll_callback(). At this point epoll + * (efd1) notices that it may have some event ready, so it needs to wake up + * the waiters on its poll wait list (efd2). So it calls ep_poll_safewake() + * that ends up in another wake_up(), after having checked about the + * recursion constraints. That are, no more than EP_MAX_NESTS, to avoid + * stack blasting. + * + * When CONFIG_DEBUG_LOCK_ALLOC is enabled, make sure lockdep can handle + * this special case of epoll. + */ +#ifdef CONFIG_DEBUG_LOCK_ALLOC + +static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi, + unsigned pollflags) +{ + struct eventpoll *ep_src; + unsigned long flags; + u8 nests = 0; + + /* + * To set the subclass or nesting level for spin_lock_irqsave_nested() + * it might be natural to create a per-cpu nest count. However, since + * we can recurse on ep->poll_wait.lock, and a non-raw spinlock can + * schedule() in the -rt kernel, the per-cpu variable are no longer + * protected. Thus, we are introducing a per eventpoll nest field. + * If we are not being call from ep_poll_callback(), epi is NULL and + * we are at the first level of nesting, 0. Otherwise, we are being + * called from ep_poll_callback() and if a previous wakeup source is + * not an epoll file itself, we are at depth 1 since the wakeup source + * is depth 0. If the wakeup source is a previous epoll file in the + * wakeup chain then we use its nests value and record ours as + * nests + 1. The previous epoll file nests value is stable since its + * already holding its own poll_wait.lock. + */ + if (epi) { + if ((is_file_epoll(epi->ffd.file))) { + ep_src = epi->ffd.file->private_data; + nests = ep_src->nests; + } else { + nests = 1; + } + } + spin_lock_irqsave_nested(&ep->poll_wait.lock, flags, nests); + ep->nests = nests + 1; + wake_up_locked_poll(&ep->poll_wait, EPOLLIN | pollflags); + ep->nests = 0; + spin_unlock_irqrestore(&ep->poll_wait.lock, flags); +} + +#else + +static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi, + __poll_t pollflags) +{ + wake_up_poll(&ep->poll_wait, EPOLLIN | pollflags); +} + +#endif + +static void ep_remove_wait_queue(struct eppoll_entry *pwq) +{ + wait_queue_head_t *whead; + + rcu_read_lock(); + /* + * POLLFREE handshake, acquire side; see "POLLFREE handshake" + * at the top of this file. + * + * A NULL load is paired with the smp_store_release(&whead, NULL) + * in ep_poll_callback()'s POLLFREE branch: the teardown is + * complete and we must not touch whead again. On a non-NULL load + * rcu_read_lock() keeps the waitqueue memory alive (POLLFREE + * firers RCU-defer the free) and whead->lock inside + * remove_wait_queue() serializes us against the store side. + */ + whead = smp_load_acquire(&pwq->whead); + if (whead) + remove_wait_queue(whead, &pwq->wait); + rcu_read_unlock(); +} + +/* + * This function unregisters poll callbacks from the associated file + * descriptor. Must be called with "mtx" held. + */ +static void ep_unregister_pollwait(struct eventpoll *ep, struct epitem *epi) +{ + struct eppoll_entry **p = &epi->pwqlist; + struct eppoll_entry *pwq; + + while ((pwq = *p) != NULL) { + *p = pwq->next; + ep_remove_wait_queue(pwq); + kmem_cache_free(pwq_cache, pwq); + } +} + +/* call only when ep->mtx is held */ +static inline struct wakeup_source *ep_wakeup_source(struct epitem *epi) +{ + return rcu_dereference_check(epi->ws, lockdep_is_held(&epi->ep->mtx)); +} + +/* call only when ep->mtx is held */ +static inline void ep_pm_stay_awake(struct epitem *epi) +{ + struct wakeup_source *ws = ep_wakeup_source(epi); + + if (ws) + __pm_stay_awake(ws); +} + +static inline bool ep_has_wakeup_source(struct epitem *epi) +{ + return rcu_access_pointer(epi->ws) ? true : false; +} + +/* call when ep->mtx cannot be held (ep_poll_callback) */ +static inline void ep_pm_stay_awake_rcu(struct epitem *epi) +{ + struct wakeup_source *ws; + + rcu_read_lock(); + ws = rcu_dereference(epi->ws); + if (ws) + __pm_stay_awake(ws); + rcu_read_unlock(); +} + + +/* + * ep->mutex needs to be held because we could be hit by + * eventpoll_release_file() and epoll_ctl(). + */ +static void ep_start_scan(struct eventpoll *ep, struct list_head *scan_batch) +{ + /* + * Steal the ready list, and re-init the original one to the + * empty list. Also, set ep->ovflist to NULL so that events + * happening while looping w/out locks, are not lost. We cannot + * have the poll callback to queue directly on ep->rdllist, + * because we want the "sproc" callback to be able to do it + * in a lockless way. + */ + lockdep_assert_irqs_enabled(); + spin_lock_irq(&ep->lock); + write_seqcount_begin(&ep->seq); + + list_splice_init(&ep->rdllist, scan_batch); + ep_enter_scan(ep); + + write_seqcount_end(&ep->seq); + spin_unlock_irq(&ep->lock); +} + +static void ep_done_scan(struct eventpoll *ep, + struct list_head *scan_batch) +{ + struct epitem *epi, *nepi; + + spin_lock_irq(&ep->lock); + /* + * During the time we spent inside the "sproc" callback, some + * other events might have been queued by the poll callback. + * We re-insert them inside the main ready-list here. + */ + for (nepi = READ_ONCE(ep->ovflist); (epi = nepi) != NULL; ) { + nepi = epi->ovflist_next; + epi_clear_ovflist(epi); + /* + * Skip items that the caller already returned via @scan_batch + * -- the list_splice() below takes care of those. + */ + if (!ep_is_linked(epi)) { + /* + * ovflist is LIFO; list_add() head-insert here + * reverses the iteration order into FIFO. + */ + list_add(&epi->rdllink, &ep->rdllist); + ep_pm_stay_awake(epi); + } + } + + write_seqcount_begin(&ep->seq); + + /* Back out of scan mode; callbacks target ep->rdllist again. */ + ep_exit_scan(ep); + + /* + * Quickly re-inject items left on "scan_batch". + */ + list_splice(scan_batch, &ep->rdllist); + + write_seqcount_end(&ep->seq); + + __pm_relax(ep->ws); + + if (!list_empty(&ep->rdllist)) { + if (waitqueue_active(&ep->wq)) + wake_up(&ep->wq); + } + + spin_unlock_irq(&ep->lock); +} + +static void ep_get(struct eventpoll *ep) +{ + refcount_inc(&ep->refcount); +} + +/* + * Drop a reference to @ep; returns true iff it was the last, in which + * case the caller is responsible for ep_free(). + */ +static bool ep_put(struct eventpoll *ep) +{ + if (!refcount_dec_and_test(&ep->refcount)) + return false; + + WARN_ON_ONCE(!RB_EMPTY_ROOT(&ep->rbr.rb_root)); + return true; +} + +static void ep_free(struct eventpoll *ep) +{ + ep_resume_napi_irqs(ep); + mutex_destroy(&ep->mtx); + free_uid(ep->user); + wakeup_source_unregister(ep->ws); + /* ep_get_upwards_depth_proc() may still hold epi->ep under RCU */ + kfree_rcu(ep, rcu); +} + +/* + * Pin @epi->ffd.file for operations that require both safe dereference + * and exclusion from __fput(). + * + * struct file uses SLAB_TYPESAFE_BY_RCU, so a freed slot can be + * reassigned at any time. The bare load of epi->ffd.file is safe here + * because the caller holds ep->mtx and eventpoll_release_file() blocks + * on that mutex while tearing down the epi, so the backing file + * allocation cannot be freed and reused under us. An rcu_read_lock() + * is therefore unnecessary for the load. + * + * A successful file_ref_get() additionally blocks __fput() from + * starting on this file: once the refcount has reached zero it cannot + * come back. ep_remove() relies on that to touch file->f_lock and + * file->f_ep without racing eventpoll_release_file() (see commit + * a6dc643c6931). A NULL return means __fput() is already in flight; + * the caller must bail without touching the file, and + * eventpoll_release_file() will clean the epi up from its side. + */ +static struct file *epi_fget(const struct epitem *epi) +{ + struct file *file; + + file = epi->ffd.file; + if (!file_ref_get(&file->f_ref)) + file = NULL; + return file; +} + +/* + * Takes &file->f_lock; returns with it released. + */ +static void ep_remove_file(struct eventpoll *ep, struct epitem *epi, + struct file *file) +{ + struct epitems_head *to_free = NULL; + struct hlist_head *head; + + lockdep_assert_held(&ep->mtx); + + spin_lock(&file->f_lock); + head = file->f_ep; + if (hlist_is_singular_node(&epi->fllink, head)) { + /* + * Last watcher: publish NULL so the eventpoll_release() + * fastpath in include/linux/eventpoll.h can skip the slow + * path on a future __fput(). Safe because every f_ep writer + * either holds a pin on @file via epi_fget() or is __fput() + * itself -- see the comment in eventpoll_release(). + */ + WRITE_ONCE(file->f_ep, NULL); + if (!is_file_epoll(file)) { + struct epitems_head *v; + v = container_of(head, struct epitems_head, epitems); + if (!smp_load_acquire(&v->next)) + to_free = v; + } + } + hlist_del_rcu(&epi->fllink); + spin_unlock(&file->f_lock); + free_ephead(to_free); +} + +static void ep_remove_epi(struct eventpoll *ep, struct epitem *epi) +{ + lockdep_assert_held(&ep->mtx); + + rb_erase_cached(&epi->rbn, &ep->rbr); + + spin_lock_irq(&ep->lock); + if (ep_is_linked(epi)) + list_del_init(&epi->rdllink); + spin_unlock_irq(&ep->lock); + + wakeup_source_unregister(ep_wakeup_source(epi)); + /* + * At this point it is safe to free the eventpoll item. Use the union + * field epi->rcu, since we are trying to minimize the size of + * 'struct epitem'. The 'rbn' field is no longer in use. Protected by + * ep->mtx. The rcu read side, reverse_path_check_proc(), does not make + * use of the rbn field. + */ + kfree_rcu(epi, rcu); + + percpu_counter_dec(&ep->user->epoll_watches); +} + +/* + * ep_remove variant for callers owing an additional reference to the ep + */ +static void ep_remove(struct eventpoll *ep, struct epitem *epi) +{ + struct file *file __free(fput) = NULL; + + lockdep_assert_irqs_enabled(); + lockdep_assert_held(&ep->mtx); + + ep_unregister_pollwait(ep, epi); + + /* + * If we manage to grab a reference it means we're not in + * eventpoll_release_file() and aren't going to be: once @file's + * refcount has reached zero, file_ref_get() cannot bring it back. + */ + file = epi_fget(epi); + if (!file) + return; + + ep_remove_file(ep, epi, file); + ep_remove_epi(ep, epi); + WARN_ON_ONCE(ep_put(ep)); +} + +/* + * Pass 1 of ep_clear_and_put(): drain every epi's pwqlist. + * ep_unregister_pollwait() takes each watched wait-queue head's lock, + * which synchronizes with any in-flight ep_poll_callback(); after + * this returns no callback can still be about to dereference an epi + * on this ep. Must strictly precede ep_drain_tree() -- fusing the + * two walks would let a callback queued on epi_i still fire after + * epi_{i+k} had already been freed. + */ +static void ep_drain_pollwaits(struct eventpoll *ep) +{ + struct rb_node *rbp; + struct epitem *epi; + + lockdep_assert_held(&ep->mtx); + + for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) { + epi = rb_entry(rbp, struct epitem, rbn); + + ep_unregister_pollwait(ep, epi); + cond_resched(); + } +} + +/* + * Pass 2 of ep_clear_and_put(): ep_remove() every epi. The per-epi + * pwqlist is already empty (ep_drain_pollwaits ran), but the rest of + * ep_remove() still runs: epi_fget() pin, f_ep clear under f_lock, + * rbtree erase, rdllist unlink, kfree_rcu(epi). rb_next() is captured + * before each erase so the iteration is stable. + * + * A concurrent eventpoll_release_file() (removal path C) on a watched + * file serializes with us via ep->mtx; ep_remove() transparently + * hands off any epi whose file is in __fput() by bailing when + * epi_fget() returns NULL, and path C will clean that epi up. + */ +static void ep_drain_tree(struct eventpoll *ep) +{ + struct rb_node *rbp, *next; + struct epitem *epi; + + lockdep_assert_held(&ep->mtx); + + for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = next) { + next = rb_next(rbp); + epi = rb_entry(rbp, struct epitem, rbn); + ep_remove(ep, epi); + cond_resched(); + } +} + +/* + * Removal path B (see "Removal paths" in the top-of-file banner): + * close of the epoll fd itself, reached via ep_eventpoll_release(). + * + * Two passes under ep->mtx: first ep_drain_pollwaits() quiesces + * in-flight callbacks, then ep_drain_tree() frees the epis. The + * ep->refcount is kept > 0 across the walk by the ep file's own + * share, which we drop below; ep_free() runs iff we were the last + * holder after the tree drained. + */ +static void ep_clear_and_put(struct eventpoll *ep) +{ + /* Release any threads blocked in poll-on-ep. */ + if (waitqueue_active(&ep->poll_wait)) + ep_poll_safewake(ep, NULL, 0); + + mutex_lock(&ep->mtx); + ep_drain_pollwaits(ep); + ep_drain_tree(ep); + mutex_unlock(&ep->mtx); + + if (ep_put(ep)) + ep_free(ep); +} + +static long ep_eventpoll_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + int ret; + + if (!is_file_epoll(file)) + return -EINVAL; + + switch (cmd) { + case EPIOCSPARAMS: + case EPIOCGPARAMS: + ret = ep_eventpoll_bp_ioctl(file, cmd, arg); + break; + default: + ret = -EINVAL; + break; + } + + return ret; +} + +static int ep_eventpoll_release(struct inode *inode, struct file *file) +{ + struct eventpoll *ep = file->private_data; + + if (ep) + ep_clear_and_put(ep); + + return 0; +} + +static __poll_t ep_item_poll(const struct epitem *epi, poll_table *pt, int depth); + +static __poll_t __ep_eventpoll_poll(struct file *file, poll_table *wait, int depth) +{ + struct eventpoll *ep = file->private_data; + LIST_HEAD(scan_batch); + struct epitem *epi, *tmp; + poll_table pt; + __poll_t res = 0; + + init_poll_funcptr(&pt, NULL); + + /* Insert inside our poll wait queue */ + poll_wait(file, &ep->poll_wait, wait); + + /* + * Proceed to find out if wanted events are really available inside + * the ready list. + */ + mutex_lock_nested(&ep->mtx, depth); + ep_start_scan(ep, &scan_batch); + list_for_each_entry_safe(epi, tmp, &scan_batch, rdllink) { + if (ep_item_poll(epi, &pt, depth + 1)) { + res = EPOLLIN | EPOLLRDNORM; + break; + } else { + /* + * Item has been dropped into the ready list by the poll + * callback, but it's not actually ready, as far as + * caller requested events goes. We can remove it here. + */ + __pm_relax(ep_wakeup_source(epi)); + list_del_init(&epi->rdllink); + } + } + ep_done_scan(ep, &scan_batch); + mutex_unlock(&ep->mtx); + return res; +} + +/* + * Differs from ep_eventpoll_poll() in that internal callers already have + * the ep->mtx so we need to start from depth=1, such that mutex_lock_nested() + * is correctly annotated. + */ +static __poll_t ep_item_poll(const struct epitem *epi, poll_table *pt, + int depth) +{ + struct file *file = epi_fget(epi); + __poll_t res; + + /* + * We could return EPOLLERR | EPOLLHUP or something, but let's + * treat this more as "file doesn't exist, poll didn't happen". + */ + if (!file) + return 0; + + pt->_key = epi->event.events; + if (!is_file_epoll(file)) + res = vfs_poll(file, pt); + else + res = __ep_eventpoll_poll(file, pt, depth); + fput(file); + return res & epi->event.events; +} + +static __poll_t ep_eventpoll_poll(struct file *file, poll_table *wait) +{ + return __ep_eventpoll_poll(file, wait, 0); +} + +#ifdef CONFIG_PROC_FS +static void ep_show_fdinfo(struct seq_file *m, struct file *f) +{ + struct eventpoll *ep = f->private_data; + struct rb_node *rbp; + + mutex_lock(&ep->mtx); + for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) { + struct epitem *epi = rb_entry(rbp, struct epitem, rbn); + struct inode *inode = file_inode(epi->ffd.file); + + seq_printf(m, "tfd: %8d events: %8x data: %16llx " + " pos:%lli ino:%llx sdev:%x\n", + epi->ffd.fd, epi->event.events, + (long long)epi->event.data, + (long long)epi->ffd.file->f_pos, + inode->i_ino, inode->i_sb->s_dev); + if (seq_has_overflowed(m)) + break; + } + mutex_unlock(&ep->mtx); +} +#endif + +/* File callbacks that implement the eventpoll file behaviour */ +static const struct file_operations eventpoll_fops = { +#ifdef CONFIG_PROC_FS + .show_fdinfo = ep_show_fdinfo, +#endif + .release = ep_eventpoll_release, + .poll = ep_eventpoll_poll, + .llseek = noop_llseek, + .unlocked_ioctl = ep_eventpoll_ioctl, + .compat_ioctl = compat_ptr_ioctl, +}; + +/* + * This is called from eventpoll_release() to unlink files from the eventpoll + * interface. We need to have this facility to cleanup correctly files that are + * closed without being removed from the eventpoll interface. + */ +void eventpoll_release_file(struct file *file) +{ + struct eventpoll *ep; + struct epitem *epi; + + /* + * A concurrent ep_remove() cannot outrace us: it pins @file via + * epi_fget(), which fails once __fput() has dropped the refcount + * to zero -- the path we're on. So any racing ep_remove() bails + * and leaves the epi for us to clean up here. + */ +again: + spin_lock(&file->f_lock); + if (file->f_ep && file->f_ep->first) { + epi = hlist_entry(file->f_ep->first, struct epitem, fllink); + spin_unlock(&file->f_lock); + + /* + * ep access is safe as we still own a reference to the ep + * struct + */ + ep = epi->ep; + mutex_lock(&ep->mtx); + + ep_unregister_pollwait(ep, epi); + + ep_remove_file(ep, epi, file); + ep_remove_epi(ep, epi); + + mutex_unlock(&ep->mtx); + + if (ep_put(ep)) + ep_free(ep); + goto again; + } + spin_unlock(&file->f_lock); +} + +static int ep_alloc(struct eventpoll **pep) +{ + struct eventpoll *ep; + + ep = kzalloc_obj(*ep); + if (unlikely(!ep)) + return -ENOMEM; + + mutex_init(&ep->mtx); + spin_lock_init(&ep->lock); + seqcount_spinlock_init(&ep->seq, &ep->lock); + init_waitqueue_head(&ep->wq); + init_waitqueue_head(&ep->poll_wait); + INIT_LIST_HEAD(&ep->rdllist); + ep->rbr = RB_ROOT_CACHED; + ep->ovflist = EP_UNACTIVE_PTR; /* not scanning */ + ep->user = get_current_user(); + refcount_set(&ep->refcount, 1); + + *pep = ep; + + return 0; +} + +/* + * Search the file inside the eventpoll tree. The RB tree operations + * are protected by the "mtx" mutex, and ep_find() must be called with + * "mtx" held. + */ +static struct epitem *ep_find(struct eventpoll *ep, struct epoll_key *tf) +{ + int kcmp; + struct rb_node *rbp; + struct epitem *epi, *epir = NULL; + + for (rbp = ep->rbr.rb_root.rb_node; rbp; ) { + epi = rb_entry(rbp, struct epitem, rbn); + kcmp = ep_cmp_ffd(tf, &epi->ffd); + if (kcmp > 0) + rbp = rbp->rb_right; + else if (kcmp < 0) + rbp = rbp->rb_left; + else { + epir = epi; + break; + } + } + + return epir; +} + +/* + * This is the callback that is passed to the wait queue wakeup + * mechanism. It is called by the stored file descriptors when they + * have events to report. + */ +static int ep_poll_callback(wait_queue_entry_t *wait, unsigned mode, int sync, void *key) +{ + int pwake = 0; + struct epitem *epi = ep_item_from_wait(wait); + struct eventpoll *ep = epi->ep; + __poll_t pollflags = key_to_poll(key); + unsigned long flags; + int ewake = 0; + + spin_lock_irqsave(&ep->lock, flags); + + ep_set_busy_poll_napi_id(epi); + + /* + * If the event mask does not contain any poll(2) event, we consider the + * descriptor to be disabled. This condition is likely the effect of the + * EPOLLONESHOT bit that disables the descriptor when an event is received, + * until the next EPOLL_CTL_MOD will be issued. + */ + if (!(epi->event.events & ~EP_PRIVATE_BITS)) + goto out_unlock; + + /* + * Check the events coming with the callback. At this stage, not + * every device reports the events in the "key" parameter of the + * callback. We need to be able to handle both cases here, hence the + * test for "key" != NULL before the event match test. + */ + if (pollflags && !(pollflags & epi->event.events)) + goto out_unlock; + + /* + * If we are transferring events to userspace, we can hold no locks + * (because we're accessing user memory, and because of linux f_op->poll() + * semantics). All the events that happen during that period of time are + * chained in ep->ovflist and requeued later on. + */ + if (ep_is_scanning(ep)) { + if (!epi_on_ovflist(epi)) { + epi->ovflist_next = READ_ONCE(ep->ovflist); + WRITE_ONCE(ep->ovflist, epi); + ep_pm_stay_awake_rcu(epi); + } + } else if (!ep_is_linked(epi)) { + /* In the usual case, add event to ready list. */ + list_add_tail(&epi->rdllink, &ep->rdllist); + ep_pm_stay_awake_rcu(epi); + } + + /* + * Wake up ( if active ) both the eventpoll wait list and the ->poll() + * wait list. + */ + if (waitqueue_active(&ep->wq)) { + if ((epi->event.events & EPOLLEXCLUSIVE) && + !(pollflags & POLLFREE)) { + switch (pollflags & EPOLLINOUT_BITS) { + case EPOLLIN: + if (epi->event.events & EPOLLIN) + ewake = 1; + break; + case EPOLLOUT: + if (epi->event.events & EPOLLOUT) + ewake = 1; + break; + case 0: + ewake = 1; + break; + } + } + if (sync) + wake_up_sync(&ep->wq); + else + wake_up(&ep->wq); + } + if (waitqueue_active(&ep->poll_wait)) + pwake++; + +out_unlock: + spin_unlock_irqrestore(&ep->lock, flags); + + /* We have to call this outside the lock */ + if (pwake) + ep_poll_safewake(ep, epi, pollflags & EPOLL_URING_WAKE); + + if (!(epi->event.events & EPOLLEXCLUSIVE)) + ewake = 1; + + if (pollflags & POLLFREE) { + /* + * POLLFREE handshake, release side; see "POLLFREE handshake" + * at the top of this file. + * + * Unlink our wait entry with list_del_init rather than + * __remove_wait_queue: a concurrent ep_remove_wait_queue() + * that already loaded a non-NULL whead may still call + * remove_wait_queue() after us, and list_del_init() tolerates + * the second delete. + * + * smp_store_release(&whead, NULL) publishes the teardown to + * ep_remove_wait_queue()'s smp_load_acquire(). Before this + * store, a racing ep_clear_and_put() / ep_remove() reaches + * ep_remove_wait_queue() which sees whead != NULL and takes + * whead->lock -- the same lock held by our caller, so it + * serializes behind us. Once whead is zeroed, nothing else + * protects ep / epi / wait. + */ + list_del_init(&wait->entry); + smp_store_release(&ep_pwq_from_wait(wait)->whead, NULL); + } + + return ewake; +} + +/* + * This is the callback that is used to add our wait queue to the + * target file wakeup lists. + */ +static void ep_ptable_queue_proc(struct file *file, wait_queue_head_t *whead, + poll_table *pt) +{ + struct ep_pqueue *epq = container_of(pt, struct ep_pqueue, pt); + struct epitem *epi = epq->epi; + struct eppoll_entry *pwq; + + if (unlikely(!epi)) // an earlier allocation has failed + return; + + pwq = kmem_cache_alloc(pwq_cache, GFP_KERNEL); + if (unlikely(!pwq)) { + epq->epi = NULL; + return; + } + + init_waitqueue_func_entry(&pwq->wait, ep_poll_callback); + pwq->whead = whead; + pwq->base = epi; + if (epi->event.events & EPOLLEXCLUSIVE) + add_wait_queue_exclusive(whead, &pwq->wait); + else + add_wait_queue(whead, &pwq->wait); + pwq->next = epi->pwqlist; + epi->pwqlist = pwq; +} + +static void ep_rbtree_insert(struct eventpoll *ep, struct epitem *epi) +{ + int kcmp; + struct rb_node **p = &ep->rbr.rb_root.rb_node, *parent = NULL; + struct epitem *epic; + bool leftmost = true; + + while (*p) { + parent = *p; + epic = rb_entry(parent, struct epitem, rbn); + kcmp = ep_cmp_ffd(&epi->ffd, &epic->ffd); + if (kcmp > 0) { + p = &parent->rb_right; + leftmost = false; + } else + p = &parent->rb_left; + } + rb_link_node(&epi->rbn, parent, p); + rb_insert_color_cached(&epi->rbn, &ep->rbr, leftmost); +} + + + +/* + * Upper bound on wakeup paths emanating from any one watched file, + * indexed by path depth (1..PATH_ARR_SIZE). For example, we allow + * 1000 paths of length 1 from each watched file. These caps limit + * the wakeup amplification that can be built from epoll-watches- + * epoll topologies without rejecting reasonable usage. + * + * Enforced at EPOLL_CTL_ADD; CTL_MOD and CTL_DEL cannot add paths. + * The running tallies live in ctx->path_count[] and are protected by + * epnested_mutex. + */ +static const int path_limits[PATH_ARR_SIZE] = { 1000, 500, 100, 50, 10 }; + +static int path_count_inc(struct ep_ctl_ctx *ctx, int nests) +{ + /* Allow an arbitrary number of depth 1 paths */ + if (nests == 0) + return 0; + + if (++ctx->path_count[nests] > path_limits[nests]) + return -1; + return 0; +} + +static void path_count_init(struct ep_ctl_ctx *ctx) +{ + int i; + + for (i = 0; i < PATH_ARR_SIZE; i++) + ctx->path_count[i] = 0; +} + +static int reverse_path_check_proc(struct ep_ctl_ctx *ctx, + struct hlist_head *refs, int depth) +{ + int error = 0; + struct epitem *epi; + + if (depth > EP_MAX_NESTS) /* too deep nesting */ + return -1; + + /* CTL_DEL can remove links here, but that can't increase our count */ + hlist_for_each_entry_rcu(epi, refs, fllink) { + struct hlist_head *refs = &epi->ep->refs; + if (hlist_empty(refs)) + error = path_count_inc(ctx, depth); + else + error = reverse_path_check_proc(ctx, refs, depth + 1); + if (error != 0) + break; + } + return error; +} + +/** + * reverse_path_check - ctx->tfile_check_list is a list of epitems_head + * anchoring files with newly proposed links; make + * sure those links don't push any path-length bucket + * over its limit in path_limits[]. + * @ctx: Per-do_epoll_ctl() scratch for the loop / path checks. + * + * Return: %zero if the proposed links don't create too many paths, + * %-1 otherwise. + */ +static int reverse_path_check(struct ep_ctl_ctx *ctx) +{ + struct epitems_head *p; + + for (p = ctx->tfile_check_list; p != EP_UNACTIVE_PTR; p = p->next) { + int error; + path_count_init(ctx); + rcu_read_lock(); + error = reverse_path_check_proc(ctx, &p->epitems, 0); + rcu_read_unlock(); + if (error) + return error; + } + return 0; +} + +static int ep_create_wakeup_source(struct epitem *epi) +{ + struct name_snapshot n; + struct wakeup_source *ws; + + if (!epi->ep->ws) { + epi->ep->ws = wakeup_source_register(NULL, "eventpoll"); + if (!epi->ep->ws) + return -ENOMEM; + } + + take_dentry_name_snapshot(&n, epi->ffd.file->f_path.dentry); + ws = wakeup_source_register(NULL, n.name.name); + release_dentry_name_snapshot(&n); + + if (!ws) + return -ENOMEM; + rcu_assign_pointer(epi->ws, ws); + + return 0; +} + +/* rare code path, only used when EPOLL_CTL_MOD removes a wakeup source */ +static noinline void ep_destroy_wakeup_source(struct epitem *epi) +{ + struct wakeup_source *ws = ep_wakeup_source(epi); + + RCU_INIT_POINTER(epi->ws, NULL); + + /* + * wait for ep_pm_stay_awake_rcu to finish, synchronize_rcu is + * used internally by wakeup_source_remove, too (called by + * wakeup_source_unregister), so we cannot use call_rcu + */ + synchronize_rcu(); + wakeup_source_unregister(ws); +} + +static int ep_attach_file(struct file *file, struct epitem *epi) +{ + struct epitems_head *to_free = NULL; + struct hlist_head *head = NULL; + struct eventpoll *ep = NULL; + + if (is_file_epoll(file)) + ep = file->private_data; + + if (ep) { + head = &ep->refs; + } else if (!READ_ONCE(file->f_ep)) { +allocate: + to_free = kmem_cache_zalloc(ephead_cache, GFP_KERNEL); + if (!to_free) + return -ENOMEM; + head = &to_free->epitems; + } + spin_lock(&file->f_lock); + if (!file->f_ep) { + if (unlikely(!head)) { + spin_unlock(&file->f_lock); + goto allocate; + } + /* See eventpoll_release() for details. */ + WRITE_ONCE(file->f_ep, head); + to_free = NULL; + } + hlist_add_head_rcu(&epi->fllink, file->f_ep); + spin_unlock(&file->f_lock); + free_ephead(to_free); + return 0; +} + +/* + * Charge the user's epoll_watches quota, allocate a fresh epitem for + * @tf, and initialize its fields. The returned item is not yet linked + * into any data structure; the caller must install it via + * ep_register_epitem() (which takes over on success) or kmem_cache_free() + * it and decrement epoll_watches on its own. + * + * Returns ERR_PTR(-ENOSPC) if the quota is exceeded, ERR_PTR(-ENOMEM) + * if the slab allocation fails. + */ +static struct epitem *ep_alloc_epitem(struct eventpoll *ep, + const struct epoll_event *event, + struct epoll_key *tf) +{ + struct epitem *epi; + + if (unlikely(percpu_counter_compare(&ep->user->epoll_watches, + max_user_watches) >= 0)) + return ERR_PTR(-ENOSPC); + percpu_counter_inc(&ep->user->epoll_watches); + + epi = kmem_cache_zalloc(epi_cache, GFP_KERNEL); + if (unlikely(!epi)) { + percpu_counter_dec(&ep->user->epoll_watches); + return ERR_PTR(-ENOMEM); + } + + INIT_LIST_HEAD(&epi->rdllink); + epi->ep = ep; + epi->ffd = *tf; + epi->event = *event; + epi_clear_ovflist(epi); + + return epi; +} + +/* + * Install @epi into its target file's f_ep hlist and into @ep's rbtree, + * taking one additional reference on @ep for the lifetime of the item. + * + * If @tep is non-NULL, the target file is itself an eventpoll; we hold + * tep->mtx at subclass 1 across the attach + rbtree insert to serialize + * with the target side. RB tree ops are protected by @ep->mtx, which + * the caller already holds. + * + * On failure the epi is freed and the epoll_watches counter decremented, + * matching ep_alloc_epitem()'s allocation. After this returns + * successfully, ep_insert()'s later error paths use ep_remove() for + * unwind; that cannot drop @ep's refcount to zero because the ep file + * itself still holds the original reference. + */ +static int ep_register_epitem(struct ep_ctl_ctx *ctx, struct eventpoll *ep, + struct epitem *epi, struct eventpoll *tep, + int full_check) +{ + struct file *tfile = epi->ffd.file; + int error; + + if (tep) + mutex_lock_nested(&tep->mtx, 1); + + error = ep_attach_file(tfile, epi); + if (unlikely(error)) { + if (tep) + mutex_unlock(&tep->mtx); + kmem_cache_free(epi_cache, epi); + percpu_counter_dec(&ep->user->epoll_watches); + return error; + } + + if (full_check && !tep) + list_file(tfile, ctx); + + ep_rbtree_insert(ep, epi); + + if (tep) + mutex_unlock(&tep->mtx); + + ep_get(ep); + return 0; +} + +/* + * Must be called with "mtx" held. + */ +static int ep_insert(struct ep_ctl_ctx *ctx, struct eventpoll *ep, + const struct epoll_event *event, struct epoll_key *tf, + int full_check) +{ + int error, pwake = 0; + __poll_t revents; + struct epitem *epi; + struct ep_pqueue epq; + struct eventpoll *tep = NULL; + + if (is_file_epoll(tf->file)) + tep = tf->file->private_data; + + lockdep_assert_irqs_enabled(); + + epi = ep_alloc_epitem(ep, event, tf); + if (IS_ERR(epi)) + return PTR_ERR(epi); + + error = ep_register_epitem(ctx, ep, epi, tep, full_check); + if (error) + return error; + + /* Reject the insert if the new link would create too many back-paths. */ + if (unlikely(full_check && reverse_path_check(ctx))) { + ep_remove(ep, epi); + return -EINVAL; + } + + if (epi->event.events & EPOLLWAKEUP) { + error = ep_create_wakeup_source(epi); + if (error) { + ep_remove(ep, epi); + return error; + } + } + + /* Initialize the poll table using the queue callback */ + epq.epi = epi; + init_poll_funcptr(&epq.pt, ep_ptable_queue_proc); + + /* + * Attach the item to the poll hooks and get current event bits. + * We can safely use the file* here because its usage count has + * been increased by the caller of this function. Note that after + * this operation completes, the poll callback can start hitting + * the new item. + */ + revents = ep_item_poll(epi, &epq.pt, 1); + + /* ep_ptable_queue_proc() signals allocation failure by clearing epq.epi. */ + if (unlikely(!epq.epi)) { + ep_remove(ep, epi); + return -ENOMEM; + } + + /* Drop the new item onto the ready list if it is already ready. */ + spin_lock_irq(&ep->lock); + + ep_set_busy_poll_napi_id(epi); + + if (revents && !ep_is_linked(epi)) { + list_add_tail(&epi->rdllink, &ep->rdllist); + ep_pm_stay_awake(epi); + + if (waitqueue_active(&ep->wq)) + wake_up(&ep->wq); + if (waitqueue_active(&ep->poll_wait)) + pwake++; + } + + spin_unlock_irq(&ep->lock); + + /* We have to call this outside the lock */ + if (pwake) + ep_poll_safewake(ep, NULL, 0); + + return 0; +} + +/* + * Modify the interest event mask by dropping an event if the new mask + * has a match in the current file status. Must be called with "mtx" held. + */ +static int ep_modify(struct eventpoll *ep, struct epitem *epi, + const struct epoll_event *event) +{ + int pwake = 0; + poll_table pt; + + lockdep_assert_irqs_enabled(); + + init_poll_funcptr(&pt, NULL); + + /* + * Set the new event interest mask before calling f_op->poll(); + * otherwise we might miss an event that happens between the + * f_op->poll() call and the new event set registering. + */ + epi->event.events = event->events; /* need barrier below */ + epi->event.data = event->data; /* protected by mtx */ + if (epi->event.events & EPOLLWAKEUP) { + if (!ep_has_wakeup_source(epi)) + ep_create_wakeup_source(epi); + } else if (ep_has_wakeup_source(epi)) { + ep_destroy_wakeup_source(epi); + } + + /* + * The following barrier has two effects: + * + * 1) Flush epi changes above to other CPUs. This ensures + * we do not miss events from ep_poll_callback if an + * event occurs immediately after we call f_op->poll(). + * We need this because we did not take ep->lock while + * changing epi above (but ep_poll_callback does take + * ep->lock). + * + * 2) We also need to ensure we do not miss _past_ events + * when calling f_op->poll(). This barrier also + * pairs with the barrier in wq_has_sleeper (see + * comments for wq_has_sleeper). + * + * This barrier will now guarantee ep_poll_callback or f_op->poll + * (or both) will notice the readiness of an item. + */ + smp_mb(); + + /* + * Get current event bits. We can safely use the file* here because + * its usage count has been increased by the caller of this function. + * If the item is "hot" and it is not registered inside the ready + * list, push it inside. + */ + if (ep_item_poll(epi, &pt, 1)) { + spin_lock_irq(&ep->lock); + if (!ep_is_linked(epi)) { + list_add_tail(&epi->rdllink, &ep->rdllist); + ep_pm_stay_awake(epi); + + /* Notify waiting tasks that events are available */ + if (waitqueue_active(&ep->wq)) + wake_up(&ep->wq); + if (waitqueue_active(&ep->poll_wait)) + pwake++; + } + spin_unlock_irq(&ep->lock); + } + + /* We have to call this outside the lock */ + if (pwake) + ep_poll_safewake(ep, NULL, 0); + + return 0; +} + +/* + * Attempt to deliver one event for @epi into @*uevents. + * + * Returns 1 if an event was delivered (with *uevents advanced to the + * next slot), 0 if the re-poll reported no caller-requested events + * (@epi drops out of the ready list; a future callback will re-add + * it), or -EFAULT if copy_to_user() faulted (in which case @epi is + * re-inserted at the head of @scan_batch so ep_done_scan() merges it + * back to rdllist for the next attempt). + * + * PM bookkeeping and level-triggered re-queue are handled here. + * Caller holds ep->mtx and the scan is active. + */ +static int ep_deliver_event(struct eventpoll *ep, struct epitem *epi, + poll_table *pt, + struct epoll_event __user **uevents, + struct list_head *scan_batch) +{ + struct epoll_event __user *next; + struct wakeup_source *ws; + __poll_t revents; + + /* + * Activate ep->ws before deactivating epi->ws to prevent + * triggering auto-suspend here (in case we reactivate epi->ws + * below). Rearranging to delay the deactivation would let + * epi->ws drift out of sync with ep_is_linked(). + */ + ws = ep_wakeup_source(epi); + if (ws) { + if (ws->active) + __pm_stay_awake(ep->ws); + __pm_relax(ws); + } + + list_del_init(&epi->rdllink); + + /* + * Re-poll under ep->mtx so userspace cannot change the item + * out from under us. If no caller-requested events remain, + * @epi stays off the ready list; the poll callback will + * re-queue it when events next appear. + */ + revents = ep_item_poll(epi, pt, 1); + if (!revents) + return 0; + + next = epoll_put_uevent(revents, epi->event.data, *uevents); + if (!next) { + /* + * copy_to_user() faulted: put the item back so + * ep_done_scan() splices it onto rdllist for the next + * attempt. + */ + list_add(&epi->rdllink, scan_batch); + ep_pm_stay_awake(epi); + return -EFAULT; + } + *uevents = next; + + if (epi->event.events & EPOLLONESHOT) { + epi->event.events &= EP_PRIVATE_BITS; + } else if (!(epi->event.events & EPOLLET)) { + /* + * Level-triggered: re-queue so the next epoll_wait() + * rechecks availability. We are the sole writer to + * rdllist here -- epoll_ctl() callers are locked out + * by ep->mtx, and the poll callback queues to ovflist + * during scans. + */ + list_add_tail(&epi->rdllink, &ep->rdllist); + ep_pm_stay_awake(epi); + } + return 1; +} + +static int ep_send_events(struct eventpoll *ep, + struct epoll_event __user *events, int maxevents) +{ + struct epitem *epi, *tmp; + LIST_HEAD(scan_batch); + poll_table pt; + int res = 0; + + /* + * Always short-circuit for fatal signals to allow threads to make a + * timely exit without the chance of finding more events available and + * fetching repeatedly. + */ + if (fatal_signal_pending(current)) + return -EINTR; + + init_poll_funcptr(&pt, NULL); + + mutex_lock(&ep->mtx); + ep_start_scan(ep, &scan_batch); + + /* + * We can loop without lock because we are passed a task-private + * scan_batch; items cannot vanish while we hold ep->mtx. + */ + list_for_each_entry_safe(epi, tmp, &scan_batch, rdllink) { + int delivered; + + if (res >= maxevents) + break; + + delivered = ep_deliver_event(ep, epi, &pt, &events, &scan_batch); + if (delivered < 0) { + if (!res) + res = delivered; + break; + } + res += delivered; + } + + ep_done_scan(ep, &scan_batch); + mutex_unlock(&ep->mtx); + + return res; +} + +static struct timespec64 *ep_timeout_to_timespec(struct timespec64 *to, long ms) +{ + struct timespec64 now; + + if (ms < 0) + return NULL; + + if (!ms) { + to->tv_sec = 0; + to->tv_nsec = 0; + return to; + } + + to->tv_sec = ms / MSEC_PER_SEC; + to->tv_nsec = NSEC_PER_MSEC * (ms % MSEC_PER_SEC); + + ktime_get_ts64(&now); + *to = timespec64_add_safe(now, *to); + return to; +} + +/* + * autoremove_wake_function, but remove even on failure to wake up, because we + * know that default_wake_function/ttwu will only fail if the thread is already + * woken, and in that case the ep_poll loop will remove the entry anyways, not + * try to reuse it. + */ +static int ep_autoremove_wake_function(struct wait_queue_entry *wq_entry, + unsigned int mode, int sync, void *key) +{ + int ret = default_wake_function(wq_entry, mode, sync, key); + + /* + * Pairs with list_empty_careful in ep_poll, and ensures future loop + * iterations see the cause of this wakeup. + */ + list_del_init_careful(&wq_entry->entry); + return ret; +} + +static int ep_try_send_events(struct eventpoll *ep, + struct epoll_event __user *events, int maxevents) +{ + int res; + + /* + * Try to transfer events to user space. In case we get 0 events and + * there's still timeout left over, we go trying again in search of + * more luck. + */ + res = ep_send_events(ep, events, maxevents); + if (res > 0) + ep_suspend_napi_irqs(ep); + return res; +} + +static int ep_schedule_timeout(ktime_t *to) +{ + if (to) + return ktime_after(*to, ktime_get()); + else + return 1; +} + +/** + * ep_poll - Retrieves ready events, and delivers them to the caller-supplied + * event buffer. + * + * @ep: Pointer to the eventpoll context. + * @events: Pointer to the userspace buffer where the ready events should be + * stored. + * @maxevents: Size (in terms of number of events) of the caller event buffer. + * @timeout: Maximum timeout for the ready events fetch operation, in + * timespec. If the timeout is zero, the function will not block, + * while if the @timeout ptr is NULL, the function will block + * until at least one event has been retrieved (or an error + * occurred). + * + * Return: the number of ready events which have been fetched, or an + * error code, in case of error. + */ +static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, + int maxevents, struct timespec64 *timeout) +{ + int res, timed_out = 0; + bool eavail; + u64 slack = 0; + wait_queue_entry_t wait; + ktime_t expires, *to = NULL; + + lockdep_assert_irqs_enabled(); + + if (timeout && (timeout->tv_sec | timeout->tv_nsec)) { + to = &expires; + *to = timespec64_to_ktime(*timeout); + } else if (timeout) { + /* + * Avoid the unnecessary trip to the wait queue loop, if the + * caller specified a non blocking operation. + */ + timed_out = 1; + } + + /* + * This call is racy: We may or may not see events that are being added + * to the ready list under the lock (e.g., in IRQ callbacks). For cases + * with a non-zero timeout, this thread will check the ready list under + * lock and will add to the wait queue. For cases with a zero + * timeout, the user by definition should not care and will have to + * recheck again. + */ + eavail = ep_events_available(ep); + + while (1) { + if (eavail) { + res = ep_try_send_events(ep, events, maxevents); + if (res) + return res; + } + + if (timed_out) + return 0; + + eavail = ep_busy_loop(ep); + if (eavail) + continue; + + if (signal_pending(current)) + return -EINTR; + + /* + * Internally init_wait() uses autoremove_wake_function(), + * thus wait entry is removed from the wait queue on each + * wakeup. Why it is important? In case of several waiters + * each new wakeup will hit the next waiter, giving it the + * chance to harvest new event. Otherwise wakeup can be + * lost. This is also good performance-wise, because on + * normal wakeup path no need to call __remove_wait_queue() + * explicitly, thus ep->lock is not taken, which halts the + * event delivery. + * + * In fact, we now use an even more aggressive function that + * unconditionally removes, because we don't reuse the wait + * entry between loop iterations. This lets us also avoid the + * performance issue if a process is killed, causing all of its + * threads to wake up without being removed normally. + */ + init_wait(&wait); + wait.func = ep_autoremove_wake_function; + + spin_lock_irq(&ep->lock); + /* + * Barrierless variant, waitqueue_active() is called under + * the same lock on wakeup ep_poll_callback() side, so it + * is safe to avoid an explicit barrier. + */ + __set_current_state(TASK_INTERRUPTIBLE); + + /* + * Do the final check under the lock. ep_start/done_scan() + * plays with two lists (->rdllist and ->ovflist) and there + * is always a race when both lists are empty for short + * period of time although events are pending, so lock is + * important. + */ + eavail = ep_events_available(ep); + if (!eavail) + __add_wait_queue_exclusive(&ep->wq, &wait); + + spin_unlock_irq(&ep->lock); + + if (!eavail) { + if (to) + slack = select_estimate_accuracy(timeout); + timed_out = !ep_schedule_timeout(to) || + !schedule_hrtimeout_range(to, slack, + HRTIMER_MODE_ABS); + } + __set_current_state(TASK_RUNNING); + + /* + * We were woken up, thus go and try to harvest some events. + * If timed out and still on the wait queue, recheck eavail + * carefully under lock, below. + */ + eavail = true; + + if (!list_empty_careful(&wait.entry)) { + spin_lock_irq(&ep->lock); + /* + * If the thread timed out and is not on the wait queue, + * it means that the thread was woken up after its + * timeout expired before it could reacquire the lock. + * Thus, when wait.entry is empty, it needs to harvest + * events. + */ + if (timed_out) + eavail = list_empty(&wait.entry); + __remove_wait_queue(&ep->wq, &wait); + spin_unlock_irq(&ep->lock); + } + } +} + +/** + * ep_loop_check_proc - verify that adding an epoll file @ep inside another + * epoll file does not create closed loops, and + * determine the depth of the subtree starting at @ep + * + * @ctx: Per-do_epoll_ctl() scratch for the loop / path checks. + * @ep: the &struct eventpoll to be currently checked. + * @depth: Current depth of the path being checked. + * + * Return: depth of the subtree, or a value bigger than EP_MAX_NESTS if we found + * a loop or went too deep. + */ +static int ep_loop_check_proc(struct ep_ctl_ctx *ctx, + struct eventpoll *ep, int depth) +{ + int result = 0; + struct rb_node *rbp; + struct epitem *epi; + + if (ep->gen == loop_check_gen) + return ep->loop_check_depth; + + mutex_lock_nested(&ep->mtx, depth + 1); + ep->gen = loop_check_gen; + for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) { + epi = rb_entry(rbp, struct epitem, rbn); + if (unlikely(is_file_epoll(epi->ffd.file))) { + struct eventpoll *ep_tovisit; + ep_tovisit = epi->ffd.file->private_data; + if (ep_tovisit == ctx->inserting_into || + depth > EP_MAX_NESTS) + result = EP_MAX_NESTS+1; + else + result = max(result, + ep_loop_check_proc(ctx, ep_tovisit, + depth + 1) + 1); + if (result > EP_MAX_NESTS) + break; + } else { + /* + * A non-epoll leaf. Queue it for the companion + * reverse_path_check() that runs after this walk so + * any new links we propose don't add too many wakeup + * paths. + */ + list_file(epi->ffd.file, ctx); + } + } + ep->loop_check_depth = result; + mutex_unlock(&ep->mtx); + + return result; +} + +/* ep_get_upwards_depth_proc - determine depth of @ep when traversed upwards */ +static int ep_get_upwards_depth_proc(struct eventpoll *ep, int depth) +{ + int result = 0; + struct epitem *epi; + + if (ep->gen == loop_check_gen) + return ep->loop_check_depth; + hlist_for_each_entry_rcu(epi, &ep->refs, fllink) + result = max(result, ep_get_upwards_depth_proc(epi->ep, depth + 1) + 1); + ep->gen = loop_check_gen; + ep->loop_check_depth = result; + return result; +} + +/** + * ep_loop_check - Performs a check to verify that adding an epoll file (@to) + * into another epoll file (represented by @ep) does not create + * closed loops or too deep chains. + * + * @ctx: Per-CTL_ADD scratch context. + * @ep: Pointer to the epoll we are inserting into. + * @to: Pointer to the epoll to be inserted. + * + * Return: %zero if adding the epoll @to inside the epoll @from + * does not violate the constraints, or %-1 otherwise. + */ +static int ep_loop_check(struct ep_ctl_ctx *ctx, struct eventpoll *ep, + struct eventpoll *to) +{ + int depth, upwards_depth; + + ctx->inserting_into = ep; + /* + * Check how deep down we can get from @to, and whether it is possible + * to loop up to @ep. + */ + depth = ep_loop_check_proc(ctx, to, 0); + if (depth > EP_MAX_NESTS) + return -1; + /* Check how far up we can go from @ep. */ + rcu_read_lock(); + upwards_depth = ep_get_upwards_depth_proc(ep, 0); + rcu_read_unlock(); + + return (depth+1+upwards_depth > EP_MAX_NESTS) ? -1 : 0; +} + +static void clear_tfile_check_list(struct ep_ctl_ctx *ctx) +{ + rcu_read_lock(); + while (ctx->tfile_check_list != EP_UNACTIVE_PTR) { + struct epitems_head *head = ctx->tfile_check_list; + ctx->tfile_check_list = head->next; + unlist_file(head); + } + rcu_read_unlock(); +} + +/* + * Open an eventpoll file descriptor. + */ +static int do_epoll_create(int flags) +{ + int error; + struct eventpoll *ep; + + /* Check the EPOLL_* constant for consistency. */ + BUILD_BUG_ON(EPOLL_CLOEXEC != O_CLOEXEC); + + if (flags & ~EPOLL_CLOEXEC) + return -EINVAL; + /* + * Create the internal data structure ("struct eventpoll"). + */ + error = ep_alloc(&ep); + if (error < 0) + return error; + /* + * Creates all the items needed to setup an eventpoll file. That is, + * a file structure and a free file descriptor. + */ + FD_PREPARE(fdf, O_RDWR | (flags & O_CLOEXEC), + anon_inode_getfile("[eventpoll]", &eventpoll_fops, ep, + O_RDWR | (flags & O_CLOEXEC))); + if (fdf.err) { + ep_clear_and_put(ep); + return fdf.err; + } + ep->file = fd_prepare_file(fdf); + return fd_publish(fdf); +} + +SYSCALL_DEFINE1(epoll_create1, int, flags) +{ + return do_epoll_create(flags); +} + +SYSCALL_DEFINE1(epoll_create, int, size) +{ + if (size <= 0) + return -EINVAL; + + return do_epoll_create(0); +} + +#ifdef CONFIG_PM_SLEEP +static inline void ep_take_care_of_epollwakeup(struct epoll_event *epev) +{ + if ((epev->events & EPOLLWAKEUP) && !capable(CAP_BLOCK_SUSPEND)) + epev->events &= ~EPOLLWAKEUP; +} +#else +static inline void ep_take_care_of_epollwakeup(struct epoll_event *epev) +{ + epev->events &= ~EPOLLWAKEUP; +} +#endif + +static inline int epoll_mutex_lock(struct mutex *mutex, bool nonblock) +{ + if (!nonblock) { + mutex_lock(mutex); + return 0; + } + return mutex_trylock(mutex) ? 0 : -EAGAIN; +} + +/* + * Acquire the locks required for do_epoll_ctl() on @ep for @op. + * + * Always takes ep->mtx. For EPOLL_CTL_ADD, additionally runs the + * loop / path check under epnested_mutex when the topology can + * change: @ep is already watched (epfile->f_ep non-NULL), @ep was + * recently loop-checked (ep->gen == loop_check_gen), or @tfile is + * itself an eventpoll. + * + * Return value encodes both outcome and lock state: + * + * 0 success; ep->mtx held. + * 1 success; ep->mtx held AND the full check ran under + * epnested_mutex (which is also still held). The value + * doubles as the @full_check argument to ep_insert(). + * -errno failure; no locks held. + * + * The caller releases what was taken with ep_ctl_unlock(ep, ret). + * + * Holding epnested_mutex on add is what prevents two racing + * EPOLL_CTL_ADDs on different eps from building a cycle without + * either walker observing it. + */ +static int ep_ctl_lock(struct ep_ctl_ctx *ctx, struct eventpoll *ep, int op, + struct file *epfile, struct file *tfile, bool nonblock) +{ + struct eventpoll *tep; + int error; + + error = epoll_mutex_lock(&ep->mtx, nonblock); + if (error) + return error; + + if (op != EPOLL_CTL_ADD) + return 0; + if (!READ_ONCE(epfile->f_ep) && ep->gen != loop_check_gen && + !is_file_epoll(tfile)) + return 0; + + /* Full check needed: drop ep->mtx so we can take epnested_mutex. */ + mutex_unlock(&ep->mtx); + error = epoll_mutex_lock(&epnested_mutex, nonblock); + if (error) + return error; + + loop_check_gen++; + + if (is_file_epoll(tfile)) { + tep = tfile->private_data; + if (ep_loop_check(ctx, ep, tep) != 0) { + error = -ELOOP; + goto err_unlock_nested; + } + } + + error = epoll_mutex_lock(&ep->mtx, nonblock); + if (error) + goto err_unlock_nested; + + return 1; + +err_unlock_nested: + clear_tfile_check_list(ctx); + loop_check_gen++; + mutex_unlock(&epnested_mutex); + return error; +} + +static void ep_ctl_unlock(struct ep_ctl_ctx *ctx, struct eventpoll *ep, + int full_check) +{ + mutex_unlock(&ep->mtx); + if (full_check) { + clear_tfile_check_list(ctx); + loop_check_gen++; + mutex_unlock(&epnested_mutex); + } +} + +int do_epoll_ctl_file(struct file *f, int op, struct epoll_key *tf, + struct epoll_event *epds, bool nonblock) +{ + int error; + int full_check; + struct eventpoll *ep; + struct epitem *epi; + struct ep_ctl_ctx ctx = { + .tfile_check_list = EP_UNACTIVE_PTR, + }; + + /* The target file descriptor must support poll */ + if (!file_can_poll(tf->file)) + return -EPERM; + + /* Check if EPOLLWAKEUP is allowed */ + if (ep_op_has_event(op)) + ep_take_care_of_epollwakeup(epds); + + /* + * The @f file must itself be an eventpoll, and we do not permit + * adding an epoll file descriptor inside itself. + */ + if (f == tf->file || !is_file_epoll(f)) + return -EINVAL; + + /* + * epoll adds to the wakeup queue at EPOLL_CTL_ADD time only, + * so EPOLLEXCLUSIVE is not allowed for a EPOLL_CTL_MOD operation. + * Also, nested exclusive wakeups are not supported. + */ + if (ep_op_has_event(op) && (epds->events & EPOLLEXCLUSIVE)) { + if (op == EPOLL_CTL_MOD) + return -EINVAL; + if (op == EPOLL_CTL_ADD && (is_file_epoll(tf->file) || + (epds->events & ~EPOLLEXCLUSIVE_OK_BITS))) + return -EINVAL; + } + + ep = f->private_data; + + full_check = ep_ctl_lock(&ctx, ep, op, f, tf->file, nonblock); + if (full_check < 0) + return full_check; + + /* + * Look the target up in ep's RB tree. We hold ep->mtx, so the + * item stays valid until we release. + */ + epi = ep_find(ep, tf); + + error = -EINVAL; + switch (op) { + case EPOLL_CTL_ADD: + if (!epi) { + epds->events |= EPOLLERR | EPOLLHUP; + error = ep_insert(&ctx, ep, epds, tf, full_check); + } else + error = -EEXIST; + break; + case EPOLL_CTL_DEL: + if (epi) { + /* + * The eventpoll itself is still alive: the refcount + * can't go to zero here. + */ + ep_remove(ep, epi); + error = 0; + } else { + error = -ENOENT; + } + break; + case EPOLL_CTL_MOD: + if (epi) { + if (!(epi->event.events & EPOLLEXCLUSIVE)) { + epds->events |= EPOLLERR | EPOLLHUP; + error = ep_modify(ep, epi, epds); + } + } else + error = -ENOENT; + break; + } + + ep_ctl_unlock(&ctx, ep, full_check); + return error; +} + +int do_epoll_ctl(int epfd, int op, int fd, struct epoll_event *epds, + bool nonblock) +{ + struct epoll_key efd; + + CLASS(fd, f)(epfd); + if (fd_empty(f)) + return -EBADF; + + /* Get the "struct file *" for the target file */ + CLASS(fd, tf)(fd); + if (fd_empty(tf)) + return -EBADF; + + efd.file = fd_file(tf); + efd.fd = fd; + return do_epoll_ctl_file(fd_file(f), op, &efd, epds, nonblock); +} + +/* + * The following function implements the controller interface for + * the eventpoll file that enables the insertion/removal/change of + * file descriptors inside the interest set. + */ +SYSCALL_DEFINE4(epoll_ctl, int, epfd, int, op, int, fd, + struct epoll_event __user *, event) +{ + struct epoll_event epds; + + if (ep_op_has_event(op) && + copy_from_user(&epds, event, sizeof(struct epoll_event))) + return -EFAULT; + + return do_epoll_ctl(epfd, op, fd, &epds, false); +} + +static int ep_check_params(struct file *file, struct epoll_event __user *evs, + int maxevents) +{ + /* The maximum number of event must be greater than zero */ + if (maxevents <= 0 || maxevents > EP_MAX_EVENTS) + return -EINVAL; + + /* Verify that the area passed by the user is writeable */ + if (!access_ok(evs, maxevents * sizeof(struct epoll_event))) + return -EFAULT; + + /* + * We have to check that the file structure underneath the fd + * the user passed to us _is_ an eventpoll file. + */ + if (!is_file_epoll(file)) + return -EINVAL; + + return 0; +} + +int epoll_sendevents(struct file *file, struct epoll_event __user *events, + int maxevents) +{ + struct eventpoll *ep; + int ret; + + ret = ep_check_params(file, events, maxevents); + if (unlikely(ret)) + return ret; + + ep = file->private_data; + /* + * Racy call, but that's ok - it should get retried based on + * poll readiness anyway. + */ + if (ep_events_available(ep)) + return ep_try_send_events(ep, events, maxevents); + return 0; +} + +/* + * Implement the event wait interface for the eventpoll file. It is the kernel + * part of the user space epoll_wait(2). + */ +static int do_epoll_wait(int epfd, struct epoll_event __user *events, + int maxevents, struct timespec64 *to) +{ + struct eventpoll *ep; + int ret; + + /* Get the "struct file *" for the eventpoll file */ + CLASS(fd, f)(epfd); + if (fd_empty(f)) + return -EBADF; + + ret = ep_check_params(fd_file(f), events, maxevents); + if (unlikely(ret)) + return ret; + + /* + * At this point it is safe to assume that the "private_data" contains + * our own data structure. + */ + ep = fd_file(f)->private_data; + + /* Time to fish for events ... */ + return ep_poll(ep, events, maxevents, to); +} + +SYSCALL_DEFINE4(epoll_wait, int, epfd, struct epoll_event __user *, events, + int, maxevents, int, timeout) +{ + struct timespec64 to; + + return do_epoll_wait(epfd, events, maxevents, + ep_timeout_to_timespec(&to, timeout)); +} + +/* + * Implement the event wait interface for the eventpoll file. It is the kernel + * part of the user space epoll_pwait(2). + */ +static int do_epoll_pwait(int epfd, struct epoll_event __user *events, + int maxevents, struct timespec64 *to, + const sigset_t __user *sigmask, size_t sigsetsize) +{ + int error; + + /* + * If the caller wants a certain signal mask to be set during the wait, + * we apply it here. + */ + error = set_user_sigmask(sigmask, sigsetsize); + if (error) + return error; + + error = do_epoll_wait(epfd, events, maxevents, to); + + restore_saved_sigmask_unless(error == -EINTR); + + return error; +} + +SYSCALL_DEFINE6(epoll_pwait, int, epfd, struct epoll_event __user *, events, + int, maxevents, int, timeout, const sigset_t __user *, sigmask, + size_t, sigsetsize) +{ + struct timespec64 to; + + return do_epoll_pwait(epfd, events, maxevents, + ep_timeout_to_timespec(&to, timeout), + sigmask, sigsetsize); +} + +SYSCALL_DEFINE6(epoll_pwait2, int, epfd, struct epoll_event __user *, events, + int, maxevents, const struct __kernel_timespec __user *, timeout, + const sigset_t __user *, sigmask, size_t, sigsetsize) +{ + struct timespec64 ts, *to = NULL; + + if (timeout) { + if (get_timespec64(&ts, timeout)) + return -EFAULT; + to = &ts; + if (poll_select_set_timeout(to, ts.tv_sec, ts.tv_nsec)) + return -EINVAL; + } + + return do_epoll_pwait(epfd, events, maxevents, to, + sigmask, sigsetsize); +} + +#ifdef CONFIG_KCMP +static struct epitem *ep_find_tfd(struct eventpoll *ep, int tfd, unsigned long toff) +{ + struct rb_node *rbp; + struct epitem *epi; + + for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) { + epi = rb_entry(rbp, struct epitem, rbn); + if (epi->ffd.fd == tfd) { + if (toff == 0) + return epi; + else + toff--; + } + cond_resched(); + } + + return NULL; +} + +struct file *get_epoll_tfile_raw_ptr(struct file *file, int tfd, + unsigned long toff) +{ + struct file *file_raw; + struct eventpoll *ep; + struct epitem *epi; + + if (!is_file_epoll(file)) + return ERR_PTR(-EINVAL); + + ep = file->private_data; + + mutex_lock(&ep->mtx); + epi = ep_find_tfd(ep, tfd, toff); + if (epi) + file_raw = epi->ffd.file; + else + file_raw = ERR_PTR(-ENOENT); + mutex_unlock(&ep->mtx); + + return file_raw; +} +#endif /* CONFIG_KCMP */ + +#ifdef CONFIG_COMPAT +static int do_compat_epoll_pwait(int epfd, struct epoll_event __user *events, + int maxevents, struct timespec64 *timeout, + const compat_sigset_t __user *sigmask, + compat_size_t sigsetsize) +{ + long err; + + /* + * If the caller wants a certain signal mask to be set during the wait, + * we apply it here. + */ + err = set_compat_user_sigmask(sigmask, sigsetsize); + if (err) + return err; + + err = do_epoll_wait(epfd, events, maxevents, timeout); + + restore_saved_sigmask_unless(err == -EINTR); + + return err; +} + +COMPAT_SYSCALL_DEFINE6(epoll_pwait, int, epfd, + struct epoll_event __user *, events, + int, maxevents, int, timeout, + const compat_sigset_t __user *, sigmask, + compat_size_t, sigsetsize) +{ + struct timespec64 to; + + return do_compat_epoll_pwait(epfd, events, maxevents, + ep_timeout_to_timespec(&to, timeout), + sigmask, sigsetsize); +} + +COMPAT_SYSCALL_DEFINE6(epoll_pwait2, int, epfd, + struct epoll_event __user *, events, + int, maxevents, + const struct __kernel_timespec __user *, timeout, + const compat_sigset_t __user *, sigmask, + compat_size_t, sigsetsize) +{ + struct timespec64 ts, *to = NULL; + + if (timeout) { + if (get_timespec64(&ts, timeout)) + return -EFAULT; + to = &ts; + if (poll_select_set_timeout(to, ts.tv_sec, ts.tv_nsec)) + return -EINVAL; + } + + return do_compat_epoll_pwait(epfd, events, maxevents, to, + sigmask, sigsetsize); +} + +#endif + +static int __init eventpoll_init(void) +{ + struct sysinfo si; + + si_meminfo(&si); + /* + * Allows top 4% of lomem to be allocated for epoll watches (per user). + */ + max_user_watches = (((si.totalram - si.totalhigh) / 25) << PAGE_SHIFT) / + EP_ITEM_COST; + BUG_ON(max_user_watches < 0); + + /* + * We can have many thousands of epitems, so prevent this from + * using an extra cache line on 64-bit (and smaller) CPUs + */ + BUILD_BUG_ON(sizeof(void *) <= 8 && sizeof(struct epitem) > 128); + + /* Allocates slab cache used to allocate "struct epitem" items */ + epi_cache = kmem_cache_create("eventpoll_epi", sizeof(struct epitem), + 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL); + + /* Allocates slab cache used to allocate "struct eppoll_entry" */ + pwq_cache = kmem_cache_create("eventpoll_pwq", + sizeof(struct eppoll_entry), 0, SLAB_PANIC|SLAB_ACCOUNT, NULL); + epoll_sysctls_init(); + + ephead_cache = kmem_cache_create("ep_head", + sizeof(struct epitems_head), 0, SLAB_PANIC|SLAB_ACCOUNT, NULL); + + return 0; +} +fs_initcall(eventpoll_init); diff --git a/fs/exec.c b/fs/exec.c new file mode 100644 index 000000000..745f6eb52 --- /dev/null +++ b/fs/exec.c @@ -0,0 +1,2098 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/exec.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + */ + +/* + * #!-checking implemented by tytso. + */ +/* + * Demand-loading implemented 01.12.91 - no need to read anything but + * the header into memory. The inode of the executable is put into + * "current->executable", and page faults do the actual loading. Clean. + * + * Once more I can proudly say that linux stood up to being changed: it + * was less than 2 hours work to get demand-loading completely implemented. + * + * Demand loading changed July 1993 by Eric Youngdale. Use mmap instead, + * current->executable is only used by the procfs. This allows a dispatch + * table to check for several different types of binary formats. We keep + * trying until we recognize the file or we run out of supported binary + * formats. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include "internal.h" + +#include + +/* For vma exec functions. */ +#include "../mm/internal.h" + +static int bprm_creds_from_file(struct linux_binprm *bprm); + +int suid_dumpable = 0; + +static LIST_HEAD(formats); +static DEFINE_RWLOCK(binfmt_lock); + +void __register_binfmt(struct linux_binfmt * fmt, int insert) +{ + write_lock(&binfmt_lock); + insert ? list_add(&fmt->lh, &formats) : + list_add_tail(&fmt->lh, &formats); + write_unlock(&binfmt_lock); +} + +EXPORT_SYMBOL(__register_binfmt); + +void unregister_binfmt(struct linux_binfmt * fmt) +{ + write_lock(&binfmt_lock); + list_del(&fmt->lh); + write_unlock(&binfmt_lock); +} + +EXPORT_SYMBOL(unregister_binfmt); + +static inline void put_binfmt(struct linux_binfmt * fmt) +{ + module_put(fmt->module); +} + +bool path_noexec(const struct path *path) +{ + /* If it's an anonymous inode make sure that we catch any shenanigans. */ + VFS_WARN_ON_ONCE(IS_ANON_FILE(d_inode(path->dentry)) && + !(path->mnt->mnt_sb->s_iflags & SB_I_NOEXEC)); + return (path->mnt->mnt_flags & MNT_NOEXEC) || + (path->mnt->mnt_sb->s_iflags & SB_I_NOEXEC); +} + +#ifdef CONFIG_MMU +/* + * The nascent bprm->mm is not visible until exec_mmap() but it can + * use a lot of memory, account these pages in current->mm temporary + * for oom_badness()->get_mm_rss(). Once exec succeeds or fails, we + * change the counter back via acct_arg_size(0). + */ +static void acct_arg_size(struct linux_binprm *bprm, unsigned long pages) +{ + struct mm_struct *mm = current->mm; + long diff = (long)(pages - bprm->vma_pages); + + if (!mm || !diff) + return; + + bprm->vma_pages = pages; + add_mm_counter(mm, MM_ANONPAGES, diff); +} + +static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos, + int write) +{ + struct page *page; + struct vm_area_struct *vma = bprm->vma; + struct mm_struct *mm = bprm->mm; + int ret; + + /* + * Avoid relying on expanding the stack down in GUP (which + * does not work for STACK_GROWSUP anyway), and just do it + * ahead of time. + */ + if (!mmap_read_lock_maybe_expand(mm, vma, pos, write)) + return NULL; + + /* + * We are doing an exec(). 'current' is the process + * doing the exec and 'mm' is the new process's mm. + */ + ret = get_user_pages_remote(mm, pos, 1, + write ? FOLL_WRITE : 0, + &page, NULL); + mmap_read_unlock(mm); + if (ret <= 0) + return NULL; + + if (write) + acct_arg_size(bprm, vma_pages(vma)); + + return page; +} + +static void put_arg_page(struct page *page) +{ + put_page(page); +} + +static void free_arg_pages(struct linux_binprm *bprm) +{ +} + +static void flush_arg_page(struct linux_binprm *bprm, unsigned long pos, + struct page *page) +{ + flush_cache_page(bprm->vma, pos, page_to_pfn(page)); +} + +static bool valid_arg_len(struct linux_binprm *bprm, long len) +{ + return len <= MAX_ARG_STRLEN; +} + +#else + +static inline void acct_arg_size(struct linux_binprm *bprm, unsigned long pages) +{ +} + +static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos, + int write) +{ + struct page *page; + + page = bprm->page[pos / PAGE_SIZE]; + if (!page && write) { + page = alloc_page(GFP_HIGHUSER|__GFP_ZERO); + if (!page) + return NULL; + bprm->page[pos / PAGE_SIZE] = page; + } + + return page; +} + +static void put_arg_page(struct page *page) +{ +} + +static void free_arg_page(struct linux_binprm *bprm, int i) +{ + if (bprm->page[i]) { + __free_page(bprm->page[i]); + bprm->page[i] = NULL; + } +} + +static void free_arg_pages(struct linux_binprm *bprm) +{ + int i; + + for (i = 0; i < MAX_ARG_PAGES; i++) + free_arg_page(bprm, i); +} + +static void flush_arg_page(struct linux_binprm *bprm, unsigned long pos, + struct page *page) +{ +} + +static bool valid_arg_len(struct linux_binprm *bprm, long len) +{ + return len <= bprm->p; +} + +#endif /* CONFIG_MMU */ + +/* + * Create a new mm_struct and populate it with a temporary stack + * vm_area_struct. We don't have enough context at this point to set the stack + * flags, permissions, and offset, so we use temporary values. We'll update + * them later in setup_arg_pages(). + */ +static int bprm_mm_init(struct linux_binprm *bprm) +{ + int err; + struct mm_struct *mm = NULL; + + bprm->mm = mm = mm_alloc(); + err = -ENOMEM; + if (!mm) + goto err; + + /* Staged for would_dump() narrowing; consumed by begin_new_exec(). */ + bprm->user_ns = get_user_ns(current_user_ns()); + + /* Save current stack limit for all calculations made during exec. */ + task_lock(current->group_leader); + bprm->rlim_stack = current->signal->rlim[RLIMIT_STACK]; + task_unlock(current->group_leader); + +#ifndef CONFIG_MMU + bprm->p = PAGE_SIZE * MAX_ARG_PAGES - sizeof(void *); +#else + err = create_init_stack_vma(bprm->mm, &bprm->vma, &bprm->p); + if (err) + goto err; +#endif + + return 0; + +err: + if (mm) { + bprm->mm = NULL; + mmdrop(mm); + } + + return err; +} + +struct user_arg_ptr { +#ifdef CONFIG_COMPAT + bool is_compat; +#endif + union { + const char __user *const __user *native; +#ifdef CONFIG_COMPAT + const compat_uptr_t __user *compat; +#endif + } ptr; +}; + +static const char __user *get_user_arg_ptr(struct user_arg_ptr argv, int nr) +{ + const char __user *native; + +#ifdef CONFIG_COMPAT + if (unlikely(argv.is_compat)) { + compat_uptr_t compat; + + if (get_user(compat, argv.ptr.compat + nr)) + return ERR_PTR(-EFAULT); + + return compat_ptr(compat); + } +#endif + + if (get_user(native, argv.ptr.native + nr)) + return ERR_PTR(-EFAULT); + + return native; +} + +/* + * count() counts the number of strings in array ARGV. + */ +static int count(struct user_arg_ptr argv, int max) +{ + int i = 0; + + if (argv.ptr.native != NULL) { + for (;;) { + const char __user *p = get_user_arg_ptr(argv, i); + + if (!p) + break; + + if (IS_ERR(p)) + return -EFAULT; + + if (i >= max) + return -E2BIG; + ++i; + + if (fatal_signal_pending(current)) + return -ERESTARTNOHAND; + cond_resched(); + } + } + return i; +} + +static int count_strings_kernel(const char *const *argv) +{ + int i; + + if (!argv) + return 0; + + for (i = 0; argv[i]; ++i) { + if (i >= MAX_ARG_STRINGS) + return -E2BIG; + if (fatal_signal_pending(current)) + return -ERESTARTNOHAND; + cond_resched(); + } + return i; +} + +static inline int bprm_set_stack_limit(struct linux_binprm *bprm, + unsigned long limit) +{ +#ifdef CONFIG_MMU + /* Avoid a pathological bprm->p. */ + if (bprm->p < limit) + return -E2BIG; + bprm->argmin = bprm->p - limit; +#endif + return 0; +} +static inline bool bprm_hit_stack_limit(struct linux_binprm *bprm) +{ +#ifdef CONFIG_MMU + return bprm->p < bprm->argmin; +#else + return false; +#endif +} + +/* + * Calculate bprm->argmin from: + * - _STK_LIM + * - ARG_MAX + * - bprm->rlim_stack.rlim_cur + * - bprm->argc + * - bprm->envc + * - bprm->p + */ +static int bprm_stack_limits(struct linux_binprm *bprm) +{ + unsigned long limit, ptr_size; + + /* + * Limit to 1/4 of the max stack size or 3/4 of _STK_LIM + * (whichever is smaller) for the argv+env strings. + * This ensures that: + * - the remaining binfmt code will not run out of stack space, + * - the program will have a reasonable amount of stack left + * to work from. + */ + limit = _STK_LIM / 4 * 3; + limit = min(limit, bprm->rlim_stack.rlim_cur / 4); + /* + * We've historically supported up to 32 pages (ARG_MAX) + * of argument strings even with small stacks + */ + limit = max_t(unsigned long, limit, ARG_MAX); + /* Reject totally pathological counts. */ + if (bprm->argc < 0 || bprm->envc < 0) + return -E2BIG; + /* + * We must account for the size of all the argv and envp pointers to + * the argv and envp strings, since they will also take up space in + * the stack. They aren't stored until much later when we can't + * signal to the parent that the child has run out of stack space. + * Instead, calculate it here so it's possible to fail gracefully. + * + * In the case of argc = 0, make sure there is space for adding a + * empty string (which will bump argc to 1), to ensure confused + * userspace programs don't start processing from argv[1], thinking + * argc can never be 0, to keep them from walking envp by accident. + * See do_execveat_common(). + */ + if (check_add_overflow(max(bprm->argc, 1), bprm->envc, &ptr_size) || + check_mul_overflow(ptr_size, sizeof(void *), &ptr_size)) + return -E2BIG; + if (limit <= ptr_size) + return -E2BIG; + limit -= ptr_size; + + return bprm_set_stack_limit(bprm, limit); +} + +/* + * 'copy_strings()' copies argument/environment strings from the old + * processes's memory to the new process's stack. The call to get_user_pages() + * ensures the destination page is created and not swapped out. + */ +static int copy_strings(int argc, struct user_arg_ptr argv, + struct linux_binprm *bprm) +{ + struct page *kmapped_page = NULL; + char *kaddr = NULL; + unsigned long kpos = 0; + int ret; + + while (argc-- > 0) { + const char __user *str; + int len; + unsigned long pos; + + ret = -EFAULT; + str = get_user_arg_ptr(argv, argc); + if (IS_ERR(str)) + goto out; + + len = strnlen_user(str, MAX_ARG_STRLEN); + if (!len) + goto out; + + ret = -E2BIG; + if (!valid_arg_len(bprm, len)) + goto out; + + /* We're going to work our way backwards. */ + pos = bprm->p; + str += len; + bprm->p -= len; + if (bprm_hit_stack_limit(bprm)) + goto out; + + while (len > 0) { + int offset, bytes_to_copy; + + if (fatal_signal_pending(current)) { + ret = -ERESTARTNOHAND; + goto out; + } + cond_resched(); + + offset = pos % PAGE_SIZE; + if (offset == 0) + offset = PAGE_SIZE; + + bytes_to_copy = offset; + if (bytes_to_copy > len) + bytes_to_copy = len; + + offset -= bytes_to_copy; + pos -= bytes_to_copy; + str -= bytes_to_copy; + len -= bytes_to_copy; + + if (!kmapped_page || kpos != (pos & PAGE_MASK)) { + struct page *page; + + page = get_arg_page(bprm, pos, 1); + if (!page) { + ret = -E2BIG; + goto out; + } + + if (kmapped_page) { + flush_dcache_page(kmapped_page); + kunmap_local(kaddr); + put_arg_page(kmapped_page); + } + kmapped_page = page; + kaddr = kmap_local_page(kmapped_page); + kpos = pos & PAGE_MASK; + flush_arg_page(bprm, kpos, kmapped_page); + } + if (copy_from_user(kaddr+offset, str, bytes_to_copy)) { + ret = -EFAULT; + goto out; + } + } + } + ret = 0; +out: + if (kmapped_page) { + flush_dcache_page(kmapped_page); + kunmap_local(kaddr); + put_arg_page(kmapped_page); + } + return ret; +} + +/* + * Copy and argument/environment string from the kernel to the processes stack. + */ +int copy_string_kernel(const char *arg, struct linux_binprm *bprm) +{ + int len = strnlen(arg, MAX_ARG_STRLEN) + 1 /* terminating NUL */; + unsigned long pos = bprm->p; + + if (len == 0) + return -EFAULT; + if (!valid_arg_len(bprm, len)) + return -E2BIG; + + /* We're going to work our way backwards. */ + arg += len; + bprm->p -= len; + if (bprm_hit_stack_limit(bprm)) + return -E2BIG; + + while (len > 0) { + unsigned int bytes_to_copy = min(len, + min_not_zero(offset_in_page(pos), PAGE_SIZE)); + struct page *page; + + pos -= bytes_to_copy; + arg -= bytes_to_copy; + len -= bytes_to_copy; + + page = get_arg_page(bprm, pos, 1); + if (!page) + return -E2BIG; + flush_arg_page(bprm, pos & PAGE_MASK, page); + memcpy_to_page(page, offset_in_page(pos), arg, bytes_to_copy); + put_arg_page(page); + } + + return 0; +} +EXPORT_SYMBOL(copy_string_kernel); + +static int copy_strings_kernel(int argc, const char *const *argv, + struct linux_binprm *bprm) +{ + while (argc-- > 0) { + int ret = copy_string_kernel(argv[argc], bprm); + if (ret < 0) + return ret; + if (fatal_signal_pending(current)) + return -ERESTARTNOHAND; + cond_resched(); + } + return 0; +} + +#ifdef CONFIG_MMU + +/* + * Finalizes the stack vm_area_struct. The flags and permissions are updated, + * the stack is optionally relocated, and some extra space is added. + */ +int setup_arg_pages(struct linux_binprm *bprm, + unsigned long stack_top, + int executable_stack) +{ + int ret; + unsigned long stack_shift; + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma = bprm->vma; + struct vm_area_struct *prev = NULL; + vm_flags_t vm_flags; + unsigned long stack_base; + unsigned long stack_size; + unsigned long stack_expand; + unsigned long rlim_stack; + struct mmu_gather tlb; + struct vma_iterator vmi; + +#ifdef CONFIG_STACK_GROWSUP + /* Limit stack size */ + stack_base = bprm->rlim_stack.rlim_max; + + stack_base = calc_max_stack_size(stack_base); + + /* Add space for stack randomization. */ + if (current->flags & PF_RANDOMIZE) + stack_base += (STACK_RND_MASK << PAGE_SHIFT); + + /* Make sure we didn't let the argument array grow too large. */ + if (vma->vm_end - vma->vm_start > stack_base) + return -ENOMEM; + + stack_base = PAGE_ALIGN(stack_top - stack_base); + + stack_shift = vma->vm_start - stack_base; + mm->arg_start = bprm->p - stack_shift; + bprm->p = vma->vm_end - stack_shift; +#else + stack_top = arch_align_stack(stack_top); + stack_top = PAGE_ALIGN(stack_top); + + if (unlikely(stack_top < mmap_min_addr) || + unlikely(vma->vm_end - vma->vm_start >= stack_top - mmap_min_addr)) + return -ENOMEM; + + stack_shift = vma->vm_end - stack_top; + + bprm->p -= stack_shift; + mm->arg_start = bprm->p; +#endif + + bprm->exec -= stack_shift; + + if (mmap_write_lock_killable(mm)) + return -EINTR; + + vm_flags = VM_STACK_FLAGS; + + /* + * Adjust stack execute permissions; explicitly enable for + * EXSTACK_ENABLE_X, disable for EXSTACK_DISABLE_X and leave alone + * (arch default) otherwise. + */ + if (unlikely(executable_stack == EXSTACK_ENABLE_X)) + vm_flags |= VM_EXEC; + else if (executable_stack == EXSTACK_DISABLE_X) + vm_flags &= ~VM_EXEC; + vm_flags |= mm->def_flags; + vm_flags |= VM_STACK_INCOMPLETE_SETUP; + + vma_iter_init(&vmi, mm, vma->vm_start); + + tlb_gather_mmu(&tlb, mm); + ret = mprotect_fixup(&vmi, &tlb, vma, &prev, vma->vm_start, vma->vm_end, + vm_flags); + tlb_finish_mmu(&tlb); + + if (ret) + goto out_unlock; + BUG_ON(prev != vma); + + if (unlikely(vm_flags & VM_EXEC)) { + pr_warn_once("process '%pD4' started with executable stack\n", + bprm->file); + } + + /* Move stack pages down in memory. */ + if (stack_shift) { + /* + * During bprm_mm_init(), we create a temporary stack at STACK_TOP_MAX. Once + * the binfmt code determines where the new stack should reside, we shift it to + * its final location. + */ + ret = relocate_vma_down(vma, stack_shift); + if (ret) + goto out_unlock; + } + + /* mprotect_fixup is overkill to remove the temporary stack flags */ + vm_flags_clear(vma, VM_STACK_INCOMPLETE_SETUP); + + stack_expand = 131072UL; /* randomly 32*4k (or 2*64k) pages */ + stack_size = vma->vm_end - vma->vm_start; + /* + * Align this down to a page boundary as expand_stack + * will align it up. + */ + rlim_stack = bprm->rlim_stack.rlim_cur & PAGE_MASK; + + stack_expand = min(rlim_stack, stack_size + stack_expand); + +#ifdef CONFIG_STACK_GROWSUP + stack_base = vma->vm_start + stack_expand; +#else + stack_base = vma->vm_end - stack_expand; +#endif + current->mm->start_stack = bprm->p; + ret = expand_stack_locked(vma, stack_base); + if (ret) + ret = -EFAULT; + +out_unlock: + mmap_write_unlock(mm); + return ret; +} +EXPORT_SYMBOL(setup_arg_pages); + +#else + +/* + * Transfer the program arguments and environment from the holding pages + * onto the stack. The provided stack pointer is adjusted accordingly. + */ +int transfer_args_to_stack(struct linux_binprm *bprm, + unsigned long *sp_location) +{ + unsigned long index, stop, sp; + int ret = 0; + + stop = bprm->p >> PAGE_SHIFT; + sp = *sp_location; + + for (index = MAX_ARG_PAGES; index-- > stop; ) { + unsigned int offset = index == stop ? bprm->p & ~PAGE_MASK : 0; + char *src = kmap_local_page(bprm->page[index]) + offset; + sp -= PAGE_SIZE - offset; + if (copy_to_user((void *) sp, src, PAGE_SIZE - offset) != 0) + ret = -EFAULT; + kunmap_local(src); + if (ret) + goto out; + } + + bprm->exec += *sp_location - MAX_ARG_PAGES * PAGE_SIZE; + *sp_location = sp; + +out: + return ret; +} +EXPORT_SYMBOL(transfer_args_to_stack); + +#endif /* CONFIG_MMU */ + +/* + * On success, caller must call do_close_execat() on the returned + * struct file to close it. + */ +static struct file *do_open_execat(int fd, struct filename *name, int flags) +{ + int err; + struct file *file __free(fput) = NULL; + struct open_flags open_exec_flags = { + .open_flag = O_LARGEFILE | O_RDONLY | __FMODE_EXEC, + .acc_mode = MAY_EXEC, + .intent = LOOKUP_OPEN, + .lookup_flags = LOOKUP_FOLLOW, + }; + + if ((flags & + ~(AT_SYMLINK_NOFOLLOW | AT_EMPTY_PATH | AT_EXECVE_CHECK)) != 0) + return ERR_PTR(-EINVAL); + if (flags & AT_SYMLINK_NOFOLLOW) + open_exec_flags.lookup_flags &= ~LOOKUP_FOLLOW; + + file = do_file_open(fd, name, &open_exec_flags); + if (IS_ERR(file)) + return file; + + if (path_noexec(&file->f_path)) + return ERR_PTR(-EACCES); + + /* + * In the past the regular type check was here. It moved to may_open() in + * 633fb6ac3980 ("exec: move S_ISREG() check earlier"). Since then it is + * an invariant that all non-regular files error out before we get here. + */ + if (WARN_ON_ONCE(!S_ISREG(file_inode(file)->i_mode))) + return ERR_PTR(-EACCES); + + err = exe_file_deny_write_access(file); + if (err) + return ERR_PTR(err); + + return no_free_ptr(file); +} + +/** + * open_exec - Open a path name for execution + * + * @name: path name to open with the intent of executing it. + * + * Returns ERR_PTR on failure or allocated struct file on success. + * + * As this is a wrapper for the internal do_open_execat(), callers + * must call exe_file_allow_write_access() before fput() on release. Also see + * do_close_execat(). + */ +struct file *open_exec(const char *name) +{ + CLASS(filename_kernel, filename)(name); + return do_open_execat(AT_FDCWD, filename, 0); +} +EXPORT_SYMBOL(open_exec); + +#if defined(CONFIG_BINFMT_FLAT) || defined(CONFIG_BINFMT_ELF_FDPIC) +ssize_t read_code(struct file *file, unsigned long addr, loff_t pos, size_t len) +{ + ssize_t res = vfs_read(file, (void __user *)addr, len, &pos); + if (res > 0) + flush_icache_user_range(addr, addr + len); + return res; +} +EXPORT_SYMBOL(read_code); +#endif + +/* + * Maps the mm_struct mm into the current task struct. + * On success, this function returns with exec_update_lock + * held for writing. The replaced address space is stashed in + * bprm->old_mm for setup_new_exec() to release outside the lock. + */ +static int exec_mmap(struct linux_binprm *bprm) +{ + struct task_exec_state *exec_state __free(put_task_exec_state) = NULL; + struct mm_struct *mm = bprm->mm; + struct task_struct *tsk; + struct mm_struct *old_mm, *active_mm; + int ret; + + exec_state = alloc_task_exec_state(bprm->user_ns); + if (!exec_state) + return -ENOMEM; + + /* Notify parent that we're no longer interested in the old VM */ + tsk = current; + old_mm = current->mm; + /* Clean up futexes and release the mm */ + mm_exit_exec_release(tsk, old_mm); + + ret = down_write_killable(&tsk->signal->exec_update_lock); + if (ret) + return ret; + + if (old_mm) { + /* + * If there is a pending fatal signal perhaps a signal + * whose default action is to create a coredump get + * out and die instead of going through with the exec. + */ + ret = mmap_read_lock_killable(old_mm); + if (ret) { + up_write(&tsk->signal->exec_update_lock); + return ret; + } + } + + task_lock(tsk); + membarrier_exec_mmap(mm); + + local_irq_disable(); + active_mm = tsk->active_mm; + tsk->active_mm = mm; + tsk->mm = mm; + mm_init_cid(mm, tsk); + exec_state = task_exec_state_replace(tsk, exec_state); + /* + * This prevents preemption while active_mm is being loaded and + * it and mm are being updated, which could cause problems for + * lazy tlb mm refcounting when these are updated by context + * switches. Not all architectures can handle irqs off over + * activate_mm yet. + */ + if (!IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM)) + local_irq_enable(); + activate_mm(active_mm, mm); + if (IS_ENABLED(CONFIG_ARCH_WANT_IRQS_OFF_ACTIVATE_MM)) + local_irq_enable(); + lru_gen_add_mm(mm); + task_unlock(tsk); + lru_gen_use_mm(mm); + if (old_mm) { + mmap_read_unlock(old_mm); + BUG_ON(active_mm != old_mm); + /* Defer teardown to setup_new_exec(), outside the exec locks. */ + bprm->old_mm = old_mm; + } else { + mmdrop_lazy_tlb(active_mm); + } + futex_exec_done(tsk); + return 0; +} + +/* Release the address space replaced by exec, outside the exec locks. */ +static void exec_mm_put_old(struct mm_struct *old_mm) +{ + setmax_mm_hiwater_rss(¤t->signal->maxrss, old_mm); + mm_update_next_owner(old_mm); + mmput(old_mm); +} + +static int de_thread(struct task_struct *tsk) +{ + struct signal_struct *sig = tsk->signal; + struct sighand_struct *oldsighand = tsk->sighand; + spinlock_t *lock = &oldsighand->siglock; + + if (thread_group_empty(tsk)) + goto no_thread_group; + + /* + * Kill all other threads in the thread group. + */ + spin_lock_irq(lock); + if ((sig->flags & SIGNAL_GROUP_EXIT) || sig->group_exec_task) { + /* + * Another group action in progress, just + * return so that the signal is processed. + */ + spin_unlock_irq(lock); + return -EAGAIN; + } + + sig->group_exec_task = tsk; + sig->notify_count = zap_other_threads(tsk); + if (!thread_group_leader(tsk)) + sig->notify_count--; + + while (sig->notify_count) { + __set_current_state(TASK_KILLABLE); + spin_unlock_irq(lock); + schedule(); + if (__fatal_signal_pending(tsk)) + goto killed; + spin_lock_irq(lock); + } + spin_unlock_irq(lock); + + /* + * At this point all other threads have exited, all we have to + * do is to wait for the thread group leader to become inactive, + * and to assume its PID: + */ + if (!thread_group_leader(tsk)) { + struct task_struct *leader = tsk->group_leader; + + for (;;) { + cgroup_threadgroup_change_begin(tsk); + write_lock_irq(&tasklist_lock); + /* + * Do this under tasklist_lock to ensure that + * exit_notify() can't miss ->group_exec_task + */ + sig->notify_count = -1; + if (likely(leader->exit_state)) + break; + __set_current_state(TASK_KILLABLE); + write_unlock_irq(&tasklist_lock); + cgroup_threadgroup_change_end(tsk); + schedule(); + if (__fatal_signal_pending(tsk)) + goto killed; + } + + /* + * The only record we have of the real-time age of a + * process, regardless of execs it's done, is start_time. + * All the past CPU time is accumulated in signal_struct + * from sister threads now dead. But in this non-leader + * exec, nothing survives from the original leader thread, + * whose birth marks the true age of this process now. + * When we take on its identity by switching to its PID, we + * also take its birthdate (always earlier than our own). + */ + tsk->start_time = leader->start_time; + tsk->start_boottime = leader->start_boottime; + + BUG_ON(!same_thread_group(leader, tsk)); + /* + * An exec() starts a new thread group with the + * TGID of the previous thread group. Rehash the + * two threads with a switched PID, and release + * the former thread group leader: + */ + + /* Become a process group leader with the old leader's pid. + * The old leader becomes a thread of the this thread group. + */ + exchange_tids(tsk, leader); + transfer_pid(leader, tsk, PIDTYPE_TGID); + transfer_pid(leader, tsk, PIDTYPE_PGID); + transfer_pid(leader, tsk, PIDTYPE_SID); + + list_replace_rcu(&leader->tasks, &tsk->tasks); + list_replace_init(&leader->sibling, &tsk->sibling); + + tsk->group_leader = tsk; + leader->group_leader = tsk; + + tsk->exit_signal = SIGCHLD; + leader->exit_signal = -1; + + BUG_ON(leader->exit_state != EXIT_ZOMBIE); + leader->exit_state = EXIT_DEAD; + /* + * We are going to release_task()->ptrace_unlink() silently, + * the tracer can sleep in do_wait(). EXIT_DEAD guarantees + * the tracer won't block again waiting for this thread. + */ + if (unlikely(leader->ptrace)) + __wake_up_parent(leader, leader->parent); + write_unlock_irq(&tasklist_lock); + cgroup_threadgroup_change_end(tsk); + + release_task(leader); + } + + sig->group_exec_task = NULL; + sig->notify_count = 0; + +no_thread_group: + /* we have changed execution domain */ + tsk->exit_signal = SIGCHLD; + + BUG_ON(!thread_group_leader(tsk)); + return 0; + +killed: + /* protects against exit_notify() and __exit_signal() */ + read_lock(&tasklist_lock); + sig->group_exec_task = NULL; + sig->notify_count = 0; + read_unlock(&tasklist_lock); + return -EAGAIN; +} + + +/* + * This function makes sure the current process has its own signal table, + * so that flush_signal_handlers can later reset the handlers without + * disturbing other processes. (Other processes might share the signal + * table via the CLONE_SIGHAND option to clone().) + */ +static int unshare_sighand(struct task_struct *me) +{ + struct sighand_struct *oldsighand = me->sighand; + + if (refcount_read(&oldsighand->count) != 1) { + struct sighand_struct *newsighand; + /* + * This ->sighand is shared with the CLONE_SIGHAND + * but not CLONE_THREAD task, switch to the new one. + */ + newsighand = kmem_cache_alloc(sighand_cachep, GFP_KERNEL); + if (!newsighand) + return -ENOMEM; + + refcount_set(&newsighand->count, 1); + + write_lock_irq(&tasklist_lock); + spin_lock(&oldsighand->siglock); + memcpy(newsighand->action, oldsighand->action, + sizeof(newsighand->action)); + rcu_assign_pointer(me->sighand, newsighand); + spin_unlock(&oldsighand->siglock); + write_unlock_irq(&tasklist_lock); + + __cleanup_sighand(oldsighand); + } + return 0; +} + +/* + * This is unlocked -- the string will always be NUL-terminated, but + * may show overlapping contents if racing concurrent reads. + */ +void __set_task_comm(struct task_struct *tsk, const char *buf, bool exec) +{ + size_t len = strnlen(buf, sizeof(tsk->comm) - 1); + + trace_task_rename(tsk, buf); + memcpy(tsk->comm, buf, len); + memset(&tsk->comm[len], 0, sizeof(tsk->comm) - len); + perf_event_comm(tsk, exec); +} + +/* + * The file the process presents as: its exe link and comm. A transparent + * dispatch presents as the binary, which is bprm->executable. + */ +static struct file *bprm_identity_file(const struct linux_binprm *bprm) +{ + if (bprm->interp_flags & BINPRM_FLAGS_TRANSPARENT_INTERP) + return bprm->executable; + return bprm->file; +} + +/* + * Calling this is the point of no return. None of the failures will be + * seen by userspace since either the process is already taking a fatal + * signal (via de_thread() or coredump), or will have SEGV raised + * (after exec_mmap()) by search_binary_handler (see below). + */ +int begin_new_exec(struct linux_binprm * bprm) +{ + struct task_struct *me = current; + int retval; + + /* A pending PT_INTERP substitution this format cannot consume. */ + if (bprm->loader) + return -ENOEXEC; + + /* Once we are committed compute the creds */ + retval = bprm_creds_from_file(bprm); + if (retval) + return retval; + + /* + * This tracepoint marks the point before flushing the old exec where + * the current task is still unchanged, but errors are fatal (point of + * no return). The later "sched_process_exec" tracepoint is called after + * the current task has successfully switched to the new exec. + */ + trace_sched_prepare_exec(current, bprm); + + /* + * Ensure all future errors are fatal. + */ + bprm->point_of_no_return = true; + + /* Make this the only thread in the thread group */ + retval = de_thread(me); + if (retval) + goto out; + /* see the comment in check_unsafe_exec() */ + current->fs->in_exec = 0; + /* + * Cancel any io_uring activity across execve + */ + io_uring_task_cancel(); + + /* Ensure the files table is not shared. */ + retval = unshare_files(); + if (retval) + goto out; + + /* + * Must be called _before_ exec_mmap() as bprm->mm is + * not visible until then. Doing it here also ensures + * we don't race against replace_mm_exe_file(). + */ + retval = set_mm_exe_file(bprm->mm, bprm_identity_file(bprm)); + if (retval) + goto out; + + /* If the binary is not readable then enforce mm->dumpable=0 */ + would_dump(bprm, bprm->file); + if (bprm->have_execfd) + would_dump(bprm, bprm->executable); + + /* + * Release all of the old mmap stuff + */ + acct_arg_size(bprm, 0); + retval = exec_mmap(bprm); + if (retval) + goto out; + + bprm->mm = NULL; + + retval = exec_task_namespaces(); + if (retval) + goto out_unlock; + +#ifdef CONFIG_POSIX_TIMERS + spin_lock_irq(&me->sighand->siglock); + posix_cpu_timers_exit(me); + spin_unlock_irq(&me->sighand->siglock); + exit_itimers(me); + flush_itimer_signals(); +#endif + + /* + * Make the signal table private. + */ + retval = unshare_sighand(me); + if (retval) + goto out_unlock; + + me->flags &= ~(PF_RANDOMIZE | PF_FORKNOEXEC | + PF_NOFREEZE | PF_NO_SETAFFINITY); + flush_thread(); + me->personality &= ~bprm->per_clear; + + clear_syscall_work_syscall_user_dispatch(me); + + /* + * We have to apply CLOEXEC before we change whether the process is + * dumpable (in setup_new_exec) to avoid a race with a process in userspace + * trying to access the should-be-closed file descriptors of a process + * undergoing exec(2). + */ + do_close_on_exec(me->files); + + if (bprm->secureexec) { + /* Make sure parent cannot signal privileged process. */ + me->pdeath_signal = 0; + + /* + * For secureexec, reset the stack limit to sane default to + * avoid bad behavior from the prior rlimits. This has to + * happen before arch_pick_mmap_layout(), which examines + * RLIMIT_STACK, but after the point of no return to avoid + * needing to clean up the change on failure. + */ + if (bprm->rlim_stack.rlim_cur > _STK_LIM) + bprm->rlim_stack.rlim_cur = _STK_LIM; + } + + me->sas_ss_sp = me->sas_ss_size = 0; + + /* + * Figure out dumpability. Note that this checking only of current + * is wrong, but userspace depends on it. This should be testing + * bprm->secureexec instead. + */ + if (bprm->interp_flags & BINPRM_FLAGS_ENFORCE_NONDUMP || + !(uid_eq(current_euid(), current_uid()) && + gid_eq(current_egid(), current_gid()))) + task_exec_state_set_dumpable(suid_dumpable); + else + task_exec_state_set_dumpable(TASK_DUMPABLE_OWNER); + + perf_event_exec(); + + /* + * If the original filename was empty, alloc_bprm() made up a path + * that will probably not be useful to admins running ps or similar. + * Let's fix it up to be something reasonable. + */ + if (bprm->comm_from_dentry) { + struct file *comm_file = bprm_identity_file(bprm); + + /* + * Hold RCU lock to keep the name from being freed behind our back. + * Use acquire semantics to make sure the terminating NUL from + * __d_alloc() is seen. + * + * Note, we're deliberately sloppy here. We don't need to care about + * detecting a concurrent rename and just want a terminated name. + */ + rcu_read_lock(); + __set_task_comm(me, smp_load_acquire(&comm_file->f_path.dentry->d_name.name), + true); + rcu_read_unlock(); + } else { + __set_task_comm(me, kbasename(bprm->filename), true); + } + + /* An exec changes our domain. We are no longer part of the thread + group */ + WRITE_ONCE(me->self_exec_id, me->self_exec_id + 1); + flush_signal_handlers(me, 0); + + retval = set_cred_ucounts(bprm->cred); + if (retval < 0) + goto out_unlock; + + /* + * install the new credentials for this executable + */ + security_bprm_committing_creds(bprm); + + commit_creds(bprm->cred); + bprm->cred = NULL; + + /* + * Disable monitoring for regular users + * when executing setuid binaries. Must + * wait until new credentials are committed + * by commit_creds() above + */ + if (task_exec_state_get_dumpable(me) != TASK_DUMPABLE_OWNER) + perf_event_exit_task(me); + /* + * cred_guard_mutex must be held at least to this point to prevent + * ptrace_attach() from altering our determination of the task's + * credentials; any time after this it may be unlocked. + */ + security_bprm_committed_creds(bprm); + + /* Pass the opened binary to the interpreter. */ + if (bprm->have_execfd) { + struct file *executable = bprm->executable; + + /* mm->exe_file carries its own write denial now so drop it. */ + exe_file_allow_write_access(executable); + bprm->executable = NULL; + retval = FD_ADD(0, executable); + if (retval < 0) { + /* The reference was not consumed. */ + fput(executable); + goto out_unlock; + } + bprm->execfd = retval; + } + return 0; + +out_unlock: + up_write(&me->signal->exec_update_lock); + if (!bprm->cred) + mutex_unlock(&me->signal->cred_guard_mutex); + +out: + return retval; +} +EXPORT_SYMBOL(begin_new_exec); + +void would_dump(struct linux_binprm *bprm, struct file *file) +{ + struct inode *inode = file_inode(file); + struct mnt_idmap *idmap = file_mnt_idmap(file); + if (inode_permission(idmap, inode, MAY_READ) < 0) { + struct user_namespace *old, *user_ns; + bprm->interp_flags |= BINPRM_FLAGS_ENFORCE_NONDUMP; + + /* Ensure bprm->user_ns contains the executable. */ + user_ns = old = bprm->user_ns; + while ((user_ns != &init_user_ns) && + !privileged_wrt_inode_uidgid(user_ns, idmap, inode)) + user_ns = user_ns->parent; + + if (old != user_ns) { + bprm->user_ns = get_user_ns(user_ns); + put_user_ns(old); + } + } +} +EXPORT_SYMBOL(would_dump); + +void setup_new_exec(struct linux_binprm * bprm) +{ + /* Setup things that can depend upon the personality */ + struct task_struct *me = current; + + arch_pick_mmap_layout(me->mm, &bprm->rlim_stack); + + arch_setup_new_exec(); + + /* Set the new mm task size. We have to do that late because it may + * depend on TIF_32BIT which is only updated in flush_thread() on + * some architectures like powerpc + */ + me->mm->task_size = TASK_SIZE; + up_write(&me->signal->exec_update_lock); + mutex_unlock(&me->signal->cred_guard_mutex); + + /* The exec locks are dropped: release the old address space now. */ + if (bprm->old_mm) { + exec_mm_put_old(bprm->old_mm); + bprm->old_mm = NULL; + } +} +EXPORT_SYMBOL(setup_new_exec); + +/* Runs immediately before start_thread() takes over. */ +void finalize_exec(struct linux_binprm *bprm) +{ + /* Store any stack rlimit changes before starting thread. */ + task_lock(current->group_leader); + current->signal->rlim[RLIMIT_STACK] = bprm->rlim_stack; + task_unlock(current->group_leader); +} +EXPORT_SYMBOL(finalize_exec); + +/* + * Prepare credentials and lock ->cred_guard_mutex. + * setup_new_exec() commits the new creds and drops the lock. + * Or, if exec fails before, free_bprm() should release ->cred + * and unlock. + */ +static int prepare_bprm_creds(struct linux_binprm *bprm) +{ + if (mutex_lock_interruptible(¤t->signal->cred_guard_mutex)) + return -ERESTARTNOINTR; + + bprm->cred = prepare_exec_creds(); + if (likely(bprm->cred)) + return 0; + + mutex_unlock(¤t->signal->cred_guard_mutex); + return -ENOMEM; +} + +/* Matches do_open_execat() */ +static void do_close_execat(struct file *file) +{ + if (!file) + return; + exe_file_allow_write_access(file); + fput(file); +} + +/** + * bprm_open_interpreter - open the interpreter the binary asks for + * @bprm: binary that is being executed + * @path: the interpreter path named in the binary's PT_INTERP + * + * A binfmt_misc loader entry substitutes for the interpreter the binary + * names. Hand out the stashed substitute if there is one and open @path + * if there is not. The caller owns the reference either way and releases + * it like any other open_exec() one. + * + * Return: the interpreter on success, an ERR_PTR on failure + */ +struct file *bprm_open_interpreter(struct linux_binprm *bprm, const char *path) +{ + if (bprm->loader) + return no_free_ptr(bprm->loader); + return open_exec(path); +} + +/** + * bprm_drop_loader - discard a PT_INTERP substitute that does not apply + * @bprm: binary that is being executed + * + * A binary without PT_INTERP has nothing to substitute for, so drop the + * override and let the binary load natively rather than have + * begin_new_exec() refuse it. A no-op once bprm_open_interpreter() took + * the substitute. + */ +void bprm_drop_loader(struct linux_binprm *bprm) +{ + do_close_execat(no_free_ptr(bprm->loader)); +} + +static void free_bprm(struct linux_binprm *bprm) +{ + if (bprm->mm) { + acct_arg_size(bprm, 0); + mmput(bprm->mm); + } + if (bprm->user_ns) + put_user_ns(bprm->user_ns); + free_arg_pages(bprm); + if (bprm->cred) { + /* in case exec fails before de_thread() succeeds */ + current->fs->in_exec = 0; + mutex_unlock(¤t->signal->cred_guard_mutex); + abort_creds(bprm->cred); + } + /* exec swapped the mm but failed before setup_new_exec() freed it */ + if (bprm->old_mm) + exec_mm_put_old(bprm->old_mm); + do_close_execat(bprm->file); + /* An unconsumed PT_INTERP substitute from a binfmt_misc loader entry. */ + bprm_drop_loader(bprm); + do_close_execat(bprm->executable); + /* If a binfmt changed the interp, free it. */ + if (bprm->interp != bprm->filename) + kfree(bprm->interp); + kfree(bprm->bpf_interp); + if (bprm->bpf_interp_file) + fput(bprm->bpf_interp_file); + kfree(bprm->bpf_interp_arg); + kfree(bprm->fdpath); + kfree(bprm); +} + +static struct linux_binprm *alloc_bprm(int fd, struct filename *filename, int flags) +{ + struct linux_binprm *bprm; + struct file *file; + int retval = -ENOMEM; + + file = do_open_execat(fd, filename, flags); + if (IS_ERR(file)) + return ERR_CAST(file); + + bprm = kzalloc_obj(*bprm); + if (!bprm) { + do_close_execat(file); + return ERR_PTR(-ENOMEM); + } + + bprm->file = file; + + if (fd == AT_FDCWD || filename->name[0] == '/') { + bprm->filename = filename->name; + } else { + if (filename->name[0] == '\0') { + bprm->fdpath = kasprintf(GFP_KERNEL, "/dev/fd/%d", fd); + bprm->comm_from_dentry = 1; + } else { + bprm->fdpath = kasprintf(GFP_KERNEL, "/dev/fd/%d/%s", + fd, filename->name); + } + if (!bprm->fdpath) + goto out_free; + + /* + * Record that a name derived from an O_CLOEXEC fd will be + * inaccessible after exec. This allows the code in exec to + * choose to fail when the executable is not mmaped into the + * interpreter and an open file descriptor is not passed to + * the interpreter. This makes for a better user experience + * than having the interpreter start and then immediately fail + * when it finds the executable is inaccessible. + */ + if (get_close_on_exec(fd)) + bprm->interp_flags |= BINPRM_FLAGS_PATH_INACCESSIBLE; + + bprm->filename = bprm->fdpath; + } + bprm->interp = bprm->filename; + + /* + * At this point, security_file_open() has already been called (with + * __FMODE_EXEC) and access control checks for AT_EXECVE_CHECK will + * stop just after the security_bprm_creds_for_exec() call in + * bprm_execve(). Indeed, the kernel should not try to parse the + * content of the file with exec_binprm() nor change the calling + * thread, which means that the following security functions will not + * be called: + * - security_bprm_check() + * - security_bprm_creds_from_file() + * - security_bprm_committing_creds() + * - security_bprm_committed_creds() + */ + bprm->is_check = !!(flags & AT_EXECVE_CHECK); + + retval = bprm_mm_init(bprm); + if (!retval) + return bprm; + +out_free: + free_bprm(bprm); + return ERR_PTR(retval); +} + +DEFINE_CLASS(bprm, struct linux_binprm *, if (!IS_ERR(_T)) free_bprm(_T), + alloc_bprm(fd, name, flags), int fd, struct filename *name, int flags) + +int bprm_change_interp(const char *interp, struct linux_binprm *bprm) +{ + /* If a binfmt changed the interp, free it first. */ + if (bprm->interp != bprm->filename) + kfree(bprm->interp); + bprm->interp = kstrdup(interp, GFP_KERNEL); + if (!bprm->interp) + return -ENOMEM; + return 0; +} +EXPORT_SYMBOL(bprm_change_interp); + +/* + * determine how safe it is to execute the proposed program + * - the caller must hold ->cred_guard_mutex to protect against + * PTRACE_ATTACH or seccomp thread-sync + */ +static void check_unsafe_exec(struct linux_binprm *bprm) +{ + struct task_struct *p = current, *t; + unsigned n_fs; + + if (p->ptrace) + bprm->unsafe |= LSM_UNSAFE_PTRACE; + + /* + * This isn't strictly necessary, but it makes it harder for LSMs to + * mess up. + */ + if (task_no_new_privs(current)) + bprm->unsafe |= LSM_UNSAFE_NO_NEW_PRIVS; + + /* + * If another task is sharing our fs, we cannot safely + * suid exec because the differently privileged task + * will be able to manipulate the current directory, etc. + * It would be nice to force an unshare instead... + * + * Otherwise we set fs->in_exec = 1 to deny clone(CLONE_FS) + * from another sub-thread until de_thread() succeeds, this + * state is protected by cred_guard_mutex we hold. + */ + n_fs = 1; + read_seqlock_excl(&p->fs->seq); + rcu_read_lock(); + for_other_threads(p, t) { + if (t->fs == p->fs) + n_fs++; + } + rcu_read_unlock(); + + /* "users" and "in_exec" locked for copy_fs() */ + if (p->fs->users > n_fs) + bprm->unsafe |= LSM_UNSAFE_SHARE; + else + p->fs->in_exec = 1; + read_sequnlock_excl(&p->fs->seq); +} + +static void bprm_fill_uid(struct linux_binprm *bprm, struct file *file) +{ + /* Handle suid and sgid on files */ + struct mnt_idmap *idmap; + struct inode *inode = file_inode(file); + unsigned int mode; + vfsuid_t vfsuid; + vfsgid_t vfsgid; + int err; + + if (!mnt_may_suid(file->f_path.mnt)) + return; + + if (task_no_new_privs(current)) + return; + + mode = READ_ONCE(inode->i_mode); + if (!(mode & (S_ISUID|S_ISGID))) + return; + + idmap = file_mnt_idmap(file); + + /* Be careful if suid/sgid is set */ + inode_lock(inode); + + /* Atomically reload and check mode/uid/gid now that lock held. */ + mode = inode->i_mode; + vfsuid = i_uid_into_vfsuid(idmap, inode); + vfsgid = i_gid_into_vfsgid(idmap, inode); + err = inode_permission(idmap, inode, MAY_EXEC); + inode_unlock(inode); + + /* Did the exec bit vanish out from under us? Give up. */ + if (err) + return; + + /* We ignore suid/sgid if there are no mappings for them in the ns */ + if (!vfsuid_has_mapping(bprm->cred->user_ns, vfsuid) || + !vfsgid_has_mapping(bprm->cred->user_ns, vfsgid)) + return; + + if (mode & S_ISUID) { + bprm->per_clear |= PER_CLEAR_ON_SETID; + bprm->cred->euid = vfsuid_into_kuid(vfsuid); + } + + if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP)) { + bprm->per_clear |= PER_CLEAR_ON_SETID; + bprm->cred->egid = vfsgid_into_kgid(vfsgid); + } +} + +/* + * Compute brpm->cred based upon the final binary. + */ +static int bprm_creds_from_file(struct linux_binprm *bprm) +{ + /* Compute creds based on which file? */ + struct file *file = bprm->execfd_creds ? bprm->executable : bprm->file; + + bprm_fill_uid(bprm, file); + return security_bprm_creds_from_file(bprm, file); +} + +/* + * Fill the binprm structure from the inode. + * Read the first BINPRM_BUF_SIZE bytes + * + * This may be called multiple times for binary chains (scripts for example). + */ +static int prepare_binprm(struct linux_binprm *bprm) +{ + loff_t pos = 0; + + memset(bprm->buf, 0, BINPRM_BUF_SIZE); + return kernel_read(bprm->file, bprm->buf, BINPRM_BUF_SIZE, &pos); +} + +/* + * Arguments are '\0' separated strings found at the location bprm->p + * points to; chop off the first by relocating brpm->p to right after + * the first '\0' encountered. + */ +int remove_arg_zero(struct linux_binprm *bprm) +{ + unsigned long offset; + char *kaddr; + struct page *page; + + if (!bprm->argc) + return 0; + + do { + offset = bprm->p & ~PAGE_MASK; + page = get_arg_page(bprm, bprm->p, 0); + if (!page) + return -EFAULT; + kaddr = kmap_local_page(page); + + for (; offset < PAGE_SIZE && kaddr[offset]; + offset++, bprm->p++) + ; + + kunmap_local(kaddr); + put_arg_page(page); + } while (offset == PAGE_SIZE); + + bprm->p++; + bprm->argc--; + + return 0; +} +EXPORT_SYMBOL(remove_arg_zero); + +/* + * cycle the list of binary formats handler, until one recognizes the image + */ +static int search_binary_handler(struct linux_binprm *bprm) +{ + struct linux_binfmt *fmt; + int retval; + + retval = prepare_binprm(bprm); + if (retval < 0) + return retval; + + retval = security_bprm_check(bprm); + if (retval) + return retval; + + read_lock(&binfmt_lock); + list_for_each_entry(fmt, &formats, lh) { + if (!try_module_get(fmt->module)) + continue; + read_unlock(&binfmt_lock); + + retval = fmt->load_binary(bprm); + + read_lock(&binfmt_lock); + put_binfmt(fmt); + if (bprm->point_of_no_return || (retval != -ENOEXEC)) { + read_unlock(&binfmt_lock); + return retval; + } + } + read_unlock(&binfmt_lock); + + return -ENOEXEC; +} + +/* binfmt handlers will call back into begin_new_exec() on success. */ +static int exec_binprm(struct linux_binprm *bprm) +{ + pid_t old_pid, old_vpid; + int ret, depth; + + /* Need to fetch pid before load_binary changes it */ + old_pid = current->pid; + rcu_read_lock(); + old_vpid = task_pid_nr_ns(current, task_active_pid_ns(current->parent)); + rcu_read_unlock(); + + /* This allows 5 levels of binfmt rewrites before failing hard. */ + for (depth = 0;; depth++) { + struct file *exec; + if (depth > 5) + return -ELOOP; + + ret = search_binary_handler(bprm); + if (ret < 0) + return ret; + if (!bprm->interpreter) + break; + + /* A stashed PT_INTERP substitute belonged to the replaced file. */ + bprm_drop_loader(bprm); + + exec = bprm->file; + bprm->file = bprm->interpreter; + bprm->interpreter = NULL; + + if (unlikely(bprm->have_execfd)) { + if (bprm->executable) { + do_close_execat(exec); + return -ENOEXEC; + } + /* Kept for AT_EXECFD; the write denial rides along until hand-over. */ + bprm->executable = exec; + } else { + do_close_execat(exec); + } + } + + audit_bprm(bprm); + trace_sched_process_exec(current, old_pid, bprm); + ptrace_event(PTRACE_EVENT_EXEC, old_vpid); + proc_exec_connector(current); + return 0; +} + +static int bprm_execve(struct linux_binprm *bprm) +{ + int retval; + + retval = prepare_bprm_creds(bprm); + if (retval) + return retval; + + /* + * Check for unsafe execution states before exec_binprm(), which + * will call back into begin_new_exec(), into bprm_creds_from_file(), + * where setuid-ness is evaluated. + */ + check_unsafe_exec(bprm); + current->in_execve = 1; + sched_mm_cid_before_execve(current); + + sched_exec(); + + /* Set the unchanging part of bprm->cred */ + retval = security_bprm_creds_for_exec(bprm); + if (retval || bprm->is_check) + goto out; + + retval = exec_binprm(bprm); + if (retval < 0) + goto out; + + sched_mm_cid_after_execve(current); + rseq_execve(current); + /* execve succeeded */ + current->in_execve = 0; + user_events_execve(current); + acct_update_integrals(current); + task_numa_free(current, false); + return retval; + +out: + /* + * If past the point of no return ensure the code never + * returns to the userspace process. Use an existing fatal + * signal if present otherwise terminate the process with + * SIGSEGV. + */ + if (bprm->point_of_no_return && !fatal_signal_pending(current)) + force_fatal_sig(SIGSEGV); + + sched_mm_cid_after_execve(current); + rseq_force_update(); + current->in_execve = 0; + + return retval; +} + +static int do_execveat_common(int fd, struct filename *filename, + struct user_arg_ptr argv, + struct user_arg_ptr envp, + int flags) +{ + int retval; + + /* + * We move the actual failure in case of RLIMIT_NPROC excess from + * set*uid() to execve() because too many poorly written programs + * don't check setuid() return code. Here we additionally recheck + * whether NPROC limit is still exceeded. + */ + if ((current->flags & PF_NPROC_EXCEEDED) && + is_rlimit_overlimit(current_ucounts(), UCOUNT_RLIMIT_NPROC, rlimit(RLIMIT_NPROC))) + return -EAGAIN; + + /* We're below the limit (still or again), so we don't want to make + * further execve() calls fail. */ + current->flags &= ~PF_NPROC_EXCEEDED; + + CLASS(bprm, bprm)(fd, filename, flags); + if (IS_ERR(bprm)) + return PTR_ERR(bprm); + + retval = count(argv, MAX_ARG_STRINGS); + if (retval < 0) + return retval; + bprm->argc = retval; + + retval = count(envp, MAX_ARG_STRINGS); + if (retval < 0) + return retval; + bprm->envc = retval; + + retval = bprm_stack_limits(bprm); + if (retval < 0) + return retval; + + retval = copy_string_kernel(bprm->filename, bprm); + if (retval < 0) + return retval; + bprm->exec = bprm->p; + + retval = copy_strings(bprm->envc, envp, bprm); + if (retval < 0) + return retval; + + retval = copy_strings(bprm->argc, argv, bprm); + if (retval < 0) + return retval; + + /* + * When argv is empty, add an empty string ("") as argv[0] to + * ensure confused userspace programs that start processing + * from argv[1] won't end up walking envp. See also + * bprm_stack_limits(). + */ + if (bprm->argc == 0) { + retval = copy_string_kernel("", bprm); + if (retval < 0) + return retval; + bprm->argc = 1; + + pr_warn_once("process '%s' launched '%s' with NULL argv: empty string added\n", + current->comm, bprm->filename); + } + + return bprm_execve(bprm); +} + +int kernel_execve(const char *kernel_filename, + const char *const *argv, const char *const *envp) +{ + int retval; + + /* It is non-sense for kernel threads to call execve */ + if (WARN_ON_ONCE(current->flags & PF_KTHREAD)) + return -EINVAL; + + CLASS(filename_kernel, filename)(kernel_filename); + CLASS(bprm, bprm)(AT_FDCWD, filename, 0); + if (IS_ERR(bprm)) + return PTR_ERR(bprm); + + retval = count_strings_kernel(argv); + if (WARN_ON_ONCE(retval == 0)) + return -EINVAL; + if (retval < 0) + return retval; + bprm->argc = retval; + + retval = count_strings_kernel(envp); + if (retval < 0) + return retval; + bprm->envc = retval; + + retval = bprm_stack_limits(bprm); + if (retval < 0) + return retval; + + retval = copy_string_kernel(bprm->filename, bprm); + if (retval < 0) + return retval; + bprm->exec = bprm->p; + + retval = copy_strings_kernel(bprm->envc, envp, bprm); + if (retval < 0) + return retval; + + retval = copy_strings_kernel(bprm->argc, argv, bprm); + if (retval < 0) + return retval; + + return bprm_execve(bprm); +} + +void set_binfmt(struct linux_binfmt *new) +{ + struct mm_struct *mm = current->mm; + + if (mm->binfmt) + module_put(mm->binfmt->module); + + mm->binfmt = new; + if (new) + __module_get(new->module); +} +EXPORT_SYMBOL(set_binfmt); + +static inline struct user_arg_ptr native_arg(const char __user *const __user *p) +{ + return (struct user_arg_ptr){.ptr.native = p}; +} + +SYSCALL_DEFINE3(execve, + const char __user *, filename, + const char __user *const __user *, argv, + const char __user *const __user *, envp) +{ + CLASS(filename, name)(filename); + return do_execveat_common(AT_FDCWD, name, + native_arg(argv), native_arg(envp), 0); +} + +SYSCALL_DEFINE5(execveat, + int, fd, const char __user *, filename, + const char __user *const __user *, argv, + const char __user *const __user *, envp, + int, flags) +{ + CLASS(filename_uflags, name)(filename, flags); + return do_execveat_common(fd, name, + native_arg(argv), native_arg(envp), flags); +} + +#ifdef CONFIG_COMPAT + +static inline struct user_arg_ptr compat_arg(const compat_uptr_t __user *p) +{ + return (struct user_arg_ptr){.is_compat = true, .ptr.compat = p}; +} + +COMPAT_SYSCALL_DEFINE3(execve, const char __user *, filename, + const compat_uptr_t __user *, argv, + const compat_uptr_t __user *, envp) +{ + CLASS(filename, name)(filename); + return do_execveat_common(AT_FDCWD, name, + compat_arg(argv), compat_arg(envp), 0); +} + +COMPAT_SYSCALL_DEFINE5(execveat, int, fd, + const char __user *, filename, + const compat_uptr_t __user *, argv, + const compat_uptr_t __user *, envp, + int, flags) +{ + CLASS(filename_uflags, name)(filename, flags); + return do_execveat_common(fd, name, + compat_arg(argv), compat_arg(envp), flags); +} +#endif + +#ifdef CONFIG_SYSCTL + +static int proc_dointvec_minmax_coredump(const struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + int error, old = READ_ONCE(suid_dumpable); + + error = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + + if (!error && write && (old != READ_ONCE(suid_dumpable))) + validate_coredump_safety(); + return error; +} + +static const struct ctl_table fs_exec_sysctls[] = { + { + .procname = "suid_dumpable", + .data = &suid_dumpable, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax_coredump, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_TWO, + }, +}; + +static int __init init_fs_exec_sysctls(void) +{ + register_sysctl_init("fs", fs_exec_sysctls); + return 0; +} + +fs_initcall(init_fs_exec_sysctls); +#endif /* CONFIG_SYSCTL */ + +#ifdef CONFIG_EXEC_KUNIT_TEST +#include "tests/exec_kunit.c" +#endif diff --git a/fs/exfat/Kconfig b/fs/exfat/Kconfig new file mode 100644 index 000000000..1fcb10c8d --- /dev/null +++ b/fs/exfat/Kconfig @@ -0,0 +1,24 @@ +# SPDX-License-Identifier: GPL-2.0-or-later + +config EXFAT_FS + tristate "exFAT filesystem support" + select BUFFER_HEAD + select NLS + select FS_IOMAP + help + This allows you to mount devices formatted with the exFAT file system. + exFAT is typically used on SD-Cards or USB sticks. + + To compile this as a module, choose M here: the module will be called + exfat. + +config EXFAT_DEFAULT_IOCHARSET + string "Default iocharset for exFAT" + default "utf8" + depends on EXFAT_FS + help + Set this to the default input/output character set to use for + converting between the encoding that is used for user visible + filenames and the UTF-16 character encoding that the exFAT + filesystem uses. This can be overridden with the "iocharset" mount + option for the exFAT filesystems. diff --git a/fs/exfat/Makefile b/fs/exfat/Makefile new file mode 100644 index 000000000..e06bf8587 --- /dev/null +++ b/fs/exfat/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Makefile for the linux exFAT filesystem support. +# +obj-$(CONFIG_EXFAT_FS) += exfat.o + +exfat-y := inode.o namei.o dir.o super.o fatent.o cache.o nls.o misc.o \ + file.o balloc.o iomap.o diff --git a/fs/exfat/balloc.c b/fs/exfat/balloc.c new file mode 100644 index 000000000..c0ddd522c --- /dev/null +++ b/fs/exfat/balloc.c @@ -0,0 +1,406 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2012-2013 Samsung Electronics Co., Ltd. + */ + +#include +#include +#include +#include +#include + +#include "exfat_raw.h" +#include "exfat_fs.h" + +#if BITS_PER_LONG == 32 +#define __le_long __le32 +#define lel_to_cpu(A) le32_to_cpu(A) +#define cpu_to_lel(A) cpu_to_le32(A) +#elif BITS_PER_LONG == 64 +#define __le_long __le64 +#define lel_to_cpu(A) le64_to_cpu(A) +#define cpu_to_lel(A) cpu_to_le64(A) +#else +#error "BITS_PER_LONG not 32 or 64" +#endif + +/* + * Allocation Bitmap Management Functions + */ +static bool exfat_test_bitmap_range(struct super_block *sb, unsigned int clu, + unsigned int count) +{ + struct exfat_sb_info *sbi = EXFAT_SB(sb); + unsigned int start = clu; + unsigned int end = clu + count; + unsigned int ent_idx, i, b; + unsigned int bit_offset, bits_to_check; + __le_long *bitmap_le; + unsigned long mask, word; + + if (!is_valid_cluster(sbi, start) || !is_valid_cluster(sbi, end - 1)) + return false; + + while (start < end) { + ent_idx = CLUSTER_TO_BITMAP_ENT(start); + i = BITMAP_OFFSET_SECTOR_INDEX(sb, ent_idx); + b = BITMAP_OFFSET_BIT_IN_SECTOR(sb, ent_idx); + + bitmap_le = (__le_long *)sbi->vol_amap[i]->b_data; + + /* Calculate how many bits we can check in the current word */ + bit_offset = b % BITS_PER_LONG; + bits_to_check = min(end - start, + (unsigned int)(BITS_PER_LONG - bit_offset)); + + /* Create a bitmask for the range of bits to check */ + if (bits_to_check >= BITS_PER_LONG) + mask = ~0UL; + else + mask = ((1UL << bits_to_check) - 1) << bit_offset; + word = lel_to_cpu(bitmap_le[b / BITS_PER_LONG]); + + /* Check if all bits in the mask are set */ + if ((word & mask) != mask) + return false; + + start += bits_to_check; + } + + return true; +} + +static int exfat_allocate_bitmap(struct super_block *sb, + struct exfat_dentry *ep) +{ + struct exfat_sb_info *sbi = EXFAT_SB(sb); + long long map_size; + unsigned int i, j, need_map_size; + sector_t sector, end, ra; + blkcnt_t ra_cnt = 0; + + sbi->map_clu = le32_to_cpu(ep->dentry.bitmap.start_clu); + map_size = le64_to_cpu(ep->dentry.bitmap.size); + need_map_size = ((EXFAT_DATA_CLUSTER_COUNT(sbi) - 1) / BITS_PER_BYTE) + + 1; + if (need_map_size != map_size) { + exfat_err(sb, "bogus allocation bitmap size(need : %u, cur : %lld)", + need_map_size, map_size); + /* + * Only allowed when bogus allocation + * bitmap size is large + */ + if (need_map_size > map_size) + return -EIO; + } + sbi->map_sectors = ((need_map_size - 1) >> + (sb->s_blocksize_bits)) + 1; + sbi->vol_amap = kvmalloc_objs(struct buffer_head *, sbi->map_sectors); + if (!sbi->vol_amap) + return -ENOMEM; + + sector = ra = exfat_cluster_to_sector(sbi, sbi->map_clu); + end = sector + sbi->map_sectors - 1; + + for (i = 0; i < sbi->map_sectors; i++) { + /* Trigger the next readahead in advance. */ + exfat_blk_readahead(sb, sector + i, &ra, &ra_cnt, end); + + sbi->vol_amap[i] = sb_bread(sb, sector + i); + if (!sbi->vol_amap[i]) + goto err_out; + } + + if (exfat_test_bitmap_range(sb, sbi->map_clu, + exfat_bytes_to_cluster_round_up(sbi, map_size)) == false) + goto err_out; + + return 0; + +err_out: + j = 0; + /* release all buffers and free vol_amap */ + while (j < i) + brelse(sbi->vol_amap[j++]); + + kvfree(sbi->vol_amap); + sbi->vol_amap = NULL; + return -EIO; +} + +int exfat_load_bitmap(struct super_block *sb) +{ + unsigned int i, type; + struct exfat_chain clu; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + + exfat_chain_set(&clu, sbi->root_dir, 0, ALLOC_FAT_CHAIN); + while (clu.dir != EXFAT_EOF_CLUSTER) { + for (i = 0; i < sbi->dentries_per_clu; i++) { + struct exfat_dentry *ep; + struct buffer_head *bh; + + ep = exfat_get_dentry(sb, &clu, i, &bh); + if (!ep) + return -EIO; + + type = exfat_get_entry_type(ep); + if (type == TYPE_BITMAP && + ep->dentry.bitmap.flags == 0x0) { + int err; + + err = exfat_allocate_bitmap(sb, ep); + brelse(bh); + return err; + } + brelse(bh); + + if (type == TYPE_UNUSED) + return -EINVAL; + } + + if (exfat_get_next_cluster(sb, &clu.dir)) + return -EIO; + } + + return -EINVAL; +} + +void exfat_free_bitmap(struct exfat_sb_info *sbi) +{ + int i; + + for (i = 0; i < sbi->map_sectors; i++) + __brelse(sbi->vol_amap[i]); + + kvfree(sbi->vol_amap); +} + +int exfat_set_bitmap(struct super_block *sb, unsigned int clu, bool sync) +{ + int i, b; + unsigned int ent_idx; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + + if (!is_valid_cluster(sbi, clu)) + return -EINVAL; + + ent_idx = CLUSTER_TO_BITMAP_ENT(clu); + i = BITMAP_OFFSET_SECTOR_INDEX(sb, ent_idx); + b = BITMAP_OFFSET_BIT_IN_SECTOR(sb, ent_idx); + + set_bit_le(b, sbi->vol_amap[i]->b_data); + exfat_update_bh(sbi->vol_amap[i], sync); + return 0; +} + +int exfat_clear_bitmap(struct super_block *sb, unsigned int clu, bool sync) +{ + int i, b; + unsigned int ent_idx; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + + if (!is_valid_cluster(sbi, clu)) + return -EIO; + + ent_idx = CLUSTER_TO_BITMAP_ENT(clu); + i = BITMAP_OFFSET_SECTOR_INDEX(sb, ent_idx); + b = BITMAP_OFFSET_BIT_IN_SECTOR(sb, ent_idx); + + if (!test_bit_le(b, sbi->vol_amap[i]->b_data)) + return -EIO; + + clear_bit_le(b, sbi->vol_amap[i]->b_data); + + exfat_update_bh(sbi->vol_amap[i], sync); + + return 0; +} + +bool exfat_test_bitmap(struct super_block *sb, unsigned int clu) +{ + int i, b; + unsigned int ent_idx; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + + if (!sbi->vol_amap) + return true; + + if (!is_valid_cluster(sbi, clu)) + return false; + + ent_idx = CLUSTER_TO_BITMAP_ENT(clu); + i = BITMAP_OFFSET_SECTOR_INDEX(sb, ent_idx); + b = BITMAP_OFFSET_BIT_IN_SECTOR(sb, ent_idx); + + if (!test_bit_le(b, sbi->vol_amap[i]->b_data)) + return false; + + return true; +} + +/* + * If the value of "clu" is 0, it means cluster 2 which is the first cluster of + * the cluster heap. + */ +unsigned int exfat_find_free_bitmap(struct super_block *sb, unsigned int clu) +{ + unsigned int i, map_i, map_b, ent_idx; + unsigned int clu_base, clu_free; + unsigned long clu_bits, clu_mask; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + __le_long bitval; + + WARN_ON(clu < EXFAT_FIRST_CLUSTER); + ent_idx = ALIGN_DOWN(CLUSTER_TO_BITMAP_ENT(clu), BITS_PER_LONG); + clu_base = BITMAP_ENT_TO_CLUSTER(ent_idx); + clu_mask = IGNORED_BITS_REMAINED(clu, clu_base); + + map_i = BITMAP_OFFSET_SECTOR_INDEX(sb, ent_idx); + map_b = BITMAP_OFFSET_BYTE_IN_SECTOR(sb, ent_idx); + + for (i = EXFAT_FIRST_CLUSTER; i < sbi->num_clusters; + i += BITS_PER_LONG) { + bitval = *(__le_long *)(sbi->vol_amap[map_i]->b_data + map_b); + if (clu_mask > 0) { + bitval |= cpu_to_lel(clu_mask); + clu_mask = 0; + } + if (lel_to_cpu(bitval) != ULONG_MAX) { + clu_bits = lel_to_cpu(bitval); + clu_free = clu_base + ffz(clu_bits); + if (clu_free < sbi->num_clusters) + return clu_free; + } + clu_base += BITS_PER_LONG; + map_b += sizeof(long); + + if (map_b >= sb->s_blocksize || + clu_base >= sbi->num_clusters) { + if (++map_i >= sbi->map_sectors) { + clu_base = EXFAT_FIRST_CLUSTER; + map_i = 0; + } + map_b = 0; + } + } + + return EXFAT_EOF_CLUSTER; +} + +int exfat_count_used_clusters(struct super_block *sb, unsigned int *ret_count) +{ + struct exfat_sb_info *sbi = EXFAT_SB(sb); + unsigned int count = 0; + unsigned int i, map_i = 0, map_b = 0; + unsigned int total_clus = EXFAT_DATA_CLUSTER_COUNT(sbi); + unsigned int last_mask = total_clus & (BITS_PER_LONG - 1); + unsigned long *bitmap, clu_bits; + + total_clus &= ~last_mask; + for (i = 0; i < total_clus; i += BITS_PER_LONG) { + bitmap = (void *)(sbi->vol_amap[map_i]->b_data + map_b); + count += hweight_long(*bitmap); + map_b += sizeof(long); + if (map_b >= (unsigned int)sb->s_blocksize) { + map_i++; + map_b = 0; + } + } + + if (last_mask) { + bitmap = (void *)(sbi->vol_amap[map_i]->b_data + map_b); + clu_bits = lel_to_cpu(*(__le_long *)bitmap); + count += hweight_long(clu_bits & BITMAP_LAST_WORD_MASK(last_mask)); + } + + *ret_count = count; + return 0; +} + +int exfat_trim_fs(struct inode *inode, struct fstrim_range *range) +{ + unsigned int trim_begin, trim_end, count, next_free_clu; + u64 clu_start, clu_end, trim_minlen, trimmed_total = 0; + struct super_block *sb = inode->i_sb; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + int err = 0; + + clu_start = max_t(u64, range->start >> sbi->cluster_size_bits, + EXFAT_FIRST_CLUSTER); + clu_end = clu_start + (range->len >> sbi->cluster_size_bits) - 1; + trim_minlen = range->minlen >> sbi->cluster_size_bits; + + if (clu_start >= sbi->num_clusters || range->len < sbi->cluster_size) + return -EINVAL; + + if (clu_end >= sbi->num_clusters) + clu_end = sbi->num_clusters - 1; + + mutex_lock(&sbi->bitmap_lock); + + trim_begin = trim_end = exfat_find_free_bitmap(sb, clu_start); + /* + * exfat_find_free_bitmap() may wrap around to the beginning of + * the bitmap. Reject a cluster outside the requested range. + */ + if (trim_begin == EXFAT_EOF_CLUSTER || + trim_begin < clu_start || trim_begin > clu_end) + goto unlock; + + for (;;) { + if (trim_end >= clu_end) + break; + + next_free_clu = exfat_find_free_bitmap(sb, trim_end + 1); + /* + * Stop if the search wrapped around or moved beyond the requested + * FITRIM range. + */ + if (next_free_clu == EXFAT_EOF_CLUSTER || + next_free_clu <= trim_end || next_free_clu > clu_end) + break; + + if (next_free_clu == trim_end + 1) { + /* extend trim range for continuous free cluster */ + trim_end++; + } else { + /* trim current range if it's larger than trim_minlen */ + count = trim_end - trim_begin + 1; + if (count >= trim_minlen) { + err = sb_issue_discard(sb, + exfat_cluster_to_sector(sbi, trim_begin), + count * sbi->sect_per_clus, GFP_NOFS, 0); + if (err) + goto unlock; + + trimmed_total += count; + } + + /* set next start point of the free hole */ + trim_begin = trim_end = next_free_clu; + } + + if (fatal_signal_pending(current)) { + err = -ERESTARTSYS; + goto unlock; + } + } + + /* try to trim remainder */ + count = trim_end - trim_begin + 1; + if (count >= trim_minlen) { + err = sb_issue_discard(sb, exfat_cluster_to_sector(sbi, trim_begin), + count * sbi->sect_per_clus, GFP_NOFS, 0); + if (err) + goto unlock; + + trimmed_total += count; + } + +unlock: + mutex_unlock(&sbi->bitmap_lock); + range->len = trimmed_total << sbi->cluster_size_bits; + + return err; +} diff --git a/fs/exfat/cache.c b/fs/exfat/cache.c new file mode 100644 index 000000000..7c8b4182f --- /dev/null +++ b/fs/exfat/cache.c @@ -0,0 +1,357 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * linux/fs/fat/cache.c + * + * Written 1992,1993 by Werner Almesberger + * + * Mar 1999. AV. Changed cache, so that it uses the starting cluster instead + * of inode number. + * May 1999. AV. Fixed the bogosity with FAT32 (read "FAT28"). Fscking lusers. + * Copyright (C) 2012-2013 Samsung Electronics Co., Ltd. + */ + +#include +#include +#include + +#include "exfat_raw.h" +#include "exfat_fs.h" + +#define EXFAT_MAX_CACHE 16 + +struct exfat_cache { + struct list_head cache_list; + unsigned int nr_contig; /* number of contiguous clusters */ + unsigned int fcluster; /* cluster number in the file. */ + unsigned int dcluster; /* cluster number on disk. */ +}; + +struct exfat_cache_id { + unsigned int id; + unsigned int nr_contig; + unsigned int fcluster; + unsigned int dcluster; +}; + +static struct kmem_cache *exfat_cachep; + +static void exfat_cache_init_once(void *c) +{ + struct exfat_cache *cache = (struct exfat_cache *)c; + + INIT_LIST_HEAD(&cache->cache_list); +} + +int exfat_cache_init(void) +{ + exfat_cachep = kmem_cache_create("exfat_cache", + sizeof(struct exfat_cache), + 0, SLAB_RECLAIM_ACCOUNT, + exfat_cache_init_once); + if (!exfat_cachep) + return -ENOMEM; + return 0; +} + +void exfat_cache_shutdown(void) +{ + if (!exfat_cachep) + return; + kmem_cache_destroy(exfat_cachep); +} + +static inline struct exfat_cache *exfat_cache_alloc(void) +{ + return kmem_cache_alloc(exfat_cachep, GFP_NOFS); +} + +static inline void exfat_cache_free(struct exfat_cache *cache) +{ + WARN_ON(!list_empty(&cache->cache_list)); + kmem_cache_free(exfat_cachep, cache); +} + +static inline void exfat_cache_update_lru(struct inode *inode, + struct exfat_cache *cache) +{ + struct exfat_inode_info *ei = EXFAT_I(inode); + + if (ei->cache_lru.next != &cache->cache_list) + list_move(&cache->cache_list, &ei->cache_lru); +} + +/* + * Find the cache that covers or precedes 'fclus' and return the last + * cluster before the next cache range. + */ +static inline unsigned int +exfat_cache_lookup(struct inode *inode, struct exfat_cache_id *cid, + unsigned int fclus, unsigned int end, + unsigned int *cached_fclus, unsigned int *cached_dclus) +{ + struct exfat_inode_info *ei = EXFAT_I(inode); + static struct exfat_cache nohit = { .fcluster = 0, }; + struct exfat_cache *hit = &nohit, *p; + unsigned int tail = 0; /* End boundary of hit cache */ + + /* + * Search range [fclus, end]. Stop early if: + * 1. Cache covers entire range, or + * 2. Next cache starts at current cache tail + */ + spin_lock(&ei->cache_lru_lock); + list_for_each_entry(p, &ei->cache_lru, cache_list) { + /* Find the cache of "fclus" or nearest cache. */ + if (p->fcluster <= fclus) { + if (p->fcluster < hit->fcluster) + continue; + + hit = p; + tail = hit->fcluster + hit->nr_contig; + + /* Current cache covers [fclus, end] completely */ + if (tail >= end) + break; + } else if (p->fcluster <= end) { + end = p->fcluster - 1; + + /* + * If we have a hit and next cache starts within/at + * its tail, caches are contiguous, stop searching. + */ + if (tail && tail >= end) + break; + } + } + if (hit != &nohit) { + unsigned int offset; + + exfat_cache_update_lru(inode, hit); + cid->id = ei->cache_valid_id; + cid->nr_contig = hit->nr_contig; + cid->fcluster = hit->fcluster; + cid->dcluster = hit->dcluster; + + offset = min(cid->nr_contig, fclus - cid->fcluster); + *cached_fclus = cid->fcluster + offset; + *cached_dclus = cid->dcluster + offset; + } + spin_unlock(&ei->cache_lru_lock); + + /* Return next cache start or 'end' if no more caches */ + return end; +} + +static struct exfat_cache *exfat_cache_merge(struct inode *inode, + struct exfat_cache_id *new) +{ + struct exfat_inode_info *ei = EXFAT_I(inode); + struct exfat_cache *p; + + list_for_each_entry(p, &ei->cache_lru, cache_list) { + /* Find the same part as "new" in cluster-chain. */ + if (p->fcluster == new->fcluster) { + if (new->nr_contig > p->nr_contig) + p->nr_contig = new->nr_contig; + return p; + } + } + return NULL; +} + +static void exfat_cache_add(struct inode *inode, + struct exfat_cache_id *new) +{ + struct exfat_inode_info *ei = EXFAT_I(inode); + struct exfat_cache *cache, *tmp; + + if (new->fcluster == EXFAT_EOF_CLUSTER) /* dummy cache */ + return; + + spin_lock(&ei->cache_lru_lock); + if (new->id != EXFAT_CACHE_VALID && + new->id != ei->cache_valid_id) + goto unlock; /* this cache was invalidated */ + + cache = exfat_cache_merge(inode, new); + if (cache == NULL) { + if (ei->nr_caches < EXFAT_MAX_CACHE) { + ei->nr_caches++; + spin_unlock(&ei->cache_lru_lock); + + tmp = exfat_cache_alloc(); + if (!tmp) { + spin_lock(&ei->cache_lru_lock); + ei->nr_caches--; + spin_unlock(&ei->cache_lru_lock); + return; + } + + spin_lock(&ei->cache_lru_lock); + cache = exfat_cache_merge(inode, new); + if (cache != NULL) { + ei->nr_caches--; + exfat_cache_free(tmp); + goto out_update_lru; + } + cache = tmp; + } else { + struct list_head *p = ei->cache_lru.prev; + + cache = list_entry(p, + struct exfat_cache, cache_list); + } + cache->fcluster = new->fcluster; + cache->dcluster = new->dcluster; + cache->nr_contig = new->nr_contig; + } +out_update_lru: + exfat_cache_update_lru(inode, cache); +unlock: + spin_unlock(&ei->cache_lru_lock); +} + +/* + * Cache invalidation occurs rarely, thus the LRU chain is not updated. It + * fixes itself after a while. + */ +static void __exfat_cache_inval_inode(struct inode *inode) +{ + struct exfat_inode_info *ei = EXFAT_I(inode); + struct exfat_cache *cache; + + while (!list_empty(&ei->cache_lru)) { + cache = list_entry(ei->cache_lru.next, + struct exfat_cache, cache_list); + list_del_init(&cache->cache_list); + ei->nr_caches--; + exfat_cache_free(cache); + } + /* Update. The copy of caches before this id is discarded. */ + ei->cache_valid_id++; + if (ei->cache_valid_id == EXFAT_CACHE_VALID) + ei->cache_valid_id++; +} + +void exfat_cache_inval_inode(struct inode *inode) +{ + struct exfat_inode_info *ei = EXFAT_I(inode); + + spin_lock(&ei->cache_lru_lock); + __exfat_cache_inval_inode(inode); + spin_unlock(&ei->cache_lru_lock); +} + +static inline int cache_contiguous(struct exfat_cache_id *cid, + unsigned int dclus) +{ + cid->nr_contig++; + return cid->dcluster + cid->nr_contig == dclus; +} + +static inline void cache_init(struct exfat_cache_id *cid, + unsigned int fclus, unsigned int dclus) +{ + cid->id = EXFAT_CACHE_VALID; + cid->fcluster = fclus; + cid->dcluster = dclus; + cid->nr_contig = 0; +} + +int exfat_get_cluster(struct inode *inode, unsigned int cluster, + unsigned int *dclus, unsigned int *count, + unsigned int *last_dclus) +{ + struct super_block *sb = inode->i_sb; + struct exfat_inode_info *ei = EXFAT_I(inode); + struct buffer_head *bh = NULL; + struct exfat_cache_id cid; + unsigned int content, fclus; + unsigned int end = cluster + *count - 1; + + if (ei->start_clu == EXFAT_FREE_CLUSTER) { + exfat_fs_error(sb, + "invalid access to exfat cache (entry 0x%08x)", + ei->start_clu); + return -EIO; + } + + fclus = 0; + *dclus = ei->start_clu; + *last_dclus = *dclus; + + /* + * This case should not exist, as exfat_map_cluster function doesn't + * call this routine when start_clu == EXFAT_EOF_CLUSTER. + * This case is retained here for routine completeness. + */ + if (*dclus == EXFAT_EOF_CLUSTER) { + *count = 0; + return 0; + } + + /* If only the first cluster is needed, return now. */ + if (fclus == cluster && *count == 1) + return 0; + + cache_init(&cid, fclus, *dclus); + /* + * Update the 'end' to exclude the next cache range, as clusters in + * different cache are typically not contiguous. + */ + end = exfat_cache_lookup(inode, &cid, cluster, end, &fclus, dclus); + + /* Return if the cache covers the entire range. */ + if (cid.fcluster + cid.nr_contig >= end) { + *count = end - cluster + 1; + return 0; + } + + /* Find the first cluster we need. */ + while (fclus < cluster) { + if (exfat_ent_get(sb, *dclus, &content, &bh)) + return -EIO; + + *last_dclus = *dclus; + *dclus = content; + fclus++; + + if (content == EXFAT_EOF_CLUSTER) + break; + + if (!cache_contiguous(&cid, *dclus)) + cache_init(&cid, fclus, *dclus); + } + + /* + * Now the cid cache contains the first cluster requested, collect + * the remaining clusters of this contiguous extent. + */ + if (*dclus != EXFAT_EOF_CLUSTER) { + unsigned int clu = *dclus; + + while (fclus < end) { + if (exfat_ent_get(sb, clu, &content, &bh)) + return -EIO; + if (++clu != content) + break; + fclus++; + } + cid.nr_contig = fclus - cid.fcluster; + *count = fclus - cluster + 1; + + /* + * Cache this discontiguous cluster, we'll definitely need + * it later + */ + if (fclus < end && content != EXFAT_EOF_CLUSTER) { + exfat_cache_add(inode, &cid); + cache_init(&cid, fclus + 1, content); + } + } else { + *count = 0; + } + brelse(bh); + exfat_cache_add(inode, &cid); + return 0; +} diff --git a/fs/exfat/dir.c b/fs/exfat/dir.c new file mode 100644 index 000000000..fe73b1380 --- /dev/null +++ b/fs/exfat/dir.c @@ -0,0 +1,1381 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2012-2013 Samsung Electronics Co., Ltd. + */ + +#include +#include +#include +#include +#include + +#include "exfat_raw.h" +#include "exfat_fs.h" + +static int exfat_extract_uni_name(struct exfat_dentry *ep, + unsigned short *uniname) +{ + int i, len = 0; + + for (i = 0; i < EXFAT_FILE_NAME_LEN; i++) { + *uniname = le16_to_cpu(ep->dentry.name.unicode_0_14[i]); + if (*uniname == 0x0) + return len; + uniname++; + len++; + } + + *uniname = 0x0; + return len; + +} + +static int exfat_get_uniname_from_ext_entry(struct super_block *sb, + struct exfat_chain *p_dir, int entry, unsigned short *uniname) +{ + int i, err; + struct exfat_entry_set_cache es; + unsigned int uni_len = 0, len; + + err = exfat_get_dentry_set(&es, sb, p_dir, entry, ES_ALL_ENTRIES); + if (err) + return err; + + /* + * First entry : file entry + * Second entry : stream-extension entry + * Third entry : first file-name entry + * So, the index of first file-name dentry should start from 2. + */ + for (i = ES_IDX_FIRST_FILENAME; i < es.num_entries; i++) { + struct exfat_dentry *ep = exfat_get_dentry_cached(&es, i); + + /* end of name entry */ + if (exfat_get_entry_type(ep) != TYPE_EXTEND) + break; + + len = exfat_extract_uni_name(ep, uniname); + uni_len += len; + if (len != EXFAT_FILE_NAME_LEN || uni_len >= MAX_NAME_LENGTH) + break; + uniname += EXFAT_FILE_NAME_LEN; + } + + exfat_put_dentry_set(&es, false); + return 0; +} + +/* read a directory entry from the opened directory */ +static int exfat_readdir(struct inode *inode, loff_t *cpos, struct exfat_dir_entry *dir_entry) +{ + int i, dentries_per_clu, num_ext, err; + unsigned int type, clu_offset, max_dentries; + struct exfat_chain dir, clu; + struct exfat_uni_name uni_name; + struct exfat_dentry *ep; + struct super_block *sb = inode->i_sb; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct exfat_inode_info *ei = EXFAT_I(inode); + unsigned int dentry = exfat_bytes_to_dentries(*cpos) & 0xFFFFFFFF; + struct buffer_head *bh; + + /* check if the given file ID is opened */ + if (ei->type != TYPE_DIR) + return -EPERM; + + exfat_chain_set(&dir, ei->start_clu, + exfat_bytes_to_cluster(sbi, i_size_read(inode)), ei->flags); + + dentries_per_clu = sbi->dentries_per_clu; + max_dentries = (unsigned int)min_t(u64, MAX_EXFAT_DENTRIES, + exfat_cluster_to_dentries(sbi, sbi->num_clusters)); + + clu_offset = exfat_dentries_to_cluster(sbi, dentry); + exfat_chain_dup(&clu, &dir); + + if (clu.flags == ALLOC_FAT_CHAIN) { + /* hint_information */ + if (clu_offset > 0 && ei->hint_bmap.off != EXFAT_EOF_CLUSTER && + ei->hint_bmap.off > 0 && clu_offset >= ei->hint_bmap.off) { + clu_offset -= ei->hint_bmap.off; + clu.dir = ei->hint_bmap.clu; + clu.size -= ei->hint_bmap.off; + } + } + + if (exfat_chain_advance(sb, &clu, clu_offset)) + return -EIO; + + while (clu.dir != EXFAT_EOF_CLUSTER && dentry < max_dentries) { + i = dentry & (dentries_per_clu - 1); + + for ( ; i < dentries_per_clu; i++, dentry++) { + ep = exfat_get_dentry(sb, &clu, i, &bh); + if (!ep) + return -EIO; + + type = exfat_get_entry_type(ep); + if (type == TYPE_UNUSED) { + brelse(bh); + goto out; + } + + if (type != TYPE_FILE && type != TYPE_DIR) { + brelse(bh); + continue; + } + + num_ext = ep->dentry.file.num_ext; + dir_entry->attr = le16_to_cpu(ep->dentry.file.attr); + + *uni_name.name = 0x0; + err = exfat_get_uniname_from_ext_entry(sb, &clu, i, + uni_name.name); + if (err) { + brelse(bh); + continue; + } + exfat_utf16_to_nls(sb, &uni_name, + dir_entry->namebuf.lfn, + dir_entry->namebuf.lfnbuf_len); + brelse(bh); + + ep = exfat_get_dentry(sb, &clu, i + 1, &bh); + if (!ep) + return -EIO; + dir_entry->entry = i; + dir_entry->dir = clu; + brelse(bh); + + ei->hint_bmap.off = exfat_dentries_to_cluster(sbi, dentry); + ei->hint_bmap.clu = clu.dir; + + *cpos = exfat_dentries_to_bytes(dentry + 1 + num_ext); + return 0; + } + + if (exfat_chain_advance(sb, &clu, 1)) + return -EIO; + } + +out: + dir_entry->namebuf.lfn[0] = '\0'; + *cpos = exfat_dentries_to_bytes(dentry); + return 0; +} + +static void exfat_init_namebuf(struct exfat_dentry_namebuf *nb) +{ + nb->lfn = NULL; + nb->lfnbuf_len = 0; +} + +static int exfat_alloc_namebuf(struct exfat_dentry_namebuf *nb) +{ + nb->lfn = __getname(); + if (!nb->lfn) + return -ENOMEM; + nb->lfnbuf_len = MAX_VFSNAME_BUF_SIZE; + return 0; +} + +static void exfat_free_namebuf(struct exfat_dentry_namebuf *nb) +{ + if (!nb->lfn) + return; + + __putname(nb->lfn); + exfat_init_namebuf(nb); +} + +/* + * Before calling dir_emit*(), sbi->s_lock should be released + * because page fault can occur in dir_emit*(). + */ +#define ITER_POS_FILLED_DOTS (2) +static int exfat_iterate(struct file *file, struct dir_context *ctx) +{ + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + struct inode *tmp; + struct exfat_dir_entry de; + struct exfat_dentry_namebuf *nb = &(de.namebuf); + struct exfat_inode_info *ei = EXFAT_I(inode); + unsigned long inum; + loff_t cpos, i_pos; + int err = 0, fake_offset = 0; + + exfat_init_namebuf(nb); + + cpos = ctx->pos; + if (!dir_emit_dots(file, ctx)) + goto out; + + if (ctx->pos == ITER_POS_FILLED_DOTS) { + cpos = 0; + fake_offset = 1; + } + + cpos = round_up(cpos, DENTRY_SIZE); + + /* name buffer should be allocated before use */ + err = exfat_alloc_namebuf(nb); + if (err) + goto out; +get_new: + mutex_lock(&EXFAT_SB(sb)->s_lock); + + if (ei->flags == ALLOC_NO_FAT_CHAIN && cpos >= i_size_read(inode)) + goto end_of_dir; + + err = exfat_readdir(inode, &cpos, &de); + if (err) { + /* + * At least we tried to read a sector. + * Move cpos to next sector position (should be aligned). + */ + if (err == -EIO) { + cpos += 1 << (sb->s_blocksize_bits); + cpos &= ~(loff_t)(sb->s_blocksize - 1); + } + + err = -EIO; + goto end_of_dir; + } + + if (!nb->lfn[0]) + goto end_of_dir; + + i_pos = ((loff_t)de.dir.dir << 32) | (de.entry & 0xffffffff); + tmp = exfat_iget(sb, i_pos); + if (tmp) { + inum = tmp->i_ino; + iput(tmp); + } else { + inum = iunique(sb, EXFAT_ROOT_INO); + } + + mutex_unlock(&EXFAT_SB(sb)->s_lock); + if (!dir_emit(ctx, nb->lfn, strlen(nb->lfn), inum, + (de.attr & EXFAT_ATTR_SUBDIR) ? DT_DIR : DT_REG)) + goto out; + ctx->pos = cpos; + goto get_new; + +end_of_dir: + if (!cpos && fake_offset) + cpos = ITER_POS_FILLED_DOTS; + ctx->pos = cpos; + mutex_unlock(&EXFAT_SB(sb)->s_lock); +out: + /* + * To improve performance, free namebuf after unlock sb_lock. + * If namebuf is not allocated, this function do nothing + */ + exfat_free_namebuf(nb); + return err; +} + +WRAP_DIR_ITER(exfat_iterate) // FIXME! +const struct file_operations exfat_dir_operations = { + .llseek = generic_file_llseek, + .read = generic_read_dir, + .iterate_shared = shared_exfat_iterate, + .unlocked_ioctl = exfat_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = exfat_compat_ioctl, +#endif + .fsync = exfat_file_fsync, + .setlease = generic_setlease, +}; + +int exfat_alloc_new_dir(struct inode *inode, struct exfat_chain *clu) +{ + int ret; + + exfat_chain_set(clu, EXFAT_EOF_CLUSTER, 0, ALLOC_NO_FAT_CHAIN); + + ret = exfat_alloc_cluster(inode, 1, clu, IS_DIRSYNC(inode), false); + if (ret) + return ret; + + ret = exfat_zeroed_cluster(inode, clu->dir); + if (ret) { + exfat_free_cluster(inode, clu); + return ret; + } + + return 0; +} + +int exfat_calc_num_entries(struct exfat_uni_name *p_uniname) +{ + int len; + + len = p_uniname->name_len; + if (len == 0) + return -EINVAL; + + /* 1 file entry + 1 stream entry + name entries */ + return ES_ENTRY_NUM(len); +} + +unsigned int exfat_get_entry_type(struct exfat_dentry *ep) +{ + if (ep->type == EXFAT_UNUSED) + return TYPE_UNUSED; + if (IS_EXFAT_DELETED(ep->type)) + return TYPE_DELETED; + if (ep->type == EXFAT_INVAL) + return TYPE_INVALID; + if (IS_EXFAT_CRITICAL_PRI(ep->type)) { + if (ep->type == EXFAT_BITMAP) + return TYPE_BITMAP; + if (ep->type == EXFAT_UPCASE) + return TYPE_UPCASE; + if (ep->type == EXFAT_VOLUME) + return TYPE_VOLUME; + if (ep->type == EXFAT_FILE) { + if (le16_to_cpu(ep->dentry.file.attr) & EXFAT_ATTR_SUBDIR) + return TYPE_DIR; + return TYPE_FILE; + } + return TYPE_CRITICAL_PRI; + } + if (IS_EXFAT_BENIGN_PRI(ep->type)) { + if (ep->type == EXFAT_GUID) + return TYPE_GUID; + if (ep->type == EXFAT_PADDING) + return TYPE_PADDING; + if (ep->type == EXFAT_ACLTAB) + return TYPE_ACLTAB; + return TYPE_BENIGN_PRI; + } + if (IS_EXFAT_CRITICAL_SEC(ep->type)) { + if (ep->type == EXFAT_STREAM) + return TYPE_STREAM; + if (ep->type == EXFAT_NAME) + return TYPE_EXTEND; + if (ep->type == EXFAT_ACL) + return TYPE_ACL; + return TYPE_CRITICAL_SEC; + } + + if (ep->type == EXFAT_VENDOR_EXT) + return TYPE_VENDOR_EXT; + if (ep->type == EXFAT_VENDOR_ALLOC) + return TYPE_VENDOR_ALLOC; + + return TYPE_BENIGN_SEC; +} + +static void exfat_set_entry_type(struct exfat_dentry *ep, unsigned int type) +{ + if (type == TYPE_UNUSED) { + ep->type = EXFAT_UNUSED; + } else if (type == TYPE_DELETED) { + ep->type &= EXFAT_DELETE; + } else if (type == TYPE_STREAM) { + ep->type = EXFAT_STREAM; + } else if (type == TYPE_EXTEND) { + ep->type = EXFAT_NAME; + } else if (type == TYPE_BITMAP) { + ep->type = EXFAT_BITMAP; + } else if (type == TYPE_UPCASE) { + ep->type = EXFAT_UPCASE; + } else if (type == TYPE_VOLUME) { + ep->type = EXFAT_VOLUME; + } else if (type == TYPE_DIR) { + ep->type = EXFAT_FILE; + ep->dentry.file.attr = cpu_to_le16(EXFAT_ATTR_SUBDIR); + } else if (type == TYPE_FILE) { + ep->type = EXFAT_FILE; + ep->dentry.file.attr = cpu_to_le16(EXFAT_ATTR_ARCHIVE); + } +} + +static void exfat_init_stream_entry(struct exfat_dentry *ep, + unsigned int start_clu, unsigned long long size) +{ + memset(ep, 0, sizeof(*ep)); + exfat_set_entry_type(ep, TYPE_STREAM); + if (size == 0) + ep->dentry.stream.flags = ALLOC_FAT_CHAIN; + else + ep->dentry.stream.flags = ALLOC_NO_FAT_CHAIN; + ep->dentry.stream.start_clu = cpu_to_le32(start_clu); + ep->dentry.stream.valid_size = cpu_to_le64(size); + ep->dentry.stream.size = cpu_to_le64(size); +} + +static void exfat_init_name_entry(struct exfat_dentry *ep, + unsigned short *uniname) +{ + int i; + + exfat_set_entry_type(ep, TYPE_EXTEND); + ep->dentry.name.flags = 0x0; + + for (i = 0; i < EXFAT_FILE_NAME_LEN; i++) { + if (*uniname != 0x0) { + ep->dentry.name.unicode_0_14[i] = cpu_to_le16(*uniname); + uniname++; + } else { + ep->dentry.name.unicode_0_14[i] = 0x0; + } + } +} + +void exfat_init_dir_entry(struct exfat_entry_set_cache *es, + unsigned int type, unsigned int start_clu, + unsigned long long size, struct timespec64 *ts) +{ + struct super_block *sb = es->sb; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct exfat_dentry *ep; + + ep = exfat_get_dentry_cached(es, ES_IDX_FILE); + memset(ep, 0, sizeof(*ep)); + exfat_set_entry_type(ep, type); + exfat_set_entry_time(sbi, ts, + &ep->dentry.file.create_tz, + &ep->dentry.file.create_time, + &ep->dentry.file.create_date, + &ep->dentry.file.create_time_cs); + exfat_set_entry_time(sbi, ts, + &ep->dentry.file.modify_tz, + &ep->dentry.file.modify_time, + &ep->dentry.file.modify_date, + &ep->dentry.file.modify_time_cs); + exfat_set_entry_time(sbi, ts, + &ep->dentry.file.access_tz, + &ep->dentry.file.access_time, + &ep->dentry.file.access_date, + NULL); + + ep = exfat_get_dentry_cached(es, ES_IDX_STREAM); + exfat_init_stream_entry(ep, start_clu, size); +} + +static void exfat_free_benign_secondary_clusters(struct inode *inode, + struct exfat_dentry *ep) +{ + struct super_block *sb = inode->i_sb; + struct exfat_chain dir; + unsigned int start_clu = + le32_to_cpu(ep->dentry.generic_secondary.start_clu); + u64 size = le64_to_cpu(ep->dentry.generic_secondary.size); + unsigned char flags = ep->dentry.generic_secondary.flags; + + if (!(flags & ALLOC_POSSIBLE) || !start_clu || !size) + return; + + exfat_chain_set(&dir, start_clu, + exfat_bytes_to_cluster_round_up(EXFAT_SB(sb), size), + flags); + exfat_free_cluster(inode, &dir); +} + +/* + * exfat_init_ext_entry - initialize extension entries in a directory entry set + * @es: target entry set + * @num_entries: number of entries excluding benign secondary entries + * @p_uniname: filename to store + * @old_es: optional source entry set with benign secondary entries, or NULL + * @num_extra: number of benign secondary entries to copy from @old_es + * + * Set up the file, stream extension, and filename entries in @es, optionally + * preserving @num_extra benign secondary entries from @old_es. @es and @old_es + * may refer to the same entry set; excess entries are marked as deleted. + */ +void exfat_init_ext_entry(struct exfat_entry_set_cache *es, int num_entries, + struct exfat_uni_name *p_uniname, + struct exfat_entry_set_cache *old_es, int num_extra) +{ + int i, src_start = 0, old_num; + unsigned short *uniname = p_uniname->name; + struct exfat_dentry *ep; + + if (WARN_ON(num_extra < 0 || (num_extra && (!old_es || + old_es->num_entries < ES_IDX_FIRST_FILENAME + num_extra)))) + num_extra = 0; + + /* + * Save old entry count and source position before modifying + * es->num_entries, since old_es and es may point to the same + * entry set. + */ + old_num = es->num_entries; + if (old_es && num_extra > 0) + src_start = old_es->num_entries - num_extra; + + es->num_entries = num_entries + num_extra; + ep = exfat_get_dentry_cached(es, ES_IDX_FILE); + ep->dentry.file.num_ext = (unsigned char)(num_entries - 1 + num_extra); + + ep = exfat_get_dentry_cached(es, ES_IDX_STREAM); + ep->dentry.stream.name_len = p_uniname->name_len; + ep->dentry.stream.name_hash = cpu_to_le16(p_uniname->name_hash); + + if (old_es && num_extra > 0) { + for (i = 0; i < num_extra; i++) + *exfat_get_dentry_cached(es, num_entries + i) = + *exfat_get_dentry_cached(old_es, src_start + i); + } + + for (i = ES_IDX_FIRST_FILENAME; i < num_entries; i++) { + ep = exfat_get_dentry_cached(es, i); + exfat_init_name_entry(ep, uniname); + uniname += EXFAT_FILE_NAME_LEN; + } + + /* Mark excess old entries as deleted (in-place shrink) */ + for (i = num_entries + num_extra; i < old_num; i++) { + ep = exfat_get_dentry_cached(es, i); + exfat_set_entry_type(ep, TYPE_DELETED); + } + + exfat_update_dir_chksum(es); +} + +void exfat_remove_entries(struct inode *inode, struct exfat_entry_set_cache *es, + int order, bool free_benign) +{ + int i; + struct exfat_dentry *ep; + + for (i = order; i < es->num_entries; i++) { + ep = exfat_get_dentry_cached(es, i); + + if (free_benign && (exfat_get_entry_type(ep) & TYPE_BENIGN_SEC)) + exfat_free_benign_secondary_clusters(inode, ep); + + exfat_set_entry_type(ep, TYPE_DELETED); + } + + if (order < es->num_entries) + es->modified = true; +} + +void exfat_update_dir_chksum(struct exfat_entry_set_cache *es) +{ + int chksum_type = CS_DIR_ENTRY, i; + unsigned short chksum = 0; + struct exfat_dentry *ep; + + for (i = ES_IDX_FILE; i < es->num_entries; i++) { + ep = exfat_get_dentry_cached(es, i); + chksum = exfat_calc_chksum16(ep, DENTRY_SIZE, chksum, + chksum_type); + chksum_type = CS_DEFAULT; + } + ep = exfat_get_dentry_cached(es, ES_IDX_FILE); + ep->dentry.file.checksum = cpu_to_le16(chksum); + es->modified = true; +} + +int exfat_put_dentry_set(struct exfat_entry_set_cache *es, int sync) +{ + int i, err = 0; + + if (es->modified) + err = exfat_update_bhs(es->bh, es->num_bh, sync); + + for (i = 0; i < es->num_bh; i++) + if (err) + bforget(es->bh[i]); + else + brelse(es->bh[i]); + + if (IS_DYNAMIC_ES(es)) + kfree(es->bh); + + return err; +} + +static int exfat_find_location(struct super_block *sb, struct exfat_chain *p_dir, + int entry, sector_t *sector, int *offset) +{ + int ret; + unsigned int off, clu = 0; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + + off = exfat_dentries_to_bytes(entry); + + clu = p_dir->dir; + ret = exfat_cluster_walk(sb, &clu, exfat_bytes_to_cluster(sbi, off), + p_dir->flags); + if (ret) + return ret; + + if (clu == EXFAT_EOF_CLUSTER) { + exfat_fs_error(sb, + "unexpected early break in cluster chain (clu : %u, len : %d)", + p_dir->dir, + exfat_bytes_to_cluster(sbi, off)); + return -EIO; + } + + if (!exfat_test_bitmap(sb, clu)) { + exfat_err(sb, "failed to test cluster bit(%u)", clu); + return -EIO; + } + + /* byte offset in cluster */ + off = exfat_cluster_offset(sbi, off); + + /* byte offset in sector */ + *offset = exfat_block_offset(sb, off); + + /* sector offset in cluster */ + *sector = exfat_bytes_to_block(sb, off); + *sector += exfat_cluster_to_sector(sbi, clu); + return 0; +} + +struct exfat_dentry *exfat_get_dentry(struct super_block *sb, + struct exfat_chain *p_dir, int entry, struct buffer_head **bh) +{ + struct exfat_sb_info *sbi = EXFAT_SB(sb); + unsigned int sect_per_clus = sbi->sect_per_clus; + unsigned int dentries_per_page = exfat_bytes_to_dentries(PAGE_SIZE); + int off; + sector_t sec; + + if (p_dir->dir == DIR_DELETED) { + exfat_err(sb, "abnormal access to deleted dentry"); + return NULL; + } + + if (exfat_find_location(sb, p_dir, entry, &sec, &off)) + return NULL; + + if (sect_per_clus > 1 && + (entry & (dentries_per_page - 1)) == 0) { + sector_t ra = sec; + blkcnt_t cnt = 0; + unsigned int ra_count = sect_per_clus; + + /* Not sector aligned with ra_count, resize ra_count to page size */ + if ((sec - sbi->data_start_sector) & (ra_count - 1)) + ra_count = PAGE_SIZE >> sb->s_blocksize_bits; + + exfat_blk_readahead(sb, sec, &ra, &cnt, sec + ra_count - 1); + } + + *bh = sb_bread(sb, sec); + if (!*bh) + return NULL; + + return (struct exfat_dentry *)((*bh)->b_data + off); +} + +enum exfat_validate_dentry_mode { + ES_MODE_GET_FILE_ENTRY, + ES_MODE_GET_STRM_ENTRY, + ES_MODE_GET_NAME_ENTRY, + ES_MODE_GET_CRITICAL_SEC_ENTRY, + ES_MODE_GET_BENIGN_SEC_ENTRY, +}; + +static bool exfat_validate_entry(unsigned int type, + enum exfat_validate_dentry_mode *mode) +{ + if (type == TYPE_UNUSED || type == TYPE_DELETED) + return false; + + switch (*mode) { + case ES_MODE_GET_FILE_ENTRY: + if (type != TYPE_STREAM) + return false; + *mode = ES_MODE_GET_STRM_ENTRY; + break; + case ES_MODE_GET_STRM_ENTRY: + if (type != TYPE_EXTEND) + return false; + *mode = ES_MODE_GET_NAME_ENTRY; + break; + case ES_MODE_GET_NAME_ENTRY: + if (type & TYPE_BENIGN_SEC) + *mode = ES_MODE_GET_BENIGN_SEC_ENTRY; + else if (type != TYPE_EXTEND) + return false; + break; + case ES_MODE_GET_BENIGN_SEC_ENTRY: + /* Assume unreconized benign secondary entry */ + if (!(type & TYPE_BENIGN_SEC)) + return false; + break; + default: + return false; + } + + return true; +} + +struct exfat_dentry *exfat_get_dentry_cached( + struct exfat_entry_set_cache *es, int num) +{ + int off = es->start_off + num * DENTRY_SIZE; + struct buffer_head *bh = es->bh[exfat_bytes_to_block(es->sb, off)]; + char *p = bh->b_data + exfat_block_offset(es->sb, off); + + return (struct exfat_dentry *)p; +} + +/* + * Returns a set of dentries. + * + * Note It provides a direct pointer to bh->data via exfat_get_dentry_cached(). + * User should call exfat_get_dentry_set() after setting 'modified' to apply + * changes made in this entry set to the real device. + * + * in: + * sb+p_dir+entry: indicates a file/dir + * num_entries: specifies how many dentries should be included. + * It will be set to es->num_entries if it is not 0. + * If num_entries is 0, es->num_entries will be obtained + * from the first dentry. + * out: + * es: pointer of entry set on success. + * return: + * 0 on success + * -error code on failure + */ +static int __exfat_get_dentry_set(struct exfat_entry_set_cache *es, + struct super_block *sb, struct exfat_chain *p_dir, int entry, + unsigned int num_entries) +{ + int ret, i, num_bh; + unsigned int off; + sector_t sec; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct buffer_head *bh; + + if (p_dir->dir == DIR_DELETED) { + exfat_err(sb, "access to deleted dentry"); + return -EIO; + } + + ret = exfat_find_location(sb, p_dir, entry, &sec, &off); + if (ret) + return ret; + + memset(es, 0, sizeof(*es)); + es->sb = sb; + es->modified = false; + es->start_off = off; + es->bh = es->__bh; + + bh = sb_bread(sb, sec); + if (!bh) + return -EIO; + es->bh[es->num_bh++] = bh; + + if (num_entries == ES_ALL_ENTRIES) { + struct exfat_dentry *ep; + + ep = exfat_get_dentry_cached(es, ES_IDX_FILE); + if (ep->type != EXFAT_FILE) { + brelse(bh); + return -EIO; + } + + num_entries = ep->dentry.file.num_ext + 1; + } + + es->num_entries = num_entries; + + num_bh = exfat_bytes_to_block_round_up(sb, off + num_entries * DENTRY_SIZE); + if (num_bh > ARRAY_SIZE(es->__bh)) { + es->bh = kmalloc_objs(*es->bh, num_bh, GFP_NOFS); + if (!es->bh) { + brelse(bh); + return -ENOMEM; + } + es->bh[0] = bh; + } + + for (i = 1; i < num_bh; i++) { + /* get the next sector */ + if (exfat_is_last_sector_in_cluster(sbi, sec)) { + unsigned int clu = exfat_sector_to_cluster(sbi, sec); + + if (exfat_cluster_walk(sb, &clu, 1, p_dir->flags)) + goto put_es; + sec = exfat_cluster_to_sector(sbi, clu); + } else { + sec++; + } + + bh = sb_bread(sb, sec); + if (!bh) + goto put_es; + es->bh[es->num_bh++] = bh; + } + + return 0; + +put_es: + exfat_put_dentry_set(es, false); + return -EIO; +} + +int exfat_get_dentry_set(struct exfat_entry_set_cache *es, + struct super_block *sb, struct exfat_chain *p_dir, + int entry, unsigned int num_entries) +{ + int ret, i; + struct exfat_dentry *ep; + enum exfat_validate_dentry_mode mode = ES_MODE_GET_FILE_ENTRY; + + ret = __exfat_get_dentry_set(es, sb, p_dir, entry, num_entries); + if (ret < 0) + return ret; + + /* validate cached dentries */ + for (i = ES_IDX_STREAM; i < es->num_entries; i++) { + ep = exfat_get_dentry_cached(es, i); + if (!exfat_validate_entry(exfat_get_entry_type(ep), &mode)) + goto put_es; + } + return 0; + +put_es: + exfat_put_dentry_set(es, false); + return -EIO; +} + +static int exfat_validate_empty_dentry_set(struct exfat_entry_set_cache *es) +{ + struct exfat_dentry *ep; + struct buffer_head *bh; + int i, off; + bool unused_hit = false; + + /* + * ONLY UNUSED OR DELETED DENTRIES ARE ALLOWED: + * Although it violates the specification for a deleted entry to + * follow an unused entry, some exFAT implementations could work + * like this. Therefore, to improve compatibility, let's allow it. + */ + for (i = 0; i < es->num_entries; i++) { + ep = exfat_get_dentry_cached(es, i); + if (ep->type == EXFAT_UNUSED) { + unused_hit = true; + } else if (!IS_EXFAT_DELETED(ep->type)) { + if (unused_hit) + goto err_used_follow_unused; + i++; + goto count_skip_entries; + } + } + + return 0; + +err_used_follow_unused: + off = es->start_off + (i << DENTRY_SIZE_BITS); + bh = es->bh[exfat_bytes_to_block(es->sb, off)]; + + exfat_fs_error(es->sb, + "in sector %lld, dentry %d should be unused, but 0x%x", + bh->b_blocknr, off >> DENTRY_SIZE_BITS, ep->type); + + return -EIO; + +count_skip_entries: + es->num_entries = + exfat_bytes_to_dentries(exfat_block_to_bytes(es->sb, es->num_bh) - es->start_off); + for (; i < es->num_entries; i++) { + ep = exfat_get_dentry_cached(es, i); + if (IS_EXFAT_DELETED(ep->type)) + break; + } + + return i; +} + +/* + * Get an empty dentry set. + * + * in: + * sb+p_dir+entry: indicates the empty dentry location + * num_entries: specifies how many empty dentries should be included. + * out: + * es: pointer of empty dentry set on success. + * return: + * 0 : on success + * >0 : the dentries are not empty, the return value is the number of + * dentries to be skipped for the next lookup. + * <0 : on failure + */ +int exfat_get_empty_dentry_set(struct exfat_entry_set_cache *es, + struct super_block *sb, struct exfat_chain *p_dir, + int entry, unsigned int num_entries) +{ + int ret; + + ret = __exfat_get_dentry_set(es, sb, p_dir, entry, num_entries); + if (ret < 0) + return ret; + + ret = exfat_validate_empty_dentry_set(es); + if (ret) + exfat_put_dentry_set(es, false); + + return ret; +} + +static inline void exfat_reset_empty_hint(struct exfat_hint_femp *hint_femp) +{ + hint_femp->eidx = EXFAT_HINT_NONE; + hint_femp->count = 0; +} + +static inline void exfat_set_empty_hint(struct exfat_inode_info *ei, + struct exfat_hint_femp *candi_empty, struct exfat_chain *clu, + int dentry, int num_entries, int entry_type) +{ + if (ei->hint_femp.eidx == EXFAT_HINT_NONE || + ei->hint_femp.eidx > dentry) { + int total_entries = exfat_bytes_to_dentries(i_size_read(&ei->vfs_inode)); + + if (candi_empty->count == 0) { + candi_empty->cur = *clu; + candi_empty->eidx = dentry; + } + + if (entry_type == TYPE_UNUSED) + candi_empty->count += total_entries - dentry; + else + candi_empty->count++; + + if (candi_empty->count == num_entries || + candi_empty->count + candi_empty->eidx == total_entries) + ei->hint_femp = *candi_empty; + } +} + +enum { + DIRENT_STEP_FILE, + DIRENT_STEP_STRM, + DIRENT_STEP_NAME, + DIRENT_STEP_SECD, +}; + +/* + * @ei: inode info of parent directory + * @p_dir: directory structure of parent directory + * @num_entries:entry size of p_uniname + * @hint_opt: If p_uniname is found, filled with optimized dir/entry + * for traversing cluster chain. + * @return: + * >= 0: file directory entry position where the name exists + * -ENOENT: entry with the name does not exist + * -EIO: I/O error + */ +int exfat_find_dir_entry(struct super_block *sb, struct exfat_inode_info *ei, + struct exfat_chain *p_dir, struct exfat_uni_name *p_uniname, + struct exfat_hint *hint_opt) +{ + int i, rewind = 0, dentry = 0, end_eidx = 0, num_ext = 0, len; + int order, step, name_len = 0; + int dentries_per_clu; + unsigned int entry_type; + unsigned short *uniname = NULL; + struct exfat_chain clu; + struct exfat_hint *hint_stat = &ei->hint_stat; + struct exfat_hint_femp candi_empty; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + int num_entries = exfat_calc_num_entries(p_uniname); + unsigned int clu_count = 0; + + if (num_entries < 0) + return num_entries; + + dentries_per_clu = sbi->dentries_per_clu; + + exfat_chain_dup(&clu, p_dir); + + if (hint_stat->eidx) { + clu.dir = hint_stat->clu; + dentry = hint_stat->eidx; + end_eidx = dentry; + } + + exfat_reset_empty_hint(&ei->hint_femp); + +rewind: + order = 0; + step = DIRENT_STEP_FILE; + exfat_reset_empty_hint(&candi_empty); + + while (clu.dir != EXFAT_EOF_CLUSTER) { + i = dentry & (dentries_per_clu - 1); + for (; i < dentries_per_clu; i++, dentry++) { + struct exfat_dentry *ep; + struct buffer_head *bh; + + if (rewind && dentry == end_eidx) + goto not_found; + + ep = exfat_get_dentry(sb, &clu, i, &bh); + if (!ep) + return -EIO; + + entry_type = exfat_get_entry_type(ep); + + if (entry_type == TYPE_UNUSED || + entry_type == TYPE_DELETED) { + step = DIRENT_STEP_FILE; + + exfat_set_empty_hint(ei, &candi_empty, &clu, + dentry, num_entries, + entry_type); + + brelse(bh); + if (entry_type == TYPE_UNUSED) + goto not_found; + continue; + } + + exfat_reset_empty_hint(&candi_empty); + + if (entry_type == TYPE_FILE || entry_type == TYPE_DIR) { + step = DIRENT_STEP_FILE; + hint_opt->clu = clu.dir; + hint_opt->eidx = i; + num_ext = ep->dentry.file.num_ext; + step = DIRENT_STEP_STRM; + brelse(bh); + continue; + } + + if (entry_type == TYPE_STREAM) { + u16 name_hash; + + if (step != DIRENT_STEP_STRM) { + step = DIRENT_STEP_FILE; + brelse(bh); + continue; + } + step = DIRENT_STEP_FILE; + name_hash = le16_to_cpu( + ep->dentry.stream.name_hash); + if (p_uniname->name_hash == name_hash && + p_uniname->name_len == + ep->dentry.stream.name_len) { + step = DIRENT_STEP_NAME; + order = 1; + name_len = 0; + } + brelse(bh); + continue; + } + + if (entry_type == TYPE_EXTEND) { + unsigned short entry_uniname[16], unichar; + unsigned int offset; + + if (step != DIRENT_STEP_NAME || + name_len >= MAX_NAME_LENGTH) { + brelse(bh); + step = DIRENT_STEP_FILE; + continue; + } + + offset = (++order - 2) * EXFAT_FILE_NAME_LEN; + len = exfat_extract_uni_name(ep, entry_uniname); + brelse(bh); + if (offset > MAX_NAME_LENGTH || + len > MAX_NAME_LENGTH - offset) { + step = DIRENT_STEP_FILE; + continue; + } + uniname = p_uniname->name + offset; + name_len += len; + + unichar = *(uniname+len); + *(uniname+len) = 0x0; + + if (exfat_uniname_ncmp(sb, uniname, + entry_uniname, len)) { + step = DIRENT_STEP_FILE; + } else if (p_uniname->name_len == name_len) { + if (order == num_ext) + goto found; + step = DIRENT_STEP_SECD; + } + + *(uniname+len) = unichar; + continue; + } + + brelse(bh); + if (entry_type & + (TYPE_CRITICAL_SEC | TYPE_BENIGN_SEC)) { + if (step == DIRENT_STEP_SECD) { + if (++order == num_ext) + goto found; + continue; + } + } + step = DIRENT_STEP_FILE; + } + + if (exfat_chain_advance(sb, &clu, 1)) + return -EIO; + + /* break if the cluster chain includes a loop */ + if (unlikely(++clu_count > EXFAT_DATA_CLUSTER_COUNT(sbi))) + goto not_found; + } + +not_found: + /* + * We started at not 0 index,so we should try to find target + * from 0 index to the index we started at. + */ + if (!rewind && end_eidx) { + rewind = 1; + dentry = 0; + clu.dir = p_dir->dir; + goto rewind; + } + + /* + * set the EXFAT_EOF_CLUSTER flag to avoid search + * from the beginning again when allocated a new cluster + */ + if (ei->hint_femp.eidx == EXFAT_HINT_NONE) { + ei->hint_femp.cur.dir = EXFAT_EOF_CLUSTER; + ei->hint_femp.eidx = p_dir->size * dentries_per_clu; + ei->hint_femp.count = 0; + } + + /* initialized hint_stat */ + hint_stat->clu = p_dir->dir; + hint_stat->eidx = 0; + return -ENOENT; + +found: + /* next dentry we'll find is out of this cluster */ + if (!((dentry + 1) & (dentries_per_clu - 1))) { + int ret = 0; + + ret = exfat_chain_advance(sb, &clu, 1); + + if (ret || clu.dir == EXFAT_EOF_CLUSTER) { + /* just initialized hint_stat */ + hint_stat->clu = p_dir->dir; + hint_stat->eidx = 0; + return (dentry - num_ext); + } + } + + hint_stat->clu = clu.dir; + hint_stat->eidx = dentry + 1; + return dentry - num_ext; +} + +int exfat_count_dir_entries(struct super_block *sb, struct exfat_chain *p_dir) +{ + int i, count = 0; + int dentries_per_clu; + unsigned int entry_type; + unsigned int clu_count = 0; + struct exfat_chain clu; + struct exfat_dentry *ep; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct buffer_head *bh; + + dentries_per_clu = sbi->dentries_per_clu; + + exfat_chain_dup(&clu, p_dir); + + while (clu.dir != EXFAT_EOF_CLUSTER) { + for (i = 0; i < dentries_per_clu; i++) { + ep = exfat_get_dentry(sb, &clu, i, &bh); + if (!ep) + return -EIO; + entry_type = exfat_get_entry_type(ep); + brelse(bh); + + if (entry_type == TYPE_UNUSED) + return count; + if (entry_type != TYPE_DIR) + continue; + count++; + } + + if (exfat_chain_advance(sb, &clu, 1)) + return -EIO; + + if (unlikely(++clu_count > sbi->used_clusters)) { + exfat_fs_error(sb, "FAT or bitmap is corrupted"); + return -EIO; + } + } + + return count; +} + +static int exfat_get_volume_label_dentry(struct super_block *sb, + struct exfat_entry_set_cache *es) +{ + int i; + int dentry = 0; + unsigned int type; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct exfat_hint_femp hint_femp; + struct exfat_inode_info *ei = EXFAT_I(sb->s_root->d_inode); + struct exfat_chain clu; + struct exfat_dentry *ep; + struct buffer_head *bh; + + hint_femp.eidx = EXFAT_HINT_NONE; + exfat_chain_set(&clu, sbi->root_dir, 0, ALLOC_FAT_CHAIN); + + while (clu.dir != EXFAT_EOF_CLUSTER) { + for (i = 0; i < sbi->dentries_per_clu; i++, dentry++) { + ep = exfat_get_dentry(sb, &clu, i, &bh); + if (!ep) + return -EIO; + + type = exfat_get_entry_type(ep); + if (hint_femp.eidx == EXFAT_HINT_NONE) { + if (type == TYPE_DELETED || type == TYPE_UNUSED) { + hint_femp.cur = clu; + hint_femp.eidx = dentry; + hint_femp.count = 1; + } + } + + if (type == TYPE_UNUSED) { + brelse(bh); + goto not_found; + } + + if (type != TYPE_VOLUME) { + brelse(bh); + continue; + } + + memset(es, 0, sizeof(*es)); + es->sb = sb; + es->bh = es->__bh; + es->bh[0] = bh; + es->num_bh = 1; + es->start_off = exfat_dentries_to_bytes(i) % sb->s_blocksize; + + return 0; + } + + if (exfat_get_next_cluster(sb, &(clu.dir))) + return -EIO; + } + +not_found: + if (hint_femp.eidx == EXFAT_HINT_NONE) { + hint_femp.cur.dir = EXFAT_EOF_CLUSTER; + hint_femp.eidx = dentry; + hint_femp.count = 0; + } + + ei->hint_femp = hint_femp; + + return -ENOENT; +} + +int exfat_read_volume_label(struct super_block *sb, struct exfat_uni_name *label_out) +{ + int ret, i; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct exfat_entry_set_cache es; + struct exfat_dentry *ep; + + mutex_lock(&sbi->s_lock); + + memset(label_out, 0, sizeof(*label_out)); + ret = exfat_get_volume_label_dentry(sb, &es); + if (ret < 0) { + /* + * ENOENT signifies that a volume label dentry doesn't exist + * We will treat this as an empty volume label and not fail. + */ + if (ret == -ENOENT) + ret = 0; + + goto unlock; + } + + ep = exfat_get_dentry_cached(&es, 0); + label_out->name_len = ep->dentry.volume_label.char_count; + if (label_out->name_len > EXFAT_VOLUME_LABEL_LEN) { + ret = -EIO; + exfat_put_dentry_set(&es, false); + goto unlock; + } + + for (i = 0; i < label_out->name_len; i++) + label_out->name[i] = le16_to_cpu(ep->dentry.volume_label.volume_label[i]); + + exfat_put_dentry_set(&es, false); +unlock: + mutex_unlock(&sbi->s_lock); + return ret; +} + +int exfat_write_volume_label(struct super_block *sb, + struct exfat_uni_name *label) +{ + int ret, i; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct inode *root_inode = sb->s_root->d_inode; + struct exfat_entry_set_cache es; + struct exfat_chain clu; + struct exfat_dentry *ep; + + if (label->name_len > EXFAT_VOLUME_LABEL_LEN) + return -EINVAL; + + mutex_lock(&sbi->s_lock); + + ret = exfat_get_volume_label_dentry(sb, &es); + if (ret == -ENOENT) { + if (label->name_len == 0) { + /* No volume label dentry, no need to clear */ + ret = 0; + goto unlock; + } + + ret = exfat_find_empty_entry(root_inode, &clu, 1, &es); + } + + if (ret < 0) + goto unlock; + + ep = exfat_get_dentry_cached(&es, 0); + + if (label->name_len == 0 && ep->dentry.volume_label.char_count == 0) { + /* volume label had been cleared */ + exfat_put_dentry_set(&es, 0); + goto unlock; + } + + memset(ep, 0, sizeof(*ep)); + ep->type = EXFAT_VOLUME; + + for (i = 0; i < label->name_len; i++) + ep->dentry.volume_label.volume_label[i] = + cpu_to_le16(label->name[i]); + + ep->dentry.volume_label.char_count = label->name_len; + es.modified = true; + + ret = exfat_put_dentry_set(&es, IS_DIRSYNC(root_inode)); + +unlock: + mutex_unlock(&sbi->s_lock); + return ret; +} diff --git a/fs/exfat/exfat_fs.h b/fs/exfat/exfat_fs.h new file mode 100644 index 000000000..a9131fe03 --- /dev/null +++ b/fs/exfat/exfat_fs.h @@ -0,0 +1,704 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2012-2013 Samsung Electronics Co., Ltd. + */ + +#ifndef _EXFAT_FS_H +#define _EXFAT_FS_H + +#include +#include +#include +#include +#include +#include +#include + +#define EXFAT_ROOT_INO 1 + +/* + * exfat error flags + */ +enum exfat_error_mode { + EXFAT_ERRORS_CONT, /* ignore error and continue */ + EXFAT_ERRORS_PANIC, /* panic on error */ + EXFAT_ERRORS_RO, /* remount r/o on error */ +}; + +/* + * exfat nls lossy flag + */ +enum { + NLS_NAME_NO_LOSSY = 0, /* no lossy */ + NLS_NAME_LOSSY = 1 << 0, /* just detected incorrect filename(s) */ +}; + +#define EXFAT_HASH_BITS 8 +#define EXFAT_HASH_SIZE (1UL << EXFAT_HASH_BITS) + +/* + * Type Definitions + */ +#define ES_2_ENTRIES 2 +#define ES_ALL_ENTRIES 0 + +#define ES_IDX_FILE 0 +#define ES_IDX_STREAM 1 +#define ES_IDX_FIRST_FILENAME 2 +#define EXFAT_FILENAME_ENTRY_NUM(name_len) \ + DIV_ROUND_UP(name_len, EXFAT_FILE_NAME_LEN) +#define ES_IDX_LAST_FILENAME(name_len) \ + (ES_IDX_FIRST_FILENAME + EXFAT_FILENAME_ENTRY_NUM(name_len) - 1) + +#define DIR_DELETED 0xFFFFFFF7 + +/* type values */ +#define TYPE_UNUSED 0x0000 +#define TYPE_DELETED 0x0001 +#define TYPE_INVALID 0x0002 +#define TYPE_CRITICAL_PRI 0x0100 +#define TYPE_BITMAP 0x0101 +#define TYPE_UPCASE 0x0102 +#define TYPE_VOLUME 0x0103 +#define TYPE_DIR 0x0104 +#define TYPE_FILE 0x011F +#define TYPE_CRITICAL_SEC 0x0200 +#define TYPE_STREAM 0x0201 +#define TYPE_EXTEND 0x0202 +#define TYPE_ACL 0x0203 +#define TYPE_BENIGN_PRI 0x0400 +#define TYPE_GUID 0x0401 +#define TYPE_PADDING 0x0402 +#define TYPE_ACLTAB 0x0403 +#define TYPE_BENIGN_SEC 0x0800 +#define TYPE_VENDOR_EXT 0x0801 +#define TYPE_VENDOR_ALLOC 0x0802 + +#define MAX_CHARSET_SIZE 6 /* max size of multi-byte character */ +#define MAX_NAME_LENGTH 255 /* max len of file name excluding NULL */ +#define MAX_VFSNAME_BUF_SIZE ((MAX_NAME_LENGTH + 1) * MAX_CHARSET_SIZE) + +#define EXFAT_HINT_NONE -1 +#define EXFAT_MIN_SUBDIR 2 + +#define EXFAT_BLK_RA_SIZE(sb) \ + (min_t(blkcnt_t, (sb)->s_bdi->ra_pages, (sb)->s_bdi->io_pages) \ + << (PAGE_SHIFT - (sb)->s_blocksize_bits)) + +/* + * helpers for fat entry. + */ +#define FAT_ENT_SIZE (4) +#define FAT_ENT_SIZE_BITS (2) +#define FAT_ENT_OFFSET_SECTOR(sb, loc) (EXFAT_SB(sb)->FAT1_start_sector + \ + (((u64)(loc) << FAT_ENT_SIZE_BITS) >> sb->s_blocksize_bits)) +#define FAT_ENT_OFFSET_BYTE_IN_SECTOR(sb, loc) \ + (((loc) << FAT_ENT_SIZE_BITS) & (sb->s_blocksize - 1)) + +/* + * helpers for bitmap. + */ +#define CLUSTER_TO_BITMAP_ENT(clu) ((clu) - EXFAT_RESERVED_CLUSTERS) +#define BITMAP_ENT_TO_CLUSTER(ent) ((ent) + EXFAT_RESERVED_CLUSTERS) +#define BITS_PER_SECTOR(sb) ((sb)->s_blocksize * BITS_PER_BYTE) +#define BITS_PER_SECTOR_MASK(sb) (BITS_PER_SECTOR(sb) - 1) +#define BITMAP_OFFSET_SECTOR_INDEX(sb, ent) \ + ((ent / BITS_PER_BYTE) >> (sb)->s_blocksize_bits) +#define BITMAP_OFFSET_BIT_IN_SECTOR(sb, ent) (ent & BITS_PER_SECTOR_MASK(sb)) +#define BITMAP_OFFSET_BYTE_IN_SECTOR(sb, ent) \ + ((ent / BITS_PER_BYTE) & ((sb)->s_blocksize - 1)) +#define IGNORED_BITS_REMAINED(clu, clu_base) ((1UL << ((clu) - (clu_base))) - 1) + +#define ES_ENTRY_NUM(name_len) (ES_IDX_LAST_FILENAME(name_len) + 1) +/* 19 entries = 1 file entry + 1 stream entry + 17 filename entries */ +#define ES_MAX_ENTRY_NUM ES_ENTRY_NUM(MAX_NAME_LENGTH) + +/* + * 19 entries x 32 bytes/entry = 608 bytes. + * The 608 bytes are in 3 sectors at most (even 512 Byte sector). + */ +#define DIR_CACHE_SIZE \ + (DIV_ROUND_UP(ES_MAX_ENTRY_NUM << DENTRY_SIZE_BITS, SECTOR_SIZE) + 1) + +/* Superblock flags */ +#define EXFAT_FLAGS_SHUTDOWN 1 + +struct exfat_dentry_namebuf { + char *lfn; + int lfnbuf_len; /* usually MAX_UNINAME_BUF_SIZE */ +}; + +/* unicode name structure */ +struct exfat_uni_name { + /* +3 for null and for converting */ + unsigned short name[MAX_NAME_LENGTH + 3]; + u16 name_hash; + unsigned char name_len; +}; + +/* directory structure */ +struct exfat_chain { + unsigned int dir; + unsigned int size; + unsigned char flags; +}; + +/* first empty entry hint information */ +struct exfat_hint_femp { + /* entry index of a directory */ + int eidx; + /* count of continuous empty entry */ + int count; + /* the cluster that first empty slot exists in */ + struct exfat_chain cur; +}; + +/* hint structure */ +struct exfat_hint { + unsigned int clu; + union { + unsigned int off; /* cluster offset */ + int eidx; /* entry index */ + }; +}; + +struct exfat_entry_set_cache { + struct super_block *sb; + unsigned int start_off; + int num_bh; + struct buffer_head *__bh[DIR_CACHE_SIZE]; + struct buffer_head **bh; + unsigned int num_entries; + bool modified; +}; + +#define IS_DYNAMIC_ES(es) ((es)->__bh != (es)->bh) + +struct exfat_dir_entry { + /* the cluster where file dentry is located */ + struct exfat_chain dir; + /* the index of file dentry in ->dir */ + int entry; + unsigned int type; + unsigned int start_clu; + unsigned char flags; + unsigned short attr; + loff_t size; + loff_t valid_size; + unsigned int num_subdirs; + struct timespec64 atime; + struct timespec64 mtime; + struct timespec64 crtime; + struct exfat_dentry_namebuf namebuf; +}; + +/* + * exfat mount in-memory data + */ +struct exfat_mount_options { + kuid_t fs_uid; + kgid_t fs_gid; + unsigned short fs_fmask; + unsigned short fs_dmask; + /* permission for setting the [am]time */ + unsigned short allow_utime; + /* charset for filename input/display */ + char *iocharset; + /* on error: continue, panic, remount-ro */ + enum exfat_error_mode errors; + unsigned utf8:1, /* Use of UTF-8 character set */ + sys_tz:1, /* Use local timezone */ + discard:1, /* Issue discard requests on deletions */ + keep_last_dots:1; /* Keep trailing periods in paths */ + int time_offset; /* Offset of timestamps from UTC (in minutes) */ + /* Support creating zero-size directory, default: false */ + bool zero_size_dir; +}; + +/* + * EXFAT file system superblock in-memory data + */ +struct exfat_sb_info { + unsigned long long num_sectors; /* num of sectors in volume */ + unsigned int num_clusters; /* num of clusters in volume */ + unsigned int cluster_size; /* cluster size in bytes */ + unsigned int cluster_size_bits; + unsigned int sect_per_clus; /* cluster size in sectors */ + unsigned int sect_per_clus_bits; + unsigned long long FAT1_start_sector; /* FAT1 start sector */ + unsigned long long FAT2_start_sector; /* FAT2 start sector */ + unsigned long long data_start_sector; /* data area start sector */ + unsigned long long data_start_bytes; + unsigned int num_FAT_sectors; /* num of FAT sectors */ + unsigned int root_dir; /* root dir cluster */ + unsigned int dentries_per_clu; /* num of dentries per cluster */ + unsigned int vol_flags; /* volume flags */ + unsigned int vol_flags_persistent; /* volume flags to retain */ + struct buffer_head *boot_bh; /* buffer_head of BOOT sector */ + + unsigned int map_clu; /* allocation bitmap start cluster */ + unsigned int map_sectors; /* num of allocation bitmap sectors */ + struct buffer_head **vol_amap; /* allocation bitmap */ + + unsigned short *vol_utbl; /* upcase table */ + + unsigned int clu_srch_ptr; /* cluster search pointer */ + unsigned int used_clusters; /* number of used clusters */ + + unsigned long s_exfat_flags; /* Exfat superblock flags */ + + struct mutex s_lock; /* superblock lock */ + struct mutex bitmap_lock; /* bitmap lock */ + struct exfat_mount_options options; + struct nls_table *nls_io; /* Charset used for input and display */ + struct ratelimit_state ratelimit; + + spinlock_t inode_hash_lock; + struct hlist_head inode_hashtable[EXFAT_HASH_SIZE]; + struct rcu_head rcu; +}; + +#define EXFAT_CACHE_VALID 0 + +/* + * EXFAT file system inode in-memory data + */ +struct exfat_inode_info { + /* the cluster where file dentry is located */ + struct exfat_chain dir; + /* the index of file dentry in ->dir */ + int entry; + unsigned int type; + unsigned short attr; + unsigned int start_clu; + unsigned char flags; + /* + * the copy of low 32bit of i_version to check + * the validation of hint_stat. + */ + unsigned int version; + + /* hint for cluster last accessed */ + struct exfat_hint hint_bmap; + /* hint for entry index we try to lookup next time */ + struct exfat_hint hint_stat; + /* hint for first empty entry */ + struct exfat_hint_femp hint_femp; + + spinlock_t cache_lru_lock; + struct list_head cache_lru; + int nr_caches; + /* for avoiding the race between alloc and free */ + unsigned int cache_valid_id; + + /* on-disk position of directory entry or 0 */ + loff_t i_pos; + loff_t valid_size; + /* block-aligned size zeroed in the page cache (>= valid_size) */ + loff_t zeroed_size; + /* hash by i_location */ + struct hlist_node i_hash_fat; + struct inode vfs_inode; + /* File creation time */ + struct timespec64 i_crtime; +}; + +static inline struct exfat_sb_info *EXFAT_SB(struct super_block *sb) +{ + return sb->s_fs_info; +} + +static inline struct exfat_inode_info *EXFAT_I(struct inode *inode) +{ + return container_of(inode, struct exfat_inode_info, vfs_inode); +} + +static inline int exfat_forced_shutdown(struct super_block *sb) +{ + return test_bit(EXFAT_FLAGS_SHUTDOWN, &EXFAT_SB(sb)->s_exfat_flags); +} + +/* + * If ->i_mode can't hold 0222 (i.e. ATTR_RO), we use ->i_attrs to + * save ATTR_RO instead of ->i_mode. + * + * If it's directory and !sbi->options.rodir, ATTR_RO isn't read-only + * bit, it's just used as flag for app. + */ +static inline int exfat_mode_can_hold_ro(struct inode *inode) +{ + struct exfat_sb_info *sbi = EXFAT_SB(inode->i_sb); + + if (S_ISDIR(inode->i_mode)) + return 0; + + if ((~sbi->options.fs_fmask) & 0222) + return 1; + return 0; +} + +/* Convert attribute bits and a mask to the UNIX mode. */ +static inline mode_t exfat_make_mode(struct exfat_sb_info *sbi, + unsigned short attr, mode_t mode) +{ + if ((attr & EXFAT_ATTR_READONLY) && !(attr & EXFAT_ATTR_SUBDIR)) + mode &= ~0222; + + if (attr & EXFAT_ATTR_SUBDIR) + return (mode & ~sbi->options.fs_dmask) | S_IFDIR; + + return (mode & ~sbi->options.fs_fmask) | S_IFREG; +} + +/* Return the FAT attribute byte for this inode */ +static inline unsigned short exfat_make_attr(struct inode *inode) +{ + unsigned short attr = EXFAT_I(inode)->attr; + + if (S_ISDIR(inode->i_mode)) + attr |= EXFAT_ATTR_SUBDIR; + if (exfat_mode_can_hold_ro(inode) && !(inode->i_mode & 0222)) + attr |= EXFAT_ATTR_READONLY; + return attr; +} + +static inline void exfat_save_attr(struct inode *inode, unsigned short attr) +{ + if (exfat_mode_can_hold_ro(inode)) + EXFAT_I(inode)->attr = attr & (EXFAT_ATTR_RWMASK | EXFAT_ATTR_READONLY); + else + EXFAT_I(inode)->attr = attr & EXFAT_ATTR_RWMASK; +} + +static inline bool exfat_is_last_sector_in_cluster(struct exfat_sb_info *sbi, + sector_t sec) +{ + return ((sec - sbi->data_start_sector + 1) & + ((1 << sbi->sect_per_clus_bits) - 1)) == 0; +} + +static inline sector_t exfat_cluster_to_sector(struct exfat_sb_info *sbi, + unsigned int clus) +{ + return ((sector_t)(clus - EXFAT_RESERVED_CLUSTERS) << sbi->sect_per_clus_bits) + + sbi->data_start_sector; +} + +static inline unsigned int exfat_sector_to_cluster(struct exfat_sb_info *sbi, + sector_t sec) +{ + return ((sec - sbi->data_start_sector) >> sbi->sect_per_clus_bits) + + EXFAT_RESERVED_CLUSTERS; +} + +static inline bool is_valid_cluster(struct exfat_sb_info *sbi, + unsigned int clus) +{ + return clus >= EXFAT_FIRST_CLUSTER && clus < sbi->num_clusters; +} + +static inline loff_t exfat_ondisk_size(const struct inode *inode) +{ + return ((loff_t)inode->i_blocks) << 9; +} + +static inline loff_t exfat_cluster_to_phys_bytes(struct exfat_sb_info *sbi, + unsigned int clus) +{ + return ((loff_t)(clus - EXFAT_RESERVED_CLUSTERS) << sbi->cluster_size_bits) + + sbi->data_start_bytes; +} + +/* + * helpers for cluster size to byte conversion. + */ +static inline loff_t exfat_cluster_to_bytes(struct exfat_sb_info *sbi, + u32 nr_clusters) +{ + return (loff_t)nr_clusters << sbi->cluster_size_bits; +} + +static inline blkcnt_t exfat_cluster_to_sectors(struct exfat_sb_info *sbi, + u32 nr_clusters) +{ + return (blkcnt_t)nr_clusters << (sbi->cluster_size_bits - 9); +} + +static inline u32 exfat_bytes_to_cluster(struct exfat_sb_info *sbi, loff_t size) +{ + return (u32)(size >> sbi->cluster_size_bits); +} + +static inline u32 exfat_bytes_to_cluster_round_up(struct exfat_sb_info *sbi, + loff_t size) +{ + if (size <= 0) + return 0; + return (u32)((size - 1) >> sbi->cluster_size_bits) + 1; +} + +static inline u32 exfat_cluster_offset(struct exfat_sb_info *sbi, loff_t off) +{ + return off & (sbi->cluster_size - 1); +} + +/* + * helpers for block size to byte conversion. + */ +static inline loff_t exfat_block_to_bytes(struct super_block *sb, + sector_t block) +{ + return (loff_t)block << sb->s_blocksize_bits; +} + +static inline sector_t exfat_bytes_to_block(struct super_block *sb, loff_t size) +{ + return (sector_t)(size >> sb->s_blocksize_bits); +} + +static inline sector_t exfat_bytes_to_block_round_up(struct super_block *sb, + loff_t size) +{ + if (size <= 0) + return 0; + return (sector_t)(((size - 1) >> sb->s_blocksize_bits) + 1); +} + +static inline u32 exfat_block_offset(struct super_block *sb, loff_t off) +{ + return (u32)(off & (sb->s_blocksize - 1)); +} + +/* + * helpers for block size to dentry size conversion. + */ +static inline u32 exfat_bytes_to_dentries(loff_t b) +{ + return (u32)(b >> DENTRY_SIZE_BITS); +} + +static inline u32 exfat_dentries_to_bytes(u32 dentry) +{ + return dentry << DENTRY_SIZE_BITS; +} + +/* + * helpers for cluster size to dentry size conversion. + */ +static inline u64 exfat_cluster_to_dentries(struct exfat_sb_info *sbi, + u32 nr_clusters) +{ + return (u64)nr_clusters << (sbi->cluster_size_bits - DENTRY_SIZE_BITS); +} + +static inline u32 exfat_dentries_to_cluster(struct exfat_sb_info *sbi, + u32 dentry) +{ + return dentry >> (sbi->cluster_size_bits - DENTRY_SIZE_BITS); +} + +/* super.c */ +int exfat_set_volume_dirty(struct super_block *sb); +int exfat_clear_volume_dirty(struct super_block *sb); + +/* fatent.c */ +#define exfat_get_next_cluster(sb, pclu) \ + exfat_cluster_walk(sb, (pclu), 1, ALLOC_FAT_CHAIN) + +int exfat_alloc_cluster(struct inode *inode, unsigned int num_alloc, + struct exfat_chain *p_chain, bool sync_bmap, bool contig); +int exfat_free_cluster(struct inode *inode, struct exfat_chain *p_chain); +int exfat_ent_get(struct super_block *sb, unsigned int loc, + unsigned int *content, struct buffer_head **last); +int exfat_ent_set(struct super_block *sb, unsigned int loc, + unsigned int content); +int exfat_chain_cont_cluster(struct super_block *sb, unsigned int chain, + unsigned int len); +int exfat_zeroed_cluster(struct inode *dir, unsigned int clu); +int exfat_find_last_cluster(struct super_block *sb, struct exfat_chain *p_chain, + unsigned int *ret_clu); +int exfat_count_num_clusters(struct super_block *sb, + struct exfat_chain *p_chain, unsigned int *ret_count); +int exfat_blk_readahead(struct super_block *sb, sector_t sec, + sector_t *ra, blkcnt_t *ra_cnt, sector_t end); + +static inline int +exfat_cluster_walk(struct super_block *sb, unsigned int *clu, + unsigned int step, int flags) +{ + struct buffer_head *bh = NULL; + + if (flags == ALLOC_NO_FAT_CHAIN) { + (*clu) += step; + return 0; + } + + while (step--) { + if (exfat_ent_get(sb, *clu, clu, &bh)) + return -EIO; + } + brelse(bh); + + return 0; +} + +/* balloc.c */ +int exfat_load_bitmap(struct super_block *sb); +void exfat_free_bitmap(struct exfat_sb_info *sbi); +int exfat_set_bitmap(struct super_block *sb, unsigned int clu, bool sync); +int exfat_clear_bitmap(struct super_block *sb, unsigned int clu, bool sync); +bool exfat_test_bitmap(struct super_block *sb, unsigned int clu); +unsigned int exfat_find_free_bitmap(struct super_block *sb, unsigned int clu); +int exfat_count_used_clusters(struct super_block *sb, unsigned int *ret_count); +int exfat_trim_fs(struct inode *inode, struct fstrim_range *range); + +/* file.c */ +extern const struct file_operations exfat_file_operations; +int __exfat_truncate(struct inode *inode); +int exfat_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr); +int exfat_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, unsigned int request_mask, + unsigned int query_flags); +struct file_kattr; +int exfat_fileattr_get(struct dentry *dentry, struct file_kattr *fa); +int exfat_file_fsync(struct file *file, loff_t start, loff_t end, int datasync); +long exfat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); +long exfat_compat_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg); +int exfat_force_shutdown(struct super_block *sb, u32 flags); + +/* namei.c */ +extern const struct dentry_operations exfat_dentry_ops; +extern const struct dentry_operations exfat_utf8_dentry_ops; +int exfat_find_empty_entry(struct inode *inode, + struct exfat_chain *p_dir, int num_entries, + struct exfat_entry_set_cache *es); + +/* cache.c */ +int exfat_cache_init(void); +void exfat_cache_shutdown(void); +void exfat_cache_inval_inode(struct inode *inode); +int exfat_get_cluster(struct inode *inode, unsigned int cluster, + unsigned int *dclus, unsigned int *count, unsigned int *last_dclus); + +/* dir.c */ +extern const struct inode_operations exfat_dir_inode_operations; +extern const struct file_operations exfat_dir_operations; +unsigned int exfat_get_entry_type(struct exfat_dentry *p_entry); +void exfat_init_dir_entry(struct exfat_entry_set_cache *es, + unsigned int type, unsigned int start_clu, + unsigned long long size, struct timespec64 *ts); +void exfat_init_ext_entry(struct exfat_entry_set_cache *es, int num_entries, + struct exfat_uni_name *p_uniname, + struct exfat_entry_set_cache *old_es, int num_extra); +void exfat_remove_entries(struct inode *inode, struct exfat_entry_set_cache *es, + int order, bool free_benign); +void exfat_update_dir_chksum(struct exfat_entry_set_cache *es); +int exfat_calc_num_entries(struct exfat_uni_name *p_uniname); +int exfat_find_dir_entry(struct super_block *sb, struct exfat_inode_info *ei, + struct exfat_chain *p_dir, struct exfat_uni_name *p_uniname, + struct exfat_hint *hint_opt); +int exfat_alloc_new_dir(struct inode *inode, struct exfat_chain *clu); +struct exfat_dentry *exfat_get_dentry(struct super_block *sb, + struct exfat_chain *p_dir, int entry, struct buffer_head **bh); +struct exfat_dentry *exfat_get_dentry_cached(struct exfat_entry_set_cache *es, + int num); +int exfat_get_dentry_set(struct exfat_entry_set_cache *es, + struct super_block *sb, struct exfat_chain *p_dir, int entry, + unsigned int num_entries); +#define exfat_get_dentry_set_by_ei(es, sb, ei) \ + exfat_get_dentry_set(es, sb, &(ei)->dir, (ei)->entry, ES_ALL_ENTRIES) +int exfat_get_empty_dentry_set(struct exfat_entry_set_cache *es, + struct super_block *sb, struct exfat_chain *p_dir, int entry, + unsigned int num_entries); +int exfat_put_dentry_set(struct exfat_entry_set_cache *es, int sync); +int exfat_count_dir_entries(struct super_block *sb, struct exfat_chain *p_dir); +int exfat_read_volume_label(struct super_block *sb, + struct exfat_uni_name *label_out); +int exfat_write_volume_label(struct super_block *sb, + struct exfat_uni_name *label); + +static inline int exfat_chain_advance(struct super_block *sb, + struct exfat_chain *chain, unsigned int step) +{ + unsigned int clu = chain->dir; + + if (unlikely(chain->size < step)) + return -EIO; + + if (exfat_cluster_walk(sb, &clu, step, chain->flags)) + return -EIO; + + chain->size -= step; + + if (chain->size == 0 && chain->flags == ALLOC_NO_FAT_CHAIN) + chain->dir = EXFAT_EOF_CLUSTER; + else + chain->dir = clu; + + return 0; +} + +/* inode.c */ +extern const struct inode_operations exfat_file_inode_operations; +void exfat_sync_inode(struct inode *inode); +struct inode *exfat_build_inode(struct super_block *sb, + struct exfat_dir_entry *info, loff_t i_pos); +void exfat_hash_inode(struct inode *inode, loff_t i_pos); +void exfat_unhash_inode(struct inode *inode); +struct inode *exfat_iget(struct super_block *sb, loff_t i_pos); +int __exfat_write_inode(struct inode *inode, int sync); +int exfat_write_inode(struct inode *inode, struct writeback_control *wbc); +void exfat_evict_inode(struct inode *inode); +int exfat_map_cluster(struct inode *inode, unsigned int clu_offset, + unsigned int *clu, unsigned int *count, int create, + bool *balloc); + +/* exfat/nls.c */ +unsigned short exfat_toupper(struct super_block *sb, unsigned short a); +int exfat_uniname_ncmp(struct super_block *sb, unsigned short *a, + unsigned short *b, unsigned int len); +int exfat_utf16_to_nls(struct super_block *sb, + struct exfat_uni_name *uniname, unsigned char *p_cstring, + int len); +int exfat_nls_to_utf16(struct super_block *sb, + const unsigned char *p_cstring, const int len, + struct exfat_uni_name *uniname, int *p_lossy); +int exfat_create_upcase_table(struct super_block *sb); +void exfat_free_upcase_table(struct exfat_sb_info *sbi); + +/* exfat/misc.c */ +void __exfat_fs_error(struct super_block *sb, int report, const char *fmt, ...) + __printf(3, 4) __cold; +#define exfat_fs_error(sb, fmt, args...) \ + __exfat_fs_error(sb, 1, fmt, ## args) +#define exfat_fs_error_ratelimit(sb, fmt, args...) \ + __exfat_fs_error(sb, __ratelimit(&EXFAT_SB(sb)->ratelimit), \ + fmt, ## args) + +/* expand to pr_*() with prefix */ +#define exfat_err(sb, fmt, ...) \ + pr_err("exFAT-fs (%s): " fmt "\n", (sb)->s_id, ##__VA_ARGS__) +#define exfat_warn(sb, fmt, ...) \ + pr_warn("exFAT-fs (%s): " fmt "\n", (sb)->s_id, ##__VA_ARGS__) +#define exfat_info(sb, fmt, ...) \ + pr_info("exFAT-fs (%s): " fmt "\n", (sb)->s_id, ##__VA_ARGS__) +#define exfat_debug(sb, fmt, ...) \ + pr_debug("exFAT-fs (%s): " fmt "\n", (sb)->s_id, ##__VA_ARGS__) + +void exfat_get_entry_time(struct exfat_sb_info *sbi, struct timespec64 *ts, + u8 tz, __le16 time, __le16 date, u8 time_cs); +void exfat_truncate_atime(struct timespec64 *ts); +void exfat_truncate_inode_atime(struct inode *inode); +void exfat_set_entry_time(struct exfat_sb_info *sbi, struct timespec64 *ts, + u8 *tz, __le16 *time, __le16 *date, u8 *time_cs); +u16 exfat_calc_chksum16(void *data, int len, u16 chksum, int type); +u32 exfat_calc_chksum32(void *data, int len, u32 chksum, int type); +int exfat_update_bh(struct buffer_head *bh, int sync); +int exfat_update_bhs(struct buffer_head **bhs, int nr_bhs, int sync); +void exfat_chain_set(struct exfat_chain *ec, unsigned int dir, + unsigned int size, unsigned char flags); +void exfat_chain_dup(struct exfat_chain *dup, struct exfat_chain *ec); + +#endif /* !_EXFAT_FS_H */ diff --git a/fs/exfat/exfat_raw.h b/fs/exfat/exfat_raw.h new file mode 100644 index 000000000..ec70cd35b --- /dev/null +++ b/fs/exfat/exfat_raw.h @@ -0,0 +1,197 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2012-2013 Samsung Electronics Co., Ltd. + */ + +#ifndef _EXFAT_RAW_H +#define _EXFAT_RAW_H + +#include + +#define BOOT_SIGNATURE 0xAA55 +#define EXBOOT_SIGNATURE 0xAA550000 +#define STR_EXFAT "EXFAT " /* size should be 8 */ + +#define EXFAT_MAX_FILE_LEN 255 + +#define VOLUME_DIRTY 0x0002 +#define MEDIA_FAILURE 0x0004 + +#define EXFAT_EOF_CLUSTER 0xFFFFFFFFu +#define EXFAT_BAD_CLUSTER 0xFFFFFFF7u +#define EXFAT_FREE_CLUSTER 0 +/* Cluster 0, 1 are reserved, the first cluster is 2 in the cluster heap. */ +#define EXFAT_RESERVED_CLUSTERS 2 +#define EXFAT_FIRST_CLUSTER 2 +#define EXFAT_DATA_CLUSTER_COUNT(sbi) \ + ((sbi)->num_clusters - EXFAT_RESERVED_CLUSTERS) +#define EXFAT_MAX_NUM_CLUSTER (0xFFFFFFF5) + +/* AllocationPossible and NoFatChain field in GeneralSecondaryFlags Field */ +#define ALLOC_POSSIBLE 0x01 +#define ALLOC_FAT_CHAIN 0x01 +#define ALLOC_NO_FAT_CHAIN 0x03 + +#define DENTRY_SIZE 32 /* directory entry size */ +#define DENTRY_SIZE_BITS 5 +/* exFAT allows 8388608(256MB) directory entries */ +#define MAX_EXFAT_DENTRIES 8388608 + +/* dentry types */ +#define EXFAT_UNUSED 0x00 /* end of directory */ +#define EXFAT_DELETE (~0x80) +#define IS_EXFAT_DELETED(x) ((x) < 0x80) /* deleted file (0x01~0x7F) */ +#define EXFAT_INVAL 0x80 /* invalid value */ +#define EXFAT_BITMAP 0x81 /* allocation bitmap */ +#define EXFAT_UPCASE 0x82 /* upcase table */ +#define EXFAT_VOLUME 0x83 /* volume label */ +#define EXFAT_FILE 0x85 /* file or dir */ +#define EXFAT_GUID 0xA0 +#define EXFAT_PADDING 0xA1 +#define EXFAT_ACLTAB 0xA2 +#define EXFAT_STREAM 0xC0 /* stream entry */ +#define EXFAT_NAME 0xC1 /* file name entry */ +#define EXFAT_ACL 0xC2 /* stream entry */ +#define EXFAT_VENDOR_EXT 0xE0 /* vendor extension entry */ +#define EXFAT_VENDOR_ALLOC 0xE1 /* vendor allocation entry */ + +#define IS_EXFAT_CRITICAL_PRI(x) (x < 0xA0) +#define IS_EXFAT_BENIGN_PRI(x) (x < 0xC0) +#define IS_EXFAT_CRITICAL_SEC(x) (x < 0xE0) + +/* checksum types */ +#define CS_DIR_ENTRY 0 +#define CS_BOOT_SECTOR 1 +#define CS_DEFAULT 2 + +/* file attributes */ +#define EXFAT_ATTR_READONLY 0x0001 +#define EXFAT_ATTR_HIDDEN 0x0002 +#define EXFAT_ATTR_SYSTEM 0x0004 +#define EXFAT_ATTR_VOLUME 0x0008 +#define EXFAT_ATTR_SUBDIR 0x0010 +#define EXFAT_ATTR_ARCHIVE 0x0020 + +#define EXFAT_ATTR_RWMASK (EXFAT_ATTR_HIDDEN | EXFAT_ATTR_SYSTEM | \ + EXFAT_ATTR_VOLUME | EXFAT_ATTR_SUBDIR | \ + EXFAT_ATTR_ARCHIVE) + +#define BOOTSEC_JUMP_BOOT_LEN 3 +#define BOOTSEC_FS_NAME_LEN 8 +#define BOOTSEC_OLDBPB_LEN 53 + +#define EXFAT_FILE_NAME_LEN 15 +#define EXFAT_VOLUME_LABEL_LEN 11 + +#define EXFAT_MIN_SECT_SIZE_BITS 9 +#define EXFAT_MAX_SECT_SIZE_BITS 12 +#define EXFAT_MAX_SECT_PER_CLUS_BITS(x) (25 - (x)->sect_size_bits) + +/* EXFAT: Main and Backup Boot Sector (512 bytes) */ +struct boot_sector { + __u8 jmp_boot[BOOTSEC_JUMP_BOOT_LEN]; + __u8 fs_name[BOOTSEC_FS_NAME_LEN]; + __u8 must_be_zero[BOOTSEC_OLDBPB_LEN]; + __le64 partition_offset; + __le64 vol_length; + __le32 fat_offset; + __le32 fat_length; + __le32 clu_offset; + __le32 clu_count; + __le32 root_cluster; + __le32 vol_serial; + __u8 fs_revision[2]; + __le16 vol_flags; + __u8 sect_size_bits; + __u8 sect_per_clus_bits; + __u8 num_fats; + __u8 drv_sel; + __u8 percent_in_use; + __u8 reserved[7]; + __u8 boot_code[390]; + __le16 signature; +} __packed; + +struct exfat_dentry { + __u8 type; + union { + struct { + __u8 num_ext; + __le16 checksum; + __le16 attr; + __le16 reserved1; + __le16 create_time; + __le16 create_date; + __le16 modify_time; + __le16 modify_date; + __le16 access_time; + __le16 access_date; + __u8 create_time_cs; + __u8 modify_time_cs; + __u8 create_tz; + __u8 modify_tz; + __u8 access_tz; + __u8 reserved2[7]; + } __packed file; /* file directory entry */ + struct { + __u8 flags; + __u8 reserved1; + __u8 name_len; + __le16 name_hash; + __le16 reserved2; + __le64 valid_size; + __le32 reserved3; + __le32 start_clu; + __le64 size; + } __packed stream; /* stream extension directory entry */ + struct { + __u8 flags; + __le16 unicode_0_14[EXFAT_FILE_NAME_LEN]; + } __packed name; /* file name directory entry */ + struct { + __u8 flags; + __u8 reserved[18]; + __le32 start_clu; + __le64 size; + } __packed bitmap; /* allocation bitmap directory entry */ + struct { + __u8 reserved1[3]; + __le32 checksum; + __u8 reserved2[12]; + __le32 start_clu; + __le64 size; + } __packed upcase; /* up-case table directory entry */ + struct { + __u8 char_count; + __le16 volume_label[EXFAT_VOLUME_LABEL_LEN]; + __u8 reserved[8]; + } __packed volume_label; /* volume label directory entry */ + struct { + __u8 flags; + __u8 vendor_guid[16]; + __u8 vendor_defined[14]; + } __packed vendor_ext; /* vendor extension directory entry */ + struct { + __u8 flags; + __u8 vendor_guid[16]; + __u8 vendor_defined[2]; + __le32 start_clu; + __le64 size; + } __packed vendor_alloc; /* vendor allocation directory entry */ + struct { + __u8 flags; + __u8 custom_defined[18]; + __le32 start_clu; + __le64 size; + } __packed generic_secondary; /* generic secondary directory entry */ + } __packed dentry; +} __packed; + +#define EXFAT_TZ_VALID (1 << 7) + +/* Jan 1 GMT 00:00:00 1980 */ +#define EXFAT_MIN_TIMESTAMP_SECS 315532800LL +/* Dec 31 GMT 23:59:59 2107 */ +#define EXFAT_MAX_TIMESTAMP_SECS 4354819199LL + +#endif /* !_EXFAT_RAW_H */ diff --git a/fs/exfat/fatent.c b/fs/exfat/fatent.c new file mode 100644 index 000000000..a8b11e2ce --- /dev/null +++ b/fs/exfat/fatent.c @@ -0,0 +1,585 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2012-2013 Samsung Electronics Co., Ltd. + */ + +#include +#include +#include +#include + +#include "exfat_raw.h" +#include "exfat_fs.h" + +static int exfat_mirror_bh(struct super_block *sb, struct buffer_head *bh) +{ + struct buffer_head *c_bh; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + sector_t sec = bh->b_blocknr; + sector_t sec2; + int err = 0; + + if (sbi->FAT2_start_sector != sbi->FAT1_start_sector) { + sec2 = sec - sbi->FAT1_start_sector + sbi->FAT2_start_sector; + c_bh = sb_getblk(sb, sec2); + if (!c_bh) + return -ENOMEM; + memcpy(c_bh->b_data, bh->b_data, sb->s_blocksize); + err = exfat_update_bh(c_bh, sb->s_flags & SB_SYNCHRONOUS); + brelse(c_bh); + } + + return err; +} + +static int exfat_end_bh(struct super_block *sb, struct buffer_head *bh) +{ + int err; + + err = exfat_update_bh(bh, sb->s_flags & SB_SYNCHRONOUS); + if (!err) + err = exfat_mirror_bh(sb, bh); + brelse(bh); + return err; +} + +static int __exfat_ent_get(struct super_block *sb, unsigned int loc, + unsigned int *content, struct buffer_head **cache) +{ + unsigned int off; + sector_t sec; + struct buffer_head *bh = *cache; + + sec = FAT_ENT_OFFSET_SECTOR(sb, loc); + off = FAT_ENT_OFFSET_BYTE_IN_SECTOR(sb, loc); + + if (!bh || bh->b_blocknr != sec || !buffer_uptodate(bh)) { + brelse(bh); + bh = sb_bread(sb, sec); + *cache = bh; + if (unlikely(!bh)) + return -EIO; + } + + *content = le32_to_cpu(*(__le32 *)(&bh->b_data[off])); + + /* remap reserved clusters to simplify code */ + if (*content > EXFAT_BAD_CLUSTER) + *content = EXFAT_EOF_CLUSTER; + + return 0; +} + +static int __exfat_ent_set(struct super_block *sb, unsigned int loc, + unsigned int content, struct buffer_head **cache) +{ + sector_t sec; + __le32 *fat_entry; + struct buffer_head *bh = cache ? *cache : NULL; + unsigned int off; + + sec = FAT_ENT_OFFSET_SECTOR(sb, loc); + off = FAT_ENT_OFFSET_BYTE_IN_SECTOR(sb, loc); + + if (!bh || bh->b_blocknr != sec || !buffer_uptodate(bh)) { + if (bh) + exfat_end_bh(sb, bh); + bh = sb_bread(sb, sec); + if (cache) + *cache = bh; + if (unlikely(!bh)) + return -EIO; + } + + fat_entry = (__le32 *)&(bh->b_data[off]); + *fat_entry = cpu_to_le32(content); + if (!cache) + exfat_end_bh(sb, bh); + return 0; +} + +int exfat_ent_set(struct super_block *sb, unsigned int loc, + unsigned int content) +{ + return __exfat_ent_set(sb, loc, content, NULL); +} + +/* + * Caller must release the buffer_head if no error return. + */ +int exfat_ent_get(struct super_block *sb, unsigned int loc, + unsigned int *content, struct buffer_head **cache) +{ + struct exfat_sb_info *sbi = EXFAT_SB(sb); + + if (!is_valid_cluster(sbi, loc)) { + exfat_fs_error_ratelimit(sb, + "invalid access to FAT (entry 0x%08x)", + loc); + goto err; + } + + if (unlikely(__exfat_ent_get(sb, loc, content, cache))) { + exfat_fs_error_ratelimit(sb, + "failed to access to FAT (entry 0x%08x)", + loc); + goto err; + } + + if (unlikely(*content == EXFAT_FREE_CLUSTER)) { + exfat_fs_error_ratelimit(sb, + "invalid access to FAT free cluster (entry 0x%08x)", + loc); + goto err; + } + + if (unlikely(*content == EXFAT_BAD_CLUSTER)) { + exfat_fs_error_ratelimit(sb, + "invalid access to FAT bad cluster (entry 0x%08x)", + loc); + goto err; + } + + if (*content != EXFAT_EOF_CLUSTER && !is_valid_cluster(sbi, *content)) { + exfat_fs_error_ratelimit(sb, + "invalid access to FAT (entry 0x%08x) bogus content (0x%08x)", + loc, *content); + goto err; + } + + return 0; + +err: + /* Avoid double release */ + brelse(*cache); + *cache = NULL; + return -EIO; +} + +int exfat_blk_readahead(struct super_block *sb, sector_t sec, + sector_t *ra, blkcnt_t *ra_cnt, sector_t end) +{ + struct blk_plug plug; + + if (sec < *ra) + return 0; + + *ra += *ra_cnt; + + /* No blocks left (or only the last block), skip readahead. */ + if (*ra >= end) + return 0; + + *ra_cnt = min(end - *ra + 1, EXFAT_BLK_RA_SIZE(sb)); + if (*ra_cnt == 0) { + /* Move 'ra' to the end to disable readahead. */ + *ra = end; + return 0; + } + + blk_start_plug(&plug); + for (unsigned int i = 0; i < *ra_cnt; i++) + sb_breadahead(sb, *ra + i); + blk_finish_plug(&plug); + return 0; +} + +int exfat_chain_cont_cluster(struct super_block *sb, unsigned int chain, + unsigned int len) +{ + struct buffer_head *bh = NULL; + sector_t sec, end, ra; + blkcnt_t ra_cnt = 0; + + if (!len) + return 0; + + ra = FAT_ENT_OFFSET_SECTOR(sb, chain); + end = FAT_ENT_OFFSET_SECTOR(sb, chain + len - 1); + + while (len > 1) { + sec = FAT_ENT_OFFSET_SECTOR(sb, chain); + exfat_blk_readahead(sb, sec, &ra, &ra_cnt, end); + + if (__exfat_ent_set(sb, chain, chain + 1, &bh)) + return -EIO; + chain++; + len--; + } + + if (__exfat_ent_set(sb, chain, EXFAT_EOF_CLUSTER, &bh)) + return -EIO; + + exfat_end_bh(sb, bh); + return 0; +} + +static inline void exfat_discard_cluster(struct super_block *sb, + unsigned int clu, unsigned int num_clusters) +{ + int ret; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + + ret = sb_issue_discard(sb, exfat_cluster_to_sector(sbi, clu), + sbi->sect_per_clus * num_clusters, GFP_NOFS, 0); + if (ret == -EOPNOTSUPP) { + exfat_err(sb, "discard not supported by device, disabling"); + sbi->options.discard = 0; + } +} + +/* This function must be called with bitmap_lock held */ +static int __exfat_free_cluster(struct inode *inode, struct exfat_chain *p_chain) +{ + struct super_block *sb = inode->i_sb; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + int cur_cmap_i, next_cmap_i; + unsigned int num_clusters = 0; + unsigned int clu; + + /* invalid cluster number */ + if (p_chain->dir == EXFAT_FREE_CLUSTER || + p_chain->dir == EXFAT_EOF_CLUSTER || + p_chain->dir < EXFAT_FIRST_CLUSTER) + return 0; + + /* no cluster to truncate */ + if (p_chain->size == 0) + return 0; + + /* check cluster validation */ + if (!is_valid_cluster(sbi, p_chain->dir)) { + exfat_err(sb, "invalid start cluster (%u)", p_chain->dir); + return -EIO; + } + + clu = p_chain->dir; + + cur_cmap_i = next_cmap_i = + BITMAP_OFFSET_SECTOR_INDEX(sb, CLUSTER_TO_BITMAP_ENT(clu)); + + if (p_chain->flags == ALLOC_NO_FAT_CHAIN) { + int err; + unsigned int last_cluster = p_chain->dir + p_chain->size - 1; + + do { + bool sync = false; + + if (clu < last_cluster) + next_cmap_i = + BITMAP_OFFSET_SECTOR_INDEX(sb, CLUSTER_TO_BITMAP_ENT(clu+1)); + + /* flush bitmap only if index would be changed or for last cluster */ + if (clu == last_cluster || cur_cmap_i != next_cmap_i) { + sync = true; + cur_cmap_i = next_cmap_i; + } + + err = exfat_clear_bitmap(sb, clu, (sync && IS_DIRSYNC(inode))); + if (err) + break; + clu++; + num_clusters++; + } while (num_clusters < p_chain->size); + + if (sbi->options.discard) + exfat_discard_cluster(sb, p_chain->dir, p_chain->size); + } else { + unsigned int nr_clu = 1; + + do { + bool sync = false; + unsigned int n_clu = clu; + int err = exfat_get_next_cluster(sb, &n_clu); + + if (err || n_clu == EXFAT_EOF_CLUSTER) + sync = true; + else + next_cmap_i = + BITMAP_OFFSET_SECTOR_INDEX(sb, CLUSTER_TO_BITMAP_ENT(n_clu)); + + if (cur_cmap_i != next_cmap_i) { + sync = true; + cur_cmap_i = next_cmap_i; + } + + if (exfat_clear_bitmap(sb, clu, (sync && IS_DIRSYNC(inode)))) + break; + + if (sbi->options.discard) { + if (n_clu == clu + 1) + nr_clu++; + else { + exfat_discard_cluster(sb, clu - nr_clu + 1, nr_clu); + nr_clu = 1; + } + } + + clu = n_clu; + num_clusters++; + + if (err) + break; + + if (num_clusters >= sbi->num_clusters - EXFAT_FIRST_CLUSTER) { + /* + * The cluster chain includes a loop, scan the + * bitmap to get the number of used clusters. + */ + exfat_count_used_clusters(sb, &sbi->used_clusters); + + return 0; + } + } while (clu != EXFAT_EOF_CLUSTER); + } + + sbi->used_clusters -= num_clusters; + return 0; +} + +int exfat_free_cluster(struct inode *inode, struct exfat_chain *p_chain) +{ + int ret = 0; + + mutex_lock(&EXFAT_SB(inode->i_sb)->bitmap_lock); + ret = __exfat_free_cluster(inode, p_chain); + mutex_unlock(&EXFAT_SB(inode->i_sb)->bitmap_lock); + + return ret; +} + +int exfat_find_last_cluster(struct super_block *sb, struct exfat_chain *p_chain, + unsigned int *ret_clu) +{ + struct buffer_head *bh = NULL; + unsigned int clu, next; + unsigned int count = 0; + + next = p_chain->dir; + if (p_chain->flags == ALLOC_NO_FAT_CHAIN) { + *ret_clu = next + p_chain->size - 1; + return 0; + } + + do { + count++; + clu = next; + if (exfat_ent_get(sb, clu, &next, &bh)) + return -EIO; + } while (next != EXFAT_EOF_CLUSTER && count <= p_chain->size); + + brelse(bh); + if (p_chain->size != count) { + exfat_fs_error(sb, + "bogus directory size (clus : ondisk(%d) != counted(%d))", + p_chain->size, count); + return -EIO; + } + + *ret_clu = clu; + return 0; +} + +int exfat_zeroed_cluster(struct inode *dir, unsigned int clu) +{ + struct super_block *sb = dir->i_sb; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct buffer_head *bh; + sector_t blknr, last_blknr, i; + + blknr = exfat_cluster_to_sector(sbi, clu); + last_blknr = blknr + sbi->sect_per_clus; + + if (last_blknr > sbi->num_sectors && sbi->num_sectors > 0) { + exfat_fs_error_ratelimit(sb, + "%s: out of range(sect:%llu len:%u)", + __func__, (unsigned long long)blknr, + sbi->sect_per_clus); + return -EIO; + } + + /* Zeroing the unused blocks on this cluster */ + for (i = blknr; i < last_blknr; i++) { + bh = sb_getblk(sb, i); + if (!bh) + return -ENOMEM; + + memset(bh->b_data, 0, sb->s_blocksize); + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + brelse(bh); + } + + if (IS_DIRSYNC(dir)) + return sync_blockdev_range(sb->s_bdev, + exfat_block_to_bytes(sb, blknr), + exfat_block_to_bytes(sb, last_blknr) - 1); + + return 0; +} + +int exfat_alloc_cluster(struct inode *inode, unsigned int num_alloc, + struct exfat_chain *p_chain, bool sync_bmap, bool contig) +{ + int ret = -ENOSPC; + unsigned int total_cnt; + unsigned int hint_clu, new_clu, last_clu = EXFAT_EOF_CLUSTER; + struct super_block *sb = inode->i_sb; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + + total_cnt = EXFAT_DATA_CLUSTER_COUNT(sbi); + + if (unlikely(total_cnt < sbi->used_clusters)) { + exfat_fs_error_ratelimit(sb, + "%s: invalid used clusters(t:%u,u:%u)\n", + __func__, total_cnt, sbi->used_clusters); + return -EIO; + } + + if (num_alloc > total_cnt - sbi->used_clusters) + return -ENOSPC; + + mutex_lock(&sbi->bitmap_lock); + + hint_clu = p_chain->dir; + /* find new cluster */ + if (hint_clu == EXFAT_EOF_CLUSTER) { + if (sbi->clu_srch_ptr < EXFAT_FIRST_CLUSTER) { + exfat_err(sb, "sbi->clu_srch_ptr is invalid (%u)", + sbi->clu_srch_ptr); + sbi->clu_srch_ptr = EXFAT_FIRST_CLUSTER; + } + + hint_clu = exfat_find_free_bitmap(sb, sbi->clu_srch_ptr); + if (hint_clu == EXFAT_EOF_CLUSTER) { + ret = -ENOSPC; + goto unlock; + } + } + + /* check cluster validation */ + if (!is_valid_cluster(sbi, hint_clu)) { + if (hint_clu != sbi->num_clusters) + exfat_err(sb, "hint_cluster is invalid (%u), rewind to the first cluster", + hint_clu); + hint_clu = EXFAT_FIRST_CLUSTER; + p_chain->flags = ALLOC_FAT_CHAIN; + } + + p_chain->dir = EXFAT_EOF_CLUSTER; + + while ((new_clu = exfat_find_free_bitmap(sb, hint_clu)) != + EXFAT_EOF_CLUSTER) { + if (new_clu != hint_clu) { + if (p_chain->flags == ALLOC_NO_FAT_CHAIN) { + if (exfat_chain_cont_cluster(sb, p_chain->dir, + p_chain->size)) { + ret = -EIO; + goto free_cluster; + } + p_chain->flags = ALLOC_FAT_CHAIN; + } + + if (contig && p_chain->size > 0) { + hint_clu = last_clu; + goto done; + } + } + + /* update allocation bitmap */ + if (exfat_set_bitmap(sb, new_clu, sync_bmap)) { + ret = -EIO; + goto free_cluster; + } + + /* update FAT table */ + if (p_chain->flags == ALLOC_FAT_CHAIN) { + if (exfat_ent_set(sb, new_clu, EXFAT_EOF_CLUSTER)) { + ret = -EIO; + goto free_cluster; + } + } + + if (p_chain->dir == EXFAT_EOF_CLUSTER) { + p_chain->dir = new_clu; + } else if (p_chain->flags == ALLOC_FAT_CHAIN) { + if (exfat_ent_set(sb, last_clu, new_clu)) { + ret = -EIO; + goto free_cluster; + } + } + p_chain->size++; + + last_clu = new_clu; + + if (p_chain->size == num_alloc) { +done: + sbi->clu_srch_ptr = hint_clu; + sbi->used_clusters += p_chain->size; + mutex_unlock(&sbi->bitmap_lock); + return 0; + } + + hint_clu = new_clu + 1; + if (hint_clu >= sbi->num_clusters) { + hint_clu = EXFAT_FIRST_CLUSTER; + + if (p_chain->flags == ALLOC_NO_FAT_CHAIN) { + if (exfat_chain_cont_cluster(sb, p_chain->dir, + p_chain->size)) { + ret = -EIO; + goto free_cluster; + } + p_chain->flags = ALLOC_FAT_CHAIN; + } + } + } +free_cluster: + __exfat_free_cluster(inode, p_chain); +unlock: + mutex_unlock(&sbi->bitmap_lock); + return ret; +} + +int exfat_count_num_clusters(struct super_block *sb, + struct exfat_chain *p_chain, unsigned int *ret_count) +{ + unsigned int i, count; + unsigned int clu; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct buffer_head *bh = NULL; + + if (!p_chain->dir || p_chain->dir == EXFAT_EOF_CLUSTER) { + *ret_count = 0; + return 0; + } + + if (p_chain->flags == ALLOC_NO_FAT_CHAIN) { + *ret_count = p_chain->size; + return 0; + } + + clu = p_chain->dir; + count = 0; + for (i = EXFAT_FIRST_CLUSTER; i < sbi->num_clusters; i++) { + count++; + if (exfat_ent_get(sb, clu, &clu, &bh)) + return -EIO; + if (clu == EXFAT_EOF_CLUSTER) + break; + } + + brelse(bh); + *ret_count = count; + + /* + * since exfat_count_used_clusters() is not called, sbi->used_clusters + * cannot be used here. + */ + if (unlikely(i == sbi->num_clusters && clu != EXFAT_EOF_CLUSTER)) { + exfat_fs_error(sb, "The cluster chain has a loop"); + return -EIO; + } + + return 0; +} diff --git a/fs/exfat/file.c b/fs/exfat/file.c new file mode 100644 index 000000000..a2a9ee1a2 --- /dev/null +++ b/fs/exfat/file.c @@ -0,0 +1,1083 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2012-2013 Samsung Electronics Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "exfat_raw.h" +#include "exfat_fs.h" +#include "iomap.h" + +static int exfat_cont_expand(struct inode *inode, loff_t size) +{ + int ret; + unsigned int num_clusters, new_num_clusters, last_clu; + struct exfat_inode_info *ei = EXFAT_I(inode); + struct super_block *sb = inode->i_sb; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct exfat_chain clu; + loff_t oldsize = i_size_read(inode); + + truncate_pagecache(inode, oldsize); + + ret = inode_newsize_ok(inode, size); + if (ret) + return ret; + + num_clusters = exfat_bytes_to_cluster(sbi, exfat_ondisk_size(inode)); + /* integer overflow is already checked in inode_newsize_ok(). */ + new_num_clusters = exfat_bytes_to_cluster_round_up(sbi, size); + + if (new_num_clusters == num_clusters) + goto out; + + if (num_clusters) { + exfat_chain_set(&clu, ei->start_clu, num_clusters, ei->flags); + ret = exfat_find_last_cluster(sb, &clu, &last_clu); + if (ret) + return ret; + + clu.dir = last_clu + 1; + } else { + last_clu = EXFAT_EOF_CLUSTER; + clu.dir = EXFAT_EOF_CLUSTER; + } + + clu.size = 0; + clu.flags = ei->flags; + + ret = exfat_alloc_cluster(inode, new_num_clusters - num_clusters, + &clu, inode_needs_sync(inode), false); + if (ret) + return ret; + + /* Append new clusters to chain */ + if (num_clusters) { + if (clu.flags != ei->flags) + if (exfat_chain_cont_cluster(sb, ei->start_clu, num_clusters)) + goto free_clu; + + if (clu.flags == ALLOC_FAT_CHAIN) + if (exfat_ent_set(sb, last_clu, clu.dir)) + goto free_clu; + } else + ei->start_clu = clu.dir; + + ei->flags = clu.flags; + +out: + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + /* Expanded range not zeroed, do not update valid_size */ + i_size_write(inode, size); + /* + * When extending file size, call truncate_pagecache() first, + * then update i_size, and call pagecache_isize_extended() + * to ensures the straddling folio is properly marked RO so + * page_mkwrite() is called and post-EOF area is zeroed. + */ + pagecache_isize_extended(inode, oldsize, inode->i_size); + + inode->i_blocks = round_up(size, sbi->cluster_size) >> 9; + mark_inode_dirty(inode); + + if (IS_SYNC(inode)) + return write_inode_now(inode, 1); + + return 0; + +free_clu: + exfat_free_cluster(inode, &clu); + return -EIO; +} + +/* + * Preallocate space for a file. This implements exfat's fallocate file + * operation, which gets called from sys_fallocate system call. User space + * requests len bytes at offset. In contrary to fat, we only support + * FALLOC_FL_ALLOCATE_RANGE because by leaving the valid data length(VDL) + * field, it is unnecessary to zero out the newly allocated clusters. + */ +static long exfat_fallocate(struct file *file, int mode, + loff_t offset, loff_t len) +{ + struct inode *inode = file->f_mapping->host; + loff_t newsize = offset + len; + int err = 0; + + /* No support for other modes */ + if (mode != FALLOC_FL_ALLOCATE_RANGE) + return -EOPNOTSUPP; + + /* No support for dir */ + if (!S_ISREG(inode->i_mode)) + return -EOPNOTSUPP; + + if (unlikely(exfat_forced_shutdown(inode->i_sb))) + return -EIO; + + inode_lock(inode); + + if (newsize <= i_size_read(inode)) + goto error; + + /* This is just an expanding truncate */ + err = exfat_cont_expand(inode, newsize); + +error: + inode_unlock(inode); + + return err; +} + +static bool exfat_allow_set_time(struct mnt_idmap *idmap, + struct exfat_sb_info *sbi, struct inode *inode) +{ + mode_t allow_utime = sbi->options.allow_utime; + + if (!vfsuid_eq_kuid(i_uid_into_vfsuid(idmap, inode), + current_fsuid())) { + if (vfsgid_in_group_p(i_gid_into_vfsgid(idmap, inode))) + allow_utime >>= 3; + if (allow_utime & MAY_WRITE) + return true; + } + + /* use a default check */ + return false; +} + +static int exfat_sanitize_mode(const struct exfat_sb_info *sbi, + struct inode *inode, umode_t *mode_ptr) +{ + mode_t i_mode, mask, perm; + + i_mode = inode->i_mode; + + mask = (S_ISREG(i_mode) || S_ISLNK(i_mode)) ? + sbi->options.fs_fmask : sbi->options.fs_dmask; + perm = *mode_ptr & ~(S_IFMT | mask); + + /* Of the r and x bits, all (subject to umask) must be present.*/ + if ((perm & 0555) != (i_mode & 0555)) + return -EPERM; + + if (exfat_mode_can_hold_ro(inode)) { + /* + * Of the w bits, either all (subject to umask) or none must + * be present. + */ + if ((perm & 0222) && ((perm & 0222) != (0222 & ~mask))) + return -EPERM; + } else { + /* + * If exfat_mode_can_hold_ro(inode) is false, can't change + * w bits. + */ + if ((perm & 0222) != (0222 & ~mask)) + return -EPERM; + } + + *mode_ptr &= S_IFMT | perm; + + return 0; +} + +/* resize the file length */ +int __exfat_truncate(struct inode *inode) +{ + unsigned int num_clusters_new, num_clusters_phys; + unsigned int last_clu = EXFAT_FREE_CLUSTER; + struct exfat_chain clu; + struct super_block *sb = inode->i_sb; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct exfat_inode_info *ei = EXFAT_I(inode); + + /* check if the given file ID is opened */ + if (ei->type != TYPE_FILE && ei->type != TYPE_DIR) + return -EPERM; + + exfat_set_volume_dirty(sb); + + num_clusters_new = exfat_bytes_to_cluster_round_up(sbi, i_size_read(inode)); + num_clusters_phys = exfat_bytes_to_cluster(sbi, exfat_ondisk_size(inode)); + + exfat_chain_set(&clu, ei->start_clu, num_clusters_phys, ei->flags); + + if (i_size_read(inode) > 0) { + /* + * Truncate FAT chain num_clusters after the first cluster + * num_clusters = min(new, phys); + */ + unsigned int num_clusters = + min(num_clusters_new, num_clusters_phys); + + /* + * Follow FAT chain + * (defensive coding - works fine even with corrupted FAT table + */ + if (clu.flags == ALLOC_NO_FAT_CHAIN) { + clu.dir += num_clusters; + clu.size -= num_clusters; + } else { + while (num_clusters > 0) { + last_clu = clu.dir; + if (exfat_get_next_cluster(sb, &(clu.dir))) + return -EIO; + + num_clusters--; + clu.size--; + } + } + } else { + ei->flags = ALLOC_NO_FAT_CHAIN; + ei->start_clu = EXFAT_EOF_CLUSTER; + } + + if (i_size_read(inode) < ei->valid_size) + ei->valid_size = ei->zeroed_size = i_size_read(inode); + + if (ei->type == TYPE_FILE) + ei->attr |= EXFAT_ATTR_ARCHIVE; + + /* + * update the directory entry + * + * If the directory entry is updated by mark_inode_dirty(), the + * directory entry will be written after a writeback cycle of + * updating the bitmap/FAT, which may result in clusters being + * freed but referenced by the directory entry in the event of a + * sudden power failure. + * __exfat_write_inode() is called for directory entry, bitmap + * and FAT to be written in a same writeback. + */ + if (__exfat_write_inode(inode, inode_needs_sync(inode))) + return -EIO; + + /* cut off from the FAT chain */ + if (ei->flags == ALLOC_FAT_CHAIN && last_clu != EXFAT_FREE_CLUSTER && + last_clu != EXFAT_EOF_CLUSTER) { + if (exfat_ent_set(sb, last_clu, EXFAT_EOF_CLUSTER)) + return -EIO; + } + + /* invalidate cache and free the clusters */ + /* clear exfat cache */ + exfat_cache_inval_inode(inode); + + /* hint information */ + ei->hint_bmap.off = EXFAT_EOF_CLUSTER; + ei->hint_bmap.clu = EXFAT_EOF_CLUSTER; + + /* hint_stat will be used if this is directory. */ + ei->hint_stat.eidx = 0; + ei->hint_stat.clu = ei->start_clu; + ei->hint_femp.eidx = EXFAT_HINT_NONE; + + /* free the clusters */ + if (exfat_free_cluster(inode, &clu)) + return -EIO; + + return 0; +} + +static void exfat_truncate(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct exfat_inode_info *ei = EXFAT_I(inode); + int err; + + mutex_lock(&sbi->s_lock); + if (ei->start_clu == 0) { + /* + * Empty start_clu != ~0 (not allocated) + */ + exfat_fs_error(sb, "tried to truncate zeroed cluster."); + goto write_size; + } + + err = __exfat_truncate(inode); + if (err) + goto write_size; + + inode->i_blocks = round_up(i_size_read(inode), sbi->cluster_size) >> 9; +write_size: + mutex_unlock(&sbi->s_lock); +} + +int exfat_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, unsigned int request_mask, + unsigned int query_flags) +{ + struct inode *inode = d_backing_inode(path->dentry); + struct exfat_inode_info *ei = EXFAT_I(inode); + + generic_fillattr(idmap, request_mask, inode, stat); + exfat_truncate_atime(&stat->atime); + stat->result_mask |= STATX_BTIME; + stat->btime.tv_sec = ei->i_crtime.tv_sec; + stat->btime.tv_nsec = ei->i_crtime.tv_nsec; + stat->blksize = EXFAT_SB(inode->i_sb)->cluster_size; + return 0; +} + +int exfat_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + /* + * exFAT compares filenames through an upcase table, so lookup + * is always case-insensitive. Long names are stored in UTF-16 + * with case intact; CASENONPRESERVING stays clear. + */ + fa->fsx_xflags |= FS_XFLAG_CASEFOLD; + fa->flags |= FS_CASEFOLD_FL; + return 0; +} + +int exfat_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +{ + struct exfat_sb_info *sbi = EXFAT_SB(dentry->d_sb); + struct inode *inode = dentry->d_inode; + unsigned int ia_valid; + int error; + + if (unlikely(exfat_forced_shutdown(inode->i_sb))) + return -EIO; + + if ((attr->ia_valid & ATTR_SIZE) && + attr->ia_size > i_size_read(inode)) { + error = exfat_cont_expand(inode, attr->ia_size); + if (error || attr->ia_valid == ATTR_SIZE) + return error; + attr->ia_valid &= ~ATTR_SIZE; + } + + /* Check for setting the inode time. */ + ia_valid = attr->ia_valid; + if ((ia_valid & (ATTR_MTIME_SET | ATTR_ATIME_SET | ATTR_TIMES_SET)) && + exfat_allow_set_time(idmap, sbi, inode)) { + attr->ia_valid &= ~(ATTR_MTIME_SET | ATTR_ATIME_SET | + ATTR_TIMES_SET); + } + + error = setattr_prepare(idmap, dentry, attr); + attr->ia_valid = ia_valid; + if (error) + goto out; + + if (((attr->ia_valid & ATTR_UID) && + (!uid_eq(from_vfsuid(idmap, i_user_ns(inode), attr->ia_vfsuid), + sbi->options.fs_uid))) || + ((attr->ia_valid & ATTR_GID) && + (!gid_eq(from_vfsgid(idmap, i_user_ns(inode), attr->ia_vfsgid), + sbi->options.fs_gid))) || + ((attr->ia_valid & ATTR_MODE) && + (attr->ia_mode & ~(S_IFREG | S_IFLNK | S_IFDIR | 0777)))) { + error = -EPERM; + goto out; + } + + /* + * We don't return -EPERM here. Yes, strange, but this is too + * old behavior. + */ + if (attr->ia_valid & ATTR_MODE) { + if (exfat_sanitize_mode(sbi, inode, &attr->ia_mode) < 0) + attr->ia_valid &= ~ATTR_MODE; + } + + if (attr->ia_valid & ATTR_SIZE) + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + + setattr_copy(idmap, inode, attr); + exfat_truncate_inode_atime(inode); + + if (attr->ia_valid & ATTR_SIZE) { + /* + * Wait for any in-flight DIO to finish before truncating to + * prevent a concurrent DIO from writing to clusters that are + * about to be freed. + */ + inode_dio_wait(inode); + truncate_setsize(inode, attr->ia_size); + + /* + * __exfat_write_inode() is called from exfat_truncate(), inode + * is already written by it, so mark_inode_dirty() is unneeded. + */ + exfat_truncate(inode); + } else + mark_inode_dirty(inode); + +out: + return error; +} + +/* + * modified ioctls from fat/file.c by Welmer Almesberger + */ +static int exfat_ioctl_get_attributes(struct inode *inode, u32 __user *user_attr) +{ + u32 attr; + + inode_lock_shared(inode); + attr = exfat_make_attr(inode); + inode_unlock_shared(inode); + + return put_user(attr, user_attr); +} + +static int exfat_ioctl_set_attributes(struct file *file, u32 __user *user_attr) +{ + struct inode *inode = file_inode(file); + struct exfat_sb_info *sbi = EXFAT_SB(inode->i_sb); + int is_dir = S_ISDIR(inode->i_mode); + u32 attr, oldattr; + struct iattr ia; + int err; + + err = get_user(attr, user_attr); + if (err) + goto out; + + err = mnt_want_write_file(file); + if (err) + goto out; + inode_lock(inode); + + oldattr = exfat_make_attr(inode); + + /* + * Mask attributes so we don't set reserved fields. + */ + attr &= (EXFAT_ATTR_READONLY | EXFAT_ATTR_HIDDEN | EXFAT_ATTR_SYSTEM | + EXFAT_ATTR_ARCHIVE); + attr |= (is_dir ? EXFAT_ATTR_SUBDIR : 0); + + /* Equivalent to a chmod() */ + ia.ia_valid = ATTR_MODE | ATTR_CTIME; + ia.ia_ctime = current_time(inode); + if (is_dir) + ia.ia_mode = exfat_make_mode(sbi, attr, 0777); + else + ia.ia_mode = exfat_make_mode(sbi, attr, 0666 | (inode->i_mode & 0111)); + + /* The root directory has no attributes */ + if (inode->i_ino == EXFAT_ROOT_INO && attr != EXFAT_ATTR_SUBDIR) { + err = -EINVAL; + goto out_unlock_inode; + } + + if (((attr | oldattr) & EXFAT_ATTR_SYSTEM) && + !capable(CAP_LINUX_IMMUTABLE)) { + err = -EPERM; + goto out_unlock_inode; + } + + /* + * The security check is questionable... We single + * out the RO attribute for checking by the security + * module, just because it maps to a file mode. + */ + err = security_inode_setattr(file_mnt_idmap(file), + file->f_path.dentry, &ia); + if (err) + goto out_unlock_inode; + + /* This MUST be done before doing anything irreversible... */ + err = exfat_setattr(file_mnt_idmap(file), file->f_path.dentry, &ia); + if (err) + goto out_unlock_inode; + + fsnotify_change(file->f_path.dentry, ia.ia_valid); + + exfat_save_attr(inode, attr); + mark_inode_dirty(inode); +out_unlock_inode: + inode_unlock(inode); + mnt_drop_write_file(file); +out: + return err; +} + +static int exfat_ioctl_fitrim(struct inode *inode, unsigned long arg) +{ + struct fstrim_range range; + int ret = 0; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (!bdev_max_discard_sectors(inode->i_sb->s_bdev)) + return -EOPNOTSUPP; + + if (copy_from_user(&range, (struct fstrim_range __user *)arg, sizeof(range))) + return -EFAULT; + + range.minlen = max_t(unsigned int, range.minlen, + bdev_discard_granularity(inode->i_sb->s_bdev)); + + ret = exfat_trim_fs(inode, &range); + if (ret < 0) + return ret; + + if (copy_to_user((struct fstrim_range __user *)arg, &range, sizeof(range))) + return -EFAULT; + + return 0; +} + +static int exfat_ioctl_shutdown(struct super_block *sb, unsigned long arg) +{ + u32 flags; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (get_user(flags, (__u32 __user *)arg)) + return -EFAULT; + + return exfat_force_shutdown(sb, flags); +} + +static int exfat_ioctl_get_volume_label(struct super_block *sb, unsigned long arg) +{ + int ret; + char label[FSLABEL_MAX] = {0}; + struct exfat_uni_name uniname; + + ret = exfat_read_volume_label(sb, &uniname); + if (ret < 0) + return ret; + + ret = exfat_utf16_to_nls(sb, &uniname, label, sizeof(label)); + if (ret < 0) + return ret; + + if (copy_to_user((char __user *)arg, label, ret + 1)) + return -EFAULT; + + return 0; +} + +static int exfat_ioctl_set_volume_label(struct super_block *sb, + unsigned long arg) +{ + int ret = 0, lossy, label_len; + char label[FSLABEL_MAX] = {0}; + struct exfat_uni_name uniname; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (copy_from_user(label, (char __user *)arg, FSLABEL_MAX)) + return -EFAULT; + + memset(&uniname, 0, sizeof(uniname)); + label_len = strnlen(label, FSLABEL_MAX - 1); + if (label[0]) { + ret = exfat_nls_to_utf16(sb, label, label_len, + &uniname, &lossy); + if (ret < 0) + return ret; + else if (lossy & NLS_NAME_LOSSY) + return -EINVAL; + } + + uniname.name_len = ret; + + return exfat_write_volume_label(sb, &uniname); +} + +long exfat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + u32 __user *user_attr = (u32 __user *)arg; + + switch (cmd) { + case FAT_IOCTL_GET_ATTRIBUTES: + return exfat_ioctl_get_attributes(inode, user_attr); + case FAT_IOCTL_SET_ATTRIBUTES: + return exfat_ioctl_set_attributes(filp, user_attr); + case EXFAT_IOC_SHUTDOWN: + return exfat_ioctl_shutdown(inode->i_sb, arg); + case FITRIM: + return exfat_ioctl_fitrim(inode, arg); + case FS_IOC_GETFSLABEL: + return exfat_ioctl_get_volume_label(inode->i_sb, arg); + case FS_IOC_SETFSLABEL: + return exfat_ioctl_set_volume_label(inode->i_sb, arg); + default: + return -ENOTTY; + } +} + +#ifdef CONFIG_COMPAT +long exfat_compat_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg) +{ + return exfat_ioctl(filp, cmd, (unsigned long)compat_ptr(arg)); +} +#endif + +int exfat_file_fsync(struct file *filp, loff_t start, loff_t end, int datasync) +{ + struct inode *inode = filp->f_mapping->host; + int err; + + if (unlikely(exfat_forced_shutdown(inode->i_sb))) + return -EIO; + + err = simple_fsync_noflush(filp, start, end, datasync); + if (err) + return err; + + err = sync_blockdev(inode->i_sb->s_bdev); + if (err) + return err; + + return blkdev_issue_flush(inode->i_sb->s_bdev); +} + +/* + * exfat_zero_new_range - zero [start, end) without overwriting uptodate blocks + * + * Uptodate blocks may contain data written through a shared mapping beyond + * valid_size. + */ +static int exfat_zero_new_range(struct inode *inode, loff_t start, loff_t end) +{ + struct address_space *mapping = inode->i_mapping; + unsigned int blocksize = i_blocksize(inode); + loff_t pos = start; + int err; + + while (pos < end) { + loff_t next = min_t(loff_t, + round_down(pos, PAGE_SIZE) + PAGE_SIZE, end); + struct folio *folio; + loff_t bpos; + + folio = filemap_get_folio(mapping, pos >> PAGE_SHIFT); + if (IS_ERR(folio)) { + err = iomap_zero_range(inode, pos, next - pos, NULL, + &exfat_iomap_ops, NULL, NULL); + if (err < 0) + return err; + pos = next; + continue; + } + + if (folio_test_uptodate(folio)) { + folio_lock(folio); + if (folio->mapping == mapping) + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + pos = next; + continue; + } + + /* + * Zero not-uptodate block runs. iomap_zero_range() requires an + * unlocked folio, so recheck ->mapping after each call. + */ + folio_lock(folio); + bpos = pos; + while (bpos < next) { + loff_t rstart, rend; + + if (folio->mapping != mapping) { + folio_unlock(folio); + err = iomap_zero_range(inode, bpos, next - bpos, + NULL, &exfat_iomap_ops, NULL, NULL); + if (err < 0) { + folio_put(folio); + return err; + } + folio_lock(folio); + break; + } + + if (iomap_is_partially_uptodate(folio, + offset_in_folio(folio, bpos), blocksize)) { + bpos += blocksize; + continue; + } + + rstart = bpos; + rend = min_t(loff_t, bpos + blocksize, next); + while (rend < next && + !iomap_is_partially_uptodate(folio, + offset_in_folio(folio, rend), blocksize)) + rend = min_t(loff_t, rend + blocksize, next); + + folio_unlock(folio); + err = iomap_zero_range(inode, rstart, rend - rstart, + NULL, &exfat_iomap_ops, NULL, NULL); + if (err < 0) { + folio_put(folio); + return err; + } + folio_lock(folio); + bpos = rend; + } + + /* + * Dirty only a fully uptodate folio. Dirtying a partial folio could + * write uninitialised cache contents over valid on-disk blocks. + */ + if (folio->mapping == mapping && folio_test_uptodate(folio)) + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + pos = next; + } + + return 0; +} + +static int exfat_extend_valid_size(struct inode *inode, loff_t new_valid_size) +{ + struct exfat_inode_info *ei = EXFAT_I(inode); + loff_t old_valid_size = ei->valid_size; + int ret = 0; + + if (old_valid_size < new_valid_size) { + /* Do not re-zero blocks already covered by zeroed_size. */ + loff_t gap_start = max(old_valid_size, ei->zeroed_size); + + if (i_size_read(inode) < new_valid_size) { + /* + * Allocate clusters before increasing i_size. The gap + * may already be zeroed, so the subsequent zeroing + * can be skipped. + */ + ret = exfat_cont_expand(inode, new_valid_size); + if (ret) + return ret; + } + + /* + * Revoke writable PTEs while zeroing the gap. A racing mmap + * store re-faults through exfat_page_mkwrite() after valid_size + * is updated. + */ + filemap_invalidate_lock(inode->i_mapping); + if (gap_start < new_valid_size) + unmap_mapping_range(inode->i_mapping, gap_start, + new_valid_size - gap_start, 0); + ret = exfat_zero_new_range(inode, gap_start, new_valid_size); + filemap_invalidate_unlock(inode->i_mapping); + if (ret) { + truncate_setsize(inode, old_valid_size); + exfat_truncate(inode); + return ret; + } + + ei->valid_size = new_valid_size; + if (ei->zeroed_size < round_up(new_valid_size, i_blocksize(inode))) + ei->zeroed_size = round_up(new_valid_size, i_blocksize(inode)); + mark_inode_dirty(inode); + } + + return ret; +} + +static ssize_t exfat_fallback_buffered_write(struct kiocb *iocb, + struct iov_iter *from) +{ + loff_t offset = iocb->ki_pos, end; + ssize_t written; + int ret; + + iocb->ki_flags &= ~IOCB_DIRECT; + + written = iomap_file_buffered_write(iocb, from, &exfat_write_iomap_ops, + NULL, NULL); + if (written < 0) + return written; + + end = iocb->ki_pos + written - 1; + ret = filemap_write_and_wait_range(iocb->ki_filp->f_mapping, + offset, end); + if (ret) + return -EIO; + + invalidate_mapping_pages(iocb->ki_filp->f_mapping, + offset >> PAGE_SHIFT, + end >> PAGE_SHIFT); + + return written; +} + +static ssize_t exfat_dio_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + ssize_t ret; + + ret = iomap_dio_rw(iocb, from, &exfat_write_iomap_ops, + &exfat_write_dio_ops, 0, NULL, 0); + if (ret == -ENOTBLK) + ret = 0; + else if (ret < 0) + return ret; + + if (iov_iter_count(from)) { + ssize_t written; + + written = exfat_fallback_buffered_write(iocb, from); + if (written < 0) + return written; + ret += written; + } + + return ret; +} + +static ssize_t exfat_file_write_iter(struct kiocb *iocb, struct iov_iter *iter) +{ + ssize_t ret; + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct exfat_inode_info *ei = EXFAT_I(inode); + loff_t pos = iocb->ki_pos; + loff_t valid_size; + int err; + + if (unlikely(exfat_forced_shutdown(inode->i_sb))) + return -EIO; + + inode_lock(inode); + + if (pos > i_size_read(inode)) + truncate_pagecache(inode, i_size_read(inode)); + + valid_size = ei->valid_size; + + ret = generic_write_checks(iocb, iter); + if (ret <= 0) + goto unlock; + + err = file_modified(iocb->ki_filp); + if (err) { + ret = err; + goto unlock; + } + + if (pos > valid_size) { + ret = exfat_extend_valid_size(inode, pos); + if (ret < 0 && ret != -ENOSPC) { + exfat_err(inode->i_sb, + "write: fail to zero from %llu to %llu(%zd)", + valid_size, pos, ret); + } + if (ret < 0) + goto unlock; + } + + if (iocb->ki_flags & IOCB_DIRECT) + ret = exfat_dio_write_iter(iocb, iter); + else + ret = iomap_file_buffered_write(iocb, iter, + &exfat_write_iomap_ops, NULL, NULL); + if (ret < 0) + goto unlock; + + inode_unlock(inode); + + if (pos > valid_size) + pos = valid_size; + + if (iocb->ki_pos > pos) { + ssize_t err = generic_write_sync(iocb, iocb->ki_pos - pos); + + if (err < 0) + return err; + } + + return ret; + +unlock: + inode_unlock(inode); + + return ret; +} + +static ssize_t exfat_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) +{ + struct inode *inode = file_inode(iocb->ki_filp); + ssize_t ret; + + if (unlikely(exfat_forced_shutdown(inode->i_sb))) + return -EIO; + + inode_lock_shared(inode); + + if (iocb->ki_flags & IOCB_DIRECT) { + file_accessed(iocb->ki_filp); + ret = iomap_dio_rw(iocb, iter, &exfat_iomap_ops, NULL, 0, + NULL, 0); + } else { + ret = generic_file_read_iter(iocb, iter); + } + + inode_unlock_shared(inode); + + return ret; +} + +static vm_fault_t exfat_page_mkwrite(struct vm_fault *vmf) +{ + struct inode *inode = file_inode(vmf->vma->vm_file); + struct exfat_inode_info *ei = EXFAT_I(inode); + vm_fault_t ret; + loff_t new_valid_size, mmap_valid_size, fault_page_start; + + if (!inode_trylock(inode)) + return VM_FAULT_RETRY; + + mmap_valid_size = ((loff_t)vmf->pgoff + 1) << PAGE_SHIFT; + fault_page_start = ((loff_t)vmf->pgoff) << PAGE_SHIFT; + new_valid_size = min(mmap_valid_size, i_size_read(inode)); + + if (ei->valid_size < new_valid_size) { + if (ei->zeroed_size < fault_page_start) { + int err; + + /* + * Zero only the gap below the faulting page. The read + * fault populated its folio and iomap_page_mkwrite() + * will dirty it. + */ + err = exfat_zero_new_range(inode, ei->zeroed_size, + fault_page_start); + if (err < 0) { + inode_unlock(inode); + return vmf_fs_error(err); + } + } + + /* + * Track zeroed_size by block, not page, because writeback stops + * at i_size recording blocks wholly beyond it could skip a + * later required zeroing. + */ + if (ei->zeroed_size < round_up(new_valid_size, i_blocksize(inode))) + ei->zeroed_size = round_up(new_valid_size, i_blocksize(inode)); + ei->valid_size = new_valid_size; + mark_inode_dirty(inode); + } + + sb_start_pagefault(inode->i_sb); + file_update_time(vmf->vma->vm_file); + + filemap_invalidate_lock_shared(inode->i_mapping); + ret = iomap_page_mkwrite(vmf, &exfat_iomap_ops, NULL); + filemap_invalidate_unlock_shared(inode->i_mapping); + sb_end_pagefault(inode->i_sb); + inode_unlock(inode); + + return ret; +} + +static const struct vm_operations_struct exfat_file_vm_ops = { + .fault = filemap_fault, + .page_mkwrite = exfat_page_mkwrite, +}; + +static int exfat_file_mmap_prepare(struct vm_area_desc *desc) +{ + struct file *file = desc->file; + + if (unlikely(exfat_forced_shutdown(file_inode(desc->file)->i_sb))) + return -EIO; + + file_accessed(file); + desc->vm_ops = &exfat_file_vm_ops; + return 0; +} + +static ssize_t exfat_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, unsigned int flags) +{ + if (unlikely(exfat_forced_shutdown(file_inode(in)->i_sb))) + return -EIO; + + return filemap_splice_read(in, ppos, pipe, len, flags); +} + +static int exfat_file_open(struct inode *inode, struct file *filp) +{ + int err; + + if (unlikely(exfat_forced_shutdown(inode->i_sb))) + return -EIO; + + err = generic_file_open(inode, filp); + if (err) + return err; + + filp->f_mode |= FMODE_CAN_ODIRECT; + + return 0; +} + +static loff_t exfat_file_llseek(struct file *file, loff_t offset, int whence) +{ + struct inode *inode = file->f_mapping->host; + + switch (whence) { + case SEEK_HOLE: + inode_lock_shared(inode); + offset = iomap_seek_hole(inode, offset, &exfat_iomap_ops); + inode_unlock_shared(inode); + break; + case SEEK_DATA: + inode_lock_shared(inode); + offset = iomap_seek_data(inode, offset, &exfat_iomap_ops); + inode_unlock_shared(inode); + break; + default: + return generic_file_llseek(file, offset, whence); + } + if (offset < 0) + return offset; + return vfs_setpos(file, offset, inode->i_sb->s_maxbytes); +} + +const struct file_operations exfat_file_operations = { + .open = exfat_file_open, + .llseek = exfat_file_llseek, + .read_iter = exfat_file_read_iter, + .write_iter = exfat_file_write_iter, + .unlocked_ioctl = exfat_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = exfat_compat_ioctl, +#endif + .mmap_prepare = exfat_file_mmap_prepare, + .fsync = exfat_file_fsync, + .splice_read = exfat_splice_read, + .splice_write = iter_file_splice_write, + .fallocate = exfat_fallocate, + .setlease = generic_setlease, +}; + +const struct inode_operations exfat_file_inode_operations = { + .setattr = exfat_setattr, + .getattr = exfat_getattr, + .fileattr_get = exfat_fileattr_get, +}; diff --git a/fs/exfat/inode.c b/fs/exfat/inode.c new file mode 100644 index 000000000..ccd136301 --- /dev/null +++ b/fs/exfat/inode.c @@ -0,0 +1,459 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2012-2013 Samsung Electronics Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "exfat_raw.h" +#include "exfat_fs.h" +#include "iomap.h" + +int __exfat_write_inode(struct inode *inode, int sync) +{ + unsigned long long on_disk_size; + unsigned long long on_disk_valid_size; + struct exfat_dentry *ep, *ep2; + struct exfat_entry_set_cache es; + struct super_block *sb = inode->i_sb; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct exfat_inode_info *ei = EXFAT_I(inode); + bool is_dir = (ei->type == TYPE_DIR); + struct timespec64 ts; + + if (inode->i_ino == EXFAT_ROOT_INO) + return 0; + + /* + * If the inode is already unlinked, there is no need for updating it. + */ + if (ei->dir.dir == DIR_DELETED) + return 0; + + if (is_dir && ei->dir.dir == sbi->root_dir && ei->entry == -1) + return 0; + + exfat_set_volume_dirty(sb); + + /* get the directory entry of given file or directory */ + if (exfat_get_dentry_set_by_ei(&es, sb, ei)) + return -EIO; + ep = exfat_get_dentry_cached(&es, ES_IDX_FILE); + ep2 = exfat_get_dentry_cached(&es, ES_IDX_STREAM); + + ep->dentry.file.attr = cpu_to_le16(exfat_make_attr(inode)); + + /* set FILE_INFO structure using the acquired struct exfat_dentry */ + exfat_set_entry_time(sbi, &ei->i_crtime, + &ep->dentry.file.create_tz, + &ep->dentry.file.create_time, + &ep->dentry.file.create_date, + &ep->dentry.file.create_time_cs); + ts = inode_get_mtime(inode); + exfat_set_entry_time(sbi, &ts, + &ep->dentry.file.modify_tz, + &ep->dentry.file.modify_time, + &ep->dentry.file.modify_date, + &ep->dentry.file.modify_time_cs); + ts = inode_get_atime(inode); + exfat_set_entry_time(sbi, &ts, + &ep->dentry.file.access_tz, + &ep->dentry.file.access_time, + &ep->dentry.file.access_date, + NULL); + + /* + * During a DIO write, valid_size is updated eagerly in iomap_end (so + * that concurrent buffered reads see IOMAP_MAPPED) while i_size is + * updated asynchronously in end_io. The FAT chain was already + * extended to cover ceil(valid_size/cluster_size) clusters. Use the + * maximum so the on-disk size field always covers the FAT chain, + * preventing fsck from reporting "more clusters are allocated". + */ + on_disk_size = max_t(unsigned long long, i_size_read(inode), + ei->valid_size); + + if (ei->start_clu == EXFAT_EOF_CLUSTER) + on_disk_size = 0; + /* + * valid_size on disk must reflect only confirmed data (up to i_size) + * and must not exceed on_disk_size. + */ + on_disk_valid_size = min_t(unsigned long long, ei->valid_size, + i_size_read(inode)); + if (ei->start_clu == EXFAT_EOF_CLUSTER) + on_disk_valid_size = 0; + + ep2->dentry.stream.size = cpu_to_le64(on_disk_size); + ep2->dentry.stream.valid_size = cpu_to_le64(on_disk_valid_size); + + if (on_disk_size) { + ep2->dentry.stream.flags = ei->flags; + ep2->dentry.stream.start_clu = cpu_to_le32(ei->start_clu); + } else { + ep2->dentry.stream.flags = ALLOC_FAT_CHAIN; + ep2->dentry.stream.start_clu = EXFAT_FREE_CLUSTER; + } + + exfat_update_dir_chksum(&es); + return exfat_put_dentry_set(&es, sync); +} + +int exfat_write_inode(struct inode *inode, struct writeback_control *wbc) +{ + int ret; + + if (unlikely(exfat_forced_shutdown(inode->i_sb))) + return -EIO; + + mutex_lock(&EXFAT_SB(inode->i_sb)->s_lock); + ret = __exfat_write_inode(inode, wbc->sync_mode == WB_SYNC_ALL); + mutex_unlock(&EXFAT_SB(inode->i_sb)->s_lock); + + return ret; +} + +void exfat_sync_inode(struct inode *inode) +{ + lockdep_assert_held(&EXFAT_SB(inode->i_sb)->s_lock); + __exfat_write_inode(inode, 1); +} + +/* + * Input: inode, (logical) clu_offset, target allocation area + * Output: errcode, cluster number + * *clu = (~0), if it's unable to allocate a new cluster + */ +int exfat_map_cluster(struct inode *inode, unsigned int clu_offset, + unsigned int *clu, unsigned int *count, int create, + bool *balloc) +{ + int ret; + unsigned int last_clu; + struct exfat_chain new_clu; + struct super_block *sb = inode->i_sb; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct exfat_inode_info *ei = EXFAT_I(inode); + unsigned int local_clu_offset = clu_offset; + unsigned int num_to_be_allocated = 0, num_clusters; + + num_clusters = exfat_bytes_to_cluster(sbi, exfat_ondisk_size(inode)); + if (clu_offset > num_clusters || + *count > num_clusters - clu_offset) + num_to_be_allocated = clu_offset + *count - num_clusters; + + if (!create && (num_to_be_allocated > 0)) { + *clu = EXFAT_EOF_CLUSTER; + return 0; + } + + *clu = last_clu = ei->start_clu; + + if (*clu == EXFAT_EOF_CLUSTER) { + *count = 0; + } else if (ei->flags == ALLOC_NO_FAT_CHAIN) { + last_clu += num_clusters - 1; + if (clu_offset < num_clusters) { + *clu += clu_offset; + *count = min(num_clusters - clu_offset, *count); + } else { + *clu = EXFAT_EOF_CLUSTER; + *count = 0; + } + } else { + int err = exfat_get_cluster(inode, clu_offset, + clu, count, &last_clu); + if (err) + return -EIO; + } + + if (*clu == EXFAT_EOF_CLUSTER) { + exfat_set_volume_dirty(sb); + + new_clu.dir = (last_clu == EXFAT_EOF_CLUSTER) ? + EXFAT_EOF_CLUSTER : last_clu + 1; + new_clu.size = 0; + new_clu.flags = ei->flags; + + /* allocate a cluster */ + if (num_to_be_allocated < 1) { + /* Broken FAT (i_sze > allocated FAT) */ + exfat_fs_error(sb, "broken FAT chain."); + return -EIO; + } + + ret = exfat_alloc_cluster(inode, num_to_be_allocated, &new_clu, + inode_needs_sync(inode), true); + if (ret) + return ret; + + if (new_clu.dir == EXFAT_EOF_CLUSTER || + new_clu.dir == EXFAT_FREE_CLUSTER) { + exfat_fs_error(sb, + "bogus cluster new allocated (last_clu : %u, new_clu : %u)", + last_clu, new_clu.dir); + return -EIO; + } + + /* append to the FAT chain */ + if (last_clu == EXFAT_EOF_CLUSTER) { + if (new_clu.flags == ALLOC_FAT_CHAIN) + ei->flags = ALLOC_FAT_CHAIN; + ei->start_clu = new_clu.dir; + } else { + if (new_clu.flags != ei->flags) { + /* no-fat-chain bit is disabled, + * so fat-chain should be synced with + * alloc-bitmap + */ + if (exfat_chain_cont_cluster(sb, ei->start_clu, + num_clusters)) + return -EIO; + ei->flags = ALLOC_FAT_CHAIN; + } + if (new_clu.flags == ALLOC_FAT_CHAIN) + if (exfat_ent_set(sb, last_clu, new_clu.dir)) + return -EIO; + } + + *clu = new_clu.dir; + *count = new_clu.size; + + inode->i_blocks += exfat_cluster_to_sectors(sbi, new_clu.size); + if (balloc) + *balloc = true; + } + + /* hint information */ + ei->hint_bmap.off = local_clu_offset; + ei->hint_bmap.clu = *clu; + + return 0; +} + +static int exfat_read_folio(struct file *file, struct folio *folio) +{ + struct iomap_read_folio_ctx ctx = { + .cur_folio = folio, + .ops = &exfat_iomap_bio_read_ops, + }; + + iomap_read_folio(&exfat_iomap_ops, &ctx, NULL); + return 0; +} + +static void exfat_readahead(struct readahead_control *rac) +{ + struct address_space *mapping = rac->mapping; + struct inode *inode = mapping->host; + struct exfat_inode_info *ei = EXFAT_I(inode); + loff_t pos = readahead_pos(rac); + struct iomap_read_folio_ctx ctx = { + .ops = &exfat_iomap_bio_read_ops, + .rac = rac, + }; + + /* Range cross valid_size, read it page by page. */ + if (ei->valid_size < i_size_read(inode) && + pos <= ei->valid_size && + ei->valid_size < pos + readahead_length(rac)) + return; + + iomap_readahead(&exfat_iomap_ops, &ctx, NULL); +} + +static int exfat_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct iomap_writepage_ctx wpc = { + .inode = mapping->host, + .wbc = wbc, + .ops = &exfat_writeback_ops, + }; + + if (unlikely(exfat_forced_shutdown(mapping->host->i_sb))) + return -EIO; + + return iomap_writepages(&wpc); +} + +static sector_t exfat_aop_bmap(struct address_space *mapping, sector_t block) +{ + sector_t blocknr; + + inode_lock_shared(mapping->host); + blocknr = iomap_bmap(mapping, block, &exfat_iomap_ops); + inode_unlock_shared(mapping->host); + return blocknr; +} + +static const struct address_space_operations exfat_aops = { + .read_folio = exfat_read_folio, + .readahead = exfat_readahead, + .writepages = exfat_writepages, + .dirty_folio = iomap_dirty_folio, + .bmap = exfat_aop_bmap, + .migrate_folio = filemap_migrate_folio, + .is_partially_uptodate = iomap_is_partially_uptodate, + .error_remove_folio = generic_error_remove_folio, + .release_folio = iomap_release_folio, + .invalidate_folio = iomap_invalidate_folio, + .swap_activate = exfat_iomap_swap_activate, +}; + +static inline unsigned long exfat_hash(loff_t i_pos) +{ + return hash_32(i_pos, EXFAT_HASH_BITS); +} + +void exfat_hash_inode(struct inode *inode, loff_t i_pos) +{ + struct exfat_sb_info *sbi = EXFAT_SB(inode->i_sb); + struct hlist_head *head = sbi->inode_hashtable + exfat_hash(i_pos); + + spin_lock(&sbi->inode_hash_lock); + EXFAT_I(inode)->i_pos = i_pos; + hlist_add_head(&EXFAT_I(inode)->i_hash_fat, head); + spin_unlock(&sbi->inode_hash_lock); +} + +void exfat_unhash_inode(struct inode *inode) +{ + struct exfat_sb_info *sbi = EXFAT_SB(inode->i_sb); + + spin_lock(&sbi->inode_hash_lock); + hlist_del_init(&EXFAT_I(inode)->i_hash_fat); + EXFAT_I(inode)->i_pos = 0; + spin_unlock(&sbi->inode_hash_lock); +} + +struct inode *exfat_iget(struct super_block *sb, loff_t i_pos) +{ + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct exfat_inode_info *info; + struct hlist_head *head = sbi->inode_hashtable + exfat_hash(i_pos); + struct inode *inode = NULL; + + spin_lock(&sbi->inode_hash_lock); + hlist_for_each_entry(info, head, i_hash_fat) { + WARN_ON(info->vfs_inode.i_sb != sb); + + if (i_pos != info->i_pos) + continue; + inode = igrab(&info->vfs_inode); + if (inode) + break; + } + spin_unlock(&sbi->inode_hash_lock); + return inode; +} + +/* doesn't deal with root inode */ +static int exfat_fill_inode(struct inode *inode, struct exfat_dir_entry *info) +{ + struct exfat_sb_info *sbi = EXFAT_SB(inode->i_sb); + struct exfat_inode_info *ei = EXFAT_I(inode); + loff_t size = info->size; + + ei->dir = info->dir; + ei->entry = info->entry; + ei->attr = info->attr; + ei->start_clu = info->start_clu; + ei->flags = info->flags; + ei->type = info->type; + ei->valid_size = info->valid_size; + ei->zeroed_size = info->valid_size; + + ei->version = 0; + ei->hint_stat.eidx = 0; + ei->hint_stat.clu = info->start_clu; + ei->hint_femp.eidx = EXFAT_HINT_NONE; + ei->hint_bmap.off = EXFAT_EOF_CLUSTER; + ei->i_pos = 0; + + inode->i_uid = sbi->options.fs_uid; + inode->i_gid = sbi->options.fs_gid; + inode_inc_iversion(inode); + inode->i_generation = get_random_u32(); + + if (info->attr & EXFAT_ATTR_SUBDIR) { /* directory */ + inode->i_generation &= ~1; + inode->i_mode = exfat_make_mode(sbi, info->attr, 0777); + inode->i_op = &exfat_dir_inode_operations; + inode->i_fop = &exfat_dir_operations; + set_nlink(inode, info->num_subdirs); + } else { /* regular file */ + inode->i_generation |= 1; + inode->i_mode = exfat_make_mode(sbi, info->attr, 0777); + inode->i_op = &exfat_file_inode_operations; + inode->i_fop = &exfat_file_operations; + inode->i_mapping->a_ops = &exfat_aops; + inode->i_mapping->nrpages = 0; + } + + i_size_write(inode, size); + + exfat_save_attr(inode, info->attr); + + inode->i_blocks = round_up(i_size_read(inode), sbi->cluster_size) >> 9; + inode_set_mtime_to_ts(inode, info->mtime); + inode_set_ctime_to_ts(inode, info->mtime); + ei->i_crtime = info->crtime; + inode_set_atime_to_ts(inode, info->atime); + + return 0; +} + +struct inode *exfat_build_inode(struct super_block *sb, + struct exfat_dir_entry *info, loff_t i_pos) +{ + struct inode *inode; + int err; + + inode = exfat_iget(sb, i_pos); + if (inode) + goto out; + inode = new_inode(sb); + if (!inode) { + inode = ERR_PTR(-ENOMEM); + goto out; + } + inode->i_ino = iunique(sb, EXFAT_ROOT_INO); + inode_set_iversion(inode, 1); + err = exfat_fill_inode(inode, info); + if (err) { + iput(inode); + inode = ERR_PTR(err); + goto out; + } + exfat_hash_inode(inode, i_pos); + insert_inode_hash(inode); +out: + return inode; +} + +void exfat_evict_inode(struct inode *inode) +{ + truncate_inode_pages_final(&inode->i_data); + + if (!inode->i_nlink) { + i_size_write(inode, 0); + mutex_lock(&EXFAT_SB(inode->i_sb)->s_lock); + __exfat_truncate(inode); + mutex_unlock(&EXFAT_SB(inode->i_sb)->s_lock); + } + + clear_inode(inode); + exfat_cache_inval_inode(inode); + exfat_unhash_inode(inode); +} diff --git a/fs/exfat/iomap.c b/fs/exfat/iomap.c new file mode 100644 index 000000000..8911aa84a --- /dev/null +++ b/fs/exfat/iomap.c @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * iomap callack functions + * + * Copyright (C) 2026 Namjae Jeon + */ + +#include +#include + +#include "exfat_raw.h" +#include "exfat_fs.h" +#include "iomap.h" + +/* + * exfat_file_write_dio_end_io - Direct I/O write completion handler + * + * Updates i_size if the write extended the file. Called from the dio layer + * after I/O completion. + */ +static int exfat_file_write_dio_end_io(struct kiocb *iocb, ssize_t size, + int error, unsigned int flags) +{ + struct inode *inode = file_inode(iocb->ki_filp); + + if (error) + return error; + + if (size && i_size_read(inode) < iocb->ki_pos + size) { + i_size_write(inode, iocb->ki_pos + size); + mark_inode_dirty(inode); + } + + return 0; +} + +const struct iomap_dio_ops exfat_write_dio_ops = { + .end_io = exfat_file_write_dio_end_io, +}; + +static int __exfat_iomap_begin(struct inode *inode, loff_t offset, loff_t length, + unsigned int flags, struct iomap *iomap, bool may_alloc) +{ + struct super_block *sb = inode->i_sb; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct exfat_inode_info *ei = EXFAT_I(inode); + unsigned int cluster, num_clusters; + loff_t cluster_offset, cluster_length; + int err; + bool balloc = false; + + if (!may_alloc) { + /* Completely beyond EOF. Treat as hole */ + if (i_size_read(inode) <= offset) { + iomap->type = IOMAP_HOLE; + iomap->addr = IOMAP_NULL_ADDR; + iomap->offset = offset; + iomap->length = length; + return 0; + } + + /* Clamp length if the requested range goes beyond i_size */ + if (offset + length > i_size_read(inode)) + length = round_up(i_size_read(inode), + i_blocksize(inode)) - offset; + } + + num_clusters = exfat_bytes_to_cluster_round_up(sbi, + offset + length) - exfat_bytes_to_cluster(sbi, offset); + + mutex_lock(&sbi->s_lock); + iomap->bdev = inode->i_sb->s_bdev; + iomap->offset = offset; + + err = exfat_map_cluster(inode, exfat_bytes_to_cluster(sbi, offset), + &cluster, &num_clusters, may_alloc, &balloc); + if (err) + goto out; + + cluster_offset = exfat_cluster_offset(sbi, offset); + cluster_length = exfat_cluster_to_bytes(sbi, num_clusters); + + iomap->length = min_t(loff_t, length, cluster_length - cluster_offset); + iomap->addr = exfat_cluster_to_phys_bytes(sbi, cluster) + cluster_offset; + iomap->type = IOMAP_MAPPED; + iomap->flags = IOMAP_F_MERGED; + + if (may_alloc || flags & IOMAP_ZERO) { + if (balloc) + iomap->flags |= IOMAP_F_NEW; + else if (iomap->offset + iomap->length >= ei->valid_size) { + /* + * This is a write that starts at or extends beyond + * the current valid_size. The region between the old + * valid_size and the end of this write needs to be + * zeroed in the page cache to prevent stale data + * exposure (see IOMAP_F_ZERO_TAIL handling in + * __iomap_write_begin()). + */ + iomap->flags |= IOMAP_F_ZERO_TAIL; + } + } else { + /* + * valid_size is tracked in byte granularity and + * marks the exact boundary between valid data and + * holes (or unwritten space). + * + * When IOMAP_REPORT is set (used by lseek(SEEK_HOLE) + * and SEEK_DATA), we return IOMAP_HOLE. This allows + * iomap_seek_hole_iter() to directly return the + * precise byte position. + * + * For normal I/O paths (without IOMAP_REPORT) we + * return IOMAP_UNWRITTEN so the write path can + * distinguish it from a real hole. + */ + if (offset >= ei->valid_size) { + iomap->type = flags & IOMAP_REPORT ? + IOMAP_HOLE : IOMAP_UNWRITTEN; + } else if (offset + iomap->length > ei->valid_size) { + if (flags & IOMAP_REPORT) { + /* + * For SEEK_HOLE/SEEK_DATA, clip the length + * to the exact byte boundary (valid_size). + * This ensures the caller gets the precise + * hole position in byte units. + */ + iomap->length = ei->valid_size - iomap->offset; + } else + iomap->length = round_up(ei->valid_size, + i_blocksize(inode)) - + iomap->offset; + } + } + + iomap->flags |= IOMAP_F_MERGED; +out: + mutex_unlock(&sbi->s_lock); + return err; +} + +static int exfat_iomap_begin(struct inode *inode, loff_t offset, loff_t length, + unsigned int flags, struct iomap *iomap, struct iomap *srcmap) +{ + return __exfat_iomap_begin(inode, offset, length, flags, iomap, false); +} + +static int exfat_write_iomap_begin(struct inode *inode, loff_t offset, loff_t length, + unsigned int flags, struct iomap *iomap, struct iomap *srcmap) +{ + return __exfat_iomap_begin(inode, offset, length, flags, iomap, true); +} + +static DEFINE_IOMAP_ITER_NEXT(exfat_iomap_next, exfat_iomap_begin); + +const struct iomap_ops exfat_iomap_ops = { + .iomap_next = exfat_iomap_next, +}; + +/* + * exfat_write_iomap_end - Update the state after write + * + * Extends ->valid_size to cover the newly written range. + * Marks the inode dirty if metadata was changed. + */ +static int exfat_write_iomap_end(struct inode *inode, loff_t pos, loff_t length, + ssize_t written, unsigned int flags, struct iomap *iomap) +{ + struct exfat_inode_info *ei = EXFAT_I(inode); + bool dirtied = false; + loff_t end; + + if (!written) + return 0; + + end = pos + written; + + if (ei->valid_size < end) { + ei->valid_size = end; + dirtied = true; + } + + /* + * IOMAP_F_ZERO_TAIL zeroes the remainder of the last block. Track that + * block as zeroed so later valid_size extensions do not zero it again. + */ + if (iomap->flags & IOMAP_F_ZERO_TAIL) + end = round_up(end, i_blocksize(inode)); + if (ei->zeroed_size < end) + ei->zeroed_size = end; + + if (dirtied || iomap->flags & IOMAP_F_SIZE_CHANGED) + mark_inode_dirty(inode); + + return written; +} + +static DEFINE_IOMAP_ITER_NEXT_END(exfat_write_iomap_next, + exfat_write_iomap_begin, exfat_write_iomap_end); + +const struct iomap_ops exfat_write_iomap_ops = { + .iomap_next = exfat_write_iomap_next, +}; + +/* + * exfat_writeback_range - Map folio during writeback + * + * Called for each folio during writeback. If the folio falls outside the + * current iomap, remaps by calling read_iomap_begin. + */ +static ssize_t exfat_writeback_range(struct iomap_writepage_ctx *wpc, + struct folio *folio, u64 offset, unsigned int len, u64 end_pos) +{ + if (offset < wpc->iomap.offset || + offset >= wpc->iomap.offset + wpc->iomap.length) { + int error; + + error = __exfat_iomap_begin(wpc->inode, offset, len, + 0, &wpc->iomap, false); + if (error) + return error; + } + + return iomap_add_to_ioend(wpc, folio, offset, end_pos, len); +} + +const struct iomap_writeback_ops exfat_writeback_ops = { + .writeback_range = exfat_writeback_range, + .writeback_submit = iomap_ioend_writeback_submit, +}; + +/** + * exfat_iomap_read_end_io - iomap read bio completion handler for exFAT + * @bio: bio that has completed reading + * + * exfat_iomap_begin() rounds up MAPPED extents to the block boundary of + * valid_size. This ensures that any subsequent blocks are treated as + * IOMAP_UNWRITTEN, but it also causes the "straddle block" containing + * valid_size to be read from disk. The disk data beyond valid_size in + * this block is stale and must be zeroed to prevent data leakage. + */ +static void exfat_iomap_read_end_io(struct bio *bio) +{ + int error = blk_status_to_errno(bio->bi_status); + struct folio_iter iter; + + bio_for_each_folio_all(iter, bio) { + struct folio *folio = iter.folio; + struct exfat_inode_info *ei = EXFAT_I(folio->mapping->host); + s64 valid_size; + loff_t pos = folio_pos(folio); + + valid_size = ei->valid_size; + if (pos + iter.offset < valid_size && + pos + iter.offset + iter.length > valid_size) + folio_zero_segment(folio, offset_in_folio(folio, valid_size), + iter.offset + iter.length); + + iomap_finish_folio_read(folio, iter.offset, iter.length, error); + } + bio_put(bio); +} + +static void exfat_iomap_bio_submit_read(const struct iomap_iter *iter, + struct iomap_read_folio_ctx *ctx) +{ + iomap_bio_submit_read_endio(iter, ctx, exfat_iomap_read_end_io); +} + +const struct iomap_read_ops exfat_iomap_bio_read_ops = { + .read_folio_range = iomap_bio_read_folio_range, + .submit_read = exfat_iomap_bio_submit_read, +}; + +int exfat_iomap_swap_activate(struct swap_info_struct *sis, + struct file *file, sector_t *span) +{ + return iomap_swapfile_activate(sis, file, span, &exfat_iomap_ops); +} diff --git a/fs/exfat/iomap.h b/fs/exfat/iomap.h new file mode 100644 index 000000000..fd8a913f7 --- /dev/null +++ b/fs/exfat/iomap.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2026 Namjae Jeon + */ + +#ifndef _LINUX_EXFAT_IOMAP_H +#define _LINUX_EXFAT_IOMAP_H + +extern const struct iomap_dio_ops exfat_write_dio_ops; +extern const struct iomap_ops exfat_iomap_ops; +extern const struct iomap_ops exfat_write_iomap_ops; +extern const struct iomap_writeback_ops exfat_writeback_ops; +extern const struct iomap_read_ops exfat_iomap_bio_read_ops; + +int exfat_iomap_swap_activate(struct swap_info_struct *sis, + struct file *file, sector_t *span); + +#endif /* _LINUX_EXFAT_IOMAP_H */ diff --git a/fs/exfat/misc.c b/fs/exfat/misc.c new file mode 100644 index 000000000..6f11a96a4 --- /dev/null +++ b/fs/exfat/misc.c @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Written 1992,1993 by Werner Almesberger + * 22/11/2000 - Fixed fat_date_unix2dos for dates earlier than 01/01/1980 + * and date_dos2unix for date==0 by Igor Zhbanov(bsg@uniyar.ac.ru) + * Copyright (C) 2012-2013 Samsung Electronics Co., Ltd. + */ + +#include +#include +#include +#include +#include + +#include "exfat_raw.h" +#include "exfat_fs.h" + +/* + * exfat_fs_error reports a file system problem that might indicate fa data + * corruption/inconsistency. Depending on 'errors' mount option the + * panic() is called, or error message is printed FAT and nothing is done, + * or filesystem is remounted read-only (default behavior). + * In case the file system is remounted read-only, it can be made writable + * again by remounting it. + */ +void __exfat_fs_error(struct super_block *sb, int report, const char *fmt, ...) +{ + struct exfat_mount_options *opts = &EXFAT_SB(sb)->options; + va_list args; + struct va_format vaf; + + if (report) { + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + exfat_err(sb, "error, %pV", &vaf); + va_end(args); + } + + if (opts->errors == EXFAT_ERRORS_PANIC) { + panic("exFAT-fs (%s): fs panic from previous error\n", + sb->s_id); + } else if (opts->errors == EXFAT_ERRORS_RO && !sb_rdonly(sb)) { + sb->s_flags |= SB_RDONLY; + exfat_err(sb, "Filesystem has been set read-only"); + } +} + +#define SECS_PER_MIN (60) +#define TIMEZONE_SEC(x) ((x) * 15 * SECS_PER_MIN) + +static void exfat_adjust_tz(struct timespec64 *ts, u8 tz_off) +{ + if (tz_off <= 0x3F) + ts->tv_sec -= TIMEZONE_SEC(tz_off); + else /* 0x40 <= (tz_off & 0x7F) <=0x7F */ + ts->tv_sec += TIMEZONE_SEC(0x80 - tz_off); +} + +static inline int exfat_tz_offset(struct exfat_sb_info *sbi) +{ + if (sbi->options.sys_tz) + return -sys_tz.tz_minuteswest; + return sbi->options.time_offset; +} + +/* Convert a EXFAT time/date pair to a UNIX date (seconds since 1 1 70). */ +void exfat_get_entry_time(struct exfat_sb_info *sbi, struct timespec64 *ts, + u8 tz, __le16 time, __le16 date, u8 time_cs) +{ + u16 t = le16_to_cpu(time); + u16 d = le16_to_cpu(date); + + ts->tv_sec = mktime64(1980 + (d >> 9), d >> 5 & 0x000F, d & 0x001F, + t >> 11, (t >> 5) & 0x003F, (t & 0x001F) << 1); + + + /* time_cs field represent 0 ~ 199cs(1990 ms) */ + if (time_cs) { + ts->tv_sec += time_cs / 100; + ts->tv_nsec = (time_cs % 100) * 10 * NSEC_PER_MSEC; + } else + ts->tv_nsec = 0; + + if (tz & EXFAT_TZ_VALID) + /* Adjust timezone to UTC0. */ + exfat_adjust_tz(ts, tz & ~EXFAT_TZ_VALID); + else + ts->tv_sec -= exfat_tz_offset(sbi) * SECS_PER_MIN; +} + +/* Convert linear UNIX date to a EXFAT time/date pair. */ +void exfat_set_entry_time(struct exfat_sb_info *sbi, struct timespec64 *ts, + u8 *tz, __le16 *time, __le16 *date, u8 *time_cs) +{ + struct tm tm; + u16 t, d; + + time64_to_tm(ts->tv_sec, 0, &tm); + t = (tm.tm_hour << 11) | (tm.tm_min << 5) | (tm.tm_sec >> 1); + d = ((tm.tm_year - 80) << 9) | ((tm.tm_mon + 1) << 5) | tm.tm_mday; + + *time = cpu_to_le16(t); + *date = cpu_to_le16(d); + + /* time_cs field represent 0 ~ 199cs(1990 ms) */ + if (time_cs) + *time_cs = (tm.tm_sec & 1) * 100 + + ts->tv_nsec / (10 * NSEC_PER_MSEC); + + /* + * Record 00h value for OffsetFromUtc field and 1 value for OffsetValid + * to indicate that local time and UTC are the same. + */ + *tz = EXFAT_TZ_VALID; +} + +/* + * The timestamp for access_time has double seconds granularity. + * (There is no 10msIncrement field for access_time unlike create/modify_time) + * atime also has only a 2-second resolution. + */ +void exfat_truncate_atime(struct timespec64 *ts) +{ + ts->tv_sec = round_down(ts->tv_sec, 2); + ts->tv_nsec = 0; +} + +void exfat_truncate_inode_atime(struct inode *inode) +{ + struct timespec64 atime = inode_get_atime(inode); + + exfat_truncate_atime(&atime); + inode_set_atime_to_ts(inode, atime); +} + +u16 exfat_calc_chksum16(void *data, int len, u16 chksum, int type) +{ + int i; + u8 *c = (u8 *)data; + + for (i = 0; i < len; i++, c++) { + if (unlikely(type == CS_DIR_ENTRY && (i == 2 || i == 3))) + continue; + chksum = ((chksum << 15) | (chksum >> 1)) + *c; + } + return chksum; +} + +u32 exfat_calc_chksum32(void *data, int len, u32 chksum, int type) +{ + int i; + u8 *c = (u8 *)data; + + for (i = 0; i < len; i++, c++) { + if (unlikely(type == CS_BOOT_SECTOR && + (i == 106 || i == 107 || i == 112))) + continue; + chksum = ((chksum << 31) | (chksum >> 1)) + *c; + } + return chksum; +} + +int exfat_update_bh(struct buffer_head *bh, int sync) +{ + int err = 0; + + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + + if (sync) + err = sync_dirty_buffer(bh); + + return err; +} + +int exfat_update_bhs(struct buffer_head **bhs, int nr_bhs, int sync) +{ + int i, err = 0; + + for (i = 0; i < nr_bhs; i++) { + set_buffer_uptodate(bhs[i]); + mark_buffer_dirty(bhs[i]); + if (sync) + write_dirty_buffer(bhs[i], REQ_SYNC); + } + + for (i = 0; i < nr_bhs && sync; i++) { + wait_on_buffer(bhs[i]); + if (!err && !buffer_uptodate(bhs[i])) + err = -EIO; + } + return err; +} + +void exfat_chain_set(struct exfat_chain *ec, unsigned int dir, + unsigned int size, unsigned char flags) +{ + ec->dir = dir; + ec->size = size; + ec->flags = flags; +} + +void exfat_chain_dup(struct exfat_chain *dup, struct exfat_chain *ec) +{ + return exfat_chain_set(dup, ec->dir, ec->size, ec->flags); +} diff --git a/fs/exfat/namei.c b/fs/exfat/namei.c new file mode 100644 index 000000000..a4dc83b59 --- /dev/null +++ b/fs/exfat/namei.c @@ -0,0 +1,1351 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2012-2013 Samsung Electronics Co., Ltd. + */ + +#include +#include +#include +#include +#include + +#include "exfat_raw.h" +#include "exfat_fs.h" + +static inline unsigned long exfat_d_version(struct dentry *dentry) +{ + return (unsigned long) dentry->d_fsdata; +} + +static inline void exfat_d_version_set(struct dentry *dentry, + unsigned long version) +{ + dentry->d_fsdata = (void *) version; +} + +/* + * If new entry was created in the parent, it could create the 8.3 alias (the + * shortname of logname). So, the parent may have the negative-dentry which + * matches the created 8.3 alias. + * + * If it happened, the negative dentry isn't actually negative anymore. So, + * drop it. + */ +static int exfat_d_revalidate(struct inode *dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + if (flags & LOOKUP_RCU) + return -ECHILD; + + /* + * This is not negative dentry. Always valid. + * + * Note, rename() to existing directory entry will have ->d_inode, and + * will use existing name which isn't specified name by user. + * + * We may be able to drop this positive dentry here. But dropping + * positive dentry isn't good idea. So it's unsupported like + * rename("filename", "FILENAME") for now. + */ + if (d_really_is_positive(dentry)) + return 1; + + /* + * Drop the negative dentry, in order to make sure to use the case + * sensitive name which is specified by user if this is for creation. + */ + if (flags & (LOOKUP_CREATE | LOOKUP_RENAME_TARGET)) + return 0; + + return inode_eq_iversion(dir, exfat_d_version(dentry)); +} + +/* returns the length of a struct qstr, ignoring trailing dots if necessary */ +static unsigned int exfat_striptail_len(unsigned int len, const char *name, + bool keep_last_dots) +{ + if (!keep_last_dots) { + while (len && name[len - 1] == '.') + len--; + } + return len; +} + +/* + * Compute the hash for the exfat name corresponding to the dentry. If the name + * is invalid, we leave the hash code unchanged so that the existing dentry can + * be used. The exfat fs routines will return ENOENT or EINVAL as appropriate. + */ +static int exfat_d_hash(const struct dentry *dentry, struct qstr *qstr) +{ + struct super_block *sb = dentry->d_sb; + struct nls_table *t = EXFAT_SB(sb)->nls_io; + const unsigned char *name = qstr->name; + unsigned int len = exfat_striptail_len(qstr->len, qstr->name, + EXFAT_SB(sb)->options.keep_last_dots); + unsigned long hash = init_name_hash(dentry); + int i, charlen; + wchar_t c; + + for (i = 0; i < len; i += charlen) { + charlen = t->char2uni(&name[i], len - i, &c); + if (charlen < 0) + return charlen; + hash = partial_name_hash(exfat_toupper(sb, c), hash); + } + + qstr->hash = end_name_hash(hash); + return 0; +} + +static int exfat_d_cmp(const struct dentry *dentry, unsigned int len, + const char *str, const struct qstr *name) +{ + struct super_block *sb = dentry->d_sb; + struct nls_table *t = EXFAT_SB(sb)->nls_io; + unsigned int alen = exfat_striptail_len(name->len, name->name, + EXFAT_SB(sb)->options.keep_last_dots); + unsigned int blen = exfat_striptail_len(len, str, + EXFAT_SB(sb)->options.keep_last_dots); + wchar_t c1, c2; + int charlen, i; + + if (alen != blen) + return 1; + + for (i = 0; i < len; i += charlen) { + charlen = t->char2uni(&name->name[i], alen - i, &c1); + if (charlen < 0) + return 1; + if (charlen != t->char2uni(&str[i], blen - i, &c2)) + return 1; + + if (exfat_toupper(sb, c1) != exfat_toupper(sb, c2)) + return 1; + } + + return 0; +} + +const struct dentry_operations exfat_dentry_ops = { + .d_revalidate = exfat_d_revalidate, + .d_hash = exfat_d_hash, + .d_compare = exfat_d_cmp, +}; + +static int exfat_utf8_d_hash(const struct dentry *dentry, struct qstr *qstr) +{ + struct super_block *sb = dentry->d_sb; + const unsigned char *name = qstr->name; + unsigned int len = exfat_striptail_len(qstr->len, qstr->name, + EXFAT_SB(sb)->options.keep_last_dots); + unsigned long hash = init_name_hash(dentry); + int i, charlen; + unicode_t u; + + for (i = 0; i < len; i += charlen) { + charlen = utf8_to_utf32(&name[i], len - i, &u); + if (charlen < 0) + return charlen; + + /* + * exfat_toupper() works only for code points up to the U+FFFF. + */ + hash = partial_name_hash(u <= 0xFFFF ? exfat_toupper(sb, u) : u, + hash); + } + + qstr->hash = end_name_hash(hash); + return 0; +} + +static int exfat_utf8_d_cmp(const struct dentry *dentry, unsigned int len, + const char *str, const struct qstr *name) +{ + struct super_block *sb = dentry->d_sb; + unsigned int alen = exfat_striptail_len(name->len, name->name, + EXFAT_SB(sb)->options.keep_last_dots); + unsigned int blen = exfat_striptail_len(len, str, + EXFAT_SB(sb)->options.keep_last_dots); + + unicode_t u_a, u_b; + int charlen, i; + + if (alen != blen) + return 1; + + for (i = 0; i < alen; i += charlen) { + charlen = utf8_to_utf32(&name->name[i], alen - i, &u_a); + if (charlen < 0) + return 1; + if (charlen != utf8_to_utf32(&str[i], blen - i, &u_b)) + return 1; + + if (u_a <= 0xFFFF && u_b <= 0xFFFF) { + if (exfat_toupper(sb, u_a) != exfat_toupper(sb, u_b)) + return 1; + } else { + if (u_a != u_b) + return 1; + } + } + + return 0; +} + +const struct dentry_operations exfat_utf8_dentry_ops = { + .d_revalidate = exfat_d_revalidate, + .d_hash = exfat_utf8_d_hash, + .d_compare = exfat_utf8_d_cmp, +}; + +/* search EMPTY CONTINUOUS "num_entries" entries */ +static int exfat_search_empty_slot(struct super_block *sb, + struct exfat_hint_femp *hint_femp, struct exfat_chain *p_dir, + int num_entries, struct exfat_entry_set_cache *es) +{ + int i, dentry, ret; + int dentries_per_clu; + struct exfat_chain clu; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + unsigned int total_entries = exfat_cluster_to_dentries(sbi, p_dir->size); + + dentries_per_clu = sbi->dentries_per_clu; + + if (hint_femp->eidx != EXFAT_HINT_NONE) { + dentry = hint_femp->eidx; + + /* + * If hint_femp->count is enough, it is needed to check if + * there are actual empty entries. + * Otherwise, and if "dentry + hint_famp->count" is also equal + * to "p_dir->size * dentries_per_clu", it means ENOSPC. + */ + if (dentry + hint_femp->count == total_entries && + num_entries > hint_femp->count) + return -ENOSPC; + + hint_femp->eidx = EXFAT_HINT_NONE; + exfat_chain_dup(&clu, &hint_femp->cur); + } else { + exfat_chain_dup(&clu, p_dir); + dentry = 0; + } + + while (dentry + num_entries <= total_entries && + clu.dir != EXFAT_EOF_CLUSTER) { + i = dentry & (dentries_per_clu - 1); + + ret = exfat_get_empty_dentry_set(es, sb, &clu, i, num_entries); + if (ret < 0) + return ret; + else if (ret == 0) + return dentry; + + dentry += ret; + i += ret; + + while (i >= dentries_per_clu) { + if (exfat_chain_advance(sb, &clu, 1)) + return -EIO; + + i -= dentries_per_clu; + } + } + + hint_femp->eidx = dentry; + hint_femp->count = 0; + if (dentry == total_entries || clu.dir == EXFAT_EOF_CLUSTER) + exfat_chain_set(&hint_femp->cur, EXFAT_EOF_CLUSTER, 0, + clu.flags); + else + hint_femp->cur = clu; + + return -ENOSPC; +} + +static int exfat_check_max_dentries(struct inode *inode) +{ + if (exfat_bytes_to_dentries(i_size_read(inode)) >= MAX_EXFAT_DENTRIES) { + /* + * exFAT spec allows a dir to grow up to 8388608(256MB) + * dentries + */ + return -ENOSPC; + } + return 0; +} + +/* + * Find an empty directory entry set. + * + * If there isn't any empty slot, expand cluster chain. + * + * in: + * inode: inode of the parent directory + * num_entries: specifies how many dentries in the empty directory entry set + * + * out: + * p_dir: the cluster where the empty directory entry set is located + * es: The found empty directory entry set + * + * return: + * the directory entry index in p_dir is returned on succeeds + * -error code is returned on failure + */ +int exfat_find_empty_entry(struct inode *inode, + struct exfat_chain *p_dir, int num_entries, + struct exfat_entry_set_cache *es) +{ + int dentry, ret; + unsigned int last_clu; + loff_t size = 0; + struct exfat_chain clu; + struct super_block *sb = inode->i_sb; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct exfat_inode_info *ei = EXFAT_I(inode); + struct exfat_hint_femp hint_femp; + + hint_femp.eidx = EXFAT_HINT_NONE; + + if (ei->hint_femp.eidx != EXFAT_HINT_NONE) { + hint_femp = ei->hint_femp; + ei->hint_femp.eidx = EXFAT_HINT_NONE; + } + + exfat_chain_set(p_dir, ei->start_clu, + exfat_bytes_to_cluster(sbi, i_size_read(inode)), + ei->flags); + + while ((dentry = exfat_search_empty_slot(sb, &hint_femp, p_dir, + num_entries, es)) < 0) { + if (dentry != -ENOSPC) + return dentry; + + if (exfat_check_max_dentries(inode)) + return -ENOSPC; + + /* + * Allocate new cluster to this directory + */ + if (ei->start_clu != EXFAT_EOF_CLUSTER) { + /* we trust p_dir->size regardless of FAT type */ + if (exfat_find_last_cluster(sb, p_dir, &last_clu)) + return -EIO; + + exfat_chain_set(&clu, last_clu + 1, 0, p_dir->flags); + } else { + /* This directory is empty */ + exfat_chain_set(&clu, EXFAT_EOF_CLUSTER, 0, + ALLOC_NO_FAT_CHAIN); + } + + /* allocate a cluster */ + ret = exfat_alloc_cluster(inode, 1, &clu, IS_DIRSYNC(inode), false); + if (ret) + return ret; + + if (exfat_zeroed_cluster(inode, clu.dir)) + return -EIO; + + if (ei->start_clu == EXFAT_EOF_CLUSTER) { + ei->start_clu = clu.dir; + p_dir->dir = clu.dir; + hint_femp.eidx = 0; + } + + /* append to the FAT chain */ + if (clu.flags != p_dir->flags) { + /* no-fat-chain bit is disabled, + * so fat-chain should be synced with alloc-bitmap + */ + if (exfat_chain_cont_cluster(sb, p_dir->dir, p_dir->size)) + return -EIO; + p_dir->flags = ALLOC_FAT_CHAIN; + hint_femp.cur.flags = ALLOC_FAT_CHAIN; + } + + if (clu.flags == ALLOC_FAT_CHAIN) + if (exfat_ent_set(sb, last_clu, clu.dir)) + return -EIO; + + if (hint_femp.cur.dir == EXFAT_EOF_CLUSTER) + exfat_chain_set(&hint_femp.cur, clu.dir, 0, clu.flags); + + hint_femp.count += sbi->dentries_per_clu; + + hint_femp.cur.size++; + p_dir->size++; + size = exfat_cluster_to_bytes(sbi, p_dir->size); + + /* directory inode should be updated in here */ + i_size_write(inode, size); + ei->valid_size += sbi->cluster_size; + ei->flags = p_dir->flags; + inode->i_blocks += sbi->cluster_size >> 9; + } + + p_dir->dir = exfat_sector_to_cluster(sbi, es->bh[0]->b_blocknr); + p_dir->size -= dentry / sbi->dentries_per_clu; + + return dentry & (sbi->dentries_per_clu - 1); +} + +/* + * Name Resolution Functions : + * Zero if it was successful; otherwise nonzero. + */ +static int __exfat_resolve_path(struct inode *inode, const unsigned char *path, + struct exfat_uni_name *p_uniname, int lookup) +{ + int namelen; + int lossy = NLS_NAME_NO_LOSSY; + struct super_block *sb = inode->i_sb; + int pathlen = strlen(path); + + /* + * get the length of the pathname excluding + * trailing periods, if any. + */ + namelen = exfat_striptail_len(pathlen, path, false); + if (EXFAT_SB(sb)->options.keep_last_dots) { + /* + * Do not allow the creation of files with names + * ending with period(s). + */ + if (!lookup && (namelen < pathlen)) + return -EINVAL; + namelen = pathlen; + } + if (!namelen) + return -ENOENT; + if (pathlen > (MAX_NAME_LENGTH * MAX_CHARSET_SIZE)) + return -ENAMETOOLONG; + + /* + * strip all leading spaces : + * "MS windows 7" supports leading spaces. + * So we should skip this preprocessing for compatibility. + */ + + /* file name conversion : + * If lookup case, we allow bad-name for compatibility. + */ + namelen = exfat_nls_to_utf16(sb, path, namelen, p_uniname, + &lossy); + if (namelen < 0) + return namelen; /* return error value */ + + if ((lossy && !lookup) || !namelen) + return -EINVAL; + + return 0; +} + +static inline int exfat_resolve_path(struct inode *inode, + const unsigned char *path, struct exfat_uni_name *uni) +{ + return __exfat_resolve_path(inode, path, uni, 0); +} + +static inline int exfat_resolve_path_for_lookup(struct inode *inode, + const unsigned char *path, struct exfat_uni_name *uni) +{ + return __exfat_resolve_path(inode, path, uni, 1); +} + +static inline loff_t exfat_make_i_pos(struct exfat_dir_entry *info) +{ + return ((loff_t) info->dir.dir << 32) | (info->entry & 0xffffffff); +} + +static int exfat_add_entry(struct inode *inode, const char *path, + unsigned int type, struct exfat_dir_entry *info) +{ + int ret, dentry, num_entries; + struct super_block *sb = inode->i_sb; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct exfat_uni_name uniname; + struct exfat_chain clu; + struct timespec64 ts = current_time(inode); + struct exfat_entry_set_cache es; + int clu_size = 0; + unsigned int start_clu = EXFAT_FREE_CLUSTER; + bool dir_allocated = false; + + ret = exfat_resolve_path(inode, path, &uniname); + if (ret) + goto out; + + num_entries = exfat_calc_num_entries(&uniname); + if (num_entries < 0) { + ret = num_entries; + goto out; + } + + /* exfat_find_empty_entry must be called before alloc_cluster() */ + dentry = exfat_find_empty_entry(inode, &info->dir, num_entries, &es); + if (dentry < 0) { + ret = dentry; /* -EIO or -ENOSPC */ + goto out; + } + + if (type == TYPE_DIR && !sbi->options.zero_size_dir) { + ret = exfat_alloc_new_dir(inode, &clu); + if (ret) { + exfat_put_dentry_set(&es, false); + goto out; + } + start_clu = clu.dir; + clu_size = sbi->cluster_size; + dir_allocated = true; + } + + /* update the directory entry */ + /* fill the dos name directory entry information of the created file. + * the first cluster is not determined yet. (0) + */ + exfat_init_dir_entry(&es, type, start_clu, clu_size, &ts); + exfat_init_ext_entry(&es, num_entries, &uniname, NULL, 0); + + ret = exfat_put_dentry_set(&es, IS_DIRSYNC(inode)); + if (ret) { + int cleanup_ret; + + cleanup_ret = exfat_get_dentry_set(&es, sb, &info->dir, + dentry, ES_ALL_ENTRIES); + if (!cleanup_ret) { + exfat_remove_entries(inode, &es, ES_IDX_FILE, false); + cleanup_ret = exfat_put_dentry_set(&es, + IS_DIRSYNC(inode)); + } + + if (!cleanup_ret && dir_allocated) + exfat_free_cluster(inode, &clu); + goto out; + } + + info->entry = dentry; + info->flags = ALLOC_NO_FAT_CHAIN; + info->type = type; + + if (type == TYPE_FILE) { + info->attr = EXFAT_ATTR_ARCHIVE; + info->start_clu = EXFAT_EOF_CLUSTER; + info->size = 0; + info->num_subdirs = 0; + } else { + info->attr = EXFAT_ATTR_SUBDIR; + if (sbi->options.zero_size_dir) + info->start_clu = EXFAT_EOF_CLUSTER; + else + info->start_clu = start_clu; + info->size = clu_size; + info->num_subdirs = EXFAT_MIN_SUBDIR; + } + info->valid_size = info->size; + + memset(&info->crtime, 0, sizeof(info->crtime)); + memset(&info->mtime, 0, sizeof(info->mtime)); + memset(&info->atime, 0, sizeof(info->atime)); +out: + return ret; +} + +static int exfat_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct super_block *sb = dir->i_sb; + struct inode *inode; + struct exfat_dir_entry info; + loff_t i_pos; + int err; + loff_t size = i_size_read(dir); + + if (unlikely(exfat_forced_shutdown(sb))) + return -EIO; + + mutex_lock(&EXFAT_SB(sb)->s_lock); + exfat_set_volume_dirty(sb); + err = exfat_add_entry(dir, dentry->d_name.name, TYPE_FILE, &info); + if (err) + goto unlock; + + inode_inc_iversion(dir); + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + if (IS_DIRSYNC(dir) && size != i_size_read(dir)) + exfat_sync_inode(dir); + else + mark_inode_dirty(dir); + + i_pos = exfat_make_i_pos(&info); + inode = exfat_build_inode(sb, &info, i_pos); + err = PTR_ERR_OR_ZERO(inode); + if (err) + goto unlock; + + inode_inc_iversion(inode); + EXFAT_I(inode)->i_crtime = simple_inode_init_ts(inode); + exfat_truncate_inode_atime(inode); + + /* timestamp is already written, so mark_inode_dirty() is unneeded. */ + + d_instantiate(dentry, inode); +unlock: + mutex_unlock(&EXFAT_SB(sb)->s_lock); + return err; +} + +/* lookup a file */ +static int exfat_find(struct inode *dir, const struct qstr *qname, + struct exfat_dir_entry *info) +{ + int ret, dentry, count; + struct exfat_chain cdir; + struct exfat_uni_name uni_name; + struct super_block *sb = dir->i_sb; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct exfat_inode_info *ei = EXFAT_I(dir); + struct exfat_dentry *ep, *ep2; + struct exfat_entry_set_cache es; + /* for optimized dir & entry to prevent long traverse of cluster chain */ + struct exfat_hint hint_opt; + + if (qname->len == 0) + return -ENOENT; + + /* check the validity of directory name in the given pathname */ + ret = exfat_resolve_path_for_lookup(dir, qname->name, &uni_name); + if (ret) + return ret; + + exfat_chain_set(&cdir, ei->start_clu, + exfat_bytes_to_cluster(sbi, i_size_read(dir)), ei->flags); + + /* check the validation of hint_stat and initialize it if required */ + if (ei->version != (inode_peek_iversion_raw(dir) & 0xffffffff)) { + ei->hint_stat.clu = cdir.dir; + ei->hint_stat.eidx = 0; + ei->version = (inode_peek_iversion_raw(dir) & 0xffffffff); + ei->hint_femp.eidx = EXFAT_HINT_NONE; + } + + /* search the file name for directories */ + dentry = exfat_find_dir_entry(sb, ei, &cdir, &uni_name, &hint_opt); + if (dentry < 0) + return dentry; /* -error value */ + + /* adjust cdir to the optimized value */ + cdir.dir = hint_opt.clu; + if (cdir.flags & ALLOC_NO_FAT_CHAIN) + cdir.size -= dentry / sbi->dentries_per_clu; + dentry = hint_opt.eidx; + + info->dir = cdir; + info->entry = dentry; + info->num_subdirs = 0; + + if (exfat_get_dentry_set(&es, sb, &cdir, dentry, ES_2_ENTRIES)) + return -EIO; + ep = exfat_get_dentry_cached(&es, ES_IDX_FILE); + ep2 = exfat_get_dentry_cached(&es, ES_IDX_STREAM); + + info->type = exfat_get_entry_type(ep); + info->attr = le16_to_cpu(ep->dentry.file.attr); + info->valid_size = le64_to_cpu(ep2->dentry.stream.valid_size); + info->size = le64_to_cpu(ep2->dentry.stream.size); + + info->start_clu = le32_to_cpu(ep2->dentry.stream.start_clu); + if (!is_valid_cluster(sbi, info->start_clu) && info->size) { + exfat_warn(sb, "start_clu is invalid cluster(0x%x)", + info->start_clu); + info->size = 0; + info->valid_size = 0; + } + + if (info->valid_size > info->size) { + exfat_warn(sb, "valid_size(%lld) is greater than size(%lld)", + info->valid_size, info->size); + info->valid_size = info->size; + } + + if (info->size == 0) { + info->flags = ALLOC_NO_FAT_CHAIN; + info->start_clu = EXFAT_EOF_CLUSTER; + } else + info->flags = ep2->dentry.stream.flags; + + exfat_get_entry_time(sbi, &info->crtime, + ep->dentry.file.create_tz, + ep->dentry.file.create_time, + ep->dentry.file.create_date, + ep->dentry.file.create_time_cs); + exfat_get_entry_time(sbi, &info->mtime, + ep->dentry.file.modify_tz, + ep->dentry.file.modify_time, + ep->dentry.file.modify_date, + ep->dentry.file.modify_time_cs); + exfat_get_entry_time(sbi, &info->atime, + ep->dentry.file.access_tz, + ep->dentry.file.access_time, + ep->dentry.file.access_date, + 0); + exfat_put_dentry_set(&es, false); + + if (info->valid_size < 0) { + exfat_fs_error(sb, "data valid size is invalid(%lld)", info->valid_size); + return -EIO; + } + + if (unlikely(exfat_bytes_to_cluster_round_up(sbi, info->size) > sbi->used_clusters)) { + exfat_fs_error(sb, "data size is invalid(%lld)", info->size); + return -EIO; + } + + if (ei->start_clu == EXFAT_FREE_CLUSTER) { + exfat_fs_error(sb, + "non-zero size file starts with zero cluster (size : %llu, p_dir : %u, entry : 0x%08x)", + i_size_read(dir), ei->dir.dir, ei->entry); + return -EIO; + } + + if (info->type == TYPE_DIR) { + exfat_chain_set(&cdir, info->start_clu, + exfat_bytes_to_cluster(sbi, info->size), + info->flags); + count = exfat_count_dir_entries(sb, &cdir); + if (count < 0) + return -EIO; + + info->num_subdirs = count + EXFAT_MIN_SUBDIR; + } + return 0; +} + +static struct dentry *exfat_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + struct super_block *sb = dir->i_sb; + struct inode *inode = NULL; + struct dentry *alias; + struct exfat_dir_entry info; + int err; + loff_t i_pos; + + mutex_lock(&EXFAT_SB(sb)->s_lock); + err = exfat_find(dir, &dentry->d_name, &info); + if (err) { + if (unlikely(err != -ENOENT)) + inode = ERR_PTR(err); + goto out; + } + + i_pos = exfat_make_i_pos(&info); + inode = exfat_build_inode(sb, &info, i_pos); + if (IS_ERR(inode) || S_ISDIR(inode->i_mode)) + goto out; + + alias = d_find_alias(inode); + + /* + * Checking "alias->d_parent == dentry->d_parent" to make sure + * FS is not corrupted (especially double linked dir). + */ + if (alias && alias->d_parent == dentry->d_parent) { + /* + * This inode has a hashed alias dentry with different + * name. This means, the user did ->lookup() by an + * another name (longname vs 8.3 alias of it) in past. + * + * Switch to new one for reason of locality if possible. + */ + d_move(alias, dentry); + iput(inode); + mutex_unlock(&EXFAT_SB(sb)->s_lock); + return alias; + } + dput(alias); +out: + mutex_unlock(&EXFAT_SB(sb)->s_lock); + if (!inode) + exfat_d_version_set(dentry, inode_query_iversion(dir)); + + return d_splice_alias(inode, dentry); +} + +/* remove an entry, BUT don't truncate */ +static int exfat_unlink(struct inode *dir, struct dentry *dentry) +{ + struct super_block *sb = dir->i_sb; + struct inode *inode = dentry->d_inode; + struct exfat_inode_info *ei = EXFAT_I(inode); + struct exfat_entry_set_cache es; + int err = 0; + + if (unlikely(exfat_forced_shutdown(sb))) + return -EIO; + + mutex_lock(&EXFAT_SB(sb)->s_lock); + if (ei->dir.dir == DIR_DELETED) { + exfat_err(sb, "abnormal access to deleted dentry"); + err = -ENOENT; + goto unlock; + } + + err = exfat_get_dentry_set_by_ei(&es, sb, ei); + if (err) { + err = -EIO; + goto unlock; + } + + exfat_set_volume_dirty(sb); + + /* update the directory entry */ + exfat_remove_entries(inode, &es, ES_IDX_FILE, true); + + err = exfat_put_dentry_set(&es, IS_DIRSYNC(inode)); + if (err) + goto unlock; + + /* This doesn't modify ei */ + ei->dir.dir = DIR_DELETED; + + inode_inc_iversion(dir); + simple_inode_init_ts(dir); + exfat_truncate_inode_atime(dir); + mark_inode_dirty(dir); + + clear_nlink(inode); + simple_inode_init_ts(inode); + exfat_truncate_inode_atime(inode); + exfat_unhash_inode(inode); + exfat_d_version_set(dentry, inode_query_iversion(dir)); +unlock: + mutex_unlock(&EXFAT_SB(sb)->s_lock); + return err; +} + +static struct dentry *exfat_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct super_block *sb = dir->i_sb; + struct inode *inode; + struct exfat_dir_entry info; + loff_t i_pos; + int err; + loff_t size = i_size_read(dir); + + if (unlikely(exfat_forced_shutdown(sb))) + return ERR_PTR(-EIO); + + mutex_lock(&EXFAT_SB(sb)->s_lock); + exfat_set_volume_dirty(sb); + err = exfat_add_entry(dir, dentry->d_name.name, TYPE_DIR, &info); + if (err) + goto unlock; + + inode_inc_iversion(dir); + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + if (IS_DIRSYNC(dir) && size != i_size_read(dir)) + exfat_sync_inode(dir); + else + mark_inode_dirty(dir); + inc_nlink(dir); + + i_pos = exfat_make_i_pos(&info); + inode = exfat_build_inode(sb, &info, i_pos); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + goto unlock; + } + + inode_inc_iversion(inode); + EXFAT_I(inode)->i_crtime = simple_inode_init_ts(inode); + exfat_truncate_inode_atime(inode); + /* timestamp is already written, so mark_inode_dirty() is unneeded. */ + + d_instantiate(dentry, inode); + +unlock: + mutex_unlock(&EXFAT_SB(sb)->s_lock); + return err ? ERR_PTR(err) : NULL; +} + +static int exfat_check_dir_empty(struct super_block *sb, + struct exfat_chain *p_dir) +{ + int i, dentries_per_clu; + unsigned int type; + unsigned int clu_count = 0; + struct exfat_chain clu; + struct exfat_dentry *ep; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct buffer_head *bh; + + dentries_per_clu = sbi->dentries_per_clu; + + if (p_dir->dir == EXFAT_EOF_CLUSTER) + return 0; + + exfat_chain_dup(&clu, p_dir); + + while (clu.dir != EXFAT_EOF_CLUSTER) { + for (i = 0; i < dentries_per_clu; i++) { + ep = exfat_get_dentry(sb, &clu, i, &bh); + if (!ep) + return -EIO; + type = exfat_get_entry_type(ep); + brelse(bh); + if (type == TYPE_UNUSED) + return 0; + + if (type != TYPE_FILE && type != TYPE_DIR) + continue; + + return -ENOTEMPTY; + } + + if (exfat_chain_advance(sb, &clu, 1)) + return -EIO; + + /* break if the cluster chain includes a loop */ + if (unlikely(++clu_count > EXFAT_DATA_CLUSTER_COUNT(sbi))) + break; + } + + return 0; +} + +static int exfat_rmdir(struct inode *dir, struct dentry *dentry) +{ + struct inode *inode = dentry->d_inode; + struct exfat_chain clu_to_free; + struct super_block *sb = inode->i_sb; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct exfat_inode_info *ei = EXFAT_I(inode); + struct exfat_entry_set_cache es; + int err; + + if (unlikely(exfat_forced_shutdown(sb))) + return -EIO; + + mutex_lock(&EXFAT_SB(inode->i_sb)->s_lock); + + if (ei->dir.dir == DIR_DELETED) { + exfat_err(sb, "abnormal access to deleted dentry"); + err = -ENOENT; + goto unlock; + } + + exfat_chain_set(&clu_to_free, ei->start_clu, + exfat_bytes_to_cluster_round_up(sbi, i_size_read(inode)), ei->flags); + + err = exfat_check_dir_empty(sb, &clu_to_free); + if (err) { + if (err == -EIO) + exfat_err(sb, "failed to exfat_check_dir_empty : err(%d)", + err); + goto unlock; + } + + err = exfat_get_dentry_set_by_ei(&es, sb, ei); + if (err) { + err = -EIO; + goto unlock; + } + + exfat_set_volume_dirty(sb); + + exfat_remove_entries(inode, &es, ES_IDX_FILE, true); + + err = exfat_put_dentry_set(&es, IS_DIRSYNC(dir)); + if (err) + goto unlock; + + ei->dir.dir = DIR_DELETED; + + inode_inc_iversion(dir); + simple_inode_init_ts(dir); + exfat_truncate_inode_atime(dir); + if (IS_DIRSYNC(dir)) + exfat_sync_inode(dir); + else + mark_inode_dirty(dir); + drop_nlink(dir); + + clear_nlink(inode); + simple_inode_init_ts(inode); + exfat_truncate_inode_atime(inode); + exfat_unhash_inode(inode); + exfat_d_version_set(dentry, inode_query_iversion(dir)); +unlock: + mutex_unlock(&EXFAT_SB(inode->i_sb)->s_lock); + return err; +} + +/* + * Count benign secondary entries beyond the filename entries. + * Returns the count, or -EIO if the entry set is inconsistent. + */ +static int exfat_count_extra_entries(struct exfat_entry_set_cache *es) +{ + struct exfat_dentry *stream; + unsigned int name_entries; + int extra; + + stream = exfat_get_dentry_cached(es, ES_IDX_STREAM); + name_entries = EXFAT_FILENAME_ENTRY_NUM(stream->dentry.stream.name_len); + extra = es->num_entries - (ES_IDX_FIRST_FILENAME + name_entries); + + return extra >= 0 ? extra : -EIO; +} + +static int exfat_rename_file(struct inode *parent_inode, + struct exfat_uni_name *p_uniname, struct exfat_inode_info *ei) +{ + int ret, num_new_entries; + struct exfat_dentry *epold, *epnew; + struct super_block *sb = parent_inode->i_sb; + struct exfat_entry_set_cache old_es, new_es; + int sync = IS_DIRSYNC(parent_inode); + unsigned int num_extra_entries, num_total_entries; + + if (unlikely(exfat_forced_shutdown(sb))) + return -EIO; + + num_new_entries = exfat_calc_num_entries(p_uniname); + if (num_new_entries < 0) + return num_new_entries; + + ret = exfat_get_dentry_set_by_ei(&old_es, sb, ei); + if (ret) + return -EIO; + + epold = exfat_get_dentry_cached(&old_es, ES_IDX_FILE); + + ret = exfat_count_extra_entries(&old_es); + if (ret < 0) + goto put_old_es; + num_extra_entries = ret; + num_total_entries = num_new_entries + num_extra_entries; + + if (old_es.num_entries < num_total_entries) { + int newentry; + struct exfat_chain dir; + + newentry = exfat_find_empty_entry(parent_inode, &dir, + num_total_entries, &new_es); + if (newentry < 0) { + ret = newentry; /* -EIO or -ENOSPC */ + goto put_old_es; + } + + epnew = exfat_get_dentry_cached(&new_es, ES_IDX_FILE); + *epnew = *epold; + if (exfat_get_entry_type(epnew) == TYPE_FILE) { + epnew->dentry.file.attr |= cpu_to_le16(EXFAT_ATTR_ARCHIVE); + ei->attr |= EXFAT_ATTR_ARCHIVE; + } + + epold = exfat_get_dentry_cached(&old_es, ES_IDX_STREAM); + epnew = exfat_get_dentry_cached(&new_es, ES_IDX_STREAM); + *epnew = *epold; + + exfat_init_ext_entry(&new_es, num_new_entries, p_uniname, + &old_es, num_extra_entries); + + ret = exfat_put_dentry_set(&new_es, sync); + if (ret) { + /* Best-effort delete to avoid duplicate entries */ + if (!exfat_get_dentry_set(&new_es, sb, + &dir, newentry, + ES_ALL_ENTRIES)) { + exfat_remove_entries(parent_inode, &new_es, + ES_IDX_FILE, false); + exfat_put_dentry_set(&new_es, false); + } + goto put_old_es; + } + + exfat_remove_entries(parent_inode, &old_es, ES_IDX_FILE, false); + ei->dir = dir; + ei->entry = newentry; + } else { + if (exfat_get_entry_type(epold) == TYPE_FILE) { + epold->dentry.file.attr |= cpu_to_le16(EXFAT_ATTR_ARCHIVE); + ei->attr |= EXFAT_ATTR_ARCHIVE; + } + + exfat_init_ext_entry(&old_es, num_new_entries, p_uniname, + &old_es, num_extra_entries); + } + return exfat_put_dentry_set(&old_es, sync); + +put_old_es: + exfat_put_dentry_set(&old_es, false); + return ret; +} + +static int exfat_move_file(struct inode *parent_inode, + struct exfat_uni_name *p_uniname, struct exfat_inode_info *ei) +{ + int ret, newentry, num_new_entries; + struct exfat_dentry *epmov, *epnew; + struct exfat_entry_set_cache mov_es, new_es; + struct exfat_chain newdir; + unsigned int num_extra_entries, num_total_entries; + + num_new_entries = exfat_calc_num_entries(p_uniname); + if (num_new_entries < 0) + return num_new_entries; + + ret = exfat_get_dentry_set_by_ei(&mov_es, parent_inode->i_sb, ei); + if (ret) + return -EIO; + + ret = exfat_count_extra_entries(&mov_es); + if (ret < 0) + goto put_mov_es; + num_extra_entries = ret; + num_total_entries = num_new_entries + num_extra_entries; + + newentry = exfat_find_empty_entry(parent_inode, &newdir, + num_total_entries, &new_es); + if (newentry < 0) { + ret = newentry; /* -EIO or -ENOSPC */ + goto put_mov_es; + } + + epmov = exfat_get_dentry_cached(&mov_es, ES_IDX_FILE); + epnew = exfat_get_dentry_cached(&new_es, ES_IDX_FILE); + *epnew = *epmov; + if (exfat_get_entry_type(epnew) == TYPE_FILE) { + epnew->dentry.file.attr |= cpu_to_le16(EXFAT_ATTR_ARCHIVE); + ei->attr |= EXFAT_ATTR_ARCHIVE; + } + + epmov = exfat_get_dentry_cached(&mov_es, ES_IDX_STREAM); + epnew = exfat_get_dentry_cached(&new_es, ES_IDX_STREAM); + *epnew = *epmov; + + exfat_init_ext_entry(&new_es, num_new_entries, p_uniname, + &mov_es, num_extra_entries); + + ret = exfat_put_dentry_set(&new_es, IS_DIRSYNC(parent_inode)); + if (ret) { + /* Best-effort delete to avoid duplicate entries */ + if (!exfat_get_dentry_set(&new_es, parent_inode->i_sb, + &newdir, newentry, + ES_ALL_ENTRIES)) { + exfat_remove_entries(parent_inode, &new_es, + ES_IDX_FILE, false); + exfat_put_dentry_set(&new_es, false); + } + goto put_mov_es; + } + + exfat_remove_entries(parent_inode, &mov_es, ES_IDX_FILE, false); + + ei->dir = newdir; + ei->entry = newentry; + + return exfat_put_dentry_set(&mov_es, IS_DIRSYNC(parent_inode)); + +put_mov_es: + exfat_put_dentry_set(&mov_es, false); + return ret; +} + +/* rename or move a old file into a new file */ +static int __exfat_rename(struct inode *old_parent_inode, + struct exfat_inode_info *ei, struct inode *new_parent_inode, + struct dentry *new_dentry) +{ + int ret; + struct exfat_uni_name uni_name; + struct super_block *sb = old_parent_inode->i_sb; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + const unsigned char *new_path = new_dentry->d_name.name; + struct inode *new_inode = new_dentry->d_inode; + struct exfat_inode_info *new_ei = NULL; + + /* check the validity of pointer parameters */ + if (new_path == NULL || strlen(new_path) == 0) + return -EINVAL; + + if (ei->dir.dir == DIR_DELETED) { + exfat_err(sb, "abnormal access to deleted source dentry"); + return -ENOENT; + } + + /* check whether new dir is existing directory and empty */ + if (new_inode) { + ret = -EIO; + new_ei = EXFAT_I(new_inode); + + if (new_ei->dir.dir == DIR_DELETED) { + exfat_err(sb, "abnormal access to deleted target dentry"); + goto out; + } + + /* if new_inode exists, update ei */ + if (S_ISDIR(new_inode->i_mode)) { + struct exfat_chain new_clu; + + new_clu.dir = new_ei->start_clu; + new_clu.size = + exfat_bytes_to_cluster_round_up(sbi, + i_size_read(new_inode)); + new_clu.flags = new_ei->flags; + + ret = exfat_check_dir_empty(sb, &new_clu); + if (ret) + goto out; + } + } + + /* check the validity of directory name in the given new pathname */ + ret = exfat_resolve_path(new_parent_inode, new_path, &uni_name); + if (ret) + goto out; + + exfat_set_volume_dirty(sb); + + if (new_parent_inode == old_parent_inode) + ret = exfat_rename_file(new_parent_inode, &uni_name, ei); + else + ret = exfat_move_file(new_parent_inode, &uni_name, ei); + + if (!ret && new_inode) { + struct exfat_entry_set_cache es; + + /* delete entries of new_dir */ + ret = exfat_get_dentry_set_by_ei(&es, sb, new_ei); + if (ret) { + ret = -EIO; + goto del_out; + } + + exfat_remove_entries(new_inode, &es, ES_IDX_FILE, true); + + ret = exfat_put_dentry_set(&es, IS_DIRSYNC(new_inode)); + if (ret) + goto del_out; + + /* Free the clusters if new_inode is a dir(as if exfat_rmdir) */ + if (S_ISDIR(new_inode->i_mode) && + new_ei->start_clu != EXFAT_EOF_CLUSTER) { + /* new_ei, new_clu_to_free */ + struct exfat_chain new_clu_to_free; + + exfat_chain_set(&new_clu_to_free, new_ei->start_clu, + exfat_bytes_to_cluster_round_up(sbi, i_size_read(new_inode)), + new_ei->flags); + + if (exfat_free_cluster(new_inode, &new_clu_to_free)) { + /* just set I/O error only */ + ret = -EIO; + } + + i_size_write(new_inode, 0); + new_ei->valid_size = 0; + new_ei->start_clu = EXFAT_EOF_CLUSTER; + new_ei->flags = ALLOC_NO_FAT_CHAIN; + } +del_out: + /* Update new_inode ei + * Prevent syncing removed new_inode + * (new_ei is already initialized above code ("if (new_inode)") + */ + new_ei->dir.dir = DIR_DELETED; + } +out: + return ret; +} + +static int exfat_rename(struct mnt_idmap *idmap, + struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry, + unsigned int flags) +{ + struct inode *old_inode, *new_inode; + struct super_block *sb = old_dir->i_sb; + loff_t i_pos; + int err; + loff_t size = i_size_read(new_dir); + + /* + * The VFS already checks for existence, so for local filesystems + * the RENAME_NOREPLACE implementation is equivalent to plain rename. + * Don't support any other flags + */ + if (flags & ~RENAME_NOREPLACE) + return -EINVAL; + + mutex_lock(&EXFAT_SB(sb)->s_lock); + old_inode = old_dentry->d_inode; + new_inode = new_dentry->d_inode; + + err = __exfat_rename(old_dir, EXFAT_I(old_inode), new_dir, new_dentry); + if (err) + goto unlock; + + inode_inc_iversion(new_dir); + simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry); + EXFAT_I(new_dir)->i_crtime = current_time(new_dir); + exfat_truncate_inode_atime(new_dir); + if (IS_DIRSYNC(new_dir) && size != i_size_read(new_dir)) + exfat_sync_inode(new_dir); + else + mark_inode_dirty(new_dir); + + i_pos = ((loff_t)EXFAT_I(old_inode)->dir.dir << 32) | + (EXFAT_I(old_inode)->entry & 0xffffffff); + exfat_unhash_inode(old_inode); + exfat_hash_inode(old_inode, i_pos); + if (IS_DIRSYNC(new_dir)) + exfat_sync_inode(old_inode); + else + mark_inode_dirty(old_inode); + + if (S_ISDIR(old_inode->i_mode) && old_dir != new_dir) { + drop_nlink(old_dir); + if (!new_inode) + inc_nlink(new_dir); + } + + inode_inc_iversion(old_dir); + if (new_dir != old_dir) + mark_inode_dirty(old_dir); + + if (new_inode) { + exfat_unhash_inode(new_inode); + + /* skip drop_nlink if new_inode already has been dropped */ + if (new_inode->i_nlink) { + drop_nlink(new_inode); + if (S_ISDIR(new_inode->i_mode)) + drop_nlink(new_inode); + } else { + exfat_warn(sb, "abnormal access to an inode dropped"); + WARN_ON(new_inode->i_nlink == 0); + } + EXFAT_I(new_inode)->i_crtime = current_time(new_inode); + } + +unlock: + mutex_unlock(&EXFAT_SB(sb)->s_lock); + return err; +} + +const struct inode_operations exfat_dir_inode_operations = { + .create = exfat_create, + .lookup = exfat_lookup, + .unlink = exfat_unlink, + .mkdir = exfat_mkdir, + .rmdir = exfat_rmdir, + .rename = exfat_rename, + .setattr = exfat_setattr, + .getattr = exfat_getattr, + .fileattr_get = exfat_fileattr_get, +}; diff --git a/fs/exfat/nls.c b/fs/exfat/nls.c new file mode 100644 index 000000000..055447edc --- /dev/null +++ b/fs/exfat/nls.c @@ -0,0 +1,809 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2012-2013 Samsung Electronics Co., Ltd. + */ + +#include +#include +#include +#include + +#include "exfat_raw.h" +#include "exfat_fs.h" + +/* Upcase table macro */ +#define EXFAT_NUM_UPCASE (2918) +#define UTBL_COUNT (0x10000) + +/* + * Upcase table in compressed format (7.2.5.1 Recommended Up-case Table + * in exfat specification, See: + * https://docs.microsoft.com/en-us/windows/win32/fileio/exfat-specification). + */ +static const unsigned short uni_def_upcase[EXFAT_NUM_UPCASE] = { + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, 0x002c, 0x002d, 0x002e, 0x002f, + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, 0x005c, 0x005d, 0x005e, 0x005f, + 0x0060, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, 0x008c, 0x008d, 0x008e, 0x008f, + 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, + 0x00a0, 0x00a1, 0x00a2, 0x00a3, 0x00a4, 0x00a5, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, + 0x00b8, 0x00b9, 0x00ba, 0x00bb, 0x00bc, 0x00bd, 0x00be, 0x00bf, + 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, + 0x00d0, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00d7, + 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x00df, + 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, + 0x00d0, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x00f7, + 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x0178, + 0x0100, 0x0100, 0x0102, 0x0102, 0x0104, 0x0104, 0x0106, 0x0106, + 0x0108, 0x0108, 0x010a, 0x010a, 0x010c, 0x010c, 0x010e, 0x010e, + 0x0110, 0x0110, 0x0112, 0x0112, 0x0114, 0x0114, 0x0116, 0x0116, + 0x0118, 0x0118, 0x011a, 0x011a, 0x011c, 0x011c, 0x011e, 0x011e, + 0x0120, 0x0120, 0x0122, 0x0122, 0x0124, 0x0124, 0x0126, 0x0126, + 0x0128, 0x0128, 0x012a, 0x012a, 0x012c, 0x012c, 0x012e, 0x012e, + 0x0130, 0x0131, 0x0132, 0x0132, 0x0134, 0x0134, 0x0136, 0x0136, + 0x0138, 0x0139, 0x0139, 0x013b, 0x013b, 0x013d, 0x013d, 0x013f, + 0x013f, 0x0141, 0x0141, 0x0143, 0x0143, 0x0145, 0x0145, 0x0147, + 0x0147, 0x0149, 0x014a, 0x014a, 0x014c, 0x014c, 0x014e, 0x014e, + 0x0150, 0x0150, 0x0152, 0x0152, 0x0154, 0x0154, 0x0156, 0x0156, + 0x0158, 0x0158, 0x015a, 0x015a, 0x015c, 0x015c, 0x015e, 0x015e, + 0x0160, 0x0160, 0x0162, 0x0162, 0x0164, 0x0164, 0x0166, 0x0166, + 0x0168, 0x0168, 0x016a, 0x016a, 0x016c, 0x016c, 0x016e, 0x016e, + 0x0170, 0x0170, 0x0172, 0x0172, 0x0174, 0x0174, 0x0176, 0x0176, + 0x0178, 0x0179, 0x0179, 0x017b, 0x017b, 0x017d, 0x017d, 0x017f, + 0x0243, 0x0181, 0x0182, 0x0182, 0x0184, 0x0184, 0x0186, 0x0187, + 0x0187, 0x0189, 0x018a, 0x018b, 0x018b, 0x018d, 0x018e, 0x018f, + 0x0190, 0x0191, 0x0191, 0x0193, 0x0194, 0x01f6, 0x0196, 0x0197, + 0x0198, 0x0198, 0x023d, 0x019b, 0x019c, 0x019d, 0x0220, 0x019f, + 0x01a0, 0x01a0, 0x01a2, 0x01a2, 0x01a4, 0x01a4, 0x01a6, 0x01a7, + 0x01a7, 0x01a9, 0x01aa, 0x01ab, 0x01ac, 0x01ac, 0x01ae, 0x01af, + 0x01af, 0x01b1, 0x01b2, 0x01b3, 0x01b3, 0x01b5, 0x01b5, 0x01b7, + 0x01b8, 0x01b8, 0x01ba, 0x01bb, 0x01bc, 0x01bc, 0x01be, 0x01f7, + 0x01c0, 0x01c1, 0x01c2, 0x01c3, 0x01c4, 0x01c5, 0x01c4, 0x01c7, + 0x01c8, 0x01c7, 0x01ca, 0x01cb, 0x01ca, 0x01cd, 0x01cd, 0x01cf, + 0x01cf, 0x01d1, 0x01d1, 0x01d3, 0x01d3, 0x01d5, 0x01d5, 0x01d7, + 0x01d7, 0x01d9, 0x01d9, 0x01db, 0x01db, 0x018e, 0x01de, 0x01de, + 0x01e0, 0x01e0, 0x01e2, 0x01e2, 0x01e4, 0x01e4, 0x01e6, 0x01e6, + 0x01e8, 0x01e8, 0x01ea, 0x01ea, 0x01ec, 0x01ec, 0x01ee, 0x01ee, + 0x01f0, 0x01f1, 0x01f2, 0x01f1, 0x01f4, 0x01f4, 0x01f6, 0x01f7, + 0x01f8, 0x01f8, 0x01fa, 0x01fa, 0x01fc, 0x01fc, 0x01fe, 0x01fe, + 0x0200, 0x0200, 0x0202, 0x0202, 0x0204, 0x0204, 0x0206, 0x0206, + 0x0208, 0x0208, 0x020a, 0x020a, 0x020c, 0x020c, 0x020e, 0x020e, + 0x0210, 0x0210, 0x0212, 0x0212, 0x0214, 0x0214, 0x0216, 0x0216, + 0x0218, 0x0218, 0x021a, 0x021a, 0x021c, 0x021c, 0x021e, 0x021e, + 0x0220, 0x0221, 0x0222, 0x0222, 0x0224, 0x0224, 0x0226, 0x0226, + 0x0228, 0x0228, 0x022a, 0x022a, 0x022c, 0x022c, 0x022e, 0x022e, + 0x0230, 0x0230, 0x0232, 0x0232, 0x0234, 0x0235, 0x0236, 0x0237, + 0x0238, 0x0239, 0x2c65, 0x023b, 0x023b, 0x023d, 0x2c66, 0x023f, + 0x0240, 0x0241, 0x0241, 0x0243, 0x0244, 0x0245, 0x0246, 0x0246, + 0x0248, 0x0248, 0x024a, 0x024a, 0x024c, 0x024c, 0x024e, 0x024e, + 0x0250, 0x0251, 0x0252, 0x0181, 0x0186, 0x0255, 0x0189, 0x018a, + 0x0258, 0x018f, 0x025a, 0x0190, 0x025c, 0x025d, 0x025e, 0x025f, + 0x0193, 0x0261, 0x0262, 0x0194, 0x0264, 0x0265, 0x0266, 0x0267, + 0x0197, 0x0196, 0x026a, 0x2c62, 0x026c, 0x026d, 0x026e, 0x019c, + 0x0270, 0x0271, 0x019d, 0x0273, 0x0274, 0x019f, 0x0276, 0x0277, + 0x0278, 0x0279, 0x027a, 0x027b, 0x027c, 0x2c64, 0x027e, 0x027f, + 0x01a6, 0x0281, 0x0282, 0x01a9, 0x0284, 0x0285, 0x0286, 0x0287, + 0x01ae, 0x0244, 0x01b1, 0x01b2, 0x0245, 0x028d, 0x028e, 0x028f, + 0x0290, 0x0291, 0x01b7, 0x0293, 0x0294, 0x0295, 0x0296, 0x0297, + 0x0298, 0x0299, 0x029a, 0x029b, 0x029c, 0x029d, 0x029e, 0x029f, + 0x02a0, 0x02a1, 0x02a2, 0x02a3, 0x02a4, 0x02a5, 0x02a6, 0x02a7, + 0x02a8, 0x02a9, 0x02aa, 0x02ab, 0x02ac, 0x02ad, 0x02ae, 0x02af, + 0x02b0, 0x02b1, 0x02b2, 0x02b3, 0x02b4, 0x02b5, 0x02b6, 0x02b7, + 0x02b8, 0x02b9, 0x02ba, 0x02bb, 0x02bc, 0x02bd, 0x02be, 0x02bf, + 0x02c0, 0x02c1, 0x02c2, 0x02c3, 0x02c4, 0x02c5, 0x02c6, 0x02c7, + 0x02c8, 0x02c9, 0x02ca, 0x02cb, 0x02cc, 0x02cd, 0x02ce, 0x02cf, + 0x02d0, 0x02d1, 0x02d2, 0x02d3, 0x02d4, 0x02d5, 0x02d6, 0x02d7, + 0x02d8, 0x02d9, 0x02da, 0x02db, 0x02dc, 0x02dd, 0x02de, 0x02df, + 0x02e0, 0x02e1, 0x02e2, 0x02e3, 0x02e4, 0x02e5, 0x02e6, 0x02e7, + 0x02e8, 0x02e9, 0x02ea, 0x02eb, 0x02ec, 0x02ed, 0x02ee, 0x02ef, + 0x02f0, 0x02f1, 0x02f2, 0x02f3, 0x02f4, 0x02f5, 0x02f6, 0x02f7, + 0x02f8, 0x02f9, 0x02fa, 0x02fb, 0x02fc, 0x02fd, 0x02fe, 0x02ff, + 0x0300, 0x0301, 0x0302, 0x0303, 0x0304, 0x0305, 0x0306, 0x0307, + 0x0308, 0x0309, 0x030a, 0x030b, 0x030c, 0x030d, 0x030e, 0x030f, + 0x0310, 0x0311, 0x0312, 0x0313, 0x0314, 0x0315, 0x0316, 0x0317, + 0x0318, 0x0319, 0x031a, 0x031b, 0x031c, 0x031d, 0x031e, 0x031f, + 0x0320, 0x0321, 0x0322, 0x0323, 0x0324, 0x0325, 0x0326, 0x0327, + 0x0328, 0x0329, 0x032a, 0x032b, 0x032c, 0x032d, 0x032e, 0x032f, + 0x0330, 0x0331, 0x0332, 0x0333, 0x0334, 0x0335, 0x0336, 0x0337, + 0x0338, 0x0339, 0x033a, 0x033b, 0x033c, 0x033d, 0x033e, 0x033f, + 0x0340, 0x0341, 0x0342, 0x0343, 0x0344, 0x0345, 0x0346, 0x0347, + 0x0348, 0x0349, 0x034a, 0x034b, 0x034c, 0x034d, 0x034e, 0x034f, + 0x0350, 0x0351, 0x0352, 0x0353, 0x0354, 0x0355, 0x0356, 0x0357, + 0x0358, 0x0359, 0x035a, 0x035b, 0x035c, 0x035d, 0x035e, 0x035f, + 0x0360, 0x0361, 0x0362, 0x0363, 0x0364, 0x0365, 0x0366, 0x0367, + 0x0368, 0x0369, 0x036a, 0x036b, 0x036c, 0x036d, 0x036e, 0x036f, + 0x0370, 0x0371, 0x0372, 0x0373, 0x0374, 0x0375, 0x0376, 0x0377, + 0x0378, 0x0379, 0x037a, 0x03fd, 0x03fe, 0x03ff, 0x037e, 0x037f, + 0x0380, 0x0381, 0x0382, 0x0383, 0x0384, 0x0385, 0x0386, 0x0387, + 0x0388, 0x0389, 0x038a, 0x038b, 0x038c, 0x038d, 0x038e, 0x038f, + 0x0390, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, + 0x0398, 0x0399, 0x039a, 0x039b, 0x039c, 0x039d, 0x039e, 0x039f, + 0x03a0, 0x03a1, 0x03a2, 0x03a3, 0x03a4, 0x03a5, 0x03a6, 0x03a7, + 0x03a8, 0x03a9, 0x03aa, 0x03ab, 0x0386, 0x0388, 0x0389, 0x038a, + 0x03b0, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, + 0x0398, 0x0399, 0x039a, 0x039b, 0x039c, 0x039d, 0x039e, 0x039f, + 0x03a0, 0x03a1, 0x03a3, 0x03a3, 0x03a4, 0x03a5, 0x03a6, 0x03a7, + 0x03a8, 0x03a9, 0x03aa, 0x03ab, 0x038c, 0x038e, 0x038f, 0x03cf, + 0x03d0, 0x03d1, 0x03d2, 0x03d3, 0x03d4, 0x03d5, 0x03d6, 0x03d7, + 0x03d8, 0x03d8, 0x03da, 0x03da, 0x03dc, 0x03dc, 0x03de, 0x03de, + 0x03e0, 0x03e0, 0x03e2, 0x03e2, 0x03e4, 0x03e4, 0x03e6, 0x03e6, + 0x03e8, 0x03e8, 0x03ea, 0x03ea, 0x03ec, 0x03ec, 0x03ee, 0x03ee, + 0x03f0, 0x03f1, 0x03f9, 0x03f3, 0x03f4, 0x03f5, 0x03f6, 0x03f7, + 0x03f7, 0x03f9, 0x03fa, 0x03fa, 0x03fc, 0x03fd, 0x03fe, 0x03ff, + 0x0400, 0x0401, 0x0402, 0x0403, 0x0404, 0x0405, 0x0406, 0x0407, + 0x0408, 0x0409, 0x040a, 0x040b, 0x040c, 0x040d, 0x040e, 0x040f, + 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, + 0x0418, 0x0419, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, 0x041f, + 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, + 0x0428, 0x0429, 0x042a, 0x042b, 0x042c, 0x042d, 0x042e, 0x042f, + 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, + 0x0418, 0x0419, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, 0x041f, + 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, + 0x0428, 0x0429, 0x042a, 0x042b, 0x042c, 0x042d, 0x042e, 0x042f, + 0x0400, 0x0401, 0x0402, 0x0403, 0x0404, 0x0405, 0x0406, 0x0407, + 0x0408, 0x0409, 0x040a, 0x040b, 0x040c, 0x040d, 0x040e, 0x040f, + 0x0460, 0x0460, 0x0462, 0x0462, 0x0464, 0x0464, 0x0466, 0x0466, + 0x0468, 0x0468, 0x046a, 0x046a, 0x046c, 0x046c, 0x046e, 0x046e, + 0x0470, 0x0470, 0x0472, 0x0472, 0x0474, 0x0474, 0x0476, 0x0476, + 0x0478, 0x0478, 0x047a, 0x047a, 0x047c, 0x047c, 0x047e, 0x047e, + 0x0480, 0x0480, 0x0482, 0x0483, 0x0484, 0x0485, 0x0486, 0x0487, + 0x0488, 0x0489, 0x048a, 0x048a, 0x048c, 0x048c, 0x048e, 0x048e, + 0x0490, 0x0490, 0x0492, 0x0492, 0x0494, 0x0494, 0x0496, 0x0496, + 0x0498, 0x0498, 0x049a, 0x049a, 0x049c, 0x049c, 0x049e, 0x049e, + 0x04a0, 0x04a0, 0x04a2, 0x04a2, 0x04a4, 0x04a4, 0x04a6, 0x04a6, + 0x04a8, 0x04a8, 0x04aa, 0x04aa, 0x04ac, 0x04ac, 0x04ae, 0x04ae, + 0x04b0, 0x04b0, 0x04b2, 0x04b2, 0x04b4, 0x04b4, 0x04b6, 0x04b6, + 0x04b8, 0x04b8, 0x04ba, 0x04ba, 0x04bc, 0x04bc, 0x04be, 0x04be, + 0x04c0, 0x04c1, 0x04c1, 0x04c3, 0x04c3, 0x04c5, 0x04c5, 0x04c7, + 0x04c7, 0x04c9, 0x04c9, 0x04cb, 0x04cb, 0x04cd, 0x04cd, 0x04c0, + 0x04d0, 0x04d0, 0x04d2, 0x04d2, 0x04d4, 0x04d4, 0x04d6, 0x04d6, + 0x04d8, 0x04d8, 0x04da, 0x04da, 0x04dc, 0x04dc, 0x04de, 0x04de, + 0x04e0, 0x04e0, 0x04e2, 0x04e2, 0x04e4, 0x04e4, 0x04e6, 0x04e6, + 0x04e8, 0x04e8, 0x04ea, 0x04ea, 0x04ec, 0x04ec, 0x04ee, 0x04ee, + 0x04f0, 0x04f0, 0x04f2, 0x04f2, 0x04f4, 0x04f4, 0x04f6, 0x04f6, + 0x04f8, 0x04f8, 0x04fa, 0x04fa, 0x04fc, 0x04fc, 0x04fe, 0x04fe, + 0x0500, 0x0500, 0x0502, 0x0502, 0x0504, 0x0504, 0x0506, 0x0506, + 0x0508, 0x0508, 0x050a, 0x050a, 0x050c, 0x050c, 0x050e, 0x050e, + 0x0510, 0x0510, 0x0512, 0x0512, 0x0514, 0x0515, 0x0516, 0x0517, + 0x0518, 0x0519, 0x051a, 0x051b, 0x051c, 0x051d, 0x051e, 0x051f, + 0x0520, 0x0521, 0x0522, 0x0523, 0x0524, 0x0525, 0x0526, 0x0527, + 0x0528, 0x0529, 0x052a, 0x052b, 0x052c, 0x052d, 0x052e, 0x052f, + 0x0530, 0x0531, 0x0532, 0x0533, 0x0534, 0x0535, 0x0536, 0x0537, + 0x0538, 0x0539, 0x053a, 0x053b, 0x053c, 0x053d, 0x053e, 0x053f, + 0x0540, 0x0541, 0x0542, 0x0543, 0x0544, 0x0545, 0x0546, 0x0547, + 0x0548, 0x0549, 0x054a, 0x054b, 0x054c, 0x054d, 0x054e, 0x054f, + 0x0550, 0x0551, 0x0552, 0x0553, 0x0554, 0x0555, 0x0556, 0x0557, + 0x0558, 0x0559, 0x055a, 0x055b, 0x055c, 0x055d, 0x055e, 0x055f, + 0x0560, 0x0531, 0x0532, 0x0533, 0x0534, 0x0535, 0x0536, 0x0537, + 0x0538, 0x0539, 0x053a, 0x053b, 0x053c, 0x053d, 0x053e, 0x053f, + 0x0540, 0x0541, 0x0542, 0x0543, 0x0544, 0x0545, 0x0546, 0x0547, + 0x0548, 0x0549, 0x054a, 0x054b, 0x054c, 0x054d, 0x054e, 0x054f, + 0x0550, 0x0551, 0x0552, 0x0553, 0x0554, 0x0555, 0x0556, 0xffff, + 0x17f6, 0x2c63, 0x1d7e, 0x1d7f, 0x1d80, 0x1d81, 0x1d82, 0x1d83, + 0x1d84, 0x1d85, 0x1d86, 0x1d87, 0x1d88, 0x1d89, 0x1d8a, 0x1d8b, + 0x1d8c, 0x1d8d, 0x1d8e, 0x1d8f, 0x1d90, 0x1d91, 0x1d92, 0x1d93, + 0x1d94, 0x1d95, 0x1d96, 0x1d97, 0x1d98, 0x1d99, 0x1d9a, 0x1d9b, + 0x1d9c, 0x1d9d, 0x1d9e, 0x1d9f, 0x1da0, 0x1da1, 0x1da2, 0x1da3, + 0x1da4, 0x1da5, 0x1da6, 0x1da7, 0x1da8, 0x1da9, 0x1daa, 0x1dab, + 0x1dac, 0x1dad, 0x1dae, 0x1daf, 0x1db0, 0x1db1, 0x1db2, 0x1db3, + 0x1db4, 0x1db5, 0x1db6, 0x1db7, 0x1db8, 0x1db9, 0x1dba, 0x1dbb, + 0x1dbc, 0x1dbd, 0x1dbe, 0x1dbf, 0x1dc0, 0x1dc1, 0x1dc2, 0x1dc3, + 0x1dc4, 0x1dc5, 0x1dc6, 0x1dc7, 0x1dc8, 0x1dc9, 0x1dca, 0x1dcb, + 0x1dcc, 0x1dcd, 0x1dce, 0x1dcf, 0x1dd0, 0x1dd1, 0x1dd2, 0x1dd3, + 0x1dd4, 0x1dd5, 0x1dd6, 0x1dd7, 0x1dd8, 0x1dd9, 0x1dda, 0x1ddb, + 0x1ddc, 0x1ddd, 0x1dde, 0x1ddf, 0x1de0, 0x1de1, 0x1de2, 0x1de3, + 0x1de4, 0x1de5, 0x1de6, 0x1de7, 0x1de8, 0x1de9, 0x1dea, 0x1deb, + 0x1dec, 0x1ded, 0x1dee, 0x1def, 0x1df0, 0x1df1, 0x1df2, 0x1df3, + 0x1df4, 0x1df5, 0x1df6, 0x1df7, 0x1df8, 0x1df9, 0x1dfa, 0x1dfb, + 0x1dfc, 0x1dfd, 0x1dfe, 0x1dff, 0x1e00, 0x1e00, 0x1e02, 0x1e02, + 0x1e04, 0x1e04, 0x1e06, 0x1e06, 0x1e08, 0x1e08, 0x1e0a, 0x1e0a, + 0x1e0c, 0x1e0c, 0x1e0e, 0x1e0e, 0x1e10, 0x1e10, 0x1e12, 0x1e12, + 0x1e14, 0x1e14, 0x1e16, 0x1e16, 0x1e18, 0x1e18, 0x1e1a, 0x1e1a, + 0x1e1c, 0x1e1c, 0x1e1e, 0x1e1e, 0x1e20, 0x1e20, 0x1e22, 0x1e22, + 0x1e24, 0x1e24, 0x1e26, 0x1e26, 0x1e28, 0x1e28, 0x1e2a, 0x1e2a, + 0x1e2c, 0x1e2c, 0x1e2e, 0x1e2e, 0x1e30, 0x1e30, 0x1e32, 0x1e32, + 0x1e34, 0x1e34, 0x1e36, 0x1e36, 0x1e38, 0x1e38, 0x1e3a, 0x1e3a, + 0x1e3c, 0x1e3c, 0x1e3e, 0x1e3e, 0x1e40, 0x1e40, 0x1e42, 0x1e42, + 0x1e44, 0x1e44, 0x1e46, 0x1e46, 0x1e48, 0x1e48, 0x1e4a, 0x1e4a, + 0x1e4c, 0x1e4c, 0x1e4e, 0x1e4e, 0x1e50, 0x1e50, 0x1e52, 0x1e52, + 0x1e54, 0x1e54, 0x1e56, 0x1e56, 0x1e58, 0x1e58, 0x1e5a, 0x1e5a, + 0x1e5c, 0x1e5c, 0x1e5e, 0x1e5e, 0x1e60, 0x1e60, 0x1e62, 0x1e62, + 0x1e64, 0x1e64, 0x1e66, 0x1e66, 0x1e68, 0x1e68, 0x1e6a, 0x1e6a, + 0x1e6c, 0x1e6c, 0x1e6e, 0x1e6e, 0x1e70, 0x1e70, 0x1e72, 0x1e72, + 0x1e74, 0x1e74, 0x1e76, 0x1e76, 0x1e78, 0x1e78, 0x1e7a, 0x1e7a, + 0x1e7c, 0x1e7c, 0x1e7e, 0x1e7e, 0x1e80, 0x1e80, 0x1e82, 0x1e82, + 0x1e84, 0x1e84, 0x1e86, 0x1e86, 0x1e88, 0x1e88, 0x1e8a, 0x1e8a, + 0x1e8c, 0x1e8c, 0x1e8e, 0x1e8e, 0x1e90, 0x1e90, 0x1e92, 0x1e92, + 0x1e94, 0x1e94, 0x1e96, 0x1e97, 0x1e98, 0x1e99, 0x1e9a, 0x1e9b, + 0x1e9c, 0x1e9d, 0x1e9e, 0x1e9f, 0x1ea0, 0x1ea0, 0x1ea2, 0x1ea2, + 0x1ea4, 0x1ea4, 0x1ea6, 0x1ea6, 0x1ea8, 0x1ea8, 0x1eaa, 0x1eaa, + 0x1eac, 0x1eac, 0x1eae, 0x1eae, 0x1eb0, 0x1eb0, 0x1eb2, 0x1eb2, + 0x1eb4, 0x1eb4, 0x1eb6, 0x1eb6, 0x1eb8, 0x1eb8, 0x1eba, 0x1eba, + 0x1ebc, 0x1ebc, 0x1ebe, 0x1ebe, 0x1ec0, 0x1ec0, 0x1ec2, 0x1ec2, + 0x1ec4, 0x1ec4, 0x1ec6, 0x1ec6, 0x1ec8, 0x1ec8, 0x1eca, 0x1eca, + 0x1ecc, 0x1ecc, 0x1ece, 0x1ece, 0x1ed0, 0x1ed0, 0x1ed2, 0x1ed2, + 0x1ed4, 0x1ed4, 0x1ed6, 0x1ed6, 0x1ed8, 0x1ed8, 0x1eda, 0x1eda, + 0x1edc, 0x1edc, 0x1ede, 0x1ede, 0x1ee0, 0x1ee0, 0x1ee2, 0x1ee2, + 0x1ee4, 0x1ee4, 0x1ee6, 0x1ee6, 0x1ee8, 0x1ee8, 0x1eea, 0x1eea, + 0x1eec, 0x1eec, 0x1eee, 0x1eee, 0x1ef0, 0x1ef0, 0x1ef2, 0x1ef2, + 0x1ef4, 0x1ef4, 0x1ef6, 0x1ef6, 0x1ef8, 0x1ef8, 0x1efa, 0x1efb, + 0x1efc, 0x1efd, 0x1efe, 0x1eff, 0x1f08, 0x1f09, 0x1f0a, 0x1f0b, + 0x1f0c, 0x1f0d, 0x1f0e, 0x1f0f, 0x1f08, 0x1f09, 0x1f0a, 0x1f0b, + 0x1f0c, 0x1f0d, 0x1f0e, 0x1f0f, 0x1f18, 0x1f19, 0x1f1a, 0x1f1b, + 0x1f1c, 0x1f1d, 0x1f16, 0x1f17, 0x1f18, 0x1f19, 0x1f1a, 0x1f1b, + 0x1f1c, 0x1f1d, 0x1f1e, 0x1f1f, 0x1f28, 0x1f29, 0x1f2a, 0x1f2b, + 0x1f2c, 0x1f2d, 0x1f2e, 0x1f2f, 0x1f28, 0x1f29, 0x1f2a, 0x1f2b, + 0x1f2c, 0x1f2d, 0x1f2e, 0x1f2f, 0x1f38, 0x1f39, 0x1f3a, 0x1f3b, + 0x1f3c, 0x1f3d, 0x1f3e, 0x1f3f, 0x1f38, 0x1f39, 0x1f3a, 0x1f3b, + 0x1f3c, 0x1f3d, 0x1f3e, 0x1f3f, 0x1f48, 0x1f49, 0x1f4a, 0x1f4b, + 0x1f4c, 0x1f4d, 0x1f46, 0x1f47, 0x1f48, 0x1f49, 0x1f4a, 0x1f4b, + 0x1f4c, 0x1f4d, 0x1f4e, 0x1f4f, 0x1f50, 0x1f59, 0x1f52, 0x1f5b, + 0x1f54, 0x1f5d, 0x1f56, 0x1f5f, 0x1f58, 0x1f59, 0x1f5a, 0x1f5b, + 0x1f5c, 0x1f5d, 0x1f5e, 0x1f5f, 0x1f68, 0x1f69, 0x1f6a, 0x1f6b, + 0x1f6c, 0x1f6d, 0x1f6e, 0x1f6f, 0x1f68, 0x1f69, 0x1f6a, 0x1f6b, + 0x1f6c, 0x1f6d, 0x1f6e, 0x1f6f, 0x1fba, 0x1fbb, 0x1fc8, 0x1fc9, + 0x1fca, 0x1fcb, 0x1fda, 0x1fdb, 0x1ff8, 0x1ff9, 0x1fea, 0x1feb, + 0x1ffa, 0x1ffb, 0x1f7e, 0x1f7f, 0x1f88, 0x1f89, 0x1f8a, 0x1f8b, + 0x1f8c, 0x1f8d, 0x1f8e, 0x1f8f, 0x1f88, 0x1f89, 0x1f8a, 0x1f8b, + 0x1f8c, 0x1f8d, 0x1f8e, 0x1f8f, 0x1f98, 0x1f99, 0x1f9a, 0x1f9b, + 0x1f9c, 0x1f9d, 0x1f9e, 0x1f9f, 0x1f98, 0x1f99, 0x1f9a, 0x1f9b, + 0x1f9c, 0x1f9d, 0x1f9e, 0x1f9f, 0x1fa8, 0x1fa9, 0x1faa, 0x1fab, + 0x1fac, 0x1fad, 0x1fae, 0x1faf, 0x1fa8, 0x1fa9, 0x1faa, 0x1fab, + 0x1fac, 0x1fad, 0x1fae, 0x1faf, 0x1fb8, 0x1fb9, 0x1fb2, 0x1fbc, + 0x1fb4, 0x1fb5, 0x1fb6, 0x1fb7, 0x1fb8, 0x1fb9, 0x1fba, 0x1fbb, + 0x1fbc, 0x1fbd, 0x1fbe, 0x1fbf, 0x1fc0, 0x1fc1, 0x1fc2, 0x1fc3, + 0x1fc4, 0x1fc5, 0x1fc6, 0x1fc7, 0x1fc8, 0x1fc9, 0x1fca, 0x1fcb, + 0x1fc3, 0x1fcd, 0x1fce, 0x1fcf, 0x1fd8, 0x1fd9, 0x1fd2, 0x1fd3, + 0x1fd4, 0x1fd5, 0x1fd6, 0x1fd7, 0x1fd8, 0x1fd9, 0x1fda, 0x1fdb, + 0x1fdc, 0x1fdd, 0x1fde, 0x1fdf, 0x1fe8, 0x1fe9, 0x1fe2, 0x1fe3, + 0x1fe4, 0x1fec, 0x1fe6, 0x1fe7, 0x1fe8, 0x1fe9, 0x1fea, 0x1feb, + 0x1fec, 0x1fed, 0x1fee, 0x1fef, 0x1ff0, 0x1ff1, 0x1ff2, 0x1ff3, + 0x1ff4, 0x1ff5, 0x1ff6, 0x1ff7, 0x1ff8, 0x1ff9, 0x1ffa, 0x1ffb, + 0x1ff3, 0x1ffd, 0x1ffe, 0x1fff, 0x2000, 0x2001, 0x2002, 0x2003, + 0x2004, 0x2005, 0x2006, 0x2007, 0x2008, 0x2009, 0x200a, 0x200b, + 0x200c, 0x200d, 0x200e, 0x200f, 0x2010, 0x2011, 0x2012, 0x2013, + 0x2014, 0x2015, 0x2016, 0x2017, 0x2018, 0x2019, 0x201a, 0x201b, + 0x201c, 0x201d, 0x201e, 0x201f, 0x2020, 0x2021, 0x2022, 0x2023, + 0x2024, 0x2025, 0x2026, 0x2027, 0x2028, 0x2029, 0x202a, 0x202b, + 0x202c, 0x202d, 0x202e, 0x202f, 0x2030, 0x2031, 0x2032, 0x2033, + 0x2034, 0x2035, 0x2036, 0x2037, 0x2038, 0x2039, 0x203a, 0x203b, + 0x203c, 0x203d, 0x203e, 0x203f, 0x2040, 0x2041, 0x2042, 0x2043, + 0x2044, 0x2045, 0x2046, 0x2047, 0x2048, 0x2049, 0x204a, 0x204b, + 0x204c, 0x204d, 0x204e, 0x204f, 0x2050, 0x2051, 0x2052, 0x2053, + 0x2054, 0x2055, 0x2056, 0x2057, 0x2058, 0x2059, 0x205a, 0x205b, + 0x205c, 0x205d, 0x205e, 0x205f, 0x2060, 0x2061, 0x2062, 0x2063, + 0x2064, 0x2065, 0x2066, 0x2067, 0x2068, 0x2069, 0x206a, 0x206b, + 0x206c, 0x206d, 0x206e, 0x206f, 0x2070, 0x2071, 0x2072, 0x2073, + 0x2074, 0x2075, 0x2076, 0x2077, 0x2078, 0x2079, 0x207a, 0x207b, + 0x207c, 0x207d, 0x207e, 0x207f, 0x2080, 0x2081, 0x2082, 0x2083, + 0x2084, 0x2085, 0x2086, 0x2087, 0x2088, 0x2089, 0x208a, 0x208b, + 0x208c, 0x208d, 0x208e, 0x208f, 0x2090, 0x2091, 0x2092, 0x2093, + 0x2094, 0x2095, 0x2096, 0x2097, 0x2098, 0x2099, 0x209a, 0x209b, + 0x209c, 0x209d, 0x209e, 0x209f, 0x20a0, 0x20a1, 0x20a2, 0x20a3, + 0x20a4, 0x20a5, 0x20a6, 0x20a7, 0x20a8, 0x20a9, 0x20aa, 0x20ab, + 0x20ac, 0x20ad, 0x20ae, 0x20af, 0x20b0, 0x20b1, 0x20b2, 0x20b3, + 0x20b4, 0x20b5, 0x20b6, 0x20b7, 0x20b8, 0x20b9, 0x20ba, 0x20bb, + 0x20bc, 0x20bd, 0x20be, 0x20bf, 0x20c0, 0x20c1, 0x20c2, 0x20c3, + 0x20c4, 0x20c5, 0x20c6, 0x20c7, 0x20c8, 0x20c9, 0x20ca, 0x20cb, + 0x20cc, 0x20cd, 0x20ce, 0x20cf, 0x20d0, 0x20d1, 0x20d2, 0x20d3, + 0x20d4, 0x20d5, 0x20d6, 0x20d7, 0x20d8, 0x20d9, 0x20da, 0x20db, + 0x20dc, 0x20dd, 0x20de, 0x20df, 0x20e0, 0x20e1, 0x20e2, 0x20e3, + 0x20e4, 0x20e5, 0x20e6, 0x20e7, 0x20e8, 0x20e9, 0x20ea, 0x20eb, + 0x20ec, 0x20ed, 0x20ee, 0x20ef, 0x20f0, 0x20f1, 0x20f2, 0x20f3, + 0x20f4, 0x20f5, 0x20f6, 0x20f7, 0x20f8, 0x20f9, 0x20fa, 0x20fb, + 0x20fc, 0x20fd, 0x20fe, 0x20ff, 0x2100, 0x2101, 0x2102, 0x2103, + 0x2104, 0x2105, 0x2106, 0x2107, 0x2108, 0x2109, 0x210a, 0x210b, + 0x210c, 0x210d, 0x210e, 0x210f, 0x2110, 0x2111, 0x2112, 0x2113, + 0x2114, 0x2115, 0x2116, 0x2117, 0x2118, 0x2119, 0x211a, 0x211b, + 0x211c, 0x211d, 0x211e, 0x211f, 0x2120, 0x2121, 0x2122, 0x2123, + 0x2124, 0x2125, 0x2126, 0x2127, 0x2128, 0x2129, 0x212a, 0x212b, + 0x212c, 0x212d, 0x212e, 0x212f, 0x2130, 0x2131, 0x2132, 0x2133, + 0x2134, 0x2135, 0x2136, 0x2137, 0x2138, 0x2139, 0x213a, 0x213b, + 0x213c, 0x213d, 0x213e, 0x213f, 0x2140, 0x2141, 0x2142, 0x2143, + 0x2144, 0x2145, 0x2146, 0x2147, 0x2148, 0x2149, 0x214a, 0x214b, + 0x214c, 0x214d, 0x2132, 0x214f, 0x2150, 0x2151, 0x2152, 0x2153, + 0x2154, 0x2155, 0x2156, 0x2157, 0x2158, 0x2159, 0x215a, 0x215b, + 0x215c, 0x215d, 0x215e, 0x215f, 0x2160, 0x2161, 0x2162, 0x2163, + 0x2164, 0x2165, 0x2166, 0x2167, 0x2168, 0x2169, 0x216a, 0x216b, + 0x216c, 0x216d, 0x216e, 0x216f, 0x2160, 0x2161, 0x2162, 0x2163, + 0x2164, 0x2165, 0x2166, 0x2167, 0x2168, 0x2169, 0x216a, 0x216b, + 0x216c, 0x216d, 0x216e, 0x216f, 0x2180, 0x2181, 0x2182, 0x2183, + 0x2183, 0xffff, 0x034b, 0x24b6, 0x24b7, 0x24b8, 0x24b9, 0x24ba, + 0x24bb, 0x24bc, 0x24bd, 0x24be, 0x24bf, 0x24c0, 0x24c1, 0x24c2, + 0x24c3, 0x24c4, 0x24c5, 0x24c6, 0x24c7, 0x24c8, 0x24c9, 0x24ca, + 0x24cb, 0x24cc, 0x24cd, 0x24ce, 0x24cf, 0xffff, 0x0746, 0x2c00, + 0x2c01, 0x2c02, 0x2c03, 0x2c04, 0x2c05, 0x2c06, 0x2c07, 0x2c08, + 0x2c09, 0x2c0a, 0x2c0b, 0x2c0c, 0x2c0d, 0x2c0e, 0x2c0f, 0x2c10, + 0x2c11, 0x2c12, 0x2c13, 0x2c14, 0x2c15, 0x2c16, 0x2c17, 0x2c18, + 0x2c19, 0x2c1a, 0x2c1b, 0x2c1c, 0x2c1d, 0x2c1e, 0x2c1f, 0x2c20, + 0x2c21, 0x2c22, 0x2c23, 0x2c24, 0x2c25, 0x2c26, 0x2c27, 0x2c28, + 0x2c29, 0x2c2a, 0x2c2b, 0x2c2c, 0x2c2d, 0x2c2e, 0x2c5f, 0x2c60, + 0x2c60, 0x2c62, 0x2c63, 0x2c64, 0x2c65, 0x2c66, 0x2c67, 0x2c67, + 0x2c69, 0x2c69, 0x2c6b, 0x2c6b, 0x2c6d, 0x2c6e, 0x2c6f, 0x2c70, + 0x2c71, 0x2c72, 0x2c73, 0x2c74, 0x2c75, 0x2c75, 0x2c77, 0x2c78, + 0x2c79, 0x2c7a, 0x2c7b, 0x2c7c, 0x2c7d, 0x2c7e, 0x2c7f, 0x2c80, + 0x2c80, 0x2c82, 0x2c82, 0x2c84, 0x2c84, 0x2c86, 0x2c86, 0x2c88, + 0x2c88, 0x2c8a, 0x2c8a, 0x2c8c, 0x2c8c, 0x2c8e, 0x2c8e, 0x2c90, + 0x2c90, 0x2c92, 0x2c92, 0x2c94, 0x2c94, 0x2c96, 0x2c96, 0x2c98, + 0x2c98, 0x2c9a, 0x2c9a, 0x2c9c, 0x2c9c, 0x2c9e, 0x2c9e, 0x2ca0, + 0x2ca0, 0x2ca2, 0x2ca2, 0x2ca4, 0x2ca4, 0x2ca6, 0x2ca6, 0x2ca8, + 0x2ca8, 0x2caa, 0x2caa, 0x2cac, 0x2cac, 0x2cae, 0x2cae, 0x2cb0, + 0x2cb0, 0x2cb2, 0x2cb2, 0x2cb4, 0x2cb4, 0x2cb6, 0x2cb6, 0x2cb8, + 0x2cb8, 0x2cba, 0x2cba, 0x2cbc, 0x2cbc, 0x2cbe, 0x2cbe, 0x2cc0, + 0x2cc0, 0x2cc2, 0x2cc2, 0x2cc4, 0x2cc4, 0x2cc6, 0x2cc6, 0x2cc8, + 0x2cc8, 0x2cca, 0x2cca, 0x2ccc, 0x2ccc, 0x2cce, 0x2cce, 0x2cd0, + 0x2cd0, 0x2cd2, 0x2cd2, 0x2cd4, 0x2cd4, 0x2cd6, 0x2cd6, 0x2cd8, + 0x2cd8, 0x2cda, 0x2cda, 0x2cdc, 0x2cdc, 0x2cde, 0x2cde, 0x2ce0, + 0x2ce0, 0x2ce2, 0x2ce2, 0x2ce4, 0x2ce5, 0x2ce6, 0x2ce7, 0x2ce8, + 0x2ce9, 0x2cea, 0x2ceb, 0x2cec, 0x2ced, 0x2cee, 0x2cef, 0x2cf0, + 0x2cf1, 0x2cf2, 0x2cf3, 0x2cf4, 0x2cf5, 0x2cf6, 0x2cf7, 0x2cf8, + 0x2cf9, 0x2cfa, 0x2cfb, 0x2cfc, 0x2cfd, 0x2cfe, 0x2cff, 0x10a0, + 0x10a1, 0x10a2, 0x10a3, 0x10a4, 0x10a5, 0x10a6, 0x10a7, 0x10a8, + 0x10a9, 0x10aa, 0x10ab, 0x10ac, 0x10ad, 0x10ae, 0x10af, 0x10b0, + 0x10b1, 0x10b2, 0x10b3, 0x10b4, 0x10b5, 0x10b6, 0x10b7, 0x10b8, + 0x10b9, 0x10ba, 0x10bb, 0x10bc, 0x10bd, 0x10be, 0x10bf, 0x10c0, + 0x10c1, 0x10c2, 0x10c3, 0x10c4, 0x10c5, 0xffff, 0xd21b, 0xff21, + 0xff22, 0xff23, 0xff24, 0xff25, 0xff26, 0xff27, 0xff28, 0xff29, + 0xff2a, 0xff2b, 0xff2c, 0xff2d, 0xff2e, 0xff2f, 0xff30, 0xff31, + 0xff32, 0xff33, 0xff34, 0xff35, 0xff36, 0xff37, 0xff38, 0xff39, + 0xff3a, 0xff5b, 0xff5c, 0xff5d, 0xff5e, 0xff5f, 0xff60, 0xff61, + 0xff62, 0xff63, 0xff64, 0xff65, 0xff66, 0xff67, 0xff68, 0xff69, + 0xff6a, 0xff6b, 0xff6c, 0xff6d, 0xff6e, 0xff6f, 0xff70, 0xff71, + 0xff72, 0xff73, 0xff74, 0xff75, 0xff76, 0xff77, 0xff78, 0xff79, + 0xff7a, 0xff7b, 0xff7c, 0xff7d, 0xff7e, 0xff7f, 0xff80, 0xff81, + 0xff82, 0xff83, 0xff84, 0xff85, 0xff86, 0xff87, 0xff88, 0xff89, + 0xff8a, 0xff8b, 0xff8c, 0xff8d, 0xff8e, 0xff8f, 0xff90, 0xff91, + 0xff92, 0xff93, 0xff94, 0xff95, 0xff96, 0xff97, 0xff98, 0xff99, + 0xff9a, 0xff9b, 0xff9c, 0xff9d, 0xff9e, 0xff9f, 0xffa0, 0xffa1, + 0xffa2, 0xffa3, 0xffa4, 0xffa5, 0xffa6, 0xffa7, 0xffa8, 0xffa9, + 0xffaa, 0xffab, 0xffac, 0xffad, 0xffae, 0xffaf, 0xffb0, 0xffb1, + 0xffb2, 0xffb3, 0xffb4, 0xffb5, 0xffb6, 0xffb7, 0xffb8, 0xffb9, + 0xffba, 0xffbb, 0xffbc, 0xffbd, 0xffbe, 0xffbf, 0xffc0, 0xffc1, + 0xffc2, 0xffc3, 0xffc4, 0xffc5, 0xffc6, 0xffc7, 0xffc8, 0xffc9, + 0xffca, 0xffcb, 0xffcc, 0xffcd, 0xffce, 0xffcf, 0xffd0, 0xffd1, + 0xffd2, 0xffd3, 0xffd4, 0xffd5, 0xffd6, 0xffd7, 0xffd8, 0xffd9, + 0xffda, 0xffdb, 0xffdc, 0xffdd, 0xffde, 0xffdf, 0xffe0, 0xffe1, + 0xffe2, 0xffe3, 0xffe4, 0xffe5, 0xffe6, 0xffe7, 0xffe8, 0xffe9, + 0xffea, 0xffeb, 0xffec, 0xffed, 0xffee, 0xffef, 0xfff0, 0xfff1, + 0xfff2, 0xfff3, 0xfff4, 0xfff5, 0xfff6, 0xfff7, 0xfff8, 0xfff9, + 0xfffa, 0xfffb, 0xfffc, 0xfffd, 0xfffe, 0xffff, +}; + +/* + * Allow full-width illegal characters : + * "MS windows 7" supports full-width-invalid-name-characters. + * So we should check half-width-invalid-name-characters(ASCII) only + * for compatibility. + * + * " * / : < > ? \ | + */ +static unsigned short bad_uni_chars[] = { + 0x0022, 0x002A, 0x002F, 0x003A, + 0x003C, 0x003E, 0x003F, 0x005C, 0x007C, + 0 +}; + +static int exfat_convert_char_to_ucs2(struct nls_table *nls, + const unsigned char *ch, int ch_len, unsigned short *ucs2, + int *lossy) +{ + int len; + + *ucs2 = 0x0; + + if (ch[0] < 0x80) { + *ucs2 = ch[0]; + return 1; + } + + len = nls->char2uni(ch, ch_len, ucs2); + if (len < 0) { + /* conversion failed */ + if (lossy != NULL) + *lossy |= NLS_NAME_LOSSY; + *ucs2 = '_'; + return 1; + } + return len; +} + +static int exfat_convert_ucs2_to_char(struct nls_table *nls, + unsigned short ucs2, unsigned char *ch, int *lossy) +{ + int len; + + ch[0] = 0x0; + + if (ucs2 < 0x0080) { + ch[0] = ucs2; + return 1; + } + + len = nls->uni2char(ucs2, ch, MAX_CHARSET_SIZE); + if (len < 0) { + /* conversion failed */ + if (lossy != NULL) + *lossy |= NLS_NAME_LOSSY; + ch[0] = '_'; + return 1; + } + return len; +} + +unsigned short exfat_toupper(struct super_block *sb, unsigned short a) +{ + struct exfat_sb_info *sbi = EXFAT_SB(sb); + + return sbi->vol_utbl[a] ? sbi->vol_utbl[a] : a; +} + +static unsigned short *exfat_wstrchr(unsigned short *str, unsigned short wchar) +{ + while (*str) { + if (*(str++) == wchar) + return str; + } + return NULL; +} + +int exfat_uniname_ncmp(struct super_block *sb, unsigned short *a, + unsigned short *b, unsigned int len) +{ + int i; + + for (i = 0; i < len; i++, a++, b++) + if (exfat_toupper(sb, *a) != exfat_toupper(sb, *b)) + return 1; + return 0; +} + +static int exfat_utf16_to_utf8(struct super_block *sb, + struct exfat_uni_name *p_uniname, unsigned char *p_cstring, + int buflen) +{ + int len; + const unsigned short *uniname = p_uniname->name; + + /* always len >= 0 */ + len = utf16s_to_utf8s(uniname, MAX_NAME_LENGTH, UTF16_HOST_ENDIAN, + p_cstring, buflen); + p_cstring[len] = '\0'; + return len; +} + +static int exfat_utf8_to_utf16(struct super_block *sb, + const unsigned char *p_cstring, const int len, + struct exfat_uni_name *p_uniname, int *p_lossy) +{ + int i, unilen, lossy = NLS_NAME_NO_LOSSY; + __le16 upname[MAX_NAME_LENGTH + 1]; + unsigned short *uniname = p_uniname->name; + + WARN_ON(!len); + + unilen = utf8s_to_utf16s(p_cstring, len, UTF16_HOST_ENDIAN, + (wchar_t *)uniname, MAX_NAME_LENGTH + 2); + if (unilen < 0) { + exfat_err(sb, "failed to %s (err : %d) nls len : %d", + __func__, unilen, len); + return unilen; + } + + if (unilen > MAX_NAME_LENGTH) { + exfat_debug(sb, "failed to %s (estr:ENAMETOOLONG) nls len : %d, unilen : %d > %d", + __func__, len, unilen, MAX_NAME_LENGTH); + return -ENAMETOOLONG; + } + + for (i = 0; i < unilen; i++) { + if (*uniname < 0x0020 || + exfat_wstrchr(bad_uni_chars, *uniname)) + lossy |= NLS_NAME_LOSSY; + + upname[i] = cpu_to_le16(exfat_toupper(sb, *uniname)); + uniname++; + } + + *uniname = '\0'; + p_uniname->name_len = unilen; + p_uniname->name_hash = exfat_calc_chksum16(upname, unilen << 1, 0, + CS_DEFAULT); + + if (p_lossy) + *p_lossy = lossy; + return unilen; +} + +#define SURROGATE_MASK 0xfffff800 +#define SURROGATE_PAIR 0x0000d800 +#define SURROGATE_LOW 0x00000400 + +static int __exfat_utf16_to_nls(struct super_block *sb, + struct exfat_uni_name *p_uniname, unsigned char *p_cstring, + int buflen) +{ + int i, j, len, out_len = 0; + unsigned char buf[MAX_CHARSET_SIZE]; + const unsigned short *uniname = p_uniname->name; + struct nls_table *nls = EXFAT_SB(sb)->nls_io; + + i = 0; + while (i < MAX_NAME_LENGTH && out_len < (buflen - 1)) { + if (*uniname == '\0') + break; + if ((*uniname & SURROGATE_MASK) != SURROGATE_PAIR) { + len = exfat_convert_ucs2_to_char(nls, *uniname, buf, + NULL); + } else { + /* Process UTF-16 surrogate pair as one character */ + if (!(*uniname & SURROGATE_LOW) && + i+1 < MAX_NAME_LENGTH && + (*(uniname+1) & SURROGATE_MASK) == SURROGATE_PAIR && + (*(uniname+1) & SURROGATE_LOW)) { + uniname++; + i++; + } + + /* + * UTF-16 surrogate pair encodes code points above + * U+FFFF. Code points above U+FFFF are not supported + * by kernel NLS framework therefore use replacement + * character + */ + len = 1; + buf[0] = '_'; + } + + if (out_len + len >= buflen) + len = buflen - 1 - out_len; + out_len += len; + + if (len > 1) { + for (j = 0; j < len; j++) + *p_cstring++ = buf[j]; + } else { /* len == 1 */ + *p_cstring++ = *buf; + } + + uniname++; + i++; + } + + *p_cstring = '\0'; + return out_len; +} + +static int exfat_nls_to_ucs2(struct super_block *sb, + const unsigned char *p_cstring, const int len, + struct exfat_uni_name *p_uniname, int *p_lossy) +{ + int i = 0, unilen = 0, lossy = NLS_NAME_NO_LOSSY; + __le16 upname[MAX_NAME_LENGTH + 1]; + unsigned short *uniname = p_uniname->name; + struct nls_table *nls = EXFAT_SB(sb)->nls_io; + + WARN_ON(!len); + + while (unilen < MAX_NAME_LENGTH && i < len) { + i += exfat_convert_char_to_ucs2(nls, p_cstring + i, len - i, + uniname, &lossy); + + if (*uniname < 0x0020 || + exfat_wstrchr(bad_uni_chars, *uniname)) + lossy |= NLS_NAME_LOSSY; + + upname[unilen] = cpu_to_le16(exfat_toupper(sb, *uniname)); + uniname++; + unilen++; + } + + *uniname = '\0'; + p_uniname->name_len = unilen; + p_uniname->name_hash = exfat_calc_chksum16(upname, unilen << 1, 0, + CS_DEFAULT); + + if (p_lossy) + *p_lossy = lossy; + return unilen; +} + +int exfat_utf16_to_nls(struct super_block *sb, struct exfat_uni_name *uniname, + unsigned char *p_cstring, int buflen) +{ + if (EXFAT_SB(sb)->options.utf8) + return exfat_utf16_to_utf8(sb, uniname, p_cstring, + buflen); + return __exfat_utf16_to_nls(sb, uniname, p_cstring, buflen); +} + +int exfat_nls_to_utf16(struct super_block *sb, const unsigned char *p_cstring, + const int len, struct exfat_uni_name *uniname, int *p_lossy) +{ + if (EXFAT_SB(sb)->options.utf8) + return exfat_utf8_to_utf16(sb, p_cstring, len, + uniname, p_lossy); + return exfat_nls_to_ucs2(sb, p_cstring, len, uniname, p_lossy); +} + +static int exfat_load_upcase_table(struct super_block *sb, + sector_t sector, unsigned long long num_sectors, + unsigned int utbl_checksum) +{ + struct exfat_sb_info *sbi = EXFAT_SB(sb); + unsigned int sect_size = sb->s_blocksize; + unsigned int i, index = 0; + u32 chksum = 0; + unsigned char skip = false; + unsigned short *upcase_table; + + upcase_table = kvcalloc(UTBL_COUNT, sizeof(unsigned short), GFP_KERNEL); + if (!upcase_table) + return -ENOMEM; + + sbi->vol_utbl = upcase_table; + num_sectors += sector; + + while (sector < num_sectors) { + struct buffer_head *bh; + + bh = sb_bread(sb, sector); + if (!bh) { + exfat_err(sb, "failed to read sector(0x%llx)", + (unsigned long long)sector); + return -EIO; + } + sector++; + for (i = 0; i < sect_size && index <= 0xFFFF; i += 2) { + unsigned short uni = get_unaligned_le16(bh->b_data + i); + + if (skip) { + index += uni; + skip = false; + } else if (uni == index) { + index++; + } else if (uni == 0xFFFF) { + skip = true; + } else { /* uni != index , uni != 0xFFFF */ + upcase_table[index] = uni; + index++; + } + } + chksum = exfat_calc_chksum32(bh->b_data, i, chksum, CS_DEFAULT); + brelse(bh); + } + + if (index >= 0xFFFF && utbl_checksum == chksum) + return 0; + + exfat_err(sb, "failed to load upcase table (idx : 0x%08x, chksum : 0x%08x, utbl_chksum : 0x%08x)", + index, chksum, utbl_checksum); + return -EINVAL; +} + +static int exfat_load_default_upcase_table(struct super_block *sb) +{ + int i; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + unsigned char skip = false; + unsigned short uni = 0, *upcase_table; + unsigned int index = 0; + + upcase_table = kvcalloc(UTBL_COUNT, sizeof(unsigned short), GFP_KERNEL); + if (!upcase_table) + return -ENOMEM; + + sbi->vol_utbl = upcase_table; + + for (i = 0; index <= 0xFFFF && i < EXFAT_NUM_UPCASE; i++) { + uni = uni_def_upcase[i]; + if (skip) { + index += uni; + skip = false; + } else if (uni == index) { + index++; + } else if (uni == 0xFFFF) { + skip = true; + } else { + upcase_table[index] = uni; + index++; + } + } + + if (index >= 0xFFFF) + return 0; + + /* FATAL error: default upcase table has error */ + return -EIO; +} + +int exfat_create_upcase_table(struct super_block *sb) +{ + int i, ret; + unsigned int tbl_clu, type; + sector_t sector; + unsigned long long tbl_size, num_sectors; + unsigned char blksize_bits = sb->s_blocksize_bits; + struct exfat_chain clu; + struct exfat_dentry *ep; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct buffer_head *bh; + + clu.dir = sbi->root_dir; + clu.flags = ALLOC_FAT_CHAIN; + + while (clu.dir != EXFAT_EOF_CLUSTER) { + for (i = 0; i < sbi->dentries_per_clu; i++) { + ep = exfat_get_dentry(sb, &clu, i, &bh); + if (!ep) + return -EIO; + + type = exfat_get_entry_type(ep); + if (type == TYPE_UNUSED) { + brelse(bh); + break; + } + + if (type != TYPE_UPCASE) { + brelse(bh); + continue; + } + + tbl_clu = le32_to_cpu(ep->dentry.upcase.start_clu); + tbl_size = le64_to_cpu(ep->dentry.upcase.size); + if (tbl_size) { + sector = exfat_cluster_to_sector(sbi, tbl_clu); + num_sectors = ((tbl_size - 1) >> blksize_bits) + 1; + ret = exfat_load_upcase_table(sb, sector, num_sectors, + le32_to_cpu(ep->dentry.upcase.checksum)); + } else { + exfat_fs_error(sb, + "bad upcase table size (0 bytes). Please run fsck"); + ret = -EINVAL; + } + brelse(bh); + + if (ret && ret != -EIO) { + /* free memory from exfat_load_upcase_table call */ + exfat_free_upcase_table(sbi); + goto load_default; + } + + /* load successfully */ + return ret; + } + + if (exfat_get_next_cluster(sb, &clu.dir)) + return -EIO; + } + + exfat_fs_error(sb, "no upcase table entry. Please run fsck"); + +load_default: + /* load default upcase table */ + return exfat_load_default_upcase_table(sb); +} + +void exfat_free_upcase_table(struct exfat_sb_info *sbi) +{ + kvfree(sbi->vol_utbl); + sbi->vol_utbl = NULL; +} diff --git a/fs/exfat/super.c b/fs/exfat/super.c new file mode 100644 index 000000000..a9ea36ba2 --- /dev/null +++ b/fs/exfat/super.c @@ -0,0 +1,943 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2012-2013 Samsung Electronics Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "exfat_raw.h" +#include "exfat_fs.h" + +static char exfat_default_iocharset[] = CONFIG_EXFAT_DEFAULT_IOCHARSET; +static struct kmem_cache *exfat_inode_cachep; + +static void exfat_free_iocharset(struct exfat_sb_info *sbi) +{ + if (sbi->options.iocharset != exfat_default_iocharset) + kfree(sbi->options.iocharset); +} + +static void exfat_set_iocharset(struct exfat_mount_options *opts, + char *iocharset) +{ + opts->iocharset = iocharset; + if (!strcmp(opts->iocharset, "utf8")) + opts->utf8 = 1; + else + opts->utf8 = 0; +} + +static void exfat_put_super(struct super_block *sb) +{ + struct exfat_sb_info *sbi = EXFAT_SB(sb); + + mutex_lock(&sbi->s_lock); + exfat_clear_volume_dirty(sb); + exfat_free_bitmap(sbi); + brelse(sbi->boot_bh); + mutex_unlock(&sbi->s_lock); +} + +static int exfat_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + unsigned long long id = huge_encode_dev(sb->s_bdev->bd_dev); + + buf->f_type = sb->s_magic; + buf->f_bsize = sbi->cluster_size; + buf->f_blocks = sbi->num_clusters - 2; /* clu 0 & 1 */ + buf->f_bfree = buf->f_blocks - sbi->used_clusters; + buf->f_bavail = buf->f_bfree; + buf->f_fsid = u64_to_fsid(id); + /* Unicode utf16 255 characters */ + buf->f_namelen = EXFAT_MAX_FILE_LEN * NLS_MAX_CHARSET_SIZE; + return 0; +} + +static int exfat_set_vol_flags(struct super_block *sb, unsigned short new_flags) +{ + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct boot_sector *p_boot = (struct boot_sector *)sbi->boot_bh->b_data; + + /* retain persistent-flags */ + new_flags |= sbi->vol_flags_persistent; + + /* flags are not changed */ + if (sbi->vol_flags == new_flags) + return 0; + + sbi->vol_flags = new_flags; + + /* skip updating volume dirty flag, + * if this volume has been mounted with read-only + */ + if (sb_rdonly(sb)) + return 0; + + p_boot->vol_flags = cpu_to_le16(new_flags); + + set_buffer_uptodate(sbi->boot_bh); + mark_buffer_dirty(sbi->boot_bh); + + __sync_dirty_buffer(sbi->boot_bh, REQ_SYNC | REQ_FUA | REQ_PREFLUSH); + + return 0; +} + +int exfat_set_volume_dirty(struct super_block *sb) +{ + struct exfat_sb_info *sbi = EXFAT_SB(sb); + + return exfat_set_vol_flags(sb, sbi->vol_flags | VOLUME_DIRTY); +} + +int exfat_clear_volume_dirty(struct super_block *sb) +{ + struct exfat_sb_info *sbi = EXFAT_SB(sb); + + return exfat_set_vol_flags(sb, sbi->vol_flags & ~VOLUME_DIRTY); +} + +static int exfat_show_options(struct seq_file *m, struct dentry *root) +{ + struct super_block *sb = root->d_sb; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct exfat_mount_options *opts = &sbi->options; + + /* Show partition info */ + if (!uid_eq(opts->fs_uid, GLOBAL_ROOT_UID)) + seq_printf(m, ",uid=%u", + from_kuid_munged(&init_user_ns, opts->fs_uid)); + if (!gid_eq(opts->fs_gid, GLOBAL_ROOT_GID)) + seq_printf(m, ",gid=%u", + from_kgid_munged(&init_user_ns, opts->fs_gid)); + seq_printf(m, ",fmask=%04o,dmask=%04o", opts->fs_fmask, opts->fs_dmask); + if (opts->allow_utime) + seq_printf(m, ",allow_utime=%04o", opts->allow_utime); + if (opts->utf8) + seq_puts(m, ",iocharset=utf8"); + else if (sbi->nls_io) + seq_printf(m, ",iocharset=%s", sbi->nls_io->charset); + if (opts->errors == EXFAT_ERRORS_CONT) + seq_puts(m, ",errors=continue"); + else if (opts->errors == EXFAT_ERRORS_PANIC) + seq_puts(m, ",errors=panic"); + else + seq_puts(m, ",errors=remount-ro"); + if (opts->discard) + seq_puts(m, ",discard"); + if (opts->keep_last_dots) + seq_puts(m, ",keep_last_dots"); + if (opts->sys_tz) + seq_puts(m, ",sys_tz"); + else if (opts->time_offset) + seq_printf(m, ",time_offset=%d", opts->time_offset); + if (opts->zero_size_dir) + seq_puts(m, ",zero_size_dir"); + return 0; +} + +int exfat_force_shutdown(struct super_block *sb, u32 flags) +{ + int ret; + struct exfat_sb_info *sbi = sb->s_fs_info; + struct exfat_mount_options *opts = &sbi->options; + + if (exfat_forced_shutdown(sb)) + return 0; + + switch (flags) { + case EXFAT_GOING_DOWN_DEFAULT: + case EXFAT_GOING_DOWN_FULLSYNC: + ret = bdev_freeze(sb->s_bdev); + if (ret) + return ret; + bdev_thaw(sb->s_bdev); + set_bit(EXFAT_FLAGS_SHUTDOWN, &sbi->s_exfat_flags); + break; + case EXFAT_GOING_DOWN_NOSYNC: + set_bit(EXFAT_FLAGS_SHUTDOWN, &sbi->s_exfat_flags); + break; + default: + return -EINVAL; + } + + if (opts->discard) + opts->discard = 0; + return 0; +} + +static void exfat_shutdown(struct super_block *sb) +{ + exfat_force_shutdown(sb, EXFAT_GOING_DOWN_NOSYNC); +} + +static struct inode *exfat_alloc_inode(struct super_block *sb) +{ + struct exfat_inode_info *ei; + + ei = alloc_inode_sb(sb, exfat_inode_cachep, GFP_NOFS); + if (!ei) + return NULL; + + return &ei->vfs_inode; +} + +static void exfat_free_inode(struct inode *inode) +{ + kmem_cache_free(exfat_inode_cachep, EXFAT_I(inode)); +} + +static const struct super_operations exfat_sops = { + .alloc_inode = exfat_alloc_inode, + .free_inode = exfat_free_inode, + .write_inode = exfat_write_inode, + .evict_inode = exfat_evict_inode, + .put_super = exfat_put_super, + .statfs = exfat_statfs, + .show_options = exfat_show_options, + .shutdown = exfat_shutdown, +}; + +enum { + Opt_uid, + Opt_gid, + Opt_umask, + Opt_dmask, + Opt_fmask, + Opt_allow_utime, + Opt_charset, + Opt_errors, + Opt_discard, + Opt_keep_last_dots, + Opt_sys_tz, + Opt_time_offset, + Opt_zero_size_dir, + + /* Deprecated options */ + Opt_utf8, + Opt_debug, + Opt_namecase, + Opt_codepage, +}; + +static const struct constant_table exfat_param_enums[] = { + { "continue", EXFAT_ERRORS_CONT }, + { "panic", EXFAT_ERRORS_PANIC }, + { "remount-ro", EXFAT_ERRORS_RO }, + {} +}; + +static const struct fs_parameter_spec exfat_parameters[] = { + fsparam_uid("uid", Opt_uid), + fsparam_gid("gid", Opt_gid), + fsparam_u32oct("umask", Opt_umask), + fsparam_u32oct("dmask", Opt_dmask), + fsparam_u32oct("fmask", Opt_fmask), + fsparam_u32oct("allow_utime", Opt_allow_utime), + fsparam_string("iocharset", Opt_charset), + fsparam_enum("errors", Opt_errors, exfat_param_enums), + fsparam_flag_no("discard", Opt_discard), + fsparam_flag("keep_last_dots", Opt_keep_last_dots), + fsparam_flag("sys_tz", Opt_sys_tz), + fsparam_s32("time_offset", Opt_time_offset), + fsparam_flag_no("zero_size_dir", Opt_zero_size_dir), + __fsparam(NULL, "utf8", Opt_utf8, fs_param_deprecated, + NULL), + __fsparam(NULL, "debug", Opt_debug, fs_param_deprecated, + NULL), + __fsparam(fs_param_is_u32, "namecase", Opt_namecase, + fs_param_deprecated, NULL), + __fsparam(fs_param_is_u32, "codepage", Opt_codepage, + fs_param_deprecated, NULL), + {} +}; + +static int exfat_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct exfat_sb_info *sbi = fc->s_fs_info; + struct exfat_mount_options *opts = &sbi->options; + struct fs_parse_result result; + int opt; + + opt = fs_parse(fc, exfat_parameters, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_uid: + opts->fs_uid = result.uid; + break; + case Opt_gid: + opts->fs_gid = result.gid; + break; + case Opt_umask: + opts->fs_fmask = result.uint_32; + opts->fs_dmask = result.uint_32; + break; + case Opt_dmask: + opts->fs_dmask = result.uint_32; + break; + case Opt_fmask: + opts->fs_fmask = result.uint_32; + break; + case Opt_allow_utime: + opts->allow_utime = result.uint_32 & 0022; + break; + case Opt_charset: + exfat_free_iocharset(sbi); + exfat_set_iocharset(opts, param->string); + param->string = NULL; + break; + case Opt_errors: + opts->errors = result.uint_32; + break; + case Opt_discard: + opts->discard = !result.negated; + break; + case Opt_keep_last_dots: + opts->keep_last_dots = 1; + break; + case Opt_sys_tz: + opts->sys_tz = 1; + break; + case Opt_time_offset: + /* + * Make the limit 24 just in case someone invents something + * unusual. + */ + if (result.int_32 < -24 * 60 || result.int_32 > 24 * 60) + return -EINVAL; + opts->time_offset = result.int_32; + break; + case Opt_zero_size_dir: + opts->zero_size_dir = !result.negated; + break; + case Opt_utf8: + case Opt_debug: + case Opt_namecase: + case Opt_codepage: + break; + default: + return -EINVAL; + } + + return 0; +} + +static void exfat_hash_init(struct super_block *sb) +{ + struct exfat_sb_info *sbi = EXFAT_SB(sb); + int i; + + spin_lock_init(&sbi->inode_hash_lock); + for (i = 0; i < EXFAT_HASH_SIZE; i++) + INIT_HLIST_HEAD(&sbi->inode_hashtable[i]); +} + +static int exfat_read_root(struct inode *inode, struct exfat_chain *root_clu) +{ + struct super_block *sb = inode->i_sb; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct exfat_inode_info *ei = EXFAT_I(inode); + int num_subdirs; + + exfat_chain_set(&ei->dir, sbi->root_dir, 0, ALLOC_FAT_CHAIN); + ei->entry = -1; + ei->start_clu = sbi->root_dir; + ei->flags = ALLOC_FAT_CHAIN; + ei->type = TYPE_DIR; + ei->version = 0; + ei->hint_bmap.off = EXFAT_EOF_CLUSTER; + ei->hint_stat.eidx = 0; + ei->hint_stat.clu = sbi->root_dir; + ei->hint_femp.eidx = EXFAT_HINT_NONE; + + i_size_write(inode, exfat_cluster_to_bytes(sbi, root_clu->size)); + + num_subdirs = exfat_count_dir_entries(sb, root_clu); + if (num_subdirs < 0) + return -EIO; + set_nlink(inode, num_subdirs + EXFAT_MIN_SUBDIR); + + inode->i_uid = sbi->options.fs_uid; + inode->i_gid = sbi->options.fs_gid; + inode_inc_iversion(inode); + inode->i_generation = 0; + inode->i_mode = exfat_make_mode(sbi, EXFAT_ATTR_SUBDIR, 0777); + inode->i_op = &exfat_dir_inode_operations; + inode->i_fop = &exfat_dir_operations; + + inode->i_blocks = round_up(i_size_read(inode), sbi->cluster_size) >> 9; + ei->i_pos = ((loff_t)sbi->root_dir << 32) | 0xffffffff; + + exfat_save_attr(inode, EXFAT_ATTR_SUBDIR); + ei->i_crtime = simple_inode_init_ts(inode); + exfat_truncate_inode_atime(inode); + return 0; +} + +static int exfat_calibrate_blocksize(struct super_block *sb, int logical_sect) +{ + struct exfat_sb_info *sbi = EXFAT_SB(sb); + + if (!is_power_of_2(logical_sect)) { + exfat_err(sb, "bogus logical sector size %u", logical_sect); + return -EIO; + } + + if (logical_sect < sb->s_blocksize) { + exfat_err(sb, "logical sector size too small for device (logical sector size = %u)", + logical_sect); + return -EIO; + } + + if (logical_sect > sb->s_blocksize) { + brelse(sbi->boot_bh); + sbi->boot_bh = NULL; + + if (!sb_set_blocksize(sb, logical_sect)) { + exfat_err(sb, "unable to set blocksize %u", + logical_sect); + return -EIO; + } + sbi->boot_bh = sb_bread(sb, 0); + if (!sbi->boot_bh) { + exfat_err(sb, "unable to read boot sector (logical sector size = %lu)", + sb->s_blocksize); + return -EIO; + } + } + return 0; +} + +static int exfat_read_boot_sector(struct super_block *sb) +{ + struct boot_sector *p_boot; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + + /* set block size to read super block */ + if (!sb_min_blocksize(sb, 512)) { + exfat_err(sb, "unable to set blocksize"); + return -EINVAL; + } + + /* read boot sector */ + sbi->boot_bh = sb_bread(sb, 0); + if (!sbi->boot_bh) { + exfat_err(sb, "unable to read boot sector"); + return -EIO; + } + p_boot = (struct boot_sector *)sbi->boot_bh->b_data; + + /* check the validity of BOOT */ + if (le16_to_cpu((p_boot->signature)) != BOOT_SIGNATURE) { + exfat_err(sb, "invalid boot record signature"); + return -EINVAL; + } + + if (memcmp(p_boot->fs_name, STR_EXFAT, BOOTSEC_FS_NAME_LEN)) { + exfat_err(sb, "invalid fs_name"); /* fs_name may unprintable */ + return -EINVAL; + } + + /* + * must_be_zero field must be filled with zero to prevent mounting + * from FAT volume. + */ + if (memchr_inv(p_boot->must_be_zero, 0, sizeof(p_boot->must_be_zero))) + return -EINVAL; + + if (p_boot->num_fats != 1 && p_boot->num_fats != 2) { + exfat_err(sb, "bogus number of FAT structure"); + return -EINVAL; + } + + /* + * sect_size_bits could be at least 9 and at most 12. + */ + if (p_boot->sect_size_bits < EXFAT_MIN_SECT_SIZE_BITS || + p_boot->sect_size_bits > EXFAT_MAX_SECT_SIZE_BITS) { + exfat_err(sb, "bogus sector size bits : %u", + p_boot->sect_size_bits); + return -EINVAL; + } + + /* + * sect_per_clus_bits could be at least 0 and at most 25 - sect_size_bits. + */ + if (p_boot->sect_per_clus_bits > EXFAT_MAX_SECT_PER_CLUS_BITS(p_boot)) { + exfat_err(sb, "bogus sectors bits per cluster : %u", + p_boot->sect_per_clus_bits); + return -EINVAL; + } + + sbi->sect_per_clus = 1 << p_boot->sect_per_clus_bits; + sbi->sect_per_clus_bits = p_boot->sect_per_clus_bits; + sbi->cluster_size_bits = p_boot->sect_per_clus_bits + + p_boot->sect_size_bits; + sbi->cluster_size = 1 << sbi->cluster_size_bits; + sbi->num_FAT_sectors = le32_to_cpu(p_boot->fat_length); + sbi->FAT1_start_sector = le32_to_cpu(p_boot->fat_offset); + sbi->FAT2_start_sector = le32_to_cpu(p_boot->fat_offset); + if (p_boot->num_fats == 2) + sbi->FAT2_start_sector += sbi->num_FAT_sectors; + sbi->data_start_sector = le32_to_cpu(p_boot->clu_offset); + sbi->data_start_bytes = sbi->data_start_sector << p_boot->sect_size_bits; + sbi->num_sectors = le64_to_cpu(p_boot->vol_length); + /* because the cluster index starts with 2 */ + sbi->num_clusters = le32_to_cpu(p_boot->clu_count) + + EXFAT_RESERVED_CLUSTERS; + + sbi->root_dir = le32_to_cpu(p_boot->root_cluster); + sbi->dentries_per_clu = 1 << + (sbi->cluster_size_bits - DENTRY_SIZE_BITS); + + sbi->vol_flags = le16_to_cpu(p_boot->vol_flags); + sbi->vol_flags_persistent = sbi->vol_flags & (VOLUME_DIRTY | MEDIA_FAILURE); + sbi->clu_srch_ptr = EXFAT_FIRST_CLUSTER; + + /* check consistencies */ + if ((u64)sbi->num_FAT_sectors << p_boot->sect_size_bits < + (u64)sbi->num_clusters * 4) { + exfat_err(sb, "bogus fat length"); + return -EINVAL; + } + + if (sbi->data_start_sector < + (u64)sbi->FAT1_start_sector + + (u64)sbi->num_FAT_sectors * p_boot->num_fats) { + exfat_err(sb, "bogus data start sector"); + return -EINVAL; + } + + if (sbi->vol_flags & VOLUME_DIRTY) + exfat_warn(sb, "Volume was not properly unmounted. Some data may be corrupt. Please run fsck."); + if (sbi->vol_flags & MEDIA_FAILURE) + exfat_warn(sb, "Medium has reported failures. Some data may be lost."); + + /* + * Set to the max possible volume size for this volume's cluster size so + * that any integer overflow from bytes to cluster size conversion is + * checked in inode_newsize_ok(). Clamped to MAX_LFS_FILESIZE for 32-bit + * machines. + */ + sb->s_maxbytes = min(MAX_LFS_FILESIZE, + exfat_cluster_to_bytes(sbi, (loff_t)EXFAT_MAX_NUM_CLUSTER)); + + /* check logical sector size */ + if (exfat_calibrate_blocksize(sb, 1 << p_boot->sect_size_bits)) + return -EIO; + + return 0; +} + +static int exfat_verify_boot_region(struct super_block *sb) +{ + struct buffer_head *bh = NULL; + u32 chksum = 0; + __le32 *p_sig, *p_chksum; + int sn, i; + + /* read boot sector sub-regions */ + for (sn = 0; sn < 11; sn++) { + bh = sb_bread(sb, sn); + if (!bh) + return -EIO; + + if (sn != 0 && sn <= 8) { + /* extended boot sector sub-regions */ + p_sig = (__le32 *)&bh->b_data[sb->s_blocksize - 4]; + if (le32_to_cpu(*p_sig) != EXBOOT_SIGNATURE) + exfat_warn(sb, "Invalid exboot-signature(sector = %d): 0x%08x", + sn, le32_to_cpu(*p_sig)); + } + + chksum = exfat_calc_chksum32(bh->b_data, sb->s_blocksize, + chksum, sn ? CS_DEFAULT : CS_BOOT_SECTOR); + brelse(bh); + } + + /* boot checksum sub-regions */ + bh = sb_bread(sb, sn); + if (!bh) + return -EIO; + + for (i = 0; i < sb->s_blocksize; i += sizeof(u32)) { + p_chksum = (__le32 *)&bh->b_data[i]; + if (le32_to_cpu(*p_chksum) != chksum) { + exfat_err(sb, "Invalid boot checksum (boot checksum : 0x%08x, checksum : 0x%08x)", + le32_to_cpu(*p_chksum), chksum); + brelse(bh); + return -EINVAL; + } + } + brelse(bh); + return 0; +} + +/* mount the file system volume */ +static int __exfat_fill_super(struct super_block *sb, + struct exfat_chain *root_clu) +{ + int ret; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + + ret = exfat_read_boot_sector(sb); + if (ret) { + exfat_err(sb, "failed to read boot sector"); + goto free_bh; + } + + ret = exfat_verify_boot_region(sb); + if (ret) { + exfat_err(sb, "invalid boot region"); + goto free_bh; + } + + /* + * Call exfat_count_num_cluster() before searching for up-case and + * bitmap directory entries to avoid infinite loop if they are missing + * and the cluster chain includes a loop. + */ + exfat_chain_set(root_clu, sbi->root_dir, 0, ALLOC_FAT_CHAIN); + ret = exfat_count_num_clusters(sb, root_clu, &root_clu->size); + if (ret) { + exfat_err(sb, "failed to count the number of clusters in root"); + goto free_bh; + } + + ret = exfat_create_upcase_table(sb); + if (ret) { + exfat_err(sb, "failed to load upcase table"); + goto free_bh; + } + + ret = exfat_load_bitmap(sb); + if (ret) { + exfat_err(sb, "failed to load alloc-bitmap"); + goto free_bh; + } + + if (!exfat_test_bitmap(sb, sbi->root_dir)) { + exfat_warn(sb, "failed to test first cluster bit of root dir(%u)", + sbi->root_dir); + /* + * The first cluster bit of the root directory should never + * be unset except when storage is corrupted. This bit is + * set to allow operations after mount. + */ + exfat_set_bitmap(sb, sbi->root_dir, false); + } + + ret = exfat_count_used_clusters(sb, &sbi->used_clusters); + if (ret) { + exfat_err(sb, "failed to scan clusters"); + goto free_alloc_bitmap; + } + + return 0; + +free_alloc_bitmap: + exfat_free_bitmap(sbi); +free_bh: + brelse(sbi->boot_bh); + return ret; +} + +static int exfat_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct exfat_sb_info *sbi = sb->s_fs_info; + struct exfat_mount_options *opts = &sbi->options; + struct inode *root_inode; + struct exfat_chain root_clu; + int err; + + if (opts->allow_utime == (unsigned short)-1) + opts->allow_utime = ~opts->fs_dmask & 0022; + + if (opts->discard && !bdev_max_discard_sectors(sb->s_bdev)) { + exfat_warn(sb, "mounting with \"discard\" option, but the device does not support discard"); + opts->discard = 0; + } + + sb->s_flags |= SB_NODIRATIME; + sb->s_magic = EXFAT_SUPER_MAGIC; + sb->s_op = &exfat_sops; + + sb->s_time_gran = 10 * NSEC_PER_MSEC; + sb->s_time_min = EXFAT_MIN_TIMESTAMP_SECS; + sb->s_time_max = EXFAT_MAX_TIMESTAMP_SECS; + + err = __exfat_fill_super(sb, &root_clu); + if (err) { + exfat_err(sb, "failed to recognize exfat type"); + goto check_nls_io; + } + + /* set up enough so that it can read an inode */ + exfat_hash_init(sb); + + if (sbi->options.utf8) + set_default_d_op(sb, &exfat_utf8_dentry_ops); + else { + sbi->nls_io = load_nls(sbi->options.iocharset); + if (!sbi->nls_io) { + exfat_err(sb, "IO charset %s not found", + sbi->options.iocharset); + err = -EINVAL; + goto free_table; + } + set_default_d_op(sb, &exfat_dentry_ops); + } + + root_inode = new_inode(sb); + if (!root_inode) { + exfat_err(sb, "failed to allocate root inode"); + err = -ENOMEM; + goto free_table; + } + + root_inode->i_ino = EXFAT_ROOT_INO; + inode_set_iversion(root_inode, 1); + err = exfat_read_root(root_inode, &root_clu); + if (err) { + exfat_err(sb, "failed to initialize root inode"); + goto put_inode; + } + + exfat_hash_inode(root_inode, EXFAT_I(root_inode)->i_pos); + insert_inode_hash(root_inode); + + sb->s_root = d_make_root(root_inode); + if (!sb->s_root) { + exfat_err(sb, "failed to get the root dentry"); + err = -ENOMEM; + goto free_table; + } + + return 0; + +put_inode: + iput(root_inode); + sb->s_root = NULL; + +free_table: + exfat_free_bitmap(sbi); + brelse(sbi->boot_bh); + +check_nls_io: + return err; +} + +static int exfat_get_tree(struct fs_context *fc) +{ + return get_tree_bdev(fc, exfat_fill_super); +} + +static void exfat_free_sbi(struct exfat_sb_info *sbi) +{ + exfat_free_iocharset(sbi); + kfree(sbi); +} + +static void exfat_free(struct fs_context *fc) +{ + struct exfat_sb_info *sbi = fc->s_fs_info; + + if (sbi) + exfat_free_sbi(sbi); +} + +static int exfat_reconfigure(struct fs_context *fc) +{ + struct super_block *sb = fc->root->d_sb; + struct exfat_sb_info *remount_sbi = fc->s_fs_info; + struct exfat_sb_info *sbi = EXFAT_SB(sb); + struct exfat_mount_options *new_opts = &remount_sbi->options; + struct exfat_mount_options *cur_opts = &sbi->options; + + fc->sb_flags |= SB_NODIRATIME; + + sync_filesystem(sb); + mutex_lock(&sbi->s_lock); + exfat_clear_volume_dirty(sb); + mutex_unlock(&sbi->s_lock); + + if (new_opts->allow_utime == (unsigned short)-1) + new_opts->allow_utime = ~new_opts->fs_dmask & 0022; + + /* + * Since the old settings of these mount options are cached in + * inodes or dentries, they cannot be modified dynamically. + */ + if (strcmp(new_opts->iocharset, cur_opts->iocharset) || + new_opts->keep_last_dots != cur_opts->keep_last_dots || + new_opts->sys_tz != cur_opts->sys_tz || + new_opts->time_offset != cur_opts->time_offset || + !uid_eq(new_opts->fs_uid, cur_opts->fs_uid) || + !gid_eq(new_opts->fs_gid, cur_opts->fs_gid) || + new_opts->fs_fmask != cur_opts->fs_fmask || + new_opts->fs_dmask != cur_opts->fs_dmask || + new_opts->allow_utime != cur_opts->allow_utime) + return -EINVAL; + + if (new_opts->discard != cur_opts->discard && + new_opts->discard && + !bdev_max_discard_sectors(sb->s_bdev)) { + exfat_warn(sb, "remounting with \"discard\" option, but the device does not support discard"); + return -EINVAL; + } + + swap(*cur_opts, *new_opts); + + return 0; +} + +static const struct fs_context_operations exfat_context_ops = { + .parse_param = exfat_parse_param, + .get_tree = exfat_get_tree, + .free = exfat_free, + .reconfigure = exfat_reconfigure, +}; + +static int exfat_init_fs_context(struct fs_context *fc) +{ + struct exfat_sb_info *sbi; + + sbi = kzalloc_obj(struct exfat_sb_info); + if (!sbi) + return -ENOMEM; + + mutex_init(&sbi->s_lock); + mutex_init(&sbi->bitmap_lock); + ratelimit_state_init(&sbi->ratelimit, DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE && fc->root) { + struct super_block *sb = fc->root->d_sb; + struct exfat_mount_options *cur_opts = &EXFAT_SB(sb)->options; + + sbi->options.fs_uid = cur_opts->fs_uid; + sbi->options.fs_gid = cur_opts->fs_gid; + sbi->options.fs_fmask = cur_opts->fs_fmask; + sbi->options.fs_dmask = cur_opts->fs_dmask; + } else { + sbi->options.fs_uid = current_uid(); + sbi->options.fs_gid = current_gid(); + sbi->options.fs_fmask = current->fs->umask; + sbi->options.fs_dmask = current->fs->umask; + } + + sbi->options.allow_utime = -1; + sbi->options.errors = EXFAT_ERRORS_RO; + exfat_set_iocharset(&sbi->options, exfat_default_iocharset); + + fc->s_fs_info = sbi; + fc->ops = &exfat_context_ops; + return 0; +} + +static void delayed_free(struct rcu_head *p) +{ + struct exfat_sb_info *sbi = container_of(p, struct exfat_sb_info, rcu); + + unload_nls(sbi->nls_io); + exfat_free_upcase_table(sbi); + exfat_free_sbi(sbi); +} + +static void exfat_kill_sb(struct super_block *sb) +{ + struct exfat_sb_info *sbi = sb->s_fs_info; + + kill_block_super(sb); + if (sbi) + call_rcu(&sbi->rcu, delayed_free); +} + +static struct file_system_type exfat_fs_type = { + .owner = THIS_MODULE, + .name = "exfat", + .init_fs_context = exfat_init_fs_context, + .parameters = exfat_parameters, + .kill_sb = exfat_kill_sb, + .fs_flags = FS_REQUIRES_DEV | FS_ALLOW_IDMAP, +}; + +static void exfat_inode_init_once(void *foo) +{ + struct exfat_inode_info *ei = (struct exfat_inode_info *)foo; + + spin_lock_init(&ei->cache_lru_lock); + ei->nr_caches = 0; + ei->cache_valid_id = EXFAT_CACHE_VALID + 1; + INIT_LIST_HEAD(&ei->cache_lru); + INIT_HLIST_NODE(&ei->i_hash_fat); + inode_init_once(&ei->vfs_inode); +} + +static int __init init_exfat_fs(void) +{ + int err; + + err = exfat_cache_init(); + if (err) + return err; + + exfat_inode_cachep = kmem_cache_create("exfat_inode_cache", + sizeof(struct exfat_inode_info), + 0, SLAB_RECLAIM_ACCOUNT, + exfat_inode_init_once); + if (!exfat_inode_cachep) { + err = -ENOMEM; + goto shutdown_cache; + } + + err = register_filesystem(&exfat_fs_type); + if (err) + goto destroy_cache; + + return 0; + +destroy_cache: + kmem_cache_destroy(exfat_inode_cachep); +shutdown_cache: + exfat_cache_shutdown(); + return err; +} + +static void __exit exit_exfat_fs(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(exfat_inode_cachep); + unregister_filesystem(&exfat_fs_type); + exfat_cache_shutdown(); +} + +module_init(init_exfat_fs); +module_exit(exit_exfat_fs); + +MODULE_ALIAS_FS("exfat"); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("exFAT filesystem support"); +MODULE_AUTHOR("Samsung Electronics Co., Ltd."); diff --git a/fs/exportfs/Makefile b/fs/exportfs/Makefile new file mode 100644 index 000000000..a04a8af83 --- /dev/null +++ b/fs/exportfs/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the filesystem export support routines. + +obj-$(CONFIG_EXPORTFS) += exportfs.o + +exportfs-objs := expfs.o diff --git a/fs/exportfs/expfs.c b/fs/exportfs/expfs.c new file mode 100644 index 000000000..eafd99507 --- /dev/null +++ b/fs/exportfs/expfs.c @@ -0,0 +1,610 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) Neil Brown 2002 + * Copyright (C) Christoph Hellwig 2007 + * + * This file contains the code mapping from inodes to NFS file handles, + * and for mapping back from file handles to dentries. + * + * For details on why we do all the strange and hairy things in here + * take a look at Documentation/filesystems/nfs/exporting.rst. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#define dprintk(fmt, args...) pr_debug(fmt, ##args) + + +static int get_name(const struct path *path, char *name, struct dentry *child); + + +static int exportfs_get_name(struct vfsmount *mnt, struct dentry *dir, + char *name, struct dentry *child) +{ + const struct export_operations *nop = dir->d_sb->s_export_op; + struct path path = {.mnt = mnt, .dentry = dir}; + + if (nop->get_name) + return nop->get_name(dir, name, child); + else + return get_name(&path, name, child); +} + +/* + * Check if the dentry or any of it's aliases is acceptable. + */ +static struct dentry * +find_acceptable_alias(struct dentry *result, + int (*acceptable)(void *context, struct dentry *dentry), + void *context) +{ + struct dentry *dentry, *toput = NULL; + struct inode *inode; + + if (acceptable(context, result)) + return result; + + inode = result->d_inode; + spin_lock(&inode->i_lock); + for_each_alias(dentry, inode) { + if (!dget_alias_ilocked(dentry)) + continue; + spin_unlock(&inode->i_lock); + dput(toput); + if (dentry != result && acceptable(context, dentry)) { + dput(result); + return dentry; + } + spin_lock(&inode->i_lock); + toput = dentry; + } + spin_unlock(&inode->i_lock); + + dput(toput); + return NULL; +} + +static bool dentry_connected(struct dentry *dentry) +{ + dget(dentry); + while (dentry->d_flags & DCACHE_DISCONNECTED) { + struct dentry *parent = dget_parent(dentry); + + dput(dentry); + if (dentry == parent) { + dput(parent); + return false; + } + dentry = parent; + } + dput(dentry); + return true; +} + +static void clear_disconnected(struct dentry *dentry) +{ + dget(dentry); + while (dentry->d_flags & DCACHE_DISCONNECTED) { + struct dentry *parent = dget_parent(dentry); + + WARN_ON_ONCE(IS_ROOT(dentry)); + + spin_lock(&dentry->d_lock); + dentry->d_flags &= ~DCACHE_DISCONNECTED; + spin_unlock(&dentry->d_lock); + + dput(dentry); + dentry = parent; + } + dput(dentry); +} + +/* + * Reconnect a directory dentry with its parent. + * + * This can return a dentry, or NULL, or an error. + * + * In the first case the returned dentry is the parent of the given + * dentry, and may itself need to be reconnected to its parent. + * + * In the NULL case, a concurrent VFS operation has either renamed or + * removed this directory. The concurrent operation has reconnected our + * dentry, so we no longer need to. + */ +static struct dentry *reconnect_one(struct vfsmount *mnt, + struct dentry *dentry, char *nbuf) +{ + struct dentry *parent; + struct dentry *tmp; + int err; + + parent = ERR_PTR(-EACCES); + if (mnt->mnt_sb->s_export_op->get_parent) + parent = mnt->mnt_sb->s_export_op->get_parent(dentry); + + if (IS_ERR(parent)) { + dprintk("get_parent of %llu failed, err %ld\n", + dentry->d_inode->i_ino, PTR_ERR(parent)); + return parent; + } + + dprintk("%s: find name of %llu in %llu\n", __func__, + dentry->d_inode->i_ino, parent->d_inode->i_ino); + err = exportfs_get_name(mnt, parent, nbuf, dentry); + if (err == -ENOENT) + goto out_reconnected; + if (err) + goto out_err; + dprintk("%s: found name: %s\n", __func__, nbuf); + tmp = lookup_one_unlocked(mnt_idmap(mnt), &QSTR(nbuf), parent); + if (IS_ERR(tmp)) { + dprintk("lookup failed: %ld\n", PTR_ERR(tmp)); + err = PTR_ERR(tmp); + goto out_err; + } + if (tmp != dentry) { + /* + * Somebody has renamed it since exportfs_get_name(); + * great, since it could've only been renamed if it + * got looked up and thus connected, and it would + * remain connected afterwards. We are done. + */ + dput(tmp); + goto out_reconnected; + } + dput(tmp); + if (IS_ROOT(dentry)) { + err = -ESTALE; + goto out_err; + } + return parent; + +out_err: + dput(parent); + return ERR_PTR(err); +out_reconnected: + dput(parent); + /* + * Someone must have renamed our entry into another parent, in + * which case it has been reconnected by the rename. + * + * Or someone removed it entirely, in which case filehandle + * lookup will succeed but the directory is now IS_DEAD and + * subsequent operations on it will fail. + * + * Alternatively, maybe there was no race at all, and the + * filesystem is just corrupt and gave us a parent that doesn't + * actually contain any entry pointing to this inode. So, + * double check that this worked and return -ESTALE if not: + */ + if (!dentry_connected(dentry)) + return ERR_PTR(-ESTALE); + return NULL; +} + +/* + * Make sure target_dir is fully connected to the dentry tree. + * + * On successful return, DCACHE_DISCONNECTED will be cleared on + * target_dir, and target_dir->d_parent->...->d_parent will reach the + * root of the filesystem. + * + * Whenever DCACHE_DISCONNECTED is unset, target_dir is fully connected. + * But the converse is not true: target_dir may have DCACHE_DISCONNECTED + * set but already be connected. In that case we'll verify the + * connection to root and then clear the flag. + * + * Note that target_dir could be removed by a concurrent operation. In + * that case reconnect_path may still succeed with target_dir fully + * connected, but further operations using the filehandle will fail when + * necessary (due to S_DEAD being set on the directory). + */ +static int +reconnect_path(struct vfsmount *mnt, struct dentry *target_dir, char *nbuf) +{ + struct dentry *dentry, *parent; + + dentry = dget(target_dir); + + while (dentry->d_flags & DCACHE_DISCONNECTED) { + BUG_ON(dentry == mnt->mnt_sb->s_root); + + if (IS_ROOT(dentry)) + parent = reconnect_one(mnt, dentry, nbuf); + else + parent = dget_parent(dentry); + + if (!parent) + break; + dput(dentry); + if (IS_ERR(parent)) + return PTR_ERR(parent); + dentry = parent; + } + dput(dentry); + clear_disconnected(target_dir); + return 0; +} + +struct getdents_callback { + struct dir_context ctx; + char *name; /* name that was found. It already points to a + buffer NAME_MAX+1 is size */ + u64 ino; /* the inum we are looking for */ + int found; /* inode matched? */ + int sequence; /* sequence counter */ +}; + +/* + * A rather strange filldir function to capture + * the name matching the specified inode number. + */ +static bool filldir_one(struct dir_context *ctx, const char *name, int len, + loff_t pos, u64 ino, unsigned int d_type) +{ + struct getdents_callback *buf = + container_of(ctx, struct getdents_callback, ctx); + + buf->sequence++; + if (buf->ino == ino && len <= NAME_MAX && + !name_is_dot_dotdot(name, len)) { + memcpy(buf->name, name, len); + buf->name[len] = '\0'; + buf->found = 1; + return false; // no more + } + return true; +} + +/** + * get_name - default export_operations->get_name function + * @path: the directory in which to find a name + * @name: a pointer to a %NAME_MAX+1 char buffer to store the name + * @child: the dentry for the child directory. + * + * calls readdir on the parent until it finds an entry with + * the same inode number as the child, and returns that. + */ +static int get_name(const struct path *path, char *name, struct dentry *child) +{ + const struct cred *cred = current_cred(); + struct inode *dir = path->dentry->d_inode; + int error; + struct file *file; + struct kstat stat; + struct path child_path = { + .mnt = path->mnt, + .dentry = child, + }; + struct getdents_callback buffer = { + .ctx.actor = filldir_one, + .ctx.count = INT_MAX, + .name = name, + }; + + error = -ENOTDIR; + if (!dir || !S_ISDIR(dir->i_mode)) + goto out; + error = -EINVAL; + if (!dir->i_fop) + goto out; + /* + * inode->i_ino is unsigned long, kstat->ino is u64, so the + * former would be insufficient on 32-bit hosts when the + * filesystem supports 64-bit inode numbers. So we need to + * actually call ->getattr, not just read i_ino: + */ + error = vfs_getattr_nosec(&child_path, &stat, + STATX_INO, AT_STATX_SYNC_AS_STAT); + if (error) + return error; + buffer.ino = stat.ino; + /* + * Open the directory ... + */ + file = dentry_open(path, O_RDONLY, cred); + error = PTR_ERR(file); + if (IS_ERR(file)) + goto out; + + error = -EINVAL; + if (!file->f_op->iterate_shared) + goto out_close; + + buffer.sequence = 0; + while (1) { + int old_seq = buffer.sequence; + + error = iterate_dir(file, &buffer.ctx); + if (buffer.found) { + error = 0; + break; + } + + if (error < 0) + break; + + error = -ENOENT; + if (old_seq == buffer.sequence) + break; + } + +out_close: + fput(file); +out: + return error; +} + +#define FILEID_INO64_GEN_LEN 3 + +/** + * exportfs_encode_ino64_fid - encode non-decodeable 64bit ino file id + * @inode: the object to encode + * @fid: where to store the file handle fragment + * @max_len: maximum length to store there (in 4 byte units) + * + * This generic function is used to encode a non-decodeable file id for + * fanotify for filesystems that do not support NFS export. + */ +static int exportfs_encode_ino64_fid(struct inode *inode, struct fid *fid, + int *max_len) +{ + if (*max_len < FILEID_INO64_GEN_LEN) { + *max_len = FILEID_INO64_GEN_LEN; + return FILEID_INVALID; + } + + fid->i64.ino = inode->i_ino; + fid->i64.gen = inode->i_generation; + *max_len = FILEID_INO64_GEN_LEN; + + return FILEID_INO64_GEN; +} + +/** + * exportfs_encode_inode_fh - encode a file handle from inode + * @inode: the object to encode + * @fid: where to store the file handle fragment + * @max_len: maximum length to store there + * @parent: parent directory inode, if wanted + * @flags: properties of the requested file handle + * + * Returns an enum fid_type or a negative errno. + */ +int exportfs_encode_inode_fh(struct inode *inode, struct fid *fid, + int *max_len, struct inode *parent, int flags) +{ + const struct export_operations *nop = inode->i_sb->s_export_op; + enum fid_type type; + + if (!exportfs_can_encode_fh(nop, flags)) + return -EOPNOTSUPP; + + if (!nop && (flags & EXPORT_FH_FID)) + type = exportfs_encode_ino64_fid(inode, fid, max_len); + else + type = nop->encode_fh(inode, fid->raw, max_len, parent); + + if (type > 0 && FILEID_USER_FLAGS(type)) { + pr_warn_once("%s: unexpected fh type value 0x%x from fstype %s.\n", + __func__, type, inode->i_sb->s_type->name); + return -EINVAL; + } + + return type; + +} +EXPORT_SYMBOL_GPL(exportfs_encode_inode_fh); + +/** + * exportfs_encode_fh - encode a file handle from dentry + * @dentry: the object to encode + * @fid: where to store the file handle fragment + * @max_len: maximum length to store there + * @flags: properties of the requested file handle + * + * Returns an enum fid_type or a negative errno. + */ +int exportfs_encode_fh(struct dentry *dentry, struct fid *fid, int *max_len, + int flags) +{ + int error; + struct dentry *p = NULL; + struct inode *inode = dentry->d_inode, *parent = NULL; + + if ((flags & EXPORT_FH_CONNECTABLE) && !S_ISDIR(inode->i_mode)) { + p = dget_parent(dentry); + /* + * note that while p might've ceased to be our parent already, + * it's still pinned by and still positive. + */ + parent = p->d_inode; + } + + error = exportfs_encode_inode_fh(inode, fid, max_len, parent, flags); + dput(p); + + return error; +} +EXPORT_SYMBOL_GPL(exportfs_encode_fh); + +struct dentry * +exportfs_decode_fh_raw(struct vfsmount *mnt, struct fid *fid, int fh_len, + int fileid_type, unsigned int flags, + int (*acceptable)(void *, struct dentry *), + void *context) +{ + const struct export_operations *nop = mnt->mnt_sb->s_export_op; + struct dentry *result, *alias; + char nbuf[NAME_MAX+1]; + int err; + + if (fileid_type < 0 || FILEID_USER_FLAGS(fileid_type)) + return ERR_PTR(-EINVAL); + + /* + * Try to get any dentry for the given file handle from the filesystem. + */ + if (!exportfs_can_decode_fh(nop)) + return ERR_PTR(-ESTALE); + result = nop->fh_to_dentry(mnt->mnt_sb, fid, fh_len, fileid_type); + if (IS_ERR_OR_NULL(result)) + return result; + + if ((flags & EXPORT_FH_DIR_ONLY) && !d_is_dir(result)) { + err = -ENOTDIR; + goto err_result; + } + + /* + * If no acceptance criteria was specified by caller, a disconnected + * dentry is also accepatable. Callers may use this mode to query if + * file handle is stale or to get a reference to an inode without + * risking the high overhead caused by directory reconnect. + */ + if (!acceptable) + return result; + + if (d_is_dir(result)) { + /* + * This request is for a directory. + * + * On the positive side there is only one dentry for each + * directory inode. On the negative side this implies that we + * to ensure our dentry is connected all the way up to the + * filesystem root. + */ + if (result->d_flags & DCACHE_DISCONNECTED) { + err = reconnect_path(mnt, result, nbuf); + if (err) + goto err_result; + } + + if (!acceptable(context, result)) { + err = -EACCES; + goto err_result; + } + + return result; + } else { + /* + * It's not a directory. Life is a little more complicated. + */ + struct dentry *target_dir, *nresult; + + /* + * See if either the dentry we just got from the filesystem + * or any alias for it is acceptable. This is always true + * if this filesystem is exported without the subtreecheck + * option. If the filesystem is exported with the subtree + * check option there's a fair chance we need to look at + * the parent directory in the file handle and make sure + * it's connected to the filesystem root. + */ + alias = find_acceptable_alias(result, acceptable, context); + if (alias) + return alias; + + /* + * Try to extract a dentry for the parent directory from the + * file handle. If this fails we'll have to give up. + */ + err = -ESTALE; + if (!nop->fh_to_parent) + goto err_result; + + target_dir = nop->fh_to_parent(mnt->mnt_sb, fid, + fh_len, fileid_type); + if (!target_dir) + goto err_result; + err = PTR_ERR(target_dir); + if (IS_ERR(target_dir)) + goto err_result; + + /* + * And as usual we need to make sure the parent directory is + * connected to the filesystem root. The VFS really doesn't + * like disconnected directories.. + */ + err = reconnect_path(mnt, target_dir, nbuf); + if (err) { + dput(target_dir); + goto err_result; + } + + /* + * Now that we've got both a well-connected parent and a + * dentry for the inode we're after, make sure that our + * inode is actually connected to the parent. + */ + err = exportfs_get_name(mnt, target_dir, nbuf, result); + if (err) { + dput(target_dir); + goto err_result; + } + + nresult = lookup_one_unlocked(mnt_idmap(mnt), &QSTR(nbuf), target_dir); + if (!IS_ERR(nresult)) { + if (unlikely(nresult->d_inode != result->d_inode)) { + dput(nresult); + nresult = ERR_PTR(-ESTALE); + } + } + /* + * At this point we are done with the parent, but it's pinned + * by the child dentry anyway. + */ + dput(target_dir); + + if (IS_ERR(nresult)) { + err = PTR_ERR(nresult); + goto err_result; + } + dput(result); + result = nresult; + + /* + * And finally make sure the dentry is actually acceptable + * to NFSD. + */ + alias = find_acceptable_alias(result, acceptable, context); + if (!alias) { + err = -EACCES; + goto err_result; + } + + return alias; + } + + err_result: + dput(result); + return ERR_PTR(err); +} +EXPORT_SYMBOL_GPL(exportfs_decode_fh_raw); + +struct dentry *exportfs_decode_fh(struct vfsmount *mnt, struct fid *fid, + int fh_len, int fileid_type, + int (*acceptable)(void *, struct dentry *), + void *context) +{ + struct dentry *ret; + + ret = exportfs_decode_fh_raw(mnt, fid, fh_len, fileid_type, 0, + acceptable, context); + if (IS_ERR_OR_NULL(ret)) { + if (ret == ERR_PTR(-ENOMEM)) + return ret; + return ERR_PTR(-ESTALE); + } + return ret; +} +EXPORT_SYMBOL_GPL(exportfs_decode_fh); + +MODULE_DESCRIPTION("Code mapping from inodes to file handles"); +MODULE_LICENSE("GPL"); diff --git a/fs/ext2/Kconfig b/fs/ext2/Kconfig new file mode 100644 index 000000000..d5bce83ad --- /dev/null +++ b/fs/ext2/Kconfig @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: GPL-2.0-only +config EXT2_FS + tristate "Second extended fs support (DEPRECATED)" + select BUFFER_HEAD + select FS_IOMAP + help + Ext2 is a standard Linux file system for hard disks. + + This filesystem driver is deprecated because it does not properly + support inode time stamps beyond 03:14:07 UTC on 19 January 2038. + + Ext2 users are advised to use ext4 driver to access their filesystem. + The driver is fully compatible, supports filesystems without journal + or extents, and also supports larger time stamps if the filesystem + is created with at least 256 byte inodes. + + This code is kept as a simple reference for filesystem developers. + + If unsure, say N. + +config EXT2_FS_XATTR + bool "Ext2 extended attributes" + depends on EXT2_FS + help + Extended attributes are name:value pairs associated with inodes by + the kernel or by users (see the attr(5) manual page for details). + + If unsure, say N. + +config EXT2_FS_POSIX_ACL + bool "Ext2 POSIX Access Control Lists" + depends on EXT2_FS_XATTR + select FS_POSIX_ACL + help + Posix Access Control Lists (ACLs) support permissions for users and + groups beyond the owner/group/world scheme. + + If you don't know what Access Control Lists are, say N + +config EXT2_FS_SECURITY + bool "Ext2 Security Labels" + depends on EXT2_FS_XATTR + help + Security labels support alternative access control models + implemented by security modules like SELinux. This option + enables an extended attribute handler for file security + labels in the ext2 filesystem. + + If you are not using a security module that requires using + extended attributes for file security labels, say N. diff --git a/fs/ext2/Makefile b/fs/ext2/Makefile new file mode 100644 index 000000000..8860948ef --- /dev/null +++ b/fs/ext2/Makefile @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the linux ext2-filesystem routines. +# + +obj-$(CONFIG_EXT2_FS) += ext2.o + +ext2-y := balloc.o dir.o file.o ialloc.o inode.o \ + ioctl.o namei.o super.o symlink.o trace.o + +# For tracepoints to include our trace.h from tracepoint infrastructure +CFLAGS_trace.o := -I$(src) + +ext2-$(CONFIG_EXT2_FS_XATTR) += xattr.o xattr_user.o xattr_trusted.o +ext2-$(CONFIG_EXT2_FS_POSIX_ACL) += acl.o +ext2-$(CONFIG_EXT2_FS_SECURITY) += xattr_security.o diff --git a/fs/ext2/acl.c b/fs/ext2/acl.c new file mode 100644 index 000000000..7e54c3158 --- /dev/null +++ b/fs/ext2/acl.c @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext2/acl.c + * + * Copyright (C) 2001-2003 Andreas Gruenbacher, + */ + +#include +#include +#include +#include +#include "ext2.h" +#include "xattr.h" +#include "acl.h" + +/* + * Convert from filesystem to in-memory representation. + */ +static struct posix_acl * +ext2_acl_from_disk(const void *value, size_t size) +{ + const char *end = (char *)value + size; + int n, count; + struct posix_acl *acl; + + if (!value) + return NULL; + if (size < sizeof(ext2_acl_header)) + return ERR_PTR(-EINVAL); + if (((ext2_acl_header *)value)->a_version != + cpu_to_le32(EXT2_ACL_VERSION)) + return ERR_PTR(-EINVAL); + value = (char *)value + sizeof(ext2_acl_header); + count = ext2_acl_count(size); + if (count < 0) + return ERR_PTR(-EINVAL); + if (count == 0) + return NULL; + acl = posix_acl_alloc(count, GFP_KERNEL); + if (!acl) + return ERR_PTR(-ENOMEM); + for (n=0; n < count; n++) { + ext2_acl_entry *entry = + (ext2_acl_entry *)value; + if ((char *)value + sizeof(ext2_acl_entry_short) > end) + goto fail; + acl->a_entries[n].e_tag = le16_to_cpu(entry->e_tag); + acl->a_entries[n].e_perm = le16_to_cpu(entry->e_perm); + switch(acl->a_entries[n].e_tag) { + case ACL_USER_OBJ: + case ACL_GROUP_OBJ: + case ACL_MASK: + case ACL_OTHER: + value = (char *)value + + sizeof(ext2_acl_entry_short); + break; + + case ACL_USER: + value = (char *)value + sizeof(ext2_acl_entry); + if ((char *)value > end) + goto fail; + acl->a_entries[n].e_uid = + make_kuid(&init_user_ns, + le32_to_cpu(entry->e_id)); + break; + case ACL_GROUP: + value = (char *)value + sizeof(ext2_acl_entry); + if ((char *)value > end) + goto fail; + acl->a_entries[n].e_gid = + make_kgid(&init_user_ns, + le32_to_cpu(entry->e_id)); + break; + + default: + goto fail; + } + } + if (value != end) + goto fail; + return acl; + +fail: + posix_acl_release(acl); + return ERR_PTR(-EINVAL); +} + +/* + * Convert from in-memory to filesystem representation. + */ +static void * +ext2_acl_to_disk(const struct posix_acl *acl, size_t *size) +{ + ext2_acl_header *ext_acl; + char *e; + size_t n; + + *size = ext2_acl_size(acl->a_count); + ext_acl = kmalloc(sizeof(ext2_acl_header) + acl->a_count * + sizeof(ext2_acl_entry), GFP_KERNEL); + if (!ext_acl) + return ERR_PTR(-ENOMEM); + ext_acl->a_version = cpu_to_le32(EXT2_ACL_VERSION); + e = (char *)ext_acl + sizeof(ext2_acl_header); + for (n=0; n < acl->a_count; n++) { + const struct posix_acl_entry *acl_e = &acl->a_entries[n]; + ext2_acl_entry *entry = (ext2_acl_entry *)e; + entry->e_tag = cpu_to_le16(acl_e->e_tag); + entry->e_perm = cpu_to_le16(acl_e->e_perm); + switch(acl_e->e_tag) { + case ACL_USER: + entry->e_id = cpu_to_le32( + from_kuid(&init_user_ns, acl_e->e_uid)); + e += sizeof(ext2_acl_entry); + break; + case ACL_GROUP: + entry->e_id = cpu_to_le32( + from_kgid(&init_user_ns, acl_e->e_gid)); + e += sizeof(ext2_acl_entry); + break; + + case ACL_USER_OBJ: + case ACL_GROUP_OBJ: + case ACL_MASK: + case ACL_OTHER: + e += sizeof(ext2_acl_entry_short); + break; + + default: + goto fail; + } + } + return (char *)ext_acl; + +fail: + kfree(ext_acl); + return ERR_PTR(-EINVAL); +} + +/* + * inode->i_mutex: don't care + */ +struct posix_acl * +ext2_get_acl(struct inode *inode, int type, bool rcu) +{ + int name_index; + char *value = NULL; + struct posix_acl *acl; + int retval; + + if (rcu) + return ERR_PTR(-ECHILD); + + switch (type) { + case ACL_TYPE_ACCESS: + name_index = EXT2_XATTR_INDEX_POSIX_ACL_ACCESS; + break; + case ACL_TYPE_DEFAULT: + name_index = EXT2_XATTR_INDEX_POSIX_ACL_DEFAULT; + break; + default: + BUG(); + } + retval = ext2_xattr_get(inode, name_index, "", NULL, 0); + if (retval > 0) { + value = kmalloc(retval, GFP_KERNEL); + if (!value) + return ERR_PTR(-ENOMEM); + retval = ext2_xattr_get(inode, name_index, "", value, retval); + } + if (retval > 0) + acl = ext2_acl_from_disk(value, retval); + else if (retval == -ENODATA || retval == -ENOSYS) + acl = NULL; + else + acl = ERR_PTR(retval); + kfree(value); + + return acl; +} + +static int +__ext2_set_acl(struct inode *inode, struct posix_acl *acl, int type) +{ + int name_index; + void *value = NULL; + size_t size = 0; + int error; + + switch(type) { + case ACL_TYPE_ACCESS: + name_index = EXT2_XATTR_INDEX_POSIX_ACL_ACCESS; + break; + + case ACL_TYPE_DEFAULT: + name_index = EXT2_XATTR_INDEX_POSIX_ACL_DEFAULT; + if (!S_ISDIR(inode->i_mode)) + return acl ? -EACCES : 0; + break; + + default: + return -EINVAL; + } + if (acl) { + value = ext2_acl_to_disk(acl, &size); + if (IS_ERR(value)) + return (int)PTR_ERR(value); + } + + error = ext2_xattr_set(inode, name_index, "", value, size, 0); + + kfree(value); + if (!error) + set_cached_acl(inode, type, acl); + return error; +} + +/* + * inode->i_mutex: down + */ +int +ext2_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type) +{ + int error; + int update_mode = 0; + struct inode *inode = d_inode(dentry); + umode_t mode = inode->i_mode; + + if (type == ACL_TYPE_ACCESS && acl) { + error = posix_acl_update_mode(&nop_mnt_idmap, inode, &mode, + &acl); + if (error) + return error; + update_mode = 1; + } + error = __ext2_set_acl(inode, acl, type); + if (!error && update_mode) { + inode->i_mode = mode; + inode_set_ctime_current(inode); + mark_inode_dirty(inode); + } + return error; +} + +/* + * Initialize the ACLs of a new inode. Called from ext2_new_inode. + * + * dir->i_mutex: down + * inode->i_mutex: up (access to inode is still exclusive) + */ +int +ext2_init_acl(struct inode *inode, struct inode *dir) +{ + struct posix_acl *default_acl, *acl; + int error; + + error = posix_acl_create(dir, &inode->i_mode, &default_acl, &acl); + if (error) + return error; + + if (default_acl) { + error = __ext2_set_acl(inode, default_acl, ACL_TYPE_DEFAULT); + posix_acl_release(default_acl); + } else { + inode->i_default_acl = NULL; + } + if (acl) { + if (!error) + error = __ext2_set_acl(inode, acl, ACL_TYPE_ACCESS); + posix_acl_release(acl); + } else { + inode->i_acl = NULL; + } + return error; +} diff --git a/fs/ext2/acl.h b/fs/ext2/acl.h new file mode 100644 index 000000000..4a8443a2b --- /dev/null +++ b/fs/ext2/acl.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + File: fs/ext2/acl.h + + (C) 2001 Andreas Gruenbacher, +*/ + +#include + +#define EXT2_ACL_VERSION 0x0001 + +typedef struct { + __le16 e_tag; + __le16 e_perm; + __le32 e_id; +} ext2_acl_entry; + +typedef struct { + __le16 e_tag; + __le16 e_perm; +} ext2_acl_entry_short; + +typedef struct { + __le32 a_version; +} ext2_acl_header; + +static inline size_t ext2_acl_size(int count) +{ + if (count <= 4) { + return sizeof(ext2_acl_header) + + count * sizeof(ext2_acl_entry_short); + } else { + return sizeof(ext2_acl_header) + + 4 * sizeof(ext2_acl_entry_short) + + (count - 4) * sizeof(ext2_acl_entry); + } +} + +static inline int ext2_acl_count(size_t size) +{ + ssize_t s; + size -= sizeof(ext2_acl_header); + s = size - 4 * sizeof(ext2_acl_entry_short); + if (s < 0) { + if (size % sizeof(ext2_acl_entry_short)) + return -1; + return size / sizeof(ext2_acl_entry_short); + } else { + if (s % sizeof(ext2_acl_entry)) + return -1; + return s / sizeof(ext2_acl_entry) + 4; + } +} + +#ifdef CONFIG_EXT2_FS_POSIX_ACL + +/* acl.c */ +extern struct posix_acl *ext2_get_acl(struct inode *inode, int type, bool rcu); +extern int ext2_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type); +extern int ext2_init_acl (struct inode *, struct inode *); + +#else +#include +#define ext2_get_acl NULL +#define ext2_set_acl NULL + +static inline int ext2_init_acl (struct inode *inode, struct inode *dir) +{ + return 0; +} +#endif + diff --git a/fs/ext2/balloc.c b/fs/ext2/balloc.c new file mode 100644 index 000000000..adf0f31fb --- /dev/null +++ b/fs/ext2/balloc.c @@ -0,0 +1,1535 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext2/balloc.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * Enhanced block allocation by Stephen Tweedie (sct@redhat.com), 1993 + * Big-endian to little-endian byte-swapping/bitmaps by + * David S. Miller (davem@caip.rutgers.edu), 1995 + */ + +#include "ext2.h" +#include +#include +#include +#include +#include +#include + +/* + * balloc.c contains the blocks allocation and deallocation routines + */ + +/* + * The free blocks are managed by bitmaps. A file system contains several + * blocks groups. Each group contains 1 bitmap block for blocks, 1 bitmap + * block for inodes, N blocks for the inode table and data blocks. + * + * The file system contains group descriptors which are located after the + * super block. Each descriptor contains the number of the bitmap block and + * the free blocks count in the block. The descriptors are loaded in memory + * when a file system is mounted (see ext2_fill_super). + */ + + +struct ext2_group_desc * ext2_get_group_desc(struct super_block * sb, + unsigned int block_group, + struct buffer_head ** bh) +{ + unsigned long group_desc; + unsigned long offset; + struct ext2_group_desc * desc; + struct ext2_sb_info *sbi = EXT2_SB(sb); + + if (block_group >= sbi->s_groups_count) { + WARN(1, "block_group >= groups_count - " + "block_group = %d, groups_count = %lu", + block_group, sbi->s_groups_count); + + return NULL; + } + + group_desc = block_group >> EXT2_DESC_PER_BLOCK_BITS(sb); + offset = block_group & (EXT2_DESC_PER_BLOCK(sb) - 1); + if (!sbi->s_group_desc[group_desc]) { + WARN(1, "Group descriptor not loaded - " + "block_group = %d, group_desc = %lu, desc = %lu", + block_group, group_desc, offset); + return NULL; + } + + desc = (struct ext2_group_desc *) sbi->s_group_desc[group_desc]->b_data; + if (bh) + *bh = sbi->s_group_desc[group_desc]; + return desc + offset; +} + +static int ext2_valid_block_bitmap(struct super_block *sb, + struct ext2_group_desc *desc, + unsigned int block_group, + struct buffer_head *bh) +{ + ext2_grpblk_t offset; + ext2_grpblk_t next_zero_bit; + ext2_fsblk_t bitmap_blk; + ext2_fsblk_t group_first_block; + ext2_grpblk_t max_bit; + + group_first_block = ext2_group_first_block_no(sb, block_group); + max_bit = ext2_group_last_block_no(sb, block_group) - group_first_block; + + /* check whether block bitmap block number is set */ + bitmap_blk = le32_to_cpu(desc->bg_block_bitmap); + offset = bitmap_blk - group_first_block; + if (offset < 0 || offset > max_bit || + !ext2_test_bit(offset, bh->b_data)) + /* bad block bitmap */ + goto err_out; + + /* check whether the inode bitmap block number is set */ + bitmap_blk = le32_to_cpu(desc->bg_inode_bitmap); + offset = bitmap_blk - group_first_block; + if (offset < 0 || offset > max_bit || + !ext2_test_bit(offset, bh->b_data)) + /* bad block bitmap */ + goto err_out; + + /* check whether the inode table block number is set */ + bitmap_blk = le32_to_cpu(desc->bg_inode_table); + offset = bitmap_blk - group_first_block; + if (offset < 0 || offset > max_bit || + offset + EXT2_SB(sb)->s_itb_per_group - 1 > max_bit) + goto err_out; + next_zero_bit = ext2_find_next_zero_bit(bh->b_data, + offset + EXT2_SB(sb)->s_itb_per_group, + offset); + if (next_zero_bit >= offset + EXT2_SB(sb)->s_itb_per_group) + /* good bitmap for inode tables */ + return 1; + +err_out: + ext2_error(sb, __func__, + "Invalid block bitmap - " + "block_group = %d, block = %lu", + block_group, bitmap_blk); + return 0; +} + +/* + * Read the bitmap for a given block_group,and validate the + * bits for block/inode/inode tables are set in the bitmaps + * + * Return buffer_head on success or NULL in case of failure. + */ +static struct buffer_head * +read_block_bitmap(struct super_block *sb, unsigned int block_group) +{ + struct ext2_group_desc * desc; + struct buffer_head * bh = NULL; + ext2_fsblk_t bitmap_blk; + int ret; + + desc = ext2_get_group_desc(sb, block_group, NULL); + if (!desc) + return NULL; + bitmap_blk = le32_to_cpu(desc->bg_block_bitmap); + bh = sb_getblk(sb, bitmap_blk); + if (unlikely(!bh)) { + ext2_error(sb, __func__, + "Cannot read block bitmap - " + "block_group = %d, block_bitmap = %u", + block_group, le32_to_cpu(desc->bg_block_bitmap)); + return NULL; + } + ret = bh_read(bh, 0); + if (ret > 0) + return bh; + if (ret < 0) { + brelse(bh); + ext2_error(sb, __func__, + "Cannot read block bitmap - " + "block_group = %d, block_bitmap = %u", + block_group, le32_to_cpu(desc->bg_block_bitmap)); + return NULL; + } + + ext2_valid_block_bitmap(sb, desc, block_group, bh); + /* + * file system mounted not to panic on error, continue with corrupt + * bitmap + */ + return bh; +} + +static void group_adjust_blocks(struct super_block *sb, int group_no, + struct ext2_group_desc *desc, struct buffer_head *bh, int count) +{ + if (count) { + struct ext2_sb_info *sbi = EXT2_SB(sb); + unsigned free_blocks; + + spin_lock(sb_bgl_lock(sbi, group_no)); + free_blocks = le16_to_cpu(desc->bg_free_blocks_count); + desc->bg_free_blocks_count = cpu_to_le16(free_blocks + count); + spin_unlock(sb_bgl_lock(sbi, group_no)); + mark_buffer_dirty(bh); + } +} + +/* + * The reservation window structure operations + * -------------------------------------------- + * Operations include: + * dump, find, add, remove, is_empty, find_next_reservable_window, etc. + * + * We use a red-black tree to represent per-filesystem reservation + * windows. + * + */ + +/** + * __rsv_window_dump() -- Dump the filesystem block allocation reservation map + * @root: root of per-filesystem reservation rb tree + * @verbose: verbose mode + * @fn: function which wishes to dump the reservation map + * + * If verbose is turned on, it will print the whole block reservation + * windows(start, end). Otherwise, it will only print out the "bad" windows, + * those windows that overlap with their immediate neighbors. + */ +#ifdef EXT2FS_DEBUG +static void __rsv_window_dump(struct rb_root *root, int verbose, + const char *fn) +{ + struct rb_node *n; + struct ext2_reserve_window_node *rsv, *prev; + int bad; + +restart: + n = rb_first(root); + bad = 0; + prev = NULL; + + printk("Block Allocation Reservation Windows Map (%s):\n", fn); + while (n) { + rsv = rb_entry(n, struct ext2_reserve_window_node, rsv_node); + if (verbose) + printk("reservation window 0x%p " + "start: %lu, end: %lu\n", + rsv, rsv->rsv_start, rsv->rsv_end); + if (rsv->rsv_start && rsv->rsv_start >= rsv->rsv_end) { + printk("Bad reservation %p (start >= end)\n", + rsv); + bad = 1; + } + if (prev && prev->rsv_end >= rsv->rsv_start) { + printk("Bad reservation %p (prev->end >= start)\n", + rsv); + bad = 1; + } + if (bad) { + if (!verbose) { + printk("Restarting reservation walk in verbose mode\n"); + verbose = 1; + goto restart; + } + } + n = rb_next(n); + prev = rsv; + } + printk("Window map complete.\n"); + BUG_ON(bad); +} +#define rsv_window_dump(root, verbose) \ + __rsv_window_dump((root), (verbose), __func__) +#else +#define rsv_window_dump(root, verbose) do {} while (0) +#endif /* EXT2FS_DEBUG */ + +/** + * goal_in_my_reservation() + * @rsv: inode's reservation window + * @grp_goal: given goal block relative to the allocation block group + * @group: the current allocation block group + * @sb: filesystem super block + * + * Test if the given goal block (group relative) is within the file's + * own block reservation window range. + * + * If the reservation window is outside the goal allocation group, return 0; + * grp_goal (given goal block) could be -1, which means no specific + * goal block. In this case, always return 1. + * If the goal block is within the reservation window, return 1; + * otherwise, return 0; + */ +static int +goal_in_my_reservation(struct ext2_reserve_window *rsv, ext2_grpblk_t grp_goal, + unsigned int group, struct super_block * sb) +{ + ext2_fsblk_t group_first_block, group_last_block; + + group_first_block = ext2_group_first_block_no(sb, group); + group_last_block = ext2_group_last_block_no(sb, group); + + if ((rsv->_rsv_start > group_last_block) || + (rsv->_rsv_end < group_first_block)) + return 0; + if ((grp_goal >= 0) && ((grp_goal + group_first_block < rsv->_rsv_start) + || (grp_goal + group_first_block > rsv->_rsv_end))) + return 0; + return 1; +} + +/** + * search_reserve_window() + * @root: root of reservation tree + * @goal: target allocation block + * + * Find the reserved window which includes the goal, or the previous one + * if the goal is not in any window. + * Returns NULL if there are no windows or if all windows start after the goal. + */ +static struct ext2_reserve_window_node * +search_reserve_window(struct rb_root *root, ext2_fsblk_t goal) +{ + struct rb_node *n = root->rb_node; + struct ext2_reserve_window_node *rsv; + + if (!n) + return NULL; + + do { + rsv = rb_entry(n, struct ext2_reserve_window_node, rsv_node); + + if (goal < rsv->rsv_start) + n = n->rb_left; + else if (goal > rsv->rsv_end) + n = n->rb_right; + else + return rsv; + } while (n); + /* + * We've fallen off the end of the tree: the goal wasn't inside + * any particular node. OK, the previous node must be to one + * side of the interval containing the goal. If it's the RHS, + * we need to back up one. + */ + if (rsv->rsv_start > goal) { + n = rb_prev(&rsv->rsv_node); + rsv = rb_entry(n, struct ext2_reserve_window_node, rsv_node); + } + return rsv; +} + +/* + * ext2_rsv_window_add() -- Insert a window to the block reservation rb tree. + * @sb: super block + * @rsv: reservation window to add + * + * Must be called with rsv_lock held. + */ +void ext2_rsv_window_add(struct super_block *sb, + struct ext2_reserve_window_node *rsv) +{ + struct rb_root *root = &EXT2_SB(sb)->s_rsv_window_root; + struct rb_node *node = &rsv->rsv_node; + ext2_fsblk_t start = rsv->rsv_start; + + struct rb_node ** p = &root->rb_node; + struct rb_node * parent = NULL; + struct ext2_reserve_window_node *this; + + while (*p) + { + parent = *p; + this = rb_entry(parent, struct ext2_reserve_window_node, rsv_node); + + if (start < this->rsv_start) + p = &(*p)->rb_left; + else if (start > this->rsv_end) + p = &(*p)->rb_right; + else { + rsv_window_dump(root, 1); + BUG(); + } + } + + rb_link_node(node, parent, p); + rb_insert_color(node, root); +} + +/** + * rsv_window_remove() -- unlink a window from the reservation rb tree + * @sb: super block + * @rsv: reservation window to remove + * + * Mark the block reservation window as not allocated, and unlink it + * from the filesystem reservation window rb tree. Must be called with + * rsv_lock held. + */ +static void rsv_window_remove(struct super_block *sb, + struct ext2_reserve_window_node *rsv) +{ + rsv->rsv_start = EXT2_RESERVE_WINDOW_NOT_ALLOCATED; + rsv->rsv_end = EXT2_RESERVE_WINDOW_NOT_ALLOCATED; + rsv->rsv_alloc_hit = 0; + rb_erase(&rsv->rsv_node, &EXT2_SB(sb)->s_rsv_window_root); +} + +/* + * rsv_is_empty() -- Check if the reservation window is allocated. + * @rsv: given reservation window to check + * + * returns 1 if the end block is EXT2_RESERVE_WINDOW_NOT_ALLOCATED. + */ +static inline int rsv_is_empty(struct ext2_reserve_window *rsv) +{ + /* a valid reservation end block could not be 0 */ + return (rsv->_rsv_end == EXT2_RESERVE_WINDOW_NOT_ALLOCATED); +} + +/** + * ext2_init_block_alloc_info() + * @inode: file inode structure + * + * Allocate and initialize the reservation window structure, and + * link the window to the ext2 inode structure at last + * + * The reservation window structure is only dynamically allocated + * and linked to ext2 inode the first time the open file + * needs a new block. So, before every ext2_new_block(s) call, for + * regular files, we should check whether the reservation window + * structure exists or not. In the latter case, this function is called. + * Fail to do so will result in block reservation being turned off for that + * open file. + * + * This function is called from ext2_get_blocks_handle(), also called + * when setting the reservation window size through ioctl before the file + * is open for write (needs block allocation). + * + * Needs truncate_mutex protection prior to calling this function. + */ +void ext2_init_block_alloc_info(struct inode *inode) +{ + struct ext2_inode_info *ei = EXT2_I(inode); + struct ext2_block_alloc_info *block_i; + struct super_block *sb = inode->i_sb; + + block_i = kmalloc_obj(*block_i); + if (block_i) { + struct ext2_reserve_window_node *rsv = &block_i->rsv_window_node; + + rsv->rsv_start = EXT2_RESERVE_WINDOW_NOT_ALLOCATED; + rsv->rsv_end = EXT2_RESERVE_WINDOW_NOT_ALLOCATED; + + /* + * if filesystem is mounted with NORESERVATION, the goal + * reservation window size is set to zero to indicate + * block reservation is off + */ + if (!test_opt(sb, RESERVATION)) + rsv->rsv_goal_size = 0; + else + rsv->rsv_goal_size = EXT2_DEFAULT_RESERVE_BLOCKS; + rsv->rsv_alloc_hit = 0; + block_i->last_alloc_logical_block = 0; + block_i->last_alloc_physical_block = 0; + } + ei->i_block_alloc_info = block_i; +} + +/** + * ext2_discard_reservation() + * @inode: inode + * + * Discard(free) block reservation window on last file close, or truncate + * or at last iput(). + * + * It is being called in three cases: + * ext2_release_file(): last writer closes the file + * ext2_clear_inode(): last iput(), when nobody links to this file. + * ext2_truncate(): when the block indirect map is about to change. + */ +void ext2_discard_reservation(struct inode *inode) +{ + struct ext2_inode_info *ei = EXT2_I(inode); + struct ext2_block_alloc_info *block_i = ei->i_block_alloc_info; + struct ext2_reserve_window_node *rsv; + spinlock_t *rsv_lock = &EXT2_SB(inode->i_sb)->s_rsv_window_lock; + + if (!block_i) + return; + + rsv = &block_i->rsv_window_node; + if (!rsv_is_empty(&rsv->rsv_window)) { + spin_lock(rsv_lock); + if (!rsv_is_empty(&rsv->rsv_window)) + rsv_window_remove(inode->i_sb, rsv); + spin_unlock(rsv_lock); + } +} + +/** + * ext2_free_blocks() -- Free given blocks and update quota and i_blocks + * @inode: inode + * @block: start physical block to free + * @count: number of blocks to free + */ +void ext2_free_blocks(struct inode * inode, ext2_fsblk_t block, + unsigned long count) +{ + struct buffer_head *bitmap_bh = NULL; + struct buffer_head * bh2; + unsigned long block_group; + unsigned long bit; + unsigned long i; + unsigned long overflow; + struct super_block * sb = inode->i_sb; + struct ext2_sb_info * sbi = EXT2_SB(sb); + struct ext2_group_desc * desc; + struct ext2_super_block * es = sbi->s_es; + unsigned freed = 0, group_freed; + + if (!ext2_data_block_valid(sbi, block, count)) { + ext2_error (sb, "ext2_free_blocks", + "Freeing blocks not in datazone - " + "block = %lu, count = %lu", block, count); + goto error_return; + } + + ext2_debug ("freeing block(s) %lu-%lu\n", block, block + count - 1); + +do_more: + overflow = 0; + block_group = (block - le32_to_cpu(es->s_first_data_block)) / + EXT2_BLOCKS_PER_GROUP(sb); + bit = (block - le32_to_cpu(es->s_first_data_block)) % + EXT2_BLOCKS_PER_GROUP(sb); + /* + * Check to see if we are freeing blocks across a group + * boundary. + */ + if (bit + count > EXT2_BLOCKS_PER_GROUP(sb)) { + overflow = bit + count - EXT2_BLOCKS_PER_GROUP(sb); + count -= overflow; + } + brelse(bitmap_bh); + bitmap_bh = read_block_bitmap(sb, block_group); + if (!bitmap_bh) + goto error_return; + + desc = ext2_get_group_desc (sb, block_group, &bh2); + if (!desc) + goto error_return; + + if (in_range (le32_to_cpu(desc->bg_block_bitmap), block, count) || + in_range (le32_to_cpu(desc->bg_inode_bitmap), block, count) || + in_range (block, le32_to_cpu(desc->bg_inode_table), + sbi->s_itb_per_group) || + in_range (block + count - 1, le32_to_cpu(desc->bg_inode_table), + sbi->s_itb_per_group)) { + ext2_error (sb, "ext2_free_blocks", + "Freeing blocks in system zones - " + "Block = %lu, count = %lu", + block, count); + goto error_return; + } + + for (i = 0, group_freed = 0; i < count; i++) { + if (!ext2_clear_bit_atomic(sb_bgl_lock(sbi, block_group), + bit + i, bitmap_bh->b_data)) { + ext2_error(sb, __func__, + "bit already cleared for block %lu", block + i); + } else { + group_freed++; + } + } + + mark_buffer_dirty(bitmap_bh); + if (sb->s_flags & SB_SYNCHRONOUS) + sync_dirty_buffer(bitmap_bh); + + group_adjust_blocks(sb, block_group, desc, bh2, group_freed); + freed += group_freed; + + if (overflow) { + block += count; + count = overflow; + goto do_more; + } +error_return: + brelse(bitmap_bh); + if (freed) { + percpu_counter_add(&sbi->s_freeblocks_counter, freed); + dquot_free_block_nodirty(inode, freed); + mark_inode_dirty(inode); + } +} + +/** + * bitmap_search_next_usable_block() + * @start: the starting block (group relative) of the search + * @bh: bufferhead contains the block group bitmap + * @maxblocks: the ending block (group relative) of the reservation + * + * The bitmap search --- search forward through the actual bitmap on disk until + * we find a bit free. + */ +static ext2_grpblk_t +bitmap_search_next_usable_block(ext2_grpblk_t start, struct buffer_head *bh, + ext2_grpblk_t maxblocks) +{ + ext2_grpblk_t next; + + next = ext2_find_next_zero_bit(bh->b_data, maxblocks, start); + if (next >= maxblocks) + return -1; + return next; +} + +/** + * find_next_usable_block() + * @start: the starting block (group relative) to find next + * allocatable block in bitmap. + * @bh: bufferhead contains the block group bitmap + * @maxblocks: the ending block (group relative) for the search + * + * Find an allocatable block in a bitmap. We perform the "most + * appropriate allocation" algorithm of looking for a free block near + * the initial goal; then for a free byte somewhere in the bitmap; + * then for any free bit in the bitmap. + */ +static ext2_grpblk_t +find_next_usable_block(int start, struct buffer_head *bh, int maxblocks) +{ + ext2_grpblk_t here, next; + char *p, *r; + + if (start > 0) { + /* + * The goal was occupied; search forward for a free + * block within the next XX blocks. + * + * end_goal is more or less random, but it has to be + * less than EXT2_BLOCKS_PER_GROUP. Aligning up to the + * next 64-bit boundary is simple.. + */ + ext2_grpblk_t end_goal = (start + 63) & ~63; + if (end_goal > maxblocks) + end_goal = maxblocks; + here = ext2_find_next_zero_bit(bh->b_data, end_goal, start); + if (here < end_goal) + return here; + ext2_debug("Bit not found near goal\n"); + } + + here = start; + if (here < 0) + here = 0; + + p = ((char *)bh->b_data) + (here >> 3); + r = memscan(p, 0, ((maxblocks + 7) >> 3) - (here >> 3)); + next = (r - ((char *)bh->b_data)) << 3; + + if (next < maxblocks && next >= here) + return next; + + here = bitmap_search_next_usable_block(here, bh, maxblocks); + return here; +} + +/** + * ext2_try_to_allocate() + * @sb: superblock + * @group: given allocation block group + * @bitmap_bh: bufferhead holds the block bitmap + * @grp_goal: given target block within the group + * @count: target number of blocks to allocate + * @my_rsv: reservation window + * + * Attempt to allocate blocks within a give range. Set the range of allocation + * first, then find the first free bit(s) from the bitmap (within the range), + * and at last, allocate the blocks by claiming the found free bit as allocated. + * + * To set the range of this allocation: + * if there is a reservation window, only try to allocate block(s) + * from the file's own reservation window; + * Otherwise, the allocation range starts from the give goal block, + * ends at the block group's last block. + * + * If we failed to allocate the desired block then we may end up crossing to a + * new bitmap. + */ +static int +ext2_try_to_allocate(struct super_block *sb, int group, + struct buffer_head *bitmap_bh, ext2_grpblk_t grp_goal, + unsigned long *count, + struct ext2_reserve_window *my_rsv) +{ + ext2_fsblk_t group_first_block = ext2_group_first_block_no(sb, group); + ext2_fsblk_t group_last_block = ext2_group_last_block_no(sb, group); + ext2_grpblk_t start, end; + unsigned long num = 0; + + start = 0; + end = group_last_block - group_first_block + 1; + /* we do allocation within the reservation window if we have a window */ + if (my_rsv) { + if (my_rsv->_rsv_start >= group_first_block) + start = my_rsv->_rsv_start - group_first_block; + if (my_rsv->_rsv_end < group_last_block) + end = my_rsv->_rsv_end - group_first_block + 1; + if (grp_goal < start || grp_goal >= end) + grp_goal = -1; + } + BUG_ON(start > EXT2_BLOCKS_PER_GROUP(sb)); + + if (grp_goal < 0) { + grp_goal = find_next_usable_block(start, bitmap_bh, end); + if (grp_goal < 0) + goto fail_access; + if (!my_rsv) { + int i; + + for (i = 0; i < 7 && grp_goal > start && + !ext2_test_bit(grp_goal - 1, + bitmap_bh->b_data); + i++, grp_goal--) + ; + } + } + + for (; num < *count && grp_goal < end; grp_goal++) { + if (ext2_set_bit_atomic(sb_bgl_lock(EXT2_SB(sb), group), + grp_goal, bitmap_bh->b_data)) { + if (num == 0) + continue; + break; + } + num++; + } + + if (num == 0) + goto fail_access; + + *count = num; + return grp_goal - num; +fail_access: + return -1; +} + +/** + * find_next_reservable_window - Find a reservable space within the given range. + * @search_head: The list to search. + * @my_rsv: The reservation we're currently using. + * @sb: The super block. + * @start_block: The first block we consider to start the real search from + * @last_block: The maximum block number that our goal reservable space + * could start from. + * + * It does not allocate the reservation window: alloc_new_reservation() + * will do the work later. + * + * We search the given range, rather than the whole reservation double + * linked list, (start_block, last_block) to find a free region that is + * of my size and has not been reserved. + * + * @search_head is not necessarily the list head of the whole filesystem. + * We have both head and @start_block to assist the search for the + * reservable space. The list starts from head, but we will shift to + * the place where start_block is, then start from there, when looking + * for a reservable space. + * + * @last_block is normally the last block in this group. The search will end + * when we found the start of next possible reservable space is out + * of this boundary. This could handle the cross boundary reservation + * window request. + * + * Return: -1 if we could not find a range of sufficient size. If we could, + * return 0 and fill in @my_rsv with the range information. + */ +static int find_next_reservable_window( + struct ext2_reserve_window_node *search_head, + struct ext2_reserve_window_node *my_rsv, + struct super_block * sb, + ext2_fsblk_t start_block, + ext2_fsblk_t last_block) +{ + struct rb_node *next; + struct ext2_reserve_window_node *rsv, *prev; + ext2_fsblk_t cur; + int size = my_rsv->rsv_goal_size; + + /* TODO: make the start of the reservation window byte-aligned */ + /* cur = *start_block & ~7;*/ + cur = start_block; + rsv = search_head; + if (!rsv) + return -1; + + while (1) { + if (cur <= rsv->rsv_end) + cur = rsv->rsv_end + 1; + + /* TODO? + * in the case we could not find a reservable space + * that is what is expected, during the re-search, we could + * remember what's the largest reservable space we could have + * and return that one. + * + * For now it will fail if we could not find the reservable + * space with expected-size (or more)... + */ + if (cur > last_block) + return -1; /* fail */ + + prev = rsv; + next = rb_next(&rsv->rsv_node); + rsv = rb_entry(next,struct ext2_reserve_window_node,rsv_node); + + /* + * Reached the last reservation, we can just append to the + * previous one. + */ + if (!next) + break; + + if (cur + size <= rsv->rsv_start) { + /* + * Found a reserveable space big enough. We could + * have a reservation across the group boundary here + */ + break; + } + } + /* + * we come here either : + * when we reach the end of the whole list, + * and there is empty reservable space after last entry in the list. + * append it to the end of the list. + * + * or we found one reservable space in the middle of the list, + * return the reservation window that we could append to. + * succeed. + */ + + if ((prev != my_rsv) && (!rsv_is_empty(&my_rsv->rsv_window))) + rsv_window_remove(sb, my_rsv); + + /* + * Let's book the whole available window for now. We will check the + * disk bitmap later and then, if there are free blocks then we adjust + * the window size if it's larger than requested. + * Otherwise, we will remove this node from the tree next time + * call find_next_reservable_window. + */ + my_rsv->rsv_start = cur; + my_rsv->rsv_end = cur + size - 1; + my_rsv->rsv_alloc_hit = 0; + + if (prev != my_rsv) + ext2_rsv_window_add(sb, my_rsv); + + return 0; +} + +/** + * alloc_new_reservation - Allocate a new reservation window. + * @my_rsv: The reservation we're currently using. + * @grp_goal: The goal block relative to the start of the group. + * @sb: The super block. + * @group: The group we are trying to allocate in. + * @bitmap_bh: The block group block bitmap. + * + * To make a new reservation, we search part of the filesystem reservation + * list (the list inside the group). We try to allocate a new + * reservation window near @grp_goal, or the beginning of the + * group, if @grp_goal is negative. + * + * We first find a reservable space after the goal, then from there, + * we check the bitmap for the first free block after it. If there is + * no free block until the end of group, then the whole group is full, + * we failed. Otherwise, check if the free block is inside the expected + * reservable space, if so, we succeed. + * + * If the first free block is outside the reservable space, then start + * from the first free block, we search for next available space, and + * go on. + * + * on succeed, a new reservation will be found and inserted into the + * list. It contains at least one free block, and it does not overlap + * with other reservation windows. + * + * Return: 0 on success, -1 if we failed to find a reservation window + * in this group + */ +static int alloc_new_reservation(struct ext2_reserve_window_node *my_rsv, + ext2_grpblk_t grp_goal, struct super_block *sb, + unsigned int group, struct buffer_head *bitmap_bh) +{ + struct ext2_reserve_window_node *search_head; + ext2_fsblk_t group_first_block, group_end_block, start_block; + ext2_grpblk_t first_free_block; + struct rb_root *fs_rsv_root = &EXT2_SB(sb)->s_rsv_window_root; + unsigned long size; + int ret; + spinlock_t *rsv_lock = &EXT2_SB(sb)->s_rsv_window_lock; + + group_first_block = ext2_group_first_block_no(sb, group); + group_end_block = ext2_group_last_block_no(sb, group); + + if (grp_goal < 0) + start_block = group_first_block; + else + start_block = grp_goal + group_first_block; + + size = my_rsv->rsv_goal_size; + + if (!rsv_is_empty(&my_rsv->rsv_window)) { + /* + * if the old reservation is cross group boundary + * and if the goal is inside the old reservation window, + * we will come here when we just failed to allocate from + * the first part of the window. We still have another part + * that belongs to the next group. In this case, there is no + * point to discard our window and try to allocate a new one + * in this group(which will fail). we should + * keep the reservation window, just simply move on. + * + * Maybe we could shift the start block of the reservation + * window to the first block of next group. + */ + + if ((my_rsv->rsv_start <= group_end_block) && + (my_rsv->rsv_end > group_end_block) && + (start_block >= my_rsv->rsv_start)) + return -1; + + if ((my_rsv->rsv_alloc_hit > + (my_rsv->rsv_end - my_rsv->rsv_start + 1) / 2)) { + /* + * if the previously allocation hit ratio is + * greater than 1/2, then we double the size of + * the reservation window the next time, + * otherwise we keep the same size window + */ + size = size * 2; + if (size > EXT2_MAX_RESERVE_BLOCKS) + size = EXT2_MAX_RESERVE_BLOCKS; + my_rsv->rsv_goal_size= size; + } + } + + spin_lock(rsv_lock); + /* + * shift the search start to the window near the goal block + */ + search_head = search_reserve_window(fs_rsv_root, start_block); + + /* + * find_next_reservable_window() simply finds a reservable window + * inside the given range(start_block, group_end_block). + * + * To make sure the reservation window has a free bit inside it, we + * need to check the bitmap after we found a reservable window. + */ +retry: + ret = find_next_reservable_window(search_head, my_rsv, sb, + start_block, group_end_block); + + if (ret == -1) { + if (!rsv_is_empty(&my_rsv->rsv_window)) + rsv_window_remove(sb, my_rsv); + spin_unlock(rsv_lock); + return -1; + } + + /* + * On success, find_next_reservable_window() returns the + * reservation window where there is a reservable space after it. + * Before we reserve this reservable space, we need + * to make sure there is at least a free block inside this region. + * + * Search the first free bit on the block bitmap. Search starts from + * the start block of the reservable space we just found. + */ + spin_unlock(rsv_lock); + first_free_block = bitmap_search_next_usable_block( + my_rsv->rsv_start - group_first_block, + bitmap_bh, group_end_block - group_first_block + 1); + + if (first_free_block < 0) { + /* + * no free block left on the bitmap, no point + * to reserve the space. return failed. + */ + spin_lock(rsv_lock); + if (!rsv_is_empty(&my_rsv->rsv_window)) + rsv_window_remove(sb, my_rsv); + spin_unlock(rsv_lock); + return -1; /* failed */ + } + + start_block = first_free_block + group_first_block; + /* + * check if the first free block is within the + * free space we just reserved + */ + if (start_block >= my_rsv->rsv_start && start_block <= my_rsv->rsv_end) + return 0; /* success */ + /* + * if the first free bit we found is out of the reservable space + * continue search for next reservable space, + * start from where the free block is, + * we also shift the list head to where we stopped last time + */ + search_head = my_rsv; + spin_lock(rsv_lock); + goto retry; +} + +/** + * try_to_extend_reservation() + * @my_rsv: given reservation window + * @sb: super block + * @size: the delta to extend + * + * Attempt to expand the reservation window large enough to have + * required number of free blocks + * + * Since ext2_try_to_allocate() will always allocate blocks within + * the reservation window range, if the window size is too small, + * multiple blocks allocation has to stop at the end of the reservation + * window. To make this more efficient, given the total number of + * blocks needed and the current size of the window, we try to + * expand the reservation window size if necessary on a best-effort + * basis before ext2_new_blocks() tries to allocate blocks. + */ +static void try_to_extend_reservation(struct ext2_reserve_window_node *my_rsv, + struct super_block *sb, int size) +{ + struct ext2_reserve_window_node *next_rsv; + struct rb_node *next; + spinlock_t *rsv_lock = &EXT2_SB(sb)->s_rsv_window_lock; + + if (!spin_trylock(rsv_lock)) + return; + + next = rb_next(&my_rsv->rsv_node); + + if (!next) + my_rsv->rsv_end += size; + else { + next_rsv = rb_entry(next, struct ext2_reserve_window_node, rsv_node); + + if ((next_rsv->rsv_start - my_rsv->rsv_end - 1) >= size) + my_rsv->rsv_end += size; + else + my_rsv->rsv_end = next_rsv->rsv_start - 1; + } + spin_unlock(rsv_lock); +} + +/** + * ext2_try_to_allocate_with_rsv() + * @sb: superblock + * @group: given allocation block group + * @bitmap_bh: bufferhead holds the block bitmap + * @grp_goal: given target block within the group + * @count: target number of blocks to allocate + * @my_rsv: reservation window + * + * This is the main function used to allocate a new block and its reservation + * window. + * + * Each time when a new block allocation is need, first try to allocate from + * its own reservation. If it does not have a reservation window, instead of + * looking for a free bit on bitmap first, then look up the reservation list to + * see if it is inside somebody else's reservation window, we try to allocate a + * reservation window for it starting from the goal first. Then do the block + * allocation within the reservation window. + * + * This will avoid keeping on searching the reservation list again and + * again when somebody is looking for a free block (without + * reservation), and there are lots of free blocks, but they are all + * being reserved. + * + * We use a red-black tree for the per-filesystem reservation list. + */ +static ext2_grpblk_t +ext2_try_to_allocate_with_rsv(struct super_block *sb, unsigned int group, + struct buffer_head *bitmap_bh, ext2_grpblk_t grp_goal, + struct ext2_reserve_window_node * my_rsv, + unsigned long *count) +{ + ext2_fsblk_t group_first_block, group_last_block; + ext2_grpblk_t ret = 0; + unsigned long num = *count; + + /* + * we don't deal with reservation when + * filesystem is mounted without reservation + * or the file is not a regular file + * or last attempt to allocate a block with reservation turned on failed + */ + if (my_rsv == NULL) { + return ext2_try_to_allocate(sb, group, bitmap_bh, + grp_goal, count, NULL); + } + /* + * grp_goal is a group relative block number (if there is a goal) + * 0 <= grp_goal < EXT2_BLOCKS_PER_GROUP(sb) + * first block is a filesystem wide block number + * first block is the block number of the first block in this group + */ + group_first_block = ext2_group_first_block_no(sb, group); + group_last_block = ext2_group_last_block_no(sb, group); + + /* + * Basically we will allocate a new block from inode's reservation + * window. + * + * We need to allocate a new reservation window, if: + * a) inode does not have a reservation window; or + * b) last attempt to allocate a block from existing reservation + * failed; or + * c) we come here with a goal and with a reservation window + * + * We do not need to allocate a new reservation window if we come here + * at the beginning with a goal and the goal is inside the window, or + * we don't have a goal but already have a reservation window. + * then we could go to allocate from the reservation window directly. + */ + while (1) { + if (rsv_is_empty(&my_rsv->rsv_window) || (ret < 0) || + !goal_in_my_reservation(&my_rsv->rsv_window, + grp_goal, group, sb)) { + if (my_rsv->rsv_goal_size < *count) + my_rsv->rsv_goal_size = *count; + ret = alloc_new_reservation(my_rsv, grp_goal, sb, + group, bitmap_bh); + if (ret < 0) + break; /* failed */ + + if (!goal_in_my_reservation(&my_rsv->rsv_window, + grp_goal, group, sb)) + grp_goal = -1; + } else if (grp_goal >= 0) { + int curr = my_rsv->rsv_end - + (grp_goal + group_first_block) + 1; + + if (curr < *count) + try_to_extend_reservation(my_rsv, sb, + *count - curr); + } + + if ((my_rsv->rsv_start > group_last_block) || + (my_rsv->rsv_end < group_first_block)) { + ext2_error(sb, __func__, + "Reservation out of group %u range goal %d fsb[%lu,%lu] rsv[%lu, %lu]", + group, grp_goal, group_first_block, + group_last_block, my_rsv->rsv_start, + my_rsv->rsv_end); + rsv_window_dump(&EXT2_SB(sb)->s_rsv_window_root, 1); + return -1; + } + ret = ext2_try_to_allocate(sb, group, bitmap_bh, grp_goal, + &num, &my_rsv->rsv_window); + if (ret >= 0) { + my_rsv->rsv_alloc_hit += num; + *count = num; + break; /* succeed */ + } + num = *count; + } + return ret; +} + +/** + * ext2_has_free_blocks() + * @sbi: in-core super block structure. + * + * Check if filesystem has at least 1 free block available for allocation. + */ +static int ext2_has_free_blocks(struct ext2_sb_info *sbi) +{ + ext2_fsblk_t free_blocks, root_blocks; + + free_blocks = percpu_counter_read_positive(&sbi->s_freeblocks_counter); + root_blocks = le32_to_cpu(sbi->s_es->s_r_blocks_count); + if (free_blocks < root_blocks + 1 && !capable(CAP_SYS_RESOURCE) && + !uid_eq(sbi->s_resuid, current_fsuid()) && + (gid_eq(sbi->s_resgid, GLOBAL_ROOT_GID) || + !in_group_p (sbi->s_resgid))) { + return 0; + } + return 1; +} + +/* + * Returns 1 if the passed-in block region is valid; 0 if some part overlaps + * with filesystem metadata blocks. + */ +int ext2_data_block_valid(struct ext2_sb_info *sbi, ext2_fsblk_t start_blk, + unsigned int count) +{ + if ((start_blk <= le32_to_cpu(sbi->s_es->s_first_data_block)) || + (start_blk + count - 1 < start_blk) || + (start_blk + count - 1 >= le32_to_cpu(sbi->s_es->s_blocks_count))) + return 0; + + /* Ensure we do not step over superblock */ + if ((start_blk <= sbi->s_sb_block) && + (start_blk + count - 1 >= sbi->s_sb_block)) + return 0; + + return 1; +} + +/* + * ext2_new_blocks() -- core block(s) allocation function + * @inode: file inode + * @goal: given target block(filesystem wide) + * @count: target number of blocks to allocate + * @errp: error code + * @flags: allocate flags + * + * ext2_new_blocks uses a goal block to assist allocation. If the goal is + * free, or there is a free block within 32 blocks of the goal, that block + * is allocated. Otherwise a forward search is made for a free block; within + * each block group the search first looks for an entire free byte in the block + * bitmap, and then for any free bit if that fails. + * This function also updates quota and i_blocks field. + */ +ext2_fsblk_t ext2_new_blocks(struct inode *inode, ext2_fsblk_t goal, + unsigned long *count, int *errp, unsigned int flags) +{ + struct buffer_head *bitmap_bh = NULL; + struct buffer_head *gdp_bh; + int group_no; + int goal_group; + ext2_grpblk_t grp_target_blk; /* blockgroup relative goal block */ + ext2_grpblk_t grp_alloc_blk; /* blockgroup-relative allocated block*/ + ext2_fsblk_t ret_block; /* filesyetem-wide allocated block */ + int bgi; /* blockgroup iteration index */ + int performed_allocation = 0; + ext2_grpblk_t free_blocks; /* number of free blocks in a group */ + struct super_block *sb; + struct ext2_group_desc *gdp; + struct ext2_super_block *es; + struct ext2_sb_info *sbi; + struct ext2_reserve_window_node *my_rsv = NULL; + struct ext2_block_alloc_info *block_i; + unsigned short windowsz = 0; + unsigned long ngroups; + unsigned long num = *count; + int ret; + + *errp = -ENOSPC; + sb = inode->i_sb; + + /* + * Check quota for allocation of this block. + */ + ret = dquot_alloc_block(inode, num); + if (ret) { + *errp = ret; + return 0; + } + + sbi = EXT2_SB(sb); + es = EXT2_SB(sb)->s_es; + ext2_debug("goal=%lu.\n", goal); + /* + * Allocate a block from reservation only when the filesystem is + * mounted with reservation(default,-o reservation), and it's a regular + * file, and the desired window size is greater than 0 (One could use + * ioctl command EXT2_IOC_SETRSVSZ to set the window size to 0 to turn + * off reservation on that particular file). Also do not use the + * reservation window if the caller asked us not to do it. + */ + block_i = EXT2_I(inode)->i_block_alloc_info; + if (!(flags & EXT2_ALLOC_NORESERVE) && block_i) { + windowsz = block_i->rsv_window_node.rsv_goal_size; + if (windowsz > 0) + my_rsv = &block_i->rsv_window_node; + } + + if (!ext2_has_free_blocks(sbi)) { + *errp = -ENOSPC; + goto out; + } + + /* + * First, test whether the goal block is free. + */ + if (goal < le32_to_cpu(es->s_first_data_block) || + goal >= le32_to_cpu(es->s_blocks_count)) + goal = le32_to_cpu(es->s_first_data_block); + group_no = (goal - le32_to_cpu(es->s_first_data_block)) / + EXT2_BLOCKS_PER_GROUP(sb); + goal_group = group_no; +retry_alloc: + gdp = ext2_get_group_desc(sb, group_no, &gdp_bh); + if (!gdp) + goto io_error; + + free_blocks = le16_to_cpu(gdp->bg_free_blocks_count); + /* + * if there is not enough free blocks to make a new resevation + * turn off reservation for this allocation + */ + if (my_rsv && (free_blocks < windowsz) + && (free_blocks > 0) + && (rsv_is_empty(&my_rsv->rsv_window))) + my_rsv = NULL; + + if (free_blocks > 0) { + grp_target_blk = ((goal - le32_to_cpu(es->s_first_data_block)) % + EXT2_BLOCKS_PER_GROUP(sb)); + /* + * In case we retry allocation (due to fs reservation not + * working out or fs corruption), the bitmap_bh is non-null + * pointer and we have to release it before calling + * read_block_bitmap(). + */ + brelse(bitmap_bh); + bitmap_bh = read_block_bitmap(sb, group_no); + if (!bitmap_bh) + goto io_error; + grp_alloc_blk = ext2_try_to_allocate_with_rsv(sb, group_no, + bitmap_bh, grp_target_blk, + my_rsv, &num); + if (grp_alloc_blk >= 0) + goto allocated; + } + + ngroups = EXT2_SB(sb)->s_groups_count; + smp_rmb(); + + /* + * Now search the rest of the groups. We assume that + * group_no and gdp correctly point to the last group visited. + */ + for (bgi = 0; bgi < ngroups; bgi++) { + group_no++; + if (group_no >= ngroups) + group_no = 0; + gdp = ext2_get_group_desc(sb, group_no, &gdp_bh); + if (!gdp) + goto io_error; + + free_blocks = le16_to_cpu(gdp->bg_free_blocks_count); + /* + * skip this group (and avoid loading bitmap) if there + * are no free blocks + */ + if (!free_blocks) + continue; + /* + * skip this group if the number of + * free blocks is less than half of the reservation + * window size. + */ + if (my_rsv && (free_blocks <= (windowsz/2))) + continue; + + brelse(bitmap_bh); + bitmap_bh = read_block_bitmap(sb, group_no); + if (!bitmap_bh) + goto io_error; + /* + * try to allocate block(s) from this group, without a goal(-1). + */ + grp_alloc_blk = ext2_try_to_allocate_with_rsv(sb, group_no, + bitmap_bh, -1, my_rsv, &num); + if (grp_alloc_blk >= 0) + goto allocated; + } + /* + * We may end up a bogus earlier ENOSPC error due to + * filesystem is "full" of reservations, but + * there maybe indeed free blocks available on disk + * In this case, we just forget about the reservations + * just do block allocation as without reservations. + */ + if (my_rsv) { + my_rsv = NULL; + windowsz = 0; + group_no = goal_group; + goto retry_alloc; + } + /* No space left on the device */ + *errp = -ENOSPC; + goto out; + +allocated: + + ext2_debug("using block group %d(%d)\n", + group_no, gdp->bg_free_blocks_count); + + ret_block = grp_alloc_blk + ext2_group_first_block_no(sb, group_no); + + if (in_range(le32_to_cpu(gdp->bg_block_bitmap), ret_block, num) || + in_range(le32_to_cpu(gdp->bg_inode_bitmap), ret_block, num) || + in_range(ret_block, le32_to_cpu(gdp->bg_inode_table), + EXT2_SB(sb)->s_itb_per_group) || + in_range(ret_block + num - 1, le32_to_cpu(gdp->bg_inode_table), + EXT2_SB(sb)->s_itb_per_group)) { + ext2_error(sb, "ext2_new_blocks", + "Allocating block in system zone - " + "blocks from "E2FSBLK", length %lu", + ret_block, num); + /* + * ext2_try_to_allocate marked the blocks we allocated as in + * use. So we may want to selectively mark some of the blocks + * as free + */ + num = *count; + goto retry_alloc; + } + + performed_allocation = 1; + + if (ret_block + num - 1 >= le32_to_cpu(es->s_blocks_count)) { + ext2_error(sb, "ext2_new_blocks", + "block("E2FSBLK") >= blocks count(%d) - " + "block_group = %d, es == %p ", ret_block, + le32_to_cpu(es->s_blocks_count), group_no, es); + goto out; + } + + group_adjust_blocks(sb, group_no, gdp, gdp_bh, -num); + percpu_counter_sub(&sbi->s_freeblocks_counter, num); + + mark_buffer_dirty(bitmap_bh); + if (sb->s_flags & SB_SYNCHRONOUS) + sync_dirty_buffer(bitmap_bh); + + *errp = 0; + brelse(bitmap_bh); + if (num < *count) { + dquot_free_block_nodirty(inode, *count-num); + mark_inode_dirty(inode); + *count = num; + } + return ret_block; + +io_error: + *errp = -EIO; +out: + /* + * Undo the block allocation + */ + if (!performed_allocation) { + dquot_free_block_nodirty(inode, *count); + mark_inode_dirty(inode); + } + brelse(bitmap_bh); + return 0; +} + +#ifdef EXT2FS_DEBUG + +unsigned long ext2_count_free(struct buffer_head *map, unsigned int numchars) +{ + return numchars * BITS_PER_BYTE - memweight(map->b_data, numchars); +} + +#endif /* EXT2FS_DEBUG */ + +unsigned long ext2_count_free_blocks (struct super_block * sb) +{ + struct ext2_group_desc * desc; + unsigned long desc_count = 0; + int i; +#ifdef EXT2FS_DEBUG + unsigned long bitmap_count, x; + struct ext2_super_block *es; + + es = EXT2_SB(sb)->s_es; + desc_count = 0; + bitmap_count = 0; + desc = NULL; + for (i = 0; i < EXT2_SB(sb)->s_groups_count; i++) { + struct buffer_head *bitmap_bh; + desc = ext2_get_group_desc (sb, i, NULL); + if (!desc) + continue; + desc_count += le16_to_cpu(desc->bg_free_blocks_count); + bitmap_bh = read_block_bitmap(sb, i); + if (!bitmap_bh) + continue; + + x = ext2_count_free(bitmap_bh, sb->s_blocksize); + printk ("group %d: stored = %d, counted = %lu\n", + i, le16_to_cpu(desc->bg_free_blocks_count), x); + bitmap_count += x; + brelse(bitmap_bh); + } + printk("ext2_count_free_blocks: stored = %lu, computed = %lu, %lu\n", + (long)le32_to_cpu(es->s_free_blocks_count), + desc_count, bitmap_count); + return bitmap_count; +#else + for (i = 0; i < EXT2_SB(sb)->s_groups_count; i++) { + desc = ext2_get_group_desc(sb, i, NULL); + if (!desc) + continue; + desc_count += le16_to_cpu(desc->bg_free_blocks_count); + } + return desc_count; +#endif +} + +static inline int test_root(int a, int b) +{ + int num = b; + + while (a > num) + num *= b; + return num == a; +} + +static int ext2_group_sparse(int group) +{ + if (group <= 1) + return 1; + return (test_root(group, 3) || test_root(group, 5) || + test_root(group, 7)); +} + +/** + * ext2_bg_has_super - number of blocks used by the superblock in group + * @sb: superblock for filesystem + * @group: group number to check + * + * Return the number of blocks used by the superblock (primary or backup) + * in this group. Currently this will be only 0 or 1. + */ +int ext2_bg_has_super(struct super_block *sb, int group) +{ + if (EXT2_HAS_RO_COMPAT_FEATURE(sb,EXT2_FEATURE_RO_COMPAT_SPARSE_SUPER)&& + !ext2_group_sparse(group)) + return 0; + return 1; +} + +/** + * ext2_bg_num_gdb - number of blocks used by the group table in group + * @sb: superblock for filesystem + * @group: group number to check + * + * Return the number of blocks used by the group descriptor table + * (primary or backup) in this group. In the future there may be a + * different number of descriptor blocks in each group. + */ +unsigned long ext2_bg_num_gdb(struct super_block *sb, int group) +{ + return ext2_bg_has_super(sb, group) ? EXT2_SB(sb)->s_gdb_count : 0; +} + diff --git a/fs/ext2/dir.c b/fs/ext2/dir.c new file mode 100644 index 000000000..e17bbc759 --- /dev/null +++ b/fs/ext2/dir.c @@ -0,0 +1,739 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext2/dir.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * from + * + * linux/fs/minix/dir.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * ext2 directory handling functions + * + * Big-endian to little-endian byte-swapping/bitmaps by + * David S. Miller (davem@caip.rutgers.edu), 1995 + * + * All code that works with directory layout had been switched to pagecache + * and moved here. AV + */ + +#include "ext2.h" +#include +#include +#include +#include +#include + +typedef struct ext2_dir_entry_2 ext2_dirent; + +/* + * Tests against MAX_REC_LEN etc were put in place for 64k block + * sizes; if that is not possible on this arch, we can skip + * those tests and speed things up. + */ +static inline unsigned ext2_rec_len_from_disk(__le16 dlen) +{ + unsigned len = le16_to_cpu(dlen); + +#if (PAGE_SIZE >= 65536) + if (len == EXT2_MAX_REC_LEN) + return 1 << 16; +#endif + return len; +} + +static inline __le16 ext2_rec_len_to_disk(unsigned len) +{ +#if (PAGE_SIZE >= 65536) + if (len == (1 << 16)) + return cpu_to_le16(EXT2_MAX_REC_LEN); + else + BUG_ON(len > (1 << 16)); +#endif + return cpu_to_le16(len); +} + +/* + * ext2 uses block-sized chunks. Arguably, sector-sized ones would be + * more robust, but we have what we have + */ +static inline unsigned ext2_chunk_size(struct inode *inode) +{ + return inode->i_sb->s_blocksize; +} + +/* + * Return the offset into page `page_nr' of the last valid + * byte in that page, plus one. + */ +static unsigned +ext2_last_byte(struct inode *inode, unsigned long page_nr) +{ + unsigned last_byte = inode->i_size; + + last_byte -= page_nr << PAGE_SHIFT; + if (last_byte > PAGE_SIZE) + last_byte = PAGE_SIZE; + return last_byte; +} + +static void ext2_commit_chunk(struct folio *folio, loff_t pos, unsigned len) +{ + struct address_space *mapping = folio->mapping; + struct inode *dir = mapping->host; + + inode_inc_iversion(dir); + block_write_end(pos, len, len, folio); + + if (pos+len > dir->i_size) { + i_size_write(dir, pos+len); + mark_inode_dirty(dir); + } + folio_unlock(folio); +} + +static bool ext2_check_folio(struct folio *folio, int quiet, char *kaddr) +{ + struct inode *dir = folio->mapping->host; + struct super_block *sb = dir->i_sb; + unsigned chunk_size = ext2_chunk_size(dir); + u32 max_inumber = le32_to_cpu(EXT2_SB(sb)->s_es->s_inodes_count); + unsigned offs, rec_len; + unsigned limit = folio_size(folio); + ext2_dirent *p; + char *error; + + if (dir->i_size < folio_pos(folio) + limit) { + limit = offset_in_folio(folio, dir->i_size); + if (limit & (chunk_size - 1)) + goto Ebadsize; + if (!limit) + goto out; + } + for (offs = 0; offs <= limit - EXT2_DIR_REC_LEN(1); offs += rec_len) { + p = (ext2_dirent *)(kaddr + offs); + rec_len = ext2_rec_len_from_disk(p->rec_len); + + if (unlikely(rec_len < EXT2_DIR_REC_LEN(1))) + goto Eshort; + if (unlikely(rec_len & 3)) + goto Ealign; + if (unlikely(rec_len < EXT2_DIR_REC_LEN(p->name_len))) + goto Enamelen; + if (unlikely(((offs + rec_len - 1) ^ offs) & ~(chunk_size-1))) + goto Espan; + if (unlikely(le32_to_cpu(p->inode) > max_inumber)) + goto Einumber; + } + if (offs != limit) + goto Eend; +out: + folio_set_checked(folio); + return true; + + /* Too bad, we had an error */ + +Ebadsize: + if (!quiet) + ext2_error(sb, __func__, + "size of directory #%llu is not a multiple " + "of chunk size", dir->i_ino); + goto fail; +Eshort: + error = "rec_len is smaller than minimal"; + goto bad_entry; +Ealign: + error = "unaligned directory entry"; + goto bad_entry; +Enamelen: + error = "rec_len is too small for name_len"; + goto bad_entry; +Espan: + error = "directory entry across blocks"; + goto bad_entry; +Einumber: + error = "inode out of bounds"; +bad_entry: + if (!quiet) + ext2_error(sb, __func__, "bad entry in directory #%llu: : %s - " + "offset=%llu, inode=%lu, rec_len=%d, name_len=%d", + dir->i_ino, error, folio_pos(folio) + offs, + (unsigned long) le32_to_cpu(p->inode), + rec_len, p->name_len); + goto fail; +Eend: + if (!quiet) { + p = (ext2_dirent *)(kaddr + offs); + ext2_error(sb, "ext2_check_folio", + "entry in directory #%llu spans the page boundary" + "offset=%llu, inode=%lu", + dir->i_ino, folio_pos(folio) + offs, + (unsigned long) le32_to_cpu(p->inode)); + } +fail: + return false; +} + +/* + * Calls to ext2_get_folio()/folio_release_kmap() must be nested according + * to the rules documented in kmap_local_folio()/kunmap_local(). + * + * NOTE: ext2_find_entry() and ext2_dotdot() act as a call + * to folio_release_kmap() and should be treated as a call to + * folio_release_kmap() for nesting purposes. + */ +static void *ext2_get_folio(struct inode *dir, unsigned long n, + int quiet, struct folio **foliop) +{ + struct address_space *mapping = dir->i_mapping; + struct folio *folio = read_mapping_folio(mapping, n, NULL); + void *kaddr; + + if (IS_ERR(folio)) + return ERR_CAST(folio); + kaddr = kmap_local_folio(folio, 0); + if (unlikely(!folio_test_checked(folio))) { + if (!ext2_check_folio(folio, quiet, kaddr)) + goto fail; + } + *foliop = folio; + return kaddr; + +fail: + folio_release_kmap(folio, kaddr); + return ERR_PTR(-EIO); +} + +/* + * NOTE! unlike strncmp, ext2_match returns 1 for success, 0 for failure. + * + * len <= EXT2_NAME_LEN and de != NULL are guaranteed by caller. + */ +static inline int ext2_match (int len, const char * const name, + struct ext2_dir_entry_2 * de) +{ + if (len != de->name_len) + return 0; + if (!de->inode) + return 0; + return !memcmp(name, de->name, len); +} + +/* + * p is at least 6 bytes before the end of page + */ +static inline ext2_dirent *ext2_next_entry(ext2_dirent *p) +{ + return (ext2_dirent *)((char *)p + + ext2_rec_len_from_disk(p->rec_len)); +} + +static inline unsigned +ext2_validate_entry(char *base, unsigned offset, unsigned mask) +{ + ext2_dirent *de = (ext2_dirent*)(base + offset); + ext2_dirent *p = (ext2_dirent*)(base + (offset&mask)); + while ((char*)p < (char*)de) { + if (p->rec_len == 0) + break; + p = ext2_next_entry(p); + } + return offset_in_page(p); +} + +static inline void ext2_set_de_type(ext2_dirent *de, struct inode *inode) +{ + if (EXT2_HAS_INCOMPAT_FEATURE(inode->i_sb, EXT2_FEATURE_INCOMPAT_FILETYPE)) + de->file_type = fs_umode_to_ftype(inode->i_mode); + else + de->file_type = 0; +} + +static int +ext2_readdir(struct file *file, struct dir_context *ctx) +{ + loff_t pos = ctx->pos; + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + unsigned int offset = pos & ~PAGE_MASK; + unsigned long n = pos >> PAGE_SHIFT; + unsigned long npages = dir_pages(inode); + unsigned chunk_mask = ~(ext2_chunk_size(inode)-1); + bool need_revalidate = !inode_eq_iversion(inode, *(u64 *)file->private_data); + bool has_filetype; + + if (pos > inode->i_size - EXT2_DIR_REC_LEN(1)) + return 0; + + has_filetype = + EXT2_HAS_INCOMPAT_FEATURE(sb, EXT2_FEATURE_INCOMPAT_FILETYPE); + + for ( ; n < npages; n++, offset = 0) { + ext2_dirent *de; + struct folio *folio; + char *kaddr = ext2_get_folio(inode, n, 0, &folio); + char *limit; + + if (IS_ERR(kaddr)) { + ext2_error(sb, __func__, + "bad page in #%llu", + inode->i_ino); + ctx->pos += PAGE_SIZE - offset; + return PTR_ERR(kaddr); + } + if (unlikely(need_revalidate)) { + if (offset) { + offset = ext2_validate_entry(kaddr, offset, chunk_mask); + ctx->pos = (n<private_data = inode_query_iversion(inode); + need_revalidate = false; + } + de = (ext2_dirent *)(kaddr+offset); + limit = kaddr + ext2_last_byte(inode, n) - EXT2_DIR_REC_LEN(1); + for ( ;(char*)de <= limit; de = ext2_next_entry(de)) { + if (de->rec_len == 0) { + ext2_error(sb, __func__, + "zero-length directory entry"); + folio_release_kmap(folio, de); + return -EIO; + } + if (de->inode) { + unsigned char d_type = DT_UNKNOWN; + + if (has_filetype) + d_type = fs_ftype_to_dtype(de->file_type); + + if (!dir_emit(ctx, de->name, de->name_len, + le32_to_cpu(de->inode), + d_type)) { + folio_release_kmap(folio, de); + return 0; + } + } + ctx->pos += ext2_rec_len_from_disk(de->rec_len); + } + folio_release_kmap(folio, kaddr); + } + return 0; +} + +/* + * ext2_find_entry() + * + * finds an entry in the specified directory with the wanted name. It + * returns the page in which the entry was found (as a parameter - res_page), + * and the entry itself. Page is returned mapped and unlocked. + * Entry is guaranteed to be valid. + * + * On Success folio_release_kmap() should be called on *foliop. + * + * NOTE: Calls to ext2_get_folio()/folio_release_kmap() must be nested + * according to the rules documented in kmap_local_folio()/kunmap_local(). + * + * ext2_find_entry() and ext2_dotdot() act as a call to ext2_get_folio() + * and should be treated as a call to ext2_get_folio() for nesting + * purposes. + */ +struct ext2_dir_entry_2 *ext2_find_entry (struct inode *dir, + const struct qstr *child, struct folio **foliop) +{ + const char *name = child->name; + int namelen = child->len; + unsigned reclen = EXT2_DIR_REC_LEN(namelen); + unsigned long start, n; + unsigned long npages = dir_pages(dir); + struct ext2_inode_info *ei = EXT2_I(dir); + ext2_dirent * de; + + if (npages == 0) + goto out; + + start = ei->i_dir_start_lookup; + if (start >= npages) + start = 0; + n = start; + do { + char *kaddr = ext2_get_folio(dir, n, 0, foliop); + if (IS_ERR(kaddr)) + return ERR_CAST(kaddr); + + de = (ext2_dirent *) kaddr; + kaddr += ext2_last_byte(dir, n) - reclen; + while ((char *) de <= kaddr) { + if (de->rec_len == 0) { + ext2_error(dir->i_sb, __func__, + "zero-length directory entry"); + folio_release_kmap(*foliop, de); + goto out; + } + if (ext2_match(namelen, name, de)) + goto found; + de = ext2_next_entry(de); + } + folio_release_kmap(*foliop, kaddr); + + if (++n >= npages) + n = 0; + /* next folio is past the blocks we've got */ + if (unlikely(n > (dir->i_blocks >> (PAGE_SHIFT - 9)))) { + ext2_error(dir->i_sb, __func__, + "dir %llu size %lld exceeds block count %llu", + dir->i_ino, dir->i_size, + (unsigned long long)dir->i_blocks); + goto out; + } + } while (n != start); +out: + return ERR_PTR(-ENOENT); + +found: + ei->i_dir_start_lookup = n; + return de; +} + +/* + * Return the '..' directory entry and the page in which the entry was found + * (as a parameter - p). + * + * On Success folio_release_kmap() should be called on *foliop. + * + * NOTE: Calls to ext2_get_folio()/folio_release_kmap() must be nested + * according to the rules documented in kmap_local_folio()/kunmap_local(). + * + * ext2_find_entry() and ext2_dotdot() act as a call to ext2_get_folio() + * and should be treated as a call to ext2_get_folio() for nesting + * purposes. + */ +struct ext2_dir_entry_2 *ext2_dotdot(struct inode *dir, struct folio **foliop) +{ + ext2_dirent *de = ext2_get_folio(dir, 0, 0, foliop); + + if (!IS_ERR(de)) + return ext2_next_entry(de); + return NULL; +} + +int ext2_inode_by_name(struct inode *dir, const struct qstr *child, ino_t *ino) +{ + struct ext2_dir_entry_2 *de; + struct folio *folio; + + de = ext2_find_entry(dir, child, &folio); + if (IS_ERR(de)) + return PTR_ERR(de); + + *ino = le32_to_cpu(de->inode); + folio_release_kmap(folio, de); + return 0; +} + +static int ext2_prepare_chunk(struct folio *folio, loff_t pos, unsigned len) +{ + return __block_write_begin(folio, pos, len, ext2_get_block); +} + +static int ext2_handle_dirsync(struct inode *dir) +{ + int err; + + err = filemap_write_and_wait(dir->i_mapping); + if (!err) + err = sync_inode_metadata(dir, 1); + return err; +} + +int ext2_set_link(struct inode *dir, struct ext2_dir_entry_2 *de, + struct folio *folio, struct inode *inode, bool update_times) +{ + loff_t pos = folio_pos(folio) + offset_in_folio(folio, de); + unsigned len = ext2_rec_len_from_disk(de->rec_len); + int err; + + folio_lock(folio); + err = ext2_prepare_chunk(folio, pos, len); + if (err) { + folio_unlock(folio); + return err; + } + de->inode = cpu_to_le32(inode->i_ino); + ext2_set_de_type(de, inode); + ext2_commit_chunk(folio, pos, len); + if (update_times) + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + EXT2_I(dir)->i_flags &= ~EXT2_BTREE_FL; + mark_inode_dirty(dir); + return ext2_handle_dirsync(dir); +} + +/* + * Parent is locked. + */ +int ext2_add_link (struct dentry *dentry, struct inode *inode) +{ + struct inode *dir = d_inode(dentry->d_parent); + const char *name = dentry->d_name.name; + int namelen = dentry->d_name.len; + unsigned chunk_size = ext2_chunk_size(dir); + unsigned reclen = EXT2_DIR_REC_LEN(namelen); + unsigned short rec_len, name_len; + struct folio *folio = NULL; + ext2_dirent * de; + unsigned long npages = dir_pages(dir); + unsigned long n; + loff_t pos; + int err; + + /* + * We take care of directory expansion in the same loop. + * This code plays outside i_size, so it locks the folio + * to protect that region. + */ + for (n = 0; n <= npages; n++) { + char *kaddr = ext2_get_folio(dir, n, 0, &folio); + char *dir_end; + + if (IS_ERR(kaddr)) + return PTR_ERR(kaddr); + folio_lock(folio); + dir_end = kaddr + ext2_last_byte(dir, n); + de = (ext2_dirent *)kaddr; + kaddr += folio_size(folio) - reclen; + while ((char *)de <= kaddr) { + if ((char *)de == dir_end) { + /* We hit i_size */ + name_len = 0; + rec_len = chunk_size; + de->rec_len = ext2_rec_len_to_disk(chunk_size); + de->inode = 0; + goto got_it; + } + if (de->rec_len == 0) { + ext2_error(dir->i_sb, __func__, + "zero-length directory entry"); + err = -EIO; + goto out_unlock; + } + err = -EEXIST; + if (ext2_match (namelen, name, de)) + goto out_unlock; + name_len = EXT2_DIR_REC_LEN(de->name_len); + rec_len = ext2_rec_len_from_disk(de->rec_len); + if (!de->inode && rec_len >= reclen) + goto got_it; + if (rec_len >= name_len + reclen) + goto got_it; + de = (ext2_dirent *) ((char *) de + rec_len); + } + folio_unlock(folio); + folio_release_kmap(folio, kaddr); + } + BUG(); + return -EINVAL; + +got_it: + pos = folio_pos(folio) + offset_in_folio(folio, de); + err = ext2_prepare_chunk(folio, pos, rec_len); + if (err) + goto out_unlock; + if (de->inode) { + ext2_dirent *de1 = (ext2_dirent *) ((char *) de + name_len); + de1->rec_len = ext2_rec_len_to_disk(rec_len - name_len); + de->rec_len = ext2_rec_len_to_disk(name_len); + de = de1; + } + de->name_len = namelen; + memcpy(de->name, name, namelen); + de->inode = cpu_to_le32(inode->i_ino); + ext2_set_de_type (de, inode); + ext2_commit_chunk(folio, pos, rec_len); + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + EXT2_I(dir)->i_flags &= ~EXT2_BTREE_FL; + mark_inode_dirty(dir); + err = ext2_handle_dirsync(dir); + /* OFFSET_CACHE */ +out_put: + folio_release_kmap(folio, de); + return err; +out_unlock: + folio_unlock(folio); + goto out_put; +} + +/* + * ext2_delete_entry deletes a directory entry by merging it with the + * previous entry. Page is up-to-date. + */ +int ext2_delete_entry(struct ext2_dir_entry_2 *dir, struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + size_t from, to; + char *kaddr; + loff_t pos; + ext2_dirent *de, *pde = NULL; + int err; + + from = offset_in_folio(folio, dir); + to = from + ext2_rec_len_from_disk(dir->rec_len); + kaddr = (char *)dir - from; + from &= ~(ext2_chunk_size(inode)-1); + de = (ext2_dirent *)(kaddr + from); + + while ((char*)de < (char*)dir) { + if (de->rec_len == 0) { + ext2_error(inode->i_sb, __func__, + "zero-length directory entry"); + return -EIO; + } + pde = de; + de = ext2_next_entry(de); + } + if (pde) + from = offset_in_folio(folio, pde); + pos = folio_pos(folio) + from; + folio_lock(folio); + err = ext2_prepare_chunk(folio, pos, to - from); + if (err) { + folio_unlock(folio); + return err; + } + if (pde) + pde->rec_len = ext2_rec_len_to_disk(to - from); + dir->inode = 0; + ext2_commit_chunk(folio, pos, to - from); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + EXT2_I(inode)->i_flags &= ~EXT2_BTREE_FL; + mark_inode_dirty(inode); + return ext2_handle_dirsync(inode); +} + +/* + * Set the first fragment of directory. + */ +int ext2_make_empty(struct inode *inode, struct inode *parent) +{ + struct folio *folio = filemap_grab_folio(inode->i_mapping, 0); + unsigned chunk_size = ext2_chunk_size(inode); + struct ext2_dir_entry_2 * de; + int err; + void *kaddr; + + if (IS_ERR(folio)) + return PTR_ERR(folio); + + err = ext2_prepare_chunk(folio, 0, chunk_size); + if (err) { + folio_unlock(folio); + goto fail; + } + kaddr = kmap_local_folio(folio, 0); + memset(kaddr, 0, chunk_size); + de = (struct ext2_dir_entry_2 *)kaddr; + de->name_len = 1; + de->rec_len = ext2_rec_len_to_disk(EXT2_DIR_REC_LEN(1)); + memcpy (de->name, ".\0\0", 4); + de->inode = cpu_to_le32(inode->i_ino); + ext2_set_de_type (de, inode); + + de = (struct ext2_dir_entry_2 *)(kaddr + EXT2_DIR_REC_LEN(1)); + de->name_len = 2; + de->rec_len = ext2_rec_len_to_disk(chunk_size - EXT2_DIR_REC_LEN(1)); + de->inode = cpu_to_le32(parent->i_ino); + memcpy (de->name, "..\0", 4); + ext2_set_de_type (de, inode); + kunmap_local(kaddr); + ext2_commit_chunk(folio, 0, chunk_size); + err = ext2_handle_dirsync(inode); +fail: + folio_put(folio); + return err; +} + +/* + * routine to check that the specified directory is empty (for rmdir) + */ +int ext2_empty_dir(struct inode *inode) +{ + struct folio *folio; + char *kaddr; + unsigned long i, npages = dir_pages(inode); + + for (i = 0; i < npages; i++) { + ext2_dirent *de; + + kaddr = ext2_get_folio(inode, i, 0, &folio); + if (IS_ERR(kaddr)) + return 0; + + de = (ext2_dirent *)kaddr; + kaddr += ext2_last_byte(inode, i) - EXT2_DIR_REC_LEN(1); + + while ((char *)de <= kaddr) { + if (de->rec_len == 0) { + ext2_error(inode->i_sb, __func__, + "zero-length directory entry"); + printk("kaddr=%p, de=%p\n", kaddr, de); + goto not_empty; + } + if (de->inode != 0) { + /* check for . and .. */ + if (de->name[0] != '.') + goto not_empty; + if (de->name_len > 2) + goto not_empty; + if (de->name_len < 2) { + if (de->inode != + cpu_to_le32(inode->i_ino)) + goto not_empty; + } else if (de->name[1] != '.') + goto not_empty; + } + de = ext2_next_entry(de); + } + folio_release_kmap(folio, kaddr); + } + return 1; + +not_empty: + folio_release_kmap(folio, kaddr); + return 0; +} + +static int ext2_dir_open(struct inode *inode, struct file *file) +{ + file->private_data = kzalloc(sizeof(u64), GFP_KERNEL); + if (!file->private_data) + return -ENOMEM; + return 0; +} + +static int ext2_dir_release(struct inode *inode, struct file *file) +{ + kfree(file->private_data); + return 0; +} + +static loff_t ext2_dir_llseek(struct file *file, loff_t offset, int whence) +{ + return generic_llseek_cookie(file, offset, whence, + (u64 *)file->private_data); +} + +const struct file_operations ext2_dir_operations = { + .open = ext2_dir_open, + .release = ext2_dir_release, + .llseek = ext2_dir_llseek, + .read = generic_read_dir, + .iterate_shared = ext2_readdir, + .unlocked_ioctl = ext2_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = ext2_compat_ioctl, +#endif + .fsync = simple_fsync, + .setlease = generic_setlease, +}; diff --git a/fs/ext2/ext2.h b/fs/ext2/ext2.h new file mode 100644 index 000000000..5642451bf --- /dev/null +++ b/fs/ext2/ext2.h @@ -0,0 +1,815 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * from + * + * linux/include/linux/minix_fs.h + * + * Copyright (C) 1991, 1992 Linus Torvalds + */ +#include +#include +#include +#include +#include +#include +#include + +/* XXX Here for now... not interested in restructing headers JUST now */ + +/* data type for block offset of block group */ +typedef int ext2_grpblk_t; + +/* data type for filesystem-wide blocks number */ +typedef unsigned long ext2_fsblk_t; + +#define E2FSBLK "%lu" + +struct ext2_reserve_window { + ext2_fsblk_t _rsv_start; /* First byte reserved */ + ext2_fsblk_t _rsv_end; /* Last byte reserved or 0 */ +}; + +struct ext2_reserve_window_node { + struct rb_node rsv_node; + __u32 rsv_goal_size; + __u32 rsv_alloc_hit; + struct ext2_reserve_window rsv_window; +}; + +struct ext2_block_alloc_info { + /* information about reservation window */ + struct ext2_reserve_window_node rsv_window_node; + /* + * was i_next_alloc_block in ext2_inode_info + * is the logical (file-relative) number of the + * most-recently-allocated block in this file. + * We use this for detecting linearly ascending allocation requests. + */ + __u32 last_alloc_logical_block; + /* + * Was i_next_alloc_goal in ext2_inode_info + * is the *physical* companion to i_next_alloc_block. + * it is the physical block number of the block which was most-recently + * allocated to this file. This gives us the goal (target) for the next + * allocation when we detect linearly ascending requests. + */ + ext2_fsblk_t last_alloc_physical_block; +}; + +#define rsv_start rsv_window._rsv_start +#define rsv_end rsv_window._rsv_end + +struct mb_cache; + +/* + * second extended-fs super-block data in memory + */ +struct ext2_sb_info { + unsigned long s_inodes_per_block;/* Number of inodes per block */ + unsigned long s_blocks_per_group;/* Number of blocks in a group */ + unsigned long s_inodes_per_group;/* Number of inodes in a group */ + unsigned long s_itb_per_group; /* Number of inode table blocks per group */ + unsigned long s_gdb_count; /* Number of group descriptor blocks */ + unsigned long s_desc_per_block; /* Number of group descriptors per block */ + unsigned long s_groups_count; /* Number of groups in the fs */ + unsigned long s_overhead_last; /* Last calculated overhead */ + unsigned long s_blocks_last; /* Last seen block count */ + struct buffer_head * s_sbh; /* Buffer containing the super block */ + struct ext2_super_block * s_es; /* Pointer to the super block in the buffer */ + struct buffer_head ** s_group_desc; + unsigned long s_mount_opt; + unsigned long s_sb_block; + kuid_t s_resuid; + kgid_t s_resgid; + unsigned short s_mount_state; + unsigned short s_pad; + int s_addr_per_block_bits; + int s_desc_per_block_bits; + int s_inode_size; + int s_first_ino; + spinlock_t s_next_gen_lock; + u32 s_next_generation; + unsigned long s_dir_count; + u8 *s_debts; + struct percpu_counter s_freeblocks_counter; + struct percpu_counter s_freeinodes_counter; + struct percpu_counter s_dirs_counter; + struct blockgroup_lock *s_blockgroup_lock; + /* root of the per fs reservation window tree */ + spinlock_t s_rsv_window_lock; + struct rb_root s_rsv_window_root; + struct ext2_reserve_window_node s_rsv_window_head; + /* + * s_lock protects against concurrent modifications of s_mount_state, + * s_blocks_last, s_overhead_last and the content of superblock's + * buffer pointed to by sbi->s_es. + * + * Note: It is used in ext2_show_options() to provide a consistent view + * of the mount options. + */ + spinlock_t s_lock; + struct mb_cache *s_ea_block_cache; +}; + +static inline spinlock_t * +sb_bgl_lock(struct ext2_sb_info *sbi, unsigned int block_group) +{ + return bgl_lock_ptr(sbi->s_blockgroup_lock, block_group); +} + +/* + * Define EXT2FS_DEBUG to produce debug messages + */ +#undef EXT2FS_DEBUG + +/* + * Define EXT2_RESERVATION to reserve data blocks for expanding files + */ +#define EXT2_DEFAULT_RESERVE_BLOCKS 8 +/*max window size: 1024(direct blocks) + 3([t,d]indirect blocks) */ +#define EXT2_MAX_RESERVE_BLOCKS 1027 +#define EXT2_RESERVE_WINDOW_NOT_ALLOCATED 0 +/* + * The second extended file system version + */ +#define EXT2FS_DATE "95/08/09" +#define EXT2FS_VERSION "0.5b" + +/* + * Debug code + */ +#ifdef EXT2FS_DEBUG +# define ext2_debug(f, a...) { \ + printk ("EXT2-fs DEBUG (%s, %d): %s:", \ + __FILE__, __LINE__, __func__); \ + printk (f, ## a); \ + } +#else +# define ext2_debug(f, a...) /**/ +#endif + +/* + * Special inode numbers + */ +#define EXT2_BAD_INO 1 /* Bad blocks inode */ +#define EXT2_ROOT_INO 2 /* Root inode */ +#define EXT2_BOOT_LOADER_INO 5 /* Boot loader inode */ +#define EXT2_UNDEL_DIR_INO 6 /* Undelete directory inode */ + +/* First non-reserved inode for old ext2 filesystems */ +#define EXT2_GOOD_OLD_FIRST_INO 11 + +static inline struct ext2_sb_info *EXT2_SB(struct super_block *sb) +{ + return sb->s_fs_info; +} + +/* + * Macro-instructions used to manage several block sizes + */ +#define EXT2_MIN_BLOCK_SIZE 1024 +#define EXT2_MAX_BLOCK_SIZE 65536 +#define EXT2_MIN_BLOCK_LOG_SIZE 10 +#define EXT2_MAX_BLOCK_LOG_SIZE 16 +#define EXT2_BLOCK_SIZE(s) ((s)->s_blocksize) +#define EXT2_ADDR_PER_BLOCK(s) (EXT2_BLOCK_SIZE(s) / sizeof (__u32)) +#define EXT2_BLOCK_SIZE_BITS(s) ((s)->s_blocksize_bits) +#define EXT2_ADDR_PER_BLOCK_BITS(s) (EXT2_SB(s)->s_addr_per_block_bits) +#define EXT2_INODE_SIZE(s) (EXT2_SB(s)->s_inode_size) +#define EXT2_FIRST_INO(s) (EXT2_SB(s)->s_first_ino) + +/* + * Structure of a blocks group descriptor + */ +struct ext2_group_desc +{ + __le32 bg_block_bitmap; /* Blocks bitmap block */ + __le32 bg_inode_bitmap; /* Inodes bitmap block */ + __le32 bg_inode_table; /* Inodes table block */ + __le16 bg_free_blocks_count; /* Free blocks count */ + __le16 bg_free_inodes_count; /* Free inodes count */ + __le16 bg_used_dirs_count; /* Directories count */ + __le16 bg_pad; + __le32 bg_reserved[3]; +}; + +/* + * Macro-instructions used to manage group descriptors + */ +#define EXT2_BLOCKS_PER_GROUP(s) (EXT2_SB(s)->s_blocks_per_group) +#define EXT2_DESC_PER_BLOCK(s) (EXT2_SB(s)->s_desc_per_block) +#define EXT2_INODES_PER_GROUP(s) (EXT2_SB(s)->s_inodes_per_group) +#define EXT2_DESC_PER_BLOCK_BITS(s) (EXT2_SB(s)->s_desc_per_block_bits) + +/* + * Constants relative to the data blocks + */ +#define EXT2_NDIR_BLOCKS 12 +#define EXT2_IND_BLOCK EXT2_NDIR_BLOCKS +#define EXT2_DIND_BLOCK (EXT2_IND_BLOCK + 1) +#define EXT2_TIND_BLOCK (EXT2_DIND_BLOCK + 1) +#define EXT2_N_BLOCKS (EXT2_TIND_BLOCK + 1) + +/* + * Inode flags (GETFLAGS/SETFLAGS) + */ +#define EXT2_SECRM_FL FS_SECRM_FL /* Secure deletion */ +#define EXT2_UNRM_FL FS_UNRM_FL /* Undelete */ +#define EXT2_COMPR_FL FS_COMPR_FL /* Compress file */ +#define EXT2_SYNC_FL FS_SYNC_FL /* Synchronous updates */ +#define EXT2_IMMUTABLE_FL FS_IMMUTABLE_FL /* Immutable file */ +#define EXT2_APPEND_FL FS_APPEND_FL /* writes to file may only append */ +#define EXT2_NODUMP_FL FS_NODUMP_FL /* do not dump file */ +#define EXT2_NOATIME_FL FS_NOATIME_FL /* do not update atime */ +/* Reserved for compression usage... */ +#define EXT2_DIRTY_FL FS_DIRTY_FL +#define EXT2_COMPRBLK_FL FS_COMPRBLK_FL /* One or more compressed clusters */ +#define EXT2_NOCOMP_FL FS_NOCOMP_FL /* Don't compress */ +#define EXT2_ECOMPR_FL FS_ECOMPR_FL /* Compression error */ +/* End compression flags --- maybe not all used */ +#define EXT2_BTREE_FL FS_BTREE_FL /* btree format dir */ +#define EXT2_INDEX_FL FS_INDEX_FL /* hash-indexed directory */ +#define EXT2_IMAGIC_FL FS_IMAGIC_FL /* AFS directory */ +#define EXT2_JOURNAL_DATA_FL FS_JOURNAL_DATA_FL /* Reserved for ext3 */ +#define EXT2_NOTAIL_FL FS_NOTAIL_FL /* file tail should not be merged */ +#define EXT2_DIRSYNC_FL FS_DIRSYNC_FL /* dirsync behaviour (directories only) */ +#define EXT2_TOPDIR_FL FS_TOPDIR_FL /* Top of directory hierarchies*/ +#define EXT2_RESERVED_FL FS_RESERVED_FL /* reserved for ext2 lib */ + +#define EXT2_FL_USER_VISIBLE FS_FL_USER_VISIBLE /* User visible flags */ +#define EXT2_FL_USER_MODIFIABLE FS_FL_USER_MODIFIABLE /* User modifiable flags */ + +/* Flags that should be inherited by new inodes from their parent. */ +#define EXT2_FL_INHERITED (EXT2_SECRM_FL | EXT2_UNRM_FL | EXT2_COMPR_FL |\ + EXT2_SYNC_FL | EXT2_NODUMP_FL |\ + EXT2_NOATIME_FL | EXT2_COMPRBLK_FL |\ + EXT2_NOCOMP_FL | EXT2_JOURNAL_DATA_FL |\ + EXT2_NOTAIL_FL | EXT2_DIRSYNC_FL) + +/* Flags that are appropriate for regular files (all but dir-specific ones). */ +#define EXT2_REG_FLMASK (~(EXT2_DIRSYNC_FL | EXT2_TOPDIR_FL)) + +/* Flags that are appropriate for non-directories/regular files. */ +#define EXT2_OTHER_FLMASK (EXT2_NODUMP_FL | EXT2_NOATIME_FL) + +/* Mask out flags that are inappropriate for the given type of inode. */ +static inline __u32 ext2_mask_flags(umode_t mode, __u32 flags) +{ + if (S_ISDIR(mode)) + return flags; + else if (S_ISREG(mode)) + return flags & EXT2_REG_FLMASK; + else + return flags & EXT2_OTHER_FLMASK; +} + +/* + * ioctl commands + */ +#define EXT2_IOC_GETVERSION FS_IOC_GETVERSION +#define EXT2_IOC_SETVERSION FS_IOC_SETVERSION +#define EXT2_IOC_GETRSVSZ _IOR('f', 5, long) +#define EXT2_IOC_SETRSVSZ _IOW('f', 6, long) + +/* + * ioctl commands in 32 bit emulation + */ +#define EXT2_IOC32_GETVERSION FS_IOC32_GETVERSION +#define EXT2_IOC32_SETVERSION FS_IOC32_SETVERSION + +/* + * Structure of an inode on the disk + */ +struct ext2_inode { + __le16 i_mode; /* File mode */ + __le16 i_uid; /* Low 16 bits of Owner Uid */ + __le32 i_size; /* Size in bytes */ + __le32 i_atime; /* Access time */ + __le32 i_ctime; /* Creation time */ + __le32 i_mtime; /* Modification time */ + __le32 i_dtime; /* Deletion Time */ + __le16 i_gid; /* Low 16 bits of Group Id */ + __le16 i_links_count; /* Links count */ + __le32 i_blocks; /* Blocks count */ + __le32 i_flags; /* File flags */ + union { + struct { + __le32 l_i_reserved1; + } linux1; + struct { + __le32 h_i_translator; + } hurd1; + struct { + __le32 m_i_reserved1; + } masix1; + } osd1; /* OS dependent 1 */ + __le32 i_block[EXT2_N_BLOCKS];/* Pointers to blocks */ + __le32 i_generation; /* File version (for NFS) */ + __le32 i_file_acl; /* File ACL */ + __le32 i_dir_acl; /* Directory ACL */ + __le32 i_faddr; /* Fragment address */ + union { + struct { + __u8 l_i_frag; /* Fragment number */ + __u8 l_i_fsize; /* Fragment size */ + __u16 i_pad1; + __le16 l_i_uid_high; /* these 2 fields */ + __le16 l_i_gid_high; /* were reserved2[0] */ + __u32 l_i_reserved2; + } linux2; + struct { + __u8 h_i_frag; /* Fragment number */ + __u8 h_i_fsize; /* Fragment size */ + __le16 h_i_mode_high; + __le16 h_i_uid_high; + __le16 h_i_gid_high; + __le32 h_i_author; + } hurd2; + struct { + __u8 m_i_frag; /* Fragment number */ + __u8 m_i_fsize; /* Fragment size */ + __u16 m_pad1; + __u32 m_i_reserved2[2]; + } masix2; + } osd2; /* OS dependent 2 */ +}; + +#define i_size_high i_dir_acl + +#define i_reserved1 osd1.linux1.l_i_reserved1 +#define i_frag osd2.linux2.l_i_frag +#define i_fsize osd2.linux2.l_i_fsize +#define i_uid_low i_uid +#define i_gid_low i_gid +#define i_uid_high osd2.linux2.l_i_uid_high +#define i_gid_high osd2.linux2.l_i_gid_high +#define i_reserved2 osd2.linux2.l_i_reserved2 + +/* + * File system states + */ +#define EXT2_VALID_FS 0x0001 /* Unmounted cleanly */ +#define EXT2_ERROR_FS 0x0002 /* Errors detected */ + +/* + * Mount flags + */ +#define EXT2_MOUNT_OLDALLOC 0x000002 /* Don't use the new Orlov allocator */ +#define EXT2_MOUNT_GRPID 0x000004 /* Create files with directory's group */ +#define EXT2_MOUNT_DEBUG 0x000008 /* Some debugging messages */ +#define EXT2_MOUNT_ERRORS_CONT 0x000010 /* Continue on errors */ +#define EXT2_MOUNT_ERRORS_RO 0x000020 /* Remount fs ro on errors */ +#define EXT2_MOUNT_ERRORS_PANIC 0x000040 /* Panic on errors */ +#define EXT2_MOUNT_ERRORS_MASK 0x000070 +#define EXT2_MOUNT_MINIX_DF 0x000080 /* Mimics the Minix statfs */ +#define EXT2_MOUNT_NOBH 0x000100 /* No buffer_heads */ +#define EXT2_MOUNT_NO_UID32 0x000200 /* Disable 32-bit UIDs */ +#define EXT2_MOUNT_XATTR_USER 0x004000 /* Extended user attributes */ +#define EXT2_MOUNT_POSIX_ACL 0x008000 /* POSIX Access Control Lists */ +#define EXT2_MOUNT_USRQUOTA 0x020000 /* user quota */ +#define EXT2_MOUNT_GRPQUOTA 0x040000 /* group quota */ +#define EXT2_MOUNT_RESERVATION 0x080000 /* Preallocation */ + + +#define clear_opt(o, opt) o &= ~EXT2_MOUNT_##opt +#define set_opt(o, opt) o |= EXT2_MOUNT_##opt +#define test_opt(sb, opt) (EXT2_SB(sb)->s_mount_opt & \ + EXT2_MOUNT_##opt) +/* + * Maximal mount counts between two filesystem checks + */ +#define EXT2_DFL_MAX_MNT_COUNT 20 /* Allow 20 mounts */ +#define EXT2_DFL_CHECKINTERVAL 0 /* Don't use interval check */ + +/* + * Behaviour when detecting errors + */ +#define EXT2_ERRORS_CONTINUE 1 /* Continue execution */ +#define EXT2_ERRORS_RO 2 /* Remount fs read-only */ +#define EXT2_ERRORS_PANIC 3 /* Panic */ +#define EXT2_ERRORS_DEFAULT EXT2_ERRORS_CONTINUE + +/* + * Allocation flags + */ +#define EXT2_ALLOC_NORESERVE 0x1 /* Do not use reservation + * window for allocation */ + +/* + * Structure of the super block + */ +struct ext2_super_block { + __le32 s_inodes_count; /* Inodes count */ + __le32 s_blocks_count; /* Blocks count */ + __le32 s_r_blocks_count; /* Reserved blocks count */ + __le32 s_free_blocks_count; /* Free blocks count */ + __le32 s_free_inodes_count; /* Free inodes count */ + __le32 s_first_data_block; /* First Data Block */ + __le32 s_log_block_size; /* Block size */ + __le32 s_log_frag_size; /* Fragment size */ + __le32 s_blocks_per_group; /* # Blocks per group */ + __le32 s_frags_per_group; /* # Fragments per group */ + __le32 s_inodes_per_group; /* # Inodes per group */ + __le32 s_mtime; /* Mount time */ + __le32 s_wtime; /* Write time */ + __le16 s_mnt_count; /* Mount count */ + __le16 s_max_mnt_count; /* Maximal mount count */ + __le16 s_magic; /* Magic signature */ + __le16 s_state; /* File system state */ + __le16 s_errors; /* Behaviour when detecting errors */ + __le16 s_minor_rev_level; /* minor revision level */ + __le32 s_lastcheck; /* time of last check */ + __le32 s_checkinterval; /* max. time between checks */ + __le32 s_creator_os; /* OS */ + __le32 s_rev_level; /* Revision level */ + __le16 s_def_resuid; /* Default uid for reserved blocks */ + __le16 s_def_resgid; /* Default gid for reserved blocks */ + /* + * These fields are for EXT2_DYNAMIC_REV superblocks only. + * + * Note: the difference between the compatible feature set and + * the incompatible feature set is that if there is a bit set + * in the incompatible feature set that the kernel doesn't + * know about, it should refuse to mount the filesystem. + * + * e2fsck's requirements are more strict; if it doesn't know + * about a feature in either the compatible or incompatible + * feature set, it must abort and not try to meddle with + * things it doesn't understand... + */ + __le32 s_first_ino; /* First non-reserved inode */ + __le16 s_inode_size; /* size of inode structure */ + __le16 s_block_group_nr; /* block group # of this superblock */ + __le32 s_feature_compat; /* compatible feature set */ + __le32 s_feature_incompat; /* incompatible feature set */ + __le32 s_feature_ro_compat; /* readonly-compatible feature set */ + __u8 s_uuid[16]; /* 128-bit uuid for volume */ + char s_volume_name[16]; /* volume name */ + char s_last_mounted[64]; /* directory where last mounted */ + __le32 s_algorithm_usage_bitmap; /* For compression */ + /* + * Performance hints. Directory preallocation should only + * happen if the EXT2_COMPAT_PREALLOC flag is on. + */ + __u8 s_prealloc_blocks; /* Nr of blocks to try to preallocate*/ + __u8 s_prealloc_dir_blocks; /* Nr to preallocate for dirs */ + __u16 s_padding1; + /* + * Journaling support valid if EXT3_FEATURE_COMPAT_HAS_JOURNAL set. + */ + __u8 s_journal_uuid[16]; /* uuid of journal superblock */ + __u32 s_journal_inum; /* inode number of journal file */ + __u32 s_journal_dev; /* device number of journal file */ + __u32 s_last_orphan; /* start of list of inodes to delete */ + __u32 s_hash_seed[4]; /* HTREE hash seed */ + __u8 s_def_hash_version; /* Default hash version to use */ + __u8 s_reserved_char_pad; + __u16 s_reserved_word_pad; + __le32 s_default_mount_opts; + __le32 s_first_meta_bg; /* First metablock block group */ + __u32 s_reserved[190]; /* Padding to the end of the block */ +}; + +/* + * Codes for operating systems + */ +#define EXT2_OS_LINUX 0 +#define EXT2_OS_HURD 1 +#define EXT2_OS_MASIX 2 +#define EXT2_OS_FREEBSD 3 +#define EXT2_OS_LITES 4 + +/* + * Revision levels + */ +#define EXT2_GOOD_OLD_REV 0 /* The good old (original) format */ +#define EXT2_DYNAMIC_REV 1 /* V2 format w/ dynamic inode sizes */ + +#define EXT2_CURRENT_REV EXT2_GOOD_OLD_REV +#define EXT2_MAX_SUPP_REV EXT2_DYNAMIC_REV + +#define EXT2_GOOD_OLD_INODE_SIZE 128 + +/* + * Feature set definitions + */ + +#define EXT2_HAS_COMPAT_FEATURE(sb,mask) \ + ( EXT2_SB(sb)->s_es->s_feature_compat & cpu_to_le32(mask) ) +#define EXT2_HAS_RO_COMPAT_FEATURE(sb,mask) \ + ( EXT2_SB(sb)->s_es->s_feature_ro_compat & cpu_to_le32(mask) ) +#define EXT2_HAS_INCOMPAT_FEATURE(sb,mask) \ + ( EXT2_SB(sb)->s_es->s_feature_incompat & cpu_to_le32(mask) ) +#define EXT2_SET_COMPAT_FEATURE(sb,mask) \ + EXT2_SB(sb)->s_es->s_feature_compat |= cpu_to_le32(mask) +#define EXT2_SET_RO_COMPAT_FEATURE(sb,mask) \ + EXT2_SB(sb)->s_es->s_feature_ro_compat |= cpu_to_le32(mask) +#define EXT2_SET_INCOMPAT_FEATURE(sb,mask) \ + EXT2_SB(sb)->s_es->s_feature_incompat |= cpu_to_le32(mask) +#define EXT2_CLEAR_COMPAT_FEATURE(sb,mask) \ + EXT2_SB(sb)->s_es->s_feature_compat &= ~cpu_to_le32(mask) +#define EXT2_CLEAR_RO_COMPAT_FEATURE(sb,mask) \ + EXT2_SB(sb)->s_es->s_feature_ro_compat &= ~cpu_to_le32(mask) +#define EXT2_CLEAR_INCOMPAT_FEATURE(sb,mask) \ + EXT2_SB(sb)->s_es->s_feature_incompat &= ~cpu_to_le32(mask) + +#define EXT2_FEATURE_COMPAT_DIR_PREALLOC 0x0001 +#define EXT2_FEATURE_COMPAT_IMAGIC_INODES 0x0002 +#define EXT3_FEATURE_COMPAT_HAS_JOURNAL 0x0004 +#define EXT2_FEATURE_COMPAT_EXT_ATTR 0x0008 +#define EXT2_FEATURE_COMPAT_RESIZE_INO 0x0010 +#define EXT2_FEATURE_COMPAT_DIR_INDEX 0x0020 +#define EXT2_FEATURE_COMPAT_ANY 0xffffffff + +#define EXT2_FEATURE_RO_COMPAT_SPARSE_SUPER 0x0001 +#define EXT2_FEATURE_RO_COMPAT_LARGE_FILE 0x0002 +#define EXT2_FEATURE_RO_COMPAT_BTREE_DIR 0x0004 +#define EXT2_FEATURE_RO_COMPAT_ANY 0xffffffff + +#define EXT2_FEATURE_INCOMPAT_COMPRESSION 0x0001 +#define EXT2_FEATURE_INCOMPAT_FILETYPE 0x0002 +#define EXT3_FEATURE_INCOMPAT_RECOVER 0x0004 +#define EXT3_FEATURE_INCOMPAT_JOURNAL_DEV 0x0008 +#define EXT2_FEATURE_INCOMPAT_META_BG 0x0010 +#define EXT2_FEATURE_INCOMPAT_ANY 0xffffffff + +#define EXT2_FEATURE_COMPAT_SUPP EXT2_FEATURE_COMPAT_EXT_ATTR +#define EXT2_FEATURE_INCOMPAT_SUPP (EXT2_FEATURE_INCOMPAT_FILETYPE| \ + EXT2_FEATURE_INCOMPAT_META_BG) +#define EXT2_FEATURE_RO_COMPAT_SUPP (EXT2_FEATURE_RO_COMPAT_SPARSE_SUPER| \ + EXT2_FEATURE_RO_COMPAT_LARGE_FILE| \ + EXT2_FEATURE_RO_COMPAT_BTREE_DIR) +#define EXT2_FEATURE_RO_COMPAT_UNSUPPORTED ~EXT2_FEATURE_RO_COMPAT_SUPP +#define EXT2_FEATURE_INCOMPAT_UNSUPPORTED ~EXT2_FEATURE_INCOMPAT_SUPP + +/* + * Default values for user and/or group using reserved blocks + */ +#define EXT2_DEF_RESUID 0 +#define EXT2_DEF_RESGID 0 + +/* + * Default mount options + */ +#define EXT2_DEFM_DEBUG 0x0001 +#define EXT2_DEFM_BSDGROUPS 0x0002 +#define EXT2_DEFM_XATTR_USER 0x0004 +#define EXT2_DEFM_ACL 0x0008 +#define EXT2_DEFM_UID16 0x0010 + /* Not used by ext2, but reserved for use by ext3 */ +#define EXT3_DEFM_JMODE 0x0060 +#define EXT3_DEFM_JMODE_DATA 0x0020 +#define EXT3_DEFM_JMODE_ORDERED 0x0040 +#define EXT3_DEFM_JMODE_WBACK 0x0060 + +/* + * Structure of a directory entry + */ + +struct ext2_dir_entry { + __le32 inode; /* Inode number */ + __le16 rec_len; /* Directory entry length */ + __le16 name_len; /* Name length */ + char name[]; /* File name, up to EXT2_NAME_LEN */ +}; + +/* + * The new version of the directory entry. Since EXT2 structures are + * stored in intel byte order, and the name_len field could never be + * bigger than 255 chars, it's safe to reclaim the extra byte for the + * file_type field. + */ +struct ext2_dir_entry_2 { + __le32 inode; /* Inode number */ + __le16 rec_len; /* Directory entry length */ + __u8 name_len; /* Name length */ + __u8 file_type; + char name[]; /* File name, up to EXT2_NAME_LEN */ +}; + +/* + * EXT2_DIR_PAD defines the directory entries boundaries + * + * NOTE: It must be a multiple of 4 + */ +#define EXT2_DIR_PAD 4 +#define EXT2_DIR_ROUND (EXT2_DIR_PAD - 1) +#define EXT2_DIR_REC_LEN(name_len) (((name_len) + 8 + EXT2_DIR_ROUND) & \ + ~EXT2_DIR_ROUND) +#define EXT2_MAX_REC_LEN ((1<<16)-1) + +static inline void verify_offsets(void) +{ +#define A(x,y) BUILD_BUG_ON(x != offsetof(struct ext2_super_block, y)); + A(EXT2_SB_MAGIC_OFFSET, s_magic); + A(EXT2_SB_BLOCKS_OFFSET, s_blocks_count); + A(EXT2_SB_BSIZE_OFFSET, s_log_block_size); +#undef A +} + +/* + * ext2 mount options + */ +struct ext2_mount_options { + unsigned long s_mount_opt; + kuid_t s_resuid; + kgid_t s_resgid; +}; + +/* + * second extended file system inode data in memory + */ +struct ext2_inode_info { + __le32 i_data[15]; + __u32 i_flags; + __u32 i_faddr; + __u8 i_frag_no; + __u8 i_frag_size; + __u16 i_state; + __u32 i_file_acl; + __u32 i_dir_acl; + __u32 i_dtime; + + /* + * i_block_group is the number of the block group which contains + * this file's inode. Constant across the lifetime of the inode, + * it is used for making block allocation decisions - we try to + * place a file's data blocks near its inode block, and new inodes + * near to their parent directory's inode. + */ + __u32 i_block_group; + + /* block reservation info */ + struct ext2_block_alloc_info *i_block_alloc_info; + + __u32 i_dir_start_lookup; +#ifdef CONFIG_EXT2_FS_XATTR + /* + * Extended attributes can be read independently of the main file + * data. Taking i_mutex even when reading would cause contention + * between readers of EAs and writers of regular file data, so + * instead we synchronize on xattr_sem when reading or changing + * EAs. + */ + struct rw_semaphore xattr_sem; +#endif + rwlock_t i_meta_lock; + + /* + * truncate_mutex is for serialising ext2_truncate() against + * ext2_getblock(). It also protects the internals of the inode's + * reservation data structures: ext2_reserve_window and + * ext2_reserve_window_node. + */ + struct mutex truncate_mutex; + struct inode vfs_inode; + struct list_head i_orphan; /* unlinked but open inodes */ +#ifdef CONFIG_QUOTA + struct dquot __rcu *i_dquot[MAXQUOTAS]; +#endif + struct mapping_metadata_bhs i_metadata_bhs; +}; + +/* + * Inode dynamic state flags + */ +#define EXT2_STATE_NEW 0x00000001 /* inode is newly created */ + + +/* + * Function prototypes + */ + +/* + * Ok, these declarations are also in but none of the + * ext2 source programs needs to include it so they are duplicated here. + */ + +static inline struct ext2_inode_info *EXT2_I(struct inode *inode) +{ + return container_of(inode, struct ext2_inode_info, vfs_inode); +} + +/* balloc.c */ +extern int ext2_bg_has_super(struct super_block *sb, int group); +extern unsigned long ext2_bg_num_gdb(struct super_block *sb, int group); +extern ext2_fsblk_t ext2_new_blocks(struct inode *, ext2_fsblk_t, + unsigned long *, int *, unsigned int); +extern int ext2_data_block_valid(struct ext2_sb_info *sbi, ext2_fsblk_t start_blk, + unsigned int count); +extern void ext2_free_blocks(struct inode *, ext2_fsblk_t, unsigned long); +extern unsigned long ext2_count_free_blocks (struct super_block *); +extern unsigned long ext2_count_dirs (struct super_block *); +extern struct ext2_group_desc * ext2_get_group_desc(struct super_block * sb, + unsigned int block_group, + struct buffer_head ** bh); +extern void ext2_discard_reservation (struct inode *); +extern int ext2_should_retry_alloc(struct super_block *sb, int *retries); +extern void ext2_init_block_alloc_info(struct inode *); +extern void ext2_rsv_window_add(struct super_block *sb, struct ext2_reserve_window_node *rsv); + +/* dir.c */ +int ext2_add_link(struct dentry *, struct inode *); +int ext2_inode_by_name(struct inode *dir, + const struct qstr *child, ino_t *ino); +int ext2_make_empty(struct inode *, struct inode *); +struct ext2_dir_entry_2 *ext2_find_entry(struct inode *, const struct qstr *, + struct folio **foliop); +int ext2_delete_entry(struct ext2_dir_entry_2 *dir, struct folio *folio); +int ext2_empty_dir(struct inode *); +struct ext2_dir_entry_2 *ext2_dotdot(struct inode *dir, struct folio **foliop); +int ext2_set_link(struct inode *dir, struct ext2_dir_entry_2 *de, + struct folio *folio, struct inode *inode, bool update_times); + +/* ialloc.c */ +extern struct inode * ext2_new_inode (struct inode *, umode_t, const struct qstr *); +extern void ext2_free_inode (struct inode *); +extern unsigned long ext2_count_free_inodes (struct super_block *); +extern unsigned long ext2_count_free (struct buffer_head *, unsigned); + +/* inode.c */ +extern struct inode *ext2_iget (struct super_block *, unsigned long); +extern int ext2_write_inode (struct inode *, struct writeback_control *); +extern int ext2_sync_inode_metadata(struct inode *, struct writeback_control *); +extern void ext2_evict_inode(struct inode *); +void ext2_write_failed(struct address_space *mapping, loff_t to); +extern int ext2_get_block(struct inode *, sector_t, struct buffer_head *, int); +extern int ext2_setattr (struct mnt_idmap *, struct dentry *, struct iattr *); +extern int ext2_getattr (struct mnt_idmap *, const struct path *, + struct kstat *, u32, unsigned int); +extern void ext2_set_inode_flags(struct inode *inode); +extern int ext2_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + u64 start, u64 len); + +/* ioctl.c */ +extern int ext2_fileattr_get(struct dentry *dentry, struct file_kattr *fa); +extern int ext2_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa); +extern long ext2_ioctl(struct file *, unsigned int, unsigned long); +extern long ext2_compat_ioctl(struct file *, unsigned int, unsigned long); + +/* namei.c */ +struct dentry *ext2_get_parent(struct dentry *child); + +/* super.c */ +extern __printf(3, 4) +void ext2_error(struct super_block *, const char *, const char *, ...); +extern __printf(3, 4) +void ext2_msg(struct super_block *, const char *, const char *, ...); +extern void ext2_update_dynamic_rev (struct super_block *sb); +extern void ext2_sync_super(struct super_block *sb, struct ext2_super_block *es, + int wait); + +/* + * Inodes and files operations + */ + +/* dir.c */ +extern const struct file_operations ext2_dir_operations; + +/* file.c */ +extern const struct inode_operations ext2_file_inode_operations; +extern const struct file_operations ext2_file_operations; + +/* inode.c */ +extern void ext2_set_file_ops(struct inode *inode); +extern const struct address_space_operations ext2_aops; +extern const struct iomap_ops ext2_iomap_ops; + +/* namei.c */ +extern const struct inode_operations ext2_dir_inode_operations; +extern const struct inode_operations ext2_special_inode_operations; + +/* symlink.c */ +extern const struct inode_operations ext2_fast_symlink_inode_operations; +extern const struct inode_operations ext2_symlink_inode_operations; + +static inline ext2_fsblk_t +ext2_group_first_block_no(struct super_block *sb, unsigned long group_no) +{ + return group_no * (ext2_fsblk_t)EXT2_BLOCKS_PER_GROUP(sb) + + le32_to_cpu(EXT2_SB(sb)->s_es->s_first_data_block); +} + +static inline ext2_fsblk_t +ext2_group_last_block_no(struct super_block *sb, unsigned long group_no) +{ + struct ext2_sb_info *sbi = EXT2_SB(sb); + + if (group_no == sbi->s_groups_count - 1) + return le32_to_cpu(sbi->s_es->s_blocks_count) - 1; + else + return ext2_group_first_block_no(sb, group_no) + + EXT2_BLOCKS_PER_GROUP(sb) - 1; +} + +#define ext2_set_bit __test_and_set_bit_le +#define ext2_clear_bit __test_and_clear_bit_le +#define ext2_test_bit test_bit_le +#define ext2_find_first_zero_bit find_first_zero_bit_le +#define ext2_find_next_zero_bit find_next_zero_bit_le diff --git a/fs/ext2/file.c b/fs/ext2/file.c new file mode 100644 index 000000000..b9020df7d --- /dev/null +++ b/fs/ext2/file.c @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext2/file.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * from + * + * linux/fs/minix/file.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * ext2 fs regular file handling primitives + * + * 64-bit file support on 64-bit platforms by Jakub Jelinek + * (jj@sunsite.ms.mff.cuni.cz) + */ + +#include +#include +#include +#include +#include +#include +#include +#include "ext2.h" +#include "xattr.h" +#include "acl.h" +#include "trace.h" + + +/* + * Called when filp is released. This happens when all file descriptors + * for a single struct file are closed. Note that different open() calls + * for the same file yield different struct file structures. + */ +static int ext2_release_file (struct inode * inode, struct file * filp) +{ + if (filp->f_mode & FMODE_WRITE) { + mutex_lock(&EXT2_I(inode)->truncate_mutex); + ext2_discard_reservation(inode); + mutex_unlock(&EXT2_I(inode)->truncate_mutex); + } + return 0; +} + +static ssize_t ext2_dio_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file->f_mapping->host; + ssize_t ret; + + trace_ext2_dio_read_begin(iocb, to, 0); + inode_lock_shared(inode); + ret = iomap_dio_rw(iocb, to, &ext2_iomap_ops, NULL, 0, NULL, 0); + inode_unlock_shared(inode); + trace_ext2_dio_read_end(iocb, to, ret); + + return ret; +} + +static int ext2_dio_write_end_io(struct kiocb *iocb, ssize_t size, + int error, unsigned int flags) +{ + loff_t pos = iocb->ki_pos; + struct inode *inode = file_inode(iocb->ki_filp); + + if (error) + goto out; + + /* + * If we are extending the file, we have to update i_size here before + * page cache gets invalidated in iomap_dio_rw(). This prevents racing + * buffered reads from zeroing out too much from page cache pages. + * Note that all extending writes always happens synchronously with + * inode lock held by ext2_dio_write_iter(). So it is safe to update + * inode size here for extending file writes. + */ + pos += size; + if (pos > i_size_read(inode)) { + i_size_write(inode, pos); + mark_inode_dirty(inode); + } +out: + trace_ext2_dio_write_endio(iocb, size, error); + return error; +} + +static const struct iomap_dio_ops ext2_dio_write_ops = { + .end_io = ext2_dio_write_end_io, +}; + +static ssize_t ext2_dio_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file->f_mapping->host; + ssize_t ret; + unsigned int flags = 0; + unsigned long blocksize = inode->i_sb->s_blocksize; + loff_t offset = iocb->ki_pos; + loff_t count = iov_iter_count(from); + ssize_t status = 0; + + trace_ext2_dio_write_begin(iocb, from, 0); + inode_lock(inode); + ret = generic_write_checks(iocb, from); + if (ret <= 0) + goto out_unlock; + + ret = kiocb_modified(iocb); + if (ret) + goto out_unlock; + + /* use IOMAP_DIO_FORCE_WAIT for unaligned or extending writes */ + if (iocb->ki_pos + iov_iter_count(from) > i_size_read(inode) || + (!IS_ALIGNED(iocb->ki_pos | iov_iter_alignment(from), blocksize))) + flags |= IOMAP_DIO_FORCE_WAIT; + + ret = iomap_dio_rw(iocb, from, &ext2_iomap_ops, &ext2_dio_write_ops, + flags, NULL, 0); + + /* ENOTBLK is magic return value for fallback to buffered-io */ + if (ret == -ENOTBLK) + ret = 0; + + if (ret < 0 && ret != -EIOCBQUEUED) + ext2_write_failed(inode->i_mapping, offset + count); + + /* handle case for partial write and for fallback to buffered write */ + if (ret >= 0 && iov_iter_count(from)) { + loff_t pos, endbyte; + int ret2; + + iocb->ki_flags &= ~IOCB_DIRECT; + pos = iocb->ki_pos; + status = generic_perform_write(iocb, from); + if (unlikely(status < 0)) { + ret = status; + goto out_unlock; + } + + ret += status; + endbyte = pos + status - 1; + ret2 = filemap_write_and_wait_range(inode->i_mapping, pos, + endbyte); + if (!ret2) { + invalidate_mapping_pages(inode->i_mapping, + pos >> PAGE_SHIFT, + endbyte >> PAGE_SHIFT); + if (ret > 0) + ret = generic_write_sync(iocb, ret); + } else { + ret = ret2; + } + } + +out_unlock: + inode_unlock(inode); + if (status) + trace_ext2_dio_write_buff_end(iocb, from, status); + trace_ext2_dio_write_end(iocb, from, ret); + return ret; +} + +static ssize_t ext2_file_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + if (iocb->ki_flags & IOCB_DIRECT) + return ext2_dio_read_iter(iocb, to); + + return generic_file_read_iter(iocb, to); +} + +static ssize_t ext2_file_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + if (iocb->ki_flags & IOCB_DIRECT) + return ext2_dio_write_iter(iocb, from); + + return generic_file_write_iter(iocb, from); +} + +static int ext2_file_open(struct inode *inode, struct file *filp) +{ + filp->f_mode |= FMODE_CAN_ODIRECT; + return dquot_file_open(inode, filp); +} + +const struct file_operations ext2_file_operations = { + .llseek = generic_file_llseek, + .read_iter = ext2_file_read_iter, + .write_iter = ext2_file_write_iter, + .unlocked_ioctl = ext2_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = ext2_compat_ioctl, +#endif + .mmap_prepare = generic_file_mmap_prepare, + .open = ext2_file_open, + .release = ext2_release_file, + .fsync = simple_fsync, + .get_unmapped_area = thp_get_unmapped_area, + .splice_read = filemap_splice_read, + .splice_write = iter_file_splice_write, + .setlease = generic_setlease, +}; + +const struct inode_operations ext2_file_inode_operations = { + .listxattr = ext2_listxattr, + .getattr = ext2_getattr, + .setattr = ext2_setattr, + .get_inode_acl = ext2_get_acl, + .set_acl = ext2_set_acl, + .fiemap = ext2_fiemap, + .fileattr_get = ext2_fileattr_get, + .fileattr_set = ext2_fileattr_set, +}; diff --git a/fs/ext2/ialloc.c b/fs/ext2/ialloc.c new file mode 100644 index 000000000..bf21b57cf --- /dev/null +++ b/fs/ext2/ialloc.c @@ -0,0 +1,673 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext2/ialloc.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * BSD ufs-inspired inode and directory allocation by + * Stephen Tweedie (sct@dcs.ed.ac.uk), 1993 + * Big-endian to little-endian byte-swapping/bitmaps by + * David S. Miller (davem@caip.rutgers.edu), 1995 + */ + +#include +#include +#include +#include +#include +#include "ext2.h" +#include "xattr.h" +#include "acl.h" + +/* + * ialloc.c contains the inodes allocation and deallocation routines + */ + +/* + * The free inodes are managed by bitmaps. A file system contains several + * blocks groups. Each group contains 1 bitmap block for blocks, 1 bitmap + * block for inodes, N blocks for the inode table and data blocks. + * + * The file system contains group descriptors which are located after the + * super block. Each descriptor contains the number of the bitmap block and + * the free blocks count in the block. + */ + + +/* + * Read the inode allocation bitmap for a given block_group, reading + * into the specified slot in the superblock's bitmap cache. + * + * Return buffer_head of bitmap on success or NULL. + */ +static struct buffer_head * +read_inode_bitmap(struct super_block * sb, unsigned long block_group) +{ + struct ext2_group_desc *desc; + struct buffer_head *bh = NULL; + + desc = ext2_get_group_desc(sb, block_group, NULL); + if (!desc) + goto error_out; + + bh = sb_bread(sb, le32_to_cpu(desc->bg_inode_bitmap)); + if (!bh) + ext2_error(sb, "read_inode_bitmap", + "Cannot read inode bitmap - " + "block_group = %lu, inode_bitmap = %u", + block_group, le32_to_cpu(desc->bg_inode_bitmap)); +error_out: + return bh; +} + +static void ext2_release_inode(struct super_block *sb, int group, int dir) +{ + struct ext2_group_desc * desc; + struct buffer_head *bh; + + desc = ext2_get_group_desc(sb, group, &bh); + if (!desc) { + ext2_error(sb, "ext2_release_inode", + "can't get descriptor for group %d", group); + return; + } + + spin_lock(sb_bgl_lock(EXT2_SB(sb), group)); + le16_add_cpu(&desc->bg_free_inodes_count, 1); + if (dir) + le16_add_cpu(&desc->bg_used_dirs_count, -1); + spin_unlock(sb_bgl_lock(EXT2_SB(sb), group)); + percpu_counter_inc(&EXT2_SB(sb)->s_freeinodes_counter); + if (dir) + percpu_counter_dec(&EXT2_SB(sb)->s_dirs_counter); + mark_buffer_dirty(bh); +} + +/* + * NOTE! When we get the inode, we're the only people + * that have access to it, and as such there are no + * race conditions we have to worry about. The inode + * is not on the hash-lists, and it cannot be reached + * through the filesystem because the directory entry + * has been deleted earlier. + * + * HOWEVER: we must make sure that we get no aliases, + * which means that we have to call "clear_inode()" + * _before_ we mark the inode not in use in the inode + * bitmaps. Otherwise a newly created file might use + * the same inode number (not actually the same pointer + * though), and then we'd have two inodes sharing the + * same inode number and space on the harddisk. + */ +void ext2_free_inode (struct inode * inode) +{ + struct super_block * sb = inode->i_sb; + int is_directory; + unsigned long ino; + struct buffer_head *bitmap_bh; + unsigned long block_group; + unsigned long bit; + struct ext2_super_block * es; + + ino = inode->i_ino; + ext2_debug ("freeing inode %lu\n", ino); + + /* + * Note: we must free any quota before locking the superblock, + * as writing the quota to disk may need the lock as well. + */ + /* Quota is already initialized in iput() */ + dquot_free_inode(inode); + dquot_drop(inode); + + es = EXT2_SB(sb)->s_es; + is_directory = S_ISDIR(inode->i_mode); + + if (ino < EXT2_FIRST_INO(sb) || + ino > le32_to_cpu(es->s_inodes_count)) { + ext2_error (sb, "ext2_free_inode", + "reserved or nonexistent inode %lu", ino); + return; + } + block_group = (ino - 1) / EXT2_INODES_PER_GROUP(sb); + bit = (ino - 1) % EXT2_INODES_PER_GROUP(sb); + bitmap_bh = read_inode_bitmap(sb, block_group); + if (!bitmap_bh) + return; + + /* Ok, now we can actually update the inode bitmaps.. */ + if (!ext2_clear_bit_atomic(sb_bgl_lock(EXT2_SB(sb), block_group), + bit, (void *) bitmap_bh->b_data)) + ext2_error (sb, "ext2_free_inode", + "bit already cleared for inode %lu", ino); + else + ext2_release_inode(sb, block_group, is_directory); + mark_buffer_dirty(bitmap_bh); + if (sb->s_flags & SB_SYNCHRONOUS) + sync_dirty_buffer(bitmap_bh); + + brelse(bitmap_bh); +} + +/* + * We perform asynchronous prereading of the new inode's inode block when + * we create the inode, in the expectation that the inode will be written + * back soon. There are two reasons: + * + * - When creating a large number of files, the async prereads will be + * nicely merged into large reads + * - When writing out a large number of inodes, we don't need to keep on + * stalling the writes while we read the inode block. + * + * FIXME: ext2_get_group_desc() needs to be simplified. + */ +static void ext2_preread_inode(struct inode *inode) +{ + unsigned long block_group; + unsigned long offset; + unsigned long block; + unsigned int ino = inode->i_ino; + struct ext2_group_desc * gdp; + + block_group = (ino - 1) / EXT2_INODES_PER_GROUP(inode->i_sb); + gdp = ext2_get_group_desc(inode->i_sb, block_group, NULL); + if (gdp == NULL) + return; + + /* + * Figure out the offset within the block group inode table + */ + offset = ((ino - 1) % EXT2_INODES_PER_GROUP(inode->i_sb)) * + EXT2_INODE_SIZE(inode->i_sb); + block = le32_to_cpu(gdp->bg_inode_table) + + (offset >> EXT2_BLOCK_SIZE_BITS(inode->i_sb)); + sb_breadahead(inode->i_sb, block); +} + +/* + * There are two policies for allocating an inode. If the new inode is + * a directory, then a forward search is made for a block group with both + * free space and a low directory-to-inode ratio; if that fails, then of + * the groups with above-average free space, that group with the fewest + * directories already is chosen. + * + * For other inodes, search forward from the parent directory\'s block + * group to find a free inode. + */ +static int find_group_dir(struct super_block *sb, struct inode *parent) +{ + int ngroups = EXT2_SB(sb)->s_groups_count; + int avefreei = ext2_count_free_inodes(sb) / ngroups; + struct ext2_group_desc *desc, *best_desc = NULL; + int group, best_group = -1; + + for (group = 0; group < ngroups; group++) { + desc = ext2_get_group_desc (sb, group, NULL); + if (!desc || !desc->bg_free_inodes_count) + continue; + if (le16_to_cpu(desc->bg_free_inodes_count) < avefreei) + continue; + if (!best_desc || + (le16_to_cpu(desc->bg_free_blocks_count) > + le16_to_cpu(best_desc->bg_free_blocks_count))) { + best_group = group; + best_desc = desc; + } + } + + return best_group; +} + +/* + * Orlov's allocator for directories. + * + * We always try to spread first-level directories. + * + * If there are blockgroups with both free inodes and free blocks counts + * not worse than average we return one with smallest directory count. + * Otherwise we simply return a random group. + * + * For the rest rules look so: + * + * It's OK to put directory into a group unless + * it has too many directories already (max_dirs) or + * it has too few free inodes left (min_inodes) or + * it has too few free blocks left (min_blocks) or + * it's already running too large debt (max_debt). + * Parent's group is preferred, if it doesn't satisfy these + * conditions we search cyclically through the rest. If none + * of the groups look good we just look for a group with more + * free inodes than average (starting at parent's group). + * + * Debt is incremented each time we allocate a directory and decremented + * when we allocate an inode, within 0--255. + */ + +#define INODE_COST 64 +#define BLOCK_COST 256 + +static int find_group_orlov(struct super_block *sb, struct inode *parent) +{ + int parent_group = EXT2_I(parent)->i_block_group; + struct ext2_sb_info *sbi = EXT2_SB(sb); + struct ext2_super_block *es = sbi->s_es; + int ngroups = sbi->s_groups_count; + int inodes_per_group = EXT2_INODES_PER_GROUP(sb); + int freei; + int avefreei; + int free_blocks; + int avefreeb; + int blocks_per_dir; + int ndirs; + int max_debt, max_dirs, min_blocks, min_inodes; + int group = -1, i; + struct ext2_group_desc *desc; + + freei = percpu_counter_read_positive(&sbi->s_freeinodes_counter); + avefreei = freei / ngroups; + free_blocks = percpu_counter_read_positive(&sbi->s_freeblocks_counter); + avefreeb = free_blocks / ngroups; + ndirs = percpu_counter_read_positive(&sbi->s_dirs_counter); + + if ((parent == d_inode(sb->s_root)) || + (EXT2_I(parent)->i_flags & EXT2_TOPDIR_FL)) { + int best_ndir = inodes_per_group; + int best_group = -1; + + parent_group = get_random_u32_below(ngroups); + for (i = 0; i < ngroups; i++) { + group = (parent_group + i) % ngroups; + desc = ext2_get_group_desc (sb, group, NULL); + if (!desc || !desc->bg_free_inodes_count) + continue; + if (le16_to_cpu(desc->bg_used_dirs_count) >= best_ndir) + continue; + if (le16_to_cpu(desc->bg_free_inodes_count) < avefreei) + continue; + if (le16_to_cpu(desc->bg_free_blocks_count) < avefreeb) + continue; + best_group = group; + best_ndir = le16_to_cpu(desc->bg_used_dirs_count); + } + if (best_group >= 0) { + group = best_group; + goto found; + } + goto fallback; + } + + if (ndirs == 0) + ndirs = 1; /* percpu_counters are approximate... */ + + blocks_per_dir = (le32_to_cpu(es->s_blocks_count)-free_blocks) / ndirs; + + max_dirs = ndirs / ngroups + inodes_per_group / 16; + min_inodes = avefreei - inodes_per_group / 4; + min_blocks = avefreeb - EXT2_BLOCKS_PER_GROUP(sb) / 4; + + max_debt = EXT2_BLOCKS_PER_GROUP(sb) / max(blocks_per_dir, BLOCK_COST); + if (max_debt * INODE_COST > inodes_per_group) + max_debt = inodes_per_group / INODE_COST; + if (max_debt > 255) + max_debt = 255; + if (max_debt == 0) + max_debt = 1; + + for (i = 0; i < ngroups; i++) { + group = (parent_group + i) % ngroups; + desc = ext2_get_group_desc (sb, group, NULL); + if (!desc || !desc->bg_free_inodes_count) + continue; + if (sbi->s_debts[group] >= max_debt) + continue; + if (le16_to_cpu(desc->bg_used_dirs_count) >= max_dirs) + continue; + if (le16_to_cpu(desc->bg_free_inodes_count) < min_inodes) + continue; + if (le16_to_cpu(desc->bg_free_blocks_count) < min_blocks) + continue; + goto found; + } + +fallback: + for (i = 0; i < ngroups; i++) { + group = (parent_group + i) % ngroups; + desc = ext2_get_group_desc (sb, group, NULL); + if (!desc || !desc->bg_free_inodes_count) + continue; + if (le16_to_cpu(desc->bg_free_inodes_count) >= avefreei) + goto found; + } + + if (avefreei) { + /* + * The free-inodes counter is approximate, and for really small + * filesystems the above test can fail to find any blockgroups + */ + avefreei = 0; + goto fallback; + } + + return -1; + +found: + return group; +} + +static int find_group_other(struct super_block *sb, struct inode *parent) +{ + int parent_group = EXT2_I(parent)->i_block_group; + int ngroups = EXT2_SB(sb)->s_groups_count; + struct ext2_group_desc *desc; + int group, i; + + /* + * Try to place the inode in its parent directory + */ + group = parent_group; + desc = ext2_get_group_desc (sb, group, NULL); + if (desc && le16_to_cpu(desc->bg_free_inodes_count) && + le16_to_cpu(desc->bg_free_blocks_count)) + goto found; + + /* + * We're going to place this inode in a different blockgroup from its + * parent. We want to cause files in a common directory to all land in + * the same blockgroup. But we want files which are in a different + * directory which shares a blockgroup with our parent to land in a + * different blockgroup. + * + * So add our directory's i_ino into the starting point for the hash. + */ + group = (group + (unsigned int)parent->i_ino) % ngroups; + + /* + * Use a quadratic hash to find a group with a free inode and some + * free blocks. + */ + for (i = 1; i < ngroups; i <<= 1) { + group += i; + if (group >= ngroups) + group -= ngroups; + desc = ext2_get_group_desc (sb, group, NULL); + if (desc && le16_to_cpu(desc->bg_free_inodes_count) && + le16_to_cpu(desc->bg_free_blocks_count)) + goto found; + } + + /* + * That failed: try linear search for a free inode, even if that group + * has no free blocks. + */ + group = parent_group; + for (i = 0; i < ngroups; i++) { + if (++group >= ngroups) + group = 0; + desc = ext2_get_group_desc (sb, group, NULL); + if (desc && le16_to_cpu(desc->bg_free_inodes_count)) + goto found; + } + + return -1; + +found: + return group; +} + +struct inode *ext2_new_inode(struct inode *dir, umode_t mode, + const struct qstr *qstr) +{ + struct super_block *sb; + struct buffer_head *bitmap_bh = NULL; + struct buffer_head *bh2; + int group, i; + ino_t ino = 0; + struct inode * inode; + struct ext2_group_desc *gdp; + struct ext2_super_block *es; + struct ext2_inode_info *ei; + struct ext2_sb_info *sbi; + int err; + + sb = dir->i_sb; + inode = new_inode(sb); + if (!inode) + return ERR_PTR(-ENOMEM); + + ei = EXT2_I(inode); + sbi = EXT2_SB(sb); + es = sbi->s_es; + if (S_ISDIR(mode)) { + if (test_opt(sb, OLDALLOC)) + group = find_group_dir(sb, dir); + else + group = find_group_orlov(sb, dir); + } else + group = find_group_other(sb, dir); + + if (group == -1) { + err = -ENOSPC; + goto fail; + } + + for (i = 0; i < sbi->s_groups_count; i++) { + gdp = ext2_get_group_desc(sb, group, &bh2); + if (!gdp) { + if (++group == sbi->s_groups_count) + group = 0; + continue; + } + brelse(bitmap_bh); + bitmap_bh = read_inode_bitmap(sb, group); + if (!bitmap_bh) { + err = -EIO; + goto fail; + } + ino = 0; + +repeat_in_this_group: + ino = ext2_find_next_zero_bit((unsigned long *)bitmap_bh->b_data, + EXT2_INODES_PER_GROUP(sb), ino); + if (ino >= EXT2_INODES_PER_GROUP(sb)) { + /* + * Rare race: find_group_xx() decided that there were + * free inodes in this group, but by the time we tried + * to allocate one, they're all gone. This can also + * occur because the counters which find_group_orlov() + * uses are approximate. So just go and search the + * next block group. + */ + if (++group == sbi->s_groups_count) + group = 0; + continue; + } + if (ext2_set_bit_atomic(sb_bgl_lock(sbi, group), + ino, bitmap_bh->b_data)) { + /* we lost this inode */ + if (++ino >= EXT2_INODES_PER_GROUP(sb)) { + /* this group is exhausted, try next group */ + if (++group == sbi->s_groups_count) + group = 0; + continue; + } + /* try to find free inode in the same group */ + goto repeat_in_this_group; + } + goto got; + } + + /* + * Scanned all blockgroups. + */ + brelse(bitmap_bh); + err = -ENOSPC; + goto fail; +got: + mark_buffer_dirty(bitmap_bh); + if (sb->s_flags & SB_SYNCHRONOUS) + sync_dirty_buffer(bitmap_bh); + brelse(bitmap_bh); + + ino += group * EXT2_INODES_PER_GROUP(sb) + 1; + if (ino < EXT2_FIRST_INO(sb) || ino > le32_to_cpu(es->s_inodes_count)) { + ext2_error (sb, "ext2_new_inode", + "reserved inode or inode > inodes count - " + "block_group = %d,inode=%lu", group, + (unsigned long) ino); + err = -EIO; + goto fail; + } + + percpu_counter_dec(&sbi->s_freeinodes_counter); + if (S_ISDIR(mode)) + percpu_counter_inc(&sbi->s_dirs_counter); + + spin_lock(sb_bgl_lock(sbi, group)); + le16_add_cpu(&gdp->bg_free_inodes_count, -1); + if (S_ISDIR(mode)) { + if (sbi->s_debts[group] < 255) + sbi->s_debts[group]++; + le16_add_cpu(&gdp->bg_used_dirs_count, 1); + } else { + if (sbi->s_debts[group]) + sbi->s_debts[group]--; + } + spin_unlock(sb_bgl_lock(sbi, group)); + + mark_buffer_dirty(bh2); + if (test_opt(sb, GRPID)) { + inode->i_mode = mode; + inode->i_uid = current_fsuid(); + inode->i_gid = dir->i_gid; + } else + inode_init_owner(&nop_mnt_idmap, inode, dir, mode); + + inode->i_ino = ino; + inode->i_blocks = 0; + simple_inode_init_ts(inode); + memset(ei->i_data, 0, sizeof(ei->i_data)); + ei->i_flags = + ext2_mask_flags(mode, EXT2_I(dir)->i_flags & EXT2_FL_INHERITED); + ei->i_faddr = 0; + ei->i_frag_no = 0; + ei->i_frag_size = 0; + ei->i_file_acl = 0; + ei->i_dir_acl = 0; + ei->i_dtime = 0; + ei->i_block_alloc_info = NULL; + ei->i_block_group = group; + ei->i_dir_start_lookup = 0; + ei->i_state = EXT2_STATE_NEW; + ext2_set_inode_flags(inode); + spin_lock(&sbi->s_next_gen_lock); + inode->i_generation = sbi->s_next_generation++; + spin_unlock(&sbi->s_next_gen_lock); + if (insert_inode_locked(inode) < 0) { + ext2_error(sb, "ext2_new_inode", + "inode number already in use - inode=%lu", + (unsigned long) ino); + err = -EIO; + goto fail; + } + + err = dquot_initialize(inode); + if (err) + goto fail_drop; + + err = dquot_alloc_inode(inode); + if (err) + goto fail_drop; + + err = ext2_init_acl(inode, dir); + if (err) + goto fail_free_drop; + + err = ext2_init_security(inode, dir, qstr); + if (err) + goto fail_free_drop; + + mark_inode_dirty(inode); + ext2_debug("allocating inode %llu\n", inode->i_ino); + ext2_preread_inode(inode); + return inode; + +fail_free_drop: + dquot_free_inode(inode); + +fail_drop: + dquot_drop(inode); + inode->i_flags |= S_NOQUOTA; + clear_nlink(inode); + discard_new_inode(inode); + return ERR_PTR(err); + +fail: + make_bad_inode(inode); + iput(inode); + return ERR_PTR(err); +} + +unsigned long ext2_count_free_inodes (struct super_block * sb) +{ + struct ext2_group_desc *desc; + unsigned long desc_count = 0; + int i; + +#ifdef EXT2FS_DEBUG + struct ext2_super_block *es; + unsigned long bitmap_count = 0; + struct buffer_head *bitmap_bh = NULL; + + es = EXT2_SB(sb)->s_es; + for (i = 0; i < EXT2_SB(sb)->s_groups_count; i++) { + unsigned x; + + desc = ext2_get_group_desc (sb, i, NULL); + if (!desc) + continue; + desc_count += le16_to_cpu(desc->bg_free_inodes_count); + brelse(bitmap_bh); + bitmap_bh = read_inode_bitmap(sb, i); + if (!bitmap_bh) + continue; + + x = ext2_count_free(bitmap_bh, EXT2_INODES_PER_GROUP(sb) / 8); + printk("group %d: stored = %d, counted = %u\n", + i, le16_to_cpu(desc->bg_free_inodes_count), x); + bitmap_count += x; + } + brelse(bitmap_bh); + printk("ext2_count_free_inodes: stored = %lu, computed = %lu, %lu\n", + (unsigned long) + percpu_counter_read(&EXT2_SB(sb)->s_freeinodes_counter), + desc_count, bitmap_count); + return desc_count; +#else + for (i = 0; i < EXT2_SB(sb)->s_groups_count; i++) { + desc = ext2_get_group_desc (sb, i, NULL); + if (!desc) + continue; + desc_count += le16_to_cpu(desc->bg_free_inodes_count); + } + return desc_count; +#endif +} + +/* Called at mount-time, super-block is locked */ +unsigned long ext2_count_dirs (struct super_block * sb) +{ + unsigned long count = 0; + int i; + + for (i = 0; i < EXT2_SB(sb)->s_groups_count; i++) { + struct ext2_group_desc *gdp = ext2_get_group_desc (sb, i, NULL); + if (!gdp) + continue; + count += le16_to_cpu(gdp->bg_used_dirs_count); + } + return count; +} + diff --git a/fs/ext2/inode.c b/fs/ext2/inode.c new file mode 100644 index 000000000..a9245f0cd --- /dev/null +++ b/fs/ext2/inode.c @@ -0,0 +1,1652 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext2/inode.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * from + * + * linux/fs/minix/inode.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * Goal-directed block allocation by Stephen Tweedie + * (sct@dcs.ed.ac.uk), 1993, 1998 + * Big-endian to little-endian byte-swapping/bitmaps by + * David S. Miller (davem@caip.rutgers.edu), 1995 + * 64-bit file support on 64-bit platforms by Jakub Jelinek + * (jj@sunsite.ms.mff.cuni.cz) + * + * Assorted race fixes, rewrite of ext2_get_block() by Al Viro, 2000 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ext2.h" +#include "acl.h" +#include "xattr.h" + +/* + * Test whether an inode is a fast symlink. + */ +static inline int ext2_inode_is_fast_symlink(struct inode *inode) +{ + int ea_blocks = EXT2_I(inode)->i_file_acl ? + (inode->i_sb->s_blocksize >> 9) : 0; + + return (S_ISLNK(inode->i_mode) && + inode->i_blocks - ea_blocks == 0); +} + +static void ext2_truncate_blocks(struct inode *inode, loff_t offset); + +void ext2_write_failed(struct address_space *mapping, loff_t to) +{ + struct inode *inode = mapping->host; + + if (to > inode->i_size) { + truncate_pagecache(inode, inode->i_size); + ext2_truncate_blocks(inode, inode->i_size); + } +} + +/* + * Called at the last iput() if i_nlink is zero. + */ +void ext2_evict_inode(struct inode * inode) +{ + struct ext2_block_alloc_info *rsv; + int want_delete = 0; + + if (!inode->i_nlink && !is_bad_inode(inode)) { + want_delete = 1; + dquot_initialize(inode); + } else { + dquot_drop(inode); + } + + truncate_inode_pages_final(&inode->i_data); + + if (want_delete) { + sb_start_intwrite(inode->i_sb); + /* set dtime */ + EXT2_I(inode)->i_dtime = ktime_get_real_seconds(); + mark_inode_dirty(inode); + sync_inode_metadata(inode, inode_needs_sync(inode)); + /* truncate to 0 */ + inode->i_size = 0; + if (inode->i_blocks) + ext2_truncate_blocks(inode, 0); + ext2_xattr_delete_inode(inode); + } else { + mmb_sync(&EXT2_I(inode)->i_metadata_bhs); + } + mmb_invalidate(&EXT2_I(inode)->i_metadata_bhs); + clear_inode(inode); + + ext2_discard_reservation(inode); + rsv = EXT2_I(inode)->i_block_alloc_info; + EXT2_I(inode)->i_block_alloc_info = NULL; + if (unlikely(rsv)) + kfree(rsv); + + if (want_delete) { + ext2_free_inode(inode); + sb_end_intwrite(inode->i_sb); + } +} + +typedef struct { + __le32 *p; + __le32 key; + struct buffer_head *bh; +} Indirect; + +static inline void add_chain(Indirect *p, struct buffer_head *bh, __le32 *v) +{ + p->key = *(p->p = v); + p->bh = bh; +} + +static inline int verify_chain(Indirect *from, Indirect *to) +{ + while (from <= to && from->key == *from->p) + from++; + return (from > to); +} + +/** + * ext2_block_to_path - parse the block number into array of offsets + * @inode: inode in question (we are only interested in its superblock) + * @i_block: block number to be parsed + * @offsets: array to store the offsets in + * @boundary: set this non-zero if the referred-to block is likely to be + * followed (on disk) by an indirect block. + * To store the locations of file's data ext2 uses a data structure common + * for UNIX filesystems - tree of pointers anchored in the inode, with + * data blocks at leaves and indirect blocks in intermediate nodes. + * This function translates the block number into path in that tree - + * return value is the path length and @offsets[n] is the offset of + * pointer to (n+1)th node in the nth one. If @block is out of range + * (negative or too large) warning is printed and zero returned. + * + * Note: function doesn't find node addresses, so no IO is needed. All + * we need to know is the capacity of indirect blocks (taken from the + * inode->i_sb). + */ + +/* + * Portability note: the last comparison (check that we fit into triple + * indirect block) is spelled differently, because otherwise on an + * architecture with 32-bit longs and 8Kb pages we might get into trouble + * if our filesystem had 8Kb blocks. We might use long long, but that would + * kill us on x86. Oh, well, at least the sign propagation does not matter - + * i_block would have to be negative in the very beginning, so we would not + * get there at all. + */ + +static int ext2_block_to_path(struct inode *inode, + long i_block, int offsets[4], int *boundary) +{ + int ptrs = EXT2_ADDR_PER_BLOCK(inode->i_sb); + int ptrs_bits = EXT2_ADDR_PER_BLOCK_BITS(inode->i_sb); + const long direct_blocks = EXT2_NDIR_BLOCKS, + indirect_blocks = ptrs, + double_blocks = (1 << (ptrs_bits * 2)); + int n = 0; + int final = 0; + + if (i_block < 0) { + ext2_msg(inode->i_sb, KERN_WARNING, + "warning: %s: block < 0", __func__); + } else if (i_block < direct_blocks) { + offsets[n++] = i_block; + final = direct_blocks; + } else if ( (i_block -= direct_blocks) < indirect_blocks) { + offsets[n++] = EXT2_IND_BLOCK; + offsets[n++] = i_block; + final = ptrs; + } else if ((i_block -= indirect_blocks) < double_blocks) { + offsets[n++] = EXT2_DIND_BLOCK; + offsets[n++] = i_block >> ptrs_bits; + offsets[n++] = i_block & (ptrs - 1); + final = ptrs; + } else if (((i_block -= double_blocks) >> (ptrs_bits * 2)) < ptrs) { + offsets[n++] = EXT2_TIND_BLOCK; + offsets[n++] = i_block >> (ptrs_bits * 2); + offsets[n++] = (i_block >> ptrs_bits) & (ptrs - 1); + offsets[n++] = i_block & (ptrs - 1); + final = ptrs; + } else { + ext2_msg(inode->i_sb, KERN_WARNING, + "warning: %s: block is too big", __func__); + } + if (boundary) + *boundary = final - 1 - (i_block & (ptrs - 1)); + + return n; +} + +/** + * ext2_get_branch - read the chain of indirect blocks leading to data + * @inode: inode in question + * @depth: depth of the chain (1 - direct pointer, etc.) + * @offsets: offsets of pointers in inode/indirect blocks + * @chain: place to store the result + * @err: here we store the error value + * + * Function fills the array of triples and returns %NULL + * if everything went OK or the pointer to the last filled triple + * (incomplete one) otherwise. Upon the return chain[i].key contains + * the number of (i+1)-th block in the chain (as it is stored in memory, + * i.e. little-endian 32-bit), chain[i].p contains the address of that + * number (it points into struct inode for i==0 and into the bh->b_data + * for i>0) and chain[i].bh points to the buffer_head of i-th indirect + * block for i>0 and NULL for i==0. In other words, it holds the block + * numbers of the chain, addresses they were taken from (and where we can + * verify that chain did not change) and buffer_heads hosting these + * numbers. + * + * Function stops when it stumbles upon zero pointer (absent block) + * (pointer to last triple returned, *@err == 0) + * or when it gets an IO error reading an indirect block + * (ditto, *@err == -EIO) + * or when it notices that chain had been changed while it was reading + * (ditto, *@err == -EAGAIN) + * or when it reads all @depth-1 indirect blocks successfully and finds + * the whole chain, all way to the data (returns %NULL, *err == 0). + */ +static Indirect *ext2_get_branch(struct inode *inode, + int depth, + int *offsets, + Indirect chain[4], + int *err) +{ + struct super_block *sb = inode->i_sb; + Indirect *p = chain; + struct buffer_head *bh; + + *err = 0; + /* i_data is not going away, no lock needed */ + add_chain (chain, NULL, EXT2_I(inode)->i_data + *offsets); + if (!p->key) + goto no_block; + while (--depth) { + bh = sb_bread(sb, le32_to_cpu(p->key)); + if (!bh) + goto failure; + read_lock(&EXT2_I(inode)->i_meta_lock); + if (!verify_chain(chain, p)) + goto changed; + add_chain(++p, bh, (__le32*)bh->b_data + *++offsets); + read_unlock(&EXT2_I(inode)->i_meta_lock); + if (!p->key) + goto no_block; + } + return NULL; + +changed: + read_unlock(&EXT2_I(inode)->i_meta_lock); + brelse(bh); + *err = -EAGAIN; + goto no_block; +failure: + *err = -EIO; +no_block: + return p; +} + +/** + * ext2_find_near - find a place for allocation with sufficient locality + * @inode: owner + * @ind: descriptor of indirect block. + * + * This function returns the preferred place for block allocation. + * It is used when heuristic for sequential allocation fails. + * Rules are: + * + if there is a block to the left of our position - allocate near it. + * + if pointer will live in indirect block - allocate near that block. + * + if pointer will live in inode - allocate in the same cylinder group. + * + * In the latter case we colour the starting block by the callers PID to + * prevent it from clashing with concurrent allocations for a different inode + * in the same block group. The PID is used here so that functionally related + * files will be close-by on-disk. + * + * Caller must make sure that @ind is valid and will stay that way. + */ + +static ext2_fsblk_t ext2_find_near(struct inode *inode, Indirect *ind) +{ + struct ext2_inode_info *ei = EXT2_I(inode); + __le32 *start = ind->bh ? (__le32 *) ind->bh->b_data : ei->i_data; + __le32 *p; + ext2_fsblk_t bg_start; + ext2_fsblk_t colour; + + /* Try to find previous block */ + for (p = ind->p - 1; p >= start; p--) + if (*p) + return le32_to_cpu(*p); + + /* No such thing, so let's try location of indirect block */ + if (ind->bh) + return ind->bh->b_blocknr; + + /* + * It is going to be referred from inode itself? OK, just put it into + * the same cylinder group then. + */ + bg_start = ext2_group_first_block_no(inode->i_sb, ei->i_block_group); + colour = (current->pid % 16) * + (EXT2_BLOCKS_PER_GROUP(inode->i_sb) / 16); + return bg_start + colour; +} + +/** + * ext2_find_goal - find a preferred place for allocation. + * @inode: owner + * @block: block we want + * @partial: pointer to the last triple within a chain + * + * Returns preferred place for a block (the goal). + */ + +static inline ext2_fsblk_t ext2_find_goal(struct inode *inode, long block, + Indirect *partial) +{ + struct ext2_block_alloc_info *block_i; + + block_i = EXT2_I(inode)->i_block_alloc_info; + + /* + * try the heuristic for sequential allocation, + * failing that at least try to get decent locality. + */ + if (block_i && (block == block_i->last_alloc_logical_block + 1) + && (block_i->last_alloc_physical_block != 0)) { + return block_i->last_alloc_physical_block + 1; + } + + return ext2_find_near(inode, partial); +} + +/** + * ext2_blks_to_allocate: Look up the block map and count the number + * of direct blocks need to be allocated for the given branch. + * + * @branch: chain of indirect blocks + * @k: number of blocks need for indirect blocks + * @blks: number of data blocks to be mapped. + * @blocks_to_boundary: the offset in the indirect block + * + * return the number of direct blocks to allocate. + */ +static int +ext2_blks_to_allocate(Indirect * branch, int k, unsigned long blks, + int blocks_to_boundary) +{ + unsigned long count = 0; + + /* + * Simple case, [t,d]Indirect block(s) has not allocated yet + * then it's clear blocks on that path have not allocated + */ + if (k > 0) { + /* right now don't hanel cross boundary allocation */ + if (blks < blocks_to_boundary + 1) + count += blks; + else + count += blocks_to_boundary + 1; + return count; + } + + count++; + while (count < blks && count <= blocks_to_boundary + && le32_to_cpu(*(branch[0].p + count)) == 0) { + count++; + } + return count; +} + +/** + * ext2_alloc_blocks: Allocate multiple blocks needed for a branch. + * @inode: Owner. + * @goal: Preferred place for allocation. + * @indirect_blks: The number of blocks needed to allocate for indirect blocks. + * @blks: The number of blocks need to allocate for direct blocks. + * @new_blocks: On return it will store the new block numbers for + * the indirect blocks(if needed) and the first direct block. + * @err: Error pointer. + * + * Return: Number of blocks allocated. + */ +static int ext2_alloc_blocks(struct inode *inode, + ext2_fsblk_t goal, int indirect_blks, int blks, + ext2_fsblk_t new_blocks[4], int *err) +{ + int target, i; + unsigned long count = 0; + int index = 0; + ext2_fsblk_t current_block = 0; + int ret = 0; + + /* + * Here we try to allocate the requested multiple blocks at once, + * on a best-effort basis. + * To build a branch, we should allocate blocks for + * the indirect blocks(if not allocated yet), and at least + * the first direct block of this branch. That's the + * minimum number of blocks need to allocate(required) + */ + target = blks + indirect_blks; + + while (1) { + count = target; + /* allocating blocks for indirect blocks and direct blocks */ + current_block = ext2_new_blocks(inode, goal, &count, err, 0); + if (*err) + goto failed_out; + + target -= count; + /* allocate blocks for indirect blocks */ + while (index < indirect_blks && count) { + new_blocks[index++] = current_block++; + count--; + } + + if (count > 0) + break; + } + + /* save the new block number for the first direct block */ + new_blocks[index] = current_block; + + /* total number of blocks allocated for direct blocks */ + ret = count; + *err = 0; + return ret; +failed_out: + for (i = 0; i key). Upon the exit we have the same + * picture as after the successful ext2_get_block(), except that in one + * place chain is disconnected - *branch->p is still zero (we did not + * set the last link), but branch->key contains the number that should + * be placed into *branch->p to fill that gap. + * + * If allocation fails we free all blocks we've allocated (and forget + * their buffer_heads) and return the error value the from failed + * ext2_alloc_block() (normally -ENOSPC). Otherwise we set the chain + * as described above and return 0. + */ + +static int ext2_alloc_branch(struct inode *inode, + int indirect_blks, int *blks, ext2_fsblk_t goal, + int *offsets, Indirect *branch) +{ + int blocksize = inode->i_sb->s_blocksize; + int i, n = 0; + int err = 0; + struct buffer_head *bh; + int num; + ext2_fsblk_t new_blocks[4]; + ext2_fsblk_t current_block; + + num = ext2_alloc_blocks(inode, goal, indirect_blks, + *blks, new_blocks, &err); + if (err) + return err; + + branch[0].key = cpu_to_le32(new_blocks[0]); + /* + * metadata blocks and data blocks are allocated. + */ + for (n = 1; n <= indirect_blks; n++) { + /* + * Get buffer_head for parent block, zero it out + * and set the pointer to new one, then send + * parent to disk. + */ + bh = sb_getblk(inode->i_sb, new_blocks[n-1]); + if (unlikely(!bh)) { + err = -ENOMEM; + goto failed; + } + branch[n].bh = bh; + lock_buffer(bh); + memset(bh->b_data, 0, blocksize); + branch[n].p = (__le32 *) bh->b_data + offsets[n]; + branch[n].key = cpu_to_le32(new_blocks[n]); + *branch[n].p = branch[n].key; + if ( n == indirect_blks) { + current_block = new_blocks[n]; + /* + * End of chain, update the last new metablock of + * the chain to point to the new allocated + * data blocks numbers + */ + for (i=1; i < num; i++) + *(branch[n].p + i) = cpu_to_le32(++current_block); + } + set_buffer_uptodate(bh); + unlock_buffer(bh); + mmb_mark_buffer_dirty(bh, &EXT2_I(inode)->i_metadata_bhs); + /* We used to sync bh here if IS_SYNC(inode). + * But we now rely upon generic_write_sync() + * and b_inode_buffers. But not for directories. + */ + if (S_ISDIR(inode->i_mode) && IS_DIRSYNC(inode)) + sync_dirty_buffer(bh); + } + *blks = num; + return err; + +failed: + for (i = 1; i < n; i++) + bforget(branch[i].bh); + for (i = 0; i < indirect_blks; i++) + ext2_free_blocks(inode, new_blocks[i], 1); + ext2_free_blocks(inode, new_blocks[i], num); + return err; +} + +/** + * ext2_splice_branch - splice the allocated branch onto inode. + * @inode: owner + * @block: (logical) number of block we are adding + * @where: location of missing link + * @num: number of indirect blocks we are adding + * @blks: number of direct blocks we are adding + * + * This function fills the missing link and does all housekeeping needed in + * inode (->i_blocks, etc.). In case of success we end up with the full + * chain to new block and return 0. + */ +static void ext2_splice_branch(struct inode *inode, + long block, Indirect *where, int num, int blks) +{ + int i; + struct ext2_block_alloc_info *block_i; + ext2_fsblk_t current_block; + + block_i = EXT2_I(inode)->i_block_alloc_info; + + /* XXX LOCKING probably should have i_meta_lock ?*/ + /* That's it */ + + *where->p = where->key; + + /* + * Update the host buffer_head or inode to point to more just allocated + * direct blocks blocks + */ + if (num == 0 && blks > 1) { + current_block = le32_to_cpu(where->key) + 1; + for (i = 1; i < blks; i++) + *(where->p + i ) = cpu_to_le32(current_block++); + } + + /* + * update the most recently allocated logical & physical block + * in i_block_alloc_info, to assist find the proper goal block for next + * allocation + */ + if (block_i) { + block_i->last_alloc_logical_block = block + blks - 1; + block_i->last_alloc_physical_block = + le32_to_cpu(where[num].key) + blks - 1; + } + + /* We are done with atomic stuff, now do the rest of housekeeping */ + + /* had we spliced it onto indirect block? */ + if (where->bh) + mmb_mark_buffer_dirty(where->bh, &EXT2_I(inode)->i_metadata_bhs); + + inode_set_ctime_current(inode); + mark_inode_dirty(inode); +} + +/* + * Allocation strategy is simple: if we have to allocate something, we will + * have to go the whole way to leaf. So let's do it before attaching anything + * to tree, set linkage between the newborn blocks, write them if sync is + * required, recheck the path, free and repeat if check fails, otherwise + * set the last missing link (that will protect us from any truncate-generated + * removals - all blocks on the path are immune now) and possibly force the + * write on the parent block. + * That has a nice additional property: no special recovery from the failed + * allocations is needed - we simply release blocks and do not touch anything + * reachable from inode. + * + * `handle' can be NULL if create == 0. + * + * return > 0, # of blocks mapped or allocated. + * return = 0, if plain lookup failed. + * return < 0, error case. + */ +static int ext2_get_blocks(struct inode *inode, + sector_t iblock, unsigned long maxblocks, + u32 *bno, bool *new, bool *boundary, + int create) +{ + int err; + int offsets[4]; + Indirect chain[4]; + Indirect *partial; + ext2_fsblk_t goal; + int indirect_blks; + int blocks_to_boundary = 0; + int depth; + struct ext2_inode_info *ei = EXT2_I(inode); + int count = 0; + ext2_fsblk_t first_block = 0; + + if (WARN_ON_ONCE(maxblocks == 0)) + return -EINVAL; + + depth = ext2_block_to_path(inode,iblock,offsets,&blocks_to_boundary); + + if (depth == 0) + return -EIO; + + partial = ext2_get_branch(inode, depth, offsets, chain, &err); + /* Simplest case - block found, no allocation needed */ + if (!partial) { + first_block = le32_to_cpu(chain[depth - 1].key); + count++; + /*map more blocks*/ + while (count < maxblocks && count <= blocks_to_boundary) { + ext2_fsblk_t blk; + + if (!verify_chain(chain, chain + depth - 1)) { + /* + * Indirect block might be removed by + * truncate while we were reading it. + * Handling of that case: forget what we've + * got now, go to reread. + */ + err = -EAGAIN; + count = 0; + partial = chain + depth - 1; + break; + } + blk = le32_to_cpu(*(chain[depth-1].p + count)); + if (blk == first_block + count) + count++; + else + break; + } + if (err != -EAGAIN) + goto got_it; + } + + /* Next simple case - plain lookup or failed read of indirect block */ + if (!create || err == -EIO) + goto cleanup; + + mutex_lock(&ei->truncate_mutex); + /* + * If the indirect block is missing while we are reading + * the chain(ext2_get_branch() returns -EAGAIN err), or + * if the chain has been changed after we grab the semaphore, + * (either because another process truncated this branch, or + * another get_block allocated this branch) re-grab the chain to see if + * the request block has been allocated or not. + * + * Since we already block the truncate/other get_block + * at this point, we will have the current copy of the chain when we + * splice the branch into the tree. + */ + if (err == -EAGAIN || !verify_chain(chain, partial)) { + while (partial > chain) { + brelse(partial->bh); + partial--; + } + partial = ext2_get_branch(inode, depth, offsets, chain, &err); + if (!partial) { + count++; + mutex_unlock(&ei->truncate_mutex); + goto got_it; + } + + if (err) { + mutex_unlock(&ei->truncate_mutex); + goto cleanup; + } + } + + /* + * Okay, we need to do block allocation. Lazily initialize the block + * allocation info here if necessary + */ + if (S_ISREG(inode->i_mode) && (!ei->i_block_alloc_info)) + ext2_init_block_alloc_info(inode); + + goal = ext2_find_goal(inode, iblock, partial); + + /* the number of blocks need to allocate for [d,t]indirect blocks */ + indirect_blks = (chain + depth) - partial - 1; + /* + * Next look up the indirect map to count the total number of + * direct blocks to allocate for this branch. + */ + count = ext2_blks_to_allocate(partial, indirect_blks, + maxblocks, blocks_to_boundary); + /* + * XXX ???? Block out ext2_truncate while we alter the tree + */ + err = ext2_alloc_branch(inode, indirect_blks, &count, goal, + offsets + (partial - chain), partial); + + if (err) { + mutex_unlock(&ei->truncate_mutex); + goto cleanup; + } + + *new = true; + + ext2_splice_branch(inode, iblock, partial, indirect_blks, count); + mutex_unlock(&ei->truncate_mutex); +got_it: + if (count > blocks_to_boundary) + *boundary = true; + err = count; + /* Clean up and exit */ + partial = chain + depth - 1; /* the whole chain */ +cleanup: + while (partial > chain) { + brelse(partial->bh); + partial--; + } + if (err > 0) + *bno = le32_to_cpu(chain[depth-1].key); + return err; +} + +int ext2_get_block(struct inode *inode, sector_t iblock, + struct buffer_head *bh_result, int create) +{ + unsigned max_blocks = bh_result->b_size >> inode->i_blkbits; + bool new = false, boundary = false; + u32 bno; + int ret; + + ret = ext2_get_blocks(inode, iblock, max_blocks, &bno, &new, &boundary, + create); + if (ret <= 0) + return ret; + + map_bh(bh_result, inode->i_sb, bno); + bh_result->b_size = (ret << inode->i_blkbits); + if (new) + set_buffer_new(bh_result); + if (boundary) + set_buffer_boundary(bh_result); + return 0; + +} + +static int ext2_iomap_begin(struct inode *inode, loff_t offset, loff_t length, + unsigned flags, struct iomap *iomap, struct iomap *srcmap) +{ + unsigned int blkbits = inode->i_blkbits; + unsigned long first_block = offset >> blkbits; + unsigned long max_blocks = (length + (1 << blkbits) - 1) >> blkbits; + bool new = false, boundary = false; + u32 bno; + int ret; + bool create = flags & IOMAP_WRITE; + + /* + * For writes that could fill holes inside i_size on a + * DIO_SKIP_HOLES filesystem we forbid block creations: only + * overwrites are permitted. + */ + if ((flags & IOMAP_DIRECT) && + (first_block << blkbits) < i_size_read(inode)) + create = 0; + + /* + * Writes that span EOF might trigger an IO size update on completion, + * so consider them to be dirty for the purposes of O_DSYNC even if + * there is no other metadata changes pending or have been made here. + */ + if ((flags & IOMAP_WRITE) && offset + length > i_size_read(inode)) + iomap->flags |= IOMAP_F_DIRTY; + + ret = ext2_get_blocks(inode, first_block, max_blocks, + &bno, &new, &boundary, create); + if (ret < 0) + return ret; + + iomap->flags = 0; + iomap->offset = (u64)first_block << blkbits; + iomap->bdev = inode->i_sb->s_bdev; + + if (ret == 0) { + /* + * Switch to buffered-io for writing to holes in a non-extent + * based filesystem to avoid stale data exposure problem. + */ + if (!create && (flags & IOMAP_WRITE) && (flags & IOMAP_DIRECT)) + return -ENOTBLK; + iomap->type = IOMAP_HOLE; + iomap->addr = IOMAP_NULL_ADDR; + iomap->length = 1 << blkbits; + } else { + iomap->type = IOMAP_MAPPED; + iomap->addr = (u64)bno << blkbits; + iomap->length = (u64)ret << blkbits; + iomap->flags |= IOMAP_F_MERGED; + } + + if (new) + iomap->flags |= IOMAP_F_NEW; + return 0; +} + +static int +ext2_iomap_end(struct inode *inode, loff_t offset, loff_t length, + ssize_t written, unsigned flags, struct iomap *iomap) +{ + /* + * Switch to buffered-io in case of any error. + * Blocks allocated can be used by the buffered-io path. + */ + if ((flags & IOMAP_DIRECT) && (flags & IOMAP_WRITE) && written == 0) + return -ENOTBLK; + + if (iomap->type == IOMAP_MAPPED && + written < length && + (flags & IOMAP_WRITE)) + ext2_write_failed(inode->i_mapping, offset + length); + return 0; +} + +static DEFINE_IOMAP_ITER_NEXT_END(ext2_iomap_next, ext2_iomap_begin, + ext2_iomap_end); + +const struct iomap_ops ext2_iomap_ops = { + .iomap_next = ext2_iomap_next, +}; + +int ext2_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + u64 start, u64 len) +{ + int ret; + loff_t i_size; + + inode_lock(inode); + i_size = i_size_read(inode); + /* + * iomap_fiemap() returns EINVAL for 0 length. Make sure we don't trim + * length to 0 but still trim the range as much as possible since + * ext2_get_blocks() iterates unmapped space block by block which is + * slow. + */ + if (i_size == 0) + i_size = 1; + len = min_t(u64, len, i_size); + ret = iomap_fiemap(inode, fieinfo, start, len, &ext2_iomap_ops); + inode_unlock(inode); + + return ret; +} + +static int ext2_read_folio(struct file *file, struct folio *folio) +{ + return mpage_read_folio(folio, ext2_get_block); +} + +static void ext2_readahead(struct readahead_control *rac) +{ + mpage_readahead(rac, ext2_get_block); +} + +static int +ext2_write_begin(const struct kiocb *iocb, struct address_space *mapping, + loff_t pos, unsigned len, struct folio **foliop, void **fsdata) +{ + int ret; + + ret = block_write_begin(mapping, pos, len, foliop, ext2_get_block); + if (ret < 0) + ext2_write_failed(mapping, pos + len); + return ret; +} + +static int ext2_write_end(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct folio *folio, void *fsdata) +{ + int ret; + + ret = generic_write_end(iocb, mapping, pos, len, copied, folio, fsdata); + if (ret < len) + ext2_write_failed(mapping, pos + len); + return ret; +} + +static sector_t ext2_bmap(struct address_space *mapping, sector_t block) +{ + return generic_block_bmap(mapping,block,ext2_get_block); +} + +static int +ext2_writepages(struct address_space *mapping, struct writeback_control *wbc) +{ + return mpage_writepages(mapping, wbc, ext2_get_block); +} + + +const struct address_space_operations ext2_aops = { + .dirty_folio = block_dirty_folio, + .invalidate_folio = block_invalidate_folio, + .read_folio = ext2_read_folio, + .readahead = ext2_readahead, + .write_begin = ext2_write_begin, + .write_end = ext2_write_end, + .bmap = ext2_bmap, + .writepages = ext2_writepages, + .migrate_folio = buffer_migrate_folio, + .is_partially_uptodate = block_is_partially_uptodate, + .error_remove_folio = generic_error_remove_folio, +}; + + +/* + * Probably it should be a library function... search for first non-zero word + * or memcmp with zero_page, whatever is better for particular architecture. + * Linus? + */ +static inline int all_zeroes(__le32 *p, __le32 *q) +{ + while (p < q) + if (*p++) + return 0; + return 1; +} + +/** + * ext2_find_shared - find the indirect blocks for partial truncation. + * @inode: inode in question + * @depth: depth of the affected branch + * @offsets: offsets of pointers in that branch (see ext2_block_to_path) + * @chain: place to store the pointers to partial indirect blocks + * @top: place to the (detached) top of branch + * + * This is a helper function used by ext2_truncate(). + * + * When we do truncate() we may have to clean the ends of several indirect + * blocks but leave the blocks themselves alive. Block is partially + * truncated if some data below the new i_size is referred from it (and + * it is on the path to the first completely truncated data block, indeed). + * We have to free the top of that path along with everything to the right + * of the path. Since no allocation past the truncation point is possible + * until ext2_truncate() finishes, we may safely do the latter, but top + * of branch may require special attention - pageout below the truncation + * point might try to populate it. + * + * We atomically detach the top of branch from the tree, store the block + * number of its root in *@top, pointers to buffer_heads of partially + * truncated blocks - in @chain[].bh and pointers to their last elements + * that should not be removed - in @chain[].p. Return value is the pointer + * to last filled element of @chain. + * + * The work left to caller to do the actual freeing of subtrees: + * a) free the subtree starting from *@top + * b) free the subtrees whose roots are stored in + * (@chain[i].p+1 .. end of @chain[i].bh->b_data) + * c) free the subtrees growing from the inode past the @chain[0].p + * (no partially truncated stuff there). + */ + +static Indirect *ext2_find_shared(struct inode *inode, + int depth, + int offsets[4], + Indirect chain[4], + __le32 *top) +{ + Indirect *partial, *p; + int k, err; + + *top = 0; + for (k = depth; k > 1 && !offsets[k-1]; k--) + ; + partial = ext2_get_branch(inode, k, offsets, chain, &err); + if (!partial) + partial = chain + k-1; + /* + * If the branch acquired continuation since we've looked at it - + * fine, it should all survive and (new) top doesn't belong to us. + */ + write_lock(&EXT2_I(inode)->i_meta_lock); + if (!partial->key && *partial->p) { + write_unlock(&EXT2_I(inode)->i_meta_lock); + goto no_top; + } + for (p=partial; p>chain && all_zeroes((__le32*)p->bh->b_data,p->p); p--) + ; + /* + * OK, we've found the last block that must survive. The rest of our + * branch should be detached before unlocking. However, if that rest + * of branch is all ours and does not grow immediately from the inode + * it's easier to cheat and just decrement partial->p. + */ + if (p == chain + k - 1 && p > chain) { + p->p--; + } else { + *top = *p->p; + *p->p = 0; + } + write_unlock(&EXT2_I(inode)->i_meta_lock); + + while(partial > p) + { + brelse(partial->bh); + partial--; + } +no_top: + return partial; +} + +/** + * ext2_free_data - free a list of data blocks + * @inode: inode we are dealing with + * @p: array of block numbers + * @q: points immediately past the end of array + * + * We are freeing all blocks referred from that array (numbers are + * stored as little-endian 32-bit) and updating @inode->i_blocks + * appropriately. + */ +static inline void ext2_free_data(struct inode *inode, __le32 *p, __le32 *q) +{ + ext2_fsblk_t block_to_free = 0, count = 0; + ext2_fsblk_t nr; + + for ( ; p < q ; p++) { + nr = le32_to_cpu(*p); + if (nr) { + *p = 0; + /* accumulate blocks to free if they're contiguous */ + if (count == 0) + goto free_this; + else if (block_to_free == nr - count) + count++; + else { + ext2_free_blocks (inode, block_to_free, count); + mark_inode_dirty(inode); + free_this: + block_to_free = nr; + count = 1; + } + } + } + if (count > 0) { + ext2_free_blocks (inode, block_to_free, count); + mark_inode_dirty(inode); + } +} + +/** + * ext2_free_branches - free an array of branches + * @inode: inode we are dealing with + * @p: array of block numbers + * @q: pointer immediately past the end of array + * @depth: depth of the branches to free + * + * We are freeing all blocks referred from these branches (numbers are + * stored as little-endian 32-bit) and updating @inode->i_blocks + * appropriately. + */ +static void ext2_free_branches(struct inode *inode, __le32 *p, __le32 *q, int depth) +{ + struct buffer_head * bh; + ext2_fsblk_t nr; + + if (depth--) { + int addr_per_block = EXT2_ADDR_PER_BLOCK(inode->i_sb); + for ( ; p < q ; p++) { + nr = le32_to_cpu(*p); + if (!nr) + continue; + *p = 0; + bh = sb_bread(inode->i_sb, nr); + /* + * A read failure? Report error and clear slot + * (should be rare). + */ + if (!bh) { + ext2_error(inode->i_sb, "ext2_free_branches", + "Read failure, inode=%llu, block=%ld", + inode->i_ino, nr); + continue; + } + ext2_free_branches(inode, + (__le32*)bh->b_data, + (__le32*)bh->b_data + addr_per_block, + depth); + bforget(bh); + ext2_free_blocks(inode, nr, 1); + mark_inode_dirty(inode); + } + } else + ext2_free_data(inode, p, q); +} + +/* mapping->invalidate_lock must be held when calling this function */ +static void __ext2_truncate_blocks(struct inode *inode, loff_t offset) +{ + __le32 *i_data = EXT2_I(inode)->i_data; + struct ext2_inode_info *ei = EXT2_I(inode); + int addr_per_block = EXT2_ADDR_PER_BLOCK(inode->i_sb); + int offsets[4]; + Indirect chain[4]; + Indirect *partial; + __le32 nr = 0; + int n; + long iblock; + unsigned blocksize; + blocksize = inode->i_sb->s_blocksize; + iblock = (offset + blocksize-1) >> EXT2_BLOCK_SIZE_BITS(inode->i_sb); + + + n = ext2_block_to_path(inode, iblock, offsets, NULL); + if (n == 0) + return; + + /* + * From here we block out all ext2_get_block() callers who want to + * modify the block allocation tree. + */ + mutex_lock(&ei->truncate_mutex); + + if (n == 1) { + ext2_free_data(inode, i_data+offsets[0], + i_data + EXT2_NDIR_BLOCKS); + goto do_indirects; + } + + partial = ext2_find_shared(inode, n, offsets, chain, &nr); + /* Kill the top of shared branch (already detached) */ + if (nr) { + if (partial == chain) + mark_inode_dirty(inode); + else + mmb_mark_buffer_dirty(partial->bh, + &EXT2_I(inode)->i_metadata_bhs); + ext2_free_branches(inode, &nr, &nr+1, (chain+n-1) - partial); + } + /* Clear the ends of indirect blocks on the shared branch */ + while (partial > chain) { + ext2_free_branches(inode, + partial->p + 1, + (__le32*)partial->bh->b_data+addr_per_block, + (chain+n-1) - partial); + mmb_mark_buffer_dirty(partial->bh, + &EXT2_I(inode)->i_metadata_bhs); + brelse (partial->bh); + partial--; + } +do_indirects: + /* Kill the remaining (whole) subtrees */ + switch (offsets[0]) { + default: + nr = i_data[EXT2_IND_BLOCK]; + if (nr) { + i_data[EXT2_IND_BLOCK] = 0; + mark_inode_dirty(inode); + ext2_free_branches(inode, &nr, &nr+1, 1); + } + fallthrough; + case EXT2_IND_BLOCK: + nr = i_data[EXT2_DIND_BLOCK]; + if (nr) { + i_data[EXT2_DIND_BLOCK] = 0; + mark_inode_dirty(inode); + ext2_free_branches(inode, &nr, &nr+1, 2); + } + fallthrough; + case EXT2_DIND_BLOCK: + nr = i_data[EXT2_TIND_BLOCK]; + if (nr) { + i_data[EXT2_TIND_BLOCK] = 0; + mark_inode_dirty(inode); + ext2_free_branches(inode, &nr, &nr+1, 3); + } + break; + case EXT2_TIND_BLOCK: + ; + } + + ext2_discard_reservation(inode); + + mutex_unlock(&ei->truncate_mutex); +} + +static void ext2_truncate_blocks(struct inode *inode, loff_t offset) +{ + if (!(S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) || + S_ISLNK(inode->i_mode))) + return; + if (ext2_inode_is_fast_symlink(inode)) + return; + + filemap_invalidate_lock(inode->i_mapping); + __ext2_truncate_blocks(inode, offset); + filemap_invalidate_unlock(inode->i_mapping); +} + +static int ext2_setsize(struct inode *inode, loff_t newsize) +{ + int error; + + if (!(S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) || + S_ISLNK(inode->i_mode))) + return -EINVAL; + if (ext2_inode_is_fast_symlink(inode)) + return -EINVAL; + if (IS_APPEND(inode) || IS_IMMUTABLE(inode)) + return -EPERM; + + inode_dio_wait(inode); + + error = block_truncate_page(inode->i_mapping, newsize, ext2_get_block); + if (error) + return error; + + filemap_invalidate_lock(inode->i_mapping); + truncate_setsize(inode, newsize); + __ext2_truncate_blocks(inode, newsize); + filemap_invalidate_unlock(inode->i_mapping); + + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + mark_inode_dirty(inode); + if (inode_needs_sync(inode)) + sync_inode_metadata(inode, 1); + + return 0; +} + +static struct ext2_inode *ext2_get_inode(struct super_block *sb, ino_t ino, + struct buffer_head **p) +{ + struct buffer_head * bh; + unsigned long block_group; + unsigned long block; + unsigned long offset; + struct ext2_group_desc * gdp; + + *p = NULL; + if ((ino != EXT2_ROOT_INO && ino < EXT2_FIRST_INO(sb)) || + ino > le32_to_cpu(EXT2_SB(sb)->s_es->s_inodes_count)) + goto Einval; + + block_group = (ino - 1) / EXT2_INODES_PER_GROUP(sb); + gdp = ext2_get_group_desc(sb, block_group, NULL); + if (!gdp) + goto Egdp; + /* + * Figure out the offset within the block group inode table + */ + offset = ((ino - 1) % EXT2_INODES_PER_GROUP(sb)) * EXT2_INODE_SIZE(sb); + block = le32_to_cpu(gdp->bg_inode_table) + + (offset >> EXT2_BLOCK_SIZE_BITS(sb)); + if (!(bh = sb_bread(sb, block))) + goto Eio; + + *p = bh; + offset &= (EXT2_BLOCK_SIZE(sb) - 1); + return (struct ext2_inode *) (bh->b_data + offset); + +Einval: + ext2_error(sb, "ext2_get_inode", "bad inode number: %lu", + (unsigned long) ino); + return ERR_PTR(-EINVAL); +Eio: + ext2_error(sb, "ext2_get_inode", + "unable to read inode block - inode=%lu, block=%lu", + (unsigned long) ino, block); +Egdp: + return ERR_PTR(-EIO); +} + +void ext2_set_inode_flags(struct inode *inode) +{ + unsigned int flags = EXT2_I(inode)->i_flags; + + inode->i_flags &= ~(S_SYNC | S_APPEND | S_IMMUTABLE | S_NOATIME | + S_DIRSYNC); + if (flags & EXT2_SYNC_FL) + inode->i_flags |= S_SYNC; + if (flags & EXT2_APPEND_FL) + inode->i_flags |= S_APPEND; + if (flags & EXT2_IMMUTABLE_FL) + inode->i_flags |= S_IMMUTABLE; + if (flags & EXT2_NOATIME_FL) + inode->i_flags |= S_NOATIME; + if (flags & EXT2_DIRSYNC_FL) + inode->i_flags |= S_DIRSYNC; +} + +void ext2_set_file_ops(struct inode *inode) +{ + inode->i_op = &ext2_file_inode_operations; + inode->i_fop = &ext2_file_operations; + inode->i_mapping->a_ops = &ext2_aops; +} + +struct inode *ext2_iget (struct super_block *sb, unsigned long ino) +{ + struct ext2_inode_info *ei; + struct buffer_head * bh = NULL; + struct ext2_inode *raw_inode; + struct inode *inode; + long ret = -EIO; + int n; + uid_t i_uid; + gid_t i_gid; + + inode = iget_locked(sb, ino); + if (!inode) + return ERR_PTR(-ENOMEM); + if (!(inode_state_read_once(inode) & I_NEW)) + return inode; + + ei = EXT2_I(inode); + ei->i_block_alloc_info = NULL; + + raw_inode = ext2_get_inode(inode->i_sb, ino, &bh); + if (IS_ERR(raw_inode)) { + ret = PTR_ERR(raw_inode); + goto bad_inode; + } + + inode->i_mode = le16_to_cpu(raw_inode->i_mode); + i_uid = (uid_t)le16_to_cpu(raw_inode->i_uid_low); + i_gid = (gid_t)le16_to_cpu(raw_inode->i_gid_low); + if (!(test_opt (inode->i_sb, NO_UID32))) { + i_uid |= le16_to_cpu(raw_inode->i_uid_high) << 16; + i_gid |= le16_to_cpu(raw_inode->i_gid_high) << 16; + } + i_uid_write(inode, i_uid); + i_gid_write(inode, i_gid); + set_nlink(inode, le16_to_cpu(raw_inode->i_links_count)); + inode->i_size = le32_to_cpu(raw_inode->i_size); + inode_set_atime(inode, (signed)le32_to_cpu(raw_inode->i_atime), 0); + inode_set_ctime(inode, (signed)le32_to_cpu(raw_inode->i_ctime), 0); + inode_set_mtime(inode, (signed)le32_to_cpu(raw_inode->i_mtime), 0); + ei->i_dtime = le32_to_cpu(raw_inode->i_dtime); + /* We now have enough fields to check if the inode was active or not. + * This is needed because nfsd might try to access dead inodes + * the test is that same one that e2fsck uses + * NeilBrown 1999oct15 + */ + if (inode->i_nlink == 0) { + if (inode->i_mode == 0 || ei->i_dtime) { + /* this inode is deleted */ + ret = -ESTALE; + } else { + ext2_error(sb, __func__, + "inode %lu has zero i_nlink with mode 0%o and no dtime, " + "filesystem may be corrupt", + ino, inode->i_mode); + ret = -EFSCORRUPTED; + } + goto bad_inode; + } + inode->i_blocks = le32_to_cpu(raw_inode->i_blocks); + ei->i_flags = le32_to_cpu(raw_inode->i_flags); + ext2_set_inode_flags(inode); + ei->i_faddr = le32_to_cpu(raw_inode->i_faddr); + ei->i_frag_no = raw_inode->i_frag; + ei->i_frag_size = raw_inode->i_fsize; + ei->i_file_acl = le32_to_cpu(raw_inode->i_file_acl); + ei->i_dir_acl = 0; + + if (ei->i_file_acl && + !ext2_data_block_valid(EXT2_SB(sb), ei->i_file_acl, 1)) { + ext2_error(sb, "ext2_iget", "bad extended attribute block %u", + ei->i_file_acl); + ret = -EFSCORRUPTED; + goto bad_inode; + } + + if (S_ISREG(inode->i_mode)) + inode->i_size |= ((__u64)le32_to_cpu(raw_inode->i_size_high)) << 32; + else + ei->i_dir_acl = le32_to_cpu(raw_inode->i_dir_acl); + if (i_size_read(inode) < 0) { + ret = -EFSCORRUPTED; + goto bad_inode; + } + ei->i_dtime = 0; + inode->i_generation = le32_to_cpu(raw_inode->i_generation); + ei->i_state = 0; + ei->i_block_group = (ino - 1) / EXT2_INODES_PER_GROUP(inode->i_sb); + ei->i_dir_start_lookup = 0; + + /* + * NOTE! The in-memory inode i_data array is in little-endian order + * even on big-endian machines: we do NOT byteswap the block numbers! + */ + for (n = 0; n < EXT2_N_BLOCKS; n++) + ei->i_data[n] = raw_inode->i_block[n]; + + if (S_ISREG(inode->i_mode)) { + ext2_set_file_ops(inode); + } else if (S_ISDIR(inode->i_mode)) { + inode->i_op = &ext2_dir_inode_operations; + inode->i_fop = &ext2_dir_operations; + inode->i_mapping->a_ops = &ext2_aops; + } else if (S_ISLNK(inode->i_mode)) { + if (ext2_inode_is_fast_symlink(inode)) { + inode->i_link = (char *)ei->i_data; + inode->i_op = &ext2_fast_symlink_inode_operations; + nd_terminate_link(ei->i_data, inode->i_size, + sizeof(ei->i_data) - 1); + } else { + inode->i_op = &ext2_symlink_inode_operations; + inode_nohighmem(inode); + inode->i_mapping->a_ops = &ext2_aops; + } + } else { + inode->i_op = &ext2_special_inode_operations; + if (raw_inode->i_block[0]) + init_special_inode(inode, inode->i_mode, + old_decode_dev(le32_to_cpu(raw_inode->i_block[0]))); + else + init_special_inode(inode, inode->i_mode, + new_decode_dev(le32_to_cpu(raw_inode->i_block[1]))); + } + brelse (bh); + unlock_new_inode(inode); + return inode; + +bad_inode: + brelse(bh); + iget_failed(inode); + return ERR_PTR(ret); +} + +int ext2_write_inode(struct inode *inode, struct writeback_control *wbc) +{ + struct ext2_inode_info *ei = EXT2_I(inode); + struct super_block *sb = inode->i_sb; + ino_t ino = inode->i_ino; + uid_t uid = i_uid_read(inode); + gid_t gid = i_gid_read(inode); + struct buffer_head * bh; + struct ext2_inode * raw_inode = ext2_get_inode(sb, ino, &bh); + int n; + int err = 0; + + if (IS_ERR(raw_inode)) + return -EIO; + + /* For fields not tracking in the in-memory inode, + * initialise them to zero for new inodes. */ + if (ei->i_state & EXT2_STATE_NEW) + memset(raw_inode, 0, EXT2_SB(sb)->s_inode_size); + + raw_inode->i_mode = cpu_to_le16(inode->i_mode); + if (!(test_opt(sb, NO_UID32))) { + raw_inode->i_uid_low = cpu_to_le16(low_16_bits(uid)); + raw_inode->i_gid_low = cpu_to_le16(low_16_bits(gid)); +/* + * Fix up interoperability with old kernels. Otherwise, old inodes get + * re-used with the upper 16 bits of the uid/gid intact + */ + if (!ei->i_dtime) { + raw_inode->i_uid_high = cpu_to_le16(high_16_bits(uid)); + raw_inode->i_gid_high = cpu_to_le16(high_16_bits(gid)); + } else { + raw_inode->i_uid_high = 0; + raw_inode->i_gid_high = 0; + } + } else { + raw_inode->i_uid_low = cpu_to_le16(fs_high2lowuid(uid)); + raw_inode->i_gid_low = cpu_to_le16(fs_high2lowgid(gid)); + raw_inode->i_uid_high = 0; + raw_inode->i_gid_high = 0; + } + raw_inode->i_links_count = cpu_to_le16(inode->i_nlink); + raw_inode->i_size = cpu_to_le32(inode->i_size); + raw_inode->i_atime = cpu_to_le32(inode_get_atime_sec(inode)); + raw_inode->i_ctime = cpu_to_le32(inode_get_ctime_sec(inode)); + raw_inode->i_mtime = cpu_to_le32(inode_get_mtime_sec(inode)); + + raw_inode->i_blocks = cpu_to_le32(inode->i_blocks); + raw_inode->i_dtime = cpu_to_le32(ei->i_dtime); + raw_inode->i_flags = cpu_to_le32(ei->i_flags); + raw_inode->i_faddr = cpu_to_le32(ei->i_faddr); + raw_inode->i_frag = ei->i_frag_no; + raw_inode->i_fsize = ei->i_frag_size; + raw_inode->i_file_acl = cpu_to_le32(ei->i_file_acl); + if (!S_ISREG(inode->i_mode)) + raw_inode->i_dir_acl = cpu_to_le32(ei->i_dir_acl); + else { + raw_inode->i_size_high = cpu_to_le32(inode->i_size >> 32); + if (inode->i_size > 0x7fffffffULL) { + if (!EXT2_HAS_RO_COMPAT_FEATURE(sb, + EXT2_FEATURE_RO_COMPAT_LARGE_FILE) || + EXT2_SB(sb)->s_es->s_rev_level == + cpu_to_le32(EXT2_GOOD_OLD_REV)) { + /* If this is the first large file + * created, add a flag to the superblock. + */ + spin_lock(&EXT2_SB(sb)->s_lock); + ext2_update_dynamic_rev(sb); + EXT2_SET_RO_COMPAT_FEATURE(sb, + EXT2_FEATURE_RO_COMPAT_LARGE_FILE); + spin_unlock(&EXT2_SB(sb)->s_lock); + ext2_sync_super(sb, EXT2_SB(sb)->s_es, 1); + } + } + } + + raw_inode->i_generation = cpu_to_le32(inode->i_generation); + if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) { + if (old_valid_dev(inode->i_rdev)) { + raw_inode->i_block[0] = + cpu_to_le32(old_encode_dev(inode->i_rdev)); + raw_inode->i_block[1] = 0; + } else { + raw_inode->i_block[0] = 0; + raw_inode->i_block[1] = + cpu_to_le32(new_encode_dev(inode->i_rdev)); + raw_inode->i_block[2] = 0; + } + } else for (n = 0; n < EXT2_N_BLOCKS; n++) + raw_inode->i_block[n] = ei->i_data[n]; + mark_buffer_dirty(bh); + ei->i_state &= ~EXT2_STATE_NEW; + brelse (bh); + set_inode_metadata_writeback(inode); + return err; +} + +int ext2_sync_inode_metadata(struct inode *inode, struct writeback_control *wbc) +{ + struct buffer_head *bh; + struct ext2_inode *raw_inode = ext2_get_inode(inode->i_sb, inode->i_ino, + &bh); + int err = 0; + + if (IS_ERR(raw_inode)) + return -EIO; + err = mmb_sync(&EXT2_I(inode)->i_metadata_bhs); + if (err) { + ext2_error(inode->i_sb, __func__, + "Error syncing inode metadata ino=%lu\n", + (unsigned long)inode->i_ino); + goto out; + } + sync_dirty_buffer(bh); + if (buffer_write_io_error(bh)) { + ext2_error(inode->i_sb, __func__, + "IO error syncing inode %lu\n", + (unsigned long)inode->i_ino); + err = -EIO; + } +out: + brelse(bh); + return err; +} + +int ext2_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int query_flags) +{ + struct inode *inode = d_inode(path->dentry); + struct ext2_inode_info *ei = EXT2_I(inode); + unsigned int flags; + + flags = ei->i_flags & EXT2_FL_USER_VISIBLE; + if (flags & EXT2_APPEND_FL) + stat->attributes |= STATX_ATTR_APPEND; + if (flags & EXT2_COMPR_FL) + stat->attributes |= STATX_ATTR_COMPRESSED; + if (flags & EXT2_IMMUTABLE_FL) + stat->attributes |= STATX_ATTR_IMMUTABLE; + if (flags & EXT2_NODUMP_FL) + stat->attributes |= STATX_ATTR_NODUMP; + stat->attributes_mask |= (STATX_ATTR_APPEND | + STATX_ATTR_COMPRESSED | + STATX_ATTR_ENCRYPTED | + STATX_ATTR_IMMUTABLE | + STATX_ATTR_NODUMP); + + generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat); + return 0; +} + +int ext2_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *iattr) +{ + struct inode *inode = d_inode(dentry); + int error; + + error = setattr_prepare(&nop_mnt_idmap, dentry, iattr); + if (error) + return error; + + if (is_quota_modification(&nop_mnt_idmap, inode, iattr)) { + error = dquot_initialize(inode); + if (error) + return error; + } + if (i_uid_needs_update(&nop_mnt_idmap, iattr, inode) || + i_gid_needs_update(&nop_mnt_idmap, iattr, inode)) { + error = dquot_transfer(&nop_mnt_idmap, inode, iattr); + if (error) + return error; + } + if (iattr->ia_valid & ATTR_SIZE && iattr->ia_size != inode->i_size) { + error = ext2_setsize(inode, iattr->ia_size); + if (error) + return error; + } + setattr_copy(&nop_mnt_idmap, inode, iattr); + if (iattr->ia_valid & ATTR_MODE) + error = posix_acl_chmod(&nop_mnt_idmap, dentry, inode->i_mode); + mark_inode_dirty(inode); + + return error; +} diff --git a/fs/ext2/ioctl.c b/fs/ext2/ioctl.c new file mode 100644 index 000000000..c3fea55b8 --- /dev/null +++ b/fs/ext2/ioctl.c @@ -0,0 +1,159 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext2/ioctl.c + * + * Copyright (C) 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + */ + +#include "ext2.h" +#include +#include +#include +#include +#include +#include +#include +#include + +int ext2_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + struct ext2_inode_info *ei = EXT2_I(d_inode(dentry)); + + fileattr_fill_flags(fa, ei->i_flags & EXT2_FL_USER_VISIBLE); + + return 0; +} + +int ext2_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + struct ext2_inode_info *ei = EXT2_I(inode); + + if (fileattr_has_fsx(fa)) + return -EOPNOTSUPP; + + /* Is it quota file? Do not allow user to mess with it */ + if (IS_NOQUOTA(inode)) + return -EPERM; + + ei->i_flags = (ei->i_flags & ~EXT2_FL_USER_MODIFIABLE) | + (fa->flags & EXT2_FL_USER_MODIFIABLE); + + ext2_set_inode_flags(inode); + inode_set_ctime_current(inode); + mark_inode_dirty(inode); + + return 0; +} + + +long ext2_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct ext2_inode_info *ei = EXT2_I(inode); + unsigned short rsv_window_size; + int ret; + + ext2_debug ("cmd = %u, arg = %lu\n", cmd, arg); + + switch (cmd) { + case EXT2_IOC_GETVERSION: + return put_user(inode->i_generation, (int __user *) arg); + case EXT2_IOC_SETVERSION: { + __u32 generation; + + if (!inode_owner_or_capable(&nop_mnt_idmap, inode)) + return -EPERM; + ret = mnt_want_write_file(filp); + if (ret) + return ret; + if (get_user(generation, (int __user *) arg)) { + ret = -EFAULT; + goto setversion_out; + } + + inode_lock(inode); + inode_set_ctime_current(inode); + inode->i_generation = generation; + inode_unlock(inode); + + mark_inode_dirty(inode); +setversion_out: + mnt_drop_write_file(filp); + return ret; + } + case EXT2_IOC_GETRSVSZ: + if (test_opt(inode->i_sb, RESERVATION) + && S_ISREG(inode->i_mode) + && ei->i_block_alloc_info) { + rsv_window_size = ei->i_block_alloc_info->rsv_window_node.rsv_goal_size; + return put_user(rsv_window_size, (int __user *)arg); + } + return -ENOTTY; + case EXT2_IOC_SETRSVSZ: { + + if (!test_opt(inode->i_sb, RESERVATION) ||!S_ISREG(inode->i_mode)) + return -ENOTTY; + + if (!inode_owner_or_capable(&nop_mnt_idmap, inode)) + return -EACCES; + + if (get_user(rsv_window_size, (int __user *)arg)) + return -EFAULT; + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + + if (rsv_window_size > EXT2_MAX_RESERVE_BLOCKS) + rsv_window_size = EXT2_MAX_RESERVE_BLOCKS; + + /* + * need to allocate reservation structure for this inode + * before set the window size + */ + /* + * XXX What lock should protect the rsv_goal_size? + * Accessed in ext2_get_block only. ext3 uses i_truncate. + */ + mutex_lock(&ei->truncate_mutex); + if (!ei->i_block_alloc_info) + ext2_init_block_alloc_info(inode); + + if (ei->i_block_alloc_info){ + struct ext2_reserve_window_node *rsv = &ei->i_block_alloc_info->rsv_window_node; + rsv->rsv_goal_size = rsv_window_size; + } else { + ret = -ENOMEM; + } + + mutex_unlock(&ei->truncate_mutex); + mnt_drop_write_file(filp); + return ret; + } + default: + return -ENOTTY; + } +} + +#ifdef CONFIG_COMPAT +long ext2_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + /* These are just misnamed, they actually get/put from/to user an int */ + switch (cmd) { + case EXT2_IOC32_GETVERSION: + cmd = EXT2_IOC_GETVERSION; + break; + case EXT2_IOC32_SETVERSION: + cmd = EXT2_IOC_SETVERSION; + break; + default: + return -ENOIOCTLCMD; + } + return ext2_ioctl(file, cmd, (unsigned long) compat_ptr(arg)); +} +#endif diff --git a/fs/ext2/namei.c b/fs/ext2/namei.c new file mode 100644 index 000000000..8666233ec --- /dev/null +++ b/fs/ext2/namei.c @@ -0,0 +1,434 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext2/namei.c + * + * Rewrite to pagecache. Almost all code had been changed, so blame me + * if the things go wrong. Please, send bug reports to + * viro@parcelfarce.linux.theplanet.co.uk + * + * Stuff here is basically a glue between the VFS and generic UNIXish + * filesystem that keeps everything in pagecache. All knowledge of the + * directory layout is in fs/ext2/dir.c - it turned out to be easily separatable + * and it's easier to debug that way. In principle we might want to + * generalize that a bit and turn it into a library. Or not. + * + * The only non-static object here is ext2_dir_inode_operations. + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * from + * + * linux/fs/minix/namei.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * Big-endian to little-endian byte-swapping/bitmaps by + * David S. Miller (davem@caip.rutgers.edu), 1995 + */ + +#include +#include +#include "ext2.h" +#include "xattr.h" +#include "acl.h" + +static inline int ext2_add_nondir(struct dentry *dentry, struct inode *inode) +{ + int err = ext2_add_link(dentry, inode); + if (!err) { + d_instantiate_new(dentry, inode); + return 0; + } + inode_dec_link_count(inode); + discard_new_inode(inode); + return err; +} + +/* + * Methods themselves. + */ + +static struct dentry *ext2_lookup(struct inode * dir, struct dentry *dentry, unsigned int flags) +{ + struct inode * inode; + ino_t ino; + int res; + + if (dentry->d_name.len > EXT2_NAME_LEN) + return ERR_PTR(-ENAMETOOLONG); + + res = ext2_inode_by_name(dir, &dentry->d_name, &ino); + if (res) { + if (res != -ENOENT) + return ERR_PTR(res); + inode = NULL; + } else { + inode = ext2_iget(dir->i_sb, ino); + if (inode == ERR_PTR(-ESTALE)) { + ext2_error(dir->i_sb, __func__, + "deleted inode referenced: %lu", + (unsigned long) ino); + return ERR_PTR(-EIO); + } + } + return d_splice_alias(inode, dentry); +} + +struct dentry *ext2_get_parent(struct dentry *child) +{ + ino_t ino; + int res; + + res = ext2_inode_by_name(d_inode(child), &dotdot_name, &ino); + if (res) + return ERR_PTR(res); + + return d_obtain_alias(ext2_iget(child->d_sb, ino)); +} + +/* + * By the time this is called, we already have created + * the directory cache entry for the new file, but it + * is so far negative - it has no inode. + * + * If the create succeeds, we fill in the inode information + * with d_instantiate(). + */ +static int ext2_create (struct mnt_idmap * idmap, + struct inode * dir, struct dentry * dentry, + umode_t mode) +{ + struct inode *inode; + int err; + + err = dquot_initialize(dir); + if (err) + return err; + + inode = ext2_new_inode(dir, mode, &dentry->d_name); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + ext2_set_file_ops(inode); + mark_inode_dirty(inode); + return ext2_add_nondir(dentry, inode); +} + +static int ext2_tmpfile(struct mnt_idmap *idmap, struct inode *dir, + struct file *file, umode_t mode) +{ + struct inode *inode = ext2_new_inode(dir, mode, NULL); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + ext2_set_file_ops(inode); + mark_inode_dirty(inode); + d_tmpfile(file, inode); + unlock_new_inode(inode); + return finish_open_simple(file, 0); +} + +static int ext2_mknod (struct mnt_idmap * idmap, struct inode * dir, + struct dentry *dentry, umode_t mode, dev_t rdev) +{ + struct inode * inode; + int err; + + err = dquot_initialize(dir); + if (err) + return err; + + inode = ext2_new_inode (dir, mode, &dentry->d_name); + err = PTR_ERR(inode); + if (!IS_ERR(inode)) { + init_special_inode(inode, inode->i_mode, rdev); + inode->i_op = &ext2_special_inode_operations; + mark_inode_dirty(inode); + err = ext2_add_nondir(dentry, inode); + } + return err; +} + +static int ext2_symlink (struct mnt_idmap * idmap, struct inode * dir, + struct dentry * dentry, const char * symname) +{ + struct super_block * sb = dir->i_sb; + int err = -ENAMETOOLONG; + unsigned l = strlen(symname)+1; + struct inode * inode; + + if (l > sb->s_blocksize) + goto out; + + err = dquot_initialize(dir); + if (err) + goto out; + + inode = ext2_new_inode (dir, S_IFLNK | S_IRWXUGO, &dentry->d_name); + err = PTR_ERR(inode); + if (IS_ERR(inode)) + goto out; + + if (l > sizeof (EXT2_I(inode)->i_data)) { + /* slow symlink */ + inode->i_op = &ext2_symlink_inode_operations; + inode_nohighmem(inode); + inode->i_mapping->a_ops = &ext2_aops; + err = page_symlink(inode, symname, l); + if (err) + goto out_fail; + } else { + /* fast symlink */ + inode->i_op = &ext2_fast_symlink_inode_operations; + inode->i_link = (char*)EXT2_I(inode)->i_data; + memcpy(inode->i_link, symname, l); + inode->i_size = l-1; + } + mark_inode_dirty(inode); + + err = ext2_add_nondir(dentry, inode); +out: + return err; + +out_fail: + inode_dec_link_count(inode); + discard_new_inode(inode); + goto out; +} + +static int ext2_link (struct dentry * old_dentry, struct inode * dir, + struct dentry *dentry) +{ + struct inode *inode = d_inode(old_dentry); + int err; + + err = dquot_initialize(dir); + if (err) + return err; + + inode_set_ctime_current(inode); + inode_inc_link_count(inode); + ihold(inode); + + err = ext2_add_link(dentry, inode); + if (!err) { + d_instantiate(dentry, inode); + return 0; + } + inode_dec_link_count(inode); + iput(inode); + return err; +} + +static struct dentry *ext2_mkdir(struct mnt_idmap * idmap, + struct inode * dir, struct dentry * dentry, + umode_t mode) +{ + struct inode * inode; + int err; + + err = dquot_initialize(dir); + if (err) + return ERR_PTR(err); + + inode_inc_link_count(dir); + + inode = ext2_new_inode(dir, mode, &dentry->d_name); + err = PTR_ERR(inode); + if (IS_ERR(inode)) + goto out_dir; + + inode->i_op = &ext2_dir_inode_operations; + inode->i_fop = &ext2_dir_operations; + inode->i_mapping->a_ops = &ext2_aops; + + inode_inc_link_count(inode); + + err = ext2_make_empty(inode, dir); + if (err) + goto out_fail; + + err = ext2_add_link(dentry, inode); + if (err) + goto out_fail; + + d_instantiate_new(dentry, inode); +out: + return ERR_PTR(err); + +out_fail: + inode_dec_link_count(inode); + inode_dec_link_count(inode); + discard_new_inode(inode); +out_dir: + inode_dec_link_count(dir); + goto out; +} + +static int ext2_unlink(struct inode *dir, struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + struct ext2_dir_entry_2 *de; + struct folio *folio; + int err; + + err = dquot_initialize(dir); + if (err) + goto out; + + de = ext2_find_entry(dir, &dentry->d_name, &folio); + if (IS_ERR(de)) { + err = PTR_ERR(de); + goto out; + } + + err = ext2_delete_entry(de, folio); + folio_release_kmap(folio, de); + if (err) + goto out; + + inode_set_ctime_to_ts(inode, inode_get_ctime(dir)); + + if (inode->i_nlink) + inode_dec_link_count(inode); + + err = 0; +out: + return err; +} + +static int ext2_rmdir (struct inode * dir, struct dentry *dentry) +{ + struct inode * inode = d_inode(dentry); + int err = -ENOTEMPTY; + + if (ext2_empty_dir(inode)) { + err = ext2_unlink(dir, dentry); + if (!err) { + inode->i_size = 0; + inode_dec_link_count(inode); + inode_dec_link_count(dir); + } + } + return err; +} + +static int ext2_rename (struct mnt_idmap * idmap, + struct inode * old_dir, struct dentry * old_dentry, + struct inode * new_dir, struct dentry * new_dentry, + unsigned int flags) +{ + struct inode * old_inode = d_inode(old_dentry); + struct inode * new_inode = d_inode(new_dentry); + struct folio *dir_folio = NULL; + struct ext2_dir_entry_2 * dir_de = NULL; + struct folio * old_folio; + struct ext2_dir_entry_2 * old_de; + bool old_is_dir = S_ISDIR(old_inode->i_mode); + int err; + + if (flags & ~RENAME_NOREPLACE) + return -EINVAL; + + err = dquot_initialize(old_dir); + if (err) + return err; + + err = dquot_initialize(new_dir); + if (err) + return err; + + old_de = ext2_find_entry(old_dir, &old_dentry->d_name, &old_folio); + if (IS_ERR(old_de)) + return PTR_ERR(old_de); + + if (old_is_dir && old_dir != new_dir) { + err = -EIO; + dir_de = ext2_dotdot(old_inode, &dir_folio); + if (!dir_de) + goto out_old; + } + + if (new_inode) { + struct folio *new_folio; + struct ext2_dir_entry_2 *new_de; + + err = -ENOTEMPTY; + if (old_is_dir && !ext2_empty_dir(new_inode)) + goto out_dir; + + new_de = ext2_find_entry(new_dir, &new_dentry->d_name, + &new_folio); + if (IS_ERR(new_de)) { + err = PTR_ERR(new_de); + goto out_dir; + } + err = ext2_set_link(new_dir, new_de, new_folio, old_inode, true); + folio_release_kmap(new_folio, new_de); + if (err) + goto out_dir; + inode_set_ctime_current(new_inode); + if (old_is_dir) + drop_nlink(new_inode); + inode_dec_link_count(new_inode); + } else { + err = ext2_add_link(new_dentry, old_inode); + if (err) + goto out_dir; + if (old_is_dir) + inode_inc_link_count(new_dir); + } + + /* + * Like most other Unix systems, set the ctime for inodes on a + * rename. + */ + inode_set_ctime_current(old_inode); + mark_inode_dirty(old_inode); + + err = ext2_delete_entry(old_de, old_folio); + if (!err && old_is_dir) { + if (old_dir != new_dir) + err = ext2_set_link(old_inode, dir_de, dir_folio, + new_dir, false); + + inode_dec_link_count(old_dir); + } +out_dir: + if (dir_de) + folio_release_kmap(dir_folio, dir_de); +out_old: + folio_release_kmap(old_folio, old_de); + return err; +} + +const struct inode_operations ext2_dir_inode_operations = { + .create = ext2_create, + .lookup = ext2_lookup, + .link = ext2_link, + .unlink = ext2_unlink, + .symlink = ext2_symlink, + .mkdir = ext2_mkdir, + .rmdir = ext2_rmdir, + .mknod = ext2_mknod, + .rename = ext2_rename, + .listxattr = ext2_listxattr, + .getattr = ext2_getattr, + .setattr = ext2_setattr, + .get_inode_acl = ext2_get_acl, + .set_acl = ext2_set_acl, + .tmpfile = ext2_tmpfile, + .fileattr_get = ext2_fileattr_get, + .fileattr_set = ext2_fileattr_set, +}; + +const struct inode_operations ext2_special_inode_operations = { + .listxattr = ext2_listxattr, + .getattr = ext2_getattr, + .setattr = ext2_setattr, + .get_inode_acl = ext2_get_acl, + .set_acl = ext2_set_acl, +}; diff --git a/fs/ext2/super.c b/fs/ext2/super.c new file mode 100644 index 000000000..a40f53087 --- /dev/null +++ b/fs/ext2/super.c @@ -0,0 +1,1701 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/ext2/super.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * from + * + * linux/fs/minix/inode.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * Big-endian to little-endian byte-swapping/bitmaps by + * David S. Miller (davem@caip.rutgers.edu), 1995 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ext2.h" +#include "xattr.h" +#include "acl.h" + +static void ext2_write_super(struct super_block *sb); +static int ext2_statfs (struct dentry * dentry, struct kstatfs * buf); +static int ext2_sync_fs(struct super_block *sb, int wait); +static int ext2_freeze(struct super_block *sb); +static int ext2_unfreeze(struct super_block *sb); + +void ext2_error(struct super_block *sb, const char *function, + const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + struct ext2_sb_info *sbi = EXT2_SB(sb); + struct ext2_super_block *es = sbi->s_es; + + if (!sb_rdonly(sb)) { + spin_lock(&sbi->s_lock); + sbi->s_mount_state |= EXT2_ERROR_FS; + es->s_state |= cpu_to_le16(EXT2_ERROR_FS); + spin_unlock(&sbi->s_lock); + ext2_sync_super(sb, es, 1); + } + + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + printk(KERN_CRIT "EXT2-fs (%s): error: %s: %pV\n", + sb->s_id, function, &vaf); + + va_end(args); + + if (test_opt(sb, ERRORS_PANIC)) + panic("EXT2-fs: panic from previous error\n"); + if (!sb_rdonly(sb) && test_opt(sb, ERRORS_RO)) { + ext2_msg(sb, KERN_CRIT, + "error: remounting filesystem read-only"); + sb->s_flags |= SB_RDONLY; + } +} + +static void ext2_msg_fc(struct fs_context *fc, const char *prefix, + const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + const char *s_id; + + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE) { + s_id = fc->root->d_sb->s_id; + } else { + /* get last path component of source */ + s_id = strrchr(fc->source, '/'); + if (s_id) + s_id++; + else + s_id = fc->source; + } + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + printk("%sEXT2-fs (%s): %pV\n", prefix, s_id, &vaf); + + va_end(args); +} + +void ext2_msg(struct super_block *sb, const char *prefix, + const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + printk("%sEXT2-fs (%s): %pV\n", prefix, sb->s_id, &vaf); + + va_end(args); +} + +/* + * This must be called with sbi->s_lock held. + */ +void ext2_update_dynamic_rev(struct super_block *sb) +{ + struct ext2_super_block *es = EXT2_SB(sb)->s_es; + + if (le32_to_cpu(es->s_rev_level) > EXT2_GOOD_OLD_REV) + return; + + ext2_msg(sb, KERN_WARNING, + "warning: updating to rev %d because of " + "new feature flag, running e2fsck is recommended", + EXT2_DYNAMIC_REV); + + es->s_first_ino = cpu_to_le32(EXT2_GOOD_OLD_FIRST_INO); + es->s_inode_size = cpu_to_le16(EXT2_GOOD_OLD_INODE_SIZE); + es->s_rev_level = cpu_to_le32(EXT2_DYNAMIC_REV); + /* leave es->s_feature_*compat flags alone */ + /* es->s_uuid will be set by e2fsck if empty */ + + /* + * The rest of the superblock fields should be zero, and if not it + * means they are likely already in use, so leave them alone. We + * can leave it up to e2fsck to clean up any inconsistencies there. + */ +} + +#ifdef CONFIG_QUOTA +static int ext2_quota_off(struct super_block *sb, int type); + +static void ext2_quota_off_umount(struct super_block *sb) +{ + int type; + + for (type = 0; type < MAXQUOTAS; type++) + ext2_quota_off(sb, type); +} +#else +static inline void ext2_quota_off_umount(struct super_block *sb) +{ +} +#endif + +static void ext2_put_super (struct super_block * sb) +{ + int db_count; + int i; + struct ext2_sb_info *sbi = EXT2_SB(sb); + + ext2_quota_off_umount(sb); + + ext2_xattr_destroy_cache(sbi->s_ea_block_cache); + sbi->s_ea_block_cache = NULL; + + if (!sb_rdonly(sb)) { + struct ext2_super_block *es = sbi->s_es; + + spin_lock(&sbi->s_lock); + es->s_state = cpu_to_le16(sbi->s_mount_state); + spin_unlock(&sbi->s_lock); + ext2_sync_super(sb, es, 1); + } + db_count = sbi->s_gdb_count; + for (i = 0; i < db_count; i++) + brelse(sbi->s_group_desc[i]); + kvfree(sbi->s_group_desc); + kfree(sbi->s_debts); + percpu_counter_destroy(&sbi->s_freeblocks_counter); + percpu_counter_destroy(&sbi->s_freeinodes_counter); + percpu_counter_destroy(&sbi->s_dirs_counter); + brelse (sbi->s_sbh); + sb->s_fs_info = NULL; + kfree(sbi->s_blockgroup_lock); + kfree(sbi); +} + +static struct kmem_cache * ext2_inode_cachep; + +static struct inode *ext2_alloc_inode(struct super_block *sb) +{ + struct ext2_inode_info *ei; + ei = alloc_inode_sb(sb, ext2_inode_cachep, GFP_KERNEL); + if (!ei) + return NULL; + ei->i_block_alloc_info = NULL; + inode_set_iversion(&ei->vfs_inode, 1); +#ifdef CONFIG_QUOTA + memset(&ei->i_dquot, 0, sizeof(ei->i_dquot)); +#endif + mmb_init(&ei->i_metadata_bhs, &ei->vfs_inode.i_data); + + return &ei->vfs_inode; +} + +static void ext2_free_in_core_inode(struct inode *inode) +{ + kmem_cache_free(ext2_inode_cachep, EXT2_I(inode)); +} + +static void init_once(void *foo) +{ + struct ext2_inode_info *ei = (struct ext2_inode_info *) foo; + + rwlock_init(&ei->i_meta_lock); +#ifdef CONFIG_EXT2_FS_XATTR + init_rwsem(&ei->xattr_sem); +#endif + mutex_init(&ei->truncate_mutex); + inode_init_once(&ei->vfs_inode); +} + +static int __init init_inodecache(void) +{ + ext2_inode_cachep = kmem_cache_create_usercopy("ext2_inode_cache", + sizeof(struct ext2_inode_info), 0, + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, + offsetof(struct ext2_inode_info, i_data), + sizeof_field(struct ext2_inode_info, i_data), + init_once); + if (ext2_inode_cachep == NULL) + return -ENOMEM; + return 0; +} + +static void destroy_inodecache(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(ext2_inode_cachep); +} + +static int ext2_show_options(struct seq_file *seq, struct dentry *root) +{ + struct super_block *sb = root->d_sb; + struct ext2_sb_info *sbi = EXT2_SB(sb); + struct ext2_super_block *es = sbi->s_es; + unsigned long def_mount_opts; + + spin_lock(&sbi->s_lock); + def_mount_opts = le32_to_cpu(es->s_default_mount_opts); + + if (sbi->s_sb_block != 1) + seq_printf(seq, ",sb=%lu", sbi->s_sb_block); + if (test_opt(sb, MINIX_DF)) + seq_puts(seq, ",minixdf"); + if (test_opt(sb, GRPID)) + seq_puts(seq, ",grpid"); + if (!test_opt(sb, GRPID) && (def_mount_opts & EXT2_DEFM_BSDGROUPS)) + seq_puts(seq, ",nogrpid"); + if (!uid_eq(sbi->s_resuid, make_kuid(&init_user_ns, EXT2_DEF_RESUID)) || + le16_to_cpu(es->s_def_resuid) != EXT2_DEF_RESUID) { + seq_printf(seq, ",resuid=%u", + from_kuid_munged(&init_user_ns, sbi->s_resuid)); + } + if (!gid_eq(sbi->s_resgid, make_kgid(&init_user_ns, EXT2_DEF_RESGID)) || + le16_to_cpu(es->s_def_resgid) != EXT2_DEF_RESGID) { + seq_printf(seq, ",resgid=%u", + from_kgid_munged(&init_user_ns, sbi->s_resgid)); + } + if (test_opt(sb, ERRORS_RO)) { + int def_errors = le16_to_cpu(es->s_errors); + + if (def_errors == EXT2_ERRORS_PANIC || + def_errors == EXT2_ERRORS_CONTINUE) { + seq_puts(seq, ",errors=remount-ro"); + } + } + if (test_opt(sb, ERRORS_CONT)) + seq_puts(seq, ",errors=continue"); + if (test_opt(sb, ERRORS_PANIC)) + seq_puts(seq, ",errors=panic"); + if (test_opt(sb, NO_UID32)) + seq_puts(seq, ",nouid32"); + if (test_opt(sb, DEBUG)) + seq_puts(seq, ",debug"); + if (test_opt(sb, OLDALLOC)) + seq_puts(seq, ",oldalloc"); + +#ifdef CONFIG_EXT2_FS_XATTR + if (test_opt(sb, XATTR_USER)) + seq_puts(seq, ",user_xattr"); + if (!test_opt(sb, XATTR_USER) && + (def_mount_opts & EXT2_DEFM_XATTR_USER)) { + seq_puts(seq, ",nouser_xattr"); + } +#endif + +#ifdef CONFIG_EXT2_FS_POSIX_ACL + if (test_opt(sb, POSIX_ACL)) + seq_puts(seq, ",acl"); + if (!test_opt(sb, POSIX_ACL) && (def_mount_opts & EXT2_DEFM_ACL)) + seq_puts(seq, ",noacl"); +#endif + + if (test_opt(sb, USRQUOTA)) + seq_puts(seq, ",usrquota"); + + if (test_opt(sb, GRPQUOTA)) + seq_puts(seq, ",grpquota"); + + + + if (!test_opt(sb, RESERVATION)) + seq_puts(seq, ",noreservation"); + + spin_unlock(&sbi->s_lock); + return 0; +} + +#ifdef CONFIG_QUOTA +static ssize_t ext2_quota_read(struct super_block *sb, int type, char *data, size_t len, loff_t off); +static ssize_t ext2_quota_write(struct super_block *sb, int type, const char *data, size_t len, loff_t off); +static int ext2_quota_on(struct super_block *sb, int type, int format_id, + const struct path *path); +static struct dquot __rcu **ext2_get_dquots(struct inode *inode) +{ + return EXT2_I(inode)->i_dquot; +} + +static const struct quotactl_ops ext2_quotactl_ops = { + .quota_on = ext2_quota_on, + .quota_off = ext2_quota_off, + .quota_sync = dquot_quota_sync, + .get_state = dquot_get_state, + .set_info = dquot_set_dqinfo, + .get_dqblk = dquot_get_dqblk, + .set_dqblk = dquot_set_dqblk, + .get_nextdqblk = dquot_get_next_dqblk, +}; +#endif + +static const struct super_operations ext2_sops = { + .alloc_inode = ext2_alloc_inode, + .free_inode = ext2_free_in_core_inode, + .write_inode = ext2_write_inode, + .sync_inode_metadata = ext2_sync_inode_metadata, + .evict_inode = ext2_evict_inode, + .put_super = ext2_put_super, + .sync_fs = ext2_sync_fs, + .freeze_fs = ext2_freeze, + .unfreeze_fs = ext2_unfreeze, + .statfs = ext2_statfs, + .show_options = ext2_show_options, +#ifdef CONFIG_QUOTA + .quota_read = ext2_quota_read, + .quota_write = ext2_quota_write, + .get_dquots = ext2_get_dquots, +#endif +}; + +static struct inode *ext2_nfs_get_inode(struct super_block *sb, + u64 ino, u32 generation) +{ + struct inode *inode; + + if (ino < EXT2_FIRST_INO(sb) && ino != EXT2_ROOT_INO) + return ERR_PTR(-ESTALE); + if (ino > le32_to_cpu(EXT2_SB(sb)->s_es->s_inodes_count)) + return ERR_PTR(-ESTALE); + + /* + * ext2_iget isn't quite right if the inode is currently unallocated! + * However ext2_iget currently does appropriate checks to handle stale + * inodes so everything is OK. + */ + inode = ext2_iget(sb, ino); + if (IS_ERR(inode)) + return ERR_CAST(inode); + if (generation && inode->i_generation != generation) { + /* we didn't find the right inode.. */ + iput(inode); + return ERR_PTR(-ESTALE); + } + return inode; +} + +static struct dentry *ext2_fh_to_dentry(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + return generic_fh_to_dentry(sb, fid, fh_len, fh_type, + ext2_nfs_get_inode); +} + +static struct dentry *ext2_fh_to_parent(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + return generic_fh_to_parent(sb, fid, fh_len, fh_type, + ext2_nfs_get_inode); +} + +static const struct export_operations ext2_export_ops = { + .encode_fh = generic_encode_ino32_fh, + .fh_to_dentry = ext2_fh_to_dentry, + .fh_to_parent = ext2_fh_to_parent, + .get_parent = ext2_get_parent, +}; + +enum { + Opt_bsd_df, Opt_minix_df, Opt_grpid, Opt_nogrpid, Opt_resgid, Opt_resuid, + Opt_sb, Opt_errors, Opt_nouid32, Opt_debug, Opt_oldalloc, Opt_orlov, + Opt_nobh, Opt_user_xattr, Opt_acl, Opt_xip, Opt_dax, Opt_ignore, + Opt_quota, Opt_usrquota, Opt_grpquota, Opt_reservation, +}; + +static const struct constant_table ext2_param_errors[] = { + {"continue", EXT2_MOUNT_ERRORS_CONT}, + {"panic", EXT2_MOUNT_ERRORS_PANIC}, + {"remount-ro", EXT2_MOUNT_ERRORS_RO}, + {} +}; + +static const struct fs_parameter_spec ext2_param_spec[] = { + fsparam_flag ("bsddf", Opt_bsd_df), + fsparam_flag ("minixdf", Opt_minix_df), + fsparam_flag ("grpid", Opt_grpid), + fsparam_flag ("bsdgroups", Opt_grpid), + fsparam_flag ("nogrpid", Opt_nogrpid), + fsparam_flag ("sysvgroups", Opt_nogrpid), + fsparam_gid ("resgid", Opt_resgid), + fsparam_uid ("resuid", Opt_resuid), + fsparam_u32 ("sb", Opt_sb), + fsparam_enum ("errors", Opt_errors, ext2_param_errors), + fsparam_flag ("nouid32", Opt_nouid32), + fsparam_flag ("debug", Opt_debug), + fsparam_flag ("oldalloc", Opt_oldalloc), + fsparam_flag ("orlov", Opt_orlov), + fsparam_flag ("nobh", Opt_nobh), + fsparam_flag_no ("user_xattr", Opt_user_xattr), + fsparam_flag_no ("acl", Opt_acl), + fsparam_flag ("xip", Opt_xip), + fsparam_flag ("dax", Opt_dax), + fsparam_flag ("grpquota", Opt_grpquota), + fsparam_flag ("noquota", Opt_ignore), + fsparam_flag ("quota", Opt_quota), + fsparam_flag ("usrquota", Opt_usrquota), + fsparam_flag_no ("reservation", Opt_reservation), + {} +}; + +#define EXT2_SPEC_s_resuid (1 << 0) +#define EXT2_SPEC_s_resgid (1 << 1) + +struct ext2_fs_context { + unsigned long vals_s_flags; /* Bits to set in s_flags */ + unsigned long mask_s_flags; /* Bits changed in s_flags */ + unsigned int vals_s_mount_opt; + unsigned int mask_s_mount_opt; + kuid_t s_resuid; + kgid_t s_resgid; + unsigned long s_sb_block; + unsigned int spec; + +}; + +static inline void ctx_set_mount_opt(struct ext2_fs_context *ctx, + unsigned long flag) +{ + ctx->mask_s_mount_opt |= flag; + ctx->vals_s_mount_opt |= flag; +} + +static inline void ctx_clear_mount_opt(struct ext2_fs_context *ctx, + unsigned long flag) +{ + ctx->mask_s_mount_opt |= flag; + ctx->vals_s_mount_opt &= ~flag; +} + +static inline unsigned long +ctx_test_mount_opt(struct ext2_fs_context *ctx, unsigned long flag) +{ + return (ctx->vals_s_mount_opt & flag); +} + +static inline bool +ctx_parsed_mount_opt(struct ext2_fs_context *ctx, unsigned long flag) +{ + return (ctx->mask_s_mount_opt & flag); +} + +static void ext2_free_fc(struct fs_context *fc) +{ + kfree(fc->fs_private); +} + +static int ext2_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct ext2_fs_context *ctx = fc->fs_private; + int opt; + struct fs_parse_result result; + + opt = fs_parse(fc, ext2_param_spec, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_bsd_df: + ctx_clear_mount_opt(ctx, EXT2_MOUNT_MINIX_DF); + break; + case Opt_minix_df: + ctx_set_mount_opt(ctx, EXT2_MOUNT_MINIX_DF); + break; + case Opt_grpid: + ctx_set_mount_opt(ctx, EXT2_MOUNT_GRPID); + break; + case Opt_nogrpid: + ctx_clear_mount_opt(ctx, EXT2_MOUNT_GRPID); + break; + case Opt_resuid: + ctx->s_resuid = result.uid; + ctx->spec |= EXT2_SPEC_s_resuid; + break; + case Opt_resgid: + ctx->s_resgid = result.gid; + ctx->spec |= EXT2_SPEC_s_resgid; + break; + case Opt_sb: + /* Note that this is silently ignored on remount */ + ctx->s_sb_block = result.uint_32; + break; + case Opt_errors: + ctx_clear_mount_opt(ctx, EXT2_MOUNT_ERRORS_MASK); + ctx_set_mount_opt(ctx, result.uint_32); + break; + case Opt_nouid32: + ctx_set_mount_opt(ctx, EXT2_MOUNT_NO_UID32); + break; + case Opt_debug: + ctx_set_mount_opt(ctx, EXT2_MOUNT_DEBUG); + break; + case Opt_oldalloc: + ctx_set_mount_opt(ctx, EXT2_MOUNT_OLDALLOC); + break; + case Opt_orlov: + ctx_clear_mount_opt(ctx, EXT2_MOUNT_OLDALLOC); + break; + case Opt_nobh: + ext2_msg_fc(fc, KERN_INFO, "nobh option not supported\n"); + break; +#ifdef CONFIG_EXT2_FS_XATTR + case Opt_user_xattr: + if (!result.negated) + ctx_set_mount_opt(ctx, EXT2_MOUNT_XATTR_USER); + else + ctx_clear_mount_opt(ctx, EXT2_MOUNT_XATTR_USER); + break; +#else + case Opt_user_xattr: + ext2_msg_fc(fc, KERN_INFO, "(no)user_xattr options not supported"); + break; +#endif +#ifdef CONFIG_EXT2_FS_POSIX_ACL + case Opt_acl: + if (!result.negated) + ctx_set_mount_opt(ctx, EXT2_MOUNT_POSIX_ACL); + else + ctx_clear_mount_opt(ctx, EXT2_MOUNT_POSIX_ACL); + break; +#else + case Opt_acl: + ext2_msg_fc(fc, KERN_INFO, "(no)acl options not supported"); + break; +#endif + case Opt_xip: + ext2_msg_fc(fc, KERN_WARNING, "DAX support has been removed. xip option ignored."); + break; + case Opt_dax: + ext2_msg_fc(fc, KERN_WARNING, "DAX support has been removed. dax option ignored."); + break; + +#if defined(CONFIG_QUOTA) + case Opt_quota: + case Opt_usrquota: + ctx_set_mount_opt(ctx, EXT2_MOUNT_USRQUOTA); + break; + + case Opt_grpquota: + ctx_set_mount_opt(ctx, EXT2_MOUNT_GRPQUOTA); + break; +#else + case Opt_quota: + case Opt_usrquota: + case Opt_grpquota: + ext2_msg_fc(fc, KERN_INFO, "quota operations not supported"); + break; +#endif + case Opt_reservation: + if (!result.negated) { + ctx_set_mount_opt(ctx, EXT2_MOUNT_RESERVATION); + ext2_msg_fc(fc, KERN_INFO, "reservations ON"); + } else { + ctx_clear_mount_opt(ctx, EXT2_MOUNT_RESERVATION); + ext2_msg_fc(fc, KERN_INFO, "reservations OFF"); + } + break; + case Opt_ignore: + break; + default: + return -EINVAL; + } + return 0; +} + +static int ext2_setup_super (struct super_block * sb, + struct ext2_super_block * es, + int read_only) +{ + int res = 0; + struct ext2_sb_info *sbi = EXT2_SB(sb); + + if (le32_to_cpu(es->s_rev_level) > EXT2_MAX_SUPP_REV) { + ext2_msg(sb, KERN_ERR, + "error: revision level too high, " + "forcing read-only mode"); + res = SB_RDONLY; + } + if (read_only) + return res; + if (!(sbi->s_mount_state & EXT2_VALID_FS)) + ext2_msg(sb, KERN_WARNING, + "warning: mounting unchecked fs, " + "running e2fsck is recommended"); + else if ((sbi->s_mount_state & EXT2_ERROR_FS)) + ext2_msg(sb, KERN_WARNING, + "warning: mounting fs with errors, " + "running e2fsck is recommended"); + else if ((__s16) le16_to_cpu(es->s_max_mnt_count) >= 0 && + le16_to_cpu(es->s_mnt_count) >= + (unsigned short) (__s16) le16_to_cpu(es->s_max_mnt_count)) + ext2_msg(sb, KERN_WARNING, + "warning: maximal mount count reached, " + "running e2fsck is recommended"); + else if (le32_to_cpu(es->s_checkinterval) && + (le32_to_cpu(es->s_lastcheck) + + le32_to_cpu(es->s_checkinterval) <= + ktime_get_real_seconds())) + ext2_msg(sb, KERN_WARNING, + "warning: checktime reached, " + "running e2fsck is recommended"); + if (!le16_to_cpu(es->s_max_mnt_count)) + es->s_max_mnt_count = cpu_to_le16(EXT2_DFL_MAX_MNT_COUNT); + le16_add_cpu(&es->s_mnt_count, 1); + if (test_opt (sb, DEBUG)) + ext2_msg(sb, KERN_INFO, "%s, %s, bs=%lu, gc=%lu, " + "bpg=%lu, ipg=%lu, mo=%04lx]", + EXT2FS_VERSION, EXT2FS_DATE, sb->s_blocksize, + sbi->s_groups_count, + EXT2_BLOCKS_PER_GROUP(sb), + EXT2_INODES_PER_GROUP(sb), + sbi->s_mount_opt); + return res; +} + +static int ext2_check_descriptors(struct super_block *sb) +{ + int i; + struct ext2_sb_info *sbi = EXT2_SB(sb); + + ext2_debug ("Checking group descriptors"); + + for (i = 0; i < sbi->s_groups_count; i++) { + struct ext2_group_desc *gdp = ext2_get_group_desc(sb, i, NULL); + ext2_fsblk_t first_block = ext2_group_first_block_no(sb, i); + ext2_fsblk_t last_block = ext2_group_last_block_no(sb, i); + + if (le32_to_cpu(gdp->bg_block_bitmap) < first_block || + le32_to_cpu(gdp->bg_block_bitmap) > last_block) + { + ext2_error (sb, "ext2_check_descriptors", + "Block bitmap for group %d" + " not in group (block %lu)!", + i, (unsigned long) le32_to_cpu(gdp->bg_block_bitmap)); + return 0; + } + if (le32_to_cpu(gdp->bg_inode_bitmap) < first_block || + le32_to_cpu(gdp->bg_inode_bitmap) > last_block) + { + ext2_error (sb, "ext2_check_descriptors", + "Inode bitmap for group %d" + " not in group (block %lu)!", + i, (unsigned long) le32_to_cpu(gdp->bg_inode_bitmap)); + return 0; + } + if (le32_to_cpu(gdp->bg_inode_table) < first_block || + le32_to_cpu(gdp->bg_inode_table) + sbi->s_itb_per_group - 1 > + last_block) + { + ext2_error (sb, "ext2_check_descriptors", + "Inode table for group %d" + " not in group (block %lu)!", + i, (unsigned long) le32_to_cpu(gdp->bg_inode_table)); + return 0; + } + } + return 1; +} + +/* + * Maximal file size. There is a direct, and {,double-,triple-}indirect + * block limit, and also a limit of (2^32 - 1) 512-byte sectors in i_blocks. + * We need to be 1 filesystem block less than the 2^32 sector limit. + */ +static loff_t ext2_max_size(int bits) +{ + loff_t res = EXT2_NDIR_BLOCKS; + int meta_blocks; + unsigned int upper_limit; + unsigned int ppb = 1 << (bits-2); + + /* This is calculated to be the largest file size for a + * dense, file such that the total number of + * sectors in the file, including data and all indirect blocks, + * does not exceed 2^32 -1 + * __u32 i_blocks representing the total number of + * 512 bytes blocks of the file + */ + upper_limit = (1LL << 32) - 1; + + /* total blocks in file system block size */ + upper_limit >>= (bits - 9); + + /* Compute how many blocks we can address by block tree */ + res += 1LL << (bits-2); + res += 1LL << (2*(bits-2)); + res += 1LL << (3*(bits-2)); + /* Compute how many metadata blocks are needed */ + meta_blocks = 1; + meta_blocks += 1 + ppb; + meta_blocks += 1 + ppb + ppb * ppb; + /* Does block tree limit file size? */ + if (res + meta_blocks <= upper_limit) + goto check_lfs; + + res = upper_limit; + /* How many metadata blocks are needed for addressing upper_limit? */ + upper_limit -= EXT2_NDIR_BLOCKS; + /* indirect blocks */ + meta_blocks = 1; + upper_limit -= ppb; + /* double indirect blocks */ + if (upper_limit < ppb * ppb) { + meta_blocks += 1 + DIV_ROUND_UP(upper_limit, ppb); + res -= meta_blocks; + goto check_lfs; + } + meta_blocks += 1 + ppb; + upper_limit -= ppb * ppb; + /* tripple indirect blocks for the rest */ + meta_blocks += 1 + DIV_ROUND_UP(upper_limit, ppb) + + DIV_ROUND_UP(upper_limit, ppb*ppb); + res -= meta_blocks; +check_lfs: + res <<= bits; + if (res > MAX_LFS_FILESIZE) + res = MAX_LFS_FILESIZE; + + return res; +} + +static unsigned long descriptor_loc(struct super_block *sb, + unsigned long logic_sb_block, + int nr) +{ + struct ext2_sb_info *sbi = EXT2_SB(sb); + unsigned long bg, first_meta_bg; + + first_meta_bg = le32_to_cpu(sbi->s_es->s_first_meta_bg); + + if (!EXT2_HAS_INCOMPAT_FEATURE(sb, EXT2_FEATURE_INCOMPAT_META_BG) || + nr < first_meta_bg) + return (logic_sb_block + nr + 1); + bg = sbi->s_desc_per_block * nr; + + return ext2_group_first_block_no(sb, bg) + ext2_bg_has_super(sb, bg); +} + +/* + * Set all mount options either from defaults on disk, or from parsed + * options. Parsed/specified options override on-disk defaults. + */ +static void ext2_set_options(struct fs_context *fc, struct ext2_sb_info *sbi) +{ + struct ext2_fs_context *ctx = fc->fs_private; + struct ext2_super_block *es = sbi->s_es; + unsigned long def_mount_opts = le32_to_cpu(es->s_default_mount_opts); + + /* Copy parsed mount options to sbi */ + sbi->s_mount_opt = ctx->vals_s_mount_opt; + + /* Use in-superblock defaults only if not specified during parsing */ + if (!ctx_parsed_mount_opt(ctx, EXT2_MOUNT_DEBUG) && + def_mount_opts & EXT2_DEFM_DEBUG) + set_opt(sbi->s_mount_opt, DEBUG); + + if (!ctx_parsed_mount_opt(ctx, EXT2_MOUNT_GRPID) && + def_mount_opts & EXT2_DEFM_BSDGROUPS) + set_opt(sbi->s_mount_opt, GRPID); + + if (!ctx_parsed_mount_opt(ctx, EXT2_MOUNT_NO_UID32) && + def_mount_opts & EXT2_DEFM_UID16) + set_opt(sbi->s_mount_opt, NO_UID32); + +#ifdef CONFIG_EXT2_FS_XATTR + if (!ctx_parsed_mount_opt(ctx, EXT2_MOUNT_XATTR_USER) && + def_mount_opts & EXT2_DEFM_XATTR_USER) + set_opt(sbi->s_mount_opt, XATTR_USER); +#endif +#ifdef CONFIG_EXT2_FS_POSIX_ACL + if (!ctx_parsed_mount_opt(ctx, EXT2_MOUNT_POSIX_ACL) && + def_mount_opts & EXT2_DEFM_ACL) + set_opt(sbi->s_mount_opt, POSIX_ACL); +#endif + + if (!ctx_parsed_mount_opt(ctx, EXT2_MOUNT_ERRORS_MASK)) { + if (le16_to_cpu(sbi->s_es->s_errors) == EXT2_ERRORS_PANIC) + set_opt(sbi->s_mount_opt, ERRORS_PANIC); + else if (le16_to_cpu(sbi->s_es->s_errors) == EXT2_ERRORS_CONTINUE) + set_opt(sbi->s_mount_opt, ERRORS_CONT); + else + set_opt(sbi->s_mount_opt, ERRORS_RO); + } + + if (ctx->spec & EXT2_SPEC_s_resuid) + sbi->s_resuid = ctx->s_resuid; + else + sbi->s_resuid = make_kuid(&init_user_ns, + le16_to_cpu(es->s_def_resuid)); + + if (ctx->spec & EXT2_SPEC_s_resgid) + sbi->s_resgid = ctx->s_resgid; + else + sbi->s_resgid = make_kgid(&init_user_ns, + le16_to_cpu(es->s_def_resgid)); +} + +static int ext2_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct ext2_fs_context *ctx = fc->fs_private; + int silent = fc->sb_flags & SB_SILENT; + struct buffer_head * bh; + struct ext2_sb_info * sbi; + struct ext2_super_block * es; + struct inode *root; + unsigned long block; + unsigned long sb_block = ctx->s_sb_block; + unsigned long logic_sb_block; + unsigned long offset = 0; + long ret = -ENOMEM; + int blocksize = BLOCK_SIZE; + int db_count; + int i, j; + __le32 features; + int err; + + sbi = kzalloc_obj(*sbi); + if (!sbi) + return -ENOMEM; + + sbi->s_blockgroup_lock = + kzalloc_obj(struct blockgroup_lock); + if (!sbi->s_blockgroup_lock) { + kfree(sbi); + return -ENOMEM; + } + sb->s_fs_info = sbi; + sbi->s_sb_block = sb_block; + + spin_lock_init(&sbi->s_lock); + ret = -EINVAL; + + /* + * See what the current blocksize for the device is, and + * use that as the blocksize. Otherwise (or if the blocksize + * is smaller than the default) use the default. + * This is important for devices that have a hardware + * sectorsize that is larger than the default. + */ + blocksize = sb_min_blocksize(sb, BLOCK_SIZE); + if (!blocksize) { + ext2_msg(sb, KERN_ERR, "error: unable to set blocksize"); + goto failed_sbi; + } + + /* + * If the superblock doesn't start on a hardware sector boundary, + * calculate the offset. + */ + if (blocksize != BLOCK_SIZE) { + logic_sb_block = (sb_block*BLOCK_SIZE) / blocksize; + offset = (sb_block*BLOCK_SIZE) % blocksize; + } else { + logic_sb_block = sb_block; + } + + if (!(bh = sb_bread(sb, logic_sb_block))) { + ext2_msg(sb, KERN_ERR, "error: unable to read superblock"); + goto failed_sbi; + } + /* + * Note: s_es must be initialized as soon as possible because + * some ext2 macro-instructions depend on its value + */ + es = (struct ext2_super_block *) (((char *)bh->b_data) + offset); + sbi->s_es = es; + sb->s_magic = le16_to_cpu(es->s_magic); + + if (sb->s_magic != EXT2_SUPER_MAGIC) + goto cantfind_ext2; + + ext2_set_options(fc, sbi); + + sb->s_flags = (sb->s_flags & ~SB_POSIXACL) | + (test_opt(sb, POSIX_ACL) ? SB_POSIXACL : 0); + sb->s_iflags |= SB_I_CGROUPWB; + + if (le32_to_cpu(es->s_rev_level) == EXT2_GOOD_OLD_REV && + (EXT2_HAS_COMPAT_FEATURE(sb, ~0U) || + EXT2_HAS_RO_COMPAT_FEATURE(sb, ~0U) || + EXT2_HAS_INCOMPAT_FEATURE(sb, ~0U))) + ext2_msg(sb, KERN_WARNING, + "warning: feature flags set on rev 0 fs, " + "running e2fsck is recommended"); + /* + * Check feature flags regardless of the revision level, since we + * previously didn't change the revision level when setting the flags, + * so there is a chance incompat flags are set on a rev 0 filesystem. + */ + features = EXT2_HAS_INCOMPAT_FEATURE(sb, ~EXT2_FEATURE_INCOMPAT_SUPP); + if (features) { + ext2_msg(sb, KERN_ERR, "error: couldn't mount because of " + "unsupported optional features (%x)", + le32_to_cpu(features)); + goto failed_mount; + } + if (!sb_rdonly(sb) && (features = EXT2_HAS_RO_COMPAT_FEATURE(sb, ~EXT2_FEATURE_RO_COMPAT_SUPP))){ + ext2_msg(sb, KERN_ERR, "error: couldn't mount RDWR because of " + "unsupported optional features (%x)", + le32_to_cpu(features)); + goto failed_mount; + } + + if (le32_to_cpu(es->s_log_block_size) > + (EXT2_MAX_BLOCK_LOG_SIZE - BLOCK_SIZE_BITS)) { + ext2_msg(sb, KERN_ERR, + "Invalid log block size: %u", + le32_to_cpu(es->s_log_block_size)); + goto failed_mount; + } + blocksize = BLOCK_SIZE << le32_to_cpu(sbi->s_es->s_log_block_size); + + /* If the blocksize doesn't match, re-read the thing.. */ + if (sb->s_blocksize != blocksize) { + brelse(bh); + + if (!sb_set_blocksize(sb, blocksize)) { + ext2_msg(sb, KERN_ERR, + "error: bad blocksize %d", blocksize); + goto failed_sbi; + } + + logic_sb_block = (sb_block*BLOCK_SIZE) / blocksize; + offset = (sb_block*BLOCK_SIZE) % blocksize; + bh = sb_bread(sb, logic_sb_block); + if(!bh) { + ext2_msg(sb, KERN_ERR, "error: couldn't read" + "superblock on 2nd try"); + goto failed_sbi; + } + es = (struct ext2_super_block *) (((char *)bh->b_data) + offset); + sbi->s_es = es; + if (es->s_magic != cpu_to_le16(EXT2_SUPER_MAGIC)) { + ext2_msg(sb, KERN_ERR, "error: magic mismatch"); + goto failed_mount; + } + } + + sb->s_maxbytes = ext2_max_size(sb->s_blocksize_bits); + sb->s_max_links = EXT2_LINK_MAX; + sb->s_time_min = S32_MIN; + sb->s_time_max = S32_MAX; + + if (le32_to_cpu(es->s_rev_level) == EXT2_GOOD_OLD_REV) { + sbi->s_inode_size = EXT2_GOOD_OLD_INODE_SIZE; + sbi->s_first_ino = EXT2_GOOD_OLD_FIRST_INO; + } else { + sbi->s_inode_size = le16_to_cpu(es->s_inode_size); + sbi->s_first_ino = le32_to_cpu(es->s_first_ino); + if ((sbi->s_inode_size < EXT2_GOOD_OLD_INODE_SIZE) || + !is_power_of_2(sbi->s_inode_size) || + (sbi->s_inode_size > blocksize)) { + ext2_msg(sb, KERN_ERR, + "error: unsupported inode size: %d", + sbi->s_inode_size); + goto failed_mount; + } + } + + sbi->s_blocks_per_group = le32_to_cpu(es->s_blocks_per_group); + sbi->s_inodes_per_group = le32_to_cpu(es->s_inodes_per_group); + + sbi->s_inodes_per_block = sb->s_blocksize / EXT2_INODE_SIZE(sb); + if (sbi->s_inodes_per_block == 0 || sbi->s_inodes_per_group == 0) + goto cantfind_ext2; + sbi->s_itb_per_group = sbi->s_inodes_per_group / + sbi->s_inodes_per_block; + sbi->s_desc_per_block = sb->s_blocksize / + sizeof (struct ext2_group_desc); + sbi->s_sbh = bh; + sbi->s_mount_state = le16_to_cpu(es->s_state); + sbi->s_addr_per_block_bits = + ilog2 (EXT2_ADDR_PER_BLOCK(sb)); + sbi->s_desc_per_block_bits = + ilog2 (EXT2_DESC_PER_BLOCK(sb)); + + if (sb->s_magic != EXT2_SUPER_MAGIC) + goto cantfind_ext2; + + if (sb->s_blocksize != bh->b_size) { + if (!silent) + ext2_msg(sb, KERN_ERR, "error: unsupported blocksize"); + goto failed_mount; + } + + if (es->s_log_frag_size != es->s_log_block_size) { + ext2_msg(sb, KERN_ERR, + "error: fragsize log %u != blocksize log %u", + le32_to_cpu(es->s_log_frag_size), sb->s_blocksize_bits); + goto failed_mount; + } + + if (sbi->s_blocks_per_group > sb->s_blocksize * 8) { + ext2_msg(sb, KERN_ERR, + "error: #blocks per group too big: %lu", + sbi->s_blocks_per_group); + goto failed_mount; + } + /* At least inode table, bitmaps, and sb have to fit in one group */ + if (sbi->s_blocks_per_group <= sbi->s_itb_per_group + 3) { + ext2_msg(sb, KERN_ERR, + "error: #blocks per group smaller than metadata size: %lu <= %lu", + sbi->s_blocks_per_group, sbi->s_inodes_per_group + 3); + goto failed_mount; + } + if (sbi->s_inodes_per_group < sbi->s_inodes_per_block || + sbi->s_inodes_per_group > sb->s_blocksize * 8) { + ext2_msg(sb, KERN_ERR, + "error: invalid #inodes per group: %lu", + sbi->s_inodes_per_group); + goto failed_mount; + } + if (sb_bdev_nr_blocks(sb) < le32_to_cpu(es->s_blocks_count)) { + ext2_msg(sb, KERN_ERR, + "bad geometry: block count %u exceeds size of device (%u blocks)", + le32_to_cpu(es->s_blocks_count), + (unsigned)sb_bdev_nr_blocks(sb)); + goto failed_mount; + } + + sbi->s_groups_count = ((le32_to_cpu(es->s_blocks_count) - + le32_to_cpu(es->s_first_data_block) - 1) + / EXT2_BLOCKS_PER_GROUP(sb)) + 1; + if ((u64)sbi->s_groups_count * sbi->s_inodes_per_group != + le32_to_cpu(es->s_inodes_count)) { + ext2_msg(sb, KERN_ERR, "error: invalid #inodes: %u vs computed %llu", + le32_to_cpu(es->s_inodes_count), + (u64)sbi->s_groups_count * sbi->s_inodes_per_group); + goto failed_mount; + } + db_count = (sbi->s_groups_count + EXT2_DESC_PER_BLOCK(sb) - 1) / + EXT2_DESC_PER_BLOCK(sb); + sbi->s_group_desc = kvmalloc_objs(struct buffer_head *, db_count); + if (sbi->s_group_desc == NULL) { + ret = -ENOMEM; + ext2_msg(sb, KERN_ERR, "error: not enough memory"); + goto failed_mount; + } + bgl_lock_init(sbi->s_blockgroup_lock); + sbi->s_debts = kcalloc(sbi->s_groups_count, sizeof(*sbi->s_debts), GFP_KERNEL); + if (!sbi->s_debts) { + ret = -ENOMEM; + ext2_msg(sb, KERN_ERR, "error: not enough memory"); + goto failed_mount_group_desc; + } + for (i = 0; i < db_count; i++) { + block = descriptor_loc(sb, logic_sb_block, i); + sbi->s_group_desc[i] = sb_bread(sb, block); + if (!sbi->s_group_desc[i]) { + for (j = 0; j < i; j++) + brelse (sbi->s_group_desc[j]); + ext2_msg(sb, KERN_ERR, + "error: unable to read group descriptors"); + goto failed_mount_group_desc; + } + } + if (!ext2_check_descriptors (sb)) { + ext2_msg(sb, KERN_ERR, "group descriptors corrupted"); + goto failed_mount2; + } + sbi->s_gdb_count = db_count; + sbi->s_next_generation = get_random_u32(); + spin_lock_init(&sbi->s_next_gen_lock); + + /* per filesystem reservation list head & lock */ + spin_lock_init(&sbi->s_rsv_window_lock); + sbi->s_rsv_window_root = RB_ROOT; + /* + * Add a single, static dummy reservation to the start of the + * reservation window list --- it gives us a placeholder for + * append-at-start-of-list which makes the allocation logic + * _much_ simpler. + */ + sbi->s_rsv_window_head.rsv_start = EXT2_RESERVE_WINDOW_NOT_ALLOCATED; + sbi->s_rsv_window_head.rsv_end = EXT2_RESERVE_WINDOW_NOT_ALLOCATED; + sbi->s_rsv_window_head.rsv_alloc_hit = 0; + sbi->s_rsv_window_head.rsv_goal_size = 0; + ext2_rsv_window_add(sb, &sbi->s_rsv_window_head); + + err = percpu_counter_init(&sbi->s_freeblocks_counter, + ext2_count_free_blocks(sb), GFP_KERNEL); + if (!err) { + err = percpu_counter_init(&sbi->s_freeinodes_counter, + ext2_count_free_inodes(sb), GFP_KERNEL); + } + if (!err) { + err = percpu_counter_init(&sbi->s_dirs_counter, + ext2_count_dirs(sb), GFP_KERNEL); + } + if (err) { + ret = err; + ext2_msg(sb, KERN_ERR, "error: insufficient memory"); + goto failed_mount3; + } + +#ifdef CONFIG_EXT2_FS_XATTR + sbi->s_ea_block_cache = ext2_xattr_create_cache(); + if (!sbi->s_ea_block_cache) { + ret = -ENOMEM; + ext2_msg(sb, KERN_ERR, "Failed to create ea_block_cache"); + goto failed_mount3; + } +#endif + /* + * set up enough so that it can read an inode + */ + sb->s_op = &ext2_sops; + sb->s_export_op = &ext2_export_ops; + sb->s_xattr = ext2_xattr_handlers; + +#ifdef CONFIG_QUOTA + sb->dq_op = &dquot_operations; + sb->s_qcop = &ext2_quotactl_ops; + sb->s_quota_types = QTYPE_MASK_USR | QTYPE_MASK_GRP; +#endif + + root = ext2_iget(sb, EXT2_ROOT_INO); + if (IS_ERR(root)) { + ret = PTR_ERR(root); + goto failed_mount3; + } + if (!S_ISDIR(root->i_mode) || !root->i_blocks || !root->i_size) { + iput(root); + ext2_msg(sb, KERN_ERR, "error: corrupt root inode, run e2fsck"); + goto failed_mount3; + } + + sb->s_root = d_make_root(root); + if (!sb->s_root) { + ext2_msg(sb, KERN_ERR, "error: get root inode failed"); + ret = -ENOMEM; + goto failed_mount3; + } + if (EXT2_HAS_COMPAT_FEATURE(sb, EXT3_FEATURE_COMPAT_HAS_JOURNAL)) + ext2_msg(sb, KERN_WARNING, + "warning: mounting ext3 filesystem as ext2"); + if (ext2_setup_super (sb, es, sb_rdonly(sb))) + sb->s_flags |= SB_RDONLY; + ext2_write_super(sb); + return 0; + +cantfind_ext2: + if (!silent) + ext2_msg(sb, KERN_ERR, + "error: can't find an ext2 filesystem on dev %s.", + sb->s_id); + goto failed_mount; +failed_mount3: + ext2_xattr_destroy_cache(sbi->s_ea_block_cache); + percpu_counter_destroy(&sbi->s_freeblocks_counter); + percpu_counter_destroy(&sbi->s_freeinodes_counter); + percpu_counter_destroy(&sbi->s_dirs_counter); +failed_mount2: + for (i = 0; i < db_count; i++) + brelse(sbi->s_group_desc[i]); +failed_mount_group_desc: + kvfree(sbi->s_group_desc); + kfree(sbi->s_debts); +failed_mount: + brelse(bh); +failed_sbi: + sb->s_fs_info = NULL; + kfree(sbi->s_blockgroup_lock); + kfree(sbi); + return ret; +} + +static void ext2_clear_super_error(struct super_block *sb) +{ + struct buffer_head *sbh = EXT2_SB(sb)->s_sbh; + + if (buffer_write_io_error(sbh)) { + /* + * Oh, dear. A previous attempt to write the + * superblock failed. This could happen because the + * USB device was yanked out. Or it could happen to + * be a transient write error and maybe the block will + * be remapped. Nothing we can do but to retry the + * write and hope for the best. + */ + ext2_msg(sb, KERN_ERR, + "previous I/O error to superblock detected"); + clear_buffer_write_io_error(sbh); + set_buffer_uptodate(sbh); + } +} + +void ext2_sync_super(struct super_block *sb, struct ext2_super_block *es, + int wait) +{ + ext2_clear_super_error(sb); + spin_lock(&EXT2_SB(sb)->s_lock); + es->s_free_blocks_count = cpu_to_le32(ext2_count_free_blocks(sb)); + es->s_free_inodes_count = cpu_to_le32(ext2_count_free_inodes(sb)); + es->s_wtime = cpu_to_le32(ktime_get_real_seconds()); + /* unlock before we do IO */ + spin_unlock(&EXT2_SB(sb)->s_lock); + mark_buffer_dirty(EXT2_SB(sb)->s_sbh); + if (wait) + sync_dirty_buffer(EXT2_SB(sb)->s_sbh); +} + +/* + * In the second extended file system, it is not necessary to + * write the super block since we use a mapping of the + * disk super block in a buffer. + * + * However, this function is still used to set the fs valid + * flags to 0. We need to set this flag to 0 since the fs + * may have been checked while mounted and e2fsck may have + * set s_state to EXT2_VALID_FS after some corrections. + */ +static int ext2_sync_fs(struct super_block *sb, int wait) +{ + struct ext2_sb_info *sbi = EXT2_SB(sb); + struct ext2_super_block *es = EXT2_SB(sb)->s_es; + + /* + * Write quota structures to quota file, sync_blockdev() will write + * them to disk later + */ + dquot_writeback_dquots(sb, -1); + + spin_lock(&sbi->s_lock); + if (es->s_state & cpu_to_le16(EXT2_VALID_FS)) { + ext2_debug("setting valid to 0\n"); + es->s_state &= cpu_to_le16(~EXT2_VALID_FS); + } + spin_unlock(&sbi->s_lock); + ext2_sync_super(sb, es, wait); + return 0; +} + +static int ext2_freeze(struct super_block *sb) +{ + struct ext2_sb_info *sbi = EXT2_SB(sb); + + /* + * Open but unlinked files present? Keep EXT2_VALID_FS flag cleared + * because we have unattached inodes and thus filesystem is not fully + * consistent. + */ + if (atomic_long_read(&sb->s_remove_count)) { + ext2_sync_fs(sb, 1); + return 0; + } + /* Set EXT2_FS_VALID flag */ + spin_lock(&sbi->s_lock); + sbi->s_es->s_state = cpu_to_le16(sbi->s_mount_state); + spin_unlock(&sbi->s_lock); + ext2_sync_super(sb, sbi->s_es, 1); + + return 0; +} + +static int ext2_unfreeze(struct super_block *sb) +{ + /* Just write sb to clear EXT2_VALID_FS flag */ + ext2_write_super(sb); + + return 0; +} + +static void ext2_write_super(struct super_block *sb) +{ + if (!sb_rdonly(sb)) + ext2_sync_fs(sb, 1); +} + +static int ext2_reconfigure(struct fs_context *fc) +{ + struct ext2_fs_context *ctx = fc->fs_private; + struct super_block *sb = fc->root->d_sb; + struct ext2_sb_info * sbi = EXT2_SB(sb); + struct ext2_super_block * es; + struct ext2_mount_options new_opts; + int flags = fc->sb_flags; + int err; + + sync_filesystem(sb); + + new_opts.s_mount_opt = ctx->vals_s_mount_opt; + new_opts.s_resuid = ctx->s_resuid; + new_opts.s_resgid = ctx->s_resgid; + + spin_lock(&sbi->s_lock); + es = sbi->s_es; + if ((bool)(flags & SB_RDONLY) == sb_rdonly(sb)) + goto out_set; + if (flags & SB_RDONLY) { + if (le16_to_cpu(es->s_state) & EXT2_VALID_FS || + !(sbi->s_mount_state & EXT2_VALID_FS)) + goto out_set; + + /* + * OK, we are remounting a valid rw partition rdonly, so set + * the rdonly flag and then mark the partition as valid again. + */ + es->s_state = cpu_to_le16(sbi->s_mount_state); + es->s_mtime = cpu_to_le32(ktime_get_real_seconds()); + spin_unlock(&sbi->s_lock); + + err = dquot_suspend(sb, -1); + if (err < 0) + return err; + + ext2_sync_super(sb, es, 1); + } else { + __le32 ret = EXT2_HAS_RO_COMPAT_FEATURE(sb, + ~EXT2_FEATURE_RO_COMPAT_SUPP); + if (ret) { + spin_unlock(&sbi->s_lock); + ext2_msg(sb, KERN_WARNING, + "warning: couldn't remount RDWR because of " + "unsupported optional features (%x).", + le32_to_cpu(ret)); + return -EROFS; + } + /* + * Mounting a RDONLY partition read-write, so reread and + * store the current valid flag. (It may have been changed + * by e2fsck since we originally mounted the partition.) + */ + sbi->s_mount_state = le16_to_cpu(es->s_state); + if (!ext2_setup_super (sb, es, 0)) + sb->s_flags &= ~SB_RDONLY; + spin_unlock(&sbi->s_lock); + + ext2_write_super(sb); + + dquot_resume(sb, -1); + } + + spin_lock(&sbi->s_lock); +out_set: + sbi->s_mount_opt = new_opts.s_mount_opt; + sbi->s_resuid = new_opts.s_resuid; + sbi->s_resgid = new_opts.s_resgid; + sb->s_flags = (sb->s_flags & ~SB_POSIXACL) | + (test_opt(sb, POSIX_ACL) ? SB_POSIXACL : 0); + spin_unlock(&sbi->s_lock); + + return 0; +} + +static int ext2_statfs (struct dentry * dentry, struct kstatfs * buf) +{ + struct super_block *sb = dentry->d_sb; + struct ext2_sb_info *sbi = EXT2_SB(sb); + struct ext2_super_block *es = sbi->s_es; + + spin_lock(&sbi->s_lock); + + if (test_opt (sb, MINIX_DF)) + sbi->s_overhead_last = 0; + else if (sbi->s_blocks_last != le32_to_cpu(es->s_blocks_count)) { + unsigned long i, overhead = 0; + smp_rmb(); + + /* + * Compute the overhead (FS structures). This is constant + * for a given filesystem unless the number of block groups + * changes so we cache the previous value until it does. + */ + + /* + * All of the blocks before first_data_block are + * overhead + */ + overhead = le32_to_cpu(es->s_first_data_block); + + /* + * Add the overhead attributed to the superblock and + * block group descriptors. If the sparse superblocks + * feature is turned on, then not all groups have this. + */ + for (i = 0; i < sbi->s_groups_count; i++) + overhead += ext2_bg_has_super(sb, i) + + ext2_bg_num_gdb(sb, i); + + /* + * Every block group has an inode bitmap, a block + * bitmap, and an inode table. + */ + overhead += (sbi->s_groups_count * + (2 + sbi->s_itb_per_group)); + sbi->s_overhead_last = overhead; + smp_wmb(); + sbi->s_blocks_last = le32_to_cpu(es->s_blocks_count); + } + + buf->f_type = EXT2_SUPER_MAGIC; + buf->f_bsize = sb->s_blocksize; + buf->f_blocks = le32_to_cpu(es->s_blocks_count) - sbi->s_overhead_last; + buf->f_bfree = ext2_count_free_blocks(sb); + es->s_free_blocks_count = cpu_to_le32(buf->f_bfree); + buf->f_bavail = buf->f_bfree - le32_to_cpu(es->s_r_blocks_count); + if (buf->f_bfree < le32_to_cpu(es->s_r_blocks_count)) + buf->f_bavail = 0; + buf->f_files = le32_to_cpu(es->s_inodes_count); + buf->f_ffree = ext2_count_free_inodes(sb); + es->s_free_inodes_count = cpu_to_le32(buf->f_ffree); + buf->f_namelen = EXT2_NAME_LEN; + buf->f_fsid = uuid_to_fsid(es->s_uuid); + spin_unlock(&sbi->s_lock); + return 0; +} + +static int ext2_get_tree(struct fs_context *fc) +{ + return get_tree_bdev(fc, ext2_fill_super); +} + +#ifdef CONFIG_QUOTA + +/* Read data from quotafile - avoid pagecache and such because we cannot afford + * acquiring the locks... As quota files are never truncated and quota code + * itself serializes the operations (and no one else should touch the files) + * we don't have to be afraid of races */ +static ssize_t ext2_quota_read(struct super_block *sb, int type, char *data, + size_t len, loff_t off) +{ + struct inode *inode = sb_dqopt(sb)->files[type]; + sector_t blk = off >> EXT2_BLOCK_SIZE_BITS(sb); + int err = 0; + int offset = off & (sb->s_blocksize - 1); + int tocopy; + size_t toread; + struct buffer_head tmp_bh; + struct buffer_head *bh; + loff_t i_size = i_size_read(inode); + + if (off > i_size) + return 0; + if (off+len > i_size) + len = i_size-off; + toread = len; + while (toread > 0) { + tocopy = min_t(size_t, sb->s_blocksize - offset, toread); + + tmp_bh.b_state = 0; + tmp_bh.b_size = sb->s_blocksize; + err = ext2_get_block(inode, blk, &tmp_bh, 0); + if (err < 0) + return err; + if (!buffer_mapped(&tmp_bh)) /* A hole? */ + memset(data, 0, tocopy); + else { + bh = sb_bread(sb, tmp_bh.b_blocknr); + if (!bh) + return -EIO; + memcpy(data, bh->b_data+offset, tocopy); + brelse(bh); + } + offset = 0; + toread -= tocopy; + data += tocopy; + blk++; + } + return len; +} + +/* Write to quotafile */ +static ssize_t ext2_quota_write(struct super_block *sb, int type, + const char *data, size_t len, loff_t off) +{ + struct inode *inode = sb_dqopt(sb)->files[type]; + sector_t blk = off >> EXT2_BLOCK_SIZE_BITS(sb); + int err = 0; + int offset = off & (sb->s_blocksize - 1); + int tocopy; + size_t towrite = len; + struct buffer_head tmp_bh; + struct buffer_head *bh; + + while (towrite > 0) { + tocopy = min_t(size_t, sb->s_blocksize - offset, towrite); + + tmp_bh.b_state = 0; + tmp_bh.b_size = sb->s_blocksize; + err = ext2_get_block(inode, blk, &tmp_bh, 1); + if (err < 0) + goto out; + if (offset || tocopy != EXT2_BLOCK_SIZE(sb)) + bh = sb_bread(sb, tmp_bh.b_blocknr); + else + bh = sb_getblk(sb, tmp_bh.b_blocknr); + if (unlikely(!bh)) { + err = -EIO; + goto out; + } + lock_buffer(bh); + memcpy(bh->b_data+offset, data, tocopy); + flush_dcache_folio(bh->b_folio); + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + unlock_buffer(bh); + brelse(bh); + offset = 0; + towrite -= tocopy; + data += tocopy; + blk++; + } +out: + if (len == towrite) + return err; + if (inode->i_size < off+len-towrite) + i_size_write(inode, off+len-towrite); + inode_inc_iversion(inode); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + mark_inode_dirty(inode); + return len - towrite; +} + +static int ext2_quota_on(struct super_block *sb, int type, int format_id, + const struct path *path) +{ + int err; + struct inode *inode; + + err = dquot_quota_on(sb, type, format_id, path); + if (err) + return err; + + inode = d_inode(path->dentry); + inode_lock(inode); + EXT2_I(inode)->i_flags |= EXT2_NOATIME_FL | EXT2_IMMUTABLE_FL; + inode_set_flags(inode, S_NOATIME | S_IMMUTABLE, + S_NOATIME | S_IMMUTABLE); + inode_unlock(inode); + mark_inode_dirty(inode); + + return 0; +} + +static int ext2_quota_off(struct super_block *sb, int type) +{ + struct inode *inode = sb_dqopt(sb)->files[type]; + int err; + + if (!inode || !igrab(inode)) + goto out; + + err = dquot_quota_off(sb, type); + if (err) + goto out_put; + + inode_lock(inode); + EXT2_I(inode)->i_flags &= ~(EXT2_NOATIME_FL | EXT2_IMMUTABLE_FL); + inode_set_flags(inode, 0, S_NOATIME | S_IMMUTABLE); + inode_unlock(inode); + mark_inode_dirty(inode); +out_put: + iput(inode); + return err; +out: + return dquot_quota_off(sb, type); +} + +#endif + +static const struct fs_context_operations ext2_context_ops = { + .parse_param = ext2_parse_param, + .get_tree = ext2_get_tree, + .reconfigure = ext2_reconfigure, + .free = ext2_free_fc, +}; + +static int ext2_init_fs_context(struct fs_context *fc) +{ + struct ext2_fs_context *ctx; + + ctx = kzalloc_obj(*ctx); + if (!ctx) + return -ENOMEM; + + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE) { + struct super_block *sb = fc->root->d_sb; + struct ext2_sb_info *sbi = EXT2_SB(sb); + + spin_lock(&sbi->s_lock); + ctx->vals_s_mount_opt = sbi->s_mount_opt; + ctx->vals_s_flags = sb->s_flags; + ctx->s_resuid = sbi->s_resuid; + ctx->s_resgid = sbi->s_resgid; + spin_unlock(&sbi->s_lock); + } else { + ctx->s_sb_block = 1; + ctx_set_mount_opt(ctx, EXT2_MOUNT_RESERVATION); + } + + fc->fs_private = ctx; + fc->ops = &ext2_context_ops; + + return 0; +} + +static struct file_system_type ext2_fs_type = { + .owner = THIS_MODULE, + .name = "ext2", + .kill_sb = kill_block_super, + .fs_flags = FS_REQUIRES_DEV, + .init_fs_context = ext2_init_fs_context, + .parameters = ext2_param_spec, +}; +MODULE_ALIAS_FS("ext2"); + +static int __init init_ext2_fs(void) +{ + int err; + + err = init_inodecache(); + if (err) + return err; + err = register_filesystem(&ext2_fs_type); + if (err) + goto out; + return 0; +out: + destroy_inodecache(); + return err; +} + +static void __exit exit_ext2_fs(void) +{ + unregister_filesystem(&ext2_fs_type); + destroy_inodecache(); +} + +MODULE_AUTHOR("Remy Card and others"); +MODULE_DESCRIPTION("Second Extended Filesystem"); +MODULE_LICENSE("GPL"); +module_init(init_ext2_fs) +module_exit(exit_ext2_fs) diff --git a/fs/ext2/symlink.c b/fs/ext2/symlink.c new file mode 100644 index 000000000..948d3a441 --- /dev/null +++ b/fs/ext2/symlink.c @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext2/symlink.c + * + * Only fast symlinks left here - the rest is done by generic code. AV, 1999 + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * from + * + * linux/fs/minix/symlink.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * ext2 symlink handling code + */ + +#include "ext2.h" +#include "xattr.h" + +const struct inode_operations ext2_symlink_inode_operations = { + .get_link = page_get_link, + .getattr = ext2_getattr, + .setattr = ext2_setattr, + .listxattr = ext2_listxattr, +}; + +const struct inode_operations ext2_fast_symlink_inode_operations = { + .get_link = simple_get_link, + .getattr = ext2_getattr, + .setattr = ext2_setattr, + .listxattr = ext2_listxattr, +}; diff --git a/fs/ext2/trace.c b/fs/ext2/trace.c new file mode 100644 index 000000000..b01cdf652 --- /dev/null +++ b/fs/ext2/trace.c @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "ext2.h" +#include + +#define CREATE_TRACE_POINTS +#include "trace.h" diff --git a/fs/ext2/trace.h b/fs/ext2/trace.h new file mode 100644 index 000000000..0922c0e6a --- /dev/null +++ b/fs/ext2/trace.h @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: GPL-2.0 + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM ext2 + +#if !defined(_EXT2_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _EXT2_TRACE_H + +#include + +DECLARE_EVENT_CLASS(ext2_dio_class, + TP_PROTO(struct kiocb *iocb, struct iov_iter *iter, ssize_t ret), + TP_ARGS(iocb, iter, ret), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u64, ino) + __field(loff_t, isize) + __field(loff_t, pos) + __field(size_t, count) + __field(int, ki_flags) + __field(bool, aio) + __field(ssize_t, ret) + ), + TP_fast_assign( + __entry->dev = file_inode(iocb->ki_filp)->i_sb->s_dev; + __entry->ino = file_inode(iocb->ki_filp)->i_ino; + __entry->isize = file_inode(iocb->ki_filp)->i_size; + __entry->pos = iocb->ki_pos; + __entry->count = iov_iter_count(iter); + __entry->ki_flags = iocb->ki_flags; + __entry->aio = !is_sync_kiocb(iocb); + __entry->ret = ret; + ), + TP_printk("dev %d:%d ino 0x%llx isize 0x%llx pos 0x%llx len %zu flags %s aio %d ret %zd", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->isize, + __entry->pos, + __entry->count, + __print_flags(__entry->ki_flags, "|", TRACE_IOCB_STRINGS), + __entry->aio, + __entry->ret) +); + +#define DEFINE_DIO_RW_EVENT(name) \ +DEFINE_EVENT(ext2_dio_class, name, \ + TP_PROTO(struct kiocb *iocb, struct iov_iter *iter, ssize_t ret), \ + TP_ARGS(iocb, iter, ret)) +DEFINE_DIO_RW_EVENT(ext2_dio_write_begin); +DEFINE_DIO_RW_EVENT(ext2_dio_write_end); +DEFINE_DIO_RW_EVENT(ext2_dio_write_buff_end); +DEFINE_DIO_RW_EVENT(ext2_dio_read_begin); +DEFINE_DIO_RW_EVENT(ext2_dio_read_end); + +TRACE_EVENT(ext2_dio_write_endio, + TP_PROTO(struct kiocb *iocb, ssize_t size, int ret), + TP_ARGS(iocb, size, ret), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u64, ino) + __field(loff_t, isize) + __field(loff_t, pos) + __field(ssize_t, size) + __field(int, ki_flags) + __field(bool, aio) + __field(int, ret) + ), + TP_fast_assign( + __entry->dev = file_inode(iocb->ki_filp)->i_sb->s_dev; + __entry->ino = file_inode(iocb->ki_filp)->i_ino; + __entry->isize = file_inode(iocb->ki_filp)->i_size; + __entry->pos = iocb->ki_pos; + __entry->size = size; + __entry->ki_flags = iocb->ki_flags; + __entry->aio = !is_sync_kiocb(iocb); + __entry->ret = ret; + ), + TP_printk("dev %d:%d ino 0x%llx isize 0x%llx pos 0x%llx len %zd flags %s aio %d ret %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->isize, + __entry->pos, + __entry->size, + __print_flags(__entry->ki_flags, "|", TRACE_IOCB_STRINGS), + __entry->aio, + __entry->ret) +); + +#endif /* _EXT2_TRACE_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE trace +#include diff --git a/fs/ext2/xattr.c b/fs/ext2/xattr.c new file mode 100644 index 000000000..9b68c490a --- /dev/null +++ b/fs/ext2/xattr.c @@ -0,0 +1,1055 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext2/xattr.c + * + * Copyright (C) 2001-2003 Andreas Gruenbacher + * + * Fix by Harrison Xing . + * Extended attributes for symlinks and special files added per + * suggestion of Luka Renko . + * xattr consolidation Copyright (c) 2004 James Morris , + * Red Hat Inc. + * + */ + +/* + * Extended attributes are stored on disk blocks allocated outside of + * any inode. The i_file_acl field is then made to point to this allocated + * block. If all extended attributes of an inode are identical, these + * inodes may share the same extended attribute block. Such situations + * are automatically detected by keeping a cache of recent attribute block + * numbers and hashes over the block's contents in memory. + * + * + * Extended attribute block layout: + * + * +------------------+ + * | header | + * | entry 1 | | + * | entry 2 | | growing downwards + * | entry 3 | v + * | four null bytes | + * | . . . | + * | value 1 | ^ + * | value 3 | | growing upwards + * | value 2 | | + * +------------------+ + * + * The block header is followed by multiple entry descriptors. These entry + * descriptors are variable in size, and aligned to EXT2_XATTR_PAD + * byte boundaries. The entry descriptors are sorted by attribute name, + * so that two extended attribute blocks can be compared efficiently. + * + * Attribute values are aligned to the end of the block, stored in + * no specific order. They are also padded to EXT2_XATTR_PAD byte + * boundaries. No additional gaps are left between them. + * + * Locking strategy + * ---------------- + * EXT2_I(inode)->i_file_acl is protected by EXT2_I(inode)->xattr_sem. + * EA blocks are only changed if they are exclusive to an inode, so + * holding xattr_sem also means that nothing but the EA block's reference + * count will change. Multiple writers to an EA block are synchronized + * by the bh lock. No more than a single bh lock is held at any time + * to avoid deadlocks. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "ext2.h" +#include "xattr.h" +#include "acl.h" + +#define HDR(bh) ((struct ext2_xattr_header *)((bh)->b_data)) +#define ENTRY(ptr) ((struct ext2_xattr_entry *)(ptr)) +#define FIRST_ENTRY(bh) ENTRY(HDR(bh)+1) +#define IS_LAST_ENTRY(entry) (*(__u32 *)(entry) == 0) + +#ifdef EXT2_XATTR_DEBUG +# define ea_idebug(inode, f...) do { \ + printk(KERN_DEBUG "inode %s:%llu: ", \ + inode->i_sb->s_id, inode->i_ino); \ + printk(f); \ + printk("\n"); \ + } while (0) +# define ea_bdebug(bh, f...) do { \ + printk(KERN_DEBUG "block %pg:%lu: ", \ + bh->b_bdev, (unsigned long) bh->b_blocknr); \ + printk(f); \ + printk("\n"); \ + } while (0) +#else +# define ea_idebug(inode, f...) no_printk(f) +# define ea_bdebug(bh, f...) no_printk(f) +#endif + +static int ext2_xattr_set2(struct inode *, struct buffer_head *, + struct ext2_xattr_header *); + +static int ext2_xattr_cache_insert(struct mb_cache *, struct buffer_head *); +static struct buffer_head *ext2_xattr_cache_find(struct inode *, + struct ext2_xattr_header *); +static void ext2_xattr_rehash(struct ext2_xattr_header *, + struct ext2_xattr_entry *); + +static const struct xattr_handler * const ext2_xattr_handler_map[] = { + [EXT2_XATTR_INDEX_USER] = &ext2_xattr_user_handler, +#ifdef CONFIG_EXT2_FS_POSIX_ACL + [EXT2_XATTR_INDEX_POSIX_ACL_ACCESS] = &nop_posix_acl_access, + [EXT2_XATTR_INDEX_POSIX_ACL_DEFAULT] = &nop_posix_acl_default, +#endif + [EXT2_XATTR_INDEX_TRUSTED] = &ext2_xattr_trusted_handler, +#ifdef CONFIG_EXT2_FS_SECURITY + [EXT2_XATTR_INDEX_SECURITY] = &ext2_xattr_security_handler, +#endif +}; + +const struct xattr_handler * const ext2_xattr_handlers[] = { + &ext2_xattr_user_handler, + &ext2_xattr_trusted_handler, +#ifdef CONFIG_EXT2_FS_SECURITY + &ext2_xattr_security_handler, +#endif + NULL +}; + +#define EA_BLOCK_CACHE(inode) (EXT2_SB(inode->i_sb)->s_ea_block_cache) + +static inline const char *ext2_xattr_prefix(int name_index, + struct dentry *dentry) +{ + const struct xattr_handler *handler = NULL; + + if (name_index > 0 && name_index < ARRAY_SIZE(ext2_xattr_handler_map)) + handler = ext2_xattr_handler_map[name_index]; + + if (!xattr_handler_can_list(handler, dentry)) + return NULL; + + return xattr_prefix(handler); +} + +static bool +ext2_xattr_header_valid(struct ext2_xattr_header *header) +{ + if (header->h_magic != cpu_to_le32(EXT2_XATTR_MAGIC) || + header->h_blocks != cpu_to_le32(1)) + return false; + + return true; +} + +static bool +ext2_xattr_entry_valid(struct ext2_xattr_entry *entry, + char *end, size_t end_offs) +{ + struct ext2_xattr_entry *next; + size_t size; + + next = EXT2_XATTR_NEXT(entry); + if ((char *)next >= end) + return false; + + if (entry->e_value_block != 0) + return false; + + size = le32_to_cpu(entry->e_value_size); + if (size > end_offs || + le16_to_cpu(entry->e_value_offs) + size > end_offs) + return false; + + return true; +} + +static int +ext2_xattr_cmp_entry(int name_index, size_t name_len, const char *name, + struct ext2_xattr_entry *entry) +{ + int cmp; + + cmp = name_index - entry->e_name_index; + if (!cmp) + cmp = name_len - entry->e_name_len; + if (!cmp) + cmp = memcmp(name, entry->e_name, name_len); + + return cmp; +} + +/* + * ext2_xattr_get() + * + * Copy an extended attribute into the buffer + * provided, or compute the buffer size required. + * Buffer is NULL to compute the size of the buffer required. + * + * Returns a negative error number on failure, or the number of bytes + * used / required on success. + */ +int +ext2_xattr_get(struct inode *inode, int name_index, const char *name, + void *buffer, size_t buffer_size) +{ + struct buffer_head *bh = NULL; + struct ext2_xattr_entry *entry; + size_t name_len, size; + char *end; + int error, not_found; + struct mb_cache *ea_block_cache = EA_BLOCK_CACHE(inode); + + ea_idebug(inode, "name=%d.%s, buffer=%p, buffer_size=%ld", + name_index, name, buffer, (long)buffer_size); + + if (name == NULL) + return -EINVAL; + name_len = strlen(name); + if (name_len > 255) + return -ERANGE; + + down_read(&EXT2_I(inode)->xattr_sem); + error = -ENODATA; + if (!EXT2_I(inode)->i_file_acl) + goto cleanup; + ea_idebug(inode, "reading block %d", EXT2_I(inode)->i_file_acl); + bh = sb_bread(inode->i_sb, EXT2_I(inode)->i_file_acl); + error = -EIO; + if (!bh) + goto cleanup; + ea_bdebug(bh, "b_count=%d, refcount=%d", + atomic_read(&(bh->b_count)), le32_to_cpu(HDR(bh)->h_refcount)); + end = bh->b_data + bh->b_size; + if (!ext2_xattr_header_valid(HDR(bh))) { +bad_block: + ext2_error(inode->i_sb, "ext2_xattr_get", + "inode %llu: bad block %d", inode->i_ino, + EXT2_I(inode)->i_file_acl); + error = -EIO; + goto cleanup; + } + + /* find named attribute */ + entry = FIRST_ENTRY(bh); + while (!IS_LAST_ENTRY(entry)) { + if (!ext2_xattr_entry_valid(entry, end, + inode->i_sb->s_blocksize)) + goto bad_block; + + not_found = ext2_xattr_cmp_entry(name_index, name_len, name, + entry); + if (!not_found) + goto found; + if (not_found < 0) + break; + + entry = EXT2_XATTR_NEXT(entry); + } + if (ext2_xattr_cache_insert(ea_block_cache, bh)) + ea_idebug(inode, "cache insert failed"); + error = -ENODATA; + goto cleanup; +found: + size = le32_to_cpu(entry->e_value_size); + if (ext2_xattr_cache_insert(ea_block_cache, bh)) + ea_idebug(inode, "cache insert failed"); + if (buffer) { + error = -ERANGE; + if (size > buffer_size) + goto cleanup; + /* return value of attribute */ + memcpy(buffer, bh->b_data + le16_to_cpu(entry->e_value_offs), + size); + } + error = size; + +cleanup: + brelse(bh); + up_read(&EXT2_I(inode)->xattr_sem); + + return error; +} + +/* + * ext2_xattr_list() + * + * Copy a list of attribute names into the buffer + * provided, or compute the buffer size required. + * Buffer is NULL to compute the size of the buffer required. + * + * Returns a negative error number on failure, or the number of bytes + * used / required on success. + */ +static int +ext2_xattr_list(struct dentry *dentry, char *buffer, size_t buffer_size) +{ + struct inode *inode = d_inode(dentry); + struct buffer_head *bh = NULL; + struct ext2_xattr_entry *entry; + char *end; + size_t rest = buffer_size; + int error; + struct mb_cache *ea_block_cache = EA_BLOCK_CACHE(inode); + + ea_idebug(inode, "buffer=%p, buffer_size=%ld", + buffer, (long)buffer_size); + + down_read(&EXT2_I(inode)->xattr_sem); + error = 0; + if (!EXT2_I(inode)->i_file_acl) + goto cleanup; + ea_idebug(inode, "reading block %d", EXT2_I(inode)->i_file_acl); + bh = sb_bread(inode->i_sb, EXT2_I(inode)->i_file_acl); + error = -EIO; + if (!bh) + goto cleanup; + ea_bdebug(bh, "b_count=%d, refcount=%d", + atomic_read(&(bh->b_count)), le32_to_cpu(HDR(bh)->h_refcount)); + end = bh->b_data + bh->b_size; + if (!ext2_xattr_header_valid(HDR(bh))) { +bad_block: + ext2_error(inode->i_sb, "ext2_xattr_list", + "inode %llu: bad block %d", inode->i_ino, + EXT2_I(inode)->i_file_acl); + error = -EIO; + goto cleanup; + } + + /* check the on-disk data structure */ + entry = FIRST_ENTRY(bh); + while (!IS_LAST_ENTRY(entry)) { + if (!ext2_xattr_entry_valid(entry, end, + inode->i_sb->s_blocksize)) + goto bad_block; + entry = EXT2_XATTR_NEXT(entry); + } + if (ext2_xattr_cache_insert(ea_block_cache, bh)) + ea_idebug(inode, "cache insert failed"); + + /* list the attribute names */ + for (entry = FIRST_ENTRY(bh); !IS_LAST_ENTRY(entry); + entry = EXT2_XATTR_NEXT(entry)) { + const char *prefix; + + prefix = ext2_xattr_prefix(entry->e_name_index, dentry); + if (prefix) { + size_t prefix_len = strlen(prefix); + size_t size = prefix_len + entry->e_name_len + 1; + + if (buffer) { + if (size > rest) { + error = -ERANGE; + goto cleanup; + } + memcpy(buffer, prefix, prefix_len); + buffer += prefix_len; + memcpy(buffer, entry->e_name, entry->e_name_len); + buffer += entry->e_name_len; + *buffer++ = 0; + } + rest -= size; + } + } + error = buffer_size - rest; /* total size */ + +cleanup: + brelse(bh); + up_read(&EXT2_I(inode)->xattr_sem); + + return error; +} + +/* + * Inode operation listxattr() + * + * d_inode(dentry)->i_mutex: don't care + */ +ssize_t +ext2_listxattr(struct dentry *dentry, char *buffer, size_t size) +{ + return ext2_xattr_list(dentry, buffer, size); +} + +/* + * If the EXT2_FEATURE_COMPAT_EXT_ATTR feature of this file system is + * not set, set it. + */ +static void ext2_xattr_update_super_block(struct super_block *sb) +{ + if (EXT2_HAS_COMPAT_FEATURE(sb, EXT2_FEATURE_COMPAT_EXT_ATTR)) + return; + + spin_lock(&EXT2_SB(sb)->s_lock); + ext2_update_dynamic_rev(sb); + EXT2_SET_COMPAT_FEATURE(sb, EXT2_FEATURE_COMPAT_EXT_ATTR); + spin_unlock(&EXT2_SB(sb)->s_lock); + mark_buffer_dirty(EXT2_SB(sb)->s_sbh); +} + +/* + * ext2_xattr_set() + * + * Create, replace or remove an extended attribute for this inode. Value + * is NULL to remove an existing extended attribute, and non-NULL to + * either replace an existing extended attribute, or create a new extended + * attribute. The flags XATTR_REPLACE and XATTR_CREATE + * specify that an extended attribute must exist and must not exist + * previous to the call, respectively. + * + * Returns 0, or a negative error number on failure. + */ +int +ext2_xattr_set(struct inode *inode, int name_index, const char *name, + const void *value, size_t value_len, int flags) +{ + struct super_block *sb = inode->i_sb; + struct buffer_head *bh = NULL; + struct ext2_xattr_header *header = NULL; + struct ext2_xattr_entry *here = NULL, *last = NULL; + size_t name_len, free, min_offs = sb->s_blocksize; + int not_found = 1, error; + char *end; + + /* + * header -- Points either into bh, or to a temporarily + * allocated buffer. + * here -- The named entry found, or the place for inserting, within + * the block pointed to by header. + * last -- Points right after the last named entry within the block + * pointed to by header. + * min_offs -- The offset of the first value (values are aligned + * towards the end of the block). + * end -- Points right after the block pointed to by header. + */ + + ea_idebug(inode, "name=%d.%s, value=%p, value_len=%ld", + name_index, name, value, (long)value_len); + + if (value == NULL) + value_len = 0; + if (name == NULL) + return -EINVAL; + name_len = strlen(name); + if (name_len > 255 || value_len > sb->s_blocksize) + return -ERANGE; + error = dquot_initialize(inode); + if (error) + return error; + down_write(&EXT2_I(inode)->xattr_sem); + if (EXT2_I(inode)->i_file_acl) { + /* The inode already has an extended attribute block. */ + bh = sb_bread(sb, EXT2_I(inode)->i_file_acl); + error = -EIO; + if (!bh) + goto cleanup; + ea_bdebug(bh, "b_count=%d, refcount=%d", + atomic_read(&(bh->b_count)), + le32_to_cpu(HDR(bh)->h_refcount)); + header = HDR(bh); + end = bh->b_data + bh->b_size; + if (!ext2_xattr_header_valid(header)) { +bad_block: + ext2_error(sb, "ext2_xattr_set", + "inode %llu: bad block %d", inode->i_ino, + EXT2_I(inode)->i_file_acl); + error = -EIO; + goto cleanup; + } + /* + * Find the named attribute. If not found, 'here' will point + * to entry where the new attribute should be inserted to + * maintain sorting. + */ + last = FIRST_ENTRY(bh); + while (!IS_LAST_ENTRY(last)) { + if (!ext2_xattr_entry_valid(last, end, sb->s_blocksize)) + goto bad_block; + if (last->e_value_size) { + size_t offs = le16_to_cpu(last->e_value_offs); + if (offs < min_offs) + min_offs = offs; + } + if (not_found > 0) { + not_found = ext2_xattr_cmp_entry(name_index, + name_len, + name, last); + if (not_found <= 0) + here = last; + } + last = EXT2_XATTR_NEXT(last); + } + if (not_found > 0) + here = last; + + /* Check whether we have enough space left. */ + free = min_offs - ((char*)last - (char*)header) - sizeof(__u32); + } else { + /* We will use a new extended attribute block. */ + free = sb->s_blocksize - + sizeof(struct ext2_xattr_header) - sizeof(__u32); + } + + if (not_found) { + /* Request to remove a nonexistent attribute? */ + error = -ENODATA; + if (flags & XATTR_REPLACE) + goto cleanup; + error = 0; + if (value == NULL) + goto cleanup; + } else { + /* Request to create an existing attribute? */ + error = -EEXIST; + if (flags & XATTR_CREATE) + goto cleanup; + free += EXT2_XATTR_SIZE(le32_to_cpu(here->e_value_size)); + free += EXT2_XATTR_LEN(name_len); + } + error = -ENOSPC; + if (free < EXT2_XATTR_LEN(name_len) + EXT2_XATTR_SIZE(value_len)) + goto cleanup; + + /* Here we know that we can set the new attribute. */ + + if (header) { + int offset; + + lock_buffer(bh); + if (header->h_refcount == cpu_to_le32(1)) { + __u32 hash = le32_to_cpu(header->h_hash); + struct mb_cache_entry *oe; + + oe = mb_cache_entry_delete_or_get(EA_BLOCK_CACHE(inode), + hash, bh->b_blocknr); + if (!oe) { + ea_bdebug(bh, "modifying in-place"); + goto update_block; + } + /* + * Someone is trying to reuse the block, leave it alone + */ + mb_cache_entry_put(EA_BLOCK_CACHE(inode), oe); + } + unlock_buffer(bh); + ea_bdebug(bh, "cloning"); + header = kmemdup(HDR(bh), bh->b_size, GFP_KERNEL); + error = -ENOMEM; + if (header == NULL) + goto cleanup; + header->h_refcount = cpu_to_le32(1); + + offset = (char *)here - bh->b_data; + here = ENTRY((char *)header + offset); + offset = (char *)last - bh->b_data; + last = ENTRY((char *)header + offset); + } else { + /* Allocate a buffer where we construct the new block. */ + header = kzalloc(sb->s_blocksize, GFP_KERNEL); + error = -ENOMEM; + if (header == NULL) + goto cleanup; + header->h_magic = cpu_to_le32(EXT2_XATTR_MAGIC); + header->h_blocks = header->h_refcount = cpu_to_le32(1); + last = here = ENTRY(header+1); + } + +update_block: + /* Iff we are modifying the block in-place, bh is locked here. */ + + if (not_found) { + /* Insert the new name. */ + size_t size = EXT2_XATTR_LEN(name_len); + size_t rest = (char *)last - (char *)here; + memmove((char *)here + size, here, rest); + memset(here, 0, size); + here->e_name_index = name_index; + here->e_name_len = name_len; + memcpy(here->e_name, name, name_len); + } else { + if (here->e_value_size) { + char *first_val = (char *)header + min_offs; + size_t offs = le16_to_cpu(here->e_value_offs); + char *val = (char *)header + offs; + size_t size = EXT2_XATTR_SIZE( + le32_to_cpu(here->e_value_size)); + + if (size == EXT2_XATTR_SIZE(value_len)) { + /* The old and the new value have the same + size. Just replace. */ + here->e_value_size = cpu_to_le32(value_len); + memset(val + size - EXT2_XATTR_PAD, 0, + EXT2_XATTR_PAD); /* Clear pad bytes. */ + memcpy(val, value, value_len); + goto skip_replace; + } + + /* Remove the old value. */ + memmove(first_val + size, first_val, val - first_val); + memset(first_val, 0, size); + min_offs += size; + + /* Adjust all value offsets. */ + last = ENTRY(header+1); + while (!IS_LAST_ENTRY(last)) { + size_t o = le16_to_cpu(last->e_value_offs); + if (o < offs) + last->e_value_offs = + cpu_to_le16(o + size); + last = EXT2_XATTR_NEXT(last); + } + + here->e_value_offs = 0; + } + if (value == NULL) { + /* Remove the old name. */ + size_t size = EXT2_XATTR_LEN(name_len); + last = ENTRY((char *)last - size); + memmove(here, (char*)here + size, + (char*)last - (char*)here); + memset(last, 0, size); + } + } + + if (value != NULL) { + /* Insert the new value. */ + here->e_value_size = cpu_to_le32(value_len); + if (value_len) { + size_t size = EXT2_XATTR_SIZE(value_len); + char *val = (char *)header + min_offs - size; + here->e_value_offs = + cpu_to_le16((char *)val - (char *)header); + memset(val + size - EXT2_XATTR_PAD, 0, + EXT2_XATTR_PAD); /* Clear the pad bytes. */ + memcpy(val, value, value_len); + } + } + +skip_replace: + if (IS_LAST_ENTRY(ENTRY(header+1))) { + /* This block is now empty. */ + if (bh && header == HDR(bh)) + unlock_buffer(bh); /* we were modifying in-place. */ + error = ext2_xattr_set2(inode, bh, NULL); + } else { + ext2_xattr_rehash(header, here); + if (bh && header == HDR(bh)) + unlock_buffer(bh); /* we were modifying in-place. */ + error = ext2_xattr_set2(inode, bh, header); + } + +cleanup: + if (!(bh && header == HDR(bh))) + kfree(header); + brelse(bh); + up_write(&EXT2_I(inode)->xattr_sem); + + return error; +} + +static void ext2_xattr_release_block(struct inode *inode, + struct buffer_head *bh) +{ + struct mb_cache *ea_block_cache = EA_BLOCK_CACHE(inode); + +retry_ref: + lock_buffer(bh); + if (HDR(bh)->h_refcount == cpu_to_le32(1)) { + __u32 hash = le32_to_cpu(HDR(bh)->h_hash); + struct mb_cache_entry *oe; + + /* + * This must happen under buffer lock to properly + * serialize with ext2_xattr_set() reusing the block. + */ + oe = mb_cache_entry_delete_or_get(ea_block_cache, hash, + bh->b_blocknr); + if (oe) { + /* + * Someone is trying to reuse the block. Wait + * and retry. + */ + unlock_buffer(bh); + mb_cache_entry_wait_unused(oe); + mb_cache_entry_put(ea_block_cache, oe); + goto retry_ref; + } + + /* Free the old block. */ + ea_bdebug(bh, "freeing"); + ext2_free_blocks(inode, bh->b_blocknr, 1); + /* We let our caller release bh, so we + * need to duplicate the buffer before. */ + get_bh(bh); + bforget(bh); + unlock_buffer(bh); + } else { + /* Decrement the refcount only. */ + le32_add_cpu(&HDR(bh)->h_refcount, -1); + dquot_free_block(inode, 1); + mark_buffer_dirty(bh); + unlock_buffer(bh); + ea_bdebug(bh, "refcount now=%d", + le32_to_cpu(HDR(bh)->h_refcount)); + if (IS_SYNC(inode)) + sync_dirty_buffer(bh); + } +} + +/* + * Second half of ext2_xattr_set(): Update the file system. + */ +static int +ext2_xattr_set2(struct inode *inode, struct buffer_head *old_bh, + struct ext2_xattr_header *header) +{ + struct super_block *sb = inode->i_sb; + struct buffer_head *new_bh = NULL; + int error; + struct mb_cache *ea_block_cache = EA_BLOCK_CACHE(inode); + + if (header) { + new_bh = ext2_xattr_cache_find(inode, header); + if (new_bh) { + /* We found an identical block in the cache. */ + if (new_bh == old_bh) { + ea_bdebug(new_bh, "keeping this block"); + } else { + /* The old block is released after updating + the inode. */ + ea_bdebug(new_bh, "reusing block"); + + error = dquot_alloc_block(inode, 1); + if (error) { + unlock_buffer(new_bh); + goto cleanup; + } + le32_add_cpu(&HDR(new_bh)->h_refcount, 1); + ea_bdebug(new_bh, "refcount now=%d", + le32_to_cpu(HDR(new_bh)->h_refcount)); + } + unlock_buffer(new_bh); + } else if (old_bh && header == HDR(old_bh)) { + /* Keep this block. No need to lock the block as we + don't need to change the reference count. */ + new_bh = old_bh; + get_bh(new_bh); + ext2_xattr_cache_insert(ea_block_cache, new_bh); + } else { + /* We need to allocate a new block */ + ext2_fsblk_t goal = ext2_group_first_block_no(sb, + EXT2_I(inode)->i_block_group); + unsigned long count = 1; + ext2_fsblk_t block = ext2_new_blocks(inode, goal, + &count, &error, + EXT2_ALLOC_NORESERVE); + if (error) + goto cleanup; + ea_idebug(inode, "creating block %lu", block); + + new_bh = sb_getblk(sb, block); + if (unlikely(!new_bh)) { + ext2_free_blocks(inode, block, 1); + mark_inode_dirty(inode); + error = -ENOMEM; + goto cleanup; + } + lock_buffer(new_bh); + memcpy(new_bh->b_data, header, new_bh->b_size); + set_buffer_uptodate(new_bh); + unlock_buffer(new_bh); + ext2_xattr_cache_insert(ea_block_cache, new_bh); + + ext2_xattr_update_super_block(sb); + } + mark_buffer_dirty(new_bh); + if (IS_SYNC(inode)) { + sync_dirty_buffer(new_bh); + error = -EIO; + if (buffer_req(new_bh) && !buffer_uptodate(new_bh)) + goto cleanup; + } + } + + /* Update the inode. */ + EXT2_I(inode)->i_file_acl = new_bh ? new_bh->b_blocknr : 0; + inode_set_ctime_current(inode); + mark_inode_dirty(inode); + if (IS_SYNC(inode)) { + error = sync_inode_metadata(inode, 1); + /* + * Inode writeout failed. Backing everything out is complex so + * let's just leave it for e2fsck to cleanup the mess. + */ + if (error) + goto cleanup; + } + + error = 0; + if (old_bh && old_bh != new_bh) { + /* + * If there was an old block and we are no longer using it, + * release the old block. + */ + ext2_xattr_release_block(inode, old_bh); + } + +cleanup: + brelse(new_bh); + + return error; +} + +/* + * ext2_xattr_delete_inode() + * + * Free extended attribute resources associated with this inode. This + * is called immediately before an inode is freed. + */ +void +ext2_xattr_delete_inode(struct inode *inode) +{ + struct buffer_head *bh = NULL; + struct ext2_sb_info *sbi = EXT2_SB(inode->i_sb); + + /* + * We are the only ones holding inode reference. The xattr_sem should + * better be unlocked! We could as well just not acquire xattr_sem at + * all but this makes the code more futureproof. OTOH we need trylock + * here to avoid false-positive warning from lockdep about reclaim + * circular dependency. + */ + if (WARN_ON_ONCE(!down_write_trylock(&EXT2_I(inode)->xattr_sem))) + return; + if (!EXT2_I(inode)->i_file_acl) + goto cleanup; + + if (!ext2_data_block_valid(sbi, EXT2_I(inode)->i_file_acl, 1)) { + ext2_error(inode->i_sb, "ext2_xattr_delete_inode", + "inode %llu: xattr block %d is out of data blocks range", + inode->i_ino, EXT2_I(inode)->i_file_acl); + goto cleanup; + } + + bh = sb_bread(inode->i_sb, EXT2_I(inode)->i_file_acl); + if (!bh) { + ext2_error(inode->i_sb, "ext2_xattr_delete_inode", + "inode %llu: block %d read error", inode->i_ino, + EXT2_I(inode)->i_file_acl); + goto cleanup; + } + ea_bdebug(bh, "b_count=%d", atomic_read(&(bh->b_count))); + if (!ext2_xattr_header_valid(HDR(bh))) { + ext2_error(inode->i_sb, "ext2_xattr_delete_inode", + "inode %llu: bad block %d", inode->i_ino, + EXT2_I(inode)->i_file_acl); + goto cleanup; + } + ext2_xattr_release_block(inode, bh); + EXT2_I(inode)->i_file_acl = 0; + +cleanup: + brelse(bh); + up_write(&EXT2_I(inode)->xattr_sem); +} + +/* + * ext2_xattr_cache_insert() + * + * Create a new entry in the extended attribute cache, and insert + * it unless such an entry is already in the cache. + * + * Returns 0, or a negative error number on failure. + */ +static int +ext2_xattr_cache_insert(struct mb_cache *cache, struct buffer_head *bh) +{ + __u32 hash = le32_to_cpu(HDR(bh)->h_hash); + int error; + + error = mb_cache_entry_create(cache, GFP_KERNEL, hash, bh->b_blocknr, + true); + if (error) { + if (error == -EBUSY) { + ea_bdebug(bh, "already in cache"); + error = 0; + } + } else + ea_bdebug(bh, "inserting [%x]", (int)hash); + return error; +} + +/* + * ext2_xattr_cmp() + * + * Compare two extended attribute blocks for equality. + * + * Returns 0 if the blocks are equal, 1 if they differ, and + * a negative error number on errors. + */ +static int +ext2_xattr_cmp(struct ext2_xattr_header *header1, + struct ext2_xattr_header *header2) +{ + struct ext2_xattr_entry *entry1, *entry2; + + entry1 = ENTRY(header1+1); + entry2 = ENTRY(header2+1); + while (!IS_LAST_ENTRY(entry1)) { + if (IS_LAST_ENTRY(entry2)) + return 1; + if (entry1->e_hash != entry2->e_hash || + entry1->e_name_index != entry2->e_name_index || + entry1->e_name_len != entry2->e_name_len || + entry1->e_value_size != entry2->e_value_size || + memcmp(entry1->e_name, entry2->e_name, entry1->e_name_len)) + return 1; + if (entry1->e_value_block != 0 || entry2->e_value_block != 0) + return -EIO; + if (memcmp((char *)header1 + le16_to_cpu(entry1->e_value_offs), + (char *)header2 + le16_to_cpu(entry2->e_value_offs), + le32_to_cpu(entry1->e_value_size))) + return 1; + + entry1 = EXT2_XATTR_NEXT(entry1); + entry2 = EXT2_XATTR_NEXT(entry2); + } + if (!IS_LAST_ENTRY(entry2)) + return 1; + return 0; +} + +/* + * ext2_xattr_cache_find() + * + * Find an identical extended attribute block. + * + * Returns a locked buffer head to the block found, or NULL if such + * a block was not found or an error occurred. + */ +static struct buffer_head * +ext2_xattr_cache_find(struct inode *inode, struct ext2_xattr_header *header) +{ + __u32 hash = le32_to_cpu(header->h_hash); + struct mb_cache_entry *ce; + struct mb_cache *ea_block_cache = EA_BLOCK_CACHE(inode); + + if (!header->h_hash) + return NULL; /* never share */ + ea_idebug(inode, "looking for cached blocks [%x]", (int)hash); + + ce = mb_cache_entry_find_first(ea_block_cache, hash); + while (ce) { + struct buffer_head *bh; + + bh = sb_bread(inode->i_sb, ce->e_value); + if (!bh) { + ext2_error(inode->i_sb, "ext2_xattr_cache_find", + "inode %llu: block %ld read error", + inode->i_ino, (unsigned long) ce->e_value); + } else { + lock_buffer(bh); + if (le32_to_cpu(HDR(bh)->h_refcount) > + EXT2_XATTR_REFCOUNT_MAX) { + ea_idebug(inode, "block %ld refcount %d>%d", + (unsigned long) ce->e_value, + le32_to_cpu(HDR(bh)->h_refcount), + EXT2_XATTR_REFCOUNT_MAX); + } else if (!ext2_xattr_cmp(header, HDR(bh))) { + ea_bdebug(bh, "b_count=%d", + atomic_read(&(bh->b_count))); + mb_cache_entry_touch(ea_block_cache, ce); + mb_cache_entry_put(ea_block_cache, ce); + return bh; + } + unlock_buffer(bh); + brelse(bh); + } + ce = mb_cache_entry_find_next(ea_block_cache, ce); + } + return NULL; +} + +#define NAME_HASH_SHIFT 5 +#define VALUE_HASH_SHIFT 16 + +/* + * ext2_xattr_hash_entry() + * + * Compute the hash of an extended attribute. + */ +static inline void ext2_xattr_hash_entry(struct ext2_xattr_header *header, + struct ext2_xattr_entry *entry) +{ + __u32 hash = 0; + char *name = entry->e_name; + int n; + + for (n=0; n < entry->e_name_len; n++) { + hash = (hash << NAME_HASH_SHIFT) ^ + (hash >> (8*sizeof(hash) - NAME_HASH_SHIFT)) ^ + *name++; + } + + if (entry->e_value_block == 0 && entry->e_value_size != 0) { + __le32 *value = (__le32 *)((char *)header + + le16_to_cpu(entry->e_value_offs)); + for (n = (le32_to_cpu(entry->e_value_size) + + EXT2_XATTR_ROUND) >> EXT2_XATTR_PAD_BITS; n; n--) { + hash = (hash << VALUE_HASH_SHIFT) ^ + (hash >> (8*sizeof(hash) - VALUE_HASH_SHIFT)) ^ + le32_to_cpu(*value++); + } + } + entry->e_hash = cpu_to_le32(hash); +} + +#undef NAME_HASH_SHIFT +#undef VALUE_HASH_SHIFT + +#define BLOCK_HASH_SHIFT 16 + +/* + * ext2_xattr_rehash() + * + * Re-compute the extended attribute hash value after an entry has changed. + */ +static void ext2_xattr_rehash(struct ext2_xattr_header *header, + struct ext2_xattr_entry *entry) +{ + struct ext2_xattr_entry *here; + __u32 hash = 0; + + ext2_xattr_hash_entry(header, entry); + here = ENTRY(header+1); + while (!IS_LAST_ENTRY(here)) { + if (!here->e_hash) { + /* Block is not shared if an entry's hash value == 0 */ + hash = 0; + break; + } + hash = (hash << BLOCK_HASH_SHIFT) ^ + (hash >> (8*sizeof(hash) - BLOCK_HASH_SHIFT)) ^ + le32_to_cpu(here->e_hash); + here = EXT2_XATTR_NEXT(here); + } + header->h_hash = cpu_to_le32(hash); +} + +#undef BLOCK_HASH_SHIFT + +#define HASH_BUCKET_BITS 10 + +struct mb_cache *ext2_xattr_create_cache(void) +{ + return mb_cache_create(HASH_BUCKET_BITS); +} + +void ext2_xattr_destroy_cache(struct mb_cache *cache) +{ + if (cache) + mb_cache_destroy(cache); +} diff --git a/fs/ext2/xattr.h b/fs/ext2/xattr.h new file mode 100644 index 000000000..6a4966949 --- /dev/null +++ b/fs/ext2/xattr.h @@ -0,0 +1,116 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + File: linux/ext2_xattr.h + + On-disk format of extended attributes for the ext2 filesystem. + + (C) 2001 Andreas Gruenbacher, +*/ + +#include +#include + +/* Magic value in attribute blocks */ +#define EXT2_XATTR_MAGIC 0xEA020000 + +/* Maximum number of references to one attribute block */ +#define EXT2_XATTR_REFCOUNT_MAX 1024 + +/* Name indexes */ +#define EXT2_XATTR_INDEX_USER 1 +#define EXT2_XATTR_INDEX_POSIX_ACL_ACCESS 2 +#define EXT2_XATTR_INDEX_POSIX_ACL_DEFAULT 3 +#define EXT2_XATTR_INDEX_TRUSTED 4 +#define EXT2_XATTR_INDEX_LUSTRE 5 +#define EXT2_XATTR_INDEX_SECURITY 6 + +struct ext2_xattr_header { + __le32 h_magic; /* magic number for identification */ + __le32 h_refcount; /* reference count */ + __le32 h_blocks; /* number of disk blocks used */ + __le32 h_hash; /* hash value of all attributes */ + __u32 h_reserved[4]; /* zero right now */ +}; + +struct ext2_xattr_entry { + __u8 e_name_len; /* length of name */ + __u8 e_name_index; /* attribute name index */ + __le16 e_value_offs; /* offset in disk block of value */ + __le32 e_value_block; /* disk block attribute is stored on (n/i) */ + __le32 e_value_size; /* size of attribute value */ + __le32 e_hash; /* hash value of name and value */ + char e_name[]; /* attribute name */ +}; + +#define EXT2_XATTR_PAD_BITS 2 +#define EXT2_XATTR_PAD (1<e_name_len)) ) +#define EXT2_XATTR_SIZE(size) \ + (((size) + EXT2_XATTR_ROUND) & ~EXT2_XATTR_ROUND) + +struct mb_cache; + +# ifdef CONFIG_EXT2_FS_XATTR + +extern const struct xattr_handler ext2_xattr_user_handler; +extern const struct xattr_handler ext2_xattr_trusted_handler; +extern const struct xattr_handler ext2_xattr_security_handler; + +extern ssize_t ext2_listxattr(struct dentry *, char *, size_t); + +extern int ext2_xattr_get(struct inode *, int, const char *, void *, size_t); +extern int ext2_xattr_set(struct inode *, int, const char *, const void *, size_t, int); + +extern void ext2_xattr_delete_inode(struct inode *); + +extern struct mb_cache *ext2_xattr_create_cache(void); +extern void ext2_xattr_destroy_cache(struct mb_cache *cache); + +extern const struct xattr_handler * const ext2_xattr_handlers[]; + +# else /* CONFIG_EXT2_FS_XATTR */ + +static inline int +ext2_xattr_get(struct inode *inode, int name_index, + const char *name, void *buffer, size_t size) +{ + return -EOPNOTSUPP; +} + +static inline int +ext2_xattr_set(struct inode *inode, int name_index, const char *name, + const void *value, size_t size, int flags) +{ + return -EOPNOTSUPP; +} + +static inline void +ext2_xattr_delete_inode(struct inode *inode) +{ +} + +static inline void ext2_xattr_destroy_cache(struct mb_cache *cache) +{ +} + +#define ext2_xattr_handlers NULL +#define ext2_listxattr NULL + +# endif /* CONFIG_EXT2_FS_XATTR */ + +#ifdef CONFIG_EXT2_FS_SECURITY +extern int ext2_init_security(struct inode *inode, struct inode *dir, + const struct qstr *qstr); +#else +static inline int ext2_init_security(struct inode *inode, struct inode *dir, + const struct qstr *qstr) +{ + return 0; +} +#endif diff --git a/fs/ext2/xattr_security.c b/fs/ext2/xattr_security.c new file mode 100644 index 000000000..db47b8ab1 --- /dev/null +++ b/fs/ext2/xattr_security.c @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext2/xattr_security.c + * Handler for storing security labels as extended attributes. + */ + +#include "ext2.h" +#include +#include "xattr.h" + +static int +ext2_xattr_security_get(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + return ext2_xattr_get(inode, EXT2_XATTR_INDEX_SECURITY, name, + buffer, size); +} + +static int +ext2_xattr_security_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +{ + return ext2_xattr_set(inode, EXT2_XATTR_INDEX_SECURITY, name, + value, size, flags); +} + +static int ext2_initxattrs(struct inode *inode, const struct xattr *xattr_array, + void *fs_info) +{ + const struct xattr *xattr; + int err = 0; + + for (xattr = xattr_array; xattr->name != NULL; xattr++) { + err = ext2_xattr_set(inode, EXT2_XATTR_INDEX_SECURITY, + xattr->name, xattr->value, + xattr->value_len, 0); + if (err < 0) + break; + } + return err; +} + +int +ext2_init_security(struct inode *inode, struct inode *dir, + const struct qstr *qstr) +{ + return security_inode_init_security(inode, dir, qstr, + &ext2_initxattrs, NULL); +} + +const struct xattr_handler ext2_xattr_security_handler = { + .prefix = XATTR_SECURITY_PREFIX, + .get = ext2_xattr_security_get, + .set = ext2_xattr_security_set, +}; diff --git a/fs/ext2/xattr_trusted.c b/fs/ext2/xattr_trusted.c new file mode 100644 index 000000000..995f93122 --- /dev/null +++ b/fs/ext2/xattr_trusted.c @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext2/xattr_trusted.c + * Handler for trusted extended attributes. + * + * Copyright (C) 2003 by Andreas Gruenbacher, + */ + +#include "ext2.h" +#include "xattr.h" + +static bool +ext2_xattr_trusted_list(struct dentry *dentry) +{ + return capable(CAP_SYS_ADMIN); +} + +static int +ext2_xattr_trusted_get(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + return ext2_xattr_get(inode, EXT2_XATTR_INDEX_TRUSTED, name, + buffer, size); +} + +static int +ext2_xattr_trusted_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +{ + return ext2_xattr_set(inode, EXT2_XATTR_INDEX_TRUSTED, name, + value, size, flags); +} + +const struct xattr_handler ext2_xattr_trusted_handler = { + .prefix = XATTR_TRUSTED_PREFIX, + .list = ext2_xattr_trusted_list, + .get = ext2_xattr_trusted_get, + .set = ext2_xattr_trusted_set, +}; diff --git a/fs/ext2/xattr_user.c b/fs/ext2/xattr_user.c new file mode 100644 index 000000000..dd1507231 --- /dev/null +++ b/fs/ext2/xattr_user.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext2/xattr_user.c + * Handler for extended user attributes. + * + * Copyright (C) 2001 by Andreas Gruenbacher, + */ + +#include +#include +#include "ext2.h" +#include "xattr.h" + +static bool +ext2_xattr_user_list(struct dentry *dentry) +{ + return test_opt(dentry->d_sb, XATTR_USER); +} + +static int +ext2_xattr_user_get(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + if (!test_opt(inode->i_sb, XATTR_USER)) + return -EOPNOTSUPP; + return ext2_xattr_get(inode, EXT2_XATTR_INDEX_USER, + name, buffer, size); +} + +static int +ext2_xattr_user_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +{ + if (!test_opt(inode->i_sb, XATTR_USER)) + return -EOPNOTSUPP; + + return ext2_xattr_set(inode, EXT2_XATTR_INDEX_USER, + name, value, size, flags); +} + +const struct xattr_handler ext2_xattr_user_handler = { + .prefix = XATTR_USER_PREFIX, + .list = ext2_xattr_user_list, + .get = ext2_xattr_user_get, + .set = ext2_xattr_user_set, +}; diff --git a/fs/ext4/.kunitconfig b/fs/ext4/.kunitconfig new file mode 100644 index 000000000..bf51da7cd --- /dev/null +++ b/fs/ext4/.kunitconfig @@ -0,0 +1,3 @@ +CONFIG_KUNIT=y +CONFIG_EXT4_FS=y +CONFIG_EXT4_KUNIT_TESTS=y diff --git a/fs/ext4/Kconfig b/fs/ext4/Kconfig new file mode 100644 index 000000000..01873c2a3 --- /dev/null +++ b/fs/ext4/Kconfig @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: GPL-2.0-only +config EXT4_FS + tristate "The Extended 4 (ext4) filesystem" + select BUFFER_HEAD + select JBD2 + select CRC16 + select CRC32 + select FS_IOMAP + select FS_ENCRYPTION_ALGS if FS_ENCRYPTION + help + This is the next generation of the ext3 filesystem. + + Unlike the change from ext2 filesystem to ext3 filesystem, + the on-disk format of ext4 is not forwards compatible with + ext3; it is based on extent maps and it supports 48-bit + physical block numbers. The ext4 filesystem also supports delayed + allocation, persistent preallocation, high resolution time stamps, + and a number of other features to improve performance and speed + up fsck time. For more information, please see the web pages at + http://ext4.wiki.kernel.org. + + The ext4 filesystem supports mounting an ext3 filesystem; while there + are some performance gains from the delayed allocation and inode + table readahead, the best performance gains require enabling ext4 + features in the filesystem using tune2fs, or formatting a new + filesystem as an ext4 filesystem initially. Without explicit enabling + of ext4 features, the on disk filesystem format stays fully backward + compatible. + + To compile this file system support as a module, choose M here. The + module will be called ext4. + + If unsure, say N. + +config EXT4_USE_FOR_EXT2 + bool "Use ext4 for ext2 file systems" + depends on EXT4_FS + depends on EXT2_FS=n + default y + help + Allow the ext4 file system driver code to be used for ext2 + file system mounts. This allows users to reduce their + compiled kernel size by using one file system driver for + ext2, ext3, and ext4 file systems. + +config EXT4_FS_POSIX_ACL + bool "Ext4 POSIX Access Control Lists" + depends on EXT4_FS + select FS_POSIX_ACL + help + POSIX Access Control Lists (ACLs) support permissions for users and + groups beyond the owner/group/world scheme. + + If you don't know what Access Control Lists are, say N + +config EXT4_FS_SECURITY + bool "Ext4 Security Labels" + depends on EXT4_FS + help + Security labels support alternative access control models + implemented by security modules like SELinux. This option + enables an extended attribute handler for file security + labels in the ext4 filesystem. + + If you are not using a security module that requires using + extended attributes for file security labels, say N. + +config EXT4_DEBUG + bool "Ext4 debugging support" + depends on EXT4_FS + help + Enables run-time debugging support for the ext4 filesystem. + + If you select Y here, then you will be able to turn on debugging + using dynamic debug control for mb_debug() / ext_debug() msgs. + +config EXT4_KUNIT_TESTS + tristate "KUnit tests for ext4" if !KUNIT_ALL_TESTS + depends on EXT4_FS && KUNIT + default KUNIT_ALL_TESTS + help + This builds the ext4 KUnit tests. + + KUnit tests run during boot and output the results to the debug log + in TAP format (https://testanything.org/). Only useful for kernel devs + running KUnit test harness and are not for inclusion into a production + build. + + For more information on KUnit and unit tests in general please refer + to the KUnit documentation in Documentation/dev-tools/kunit/. + + If unsure, say N. diff --git a/fs/ext4/Makefile b/fs/ext4/Makefile new file mode 100644 index 000000000..3f9fc0eb8 --- /dev/null +++ b/fs/ext4/Makefile @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the linux ext4-filesystem routines. +# + +obj-$(CONFIG_EXT4_FS) += ext4.o + +ext4-y := balloc.o bitmap.o block_validity.o dir.o ext4_jbd2.o extents.o \ + extents_status.o file.o fsmap.o fsync.o hash.o ialloc.o \ + indirect.o inline.o inode.o ioctl.o mballoc.o migrate.o \ + mmp.o move_extent.o namei.o page-io.o readpage.o resize.o \ + super.o symlink.o sysfs.o xattr.o xattr_hurd.o xattr_trusted.o \ + xattr_user.o fast_commit.o orphan.o + +ext4-$(CONFIG_EXT4_FS_POSIX_ACL) += acl.o +ext4-$(CONFIG_EXT4_FS_SECURITY) += xattr_security.o +ext4-test-objs += inode-test.o mballoc-test.o \ + extents-test.o hash-test.o +obj-$(CONFIG_EXT4_KUNIT_TESTS) += ext4-test.o +ext4-$(CONFIG_FS_VERITY) += verity.o +ext4-$(CONFIG_FS_ENCRYPTION) += crypto.o diff --git a/fs/ext4/acl.c b/fs/ext4/acl.c new file mode 100644 index 000000000..3bffe862f --- /dev/null +++ b/fs/ext4/acl.c @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext4/acl.c + * + * Copyright (C) 2001-2003 Andreas Gruenbacher, + */ + +#include +#include "ext4_jbd2.h" +#include "ext4.h" +#include "xattr.h" +#include "acl.h" + +/* + * Convert from filesystem to in-memory representation. + */ +static struct posix_acl * +ext4_acl_from_disk(const void *value, size_t size) +{ + const char *end = (char *)value + size; + int n, count; + struct posix_acl *acl; + + if (!value) + return NULL; + if (size < sizeof(ext4_acl_header)) + return ERR_PTR(-EINVAL); + if (((ext4_acl_header *)value)->a_version != + cpu_to_le32(EXT4_ACL_VERSION)) + return ERR_PTR(-EINVAL); + value = (char *)value + sizeof(ext4_acl_header); + count = ext4_acl_count(size); + if (count < 0) + return ERR_PTR(-EINVAL); + if (count == 0) + return NULL; + acl = posix_acl_alloc(count, GFP_NOFS); + if (!acl) + return ERR_PTR(-ENOMEM); + for (n = 0; n < count; n++) { + ext4_acl_entry *entry = + (ext4_acl_entry *)value; + if ((char *)value + sizeof(ext4_acl_entry_short) > end) + goto fail; + acl->a_entries[n].e_tag = le16_to_cpu(entry->e_tag); + acl->a_entries[n].e_perm = le16_to_cpu(entry->e_perm); + + switch (acl->a_entries[n].e_tag) { + case ACL_USER_OBJ: + case ACL_GROUP_OBJ: + case ACL_MASK: + case ACL_OTHER: + value = (char *)value + + sizeof(ext4_acl_entry_short); + break; + + case ACL_USER: + value = (char *)value + sizeof(ext4_acl_entry); + if ((char *)value > end) + goto fail; + acl->a_entries[n].e_uid = + make_kuid(&init_user_ns, + le32_to_cpu(entry->e_id)); + break; + case ACL_GROUP: + value = (char *)value + sizeof(ext4_acl_entry); + if ((char *)value > end) + goto fail; + acl->a_entries[n].e_gid = + make_kgid(&init_user_ns, + le32_to_cpu(entry->e_id)); + break; + + default: + goto fail; + } + } + if (value != end) + goto fail; + return acl; + +fail: + posix_acl_release(acl); + return ERR_PTR(-EINVAL); +} + +/* + * Convert from in-memory to filesystem representation. + */ +static void * +ext4_acl_to_disk(const struct posix_acl *acl, size_t *size) +{ + ext4_acl_header *ext_acl; + char *e; + size_t n; + + *size = ext4_acl_size(acl->a_count); + ext_acl = kmalloc(sizeof(ext4_acl_header) + acl->a_count * + sizeof(ext4_acl_entry), GFP_NOFS); + if (!ext_acl) + return ERR_PTR(-ENOMEM); + ext_acl->a_version = cpu_to_le32(EXT4_ACL_VERSION); + e = (char *)ext_acl + sizeof(ext4_acl_header); + for (n = 0; n < acl->a_count; n++) { + const struct posix_acl_entry *acl_e = &acl->a_entries[n]; + ext4_acl_entry *entry = (ext4_acl_entry *)e; + entry->e_tag = cpu_to_le16(acl_e->e_tag); + entry->e_perm = cpu_to_le16(acl_e->e_perm); + switch (acl_e->e_tag) { + case ACL_USER: + entry->e_id = cpu_to_le32( + from_kuid(&init_user_ns, acl_e->e_uid)); + e += sizeof(ext4_acl_entry); + break; + case ACL_GROUP: + entry->e_id = cpu_to_le32( + from_kgid(&init_user_ns, acl_e->e_gid)); + e += sizeof(ext4_acl_entry); + break; + + case ACL_USER_OBJ: + case ACL_GROUP_OBJ: + case ACL_MASK: + case ACL_OTHER: + e += sizeof(ext4_acl_entry_short); + break; + + default: + goto fail; + } + } + return (char *)ext_acl; + +fail: + kfree(ext_acl); + return ERR_PTR(-EINVAL); +} + +/* + * Inode operation get_posix_acl(). + * + * inode->i_rwsem: don't care + */ +struct posix_acl * +ext4_get_acl(struct inode *inode, int type, bool rcu) +{ + int name_index; + char *value = NULL; + struct posix_acl *acl; + int retval; + + if (rcu) + return ERR_PTR(-ECHILD); + + switch (type) { + case ACL_TYPE_ACCESS: + name_index = EXT4_XATTR_INDEX_POSIX_ACL_ACCESS; + break; + case ACL_TYPE_DEFAULT: + name_index = EXT4_XATTR_INDEX_POSIX_ACL_DEFAULT; + break; + default: + BUG(); + } + retval = ext4_xattr_get(inode, name_index, "", NULL, 0); + if (retval > 0) { + value = kmalloc(retval, GFP_NOFS); + if (!value) + return ERR_PTR(-ENOMEM); + retval = ext4_xattr_get(inode, name_index, "", value, retval); + } + if (retval > 0) + acl = ext4_acl_from_disk(value, retval); + else if (retval == -ENODATA || retval == -ENOSYS) + acl = NULL; + else + acl = ERR_PTR(retval); + kfree(value); + + return acl; +} + +/* + * Set the access or default ACL of an inode. + * + * inode->i_rwsem: down unless called from ext4_new_inode + */ +static int +__ext4_set_acl(handle_t *handle, struct inode *inode, int type, + struct posix_acl *acl, int xattr_flags) +{ + int name_index; + void *value = NULL; + size_t size = 0; + int error; + + switch (type) { + case ACL_TYPE_ACCESS: + name_index = EXT4_XATTR_INDEX_POSIX_ACL_ACCESS; + break; + + case ACL_TYPE_DEFAULT: + name_index = EXT4_XATTR_INDEX_POSIX_ACL_DEFAULT; + if (!S_ISDIR(inode->i_mode)) + return acl ? -EACCES : 0; + break; + + default: + return -EINVAL; + } + if (acl) { + value = ext4_acl_to_disk(acl, &size); + if (IS_ERR(value)) + return (int)PTR_ERR(value); + } + + error = ext4_xattr_set_handle(handle, inode, name_index, "", + value, size, xattr_flags); + + kfree(value); + if (!error) + set_cached_acl(inode, type, acl); + + return error; +} + +int +ext4_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type) +{ + handle_t *handle; + int error, credits, retries = 0; + size_t acl_size = acl ? ext4_acl_size(acl->a_count) : 0; + struct inode *inode = d_inode(dentry); + umode_t mode = inode->i_mode; + int update_mode = 0; + + error = dquot_initialize(inode); + if (error) + return error; +retry: + error = ext4_xattr_set_credits(inode, acl_size, false /* is_create */, + &credits); + if (error) + return error; + + handle = ext4_journal_start(inode, EXT4_HT_XATTR, credits); + if (IS_ERR(handle)) + return PTR_ERR(handle); + + if ((type == ACL_TYPE_ACCESS) && acl) { + error = posix_acl_update_mode(idmap, inode, &mode, &acl); + if (error) + goto out_stop; + if (mode != inode->i_mode) + update_mode = 1; + } + + error = __ext4_set_acl(handle, inode, type, acl, 0 /* xattr_flags */); + if (!error && update_mode) { + inode->i_mode = mode; + inode_set_ctime_current(inode); + error = ext4_mark_inode_dirty(handle, inode); + } +out_stop: + ext4_journal_stop(handle); + if (error == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)) + goto retry; + return error; +} + +/* + * Initialize the ACLs of a new inode. Called from ext4_new_inode. + * + * dir->i_rwsem: down + * inode->i_rwsem: up (access to inode is still exclusive) + */ +int +ext4_init_acl(handle_t *handle, struct inode *inode, struct inode *dir) +{ + struct posix_acl *default_acl, *acl; + int error; + + error = posix_acl_create(dir, &inode->i_mode, &default_acl, &acl); + if (error) + return error; + + if (default_acl) { + error = __ext4_set_acl(handle, inode, ACL_TYPE_DEFAULT, + default_acl, XATTR_CREATE); + posix_acl_release(default_acl); + } else { + inode->i_default_acl = NULL; + } + if (acl) { + if (!error) + error = __ext4_set_acl(handle, inode, ACL_TYPE_ACCESS, + acl, XATTR_CREATE); + posix_acl_release(acl); + } else { + inode->i_acl = NULL; + } + return error; +} diff --git a/fs/ext4/acl.h b/fs/ext4/acl.h new file mode 100644 index 000000000..0c5a79c3b --- /dev/null +++ b/fs/ext4/acl.h @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + File: fs/ext4/acl.h + + (C) 2001 Andreas Gruenbacher, +*/ + +#include + +#define EXT4_ACL_VERSION 0x0001 + +typedef struct { + __le16 e_tag; + __le16 e_perm; + __le32 e_id; +} ext4_acl_entry; + +typedef struct { + __le16 e_tag; + __le16 e_perm; +} ext4_acl_entry_short; + +typedef struct { + __le32 a_version; +} ext4_acl_header; + +static inline size_t ext4_acl_size(int count) +{ + if (count <= 4) { + return sizeof(ext4_acl_header) + + count * sizeof(ext4_acl_entry_short); + } else { + return sizeof(ext4_acl_header) + + 4 * sizeof(ext4_acl_entry_short) + + (count - 4) * sizeof(ext4_acl_entry); + } +} + +static inline int ext4_acl_count(size_t size) +{ + ssize_t s; + size -= sizeof(ext4_acl_header); + s = size - 4 * sizeof(ext4_acl_entry_short); + if (s < 0) { + if (size % sizeof(ext4_acl_entry_short)) + return -1; + return size / sizeof(ext4_acl_entry_short); + } else { + if (s % sizeof(ext4_acl_entry)) + return -1; + return s / sizeof(ext4_acl_entry) + 4; + } +} + +#ifdef CONFIG_EXT4_FS_POSIX_ACL + +/* acl.c */ +struct posix_acl *ext4_get_acl(struct inode *inode, int type, bool rcu); +int ext4_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type); +extern int ext4_init_acl(handle_t *, struct inode *, struct inode *); + +#else /* CONFIG_EXT4_FS_POSIX_ACL */ +#include +#define ext4_get_acl NULL +#define ext4_set_acl NULL + +static inline int +ext4_init_acl(handle_t *handle, struct inode *inode, struct inode *dir) +{ + return 0; +} +#endif /* CONFIG_EXT4_FS_POSIX_ACL */ + diff --git a/fs/ext4/balloc.c b/fs/ext4/balloc.c new file mode 100644 index 000000000..52f4c5169 --- /dev/null +++ b/fs/ext4/balloc.c @@ -0,0 +1,1007 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext4/balloc.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * Enhanced block allocation by Stephen Tweedie (sct@redhat.com), 1993 + * Big-endian to little-endian byte-swapping/bitmaps by + * David S. Miller (davem@caip.rutgers.edu), 1995 + */ + +#include +#include +#include +#include +#include +#include "ext4.h" +#include "ext4_jbd2.h" +#include "mballoc.h" + +#include +#include + +static unsigned ext4_num_base_meta_clusters(struct super_block *sb, + ext4_group_t block_group); +/* + * balloc.c contains the blocks allocation and deallocation routines + */ + +/* + * Calculate block group number for a given block number + */ +ext4_group_t ext4_get_group_number(struct super_block *sb, + ext4_fsblk_t block) +{ + ext4_group_t group; + + if (test_opt2(sb, STD_GROUP_SIZE)) + group = (block - + le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block)) >> + (EXT4_BLOCK_SIZE_BITS(sb) + EXT4_CLUSTER_BITS(sb) + 3); + else + ext4_get_group_no_and_offset(sb, block, &group, NULL); + return group; +} + +/* + * Calculate the block group number and offset into the block/cluster + * allocation bitmap, given a block number + */ +void ext4_get_group_no_and_offset(struct super_block *sb, ext4_fsblk_t blocknr, + ext4_group_t *blockgrpp, ext4_grpblk_t *offsetp) +{ + struct ext4_super_block *es = EXT4_SB(sb)->s_es; + ext4_grpblk_t offset; + + blocknr = blocknr - le32_to_cpu(es->s_first_data_block); + offset = do_div(blocknr, EXT4_BLOCKS_PER_GROUP(sb)) >> + EXT4_SB(sb)->s_cluster_bits; + if (offsetp) + *offsetp = offset; + if (blockgrpp) + *blockgrpp = blocknr; + +} + +/* + * Check whether the 'block' lives within the 'block_group'. Returns 1 if so + * and 0 otherwise. + */ +static inline int ext4_block_in_group(struct super_block *sb, + ext4_fsblk_t block, + ext4_group_t block_group) +{ + ext4_group_t actual_group; + + actual_group = ext4_get_group_number(sb, block); + return (actual_group == block_group) ? 1 : 0; +} + +/* + * Return the number of clusters used for file system metadata; this + * represents the overhead needed by the file system. + */ +static unsigned ext4_num_overhead_clusters(struct super_block *sb, + ext4_group_t block_group, + struct ext4_group_desc *gdp) +{ + unsigned base_clusters, num_clusters; + int block_cluster = -1, inode_cluster; + int itbl_cluster_start = -1, itbl_cluster_end = -1; + ext4_fsblk_t start = ext4_group_first_block_no(sb, block_group); + ext4_fsblk_t end = start + EXT4_BLOCKS_PER_GROUP(sb) - 1; + ext4_fsblk_t itbl_blk_start, itbl_blk_end; + struct ext4_sb_info *sbi = EXT4_SB(sb); + + /* This is the number of clusters used by the superblock, + * block group descriptors, and reserved block group + * descriptor blocks */ + base_clusters = ext4_num_base_meta_clusters(sb, block_group); + num_clusters = base_clusters; + + /* + * Account and record inode table clusters if any cluster + * is in the block group, or inode table cluster range is + * [-1, -1] and won't overlap with block/inode bitmap cluster + * accounted below. + */ + itbl_blk_start = ext4_inode_table(sb, gdp); + itbl_blk_end = itbl_blk_start + sbi->s_itb_per_group - 1; + if (itbl_blk_start <= end && itbl_blk_end >= start) { + itbl_blk_start = max(itbl_blk_start, start); + itbl_blk_end = min(itbl_blk_end, end); + + itbl_cluster_start = EXT4_B2C(sbi, itbl_blk_start - start); + itbl_cluster_end = EXT4_B2C(sbi, itbl_blk_end - start); + + num_clusters += itbl_cluster_end - itbl_cluster_start + 1; + /* check if border cluster is overlapped */ + if (itbl_cluster_start == base_clusters - 1) + num_clusters--; + } + + /* + * For the allocation bitmaps, we first need to check to see + * if the block is in the block group. If it is, then check + * to see if the cluster is already accounted for in the clusters + * used for the base metadata cluster and inode tables cluster. + * Normally all of these blocks are contiguous, so the special + * case handling shouldn't be necessary except for *very* + * unusual file system layouts. + */ + if (ext4_block_in_group(sb, ext4_block_bitmap(sb, gdp), block_group)) { + block_cluster = EXT4_B2C(sbi, + ext4_block_bitmap(sb, gdp) - start); + if (block_cluster >= base_clusters && + (block_cluster < itbl_cluster_start || + block_cluster > itbl_cluster_end)) + num_clusters++; + } + + if (ext4_block_in_group(sb, ext4_inode_bitmap(sb, gdp), block_group)) { + inode_cluster = EXT4_B2C(sbi, + ext4_inode_bitmap(sb, gdp) - start); + /* + * Additional check if inode bitmap is in just accounted + * block_cluster + */ + if (inode_cluster != block_cluster && + inode_cluster >= base_clusters && + (inode_cluster < itbl_cluster_start || + inode_cluster > itbl_cluster_end)) + num_clusters++; + } + + return num_clusters; +} + +static unsigned int num_clusters_in_group(struct super_block *sb, + ext4_group_t block_group) +{ + unsigned int blocks; + + if (block_group == ext4_get_groups_count(sb) - 1) { + /* + * Even though mke2fs always initializes the first and + * last group, just in case some other tool was used, + * we need to make sure we calculate the right free + * blocks. + */ + blocks = ext4_blocks_count(EXT4_SB(sb)->s_es) - + ext4_group_first_block_no(sb, block_group); + } else + blocks = EXT4_BLOCKS_PER_GROUP(sb); + return EXT4_NUM_B2C(EXT4_SB(sb), blocks); +} + +/* Initializes an uninitialized block bitmap */ +static int ext4_init_block_bitmap(struct super_block *sb, + struct buffer_head *bh, + ext4_group_t block_group, + struct ext4_group_desc *gdp) +{ + unsigned int bit, bit_max; + struct ext4_sb_info *sbi = EXT4_SB(sb); + ext4_fsblk_t start, tmp; + + ASSERT(buffer_locked(bh)); + + if (!ext4_group_desc_csum_verify(sb, block_group, gdp)) { + ext4_mark_group_bitmap_corrupted(sb, block_group, + EXT4_GROUP_INFO_BBITMAP_CORRUPT | + EXT4_GROUP_INFO_IBITMAP_CORRUPT); + return -EFSBADCRC; + } + memset(bh->b_data, 0, sb->s_blocksize); + + bit_max = ext4_num_base_meta_clusters(sb, block_group); + if ((bit_max >> 3) >= bh->b_size) + return -EFSCORRUPTED; + + for (bit = 0; bit < bit_max; bit++) + ext4_set_bit(bit, bh->b_data); + + start = ext4_group_first_block_no(sb, block_group); + + /* Set bits for block and inode bitmaps, and inode table */ + tmp = ext4_block_bitmap(sb, gdp); + if (ext4_block_in_group(sb, tmp, block_group)) + ext4_set_bit(EXT4_B2C(sbi, tmp - start), bh->b_data); + + tmp = ext4_inode_bitmap(sb, gdp); + if (ext4_block_in_group(sb, tmp, block_group)) + ext4_set_bit(EXT4_B2C(sbi, tmp - start), bh->b_data); + + tmp = ext4_inode_table(sb, gdp); + for (; tmp < ext4_inode_table(sb, gdp) + + sbi->s_itb_per_group; tmp++) { + if (ext4_block_in_group(sb, tmp, block_group)) + ext4_set_bit(EXT4_B2C(sbi, tmp - start), bh->b_data); + } + + /* + * Also if the number of blocks within the group is less than + * the blocksize * 8 ( which is the size of bitmap ), set rest + * of the block bitmap to 1 + */ + ext4_mark_bitmap_end(num_clusters_in_group(sb, block_group), + sb->s_blocksize * 8, bh->b_data); + return 0; +} + +/* Return the number of free blocks in a block group. It is used when + * the block bitmap is uninitialized, so we can't just count the bits + * in the bitmap. */ +unsigned ext4_free_clusters_after_init(struct super_block *sb, + ext4_group_t block_group, + struct ext4_group_desc *gdp) +{ + return num_clusters_in_group(sb, block_group) - + ext4_num_overhead_clusters(sb, block_group, gdp); +} + +/* + * The free blocks are managed by bitmaps. A file system contains several + * blocks groups. Each group contains 1 bitmap block for blocks, 1 bitmap + * block for inodes, N blocks for the inode table and data blocks. + * + * The file system contains group descriptors which are located after the + * super block. Each descriptor contains the number of the bitmap block and + * the free blocks count in the block. The descriptors are loaded in memory + * when a file system is mounted (see ext4_fill_super). + */ + +/** + * ext4_get_group_desc() -- load group descriptor from disk + * @sb: super block + * @block_group: given block group + * @bh: pointer to the buffer head to store the block + * group descriptor + */ +struct ext4_group_desc * ext4_get_group_desc(struct super_block *sb, + ext4_group_t block_group, + struct buffer_head **bh) +{ + unsigned int group_desc; + unsigned int offset; + ext4_group_t ngroups = ext4_get_groups_count(sb); + struct ext4_group_desc *desc; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct buffer_head *bh_p; + + KUNIT_STATIC_STUB_REDIRECT(ext4_get_group_desc, + sb, block_group, bh); + + if (block_group >= ngroups) { + ext4_error(sb, "block_group >= groups_count - block_group = %u," + " groups_count = %u", block_group, ngroups); + + return NULL; + } + + group_desc = block_group >> EXT4_DESC_PER_BLOCK_BITS(sb); + offset = block_group & (EXT4_DESC_PER_BLOCK(sb) - 1); + bh_p = sbi_array_rcu_deref(sbi, s_group_desc, group_desc); + /* + * sbi_array_rcu_deref returns with rcu unlocked, this is ok since + * the pointer being dereferenced won't be dereferenced again. By + * looking at the usage in add_new_gdb() the value isn't modified, + * just the pointer, and so it remains valid. + */ + if (!bh_p) { + ext4_error(sb, "Group descriptor not loaded - " + "block_group = %u, group_desc = %u, desc = %u", + block_group, group_desc, offset); + return NULL; + } + + desc = (struct ext4_group_desc *)( + (__u8 *)bh_p->b_data + + offset * EXT4_DESC_SIZE(sb)); + if (bh) + *bh = bh_p; + return desc; +} + +static ext4_fsblk_t ext4_valid_block_bitmap_padding(struct super_block *sb, + ext4_group_t block_group, + struct buffer_head *bh) +{ + ext4_grpblk_t next_zero_bit; + unsigned long bitmap_size = sb->s_blocksize * 8; + unsigned int offset = num_clusters_in_group(sb, block_group); + + if (bitmap_size <= offset) + return 0; + + next_zero_bit = ext4_find_next_zero_bit(bh->b_data, bitmap_size, offset); + + return (next_zero_bit < bitmap_size ? next_zero_bit : 0); +} + +struct ext4_group_info *ext4_get_group_info(struct super_block *sb, + ext4_group_t group) +{ + struct ext4_group_info **grp_info; + long indexv, indexh; + + if (unlikely(group >= EXT4_SB(sb)->s_groups_count)) + return NULL; + if (unlikely(!EXT4_SB(sb)->s_group_info)) + return NULL; + indexv = group >> (EXT4_DESC_PER_BLOCK_BITS(sb)); + indexh = group & ((EXT4_DESC_PER_BLOCK(sb)) - 1); + grp_info = sbi_array_rcu_deref(EXT4_SB(sb), s_group_info, indexv); + if (unlikely(!grp_info)) + return NULL; + return grp_info[indexh]; +} + +/* + * Return the block number which was discovered to be invalid, or 0 if + * the block bitmap is valid. + */ +static ext4_fsblk_t ext4_valid_block_bitmap(struct super_block *sb, + struct ext4_group_desc *desc, + ext4_group_t block_group, + struct buffer_head *bh) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + ext4_grpblk_t offset; + ext4_grpblk_t next_zero_bit; + ext4_grpblk_t max_bit = EXT4_CLUSTERS_PER_GROUP(sb); + ext4_fsblk_t blk; + ext4_fsblk_t group_first_block; + + if (ext4_has_feature_flex_bg(sb)) { + /* with FLEX_BG, the inode/block bitmaps and itable + * blocks may not be in the group at all + * so the bitmap validation will be skipped for those groups + * or it has to also read the block group where the bitmaps + * are located to verify they are set. + */ + return 0; + } + group_first_block = ext4_group_first_block_no(sb, block_group); + + /* check whether block bitmap block number is set */ + blk = ext4_block_bitmap(sb, desc); + offset = blk - group_first_block; + if (offset < 0 || EXT4_B2C(sbi, offset) >= max_bit || + !ext4_test_bit(EXT4_B2C(sbi, offset), bh->b_data)) + /* bad block bitmap */ + return blk; + + /* check whether the inode bitmap block number is set */ + blk = ext4_inode_bitmap(sb, desc); + offset = blk - group_first_block; + if (offset < 0 || EXT4_B2C(sbi, offset) >= max_bit || + !ext4_test_bit(EXT4_B2C(sbi, offset), bh->b_data)) + /* bad block bitmap */ + return blk; + + /* check whether the inode table block number is set */ + blk = ext4_inode_table(sb, desc); + offset = blk - group_first_block; + if (offset < 0 || EXT4_B2C(sbi, offset) >= max_bit || + EXT4_B2C(sbi, offset + sbi->s_itb_per_group - 1) >= max_bit) + return blk; + next_zero_bit = ext4_find_next_zero_bit(bh->b_data, + EXT4_B2C(sbi, offset + sbi->s_itb_per_group - 1) + 1, + EXT4_B2C(sbi, offset)); + if (next_zero_bit < + EXT4_B2C(sbi, offset + sbi->s_itb_per_group - 1) + 1) + /* bad bitmap for inode tables */ + return blk; + return 0; +} + +static int ext4_validate_block_bitmap(struct super_block *sb, + struct ext4_group_desc *desc, + ext4_group_t block_group, + struct buffer_head *bh) +{ + ext4_fsblk_t blk; + struct ext4_group_info *grp; + + if (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY) + return 0; + + grp = ext4_get_group_info(sb, block_group); + + if (buffer_verified(bh)) + return 0; + if (!grp || EXT4_MB_GRP_BBITMAP_CORRUPT(grp)) + return -EFSCORRUPTED; + + ext4_lock_group(sb, block_group); + if (buffer_verified(bh)) + goto verified; + if (unlikely(!ext4_block_bitmap_csum_verify(sb, desc, bh) || + ext4_simulate_fail(sb, EXT4_SIM_BBITMAP_CRC))) { + ext4_unlock_group(sb, block_group); + ext4_error(sb, "bg %u: bad block bitmap checksum", block_group); + ext4_mark_group_bitmap_corrupted(sb, block_group, + EXT4_GROUP_INFO_BBITMAP_CORRUPT); + return -EFSBADCRC; + } + blk = ext4_valid_block_bitmap(sb, desc, block_group, bh); + if (unlikely(blk != 0)) { + ext4_unlock_group(sb, block_group); + ext4_error(sb, "bg %u: block %llu: invalid block bitmap", + block_group, blk); + ext4_mark_group_bitmap_corrupted(sb, block_group, + EXT4_GROUP_INFO_BBITMAP_CORRUPT); + return -EFSCORRUPTED; + } + blk = ext4_valid_block_bitmap_padding(sb, block_group, bh); + if (unlikely(blk != 0)) { + ext4_unlock_group(sb, block_group); + ext4_error(sb, "bg %u: block %llu: padding at end of block bitmap is not set", + block_group, blk); + ext4_mark_group_bitmap_corrupted(sb, block_group, + EXT4_GROUP_INFO_BBITMAP_CORRUPT); + return -EFSCORRUPTED; + } + set_buffer_verified(bh); +verified: + ext4_unlock_group(sb, block_group); + return 0; +} + +/** + * ext4_read_block_bitmap_nowait() + * @sb: super block + * @block_group: given block group + * @ignore_locked: ignore locked buffers + * + * Read the bitmap for a given block_group,and validate the + * bits for block/inode/inode tables are set in the bitmaps + * + * Return buffer_head on success or an ERR_PTR in case of failure. + */ +struct buffer_head * +ext4_read_block_bitmap_nowait(struct super_block *sb, ext4_group_t block_group, + bool ignore_locked) +{ + struct ext4_group_desc *desc; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct buffer_head *bh; + ext4_fsblk_t bitmap_blk; + int err; + + KUNIT_STATIC_STUB_REDIRECT(ext4_read_block_bitmap_nowait, + sb, block_group, ignore_locked); + + desc = ext4_get_group_desc(sb, block_group, NULL); + if (!desc) + return ERR_PTR(-EFSCORRUPTED); + bitmap_blk = ext4_block_bitmap(sb, desc); + if ((bitmap_blk <= le32_to_cpu(sbi->s_es->s_first_data_block)) || + (bitmap_blk >= ext4_blocks_count(sbi->s_es))) { + ext4_error(sb, "Invalid block bitmap block %llu in " + "block_group %u", bitmap_blk, block_group); + ext4_mark_group_bitmap_corrupted(sb, block_group, + EXT4_GROUP_INFO_BBITMAP_CORRUPT); + return ERR_PTR(-EFSCORRUPTED); + } + bh = sb_getblk(sb, bitmap_blk); + if (unlikely(!bh)) { + ext4_warning(sb, "Cannot get buffer for block bitmap - " + "block_group = %u, block_bitmap = %llu", + block_group, bitmap_blk); + return ERR_PTR(-ENOMEM); + } + + if (ignore_locked && buffer_locked(bh)) { + /* buffer under IO already, return if called for prefetching */ + put_bh(bh); + return NULL; + } + + if (bitmap_uptodate(bh)) + goto verify; + + lock_buffer(bh); + if (bitmap_uptodate(bh)) { + unlock_buffer(bh); + goto verify; + } + ext4_lock_group(sb, block_group); + if (ext4_has_group_desc_csum(sb) && + (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) { + if (block_group == 0) { + ext4_unlock_group(sb, block_group); + unlock_buffer(bh); + ext4_error(sb, "Block bitmap for bg 0 marked " + "uninitialized"); + err = -EFSCORRUPTED; + goto out; + } + err = ext4_init_block_bitmap(sb, bh, block_group, desc); + if (err) { + ext4_unlock_group(sb, block_group); + unlock_buffer(bh); + ext4_error(sb, "Failed to init block bitmap for group " + "%u: %d", block_group, err); + goto out; + } + set_bitmap_uptodate(bh); + set_buffer_uptodate(bh); + set_buffer_verified(bh); + ext4_unlock_group(sb, block_group); + unlock_buffer(bh); + return bh; + } + ext4_unlock_group(sb, block_group); + if (buffer_uptodate(bh)) { + /* + * if not uninit if bh is uptodate, + * bitmap is also uptodate + */ + set_bitmap_uptodate(bh); + unlock_buffer(bh); + goto verify; + } + /* + * submit the buffer_head for reading + */ + set_buffer_new(bh); + trace_ext4_read_block_bitmap_load(sb, block_group, ignore_locked); + ext4_read_bh_nowait(bh, REQ_META | REQ_PRIO | + (ignore_locked ? REQ_RAHEAD : 0), + ext4_end_bitmap_read, + ext4_simulate_fail(sb, EXT4_SIM_BBITMAP_EIO)); + return bh; +verify: + err = ext4_validate_block_bitmap(sb, desc, block_group, bh); + if (err) + goto out; + return bh; +out: + put_bh(bh); + return ERR_PTR(err); +} + +/* Returns 0 on success, -errno on error */ +int ext4_wait_block_bitmap(struct super_block *sb, ext4_group_t block_group, + struct buffer_head *bh) +{ + struct ext4_group_desc *desc; + + KUNIT_STATIC_STUB_REDIRECT(ext4_wait_block_bitmap, + sb, block_group, bh); + + if (!buffer_new(bh)) + return 0; + desc = ext4_get_group_desc(sb, block_group, NULL); + if (!desc) + return -EFSCORRUPTED; + wait_on_buffer(bh); + if (!buffer_uptodate(bh)) { + ext4_error_err(sb, EIO, "Cannot read block bitmap - " + "block_group = %u, block_bitmap = %llu", + block_group, (unsigned long long) bh->b_blocknr); + ext4_mark_group_bitmap_corrupted(sb, block_group, + EXT4_GROUP_INFO_BBITMAP_CORRUPT); + return -EIO; + } + clear_buffer_new(bh); + /* Panic or remount fs read-only if block bitmap is invalid */ + return ext4_validate_block_bitmap(sb, desc, block_group, bh); +} + +struct buffer_head * +ext4_read_block_bitmap(struct super_block *sb, ext4_group_t block_group) +{ + struct buffer_head *bh; + int err; + + bh = ext4_read_block_bitmap_nowait(sb, block_group, false); + if (IS_ERR(bh)) + return bh; + err = ext4_wait_block_bitmap(sb, block_group, bh); + if (err) { + put_bh(bh); + return ERR_PTR(err); + } + return bh; +} + +/** + * ext4_has_free_clusters() + * @sbi: in-core super block structure. + * @nclusters: number of needed blocks + * @flags: flags from ext4_mb_new_blocks() + * + * Check if filesystem has nclusters free & available for allocation. + * On success return 1, return 0 on failure. + */ +static int ext4_has_free_clusters(struct ext4_sb_info *sbi, + s64 nclusters, unsigned int flags) +{ + s64 free_clusters, dirty_clusters, rsv, resv_clusters; + struct percpu_counter *fcc = &sbi->s_freeclusters_counter; + struct percpu_counter *dcc = &sbi->s_dirtyclusters_counter; + + free_clusters = percpu_counter_read_positive(fcc); + dirty_clusters = percpu_counter_read_positive(dcc); + resv_clusters = atomic64_read(&sbi->s_resv_clusters); + + /* + * r_blocks_count should always be multiple of the cluster ratio so + * we are safe to do a plane bit shift only. + */ + rsv = (ext4_r_blocks_count(sbi->s_es) >> sbi->s_cluster_bits) + + resv_clusters; + + if (free_clusters - (nclusters + rsv + dirty_clusters) < + EXT4_FREECLUSTERS_WATERMARK) { + free_clusters = percpu_counter_sum_positive(fcc); + dirty_clusters = percpu_counter_sum_positive(dcc); + } + /* Check whether we have space after accounting for current + * dirty clusters & root reserved clusters. + */ + if (free_clusters >= (rsv + nclusters + dirty_clusters)) + return 1; + + /* Hm, nope. Are (enough) root reserved clusters available? */ + if (uid_eq(sbi->s_resuid, current_fsuid()) || + (!gid_eq(sbi->s_resgid, GLOBAL_ROOT_GID) && in_group_p(sbi->s_resgid)) || + (flags & EXT4_MB_USE_ROOT_BLOCKS) || + capable(CAP_SYS_RESOURCE)) { + + if (free_clusters >= (nclusters + dirty_clusters + + resv_clusters)) + return 1; + } + /* No free blocks. Let's see if we can dip into reserved pool */ + if (flags & EXT4_MB_USE_RESERVED) { + if (free_clusters >= (nclusters + dirty_clusters)) + return 1; + } + + return 0; +} + +int ext4_claim_free_clusters(struct ext4_sb_info *sbi, + s64 nclusters, unsigned int flags) +{ + if (ext4_has_free_clusters(sbi, nclusters, flags)) { + percpu_counter_add(&sbi->s_dirtyclusters_counter, nclusters); + return 0; + } else + return -ENOSPC; +} + +/** + * ext4_should_retry_alloc() - check if a block allocation should be retried + * @sb: superblock + * @retries: number of retry attempts made so far + * + * ext4_should_retry_alloc() is called when ENOSPC is returned while + * attempting to allocate blocks. If there's an indication that a pending + * journal transaction might free some space and allow another attempt to + * succeed, this function will wait for the current or committing transaction + * to complete and then return TRUE. + */ +int ext4_should_retry_alloc(struct super_block *sb, int *retries) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + + if (!sbi->s_journal) + return 0; + + if (++(*retries) > 3) { + percpu_counter_inc(&sbi->s_sra_exceeded_retry_limit); + return 0; + } + + /* + * if there's no indication that blocks are about to be freed it's + * possible we just missed a transaction commit that did so + */ + smp_mb(); + if (atomic_read(&sbi->s_mb_free_pending) == 0) { + if (test_opt(sb, DISCARD)) { + atomic_inc(&sbi->s_retry_alloc_pending); + flush_work(&sbi->s_discard_work); + atomic_dec(&sbi->s_retry_alloc_pending); + } + return ext4_has_free_clusters(sbi, 1, 0); + } + + /* + * it's possible we've just missed a transaction commit here, + * so ignore the returned status + */ + ext4_debug("%s: retrying operation after ENOSPC\n", sb->s_id); + (void) jbd2_journal_force_commit_nested(sbi->s_journal); + return 1; +} + +/* + * ext4_new_meta_blocks() -- allocate block for meta data (indexing) blocks + * + * @handle: handle to this transaction + * @inode: file inode + * @goal: given target block(filesystem wide) + * @count: pointer to total number of clusters needed + * @errp: error code + * + * Return 1st allocated block number on success, *count stores total account + * error stores in errp pointer + */ +ext4_fsblk_t ext4_new_meta_blocks(handle_t *handle, struct inode *inode, + ext4_fsblk_t goal, unsigned int flags, + unsigned long *count, int *errp) +{ + struct ext4_allocation_request ar; + ext4_fsblk_t ret; + + memset(&ar, 0, sizeof(ar)); + /* Fill with neighbour allocated blocks */ + ar.inode = inode; + ar.goal = goal; + ar.len = count ? *count : 1; + ar.flags = flags; + + ret = ext4_mb_new_blocks(handle, &ar, errp); + if (count) + *count = ar.len; + /* + * Account for the allocated meta blocks. We will never + * fail EDQUOT for metadata, but we do account for it. + */ + if (!(*errp) && (flags & EXT4_MB_DELALLOC_RESERVED)) { + dquot_alloc_block_nofail(inode, + EXT4_C2B(EXT4_SB(inode->i_sb), ar.len)); + } + return ret; +} + +/** + * ext4_count_free_clusters() -- count filesystem free clusters + * @sb: superblock + * + * Adds up the number of free clusters from each block group. + */ +ext4_fsblk_t ext4_count_free_clusters(struct super_block *sb) +{ + ext4_fsblk_t desc_count; + struct ext4_group_desc *gdp; + ext4_group_t i; + ext4_group_t ngroups = ext4_get_groups_count(sb); + struct ext4_group_info *grp; +#ifdef EXT4FS_DEBUG + struct ext4_super_block *es; + ext4_fsblk_t bitmap_count; + unsigned int x; + struct buffer_head *bitmap_bh = NULL; + + es = EXT4_SB(sb)->s_es; + desc_count = 0; + bitmap_count = 0; + gdp = NULL; + + for (i = 0; i < ngroups; i++) { + gdp = ext4_get_group_desc(sb, i, NULL); + if (!gdp) + continue; + grp = NULL; + if (EXT4_SB(sb)->s_group_info) + grp = ext4_get_group_info(sb, i); + if (!grp || !EXT4_MB_GRP_BBITMAP_CORRUPT(grp)) + desc_count += ext4_free_group_clusters(sb, gdp); + brelse(bitmap_bh); + bitmap_bh = ext4_read_block_bitmap(sb, i); + if (IS_ERR(bitmap_bh)) { + bitmap_bh = NULL; + continue; + } + + x = ext4_count_free(bitmap_bh->b_data, + EXT4_CLUSTERS_PER_GROUP(sb) / 8); + printk(KERN_DEBUG "group %u: stored = %d, counted = %u\n", + i, ext4_free_group_clusters(sb, gdp), x); + bitmap_count += x; + } + brelse(bitmap_bh); + printk(KERN_DEBUG "ext4_count_free_clusters: stored = %llu" + ", computed = %llu, %llu\n", + EXT4_NUM_B2C(EXT4_SB(sb), ext4_free_blocks_count(es)), + desc_count, bitmap_count); + return bitmap_count; +#else + desc_count = 0; + for (i = 0; i < ngroups; i++) { + gdp = ext4_get_group_desc(sb, i, NULL); + if (!gdp) + continue; + grp = NULL; + if (EXT4_SB(sb)->s_group_info) + grp = ext4_get_group_info(sb, i); + if (!grp || !EXT4_MB_GRP_BBITMAP_CORRUPT(grp)) + desc_count += ext4_free_group_clusters(sb, gdp); + } + + return desc_count; +#endif +} + +static inline int test_root(ext4_group_t a, int b) +{ + while (1) { + if (a < b) + return 0; + if (a == b) + return 1; + if ((a % b) != 0) + return 0; + a = a / b; + } +} + +/** + * ext4_bg_has_super - number of blocks used by the superblock in group + * @sb: superblock for filesystem + * @group: group number to check + * + * Return the number of blocks used by the superblock (primary or backup) + * in this group. Currently this will be only 0 or 1. + */ +int ext4_bg_has_super(struct super_block *sb, ext4_group_t group) +{ + struct ext4_super_block *es = EXT4_SB(sb)->s_es; + + if (group == 0) + return 1; + if (ext4_has_feature_sparse_super2(sb)) { + if (group == le32_to_cpu(es->s_backup_bgs[0]) || + group == le32_to_cpu(es->s_backup_bgs[1])) + return 1; + return 0; + } + if ((group <= 1) || !ext4_has_feature_sparse_super(sb)) + return 1; + if (!(group & 1)) + return 0; + if (test_root(group, 3) || (test_root(group, 5)) || + test_root(group, 7)) + return 1; + + return 0; +} + +static unsigned long ext4_bg_num_gdb_meta(struct super_block *sb, + ext4_group_t group) +{ + unsigned long metagroup = group / EXT4_DESC_PER_BLOCK(sb); + ext4_group_t first = metagroup * EXT4_DESC_PER_BLOCK(sb); + ext4_group_t last = first + EXT4_DESC_PER_BLOCK(sb) - 1; + + if (group == first || group == first + 1 || group == last) + return 1; + return 0; +} + +static unsigned long ext4_bg_num_gdb_nometa(struct super_block *sb, + ext4_group_t group) +{ + if (!ext4_bg_has_super(sb, group)) + return 0; + + if (ext4_has_feature_meta_bg(sb)) + return le32_to_cpu(EXT4_SB(sb)->s_es->s_first_meta_bg); + else + return EXT4_SB(sb)->s_gdb_count; +} + +/** + * ext4_bg_num_gdb - number of blocks used by the group table in group + * @sb: superblock for filesystem + * @group: group number to check + * + * Return the number of blocks used by the group descriptor table + * (primary or backup) in this group. In the future there may be a + * different number of descriptor blocks in each group. + */ +unsigned long ext4_bg_num_gdb(struct super_block *sb, ext4_group_t group) +{ + unsigned long first_meta_bg = + le32_to_cpu(EXT4_SB(sb)->s_es->s_first_meta_bg); + unsigned long metagroup = group / EXT4_DESC_PER_BLOCK(sb); + + if (!ext4_has_feature_meta_bg(sb) || metagroup < first_meta_bg) + return ext4_bg_num_gdb_nometa(sb, group); + + return ext4_bg_num_gdb_meta(sb,group); + +} + +/* + * This function returns the number of file system metadata blocks at + * the beginning of a block group, including the reserved gdt blocks. + */ +unsigned int ext4_num_base_meta_blocks(struct super_block *sb, + ext4_group_t block_group) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + unsigned num; + + /* Check for superblock and gdt backups in this group */ + num = ext4_bg_has_super(sb, block_group); + + if (!ext4_has_feature_meta_bg(sb) || + block_group < le32_to_cpu(sbi->s_es->s_first_meta_bg) * + sbi->s_desc_per_block) { + if (num) { + num += ext4_bg_num_gdb_nometa(sb, block_group); + num += le16_to_cpu(sbi->s_es->s_reserved_gdt_blocks); + } + } else { /* For META_BG_BLOCK_GROUPS */ + num += ext4_bg_num_gdb_meta(sb, block_group); + } + return num; +} + +static unsigned int ext4_num_base_meta_clusters(struct super_block *sb, + ext4_group_t block_group) +{ + return EXT4_NUM_B2C(EXT4_SB(sb), ext4_num_base_meta_blocks(sb, block_group)); +} + +/** + * ext4_inode_to_goal_block - return a hint for block allocation + * @inode: inode for block allocation + * + * Return the ideal location to start allocating blocks for a + * newly created inode. + */ +ext4_fsblk_t ext4_inode_to_goal_block(struct inode *inode) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + ext4_group_t block_group; + ext4_grpblk_t colour; + int flex_size = ext4_flex_bg_size(EXT4_SB(inode->i_sb)); + ext4_fsblk_t bg_start; + ext4_fsblk_t last_block; + + block_group = ei->i_block_group; + if (flex_size >= EXT4_FLEX_SIZE_DIR_ALLOC_SCHEME) { + /* + * If there are at least EXT4_FLEX_SIZE_DIR_ALLOC_SCHEME + * block groups per flexgroup, reserve the first block + * group for directories and special files. Regular + * files will start at the second block group. This + * tends to speed up directory access and improves + * fsck times. + */ + block_group &= ~(flex_size-1); + if (S_ISREG(inode->i_mode)) + block_group++; + } + bg_start = ext4_group_first_block_no(inode->i_sb, block_group); + last_block = ext4_blocks_count(EXT4_SB(inode->i_sb)->s_es) - 1; + + /* + * If we are doing delayed allocation, we don't need take + * colour into account. + */ + if (test_opt(inode->i_sb, DELALLOC)) + return bg_start; + + if (bg_start + EXT4_BLOCKS_PER_GROUP(inode->i_sb) <= last_block) + colour = (task_pid_nr(current) % 16) * + (EXT4_BLOCKS_PER_GROUP(inode->i_sb) / 16); + else + colour = (task_pid_nr(current) % 16) * + ((last_block - bg_start) / 16); + return bg_start + colour; +} + diff --git a/fs/ext4/bitmap.c b/fs/ext4/bitmap.c new file mode 100644 index 000000000..87760fabd --- /dev/null +++ b/fs/ext4/bitmap.c @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext4/bitmap.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + */ + +#include +#include "ext4.h" + +unsigned int ext4_count_free(char *bitmap, unsigned int numchars) +{ + return numchars * BITS_PER_BYTE - memweight(bitmap, numchars); +} + +int ext4_inode_bitmap_csum_verify(struct super_block *sb, + struct ext4_group_desc *gdp, + struct buffer_head *bh) +{ + __u32 hi; + __u32 provided, calculated; + struct ext4_sb_info *sbi = EXT4_SB(sb); + int sz; + + if (!ext4_has_feature_metadata_csum(sb)) + return 1; + + sz = EXT4_INODES_PER_GROUP(sb) >> 3; + provided = le16_to_cpu(gdp->bg_inode_bitmap_csum_lo); + calculated = ext4_chksum(sbi->s_csum_seed, (__u8 *)bh->b_data, sz); + if (sbi->s_desc_size >= EXT4_BG_INODE_BITMAP_CSUM_HI_END) { + hi = le16_to_cpu(gdp->bg_inode_bitmap_csum_hi); + provided |= (hi << 16); + } else + calculated &= 0xFFFF; + + return provided == calculated; +} + +void ext4_inode_bitmap_csum_set(struct super_block *sb, + struct ext4_group_desc *gdp, + struct buffer_head *bh) +{ + __u32 csum; + struct ext4_sb_info *sbi = EXT4_SB(sb); + int sz; + + if (!ext4_has_feature_metadata_csum(sb)) + return; + + sz = EXT4_INODES_PER_GROUP(sb) >> 3; + csum = ext4_chksum(sbi->s_csum_seed, (__u8 *)bh->b_data, sz); + gdp->bg_inode_bitmap_csum_lo = cpu_to_le16(csum & 0xFFFF); + if (sbi->s_desc_size >= EXT4_BG_INODE_BITMAP_CSUM_HI_END) + gdp->bg_inode_bitmap_csum_hi = cpu_to_le16(csum >> 16); +} + +int ext4_block_bitmap_csum_verify(struct super_block *sb, + struct ext4_group_desc *gdp, + struct buffer_head *bh) +{ + __u32 hi; + __u32 provided, calculated; + struct ext4_sb_info *sbi = EXT4_SB(sb); + int sz = EXT4_CLUSTERS_PER_GROUP(sb) / 8; + + if (!ext4_has_feature_metadata_csum(sb)) + return 1; + + provided = le16_to_cpu(gdp->bg_block_bitmap_csum_lo); + calculated = ext4_chksum(sbi->s_csum_seed, (__u8 *)bh->b_data, sz); + if (sbi->s_desc_size >= EXT4_BG_BLOCK_BITMAP_CSUM_HI_END) { + hi = le16_to_cpu(gdp->bg_block_bitmap_csum_hi); + provided |= (hi << 16); + } else + calculated &= 0xFFFF; + + return provided == calculated; +} + +void ext4_block_bitmap_csum_set(struct super_block *sb, + struct ext4_group_desc *gdp, + struct buffer_head *bh) +{ + int sz = EXT4_CLUSTERS_PER_GROUP(sb) / 8; + __u32 csum; + struct ext4_sb_info *sbi = EXT4_SB(sb); + + if (!ext4_has_feature_metadata_csum(sb)) + return; + + csum = ext4_chksum(sbi->s_csum_seed, (__u8 *)bh->b_data, sz); + gdp->bg_block_bitmap_csum_lo = cpu_to_le16(csum & 0xFFFF); + if (sbi->s_desc_size >= EXT4_BG_BLOCK_BITMAP_CSUM_HI_END) + gdp->bg_block_bitmap_csum_hi = cpu_to_le16(csum >> 16); +} diff --git a/fs/ext4/block_validity.c b/fs/ext4/block_validity.c new file mode 100644 index 000000000..3a31e3d71 --- /dev/null +++ b/fs/ext4/block_validity.c @@ -0,0 +1,370 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext4/block_validity.c + * + * Copyright (C) 2009 + * Theodore Ts'o (tytso@mit.edu) + * + * Track which blocks in the filesystem are metadata blocks that + * should never be used as data blocks by files or directories. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ext4.h" + +struct ext4_system_zone { + struct rb_node node; + ext4_fsblk_t start_blk; + unsigned int count; + u32 ino; +}; + +static struct kmem_cache *ext4_system_zone_cachep; + +int __init ext4_init_system_zone(void) +{ + ext4_system_zone_cachep = KMEM_CACHE(ext4_system_zone, 0); + if (ext4_system_zone_cachep == NULL) + return -ENOMEM; + return 0; +} + +void ext4_exit_system_zone(void) +{ + rcu_barrier(); + kmem_cache_destroy(ext4_system_zone_cachep); +} + +static inline int can_merge(struct ext4_system_zone *entry1, + struct ext4_system_zone *entry2) +{ + if ((entry1->start_blk + entry1->count) == entry2->start_blk && + entry1->ino == entry2->ino) + return 1; + return 0; +} + +static void release_system_zone(struct ext4_system_blocks *system_blks) +{ + struct ext4_system_zone *entry, *n; + + rbtree_postorder_for_each_entry_safe(entry, n, + &system_blks->root, node) + kmem_cache_free(ext4_system_zone_cachep, entry); +} + +/* + * Mark a range of blocks as belonging to the "system zone" --- that + * is, filesystem metadata blocks which should never be used by + * inodes. + */ +static int add_system_zone(struct ext4_system_blocks *system_blks, + ext4_fsblk_t start_blk, + unsigned int count, u32 ino) +{ + struct ext4_system_zone *new_entry, *entry; + struct rb_node **n = &system_blks->root.rb_node, *node; + struct rb_node *parent = NULL, *new_node; + + while (*n) { + parent = *n; + entry = rb_entry(parent, struct ext4_system_zone, node); + if (start_blk < entry->start_blk) + n = &(*n)->rb_left; + else if (start_blk >= (entry->start_blk + entry->count)) + n = &(*n)->rb_right; + else /* Unexpected overlap of system zones. */ + return -EFSCORRUPTED; + } + + new_entry = kmem_cache_alloc(ext4_system_zone_cachep, + GFP_KERNEL); + if (!new_entry) + return -ENOMEM; + new_entry->start_blk = start_blk; + new_entry->count = count; + new_entry->ino = ino; + new_node = &new_entry->node; + + rb_link_node(new_node, parent, n); + rb_insert_color(new_node, &system_blks->root); + + /* Can we merge to the left? */ + node = rb_prev(new_node); + if (node) { + entry = rb_entry(node, struct ext4_system_zone, node); + if (can_merge(entry, new_entry)) { + new_entry->start_blk = entry->start_blk; + new_entry->count += entry->count; + rb_erase(node, &system_blks->root); + kmem_cache_free(ext4_system_zone_cachep, entry); + } + } + + /* Can we merge to the right? */ + node = rb_next(new_node); + if (node) { + entry = rb_entry(node, struct ext4_system_zone, node); + if (can_merge(new_entry, entry)) { + new_entry->count += entry->count; + rb_erase(node, &system_blks->root); + kmem_cache_free(ext4_system_zone_cachep, entry); + } + } + return 0; +} + +static void debug_print_tree(struct ext4_sb_info *sbi) +{ + struct rb_node *node; + struct ext4_system_zone *entry; + struct ext4_system_blocks *system_blks; + int first = 1; + + printk(KERN_INFO "System zones: "); + rcu_read_lock(); + system_blks = rcu_dereference(sbi->s_system_blks); + node = rb_first(&system_blks->root); + while (node) { + entry = rb_entry(node, struct ext4_system_zone, node); + printk(KERN_CONT "%s%llu-%llu", first ? "" : ", ", + entry->start_blk, entry->start_blk + entry->count - 1); + first = 0; + node = rb_next(node); + } + rcu_read_unlock(); + printk(KERN_CONT "\n"); +} + +static int ext4_protect_reserved_inode(struct super_block *sb, + struct ext4_system_blocks *system_blks, + u32 ino) +{ + struct inode *inode; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_map_blocks map; + u32 i = 0, num; + int err = 0, n; + + if ((ino < EXT4_ROOT_INO) || + (ino > le32_to_cpu(sbi->s_es->s_inodes_count))) + return -EINVAL; + inode = ext4_iget(sb, ino, EXT4_IGET_SPECIAL); + if (IS_ERR(inode)) + return PTR_ERR(inode); + num = (inode->i_size + sb->s_blocksize - 1) >> sb->s_blocksize_bits; + while (i < num) { + cond_resched(); + map.m_lblk = i; + map.m_len = num - i; + n = ext4_map_blocks(NULL, inode, &map, 0); + if (n < 0) { + err = n; + break; + } + if (n == 0) { + i++; + } else { + err = add_system_zone(system_blks, map.m_pblk, n, ino); + if (err < 0) { + if (err == -EFSCORRUPTED) { + EXT4_ERROR_INODE_ERR(inode, -err, + "blocks %llu-%llu from inode overlap system zone", + map.m_pblk, + map.m_pblk + map.m_len - 1); + } + break; + } + i += n; + } + } + iput(inode); + return err; +} + +static void ext4_destroy_system_zone(struct rcu_head *rcu) +{ + struct ext4_system_blocks *system_blks; + + system_blks = container_of(rcu, struct ext4_system_blocks, rcu); + release_system_zone(system_blks); + kfree(system_blks); +} + +/* + * Build system zone rbtree which is used for block validity checking. + * + * The update of system_blks pointer in this function is protected by + * sb->s_umount semaphore. However we have to be careful as we can be + * racing with ext4_inode_block_valid() calls reading system_blks rbtree + * protected only by RCU. That's why we first build the rbtree and then + * swap it in place. + */ +int ext4_setup_system_zone(struct super_block *sb) +{ + ext4_group_t ngroups = ext4_get_groups_count(sb); + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_system_blocks *system_blks; + struct ext4_group_desc *gdp; + ext4_group_t i; + int ret; + + system_blks = kzalloc_obj(*system_blks); + if (!system_blks) + return -ENOMEM; + + for (i=0; i < ngroups; i++) { + unsigned int meta_blks = ext4_num_base_meta_blocks(sb, i); + + cond_resched(); + if (meta_blks != 0) { + ret = add_system_zone(system_blks, + ext4_group_first_block_no(sb, i), + meta_blks, 0); + if (ret) + goto err; + } + gdp = ext4_get_group_desc(sb, i, NULL); + ret = add_system_zone(system_blks, + ext4_block_bitmap(sb, gdp), 1, 0); + if (ret) + goto err; + ret = add_system_zone(system_blks, + ext4_inode_bitmap(sb, gdp), 1, 0); + if (ret) + goto err; + ret = add_system_zone(system_blks, + ext4_inode_table(sb, gdp), + sbi->s_itb_per_group, 0); + if (ret) + goto err; + } + if (ext4_has_feature_journal(sb) && sbi->s_es->s_journal_inum) { + ret = ext4_protect_reserved_inode(sb, system_blks, + le32_to_cpu(sbi->s_es->s_journal_inum)); + if (ret) + goto err; + } + + /* + * System blks rbtree complete, announce it once to prevent racing + * with ext4_inode_block_valid() accessing the rbtree at the same + * time. + */ + rcu_assign_pointer(sbi->s_system_blks, system_blks); + + if (test_opt(sb, DEBUG)) + debug_print_tree(sbi); + return 0; +err: + release_system_zone(system_blks); + kfree(system_blks); + return ret; +} + +/* + * Called when the filesystem is unmounted or when remounting it with + * noblock_validity specified. + * + * The update of system_blks pointer in this function is protected by + * sb->s_umount semaphore. However we have to be careful as we can be + * racing with ext4_inode_block_valid() calls reading system_blks rbtree + * protected only by RCU. So we first clear the system_blks pointer and + * then free the rbtree only after RCU grace period expires. + */ +void ext4_release_system_zone(struct super_block *sb) +{ + struct ext4_system_blocks *system_blks; + + system_blks = rcu_dereference_protected(EXT4_SB(sb)->s_system_blks, + lockdep_is_held(&sb->s_umount)); + rcu_assign_pointer(EXT4_SB(sb)->s_system_blks, NULL); + + if (system_blks) + call_rcu(&system_blks->rcu, ext4_destroy_system_zone); +} + +int ext4_sb_block_valid(struct super_block *sb, struct inode *inode, + ext4_fsblk_t start_blk, unsigned int count) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_system_blocks *system_blks; + struct ext4_system_zone *entry; + struct rb_node *n; + int ret = 1; + + if ((start_blk <= le32_to_cpu(sbi->s_es->s_first_data_block)) || + (start_blk + count < start_blk) || + (start_blk + count > ext4_blocks_count(sbi->s_es))) + return 0; + + /* + * Lock the system zone to prevent it being released concurrently + * when doing a remount which inverse current "[no]block_validity" + * mount option. + */ + rcu_read_lock(); + system_blks = rcu_dereference(sbi->s_system_blks); + if (system_blks == NULL) + goto out_rcu; + + n = system_blks->root.rb_node; + while (n) { + entry = rb_entry(n, struct ext4_system_zone, node); + if (start_blk + count - 1 < entry->start_blk) + n = n->rb_left; + else if (start_blk >= (entry->start_blk + entry->count)) + n = n->rb_right; + else { + ret = 0; + if (inode) + ret = (entry->ino == inode->i_ino); + break; + } + } +out_rcu: + rcu_read_unlock(); + return ret; +} + +/* + * Returns 1 if the passed-in block region (start_blk, + * start_blk+count) is valid; 0 if some part of the block region + * overlaps with some other filesystem metadata blocks. + */ +int ext4_inode_block_valid(struct inode *inode, ext4_fsblk_t start_blk, + unsigned int count) +{ + return ext4_sb_block_valid(inode->i_sb, inode, start_blk, count); +} + +int ext4_check_blockref(const char *function, unsigned int line, + struct inode *inode, __le32 *p, unsigned int max) +{ + __le32 *bref = p; + unsigned int blk; + journal_t *journal = EXT4_SB(inode->i_sb)->s_journal; + + if (journal && inode == journal->j_inode) + return 0; + + while (bref < p+max) { + blk = le32_to_cpu(*bref++); + if (blk && + unlikely(!ext4_inode_block_valid(inode, blk, 1))) { + ext4_error_inode(inode, function, line, blk, + "invalid block"); + return -EFSCORRUPTED; + } + } + return 0; +} + diff --git a/fs/ext4/crypto.c b/fs/ext4/crypto.c new file mode 100644 index 000000000..1a0fccb08 --- /dev/null +++ b/fs/ext4/crypto.c @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +#include "ext4.h" +#include "xattr.h" +#include "ext4_jbd2.h" + +static void ext4_fname_from_fscrypt_name(struct ext4_filename *dst, + const struct fscrypt_name *src) +{ + memset(dst, 0, sizeof(*dst)); + + dst->usr_fname = src->usr_fname; + dst->disk_name = src->disk_name; + dst->hinfo.hash = src->hash; + dst->hinfo.minor_hash = src->minor_hash; + dst->crypto_buf = src->crypto_buf; +} + +int ext4_fname_setup_filename(struct inode *dir, const struct qstr *iname, + int lookup, struct ext4_filename *fname) +{ + struct fscrypt_name name; + int err; + + err = fscrypt_setup_filename(dir, iname, lookup, &name); + if (err) + return err; + + ext4_fname_from_fscrypt_name(fname, &name); + + err = ext4_fname_setup_ci_filename(dir, iname, fname); + if (err) + ext4_fname_free_filename(fname); + + return err; +} + +int ext4_fname_prepare_lookup(struct inode *dir, struct dentry *dentry, + struct ext4_filename *fname) +{ + struct fscrypt_name name; + int err; + + err = fscrypt_prepare_lookup(dir, dentry, &name); + if (err) + return err; + + ext4_fname_from_fscrypt_name(fname, &name); + + err = ext4_fname_setup_ci_filename(dir, &dentry->d_name, fname); + if (err) + ext4_fname_free_filename(fname); + return err; +} + +void ext4_fname_free_filename(struct ext4_filename *fname) +{ + struct fscrypt_name name; + + name.crypto_buf = fname->crypto_buf; + fscrypt_free_filename(&name); + + fname->crypto_buf.name = NULL; + fname->usr_fname = NULL; + fname->disk_name.name = NULL; + + ext4_fname_free_ci_filename(fname); +} + +static bool uuid_is_zero(__u8 u[16]) +{ + int i; + + for (i = 0; i < 16; i++) + if (u[i]) + return false; + return true; +} + +int ext4_ioctl_get_encryption_pwsalt(struct file *filp, void __user *arg) +{ + struct super_block *sb = file_inode(filp)->i_sb; + struct ext4_sb_info *sbi = EXT4_SB(sb); + int err, err2; + handle_t *handle; + + if (!ext4_has_feature_encrypt(sb)) + return -EOPNOTSUPP; + + if (uuid_is_zero(sbi->s_es->s_encrypt_pw_salt)) { + err = mnt_want_write_file(filp); + if (err) + return err; + handle = ext4_journal_start_sb(sb, EXT4_HT_MISC, 1); + if (IS_ERR(handle)) { + err = PTR_ERR(handle); + goto pwsalt_err_exit; + } + err = ext4_journal_get_write_access(handle, sb, sbi->s_sbh, + EXT4_JTR_NONE); + if (err) + goto pwsalt_err_journal; + lock_buffer(sbi->s_sbh); + generate_random_uuid(sbi->s_es->s_encrypt_pw_salt); + ext4_superblock_csum_set(sb); + unlock_buffer(sbi->s_sbh); + err = ext4_handle_dirty_metadata(handle, NULL, sbi->s_sbh); +pwsalt_err_journal: + err2 = ext4_journal_stop(handle); + if (err2 && !err) + err = err2; +pwsalt_err_exit: + mnt_drop_write_file(filp); + if (err) + return err; + } + + if (copy_to_user(arg, sbi->s_es->s_encrypt_pw_salt, 16)) + return -EFAULT; + return 0; +} + +static int ext4_get_context(struct inode *inode, void *ctx, size_t len) +{ + return ext4_xattr_get(inode, EXT4_XATTR_INDEX_ENCRYPTION, + EXT4_XATTR_NAME_ENCRYPTION_CONTEXT, ctx, len); +} + +static int ext4_set_context(struct inode *inode, const void *ctx, size_t len, + void *fs_data) +{ + handle_t *handle = fs_data; + int res, res2, credits, retries = 0; + + /* + * Encrypting the root directory is not allowed because e2fsck expects + * lost+found to exist and be unencrypted, and encrypting the root + * directory would imply encrypting the lost+found directory as well as + * the filename "lost+found" itself. + */ + if (inode->i_ino == EXT4_ROOT_INO) + return -EPERM; + + /* + * For new encrypted inodes, S_DAX is never set in the first place. + * + * For existing inodes, this is called only on empty directories. ext4 + * never sets S_DAX on directories. + */ + if (WARN_ON_ONCE(IS_DAX(inode))) + return -EINVAL; + + if (ext4_test_inode_flag(inode, EXT4_INODE_DAX)) + return -EOPNOTSUPP; + + res = ext4_convert_inline_data(inode); + if (res) + return res; + + /* + * If a journal handle was specified, then the encryption context is + * being set on a new inode via inheritance and is part of a larger + * transaction to create the inode. Otherwise the encryption context is + * being set on an existing inode in its own transaction. Only in the + * latter case should the "retry on ENOSPC" logic be used. + */ + + if (handle) { + /* + * __ext4_new_inode() should have already set the encrypt flag + * on the inode and avoided enabling inline data. + */ + if (WARN_ON_ONCE(!IS_ENCRYPTED(inode))) + return -EINVAL; + if (WARN_ON_ONCE(ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA))) + return -EINVAL; + /* + * Since the inode is new it is ok to pass the + * XATTR_CREATE flag. This is necessary to match the + * remaining journal credits check in the set_handle + * function with the credits allocated for the new + * inode. + */ + return ext4_xattr_set_handle(handle, inode, + EXT4_XATTR_INDEX_ENCRYPTION, + EXT4_XATTR_NAME_ENCRYPTION_CONTEXT, + ctx, len, XATTR_CREATE); + } + + res = dquot_initialize(inode); + if (res) + return res; +retry: + res = ext4_xattr_set_credits(inode, len, false /* is_create */, + &credits); + if (res) + return res; + + handle = ext4_journal_start(inode, EXT4_HT_MISC, credits); + if (IS_ERR(handle)) + return PTR_ERR(handle); + + res = ext4_xattr_set_handle(handle, inode, EXT4_XATTR_INDEX_ENCRYPTION, + EXT4_XATTR_NAME_ENCRYPTION_CONTEXT, + ctx, len, 0); + if (!res) { + ext4_set_inode_flag(inode, EXT4_INODE_ENCRYPT); + /* Update inode->i_flags to set S_ENCRYPTED. */ + ext4_set_inode_flags(inode, false); + res = ext4_mark_inode_dirty(handle, inode); + if (res) + EXT4_ERROR_INODE(inode, "Failed to mark inode dirty"); + } + res2 = ext4_journal_stop(handle); + + if (res == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)) + goto retry; + if (!res) + res = res2; + return res; +} + +static const union fscrypt_policy *ext4_get_dummy_policy(struct super_block *sb) +{ + return EXT4_SB(sb)->s_dummy_enc_policy.policy; +} + +static bool ext4_has_stable_inodes(struct super_block *sb) +{ + return ext4_has_feature_stable_inodes(sb); +} + +const struct fscrypt_operations ext4_cryptops = { + .inode_info_offs = (int)offsetof(struct ext4_inode_info, i_crypt_info) - + (int)offsetof(struct ext4_inode_info, vfs_inode), + .is_block_based = 1, + .has_32bit_inodes = 1, + .supports_subblock_data_units = 1, + .legacy_key_prefix = "ext4:", + .get_context = ext4_get_context, + .set_context = ext4_set_context, + .get_dummy_policy = ext4_get_dummy_policy, + .empty_dir = ext4_empty_dir, + .has_stable_inodes = ext4_has_stable_inodes, +}; diff --git a/fs/ext4/dir.c b/fs/ext4/dir.c new file mode 100644 index 000000000..8d7b81e69 --- /dev/null +++ b/fs/ext4/dir.c @@ -0,0 +1,710 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext4/dir.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * from + * + * linux/fs/minix/dir.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * ext4 directory handling functions + * + * Big-endian to little-endian byte-swapping/bitmaps by + * David S. Miller (davem@caip.rutgers.edu), 1995 + * + * Hash Tree Directory indexing (c) 2001 Daniel Phillips + * + */ + +#include +#include +#include +#include +#include +#include +#include "ext4.h" +#include "xattr.h" + +static int ext4_dx_readdir(struct file *, struct dir_context *); + +/** + * is_dx_dir() - check if a directory is using htree indexing + * @inode: directory inode + * + * Check if the given dir-inode refers to an htree-indexed directory + * (or a directory which could potentially get converted to use htree + * indexing). + * + * Return 1 if it is a dx dir, 0 if not + */ +static int is_dx_dir(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + + if (ext4_has_feature_dir_index(inode->i_sb) && + ((ext4_test_inode_flag(inode, EXT4_INODE_INDEX)) || + ((inode->i_size >> sb->s_blocksize_bits) == 1) || + ext4_has_inline_data(inode))) + return 1; + + return 0; +} + +static bool is_fake_dir_entry(struct ext4_dir_entry_2 *de) +{ + /* Check if . or .. , or skip if namelen is 0 */ + if ((de->name_len > 0) && (de->name_len <= 2) && (de->name[0] == '.') && + (de->name[1] == '.' || de->name[1] == '\0')) + return true; + /* Check if this is a csum entry */ + if (de->file_type == EXT4_FT_DIR_CSUM) + return true; + return false; +} + +/* + * Return 0 if the directory entry is OK, and 1 if there is a problem + * + * Note: this is the opposite of what ext2 and ext3 historically returned... + * + * bh passed here can be an inode block or a dir data block, depending + * on the inode inline data flag. + */ +int __ext4_check_dir_entry(const char *function, unsigned int line, + struct inode *dir, struct file *filp, + struct ext4_dir_entry_2 *de, + struct buffer_head *bh, char *buf, int size, + unsigned int offset) +{ + const char *error_msg = NULL; + const int rlen = ext4_rec_len_from_disk(de->rec_len, + dir->i_sb->s_blocksize); + const int next_offset = ((char *) de - buf) + rlen; + bool fake = is_fake_dir_entry(de); + bool has_csum = ext4_has_feature_metadata_csum(dir->i_sb); + + if (unlikely(rlen < ext4_dir_rec_len(1, fake ? NULL : dir))) + error_msg = "rec_len is smaller than minimal"; + else if (unlikely(rlen % 4 != 0)) + error_msg = "rec_len % 4 != 0"; + else if (unlikely(rlen < ext4_dir_rec_len(de->name_len, + fake ? NULL : dir))) + error_msg = "rec_len is too small for name_len"; + else if (unlikely(next_offset > size)) + error_msg = "directory entry overrun"; + else if (unlikely(next_offset > size - ext4_dir_rec_len(1, + has_csum ? NULL : dir) && + next_offset != size)) + error_msg = "directory entry too close to block end"; + else if (unlikely(le32_to_cpu(de->inode) > + le32_to_cpu(EXT4_SB(dir->i_sb)->s_es->s_inodes_count))) + error_msg = "inode out of bounds"; + else if (unlikely(next_offset == size && de->name_len == 1 && + de->name[0] == '.')) + error_msg = "'.' directory cannot be the last in data block"; + else + return 0; + + if (filp) + ext4_error_file(filp, function, line, bh->b_blocknr, + "bad entry in directory: %s - offset=%u, " + "inode=%u, rec_len=%d, size=%d fake=%d", + error_msg, offset, le32_to_cpu(de->inode), + rlen, size, fake); + else + ext4_error_inode(dir, function, line, bh->b_blocknr, + "bad entry in directory: %s - offset=%u, " + "inode=%u, rec_len=%d, size=%d fake=%d", + error_msg, offset, le32_to_cpu(de->inode), + rlen, size, fake); + + return 1; +} + +static int ext4_readdir(struct file *file, struct dir_context *ctx) +{ + unsigned int offset; + int i; + struct ext4_dir_entry_2 *de; + int err; + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + struct buffer_head *bh = NULL; + struct fscrypt_str fstr = FSTR_INIT(NULL, 0); + struct dir_private_info *info = file->private_data; + bool has_csum = ext4_has_feature_metadata_csum(sb); + + err = fscrypt_prepare_readdir(inode); + if (err) + return err; + + if (is_dx_dir(inode)) { + err = ext4_dx_readdir(file, ctx); + if (err != ERR_BAD_DX_DIR) + return err; + + /* Can we just clear INDEX flag to ignore htree information? */ + if (!has_csum) { + /* + * We don't set the inode dirty flag since it's not + * critical that it gets flushed back to the disk. + */ + ext4_clear_inode_flag(inode, EXT4_INODE_INDEX); + } + } + + if (ext4_has_inline_data(inode)) { + int has_inline_data = 1; + err = ext4_read_inline_dir(file, ctx, + &has_inline_data); + if (has_inline_data) + return err; + } + + if (IS_ENCRYPTED(inode)) { + err = fscrypt_fname_alloc_buffer(EXT4_NAME_LEN, &fstr); + if (err < 0) + return err; + } + + while (ctx->pos < inode->i_size) { + struct ext4_map_blocks map; + + if (fatal_signal_pending(current)) { + err = -ERESTARTSYS; + goto errout; + } + cond_resched(); + offset = ctx->pos & (sb->s_blocksize - 1); + map.m_lblk = ctx->pos >> EXT4_BLOCK_SIZE_BITS(sb); + map.m_len = 1; + err = ext4_map_blocks(NULL, inode, &map, 0); + if (err == 0) { + /* m_len should never be zero but let's avoid + * an infinite loop if it somehow is */ + if (map.m_len == 0) + map.m_len = 1; + ctx->pos += map.m_len * sb->s_blocksize; + continue; + } + if (err > 0) { + pgoff_t index = map.m_pblk << inode->i_blkbits >> + PAGE_SHIFT; + if (!ra_has_index(&file->f_ra, index)) + page_cache_sync_readahead( + sb->s_bdev->bd_mapping, + &file->f_ra, file, index, + 1 << EXT4_SB(sb)->s_min_folio_order); + file->f_ra.prev_pos = (loff_t)index << PAGE_SHIFT; + bh = ext4_bread(NULL, inode, map.m_lblk, 0); + if (IS_ERR(bh)) { + err = PTR_ERR(bh); + bh = NULL; + goto errout; + } + } + + if (!bh) { + /* corrupt size? Maybe no more blocks to read */ + if (ctx->pos > inode->i_blocks << 9) + break; + ctx->pos += sb->s_blocksize - offset; + continue; + } + + /* Check the checksum */ + if (!buffer_verified(bh) && + !ext4_dirblock_csum_verify(inode, bh)) { + EXT4_ERROR_FILE(file, 0, "directory fails checksum " + "at offset %llu", + (unsigned long long)ctx->pos); + ctx->pos += sb->s_blocksize - offset; + brelse(bh); + bh = NULL; + continue; + } + set_buffer_verified(bh); + + /* If the dir block has changed since the last call to + * readdir(2), then we might be pointing to an invalid + * dirent right now. Scan from the start of the block + * to make sure. */ + if (!inode_eq_iversion(inode, info->cookie)) { + for (i = 0; + i <= sb->s_blocksize - + ext4_dir_rec_len(1, has_csum ? NULL : inode) && + i < offset;) { + de = (struct ext4_dir_entry_2 *) + (bh->b_data + i); + /* It's too expensive to do a full + * dirent test each time round this + * loop, but we do have to test at + * least that it is non-zero. A + * failure will be detected in the + * dirent test below. */ + if (ext4_rec_len_from_disk(de->rec_len, + sb->s_blocksize) < ext4_dir_rec_len(1, + inode)) + break; + i += ext4_rec_len_from_disk(de->rec_len, + sb->s_blocksize); + } + offset = i; + ctx->pos = (ctx->pos & ~(sb->s_blocksize - 1)) + | offset; + info->cookie = inode_query_iversion(inode); + } + + if (unlikely(offset < sb->s_blocksize && + offset > sb->s_blocksize - + ext4_dir_rec_len(1, has_csum ? NULL : inode))) { + EXT4_ERROR_FILE(file, bh->b_blocknr, + "bad entry in directory: %s - offset=%u, size=%lu", + "directory entry too close to block end", + offset, sb->s_blocksize); + ctx->pos = round_up(ctx->pos, sb->s_blocksize); + goto next_block; + } + + while (ctx->pos < inode->i_size + && offset < sb->s_blocksize) { + de = (struct ext4_dir_entry_2 *) (bh->b_data + offset); + if (ext4_check_dir_entry(inode, file, de, bh, + bh->b_data, bh->b_size, + offset)) { + /* + * On error, skip to the next block + */ + ctx->pos = (ctx->pos | + (sb->s_blocksize - 1)) + 1; + break; + } + offset += ext4_rec_len_from_disk(de->rec_len, + sb->s_blocksize); + if (le32_to_cpu(de->inode)) { + if (!IS_ENCRYPTED(inode)) { + if (!dir_emit(ctx, de->name, + de->name_len, + le32_to_cpu(de->inode), + get_dtype(sb, de->file_type))) + goto done; + } else { + int save_len = fstr.len; + struct fscrypt_str de_name = + FSTR_INIT(de->name, + de->name_len); + u32 hash; + u32 minor_hash; + + if (IS_CASEFOLDED(inode)) { + hash = EXT4_DIRENT_HASH(de); + minor_hash = EXT4_DIRENT_MINOR_HASH(de); + } else { + hash = 0; + minor_hash = 0; + } + + /* Directory is encrypted */ + err = fscrypt_fname_disk_to_usr(inode, + hash, minor_hash, &de_name, &fstr); + de_name = fstr; + fstr.len = save_len; + if (err) + goto errout; + if (!dir_emit(ctx, + de_name.name, de_name.len, + le32_to_cpu(de->inode), + get_dtype(sb, de->file_type))) + goto done; + } + } + ctx->pos += ext4_rec_len_from_disk(de->rec_len, + sb->s_blocksize); + } +next_block: + if ((ctx->pos < inode->i_size) && !dir_relax_shared(inode)) + goto done; + brelse(bh); + bh = NULL; + } +done: + err = 0; +errout: + fscrypt_fname_free_buffer(&fstr); + brelse(bh); + return err; +} + +static inline int is_32bit_api(void) +{ +#ifdef CONFIG_COMPAT + return in_compat_syscall(); +#else + return (BITS_PER_LONG == 32); +#endif +} + +/* + * These functions convert from the major/minor hash to an f_pos + * value for dx directories + * + * Upper layer (for example NFS) should specify FMODE_32BITHASH or + * FMODE_64BITHASH explicitly. On the other hand, we allow ext4 to be mounted + * directly on both 32-bit and 64-bit nodes, under such case, neither + * FMODE_32BITHASH nor FMODE_64BITHASH is specified. + */ +static inline loff_t hash2pos(struct file *filp, __u32 major, __u32 minor) +{ + if ((filp->f_mode & FMODE_32BITHASH) || + (!(filp->f_mode & FMODE_64BITHASH) && is_32bit_api())) + return major >> 1; + else + return ((__u64)(major >> 1) << 32) | (__u64)minor; +} + +static inline __u32 pos2maj_hash(struct file *filp, loff_t pos) +{ + if ((filp->f_mode & FMODE_32BITHASH) || + (!(filp->f_mode & FMODE_64BITHASH) && is_32bit_api())) + return (pos << 1) & 0xffffffff; + else + return ((pos >> 32) << 1) & 0xffffffff; +} + +static inline __u32 pos2min_hash(struct file *filp, loff_t pos) +{ + if ((filp->f_mode & FMODE_32BITHASH) || + (!(filp->f_mode & FMODE_64BITHASH) && is_32bit_api())) + return 0; + else + return pos & 0xffffffff; +} + +/* + * Return 32- or 64-bit end-of-file for dx directories + */ +static inline loff_t ext4_get_htree_eof(struct file *filp) +{ + if ((filp->f_mode & FMODE_32BITHASH) || + (!(filp->f_mode & FMODE_64BITHASH) && is_32bit_api())) + return EXT4_HTREE_EOF_32BIT; + else + return EXT4_HTREE_EOF_64BIT; +} + + +/* + * ext4_dir_llseek() calls generic_file_llseek_size to handle htree + * directories, where the "offset" is in terms of the filename hash + * value instead of the byte offset. + * + * Because we may return a 64-bit hash that is well beyond offset limits, + * we need to pass the max hash as the maximum allowable offset in + * the htree directory case. + * + * For non-htree, ext4_llseek already chooses the proper max offset. + */ +static loff_t ext4_dir_llseek(struct file *file, loff_t offset, int whence) +{ + struct inode *inode = file->f_mapping->host; + struct dir_private_info *info = file->private_data; + int dx_dir = is_dx_dir(inode); + loff_t ret, htree_max = ext4_get_htree_eof(file); + + if (likely(dx_dir)) + ret = generic_file_llseek_size(file, offset, whence, + htree_max, htree_max); + else + ret = ext4_llseek(file, offset, whence); + info->cookie = inode_peek_iversion(inode) - 1; + return ret; +} + +/* + * This structure holds the nodes of the red-black tree used to store + * the directory entry in hash order. + */ +struct fname { + __u32 hash; + __u32 minor_hash; + struct rb_node rb_hash; + struct fname *next; + __u32 inode; + __u8 name_len; + __u8 file_type; + char name[] __counted_by(name_len); +}; + +/* + * This function implements a non-recursive way of freeing all of the + * nodes in the red-black tree. + */ +static void free_rb_tree_fname(struct rb_root *root) +{ + struct fname *fname, *next; + + rbtree_postorder_for_each_entry_safe(fname, next, root, rb_hash) + while (fname) { + struct fname *old = fname; + fname = fname->next; + kfree(old); + } + + *root = RB_ROOT; +} + +static void ext4_htree_init_dir_info(struct file *filp, loff_t pos) +{ + struct dir_private_info *p = filp->private_data; + + if (is_dx_dir(file_inode(filp)) && !p->initialized) { + p->curr_hash = pos2maj_hash(filp, pos); + p->curr_minor_hash = pos2min_hash(filp, pos); + p->initialized = true; + } +} + +void ext4_htree_free_dir_info(struct dir_private_info *p) +{ + free_rb_tree_fname(&p->root); + kfree(p); +} + +/* + * Given a directory entry, enter it into the fname rb tree. + * + * When filename encryption is enabled, the dirent will hold the + * encrypted filename, while the htree will hold decrypted filename. + * The decrypted filename is passed in via ent_name. parameter. + */ +int ext4_htree_store_dirent(struct file *dir_file, __u32 hash, + __u32 minor_hash, + struct ext4_dir_entry_2 *dirent, + struct fscrypt_str *ent_name) +{ + struct rb_node **p, *parent = NULL; + struct fname *fname, *new_fn; + struct dir_private_info *info; + + info = dir_file->private_data; + p = &info->root.rb_node; + + /* Create and allocate the fname structure */ + new_fn = kzalloc_flex(*new_fn, name, ent_name->len + 1); + if (!new_fn) + return -ENOMEM; + new_fn->hash = hash; + new_fn->minor_hash = minor_hash; + new_fn->inode = le32_to_cpu(dirent->inode); + new_fn->name_len = ent_name->len; + new_fn->file_type = dirent->file_type; + memcpy(new_fn->name, ent_name->name, ent_name->len); + + while (*p) { + parent = *p; + fname = rb_entry(parent, struct fname, rb_hash); + + /* + * If the hash and minor hash match up, then we put + * them on a linked list. This rarely happens... + */ + if ((new_fn->hash == fname->hash) && + (new_fn->minor_hash == fname->minor_hash)) { + new_fn->next = fname->next; + fname->next = new_fn; + return 0; + } + + if (new_fn->hash < fname->hash) + p = &(*p)->rb_left; + else if (new_fn->hash > fname->hash) + p = &(*p)->rb_right; + else if (new_fn->minor_hash < fname->minor_hash) + p = &(*p)->rb_left; + else /* if (new_fn->minor_hash > fname->minor_hash) */ + p = &(*p)->rb_right; + } + + rb_link_node(&new_fn->rb_hash, parent, p); + rb_insert_color(&new_fn->rb_hash, &info->root); + return 0; +} + + + +/* + * This is a helper function for ext4_dx_readdir. It calls filldir + * for all entries on the fname linked list. (Normally there is only + * one entry on the linked list, unless there are 62 bit hash collisions.) + */ +static int call_filldir(struct file *file, struct dir_context *ctx, + struct fname *fname) +{ + struct dir_private_info *info = file->private_data; + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + + if (!fname) { + ext4_msg(sb, KERN_ERR, "%s:%d: inode #%llu: comm %s: " + "called with null fname?!?", __func__, __LINE__, + inode->i_ino, current->comm); + return 0; + } + ctx->pos = hash2pos(file, fname->hash, fname->minor_hash); + while (fname) { + if (!dir_emit(ctx, fname->name, + fname->name_len, + fname->inode, + get_dtype(sb, fname->file_type))) { + info->extra_fname = fname; + return 1; + } + fname = fname->next; + } + return 0; +} + +static int ext4_dx_readdir(struct file *file, struct dir_context *ctx) +{ + struct dir_private_info *info = file->private_data; + struct inode *inode = file_inode(file); + struct fname *fname; + int ret = 0; + + ext4_htree_init_dir_info(file, ctx->pos); + + if (ctx->pos == ext4_get_htree_eof(file)) + return 0; /* EOF */ + + /* Some one has messed with f_pos; reset the world */ + if (info->last_pos != ctx->pos) { + free_rb_tree_fname(&info->root); + info->curr_node = NULL; + info->extra_fname = NULL; + info->curr_hash = pos2maj_hash(file, ctx->pos); + info->curr_minor_hash = pos2min_hash(file, ctx->pos); + } + + /* + * If there are any leftover names on the hash collision + * chain, return them first. + */ + if (info->extra_fname) { + if (call_filldir(file, ctx, info->extra_fname)) + goto finished; + info->extra_fname = NULL; + goto next_node; + } else if (!info->curr_node) + info->curr_node = rb_first(&info->root); + + while (1) { + /* + * Fill the rbtree if we have no more entries, + * or the inode has changed since we last read in the + * cached entries. + */ + if ((!info->curr_node) || + !inode_eq_iversion(inode, info->cookie)) { + info->curr_node = NULL; + free_rb_tree_fname(&info->root); + info->cookie = inode_query_iversion(inode); + ret = ext4_htree_fill_tree(file, info->curr_hash, + info->curr_minor_hash, + &info->next_hash); + if (ret < 0) + goto finished; + if (ret == 0) { + ctx->pos = ext4_get_htree_eof(file); + break; + } + info->curr_node = rb_first(&info->root); + } + + fname = rb_entry(info->curr_node, struct fname, rb_hash); + info->curr_hash = fname->hash; + info->curr_minor_hash = fname->minor_hash; + if (call_filldir(file, ctx, fname)) + break; + next_node: + info->curr_node = rb_next(info->curr_node); + if (info->curr_node) { + fname = rb_entry(info->curr_node, struct fname, + rb_hash); + info->curr_hash = fname->hash; + info->curr_minor_hash = fname->minor_hash; + } else { + if (info->next_hash == ~0) { + ctx->pos = ext4_get_htree_eof(file); + break; + } + info->curr_hash = info->next_hash; + info->curr_minor_hash = 0; + } + } +finished: + info->last_pos = ctx->pos; + return ret < 0 ? ret : 0; +} + +static int ext4_release_dir(struct inode *inode, struct file *filp) +{ + if (filp->private_data) + ext4_htree_free_dir_info(filp->private_data); + + return 0; +} + +int ext4_check_all_de(struct inode *dir, struct buffer_head *bh, void *buf, + int buf_size) +{ + struct ext4_dir_entry_2 *de; + int rlen; + unsigned int offset = 0; + char *top; + + de = buf; + top = buf + buf_size; + while ((char *) de < top) { + if (ext4_check_dir_entry(dir, NULL, de, bh, + buf, buf_size, offset)) + return -EFSCORRUPTED; + rlen = ext4_rec_len_from_disk(de->rec_len, buf_size); + de = (struct ext4_dir_entry_2 *)((char *)de + rlen); + offset += rlen; + } + if ((char *) de > top) + return -EFSCORRUPTED; + + return 0; +} + +static int ext4_dir_open(struct inode *inode, struct file *file) +{ + struct dir_private_info *info; + + info = kzalloc_obj(*info); + if (!info) + return -ENOMEM; + file->private_data = info; + return 0; +} + +const struct file_operations ext4_dir_operations = { + .open = ext4_dir_open, + .llseek = ext4_dir_llseek, + .read = generic_read_dir, + .iterate_shared = ext4_readdir, + .unlocked_ioctl = ext4_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = ext4_compat_ioctl, +#endif + .fsync = ext4_sync_file, + .release = ext4_release_dir, + .setlease = generic_setlease, +}; diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h new file mode 100644 index 000000000..724a27e8b --- /dev/null +++ b/fs/ext4/ext4.h @@ -0,0 +1,4078 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ext4.h + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * from + * + * linux/include/linux/minix_fs.h + * + * Copyright (C) 1991, 1992 Linus Torvalds + */ + +#ifndef _EXT4_H +#define _EXT4_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef __KERNEL__ +#include +#endif +#include + +#include +#include + +#include + +/* + * The fourth extended filesystem constants/structures + */ + +/* + * with AGGRESSIVE_CHECK allocator runs consistency checks over + * structures. these checks slow things down a lot + */ +#define AGGRESSIVE_CHECK__ + +/* + * with DOUBLE_CHECK defined mballoc creates persistent in-core + * bitmaps, maintains and uses them to check for double allocations + */ +#define DOUBLE_CHECK__ + +/* + * Define EXT4FS_DEBUG to produce debug messages + */ +#undef EXT4FS_DEBUG + +/* + * Debug code + */ +#ifdef EXT4FS_DEBUG +#define ext4_debug(f, a...) \ + do { \ + printk(KERN_DEBUG "EXT4-fs DEBUG (%s, %d): %s:", \ + __FILE__, __LINE__, __func__); \ + printk(KERN_DEBUG f, ## a); \ + } while (0) +#else +#define ext4_debug(fmt, ...) no_printk(fmt, ##__VA_ARGS__) +#endif + + /* + * Turn on EXT_DEBUG to enable ext4_ext_show_path/leaf/move in extents.c + */ +#define EXT_DEBUG__ + +/* + * Dynamic printk for controlled extents debugging. + */ +#ifdef CONFIG_EXT4_DEBUG +#define ext_debug(ino, fmt, ...) \ + pr_debug("[%s/%d] EXT4-fs (%s): ino %llu: (%s, %d): %s:" fmt, \ + current->comm, task_pid_nr(current), \ + ino->i_sb->s_id, ino->i_ino, __FILE__, __LINE__, \ + __func__, ##__VA_ARGS__) +#else +#define ext_debug(ino, fmt, ...) no_printk(fmt, ##__VA_ARGS__) +#endif + +#define ASSERT(assert) \ +do { \ + if (unlikely(!(assert))) { \ + printk(KERN_EMERG \ + "Assertion failure in %s() at %s:%d: '%s'\n", \ + __func__, __FILE__, __LINE__, #assert); \ + BUG(); \ + } \ +} while (0) + +/* data type for block offset of block group */ +typedef int ext4_grpblk_t; + +/* data type for filesystem-wide blocks number */ +typedef unsigned long long ext4_fsblk_t; + +/* data type for file logical block number */ +typedef __u32 ext4_lblk_t; + +/* data type for block group number */ +typedef unsigned int ext4_group_t; + +enum SHIFT_DIRECTION { + SHIFT_LEFT = 0, + SHIFT_RIGHT, +}; + +/* + * For each criteria, mballoc has slightly different way of finding + * the required blocks nad usually, higher the criteria the slower the + * allocation. We start at lower criterias and keep falling back to + * higher ones if we are not able to find any blocks. Lower (earlier) + * criteria are faster. + */ +enum criteria { + /* + * Used when number of blocks needed is a power of 2. This + * doesn't trigger any disk IO except prefetch and is the + * fastest criteria. + */ + CR_POWER2_ALIGNED, + + /* + * Tries to lookup in-memory data structures to find the most + * suitable group that satisfies goal request. No disk IO + * except block prefetch. + */ + CR_GOAL_LEN_FAST, + + /* + * Same as CR_GOAL_LEN_FAST but is allowed to reduce the goal + * length to the best available length for faster allocation. + */ + CR_BEST_AVAIL_LEN, + + /* + * Reads each block group sequentially, performing disk IO if + * necessary, to find suitable block group. Tries to + * allocate goal length but might trim the request if nothing + * is found after enough tries. + */ + CR_GOAL_LEN_SLOW, + + /* + * Finds the first free set of blocks and allocates + * those. This is only used in rare cases when + * CR_GOAL_LEN_SLOW also fails to allocate anything. + */ + CR_ANY_FREE, + + /* + * Number of criterias defined. + */ + EXT4_MB_NUM_CRS +}; + +/* + * Flags used in mballoc's allocation_context flags field. + * + * Also used to show what's going on for debugging purposes when the + * flag field is exported via the traceport interface + */ + +/* prefer goal again. length */ +#define EXT4_MB_HINT_MERGE 0x0001 +/* first blocks in the file */ +#define EXT4_MB_HINT_FIRST 0x0008 +/* data is being allocated */ +#define EXT4_MB_HINT_DATA 0x0020 +/* don't preallocate (for tails) */ +#define EXT4_MB_HINT_NOPREALLOC 0x0040 +/* allocate for locality group */ +#define EXT4_MB_HINT_GROUP_ALLOC 0x0080 +/* allocate goal blocks or none */ +#define EXT4_MB_HINT_GOAL_ONLY 0x0100 +/* goal is meaningful */ +#define EXT4_MB_HINT_TRY_GOAL 0x0200 +/* blocks already pre-reserved by delayed allocation */ +#define EXT4_MB_DELALLOC_RESERVED 0x0400 +/* We are doing stream allocation */ +#define EXT4_MB_STREAM_ALLOC 0x0800 +/* Use reserved root blocks if needed */ +#define EXT4_MB_USE_ROOT_BLOCKS 0x1000 +/* Use blocks from reserved pool */ +#define EXT4_MB_USE_RESERVED 0x2000 +/* Do strict check for free blocks while retrying block allocation */ +#define EXT4_MB_STRICT_CHECK 0x4000 + +struct ext4_allocation_request { + /* target inode for block we're allocating */ + struct inode *inode; + /* how many blocks we want to allocate */ + unsigned int len; + /* logical block in target inode */ + ext4_lblk_t logical; + /* the closest logical allocated block to the left */ + ext4_lblk_t lleft; + /* the closest logical allocated block to the right */ + ext4_lblk_t lright; + /* phys. target (a hint) */ + ext4_fsblk_t goal; + /* phys. block for the closest logical allocated block to the left */ + ext4_fsblk_t pleft; + /* phys. block for the closest logical allocated block to the right */ + ext4_fsblk_t pright; + /* flags. see above EXT4_MB_HINT_* */ + unsigned int flags; +}; + +/* + * Logical to physical block mapping, used by ext4_map_blocks() + * + * This structure is used to pass requests into ext4_map_blocks() as + * well as to store the information returned by ext4_map_blocks(). It + * takes less room on the stack than a struct buffer_head. + */ +#define EXT4_MAP_NEW BIT(BH_New) +#define EXT4_MAP_MAPPED BIT(BH_Mapped) +#define EXT4_MAP_UNWRITTEN BIT(BH_Unwritten) +#define EXT4_MAP_BOUNDARY BIT(BH_Boundary) +#define EXT4_MAP_DELAYED BIT(BH_Delay) +/* + * This is for use in ext4_map_query_blocks() for a special case where we can + * have a physically and logically contiguous blocks split across two leaf + * nodes instead of a single extent. This is required in case of atomic writes + * to know whether the returned extent is last in leaf. If yes, then lookup for + * next in leaf block in ext4_map_query_blocks_next_in_leaf(). + * - This is never going to be added to any buffer head state. + * - We use the next available bit after BH_BITMAP_UPTODATE. + */ +#define EXT4_MAP_QUERY_LAST_IN_LEAF BIT(BH_BITMAP_UPTODATE + 1) +#define EXT4_MAP_FLAGS (EXT4_MAP_NEW | EXT4_MAP_MAPPED |\ + EXT4_MAP_UNWRITTEN | EXT4_MAP_BOUNDARY |\ + EXT4_MAP_DELAYED | EXT4_MAP_QUERY_LAST_IN_LEAF) + +struct ext4_map_blocks { + ext4_fsblk_t m_pblk; + ext4_lblk_t m_lblk; + unsigned int m_len; + unsigned int m_flags; + u64 m_seq; +}; + +/* + * Block validity checking, system zone rbtree. + */ +struct ext4_system_blocks { + struct rb_root root; + struct rcu_head rcu; +}; + +/* + * Flags for ext4_io_end->flags + */ +#define EXT4_IO_END_UNWRITTEN 0x0001 +#define EXT4_IO_END_FAILED 0x0002 + +#define EXT4_IO_END_DEFER_COMPLETION (EXT4_IO_END_UNWRITTEN | EXT4_IO_END_FAILED) + +struct ext4_io_end_vec { + struct list_head list; /* list of io_end_vec */ + loff_t offset; /* offset in the file */ + ssize_t size; /* size of the extent */ +}; + +/* + * For converting unwritten extents on a work queue. 'handle' is used for + * buffered writeback. + */ +typedef struct ext4_io_end { + struct list_head list; /* per-file finished IO list */ + handle_t *handle; /* handle reserved for extent + * conversion */ + struct inode *inode; /* file being written to */ + struct bio *bio; /* Linked list of completed + * bios covering the extent */ + unsigned int flag; /* unwritten or not */ + refcount_t count; /* reference counter */ + struct list_head list_vec; /* list of ext4_io_end_vec */ +} ext4_io_end_t; + +struct ext4_io_submit { + struct writeback_control *io_wbc; + struct bio *io_bio; + ext4_io_end_t *io_end; + sector_t io_next_block; +}; + +/* + * Special inodes numbers + */ +#define EXT4_BAD_INO 1 /* Bad blocks inode */ +#define EXT4_ROOT_INO 2 /* Root inode */ +#define EXT4_USR_QUOTA_INO 3 /* User quota inode */ +#define EXT4_GRP_QUOTA_INO 4 /* Group quota inode */ +#define EXT4_BOOT_LOADER_INO 5 /* Boot loader inode */ +#define EXT4_UNDEL_DIR_INO 6 /* Undelete directory inode */ +#define EXT4_RESIZE_INO 7 /* Reserved group descriptors inode */ +#define EXT4_JOURNAL_INO 8 /* Journal inode */ + +/* First non-reserved inode for old ext4 filesystems */ +#define EXT4_GOOD_OLD_FIRST_INO 11 + +/* + * Maximal count of links to a file + */ +#define EXT4_LINK_MAX 65000 + +/* + * Macro-instructions used to manage several block sizes + */ +#define EXT4_MIN_BLOCK_SIZE 1024 +#define EXT4_MAX_BLOCK_SIZE 65536 +#define EXT4_MIN_BLOCK_LOG_SIZE 10 +#define EXT4_MAX_BLOCK_LOG_SIZE 16 +#define EXT4_MAX_CLUSTER_LOG_SIZE 28 +#ifdef __KERNEL__ +# define EXT4_BLOCK_SIZE(s) ((s)->s_blocksize) +#else +# define EXT4_BLOCK_SIZE(s) (EXT4_MIN_BLOCK_SIZE << (s)->s_log_block_size) +#endif +#define EXT4_ADDR_PER_BLOCK(s) (EXT4_BLOCK_SIZE(s) / sizeof(__u32)) +#define EXT4_CLUSTER_SIZE(s) (EXT4_BLOCK_SIZE(s) << \ + EXT4_SB(s)->s_cluster_bits) +#ifdef __KERNEL__ +# define EXT4_BLOCK_SIZE_BITS(s) ((s)->s_blocksize_bits) +# define EXT4_CLUSTER_BITS(s) (EXT4_SB(s)->s_cluster_bits) +#else +# define EXT4_BLOCK_SIZE_BITS(s) ((s)->s_log_block_size + 10) +#endif +#ifdef __KERNEL__ +#define EXT4_ADDR_PER_BLOCK_BITS(s) (EXT4_SB(s)->s_addr_per_block_bits) +#define EXT4_INODE_SIZE(s) (EXT4_SB(s)->s_inode_size) +#define EXT4_FIRST_INO(s) (EXT4_SB(s)->s_first_ino) +#else +#define EXT4_INODE_SIZE(s) (((s)->s_rev_level == EXT4_GOOD_OLD_REV) ? \ + EXT4_GOOD_OLD_INODE_SIZE : \ + (s)->s_inode_size) +#define EXT4_FIRST_INO(s) (((s)->s_rev_level == EXT4_GOOD_OLD_REV) ? \ + EXT4_GOOD_OLD_FIRST_INO : \ + (s)->s_first_ino) +#endif +#define EXT4_BLOCK_ALIGN(size, blkbits) ALIGN((size), (1 << (blkbits))) +#define EXT4_MAX_BLOCKS(size, offset, blkbits) \ + ((EXT4_BLOCK_ALIGN(size + offset, blkbits) >> blkbits) - (offset >> \ + blkbits)) +#define EXT4_B_TO_LBLK(inode, offset) \ + (round_up((offset), i_blocksize(inode)) >> (inode)->i_blkbits) +#define EXT4_LBLK_TO_B(inode, lblk) ((loff_t)(lblk) << (inode)->i_blkbits) + +/* Translate a block number to a page index */ +#define EXT4_LBLK_TO_PG(inode, lblk) (EXT4_LBLK_TO_B((inode), (lblk)) >> \ + PAGE_SHIFT) +/* Translate a page index to a block number */ +#define EXT4_PG_TO_LBLK(inode, pnum) (((loff_t)(pnum) << PAGE_SHIFT) >> \ + (inode)->i_blkbits) +/* Translate a block number to a cluster number */ +#define EXT4_B2C(sbi, blk) ((blk) >> (sbi)->s_cluster_bits) +/* Translate a cluster number to a block number */ +#define EXT4_C2B(sbi, cluster) ((cluster) << (sbi)->s_cluster_bits) +/* Translate # of blks to # of clusters */ +#define EXT4_NUM_B2C(sbi, blks) (((blks) + (sbi)->s_cluster_ratio - 1) >> \ + (sbi)->s_cluster_bits) +/* Mask out the low bits to get the starting block of the cluster */ +#define EXT4_PBLK_CMASK(s, pblk) ((pblk) & \ + ~((ext4_fsblk_t) (s)->s_cluster_ratio - 1)) +#define EXT4_LBLK_CMASK(s, lblk) ((lblk) & \ + ~((ext4_lblk_t) (s)->s_cluster_ratio - 1)) +/* Fill in the low bits to get the last block of the cluster */ +#define EXT4_LBLK_CFILL(sbi, lblk) ((lblk) | \ + ((ext4_lblk_t) (sbi)->s_cluster_ratio - 1)) +/* Get the cluster offset */ +#define EXT4_PBLK_COFF(s, pblk) ((pblk) & \ + ((ext4_fsblk_t) (s)->s_cluster_ratio - 1)) +#define EXT4_LBLK_COFF(s, lblk) ((lblk) & \ + ((ext4_lblk_t) (s)->s_cluster_ratio - 1)) + +/* + * Structure of a blocks group descriptor + */ +struct ext4_group_desc +{ + __le32 bg_block_bitmap_lo; /* Blocks bitmap block */ + __le32 bg_inode_bitmap_lo; /* Inodes bitmap block */ + __le32 bg_inode_table_lo; /* Inodes table block */ + __le16 bg_free_blocks_count_lo;/* Free blocks count */ + __le16 bg_free_inodes_count_lo;/* Free inodes count */ + __le16 bg_used_dirs_count_lo; /* Directories count */ + __le16 bg_flags; /* EXT4_BG_flags (INODE_UNINIT, etc) */ + __le32 bg_exclude_bitmap_lo; /* Exclude bitmap for snapshots */ + __le16 bg_block_bitmap_csum_lo;/* crc32c(s_uuid+grp_num+bbitmap) LE */ + __le16 bg_inode_bitmap_csum_lo;/* crc32c(s_uuid+grp_num+ibitmap) LE */ + __le16 bg_itable_unused_lo; /* Unused inodes count */ + __le16 bg_checksum; /* crc16(sb_uuid+group+desc) */ + __le32 bg_block_bitmap_hi; /* Blocks bitmap block MSB */ + __le32 bg_inode_bitmap_hi; /* Inodes bitmap block MSB */ + __le32 bg_inode_table_hi; /* Inodes table block MSB */ + __le16 bg_free_blocks_count_hi;/* Free blocks count MSB */ + __le16 bg_free_inodes_count_hi;/* Free inodes count MSB */ + __le16 bg_used_dirs_count_hi; /* Directories count MSB */ + __le16 bg_itable_unused_hi; /* Unused inodes count MSB */ + __le32 bg_exclude_bitmap_hi; /* Exclude bitmap block MSB */ + __le16 bg_block_bitmap_csum_hi;/* crc32c(s_uuid+grp_num+bbitmap) BE */ + __le16 bg_inode_bitmap_csum_hi;/* crc32c(s_uuid+grp_num+ibitmap) BE */ + __u32 bg_reserved; +}; + +#define EXT4_BG_INODE_BITMAP_CSUM_HI_END \ + (offsetof(struct ext4_group_desc, bg_inode_bitmap_csum_hi) + \ + sizeof(__le16)) +#define EXT4_BG_BLOCK_BITMAP_CSUM_HI_END \ + (offsetof(struct ext4_group_desc, bg_block_bitmap_csum_hi) + \ + sizeof(__le16)) + +/* + * Structure of a flex block group info + */ + +struct flex_groups { + atomic64_t free_clusters; + atomic_t free_inodes; + atomic_t used_dirs; +}; + +#define EXT4_BG_INODE_UNINIT 0x0001 /* Inode table/bitmap not in use */ +#define EXT4_BG_BLOCK_UNINIT 0x0002 /* Block bitmap not in use */ +#define EXT4_BG_INODE_ZEROED 0x0004 /* On-disk itable initialized to zero */ + +/* + * Macro-instructions used to manage group descriptors + */ +#define EXT4_MIN_DESC_SIZE 32 +#define EXT4_MIN_DESC_SIZE_64BIT 64 +#define EXT4_MAX_DESC_SIZE EXT4_MIN_BLOCK_SIZE +#define EXT4_DESC_SIZE(s) (EXT4_SB(s)->s_desc_size) +#ifdef __KERNEL__ +# define EXT4_BLOCKS_PER_GROUP(s) (EXT4_SB(s)->s_blocks_per_group) +# define EXT4_CLUSTERS_PER_GROUP(s) (EXT4_SB(s)->s_clusters_per_group) +# define EXT4_DESC_PER_BLOCK(s) (EXT4_SB(s)->s_desc_per_block) +# define EXT4_INODES_PER_GROUP(s) (EXT4_SB(s)->s_inodes_per_group) +# define EXT4_DESC_PER_BLOCK_BITS(s) (EXT4_SB(s)->s_desc_per_block_bits) +#else +# define EXT4_BLOCKS_PER_GROUP(s) ((s)->s_blocks_per_group) +# define EXT4_DESC_PER_BLOCK(s) (EXT4_BLOCK_SIZE(s) / EXT4_DESC_SIZE(s)) +# define EXT4_INODES_PER_GROUP(s) ((s)->s_inodes_per_group) +#endif + +/* + * Constants relative to the data blocks + */ +#define EXT4_NDIR_BLOCKS 12 +#define EXT4_IND_BLOCK EXT4_NDIR_BLOCKS +#define EXT4_DIND_BLOCK (EXT4_IND_BLOCK + 1) +#define EXT4_TIND_BLOCK (EXT4_DIND_BLOCK + 1) +#define EXT4_N_BLOCKS (EXT4_TIND_BLOCK + 1) + +/* + * Inode flags + */ +#define EXT4_SECRM_FL 0x00000001 /* Secure deletion */ +#define EXT4_UNRM_FL 0x00000002 /* Undelete */ +#define EXT4_COMPR_FL 0x00000004 /* Compress file */ +#define EXT4_SYNC_FL 0x00000008 /* Synchronous updates */ +#define EXT4_IMMUTABLE_FL 0x00000010 /* Immutable file */ +#define EXT4_APPEND_FL 0x00000020 /* writes to file may only append */ +#define EXT4_NODUMP_FL 0x00000040 /* do not dump file */ +#define EXT4_NOATIME_FL 0x00000080 /* do not update atime */ +/* Reserved for compression usage... */ +#define EXT4_DIRTY_FL 0x00000100 +#define EXT4_COMPRBLK_FL 0x00000200 /* One or more compressed clusters */ +#define EXT4_NOCOMPR_FL 0x00000400 /* Don't compress */ + /* nb: was previously EXT2_ECOMPR_FL */ +#define EXT4_ENCRYPT_FL 0x00000800 /* encrypted file */ +/* End compression flags --- maybe not all used */ +#define EXT4_INDEX_FL 0x00001000 /* hash-indexed directory */ +#define EXT4_IMAGIC_FL 0x00002000 /* AFS directory */ +#define EXT4_JOURNAL_DATA_FL 0x00004000 /* file data should be journaled */ +#define EXT4_NOTAIL_FL 0x00008000 /* file tail should not be merged */ +#define EXT4_DIRSYNC_FL 0x00010000 /* dirsync behaviour (directories only) */ +#define EXT4_TOPDIR_FL 0x00020000 /* Top of directory hierarchies*/ +#define EXT4_HUGE_FILE_FL 0x00040000 /* Set to each huge file */ +#define EXT4_EXTENTS_FL 0x00080000 /* Inode uses extents */ +#define EXT4_VERITY_FL 0x00100000 /* Verity protected inode */ +#define EXT4_EA_INODE_FL 0x00200000 /* Inode used for large EA */ +/* 0x00400000 was formerly EXT4_EOFBLOCKS_FL */ + +#define EXT4_DAX_FL 0x02000000 /* Inode is DAX */ + +#define EXT4_INLINE_DATA_FL 0x10000000 /* Inode has inline data. */ +#define EXT4_PROJINHERIT_FL 0x20000000 /* Create with parents projid */ +#define EXT4_CASEFOLD_FL 0x40000000 /* Casefolded directory */ +#define EXT4_RESERVED_FL 0x80000000 /* reserved for ext4 lib */ + +/* User modifiable flags */ +#define EXT4_FL_USER_MODIFIABLE (EXT4_SECRM_FL | \ + EXT4_UNRM_FL | \ + EXT4_COMPR_FL | \ + EXT4_SYNC_FL | \ + EXT4_IMMUTABLE_FL | \ + EXT4_APPEND_FL | \ + EXT4_NODUMP_FL | \ + EXT4_NOATIME_FL | \ + EXT4_JOURNAL_DATA_FL | \ + EXT4_NOTAIL_FL | \ + EXT4_DIRSYNC_FL | \ + EXT4_TOPDIR_FL | \ + EXT4_EXTENTS_FL | \ + 0x00400000 /* EXT4_EOFBLOCKS_FL */ | \ + EXT4_DAX_FL | \ + EXT4_PROJINHERIT_FL | \ + EXT4_CASEFOLD_FL) + +/* User visible flags */ +#define EXT4_FL_USER_VISIBLE (EXT4_FL_USER_MODIFIABLE | \ + EXT4_DIRTY_FL | \ + EXT4_COMPRBLK_FL | \ + EXT4_NOCOMPR_FL | \ + EXT4_ENCRYPT_FL | \ + EXT4_INDEX_FL | \ + EXT4_VERITY_FL | \ + EXT4_INLINE_DATA_FL) + +/* Flags that should be inherited by new inodes from their parent. */ +#define EXT4_FL_INHERITED (EXT4_SECRM_FL | EXT4_UNRM_FL | EXT4_COMPR_FL |\ + EXT4_SYNC_FL | EXT4_NODUMP_FL | EXT4_NOATIME_FL |\ + EXT4_NOCOMPR_FL | EXT4_JOURNAL_DATA_FL |\ + EXT4_NOTAIL_FL | EXT4_DIRSYNC_FL |\ + EXT4_PROJINHERIT_FL | EXT4_CASEFOLD_FL |\ + EXT4_DAX_FL) + +/* Flags that are appropriate for regular files (all but dir-specific ones). */ +#define EXT4_REG_FLMASK (~(EXT4_DIRSYNC_FL | EXT4_TOPDIR_FL | EXT4_CASEFOLD_FL |\ + EXT4_PROJINHERIT_FL)) + +/* Flags that are appropriate for non-directories/regular files. */ +#define EXT4_OTHER_FLMASK (EXT4_NODUMP_FL | EXT4_NOATIME_FL) + +/* The only flags that should be swapped */ +#define EXT4_FL_SHOULD_SWAP (EXT4_HUGE_FILE_FL | EXT4_EXTENTS_FL) + +/* Flags which are mutually exclusive to DAX */ +#define EXT4_DAX_MUT_EXCL (EXT4_VERITY_FL | EXT4_ENCRYPT_FL |\ + EXT4_JOURNAL_DATA_FL | EXT4_INLINE_DATA_FL) + +/* Mask out flags that are inappropriate for the given type of inode. */ +static inline __u32 ext4_mask_flags(umode_t mode, __u32 flags) +{ + if (S_ISDIR(mode)) + return flags; + else if (S_ISREG(mode)) + return flags & EXT4_REG_FLMASK; + else + return flags & EXT4_OTHER_FLMASK; +} + +/* + * Inode flags used for atomic set/get + */ +enum { + EXT4_INODE_SECRM = 0, /* Secure deletion */ + EXT4_INODE_UNRM = 1, /* Undelete */ + EXT4_INODE_COMPR = 2, /* Compress file */ + EXT4_INODE_SYNC = 3, /* Synchronous updates */ + EXT4_INODE_IMMUTABLE = 4, /* Immutable file */ + EXT4_INODE_APPEND = 5, /* writes to file may only append */ + EXT4_INODE_NODUMP = 6, /* do not dump file */ + EXT4_INODE_NOATIME = 7, /* do not update atime */ +/* Reserved for compression usage... */ + EXT4_INODE_DIRTY = 8, + EXT4_INODE_COMPRBLK = 9, /* One or more compressed clusters */ + EXT4_INODE_NOCOMPR = 10, /* Don't compress */ + EXT4_INODE_ENCRYPT = 11, /* Encrypted file */ +/* End compression flags --- maybe not all used */ + EXT4_INODE_INDEX = 12, /* hash-indexed directory */ + EXT4_INODE_IMAGIC = 13, /* AFS directory */ + EXT4_INODE_JOURNAL_DATA = 14, /* file data should be journaled */ + EXT4_INODE_NOTAIL = 15, /* file tail should not be merged */ + EXT4_INODE_DIRSYNC = 16, /* dirsync behaviour (directories only) */ + EXT4_INODE_TOPDIR = 17, /* Top of directory hierarchies*/ + EXT4_INODE_HUGE_FILE = 18, /* Set to each huge file */ + EXT4_INODE_EXTENTS = 19, /* Inode uses extents */ + EXT4_INODE_VERITY = 20, /* Verity protected inode */ + EXT4_INODE_EA_INODE = 21, /* Inode used for large EA */ +/* 22 was formerly EXT4_INODE_EOFBLOCKS */ + EXT4_INODE_DAX = 25, /* Inode is DAX */ + EXT4_INODE_INLINE_DATA = 28, /* Data in inode. */ + EXT4_INODE_PROJINHERIT = 29, /* Create with parents projid */ + EXT4_INODE_CASEFOLD = 30, /* Casefolded directory */ + EXT4_INODE_RESERVED = 31, /* reserved for ext4 lib */ +}; + +/* + * Since it's pretty easy to mix up bit numbers and hex values, we use a + * build-time check to make sure that EXT4_XXX_FL is consistent with respect to + * EXT4_INODE_XXX. If all is well, the macros will be dropped, so, it won't cost + * any extra space in the compiled kernel image, otherwise, the build will fail. + * It's important that these values are the same, since we are using + * EXT4_INODE_XXX to test for flag values, but EXT4_XXX_FL must be consistent + * with the values of FS_XXX_FL defined in include/linux/fs.h and the on-disk + * values found in ext2, ext3 and ext4 filesystems, and of course the values + * defined in e2fsprogs. + * + * It's not paranoia if the Murphy's Law really *is* out to get you. :-) + */ +#define TEST_FLAG_VALUE(FLAG) (EXT4_##FLAG##_FL == (1U << EXT4_INODE_##FLAG)) +#define CHECK_FLAG_VALUE(FLAG) BUILD_BUG_ON(!TEST_FLAG_VALUE(FLAG)) + +static inline void ext4_check_flag_values(void) +{ + CHECK_FLAG_VALUE(SECRM); + CHECK_FLAG_VALUE(UNRM); + CHECK_FLAG_VALUE(COMPR); + CHECK_FLAG_VALUE(SYNC); + CHECK_FLAG_VALUE(IMMUTABLE); + CHECK_FLAG_VALUE(APPEND); + CHECK_FLAG_VALUE(NODUMP); + CHECK_FLAG_VALUE(NOATIME); + CHECK_FLAG_VALUE(DIRTY); + CHECK_FLAG_VALUE(COMPRBLK); + CHECK_FLAG_VALUE(NOCOMPR); + CHECK_FLAG_VALUE(ENCRYPT); + CHECK_FLAG_VALUE(INDEX); + CHECK_FLAG_VALUE(IMAGIC); + CHECK_FLAG_VALUE(JOURNAL_DATA); + CHECK_FLAG_VALUE(NOTAIL); + CHECK_FLAG_VALUE(DIRSYNC); + CHECK_FLAG_VALUE(TOPDIR); + CHECK_FLAG_VALUE(HUGE_FILE); + CHECK_FLAG_VALUE(EXTENTS); + CHECK_FLAG_VALUE(VERITY); + CHECK_FLAG_VALUE(EA_INODE); + CHECK_FLAG_VALUE(INLINE_DATA); + CHECK_FLAG_VALUE(PROJINHERIT); + CHECK_FLAG_VALUE(CASEFOLD); + CHECK_FLAG_VALUE(RESERVED); +} + +#if defined(__KERNEL__) && defined(CONFIG_COMPAT) +struct compat_ext4_new_group_input { + u32 group; + compat_u64 block_bitmap; + compat_u64 inode_bitmap; + compat_u64 inode_table; + u32 blocks_count; + u16 reserved_blocks; + u16 unused; +}; +#endif + +/* The struct ext4_new_group_input in kernel space, with free_blocks_count */ +struct ext4_new_group_data { + __u32 group; + __u64 block_bitmap; + __u64 inode_bitmap; + __u64 inode_table; + __u32 blocks_count; + __u16 reserved_blocks; + __u16 mdata_blocks; + __u32 free_clusters_count; +}; + +/* Indexes used to index group tables in ext4_new_group_data */ +enum { + BLOCK_BITMAP = 0, /* block bitmap */ + INODE_BITMAP, /* inode bitmap */ + INODE_TABLE, /* inode tables */ + GROUP_TABLE_COUNT, +}; + +/* + * Flags used by ext4_map_blocks() + */ + /* Allocate any needed blocks and/or convert an unwritten + extent to be an initialized ext4 */ +#define EXT4_GET_BLOCKS_CREATE 0x0001 + /* Request the creation of an unwritten extent */ +#define EXT4_GET_BLOCKS_UNWRIT_EXT 0x0002 +#define EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT (EXT4_GET_BLOCKS_UNWRIT_EXT|\ + EXT4_GET_BLOCKS_CREATE) + /* Caller is from the delayed allocation writeout path + * finally doing the actual allocation of delayed blocks */ +#define EXT4_GET_BLOCKS_DELALLOC_RESERVE 0x0004 + /* + * This means that we cannot merge newly allocated extents, and if we + * found an unwritten extent, we need to split it. + */ +#define EXT4_GET_BLOCKS_SPLIT_NOMERGE 0x0008 + /* Convert unwritten extent to initialized. */ +#define EXT4_GET_BLOCKS_CONVERT 0x0010 + /* Eventual metadata allocation (due to growing extent tree) + * should not fail, so try to use reserved blocks for that.*/ +#define EXT4_GET_BLOCKS_METADATA_NOFAIL 0x0020 + /* Don't normalize allocation size (used for fallocate) */ +#define EXT4_GET_BLOCKS_NO_NORMALIZE 0x0040 + /* Convert written extents to unwritten */ +#define EXT4_GET_BLOCKS_CONVERT_UNWRITTEN 0x0100 + /* Write zeros to newly created written extents */ +#define EXT4_GET_BLOCKS_ZERO 0x0200 +#define EXT4_GET_BLOCKS_CREATE_ZERO (EXT4_GET_BLOCKS_CREATE |\ + EXT4_GET_BLOCKS_ZERO) + /* Caller is in the context of data submission, such as writeback, + * fsync, etc. Especially, in the generic writeback path, caller will + * submit data before dropping transaction handle. This allows jbd2 + * to avoid submitting data before commit. */ +#define EXT4_GET_BLOCKS_IO_SUBMIT 0x0400 + /* Convert extent to initialized after IO complete */ +#define EXT4_GET_BLOCKS_IO_CONVERT_EXT (EXT4_GET_BLOCKS_CONVERT |\ + EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT |\ + EXT4_GET_BLOCKS_IO_SUBMIT) + /* Caller is in the atomic contex, find extent if it has been cached */ +#define EXT4_GET_BLOCKS_CACHED_NOWAIT 0x0800 +/* + * Atomic write caller needs this to query in the slow path of mixed mapping + * case, when a contiguous extent can be split across two adjacent leaf nodes. + * Look EXT4_MAP_QUERY_LAST_IN_LEAF. + */ +#define EXT4_GET_BLOCKS_QUERY_LAST_IN_LEAF 0x1000 + +/* + * The bit position of these flags must not overlap with any of the + * EXT4_GET_BLOCKS_*. They are used by ext4_find_extent(), + * read_extent_tree_block(), ext4_split_extent_at(), + * ext4_ext_insert_extent(), and ext4_ext_create_new_leaf(). + * EXT4_EX_NOCACHE is used to indicate that the we shouldn't be + * caching the extents when reading from the extent tree while a + * truncate or punch hole operation is in progress. + */ +#define EXT4_EX_NOCACHE 0x40000000 +#define EXT4_EX_FORCE_CACHE 0x20000000 +#define EXT4_EX_NOFAIL 0x10000000 +/* + * ext4_map_query_blocks() uses this filter mask to filter the flags needed to + * pass while lookup/querying of on disk extent tree. + */ +#define EXT4_EX_QUERY_FILTER (EXT4_EX_NOCACHE | EXT4_EX_FORCE_CACHE |\ + EXT4_EX_NOFAIL |\ + EXT4_GET_BLOCKS_QUERY_LAST_IN_LEAF) + +/* + * Flags used by ext4_free_blocks + */ +#define EXT4_FREE_BLOCKS_METADATA 0x0001 +#define EXT4_FREE_BLOCKS_FORGET 0x0002 +#define EXT4_FREE_BLOCKS_VALIDATED 0x0004 +#define EXT4_FREE_BLOCKS_NO_QUOT_UPDATE 0x0008 +#define EXT4_FREE_BLOCKS_NOFREE_FIRST_CLUSTER 0x0010 +#define EXT4_FREE_BLOCKS_NOFREE_LAST_CLUSTER 0x0020 +#define EXT4_FREE_BLOCKS_RERESERVE_CLUSTER 0x0040 + +#if defined(__KERNEL__) && defined(CONFIG_COMPAT) +/* + * ioctl commands in 32 bit emulation + */ +#define EXT4_IOC32_GETVERSION _IOR('f', 3, int) +#define EXT4_IOC32_SETVERSION _IOW('f', 4, int) +#define EXT4_IOC32_GETRSVSZ _IOR('f', 5, int) +#define EXT4_IOC32_SETRSVSZ _IOW('f', 6, int) +#define EXT4_IOC32_GROUP_EXTEND _IOW('f', 7, unsigned int) +#define EXT4_IOC32_GROUP_ADD _IOW('f', 8, struct compat_ext4_new_group_input) +#define EXT4_IOC32_GETVERSION_OLD FS_IOC32_GETVERSION +#define EXT4_IOC32_SETVERSION_OLD FS_IOC32_SETVERSION +#endif + +/* Max physical block we can address w/o extents */ +#define EXT4_MAX_BLOCK_FILE_PHYS 0xFFFFFFFF + +/* Max logical block we can support */ +#define EXT4_MAX_LOGICAL_BLOCK 0xFFFFFFFE + +/* + * Structure of an inode on the disk + */ +struct ext4_inode { + __le16 i_mode; /* File mode */ + __le16 i_uid; /* Low 16 bits of Owner Uid */ + __le32 i_size_lo; /* Size in bytes */ + __le32 i_atime; /* Access time */ + __le32 i_ctime; /* Inode Change time */ + __le32 i_mtime; /* Modification time */ + __le32 i_dtime; /* Deletion Time */ + __le16 i_gid; /* Low 16 bits of Group Id */ + __le16 i_links_count; /* Links count */ + __le32 i_blocks_lo; /* Blocks count */ + __le32 i_flags; /* File flags */ + union { + struct { + __le32 l_i_version; + } linux1; + struct { + __u32 h_i_translator; + } hurd1; + struct { + __u32 m_i_reserved1; + } masix1; + } osd1; /* OS dependent 1 */ + __le32 i_block[EXT4_N_BLOCKS];/* Pointers to blocks */ + __le32 i_generation; /* File version (for NFS) */ + __le32 i_file_acl_lo; /* File ACL */ + __le32 i_size_high; + __le32 i_obso_faddr; /* Obsoleted fragment address */ + union { + struct { + __le16 l_i_blocks_high; /* were l_i_reserved1 */ + __le16 l_i_file_acl_high; + __le16 l_i_uid_high; /* these 2 fields */ + __le16 l_i_gid_high; /* were reserved2[0] */ + __le16 l_i_checksum_lo;/* crc32c(uuid+inum+inode) LE */ + __le16 l_i_reserved; + } linux2; + struct { + __le16 h_i_reserved1; /* Obsoleted fragment number/size which are removed in ext4 */ + __u16 h_i_mode_high; + __u16 h_i_uid_high; + __u16 h_i_gid_high; + __u32 h_i_author; + } hurd2; + struct { + __le16 h_i_reserved1; /* Obsoleted fragment number/size which are removed in ext4 */ + __le16 m_i_file_acl_high; + __u32 m_i_reserved2[2]; + } masix2; + } osd2; /* OS dependent 2 */ + __le16 i_extra_isize; + __le16 i_checksum_hi; /* crc32c(uuid+inum+inode) BE */ + __le32 i_ctime_extra; /* extra Change time (nsec << 2 | epoch) */ + __le32 i_mtime_extra; /* extra Modification time(nsec << 2 | epoch) */ + __le32 i_atime_extra; /* extra Access time (nsec << 2 | epoch) */ + __le32 i_crtime; /* File Creation time */ + __le32 i_crtime_extra; /* extra FileCreationtime (nsec << 2 | epoch) */ + __le32 i_version_hi; /* high 32 bits for 64-bit version */ + __le32 i_projid; /* Project ID */ +}; + +#define EXT4_EPOCH_BITS 2 +#define EXT4_EPOCH_MASK ((1 << EXT4_EPOCH_BITS) - 1) +#define EXT4_NSEC_MASK (~0UL << EXT4_EPOCH_BITS) + +/* + * Extended fields will fit into an inode if the filesystem was formatted + * with large inodes (-I 256 or larger) and there are not currently any EAs + * consuming all of the available space. For new inodes we always reserve + * enough space for the kernel's known extended fields, but for inodes + * created with an old kernel this might not have been the case. None of + * the extended inode fields is critical for correct filesystem operation. + * This macro checks if a certain field fits in the inode. Note that + * inode-size = GOOD_OLD_INODE_SIZE + i_extra_isize + */ +#define EXT4_FITS_IN_INODE(ext4_inode, einode, field) \ + ((offsetof(typeof(*ext4_inode), field) + \ + sizeof((ext4_inode)->field)) \ + <= (EXT4_GOOD_OLD_INODE_SIZE + \ + (einode)->i_extra_isize)) \ + +/* + * We use an encoding that preserves the times for extra epoch "00": + * + * extra msb of adjust for signed + * epoch 32-bit 32-bit tv_sec to + * bits time decoded 64-bit tv_sec 64-bit tv_sec valid time range + * 0 0 1 -0x80000000..-0x00000001 0x000000000 1901-12-13..1969-12-31 + * 0 0 0 0x000000000..0x07fffffff 0x000000000 1970-01-01..2038-01-19 + * 0 1 1 0x080000000..0x0ffffffff 0x100000000 2038-01-19..2106-02-07 + * 0 1 0 0x100000000..0x17fffffff 0x100000000 2106-02-07..2174-02-25 + * 1 0 1 0x180000000..0x1ffffffff 0x200000000 2174-02-25..2242-03-16 + * 1 0 0 0x200000000..0x27fffffff 0x200000000 2242-03-16..2310-04-04 + * 1 1 1 0x280000000..0x2ffffffff 0x300000000 2310-04-04..2378-04-22 + * 1 1 0 0x300000000..0x37fffffff 0x300000000 2378-04-22..2446-05-10 + * + * Note that previous versions of the kernel on 64-bit systems would + * incorrectly use extra epoch bits 1,1 for dates between 1901 and + * 1970. e2fsck will correct this, assuming that it is run on the + * affected filesystem before 2242. + */ + +static inline __le32 ext4_encode_extra_time(struct timespec64 ts) +{ + u32 extra = ((ts.tv_sec - (s32)ts.tv_sec) >> 32) & EXT4_EPOCH_MASK; + return cpu_to_le32(extra | (ts.tv_nsec << EXT4_EPOCH_BITS)); +} + +static inline struct timespec64 ext4_decode_extra_time(__le32 base, + __le32 extra) +{ + struct timespec64 ts = { .tv_sec = (signed)le32_to_cpu(base) }; + + if (unlikely(extra & cpu_to_le32(EXT4_EPOCH_MASK))) + ts.tv_sec += (u64)(le32_to_cpu(extra) & EXT4_EPOCH_MASK) << 32; + ts.tv_nsec = (le32_to_cpu(extra) & EXT4_NSEC_MASK) >> EXT4_EPOCH_BITS; + return ts; +} + +#define EXT4_INODE_SET_XTIME_VAL(xtime, inode, raw_inode, ts) \ +do { \ + if (EXT4_FITS_IN_INODE(raw_inode, EXT4_I(inode), xtime ## _extra)) { \ + (raw_inode)->xtime = cpu_to_le32((ts).tv_sec); \ + (raw_inode)->xtime ## _extra = ext4_encode_extra_time(ts); \ + } else \ + (raw_inode)->xtime = cpu_to_le32(clamp_t(int32_t, (ts).tv_sec, S32_MIN, S32_MAX)); \ +} while (0) + +#define EXT4_INODE_SET_ATIME(inode, raw_inode) \ + EXT4_INODE_SET_XTIME_VAL(i_atime, inode, raw_inode, inode_get_atime(inode)) + +#define EXT4_INODE_SET_MTIME(inode, raw_inode) \ + EXT4_INODE_SET_XTIME_VAL(i_mtime, inode, raw_inode, inode_get_mtime(inode)) + +#define EXT4_INODE_SET_CTIME(inode, raw_inode) \ + EXT4_INODE_SET_XTIME_VAL(i_ctime, inode, raw_inode, inode_get_ctime(inode)) + +#define EXT4_EINODE_SET_XTIME(xtime, einode, raw_inode) \ + if (EXT4_FITS_IN_INODE(raw_inode, einode, xtime)) \ + EXT4_INODE_SET_XTIME_VAL(xtime, &((einode)->vfs_inode), \ + raw_inode, (einode)->xtime) + +#define EXT4_INODE_GET_XTIME_VAL(xtime, inode, raw_inode) \ + (EXT4_FITS_IN_INODE(raw_inode, EXT4_I(inode), xtime ## _extra) ? \ + ext4_decode_extra_time((raw_inode)->xtime, \ + (raw_inode)->xtime ## _extra) : \ + (struct timespec64) { \ + .tv_sec = (signed)le32_to_cpu((raw_inode)->xtime) \ + }) + +#define EXT4_INODE_GET_ATIME(inode, raw_inode) \ +do { \ + inode_set_atime_to_ts(inode, \ + EXT4_INODE_GET_XTIME_VAL(i_atime, inode, raw_inode)); \ +} while (0) + +#define EXT4_INODE_GET_MTIME(inode, raw_inode) \ +do { \ + inode_set_mtime_to_ts(inode, \ + EXT4_INODE_GET_XTIME_VAL(i_mtime, inode, raw_inode)); \ +} while (0) + +#define EXT4_INODE_GET_CTIME(inode, raw_inode) \ +do { \ + inode_set_ctime_to_ts(inode, \ + EXT4_INODE_GET_XTIME_VAL(i_ctime, inode, raw_inode)); \ +} while (0) + +#define EXT4_EINODE_GET_XTIME(xtime, einode, raw_inode) \ +do { \ + if (EXT4_FITS_IN_INODE(raw_inode, einode, xtime)) \ + (einode)->xtime = \ + EXT4_INODE_GET_XTIME_VAL(xtime, &(einode->vfs_inode), \ + raw_inode); \ + else \ + (einode)->xtime = (struct timespec64){0, 0}; \ +} while (0) + +#define i_disk_version osd1.linux1.l_i_version + +#if defined(__KERNEL__) || defined(__linux__) +#define i_reserved1 osd1.linux1.l_i_reserved1 +#define i_file_acl_high osd2.linux2.l_i_file_acl_high +#define i_blocks_high osd2.linux2.l_i_blocks_high +#define i_uid_low i_uid +#define i_gid_low i_gid +#define i_uid_high osd2.linux2.l_i_uid_high +#define i_gid_high osd2.linux2.l_i_gid_high +#define i_checksum_lo osd2.linux2.l_i_checksum_lo + +#elif defined(__GNU__) + +#define i_translator osd1.hurd1.h_i_translator +#define i_uid_high osd2.hurd2.h_i_uid_high +#define i_gid_high osd2.hurd2.h_i_gid_high +#define i_author osd2.hurd2.h_i_author + +#elif defined(__masix__) + +#define i_reserved1 osd1.masix1.m_i_reserved1 +#define i_file_acl_high osd2.masix2.m_i_file_acl_high +#define i_reserved2 osd2.masix2.m_i_reserved2 + +#endif /* defined(__KERNEL__) || defined(__linux__) */ + +#include "extents_status.h" +#include "fast_commit.h" + +/* + * Lock subclasses for i_data_sem in the ext4_inode_info structure. + * + * These are needed to avoid lockdep false positives when we need to + * allocate blocks to the quota inode during ext4_map_blocks(), while + * holding i_data_sem for a normal (non-quota) inode. Since we don't + * do quota tracking for the quota inode, this avoids deadlock (as + * well as infinite recursion, since it isn't turtles all the way + * down...) + * + * I_DATA_SEM_NORMAL - Used for most inodes + * I_DATA_SEM_OTHER - Used by move_inode.c for the second normal inode + * where the second inode has larger inode number + * than the first + * I_DATA_SEM_QUOTA - Used for quota inodes only + * I_DATA_SEM_EA - Used for ea_inodes only + * I_DATA_SEM_JOURNAL - Used for journal inode only + */ +enum { + I_DATA_SEM_NORMAL = 0, + I_DATA_SEM_OTHER, + I_DATA_SEM_QUOTA, + I_DATA_SEM_EA, + I_DATA_SEM_JOURNAL +}; + +struct ext4_fc_inode_snap; + +/* + * Snapshot failure reasons for ext4_fc_lock_updates tracepoint. + * Keep these stable for tooling. + */ +enum ext4_fc_snap_err { + EXT4_FC_SNAP_ERR_NONE = 0, + EXT4_FC_SNAP_ERR_ES_MISS, + EXT4_FC_SNAP_ERR_ES_DELAYED, + EXT4_FC_SNAP_ERR_ES_OTHER, + EXT4_FC_SNAP_ERR_INODES_CAP, + EXT4_FC_SNAP_ERR_RANGES_CAP, + EXT4_FC_SNAP_ERR_NOMEM, + EXT4_FC_SNAP_ERR_INODE_LOC, +}; + +/* + * fourth extended file system inode data in memory + */ +struct ext4_inode_info { + __le32 i_data[15]; /* unconverted */ + __u32 i_dtime; + ext4_fsblk_t i_file_acl; + + /* + * i_block_group is the number of the block group which contains + * this file's inode. Constant across the lifetime of the inode, + * it is used for making block allocation decisions - we try to + * place a file's data blocks near its inode block, and new inodes + * near to their parent directory's inode. + */ + ext4_group_t i_block_group; + ext4_lblk_t i_dir_start_lookup; +#if (BITS_PER_LONG < 64) + unsigned long i_state_flags; /* Dynamic state flags */ +#endif + unsigned long i_flags; + + /* + * Extended attributes can be read independently of the main file + * data. Taking i_rwsem even when reading would cause contention + * between readers of EAs and writers of regular file data, so + * instead we synchronize on xattr_sem when reading or changing + * EAs. + * + * EA inodes (EXT4_EA_INODE_FL) do not use xattr_sem; they reuse + * the space for deferred iput linkage. + */ + union { + struct rw_semaphore xattr_sem; + struct llist_node i_ea_iput_node; + }; + + /* + * Inodes with EXT4_STATE_ORPHAN_FILE use i_orphan_idx. Otherwise + * i_orphan is used. + */ + union { + struct list_head i_orphan; /* unlinked but open inodes */ + unsigned int i_orphan_idx; /* Index in orphan file */ + }; + + /* Fast commit related info */ + + /* For tracking dentry create updates */ + struct list_head i_fc_dilist; + struct list_head i_fc_list; /* + * inodes that need fast commit + * protected by sbi->s_fc_lock. + */ + + /* Start of lblk range that needs to be committed in this fast commit */ + ext4_lblk_t i_fc_lblk_start; + + /* End of lblk range that needs to be committed in this fast commit */ + ext4_lblk_t i_fc_lblk_len; + + /* + * Commit-time fast commit snapshots. + * + * i_fc_snap is installed and freed under sbi->s_fc_lock. The fast + * commit log writing path reads the snapshot under sbi->s_fc_lock while + * serializing fast commit TLVs. + * + * The snapshot lifetime is bounded by EXT4_STATE_FC_COMMITTING and the + * corresponding cleanup / eviction paths. + * + * i_fc_snap points to per-inode snapshot data for fast commit: + * - a raw inode snapshot for EXT4_FC_TAG_INODE + * - data range records for EXT4_FC_TAG_{ADD,DEL}_RANGE + */ + struct ext4_fc_inode_snap *i_fc_snap; + + spinlock_t i_raw_lock; /* protects updates to the raw inode */ + + /* + * Protect concurrent accesses on i_fc_lblk_start, i_fc_lblk_len + * and inode's EXT4_FC_STATE_COMMITTING state bit. + */ + spinlock_t i_fc_lock; + + /* + * i_disksize keeps track of what the inode size is ON DISK, not + * in memory. During truncate, i_size is set to the new size by + * the VFS prior to calling ext4_truncate(), but the filesystem won't + * set i_disksize to 0 until the truncate is actually under way. + * + * The intent is that i_disksize always represents the blocks which + * are used by this file. This allows recovery to restart truncate + * on orphans if we crash during truncate. We actually write i_disksize + * into the on-disk inode when writing inodes out, instead of i_size. + * + * The only time when i_disksize and i_size may be different is when + * a truncate is in progress. The only things which change i_disksize + * are ext4_get_block (growth) and ext4_truncate (shrinkth). + */ + loff_t i_disksize; + + /* + * i_data_sem is for serialising ext4_truncate() against + * ext4_getblock(). In the 2.4 ext2 design, great chunks of inode's + * data tree are chopped off during truncate. We can't do that in + * ext4 because whenever we perform intermediate commits during + * truncate, the inode and all the metadata blocks *must* be in a + * consistent state which allows truncation of the orphans to restart + * during recovery. Hence we must fix the get_block-vs-truncate race + * by other means, so we have i_data_sem. + */ + struct rw_semaphore i_data_sem; + struct inode vfs_inode; + struct jbd2_inode *jinode; + struct mapping_metadata_bhs *i_metadata_bhs; + + /* + * File creation time. Its function is same as that of + * struct timespec64 i_{a,c,m}time in the generic inode. + */ + struct timespec64 i_crtime; + + /* mballoc */ + atomic_t i_prealloc_active; + + /* allocation reservation info for delalloc */ + /* In case of bigalloc, this refer to clusters rather than blocks */ + unsigned int i_reserved_data_blocks; + struct rb_root i_prealloc_node; + rwlock_t i_prealloc_lock; + + /* extents status tree */ + struct ext4_es_tree i_es_tree; + rwlock_t i_es_lock; + struct list_head i_es_list; + unsigned int i_es_all_nr; /* protected by i_es_lock */ + unsigned int i_es_shk_nr; /* protected by i_es_lock */ + ext4_lblk_t i_es_shrink_lblk; /* Offset where we start searching for + extents to shrink. Protected by + i_es_lock */ + u64 i_es_seq; /* Change counter for extents. + Protected by i_es_lock */ + + /* ialloc */ + ext4_group_t i_last_alloc_group; + + /* pending cluster reservations for bigalloc file systems */ + struct ext4_pending_tree i_pending_tree; + + /* on-disk additional length */ + __u16 i_extra_isize; + + /* Indicate the inline data space. */ + u16 i_inline_off; + u16 i_inline_size; + +#ifdef CONFIG_QUOTA + /* quota space reservation, managed internally by quota code */ + qsize_t i_reserved_quota; +#endif + spinlock_t i_block_reservation_lock; + + /* Lock protecting lists below */ + spinlock_t i_completed_io_lock; + /* + * Completed IOs that need unwritten extents handling and have + * transaction reserved + */ + struct list_head i_rsv_conversion_list; + struct work_struct i_rsv_conversion_work; + + /* + * Transactions that contain inode's metadata needed to complete + * fsync and fdatasync, respectively. + */ + tid_t i_sync_tid; + tid_t i_datasync_tid; + +#ifdef CONFIG_QUOTA + struct dquot __rcu *i_dquot[MAXQUOTAS]; +#endif + + /* Precomputed uuid+inum+igen checksum for seeding inode checksums */ + __u32 i_csum_seed; + + kprojid_t i_projid; + +#ifdef CONFIG_FS_ENCRYPTION + struct fscrypt_inode_info *i_crypt_info; +#endif +}; + +/* + * File system states + */ +#define EXT4_VALID_FS 0x0001 /* Unmounted cleanly */ +#define EXT4_ERROR_FS 0x0002 /* Errors detected */ +#define EXT4_ORPHAN_FS 0x0004 /* Orphans being recovered */ +#define EXT4_FC_REPLAY 0x0020 /* Fast commit replay ongoing */ + +/* + * Misc. filesystem flags + */ +#define EXT2_FLAGS_SIGNED_HASH 0x0001 /* Signed dirhash in use */ +#define EXT2_FLAGS_UNSIGNED_HASH 0x0002 /* Unsigned dirhash in use */ +#define EXT2_FLAGS_TEST_FILESYS 0x0004 /* to test development code */ + +/* + * Mount flags set via mount options or defaults + */ +#define EXT4_MOUNT_NO_MBCACHE 0x00001 /* Do not use mbcache */ +#define EXT4_MOUNT_GRPID 0x00004 /* Create files with directory's group */ +#define EXT4_MOUNT_DEBUG 0x00008 /* Some debugging messages */ +#define EXT4_MOUNT_ERRORS_CONT 0x00010 /* Continue on errors */ +#define EXT4_MOUNT_ERRORS_RO 0x00020 /* Remount fs ro on errors */ +#define EXT4_MOUNT_ERRORS_PANIC 0x00040 /* Panic on errors */ +#define EXT4_MOUNT_ERRORS_MASK 0x00070 +#define EXT4_MOUNT_MINIX_DF 0x00080 /* Mimics the Minix statfs */ +#define EXT4_MOUNT_NOLOAD 0x00100 /* Don't use existing journal*/ +#ifdef CONFIG_FS_DAX +#define EXT4_MOUNT_DAX_ALWAYS 0x00200 /* Direct Access */ +#else +#define EXT4_MOUNT_DAX_ALWAYS 0 +#endif +#define EXT4_MOUNT_DATA_FLAGS 0x00C00 /* Mode for data writes: */ +#define EXT4_MOUNT_JOURNAL_DATA 0x00400 /* Write data to journal */ +#define EXT4_MOUNT_ORDERED_DATA 0x00800 /* Flush data before commit */ +#define EXT4_MOUNT_WRITEBACK_DATA 0x00C00 /* No data ordering */ +#define EXT4_MOUNT_UPDATE_JOURNAL 0x01000 /* Update the journal format */ +#define EXT4_MOUNT_NO_UID32 0x02000 /* Disable 32-bit UIDs */ +#define EXT4_MOUNT_XATTR_USER 0x04000 /* Extended user attributes */ +#define EXT4_MOUNT_POSIX_ACL 0x08000 /* POSIX Access Control Lists */ +#define EXT4_MOUNT_NO_AUTO_DA_ALLOC 0x10000 /* No auto delalloc mapping */ +#define EXT4_MOUNT_BARRIER 0x20000 /* Use block barriers */ +#define EXT4_MOUNT_QUOTA 0x40000 /* Some quota option set */ +#define EXT4_MOUNT_USRQUOTA 0x80000 /* "old" user quota, + * enable enforcement for hidden + * quota files */ +#define EXT4_MOUNT_GRPQUOTA 0x100000 /* "old" group quota, enable + * enforcement for hidden quota + * files */ +#define EXT4_MOUNT_PRJQUOTA 0x200000 /* Enable project quota + * enforcement */ +#define EXT4_MOUNT_DIOREAD_NOLOCK 0x400000 /* Enable support for dio read nolocking */ +#define EXT4_MOUNT_JOURNAL_CHECKSUM 0x800000 /* Journal checksums */ +#define EXT4_MOUNT_JOURNAL_ASYNC_COMMIT 0x1000000 /* Journal Async Commit */ +#define EXT4_MOUNT_WARN_ON_ERROR 0x2000000 /* Trigger WARN_ON on error */ +#define EXT4_MOUNT_NO_PREFETCH_BLOCK_BITMAPS 0x4000000 +#define EXT4_MOUNT_DELALLOC 0x8000000 /* Delalloc support */ +#define EXT4_MOUNT_DATA_ERR_ABORT 0x10000000 /* Abort on file data write */ +#define EXT4_MOUNT_BLOCK_VALIDITY 0x20000000 /* Block validity checking */ +#define EXT4_MOUNT_DISCARD 0x40000000 /* Issue DISCARD requests */ +#define EXT4_MOUNT_INIT_INODE_TABLE 0x80000000 /* Initialize uninitialized itables */ + +/* + * Mount flags set either automatically (could not be set by mount option) + * based on per file system feature or property or in special cases such as + * distinguishing between explicit mount option definition and default. + */ +#define EXT4_MOUNT2_EXPLICIT_DELALLOC 0x00000001 /* User explicitly + specified delalloc */ +#define EXT4_MOUNT2_STD_GROUP_SIZE 0x00000002 /* We have standard group + size of blocksize * 8 + blocks */ +#define EXT4_MOUNT2_HURD_COMPAT 0x00000004 /* Support HURD-castrated + file systems */ +#define EXT4_MOUNT2_EXPLICIT_JOURNAL_CHECKSUM 0x00000008 /* User explicitly + specified journal checksum */ + +#define EXT4_MOUNT2_JOURNAL_FAST_COMMIT 0x00000010 /* Journal fast commit */ +#define EXT4_MOUNT2_DAX_NEVER 0x00000020 /* Do not allow Direct Access */ +#define EXT4_MOUNT2_DAX_INODE 0x00000040 /* For printing options only */ +#define EXT4_MOUNT2_MB_OPTIMIZE_SCAN 0x00000080 /* Optimize group + * scanning in mballoc + */ +#define EXT4_MOUNT2_ABORT 0x00000100 /* Abort filesystem */ + +#define clear_opt(sb, opt) EXT4_SB(sb)->s_mount_opt &= \ + ~EXT4_MOUNT_##opt +#define set_opt(sb, opt) EXT4_SB(sb)->s_mount_opt |= \ + EXT4_MOUNT_##opt +#define test_opt(sb, opt) (EXT4_SB(sb)->s_mount_opt & \ + EXT4_MOUNT_##opt) + +#define clear_opt2(sb, opt) EXT4_SB(sb)->s_mount_opt2 &= \ + ~EXT4_MOUNT2_##opt +#define set_opt2(sb, opt) EXT4_SB(sb)->s_mount_opt2 |= \ + EXT4_MOUNT2_##opt +#define test_opt2(sb, opt) (EXT4_SB(sb)->s_mount_opt2 & \ + EXT4_MOUNT2_##opt) + +#define ext4_test_and_set_bit __test_and_set_bit_le +#define ext4_set_bit __set_bit_le +#define ext4_test_and_clear_bit __test_and_clear_bit_le +#define ext4_clear_bit __clear_bit_le +#define ext4_test_bit test_bit_le +#define ext4_find_next_zero_bit find_next_zero_bit_le +#define ext4_find_next_bit find_next_bit_le + +extern void mb_set_bits(void *bm, int cur, int len); + +/* + * Maximal mount counts between two filesystem checks + */ +#define EXT4_DFL_MAX_MNT_COUNT 20 /* Allow 20 mounts */ +#define EXT4_DFL_CHECKINTERVAL 0 /* Don't use interval check */ + +/* + * Behaviour when detecting errors + */ +#define EXT4_ERRORS_CONTINUE 1 /* Continue execution */ +#define EXT4_ERRORS_RO 2 /* Remount fs read-only */ +#define EXT4_ERRORS_PANIC 3 /* Panic */ +#define EXT4_ERRORS_DEFAULT EXT4_ERRORS_CONTINUE + +/* Metadata checksum algorithm codes */ +#define EXT4_CRC32C_CHKSUM 1 + +#define EXT4_LABEL_MAX 16 + +/* + * Structure of the super block + */ +struct ext4_super_block { +/*00*/ __le32 s_inodes_count; /* Inodes count */ + __le32 s_blocks_count_lo; /* Blocks count */ + __le32 s_r_blocks_count_lo; /* Reserved blocks count */ + __le32 s_free_blocks_count_lo; /* Free blocks count */ +/*10*/ __le32 s_free_inodes_count; /* Free inodes count */ + __le32 s_first_data_block; /* First Data Block */ + __le32 s_log_block_size; /* Block size */ + __le32 s_log_cluster_size; /* Allocation cluster size */ +/*20*/ __le32 s_blocks_per_group; /* # Blocks per group */ + __le32 s_clusters_per_group; /* # Clusters per group */ + __le32 s_inodes_per_group; /* # Inodes per group */ + __le32 s_mtime; /* Mount time */ +/*30*/ __le32 s_wtime; /* Write time */ + __le16 s_mnt_count; /* Mount count */ + __le16 s_max_mnt_count; /* Maximal mount count */ + __le16 s_magic; /* Magic signature */ + __le16 s_state; /* File system state */ + __le16 s_errors; /* Behaviour when detecting errors */ + __le16 s_minor_rev_level; /* minor revision level */ +/*40*/ __le32 s_lastcheck; /* time of last check */ + __le32 s_checkinterval; /* max. time between checks */ + __le32 s_creator_os; /* OS */ + __le32 s_rev_level; /* Revision level */ +/*50*/ __le16 s_def_resuid; /* Default uid for reserved blocks */ + __le16 s_def_resgid; /* Default gid for reserved blocks */ + /* + * These fields are for EXT4_DYNAMIC_REV superblocks only. + * + * Note: the difference between the compatible feature set and + * the incompatible feature set is that if there is a bit set + * in the incompatible feature set that the kernel doesn't + * know about, it should refuse to mount the filesystem. + * + * e2fsck's requirements are more strict; if it doesn't know + * about a feature in either the compatible or incompatible + * feature set, it must abort and not try to meddle with + * things it doesn't understand... + */ + __le32 s_first_ino; /* First non-reserved inode */ + __le16 s_inode_size; /* size of inode structure */ + __le16 s_block_group_nr; /* block group # of this superblock */ + __le32 s_feature_compat; /* compatible feature set */ +/*60*/ __le32 s_feature_incompat; /* incompatible feature set */ + __le32 s_feature_ro_compat; /* readonly-compatible feature set */ +/*68*/ __u8 s_uuid[16]; /* 128-bit uuid for volume */ +/*78*/ char s_volume_name[EXT4_LABEL_MAX] __nonstring; /* volume name */ +/*88*/ char s_last_mounted[64] __nonstring; /* directory where last mounted */ +/*C8*/ __le32 s_algorithm_usage_bitmap; /* For compression */ + /* + * Performance hints. Directory preallocation should only + * happen if the EXT4_FEATURE_COMPAT_DIR_PREALLOC flag is on. + */ + __u8 s_prealloc_blocks; /* Nr of blocks to try to preallocate*/ + __u8 s_prealloc_dir_blocks; /* Nr to preallocate for dirs */ + __le16 s_reserved_gdt_blocks; /* Per group desc for online growth */ + /* + * Journaling support valid if EXT4_FEATURE_COMPAT_HAS_JOURNAL set. + */ +/*D0*/ __u8 s_journal_uuid[16]; /* uuid of journal superblock */ +/*E0*/ __le32 s_journal_inum; /* inode number of journal file */ + __le32 s_journal_dev; /* device number of journal file */ + __le32 s_last_orphan; /* start of list of inodes to delete */ + __le32 s_hash_seed[4]; /* HTREE hash seed */ + __u8 s_def_hash_version; /* Default hash version to use */ + __u8 s_jnl_backup_type; + __le16 s_desc_size; /* size of group descriptor */ +/*100*/ __le32 s_default_mount_opts; + __le32 s_first_meta_bg; /* First metablock block group */ + __le32 s_mkfs_time; /* When the filesystem was created */ + __le32 s_jnl_blocks[17]; /* Backup of the journal inode */ + /* 64bit support valid if EXT4_FEATURE_INCOMPAT_64BIT */ +/*150*/ __le32 s_blocks_count_hi; /* Blocks count */ + __le32 s_r_blocks_count_hi; /* Reserved blocks count */ + __le32 s_free_blocks_count_hi; /* Free blocks count */ + __le16 s_min_extra_isize; /* All inodes have at least # bytes */ + __le16 s_want_extra_isize; /* New inodes should reserve # bytes */ + __le32 s_flags; /* Miscellaneous flags */ + __le16 s_raid_stride; /* RAID stride */ + __le16 s_mmp_update_interval; /* # seconds to wait in MMP checking */ + __le64 s_mmp_block; /* Block for multi-mount protection */ + __le32 s_raid_stripe_width; /* blocks on all data disks (N*stride)*/ + __u8 s_log_groups_per_flex; /* FLEX_BG group size */ + __u8 s_checksum_type; /* metadata checksum algorithm used */ + __u8 s_encryption_level; /* versioning level for encryption */ + __u8 s_reserved_pad; /* Padding to next 32bits */ + __le64 s_kbytes_written; /* nr of lifetime kilobytes written */ + __le32 s_snapshot_inum; /* Inode number of active snapshot */ + __le32 s_snapshot_id; /* sequential ID of active snapshot */ + __le64 s_snapshot_r_blocks_count; /* reserved blocks for active + snapshot's future use */ + __le32 s_snapshot_list; /* inode number of the head of the + on-disk snapshot list */ +#define EXT4_S_ERR_START offsetof(struct ext4_super_block, s_error_count) + __le32 s_error_count; /* number of fs errors */ + __le32 s_first_error_time; /* first time an error happened */ + __le32 s_first_error_ino; /* inode involved in first error */ + __le64 s_first_error_block; /* block involved of first error */ + __u8 s_first_error_func[32] __nonstring; /* function where the error happened */ + __le32 s_first_error_line; /* line number where error happened */ + __le32 s_last_error_time; /* most recent time of an error */ + __le32 s_last_error_ino; /* inode involved in last error */ + __le32 s_last_error_line; /* line number where error happened */ + __le64 s_last_error_block; /* block involved of last error */ + __u8 s_last_error_func[32] __nonstring; /* function where the error happened */ +#define EXT4_S_ERR_END offsetof(struct ext4_super_block, s_mount_opts) + __u8 s_mount_opts[64]; + __le32 s_usr_quota_inum; /* inode for tracking user quota */ + __le32 s_grp_quota_inum; /* inode for tracking group quota */ + __le32 s_overhead_clusters; /* overhead blocks/clusters in fs */ + __le32 s_backup_bgs[2]; /* groups with sparse_super2 SBs */ + __u8 s_encrypt_algos[4]; /* Encryption algorithms in use */ + __u8 s_encrypt_pw_salt[16]; /* Salt used for string2key algorithm */ + __le32 s_lpf_ino; /* Location of the lost+found inode */ + __le32 s_prj_quota_inum; /* inode for tracking project quota */ + __le32 s_checksum_seed; /* crc32c(uuid) if csum_seed set */ + __u8 s_wtime_hi; + __u8 s_mtime_hi; + __u8 s_mkfs_time_hi; + __u8 s_lastcheck_hi; + __u8 s_first_error_time_hi; + __u8 s_last_error_time_hi; + __u8 s_first_error_errcode; + __u8 s_last_error_errcode; + __le16 s_encoding; /* Filename charset encoding */ + __le16 s_encoding_flags; /* Filename charset encoding flags */ + __le32 s_orphan_file_inum; /* Inode for tracking orphan inodes */ + __le16 s_def_resuid_hi; + __le16 s_def_resgid_hi; + __le32 s_reserved[93]; /* Padding to the end of the block */ + __le32 s_checksum; /* crc32c(superblock) */ +}; + +#define EXT4_S_ERR_LEN (EXT4_S_ERR_END - EXT4_S_ERR_START) + +#ifdef __KERNEL__ + +/* Number of quota types we support */ +#define EXT4_MAXQUOTAS 3 + +#define EXT4_ENC_UTF8_12_1 1 + +/* Types of ext4 journal triggers */ +enum ext4_journal_trigger_type { + EXT4_JTR_ORPHAN_FILE, + EXT4_JTR_NONE /* This must be the last entry for indexing to work! */ +}; + +#define EXT4_JOURNAL_TRIGGER_COUNT EXT4_JTR_NONE + +struct ext4_journal_trigger { + struct jbd2_buffer_trigger_type tr_triggers; + struct super_block *sb; +}; + +static inline struct ext4_journal_trigger *EXT4_TRIGGER( + struct jbd2_buffer_trigger_type *trigger) +{ + return container_of(trigger, struct ext4_journal_trigger, tr_triggers); +} + +#define EXT4_ORPHAN_BLOCK_MAGIC 0x0b10ca04 + +/* Structure at the tail of orphan block */ +struct ext4_orphan_block_tail { + __le32 ob_magic; + __le32 ob_checksum; +}; + +static inline int ext4_inodes_per_orphan_block(struct super_block *sb) +{ + return (sb->s_blocksize - sizeof(struct ext4_orphan_block_tail)) / + sizeof(u32); +} + +struct ext4_orphan_block { + atomic_t ob_free_entries; /* Number of free orphan entries in block */ + struct buffer_head *ob_bh; /* Buffer for orphan block */ +}; + +/* + * Info about orphan file. + */ +struct ext4_orphan_info { + int of_blocks; /* Number of orphan blocks in a file */ + __u32 of_csum_seed; /* Checksum seed for orphan file */ + struct ext4_orphan_block *of_binfo; /* Array with info about orphan + * file blocks */ +}; + +/* + * Ext4 fast commit snapshot statistics. + * + * These are best-effort counters intended for debugging / performance + * introspection; they are not exact under concurrent updates. + */ +struct ext4_fc_snap_stats { + atomic64_t lock_updates_ns_total; + atomic64_t lock_updates_ns_max; + atomic64_t lock_updates_samples; + + atomic64_t snap_inodes; + atomic64_t snap_ranges; + + atomic64_t snap_fail_es_miss; + atomic64_t snap_fail_es_delayed; + atomic64_t snap_fail_es_other; + + atomic64_t snap_fail_inodes_cap; + atomic64_t snap_fail_ranges_cap; + atomic64_t snap_fail_nomem; + atomic64_t snap_fail_inode_loc; + + /* + * Missing inode snapshots during log writing should never happen. + * Keep this counter to help catch unexpected regressions. + */ + atomic64_t snap_fail_no_snap; +}; + +/* + * fourth extended-fs super-block data in memory + */ +struct ext4_sb_info { + unsigned long s_desc_size; /* Size of a group descriptor in bytes */ + unsigned long s_inodes_per_block;/* Number of inodes per block */ + unsigned long s_blocks_per_group;/* Number of blocks in a group */ + unsigned long s_clusters_per_group; /* Number of clusters in a group */ + unsigned long s_inodes_per_group;/* Number of inodes in a group */ + unsigned long s_itb_per_group; /* Number of inode table blocks per group */ + unsigned long s_gdb_count; /* Number of group descriptor blocks */ + unsigned long s_desc_per_block; /* Number of group descriptors per block */ + ext4_group_t s_groups_count; /* Number of groups in the fs */ + ext4_group_t s_blockfile_groups;/* Groups acceptable for non-extent files */ + unsigned long s_overhead; /* # of fs overhead clusters */ + unsigned int s_cluster_ratio; /* Number of blocks per cluster */ + unsigned int s_cluster_bits; /* log2 of s_cluster_ratio */ + loff_t s_bitmap_maxbytes; /* max bytes for bitmap files */ + struct buffer_head * s_sbh; /* Buffer containing the super block */ + struct ext4_super_block *s_es; /* Pointer to the super block in the buffer */ + /* Array of bh's for the block group descriptors */ + struct buffer_head * __rcu *s_group_desc; + unsigned int s_mount_opt; + unsigned int s_mount_opt2; + unsigned long s_mount_flags; + unsigned int s_def_mount_opt; + unsigned int s_def_mount_opt2; + ext4_fsblk_t s_sb_block; + atomic64_t s_resv_clusters; + kuid_t s_resuid; + kgid_t s_resgid; + unsigned short s_mount_state; + unsigned short s_pad; + int s_addr_per_block_bits; + int s_desc_per_block_bits; + int s_inode_size; + int s_first_ino; + unsigned int s_inode_readahead_blks; + unsigned int s_inode_goal; + u32 s_hash_seed[4]; + int s_def_hash_version; + int s_hash_unsigned; /* 3 if hash should be unsigned, 0 if not */ + struct percpu_counter s_freeclusters_counter; + struct percpu_counter s_freeinodes_counter; + struct percpu_counter s_dirs_counter; + struct percpu_counter s_dirtyclusters_counter; + struct percpu_counter s_sra_exceeded_retry_limit; + struct blockgroup_lock *s_blockgroup_lock; + struct proc_dir_entry *s_proc; + struct kobject s_kobj; + struct completion s_kobj_unregister; + struct mutex s_error_notify_mutex; /* protects sysfs_notify vs kobject_del */ + struct super_block *s_sb; + struct buffer_head *s_mmp_bh; + + /* Journaling */ + struct journal_s *s_journal; + unsigned long s_ext4_flags; /* Ext4 superblock flags */ + struct mutex s_orphan_lock; /* Protects on disk list changes */ + struct list_head s_orphan; /* List of orphaned inodes in on disk + list */ + struct ext4_orphan_info s_orphan_info; + unsigned long s_commit_interval; + u32 s_max_batch_time; + u32 s_min_batch_time; + struct file *s_journal_bdev_file; +#ifdef CONFIG_QUOTA + /* Names of quota files with journalled quota */ + char __rcu *s_qf_names[EXT4_MAXQUOTAS]; + int s_jquota_fmt; /* Format of quota to use */ +#endif + unsigned int s_want_extra_isize; /* New inodes should reserve # bytes */ + struct ext4_system_blocks __rcu *s_system_blks; + +#ifdef EXTENTS_STATS + /* ext4 extents stats */ + unsigned long s_ext_min; + unsigned long s_ext_max; + unsigned long s_depth_max; + spinlock_t s_ext_stats_lock; + unsigned long s_ext_blocks; + unsigned long s_ext_extents; +#endif + + /* for buddy allocator */ + struct ext4_group_info ** __rcu *s_group_info; + struct inode *s_buddy_cache; + spinlock_t s_md_lock; + unsigned short *s_mb_offsets; + unsigned int *s_mb_maxs; + unsigned int s_group_info_size; + atomic_t s_mb_free_pending; + struct list_head s_freed_data_list[2]; /* List of blocks to be freed + after commit completed */ + struct list_head s_discard_list; + struct work_struct s_discard_work; + atomic_t s_retry_alloc_pending; + struct xarray *s_mb_avg_fragment_size; + struct xarray *s_mb_largest_free_orders; + + /* tunables */ + unsigned long s_stripe; + unsigned int s_mb_max_linear_groups; + unsigned int s_mb_stream_request; + unsigned int s_mb_max_to_scan; + unsigned int s_mb_min_to_scan; + unsigned int s_mb_stats; + unsigned int s_mb_order2_reqs; + unsigned int s_mb_group_prealloc; + unsigned int s_max_dir_size_kb; + unsigned int s_mb_prefetch; + unsigned int s_mb_prefetch_limit; + unsigned int s_mb_best_avail_max_trim_order; + unsigned int s_sb_update_sec; + unsigned int s_sb_update_kb; + + /* where last allocation was done - for stream allocation */ + ext4_group_t *s_mb_last_groups; + unsigned int s_mb_nr_global_goals; + + /* stats for buddy allocator */ + atomic_t s_bal_reqs; /* number of reqs with len > 1 */ + atomic_t s_bal_success; /* we found long enough chunks */ + atomic_t s_bal_allocated; /* in blocks */ + atomic_t s_bal_ex_scanned; /* total extents scanned */ + atomic_t s_bal_cX_ex_scanned[EXT4_MB_NUM_CRS]; /* total extents scanned */ + atomic_t s_bal_groups_scanned; /* number of groups scanned */ + atomic_t s_bal_goals; /* goal hits */ + atomic_t s_bal_stream_goals; /* stream allocation global goal hits */ + atomic_t s_bal_len_goals; /* len goal hits */ + atomic_t s_bal_breaks; /* too long searches */ + atomic_t s_bal_2orders; /* 2^order hits */ + atomic64_t s_bal_cX_groups_considered[EXT4_MB_NUM_CRS]; + atomic64_t s_bal_cX_hits[EXT4_MB_NUM_CRS]; + atomic64_t s_bal_cX_failed[EXT4_MB_NUM_CRS]; /* cX loop didn't find blocks */ + atomic_t s_mb_buddies_generated; /* number of buddies generated */ + atomic64_t s_mb_generation_time; + atomic_t s_mb_lost_chunks; + atomic_t s_mb_preallocated; + atomic_t s_mb_discarded; + atomic_t s_lock_busy; + + /* locality groups */ + struct ext4_locality_group __percpu *s_locality_groups; + + /* for write statistics */ + unsigned long s_sectors_written_start; + u64 s_kbytes_written; + + /* the size of zero-out chunk */ + unsigned int s_extent_max_zeroout_kb; + + unsigned int s_log_groups_per_flex; + struct flex_groups * __rcu *s_flex_groups; + ext4_group_t s_flex_groups_allocated; + + /* workqueue for reserved extent conversions (buffered io) */ + struct workqueue_struct *rsv_conversion_wq; + + /* timer for periodic error stats printing */ + struct timer_list s_err_report; + /* timeout in seconds for s_err_report; 0 disables the timer. */ + unsigned long s_err_report_sec; + + /* Lazy inode table initialization info */ + struct ext4_li_request *s_li_request; + /* Wait multiplier for lazy initialization thread */ + unsigned int s_li_wait_mult; + + /* Kernel thread for multiple mount protection */ + struct task_struct *s_mmp_tsk; + + /* record the last minlen when FITRIM is called. */ + unsigned long s_last_trim_minblks; + + /* minimum folio order of a page cache allocation */ + u16 s_min_folio_order; + /* supported maximum folio order, 0 means not supported */ + u16 s_max_folio_order; + + /* Precomputed FS UUID checksum for seeding other checksums */ + __u32 s_csum_seed; + + /* Reclaim extents from extent status tree */ + struct shrinker *s_es_shrinker; + struct list_head s_es_list; /* List of inodes with reclaimable extents */ + long s_es_nr_inode; + struct ext4_es_stats s_es_stats; + struct mb_cache *s_ea_block_cache; + struct mb_cache *s_ea_inode_cache; + + /* Deferred iput for EA inodes to avoid lock ordering issues */ + struct llist_head s_ea_inode_to_free; + struct delayed_work s_ea_inode_work; + + spinlock_t s_es_lock ____cacheline_aligned_in_smp; + + /* Journal triggers for checksum computation */ + struct ext4_journal_trigger s_journal_triggers[EXT4_JOURNAL_TRIGGER_COUNT]; + + /* Ratelimit ext4 messages. */ + struct ratelimit_state s_err_ratelimit_state; + struct ratelimit_state s_warning_ratelimit_state; + struct ratelimit_state s_msg_ratelimit_state; + atomic_t s_warning_count; + atomic_t s_msg_count; + + /* Encryption policy for '-o test_dummy_encryption' */ + struct fscrypt_dummy_policy s_dummy_enc_policy; + + /* + * Barrier between writepages ops and changing any inode's JOURNAL_DATA + * or EXTENTS flag or between writepages ops and changing DELALLOC or + * DIOREAD_NOLOCK mount options on remount. + */ + struct percpu_rw_semaphore s_writepages_rwsem; + struct dax_device *s_daxdev; + u64 s_dax_part_off; +#ifdef CONFIG_EXT4_DEBUG + unsigned long s_simulate_fail; +#endif + /* Record the errseq of the backing block device */ + errseq_t s_bdev_wb_err; + spinlock_t s_bdev_wb_lock; + + /* Information about errors that happened during this mount */ + spinlock_t s_error_lock; + int s_add_error_count; + int s_first_error_code; + __u32 s_first_error_line; + __u32 s_first_error_ino; + __u64 s_first_error_block; + const char *s_first_error_func; + time64_t s_first_error_time; + int s_last_error_code; + __u32 s_last_error_line; + __u32 s_last_error_ino; + __u64 s_last_error_block; + const char *s_last_error_func; + time64_t s_last_error_time; + /* + * If we are in a context where we cannot update the on-disk + * superblock, we queue the work here. This is used to update + * the error information in the superblock, and for periodic + * updates of the superblock called from the commit callback + * function. + */ + struct work_struct s_sb_upd_work; + + /* Atomic write unit values in bytes */ + unsigned int s_awu_min; + unsigned int s_awu_max; + + /* Ext4 fast commit sub transaction ID */ + atomic_t s_fc_subtid; + + /* + * After commit starts, the main queue gets locked, and the further + * updates get added in the staging queue. + */ +#define FC_Q_MAIN 0 +#define FC_Q_STAGING 1 + struct list_head s_fc_q[2]; /* Inodes staged for fast commit + * that have data changes in them. + */ + struct list_head s_fc_dentry_q[2]; /* directory entry updates */ + unsigned int s_fc_bytes; + /* + * Main fast commit lock. This lock protects accesses to the + * following fields: + * ei->i_fc_list, s_fc_dentry_q, s_fc_q, s_fc_bytes, s_fc_bh. + * + * s_fc_lock can be taken from reclaim context (inode eviction) and is + * thus reclaim unsafe. Use ext4_fc_lock()/ext4_fc_unlock() helpers + * when acquiring / releasing the lock. + */ + struct mutex s_fc_lock; + struct buffer_head *s_fc_bh; + struct ext4_fc_stats s_fc_stats; + struct ext4_fc_snap_stats s_fc_snap_stats; + tid_t s_fc_ineligible_tid; +#ifdef CONFIG_EXT4_DEBUG + int s_fc_debug_max_replay; +#endif + struct ext4_fc_replay_state s_fc_replay_state; +}; + +static inline struct ext4_sb_info *EXT4_SB(struct super_block *sb) +{ + return sb->s_fs_info; +} +static inline struct ext4_inode_info *EXT4_I(struct inode *inode) +{ + return container_of(inode, struct ext4_inode_info, vfs_inode); +} + +static inline int ext4_writepages_down_read(struct super_block *sb) +{ + percpu_down_read(&EXT4_SB(sb)->s_writepages_rwsem); + return memalloc_nofs_save(); +} + +static inline void ext4_writepages_up_read(struct super_block *sb, int ctx) +{ + memalloc_nofs_restore(ctx); + percpu_up_read(&EXT4_SB(sb)->s_writepages_rwsem); +} + +static inline int ext4_writepages_down_write(struct super_block *sb) +{ + percpu_down_write(&EXT4_SB(sb)->s_writepages_rwsem); + return memalloc_nofs_save(); +} + +static inline void ext4_writepages_up_write(struct super_block *sb, int ctx) +{ + memalloc_nofs_restore(ctx); + percpu_up_write(&EXT4_SB(sb)->s_writepages_rwsem); +} + +static inline int ext4_fc_lock(struct super_block *sb) +{ + mutex_lock(&EXT4_SB(sb)->s_fc_lock); + return memalloc_nofs_save(); +} + +static inline void ext4_fc_unlock(struct super_block *sb, int ctx) +{ + memalloc_nofs_restore(ctx); + mutex_unlock(&EXT4_SB(sb)->s_fc_lock); +} + +static inline int ext4_valid_inum(struct super_block *sb, unsigned long ino) +{ + return ino == EXT4_ROOT_INO || + (ino >= EXT4_FIRST_INO(sb) && + ino <= le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count)); +} + +static inline int ext4_get_resuid(struct ext4_super_block *es) +{ + return le16_to_cpu(es->s_def_resuid) | + le16_to_cpu(es->s_def_resuid_hi) << 16; +} + +static inline int ext4_get_resgid(struct ext4_super_block *es) +{ + return le16_to_cpu(es->s_def_resgid) | + le16_to_cpu(es->s_def_resgid_hi) << 16; +} + +/* + * Returns: sbi->field[index] + * Used to access an array element from the following sbi fields which require + * rcu protection to avoid dereferencing an invalid pointer due to reassignment + * - s_group_desc + * - s_group_info + * - s_flex_group + */ +#define sbi_array_rcu_deref(sbi, field, index) \ +({ \ + typeof(*((sbi)->field)) _v; \ + rcu_read_lock(); \ + _v = ((typeof(_v)*)rcu_dereference((sbi)->field))[index]; \ + rcu_read_unlock(); \ + _v; \ +}) + +/* + * run-time mount flags + */ +enum { + EXT4_MF_MNTDIR_SAMPLED, + EXT4_MF_FC_INELIGIBLE, /* Fast commit ineligible */ + EXT4_MF_JOURNAL_DESTROY /* Journal is in process of destroying */ +}; + +static inline void ext4_set_mount_flag(struct super_block *sb, int bit) +{ + set_bit(bit, &EXT4_SB(sb)->s_mount_flags); +} + +static inline void ext4_clear_mount_flag(struct super_block *sb, int bit) +{ + clear_bit(bit, &EXT4_SB(sb)->s_mount_flags); +} + +static inline int ext4_test_mount_flag(struct super_block *sb, int bit) +{ + return test_bit(bit, &EXT4_SB(sb)->s_mount_flags); +} + + +/* + * Simulate_fail codes + */ +#define EXT4_SIM_BBITMAP_EIO 1 +#define EXT4_SIM_BBITMAP_CRC 2 +#define EXT4_SIM_IBITMAP_EIO 3 +#define EXT4_SIM_IBITMAP_CRC 4 +#define EXT4_SIM_INODE_EIO 5 +#define EXT4_SIM_INODE_CRC 6 +#define EXT4_SIM_DIRBLOCK_EIO 7 +#define EXT4_SIM_DIRBLOCK_CRC 8 + +static inline bool ext4_simulate_fail(struct super_block *sb, + unsigned long code) +{ +#ifdef CONFIG_EXT4_DEBUG + struct ext4_sb_info *sbi = EXT4_SB(sb); + + if (unlikely(sbi->s_simulate_fail == code)) { + sbi->s_simulate_fail = 0; + return true; + } +#endif + return false; +} + +/* + * Error number codes for s_{first,last}_error_errno + * + * Linux errno numbers are architecture specific, so we need to translate + * them into something which is architecture independent. We don't define + * codes for all errno's; just the ones which are most likely to be the cause + * of an ext4_error() call. + */ +#define EXT4_ERR_UNKNOWN 1 +#define EXT4_ERR_EIO 2 +#define EXT4_ERR_ENOMEM 3 +#define EXT4_ERR_EFSBADCRC 4 +#define EXT4_ERR_EFSCORRUPTED 5 +#define EXT4_ERR_ENOSPC 6 +#define EXT4_ERR_ENOKEY 7 +#define EXT4_ERR_EROFS 8 +#define EXT4_ERR_EFBIG 9 +#define EXT4_ERR_EEXIST 10 +#define EXT4_ERR_ERANGE 11 +#define EXT4_ERR_EOVERFLOW 12 +#define EXT4_ERR_EBUSY 13 +#define EXT4_ERR_ENOTDIR 14 +#define EXT4_ERR_ENOTEMPTY 15 +#define EXT4_ERR_ESHUTDOWN 16 +#define EXT4_ERR_EFAULT 17 + +/* + * Inode dynamic state flags + */ +enum { + EXT4_STATE_NEW, /* inode is newly created */ + EXT4_STATE_XATTR, /* has in-inode xattrs */ + EXT4_STATE_NO_EXPAND, /* No space for expansion */ + EXT4_STATE_DA_ALLOC_CLOSE, /* Alloc DA blks on close */ + EXT4_STATE_EXT_MIGRATE, /* Inode is migrating */ + EXT4_STATE_NEWENTRY, /* File just added to dir */ + EXT4_STATE_MAY_INLINE_DATA, /* may have in-inode data */ + EXT4_STATE_EXT_PRECACHED, /* extents have been precached */ + EXT4_STATE_LUSTRE_EA_INODE, /* Lustre-style ea_inode */ + EXT4_STATE_VERITY_IN_PROGRESS, /* building fs-verity Merkle tree */ + EXT4_STATE_FC_COMMITTING, /* Fast commit ongoing */ + EXT4_STATE_FC_FLUSHING_DATA, /* Fast commit flushing data */ + EXT4_STATE_ORPHAN_FILE, /* Inode orphaned in orphan file */ + EXT4_STATE_FC_REQUEUE, /* Inode modified during fast commit */ +}; + +#define EXT4_INODE_BIT_FNS(name, field, offset) \ +static inline int ext4_test_inode_##name(struct inode *inode, int bit) \ +{ \ + return test_bit(bit + (offset), &EXT4_I(inode)->i_##field); \ +} \ +static inline void ext4_set_inode_##name(struct inode *inode, int bit) \ +{ \ + set_bit(bit + (offset), &EXT4_I(inode)->i_##field); \ +} \ +static inline void ext4_clear_inode_##name(struct inode *inode, int bit) \ +{ \ + clear_bit(bit + (offset), &EXT4_I(inode)->i_##field); \ +} + +/* Add these declarations here only so that these functions can be + * found by name. Otherwise, they are very hard to locate. */ +static inline int ext4_test_inode_flag(struct inode *inode, int bit); +static inline void ext4_set_inode_flag(struct inode *inode, int bit); +static inline void ext4_clear_inode_flag(struct inode *inode, int bit); +EXT4_INODE_BIT_FNS(flag, flags, 0) + +/* Add these declarations here only so that these functions can be + * found by name. Otherwise, they are very hard to locate. */ +static inline int ext4_test_inode_state(struct inode *inode, int bit); +static inline void ext4_set_inode_state(struct inode *inode, int bit); +static inline void ext4_clear_inode_state(struct inode *inode, int bit); +static inline unsigned long *ext4_inode_state_wait_word(struct inode *inode); +static inline int ext4_inode_state_wait_bit(int bit); +#if (BITS_PER_LONG < 64) +EXT4_INODE_BIT_FNS(state, state_flags, 0) + +static inline void ext4_clear_state_flags(struct ext4_inode_info *ei) +{ + (ei)->i_state_flags = 0; +} +#else +EXT4_INODE_BIT_FNS(state, flags, 32) + +static inline void ext4_clear_state_flags(struct ext4_inode_info *ei) +{ + /* We depend on the fact that callers will set i_flags */ +} +#endif + +static inline unsigned long *ext4_inode_state_wait_word(struct inode *inode) +{ +#if (BITS_PER_LONG < 64) + return &EXT4_I(inode)->i_state_flags; +#else + return &EXT4_I(inode)->i_flags; +#endif +} + +static inline int ext4_inode_state_wait_bit(int bit) +{ +#if (BITS_PER_LONG < 64) + return bit; +#else + return bit + 32; +#endif +} +#else +/* Assume that user mode programs are passing in an ext4fs superblock, not + * a kernel struct super_block. This will allow us to call the feature-test + * macros from user land. */ +#define EXT4_SB(sb) (sb) +#endif + +static inline bool ext4_verity_in_progress(struct inode *inode) +{ + return IS_ENABLED(CONFIG_FS_VERITY) && + ext4_test_inode_state(inode, EXT4_STATE_VERITY_IN_PROGRESS); +} + +#define NEXT_ORPHAN(inode) EXT4_I(inode)->i_dtime + +/* + * Check whether the inode is tracked as orphan (either in orphan file or + * orphan list). + */ +static inline bool ext4_inode_orphan_tracked(struct inode *inode) +{ + return ext4_test_inode_state(inode, EXT4_STATE_ORPHAN_FILE) || + !list_empty(&EXT4_I(inode)->i_orphan); +} + +static inline struct mapping_metadata_bhs *ext4_i_metadata_bhs( + struct inode *inode) +{ + /* + * i_metadata_bhs is set in ext4_inode_attach_mmb() using cmpxchg(). + * We use READ_ONCE when accessing i_metadata_bhs to make sure we get + * consistent view for all accesses. + */ + return READ_ONCE(EXT4_I(inode)->i_metadata_bhs); +} + +/* + * Codes for operating systems + */ +#define EXT4_OS_LINUX 0 +#define EXT4_OS_HURD 1 +#define EXT4_OS_MASIX 2 +#define EXT4_OS_FREEBSD 3 +#define EXT4_OS_LITES 4 + +/* + * Revision levels + */ +#define EXT4_GOOD_OLD_REV 0 /* The good old (original) format */ +#define EXT4_DYNAMIC_REV 1 /* V2 format w/ dynamic inode sizes */ + +#define EXT4_MAX_SUPP_REV EXT4_DYNAMIC_REV + +#define EXT4_GOOD_OLD_INODE_SIZE 128 + +#define EXT4_EXTRA_TIMESTAMP_MAX (((s64)1 << 34) - 1 + S32_MIN) +#define EXT4_NON_EXTRA_TIMESTAMP_MAX S32_MAX +#define EXT4_TIMESTAMP_MIN S32_MIN + +/* + * Feature set definitions + */ + +#define EXT4_FEATURE_COMPAT_DIR_PREALLOC 0x0001 +#define EXT4_FEATURE_COMPAT_IMAGIC_INODES 0x0002 +#define EXT4_FEATURE_COMPAT_HAS_JOURNAL 0x0004 +#define EXT4_FEATURE_COMPAT_EXT_ATTR 0x0008 +#define EXT4_FEATURE_COMPAT_RESIZE_INODE 0x0010 +#define EXT4_FEATURE_COMPAT_DIR_INDEX 0x0020 +#define EXT4_FEATURE_COMPAT_SPARSE_SUPER2 0x0200 +/* + * The reason why "FAST_COMMIT" is a compat feature is that, FS becomes + * incompatible only if fast commit blocks are present in the FS. Since we + * clear the journal (and thus the fast commit blocks), we don't mark FS as + * incompatible. We also have a JBD2 incompat feature, which gets set when + * there are fast commit blocks present in the journal. + */ +#define EXT4_FEATURE_COMPAT_FAST_COMMIT 0x0400 +#define EXT4_FEATURE_COMPAT_STABLE_INODES 0x0800 +#define EXT4_FEATURE_COMPAT_ORPHAN_FILE 0x1000 /* Orphan file exists */ + +#define EXT4_FEATURE_RO_COMPAT_SPARSE_SUPER 0x0001 +#define EXT4_FEATURE_RO_COMPAT_LARGE_FILE 0x0002 +#define EXT4_FEATURE_RO_COMPAT_BTREE_DIR 0x0004 +#define EXT4_FEATURE_RO_COMPAT_HUGE_FILE 0x0008 +#define EXT4_FEATURE_RO_COMPAT_GDT_CSUM 0x0010 +#define EXT4_FEATURE_RO_COMPAT_DIR_NLINK 0x0020 +#define EXT4_FEATURE_RO_COMPAT_EXTRA_ISIZE 0x0040 +#define EXT4_FEATURE_RO_COMPAT_QUOTA 0x0100 +#define EXT4_FEATURE_RO_COMPAT_BIGALLOC 0x0200 +/* + * METADATA_CSUM also enables group descriptor checksums (GDT_CSUM). When + * METADATA_CSUM is set, group descriptor checksums use the same algorithm as + * all other data structures' checksums. However, the METADATA_CSUM and + * GDT_CSUM bits are mutually exclusive. + */ +#define EXT4_FEATURE_RO_COMPAT_METADATA_CSUM 0x0400 +#define EXT4_FEATURE_RO_COMPAT_READONLY 0x1000 +#define EXT4_FEATURE_RO_COMPAT_PROJECT 0x2000 +#define EXT4_FEATURE_RO_COMPAT_VERITY 0x8000 +#define EXT4_FEATURE_RO_COMPAT_ORPHAN_PRESENT 0x10000 /* Orphan file may be + non-empty */ + +#define EXT4_FEATURE_INCOMPAT_COMPRESSION 0x0001 +#define EXT4_FEATURE_INCOMPAT_FILETYPE 0x0002 +#define EXT4_FEATURE_INCOMPAT_RECOVER 0x0004 /* Needs recovery */ +#define EXT4_FEATURE_INCOMPAT_JOURNAL_DEV 0x0008 /* Journal device */ +#define EXT4_FEATURE_INCOMPAT_META_BG 0x0010 +#define EXT4_FEATURE_INCOMPAT_EXTENTS 0x0040 /* extents support */ +#define EXT4_FEATURE_INCOMPAT_64BIT 0x0080 +#define EXT4_FEATURE_INCOMPAT_MMP 0x0100 +#define EXT4_FEATURE_INCOMPAT_FLEX_BG 0x0200 +#define EXT4_FEATURE_INCOMPAT_EA_INODE 0x0400 /* EA in inode */ +#define EXT4_FEATURE_INCOMPAT_DIRDATA 0x1000 /* data in dirent */ +#define EXT4_FEATURE_INCOMPAT_CSUM_SEED 0x2000 +#define EXT4_FEATURE_INCOMPAT_LARGEDIR 0x4000 /* >2GB or 3-lvl htree */ +#define EXT4_FEATURE_INCOMPAT_INLINE_DATA 0x8000 /* data in inode */ +#define EXT4_FEATURE_INCOMPAT_ENCRYPT 0x10000 +#define EXT4_FEATURE_INCOMPAT_CASEFOLD 0x20000 + +extern void ext4_update_dynamic_rev(struct super_block *sb); + +#define EXT4_FEATURE_COMPAT_FUNCS(name, flagname) \ +static inline bool ext4_has_feature_##name(struct super_block *sb) \ +{ \ + return ((EXT4_SB(sb)->s_es->s_feature_compat & \ + cpu_to_le32(EXT4_FEATURE_COMPAT_##flagname)) != 0); \ +} \ +static inline void ext4_set_feature_##name(struct super_block *sb) \ +{ \ + ext4_update_dynamic_rev(sb); \ + EXT4_SB(sb)->s_es->s_feature_compat |= \ + cpu_to_le32(EXT4_FEATURE_COMPAT_##flagname); \ +} \ +static inline void ext4_clear_feature_##name(struct super_block *sb) \ +{ \ + EXT4_SB(sb)->s_es->s_feature_compat &= \ + ~cpu_to_le32(EXT4_FEATURE_COMPAT_##flagname); \ +} + +#define EXT4_FEATURE_RO_COMPAT_FUNCS(name, flagname) \ +static inline bool ext4_has_feature_##name(struct super_block *sb) \ +{ \ + return ((EXT4_SB(sb)->s_es->s_feature_ro_compat & \ + cpu_to_le32(EXT4_FEATURE_RO_COMPAT_##flagname)) != 0); \ +} \ +static inline void ext4_set_feature_##name(struct super_block *sb) \ +{ \ + ext4_update_dynamic_rev(sb); \ + EXT4_SB(sb)->s_es->s_feature_ro_compat |= \ + cpu_to_le32(EXT4_FEATURE_RO_COMPAT_##flagname); \ +} \ +static inline void ext4_clear_feature_##name(struct super_block *sb) \ +{ \ + EXT4_SB(sb)->s_es->s_feature_ro_compat &= \ + ~cpu_to_le32(EXT4_FEATURE_RO_COMPAT_##flagname); \ +} + +#define EXT4_FEATURE_INCOMPAT_FUNCS(name, flagname) \ +static inline bool ext4_has_feature_##name(struct super_block *sb) \ +{ \ + return ((EXT4_SB(sb)->s_es->s_feature_incompat & \ + cpu_to_le32(EXT4_FEATURE_INCOMPAT_##flagname)) != 0); \ +} \ +static inline void ext4_set_feature_##name(struct super_block *sb) \ +{ \ + ext4_update_dynamic_rev(sb); \ + EXT4_SB(sb)->s_es->s_feature_incompat |= \ + cpu_to_le32(EXT4_FEATURE_INCOMPAT_##flagname); \ +} \ +static inline void ext4_clear_feature_##name(struct super_block *sb) \ +{ \ + EXT4_SB(sb)->s_es->s_feature_incompat &= \ + ~cpu_to_le32(EXT4_FEATURE_INCOMPAT_##flagname); \ +} + +EXT4_FEATURE_COMPAT_FUNCS(dir_prealloc, DIR_PREALLOC) +EXT4_FEATURE_COMPAT_FUNCS(imagic_inodes, IMAGIC_INODES) +EXT4_FEATURE_COMPAT_FUNCS(journal, HAS_JOURNAL) +EXT4_FEATURE_COMPAT_FUNCS(xattr, EXT_ATTR) +EXT4_FEATURE_COMPAT_FUNCS(resize_inode, RESIZE_INODE) +EXT4_FEATURE_COMPAT_FUNCS(dir_index, DIR_INDEX) +EXT4_FEATURE_COMPAT_FUNCS(sparse_super2, SPARSE_SUPER2) +EXT4_FEATURE_COMPAT_FUNCS(fast_commit, FAST_COMMIT) +EXT4_FEATURE_COMPAT_FUNCS(stable_inodes, STABLE_INODES) +EXT4_FEATURE_COMPAT_FUNCS(orphan_file, ORPHAN_FILE) + +EXT4_FEATURE_RO_COMPAT_FUNCS(sparse_super, SPARSE_SUPER) +EXT4_FEATURE_RO_COMPAT_FUNCS(large_file, LARGE_FILE) +EXT4_FEATURE_RO_COMPAT_FUNCS(btree_dir, BTREE_DIR) +EXT4_FEATURE_RO_COMPAT_FUNCS(huge_file, HUGE_FILE) +EXT4_FEATURE_RO_COMPAT_FUNCS(gdt_csum, GDT_CSUM) +EXT4_FEATURE_RO_COMPAT_FUNCS(dir_nlink, DIR_NLINK) +EXT4_FEATURE_RO_COMPAT_FUNCS(extra_isize, EXTRA_ISIZE) +EXT4_FEATURE_RO_COMPAT_FUNCS(quota, QUOTA) +EXT4_FEATURE_RO_COMPAT_FUNCS(bigalloc, BIGALLOC) +EXT4_FEATURE_RO_COMPAT_FUNCS(metadata_csum, METADATA_CSUM) +EXT4_FEATURE_RO_COMPAT_FUNCS(readonly, READONLY) +EXT4_FEATURE_RO_COMPAT_FUNCS(project, PROJECT) +EXT4_FEATURE_RO_COMPAT_FUNCS(verity, VERITY) +EXT4_FEATURE_RO_COMPAT_FUNCS(orphan_present, ORPHAN_PRESENT) + +EXT4_FEATURE_INCOMPAT_FUNCS(compression, COMPRESSION) +EXT4_FEATURE_INCOMPAT_FUNCS(filetype, FILETYPE) +EXT4_FEATURE_INCOMPAT_FUNCS(journal_needs_recovery, RECOVER) +EXT4_FEATURE_INCOMPAT_FUNCS(journal_dev, JOURNAL_DEV) +EXT4_FEATURE_INCOMPAT_FUNCS(meta_bg, META_BG) +EXT4_FEATURE_INCOMPAT_FUNCS(extents, EXTENTS) +EXT4_FEATURE_INCOMPAT_FUNCS(64bit, 64BIT) +EXT4_FEATURE_INCOMPAT_FUNCS(mmp, MMP) +EXT4_FEATURE_INCOMPAT_FUNCS(flex_bg, FLEX_BG) +EXT4_FEATURE_INCOMPAT_FUNCS(ea_inode, EA_INODE) +EXT4_FEATURE_INCOMPAT_FUNCS(dirdata, DIRDATA) +EXT4_FEATURE_INCOMPAT_FUNCS(csum_seed, CSUM_SEED) +EXT4_FEATURE_INCOMPAT_FUNCS(largedir, LARGEDIR) +EXT4_FEATURE_INCOMPAT_FUNCS(inline_data, INLINE_DATA) +EXT4_FEATURE_INCOMPAT_FUNCS(encrypt, ENCRYPT) +EXT4_FEATURE_INCOMPAT_FUNCS(casefold, CASEFOLD) + +#define EXT2_FEATURE_COMPAT_SUPP EXT4_FEATURE_COMPAT_EXT_ATTR +#define EXT2_FEATURE_INCOMPAT_SUPP (EXT4_FEATURE_INCOMPAT_FILETYPE| \ + EXT4_FEATURE_INCOMPAT_META_BG) +#define EXT2_FEATURE_RO_COMPAT_SUPP (EXT4_FEATURE_RO_COMPAT_SPARSE_SUPER| \ + EXT4_FEATURE_RO_COMPAT_LARGE_FILE| \ + EXT4_FEATURE_RO_COMPAT_BTREE_DIR) + +#define EXT3_FEATURE_COMPAT_SUPP EXT4_FEATURE_COMPAT_EXT_ATTR +#define EXT3_FEATURE_INCOMPAT_SUPP (EXT4_FEATURE_INCOMPAT_FILETYPE| \ + EXT4_FEATURE_INCOMPAT_RECOVER| \ + EXT4_FEATURE_INCOMPAT_META_BG) +#define EXT3_FEATURE_RO_COMPAT_SUPP (EXT4_FEATURE_RO_COMPAT_SPARSE_SUPER| \ + EXT4_FEATURE_RO_COMPAT_LARGE_FILE| \ + EXT4_FEATURE_RO_COMPAT_BTREE_DIR) + +#define EXT4_FEATURE_COMPAT_SUPP (EXT4_FEATURE_COMPAT_EXT_ATTR| \ + EXT4_FEATURE_COMPAT_ORPHAN_FILE) +#define EXT4_FEATURE_INCOMPAT_SUPP (EXT4_FEATURE_INCOMPAT_FILETYPE| \ + EXT4_FEATURE_INCOMPAT_RECOVER| \ + EXT4_FEATURE_INCOMPAT_META_BG| \ + EXT4_FEATURE_INCOMPAT_EXTENTS| \ + EXT4_FEATURE_INCOMPAT_64BIT| \ + EXT4_FEATURE_INCOMPAT_FLEX_BG| \ + EXT4_FEATURE_INCOMPAT_EA_INODE| \ + EXT4_FEATURE_INCOMPAT_MMP | \ + EXT4_FEATURE_INCOMPAT_INLINE_DATA | \ + EXT4_FEATURE_INCOMPAT_ENCRYPT | \ + EXT4_FEATURE_INCOMPAT_CASEFOLD | \ + EXT4_FEATURE_INCOMPAT_CSUM_SEED | \ + EXT4_FEATURE_INCOMPAT_LARGEDIR) +#define EXT4_FEATURE_RO_COMPAT_SUPP (EXT4_FEATURE_RO_COMPAT_SPARSE_SUPER| \ + EXT4_FEATURE_RO_COMPAT_LARGE_FILE| \ + EXT4_FEATURE_RO_COMPAT_GDT_CSUM| \ + EXT4_FEATURE_RO_COMPAT_DIR_NLINK | \ + EXT4_FEATURE_RO_COMPAT_EXTRA_ISIZE | \ + EXT4_FEATURE_RO_COMPAT_BTREE_DIR |\ + EXT4_FEATURE_RO_COMPAT_HUGE_FILE |\ + EXT4_FEATURE_RO_COMPAT_BIGALLOC |\ + EXT4_FEATURE_RO_COMPAT_METADATA_CSUM|\ + EXT4_FEATURE_RO_COMPAT_QUOTA |\ + EXT4_FEATURE_RO_COMPAT_PROJECT |\ + EXT4_FEATURE_RO_COMPAT_VERITY |\ + EXT4_FEATURE_RO_COMPAT_ORPHAN_PRESENT) + +#define EXTN_FEATURE_FUNCS(ver) \ +static inline bool ext4_has_unknown_ext##ver##_compat_features(struct super_block *sb) \ +{ \ + return ((EXT4_SB(sb)->s_es->s_feature_compat & \ + cpu_to_le32(~EXT##ver##_FEATURE_COMPAT_SUPP)) != 0); \ +} \ +static inline bool ext4_has_unknown_ext##ver##_ro_compat_features(struct super_block *sb) \ +{ \ + return ((EXT4_SB(sb)->s_es->s_feature_ro_compat & \ + cpu_to_le32(~EXT##ver##_FEATURE_RO_COMPAT_SUPP)) != 0); \ +} \ +static inline bool ext4_has_unknown_ext##ver##_incompat_features(struct super_block *sb) \ +{ \ + return ((EXT4_SB(sb)->s_es->s_feature_incompat & \ + cpu_to_le32(~EXT##ver##_FEATURE_INCOMPAT_SUPP)) != 0); \ +} + +EXTN_FEATURE_FUNCS(2) +EXTN_FEATURE_FUNCS(3) +EXTN_FEATURE_FUNCS(4) + +static inline bool ext4_has_compat_features(struct super_block *sb) +{ + return (EXT4_SB(sb)->s_es->s_feature_compat != 0); +} +static inline bool ext4_has_ro_compat_features(struct super_block *sb) +{ + return (EXT4_SB(sb)->s_es->s_feature_ro_compat != 0); +} +static inline bool ext4_has_incompat_features(struct super_block *sb) +{ + return (EXT4_SB(sb)->s_es->s_feature_incompat != 0); +} + +extern int ext4_feature_set_ok(struct super_block *sb, int readonly); + +/* + * Superblock flags + */ +enum { + EXT4_FLAGS_RESIZING, /* Avoid superblock update and resize race */ + EXT4_FLAGS_SHUTDOWN, /* Prevent access to the file system */ + EXT4_FLAGS_BDEV_IS_DAX, /* Current block device support DAX */ + EXT4_FLAGS_EMERGENCY_RO,/* Emergency read-only due to fs errors */ +}; + +static inline int ext4_forced_shutdown(struct super_block *sb) +{ + return test_bit(EXT4_FLAGS_SHUTDOWN, &EXT4_SB(sb)->s_ext4_flags); +} + +static inline int ext4_emergency_ro(struct super_block *sb) +{ + return test_bit(EXT4_FLAGS_EMERGENCY_RO, &EXT4_SB(sb)->s_ext4_flags); +} + +static inline int ext4_emergency_state(struct super_block *sb) +{ + if (unlikely(ext4_forced_shutdown(sb))) + return -EIO; + if (unlikely(ext4_emergency_ro(sb))) + return -EROFS; + return 0; +} + +/* + * Default values for user and/or group using reserved blocks + */ +#define EXT4_DEF_RESUID 0 +#define EXT4_DEF_RESGID 0 + +/* + * Default project ID + */ +#define EXT4_DEF_PROJID 0 + +#define EXT4_DEF_INODE_READAHEAD_BLKS 32 + +/* + * Default mount options + */ +#define EXT4_DEFM_DEBUG 0x0001 +#define EXT4_DEFM_BSDGROUPS 0x0002 +#define EXT4_DEFM_XATTR_USER 0x0004 +#define EXT4_DEFM_ACL 0x0008 +#define EXT4_DEFM_UID16 0x0010 +#define EXT4_DEFM_JMODE 0x0060 +#define EXT4_DEFM_JMODE_DATA 0x0020 +#define EXT4_DEFM_JMODE_ORDERED 0x0040 +#define EXT4_DEFM_JMODE_WBACK 0x0060 +#define EXT4_DEFM_NOBARRIER 0x0100 +#define EXT4_DEFM_BLOCK_VALIDITY 0x0200 +#define EXT4_DEFM_DISCARD 0x0400 +#define EXT4_DEFM_NODELALLOC 0x0800 + +/* + * Default journal batch times and ioprio. + */ +#define EXT4_DEF_MIN_BATCH_TIME 0 +#define EXT4_DEF_MAX_BATCH_TIME 15000 /* 15ms */ +#define EXT4_DEF_JOURNAL_IOPRIO (IOPRIO_PRIO_VALUE(IOPRIO_CLASS_BE, 3)) + + +/* + * Default values for superblock update + */ +#define EXT4_DEF_SB_UPDATE_INTERVAL_SEC (3600) /* seconds (1 hour) */ +#define EXT4_DEF_SB_UPDATE_INTERVAL_KB (16384) /* kilobytes (16MB) */ + +/* + * Minimum number of groups in a flexgroup before we separate out + * directories into the first block group of a flexgroup + */ +#define EXT4_FLEX_SIZE_DIR_ALLOC_SCHEME 4 + +/* + * Structure of a directory entry + */ +#define EXT4_NAME_LEN 255 +/* + * Base length of the ext4 directory entry excluding the name length + */ +#define EXT4_BASE_DIR_LEN (sizeof(struct ext4_dir_entry_2) - EXT4_NAME_LEN) + +struct ext4_dir_entry { + __le32 inode; /* Inode number */ + __le16 rec_len; /* Directory entry length */ + __le16 name_len; /* Name length */ + char name[EXT4_NAME_LEN]; /* File name */ +}; + + +/* + * Encrypted Casefolded entries require saving the hash on disk. This structure + * followed ext4_dir_entry_2's name[name_len] at the next 4 byte aligned + * boundary. + */ +struct ext4_dir_entry_hash { + __le32 hash; + __le32 minor_hash; +}; + +/* + * The new version of the directory entry. Since EXT4 structures are + * stored in intel byte order, and the name_len field could never be + * bigger than 255 chars, it's safe to reclaim the extra byte for the + * file_type field. + */ +struct ext4_dir_entry_2 { + __le32 inode; /* Inode number */ + __le16 rec_len; /* Directory entry length */ + __u8 name_len; /* Name length */ + __u8 file_type; /* See file type macros EXT4_FT_* below */ + char name[EXT4_NAME_LEN]; /* File name */ +}; + +/* + * Access the hashes at the end of ext4_dir_entry_2 + */ +#define EXT4_DIRENT_HASHES(entry) \ + ((struct ext4_dir_entry_hash *) \ + (((void *)(entry)) + \ + ((8 + (entry)->name_len + EXT4_DIR_ROUND) & ~EXT4_DIR_ROUND))) +#define EXT4_DIRENT_HASH(entry) le32_to_cpu(EXT4_DIRENT_HASHES(entry)->hash) +#define EXT4_DIRENT_MINOR_HASH(entry) \ + le32_to_cpu(EXT4_DIRENT_HASHES(entry)->minor_hash) + +static inline bool ext4_hash_in_dirent(const struct inode *inode) +{ + return IS_CASEFOLDED(inode) && IS_ENCRYPTED(inode); +} + +/* + * This is a bogus directory entry at the end of each leaf block that + * records checksums. + */ +struct ext4_dir_entry_tail { + __le32 det_reserved_zero1; /* Pretend to be unused */ + __le16 det_rec_len; /* 12 */ + __u8 det_reserved_zero2; /* Zero name length */ + __u8 det_reserved_ft; /* 0xDE, fake file type */ + __le32 det_checksum; /* crc32c(uuid+inum+dirblock) */ +}; + +#define EXT4_DIRENT_TAIL(block, blocksize) \ + ((struct ext4_dir_entry_tail *)(((void *)(block)) + \ + ((blocksize) - \ + sizeof(struct ext4_dir_entry_tail)))) + +/* + * Ext4 directory file types. Only the low 3 bits are used. The + * other bits are reserved for now. + */ +#define EXT4_FT_UNKNOWN 0 +#define EXT4_FT_REG_FILE 1 +#define EXT4_FT_DIR 2 +#define EXT4_FT_CHRDEV 3 +#define EXT4_FT_BLKDEV 4 +#define EXT4_FT_FIFO 5 +#define EXT4_FT_SOCK 6 +#define EXT4_FT_SYMLINK 7 + +#define EXT4_FT_MAX 8 + +#define EXT4_FT_DIR_CSUM 0xDE + +/* + * EXT4_DIR_PAD defines the directory entries boundaries + * + * NOTE: It must be a multiple of 4 + */ +#define EXT4_DIR_PAD 4 +#define EXT4_DIR_ROUND (EXT4_DIR_PAD - 1) +#define EXT4_MAX_REC_LEN ((1<<16)-1) + +/* + * The rec_len is dependent on the type of directory. Directories that are + * casefolded and encrypted need to store the hash as well, so we add room for + * ext4_extended_dir_entry_2. For all entries related to '.' or '..' you should + * pass NULL for dir, as those entries do not use the extra fields. + */ +static inline unsigned int ext4_dir_rec_len(__u8 name_len, + const struct inode *dir) +{ + int rec_len = (name_len + 8 + EXT4_DIR_ROUND); + + if (dir && ext4_hash_in_dirent(dir)) + rec_len += sizeof(struct ext4_dir_entry_hash); + return (rec_len & ~EXT4_DIR_ROUND); +} + +static inline unsigned int +ext4_rec_len_from_disk(__le16 dlen, unsigned blocksize) +{ + unsigned len = le16_to_cpu(dlen); + + if (len == EXT4_MAX_REC_LEN || len == 0) + return blocksize; + return (len & 65532) | ((len & 3) << 16); +} + +static inline __le16 ext4_rec_len_to_disk(unsigned len, unsigned blocksize) +{ + BUG_ON((len > blocksize) || (blocksize > (1 << 18)) || (len & 3)); + if (len < 65536) + return cpu_to_le16(len); + if (len == blocksize) { + if (blocksize == 65536) + return cpu_to_le16(EXT4_MAX_REC_LEN); + else + return cpu_to_le16(0); + } + return cpu_to_le16((len & 65532) | ((len >> 16) & 3)); +} + +/* + * Hash Tree Directory indexing + * (c) Daniel Phillips, 2001 + */ + +#define is_dx(dir) (ext4_has_feature_dir_index((dir)->i_sb) && \ + ext4_test_inode_flag((dir), EXT4_INODE_INDEX)) +#define EXT4_DIR_LINK_MAX(dir) unlikely((dir)->i_nlink >= EXT4_LINK_MAX && \ + !(ext4_has_feature_dir_nlink((dir)->i_sb) && is_dx(dir))) +#define EXT4_DIR_LINK_EMPTY(dir) ((dir)->i_nlink == 2 || (dir)->i_nlink == 1) + +/* Legal values for the dx_root hash_version field: */ + +#define DX_HASH_LEGACY 0 +#define DX_HASH_HALF_MD4 1 +#define DX_HASH_TEA 2 +#define DX_HASH_LEGACY_UNSIGNED 3 +#define DX_HASH_HALF_MD4_UNSIGNED 4 +#define DX_HASH_TEA_UNSIGNED 5 +#define DX_HASH_SIPHASH 6 +#define DX_HASH_LAST DX_HASH_SIPHASH + +static inline u32 ext4_chksum(u32 crc, const void *address, unsigned int length) +{ + return crc32c(crc, address, length); +} + +#ifdef __KERNEL__ + +/* hash info structure used by the directory hash */ +struct dx_hash_info +{ + u32 hash; + u32 minor_hash; + int hash_version; + u32 *seed; +}; + + +/* 32 and 64 bit signed EOF for dx directories */ +#define EXT4_HTREE_EOF_32BIT ((1UL << (32 - 1)) - 1) +#define EXT4_HTREE_EOF_64BIT ((1ULL << (64 - 1)) - 1) + + +/* + * Control parameters used by ext4_htree_next_block + */ +#define HASH_NB_ALWAYS 1 + +struct ext4_filename { + const struct qstr *usr_fname; + struct fscrypt_str disk_name; + struct dx_hash_info hinfo; +#ifdef CONFIG_FS_ENCRYPTION + struct fscrypt_str crypto_buf; +#endif +#if IS_ENABLED(CONFIG_UNICODE) + struct qstr cf_name; +#endif +}; + +#define fname_name(p) ((p)->disk_name.name) +#define fname_usr_name(p) ((p)->usr_fname->name) +#define fname_len(p) ((p)->disk_name.len) + +/* + * Describe an inode's exact location on disk and in memory + */ +struct ext4_iloc +{ + struct buffer_head *bh; + unsigned long offset; + ext4_group_t block_group; +}; + +static inline struct ext4_inode *ext4_raw_inode(struct ext4_iloc *iloc) +{ + return (struct ext4_inode *) (iloc->bh->b_data + iloc->offset); +} + +static inline bool ext4_is_quota_file(struct inode *inode) +{ + return IS_NOQUOTA(inode) && + !(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL); +} + +/* + * This structure is stuffed into the struct file's private_data field + * for directories. It is where we put information so that we can do + * readdir operations in hash tree order. + */ +struct dir_private_info { + struct rb_root root; + struct rb_node *curr_node; + struct fname *extra_fname; + loff_t last_pos; + __u32 curr_hash; + __u32 curr_minor_hash; + __u32 next_hash; + u64 cookie; + bool initialized; +}; + +/* calculate the first block number of the group */ +static inline ext4_fsblk_t +ext4_group_first_block_no(struct super_block *sb, ext4_group_t group_no) +{ + return group_no * (ext4_fsblk_t)EXT4_BLOCKS_PER_GROUP(sb) + + le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block); +} + +/* + * Special error return code only used by dx_probe() and its callers. + */ +#define ERR_BAD_DX_DIR (-(MAX_ERRNO - 1)) + +/* htree levels for ext4 */ +#define EXT4_HTREE_LEVEL_COMPAT 2 +#define EXT4_HTREE_LEVEL 3 + +static inline int ext4_dir_htree_level(struct super_block *sb) +{ + return ext4_has_feature_largedir(sb) ? + EXT4_HTREE_LEVEL : EXT4_HTREE_LEVEL_COMPAT; +} + +/* + * Timeout and state flag for lazy initialization inode thread. + */ +#define EXT4_DEF_LI_WAIT_MULT 10 +#define EXT4_DEF_LI_MAX_START_DELAY 5 +#define EXT4_LAZYINIT_QUIT 0x0001 +#define EXT4_LAZYINIT_RUNNING 0x0002 + +/* + * Lazy inode table initialization info + */ +struct ext4_lazy_init { + unsigned long li_state; + struct list_head li_request_list; + struct mutex li_list_mtx; +}; + +enum ext4_li_mode { + EXT4_LI_MODE_PREFETCH_BBITMAP, + EXT4_LI_MODE_ITABLE, +}; + +struct ext4_li_request { + struct super_block *lr_super; + enum ext4_li_mode lr_mode; + ext4_group_t lr_first_not_zeroed; + ext4_group_t lr_next_group; + struct list_head lr_request; + unsigned long lr_next_sched; + unsigned long lr_timeout; +}; + +struct ext4_features { + struct kobject f_kobj; + struct completion f_kobj_unregister; +}; + +/* + * This structure will be used for multiple mount protection. It will be + * written into the block number saved in the s_mmp_block field in the + * superblock. Programs that check MMP should assume that if + * SEQ_FSCK (or any unknown code above SEQ_MAX) is present then it is NOT safe + * to use the filesystem, regardless of how old the timestamp is. + */ +#define EXT4_MMP_MAGIC 0x004D4D50U /* ASCII for MMP */ +#define EXT4_MMP_SEQ_CLEAN 0xFF4D4D50U /* mmp_seq value for clean unmount */ +#define EXT4_MMP_SEQ_FSCK 0xE24D4D50U /* mmp_seq value when being fscked */ +#define EXT4_MMP_SEQ_MAX 0xE24D4D4FU /* maximum valid mmp_seq value */ + +struct mmp_struct { + __le32 mmp_magic; /* Magic number for MMP */ + __le32 mmp_seq; /* Sequence no. updated periodically */ + + /* + * mmp_time, mmp_nodename & mmp_bdevname are only used for information + * purposes and do not affect the correctness of the algorithm + */ + __le64 mmp_time; /* Time last updated */ + char mmp_nodename[64]; /* Node which last updated MMP block */ + char mmp_bdevname[32]; /* Bdev which last updated MMP block */ + + /* + * mmp_check_interval is used to verify if the MMP block has been + * updated on the block device. The value is updated based on the + * maximum time to write the MMP block during an update cycle. + */ + __le16 mmp_check_interval; + + __le16 mmp_pad1; + __le32 mmp_pad2[226]; + __le32 mmp_checksum; /* crc32c(uuid+mmp_block) */ +}; + +/* arguments passed to the mmp thread */ +struct mmpd_data { + struct buffer_head *bh; /* bh from initial read_mmp_block() */ + struct super_block *sb; /* super block of the fs */ +}; + +/* + * Check interval multiplier + * The MMP block is written every update interval and initially checked every + * update interval x the multiplier (the value is then adapted based on the + * write latency). The reason is that writes can be delayed under load and we + * don't want readers to incorrectly assume that the filesystem is no longer + * in use. + */ +#define EXT4_MMP_CHECK_MULT 2UL + +/* + * Minimum interval for MMP checking in seconds. + */ +#define EXT4_MMP_MIN_CHECK_INTERVAL 5UL + +/* + * Maximum interval for MMP checking in seconds. + */ +#define EXT4_MMP_MAX_CHECK_INTERVAL 300UL + +/* + * Function prototypes + */ + +/* + * Ok, these declarations are also in but none of the + * ext4 source programs needs to include it so they are duplicated here. + */ +# define NORET_TYPE /**/ +# define ATTRIB_NORET __attribute__((noreturn)) +# define NORET_AND noreturn, + +/* bitmap.c */ +extern unsigned int ext4_count_free(char *bitmap, unsigned numchars); +void ext4_inode_bitmap_csum_set(struct super_block *sb, + struct ext4_group_desc *gdp, + struct buffer_head *bh); +int ext4_inode_bitmap_csum_verify(struct super_block *sb, + struct ext4_group_desc *gdp, + struct buffer_head *bh); +void ext4_block_bitmap_csum_set(struct super_block *sb, + struct ext4_group_desc *gdp, + struct buffer_head *bh); +int ext4_block_bitmap_csum_verify(struct super_block *sb, + struct ext4_group_desc *gdp, + struct buffer_head *bh); + +/* balloc.c */ +extern void ext4_get_group_no_and_offset(struct super_block *sb, + ext4_fsblk_t blocknr, + ext4_group_t *blockgrpp, + ext4_grpblk_t *offsetp); +extern ext4_group_t ext4_get_group_number(struct super_block *sb, + ext4_fsblk_t block); + +extern int ext4_bg_has_super(struct super_block *sb, ext4_group_t group); +extern unsigned long ext4_bg_num_gdb(struct super_block *sb, + ext4_group_t group); +extern ext4_fsblk_t ext4_new_meta_blocks(handle_t *handle, struct inode *inode, + ext4_fsblk_t goal, + unsigned int flags, + unsigned long *count, + int *errp); +extern int ext4_claim_free_clusters(struct ext4_sb_info *sbi, + s64 nclusters, unsigned int flags); +extern ext4_fsblk_t ext4_count_free_clusters(struct super_block *); +extern struct ext4_group_desc * ext4_get_group_desc(struct super_block * sb, + ext4_group_t block_group, + struct buffer_head ** bh); +extern struct ext4_group_info *ext4_get_group_info(struct super_block *sb, + ext4_group_t group); +extern int ext4_should_retry_alloc(struct super_block *sb, int *retries); + +extern struct buffer_head *ext4_read_block_bitmap_nowait(struct super_block *sb, + ext4_group_t block_group, + bool ignore_locked); +extern int ext4_wait_block_bitmap(struct super_block *sb, + ext4_group_t block_group, + struct buffer_head *bh); +extern struct buffer_head *ext4_read_block_bitmap(struct super_block *sb, + ext4_group_t block_group); +extern unsigned ext4_free_clusters_after_init(struct super_block *sb, + ext4_group_t block_group, + struct ext4_group_desc *gdp); +ext4_fsblk_t ext4_inode_to_goal_block(struct inode *); + +#if IS_ENABLED(CONFIG_UNICODE) +extern int ext4_fname_setup_ci_filename(struct inode *dir, + const struct qstr *iname, + struct ext4_filename *fname); + +static inline void ext4_fname_free_ci_filename(struct ext4_filename *fname) +{ + kfree(fname->cf_name.name); + fname->cf_name.name = NULL; +} +#else +static inline int ext4_fname_setup_ci_filename(struct inode *dir, + const struct qstr *iname, + struct ext4_filename *fname) +{ + return 0; +} + +static inline void ext4_fname_free_ci_filename(struct ext4_filename *fname) +{ +} +#endif + +/* ext4 encryption related stuff goes here crypto.c */ +#ifdef CONFIG_FS_ENCRYPTION +extern const struct fscrypt_operations ext4_cryptops; + +int ext4_fname_setup_filename(struct inode *dir, const struct qstr *iname, + int lookup, struct ext4_filename *fname); + +int ext4_fname_prepare_lookup(struct inode *dir, struct dentry *dentry, + struct ext4_filename *fname); + +void ext4_fname_free_filename(struct ext4_filename *fname); + +int ext4_ioctl_get_encryption_pwsalt(struct file *filp, void __user *arg); + +#else /* !CONFIG_FS_ENCRYPTION */ +static inline int ext4_fname_setup_filename(struct inode *dir, + const struct qstr *iname, + int lookup, + struct ext4_filename *fname) +{ + fname->usr_fname = iname; + fname->disk_name.name = (unsigned char *) iname->name; + fname->disk_name.len = iname->len; + + return ext4_fname_setup_ci_filename(dir, iname, fname); +} + +static inline int ext4_fname_prepare_lookup(struct inode *dir, + struct dentry *dentry, + struct ext4_filename *fname) +{ + return ext4_fname_setup_filename(dir, &dentry->d_name, 1, fname); +} + +static inline void ext4_fname_free_filename(struct ext4_filename *fname) +{ + ext4_fname_free_ci_filename(fname); +} + +static inline int ext4_ioctl_get_encryption_pwsalt(struct file *filp, + void __user *arg) +{ + return -EOPNOTSUPP; +} +#endif /* !CONFIG_FS_ENCRYPTION */ + +/* dir.c */ +extern int __ext4_check_dir_entry(const char *, unsigned int, struct inode *, + struct file *, + struct ext4_dir_entry_2 *, + struct buffer_head *, char *, int, + unsigned int); +#define ext4_check_dir_entry(dir, filp, de, bh, buf, size, offset) \ + unlikely(__ext4_check_dir_entry(__func__, __LINE__, (dir), (filp), \ + (de), (bh), (buf), (size), (offset))) +extern int ext4_htree_store_dirent(struct file *dir_file, __u32 hash, + __u32 minor_hash, + struct ext4_dir_entry_2 *dirent, + struct fscrypt_str *ent_name); +extern void ext4_htree_free_dir_info(struct dir_private_info *p); +extern int ext4_find_dest_de(struct inode *dir, struct buffer_head *bh, + void *buf, int buf_size, + struct ext4_filename *fname, + struct ext4_dir_entry_2 **dest_de); +void ext4_insert_dentry(struct inode *dir, struct inode *inode, + struct ext4_dir_entry_2 *de, + int buf_size, + struct ext4_filename *fname); +static inline void ext4_update_dx_flag(struct inode *inode) +{ + if (!ext4_has_feature_dir_index(inode->i_sb) && + ext4_test_inode_flag(inode, EXT4_INODE_INDEX)) { + /* ext4_iget() should have caught this... */ + WARN_ON_ONCE(ext4_has_feature_metadata_csum(inode->i_sb)); + ext4_clear_inode_flag(inode, EXT4_INODE_INDEX); + } +} +static const unsigned char ext4_filetype_table[] = { + DT_UNKNOWN, DT_REG, DT_DIR, DT_CHR, DT_BLK, DT_FIFO, DT_SOCK, DT_LNK +}; + +static inline unsigned char get_dtype(struct super_block *sb, int filetype) +{ + if (!ext4_has_feature_filetype(sb) || filetype >= EXT4_FT_MAX) + return DT_UNKNOWN; + + return ext4_filetype_table[filetype]; +} +extern int ext4_check_all_de(struct inode *dir, struct buffer_head *bh, + void *buf, int buf_size); + +/* fsync.c */ +extern int ext4_sync_file(struct file *, loff_t, loff_t, int); + +/* hash.c */ +extern int ext4fs_dirhash(const struct inode *dir, const char *name, int len, + struct dx_hash_info *hinfo); + +/* ialloc.c */ +extern int ext4_mark_inode_used(struct super_block *sb, int ino); +extern struct inode *__ext4_new_inode(struct mnt_idmap *, handle_t *, + struct inode *, umode_t, + const struct qstr *qstr, __u32 goal, + uid_t *owner, __u32 i_flags, + int handle_type, unsigned int line_no, + int nblocks); + +#define ext4_new_inode(handle, dir, mode, qstr, goal, owner, i_flags) \ + __ext4_new_inode(&nop_mnt_idmap, (handle), (dir), (mode), (qstr), \ + (goal), (owner), i_flags, 0, 0, 0) +#define ext4_new_inode_start_handle(idmap, dir, mode, qstr, goal, owner, \ + type, nblocks) \ + __ext4_new_inode((idmap), NULL, (dir), (mode), (qstr), (goal), (owner), \ + 0, (type), __LINE__, (nblocks)) + + +extern void ext4_free_inode(handle_t *, struct inode *); +extern struct inode * ext4_orphan_get(struct super_block *, unsigned long); +extern unsigned long ext4_count_free_inodes(struct super_block *); +extern unsigned long ext4_count_dirs(struct super_block *); +extern void ext4_mark_bitmap_end(int start_bit, int end_bit, char *bitmap); +extern int ext4_init_inode_table(struct super_block *sb, + ext4_group_t group, int barrier); +void ext4_end_bitmap_read(struct bio *bio); + +/* fast_commit.c */ +int ext4_fc_info_show(struct seq_file *seq, void *v); +void ext4_fc_init(struct super_block *sb, journal_t *journal); +void ext4_fc_init_inode(struct inode *inode); +void ext4_fc_track_range(handle_t *handle, struct inode *inode, ext4_lblk_t start, + ext4_lblk_t end); +void __ext4_fc_track_unlink(handle_t *handle, struct inode *inode, + struct dentry *dentry); +void __ext4_fc_track_link(handle_t *handle, struct inode *inode, + struct dentry *dentry); +void ext4_fc_track_unlink(handle_t *handle, struct dentry *dentry); +void ext4_fc_track_link(handle_t *handle, struct inode *inode, + struct dentry *dentry); +void __ext4_fc_track_create(handle_t *handle, struct inode *inode, + struct dentry *dentry); +void ext4_fc_track_create(handle_t *handle, struct dentry *dentry); +void ext4_fc_track_inode(handle_t *handle, struct inode *inode); +void ext4_fc_mark_ineligible(struct super_block *sb, int reason, handle_t *handle); +void ext4_fc_del(struct inode *inode); +bool ext4_fc_replay_check_excluded(struct super_block *sb, ext4_fsblk_t block); +void ext4_fc_replay_cleanup(struct super_block *sb); +int ext4_fc_commit(journal_t *journal, tid_t commit_tid); +int __init ext4_fc_init_dentry_cache(void); +void ext4_fc_destroy_dentry_cache(void); +int ext4_fc_record_regions(struct super_block *sb, int ino, + ext4_lblk_t lblk, ext4_fsblk_t pblk, + int len, int replay); + +/* mballoc.c */ +extern const struct seq_operations ext4_mb_seq_groups_ops; +extern const struct seq_operations ext4_mb_seq_structs_summary_ops; +extern int ext4_seq_mb_stats_show(struct seq_file *seq, void *offset); +extern int ext4_mb_init(struct super_block *); +extern void ext4_mb_release(struct super_block *); +extern ext4_fsblk_t ext4_mb_new_blocks(handle_t *, + struct ext4_allocation_request *, int *); +extern void ext4_discard_preallocations(struct inode *); +extern int __init ext4_init_mballoc(void); +extern void ext4_exit_mballoc(void); +extern ext4_group_t ext4_mb_prefetch(struct super_block *sb, + ext4_group_t group, + unsigned int nr, int *cnt); +extern void ext4_mb_prefetch_fini(struct super_block *sb, ext4_group_t group, + unsigned int nr); + +extern void ext4_free_blocks(handle_t *handle, struct inode *inode, + struct buffer_head *bh, ext4_fsblk_t block, + unsigned long count, int flags); +extern int ext4_mb_alloc_groupinfo(struct super_block *sb, + ext4_group_t ngroups); +extern int ext4_mb_add_groupinfo(struct super_block *sb, + ext4_group_t i, struct ext4_group_desc *desc); +extern int ext4_group_add_blocks(handle_t *handle, struct super_block *sb, + ext4_fsblk_t block, unsigned long count); +extern int ext4_trim_fs(struct super_block *, struct fstrim_range *); +extern void ext4_process_freed_data(struct super_block *sb, tid_t commit_tid); +extern void ext4_mb_mark_bb(struct super_block *sb, ext4_fsblk_t block, + int len, bool state); +static inline bool ext4_mb_cr_expensive(enum criteria cr) +{ + return cr >= CR_GOAL_LEN_SLOW; +} + +/* inode.c */ +void ext4_inode_csum_set(struct inode *inode, struct ext4_inode *raw, + struct ext4_inode_info *ei); +int ext4_inode_is_fast_symlink(struct inode *inode); +void ext4_check_map_extents_env(struct inode *inode); +struct buffer_head *ext4_getblk(handle_t *, struct inode *, ext4_lblk_t, int); +struct buffer_head *ext4_bread(handle_t *, struct inode *, ext4_lblk_t, int); +int ext4_bread_batch(struct inode *inode, ext4_lblk_t block, int bh_count, + bool wait, struct buffer_head **bhs); +int ext4_get_block_unwritten(struct inode *inode, sector_t iblock, + struct buffer_head *bh_result, int create); +int ext4_get_block(struct inode *inode, sector_t iblock, + struct buffer_head *bh_result, int create); +int ext4_da_get_block_prep(struct inode *inode, sector_t iblock, + struct buffer_head *bh, int create); +int ext4_walk_page_buffers(handle_t *handle, + struct inode *inode, + struct buffer_head *head, + unsigned from, + unsigned to, + int *partial, + int (*fn)(handle_t *handle, struct inode *inode, + struct buffer_head *bh)); +int do_journal_get_write_access(handle_t *handle, struct inode *inode, + struct buffer_head *bh); +void ext4_set_inode_mapping_order(struct inode *inode); +#define FALL_BACK_TO_NONDELALLOC 1 +#define EXT4_WRITE_DATA_INLINE 2 + +typedef enum { + EXT4_IGET_NORMAL = 0, + EXT4_IGET_SPECIAL = 0x0001, /* OK to iget a system inode */ + EXT4_IGET_HANDLE = 0x0002, /* Inode # is from a handle */ + EXT4_IGET_BAD = 0x0004, /* Allow to iget a bad inode */ + EXT4_IGET_EA_INODE = 0x0008, /* Inode should contain an EA value */ + EXT4_IGET_NOWAIT = 0x0010 /* Non-blocking lookup (skip if freeing) */ +} ext4_iget_flags; + +extern struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, + ext4_iget_flags flags, const char *function, + unsigned int line); + +#define ext4_iget(sb, ino, flags) \ + __ext4_iget((sb), (ino), (flags), __func__, __LINE__) + +extern int ext4_write_inode(struct inode *, struct writeback_control *); +extern int ext4_sync_inode_metadata(struct inode *, struct writeback_control *); +extern int ext4_setattr(struct mnt_idmap *, struct dentry *, + struct iattr *); +extern u32 ext4_dio_alignment(struct inode *inode); +extern int ext4_getattr(struct mnt_idmap *, const struct path *, + struct kstat *, u32, unsigned int); +extern void ext4_evict_inode(struct inode *); +extern void ext4_clear_inode(struct inode *); +extern int ext4_file_getattr(struct mnt_idmap *, const struct path *, + struct kstat *, u32, unsigned int); +extern void ext4_dirty_inode(struct inode *, int); +extern int ext4_change_inode_journal_flag(struct inode *, int); +int ext4_get_inode_loc(struct inode *inode, struct ext4_iloc *iloc); +int ext4_get_inode_loc_noio(struct inode *inode, struct ext4_iloc *iloc); +int ext4_get_fc_inode_loc(struct super_block *sb, unsigned long ino, + struct ext4_iloc *iloc); +extern int ext4_inode_attach_jinode(struct inode *inode); +extern int ext4_can_truncate(struct inode *inode); +extern int ext4_truncate(struct inode *); +extern int ext4_break_layouts(struct inode *); +extern int ext4_truncate_page_cache_block_range(struct inode *inode, + loff_t start, loff_t end); +extern int ext4_punch_hole(struct file *file, loff_t offset, loff_t length); +extern void ext4_set_inode_flags(struct inode *, bool init); +extern int ext4_alloc_da_blocks(struct inode *inode); +extern void ext4_set_aops(struct inode *inode); +extern int ext4_normal_submit_inode_data_buffers(struct jbd2_inode *jinode); +extern int ext4_chunk_trans_blocks(struct inode *, int nrblocks); +extern int ext4_chunk_trans_extent(struct inode *inode, int nrblocks); +extern int ext4_meta_trans_blocks(struct inode *inode, int lblocks, + int pextents, int alloc_extents); +extern int ext4_block_zero_eof(struct inode *inode, loff_t from, loff_t end); + +#define EXT4_PARTIAL_ZERO_START 0x1 +#define EXT4_PARTIAL_ZERO_END 0x2 +extern int ext4_zero_partial_blocks(struct inode *inode, loff_t lstart, + loff_t length, unsigned int *partial_zeroed); +extern vm_fault_t ext4_page_mkwrite(struct vm_fault *vmf); +extern qsize_t *ext4_get_reserved_space(struct inode *inode); +extern int ext4_get_projid(struct inode *inode, kprojid_t *projid); +extern void ext4_da_release_space(struct inode *inode, int to_free); +extern void ext4_da_update_reserve_space(struct inode *inode, + int used, int quota_claim); +extern int ext4_issue_zeroout(struct inode *inode, ext4_lblk_t lblk, + ext4_fsblk_t pblk, ext4_lblk_t len); + +static inline bool is_special_ino(struct super_block *sb, unsigned long ino) +{ + struct ext4_super_block *es = EXT4_SB(sb)->s_es; + + return (ino < EXT4_FIRST_INO(sb) && ino != EXT4_ROOT_INO) || + ino == le32_to_cpu(es->s_usr_quota_inum) || + ino == le32_to_cpu(es->s_grp_quota_inum) || + ino == le32_to_cpu(es->s_prj_quota_inum) || + ino == le32_to_cpu(es->s_orphan_file_inum); +} + +/* indirect.c */ +extern int ext4_ind_map_blocks(handle_t *handle, struct inode *inode, + struct ext4_map_blocks *map, int flags); +extern int ext4_ind_trans_blocks(struct inode *inode, int nrblocks); +extern void ext4_ind_truncate(handle_t *, struct inode *inode); +extern int ext4_ind_remove_space(handle_t *handle, struct inode *inode, + ext4_lblk_t start, ext4_lblk_t end); + +/* ioctl.c */ +extern long ext4_ioctl(struct file *, unsigned int, unsigned long); +extern long ext4_compat_ioctl(struct file *, unsigned int, unsigned long); +int ext4_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa); +int ext4_fileattr_get(struct dentry *dentry, struct file_kattr *fa); +extern void ext4_reset_inode_seed(struct inode *inode); +int ext4_update_overhead(struct super_block *sb, bool force); +int ext4_force_shutdown(struct super_block *sb, u32 flags); + +/* migrate.c */ +extern int ext4_ext_migrate(struct inode *); +extern int ext4_ind_migrate(struct inode *inode); + +/* namei.c */ +extern int ext4_init_new_dir(handle_t *handle, struct inode *dir, + struct inode *inode); +extern int ext4_dirblock_csum_verify(struct inode *inode, + struct buffer_head *bh); +extern int ext4_htree_fill_tree(struct file *dir_file, __u32 start_hash, + __u32 start_minor_hash, __u32 *next_hash); +extern int ext4_search_dir(struct buffer_head *bh, + char *search_buf, + int buf_size, + struct inode *dir, + struct ext4_filename *fname, + unsigned int offset, + struct ext4_dir_entry_2 **res_dir); +extern int ext4_generic_delete_entry(struct inode *dir, + struct ext4_dir_entry_2 *de_del, + struct buffer_head *bh, + void *entry_buf, + int buf_size, + int csum_size); +extern bool ext4_empty_dir(struct inode *inode); + +/* resize.c */ +extern void ext4_kvfree_array_rcu(void *to_free); +extern int ext4_group_add(struct super_block *sb, + struct ext4_new_group_data *input); +extern int ext4_group_extend(struct super_block *sb, + struct ext4_super_block *es, + ext4_fsblk_t n_blocks_count); +extern int ext4_resize_fs(struct super_block *sb, ext4_fsblk_t n_blocks_count); +extern unsigned int ext4_list_backups(struct super_block *sb, + unsigned int *three, unsigned int *five, + unsigned int *seven); + +/* super.c */ +extern struct buffer_head *ext4_sb_bread(struct super_block *sb, + sector_t block, blk_opf_t op_flags); +extern struct buffer_head *ext4_sb_bread_unmovable(struct super_block *sb, + sector_t block); +extern struct buffer_head *ext4_sb_bread_nofail(struct super_block *sb, + sector_t block); +void ext4_read_bh_nowait(struct buffer_head *bh, blk_opf_t op_flags, + bio_end_io_t end_io, bool simu_fail); +int ext4_read_bh(struct buffer_head *bh, blk_opf_t op_flags, + bio_end_io_t end_io, bool simu_fail); +extern int ext4_read_bh_lock(struct buffer_head *bh, blk_opf_t op_flags, bool wait); +extern void ext4_sb_breadahead_unmovable(struct super_block *sb, sector_t block); +extern int ext4_seq_options_show(struct seq_file *seq, void *offset); +extern int ext4_calculate_overhead(struct super_block *sb); +extern __le32 ext4_superblock_csum(struct ext4_super_block *es); +extern void ext4_superblock_csum_set(struct super_block *sb); +extern int ext4_alloc_flex_bg_array(struct super_block *sb, + ext4_group_t ngroup); +extern const char *ext4_decode_error(struct super_block *sb, int errno, + char nbuf[16]); +extern void ext4_mark_group_bitmap_corrupted(struct super_block *sb, + ext4_group_t block_group, + unsigned int flags); +extern unsigned int ext4_num_base_meta_blocks(struct super_block *sb, + ext4_group_t block_group); +extern void print_daily_error_info(struct timer_list *t); + +extern __printf(7, 8) +void __ext4_error(struct super_block *, const char *, unsigned int, bool, + int, __u64, const char *, ...); +extern __printf(6, 7) +void __ext4_error_inode(struct inode *, const char *, unsigned int, + ext4_fsblk_t, int, const char *, ...); +extern __printf(5, 6) +void __ext4_error_file(struct file *, const char *, unsigned int, ext4_fsblk_t, + const char *, ...); +extern void __ext4_std_error(struct super_block *, const char *, + unsigned int, int); +extern __printf(4, 5) +void __ext4_warning(struct super_block *, const char *, unsigned int, + const char *, ...); +extern __printf(4, 5) +void __ext4_warning_inode(const struct inode *inode, const char *function, + unsigned int line, const char *fmt, ...); +extern __printf(3, 4) +void __ext4_msg(struct super_block *, const char *, const char *, ...); +extern void __dump_mmp_msg(struct super_block *, struct mmp_struct *mmp, + const char *, unsigned int, const char *); +extern __printf(7, 8) +void __ext4_grp_locked_error(const char *, unsigned int, + struct super_block *, ext4_group_t, + u64, ext4_fsblk_t, + const char *, ...); + +#define EXT4_ERROR_INODE(inode, fmt, a...) \ + ext4_error_inode((inode), __func__, __LINE__, 0, (fmt), ## a) + +#define EXT4_ERROR_INODE_ERR(inode, err, fmt, a...) \ + __ext4_error_inode((inode), __func__, __LINE__, 0, (err), (fmt), ## a) + +#define ext4_error_inode_block(inode, block, err, fmt, a...) \ + __ext4_error_inode((inode), __func__, __LINE__, (block), (err), \ + (fmt), ## a) + +#define EXT4_ERROR_FILE(file, block, fmt, a...) \ + ext4_error_file((file), __func__, __LINE__, (block), (fmt), ## a) + +#define ext4_abort(sb, err, fmt, a...) \ + __ext4_error((sb), __func__, __LINE__, true, (err), 0, (fmt), ## a) + +#ifdef CONFIG_PRINTK + +#define ext4_error_inode(inode, func, line, block, fmt, ...) \ + __ext4_error_inode(inode, func, line, block, 0, fmt, ##__VA_ARGS__) +#define ext4_error_inode_err(inode, func, line, block, err, fmt, ...) \ + __ext4_error_inode((inode), (func), (line), (block), \ + (err), (fmt), ##__VA_ARGS__) +#define ext4_error_file(file, func, line, block, fmt, ...) \ + __ext4_error_file(file, func, line, block, fmt, ##__VA_ARGS__) +#define ext4_error(sb, fmt, ...) \ + __ext4_error((sb), __func__, __LINE__, false, 0, 0, (fmt), \ + ##__VA_ARGS__) +#define ext4_error_err(sb, err, fmt, ...) \ + __ext4_error((sb), __func__, __LINE__, false, (err), 0, (fmt), \ + ##__VA_ARGS__) +#define ext4_warning(sb, fmt, ...) \ + __ext4_warning(sb, __func__, __LINE__, fmt, ##__VA_ARGS__) +#define ext4_warning_inode(inode, fmt, ...) \ + __ext4_warning_inode(inode, __func__, __LINE__, fmt, ##__VA_ARGS__) +#define ext4_msg(sb, level, fmt, ...) \ + __ext4_msg(sb, level, fmt, ##__VA_ARGS__) +#define dump_mmp_msg(sb, mmp, msg) \ + __dump_mmp_msg(sb, mmp, __func__, __LINE__, msg) +#define ext4_grp_locked_error(sb, grp, ino, block, fmt, ...) \ + __ext4_grp_locked_error(__func__, __LINE__, sb, grp, ino, block, \ + fmt, ##__VA_ARGS__) + +#else + +#define ext4_error_inode(inode, func, line, block, fmt, ...) \ +do { \ + no_printk(fmt, ##__VA_ARGS__); \ + __ext4_error_inode(inode, "", 0, block, 0, " "); \ +} while (0) +#define ext4_error_inode_err(inode, func, line, block, err, fmt, ...) \ +do { \ + no_printk(fmt, ##__VA_ARGS__); \ + __ext4_error_inode(inode, "", 0, block, err, " "); \ +} while (0) +#define ext4_error_file(file, func, line, block, fmt, ...) \ +do { \ + no_printk(fmt, ##__VA_ARGS__); \ + __ext4_error_file(file, "", 0, block, " "); \ +} while (0) +#define ext4_error(sb, fmt, ...) \ +do { \ + no_printk(fmt, ##__VA_ARGS__); \ + __ext4_error(sb, "", 0, false, 0, 0, " "); \ +} while (0) +#define ext4_error_err(sb, err, fmt, ...) \ +do { \ + no_printk(fmt, ##__VA_ARGS__); \ + __ext4_error(sb, "", 0, false, err, 0, " "); \ +} while (0) +#define ext4_warning(sb, fmt, ...) \ +do { \ + no_printk(fmt, ##__VA_ARGS__); \ + __ext4_warning(sb, "", 0, " "); \ +} while (0) +#define ext4_warning_inode(inode, fmt, ...) \ +do { \ + no_printk(fmt, ##__VA_ARGS__); \ + __ext4_warning_inode(inode, "", 0, " "); \ +} while (0) +#define ext4_msg(sb, level, fmt, ...) \ +do { \ + no_printk(fmt, ##__VA_ARGS__); \ + __ext4_msg(sb, "", " "); \ +} while (0) +#define dump_mmp_msg(sb, mmp, msg) \ + __dump_mmp_msg(sb, mmp, "", 0, "") +#define ext4_grp_locked_error(sb, grp, ino, block, fmt, ...) \ +do { \ + no_printk(fmt, ##__VA_ARGS__); \ + __ext4_grp_locked_error("", 0, sb, grp, ino, block, " "); \ +} while (0) + +#endif + +extern ext4_fsblk_t ext4_block_bitmap(struct super_block *sb, + struct ext4_group_desc *bg); +extern ext4_fsblk_t ext4_inode_bitmap(struct super_block *sb, + struct ext4_group_desc *bg); +extern ext4_fsblk_t ext4_inode_table(struct super_block *sb, + struct ext4_group_desc *bg); +extern __u32 ext4_free_group_clusters(struct super_block *sb, + struct ext4_group_desc *bg); +extern __u32 ext4_free_inodes_count(struct super_block *sb, + struct ext4_group_desc *bg); +extern __u32 ext4_used_dirs_count(struct super_block *sb, + struct ext4_group_desc *bg); +extern __u32 ext4_itable_unused_count(struct super_block *sb, + struct ext4_group_desc *bg); +extern void ext4_block_bitmap_set(struct super_block *sb, + struct ext4_group_desc *bg, ext4_fsblk_t blk); +extern void ext4_inode_bitmap_set(struct super_block *sb, + struct ext4_group_desc *bg, ext4_fsblk_t blk); +extern void ext4_inode_table_set(struct super_block *sb, + struct ext4_group_desc *bg, ext4_fsblk_t blk); +extern void ext4_free_group_clusters_set(struct super_block *sb, + struct ext4_group_desc *bg, + __u32 count); +extern void ext4_free_inodes_set(struct super_block *sb, + struct ext4_group_desc *bg, __u32 count); +extern void ext4_used_dirs_set(struct super_block *sb, + struct ext4_group_desc *bg, __u32 count); +extern void ext4_itable_unused_set(struct super_block *sb, + struct ext4_group_desc *bg, __u32 count); +extern int ext4_group_desc_csum_verify(struct super_block *sb, __u32 group, + struct ext4_group_desc *gdp); +extern void ext4_group_desc_csum_set(struct super_block *sb, __u32 group, + struct ext4_group_desc *gdp); +extern int ext4_register_li_request(struct super_block *sb, + ext4_group_t first_not_zeroed); + +static inline int ext4_has_group_desc_csum(struct super_block *sb) +{ + return ext4_has_feature_gdt_csum(sb) || + ext4_has_feature_metadata_csum(sb); +} + +#define ext4_read_incompat_64bit_val(es, name) \ + (((es)->s_feature_incompat & cpu_to_le32(EXT4_FEATURE_INCOMPAT_64BIT) \ + ? (ext4_fsblk_t)le32_to_cpu(es->name##_hi) << 32 : 0) | \ + le32_to_cpu(es->name##_lo)) + +static inline ext4_fsblk_t ext4_blocks_count(struct ext4_super_block *es) +{ + return ext4_read_incompat_64bit_val(es, s_blocks_count); +} + +static inline ext4_fsblk_t ext4_r_blocks_count(struct ext4_super_block *es) +{ + return ext4_read_incompat_64bit_val(es, s_r_blocks_count); +} + +static inline ext4_fsblk_t ext4_free_blocks_count(struct ext4_super_block *es) +{ + return ext4_read_incompat_64bit_val(es, s_free_blocks_count); +} + +static inline void ext4_blocks_count_set(struct ext4_super_block *es, + ext4_fsblk_t blk) +{ + es->s_blocks_count_lo = cpu_to_le32((u32)blk); + es->s_blocks_count_hi = cpu_to_le32(blk >> 32); +} + +static inline void ext4_free_blocks_count_set(struct ext4_super_block *es, + ext4_fsblk_t blk) +{ + es->s_free_blocks_count_lo = cpu_to_le32((u32)blk); + es->s_free_blocks_count_hi = cpu_to_le32(blk >> 32); +} + +static inline void ext4_r_blocks_count_set(struct ext4_super_block *es, + ext4_fsblk_t blk) +{ + es->s_r_blocks_count_lo = cpu_to_le32((u32)blk); + es->s_r_blocks_count_hi = cpu_to_le32(blk >> 32); +} + +static inline loff_t ext4_isize(struct super_block *sb, + struct ext4_inode *raw_inode) +{ + if (ext4_has_feature_largedir(sb) || + S_ISREG(le16_to_cpu(raw_inode->i_mode))) + return ((loff_t)le32_to_cpu(raw_inode->i_size_high) << 32) | + le32_to_cpu(raw_inode->i_size_lo); + + return (loff_t) le32_to_cpu(raw_inode->i_size_lo); +} + +static inline void ext4_isize_set(struct ext4_inode *raw_inode, loff_t i_size) +{ + raw_inode->i_size_lo = cpu_to_le32(i_size); + raw_inode->i_size_high = cpu_to_le32(i_size >> 32); +} + +/* + * Reading s_groups_count requires using smp_rmb() afterwards. See + * the locking protocol documented in the comments of ext4_group_add() + * in resize.c + */ +static inline ext4_group_t ext4_get_groups_count(struct super_block *sb) +{ + ext4_group_t ngroups = EXT4_SB(sb)->s_groups_count; + + smp_rmb(); + return ngroups; +} + +static inline ext4_group_t ext4_flex_group(struct ext4_sb_info *sbi, + ext4_group_t block_group) +{ + return block_group >> sbi->s_log_groups_per_flex; +} + +static inline unsigned int ext4_flex_bg_size(struct ext4_sb_info *sbi) +{ + return 1 << sbi->s_log_groups_per_flex; +} + +static inline loff_t ext4_get_maxbytes(struct inode *inode) +{ + if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) + return inode->i_sb->s_maxbytes; + return EXT4_SB(inode->i_sb)->s_bitmap_maxbytes; +} + +#define ext4_std_error(sb, errno) \ +do { \ + if ((errno)) \ + __ext4_std_error((sb), __func__, __LINE__, (errno)); \ +} while (0) + +#ifdef CONFIG_SMP +/* Each CPU can accumulate percpu_counter_batch clusters in their local + * counters. So we need to make sure we have free clusters more + * than percpu_counter_batch * nr_cpu_ids. Also add a window of 4 times. + */ +#define EXT4_FREECLUSTERS_WATERMARK (4 * (percpu_counter_batch * nr_cpu_ids)) +#else +#define EXT4_FREECLUSTERS_WATERMARK 0 +#endif + +/* Update i_disksize. Requires i_rwsem to avoid races with truncate */ +static inline void ext4_update_i_disksize(struct inode *inode, loff_t newsize) +{ + WARN_ON_ONCE(S_ISREG(inode->i_mode) && + !inode_is_locked(inode)); + down_write(&EXT4_I(inode)->i_data_sem); + if (newsize > EXT4_I(inode)->i_disksize) + WRITE_ONCE(EXT4_I(inode)->i_disksize, newsize); + up_write(&EXT4_I(inode)->i_data_sem); +} + +/* Update i_size, i_disksize. Requires i_rwsem to avoid races with truncate */ +static inline int ext4_update_inode_size(struct inode *inode, loff_t newsize) +{ + int changed = 0; + + if (newsize > inode->i_size) { + i_size_write(inode, newsize); + changed = 1; + } + if (newsize > EXT4_I(inode)->i_disksize) { + ext4_update_i_disksize(inode, newsize); + changed |= 2; + } + return changed; +} + +int ext4_update_disksize_before_punch(struct inode *inode, loff_t offset, + loff_t len); + +struct ext4_group_info { + unsigned long bb_state; +#ifdef AGGRESSIVE_CHECK + unsigned long bb_check_counter; +#endif + struct rb_root bb_free_root; + ext4_grpblk_t bb_first_free; /* first free block */ + ext4_grpblk_t bb_free; /* total free blocks */ + ext4_grpblk_t bb_fragments; /* nr of freespace fragments */ + int bb_avg_fragment_size_order; /* order of average + fragment in BG */ + ext4_grpblk_t bb_largest_free_order;/* order of largest frag in BG */ + ext4_group_t bb_group; /* Group number */ + struct list_head bb_prealloc_list; +#ifdef DOUBLE_CHECK + void *bb_bitmap; +#endif + struct rw_semaphore alloc_sem; + ext4_grpblk_t bb_counters[]; /* Nr of free power-of-two-block + * regions, index is order. + * bb_counters[3] = 5 means + * 5 free 8-block regions. */ +}; + +#define EXT4_GROUP_INFO_NEED_INIT_BIT 0 +#define EXT4_GROUP_INFO_WAS_TRIMMED_BIT 1 +#define EXT4_GROUP_INFO_BBITMAP_CORRUPT_BIT 2 +#define EXT4_GROUP_INFO_IBITMAP_CORRUPT_BIT 3 +#define EXT4_GROUP_INFO_BBITMAP_CORRUPT \ + (1 << EXT4_GROUP_INFO_BBITMAP_CORRUPT_BIT) +#define EXT4_GROUP_INFO_IBITMAP_CORRUPT \ + (1 << EXT4_GROUP_INFO_IBITMAP_CORRUPT_BIT) +#define EXT4_GROUP_INFO_BBITMAP_READ_BIT 4 + +#define EXT4_MB_GRP_NEED_INIT(grp) \ + (test_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &((grp)->bb_state))) +#define EXT4_MB_GRP_BBITMAP_CORRUPT(grp) \ + (test_bit(EXT4_GROUP_INFO_BBITMAP_CORRUPT_BIT, &((grp)->bb_state))) +#define EXT4_MB_GRP_IBITMAP_CORRUPT(grp) \ + (test_bit(EXT4_GROUP_INFO_IBITMAP_CORRUPT_BIT, &((grp)->bb_state))) + +#define EXT4_MB_GRP_WAS_TRIMMED(grp) \ + (test_bit(EXT4_GROUP_INFO_WAS_TRIMMED_BIT, &((grp)->bb_state))) +#define EXT4_MB_GRP_SET_TRIMMED(grp) \ + (set_bit(EXT4_GROUP_INFO_WAS_TRIMMED_BIT, &((grp)->bb_state))) +#define EXT4_MB_GRP_CLEAR_TRIMMED(grp) \ + (clear_bit(EXT4_GROUP_INFO_WAS_TRIMMED_BIT, &((grp)->bb_state))) +#define EXT4_MB_GRP_TEST_AND_SET_READ(grp) \ + (ext4_mb_grp_test_and_set_read((grp))) + +static inline int ext4_mb_grp_test_and_set_read(struct ext4_group_info *grp) +{ + return (test_bit(EXT4_GROUP_INFO_BBITMAP_READ_BIT, &grp->bb_state) || + test_and_set_bit(EXT4_GROUP_INFO_BBITMAP_READ_BIT, &grp->bb_state)); +} + +#define EXT4_MAX_CONTENTION 8 +#define EXT4_CONTENTION_THRESHOLD 2 + +static inline spinlock_t *ext4_group_lock_ptr(struct super_block *sb, + ext4_group_t group) +{ + return bgl_lock_ptr(EXT4_SB(sb)->s_blockgroup_lock, group); +} + +/* + * Returns true if the filesystem is busy enough that attempts to + * access the block group locks has run into contention. + */ +static inline int ext4_fs_is_busy(struct ext4_sb_info *sbi) +{ + return (atomic_read(&sbi->s_lock_busy) > EXT4_CONTENTION_THRESHOLD); +} + +static inline bool ext4_try_lock_group(struct super_block *sb, ext4_group_t group) +{ + if (!spin_trylock(ext4_group_lock_ptr(sb, group))) + return false; + /* + * We're able to grab the lock right away, so drop the lock + * contention counter. + */ + atomic_add_unless(&EXT4_SB(sb)->s_lock_busy, -1, 0); + return true; +} + +static inline void ext4_lock_group(struct super_block *sb, ext4_group_t group) +{ + if (!ext4_try_lock_group(sb, group)) { + /* + * The lock is busy, so bump the contention counter, + * and then wait on the spin lock. + */ + atomic_add_unless(&EXT4_SB(sb)->s_lock_busy, 1, + EXT4_MAX_CONTENTION); + spin_lock(ext4_group_lock_ptr(sb, group)); + } +} + +static inline void ext4_unlock_group(struct super_block *sb, + ext4_group_t group) +{ + spin_unlock(ext4_group_lock_ptr(sb, group)); +} + +#ifdef CONFIG_QUOTA +static inline bool ext4_quota_capable(struct super_block *sb) +{ + return (test_opt(sb, QUOTA) || ext4_has_feature_quota(sb)); +} + +static inline bool ext4_is_quota_journalled(struct super_block *sb) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + + return (ext4_has_feature_quota(sb) || + sbi->s_qf_names[USRQUOTA] || sbi->s_qf_names[GRPQUOTA]); +} +int ext4_enable_quotas(struct super_block *sb); +#endif + +/* + * Block validity checking + */ +#define ext4_check_indirect_blockref(inode, bh) \ + ext4_check_blockref(__func__, __LINE__, inode, \ + (__le32 *)(bh)->b_data, \ + EXT4_ADDR_PER_BLOCK((inode)->i_sb)) + +#define ext4_ind_check_inode(inode) \ + ext4_check_blockref(__func__, __LINE__, inode, \ + EXT4_I(inode)->i_data, \ + EXT4_NDIR_BLOCKS) + +/* + * Inodes and files operations + */ + +/* dir.c */ +extern const struct file_operations ext4_dir_operations; + +/* file.c */ +extern const struct inode_operations ext4_file_inode_operations; +extern const struct file_operations ext4_file_operations; +extern loff_t ext4_llseek(struct file *file, loff_t offset, int origin); + +/* inline.c */ +extern int ext4_get_max_inline_size(struct inode *inode); +extern int ext4_find_inline_data_nolock(struct inode *inode); +extern int ext4_destroy_inline_data(handle_t *handle, struct inode *inode); +extern void ext4_update_final_de(void *de_buf, int old_size, int new_size); + +int ext4_readpage_inline(struct inode *inode, struct folio *folio); +extern int ext4_try_to_write_inline_data(struct address_space *mapping, + struct inode *inode, + loff_t pos, unsigned len, + struct folio **foliop); +int ext4_write_inline_data_end(struct inode *inode, loff_t pos, unsigned len, + unsigned copied, struct folio *folio); +extern int ext4_generic_write_inline_data(struct address_space *mapping, + struct inode *inode, + loff_t pos, unsigned len, + struct folio **foliop, + bool da); +extern int ext4_try_add_inline_entry(handle_t *handle, + struct ext4_filename *fname, + struct inode *dir, struct inode *inode); +extern int ext4_try_create_inline_dir(handle_t *handle, + struct inode *parent, + struct inode *inode); +extern int ext4_read_inline_dir(struct file *filp, + struct dir_context *ctx, + int *has_inline_data); +extern int ext4_inlinedir_to_tree(struct file *dir_file, + struct inode *dir, ext4_lblk_t block, + struct dx_hash_info *hinfo, + __u32 start_hash, __u32 start_minor_hash, + int *has_inline_data); +extern struct buffer_head *ext4_find_inline_entry(struct inode *dir, + struct ext4_filename *fname, + struct ext4_dir_entry_2 **res_dir, + int *has_inline_data); +extern int ext4_delete_inline_entry(handle_t *handle, + struct inode *dir, + struct ext4_dir_entry_2 *de_del, + struct buffer_head *bh, + int *has_inline_data); +extern bool empty_inline_dir(struct inode *dir, int *has_inline_data); +extern struct buffer_head *ext4_get_first_inline_block(struct inode *inode, + struct ext4_dir_entry_2 **parent_de, + int *retval); +extern void *ext4_read_inline_link(struct inode *inode); + +struct iomap; +extern int ext4_inline_data_iomap(struct inode *inode, struct iomap *iomap); + +extern int ext4_inline_data_truncate(struct inode *inode, int *has_inline); + +extern int ext4_convert_inline_data(struct inode *inode); + +static inline int ext4_has_inline_data(struct inode *inode) +{ + return ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA) && + EXT4_I(inode)->i_inline_off; +} + +/* namei.c */ +extern const struct inode_operations ext4_dir_inode_operations; +extern const struct inode_operations ext4_special_inode_operations; +extern struct dentry *ext4_get_parent(struct dentry *child); +extern int ext4_init_dirblock(handle_t *handle, struct inode *inode, + struct buffer_head *dir_block, + unsigned int parent_ino, void *inline_buf, + int inline_size); +extern void ext4_initialize_dirent_tail(struct buffer_head *bh, + unsigned int blocksize); +extern int ext4_handle_dirty_dirblock(handle_t *handle, struct inode *inode, + struct buffer_head *bh); +extern int __ext4_unlink(struct inode *dir, const struct qstr *d_name, + struct inode *inode, struct dentry *dentry); +extern int __ext4_link(struct inode *dir, struct inode *inode, + const struct qstr *d_name, struct dentry *dentry); + +#define S_SHIFT 12 +static const unsigned char ext4_type_by_mode[(S_IFMT >> S_SHIFT) + 1] = { + [S_IFREG >> S_SHIFT] = EXT4_FT_REG_FILE, + [S_IFDIR >> S_SHIFT] = EXT4_FT_DIR, + [S_IFCHR >> S_SHIFT] = EXT4_FT_CHRDEV, + [S_IFBLK >> S_SHIFT] = EXT4_FT_BLKDEV, + [S_IFIFO >> S_SHIFT] = EXT4_FT_FIFO, + [S_IFSOCK >> S_SHIFT] = EXT4_FT_SOCK, + [S_IFLNK >> S_SHIFT] = EXT4_FT_SYMLINK, +}; + +static inline void ext4_set_de_type(struct super_block *sb, + struct ext4_dir_entry_2 *de, + umode_t mode) { + if (ext4_has_feature_filetype(sb)) + de->file_type = ext4_type_by_mode[(mode & S_IFMT)>>S_SHIFT]; +} + +/* readpages.c */ +int ext4_read_folio(struct file *file, struct folio *folio); +void ext4_readahead(struct readahead_control *rac); +int __init ext4_init_verity_caches(void); +void ext4_exit_verity_caches(void); + +/* symlink.c */ +extern const struct inode_operations ext4_encrypted_symlink_inode_operations; +extern const struct inode_operations ext4_symlink_inode_operations; +extern const struct inode_operations ext4_fast_symlink_inode_operations; + +/* sysfs.c */ +extern void ext4_notify_error_sysfs(struct ext4_sb_info *sbi); +extern int ext4_register_sysfs(struct super_block *sb); +extern void ext4_unregister_sysfs(struct super_block *sb); +extern int __init ext4_init_sysfs(void); +extern void ext4_exit_sysfs(void); + +/* block_validity */ +extern void ext4_release_system_zone(struct super_block *sb); +extern int ext4_setup_system_zone(struct super_block *sb); +extern int __init ext4_init_system_zone(void); +extern void ext4_exit_system_zone(void); +extern int ext4_inode_block_valid(struct inode *inode, + ext4_fsblk_t start_blk, + unsigned int count); +extern int ext4_check_blockref(const char *, unsigned int, + struct inode *, __le32 *, unsigned int); +extern int ext4_sb_block_valid(struct super_block *sb, struct inode *inode, + ext4_fsblk_t start_blk, unsigned int count); + + +/* extents.c */ +struct ext4_ext_path; +struct ext4_extent; + +/* + * Maximum number of logical blocks in a file; ext4_extent's ee_block is + * __le32. + */ +#define EXT_MAX_BLOCKS 0xffffffff + +extern void ext4_ext_tree_init(handle_t *handle, struct inode *inode); +extern int ext4_ext_index_trans_blocks(struct inode *inode, int extents); +extern int ext4_ext_map_blocks(handle_t *handle, struct inode *inode, + struct ext4_map_blocks *map, int flags); +extern int ext4_ext_truncate(handle_t *, struct inode *); +extern int ext4_ext_remove_space(struct inode *inode, ext4_lblk_t start, + ext4_lblk_t end); +extern void ext4_ext_init(struct super_block *); +extern void ext4_ext_release(struct super_block *); +extern long ext4_fallocate(struct file *file, int mode, loff_t offset, + loff_t len); +extern int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode, + loff_t offset, ssize_t len, + ext4_lblk_t *converted); +extern int ext4_convert_unwritten_extents_atomic(handle_t *handle, + struct inode *inode, loff_t offset, ssize_t len); +extern int ext4_convert_unwritten_io_end_vec(handle_t *handle, + ext4_io_end_t *io_end); +extern int ext4_map_blocks(handle_t *handle, struct inode *inode, + struct ext4_map_blocks *map, int flags); +extern int ext4_map_query_blocks(handle_t *handle, struct inode *inode, + struct ext4_map_blocks *map, int flags); +extern int ext4_map_create_blocks(handle_t *handle, struct inode *inode, + struct ext4_map_blocks *map, int flags); +extern int ext4_ext_calc_credits_for_single_extent(struct inode *inode, + int num, + struct ext4_ext_path *path); +extern struct ext4_ext_path *ext4_ext_insert_extent( + handle_t *handle, struct inode *inode, + struct ext4_ext_path *path, + struct ext4_extent *newext, int gb_flags); +extern struct ext4_ext_path *ext4_find_extent(struct inode *, ext4_lblk_t, + struct ext4_ext_path *, + int flags); +extern void ext4_free_ext_path(struct ext4_ext_path *); +extern int ext4_ext_check_inode(struct inode *inode); +extern ext4_lblk_t ext4_ext_next_allocated_block(struct ext4_ext_path *path); +extern int ext4_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + __u64 start, __u64 len); +extern int ext4_get_es_cache(struct inode *inode, + struct fiemap_extent_info *fieinfo, + __u64 start, __u64 len); +extern int ext4_ext_precache(struct inode *inode); +extern int ext4_swap_extents(handle_t *handle, struct inode *inode1, + struct inode *inode2, ext4_lblk_t lblk1, + ext4_lblk_t lblk2, ext4_lblk_t count, + int mark_unwritten,int *err); +extern int ext4_clu_mapped(struct inode *inode, ext4_lblk_t lclu); +extern int ext4_datasem_ensure_credits(handle_t *handle, struct inode *inode, + int check_cred, int restart_cred, + int revoke_cred); +extern void ext4_ext_replay_shrink_inode(struct inode *inode, ext4_lblk_t end); +extern int ext4_ext_replay_set_iblocks(struct inode *inode); +extern int ext4_ext_replay_update_ex(struct inode *inode, ext4_lblk_t start, + int len, int unwritten, ext4_fsblk_t pblk); +extern int ext4_ext_clear_bb(struct inode *inode); + + +/* move_extent.c */ +extern void ext4_double_down_write_data_sem(struct inode *first, + struct inode *second); +extern void ext4_double_up_write_data_sem(struct inode *orig_inode, + struct inode *donor_inode); +extern int ext4_move_extents(struct file *o_filp, struct file *d_filp, + __u64 start_orig, __u64 start_donor, + __u64 len, __u64 *moved_len); + +/* page-io.c */ +extern int __init ext4_init_pageio(void); +extern void ext4_exit_pageio(void); +extern ext4_io_end_t *ext4_init_io_end(struct inode *inode, gfp_t flags); +extern ext4_io_end_t *ext4_get_io_end(ext4_io_end_t *io_end); +extern int ext4_put_io_end(ext4_io_end_t *io_end); +extern void ext4_put_io_end_defer(ext4_io_end_t *io_end); +extern void ext4_io_submit_init(struct ext4_io_submit *io, + struct writeback_control *wbc); +extern void ext4_end_io_rsv_work(struct work_struct *work); +extern void ext4_io_submit(struct ext4_io_submit *io); +void ext4_bio_write_folio(struct ext4_io_submit *io, struct folio *page, + size_t len); +extern struct ext4_io_end_vec *ext4_alloc_io_end_vec(ext4_io_end_t *io_end); +extern struct ext4_io_end_vec *ext4_last_io_end_vec(ext4_io_end_t *io_end); + +/* mmp.c */ +extern int ext4_multi_mount_protect(struct super_block *, ext4_fsblk_t); + +/* mmp.c */ +extern void ext4_stop_mmpd(struct ext4_sb_info *sbi); + +/* verity.c */ +extern const struct fsverity_operations ext4_verityops; + +/* orphan.c */ +extern int ext4_orphan_add(handle_t *, struct inode *); +extern int ext4_orphan_del(handle_t *, struct inode *); +extern void ext4_orphan_cleanup(struct super_block *sb, + struct ext4_super_block *es); +extern void ext4_release_orphan_info(struct super_block *sb); +extern int ext4_init_orphan_info(struct super_block *sb); +extern int ext4_orphan_file_empty(struct super_block *sb); +extern void ext4_orphan_file_block_trigger( + struct jbd2_buffer_trigger_type *triggers, + struct buffer_head *bh, + void *data, size_t size); + +/* + * Add new method to test whether block and inode bitmaps are properly + * initialized. With uninit_bg reading the block from disk is not enough + * to mark the bitmap uptodate. We need to also zero-out the bitmap + */ +#define BH_BITMAP_UPTODATE BH_JBDPrivateStart + +static inline int bitmap_uptodate(struct buffer_head *bh) +{ + return (buffer_uptodate(bh) && + test_bit(BH_BITMAP_UPTODATE, &(bh)->b_state)); +} +static inline void set_bitmap_uptodate(struct buffer_head *bh) +{ + set_bit(BH_BITMAP_UPTODATE, &(bh)->b_state); +} + +extern int ext4_resize_begin(struct super_block *sb); +extern int ext4_resize_end(struct super_block *sb, bool update_backups); + +static inline void ext4_set_io_unwritten_flag(struct ext4_io_end *io_end) +{ + if (!(io_end->flag & EXT4_IO_END_UNWRITTEN)) + io_end->flag |= EXT4_IO_END_UNWRITTEN; +} + +static inline void ext4_clear_io_unwritten_flag(ext4_io_end_t *io_end) +{ + if (io_end->flag & EXT4_IO_END_UNWRITTEN) + io_end->flag &= ~EXT4_IO_END_UNWRITTEN; +} + +extern const struct iomap_ops ext4_iomap_ops; +extern const struct iomap_ops ext4_iomap_report_ops; + +int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length, + unsigned flags, struct iomap *iomap, struct iomap *srcmap); + +static inline int ext4_buffer_uptodate(struct buffer_head *bh) +{ + /* + * If the buffer has the write error flag, we have failed + * to write out data in the block. In this case, we don't + * have to read the block because we may read the old data + * successfully. + */ + if (buffer_write_io_error(bh)) + set_buffer_uptodate(bh); + return buffer_uptodate(bh); +} + +static inline bool ext4_inode_can_atomic_write(struct inode *inode) +{ + + return S_ISREG(inode->i_mode) && + ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS) && + EXT4_SB(inode->i_sb)->s_awu_min > 0; +} + +extern int ext4_block_write_begin(handle_t *handle, struct folio *folio, + loff_t pos, unsigned len, + get_block_t *get_block); + +#if IS_ENABLED(CONFIG_EXT4_KUNIT_TESTS) +#define EXPORT_SYMBOL_FOR_EXT4_TEST(sym) \ + EXPORT_SYMBOL_FOR_MODULES(sym, "ext4-test") +#endif +#endif /* __KERNEL__ */ + +#endif /* _EXT4_H */ diff --git a/fs/ext4/ext4_extents.h b/fs/ext4/ext4_extents.h new file mode 100644 index 000000000..ebaf7cc42 --- /dev/null +++ b/fs/ext4/ext4_extents.h @@ -0,0 +1,280 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2003-2006, Cluster File Systems, Inc, info@clusterfs.com + * Written by Alex Tomas + */ + +#ifndef _EXT4_EXTENTS +#define _EXT4_EXTENTS + +#include "ext4.h" + +/* + * With AGGRESSIVE_TEST defined, the capacity of index/leaf blocks + * becomes very small, so index split, in-depth growing and + * other hard changes happen much more often. + * This is for debug purposes only. + */ +#define AGGRESSIVE_TEST_ + +/* + * With EXTENTS_STATS defined, the number of blocks and extents + * are collected in the truncate path. They'll be shown at + * umount time. + */ +#define EXTENTS_STATS__ + +/* + * If CHECK_BINSEARCH is defined, then the results of the binary search + * will also be checked by linear search. + */ +#define CHECK_BINSEARCH__ + +/* + * ext4_inode has i_block array (60 bytes total). + * The first 12 bytes store ext4_extent_header; + * the remainder stores an array of ext4_extent. + * For non-inode extent blocks, ext4_extent_tail + * follows the array. + */ + +/* + * This is the extent tail on-disk structure. + * All other extent structures are 12 bytes long. It turns out that + * block_size % 12 >= 4 for at least all powers of 2 greater than 512, which + * covers all valid ext4 block sizes. Therefore, this tail structure can be + * crammed into the end of the block without having to rebalance the tree. + */ +struct ext4_extent_tail { + __le32 et_checksum; /* crc32c(uuid+inum+extent_block) */ +}; + +/* + * This is the extent on-disk structure. + * It's used at the bottom of the tree. + */ +struct ext4_extent { + __le32 ee_block; /* first logical block extent covers */ + __le16 ee_len; /* number of blocks covered by extent */ + __le16 ee_start_hi; /* high 16 bits of physical block */ + __le32 ee_start_lo; /* low 32 bits of physical block */ +}; + +/* + * This is index on-disk structure. + * It's used at all the levels except the bottom. + */ +struct ext4_extent_idx { + __le32 ei_block; /* index covers logical blocks from 'block' */ + __le32 ei_leaf_lo; /* pointer to the physical block of the next * + * level. leaf or next index could be there */ + __le16 ei_leaf_hi; /* high 16 bits of physical block */ + __u16 ei_unused; +}; + +/* + * Each block (leaves and indexes), even inode-stored has header. + */ +struct ext4_extent_header { + __le16 eh_magic; /* probably will support different formats */ + __le16 eh_entries; /* number of valid entries */ + __le16 eh_max; /* capacity of store in entries */ + __le16 eh_depth; /* has tree real underlying blocks? */ + __le32 eh_generation; /* generation of the tree */ +}; + +#define EXT4_EXT_MAGIC cpu_to_le16(0xf30a) +#define EXT4_MAX_EXTENT_DEPTH 5 + +#define EXT4_EXTENT_TAIL_OFFSET(hdr) \ + (sizeof(struct ext4_extent_header) + \ + (sizeof(struct ext4_extent) * le16_to_cpu((hdr)->eh_max))) + +static inline struct ext4_extent_tail * +find_ext4_extent_tail(struct ext4_extent_header *eh) +{ + return (struct ext4_extent_tail *)(((void *)eh) + + EXT4_EXTENT_TAIL_OFFSET(eh)); +} + +/* + * Array of ext4_ext_path contains path to some extent. + * Creation/lookup routines use it for traversal/splitting/etc. + * Truncate uses it to simulate recursive walking. + */ +struct ext4_ext_path { + ext4_fsblk_t p_block; + __u16 p_depth; + __u16 p_maxdepth; + struct ext4_extent *p_ext; + struct ext4_extent_idx *p_idx; + struct ext4_extent_header *p_hdr; + struct buffer_head *p_bh; +}; + +/* + * Used to record a portion of a cluster found at the beginning or end + * of an extent while traversing the extent tree during space removal. + * A partial cluster may be removed if it does not contain blocks shared + * with extents that aren't being deleted (tofree state). Otherwise, + * it cannot be removed (nofree state). + */ +struct partial_cluster { + ext4_fsblk_t pclu; /* physical cluster number */ + ext4_lblk_t lblk; /* logical block number within logical cluster */ + enum {initial, tofree, nofree} state; +}; + +/* + * structure for external API + */ + +/* + * EXT_INIT_MAX_LEN is the maximum number of blocks we can have in an + * initialized extent. This is 2^15 and not (2^16 - 1), since we use the + * MSB of ee_len field in the extent datastructure to signify if this + * particular extent is an initialized extent or an unwritten (i.e. + * preallocated). + * EXT_UNWRITTEN_MAX_LEN is the maximum number of blocks we can have in an + * unwritten extent. + * If ee_len is <= 0x8000, it is an initialized extent. Otherwise, it is an + * unwritten one. In other words, if MSB of ee_len is set, it is an + * unwritten extent with only one special scenario when ee_len = 0x8000. + * In this case we can not have an unwritten extent of zero length and + * thus we make it as a special case of initialized extent with 0x8000 length. + * This way we get better extent-to-group alignment for initialized extents. + * Hence, the maximum number of blocks we can have in an *initialized* + * extent is 2^15 (32768) and in an *unwritten* extent is 2^15-1 (32767). + */ +#define EXT_INIT_MAX_LEN (1UL << 15) +#define EXT_UNWRITTEN_MAX_LEN (EXT_INIT_MAX_LEN - 1) + + +#define EXT_FIRST_EXTENT(__hdr__) \ + ((struct ext4_extent *) (((char *) (__hdr__)) + \ + sizeof(struct ext4_extent_header))) +#define EXT_FIRST_INDEX(__hdr__) \ + ((struct ext4_extent_idx *) (((char *) (__hdr__)) + \ + sizeof(struct ext4_extent_header))) +#define EXT_HAS_FREE_INDEX(__path__) \ + (le16_to_cpu((__path__)->p_hdr->eh_entries) \ + < le16_to_cpu((__path__)->p_hdr->eh_max)) +#define EXT_LAST_EXTENT(__hdr__) \ + (EXT_FIRST_EXTENT((__hdr__)) + le16_to_cpu((__hdr__)->eh_entries) - 1) +#define EXT_LAST_INDEX(__hdr__) \ + (EXT_FIRST_INDEX((__hdr__)) + le16_to_cpu((__hdr__)->eh_entries) - 1) +#define EXT_MAX_EXTENT(__hdr__) \ + ((le16_to_cpu((__hdr__)->eh_max)) ? \ + ((EXT_FIRST_EXTENT((__hdr__)) + le16_to_cpu((__hdr__)->eh_max) - 1)) \ + : NULL) +#define EXT_MAX_INDEX(__hdr__) \ + ((le16_to_cpu((__hdr__)->eh_max)) ? \ + ((EXT_FIRST_INDEX((__hdr__)) + le16_to_cpu((__hdr__)->eh_max) - 1)) \ + : NULL) + +static inline struct ext4_extent_header *ext_inode_hdr(struct inode *inode) +{ + return (struct ext4_extent_header *) EXT4_I(inode)->i_data; +} + +static inline struct ext4_extent_header *ext_block_hdr(struct buffer_head *bh) +{ + return (struct ext4_extent_header *) bh->b_data; +} + +static inline unsigned short ext_depth(struct inode *inode) +{ + return le16_to_cpu(ext_inode_hdr(inode)->eh_depth); +} + +static inline void ext4_ext_mark_unwritten(struct ext4_extent *ext) +{ + /* We can not have an unwritten extent of zero length! */ + BUG_ON((le16_to_cpu(ext->ee_len) & ~EXT_INIT_MAX_LEN) == 0); + ext->ee_len |= cpu_to_le16(EXT_INIT_MAX_LEN); +} + +static inline int ext4_ext_is_unwritten(struct ext4_extent *ext) +{ + /* Extent with ee_len of 0x8000 is treated as an initialized extent */ + return (le16_to_cpu(ext->ee_len) > EXT_INIT_MAX_LEN); +} + +static inline int ext4_ext_get_actual_len(struct ext4_extent *ext) +{ + return (le16_to_cpu(ext->ee_len) <= EXT_INIT_MAX_LEN ? + le16_to_cpu(ext->ee_len) : + (le16_to_cpu(ext->ee_len) - EXT_INIT_MAX_LEN)); +} + +static inline void ext4_ext_mark_initialized(struct ext4_extent *ext) +{ + ext->ee_len = cpu_to_le16(ext4_ext_get_actual_len(ext)); +} + +/* + * ext4_ext_pblock: + * combine low and high parts of physical block number into ext4_fsblk_t + */ +static inline ext4_fsblk_t ext4_ext_pblock(struct ext4_extent *ex) +{ + ext4_fsblk_t block; + + block = le32_to_cpu(ex->ee_start_lo); + block |= ((ext4_fsblk_t) le16_to_cpu(ex->ee_start_hi) << 31) << 1; + return block; +} + +/* + * ext4_idx_pblock: + * combine low and high parts of a leaf physical block number into ext4_fsblk_t + */ +static inline ext4_fsblk_t ext4_idx_pblock(struct ext4_extent_idx *ix) +{ + ext4_fsblk_t block; + + block = le32_to_cpu(ix->ei_leaf_lo); + block |= ((ext4_fsblk_t) le16_to_cpu(ix->ei_leaf_hi) << 31) << 1; + return block; +} + +/* + * ext4_ext_store_pblock: + * stores a large physical block number into an extent struct, + * breaking it into parts + */ +static inline void ext4_ext_store_pblock(struct ext4_extent *ex, + ext4_fsblk_t pb) +{ + ex->ee_start_lo = cpu_to_le32((unsigned long) (pb & 0xffffffff)); + ex->ee_start_hi = cpu_to_le16((unsigned long) ((pb >> 31) >> 1) & + 0xffff); +} + +/* + * ext4_idx_store_pblock: + * stores a large physical block number into an index struct, + * breaking it into parts + */ +static inline void ext4_idx_store_pblock(struct ext4_extent_idx *ix, + ext4_fsblk_t pb) +{ + ix->ei_leaf_lo = cpu_to_le32((unsigned long) (pb & 0xffffffff)); + ix->ei_leaf_hi = cpu_to_le16((unsigned long) ((pb >> 31) >> 1) & + 0xffff); +} + +extern int __ext4_ext_dirty(const char *where, unsigned int line, + handle_t *handle, struct inode *inode, + struct ext4_ext_path *path); +extern int ext4_ext_zeroout(struct inode *inode, struct ext4_extent *ex); +#if IS_ENABLED(CONFIG_EXT4_KUNIT_TESTS) +extern int ext4_ext_space_root_idx_test(struct inode *inode, int check); +extern struct ext4_ext_path *ext4_split_convert_extents_test( + handle_t *handle, struct inode *inode, + struct ext4_map_blocks *map, + struct ext4_ext_path *path, + int flags, unsigned int *allocated); +#endif +#endif /* _EXT4_EXTENTS */ + diff --git a/fs/ext4/ext4_jbd2.c b/fs/ext4/ext4_jbd2.c new file mode 100644 index 000000000..53ddedb52 --- /dev/null +++ b/fs/ext4/ext4_jbd2.c @@ -0,0 +1,433 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Interface between ext4 and JBD + */ + +#include "ext4_jbd2.h" + +#include + +int ext4_inode_journal_mode(struct inode *inode) +{ + if (EXT4_JOURNAL(inode) == NULL) + return EXT4_INODE_WRITEBACK_DATA_MODE; /* writeback */ + /* We do not support data journalling with delayed allocation */ + if (!S_ISREG(inode->i_mode) || + ext4_test_inode_flag(inode, EXT4_INODE_EA_INODE) || + test_opt(inode->i_sb, DATA_FLAGS) == EXT4_MOUNT_JOURNAL_DATA || + (ext4_test_inode_flag(inode, EXT4_INODE_JOURNAL_DATA) && + !test_opt(inode->i_sb, DELALLOC))) { + /* We do not support data journalling for encrypted data */ + if (S_ISREG(inode->i_mode) && IS_ENCRYPTED(inode)) + return EXT4_INODE_ORDERED_DATA_MODE; /* ordered */ + return EXT4_INODE_JOURNAL_DATA_MODE; /* journal data */ + } + if (test_opt(inode->i_sb, DATA_FLAGS) == EXT4_MOUNT_ORDERED_DATA) + return EXT4_INODE_ORDERED_DATA_MODE; /* ordered */ + if (test_opt(inode->i_sb, DATA_FLAGS) == EXT4_MOUNT_WRITEBACK_DATA) + return EXT4_INODE_WRITEBACK_DATA_MODE; /* writeback */ + BUG(); +} + +/* Just increment the non-pointer handle value */ +static handle_t *ext4_get_nojournal(void) +{ + handle_t *handle = current->journal_info; + + BUG_ON(handle && !handle->h_invalid); + + if (!handle) { + handle = jbd2_alloc_handle(GFP_NOFS); + if (!handle) + return ERR_PTR(-ENOMEM); + handle->h_invalid = 1; + /* + * This is done by start_this_handle() if journalling + * is enabled. + */ + handle->saved_alloc_context = memalloc_nofs_save(); + current->journal_info = handle; + } + handle->h_ref++; + return handle; +} + + +/* Decrement the non-pointer handle value */ +static void ext4_put_nojournal(handle_t *handle) +{ + BUG_ON(handle->h_ref == 0); + + handle->h_ref--; + if (handle->h_ref == 0) { + memalloc_nofs_restore(handle->saved_alloc_context); + jbd2_free_handle(handle); + current->journal_info = NULL; + } +} + +/* + * Wrappers for jbd2_journal_start/end. + */ +static int ext4_journal_check_start(struct super_block *sb) +{ + int ret; + journal_t *journal; + + might_sleep(); + + ret = ext4_emergency_state(sb); + if (unlikely(ret)) + return ret; + + if (WARN_ON_ONCE(sb_rdonly(sb))) + return -EROFS; + + WARN_ON(sb->s_writers.frozen == SB_FREEZE_COMPLETE); + journal = EXT4_SB(sb)->s_journal; + /* + * Special case here: if the journal has aborted behind our + * backs (eg. EIO in the commit thread), then we still need to + * take the FS itself readonly cleanly. + */ + if (journal && is_journal_aborted(journal)) { + ext4_abort(sb, -journal->j_errno, "Detected aborted journal"); + return -EROFS; + } + return 0; +} + +handle_t *__ext4_journal_start_sb(struct inode *inode, + struct super_block *sb, unsigned int line, + int type, int blocks, int rsv_blocks, + int revoke_creds) +{ + journal_t *journal; + int err; + if (inode) + trace_ext4_journal_start_inode(inode, blocks, rsv_blocks, + revoke_creds, type, + _RET_IP_); + else + trace_ext4_journal_start_sb(sb, blocks, rsv_blocks, + revoke_creds, type, + _RET_IP_); + err = ext4_journal_check_start(sb); + if (err < 0) + return ERR_PTR(err); + + journal = EXT4_SB(sb)->s_journal; + if (!journal || (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY)) + return ext4_get_nojournal(); + return jbd2__journal_start(journal, blocks, rsv_blocks, revoke_creds, + GFP_NOFS, type, line); +} + +int __ext4_journal_stop(const char *where, unsigned int line, handle_t *handle) +{ + struct super_block *sb; + int err; + int rc; + + if (!ext4_handle_valid(handle)) { + ext4_put_nojournal(handle); + return 0; + } + + err = handle->h_err; + if (!handle->h_transaction) { + rc = jbd2_journal_stop(handle); + return err ? err : rc; + } + + sb = handle->h_transaction->t_journal->j_private; + rc = jbd2_journal_stop(handle); + + if (!err) + err = rc; + if (err) + __ext4_std_error(sb, where, line, err); + return err; +} + +handle_t *__ext4_journal_start_reserved(handle_t *handle, unsigned int line, + int type) +{ + struct super_block *sb; + int err; + + if (!ext4_handle_valid(handle)) + return ext4_get_nojournal(); + + sb = handle->h_journal->j_private; + trace_ext4_journal_start_reserved(sb, + jbd2_handle_buffer_credits(handle), _RET_IP_); + err = ext4_journal_check_start(sb); + if (err < 0) { + jbd2_journal_free_reserved(handle); + return ERR_PTR(err); + } + + err = jbd2_journal_start_reserved(handle, type, line); + if (err < 0) + return ERR_PTR(err); + return handle; +} + +int __ext4_journal_ensure_credits(handle_t *handle, int check_cred, + int extend_cred, int revoke_cred) +{ + if (!ext4_handle_valid(handle)) + return 0; + if (is_handle_aborted(handle)) + return -EROFS; + if (jbd2_handle_buffer_credits(handle) >= check_cred && + handle->h_revoke_credits >= revoke_cred) + return 0; + extend_cred = max(0, extend_cred - jbd2_handle_buffer_credits(handle)); + revoke_cred = max(0, revoke_cred - handle->h_revoke_credits); + return ext4_journal_extend(handle, extend_cred, revoke_cred); +} + +static void ext4_journal_abort_handle(const char *caller, unsigned int line, + const char *err_fn, + struct buffer_head *bh, + handle_t *handle, int err) +{ + char nbuf[16]; + const char *errstr = ext4_decode_error(NULL, err, nbuf); + + BUG_ON(!ext4_handle_valid(handle)); + + if (bh) + BUFFER_TRACE(bh, "abort"); + + if (!handle->h_err) + handle->h_err = err; + + if (is_handle_aborted(handle)) + return; + + printk(KERN_ERR "EXT4-fs: %s:%d: aborting transaction: %s in %s\n", + caller, line, errstr, err_fn); + + jbd2_journal_abort_handle(handle); +} + +static void ext4_check_bdev_write_error(struct super_block *sb) +{ + struct address_space *mapping = sb->s_bdev->bd_mapping; + struct ext4_sb_info *sbi = EXT4_SB(sb); + int err; + + /* + * If the block device has write error flag, it may have failed to + * async write out metadata buffers in the background. In this case, + * we could read old data from disk and write it out again, which + * may lead to on-disk filesystem inconsistency. + */ + if (errseq_check(&mapping->wb_err, READ_ONCE(sbi->s_bdev_wb_err))) { + spin_lock(&sbi->s_bdev_wb_lock); + err = errseq_check_and_advance(&mapping->wb_err, &sbi->s_bdev_wb_err); + spin_unlock(&sbi->s_bdev_wb_lock); + if (err) + ext4_error_err(sb, -err, + "Error while async write back metadata"); + } +} + +int __ext4_journal_get_write_access(const char *where, unsigned int line, + handle_t *handle, struct super_block *sb, + struct buffer_head *bh, + enum ext4_journal_trigger_type trigger_type) +{ + int err; + + might_sleep(); + + if (ext4_handle_valid(handle)) { + err = jbd2_journal_get_write_access(handle, bh); + if (err) { + ext4_journal_abort_handle(where, line, __func__, bh, + handle, err); + return err; + } + } else + ext4_check_bdev_write_error(sb); + if (trigger_type == EXT4_JTR_NONE || + !ext4_has_feature_metadata_csum(sb)) + return 0; + BUG_ON(trigger_type >= EXT4_JOURNAL_TRIGGER_COUNT); + jbd2_journal_set_triggers(bh, + &EXT4_SB(sb)->s_journal_triggers[trigger_type].tr_triggers); + return 0; +} + +/* + * The ext4 forget function must perform a revoke if we are freeing data + * which has been journaled. Metadata (eg. indirect blocks) must be + * revoked in all cases. + * + * "bh" may be NULL: a metadata block may have been freed from memory + * but there may still be a record of it in the journal, and that record + * still needs to be revoked. + */ +int __ext4_forget(const char *where, unsigned int line, handle_t *handle, + int is_metadata, struct inode *inode, + struct buffer_head *bh, ext4_fsblk_t blocknr) +{ + int err; + + might_sleep(); + + trace_ext4_forget(inode, is_metadata, blocknr); + BUFFER_TRACE(bh, "enter"); + + ext4_debug("forgetting bh %p: is_metadata=%d, mode %o, data mode %x\n", + bh, is_metadata, inode->i_mode, + test_opt(inode->i_sb, DATA_FLAGS)); + + /* + * In the no journal case, we should wait for the ongoing buffer + * to complete and do a forget. + */ + if (!ext4_handle_valid(handle)) { + if (bh) { + clear_buffer_dirty(bh); + wait_on_buffer(bh); + __bforget(bh); + } + return 0; + } + + /* Never use the revoke function if we are doing full data + * journaling: there is no need to, and a V1 superblock won't + * support it. Otherwise, only skip the revoke on un-journaled + * data blocks. */ + + if (test_opt(inode->i_sb, DATA_FLAGS) == EXT4_MOUNT_JOURNAL_DATA || + (!is_metadata && !ext4_should_journal_data(inode))) { + if (bh) { + BUFFER_TRACE(bh, "call jbd2_journal_forget"); + err = jbd2_journal_forget(handle, bh); + if (err) + ext4_journal_abort_handle(where, line, __func__, + bh, handle, err); + return err; + } + return 0; + } + + /* + * data!=journal && (is_metadata || should_journal_data(inode)) + */ + BUFFER_TRACE(bh, "call jbd2_journal_revoke"); + err = jbd2_journal_revoke(handle, blocknr, bh); + if (err) { + ext4_journal_abort_handle(where, line, __func__, + bh, handle, err); + __ext4_error(inode->i_sb, where, line, true, -err, 0, + "error %d when attempting revoke", err); + } + BUFFER_TRACE(bh, "exit"); + return err; +} + +int __ext4_journal_get_create_access(const char *where, unsigned int line, + handle_t *handle, struct super_block *sb, + struct buffer_head *bh, + enum ext4_journal_trigger_type trigger_type) +{ + int err; + + if (!ext4_handle_valid(handle)) + return 0; + + err = jbd2_journal_get_create_access(handle, bh); + if (err) { + ext4_journal_abort_handle(where, line, __func__, bh, handle, + err); + return err; + } + if (trigger_type == EXT4_JTR_NONE || + !ext4_has_feature_metadata_csum(sb)) + return 0; + BUG_ON(trigger_type >= EXT4_JOURNAL_TRIGGER_COUNT); + jbd2_journal_set_triggers(bh, + &EXT4_SB(sb)->s_journal_triggers[trigger_type].tr_triggers); + return 0; +} + +static void ext4_inode_attach_mmb(struct inode *inode) +{ + struct mapping_metadata_bhs *mmb; + + /* + * It's difficult to handle failure when marking buffer dirty without + * leaving filesystem corrupted + */ + mmb = kmalloc_obj(*mmb, GFP_NOFS | __GFP_NOFAIL | __GFP_ACCOUNT); + mmb_init(mmb, &inode->i_data); + /* Someone swapped another mmb before us? */ + if (cmpxchg(&EXT4_I(inode)->i_metadata_bhs, NULL, mmb)) + kfree(mmb); +} + +int __ext4_handle_dirty_metadata(const char *where, unsigned int line, + handle_t *handle, struct inode *inode, + struct buffer_head *bh) +{ + int err = 0; + + might_sleep(); + + set_buffer_meta(bh); + set_buffer_prio(bh); + set_buffer_uptodate(bh); + if (ext4_handle_valid(handle)) { + err = jbd2_journal_dirty_metadata(handle, bh); + /* Errors can only happen due to aborted journal or a nasty bug */ + if (!is_handle_aborted(handle) && WARN_ON_ONCE(err)) { + ext4_journal_abort_handle(where, line, __func__, bh, + handle, err); + if (inode == NULL) { + pr_err("EXT4: jbd2_journal_dirty_metadata " + "failed: handle type %u started at " + "line %u, credits %u/%u, errcode %d", + handle->h_type, + handle->h_line_no, + handle->h_requested_credits, + jbd2_handle_buffer_credits(handle), err); + return err; + } + ext4_error_inode(inode, where, line, + bh->b_blocknr, + "journal_dirty_metadata failed: " + "handle type %u started at line %u, " + "credits %u/%u, errcode %d", + handle->h_type, + handle->h_line_no, + handle->h_requested_credits, + jbd2_handle_buffer_credits(handle), + err); + } + } else { + if (inode) { + if (!ext4_i_metadata_bhs(inode)) + ext4_inode_attach_mmb(inode); + mmb_mark_buffer_dirty(bh, ext4_i_metadata_bhs(inode)); + } else { + mark_buffer_dirty(bh); + } + if (inode && inode_needs_sync(inode)) { + sync_dirty_buffer(bh); + if (buffer_req(bh) && !buffer_uptodate(bh)) { + ext4_error_inode_err(inode, where, line, + bh->b_blocknr, EIO, + "IO error syncing itable block"); + err = -EIO; + } + } + } + return err; +} diff --git a/fs/ext4/ext4_jbd2.h b/fs/ext4/ext4_jbd2.h new file mode 100644 index 000000000..2fbf48b3d --- /dev/null +++ b/fs/ext4/ext4_jbd2.h @@ -0,0 +1,457 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * ext4_jbd2.h + * + * Written by Stephen C. Tweedie , 1999 + * + * Copyright 1998--1999 Red Hat corp --- All Rights Reserved + * + * Ext4-specific journaling extensions. + */ + +#ifndef _EXT4_JBD2_H +#define _EXT4_JBD2_H + +#include +#include +#include "ext4.h" + +#define EXT4_JOURNAL(inode) (EXT4_SB((inode)->i_sb)->s_journal) + +/* Define the number of blocks we need to account to a transaction to + * modify one block of data. + * + * We may have to touch one inode, one bitmap buffer, up to three + * indirection blocks, the group and superblock summaries, and the data + * block to complete the transaction. + * + * For extents-enabled fs we may have to allocate and modify up to + * 5 levels of tree, data block (for each of these we need bitmap + group + * summaries), root which is stored in the inode, sb + */ + +#define EXT4_SINGLEDATA_TRANS_BLOCKS(sb) \ + (ext4_has_feature_extents(sb) ? 20U : 8U) + +/* Extended attribute operations touch at most two data buffers, + * two bitmap buffers, and two group summaries, in addition to the inode + * and the superblock, which are already accounted for. */ + +#define EXT4_XATTR_TRANS_BLOCKS 6U + +/* Define the minimum size for a transaction which modifies data. This + * needs to take into account the fact that we may end up modifying two + * quota files too (one for the group, one for the user quota). The + * superblock only gets updated once, of course, so don't bother + * counting that again for the quota updates. */ + +#define EXT4_DATA_TRANS_BLOCKS(sb) (EXT4_SINGLEDATA_TRANS_BLOCKS(sb) + \ + EXT4_XATTR_TRANS_BLOCKS - 2 + \ + EXT4_MAXQUOTAS_TRANS_BLOCKS(sb)) + +/* + * Define the number of metadata blocks we need to account to modify data. + * + * This include super block, inode block, quota blocks and xattr blocks + */ +#define EXT4_META_TRANS_BLOCKS(sb) (EXT4_XATTR_TRANS_BLOCKS + \ + EXT4_MAXQUOTAS_TRANS_BLOCKS(sb)) + +/* Define an arbitrary limit for the amount of data we will anticipate + * writing to any given transaction. For unbounded transactions such as + * write(2) and truncate(2) we can write more than this, but we always + * start off at the maximum transaction size and grow the transaction + * optimistically as we go. */ + +#define EXT4_MAX_TRANS_DATA 64U + +/* We break up a large truncate or write transaction once the handle's + * buffer credits gets this low, we need either to extend the + * transaction or to start a new one. Reserve enough space here for + * inode, bitmap, superblock, group and indirection updates for at least + * one block, plus two quota updates. Quota allocations are not + * needed. */ + +#define EXT4_RESERVE_TRANS_BLOCKS 12U + +/* + * Number of credits needed if we need to insert an entry into a + * directory. For each new index block, we need 4 blocks (old index + * block, new index block, bitmap block, bg summary). For normal + * htree directories there are 2 levels; if the largedir feature + * enabled it's 3 levels. + */ +#define EXT4_INDEX_EXTRA_TRANS_BLOCKS 12U + +#ifdef CONFIG_QUOTA +/* Amount of blocks needed for quota update - we know that the structure was + * allocated so we need to update only data block */ +#define EXT4_QUOTA_TRANS_BLOCKS(sb) ((ext4_quota_capable(sb)) ? 1 : 0) +/* Amount of blocks needed for quota insert/delete - we do some block writes + * but inode, sb and group updates are done only once */ +#define EXT4_QUOTA_INIT_BLOCKS(sb) ((ext4_quota_capable(sb)) ?\ + (DQUOT_INIT_ALLOC*(EXT4_SINGLEDATA_TRANS_BLOCKS(sb)-3)\ + +3+DQUOT_INIT_REWRITE) : 0) + +#define EXT4_QUOTA_DEL_BLOCKS(sb) ((ext4_quota_capable(sb)) ?\ + (DQUOT_DEL_ALLOC*(EXT4_SINGLEDATA_TRANS_BLOCKS(sb)-3)\ + +3+DQUOT_DEL_REWRITE) : 0) +#else +#define EXT4_QUOTA_TRANS_BLOCKS(sb) 0 +#define EXT4_QUOTA_INIT_BLOCKS(sb) 0 +#define EXT4_QUOTA_DEL_BLOCKS(sb) 0 +#endif +#define EXT4_MAXQUOTAS_TRANS_BLOCKS(sb) (EXT4_MAXQUOTAS*EXT4_QUOTA_TRANS_BLOCKS(sb)) +#define EXT4_MAXQUOTAS_INIT_BLOCKS(sb) (EXT4_MAXQUOTAS*EXT4_QUOTA_INIT_BLOCKS(sb)) +#define EXT4_MAXQUOTAS_DEL_BLOCKS(sb) (EXT4_MAXQUOTAS*EXT4_QUOTA_DEL_BLOCKS(sb)) + +/* + * Ext4 handle operation types -- for logging purposes + */ +#define EXT4_HT_MISC 0 +#define EXT4_HT_INODE 1 +#define EXT4_HT_WRITE_PAGE 2 +#define EXT4_HT_MAP_BLOCKS 3 +#define EXT4_HT_DIR 4 +#define EXT4_HT_TRUNCATE 5 +#define EXT4_HT_QUOTA 6 +#define EXT4_HT_RESIZE 7 +#define EXT4_HT_MIGRATE 8 +#define EXT4_HT_MOVE_EXTENTS 9 +#define EXT4_HT_XATTR 10 +#define EXT4_HT_EXT_CONVERT 11 +#define EXT4_HT_MAX 12 + +int +ext4_mark_iloc_dirty(handle_t *handle, + struct inode *inode, + struct ext4_iloc *iloc); + +/* + * On success, We end up with an outstanding reference count against + * iloc->bh. This _must_ be cleaned up later. + */ + +int ext4_reserve_inode_write(handle_t *handle, struct inode *inode, + struct ext4_iloc *iloc); + +#define ext4_mark_inode_dirty(__h, __i) \ + __ext4_mark_inode_dirty((__h), (__i), __func__, __LINE__) +int __ext4_mark_inode_dirty(handle_t *handle, struct inode *inode, + const char *func, unsigned int line); + +int ext4_expand_extra_isize(struct inode *inode, + unsigned int new_extra_isize, + struct ext4_iloc *iloc); +/* + * Wrapper functions with which ext4 calls into JBD. + */ +int __ext4_journal_get_write_access(const char *where, unsigned int line, + handle_t *handle, struct super_block *sb, + struct buffer_head *bh, + enum ext4_journal_trigger_type trigger_type); + +int __ext4_forget(const char *where, unsigned int line, handle_t *handle, + int is_metadata, struct inode *inode, + struct buffer_head *bh, ext4_fsblk_t blocknr); + +int __ext4_journal_get_create_access(const char *where, unsigned int line, + handle_t *handle, struct super_block *sb, + struct buffer_head *bh, + enum ext4_journal_trigger_type trigger_type); + +int __ext4_handle_dirty_metadata(const char *where, unsigned int line, + handle_t *handle, struct inode *inode, + struct buffer_head *bh); + +#define ext4_journal_get_write_access(handle, sb, bh, trigger_type) \ + __ext4_journal_get_write_access(__func__, __LINE__, (handle), (sb), \ + (bh), (trigger_type)) +#define ext4_forget(handle, is_metadata, inode, bh, block_nr) \ + __ext4_forget(__func__, __LINE__, (handle), (is_metadata), (inode), \ + (bh), (block_nr)) +#define ext4_journal_get_create_access(handle, sb, bh, trigger_type) \ + __ext4_journal_get_create_access(__func__, __LINE__, (handle), (sb), \ + (bh), (trigger_type)) +#define ext4_handle_dirty_metadata(handle, inode, bh) \ + __ext4_handle_dirty_metadata(__func__, __LINE__, (handle), (inode), \ + (bh)) + +handle_t *__ext4_journal_start_sb(struct inode *inode, struct super_block *sb, + unsigned int line, int type, int blocks, + int rsv_blocks, int revoke_creds); +int __ext4_journal_stop(const char *where, unsigned int line, handle_t *handle); + +/* Note: Do not use this for NULL handles. This is only to determine if + * a properly allocated handle is using a journal or not. */ +static inline int ext4_handle_valid(handle_t *handle) +{ + return (handle && !handle->h_invalid); +} + +static inline void ext4_handle_sync(handle_t *handle) +{ + if (ext4_handle_valid(handle)) + handle->h_sync = 1; +} + +static inline int ext4_handle_is_aborted(handle_t *handle) +{ + if (ext4_handle_valid(handle)) + return is_handle_aborted(handle); + return 0; +} + +static inline int ext4_free_metadata_revoke_credits(struct super_block *sb, + int blocks) +{ + /* Freeing each metadata block can result in freeing one cluster */ + return blocks * EXT4_SB(sb)->s_cluster_ratio; +} + +static inline int ext4_trans_default_revoke_credits(struct super_block *sb) +{ + return ext4_free_metadata_revoke_credits(sb, 8); +} + +#define ext4_journal_start_sb(sb, type, nblocks) \ + __ext4_journal_start_sb(NULL, (sb), __LINE__, (type), (nblocks), 0,\ + ext4_trans_default_revoke_credits(sb)) + +#define ext4_journal_start(inode, type, nblocks) \ + __ext4_journal_start((inode), __LINE__, (type), (nblocks), 0, \ + ext4_trans_default_revoke_credits((inode)->i_sb)) + +#define ext4_journal_start_with_reserve(inode, type, blocks, rsv_blocks)\ + __ext4_journal_start((inode), __LINE__, (type), (blocks), (rsv_blocks),\ + ext4_trans_default_revoke_credits((inode)->i_sb)) + +#define ext4_journal_start_with_revoke(inode, type, blocks, revoke_creds) \ + __ext4_journal_start((inode), __LINE__, (type), (blocks), 0, \ + (revoke_creds)) + +static inline handle_t *__ext4_journal_start(struct inode *inode, + unsigned int line, int type, + int blocks, int rsv_blocks, + int revoke_creds) +{ + return __ext4_journal_start_sb(inode, inode->i_sb, line, type, blocks, + rsv_blocks, revoke_creds); +} + +#define ext4_journal_stop(handle) \ + __ext4_journal_stop(__func__, __LINE__, (handle)) + +#define ext4_journal_start_reserved(handle, type) \ + __ext4_journal_start_reserved((handle), __LINE__, (type)) + +handle_t *__ext4_journal_start_reserved(handle_t *handle, unsigned int line, + int type); + +static inline handle_t *ext4_journal_current_handle(void) +{ + return journal_current_handle(); +} + +static inline int ext4_journal_extend(handle_t *handle, int nblocks, int revoke) +{ + if (ext4_handle_valid(handle)) + return jbd2_journal_extend(handle, nblocks, revoke); + return 0; +} + +static inline int ext4_journal_restart(handle_t *handle, int nblocks, + int revoke) +{ + if (ext4_handle_valid(handle)) + return jbd2__journal_restart(handle, nblocks, revoke, GFP_NOFS); + return 0; +} + +int __ext4_journal_ensure_credits(handle_t *handle, int check_cred, + int extend_cred, int revoke_cred); + + +/* + * Ensure @handle has at least @check_creds credits available. If not, + * transaction will be extended or restarted to contain at least @extend_cred + * credits. Before restarting transaction @fn is executed to allow for cleanup + * before the transaction is restarted. + * + * The return value is < 0 in case of error, 0 in case the handle has enough + * credits or transaction extension succeeded, 1 in case transaction had to be + * restarted. + */ +#define ext4_journal_ensure_credits_fn(handle, check_cred, extend_cred, \ + revoke_cred, fn) \ +({ \ + __label__ __ensure_end; \ + int err = __ext4_journal_ensure_credits((handle), (check_cred), \ + (extend_cred), (revoke_cred)); \ + \ + if (err <= 0) \ + goto __ensure_end; \ + err = (fn); \ + if (err < 0) \ + goto __ensure_end; \ + err = ext4_journal_restart((handle), (extend_cred), (revoke_cred)); \ + if (err == 0) \ + err = 1; \ +__ensure_end: \ + err; \ +}) + +/* + * Ensure given handle has at least requested amount of credits available, + * possibly restarting transaction if needed. We also make sure the transaction + * has space for at least ext4_trans_default_revoke_credits(sb) revoke records + * as freeing one or two blocks is very common pattern and requesting this is + * very cheap. + */ +static inline int ext4_journal_ensure_credits(handle_t *handle, int credits, + int revoke_creds) +{ + return ext4_journal_ensure_credits_fn(handle, credits, credits, + revoke_creds, 0); +} + +static inline int ext4_journal_blocks_per_folio(struct inode *inode) +{ + if (EXT4_JOURNAL(inode) != NULL) + return jbd2_journal_blocks_per_folio(inode); + return 0; +} + +static inline int ext4_journal_force_commit(journal_t *journal) +{ + if (journal) + return jbd2_journal_force_commit(journal); + return 0; +} + +static inline int ext4_jbd2_inode_add_write(handle_t *handle, + struct inode *inode, loff_t start_byte, loff_t length) +{ + if (ext4_handle_valid(handle)) + return jbd2_journal_inode_ranged_write(handle, + EXT4_I(inode)->jinode, start_byte, length); + return 0; +} + +static inline int ext4_jbd2_inode_add_wait(handle_t *handle, + struct inode *inode, loff_t start_byte, loff_t length) +{ + if (ext4_handle_valid(handle)) + return jbd2_journal_inode_ranged_wait(handle, + EXT4_I(inode)->jinode, start_byte, length); + return 0; +} + +static inline void ext4_update_inode_fsync_trans(handle_t *handle, + struct inode *inode, + int datasync) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + + if (ext4_handle_valid(handle) && !is_handle_aborted(handle)) { + ei->i_sync_tid = handle->h_transaction->t_tid; + if (datasync) + ei->i_datasync_tid = handle->h_transaction->t_tid; + } +} + +/* super.c */ +int ext4_force_commit(struct super_block *sb); + +/* + * Ext4 inode journal modes + */ +#define EXT4_INODE_JOURNAL_DATA_MODE 0x01 /* journal data mode */ +#define EXT4_INODE_ORDERED_DATA_MODE 0x02 /* ordered data mode */ +#define EXT4_INODE_WRITEBACK_DATA_MODE 0x04 /* writeback data mode */ + +int ext4_inode_journal_mode(struct inode *inode); + +static inline int ext4_should_journal_data(struct inode *inode) +{ + return ext4_inode_journal_mode(inode) & EXT4_INODE_JOURNAL_DATA_MODE; +} + +static inline int ext4_should_order_data(struct inode *inode) +{ + return ext4_inode_journal_mode(inode) & EXT4_INODE_ORDERED_DATA_MODE; +} + +static inline int ext4_should_writeback_data(struct inode *inode) +{ + return ext4_inode_journal_mode(inode) & EXT4_INODE_WRITEBACK_DATA_MODE; +} + +static inline int ext4_free_data_revoke_credits(struct inode *inode, int blocks) +{ + if (test_opt(inode->i_sb, DATA_FLAGS) == EXT4_MOUNT_JOURNAL_DATA) + return 0; + if (!ext4_should_journal_data(inode)) + return 0; + /* + * Data blocks in one extent are contiguous, just account for partial + * clusters at extent boundaries + */ + return blocks + 2*(EXT4_SB(inode->i_sb)->s_cluster_ratio - 1); +} + +/* + * This function controls whether or not we should try to go down the + * dioread_nolock code paths, which makes it safe to avoid taking + * i_rwsem for direct I/O reads. This only works for extent-based + * files, and it doesn't work if data journaling is enabled, since the + * dioread_nolock code uses b_private to pass information back to the + * I/O completion handler, and this conflicts with the jbd's use of + * b_private. + */ +static inline int ext4_should_dioread_nolock(struct inode *inode) +{ + if (!test_opt(inode->i_sb, DIOREAD_NOLOCK)) + return 0; + if (!S_ISREG(inode->i_mode)) + return 0; + if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) + return 0; + if (ext4_should_journal_data(inode)) + return 0; + /* temporary fix to prevent generic/422 test failures */ + if (!test_opt(inode->i_sb, DELALLOC)) + return 0; + return 1; +} + +/* + * Pass journal explicitly as it may not be cached in the sbi->s_journal in some + * cases + */ +static inline int ext4_journal_destroy(struct ext4_sb_info *sbi, journal_t *journal) +{ + int err = 0; + + /* + * At this point only two things can be operating on the journal. + * JBD2 thread performing transaction commit and s_sb_upd_work + * issuing sb update through the journal. Once we set + * EXT4_JOURNAL_DESTROY, new ext4_handle_error() calls will not + * queue s_sb_upd_work and ext4_force_commit() makes sure any + * ext4_handle_error() calls from the running transaction commit are + * finished. Hence no new s_sb_upd_work can be queued after we + * flush it here. + */ + ext4_set_mount_flag(sbi->s_sb, EXT4_MF_JOURNAL_DESTROY); + + ext4_force_commit(sbi->s_sb); + flush_work(&sbi->s_sb_upd_work); + + err = jbd2_journal_destroy(journal); + sbi->s_journal = NULL; + + return err; +} + +#endif /* _EXT4_JBD2_H */ diff --git a/fs/ext4/extents-test.c b/fs/ext4/extents-test.c new file mode 100644 index 000000000..c3836ecb8 --- /dev/null +++ b/fs/ext4/extents-test.c @@ -0,0 +1,1064 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Written by Ojaswin Mujoo (IBM) + * + * These Kunit tests are designed to test the functionality of + * extent split and conversion in ext4. + * + * Currently, ext4 can split extents in 2 ways: + * 1. By splitting the extents in the extent tree and optionally converting them + * to written or unwritten based on flags passed. + * 2. In case 1 encounters an error, ext4 instead zerooes out the unwritten + * areas of the extent and marks the complete extent written. + * + * The primary function that handles this is ext4_split_convert_extents(). + * + * We test both of the methods of split. The behavior we try to enforce is: + * 1. When passing EXT4_GET_BLOCKS_CONVERT flag to ext4_split_convert_extents(), + * the split extent should be converted to initialized. + * 2. When passing EXT4_GET_BLOCKS_CONVERT_UNWRITTEN flag to + * ext4_split_convert_extents(), the split extent should be converted to + * uninitialized. + * 3. In case we use the zeroout method, then we should correctly write zeroes + * to the unwritten areas of the extent and we should not corrupt/leak any + * data. + * + * Enforcing 1 and 2 is straight forward, we just setup a minimal inode with + * extent tree, call ext4_split_convert_extents() and check the final state of + * the extent tree. + * + * For zeroout testing, we maintain a separate buffer which represents the disk + * data corresponding to the extents. We then override ext4's zeroout functions + * to instead write zeroes to our buffer. Then, we override + * ext4_ext_insert_extent() to return -ENOSPC, which triggers the zeroout. + * Finally, we check the state of the extent tree and zeroout buffer to confirm + * everything went well. + */ + +#include +#include +#include +#include +#include + +#include "ext4.h" +#include "ext4_extents.h" + +#define EXT_DATA_PBLK 100 +#define EXT_DATA_LBLK 10 +#define EXT_DATA_LEN 3 + +struct kunit_ctx { + /* + * Ext4 inode which has only 1 unwrit extent + */ + struct ext4_inode_info *k_ei; + /* + * Represents the underlying data area (used for zeroout testing) + */ + char *k_data; +} k_ctx; + +/* + * describes the state of an expected extent in extent tree. + */ +struct kunit_ext_state { + ext4_lblk_t ex_lblk; + ext4_lblk_t ex_len; + bool is_unwrit; +}; + +/* + * describes the state of the data area of a writ extent. Used for testing + * correctness of zeroout. + */ +struct kunit_ext_data_state { + char exp_char; + ext4_lblk_t off_blk; + ext4_lblk_t len_blk; +}; + +enum kunit_test_types { + TEST_SPLIT_CONVERT, + TEST_CREATE_BLOCKS, +}; + +struct kunit_ext_test_param { + /* description of test */ + char *desc; + + /* determines which function will be tested */ + int type; + + /* is extent unwrit at beginning of test */ + bool is_unwrit_at_start; + + /* flags to pass while splitting */ + int split_flags; + + /* map describing range to split */ + struct ext4_map_blocks split_map; + + /* disable zeroout */ + bool disable_zeroout; + + /* no of extents expected after split */ + int nr_exp_ext; + + /* + * expected state of extents after split. We will never split into more + * than 3 extents + */ + struct kunit_ext_state exp_ext_state[3]; + + /* Below fields used for zeroout tests */ + + bool is_zeroout_test; + /* + * no of expected data segments (zeroout tests). Example, if we expect + * data to be 4kb 0s, followed by 8kb non-zero, then nr_exp_data_segs==2 + */ + int nr_exp_data_segs; + + /* + * expected state of data area after zeroout. + */ + struct kunit_ext_data_state exp_data_state[3]; +}; + +static int ext_init_fs_context(struct fs_context *fc) +{ + return 0; +} + +static int ext_set(struct super_block *sb, struct fs_context *fc) +{ + return set_anon_super_fc(sb, fc); +} + +static struct file_system_type ext_fs_type = { + .name = "extents test", + .init_fs_context = ext_init_fs_context, + .kill_sb = kill_anon_super, +}; + +static void extents_kunit_exit(struct kunit *test) +{ + struct ext4_sb_info *sbi; + + if (!k_ctx.k_ei) + return; + + sbi = k_ctx.k_ei->vfs_inode.i_sb->s_fs_info; + ext4_es_unregister_shrinker(sbi); + deactivate_super(sbi->s_sb); + kfree(sbi); + kfree(k_ctx.k_ei); + kfree(k_ctx.k_data); +} + +static int __ext4_ext_dirty_stub(const char *where, unsigned int line, + handle_t *handle, struct inode *inode, + struct ext4_ext_path *path) +{ + return 0; +} + +static struct ext4_ext_path * +ext4_ext_insert_extent_stub(handle_t *handle, struct inode *inode, + struct ext4_ext_path *path, + struct ext4_extent *newext, int gb_flags) +{ + return ERR_PTR(-ENOSPC); +} + +/* + * We will zeroout the equivalent range in the data area + */ +static int ext4_ext_zeroout_stub(struct inode *inode, struct ext4_extent *ex) +{ + ext4_lblk_t ee_block, off_blk; + loff_t ee_len; + loff_t off_bytes; + struct kunit *test = kunit_get_current_test(); + + ee_block = le32_to_cpu(ex->ee_block); + ee_len = ext4_ext_get_actual_len(ex); + + KUNIT_EXPECT_EQ_MSG(test, 1, ee_block >= EXT_DATA_LBLK, "ee_block=%d", + ee_block); + KUNIT_EXPECT_EQ(test, 1, + ee_block + ee_len <= EXT_DATA_LBLK + EXT_DATA_LEN); + + off_blk = ee_block - EXT_DATA_LBLK; + off_bytes = off_blk << inode->i_sb->s_blocksize_bits; + memset(k_ctx.k_data + off_bytes, 0, + ee_len << inode->i_sb->s_blocksize_bits); + + return 0; +} + +static int ext4_issue_zeroout_stub(struct inode *inode, ext4_lblk_t lblk, + ext4_fsblk_t pblk, ext4_lblk_t len) +{ + ext4_lblk_t off_blk; + loff_t off_bytes; + struct kunit *test = kunit_get_current_test(); + + kunit_log(KERN_ALERT, test, + "%s: lblk=%u pblk=%llu len=%u", __func__, lblk, pblk, len); + KUNIT_EXPECT_EQ(test, 1, lblk >= EXT_DATA_LBLK); + KUNIT_EXPECT_EQ(test, 1, lblk + len <= EXT_DATA_LBLK + EXT_DATA_LEN); + KUNIT_EXPECT_EQ(test, 1, lblk - EXT_DATA_LBLK == pblk - EXT_DATA_PBLK); + + off_blk = lblk - EXT_DATA_LBLK; + off_bytes = off_blk << inode->i_sb->s_blocksize_bits; + memset(k_ctx.k_data + off_bytes, 0, + len << inode->i_sb->s_blocksize_bits); + + return 0; +} + +static int extents_kunit_init(struct kunit *test) +{ + struct ext4_extent_header *eh = NULL; + struct ext4_inode_info *ei; + struct inode *inode; + struct super_block *sb; + struct fs_context *fc; + struct ext4_sb_info *sbi = NULL; + struct kunit_ext_test_param *param = + (struct kunit_ext_test_param *)(test->param_value); + int err; + + sbi = kzalloc_obj(struct ext4_sb_info); + if (sbi == NULL) + return -ENOMEM; + + fc = fs_context_for_mount(&ext_fs_type, 0); + if (IS_ERR(fc)) { + kfree(sbi); + return PTR_ERR(fc); + } + sb = sget_fc(fc, NULL, ext_set); + put_fs_context(fc); + if (IS_ERR(sb)) { + kfree(sbi); + return PTR_ERR(sb); + } + + sbi->s_sb = sb; + sb->s_fs_info = sbi; + + sb->s_blocksize = 4096; + sb->s_blocksize_bits = 12; + + if (!param || !param->disable_zeroout) + sbi->s_extent_max_zeroout_kb = 32; + + err = ext4_es_register_shrinker(sbi); + if (err) + goto out_deactivate; + + /* setup the mock inode */ + k_ctx.k_ei = kzalloc_obj(struct ext4_inode_info); + if (k_ctx.k_ei == NULL) { + err = -ENOMEM; + goto out; + } + ei = k_ctx.k_ei; + inode = &ei->vfs_inode; + + ext4_es_init_tree(&ei->i_es_tree); + rwlock_init(&ei->i_es_lock); + INIT_LIST_HEAD(&ei->i_es_list); + ei->i_es_all_nr = 0; + ei->i_es_shk_nr = 0; + ei->i_es_shrink_lblk = 0; + + ei->i_disksize = (EXT_DATA_LBLK + EXT_DATA_LEN + 10) + << sb->s_blocksize_bits; + ei->i_flags = 0; + ext4_set_inode_flag(inode, EXT4_INODE_EXTENTS); + inode->i_sb = sb; + + k_ctx.k_data = kzalloc(EXT_DATA_LEN * 4096, GFP_KERNEL); + if (k_ctx.k_data == NULL) { + err = -ENOMEM; + goto out; + } + + /* + * set the data area to a junk value + */ + memset(k_ctx.k_data, 'X', EXT_DATA_LEN * 4096); + + /* create a tree with depth 0 */ + eh = (struct ext4_extent_header *)k_ctx.k_ei->i_data; + + /* Fill extent header */ + eh = ext_inode_hdr(&k_ctx.k_ei->vfs_inode); + eh->eh_depth = 0; + eh->eh_entries = cpu_to_le16(1); + eh->eh_magic = EXT4_EXT_MAGIC; + eh->eh_max = cpu_to_le16(ext4_ext_space_root_idx_test( + &k_ctx.k_ei->vfs_inode, 0)); + eh->eh_generation = 0; + + /* + * add 1 extent in leaf node covering: + * - lblks: [EXT_DATA_LBLK, EXT_DATA_LBLK * + EXT_DATA_LEN) + * - pblks: [EXT_DATA_PBLK, EXT_DATA_PBLK + EXT_DATA_LEN) + */ + EXT_FIRST_EXTENT(eh)->ee_block = cpu_to_le32(EXT_DATA_LBLK); + EXT_FIRST_EXTENT(eh)->ee_len = cpu_to_le16(EXT_DATA_LEN); + ext4_ext_store_pblock(EXT_FIRST_EXTENT(eh), EXT_DATA_PBLK); + if (!param || param->is_unwrit_at_start) + ext4_ext_mark_unwritten(EXT_FIRST_EXTENT(eh)); + + ext4_es_insert_extent(inode, EXT_DATA_LBLK, EXT_DATA_LEN, EXT_DATA_PBLK, + ext4_ext_is_unwritten(EXT_FIRST_EXTENT(eh)) ? + EXTENT_STATUS_UNWRITTEN : + EXTENT_STATUS_WRITTEN, + 0); + + /* Add stubs */ + kunit_activate_static_stub(test, __ext4_ext_dirty, + __ext4_ext_dirty_stub); + kunit_activate_static_stub(test, ext4_ext_zeroout, ext4_ext_zeroout_stub); + kunit_activate_static_stub(test, ext4_issue_zeroout, + ext4_issue_zeroout_stub); + up_write(&sb->s_umount); + + return 0; + +out: + kfree(k_ctx.k_ei); + k_ctx.k_ei = NULL; + + kfree(k_ctx.k_data); + k_ctx.k_data = NULL; + + ext4_es_unregister_shrinker(sbi); +out_deactivate: + deactivate_locked_super(sb); + kfree(sbi); + + return err; +} + +/* + * Return 1 if all bytes in the buf equal to c, else return the offset of first mismatch + */ +static int check_buffer(char *buf, int c, int size) +{ + void *ret = NULL; + + ret = memchr_inv(buf, c, size); + if (ret == NULL) + return 0; + + kunit_log(KERN_ALERT, kunit_get_current_test(), + "# %s: wrong char found at offset %u (expected:%d got:%d)", __func__, + (u32)((char *)ret - buf), c, *((char *)ret)); + return 1; +} + +/* + * Simulate a map block call by first calling ext4_map_query_blocks() to + * correctly populate map flags and pblk and then call the + * ext4_map_create_blocks() to do actual split and conversion. This is easier + * than calling ext4_map_blocks() because that needs mocking a lot of unrelated + * functions. + */ +static void ext4_map_create_blocks_helper(struct kunit *test, + struct inode *inode, + struct ext4_map_blocks *map, + int flags) +{ + int retval = 0; + + retval = ext4_map_query_blocks(NULL, inode, map, flags); + if (retval < 0) { + KUNIT_FAIL(test, + "ext4_map_query_blocks() failed. Cannot proceed\n"); + return; + } + + ext4_map_create_blocks(NULL, inode, map, flags); +} + +static void test_split_convert(struct kunit *test) +{ + struct ext4_ext_path *path; + struct inode *inode = &k_ctx.k_ei->vfs_inode; + struct ext4_extent *ex; + struct ext4_map_blocks map; + const struct kunit_ext_test_param *param = + (const struct kunit_ext_test_param *)(test->param_value); + int blkbits = inode->i_sb->s_blocksize_bits; + + if (param->is_zeroout_test) + /* + * Force zeroout by making ext4_ext_insert_extent return ENOSPC + */ + kunit_activate_static_stub(test, ext4_ext_insert_extent, + ext4_ext_insert_extent_stub); + + path = ext4_find_extent(inode, EXT_DATA_LBLK, NULL, EXT4_EX_NOCACHE); + ex = path->p_ext; + KUNIT_EXPECT_EQ(test, EXT_DATA_LBLK, le32_to_cpu(ex->ee_block)); + KUNIT_EXPECT_EQ(test, EXT_DATA_LEN, ext4_ext_get_actual_len(ex)); + KUNIT_EXPECT_EQ(test, param->is_unwrit_at_start, + ext4_ext_is_unwritten(ex)); + if (param->is_zeroout_test) + KUNIT_EXPECT_EQ(test, 0, + check_buffer(k_ctx.k_data, 'X', + EXT_DATA_LEN << blkbits)); + + map.m_lblk = param->split_map.m_lblk; + map.m_len = param->split_map.m_len; + + switch (param->type) { + case TEST_SPLIT_CONVERT: + path = ext4_split_convert_extents_test(NULL, inode, &map, + path, param->split_flags, NULL); + break; + case TEST_CREATE_BLOCKS: + ext4_map_create_blocks_helper(test, inode, &map, param->split_flags); + break; + default: + KUNIT_FAIL(test, "param->type %d not support.", param->type); + } + + path = ext4_find_extent(inode, EXT_DATA_LBLK, NULL, EXT4_EX_NOCACHE); + ex = path->p_ext; + + for (int i = 0; i < param->nr_exp_ext; i++) { + struct kunit_ext_state exp_ext = param->exp_ext_state[i]; + bool es_check_needed = param->type != TEST_SPLIT_CONVERT; + struct extent_status es; + int contains_ex, ex_end, es_end, es_pblk; + + KUNIT_EXPECT_EQ(test, exp_ext.ex_lblk, + le32_to_cpu(ex->ee_block)); + KUNIT_EXPECT_EQ(test, exp_ext.ex_len, + ext4_ext_get_actual_len(ex)); + KUNIT_EXPECT_EQ(test, exp_ext.is_unwrit, + ext4_ext_is_unwritten(ex)); + /* + * Confirm extent cache is in sync. Note that es cache can be + * merged even when on-disk extents are not so take that into + * account. + * + * Also, ext4_split_convert_extents() forces EXT4_EX_NOCACHE hence + * es status are ignored for that case. + */ + if (es_check_needed) { + ext4_es_lookup_extent(inode, le32_to_cpu(ex->ee_block), + NULL, &es, NULL); + + ex_end = exp_ext.ex_lblk + exp_ext.ex_len; + es_end = es.es_lblk + es.es_len; + contains_ex = es.es_lblk <= exp_ext.ex_lblk && + es_end >= ex_end; + es_pblk = ext4_es_pblock(&es) + + (exp_ext.ex_lblk - es.es_lblk); + + KUNIT_EXPECT_EQ(test, contains_ex, 1); + KUNIT_EXPECT_EQ(test, ext4_ext_pblock(ex), es_pblk); + KUNIT_EXPECT_EQ(test, 1, + (exp_ext.is_unwrit && + ext4_es_is_unwritten(&es)) || + (!exp_ext.is_unwrit && + ext4_es_is_written(&es))); + } + + /* Only printed on failure */ + kunit_log(KERN_INFO, test, + "# [extent %d] exp: lblk:%d len:%d unwrit:%d \n", i, + exp_ext.ex_lblk, exp_ext.ex_len, exp_ext.is_unwrit); + kunit_log(KERN_INFO, test, + "# [extent %d] got: lblk:%d len:%d unwrit:%d\n", i, + le32_to_cpu(ex->ee_block), + ext4_ext_get_actual_len(ex), + ext4_ext_is_unwritten(ex)); + if (es_check_needed) + kunit_log( + KERN_INFO, test, + "# [extent %d] es: lblk:%d len:%d pblk:%lld type:0x%x\n", + i, es.es_lblk, es.es_len, ext4_es_pblock(&es), + ext4_es_type(&es)); + kunit_log(KERN_INFO, test, "------------------\n"); + + ex = ex + 1; + } + + if (!param->is_zeroout_test) + return; + + /* + * Check that then data area has been zeroed out correctly + */ + for (int i = 0; i < param->nr_exp_data_segs; i++) { + loff_t off, len; + struct kunit_ext_data_state exp_data_seg = param->exp_data_state[i]; + + off = exp_data_seg.off_blk << blkbits; + len = exp_data_seg.len_blk << blkbits; + KUNIT_EXPECT_EQ_MSG(test, 0, + check_buffer(k_ctx.k_data + off, + exp_data_seg.exp_char, len), + "# corruption in byte range [%lld, %lld)", + off, len); + } + + return; +} + +static const struct kunit_ext_test_param test_split_convert_params[] = { + /* unwrit to writ splits */ + { .desc = "split unwrit extent to 2 extents and convert 1st half writ", + .type = TEST_SPLIT_CONVERT, + .is_unwrit_at_start = 1, + .split_flags = EXT4_GET_BLOCKS_CONVERT, + .split_map = { .m_lblk = EXT_DATA_LBLK, .m_len = 1 }, + .nr_exp_ext = 2, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = 1, + .is_unwrit = 0 }, + { .ex_lblk = EXT_DATA_LBLK + 1, + .ex_len = EXT_DATA_LEN - 1, + .is_unwrit = 1 } }, + .is_zeroout_test = 0 }, + { .desc = "split unwrit extent to 2 extents and convert 2nd half writ", + .type = TEST_SPLIT_CONVERT, + .is_unwrit_at_start = 1, + .split_flags = EXT4_GET_BLOCKS_CONVERT, + .split_map = { .m_lblk = EXT_DATA_LBLK + 1, .m_len = EXT_DATA_LEN - 1 }, + .nr_exp_ext = 2, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = 1, + .is_unwrit = 1 }, + { .ex_lblk = EXT_DATA_LBLK + 1, + .ex_len = EXT_DATA_LEN - 1, + .is_unwrit = 0 } }, + .is_zeroout_test = 0 }, + { .desc = "split unwrit extent to 3 extents and convert 2nd half to writ", + .type = TEST_SPLIT_CONVERT, + .is_unwrit_at_start = 1, + .split_flags = EXT4_GET_BLOCKS_CONVERT, + .split_map = { .m_lblk = EXT_DATA_LBLK + 1, .m_len = EXT_DATA_LEN - 2 }, + .nr_exp_ext = 3, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = 1, + .is_unwrit = 1 }, + { .ex_lblk = EXT_DATA_LBLK + 1, + .ex_len = EXT_DATA_LEN - 2, + .is_unwrit = 0 }, + { .ex_lblk = EXT_DATA_LBLK + 1 + (EXT_DATA_LEN - 2), + .ex_len = 1, + .is_unwrit = 1 } }, + .is_zeroout_test = 0 }, + + /* writ to unwrit splits */ + { .desc = "split writ extent to 2 extents and convert 1st half unwrit", + .type = TEST_SPLIT_CONVERT, + .is_unwrit_at_start = 0, + .split_flags = EXT4_GET_BLOCKS_CONVERT_UNWRITTEN, + .split_map = { .m_lblk = EXT_DATA_LBLK, .m_len = 1 }, + .nr_exp_ext = 2, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = 1, + .is_unwrit = 1 }, + { .ex_lblk = EXT_DATA_LBLK + 1, + .ex_len = EXT_DATA_LEN - 1, + .is_unwrit = 0 } }, + .is_zeroout_test = 0 }, + { .desc = "split writ extent to 2 extents and convert 2nd half unwrit", + .type = TEST_SPLIT_CONVERT, + .is_unwrit_at_start = 0, + .split_flags = EXT4_GET_BLOCKS_CONVERT_UNWRITTEN, + .split_map = { .m_lblk = EXT_DATA_LBLK + 1, .m_len = EXT_DATA_LEN - 1 }, + .nr_exp_ext = 2, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = 1, + .is_unwrit = 0 }, + { .ex_lblk = EXT_DATA_LBLK + 1, + .ex_len = EXT_DATA_LEN - 1, + .is_unwrit = 1 } }, + .is_zeroout_test = 0 }, + { .desc = "split writ extent to 3 extents and convert 2nd half to unwrit", + .type = TEST_SPLIT_CONVERT, + .is_unwrit_at_start = 0, + .split_flags = EXT4_GET_BLOCKS_CONVERT_UNWRITTEN, + .split_map = { .m_lblk = EXT_DATA_LBLK + 1, .m_len = EXT_DATA_LEN - 2 }, + .nr_exp_ext = 3, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = 1, + .is_unwrit = 0 }, + { .ex_lblk = EXT_DATA_LBLK + 1, + .ex_len = EXT_DATA_LEN - 2, + .is_unwrit = 1 }, + { .ex_lblk = EXT_DATA_LBLK + 1 + (EXT_DATA_LEN - 2), + .ex_len = 1, + .is_unwrit = 0 } }, + .is_zeroout_test = 0 }, + + /* + * ***** zeroout tests ***** + */ + /* unwrit to writ splits */ + { .desc = "split unwrit extent to 2 extents and convert 1st half writ (zeroout)", + .type = TEST_SPLIT_CONVERT, + .is_unwrit_at_start = 1, + .split_flags = EXT4_GET_BLOCKS_CONVERT, + .split_map = { .m_lblk = EXT_DATA_LBLK, .m_len = 1 }, + .nr_exp_ext = 1, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = EXT_DATA_LEN, + .is_unwrit = 0 } }, + .is_zeroout_test = 1, + .nr_exp_data_segs = 2, + .exp_data_state = { { .exp_char = 'X', .off_blk = 0, .len_blk = 1 }, + { .exp_char = 0, + .off_blk = 1, + .len_blk = EXT_DATA_LEN - 1 } } }, + { .desc = "split unwrit extent to 2 extents and convert 2nd half writ (zeroout)", + .type = TEST_SPLIT_CONVERT, + .is_unwrit_at_start = 1, + .split_flags = EXT4_GET_BLOCKS_CONVERT, + .split_map = { .m_lblk = EXT_DATA_LBLK + 1, .m_len = EXT_DATA_LEN - 1 }, + .nr_exp_ext = 1, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = EXT_DATA_LEN, + .is_unwrit = 0 } }, + .is_zeroout_test = 1, + .nr_exp_data_segs = 2, + .exp_data_state = { { .exp_char = 0, .off_blk = 0, .len_blk = 1 }, + { .exp_char = 'X', + .off_blk = 1, + .len_blk = EXT_DATA_LEN - 1 } } }, + { .desc = "split unwrit extent to 3 extents and convert 2nd half writ (zeroout)", + .type = TEST_SPLIT_CONVERT, + .is_unwrit_at_start = 1, + .split_flags = EXT4_GET_BLOCKS_CONVERT, + .split_map = { .m_lblk = EXT_DATA_LBLK + 1, .m_len = EXT_DATA_LEN - 2 }, + .nr_exp_ext = 1, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = EXT_DATA_LEN, + .is_unwrit = 0 } }, + .is_zeroout_test = 1, + .nr_exp_data_segs = 3, + .exp_data_state = { { .exp_char = 0, .off_blk = 0, .len_blk = 1 }, + { .exp_char = 'X', .off_blk = 1, .len_blk = EXT_DATA_LEN - 2 }, + { .exp_char = 0, .off_blk = EXT_DATA_LEN - 1, .len_blk = 1 } } }, + + /* writ to unwrit splits */ + { .desc = "split writ extent to 2 extents and convert 1st half unwrit (zeroout)", + .type = TEST_SPLIT_CONVERT, + .is_unwrit_at_start = 0, + .split_flags = EXT4_GET_BLOCKS_CONVERT_UNWRITTEN, + .split_map = { .m_lblk = EXT_DATA_LBLK, .m_len = 1 }, + .nr_exp_ext = 1, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = EXT_DATA_LEN, + .is_unwrit = 0 } }, + .is_zeroout_test = 1, + .nr_exp_data_segs = 2, + .exp_data_state = { { .exp_char = 0, .off_blk = 0, .len_blk = 1 }, + { .exp_char = 'X', + .off_blk = 1, + .len_blk = EXT_DATA_LEN - 1 } } }, + { .desc = "split writ extent to 2 extents and convert 2nd half unwrit (zeroout)", + .type = TEST_SPLIT_CONVERT, + .is_unwrit_at_start = 0, + .split_flags = EXT4_GET_BLOCKS_CONVERT_UNWRITTEN, + .split_map = { .m_lblk = EXT_DATA_LBLK + 1, .m_len = EXT_DATA_LEN - 1 }, + .nr_exp_ext = 1, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = EXT_DATA_LEN, + .is_unwrit = 0 } }, + .is_zeroout_test = 1, + .nr_exp_data_segs = 2, + .exp_data_state = { { .exp_char = 'X', .off_blk = 0, .len_blk = 1 }, + { .exp_char = 0, + .off_blk = 1, + .len_blk = EXT_DATA_LEN - 1 } } }, + { .desc = "split writ extent to 3 extents and convert 2nd half unwrit (zeroout)", + .type = TEST_SPLIT_CONVERT, + .is_unwrit_at_start = 0, + .split_flags = EXT4_GET_BLOCKS_CONVERT_UNWRITTEN, + .split_map = { .m_lblk = EXT_DATA_LBLK + 1, .m_len = EXT_DATA_LEN - 2 }, + .nr_exp_ext = 1, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = EXT_DATA_LEN, + .is_unwrit = 0 } }, + .is_zeroout_test = 1, + .nr_exp_data_segs = 3, + .exp_data_state = { { .exp_char = 'X', .off_blk = 0, .len_blk = 1 }, + { .exp_char = 0, + .off_blk = 1, + .len_blk = EXT_DATA_LEN - 2 }, + { .exp_char = 'X', + .off_blk = EXT_DATA_LEN - 1, + .len_blk = 1 } } }, +}; + +/* Tests to trigger ext4_ext_map_blocks() -> convert_initialized_extent() */ +static const struct kunit_ext_test_param test_convert_initialized_params[] = { + /* writ to unwrit splits */ + { .desc = "split writ extent to 2 extents and convert 1st half unwrit", + .type = TEST_CREATE_BLOCKS, + .split_flags = EXT4_GET_BLOCKS_CONVERT_UNWRITTEN, + .is_unwrit_at_start = 0, + .split_map = { .m_lblk = EXT_DATA_LBLK, .m_len = 1 }, + .nr_exp_ext = 2, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = 1, + .is_unwrit = 1 }, + { .ex_lblk = EXT_DATA_LBLK + 1, + .ex_len = EXT_DATA_LEN - 1, + .is_unwrit = 0 } }, + .is_zeroout_test = 0 }, + { .desc = "split writ extent to 2 extents and convert 2nd half unwrit", + .type = TEST_CREATE_BLOCKS, + .split_flags = EXT4_GET_BLOCKS_CONVERT_UNWRITTEN, + .is_unwrit_at_start = 0, + .split_map = { .m_lblk = EXT_DATA_LBLK + 1, .m_len = EXT_DATA_LEN - 1 }, + .nr_exp_ext = 2, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = 1, + .is_unwrit = 0 }, + { .ex_lblk = EXT_DATA_LBLK + 1, + .ex_len = EXT_DATA_LEN - 1, + .is_unwrit = 1 } }, + .is_zeroout_test = 0 }, + { .desc = "split writ extent to 3 extents and convert 2nd half to unwrit", + .type = TEST_CREATE_BLOCKS, + .split_flags = EXT4_GET_BLOCKS_CONVERT_UNWRITTEN, + .is_unwrit_at_start = 0, + .split_map = { .m_lblk = EXT_DATA_LBLK + 1, .m_len = EXT_DATA_LEN - 2 }, + .nr_exp_ext = 3, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = 1, + .is_unwrit = 0 }, + { .ex_lblk = EXT_DATA_LBLK + 1, + .ex_len = EXT_DATA_LEN - 2, + .is_unwrit = 1 }, + { .ex_lblk = EXT_DATA_LBLK + 1 + (EXT_DATA_LEN - 2), + .ex_len = 1, + .is_unwrit = 0 } }, + .is_zeroout_test = 0 }, + + /* writ to unwrit splits (zeroout) */ + { .desc = "split writ extent to 2 extents and convert 1st half unwrit (zeroout)", + .type = TEST_CREATE_BLOCKS, + .is_unwrit_at_start = 0, + .split_flags = EXT4_GET_BLOCKS_CONVERT_UNWRITTEN, + .split_map = { .m_lblk = EXT_DATA_LBLK, .m_len = 1 }, + .nr_exp_ext = 1, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = EXT_DATA_LEN, + .is_unwrit = 0 } }, + .is_zeroout_test = 1, + .nr_exp_data_segs = 2, + .exp_data_state = { { .exp_char = 0, .off_blk = 0, .len_blk = 1 }, + { .exp_char = 'X', + .off_blk = 1, + .len_blk = EXT_DATA_LEN - 1 } } }, + { .desc = "split writ extent to 2 extents and convert 2nd half unwrit (zeroout)", + .type = TEST_CREATE_BLOCKS, + .is_unwrit_at_start = 0, + .split_flags = EXT4_GET_BLOCKS_CONVERT_UNWRITTEN, + .split_map = { .m_lblk = EXT_DATA_LBLK + 1, .m_len = EXT_DATA_LEN - 1 }, + .nr_exp_ext = 1, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = EXT_DATA_LEN, + .is_unwrit = 0 } }, + .is_zeroout_test = 1, + .nr_exp_data_segs = 2, + .exp_data_state = { { .exp_char = 'X', .off_blk = 0, .len_blk = 1 }, + { .exp_char = 0, + .off_blk = 1, + .len_blk = EXT_DATA_LEN - 1 } } }, + { .desc = "split writ extent to 3 extents and convert 2nd half unwrit (zeroout)", + .type = TEST_CREATE_BLOCKS, + .is_unwrit_at_start = 0, + .split_flags = EXT4_GET_BLOCKS_CONVERT_UNWRITTEN, + .split_map = { .m_lblk = EXT_DATA_LBLK + 1, .m_len = EXT_DATA_LEN - 2 }, + .nr_exp_ext = 1, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = EXT_DATA_LEN, + .is_unwrit = 0 } }, + .is_zeroout_test = 1, + .nr_exp_data_segs = 3, + .exp_data_state = { { .exp_char = 'X', .off_blk = 0, .len_blk = 1 }, + { .exp_char = 0, + .off_blk = 1, + .len_blk = EXT_DATA_LEN - 2 }, + { .exp_char = 'X', + .off_blk = EXT_DATA_LEN - 1, + .len_blk = 1 } } }, +}; + +/* Tests to trigger ext4_ext_map_blocks() -> ext4_ext_handle_unwritten_exntents() */ +static const struct kunit_ext_test_param test_handle_unwritten_params[] = { + /* unwrit to writ splits via endio path */ + { .desc = "split unwrit extent to 2 extents and convert 1st half writ (endio)", + .type = TEST_CREATE_BLOCKS, + .is_unwrit_at_start = 1, + .split_flags = EXT4_GET_BLOCKS_CONVERT, + .split_map = { .m_lblk = EXT_DATA_LBLK, .m_len = 1 }, + .nr_exp_ext = 2, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = 1, + .is_unwrit = 0 }, + { .ex_lblk = EXT_DATA_LBLK + 1, + .ex_len = EXT_DATA_LEN - 1, + .is_unwrit = 1 } }, + .is_zeroout_test = 0 }, + { .desc = "split unwrit extent to 2 extents and convert 2nd half writ (endio)", + .type = TEST_CREATE_BLOCKS, + .is_unwrit_at_start = 1, + .split_flags = EXT4_GET_BLOCKS_CONVERT, + .split_map = { .m_lblk = EXT_DATA_LBLK + 1, .m_len = EXT_DATA_LEN - 1 }, + .nr_exp_ext = 2, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = 1, + .is_unwrit = 1 }, + { .ex_lblk = EXT_DATA_LBLK + 1, + .ex_len = EXT_DATA_LEN - 1, + .is_unwrit = 0 } }, + .is_zeroout_test = 0 }, + { .desc = "split unwrit extent to 3 extents and convert 2nd half to writ (endio)", + .type = TEST_CREATE_BLOCKS, + .is_unwrit_at_start = 1, + .split_flags = EXT4_GET_BLOCKS_CONVERT, + .split_map = { .m_lblk = EXT_DATA_LBLK + 1, .m_len = EXT_DATA_LEN - 2 }, + .nr_exp_ext = 3, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = 1, + .is_unwrit = 1 }, + { .ex_lblk = EXT_DATA_LBLK + 1, + .ex_len = EXT_DATA_LEN - 2, + .is_unwrit = 0 }, + { .ex_lblk = EXT_DATA_LBLK + 1 + (EXT_DATA_LEN - 2), + .ex_len = 1, + .is_unwrit = 1 } }, + .is_zeroout_test = 0 }, + + /* unwrit to writ splits via non-endio path */ + { .desc = "split unwrit extent to 2 extents and convert 1st half writ (non endio)", + .type = TEST_CREATE_BLOCKS, + .is_unwrit_at_start = 1, + .split_flags = EXT4_GET_BLOCKS_CREATE, + .split_map = { .m_lblk = EXT_DATA_LBLK, .m_len = 1 }, + .nr_exp_ext = 2, + .disable_zeroout = true, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = 1, + .is_unwrit = 0 }, + { .ex_lblk = EXT_DATA_LBLK + 1, + .ex_len = EXT_DATA_LEN - 1, + .is_unwrit = 1 } }, + .is_zeroout_test = 0 }, + { .desc = "split unwrit extent to 2 extents and convert 2nd half writ (non endio)", + .type = TEST_CREATE_BLOCKS, + .is_unwrit_at_start = 1, + .split_flags = EXT4_GET_BLOCKS_CREATE, + .split_map = { .m_lblk = EXT_DATA_LBLK + 1, .m_len = EXT_DATA_LEN - 1 }, + .nr_exp_ext = 2, + .disable_zeroout = true, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = 1, + .is_unwrit = 1 }, + { .ex_lblk = EXT_DATA_LBLK + 1, + .ex_len = EXT_DATA_LEN - 1, + .is_unwrit = 0 } }, + .is_zeroout_test = 0 }, + { .desc = "split unwrit extent to 3 extents and convert 2nd half to writ (non endio)", + .type = TEST_CREATE_BLOCKS, + .is_unwrit_at_start = 1, + .split_flags = EXT4_GET_BLOCKS_CREATE, + .split_map = { .m_lblk = EXT_DATA_LBLK + 1, .m_len = EXT_DATA_LEN - 2 }, + .nr_exp_ext = 3, + .disable_zeroout = true, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = 1, + .is_unwrit = 1 }, + { .ex_lblk = EXT_DATA_LBLK + 1, + .ex_len = EXT_DATA_LEN - 2, + .is_unwrit = 0 }, + { .ex_lblk = EXT_DATA_LBLK + 1 + (EXT_DATA_LEN - 2), + .ex_len = 1, + .is_unwrit = 1 } }, + .is_zeroout_test = 0 }, + + /* + * ***** zeroout tests ***** + */ + /* unwrit to writ splits (endio)*/ + { .desc = "split unwrit extent to 2 extents and convert 1st half writ (endio, zeroout)", + .type = TEST_CREATE_BLOCKS, + .is_unwrit_at_start = 1, + .split_flags = EXT4_GET_BLOCKS_CONVERT, + .split_map = { .m_lblk = EXT_DATA_LBLK, .m_len = 1 }, + .nr_exp_ext = 1, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = EXT_DATA_LEN, + .is_unwrit = 0 } }, + .is_zeroout_test = 1, + .nr_exp_data_segs = 2, + .exp_data_state = { { .exp_char = 'X', .off_blk = 0, .len_blk = 1 }, + { .exp_char = 0, + .off_blk = 1, + .len_blk = EXT_DATA_LEN - 1 } } }, + { .desc = "split unwrit extent to 2 extents and convert 2nd half writ (endio, zeroout)", + .type = TEST_CREATE_BLOCKS, + .is_unwrit_at_start = 1, + .split_flags = EXT4_GET_BLOCKS_CONVERT, + .split_map = { .m_lblk = EXT_DATA_LBLK + 1, .m_len = EXT_DATA_LEN - 1 }, + .nr_exp_ext = 1, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = EXT_DATA_LEN, + .is_unwrit = 0 } }, + .is_zeroout_test = 1, + .nr_exp_data_segs = 2, + .exp_data_state = { { .exp_char = 0, .off_blk = 0, .len_blk = 1 }, + { .exp_char = 'X', + .off_blk = 1, + .len_blk = EXT_DATA_LEN - 1 } } }, + { .desc = "split unwrit extent to 3 extents and convert 2nd half writ (endio, zeroout)", + .type = TEST_CREATE_BLOCKS, + .is_unwrit_at_start = 1, + .split_flags = EXT4_GET_BLOCKS_CONVERT, + .split_map = { .m_lblk = EXT_DATA_LBLK + 1, .m_len = EXT_DATA_LEN - 2 }, + .nr_exp_ext = 1, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = EXT_DATA_LEN, + .is_unwrit = 0 } }, + .is_zeroout_test = 1, + .nr_exp_data_segs = 3, + .exp_data_state = { { .exp_char = 0, .off_blk = 0, .len_blk = 1 }, + { .exp_char = 'X', + .off_blk = 1, + .len_blk = EXT_DATA_LEN - 2 }, + { .exp_char = 0, + .off_blk = EXT_DATA_LEN - 1, + .len_blk = 1 } } }, + + /* unwrit to writ splits (non-endio)*/ + { .desc = "split unwrit extent to 2 extents and convert 1st half writ (non-endio, zeroout)", + .type = TEST_CREATE_BLOCKS, + .is_unwrit_at_start = 1, + .split_flags = EXT4_GET_BLOCKS_CREATE, + .split_map = { .m_lblk = EXT_DATA_LBLK, .m_len = 1 }, + .nr_exp_ext = 1, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = EXT_DATA_LEN, + .is_unwrit = 0 } }, + .is_zeroout_test = 1, + .nr_exp_data_segs = 2, + .exp_data_state = { { .exp_char = 'X', .off_blk = 0, .len_blk = 1 }, + { .exp_char = 0, + .off_blk = 1, + .len_blk = EXT_DATA_LEN - 1 } } }, + { .desc = "split unwrit extent to 2 extents and convert 2nd half writ (non-endio, zeroout)", + .type = TEST_CREATE_BLOCKS, + .is_unwrit_at_start = 1, + .split_flags = EXT4_GET_BLOCKS_CREATE, + .split_map = { .m_lblk = EXT_DATA_LBLK + 1, .m_len = EXT_DATA_LEN - 1 }, + .nr_exp_ext = 1, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = EXT_DATA_LEN, + .is_unwrit = 0 } }, + .is_zeroout_test = 1, + .nr_exp_data_segs = 2, + .exp_data_state = { { .exp_char = 0, .off_blk = 0, .len_blk = 1 }, + { .exp_char = 'X', + .off_blk = 1, + .len_blk = EXT_DATA_LEN - 1 } } }, + { .desc = "split unwrit extent to 3 extents and convert 2nd half writ (non-endio, zeroout)", + .type = TEST_CREATE_BLOCKS, + .is_unwrit_at_start = 1, + .split_flags = EXT4_GET_BLOCKS_CREATE, + .split_map = { .m_lblk = EXT_DATA_LBLK + 1, .m_len = EXT_DATA_LEN - 2 }, + .nr_exp_ext = 1, + .exp_ext_state = { { .ex_lblk = EXT_DATA_LBLK, + .ex_len = EXT_DATA_LEN, + .is_unwrit = 0 } }, + .is_zeroout_test = 1, + .nr_exp_data_segs = 3, + .exp_data_state = { { .exp_char = 0, .off_blk = 0, .len_blk = 1 }, + { .exp_char = 'X', + .off_blk = 1, + .len_blk = EXT_DATA_LEN - 2 }, + { .exp_char = 0, + .off_blk = EXT_DATA_LEN - 1, + .len_blk = 1 } } }, +}; + +static void ext_get_desc(struct kunit *test, const void *p, char *desc) + +{ + struct kunit_ext_test_param *param = (struct kunit_ext_test_param *)p; + + snprintf(desc, KUNIT_PARAM_DESC_SIZE, "%s %s\n", param->desc, + (param->type & TEST_CREATE_BLOCKS) ? "(highlevel)" : ""); +} + +static int test_split_convert_param_init(struct kunit *test) +{ + size_t arr_size = ARRAY_SIZE(test_split_convert_params); + + kunit_register_params_array(test, test_split_convert_params, arr_size, + ext_get_desc); + return 0; +} + +static int test_convert_initialized_param_init(struct kunit *test) +{ + size_t arr_size = ARRAY_SIZE(test_convert_initialized_params); + + kunit_register_params_array(test, test_convert_initialized_params, + arr_size, ext_get_desc); + return 0; +} + +static int test_handle_unwritten_init(struct kunit *test) +{ + size_t arr_size = ARRAY_SIZE(test_handle_unwritten_params); + + kunit_register_params_array(test, test_handle_unwritten_params, + arr_size, ext_get_desc); + return 0; +} + +/* + * Note that we use KUNIT_CASE_PARAM_WITH_INIT() instead of the more compact + * KUNIT_ARRAY_PARAM() because the later currently has a limitation causing the + * output parsing to be prone to error. For more context: + * + * https://lore.kernel.org/linux-kselftest/aULJpTvJDw9ctUDe@li-dc0c254c-257c-11b2-a85c-98b6c1322444.ibm.com/ + */ +static struct kunit_case extents_test_cases[] = { + KUNIT_CASE_PARAM_WITH_INIT(test_split_convert, kunit_array_gen_params, + test_split_convert_param_init, NULL), + KUNIT_CASE_PARAM_WITH_INIT(test_split_convert, kunit_array_gen_params, + test_convert_initialized_param_init, NULL), + KUNIT_CASE_PARAM_WITH_INIT(test_split_convert, kunit_array_gen_params, + test_handle_unwritten_init, NULL), + {} +}; + +static struct kunit_suite extents_test_suite = { + .name = "ext4_extents_test", + .init = extents_kunit_init, + .exit = extents_kunit_exit, + .test_cases = extents_test_cases, +}; + +kunit_test_suites(&extents_test_suite); + +MODULE_LICENSE("GPL"); diff --git a/fs/ext4/extents.c b/fs/ext4/extents.c new file mode 100644 index 000000000..76038b6c3 --- /dev/null +++ b/fs/ext4/extents.c @@ -0,0 +1,6420 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2003-2006, Cluster File Systems, Inc, info@clusterfs.com + * Written by Alex Tomas + * + * Architecture independence: + * Copyright (c) 2005, Bull S.A. + * Written by Pierre Peiffer + */ + +/* + * Extents support for EXT4 + * + * TODO: + * - ext4*_error() should be used in some situations + * - analyze all BUG()/BUG_ON(), use -EIO where appropriate + * - smart tree reduction + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ext4_jbd2.h" +#include "ext4_extents.h" +#include "xattr.h" +#include + +#include + +/* + * used by extent splitting. + */ +#define EXT4_EXT_MAY_ZEROOUT 0x1 /* safe to zeroout if split fails \ + due to ENOSPC */ +static struct ext4_ext_path *ext4_split_convert_extents( + handle_t *handle, struct inode *inode, struct ext4_map_blocks *map, + struct ext4_ext_path *path, int flags, unsigned int *allocated); + +static __le32 ext4_extent_block_csum(struct inode *inode, + struct ext4_extent_header *eh) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + __u32 csum; + + csum = ext4_chksum(ei->i_csum_seed, (__u8 *)eh, + EXT4_EXTENT_TAIL_OFFSET(eh)); + return cpu_to_le32(csum); +} + +static int ext4_extent_block_csum_verify(struct inode *inode, + struct ext4_extent_header *eh) +{ + struct ext4_extent_tail *et; + + if (!ext4_has_feature_metadata_csum(inode->i_sb)) + return 1; + + et = find_ext4_extent_tail(eh); + if (et->et_checksum != ext4_extent_block_csum(inode, eh)) + return 0; + return 1; +} + +static void ext4_extent_block_csum_set(struct inode *inode, + struct ext4_extent_header *eh) +{ + struct ext4_extent_tail *et; + + if (!ext4_has_feature_metadata_csum(inode->i_sb)) + return; + + et = find_ext4_extent_tail(eh); + et->et_checksum = ext4_extent_block_csum(inode, eh); +} + +static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, + struct inode *inode, + struct ext4_ext_path *path, + ext4_lblk_t split, int flags); + +static int ext4_ext_trunc_restart_fn(struct inode *inode, int *dropped) +{ + /* + * Drop i_data_sem to avoid deadlock with ext4_map_blocks. At this + * moment, get_block can be called only for blocks inside i_size since + * page cache has been already dropped and writes are blocked by + * i_rwsem. So we can safely drop the i_data_sem here. + */ + BUG_ON(EXT4_JOURNAL(inode) == NULL); + ext4_discard_preallocations(inode); + up_write(&EXT4_I(inode)->i_data_sem); + *dropped = 1; + return 0; +} + +static inline void ext4_ext_path_brelse(struct ext4_ext_path *path) +{ + brelse(path->p_bh); + path->p_bh = NULL; +} + +static void ext4_ext_drop_refs(struct ext4_ext_path *path) +{ + int depth, i; + + if (IS_ERR_OR_NULL(path)) + return; + depth = path->p_depth; + for (i = 0; i <= depth; i++, path++) + ext4_ext_path_brelse(path); +} + +void ext4_free_ext_path(struct ext4_ext_path *path) +{ + if (IS_ERR_OR_NULL(path)) + return; + ext4_ext_drop_refs(path); + kfree(path); +} + +/* + * Make sure 'handle' has at least 'check_cred' credits. If not, restart + * transaction with 'restart_cred' credits. The function drops i_data_sem + * when restarting transaction and gets it after transaction is restarted. + * + * The function returns 0 on success, 1 if transaction had to be restarted, + * and < 0 in case of fatal error. + */ +int ext4_datasem_ensure_credits(handle_t *handle, struct inode *inode, + int check_cred, int restart_cred, + int revoke_cred) +{ + int ret; + int dropped = 0; + + ret = ext4_journal_ensure_credits_fn(handle, check_cred, restart_cred, + revoke_cred, ext4_ext_trunc_restart_fn(inode, &dropped)); + if (dropped) + down_write(&EXT4_I(inode)->i_data_sem); + return ret; +} + +/* + * could return: + * - EROFS + * - ENOMEM + */ +static int ext4_ext_get_access(handle_t *handle, struct inode *inode, + struct ext4_ext_path *path) +{ + int err = 0; + + if (path->p_bh) { + /* path points to block */ + BUFFER_TRACE(path->p_bh, "get_write_access"); + err = ext4_journal_get_write_access(handle, inode->i_sb, + path->p_bh, EXT4_JTR_NONE); + /* + * The extent buffer's verified bit will be set again in + * __ext4_ext_dirty(). We could leave an inconsistent + * buffer if the extents updating procudure break off du + * to some error happens, force to check it again. + */ + if (!err) + clear_buffer_verified(path->p_bh); + } + /* path points to leaf/index in inode body */ + /* we use in-core data, no need to protect them */ + return err; +} + +/* + * could return: + * - EROFS + * - ENOMEM + * - EIO + */ +int __ext4_ext_dirty(const char *where, unsigned int line, + handle_t *handle, struct inode *inode, + struct ext4_ext_path *path) +{ + int err; + + KUNIT_STATIC_STUB_REDIRECT(__ext4_ext_dirty, where, line, handle, inode, + path); + + WARN_ON(!rwsem_is_locked(&EXT4_I(inode)->i_data_sem)); + if (path->p_bh) { + ext4_extent_block_csum_set(inode, ext_block_hdr(path->p_bh)); + /* path points to block */ + err = __ext4_handle_dirty_metadata(where, line, handle, + inode, path->p_bh); + /* Extents updating done, re-set verified flag */ + if (!err) + set_buffer_verified(path->p_bh); + } else { + /* path points to leaf/index in inode body */ + err = ext4_mark_inode_dirty(handle, inode); + } + return err; +} + +#define ext4_ext_dirty(handle, inode, path) \ + __ext4_ext_dirty(__func__, __LINE__, (handle), (inode), (path)) + +static ext4_fsblk_t ext4_ext_find_goal(struct inode *inode, + struct ext4_ext_path *path, + ext4_lblk_t block) +{ + if (path) { + int depth = path->p_depth; + struct ext4_extent *ex; + + /* + * Try to predict block placement assuming that we are + * filling in a file which will eventually be + * non-sparse --- i.e., in the case of libbfd writing + * an ELF object sections out-of-order but in a way + * the eventually results in a contiguous object or + * executable file, or some database extending a table + * space file. However, this is actually somewhat + * non-ideal if we are writing a sparse file such as + * qemu or KVM writing a raw image file that is going + * to stay fairly sparse, since it will end up + * fragmenting the file system's free space. Maybe we + * should have some hueristics or some way to allow + * userspace to pass a hint to file system, + * especially if the latter case turns out to be + * common. + */ + ex = path[depth].p_ext; + if (ex) { + ext4_fsblk_t ext_pblk = ext4_ext_pblock(ex); + ext4_lblk_t ext_block = le32_to_cpu(ex->ee_block); + + if (block > ext_block) + return ext_pblk + (block - ext_block); + else + return ext_pblk - (ext_block - block); + } + + /* it looks like index is empty; + * try to find starting block from index itself */ + if (path[depth].p_bh) + return path[depth].p_bh->b_blocknr; + } + + /* OK. use inode's group */ + return ext4_inode_to_goal_block(inode); +} + +/* + * Allocation for a meta data block + */ +static ext4_fsblk_t +ext4_ext_new_meta_block(handle_t *handle, struct inode *inode, + struct ext4_ext_path *path, + struct ext4_extent *ex, int *err, unsigned int flags) +{ + ext4_fsblk_t goal, newblock; + + goal = ext4_ext_find_goal(inode, path, le32_to_cpu(ex->ee_block)); + newblock = ext4_new_meta_blocks(handle, inode, goal, flags, + NULL, err); + return newblock; +} + +static inline int ext4_ext_space_block(struct inode *inode, int check) +{ + int size; + + size = (inode->i_sb->s_blocksize - sizeof(struct ext4_extent_header)) + / sizeof(struct ext4_extent); +#ifdef AGGRESSIVE_TEST + if (!check && size > 6) + size = 6; +#endif + return size; +} + +static inline int ext4_ext_space_block_idx(struct inode *inode, int check) +{ + int size; + + size = (inode->i_sb->s_blocksize - sizeof(struct ext4_extent_header)) + / sizeof(struct ext4_extent_idx); +#ifdef AGGRESSIVE_TEST + if (!check && size > 5) + size = 5; +#endif + return size; +} + +static inline int ext4_ext_space_root(struct inode *inode, int check) +{ + int size; + + size = sizeof(EXT4_I(inode)->i_data); + size -= sizeof(struct ext4_extent_header); + size /= sizeof(struct ext4_extent); +#ifdef AGGRESSIVE_TEST + if (!check && size > 3) + size = 3; +#endif + return size; +} + +static inline int ext4_ext_space_root_idx(struct inode *inode, int check) +{ + int size; + + size = sizeof(EXT4_I(inode)->i_data); + size -= sizeof(struct ext4_extent_header); + size /= sizeof(struct ext4_extent_idx); +#ifdef AGGRESSIVE_TEST + if (!check && size > 4) + size = 4; +#endif + return size; +} + +static inline struct ext4_ext_path * +ext4_force_split_extent_at(handle_t *handle, struct inode *inode, + struct ext4_ext_path *path, ext4_lblk_t lblk, + int nofail) +{ + int flags = EXT4_EX_NOCACHE | EXT4_GET_BLOCKS_SPLIT_NOMERGE; + + if (nofail) + flags |= EXT4_GET_BLOCKS_METADATA_NOFAIL | EXT4_EX_NOFAIL; + + return ext4_split_extent_at(handle, inode, path, lblk, flags); +} + +static int +ext4_ext_max_entries(struct inode *inode, int depth) +{ + int max; + + if (depth == ext_depth(inode)) { + if (depth == 0) + max = ext4_ext_space_root(inode, 1); + else + max = ext4_ext_space_root_idx(inode, 1); + } else { + if (depth == 0) + max = ext4_ext_space_block(inode, 1); + else + max = ext4_ext_space_block_idx(inode, 1); + } + + return max; +} + +static int ext4_valid_extent(struct inode *inode, struct ext4_extent *ext) +{ + ext4_fsblk_t block = ext4_ext_pblock(ext); + int len = ext4_ext_get_actual_len(ext); + ext4_lblk_t lblock = le32_to_cpu(ext->ee_block); + + /* + * We allow neither: + * - zero length + * - overflow/wrap-around + */ + if (lblock + len <= lblock) + return 0; + return ext4_inode_block_valid(inode, block, len); +} + +static int ext4_valid_extent_idx(struct inode *inode, + struct ext4_extent_idx *ext_idx) +{ + ext4_fsblk_t block = ext4_idx_pblock(ext_idx); + + return ext4_inode_block_valid(inode, block, 1); +} + +static int ext4_valid_extent_entries(struct inode *inode, + struct ext4_extent_header *eh, + ext4_lblk_t lblk, ext4_fsblk_t *pblk, + int depth) +{ + unsigned short entries; + ext4_lblk_t lblock = 0; + ext4_lblk_t cur = 0; + + if (eh->eh_entries == 0) + return 1; + + entries = le16_to_cpu(eh->eh_entries); + + if (depth == 0) { + /* leaf entries */ + struct ext4_extent *ext = EXT_FIRST_EXTENT(eh); + + /* + * The logical block in the first entry should equal to + * the number in the index block. + */ + if (depth != ext_depth(inode) && + lblk != le32_to_cpu(ext->ee_block)) + return 0; + while (entries) { + if (!ext4_valid_extent(inode, ext)) + return 0; + + /* Check for overlapping extents */ + lblock = le32_to_cpu(ext->ee_block); + if (lblock < cur) { + *pblk = ext4_ext_pblock(ext); + return 0; + } + cur = lblock + ext4_ext_get_actual_len(ext); + ext++; + entries--; + } + } else { + struct ext4_extent_idx *ext_idx = EXT_FIRST_INDEX(eh); + + /* + * The logical block in the first entry should equal to + * the number in the parent index block. + */ + if (depth != ext_depth(inode) && + lblk != le32_to_cpu(ext_idx->ei_block)) + return 0; + while (entries) { + if (!ext4_valid_extent_idx(inode, ext_idx)) + return 0; + + /* Check for overlapping index extents */ + lblock = le32_to_cpu(ext_idx->ei_block); + if (lblock < cur) { + *pblk = ext4_idx_pblock(ext_idx); + return 0; + } + ext_idx++; + entries--; + cur = lblock + 1; + } + } + return 1; +} + +static int __ext4_ext_check(const char *function, unsigned int line, + struct inode *inode, struct ext4_extent_header *eh, + int depth, ext4_fsblk_t pblk, ext4_lblk_t lblk) +{ + const char *error_msg; + int max = 0, err = -EFSCORRUPTED; + + if (unlikely(eh->eh_magic != EXT4_EXT_MAGIC)) { + error_msg = "invalid magic"; + goto corrupted; + } + if (unlikely(le16_to_cpu(eh->eh_depth) != depth)) { + error_msg = "unexpected eh_depth"; + goto corrupted; + } + if (unlikely(eh->eh_max == 0)) { + error_msg = "invalid eh_max"; + goto corrupted; + } + max = ext4_ext_max_entries(inode, depth); + if (unlikely(le16_to_cpu(eh->eh_max) > max)) { + error_msg = "too large eh_max"; + goto corrupted; + } + if (unlikely(le16_to_cpu(eh->eh_entries) > le16_to_cpu(eh->eh_max))) { + error_msg = "invalid eh_entries"; + goto corrupted; + } + if (unlikely((eh->eh_entries == 0) && (depth > 0))) { + error_msg = "eh_entries is 0 but eh_depth is > 0"; + goto corrupted; + } + if (!ext4_valid_extent_entries(inode, eh, lblk, &pblk, depth)) { + error_msg = "invalid extent entries"; + goto corrupted; + } + if (unlikely(depth > 32)) { + error_msg = "too large eh_depth"; + goto corrupted; + } + /* Verify checksum on non-root extent tree nodes */ + if (ext_depth(inode) != depth && + !ext4_extent_block_csum_verify(inode, eh)) { + error_msg = "extent tree corrupted"; + err = -EFSBADCRC; + goto corrupted; + } + return 0; + +corrupted: + ext4_error_inode_err(inode, function, line, 0, -err, + "pblk %llu bad header/extent: %s - magic %x, " + "entries %u, max %u(%u), depth %u(%u)", + (unsigned long long) pblk, error_msg, + le16_to_cpu(eh->eh_magic), + le16_to_cpu(eh->eh_entries), + le16_to_cpu(eh->eh_max), + max, le16_to_cpu(eh->eh_depth), depth); + return err; +} + +#define ext4_ext_check(inode, eh, depth, pblk) \ + __ext4_ext_check(__func__, __LINE__, (inode), (eh), (depth), (pblk), 0) + +int ext4_ext_check_inode(struct inode *inode) +{ + return ext4_ext_check(inode, ext_inode_hdr(inode), ext_depth(inode), 0); +} + +static void ext4_cache_extents(struct inode *inode, + struct ext4_extent_header *eh) +{ + struct ext4_extent *ex = EXT_FIRST_EXTENT(eh); + ext4_lblk_t prev = 0; + int i; + + KUNIT_STATIC_STUB_REDIRECT(ext4_cache_extents, inode, eh); + + for (i = le16_to_cpu(eh->eh_entries); i > 0; i--, ex++) { + unsigned int status = EXTENT_STATUS_WRITTEN; + ext4_lblk_t lblk = le32_to_cpu(ex->ee_block); + int len = ext4_ext_get_actual_len(ex); + + if (prev && (prev != lblk)) + ext4_es_cache_extent(inode, prev, lblk - prev, ~0, + EXTENT_STATUS_HOLE); + + if (ext4_ext_is_unwritten(ex)) + status = EXTENT_STATUS_UNWRITTEN; + ext4_es_cache_extent(inode, lblk, len, + ext4_ext_pblock(ex), status); + prev = lblk + len; + } +} + +static struct buffer_head * +__read_extent_tree_block(const char *function, unsigned int line, + struct inode *inode, struct ext4_extent_idx *idx, + int depth, int flags) +{ + struct buffer_head *bh; + int err; + gfp_t gfp_flags = __GFP_MOVABLE | GFP_NOFS; + ext4_fsblk_t pblk; + + if (flags & EXT4_EX_NOFAIL) + gfp_flags |= __GFP_NOFAIL; + + pblk = ext4_idx_pblock(idx); + bh = sb_getblk_gfp(inode->i_sb, pblk, gfp_flags); + if (unlikely(!bh)) + return ERR_PTR(-ENOMEM); + + if (!bh_uptodate_or_lock(bh)) { + trace_ext4_ext_load_extent(inode, pblk, _RET_IP_); + err = ext4_read_bh(bh, 0, NULL, false); + if (err < 0) + goto errout; + } + if (buffer_verified(bh) && !(flags & EXT4_EX_FORCE_CACHE)) + return bh; + err = __ext4_ext_check(function, line, inode, ext_block_hdr(bh), + depth, pblk, le32_to_cpu(idx->ei_block)); + if (err) + goto errout; + set_buffer_verified(bh); + /* + * If this is a leaf block, cache all of its entries + */ + if (!(flags & EXT4_EX_NOCACHE) && depth == 0) { + struct ext4_extent_header *eh = ext_block_hdr(bh); + ext4_cache_extents(inode, eh); + } + return bh; +errout: + put_bh(bh); + return ERR_PTR(err); + +} + +#define read_extent_tree_block(inode, idx, depth, flags) \ + __read_extent_tree_block(__func__, __LINE__, (inode), (idx), \ + (depth), (flags)) + +/* + * This function is called to cache a file's extent information in the + * extent status tree + */ +int ext4_ext_precache(struct inode *inode) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + struct ext4_ext_path *path = NULL; + struct buffer_head *bh; + int i = 0, depth, ret = 0; + + if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) + return 0; /* not an extent-mapped inode */ + + ext4_check_map_extents_env(inode); + + down_read(&ei->i_data_sem); + depth = ext_depth(inode); + + /* Don't cache anything if there are no external extent blocks */ + if (!depth) { + up_read(&ei->i_data_sem); + return ret; + } + + path = kzalloc_objs(struct ext4_ext_path, depth + 1, GFP_NOFS); + if (path == NULL) { + up_read(&ei->i_data_sem); + return -ENOMEM; + } + + path[0].p_hdr = ext_inode_hdr(inode); + ret = ext4_ext_check(inode, path[0].p_hdr, depth, 0); + if (ret) + goto out; + path[0].p_idx = EXT_FIRST_INDEX(path[0].p_hdr); + while (i >= 0) { + /* + * If this is a leaf block or we've reached the end of + * the index block, go up + */ + if ((i == depth) || + path[i].p_idx > EXT_LAST_INDEX(path[i].p_hdr)) { + ext4_ext_path_brelse(path + i); + i--; + continue; + } + bh = read_extent_tree_block(inode, path[i].p_idx++, + depth - i - 1, + EXT4_EX_FORCE_CACHE); + if (IS_ERR(bh)) { + ret = PTR_ERR(bh); + break; + } + i++; + path[i].p_bh = bh; + path[i].p_hdr = ext_block_hdr(bh); + path[i].p_idx = EXT_FIRST_INDEX(path[i].p_hdr); + } + ext4_set_inode_state(inode, EXT4_STATE_EXT_PRECACHED); +out: + up_read(&ei->i_data_sem); + ext4_free_ext_path(path); + return ret; +} + +#ifdef EXT_DEBUG +static void ext4_ext_show_path(struct inode *inode, struct ext4_ext_path *path) +{ + int k, l = path->p_depth; + + ext_debug(inode, "path:"); + for (k = 0; k <= l; k++, path++) { + if (path->p_idx) { + ext_debug(inode, " %d->%llu", + le32_to_cpu(path->p_idx->ei_block), + ext4_idx_pblock(path->p_idx)); + } else if (path->p_ext) { + ext_debug(inode, " %d:[%d]%d:%llu ", + le32_to_cpu(path->p_ext->ee_block), + ext4_ext_is_unwritten(path->p_ext), + ext4_ext_get_actual_len(path->p_ext), + ext4_ext_pblock(path->p_ext)); + } else + ext_debug(inode, " []"); + } + ext_debug(inode, "\n"); +} + +static void ext4_ext_show_leaf(struct inode *inode, struct ext4_ext_path *path) +{ + int depth = ext_depth(inode); + struct ext4_extent_header *eh; + struct ext4_extent *ex; + int i; + + if (IS_ERR_OR_NULL(path)) + return; + + eh = path[depth].p_hdr; + ex = EXT_FIRST_EXTENT(eh); + + ext_debug(inode, "Displaying leaf extents\n"); + + for (i = 0; i < le16_to_cpu(eh->eh_entries); i++, ex++) { + ext_debug(inode, "%d:[%d]%d:%llu ", le32_to_cpu(ex->ee_block), + ext4_ext_is_unwritten(ex), + ext4_ext_get_actual_len(ex), ext4_ext_pblock(ex)); + } + ext_debug(inode, "\n"); +} + +static void ext4_ext_show_move(struct inode *inode, struct ext4_ext_path *path, + ext4_fsblk_t newblock, int level) +{ + int depth = ext_depth(inode); + struct ext4_extent *ex; + + if (depth != level) { + struct ext4_extent_idx *idx; + idx = path[level].p_idx; + while (idx <= EXT_MAX_INDEX(path[level].p_hdr)) { + ext_debug(inode, "%d: move %d:%llu in new index %llu\n", + level, le32_to_cpu(idx->ei_block), + ext4_idx_pblock(idx), newblock); + idx++; + } + + return; + } + + ex = path[depth].p_ext; + while (ex <= EXT_MAX_EXTENT(path[depth].p_hdr)) { + ext_debug(inode, "move %d:%llu:[%d]%d in new leaf %llu\n", + le32_to_cpu(ex->ee_block), + ext4_ext_pblock(ex), + ext4_ext_is_unwritten(ex), + ext4_ext_get_actual_len(ex), + newblock); + ex++; + } +} + +#else +#define ext4_ext_show_path(inode, path) +#define ext4_ext_show_leaf(inode, path) +#define ext4_ext_show_move(inode, path, newblock, level) +#endif + +/* + * ext4_ext_binsearch_idx: + * binary search for the closest index of the given block + * the header must be checked before calling this + */ +static void +ext4_ext_binsearch_idx(struct inode *inode, + struct ext4_ext_path *path, ext4_lblk_t block) +{ + struct ext4_extent_header *eh = path->p_hdr; + struct ext4_extent_idx *r, *l, *m; + + + ext_debug(inode, "binsearch for %u(idx): ", block); + + l = EXT_FIRST_INDEX(eh) + 1; + r = EXT_LAST_INDEX(eh); + while (l <= r) { + m = l + (r - l) / 2; + ext_debug(inode, "%p(%u):%p(%u):%p(%u) ", l, + le32_to_cpu(l->ei_block), m, le32_to_cpu(m->ei_block), + r, le32_to_cpu(r->ei_block)); + + if (block < le32_to_cpu(m->ei_block)) + r = m - 1; + else + l = m + 1; + } + + path->p_idx = l - 1; + ext_debug(inode, " -> %u->%lld ", le32_to_cpu(path->p_idx->ei_block), + ext4_idx_pblock(path->p_idx)); + +#ifdef CHECK_BINSEARCH + { + struct ext4_extent_idx *chix, *ix; + int k; + + chix = ix = EXT_FIRST_INDEX(eh); + for (k = 0; k < le16_to_cpu(eh->eh_entries); k++, ix++) { + if (k != 0 && le32_to_cpu(ix->ei_block) <= + le32_to_cpu(ix[-1].ei_block)) { + printk(KERN_DEBUG "k=%d, ix=0x%p, " + "first=0x%p\n", k, + ix, EXT_FIRST_INDEX(eh)); + printk(KERN_DEBUG "%u <= %u\n", + le32_to_cpu(ix->ei_block), + le32_to_cpu(ix[-1].ei_block)); + } + BUG_ON(k && le32_to_cpu(ix->ei_block) + <= le32_to_cpu(ix[-1].ei_block)); + if (block < le32_to_cpu(ix->ei_block)) + break; + chix = ix; + } + BUG_ON(chix != path->p_idx); + } +#endif + +} + +/* + * ext4_ext_binsearch: + * binary search for closest extent of the given block + * the header must be checked before calling this + */ +static void +ext4_ext_binsearch(struct inode *inode, + struct ext4_ext_path *path, ext4_lblk_t block) +{ + struct ext4_extent_header *eh = path->p_hdr; + struct ext4_extent *r, *l, *m; + + if (eh->eh_entries == 0) { + /* + * this leaf is empty: + * we get such a leaf in split/add case + */ + return; + } + + ext_debug(inode, "binsearch for %u: ", block); + + l = EXT_FIRST_EXTENT(eh) + 1; + r = EXT_LAST_EXTENT(eh); + + while (l <= r) { + m = l + (r - l) / 2; + ext_debug(inode, "%p(%u):%p(%u):%p(%u) ", l, + le32_to_cpu(l->ee_block), m, le32_to_cpu(m->ee_block), + r, le32_to_cpu(r->ee_block)); + + if (block < le32_to_cpu(m->ee_block)) + r = m - 1; + else + l = m + 1; + } + + path->p_ext = l - 1; + ext_debug(inode, " -> %d:%llu:[%d]%d ", + le32_to_cpu(path->p_ext->ee_block), + ext4_ext_pblock(path->p_ext), + ext4_ext_is_unwritten(path->p_ext), + ext4_ext_get_actual_len(path->p_ext)); + +#ifdef CHECK_BINSEARCH + { + struct ext4_extent *chex, *ex; + int k; + + chex = ex = EXT_FIRST_EXTENT(eh); + for (k = 0; k < le16_to_cpu(eh->eh_entries); k++, ex++) { + BUG_ON(k && le32_to_cpu(ex->ee_block) + <= le32_to_cpu(ex[-1].ee_block)); + if (block < le32_to_cpu(ex->ee_block)) + break; + chex = ex; + } + BUG_ON(chex != path->p_ext); + } +#endif + +} + +void ext4_ext_tree_init(handle_t *handle, struct inode *inode) +{ + struct ext4_extent_header *eh; + + eh = ext_inode_hdr(inode); + eh->eh_depth = 0; + eh->eh_entries = 0; + eh->eh_magic = EXT4_EXT_MAGIC; + eh->eh_max = cpu_to_le16(ext4_ext_space_root(inode, 0)); + eh->eh_generation = 0; + ext4_mark_inode_dirty(handle, inode); +} + +struct ext4_ext_path * +ext4_find_extent(struct inode *inode, ext4_lblk_t block, + struct ext4_ext_path *path, int flags) +{ + struct ext4_extent_header *eh; + struct buffer_head *bh; + short int depth, i, ppos = 0; + int ret; + gfp_t gfp_flags = GFP_NOFS; + + KUNIT_STATIC_STUB_REDIRECT(ext4_find_extent, inode, block, path, flags); + + if (flags & EXT4_EX_NOFAIL) + gfp_flags |= __GFP_NOFAIL; + + eh = ext_inode_hdr(inode); + depth = ext_depth(inode); + if (depth < 0 || depth > EXT4_MAX_EXTENT_DEPTH) { + EXT4_ERROR_INODE(inode, "inode has invalid extent depth: %d", + depth); + ret = -EFSCORRUPTED; + goto err; + } + + if (path) { + ext4_ext_drop_refs(path); + if (depth > path[0].p_maxdepth) { + kfree(path); + path = NULL; + } + } + if (!path) { + /* account possible depth increase */ + path = kzalloc_objs(struct ext4_ext_path, depth + 2, gfp_flags); + if (unlikely(!path)) + return ERR_PTR(-ENOMEM); + path[0].p_maxdepth = depth + 1; + } + path[0].p_hdr = eh; + path[0].p_bh = NULL; + + i = depth; + if (!(flags & EXT4_EX_NOCACHE) && depth == 0) + ext4_cache_extents(inode, eh); + /* walk through the tree */ + while (i) { + ext_debug(inode, "depth %d: num %d, max %d\n", + ppos, le16_to_cpu(eh->eh_entries), le16_to_cpu(eh->eh_max)); + + ext4_ext_binsearch_idx(inode, path + ppos, block); + path[ppos].p_block = ext4_idx_pblock(path[ppos].p_idx); + path[ppos].p_depth = i; + path[ppos].p_ext = NULL; + + bh = read_extent_tree_block(inode, path[ppos].p_idx, --i, flags); + if (IS_ERR(bh)) { + ret = PTR_ERR(bh); + goto err; + } + + eh = ext_block_hdr(bh); + ppos++; + path[ppos].p_bh = bh; + path[ppos].p_hdr = eh; + } + + path[ppos].p_depth = i; + path[ppos].p_ext = NULL; + path[ppos].p_idx = NULL; + + /* find extent */ + ext4_ext_binsearch(inode, path + ppos, block); + /* if not an empty leaf */ + if (path[ppos].p_ext) + path[ppos].p_block = ext4_ext_pblock(path[ppos].p_ext); + + ext4_ext_show_path(inode, path); + + return path; + +err: + ext4_free_ext_path(path); + return ERR_PTR(ret); +} + +/* + * ext4_ext_insert_index: + * insert new index [@logical;@ptr] into the block at @curp; + * check where to insert: before @curp or after @curp + */ +static int ext4_ext_insert_index(handle_t *handle, struct inode *inode, + struct ext4_ext_path *curp, + int logical, ext4_fsblk_t ptr) +{ + struct ext4_extent_idx *ix; + int len, err; + + err = ext4_ext_get_access(handle, inode, curp); + if (err) + return err; + + if (unlikely(logical == le32_to_cpu(curp->p_idx->ei_block))) { + EXT4_ERROR_INODE(inode, + "logical %d == ei_block %d!", + logical, le32_to_cpu(curp->p_idx->ei_block)); + return -EFSCORRUPTED; + } + + if (unlikely(le16_to_cpu(curp->p_hdr->eh_entries) + >= le16_to_cpu(curp->p_hdr->eh_max))) { + EXT4_ERROR_INODE(inode, + "eh_entries %d >= eh_max %d!", + le16_to_cpu(curp->p_hdr->eh_entries), + le16_to_cpu(curp->p_hdr->eh_max)); + return -EFSCORRUPTED; + } + + if (logical > le32_to_cpu(curp->p_idx->ei_block)) { + /* insert after */ + ext_debug(inode, "insert new index %d after: %llu\n", + logical, ptr); + ix = curp->p_idx + 1; + } else { + /* insert before */ + ext_debug(inode, "insert new index %d before: %llu\n", + logical, ptr); + ix = curp->p_idx; + } + + if (unlikely(ix > EXT_MAX_INDEX(curp->p_hdr))) { + EXT4_ERROR_INODE(inode, "ix > EXT_MAX_INDEX!"); + return -EFSCORRUPTED; + } + + len = EXT_LAST_INDEX(curp->p_hdr) - ix + 1; + BUG_ON(len < 0); + if (len > 0) { + ext_debug(inode, "insert new index %d: " + "move %d indices from 0x%p to 0x%p\n", + logical, len, ix, ix + 1); + memmove(ix + 1, ix, len * sizeof(struct ext4_extent_idx)); + } + + ix->ei_block = cpu_to_le32(logical); + ext4_idx_store_pblock(ix, ptr); + le16_add_cpu(&curp->p_hdr->eh_entries, 1); + + if (unlikely(ix > EXT_LAST_INDEX(curp->p_hdr))) { + EXT4_ERROR_INODE(inode, "ix > EXT_LAST_INDEX!"); + return -EFSCORRUPTED; + } + + err = ext4_ext_dirty(handle, inode, curp); + ext4_std_error(inode->i_sb, err); + + return err; +} + +/* + * ext4_ext_split: + * inserts new subtree into the path, using free index entry + * at depth @at: + * - allocates all needed blocks (new leaf and all intermediate index blocks) + * - makes decision where to split + * - moves remaining extents and index entries (right to the split point) + * into the newly allocated blocks + * - initializes subtree + */ +static int ext4_ext_split(handle_t *handle, struct inode *inode, + unsigned int flags, + struct ext4_ext_path *path, + struct ext4_extent *newext, int at) +{ + struct buffer_head *bh = NULL; + int depth = ext_depth(inode); + struct ext4_extent_header *neh; + struct ext4_extent_idx *fidx; + int i = at, k, m, a; + ext4_fsblk_t newblock, oldblock; + __le32 border; + ext4_fsblk_t *ablocks = NULL; /* array of allocated blocks */ + gfp_t gfp_flags = GFP_NOFS; + int err = 0; + size_t ext_size = 0; + + if (flags & EXT4_EX_NOFAIL) + gfp_flags |= __GFP_NOFAIL; + + /* make decision: where to split? */ + /* FIXME: now decision is simplest: at current extent */ + + /* if current leaf will be split, then we should use + * border from split point */ + if (unlikely(path[depth].p_ext > EXT_MAX_EXTENT(path[depth].p_hdr))) { + EXT4_ERROR_INODE(inode, "p_ext > EXT_MAX_EXTENT!"); + return -EFSCORRUPTED; + } + if (path[depth].p_ext != EXT_MAX_EXTENT(path[depth].p_hdr)) { + border = path[depth].p_ext[1].ee_block; + ext_debug(inode, "leaf will be split." + " next leaf starts at %d\n", + le32_to_cpu(border)); + } else { + border = newext->ee_block; + ext_debug(inode, "leaf will be added." + " next leaf starts at %d\n", + le32_to_cpu(border)); + } + + /* + * If error occurs, then we break processing + * and mark filesystem read-only. index won't + * be inserted and tree will be in consistent + * state. Next mount will repair buffers too. + */ + + /* + * Get array to track all allocated blocks. + * We need this to handle errors and free blocks + * upon them. + */ + ablocks = kzalloc_objs(ext4_fsblk_t, depth, gfp_flags); + if (!ablocks) + return -ENOMEM; + + /* allocate all needed blocks */ + ext_debug(inode, "allocate %d blocks for indexes/leaf\n", depth - at); + for (a = 0; a < depth - at; a++) { + newblock = ext4_ext_new_meta_block(handle, inode, path, + newext, &err, flags); + if (newblock == 0) + goto cleanup; + ablocks[a] = newblock; + } + + /* initialize new leaf */ + newblock = ablocks[--a]; + if (unlikely(newblock == 0)) { + EXT4_ERROR_INODE(inode, "newblock == 0!"); + err = -EFSCORRUPTED; + goto cleanup; + } + bh = sb_getblk_gfp(inode->i_sb, newblock, __GFP_MOVABLE | GFP_NOFS); + if (unlikely(!bh)) { + err = -ENOMEM; + goto cleanup; + } + lock_buffer(bh); + + err = ext4_journal_get_create_access(handle, inode->i_sb, bh, + EXT4_JTR_NONE); + if (err) + goto cleanup; + + neh = ext_block_hdr(bh); + neh->eh_entries = 0; + neh->eh_max = cpu_to_le16(ext4_ext_space_block(inode, 0)); + neh->eh_magic = EXT4_EXT_MAGIC; + neh->eh_depth = 0; + neh->eh_generation = 0; + + /* move remainder of path[depth] to the new leaf */ + if (unlikely(path[depth].p_hdr->eh_entries != + path[depth].p_hdr->eh_max)) { + EXT4_ERROR_INODE(inode, "eh_entries %d != eh_max %d!", + path[depth].p_hdr->eh_entries, + path[depth].p_hdr->eh_max); + err = -EFSCORRUPTED; + goto cleanup; + } + /* start copy from next extent */ + m = EXT_MAX_EXTENT(path[depth].p_hdr) - path[depth].p_ext++; + ext4_ext_show_move(inode, path, newblock, depth); + if (m) { + struct ext4_extent *ex; + ex = EXT_FIRST_EXTENT(neh); + memmove(ex, path[depth].p_ext, sizeof(struct ext4_extent) * m); + le16_add_cpu(&neh->eh_entries, m); + } + + /* zero out unused area in the extent block */ + ext_size = sizeof(struct ext4_extent_header) + + sizeof(struct ext4_extent) * le16_to_cpu(neh->eh_entries); + memset(bh->b_data + ext_size, 0, inode->i_sb->s_blocksize - ext_size); + ext4_extent_block_csum_set(inode, neh); + set_buffer_uptodate(bh); + unlock_buffer(bh); + + err = ext4_handle_dirty_metadata(handle, inode, bh); + if (err) + goto cleanup; + brelse(bh); + bh = NULL; + + /* correct old leaf */ + if (m) { + err = ext4_ext_get_access(handle, inode, path + depth); + if (err) + goto cleanup; + le16_add_cpu(&path[depth].p_hdr->eh_entries, -m); + err = ext4_ext_dirty(handle, inode, path + depth); + if (err) + goto cleanup; + + } + + /* create intermediate indexes */ + k = depth - at - 1; + if (unlikely(k < 0)) { + EXT4_ERROR_INODE(inode, "k %d < 0!", k); + err = -EFSCORRUPTED; + goto cleanup; + } + if (k) + ext_debug(inode, "create %d intermediate indices\n", k); + /* insert new index into current index block */ + /* current depth stored in i var */ + i = depth - 1; + while (k--) { + oldblock = newblock; + newblock = ablocks[--a]; + bh = sb_getblk(inode->i_sb, newblock); + if (unlikely(!bh)) { + err = -ENOMEM; + goto cleanup; + } + lock_buffer(bh); + + err = ext4_journal_get_create_access(handle, inode->i_sb, bh, + EXT4_JTR_NONE); + if (err) + goto cleanup; + + neh = ext_block_hdr(bh); + neh->eh_entries = cpu_to_le16(1); + neh->eh_magic = EXT4_EXT_MAGIC; + neh->eh_max = cpu_to_le16(ext4_ext_space_block_idx(inode, 0)); + neh->eh_depth = cpu_to_le16(depth - i); + neh->eh_generation = 0; + fidx = EXT_FIRST_INDEX(neh); + fidx->ei_block = border; + ext4_idx_store_pblock(fidx, oldblock); + + ext_debug(inode, "int.index at %d (block %llu): %u -> %llu\n", + i, newblock, le32_to_cpu(border), oldblock); + + /* move remainder of path[i] to the new index block */ + if (unlikely(EXT_MAX_INDEX(path[i].p_hdr) != + EXT_LAST_INDEX(path[i].p_hdr))) { + EXT4_ERROR_INODE(inode, + "EXT_MAX_INDEX != EXT_LAST_INDEX ee_block %d!", + le32_to_cpu(path[i].p_ext->ee_block)); + err = -EFSCORRUPTED; + goto cleanup; + } + /* start copy indexes */ + m = EXT_MAX_INDEX(path[i].p_hdr) - path[i].p_idx++; + ext_debug(inode, "cur 0x%p, last 0x%p\n", path[i].p_idx, + EXT_MAX_INDEX(path[i].p_hdr)); + ext4_ext_show_move(inode, path, newblock, i); + if (m) { + memmove(++fidx, path[i].p_idx, + sizeof(struct ext4_extent_idx) * m); + le16_add_cpu(&neh->eh_entries, m); + } + /* zero out unused area in the extent block */ + ext_size = sizeof(struct ext4_extent_header) + + (sizeof(struct ext4_extent) * le16_to_cpu(neh->eh_entries)); + memset(bh->b_data + ext_size, 0, + inode->i_sb->s_blocksize - ext_size); + ext4_extent_block_csum_set(inode, neh); + set_buffer_uptodate(bh); + unlock_buffer(bh); + + err = ext4_handle_dirty_metadata(handle, inode, bh); + if (err) + goto cleanup; + brelse(bh); + bh = NULL; + + /* correct old index */ + if (m) { + err = ext4_ext_get_access(handle, inode, path + i); + if (err) + goto cleanup; + le16_add_cpu(&path[i].p_hdr->eh_entries, -m); + err = ext4_ext_dirty(handle, inode, path + i); + if (err) + goto cleanup; + } + + i--; + } + + /* insert new index */ + err = ext4_ext_insert_index(handle, inode, path + at, + le32_to_cpu(border), newblock); + +cleanup: + if (bh) { + if (buffer_locked(bh)) + unlock_buffer(bh); + brelse(bh); + } + + if (err) { + /* free all allocated blocks in error case */ + for (i = 0; i < depth; i++) { + if (!ablocks[i]) + continue; + ext4_free_blocks(handle, inode, NULL, ablocks[i], 1, + EXT4_FREE_BLOCKS_METADATA); + } + } + kfree(ablocks); + + return err; +} + +/* + * ext4_ext_grow_indepth: + * implements tree growing procedure: + * - allocates new block + * - moves top-level data (index block or leaf) into the new block + * - initializes new top-level, creating index that points to the + * just created block + */ +static int ext4_ext_grow_indepth(handle_t *handle, struct inode *inode, + unsigned int flags) +{ + struct ext4_extent_header *neh; + struct buffer_head *bh; + ext4_fsblk_t newblock, goal = 0; + struct ext4_super_block *es = EXT4_SB(inode->i_sb)->s_es; + int err = 0; + size_t ext_size = 0; + + /* Try to prepend new index to old one */ + if (ext_depth(inode)) + goal = ext4_idx_pblock(EXT_FIRST_INDEX(ext_inode_hdr(inode))); + if (goal > le32_to_cpu(es->s_first_data_block)) { + flags |= EXT4_MB_HINT_TRY_GOAL; + goal--; + } else + goal = ext4_inode_to_goal_block(inode); + newblock = ext4_new_meta_blocks(handle, inode, goal, flags, + NULL, &err); + if (newblock == 0) + return err; + + bh = sb_getblk_gfp(inode->i_sb, newblock, __GFP_MOVABLE | GFP_NOFS); + if (unlikely(!bh)) + return -ENOMEM; + lock_buffer(bh); + + err = ext4_journal_get_create_access(handle, inode->i_sb, bh, + EXT4_JTR_NONE); + if (err) { + unlock_buffer(bh); + goto out; + } + + ext_size = sizeof(EXT4_I(inode)->i_data); + /* move top-level index/leaf into new block */ + memmove(bh->b_data, EXT4_I(inode)->i_data, ext_size); + /* zero out unused area in the extent block */ + memset(bh->b_data + ext_size, 0, inode->i_sb->s_blocksize - ext_size); + + /* set size of new block */ + neh = ext_block_hdr(bh); + /* old root could have indexes or leaves + * so calculate e_max right way */ + if (ext_depth(inode)) + neh->eh_max = cpu_to_le16(ext4_ext_space_block_idx(inode, 0)); + else + neh->eh_max = cpu_to_le16(ext4_ext_space_block(inode, 0)); + neh->eh_magic = EXT4_EXT_MAGIC; + ext4_extent_block_csum_set(inode, neh); + set_buffer_uptodate(bh); + set_buffer_verified(bh); + unlock_buffer(bh); + + err = ext4_handle_dirty_metadata(handle, inode, bh); + if (err) + goto out; + + /* Update top-level index: num,max,pointer */ + neh = ext_inode_hdr(inode); + neh->eh_entries = cpu_to_le16(1); + ext4_idx_store_pblock(EXT_FIRST_INDEX(neh), newblock); + if (neh->eh_depth == 0) { + /* Root extent block becomes index block */ + neh->eh_max = cpu_to_le16(ext4_ext_space_root_idx(inode, 0)); + EXT_FIRST_INDEX(neh)->ei_block = + EXT_FIRST_EXTENT(neh)->ee_block; + } + ext_debug(inode, "new root: num %d(%d), lblock %d, ptr %llu\n", + le16_to_cpu(neh->eh_entries), le16_to_cpu(neh->eh_max), + le32_to_cpu(EXT_FIRST_INDEX(neh)->ei_block), + ext4_idx_pblock(EXT_FIRST_INDEX(neh))); + + le16_add_cpu(&neh->eh_depth, 1); + err = ext4_mark_inode_dirty(handle, inode); +out: + brelse(bh); + + return err; +} + +/* + * ext4_ext_create_new_leaf: + * finds empty index and adds new leaf. + * if no free index is found, then it requests in-depth growing. + */ +static struct ext4_ext_path * +ext4_ext_create_new_leaf(handle_t *handle, struct inode *inode, + unsigned int mb_flags, unsigned int gb_flags, + struct ext4_ext_path *path, + struct ext4_extent *newext) +{ + struct ext4_ext_path *curp; + int depth, i, err = 0; + ext4_lblk_t ee_block = le32_to_cpu(newext->ee_block); + +repeat: + i = depth = ext_depth(inode); + + /* walk up to the tree and look for free index entry */ + curp = path + depth; + while (i > 0 && !EXT_HAS_FREE_INDEX(curp)) { + i--; + curp--; + } + + /* we use already allocated block for index block, + * so subsequent data blocks should be contiguous */ + if (EXT_HAS_FREE_INDEX(curp)) { + /* if we found index with free entry, then use that + * entry: create all needed subtree and add new leaf */ + err = ext4_ext_split(handle, inode, mb_flags, path, newext, i); + if (err) + goto errout; + + /* refill path */ + path = ext4_find_extent(inode, ee_block, path, gb_flags); + return path; + } + + /* tree is full, time to grow in depth */ + err = ext4_ext_grow_indepth(handle, inode, mb_flags); + if (err) + goto errout; + + /* refill path */ + path = ext4_find_extent(inode, ee_block, path, gb_flags); + if (IS_ERR(path)) + return path; + + /* + * only first (depth 0 -> 1) produces free space; + * in all other cases we have to split the grown tree + */ + depth = ext_depth(inode); + if (path[depth].p_hdr->eh_entries == path[depth].p_hdr->eh_max) { + /* now we need to split */ + goto repeat; + } + + return path; + +errout: + ext4_free_ext_path(path); + return ERR_PTR(err); +} + +/* + * search the closest allocated block to the left for *logical + * and returns it at @logical + it's physical address at @phys + * if *logical is the smallest allocated block, the function + * returns 0 at @phys + * return value contains 0 (success) or error code + */ +static int ext4_ext_search_left(struct inode *inode, + struct ext4_ext_path *path, + ext4_lblk_t *logical, ext4_fsblk_t *phys) +{ + struct ext4_extent_idx *ix; + struct ext4_extent *ex; + int depth, ee_len; + + if (unlikely(path == NULL)) { + EXT4_ERROR_INODE(inode, "path == NULL *logical %d!", *logical); + return -EFSCORRUPTED; + } + depth = path->p_depth; + *phys = 0; + + if (depth == 0 && path->p_ext == NULL) + return 0; + + /* usually extent in the path covers blocks smaller + * then *logical, but it can be that extent is the + * first one in the file */ + + ex = path[depth].p_ext; + ee_len = ext4_ext_get_actual_len(ex); + if (*logical < le32_to_cpu(ex->ee_block)) { + if (unlikely(EXT_FIRST_EXTENT(path[depth].p_hdr) != ex)) { + EXT4_ERROR_INODE(inode, + "EXT_FIRST_EXTENT != ex *logical %d ee_block %d!", + *logical, le32_to_cpu(ex->ee_block)); + return -EFSCORRUPTED; + } + while (--depth >= 0) { + ix = path[depth].p_idx; + if (unlikely(ix != EXT_FIRST_INDEX(path[depth].p_hdr))) { + EXT4_ERROR_INODE(inode, + "ix (%d) != EXT_FIRST_INDEX (%d) (depth %d)!", + ix != NULL ? le32_to_cpu(ix->ei_block) : 0, + le32_to_cpu(EXT_FIRST_INDEX(path[depth].p_hdr)->ei_block), + depth); + return -EFSCORRUPTED; + } + } + return 0; + } + + if (unlikely(*logical < (le32_to_cpu(ex->ee_block) + ee_len))) { + EXT4_ERROR_INODE(inode, + "logical %d < ee_block %d + ee_len %d!", + *logical, le32_to_cpu(ex->ee_block), ee_len); + return -EFSCORRUPTED; + } + + *logical = le32_to_cpu(ex->ee_block) + ee_len - 1; + *phys = ext4_ext_pblock(ex) + ee_len - 1; + return 0; +} + +/* + * Search the closest allocated block to the right for *logical + * and returns it at @logical + it's physical address at @phys. + * If not exists, return 0 and @phys is set to 0. We will return + * 1 which means we found an allocated block and ret_ex is valid. + * Or return a (< 0) error code. + */ +static int ext4_ext_search_right(struct inode *inode, + struct ext4_ext_path *path, + ext4_lblk_t *logical, ext4_fsblk_t *phys, + struct ext4_extent *ret_ex, int flags) +{ + struct buffer_head *bh = NULL; + struct ext4_extent_header *eh; + struct ext4_extent_idx *ix; + struct ext4_extent *ex; + int depth; /* Note, NOT eh_depth; depth from top of tree */ + int ee_len; + + if (unlikely(path == NULL)) { + EXT4_ERROR_INODE(inode, "path == NULL *logical %d!", *logical); + return -EFSCORRUPTED; + } + depth = path->p_depth; + *phys = 0; + + if (depth == 0 && path->p_ext == NULL) + return 0; + + /* usually extent in the path covers blocks smaller + * then *logical, but it can be that extent is the + * first one in the file */ + + ex = path[depth].p_ext; + ee_len = ext4_ext_get_actual_len(ex); + if (*logical < le32_to_cpu(ex->ee_block)) { + if (unlikely(EXT_FIRST_EXTENT(path[depth].p_hdr) != ex)) { + EXT4_ERROR_INODE(inode, + "first_extent(path[%d].p_hdr) != ex", + depth); + return -EFSCORRUPTED; + } + while (--depth >= 0) { + ix = path[depth].p_idx; + if (unlikely(ix != EXT_FIRST_INDEX(path[depth].p_hdr))) { + EXT4_ERROR_INODE(inode, + "ix != EXT_FIRST_INDEX *logical %d!", + *logical); + return -EFSCORRUPTED; + } + } + goto found_extent; + } + + if (unlikely(*logical < (le32_to_cpu(ex->ee_block) + ee_len))) { + EXT4_ERROR_INODE(inode, + "logical %d < ee_block %d + ee_len %d!", + *logical, le32_to_cpu(ex->ee_block), ee_len); + return -EFSCORRUPTED; + } + + if (ex != EXT_LAST_EXTENT(path[depth].p_hdr)) { + /* next allocated block in this leaf */ + ex++; + goto found_extent; + } + + /* go up and search for index to the right */ + while (--depth >= 0) { + ix = path[depth].p_idx; + if (ix != EXT_LAST_INDEX(path[depth].p_hdr)) + goto got_index; + } + + /* we've gone up to the root and found no index to the right */ + return 0; + +got_index: + /* we've found index to the right, let's + * follow it and find the closest allocated + * block to the right */ + ix++; + while (++depth < path->p_depth) { + /* subtract from p_depth to get proper eh_depth */ + bh = read_extent_tree_block(inode, ix, path->p_depth - depth, + flags); + if (IS_ERR(bh)) + return PTR_ERR(bh); + eh = ext_block_hdr(bh); + ix = EXT_FIRST_INDEX(eh); + put_bh(bh); + } + + bh = read_extent_tree_block(inode, ix, path->p_depth - depth, flags); + if (IS_ERR(bh)) + return PTR_ERR(bh); + eh = ext_block_hdr(bh); + ex = EXT_FIRST_EXTENT(eh); +found_extent: + *logical = le32_to_cpu(ex->ee_block); + *phys = ext4_ext_pblock(ex); + if (ret_ex) + *ret_ex = *ex; + if (bh) + put_bh(bh); + return 1; +} + +/* + * ext4_ext_next_allocated_block: + * returns allocated block in subsequent extent or EXT_MAX_BLOCKS. + * NOTE: it considers block number from index entry as + * allocated block. Thus, index entries have to be consistent + * with leaves. + */ +ext4_lblk_t +ext4_ext_next_allocated_block(struct ext4_ext_path *path) +{ + int depth; + + BUG_ON(path == NULL); + depth = path->p_depth; + + if (depth == 0 && path->p_ext == NULL) + return EXT_MAX_BLOCKS; + + while (depth >= 0) { + struct ext4_ext_path *p = &path[depth]; + + if (depth == path->p_depth) { + /* leaf */ + if (p->p_ext && p->p_ext != EXT_LAST_EXTENT(p->p_hdr)) + return le32_to_cpu(p->p_ext[1].ee_block); + } else { + /* index */ + if (p->p_idx != EXT_LAST_INDEX(p->p_hdr)) + return le32_to_cpu(p->p_idx[1].ei_block); + } + depth--; + } + + return EXT_MAX_BLOCKS; +} + +/* + * ext4_ext_next_leaf_block: + * returns first allocated block from next leaf or EXT_MAX_BLOCKS + */ +static ext4_lblk_t ext4_ext_next_leaf_block(struct ext4_ext_path *path) +{ + int depth; + + BUG_ON(path == NULL); + depth = path->p_depth; + + /* zero-tree has no leaf blocks at all */ + if (depth == 0) + return EXT_MAX_BLOCKS; + + /* go to index block */ + depth--; + + while (depth >= 0) { + if (path[depth].p_idx != + EXT_LAST_INDEX(path[depth].p_hdr)) + return (ext4_lblk_t) + le32_to_cpu(path[depth].p_idx[1].ei_block); + depth--; + } + + return EXT_MAX_BLOCKS; +} + +/* + * ext4_ext_correct_indexes: + * if leaf gets modified and modified extent is first in the leaf, + * then we have to correct all indexes above. + * TODO: do we need to correct tree in all cases? + */ +static int ext4_ext_correct_indexes(handle_t *handle, struct inode *inode, + struct ext4_ext_path *path) +{ + struct ext4_extent_header *eh; + int depth = ext_depth(inode); + struct ext4_extent *ex; + __le32 border; + int k, err = 0; + + eh = path[depth].p_hdr; + ex = path[depth].p_ext; + + if (unlikely(ex == NULL || eh == NULL)) { + EXT4_ERROR_INODE(inode, + "ex %p == NULL or eh %p == NULL", ex, eh); + return -EFSCORRUPTED; + } + + if (depth == 0) { + /* there is no tree at all */ + return 0; + } + + if (ex != EXT_FIRST_EXTENT(eh)) { + /* we correct tree if first leaf got modified only */ + return 0; + } + + /* + * TODO: we need correction if border is smaller than current one + */ + k = depth - 1; + border = path[depth].p_ext->ee_block; + err = ext4_ext_get_access(handle, inode, path + k); + if (err) + return err; + if (unlikely(path[k].p_idx > EXT_LAST_INDEX(path[k].p_hdr))) { + EXT4_ERROR_INODE(inode, + "path[%d].p_idx %p > EXT_LAST_INDEX %p", + k, path[k].p_idx, + EXT_LAST_INDEX(path[k].p_hdr)); + return -EFSCORRUPTED; + } + path[k].p_idx->ei_block = border; + err = ext4_ext_dirty(handle, inode, path + k); + if (err) + return err; + + while (k--) { + /* change all left-side indexes */ + if (path[k+1].p_idx != EXT_FIRST_INDEX(path[k+1].p_hdr)) + break; + err = ext4_ext_get_access(handle, inode, path + k); + if (err) + goto clean; + if (unlikely(path[k].p_idx > EXT_LAST_INDEX(path[k].p_hdr))) { + EXT4_ERROR_INODE(inode, + "path[%d].p_idx %p > EXT_LAST_INDEX %p", + k, path[k].p_idx, + EXT_LAST_INDEX(path[k].p_hdr)); + err = -EFSCORRUPTED; + goto clean; + } + path[k].p_idx->ei_block = border; + err = ext4_ext_dirty(handle, inode, path + k); + if (err) + goto clean; + } + return 0; + +clean: + /* + * The path[k].p_bh is either unmodified or with no verified bit + * set (see ext4_ext_get_access()). So just clear the verified bit + * of the successfully modified extents buffers, which will force + * these extents to be checked to avoid using inconsistent data. + */ + while (++k < depth) + clear_buffer_verified(path[k].p_bh); + + return err; +} + +static int ext4_can_extents_be_merged(struct inode *inode, + struct ext4_extent *ex1, + struct ext4_extent *ex2) +{ + unsigned short ext1_ee_len, ext2_ee_len; + + if (ext4_ext_is_unwritten(ex1) != ext4_ext_is_unwritten(ex2)) + return 0; + + ext1_ee_len = ext4_ext_get_actual_len(ex1); + ext2_ee_len = ext4_ext_get_actual_len(ex2); + + if (le32_to_cpu(ex1->ee_block) + ext1_ee_len != + le32_to_cpu(ex2->ee_block)) + return 0; + + if (ext1_ee_len + ext2_ee_len > EXT_INIT_MAX_LEN) + return 0; + + if (ext4_ext_is_unwritten(ex1) && + ext1_ee_len + ext2_ee_len > EXT_UNWRITTEN_MAX_LEN) + return 0; +#ifdef AGGRESSIVE_TEST + if (ext1_ee_len >= 4) + return 0; +#endif + + if (ext4_ext_pblock(ex1) + ext1_ee_len == ext4_ext_pblock(ex2)) + return 1; + return 0; +} + +/* + * This function tries to merge the "ex" extent to the next extent in the tree. + * It always tries to merge towards right. If you want to merge towards + * left, pass "ex - 1" as argument instead of "ex". + * Returns 0 if the extents (ex and ex+1) were _not_ merged and returns + * 1 if they got merged. + */ +static int ext4_ext_try_to_merge_right(struct inode *inode, + struct ext4_ext_path *path, + struct ext4_extent *ex) +{ + struct ext4_extent_header *eh; + unsigned int depth, len; + int merge_done = 0, unwritten; + + depth = ext_depth(inode); + BUG_ON(path[depth].p_hdr == NULL); + eh = path[depth].p_hdr; + + while (ex < EXT_LAST_EXTENT(eh)) { + if (!ext4_can_extents_be_merged(inode, ex, ex + 1)) + break; + /* merge with next extent! */ + unwritten = ext4_ext_is_unwritten(ex); + ex->ee_len = cpu_to_le16(ext4_ext_get_actual_len(ex) + + ext4_ext_get_actual_len(ex + 1)); + if (unwritten) + ext4_ext_mark_unwritten(ex); + + if (ex + 1 < EXT_LAST_EXTENT(eh)) { + len = (EXT_LAST_EXTENT(eh) - ex - 1) + * sizeof(struct ext4_extent); + memmove(ex + 1, ex + 2, len); + } + le16_add_cpu(&eh->eh_entries, -1); + merge_done = 1; + WARN_ON(eh->eh_entries == 0); + if (!eh->eh_entries) + EXT4_ERROR_INODE(inode, "eh->eh_entries = 0!"); + } + + return merge_done; +} + +/* + * This function does a very simple check to see if we can collapse + * an extent tree with a single extent tree leaf block into the inode. + */ +static void ext4_ext_try_to_merge_up(handle_t *handle, + struct inode *inode, + struct ext4_ext_path *path) +{ + size_t s; + unsigned max_root = ext4_ext_space_root(inode, 0); + ext4_fsblk_t blk; + + if ((path[0].p_depth != 1) || + (le16_to_cpu(path[0].p_hdr->eh_entries) != 1) || + (le16_to_cpu(path[1].p_hdr->eh_entries) > max_root)) + return; + + /* + * We need to modify the block allocation bitmap and the block + * group descriptor to release the extent tree block. If we + * can't get the journal credits, give up. + */ + if (ext4_journal_extend(handle, 2, + ext4_free_metadata_revoke_credits(inode->i_sb, 1))) + return; + + /* + * Copy the extent data up to the inode + */ + blk = ext4_idx_pblock(path[0].p_idx); + s = le16_to_cpu(path[1].p_hdr->eh_entries) * + sizeof(struct ext4_extent_idx); + s += sizeof(struct ext4_extent_header); + + path[1].p_maxdepth = path[0].p_maxdepth; + memcpy(path[0].p_hdr, path[1].p_hdr, s); + path[0].p_depth = 0; + path[0].p_ext = EXT_FIRST_EXTENT(path[0].p_hdr) + + (path[1].p_ext - EXT_FIRST_EXTENT(path[1].p_hdr)); + path[0].p_hdr->eh_max = cpu_to_le16(max_root); + + ext4_ext_path_brelse(path + 1); + ext4_free_blocks(handle, inode, NULL, blk, 1, + EXT4_FREE_BLOCKS_METADATA | EXT4_FREE_BLOCKS_FORGET); +} + +/* + * This function tries to merge the @ex extent to neighbours in the tree, then + * tries to collapse the extent tree into the inode. + */ +static void ext4_ext_try_to_merge(handle_t *handle, + struct inode *inode, + struct ext4_ext_path *path, + struct ext4_extent *ex) +{ + struct ext4_extent_header *eh; + unsigned int depth; + int merge_done = 0; + + depth = ext_depth(inode); + BUG_ON(path[depth].p_hdr == NULL); + eh = path[depth].p_hdr; + + if (ex > EXT_FIRST_EXTENT(eh)) + merge_done = ext4_ext_try_to_merge_right(inode, path, ex - 1); + + if (!merge_done) + (void) ext4_ext_try_to_merge_right(inode, path, ex); + + ext4_ext_try_to_merge_up(handle, inode, path); +} + +/* + * check if a portion of the "newext" extent overlaps with an + * existing extent. + * + * If there is an overlap discovered, it updates the length of the newext + * such that there will be no overlap, and then returns 1. + * If there is no overlap found, it returns 0. + */ +static unsigned int ext4_ext_check_overlap(struct ext4_sb_info *sbi, + struct inode *inode, + struct ext4_extent *newext, + struct ext4_ext_path *path) +{ + ext4_lblk_t b1, b2; + unsigned int depth, len1; + unsigned int ret = 0; + + b1 = le32_to_cpu(newext->ee_block); + len1 = ext4_ext_get_actual_len(newext); + depth = ext_depth(inode); + if (!path[depth].p_ext) + goto out; + b2 = EXT4_LBLK_CMASK(sbi, le32_to_cpu(path[depth].p_ext->ee_block)); + + /* + * get the next allocated block if the extent in the path + * is before the requested block(s) + */ + if (b2 < b1) { + b2 = ext4_ext_next_allocated_block(path); + if (b2 == EXT_MAX_BLOCKS) + goto out; + b2 = EXT4_LBLK_CMASK(sbi, b2); + } + + /* check for wrap through zero on extent logical start block*/ + if (b1 + len1 < b1) { + len1 = EXT_MAX_BLOCKS - b1; + newext->ee_len = cpu_to_le16(len1); + ret = 1; + } + + /* check for overlap */ + if (b1 + len1 > b2) { + newext->ee_len = cpu_to_le16(b2 - b1); + ret = 1; + } +out: + return ret; +} + +/* + * ext4_ext_insert_extent: + * tries to merge requested extent into the existing extent or + * inserts requested extent as new one into the tree, + * creating new leaf in the no-space case. + */ +struct ext4_ext_path * +ext4_ext_insert_extent(handle_t *handle, struct inode *inode, + struct ext4_ext_path *path, + struct ext4_extent *newext, int gb_flags) +{ + struct ext4_extent_header *eh; + struct ext4_extent *ex, *fex; + struct ext4_extent *nearex; /* nearest extent */ + int depth, len, err = 0; + ext4_lblk_t next; + int mb_flags = 0, unwritten; + + KUNIT_STATIC_STUB_REDIRECT(ext4_ext_insert_extent, handle, inode, path, + newext, gb_flags); + + if (gb_flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) + mb_flags |= EXT4_MB_DELALLOC_RESERVED; + if (unlikely(ext4_ext_get_actual_len(newext) == 0)) { + EXT4_ERROR_INODE(inode, "ext4_ext_get_actual_len(newext) == 0"); + err = -EFSCORRUPTED; + goto errout; + } + depth = ext_depth(inode); + ex = path[depth].p_ext; + eh = path[depth].p_hdr; + if (unlikely(path[depth].p_hdr == NULL)) { + EXT4_ERROR_INODE(inode, "path[%d].p_hdr == NULL", depth); + err = -EFSCORRUPTED; + goto errout; + } + + /* try to insert block into found extent and return */ + if (ex && !(gb_flags & EXT4_GET_BLOCKS_SPLIT_NOMERGE)) { + + /* + * Try to see whether we should rather test the extent on + * right from ex, or from the left of ex. This is because + * ext4_find_extent() can return either extent on the + * left, or on the right from the searched position. This + * will make merging more effective. + */ + if (ex < EXT_LAST_EXTENT(eh) && + (le32_to_cpu(ex->ee_block) + + ext4_ext_get_actual_len(ex) < + le32_to_cpu(newext->ee_block))) { + ex += 1; + goto prepend; + } else if ((ex > EXT_FIRST_EXTENT(eh)) && + (le32_to_cpu(newext->ee_block) + + ext4_ext_get_actual_len(newext) < + le32_to_cpu(ex->ee_block))) + ex -= 1; + + /* Try to append newex to the ex */ + if (ext4_can_extents_be_merged(inode, ex, newext)) { + ext_debug(inode, "append [%d]%d block to %u:[%d]%d" + "(from %llu)\n", + ext4_ext_is_unwritten(newext), + ext4_ext_get_actual_len(newext), + le32_to_cpu(ex->ee_block), + ext4_ext_is_unwritten(ex), + ext4_ext_get_actual_len(ex), + ext4_ext_pblock(ex)); + err = ext4_ext_get_access(handle, inode, + path + depth); + if (err) + goto errout; + unwritten = ext4_ext_is_unwritten(ex); + ex->ee_len = cpu_to_le16(ext4_ext_get_actual_len(ex) + + ext4_ext_get_actual_len(newext)); + if (unwritten) + ext4_ext_mark_unwritten(ex); + nearex = ex; + goto merge; + } + +prepend: + /* Try to prepend newex to the ex */ + if (ext4_can_extents_be_merged(inode, newext, ex)) { + ext_debug(inode, "prepend %u[%d]%d block to %u:[%d]%d" + "(from %llu)\n", + le32_to_cpu(newext->ee_block), + ext4_ext_is_unwritten(newext), + ext4_ext_get_actual_len(newext), + le32_to_cpu(ex->ee_block), + ext4_ext_is_unwritten(ex), + ext4_ext_get_actual_len(ex), + ext4_ext_pblock(ex)); + err = ext4_ext_get_access(handle, inode, + path + depth); + if (err) + goto errout; + + unwritten = ext4_ext_is_unwritten(ex); + ex->ee_block = newext->ee_block; + ext4_ext_store_pblock(ex, ext4_ext_pblock(newext)); + ex->ee_len = cpu_to_le16(ext4_ext_get_actual_len(ex) + + ext4_ext_get_actual_len(newext)); + if (unwritten) + ext4_ext_mark_unwritten(ex); + nearex = ex; + goto merge; + } + } + + depth = ext_depth(inode); + eh = path[depth].p_hdr; + if (le16_to_cpu(eh->eh_entries) < le16_to_cpu(eh->eh_max)) + goto has_space; + + /* probably next leaf has space for us? */ + fex = EXT_LAST_EXTENT(eh); + next = EXT_MAX_BLOCKS; + if (le32_to_cpu(newext->ee_block) > le32_to_cpu(fex->ee_block)) + next = ext4_ext_next_leaf_block(path); + if (next != EXT_MAX_BLOCKS) { + struct ext4_ext_path *npath; + + ext_debug(inode, "next leaf block - %u\n", next); + npath = ext4_find_extent(inode, next, NULL, gb_flags); + if (IS_ERR(npath)) { + err = PTR_ERR(npath); + goto errout; + } + BUG_ON(npath->p_depth != path->p_depth); + eh = npath[depth].p_hdr; + if (le16_to_cpu(eh->eh_entries) < le16_to_cpu(eh->eh_max)) { + ext_debug(inode, "next leaf isn't full(%d)\n", + le16_to_cpu(eh->eh_entries)); + ext4_free_ext_path(path); + path = npath; + goto has_space; + } + ext_debug(inode, "next leaf has no free space(%d,%d)\n", + le16_to_cpu(eh->eh_entries), le16_to_cpu(eh->eh_max)); + ext4_free_ext_path(npath); + } + + /* + * There is no free space in the found leaf. + * We're gonna add a new leaf in the tree. + */ + if (gb_flags & EXT4_GET_BLOCKS_METADATA_NOFAIL) + mb_flags |= EXT4_MB_USE_RESERVED; + path = ext4_ext_create_new_leaf(handle, inode, mb_flags, gb_flags, + path, newext); + if (IS_ERR(path)) + return path; + depth = ext_depth(inode); + eh = path[depth].p_hdr; + +has_space: + nearex = path[depth].p_ext; + + err = ext4_ext_get_access(handle, inode, path + depth); + if (err) + goto errout; + + if (!nearex) { + /* there is no extent in this leaf, create first one */ + ext_debug(inode, "first extent in the leaf: %u:%llu:[%d]%d\n", + le32_to_cpu(newext->ee_block), + ext4_ext_pblock(newext), + ext4_ext_is_unwritten(newext), + ext4_ext_get_actual_len(newext)); + nearex = EXT_FIRST_EXTENT(eh); + } else { + if (le32_to_cpu(newext->ee_block) + > le32_to_cpu(nearex->ee_block)) { + /* Insert after */ + ext_debug(inode, "insert %u:%llu:[%d]%d before: " + "nearest %p\n", + le32_to_cpu(newext->ee_block), + ext4_ext_pblock(newext), + ext4_ext_is_unwritten(newext), + ext4_ext_get_actual_len(newext), + nearex); + nearex++; + } else { + /* Insert before */ + BUG_ON(newext->ee_block == nearex->ee_block); + ext_debug(inode, "insert %u:%llu:[%d]%d after: " + "nearest %p\n", + le32_to_cpu(newext->ee_block), + ext4_ext_pblock(newext), + ext4_ext_is_unwritten(newext), + ext4_ext_get_actual_len(newext), + nearex); + } + len = EXT_LAST_EXTENT(eh) - nearex + 1; + if (len > 0) { + ext_debug(inode, "insert %u:%llu:[%d]%d: " + "move %d extents from 0x%p to 0x%p\n", + le32_to_cpu(newext->ee_block), + ext4_ext_pblock(newext), + ext4_ext_is_unwritten(newext), + ext4_ext_get_actual_len(newext), + len, nearex, nearex + 1); + memmove(nearex + 1, nearex, + len * sizeof(struct ext4_extent)); + } + } + + le16_add_cpu(&eh->eh_entries, 1); + path[depth].p_ext = nearex; + nearex->ee_block = newext->ee_block; + ext4_ext_store_pblock(nearex, ext4_ext_pblock(newext)); + nearex->ee_len = newext->ee_len; + +merge: + /* try to merge extents */ + if (!(gb_flags & EXT4_GET_BLOCKS_SPLIT_NOMERGE)) + ext4_ext_try_to_merge(handle, inode, path, nearex); + + /* time to correct all indexes above */ + err = ext4_ext_correct_indexes(handle, inode, path); + if (err) + goto errout; + + err = ext4_ext_dirty(handle, inode, path + path->p_depth); + if (err) + goto errout; + + return path; + +errout: + ext4_free_ext_path(path); + return ERR_PTR(err); +} + +static int ext4_fill_es_cache_info(struct inode *inode, + ext4_lblk_t block, ext4_lblk_t num, + struct fiemap_extent_info *fieinfo) +{ + ext4_lblk_t next, end = block + num - 1; + struct extent_status es; + unsigned char blksize_bits = inode->i_sb->s_blocksize_bits; + unsigned int flags; + int err; + + while (block <= end) { + next = 0; + flags = 0; + if (!ext4_es_lookup_extent(inode, block, &next, &es, NULL)) + break; + if (ext4_es_is_unwritten(&es)) + flags |= FIEMAP_EXTENT_UNWRITTEN; + if (ext4_es_is_delayed(&es)) + flags |= (FIEMAP_EXTENT_DELALLOC | + FIEMAP_EXTENT_UNKNOWN); + if (ext4_es_is_hole(&es)) + flags |= EXT4_FIEMAP_EXTENT_HOLE; + if (next == 0) + flags |= FIEMAP_EXTENT_LAST; + if (flags & (FIEMAP_EXTENT_DELALLOC| + EXT4_FIEMAP_EXTENT_HOLE)) + es.es_pblk = 0; + else + es.es_pblk = ext4_es_pblock(&es); + err = fiemap_fill_next_extent(fieinfo, + (__u64)es.es_lblk << blksize_bits, + (__u64)es.es_pblk << blksize_bits, + (__u64)es.es_len << blksize_bits, + flags); + if (next == 0) + break; + block = next; + if (err < 0) + return err; + if (err == 1) + return 0; + } + return 0; +} + + +/* + * ext4_ext_find_hole - find hole around given block according to the given path + * @inode: inode we lookup in + * @path: path in extent tree to @lblk + * @lblk: pointer to logical block around which we want to determine hole + * + * Determine hole length (and start if easily possible) around given logical + * block. We don't try too hard to find the beginning of the hole but @path + * actually points to extent before @lblk, we provide it. + * + * The function returns the length of a hole starting at @lblk. We update @lblk + * to the beginning of the hole if we managed to find it. + */ +static ext4_lblk_t ext4_ext_find_hole(struct inode *inode, + struct ext4_ext_path *path, + ext4_lblk_t *lblk) +{ + int depth = ext_depth(inode); + struct ext4_extent *ex; + ext4_lblk_t len; + + ex = path[depth].p_ext; + if (ex == NULL) { + /* there is no extent yet, so gap is [0;-] */ + *lblk = 0; + len = EXT_MAX_BLOCKS; + } else if (*lblk < le32_to_cpu(ex->ee_block)) { + len = le32_to_cpu(ex->ee_block) - *lblk; + } else if (*lblk >= le32_to_cpu(ex->ee_block) + + ext4_ext_get_actual_len(ex)) { + ext4_lblk_t next; + + *lblk = le32_to_cpu(ex->ee_block) + ext4_ext_get_actual_len(ex); + next = ext4_ext_next_allocated_block(path); + BUG_ON(next == *lblk); + len = next - *lblk; + } else { + BUG(); + } + return len; +} + +/* + * ext4_ext_rm_idx: + * removes index from the index block. + */ +static int ext4_ext_rm_idx(handle_t *handle, struct inode *inode, + struct ext4_ext_path *path, int depth) +{ + int err; + ext4_fsblk_t leaf; + int k = depth - 1; + + /* free index block */ + leaf = ext4_idx_pblock(path[k].p_idx); + if (unlikely(path[k].p_hdr->eh_entries == 0)) { + EXT4_ERROR_INODE(inode, "path[%d].p_hdr->eh_entries == 0", k); + return -EFSCORRUPTED; + } + err = ext4_ext_get_access(handle, inode, path + k); + if (err) + return err; + + if (path[k].p_idx != EXT_LAST_INDEX(path[k].p_hdr)) { + int len = EXT_LAST_INDEX(path[k].p_hdr) - path[k].p_idx; + len *= sizeof(struct ext4_extent_idx); + memmove(path[k].p_idx, path[k].p_idx + 1, len); + } + + le16_add_cpu(&path[k].p_hdr->eh_entries, -1); + err = ext4_ext_dirty(handle, inode, path + k); + if (err) + return err; + ext_debug(inode, "index is empty, remove it, free block %llu\n", leaf); + trace_ext4_ext_rm_idx(inode, leaf); + + ext4_free_blocks(handle, inode, NULL, leaf, 1, + EXT4_FREE_BLOCKS_METADATA | EXT4_FREE_BLOCKS_FORGET); + + while (--k >= 0) { + if (path[k + 1].p_idx != EXT_FIRST_INDEX(path[k + 1].p_hdr)) + break; + err = ext4_ext_get_access(handle, inode, path + k); + if (err) + goto clean; + path[k].p_idx->ei_block = path[k + 1].p_idx->ei_block; + err = ext4_ext_dirty(handle, inode, path + k); + if (err) + goto clean; + } + return 0; + +clean: + /* + * The path[k].p_bh is either unmodified or with no verified bit + * set (see ext4_ext_get_access()). So just clear the verified bit + * of the successfully modified extents buffers, which will force + * these extents to be checked to avoid using inconsistent data. + */ + while (++k < depth) + clear_buffer_verified(path[k].p_bh); + + return err; +} + +/* + * ext4_ext_calc_credits_for_single_extent: + * This routine returns max. credits that needed to insert an extent + * to the extent tree. + * When pass the actual path, the caller should calculate credits + * under i_data_sem. + */ +int ext4_ext_calc_credits_for_single_extent(struct inode *inode, int nrblocks, + struct ext4_ext_path *path) +{ + if (path) { + int depth = ext_depth(inode); + int ret = 0; + + /* probably there is space in leaf? */ + if (le16_to_cpu(path[depth].p_hdr->eh_entries) + < le16_to_cpu(path[depth].p_hdr->eh_max)) { + + /* + * There are some space in the leaf tree, no + * need to account for leaf block credit + * + * bitmaps and block group descriptor blocks + * and other metadata blocks still need to be + * accounted. + */ + /* 1 bitmap, 1 block group descriptor */ + ret = 2 + EXT4_META_TRANS_BLOCKS(inode->i_sb); + return ret; + } + } + + return ext4_chunk_trans_blocks(inode, nrblocks); +} + +/* + * How many index/leaf blocks need to change/allocate to add @extents extents? + * + * If we add a single extent, then in the worse case, each tree level + * index/leaf need to be changed in case of the tree split. + * + * If more extents are inserted, they could cause the whole tree split more + * than once, but this is really rare. + */ +int ext4_ext_index_trans_blocks(struct inode *inode, int extents) +{ + int index; + + /* If we are converting the inline data, only one is needed here. */ + if (ext4_has_inline_data(inode)) + return 1; + + /* + * Extent tree can change between the time we estimate credits and + * the time we actually modify the tree. Assume the worst case. + */ + if (extents <= 1) + index = (EXT4_MAX_EXTENT_DEPTH * 2) + extents; + else { + int ext_max = ext4_ext_space_block(inode, 0); + + index = EXT4_MAX_EXTENT_DEPTH * 3; + /* + * Modified extents need not start at the beginning of the + * leaf. Already two extents may need two leaf block + * modifications... + */ + index += DIV_ROUND_UP(extents + ext_max - 1, ext_max); + } + + return index; +} + +static inline int get_default_free_blocks_flags(struct inode *inode) +{ + if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode) || + ext4_test_inode_flag(inode, EXT4_INODE_EA_INODE)) + return EXT4_FREE_BLOCKS_METADATA | EXT4_FREE_BLOCKS_FORGET; + else if (ext4_should_journal_data(inode)) + return EXT4_FREE_BLOCKS_FORGET; + return 0; +} + +/* + * ext4_rereserve_cluster - increment the reserved cluster count when + * freeing a cluster with a pending reservation + * + * @inode - file containing the cluster + * @lblk - logical block in cluster to be reserved + * + * Increments the reserved cluster count and adjusts quota in a bigalloc + * file system when freeing a partial cluster containing at least one + * delayed and unwritten block. A partial cluster meeting that + * requirement will have a pending reservation. If so, the + * RERESERVE_CLUSTER flag is used when calling ext4_free_blocks() to + * defer reserved and allocated space accounting to a subsequent call + * to this function. + */ +static void ext4_rereserve_cluster(struct inode *inode, ext4_lblk_t lblk) +{ + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + struct ext4_inode_info *ei = EXT4_I(inode); + + dquot_reclaim_block(inode, EXT4_C2B(sbi, 1)); + + spin_lock(&ei->i_block_reservation_lock); + ei->i_reserved_data_blocks++; + percpu_counter_add(&sbi->s_dirtyclusters_counter, 1); + spin_unlock(&ei->i_block_reservation_lock); + + percpu_counter_add(&sbi->s_freeclusters_counter, 1); + ext4_remove_pending(inode, lblk); +} + +static int ext4_remove_blocks(handle_t *handle, struct inode *inode, + struct ext4_extent *ex, + struct partial_cluster *partial, + ext4_lblk_t from, ext4_lblk_t to) +{ + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + unsigned short ee_len = ext4_ext_get_actual_len(ex); + ext4_fsblk_t last_pblk, pblk; + ext4_lblk_t num; + int flags; + + /* only extent tail removal is allowed */ + if (from < le32_to_cpu(ex->ee_block) || + to != le32_to_cpu(ex->ee_block) + ee_len - 1) { + ext4_error(sbi->s_sb, + "strange request: removal(2) %u-%u from %u:%u", + from, to, le32_to_cpu(ex->ee_block), ee_len); + return 0; + } + +#ifdef EXTENTS_STATS + spin_lock(&sbi->s_ext_stats_lock); + sbi->s_ext_blocks += ee_len; + sbi->s_ext_extents++; + if (ee_len < sbi->s_ext_min) + sbi->s_ext_min = ee_len; + if (ee_len > sbi->s_ext_max) + sbi->s_ext_max = ee_len; + if (ext_depth(inode) > sbi->s_depth_max) + sbi->s_depth_max = ext_depth(inode); + spin_unlock(&sbi->s_ext_stats_lock); +#endif + + trace_ext4_remove_blocks(inode, ex, from, to, partial); + + /* + * if we have a partial cluster, and it's different from the + * cluster of the last block in the extent, we free it + */ + last_pblk = ext4_ext_pblock(ex) + ee_len - 1; + + if (partial->state != initial && + partial->pclu != EXT4_B2C(sbi, last_pblk)) { + if (partial->state == tofree) { + flags = get_default_free_blocks_flags(inode); + if (ext4_is_pending(inode, partial->lblk)) + flags |= EXT4_FREE_BLOCKS_RERESERVE_CLUSTER; + ext4_free_blocks(handle, inode, NULL, + EXT4_C2B(sbi, partial->pclu), + sbi->s_cluster_ratio, flags); + if (flags & EXT4_FREE_BLOCKS_RERESERVE_CLUSTER) + ext4_rereserve_cluster(inode, partial->lblk); + } + partial->state = initial; + } + + num = le32_to_cpu(ex->ee_block) + ee_len - from; + pblk = ext4_ext_pblock(ex) + ee_len - num; + + /* + * We free the partial cluster at the end of the extent (if any), + * unless the cluster is used by another extent (partial_cluster + * state is nofree). If a partial cluster exists here, it must be + * shared with the last block in the extent. + */ + flags = get_default_free_blocks_flags(inode); + + /* partial, left end cluster aligned, right end unaligned */ + if ((EXT4_LBLK_COFF(sbi, to) != sbi->s_cluster_ratio - 1) && + (EXT4_LBLK_CMASK(sbi, to) >= from) && + (partial->state != nofree)) { + if (ext4_is_pending(inode, to)) + flags |= EXT4_FREE_BLOCKS_RERESERVE_CLUSTER; + ext4_free_blocks(handle, inode, NULL, + EXT4_PBLK_CMASK(sbi, last_pblk), + sbi->s_cluster_ratio, flags); + if (flags & EXT4_FREE_BLOCKS_RERESERVE_CLUSTER) + ext4_rereserve_cluster(inode, to); + partial->state = initial; + flags = get_default_free_blocks_flags(inode); + } + + flags |= EXT4_FREE_BLOCKS_NOFREE_LAST_CLUSTER; + + /* + * For bigalloc file systems, we never free a partial cluster + * at the beginning of the extent. Instead, we check to see if we + * need to free it on a subsequent call to ext4_remove_blocks, + * or at the end of ext4_ext_rm_leaf or ext4_ext_remove_space. + */ + flags |= EXT4_FREE_BLOCKS_NOFREE_FIRST_CLUSTER; + ext4_free_blocks(handle, inode, NULL, pblk, num, flags); + + /* reset the partial cluster if we've freed past it */ + if (partial->state != initial && partial->pclu != EXT4_B2C(sbi, pblk)) + partial->state = initial; + + /* + * If we've freed the entire extent but the beginning is not left + * cluster aligned and is not marked as ineligible for freeing we + * record the partial cluster at the beginning of the extent. It + * wasn't freed by the preceding ext4_free_blocks() call, and we + * need to look farther to the left to determine if it's to be freed + * (not shared with another extent). Else, reset the partial + * cluster - we're either done freeing or the beginning of the + * extent is left cluster aligned. + */ + if (EXT4_LBLK_COFF(sbi, from) && num == ee_len) { + if (partial->state == initial) { + partial->pclu = EXT4_B2C(sbi, pblk); + partial->lblk = from; + partial->state = tofree; + } + } else { + partial->state = initial; + } + + return 0; +} + +/* + * ext4_ext_rm_leaf() Removes the extents associated with the + * blocks appearing between "start" and "end". Both "start" + * and "end" must appear in the same extent or EIO is returned. + * + * @handle: The journal handle + * @inode: The files inode + * @path: The path to the leaf + * @partial_cluster: The cluster which we'll have to free if all extents + * has been released from it. However, if this value is + * negative, it's a cluster just to the right of the + * punched region and it must not be freed. + * @start: The first block to remove + * @end: The last block to remove + */ +static int +ext4_ext_rm_leaf(handle_t *handle, struct inode *inode, + struct ext4_ext_path *path, + struct partial_cluster *partial, + ext4_lblk_t start, ext4_lblk_t end) +{ + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + int err = 0, correct_index = 0; + int depth = ext_depth(inode), credits, revoke_credits; + struct ext4_extent_header *eh; + ext4_lblk_t a, b; + unsigned num; + ext4_lblk_t ex_ee_block; + unsigned short ex_ee_len; + unsigned unwritten = 0; + struct ext4_extent *ex; + ext4_fsblk_t pblk; + + /* the header must be checked already in ext4_ext_remove_space() */ + ext_debug(inode, "truncate since %u in leaf to %u\n", start, end); + if (!path[depth].p_hdr) + path[depth].p_hdr = ext_block_hdr(path[depth].p_bh); + eh = path[depth].p_hdr; + if (unlikely(path[depth].p_hdr == NULL)) { + EXT4_ERROR_INODE(inode, "path[%d].p_hdr == NULL", depth); + return -EFSCORRUPTED; + } + /* find where to start removing */ + ex = path[depth].p_ext; + if (!ex) + ex = EXT_LAST_EXTENT(eh); + + ex_ee_block = le32_to_cpu(ex->ee_block); + ex_ee_len = ext4_ext_get_actual_len(ex); + + trace_ext4_ext_rm_leaf(inode, start, ex, partial); + + while (ex >= EXT_FIRST_EXTENT(eh) && + ex_ee_block + ex_ee_len > start) { + + if (ext4_ext_is_unwritten(ex)) + unwritten = 1; + else + unwritten = 0; + + ext_debug(inode, "remove ext %u:[%d]%d\n", ex_ee_block, + unwritten, ex_ee_len); + path[depth].p_ext = ex; + + a = max(ex_ee_block, start); + b = min(ex_ee_block + ex_ee_len - 1, end); + + ext_debug(inode, " border %u:%u\n", a, b); + + /* If this extent is beyond the end of the hole, skip it */ + if (end < ex_ee_block) { + /* + * We're going to skip this extent and move to another, + * so note that its first cluster is in use to avoid + * freeing it when removing blocks. Eventually, the + * right edge of the truncated/punched region will + * be just to the left. + */ + if (sbi->s_cluster_ratio > 1) { + pblk = ext4_ext_pblock(ex); + partial->pclu = EXT4_B2C(sbi, pblk); + partial->state = nofree; + } + ex--; + ex_ee_block = le32_to_cpu(ex->ee_block); + ex_ee_len = ext4_ext_get_actual_len(ex); + continue; + } else if (b != ex_ee_block + ex_ee_len - 1) { + EXT4_ERROR_INODE(inode, + "can not handle truncate %u:%u " + "on extent %u:%u", + start, end, ex_ee_block, + ex_ee_block + ex_ee_len - 1); + err = -EFSCORRUPTED; + goto out; + } else if (a != ex_ee_block) { + /* remove tail of the extent */ + num = a - ex_ee_block; + } else { + /* remove whole extent: excellent! */ + num = 0; + } + /* + * 3 for leaf, sb, and inode plus 2 (bmap and group + * descriptor) for each block group; assume two block + * groups plus ex_ee_len/blocks_per_block_group for + * the worst case + */ + credits = 7 + 2*(ex_ee_len/EXT4_BLOCKS_PER_GROUP(inode->i_sb)); + if (ex == EXT_FIRST_EXTENT(eh)) { + correct_index = 1; + credits += (ext_depth(inode)) + 1; + } + credits += EXT4_MAXQUOTAS_TRANS_BLOCKS(inode->i_sb); + /* + * We may end up freeing some index blocks and data from the + * punched range. Note that partial clusters are accounted for + * by ext4_free_data_revoke_credits(). + */ + revoke_credits = + ext4_free_metadata_revoke_credits(inode->i_sb, + ext_depth(inode)) + + ext4_free_data_revoke_credits(inode, b - a + 1); + + err = ext4_datasem_ensure_credits(handle, inode, credits, + credits, revoke_credits); + if (err) { + if (err > 0) + err = -EAGAIN; + goto out; + } + + err = ext4_ext_get_access(handle, inode, path + depth); + if (err) + goto out; + + err = ext4_remove_blocks(handle, inode, ex, partial, a, b); + if (err) + goto out; + + if (num == 0) + /* this extent is removed; mark slot entirely unused */ + ext4_ext_store_pblock(ex, 0); + + ex->ee_len = cpu_to_le16(num); + /* + * Do not mark unwritten if all the blocks in the + * extent have been removed. + */ + if (unwritten && num) + ext4_ext_mark_unwritten(ex); + /* + * If the extent was completely released, + * we need to remove it from the leaf + */ + if (num == 0) { + if (end != EXT_MAX_BLOCKS - 1) { + /* + * For hole punching, we need to scoot all the + * extents up when an extent is removed so that + * we dont have blank extents in the middle + */ + memmove(ex, ex+1, (EXT_LAST_EXTENT(eh) - ex) * + sizeof(struct ext4_extent)); + + /* Now get rid of the one at the end */ + memset(EXT_LAST_EXTENT(eh), 0, + sizeof(struct ext4_extent)); + } + le16_add_cpu(&eh->eh_entries, -1); + } + + err = ext4_ext_dirty(handle, inode, path + depth); + if (err) + goto out; + + ext_debug(inode, "new extent: %u:%u:%llu\n", ex_ee_block, num, + ext4_ext_pblock(ex)); + ex--; + ex_ee_block = le32_to_cpu(ex->ee_block); + ex_ee_len = ext4_ext_get_actual_len(ex); + } + + if (correct_index && eh->eh_entries) + err = ext4_ext_correct_indexes(handle, inode, path); + + /* + * If there's a partial cluster and at least one extent remains in + * the leaf, free the partial cluster if it isn't shared with the + * current extent. If it is shared with the current extent + * we reset the partial cluster because we've reached the start of the + * truncated/punched region and we're done removing blocks. + */ + if (partial->state == tofree && ex >= EXT_FIRST_EXTENT(eh)) { + pblk = ext4_ext_pblock(ex) + ex_ee_len - 1; + if (partial->pclu != EXT4_B2C(sbi, pblk)) { + int flags = get_default_free_blocks_flags(inode); + + if (ext4_is_pending(inode, partial->lblk)) + flags |= EXT4_FREE_BLOCKS_RERESERVE_CLUSTER; + ext4_free_blocks(handle, inode, NULL, + EXT4_C2B(sbi, partial->pclu), + sbi->s_cluster_ratio, flags); + if (flags & EXT4_FREE_BLOCKS_RERESERVE_CLUSTER) + ext4_rereserve_cluster(inode, partial->lblk); + } + partial->state = initial; + } + + /* if this leaf is free, then we should + * remove it from index block above */ + if (err == 0 && eh->eh_entries == 0 && path[depth].p_bh != NULL) + err = ext4_ext_rm_idx(handle, inode, path, depth); + +out: + return err; +} + +/* + * ext4_ext_more_to_rm: + * returns 1 if current index has to be freed (even partial) + */ +static int +ext4_ext_more_to_rm(struct ext4_ext_path *path) +{ + BUG_ON(path->p_idx == NULL); + + if (path->p_idx < EXT_FIRST_INDEX(path->p_hdr)) + return 0; + + /* + * if truncate on deeper level happened, it wasn't partial, + * so we have to consider current index for truncation + */ + if (le16_to_cpu(path->p_hdr->eh_entries) == path->p_block) + return 0; + return 1; +} + +int ext4_ext_remove_space(struct inode *inode, ext4_lblk_t start, + ext4_lblk_t end) +{ + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + int depth = ext_depth(inode); + struct ext4_ext_path *path = NULL; + struct partial_cluster partial; + handle_t *handle; + int i = 0, err = 0; + int flags = EXT4_EX_NOCACHE | EXT4_EX_NOFAIL; + + partial.pclu = 0; + partial.lblk = 0; + partial.state = initial; + + ext_debug(inode, "truncate since %u to %u\n", start, end); + + /* probably first extent we're gonna free will be last in block */ + handle = ext4_journal_start_with_revoke(inode, EXT4_HT_TRUNCATE, + depth + 1, + ext4_free_metadata_revoke_credits(inode->i_sb, depth)); + if (IS_ERR(handle)) + return PTR_ERR(handle); + +again: + trace_ext4_ext_remove_space(inode, start, end, depth); + + /* + * Check if we are removing extents inside the extent tree. If that + * is the case, we are going to punch a hole inside the extent tree + * so we have to check whether we need to split the extent covering + * the last block to remove so we can easily remove the part of it + * in ext4_ext_rm_leaf(). + */ + if (end < EXT_MAX_BLOCKS - 1) { + struct ext4_extent *ex; + ext4_lblk_t ee_block, ex_end, lblk; + ext4_fsblk_t pblk; + + /* find extent for or closest extent to this block */ + path = ext4_find_extent(inode, end, NULL, flags); + if (IS_ERR(path)) { + ext4_journal_stop(handle); + return PTR_ERR(path); + } + depth = ext_depth(inode); + /* Leaf not may not exist only if inode has no blocks at all */ + ex = path[depth].p_ext; + if (!ex) { + if (depth) { + EXT4_ERROR_INODE(inode, + "path[%d].p_hdr == NULL", + depth); + err = -EFSCORRUPTED; + } + goto out; + } + + ee_block = le32_to_cpu(ex->ee_block); + ex_end = ee_block + ext4_ext_get_actual_len(ex) - 1; + + /* + * See if the last block is inside the extent, if so split + * the extent at 'end' block so we can easily remove the + * tail of the first part of the split extent in + * ext4_ext_rm_leaf(). + */ + if (end >= ee_block && end < ex_end) { + + /* + * If we're going to split the extent, note that + * the cluster containing the block after 'end' is + * in use to avoid freeing it when removing blocks. + */ + if (sbi->s_cluster_ratio > 1) { + pblk = ext4_ext_pblock(ex) + end - ee_block + 1; + partial.pclu = EXT4_B2C(sbi, pblk); + partial.state = nofree; + } + + /* + * Split the extent in two so that 'end' is the last + * block in the first new extent. Also we should not + * fail removing space due to ENOSPC so try to use + * reserved block if that happens. + */ + path = ext4_force_split_extent_at(handle, inode, path, + end + 1, 1); + if (IS_ERR(path)) { + err = PTR_ERR(path); + goto out; + } + } else if (sbi->s_cluster_ratio > 1 && end >= ex_end && + partial.state == initial) { + /* + * If we're punching, there's an extent to the right. + * If the partial cluster hasn't been set, set it to + * that extent's first cluster and its state to nofree + * so it won't be freed should it contain blocks to be + * removed. If it's already set (tofree/nofree), we're + * retrying and keep the original partial cluster info + * so a cluster marked tofree as a result of earlier + * extent removal is not lost. + */ + lblk = ex_end + 1; + err = ext4_ext_search_right(inode, path, &lblk, &pblk, + NULL, flags); + if (err < 0) + goto out; + if (pblk) { + partial.pclu = EXT4_B2C(sbi, pblk); + partial.state = nofree; + } + } + } + /* + * We start scanning from right side, freeing all the blocks + * after i_size and walking into the tree depth-wise. + */ + depth = ext_depth(inode); + if (path) { + int k = i = depth; + while (--k > 0) + path[k].p_block = + le16_to_cpu(path[k].p_hdr->eh_entries)+1; + } else { + path = kzalloc_objs(struct ext4_ext_path, depth + 1, + GFP_NOFS | __GFP_NOFAIL); + path[0].p_maxdepth = path[0].p_depth = depth; + path[0].p_hdr = ext_inode_hdr(inode); + i = 0; + + if (ext4_ext_check(inode, path[0].p_hdr, depth, 0)) { + err = -EFSCORRUPTED; + goto out; + } + } + err = 0; + + while (i >= 0 && err == 0) { + if (i == depth) { + /* this is leaf block */ + err = ext4_ext_rm_leaf(handle, inode, path, + &partial, start, end); + /* root level has p_bh == NULL, brelse() eats this */ + ext4_ext_path_brelse(path + i); + i--; + continue; + } + + /* this is index block */ + if (!path[i].p_hdr) { + ext_debug(inode, "initialize header\n"); + path[i].p_hdr = ext_block_hdr(path[i].p_bh); + } + + if (!path[i].p_idx) { + /* this level hasn't been touched yet */ + path[i].p_idx = EXT_LAST_INDEX(path[i].p_hdr); + path[i].p_block = le16_to_cpu(path[i].p_hdr->eh_entries)+1; + ext_debug(inode, "init index ptr: hdr 0x%p, num %d\n", + path[i].p_hdr, + le16_to_cpu(path[i].p_hdr->eh_entries)); + } else { + /* we were already here, see at next index */ + path[i].p_idx--; + } + + ext_debug(inode, "level %d - index, first 0x%p, cur 0x%p\n", + i, EXT_FIRST_INDEX(path[i].p_hdr), + path[i].p_idx); + if (ext4_ext_more_to_rm(path + i)) { + struct buffer_head *bh; + /* go to the next level */ + ext_debug(inode, "move to level %d (block %llu)\n", + i + 1, ext4_idx_pblock(path[i].p_idx)); + memset(path + i + 1, 0, sizeof(*path)); + bh = read_extent_tree_block(inode, path[i].p_idx, + depth - i - 1, flags); + if (IS_ERR(bh)) { + /* should we reset i_size? */ + err = PTR_ERR(bh); + break; + } + /* Yield here to deal with large extent trees. + * Should be a no-op if we did IO above. */ + cond_resched(); + if (WARN_ON(i + 1 > depth)) { + err = -EFSCORRUPTED; + break; + } + path[i + 1].p_bh = bh; + + /* save actual number of indexes since this + * number is changed at the next iteration */ + path[i].p_block = le16_to_cpu(path[i].p_hdr->eh_entries); + i++; + } else { + /* we finished processing this index, go up */ + if (path[i].p_hdr->eh_entries == 0 && i > 0) { + /* index is empty, remove it; + * handle must be already prepared by the + * truncatei_leaf() */ + err = ext4_ext_rm_idx(handle, inode, path, i); + } + /* root level has p_bh == NULL, brelse() eats this */ + ext4_ext_path_brelse(path + i); + i--; + ext_debug(inode, "return to level %d\n", i); + } + } + + trace_ext4_ext_remove_space_done(inode, start, end, depth, &partial, + path->p_hdr->eh_entries); + + /* + * if there's a partial cluster and we have removed the first extent + * in the file, then we also free the partial cluster, if any + */ + if (partial.state == tofree && err == 0) { + int flags = get_default_free_blocks_flags(inode); + + if (ext4_is_pending(inode, partial.lblk)) + flags |= EXT4_FREE_BLOCKS_RERESERVE_CLUSTER; + ext4_free_blocks(handle, inode, NULL, + EXT4_C2B(sbi, partial.pclu), + sbi->s_cluster_ratio, flags); + if (flags & EXT4_FREE_BLOCKS_RERESERVE_CLUSTER) + ext4_rereserve_cluster(inode, partial.lblk); + partial.state = initial; + } + + /* TODO: flexible tree reduction should be here */ + if (path->p_hdr->eh_entries == 0) { + /* + * truncate to zero freed all the tree, + * so we need to correct eh_depth + */ + err = ext4_ext_get_access(handle, inode, path); + if (err == 0) { + ext_inode_hdr(inode)->eh_depth = 0; + ext_inode_hdr(inode)->eh_max = + cpu_to_le16(ext4_ext_space_root(inode, 0)); + err = ext4_ext_dirty(handle, inode, path); + } + } +out: + ext4_free_ext_path(path); + path = NULL; + if (err == -EAGAIN) + goto again; + ext4_journal_stop(handle); + + return err; +} + +/* + * called at mount time + */ +void ext4_ext_init(struct super_block *sb) +{ + /* + * possible initialization would be here + */ + + if (ext4_has_feature_extents(sb)) { +#if defined(AGGRESSIVE_TEST) || defined(CHECK_BINSEARCH) || defined(EXTENTS_STATS) + printk(KERN_INFO "EXT4-fs: file extents enabled" +#ifdef AGGRESSIVE_TEST + ", aggressive tests" +#endif +#ifdef CHECK_BINSEARCH + ", check binsearch" +#endif +#ifdef EXTENTS_STATS + ", stats" +#endif + "\n"); +#endif +#ifdef EXTENTS_STATS + spin_lock_init(&EXT4_SB(sb)->s_ext_stats_lock); + EXT4_SB(sb)->s_ext_min = 1 << 30; + EXT4_SB(sb)->s_ext_max = 0; +#endif + } +} + +/* + * called at umount time + */ +void ext4_ext_release(struct super_block *sb) +{ + if (!ext4_has_feature_extents(sb)) + return; + +#ifdef EXTENTS_STATS + if (EXT4_SB(sb)->s_ext_blocks && EXT4_SB(sb)->s_ext_extents) { + struct ext4_sb_info *sbi = EXT4_SB(sb); + printk(KERN_ERR "EXT4-fs: %lu blocks in %lu extents (%lu ave)\n", + sbi->s_ext_blocks, sbi->s_ext_extents, + sbi->s_ext_blocks / sbi->s_ext_extents); + printk(KERN_ERR "EXT4-fs: extents: %lu min, %lu max, max depth %lu\n", + sbi->s_ext_min, sbi->s_ext_max, sbi->s_depth_max); + } +#endif +} + +static void ext4_zeroout_es(struct inode *inode, struct ext4_extent *ex) +{ + ext4_lblk_t ee_block; + ext4_fsblk_t ee_pblock; + unsigned int ee_len; + + ee_block = le32_to_cpu(ex->ee_block); + ee_len = ext4_ext_get_actual_len(ex); + ee_pblock = ext4_ext_pblock(ex); + + if (ee_len == 0) + return; + + ext4_es_insert_extent(inode, ee_block, ee_len, ee_pblock, + EXTENT_STATUS_WRITTEN, false); +} + +/* FIXME!! we need to try to merge to left or right after zero-out */ +int ext4_ext_zeroout(struct inode *inode, struct ext4_extent *ex) +{ + ext4_fsblk_t ee_pblock; + unsigned int ee_len; + + KUNIT_STATIC_STUB_REDIRECT(ext4_ext_zeroout, inode, ex); + + ee_len = ext4_ext_get_actual_len(ex); + ee_pblock = ext4_ext_pblock(ex); + return ext4_issue_zeroout(inode, le32_to_cpu(ex->ee_block), ee_pblock, + ee_len); +} + +/* + * ext4_split_extent_at() splits an extent at given block. + * + * @handle: the journal handle + * @inode: the file inode + * @path: the path to the extent + * @split: the logical block where the extent is splitted. + * @flags: flags used to insert new extent to extent tree. + * + * + * Splits extent [a, b] into two extents [a, @split) and [@split, b], states + * of which are same as the original extent. No conversion is performed. + * + * Return an extent path pointer on success, or an error pointer on failure. On + * failure, the extent is restored to original state. + */ +static struct ext4_ext_path *ext4_split_extent_at(handle_t *handle, + struct inode *inode, + struct ext4_ext_path *path, + ext4_lblk_t split, + int flags) +{ + ext4_fsblk_t newblock; + ext4_lblk_t ee_block; + struct ext4_extent *ex, newex, orig_ex; + struct ext4_extent *ex2 = NULL; + unsigned int ee_len, depth; + int err = 0, insert_err = 0, is_unwrit = 0; + + /* Do not cache extents that are in the process of being modified. */ + flags |= EXT4_EX_NOCACHE; + + ext_debug(inode, "logical block %llu\n", (unsigned long long)split); + + ext4_ext_show_leaf(inode, path); + + depth = ext_depth(inode); + ex = path[depth].p_ext; + ee_block = le32_to_cpu(ex->ee_block); + ee_len = ext4_ext_get_actual_len(ex); + newblock = split - ee_block + ext4_ext_pblock(ex); + is_unwrit = ext4_ext_is_unwritten(ex); + + BUG_ON(split < ee_block || split >= (ee_block + ee_len)); + + /* + * No split needed + */ + if (split == ee_block) + goto out; + + err = ext4_ext_get_access(handle, inode, path + depth); + if (err) + goto out; + + /* case a */ + memcpy(&orig_ex, ex, sizeof(orig_ex)); + ex->ee_len = cpu_to_le16(split - ee_block); + if (is_unwrit) + ext4_ext_mark_unwritten(ex); + + /* + * path may lead to new leaf, not to original leaf any more + * after ext4_ext_insert_extent() returns, + */ + err = ext4_ext_dirty(handle, inode, path + depth); + if (err) + goto fix_extent_len; + + ex2 = &newex; + ex2->ee_block = cpu_to_le32(split); + ex2->ee_len = cpu_to_le16(ee_len - (split - ee_block)); + ext4_ext_store_pblock(ex2, newblock); + if (is_unwrit) + ext4_ext_mark_unwritten(ex2); + + path = ext4_ext_insert_extent(handle, inode, path, &newex, flags); + if (!IS_ERR(path)) + return path; + + insert_err = PTR_ERR(path); + err = 0; + if (insert_err != -ENOSPC && insert_err != -EDQUOT && + insert_err != -ENOMEM) + goto out_path; + + /* + * Get a new path to try to zeroout or fix the extent length. + * Using EXT4_EX_NOFAIL guarantees that ext4_find_extent() + * will not return -ENOMEM, otherwise -ENOMEM will cause a + * retry in do_writepages(), and a WARN_ON may be triggered + * in ext4_da_update_reserve_space() due to an incorrect + * ee_len causing the i_reserved_data_blocks exception. + */ + path = ext4_find_extent(inode, ee_block, NULL, flags | EXT4_EX_NOFAIL); + if (IS_ERR(path)) { + EXT4_ERROR_INODE(inode, "Failed split extent on %u, err %pe", + split, path); + goto out_path; + } + + depth = ext_depth(inode); + ex = path[depth].p_ext; + if (!ex) { + EXT4_ERROR_INODE(inode, + "bad extent address lblock: %lu, depth: %d pblock %llu", + (unsigned long)ee_block, depth, path[depth].p_block); + err = -EFSCORRUPTED; + goto out; + } + + err = ext4_ext_get_access(handle, inode, path + depth); + if (err) + goto out; + +fix_extent_len: + ex->ee_len = orig_ex.ee_len; + err = ext4_ext_dirty(handle, inode, path + path->p_depth); +out: + if (err || insert_err) { + ext4_free_ext_path(path); + path = err ? ERR_PTR(err) : ERR_PTR(insert_err); + } +out_path: + if (IS_ERR(path)) + /* Remove all remaining potentially stale extents. */ + ext4_es_remove_extent(inode, ee_block, ee_len); + ext4_ext_show_leaf(inode, path); + return path; +} + +static int ext4_split_extent_zeroout(handle_t *handle, struct inode *inode, + struct ext4_ext_path *path, + struct ext4_map_blocks *map, int flags) +{ + struct ext4_extent *ex; + unsigned int ee_len, depth; + ext4_lblk_t ee_block; + uint64_t lblk, pblk, len; + int is_unwrit; + int err = 0; + + depth = ext_depth(inode); + ex = path[depth].p_ext; + ee_block = le32_to_cpu(ex->ee_block); + ee_len = ext4_ext_get_actual_len(ex); + is_unwrit = ext4_ext_is_unwritten(ex); + + if (flags & EXT4_GET_BLOCKS_CONVERT) { + /* + * EXT4_GET_BLOCKS_CONVERT: Caller wants the range specified by + * map to be initialized. Zeroout everything except the map + * range. + */ + + loff_t map_end = (loff_t) map->m_lblk + map->m_len; + loff_t ex_end = (loff_t) ee_block + ee_len; + + if (!is_unwrit) + /* Shouldn't happen. Just exit */ + return -EINVAL; + + /* zeroout left */ + if (map->m_lblk > ee_block) { + lblk = ee_block; + len = map->m_lblk - ee_block; + pblk = ext4_ext_pblock(ex); + err = ext4_issue_zeroout(inode, lblk, pblk, len); + if (err) + /* ZEROOUT failed, just return original error */ + return err; + } + + /* zeroout right */ + if (map_end < ex_end) { + lblk = map_end; + len = ex_end - map_end; + pblk = ext4_ext_pblock(ex) + (map_end - ee_block); + err = ext4_issue_zeroout(inode, lblk, pblk, len); + if (err) + /* ZEROOUT failed, just return original error */ + return err; + } + } else if (flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN) { + /* + * EXT4_GET_BLOCKS_CONVERT_UNWRITTEN: Caller wants the + * range specified by map to be marked unwritten. + * Zeroout the map range leaving rest as it is. + */ + + if (is_unwrit) + /* Shouldn't happen. Just exit */ + return -EINVAL; + + lblk = map->m_lblk; + len = map->m_len; + pblk = ext4_ext_pblock(ex) + (map->m_lblk - ee_block); + err = ext4_issue_zeroout(inode, lblk, pblk, len); + if (err) + /* ZEROOUT failed, just return original error */ + return err; + } else { + /* + * We no longer perform unwritten to unwritten splits in IO paths. + * Hence this should not happen. + */ + WARN_ON_ONCE(true); + return -EINVAL; + } + + err = ext4_ext_get_access(handle, inode, path + depth); + if (err) + return err; + + ext4_ext_mark_initialized(ex); + + err = ext4_ext_dirty(handle, inode, path + depth); + if (err) + return err; + + return 0; +} + +/* + * ext4_split_extent() splits an extent and mark extent which is covered + * by @map as split_flags indicates + * + * It may result in splitting the extent into multiple extents (up to three) + * There are three possibilities: + * a> There is no split required + * b> Splits in two extents: Split is happening at either end of the extent + * c> Splits in three extents: Somone is splitting in middle of the extent + * + */ +static struct ext4_ext_path *ext4_split_extent(handle_t *handle, + struct inode *inode, + struct ext4_ext_path *path, + struct ext4_map_blocks *map, + int split_flag, int flags, + unsigned int *allocated, bool *did_zeroout) +{ + ext4_lblk_t ee_block, orig_ee_block; + struct ext4_extent *ex; + unsigned int ee_len, orig_ee_len, depth; + int unwritten, orig_unwritten; + int orig_err = 0; + + depth = ext_depth(inode); + ex = path[depth].p_ext; + ee_block = le32_to_cpu(ex->ee_block); + ee_len = ext4_ext_get_actual_len(ex); + unwritten = ext4_ext_is_unwritten(ex); + + orig_ee_block = ee_block; + orig_ee_len = ee_len; + orig_unwritten = unwritten; + + /* Do not cache extents that are in the process of being modified. */ + flags |= EXT4_EX_NOCACHE; + + if (map->m_lblk + map->m_len < ee_block + ee_len) { + path = ext4_split_extent_at(handle, inode, path, + map->m_lblk + map->m_len, flags); + if (IS_ERR(path)) + goto try_zeroout; + + /* + * Update path is required because previous ext4_split_extent_at + * may result in split of original leaf or extent zeroout. + */ + path = ext4_find_extent(inode, map->m_lblk, path, flags); + if (IS_ERR(path)) + goto try_zeroout; + + depth = ext_depth(inode); + ex = path[depth].p_ext; + if (!ex) { + EXT4_ERROR_INODE(inode, "unexpected hole at %lu", + (unsigned long) map->m_lblk); + ext4_free_ext_path(path); + return ERR_PTR(-EFSCORRUPTED); + } + + /* extent would have changed so update original values */ + orig_ee_block = le32_to_cpu(ex->ee_block); + orig_ee_len = ext4_ext_get_actual_len(ex); + orig_unwritten = ext4_ext_is_unwritten(ex); + } + + if (map->m_lblk >= ee_block) { + path = ext4_split_extent_at(handle, inode, path, map->m_lblk, + flags); + if (IS_ERR(path)) + goto try_zeroout; + } + + goto success; + +try_zeroout: + /* + * There was an error in splitting the extent. So instead, just zeroout + * unwritten portions and convert it to initialized as a last resort. If + * there is any failure here we just return the original error + */ + + orig_err = PTR_ERR(path); + if (orig_err != -ENOSPC && orig_err != -EDQUOT && orig_err != -ENOMEM) + goto out_orig_err; + + /* we can't zeroout? just return the original err */ + if (!(split_flag & EXT4_EXT_MAY_ZEROOUT)) + goto out_orig_err; + + if (flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN) { + int max_zeroout_blks = + EXT4_SB(inode->i_sb)->s_extent_max_zeroout_kb >> + (inode->i_sb->s_blocksize_bits - 10); + + if (map->m_len > max_zeroout_blks) + goto out_orig_err; + } + + path = ext4_find_extent(inode, map->m_lblk, NULL, flags); + if (IS_ERR(path)) + goto out_orig_err; + + depth = ext_depth(inode); + ex = path[depth].p_ext; + ee_block = le32_to_cpu(ex->ee_block); + ee_len = ext4_ext_get_actual_len(ex); + unwritten = ext4_ext_is_unwritten(ex); + + /* extent to zeroout should have been unchanged but its not */ + if (WARN_ON(ee_block != orig_ee_block || ee_len != orig_ee_len || + unwritten != orig_unwritten)) + goto out_free_path; + + if (ext4_split_extent_zeroout(handle, inode, path, map, flags)) + goto out_free_path; + + /* zeroout succeeded */ + if (did_zeroout) + *did_zeroout = true; + +success: + if (allocated) { + if (map->m_lblk + map->m_len > ee_block + ee_len) + *allocated = ee_len - (map->m_lblk - ee_block); + else + *allocated = map->m_len; + } + ext4_ext_show_leaf(inode, path); + return path; + +out_free_path: + ext4_free_ext_path(path); +out_orig_err: + return ERR_PTR(orig_err); + +} + +/* + * This function is called by ext4_ext_map_blocks() if someone tries to write + * to an unwritten extent. It may result in splitting the unwritten + * extent into multiple extents (up to three - one initialized and two + * unwritten). + * There are three possibilities: + * a> There is no split required: Entire extent should be initialized + * b> Splits in two extents: Write is happening at either end of the extent + * c> Splits in three extents: Somone is writing in middle of the extent + * + * Pre-conditions: + * - The extent pointed to by 'path' is unwritten. + * - The extent pointed to by 'path' contains a superset + * of the logical span [map->m_lblk, map->m_lblk + map->m_len). + * + * Post-conditions on success: + * - the returned value is the number of blocks beyond map->l_lblk + * that are allocated and initialized. + * It is guaranteed to be >= map->m_len. + */ +static struct ext4_ext_path * +ext4_ext_convert_to_initialized(handle_t *handle, struct inode *inode, + struct ext4_map_blocks *map, struct ext4_ext_path *path, + int flags, unsigned int *allocated) +{ + struct ext4_sb_info *sbi; + struct ext4_extent_header *eh; + struct ext4_map_blocks split_map; + struct ext4_extent zero_ex1, zero_ex2; + struct ext4_extent *ex, *abut_ex; + ext4_lblk_t ee_block, eof_block; + unsigned int ee_len, depth, map_len = map->m_len; + int err = 0; + unsigned int max_zeroout = 0; + + ext_debug(inode, "logical block %llu, max_blocks %u\n", + (unsigned long long)map->m_lblk, map_len); + + sbi = EXT4_SB(inode->i_sb); + eof_block = (EXT4_I(inode)->i_disksize + inode->i_sb->s_blocksize - 1) + >> inode->i_sb->s_blocksize_bits; + if (eof_block < map->m_lblk + map_len) + eof_block = map->m_lblk + map_len; + + depth = ext_depth(inode); + eh = path[depth].p_hdr; + ex = path[depth].p_ext; + ee_block = le32_to_cpu(ex->ee_block); + ee_len = ext4_ext_get_actual_len(ex); + zero_ex1.ee_len = 0; + zero_ex2.ee_len = 0; + + trace_ext4_ext_convert_to_initialized_enter(inode, map, ex); + + /* Pre-conditions */ + BUG_ON(!ext4_ext_is_unwritten(ex)); + BUG_ON(!in_range(map->m_lblk, ee_block, ee_len)); + + /* + * Attempt to transfer newly initialized blocks from the currently + * unwritten extent to its neighbor. This is much cheaper + * than an insertion followed by a merge as those involve costly + * memmove() calls. Transferring to the left is the common case in + * steady state for workloads doing fallocate(FALLOC_FL_KEEP_SIZE) + * followed by append writes. + * + * Limitations of the current logic: + * - L1: we do not deal with writes covering the whole extent. + * This would require removing the extent if the transfer + * is possible. + * - L2: we only attempt to merge with an extent stored in the + * same extent tree node. + */ + *allocated = 0; + if ((map->m_lblk == ee_block) && + /* See if we can merge left */ + (map_len < ee_len) && /*L1*/ + (ex > EXT_FIRST_EXTENT(eh))) { /*L2*/ + ext4_lblk_t prev_lblk; + ext4_fsblk_t prev_pblk, ee_pblk; + unsigned int prev_len; + + abut_ex = ex - 1; + prev_lblk = le32_to_cpu(abut_ex->ee_block); + prev_len = ext4_ext_get_actual_len(abut_ex); + prev_pblk = ext4_ext_pblock(abut_ex); + ee_pblk = ext4_ext_pblock(ex); + + /* + * A transfer of blocks from 'ex' to 'abut_ex' is allowed + * upon those conditions: + * - C1: abut_ex is initialized, + * - C2: abut_ex is logically abutting ex, + * - C3: abut_ex is physically abutting ex, + * - C4: abut_ex can receive the additional blocks without + * overflowing the (initialized) length limit. + */ + if ((!ext4_ext_is_unwritten(abut_ex)) && /*C1*/ + ((prev_lblk + prev_len) == ee_block) && /*C2*/ + ((prev_pblk + prev_len) == ee_pblk) && /*C3*/ + (prev_len < (EXT_INIT_MAX_LEN - map_len))) { /*C4*/ + err = ext4_ext_get_access(handle, inode, path + depth); + if (err) + goto errout; + + trace_ext4_ext_convert_to_initialized_fastpath(inode, + map, ex, abut_ex); + + /* Shift the start of ex by 'map_len' blocks */ + ex->ee_block = cpu_to_le32(ee_block + map_len); + ext4_ext_store_pblock(ex, ee_pblk + map_len); + ex->ee_len = cpu_to_le16(ee_len - map_len); + ext4_ext_mark_unwritten(ex); /* Restore the flag */ + + /* Extend abut_ex by 'map_len' blocks */ + abut_ex->ee_len = cpu_to_le16(prev_len + map_len); + + /* Result: number of initialized blocks past m_lblk */ + *allocated = map_len; + } + } else if (((map->m_lblk + map_len) == (ee_block + ee_len)) && + (map_len < ee_len) && /*L1*/ + ex < EXT_LAST_EXTENT(eh)) { /*L2*/ + /* See if we can merge right */ + ext4_lblk_t next_lblk; + ext4_fsblk_t next_pblk, ee_pblk; + unsigned int next_len; + + abut_ex = ex + 1; + next_lblk = le32_to_cpu(abut_ex->ee_block); + next_len = ext4_ext_get_actual_len(abut_ex); + next_pblk = ext4_ext_pblock(abut_ex); + ee_pblk = ext4_ext_pblock(ex); + + /* + * A transfer of blocks from 'ex' to 'abut_ex' is allowed + * upon those conditions: + * - C1: abut_ex is initialized, + * - C2: abut_ex is logically abutting ex, + * - C3: abut_ex is physically abutting ex, + * - C4: abut_ex can receive the additional blocks without + * overflowing the (initialized) length limit. + */ + if ((!ext4_ext_is_unwritten(abut_ex)) && /*C1*/ + ((map->m_lblk + map_len) == next_lblk) && /*C2*/ + ((ee_pblk + ee_len) == next_pblk) && /*C3*/ + (next_len < (EXT_INIT_MAX_LEN - map_len))) { /*C4*/ + err = ext4_ext_get_access(handle, inode, path + depth); + if (err) + goto errout; + + trace_ext4_ext_convert_to_initialized_fastpath(inode, + map, ex, abut_ex); + + /* Shift the start of abut_ex by 'map_len' blocks */ + abut_ex->ee_block = cpu_to_le32(next_lblk - map_len); + ext4_ext_store_pblock(abut_ex, next_pblk - map_len); + ex->ee_len = cpu_to_le16(ee_len - map_len); + ext4_ext_mark_unwritten(ex); /* Restore the flag */ + + /* Extend abut_ex by 'map_len' blocks */ + abut_ex->ee_len = cpu_to_le16(next_len + map_len); + + /* Result: number of initialized blocks past m_lblk */ + *allocated = map_len; + } + } + if (*allocated) { + /* Mark the block containing both extents as dirty */ + err = ext4_ext_dirty(handle, inode, path + depth); + + /* Update path to point to the right extent */ + path[depth].p_ext = abut_ex; + if (err) + goto errout; + goto out; + } else + *allocated = ee_len - (map->m_lblk - ee_block); + + WARN_ON(map->m_lblk < ee_block); + /* + * It is safe to convert extent to initialized via explicit + * zeroout only if extent is fully inside i_size or new_size. + */ + if (ee_block + ee_len <= eof_block) + max_zeroout = sbi->s_extent_max_zeroout_kb >> + (inode->i_sb->s_blocksize_bits - 10); + + /* + * five cases: + * 1. split the extent into three extents. + * 2. split the extent into two extents, zeroout the head of the first + * extent. + * 3. split the extent into two extents, zeroout the tail of the second + * extent. + * 4. split the extent into two extents with out zeroout. + * 5. no splitting needed, just possibly zeroout the head and / or the + * tail of the extent. + */ + split_map.m_lblk = map->m_lblk; + split_map.m_len = map->m_len; + + if (max_zeroout && (*allocated > split_map.m_len)) { + if (*allocated <= max_zeroout) { + /* case 3 or 5 */ + zero_ex1.ee_block = + cpu_to_le32(split_map.m_lblk + + split_map.m_len); + zero_ex1.ee_len = + cpu_to_le16(*allocated - split_map.m_len); + ext4_ext_store_pblock(&zero_ex1, + ext4_ext_pblock(ex) + split_map.m_lblk + + split_map.m_len - ee_block); + err = ext4_ext_zeroout(inode, &zero_ex1); + if (err) + goto fallback; + split_map.m_len = *allocated; + } + if (split_map.m_lblk - ee_block + split_map.m_len < + max_zeroout) { + /* case 2 or 5 */ + if (split_map.m_lblk != ee_block) { + zero_ex2.ee_block = ex->ee_block; + zero_ex2.ee_len = cpu_to_le16(split_map.m_lblk - + ee_block); + ext4_ext_store_pblock(&zero_ex2, + ext4_ext_pblock(ex)); + err = ext4_ext_zeroout(inode, &zero_ex2); + if (err) + goto fallback; + } + + split_map.m_len += split_map.m_lblk - ee_block; + split_map.m_lblk = ee_block; + *allocated = map->m_len; + } + } + +fallback: + path = ext4_split_convert_extents(handle, inode, &split_map, path, + flags | EXT4_GET_BLOCKS_CONVERT, NULL); + if (IS_ERR(path)) + return path; +out: + /* If we have gotten a failure, don't zero out status tree */ + ext4_zeroout_es(inode, &zero_ex1); + ext4_zeroout_es(inode, &zero_ex2); + return path; + +errout: + ext4_free_ext_path(path); + return ERR_PTR(err); +} + +/* + * This function is called by ext4_ext_map_blocks() from + * ext4_get_blocks_dio_write() when DIO to write + * to an unwritten extent. + * + * Writing to an unwritten extent may result in splitting the unwritten + * extent into multiple initialized/unwritten extents (up to three) + * There are three possibilities: + * a> There is no split required: Entire extent should be unwritten + * b> Splits in two extents: Write is happening at either end of the extent + * c> Splits in three extents: Somone is writing in middle of the extent + * + * This works the same way in the case of initialized -> unwritten conversion. + * + * One of more index blocks maybe needed if the extent tree grow after + * the unwritten extent split. To prevent ENOSPC occur at the IO + * complete, we need to split the unwritten extent before DIO submit + * the IO. The unwritten extent called at this time will be split + * into three unwritten extent(at most). After IO complete, the part + * being filled will be convert to initialized by the end_io callback function + * via ext4_convert_unwritten_extents(). + * + * The size of unwritten extent to be written is passed to the caller via the + * allocated pointer. Return an extent path pointer on success, or an error + * pointer on failure. + */ +static struct ext4_ext_path *ext4_split_convert_extents(handle_t *handle, + struct inode *inode, + struct ext4_map_blocks *map, + struct ext4_ext_path *path, + int flags, unsigned int *allocated) +{ + ext4_lblk_t eof_block; + ext4_lblk_t ee_block; + struct ext4_extent *ex; + unsigned int ee_len; + int split_flag = 0, depth, err = 0; + bool did_zeroout = false; + + ext_debug(inode, "logical block %llu, max_blocks %u\n", + (unsigned long long)map->m_lblk, map->m_len); + + eof_block = (EXT4_I(inode)->i_disksize + inode->i_sb->s_blocksize - 1) + >> inode->i_sb->s_blocksize_bits; + if (eof_block < map->m_lblk + map->m_len) + eof_block = map->m_lblk + map->m_len; + depth = ext_depth(inode); + ex = path[depth].p_ext; + ee_block = le32_to_cpu(ex->ee_block); + ee_len = ext4_ext_get_actual_len(ex); + + /* No split needed */ + if (ee_block == map->m_lblk && ee_len == map->m_len) + goto convert; + + /* + * It is only safe to convert extent to initialized via explicit + * zeroout only if extent is fully inside i_size or new_size. + */ + split_flag |= ee_block + ee_len <= eof_block ? EXT4_EXT_MAY_ZEROOUT : 0; + + /* + * pass SPLIT_NOMERGE explicitly so we don't end up merging extents we + * just split. + */ + path = ext4_split_extent(handle, inode, path, map, split_flag, + flags | EXT4_GET_BLOCKS_SPLIT_NOMERGE, + allocated, &did_zeroout); + if (IS_ERR(path)) + return path; + +convert: + path = ext4_find_extent(inode, map->m_lblk, path, flags); + if (IS_ERR(path)) + return path; + + depth = ext_depth(inode); + ex = path[depth].p_ext; + + /* + * Conversion is already handled in case of zeroout + */ + if (!did_zeroout) { + err = ext4_ext_get_access(handle, inode, path + depth); + if (err) + goto err; + + if (flags & EXT4_GET_BLOCKS_CONVERT) + ext4_ext_mark_initialized(ex); + else if (flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN) + ext4_ext_mark_unwritten(ex); + + if (!(flags & EXT4_GET_BLOCKS_SPLIT_NOMERGE)) + /* + * note: ext4_ext_correct_indexes() isn't needed here because + * borders are not changed + */ + ext4_ext_try_to_merge(handle, inode, path, ex); + + err = ext4_ext_dirty(handle, inode, path + depth); + if (err) + goto err; + } + + /* Lets update the extent status tree after conversion */ + if (!(flags & EXT4_EX_NOCACHE)) + ext4_es_insert_extent(inode, le32_to_cpu(ex->ee_block), + ext4_ext_get_actual_len(ex), + ext4_ext_pblock(ex), + ext4_ext_is_unwritten(ex) ? + EXTENT_STATUS_UNWRITTEN : + EXTENT_STATUS_WRITTEN, + false); + +err: + if (err) { + ext4_free_ext_path(path); + return ERR_PTR(err); + } + + return path; +} + +static struct ext4_ext_path * +ext4_convert_unwritten_extents_endio(handle_t *handle, struct inode *inode, + struct ext4_map_blocks *map, + struct ext4_ext_path *path, int flags) +{ + struct ext4_extent *ex; + ext4_lblk_t ee_block; + unsigned int ee_len; + int depth; + + depth = ext_depth(inode); + ex = path[depth].p_ext; + ee_block = le32_to_cpu(ex->ee_block); + ee_len = ext4_ext_get_actual_len(ex); + + ext_debug(inode, "logical block %llu, max_blocks %u\n", + (unsigned long long)ee_block, ee_len); + + return ext4_split_convert_extents(handle, inode, map, path, flags, + NULL); +} + +static struct ext4_ext_path * +convert_initialized_extent(handle_t *handle, struct inode *inode, + struct ext4_map_blocks *map, + struct ext4_ext_path *path, + int flags, + unsigned int *allocated) +{ + struct ext4_extent *ex; + ext4_lblk_t ee_block; + unsigned int ee_len; + int depth; + + /* + * Make sure that the extent is no bigger than we support with + * unwritten extent + */ + if (map->m_len > EXT_UNWRITTEN_MAX_LEN) + map->m_len = EXT_UNWRITTEN_MAX_LEN / 2; + + depth = ext_depth(inode); + ex = path[depth].p_ext; + ee_block = le32_to_cpu(ex->ee_block); + ee_len = ext4_ext_get_actual_len(ex); + + ext_debug(inode, "logical block %llu, max_blocks %u\n", + (unsigned long long)ee_block, ee_len); + + path = ext4_split_convert_extents(handle, inode, map, path, flags, + NULL); + if (IS_ERR(path)) + return path; + + ext4_ext_show_leaf(inode, path); + + ext4_update_inode_fsync_trans(handle, inode, 1); + + /* + * The extent might be initialized in case of zeroout. + */ + path = ext4_find_extent(inode, map->m_lblk, path, flags); + if (IS_ERR(path)) + return path; + + depth = ext_depth(inode); + ex = path[depth].p_ext; + + if (ext4_ext_is_unwritten(ex)) + map->m_flags |= EXT4_MAP_UNWRITTEN; + else + map->m_flags |= EXT4_MAP_MAPPED; + if (*allocated > map->m_len) + *allocated = map->m_len; + map->m_len = *allocated; + return path; +} + +static struct ext4_ext_path * +ext4_ext_handle_unwritten_extents(handle_t *handle, struct inode *inode, + struct ext4_map_blocks *map, + struct ext4_ext_path *path, int flags, + unsigned int *allocated, ext4_fsblk_t newblock) +{ + int err = 0; + + ext_debug(inode, "logical block %llu, max_blocks %u, flags 0x%x, allocated %u\n", + (unsigned long long)map->m_lblk, map->m_len, flags, + *allocated); + ext4_ext_show_leaf(inode, path); + + /* + * When writing into unwritten space, we should not fail to + * allocate metadata blocks for the new extent block if needed. + */ + flags |= EXT4_GET_BLOCKS_METADATA_NOFAIL; + + trace_ext4_ext_handle_unwritten_extents(inode, map, flags, + *allocated, newblock); + + /* IO end_io complete, convert the filled extent to written */ + if (flags & EXT4_GET_BLOCKS_CONVERT) { + path = ext4_convert_unwritten_extents_endio(handle, inode, + map, path, flags); + if (IS_ERR(path)) + return path; + ext4_update_inode_fsync_trans(handle, inode, 1); + goto map_out; + } + /* buffered IO cases */ + /* + * repeat fallocate creation request + * we already have an unwritten extent + */ + if (flags & EXT4_GET_BLOCKS_UNWRIT_EXT) { + map->m_flags |= EXT4_MAP_UNWRITTEN; + goto map_out; + } + + /* buffered READ or buffered write_begin() lookup */ + if ((flags & EXT4_GET_BLOCKS_CREATE) == 0) { + /* + * We have blocks reserved already. We + * return allocated blocks so that delalloc + * won't do block reservation for us. But + * the buffer head will be unmapped so that + * a read from the block returns 0s. + */ + map->m_flags |= EXT4_MAP_UNWRITTEN; + goto out1; + } + + /* + * Default case when (flags & EXT4_GET_BLOCKS_CREATE) == 1. + * For buffered writes, at writepage time, etc. Convert a + * discovered unwritten extent to written. + */ + path = ext4_ext_convert_to_initialized(handle, inode, map, path, + flags, allocated); + if (IS_ERR(path)) + return path; + ext4_update_inode_fsync_trans(handle, inode, 1); + /* + * shouldn't get a 0 allocated when converting an unwritten extent + * unless m_len is 0 (bug) or extent has been corrupted + */ + if (unlikely(*allocated == 0)) { + EXT4_ERROR_INODE(inode, "unexpected allocated == 0, m_len = %u", + map->m_len); + err = -EFSCORRUPTED; + goto errout; + } + + map->m_flags |= EXT4_MAP_NEW; +map_out: + map->m_flags |= EXT4_MAP_MAPPED; +out1: + map->m_pblk = newblock; + if (*allocated > map->m_len) + *allocated = map->m_len; + map->m_len = *allocated; + ext4_ext_show_leaf(inode, path); + return path; + +errout: + ext4_free_ext_path(path); + return ERR_PTR(err); +} + +/* + * get_implied_cluster_alloc - check to see if the requested + * allocation (in the map structure) overlaps with a cluster already + * allocated in an extent. + * @sb The filesystem superblock structure + * @map The requested lblk->pblk mapping + * @ex The extent structure which might contain an implied + * cluster allocation + * + * This function is called by ext4_ext_map_blocks() after we failed to + * find blocks that were already in the inode's extent tree. Hence, + * we know that the beginning of the requested region cannot overlap + * the extent from the inode's extent tree. There are three cases we + * want to catch. The first is this case: + * + * |--- cluster # N--| + * |--- extent ---| |---- requested region ---| + * |==========| + * + * The second case that we need to test for is this one: + * + * |--------- cluster # N ----------------| + * |--- requested region --| |------- extent ----| + * |=======================| + * + * The third case is when the requested region lies between two extents + * within the same cluster: + * |------------- cluster # N-------------| + * |----- ex -----| |---- ex_right ----| + * |------ requested region ------| + * |================| + * + * In each of the above cases, we need to set the map->m_pblk and + * map->m_len so it corresponds to the return the extent labelled as + * "|====|" from cluster #N, since it is already in use for data in + * cluster EXT4_B2C(sbi, map->m_lblk). We will then return 1 to + * signal to ext4_ext_map_blocks() that map->m_pblk should be treated + * as a new "allocated" block region. Otherwise, we will return 0 and + * ext4_ext_map_blocks() will then allocate one or more new clusters + * by calling ext4_mb_new_blocks(). + */ +static int get_implied_cluster_alloc(struct super_block *sb, + struct ext4_map_blocks *map, + struct ext4_extent *ex, + struct ext4_ext_path *path) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + ext4_lblk_t c_offset = EXT4_LBLK_COFF(sbi, map->m_lblk); + ext4_lblk_t ex_cluster_start, ex_cluster_end; + ext4_lblk_t rr_cluster_start; + ext4_lblk_t ee_block = le32_to_cpu(ex->ee_block); + ext4_fsblk_t ee_start = ext4_ext_pblock(ex); + unsigned short ee_len = ext4_ext_get_actual_len(ex); + + /* The extent passed in that we are trying to match */ + ex_cluster_start = EXT4_B2C(sbi, ee_block); + ex_cluster_end = EXT4_B2C(sbi, ee_block + ee_len - 1); + + /* The requested region passed into ext4_map_blocks() */ + rr_cluster_start = EXT4_B2C(sbi, map->m_lblk); + + if ((rr_cluster_start == ex_cluster_end) || + (rr_cluster_start == ex_cluster_start)) { + if (rr_cluster_start == ex_cluster_end) + ee_start += ee_len - 1; + map->m_pblk = EXT4_PBLK_CMASK(sbi, ee_start) + c_offset; + map->m_len = min(map->m_len, + (unsigned) sbi->s_cluster_ratio - c_offset); + /* + * Check for and handle this case: + * + * |--------- cluster # N-------------| + * |------- extent ----| + * |--- requested region ---| + * |===========| + */ + + if (map->m_lblk < ee_block) + map->m_len = min(map->m_len, ee_block - map->m_lblk); + + /* + * Check for the case where there is already another allocated + * block to the right of 'ex' but before the end of the cluster. + * + * |------------- cluster # N-------------| + * |----- ex -----| |---- ex_right ----| + * |------ requested region ------| + * |================| + */ + if (map->m_lblk > ee_block) { + ext4_lblk_t next = ext4_ext_next_allocated_block(path); + map->m_len = min(map->m_len, next - map->m_lblk); + } + + trace_ext4_get_implied_cluster_alloc_exit(sb, map, 1); + return 1; + } + + trace_ext4_get_implied_cluster_alloc_exit(sb, map, 0); + return 0; +} + +/* + * Determine hole length around the given logical block, first try to + * locate and expand the hole from the given @path, and then adjust it + * if it's partially or completely converted to delayed extents, insert + * it into the extent cache tree if it's indeed a hole, finally return + * the length of the determined extent. + */ +static ext4_lblk_t ext4_ext_determine_insert_hole(struct inode *inode, + struct ext4_ext_path *path, + ext4_lblk_t lblk) +{ + ext4_lblk_t hole_start, len; + struct extent_status es; + + hole_start = lblk; + len = ext4_ext_find_hole(inode, path, &hole_start); +again: + ext4_es_find_extent_range(inode, &ext4_es_is_delayed, hole_start, + hole_start + len - 1, &es); + if (!es.es_len) + goto insert_hole; + + /* + * There's a delalloc extent in the hole, handle it if the delalloc + * extent is in front of, behind and straddle the queried range. + */ + if (lblk >= es.es_lblk + es.es_len) { + /* + * The delalloc extent is in front of the queried range, + * find again from the queried start block. + */ + len -= lblk - hole_start; + hole_start = lblk; + goto again; + } else if (in_range(lblk, es.es_lblk, es.es_len)) { + /* + * The delalloc extent containing lblk, it must have been + * added after ext4_map_blocks() checked the extent status + * tree so we are not holding i_rwsem and delalloc info is + * only stabilized by i_data_sem we are going to release + * soon. Don't modify the extent status tree and report + * extent as a hole, just adjust the length to the delalloc + * extent's after lblk. + */ + len = es.es_lblk + es.es_len - lblk; + return len; + } else { + /* + * The delalloc extent is partially or completely behind + * the queried range, update hole length until the + * beginning of the delalloc extent. + */ + len = min(es.es_lblk - hole_start, len); + } + +insert_hole: + /* Put just found gap into cache to speed up subsequent requests */ + ext_debug(inode, " -> %u:%u\n", hole_start, len); + ext4_es_cache_extent(inode, hole_start, len, ~0, EXTENT_STATUS_HOLE); + + /* Update hole_len to reflect hole size after lblk */ + if (hole_start != lblk) + len -= lblk - hole_start; + + return len; +} + +/* + * Block allocation/map/preallocation routine for extents based files + * + * + * Need to be called with + * down_read(&EXT4_I(inode)->i_data_sem) if not allocating file system block + * (ie, flags is zero). Otherwise down_write(&EXT4_I(inode)->i_data_sem) + * + * return > 0, number of blocks already mapped/allocated + * if flags doesn't contain EXT4_GET_BLOCKS_CREATE and these are pre-allocated blocks + * buffer head is unmapped + * otherwise blocks are mapped + * + * return = 0, if plain look up failed (blocks have not been allocated) + * buffer head is unmapped + * + * return < 0, error case. + */ +int ext4_ext_map_blocks(handle_t *handle, struct inode *inode, + struct ext4_map_blocks *map, int flags) +{ + struct ext4_ext_path *path = NULL; + struct ext4_extent newex, *ex, ex2; + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + ext4_fsblk_t newblock = 0, pblk; + int err = 0, depth; + unsigned int allocated = 0, offset = 0; + unsigned int allocated_clusters = 0; + struct ext4_allocation_request ar; + ext4_lblk_t cluster_offset; + + ext_debug(inode, "blocks %u/%u requested\n", map->m_lblk, map->m_len); + trace_ext4_ext_map_blocks_enter(inode, map->m_lblk, map->m_len, flags); + + /* find extent for this block */ + path = ext4_find_extent(inode, map->m_lblk, NULL, flags); + if (IS_ERR(path)) { + err = PTR_ERR(path); + goto out; + } + + depth = ext_depth(inode); + + /* + * consistent leaf must not be empty; + * this situation is possible, though, _during_ tree modification; + * this is why assert can't be put in ext4_find_extent() + */ + if (unlikely(path[depth].p_ext == NULL && depth != 0)) { + EXT4_ERROR_INODE(inode, "bad extent address " + "lblock: %lu, depth: %d pblock %lld", + (unsigned long) map->m_lblk, depth, + path[depth].p_block); + err = -EFSCORRUPTED; + goto out; + } + + ex = path[depth].p_ext; + if (ex) { + ext4_lblk_t ee_block = le32_to_cpu(ex->ee_block); + ext4_fsblk_t ee_start = ext4_ext_pblock(ex); + unsigned short ee_len; + + + /* + * unwritten extents are treated as holes, except that + * we split out initialized portions during a write. + */ + ee_len = ext4_ext_get_actual_len(ex); + + trace_ext4_ext_show_extent(inode, ee_block, ee_start, ee_len); + + /* if found extent covers block, simply return it */ + if (in_range(map->m_lblk, ee_block, ee_len)) { + newblock = map->m_lblk - ee_block + ee_start; + /* number of remaining blocks in the extent */ + allocated = ee_len - (map->m_lblk - ee_block); + ext_debug(inode, "%u fit into %u:%d -> %llu\n", + map->m_lblk, ee_block, ee_len, newblock); + + /* + * If the extent is initialized check whether the + * caller wants to convert it to unwritten. + */ + if ((!ext4_ext_is_unwritten(ex)) && + (flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN)) { + path = convert_initialized_extent(handle, + inode, map, path, flags, &allocated); + if (IS_ERR(path)) + err = PTR_ERR(path); + goto out; + } else if (!ext4_ext_is_unwritten(ex)) { + map->m_flags |= EXT4_MAP_MAPPED; + map->m_pblk = newblock; + if (allocated > map->m_len) + allocated = map->m_len; + map->m_len = allocated; + ext4_ext_show_leaf(inode, path); + goto out; + } + + path = ext4_ext_handle_unwritten_extents( + handle, inode, map, path, flags, + &allocated, newblock); + if (IS_ERR(path)) + err = PTR_ERR(path); + goto out; + } + } + + /* + * requested block isn't allocated yet; + * we couldn't try to create block if flags doesn't contain EXT4_GET_BLOCKS_CREATE + */ + if ((flags & EXT4_GET_BLOCKS_CREATE) == 0) { + ext4_lblk_t len; + + len = ext4_ext_determine_insert_hole(inode, path, map->m_lblk); + + map->m_pblk = 0; + map->m_len = min_t(unsigned int, map->m_len, len); + goto out; + } + + /* + * Okay, we need to do block allocation. + */ + newex.ee_block = cpu_to_le32(map->m_lblk); + cluster_offset = EXT4_LBLK_COFF(sbi, map->m_lblk); + + /* + * If we are doing bigalloc, check to see if the extent returned + * by ext4_find_extent() implies a cluster we can use. + */ + if (cluster_offset && ex && + get_implied_cluster_alloc(inode->i_sb, map, ex, path)) { + ar.len = allocated = map->m_len; + newblock = map->m_pblk; + goto got_allocated_blocks; + } + + /* find neighbour allocated blocks */ + ar.lleft = map->m_lblk; + err = ext4_ext_search_left(inode, path, &ar.lleft, &ar.pleft); + if (err) + goto out; + ar.lright = map->m_lblk; + err = ext4_ext_search_right(inode, path, &ar.lright, &ar.pright, + &ex2, flags); + if (err < 0) + goto out; + + /* Check if the extent after searching to the right implies a + * cluster we can use. */ + if ((sbi->s_cluster_ratio > 1) && err && + get_implied_cluster_alloc(inode->i_sb, map, &ex2, path)) { + ar.len = allocated = map->m_len; + newblock = map->m_pblk; + err = 0; + goto got_allocated_blocks; + } + + /* + * See if request is beyond maximum number of blocks we can have in + * a single extent. For an initialized extent this limit is + * EXT_INIT_MAX_LEN and for an unwritten extent this limit is + * EXT_UNWRITTEN_MAX_LEN. + */ + if (map->m_len > EXT_INIT_MAX_LEN && + !(flags & EXT4_GET_BLOCKS_UNWRIT_EXT)) + map->m_len = EXT_INIT_MAX_LEN; + else if (map->m_len > EXT_UNWRITTEN_MAX_LEN && + (flags & EXT4_GET_BLOCKS_UNWRIT_EXT)) + map->m_len = EXT_UNWRITTEN_MAX_LEN; + + /* Check if we can really insert (m_lblk)::(m_lblk + m_len) extent */ + newex.ee_len = cpu_to_le16(map->m_len); + err = ext4_ext_check_overlap(sbi, inode, &newex, path); + if (err) + allocated = ext4_ext_get_actual_len(&newex); + else + allocated = map->m_len; + + /* allocate new block */ + ar.inode = inode; + ar.goal = ext4_ext_find_goal(inode, path, map->m_lblk); + ar.logical = map->m_lblk; + /* + * We calculate the offset from the beginning of the cluster + * for the logical block number, since when we allocate a + * physical cluster, the physical block should start at the + * same offset from the beginning of the cluster. This is + * needed so that future calls to get_implied_cluster_alloc() + * work correctly. + */ + offset = EXT4_LBLK_COFF(sbi, map->m_lblk); + ar.len = EXT4_NUM_B2C(sbi, offset+allocated); + ar.goal -= offset; + ar.logical -= offset; + if (S_ISREG(inode->i_mode)) + ar.flags = EXT4_MB_HINT_DATA; + else + /* disable in-core preallocation for non-regular files */ + ar.flags = 0; + if (flags & EXT4_GET_BLOCKS_NO_NORMALIZE) + ar.flags |= EXT4_MB_HINT_NOPREALLOC; + if (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) + ar.flags |= EXT4_MB_DELALLOC_RESERVED; + if (flags & EXT4_GET_BLOCKS_METADATA_NOFAIL) + ar.flags |= EXT4_MB_USE_RESERVED; + newblock = ext4_mb_new_blocks(handle, &ar, &err); + if (!newblock) + goto out; + allocated_clusters = ar.len; + ar.len = EXT4_C2B(sbi, ar.len) - offset; + ext_debug(inode, "allocate new block: goal %llu, found %llu/%u, requested %u\n", + ar.goal, newblock, ar.len, allocated); + if (ar.len > allocated) + ar.len = allocated; + +got_allocated_blocks: + /* try to insert new extent into found leaf and return */ + pblk = newblock + offset; + ext4_ext_store_pblock(&newex, pblk); + newex.ee_len = cpu_to_le16(ar.len); + /* Mark unwritten */ + if (flags & EXT4_GET_BLOCKS_UNWRIT_EXT) { + ext4_ext_mark_unwritten(&newex); + map->m_flags |= EXT4_MAP_UNWRITTEN; + } + + path = ext4_ext_insert_extent(handle, inode, path, &newex, flags); + if (IS_ERR(path)) { + err = PTR_ERR(path); + /* + * Gracefully handle out of space conditions. If the filesystem + * is inconsistent, we'll just leak allocated blocks to avoid + * causing even more damage. + */ + if (allocated_clusters && (err == -EDQUOT || err == -ENOSPC)) { + int fb_flags = 0; + /* + * free data blocks we just allocated. + * not a good idea to call discard here directly, + * but otherwise we'd need to call it every free(). + */ + ext4_discard_preallocations(inode); + if (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) + fb_flags = EXT4_FREE_BLOCKS_NO_QUOT_UPDATE; + ext4_free_blocks(handle, inode, NULL, newblock, + EXT4_C2B(sbi, allocated_clusters), + fb_flags); + } + goto out; + } + + /* + * Cache the extent and update transaction to commit on fdatasync only + * when it is _not_ an unwritten extent. + */ + if ((flags & EXT4_GET_BLOCKS_UNWRIT_EXT) == 0) + ext4_update_inode_fsync_trans(handle, inode, 1); + else + ext4_update_inode_fsync_trans(handle, inode, 0); + + map->m_flags |= (EXT4_MAP_NEW | EXT4_MAP_MAPPED); + map->m_pblk = pblk; + map->m_len = ar.len; + allocated = map->m_len; + ext4_ext_show_leaf(inode, path); +out: + /* + * We never use EXT4_GET_BLOCKS_QUERY_LAST_IN_LEAF with CREATE flag. + * So we know that the depth used here is correct, since there was no + * block allocation done if EXT4_GET_BLOCKS_QUERY_LAST_IN_LEAF is set. + * If tomorrow we start using this QUERY flag with CREATE, then we will + * need to re-calculate the depth as it might have changed due to block + * allocation. + */ + if (flags & EXT4_GET_BLOCKS_QUERY_LAST_IN_LEAF) { + WARN_ON_ONCE(flags & EXT4_GET_BLOCKS_CREATE); + if (!err && ex && (ex == EXT_LAST_EXTENT(path[depth].p_hdr))) + map->m_flags |= EXT4_MAP_QUERY_LAST_IN_LEAF; + } + + ext4_free_ext_path(path); + + trace_ext4_ext_map_blocks_exit(inode, flags, map, + err ? err : allocated); + return err ? err : allocated; +} + +int ext4_ext_truncate(handle_t *handle, struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + ext4_lblk_t last_block; + int err = 0; + + /* + * TODO: optimization is possible here. + * Probably we need not scan at all, + * because page truncation is enough. + */ + + /* we have to know where to truncate from in crash case */ + EXT4_I(inode)->i_disksize = inode->i_size; + err = ext4_mark_inode_dirty(handle, inode); + if (err) + return err; + + last_block = (inode->i_size + sb->s_blocksize - 1) + >> EXT4_BLOCK_SIZE_BITS(sb); + ext4_es_remove_extent(inode, last_block, EXT_MAX_BLOCKS - last_block); + +retry_remove_space: + err = ext4_ext_remove_space(inode, last_block, EXT_MAX_BLOCKS - 1); + if (err == -ENOMEM) { + memalloc_retry_wait(GFP_ATOMIC); + goto retry_remove_space; + } + return err; +} + +/* + * Pre-allocate blocks for the range [@offset, @offset + @len). Allocated + * blocks are marked as unwritten by default. If EXT4_GET_BLOCKS_ZERO is + * set, the allocated blocks are zeroed on disk and their extents are + * converted to written state. + * + * When @new_size is nonzero, the caller intends to extend the file, and + * the file size should be updated to the end of the allocated blocks. + * + * Allocation may partially succeed due to some non-fatal issues. In that + * case, i_disksize (and i_size) is advanced up to the successfully + * processed portion of the range. + * + * Return 0 on success, or a negative error code on failure or partial + * failure. + */ +static int ext4_alloc_file_blocks(struct file *file, loff_t offset, loff_t len, + loff_t new_size, int flags) +{ + struct inode *inode = file_inode(file); + handle_t *handle; + int ret = 0, ret2 = 0, ret3 = 0; + int retries = 0; + int depth = 0; + ext4_lblk_t len_lblk; + struct ext4_map_blocks map; + unsigned int credits; + loff_t epos = 0, old_size = i_size_read(inode); + unsigned int blkbits = inode->i_blkbits; + bool alloc_zero = false; + bool orphan = false; + + BUG_ON(!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)); + map.m_lblk = offset >> blkbits; + map.m_len = len_lblk = EXT4_MAX_BLOCKS(len, offset, blkbits); + /* + * Don't normalize the request if it can fit in one extent so + * that it doesn't get unnecessarily split into multiple + * extents. + */ + if (len_lblk <= EXT_UNWRITTEN_MAX_LEN) + flags |= EXT4_GET_BLOCKS_NO_NORMALIZE; + + /* + * Do the actual write zero during a running journal transaction + * costs a lot. First allocate an unwritten extent and then + * convert it to written after zeroing it out. + */ + if (flags & EXT4_GET_BLOCKS_ZERO) { + flags &= ~EXT4_GET_BLOCKS_ZERO; + flags |= EXT4_GET_BLOCKS_UNWRIT_EXT; + alloc_zero = true; + } + + /* + * credits to insert 1 extent into extent tree + */ + credits = ext4_chunk_trans_blocks(inode, len_lblk); + depth = ext_depth(inode); + + /* Zero to the end of the block containing i_size */ + if (new_size > old_size) { + ret = ext4_block_zero_eof(inode, old_size, LLONG_MAX); + if (ret) + return ret; + } + +retry: + while (len_lblk) { + /* + * Recalculate credits when extent tree depth changes. + */ + if (depth != ext_depth(inode)) { + credits = ext4_chunk_trans_blocks(inode, len_lblk); + depth = ext_depth(inode); + } + + handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, + credits); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + break; + } + ret = ext4_map_blocks(handle, inode, &map, flags); + if (ret <= 0) { + ext4_debug("inode #%llu: block %u: len %u: " + "ext4_ext_map_blocks returned %d", + inode->i_ino, map.m_lblk, + map.m_len, ret); + ext4_mark_inode_dirty(handle, inode); + ext4_journal_stop(handle); + break; + } + ext4_update_inode_fsync_trans(handle, inode, 1); + ret = ext4_journal_stop(handle); + if (unlikely(ret)) + break; + + /* + * allow a full retry cycle for any remaining allocations + */ + retries = 0; + + if (alloc_zero && + (map.m_flags & (EXT4_MAP_MAPPED | EXT4_MAP_UNWRITTEN))) { + ext4_lblk_t converted; + + WARN_ON_ONCE(map.m_lblk + map.m_len > + EXT4_B_TO_LBLK(inode, new_size ?: old_size)); + + ret = ext4_issue_zeroout(inode, map.m_lblk, map.m_pblk, + map.m_len); + if (unlikely(ret)) + break; + + handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, + credits); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + break; + } + + ret = ext4_convert_unwritten_extents(handle, + inode, (loff_t)map.m_lblk << blkbits, + (loff_t)map.m_len << blkbits, + &converted); + if (ret) + map.m_len = converted; + + /* + * If blocks beyond i_disksize are converted, add + * the inode to the orphan list and advance the epos. + */ + if (new_size && converted) { + ret2 = ext4_orphan_add(handle, inode); + ret = ret ? ret : ret2; + orphan = true; + } + + ret3 = ext4_journal_stop(handle); + ret = ret ? ret : ret3; + } + + map.m_lblk += map.m_len; + map.m_len = len_lblk = len_lblk - map.m_len; + epos = EXT4_LBLK_TO_B(inode, map.m_lblk); + if (ret) + break; + } + + if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)) + goto retry; + + if (!epos || !new_size) + return ret; + + /* + * Allocate blocks, update the file size to match the size of the + * already successfully allocated blocks. + */ + if (epos > new_size) + epos = new_size; + + handle = ext4_journal_start(inode, EXT4_HT_MISC, 2); + if (IS_ERR(handle)) { + /* + * The conversion has successfully completed. Not much to + * do with the error here so just cleanup the orphan list + * and hope for the best. + */ + if (orphan && inode->i_nlink) + ext4_orphan_del(NULL, inode); + ret2 = PTR_ERR(handle); + goto out; + } + + ext4_update_inode_size(inode, epos); + if (orphan && inode->i_nlink) + ext4_orphan_del(handle, inode); + + ret2 = ext4_mark_inode_dirty(handle, inode); + ext4_update_inode_fsync_trans(handle, inode, 1); + ret3 = ext4_journal_stop(handle); + ret2 = ret3 ? ret3 : ret2; + + if (epos > old_size) + pagecache_isize_extended(inode, old_size, epos); +out: + if (ret2) + ext4_std_error(inode->i_sb, ret2); + + return ret ? ret : ret2; +} + +static int ext4_collapse_range(struct file *file, loff_t offset, loff_t len); + +static int ext4_insert_range(struct file *file, loff_t offset, loff_t len); + +static long ext4_zero_range(struct file *file, loff_t offset, + loff_t len, int mode) +{ + struct inode *inode = file_inode(file); + handle_t *handle = NULL; + loff_t align_start, align_end, new_size = 0; + loff_t end = offset + len; + unsigned int blocksize = i_blocksize(inode); + unsigned int partial_zeroed = 0; + int ret, flags; + + trace_ext4_zero_range(inode, offset, len, mode); + WARN_ON_ONCE(!inode_is_locked(inode)); + + /* Indirect files do not support unwritten extents */ + if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) + return -EOPNOTSUPP; + + if (!(mode & FALLOC_FL_KEEP_SIZE) && + (end > inode->i_size || end > EXT4_I(inode)->i_disksize)) { + new_size = end; + ret = inode_newsize_ok(inode, new_size); + if (ret) + return ret; + } + + flags = EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT; + /* + * Preallocate the range including the unaligned edges, and zero + * out partial blocks if they already contain data. + */ + if (!IS_ALIGNED(offset | end, blocksize)) { + ret = ext4_alloc_file_blocks(file, offset, len, new_size, + flags); + if (!ret) + ret = ext4_zero_partial_blocks(inode, offset, len, + &partial_zeroed); + if (ret) + return ret; + } + + ret = ext4_update_disksize_before_punch(inode, offset, len); + if (ret) + return ret; + + /* Now release the pages and zero block aligned part of pages */ + ret = ext4_truncate_page_cache_block_range(inode, offset, end); + if (ret) + return ret; + + /* Zero range excluding the unaligned edges */ + align_start = round_up(offset, blocksize); + align_end = round_down(end, blocksize); + + /* + * In WRITE_ZEROES mode, edges that were not partial-zeroed (clean + * unwritten or hole) must be allocated and zeroed as whole blocks. + * Expand the aligned range outward to cover them. + */ + if (mode & FALLOC_FL_WRITE_ZEROES) { + if (!IS_ALIGNED(offset, blocksize) && + !(partial_zeroed & EXT4_PARTIAL_ZERO_START)) + align_start = round_down(offset, blocksize); + if (!IS_ALIGNED(end, blocksize) && + !(partial_zeroed & EXT4_PARTIAL_ZERO_END)) + align_end = round_up(end, blocksize); + } + + if (align_end > align_start) { + if (mode & FALLOC_FL_WRITE_ZEROES) + flags = EXT4_GET_BLOCKS_CREATE_ZERO | EXT4_EX_NOCACHE; + else + flags |= (EXT4_GET_BLOCKS_CONVERT_UNWRITTEN | + EXT4_EX_NOCACHE); + ret = ext4_alloc_file_blocks(file, align_start, + align_end - align_start, new_size, + flags); + if (ret) + return ret; + } + /* Finish zeroing out if it doesn't contain partial block */ + if (IS_ALIGNED(offset | end, blocksize)) + return ret; + + /* + * In FALLOC_FL_WRITE_ZEROES mode, edges that have been partially + * zeroed must be written back to ensure the entire zeroed range + * is converted to the written state. In SYNC mode, writeback is + * also required to persist the zeroed data to disk. + */ + if (partial_zeroed && + ((mode & FALLOC_FL_WRITE_ZEROES) || + (file->f_flags & O_SYNC) || IS_SYNC(inode))) { + ret = filemap_write_and_wait_range(inode->i_mapping, offset, + end - 1); + if (ret) + return ret; + } + + handle = ext4_journal_start(inode, EXT4_HT_MISC, 1); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + ext4_std_error(inode->i_sb, ret); + return ret; + } + + if (new_size) + ext4_update_inode_size(inode, new_size); + ret = ext4_mark_inode_dirty(handle, inode); + if (unlikely(ret)) + goto out_handle; + + ext4_update_inode_fsync_trans(handle, inode, 1); + if ((file->f_flags & O_SYNC) || IS_SYNC(inode)) + ext4_handle_sync(handle); + +out_handle: + ext4_journal_stop(handle); + return ret; +} + +static long ext4_do_fallocate(struct file *file, loff_t offset, + loff_t len, int mode) +{ + struct inode *inode = file_inode(file); + loff_t end = offset + len; + loff_t new_size = 0; + int ret; + + trace_ext4_fallocate_enter(inode, offset, len, mode); + WARN_ON_ONCE(!inode_is_locked(inode)); + + /* We only support preallocation for extent-based files only. */ + if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) { + ret = -EOPNOTSUPP; + goto out; + } + + if (!(mode & FALLOC_FL_KEEP_SIZE) && + (end > inode->i_size || end > EXT4_I(inode)->i_disksize)) { + new_size = end; + ret = inode_newsize_ok(inode, new_size); + if (ret) + goto out; + } + + ret = ext4_alloc_file_blocks(file, offset, len, new_size, + EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT); + if (ret) + goto out; + + if (((file->f_flags & O_SYNC) || IS_SYNC(inode)) && + EXT4_SB(inode->i_sb)->s_journal) { + ret = ext4_fc_commit(EXT4_SB(inode->i_sb)->s_journal, + EXT4_I(inode)->i_sync_tid); + } +out: + trace_ext4_fallocate_exit(inode, offset, + EXT4_MAX_BLOCKS(len, offset, inode->i_blkbits), ret); + return ret; +} + +/* + * preallocate space for a file. This implements ext4's fallocate file + * operation, which gets called from sys_fallocate system call. + * For block-mapped files, posix_fallocate should fall back to the method + * of writing zeroes to the required new blocks (the same behavior which is + * expected for file systems which do not support fallocate() system call). + */ +long ext4_fallocate(struct file *file, int mode, loff_t offset, loff_t len) +{ + struct inode *inode = file_inode(file); + struct address_space *mapping = file->f_mapping; + int ret; + + /* + * Encrypted inodes can't handle collapse range or insert + * range since we would need to re-encrypt blocks with a + * different IV or XTS tweak (which are based on the logical + * block number). + */ + if (IS_ENCRYPTED(inode) && + (mode & (FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_INSERT_RANGE))) + return -EOPNOTSUPP; + /* + * Don't allow writing zeroes if the underlying device does not + * enable the unmap write zeroes operation. + */ + if ((mode & FALLOC_FL_WRITE_ZEROES) && + !bdev_write_zeroes_unmap_sectors(inode->i_sb->s_bdev)) + return -EOPNOTSUPP; + + /* Return error if mode is not supported */ + if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE | + FALLOC_FL_ZERO_RANGE | FALLOC_FL_COLLAPSE_RANGE | + FALLOC_FL_INSERT_RANGE | FALLOC_FL_WRITE_ZEROES)) + return -EOPNOTSUPP; + + inode_lock(inode); + ret = ext4_convert_inline_data(inode); + if (ret) + goto out_inode_lock; + + /* Wait all existing dio workers, newcomers will block on i_rwsem */ + inode_dio_wait(inode); + + ret = file_modified(file); + if (ret) + goto out_inode_lock; + + if ((mode & FALLOC_FL_MODE_MASK) == FALLOC_FL_ALLOCATE_RANGE) { + ret = ext4_do_fallocate(file, offset, len, mode); + goto out_inode_lock; + } + + /* + * Follow-up operations will drop page cache, hold invalidate lock + * to prevent page faults from reinstantiating pages we have + * released from page cache. + */ + filemap_invalidate_lock(mapping); + + ret = ext4_break_layouts(inode); + if (ret) + goto out_invalidate_lock; + + switch (mode & FALLOC_FL_MODE_MASK) { + case FALLOC_FL_PUNCH_HOLE: + ret = ext4_punch_hole(file, offset, len); + break; + case FALLOC_FL_COLLAPSE_RANGE: + ret = ext4_collapse_range(file, offset, len); + break; + case FALLOC_FL_INSERT_RANGE: + ret = ext4_insert_range(file, offset, len); + break; + case FALLOC_FL_ZERO_RANGE: + case FALLOC_FL_WRITE_ZEROES: + ret = ext4_zero_range(file, offset, len, mode); + break; + default: + ret = -EOPNOTSUPP; + } + +out_invalidate_lock: + filemap_invalidate_unlock(mapping); +out_inode_lock: + inode_unlock(inode); + return ret; +} + +/* + * This function converts a range of blocks to written extents. The caller of + * this function will pass the start offset and the size. all unwritten extents + * within this range will be converted to written extents. + * + * This function is called from the direct IO end io call back function for + * atomic writes, to convert the unwritten extents after IO is completed. + * + * Note that the requirement for atomic writes is that all conversion should + * happen atomically in a single fs journal transaction. We mainly only allocate + * unwritten extents either on a hole on a pre-exiting unwritten extent range in + * ext4_map_blocks_atomic_write(). The only case where we can have multiple + * unwritten extents in a range [offset, offset+len) is when there is a split + * unwritten extent between two leaf nodes which was cached in extent status + * cache during ext4_iomap_alloc() time. That will allow + * ext4_map_blocks_atomic_write() to return the unwritten extent range w/o going + * into the slow path. That means we might need a loop for conversion of this + * unwritten extent split across leaf block within a single journal transaction. + * Split extents across leaf nodes is a rare case, but let's still handle that + * to meet the requirements of multi-fsblock atomic writes. + * + * Returns 0 on success. + */ +int ext4_convert_unwritten_extents_atomic(handle_t *handle, struct inode *inode, + loff_t offset, ssize_t len) +{ + unsigned int max_blocks; + int ret = 0, ret2 = 0, ret3 = 0; + struct ext4_map_blocks map; + unsigned int blkbits = inode->i_blkbits; + unsigned int credits = 0; + int flags = EXT4_GET_BLOCKS_IO_CONVERT_EXT | EXT4_EX_NOCACHE; + + map.m_lblk = offset >> blkbits; + max_blocks = EXT4_MAX_BLOCKS(len, offset, blkbits); + + if (!handle) { + /* + * TODO: An optimization can be added later by having an extent + * status flag e.g. EXTENT_STATUS_SPLIT_LEAF. If we query that + * it can tell if the extent in the cache is a split extent. + * But for now let's assume pextents as 2 always. + */ + credits = ext4_meta_trans_blocks(inode, max_blocks, 2, 0); + } + + if (credits) { + handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, credits); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + return ret; + } + } + + while (ret >= 0 && ret < max_blocks) { + map.m_lblk += ret; + map.m_len = (max_blocks -= ret); + ret = ext4_map_blocks(handle, inode, &map, flags); + if (ret != max_blocks) + ext4_msg(inode->i_sb, KERN_INFO, + "inode #%llu: block %u: len %u: " + "split block mapping found for atomic write, " + "ret = %d", + inode->i_ino, map.m_lblk, + map.m_len, ret); + if (ret <= 0) + break; + } + + ret2 = ext4_mark_inode_dirty(handle, inode); + + if (credits) { + ret3 = ext4_journal_stop(handle); + if (unlikely(ret3)) + ret2 = ret3; + } + + if (ret <= 0 || ret2) + ext4_warning(inode->i_sb, + "inode #%llu: block %u: len %u: " + "returned %d or %d", + inode->i_ino, map.m_lblk, + map.m_len, ret, ret2); + + return ret > 0 ? ret2 : ret; +} + +/* + * This function convert a range of blocks to written extents + * The caller of this function will pass the start offset and the size. + * all unwritten extents within this range will be converted to + * written extents. + * + * This function is called from the direct/buffered I/O end io call back + * function and FALLOC_FL_WRITE_ZEROES, to convert the fallocated + * unwritten extents after data I/O is completed. + * + * Returns 0 on full success, or a negative error code on partial + * success or failure. The number of blocks converted is returned via + * @converted. + */ +int ext4_convert_unwritten_extents(handle_t *handle, struct inode *inode, + loff_t offset, ssize_t len, + ext4_lblk_t *converted) +{ + ext4_lblk_t max_blocks, conv_blocks = 0; + int ret = 0, ret2 = 0, ret3 = 0; + struct ext4_map_blocks map; + unsigned int blkbits = inode->i_blkbits; + unsigned int credits = 0; + + map.m_lblk = offset >> blkbits; + map.m_len = max_blocks = EXT4_MAX_BLOCKS(len, offset, blkbits); + + if (!handle) { + /* + * credits to insert 1 extent into extent tree + */ + credits = ext4_chunk_trans_blocks(inode, max_blocks); + } + + while (max_blocks) { + if (credits) { + handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, + credits); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + break; + } + } + /* + * Do not cache any unrelated extents, as it does not hold the + * i_rwsem or invalidate_lock, which could corrupt the extent + * status tree. + */ + ret = ext4_map_blocks(handle, inode, &map, + EXT4_GET_BLOCKS_IO_CONVERT_EXT | + EXT4_EX_NOCACHE); + if (ret <= 0) { + ext4_warning(inode->i_sb, + "inode #%llu: block %u: len %u: ext4_map_blocks returned %d", + inode->i_ino, map.m_lblk, map.m_len, ret); + if (unlikely(ret == 0)) + ret = -EINVAL; + } else { + conv_blocks += map.m_len; + } + + ret2 = ext4_mark_inode_dirty(handle, inode); + if (credits) { + ret3 = ext4_journal_stop(handle); + if (unlikely(ret3)) + ret2 = ret3; + } + ret = ret < 0 ? ret : ret2; + if (ret) + break; + + map.m_lblk += map.m_len; + map.m_len = (max_blocks -= map.m_len); + } + /* Converted some or all blocks successfully? */ + if (converted) + *converted = conv_blocks; + + return ret; +} + +int ext4_convert_unwritten_io_end_vec(handle_t *handle, ext4_io_end_t *io_end) +{ + int ret = 0, err = 0; + struct ext4_io_end_vec *io_end_vec; + + /* + * This is somewhat ugly but the idea is clear: When transaction is + * reserved, everything goes into it. Otherwise we rather start several + * smaller transactions for conversion of each extent separately. + */ + if (handle) { + handle = ext4_journal_start_reserved(handle, + EXT4_HT_EXT_CONVERT); + if (IS_ERR(handle)) + return PTR_ERR(handle); + } + + list_for_each_entry(io_end_vec, &io_end->list_vec, list) { + ret = ext4_convert_unwritten_extents(handle, io_end->inode, + io_end_vec->offset, + io_end_vec->size, NULL); + if (ret) + break; + } + + if (handle) + err = ext4_journal_stop(handle); + + return ret < 0 ? ret : err; +} + +static int ext4_iomap_xattr_fiemap(struct inode *inode, struct iomap *iomap) +{ + __u64 physical = 0; + __u64 length = 0; + int blockbits = inode->i_sb->s_blocksize_bits; + int error = 0; + u16 iomap_type; + + /* in-inode? */ + if (ext4_test_inode_state(inode, EXT4_STATE_XATTR)) { + struct ext4_iloc iloc; + int offset; /* offset of xattr in inode */ + + error = ext4_get_inode_loc(inode, &iloc); + if (error) + return error; + physical = (__u64)iloc.bh->b_blocknr << blockbits; + offset = EXT4_GOOD_OLD_INODE_SIZE + + EXT4_I(inode)->i_extra_isize; + physical += offset; + length = EXT4_SB(inode->i_sb)->s_inode_size - offset; + brelse(iloc.bh); + iomap_type = IOMAP_INLINE; + } else if (EXT4_I(inode)->i_file_acl) { /* external block */ + physical = (__u64)EXT4_I(inode)->i_file_acl << blockbits; + length = inode->i_sb->s_blocksize; + iomap_type = IOMAP_MAPPED; + } else { + /* no in-inode or external block for xattr, so return -ENOENT */ + error = -ENOENT; + goto out; + } + + iomap->addr = physical; + iomap->offset = 0; + iomap->length = length; + iomap->type = iomap_type; + iomap->flags = 0; +out: + return error; +} + +static int ext4_iomap_xattr_begin(struct inode *inode, loff_t offset, + loff_t length, unsigned flags, + struct iomap *iomap, struct iomap *srcmap) +{ + int error; + + error = ext4_iomap_xattr_fiemap(inode, iomap); + if (error == 0 && (offset >= iomap->length)) + error = -ENOENT; + return error; +} + +static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_xattr_next, ext4_iomap_xattr_begin); + +static const struct iomap_ops ext4_iomap_xattr_ops = { + .iomap_next = ext4_iomap_xattr_next, +}; + +static int ext4_fiemap_check_ranges(struct inode *inode, u64 start, u64 *len) +{ + u64 maxbytes = ext4_get_maxbytes(inode); + + if (*len == 0) + return -EINVAL; + if (start > maxbytes) + return -EFBIG; + + /* + * Shrink request scope to what the fs can actually handle. + */ + if (*len > maxbytes || (maxbytes - *len) < start) + *len = maxbytes - start; + return 0; +} + +int ext4_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + u64 start, u64 len) +{ + int error = 0; + + inode_lock_shared(inode); + if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) { + error = ext4_ext_precache(inode); + if (error) + goto unlock; + fieinfo->fi_flags &= ~FIEMAP_FLAG_CACHE; + } + + /* + * For bitmap files the maximum size limit could be smaller than + * s_maxbytes, so check len here manually instead of just relying on the + * generic check. + */ + error = ext4_fiemap_check_ranges(inode, start, &len); + if (error) + goto unlock; + + if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) { + fieinfo->fi_flags &= ~FIEMAP_FLAG_XATTR; + error = iomap_fiemap(inode, fieinfo, start, len, + &ext4_iomap_xattr_ops); + } else { + error = iomap_fiemap(inode, fieinfo, start, len, + &ext4_iomap_report_ops); + } +unlock: + inode_unlock_shared(inode); + return error; +} + +int ext4_get_es_cache(struct inode *inode, struct fiemap_extent_info *fieinfo, + __u64 start, __u64 len) +{ + ext4_lblk_t start_blk, len_blks; + __u64 last_blk; + int error = 0; + + if (ext4_has_inline_data(inode)) { + int has_inline; + + down_read(&EXT4_I(inode)->xattr_sem); + has_inline = ext4_has_inline_data(inode); + up_read(&EXT4_I(inode)->xattr_sem); + if (has_inline) + return 0; + } + + if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) { + inode_lock_shared(inode); + error = ext4_ext_precache(inode); + inode_unlock_shared(inode); + if (error) + return error; + fieinfo->fi_flags &= ~FIEMAP_FLAG_CACHE; + } + + error = fiemap_prep(inode, fieinfo, start, &len, 0); + if (error) + return error; + + error = ext4_fiemap_check_ranges(inode, start, &len); + if (error) + return error; + + start_blk = start >> inode->i_sb->s_blocksize_bits; + last_blk = (start + len - 1) >> inode->i_sb->s_blocksize_bits; + if (last_blk >= EXT_MAX_BLOCKS) + last_blk = EXT_MAX_BLOCKS-1; + len_blks = ((ext4_lblk_t) last_blk) - start_blk + 1; + + /* + * Walk the extent tree gathering extent information + * and pushing extents back to the user. + */ + return ext4_fill_es_cache_info(inode, start_blk, len_blks, fieinfo); +} + +/* + * ext4_ext_shift_path_extents: + * Shift the extents of a path structure lying between path[depth].p_ext + * and EXT_LAST_EXTENT(path[depth].p_hdr), by @shift blocks. @SHIFT tells + * if it is right shift or left shift operation. + */ +static int +ext4_ext_shift_path_extents(struct ext4_ext_path *path, ext4_lblk_t shift, + struct inode *inode, handle_t *handle, + enum SHIFT_DIRECTION SHIFT) +{ + int depth, err = 0; + struct ext4_extent *ex_start, *ex_last; + bool update = false; + int credits, restart_credits; + depth = path->p_depth; + + while (depth >= 0) { + if (depth == path->p_depth) { + ex_start = path[depth].p_ext; + if (!ex_start) + return -EFSCORRUPTED; + + ex_last = EXT_LAST_EXTENT(path[depth].p_hdr); + /* leaf + sb + inode */ + credits = 3; + if (ex_start == EXT_FIRST_EXTENT(path[depth].p_hdr)) { + update = true; + /* extent tree + sb + inode */ + credits = depth + 2; + } + + restart_credits = ext4_chunk_trans_extent(inode, 0); + err = ext4_datasem_ensure_credits(handle, inode, credits, + restart_credits, 0); + if (err) { + if (err > 0) + err = -EAGAIN; + goto out; + } + + err = ext4_ext_get_access(handle, inode, path + depth); + if (err) + goto out; + + while (ex_start <= ex_last) { + if (SHIFT == SHIFT_LEFT) { + le32_add_cpu(&ex_start->ee_block, + -shift); + /* Try to merge to the left. */ + if ((ex_start > + EXT_FIRST_EXTENT(path[depth].p_hdr)) + && + ext4_ext_try_to_merge_right(inode, + path, ex_start - 1)) + ex_last--; + else + ex_start++; + } else { + le32_add_cpu(&ex_last->ee_block, shift); + ext4_ext_try_to_merge_right(inode, path, + ex_last); + ex_last--; + } + } + err = ext4_ext_dirty(handle, inode, path + depth); + if (err) + goto out; + + if (--depth < 0 || !update) + break; + } + + /* Update index too */ + err = ext4_ext_get_access(handle, inode, path + depth); + if (err) + goto out; + + if (SHIFT == SHIFT_LEFT) + le32_add_cpu(&path[depth].p_idx->ei_block, -shift); + else + le32_add_cpu(&path[depth].p_idx->ei_block, shift); + err = ext4_ext_dirty(handle, inode, path + depth); + if (err) + goto out; + + /* we are done if current index is not a starting index */ + if (path[depth].p_idx != EXT_FIRST_INDEX(path[depth].p_hdr)) + break; + + depth--; + } + +out: + return err; +} + +/* + * ext4_ext_shift_extents: + * All the extents which lies in the range from @start to the last allocated + * block for the @inode are shifted either towards left or right (depending + * upon @SHIFT) by @shift blocks. + * On success, 0 is returned, error otherwise. + */ +static int +ext4_ext_shift_extents(struct inode *inode, handle_t *handle, + ext4_lblk_t start, ext4_lblk_t shift, + enum SHIFT_DIRECTION SHIFT) +{ + struct ext4_ext_path *path; + int ret = 0, depth; + struct ext4_extent *extent; + ext4_lblk_t stop, *iterator, ex_start, ex_end; + ext4_lblk_t tmp = EXT_MAX_BLOCKS; + + /* Let path point to the last extent */ + path = ext4_find_extent(inode, EXT_MAX_BLOCKS - 1, NULL, + EXT4_EX_NOCACHE); + if (IS_ERR(path)) + return PTR_ERR(path); + + depth = path->p_depth; + extent = path[depth].p_ext; + if (!extent) + goto out; + + stop = le32_to_cpu(extent->ee_block); + + /* + * For left shifts, make sure the hole on the left is big enough to + * accommodate the shift. For right shifts, make sure the last extent + * won't be shifted beyond EXT_MAX_BLOCKS. + */ + if (SHIFT == SHIFT_LEFT) { + path = ext4_find_extent(inode, start - 1, path, + EXT4_EX_NOCACHE); + if (IS_ERR(path)) + return PTR_ERR(path); + depth = path->p_depth; + extent = path[depth].p_ext; + if (extent) { + ex_start = le32_to_cpu(extent->ee_block); + ex_end = le32_to_cpu(extent->ee_block) + + ext4_ext_get_actual_len(extent); + } else { + ex_start = 0; + ex_end = 0; + } + + if ((start == ex_start && shift > ex_start) || + (shift > start - ex_end)) { + ret = -EINVAL; + goto out; + } + } else { + if (shift > EXT_MAX_BLOCKS - + (stop + ext4_ext_get_actual_len(extent))) { + ret = -EINVAL; + goto out; + } + } + + /* + * In case of left shift, iterator points to start and it is increased + * till we reach stop. In case of right shift, iterator points to stop + * and it is decreased till we reach start. + */ +again: + ret = 0; + if (SHIFT == SHIFT_LEFT) + iterator = &start; + else + iterator = &stop; + + if (tmp != EXT_MAX_BLOCKS) + *iterator = tmp; + + /* + * Its safe to start updating extents. Start and stop are unsigned, so + * in case of right shift if extent with 0 block is reached, iterator + * becomes NULL to indicate the end of the loop. + */ + while (iterator && start <= stop) { + path = ext4_find_extent(inode, *iterator, path, + EXT4_EX_NOCACHE); + if (IS_ERR(path)) + return PTR_ERR(path); + depth = path->p_depth; + extent = path[depth].p_ext; + if (!extent) { + EXT4_ERROR_INODE(inode, "unexpected hole at %lu", + (unsigned long) *iterator); + ret = -EFSCORRUPTED; + goto out; + } + if (SHIFT == SHIFT_LEFT && *iterator > + le32_to_cpu(extent->ee_block)) { + /* Hole, move to the next extent */ + if (extent < EXT_LAST_EXTENT(path[depth].p_hdr)) { + path[depth].p_ext++; + } else { + *iterator = ext4_ext_next_allocated_block(path); + continue; + } + } + + tmp = *iterator; + if (SHIFT == SHIFT_LEFT) { + extent = EXT_LAST_EXTENT(path[depth].p_hdr); + *iterator = le32_to_cpu(extent->ee_block) + + ext4_ext_get_actual_len(extent); + } else { + extent = EXT_FIRST_EXTENT(path[depth].p_hdr); + if (le32_to_cpu(extent->ee_block) > start) + *iterator = le32_to_cpu(extent->ee_block) - 1; + else if (le32_to_cpu(extent->ee_block) == start) + iterator = NULL; + else { + extent = EXT_LAST_EXTENT(path[depth].p_hdr); + while (le32_to_cpu(extent->ee_block) >= start) + extent--; + + if (extent == EXT_LAST_EXTENT(path[depth].p_hdr)) + break; + + extent++; + iterator = NULL; + } + path[depth].p_ext = extent; + } + ret = ext4_ext_shift_path_extents(path, shift, inode, + handle, SHIFT); + /* iterator can be NULL which means we should break */ + if (ret == -EAGAIN) + goto again; + if (ret) + break; + } +out: + ext4_free_ext_path(path); + return ret; +} + +/* + * ext4_collapse_range: + * This implements the fallocate's collapse range functionality for ext4 + * Returns: 0 and non-zero on error. + */ +static int ext4_collapse_range(struct file *file, loff_t offset, loff_t len) +{ + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + struct address_space *mapping = inode->i_mapping; + loff_t end = offset + len; + ext4_lblk_t start_lblk, end_lblk; + handle_t *handle; + unsigned int credits; + loff_t start, new_size; + int ret; + + trace_ext4_collapse_range(inode, offset, len); + WARN_ON_ONCE(!inode_is_locked(inode)); + + /* Currently just for extent based files */ + if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) + return -EOPNOTSUPP; + /* Collapse range works only on fs cluster size aligned regions. */ + if (!IS_ALIGNED(offset | len, EXT4_CLUSTER_SIZE(sb))) + return -EINVAL; + /* + * There is no need to overlap collapse range with EOF, in which case + * it is effectively a truncate operation + */ + if (end >= inode->i_size) + return -EINVAL; + + /* + * Write tail of the last page before removed range and data that + * will be shifted since they will get removed from the page cache + * below. We are also protected from pages becoming dirty by + * i_rwsem and invalidate_lock. + * Need to round down offset to be aligned with page size boundary + * for page size > block size. + */ + start = round_down(offset, PAGE_SIZE); + ret = filemap_write_and_wait_range(mapping, start, offset); + if (!ret) + ret = filemap_write_and_wait_range(mapping, end, LLONG_MAX); + if (ret) + return ret; + + truncate_pagecache(inode, start); + + credits = ext4_chunk_trans_extent(inode, 0); + handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits); + if (IS_ERR(handle)) + return PTR_ERR(handle); + + ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_FALLOC_RANGE, handle); + + start_lblk = offset >> inode->i_blkbits; + end_lblk = (offset + len) >> inode->i_blkbits; + + ext4_check_map_extents_env(inode); + + down_write(&EXT4_I(inode)->i_data_sem); + ext4_discard_preallocations(inode); + ext4_es_remove_extent(inode, start_lblk, EXT_MAX_BLOCKS - start_lblk); + + ret = ext4_ext_remove_space(inode, start_lblk, end_lblk - 1); + if (ret) { + up_write(&EXT4_I(inode)->i_data_sem); + goto out_handle; + } + ext4_discard_preallocations(inode); + + ret = ext4_ext_shift_extents(inode, handle, end_lblk, + end_lblk - start_lblk, SHIFT_LEFT); + if (ret) { + up_write(&EXT4_I(inode)->i_data_sem); + goto out_handle; + } + + new_size = inode->i_size - len; + i_size_write(inode, new_size); + EXT4_I(inode)->i_disksize = new_size; + + up_write(&EXT4_I(inode)->i_data_sem); + ret = ext4_mark_inode_dirty(handle, inode); + if (ret) + goto out_handle; + + ext4_update_inode_fsync_trans(handle, inode, 1); + if ((file->f_flags & O_SYNC) || IS_SYNC(inode)) + ext4_handle_sync(handle); + +out_handle: + ext4_journal_stop(handle); + return ret; +} + +/* + * ext4_insert_range: + * This function implements the FALLOC_FL_INSERT_RANGE flag of fallocate. + * The data blocks starting from @offset to the EOF are shifted by @len + * towards right to create a hole in the @inode. Inode size is increased + * by len bytes. + * Returns 0 on success, error otherwise. + */ +static int ext4_insert_range(struct file *file, loff_t offset, loff_t len) +{ + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + struct address_space *mapping = inode->i_mapping; + handle_t *handle; + struct ext4_ext_path *path; + struct ext4_extent *extent; + ext4_lblk_t start_lblk, len_lblk, ee_start_lblk = 0; + unsigned int credits, ee_len; + int ret, depth; + loff_t start; + + trace_ext4_insert_range(inode, offset, len); + WARN_ON_ONCE(!inode_is_locked(inode)); + + /* Currently just for extent based files */ + if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) + return -EOPNOTSUPP; + /* Insert range works only on fs cluster size aligned regions. */ + if (!IS_ALIGNED(offset | len, EXT4_CLUSTER_SIZE(sb))) + return -EINVAL; + /* Offset must be less than i_size */ + if (offset >= inode->i_size) + return -EINVAL; + /* Check whether the maximum file size would be exceeded */ + if (len > inode->i_sb->s_maxbytes - inode->i_size) + return -EFBIG; + + /* + * Write out all dirty pages. Need to round down to align start offset + * to page size boundary for page size > block size. + */ + start = round_down(offset, PAGE_SIZE); + ret = filemap_write_and_wait_range(mapping, start, LLONG_MAX); + if (ret) + return ret; + + truncate_pagecache(inode, start); + + credits = ext4_chunk_trans_extent(inode, 0); + handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits); + if (IS_ERR(handle)) + return PTR_ERR(handle); + + ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_FALLOC_RANGE, handle); + + /* Expand file to avoid data loss if there is error while shifting */ + inode->i_size += len; + EXT4_I(inode)->i_disksize += len; + ret = ext4_mark_inode_dirty(handle, inode); + if (ret) + goto out_handle; + + start_lblk = offset >> inode->i_blkbits; + len_lblk = len >> inode->i_blkbits; + + ext4_check_map_extents_env(inode); + + down_write(&EXT4_I(inode)->i_data_sem); + ext4_discard_preallocations(inode); + + path = ext4_find_extent(inode, start_lblk, NULL, 0); + if (IS_ERR(path)) { + up_write(&EXT4_I(inode)->i_data_sem); + ret = PTR_ERR(path); + goto out_handle; + } + + depth = ext_depth(inode); + extent = path[depth].p_ext; + if (extent) { + ee_start_lblk = le32_to_cpu(extent->ee_block); + ee_len = ext4_ext_get_actual_len(extent); + + /* + * If start_lblk is not the starting block of extent, split + * the extent @start_lblk + */ + if ((start_lblk > ee_start_lblk) && + (start_lblk < (ee_start_lblk + ee_len))) { + path = ext4_split_extent_at(handle, inode, path, + start_lblk, EXT4_EX_NOCACHE | + EXT4_GET_BLOCKS_SPLIT_NOMERGE | + EXT4_GET_BLOCKS_METADATA_NOFAIL); + } + + if (IS_ERR(path)) { + up_write(&EXT4_I(inode)->i_data_sem); + ret = PTR_ERR(path); + goto out_handle; + } + } + + ext4_free_ext_path(path); + ext4_es_remove_extent(inode, start_lblk, EXT_MAX_BLOCKS - start_lblk); + + /* + * if start_lblk lies in a hole which is at start of file, use + * ee_start_lblk to shift extents + */ + ret = ext4_ext_shift_extents(inode, handle, + max(ee_start_lblk, start_lblk), len_lblk, SHIFT_RIGHT); + up_write(&EXT4_I(inode)->i_data_sem); + if (ret) + goto out_handle; + + ext4_update_inode_fsync_trans(handle, inode, 1); + if ((file->f_flags & O_SYNC) || IS_SYNC(inode)) + ext4_handle_sync(handle); + +out_handle: + ext4_journal_stop(handle); + return ret; +} + +/** + * ext4_swap_extents() - Swap extents between two inodes + * @handle: handle for this transaction + * @inode1: First inode + * @inode2: Second inode + * @lblk1: Start block for first inode + * @lblk2: Start block for second inode + * @count: Number of blocks to swap + * @unwritten: Mark second inode's extents as unwritten after swap + * @erp: Pointer to save error value + * + * This helper routine does exactly what is promise "swap extents". All other + * stuff such as page-cache locking consistency, bh mapping consistency or + * extent's data copying must be performed by caller. + * Locking: + * i_rwsem is held for both inodes + * i_data_sem is locked for write for both inodes + * Assumptions: + * All pages from requested range are locked for both inodes + */ +int +ext4_swap_extents(handle_t *handle, struct inode *inode1, + struct inode *inode2, ext4_lblk_t lblk1, ext4_lblk_t lblk2, + ext4_lblk_t count, int unwritten, int *erp) +{ + struct ext4_ext_path *path1 = NULL; + struct ext4_ext_path *path2 = NULL; + int replaced_count = 0; + + BUG_ON(!rwsem_is_locked(&EXT4_I(inode1)->i_data_sem)); + BUG_ON(!rwsem_is_locked(&EXT4_I(inode2)->i_data_sem)); + BUG_ON(!inode_is_locked(inode1)); + BUG_ON(!inode_is_locked(inode2)); + + ext4_es_remove_extent(inode1, lblk1, count); + ext4_es_remove_extent(inode2, lblk2, count); + + while (count) { + struct ext4_extent *ex1, *ex2, tmp_ex; + ext4_lblk_t e1_blk, e2_blk; + int e1_len, e2_len, len; + int split = 0; + + path1 = ext4_find_extent(inode1, lblk1, path1, EXT4_EX_NOCACHE); + if (IS_ERR(path1)) { + *erp = PTR_ERR(path1); + goto errout; + } + path2 = ext4_find_extent(inode2, lblk2, path2, EXT4_EX_NOCACHE); + if (IS_ERR(path2)) { + *erp = PTR_ERR(path2); + goto errout; + } + ex1 = path1[path1->p_depth].p_ext; + ex2 = path2[path2->p_depth].p_ext; + /* Do we have something to swap ? */ + if (unlikely(!ex2 || !ex1)) + goto errout; + + e1_blk = le32_to_cpu(ex1->ee_block); + e2_blk = le32_to_cpu(ex2->ee_block); + e1_len = ext4_ext_get_actual_len(ex1); + e2_len = ext4_ext_get_actual_len(ex2); + + /* Hole handling */ + if (!in_range(lblk1, e1_blk, e1_len) || + !in_range(lblk2, e2_blk, e2_len)) { + ext4_lblk_t next1, next2; + + /* if hole after extent, then go to next extent */ + next1 = ext4_ext_next_allocated_block(path1); + next2 = ext4_ext_next_allocated_block(path2); + /* If hole before extent, then shift to that extent */ + if (e1_blk > lblk1) + next1 = e1_blk; + if (e2_blk > lblk2) + next2 = e2_blk; + /* Do we have something to swap */ + if (next1 == EXT_MAX_BLOCKS || next2 == EXT_MAX_BLOCKS) + goto errout; + /* Move to the rightest boundary */ + len = next1 - lblk1; + if (len < next2 - lblk2) + len = next2 - lblk2; + if (len > count) + len = count; + lblk1 += len; + lblk2 += len; + count -= len; + continue; + } + + /* Prepare left boundary */ + if (e1_blk < lblk1) { + split = 1; + path1 = ext4_force_split_extent_at(handle, inode1, + path1, lblk1, 0); + if (IS_ERR(path1)) { + *erp = PTR_ERR(path1); + goto errout; + } + } + if (e2_blk < lblk2) { + split = 1; + path2 = ext4_force_split_extent_at(handle, inode2, + path2, lblk2, 0); + if (IS_ERR(path2)) { + *erp = PTR_ERR(path2); + goto errout; + } + } + /* ext4_split_extent_at() may result in leaf extent split, + * path must to be revalidated. */ + if (split) + continue; + + /* Prepare right boundary */ + len = count; + if (len > e1_blk + e1_len - lblk1) + len = e1_blk + e1_len - lblk1; + if (len > e2_blk + e2_len - lblk2) + len = e2_blk + e2_len - lblk2; + + if (len != e1_len) { + split = 1; + path1 = ext4_force_split_extent_at(handle, inode1, + path1, lblk1 + len, 0); + if (IS_ERR(path1)) { + *erp = PTR_ERR(path1); + goto errout; + } + } + if (len != e2_len) { + split = 1; + path2 = ext4_force_split_extent_at(handle, inode2, + path2, lblk2 + len, 0); + if (IS_ERR(path2)) { + *erp = PTR_ERR(path2); + goto errout; + } + } + /* ext4_split_extent_at() may result in leaf extent split, + * path must to be revalidated. */ + if (split) + continue; + + BUG_ON(e2_len != e1_len); + *erp = ext4_ext_get_access(handle, inode1, path1 + path1->p_depth); + if (unlikely(*erp)) + goto errout; + *erp = ext4_ext_get_access(handle, inode2, path2 + path2->p_depth); + if (unlikely(*erp)) + goto errout; + + /* Both extents are fully inside boundaries. Swap it now */ + tmp_ex = *ex1; + ext4_ext_store_pblock(ex1, ext4_ext_pblock(ex2)); + ext4_ext_store_pblock(ex2, ext4_ext_pblock(&tmp_ex)); + ex1->ee_len = cpu_to_le16(e2_len); + ex2->ee_len = cpu_to_le16(e1_len); + if (unwritten) + ext4_ext_mark_unwritten(ex2); + if (ext4_ext_is_unwritten(&tmp_ex)) + ext4_ext_mark_unwritten(ex1); + + ext4_ext_try_to_merge(handle, inode2, path2, ex2); + ext4_ext_try_to_merge(handle, inode1, path1, ex1); + *erp = ext4_ext_dirty(handle, inode2, path2 + + path2->p_depth); + if (unlikely(*erp)) + goto errout; + *erp = ext4_ext_dirty(handle, inode1, path1 + + path1->p_depth); + /* + * Looks scarry ah..? second inode already points to new blocks, + * and it was successfully dirtied. But luckily error may happen + * only due to journal error, so full transaction will be + * aborted anyway. + */ + if (unlikely(*erp)) + goto errout; + + lblk1 += len; + lblk2 += len; + replaced_count += len; + count -= len; + } + +errout: + ext4_free_ext_path(path1); + ext4_free_ext_path(path2); + return replaced_count; +} + +/* + * ext4_clu_mapped - determine whether any block in a logical cluster has + * been mapped to a physical cluster + * + * @inode - file containing the logical cluster + * @lclu - logical cluster of interest + * + * Returns 1 if any block in the logical cluster is mapped, signifying + * that a physical cluster has been allocated for it. Otherwise, + * returns 0. Can also return negative error codes. Derived from + * ext4_ext_map_blocks(). + */ +int ext4_clu_mapped(struct inode *inode, ext4_lblk_t lclu) +{ + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + struct ext4_ext_path *path; + int depth, mapped = 0, err = 0; + struct ext4_extent *extent; + ext4_lblk_t first_lblk, first_lclu, last_lclu; + + /* + * if data can be stored inline, the logical cluster isn't + * mapped - no physical clusters have been allocated, and the + * file has no extents + */ + if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA) || + ext4_has_inline_data(inode)) + return 0; + + /* search for the extent closest to the first block in the cluster */ + path = ext4_find_extent(inode, EXT4_C2B(sbi, lclu), NULL, 0); + if (IS_ERR(path)) + return PTR_ERR(path); + + depth = ext_depth(inode); + + /* + * A consistent leaf must not be empty. This situation is possible, + * though, _during_ tree modification, and it's why an assert can't + * be put in ext4_find_extent(). + */ + if (unlikely(path[depth].p_ext == NULL && depth != 0)) { + EXT4_ERROR_INODE(inode, + "bad extent address - lblock: %lu, depth: %d, pblock: %lld", + (unsigned long) EXT4_C2B(sbi, lclu), + depth, path[depth].p_block); + err = -EFSCORRUPTED; + goto out; + } + + extent = path[depth].p_ext; + + /* can't be mapped if the extent tree is empty */ + if (extent == NULL) + goto out; + + first_lblk = le32_to_cpu(extent->ee_block); + first_lclu = EXT4_B2C(sbi, first_lblk); + + /* + * Three possible outcomes at this point - found extent spanning + * the target cluster, to the left of the target cluster, or to the + * right of the target cluster. The first two cases are handled here. + * The last case indicates the target cluster is not mapped. + */ + if (lclu >= first_lclu) { + last_lclu = EXT4_B2C(sbi, first_lblk + + ext4_ext_get_actual_len(extent) - 1); + if (lclu <= last_lclu) { + mapped = 1; + } else { + first_lblk = ext4_ext_next_allocated_block(path); + first_lclu = EXT4_B2C(sbi, first_lblk); + if (lclu == first_lclu) + mapped = 1; + } + } + +out: + ext4_free_ext_path(path); + + return err ? err : mapped; +} + +/* + * Updates physical block address and unwritten status of extent + * starting at lblk start and of len. If such an extent doesn't exist, + * this function splits the extent tree appropriately to create an + * extent like this. This function is called in the fast commit + * replay path. Returns 0 on success and error on failure. + */ +int ext4_ext_replay_update_ex(struct inode *inode, ext4_lblk_t start, + int len, int unwritten, ext4_fsblk_t pblk) +{ + struct ext4_ext_path *path; + struct ext4_extent *ex; + int ret; + + path = ext4_find_extent(inode, start, NULL, 0); + if (IS_ERR(path)) + return PTR_ERR(path); + ex = path[path->p_depth].p_ext; + if (!ex) { + ret = -EFSCORRUPTED; + goto out; + } + + if (le32_to_cpu(ex->ee_block) != start || + ext4_ext_get_actual_len(ex) != len) { + /* We need to split this extent to match our extent first */ + down_write(&EXT4_I(inode)->i_data_sem); + path = ext4_force_split_extent_at(NULL, inode, path, start, 1); + up_write(&EXT4_I(inode)->i_data_sem); + if (IS_ERR(path)) { + ret = PTR_ERR(path); + goto out; + } + + path = ext4_find_extent(inode, start, path, 0); + if (IS_ERR(path)) + return PTR_ERR(path); + + ex = path[path->p_depth].p_ext; + WARN_ON(le32_to_cpu(ex->ee_block) != start); + + if (ext4_ext_get_actual_len(ex) != len) { + down_write(&EXT4_I(inode)->i_data_sem); + path = ext4_force_split_extent_at(NULL, inode, path, + start + len, 1); + up_write(&EXT4_I(inode)->i_data_sem); + if (IS_ERR(path)) { + ret = PTR_ERR(path); + goto out; + } + + path = ext4_find_extent(inode, start, path, 0); + if (IS_ERR(path)) + return PTR_ERR(path); + ex = path[path->p_depth].p_ext; + } + } + if (unwritten) + ext4_ext_mark_unwritten(ex); + else + ext4_ext_mark_initialized(ex); + ext4_ext_store_pblock(ex, pblk); + down_write(&EXT4_I(inode)->i_data_sem); + ret = ext4_ext_dirty(NULL, inode, &path[path->p_depth]); + up_write(&EXT4_I(inode)->i_data_sem); +out: + ext4_free_ext_path(path); + ext4_mark_inode_dirty(NULL, inode); + return ret; +} + +/* Try to shrink the extent tree */ +void ext4_ext_replay_shrink_inode(struct inode *inode, ext4_lblk_t end) +{ + struct ext4_ext_path *path = NULL; + struct ext4_extent *ex; + ext4_lblk_t old_cur, cur = 0; + + while (cur < end) { + path = ext4_find_extent(inode, cur, NULL, 0); + if (IS_ERR(path)) + return; + ex = path[path->p_depth].p_ext; + if (!ex) { + ext4_free_ext_path(path); + ext4_mark_inode_dirty(NULL, inode); + return; + } + old_cur = cur; + cur = le32_to_cpu(ex->ee_block) + ext4_ext_get_actual_len(ex); + if (cur <= old_cur) + cur = old_cur + 1; + ext4_ext_try_to_merge(NULL, inode, path, ex); + down_write(&EXT4_I(inode)->i_data_sem); + ext4_ext_dirty(NULL, inode, &path[path->p_depth]); + up_write(&EXT4_I(inode)->i_data_sem); + ext4_mark_inode_dirty(NULL, inode); + ext4_free_ext_path(path); + } +} + +/* Check if *cur is a hole and if it is, skip it */ +static int skip_hole(struct inode *inode, ext4_lblk_t *cur) +{ + int ret; + struct ext4_map_blocks map; + + map.m_lblk = *cur; + map.m_len = ((inode->i_size) >> inode->i_sb->s_blocksize_bits) - *cur; + + ret = ext4_map_blocks(NULL, inode, &map, 0); + if (ret < 0) + return ret; + if (ret != 0) + return 0; + *cur = *cur + map.m_len; + return 0; +} + +/* Count number of blocks used by this inode and update i_blocks */ +int ext4_ext_replay_set_iblocks(struct inode *inode) +{ + struct ext4_ext_path *path = NULL, *path2 = NULL; + struct ext4_extent *ex; + ext4_lblk_t cur = 0, end; + int numblks = 0, i, ret = 0; + ext4_fsblk_t cmp1, cmp2; + struct ext4_map_blocks map; + + /* Determin the size of the file first */ + path = ext4_find_extent(inode, EXT_MAX_BLOCKS - 1, NULL, + EXT4_EX_NOCACHE); + if (IS_ERR(path)) + return PTR_ERR(path); + ex = path[path->p_depth].p_ext; + if (!ex) + goto out; + end = le32_to_cpu(ex->ee_block) + ext4_ext_get_actual_len(ex); + + /* Count the number of data blocks */ + cur = 0; + while (cur < end) { + map.m_lblk = cur; + map.m_len = end - cur; + ret = ext4_map_blocks(NULL, inode, &map, 0); + if (ret < 0) + break; + if (ret > 0) + numblks += ret; + cur = cur + map.m_len; + } + + /* + * Count the number of extent tree blocks. We do it by looking up + * two successive extents and determining the difference between + * their paths. When path is different for 2 successive extents + * we compare the blocks in the path at each level and increment + * iblocks by total number of differences found. + */ + cur = 0; + ret = skip_hole(inode, &cur); + if (ret < 0) + goto out; + path = ext4_find_extent(inode, cur, path, 0); + if (IS_ERR(path)) + goto out; + numblks += path->p_depth; + while (cur < end) { + path = ext4_find_extent(inode, cur, path, 0); + if (IS_ERR(path)) + break; + ex = path[path->p_depth].p_ext; + if (!ex) + goto cleanup; + + cur = max(cur + 1, le32_to_cpu(ex->ee_block) + + ext4_ext_get_actual_len(ex)); + ret = skip_hole(inode, &cur); + if (ret < 0) + break; + + path2 = ext4_find_extent(inode, cur, path2, 0); + if (IS_ERR(path2)) + break; + + for (i = 0; i <= max(path->p_depth, path2->p_depth); i++) { + cmp1 = cmp2 = 0; + if (i <= path->p_depth) + cmp1 = path[i].p_bh ? + path[i].p_bh->b_blocknr : 0; + if (i <= path2->p_depth) + cmp2 = path2[i].p_bh ? + path2[i].p_bh->b_blocknr : 0; + if (cmp1 != cmp2 && cmp2 != 0) + numblks++; + } + } + +out: + inode->i_blocks = numblks << (inode->i_sb->s_blocksize_bits - 9); + ext4_mark_inode_dirty(NULL, inode); +cleanup: + ext4_free_ext_path(path); + ext4_free_ext_path(path2); + return 0; +} + +int ext4_ext_clear_bb(struct inode *inode) +{ + struct ext4_ext_path *path = NULL; + struct ext4_extent *ex; + ext4_lblk_t cur = 0, end; + int j, ret = 0; + struct ext4_map_blocks map; + + if (ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA)) + return 0; + + /* Determin the size of the file first */ + path = ext4_find_extent(inode, EXT_MAX_BLOCKS - 1, NULL, + EXT4_EX_NOCACHE); + if (IS_ERR(path)) + return PTR_ERR(path); + ex = path[path->p_depth].p_ext; + if (!ex) + goto out; + end = le32_to_cpu(ex->ee_block) + ext4_ext_get_actual_len(ex); + + cur = 0; + while (cur < end) { + map.m_lblk = cur; + map.m_len = end - cur; + ret = ext4_map_blocks(NULL, inode, &map, 0); + if (ret < 0) + break; + if (ret > 0) { + path = ext4_find_extent(inode, map.m_lblk, path, 0); + if (!IS_ERR(path)) { + for (j = 0; j < path->p_depth; j++) { + ext4_mb_mark_bb(inode->i_sb, + path[j].p_block, 1, false); + ext4_fc_record_regions(inode->i_sb, inode->i_ino, + 0, path[j].p_block, 1, 1); + } + } else { + path = NULL; + } + ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, false); + ext4_fc_record_regions(inode->i_sb, inode->i_ino, + map.m_lblk, map.m_pblk, map.m_len, 1); + } + cur = cur + map.m_len; + } + +out: + ext4_free_ext_path(path); + return 0; +} + +#if IS_ENABLED(CONFIG_EXT4_KUNIT_TESTS) +int ext4_ext_space_root_idx_test(struct inode *inode, int check) +{ + return ext4_ext_space_root_idx(inode, check); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_ext_space_root_idx_test); + +struct ext4_ext_path *ext4_split_convert_extents_test(handle_t *handle, + struct inode *inode, struct ext4_map_blocks *map, + struct ext4_ext_path *path, int flags, + unsigned int *allocated) +{ + return ext4_split_convert_extents(handle, inode, map, path, + flags, allocated); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_split_convert_extents_test); + +EXPORT_SYMBOL_FOR_EXT4_TEST(__ext4_ext_dirty); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_ext_zeroout); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_es_register_shrinker); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_es_unregister_shrinker); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_map_create_blocks); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_es_init_tree); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_es_lookup_extent); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_es_insert_extent); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_ext_insert_extent); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_find_extent); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_issue_zeroout); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_map_query_blocks); +#endif diff --git a/fs/ext4/extents_status.c b/fs/ext4/extents_status.c new file mode 100644 index 000000000..6e4a191e8 --- /dev/null +++ b/fs/ext4/extents_status.c @@ -0,0 +1,2392 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/ext4/extents_status.c + * + * Written by Yongqiang Yang + * Modified by + * Allison Henderson + * Hugh Dickins + * Zheng Liu + * + * Ext4 extents status tree core functions. + */ +#include +#include +#include +#include "ext4.h" + +#include +#include + +/* + * According to previous discussion in Ext4 Developer Workshop, we + * will introduce a new structure called io tree to track all extent + * status in order to solve some problems that we have met + * (e.g. Reservation space warning), and provide extent-level locking. + * Delay extent tree is the first step to achieve this goal. It is + * original built by Yongqiang Yang. At that time it is called delay + * extent tree, whose goal is only track delayed extents in memory to + * simplify the implementation of fiemap and bigalloc, and introduce + * lseek SEEK_DATA/SEEK_HOLE support. That is why it is still called + * delay extent tree at the first commit. But for better understand + * what it does, it has been rename to extent status tree. + * + * Step1: + * Currently the first step has been done. All delayed extents are + * tracked in the tree. It maintains the delayed extent when a delayed + * allocation is issued, and the delayed extent is written out or + * invalidated. Therefore the implementation of fiemap and bigalloc + * are simplified, and SEEK_DATA/SEEK_HOLE are introduced. + * + * The following comment describes the implemenmtation of extent + * status tree and future works. + * + * Step2: + * In this step all extent status are tracked by extent status tree. + * Thus, we can first try to lookup a block mapping in this tree before + * finding it in extent tree. Hence, single extent cache can be removed + * because extent status tree can do a better job. Extents in status + * tree are loaded on-demand. Therefore, the extent status tree may not + * contain all of the extents in a file. Meanwhile we define a shrinker + * to reclaim memory from extent status tree because fragmented extent + * tree will make status tree cost too much memory. written/unwritten/- + * hole extents in the tree will be reclaimed by this shrinker when we + * are under high memory pressure. Delayed extents will not be + * reclimed because fiemap, bigalloc, and seek_data/hole need it. + */ + +/* + * Extent status tree implementation for ext4. + * + * + * ========================================================================== + * Extent status tree tracks all extent status. + * + * 1. Why we need to implement extent status tree? + * + * Without extent status tree, ext4 identifies a delayed extent by looking + * up page cache, this has several deficiencies - complicated, buggy, + * and inefficient code. + * + * FIEMAP, SEEK_HOLE/DATA, bigalloc, and writeout all need to know if a + * block or a range of blocks are belonged to a delayed extent. + * + * Let us have a look at how they do without extent status tree. + * -- FIEMAP + * FIEMAP looks up page cache to identify delayed allocations from holes. + * + * -- SEEK_HOLE/DATA + * SEEK_HOLE/DATA has the same problem as FIEMAP. + * + * -- bigalloc + * bigalloc looks up page cache to figure out if a block is + * already under delayed allocation or not to determine whether + * quota reserving is needed for the cluster. + * + * -- writeout + * Writeout looks up whole page cache to see if a buffer is + * mapped, If there are not very many delayed buffers, then it is + * time consuming. + * + * With extent status tree implementation, FIEMAP, SEEK_HOLE/DATA, + * bigalloc and writeout can figure out if a block or a range of + * blocks is under delayed allocation(belonged to a delayed extent) or + * not by searching the extent tree. + * + * + * ========================================================================== + * 2. Ext4 extent status tree impelmentation + * + * -- extent + * A extent is a range of blocks which are contiguous logically and + * physically. Unlike extent in extent tree, this extent in ext4 is + * a in-memory struct, there is no corresponding on-disk data. There + * is no limit on length of extent, so an extent can contain as many + * blocks as they are contiguous logically and physically. + * + * -- extent status tree + * Every inode has an extent status tree and all allocation blocks + * are added to the tree with different status. The extent in the + * tree are ordered by logical block no. + * + * -- operations on a extent status tree + * There are three important operations on a delayed extent tree: find + * next extent, adding a extent(a range of blocks) and removing a extent. + * + * -- race on a extent status tree + * Extent status tree is protected by inode->i_es_lock. + * + * -- memory consumption + * Fragmented extent tree will make extent status tree cost too much + * memory. Hence, we will reclaim written/unwritten/hole extents from + * the tree under a heavy memory pressure. + * + * ========================================================================== + * 3. Assurance of Ext4 extent status tree consistency + * + * When mapping blocks, Ext4 queries the extent status tree first and should + * always trusts that the extent status tree is consistent and up to date. + * Therefore, it is important to adheres to the following rules when createing, + * modifying and removing extents. + * + * 1. Besides fastcommit replay, when Ext4 creates or queries block mappings, + * the extent information should always be processed through the extent + * status tree instead of being organized manually through the on-disk + * extent tree. + * + * 2. When updating the extent tree, Ext4 should acquire the i_data_sem + * exclusively and update the extent status tree atomically. If the extents + * to be modified are large enough to exceed the range that a single + * i_data_sem can process (as ext4_datasem_ensure_credits() may drop + * i_data_sem to restart a transaction), it must (e.g. as ext4_punch_hole() + * does): + * + * a) Hold the i_rwsem and invalidate_lock exclusively. This ensures + * exclusion against page faults, as well as reads and writes that may + * concurrently modify the extent status tree. + * b) Evict all page cache in the affected range and recommend rebuilding + * or dropping the extent status tree after modifying the on-disk + * extent tree. This ensures exclusion against concurrent writebacks + * that do not hold those locks but only holds a folio lock. + * + * 3. Based on the rules above, when querying block mappings, Ext4 should at + * least hold the i_rwsem or invalidate_lock or folio lock(s) for the + * specified querying range. + * + * ========================================================================== + * 4. Performance analysis + * + * -- overhead + * 1. There is a cache extent for write access, so if writes are + * not very random, adding space operaions are in O(1) time. + * + * -- gain + * 2. Code is much simpler, more readable, more maintainable and + * more efficient. + * + * + * ========================================================================== + * 5. TODO list + * + * -- Refactor delayed space reservation + * + * -- Extent-level locking + */ + +static struct kmem_cache *ext4_es_cachep; +static struct kmem_cache *ext4_pending_cachep; + +static int __es_insert_extent(struct inode *inode, struct extent_status *newes, + struct extent_status *prealloc); +static int __es_remove_extent(struct inode *inode, ext4_lblk_t lblk, + ext4_lblk_t end, unsigned int status, + int *reserved, struct extent_status *res, + struct extent_status *prealloc); +static int es_reclaim_extents(struct ext4_inode_info *ei, int *nr_to_scan); +static int __es_shrink(struct ext4_sb_info *sbi, int nr_to_scan, + struct ext4_inode_info *locked_ei); +static int __revise_pending(struct inode *inode, ext4_lblk_t lblk, + ext4_lblk_t len, + struct pending_reservation **prealloc); + +int __init ext4_init_es(void) +{ + ext4_es_cachep = KMEM_CACHE(extent_status, SLAB_RECLAIM_ACCOUNT); + if (ext4_es_cachep == NULL) + return -ENOMEM; + return 0; +} + +void ext4_exit_es(void) +{ + kmem_cache_destroy(ext4_es_cachep); +} + +void ext4_es_init_tree(struct ext4_es_tree *tree) +{ + tree->root = RB_ROOT; + tree->cache_es = NULL; +} + +#ifdef ES_DEBUG__ +static void ext4_es_print_tree(struct inode *inode) +{ + struct ext4_es_tree *tree; + struct rb_node *node; + + printk(KERN_DEBUG "status extents for inode %llu:", inode->i_ino); + tree = &EXT4_I(inode)->i_es_tree; + node = rb_first(&tree->root); + while (node) { + struct extent_status *es; + es = rb_entry(node, struct extent_status, rb_node); + printk(KERN_DEBUG " [%u/%u) %llu %x", + es->es_lblk, es->es_len, + ext4_es_pblock(es), ext4_es_status(es)); + node = rb_next(node); + } + printk(KERN_DEBUG "\n"); +} +#else +#define ext4_es_print_tree(inode) +#endif + +static inline ext4_lblk_t ext4_es_end(struct extent_status *es) +{ + BUG_ON(es->es_lblk + es->es_len < es->es_lblk); + return es->es_lblk + es->es_len - 1; +} + +static inline void ext4_es_inc_seq(struct inode *inode) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + + WRITE_ONCE(ei->i_es_seq, ei->i_es_seq + 1); +} + +static inline int __es_check_extent_status(struct extent_status *es, + unsigned int status, + struct extent_status *res) +{ + if (ext4_es_type(es) & status) + return 0; + + if (res) { + res->es_lblk = es->es_lblk; + res->es_len = es->es_len; + res->es_pblk = es->es_pblk; + } + return -EINVAL; +} + +/* + * search through the tree for an delayed extent with a given offset. If + * it can't be found, try to find next extent. + */ +static struct extent_status *__es_tree_search(struct rb_root *root, + ext4_lblk_t lblk) +{ + struct rb_node *node = root->rb_node; + struct extent_status *es = NULL; + + while (node) { + es = rb_entry(node, struct extent_status, rb_node); + if (lblk < es->es_lblk) + node = node->rb_left; + else if (lblk > ext4_es_end(es)) + node = node->rb_right; + else + return es; + } + + if (es && lblk < es->es_lblk) + return es; + + if (es && lblk > ext4_es_end(es)) { + node = rb_next(&es->rb_node); + return node ? rb_entry(node, struct extent_status, rb_node) : + NULL; + } + + return NULL; +} + +/* + * ext4_es_find_extent_range - find extent with specified status within block + * range or next extent following block range in + * extents status tree + * + * @inode - file containing the range + * @matching_fn - pointer to function that matches extents with desired status + * @lblk - logical block defining start of range + * @end - logical block defining end of range + * @es - extent found, if any + * + * Find the first extent within the block range specified by @lblk and @end + * in the extents status tree that satisfies @matching_fn. If a match + * is found, it's returned in @es. If not, and a matching extent is found + * beyond the block range, it's returned in @es. If no match is found, an + * extent is returned in @es whose es_lblk, es_len, and es_pblk components + * are 0. + */ +static void __es_find_extent_range(struct inode *inode, + int (*matching_fn)(struct extent_status *es), + ext4_lblk_t lblk, ext4_lblk_t end, + struct extent_status *es) +{ + struct ext4_es_tree *tree = NULL; + struct extent_status *es1 = NULL; + struct rb_node *node; + + WARN_ON(es == NULL); + WARN_ON(end < lblk); + + tree = &EXT4_I(inode)->i_es_tree; + + /* see if the extent has been cached */ + es->es_lblk = es->es_len = es->es_pblk = 0; + es1 = READ_ONCE(tree->cache_es); + if (es1 && in_range(lblk, es1->es_lblk, es1->es_len)) { + es_debug("%u cached by [%u/%u) %llu %x\n", + lblk, es1->es_lblk, es1->es_len, + ext4_es_pblock(es1), ext4_es_status(es1)); + goto out; + } + + es1 = __es_tree_search(&tree->root, lblk); + +out: + if (es1 && !matching_fn(es1)) { + while ((node = rb_next(&es1->rb_node)) != NULL) { + es1 = rb_entry(node, struct extent_status, rb_node); + if (es1->es_lblk > end) { + es1 = NULL; + break; + } + if (matching_fn(es1)) + break; + } + } + + if (es1 && matching_fn(es1)) { + WRITE_ONCE(tree->cache_es, es1); + es->es_lblk = es1->es_lblk; + es->es_len = es1->es_len; + es->es_pblk = es1->es_pblk; + } + +} + +/* + * Locking for __es_find_extent_range() for external use + */ +void ext4_es_find_extent_range(struct inode *inode, + int (*matching_fn)(struct extent_status *es), + ext4_lblk_t lblk, ext4_lblk_t end, + struct extent_status *es) +{ + es->es_lblk = es->es_len = es->es_pblk = 0; + + if (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY) + return; + + trace_ext4_es_find_extent_range_enter(inode, lblk); + + read_lock(&EXT4_I(inode)->i_es_lock); + __es_find_extent_range(inode, matching_fn, lblk, end, es); + read_unlock(&EXT4_I(inode)->i_es_lock); + + trace_ext4_es_find_extent_range_exit(inode, es); +} + +/* + * __es_scan_range - search block range for block with specified status + * in extents status tree + * + * @inode - file containing the range + * @matching_fn - pointer to function that matches extents with desired status + * @lblk - logical block defining start of range + * @end - logical block defining end of range + * + * Returns true if at least one block in the specified block range satisfies + * the criterion specified by @matching_fn, and false if not. If at least + * one extent has the specified status, then there is at least one block + * in the cluster with that status. Should only be called by code that has + * taken i_es_lock. + */ +static bool __es_scan_range(struct inode *inode, + int (*matching_fn)(struct extent_status *es), + ext4_lblk_t start, ext4_lblk_t end) +{ + struct extent_status es; + + __es_find_extent_range(inode, matching_fn, start, end, &es); + if (es.es_len == 0) + return false; /* no matching extent in the tree */ + else if (es.es_lblk <= start && + start < es.es_lblk + es.es_len) + return true; + else if (start <= es.es_lblk && es.es_lblk <= end) + return true; + else + return false; +} +/* + * Locking for __es_scan_range() for external use + */ +bool ext4_es_scan_range(struct inode *inode, + int (*matching_fn)(struct extent_status *es), + ext4_lblk_t lblk, ext4_lblk_t end) +{ + bool ret; + + if (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY) + return false; + + read_lock(&EXT4_I(inode)->i_es_lock); + ret = __es_scan_range(inode, matching_fn, lblk, end); + read_unlock(&EXT4_I(inode)->i_es_lock); + + return ret; +} + +/* + * __es_scan_clu - search cluster for block with specified status in + * extents status tree + * + * @inode - file containing the cluster + * @matching_fn - pointer to function that matches extents with desired status + * @lblk - logical block in cluster to be searched + * + * Returns true if at least one extent in the cluster containing @lblk + * satisfies the criterion specified by @matching_fn, and false if not. If at + * least one extent has the specified status, then there is at least one block + * in the cluster with that status. Should only be called by code that has + * taken i_es_lock. + */ +static bool __es_scan_clu(struct inode *inode, + int (*matching_fn)(struct extent_status *es), + ext4_lblk_t lblk) +{ + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + ext4_lblk_t lblk_start, lblk_end; + + lblk_start = EXT4_LBLK_CMASK(sbi, lblk); + lblk_end = lblk_start + sbi->s_cluster_ratio - 1; + + return __es_scan_range(inode, matching_fn, lblk_start, lblk_end); +} + +/* + * Locking for __es_scan_clu() for external use + */ +bool ext4_es_scan_clu(struct inode *inode, + int (*matching_fn)(struct extent_status *es), + ext4_lblk_t lblk) +{ + bool ret; + + if (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY) + return false; + + read_lock(&EXT4_I(inode)->i_es_lock); + ret = __es_scan_clu(inode, matching_fn, lblk); + read_unlock(&EXT4_I(inode)->i_es_lock); + + return ret; +} + +static void ext4_es_list_add(struct inode *inode) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + + if (!list_empty(&ei->i_es_list)) + return; + + spin_lock(&sbi->s_es_lock); + if (list_empty(&ei->i_es_list)) { + list_add_tail(&ei->i_es_list, &sbi->s_es_list); + sbi->s_es_nr_inode++; + } + spin_unlock(&sbi->s_es_lock); +} + +static void ext4_es_list_del(struct inode *inode) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + + spin_lock(&sbi->s_es_lock); + if (!list_empty(&ei->i_es_list)) { + list_del_init(&ei->i_es_list); + sbi->s_es_nr_inode--; + WARN_ON_ONCE(sbi->s_es_nr_inode < 0); + } + spin_unlock(&sbi->s_es_lock); +} + +static inline struct pending_reservation *__alloc_pending(bool nofail) +{ + if (!nofail) + return kmem_cache_alloc(ext4_pending_cachep, GFP_ATOMIC); + + return kmem_cache_zalloc(ext4_pending_cachep, GFP_KERNEL | __GFP_NOFAIL); +} + +static inline void __free_pending(struct pending_reservation *pr) +{ + kmem_cache_free(ext4_pending_cachep, pr); +} + +/* + * Returns true if we cannot fail to allocate memory for this extent_status + * entry and cannot reclaim it until its status changes. + */ +static inline bool ext4_es_must_keep(struct extent_status *es) +{ + /* fiemap, bigalloc, and seek_data/hole need to use it. */ + if (ext4_es_is_delayed(es)) + return true; + + return false; +} + +static inline struct extent_status *__es_alloc_extent(bool nofail) +{ + if (!nofail) + return kmem_cache_alloc(ext4_es_cachep, GFP_ATOMIC); + + return kmem_cache_zalloc(ext4_es_cachep, GFP_KERNEL | __GFP_NOFAIL); +} + +static void ext4_es_init_extent(struct inode *inode, struct extent_status *es, + ext4_lblk_t lblk, ext4_lblk_t len, ext4_fsblk_t pblk) +{ + es->es_lblk = lblk; + es->es_len = len; + es->es_pblk = pblk; + + /* We never try to reclaim a must kept extent, so we don't count it. */ + if (!ext4_es_must_keep(es)) { + if (!EXT4_I(inode)->i_es_shk_nr++) + ext4_es_list_add(inode); + percpu_counter_inc(&EXT4_SB(inode->i_sb)-> + s_es_stats.es_stats_shk_cnt); + } + + EXT4_I(inode)->i_es_all_nr++; + percpu_counter_inc(&EXT4_SB(inode->i_sb)->s_es_stats.es_stats_all_cnt); +} + +static inline void __es_free_extent(struct extent_status *es) +{ + kmem_cache_free(ext4_es_cachep, es); +} + +static void ext4_es_free_extent(struct inode *inode, struct extent_status *es) +{ + EXT4_I(inode)->i_es_all_nr--; + percpu_counter_dec(&EXT4_SB(inode->i_sb)->s_es_stats.es_stats_all_cnt); + + /* Decrease the shrink counter when we can reclaim the extent. */ + if (!ext4_es_must_keep(es)) { + BUG_ON(EXT4_I(inode)->i_es_shk_nr == 0); + if (!--EXT4_I(inode)->i_es_shk_nr) + ext4_es_list_del(inode); + percpu_counter_dec(&EXT4_SB(inode->i_sb)-> + s_es_stats.es_stats_shk_cnt); + } + + __es_free_extent(es); +} + +/* + * Check whether or not two extents can be merged + * Condition: + * - logical block number is contiguous + * - physical block number is contiguous + * - status is equal + */ +static int ext4_es_can_be_merged(struct extent_status *es1, + struct extent_status *es2) +{ + if (ext4_es_type(es1) != ext4_es_type(es2)) + return 0; + + if (((__u64) es1->es_len) + es2->es_len > EXT_MAX_BLOCKS) { + pr_warn("ES assertion failed when merging extents. " + "The sum of lengths of es1 (%d) and es2 (%d) " + "is bigger than allowed file size (%d)\n", + es1->es_len, es2->es_len, EXT_MAX_BLOCKS); + WARN_ON(1); + return 0; + } + + if (((__u64) es1->es_lblk) + es1->es_len != es2->es_lblk) + return 0; + + if ((ext4_es_is_written(es1) || ext4_es_is_unwritten(es1)) && + (ext4_es_pblock(es1) + es1->es_len == ext4_es_pblock(es2))) + return 1; + + if (ext4_es_is_hole(es1)) + return 1; + + /* we need to check delayed extent */ + if (ext4_es_is_delayed(es1)) + return 1; + + return 0; +} + +static struct extent_status * +ext4_es_try_to_merge_left(struct inode *inode, struct extent_status *es) +{ + struct ext4_es_tree *tree = &EXT4_I(inode)->i_es_tree; + struct extent_status *es1; + struct rb_node *node; + + node = rb_prev(&es->rb_node); + if (!node) + return es; + + es1 = rb_entry(node, struct extent_status, rb_node); + if (ext4_es_can_be_merged(es1, es)) { + es1->es_len += es->es_len; + if (ext4_es_is_referenced(es)) + ext4_es_set_referenced(es1); + rb_erase(&es->rb_node, &tree->root); + ext4_es_free_extent(inode, es); + es = es1; + } + + return es; +} + +static struct extent_status * +ext4_es_try_to_merge_right(struct inode *inode, struct extent_status *es) +{ + struct ext4_es_tree *tree = &EXT4_I(inode)->i_es_tree; + struct extent_status *es1; + struct rb_node *node; + + node = rb_next(&es->rb_node); + if (!node) + return es; + + es1 = rb_entry(node, struct extent_status, rb_node); + if (ext4_es_can_be_merged(es, es1)) { + es->es_len += es1->es_len; + if (ext4_es_is_referenced(es1)) + ext4_es_set_referenced(es); + rb_erase(node, &tree->root); + ext4_es_free_extent(inode, es1); + } + + return es; +} + +#ifdef ES_AGGRESSIVE_TEST +#include "ext4_extents.h" /* Needed when ES_AGGRESSIVE_TEST is defined */ + +static void ext4_es_insert_extent_ext_check(struct inode *inode, + struct extent_status *es) +{ + struct ext4_ext_path *path = NULL; + struct ext4_extent *ex; + ext4_lblk_t ee_block; + ext4_fsblk_t ee_start; + unsigned short ee_len; + int depth, ee_status, es_status; + + path = ext4_find_extent(inode, es->es_lblk, NULL, EXT4_EX_NOCACHE); + if (IS_ERR(path)) + return; + + depth = ext_depth(inode); + ex = path[depth].p_ext; + + if (ex) { + + ee_block = le32_to_cpu(ex->ee_block); + ee_start = ext4_ext_pblock(ex); + ee_len = ext4_ext_get_actual_len(ex); + + ee_status = ext4_ext_is_unwritten(ex) ? 1 : 0; + es_status = ext4_es_is_unwritten(es) ? 1 : 0; + + /* + * Make sure ex and es are not overlap when we try to insert + * a delayed/hole extent. + */ + if (!ext4_es_is_written(es) && !ext4_es_is_unwritten(es)) { + if (in_range(es->es_lblk, ee_block, ee_len)) { + pr_warn("ES insert assertion failed for " + "inode: %llu we can find an extent " + "at block [%d/%d/%llu/%c], but we " + "want to add a delayed/hole extent " + "[%d/%d/%llu/%x]\n", + inode->i_ino, ee_block, ee_len, + ee_start, ee_status ? 'u' : 'w', + es->es_lblk, es->es_len, + ext4_es_pblock(es), ext4_es_status(es)); + } + goto out; + } + + /* + * We don't check ee_block == es->es_lblk, etc. because es + * might be a part of whole extent, vice versa. + */ + if (es->es_lblk < ee_block || + ext4_es_pblock(es) != ee_start + es->es_lblk - ee_block) { + pr_warn("ES insert assertion failed for inode: %llu " + "ex_status [%d/%d/%llu/%c] != " + "es_status [%d/%d/%llu/%c]\n", inode->i_ino, + ee_block, ee_len, ee_start, + ee_status ? 'u' : 'w', es->es_lblk, es->es_len, + ext4_es_pblock(es), es_status ? 'u' : 'w'); + goto out; + } + + if (ee_status ^ es_status) { + pr_warn("ES insert assertion failed for inode: %llu " + "ex_status [%d/%d/%llu/%c] != " + "es_status [%d/%d/%llu/%c]\n", inode->i_ino, + ee_block, ee_len, ee_start, + ee_status ? 'u' : 'w', es->es_lblk, es->es_len, + ext4_es_pblock(es), es_status ? 'u' : 'w'); + } + } else { + /* + * We can't find an extent on disk. So we need to make sure + * that we don't want to add an written/unwritten extent. + */ + if (!ext4_es_is_delayed(es) && !ext4_es_is_hole(es)) { + pr_warn("ES insert assertion failed for inode: %llu " + "can't find an extent at block %d but we want " + "to add a written/unwritten extent " + "[%d/%d/%llu/%x]\n", inode->i_ino, + es->es_lblk, es->es_lblk, es->es_len, + ext4_es_pblock(es), ext4_es_status(es)); + } + } +out: + ext4_free_ext_path(path); +} + +static void ext4_es_insert_extent_ind_check(struct inode *inode, + struct extent_status *es) +{ + struct ext4_map_blocks map; + int retval; + + /* + * Here we call ext4_ind_map_blocks to lookup a block mapping because + * 'Indirect' structure is defined in indirect.c. So we couldn't + * access direct/indirect tree from outside. It is too dirty to define + * this function in indirect.c file. + */ + + map.m_lblk = es->es_lblk; + map.m_len = es->es_len; + + retval = ext4_ind_map_blocks(NULL, inode, &map, 0); + if (retval > 0) { + if (ext4_es_is_delayed(es) || ext4_es_is_hole(es)) { + /* + * We want to add a delayed/hole extent but this + * block has been allocated. + */ + pr_warn("ES insert assertion failed for inode: %llu " + "We can find blocks but we want to add a " + "delayed/hole extent [%d/%d/%llu/%x]\n", + inode->i_ino, es->es_lblk, es->es_len, + ext4_es_pblock(es), ext4_es_status(es)); + return; + } else if (ext4_es_is_written(es)) { + if (retval != es->es_len) { + pr_warn("ES insert assertion failed for " + "inode: %llu retval %d != es_len %d\n", + inode->i_ino, retval, es->es_len); + return; + } + if (map.m_pblk != ext4_es_pblock(es)) { + pr_warn("ES insert assertion failed for " + "inode: %llu m_pblk %llu != " + "es_pblk %llu\n", + inode->i_ino, map.m_pblk, + ext4_es_pblock(es)); + return; + } + } else { + /* + * We don't need to check unwritten extent because + * indirect-based file doesn't have it. + */ + BUG(); + } + } else if (retval == 0) { + if (ext4_es_is_written(es)) { + pr_warn("ES insert assertion failed for inode: %llu " + "We can't find the block but we want to add " + "a written extent [%d/%d/%llu/%x]\n", + inode->i_ino, es->es_lblk, es->es_len, + ext4_es_pblock(es), ext4_es_status(es)); + return; + } + } +} + +static inline void ext4_es_insert_extent_check(struct inode *inode, + struct extent_status *es) +{ + /* + * We don't need to worry about the race condition because + * caller takes i_data_sem locking. + */ + BUG_ON(!rwsem_is_locked(&EXT4_I(inode)->i_data_sem)); + if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) + ext4_es_insert_extent_ext_check(inode, es); + else + ext4_es_insert_extent_ind_check(inode, es); +} +#else +static inline void ext4_es_insert_extent_check(struct inode *inode, + struct extent_status *es) +{ +} +#endif + +static int __es_insert_extent(struct inode *inode, struct extent_status *newes, + struct extent_status *prealloc) +{ + struct ext4_es_tree *tree = &EXT4_I(inode)->i_es_tree; + struct rb_node **p = &tree->root.rb_node; + struct rb_node *parent = NULL; + struct extent_status *es; + + while (*p) { + parent = *p; + es = rb_entry(parent, struct extent_status, rb_node); + + if (newes->es_lblk < es->es_lblk) { + if (ext4_es_can_be_merged(newes, es)) { + /* + * Here we can modify es_lblk directly + * because it isn't overlapped. + */ + es->es_lblk = newes->es_lblk; + es->es_len += newes->es_len; + if (ext4_es_is_written(es) || + ext4_es_is_unwritten(es)) + ext4_es_store_pblock(es, + newes->es_pblk); + es = ext4_es_try_to_merge_left(inode, es); + goto out; + } + p = &(*p)->rb_left; + } else if (newes->es_lblk > ext4_es_end(es)) { + if (ext4_es_can_be_merged(es, newes)) { + es->es_len += newes->es_len; + es = ext4_es_try_to_merge_right(inode, es); + goto out; + } + p = &(*p)->rb_right; + } else { + BUG(); + return -EINVAL; + } + } + + if (prealloc) + es = prealloc; + else + es = __es_alloc_extent(false); + if (!es) + return -ENOMEM; + ext4_es_init_extent(inode, es, newes->es_lblk, newes->es_len, + newes->es_pblk); + + rb_link_node(&es->rb_node, parent, p); + rb_insert_color(&es->rb_node, &tree->root); + +out: + tree->cache_es = es; + return 0; +} + +/* + * ext4_es_insert_extent() adds information to an inode's extent + * status tree. This interface is used for modifying extents. To cache + * on-disk extents, use ext4_es_cache_extent() instead. + */ +void ext4_es_insert_extent(struct inode *inode, ext4_lblk_t lblk, + ext4_lblk_t len, ext4_fsblk_t pblk, + unsigned int status, bool delalloc_reserve_used) +{ + struct extent_status newes; + ext4_lblk_t end = lblk + len - 1; + int err1 = 0, err2 = 0, err3 = 0; + int resv_used = 0, pending = 0; + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + struct extent_status *es1 = NULL; + struct extent_status *es2 = NULL; + struct pending_reservation *pr = NULL; + bool revise_pending = false; + + if (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY) + return; + + es_debug("add [%u/%u) %llu %x %d to extent status tree of inode %llu\n", + lblk, len, pblk, status, delalloc_reserve_used, inode->i_ino); + + if (!len) + return; + + BUG_ON(end < lblk); + WARN_ON_ONCE(status & EXTENT_STATUS_DELAYED); + + newes.es_lblk = lblk; + newes.es_len = len; + ext4_es_store_pblock_status(&newes, pblk, status); + + ext4_es_insert_extent_check(inode, &newes); + + revise_pending = sbi->s_cluster_ratio > 1 && + test_opt(inode->i_sb, DELALLOC) && + (status & (EXTENT_STATUS_WRITTEN | + EXTENT_STATUS_UNWRITTEN)); +retry: + if (err1 && !es1) + es1 = __es_alloc_extent(true); + if ((err1 || err2) && !es2) + es2 = __es_alloc_extent(true); + if ((err1 || err2 || err3 < 0) && revise_pending && !pr) + pr = __alloc_pending(true); + write_lock(&EXT4_I(inode)->i_es_lock); + + err1 = __es_remove_extent(inode, lblk, end, 0, &resv_used, NULL, es1); + if (err1 != 0) + goto error; + /* Free preallocated extent if it didn't get used. */ + if (es1) { + if (!es1->es_len) + __es_free_extent(es1); + es1 = NULL; + } + + err2 = __es_insert_extent(inode, &newes, es2); + if (err2 == -ENOMEM && !ext4_es_must_keep(&newes)) + err2 = 0; + if (err2 != 0) + goto error; + /* Free preallocated extent if it didn't get used. */ + if (es2) { + if (!es2->es_len) + __es_free_extent(es2); + es2 = NULL; + } + + if (revise_pending) { + err3 = __revise_pending(inode, lblk, len, &pr); + if (err3 < 0) + goto error; + if (pr) { + __free_pending(pr); + pr = NULL; + } + pending = err3; + } + ext4_es_inc_seq(inode); +error: + write_unlock(&EXT4_I(inode)->i_es_lock); + /* + * Reduce the reserved cluster count to reflect successful deferred + * allocation of delayed allocated clusters or direct allocation of + * clusters discovered to be delayed allocated. Once allocated, a + * cluster is not included in the reserved count. + * + * When direct allocating (from fallocate, filemap, DIO, or clusters + * allocated when delalloc has been disabled by ext4_nonda_switch()) + * an extent either 1) contains delayed blocks but start with + * non-delayed allocated blocks (e.g. hole) or 2) contains non-delayed + * allocated blocks which belong to delayed allocated clusters when + * bigalloc feature is enabled, quota has already been claimed by + * ext4_mb_new_blocks(), so release the quota reservations made for + * any previously delayed allocated clusters instead of claim them + * again. + */ + resv_used += pending; + if (resv_used) + ext4_da_update_reserve_space(inode, resv_used, + delalloc_reserve_used); + + if (err1 || err2 || err3 < 0) + goto retry; + + trace_ext4_es_insert_extent(inode, &newes); + ext4_es_print_tree(inode); + return; +} + +/* + * ext4_es_cache_extent() inserts information into the extent status tree + * only if there is no existing information about the specified range or + * if the existing extents have the same status. + * + * Note that this interface is only used for caching on-disk extent + * information and cannot be used to convert existing extents in the extent + * status tree. To convert existing extents, use ext4_es_insert_extent() + * instead. + */ +void ext4_es_cache_extent(struct inode *inode, ext4_lblk_t lblk, + ext4_lblk_t len, ext4_fsblk_t pblk, + unsigned int status) +{ + struct extent_status *es; + struct extent_status chkes, newes; + ext4_lblk_t end = lblk + len - 1; + bool conflict = false; + int err; + + if (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY) + return; + + newes.es_lblk = lblk; + newes.es_len = len; + ext4_es_store_pblock_status(&newes, pblk, status); + + if (!len) + return; + + BUG_ON(end < lblk); + + write_lock(&EXT4_I(inode)->i_es_lock); + es = __es_tree_search(&EXT4_I(inode)->i_es_tree.root, lblk); + if (es && es->es_lblk <= end) { + /* Found an extent that covers the entire range. */ + if (es->es_lblk <= lblk && es->es_lblk + es->es_len > end) { + if (__es_check_extent_status(es, status, &chkes)) + conflict = true; + goto unlock; + } + /* Check and remove all extents in range. */ + err = __es_remove_extent(inode, lblk, end, status, NULL, + &chkes, NULL); + if (err) { + if (err == -EINVAL) + conflict = true; + goto unlock; + } + } + __es_insert_extent(inode, &newes, NULL); + trace_ext4_es_cache_extent(inode, &newes); + ext4_es_print_tree(inode); +unlock: + write_unlock(&EXT4_I(inode)->i_es_lock); + if (!conflict) + return; + /* + * A hole in the on-disk extent but a delayed extent in the extent + * status tree, is allowed. + */ + if (status == EXTENT_STATUS_HOLE && + ext4_es_type(&chkes) == EXTENT_STATUS_DELAYED) + return; + + ext4_warning_inode(inode, + "ES cache extent failed: add [%d,%d,%llu,0x%x] conflict with existing [%d,%d,%llu,0x%x]\n", + lblk, len, pblk, status, chkes.es_lblk, chkes.es_len, + ext4_es_pblock(&chkes), ext4_es_status(&chkes)); +} + +/* + * ext4_es_lookup_extent() looks up an extent in extent status tree. + * + * ext4_es_lookup_extent is called by ext4_map_blocks/ext4_da_map_blocks. + * + * Return: 1 on found, 0 on not + */ +int ext4_es_lookup_extent(struct inode *inode, ext4_lblk_t lblk, + ext4_lblk_t *next_lblk, struct extent_status *es, + u64 *pseq) +{ + struct ext4_es_tree *tree; + struct ext4_es_stats *stats; + struct extent_status *es1 = NULL; + struct rb_node *node; + int found = 0; + + if (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY) + return 0; + + trace_ext4_es_lookup_extent_enter(inode, lblk); + es_debug("lookup extent in block %u\n", lblk); + + tree = &EXT4_I(inode)->i_es_tree; + read_lock(&EXT4_I(inode)->i_es_lock); + + /* find extent in cache firstly */ + es->es_lblk = es->es_len = es->es_pblk = 0; + es1 = READ_ONCE(tree->cache_es); + if (es1 && in_range(lblk, es1->es_lblk, es1->es_len)) { + es_debug("%u cached by [%u/%u)\n", + lblk, es1->es_lblk, es1->es_len); + found = 1; + goto out; + } + + node = tree->root.rb_node; + while (node) { + es1 = rb_entry(node, struct extent_status, rb_node); + if (lblk < es1->es_lblk) + node = node->rb_left; + else if (lblk > ext4_es_end(es1)) + node = node->rb_right; + else { + found = 1; + break; + } + } + +out: + stats = &EXT4_SB(inode->i_sb)->s_es_stats; + if (found) { + BUG_ON(!es1); + es->es_lblk = es1->es_lblk; + es->es_len = es1->es_len; + es->es_pblk = es1->es_pblk; + if (!ext4_es_is_referenced(es1)) + ext4_es_set_referenced(es1); + percpu_counter_inc(&stats->es_stats_cache_hits); + if (next_lblk) { + node = rb_next(&es1->rb_node); + if (node) { + es1 = rb_entry(node, struct extent_status, + rb_node); + *next_lblk = es1->es_lblk; + } else + *next_lblk = 0; + } + if (pseq) + *pseq = EXT4_I(inode)->i_es_seq; + } else { + percpu_counter_inc(&stats->es_stats_cache_misses); + } + + read_unlock(&EXT4_I(inode)->i_es_lock); + + trace_ext4_es_lookup_extent_exit(inode, es, found); + return found; +} + +struct rsvd_count { + int ndelayed; + bool first_do_lblk_found; + ext4_lblk_t first_do_lblk; + ext4_lblk_t last_do_lblk; + struct extent_status *left_es; + bool partial; + ext4_lblk_t lclu; +}; + +/* + * init_rsvd - initialize reserved count data before removing block range + * in file from extent status tree + * + * @inode - file containing range + * @lblk - first block in range + * @es - pointer to first extent in range + * @rc - pointer to reserved count data + * + * Assumes es is not NULL + */ +static void init_rsvd(struct inode *inode, ext4_lblk_t lblk, + struct extent_status *es, struct rsvd_count *rc) +{ + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + struct rb_node *node; + + rc->ndelayed = 0; + + /* + * for bigalloc, note the first delayed block in the range has not + * been found, record the extent containing the block to the left of + * the region to be removed, if any, and note that there's no partial + * cluster to track + */ + if (sbi->s_cluster_ratio > 1) { + rc->first_do_lblk_found = false; + if (lblk > es->es_lblk) { + rc->left_es = es; + } else { + node = rb_prev(&es->rb_node); + rc->left_es = node ? rb_entry(node, + struct extent_status, + rb_node) : NULL; + } + rc->partial = false; + } +} + +/* + * count_rsvd - count the clusters containing delayed blocks in a range + * within an extent and add to the running tally in rsvd_count + * + * @inode - file containing extent + * @lblk - first block in range + * @len - length of range in blocks + * @es - pointer to extent containing clusters to be counted + * @rc - pointer to reserved count data + * + * Tracks partial clusters found at the beginning and end of extents so + * they aren't overcounted when they span adjacent extents + */ +static void count_rsvd(struct inode *inode, ext4_lblk_t lblk, long len, + struct extent_status *es, struct rsvd_count *rc) +{ + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + ext4_lblk_t i, end, nclu; + + if (!ext4_es_is_delayed(es)) + return; + + WARN_ON(len <= 0); + + if (sbi->s_cluster_ratio == 1) { + rc->ndelayed += (int) len; + return; + } + + /* bigalloc */ + + i = (lblk < es->es_lblk) ? es->es_lblk : lblk; + end = lblk + (ext4_lblk_t) len - 1; + end = (end > ext4_es_end(es)) ? ext4_es_end(es) : end; + + /* record the first block of the first delayed extent seen */ + if (!rc->first_do_lblk_found) { + rc->first_do_lblk = i; + rc->first_do_lblk_found = true; + } + + /* update the last lblk in the region seen so far */ + rc->last_do_lblk = end; + + /* + * if we're tracking a partial cluster and the current extent + * doesn't start with it, count it and stop tracking + */ + if (rc->partial && (rc->lclu != EXT4_B2C(sbi, i))) { + rc->ndelayed++; + rc->partial = false; + } + + /* + * if the first cluster doesn't start on a cluster boundary but + * ends on one, count it + */ + if (EXT4_LBLK_COFF(sbi, i) != 0) { + if (end >= EXT4_LBLK_CFILL(sbi, i)) { + rc->ndelayed++; + rc->partial = false; + i = EXT4_LBLK_CFILL(sbi, i) + 1; + } + } + + /* + * if the current cluster starts on a cluster boundary, count the + * number of whole delayed clusters in the extent + */ + if ((i + sbi->s_cluster_ratio - 1) <= end) { + nclu = (end - i + 1) >> sbi->s_cluster_bits; + rc->ndelayed += nclu; + i += nclu << sbi->s_cluster_bits; + } + + /* + * start tracking a partial cluster if there's a partial at the end + * of the current extent and we're not already tracking one + */ + if (!rc->partial && i <= end) { + rc->partial = true; + rc->lclu = EXT4_B2C(sbi, i); + } +} + +/* + * __pr_tree_search - search for a pending cluster reservation + * + * @root - root of pending reservation tree + * @lclu - logical cluster to search for + * + * Returns the pending reservation for the cluster identified by @lclu + * if found. If not, returns a reservation for the next cluster if any, + * and if not, returns NULL. + */ +static struct pending_reservation *__pr_tree_search(struct rb_root *root, + ext4_lblk_t lclu) +{ + struct rb_node *node = root->rb_node; + struct pending_reservation *pr = NULL; + + while (node) { + pr = rb_entry(node, struct pending_reservation, rb_node); + if (lclu < pr->lclu) + node = node->rb_left; + else if (lclu > pr->lclu) + node = node->rb_right; + else + return pr; + } + if (pr && lclu < pr->lclu) + return pr; + if (pr && lclu > pr->lclu) { + node = rb_next(&pr->rb_node); + return node ? rb_entry(node, struct pending_reservation, + rb_node) : NULL; + } + return NULL; +} + +/* + * get_rsvd - calculates and returns the number of cluster reservations to be + * released when removing a block range from the extent status tree + * and releases any pending reservations within the range + * + * @inode - file containing block range + * @end - last block in range + * @right_es - pointer to extent containing next block beyond end or NULL + * @rc - pointer to reserved count data + * + * The number of reservations to be released is equal to the number of + * clusters containing delayed blocks within the range, minus the number of + * clusters still containing delayed blocks at the ends of the range, and + * minus the number of pending reservations within the range. + */ +static unsigned int get_rsvd(struct inode *inode, ext4_lblk_t end, + struct extent_status *right_es, + struct rsvd_count *rc) +{ + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + struct pending_reservation *pr; + struct ext4_pending_tree *tree = &EXT4_I(inode)->i_pending_tree; + struct rb_node *node; + ext4_lblk_t first_lclu, last_lclu; + bool left_delayed, right_delayed, count_pending; + struct extent_status *es; + + if (sbi->s_cluster_ratio > 1) { + /* count any remaining partial cluster */ + if (rc->partial) + rc->ndelayed++; + + if (rc->ndelayed == 0) + return 0; + + first_lclu = EXT4_B2C(sbi, rc->first_do_lblk); + last_lclu = EXT4_B2C(sbi, rc->last_do_lblk); + + /* + * decrease the delayed count by the number of clusters at the + * ends of the range that still contain delayed blocks - + * these clusters still need to be reserved + */ + left_delayed = right_delayed = false; + + es = rc->left_es; + while (es && ext4_es_end(es) >= + EXT4_LBLK_CMASK(sbi, rc->first_do_lblk)) { + if (ext4_es_is_delayed(es)) { + rc->ndelayed--; + left_delayed = true; + break; + } + node = rb_prev(&es->rb_node); + if (!node) + break; + es = rb_entry(node, struct extent_status, rb_node); + } + if (right_es && (!left_delayed || first_lclu != last_lclu)) { + if (end < ext4_es_end(right_es)) { + es = right_es; + } else { + node = rb_next(&right_es->rb_node); + es = node ? rb_entry(node, struct extent_status, + rb_node) : NULL; + } + while (es && es->es_lblk <= + EXT4_LBLK_CFILL(sbi, rc->last_do_lblk)) { + if (ext4_es_is_delayed(es)) { + rc->ndelayed--; + right_delayed = true; + break; + } + node = rb_next(&es->rb_node); + if (!node) + break; + es = rb_entry(node, struct extent_status, + rb_node); + } + } + + /* + * Determine the block range that should be searched for + * pending reservations, if any. Clusters on the ends of the + * original removed range containing delayed blocks are + * excluded. They've already been accounted for and it's not + * possible to determine if an associated pending reservation + * should be released with the information available in the + * extents status tree. + */ + if (first_lclu == last_lclu) { + if (left_delayed | right_delayed) + count_pending = false; + else + count_pending = true; + } else { + if (left_delayed) + first_lclu++; + if (right_delayed) + last_lclu--; + if (first_lclu <= last_lclu) + count_pending = true; + else + count_pending = false; + } + + /* + * a pending reservation found between first_lclu and last_lclu + * represents an allocated cluster that contained at least one + * delayed block, so the delayed total must be reduced by one + * for each pending reservation found and released + */ + if (count_pending) { + pr = __pr_tree_search(&tree->root, first_lclu); + while (pr && pr->lclu <= last_lclu) { + rc->ndelayed--; + node = rb_next(&pr->rb_node); + rb_erase(&pr->rb_node, &tree->root); + __free_pending(pr); + if (!node) + break; + pr = rb_entry(node, struct pending_reservation, + rb_node); + } + } + } + return rc->ndelayed; +} + +/* + * __es_remove_extent - removes block range from extent status tree + * + * @inode - file containing range + * @lblk - first block in range + * @end - last block in range + * @status - the extent status to be checked + * @reserved - number of cluster reservations released + * @res - return the extent if the status is not match + * @prealloc - pre-allocated es to avoid memory allocation failures + * + * If @reserved is not NULL and delayed allocation is enabled, counts + * block/cluster reservations freed by removing range and if bigalloc + * enabled cancels pending reservations as needed. If @status is not + * zero, check extent status type while removing extent, return -EINVAL + * and pass out the extent through @res if not match. Returns 0 on + * success, error code on failure. + */ +static int __es_remove_extent(struct inode *inode, ext4_lblk_t lblk, + ext4_lblk_t end, unsigned int status, + int *reserved, struct extent_status *res, + struct extent_status *prealloc) +{ + struct ext4_es_tree *tree = &EXT4_I(inode)->i_es_tree; + struct rb_node *node; + struct extent_status *es; + struct extent_status orig_es; + ext4_lblk_t len1, len2; + ext4_fsblk_t block; + int err; + bool count_reserved = true; + struct rsvd_count rc; + + if (reserved == NULL || !test_opt(inode->i_sb, DELALLOC)) + count_reserved = false; + if (status == 0) + status = ES_TYPE_MASK; + + es = __es_tree_search(&tree->root, lblk); + if (!es) + return 0; + if (es->es_lblk > end) + return 0; + + err = __es_check_extent_status(es, status, res); + if (err) + return err; + + /* Simply invalidate cache_es. */ + tree->cache_es = NULL; + if (count_reserved) + init_rsvd(inode, lblk, es, &rc); + + orig_es.es_lblk = es->es_lblk; + orig_es.es_len = es->es_len; + orig_es.es_pblk = es->es_pblk; + + len1 = lblk > es->es_lblk ? lblk - es->es_lblk : 0; + len2 = ext4_es_end(es) > end ? ext4_es_end(es) - end : 0; + if (len1 > 0) + es->es_len = len1; + if (len2 > 0) { + if (len1 > 0) { + struct extent_status newes; + + newes.es_lblk = end + 1; + newes.es_len = len2; + block = 0x7FDEADBEEFULL; + if (ext4_es_is_written(&orig_es) || + ext4_es_is_unwritten(&orig_es)) + block = ext4_es_pblock(&orig_es) + + orig_es.es_len - len2; + ext4_es_store_pblock_status(&newes, block, + ext4_es_status(&orig_es)); + err = __es_insert_extent(inode, &newes, prealloc); + if (err) { + if (!ext4_es_must_keep(&newes)) + return 0; + + es->es_lblk = orig_es.es_lblk; + es->es_len = orig_es.es_len; + return err; + } + } else { + es->es_lblk = end + 1; + es->es_len = len2; + if (ext4_es_is_written(es) || + ext4_es_is_unwritten(es)) { + block = orig_es.es_pblk + orig_es.es_len - len2; + ext4_es_store_pblock(es, block); + } + } + if (count_reserved) + count_rsvd(inode, orig_es.es_lblk + len1, + orig_es.es_len - len1 - len2, &orig_es, &rc); + goto out; + } + + if (len1 > 0) { + if (count_reserved) + count_rsvd(inode, lblk, orig_es.es_len - len1, + &orig_es, &rc); + node = rb_next(&es->rb_node); + if (node) + es = rb_entry(node, struct extent_status, rb_node); + else + es = NULL; + } + + while (es && ext4_es_end(es) <= end) { + err = __es_check_extent_status(es, status, res); + if (err) + return err; + if (count_reserved) + count_rsvd(inode, es->es_lblk, es->es_len, es, &rc); + node = rb_next(&es->rb_node); + rb_erase(&es->rb_node, &tree->root); + ext4_es_free_extent(inode, es); + if (!node) { + es = NULL; + break; + } + es = rb_entry(node, struct extent_status, rb_node); + } + + if (es && es->es_lblk < end + 1) { + ext4_lblk_t orig_len = es->es_len; + + err = __es_check_extent_status(es, status, res); + if (err) + return err; + + len1 = ext4_es_end(es) - end; + if (count_reserved) + count_rsvd(inode, es->es_lblk, orig_len - len1, + es, &rc); + es->es_lblk = end + 1; + es->es_len = len1; + if (ext4_es_is_written(es) || ext4_es_is_unwritten(es)) { + block = es->es_pblk + orig_len - len1; + ext4_es_store_pblock(es, block); + } + } + +out: + if (count_reserved) + *reserved = get_rsvd(inode, end, es, &rc); + return 0; +} + +/* + * ext4_es_remove_extent - removes block range from extent status tree + * + * @inode - file containing range + * @lblk - first block in range + * @len - number of blocks to remove + * + * Reduces block/cluster reservation count and for bigalloc cancels pending + * reservations as needed. + */ +void ext4_es_remove_extent(struct inode *inode, ext4_lblk_t lblk, + ext4_lblk_t len) +{ + ext4_lblk_t end; + int err = 0; + int reserved = 0; + struct extent_status *es = NULL; + + if (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY) + return; + + es_debug("remove [%u/%u) from extent status tree of inode %llu\n", + lblk, len, inode->i_ino); + + if (!len) + return; + + end = lblk + len - 1; + BUG_ON(end < lblk); + +retry: + if (err && !es) + es = __es_alloc_extent(true); + /* + * ext4_clear_inode() depends on us taking i_es_lock unconditionally + * so that we are sure __es_shrink() is done with the inode before it + * is reclaimed. + */ + write_lock(&EXT4_I(inode)->i_es_lock); + err = __es_remove_extent(inode, lblk, end, 0, &reserved, NULL, es); + if (err) + goto error; + /* Free preallocated extent if it didn't get used. */ + if (es) { + if (!es->es_len) + __es_free_extent(es); + es = NULL; + } + ext4_es_inc_seq(inode); +error: + write_unlock(&EXT4_I(inode)->i_es_lock); + if (err) + goto retry; + + trace_ext4_es_remove_extent(inode, lblk, len); + ext4_es_print_tree(inode); + ext4_da_release_space(inode, reserved); +} + +static int __es_shrink(struct ext4_sb_info *sbi, int nr_to_scan, + struct ext4_inode_info *locked_ei) +{ + struct ext4_inode_info *ei; + struct ext4_es_stats *es_stats; + ktime_t start_time; + u64 scan_time; + int nr_to_walk; + int nr_shrunk = 0; + int retried = 0, nr_skipped = 0; + + es_stats = &sbi->s_es_stats; + start_time = ktime_get(); + +retry: + spin_lock(&sbi->s_es_lock); + nr_to_walk = sbi->s_es_nr_inode; + while (nr_to_walk-- > 0) { + if (list_empty(&sbi->s_es_list)) { + spin_unlock(&sbi->s_es_lock); + goto out; + } + ei = list_first_entry(&sbi->s_es_list, struct ext4_inode_info, + i_es_list); + /* Move the inode to the tail */ + list_move_tail(&ei->i_es_list, &sbi->s_es_list); + + /* + * Normally we try hard to avoid shrinking precached inodes, + * but we will as a last resort. + */ + if (!retried && ext4_test_inode_state(&ei->vfs_inode, + EXT4_STATE_EXT_PRECACHED)) { + nr_skipped++; + continue; + } + + if (ei == locked_ei || !write_trylock(&ei->i_es_lock)) { + nr_skipped++; + continue; + } + /* + * Now we hold i_es_lock which protects us from inode reclaim + * freeing inode under us + */ + spin_unlock(&sbi->s_es_lock); + + nr_shrunk += es_reclaim_extents(ei, &nr_to_scan); + write_unlock(&ei->i_es_lock); + + if (nr_to_scan <= 0) + goto out; + spin_lock(&sbi->s_es_lock); + } + spin_unlock(&sbi->s_es_lock); + + /* + * If we skipped any inodes, and we weren't able to make any + * forward progress, try again to scan precached inodes. + */ + if ((nr_shrunk == 0) && nr_skipped && !retried) { + retried++; + goto retry; + } + + if (locked_ei && nr_shrunk == 0) + nr_shrunk = es_reclaim_extents(locked_ei, &nr_to_scan); + +out: + scan_time = ktime_to_ns(ktime_sub(ktime_get(), start_time)); + if (likely(es_stats->es_stats_scan_time)) + es_stats->es_stats_scan_time = (scan_time + + es_stats->es_stats_scan_time*3) / 4; + else + es_stats->es_stats_scan_time = scan_time; + if (scan_time > es_stats->es_stats_max_scan_time) + es_stats->es_stats_max_scan_time = scan_time; + if (likely(es_stats->es_stats_shrunk)) + es_stats->es_stats_shrunk = (nr_shrunk + + es_stats->es_stats_shrunk*3) / 4; + else + es_stats->es_stats_shrunk = nr_shrunk; + + trace_ext4_es_shrink(sbi->s_sb, nr_shrunk, scan_time, + nr_skipped, retried); + return nr_shrunk; +} + +static unsigned long ext4_es_count(struct shrinker *shrink, + struct shrink_control *sc) +{ + unsigned long nr; + struct ext4_sb_info *sbi; + + sbi = shrink->private_data; + nr = percpu_counter_read_positive(&sbi->s_es_stats.es_stats_shk_cnt); + trace_ext4_es_shrink_count(sbi->s_sb, sc->nr_to_scan, nr); + return nr; +} + +static unsigned long ext4_es_scan(struct shrinker *shrink, + struct shrink_control *sc) +{ + struct ext4_sb_info *sbi = shrink->private_data; + int nr_to_scan = sc->nr_to_scan; + int ret, nr_shrunk; + + ret = percpu_counter_read_positive(&sbi->s_es_stats.es_stats_shk_cnt); + trace_ext4_es_shrink_scan_enter(sbi->s_sb, nr_to_scan, ret); + + nr_shrunk = __es_shrink(sbi, nr_to_scan, NULL); + + ret = percpu_counter_read_positive(&sbi->s_es_stats.es_stats_shk_cnt); + trace_ext4_es_shrink_scan_exit(sbi->s_sb, nr_shrunk, ret); + return nr_shrunk; +} + +int ext4_seq_es_shrinker_info_show(struct seq_file *seq, void *v) +{ + struct ext4_sb_info *sbi = EXT4_SB((struct super_block *) seq->private); + struct ext4_es_stats *es_stats = &sbi->s_es_stats; + struct ext4_inode_info *ei, *max = NULL; + unsigned int inode_cnt = 0; + + if (v != SEQ_START_TOKEN) + return 0; + + /* here we just find an inode that has the max nr. of objects */ + spin_lock(&sbi->s_es_lock); + list_for_each_entry(ei, &sbi->s_es_list, i_es_list) { + inode_cnt++; + if (max && max->i_es_all_nr < ei->i_es_all_nr) + max = ei; + else if (!max) + max = ei; + } + spin_unlock(&sbi->s_es_lock); + + seq_printf(seq, "stats:\n %lld objects\n %lld reclaimable objects\n", + percpu_counter_sum_positive(&es_stats->es_stats_all_cnt), + percpu_counter_sum_positive(&es_stats->es_stats_shk_cnt)); + seq_printf(seq, " %lld/%lld cache hits/misses\n", + percpu_counter_sum_positive(&es_stats->es_stats_cache_hits), + percpu_counter_sum_positive(&es_stats->es_stats_cache_misses)); + if (inode_cnt) + seq_printf(seq, " %d inodes on list\n", inode_cnt); + + seq_printf(seq, "average:\n %llu us scan time\n", + div_u64(es_stats->es_stats_scan_time, 1000)); + seq_printf(seq, " %lu shrunk objects\n", es_stats->es_stats_shrunk); + if (inode_cnt) + seq_printf(seq, + "maximum:\n %llu inode (%u objects, %u reclaimable)\n" + " %llu us max scan time\n", + max->vfs_inode.i_ino, max->i_es_all_nr, max->i_es_shk_nr, + div_u64(es_stats->es_stats_max_scan_time, 1000)); + + return 0; +} + +int ext4_es_register_shrinker(struct ext4_sb_info *sbi) +{ + int err; + + /* Make sure we have enough bits for physical block number */ + BUILD_BUG_ON(ES_SHIFT < 48); + INIT_LIST_HEAD(&sbi->s_es_list); + sbi->s_es_nr_inode = 0; + spin_lock_init(&sbi->s_es_lock); + sbi->s_es_stats.es_stats_shrunk = 0; + err = percpu_counter_init(&sbi->s_es_stats.es_stats_cache_hits, 0, + GFP_KERNEL); + if (err) + return err; + err = percpu_counter_init(&sbi->s_es_stats.es_stats_cache_misses, 0, + GFP_KERNEL); + if (err) + goto err1; + sbi->s_es_stats.es_stats_scan_time = 0; + sbi->s_es_stats.es_stats_max_scan_time = 0; + err = percpu_counter_init(&sbi->s_es_stats.es_stats_all_cnt, 0, GFP_KERNEL); + if (err) + goto err2; + err = percpu_counter_init(&sbi->s_es_stats.es_stats_shk_cnt, 0, GFP_KERNEL); + if (err) + goto err3; + + sbi->s_es_shrinker = shrinker_alloc(0, "ext4-es:%s", sbi->s_sb->s_id); + if (!sbi->s_es_shrinker) { + err = -ENOMEM; + goto err4; + } + + sbi->s_es_shrinker->scan_objects = ext4_es_scan; + sbi->s_es_shrinker->count_objects = ext4_es_count; + sbi->s_es_shrinker->private_data = sbi; + + shrinker_register(sbi->s_es_shrinker); + + return 0; +err4: + percpu_counter_destroy(&sbi->s_es_stats.es_stats_shk_cnt); +err3: + percpu_counter_destroy(&sbi->s_es_stats.es_stats_all_cnt); +err2: + percpu_counter_destroy(&sbi->s_es_stats.es_stats_cache_misses); +err1: + percpu_counter_destroy(&sbi->s_es_stats.es_stats_cache_hits); + return err; +} + +void ext4_es_unregister_shrinker(struct ext4_sb_info *sbi) +{ + percpu_counter_destroy(&sbi->s_es_stats.es_stats_cache_hits); + percpu_counter_destroy(&sbi->s_es_stats.es_stats_cache_misses); + percpu_counter_destroy(&sbi->s_es_stats.es_stats_all_cnt); + percpu_counter_destroy(&sbi->s_es_stats.es_stats_shk_cnt); + shrinker_free(sbi->s_es_shrinker); +} + +/* + * Shrink extents in given inode from ei->i_es_shrink_lblk till end. Scan at + * most *nr_to_scan extents, update *nr_to_scan accordingly. + * + * Return 0 if we hit end of tree / interval, 1 if we exhausted nr_to_scan. + * Increment *nr_shrunk by the number of reclaimed extents. Also update + * ei->i_es_shrink_lblk to where we should continue scanning. + */ +static int es_do_reclaim_extents(struct ext4_inode_info *ei, ext4_lblk_t end, + int *nr_to_scan, int *nr_shrunk) +{ + struct inode *inode = &ei->vfs_inode; + struct ext4_es_tree *tree = &ei->i_es_tree; + struct extent_status *es; + struct rb_node *node; + + es = __es_tree_search(&tree->root, ei->i_es_shrink_lblk); + if (!es) + goto out_wrap; + + while (*nr_to_scan > 0) { + if (es->es_lblk > end) { + ei->i_es_shrink_lblk = end + 1; + return 0; + } + + (*nr_to_scan)--; + node = rb_next(&es->rb_node); + + if (ext4_es_must_keep(es)) + goto next; + if (ext4_es_is_referenced(es)) { + ext4_es_clear_referenced(es); + goto next; + } + + rb_erase(&es->rb_node, &tree->root); + ext4_es_free_extent(inode, es); + (*nr_shrunk)++; +next: + if (!node) + goto out_wrap; + es = rb_entry(node, struct extent_status, rb_node); + } + ei->i_es_shrink_lblk = es->es_lblk; + return 1; +out_wrap: + ei->i_es_shrink_lblk = 0; + return 0; +} + +static int es_reclaim_extents(struct ext4_inode_info *ei, int *nr_to_scan) +{ + struct inode *inode = &ei->vfs_inode; + int nr_shrunk = 0; + ext4_lblk_t start = ei->i_es_shrink_lblk; + static DEFINE_RATELIMIT_STATE(_rs, DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + + if (ei->i_es_shk_nr == 0) + return 0; + + if (ext4_test_inode_state(inode, EXT4_STATE_EXT_PRECACHED) && + __ratelimit(&_rs)) + ext4_warning(inode->i_sb, "forced shrink of precached extents"); + + if (!es_do_reclaim_extents(ei, EXT_MAX_BLOCKS, nr_to_scan, &nr_shrunk) && + start != 0) + es_do_reclaim_extents(ei, start - 1, nr_to_scan, &nr_shrunk); + + ei->i_es_tree.cache_es = NULL; + return nr_shrunk; +} + +/* + * Called to support EXT4_IOC_CLEAR_ES_CACHE. We can only remove + * discretionary entries from the extent status cache. (Some entries + * must be present for proper operations.) + */ +void ext4_clear_inode_es(struct inode *inode) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + struct extent_status *es; + struct ext4_es_tree *tree; + struct rb_node *node; + + write_lock(&ei->i_es_lock); + tree = &EXT4_I(inode)->i_es_tree; + tree->cache_es = NULL; + node = rb_first(&tree->root); + while (node) { + es = rb_entry(node, struct extent_status, rb_node); + node = rb_next(node); + if (!ext4_es_must_keep(es)) { + rb_erase(&es->rb_node, &tree->root); + ext4_es_free_extent(inode, es); + } + } + ext4_clear_inode_state(inode, EXT4_STATE_EXT_PRECACHED); + write_unlock(&ei->i_es_lock); +} + +#ifdef ES_DEBUG__ +static void ext4_print_pending_tree(struct inode *inode) +{ + struct ext4_pending_tree *tree; + struct rb_node *node; + struct pending_reservation *pr; + + printk(KERN_DEBUG "pending reservations for inode %llu:", inode->i_ino); + tree = &EXT4_I(inode)->i_pending_tree; + node = rb_first(&tree->root); + while (node) { + pr = rb_entry(node, struct pending_reservation, rb_node); + printk(KERN_DEBUG " %u", pr->lclu); + node = rb_next(node); + } + printk(KERN_DEBUG "\n"); +} +#else +#define ext4_print_pending_tree(inode) +#endif + +int __init ext4_init_pending(void) +{ + ext4_pending_cachep = KMEM_CACHE(pending_reservation, SLAB_RECLAIM_ACCOUNT); + if (ext4_pending_cachep == NULL) + return -ENOMEM; + return 0; +} + +void ext4_exit_pending(void) +{ + kmem_cache_destroy(ext4_pending_cachep); +} + +void ext4_init_pending_tree(struct ext4_pending_tree *tree) +{ + tree->root = RB_ROOT; +} + +/* + * __get_pending - retrieve a pointer to a pending reservation + * + * @inode - file containing the pending cluster reservation + * @lclu - logical cluster of interest + * + * Returns a pointer to a pending reservation if it's a member of + * the set, and NULL if not. Must be called holding i_es_lock. + */ +static struct pending_reservation *__get_pending(struct inode *inode, + ext4_lblk_t lclu) +{ + struct ext4_pending_tree *tree; + struct rb_node *node; + struct pending_reservation *pr = NULL; + + tree = &EXT4_I(inode)->i_pending_tree; + node = (&tree->root)->rb_node; + + while (node) { + pr = rb_entry(node, struct pending_reservation, rb_node); + if (lclu < pr->lclu) + node = node->rb_left; + else if (lclu > pr->lclu) + node = node->rb_right; + else if (lclu == pr->lclu) + return pr; + } + return NULL; +} + +/* + * __insert_pending - adds a pending cluster reservation to the set of + * pending reservations + * + * @inode - file containing the cluster + * @lblk - logical block in the cluster to be added + * @prealloc - preallocated pending entry + * + * Returns 1 on successful insertion and -ENOMEM on failure. If the + * pending reservation is already in the set, returns successfully. + */ +static int __insert_pending(struct inode *inode, ext4_lblk_t lblk, + struct pending_reservation **prealloc) +{ + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + struct ext4_pending_tree *tree = &EXT4_I(inode)->i_pending_tree; + struct rb_node **p = &tree->root.rb_node; + struct rb_node *parent = NULL; + struct pending_reservation *pr; + ext4_lblk_t lclu; + int ret = 0; + + lclu = EXT4_B2C(sbi, lblk); + /* search to find parent for insertion */ + while (*p) { + parent = *p; + pr = rb_entry(parent, struct pending_reservation, rb_node); + + if (lclu < pr->lclu) { + p = &(*p)->rb_left; + } else if (lclu > pr->lclu) { + p = &(*p)->rb_right; + } else { + /* pending reservation already inserted */ + goto out; + } + } + + if (likely(*prealloc == NULL)) { + pr = __alloc_pending(false); + if (!pr) { + ret = -ENOMEM; + goto out; + } + } else { + pr = *prealloc; + *prealloc = NULL; + } + pr->lclu = lclu; + + rb_link_node(&pr->rb_node, parent, p); + rb_insert_color(&pr->rb_node, &tree->root); + ret = 1; + +out: + return ret; +} + +/* + * __remove_pending - removes a pending cluster reservation from the set + * of pending reservations + * + * @inode - file containing the cluster + * @lblk - logical block in the pending cluster reservation to be removed + * + * Returns successfully if pending reservation is not a member of the set. + */ +static void __remove_pending(struct inode *inode, ext4_lblk_t lblk) +{ + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + struct pending_reservation *pr; + struct ext4_pending_tree *tree; + + pr = __get_pending(inode, EXT4_B2C(sbi, lblk)); + if (pr != NULL) { + tree = &EXT4_I(inode)->i_pending_tree; + rb_erase(&pr->rb_node, &tree->root); + __free_pending(pr); + } +} + +/* + * ext4_remove_pending - removes a pending cluster reservation from the set + * of pending reservations + * + * @inode - file containing the cluster + * @lblk - logical block in the pending cluster reservation to be removed + * + * Locking for external use of __remove_pending. + */ +void ext4_remove_pending(struct inode *inode, ext4_lblk_t lblk) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + + write_lock(&ei->i_es_lock); + __remove_pending(inode, lblk); + write_unlock(&ei->i_es_lock); +} + +/* + * ext4_is_pending - determine whether a cluster has a pending reservation + * on it + * + * @inode - file containing the cluster + * @lblk - logical block in the cluster + * + * Returns true if there's a pending reservation for the cluster in the + * set of pending reservations, and false if not. + */ +bool ext4_is_pending(struct inode *inode, ext4_lblk_t lblk) +{ + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + struct ext4_inode_info *ei = EXT4_I(inode); + bool ret; + + read_lock(&ei->i_es_lock); + ret = (bool)(__get_pending(inode, EXT4_B2C(sbi, lblk)) != NULL); + read_unlock(&ei->i_es_lock); + + return ret; +} + +/* + * ext4_es_insert_delayed_extent - adds some delayed blocks to the extents + * status tree, adding a pending reservation + * where needed + * + * @inode - file containing the newly added block + * @lblk - start logical block to be added + * @len - length of blocks to be added + * @lclu_allocated/end_allocated - indicates whether a physical cluster has + * been allocated for the logical cluster + * that contains the start/end block. Note that + * end_allocated should always be set to false + * if the start and the end block are in the + * same cluster + */ +void ext4_es_insert_delayed_extent(struct inode *inode, ext4_lblk_t lblk, + ext4_lblk_t len, bool lclu_allocated, + bool end_allocated) +{ + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + struct extent_status newes; + ext4_lblk_t end = lblk + len - 1; + int err1 = 0, err2 = 0, err3 = 0; + struct extent_status *es1 = NULL; + struct extent_status *es2 = NULL; + struct pending_reservation *pr1 = NULL; + struct pending_reservation *pr2 = NULL; + + if (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY) + return; + + es_debug("add [%u/%u) delayed to extent status tree of inode %llu\n", + lblk, len, inode->i_ino); + if (!len) + return; + + WARN_ON_ONCE((EXT4_B2C(sbi, lblk) == EXT4_B2C(sbi, end)) && + end_allocated); + + newes.es_lblk = lblk; + newes.es_len = len; + ext4_es_store_pblock_status(&newes, ~0, EXTENT_STATUS_DELAYED); + + ext4_es_insert_extent_check(inode, &newes); + +retry: + if (err1 && !es1) + es1 = __es_alloc_extent(true); + if ((err1 || err2) && !es2) + es2 = __es_alloc_extent(true); + if (err1 || err2 || err3 < 0) { + if (lclu_allocated && !pr1) + pr1 = __alloc_pending(true); + if (end_allocated && !pr2) + pr2 = __alloc_pending(true); + } + write_lock(&EXT4_I(inode)->i_es_lock); + + err1 = __es_remove_extent(inode, lblk, end, 0, NULL, NULL, es1); + if (err1 != 0) + goto error; + /* Free preallocated extent if it didn't get used. */ + if (es1) { + if (!es1->es_len) + __es_free_extent(es1); + es1 = NULL; + } + + err2 = __es_insert_extent(inode, &newes, es2); + if (err2 != 0) + goto error; + /* Free preallocated extent if it didn't get used. */ + if (es2) { + if (!es2->es_len) + __es_free_extent(es2); + es2 = NULL; + } + + if (lclu_allocated) { + err3 = __insert_pending(inode, lblk, &pr1); + if (err3 < 0) + goto error; + if (pr1) { + __free_pending(pr1); + pr1 = NULL; + } + } + if (end_allocated) { + err3 = __insert_pending(inode, end, &pr2); + if (err3 < 0) + goto error; + if (pr2) { + __free_pending(pr2); + pr2 = NULL; + } + } + ext4_es_inc_seq(inode); +error: + write_unlock(&EXT4_I(inode)->i_es_lock); + if (err1 || err2 || err3 < 0) + goto retry; + + trace_ext4_es_insert_delayed_extent(inode, &newes, lclu_allocated, + end_allocated); + ext4_es_print_tree(inode); + ext4_print_pending_tree(inode); + return; +} + +/* + * __revise_pending - makes, cancels, or leaves unchanged pending cluster + * reservations for a specified block range depending + * upon the presence or absence of delayed blocks + * outside the range within clusters at the ends of the + * range + * + * @inode - file containing the range + * @lblk - logical block defining the start of range + * @len - length of range in blocks + * @prealloc - preallocated pending entry + * + * Used after a newly allocated extent is added to the extents status tree. + * Requires that the extents in the range have either written or unwritten + * status. Must be called while holding i_es_lock. Returns number of new + * inserts pending cluster on insert pendings, returns 0 on remove pendings, + * return -ENOMEM on failure. + */ +static int __revise_pending(struct inode *inode, ext4_lblk_t lblk, + ext4_lblk_t len, + struct pending_reservation **prealloc) +{ + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + ext4_lblk_t end = lblk + len - 1; + ext4_lblk_t first, last; + bool f_del = false, l_del = false; + int pendings = 0; + int ret = 0; + + if (len == 0) + return 0; + + /* + * Two cases - block range within single cluster and block range + * spanning two or more clusters. Note that a cluster belonging + * to a range starting and/or ending on a cluster boundary is treated + * as if it does not contain a delayed extent. The new range may + * have allocated space for previously delayed blocks out to the + * cluster boundary, requiring that any pre-existing pending + * reservation be canceled. Because this code only looks at blocks + * outside the range, it should revise pending reservations + * correctly even if the extent represented by the range can't be + * inserted in the extents status tree due to ENOSPC. + */ + + if (EXT4_B2C(sbi, lblk) == EXT4_B2C(sbi, end)) { + first = EXT4_LBLK_CMASK(sbi, lblk); + if (first != lblk) + f_del = __es_scan_range(inode, &ext4_es_is_delayed, + first, lblk - 1); + if (f_del) { + ret = __insert_pending(inode, first, prealloc); + if (ret < 0) + goto out; + pendings += ret; + } else { + last = EXT4_LBLK_CMASK(sbi, end) + + sbi->s_cluster_ratio - 1; + if (last != end) + l_del = __es_scan_range(inode, + &ext4_es_is_delayed, + end + 1, last); + if (l_del) { + ret = __insert_pending(inode, last, prealloc); + if (ret < 0) + goto out; + pendings += ret; + } else + __remove_pending(inode, last); + } + } else { + first = EXT4_LBLK_CMASK(sbi, lblk); + if (first != lblk) + f_del = __es_scan_range(inode, &ext4_es_is_delayed, + first, lblk - 1); + if (f_del) { + ret = __insert_pending(inode, first, prealloc); + if (ret < 0) + goto out; + pendings += ret; + } else + __remove_pending(inode, first); + + last = EXT4_LBLK_CMASK(sbi, end) + sbi->s_cluster_ratio - 1; + if (last != end) + l_del = __es_scan_range(inode, &ext4_es_is_delayed, + end + 1, last); + if (l_del) { + ret = __insert_pending(inode, last, prealloc); + if (ret < 0) + goto out; + pendings += ret; + } else + __remove_pending(inode, last); + } +out: + return (ret < 0) ? ret : pendings; +} diff --git a/fs/ext4/extents_status.h b/fs/ext4/extents_status.h new file mode 100644 index 000000000..f3396cf32 --- /dev/null +++ b/fs/ext4/extents_status.h @@ -0,0 +1,254 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/ext4/extents_status.h + * + * Written by Yongqiang Yang + * Modified by + * Allison Henderson + * Zheng Liu + * + */ + +#ifndef _EXT4_EXTENTS_STATUS_H +#define _EXT4_EXTENTS_STATUS_H + +/* + * Turn on ES_DEBUG__ to get lots of info about extent status operations. + */ +#ifdef ES_DEBUG__ +#define es_debug(fmt, ...) printk(fmt, ##__VA_ARGS__) +#else +#define es_debug(fmt, ...) no_printk(fmt, ##__VA_ARGS__) +#endif + +/* + * With ES_AGGRESSIVE_TEST defined, the result of es caching will be + * checked with old map_block's result. + */ +#define ES_AGGRESSIVE_TEST__ + +/* + * These flags live in the high bits of extent_status.es_pblk + */ +enum { + ES_WRITTEN_B, + ES_UNWRITTEN_B, + ES_DELAYED_B, + ES_HOLE_B, + ES_REFERENCED_B, + ES_FLAGS +}; + +#define ES_SHIFT (sizeof(ext4_fsblk_t)*8 - ES_FLAGS) +#define ES_MASK (~((ext4_fsblk_t)0) << ES_SHIFT) + +/* + * Besides EXTENT_STATUS_REFERENCED, all these extent type masks + * are exclusive, only one type can be set at a time. + */ +#define EXTENT_STATUS_WRITTEN (1 << ES_WRITTEN_B) +#define EXTENT_STATUS_UNWRITTEN (1 << ES_UNWRITTEN_B) +#define EXTENT_STATUS_DELAYED (1 << ES_DELAYED_B) +#define EXTENT_STATUS_HOLE (1 << ES_HOLE_B) +#define EXTENT_STATUS_REFERENCED (1 << ES_REFERENCED_B) + +#define ES_TYPE_MASK ((ext4_fsblk_t)(EXTENT_STATUS_WRITTEN | \ + EXTENT_STATUS_UNWRITTEN | \ + EXTENT_STATUS_DELAYED | \ + EXTENT_STATUS_HOLE)) + +#define ES_TYPE_VALID(type) ((type) && !((type) & ((type) - 1))) + +struct ext4_sb_info; +struct ext4_extent; + +struct extent_status { + struct rb_node rb_node; + ext4_lblk_t es_lblk; /* first logical block extent covers */ + ext4_lblk_t es_len; /* length of extent in block */ + ext4_fsblk_t es_pblk; /* first physical block */ +}; + +struct ext4_es_tree { + struct rb_root root; + struct extent_status *cache_es; /* recently accessed extent */ +}; + +struct ext4_es_stats { + unsigned long es_stats_shrunk; + struct percpu_counter es_stats_cache_hits; + struct percpu_counter es_stats_cache_misses; + u64 es_stats_scan_time; + u64 es_stats_max_scan_time; + struct percpu_counter es_stats_all_cnt; + struct percpu_counter es_stats_shk_cnt; +}; + +/* + * Pending cluster reservations for bigalloc file systems + * + * A cluster with a pending reservation is a logical cluster shared by at + * least one extent in the extents status tree with delayed and unwritten + * status and at least one other written or unwritten extent. The + * reservation is said to be pending because a cluster reservation would + * have to be taken in the event all blocks in the cluster shared with + * written or unwritten extents were deleted while the delayed and + * unwritten blocks remained. + * + * The set of pending cluster reservations is an auxiliary data structure + * used with the extents status tree to implement reserved cluster/block + * accounting for bigalloc file systems. The set is kept in memory and + * records all pending cluster reservations. + * + * Its primary function is to avoid the need to read extents from the + * disk when invalidating pages as a result of a truncate, punch hole, or + * collapse range operation. Page invalidation requires a decrease in the + * reserved cluster count if it results in the removal of all delayed + * and unwritten extents (blocks) from a cluster that is not shared with a + * written or unwritten extent, and no decrease otherwise. Determining + * whether the cluster is shared can be done by searching for a pending + * reservation on it. + * + * Secondarily, it provides a potentially faster method for determining + * whether the reserved cluster count should be increased when a physical + * cluster is deallocated as a result of a truncate, punch hole, or + * collapse range operation. The necessary information is also present + * in the extents status tree, but might be more rapidly accessed in + * the pending reservation set in many cases due to smaller size. + * + * The pending cluster reservation set is implemented as a red-black tree + * with the goal of minimizing per page search time overhead. + */ + +struct pending_reservation { + struct rb_node rb_node; + ext4_lblk_t lclu; +}; + +struct ext4_pending_tree { + struct rb_root root; +}; + +extern int __init ext4_init_es(void); +extern void ext4_exit_es(void); +extern void ext4_es_init_tree(struct ext4_es_tree *tree); + +extern void ext4_es_insert_extent(struct inode *inode, ext4_lblk_t lblk, + ext4_lblk_t len, ext4_fsblk_t pblk, + unsigned int status, + bool delalloc_reserve_used); +extern void ext4_es_cache_extent(struct inode *inode, ext4_lblk_t lblk, + ext4_lblk_t len, ext4_fsblk_t pblk, + unsigned int status); +extern void ext4_es_remove_extent(struct inode *inode, ext4_lblk_t lblk, + ext4_lblk_t len); +extern void ext4_es_find_extent_range(struct inode *inode, + int (*match_fn)(struct extent_status *es), + ext4_lblk_t lblk, ext4_lblk_t end, + struct extent_status *es); +extern int ext4_es_lookup_extent(struct inode *inode, ext4_lblk_t lblk, + ext4_lblk_t *next_lblk, + struct extent_status *es, u64 *pseq); +extern bool ext4_es_scan_range(struct inode *inode, + int (*matching_fn)(struct extent_status *es), + ext4_lblk_t lblk, ext4_lblk_t end); +extern bool ext4_es_scan_clu(struct inode *inode, + int (*matching_fn)(struct extent_status *es), + ext4_lblk_t lblk); + +static inline unsigned int ext4_es_status(struct extent_status *es) +{ + return es->es_pblk >> ES_SHIFT; +} + +static inline unsigned int ext4_es_type(struct extent_status *es) +{ + return (es->es_pblk >> ES_SHIFT) & ES_TYPE_MASK; +} + +static inline int ext4_es_is_written(struct extent_status *es) +{ + return (ext4_es_type(es) & EXTENT_STATUS_WRITTEN) != 0; +} + +static inline int ext4_es_is_unwritten(struct extent_status *es) +{ + return (ext4_es_type(es) & EXTENT_STATUS_UNWRITTEN) != 0; +} + +static inline int ext4_es_is_delayed(struct extent_status *es) +{ + return (ext4_es_type(es) & EXTENT_STATUS_DELAYED) != 0; +} + +static inline int ext4_es_is_hole(struct extent_status *es) +{ + return (ext4_es_type(es) & EXTENT_STATUS_HOLE) != 0; +} + +static inline int ext4_es_is_mapped(struct extent_status *es) +{ + return (ext4_es_is_written(es) || ext4_es_is_unwritten(es)); +} + +static inline void ext4_es_set_referenced(struct extent_status *es) +{ + es->es_pblk |= ((ext4_fsblk_t)EXTENT_STATUS_REFERENCED) << ES_SHIFT; +} + +static inline void ext4_es_clear_referenced(struct extent_status *es) +{ + es->es_pblk &= ~(((ext4_fsblk_t)EXTENT_STATUS_REFERENCED) << ES_SHIFT); +} + +static inline int ext4_es_is_referenced(struct extent_status *es) +{ + return (ext4_es_status(es) & EXTENT_STATUS_REFERENCED) != 0; +} + +static inline ext4_fsblk_t ext4_es_pblock(struct extent_status *es) +{ + return es->es_pblk & ~ES_MASK; +} + +static inline ext4_fsblk_t ext4_es_show_pblock(struct extent_status *es) +{ + ext4_fsblk_t pblock = ext4_es_pblock(es); + return pblock == ~ES_MASK ? 0 : pblock; +} + +static inline void ext4_es_store_pblock(struct extent_status *es, + ext4_fsblk_t pb) +{ + ext4_fsblk_t block; + + block = (pb & ~ES_MASK) | (es->es_pblk & ES_MASK); + es->es_pblk = block; +} + +static inline void ext4_es_store_pblock_status(struct extent_status *es, + ext4_fsblk_t pb, + unsigned int status) +{ + WARN_ON_ONCE(!ES_TYPE_VALID(status & ES_TYPE_MASK)); + + es->es_pblk = (((ext4_fsblk_t)status << ES_SHIFT) & ES_MASK) | + (pb & ~ES_MASK); +} + +extern int ext4_es_register_shrinker(struct ext4_sb_info *sbi); +extern void ext4_es_unregister_shrinker(struct ext4_sb_info *sbi); + +extern int ext4_seq_es_shrinker_info_show(struct seq_file *seq, void *v); + +extern int __init ext4_init_pending(void); +extern void ext4_exit_pending(void); +extern void ext4_init_pending_tree(struct ext4_pending_tree *tree); +extern void ext4_remove_pending(struct inode *inode, ext4_lblk_t lblk); +extern bool ext4_is_pending(struct inode *inode, ext4_lblk_t lblk); +extern void ext4_es_insert_delayed_extent(struct inode *inode, ext4_lblk_t lblk, + ext4_lblk_t len, bool lclu_allocated, + bool end_allocated); +extern void ext4_clear_inode_es(struct inode *inode); + +#endif /* _EXT4_EXTENTS_STATUS_H */ diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c new file mode 100644 index 000000000..062103e42 --- /dev/null +++ b/fs/ext4/fast_commit.c @@ -0,0 +1,2766 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * fs/ext4/fast_commit.c + * + * Written by Harshad Shirwadkar + * + * Ext4 fast commits routines. + */ +#include "ext4.h" +#include "ext4_jbd2.h" +#include "ext4_extents.h" +#include "mballoc.h" + +#include +#include +/* + * Ext4 Fast Commits + * ----------------- + * + * Ext4 fast commits implement fine grained journalling for Ext4. + * + * Fast commits are organized as a log of tag-length-value (TLV) structs. (See + * struct ext4_fc_tl). Each TLV contains some delta that is replayed TLV by + * TLV during the recovery phase. For the scenarios for which we currently + * don't have replay code, fast commit falls back to full commits. + * Fast commits record delta in one of the following three categories. + * + * (A) Directory entry updates: + * + * - EXT4_FC_TAG_UNLINK - records directory entry unlink + * - EXT4_FC_TAG_LINK - records directory entry link + * - EXT4_FC_TAG_CREAT - records inode and directory entry creation + * + * (B) File specific data range updates: + * + * - EXT4_FC_TAG_ADD_RANGE - records addition of new blocks to an inode + * - EXT4_FC_TAG_DEL_RANGE - records deletion of blocks from an inode + * + * (C) Inode metadata (mtime / ctime etc): + * + * - EXT4_FC_TAG_INODE - record the inode that should be replayed + * during recovery. Note that iblocks field is + * not replayed and instead derived during + * replay. + * Commit Operation + * ---------------- + * With fast commits, we maintain all the directory entry operations in the + * order in which they are issued in an in-memory queue. This queue is flushed + * to disk during the commit operation. We also maintain a list of inodes + * that need to be committed during a fast commit in another in memory queue of + * inodes. During the commit operation, we commit in the following order: + * + * [1] Prepare all the inodes to write out their data by setting + * "EXT4_STATE_FC_FLUSHING_DATA". This ensures that inode cannot be + * deleted while it is being flushed. + * [2] Flush data buffers to disk and clear "EXT4_STATE_FC_FLUSHING_DATA" + * state. + * [3] Lock the journal by calling jbd2_journal_lock_updates(). This ensures + * that all the existing handles finish and no new handles can start. + * [4] Mark all the fast commit eligible inodes as undergoing fast commit by + * setting "EXT4_STATE_FC_COMMITTING" state, and snapshot the inode state + * needed for log writing. + * [5] Unlock the journal by calling jbd2_journal_unlock_updates(). This allows + * starting of new handles. Updates to inodes being fast committed are + * tracked for requeue rather than blocking. + * [6] Commit all the directory entry updates in the fast commit space. + * [7] Commit all the changed inodes in the fast commit space. + * [8] Write tail tag (this tag ensures the atomicity, please read the following + * section for more details). + * [9] Clear "EXT4_STATE_FC_COMMITTING" and wake up waiters in + * ext4_fc_cleanup(). + * + * All the inode updates must be enclosed within jbd2_journal_start() + * and jbd2_journal_stop() similar to JBD2 journaling. + * + * Fast Commit Ineligibility + * ------------------------- + * + * Not all operations are supported by fast commits today (e.g extended + * attributes). Fast commit ineligibility is marked by calling + * ext4_fc_mark_ineligible(): This makes next fast commit operation to fall back + * to full commit. + * + * Atomicity of commits + * -------------------- + * In order to guarantee atomicity during the commit operation, fast commit + * uses "EXT4_FC_TAG_TAIL" tag that marks a fast commit as complete. Tail + * tag contains CRC of the contents and TID of the transaction after which + * this fast commit should be applied. Recovery code replays fast commit + * logs only if there's at least 1 valid tail present. For every fast commit + * operation, there is 1 tail. This means, we may end up with multiple tails + * in the fast commit space. Here's an example: + * + * - Create a new file A and remove existing file B + * - fsync() + * - Append contents to file A + * - Truncate file A + * - fsync() + * + * The fast commit space at the end of above operations would look like this: + * [HEAD] [CREAT A] [UNLINK B] [TAIL] [ADD_RANGE A] [DEL_RANGE A] [TAIL] + * |<--- Fast Commit 1 --->|<--- Fast Commit 2 ---->| + * + * Replay code should thus check for all the valid tails in the FC area. + * + * Fast Commit Replay Idempotence + * ------------------------------ + * + * Fast commits tags are idempotent in nature provided the recovery code follows + * certain rules. The guiding principle that the commit path follows while + * committing is that it stores the result of a particular operation instead of + * storing the procedure. + * + * Let's consider this rename operation: 'mv /a /b'. Let's assume dirent '/a' + * was associated with inode 10. During fast commit, instead of storing this + * operation as a procedure "rename a to b", we store the resulting file system + * state as a "series" of outcomes: + * + * - Link dirent b to inode 10 + * - Unlink dirent a + * - Inode <10> with valid refcount + * + * Now when recovery code runs, it needs "enforce" this state on the file + * system. This is what guarantees idempotence of fast commit replay. + * + * Let's take an example of a procedure that is not idempotent and see how fast + * commits make it idempotent. Consider following sequence of operations: + * + * rm A; mv B A; read A + * (x) (y) (z) + * + * (x), (y) and (z) are the points at which we can crash. If we store this + * sequence of operations as is then the replay is not idempotent. Let's say + * while in replay, we crash at (z). During the second replay, file A (which was + * actually created as a result of "mv B A" operation) would get deleted. Thus, + * file named A would be absent when we try to read A. So, this sequence of + * operations is not idempotent. However, as mentioned above, instead of storing + * the procedure fast commits store the outcome of each procedure. Thus the fast + * commit log for above procedure would be as follows: + * + * (Let's assume dirent A was linked to inode 10 and dirent B was linked to + * inode 11 before the replay) + * + * [Unlink A] [Link A to inode 11] [Unlink B] [Inode 11] + * (w) (x) (y) (z) + * + * If we crash at (z), we will have file A linked to inode 11. During the second + * replay, we will remove file A (inode 11). But we will create it back and make + * it point to inode 11. We won't find B, so we'll just skip that step. At this + * point, the refcount for inode 11 is not reliable, but that gets fixed by the + * replay of last inode 11 tag. Crashes at points (w), (x) and (y) get handled + * similarly. Thus, by converting a non-idempotent procedure into a series of + * idempotent outcomes, fast commits ensured idempotence during the replay. + * + * Locking + * ------- + * sbi->s_fc_lock protects the fast commit inodes queue and the fast commit + * dentry queue. ei->i_fc_lock protects the fast commit related info in a given + * inode. Most of the code avoids acquiring both the locks, but if one must do + * that then sbi->s_fc_lock must be acquired before ei->i_fc_lock. + * + * TODOs + * ----- + * + * 0) Fast commit replay path hardening: Fast commit replay code should use + * journal handles to make sure all the updates it does during the replay + * path are atomic. With that if we crash during fast commit replay, after + * trying to do recovery again, we will find a file system where fast commit + * area is invalid (because new full commit would be found). In order to deal + * with that, fast commit replay code should ensure that the "FC_REPLAY" + * superblock state is persisted before starting the replay, so that after + * the crash, fast commit recovery code can look at that flag and perform + * fast commit recovery even if that area is invalidated by later full + * commits. + * + * 1) Handle more ineligible cases. + * + * 2) Change ext4_fc_commit() to lookup logical to physical mapping using extent + * status tree. This would get rid of the need to call ext4_fc_track_inode() + * before acquiring i_data_sem. To do that we would need to ensure that + * modified extents from the extent status tree are not evicted from memory. + */ + +#include +static struct kmem_cache *ext4_fc_dentry_cachep; +static struct kmem_cache *ext4_fc_range_cachep; + +/* + * Avoid spending unbounded time/memory snapshotting highly fragmented files + * under jbd2_journal_lock_updates(). If we exceed this limit, fall back to + * full commit. + */ +#define EXT4_FC_SNAPSHOT_MAX_INODES 1024 +#define EXT4_FC_SNAPSHOT_MAX_RANGES 2048 + +static inline void ext4_fc_set_snap_err(int *snap_err, int err) +{ + if (snap_err && *snap_err == EXT4_FC_SNAP_ERR_NONE) + *snap_err = err; +} + +static void ext4_fc_free_inode_snap(struct inode *inode); + +static inline void ext4_fc_reset_inode(struct inode *inode) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + + ei->i_fc_lblk_start = 0; + ei->i_fc_lblk_len = 0; +} + +void ext4_fc_init_inode(struct inode *inode) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + + ext4_fc_reset_inode(inode); + ext4_clear_inode_state(inode, EXT4_STATE_FC_COMMITTING); + ext4_clear_inode_state(inode, EXT4_STATE_FC_REQUEUE); + INIT_LIST_HEAD(&ei->i_fc_list); + INIT_LIST_HEAD(&ei->i_fc_dilist); + ei->i_fc_snap = NULL; +} + +static bool ext4_fc_disabled(struct super_block *sb) +{ + return (!test_opt2(sb, JOURNAL_FAST_COMMIT) || + (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY)); +} + +static bool ext4_fc_eligible(struct super_block *sb) +{ + return !ext4_fc_disabled(sb) && + !(ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE)); +} + +/* + * Wait for an inode fast-commit state bit to clear while dropping the + * fast-commit lock around schedule(). + */ +static void ext4_fc_wait_inode_state(struct inode *inode, int bit, + int *alloc_ctx) +{ + wait_queue_head_t *wq; + unsigned long *wait_word = ext4_inode_state_wait_word(inode); + int wait_bit = ext4_inode_state_wait_bit(bit); + + while (ext4_test_inode_state(inode, bit)) { + DEFINE_WAIT_BIT(wait, wait_word, wait_bit); + + wq = bit_waitqueue(wait_word, wait_bit); + prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE); + if (ext4_test_inode_state(inode, bit)) { + ext4_fc_unlock(inode->i_sb, *alloc_ctx); + schedule(); + *alloc_ctx = ext4_fc_lock(inode->i_sb); + } + finish_wait(wq, &wait.wq_entry); + } +} + +static inline void ext4_fc_wake_inode_state(struct inode *inode, int bit) +{ + wake_up_bit(ext4_inode_state_wait_word(inode), + ext4_inode_state_wait_bit(bit)); +} + +static void ext4_fc_snap_stats_update_max(atomic64_t *stat, u64 value) +{ + u64 old = atomic64_read(stat); + + while (value > old) { + u64 prev = atomic64_cmpxchg(stat, old, value); + + if (prev == old) + break; + old = prev; + } +} + +/* + * Remove inode from fast commit list. If the inode is being committed + * we wait until inode commit is done. + */ +void ext4_fc_del(struct inode *inode) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + struct ext4_fc_dentry_update *fc_dentry; + int alloc_ctx; + + if (ext4_fc_disabled(inode->i_sb)) + return; + + alloc_ctx = ext4_fc_lock(inode->i_sb); + if (list_empty(&ei->i_fc_list) && list_empty(&ei->i_fc_dilist)) { + ext4_fc_free_inode_snap(inode); + ext4_fc_unlock(inode->i_sb, alloc_ctx); + return; + } + + /* + * Wait for ongoing fast commit to finish. We cannot remove the inode + * from fast commit lists while it is being committed. If we wake from + * FC_FLUSHING_DATA, re-check FC_COMMITTING before deleting because the + * commit thread sets FC_COMMITTING only after clearing FLUSHING_DATA. + */ + for (;;) { + ext4_fc_wait_inode_state(inode, EXT4_STATE_FC_COMMITTING, + &alloc_ctx); + + if (!ext4_test_inode_state(inode, EXT4_STATE_FC_FLUSHING_DATA)) + break; + + ext4_fc_wait_inode_state(inode, EXT4_STATE_FC_FLUSHING_DATA, + &alloc_ctx); + } + + ext4_fc_free_inode_snap(inode); + list_del_init(&ei->i_fc_list); + + /* + * Since this inode is getting removed, let's also remove all FC dentry + * create references, since it is not needed to log it anyways. + */ + if (list_empty(&ei->i_fc_dilist)) { + ext4_fc_unlock(inode->i_sb, alloc_ctx); + return; + } + + fc_dentry = list_first_entry(&ei->i_fc_dilist, + struct ext4_fc_dentry_update, + fcd_dilist); + WARN_ON(fc_dentry->fcd_op != EXT4_FC_TAG_CREAT); + list_del_init(&fc_dentry->fcd_list); + list_del_init(&fc_dentry->fcd_dilist); + + WARN_ON(!list_empty(&ei->i_fc_dilist)); + ext4_fc_unlock(inode->i_sb, alloc_ctx); + + release_dentry_name_snapshot(&fc_dentry->fcd_name); + kmem_cache_free(ext4_fc_dentry_cachep, fc_dentry); +} + +/* + * Mark file system as fast commit ineligible, and record latest + * ineligible transaction tid. This means until the recorded + * transaction, commit operation would result in a full jbd2 commit. + */ +void ext4_fc_mark_ineligible(struct super_block *sb, int reason, handle_t *handle) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + tid_t tid; + bool has_transaction = true; + bool is_ineligible; + int alloc_ctx; + + if (ext4_fc_disabled(sb)) + return; + + if (!IS_ERR_OR_NULL(handle)) + tid = handle->h_transaction->t_tid; + else { + read_lock(&sbi->s_journal->j_state_lock); + if (sbi->s_journal->j_running_transaction) + tid = sbi->s_journal->j_running_transaction->t_tid; + else + has_transaction = false; + read_unlock(&sbi->s_journal->j_state_lock); + } + alloc_ctx = ext4_fc_lock(sb); + is_ineligible = ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE); + if (has_transaction && (!is_ineligible || tid_gt(tid, sbi->s_fc_ineligible_tid))) + sbi->s_fc_ineligible_tid = tid; + ext4_set_mount_flag(sb, EXT4_MF_FC_INELIGIBLE); + ext4_fc_unlock(sb, alloc_ctx); + WARN_ON(reason >= EXT4_FC_REASON_MAX); + sbi->s_fc_stats.fc_ineligible_reason_count[reason]++; +} + +/* + * Generic fast commit tracking function. If this is the first time this we are + * called after a full commit, we initialize fast commit fields and then call + * __fc_track_fn() with update = 0. If we have already been called after a full + * commit, we pass update = 1. Based on that, the track function can determine + * if it needs to track a field for the first time or if it needs to just + * update the previously tracked value. + * + * If enqueue is set, this function enqueues the inode in fast commit list. + */ +static int ext4_fc_track_template( + handle_t *handle, struct inode *inode, + int (*__fc_track_fn)(handle_t *handle, struct inode *, void *, bool), + void *args, int enqueue) +{ + bool update = false; + struct ext4_inode_info *ei = EXT4_I(inode); + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + tid_t tid = 0; + int alloc_ctx; + int ret; + + tid = handle->h_transaction->t_tid; + spin_lock(&ei->i_fc_lock); + if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) + ext4_set_inode_state(inode, EXT4_STATE_FC_REQUEUE); + if (tid == ei->i_sync_tid) { + update = true; + } else { + ext4_fc_reset_inode(inode); + ei->i_sync_tid = tid; + } + ret = __fc_track_fn(handle, inode, args, update); + spin_unlock(&ei->i_fc_lock); + if (!enqueue) + return ret; + + alloc_ctx = ext4_fc_lock(inode->i_sb); + if (list_empty(&EXT4_I(inode)->i_fc_list)) + list_add_tail(&EXT4_I(inode)->i_fc_list, + (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING || + sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING) ? + &sbi->s_fc_q[FC_Q_STAGING] : + &sbi->s_fc_q[FC_Q_MAIN]); + ext4_fc_unlock(inode->i_sb, alloc_ctx); + + return ret; +} + +struct __track_dentry_update_args { + struct dentry *dentry; + int op; +}; + +/* __track_fn for directory entry updates. Called with ei->i_fc_lock. */ +static int __track_dentry_update(handle_t *handle, struct inode *inode, + void *arg, bool update) +{ + struct ext4_fc_dentry_update *node; + struct ext4_inode_info *ei = EXT4_I(inode); + struct __track_dentry_update_args *dentry_update = + (struct __track_dentry_update_args *)arg; + struct dentry *dentry = dentry_update->dentry; + struct inode *dir = dentry->d_parent->d_inode; + struct super_block *sb = inode->i_sb; + struct ext4_sb_info *sbi = EXT4_SB(sb); + int alloc_ctx; + + spin_unlock(&ei->i_fc_lock); + + if (IS_ENCRYPTED(dir)) { + ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_ENCRYPTED_FILENAME, + handle); + spin_lock(&ei->i_fc_lock); + return -EOPNOTSUPP; + } + + node = kmem_cache_alloc(ext4_fc_dentry_cachep, GFP_NOFS); + if (!node) { + ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_NOMEM, handle); + spin_lock(&ei->i_fc_lock); + return -ENOMEM; + } + + node->fcd_op = dentry_update->op; + node->fcd_parent = dir->i_ino; + node->fcd_ino = inode->i_ino; + take_dentry_name_snapshot(&node->fcd_name, dentry); + INIT_LIST_HEAD(&node->fcd_dilist); + INIT_LIST_HEAD(&node->fcd_list); + alloc_ctx = ext4_fc_lock(sb); + if (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING || + sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING) + list_add_tail(&node->fcd_list, + &sbi->s_fc_dentry_q[FC_Q_STAGING]); + else + list_add_tail(&node->fcd_list, &sbi->s_fc_dentry_q[FC_Q_MAIN]); + + /* + * This helps us keep a track of all fc_dentry updates which is part of + * this ext4 inode. So in case the inode is getting unlinked, before + * even we get a chance to fsync, we could remove all fc_dentry + * references while evicting the inode in ext4_fc_del(). + * Also with this, we don't need to loop over all the inodes in + * sbi->s_fc_q to get the corresponding inode in + * ext4_fc_commit_dentry_updates(). + */ + if (dentry_update->op == EXT4_FC_TAG_CREAT) { + WARN_ON(!list_empty(&ei->i_fc_dilist)); + list_add_tail(&node->fcd_dilist, &ei->i_fc_dilist); + } + ext4_fc_unlock(sb, alloc_ctx); + spin_lock(&ei->i_fc_lock); + + return 0; +} + +void __ext4_fc_track_unlink(handle_t *handle, + struct inode *inode, struct dentry *dentry) +{ + struct __track_dentry_update_args args; + int ret; + + args.dentry = dentry; + args.op = EXT4_FC_TAG_UNLINK; + + ret = ext4_fc_track_template(handle, inode, __track_dentry_update, + (void *)&args, 0); + trace_ext4_fc_track_unlink(handle, inode, dentry, ret); +} + +void ext4_fc_track_unlink(handle_t *handle, struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + + if (ext4_fc_eligible(inode->i_sb)) + __ext4_fc_track_unlink(handle, inode, dentry); +} + +void __ext4_fc_track_link(handle_t *handle, + struct inode *inode, struct dentry *dentry) +{ + struct __track_dentry_update_args args; + int ret; + + args.dentry = dentry; + args.op = EXT4_FC_TAG_LINK; + + ret = ext4_fc_track_template(handle, inode, __track_dentry_update, + (void *)&args, 0); + trace_ext4_fc_track_link(handle, inode, dentry, ret); +} + +void ext4_fc_track_link(handle_t *handle, struct inode *inode, + struct dentry *dentry) +{ + if (ext4_fc_eligible(inode->i_sb)) + __ext4_fc_track_link(handle, inode, dentry); +} + +void __ext4_fc_track_create(handle_t *handle, struct inode *inode, + struct dentry *dentry) +{ + struct __track_dentry_update_args args; + int ret; + + args.dentry = dentry; + args.op = EXT4_FC_TAG_CREAT; + + ret = ext4_fc_track_template(handle, inode, __track_dentry_update, + (void *)&args, 0); + trace_ext4_fc_track_create(handle, inode, dentry, ret); +} + +void ext4_fc_track_create(handle_t *handle, struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + + if (ext4_fc_eligible(inode->i_sb)) + __ext4_fc_track_create(handle, inode, dentry); +} + +/* __track_fn for inode tracking */ +static int __track_inode(handle_t *handle, struct inode *inode, void *arg, + bool update) +{ + if (update) + return -EEXIST; + + EXT4_I(inode)->i_fc_lblk_len = 0; + + return 0; +} + +void ext4_fc_track_inode(handle_t *handle, struct inode *inode) +{ + int ret; + + if (S_ISDIR(inode->i_mode)) + return; + + if (ext4_should_journal_data(inode)) { + ext4_fc_mark_ineligible(inode->i_sb, + EXT4_FC_REASON_INODE_JOURNAL_DATA, handle); + return; + } + + if (!ext4_fc_eligible(inode->i_sb)) + return; + + /* + * Fast commit snapshots inode state at commit time, so there's no need + * to wait for EXT4_STATE_FC_COMMITTING here. If the inode is already + * on the commit queue, ext4_fc_cleanup() will requeue it for the new + * transaction once the current commit finishes. + */ + + /* + * From this point on, this inode will not be committed either + * by fast or full commit as long as the handle is open. + */ + ret = ext4_fc_track_template(handle, inode, __track_inode, NULL, 1); + trace_ext4_fc_track_inode(handle, inode, ret); +} + +struct __track_range_args { + ext4_lblk_t start, end; +}; + +/* __track_fn for tracking data updates */ +static int __track_range(handle_t *handle, struct inode *inode, void *arg, + bool update) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + ext4_lblk_t oldstart; + struct __track_range_args *__arg = + (struct __track_range_args *)arg; + + if (inode->i_ino < EXT4_FIRST_INO(inode->i_sb)) { + ext4_debug("Special inode %llu being modified\n", inode->i_ino); + return -ECANCELED; + } + + oldstart = ei->i_fc_lblk_start; + + if (update && ei->i_fc_lblk_len > 0) { + ei->i_fc_lblk_start = min(ei->i_fc_lblk_start, __arg->start); + ei->i_fc_lblk_len = + max(oldstart + ei->i_fc_lblk_len - 1, __arg->end) - + ei->i_fc_lblk_start + 1; + } else { + ei->i_fc_lblk_start = __arg->start; + ei->i_fc_lblk_len = __arg->end - __arg->start + 1; + } + + return 0; +} + +void ext4_fc_track_range(handle_t *handle, struct inode *inode, ext4_lblk_t start, + ext4_lblk_t end) +{ + struct __track_range_args args; + int ret; + + if (S_ISDIR(inode->i_mode)) + return; + + if (!ext4_fc_eligible(inode->i_sb)) + return; + + if (ext4_has_inline_data(inode)) { + ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_XATTR, + handle); + return; + } + + args.start = start; + args.end = end; + + ret = ext4_fc_track_template(handle, inode, __track_range, &args, 1); + + trace_ext4_fc_track_range(handle, inode, start, end, ret); +} + +static void ext4_fc_submit_bh(struct super_block *sb, bool is_tail) +{ + blk_opf_t write_flags = JBD2_JOURNAL_REQ_FLAGS; + struct buffer_head *bh = EXT4_SB(sb)->s_fc_bh; + + /* Add REQ_FUA | REQ_PREFLUSH only its tail */ + if (test_opt(sb, BARRIER) && is_tail) + write_flags |= REQ_FUA | REQ_PREFLUSH; + lock_buffer(bh); + set_buffer_dirty(bh); + set_buffer_uptodate(bh); + bh_submit(bh, REQ_OP_WRITE | write_flags, bh_end_write); + EXT4_SB(sb)->s_fc_bh = NULL; +} + +/* Ext4 commit path routines */ + +/* + * Allocate len bytes on a fast commit buffer. + * + * During the commit time this function is used to manage fast commit + * block space. We don't split a fast commit log onto different + * blocks. So this function makes sure that if there's not enough space + * on the current block, the remaining space in the current block is + * marked as unused by adding EXT4_FC_TAG_PAD tag. In that case, + * new block is from jbd2 and CRC is updated to reflect the padding + * we added. + */ +static u8 *ext4_fc_reserve_space(struct super_block *sb, int len, u32 *crc) +{ + struct ext4_fc_tl tl; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct buffer_head *bh; + int bsize = sbi->s_journal->j_blocksize; + int ret, off = sbi->s_fc_bytes % bsize; + int remaining; + u8 *dst; + + /* + * If 'len' is too long to fit in any block alongside a PAD tlv, then we + * cannot fulfill the request. + */ + if (len > bsize - EXT4_FC_TAG_BASE_LEN) + return NULL; + + if (!sbi->s_fc_bh) { + ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh); + if (ret) + return NULL; + sbi->s_fc_bh = bh; + } + dst = sbi->s_fc_bh->b_data + off; + + /* + * Allocate the bytes in the current block if we can do so while still + * leaving enough space for a PAD tlv. + */ + remaining = bsize - EXT4_FC_TAG_BASE_LEN - off; + if (len <= remaining) { + sbi->s_fc_bytes += len; + return dst; + } + + /* + * Else, terminate the current block with a PAD tlv, then allocate a new + * block and allocate the bytes at the start of that new block. + */ + + tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_PAD); + tl.fc_len = cpu_to_le16(remaining); + memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN); + memset(dst + EXT4_FC_TAG_BASE_LEN, 0, remaining); + *crc = ext4_chksum(*crc, sbi->s_fc_bh->b_data, bsize); + + ext4_fc_submit_bh(sb, false); + + ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh); + if (ret) + return NULL; + sbi->s_fc_bh = bh; + sbi->s_fc_bytes += bsize - off + len; + return sbi->s_fc_bh->b_data; +} + +/* + * Complete a fast commit by writing tail tag. + * + * Writing tail tag marks the end of a fast commit. In order to guarantee + * atomicity, after writing tail tag, even if there's space remaining + * in the block, next commit shouldn't use it. That's why tail tag + * has the length as that of the remaining space on the block. + */ +static int ext4_fc_write_tail(struct super_block *sb, u32 crc) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_fc_tl tl; + struct ext4_fc_tail tail; + int off, bsize = sbi->s_journal->j_blocksize; + u8 *dst; + + /* + * ext4_fc_reserve_space takes care of allocating an extra block if + * there's no enough space on this block for accommodating this tail. + */ + dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + sizeof(tail), &crc); + if (!dst) + return -ENOSPC; + + off = sbi->s_fc_bytes % bsize; + + tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_TAIL); + tl.fc_len = cpu_to_le16(bsize - off + sizeof(struct ext4_fc_tail)); + sbi->s_fc_bytes = round_up(sbi->s_fc_bytes, bsize); + + memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN); + dst += EXT4_FC_TAG_BASE_LEN; + tail.fc_tid = cpu_to_le32(sbi->s_journal->j_running_transaction->t_tid); + memcpy(dst, &tail.fc_tid, sizeof(tail.fc_tid)); + dst += sizeof(tail.fc_tid); + crc = ext4_chksum(crc, sbi->s_fc_bh->b_data, + dst - (u8 *)sbi->s_fc_bh->b_data); + tail.fc_crc = cpu_to_le32(crc); + memcpy(dst, &tail.fc_crc, sizeof(tail.fc_crc)); + dst += sizeof(tail.fc_crc); + memset(dst, 0, bsize - off); /* Don't leak uninitialized memory. */ + + ext4_fc_submit_bh(sb, true); + + return 0; +} + +/* + * Adds tag, length, value and updates CRC. Returns true if tlv was added. + * Returns false if there's not enough space. + */ +static bool ext4_fc_add_tlv(struct super_block *sb, u16 tag, u16 len, u8 *val, + u32 *crc) +{ + struct ext4_fc_tl tl; + u8 *dst; + + dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + len, crc); + if (!dst) + return false; + + tl.fc_tag = cpu_to_le16(tag); + tl.fc_len = cpu_to_le16(len); + + memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN); + memcpy(dst + EXT4_FC_TAG_BASE_LEN, val, len); + + return true; +} + +/* Same as above, but adds dentry tlv. */ +static bool ext4_fc_add_dentry_tlv(struct super_block *sb, u32 *crc, + struct ext4_fc_dentry_update *fc_dentry) +{ + struct ext4_fc_dentry_info fcd; + struct ext4_fc_tl tl; + int dlen = fc_dentry->fcd_name.name.len; + u8 *dst = ext4_fc_reserve_space(sb, + EXT4_FC_TAG_BASE_LEN + sizeof(fcd) + dlen, crc); + + if (!dst) + return false; + + fcd.fc_parent_ino = cpu_to_le32(fc_dentry->fcd_parent); + fcd.fc_ino = cpu_to_le32(fc_dentry->fcd_ino); + tl.fc_tag = cpu_to_le16(fc_dentry->fcd_op); + tl.fc_len = cpu_to_le16(sizeof(fcd) + dlen); + memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN); + dst += EXT4_FC_TAG_BASE_LEN; + memcpy(dst, &fcd, sizeof(fcd)); + dst += sizeof(fcd); + memcpy(dst, fc_dentry->fcd_name.name.name, dlen); + + return true; +} + +struct ext4_fc_range { + struct list_head list; + u16 tag; + ext4_lblk_t lblk; + ext4_lblk_t len; + ext4_fsblk_t pblk; + bool unwritten; +}; + +struct ext4_fc_inode_snap { + struct list_head data_list; + unsigned int inode_len; + u8 inode_buf[]; +}; + +/* + * Writes inode in the fast commit space under TLV with tag @tag. + * Returns 0 on success, error on failure. + */ +static int ext4_fc_write_inode(struct inode *inode, u32 *crc) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + struct ext4_fc_inode_snap *snap = ei->i_fc_snap; + struct ext4_fc_snap_stats *stats = + &EXT4_SB(inode->i_sb)->s_fc_snap_stats; + struct ext4_fc_inode fc_inode; + struct ext4_fc_tl tl; + u8 *dst; + u8 *src; + int inode_len; + int ret; + + if (!snap) { + atomic64_inc(&stats->snap_fail_no_snap); + return -ECANCELED; + } + + src = snap->inode_buf; + inode_len = snap->inode_len; + if (!src || inode_len == 0) { + atomic64_inc(&stats->snap_fail_no_snap); + return -ECANCELED; + } + + fc_inode.fc_ino = cpu_to_le32(inode->i_ino); + tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_INODE); + tl.fc_len = cpu_to_le16(inode_len + sizeof(fc_inode.fc_ino)); + + ret = -ECANCELED; + dst = ext4_fc_reserve_space(inode->i_sb, + EXT4_FC_TAG_BASE_LEN + inode_len + sizeof(fc_inode.fc_ino), crc); + if (!dst) + goto err; + + memcpy(dst, &tl, EXT4_FC_TAG_BASE_LEN); + dst += EXT4_FC_TAG_BASE_LEN; + memcpy(dst, &fc_inode, sizeof(fc_inode)); + dst += sizeof(fc_inode); + memcpy(dst, src, inode_len); + ret = 0; +err: + return ret; +} + +/* + * Writes updated data ranges for the inode in question. Updates CRC. + * Returns 0 on success, error otherwise. + */ +static int ext4_fc_write_inode_data(struct inode *inode, u32 *crc) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + struct ext4_fc_inode_snap *snap = ei->i_fc_snap; + struct ext4_fc_snap_stats *stats = + &EXT4_SB(inode->i_sb)->s_fc_snap_stats; + struct ext4_fc_add_range fc_ext; + struct ext4_fc_del_range lrange; + struct ext4_extent *ex; + struct ext4_fc_range *range; + + if (!snap) { + atomic64_inc(&stats->snap_fail_no_snap); + return -ECANCELED; + } + + list_for_each_entry(range, &snap->data_list, list) { + if (range->tag == EXT4_FC_TAG_DEL_RANGE) { + lrange.fc_ino = cpu_to_le32(inode->i_ino); + lrange.fc_lblk = cpu_to_le32(range->lblk); + lrange.fc_len = cpu_to_le32(range->len); + if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_DEL_RANGE, + sizeof(lrange), (u8 *)&lrange, crc)) + return -ENOSPC; + continue; + } + + fc_ext.fc_ino = cpu_to_le32(inode->i_ino); + ex = (struct ext4_extent *)&fc_ext.fc_ex; + ex->ee_block = cpu_to_le32(range->lblk); + ex->ee_len = cpu_to_le16(range->len); + ext4_ext_store_pblock(ex, range->pblk); + if (range->unwritten) + ext4_ext_mark_unwritten(ex); + else + ext4_ext_mark_initialized(ex); + + if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_ADD_RANGE, + sizeof(fc_ext), (u8 *)&fc_ext, crc)) + return -ENOSPC; + } + + return 0; +} + +static void ext4_fc_free_ranges(struct list_head *head) +{ + struct ext4_fc_range *range, *range_n; + + list_for_each_entry_safe(range, range_n, head, list) { + list_del(&range->list); + kmem_cache_free(ext4_fc_range_cachep, range); + } +} + +static void ext4_fc_free_inode_snap(struct inode *inode) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + struct ext4_fc_inode_snap *snap = ei->i_fc_snap; + + if (!snap) + return; + + ext4_fc_free_ranges(&snap->data_list); + kfree(snap); + ei->i_fc_snap = NULL; +} + +static int ext4_fc_snapshot_inode_data(struct inode *inode, + struct list_head *ranges, + unsigned int nr_ranges_total, + unsigned int *nr_rangesp, + int *snap_err) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + struct ext4_fc_snap_stats *stats = + &EXT4_SB(inode->i_sb)->s_fc_snap_stats; + ext4_lblk_t start_lblk, end_lblk, cur_lblk; + unsigned int nr_ranges = 0; + + spin_lock(&ei->i_fc_lock); + if (ei->i_fc_lblk_len == 0) { + spin_unlock(&ei->i_fc_lock); + if (nr_rangesp) + *nr_rangesp = 0; + return 0; + } + start_lblk = ei->i_fc_lblk_start; + end_lblk = ei->i_fc_lblk_start + ei->i_fc_lblk_len - 1; + ei->i_fc_lblk_len = 0; + spin_unlock(&ei->i_fc_lock); + + cur_lblk = start_lblk; + ext4_debug("snapshot data ranges %u-%u for inode %llu\n", + start_lblk, end_lblk, + (unsigned long long)inode->i_ino); + + while (cur_lblk <= end_lblk) { + struct extent_status es; + struct ext4_fc_range *range; + ext4_lblk_t len; + u64 remaining = (u64)end_lblk - cur_lblk + 1; + + if (!ext4_es_lookup_extent(inode, cur_lblk, NULL, &es, NULL)) { + atomic64_inc(&stats->snap_fail_es_miss); + ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_ES_MISS); + return -EAGAIN; + } + + if (ext4_es_is_delayed(&es)) { + atomic64_inc(&stats->snap_fail_es_delayed); + ext4_fc_set_snap_err(snap_err, + EXT4_FC_SNAP_ERR_ES_DELAYED); + return -EAGAIN; + } + + len = es.es_len - (cur_lblk - es.es_lblk); + if (len > remaining) + len = remaining; + if (len == 0) { + cur_lblk++; + continue; + } + + if (nr_ranges_total + nr_ranges >= EXT4_FC_SNAPSHOT_MAX_RANGES) { + atomic64_inc(&stats->snap_fail_ranges_cap); + ext4_fc_set_snap_err(snap_err, + EXT4_FC_SNAP_ERR_RANGES_CAP); + return -E2BIG; + } + + range = kmem_cache_alloc(ext4_fc_range_cachep, GFP_NOFS); + if (!range) { + atomic64_inc(&stats->snap_fail_nomem); + ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_NOMEM); + return -ENOMEM; + } + nr_ranges++; + + range->lblk = cur_lblk; + range->len = len; + range->pblk = 0; + range->unwritten = false; + + if (ext4_es_is_hole(&es)) { + range->tag = EXT4_FC_TAG_DEL_RANGE; + } else if (ext4_es_is_written(&es) || + ext4_es_is_unwritten(&es)) { + unsigned int max; + + range->tag = EXT4_FC_TAG_ADD_RANGE; + range->pblk = ext4_es_pblock(&es) + + (cur_lblk - es.es_lblk); + range->unwritten = ext4_es_is_unwritten(&es); + + max = range->unwritten ? EXT_UNWRITTEN_MAX_LEN : + EXT_INIT_MAX_LEN; + if (range->len > max) + range->len = max; + } else { + kmem_cache_free(ext4_fc_range_cachep, range); + atomic64_inc(&stats->snap_fail_es_other); + ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_ES_OTHER); + return -EAGAIN; + } + + INIT_LIST_HEAD(&range->list); + list_add_tail(&range->list, ranges); + + if ((u64)range->len > (u64)end_lblk - cur_lblk) + break; + + cur_lblk += range->len; + } + + if (nr_rangesp) + *nr_rangesp = nr_ranges; + return 0; +} + +static int ext4_fc_snapshot_inode(struct inode *inode, + unsigned int nr_ranges_total, + unsigned int *nr_rangesp, int *snap_err) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + struct ext4_fc_snap_stats *stats = + &EXT4_SB(inode->i_sb)->s_fc_snap_stats; + struct ext4_fc_inode_snap *snap; + int inode_len = EXT4_GOOD_OLD_INODE_SIZE; + struct ext4_iloc iloc; + LIST_HEAD(ranges); + unsigned int nr_ranges = 0; + int ret; + int alloc_ctx; + + ret = ext4_get_inode_loc_noio(inode, &iloc); + if (ret) { + atomic64_inc(&stats->snap_fail_inode_loc); + ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_INODE_LOC); + return ret; + } + + if (ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA)) + inode_len = EXT4_INODE_SIZE(inode->i_sb); + else if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) + inode_len += ei->i_extra_isize; + + snap = kmalloc(struct_size(snap, inode_buf, inode_len), GFP_NOFS); + if (!snap) { + atomic64_inc(&stats->snap_fail_nomem); + ext4_fc_set_snap_err(snap_err, EXT4_FC_SNAP_ERR_NOMEM); + brelse(iloc.bh); + return -ENOMEM; + } + INIT_LIST_HEAD(&snap->data_list); + snap->inode_len = inode_len; + + memcpy(snap->inode_buf, (u8 *)ext4_raw_inode(&iloc), inode_len); + brelse(iloc.bh); + + ret = ext4_fc_snapshot_inode_data(inode, &ranges, nr_ranges_total, + &nr_ranges, snap_err); + if (ret) { + kfree(snap); + ext4_fc_free_ranges(&ranges); + return ret; + } + + alloc_ctx = ext4_fc_lock(inode->i_sb); + ext4_fc_free_inode_snap(inode); + ei->i_fc_snap = snap; + list_splice_tail_init(&ranges, &snap->data_list); + ext4_fc_unlock(inode->i_sb, alloc_ctx); + + atomic64_inc(&stats->snap_inodes); + atomic64_add(nr_ranges, &stats->snap_ranges); + if (nr_rangesp) + *nr_rangesp = nr_ranges; + return 0; +} + +/* Flushes data of all the inodes in the commit queue. */ +static int ext4_fc_flush_data(journal_t *journal) +{ + struct super_block *sb = journal->j_private; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_inode_info *ei; + int ret = 0; + + list_for_each_entry(ei, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { + ret = jbd2_submit_inode_data(journal, READ_ONCE(ei->jinode)); + if (ret) + return ret; + } + + list_for_each_entry(ei, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { + ret = jbd2_wait_inode_data(journal, READ_ONCE(ei->jinode)); + if (ret) + return ret; + } + + return 0; +} + +/* Commit all the directory entry updates */ +static int ext4_fc_commit_dentry_updates(journal_t *journal, u32 *crc) +{ + struct super_block *sb = journal->j_private; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_fc_dentry_update *fc_dentry, *fc_dentry_n; + struct inode *inode; + struct ext4_inode_info *ei; + int ret; + + if (list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN])) + return 0; + list_for_each_entry_safe(fc_dentry, fc_dentry_n, + &sbi->s_fc_dentry_q[FC_Q_MAIN], fcd_list) { + if (fc_dentry->fcd_op != EXT4_FC_TAG_CREAT) { + if (!ext4_fc_add_dentry_tlv(sb, crc, fc_dentry)) + return -ENOSPC; + continue; + } + /* + * With fcd_dilist we need not loop in sbi->s_fc_q to get the + * corresponding inode. Also, the corresponding inode could have been + * deleted, in which case, we don't need to do anything. + */ + if (list_empty(&fc_dentry->fcd_dilist)) + continue; + /* + * For EXT4_FC_TAG_CREAT, fcd_dilist is linked on the created + * inode's i_fc_dilist list (kept singular), so we can recover the + * inode through it. + */ + ei = list_first_entry(&fc_dentry->fcd_dilist, + struct ext4_inode_info, i_fc_dilist); + inode = &ei->vfs_inode; + WARN_ON(inode->i_ino != fc_dentry->fcd_ino); + + /* + * We first write the inode and then the create dirent. This + * allows the recovery code to create an unnamed inode first + * and then link it to a directory entry. This allows us + * to use namei.c routines almost as is and simplifies + * the recovery code. + */ + ret = ext4_fc_write_inode(inode, crc); + if (ret) + return ret; + ret = ext4_fc_write_inode_data(inode, crc); + if (ret) + return ret; + if (!ext4_fc_add_dentry_tlv(sb, crc, fc_dentry)) + return -ENOSPC; + } + return 0; +} + +static int ext4_fc_alloc_snapshot_inodes(struct super_block *sb, + struct inode ***inodesp, + unsigned int *nr_inodesp); + +static int ext4_fc_snapshot_inodes(journal_t *journal, struct inode **inodes, + unsigned int inodes_size, + unsigned int *nr_inodesp, + unsigned int *nr_rangesp, + int *snap_err) +{ + struct super_block *sb = journal->j_private; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_inode_info *iter; + struct ext4_fc_dentry_update *fc_dentry; + unsigned int i = 0; + unsigned int idx; + unsigned int nr_ranges = 0; + int ret = 0; + int alloc_ctx; + + alloc_ctx = ext4_fc_lock(sb); + list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { + if (i >= inodes_size) { + atomic64_inc(&sbi->s_fc_snap_stats.snap_fail_inodes_cap); + ext4_fc_set_snap_err(snap_err, + EXT4_FC_SNAP_ERR_INODES_CAP); + ret = -E2BIG; + goto unlock; + } + inodes[i++] = &iter->vfs_inode; + } + + list_for_each_entry(fc_dentry, &sbi->s_fc_dentry_q[FC_Q_MAIN], fcd_list) { + struct ext4_inode_info *ei; + struct inode *inode; + + if (fc_dentry->fcd_op != EXT4_FC_TAG_CREAT) + continue; + if (list_empty(&fc_dentry->fcd_dilist)) + continue; + + /* See the comment in ext4_fc_commit_dentry_updates(). */ + ei = list_first_entry(&fc_dentry->fcd_dilist, + struct ext4_inode_info, i_fc_dilist); + inode = &ei->vfs_inode; + if (!list_empty(&ei->i_fc_list)) + continue; + + if (i >= inodes_size) { + atomic64_inc(&sbi->s_fc_snap_stats.snap_fail_inodes_cap); + ext4_fc_set_snap_err(snap_err, + EXT4_FC_SNAP_ERR_INODES_CAP); + ret = -E2BIG; + goto unlock; + } + /* + * Create-only inodes may only be referenced via fcd_dilist and + * not appear on s_fc_q[MAIN]. They may hit the last iput while + * we are snapshotting, but inode eviction calls ext4_fc_del(), + * which waits for FC_COMMITTING to clear. Mark them FC_COMMITTING + * so the inode stays pinned and the snapshot stays valid until + * ext4_fc_cleanup(). + */ + ext4_set_inode_state(inode, EXT4_STATE_FC_COMMITTING); + inodes[i++] = inode; + } +unlock: + ext4_fc_unlock(sb, alloc_ctx); + + if (ret) + return ret; + + for (idx = 0; idx < i; idx++) { + unsigned int inode_ranges = 0; + + ret = ext4_fc_snapshot_inode(inodes[idx], nr_ranges, + &inode_ranges, snap_err); + if (ret) + break; + nr_ranges += inode_ranges; + } + + if (nr_inodesp) + *nr_inodesp = idx; + if (nr_rangesp) + *nr_rangesp = nr_ranges; + return ret; +} + +static int ext4_fc_perform_commit(journal_t *journal, tid_t commit_tid) +{ + struct super_block *sb = journal->j_private; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_fc_snap_stats *snap_stats = &sbi->s_fc_snap_stats; + struct ext4_inode_info *iter; + struct ext4_fc_head head; + struct inode *inode; + struct inode **inodes; + unsigned int inodes_size; + unsigned int snap_inodes = 0; + unsigned int snap_ranges = 0; + int snap_err = EXT4_FC_SNAP_ERR_NONE; + struct blk_plug plug; + int ret = 0; + u32 crc = 0; + int alloc_ctx; + ktime_t lock_start; + u64 locked_ns; + + /* + * Step 1: Mark all inodes on s_fc_q[MAIN] with + * EXT4_STATE_FC_FLUSHING_DATA. This prevents these inodes from being + * freed until the data flush is over. + */ + alloc_ctx = ext4_fc_lock(sb); + list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { + ext4_set_inode_state(&iter->vfs_inode, + EXT4_STATE_FC_FLUSHING_DATA); + } + ext4_fc_unlock(sb, alloc_ctx); + + /* Step 2: Flush data for all the eligible inodes. */ + ret = ext4_fc_flush_data(journal); + + /* + * Step 3: Clear EXT4_STATE_FC_FLUSHING_DATA flag, before returning + * any error from step 2. This ensures that waiters waiting on + * EXT4_STATE_FC_FLUSHING_DATA can resume. + */ + alloc_ctx = ext4_fc_lock(sb); + list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { + ext4_clear_inode_state(&iter->vfs_inode, + EXT4_STATE_FC_FLUSHING_DATA); + ext4_fc_wake_inode_state(&iter->vfs_inode, + EXT4_STATE_FC_FLUSHING_DATA); + } + + /* + * Make sure clearing of EXT4_STATE_FC_FLUSHING_DATA is visible before + * the waiter checks the bit. Pairs with implicit barrier in + * prepare_to_wait() in ext4_fc_del(). + */ + smp_mb(); + ext4_fc_unlock(sb, alloc_ctx); + + /* + * If we encountered error in Step 2, return it now after clearing + * EXT4_STATE_FC_FLUSHING_DATA bit. + */ + if (ret) + return ret; + + ret = ext4_fc_alloc_snapshot_inodes(sb, &inodes, &inodes_size); + if (ret) { + if (ret == -E2BIG) + atomic64_inc(&snap_stats->snap_fail_inodes_cap); + else if (ret == -ENOMEM) + atomic64_inc(&snap_stats->snap_fail_nomem); + return ret; + } + + /* Step 4: Mark all inodes as being committed. */ + jbd2_journal_lock_updates(journal); + lock_start = ktime_get(); + /* + * The journal is now locked. No more handles can start and all the + * previous handles are now drained. Snapshotting happens in this + * window so log writing can consume only stable snapshots without + * doing logical-to-physical mapping. + */ + alloc_ctx = ext4_fc_lock(sb); + list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { + ext4_set_inode_state(&iter->vfs_inode, + EXT4_STATE_FC_COMMITTING); + } + ext4_fc_unlock(sb, alloc_ctx); + + ret = ext4_fc_snapshot_inodes(journal, inodes, inodes_size, + &snap_inodes, &snap_ranges, &snap_err); + jbd2_journal_unlock_updates(journal); + locked_ns = ktime_to_ns(ktime_sub(ktime_get(), lock_start)); + atomic64_add(locked_ns, &snap_stats->lock_updates_ns_total); + atomic64_inc(&snap_stats->lock_updates_samples); + ext4_fc_snap_stats_update_max(&snap_stats->lock_updates_ns_max, + locked_ns); + if (trace_ext4_fc_lock_updates_enabled()) + trace_call__ext4_fc_lock_updates(sb, commit_tid, locked_ns, + snap_inodes, snap_ranges, + ret, snap_err); + kvfree(inodes); + if (ret) + return ret; + + /* + * Step 5: If file system device is different from journal device, + * issue a cache flush before we start writing fast commit blocks. + */ + if (journal->j_fs_dev != journal->j_dev) + blkdev_issue_flush(journal->j_fs_dev); + + blk_start_plug(&plug); + alloc_ctx = ext4_fc_lock(sb); + /* Step 6: Write fast commit blocks to disk. */ + if (sbi->s_fc_bytes == 0) { + /* + * Step 6.1: Add a head tag only if this is the first fast + * commit in this TID. + */ + head.fc_features = cpu_to_le32(EXT4_FC_SUPPORTED_FEATURES); + head.fc_tid = cpu_to_le32( + sbi->s_journal->j_running_transaction->t_tid); + if (!ext4_fc_add_tlv(sb, EXT4_FC_TAG_HEAD, sizeof(head), + (u8 *)&head, &crc)) { + ret = -ENOSPC; + goto out; + } + } + + /* Step 6.2: Now write all the dentry updates. */ + ret = ext4_fc_commit_dentry_updates(journal, &crc); + if (ret) + goto out; + + /* Step 6.3: Now write all the changed inodes to disk. */ + list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { + inode = &iter->vfs_inode; + if (!ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) + continue; + + ret = ext4_fc_write_inode_data(inode, &crc); + if (ret) + goto out; + ret = ext4_fc_write_inode(inode, &crc); + if (ret) + goto out; + } + /* Step 6.4: Finally write tail tag to conclude this fast commit. */ + ret = ext4_fc_write_tail(sb, crc); + +out: + ext4_fc_unlock(sb, alloc_ctx); + blk_finish_plug(&plug); + return ret; +} + +static unsigned int ext4_fc_count_snapshot_inodes(struct super_block *sb) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_inode_info *iter; + struct ext4_fc_dentry_update *fc_dentry; + unsigned int nr_inodes = 0; + int alloc_ctx; + + alloc_ctx = ext4_fc_lock(sb); + list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) + nr_inodes++; + + list_for_each_entry(fc_dentry, &sbi->s_fc_dentry_q[FC_Q_MAIN], fcd_list) { + struct ext4_inode_info *ei; + + if (fc_dentry->fcd_op != EXT4_FC_TAG_CREAT) + continue; + if (list_empty(&fc_dentry->fcd_dilist)) + continue; + + /* See the comment in ext4_fc_commit_dentry_updates(). */ + ei = list_first_entry(&fc_dentry->fcd_dilist, + struct ext4_inode_info, i_fc_dilist); + if (!list_empty(&ei->i_fc_list)) + continue; + + nr_inodes++; + } + ext4_fc_unlock(sb, alloc_ctx); + + return nr_inodes; +} + +static int ext4_fc_alloc_snapshot_inodes(struct super_block *sb, + struct inode ***inodesp, + unsigned int *nr_inodesp) +{ + unsigned int nr_inodes = ext4_fc_count_snapshot_inodes(sb); + struct inode **inodes; + + *inodesp = NULL; + *nr_inodesp = 0; + + if (!nr_inodes) + return 0; + + if (nr_inodes > EXT4_FC_SNAPSHOT_MAX_INODES) + return -E2BIG; + + inodes = kvcalloc(nr_inodes, sizeof(*inodes), GFP_NOFS); + if (!inodes) + return -ENOMEM; + + *inodesp = inodes; + *nr_inodesp = nr_inodes; + return 0; +} + +static void ext4_fc_update_stats(struct super_block *sb, int status, + u64 commit_time, int nblks, tid_t commit_tid) +{ + struct ext4_fc_stats *stats = &EXT4_SB(sb)->s_fc_stats; + + ext4_debug("Fast commit ended with status = %d for tid %u", + status, commit_tid); + if (status == EXT4_FC_STATUS_OK) { + stats->fc_num_commits++; + stats->fc_numblks += nblks; + if (likely(stats->s_fc_avg_commit_time)) + stats->s_fc_avg_commit_time = + (commit_time + + stats->s_fc_avg_commit_time * 3) / 4; + else + stats->s_fc_avg_commit_time = commit_time; + } else if (status == EXT4_FC_STATUS_FAILED || + status == EXT4_FC_STATUS_INELIGIBLE) { + if (status == EXT4_FC_STATUS_FAILED) + stats->fc_failed_commits++; + stats->fc_ineligible_commits++; + } else { + stats->fc_skipped_commits++; + } + trace_ext4_fc_commit_stop(sb, nblks, status, commit_tid); +} + +/* + * The main commit entry point. Performs a fast commit for transaction + * commit_tid if needed. If it's not possible to perform a fast commit + * due to various reasons, we fall back to full commit. Returns 0 + * on success, error otherwise. + */ +int ext4_fc_commit(journal_t *journal, tid_t commit_tid) +{ + struct super_block *sb = journal->j_private; + struct ext4_sb_info *sbi = EXT4_SB(sb); + int nblks = 0, ret, bsize = journal->j_blocksize; + int subtid = atomic_read(&sbi->s_fc_subtid); + int status = EXT4_FC_STATUS_OK, fc_bufs_before = 0; + ktime_t start_time, commit_time; + int old_ioprio, journal_ioprio; + + if (!test_opt2(sb, JOURNAL_FAST_COMMIT)) + return jbd2_complete_transaction(journal, commit_tid); + + trace_ext4_fc_commit_start(sb, commit_tid); + + start_time = ktime_get(); + old_ioprio = get_current_ioprio(); + +restart_fc: + ret = jbd2_fc_begin_commit(journal, commit_tid); + if (ret == -EALREADY) { + /* There was an ongoing commit, check if we need to restart */ + if (atomic_read(&sbi->s_fc_subtid) <= subtid && + tid_gt(commit_tid, journal->j_commit_sequence)) + goto restart_fc; + ext4_fc_update_stats(sb, EXT4_FC_STATUS_SKIPPED, 0, 0, + commit_tid); + return 0; + } else if (ret) { + /* + * Commit couldn't start. Just update stats and perform a + * full commit. + */ + ext4_fc_update_stats(sb, EXT4_FC_STATUS_FAILED, 0, 0, + commit_tid); + return jbd2_complete_transaction(journal, commit_tid); + } + + /* + * After establishing journal barrier via jbd2_fc_begin_commit(), check + * if we are fast commit ineligible. + */ + if (ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE)) { + status = EXT4_FC_STATUS_INELIGIBLE; + goto fallback; + } + + /* + * Now that we know that this thread is going to do a fast commit, + * elevate the priority to match that of the journal thread. + */ + if (journal->j_task->io_context) + journal_ioprio = sbi->s_journal->j_task->io_context->ioprio; + else + journal_ioprio = EXT4_DEF_JOURNAL_IOPRIO; + set_task_ioprio(current, journal_ioprio); + fc_bufs_before = (sbi->s_fc_bytes + bsize - 1) / bsize; + ret = ext4_fc_perform_commit(journal, commit_tid); + if (ret < 0) { + if (ret == -EAGAIN || ret == -E2BIG || ret == -ECANCELED) + status = EXT4_FC_STATUS_INELIGIBLE; + else + status = EXT4_FC_STATUS_FAILED; + goto fallback; + } + nblks = (sbi->s_fc_bytes + bsize - 1) / bsize - fc_bufs_before; + ret = jbd2_fc_wait_bufs(journal, nblks); + if (ret < 0) { + status = EXT4_FC_STATUS_FAILED; + goto fallback; + } + atomic_inc(&sbi->s_fc_subtid); + ret = jbd2_fc_end_commit(journal); + set_task_ioprio(current, old_ioprio); + /* + * weight the commit time higher than the average time so we + * don't react too strongly to vast changes in the commit time + */ + commit_time = ktime_to_ns(ktime_sub(ktime_get(), start_time)); + ext4_fc_update_stats(sb, status, commit_time, nblks, commit_tid); + return ret; + +fallback: + set_task_ioprio(current, old_ioprio); + ret = jbd2_fc_end_commit_fallback(journal); + ext4_fc_update_stats(sb, status, 0, 0, commit_tid); + return ret; +} + +/* + * Fast commit cleanup routine. This is called after every fast commit and + * full commit. full is true if we are called after a full commit. + */ +static void ext4_fc_cleanup(journal_t *journal, int full, tid_t tid) +{ + struct super_block *sb = journal->j_private; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_inode_info *ei; + struct ext4_fc_dentry_update *fc_dentry; + int alloc_ctx; + + if (full && sbi->s_fc_bh) + sbi->s_fc_bh = NULL; + + trace_ext4_fc_cleanup(journal, full, tid); + jbd2_fc_release_bufs(journal); + + alloc_ctx = ext4_fc_lock(sb); + while (!list_empty(&sbi->s_fc_q[FC_Q_MAIN])) { + bool requeue; + + ei = list_first_entry(&sbi->s_fc_q[FC_Q_MAIN], + struct ext4_inode_info, + i_fc_list); + list_del_init(&ei->i_fc_list); + ext4_fc_free_inode_snap(&ei->vfs_inode); + spin_lock(&ei->i_fc_lock); + if (full) + requeue = !tid_geq(tid, ei->i_sync_tid); + else + requeue = ext4_test_inode_state(&ei->vfs_inode, + EXT4_STATE_FC_REQUEUE); + if (!requeue) + ext4_fc_reset_inode(&ei->vfs_inode); + ext4_clear_inode_state(&ei->vfs_inode, EXT4_STATE_FC_REQUEUE); + ext4_clear_inode_state(&ei->vfs_inode, + EXT4_STATE_FC_COMMITTING); + spin_unlock(&ei->i_fc_lock); + if (requeue) + list_add_tail(&ei->i_fc_list, + &sbi->s_fc_q[FC_Q_STAGING]); + /* + * Make sure clearing of EXT4_STATE_FC_COMMITTING is + * visible before we send the wakeup. Pairs with implicit + * barrier in prepare_to_wait() in ext4_fc_del(). + */ + smp_mb(); + ext4_fc_wake_inode_state(&ei->vfs_inode, + EXT4_STATE_FC_COMMITTING); + } + + while (!list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN])) { + fc_dentry = list_first_entry(&sbi->s_fc_dentry_q[FC_Q_MAIN], + struct ext4_fc_dentry_update, + fcd_list); + list_del_init(&fc_dentry->fcd_list); + if (fc_dentry->fcd_op == EXT4_FC_TAG_CREAT && + !list_empty(&fc_dentry->fcd_dilist)) { + /* See the comment in ext4_fc_commit_dentry_updates(). */ + ei = list_first_entry(&fc_dentry->fcd_dilist, + struct ext4_inode_info, + i_fc_dilist); + ext4_fc_free_inode_snap(&ei->vfs_inode); + spin_lock(&ei->i_fc_lock); + ext4_clear_inode_state(&ei->vfs_inode, + EXT4_STATE_FC_REQUEUE); + ext4_clear_inode_state(&ei->vfs_inode, + EXT4_STATE_FC_COMMITTING); + spin_unlock(&ei->i_fc_lock); + /* + * Make sure clearing of EXT4_STATE_FC_COMMITTING is + * visible before we send the wakeup. Pairs with + * implicit barrier in prepare_to_wait() in + * ext4_fc_del(). + */ + smp_mb(); + ext4_fc_wake_inode_state(&ei->vfs_inode, + EXT4_STATE_FC_COMMITTING); + } + list_del_init(&fc_dentry->fcd_dilist); + + release_dentry_name_snapshot(&fc_dentry->fcd_name); + kmem_cache_free(ext4_fc_dentry_cachep, fc_dentry); + } + + list_splice_init(&sbi->s_fc_dentry_q[FC_Q_STAGING], + &sbi->s_fc_dentry_q[FC_Q_MAIN]); + list_splice_init(&sbi->s_fc_q[FC_Q_STAGING], + &sbi->s_fc_q[FC_Q_MAIN]); + + if (tid_geq(tid, sbi->s_fc_ineligible_tid)) { + sbi->s_fc_ineligible_tid = 0; + ext4_clear_mount_flag(sb, EXT4_MF_FC_INELIGIBLE); + } + + if (full) + sbi->s_fc_bytes = 0; + ext4_fc_unlock(sb, alloc_ctx); + trace_ext4_fc_stats(sb); +} + +/* Ext4 Replay Path Routines */ + +/* Helper struct for dentry replay routines */ +struct dentry_info_args { + int parent_ino, dname_len, ino, inode_len; + char *dname; +}; + +/* Same as struct ext4_fc_tl, but uses native endianness fields */ +struct ext4_fc_tl_mem { + u16 fc_tag; + u16 fc_len; +}; + +static inline void tl_to_darg(struct dentry_info_args *darg, + struct ext4_fc_tl_mem *tl, u8 *val) +{ + struct ext4_fc_dentry_info fcd; + + memcpy(&fcd, val, sizeof(fcd)); + + darg->parent_ino = le32_to_cpu(fcd.fc_parent_ino); + darg->ino = le32_to_cpu(fcd.fc_ino); + darg->dname = val + offsetof(struct ext4_fc_dentry_info, fc_dname); + darg->dname_len = tl->fc_len - sizeof(struct ext4_fc_dentry_info); +} + +static inline void ext4_fc_get_tl(struct ext4_fc_tl_mem *tl, u8 *val) +{ + struct ext4_fc_tl tl_disk; + + memcpy(&tl_disk, val, EXT4_FC_TAG_BASE_LEN); + tl->fc_len = le16_to_cpu(tl_disk.fc_len); + tl->fc_tag = le16_to_cpu(tl_disk.fc_tag); +} + +/* Unlink replay function */ +static int ext4_fc_replay_unlink(struct super_block *sb, + struct ext4_fc_tl_mem *tl, u8 *val) +{ + struct inode *inode, *old_parent; + struct qstr entry; + struct dentry_info_args darg; + int ret = 0; + + tl_to_darg(&darg, tl, val); + + trace_ext4_fc_replay(sb, EXT4_FC_TAG_UNLINK, darg.ino, + darg.parent_ino, darg.dname_len); + + entry.name = darg.dname; + entry.len = darg.dname_len; + inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL); + + if (IS_ERR(inode)) { + ext4_debug("Inode %d not found", darg.ino); + return 0; + } + + old_parent = ext4_iget(sb, darg.parent_ino, + EXT4_IGET_NORMAL); + if (IS_ERR(old_parent)) { + ext4_debug("Dir with inode %d not found", darg.parent_ino); + iput(inode); + return 0; + } + + ret = __ext4_unlink(old_parent, &entry, inode, NULL); + /* -ENOENT ok coz it might not exist anymore. */ + if (ret == -ENOENT) + ret = 0; + iput(old_parent); + iput(inode); + return ret; +} + +static int ext4_fc_replay_link_internal(struct super_block *sb, + struct dentry_info_args *darg, + struct inode *inode) +{ + struct inode *dir = NULL; + struct qstr qstr_dname = QSTR_INIT(darg->dname, darg->dname_len); + int ret = 0; + + dir = ext4_iget(sb, darg->parent_ino, EXT4_IGET_NORMAL); + if (IS_ERR(dir)) { + ext4_debug("Dir with inode %d not found.", darg->parent_ino); + dir = NULL; + goto out; + } + + ret = __ext4_link(dir, inode, &qstr_dname, NULL); + /* + * It's possible that link already existed since data blocks + * for the dir in question got persisted before we crashed OR + * we replayed this tag and crashed before the entire replay + * could complete. + */ + if (ret && ret != -EEXIST) { + ext4_debug("Failed to link\n"); + goto out; + } + + ret = 0; +out: + if (dir) + iput(dir); + + return ret; +} + +/* Link replay function */ +static int ext4_fc_replay_link(struct super_block *sb, + struct ext4_fc_tl_mem *tl, u8 *val) +{ + struct inode *inode; + struct dentry_info_args darg; + int ret = 0; + + tl_to_darg(&darg, tl, val); + trace_ext4_fc_replay(sb, EXT4_FC_TAG_LINK, darg.ino, + darg.parent_ino, darg.dname_len); + + inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL); + if (IS_ERR(inode)) { + ext4_debug("Inode not found."); + return 0; + } + + ret = ext4_fc_replay_link_internal(sb, &darg, inode); + iput(inode); + return ret; +} + +/* + * Record all the modified inodes during replay. We use this later to setup + * block bitmaps correctly. + */ +static int ext4_fc_record_modified_inode(struct super_block *sb, int ino) +{ + struct ext4_fc_replay_state *state; + int i; + + state = &EXT4_SB(sb)->s_fc_replay_state; + for (i = 0; i < state->fc_modified_inodes_used; i++) + if (state->fc_modified_inodes[i] == ino) + return 0; + if (state->fc_modified_inodes_used == state->fc_modified_inodes_size) { + int *fc_modified_inodes; + + fc_modified_inodes = krealloc(state->fc_modified_inodes, + sizeof(int) * (state->fc_modified_inodes_size + + EXT4_FC_REPLAY_REALLOC_INCREMENT), + GFP_KERNEL); + if (!fc_modified_inodes) + return -ENOMEM; + state->fc_modified_inodes = fc_modified_inodes; + state->fc_modified_inodes_size += + EXT4_FC_REPLAY_REALLOC_INCREMENT; + } + state->fc_modified_inodes[state->fc_modified_inodes_used++] = ino; + return 0; +} + +/* + * Inode replay function + */ +static int ext4_fc_replay_inode(struct super_block *sb, + struct ext4_fc_tl_mem *tl, u8 *val) +{ + struct ext4_fc_inode fc_inode; + struct ext4_inode *raw_inode; + struct ext4_inode *raw_fc_inode; + struct inode *inode = NULL; + struct ext4_iloc iloc; + int inode_len, ino, ret, tag = tl->fc_tag; + struct ext4_extent_header *eh; + size_t off_gen = offsetof(struct ext4_inode, i_generation); + + memcpy(&fc_inode, val, sizeof(fc_inode)); + + ino = le32_to_cpu(fc_inode.fc_ino); + trace_ext4_fc_replay(sb, tag, ino, 0, 0); + + inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL); + if (!IS_ERR(inode)) { + ext4_ext_clear_bb(inode); + iput(inode); + } + inode = NULL; + + ret = ext4_fc_record_modified_inode(sb, ino); + if (ret) + goto out; + + raw_fc_inode = (struct ext4_inode *) + (val + offsetof(struct ext4_fc_inode, fc_raw_inode)); + ret = ext4_get_fc_inode_loc(sb, ino, &iloc); + if (ret) + goto out; + + inode_len = tl->fc_len - sizeof(struct ext4_fc_inode); + raw_inode = ext4_raw_inode(&iloc); + + memcpy(raw_inode, raw_fc_inode, offsetof(struct ext4_inode, i_block)); + memcpy((u8 *)raw_inode + off_gen, (u8 *)raw_fc_inode + off_gen, + inode_len - off_gen); + if (le32_to_cpu(raw_inode->i_flags) & EXT4_EXTENTS_FL) { + eh = (struct ext4_extent_header *)(&raw_inode->i_block[0]); + if (eh->eh_magic != EXT4_EXT_MAGIC) { + memset(eh, 0, sizeof(*eh)); + eh->eh_magic = EXT4_EXT_MAGIC; + eh->eh_max = cpu_to_le16( + (sizeof(raw_inode->i_block) - + sizeof(struct ext4_extent_header)) + / sizeof(struct ext4_extent)); + } + } else if (le32_to_cpu(raw_inode->i_flags) & EXT4_INLINE_DATA_FL) { + memcpy(raw_inode->i_block, raw_fc_inode->i_block, + sizeof(raw_inode->i_block)); + } + + /* Immediately update the inode on disk. */ + ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh); + if (ret) + goto out_brelse; + ret = sync_dirty_buffer(iloc.bh); + if (ret) + goto out_brelse; + ret = ext4_mark_inode_used(sb, ino); + if (ret) + goto out_brelse; + + /* Given that we just wrote the inode on disk, this SHOULD succeed. */ + inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL); + if (IS_ERR(inode)) { + ext4_debug("Inode not found."); + inode = NULL; + ret = -EFSCORRUPTED; + goto out_brelse; + } + + /* + * Our allocator could have made different decisions than before + * crashing. This should be fixed but until then, we calculate + * the number of blocks the inode. + */ + if (!ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA)) + ext4_ext_replay_set_iblocks(inode); + + inode->i_generation = le32_to_cpu(ext4_raw_inode(&iloc)->i_generation); + ext4_reset_inode_seed(inode); + + ext4_inode_csum_set(inode, ext4_raw_inode(&iloc), EXT4_I(inode)); + ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh); + sync_dirty_buffer(iloc.bh); +out_brelse: + brelse(iloc.bh); +out: + iput(inode); + if (!ret) + blkdev_issue_flush(sb->s_bdev); + + return ret; +} + +/* + * Dentry create replay function. + * + * EXT4_FC_TAG_CREAT is preceded by EXT4_FC_TAG_INODE_FULL. Which means, the + * inode for which we are trying to create a dentry here, should already have + * been replayed before we start here. + */ +static int ext4_fc_replay_create(struct super_block *sb, + struct ext4_fc_tl_mem *tl, u8 *val) +{ + int ret = 0; + struct inode *inode = NULL; + struct inode *dir = NULL; + struct dentry_info_args darg; + + tl_to_darg(&darg, tl, val); + + trace_ext4_fc_replay(sb, EXT4_FC_TAG_CREAT, darg.ino, + darg.parent_ino, darg.dname_len); + + /* This takes care of update group descriptor and other metadata */ + ret = ext4_mark_inode_used(sb, darg.ino); + if (ret) + goto out; + + inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL); + if (IS_ERR(inode)) { + ext4_debug("inode %d not found.", darg.ino); + inode = NULL; + ret = -EINVAL; + goto out; + } + + if (S_ISDIR(inode->i_mode)) { + /* + * If we are creating a directory, we need to make sure that the + * dot and dot dot dirents are setup properly. + */ + dir = ext4_iget(sb, darg.parent_ino, EXT4_IGET_NORMAL); + if (IS_ERR(dir)) { + ext4_debug("Dir %d not found.", darg.ino); + goto out; + } + ret = ext4_init_new_dir(NULL, dir, inode); + iput(dir); + if (ret) { + ret = 0; + goto out; + } + } + ret = ext4_fc_replay_link_internal(sb, &darg, inode); + if (ret) + goto out; + set_nlink(inode, 1); + ext4_mark_inode_dirty(NULL, inode); +out: + iput(inode); + return ret; +} + +/* + * Record physical disk regions which are in use as per fast commit area, + * and used by inodes during replay phase. Our simple replay phase + * allocator excludes these regions from allocation. + */ +int ext4_fc_record_regions(struct super_block *sb, int ino, + ext4_lblk_t lblk, ext4_fsblk_t pblk, int len, int replay) +{ + struct ext4_fc_replay_state *state; + struct ext4_fc_alloc_region *region; + + state = &EXT4_SB(sb)->s_fc_replay_state; + /* + * during replay phase, the fc_regions_valid may not same as + * fc_regions_used, update it when do new additions. + */ + if (replay && state->fc_regions_used != state->fc_regions_valid) + state->fc_regions_used = state->fc_regions_valid; + if (state->fc_regions_used == state->fc_regions_size) { + struct ext4_fc_alloc_region *fc_regions; + + fc_regions = krealloc(state->fc_regions, + sizeof(struct ext4_fc_alloc_region) * + (state->fc_regions_size + + EXT4_FC_REPLAY_REALLOC_INCREMENT), + GFP_KERNEL); + if (!fc_regions) + return -ENOMEM; + state->fc_regions_size += + EXT4_FC_REPLAY_REALLOC_INCREMENT; + state->fc_regions = fc_regions; + } + region = &state->fc_regions[state->fc_regions_used++]; + region->ino = ino; + region->lblk = lblk; + region->pblk = pblk; + region->len = len; + + if (replay) + state->fc_regions_valid++; + + return 0; +} + +/* Replay add range tag */ +static int ext4_fc_replay_add_range(struct super_block *sb, u8 *val) +{ + struct ext4_fc_add_range fc_add_ex; + struct ext4_extent newex, *ex; + struct inode *inode; + ext4_lblk_t start, cur; + int remaining, len; + ext4_fsblk_t start_pblk; + struct ext4_map_blocks map; + struct ext4_ext_path *path = NULL; + int ret; + + memcpy(&fc_add_ex, val, sizeof(fc_add_ex)); + ex = (struct ext4_extent *)&fc_add_ex.fc_ex; + + trace_ext4_fc_replay(sb, EXT4_FC_TAG_ADD_RANGE, + le32_to_cpu(fc_add_ex.fc_ino), le32_to_cpu(ex->ee_block), + ext4_ext_get_actual_len(ex)); + + inode = ext4_iget(sb, le32_to_cpu(fc_add_ex.fc_ino), EXT4_IGET_NORMAL); + if (IS_ERR(inode)) { + ext4_debug("Inode not found."); + return 0; + } + + ret = ext4_fc_record_modified_inode(sb, inode->i_ino); + if (ret) + goto out; + + start = le32_to_cpu(ex->ee_block); + start_pblk = ext4_ext_pblock(ex); + len = ext4_ext_get_actual_len(ex); + + cur = start; + remaining = len; + ext4_debug("ADD_RANGE, lblk %d, pblk %lld, len %d, unwritten %d, inode %llu\n", + start, start_pblk, len, ext4_ext_is_unwritten(ex), + inode->i_ino); + + while (remaining > 0) { + map.m_lblk = cur; + map.m_len = remaining; + map.m_pblk = 0; + ret = ext4_map_blocks(NULL, inode, &map, 0); + + if (ret < 0) + goto out; + + if (ret == 0) { + /* Range is not mapped */ + path = ext4_find_extent(inode, cur, path, 0); + if (IS_ERR(path)) { + ret = PTR_ERR(path); + path = NULL; + goto out; + } + memset(&newex, 0, sizeof(newex)); + newex.ee_block = cpu_to_le32(cur); + ext4_ext_store_pblock( + &newex, start_pblk + cur - start); + newex.ee_len = cpu_to_le16(map.m_len); + if (ext4_ext_is_unwritten(ex)) + ext4_ext_mark_unwritten(&newex); + down_write(&EXT4_I(inode)->i_data_sem); + path = ext4_ext_insert_extent(NULL, inode, + path, &newex, 0); + up_write((&EXT4_I(inode)->i_data_sem)); + if (IS_ERR(path)) { + ret = PTR_ERR(path); + path = NULL; + goto out; + } + goto next; + } + + if (start_pblk + cur - start != map.m_pblk) { + /* + * Logical to physical mapping changed. This can happen + * if this range was removed and then reallocated to + * map to new physical blocks during a fast commit. + */ + ret = ext4_ext_replay_update_ex(inode, cur, map.m_len, + ext4_ext_is_unwritten(ex), + start_pblk + cur - start); + if (ret) + goto out; + /* + * Mark the old blocks as free since they aren't used + * anymore. We maintain an array of all the modified + * inodes. In case these blocks are still used at either + * a different logical range in the same inode or in + * some different inode, we will mark them as allocated + * at the end of the FC replay using our array of + * modified inodes. + */ + ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, false); + goto next; + } + + /* Range is mapped and needs a state change */ + ext4_debug("Converting from %ld to %d %lld", + map.m_flags & EXT4_MAP_UNWRITTEN, + ext4_ext_is_unwritten(ex), map.m_pblk); + ret = ext4_ext_replay_update_ex(inode, cur, map.m_len, + ext4_ext_is_unwritten(ex), map.m_pblk); + if (ret) + goto out; + /* + * We may have split the extent tree while toggling the state. + * Try to shrink the extent tree now. + */ + ext4_ext_replay_shrink_inode(inode, start + len); +next: + cur += map.m_len; + remaining -= map.m_len; + } + ext4_ext_replay_shrink_inode(inode, i_size_read(inode) >> + sb->s_blocksize_bits); + ret = 0; +out: + ext4_free_ext_path(path); + iput(inode); + return ret; +} + +/* Replay DEL_RANGE tag */ +static int +ext4_fc_replay_del_range(struct super_block *sb, u8 *val) +{ + struct inode *inode; + struct ext4_fc_del_range lrange; + struct ext4_map_blocks map; + ext4_lblk_t cur, remaining; + int ret; + + memcpy(&lrange, val, sizeof(lrange)); + cur = le32_to_cpu(lrange.fc_lblk); + remaining = le32_to_cpu(lrange.fc_len); + + trace_ext4_fc_replay(sb, EXT4_FC_TAG_DEL_RANGE, + le32_to_cpu(lrange.fc_ino), cur, remaining); + + inode = ext4_iget(sb, le32_to_cpu(lrange.fc_ino), EXT4_IGET_NORMAL); + if (IS_ERR(inode)) { + ext4_debug("Inode %d not found", le32_to_cpu(lrange.fc_ino)); + return 0; + } + + ret = ext4_fc_record_modified_inode(sb, inode->i_ino); + if (ret) + goto out; + + ext4_debug("DEL_RANGE, inode %llu, lblk %d, len %d\n", + inode->i_ino, le32_to_cpu(lrange.fc_lblk), + le32_to_cpu(lrange.fc_len)); + while (remaining > 0) { + map.m_lblk = cur; + map.m_len = remaining; + + ret = ext4_map_blocks(NULL, inode, &map, 0); + if (ret < 0) + goto out; + if (ret > 0) { + remaining -= ret; + cur += ret; + ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, false); + } else { + remaining -= map.m_len; + cur += map.m_len; + } + } + + down_write(&EXT4_I(inode)->i_data_sem); + ret = ext4_ext_remove_space(inode, le32_to_cpu(lrange.fc_lblk), + le32_to_cpu(lrange.fc_lblk) + + le32_to_cpu(lrange.fc_len) - 1); + up_write(&EXT4_I(inode)->i_data_sem); + if (ret) + goto out; + ext4_ext_replay_shrink_inode(inode, + i_size_read(inode) >> sb->s_blocksize_bits); + ext4_mark_inode_dirty(NULL, inode); + ret = 0; +out: + iput(inode); + return ret; +} + +static void ext4_fc_set_bitmaps_and_counters(struct super_block *sb) +{ + struct ext4_fc_replay_state *state; + struct inode *inode; + struct ext4_ext_path *path = NULL; + struct ext4_map_blocks map; + int i, ret, j; + ext4_lblk_t cur, end; + + state = &EXT4_SB(sb)->s_fc_replay_state; + for (i = 0; i < state->fc_modified_inodes_used; i++) { + inode = ext4_iget(sb, state->fc_modified_inodes[i], + EXT4_IGET_NORMAL); + if (IS_ERR(inode)) { + ext4_debug("Inode %d not found.", + state->fc_modified_inodes[i]); + continue; + } + cur = 0; + end = EXT_MAX_BLOCKS; + if (ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA)) { + iput(inode); + continue; + } + while (cur < end) { + map.m_lblk = cur; + map.m_len = end - cur; + + ret = ext4_map_blocks(NULL, inode, &map, 0); + if (ret < 0) + break; + + if (ret > 0) { + path = ext4_find_extent(inode, map.m_lblk, path, 0); + if (!IS_ERR(path)) { + for (j = 0; j < path->p_depth; j++) + ext4_mb_mark_bb(inode->i_sb, + path[j].p_block, 1, true); + } else { + path = NULL; + } + cur += ret; + ext4_mb_mark_bb(inode->i_sb, map.m_pblk, + map.m_len, true); + } else { + cur = cur + (map.m_len ? map.m_len : 1); + } + } + iput(inode); + } + + ext4_free_ext_path(path); +} + +/* + * Check if block is in excluded regions for block allocation. The simple + * allocator that runs during replay phase is calls this function to see + * if it is okay to use a block. + */ +bool ext4_fc_replay_check_excluded(struct super_block *sb, ext4_fsblk_t blk) +{ + int i; + struct ext4_fc_replay_state *state; + + state = &EXT4_SB(sb)->s_fc_replay_state; + for (i = 0; i < state->fc_regions_valid; i++) { + if (state->fc_regions[i].ino == 0 || + state->fc_regions[i].len == 0) + continue; + if (in_range(blk, state->fc_regions[i].pblk, + state->fc_regions[i].len)) + return true; + } + return false; +} + +/* Cleanup function called after replay */ +void ext4_fc_replay_cleanup(struct super_block *sb) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + + sbi->s_mount_state &= ~EXT4_FC_REPLAY; + kfree(sbi->s_fc_replay_state.fc_regions); + kfree(sbi->s_fc_replay_state.fc_modified_inodes); +} + +static bool ext4_fc_value_len_isvalid(struct ext4_sb_info *sbi, + int tag, int len) +{ + switch (tag) { + case EXT4_FC_TAG_ADD_RANGE: + return len == sizeof(struct ext4_fc_add_range); + case EXT4_FC_TAG_DEL_RANGE: + return len == sizeof(struct ext4_fc_del_range); + case EXT4_FC_TAG_CREAT: + case EXT4_FC_TAG_LINK: + case EXT4_FC_TAG_UNLINK: + len -= sizeof(struct ext4_fc_dentry_info); + return len >= 1 && len <= EXT4_NAME_LEN; + case EXT4_FC_TAG_INODE: + len -= sizeof(struct ext4_fc_inode); + return len >= EXT4_GOOD_OLD_INODE_SIZE && + len <= sbi->s_inode_size; + case EXT4_FC_TAG_PAD: + return true; /* padding can have any length */ + case EXT4_FC_TAG_TAIL: + return len >= sizeof(struct ext4_fc_tail); + case EXT4_FC_TAG_HEAD: + return len == sizeof(struct ext4_fc_head); + } + return false; +} + +/* + * Recovery Scan phase handler + * + * This function is called during the scan phase and is responsible + * for doing following things: + * - Make sure the fast commit area has valid tags for replay + * - Count number of tags that need to be replayed by the replay handler + * - Verify CRC + * - Create a list of excluded blocks for allocation during replay phase + * + * This function returns JBD2_FC_REPLAY_CONTINUE to indicate that SCAN is + * incomplete and JBD2 should send more blocks. It returns JBD2_FC_REPLAY_STOP + * to indicate that scan has finished and JBD2 can now start replay phase. + * It returns a negative error to indicate that there was an error. At the end + * of a successful scan phase, sbi->s_fc_replay_state.fc_replay_num_tags is set + * to indicate the number of tags that need to replayed during the replay phase. + */ +static int ext4_fc_replay_scan(journal_t *journal, + struct buffer_head *bh, int off, + tid_t expected_tid) +{ + struct super_block *sb = journal->j_private; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_fc_replay_state *state; + int ret = JBD2_FC_REPLAY_CONTINUE; + struct ext4_fc_add_range ext; + struct ext4_fc_tl_mem tl; + struct ext4_fc_tail tail; + __u8 *start, *end, *cur, *val; + struct ext4_fc_head head; + struct ext4_extent *ex; + + state = &sbi->s_fc_replay_state; + + start = (u8 *)bh->b_data; + end = start + journal->j_blocksize; + + if (state->fc_replay_expected_off == 0) { + state->fc_cur_tag = 0; + state->fc_replay_num_tags = 0; + state->fc_crc = 0; + state->fc_regions = NULL; + state->fc_regions_valid = state->fc_regions_used = + state->fc_regions_size = 0; + /* Check if we can stop early */ + if (le16_to_cpu(((struct ext4_fc_tl *)start)->fc_tag) + != EXT4_FC_TAG_HEAD) + return 0; + } + + if (off != state->fc_replay_expected_off) { + ret = -EFSCORRUPTED; + goto out_err; + } + + state->fc_replay_expected_off++; + for (cur = start; cur <= end - EXT4_FC_TAG_BASE_LEN; + cur = cur + EXT4_FC_TAG_BASE_LEN + tl.fc_len) { + ext4_fc_get_tl(&tl, cur); + val = cur + EXT4_FC_TAG_BASE_LEN; + if (tl.fc_len > end - val || + !ext4_fc_value_len_isvalid(sbi, tl.fc_tag, tl.fc_len)) { + ret = state->fc_replay_num_tags ? + JBD2_FC_REPLAY_STOP : -ECANCELED; + goto out_err; + } + ext4_debug("Scan phase, tag:%s, blk %lld\n", + tag2str(tl.fc_tag), bh->b_blocknr); + switch (tl.fc_tag) { + case EXT4_FC_TAG_ADD_RANGE: + memcpy(&ext, val, sizeof(ext)); + ex = (struct ext4_extent *)&ext.fc_ex; + ret = ext4_fc_record_regions(sb, + le32_to_cpu(ext.fc_ino), + le32_to_cpu(ex->ee_block), ext4_ext_pblock(ex), + ext4_ext_get_actual_len(ex), 0); + if (ret < 0) + break; + ret = JBD2_FC_REPLAY_CONTINUE; + fallthrough; + case EXT4_FC_TAG_DEL_RANGE: + case EXT4_FC_TAG_LINK: + case EXT4_FC_TAG_UNLINK: + case EXT4_FC_TAG_CREAT: + case EXT4_FC_TAG_INODE: + case EXT4_FC_TAG_PAD: + state->fc_cur_tag++; + state->fc_crc = ext4_chksum(state->fc_crc, cur, + EXT4_FC_TAG_BASE_LEN + tl.fc_len); + break; + case EXT4_FC_TAG_TAIL: + state->fc_cur_tag++; + memcpy(&tail, val, sizeof(tail)); + state->fc_crc = ext4_chksum(state->fc_crc, cur, + EXT4_FC_TAG_BASE_LEN + + offsetof(struct ext4_fc_tail, + fc_crc)); + if (le32_to_cpu(tail.fc_tid) == expected_tid && + le32_to_cpu(tail.fc_crc) == state->fc_crc) { + state->fc_replay_num_tags = state->fc_cur_tag; + state->fc_regions_valid = + state->fc_regions_used; + } else { + ret = state->fc_replay_num_tags ? + JBD2_FC_REPLAY_STOP : -EFSBADCRC; + } + state->fc_crc = 0; + break; + case EXT4_FC_TAG_HEAD: + memcpy(&head, val, sizeof(head)); + if (le32_to_cpu(head.fc_features) & + ~EXT4_FC_SUPPORTED_FEATURES) { + ret = -EOPNOTSUPP; + break; + } + if (le32_to_cpu(head.fc_tid) != expected_tid) { + ret = JBD2_FC_REPLAY_STOP; + break; + } + state->fc_cur_tag++; + state->fc_crc = ext4_chksum(state->fc_crc, cur, + EXT4_FC_TAG_BASE_LEN + tl.fc_len); + break; + default: + ret = state->fc_replay_num_tags ? + JBD2_FC_REPLAY_STOP : -ECANCELED; + } + if (ret < 0 || ret == JBD2_FC_REPLAY_STOP) + break; + } + +out_err: + trace_ext4_fc_replay_scan(sb, ret, off); + return ret; +} + +/* + * Main recovery path entry point. + * The meaning of return codes is similar as above. + */ +static int ext4_fc_replay(journal_t *journal, struct buffer_head *bh, + enum passtype pass, int off, tid_t expected_tid) +{ + struct super_block *sb = journal->j_private; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_fc_tl_mem tl; + __u8 *start, *end, *cur, *val; + int ret = JBD2_FC_REPLAY_CONTINUE; + struct ext4_fc_replay_state *state = &sbi->s_fc_replay_state; + struct ext4_fc_tail tail; + + if (pass == PASS_SCAN) { + state->fc_current_pass = PASS_SCAN; + return ext4_fc_replay_scan(journal, bh, off, expected_tid); + } + + if (state->fc_current_pass != pass) { + state->fc_current_pass = pass; + sbi->s_mount_state |= EXT4_FC_REPLAY; + } + if (!sbi->s_fc_replay_state.fc_replay_num_tags) { + ext4_debug("Replay stops\n"); + ext4_fc_set_bitmaps_and_counters(sb); + return 0; + } + +#ifdef CONFIG_EXT4_DEBUG + if (sbi->s_fc_debug_max_replay && off >= sbi->s_fc_debug_max_replay) { + pr_warn("Dropping fc block %d because max_replay set\n", off); + return JBD2_FC_REPLAY_STOP; + } +#endif + + start = (u8 *)bh->b_data; + end = start + journal->j_blocksize; + + for (cur = start; cur <= end - EXT4_FC_TAG_BASE_LEN; + cur = cur + EXT4_FC_TAG_BASE_LEN + tl.fc_len) { + ext4_fc_get_tl(&tl, cur); + val = cur + EXT4_FC_TAG_BASE_LEN; + + if (state->fc_replay_num_tags == 0) { + ret = JBD2_FC_REPLAY_STOP; + ext4_fc_set_bitmaps_and_counters(sb); + break; + } + + ext4_debug("Replay phase, tag:%s\n", tag2str(tl.fc_tag)); + state->fc_replay_num_tags--; + switch (tl.fc_tag) { + case EXT4_FC_TAG_LINK: + ret = ext4_fc_replay_link(sb, &tl, val); + break; + case EXT4_FC_TAG_UNLINK: + ret = ext4_fc_replay_unlink(sb, &tl, val); + break; + case EXT4_FC_TAG_ADD_RANGE: + ret = ext4_fc_replay_add_range(sb, val); + break; + case EXT4_FC_TAG_CREAT: + ret = ext4_fc_replay_create(sb, &tl, val); + break; + case EXT4_FC_TAG_DEL_RANGE: + ret = ext4_fc_replay_del_range(sb, val); + break; + case EXT4_FC_TAG_INODE: + ret = ext4_fc_replay_inode(sb, &tl, val); + break; + case EXT4_FC_TAG_PAD: + trace_ext4_fc_replay(sb, EXT4_FC_TAG_PAD, 0, + tl.fc_len, 0); + break; + case EXT4_FC_TAG_TAIL: + trace_ext4_fc_replay(sb, EXT4_FC_TAG_TAIL, + 0, tl.fc_len, 0); + memcpy(&tail, val, sizeof(tail)); + WARN_ON(le32_to_cpu(tail.fc_tid) != expected_tid); + break; + case EXT4_FC_TAG_HEAD: + break; + default: + trace_ext4_fc_replay(sb, tl.fc_tag, 0, tl.fc_len, 0); + ret = -ECANCELED; + break; + } + if (ret < 0) + break; + ret = JBD2_FC_REPLAY_CONTINUE; + } + return ret; +} + +void ext4_fc_init(struct super_block *sb, journal_t *journal) +{ + /* + * We set replay callback even if fast commit disabled because we may + * could still have fast commit blocks that need to be replayed even if + * fast commit has now been turned off. + */ + journal->j_fc_replay_callback = ext4_fc_replay; + if (!test_opt2(sb, JOURNAL_FAST_COMMIT)) + return; + journal->j_fc_cleanup_callback = ext4_fc_cleanup; +} + +static const char * const fc_ineligible_reasons[] = { + [EXT4_FC_REASON_XATTR] = "Extended attributes changed", + [EXT4_FC_REASON_CROSS_RENAME] = "Cross rename", + [EXT4_FC_REASON_JOURNAL_FLAG_CHANGE] = "Journal flag changed", + [EXT4_FC_REASON_NOMEM] = "Insufficient memory", + [EXT4_FC_REASON_SWAP_BOOT] = "Swap boot", + [EXT4_FC_REASON_RESIZE] = "Resize", + [EXT4_FC_REASON_RENAME_DIR] = "Dir renamed", + [EXT4_FC_REASON_FALLOC_RANGE] = "Falloc range op", + [EXT4_FC_REASON_INODE_JOURNAL_DATA] = "Data journalling", + [EXT4_FC_REASON_ENCRYPTED_FILENAME] = "Encrypted filename", + [EXT4_FC_REASON_MIGRATE] = "Inode format migration", + [EXT4_FC_REASON_VERITY] = "fs-verity enable", + [EXT4_FC_REASON_MOVE_EXT] = "Move extents", +}; + +int ext4_fc_info_show(struct seq_file *seq, void *v) +{ + struct ext4_sb_info *sbi = EXT4_SB((struct super_block *)seq->private); + struct ext4_fc_stats *stats = &sbi->s_fc_stats; + struct ext4_fc_snap_stats *snap_stats = &sbi->s_fc_snap_stats; + u64 lock_avg_ns = 0; + u64 lock_updates_samples; + u64 lock_updates_ns_total; + u64 lock_updates_ns_max; + int i; + + if (v != SEQ_START_TOKEN) + return 0; + + lock_updates_samples = + atomic64_read(&snap_stats->lock_updates_samples); + lock_updates_ns_total = + atomic64_read(&snap_stats->lock_updates_ns_total); + lock_updates_ns_max = + atomic64_read(&snap_stats->lock_updates_ns_max); + if (lock_updates_samples) + lock_avg_ns = div64_u64(lock_updates_ns_total, + lock_updates_samples); + + seq_printf(seq, + "fc stats:\n%ld commits\n%ld ineligible\n%ld numblks\n%lluus avg_commit_time\n", + stats->fc_num_commits, stats->fc_ineligible_commits, + stats->fc_numblks, + div_u64(stats->s_fc_avg_commit_time, 1000)); + seq_puts(seq, "Ineligible reasons:\n"); + for (i = 0; i < EXT4_FC_REASON_MAX; i++) + seq_printf(seq, "\"%s\":\t%d\n", fc_ineligible_reasons[i], + stats->fc_ineligible_reason_count[i]); + + seq_printf(seq, + "Snapshot stats:\n%llu inodes\n%llu ranges\n%lluus lock_updates_avg\n%lluus lock_updates_max\n", + atomic64_read(&snap_stats->snap_inodes), + atomic64_read(&snap_stats->snap_ranges), + div_u64(lock_avg_ns, 1000), + div_u64(lock_updates_ns_max, 1000)); + seq_printf(seq, + "Snapshot failures:\n%llu es_miss\n%llu es_delayed\n%llu es_other\n%llu inodes_cap\n%llu ranges_cap\n%llu nomem\n%llu inode_loc\n%llu no_snap\n", + atomic64_read(&snap_stats->snap_fail_es_miss), + atomic64_read(&snap_stats->snap_fail_es_delayed), + atomic64_read(&snap_stats->snap_fail_es_other), + atomic64_read(&snap_stats->snap_fail_inodes_cap), + atomic64_read(&snap_stats->snap_fail_ranges_cap), + atomic64_read(&snap_stats->snap_fail_nomem), + atomic64_read(&snap_stats->snap_fail_inode_loc), + atomic64_read(&snap_stats->snap_fail_no_snap)); + + return 0; +} + +int __init ext4_fc_init_dentry_cache(void) +{ + ext4_fc_dentry_cachep = KMEM_CACHE(ext4_fc_dentry_update, + SLAB_RECLAIM_ACCOUNT); + + if (!ext4_fc_dentry_cachep) + return -ENOMEM; + + ext4_fc_range_cachep = KMEM_CACHE(ext4_fc_range, SLAB_RECLAIM_ACCOUNT); + if (!ext4_fc_range_cachep) { + kmem_cache_destroy(ext4_fc_dentry_cachep); + return -ENOMEM; + } + + return 0; +} + +void ext4_fc_destroy_dentry_cache(void) +{ + kmem_cache_destroy(ext4_fc_range_cachep); + kmem_cache_destroy(ext4_fc_dentry_cachep); +} diff --git a/fs/ext4/fast_commit.h b/fs/ext4/fast_commit.h new file mode 100644 index 000000000..2f77a37fb --- /dev/null +++ b/fs/ext4/fast_commit.h @@ -0,0 +1,189 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __FAST_COMMIT_H__ +#define __FAST_COMMIT_H__ + +/* + * Note this file is present in e2fsprogs/lib/ext2fs/fast_commit.h and + * linux/fs/ext4/fast_commit.h. These file should always be byte identical. + */ + +/* Fast commit tags */ +#define EXT4_FC_TAG_ADD_RANGE 0x0001 +#define EXT4_FC_TAG_DEL_RANGE 0x0002 +#define EXT4_FC_TAG_CREAT 0x0003 +#define EXT4_FC_TAG_LINK 0x0004 +#define EXT4_FC_TAG_UNLINK 0x0005 +#define EXT4_FC_TAG_INODE 0x0006 +#define EXT4_FC_TAG_PAD 0x0007 +#define EXT4_FC_TAG_TAIL 0x0008 +#define EXT4_FC_TAG_HEAD 0x0009 + +#define EXT4_FC_SUPPORTED_FEATURES 0x0 + +/* On disk fast commit tlv value structures */ + +/* Fast commit on disk tag length structure */ +struct ext4_fc_tl { + __le16 fc_tag; + __le16 fc_len; +}; + +/* Value structure for tag EXT4_FC_TAG_HEAD. */ +struct ext4_fc_head { + __le32 fc_features; + __le32 fc_tid; +}; + +/* Value structure for EXT4_FC_TAG_ADD_RANGE. */ +struct ext4_fc_add_range { + __le32 fc_ino; + __u8 fc_ex[12]; +}; + +/* Value structure for tag EXT4_FC_TAG_DEL_RANGE. */ +struct ext4_fc_del_range { + __le32 fc_ino; + __le32 fc_lblk; + __le32 fc_len; +}; + +/* + * This is the value structure for tags EXT4_FC_TAG_CREAT, EXT4_FC_TAG_LINK + * and EXT4_FC_TAG_UNLINK. + */ +struct ext4_fc_dentry_info { + __le32 fc_parent_ino; + __le32 fc_ino; + __u8 fc_dname[]; +}; + +/* Value structure for EXT4_FC_TAG_INODE. */ +struct ext4_fc_inode { + __le32 fc_ino; + __u8 fc_raw_inode[]; +}; + +/* Value structure for tag EXT4_FC_TAG_TAIL. */ +struct ext4_fc_tail { + __le32 fc_tid; + __le32 fc_crc; +}; + +/* Tag base length */ +#define EXT4_FC_TAG_BASE_LEN (sizeof(struct ext4_fc_tl)) + +/* + * Fast commit status codes + */ +enum { + EXT4_FC_STATUS_OK = 0, + EXT4_FC_STATUS_INELIGIBLE, + EXT4_FC_STATUS_SKIPPED, + EXT4_FC_STATUS_FAILED, +}; + +/* + * Fast commit ineligiblity reasons: + */ +enum { + EXT4_FC_REASON_XATTR = 0, + EXT4_FC_REASON_CROSS_RENAME, + EXT4_FC_REASON_JOURNAL_FLAG_CHANGE, + EXT4_FC_REASON_NOMEM, + EXT4_FC_REASON_SWAP_BOOT, + EXT4_FC_REASON_RESIZE, + EXT4_FC_REASON_RENAME_DIR, + EXT4_FC_REASON_FALLOC_RANGE, + EXT4_FC_REASON_INODE_JOURNAL_DATA, + EXT4_FC_REASON_ENCRYPTED_FILENAME, + EXT4_FC_REASON_MIGRATE, + EXT4_FC_REASON_VERITY, + EXT4_FC_REASON_MOVE_EXT, + EXT4_FC_REASON_MAX +}; + +#ifdef __KERNEL__ +/* + * In memory list of dentry updates that are performed on the file + * system used by fast commit code. + */ +struct ext4_fc_dentry_update { + int fcd_op; /* Type of update create / unlink / link */ + int fcd_parent; /* Parent inode number */ + int fcd_ino; /* Inode number */ + struct name_snapshot fcd_name; /* Dirent name */ + struct list_head fcd_list; + struct list_head fcd_dilist; +}; + +struct ext4_fc_stats { + unsigned int fc_ineligible_reason_count[EXT4_FC_REASON_MAX]; + unsigned long fc_num_commits; + unsigned long fc_ineligible_commits; + unsigned long fc_failed_commits; + unsigned long fc_skipped_commits; + unsigned long fc_numblks; + u64 s_fc_avg_commit_time; +}; + +#define EXT4_FC_REPLAY_REALLOC_INCREMENT 4 + +/* + * Physical block regions added to different inodes due to fast commit + * recovery. These are set during the SCAN phase. During the replay phase, + * our allocator excludes these from its allocation. This ensures that + * we don't accidentally allocating a block that is going to be used by + * another inode. + */ +struct ext4_fc_alloc_region { + ext4_lblk_t lblk; + ext4_fsblk_t pblk; + int ino, len; +}; + +/* + * Fast commit replay state. + */ +struct ext4_fc_replay_state { + int fc_replay_num_tags; + int fc_replay_expected_off; + int fc_current_pass; + int fc_cur_tag; + int fc_crc; + struct ext4_fc_alloc_region *fc_regions; + int fc_regions_size, fc_regions_used, fc_regions_valid; + int *fc_modified_inodes; + int fc_modified_inodes_used, fc_modified_inodes_size; +}; + +#define region_last(__region) (((__region)->lblk) + ((__region)->len) - 1) +#endif + +static inline const char *tag2str(__u16 tag) +{ + switch (tag) { + case EXT4_FC_TAG_LINK: + return "ADD_ENTRY"; + case EXT4_FC_TAG_UNLINK: + return "DEL_ENTRY"; + case EXT4_FC_TAG_ADD_RANGE: + return "ADD_RANGE"; + case EXT4_FC_TAG_CREAT: + return "CREAT_DENTRY"; + case EXT4_FC_TAG_DEL_RANGE: + return "DEL_RANGE"; + case EXT4_FC_TAG_INODE: + return "INODE"; + case EXT4_FC_TAG_PAD: + return "PAD"; + case EXT4_FC_TAG_TAIL: + return "TAIL"; + case EXT4_FC_TAG_HEAD: + return "HEAD"; + default: + return "ERROR"; + } +} + +#endif /* __FAST_COMMIT_H__ */ diff --git a/fs/ext4/file.c b/fs/ext4/file.c new file mode 100644 index 000000000..374b4bc25 --- /dev/null +++ b/fs/ext4/file.c @@ -0,0 +1,1068 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext4/file.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * from + * + * linux/fs/minix/file.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * ext4 fs regular file handling primitives + * + * 64-bit file support on 64-bit platforms by Jakub Jelinek + * (jj@sunsite.ms.mff.cuni.cz) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ext4.h" +#include "ext4_jbd2.h" +#include "xattr.h" +#include "acl.h" +#include "truncate.h" + +/* + * Returns %true if the given DIO request should be attempted with DIO, or + * %false if it should fall back to buffered I/O. + * + * DIO isn't well specified; when it's unsupported (either due to the request + * being misaligned, or due to the file not supporting DIO at all), filesystems + * either fall back to buffered I/O or return EINVAL. For files that don't use + * any special features like encryption or verity, ext4 has traditionally + * returned EINVAL for misaligned DIO. iomap_dio_rw() uses this convention too. + * In this case, we should attempt the DIO, *not* fall back to buffered I/O. + * + * In contrast, in cases where DIO is unsupported due to ext4 features, ext4 + * traditionally falls back to buffered I/O. + * + * This function implements the traditional ext4 behavior in all these cases. + */ +static bool ext4_should_use_dio(struct kiocb *iocb, struct iov_iter *iter) +{ + struct inode *inode = file_inode(iocb->ki_filp); + u32 dio_align = ext4_dio_alignment(inode); + + if (dio_align == 0) + return false; + + if (dio_align == 1) + return true; + + return IS_ALIGNED(iocb->ki_pos | iov_iter_alignment(iter), dio_align); +} + +static ssize_t ext4_dio_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + ssize_t ret; + struct inode *inode = file_inode(iocb->ki_filp); + + if (iocb->ki_flags & IOCB_NOWAIT) { + if (!inode_trylock_shared(inode)) + return -EAGAIN; + } else { + inode_lock_shared(inode); + } + + if (!ext4_should_use_dio(iocb, to)) { + inode_unlock_shared(inode); + /* + * Fallback to buffered I/O if the operation being performed on + * the inode is not supported by direct I/O. The IOCB_DIRECT + * flag needs to be cleared here in order to ensure that the + * direct I/O path within generic_file_read_iter() is not + * taken. + */ + iocb->ki_flags &= ~IOCB_DIRECT; + return generic_file_read_iter(iocb, to); + } + + ret = iomap_dio_read_simple(iocb, to, ext4_iomap_begin); + if (ret == -ENOTBLK) + ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL, 0, NULL, 0); + inode_unlock_shared(inode); + + file_accessed(iocb->ki_filp); + return ret; +} + +#ifdef CONFIG_FS_DAX +static ssize_t ext4_dax_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct inode *inode = file_inode(iocb->ki_filp); + ssize_t ret; + + if (iocb->ki_flags & IOCB_NOWAIT) { + if (!inode_trylock_shared(inode)) + return -EAGAIN; + } else { + inode_lock_shared(inode); + } + /* + * Recheck under inode lock - at this point we are sure it cannot + * change anymore + */ + if (!IS_DAX(inode)) { + inode_unlock_shared(inode); + /* Fallback to buffered IO in case we cannot support DAX */ + return generic_file_read_iter(iocb, to); + } + ret = dax_iomap_rw(iocb, to, &ext4_iomap_ops); + inode_unlock_shared(inode); + + file_accessed(iocb->ki_filp); + return ret; +} +#endif + +static ssize_t ext4_file_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct inode *inode = file_inode(iocb->ki_filp); + + if (unlikely(ext4_forced_shutdown(inode->i_sb))) + return -EIO; + + if (!iov_iter_count(to)) + return 0; /* skip atime */ + +#ifdef CONFIG_FS_DAX + if (IS_DAX(inode)) + return ext4_dax_read_iter(iocb, to); +#endif + if (iocb->ki_flags & IOCB_DIRECT) + return ext4_dio_read_iter(iocb, to); + + return generic_file_read_iter(iocb, to); +} + +static ssize_t ext4_file_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, + size_t len, unsigned int flags) +{ + struct inode *inode = file_inode(in); + + if (unlikely(ext4_forced_shutdown(inode->i_sb))) + return -EIO; + return filemap_splice_read(in, ppos, pipe, len, flags); +} + +/* + * Called when an inode is released. Note that this is different + * from ext4_file_open: open gets called at every open, but release + * gets called only when /all/ the files are closed. + */ +static int ext4_release_file(struct inode *inode, struct file *filp) +{ + if (ext4_test_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE)) { + ext4_alloc_da_blocks(inode); + ext4_clear_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE); + } + /* if we are the last writer on the inode, drop the block reservation */ + if ((filp->f_mode & FMODE_WRITE) && + (atomic_read(&inode->i_writecount) == 1) && + !EXT4_I(inode)->i_reserved_data_blocks) { + down_write(&EXT4_I(inode)->i_data_sem); + ext4_discard_preallocations(inode); + up_write(&EXT4_I(inode)->i_data_sem); + } + if (is_dx(inode) && filp->private_data) + ext4_htree_free_dir_info(filp->private_data); + + return 0; +} + +/* + * This tests whether the IO in question is block-aligned or not. + * Ext4 utilizes unwritten extents when hole-filling during direct IO, and they + * are converted to written only after the IO is complete. Until they are + * mapped, these blocks appear as holes, so dio_zero_block() will assume that + * it needs to zero out portions of the start and/or end block. If 2 AIO + * threads are at work on the same unwritten block, they must be synchronized + * or one thread will zero the other's data, causing corruption. + */ +static bool +ext4_unaligned_io(struct inode *inode, struct iov_iter *from, loff_t pos) +{ + struct super_block *sb = inode->i_sb; + unsigned long blockmask = sb->s_blocksize - 1; + + if ((pos | iov_iter_alignment(from)) & blockmask) + return true; + + return false; +} + +static bool +ext4_extending_io(struct inode *inode, loff_t offset, size_t len) +{ + if (offset + len > i_size_read(inode) || + offset + len > EXT4_I(inode)->i_disksize) + return true; + return false; +} + +/* + * Does an unaligned DIO write require partial block zeroing? + * + * Partial block zeroing is performed only for the head and tail blocks + * when they are partially covered by the write and the underlying extent + * is a hole or unwritten. Middle blocks (fully covered by the write) + * are written as whole blocks without zeroing. + * + * When zeroing is required, two concurrent unaligned DIO writes to the + * same partial block can race and corrupt each other's data, so the + * caller must take the exclusive i_rwsem and drain in-flight DIO. When + * zeroing is not required, shared lock is safe -- block allocation and + * unwritten conversion for middle blocks are protected by i_data_sem + * and inode_dio_begin(). + */ +static bool ext4_dio_needs_zeroing(struct inode *inode, loff_t pos, loff_t len) +{ + struct ext4_map_blocks map; + unsigned int blkbits = inode->i_blkbits; + unsigned long blockmask = inode->i_sb->s_blocksize - 1; + bool head_partial, tail_partial; + ext4_lblk_t head_lblk, tail_lblk; + int err; + + if (pos + len > i_size_read(inode)) + return true; + + head_partial = (pos & blockmask) != 0; + tail_partial = ((pos + len) & blockmask) != 0; + head_lblk = pos >> blkbits; + tail_lblk = (pos + len - 1) >> blkbits; + + /* Check the head partial block. */ + if (head_partial) { + map.m_lblk = head_lblk; + map.m_len = tail_lblk - head_lblk + 1; + err = ext4_map_blocks(NULL, inode, &map, 0); + if (err <= 0 || !(map.m_flags & EXT4_MAP_MAPPED)) + return true; + /* If this mapping already covers the tail block, we're done. */ + if (!tail_partial || map.m_lblk + err > tail_lblk) + return false; + } + + /* Check the tail partial block. */ + if (tail_partial) { + map.m_lblk = tail_lblk; + map.m_len = 1; + err = ext4_map_blocks(NULL, inode, &map, 0); + if (err <= 0 || !(map.m_flags & EXT4_MAP_MAPPED)) + return true; + } + + return false; +} + +static ssize_t ext4_generic_write_checks(struct kiocb *iocb, + struct iov_iter *from) +{ + struct inode *inode = file_inode(iocb->ki_filp); + ssize_t ret; + + if (unlikely(IS_IMMUTABLE(inode))) + return -EPERM; + + ret = generic_write_checks(iocb, from); + if (ret <= 0) + return ret; + + /* + * If we have encountered a bitmap-format file, the size limit + * is smaller than s_maxbytes, which is for extent-mapped files. + */ + if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) { + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + + if (iocb->ki_pos >= sbi->s_bitmap_maxbytes) + return -EFBIG; + iov_iter_truncate(from, sbi->s_bitmap_maxbytes - iocb->ki_pos); + } + + return iov_iter_count(from); +} + +static ssize_t ext4_write_checks(struct kiocb *iocb, struct iov_iter *from) +{ + struct inode *inode = file_inode(iocb->ki_filp); + loff_t old_size = i_size_read(inode); + ssize_t ret, count; + + count = ext4_generic_write_checks(iocb, from); + if (count <= 0) + return count; + + ret = kiocb_modified(iocb); + if (ret) + return ret; + + /* + * If the position is beyond the EOF, it is necessary to zero out the + * partial block that beyond the existing EOF, as it may contains + * stale data written through mmap. + */ + if (iocb->ki_pos > old_size && !ext4_verity_in_progress(inode)) { + if (iocb->ki_flags & IOCB_NOWAIT) + return -EAGAIN; + + ret = ext4_block_zero_eof(inode, old_size, iocb->ki_pos); + if (ret) + return ret; + } + + return count; +} + +static ssize_t ext4_buffered_write_iter(struct kiocb *iocb, + struct iov_iter *from) +{ + ssize_t ret; + struct inode *inode = file_inode(iocb->ki_filp); + + if (iocb->ki_flags & IOCB_NOWAIT) + return -EOPNOTSUPP; + + inode_lock(inode); + + /* + * Prevent concurrent direct I/O and buffered I/O to the same file + * range. Wait for in-flight DIO to finish before dirtying pages. + */ + inode_dio_wait(inode); + + ret = ext4_write_checks(iocb, from); + if (ret <= 0) + goto out; + + ret = generic_perform_write(iocb, from); + +out: + inode_unlock(inode); + if (unlikely(ret <= 0)) + return ret; + return generic_write_sync(iocb, ret); +} + +static ssize_t ext4_handle_inode_extension(struct inode *inode, loff_t offset, + ssize_t written, ssize_t count) +{ + handle_t *handle; + + lockdep_assert_held_write(&inode->i_rwsem); + handle = ext4_journal_start(inode, EXT4_HT_INODE, 2); + if (IS_ERR(handle)) + return PTR_ERR(handle); + + if (ext4_update_inode_size(inode, offset + written)) { + int ret = ext4_mark_inode_dirty(handle, inode); + if (unlikely(ret)) { + ext4_journal_stop(handle); + return ret; + } + } + + if ((written == count) && inode->i_nlink) + ext4_orphan_del(handle, inode); + ext4_journal_stop(handle); + + return written; +} + +/* + * Clean up the inode after DIO or DAX extending write has completed and the + * inode size has been updated using ext4_handle_inode_extension(). + */ +static void ext4_inode_extension_cleanup(struct inode *inode, bool need_trunc) +{ + lockdep_assert_held_write(&inode->i_rwsem); + if (need_trunc) { + ext4_truncate_failed_write(inode); + /* + * If the truncate operation failed early, then the inode may + * still be on the orphan list. In that case, we need to try + * remove the inode from the in-memory linked list. + */ + if (inode->i_nlink) + ext4_orphan_del(NULL, inode); + return; + } + /* + * If i_disksize got extended either due to writeback of delalloc + * blocks or extending truncate while the DIO was running we could fail + * to cleanup the orphan list in ext4_handle_inode_extension(). Do it + * now. + */ + if (ext4_inode_orphan_tracked(inode) && inode->i_nlink) { + handle_t *handle = ext4_journal_start(inode, EXT4_HT_INODE, 2); + + if (IS_ERR(handle)) { + /* + * The write has successfully completed. Not much to + * do with the error here so just cleanup the orphan + * list and hope for the best. + */ + ext4_orphan_del(NULL, inode); + return; + } + ext4_orphan_del(handle, inode); + ext4_journal_stop(handle); + } +} + +static int ext4_dio_write_end_io(struct kiocb *iocb, ssize_t size, + int error, unsigned int flags) +{ + loff_t pos = iocb->ki_pos; + struct inode *inode = file_inode(iocb->ki_filp); + + + if (!error && size && (flags & IOMAP_DIO_UNWRITTEN) && + (iocb->ki_flags & IOCB_ATOMIC)) + error = ext4_convert_unwritten_extents_atomic(NULL, inode, pos, + size); + else if (!error && size && flags & IOMAP_DIO_UNWRITTEN) + error = ext4_convert_unwritten_extents(NULL, inode, pos, size, + NULL); + if (error) + return error; + /* + * Note that EXT4_I(inode)->i_disksize can get extended up to + * inode->i_size while the I/O was running due to writeback of delalloc + * blocks. But the code in ext4_iomap_alloc() is careful to use + * zeroed/unwritten extents if this is possible; thus we won't leave + * uninitialized blocks in a file even if we didn't succeed in writing + * as much as we intended. Also we can race with truncate or write + * expanding the file so we have to be a bit careful here. + */ + if (pos + size <= READ_ONCE(EXT4_I(inode)->i_disksize) && + pos + size <= i_size_read(inode)) + return 0; + error = ext4_handle_inode_extension(inode, pos, size, size); + return error < 0 ? error : 0; +} + +static const struct iomap_dio_ops ext4_dio_write_ops = { + .end_io = ext4_dio_write_end_io, +}; + +/* + * The intention here is to start with shared lock acquired then see if any + * condition requires an exclusive inode lock. If yes, then we restart the + * whole operation by releasing the shared lock and acquiring exclusive lock. + * + * The decision is layered, evaluated in this order: + * + * 1. If kiocb_modified() needs to update security info (!IS_NOSEC), upgrade + * to the exclusive lock -- the security update itself requires it, + * regardless of whether the write extends the file or is aligned. + * + * 2. If the write extends i_size or i_disksize, upgrade to the exclusive + * lock to safely update i_disksize and the orphan list, regardless of + * alignment. + * + * 3. Otherwise, for aligned non-extending writes, shared lock is always + * sufficient regardless of extent state (written, unwritten, or hole). + * truncate/punch_hole cannot run while we hold the shared i_rwsem + * (they need it exclusively); after we release it, inode_dio_begin() + * keeps their inode_dio_wait() blocked until in-flight bios complete. + * i_data_sem serializes concurrent extent tree modifications. + * + * 4. Otherwise, the write is unaligned and non-extending. Shared lock is + * safe unless the DIO layer needs to perform partial block zeroing -- + * i.e. the head or tail partial block sits on a hole or unwritten + * extent. In that case upgrade to the exclusive lock and drain + * in-flight DIO to avoid races with concurrent partial block zeroing. + */ +static ssize_t ext4_dio_write_checks(struct kiocb *iocb, struct iov_iter *from, + bool *ilock_shared, bool *extend, + int *dio_flags) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + loff_t offset; + size_t count; + ssize_t ret; + bool needs_zeroing = false; + +restart: + ret = ext4_generic_write_checks(iocb, from); + if (ret <= 0) + goto out; + + offset = iocb->ki_pos; + count = ret; + + *extend = ext4_extending_io(inode, offset, count); + + /* + * For unaligned writes, check whether partial block zeroing will be + * needed. If so, exclusive lock is required to serialize against + * concurrent DIO that could race with the zeroing. + * + * For aligned writes we skip this check entirely since allocation + * under shared lock is safe. + */ + if (ext4_unaligned_io(inode, from, offset)) + needs_zeroing = ext4_dio_needs_zeroing(inode, offset, count); + + /* Determine whether we need to upgrade to an exclusive lock. */ + if (*ilock_shared && + (!IS_NOSEC(inode) || *extend || needs_zeroing)) { + if (iocb->ki_flags & IOCB_NOWAIT) { + ret = -EAGAIN; + goto out; + } + inode_unlock_shared(inode); + *ilock_shared = false; + inode_lock(inode); + goto restart; + } + + /* + * Now that locking is settled, determine dio flags and exclusivity + * requirements. We don't use DIO_OVERWRITE_ONLY because we enforce + * behavior already. When holding the exclusive lock for a write that + * needs partial block zeroing or is extending the file, we must wait + * for the I/O to complete synchronously: + * + * - needs_zeroing: drain in-flight DIO whose end_io could race with + * our partial block zeroing, and force synchronous completion so we + * don't leave in-flight zeroing bios for the next writer to drain. + * + * - extend: the caller must update i_disksize after I/O completion, + * which requires the data to be on disk first. + */ + if (!*ilock_shared && (needs_zeroing || *extend)) { + if (iocb->ki_flags & IOCB_NOWAIT) { + ret = -EAGAIN; + goto out; + } + if (needs_zeroing) + inode_dio_wait(inode); + *dio_flags = IOMAP_DIO_FORCE_WAIT; + } + + ret = kiocb_modified(iocb); + if (ret < 0) + goto out; + + return count; +out: + if (*ilock_shared) + inode_unlock_shared(inode); + else + inode_unlock(inode); + return ret; +} + +static ssize_t ext4_dio_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + ssize_t ret; + handle_t *handle; + struct inode *inode = file_inode(iocb->ki_filp); + loff_t offset = iocb->ki_pos; + size_t count = iov_iter_count(from); + bool extend = false; + bool ilock_shared = true; + int dio_flags = 0; + + /* + * Quick check here without any i_rwsem lock to see if it is extending + * IO. A more reliable check is done in ext4_dio_write_checks() with + * proper locking in place. + */ + if (offset + count > i_size_read(inode)) + ilock_shared = false; + + if (iocb->ki_flags & IOCB_NOWAIT) { + if (ilock_shared) { + if (!inode_trylock_shared(inode)) + return -EAGAIN; + } else { + if (!inode_trylock(inode)) + return -EAGAIN; + } + } else { + if (ilock_shared) + inode_lock_shared(inode); + else + inode_lock(inode); + } + + /* Fallback to buffered I/O if the inode does not support direct I/O. */ + if (!ext4_should_use_dio(iocb, from)) { + if (ilock_shared) + inode_unlock_shared(inode); + else + inode_unlock(inode); + return ext4_buffered_write_iter(iocb, from); + } + + /* + * Prevent inline data from being created since we are going to allocate + * blocks for DIO. We know the inode does not currently have inline data + * because ext4_should_use_dio() checked for it, but we have to clear + * the state flag before the write checks because a lock cycle could + * introduce races with other writers. + */ + ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); + + ret = ext4_dio_write_checks(iocb, from, &ilock_shared, &extend, + &dio_flags); + if (ret <= 0) + return ret; + + offset = iocb->ki_pos; + count = ret; + + if (extend) { + handle = ext4_journal_start(inode, EXT4_HT_INODE, 2); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + goto out; + } + + ret = ext4_orphan_add(handle, inode); + ext4_journal_stop(handle); + if (ret) + goto out; + } + + ret = iomap_dio_rw(iocb, from, &ext4_iomap_ops, &ext4_dio_write_ops, + dio_flags, NULL, 0); + if (ret == -ENOTBLK) + ret = 0; + if (extend) { + /* + * We always perform extending DIO write synchronously so by + * now the IO is completed and ext4_handle_inode_extension() + * was called. Cleanup the inode in case of error or race with + * writeback of delalloc blocks. + */ + WARN_ON_ONCE(ret == -EIOCBQUEUED); + ext4_inode_extension_cleanup(inode, ret < 0); + } + +out: + if (ilock_shared) + inode_unlock_shared(inode); + else + inode_unlock(inode); + + if (ret >= 0 && iov_iter_count(from)) { + ssize_t err; + loff_t endbyte; + + /* + * There is no support for atomic writes on buffered-io yet, + * we should never fallback to buffered-io for DIO atomic + * writes. + */ + WARN_ON_ONCE(iocb->ki_flags & IOCB_ATOMIC); + + offset = iocb->ki_pos; + err = ext4_buffered_write_iter(iocb, from); + if (err < 0) + return err; + + /* + * We need to ensure that the pages within the page cache for + * the range covered by this I/O are written to disk and + * invalidated. This is in attempt to preserve the expected + * direct I/O semantics in the case we fallback to buffered I/O + * to complete off the I/O request. + */ + ret += err; + endbyte = offset + err - 1; + err = filemap_write_and_wait_range(iocb->ki_filp->f_mapping, + offset, endbyte); + if (!err) + invalidate_mapping_pages(iocb->ki_filp->f_mapping, + offset >> PAGE_SHIFT, + endbyte >> PAGE_SHIFT); + } + + return ret; +} + +#ifdef CONFIG_FS_DAX +static ssize_t +ext4_dax_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + ssize_t ret; + size_t count; + loff_t offset; + handle_t *handle; + bool extend = false; + struct inode *inode = file_inode(iocb->ki_filp); + + if (iocb->ki_flags & IOCB_NOWAIT) { + if (!inode_trylock(inode)) + return -EAGAIN; + } else { + inode_lock(inode); + } + + ret = ext4_write_checks(iocb, from); + if (ret <= 0) + goto out; + + offset = iocb->ki_pos; + count = iov_iter_count(from); + + if (offset + count > EXT4_I(inode)->i_disksize) { + if (iocb->ki_flags & IOCB_NOWAIT) { + ret = -EAGAIN; + goto out; + } + + handle = ext4_journal_start(inode, EXT4_HT_INODE, 2); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + goto out; + } + + ret = ext4_orphan_add(handle, inode); + if (ret) { + ext4_journal_stop(handle); + goto out; + } + + extend = true; + ext4_journal_stop(handle); + } + + ret = dax_iomap_rw(iocb, from, &ext4_iomap_ops); + + if (extend) { + ret = ext4_handle_inode_extension(inode, offset, ret, count); + ext4_inode_extension_cleanup(inode, ret < (ssize_t)count); + } +out: + inode_unlock(inode); + if (ret > 0) + ret = generic_write_sync(iocb, ret); + return ret; +} +#endif + +static ssize_t +ext4_file_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + int ret; + struct inode *inode = file_inode(iocb->ki_filp); + + ret = ext4_emergency_state(inode->i_sb); + if (unlikely(ret)) + return ret; + +#ifdef CONFIG_FS_DAX + if (IS_DAX(inode)) + return ext4_dax_write_iter(iocb, from); +#endif + + if (iocb->ki_flags & IOCB_ATOMIC) { + size_t len = iov_iter_count(from); + + if (len < EXT4_SB(inode->i_sb)->s_awu_min || + len > EXT4_SB(inode->i_sb)->s_awu_max) + return -EINVAL; + + ret = generic_atomic_write_valid(iocb, from); + if (ret) + return ret; + } + + if (iocb->ki_flags & IOCB_DIRECT) + return ext4_dio_write_iter(iocb, from); + else + return ext4_buffered_write_iter(iocb, from); +} + +#ifdef CONFIG_FS_DAX +static vm_fault_t ext4_dax_huge_fault(struct vm_fault *vmf, unsigned int order) +{ + int error = 0; + vm_fault_t result; + int retries = 0; + handle_t *handle = NULL; + struct inode *inode = file_inode(vmf->vma->vm_file); + struct super_block *sb = inode->i_sb; + + /* + * We have to distinguish real writes from writes which will result in a + * COW page; COW writes should *not* poke the journal (the file will not + * be changed). Doing so would cause unintended failures when mounted + * read-only. + * + * We check for VM_SHARED rather than vmf->cow_page since the latter is + * unset for order != 0 (i.e. only in do_cow_fault); for + * other sizes, dax_iomap_fault will handle splitting / fallback so that + * we eventually come back with a COW page. + */ + bool write = (vmf->flags & FAULT_FLAG_WRITE) && + (vmf->vma->vm_flags & VM_SHARED); + struct address_space *mapping = vmf->vma->vm_file->f_mapping; + unsigned long pfn; + + if (write) { + sb_start_pagefault(sb); + file_update_time(vmf->vma->vm_file); + filemap_invalidate_lock_shared(mapping); +retry: + handle = ext4_journal_start_sb(sb, EXT4_HT_WRITE_PAGE, + EXT4_DATA_TRANS_BLOCKS(sb)); + if (IS_ERR(handle)) { + filemap_invalidate_unlock_shared(mapping); + sb_end_pagefault(sb); + return VM_FAULT_SIGBUS; + } + } else { + filemap_invalidate_lock_shared(mapping); + } + result = dax_iomap_fault(vmf, order, &pfn, &error, &ext4_iomap_ops); + if (write) { + ext4_journal_stop(handle); + + if ((result & VM_FAULT_ERROR) && error == -ENOSPC && + ext4_should_retry_alloc(sb, &retries)) + goto retry; + /* Handling synchronous page fault? */ + if (result & VM_FAULT_NEEDDSYNC) + result = dax_finish_sync_fault(vmf, order, pfn); + filemap_invalidate_unlock_shared(mapping); + sb_end_pagefault(sb); + } else { + filemap_invalidate_unlock_shared(mapping); + } + + return result; +} + +static vm_fault_t ext4_dax_fault(struct vm_fault *vmf) +{ + return ext4_dax_huge_fault(vmf, 0); +} + +static const struct vm_operations_struct ext4_dax_vm_ops = { + .fault = ext4_dax_fault, + .huge_fault = ext4_dax_huge_fault, + .page_mkwrite = ext4_dax_fault, + .pfn_mkwrite = ext4_dax_fault, +}; +#else +#define ext4_dax_vm_ops ext4_file_vm_ops +#endif + +static const struct vm_operations_struct ext4_file_vm_ops = { + .fault = filemap_fault, + .map_pages = filemap_map_pages, + .page_mkwrite = ext4_page_mkwrite, +}; + +static int ext4_file_mmap_prepare(struct vm_area_desc *desc) +{ + int ret; + struct file *file = desc->file; + struct inode *inode = file->f_mapping->host; + struct dax_device *dax_dev = EXT4_SB(inode->i_sb)->s_daxdev; + + if (file->f_mode & FMODE_WRITE) + ret = ext4_emergency_state(inode->i_sb); + else + ret = ext4_forced_shutdown(inode->i_sb) ? -EIO : 0; + if (unlikely(ret)) + return ret; + + /* + * We don't support synchronous mappings for non-DAX files and + * for DAX files if underneath dax_device is not synchronous. + */ + if (!daxdev_mapping_supported(desc, file_inode(file), dax_dev)) + return -EOPNOTSUPP; + + file_accessed(file); + if (IS_DAX(file_inode(file))) { + desc->vm_ops = &ext4_dax_vm_ops; + vma_desc_set_flags(desc, VMA_HUGEPAGE_BIT); + } else { + desc->vm_ops = &ext4_file_vm_ops; + } + return 0; +} + +static int ext4_sample_last_mounted(struct super_block *sb, + struct vfsmount *mnt) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct path path; + char buf[64], *cp; + handle_t *handle; + int err; + + if (likely(ext4_test_mount_flag(sb, EXT4_MF_MNTDIR_SAMPLED))) + return 0; + + if (ext4_emergency_state(sb) || sb_rdonly(sb) || + !sb_start_intwrite_trylock(sb)) + return 0; + + ext4_set_mount_flag(sb, EXT4_MF_MNTDIR_SAMPLED); + /* + * Sample where the filesystem has been mounted and + * store it in the superblock for sysadmin convenience + * when trying to sort through large numbers of block + * devices or filesystem images. + */ + path.mnt = mnt; + path.dentry = mnt->mnt_root; + cp = d_path(&path, buf, sizeof(buf)); + err = 0; + if (IS_ERR(cp)) + goto out; + + handle = ext4_journal_start_sb(sb, EXT4_HT_MISC, 1); + err = PTR_ERR(handle); + if (IS_ERR(handle)) + goto out; + BUFFER_TRACE(sbi->s_sbh, "get_write_access"); + err = ext4_journal_get_write_access(handle, sb, sbi->s_sbh, + EXT4_JTR_NONE); + if (err) + goto out_journal; + lock_buffer(sbi->s_sbh); + strtomem_pad(sbi->s_es->s_last_mounted, cp, 0); + ext4_superblock_csum_set(sb); + unlock_buffer(sbi->s_sbh); + ext4_handle_dirty_metadata(handle, NULL, sbi->s_sbh); +out_journal: + ext4_journal_stop(handle); +out: + sb_end_intwrite(sb); + return err; +} + +static int ext4_file_open(struct inode *inode, struct file *filp) +{ + int ret; + + if (filp->f_mode & FMODE_WRITE) + ret = ext4_emergency_state(inode->i_sb); + else + ret = ext4_forced_shutdown(inode->i_sb) ? -EIO : 0; + if (unlikely(ret)) + return ret; + + ret = ext4_sample_last_mounted(inode->i_sb, filp->f_path.mnt); + if (ret) + return ret; + + ret = fscrypt_file_open(inode, filp); + if (ret) + return ret; + + ret = fsverity_file_open(inode, filp); + if (ret) + return ret; + + /* + * Set up the jbd2_inode if we are opening the inode for + * writing and the journal is present + */ + if (filp->f_mode & FMODE_WRITE) { + ret = ext4_inode_attach_jinode(inode); + if (ret < 0) + return ret; + } + + if (ext4_inode_can_atomic_write(inode)) + filp->f_mode |= FMODE_CAN_ATOMIC_WRITE; + + filp->f_mode |= FMODE_NOWAIT | FMODE_CAN_ODIRECT; + return dquot_file_open(inode, filp); +} + +/* + * ext4_llseek() handles both block-mapped and extent-mapped maxbytes values + * by calling generic_file_llseek_size() with the appropriate maxbytes + * value for each. + */ +loff_t ext4_llseek(struct file *file, loff_t offset, int whence) +{ + struct inode *inode = file->f_mapping->host; + loff_t maxbytes = ext4_get_maxbytes(inode); + + switch (whence) { + default: + return generic_file_llseek_size(file, offset, whence, + maxbytes, i_size_read(inode)); + case SEEK_HOLE: + inode_lock_shared(inode); + offset = iomap_seek_hole(inode, offset, + &ext4_iomap_report_ops); + inode_unlock_shared(inode); + break; + case SEEK_DATA: + inode_lock_shared(inode); + offset = iomap_seek_data(inode, offset, + &ext4_iomap_report_ops); + inode_unlock_shared(inode); + break; + } + + if (offset < 0) + return offset; + return vfs_setpos(file, offset, maxbytes); +} + +const struct file_operations ext4_file_operations = { + .llseek = ext4_llseek, + .read_iter = ext4_file_read_iter, + .write_iter = ext4_file_write_iter, + .iopoll = iocb_bio_iopoll, + .unlocked_ioctl = ext4_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = ext4_compat_ioctl, +#endif + .mmap_prepare = ext4_file_mmap_prepare, + .open = ext4_file_open, + .release = ext4_release_file, + .fsync = ext4_sync_file, + .get_unmapped_area = thp_get_unmapped_area, + .splice_read = ext4_file_splice_read, + .splice_write = iter_file_splice_write, + .fallocate = ext4_fallocate, + .fop_flags = FOP_MMAP_SYNC | FOP_BUFFER_RASYNC | + FOP_DIO_PARALLEL_WRITE | + FOP_DONTCACHE, + .setlease = generic_setlease, +}; + +const struct inode_operations ext4_file_inode_operations = { + .setattr = ext4_setattr, + .getattr = ext4_file_getattr, + .listxattr = ext4_listxattr, + .get_inode_acl = ext4_get_acl, + .set_acl = ext4_set_acl, + .fiemap = ext4_fiemap, + .fileattr_get = ext4_fileattr_get, + .fileattr_set = ext4_fileattr_set, +}; + diff --git a/fs/ext4/fsmap.c b/fs/ext4/fsmap.c new file mode 100644 index 000000000..8b1138d2d --- /dev/null +++ b/fs/ext4/fsmap.c @@ -0,0 +1,792 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2017 Oracle. All Rights Reserved. + * + * Author: Darrick J. Wong + */ +#include "ext4.h" +#include +#include "fsmap.h" +#include "mballoc.h" +#include +#include +#include + +/* Convert an ext4_fsmap to an fsmap. */ +void ext4_fsmap_from_internal(struct super_block *sb, struct fsmap *dest, + struct ext4_fsmap *src) +{ + dest->fmr_device = src->fmr_device; + dest->fmr_flags = src->fmr_flags; + dest->fmr_physical = src->fmr_physical << sb->s_blocksize_bits; + dest->fmr_owner = src->fmr_owner; + dest->fmr_offset = 0; + dest->fmr_length = src->fmr_length << sb->s_blocksize_bits; + dest->fmr_reserved[0] = 0; + dest->fmr_reserved[1] = 0; + dest->fmr_reserved[2] = 0; +} + +/* Convert an fsmap to an ext4_fsmap. */ +void ext4_fsmap_to_internal(struct super_block *sb, struct ext4_fsmap *dest, + struct fsmap *src) +{ + dest->fmr_device = src->fmr_device; + dest->fmr_flags = src->fmr_flags; + dest->fmr_physical = src->fmr_physical >> sb->s_blocksize_bits; + dest->fmr_owner = src->fmr_owner; + dest->fmr_length = src->fmr_length >> sb->s_blocksize_bits; +} + +/* getfsmap query state */ +struct ext4_getfsmap_info { + struct ext4_fsmap_head *gfi_head; + ext4_fsmap_format_t gfi_formatter; /* formatting fn */ + void *gfi_format_arg;/* format buffer */ + ext4_fsblk_t gfi_next_fsblk; /* next fsblock we expect */ + u32 gfi_dev; /* device id */ + ext4_group_t gfi_agno; /* bg number, if applicable */ + struct ext4_fsmap gfi_low; /* low rmap key */ + struct ext4_fsmap gfi_high; /* high rmap key */ + struct ext4_fsmap gfi_lastfree; /* free ext at end of last bg */ + struct list_head gfi_meta_list; /* fixed metadata list */ + bool gfi_last; /* last extent? */ +}; + +/* Associate a device with a getfsmap handler. */ +struct ext4_getfsmap_dev { + int (*gfd_fn)(struct super_block *sb, + struct ext4_fsmap *keys, + struct ext4_getfsmap_info *info); + u32 gfd_dev; +}; + +/* Compare two getfsmap device handlers. */ +static int ext4_getfsmap_dev_compare(const void *p1, const void *p2) +{ + const struct ext4_getfsmap_dev *d1 = p1; + const struct ext4_getfsmap_dev *d2 = p2; + + return d1->gfd_dev - d2->gfd_dev; +} + +/* Compare a record against our starting point */ +static bool ext4_getfsmap_rec_before_low_key(struct ext4_getfsmap_info *info, + struct ext4_fsmap *rec) +{ + return rec->fmr_physical + rec->fmr_length <= + info->gfi_low.fmr_physical; +} + +/* + * Format a reverse mapping for getfsmap, having translated rm_startblock + * into the appropriate daddr units. + */ +static int ext4_getfsmap_helper(struct super_block *sb, + struct ext4_getfsmap_info *info, + struct ext4_fsmap *rec) +{ + struct ext4_fsmap fmr; + struct ext4_sb_info *sbi = EXT4_SB(sb); + ext4_fsblk_t rec_fsblk = rec->fmr_physical; + ext4_group_t agno; + ext4_grpblk_t cno; + int error; + + if (fatal_signal_pending(current)) + return -EINTR; + + /* + * Filter out records that start before our startpoint, if the + * caller requested that. + */ + if (ext4_getfsmap_rec_before_low_key(info, rec)) { + rec_fsblk += rec->fmr_length; + if (info->gfi_next_fsblk < rec_fsblk) + info->gfi_next_fsblk = rec_fsblk; + return EXT4_QUERY_RANGE_CONTINUE; + } + + /* Are we just counting mappings? */ + if (info->gfi_head->fmh_count == 0) { + if (info->gfi_head->fmh_entries == UINT_MAX) + return EXT4_QUERY_RANGE_ABORT; + + if (rec_fsblk > info->gfi_next_fsblk) + info->gfi_head->fmh_entries++; + + if (info->gfi_last) + return EXT4_QUERY_RANGE_CONTINUE; + + info->gfi_head->fmh_entries++; + + rec_fsblk += rec->fmr_length; + if (info->gfi_next_fsblk < rec_fsblk) + info->gfi_next_fsblk = rec_fsblk; + return EXT4_QUERY_RANGE_CONTINUE; + } + + /* + * If the record starts past the last physical block we saw, + * then we've found a gap. Report the gap as being owned by + * whatever the caller specified is the missing owner. + */ + if (rec_fsblk > info->gfi_next_fsblk) { + if (info->gfi_head->fmh_entries >= info->gfi_head->fmh_count) + return EXT4_QUERY_RANGE_ABORT; + + ext4_get_group_no_and_offset(sb, info->gfi_next_fsblk, + &agno, &cno); + trace_ext4_fsmap_mapping(sb, info->gfi_dev, agno, + EXT4_C2B(sbi, cno), + rec_fsblk - info->gfi_next_fsblk, + EXT4_FMR_OWN_UNKNOWN); + + fmr.fmr_device = info->gfi_dev; + fmr.fmr_physical = info->gfi_next_fsblk; + fmr.fmr_owner = EXT4_FMR_OWN_UNKNOWN; + fmr.fmr_length = rec_fsblk - info->gfi_next_fsblk; + fmr.fmr_flags = FMR_OF_SPECIAL_OWNER; + error = info->gfi_formatter(&fmr, info->gfi_format_arg); + if (error) + return error; + info->gfi_head->fmh_entries++; + } + + if (info->gfi_last) + goto out; + + /* Fill out the extent we found */ + if (info->gfi_head->fmh_entries >= info->gfi_head->fmh_count) + return EXT4_QUERY_RANGE_ABORT; + + ext4_get_group_no_and_offset(sb, rec_fsblk, &agno, &cno); + trace_ext4_fsmap_mapping(sb, info->gfi_dev, agno, EXT4_C2B(sbi, cno), + rec->fmr_length, rec->fmr_owner); + + fmr.fmr_device = info->gfi_dev; + fmr.fmr_physical = rec_fsblk; + fmr.fmr_owner = rec->fmr_owner; + fmr.fmr_flags = FMR_OF_SPECIAL_OWNER; + fmr.fmr_length = rec->fmr_length; + error = info->gfi_formatter(&fmr, info->gfi_format_arg); + if (error) + return error; + info->gfi_head->fmh_entries++; + +out: + rec_fsblk += rec->fmr_length; + if (info->gfi_next_fsblk < rec_fsblk) + info->gfi_next_fsblk = rec_fsblk; + return EXT4_QUERY_RANGE_CONTINUE; +} + +static inline ext4_fsblk_t ext4_fsmap_next_pblk(struct ext4_fsmap *fmr) +{ + return fmr->fmr_physical + fmr->fmr_length; +} + +static int ext4_getfsmap_meta_helper(struct super_block *sb, + ext4_group_t agno, ext4_grpblk_t start, + ext4_grpblk_t len, void *priv) +{ + struct ext4_getfsmap_info *info = priv; + struct ext4_fsmap *p; + struct ext4_fsmap *tmp; + struct ext4_sb_info *sbi = EXT4_SB(sb); + ext4_fsblk_t fsb, fs_start, fs_end; + int error; + + fs_start = fsb = (EXT4_C2B(sbi, start) + + ext4_group_first_block_no(sb, agno)); + fs_end = fs_start + EXT4_C2B(sbi, len); + + /* + * Return relevant extents from the meta_list. We emit all extents that + * partially/fully overlap with the query range + */ + list_for_each_entry_safe(p, tmp, &info->gfi_meta_list, fmr_list) { + if (p->fmr_physical + p->fmr_length <= info->gfi_next_fsblk) { + list_del(&p->fmr_list); + kfree(p); + continue; + } + if (p->fmr_physical <= fs_end && + p->fmr_physical + p->fmr_length > fs_start) { + /* Emit the retained free extent record if present */ + if (info->gfi_lastfree.fmr_owner) { + error = ext4_getfsmap_helper(sb, info, + &info->gfi_lastfree); + if (error) + return error; + info->gfi_lastfree.fmr_owner = 0; + } + error = ext4_getfsmap_helper(sb, info, p); + if (error) + return error; + fsb = p->fmr_physical + p->fmr_length; + if (info->gfi_next_fsblk < fsb) + info->gfi_next_fsblk = fsb; + list_del(&p->fmr_list); + kfree(p); + continue; + } + } + if (info->gfi_next_fsblk < fsb) + info->gfi_next_fsblk = fsb; + + return 0; +} + + +/* Transform a blockgroup's free record into a fsmap */ +static int ext4_getfsmap_datadev_helper(struct super_block *sb, + ext4_group_t agno, ext4_grpblk_t start, + ext4_grpblk_t len, void *priv) +{ + struct ext4_fsmap irec; + struct ext4_getfsmap_info *info = priv; + struct ext4_fsmap *p; + struct ext4_fsmap *tmp; + struct ext4_sb_info *sbi = EXT4_SB(sb); + ext4_fsblk_t fsb; + ext4_fsblk_t fslen; + int error; + + fsb = (EXT4_C2B(sbi, start) + ext4_group_first_block_no(sb, agno)); + fslen = EXT4_C2B(sbi, len); + + /* If the retained free extent record is set... */ + if (info->gfi_lastfree.fmr_owner) { + /* ...and abuts this one, lengthen it and return. */ + if (ext4_fsmap_next_pblk(&info->gfi_lastfree) == fsb) { + info->gfi_lastfree.fmr_length += fslen; + return 0; + } + + /* + * There's a gap between the two free extents; emit the + * retained extent prior to merging the meta_list. + */ + error = ext4_getfsmap_helper(sb, info, &info->gfi_lastfree); + if (error) + return error; + info->gfi_lastfree.fmr_owner = 0; + } + + /* Merge in any relevant extents from the meta_list */ + list_for_each_entry_safe(p, tmp, &info->gfi_meta_list, fmr_list) { + if (p->fmr_physical + p->fmr_length <= info->gfi_next_fsblk) { + list_del(&p->fmr_list); + kfree(p); + } else if (p->fmr_physical < fsb) { + error = ext4_getfsmap_helper(sb, info, p); + if (error) + return error; + + list_del(&p->fmr_list); + kfree(p); + } + } + + irec.fmr_device = 0; + irec.fmr_physical = fsb; + irec.fmr_length = fslen; + irec.fmr_owner = EXT4_FMR_OWN_FREE; + irec.fmr_flags = 0; + + /* If this is a free extent at the end of a bg, buffer it. */ + if (ext4_fsmap_next_pblk(&irec) == + ext4_group_first_block_no(sb, agno + 1)) { + info->gfi_lastfree = irec; + return 0; + } + + /* Otherwise, emit it */ + return ext4_getfsmap_helper(sb, info, &irec); +} + +/* Execute a getfsmap query against the log device. */ +static int ext4_getfsmap_logdev(struct super_block *sb, struct ext4_fsmap *keys, + struct ext4_getfsmap_info *info) +{ + journal_t *journal = EXT4_SB(sb)->s_journal; + struct ext4_fsmap irec; + + /* Set up search keys */ + info->gfi_low = keys[0]; + info->gfi_low.fmr_length = 0; + + memset(&info->gfi_high, 0xFF, sizeof(info->gfi_high)); + + trace_ext4_fsmap_low_key(sb, info->gfi_dev, 0, + info->gfi_low.fmr_physical, + info->gfi_low.fmr_length, + info->gfi_low.fmr_owner); + + trace_ext4_fsmap_high_key(sb, info->gfi_dev, 0, + info->gfi_high.fmr_physical, + info->gfi_high.fmr_length, + info->gfi_high.fmr_owner); + + if (keys[0].fmr_physical > 0) + return 0; + + /* Fabricate an rmap entry for the external log device. */ + irec.fmr_physical = journal->j_blk_offset; + irec.fmr_length = journal->j_total_len; + irec.fmr_owner = EXT4_FMR_OWN_LOG; + irec.fmr_flags = 0; + + return ext4_getfsmap_helper(sb, info, &irec); +} + +/* Helper to fill out an ext4_fsmap. */ +static inline int ext4_getfsmap_fill(struct list_head *meta_list, + ext4_fsblk_t fsb, ext4_fsblk_t len, + uint64_t owner) +{ + struct ext4_fsmap *fsm; + + fsm = kmalloc_obj(*fsm, GFP_NOFS); + if (!fsm) + return -ENOMEM; + fsm->fmr_device = 0; + fsm->fmr_flags = 0; + fsm->fmr_physical = fsb; + fsm->fmr_owner = owner; + fsm->fmr_length = len; + list_add_tail(&fsm->fmr_list, meta_list); + + return 0; +} + +/* + * This function returns the number of file system metadata blocks at + * the beginning of a block group, including the reserved gdt blocks. + */ +static unsigned int ext4_getfsmap_find_sb(struct super_block *sb, + ext4_group_t agno, + struct list_head *meta_list) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + ext4_fsblk_t fsb = ext4_group_first_block_no(sb, agno); + ext4_fsblk_t len; + unsigned long first_meta_bg = le32_to_cpu(sbi->s_es->s_first_meta_bg); + unsigned long metagroup = agno / EXT4_DESC_PER_BLOCK(sb); + int error; + + /* Record the superblock. */ + if (ext4_bg_has_super(sb, agno)) { + error = ext4_getfsmap_fill(meta_list, fsb, 1, EXT4_FMR_OWN_FS); + if (error) + return error; + fsb++; + } + + /* Record the group descriptors. */ + len = ext4_bg_num_gdb(sb, agno); + if (!len) + return 0; + error = ext4_getfsmap_fill(meta_list, fsb, len, + EXT4_FMR_OWN_GDT); + if (error) + return error; + fsb += len; + + /* Reserved GDT blocks */ + if (!ext4_has_feature_meta_bg(sb) || metagroup < first_meta_bg) { + len = le16_to_cpu(sbi->s_es->s_reserved_gdt_blocks); + + /* + * mkfs.ext4 can set s_reserved_gdt_blocks as 0 in some cases, + * check for that. + */ + if (!len) + return 0; + + error = ext4_getfsmap_fill(meta_list, fsb, len, + EXT4_FMR_OWN_RESV_GDT); + if (error) + return error; + } + + return 0; +} + +/* Compare two fsmap items. */ +static int ext4_getfsmap_compare(void *priv, + const struct list_head *a, + const struct list_head *b) +{ + struct ext4_fsmap *fa; + struct ext4_fsmap *fb; + + fa = container_of(a, struct ext4_fsmap, fmr_list); + fb = container_of(b, struct ext4_fsmap, fmr_list); + if (fa->fmr_physical < fb->fmr_physical) + return -1; + else if (fa->fmr_physical > fb->fmr_physical) + return 1; + return 0; +} + +/* Merge adjacent extents of fixed metadata. */ +static void ext4_getfsmap_merge_fixed_metadata(struct list_head *meta_list) +{ + struct ext4_fsmap *p; + struct ext4_fsmap *prev = NULL; + struct ext4_fsmap *tmp; + + list_for_each_entry_safe(p, tmp, meta_list, fmr_list) { + if (!prev) { + prev = p; + continue; + } + + if (prev->fmr_owner == p->fmr_owner && + prev->fmr_physical + prev->fmr_length == p->fmr_physical) { + prev->fmr_length += p->fmr_length; + list_del(&p->fmr_list); + kfree(p); + } else + prev = p; + } +} + +/* Free a list of fixed metadata. */ +static void ext4_getfsmap_free_fixed_metadata(struct list_head *meta_list) +{ + struct ext4_fsmap *p; + struct ext4_fsmap *tmp; + + list_for_each_entry_safe(p, tmp, meta_list, fmr_list) { + list_del(&p->fmr_list); + kfree(p); + } +} + +/* Find all the fixed metadata in the filesystem. */ +static int ext4_getfsmap_find_fixed_metadata(struct super_block *sb, + struct list_head *meta_list) +{ + struct ext4_group_desc *gdp; + ext4_group_t agno; + int error; + + INIT_LIST_HEAD(meta_list); + + /* Collect everything. */ + for (agno = 0; agno < EXT4_SB(sb)->s_groups_count; agno++) { + gdp = ext4_get_group_desc(sb, agno, NULL); + if (!gdp) { + error = -EFSCORRUPTED; + goto err; + } + + /* Superblock & GDT */ + error = ext4_getfsmap_find_sb(sb, agno, meta_list); + if (error) + goto err; + + /* Block bitmap */ + error = ext4_getfsmap_fill(meta_list, + ext4_block_bitmap(sb, gdp), 1, + EXT4_FMR_OWN_BLKBM); + if (error) + goto err; + + /* Inode bitmap */ + error = ext4_getfsmap_fill(meta_list, + ext4_inode_bitmap(sb, gdp), 1, + EXT4_FMR_OWN_INOBM); + if (error) + goto err; + + /* Inodes */ + error = ext4_getfsmap_fill(meta_list, + ext4_inode_table(sb, gdp), + EXT4_SB(sb)->s_itb_per_group, + EXT4_FMR_OWN_INODES); + if (error) + goto err; + } + + /* Sort the list */ + list_sort(NULL, meta_list, ext4_getfsmap_compare); + + /* Merge adjacent extents */ + ext4_getfsmap_merge_fixed_metadata(meta_list); + + return 0; +err: + ext4_getfsmap_free_fixed_metadata(meta_list); + return error; +} + +/* Execute a getfsmap query against the buddy bitmaps */ +static int ext4_getfsmap_datadev(struct super_block *sb, + struct ext4_fsmap *keys, + struct ext4_getfsmap_info *info) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + ext4_fsblk_t start_fsb; + ext4_fsblk_t end_fsb; + ext4_fsblk_t bofs; + ext4_fsblk_t eofs; + ext4_group_t start_ag; + ext4_group_t end_ag; + ext4_grpblk_t first_cluster; + ext4_grpblk_t last_cluster; + struct ext4_fsmap irec; + int error = 0; + + bofs = le32_to_cpu(sbi->s_es->s_first_data_block); + eofs = ext4_blocks_count(sbi->s_es); + if (keys[0].fmr_physical >= eofs) + return 0; + else if (keys[0].fmr_physical < bofs) + keys[0].fmr_physical = bofs; + if (keys[1].fmr_physical >= eofs) + keys[1].fmr_physical = eofs - 1; + if (keys[1].fmr_physical < keys[0].fmr_physical) + return 0; + start_fsb = keys[0].fmr_physical; + end_fsb = keys[1].fmr_physical; + + /* Determine first and last group to examine based on start and end */ + ext4_get_group_no_and_offset(sb, start_fsb, &start_ag, &first_cluster); + ext4_get_group_no_and_offset(sb, end_fsb, &end_ag, &last_cluster); + + /* + * Convert the fsmap low/high keys to bg based keys. Initialize + * low to the fsmap low key and max out the high key to the end + * of the bg. + */ + info->gfi_low = keys[0]; + info->gfi_low.fmr_physical = EXT4_C2B(sbi, first_cluster); + info->gfi_low.fmr_length = 0; + + memset(&info->gfi_high, 0xFF, sizeof(info->gfi_high)); + + /* Assemble a list of all the fixed-location metadata. */ + error = ext4_getfsmap_find_fixed_metadata(sb, &info->gfi_meta_list); + if (error) + goto err; + + /* Query each bg */ + for (info->gfi_agno = start_ag; + info->gfi_agno <= end_ag; + info->gfi_agno++) { + /* + * Set the bg high key from the fsmap high key if this + * is the last bg that we're querying. + */ + if (info->gfi_agno == end_ag) { + info->gfi_high = keys[1]; + info->gfi_high.fmr_physical = EXT4_C2B(sbi, + last_cluster); + info->gfi_high.fmr_length = 0; + } + + trace_ext4_fsmap_low_key(sb, info->gfi_dev, info->gfi_agno, + info->gfi_low.fmr_physical, + info->gfi_low.fmr_length, + info->gfi_low.fmr_owner); + + trace_ext4_fsmap_high_key(sb, info->gfi_dev, info->gfi_agno, + info->gfi_high.fmr_physical, + info->gfi_high.fmr_length, + info->gfi_high.fmr_owner); + + error = ext4_mballoc_query_range(sb, info->gfi_agno, + EXT4_B2C(sbi, info->gfi_low.fmr_physical), + EXT4_B2C(sbi, info->gfi_high.fmr_physical), + ext4_getfsmap_meta_helper, + ext4_getfsmap_datadev_helper, info); + if (error) + goto err; + + /* + * Set the bg low key to the start of the bg prior to + * moving on to the next bg. + */ + if (info->gfi_agno == start_ag) + memset(&info->gfi_low, 0, sizeof(info->gfi_low)); + } + + /* Do we have a retained free extent? */ + if (info->gfi_lastfree.fmr_owner) { + error = ext4_getfsmap_helper(sb, info, &info->gfi_lastfree); + if (error) + goto err; + } + + /* + * The dummy record below will cause ext4_getfsmap_helper() to report + * any allocated blocks at the end of the range. + */ + irec.fmr_device = 0; + irec.fmr_physical = end_fsb + 1; + irec.fmr_length = 0; + irec.fmr_owner = EXT4_FMR_OWN_FREE; + irec.fmr_flags = 0; + + info->gfi_last = true; + error = ext4_getfsmap_helper(sb, info, &irec); + if (error) + goto err; + +err: + ext4_getfsmap_free_fixed_metadata(&info->gfi_meta_list); + return error; +} + +/* Do we recognize the device? */ +static bool ext4_getfsmap_is_valid_device(struct super_block *sb, + struct ext4_fsmap *fm) +{ + if (fm->fmr_device == 0 || fm->fmr_device == UINT_MAX || + fm->fmr_device == new_encode_dev(sb->s_bdev->bd_dev)) + return true; + if (EXT4_SB(sb)->s_journal_bdev_file && + fm->fmr_device == + new_encode_dev(file_bdev(EXT4_SB(sb)->s_journal_bdev_file)->bd_dev)) + return true; + return false; +} + +/* Ensure that the low key is less than the high key. */ +static bool ext4_getfsmap_check_keys(struct ext4_fsmap *low_key, + struct ext4_fsmap *high_key) +{ + if (low_key->fmr_device > high_key->fmr_device) + return false; + if (low_key->fmr_device < high_key->fmr_device) + return true; + + if (low_key->fmr_physical > high_key->fmr_physical) + return false; + if (low_key->fmr_physical < high_key->fmr_physical) + return true; + + if (low_key->fmr_owner > high_key->fmr_owner) + return false; + if (low_key->fmr_owner < high_key->fmr_owner) + return true; + + return false; +} + +#define EXT4_GETFSMAP_DEVS 2 +/* + * Get filesystem's extents as described in head, and format for + * output. Calls formatter to fill the user's buffer until all + * extents are mapped, until the passed-in head->fmh_count slots have + * been filled, or until the formatter short-circuits the loop, if it + * is tracking filled-in extents on its own. + * + * Key to Confusion + * ---------------- + * There are multiple levels of keys and counters at work here: + * _fsmap_head.fmh_keys -- low and high fsmap keys passed in; + * these reflect fs-wide block addrs. + * dkeys -- fmh_keys used to query each device; + * these are fmh_keys but w/ the low key + * bumped up by fmr_length. + * _getfsmap_info.gfi_next_fsblk-- next fs block we expect to see; this + * is how we detect gaps in the fsmap + * records and report them. + * _getfsmap_info.gfi_low/high -- per-bg low/high keys computed from + * dkeys; used to query the free space. + */ +int ext4_getfsmap(struct super_block *sb, struct ext4_fsmap_head *head, + ext4_fsmap_format_t formatter, void *arg) +{ + struct ext4_fsmap dkeys[2]; /* per-dev keys */ + struct ext4_getfsmap_dev handlers[EXT4_GETFSMAP_DEVS]; + struct ext4_getfsmap_info info = { NULL }; + int i; + int error = 0; + + if (head->fmh_iflags & ~FMH_IF_VALID) + return -EINVAL; + if (!ext4_getfsmap_is_valid_device(sb, &head->fmh_keys[0]) || + !ext4_getfsmap_is_valid_device(sb, &head->fmh_keys[1])) + return -EINVAL; + + head->fmh_entries = 0; + + /* Set up our device handlers. */ + memset(handlers, 0, sizeof(handlers)); + handlers[0].gfd_dev = new_encode_dev(sb->s_bdev->bd_dev); + handlers[0].gfd_fn = ext4_getfsmap_datadev; + if (EXT4_SB(sb)->s_journal_bdev_file) { + handlers[1].gfd_dev = new_encode_dev( + file_bdev(EXT4_SB(sb)->s_journal_bdev_file)->bd_dev); + handlers[1].gfd_fn = ext4_getfsmap_logdev; + } + + sort(handlers, EXT4_GETFSMAP_DEVS, sizeof(struct ext4_getfsmap_dev), + ext4_getfsmap_dev_compare, NULL); + + /* + * To continue where we left off, we allow userspace to use the + * last mapping from a previous call as the low key of the next. + * This is identified by a non-zero length in the low key. We + * have to increment the low key in this scenario to ensure we + * don't return the same mapping again, and instead return the + * very next mapping. + * + * Bump the physical offset as there can be no other mapping for + * the same physical block range. + */ + dkeys[0] = head->fmh_keys[0]; + dkeys[0].fmr_physical += dkeys[0].fmr_length; + dkeys[0].fmr_owner = 0; + dkeys[0].fmr_length = 0; + memset(&dkeys[1], 0xFF, sizeof(struct ext4_fsmap)); + + if (!ext4_getfsmap_check_keys(dkeys, &head->fmh_keys[1])) + return -EINVAL; + + info.gfi_next_fsblk = head->fmh_keys[0].fmr_physical + + head->fmh_keys[0].fmr_length; + info.gfi_formatter = formatter; + info.gfi_format_arg = arg; + info.gfi_head = head; + + /* For each device we support... */ + for (i = 0; i < EXT4_GETFSMAP_DEVS; i++) { + /* Is this device within the range the user asked for? */ + if (!handlers[i].gfd_fn) + continue; + if (head->fmh_keys[0].fmr_device > handlers[i].gfd_dev) + continue; + if (head->fmh_keys[1].fmr_device < handlers[i].gfd_dev) + break; + + /* + * If this device number matches the high key, we have + * to pass the high key to the handler to limit the + * query results. If the device number exceeds the + * low key, zero out the low key so that we get + * everything from the beginning. + */ + if (handlers[i].gfd_dev == head->fmh_keys[1].fmr_device) + dkeys[1] = head->fmh_keys[1]; + if (handlers[i].gfd_dev > head->fmh_keys[0].fmr_device) + memset(&dkeys[0], 0, sizeof(struct ext4_fsmap)); + + info.gfi_dev = handlers[i].gfd_dev; + info.gfi_last = false; + info.gfi_agno = -1; + error = handlers[i].gfd_fn(sb, dkeys, &info); + if (error) + break; + info.gfi_next_fsblk = 0; + } + + head->fmh_oflags = FMH_OF_DEV_T; + return error; +} diff --git a/fs/ext4/fsmap.h b/fs/ext4/fsmap.h new file mode 100644 index 000000000..ac642be23 --- /dev/null +++ b/fs/ext4/fsmap.h @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2017 Oracle. All Rights Reserved. + * + * Author: Darrick J. Wong + */ +#ifndef __EXT4_FSMAP_H__ +#define __EXT4_FSMAP_H__ + +struct fsmap; + +/* internal fsmap representation */ +struct ext4_fsmap { + struct list_head fmr_list; + dev_t fmr_device; /* device id */ + uint32_t fmr_flags; /* mapping flags */ + uint64_t fmr_physical; /* device offset of segment */ + uint64_t fmr_owner; /* owner id */ + uint64_t fmr_length; /* length of segment, blocks */ +}; + +struct ext4_fsmap_head { + uint32_t fmh_iflags; /* control flags */ + uint32_t fmh_oflags; /* output flags */ + unsigned int fmh_count; /* # of entries in array incl. input */ + unsigned int fmh_entries; /* # of entries filled in (output). */ + + struct ext4_fsmap fmh_keys[2]; /* low and high keys */ +}; + +void ext4_fsmap_from_internal(struct super_block *sb, struct fsmap *dest, + struct ext4_fsmap *src); +void ext4_fsmap_to_internal(struct super_block *sb, struct ext4_fsmap *dest, + struct fsmap *src); + +/* fsmap to userspace formatter - copy to user & advance pointer */ +typedef int (*ext4_fsmap_format_t)(struct ext4_fsmap *, void *); + +int ext4_getfsmap(struct super_block *sb, struct ext4_fsmap_head *head, + ext4_fsmap_format_t formatter, void *arg); + +#define EXT4_QUERY_RANGE_ABORT 1 +#define EXT4_QUERY_RANGE_CONTINUE 0 + +/* fmr_owner special values for FS_IOC_GETFSMAP; some share w/ XFS */ +#define EXT4_FMR_OWN_FREE FMR_OWN_FREE /* free space */ +#define EXT4_FMR_OWN_UNKNOWN FMR_OWN_UNKNOWN /* unknown owner */ +#define EXT4_FMR_OWN_FS FMR_OWNER('X', 1) /* static fs metadata */ +#define EXT4_FMR_OWN_LOG FMR_OWNER('X', 2) /* journalling log */ +#define EXT4_FMR_OWN_INODES FMR_OWNER('X', 5) /* inodes */ +#define EXT4_FMR_OWN_GDT FMR_OWNER('f', 1) /* group descriptors */ +#define EXT4_FMR_OWN_RESV_GDT FMR_OWNER('f', 2) /* reserved gdt blocks */ +#define EXT4_FMR_OWN_BLKBM FMR_OWNER('f', 3) /* block bitmap */ +#define EXT4_FMR_OWN_INOBM FMR_OWNER('f', 4) /* inode bitmap */ + +#endif /* __EXT4_FSMAP_H__ */ diff --git a/fs/ext4/fsync.c b/fs/ext4/fsync.c new file mode 100644 index 000000000..2999c2cc8 --- /dev/null +++ b/fs/ext4/fsync.c @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext4/fsync.c + * + * Copyright (C) 1993 Stephen Tweedie (sct@redhat.com) + * from + * Copyright (C) 1992 Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * from + * linux/fs/minix/truncate.c Copyright (C) 1991, 1992 Linus Torvalds + * + * ext4fs fsync primitive + * + * Big-endian to little-endian byte-swapping/bitmaps by + * David S. Miller (davem@caip.rutgers.edu), 1995 + * + * Removed unnecessary code duplication for little endian machines + * and excessive __inline__s. + * Andi Kleen, 1997 + * + * Major simplications and cleanup - we only need to do the metadata, because + * we can depend on generic_block_fdatasync() to sync the data blocks. + */ + +#include +#include +#include +#include +#include +#include + +#include "ext4.h" +#include "ext4_jbd2.h" + +#include + +/* + * If we're not journaling and this is a just-created file, we have to + * sync our parent directory (if it was freshly created) since + * otherwise it will only be written by writeback, leaving a huge + * window during which a crash may lose the file. This may apply for + * the parent directory's parent as well, and so on recursively, if + * they are also freshly created. + */ +static int ext4_sync_parent(struct inode *inode) +{ + struct dentry *dentry, *next; + int ret = 0; + + if (!ext4_test_inode_state(inode, EXT4_STATE_NEWENTRY)) + return 0; + dentry = d_find_any_alias(inode); + if (!dentry) + return 0; + while (ext4_test_inode_state(inode, EXT4_STATE_NEWENTRY)) { + ext4_clear_inode_state(inode, EXT4_STATE_NEWENTRY); + + next = dget_parent(dentry); + dput(dentry); + dentry = next; + inode = dentry->d_inode; + + /* + * The directory inode may have gone through rmdir by now. But + * the inode itself and its blocks are still allocated (we hold + * a reference to the inode via its dentry), so it didn't go + * through ext4_evict_inode()) and so we are safe to flush + * metadata blocks and the inode. + */ + ret = sync_inode_metadata(inode, 1); + if (ret) + break; + } + dput(dentry); + return ret; +} + +static int ext4_fsync_nojournal(struct file *file, loff_t start, loff_t end, + int datasync, bool *needs_barrier) +{ + struct inode *inode = file->f_inode; + int ret; + + ret = sync_inode_metadata(inode, 1); + if (ret) + return ret; + ret = ext4_sync_parent(inode); + + if (test_opt(inode->i_sb, BARRIER)) + *needs_barrier = true; + + return ret; +} + +static int ext4_fsync_journal(struct inode *inode, bool datasync, + bool *needs_barrier) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + journal_t *journal = EXT4_SB(inode->i_sb)->s_journal; + tid_t commit_tid = datasync ? ei->i_datasync_tid : ei->i_sync_tid; + + /* + * Fastcommit does not really support fsync on directories or other + * special files. Force a full commit. + */ + if (!S_ISREG(inode->i_mode)) + return ext4_force_commit(inode->i_sb); + + if (journal->j_flags & JBD2_BARRIER && + !jbd2_trans_will_send_data_barrier(journal, commit_tid)) + *needs_barrier = true; + + return ext4_fc_commit(journal, commit_tid); +} + +/* + * akpm: A new design for ext4_sync_file(). + * + * This is only called from sys_fsync(), sys_fdatasync() and sys_msync(). + * There cannot be a transaction open by this task. + * Another task could have dirtied this inode. Its data can be in any + * state in the journalling system. + * + * What we do is just kick off a commit and wait on it. This will snapshot the + * inode to disk. + */ +int ext4_sync_file(struct file *file, loff_t start, loff_t end, int datasync) +{ + int ret = 0, err; + bool needs_barrier = false; + struct inode *inode = file->f_mapping->host; + + ret = ext4_emergency_state(inode->i_sb); + if (unlikely(ret)) + return ret; + + ASSERT(ext4_journal_current_handle() == NULL); + + trace_ext4_sync_file_enter(file, datasync); + + if (sb_rdonly(inode->i_sb)) + goto out; + + ret = file_write_and_wait_range(file, start, end); + if (ret) + goto out; + + if (!EXT4_SB(inode->i_sb)->s_journal) { + ret = ext4_fsync_nojournal(file, start, end, datasync, + &needs_barrier); + if (needs_barrier) + goto issue_flush; + goto out; + } + + /* + * The caller's filemap_fdatawrite()/wait will sync the data. + * Metadata is in the journal, we wait for proper transaction to + * commit here. + */ + ret = ext4_fsync_journal(inode, datasync, &needs_barrier); + +issue_flush: + if (needs_barrier) { + err = blkdev_issue_flush(inode->i_sb->s_bdev); + if (!ret) + ret = err; + } +out: + err = file_check_and_advance_wb_err(file); + if (ret == 0) + ret = err; + trace_ext4_sync_file_exit(inode, ret); + return ret; +} diff --git a/fs/ext4/hash-test.c b/fs/ext4/hash-test.c new file mode 100644 index 000000000..49b0d874c --- /dev/null +++ b/fs/ext4/hash-test.c @@ -0,0 +1,567 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit tests for ext4 directory hash computation. + */ + +#include +#include +#include +#include +#include +#include +#include "ext4.h" + +static void ext4_hash_init_fake_dir(struct inode *dir, struct super_block *sb) +{ + memset(sb, 0, sizeof(*sb)); + memset(dir, 0, sizeof(*dir)); + dir->i_sb = sb; + strscpy(sb->s_id, "kunit-ext4", sizeof(sb->s_id)); +} + +static void ext4_hash_init_fake_dir_with_sbi(struct inode *dir, + struct super_block *sb, + struct ext4_sb_info *sbi) +{ + ext4_hash_init_fake_dir(dir, sb); + memset(sbi, 0, sizeof(*sbi)); + sb->s_fs_info = sbi; + sbi->s_sb = sb; +} + +#ifdef CONFIG_FS_ENCRYPTION +static const struct fscrypt_operations ext4_hash_test_cryptops = { + .inode_info_offs = + (int)offsetof(struct ext4_inode_info, i_crypt_info) - + (int)offsetof(struct ext4_inode_info, vfs_inode), +}; +#endif + +static void ext4_hash_init_fake_ext4_dir(struct ext4_inode_info *ei, + struct super_block *sb, + struct ext4_sb_info *sbi) +{ + struct inode *dir = &ei->vfs_inode; + + memset(sb, 0, sizeof(*sb)); + memset(ei, 0, sizeof(*ei)); + memset(sbi, 0, sizeof(*sbi)); + + strscpy(sb->s_id, "kunit-ext4", sizeof(sb->s_id)); + sb->s_fs_info = sbi; + sbi->s_sb = sb; + + dir->i_sb = sb; + dir->i_mode = S_IFDIR; + +#ifdef CONFIG_FS_ENCRYPTION + fscrypt_set_ops(sb, &ext4_hash_test_cryptops); +#endif +} + +struct ext4_dirhash_test_case { + const char *name; + u32 hash_version; + const char *input; + int len; + u32 seed[4]; + bool use_seed; + u32 expected_hash; + u32 expected_minor_hash; +}; + +static const struct ext4_dirhash_test_case ext4_dirhash_test_cases[] = { + { + .name = "legacy_abc", + .hash_version = DX_HASH_LEGACY, + .input = "abc", + .len = 3, + .use_seed = false, + .expected_hash = 0x75afd992, + .expected_minor_hash = 0x00000000, + }, + { + .name = "legacy_unsigned_abc", + .hash_version = DX_HASH_LEGACY_UNSIGNED, + .input = "abc", + .len = 3, + .use_seed = false, + .expected_hash = 0x75afd992, + .expected_minor_hash = 0x00000000, + }, + { + .name = "half_md4_abc", + .hash_version = DX_HASH_HALF_MD4, + .input = "abc", + .len = 3, + .use_seed = false, + .expected_hash = 0xd196a868, + .expected_minor_hash = 0xc420eb28, + }, + { + .name = "half_md4_unsigned_abc", + .hash_version = DX_HASH_HALF_MD4_UNSIGNED, + .input = "abc", + .len = 3, + .use_seed = false, + .expected_hash = 0xd196a868, + .expected_minor_hash = 0xc420eb28, + }, + { + .name = "tea_abc", + .hash_version = DX_HASH_TEA, + .input = "abc", + .len = 3, + .use_seed = false, + .expected_hash = 0xb1435ec4, + .expected_minor_hash = 0x3f7eaa0e, + }, + { + .name = "tea_unsigned_abc", + .hash_version = DX_HASH_TEA_UNSIGNED, + .input = "abc", + .len = 3, + .use_seed = false, + .expected_hash = 0xb1435ec4, + .expected_minor_hash = 0x3f7eaa0e, + }, + { + .name = "empty_half_md4", + .hash_version = DX_HASH_HALF_MD4, + .input = "", + .len = 0, + .use_seed = false, + .expected_hash = 0xefcdab88, + .expected_minor_hash = 0x98badcfe, + }, + { + .name = "half_md4_31bytes", + .hash_version = DX_HASH_HALF_MD4, + .input = "1234567890123456789012345678901", + .len = 31, + .use_seed = false, + .expected_hash = 0xc4db1f78, + .expected_minor_hash = 0xea23921b, + }, + { + .name = "half_md4_32bytes", + .hash_version = DX_HASH_HALF_MD4, + .input = "12345678901234567890123456789012", + .len = 32, + .use_seed = false, + .expected_hash = 0xfa6cc63e, + .expected_minor_hash = 0x2f77bd1c, + }, + { + .name = "half_md4_33bytes", + .hash_version = DX_HASH_HALF_MD4, + .input = "123456789012345678901234567890123", + .len = 33, + .use_seed = false, + .expected_hash = 0xdc0c2dec, + .expected_minor_hash = 0x5ca23365, + }, + { + .name = "half_md4_unsigned_31bytes", + .hash_version = DX_HASH_HALF_MD4_UNSIGNED, + .input = "1234567890123456789012345678901", + .len = 31, + .use_seed = false, + .expected_hash = 0xc4db1f78, + .expected_minor_hash = 0xea23921b, + }, + { + .name = "half_md4_unsigned_32bytes", + .hash_version = DX_HASH_HALF_MD4_UNSIGNED, + .input = "12345678901234567890123456789012", + .len = 32, + .use_seed = false, + .expected_hash = 0xfa6cc63e, + .expected_minor_hash = 0x2f77bd1c, + }, + { + .name = "half_md4_unsigned_33bytes", + .hash_version = DX_HASH_HALF_MD4_UNSIGNED, + .input = "123456789012345678901234567890123", + .len = 33, + .use_seed = false, + .expected_hash = 0xdc0c2dec, + .expected_minor_hash = 0x5ca23365, + }, + { + .name = "tea_15bytes", + .hash_version = DX_HASH_TEA, + .input = "123456789abcdef", + .len = 15, + .use_seed = false, + .expected_hash = 0xa562903a, + .expected_minor_hash = 0x6174a00f, + }, + { + .name = "tea_16bytes", + .hash_version = DX_HASH_TEA, + .input = "1234567890abcdef", + .len = 16, + .use_seed = false, + .expected_hash = 0x8449f258, + .expected_minor_hash = 0x49a16d46, + }, + { + .name = "tea_17bytes", + .hash_version = DX_HASH_TEA, + .input = "123456789abcdefgh", + .len = 17, + .use_seed = false, + .expected_hash = 0xf32ec10c, + .expected_minor_hash = 0x58ceae61, + }, + { + .name = "half_md4_seeded", + .hash_version = DX_HASH_HALF_MD4, + .input = "same-name", + .len = 9, + .seed = { 0x11111111, 0x22222222, 0x33333333, 0x44444444 }, + .use_seed = true, + .expected_hash = 0x8aebf604, + .expected_minor_hash = 0x66ce48fe, + }, + { + .name = "half_md4_non_ascii_signed", + .hash_version = DX_HASH_HALF_MD4, + .input = "\x80\x81\x82\x83\x84", + .len = 5, + .use_seed = false, + .expected_hash = 0x8bab0498, + .expected_minor_hash = 0xc326632d, + }, + { + .name = "half_md4_non_ascii_unsigned", + .hash_version = DX_HASH_HALF_MD4_UNSIGNED, + .input = "\x80\x81\x82\x83\x84", + .len = 5, + .use_seed = false, + .expected_hash = 0xbc48596e, + .expected_minor_hash = 0xde0fad41, + }, + { + .name = "tea_non_ascii_signed", + .hash_version = DX_HASH_TEA, + .input = "\x80\x81\x82\x83\x84", + .len = 5, + .use_seed = false, + .expected_hash = 0x21e3a154, + .expected_minor_hash = 0x90112c3d, + }, + { + .name = "tea_non_ascii_unsigned", + .hash_version = DX_HASH_TEA_UNSIGNED, + .input = "\x80\x81\x82\x83\x84", + .len = 5, + .use_seed = false, + .expected_hash = 0x9b648616, + .expected_minor_hash = 0x011dd507, + }, +}; + +static void test_ext4fs_dirhash_vectors(struct kunit *test) +{ + struct super_block *sb; + struct inode *dir; + int i; + + sb = kunit_kzalloc(test, sizeof(*sb), GFP_KERNEL); + dir = kunit_kzalloc(test, sizeof(*dir), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, sb); + KUNIT_ASSERT_NOT_NULL(test, dir); + + ext4_hash_init_fake_dir(dir, sb); + + for (i = 0; i < ARRAY_SIZE(ext4_dirhash_test_cases); i++) { + const struct ext4_dirhash_test_case *tc = + &ext4_dirhash_test_cases[i]; + struct dx_hash_info hinfo; + int ret; + + memset(&hinfo, 0, sizeof(hinfo)); + hinfo.hash_version = tc->hash_version; + hinfo.seed = tc->use_seed ? (u32 *)tc->seed : NULL; + + ret = ext4fs_dirhash(dir, tc->input, tc->len, &hinfo); + + KUNIT_ASSERT_EQ_MSG(test, ret, 0, "case=%s", tc->name); + KUNIT_EXPECT_EQ_MSG(test, hinfo.hash, tc->expected_hash, + "case=%s", tc->name); + KUNIT_EXPECT_EQ_MSG(test, hinfo.minor_hash, + tc->expected_minor_hash, + "case=%s", tc->name); + } +} + +static void test_ext4fs_dirhash_seed_changes_result(struct kunit *test) +{ + struct super_block *sb; + struct inode *dir; + u32 seed[4] = { 0x11111111, 0x22222222, 0x33333333, 0x44444444 }; + struct dx_hash_info plain = { + .hash_version = DX_HASH_HALF_MD4, + }; + struct dx_hash_info seeded = { + .hash_version = DX_HASH_HALF_MD4, + .seed = seed, + }; + int ret_plain, ret_seeded; + + sb = kunit_kzalloc(test, sizeof(*sb), GFP_KERNEL); + dir = kunit_kzalloc(test, sizeof(*dir), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, sb); + KUNIT_ASSERT_NOT_NULL(test, dir); + + ext4_hash_init_fake_dir(dir, sb); + + ret_plain = ext4fs_dirhash(dir, "same-name", 9, &plain); + ret_seeded = ext4fs_dirhash(dir, "same-name", 9, &seeded); + + KUNIT_ASSERT_EQ(test, ret_plain, 0); + KUNIT_ASSERT_EQ(test, ret_seeded, 0); + + KUNIT_EXPECT_TRUE(test, + plain.hash != seeded.hash || + plain.minor_hash != seeded.minor_hash); +} + +static void test_ext4fs_dirhash_invalid_version_returns_einval(struct kunit *test) +{ + struct super_block *sb; + struct inode *dir; + struct ext4_sb_info *sbi; + struct dx_hash_info hinfo = { + .hash = 0xdeadbeef, + .minor_hash = 0xcafebabe, + .hash_version = DX_HASH_LAST + 1, + }; + int ret; + + sb = kunit_kzalloc(test, sizeof(*sb), GFP_KERNEL); + dir = kunit_kzalloc(test, sizeof(*dir), GFP_KERNEL); + sbi = kunit_kzalloc(test, sizeof(*sbi), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, sb); + KUNIT_ASSERT_NOT_NULL(test, dir); + KUNIT_ASSERT_NOT_NULL(test, sbi); + + ext4_hash_init_fake_dir_with_sbi(dir, sb, sbi); + + ret = ext4fs_dirhash(dir, "abc", 3, &hinfo); + + KUNIT_EXPECT_EQ(test, ret, -EINVAL); + KUNIT_EXPECT_EQ(test, hinfo.hash, 0); + KUNIT_EXPECT_EQ(test, hinfo.minor_hash, 0); +} + +static void test_ext4fs_dirhash_siphash_without_key_returns_einval(struct kunit *test) +{ + struct super_block *sb; + struct ext4_inode_info *ei; + struct inode *dir; + struct ext4_sb_info *sbi; + struct dx_hash_info hinfo = { + .hash_version = DX_HASH_SIPHASH, + }; + int ret; + + sb = kunit_kzalloc(test, sizeof(*sb), GFP_KERNEL); + ei = kunit_kzalloc(test, sizeof(*ei), GFP_KERNEL); + sbi = kunit_kzalloc(test, sizeof(*sbi), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, sb); + KUNIT_ASSERT_NOT_NULL(test, ei); + KUNIT_ASSERT_NOT_NULL(test, sbi); + + ext4_hash_init_fake_ext4_dir(ei, sb, sbi); + dir = &ei->vfs_inode; + + ret = ext4fs_dirhash(dir, "name", strlen("name"), &hinfo); + + KUNIT_EXPECT_EQ(test, ret, -EINVAL); +} + +static void test_ext4fs_dirhash_signed_unsigned_differ_on_nonascii(struct kunit *test) +{ + struct super_block *sb; + struct inode *dir; + static const char input[] = "\x80\xff\x81\xfe\101bc"; + struct dx_hash_info legacy_signed = { + .hash_version = DX_HASH_LEGACY, + }; + struct dx_hash_info legacy_unsigned = { + .hash_version = DX_HASH_LEGACY_UNSIGNED, + }; + struct dx_hash_info md4_signed = { + .hash_version = DX_HASH_HALF_MD4, + }; + struct dx_hash_info md4_unsigned = { + .hash_version = DX_HASH_HALF_MD4_UNSIGNED, + }; + struct dx_hash_info tea_signed = { + .hash_version = DX_HASH_TEA, + }; + struct dx_hash_info tea_unsigned = { + .hash_version = DX_HASH_TEA_UNSIGNED, + }; + int ret; + + sb = kunit_kzalloc(test, sizeof(*sb), GFP_KERNEL); + dir = kunit_kzalloc(test, sizeof(*dir), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, sb); + KUNIT_ASSERT_NOT_NULL(test, dir); + + ext4_hash_init_fake_dir(dir, sb); + + ret = ext4fs_dirhash(dir, input, sizeof(input) - 1, &legacy_signed); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = ext4fs_dirhash(dir, input, sizeof(input) - 1, &legacy_unsigned); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_NE(test, legacy_signed.hash, legacy_unsigned.hash); + + ret = ext4fs_dirhash(dir, input, sizeof(input) - 1, &md4_signed); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = ext4fs_dirhash(dir, input, sizeof(input) - 1, &md4_unsigned); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, + md4_signed.hash != md4_unsigned.hash || + md4_signed.minor_hash != md4_unsigned.minor_hash); + + ret = ext4fs_dirhash(dir, input, sizeof(input) - 1, &tea_signed); + KUNIT_ASSERT_EQ(test, ret, 0); + ret = ext4fs_dirhash(dir, input, sizeof(input) - 1, &tea_unsigned); + KUNIT_ASSERT_EQ(test, ret, 0); + KUNIT_EXPECT_TRUE(test, + tea_signed.hash != tea_unsigned.hash || + tea_signed.minor_hash != tea_unsigned.minor_hash); +} + +#if IS_ENABLED(CONFIG_UNICODE) +KUNIT_DEFINE_ACTION_WRAPPER(utf8_unload_action, utf8_unload, + struct unicode_map *); +static void test_ext4fs_dirhash_casefolded_names_hash_consistently(struct kunit *test) +{ + struct super_block *sb; + struct ext4_inode_info *ei; + struct ext4_sb_info *sbi; + struct unicode_map *um; + struct dx_hash_info h1 = { + .hash_version = DX_HASH_HALF_MD4, + }; + struct dx_hash_info h2 = { + .hash_version = DX_HASH_HALF_MD4, + }; + int ret, ret1, ret2; + + sb = kunit_kzalloc(test, sizeof(*sb), GFP_KERNEL); + ei = kunit_kzalloc(test, sizeof(*ei), GFP_KERNEL); + sbi = kunit_kzalloc(test, sizeof(*sbi), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, sb); + KUNIT_ASSERT_NOT_NULL(test, ei); + KUNIT_ASSERT_NOT_NULL(test, sbi); + + um = utf8_load(UTF8_LATEST); + if (IS_ERR(um)) { + kunit_skip(test, "utf8_load(UTF8_LATEST) failed: %pe", + um); + return; + } + + ret = kunit_add_action_or_reset(test, utf8_unload_action, um); + KUNIT_ASSERT_EQ(test, ret, 0); + + ext4_hash_init_fake_ext4_dir(ei, sb, sbi); + sb->s_encoding = um; + ei->vfs_inode.i_flags |= S_CASEFOLD; + + KUNIT_ASSERT_TRUE(test, IS_CASEFOLDED(&ei->vfs_inode)); + + ret1 = ext4fs_dirhash(&ei->vfs_inode, "Alpha", 5, &h1); + ret2 = ext4fs_dirhash(&ei->vfs_inode, "aLPHa", 5, &h2); + + KUNIT_ASSERT_EQ(test, ret1, 0); + KUNIT_ASSERT_EQ(test, ret2, 0); + KUNIT_EXPECT_EQ(test, h1.hash, h2.hash); + KUNIT_EXPECT_EQ(test, h1.minor_hash, h2.minor_hash); +} + +static void test_ext4fs_dirhash_casefold_fallback(struct kunit *test) +{ + struct super_block *sb_cf, *sb_plain; + struct ext4_inode_info *ei; + struct ext4_sb_info *sbi; + struct inode *plain_dir; + struct unicode_map *um; + static const char invalid_utf8[] = "\xc3\x28"; + struct dx_hash_info folded_dir = { + .hash_version = DX_HASH_HALF_MD4, + }; + struct dx_hash_info plain = { + .hash_version = DX_HASH_HALF_MD4, + }; + int ret, ret_cf, ret_plain; + + sb_cf = kunit_kzalloc(test, sizeof(*sb_cf), GFP_KERNEL); + sb_plain = kunit_kzalloc(test, sizeof(*sb_plain), GFP_KERNEL); + ei = kunit_kzalloc(test, sizeof(*ei), GFP_KERNEL); + sbi = kunit_kzalloc(test, sizeof(*sbi), GFP_KERNEL); + plain_dir = kunit_kzalloc(test, sizeof(*plain_dir), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, sb_cf); + KUNIT_ASSERT_NOT_NULL(test, sb_plain); + KUNIT_ASSERT_NOT_NULL(test, ei); + KUNIT_ASSERT_NOT_NULL(test, sbi); + KUNIT_ASSERT_NOT_NULL(test, plain_dir); + + um = utf8_load(UTF8_LATEST); + if (IS_ERR(um)) { + kunit_skip(test, "utf8_load(UTF8_LATEST) failed: %pe", + um); + return; + } + + ret = kunit_add_action_or_reset(test, utf8_unload_action, um); + KUNIT_ASSERT_EQ(test, ret, 0); + + ext4_hash_init_fake_ext4_dir(ei, sb_cf, sbi); + sb_cf->s_encoding = um; + ei->vfs_inode.i_flags |= S_CASEFOLD; + + KUNIT_ASSERT_TRUE(test, IS_CASEFOLDED(&ei->vfs_inode)); + + ext4_hash_init_fake_dir(plain_dir, sb_plain); + + ret_cf = ext4fs_dirhash(&ei->vfs_inode, invalid_utf8, + sizeof(invalid_utf8) - 1, &folded_dir); + ret_plain = ext4fs_dirhash(plain_dir, invalid_utf8, + sizeof(invalid_utf8) - 1, &plain); + + KUNIT_ASSERT_EQ(test, ret_cf, 0); + KUNIT_ASSERT_EQ(test, ret_plain, 0); + KUNIT_EXPECT_EQ(test, folded_dir.hash, plain.hash); + KUNIT_EXPECT_EQ(test, folded_dir.minor_hash, plain.minor_hash); +} +#endif + +static struct kunit_case ext4_hash_test_cases[] = { + KUNIT_CASE(test_ext4fs_dirhash_vectors), + KUNIT_CASE(test_ext4fs_dirhash_seed_changes_result), + KUNIT_CASE(test_ext4fs_dirhash_invalid_version_returns_einval), + KUNIT_CASE(test_ext4fs_dirhash_siphash_without_key_returns_einval), + KUNIT_CASE(test_ext4fs_dirhash_signed_unsigned_differ_on_nonascii), +#if IS_ENABLED(CONFIG_UNICODE) + KUNIT_CASE(test_ext4fs_dirhash_casefolded_names_hash_consistently), + KUNIT_CASE(test_ext4fs_dirhash_casefold_fallback), +#endif + {} +}; + +static struct kunit_suite ext4_hash_test_suite = { + .name = "ext4_hash", + .test_cases = ext4_hash_test_cases, +}; + +kunit_test_suites(&ext4_hash_test_suite); + +MODULE_LICENSE("GPL"); diff --git a/fs/ext4/hash.c b/fs/ext4/hash.c new file mode 100644 index 000000000..978bd92da --- /dev/null +++ b/fs/ext4/hash.c @@ -0,0 +1,347 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext4/hash.c + * + * Copyright (C) 2002 by Theodore Ts'o + */ + +#include +#include +#include +#include +#include +#include "ext4.h" + +#define DELTA 0x9E3779B9 + +static void TEA_transform(__u32 buf[4], __u32 const in[]) +{ + __u32 sum = 0; + __u32 b0 = buf[0], b1 = buf[1]; + __u32 a = in[0], b = in[1], c = in[2], d = in[3]; + int n = 16; + + do { + sum += DELTA; + b0 += ((b1 << 4)+a) ^ (b1+sum) ^ ((b1 >> 5)+b); + b1 += ((b0 << 4)+c) ^ (b0+sum) ^ ((b0 >> 5)+d); + } while (--n); + + buf[0] += b0; + buf[1] += b1; +} + +/* F, G and H are basic MD4 functions: selection, majority, parity */ +#define F(x, y, z) ((z) ^ ((x) & ((y) ^ (z)))) +#define G(x, y, z) (((x) & (y)) + (((x) ^ (y)) & (z))) +#define H(x, y, z) ((x) ^ (y) ^ (z)) + +/* + * The generic round function. The application is so specific that + * we don't bother protecting all the arguments with parens, as is generally + * good macro practice, in favor of extra legibility. + * Rotation is separate from addition to prevent recomputation + */ +#define ROUND(f, a, b, c, d, x, s) \ + (a += f(b, c, d) + x, a = rol32(a, s)) +#define K1 0 +#define K2 013240474631UL +#define K3 015666365641UL + +/* + * Basic cut-down MD4 transform. Returns only 32 bits of result. + */ +static __u32 half_md4_transform(__u32 buf[4], __u32 const in[8]) +{ + __u32 a = buf[0], b = buf[1], c = buf[2], d = buf[3]; + + /* Round 1 */ + ROUND(F, a, b, c, d, in[0] + K1, 3); + ROUND(F, d, a, b, c, in[1] + K1, 7); + ROUND(F, c, d, a, b, in[2] + K1, 11); + ROUND(F, b, c, d, a, in[3] + K1, 19); + ROUND(F, a, b, c, d, in[4] + K1, 3); + ROUND(F, d, a, b, c, in[5] + K1, 7); + ROUND(F, c, d, a, b, in[6] + K1, 11); + ROUND(F, b, c, d, a, in[7] + K1, 19); + + /* Round 2 */ + ROUND(G, a, b, c, d, in[1] + K2, 3); + ROUND(G, d, a, b, c, in[3] + K2, 5); + ROUND(G, c, d, a, b, in[5] + K2, 9); + ROUND(G, b, c, d, a, in[7] + K2, 13); + ROUND(G, a, b, c, d, in[0] + K2, 3); + ROUND(G, d, a, b, c, in[2] + K2, 5); + ROUND(G, c, d, a, b, in[4] + K2, 9); + ROUND(G, b, c, d, a, in[6] + K2, 13); + + /* Round 3 */ + ROUND(H, a, b, c, d, in[3] + K3, 3); + ROUND(H, d, a, b, c, in[7] + K3, 9); + ROUND(H, c, d, a, b, in[2] + K3, 11); + ROUND(H, b, c, d, a, in[6] + K3, 15); + ROUND(H, a, b, c, d, in[1] + K3, 3); + ROUND(H, d, a, b, c, in[5] + K3, 9); + ROUND(H, c, d, a, b, in[0] + K3, 11); + ROUND(H, b, c, d, a, in[4] + K3, 15); + + buf[0] += a; + buf[1] += b; + buf[2] += c; + buf[3] += d; + + return buf[1]; /* "most hashed" word */ +} +#undef ROUND +#undef K1 +#undef K2 +#undef K3 +#undef F +#undef G +#undef H + +/* The old legacy hash */ +static __u32 dx_hack_hash_unsigned(const char *name, int len) +{ + __u32 hash, hash0 = 0x12a3fe2d, hash1 = 0x37abe8f9; + const unsigned char *ucp = (const unsigned char *) name; + + while (len--) { + hash = hash1 + (hash0 ^ (((int) *ucp++) * 7152373)); + + if (hash & 0x80000000) + hash -= 0x7fffffff; + hash1 = hash0; + hash0 = hash; + } + return hash0 << 1; +} + +static __u32 dx_hack_hash_signed(const char *name, int len) +{ + __u32 hash, hash0 = 0x12a3fe2d, hash1 = 0x37abe8f9; + const signed char *scp = (const signed char *) name; + + while (len--) { + hash = hash1 + (hash0 ^ (((int) *scp++) * 7152373)); + + if (hash & 0x80000000) + hash -= 0x7fffffff; + hash1 = hash0; + hash0 = hash; + } + return hash0 << 1; +} + +static void str2hashbuf_signed(const char *msg, int len, __u32 *buf, int num) +{ + __u32 pad, val; + int i; + const signed char *scp = (const signed char *) msg; + + pad = (__u32)len | ((__u32)len << 8); + pad |= pad << 16; + + if (len > num*4) + len = num * 4; + + while (len >= 4) { + val = ((__u32)scp[0] << 24) + ((__u32)scp[1] << 16) + ((__u32)scp[2] << 8) + scp[3]; + *buf++ = val; + scp += 4; + len -= 4; + num--; + } + + val = pad; + + for (i = 0; i < len; i++) + val = scp[i] + (val << 8); + + if (--num >= 0) + *buf++ = val; + + while (--num >= 0) + *buf++ = pad; + +} + +static void str2hashbuf_unsigned(const char *msg, int len, __u32 *buf, int num) +{ + __u32 pad, val; + int i; + const unsigned char *ucp = (const unsigned char *) msg; + + pad = (__u32)len | ((__u32)len << 8); + pad |= pad << 16; + + if (len > num*4) + len = num * 4; + + while (len >= 4) { + val = get_unaligned_be32(ucp); + *buf++ = val; + ucp += 4; + len -= 4; + num--; + } + + val = pad; + + for (i = 0; i < len; i++) + val = ucp[i] + (val << 8); + + if (--num >= 0) + *buf++ = val; + + while (--num >= 0) + *buf++ = pad; + +} + +/* + * Returns the hash of a filename. If len is 0 and name is NULL, then + * this function can be used to test whether or not a hash version is + * supported. + * + * The seed is an 4 longword (32 bits) "secret" which can be used to + * uniquify a hash. If the seed is all zero's, then some default seed + * may be used. + * + * A particular hash version specifies whether or not the seed is + * represented, and whether or not the returned hash is 32 bits or 64 + * bits. 32 bit hashes will return 0 for the minor hash. + */ +static int __ext4fs_dirhash(const struct inode *dir, const char *name, int len, + struct dx_hash_info *hinfo) +{ + __u32 hash; + __u32 minor_hash = 0; + const char *p; + int i; + __u32 in[8], buf[4]; + bool use_unsigned = false; + + /* Initialize the default seed for the hash checksum functions */ + buf[0] = 0x67452301; + buf[1] = 0xefcdab89; + buf[2] = 0x98badcfe; + buf[3] = 0x10325476; + + /* Check to see if the seed is all zero's */ + if (hinfo->seed) { + for (i = 0; i < 4; i++) { + if (hinfo->seed[i]) { + memcpy(buf, hinfo->seed, sizeof(buf)); + break; + } + } + } + + switch (hinfo->hash_version) { + case DX_HASH_LEGACY_UNSIGNED: + hash = dx_hack_hash_unsigned(name, len); + break; + case DX_HASH_LEGACY: + hash = dx_hack_hash_signed(name, len); + break; + case DX_HASH_HALF_MD4_UNSIGNED: + use_unsigned = true; + fallthrough; + case DX_HASH_HALF_MD4: + p = name; + while (len > 0) { + if (use_unsigned) + str2hashbuf_unsigned(p, len, in, 8); + else + str2hashbuf_signed(p, len, in, 8); + half_md4_transform(buf, in); + len -= 32; + p += 32; + } + minor_hash = buf[2]; + hash = buf[1]; + break; + case DX_HASH_TEA_UNSIGNED: + use_unsigned = true; + fallthrough; + case DX_HASH_TEA: + p = name; + while (len > 0) { + if (use_unsigned) + str2hashbuf_unsigned(p, len, in, 4); + else + str2hashbuf_signed(p, len, in, 4); + TEA_transform(buf, in); + len -= 16; + p += 16; + } + hash = buf[0]; + minor_hash = buf[1]; + break; + case DX_HASH_SIPHASH: + { + struct qstr qname = QSTR_INIT(name, len); + __u64 combined_hash; + + if (fscrypt_has_encryption_key(dir)) { + combined_hash = fscrypt_fname_siphash(dir, &qname); + } else { + ext4_warning_inode(dir, "Siphash requires key"); + return -EINVAL; + } + + hash = (__u32)(combined_hash >> 32); + minor_hash = (__u32)combined_hash; + break; + } + default: + hinfo->hash = 0; + hinfo->minor_hash = 0; + ext4_warning(dir->i_sb, + "invalid/unsupported hash tree version %u", + hinfo->hash_version); + return -EINVAL; + } + hash = hash & ~1; + if (hash == (EXT4_HTREE_EOF_32BIT << 1)) + hash = (EXT4_HTREE_EOF_32BIT - 1) << 1; + hinfo->hash = hash; + hinfo->minor_hash = minor_hash; + return 0; +} + +int ext4fs_dirhash(const struct inode *dir, const char *name, int len, + struct dx_hash_info *hinfo) +{ +#if IS_ENABLED(CONFIG_UNICODE) + const struct unicode_map *um = dir->i_sb->s_encoding; + int r, dlen; + unsigned char *buff; + struct qstr qstr = {.name = name, .len = len }; + + if (len && IS_CASEFOLDED(dir) && + (!IS_ENCRYPTED(dir) || fscrypt_has_encryption_key(dir))) { + buff = kzalloc(PATH_MAX, GFP_KERNEL); + if (!buff) + return -ENOMEM; + + dlen = utf8_casefold(um, &qstr, buff, PATH_MAX); + if (dlen < 0) { + kfree(buff); + goto opaque_seq; + } + + r = __ext4fs_dirhash(dir, buff, dlen, hinfo); + + kfree(buff); + return r; + } +opaque_seq: +#endif + return __ext4fs_dirhash(dir, name, len, hinfo); +} + +#if IS_ENABLED(CONFIG_EXT4_KUNIT_TESTS) +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4fs_dirhash); +#endif diff --git a/fs/ext4/ialloc.c b/fs/ext4/ialloc.c new file mode 100644 index 000000000..a5831fc53 --- /dev/null +++ b/fs/ext4/ialloc.c @@ -0,0 +1,1632 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext4/ialloc.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * BSD ufs-inspired inode and directory allocation by + * Stephen Tweedie (sct@redhat.com), 1993 + * Big-endian to little-endian byte-swapping/bitmaps by + * David S. Miller (davem@caip.rutgers.edu), 1995 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ext4.h" +#include "ext4_jbd2.h" +#include "xattr.h" +#include "acl.h" + +#include + +/* + * ialloc.c contains the inodes allocation and deallocation routines + */ + +/* + * The free inodes are managed by bitmaps. A file system contains several + * blocks groups. Each group contains 1 bitmap block for blocks, 1 bitmap + * block for inodes, N blocks for the inode table and data blocks. + * + * The file system contains group descriptors which are located after the + * super block. Each descriptor contains the number of the bitmap block and + * the free blocks count in the block. + */ + +/* + * To avoid calling the atomic setbit hundreds or thousands of times, we only + * need to use it within a single byte (to ensure we get endianness right). + * We can use memset for the rest of the bitmap as there are no other users. + */ +void ext4_mark_bitmap_end(int start_bit, int end_bit, char *bitmap) +{ + int i; + + if (start_bit >= end_bit) + return; + + ext4_debug("mark end bits +%d through +%d used\n", start_bit, end_bit); + for (i = start_bit; i < ((start_bit + 7) & ~7UL); i++) + ext4_set_bit(i, bitmap); + if (i < end_bit) + memset(bitmap + (i >> 3), 0xff, (end_bit - i) >> 3); +} + +void ext4_end_bitmap_read(struct bio *bio) +{ + struct buffer_head *bh; + bool uptodate = bio_endio_bh(bio, &bh); + + if (uptodate) { + set_buffer_uptodate(bh); + set_bitmap_uptodate(bh); + } + unlock_buffer(bh); +} + +static int ext4_validate_inode_bitmap(struct super_block *sb, + struct ext4_group_desc *desc, + ext4_group_t block_group, + struct buffer_head *bh) +{ + ext4_fsblk_t blk; + struct ext4_group_info *grp; + + if (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY) + return 0; + + if (buffer_verified(bh)) + return 0; + + grp = ext4_get_group_info(sb, block_group); + if (!grp || EXT4_MB_GRP_IBITMAP_CORRUPT(grp)) + return -EFSCORRUPTED; + + ext4_lock_group(sb, block_group); + if (buffer_verified(bh)) + goto verified; + blk = ext4_inode_bitmap(sb, desc); + if (!ext4_inode_bitmap_csum_verify(sb, desc, bh) || + ext4_simulate_fail(sb, EXT4_SIM_IBITMAP_CRC)) { + ext4_unlock_group(sb, block_group); + ext4_error(sb, "Corrupt inode bitmap - block_group = %u, " + "inode_bitmap = %llu", block_group, blk); + ext4_mark_group_bitmap_corrupted(sb, block_group, + EXT4_GROUP_INFO_IBITMAP_CORRUPT); + return -EFSBADCRC; + } + set_buffer_verified(bh); +verified: + ext4_unlock_group(sb, block_group); + return 0; +} + +/* + * Read the inode allocation bitmap for a given block_group, reading + * into the specified slot in the superblock's bitmap cache. + * + * Return buffer_head of bitmap on success, or an ERR_PTR on error. + */ +static struct buffer_head * +ext4_read_inode_bitmap(struct super_block *sb, ext4_group_t block_group) +{ + struct ext4_group_desc *desc; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct buffer_head *bh = NULL; + ext4_fsblk_t bitmap_blk; + int err; + + desc = ext4_get_group_desc(sb, block_group, NULL); + if (!desc) + return ERR_PTR(-EFSCORRUPTED); + + bitmap_blk = ext4_inode_bitmap(sb, desc); + if ((bitmap_blk <= le32_to_cpu(sbi->s_es->s_first_data_block)) || + (bitmap_blk >= ext4_blocks_count(sbi->s_es))) { + ext4_error(sb, "Invalid inode bitmap blk %llu in " + "block_group %u", bitmap_blk, block_group); + ext4_mark_group_bitmap_corrupted(sb, block_group, + EXT4_GROUP_INFO_IBITMAP_CORRUPT); + return ERR_PTR(-EFSCORRUPTED); + } + bh = sb_getblk(sb, bitmap_blk); + if (unlikely(!bh)) { + ext4_warning(sb, "Cannot read inode bitmap - " + "block_group = %u, inode_bitmap = %llu", + block_group, bitmap_blk); + return ERR_PTR(-ENOMEM); + } + if (bitmap_uptodate(bh)) + goto verify; + + lock_buffer(bh); + if (bitmap_uptodate(bh)) { + unlock_buffer(bh); + goto verify; + } + + ext4_lock_group(sb, block_group); + if (ext4_has_group_desc_csum(sb) && + (desc->bg_flags & cpu_to_le16(EXT4_BG_INODE_UNINIT))) { + if (block_group == 0) { + ext4_unlock_group(sb, block_group); + unlock_buffer(bh); + ext4_error(sb, "Inode bitmap for bg 0 marked " + "uninitialized"); + err = -EFSCORRUPTED; + goto out; + } + memset(bh->b_data, 0, (EXT4_INODES_PER_GROUP(sb) + 7) / 8); + ext4_mark_bitmap_end(EXT4_INODES_PER_GROUP(sb), + sb->s_blocksize * 8, bh->b_data); + set_bitmap_uptodate(bh); + set_buffer_uptodate(bh); + set_buffer_verified(bh); + ext4_unlock_group(sb, block_group); + unlock_buffer(bh); + return bh; + } + ext4_unlock_group(sb, block_group); + + if (buffer_uptodate(bh)) { + /* + * if not uninit if bh is uptodate, + * bitmap is also uptodate + */ + set_bitmap_uptodate(bh); + unlock_buffer(bh); + goto verify; + } + /* + * submit the buffer_head for reading + */ + trace_ext4_load_inode_bitmap(sb, block_group); + ext4_read_bh(bh, REQ_META | REQ_PRIO, + ext4_end_bitmap_read, + ext4_simulate_fail(sb, EXT4_SIM_IBITMAP_EIO)); + if (!buffer_uptodate(bh)) { + put_bh(bh); + ext4_error_err(sb, EIO, "Cannot read inode bitmap - " + "block_group = %u, inode_bitmap = %llu", + block_group, bitmap_blk); + ext4_mark_group_bitmap_corrupted(sb, block_group, + EXT4_GROUP_INFO_IBITMAP_CORRUPT); + return ERR_PTR(-EIO); + } + +verify: + err = ext4_validate_inode_bitmap(sb, desc, block_group, bh); + if (err) + goto out; + return bh; +out: + put_bh(bh); + return ERR_PTR(err); +} + +/* + * NOTE! When we get the inode, we're the only people + * that have access to it, and as such there are no + * race conditions we have to worry about. The inode + * is not on the hash-lists, and it cannot be reached + * through the filesystem because the directory entry + * has been deleted earlier. + * + * HOWEVER: we must make sure that we get no aliases, + * which means that we have to call "clear_inode()" + * _before_ we mark the inode not in use in the inode + * bitmaps. Otherwise a newly created file might use + * the same inode number (not actually the same pointer + * though), and then we'd have two inodes sharing the + * same inode number and space on the harddisk. + */ +void ext4_free_inode(handle_t *handle, struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + int is_directory; + unsigned long ino; + struct buffer_head *bitmap_bh = NULL; + struct buffer_head *bh2; + ext4_group_t block_group; + unsigned long bit; + struct ext4_group_desc *gdp; + struct ext4_super_block *es; + struct ext4_sb_info *sbi; + int fatal = 0, err, count, cleared; + struct ext4_group_info *grp; + + if (!sb) { + printk(KERN_ERR "EXT4-fs: %s:%d: inode on " + "nonexistent device\n", __func__, __LINE__); + return; + } + if (icount_read_once(inode) > 1) { + ext4_msg(sb, KERN_ERR, "%s:%d: inode #%llu: count=%d", + __func__, __LINE__, inode->i_ino, + icount_read_once(inode)); + return; + } + if (inode->i_nlink) { + ext4_msg(sb, KERN_ERR, "%s:%d: inode #%llu: nlink=%d\n", + __func__, __LINE__, inode->i_ino, inode->i_nlink); + return; + } + sbi = EXT4_SB(sb); + + ino = inode->i_ino; + ext4_debug("freeing inode %lu\n", ino); + trace_ext4_free_inode(inode); + + dquot_initialize(inode); + dquot_free_inode(inode); + + is_directory = S_ISDIR(inode->i_mode); + + /* Do this BEFORE marking the inode not in use or returning an error */ + ext4_clear_inode(inode); + + es = sbi->s_es; + if (ino < EXT4_FIRST_INO(sb) || ino > le32_to_cpu(es->s_inodes_count)) { + ext4_error(sb, "reserved or nonexistent inode %lu", ino); + goto error_return; + } + block_group = (ino - 1) / EXT4_INODES_PER_GROUP(sb); + bit = (ino - 1) % EXT4_INODES_PER_GROUP(sb); + bitmap_bh = ext4_read_inode_bitmap(sb, block_group); + /* Don't bother if the inode bitmap is corrupt. */ + if (IS_ERR(bitmap_bh)) { + fatal = PTR_ERR(bitmap_bh); + bitmap_bh = NULL; + goto error_return; + } + if (!(sbi->s_mount_state & EXT4_FC_REPLAY)) { + grp = ext4_get_group_info(sb, block_group); + if (!grp || unlikely(EXT4_MB_GRP_IBITMAP_CORRUPT(grp))) { + fatal = -EFSCORRUPTED; + goto error_return; + } + } + + BUFFER_TRACE(bitmap_bh, "get_write_access"); + fatal = ext4_journal_get_write_access(handle, sb, bitmap_bh, + EXT4_JTR_NONE); + if (fatal) + goto error_return; + + fatal = -ESRCH; + gdp = ext4_get_group_desc(sb, block_group, &bh2); + if (gdp) { + BUFFER_TRACE(bh2, "get_write_access"); + fatal = ext4_journal_get_write_access(handle, sb, bh2, + EXT4_JTR_NONE); + } + ext4_lock_group(sb, block_group); + cleared = ext4_test_and_clear_bit(bit, bitmap_bh->b_data); + if (fatal || !cleared) { + ext4_unlock_group(sb, block_group); + goto out; + } + + count = ext4_free_inodes_count(sb, gdp) + 1; + ext4_free_inodes_set(sb, gdp, count); + if (is_directory) { + count = ext4_used_dirs_count(sb, gdp) - 1; + ext4_used_dirs_set(sb, gdp, count); + if (percpu_counter_initialized(&sbi->s_dirs_counter)) + percpu_counter_dec(&sbi->s_dirs_counter); + } + ext4_inode_bitmap_csum_set(sb, gdp, bitmap_bh); + ext4_group_desc_csum_set(sb, block_group, gdp); + ext4_unlock_group(sb, block_group); + + if (percpu_counter_initialized(&sbi->s_freeinodes_counter)) + percpu_counter_inc(&sbi->s_freeinodes_counter); + if (sbi->s_log_groups_per_flex) { + struct flex_groups *fg; + + fg = sbi_array_rcu_deref(sbi, s_flex_groups, + ext4_flex_group(sbi, block_group)); + atomic_inc(&fg->free_inodes); + if (is_directory) + atomic_dec(&fg->used_dirs); + } + BUFFER_TRACE(bh2, "call ext4_handle_dirty_metadata"); + fatal = ext4_handle_dirty_metadata(handle, NULL, bh2); +out: + if (cleared) { + BUFFER_TRACE(bitmap_bh, "call ext4_handle_dirty_metadata"); + err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); + if (!fatal) + fatal = err; + } else { + ext4_error(sb, "bit already cleared for inode %lu", ino); + ext4_mark_group_bitmap_corrupted(sb, block_group, + EXT4_GROUP_INFO_IBITMAP_CORRUPT); + } + +error_return: + brelse(bitmap_bh); + ext4_std_error(sb, fatal); +} + +struct orlov_stats { + __u64 free_clusters; + __u32 free_inodes; + __u32 used_dirs; +}; + +/* + * Helper function for Orlov's allocator; returns critical information + * for a particular block group or flex_bg. If flex_size is 1, then g + * is a block group number; otherwise it is flex_bg number. + */ +static void get_orlov_stats(struct super_block *sb, ext4_group_t g, + int flex_size, struct orlov_stats *stats) +{ + struct ext4_group_desc *desc; + + if (flex_size > 1) { + struct flex_groups *fg = sbi_array_rcu_deref(EXT4_SB(sb), + s_flex_groups, g); + stats->free_inodes = atomic_read(&fg->free_inodes); + stats->free_clusters = atomic64_read(&fg->free_clusters); + stats->used_dirs = atomic_read(&fg->used_dirs); + return; + } + + desc = ext4_get_group_desc(sb, g, NULL); + if (desc) { + stats->free_inodes = ext4_free_inodes_count(sb, desc); + stats->free_clusters = ext4_free_group_clusters(sb, desc); + stats->used_dirs = ext4_used_dirs_count(sb, desc); + } else { + stats->free_inodes = 0; + stats->free_clusters = 0; + stats->used_dirs = 0; + } +} + +/* + * Orlov's allocator for directories. + * + * We always try to spread first-level directories. + * + * If there are blockgroups with both free inodes and free clusters counts + * not worse than average we return one with smallest directory count. + * Otherwise we simply return a random group. + * + * For the rest rules look so: + * + * It's OK to put directory into a group unless + * it has too many directories already (max_dirs) or + * it has too few free inodes left (min_inodes) or + * it has too few free clusters left (min_clusters) or + * Parent's group is preferred, if it doesn't satisfy these + * conditions we search cyclically through the rest. If none + * of the groups look good we just look for a group with more + * free inodes than average (starting at parent's group). + */ + +static int find_group_orlov(struct super_block *sb, struct inode *parent, + ext4_group_t *group, umode_t mode, + const struct qstr *qstr) +{ + ext4_group_t parent_group = EXT4_I(parent)->i_block_group; + struct ext4_sb_info *sbi = EXT4_SB(sb); + ext4_group_t real_ngroups = ext4_get_groups_count(sb); + int inodes_per_group = EXT4_INODES_PER_GROUP(sb); + unsigned int freei, avefreei, grp_free; + ext4_fsblk_t freec, avefreec; + unsigned int ndirs; + int max_dirs, min_inodes; + ext4_grpblk_t min_clusters; + ext4_group_t i, grp, g, ngroups; + struct ext4_group_desc *desc; + struct orlov_stats stats; + int flex_size = ext4_flex_bg_size(sbi); + struct dx_hash_info hinfo; + + ngroups = real_ngroups; + if (flex_size > 1) { + ngroups = (real_ngroups + flex_size - 1) >> + sbi->s_log_groups_per_flex; + parent_group >>= sbi->s_log_groups_per_flex; + } + + freei = percpu_counter_read_positive(&sbi->s_freeinodes_counter); + avefreei = freei / ngroups; + freec = percpu_counter_read_positive(&sbi->s_freeclusters_counter); + avefreec = freec; + do_div(avefreec, ngroups); + ndirs = percpu_counter_read_positive(&sbi->s_dirs_counter); + + if (S_ISDIR(mode) && + ((parent == d_inode(sb->s_root)) || + (ext4_test_inode_flag(parent, EXT4_INODE_TOPDIR)))) { + int best_ndir = inodes_per_group; + int ret = -1; + + if (qstr) { + hinfo.hash_version = DX_HASH_HALF_MD4; + hinfo.seed = sbi->s_hash_seed; + ext4fs_dirhash(parent, qstr->name, qstr->len, &hinfo); + parent_group = hinfo.hash % ngroups; + } else + parent_group = get_random_u32_below(ngroups); + for (i = 0; i < ngroups; i++) { + g = (parent_group + i) % ngroups; + get_orlov_stats(sb, g, flex_size, &stats); + if (!stats.free_inodes) + continue; + if (stats.used_dirs >= best_ndir) + continue; + if (stats.free_inodes < avefreei) + continue; + if (stats.free_clusters < avefreec) + continue; + grp = g; + ret = 0; + best_ndir = stats.used_dirs; + } + if (ret) + goto fallback; + found_flex_bg: + if (flex_size == 1) { + *group = grp; + return 0; + } + + /* + * We pack inodes at the beginning of the flexgroup's + * inode tables. Block allocation decisions will do + * something similar, although regular files will + * start at 2nd block group of the flexgroup. See + * ext4_ext_find_goal() and ext4_find_near(). + */ + grp *= flex_size; + for (i = 0; i < flex_size; i++) { + if (grp+i >= real_ngroups) + break; + desc = ext4_get_group_desc(sb, grp+i, NULL); + if (desc && ext4_free_inodes_count(sb, desc)) { + *group = grp+i; + return 0; + } + } + goto fallback; + } + + max_dirs = ndirs / ngroups + inodes_per_group*flex_size / 16; + min_inodes = avefreei - inodes_per_group*flex_size / 4; + if (min_inodes < 1) + min_inodes = 1; + min_clusters = avefreec - EXT4_CLUSTERS_PER_GROUP(sb)*flex_size / 4; + if (min_clusters < 0) + min_clusters = 0; + + /* + * Start looking in the flex group where we last allocated an + * inode for this parent directory + */ + if (EXT4_I(parent)->i_last_alloc_group != ~0) { + parent_group = EXT4_I(parent)->i_last_alloc_group; + if (flex_size > 1) + parent_group >>= sbi->s_log_groups_per_flex; + } + + for (i = 0; i < ngroups; i++) { + grp = (parent_group + i) % ngroups; + get_orlov_stats(sb, grp, flex_size, &stats); + if (stats.used_dirs >= max_dirs) + continue; + if (stats.free_inodes < min_inodes) + continue; + if (stats.free_clusters < min_clusters) + continue; + goto found_flex_bg; + } + +fallback: + ngroups = real_ngroups; + avefreei = freei / ngroups; +fallback_retry: + parent_group = EXT4_I(parent)->i_block_group; + for (i = 0; i < ngroups; i++) { + grp = (parent_group + i) % ngroups; + desc = ext4_get_group_desc(sb, grp, NULL); + if (desc) { + grp_free = ext4_free_inodes_count(sb, desc); + if (grp_free && grp_free >= avefreei) { + *group = grp; + return 0; + } + } + } + + if (avefreei) { + /* + * The free-inodes counter is approximate, and for really small + * filesystems the above test can fail to find any blockgroups + */ + avefreei = 0; + goto fallback_retry; + } + + return -1; +} + +static int find_group_other(struct super_block *sb, struct inode *parent, + ext4_group_t *group, umode_t mode) +{ + ext4_group_t parent_group = EXT4_I(parent)->i_block_group; + ext4_group_t i, last, ngroups = ext4_get_groups_count(sb); + struct ext4_group_desc *desc; + int flex_size = ext4_flex_bg_size(EXT4_SB(sb)); + + /* + * Try to place the inode is the same flex group as its + * parent. If we can't find space, use the Orlov algorithm to + * find another flex group, and store that information in the + * parent directory's inode information so that use that flex + * group for future allocations. + */ + if (flex_size > 1) { + int retry = 0; + + try_again: + parent_group &= ~(flex_size-1); + last = parent_group + flex_size; + if (last > ngroups) + last = ngroups; + for (i = parent_group; i < last; i++) { + desc = ext4_get_group_desc(sb, i, NULL); + if (desc && ext4_free_inodes_count(sb, desc)) { + *group = i; + return 0; + } + } + if (!retry && EXT4_I(parent)->i_last_alloc_group != ~0) { + retry = 1; + parent_group = EXT4_I(parent)->i_last_alloc_group; + goto try_again; + } + /* + * If this didn't work, use the Orlov search algorithm + * to find a new flex group; we pass in the mode to + * avoid the topdir algorithms. + */ + *group = parent_group + flex_size; + if (*group > ngroups) + *group = 0; + return find_group_orlov(sb, parent, group, mode, NULL); + } + + /* + * Try to place the inode in its parent directory + */ + *group = parent_group; + desc = ext4_get_group_desc(sb, *group, NULL); + if (desc && ext4_free_inodes_count(sb, desc) && + ext4_free_group_clusters(sb, desc)) + return 0; + + /* + * We're going to place this inode in a different blockgroup from its + * parent. We want to cause files in a common directory to all land in + * the same blockgroup. But we want files which are in a different + * directory which shares a blockgroup with our parent to land in a + * different blockgroup. + * + * So add our directory's i_ino into the starting point for the hash. + */ + *group = (*group + (unsigned int)parent->i_ino) % ngroups; + + /* + * Use a quadratic hash to find a group with a free inode and some free + * blocks. + */ + for (i = 1; i < ngroups; i <<= 1) { + *group += i; + if (*group >= ngroups) + *group -= ngroups; + desc = ext4_get_group_desc(sb, *group, NULL); + if (desc && ext4_free_inodes_count(sb, desc) && + ext4_free_group_clusters(sb, desc)) + return 0; + } + + /* + * That failed: try linear search for a free inode, even if that group + * has no free blocks. + */ + *group = parent_group; + for (i = 0; i < ngroups; i++) { + if (++*group >= ngroups) + *group = 0; + desc = ext4_get_group_desc(sb, *group, NULL); + if (desc && ext4_free_inodes_count(sb, desc)) + return 0; + } + + return -1; +} + +/* + * In no journal mode, if an inode has recently been deleted, we want + * to avoid reusing it until we're reasonably sure the inode table + * block has been written back to disk. (Yes, these values are + * somewhat arbitrary...) + */ +#define RECENTCY_MIN 60 +#define RECENTCY_DIRTY 300 + +static int recently_deleted(struct super_block *sb, ext4_group_t group, int ino) +{ + struct ext4_group_desc *gdp; + struct ext4_inode *raw_inode; + struct buffer_head *bh; + int inodes_per_block = EXT4_SB(sb)->s_inodes_per_block; + int offset, ret = 0; + int recentcy = RECENTCY_MIN; + u32 dtime, now; + + gdp = ext4_get_group_desc(sb, group, NULL); + if (unlikely(!gdp)) + return 0; + + /* Inode was never used in this filesystem? */ + if (ext4_has_group_desc_csum(sb) && + (gdp->bg_flags & cpu_to_le16(EXT4_BG_INODE_UNINIT) || + ino >= EXT4_INODES_PER_GROUP(sb) - ext4_itable_unused_count(sb, gdp))) + return 0; + + bh = sb_find_get_block(sb, ext4_inode_table(sb, gdp) + + (ino / inodes_per_block)); + if (!bh || !buffer_uptodate(bh)) + /* + * If the block is not in the buffer cache, then it + * must have been written out, or, most unlikely, is + * being migrated - false failure should be OK here. + */ + goto out; + + offset = (ino % inodes_per_block) * EXT4_INODE_SIZE(sb); + raw_inode = (struct ext4_inode *) (bh->b_data + offset); + + /* i_dtime is only 32 bits on disk, but we only care about relative + * times in the range of a few minutes (i.e. long enough to sync a + * recently-deleted inode to disk), so using the low 32 bits of the + * clock (a 68 year range) is enough, see time_before32() */ + dtime = le32_to_cpu(raw_inode->i_dtime); + now = ktime_get_real_seconds(); + if (buffer_dirty(bh)) + recentcy += RECENTCY_DIRTY; + + if (dtime && time_before32(dtime, now) && + time_before32(now, dtime + recentcy)) + ret = 1; +out: + brelse(bh); + return ret; +} + +static int find_inode_bit(struct super_block *sb, ext4_group_t group, + struct buffer_head *bitmap, unsigned long *ino) +{ + bool check_recently_deleted = EXT4_SB(sb)->s_journal == NULL; + unsigned long recently_deleted_ino = EXT4_INODES_PER_GROUP(sb); + +next: + *ino = ext4_find_next_zero_bit((unsigned long *) + bitmap->b_data, + EXT4_INODES_PER_GROUP(sb), *ino); + if (*ino >= EXT4_INODES_PER_GROUP(sb)) + goto not_found; + + if (check_recently_deleted && recently_deleted(sb, group, *ino)) { + recently_deleted_ino = *ino; + *ino = *ino + 1; + if (*ino < EXT4_INODES_PER_GROUP(sb)) + goto next; + goto not_found; + } + return 1; +not_found: + if (recently_deleted_ino >= EXT4_INODES_PER_GROUP(sb)) + return 0; + /* + * Not reusing recently deleted inodes is mostly a preference. We don't + * want to report ENOSPC or skew allocation patterns because of that. + * So return even recently deleted inode if we could find better in the + * given range. + */ + *ino = recently_deleted_ino; + return 1; +} + +int ext4_mark_inode_used(struct super_block *sb, int ino) +{ + unsigned long max_ino = le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count); + struct buffer_head *inode_bitmap_bh = NULL, *group_desc_bh = NULL; + struct ext4_group_desc *gdp; + ext4_group_t group; + int bit; + int err; + + if (ino < EXT4_FIRST_INO(sb) || ino > max_ino) + return -EFSCORRUPTED; + + group = (ino - 1) / EXT4_INODES_PER_GROUP(sb); + bit = (ino - 1) % EXT4_INODES_PER_GROUP(sb); + inode_bitmap_bh = ext4_read_inode_bitmap(sb, group); + if (IS_ERR(inode_bitmap_bh)) + return PTR_ERR(inode_bitmap_bh); + + if (ext4_test_bit(bit, inode_bitmap_bh->b_data)) { + err = 0; + goto out; + } + + gdp = ext4_get_group_desc(sb, group, &group_desc_bh); + if (!gdp) { + err = -EINVAL; + goto out; + } + + ext4_set_bit(bit, inode_bitmap_bh->b_data); + + BUFFER_TRACE(inode_bitmap_bh, "call ext4_handle_dirty_metadata"); + err = ext4_handle_dirty_metadata(NULL, NULL, inode_bitmap_bh); + if (err) { + ext4_std_error(sb, err); + goto out; + } + err = sync_dirty_buffer(inode_bitmap_bh); + if (err) { + ext4_std_error(sb, err); + goto out; + } + + /* We may have to initialize the block bitmap if it isn't already */ + if (ext4_has_group_desc_csum(sb) && + gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) { + struct buffer_head *block_bitmap_bh; + + block_bitmap_bh = ext4_read_block_bitmap(sb, group); + if (IS_ERR(block_bitmap_bh)) { + err = PTR_ERR(block_bitmap_bh); + goto out; + } + + BUFFER_TRACE(block_bitmap_bh, "dirty block bitmap"); + err = ext4_handle_dirty_metadata(NULL, NULL, block_bitmap_bh); + sync_dirty_buffer(block_bitmap_bh); + + /* recheck and clear flag under lock if we still need to */ + ext4_lock_group(sb, group); + if (ext4_has_group_desc_csum(sb) && + (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) { + gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT); + ext4_free_group_clusters_set(sb, gdp, + ext4_free_clusters_after_init(sb, group, gdp)); + ext4_block_bitmap_csum_set(sb, gdp, block_bitmap_bh); + ext4_group_desc_csum_set(sb, group, gdp); + } + ext4_unlock_group(sb, group); + brelse(block_bitmap_bh); + + if (err) { + ext4_std_error(sb, err); + goto out; + } + } + + /* Update the relevant bg descriptor fields */ + if (ext4_has_group_desc_csum(sb)) { + int free; + + ext4_lock_group(sb, group); /* while we modify the bg desc */ + free = EXT4_INODES_PER_GROUP(sb) - + ext4_itable_unused_count(sb, gdp); + if (gdp->bg_flags & cpu_to_le16(EXT4_BG_INODE_UNINIT)) { + gdp->bg_flags &= cpu_to_le16(~EXT4_BG_INODE_UNINIT); + free = 0; + } + + /* + * Check the relative inode number against the last used + * relative inode number in this group. if it is greater + * we need to update the bg_itable_unused count + */ + if (bit >= free) + ext4_itable_unused_set(sb, gdp, + (EXT4_INODES_PER_GROUP(sb) - bit - 1)); + } else { + ext4_lock_group(sb, group); + } + + ext4_free_inodes_set(sb, gdp, ext4_free_inodes_count(sb, gdp) - 1); + if (ext4_has_group_desc_csum(sb)) { + ext4_inode_bitmap_csum_set(sb, gdp, inode_bitmap_bh); + ext4_group_desc_csum_set(sb, group, gdp); + } + + ext4_unlock_group(sb, group); + err = ext4_handle_dirty_metadata(NULL, NULL, group_desc_bh); + sync_dirty_buffer(group_desc_bh); +out: + brelse(inode_bitmap_bh); + return err; +} + +static int ext4_xattr_credits_for_new_inode(struct inode *dir, mode_t mode, + bool encrypt) +{ + struct super_block *sb = dir->i_sb; + int nblocks = 0; +#ifdef CONFIG_EXT4_FS_POSIX_ACL + struct posix_acl *p = get_inode_acl(dir, ACL_TYPE_DEFAULT); + + if (IS_ERR(p)) + return PTR_ERR(p); + if (p) { + int acl_size = p->a_count * sizeof(ext4_acl_entry); + + nblocks += (S_ISDIR(mode) ? 2 : 1) * + __ext4_xattr_set_credits(sb, NULL /* inode */, + NULL /* block_bh */, acl_size, + true /* is_create */); + posix_acl_release(p); + } +#endif + +#ifdef CONFIG_SECURITY + { + int num_security_xattrs = 1; + +#ifdef CONFIG_INTEGRITY + num_security_xattrs++; +#endif + /* + * We assume that security xattrs are never more than 1k. + * In practice they are under 128 bytes. + */ + nblocks += num_security_xattrs * + __ext4_xattr_set_credits(sb, NULL /* inode */, + NULL /* block_bh */, 1024, + true /* is_create */); + } +#endif + if (encrypt) + nblocks += __ext4_xattr_set_credits(sb, + NULL /* inode */, + NULL /* block_bh */, + FSCRYPT_SET_CONTEXT_MAX_SIZE, + true /* is_create */); + return nblocks; +} + +/* + * There are two policies for allocating an inode. If the new inode is + * a directory, then a forward search is made for a block group with both + * free space and a low directory-to-inode ratio; if that fails, then of + * the groups with above-average free space, that group with the fewest + * directories already is chosen. + * + * For other inodes, search forward from the parent directory's block + * group to find a free inode. + */ +struct inode *__ext4_new_inode(struct mnt_idmap *idmap, + handle_t *handle, struct inode *dir, + umode_t mode, const struct qstr *qstr, + __u32 goal, uid_t *owner, __u32 i_flags, + int handle_type, unsigned int line_no, + int nblocks) +{ + struct super_block *sb; + struct buffer_head *inode_bitmap_bh = NULL; + struct buffer_head *group_desc_bh; + ext4_group_t ngroups, group = 0; + unsigned long ino = 0; + struct inode *inode; + struct ext4_group_desc *gdp = NULL; + struct ext4_inode_info *ei; + struct ext4_sb_info *sbi; + int ret2, err; + struct inode *ret; + ext4_group_t i; + ext4_group_t flex_group; + struct ext4_group_info *grp = NULL; + bool encrypt = false; + + /* Cannot create files in a deleted directory */ + if (!dir || !dir->i_nlink) + return ERR_PTR(-EPERM); + + sb = dir->i_sb; + sbi = EXT4_SB(sb); + + ret2 = ext4_emergency_state(sb); + if (unlikely(ret2)) + return ERR_PTR(ret2); + + ngroups = ext4_get_groups_count(sb); + trace_ext4_request_inode(dir, mode); + inode = new_inode(sb); + if (!inode) + return ERR_PTR(-ENOMEM); + ei = EXT4_I(inode); + + /* + * Initialize owners and quota early so that we don't have to account + * for quota initialization worst case in standard inode creating + * transaction + */ + if (owner) { + inode->i_mode = mode; + i_uid_write(inode, owner[0]); + i_gid_write(inode, owner[1]); + } else if (test_opt(sb, GRPID)) { + inode->i_mode = mode; + inode_fsuid_set(inode, idmap); + inode->i_gid = dir->i_gid; + } else + inode_init_owner(idmap, inode, dir, mode); + + if (ext4_has_feature_project(sb) && + ext4_test_inode_flag(dir, EXT4_INODE_PROJINHERIT)) + ei->i_projid = EXT4_I(dir)->i_projid; + else + ei->i_projid = make_kprojid(&init_user_ns, EXT4_DEF_PROJID); + + if (!(i_flags & EXT4_EA_INODE_FL)) { + err = fscrypt_prepare_new_inode(dir, inode, &encrypt); + if (err) + goto out; + if (encrypt) + i_flags |= EXT4_ENCRYPT_FL; + } + + err = dquot_initialize(inode); + if (err) + goto out; + + if (!handle && sbi->s_journal && !(i_flags & EXT4_EA_INODE_FL)) { + ret2 = ext4_xattr_credits_for_new_inode(dir, mode, encrypt); + if (ret2 < 0) { + err = ret2; + goto out; + } + nblocks += ret2; + } + + if (!goal) + goal = sbi->s_inode_goal; + + if (goal && goal <= le32_to_cpu(sbi->s_es->s_inodes_count)) { + group = (goal - 1) / EXT4_INODES_PER_GROUP(sb); + ino = (goal - 1) % EXT4_INODES_PER_GROUP(sb); + ret2 = 0; + goto got_group; + } + + if (S_ISDIR(mode)) + ret2 = find_group_orlov(sb, dir, &group, mode, qstr); + else + ret2 = find_group_other(sb, dir, &group, mode); + +got_group: + EXT4_I(dir)->i_last_alloc_group = group; + err = -ENOSPC; + if (ret2 == -1) + goto out; + + /* + * Normally we will only go through one pass of this loop, + * unless we get unlucky and it turns out the group we selected + * had its last inode grabbed by someone else. + */ + for (i = 0; i < ngroups; i++, ino = 0) { + err = -EIO; + + gdp = ext4_get_group_desc(sb, group, &group_desc_bh); + if (!gdp) + goto out; + + /* + * Check free inodes count before loading bitmap. + */ + if (ext4_free_inodes_count(sb, gdp) == 0) + goto next_group; + + if (!(sbi->s_mount_state & EXT4_FC_REPLAY)) { + grp = ext4_get_group_info(sb, group); + /* + * Skip groups with already-known suspicious inode + * tables + */ + if (!grp || EXT4_MB_GRP_IBITMAP_CORRUPT(grp)) + goto next_group; + } + + brelse(inode_bitmap_bh); + inode_bitmap_bh = ext4_read_inode_bitmap(sb, group); + /* Skip groups with suspicious inode tables */ + if (IS_ERR(inode_bitmap_bh)) { + inode_bitmap_bh = NULL; + goto next_group; + } + if (!(sbi->s_mount_state & EXT4_FC_REPLAY) && + EXT4_MB_GRP_IBITMAP_CORRUPT(grp)) + goto next_group; + + ret2 = find_inode_bit(sb, group, inode_bitmap_bh, &ino); + if (!ret2) + goto next_group; + + if (group == 0 && (ino + 1) < EXT4_FIRST_INO(sb)) { + ext4_error(sb, "reserved inode found cleared - " + "inode=%lu", ino + 1); + ext4_mark_group_bitmap_corrupted(sb, group, + EXT4_GROUP_INFO_IBITMAP_CORRUPT); + goto next_group; + } + + if ((!(sbi->s_mount_state & EXT4_FC_REPLAY)) && !handle) { + BUG_ON(nblocks <= 0); + handle = __ext4_journal_start_sb(NULL, dir->i_sb, + line_no, handle_type, nblocks, 0, + ext4_trans_default_revoke_credits(sb)); + if (IS_ERR(handle)) { + err = PTR_ERR(handle); + ext4_std_error(sb, err); + goto out; + } + } + BUFFER_TRACE(inode_bitmap_bh, "get_write_access"); + err = ext4_journal_get_write_access(handle, sb, inode_bitmap_bh, + EXT4_JTR_NONE); + if (err) { + ext4_std_error(sb, err); + goto out; + } + ext4_lock_group(sb, group); + ret2 = ext4_test_and_set_bit(ino, inode_bitmap_bh->b_data); + if (ret2) { + /* Someone already took the bit. Repeat the search + * with lock held. + */ + ret2 = find_inode_bit(sb, group, inode_bitmap_bh, &ino); + if (ret2) { + ext4_set_bit(ino, inode_bitmap_bh->b_data); + ret2 = 0; + } else { + ret2 = 1; /* we didn't grab the inode */ + } + } + ext4_unlock_group(sb, group); + ino++; /* the inode bitmap is zero-based */ + if (!ret2) + goto got; /* we grabbed the inode! */ + +next_group: + if (++group == ngroups) + group = 0; + } + err = -ENOSPC; + goto out; + +got: + BUFFER_TRACE(inode_bitmap_bh, "call ext4_handle_dirty_metadata"); + err = ext4_handle_dirty_metadata(handle, NULL, inode_bitmap_bh); + if (err) { + ext4_std_error(sb, err); + goto out; + } + + BUFFER_TRACE(group_desc_bh, "get_write_access"); + err = ext4_journal_get_write_access(handle, sb, group_desc_bh, + EXT4_JTR_NONE); + if (err) { + ext4_std_error(sb, err); + goto out; + } + + /* We may have to initialize the block bitmap if it isn't already */ + if (ext4_has_group_desc_csum(sb) && + gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) { + struct buffer_head *block_bitmap_bh; + + block_bitmap_bh = ext4_read_block_bitmap(sb, group); + if (IS_ERR(block_bitmap_bh)) { + err = PTR_ERR(block_bitmap_bh); + goto out; + } + BUFFER_TRACE(block_bitmap_bh, "get block bitmap access"); + err = ext4_journal_get_write_access(handle, sb, block_bitmap_bh, + EXT4_JTR_NONE); + if (err) { + brelse(block_bitmap_bh); + ext4_std_error(sb, err); + goto out; + } + + BUFFER_TRACE(block_bitmap_bh, "dirty block bitmap"); + err = ext4_handle_dirty_metadata(handle, NULL, block_bitmap_bh); + + /* recheck and clear flag under lock if we still need to */ + ext4_lock_group(sb, group); + if (ext4_has_group_desc_csum(sb) && + (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) { + gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT); + ext4_free_group_clusters_set(sb, gdp, + ext4_free_clusters_after_init(sb, group, gdp)); + ext4_block_bitmap_csum_set(sb, gdp, block_bitmap_bh); + ext4_group_desc_csum_set(sb, group, gdp); + } + ext4_unlock_group(sb, group); + brelse(block_bitmap_bh); + + if (err) { + ext4_std_error(sb, err); + goto out; + } + } + + /* Update the relevant bg descriptor fields */ + if (ext4_has_group_desc_csum(sb)) { + int free; + struct ext4_group_info *grp = NULL; + + if (!(sbi->s_mount_state & EXT4_FC_REPLAY)) { + grp = ext4_get_group_info(sb, group); + if (!grp) { + err = -EFSCORRUPTED; + goto out; + } + down_read(&grp->alloc_sem); /* + * protect vs itable + * lazyinit + */ + } + ext4_lock_group(sb, group); /* while we modify the bg desc */ + free = EXT4_INODES_PER_GROUP(sb) - + ext4_itable_unused_count(sb, gdp); + if (gdp->bg_flags & cpu_to_le16(EXT4_BG_INODE_UNINIT)) { + gdp->bg_flags &= cpu_to_le16(~EXT4_BG_INODE_UNINIT); + free = 0; + } + /* + * Check the relative inode number against the last used + * relative inode number in this group. if it is greater + * we need to update the bg_itable_unused count + */ + if (ino > free) + ext4_itable_unused_set(sb, gdp, + (EXT4_INODES_PER_GROUP(sb) - ino)); + if (!(sbi->s_mount_state & EXT4_FC_REPLAY)) + up_read(&grp->alloc_sem); + } else { + ext4_lock_group(sb, group); + } + + ext4_free_inodes_set(sb, gdp, ext4_free_inodes_count(sb, gdp) - 1); + if (S_ISDIR(mode)) { + ext4_used_dirs_set(sb, gdp, ext4_used_dirs_count(sb, gdp) + 1); + if (sbi->s_log_groups_per_flex) { + ext4_group_t f = ext4_flex_group(sbi, group); + + atomic_inc(&sbi_array_rcu_deref(sbi, s_flex_groups, + f)->used_dirs); + } + } + if (ext4_has_group_desc_csum(sb)) { + ext4_inode_bitmap_csum_set(sb, gdp, inode_bitmap_bh); + ext4_group_desc_csum_set(sb, group, gdp); + } + ext4_unlock_group(sb, group); + + BUFFER_TRACE(group_desc_bh, "call ext4_handle_dirty_metadata"); + err = ext4_handle_dirty_metadata(handle, NULL, group_desc_bh); + if (err) { + ext4_std_error(sb, err); + goto out; + } + + percpu_counter_dec(&sbi->s_freeinodes_counter); + if (S_ISDIR(mode)) + percpu_counter_inc(&sbi->s_dirs_counter); + + if (sbi->s_log_groups_per_flex) { + flex_group = ext4_flex_group(sbi, group); + atomic_dec(&sbi_array_rcu_deref(sbi, s_flex_groups, + flex_group)->free_inodes); + } + + inode->i_ino = ino + group * EXT4_INODES_PER_GROUP(sb); + /* This is the optimal IO size (for stat), not the fs block size */ + inode->i_blocks = 0; + simple_inode_init_ts(inode); + ei->i_crtime = inode_get_mtime(inode); + + memset(ei->i_data, 0, sizeof(ei->i_data)); + ei->i_dir_start_lookup = 0; + ei->i_disksize = 0; + + /* Don't inherit extent flag from directory, amongst others. */ + ei->i_flags = + ext4_mask_flags(mode, EXT4_I(dir)->i_flags & EXT4_FL_INHERITED); + ei->i_flags |= i_flags; + ei->i_file_acl = 0; + ei->i_dtime = 0; + ei->i_block_group = group; + ei->i_last_alloc_group = ~0; + + ext4_set_inode_flags(inode, true); + if (IS_DIRSYNC(inode)) + ext4_handle_sync(handle); + if (insert_inode_locked(inode) < 0) { + /* + * Likely a bitmap corruption causing inode to be allocated + * twice. + */ + err = -EIO; + ext4_error(sb, "failed to insert inode %llu: doubly allocated?", + inode->i_ino); + ext4_mark_group_bitmap_corrupted(sb, group, + EXT4_GROUP_INFO_IBITMAP_CORRUPT); + goto out; + } + inode->i_generation = get_random_u32(); + + /* Precompute checksum seed for inode metadata */ + if (ext4_has_feature_metadata_csum(sb)) { + __u32 csum; + __le32 inum = cpu_to_le32(inode->i_ino); + __le32 gen = cpu_to_le32(inode->i_generation); + csum = ext4_chksum(sbi->s_csum_seed, (__u8 *)&inum, + sizeof(inum)); + ei->i_csum_seed = ext4_chksum(csum, (__u8 *)&gen, sizeof(gen)); + } + + ext4_set_inode_state(inode, EXT4_STATE_NEW); + + ei->i_extra_isize = sbi->s_want_extra_isize; + ei->i_inline_off = 0; + if (ext4_has_feature_inline_data(sb) && + /* Encrypted inodes cannot have inline data */ + !(ei->i_flags & EXT4_ENCRYPT_FL) && + (!(ei->i_flags & (EXT4_DAX_FL|EXT4_EA_INODE_FL)) || S_ISDIR(mode))) + ext4_set_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); + ret = inode; + err = dquot_alloc_inode(inode); + if (err) + goto fail_drop; + + /* + * Since the encryption xattr will always be unique, create it first so + * that it's less likely to end up in an external xattr block and + * prevent its deduplication. + */ + if (encrypt) { + err = fscrypt_set_context(inode, handle); + if (err) + goto fail_free_drop; + } + + if (!(ei->i_flags & EXT4_EA_INODE_FL)) { + err = ext4_init_acl(handle, inode, dir); + if (err) + goto fail_free_drop; + + err = ext4_init_security(handle, inode, dir, qstr); + if (err) + goto fail_free_drop; + } + + if (ext4_has_feature_extents(sb)) { + /* set extent flag only for directory, file and normal symlink*/ + if (S_ISDIR(mode) || S_ISREG(mode) || S_ISLNK(mode)) { + ext4_set_inode_flag(inode, EXT4_INODE_EXTENTS); + ext4_ext_tree_init(handle, inode); + } + } + + ext4_set_inode_mapping_order(inode); + + ext4_update_inode_fsync_trans(handle, inode, 1); + + err = ext4_mark_inode_dirty(handle, inode); + if (err) { + ext4_std_error(sb, err); + goto fail_free_drop; + } + + ext4_debug("allocating inode %llu\n", inode->i_ino); + trace_ext4_allocate_inode(inode, dir, mode); + brelse(inode_bitmap_bh); + return ret; + +fail_free_drop: + dquot_free_inode(inode); +fail_drop: + clear_nlink(inode); + unlock_new_inode(inode); +out: + dquot_drop(inode); + inode->i_flags |= S_NOQUOTA; + iput(inode); + brelse(inode_bitmap_bh); + return ERR_PTR(err); +} + +/* Verify that we are loading a valid orphan from disk */ +struct inode *ext4_orphan_get(struct super_block *sb, unsigned long ino) +{ + unsigned long max_ino = le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count); + ext4_group_t block_group; + int bit; + struct buffer_head *bitmap_bh = NULL; + struct inode *inode = NULL; + int err = -EFSCORRUPTED; + + if (ino < EXT4_FIRST_INO(sb) || ino > max_ino) + goto bad_orphan; + + block_group = (ino - 1) / EXT4_INODES_PER_GROUP(sb); + bit = (ino - 1) % EXT4_INODES_PER_GROUP(sb); + bitmap_bh = ext4_read_inode_bitmap(sb, block_group); + if (IS_ERR(bitmap_bh)) + return ERR_CAST(bitmap_bh); + + /* Having the inode bit set should be a 100% indicator that this + * is a valid orphan (no e2fsck run on fs). Orphans also include + * inodes that were being truncated, so we can't check i_nlink==0. + */ + if (!ext4_test_bit(bit, bitmap_bh->b_data)) + goto bad_orphan; + + inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + ext4_error_err(sb, -err, + "couldn't read orphan inode %lu (err %d)", + ino, err); + brelse(bitmap_bh); + return inode; + } + + /* + * If the orphans has i_nlinks > 0 then it should be able to + * be truncated, otherwise it won't be removed from the orphan + * list during processing and an infinite loop will result. + * Similarly, it must not be a bad inode. + */ + if ((inode->i_nlink && !ext4_can_truncate(inode)) || + is_bad_inode(inode)) + goto bad_orphan; + + if (NEXT_ORPHAN(inode) > max_ino) + goto bad_orphan; + brelse(bitmap_bh); + return inode; + +bad_orphan: + ext4_error(sb, "bad orphan inode %lu", ino); + if (bitmap_bh) + printk(KERN_ERR "ext4_test_bit(bit=%d, block=%llu) = %d\n", + bit, (unsigned long long)bitmap_bh->b_blocknr, + ext4_test_bit(bit, bitmap_bh->b_data)); + if (inode) { + printk(KERN_ERR "is_bad_inode(inode)=%d\n", + is_bad_inode(inode)); + printk(KERN_ERR "NEXT_ORPHAN(inode)=%u\n", + NEXT_ORPHAN(inode)); + printk(KERN_ERR "max_ino=%lu\n", max_ino); + printk(KERN_ERR "i_nlink=%u\n", inode->i_nlink); + /* Avoid freeing blocks if we got a bad deleted inode */ + if (inode->i_nlink == 0) + inode->i_blocks = 0; + iput(inode); + } + brelse(bitmap_bh); + return ERR_PTR(err); +} + +unsigned long ext4_count_free_inodes(struct super_block *sb) +{ + unsigned long desc_count; + struct ext4_group_desc *gdp; + ext4_group_t i, ngroups = ext4_get_groups_count(sb); +#ifdef EXT4FS_DEBUG + struct ext4_super_block *es; + unsigned long bitmap_count, x; + struct buffer_head *bitmap_bh = NULL; + + es = EXT4_SB(sb)->s_es; + desc_count = 0; + bitmap_count = 0; + gdp = NULL; + for (i = 0; i < ngroups; i++) { + gdp = ext4_get_group_desc(sb, i, NULL); + if (!gdp) + continue; + desc_count += ext4_free_inodes_count(sb, gdp); + brelse(bitmap_bh); + bitmap_bh = ext4_read_inode_bitmap(sb, i); + if (IS_ERR(bitmap_bh)) { + bitmap_bh = NULL; + continue; + } + + x = ext4_count_free(bitmap_bh->b_data, + EXT4_INODES_PER_GROUP(sb) / 8); + printk(KERN_DEBUG "group %lu: stored = %d, counted = %lu\n", + (unsigned long) i, ext4_free_inodes_count(sb, gdp), x); + bitmap_count += x; + } + brelse(bitmap_bh); + printk(KERN_DEBUG "ext4_count_free_inodes: " + "stored = %u, computed = %lu, %lu\n", + le32_to_cpu(es->s_free_inodes_count), desc_count, bitmap_count); + return desc_count; +#else + desc_count = 0; + for (i = 0; i < ngroups; i++) { + gdp = ext4_get_group_desc(sb, i, NULL); + if (!gdp) + continue; + desc_count += ext4_free_inodes_count(sb, gdp); + cond_resched(); + } + return desc_count; +#endif +} + +/* Called at mount-time, super-block is locked */ +unsigned long ext4_count_dirs(struct super_block * sb) +{ + unsigned long count = 0; + ext4_group_t i, ngroups = ext4_get_groups_count(sb); + + for (i = 0; i < ngroups; i++) { + struct ext4_group_desc *gdp = ext4_get_group_desc(sb, i, NULL); + if (!gdp) + continue; + count += ext4_used_dirs_count(sb, gdp); + } + return count; +} + +/* + * Zeroes not yet zeroed inode table - just write zeroes through the whole + * inode table. Must be called without any spinlock held. The only place + * where it is called from on active part of filesystem is ext4lazyinit + * thread, so we do not need any special locks, however we have to prevent + * inode allocation from the current group, so we take alloc_sem lock, to + * block ext4_new_inode() until we are finished. + */ +int ext4_init_inode_table(struct super_block *sb, ext4_group_t group, + int barrier) +{ + struct ext4_group_info *grp = ext4_get_group_info(sb, group); + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_group_desc *gdp = NULL; + struct buffer_head *group_desc_bh; + handle_t *handle; + ext4_fsblk_t blk; + int num, ret = 0, used_blks = 0; + unsigned long used_inos = 0; + + gdp = ext4_get_group_desc(sb, group, &group_desc_bh); + if (!gdp || !grp) + goto out; + + /* + * We do not need to lock this, because we are the only one + * handling this flag. + */ + if (gdp->bg_flags & cpu_to_le16(EXT4_BG_INODE_ZEROED)) + goto out; + + handle = ext4_journal_start_sb(sb, EXT4_HT_MISC, 1); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + goto out; + } + + down_write(&grp->alloc_sem); + /* + * If inode bitmap was already initialized there may be some + * used inodes so we need to skip blocks with used inodes in + * inode table. + */ + if (!(gdp->bg_flags & cpu_to_le16(EXT4_BG_INODE_UNINIT))) { + used_inos = EXT4_INODES_PER_GROUP(sb) - + ext4_itable_unused_count(sb, gdp); + used_blks = DIV_ROUND_UP(used_inos, sbi->s_inodes_per_block); + + /* Bogus inode unused count? */ + if (used_blks < 0 || used_blks > sbi->s_itb_per_group) { + ext4_error(sb, "Something is wrong with group %u: " + "used itable blocks: %d; " + "itable unused count: %u", + group, used_blks, + ext4_itable_unused_count(sb, gdp)); + ret = 1; + goto err_out; + } + + used_inos += group * EXT4_INODES_PER_GROUP(sb); + /* + * Are there some uninitialized inodes in the inode table + * before the first normal inode? + */ + if ((used_blks != sbi->s_itb_per_group) && + (used_inos < EXT4_FIRST_INO(sb))) { + ext4_error(sb, "Something is wrong with group %u: " + "itable unused count: %u; " + "itables initialized count: %ld", + group, ext4_itable_unused_count(sb, gdp), + used_inos); + ret = 1; + goto err_out; + } + } + + blk = ext4_inode_table(sb, gdp) + used_blks; + num = sbi->s_itb_per_group - used_blks; + + BUFFER_TRACE(group_desc_bh, "get_write_access"); + ret = ext4_journal_get_write_access(handle, sb, group_desc_bh, + EXT4_JTR_NONE); + if (ret) + goto err_out; + + /* + * Skip zeroout if the inode table is full. But we set the ZEROED + * flag anyway, because obviously, when it is full it does not need + * further zeroing. + */ + if (unlikely(num == 0)) + goto skip_zeroout; + + ext4_debug("going to zero out inode table in group %d\n", + group); + ret = sb_issue_zeroout(sb, blk, num, GFP_NOFS); + if (ret < 0) + goto err_out; + if (barrier) + blkdev_issue_flush(sb->s_bdev); + +skip_zeroout: + ext4_lock_group(sb, group); + gdp->bg_flags |= cpu_to_le16(EXT4_BG_INODE_ZEROED); + ext4_group_desc_csum_set(sb, group, gdp); + ext4_unlock_group(sb, group); + + BUFFER_TRACE(group_desc_bh, + "call ext4_handle_dirty_metadata"); + ret = ext4_handle_dirty_metadata(handle, NULL, + group_desc_bh); + +err_out: + up_write(&grp->alloc_sem); + ext4_journal_stop(handle); +out: + return ret; +} diff --git a/fs/ext4/indirect.c b/fs/ext4/indirect.c new file mode 100644 index 000000000..5aec759ee --- /dev/null +++ b/fs/ext4/indirect.c @@ -0,0 +1,1474 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext4/indirect.c + * + * from + * + * linux/fs/ext4/inode.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * from + * + * linux/fs/minix/inode.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * Goal-directed block allocation by Stephen Tweedie + * (sct@redhat.com), 1993, 1998 + */ + +#include "ext4_jbd2.h" +#include "truncate.h" +#include +#include + +#include + +typedef struct { + __le32 *p; + __le32 key; + struct buffer_head *bh; +} Indirect; + +static inline void add_chain(Indirect *p, struct buffer_head *bh, __le32 *v) +{ + p->key = *(p->p = v); + p->bh = bh; +} + +/** + * ext4_block_to_path - parse the block number into array of offsets + * @inode: inode in question (we are only interested in its superblock) + * @i_block: block number to be parsed + * @offsets: array to store the offsets in + * @boundary: set this non-zero if the referred-to block is likely to be + * followed (on disk) by an indirect block. + * + * To store the locations of file's data ext4 uses a data structure common + * for UNIX filesystems - tree of pointers anchored in the inode, with + * data blocks at leaves and indirect blocks in intermediate nodes. + * This function translates the block number into path in that tree - + * return value is the path length and @offsets[n] is the offset of + * pointer to (n+1)th node in the nth one. If @block is out of range + * (negative or too large) warning is printed and zero returned. + * + * Note: function doesn't find node addresses, so no IO is needed. All + * we need to know is the capacity of indirect blocks (taken from the + * inode->i_sb). + */ + +/* + * Portability note: the last comparison (check that we fit into triple + * indirect block) is spelled differently, because otherwise on an + * architecture with 32-bit longs and 8Kb pages we might get into trouble + * if our filesystem had 8Kb blocks. We might use long long, but that would + * kill us on x86. Oh, well, at least the sign propagation does not matter - + * i_block would have to be negative in the very beginning, so we would not + * get there at all. + */ + +static int ext4_block_to_path(struct inode *inode, + ext4_lblk_t i_block, + ext4_lblk_t offsets[4], int *boundary) +{ + int ptrs = EXT4_ADDR_PER_BLOCK(inode->i_sb); + int ptrs_bits = EXT4_ADDR_PER_BLOCK_BITS(inode->i_sb); + const long direct_blocks = EXT4_NDIR_BLOCKS, + indirect_blocks = ptrs, + double_blocks = (1 << (ptrs_bits * 2)); + int n = 0; + int final = 0; + + if (i_block < direct_blocks) { + offsets[n++] = i_block; + final = direct_blocks; + } else if ((i_block -= direct_blocks) < indirect_blocks) { + offsets[n++] = EXT4_IND_BLOCK; + offsets[n++] = i_block; + final = ptrs; + } else if ((i_block -= indirect_blocks) < double_blocks) { + offsets[n++] = EXT4_DIND_BLOCK; + offsets[n++] = i_block >> ptrs_bits; + offsets[n++] = i_block & (ptrs - 1); + final = ptrs; + } else if (((i_block -= double_blocks) >> (ptrs_bits * 2)) < ptrs) { + offsets[n++] = EXT4_TIND_BLOCK; + offsets[n++] = i_block >> (ptrs_bits * 2); + offsets[n++] = (i_block >> ptrs_bits) & (ptrs - 1); + offsets[n++] = i_block & (ptrs - 1); + final = ptrs; + } else { + ext4_warning(inode->i_sb, "block %lu > max in inode %llu", + i_block + direct_blocks + + indirect_blocks + double_blocks, inode->i_ino); + } + if (boundary) + *boundary = final - 1 - (i_block & (ptrs - 1)); + return n; +} + +/** + * ext4_get_branch - read the chain of indirect blocks leading to data + * @inode: inode in question + * @depth: depth of the chain (1 - direct pointer, etc.) + * @offsets: offsets of pointers in inode/indirect blocks + * @chain: place to store the result + * @err: here we store the error value + * + * Function fills the array of triples and returns %NULL + * if everything went OK or the pointer to the last filled triple + * (incomplete one) otherwise. Upon the return chain[i].key contains + * the number of (i+1)-th block in the chain (as it is stored in memory, + * i.e. little-endian 32-bit), chain[i].p contains the address of that + * number (it points into struct inode for i==0 and into the bh->b_data + * for i>0) and chain[i].bh points to the buffer_head of i-th indirect + * block for i>0 and NULL for i==0. In other words, it holds the block + * numbers of the chain, addresses they were taken from (and where we can + * verify that chain did not change) and buffer_heads hosting these + * numbers. + * + * Function stops when it stumbles upon zero pointer (absent block) + * (pointer to last triple returned, *@err == 0) + * or when it gets an IO error reading an indirect block + * (ditto, *@err == -EIO) + * or when it reads all @depth-1 indirect blocks successfully and finds + * the whole chain, all way to the data (returns %NULL, *err == 0). + * + * Need to be called with + * down_read(&EXT4_I(inode)->i_data_sem) + */ +static Indirect *ext4_get_branch(struct inode *inode, int depth, + ext4_lblk_t *offsets, + Indirect chain[4], int *err) +{ + struct super_block *sb = inode->i_sb; + Indirect *p = chain; + struct buffer_head *bh; + unsigned int key; + int ret = -EIO; + + *err = 0; + /* i_data is not going away, no lock needed */ + add_chain(chain, NULL, EXT4_I(inode)->i_data + *offsets); + if (!p->key) + goto no_block; + while (--depth) { + key = le32_to_cpu(p->key); + if (key > ext4_blocks_count(EXT4_SB(sb)->s_es)) { + /* the block was out of range */ + ret = -EFSCORRUPTED; + goto failure; + } + bh = sb_getblk(sb, key); + if (unlikely(!bh)) { + ret = -ENOMEM; + goto failure; + } + + if (!bh_uptodate_or_lock(bh)) { + if (ext4_read_bh(bh, 0, NULL, false) < 0) { + put_bh(bh); + goto failure; + } + /* validate block references */ + if (ext4_check_indirect_blockref(inode, bh)) { + put_bh(bh); + goto failure; + } + } + + add_chain(++p, bh, (__le32 *)bh->b_data + *++offsets); + /* Reader: end */ + if (!p->key) + goto no_block; + } + return NULL; + +failure: + *err = ret; +no_block: + return p; +} + +/** + * ext4_find_near - find a place for allocation with sufficient locality + * @inode: owner + * @ind: descriptor of indirect block. + * + * This function returns the preferred place for block allocation. + * It is used when heuristic for sequential allocation fails. + * Rules are: + * + if there is a block to the left of our position - allocate near it. + * + if pointer will live in indirect block - allocate near that block. + * + if pointer will live in inode - allocate in the same + * cylinder group. + * + * In the latter case we colour the starting block by the callers PID to + * prevent it from clashing with concurrent allocations for a different inode + * in the same block group. The PID is used here so that functionally related + * files will be close-by on-disk. + * + * Caller must make sure that @ind is valid and will stay that way. + */ +static ext4_fsblk_t ext4_find_near(struct inode *inode, Indirect *ind) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + __le32 *start = ind->bh ? (__le32 *) ind->bh->b_data : ei->i_data; + __le32 *p; + + /* Try to find previous block */ + for (p = ind->p - 1; p >= start; p--) { + if (*p) + return le32_to_cpu(*p); + } + + /* No such thing, so let's try location of indirect block */ + if (ind->bh) + return ind->bh->b_blocknr; + + /* + * It is going to be referred to from the inode itself? OK, just put it + * into the same cylinder group then. + */ + return ext4_inode_to_goal_block(inode); +} + +/** + * ext4_find_goal - find a preferred place for allocation. + * @inode: owner + * @block: block we want + * @partial: pointer to the last triple within a chain + * + * Normally this function find the preferred place for block allocation, + * returns it. + * Because this is only used for non-extent files, we limit the block nr + * to 32 bits. + */ +static ext4_fsblk_t ext4_find_goal(struct inode *inode, ext4_lblk_t block, + Indirect *partial) +{ + ext4_fsblk_t goal; + + /* + * XXX need to get goal block from mballoc's data structures + */ + + goal = ext4_find_near(inode, partial); + goal = goal & EXT4_MAX_BLOCK_FILE_PHYS; + return goal; +} + +/** + * ext4_blks_to_allocate - Look up the block map and count the number + * of direct blocks need to be allocated for the given branch. + * + * @branch: chain of indirect blocks + * @k: number of blocks need for indirect blocks + * @blks: number of data blocks to be mapped. + * @blocks_to_boundary: the offset in the indirect block + * + * return the total number of blocks to be allocate, including the + * direct and indirect blocks. + */ +static int ext4_blks_to_allocate(Indirect *branch, int k, unsigned int blks, + int blocks_to_boundary) +{ + unsigned int count = 0; + + /* + * Simple case, [t,d]Indirect block(s) has not allocated yet + * then it's clear blocks on that path have not allocated + */ + if (k > 0) { + /* right now we don't handle cross boundary allocation */ + if (blks < blocks_to_boundary + 1) + count += blks; + else + count += blocks_to_boundary + 1; + return count; + } + + count++; + while (count < blks && count <= blocks_to_boundary && + le32_to_cpu(*(branch[0].p + count)) == 0) { + count++; + } + return count; +} + +/** + * ext4_alloc_branch() - allocate and set up a chain of blocks + * @handle: handle for this transaction + * @ar: structure describing the allocation request + * @indirect_blks: number of allocated indirect blocks + * @offsets: offsets (in the blocks) to store the pointers to next. + * @branch: place to store the chain in. + * + * This function allocates blocks, zeroes out all but the last one, + * links them into chain and (if we are synchronous) writes them to disk. + * In other words, it prepares a branch that can be spliced onto the + * inode. It stores the information about that chain in the branch[], in + * the same format as ext4_get_branch() would do. We are calling it after + * we had read the existing part of chain and partial points to the last + * triple of that (one with zero ->key). Upon the exit we have the same + * picture as after the successful ext4_get_block(), except that in one + * place chain is disconnected - *branch->p is still zero (we did not + * set the last link), but branch->key contains the number that should + * be placed into *branch->p to fill that gap. + * + * If allocation fails we free all blocks we've allocated (and forget + * their buffer_heads) and return the error value the from failed + * ext4_alloc_block() (normally -ENOSPC). Otherwise we set the chain + * as described above and return 0. + */ +static int ext4_alloc_branch(handle_t *handle, + struct ext4_allocation_request *ar, + int indirect_blks, ext4_lblk_t *offsets, + Indirect *branch) +{ + struct buffer_head * bh; + ext4_fsblk_t b, new_blocks[4]; + __le32 *p; + int i, j, err, len = 1; + + for (i = 0; i <= indirect_blks; i++) { + if (i == indirect_blks) { + new_blocks[i] = ext4_mb_new_blocks(handle, ar, &err); + } else { + ar->goal = new_blocks[i] = ext4_new_meta_blocks(handle, + ar->inode, ar->goal, + ar->flags & EXT4_MB_DELALLOC_RESERVED, + NULL, &err); + /* Simplify error cleanup... */ + branch[i+1].bh = NULL; + } + if (err) { + i--; + goto failed; + } + branch[i].key = cpu_to_le32(new_blocks[i]); + if (i == 0) + continue; + + bh = branch[i].bh = sb_getblk(ar->inode->i_sb, new_blocks[i-1]); + if (unlikely(!bh)) { + err = -ENOMEM; + goto failed; + } + lock_buffer(bh); + BUFFER_TRACE(bh, "call get_create_access"); + err = ext4_journal_get_create_access(handle, ar->inode->i_sb, + bh, EXT4_JTR_NONE); + if (err) { + unlock_buffer(bh); + goto failed; + } + + memset(bh->b_data, 0, bh->b_size); + p = branch[i].p = (__le32 *) bh->b_data + offsets[i]; + b = new_blocks[i]; + + if (i == indirect_blks) + len = ar->len; + for (j = 0; j < len; j++) + *p++ = cpu_to_le32(b++); + + BUFFER_TRACE(bh, "marking uptodate"); + set_buffer_uptodate(bh); + unlock_buffer(bh); + + BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata"); + err = ext4_handle_dirty_metadata(handle, ar->inode, bh); + if (err) + goto failed; + } + return 0; +failed: + if (i == indirect_blks) { + /* Free data blocks */ + ext4_free_blocks(handle, ar->inode, NULL, new_blocks[i], + ar->len, 0); + i--; + } + for (; i >= 0; i--) { + /* + * We want to ext4_forget() only freshly allocated indirect + * blocks. Buffer for new_blocks[i] is at branch[i+1].bh + * (buffer at branch[0].bh is indirect block / inode already + * existing before ext4_alloc_branch() was called). Also + * because blocks are freshly allocated, we don't need to + * revoke them which is why we don't set + * EXT4_FREE_BLOCKS_METADATA. + */ + ext4_free_blocks(handle, ar->inode, branch[i+1].bh, + new_blocks[i], 1, + branch[i+1].bh ? EXT4_FREE_BLOCKS_FORGET : 0); + } + return err; +} + +/** + * ext4_splice_branch() - splice the allocated branch onto inode. + * @handle: handle for this transaction + * @ar: structure describing the allocation request + * @where: location of missing link + * @num: number of indirect blocks we are adding + * + * This function fills the missing link and does all housekeeping needed in + * inode (->i_blocks, etc.). In case of success we end up with the full + * chain to new block and return 0. + */ +static int ext4_splice_branch(handle_t *handle, + struct ext4_allocation_request *ar, + Indirect *where, int num) +{ + int i; + int err = 0; + ext4_fsblk_t current_block; + + /* + * If we're splicing into a [td]indirect block (as opposed to the + * inode) then we need to get write access to the [td]indirect block + * before the splice. + */ + if (where->bh) { + BUFFER_TRACE(where->bh, "get_write_access"); + err = ext4_journal_get_write_access(handle, ar->inode->i_sb, + where->bh, EXT4_JTR_NONE); + if (err) + goto err_out; + } + /* That's it */ + + *where->p = where->key; + + /* + * Update the host buffer_head or inode to point to more just allocated + * direct blocks blocks + */ + if (num == 0 && ar->len > 1) { + current_block = le32_to_cpu(where->key) + 1; + for (i = 1; i < ar->len; i++) + *(where->p + i) = cpu_to_le32(current_block++); + } + + /* We are done with atomic stuff, now do the rest of housekeeping */ + /* had we spliced it onto indirect block? */ + if (where->bh) { + /* + * If we spliced it onto an indirect block, we haven't + * altered the inode. Note however that if it is being spliced + * onto an indirect block at the very end of the file (the + * file is growing) then we *will* alter the inode to reflect + * the new i_size. But that is not done here - it is done in + * generic_commit_write->__mark_inode_dirty->ext4_dirty_inode. + */ + ext4_debug("splicing indirect only\n"); + BUFFER_TRACE(where->bh, "call ext4_handle_dirty_metadata"); + err = ext4_handle_dirty_metadata(handle, ar->inode, where->bh); + if (err) + goto err_out; + } else { + /* + * OK, we spliced it into the inode itself on a direct block. + */ + err = ext4_mark_inode_dirty(handle, ar->inode); + if (unlikely(err)) + goto err_out; + ext4_debug("splicing direct\n"); + } + return err; + +err_out: + for (i = 1; i <= num; i++) { + /* + * branch[i].bh is newly allocated, so there is no + * need to revoke the block, which is why we don't + * need to set EXT4_FREE_BLOCKS_METADATA. + */ + ext4_free_blocks(handle, ar->inode, where[i].bh, 0, 1, + EXT4_FREE_BLOCKS_FORGET); + } + ext4_free_blocks(handle, ar->inode, NULL, le32_to_cpu(where[num].key), + ar->len, 0); + + return err; +} + +/* + * The ext4_ind_map_blocks() function handles non-extents inodes + * (i.e., using the traditional indirect/double-indirect i_blocks + * scheme) for ext4_map_blocks(). + * + * Allocation strategy is simple: if we have to allocate something, we will + * have to go the whole way to leaf. So let's do it before attaching anything + * to tree, set linkage between the newborn blocks, write them if sync is + * required, recheck the path, free and repeat if check fails, otherwise + * set the last missing link (that will protect us from any truncate-generated + * removals - all blocks on the path are immune now) and possibly force the + * write on the parent block. + * That has a nice additional property: no special recovery from the failed + * allocations is needed - we simply release blocks and do not touch anything + * reachable from inode. + * + * `handle' can be NULL if create == 0. + * + * return > 0, # of blocks mapped or allocated. + * return = 0, if plain lookup failed. + * return < 0, error case. + * + * The ext4_ind_get_blocks() function should be called with + * down_write(&EXT4_I(inode)->i_data_sem) if allocating filesystem + * blocks (i.e., flags has EXT4_GET_BLOCKS_CREATE set) or + * down_read(&EXT4_I(inode)->i_data_sem) if not allocating file system + * blocks. + */ +int ext4_ind_map_blocks(handle_t *handle, struct inode *inode, + struct ext4_map_blocks *map, + int flags) +{ + struct ext4_allocation_request ar; + int err = -EIO; + ext4_lblk_t offsets[4]; + Indirect chain[4]; + Indirect *partial; + int indirect_blks; + int blocks_to_boundary = 0; + int depth; + u64 count = 0; + ext4_fsblk_t first_block = 0; + + trace_ext4_ind_map_blocks_enter(inode, map->m_lblk, map->m_len, flags); + ASSERT(!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))); + ASSERT(handle != NULL || (flags & EXT4_GET_BLOCKS_CREATE) == 0); + depth = ext4_block_to_path(inode, map->m_lblk, offsets, + &blocks_to_boundary); + + if (depth == 0) + goto out; + + partial = ext4_get_branch(inode, depth, offsets, chain, &err); + + /* Simplest case - block found, no allocation needed */ + if (!partial) { + first_block = le32_to_cpu(chain[depth - 1].key); + count++; + /*map more blocks*/ + while (count < map->m_len && count <= blocks_to_boundary) { + ext4_fsblk_t blk; + + blk = le32_to_cpu(*(chain[depth-1].p + count)); + + if (blk == first_block + count) + count++; + else + break; + } + goto got_it; + } + + /* Next simple case - plain lookup failed */ + if ((flags & EXT4_GET_BLOCKS_CREATE) == 0) { + unsigned epb = inode->i_sb->s_blocksize / sizeof(u32); + int i; + + /* + * Count number blocks in a subtree under 'partial'. At each + * level we count number of complete empty subtrees beyond + * current offset and then descend into the subtree only + * partially beyond current offset. + */ + count = 0; + for (i = partial - chain + 1; i < depth; i++) + count = count * epb + (epb - offsets[i] - 1); + count++; + /* Fill in size of a hole we found */ + map->m_pblk = 0; + map->m_len = umin(map->m_len, count); + goto cleanup; + } + + /* Failed read of indirect block */ + if (err == -EIO) + goto cleanup; + + /* + * Okay, we need to do block allocation. + */ + if (ext4_has_feature_bigalloc(inode->i_sb)) { + EXT4_ERROR_INODE(inode, "Can't allocate blocks for " + "non-extent mapped inodes with bigalloc"); + err = -EFSCORRUPTED; + goto out; + } + + /* Set up for the direct block allocation */ + memset(&ar, 0, sizeof(ar)); + ar.inode = inode; + ar.logical = map->m_lblk; + if (S_ISREG(inode->i_mode)) + ar.flags = EXT4_MB_HINT_DATA; + if (flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) + ar.flags |= EXT4_MB_DELALLOC_RESERVED; + if (flags & EXT4_GET_BLOCKS_METADATA_NOFAIL) + ar.flags |= EXT4_MB_USE_RESERVED; + + ar.goal = ext4_find_goal(inode, map->m_lblk, partial); + + /* the number of blocks need to allocate for [d,t]indirect blocks */ + indirect_blks = (chain + depth) - partial - 1; + + /* + * Next look up the indirect map to count the totoal number of + * direct blocks to allocate for this branch. + */ + ar.len = ext4_blks_to_allocate(partial, indirect_blks, + map->m_len, blocks_to_boundary); + + /* + * Block out ext4_truncate while we alter the tree + */ + err = ext4_alloc_branch(handle, &ar, indirect_blks, + offsets + (partial - chain), partial); + + /* + * The ext4_splice_branch call will free and forget any buffers + * on the new chain if there is a failure, but that risks using + * up transaction credits, especially for bitmaps where the + * credits cannot be returned. Can we handle this somehow? We + * may need to return -EAGAIN upwards in the worst case. --sct + */ + if (!err) + err = ext4_splice_branch(handle, &ar, partial, indirect_blks); + if (err) + goto cleanup; + + map->m_flags |= EXT4_MAP_NEW; + + ext4_update_inode_fsync_trans(handle, inode, 1); + count = ar.len; + +got_it: + map->m_flags |= EXT4_MAP_MAPPED; + map->m_pblk = le32_to_cpu(chain[depth-1].key); + map->m_len = count; + if (count > blocks_to_boundary) + map->m_flags |= EXT4_MAP_BOUNDARY; + err = count; + /* Clean up and exit */ + partial = chain + depth - 1; /* the whole chain */ +cleanup: + while (partial > chain) { + BUFFER_TRACE(partial->bh, "call brelse"); + brelse(partial->bh); + partial--; + } +out: + trace_ext4_ind_map_blocks_exit(inode, flags, map, err); + return err; +} + +/* + * Calculate number of indirect blocks touched by mapping @nrblocks logically + * contiguous blocks + */ +int ext4_ind_trans_blocks(struct inode *inode, int nrblocks) +{ + /* + * With N contiguous data blocks, we need at most + * N/EXT4_ADDR_PER_BLOCK(inode->i_sb) + 1 indirect blocks, + * 2 dindirect blocks, and 1 tindirect block + */ + return DIV_ROUND_UP(nrblocks, EXT4_ADDR_PER_BLOCK(inode->i_sb)) + 4; +} + +static int ext4_ind_trunc_restart_fn(handle_t *handle, struct inode *inode, + struct buffer_head *bh, int *dropped) +{ + int err; + + if (bh) { + BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata"); + err = ext4_handle_dirty_metadata(handle, inode, bh); + if (unlikely(err)) + return err; + } + err = ext4_mark_inode_dirty(handle, inode); + if (unlikely(err)) + return err; + /* + * Drop i_data_sem to avoid deadlock with ext4_map_blocks. At this + * moment, get_block can be called only for blocks inside i_size since + * page cache has been already dropped and writes are blocked by + * i_rwsem. So we can safely drop the i_data_sem here. + */ + BUG_ON(EXT4_JOURNAL(inode) == NULL); + ext4_discard_preallocations(inode); + up_write(&EXT4_I(inode)->i_data_sem); + *dropped = 1; + return 0; +} + +/* + * Truncate transactions can be complex and absolutely huge. So we need to + * be able to restart the transaction at a convenient checkpoint to make + * sure we don't overflow the journal. + * + * Try to extend this transaction for the purposes of truncation. If + * extend fails, we restart transaction. + */ +static int ext4_ind_truncate_ensure_credits(handle_t *handle, + struct inode *inode, + struct buffer_head *bh, + int revoke_creds) +{ + int ret; + int dropped = 0; + + ret = ext4_journal_ensure_credits_fn(handle, EXT4_RESERVE_TRANS_BLOCKS, + ext4_blocks_for_truncate(inode), revoke_creds, + ext4_ind_trunc_restart_fn(handle, inode, bh, &dropped)); + if (dropped) + down_write(&EXT4_I(inode)->i_data_sem); + if (ret <= 0) + return ret; + if (bh) { + BUFFER_TRACE(bh, "retaking write access"); + ret = ext4_journal_get_write_access(handle, inode->i_sb, bh, + EXT4_JTR_NONE); + if (unlikely(ret)) + return ret; + } + return 0; +} + +/* + * Probably it should be a library function... search for first non-zero word + * or memcmp with zero_page, whatever is better for particular architecture. + * Linus? + */ +static inline int all_zeroes(__le32 *p, __le32 *q) +{ + while (p < q) + if (*p++) + return 0; + return 1; +} + +/** + * ext4_find_shared - find the indirect blocks for partial truncation. + * @inode: inode in question + * @depth: depth of the affected branch + * @offsets: offsets of pointers in that branch (see ext4_block_to_path) + * @chain: place to store the pointers to partial indirect blocks + * @top: place to the (detached) top of branch + * + * This is a helper function used by ext4_truncate(). + * + * When we do truncate() we may have to clean the ends of several + * indirect blocks but leave the blocks themselves alive. Block is + * partially truncated if some data below the new i_size is referred + * from it (and it is on the path to the first completely truncated + * data block, indeed). We have to free the top of that path along + * with everything to the right of the path. Since no allocation + * past the truncation point is possible until ext4_truncate() + * finishes, we may safely do the latter, but top of branch may + * require special attention - pageout below the truncation point + * might try to populate it. + * + * We atomically detach the top of branch from the tree, store the + * block number of its root in *@top, pointers to buffer_heads of + * partially truncated blocks - in @chain[].bh and pointers to + * their last elements that should not be removed - in + * @chain[].p. Return value is the pointer to last filled element + * of @chain. + * + * The work left to caller to do the actual freeing of subtrees: + * a) free the subtree starting from *@top + * b) free the subtrees whose roots are stored in + * (@chain[i].p+1 .. end of @chain[i].bh->b_data) + * c) free the subtrees growing from the inode past the @chain[0]. + * (no partially truncated stuff there). */ + +static Indirect *ext4_find_shared(struct inode *inode, int depth, + ext4_lblk_t offsets[4], Indirect chain[4], + __le32 *top) +{ + Indirect *partial, *p; + int k, err; + + *top = 0; + /* Make k index the deepest non-null offset + 1 */ + for (k = depth; k > 1 && !offsets[k-1]; k--) + ; + partial = ext4_get_branch(inode, k, offsets, chain, &err); + /* Writer: pointers */ + if (!partial) + partial = chain + k-1; + /* + * If the branch acquired continuation since we've looked at it - + * fine, it should all survive and (new) top doesn't belong to us. + */ + if (!partial->key && *partial->p) + /* Writer: end */ + goto no_top; + for (p = partial; (p > chain) && all_zeroes((__le32 *) p->bh->b_data, p->p); p--) + ; + /* + * OK, we've found the last block that must survive. The rest of our + * branch should be detached before unlocking. However, if that rest + * of branch is all ours and does not grow immediately from the inode + * it's easier to cheat and just decrement partial->p. + */ + if (p == chain + k - 1 && p > chain) { + p->p--; + } else { + *top = *p->p; + /* Nope, don't do this in ext4. Must leave the tree intact */ +#if 0 + *p->p = 0; +#endif + } + /* Writer: end */ + + while (partial > p) { + brelse(partial->bh); + partial--; + } +no_top: + return partial; +} + +/* + * Zero a number of block pointers in either an inode or an indirect block. + * If we restart the transaction we must again get write access to the + * indirect block for further modification. + * + * We release `count' blocks on disk, but (last - first) may be greater + * than `count' because there can be holes in there. + * + * Return 0 on success, 1 on invalid block range + * and < 0 on fatal error. + */ +static int ext4_clear_blocks(handle_t *handle, struct inode *inode, + struct buffer_head *bh, + ext4_fsblk_t block_to_free, + unsigned long count, __le32 *first, + __le32 *last) +{ + __le32 *p; + int flags = EXT4_FREE_BLOCKS_VALIDATED; + int err; + + if (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode) || + ext4_test_inode_flag(inode, EXT4_INODE_EA_INODE)) + flags |= EXT4_FREE_BLOCKS_FORGET | EXT4_FREE_BLOCKS_METADATA; + else if (ext4_should_journal_data(inode)) + flags |= EXT4_FREE_BLOCKS_FORGET; + + if (!ext4_inode_block_valid(inode, block_to_free, count)) { + EXT4_ERROR_INODE(inode, "attempt to clear invalid " + "blocks %llu len %lu", + (unsigned long long) block_to_free, count); + return 1; + } + + err = ext4_ind_truncate_ensure_credits(handle, inode, bh, + ext4_free_data_revoke_credits(inode, count)); + if (err < 0) + goto out_err; + + for (p = first; p < last; p++) + *p = 0; + + ext4_free_blocks(handle, inode, NULL, block_to_free, count, flags); + return 0; +out_err: + ext4_std_error(inode->i_sb, err); + return err; +} + +/** + * ext4_free_data - free a list of data blocks + * @handle: handle for this transaction + * @inode: inode we are dealing with + * @this_bh: indirect buffer_head which contains *@first and *@last + * @first: array of block numbers + * @last: points immediately past the end of array + * + * We are freeing all blocks referred from that array (numbers are stored as + * little-endian 32-bit) and updating @inode->i_blocks appropriately. + * + * We accumulate contiguous runs of blocks to free. Conveniently, if these + * blocks are contiguous then releasing them at one time will only affect one + * or two bitmap blocks (+ group descriptor(s) and superblock) and we won't + * actually use a lot of journal space. + * + * @this_bh will be %NULL if @first and @last point into the inode's direct + * block pointers. + */ +static void ext4_free_data(handle_t *handle, struct inode *inode, + struct buffer_head *this_bh, + __le32 *first, __le32 *last) +{ + ext4_fsblk_t block_to_free = 0; /* Starting block # of a run */ + unsigned long count = 0; /* Number of blocks in the run */ + __le32 *block_to_free_p = NULL; /* Pointer into inode/ind + corresponding to + block_to_free */ + ext4_fsblk_t nr; /* Current block # */ + __le32 *p; /* Pointer into inode/ind + for current block */ + int err = 0; + + if (this_bh) { /* For indirect block */ + BUFFER_TRACE(this_bh, "get_write_access"); + err = ext4_journal_get_write_access(handle, inode->i_sb, + this_bh, EXT4_JTR_NONE); + /* Important: if we can't update the indirect pointers + * to the blocks, we can't free them. */ + if (err) + return; + } + + for (p = first; p < last; p++) { + nr = le32_to_cpu(*p); + if (nr) { + /* accumulate blocks to free if they're contiguous */ + if (count == 0) { + block_to_free = nr; + block_to_free_p = p; + count = 1; + } else if (nr == block_to_free + count) { + count++; + } else { + err = ext4_clear_blocks(handle, inode, this_bh, + block_to_free, count, + block_to_free_p, p); + if (err) + break; + block_to_free = nr; + block_to_free_p = p; + count = 1; + } + } + } + + if (!err && count > 0) + err = ext4_clear_blocks(handle, inode, this_bh, block_to_free, + count, block_to_free_p, p); + if (err < 0) + /* fatal error */ + return; + + if (this_bh) { + BUFFER_TRACE(this_bh, "call ext4_handle_dirty_metadata"); + + /* + * The buffer head should have an attached journal head at this + * point. However, if the data is corrupted and an indirect + * block pointed to itself, it would have been detached when + * the block was cleared. Check for this instead of OOPSing. + */ + if ((EXT4_JOURNAL(inode) == NULL) || bh2jh(this_bh)) + ext4_handle_dirty_metadata(handle, inode, this_bh); + else + EXT4_ERROR_INODE(inode, + "circular indirect block detected at " + "block %llu", + (unsigned long long) this_bh->b_blocknr); + } +} + +/** + * ext4_free_branches - free an array of branches + * @handle: JBD handle for this transaction + * @inode: inode we are dealing with + * @parent_bh: the buffer_head which contains *@first and *@last + * @first: array of block numbers + * @last: pointer immediately past the end of array + * @depth: depth of the branches to free + * + * We are freeing all blocks referred from these branches (numbers are + * stored as little-endian 32-bit) and updating @inode->i_blocks + * appropriately. + */ +static void ext4_free_branches(handle_t *handle, struct inode *inode, + struct buffer_head *parent_bh, + __le32 *first, __le32 *last, int depth) +{ + ext4_fsblk_t nr; + __le32 *p; + + if (ext4_handle_is_aborted(handle)) + return; + + if (depth--) { + struct buffer_head *bh; + int addr_per_block = EXT4_ADDR_PER_BLOCK(inode->i_sb); + p = last; + while (--p >= first) { + nr = le32_to_cpu(*p); + if (!nr) + continue; /* A hole */ + + if (!ext4_inode_block_valid(inode, nr, 1)) { + EXT4_ERROR_INODE(inode, + "invalid indirect mapped " + "block %lu (level %d)", + (unsigned long) nr, depth); + break; + } + + /* Go read the buffer for the next level down */ + bh = ext4_sb_bread_nofail(inode->i_sb, nr); + + /* + * A read failure? Report error and clear slot + * (should be rare). + */ + if (IS_ERR(bh)) { + ext4_error_inode_block(inode, nr, -PTR_ERR(bh), + "Read failure"); + continue; + } + + /* This zaps the entire block. Bottom up. */ + BUFFER_TRACE(bh, "free child branches"); + ext4_free_branches(handle, inode, bh, + (__le32 *) bh->b_data, + (__le32 *) bh->b_data + addr_per_block, + depth); + brelse(bh); + + /* + * Everything below this pointer has been + * released. Now let this top-of-subtree go. + * + * We want the freeing of this indirect block to be + * atomic in the journal with the updating of the + * bitmap block which owns it. So make some room in + * the journal. + * + * We zero the parent pointer *after* freeing its + * pointee in the bitmaps, so if extend_transaction() + * for some reason fails to put the bitmap changes and + * the release into the same transaction, recovery + * will merely complain about releasing a free block, + * rather than leaking blocks. + */ + if (ext4_handle_is_aborted(handle)) + return; + if (ext4_ind_truncate_ensure_credits(handle, inode, + NULL, + ext4_free_metadata_revoke_credits( + inode->i_sb, 1)) < 0) + return; + + /* + * The forget flag here is critical because if + * we are journaling (and not doing data + * journaling), we have to make sure a revoke + * record is written to prevent the journal + * replay from overwriting the (former) + * indirect block if it gets reallocated as a + * data block. This must happen in the same + * transaction where the data blocks are + * actually freed. + */ + ext4_free_blocks(handle, inode, NULL, nr, 1, + EXT4_FREE_BLOCKS_METADATA| + EXT4_FREE_BLOCKS_FORGET); + + if (parent_bh) { + /* + * The block which we have just freed is + * pointed to by an indirect block: journal it + */ + BUFFER_TRACE(parent_bh, "get_write_access"); + if (!ext4_journal_get_write_access(handle, + inode->i_sb, parent_bh, + EXT4_JTR_NONE)) { + *p = 0; + BUFFER_TRACE(parent_bh, + "call ext4_handle_dirty_metadata"); + ext4_handle_dirty_metadata(handle, + inode, + parent_bh); + } + } + } + } else { + /* We have reached the bottom of the tree. */ + BUFFER_TRACE(parent_bh, "free data blocks"); + ext4_free_data(handle, inode, parent_bh, first, last); + } +} + +void ext4_ind_truncate(handle_t *handle, struct inode *inode) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + __le32 *i_data = ei->i_data; + int addr_per_block = EXT4_ADDR_PER_BLOCK(inode->i_sb); + ext4_lblk_t offsets[4]; + Indirect chain[4]; + Indirect *partial; + __le32 nr = 0; + int n = 0; + ext4_lblk_t last_block, max_block; + unsigned blocksize = inode->i_sb->s_blocksize; + + last_block = (inode->i_size + blocksize-1) + >> EXT4_BLOCK_SIZE_BITS(inode->i_sb); + max_block = (EXT4_SB(inode->i_sb)->s_bitmap_maxbytes + blocksize-1) + >> EXT4_BLOCK_SIZE_BITS(inode->i_sb); + + if (last_block != max_block) { + n = ext4_block_to_path(inode, last_block, offsets, NULL); + if (n == 0) + return; + } + + ext4_es_remove_extent(inode, last_block, EXT_MAX_BLOCKS - last_block); + + /* + * The orphan list entry will now protect us from any crash which + * occurs before the truncate completes, so it is now safe to propagate + * the new, shorter inode size (held for now in i_size) into the + * on-disk inode. We do this via i_disksize, which is the value which + * ext4 *really* writes onto the disk inode. + */ + ei->i_disksize = inode->i_size; + + if (last_block == max_block) { + /* + * It is unnecessary to free any data blocks if last_block is + * equal to the indirect block limit. + */ + return; + } else if (n == 1) { /* direct blocks */ + ext4_free_data(handle, inode, NULL, i_data+offsets[0], + i_data + EXT4_NDIR_BLOCKS); + goto do_indirects; + } + + partial = ext4_find_shared(inode, n, offsets, chain, &nr); + /* Kill the top of shared branch (not detached) */ + if (nr) { + if (partial == chain) { + /* Shared branch grows from the inode */ + ext4_free_branches(handle, inode, NULL, + &nr, &nr+1, (chain+n-1) - partial); + *partial->p = 0; + /* + * We mark the inode dirty prior to restart, + * and prior to stop. No need for it here. + */ + } else { + /* Shared branch grows from an indirect block */ + BUFFER_TRACE(partial->bh, "get_write_access"); + ext4_free_branches(handle, inode, partial->bh, + partial->p, + partial->p+1, (chain+n-1) - partial); + } + } + /* Clear the ends of indirect blocks on the shared branch */ + while (partial > chain) { + ext4_free_branches(handle, inode, partial->bh, partial->p + 1, + (__le32*)partial->bh->b_data+addr_per_block, + (chain+n-1) - partial); + BUFFER_TRACE(partial->bh, "call brelse"); + brelse(partial->bh); + partial--; + } +do_indirects: + /* Kill the remaining (whole) subtrees */ + switch (offsets[0]) { + default: + nr = i_data[EXT4_IND_BLOCK]; + if (nr) { + ext4_free_branches(handle, inode, NULL, &nr, &nr+1, 1); + i_data[EXT4_IND_BLOCK] = 0; + } + fallthrough; + case EXT4_IND_BLOCK: + nr = i_data[EXT4_DIND_BLOCK]; + if (nr) { + ext4_free_branches(handle, inode, NULL, &nr, &nr+1, 2); + i_data[EXT4_DIND_BLOCK] = 0; + } + fallthrough; + case EXT4_DIND_BLOCK: + nr = i_data[EXT4_TIND_BLOCK]; + if (nr) { + ext4_free_branches(handle, inode, NULL, &nr, &nr+1, 3); + i_data[EXT4_TIND_BLOCK] = 0; + } + fallthrough; + case EXT4_TIND_BLOCK: + ; + } +} + +/** + * ext4_ind_remove_space - remove space from the range + * @handle: JBD handle for this transaction + * @inode: inode we are dealing with + * @start: First block to remove + * @end: One block after the last block to remove (exclusive) + * + * Free the blocks in the defined range (end is exclusive endpoint of + * range). This is used by ext4_punch_hole(). + */ +int ext4_ind_remove_space(handle_t *handle, struct inode *inode, + ext4_lblk_t start, ext4_lblk_t end) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + __le32 *i_data = ei->i_data; + int addr_per_block = EXT4_ADDR_PER_BLOCK(inode->i_sb); + ext4_lblk_t offsets[4], offsets2[4]; + Indirect chain[4], chain2[4]; + Indirect *partial, *partial2; + Indirect *p = NULL, *p2 = NULL; + ext4_lblk_t max_block; + __le32 nr = 0, nr2 = 0; + int n = 0, n2 = 0; + unsigned blocksize = inode->i_sb->s_blocksize; + + max_block = (EXT4_SB(inode->i_sb)->s_bitmap_maxbytes + blocksize-1) + >> EXT4_BLOCK_SIZE_BITS(inode->i_sb); + if (end >= max_block) + end = max_block; + if ((start >= end) || (start > max_block)) + return 0; + + n = ext4_block_to_path(inode, start, offsets, NULL); + n2 = ext4_block_to_path(inode, end, offsets2, NULL); + + BUG_ON(n > n2); + + if ((n == 1) && (n == n2)) { + /* We're punching only within direct block range */ + ext4_free_data(handle, inode, NULL, i_data + offsets[0], + i_data + offsets2[0]); + return 0; + } else if (n2 > n) { + /* + * Start and end are on a different levels so we're going to + * free partial block at start, and partial block at end of + * the range. If there are some levels in between then + * do_indirects label will take care of that. + */ + + if (n == 1) { + /* + * Start is at the direct block level, free + * everything to the end of the level. + */ + ext4_free_data(handle, inode, NULL, i_data + offsets[0], + i_data + EXT4_NDIR_BLOCKS); + goto end_range; + } + + + partial = p = ext4_find_shared(inode, n, offsets, chain, &nr); + if (nr) { + if (partial == chain) { + /* Shared branch grows from the inode */ + ext4_free_branches(handle, inode, NULL, + &nr, &nr+1, (chain+n-1) - partial); + *partial->p = 0; + } else { + /* Shared branch grows from an indirect block */ + BUFFER_TRACE(partial->bh, "get_write_access"); + ext4_free_branches(handle, inode, partial->bh, + partial->p, + partial->p+1, (chain+n-1) - partial); + } + } + + /* + * Clear the ends of indirect blocks on the shared branch + * at the start of the range + */ + while (partial > chain) { + ext4_free_branches(handle, inode, partial->bh, + partial->p + 1, + (__le32 *)partial->bh->b_data+addr_per_block, + (chain+n-1) - partial); + partial--; + } + +end_range: + partial2 = p2 = ext4_find_shared(inode, n2, offsets2, chain2, &nr2); + if (nr2) { + if (partial2 == chain2) { + /* + * Remember, end is exclusive so here we're at + * the start of the next level we're not going + * to free. Everything was covered by the start + * of the range. + */ + goto do_indirects; + } + } else { + /* + * ext4_find_shared returns Indirect structure which + * points to the last element which should not be + * removed by truncate. But this is end of the range + * in punch_hole so we need to point to the next element + */ + partial2->p++; + } + + /* + * Clear the ends of indirect blocks on the shared branch + * at the end of the range + */ + while (partial2 > chain2) { + ext4_free_branches(handle, inode, partial2->bh, + (__le32 *)partial2->bh->b_data, + partial2->p, + (chain2+n2-1) - partial2); + partial2--; + } + goto do_indirects; + } + + /* Punch happened within the same level (n == n2) */ + partial = p = ext4_find_shared(inode, n, offsets, chain, &nr); + partial2 = p2 = ext4_find_shared(inode, n2, offsets2, chain2, &nr2); + + /* Free top, but only if partial2 isn't its subtree. */ + if (nr) { + int level = min(partial - chain, partial2 - chain2); + int i; + int subtree = 1; + + for (i = 0; i <= level; i++) { + if (offsets[i] != offsets2[i]) { + subtree = 0; + break; + } + } + + if (!subtree) { + if (partial == chain) { + /* Shared branch grows from the inode */ + ext4_free_branches(handle, inode, NULL, + &nr, &nr+1, + (chain+n-1) - partial); + *partial->p = 0; + } else { + /* Shared branch grows from an indirect block */ + BUFFER_TRACE(partial->bh, "get_write_access"); + ext4_free_branches(handle, inode, partial->bh, + partial->p, + partial->p+1, + (chain+n-1) - partial); + } + } + } + + if (!nr2) { + /* + * ext4_find_shared returns Indirect structure which + * points to the last element which should not be + * removed by truncate. But this is end of the range + * in punch_hole so we need to point to the next element + */ + partial2->p++; + } + + while (partial > chain || partial2 > chain2) { + int depth = (chain+n-1) - partial; + int depth2 = (chain2+n2-1) - partial2; + + if (partial > chain && partial2 > chain2 && + partial->bh->b_blocknr == partial2->bh->b_blocknr) { + /* + * We've converged on the same block. Clear the range, + * then we're done. + */ + ext4_free_branches(handle, inode, partial->bh, + partial->p + 1, + partial2->p, + (chain+n-1) - partial); + goto cleanup; + } + + /* + * The start and end partial branches may not be at the same + * level even though the punch happened within one level. So, we + * give them a chance to arrive at the same level, then walk + * them in step with each other until we converge on the same + * block. + */ + if (partial > chain && depth <= depth2) { + ext4_free_branches(handle, inode, partial->bh, + partial->p + 1, + (__le32 *)partial->bh->b_data+addr_per_block, + (chain+n-1) - partial); + partial--; + } + if (partial2 > chain2 && depth2 <= depth) { + ext4_free_branches(handle, inode, partial2->bh, + (__le32 *)partial2->bh->b_data, + partial2->p, + (chain2+n2-1) - partial2); + partial2--; + } + } + +cleanup: + while (p && p > chain) { + BUFFER_TRACE(p->bh, "call brelse"); + brelse(p->bh); + p--; + } + while (p2 && p2 > chain2) { + BUFFER_TRACE(p2->bh, "call brelse"); + brelse(p2->bh); + p2--; + } + return 0; + +do_indirects: + /* Kill the remaining (whole) subtrees */ + switch (offsets[0]) { + default: + if (++n >= n2) + break; + nr = i_data[EXT4_IND_BLOCK]; + if (nr) { + ext4_free_branches(handle, inode, NULL, &nr, &nr+1, 1); + i_data[EXT4_IND_BLOCK] = 0; + } + fallthrough; + case EXT4_IND_BLOCK: + if (++n >= n2) + break; + nr = i_data[EXT4_DIND_BLOCK]; + if (nr) { + ext4_free_branches(handle, inode, NULL, &nr, &nr+1, 2); + i_data[EXT4_DIND_BLOCK] = 0; + } + fallthrough; + case EXT4_DIND_BLOCK: + if (++n >= n2) + break; + nr = i_data[EXT4_TIND_BLOCK]; + if (nr) { + ext4_free_branches(handle, inode, NULL, &nr, &nr+1, 3); + i_data[EXT4_TIND_BLOCK] = 0; + } + fallthrough; + case EXT4_TIND_BLOCK: + ; + } + goto cleanup; +} diff --git a/fs/ext4/inline.c b/fs/ext4/inline.c new file mode 100644 index 000000000..ceee69a66 --- /dev/null +++ b/fs/ext4/inline.c @@ -0,0 +1,2018 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * Copyright (c) 2012 Taobao. + * Written by Tao Ma + */ + +#include +#include +#include +#include +#include + +#include "ext4_jbd2.h" +#include "ext4.h" +#include "xattr.h" +#include "truncate.h" + +#define EXT4_XATTR_SYSTEM_DATA "data" +#define EXT4_MIN_INLINE_DATA_SIZE ((sizeof(__le32) * EXT4_N_BLOCKS)) +#define EXT4_INLINE_DOTDOT_OFFSET 2 +#define EXT4_INLINE_DOTDOT_SIZE 4 + + +static int ext4_da_convert_inline_data_to_extent(struct address_space *mapping, + struct inode *inode); + +static int ext4_get_inline_size(struct inode *inode) +{ + if (EXT4_I(inode)->i_inline_off) + return EXT4_I(inode)->i_inline_size; + + return 0; +} + +static int get_max_inline_xattr_value_size(struct inode *inode, + struct ext4_iloc *iloc) +{ + struct ext4_xattr_ibody_header *header; + struct ext4_xattr_entry *entry; + struct ext4_inode *raw_inode; + void *end; + int free, min_offs; + + if (!EXT4_INODE_HAS_XATTR_SPACE(inode)) + return 0; + + min_offs = EXT4_SB(inode->i_sb)->s_inode_size - + EXT4_GOOD_OLD_INODE_SIZE - + EXT4_I(inode)->i_extra_isize - + sizeof(struct ext4_xattr_ibody_header); + + /* + * We need to subtract another sizeof(__u32) since an in-inode xattr + * needs an empty 4 bytes to indicate the gap between the xattr entry + * and the name/value pair. + */ + if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR)) + return EXT4_XATTR_SIZE(min_offs - + EXT4_XATTR_LEN(strlen(EXT4_XATTR_SYSTEM_DATA)) - + EXT4_XATTR_ROUND - sizeof(__u32)); + + raw_inode = ext4_raw_inode(iloc); + header = IHDR(inode, raw_inode); + entry = IFIRST(header); + end = (void *)raw_inode + EXT4_SB(inode->i_sb)->s_inode_size; + + /* Compute min_offs. */ + while (!IS_LAST_ENTRY(entry)) { + void *next = EXT4_XATTR_NEXT(entry); + + if (next >= end) { + EXT4_ERROR_INODE(inode, + "corrupt xattr in inline inode"); + return 0; + } + if (!entry->e_value_inum && entry->e_value_size) { + size_t offs = le16_to_cpu(entry->e_value_offs); + if (offs < min_offs) + min_offs = offs; + } + entry = next; + } + free = min_offs - + ((void *)entry - (void *)IFIRST(header)) - sizeof(__u32); + + if (EXT4_I(inode)->i_inline_off) { + entry = (struct ext4_xattr_entry *) + ((void *)raw_inode + EXT4_I(inode)->i_inline_off); + + free += EXT4_XATTR_SIZE(le32_to_cpu(entry->e_value_size)); + goto out; + } + + free -= EXT4_XATTR_LEN(strlen(EXT4_XATTR_SYSTEM_DATA)); + + if (free > EXT4_XATTR_ROUND) + free = EXT4_XATTR_SIZE(free - EXT4_XATTR_ROUND); + else + free = 0; + +out: + return free; +} + +/* + * Get the maximum size we now can store in an inode. + * If we can't find the space for a xattr entry, don't use the space + * of the extents since we have no space to indicate the inline data. + */ +int ext4_get_max_inline_size(struct inode *inode) +{ + int error, max_inline_size; + struct ext4_iloc iloc; + + if (EXT4_I(inode)->i_extra_isize == 0) + return 0; + + error = ext4_get_inode_loc(inode, &iloc); + if (error) { + ext4_error_inode_err(inode, __func__, __LINE__, 0, -error, + "can't get inode location %llu", + inode->i_ino); + return 0; + } + + down_read(&EXT4_I(inode)->xattr_sem); + max_inline_size = get_max_inline_xattr_value_size(inode, &iloc); + up_read(&EXT4_I(inode)->xattr_sem); + + brelse(iloc.bh); + + if (!max_inline_size) + return 0; + + return max_inline_size + EXT4_MIN_INLINE_DATA_SIZE; +} + +/* + * this function does not take xattr_sem, which is OK because it is + * currently only used in a code path coming form ext4_iget, before + * the new inode has been unlocked + */ +int ext4_find_inline_data_nolock(struct inode *inode) +{ + struct ext4_xattr_ibody_find is = { + .s = { .not_found = -ENODATA, }, + }; + struct ext4_xattr_info i = { + .name_index = EXT4_XATTR_INDEX_SYSTEM, + .name = EXT4_XATTR_SYSTEM_DATA, + }; + int error; + + if (EXT4_I(inode)->i_extra_isize == 0) + return 0; + + error = ext4_get_inode_loc(inode, &is.iloc); + if (error) + return error; + + error = ext4_xattr_ibody_find(inode, &i, &is); + if (error) + goto out; + + if (!is.s.not_found) { + if (is.s.here->e_value_inum) { + EXT4_ERROR_INODE(inode, "inline data xattr refers " + "to an external xattr inode"); + error = -EFSCORRUPTED; + goto out; + } + EXT4_I(inode)->i_inline_off = (u16)((void *)is.s.here - + (void *)ext4_raw_inode(&is.iloc)); + EXT4_I(inode)->i_inline_size = EXT4_MIN_INLINE_DATA_SIZE + + le32_to_cpu(is.s.here->e_value_size); + } +out: + brelse(is.iloc.bh); + return error; +} + +static int ext4_read_inline_data(struct inode *inode, void *buffer, + unsigned int len, + struct ext4_iloc *iloc) +{ + struct ext4_xattr_entry *entry; + struct ext4_xattr_ibody_header *header; + int cp_len = 0; + struct ext4_inode *raw_inode; + + if (!len) + return 0; + + BUG_ON(len > EXT4_I(inode)->i_inline_size); + + cp_len = min_t(unsigned int, len, EXT4_MIN_INLINE_DATA_SIZE); + + raw_inode = ext4_raw_inode(iloc); + memcpy(buffer, (void *)(raw_inode->i_block), cp_len); + + len -= cp_len; + buffer += cp_len; + + if (!len) + goto out; + + header = IHDR(inode, raw_inode); + entry = (struct ext4_xattr_entry *)((void *)raw_inode + + EXT4_I(inode)->i_inline_off); + len = min_t(unsigned int, len, + (unsigned int)le32_to_cpu(entry->e_value_size)); + + memcpy(buffer, + (void *)IFIRST(header) + le16_to_cpu(entry->e_value_offs), len); + cp_len += len; + +out: + return cp_len; +} + +/* + * write the buffer to the inline inode. + * If 'create' is set, we don't need to do the extra copy in the xattr + * value since it is already handled by ext4_xattr_ibody_set. + * That saves us one memcpy. + */ +static void ext4_write_inline_data(struct inode *inode, struct ext4_iloc *iloc, + void *buffer, loff_t pos, unsigned int len) +{ + struct ext4_xattr_entry *entry; + struct ext4_xattr_ibody_header *header; + struct ext4_inode *raw_inode; + int cp_len = 0; + + if (unlikely(ext4_emergency_state(inode->i_sb))) + return; + + BUG_ON(!EXT4_I(inode)->i_inline_off); + BUG_ON(pos + len > EXT4_I(inode)->i_inline_size); + + raw_inode = ext4_raw_inode(iloc); + buffer += pos; + + if (pos < EXT4_MIN_INLINE_DATA_SIZE) { + cp_len = pos + len > EXT4_MIN_INLINE_DATA_SIZE ? + EXT4_MIN_INLINE_DATA_SIZE - pos : len; + memcpy((void *)raw_inode->i_block + pos, buffer, cp_len); + + len -= cp_len; + buffer += cp_len; + pos += cp_len; + } + + if (!len) + return; + + pos -= EXT4_MIN_INLINE_DATA_SIZE; + header = IHDR(inode, raw_inode); + entry = (struct ext4_xattr_entry *)((void *)raw_inode + + EXT4_I(inode)->i_inline_off); + + memcpy((void *)IFIRST(header) + le16_to_cpu(entry->e_value_offs) + pos, + buffer, len); +} + +static int ext4_create_inline_data(handle_t *handle, + struct inode *inode, unsigned len) +{ + int error; + void *value = NULL; + struct ext4_xattr_ibody_find is = { + .s = { .not_found = -ENODATA, }, + }; + struct ext4_xattr_info i = { + .name_index = EXT4_XATTR_INDEX_SYSTEM, + .name = EXT4_XATTR_SYSTEM_DATA, + }; + + error = ext4_get_inode_loc(inode, &is.iloc); + if (error) + return error; + + BUFFER_TRACE(is.iloc.bh, "get_write_access"); + error = ext4_journal_get_write_access(handle, inode->i_sb, is.iloc.bh, + EXT4_JTR_NONE); + if (error) + goto out; + + if (len > EXT4_MIN_INLINE_DATA_SIZE) { + value = EXT4_ZERO_XATTR_VALUE; + len -= EXT4_MIN_INLINE_DATA_SIZE; + } else { + value = ""; + len = 0; + } + + /* Insert the xttr entry. */ + i.value = value; + i.value_len = len; + + error = ext4_xattr_ibody_find(inode, &i, &is); + if (error) + goto out; + + if (!is.s.not_found) { + EXT4_ERROR_INODE(inode, "unexpected inline data xattr"); + error = -EFSCORRUPTED; + goto out; + } + + error = ext4_xattr_ibody_set(handle, inode, &i, &is); + if (error) { + if (error == -ENOSPC) + ext4_clear_inode_state(inode, + EXT4_STATE_MAY_INLINE_DATA); + goto out; + } + + memset((void *)ext4_raw_inode(&is.iloc)->i_block, + 0, EXT4_MIN_INLINE_DATA_SIZE); + + EXT4_I(inode)->i_inline_off = (u16)((void *)is.s.here - + (void *)ext4_raw_inode(&is.iloc)); + EXT4_I(inode)->i_inline_size = len + EXT4_MIN_INLINE_DATA_SIZE; + ext4_clear_inode_flag(inode, EXT4_INODE_EXTENTS); + ext4_set_inode_flag(inode, EXT4_INODE_INLINE_DATA); + get_bh(is.iloc.bh); + error = ext4_mark_iloc_dirty(handle, inode, &is.iloc); + +out: + brelse(is.iloc.bh); + return error; +} + +static int ext4_update_inline_data(handle_t *handle, struct inode *inode, + unsigned int len) +{ + int error; + void *value = NULL; + struct ext4_xattr_ibody_find is = { + .s = { .not_found = -ENODATA, }, + }; + struct ext4_xattr_info i = { + .name_index = EXT4_XATTR_INDEX_SYSTEM, + .name = EXT4_XATTR_SYSTEM_DATA, + }; + + /* If the old space is ok, write the data directly. */ + if (len <= EXT4_I(inode)->i_inline_size) + return 0; + + error = ext4_get_inode_loc(inode, &is.iloc); + if (error) + return error; + + error = ext4_xattr_ibody_find(inode, &i, &is); + if (error) + goto out; + + if (is.s.not_found) { + EXT4_ERROR_INODE(inode, "missing inline data xattr"); + error = -EFSCORRUPTED; + goto out; + } + + len -= EXT4_MIN_INLINE_DATA_SIZE; + value = kzalloc(len, GFP_NOFS); + if (!value) { + error = -ENOMEM; + goto out; + } + + error = ext4_xattr_ibody_get(inode, i.name_index, i.name, + value, len); + if (error < 0) + goto out; + + BUFFER_TRACE(is.iloc.bh, "get_write_access"); + error = ext4_journal_get_write_access(handle, inode->i_sb, is.iloc.bh, + EXT4_JTR_NONE); + if (error) + goto out; + + /* Update the xattr entry. */ + i.value = value; + i.value_len = len; + + error = ext4_xattr_ibody_set(handle, inode, &i, &is); + if (error) + goto out; + + EXT4_I(inode)->i_inline_off = (u16)((void *)is.s.here - + (void *)ext4_raw_inode(&is.iloc)); + EXT4_I(inode)->i_inline_size = EXT4_MIN_INLINE_DATA_SIZE + + le32_to_cpu(is.s.here->e_value_size); + ext4_set_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); + get_bh(is.iloc.bh); + error = ext4_mark_iloc_dirty(handle, inode, &is.iloc); + +out: + kfree(value); + brelse(is.iloc.bh); + return error; +} + +static int ext4_prepare_inline_data(handle_t *handle, struct inode *inode, + loff_t len) +{ + int ret, size, no_expand; + struct ext4_inode_info *ei = EXT4_I(inode); + + if (!ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) + return -ENOSPC; + + size = ext4_get_max_inline_size(inode); + if (size < len) + return -ENOSPC; + + ext4_write_lock_xattr(inode, &no_expand); + /* + * ei->i_inline_size may have changed since the initial check + * if other xattrs were added. Recalculate to ensure + * ext4_update_inline_data() validates against current capacity. + */ + (void) ext4_find_inline_data_nolock(inode); + if (ei->i_inline_off) + ret = ext4_update_inline_data(handle, inode, len); + else + ret = ext4_create_inline_data(handle, inode, len); + + ext4_write_unlock_xattr(inode, &no_expand); + return ret; +} + +static int ext4_destroy_inline_data_nolock(handle_t *handle, + struct inode *inode) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + struct ext4_xattr_ibody_find is = { + .s = { .not_found = 0, }, + }; + struct ext4_xattr_info i = { + .name_index = EXT4_XATTR_INDEX_SYSTEM, + .name = EXT4_XATTR_SYSTEM_DATA, + .value = NULL, + .value_len = 0, + }; + int error; + + if (!ei->i_inline_off) + return 0; + + down_write(&ei->i_data_sem); + + error = ext4_get_inode_loc(inode, &is.iloc); + if (error) { + up_write(&ei->i_data_sem); + return error; + } + + error = ext4_xattr_ibody_find(inode, &i, &is); + if (error) + goto out; + + BUFFER_TRACE(is.iloc.bh, "get_write_access"); + error = ext4_journal_get_write_access(handle, inode->i_sb, is.iloc.bh, + EXT4_JTR_NONE); + if (error) + goto out; + + error = ext4_xattr_ibody_set(handle, inode, &i, &is); + if (error) + goto out; + + memset((void *)ext4_raw_inode(&is.iloc)->i_block, + 0, EXT4_MIN_INLINE_DATA_SIZE); + memset(ei->i_data, 0, EXT4_MIN_INLINE_DATA_SIZE); + + if (ext4_has_feature_extents(inode->i_sb)) { + if (S_ISDIR(inode->i_mode) || + S_ISREG(inode->i_mode) || S_ISLNK(inode->i_mode)) { + ext4_set_inode_flag(inode, EXT4_INODE_EXTENTS); + ext4_ext_tree_init(handle, inode); + } + } + ext4_clear_inode_flag(inode, EXT4_INODE_INLINE_DATA); + + get_bh(is.iloc.bh); + error = ext4_mark_iloc_dirty(handle, inode, &is.iloc); + + EXT4_I(inode)->i_inline_off = 0; + EXT4_I(inode)->i_inline_size = 0; + ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); +out: + brelse(is.iloc.bh); + if (error == -ENODATA) + error = 0; + up_write(&ei->i_data_sem); + return error; +} + +static int ext4_read_inline_folio(struct inode *inode, struct folio *folio) +{ + void *kaddr; + int ret = 0; + size_t len; + struct ext4_iloc iloc; + + BUG_ON(!folio_test_locked(folio)); + BUG_ON(!ext4_has_inline_data(inode)); + BUG_ON(folio->index); + + if (!EXT4_I(inode)->i_inline_off) { + ext4_warning(inode->i_sb, "inode %llu doesn't have inline data.", + inode->i_ino); + goto out; + } + + ret = ext4_get_inode_loc(inode, &iloc); + if (ret) + goto out; + + len = min_t(size_t, ext4_get_inline_size(inode), i_size_read(inode)); + + if (len > PAGE_SIZE) { + ext4_error_inode(inode, __func__, __LINE__, 0, + "inline size %zu exceeds PAGE_SIZE", len); + ret = -EFSCORRUPTED; + brelse(iloc.bh); + goto out; + } + + kaddr = kmap_local_folio(folio, 0); + ret = ext4_read_inline_data(inode, kaddr, len, &iloc); + kaddr = folio_zero_tail(folio, len, kaddr + len); + kunmap_local(kaddr); + folio_mark_uptodate(folio); + brelse(iloc.bh); + +out: + return ret; +} + +int ext4_readpage_inline(struct inode *inode, struct folio *folio) +{ + int ret = 0; + + down_read(&EXT4_I(inode)->xattr_sem); + if (!ext4_has_inline_data(inode)) { + up_read(&EXT4_I(inode)->xattr_sem); + return -EAGAIN; + } + + /* + * Current inline data can only exist in the 1st page, + * So for all the other pages, just set them uptodate. + */ + if (!folio->index) + ret = ext4_read_inline_folio(inode, folio); + else if (!folio_test_uptodate(folio)) { + folio_zero_segment(folio, 0, folio_size(folio)); + folio_mark_uptodate(folio); + } + + up_read(&EXT4_I(inode)->xattr_sem); + + folio_unlock(folio); + return ret >= 0 ? 0 : ret; +} + +static int ext4_convert_inline_data_to_extent(struct address_space *mapping, + struct inode *inode) +{ + int ret, needed_blocks, no_expand; + handle_t *handle = NULL; + int retries = 0, sem_held = 0; + struct folio *folio = NULL; + unsigned from, to; + struct ext4_iloc iloc; + + if (!ext4_has_inline_data(inode)) { + /* + * clear the flag so that no new write + * will trap here again. + */ + ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); + return 0; + } + + needed_blocks = ext4_chunk_trans_extent(inode, 1); + + ret = ext4_get_inode_loc(inode, &iloc); + if (ret) + return ret; + +retry: + handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, needed_blocks); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + handle = NULL; + goto out; + } + + /* We cannot recurse into the filesystem as the transaction is already + * started */ + folio = __filemap_get_folio(mapping, 0, FGP_WRITEBEGIN | FGP_NOFS, + mapping_gfp_mask(mapping)); + if (IS_ERR(folio)) { + ret = PTR_ERR(folio); + goto out_nofolio; + } + + ext4_write_lock_xattr(inode, &no_expand); + sem_held = 1; + /* If some one has already done this for us, just exit. */ + if (!ext4_has_inline_data(inode)) { + ret = 0; + goto out; + } + + from = 0; + to = ext4_get_inline_size(inode); + if (!folio_test_uptodate(folio)) { + ret = ext4_read_inline_folio(inode, folio); + if (ret < 0) + goto out; + } + + ext4_fc_track_inode(handle, inode); + ret = ext4_destroy_inline_data_nolock(handle, inode); + if (ret) + goto out; + + if (ext4_should_dioread_nolock(inode)) { + ret = ext4_block_write_begin(handle, folio, from, to, + ext4_get_block_unwritten); + } else + ret = ext4_block_write_begin(handle, folio, from, to, + ext4_get_block); + clear_buffer_new(folio_buffers(folio)); + + if (!ret && ext4_should_journal_data(inode)) { + ret = ext4_walk_page_buffers(handle, inode, + folio_buffers(folio), from, to, + NULL, do_journal_get_write_access); + } + + if (ret) { + folio_unlock(folio); + folio_put(folio); + folio = NULL; + ext4_orphan_add(handle, inode); + ext4_write_unlock_xattr(inode, &no_expand); + sem_held = 0; + ext4_journal_stop(handle); + handle = NULL; + ext4_truncate_failed_write(inode); + /* + * If truncate failed early the inode might + * still be on the orphan list; we need to + * make sure the inode is removed from the + * orphan list in that case. + */ + if (inode->i_nlink) + ext4_orphan_del(NULL, inode); + } + + if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)) + goto retry; + + if (folio) + block_commit_write(folio, from, to); +out: + if (folio) { + folio_unlock(folio); + folio_put(folio); + } +out_nofolio: + if (sem_held) + ext4_write_unlock_xattr(inode, &no_expand); + if (handle) + ext4_journal_stop(handle); + brelse(iloc.bh); + return ret; +} + +/* + * Prepare the write for the inline data. + * If the data can be written into the inode, we just read + * the page and make it uptodate, and start the journal. + * Otherwise read the page, makes it dirty so that it can be + * handle in writepages(the i_disksize update is left to the + * normal ext4_da_write_end). + */ +int ext4_generic_write_inline_data(struct address_space *mapping, + struct inode *inode, + loff_t pos, unsigned len, + struct folio **foliop, + bool da) +{ + int ret; + handle_t *handle; + struct folio *folio; + struct ext4_iloc iloc; + int retries = 0; + + ret = ext4_get_inode_loc(inode, &iloc); + if (ret) + return ret; + +retry_journal: + handle = ext4_journal_start(inode, EXT4_HT_INODE, 1); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + goto out_release_bh; + } + + ret = ext4_prepare_inline_data(handle, inode, pos + len); + if (ret && ret != -ENOSPC) + goto out_stop_journal; + + if (ret == -ENOSPC) { + ext4_journal_stop(handle); + if (!da) { + brelse(iloc.bh); + /* Retry inside */ + return ext4_convert_inline_data_to_extent(mapping, inode); + } + + ret = ext4_da_convert_inline_data_to_extent(mapping, inode); + if (ret == -ENOSPC && + ext4_should_retry_alloc(inode->i_sb, &retries)) + goto retry_journal; + goto out_release_bh; + } + + folio = __filemap_get_folio(mapping, 0, FGP_WRITEBEGIN | FGP_NOFS, + mapping_gfp_mask(mapping)); + if (IS_ERR(folio)) { + ret = PTR_ERR(folio); + goto out_stop_journal; + } + + down_read(&EXT4_I(inode)->xattr_sem); + /* Someone else had converted it to extent */ + if (!ext4_has_inline_data(inode)) { + ret = 0; + goto out_release_folio; + } + + if (!folio_test_uptodate(folio)) { + ret = ext4_read_inline_folio(inode, folio); + if (ret < 0) + goto out_release_folio; + } + + ret = ext4_journal_get_write_access(handle, inode->i_sb, iloc.bh, EXT4_JTR_NONE); + if (ret) + goto out_release_folio; + *foliop = folio; + up_read(&EXT4_I(inode)->xattr_sem); + brelse(iloc.bh); + return 1; + +out_release_folio: + up_read(&EXT4_I(inode)->xattr_sem); + folio_unlock(folio); + folio_put(folio); +out_stop_journal: + ext4_journal_stop(handle); +out_release_bh: + brelse(iloc.bh); + return ret; +} + +/* + * Try to write data in the inode. + * If the inode has inline data, check whether the new write can be + * in the inode also. If not, create the page the handle, move the data + * to the page make it update and let the later codes create extent for it. + */ +int ext4_try_to_write_inline_data(struct address_space *mapping, + struct inode *inode, + loff_t pos, unsigned len, + struct folio **foliop) +{ + if (pos + len > ext4_get_max_inline_size(inode)) + return ext4_convert_inline_data_to_extent(mapping, inode); + return ext4_generic_write_inline_data(mapping, inode, pos, len, + foliop, false); +} + +int ext4_write_inline_data_end(struct inode *inode, loff_t pos, unsigned len, + unsigned copied, struct folio *folio) +{ + handle_t *handle = ext4_journal_current_handle(); + int no_expand; + void *kaddr; + struct ext4_iloc iloc; + int ret = 0, ret2; + + if (unlikely(copied < len) && !folio_test_uptodate(folio)) + copied = 0; + + if (likely(copied)) { + ret = ext4_get_inode_loc(inode, &iloc); + if (ret) { + folio_unlock(folio); + folio_put(folio); + ext4_std_error(inode->i_sb, ret); + goto out; + } + ext4_write_lock_xattr(inode, &no_expand); + /* + * We could have raced with ext4_page_mkwrite() converting + * the inode and clearing the inline data flag, so we just + * release resources and retry the whole write. + */ + if (unlikely(!ext4_has_inline_data(inode))) { + ext4_write_unlock_xattr(inode, &no_expand); + brelse(iloc.bh); + folio_unlock(folio); + folio_put(folio); + ext4_journal_stop(handle); + return 0; + } + + /* + * ei->i_inline_off may have changed since + * ext4_write_begin() called + * ext4_try_to_write_inline_data() + */ + (void) ext4_find_inline_data_nolock(inode); + + kaddr = kmap_local_folio(folio, 0); + ext4_write_inline_data(inode, &iloc, kaddr, pos, copied); + kunmap_local(kaddr); + folio_mark_uptodate(folio); + /* clear dirty flag so that writepages wouldn't work for us. */ + folio_clear_dirty(folio); + + ext4_write_unlock_xattr(inode, &no_expand); + brelse(iloc.bh); + + /* + * It's important to update i_size while still holding folio + * lock: page writeout could otherwise come in and zero + * beyond i_size. + */ + ext4_update_inode_size(inode, pos + copied); + } + folio_unlock(folio); + folio_put(folio); + + /* + * Don't mark the inode dirty under folio lock. First, it unnecessarily + * makes the holding time of folio lock longer. Second, it forces lock + * ordering of folio lock and transaction start for journaling + * filesystems. + */ + if (likely(copied)) + mark_inode_dirty(inode); +out: + /* + * If we didn't copy as much data as expected, we need to trim back + * size of xattr containing inline data. + */ + if (pos + len > inode->i_size && ext4_can_truncate(inode)) + ext4_orphan_add(handle, inode); + + ret2 = ext4_journal_stop(handle); + if (!ret) + ret = ret2; + if (pos + len > inode->i_size) { + ext4_truncate_failed_write(inode); + /* + * If truncate failed early the inode might still be + * on the orphan list; we need to make sure the inode + * is removed from the orphan list in that case. + */ + if (inode->i_nlink) + ext4_orphan_del(NULL, inode); + } + return ret ? ret : copied; +} + +/* + * Try to make the page cache and handle ready for the inline data case. + * We can call this function in 2 cases: + * 1. The inode is created and the first write exceeds inline size. We can + * clear the inode state safely. + * 2. The inode has inline data, then we need to read the data, make it + * update and dirty so that ext4_da_writepages can handle it. We don't + * need to start the journal since the file's metadata isn't changed now. + */ +static int ext4_da_convert_inline_data_to_extent(struct address_space *mapping, + struct inode *inode) +{ + int ret = 0, inline_size; + struct folio *folio; + + folio = __filemap_get_folio(mapping, 0, FGP_WRITEBEGIN, + mapping_gfp_mask(mapping)); + if (IS_ERR(folio)) + return PTR_ERR(folio); + + down_read(&EXT4_I(inode)->xattr_sem); + if (!ext4_has_inline_data(inode)) { + ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); + goto out; + } + + inline_size = ext4_get_inline_size(inode); + + if (!folio_test_uptodate(folio)) { + ret = ext4_read_inline_folio(inode, folio); + if (ret < 0) + goto out; + } + + ret = ext4_block_write_begin(NULL, folio, 0, inline_size, + ext4_da_get_block_prep); + if (ret) { + up_read(&EXT4_I(inode)->xattr_sem); + folio_unlock(folio); + folio_put(folio); + ext4_truncate_failed_write(inode); + return ret; + } + + clear_buffer_new(folio_buffers(folio)); + folio_mark_dirty(folio); + folio_mark_uptodate(folio); + ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); + +out: + up_read(&EXT4_I(inode)->xattr_sem); + if (folio) { + folio_unlock(folio); + folio_put(folio); + } + return ret; +} + +#ifdef INLINE_DIR_DEBUG +void ext4_show_inline_dir(struct inode *dir, struct buffer_head *bh, + void *inline_start, int inline_size) +{ + int offset; + unsigned short de_len; + struct ext4_dir_entry_2 *de = inline_start; + void *dlimit = inline_start + inline_size; + + trace_printk("inode %llu\n", dir->i_ino); + offset = 0; + while ((void *)de < dlimit) { + de_len = ext4_rec_len_from_disk(de->rec_len, inline_size); + trace_printk("de: off %u rlen %u name %.*s nlen %u ino %u\n", + offset, de_len, de->name_len, de->name, + de->name_len, le32_to_cpu(de->inode)); + if (ext4_check_dir_entry(dir, NULL, de, bh, + inline_start, inline_size, offset)) + BUG(); + + offset += de_len; + de = (struct ext4_dir_entry_2 *) ((char *) de + de_len); + } +} +#else +#define ext4_show_inline_dir(dir, bh, inline_start, inline_size) +#endif + +/* + * Add a new entry into a inline dir. + * It will return -ENOSPC if no space is available, and -EIO + * and -EEXIST if directory entry already exists. + */ +static int ext4_add_dirent_to_inline(handle_t *handle, + struct ext4_filename *fname, + struct inode *dir, + struct inode *inode, + struct ext4_iloc *iloc, + void *inline_start, int inline_size) +{ + int err; + struct ext4_dir_entry_2 *de; + + err = ext4_find_dest_de(dir, iloc->bh, inline_start, + inline_size, fname, &de); + if (err) + return err; + + BUFFER_TRACE(iloc->bh, "get_write_access"); + err = ext4_journal_get_write_access(handle, dir->i_sb, iloc->bh, + EXT4_JTR_NONE); + if (err) + return err; + ext4_insert_dentry(dir, inode, de, inline_size, fname); + + ext4_show_inline_dir(dir, iloc->bh, inline_start, inline_size); + + /* + * XXX shouldn't update any times until successful + * completion of syscall, but too many callers depend + * on this. + * + * XXX similarly, too many callers depend on + * ext4_new_inode() setting the times, but error + * recovery deletes the inode, so the worst that can + * happen is that the times are slightly out of date + * and/or different from the directory change time. + */ + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + ext4_update_dx_flag(dir); + inode_inc_iversion(dir); + return 1; +} + +static void *ext4_get_inline_xattr_pos(struct inode *inode, + struct ext4_iloc *iloc) +{ + struct ext4_xattr_entry *entry; + struct ext4_xattr_ibody_header *header; + + BUG_ON(!EXT4_I(inode)->i_inline_off); + + header = IHDR(inode, ext4_raw_inode(iloc)); + entry = (struct ext4_xattr_entry *)((void *)ext4_raw_inode(iloc) + + EXT4_I(inode)->i_inline_off); + + return (void *)IFIRST(header) + le16_to_cpu(entry->e_value_offs); +} + +/* Set the final de to cover the whole block. */ +void ext4_update_final_de(void *de_buf, int old_size, int new_size) +{ + struct ext4_dir_entry_2 *de, *prev_de; + void *limit; + int de_len; + + de = de_buf; + if (old_size) { + limit = de_buf + old_size; + do { + prev_de = de; + de_len = ext4_rec_len_from_disk(de->rec_len, old_size); + de_buf += de_len; + de = de_buf; + } while (de_buf < limit); + + prev_de->rec_len = ext4_rec_len_to_disk(de_len + new_size - + old_size, new_size); + } else { + /* this is just created, so create an empty entry. */ + de->inode = 0; + de->rec_len = ext4_rec_len_to_disk(new_size, new_size); + } +} + +static int ext4_update_inline_dir(handle_t *handle, struct inode *dir, + struct ext4_iloc *iloc) +{ + int ret; + int old_size = EXT4_I(dir)->i_inline_size - EXT4_MIN_INLINE_DATA_SIZE; + int new_size = get_max_inline_xattr_value_size(dir, iloc); + + if (new_size - old_size <= ext4_dir_rec_len(1, NULL)) + return -ENOSPC; + + ret = ext4_update_inline_data(handle, dir, + new_size + EXT4_MIN_INLINE_DATA_SIZE); + if (ret) + return ret; + + ext4_update_final_de(ext4_get_inline_xattr_pos(dir, iloc), old_size, + EXT4_I(dir)->i_inline_size - + EXT4_MIN_INLINE_DATA_SIZE); + dir->i_size = EXT4_I(dir)->i_disksize = EXT4_I(dir)->i_inline_size; + return 0; +} + +static void ext4_restore_inline_data(handle_t *handle, struct inode *inode, + struct ext4_iloc *iloc, + void *buf, int inline_size) +{ + int ret; + + ret = ext4_create_inline_data(handle, inode, inline_size); + if (ret) { + ext4_msg(inode->i_sb, KERN_EMERG, + "error restoring inline_data for inode -- potential data loss! (inode %llu, error %d)", + inode->i_ino, ret); + return; + } + ext4_write_inline_data(inode, iloc, buf, 0, inline_size); + ext4_set_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); +} + +static int ext4_convert_inline_data_nolock(handle_t *handle, + struct inode *inode, + struct ext4_iloc *iloc) +{ + int error; + void *buf = NULL; + struct buffer_head *data_bh = NULL; + struct ext4_map_blocks map; + int inline_size; + + inline_size = ext4_get_inline_size(inode); + buf = kmalloc(inline_size, GFP_NOFS); + if (!buf) { + error = -ENOMEM; + goto out; + } + + error = ext4_read_inline_data(inode, buf, inline_size, iloc); + if (error < 0) + goto out; + + /* + * Make sure the inline directory entries pass checks before we try to + * convert them, so that we avoid touching stuff that needs fsck. + */ + if (S_ISDIR(inode->i_mode)) { + error = ext4_check_all_de(inode, iloc->bh, + buf + EXT4_INLINE_DOTDOT_SIZE, + inline_size - EXT4_INLINE_DOTDOT_SIZE); + if (error) + goto out; + } + + error = ext4_destroy_inline_data_nolock(handle, inode); + if (error) + goto out; + + map.m_lblk = 0; + map.m_len = 1; + map.m_flags = 0; + error = ext4_map_blocks(handle, inode, &map, EXT4_GET_BLOCKS_CREATE); + if (error < 0) + goto out_restore; + if (!(map.m_flags & EXT4_MAP_MAPPED)) { + error = -EIO; + goto out_restore; + } + + data_bh = sb_getblk(inode->i_sb, map.m_pblk); + if (!data_bh) { + error = -ENOMEM; + goto out_restore; + } + + lock_buffer(data_bh); + error = ext4_journal_get_create_access(handle, inode->i_sb, data_bh, + EXT4_JTR_NONE); + if (error) { + unlock_buffer(data_bh); + error = -EIO; + goto out_restore; + } + memset(data_bh->b_data, 0, inode->i_sb->s_blocksize); + + if (!S_ISDIR(inode->i_mode)) { + memcpy(data_bh->b_data, buf, inline_size); + set_buffer_uptodate(data_bh); + unlock_buffer(data_bh); + error = ext4_handle_dirty_metadata(handle, + inode, data_bh); + } else { + unlock_buffer(data_bh); + inode->i_size = inode->i_sb->s_blocksize; + i_size_write(inode, inode->i_sb->s_blocksize); + EXT4_I(inode)->i_disksize = inode->i_sb->s_blocksize; + + error = ext4_init_dirblock(handle, inode, data_bh, + le32_to_cpu(((struct ext4_dir_entry_2 *)buf)->inode), + buf + EXT4_INLINE_DOTDOT_SIZE, + inline_size - EXT4_INLINE_DOTDOT_SIZE); + if (!error) + error = ext4_mark_inode_dirty(handle, inode); + } + +out_restore: + if (error) + ext4_restore_inline_data(handle, inode, iloc, buf, inline_size); + +out: + brelse(data_bh); + kfree(buf); + return error; +} + +/* + * Try to add the new entry to the inline data. + * If succeeds, return 0. If not, extended the inline dir and copied data to + * the new created block. + */ +int ext4_try_add_inline_entry(handle_t *handle, struct ext4_filename *fname, + struct inode *dir, struct inode *inode) +{ + int ret, ret2, inline_size, no_expand; + void *inline_start; + struct ext4_iloc iloc; + + ret = ext4_get_inode_loc(dir, &iloc); + if (ret) + return ret; + + ext4_write_lock_xattr(dir, &no_expand); + if (!ext4_has_inline_data(dir)) + goto out; + + inline_start = (void *)ext4_raw_inode(&iloc)->i_block + + EXT4_INLINE_DOTDOT_SIZE; + inline_size = EXT4_MIN_INLINE_DATA_SIZE - EXT4_INLINE_DOTDOT_SIZE; + + ret = ext4_add_dirent_to_inline(handle, fname, dir, inode, &iloc, + inline_start, inline_size); + if (ret != -ENOSPC) + goto out; + + /* check whether it can be inserted to inline xattr space. */ + inline_size = EXT4_I(dir)->i_inline_size - + EXT4_MIN_INLINE_DATA_SIZE; + if (!inline_size) { + /* Try to use the xattr space.*/ + ret = ext4_update_inline_dir(handle, dir, &iloc); + if (ret && ret != -ENOSPC) + goto out; + + inline_size = EXT4_I(dir)->i_inline_size - + EXT4_MIN_INLINE_DATA_SIZE; + } + + if (inline_size) { + inline_start = ext4_get_inline_xattr_pos(dir, &iloc); + + ret = ext4_add_dirent_to_inline(handle, fname, dir, + inode, &iloc, inline_start, + inline_size); + + if (ret != -ENOSPC) + goto out; + } + + /* + * The inline space is filled up, so create a new block for it. + * As the extent tree will be created, we have to save the inline + * dir first. + */ + ret = ext4_convert_inline_data_nolock(handle, dir, &iloc); + +out: + ext4_write_unlock_xattr(dir, &no_expand); + ret2 = ext4_mark_inode_dirty(handle, dir); + if (unlikely(ret2 && !ret)) + ret = ret2; + brelse(iloc.bh); + return ret; +} + +/* + * This function fills a red-black tree with information from an + * inlined dir. It returns the number directory entries loaded + * into the tree. If there is an error it is returned in err. + */ +int ext4_inlinedir_to_tree(struct file *dir_file, + struct inode *dir, ext4_lblk_t block, + struct dx_hash_info *hinfo, + __u32 start_hash, __u32 start_minor_hash, + int *has_inline_data) +{ + int err = 0, count = 0; + unsigned int parent_ino; + int pos; + struct ext4_dir_entry_2 *de; + struct inode *inode = file_inode(dir_file); + int ret, inline_size = 0; + struct ext4_iloc iloc; + void *dir_buf = NULL; + struct ext4_dir_entry_2 fake; + struct fscrypt_str tmp_str; + + ret = ext4_get_inode_loc(inode, &iloc); + if (ret) + return ret; + + down_read(&EXT4_I(inode)->xattr_sem); + if (!ext4_has_inline_data(inode)) { + up_read(&EXT4_I(inode)->xattr_sem); + *has_inline_data = 0; + goto out; + } + + inline_size = ext4_get_inline_size(inode); + dir_buf = kmalloc(inline_size, GFP_NOFS); + if (!dir_buf) { + ret = -ENOMEM; + up_read(&EXT4_I(inode)->xattr_sem); + goto out; + } + + ret = ext4_read_inline_data(inode, dir_buf, inline_size, &iloc); + up_read(&EXT4_I(inode)->xattr_sem); + if (ret < 0) + goto out; + + pos = 0; + parent_ino = le32_to_cpu(((struct ext4_dir_entry_2 *)dir_buf)->inode); + while (pos < inline_size) { + /* + * As inlined dir doesn't store any information about '.' and + * only the inode number of '..' is stored, we have to handle + * them differently. + */ + if (pos == 0) { + fake.inode = cpu_to_le32(inode->i_ino); + fake.name_len = 1; + memcpy(fake.name, ".", 2); + fake.rec_len = ext4_rec_len_to_disk( + ext4_dir_rec_len(fake.name_len, NULL), + inline_size); + ext4_set_de_type(inode->i_sb, &fake, S_IFDIR); + de = &fake; + pos = EXT4_INLINE_DOTDOT_OFFSET; + } else if (pos == EXT4_INLINE_DOTDOT_OFFSET) { + fake.inode = cpu_to_le32(parent_ino); + fake.name_len = 2; + memcpy(fake.name, "..", 3); + fake.rec_len = ext4_rec_len_to_disk( + ext4_dir_rec_len(fake.name_len, NULL), + inline_size); + ext4_set_de_type(inode->i_sb, &fake, S_IFDIR); + de = &fake; + pos = EXT4_INLINE_DOTDOT_SIZE; + } else { + de = (struct ext4_dir_entry_2 *)(dir_buf + pos); + pos += ext4_rec_len_from_disk(de->rec_len, inline_size); + if (ext4_check_dir_entry(inode, dir_file, de, + iloc.bh, dir_buf, + inline_size, pos)) { + ret = count; + goto out; + } + } + + if (ext4_hash_in_dirent(dir)) { + hinfo->hash = EXT4_DIRENT_HASH(de); + hinfo->minor_hash = EXT4_DIRENT_MINOR_HASH(de); + } else { + err = ext4fs_dirhash(dir, de->name, de->name_len, hinfo); + if (err) { + ret = err; + goto out; + } + } + if ((hinfo->hash < start_hash) || + ((hinfo->hash == start_hash) && + (hinfo->minor_hash < start_minor_hash))) + continue; + if (de->inode == 0) + continue; + tmp_str.name = de->name; + tmp_str.len = de->name_len; + err = ext4_htree_store_dirent(dir_file, hinfo->hash, + hinfo->minor_hash, de, &tmp_str); + if (err) { + ret = err; + goto out; + } + count++; + } + ret = count; +out: + kfree(dir_buf); + brelse(iloc.bh); + return ret; +} + +/* + * So this function is called when the volume is mkfsed with + * dir_index disabled. In order to keep f_pos persistent + * after we convert from an inlined dir to a blocked based, + * we just pretend that we are a normal dir and return the + * offset as if '.' and '..' really take place. + * + */ +int ext4_read_inline_dir(struct file *file, + struct dir_context *ctx, + int *has_inline_data) +{ + unsigned int offset, parent_ino; + int i; + struct ext4_dir_entry_2 *de; + struct super_block *sb; + struct inode *inode = file_inode(file); + int ret, inline_size = 0; + struct ext4_iloc iloc; + void *dir_buf = NULL; + int dotdot_offset, dotdot_size, extra_offset, extra_size; + struct dir_private_info *info = file->private_data; + + ret = ext4_get_inode_loc(inode, &iloc); + if (ret) + return ret; + + down_read(&EXT4_I(inode)->xattr_sem); + if (!ext4_has_inline_data(inode)) { + up_read(&EXT4_I(inode)->xattr_sem); + *has_inline_data = 0; + goto out; + } + + inline_size = ext4_get_inline_size(inode); + dir_buf = kmalloc(inline_size, GFP_NOFS); + if (!dir_buf) { + ret = -ENOMEM; + up_read(&EXT4_I(inode)->xattr_sem); + goto out; + } + + ret = ext4_read_inline_data(inode, dir_buf, inline_size, &iloc); + up_read(&EXT4_I(inode)->xattr_sem); + if (ret < 0) + goto out; + + ret = 0; + sb = inode->i_sb; + parent_ino = le32_to_cpu(((struct ext4_dir_entry_2 *)dir_buf)->inode); + offset = ctx->pos; + + /* + * dotdot_offset and dotdot_size is the real offset and + * size for ".." and "." if the dir is block based while + * the real size for them are only EXT4_INLINE_DOTDOT_SIZE. + * So we will use extra_offset and extra_size to indicate them + * during the inline dir iteration. + */ + dotdot_offset = ext4_dir_rec_len(1, NULL); + dotdot_size = dotdot_offset + ext4_dir_rec_len(2, NULL); + extra_offset = dotdot_size - EXT4_INLINE_DOTDOT_SIZE; + extra_size = extra_offset + inline_size; + + /* + * If the cookie has changed since the last call to + * readdir(2), then we might be pointing to an invalid + * dirent right now. Scan from the start of the inline + * dir to make sure. + */ + if (!inode_eq_iversion(inode, info->cookie)) { + for (i = 0; i < extra_size && i < offset;) { + /* + * "." is with offset 0 and + * ".." is dotdot_offset. + */ + if (!i) { + i = dotdot_offset; + continue; + } else if (i == dotdot_offset) { + i = dotdot_size; + continue; + } + /* for other entry, the real offset in + * the buf has to be tuned accordingly. + */ + if (i + ext4_dir_rec_len(1, NULL) > extra_size) + break; + de = (struct ext4_dir_entry_2 *) + (dir_buf + i - extra_offset); + /* It's too expensive to do a full + * dirent test each time round this + * loop, but we do have to test at + * least that it is non-zero. A + * failure will be detected in the + * dirent test below. */ + if (ext4_rec_len_from_disk(de->rec_len, extra_size) + < ext4_dir_rec_len(1, NULL)) + break; + i += ext4_rec_len_from_disk(de->rec_len, + extra_size); + } + offset = i; + ctx->pos = offset; + info->cookie = inode_query_iversion(inode); + } + + while (ctx->pos < extra_size) { + if (ctx->pos == 0) { + if (!dir_emit(ctx, ".", 1, inode->i_ino, DT_DIR)) + goto out; + ctx->pos = dotdot_offset; + continue; + } + + if (ctx->pos == dotdot_offset) { + if (!dir_emit(ctx, "..", 2, parent_ino, DT_DIR)) + goto out; + ctx->pos = dotdot_size; + continue; + } + + /* + * de lives at dir_buf + ctx->pos - extra_offset, within the + * kmalloc(inline_size) buffer. Make sure its header fits before + * ext4_check_dir_entry() dereferences de->rec_len. + */ + if (ctx->pos + ext4_dir_rec_len(1, NULL) > extra_size) + goto out; + de = (struct ext4_dir_entry_2 *) + (dir_buf + ctx->pos - extra_offset); + if (ext4_check_dir_entry(inode, file, de, iloc.bh, dir_buf, + inline_size, ctx->pos)) + goto out; + if (le32_to_cpu(de->inode)) { + if (!dir_emit(ctx, de->name, de->name_len, + le32_to_cpu(de->inode), + get_dtype(sb, de->file_type))) + goto out; + } + ctx->pos += ext4_rec_len_from_disk(de->rec_len, extra_size); + } +out: + kfree(dir_buf); + brelse(iloc.bh); + return ret; +} + +void *ext4_read_inline_link(struct inode *inode) +{ + struct ext4_iloc iloc; + int ret, inline_size; + void *link; + + ret = ext4_get_inode_loc(inode, &iloc); + if (ret) + return ERR_PTR(ret); + + ret = -ENOMEM; + inline_size = ext4_get_inline_size(inode); + link = kmalloc(inline_size + 1, GFP_NOFS); + if (!link) + goto out; + + ret = ext4_read_inline_data(inode, link, inline_size, &iloc); + if (ret < 0) { + kfree(link); + goto out; + } + nd_terminate_link(link, inode->i_size, ret); +out: + if (ret < 0) + link = ERR_PTR(ret); + brelse(iloc.bh); + return link; +} + +struct buffer_head *ext4_get_first_inline_block(struct inode *inode, + struct ext4_dir_entry_2 **parent_de, + int *retval) +{ + struct ext4_iloc iloc; + + *retval = ext4_get_inode_loc(inode, &iloc); + if (*retval) + return NULL; + + *parent_de = (struct ext4_dir_entry_2 *)ext4_raw_inode(&iloc)->i_block; + + return iloc.bh; +} + +/* + * Try to create the inline data for the new dir. + * If it succeeds, return 0, otherwise return the error. + * In case of ENOSPC, the caller should create the normal disk layout dir. + */ +int ext4_try_create_inline_dir(handle_t *handle, struct inode *parent, + struct inode *inode) +{ + int ret, inline_size = EXT4_MIN_INLINE_DATA_SIZE; + struct ext4_iloc iloc; + struct ext4_dir_entry_2 *de; + + ret = ext4_get_inode_loc(inode, &iloc); + if (ret) + return ret; + + ret = ext4_prepare_inline_data(handle, inode, inline_size); + if (ret) + goto out; + + /* + * For inline dir, we only save the inode information for the ".." + * and create a fake dentry to cover the left space. + */ + de = (struct ext4_dir_entry_2 *)ext4_raw_inode(&iloc)->i_block; + de->inode = cpu_to_le32(parent->i_ino); + de = (struct ext4_dir_entry_2 *)((void *)de + EXT4_INLINE_DOTDOT_SIZE); + de->inode = 0; + de->rec_len = ext4_rec_len_to_disk( + inline_size - EXT4_INLINE_DOTDOT_SIZE, + inline_size); + set_nlink(inode, 2); + inode->i_size = EXT4_I(inode)->i_disksize = inline_size; +out: + brelse(iloc.bh); + return ret; +} + +struct buffer_head *ext4_find_inline_entry(struct inode *dir, + struct ext4_filename *fname, + struct ext4_dir_entry_2 **res_dir, + int *has_inline_data) +{ + struct ext4_xattr_ibody_find is = { + .s = { .not_found = -ENODATA, }, + }; + struct ext4_xattr_info i = { + .name_index = EXT4_XATTR_INDEX_SYSTEM, + .name = EXT4_XATTR_SYSTEM_DATA, + }; + int ret; + void *inline_start; + int inline_size; + + ret = ext4_get_inode_loc(dir, &is.iloc); + if (ret) + return ERR_PTR(ret); + + down_read(&EXT4_I(dir)->xattr_sem); + + ret = ext4_xattr_ibody_find(dir, &i, &is); + if (ret) + goto out; + + if (!ext4_has_inline_data(dir)) { + *has_inline_data = 0; + goto out; + } + + inline_start = (void *)ext4_raw_inode(&is.iloc)->i_block + + EXT4_INLINE_DOTDOT_SIZE; + inline_size = EXT4_MIN_INLINE_DATA_SIZE - EXT4_INLINE_DOTDOT_SIZE; + ret = ext4_search_dir(is.iloc.bh, inline_start, inline_size, + dir, fname, 0, res_dir); + if (ret == 1) + goto out_find; + if (ret < 0) + goto out; + + if (ext4_get_inline_size(dir) == EXT4_MIN_INLINE_DATA_SIZE) + goto out; + + inline_start = ext4_get_inline_xattr_pos(dir, &is.iloc); + inline_size = ext4_get_inline_size(dir) - EXT4_MIN_INLINE_DATA_SIZE; + + ret = ext4_search_dir(is.iloc.bh, inline_start, inline_size, + dir, fname, 0, res_dir); + if (ret == 1) + goto out_find; + +out: + brelse(is.iloc.bh); + if (ret < 0) + is.iloc.bh = ERR_PTR(ret); + else + is.iloc.bh = NULL; +out_find: + up_read(&EXT4_I(dir)->xattr_sem); + return is.iloc.bh; +} + +int ext4_delete_inline_entry(handle_t *handle, + struct inode *dir, + struct ext4_dir_entry_2 *de_del, + struct buffer_head *bh, + int *has_inline_data) +{ + int err, inline_size, no_expand; + struct ext4_iloc iloc; + void *inline_start; + + err = ext4_get_inode_loc(dir, &iloc); + if (err) + return err; + + ext4_write_lock_xattr(dir, &no_expand); + if (!ext4_has_inline_data(dir)) { + *has_inline_data = 0; + goto out; + } + + if ((void *)de_del - ((void *)ext4_raw_inode(&iloc)->i_block) < + EXT4_MIN_INLINE_DATA_SIZE) { + inline_start = (void *)ext4_raw_inode(&iloc)->i_block + + EXT4_INLINE_DOTDOT_SIZE; + inline_size = EXT4_MIN_INLINE_DATA_SIZE - + EXT4_INLINE_DOTDOT_SIZE; + } else { + inline_start = ext4_get_inline_xattr_pos(dir, &iloc); + inline_size = ext4_get_inline_size(dir) - + EXT4_MIN_INLINE_DATA_SIZE; + } + + BUFFER_TRACE(bh, "get_write_access"); + err = ext4_journal_get_write_access(handle, dir->i_sb, bh, + EXT4_JTR_NONE); + if (err) + goto out; + + err = ext4_generic_delete_entry(dir, de_del, bh, + inline_start, inline_size, 0); + if (err) + goto out; + + ext4_show_inline_dir(dir, iloc.bh, inline_start, inline_size); +out: + ext4_write_unlock_xattr(dir, &no_expand); + if (likely(err == 0)) + err = ext4_mark_inode_dirty(handle, dir); + brelse(iloc.bh); + if (err != -ENOENT) + ext4_std_error(dir->i_sb, err); + return err; +} + +/* + * Get the inline dentry at offset. + */ +static inline struct ext4_dir_entry_2 * +ext4_get_inline_entry(struct inode *inode, + struct ext4_iloc *iloc, + unsigned int offset, + void **inline_start, + int *inline_size) +{ + void *inline_pos; + + BUG_ON(offset > ext4_get_inline_size(inode)); + + if (offset < EXT4_MIN_INLINE_DATA_SIZE) { + inline_pos = (void *)ext4_raw_inode(iloc)->i_block; + *inline_size = EXT4_MIN_INLINE_DATA_SIZE; + } else { + inline_pos = ext4_get_inline_xattr_pos(inode, iloc); + offset -= EXT4_MIN_INLINE_DATA_SIZE; + *inline_size = ext4_get_inline_size(inode) - + EXT4_MIN_INLINE_DATA_SIZE; + } + + if (inline_start) + *inline_start = inline_pos; + return (struct ext4_dir_entry_2 *)(inline_pos + offset); +} + +bool empty_inline_dir(struct inode *dir, int *has_inline_data) +{ + int err, inline_size; + struct ext4_iloc iloc; + size_t inline_len; + void *inline_pos; + unsigned int offset; + struct ext4_dir_entry_2 *de; + bool ret = false; + + err = ext4_get_inode_loc(dir, &iloc); + if (err) { + EXT4_ERROR_INODE_ERR(dir, -err, + "error %d getting inode %llu block", + err, dir->i_ino); + return false; + } + + down_read(&EXT4_I(dir)->xattr_sem); + if (!ext4_has_inline_data(dir)) { + *has_inline_data = 0; + ret = true; + goto out; + } + + de = (struct ext4_dir_entry_2 *)ext4_raw_inode(&iloc)->i_block; + if (!le32_to_cpu(de->inode)) { + ext4_warning(dir->i_sb, + "bad inline directory (dir #%llu) - no `..'", + dir->i_ino); + goto out; + } + + inline_len = ext4_get_inline_size(dir); + offset = EXT4_INLINE_DOTDOT_SIZE; + while (offset < inline_len) { + de = ext4_get_inline_entry(dir, &iloc, offset, + &inline_pos, &inline_size); + if (ext4_check_dir_entry(dir, NULL, de, + iloc.bh, inline_pos, + inline_size, offset)) { + ext4_warning(dir->i_sb, + "bad inline directory (dir #%llu) - " + "inode %u, rec_len %u, name_len %d" + "inline size %d", + dir->i_ino, le32_to_cpu(de->inode), + le16_to_cpu(de->rec_len), de->name_len, + inline_size); + goto out; + } + if (le32_to_cpu(de->inode)) { + goto out; + } + offset += ext4_rec_len_from_disk(de->rec_len, inline_size); + } + + ret = true; +out: + up_read(&EXT4_I(dir)->xattr_sem); + brelse(iloc.bh); + return ret; +} + +int ext4_destroy_inline_data(handle_t *handle, struct inode *inode) +{ + int ret, no_expand; + + ext4_write_lock_xattr(inode, &no_expand); + ret = ext4_destroy_inline_data_nolock(handle, inode); + ext4_write_unlock_xattr(inode, &no_expand); + + return ret; +} + +int ext4_inline_data_iomap(struct inode *inode, struct iomap *iomap) +{ + __u64 addr; + int error = -EAGAIN; + struct ext4_iloc iloc; + + down_read(&EXT4_I(inode)->xattr_sem); + if (!ext4_has_inline_data(inode)) + goto out; + + error = ext4_get_inode_loc(inode, &iloc); + if (error) + goto out; + + addr = (__u64)iloc.bh->b_blocknr << inode->i_sb->s_blocksize_bits; + addr += (char *)ext4_raw_inode(&iloc) - iloc.bh->b_data; + addr += offsetof(struct ext4_inode, i_block); + + brelse(iloc.bh); + + iomap->addr = addr; + iomap->offset = 0; + iomap->length = min_t(loff_t, ext4_get_inline_size(inode), + i_size_read(inode)); + iomap->type = IOMAP_INLINE; + iomap->flags = 0; + +out: + up_read(&EXT4_I(inode)->xattr_sem); + return error; +} + +int ext4_inline_data_truncate(struct inode *inode, int *has_inline) +{ + handle_t *handle; + int inline_size, value_len, needed_blocks, no_expand, err = 0; + size_t i_size; + void *value = NULL; + struct ext4_xattr_ibody_find is = { + .s = { .not_found = -ENODATA, }, + }; + struct ext4_xattr_info i = { + .name_index = EXT4_XATTR_INDEX_SYSTEM, + .name = EXT4_XATTR_SYSTEM_DATA, + }; + + + needed_blocks = ext4_chunk_trans_extent(inode, 1); + handle = ext4_journal_start(inode, EXT4_HT_INODE, needed_blocks); + if (IS_ERR(handle)) + return PTR_ERR(handle); + + ext4_write_lock_xattr(inode, &no_expand); + if (!ext4_has_inline_data(inode)) { + ext4_write_unlock_xattr(inode, &no_expand); + *has_inline = 0; + ext4_journal_stop(handle); + return 0; + } + + if ((err = ext4_orphan_add(handle, inode)) != 0) + goto out; + + if ((err = ext4_get_inode_loc(inode, &is.iloc)) != 0) + goto out; + + down_write(&EXT4_I(inode)->i_data_sem); + i_size = inode->i_size; + inline_size = ext4_get_inline_size(inode); + EXT4_I(inode)->i_disksize = i_size; + + if (i_size < inline_size) { + /* + * if there's inline data to truncate and this file was + * converted to extents after that inline data was written, + * the extent status cache must be cleared to avoid leaving + * behind stale delayed allocated extent entries + */ + if (!ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) + ext4_es_remove_extent(inode, 0, EXT_MAX_BLOCKS); + + /* Clear the content in the xattr space. */ + if (inline_size > EXT4_MIN_INLINE_DATA_SIZE) { + if ((err = ext4_xattr_ibody_find(inode, &i, &is)) != 0) + goto out_error; + + if (is.s.not_found) { + EXT4_ERROR_INODE(inode, + "missing inline data xattr"); + err = -EFSCORRUPTED; + goto out_error; + } + + value_len = le32_to_cpu(is.s.here->e_value_size); + value = kmalloc(value_len, GFP_NOFS); + if (!value) { + err = -ENOMEM; + goto out_error; + } + + err = ext4_xattr_ibody_get(inode, i.name_index, + i.name, value, value_len); + if (err <= 0) + goto out_error; + + i.value = value; + i.value_len = i_size > EXT4_MIN_INLINE_DATA_SIZE ? + i_size - EXT4_MIN_INLINE_DATA_SIZE : 0; + err = ext4_xattr_ibody_set(handle, inode, &i, &is); + if (err) + goto out_error; + } + + /* Clear the content within i_blocks. */ + if (i_size < EXT4_MIN_INLINE_DATA_SIZE) { + void *p = (void *) ext4_raw_inode(&is.iloc)->i_block; + memset(p + i_size, 0, + EXT4_MIN_INLINE_DATA_SIZE - i_size); + } + + EXT4_I(inode)->i_inline_size = i_size < + EXT4_MIN_INLINE_DATA_SIZE ? + EXT4_MIN_INLINE_DATA_SIZE : i_size; + } + +out_error: + up_write(&EXT4_I(inode)->i_data_sem); +out: + brelse(is.iloc.bh); + ext4_write_unlock_xattr(inode, &no_expand); + kfree(value); + if (inode->i_nlink) + ext4_orphan_del(handle, inode); + + if (err == 0) { + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + err = ext4_mark_inode_dirty(handle, inode); + if (IS_SYNC(inode)) + ext4_handle_sync(handle); + } + ext4_journal_stop(handle); + return err; +} + +int ext4_convert_inline_data(struct inode *inode) +{ + int error, needed_blocks, no_expand; + handle_t *handle; + struct ext4_iloc iloc; + + if (!ext4_has_inline_data(inode)) { + ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); + return 0; + } else if (!ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) { + /* + * Inode has inline data but EXT4_STATE_MAY_INLINE_DATA is + * cleared. This means we are in the middle of moving of + * inline data to delay allocated block. Just force writeout + * here to finish conversion. + */ + error = filemap_flush(inode->i_mapping); + if (error) + return error; + if (!ext4_has_inline_data(inode)) + return 0; + } + + needed_blocks = ext4_chunk_trans_extent(inode, 1); + + iloc.bh = NULL; + error = ext4_get_inode_loc(inode, &iloc); + if (error) + return error; + + handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, needed_blocks); + if (IS_ERR(handle)) { + error = PTR_ERR(handle); + goto out_free; + } + + ext4_write_lock_xattr(inode, &no_expand); + if (ext4_has_inline_data(inode)) + error = ext4_convert_inline_data_nolock(handle, inode, &iloc); + ext4_write_unlock_xattr(inode, &no_expand); + ext4_journal_stop(handle); +out_free: + brelse(iloc.bh); + return error; +} diff --git a/fs/ext4/inode-test.c b/fs/ext4/inode-test.c new file mode 100644 index 000000000..749af7ad4 --- /dev/null +++ b/fs/ext4/inode-test.c @@ -0,0 +1,283 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit test of ext4 inode that verify the seconds part of [a/c/m] + * timestamps in ext4 inode structs are decoded correctly. + */ + +#include +#include +#include + +#include "ext4.h" + +/* + * For constructing the nonnegative timestamp lower bound value. + * binary: 00000000 00000000 00000000 00000000 + */ +#define LOWER_MSB_0 0L +/* + * For constructing the nonnegative timestamp upper bound value. + * binary: 01111111 11111111 11111111 11111111 + * + */ +#define UPPER_MSB_0 0x7fffffffL +/* + * For constructing the negative timestamp lower bound value. + * binary: 10000000 00000000 00000000 00000000 + */ +#define LOWER_MSB_1 (-(UPPER_MSB_0) - 1L) /* avoid overflow */ +/* + * For constructing the negative timestamp upper bound value. + * binary: 11111111 11111111 11111111 11111111 + */ +#define UPPER_MSB_1 (-1L) +/* + * Upper bound for nanoseconds value supported by the encoding. + * binary: 00111111 11111111 11111111 11111111 + */ +#define MAX_NANOSECONDS ((1L << 30) - 1) + +#define CASE_NAME_FORMAT "%s: msb:%x lower_bound:%x extra_bits: %x" + +#define LOWER_BOUND_NEG_NO_EXTRA_BITS_CASE\ + "1901-12-13 Lower bound of 32bit < 0 timestamp, no extra bits" +#define UPPER_BOUND_NEG_NO_EXTRA_BITS_CASE\ + "1969-12-31 Upper bound of 32bit < 0 timestamp, no extra bits" +#define LOWER_BOUND_NONNEG_NO_EXTRA_BITS_CASE\ + "1970-01-01 Lower bound of 32bit >=0 timestamp, no extra bits" +#define UPPER_BOUND_NONNEG_NO_EXTRA_BITS_CASE\ + "2038-01-19 Upper bound of 32bit >=0 timestamp, no extra bits" +#define LOWER_BOUND_NEG_LO_1_CASE\ + "2038-01-19 Lower bound of 32bit <0 timestamp, lo extra sec bit on" +#define UPPER_BOUND_NEG_LO_1_CASE\ + "2106-02-07 Upper bound of 32bit <0 timestamp, lo extra sec bit on" +#define LOWER_BOUND_NONNEG_LO_1_CASE\ + "2106-02-07 Lower bound of 32bit >=0 timestamp, lo extra sec bit on" +#define UPPER_BOUND_NONNEG_LO_1_CASE\ + "2174-02-25 Upper bound of 32bit >=0 timestamp, lo extra sec bit on" +#define LOWER_BOUND_NEG_HI_1_CASE\ + "2174-02-25 Lower bound of 32bit <0 timestamp, hi extra sec bit on" +#define UPPER_BOUND_NEG_HI_1_CASE\ + "2242-03-16 Upper bound of 32bit <0 timestamp, hi extra sec bit on" +#define LOWER_BOUND_NONNEG_HI_1_CASE\ + "2242-03-16 Lower bound of 32bit >=0 timestamp, hi extra sec bit on" +#define UPPER_BOUND_NONNEG_HI_1_CASE\ + "2310-04-04 Upper bound of 32bit >=0 timestamp, hi extra sec bit on" +#define UPPER_BOUND_NONNEG_HI_1_NS_1_CASE\ + "2310-04-04 Upper bound of 32bit>=0 timestamp, hi extra sec bit 1. 1 ns" +#define LOWER_BOUND_NONNEG_HI_1_NS_MAX_CASE\ + "2378-04-22 Lower bound of 32bit>= timestamp. Extra sec bits 1. Max ns" +#define LOWER_BOUND_NONNEG_EXTRA_BITS_1_CASE\ + "2378-04-22 Lower bound of 32bit >=0 timestamp. All extra sec bits on" +#define UPPER_BOUND_NONNEG_EXTRA_BITS_1_CASE\ + "2446-05-10 Upper bound of 32bit >=0 timestamp. All extra sec bits on" + +struct timestamp_expectation { + const char *test_case_name; + struct timespec64 expected; + u32 extra_bits; + bool msb_set; + bool lower_bound; +}; + +static const struct timestamp_expectation test_data[] = { + { + .test_case_name = LOWER_BOUND_NEG_NO_EXTRA_BITS_CASE, + .msb_set = true, + .lower_bound = true, + .extra_bits = 0, + .expected = {.tv_sec = -0x80000000LL, .tv_nsec = 0L}, + }, + + { + .test_case_name = UPPER_BOUND_NEG_NO_EXTRA_BITS_CASE, + .msb_set = true, + .lower_bound = false, + .extra_bits = 0, + .expected = {.tv_sec = -1LL, .tv_nsec = 0L}, + }, + + { + .test_case_name = LOWER_BOUND_NONNEG_NO_EXTRA_BITS_CASE, + .msb_set = false, + .lower_bound = true, + .extra_bits = 0, + .expected = {0LL, 0L}, + }, + + { + .test_case_name = UPPER_BOUND_NONNEG_NO_EXTRA_BITS_CASE, + .msb_set = false, + .lower_bound = false, + .extra_bits = 0, + .expected = {.tv_sec = 0x7fffffffLL, .tv_nsec = 0L}, + }, + + { + .test_case_name = LOWER_BOUND_NEG_LO_1_CASE, + .msb_set = true, + .lower_bound = true, + .extra_bits = 1, + .expected = {.tv_sec = 0x80000000LL, .tv_nsec = 0L}, + }, + + { + .test_case_name = UPPER_BOUND_NEG_LO_1_CASE, + .msb_set = true, + .lower_bound = false, + .extra_bits = 1, + .expected = {.tv_sec = 0xffffffffLL, .tv_nsec = 0L}, + }, + + { + .test_case_name = LOWER_BOUND_NONNEG_LO_1_CASE, + .msb_set = false, + .lower_bound = true, + .extra_bits = 1, + .expected = {.tv_sec = 0x100000000LL, .tv_nsec = 0L}, + }, + + { + .test_case_name = UPPER_BOUND_NONNEG_LO_1_CASE, + .msb_set = false, + .lower_bound = false, + .extra_bits = 1, + .expected = {.tv_sec = 0x17fffffffLL, .tv_nsec = 0L}, + }, + + { + .test_case_name = LOWER_BOUND_NEG_HI_1_CASE, + .msb_set = true, + .lower_bound = true, + .extra_bits = 2, + .expected = {.tv_sec = 0x180000000LL, .tv_nsec = 0L}, + }, + + { + .test_case_name = UPPER_BOUND_NEG_HI_1_CASE, + .msb_set = true, + .lower_bound = false, + .extra_bits = 2, + .expected = {.tv_sec = 0x1ffffffffLL, .tv_nsec = 0L}, + }, + + { + .test_case_name = LOWER_BOUND_NONNEG_HI_1_CASE, + .msb_set = false, + .lower_bound = true, + .extra_bits = 2, + .expected = {.tv_sec = 0x200000000LL, .tv_nsec = 0L}, + }, + + { + .test_case_name = UPPER_BOUND_NONNEG_HI_1_CASE, + .msb_set = false, + .lower_bound = false, + .extra_bits = 2, + .expected = {.tv_sec = 0x27fffffffLL, .tv_nsec = 0L}, + }, + + { + .test_case_name = UPPER_BOUND_NONNEG_HI_1_NS_1_CASE, + .msb_set = false, + .lower_bound = false, + .extra_bits = 6, + .expected = {.tv_sec = 0x27fffffffLL, .tv_nsec = 1L}, + }, + + { + .test_case_name = LOWER_BOUND_NONNEG_HI_1_NS_MAX_CASE, + .msb_set = false, + .lower_bound = true, + .extra_bits = 0xFFFFFFFF, + .expected = {.tv_sec = 0x300000000LL, + .tv_nsec = MAX_NANOSECONDS}, + }, + + { + .test_case_name = LOWER_BOUND_NONNEG_EXTRA_BITS_1_CASE, + .msb_set = false, + .lower_bound = true, + .extra_bits = 3, + .expected = {.tv_sec = 0x300000000LL, .tv_nsec = 0L}, + }, + + { + .test_case_name = UPPER_BOUND_NONNEG_EXTRA_BITS_1_CASE, + .msb_set = false, + .lower_bound = false, + .extra_bits = 3, + .expected = {.tv_sec = 0x37fffffffLL, .tv_nsec = 0L}, + } +}; + +static void timestamp_expectation_to_desc(const struct timestamp_expectation *t, + char *desc) +{ + strscpy(desc, t->test_case_name, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(ext4_inode, test_data, timestamp_expectation_to_desc); + +static time64_t get_32bit_time(const struct timestamp_expectation * const test) +{ + if (test->msb_set) { + if (test->lower_bound) + return LOWER_MSB_1; + + return UPPER_MSB_1; + } + + if (test->lower_bound) + return LOWER_MSB_0; + return UPPER_MSB_0; +} + + +/* + * Test data is derived from the table in the Inode Timestamps section of + * Documentation/filesystems/ext4/inodes.rst. + */ +static void inode_test_xtimestamp_decoding(struct kunit *test) +{ + struct timespec64 timestamp; + + struct timestamp_expectation *test_param = + (struct timestamp_expectation *)(test->param_value); + + timestamp = ext4_decode_extra_time( + cpu_to_le32(get_32bit_time(test_param)), + cpu_to_le32(test_param->extra_bits)); + + KUNIT_EXPECT_EQ_MSG(test, + test_param->expected.tv_sec, + timestamp.tv_sec, + CASE_NAME_FORMAT, + test_param->test_case_name, + test_param->msb_set, + test_param->lower_bound, + test_param->extra_bits); + KUNIT_EXPECT_EQ_MSG(test, + test_param->expected.tv_nsec, + timestamp.tv_nsec, + CASE_NAME_FORMAT, + test_param->test_case_name, + test_param->msb_set, + test_param->lower_bound, + test_param->extra_bits); +} + +static struct kunit_case ext4_inode_test_cases[] = { + KUNIT_CASE_PARAM(inode_test_xtimestamp_decoding, ext4_inode_gen_params), + {} +}; + +static struct kunit_suite ext4_inode_test_suite = { + .name = "ext4_inode_test", + .test_cases = ext4_inode_test_cases, +}; + +kunit_test_suites(&ext4_inode_test_suite); + +MODULE_DESCRIPTION("KUnit test of ext4 inode timestamp decoding"); +MODULE_LICENSE("GPL v2"); diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c new file mode 100644 index 000000000..bd4b778df --- /dev/null +++ b/fs/ext4/inode.c @@ -0,0 +1,6970 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext4/inode.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * from + * + * linux/fs/minix/inode.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * 64-bit file support on 64-bit platforms by Jakub Jelinek + * (jj@sunsite.ms.mff.cuni.cz) + * + * Assorted race fixes, rewrite of ext4_get_block() by Al Viro, 2000 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ext4_jbd2.h" +#include "xattr.h" +#include "acl.h" +#include "truncate.h" + +#include + +#include + +static void ext4_journalled_zero_new_buffers(handle_t *handle, + struct inode *inode, + struct folio *folio, + unsigned from, unsigned to); + +static __u32 ext4_inode_csum(struct inode *inode, struct ext4_inode *raw, + struct ext4_inode_info *ei) +{ + __u32 csum; + __u16 dummy_csum = 0; + int offset = offsetof(struct ext4_inode, i_checksum_lo); + unsigned int csum_size = sizeof(dummy_csum); + + csum = ext4_chksum(ei->i_csum_seed, (__u8 *)raw, offset); + csum = ext4_chksum(csum, (__u8 *)&dummy_csum, csum_size); + offset += csum_size; + csum = ext4_chksum(csum, (__u8 *)raw + offset, + EXT4_GOOD_OLD_INODE_SIZE - offset); + + if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) { + offset = offsetof(struct ext4_inode, i_checksum_hi); + csum = ext4_chksum(csum, (__u8 *)raw + EXT4_GOOD_OLD_INODE_SIZE, + offset - EXT4_GOOD_OLD_INODE_SIZE); + if (EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) { + csum = ext4_chksum(csum, (__u8 *)&dummy_csum, + csum_size); + offset += csum_size; + } + csum = ext4_chksum(csum, (__u8 *)raw + offset, + EXT4_INODE_SIZE(inode->i_sb) - offset); + } + + return csum; +} + +static int ext4_inode_csum_verify(struct inode *inode, struct ext4_inode *raw, + struct ext4_inode_info *ei) +{ + __u32 provided, calculated; + + if (EXT4_SB(inode->i_sb)->s_es->s_creator_os != + cpu_to_le32(EXT4_OS_LINUX) || + !ext4_has_feature_metadata_csum(inode->i_sb)) + return 1; + + provided = le16_to_cpu(raw->i_checksum_lo); + calculated = ext4_inode_csum(inode, raw, ei); + if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE && + EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) + provided |= ((__u32)le16_to_cpu(raw->i_checksum_hi)) << 16; + else + calculated &= 0xFFFF; + + return provided == calculated; +} + +void ext4_inode_csum_set(struct inode *inode, struct ext4_inode *raw, + struct ext4_inode_info *ei) +{ + __u32 csum; + + if (EXT4_SB(inode->i_sb)->s_es->s_creator_os != + cpu_to_le32(EXT4_OS_LINUX) || + !ext4_has_feature_metadata_csum(inode->i_sb)) + return; + + csum = ext4_inode_csum(inode, raw, ei); + raw->i_checksum_lo = cpu_to_le16(csum & 0xFFFF); + if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE && + EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) + raw->i_checksum_hi = cpu_to_le16(csum >> 16); +} + +static inline int ext4_begin_ordered_truncate(struct inode *inode, + loff_t new_size) +{ + struct jbd2_inode *jinode = READ_ONCE(EXT4_I(inode)->jinode); + + trace_ext4_begin_ordered_truncate(inode, new_size); + /* + * If jinode is zero, then we never opened the file for + * writing, so there's no need to call + * jbd2_journal_begin_ordered_truncate() since there's no + * outstanding writes we need to flush. + */ + if (!jinode) + return 0; + return jbd2_journal_begin_ordered_truncate(EXT4_JOURNAL(inode), + jinode, + new_size); +} + +/* + * Test whether an inode is a fast symlink. + * A fast symlink has its symlink data stored in ext4_inode_info->i_data. + */ +int ext4_inode_is_fast_symlink(struct inode *inode) +{ + if (!ext4_has_feature_ea_inode(inode->i_sb)) { + int ea_blocks = EXT4_I(inode)->i_file_acl ? + EXT4_CLUSTER_SIZE(inode->i_sb) >> 9 : 0; + + if (ext4_has_inline_data(inode)) + return 0; + + return (S_ISLNK(inode->i_mode) && inode->i_blocks - ea_blocks == 0); + } + return S_ISLNK(inode->i_mode) && inode->i_size && + (inode->i_size < EXT4_N_BLOCKS * 4); +} + +/* + * Called at the last iput() if i_nlink is zero. + */ +void ext4_evict_inode(struct inode *inode) +{ + handle_t *handle; + int err; + /* + * Credits for final inode cleanup and freeing: + * sb + inode (ext4_orphan_del()), block bitmap, group descriptor + * (xattr block freeing), bitmap, group descriptor (inode freeing) + */ + int extra_credits = 6; + bool freeze_protected = false; + + trace_ext4_evict_inode(inode); + + dax_break_layout_final(inode); + + if (EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL) + ext4_evict_ea_inode(inode); + if (inode->i_nlink) { + struct mapping_metadata_bhs *mmb; + + /* + * If there's dirty page will lead to data loss, user + * could see stale data. + */ + if (unlikely(!ext4_emergency_state(inode->i_sb) && + mapping_tagged(&inode->i_data, PAGECACHE_TAG_DIRTY))) + ext4_warning_inode(inode, "data will be lost"); + + truncate_inode_pages_final(&inode->i_data); + mmb = ext4_i_metadata_bhs(inode); + if (mmb) + mmb_sync(mmb); + goto no_delete; + } + + if (is_bad_inode(inode)) + goto no_delete; + dquot_initialize(inode); + + if (ext4_should_order_data(inode)) + ext4_begin_ordered_truncate(inode, 0); + truncate_inode_pages_final(&inode->i_data); + + /* + * For inodes with journalled data, transaction commit could have + * dirtied the inode. And for inodes with dioread_nolock, unwritten + * extents converting worker could merge extents and also have dirtied + * the inode. Flush worker is ignoring it because of I_FREEING flag but + * we still need to remove the inode from the writeback lists. + */ + inode_io_list_del(inode); + + /* + * Protect us against freezing - iput() caller didn't have to have any + * protection against it. When we are in a running transaction though, + * we are already protected against freezing and we cannot grab further + * protection due to lock ordering constraints. + */ + if (!ext4_journal_current_handle()) { + sb_start_intwrite(inode->i_sb); + freeze_protected = true; + } + + if (!IS_NOQUOTA(inode)) + extra_credits += EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb); + + /* + * Block bitmap, group descriptor, and inode are accounted in both + * ext4_blocks_for_truncate() and extra_credits. So subtract 3. + */ + handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, + ext4_blocks_for_truncate(inode) + extra_credits - 3); + if (IS_ERR(handle)) { + ext4_std_error(inode->i_sb, PTR_ERR(handle)); + /* + * If we're going to skip the normal cleanup, we still need to + * make sure that the in-core orphan linked list is properly + * cleaned up. + */ + ext4_orphan_del(NULL, inode); + if (freeze_protected) + sb_end_intwrite(inode->i_sb); + goto no_delete; + } + + if (IS_SYNC(inode)) + ext4_handle_sync(handle); + + /* + * Set inode->i_size to 0 before calling ext4_truncate(). We need + * special handling of symlinks here because i_size is used to + * determine whether ext4_inode_info->i_data contains symlink data or + * block mappings. Setting i_size to 0 will remove its fast symlink + * status. Erase i_data so that it becomes a valid empty block map. + */ + if (ext4_inode_is_fast_symlink(inode)) + memset(EXT4_I(inode)->i_data, 0, sizeof(EXT4_I(inode)->i_data)); + inode->i_size = 0; + ext4_set_inode_state(inode, EXT4_STATE_NO_EXPAND); + err = ext4_mark_inode_dirty(handle, inode); + if (err) { + ext4_warning(inode->i_sb, + "couldn't mark inode dirty (err %d)", err); + goto stop_handle; + } + if (inode->i_blocks) { + err = ext4_truncate(inode); + if (err) { + ext4_error_err(inode->i_sb, -err, + "couldn't truncate inode %llu (err %d)", + inode->i_ino, err); + goto stop_handle; + } + } + + /* Remove xattr references. */ + err = ext4_xattr_delete_inode(handle, inode, extra_credits); + if (err) { + ext4_warning(inode->i_sb, "xattr delete (err %d)", err); +stop_handle: + ext4_journal_stop(handle); + ext4_orphan_del(NULL, inode); + if (freeze_protected) + sb_end_intwrite(inode->i_sb); + goto no_delete; + } + + /* + * Kill off the orphan record which ext4_truncate created. + * AKPM: I think this can be inside the above `if'. + * Note that ext4_orphan_del() has to be able to cope with the + * deletion of a non-existent orphan - this is because we don't + * know if ext4_truncate() actually created an orphan record. + * (Well, we could do this if we need to, but heck - it works) + */ + ext4_orphan_del(handle, inode); + EXT4_I(inode)->i_dtime = (__u32)ktime_get_real_seconds(); + + /* + * One subtle ordering requirement: if anything has gone wrong + * (transaction abort, IO errors, whatever), then we can still + * do these next steps (the fs will already have been marked as + * having errors), but we can't free the inode if the mark_dirty + * fails. + */ + if (ext4_mark_inode_dirty(handle, inode)) + /* If that failed, just do the required in-core inode clear. */ + ext4_clear_inode(inode); + else + ext4_free_inode(handle, inode); + ext4_journal_stop(handle); + if (freeze_protected) + sb_end_intwrite(inode->i_sb); + return; +no_delete: + /* + * Check out some where else accidentally dirty the evicting inode, + * which may probably cause inode use-after-free issues later. + */ + WARN_ON_ONCE(!list_empty_careful(&inode->i_io_list)); + + if (!list_empty(&EXT4_I(inode)->i_fc_list)) + ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_NOMEM, NULL); + ext4_clear_inode(inode); /* We must guarantee clearing of inode... */ +} + +#ifdef CONFIG_QUOTA +qsize_t *ext4_get_reserved_space(struct inode *inode) +{ + return &EXT4_I(inode)->i_reserved_quota; +} +#endif + +/* + * Called with i_data_sem down, which is important since we can call + * ext4_discard_preallocations() from here. + */ +void ext4_da_update_reserve_space(struct inode *inode, + int used, int quota_claim) +{ + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + struct ext4_inode_info *ei = EXT4_I(inode); + + spin_lock(&ei->i_block_reservation_lock); + trace_ext4_da_update_reserve_space(inode, used, quota_claim); + if (unlikely(used > ei->i_reserved_data_blocks)) { + ext4_warning(inode->i_sb, "%s: ino %llu, used %d " + "with only %d reserved data blocks", + __func__, inode->i_ino, used, + ei->i_reserved_data_blocks); + WARN_ON(1); + used = ei->i_reserved_data_blocks; + } + + /* Update per-inode reservations */ + ei->i_reserved_data_blocks -= used; + percpu_counter_sub(&sbi->s_dirtyclusters_counter, used); + + spin_unlock(&ei->i_block_reservation_lock); + + /* Update quota subsystem for data blocks */ + if (quota_claim) + dquot_claim_block(inode, EXT4_C2B(sbi, used)); + else { + /* + * We did fallocate with an offset that is already delayed + * allocated. So on delayed allocated writeback we should + * not re-claim the quota for fallocated blocks. + */ + dquot_release_reservation_block(inode, EXT4_C2B(sbi, used)); + } + + /* + * If we have done all the pending block allocations and if + * there aren't any writers on the inode, we can discard the + * inode's preallocations. + */ + if ((ei->i_reserved_data_blocks == 0) && + !inode_is_open_for_write(inode)) + ext4_discard_preallocations(inode); +} + +static int __check_block_validity(struct inode *inode, const char *func, + unsigned int line, + struct ext4_map_blocks *map) +{ + journal_t *journal = EXT4_SB(inode->i_sb)->s_journal; + + if (journal && inode == journal->j_inode) + return 0; + + if (!ext4_inode_block_valid(inode, map->m_pblk, map->m_len)) { + ext4_error_inode(inode, func, line, map->m_pblk, + "lblock %lu mapped to illegal pblock %llu " + "(length %d)", (unsigned long) map->m_lblk, + map->m_pblk, map->m_len); + return -EFSCORRUPTED; + } + return 0; +} + +int ext4_issue_zeroout(struct inode *inode, ext4_lblk_t lblk, ext4_fsblk_t pblk, + ext4_lblk_t len) +{ + int ret; + + KUNIT_STATIC_STUB_REDIRECT(ext4_issue_zeroout, inode, lblk, pblk, len); + + if (IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode)) + return fscrypt_zeroout_range(inode, + (loff_t)lblk << inode->i_blkbits, + pblk << (inode->i_blkbits - SECTOR_SHIFT), + (u64)len << inode->i_blkbits); + + ret = sb_issue_zeroout(inode->i_sb, pblk, len, GFP_NOFS); + if (ret > 0) + ret = 0; + + return ret; +} + +/* + * For generic regular files, when updating the extent tree, Ext4 should + * hold the i_rwsem and invalidate_lock exclusively. This ensures + * exclusion against concurrent page faults, as well as reads and writes. + */ +#ifdef CONFIG_EXT4_DEBUG +void ext4_check_map_extents_env(struct inode *inode) +{ + if (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY) + return; + + if (!S_ISREG(inode->i_mode) || + IS_NOQUOTA(inode) || IS_VERITY(inode) || + is_special_ino(inode->i_sb, inode->i_ino) || + (inode_state_read_once(inode) & (I_FREEING | I_WILL_FREE | I_NEW)) || + ext4_test_inode_flag(inode, EXT4_INODE_EA_INODE) || + ext4_verity_in_progress(inode)) + return; + + WARN_ON_ONCE(!inode_is_locked(inode) && + !rwsem_is_locked(&inode->i_mapping->invalidate_lock)); +} +#else +void ext4_check_map_extents_env(struct inode *inode) {} +#endif + +#define check_block_validity(inode, map) \ + __check_block_validity((inode), __func__, __LINE__, (map)) + +#ifdef ES_AGGRESSIVE_TEST +static void ext4_map_blocks_es_recheck(handle_t *handle, + struct inode *inode, + struct ext4_map_blocks *es_map, + struct ext4_map_blocks *map, + int flags) +{ + int retval; + + map->m_flags = 0; + /* + * There is a race window that the result is not the same. + * e.g. xfstests #223 when dioread_nolock enables. The reason + * is that we lookup a block mapping in extent status tree with + * out taking i_data_sem. So at the time the unwritten extent + * could be converted. + */ + down_read(&EXT4_I(inode)->i_data_sem); + if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) { + retval = ext4_ext_map_blocks(handle, inode, map, 0); + } else { + retval = ext4_ind_map_blocks(handle, inode, map, 0); + } + up_read((&EXT4_I(inode)->i_data_sem)); + + /* + * We don't check m_len because extent will be collpased in status + * tree. So the m_len might not equal. + */ + if (es_map->m_lblk != map->m_lblk || + es_map->m_flags != map->m_flags || + es_map->m_pblk != map->m_pblk) { + printk("ES cache assertion failed for inode: %llu " + "es_cached ex [%d/%d/%llu/%x] != " + "found ex [%d/%d/%llu/%x] retval %d flags %x\n", + inode->i_ino, es_map->m_lblk, es_map->m_len, + es_map->m_pblk, es_map->m_flags, map->m_lblk, + map->m_len, map->m_pblk, map->m_flags, + retval, flags); + } +} +#endif /* ES_AGGRESSIVE_TEST */ + +static int ext4_map_query_blocks_next_in_leaf(handle_t *handle, + struct inode *inode, struct ext4_map_blocks *map, + unsigned int orig_mlen) +{ + struct ext4_map_blocks map2; + unsigned int status, status2; + int retval; + + status = map->m_flags & EXT4_MAP_UNWRITTEN ? + EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN; + + WARN_ON_ONCE(!(map->m_flags & EXT4_MAP_QUERY_LAST_IN_LEAF)); + WARN_ON_ONCE(orig_mlen <= map->m_len); + + /* Prepare map2 for lookup in next leaf block */ + map2.m_lblk = map->m_lblk + map->m_len; + map2.m_len = orig_mlen - map->m_len; + map2.m_flags = 0; + retval = ext4_ext_map_blocks(handle, inode, &map2, 0); + + if (retval <= 0) { + ext4_es_cache_extent(inode, map->m_lblk, map->m_len, + map->m_pblk, status); + return map->m_len; + } + + if (unlikely(retval != map2.m_len)) { + ext4_warning(inode->i_sb, + "ES len assertion failed for inode " + "%llu: retval %d != map->m_len %d", + inode->i_ino, retval, map2.m_len); + WARN_ON(1); + } + + status2 = map2.m_flags & EXT4_MAP_UNWRITTEN ? + EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN; + + /* + * If map2 is contiguous with map, then let's insert it as a single + * extent in es cache and return the combined length of both the maps. + */ + if (map->m_pblk + map->m_len == map2.m_pblk && + status == status2) { + ext4_es_cache_extent(inode, map->m_lblk, + map->m_len + map2.m_len, map->m_pblk, + status); + map->m_len += map2.m_len; + } else { + ext4_es_cache_extent(inode, map->m_lblk, map->m_len, + map->m_pblk, status); + } + + return map->m_len; +} + +int ext4_map_query_blocks(handle_t *handle, struct inode *inode, + struct ext4_map_blocks *map, int flags) +{ + unsigned int status; + int retval; + unsigned int orig_mlen = map->m_len; + + flags &= EXT4_EX_QUERY_FILTER; + if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) + retval = ext4_ext_map_blocks(handle, inode, map, flags); + else + retval = ext4_ind_map_blocks(handle, inode, map, flags); + if (retval < 0) + return retval; + + /* A hole? */ + if (retval == 0) + goto out; + + if (unlikely(retval != map->m_len)) { + ext4_warning(inode->i_sb, + "ES len assertion failed for inode " + "%llu: retval %d != map->m_len %d", + inode->i_ino, retval, map->m_len); + WARN_ON(1); + } + + /* + * No need to query next in leaf: + * - if returned extent is not last in leaf or + * - if the last in leaf is the full requested range + */ + if (!(map->m_flags & EXT4_MAP_QUERY_LAST_IN_LEAF) || + map->m_len == orig_mlen) { + status = map->m_flags & EXT4_MAP_UNWRITTEN ? + EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN; + ext4_es_cache_extent(inode, map->m_lblk, map->m_len, + map->m_pblk, status); + } else { + retval = ext4_map_query_blocks_next_in_leaf(handle, inode, map, + orig_mlen); + } +out: + map->m_seq = READ_ONCE(EXT4_I(inode)->i_es_seq); + return retval; +} + +int ext4_map_create_blocks(handle_t *handle, struct inode *inode, + struct ext4_map_blocks *map, int flags) +{ + unsigned int status; + int err, retval = 0; + + /* + * We pass in the magic EXT4_GET_BLOCKS_DELALLOC_RESERVE + * indicates that the blocks and quotas has already been + * checked when the data was copied into the page cache. + */ + if (map->m_flags & EXT4_MAP_DELAYED) + flags |= EXT4_GET_BLOCKS_DELALLOC_RESERVE; + + /* + * Here we clear m_flags because after allocating an new extent, + * it will be set again. + */ + map->m_flags &= ~EXT4_MAP_FLAGS; + + /* + * We need to check for EXT4 here because migrate could have + * changed the inode type in between. + */ + if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) { + retval = ext4_ext_map_blocks(handle, inode, map, flags); + } else { + retval = ext4_ind_map_blocks(handle, inode, map, flags); + + /* + * We allocated new blocks which will result in i_data's + * format changing. Force the migrate to fail by clearing + * migrate flags. + */ + if (retval > 0 && map->m_flags & EXT4_MAP_NEW) + ext4_clear_inode_state(inode, EXT4_STATE_EXT_MIGRATE); + } + if (retval <= 0) + return retval; + + if (unlikely(retval != map->m_len)) { + ext4_warning(inode->i_sb, + "ES len assertion failed for inode %llu: " + "retval %d != map->m_len %d", + inode->i_ino, retval, map->m_len); + WARN_ON(1); + } + + /* + * We have to zeroout blocks before inserting them into extent + * status tree. Otherwise someone could look them up there and + * use them before they are really zeroed. We also have to + * unmap metadata before zeroing as otherwise writeback can + * overwrite zeros with stale data from block device. + */ + if (flags & EXT4_GET_BLOCKS_ZERO && + map->m_flags & EXT4_MAP_MAPPED && map->m_flags & EXT4_MAP_NEW) { + err = ext4_issue_zeroout(inode, map->m_lblk, map->m_pblk, + map->m_len); + if (err) + return err; + } + + status = map->m_flags & EXT4_MAP_UNWRITTEN ? + EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN; + ext4_es_insert_extent(inode, map->m_lblk, map->m_len, map->m_pblk, + status, flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE); + map->m_seq = READ_ONCE(EXT4_I(inode)->i_es_seq); + + return retval; +} + +/* + * The ext4_map_blocks() function tries to look up the requested blocks, + * and returns if the blocks are already mapped. + * + * Otherwise it takes the write lock of the i_data_sem and allocate blocks + * and store the allocated blocks in the result buffer head and mark it + * mapped. + * + * If file type is extents based, it will call ext4_ext_map_blocks(), + * Otherwise, call with ext4_ind_map_blocks() to handle indirect mapping + * based files + * + * On success, it returns the number of blocks being mapped or allocated. + * If flags doesn't contain EXT4_GET_BLOCKS_CREATE the blocks are + * pre-allocated and unwritten, the resulting @map is marked as unwritten. + * If the flags contain EXT4_GET_BLOCKS_CREATE, it will mark @map as mapped. + * + * It returns 0 if plain look up failed (blocks have not been allocated), in + * that case, @map is returned as unmapped but we still do fill map->m_len to + * indicate the length of a hole starting at map->m_lblk. + * + * It returns the error in case of allocation failure. + */ +int ext4_map_blocks(handle_t *handle, struct inode *inode, + struct ext4_map_blocks *map, int flags) +{ + struct extent_status es; + int retval; + int ret = 0; + unsigned int orig_mlen = map->m_len; +#ifdef ES_AGGRESSIVE_TEST + struct ext4_map_blocks orig_map; + + memcpy(&orig_map, map, sizeof(*map)); +#endif + + map->m_flags = 0; + ext_debug(inode, "flag 0x%x, max_blocks %u, logical block %lu\n", + flags, map->m_len, (unsigned long) map->m_lblk); + + /* + * ext4_map_blocks returns an int, and m_len is an unsigned int + */ + if (unlikely(map->m_len > INT_MAX)) + map->m_len = INT_MAX; + + /* We can handle the block number less than EXT_MAX_BLOCKS */ + if (unlikely(map->m_lblk >= EXT_MAX_BLOCKS)) + return -EFSCORRUPTED; + + /* + * Callers from the context of data submission are the only exceptions + * for regular files that do not hold the i_rwsem or invalidate_lock. + * However, caching unrelated ranges is not permitted. + */ + if (flags & EXT4_GET_BLOCKS_IO_SUBMIT) + WARN_ON_ONCE(!(flags & EXT4_EX_NOCACHE)); + else + ext4_check_map_extents_env(inode); + + /* Lookup extent status tree firstly */ + if (ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es, &map->m_seq)) { + if (ext4_es_is_written(&es) || ext4_es_is_unwritten(&es)) { + map->m_pblk = ext4_es_pblock(&es) + + map->m_lblk - es.es_lblk; + map->m_flags |= ext4_es_is_written(&es) ? + EXT4_MAP_MAPPED : EXT4_MAP_UNWRITTEN; + retval = es.es_len - (map->m_lblk - es.es_lblk); + if (retval > map->m_len) + retval = map->m_len; + map->m_len = retval; + } else if (ext4_es_is_delayed(&es) || ext4_es_is_hole(&es)) { + map->m_pblk = 0; + map->m_flags |= ext4_es_is_delayed(&es) ? + EXT4_MAP_DELAYED : 0; + retval = es.es_len - (map->m_lblk - es.es_lblk); + if (retval > map->m_len) + retval = map->m_len; + map->m_len = retval; + retval = 0; + } else { + BUG(); + } + + if (flags & EXT4_GET_BLOCKS_CACHED_NOWAIT) + return retval; +#ifdef ES_AGGRESSIVE_TEST + ext4_map_blocks_es_recheck(handle, inode, map, + &orig_map, flags); +#endif + if (!(flags & EXT4_GET_BLOCKS_QUERY_LAST_IN_LEAF) || + orig_mlen == map->m_len) + goto found; + + map->m_len = orig_mlen; + } + /* + * In the query cache no-wait mode, nothing we can do more if we + * cannot find extent in the cache. + */ + if (flags & EXT4_GET_BLOCKS_CACHED_NOWAIT) + return 0; + + /* + * Try to see if we can get the block without requesting a new + * file system block. + */ + down_read(&EXT4_I(inode)->i_data_sem); + retval = ext4_map_query_blocks(handle, inode, map, flags); + up_read((&EXT4_I(inode)->i_data_sem)); + +found: + if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) { + ret = check_block_validity(inode, map); + if (ret != 0) + return ret; + } + + /* If it is only a block(s) look up */ + if ((flags & EXT4_GET_BLOCKS_CREATE) == 0) + return retval; + + /* + * Returns if the blocks have already allocated + * + * Note that if blocks have been preallocated + * ext4_ext_map_blocks() returns with buffer head unmapped + */ + if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) + /* + * If we need to convert extent to unwritten + * we continue and do the actual work in + * ext4_ext_map_blocks() + */ + if (!(flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN)) + return retval; + + + ext4_fc_track_inode(handle, inode); + /* + * New blocks allocate and/or writing to unwritten extent + * will possibly result in updating i_data, so we take + * the write lock of i_data_sem, and call get_block() + * with create == 1 flag. + */ + down_write(&EXT4_I(inode)->i_data_sem); + retval = ext4_map_create_blocks(handle, inode, map, flags); + up_write((&EXT4_I(inode)->i_data_sem)); + + if (retval < 0) + ext_debug(inode, "failed with err %d\n", retval); + if (retval <= 0) + return retval; + + if (map->m_flags & EXT4_MAP_MAPPED) { + ret = check_block_validity(inode, map); + if (ret != 0) + return ret; + + /* + * Inodes with freshly allocated blocks where contents will be + * visible after transaction commit must be on transaction's + * ordered data list. + */ + if (map->m_flags & EXT4_MAP_NEW && + !(map->m_flags & EXT4_MAP_UNWRITTEN) && + !(flags & EXT4_GET_BLOCKS_ZERO) && + !ext4_is_quota_file(inode) && + ext4_should_order_data(inode)) { + loff_t start_byte = EXT4_LBLK_TO_B(inode, map->m_lblk); + loff_t length = EXT4_LBLK_TO_B(inode, map->m_len); + + if (flags & EXT4_GET_BLOCKS_IO_SUBMIT) + ret = ext4_jbd2_inode_add_wait(handle, inode, + start_byte, length); + else + ret = ext4_jbd2_inode_add_write(handle, inode, + start_byte, length); + if (ret) + return ret; + } + } + ext4_fc_track_range(handle, inode, map->m_lblk, map->m_lblk + + map->m_len - 1); + return retval; +} + +/* + * Update EXT4_MAP_FLAGS in bh->b_state. For buffer heads attached to pages + * we have to be careful as someone else may be manipulating b_state as well. + */ +static void ext4_update_bh_state(struct buffer_head *bh, unsigned long flags) +{ + unsigned long old_state; + unsigned long new_state; + + flags &= EXT4_MAP_FLAGS; + + /* Dummy buffer_head? Set non-atomically. */ + if (!bh->b_folio) { + bh->b_state = (bh->b_state & ~EXT4_MAP_FLAGS) | flags; + return; + } + /* + * Someone else may be modifying b_state. Be careful! This is ugly but + * once we get rid of using bh as a container for mapping information + * to pass to / from get_block functions, this can go away. + */ + old_state = READ_ONCE(bh->b_state); + do { + new_state = (old_state & ~EXT4_MAP_FLAGS) | flags; + } while (unlikely(!try_cmpxchg(&bh->b_state, &old_state, new_state))); +} + +/* + * Make sure that the current journal transaction has enough credits to map + * one extent. Return -EAGAIN if it cannot extend the current running + * transaction. + */ +static inline int ext4_journal_ensure_extent_credits(handle_t *handle, + struct inode *inode) +{ + int credits; + int ret; + + /* Called from ext4_da_write_begin() which has no handle started? */ + if (!handle) + return 0; + + credits = ext4_chunk_trans_blocks(inode, 1); + ret = __ext4_journal_ensure_credits(handle, credits, credits, 0); + return ret <= 0 ? ret : -EAGAIN; +} + +static int _ext4_get_block(struct inode *inode, sector_t iblock, + struct buffer_head *bh, int flags) +{ + struct ext4_map_blocks map; + int ret = 0; + + if (ext4_has_inline_data(inode)) + return -ERANGE; + + map.m_lblk = iblock; + map.m_len = bh->b_size >> inode->i_blkbits; + + ret = ext4_map_blocks(ext4_journal_current_handle(), inode, &map, + flags); + if (ret > 0) { + map_bh(bh, inode->i_sb, map.m_pblk); + ext4_update_bh_state(bh, map.m_flags); + bh->b_size = inode->i_sb->s_blocksize * map.m_len; + ret = 0; + } else if (ret == 0) { + /* hole case, need to fill in bh->b_size */ + bh->b_size = inode->i_sb->s_blocksize * map.m_len; + } + return ret; +} + +int ext4_get_block(struct inode *inode, sector_t iblock, + struct buffer_head *bh, int create) +{ + return _ext4_get_block(inode, iblock, bh, + create ? EXT4_GET_BLOCKS_CREATE : 0); +} + +/* + * Get block function used when preparing for buffered write if we require + * creating an unwritten extent if blocks haven't been allocated. The extent + * will be converted to written after the IO is complete. + */ +int ext4_get_block_unwritten(struct inode *inode, sector_t iblock, + struct buffer_head *bh_result, int create) +{ + int ret = 0; + + ext4_debug("ext4_get_block_unwritten: inode %llu, create flag %d\n", + inode->i_ino, create); + ret = _ext4_get_block(inode, iblock, bh_result, + EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT); + + /* + * If the buffer is marked unwritten, mark it as new to make sure it is + * zeroed out correctly in case of partial writes. Otherwise, there is + * a chance of stale data getting exposed. + */ + if (ret == 0 && buffer_unwritten(bh_result)) + set_buffer_new(bh_result); + + return ret; +} + +/* Maximum number of blocks we map for direct IO at once. */ +#define DIO_MAX_BLOCKS 4096 + +/* + * `handle' can be NULL if create is zero + */ +struct buffer_head *ext4_getblk(handle_t *handle, struct inode *inode, + ext4_lblk_t block, int map_flags) +{ + struct ext4_map_blocks map; + struct buffer_head *bh; + int create = map_flags & EXT4_GET_BLOCKS_CREATE; + bool nowait = map_flags & EXT4_GET_BLOCKS_CACHED_NOWAIT; + int err; + + ASSERT((EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY) + || handle != NULL || create == 0); + ASSERT(create == 0 || !nowait); + + map.m_lblk = block; + map.m_len = 1; + err = ext4_map_blocks(handle, inode, &map, map_flags); + + if (err == 0) + return create ? ERR_PTR(-ENOSPC) : NULL; + if (err < 0) + return ERR_PTR(err); + + if (nowait) + return sb_find_get_block(inode->i_sb, map.m_pblk); + + /* + * Since bh could introduce extra ref count such as referred by + * journal_head etc. Try to avoid using __GFP_MOVABLE here + * as it may fail the migration when journal_head remains. + */ + bh = getblk_unmovable(inode->i_sb->s_bdev, map.m_pblk, + inode->i_sb->s_blocksize); + + if (unlikely(!bh)) + return ERR_PTR(-ENOMEM); + if (map.m_flags & EXT4_MAP_NEW) { + ASSERT(create != 0); + ASSERT((EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY) + || (handle != NULL)); + + /* + * Now that we do not always journal data, we should + * keep in mind whether this should always journal the + * new buffer as metadata. For now, regular file + * writes use ext4_get_block instead, so it's not a + * problem. + */ + lock_buffer(bh); + BUFFER_TRACE(bh, "call get_create_access"); + err = ext4_journal_get_create_access(handle, inode->i_sb, bh, + EXT4_JTR_NONE); + if (unlikely(err)) { + unlock_buffer(bh); + goto errout; + } + if (!buffer_uptodate(bh)) { + memset(bh->b_data, 0, inode->i_sb->s_blocksize); + set_buffer_uptodate(bh); + } + unlock_buffer(bh); + BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata"); + err = ext4_handle_dirty_metadata(handle, inode, bh); + if (unlikely(err)) + goto errout; + } else + BUFFER_TRACE(bh, "not a new buffer"); + return bh; +errout: + brelse(bh); + return ERR_PTR(err); +} + +struct buffer_head *ext4_bread(handle_t *handle, struct inode *inode, + ext4_lblk_t block, int map_flags) +{ + struct buffer_head *bh; + int ret; + + bh = ext4_getblk(handle, inode, block, map_flags); + if (IS_ERR(bh)) + return bh; + if (!bh || ext4_buffer_uptodate(bh)) + return bh; + + ret = ext4_read_bh_lock(bh, REQ_META | REQ_PRIO, true); + if (ret) { + put_bh(bh); + return ERR_PTR(ret); + } + return bh; +} + +/* Read a contiguous batch of blocks. */ +int ext4_bread_batch(struct inode *inode, ext4_lblk_t block, int bh_count, + bool wait, struct buffer_head **bhs) +{ + int i, err; + + for (i = 0; i < bh_count; i++) { + bhs[i] = ext4_getblk(NULL, inode, block + i, 0 /* map_flags */); + if (IS_ERR(bhs[i])) { + err = PTR_ERR(bhs[i]); + bh_count = i; + goto out_brelse; + } + } + + for (i = 0; i < bh_count; i++) + /* Note that NULL bhs[i] is valid because of holes. */ + if (bhs[i] && !ext4_buffer_uptodate(bhs[i])) + ext4_read_bh_lock(bhs[i], REQ_META | REQ_PRIO, false); + + if (!wait) + return 0; + + for (i = 0; i < bh_count; i++) + if (bhs[i]) + wait_on_buffer(bhs[i]); + + for (i = 0; i < bh_count; i++) { + if (bhs[i] && !buffer_uptodate(bhs[i])) { + err = -EIO; + goto out_brelse; + } + } + return 0; + +out_brelse: + for (i = 0; i < bh_count; i++) { + brelse(bhs[i]); + bhs[i] = NULL; + } + return err; +} + +int ext4_walk_page_buffers(handle_t *handle, struct inode *inode, + struct buffer_head *head, + unsigned from, + unsigned to, + int *partial, + int (*fn)(handle_t *handle, struct inode *inode, + struct buffer_head *bh)) +{ + struct buffer_head *bh; + unsigned block_start, block_end; + unsigned blocksize = head->b_size; + int err, ret = 0; + struct buffer_head *next; + + for (bh = head, block_start = 0; + ret == 0 && (bh != head || !block_start); + block_start = block_end, bh = next) { + next = bh->b_this_page; + block_end = block_start + blocksize; + if (block_end <= from || block_start >= to) { + if (partial && !buffer_uptodate(bh)) + *partial = 1; + continue; + } + err = (*fn)(handle, inode, bh); + if (!ret) + ret = err; + } + return ret; +} + +/* + * Helper for handling dirtying of journalled data. We also mark the folio as + * dirty so that writeback code knows about this page (and inode) contains + * dirty data. ext4_writepages() then commits appropriate transaction to + * make data stable. + */ +static int ext4_dirty_journalled_data(handle_t *handle, struct buffer_head *bh) +{ + struct folio *folio = bh->b_folio; + struct inode *inode = folio->mapping->host; + + /* only regular files have a_ops */ + if (S_ISREG(inode->i_mode)) + folio_mark_dirty(folio); + return ext4_handle_dirty_metadata(handle, NULL, bh); +} + +int do_journal_get_write_access(handle_t *handle, struct inode *inode, + struct buffer_head *bh) +{ + if (!buffer_mapped(bh) || buffer_freed(bh)) + return 0; + BUFFER_TRACE(bh, "get write access"); + return ext4_journal_get_write_access(handle, inode->i_sb, bh, + EXT4_JTR_NONE); +} + +int ext4_block_write_begin(handle_t *handle, struct folio *folio, + loff_t pos, unsigned len, + get_block_t *get_block) +{ + unsigned int from = offset_in_folio(folio, pos); + unsigned to = from + len; + struct inode *inode = folio->mapping->host; + unsigned block_start, block_end; + sector_t block; + int err = 0; + unsigned int blocksize = i_blocksize(inode); + struct buffer_head *bh, *head, *wait[2]; + int nr_wait = 0; + int i; + bool should_journal_data = ext4_should_journal_data(inode); + bool folio_uptodate = folio_test_uptodate(folio); + + BUG_ON(!folio_test_locked(folio)); + BUG_ON(to > folio_size(folio)); + BUG_ON(from > to); + WARN_ON_ONCE(blocksize > folio_size(folio)); + + head = folio_buffers(folio); + if (!head) + head = create_empty_buffers(folio, blocksize, 0); + block = EXT4_PG_TO_LBLK(inode, folio->index); + + for (bh = head, block_start = 0; + block_start < to || (!folio_uptodate && bh != head); + block++, block_start = block_end, bh = bh->b_this_page) { + block_end = block_start + blocksize; + if (block_end <= from || block_start >= to) { + if (folio_uptodate) + set_buffer_uptodate(bh); + continue; + } + if (WARN_ON_ONCE(buffer_new(bh))) + clear_buffer_new(bh); + if (!buffer_mapped(bh)) { + WARN_ON(bh->b_size != blocksize); + err = ext4_journal_ensure_extent_credits(handle, inode); + if (!err) + err = get_block(inode, block, bh, 1); + if (err) + break; + if (buffer_new(bh)) { + /* + * We may be zeroing partial buffers or all new + * buffers in case of failure. Prepare JBD2 for + * that. + */ + if (should_journal_data) + do_journal_get_write_access(handle, + inode, bh); + if (folio_uptodate) { + /* + * Unlike __block_write_begin() we leave + * dirtying of new uptodate buffers to + * ->write_end() time or + * folio_zero_new_buffers(). + */ + set_buffer_uptodate(bh); + continue; + } + if (block_end > to || block_start < from) + folio_zero_segments(folio, to, + block_end, + block_start, from); + continue; + } + } + if (folio_uptodate) { + set_buffer_uptodate(bh); + continue; + } + if (!buffer_uptodate(bh) && !buffer_delay(bh) && + !buffer_unwritten(bh) && + (block_start < from || block_end > to)) { + ext4_read_bh_lock(bh, 0, false); + wait[nr_wait++] = bh; + } + } + /* + * If we issued read requests, let them complete. + */ + for (i = 0; i < nr_wait; i++) { + wait_on_buffer(wait[i]); + if (!buffer_uptodate(wait[i])) + err = -EIO; + } + if (unlikely(err)) { + if (should_journal_data) + ext4_journalled_zero_new_buffers(handle, inode, folio, + from, to); + else + folio_zero_new_buffers(folio, from, to); + } + + return err; +} + +/* + * To preserve ordering, it is essential that the hole instantiation and + * the data write be encapsulated in a single transaction. We cannot + * close off a transaction and start a new one between the ext4_get_block() + * and the ext4_write_end(). So doing the jbd2_journal_start at the start of + * ext4_write_begin() is the right place. + */ +static int ext4_write_begin(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, + struct folio **foliop, void **fsdata) +{ + struct inode *inode = mapping->host; + int ret, needed_blocks; + handle_t *handle; + int retries = 0; + struct folio *folio; + pgoff_t index; + unsigned from, to; + + ret = ext4_emergency_state(inode->i_sb); + if (unlikely(ret)) + return ret; + + *fsdata = (void *)((unsigned long)*fsdata & ~EXT4_WRITE_DATA_INLINE); + + trace_ext4_write_begin(inode, pos, len); + /* + * Reserve one block more for addition to orphan list in case + * we allocate blocks but write fails for some reason + */ + needed_blocks = ext4_chunk_trans_extent(inode, + ext4_journal_blocks_per_folio(inode)) + 1; + index = pos >> PAGE_SHIFT; + + if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) { + ret = ext4_try_to_write_inline_data(mapping, inode, pos, len, + foliop); + if (ret < 0) + return ret; + if (ret == 1) { + *fsdata = (void *)((unsigned long)*fsdata | EXT4_WRITE_DATA_INLINE); + return 0; + } + } + + /* + * write_begin_get_folio() can take a long time if the + * system is thrashing due to memory pressure, or if the folio + * is being written back. So grab it first before we start + * the transaction handle. This also allows us to allocate + * the folio (if needed) without using GFP_NOFS. + */ +retry_grab: + folio = write_begin_get_folio(iocb, mapping, index, len); + if (IS_ERR(folio)) + return PTR_ERR(folio); + + if (len > folio_next_pos(folio) - pos) + len = folio_next_pos(folio) - pos; + + from = offset_in_folio(folio, pos); + to = from + len; + + /* + * The same as page allocation, we prealloc buffer heads before + * starting the handle. + */ + if (!folio_buffers(folio)) + create_empty_buffers(folio, inode->i_sb->s_blocksize, 0); + + folio_unlock(folio); + +retry_journal: + handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, needed_blocks); + if (IS_ERR(handle)) { + folio_put(folio); + return PTR_ERR(handle); + } + + folio_lock(folio); + if (folio->mapping != mapping) { + /* The folio got truncated from under us */ + folio_unlock(folio); + folio_put(folio); + ext4_journal_stop(handle); + goto retry_grab; + } + /* In case writeback began while the folio was unlocked */ + folio_wait_stable(folio); + + if (ext4_should_dioread_nolock(inode)) + ret = ext4_block_write_begin(handle, folio, pos, len, + ext4_get_block_unwritten); + else + ret = ext4_block_write_begin(handle, folio, pos, len, + ext4_get_block); + if (!ret && ext4_should_journal_data(inode)) { + ret = ext4_walk_page_buffers(handle, inode, + folio_buffers(folio), from, to, + NULL, do_journal_get_write_access); + } + + if (ret) { + bool extended = (pos + len > inode->i_size) && + !ext4_verity_in_progress(inode); + + folio_unlock(folio); + /* + * ext4_block_write_begin may have instantiated a few blocks + * outside i_size. Trim these off again. Don't need + * i_size_read because we hold i_rwsem. + * + * Add inode to orphan list in case we crash before + * truncate finishes + */ + if (extended && ext4_can_truncate(inode)) + ext4_orphan_add(handle, inode); + + ext4_journal_stop(handle); + if (extended) { + ext4_truncate_failed_write(inode); + /* + * If truncate failed early the inode might + * still be on the orphan list; we need to + * make sure the inode is removed from the + * orphan list in that case. + */ + if (inode->i_nlink) + ext4_orphan_del(NULL, inode); + } + + if (ret == -EAGAIN || + (ret == -ENOSPC && + ext4_should_retry_alloc(inode->i_sb, &retries))) + goto retry_journal; + folio_put(folio); + return ret; + } + *foliop = folio; + return ret; +} + +/* For write_end() in data=journal mode */ +static int write_end_fn(handle_t *handle, struct inode *inode, + struct buffer_head *bh) +{ + int ret; + if (!buffer_mapped(bh) || buffer_freed(bh)) + return 0; + set_buffer_uptodate(bh); + ret = ext4_dirty_journalled_data(handle, bh); + clear_buffer_meta(bh); + clear_buffer_prio(bh); + clear_buffer_new(bh); + return ret; +} + +/* + * We need to pick up the new inode size which generic_commit_write gave us + * `iocb` can be NULL - eg, when called from page_symlink(). + */ +static int ext4_write_end(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct folio *folio, void *fsdata) +{ + handle_t *handle = ext4_journal_current_handle(); + struct inode *inode = mapping->host; + loff_t old_size = inode->i_size; + int ret = 0, ret2; + int i_size_changed = 0; + bool verity = ext4_verity_in_progress(inode); + + trace_ext4_write_end(inode, pos, len, copied); + + if ((unsigned long)fsdata & EXT4_WRITE_DATA_INLINE) + return ext4_write_inline_data_end(inode, pos, len, copied, + folio); + + copied = block_write_end(pos, len, copied, folio); + /* + * it's important to update i_size while still holding folio lock: + * page writeout could otherwise come in and zero beyond i_size. + * + * If FS_IOC_ENABLE_VERITY is running on this inode, then Merkle tree + * blocks are being written past EOF, so skip the i_size update. + */ + if (!verity) + i_size_changed = ext4_update_inode_size(inode, pos + copied); + folio_unlock(folio); + folio_put(folio); + + if (old_size < pos && !verity) + pagecache_isize_extended(inode, old_size, pos); + + /* + * Don't mark the inode dirty under folio lock. First, it unnecessarily + * makes the holding time of folio lock longer. Second, it forces lock + * ordering of folio lock and transaction start for journaling + * filesystems. + */ + if (i_size_changed) + ret = ext4_mark_inode_dirty(handle, inode); + + if (pos + len > inode->i_size && !verity && ext4_can_truncate(inode)) + /* if we have allocated more blocks and copied + * less. We will have blocks allocated outside + * inode->i_size. So truncate them + */ + ext4_orphan_add(handle, inode); + + ret2 = ext4_journal_stop(handle); + if (!ret) + ret = ret2; + + if (pos + len > inode->i_size && !verity) { + ext4_truncate_failed_write(inode); + /* + * If truncate failed early the inode might still be + * on the orphan list; we need to make sure the inode + * is removed from the orphan list in that case. + */ + if (inode->i_nlink) + ext4_orphan_del(NULL, inode); + } + + return ret ? ret : copied; +} + +/* + * This is a private version of folio_zero_new_buffers() which doesn't + * set the buffer to be dirty, since in data=journalled mode we need + * to call ext4_dirty_journalled_data() instead. + */ +static void ext4_journalled_zero_new_buffers(handle_t *handle, + struct inode *inode, + struct folio *folio, + unsigned from, unsigned to) +{ + unsigned int block_start = 0, block_end; + struct buffer_head *head, *bh; + + bh = head = folio_buffers(folio); + do { + block_end = block_start + bh->b_size; + if (buffer_new(bh)) { + if (block_end > from && block_start < to) { + if (!folio_test_uptodate(folio)) { + unsigned start, size; + + start = max(from, block_start); + size = min(to, block_end) - start; + + folio_zero_range(folio, start, size); + } + clear_buffer_new(bh); + write_end_fn(handle, inode, bh); + } + } + block_start = block_end; + bh = bh->b_this_page; + } while (bh != head); +} + +static int ext4_journalled_write_end(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct folio *folio, void *fsdata) +{ + handle_t *handle = ext4_journal_current_handle(); + struct inode *inode = mapping->host; + loff_t old_size = inode->i_size; + int ret = 0, ret2; + int partial = 0; + unsigned from, to; + int size_changed = 0; + bool verity = ext4_verity_in_progress(inode); + + trace_ext4_journalled_write_end(inode, pos, len, copied); + from = pos & (PAGE_SIZE - 1); + to = from + len; + + BUG_ON(!ext4_handle_valid(handle)); + + if ((unsigned long)fsdata & EXT4_WRITE_DATA_INLINE) + return ext4_write_inline_data_end(inode, pos, len, copied, + folio); + + if (unlikely(copied < len) && !folio_test_uptodate(folio)) { + copied = 0; + ext4_journalled_zero_new_buffers(handle, inode, folio, + from, to); + } else { + if (unlikely(copied < len)) + ext4_journalled_zero_new_buffers(handle, inode, folio, + from + copied, to); + ret = ext4_walk_page_buffers(handle, inode, + folio_buffers(folio), + from, from + copied, &partial, + write_end_fn); + if (!partial) + folio_mark_uptodate(folio); + } + if (!verity) + size_changed = ext4_update_inode_size(inode, pos + copied); + EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid; + folio_unlock(folio); + folio_put(folio); + + if (old_size < pos && !verity) + pagecache_isize_extended(inode, old_size, pos); + + if (size_changed) { + ret2 = ext4_mark_inode_dirty(handle, inode); + if (!ret) + ret = ret2; + } + + if (pos + len > inode->i_size && !verity && ext4_can_truncate(inode)) + /* if we have allocated more blocks and copied + * less. We will have blocks allocated outside + * inode->i_size. So truncate them + */ + ext4_orphan_add(handle, inode); + + ret2 = ext4_journal_stop(handle); + if (!ret) + ret = ret2; + if (pos + len > inode->i_size && !verity) { + ext4_truncate_failed_write(inode); + /* + * If truncate failed early the inode might still be + * on the orphan list; we need to make sure the inode + * is removed from the orphan list in that case. + */ + if (inode->i_nlink) + ext4_orphan_del(NULL, inode); + } + + return ret ? ret : copied; +} + +/* + * Reserve space for 'nr_resv' clusters + */ +static int ext4_da_reserve_space(struct inode *inode, int nr_resv) +{ + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + struct ext4_inode_info *ei = EXT4_I(inode); + int ret; + + /* + * We will charge metadata quota at writeout time; this saves + * us from metadata over-estimation, though we may go over by + * a small amount in the end. Here we just reserve for data. + */ + ret = dquot_reserve_block(inode, EXT4_C2B(sbi, nr_resv)); + if (ret) + return ret; + + spin_lock(&ei->i_block_reservation_lock); + if (ext4_claim_free_clusters(sbi, nr_resv, 0)) { + spin_unlock(&ei->i_block_reservation_lock); + dquot_release_reservation_block(inode, EXT4_C2B(sbi, nr_resv)); + return -ENOSPC; + } + ei->i_reserved_data_blocks += nr_resv; + trace_ext4_da_reserve_space(inode, nr_resv); + spin_unlock(&ei->i_block_reservation_lock); + + return 0; /* success */ +} + +void ext4_da_release_space(struct inode *inode, int to_free) +{ + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + struct ext4_inode_info *ei = EXT4_I(inode); + + if (!to_free) + return; /* Nothing to release, exit */ + + spin_lock(&EXT4_I(inode)->i_block_reservation_lock); + + trace_ext4_da_release_space(inode, to_free); + if (unlikely(to_free > ei->i_reserved_data_blocks)) { + /* + * if there aren't enough reserved blocks, then the + * counter is messed up somewhere. Since this + * function is called from invalidate page, it's + * harmless to return without any action. + */ + ext4_warning(inode->i_sb, "ext4_da_release_space: " + "ino %llu, to_free %d with only %d reserved " + "data blocks", inode->i_ino, to_free, + ei->i_reserved_data_blocks); + WARN_ON(1); + to_free = ei->i_reserved_data_blocks; + } + ei->i_reserved_data_blocks -= to_free; + + /* update fs dirty data blocks counter */ + percpu_counter_sub(&sbi->s_dirtyclusters_counter, to_free); + + spin_unlock(&EXT4_I(inode)->i_block_reservation_lock); + + dquot_release_reservation_block(inode, EXT4_C2B(sbi, to_free)); +} + +/* + * Delayed allocation stuff + */ + +struct mpage_da_data { + /* These are input fields for ext4_do_writepages() */ + struct inode *inode; + struct writeback_control *wbc; + unsigned int can_map:1; /* Can writepages call map blocks? */ + + /* These are internal state of ext4_do_writepages() */ + loff_t start_pos; /* The start pos to write */ + loff_t next_pos; /* Current pos to examine */ + loff_t end_pos; /* Last pos to examine */ + + /* + * Extent to map - this can be after start_pos because that can be + * fully mapped. We somewhat abuse m_flags to store whether the extent + * is delalloc or unwritten. + */ + struct ext4_map_blocks map; + struct ext4_io_submit io_submit; /* IO submission data */ + unsigned int do_map:1; + unsigned int scanned_until_end:1; + unsigned int journalled_more_data:1; +}; + +static void mpage_release_unused_pages(struct mpage_da_data *mpd, + bool invalidate) +{ + unsigned nr, i; + pgoff_t index, end; + struct folio_batch fbatch; + struct inode *inode = mpd->inode; + struct address_space *mapping = inode->i_mapping; + + /* This is necessary when next_pos == 0. */ + if (mpd->start_pos >= mpd->next_pos) + return; + + mpd->scanned_until_end = 0; + if (invalidate) { + ext4_lblk_t start, last; + start = EXT4_B_TO_LBLK(inode, mpd->start_pos); + last = mpd->next_pos >> inode->i_blkbits; + + /* + * avoid racing with extent status tree scans made by + * ext4_insert_delayed_block() + */ + down_write(&EXT4_I(inode)->i_data_sem); + ext4_es_remove_extent(inode, start, last - start); + up_write(&EXT4_I(inode)->i_data_sem); + } + + folio_batch_init(&fbatch); + index = mpd->start_pos >> PAGE_SHIFT; + end = mpd->next_pos >> PAGE_SHIFT; + while (index < end) { + nr = filemap_get_folios(mapping, &index, end - 1, &fbatch); + if (nr == 0) + break; + for (i = 0; i < nr; i++) { + struct folio *folio = fbatch.folios[i]; + + if (folio_pos(folio) < mpd->start_pos) + continue; + if (folio_next_index(folio) > end) + continue; + BUG_ON(!folio_test_locked(folio)); + BUG_ON(folio_test_writeback(folio)); + if (invalidate) { + if (folio_mapped(folio)) { + folio_clear_dirty_for_io(folio); + /* + * Unmap folio from page + * tables to prevent + * subsequent accesses through + * stale PTEs. This ensures + * future accesses trigger new + * page faults rather than + * reusing the invalidated + * folio. + */ + unmap_mapping_pages(folio->mapping, + folio->index, + folio_nr_pages(folio), false); + } + block_invalidate_folio(folio, 0, + folio_size(folio)); + folio_clear_uptodate(folio); + } + folio_unlock(folio); + } + folio_batch_release(&fbatch); + } +} + +static void ext4_print_free_blocks(struct inode *inode) +{ + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + struct super_block *sb = inode->i_sb; + struct ext4_inode_info *ei = EXT4_I(inode); + + ext4_msg(sb, KERN_CRIT, "Total free blocks count %lld", + EXT4_C2B(EXT4_SB(inode->i_sb), + ext4_count_free_clusters(sb))); + ext4_msg(sb, KERN_CRIT, "Free/Dirty block details"); + ext4_msg(sb, KERN_CRIT, "free_blocks=%lld", + (long long) EXT4_C2B(EXT4_SB(sb), + percpu_counter_sum(&sbi->s_freeclusters_counter))); + ext4_msg(sb, KERN_CRIT, "dirty_blocks=%lld", + (long long) EXT4_C2B(EXT4_SB(sb), + percpu_counter_sum(&sbi->s_dirtyclusters_counter))); + ext4_msg(sb, KERN_CRIT, "Block reservation details"); + ext4_msg(sb, KERN_CRIT, "i_reserved_data_blocks=%u", + ei->i_reserved_data_blocks); + return; +} + +/* + * Check whether the cluster containing lblk has been allocated or has + * delalloc reservation. + * + * Returns 0 if the cluster doesn't have either, 1 if it has delalloc + * reservation, 2 if it's already been allocated, negative error code on + * failure. + */ +static int ext4_clu_alloc_state(struct inode *inode, ext4_lblk_t lblk) +{ + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + int ret; + + /* Has delalloc reservation? */ + if (ext4_es_scan_clu(inode, &ext4_es_is_delayed, lblk)) + return 1; + + /* Already been allocated? */ + if (ext4_es_scan_clu(inode, &ext4_es_is_mapped, lblk)) + return 2; + ret = ext4_clu_mapped(inode, EXT4_B2C(sbi, lblk)); + if (ret < 0) + return ret; + if (ret > 0) + return 2; + + return 0; +} + +/* + * ext4_insert_delayed_blocks - adds a multiple delayed blocks to the extents + * status tree, incrementing the reserved + * cluster/block count or making pending + * reservations where needed + * + * @inode - file containing the newly added block + * @lblk - start logical block to be added + * @len - length of blocks to be added + * + * Returns 0 on success, negative error code on failure. + */ +static int ext4_insert_delayed_blocks(struct inode *inode, ext4_lblk_t lblk, + ext4_lblk_t len) +{ + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + int ret; + bool lclu_allocated = false; + bool end_allocated = false; + ext4_lblk_t resv_clu; + ext4_lblk_t end = lblk + len - 1; + + /* + * If the cluster containing lblk or end is shared with a delayed, + * written, or unwritten extent in a bigalloc file system, it's + * already been accounted for and does not need to be reserved. + * A pending reservation must be made for the cluster if it's + * shared with a written or unwritten extent and doesn't already + * have one. Written and unwritten extents can be purged from the + * extents status tree if the system is under memory pressure, so + * it's necessary to examine the extent tree if a search of the + * extents status tree doesn't get a match. + */ + if (sbi->s_cluster_ratio == 1) { + ret = ext4_da_reserve_space(inode, len); + if (ret != 0) /* ENOSPC */ + return ret; + } else { /* bigalloc */ + resv_clu = EXT4_B2C(sbi, end) - EXT4_B2C(sbi, lblk) + 1; + + ret = ext4_clu_alloc_state(inode, lblk); + if (ret < 0) + return ret; + if (ret > 0) { + resv_clu--; + lclu_allocated = (ret == 2); + } + + if (EXT4_B2C(sbi, lblk) != EXT4_B2C(sbi, end)) { + ret = ext4_clu_alloc_state(inode, end); + if (ret < 0) + return ret; + if (ret > 0) { + resv_clu--; + end_allocated = (ret == 2); + } + } + + if (resv_clu) { + ret = ext4_da_reserve_space(inode, resv_clu); + if (ret != 0) /* ENOSPC */ + return ret; + } + } + + ext4_es_insert_delayed_extent(inode, lblk, len, lclu_allocated, + end_allocated); + return 0; +} + +/* + * Looks up the requested blocks and sets the delalloc extent map. + * First try to look up for the extent entry that contains the requested + * blocks in the extent status tree without i_data_sem, then try to look + * up for the ondisk extent mapping with i_data_sem in read mode, + * finally hold i_data_sem in write mode, looks up again and add a + * delalloc extent entry if it still couldn't find any extent. Pass out + * the mapped extent through @map and return 0 on success. + */ +static int ext4_da_map_blocks(struct inode *inode, struct ext4_map_blocks *map) +{ + struct extent_status es; + int retval; +#ifdef ES_AGGRESSIVE_TEST + struct ext4_map_blocks orig_map; + + memcpy(&orig_map, map, sizeof(*map)); +#endif + + map->m_flags = 0; + ext_debug(inode, "max_blocks %u, logical block %lu\n", map->m_len, + (unsigned long) map->m_lblk); + + ext4_check_map_extents_env(inode); + + /* Lookup extent status tree firstly */ + if (ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es, NULL)) { + map->m_len = min_t(unsigned int, map->m_len, + es.es_len - (map->m_lblk - es.es_lblk)); + + if (ext4_es_is_hole(&es)) + goto add_delayed; + +found: + /* + * Delayed extent could be allocated by fallocate. + * So we need to check it. + */ + if (ext4_es_is_delayed(&es)) { + map->m_flags |= EXT4_MAP_DELAYED; + return 0; + } + + map->m_pblk = ext4_es_pblock(&es) + map->m_lblk - es.es_lblk; + if (ext4_es_is_written(&es)) + map->m_flags |= EXT4_MAP_MAPPED; + else if (ext4_es_is_unwritten(&es)) + map->m_flags |= EXT4_MAP_UNWRITTEN; + else + BUG(); + +#ifdef ES_AGGRESSIVE_TEST + ext4_map_blocks_es_recheck(NULL, inode, map, &orig_map, 0); +#endif + return 0; + } + + /* + * Try to see if we can get the block without requesting a new + * file system block. + */ + down_read(&EXT4_I(inode)->i_data_sem); + if (ext4_has_inline_data(inode)) + retval = 0; + else + retval = ext4_map_query_blocks(NULL, inode, map, 0); + up_read(&EXT4_I(inode)->i_data_sem); + if (retval) + return retval < 0 ? retval : 0; + +add_delayed: + down_write(&EXT4_I(inode)->i_data_sem); + /* + * Page fault path (ext4_page_mkwrite does not take i_rwsem) + * and fallocate path (no folio lock) can race. Make sure we + * lookup the extent status tree here again while i_data_sem + * is held in write mode, before inserting a new da entry in + * the extent status tree. + */ + if (ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es, NULL)) { + map->m_len = min_t(unsigned int, map->m_len, + es.es_len - (map->m_lblk - es.es_lblk)); + + if (!ext4_es_is_hole(&es)) { + up_write(&EXT4_I(inode)->i_data_sem); + goto found; + } + } else if (!ext4_has_inline_data(inode)) { + retval = ext4_map_query_blocks(NULL, inode, map, 0); + if (retval) { + up_write(&EXT4_I(inode)->i_data_sem); + return retval < 0 ? retval : 0; + } + } + + map->m_flags |= EXT4_MAP_DELAYED; + retval = ext4_insert_delayed_blocks(inode, map->m_lblk, map->m_len); + if (!retval) + map->m_seq = READ_ONCE(EXT4_I(inode)->i_es_seq); + up_write(&EXT4_I(inode)->i_data_sem); + + return retval; +} + +/* + * This is a special get_block_t callback which is used by + * ext4_da_write_begin(). It will either return mapped block or + * reserve space for a single block. + * + * For delayed buffer_head we have BH_Mapped, BH_New, BH_Delay set. + * We also have b_blocknr = -1 and b_bdev initialized properly + * + * For unwritten buffer_head we have BH_Mapped, BH_New, BH_Unwritten set. + * We also have b_blocknr = physicalblock mapping unwritten extent and b_bdev + * initialized properly. + */ +int ext4_da_get_block_prep(struct inode *inode, sector_t iblock, + struct buffer_head *bh, int create) +{ + struct ext4_map_blocks map; + sector_t invalid_block = ~((sector_t) 0xffff); + int ret = 0; + + BUG_ON(create == 0); + BUG_ON(bh->b_size != inode->i_sb->s_blocksize); + + if (invalid_block < ext4_blocks_count(EXT4_SB(inode->i_sb)->s_es)) + invalid_block = ~0; + + map.m_lblk = iblock; + map.m_len = 1; + + /* + * first, we need to know whether the block is allocated already + * preallocated blocks are unmapped but should treated + * the same as allocated blocks. + */ + ret = ext4_da_map_blocks(inode, &map); + if (ret < 0) + return ret; + + if (map.m_flags & EXT4_MAP_DELAYED) { + map_bh(bh, inode->i_sb, invalid_block); + set_buffer_new(bh); + set_buffer_delay(bh); + return 0; + } + + map_bh(bh, inode->i_sb, map.m_pblk); + ext4_update_bh_state(bh, map.m_flags); + + if (buffer_unwritten(bh)) { + /* A delayed write to unwritten bh should be marked + * new and mapped. Mapped ensures that we don't do + * get_block multiple times when we write to the same + * offset and new ensures that we do proper zero out + * for partial write. + */ + set_buffer_new(bh); + set_buffer_mapped(bh); + } + return 0; +} + +static void mpage_folio_done(struct mpage_da_data *mpd, struct folio *folio) +{ + mpd->start_pos += folio_size(folio); + mpd->wbc->nr_to_write -= folio_nr_pages(folio); + folio_unlock(folio); +} + +static void mpage_submit_folio(struct mpage_da_data *mpd, struct folio *folio) +{ + size_t len; + loff_t size; + + WARN_ON_ONCE(folio_pos(folio) != mpd->start_pos); + folio_clear_dirty_for_io(folio); + /* + * We have to be very careful here! Nothing protects writeback path + * against i_size changes and the page can be writeably mapped into + * page tables. So an application can be growing i_size and writing + * data through mmap while writeback runs. folio_clear_dirty_for_io() + * write-protects our page in page tables and the page cannot get + * written to again until we release folio lock. So only after + * folio_clear_dirty_for_io() we are safe to sample i_size for + * ext4_bio_write_folio() to zero-out tail of the written page. We rely + * on the barrier provided by folio_test_clear_dirty() in + * folio_clear_dirty_for_io() to make sure i_size is really sampled only + * after page tables are updated. + */ + size = i_size_read(mpd->inode); + len = folio_size(folio); + if (folio_pos(folio) + len > size && + !ext4_verity_in_progress(mpd->inode)) + len = size & (len - 1); + ext4_bio_write_folio(&mpd->io_submit, folio, len); +} + +#define BH_FLAGS (BIT(BH_Unwritten) | BIT(BH_Delay)) + +/* + * mballoc gives us at most this number of blocks... + * XXX: That seems to be only a limitation of ext4_mb_normalize_request(). + * The rest of mballoc seems to handle chunks up to full group size. + */ +#define MAX_WRITEPAGES_EXTENT_LEN 2048 + +/* + * mpage_add_bh_to_extent - try to add bh to extent of blocks to map + * + * @mpd - extent of blocks + * @lblk - logical number of the block in the file + * @bh - buffer head we want to add to the extent + * + * The function is used to collect contig. blocks in the same state. If the + * buffer doesn't require mapping for writeback and we haven't started the + * extent of buffers to map yet, the function returns 'true' immediately - the + * caller can write the buffer right away. Otherwise the function returns true + * if the block has been added to the extent, false if the block couldn't be + * added. + */ +static bool mpage_add_bh_to_extent(struct mpage_da_data *mpd, ext4_lblk_t lblk, + struct buffer_head *bh) +{ + struct ext4_map_blocks *map = &mpd->map; + + /* Buffer that doesn't need mapping for writeback? */ + if (!buffer_dirty(bh) || !buffer_mapped(bh) || + (!buffer_delay(bh) && !buffer_unwritten(bh))) { + /* So far no extent to map => we write the buffer right away */ + if (map->m_len == 0) + return true; + return false; + } + + /* First block in the extent? */ + if (map->m_len == 0) { + /* We cannot map unless handle is started... */ + if (!mpd->do_map) + return false; + map->m_lblk = lblk; + map->m_len = 1; + map->m_flags = bh->b_state & BH_FLAGS; + return true; + } + + /* Don't go larger than mballoc is willing to allocate */ + if (map->m_len >= MAX_WRITEPAGES_EXTENT_LEN) + return false; + + /* Can we merge the block to our big extent? */ + if (lblk == map->m_lblk + map->m_len && + (bh->b_state & BH_FLAGS) == map->m_flags) { + map->m_len++; + return true; + } + return false; +} + +/* + * mpage_process_page_bufs - submit page buffers for IO or add them to extent + * + * @mpd - extent of blocks for mapping + * @head - the first buffer in the page + * @bh - buffer we should start processing from + * @lblk - logical number of the block in the file corresponding to @bh + * + * Walk through page buffers from @bh upto @head (exclusive) and either submit + * the page for IO if all buffers in this page were mapped and there's no + * accumulated extent of buffers to map or add buffers in the page to the + * extent of buffers to map. The function returns 1 if the caller can continue + * by processing the next page, 0 if it should stop adding buffers to the + * extent to map because we cannot extend it anymore. + */ +static int mpage_process_page_bufs(struct mpage_da_data *mpd, + struct buffer_head *head, + struct buffer_head *bh, + ext4_lblk_t lblk) +{ + struct inode *inode = mpd->inode; + ext4_lblk_t blocks = (i_size_read(inode) + i_blocksize(inode) - 1) + >> inode->i_blkbits; + + if (ext4_verity_in_progress(inode)) + blocks = EXT_MAX_BLOCKS; + + do { + BUG_ON(buffer_locked(bh)); + + if (lblk >= blocks || !mpage_add_bh_to_extent(mpd, lblk, bh)) { + /* Found extent to map? */ + if (mpd->map.m_len) + return 0; + /* Buffer needs mapping and handle is not started? */ + if (!mpd->do_map) + return 0; + /* Everything mapped so far and we hit EOF */ + break; + } + } while (lblk++, (bh = bh->b_this_page) != head); + /* So far everything mapped? Submit the page for IO. */ + if (mpd->map.m_len == 0) { + mpage_submit_folio(mpd, head->b_folio); + mpage_folio_done(mpd, head->b_folio); + } + if (lblk >= blocks) { + mpd->scanned_until_end = 1; + return 0; + } + return 1; +} + +/* + * mpage_process_folio - update folio buffers corresponding to changed extent + * and may submit fully mapped page for IO + * @mpd: description of extent to map, on return next extent to map + * @folio: Contains these buffers. + * @m_lblk: logical block mapping. + * @m_pblk: corresponding physical mapping. + * @map_bh: determines on return whether this page requires any further + * mapping or not. + * + * Scan given folio buffers corresponding to changed extent and update buffer + * state according to new extent state. + * We map delalloc buffers to their physical location, clear unwritten bits. + * If the given folio is not fully mapped, we update @mpd to the next extent in + * the given folio that needs mapping & return @map_bh as true. + */ +static int mpage_process_folio(struct mpage_da_data *mpd, struct folio *folio, + ext4_lblk_t *m_lblk, ext4_fsblk_t *m_pblk, + bool *map_bh) +{ + struct buffer_head *head, *bh; + ext4_io_end_t *io_end = mpd->io_submit.io_end; + ext4_lblk_t lblk = *m_lblk; + ext4_fsblk_t pblock = *m_pblk; + int err = 0; + ssize_t io_end_size = 0; + struct ext4_io_end_vec *io_end_vec = ext4_last_io_end_vec(io_end); + + bh = head = folio_buffers(folio); + do { + if (lblk < mpd->map.m_lblk) + continue; + if (lblk >= mpd->map.m_lblk + mpd->map.m_len) { + /* + * Buffer after end of mapped extent. + * Find next buffer in the folio to map. + */ + mpd->map.m_len = 0; + mpd->map.m_flags = 0; + io_end_vec->size += io_end_size; + + err = mpage_process_page_bufs(mpd, head, bh, lblk); + if (err > 0) + err = 0; + if (!err && mpd->map.m_len && mpd->map.m_lblk > lblk) { + io_end_vec = ext4_alloc_io_end_vec(io_end); + if (IS_ERR(io_end_vec)) { + err = PTR_ERR(io_end_vec); + goto out; + } + io_end_vec->offset = EXT4_LBLK_TO_B(mpd->inode, + mpd->map.m_lblk); + } + *map_bh = true; + goto out; + } + if (buffer_delay(bh)) { + clear_buffer_delay(bh); + bh->b_blocknr = pblock++; + } + clear_buffer_unwritten(bh); + io_end_size += i_blocksize(mpd->inode); + } while (lblk++, (bh = bh->b_this_page) != head); + + io_end_vec->size += io_end_size; + *map_bh = false; +out: + *m_lblk = lblk; + *m_pblk = pblock; + return err; +} + +/* + * mpage_map_buffers - update buffers corresponding to changed extent and + * submit fully mapped pages for IO + * + * @mpd - description of extent to map, on return next extent to map + * + * Scan buffers corresponding to changed extent (we expect corresponding pages + * to be already locked) and update buffer state according to new extent state. + * We map delalloc buffers to their physical location, clear unwritten bits, + * and mark buffers as uninit when we perform writes to unwritten extents + * and do extent conversion after IO is finished. If the last page is not fully + * mapped, we update @map to the next extent in the last page that needs + * mapping. Otherwise we submit the page for IO. + */ +static int mpage_map_and_submit_buffers(struct mpage_da_data *mpd) +{ + struct folio_batch fbatch; + unsigned nr, i; + struct inode *inode = mpd->inode; + pgoff_t start, end; + ext4_lblk_t lblk; + ext4_fsblk_t pblock; + int err; + bool map_bh = false; + + start = EXT4_LBLK_TO_PG(inode, mpd->map.m_lblk); + end = EXT4_LBLK_TO_PG(inode, mpd->map.m_lblk + mpd->map.m_len - 1); + pblock = mpd->map.m_pblk; + + folio_batch_init(&fbatch); + while (start <= end) { + nr = filemap_get_folios(inode->i_mapping, &start, end, &fbatch); + if (nr == 0) + break; + for (i = 0; i < nr; i++) { + struct folio *folio = fbatch.folios[i]; + + lblk = EXT4_PG_TO_LBLK(inode, folio->index); + err = mpage_process_folio(mpd, folio, &lblk, &pblock, + &map_bh); + /* + * If map_bh is true, means page may require further bh + * mapping, or maybe the page was submitted for IO. + * So we return to call further extent mapping. + */ + if (err < 0 || map_bh) + goto out; + /* Page fully mapped - let IO run! */ + mpage_submit_folio(mpd, folio); + mpage_folio_done(mpd, folio); + } + folio_batch_release(&fbatch); + } + /* Extent fully mapped and matches with page boundary. We are done. */ + mpd->map.m_len = 0; + mpd->map.m_flags = 0; + return 0; +out: + folio_batch_release(&fbatch); + return err; +} + +static int mpage_map_one_extent(handle_t *handle, struct mpage_da_data *mpd) +{ + struct inode *inode = mpd->inode; + struct ext4_map_blocks *map = &mpd->map; + int get_blocks_flags; + int err, dioread_nolock; + + /* Make sure transaction has enough credits for this extent */ + err = ext4_journal_ensure_extent_credits(handle, inode); + if (err < 0) + return err; + + trace_ext4_da_write_pages_extent(inode, map); + /* + * Call ext4_map_blocks() to allocate any delayed allocation blocks, or + * to convert an unwritten extent to be initialized (in the case + * where we have written into one or more preallocated blocks). It is + * possible that we're going to need more metadata blocks than + * previously reserved. However we must not fail because we're in + * writeback and there is nothing we can do about it so it might result + * in data loss. So use reserved blocks to allocate metadata if + * possible. In addition, do not cache any unrelated extents, as it + * only holds the folio lock but does not hold the i_rwsem or + * invalidate_lock, which could corrupt the extent status tree. + */ + get_blocks_flags = EXT4_GET_BLOCKS_CREATE | + EXT4_GET_BLOCKS_METADATA_NOFAIL | + EXT4_GET_BLOCKS_IO_SUBMIT | + EXT4_EX_NOCACHE; + + dioread_nolock = ext4_should_dioread_nolock(inode); + if (dioread_nolock) + get_blocks_flags |= EXT4_GET_BLOCKS_UNWRIT_EXT; + + err = ext4_map_blocks(handle, inode, map, get_blocks_flags); + if (err < 0) + return err; + if (dioread_nolock && (map->m_flags & EXT4_MAP_UNWRITTEN)) { + if (!mpd->io_submit.io_end->handle && + ext4_handle_valid(handle)) { + mpd->io_submit.io_end->handle = handle->h_rsv_handle; + handle->h_rsv_handle = NULL; + } + ext4_set_io_unwritten_flag(mpd->io_submit.io_end); + } + + BUG_ON(map->m_len == 0); + return 0; +} + +/* + * This is used to submit mapped buffers in a single folio that is not fully + * mapped for various reasons, such as insufficient space or journal credits. + */ +static int mpage_submit_partial_folio(struct mpage_da_data *mpd) +{ + struct inode *inode = mpd->inode; + struct folio *folio; + loff_t pos; + + folio = filemap_get_folio(inode->i_mapping, + mpd->start_pos >> PAGE_SHIFT); + if (IS_ERR(folio)) + return PTR_ERR(folio); + /* + * The mapped position should be within the current processing folio + * but must not be the folio start position. + */ + pos = ((loff_t)mpd->map.m_lblk) << inode->i_blkbits; + if (WARN_ON_ONCE((folio_pos(folio) == pos) || + !folio_contains(folio, pos >> PAGE_SHIFT))) + return -EINVAL; + + mpage_submit_folio(mpd, folio); + /* + * Update start_pos to prevent this folio from being released in + * mpage_release_unused_pages(), it will be reset to the aligned folio + * pos when this folio is written again in the next round. Additionally, + * do not update wbc->nr_to_write here, as it will be updated once the + * entire folio has finished processing. + */ + mpd->start_pos = pos; + folio_unlock(folio); + folio_put(folio); + return 0; +} + +/* + * mpage_map_and_submit_extent - map extent starting at mpd->lblk of length + * mpd->len and submit pages underlying it for IO + * + * @handle - handle for journal operations + * @mpd - extent to map + * @give_up_on_write - we set this to true iff there is a fatal error and there + * is no hope of writing the data. The caller should discard + * dirty pages to avoid infinite loops. + * + * The function maps extent starting at mpd->lblk of length mpd->len. If it is + * delayed, blocks are allocated, if it is unwritten, we may need to convert + * them to initialized or split the described range from larger unwritten + * extent. Note that we need not map all the described range since allocation + * can return less blocks or the range is covered by more unwritten extents. We + * cannot map more because we are limited by reserved transaction credits. On + * the other hand we always make sure that the last touched page is fully + * mapped so that it can be written out (and thus forward progress is + * guaranteed). After mapping we submit all mapped pages for IO. + */ +static int mpage_map_and_submit_extent(handle_t *handle, + struct mpage_da_data *mpd, + bool *give_up_on_write) +{ + struct inode *inode = mpd->inode; + struct ext4_map_blocks *map = &mpd->map; + int err; + loff_t disksize; + int progress = 0; + ext4_io_end_t *io_end = mpd->io_submit.io_end; + struct ext4_io_end_vec *io_end_vec; + + io_end_vec = ext4_alloc_io_end_vec(io_end); + if (IS_ERR(io_end_vec)) + return PTR_ERR(io_end_vec); + io_end_vec->offset = EXT4_LBLK_TO_B(inode, map->m_lblk); + do { + err = mpage_map_one_extent(handle, mpd); + if (err < 0) { + struct super_block *sb = inode->i_sb; + + if (ext4_emergency_state(sb)) + goto invalidate_dirty_pages; + /* + * Let the uper layers retry transient errors. + * In the case of ENOSPC, if ext4_count_free_blocks() + * is non-zero, a commit should free up blocks. + */ + if ((err == -ENOMEM) || (err == -EAGAIN) || + (err == -ENOSPC && ext4_count_free_clusters(sb))) { + /* + * We may have already allocated extents for + * some bhs inside the folio, issue the + * corresponding data to prevent stale data. + */ + if (progress) { + if (mpage_submit_partial_folio(mpd)) + goto invalidate_dirty_pages; + goto update_disksize; + } + return err; + } + ext4_msg(sb, KERN_CRIT, + "Delayed block allocation failed for " + "inode %llu at logical offset %llu with" + " max blocks %u with error %d", + inode->i_ino, + (unsigned long long)map->m_lblk, + (unsigned)map->m_len, -err); + ext4_msg(sb, KERN_CRIT, + "This should not happen!! Data will " + "be lost\n"); + if (err == -ENOSPC) + ext4_print_free_blocks(inode); + invalidate_dirty_pages: + *give_up_on_write = true; + return err; + } + progress = 1; + /* + * Update buffer state, submit mapped pages, and get us new + * extent to map + */ + err = mpage_map_and_submit_buffers(mpd); + if (err < 0) + goto update_disksize; + } while (map->m_len); + +update_disksize: + /* + * Update on-disk size after IO is submitted. Races with + * truncate are avoided by checking i_size under i_data_sem. + */ + disksize = mpd->start_pos; + if (disksize > READ_ONCE(EXT4_I(inode)->i_disksize)) { + int err2; + loff_t i_size; + + down_write(&EXT4_I(inode)->i_data_sem); + i_size = i_size_read(inode); + if (disksize > i_size) + disksize = i_size; + if (disksize > EXT4_I(inode)->i_disksize) + EXT4_I(inode)->i_disksize = disksize; + up_write(&EXT4_I(inode)->i_data_sem); + err2 = ext4_mark_inode_dirty(handle, inode); + if (err2) { + ext4_error_err(inode->i_sb, -err2, + "Failed to mark inode %llu dirty", + inode->i_ino); + } + if (!err) + err = err2; + } + return err; +} + +static int ext4_journal_folio_buffers(handle_t *handle, struct folio *folio, + size_t len) +{ + struct buffer_head *page_bufs = folio_buffers(folio); + struct inode *inode = folio->mapping->host; + int ret, err; + + ret = ext4_walk_page_buffers(handle, inode, page_bufs, 0, len, + NULL, do_journal_get_write_access); + err = ext4_walk_page_buffers(handle, inode, page_bufs, 0, len, + NULL, write_end_fn); + if (ret == 0) + ret = err; + err = ext4_jbd2_inode_add_write(handle, inode, folio_pos(folio), len); + if (ret == 0) + ret = err; + EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid; + + return ret; +} + +static int mpage_journal_page_buffers(handle_t *handle, + struct mpage_da_data *mpd, + struct folio *folio) +{ + struct inode *inode = mpd->inode; + loff_t size = i_size_read(inode); + size_t len = folio_size(folio); + + folio_clear_checked(folio); + mpd->wbc->nr_to_write -= folio_nr_pages(folio); + + if (folio_pos(folio) + len > size && + !ext4_verity_in_progress(inode)) + len = size & (len - 1); + + return ext4_journal_folio_buffers(handle, folio, len); +} + +/* + * mpage_prepare_extent_to_map - find & lock contiguous range of dirty pages + * needing mapping, submit mapped pages + * + * @mpd - where to look for pages + * + * Walk dirty pages in the mapping. If they are fully mapped, submit them for + * IO immediately. If we cannot map blocks, we submit just already mapped + * buffers in the page for IO and keep page dirty. When we can map blocks and + * we find a page which isn't mapped we start accumulating extent of buffers + * underlying these pages that needs mapping (formed by either delayed or + * unwritten buffers). We also lock the pages containing these buffers. The + * extent found is returned in @mpd structure (starting at mpd->lblk with + * length mpd->len blocks). + * + * Note that this function can attach bios to one io_end structure which are + * neither logically nor physically contiguous. Although it may seem as an + * unnecessary complication, it is actually inevitable in blocksize < pagesize + * case as we need to track IO to all buffers underlying a page in one io_end. + */ +static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) +{ + struct address_space *mapping = mpd->inode->i_mapping; + struct folio_batch fbatch; + unsigned int nr_folios; + pgoff_t index = mpd->start_pos >> PAGE_SHIFT; + pgoff_t end = mpd->end_pos >> PAGE_SHIFT; + xa_mark_t tag; + int i, err = 0; + ext4_lblk_t lblk; + struct buffer_head *head; + handle_t *handle = NULL; + int bpp = ext4_journal_blocks_per_folio(mpd->inode); + + tag = wbc_to_tag(mpd->wbc); + + mpd->map.m_len = 0; + mpd->next_pos = mpd->start_pos; + if (ext4_should_journal_data(mpd->inode)) { + handle = ext4_journal_start(mpd->inode, EXT4_HT_WRITE_PAGE, + bpp); + if (IS_ERR(handle)) + return PTR_ERR(handle); + } + folio_batch_init(&fbatch); + while (index <= end) { + nr_folios = filemap_get_folios_tag(mapping, &index, end, + tag, &fbatch); + if (nr_folios == 0) + break; + + for (i = 0; i < nr_folios; i++) { + struct folio *folio = fbatch.folios[i]; + + /* + * Accumulated enough dirty pages? This doesn't apply + * to WB_SYNC_ALL mode. For integrity sync we have to + * keep going because someone may be concurrently + * dirtying pages, and we might have synced a lot of + * newly appeared dirty pages, but have not synced all + * of the old dirty pages. + */ + if (mpd->wbc->sync_mode == WB_SYNC_NONE && + mpd->wbc->nr_to_write <= + EXT4_LBLK_TO_PG(mpd->inode, mpd->map.m_len)) + goto out; + + /* If we can't merge this page, we are done. */ + if (mpd->map.m_len > 0 && + mpd->next_pos != folio_pos(folio)) + goto out; + + if (handle) { + err = ext4_journal_ensure_credits(handle, bpp, + 0); + if (err < 0) + goto out; + } + + folio_lock(folio); + /* + * If the page is no longer dirty, or its mapping no + * longer corresponds to inode we are writing (which + * means it has been truncated or invalidated), or the + * page is already under writeback and we are not doing + * a data integrity writeback, skip the page + */ + if ((folio_test_writeback(folio) && + mpd->wbc->sync_mode == WB_SYNC_NONE) || + unlikely(folio->mapping != mapping)) { + folio_unlock(folio); + continue; + } + /* + * If the folio is clean, skip writing it back. + * Cycle the folio through the writeback state + * though, to clear stale xarray tags. + */ + if (!folio_test_dirty(folio)) { + if (!folio_test_writeback(folio)) { + __folio_start_writeback(folio, false); + folio_end_writeback(folio); + } + folio_unlock(folio); + continue; + } + + folio_wait_writeback(folio); + BUG_ON(folio_test_writeback(folio)); + + /* + * Should never happen but for buggy code in + * other subsystems that call + * set_page_dirty() without properly warning + * the file system first. See [1] for more + * information. + * + * [1] https://lore.kernel.org/linux-mm/20180103100430.GE4911@quack2.suse.cz + */ + if (!folio_buffers(folio)) { + ext4_warning_inode(mpd->inode, "page %lu does not have buffers attached", folio->index); + folio_clear_dirty(folio); + folio_unlock(folio); + continue; + } + + if (mpd->map.m_len == 0) + mpd->start_pos = folio_pos(folio); + mpd->next_pos = folio_next_pos(folio); + /* + * Writeout when we cannot modify metadata is simple. + * Just submit the page. For data=journal mode we + * first handle writeout of the page for checkpoint and + * only after that handle delayed page dirtying. This + * makes sure current data is checkpointed to the final + * location before possibly journalling it again which + * is desirable when the page is frequently dirtied + * through a pin. + */ + if (!mpd->can_map) { + mpage_submit_folio(mpd, folio); + err = 0; + /* Pending dirtying of journalled data? */ + if (folio_test_checked(folio)) { + err = mpage_journal_page_buffers(handle, + mpd, folio); + if (err < 0) + goto out; + mpd->journalled_more_data = 1; + } + mpage_folio_done(mpd, folio); + } else { + /* Add all dirty buffers to mpd */ + lblk = EXT4_PG_TO_LBLK(mpd->inode, folio->index); + head = folio_buffers(folio); + err = mpage_process_page_bufs(mpd, head, head, + lblk); + if (err <= 0) + goto out; + err = 0; + } + } + folio_batch_release(&fbatch); + cond_resched(); + } + mpd->scanned_until_end = 1; + if (handle) + ext4_journal_stop(handle); + return 0; +out: + folio_batch_release(&fbatch); + if (handle) + ext4_journal_stop(handle); + return err; +} + +static int ext4_do_writepages(struct mpage_da_data *mpd) +{ + struct writeback_control *wbc = mpd->wbc; + pgoff_t writeback_index = 0; + long nr_to_write = wbc->nr_to_write; + int range_whole = 0; + int cycled = 1; + handle_t *handle = NULL; + struct inode *inode = mpd->inode; + struct address_space *mapping = inode->i_mapping; + int needed_blocks, rsv_blocks = 0, ret = 0; + struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb); + struct blk_plug plug; + bool give_up_on_write = false; + + trace_ext4_writepages(inode, wbc); + + /* + * No pages to write? This is mainly a kludge to avoid starting + * a transaction for special inodes like journal inode on last iput() + * because that could violate lock ordering on umount + */ + if (!mapping->nrpages || !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) + goto out_writepages; + + /* + * If the filesystem has aborted, it is read-only, so return + * right away instead of dumping stack traces later on that + * will obscure the real source of the problem. We test + * fs shutdown state instead of sb->s_flag's SB_RDONLY because + * the latter could be true if the filesystem is mounted + * read-only, and in that case, ext4_writepages should + * *never* be called, so if that ever happens, we would want + * the stack trace. + */ + ret = ext4_emergency_state(mapping->host->i_sb); + if (unlikely(ret)) + goto out_writepages; + + /* + * If we have inline data and arrive here, it means that + * we will soon create the block for the 1st page, so + * we'd better clear the inline data here. + */ + if (ext4_has_inline_data(inode)) { + /* Just inode will be modified... */ + handle = ext4_journal_start(inode, EXT4_HT_INODE, 1); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + goto out_writepages; + } + BUG_ON(ext4_test_inode_state(inode, + EXT4_STATE_MAY_INLINE_DATA)); + ext4_destroy_inline_data(handle, inode); + ext4_journal_stop(handle); + } + + /* + * data=journal mode does not do delalloc so we just need to writeout / + * journal already mapped buffers. On the other hand we need to commit + * transaction to make data stable. We expect all the data to be + * already in the journal (the only exception are DMA pinned pages + * dirtied behind our back) so we commit transaction here and run the + * writeback loop to checkpoint them. The checkpointing is not actually + * necessary to make data persistent *but* quite a few places (extent + * shifting operations, fsverity, ...) depend on being able to drop + * pagecache pages after calling filemap_write_and_wait() and for that + * checkpointing needs to happen. + */ + if (ext4_should_journal_data(inode)) { + mpd->can_map = 0; + if (wbc->sync_mode == WB_SYNC_ALL) + ext4_fc_commit(sbi->s_journal, + EXT4_I(inode)->i_datasync_tid); + } + mpd->journalled_more_data = 0; + + if (ext4_should_dioread_nolock(inode)) { + int bpf = ext4_journal_blocks_per_folio(inode); + /* + * We may need to convert up to one extent per block in the + * folio. + */ + rsv_blocks = ext4_meta_trans_blocks(inode, bpf, bpf, 0); + } + + if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) + range_whole = 1; + + if (wbc->range_cyclic) { + writeback_index = mapping->writeback_index; + if (writeback_index) + cycled = 0; + mpd->start_pos = writeback_index << PAGE_SHIFT; + mpd->end_pos = LLONG_MAX; + } else { + mpd->start_pos = wbc->range_start; + mpd->end_pos = wbc->range_end; + } + + ext4_io_submit_init(&mpd->io_submit, wbc); +retry: + if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) + tag_pages_for_writeback(mapping, mpd->start_pos >> PAGE_SHIFT, + mpd->end_pos >> PAGE_SHIFT); + blk_start_plug(&plug); + + /* + * First writeback pages that don't need mapping - we can avoid + * starting a transaction unnecessarily and also avoid being blocked + * in the block layer on device congestion while having transaction + * started. + */ + mpd->do_map = 0; + mpd->scanned_until_end = 0; + mpd->io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL); + if (!mpd->io_submit.io_end) { + ret = -ENOMEM; + goto unplug; + } + ret = mpage_prepare_extent_to_map(mpd); + /* Unlock pages we didn't use */ + mpage_release_unused_pages(mpd, false); + /* Submit prepared bio */ + ext4_io_submit(&mpd->io_submit); + ext4_put_io_end_defer(mpd->io_submit.io_end); + mpd->io_submit.io_end = NULL; + if (ret < 0) + goto unplug; + + while (!mpd->scanned_until_end && wbc->nr_to_write > 0) { + /* For each extent of pages we use new io_end */ + mpd->io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL); + if (!mpd->io_submit.io_end) { + ret = -ENOMEM; + break; + } + + WARN_ON_ONCE(!mpd->can_map); + /* + * We have two constraints: We find one extent to map and we + * must always write out whole page (makes a difference when + * blocksize < pagesize) so that we don't block on IO when we + * try to write out the rest of the page. Journalled mode is + * not supported by delalloc. + */ + BUG_ON(ext4_should_journal_data(inode)); + /* + * Calculate the number of credits needed to reserve for one + * extent of up to MAX_WRITEPAGES_EXTENT_LEN blocks. It will + * attempt to extend the transaction or start a new iteration + * if the reserved credits are insufficient. + */ + needed_blocks = ext4_chunk_trans_blocks(inode, + MAX_WRITEPAGES_EXTENT_LEN); + /* start a new transaction */ + handle = ext4_journal_start_with_reserve(inode, + EXT4_HT_WRITE_PAGE, needed_blocks, rsv_blocks); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + ext4_msg(inode->i_sb, KERN_CRIT, "%s: jbd2_start: " + "%ld pages, ino %llu; err %d", __func__, + wbc->nr_to_write, inode->i_ino, ret); + /* Release allocated io_end */ + ext4_put_io_end(mpd->io_submit.io_end); + mpd->io_submit.io_end = NULL; + break; + } + mpd->do_map = 1; + + trace_ext4_da_write_folios_start(inode, mpd->start_pos, + mpd->next_pos, wbc); + ret = mpage_prepare_extent_to_map(mpd); + if (!ret && mpd->map.m_len) + ret = mpage_map_and_submit_extent(handle, mpd, + &give_up_on_write); + /* + * Caution: If the handle is synchronous, + * ext4_journal_stop() can wait for transaction commit + * to finish which may depend on writeback of pages to + * complete or on page lock to be released. In that + * case, we have to wait until after we have + * submitted all the IO, released page locks we hold, + * and dropped io_end reference (for extent conversion + * to be able to complete) before stopping the handle. + */ + if (!ext4_handle_valid(handle) || handle->h_sync == 0) { + ext4_journal_stop(handle); + handle = NULL; + mpd->do_map = 0; + } + /* Unlock pages we didn't use */ + mpage_release_unused_pages(mpd, give_up_on_write); + /* Submit prepared bio */ + ext4_io_submit(&mpd->io_submit); + + /* + * Drop our io_end reference we got from init. We have + * to be careful and use deferred io_end finishing if + * we are still holding the transaction as we can + * release the last reference to io_end which may end + * up doing unwritten extent conversion. + */ + if (handle) { + ext4_put_io_end_defer(mpd->io_submit.io_end); + ext4_journal_stop(handle); + } else + ext4_put_io_end(mpd->io_submit.io_end); + mpd->io_submit.io_end = NULL; + trace_ext4_da_write_folios_end(inode, mpd->start_pos, + mpd->next_pos, wbc, ret); + + if (ret == -ENOSPC && sbi->s_journal) { + /* + * Commit the transaction which would + * free blocks released in the transaction + * and try again + */ + jbd2_journal_force_commit_nested(sbi->s_journal); + ret = 0; + continue; + } + if (ret == -EAGAIN) + ret = 0; + /* Fatal error - ENOMEM, EIO... */ + if (ret) + break; + } +unplug: + blk_finish_plug(&plug); + if (!ret && !cycled && wbc->nr_to_write > 0) { + cycled = 1; + mpd->end_pos = (writeback_index << PAGE_SHIFT) - 1; + mpd->start_pos = 0; + goto retry; + } + + /* Update index */ + if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) + /* + * Set the writeback_index so that range_cyclic + * mode will write it back later + */ + mapping->writeback_index = mpd->start_pos >> PAGE_SHIFT; + +out_writepages: + trace_ext4_writepages_result(inode, wbc, ret, + nr_to_write - wbc->nr_to_write); + return ret; +} + +static int ext4_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct super_block *sb = mapping->host->i_sb; + struct mpage_da_data mpd = { + .inode = mapping->host, + .wbc = wbc, + .can_map = 1, + }; + int ret; + int alloc_ctx; + + ret = ext4_emergency_state(sb); + if (unlikely(ret)) + return ret; + + alloc_ctx = ext4_writepages_down_read(sb); + ret = ext4_do_writepages(&mpd); + /* + * For data=journal writeback we could have come across pages marked + * for delayed dirtying (PageChecked) which were just added to the + * running transaction. Try once more to get them to stable storage. + */ + if (!ret && mpd.journalled_more_data) + ret = ext4_do_writepages(&mpd); + ext4_writepages_up_read(sb, alloc_ctx); + + return ret; +} + +int ext4_normal_submit_inode_data_buffers(struct jbd2_inode *jinode) +{ + loff_t range_start, range_end; + struct writeback_control wbc = { + .sync_mode = WB_SYNC_ALL, + .nr_to_write = LONG_MAX, + }; + struct mpage_da_data mpd = { + .inode = jinode->i_vfs_inode, + .wbc = &wbc, + .can_map = 0, + }; + + if (!jbd2_jinode_get_dirty_range(jinode, &range_start, &range_end)) + return 0; + + wbc.range_start = range_start; + wbc.range_end = range_end; + + return ext4_do_writepages(&mpd); +} + +static int ext4_dax_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + int ret; + long nr_to_write = wbc->nr_to_write; + struct inode *inode = mapping->host; + int alloc_ctx; + + ret = ext4_emergency_state(inode->i_sb); + if (unlikely(ret)) + return ret; + + alloc_ctx = ext4_writepages_down_read(inode->i_sb); + trace_ext4_writepages(inode, wbc); + + ret = dax_writeback_mapping_range(mapping, + EXT4_SB(inode->i_sb)->s_daxdev, wbc); + trace_ext4_writepages_result(inode, wbc, ret, + nr_to_write - wbc->nr_to_write); + ext4_writepages_up_read(inode->i_sb, alloc_ctx); + return ret; +} + +static int ext4_nonda_switch(struct super_block *sb) +{ + s64 free_clusters, dirty_clusters; + struct ext4_sb_info *sbi = EXT4_SB(sb); + + /* + * switch to non delalloc mode if we are running low + * on free block. The free block accounting via percpu + * counters can get slightly wrong with percpu_counter_batch getting + * accumulated on each CPU without updating global counters + * Delalloc need an accurate free block accounting. So switch + * to non delalloc when we are near to error range. + */ + free_clusters = + percpu_counter_read_positive(&sbi->s_freeclusters_counter); + dirty_clusters = + percpu_counter_read_positive(&sbi->s_dirtyclusters_counter); + /* + * Start pushing delalloc when 1/2 of free blocks are dirty. + */ + if (dirty_clusters && (free_clusters < 2 * dirty_clusters)) + try_to_writeback_inodes_sb(sb, WB_REASON_FS_FREE_SPACE); + + if (2 * free_clusters < 3 * dirty_clusters || + free_clusters < (dirty_clusters + EXT4_FREECLUSTERS_WATERMARK)) { + /* + * free block count is less than 150% of dirty blocks + * or free blocks is less than watermark + */ + return 1; + } + return 0; +} + +static int ext4_da_write_begin(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, + struct folio **foliop, void **fsdata) +{ + int ret, retries = 0; + struct folio *folio; + pgoff_t index; + struct inode *inode = mapping->host; + + ret = ext4_emergency_state(inode->i_sb); + if (unlikely(ret)) + return ret; + + index = pos >> PAGE_SHIFT; + + if (ext4_nonda_switch(inode->i_sb) || ext4_verity_in_progress(inode)) { + *fsdata = (void *)FALL_BACK_TO_NONDELALLOC; + return ext4_write_begin(iocb, mapping, pos, + len, foliop, fsdata); + } + *fsdata = (void *)0; + trace_ext4_da_write_begin(inode, pos, len); + + if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) { + ret = ext4_generic_write_inline_data(mapping, inode, pos, len, + foliop, true); + if (ret < 0) + return ret; + if (ret == 1) { + *fsdata = (void *)((unsigned long)*fsdata | EXT4_WRITE_DATA_INLINE); + return 0; + } + } + +retry: + folio = write_begin_get_folio(iocb, mapping, index, len); + if (IS_ERR(folio)) + return PTR_ERR(folio); + + if (len > folio_next_pos(folio) - pos) + len = folio_next_pos(folio) - pos; + + ret = ext4_block_write_begin(NULL, folio, pos, len, + ext4_da_get_block_prep); + if (ret < 0) { + folio_unlock(folio); + folio_put(folio); + /* + * ext4_block_write_begin may have instantiated a few blocks + * outside i_size. Trim these off again. Don't need + * i_size_read because we hold inode lock. + */ + if (pos + len > inode->i_size) + ext4_truncate_failed_write(inode); + + if (ret == -ENOSPC && + ext4_should_retry_alloc(inode->i_sb, &retries)) + goto retry; + return ret; + } + + *foliop = folio; + return ret; +} + +/* + * Check if we should update i_disksize + * when write to the end of file but not require block allocation + */ +static int ext4_da_should_update_i_disksize(struct folio *folio, + unsigned long offset) +{ + struct buffer_head *bh; + struct inode *inode = folio->mapping->host; + unsigned int idx; + int i; + + bh = folio_buffers(folio); + idx = offset >> inode->i_blkbits; + + for (i = 0; i < idx; i++) + bh = bh->b_this_page; + + if (!buffer_mapped(bh) || (buffer_delay(bh)) || buffer_unwritten(bh)) + return 0; + return 1; +} + +static int ext4_da_do_write_end(struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct folio *folio) +{ + struct inode *inode = mapping->host; + loff_t old_size = inode->i_size; + bool disksize_changed = false; + loff_t new_i_size; + handle_t *handle; + + if (unlikely(!folio_buffers(folio))) { + folio_unlock(folio); + folio_put(folio); + return -EIO; + } + /* + * block_write_end() will mark the inode as dirty with I_DIRTY_PAGES + * flag, which all that's needed to trigger page writeback. + */ + copied = block_write_end(pos, len, copied, folio); + new_i_size = pos + copied; + + /* + * It's important to update i_size while still holding folio lock, + * because folio writeout could otherwise come in and zero beyond + * i_size. + * + * Since we are holding inode lock, we are sure i_disksize <= + * i_size. We also know that if i_disksize < i_size, there are + * delalloc writes pending in the range up to i_size. If the end of + * the current write is <= i_size, there's no need to touch + * i_disksize since writeback will push i_disksize up to i_size + * eventually. If the end of the current write is > i_size and + * inside an allocated block which ext4_da_should_update_i_disksize() + * checked, we need to update i_disksize here as certain + * ext4_writepages() paths not allocating blocks and update i_disksize. + */ + if (new_i_size > inode->i_size) { + unsigned long end; + + i_size_write(inode, new_i_size); + end = offset_in_folio(folio, new_i_size - 1); + if (copied && ext4_da_should_update_i_disksize(folio, end)) { + ext4_update_i_disksize(inode, new_i_size); + disksize_changed = true; + } + } + + folio_unlock(folio); + folio_put(folio); + + if (pos > old_size) + pagecache_isize_extended(inode, old_size, pos); + + if (!disksize_changed) + return copied; + + handle = ext4_journal_start(inode, EXT4_HT_INODE, 1); + if (IS_ERR(handle)) + return PTR_ERR(handle); + ext4_mark_inode_dirty(handle, inode); + ext4_journal_stop(handle); + + return copied; +} + +static int ext4_da_write_end(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct folio *folio, void *fsdata) +{ + struct inode *inode = mapping->host; + unsigned long write_mode = (unsigned long)fsdata; + + if (write_mode & FALL_BACK_TO_NONDELALLOC) + return ext4_write_end(iocb, mapping, pos, + len, copied, folio, fsdata); + + trace_ext4_da_write_end(inode, pos, len, copied); + + if (write_mode & EXT4_WRITE_DATA_INLINE) + return ext4_write_inline_data_end(inode, pos, len, copied, + folio); + + if (unlikely(copied < len) && !folio_test_uptodate(folio)) + copied = 0; + + return ext4_da_do_write_end(mapping, pos, len, copied, folio); +} + +/* + * Force all delayed allocation blocks to be allocated for a given inode. + */ +int ext4_alloc_da_blocks(struct inode *inode) +{ + trace_ext4_alloc_da_blocks(inode); + + if (!EXT4_I(inode)->i_reserved_data_blocks) + return 0; + + /* + * We do something simple for now. The filemap_flush() will + * also start triggering a write of the data blocks, which is + * not strictly speaking necessary. However, to do otherwise + * would require replicating code paths in: + * + * ext4_writepages() -> + * write_cache_pages() ---> (via passed in callback function) + * __mpage_da_writepage() --> + * mpage_add_bh_to_extent() + * mpage_da_map_blocks() + * + * The problem is that write_cache_pages(), located in + * mm/page-writeback.c, marks pages clean in preparation for + * doing I/O, which is not desirable if we're not planning on + * doing I/O at all. + * + * We could call write_cache_pages(), and then redirty all of + * the pages by calling redirty_page_for_writepage() but that + * would be ugly in the extreme. So instead we would need to + * replicate parts of the code in the above functions, + * simplifying them because we wouldn't actually intend to + * write out the pages, but rather only collect contiguous + * logical block extents, call the multi-block allocator, and + * then update the buffer heads with the block allocations. + * + * For now, though, we'll cheat by calling filemap_flush(), + * which will map the blocks, and start the I/O, but not + * actually wait for the I/O to complete. + */ + return filemap_flush(inode->i_mapping); +} + +/* + * bmap() is special. It gets used by applications such as lilo and by + * the swapper to find the on-disk block of a specific piece of data. + * + * Naturally, this is dangerous if the block concerned is still in the + * journal. If somebody makes a swapfile on an ext4 data-journaling + * filesystem and enables swap, then they may get a nasty shock when the + * data getting swapped to that swapfile suddenly gets overwritten by + * the original zero's written out previously to the journal and + * awaiting writeback in the kernel's buffer cache. + * + * So, if we see any bmap calls here on a modified, data-journaled file, + * take extra steps to flush any blocks which might be in the cache. + */ +static sector_t ext4_bmap(struct address_space *mapping, sector_t block) +{ + struct inode *inode = mapping->host; + sector_t ret = 0; + + inode_lock_shared(inode); + /* + * We can get here for an inline file via the FIBMAP ioctl + */ + if (ext4_has_inline_data(inode)) + goto out; + + if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) && + (test_opt(inode->i_sb, DELALLOC) || + ext4_should_journal_data(inode))) { + /* + * With delalloc or journalled data we want to sync the file so + * that we can make sure we allocate blocks for file and data + * is in place for the user to see it + */ + filemap_write_and_wait(mapping); + } + + ret = iomap_bmap(mapping, block, &ext4_iomap_ops); + +out: + inode_unlock_shared(inode); + return ret; +} + +static void ext4_invalidate_folio(struct folio *folio, size_t offset, + size_t length) +{ + trace_ext4_invalidate_folio(folio, offset, length); + + /* No journalling happens on data buffers when this function is used */ + WARN_ON(folio_buffers(folio) && buffer_jbd(folio_buffers(folio))); + + block_invalidate_folio(folio, offset, length); +} + +static int __ext4_journalled_invalidate_folio(struct folio *folio, + size_t offset, size_t length) +{ + journal_t *journal = EXT4_JOURNAL(folio->mapping->host); + + trace_ext4_journalled_invalidate_folio(folio, offset, length); + + /* + * If it's a full truncate we just forget about the pending dirtying + */ + if (offset == 0 && length == folio_size(folio)) + folio_clear_checked(folio); + + return jbd2_journal_invalidate_folio(journal, folio, offset, length); +} + +/* Wrapper for aops... */ +static void ext4_journalled_invalidate_folio(struct folio *folio, + size_t offset, + size_t length) +{ + WARN_ON(__ext4_journalled_invalidate_folio(folio, offset, length) < 0); +} + +static bool ext4_release_folio(struct folio *folio, gfp_t wait) +{ + struct inode *inode = folio->mapping->host; + journal_t *journal = EXT4_JOURNAL(inode); + + trace_ext4_release_folio(inode, folio); + + /* Page has dirty journalled data -> cannot release */ + if (folio_test_checked(folio)) + return false; + if (journal) + return jbd2_journal_try_to_free_buffers(journal, folio); + else + return try_to_free_buffers(folio); +} + +static bool ext4_inode_datasync_dirty(struct inode *inode) +{ + journal_t *journal = EXT4_SB(inode->i_sb)->s_journal; + struct mapping_metadata_bhs *mmb; + + if (journal) { + if (jbd2_transaction_committed(journal, + EXT4_I(inode)->i_datasync_tid)) + return false; + if (test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT)) + return !list_empty(&EXT4_I(inode)->i_fc_list); + return true; + } + + mmb = ext4_i_metadata_bhs(inode); + /* Any metadata buffers to write? */ + if (mmb && mmb_has_buffers(mmb)) + return true; + return inode_state_read_once(inode) & I_DIRTY_DATASYNC; +} + +static void ext4_set_iomap(struct inode *inode, struct iomap *iomap, + struct ext4_map_blocks *map, loff_t offset, + loff_t length, unsigned int flags) +{ + u8 blkbits = inode->i_blkbits; + + /* + * Writes that span EOF might trigger an I/O size update on completion, + * so consider them to be dirty for the purpose of O_DSYNC, even if + * there is no other metadata changes being made or are pending. + */ + iomap->flags = 0; + if (ext4_inode_datasync_dirty(inode) || + offset + length > i_size_read(inode)) + iomap->flags |= IOMAP_F_DIRTY; + + if (map->m_flags & EXT4_MAP_NEW) + iomap->flags |= IOMAP_F_NEW; + + /* HW-offload atomics are always used */ + if (flags & IOMAP_ATOMIC) + iomap->flags |= IOMAP_F_ATOMIC_BIO; + + if (flags & IOMAP_DAX) + iomap->dax_dev = EXT4_SB(inode->i_sb)->s_daxdev; + else + iomap->bdev = inode->i_sb->s_bdev; + iomap->offset = EXT4_LBLK_TO_B(inode, map->m_lblk); + iomap->length = EXT4_LBLK_TO_B(inode, map->m_len); + + if ((map->m_flags & EXT4_MAP_MAPPED) && + !ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) + iomap->flags |= IOMAP_F_MERGED; + + /* + * Flags passed to ext4_map_blocks() for direct I/O writes can result + * in m_flags having both EXT4_MAP_MAPPED and EXT4_MAP_UNWRITTEN bits + * set. In order for any allocated unwritten extents to be converted + * into written extents correctly within the ->end_io() handler, we + * need to ensure that the iomap->type is set appropriately. Hence, the + * reason why we need to check whether the EXT4_MAP_UNWRITTEN bit has + * been set first. + */ + if (map->m_flags & EXT4_MAP_UNWRITTEN) { + iomap->type = IOMAP_UNWRITTEN; + iomap->addr = (u64) map->m_pblk << blkbits; + if (flags & IOMAP_DAX) + iomap->addr += EXT4_SB(inode->i_sb)->s_dax_part_off; + } else if (map->m_flags & EXT4_MAP_MAPPED) { + iomap->type = IOMAP_MAPPED; + iomap->addr = (u64) map->m_pblk << blkbits; + if (flags & IOMAP_DAX) + iomap->addr += EXT4_SB(inode->i_sb)->s_dax_part_off; + } else if (map->m_flags & EXT4_MAP_DELAYED) { + iomap->type = IOMAP_DELALLOC; + iomap->addr = IOMAP_NULL_ADDR; + } else { + iomap->type = IOMAP_HOLE; + iomap->addr = IOMAP_NULL_ADDR; + } +} + +static int ext4_map_blocks_atomic_write_slow(handle_t *handle, + struct inode *inode, struct ext4_map_blocks *map) +{ + ext4_lblk_t m_lblk = map->m_lblk; + unsigned int m_len = map->m_len; + unsigned int mapped_len = 0, m_flags = 0; + ext4_fsblk_t next_pblk = 0; + bool check_next_pblk = false; + int ret = 0; + + WARN_ON_ONCE(!ext4_has_feature_bigalloc(inode->i_sb)); + + /* + * This is a slow path in case of mixed mapping. We use + * EXT4_GET_BLOCKS_CREATE_ZERO flag here to make sure we get a single + * contiguous mapped mapping. This will ensure any unwritten or hole + * regions within the requested range is zeroed out and we return + * a single contiguous mapped extent. + */ + m_flags = EXT4_GET_BLOCKS_CREATE_ZERO; + + do { + ret = ext4_map_blocks(handle, inode, map, m_flags); + if (ret < 0 && ret != -ENOSPC) + goto out_err; + /* + * This should never happen, but let's return an error code to + * avoid an infinite loop in here. + */ + if (ret == 0) { + ret = -EFSCORRUPTED; + ext4_warning_inode(inode, + "ext4_map_blocks() couldn't allocate blocks m_flags: 0x%x, ret:%d", + m_flags, ret); + goto out_err; + } + /* + * With bigalloc we should never get ENOSPC nor discontiguous + * physical extents. + */ + if ((check_next_pblk && next_pblk != map->m_pblk) || + ret == -ENOSPC) { + ext4_warning_inode(inode, + "Non-contiguous allocation detected: expected %llu, got %llu, " + "or ext4_map_blocks() returned out of space ret: %d", + next_pblk, map->m_pblk, ret); + ret = -EFSCORRUPTED; + goto out_err; + } + next_pblk = map->m_pblk + map->m_len; + check_next_pblk = true; + + mapped_len += map->m_len; + map->m_lblk += map->m_len; + map->m_len = m_len - mapped_len; + } while (mapped_len < m_len); + + /* + * We might have done some work in above loop, so we need to query the + * start of the physical extent, based on the origin m_lblk and m_len. + * Let's also ensure we were able to allocate the required range for + * mixed mapping case. + */ + map->m_lblk = m_lblk; + map->m_len = m_len; + map->m_flags = 0; + + ret = ext4_map_blocks(handle, inode, map, + EXT4_GET_BLOCKS_QUERY_LAST_IN_LEAF); + if (ret != m_len) { + ext4_warning_inode(inode, + "allocation failed for atomic write request m_lblk:%u, m_len:%u, ret:%d\n", + m_lblk, m_len, ret); + ret = -EINVAL; + } + return ret; + +out_err: + /* reset map before returning an error */ + map->m_lblk = m_lblk; + map->m_len = m_len; + map->m_flags = 0; + return ret; +} + +/* + * ext4_map_blocks_atomic: Helper routine to ensure the entire requested + * range in @map [lblk, lblk + len) is one single contiguous extent with no + * mixed mappings. + * + * We first use m_flags passed to us by our caller (ext4_iomap_alloc()). + * We only call EXT4_GET_BLOCKS_ZERO in the slow path, when the underlying + * physical extent for the requested range does not have a single contiguous + * mapping type i.e. (Hole, Mapped, or Unwritten) throughout. + * In that case we will loop over the requested range to allocate and zero out + * the unwritten / holes in between, to get a single mapped extent from + * [m_lblk, m_lblk + m_len). Note that this is only possible because we know + * this can be called only with bigalloc enabled filesystem where the underlying + * cluster is already allocated. This avoids allocating discontiguous extents + * in the slow path due to multiple calls to ext4_map_blocks(). + * The slow path is mostly non-performance critical path, so it should be ok to + * loop using ext4_map_blocks() with appropriate flags to allocate & zero the + * underlying short holes/unwritten extents within the requested range. + */ +static int ext4_map_blocks_atomic_write(handle_t *handle, struct inode *inode, + struct ext4_map_blocks *map, int m_flags, + bool *force_commit) +{ + ext4_lblk_t m_lblk = map->m_lblk; + unsigned int m_len = map->m_len; + int ret = 0; + + WARN_ON_ONCE(m_len > 1 && !ext4_has_feature_bigalloc(inode->i_sb)); + + ret = ext4_map_blocks(handle, inode, map, m_flags); + if (ret < 0 || ret == m_len) + goto out; + /* + * This is a mixed mapping case where we were not able to allocate + * a single contiguous extent. In that case let's reset requested + * mapping and call the slow path. + */ + map->m_lblk = m_lblk; + map->m_len = m_len; + map->m_flags = 0; + + /* + * slow path means we have mixed mapping, that means we will need + * to force txn commit. + */ + *force_commit = true; + return ext4_map_blocks_atomic_write_slow(handle, inode, map); +out: + return ret; +} + +static int ext4_iomap_alloc(struct inode *inode, struct ext4_map_blocks *map, + unsigned int flags) +{ + handle_t *handle; + int ret, dio_credits, m_flags = 0, retries = 0; + bool force_commit = false; + + if (flags & IOMAP_NOWAIT) + return -EAGAIN; + + /* + * Trim the mapping request to the maximum value that we can map at + * once for direct I/O. + */ + if (map->m_len > DIO_MAX_BLOCKS) + map->m_len = DIO_MAX_BLOCKS; + + /* + * journal credits estimation for atomic writes. We call + * ext4_map_blocks(), to find if there could be a mixed mapping. If yes, + * then let's assume the no. of pextents required can be m_len i.e. + * every alternate block can be unwritten and hole. + */ + if (flags & IOMAP_ATOMIC) { + unsigned int orig_mlen = map->m_len; + + ret = ext4_map_blocks(NULL, inode, map, 0); + if (ret < 0) + return ret; + if (map->m_len < orig_mlen) { + map->m_len = orig_mlen; + dio_credits = ext4_meta_trans_blocks(inode, map->m_len, + map->m_len, 0); + } else { + dio_credits = ext4_chunk_trans_blocks(inode, + map->m_len); + } + } else { + dio_credits = ext4_chunk_trans_blocks(inode, map->m_len); + } + +retry: + /* + * Either we allocate blocks and then don't get an unwritten extent, so + * in that case we have reserved enough credits. Or, the blocks are + * already allocated and unwritten. In that case, the extent conversion + * fits into the credits as well. + */ + handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, dio_credits); + if (IS_ERR(handle)) + return PTR_ERR(handle); + + /* + * DAX and direct I/O are the only two operations that are currently + * supported with IOMAP_WRITE. + */ + WARN_ON(!(flags & (IOMAP_DAX | IOMAP_DIRECT))); + if (flags & IOMAP_DAX) + m_flags = EXT4_GET_BLOCKS_CREATE_ZERO; + /* + * We use i_size instead of i_disksize here because delalloc writeback + * can complete at any point during the I/O and subsequently push the + * i_disksize out to i_size. This could be beyond where direct I/O is + * happening and thus expose allocated blocks to direct I/O reads. + */ + else if (EXT4_LBLK_TO_B(inode, map->m_lblk) >= i_size_read(inode)) + m_flags = EXT4_GET_BLOCKS_CREATE; + else if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) + m_flags = EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT; + + if (flags & IOMAP_ATOMIC) + ret = ext4_map_blocks_atomic_write(handle, inode, map, m_flags, + &force_commit); + else + ret = ext4_map_blocks(handle, inode, map, m_flags); + + /* + * We cannot fill holes in indirect tree based inodes as that could + * expose stale data in the case of a crash. Use the magic error code + * to fallback to buffered I/O. + */ + if (!m_flags && !ret) + ret = -ENOTBLK; + + ext4_journal_stop(handle); + if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries)) + goto retry; + + /* + * Force commit the current transaction if the allocation spans a mixed + * mapping range. This ensures any pending metadata updates (like + * unwritten to written extents conversion) in this range are in + * consistent state with the file data blocks, before performing the + * actual write I/O. If the commit fails, the whole I/O must be aborted + * to prevent any possible torn writes. + */ + if (ret > 0 && force_commit) { + int ret2; + + ret2 = ext4_force_commit(inode->i_sb); + if (ret2) + return ret2; + } + + return ret; +} + + +int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length, + unsigned flags, struct iomap *iomap, struct iomap *srcmap) +{ + int ret; + struct ext4_map_blocks map; + u8 blkbits = inode->i_blkbits; + unsigned int orig_mlen; + + if ((offset >> blkbits) > EXT4_MAX_LOGICAL_BLOCK) + return -EINVAL; + + if (WARN_ON_ONCE(ext4_has_inline_data(inode))) + return -ERANGE; + + /* + * Calculate the first and last logical blocks respectively. + */ + map.m_lblk = offset >> blkbits; + map.m_len = min_t(loff_t, (offset + length - 1) >> blkbits, + EXT4_MAX_LOGICAL_BLOCK) - map.m_lblk + 1; + orig_mlen = map.m_len; + + if (flags & IOMAP_WRITE) { + /* + * We check here if the blocks are already allocated, then we + * don't need to start a journal txn and we can directly return + * the mapping information. This could boost performance + * especially in multi-threaded overwrite requests. + */ + if (offset + length <= i_size_read(inode)) { + ret = ext4_map_blocks(NULL, inode, &map, 0); + /* + * For DAX we convert extents to initialized ones before + * copying the data, otherwise we do it after I/O so + * there's no need to call into ext4_iomap_alloc(). + */ + if ((map.m_flags & EXT4_MAP_MAPPED) || + (!(flags & IOMAP_DAX) && + (map.m_flags & EXT4_MAP_UNWRITTEN))) { + /* + * For atomic writes the entire requested + * length should be mapped. + */ + if (ret == orig_mlen || + (!(flags & IOMAP_ATOMIC) && ret > 0)) + goto out; + } + map.m_len = orig_mlen; + } + ret = ext4_iomap_alloc(inode, &map, flags); + } else { + ret = ext4_map_blocks(NULL, inode, &map, 0); + } + + if (ret < 0) + return ret; +out: + /* + * Sometimes I/O to an encrypted file has to be broken up to guarantee + * DUN contiguity. Handle this by limiting the length of the mapping + * returned. + */ + map.m_len = fscrypt_limit_io_blocks(inode, map.m_lblk, map.m_len); + + /* + * Before returning to iomap, let's ensure the allocated mapping + * covers the entire requested length for atomic writes. + */ + if (flags & IOMAP_ATOMIC) { + if (map.m_len < (length >> blkbits)) { + WARN_ON_ONCE(1); + return -EINVAL; + } + } + ext4_set_iomap(inode, iomap, &map, offset, length, flags); + + return 0; +} + +static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_next, ext4_iomap_begin); + +const struct iomap_ops ext4_iomap_ops = { + .iomap_next = ext4_iomap_next, +}; + +static int ext4_iomap_begin_report(struct inode *inode, loff_t offset, + loff_t length, unsigned int flags, + struct iomap *iomap, struct iomap *srcmap) +{ + int ret; + struct ext4_map_blocks map; + u8 blkbits = inode->i_blkbits; + + if ((offset >> blkbits) > EXT4_MAX_LOGICAL_BLOCK) + return -EINVAL; + + if (ext4_has_inline_data(inode)) { + ret = ext4_inline_data_iomap(inode, iomap); + if (ret != -EAGAIN) { + if (ret == 0 && offset >= iomap->length) + ret = -ENOENT; + return ret; + } + } + + /* + * Calculate the first and last logical block respectively. + */ + map.m_lblk = offset >> blkbits; + map.m_len = min_t(loff_t, (offset + length - 1) >> blkbits, + EXT4_MAX_LOGICAL_BLOCK) - map.m_lblk + 1; + + /* + * Fiemap callers may call for offset beyond s_bitmap_maxbytes. + * So handle it here itself instead of querying ext4_map_blocks(). + * Since ext4_map_blocks() will warn about it and will return + * -EIO error. + */ + if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) { + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + + if (offset >= sbi->s_bitmap_maxbytes) { + map.m_flags = 0; + goto set_iomap; + } + } + + ret = ext4_map_blocks(NULL, inode, &map, 0); + if (ret < 0) + return ret; +set_iomap: + ext4_set_iomap(inode, iomap, &map, offset, length, flags); + + return 0; +} + +static DEFINE_IOMAP_ITER_NEXT(ext4_iomap_next_report, ext4_iomap_begin_report); + +const struct iomap_ops ext4_iomap_report_ops = { + .iomap_next = ext4_iomap_next_report, +}; + +/* + * For data=journal mode, folio should be marked dirty only when it was + * writeably mapped. When that happens, it was already attached to the + * transaction and marked as jbddirty (we take care of this in + * ext4_page_mkwrite()). On transaction commit, we writeprotect page mappings + * so we should have nothing to do here, except for the case when someone + * had the page pinned and dirtied the page through this pin (e.g. by doing + * direct IO to it). In that case we'd need to attach buffers here to the + * transaction but we cannot due to lock ordering. We cannot just dirty the + * folio and leave attached buffers clean, because the buffers' dirty state is + * "definitive". We cannot just set the buffers dirty or jbddirty because all + * the journalling code will explode. So what we do is to mark the folio + * "pending dirty" and next time ext4_writepages() is called, attach buffers + * to the transaction appropriately. + */ +static bool ext4_journalled_dirty_folio(struct address_space *mapping, + struct folio *folio) +{ + WARN_ON_ONCE(!folio_buffers(folio)); + if (folio_maybe_dma_pinned(folio)) + folio_set_checked(folio); + return filemap_dirty_folio(mapping, folio); +} + +static bool ext4_dirty_folio(struct address_space *mapping, struct folio *folio) +{ + WARN_ON_ONCE(!folio_test_locked(folio) && !folio_test_dirty(folio)); + WARN_ON_ONCE(!folio_buffers(folio)); + return block_dirty_folio(mapping, folio); +} + +static int ext4_iomap_swap_activate(struct swap_info_struct *sis, + struct file *file, sector_t *span) +{ + return iomap_swapfile_activate(sis, file, span, + &ext4_iomap_report_ops); +} + +static const struct address_space_operations ext4_aops = { + .read_folio = ext4_read_folio, + .readahead = ext4_readahead, + .writepages = ext4_writepages, + .write_begin = ext4_write_begin, + .write_end = ext4_write_end, + .dirty_folio = ext4_dirty_folio, + .bmap = ext4_bmap, + .invalidate_folio = ext4_invalidate_folio, + .release_folio = ext4_release_folio, + .migrate_folio = buffer_migrate_folio, + .is_partially_uptodate = block_is_partially_uptodate, + .error_remove_folio = generic_error_remove_folio, + .swap_activate = ext4_iomap_swap_activate, +}; + +static const struct address_space_operations ext4_journalled_aops = { + .read_folio = ext4_read_folio, + .readahead = ext4_readahead, + .writepages = ext4_writepages, + .write_begin = ext4_write_begin, + .write_end = ext4_journalled_write_end, + .dirty_folio = ext4_journalled_dirty_folio, + .bmap = ext4_bmap, + .invalidate_folio = ext4_journalled_invalidate_folio, + .release_folio = ext4_release_folio, + .migrate_folio = buffer_migrate_folio_norefs, + .is_partially_uptodate = block_is_partially_uptodate, + .error_remove_folio = generic_error_remove_folio, + .swap_activate = ext4_iomap_swap_activate, +}; + +static const struct address_space_operations ext4_da_aops = { + .read_folio = ext4_read_folio, + .readahead = ext4_readahead, + .writepages = ext4_writepages, + .write_begin = ext4_da_write_begin, + .write_end = ext4_da_write_end, + .dirty_folio = ext4_dirty_folio, + .bmap = ext4_bmap, + .invalidate_folio = ext4_invalidate_folio, + .release_folio = ext4_release_folio, + .migrate_folio = buffer_migrate_folio, + .is_partially_uptodate = block_is_partially_uptodate, + .error_remove_folio = generic_error_remove_folio, + .swap_activate = ext4_iomap_swap_activate, +}; + +static const struct address_space_operations ext4_dax_aops = { + .writepages = ext4_dax_writepages, + .dirty_folio = noop_dirty_folio, + .bmap = ext4_bmap, + .swap_activate = ext4_iomap_swap_activate, +}; + +void ext4_set_aops(struct inode *inode) +{ + switch (ext4_inode_journal_mode(inode)) { + case EXT4_INODE_ORDERED_DATA_MODE: + case EXT4_INODE_WRITEBACK_DATA_MODE: + break; + case EXT4_INODE_JOURNAL_DATA_MODE: + inode->i_mapping->a_ops = &ext4_journalled_aops; + return; + default: + BUG(); + } + if (IS_DAX(inode)) + inode->i_mapping->a_ops = &ext4_dax_aops; + else if (test_opt(inode->i_sb, DELALLOC)) + inode->i_mapping->a_ops = &ext4_da_aops; + else + inode->i_mapping->a_ops = &ext4_aops; +} + +/* + * Here we can't skip an unwritten buffer even though it usually reads zero + * because it might have data in pagecache (eg, if called from ext4_zero_range, + * ext4_punch_hole, etc) which needs to be properly zeroed out. Otherwise a + * racing writeback can come later and flush the stale pagecache to disk. + * + * Return the loaded bh if it actually needs zeroing - in written, dirty + * unwritten, or delalloc state. Return NULL if it's clean (i.e., a hole or + * a clean unwritten block). + */ +static struct buffer_head *ext4_load_tail_bh(struct inode *inode, loff_t from) +{ + unsigned int offset, blocksize, pos; + ext4_lblk_t iblock; + struct address_space *mapping = inode->i_mapping; + struct buffer_head *bh; + struct folio *folio; + int err = 0; + + folio = __filemap_get_folio(mapping, from >> PAGE_SHIFT, + FGP_WRITEBEGIN | FGP_ACCESSED, + mapping_gfp_constraint(mapping, ~__GFP_FS)); + if (IS_ERR(folio)) + return ERR_CAST(folio); + + blocksize = inode->i_sb->s_blocksize; + + iblock = EXT4_PG_TO_LBLK(inode, folio->index); + + bh = folio_buffers(folio); + if (!bh) + bh = create_empty_buffers(folio, blocksize, 0); + + /* Find the buffer that contains "offset" */ + offset = offset_in_folio(folio, from); + pos = blocksize; + while (offset >= pos) { + bh = bh->b_this_page; + iblock++; + pos += blocksize; + } + if (buffer_freed(bh)) { + BUFFER_TRACE(bh, "freed: skip"); + goto unlock; + } + if (!buffer_mapped(bh)) { + BUFFER_TRACE(bh, "unmapped"); + err = ext4_get_block(inode, iblock, bh, 0); + if (err < 0) + goto unlock; + /* + * It's a hole or a clean unwritten block - nothing to do. + * Note that a lookup-only get_block (without + * EXT4_GET_BLOCKS_CREATE) never sets BH_Mapped for clean + * unwritten extents. + */ + if (!buffer_mapped(bh)) { + BUFFER_TRACE(bh, "still unmapped"); + goto unlock; + } + } + + /* Ok, it's mapped. Make sure it's up-to-date */ + if (folio_test_uptodate(folio)) + set_buffer_uptodate(bh); + + if (!buffer_uptodate(bh)) { + err = ext4_read_bh_lock(bh, 0, true); + if (err) + goto unlock; + } + return bh; + +unlock: + folio_unlock(folio); + folio_put(folio); + return err ? ERR_PTR(err) : NULL; +} + +static int ext4_block_do_zero_range(struct inode *inode, loff_t from, + loff_t length, bool *did_zero, + bool *zero_written) +{ + struct buffer_head *bh; + struct folio *folio; + + bh = ext4_load_tail_bh(inode, from); + if (IS_ERR_OR_NULL(bh)) + return PTR_ERR_OR_ZERO(bh); + + folio = bh->b_folio; + folio_zero_range(folio, offset_in_folio(folio, from), length); + BUFFER_TRACE(bh, "zeroed end of block"); + + mark_buffer_dirty(bh); + if (did_zero) + *did_zero = true; + if (zero_written && !buffer_unwritten(bh) && !buffer_delay(bh)) + *zero_written = true; + + folio_unlock(folio); + folio_put(folio); + return 0; +} + +static int ext4_block_journalled_zero_range(struct inode *inode, loff_t from, + loff_t length, bool *did_zero) +{ + struct buffer_head *bh; + struct folio *folio; + handle_t *handle; + int err; + + handle = ext4_journal_start(inode, EXT4_HT_MISC, 1); + if (IS_ERR(handle)) + return PTR_ERR(handle); + + bh = ext4_load_tail_bh(inode, from); + if (IS_ERR_OR_NULL(bh)) { + err = PTR_ERR_OR_ZERO(bh); + goto out_handle; + } + folio = bh->b_folio; + + BUFFER_TRACE(bh, "get write access"); + err = ext4_journal_get_write_access(handle, inode->i_sb, bh, + EXT4_JTR_NONE); + if (err) + goto out; + + folio_zero_range(folio, offset_in_folio(folio, from), length); + BUFFER_TRACE(bh, "zeroed end of block"); + + err = ext4_dirty_journalled_data(handle, bh); + if (err) + goto out; + + if (did_zero) + *did_zero = true; +out: + folio_unlock(folio); + folio_put(folio); +out_handle: + ext4_journal_stop(handle); + return err; +} + +/* + * Zeros out a mapping of length 'length' starting from file offset + * 'from'. The range to be zero'd must be contained with in one block. + * If the specified range exceeds the end of the block it will be + * shortened to end of the block that corresponds to 'from'. + */ +static int ext4_block_zero_range(struct inode *inode, + loff_t from, loff_t length, bool *did_zero, + bool *zero_written) +{ + unsigned blocksize = inode->i_sb->s_blocksize; + unsigned int max = blocksize - (from & (blocksize - 1)); + + /* + * correct length if it does not fall between + * 'from' and the end of the block + */ + if (length > max || length < 0) + length = max; + + if (IS_DAX(inode)) { + return dax_zero_range(inode, from, length, did_zero, + &ext4_iomap_ops); + } else if (ext4_should_journal_data(inode)) { + return ext4_block_journalled_zero_range(inode, from, length, + did_zero); + } + return ext4_block_do_zero_range(inode, from, length, did_zero, + zero_written); +} + +/* + * Zero out a mapping from file offset 'from' up to the end of the block + * which corresponds to 'from' or to the given 'end' inside this block. + * This required during truncate up and performing append writes. We need + * to physically zero the tail end of that block so it doesn't yield old + * data if the file is grown. + */ +int ext4_block_zero_eof(struct inode *inode, loff_t from, loff_t end) +{ + unsigned int blocksize = i_blocksize(inode); + unsigned int offset; + loff_t length = end - from; + bool did_zero = false; + bool zero_written = false; + int err; + + offset = from & (blocksize - 1); + if (!offset || from >= end) + return 0; + /* + * Inline data has no tail block to zero out. Note that a race with + * ext4_page_mkwrite() converting inline data to an extent without + * holding i_rwsem is safe, as that path zeroes the full block before + * copying in the inline data. + */ + if (ext4_has_inline_data(inode)) + return 0; + /* If we are processing an encrypted inode during orphan list handling */ + if (IS_ENCRYPTED(inode) && !fscrypt_has_encryption_key(inode)) + return 0; + + if (length > blocksize - offset) + length = blocksize - offset; + + err = ext4_block_zero_range(inode, from, length, + &did_zero, &zero_written); + if (err) + return err; + /* + * It's necessary to order zeroed data before update i_disksize when + * truncating up or performing an append write, because there might be + * exposing stale on-disk data which may caused by concurrent post-EOF + * mmap write during folio writeback. + */ + if (ext4_should_order_data(inode) && + did_zero && zero_written && !IS_DAX(inode)) { + handle_t *handle; + + handle = ext4_journal_start(inode, EXT4_HT_MISC, 1); + if (IS_ERR(handle)) + return PTR_ERR(handle); + + err = ext4_jbd2_inode_add_write(handle, inode, from, length); + ext4_journal_stop(handle); + if (err) + return err; + } + + return 0; +} + +/* + * Zero out the unaligned head and tail of the [lstart, lstart+length) + * range. + * + * On return, @partial_zeroed records which edges actually got + * partial-zeroed. Set EXT4_PARTIAL_ZERO_START/EXT4_PARTIAL_ZERO_END if + * the head/tail block got actually partially zeroed (in written, dirty + * unwritten or delalloc state). Cleared if the head/tail block is a + * hole or a clean unwritten block, in which case there is nothing that + * needs zeroing. When the head and tail land in the same block, both + * bits are set together on a successful zeroing. + */ +int ext4_zero_partial_blocks(struct inode *inode, loff_t lstart, loff_t length, + unsigned int *partial_zeroed) +{ + struct super_block *sb = inode->i_sb; + unsigned partial_start, partial_end; + ext4_fsblk_t start, end; + loff_t byte_end = (lstart + length - 1); + bool did_zero = false; + int err = 0; + + partial_start = lstart & (sb->s_blocksize - 1); + partial_end = byte_end & (sb->s_blocksize - 1); + + start = lstart >> sb->s_blocksize_bits; + end = byte_end >> sb->s_blocksize_bits; + + /* Handle partial zero within the single block */ + if (start == end && + (partial_start || (partial_end != sb->s_blocksize - 1))) { + err = ext4_block_zero_range(inode, lstart, length, &did_zero, + NULL); + if (did_zero) + *partial_zeroed |= (EXT4_PARTIAL_ZERO_START | + EXT4_PARTIAL_ZERO_END); + return err; + } + /* Handle partial zero out on the start of the range */ + if (partial_start) { + err = ext4_block_zero_range(inode, lstart, sb->s_blocksize, + &did_zero, NULL); + if (err) + return err; + if (did_zero) + *partial_zeroed |= EXT4_PARTIAL_ZERO_START; + } + /* Handle partial zero out on the end of the range */ + if (partial_end != sb->s_blocksize - 1) { + did_zero = false; + err = ext4_block_zero_range(inode, byte_end - partial_end, + partial_end + 1, &did_zero, NULL); + if (err) + return err; + if (did_zero) + *partial_zeroed |= EXT4_PARTIAL_ZERO_END; + } + return err; +} + +int ext4_can_truncate(struct inode *inode) +{ + if (S_ISREG(inode->i_mode)) + return 1; + if (S_ISDIR(inode->i_mode)) + return 1; + if (S_ISLNK(inode->i_mode)) + return !ext4_inode_is_fast_symlink(inode); + return 0; +} + +/* + * We have to make sure i_disksize gets properly updated before we truncate + * page cache due to hole punching or zero range. Otherwise i_disksize update + * can get lost as it may have been postponed to submission of writeback but + * that will never happen if we remove the folio containing i_size from the + * page cache. Also if we punch hole within i_size but above i_disksize, + * following ext4_page_mkwrite() may mistakenly allocate written blocks over + * the hole and thus introduce allocated blocks beyond i_disksize which is + * not allowed (e2fsck would complain in case of crash). + */ +int ext4_update_disksize_before_punch(struct inode *inode, loff_t offset, + loff_t len) +{ + handle_t *handle; + int ret; + + loff_t size = i_size_read(inode); + + WARN_ON(!inode_is_locked(inode)); + if (offset > size) + return 0; + + if (offset + len < size) + size = offset + len; + if (EXT4_I(inode)->i_disksize >= size) + return 0; + + handle = ext4_journal_start(inode, EXT4_HT_MISC, 1); + if (IS_ERR(handle)) + return PTR_ERR(handle); + ext4_update_i_disksize(inode, size); + ret = ext4_mark_inode_dirty(handle, inode); + ext4_journal_stop(handle); + + return ret; +} + +static inline void ext4_truncate_folio(struct inode *inode, + loff_t start, loff_t end) +{ + unsigned long blocksize = i_blocksize(inode); + struct folio *folio; + + /* Nothing to be done if no complete block needs to be truncated. */ + if (round_up(start, blocksize) >= round_down(end, blocksize)) + return; + + folio = filemap_lock_folio(inode->i_mapping, start >> PAGE_SHIFT); + if (IS_ERR(folio)) + return; + + if (folio_mkclean(folio)) + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); +} + +int ext4_truncate_page_cache_block_range(struct inode *inode, + loff_t start, loff_t end) +{ + unsigned long blocksize = i_blocksize(inode); + int ret; + + /* + * For journalled data we need to write (and checkpoint) pages + * before discarding page cache to avoid inconsitent data on disk + * in case of crash before freeing or unwritten converting trans + * is committed. + */ + if (ext4_should_journal_data(inode)) { + ret = filemap_write_and_wait_range(inode->i_mapping, start, + end - 1); + if (ret) + return ret; + goto truncate_pagecache; + } + + /* + * If the block size is less than the page size, the file's mapped + * blocks within one page could be freed or converted to unwritten. + * So it's necessary to remove writable userspace mappings, and then + * ext4_page_mkwrite() can be called during subsequent write access + * to these partial folios. + */ + if (!IS_ALIGNED(start | end, PAGE_SIZE) && + blocksize < PAGE_SIZE && start < inode->i_size) { + loff_t page_boundary = round_up(start, PAGE_SIZE); + + ext4_truncate_folio(inode, start, min(page_boundary, end)); + if (end > page_boundary) + ext4_truncate_folio(inode, + round_down(end, PAGE_SIZE), end); + } + +truncate_pagecache: + truncate_pagecache_range(inode, start, end - 1); + return 0; +} + +static void ext4_wait_dax_page(struct inode *inode) +{ + filemap_invalidate_unlock(inode->i_mapping); + schedule(); + filemap_invalidate_lock(inode->i_mapping); +} + +int ext4_break_layouts(struct inode *inode) +{ + if (WARN_ON_ONCE(!rwsem_is_locked(&inode->i_mapping->invalidate_lock))) + return -EINVAL; + + return dax_break_layout_inode(inode, ext4_wait_dax_page); +} + +/* + * ext4_punch_hole: punches a hole in a file by releasing the blocks + * associated with the given offset and length + * + * @inode: File inode + * @offset: The offset where the hole will begin + * @len: The length of the hole + * + * Returns: 0 on success or negative on failure + */ + +int ext4_punch_hole(struct file *file, loff_t offset, loff_t length) +{ + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + ext4_lblk_t start_lblk, end_lblk; + loff_t max_end = sb->s_maxbytes; + loff_t end = offset + length; + handle_t *handle; + unsigned int credits; + unsigned int partial_zeroed = 0; + int ret; + + trace_ext4_punch_hole(inode, offset, length, 0); + WARN_ON_ONCE(!inode_is_locked(inode)); + + /* + * For indirect-block based inodes, make sure that the hole within + * one block before last range. + */ + if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) + max_end = EXT4_SB(sb)->s_bitmap_maxbytes - sb->s_blocksize; + + /* No need to punch hole beyond i_size */ + if (offset >= inode->i_size || offset >= max_end) + return 0; + + /* + * If the hole extends beyond i_size, set the hole to end after + * the block that contains i_size to save pointless tail block zeroing. + */ + if (end >= inode->i_size) + end = round_up(inode->i_size, sb->s_blocksize); + if (end > max_end) + end = max_end; + length = end - offset; + + ret = ext4_update_disksize_before_punch(inode, offset, length); + if (ret) + return ret; + + /* Now release the pages and zero block aligned part of pages*/ + ret = ext4_truncate_page_cache_block_range(inode, offset, end); + if (ret) + return ret; + + ret = ext4_zero_partial_blocks(inode, offset, length, &partial_zeroed); + if (ret) + return ret; + if (((file->f_flags & O_SYNC) || IS_SYNC(inode)) && partial_zeroed) { + ret = filemap_write_and_wait_range(inode->i_mapping, offset, + end - 1); + if (ret) + return ret; + } + + if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) + credits = ext4_chunk_trans_extent(inode, 0); + else + credits = ext4_blocks_for_truncate(inode); + handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + ext4_std_error(sb, ret); + return ret; + } + + /* If there are blocks to remove, do it */ + start_lblk = EXT4_B_TO_LBLK(inode, offset); + end_lblk = end >> inode->i_blkbits; + + if (end_lblk > start_lblk) { + ext4_lblk_t hole_len = end_lblk - start_lblk; + + ext4_fc_track_inode(handle, inode); + ext4_check_map_extents_env(inode); + down_write(&EXT4_I(inode)->i_data_sem); + ext4_discard_preallocations(inode); + + ext4_es_remove_extent(inode, start_lblk, hole_len); + + if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) + ret = ext4_ext_remove_space(inode, start_lblk, + end_lblk - 1); + else + ret = ext4_ind_remove_space(handle, inode, start_lblk, + end_lblk); + if (ret) { + up_write(&EXT4_I(inode)->i_data_sem); + goto out_handle; + } + + ext4_es_insert_extent(inode, start_lblk, hole_len, ~0, + EXTENT_STATUS_HOLE, 0); + up_write(&EXT4_I(inode)->i_data_sem); + } + ext4_fc_track_range(handle, inode, start_lblk, end_lblk); + + ret = ext4_mark_inode_dirty(handle, inode); + if (unlikely(ret)) + goto out_handle; + + ext4_update_inode_fsync_trans(handle, inode, 1); + if ((file->f_flags & O_SYNC) || IS_SYNC(inode)) + ext4_handle_sync(handle); +out_handle: + ext4_journal_stop(handle); + return ret; +} + +int ext4_inode_attach_jinode(struct inode *inode) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + struct jbd2_inode *jinode; + + if (ei->jinode || !EXT4_SB(inode->i_sb)->s_journal) + return 0; + + jinode = jbd2_alloc_inode(GFP_KERNEL); + spin_lock(&inode->i_lock); + if (!ei->jinode) { + if (!jinode) { + spin_unlock(&inode->i_lock); + return -ENOMEM; + } + jbd2_journal_init_jbd_inode(jinode, inode); + /* + * Publish ->jinode only after it is fully initialized so that + * readers never observe a partially initialized jbd2_inode. + */ + smp_wmb(); + WRITE_ONCE(ei->jinode, jinode); + jinode = NULL; + } + spin_unlock(&inode->i_lock); + if (unlikely(jinode != NULL)) + jbd2_free_inode(jinode); + return 0; +} + +/* + * ext4_truncate() + * + * We block out ext4_get_block() block instantiations across the entire + * transaction, and VFS/VM ensures that ext4_truncate() cannot run + * simultaneously on behalf of the same inode. + * + * As we work through the truncate and commit bits of it to the journal there + * is one core, guiding principle: the file's tree must always be consistent on + * disk. We must be able to restart the truncate after a crash. + * + * The file's tree may be transiently inconsistent in memory (although it + * probably isn't), but whenever we close off and commit a journal transaction, + * the contents of (the filesystem + the journal) must be consistent and + * restartable. It's pretty simple, really: bottom up, right to left (although + * left-to-right works OK too). + * + * Note that at recovery time, journal replay occurs *before* the restart of + * truncate against the orphan inode list. + * + * The committed inode has the new, desired i_size (which is the same as + * i_disksize in this case). After a crash, ext4_orphan_cleanup() will see + * that this inode's truncate did not complete and it will again call + * ext4_truncate() to have another go. So there will be instantiated blocks + * to the right of the truncation point in a crashed ext4 filesystem. But + * that's fine - as long as they are linked from the inode, the post-crash + * ext4_truncate() run will find them and release them. + */ +int ext4_truncate(struct inode *inode) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + unsigned int credits; + int err = 0, err2; + handle_t *handle; + + /* + * There is a possibility that we're either freeing the inode + * or it's a completely new inode. In those cases we might not + * have i_rwsem locked because it's not necessary. + */ + if (!(inode_state_read_once(inode) & (I_NEW | I_FREEING))) + WARN_ON(!inode_is_locked(inode)); + trace_ext4_truncate_enter(inode); + + if (!ext4_can_truncate(inode)) + goto out_trace; + + if (inode->i_size == 0 && !test_opt(inode->i_sb, NO_AUTO_DA_ALLOC)) + ext4_set_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE); + + if (ext4_has_inline_data(inode)) { + int has_inline = 1; + + err = ext4_inline_data_truncate(inode, &has_inline); + if (err || has_inline) + goto out_trace; + } + + /* If we zero-out tail of the page, we have to create jinode for jbd2 */ + if (inode->i_size & (inode->i_sb->s_blocksize - 1)) { + err = ext4_inode_attach_jinode(inode); + if (err) + goto out_trace; + + /* Zero to the end of the block containing i_size */ + err = ext4_block_zero_eof(inode, inode->i_size, LLONG_MAX); + if (err) + goto out_trace; + } + + if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) + credits = ext4_chunk_trans_extent(inode, 1); + else + credits = ext4_blocks_for_truncate(inode); + + handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits); + if (IS_ERR(handle)) { + err = PTR_ERR(handle); + goto out_trace; + } + + /* + * We add the inode to the orphan list, so that if this + * truncate spans multiple transactions, and we crash, we will + * resume the truncate when the filesystem recovers. It also + * marks the inode dirty, to catch the new size. + * + * Implication: the file must always be in a sane, consistent + * truncatable state while each transaction commits. + */ + err = ext4_orphan_add(handle, inode); + if (err) + goto out_stop; + + ext4_fc_track_inode(handle, inode); + ext4_check_map_extents_env(inode); + + down_write(&EXT4_I(inode)->i_data_sem); + ext4_discard_preallocations(inode); + + if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) + err = ext4_ext_truncate(handle, inode); + else + ext4_ind_truncate(handle, inode); + + up_write(&ei->i_data_sem); + if (err) + goto out_stop; + + if (IS_SYNC(inode)) + ext4_handle_sync(handle); + +out_stop: + /* + * If this was a simple ftruncate() and the file will remain alive, + * then we need to clear up the orphan record which we created above. + * However, if this was a real unlink then we were called by + * ext4_evict_inode(), and we allow that function to clean up the + * orphan info for us. + */ + if (inode->i_nlink) + ext4_orphan_del(handle, inode); + + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + err2 = ext4_mark_inode_dirty(handle, inode); + if (unlikely(err2 && !err)) + err = err2; + ext4_journal_stop(handle); + +out_trace: + trace_ext4_truncate_exit(inode); + return err; +} + +static inline u64 ext4_inode_peek_iversion(const struct inode *inode) +{ + if (unlikely(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) + return inode_peek_iversion_raw(inode); + else + return inode_peek_iversion(inode); +} + +static int ext4_inode_blocks_set(struct ext4_inode *raw_inode, + struct ext4_inode_info *ei) +{ + struct inode *inode = &(ei->vfs_inode); + u64 i_blocks = READ_ONCE(inode->i_blocks); + struct super_block *sb = inode->i_sb; + + if (i_blocks <= ~0U) { + /* + * i_blocks can be represented in a 32 bit variable + * as multiple of 512 bytes + */ + raw_inode->i_blocks_lo = cpu_to_le32(i_blocks); + raw_inode->i_blocks_high = 0; + ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE); + return 0; + } + + /* + * This should never happen since sb->s_maxbytes should not have + * allowed this, sb->s_maxbytes was set according to the huge_file + * feature in ext4_fill_super(). + */ + if (!ext4_has_feature_huge_file(sb)) + return -EFSCORRUPTED; + + if (i_blocks <= 0xffffffffffffULL) { + /* + * i_blocks can be represented in a 48 bit variable + * as multiple of 512 bytes + */ + raw_inode->i_blocks_lo = cpu_to_le32(i_blocks); + raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32); + ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE); + } else { + ext4_set_inode_flag(inode, EXT4_INODE_HUGE_FILE); + /* i_block is stored in file system block size */ + i_blocks = i_blocks >> (inode->i_blkbits - 9); + raw_inode->i_blocks_lo = cpu_to_le32(i_blocks); + raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32); + } + return 0; +} + +static int ext4_fill_raw_inode(struct inode *inode, struct ext4_inode *raw_inode) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + uid_t i_uid; + gid_t i_gid; + projid_t i_projid; + int block; + int err; + + err = ext4_inode_blocks_set(raw_inode, ei); + + raw_inode->i_mode = cpu_to_le16(inode->i_mode); + i_uid = i_uid_read(inode); + i_gid = i_gid_read(inode); + i_projid = from_kprojid(&init_user_ns, ei->i_projid); + if (!(test_opt(inode->i_sb, NO_UID32))) { + raw_inode->i_uid_low = cpu_to_le16(low_16_bits(i_uid)); + raw_inode->i_gid_low = cpu_to_le16(low_16_bits(i_gid)); + /* + * Fix up interoperability with old kernels. Otherwise, + * old inodes get re-used with the upper 16 bits of the + * uid/gid intact. + */ + if (ei->i_dtime && !ext4_inode_orphan_tracked(inode)) { + raw_inode->i_uid_high = 0; + raw_inode->i_gid_high = 0; + } else { + raw_inode->i_uid_high = + cpu_to_le16(high_16_bits(i_uid)); + raw_inode->i_gid_high = + cpu_to_le16(high_16_bits(i_gid)); + } + } else { + raw_inode->i_uid_low = cpu_to_le16(fs_high2lowuid(i_uid)); + raw_inode->i_gid_low = cpu_to_le16(fs_high2lowgid(i_gid)); + raw_inode->i_uid_high = 0; + raw_inode->i_gid_high = 0; + } + raw_inode->i_links_count = cpu_to_le16(inode->i_nlink); + + EXT4_INODE_SET_CTIME(inode, raw_inode); + EXT4_INODE_SET_MTIME(inode, raw_inode); + EXT4_INODE_SET_ATIME(inode, raw_inode); + EXT4_EINODE_SET_XTIME(i_crtime, ei, raw_inode); + + raw_inode->i_dtime = cpu_to_le32(ei->i_dtime); + raw_inode->i_flags = cpu_to_le32(ei->i_flags & 0xFFFFFFFF); + if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) + raw_inode->i_file_acl_high = + cpu_to_le16(ei->i_file_acl >> 32); + raw_inode->i_file_acl_lo = cpu_to_le32(ei->i_file_acl); + ext4_isize_set(raw_inode, ei->i_disksize); + + raw_inode->i_generation = cpu_to_le32(inode->i_generation); + if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) { + if (old_valid_dev(inode->i_rdev)) { + raw_inode->i_block[0] = + cpu_to_le32(old_encode_dev(inode->i_rdev)); + raw_inode->i_block[1] = 0; + } else { + raw_inode->i_block[0] = 0; + raw_inode->i_block[1] = + cpu_to_le32(new_encode_dev(inode->i_rdev)); + raw_inode->i_block[2] = 0; + } + } else if (!ext4_has_inline_data(inode)) { + for (block = 0; block < EXT4_N_BLOCKS; block++) + raw_inode->i_block[block] = ei->i_data[block]; + } + + if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) { + u64 ivers = ext4_inode_peek_iversion(inode); + + raw_inode->i_disk_version = cpu_to_le32(ivers); + if (ei->i_extra_isize) { + if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi)) + raw_inode->i_version_hi = + cpu_to_le32(ivers >> 32); + raw_inode->i_extra_isize = + cpu_to_le16(ei->i_extra_isize); + } + } + + if (i_projid != EXT4_DEF_PROJID && + !ext4_has_feature_project(inode->i_sb)) + err = err ?: -EFSCORRUPTED; + + if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE && + EXT4_FITS_IN_INODE(raw_inode, ei, i_projid)) + raw_inode->i_projid = cpu_to_le32(i_projid); + + ext4_inode_csum_set(inode, raw_inode, ei); + return err; +} + +/* + * ext4_get_inode_loc returns with an extra refcount against the inode's + * underlying buffer_head on success. If we pass 'inode' and it does not + * have in-inode xattr, we have all inode data in memory that is needed + * to recreate the on-disk version of this inode. + */ +static int __ext4_get_inode_loc(struct super_block *sb, unsigned long ino, + struct inode *inode, struct ext4_iloc *iloc, + ext4_fsblk_t *ret_block) +{ + struct ext4_group_desc *gdp; + struct buffer_head *bh; + ext4_fsblk_t block; + struct blk_plug plug; + int inodes_per_block, inode_offset; + + iloc->bh = NULL; + if (ino < EXT4_ROOT_INO || + ino > le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count)) + return -EFSCORRUPTED; + + iloc->block_group = (ino - 1) / EXT4_INODES_PER_GROUP(sb); + gdp = ext4_get_group_desc(sb, iloc->block_group, NULL); + if (!gdp) + return -EIO; + + /* + * Figure out the offset within the block group inode table + */ + inodes_per_block = EXT4_SB(sb)->s_inodes_per_block; + inode_offset = ((ino - 1) % + EXT4_INODES_PER_GROUP(sb)); + iloc->offset = (inode_offset % inodes_per_block) * EXT4_INODE_SIZE(sb); + + block = ext4_inode_table(sb, gdp); + if ((block <= le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block)) || + (block >= ext4_blocks_count(EXT4_SB(sb)->s_es))) { + ext4_error(sb, "Invalid inode table block %llu in " + "block_group %u", block, iloc->block_group); + return -EFSCORRUPTED; + } + block += (inode_offset / inodes_per_block); + + bh = sb_getblk(sb, block); + if (unlikely(!bh)) + return -ENOMEM; + if (ext4_buffer_uptodate(bh)) + goto has_buffer; + + lock_buffer(bh); + if (ext4_buffer_uptodate(bh)) { + /* Someone brought it uptodate while we waited */ + unlock_buffer(bh); + goto has_buffer; + } + + /* + * If we have all information of the inode in memory and this + * is the only valid inode in the block, we need not read the + * block. + */ + if (inode && !ext4_test_inode_state(inode, EXT4_STATE_XATTR)) { + struct buffer_head *bitmap_bh; + int i, start; + + start = inode_offset & ~(inodes_per_block - 1); + + /* Is the inode bitmap in cache? */ + bitmap_bh = sb_getblk(sb, ext4_inode_bitmap(sb, gdp)); + if (unlikely(!bitmap_bh)) + goto make_io; + + /* + * If the inode bitmap isn't in cache then the + * optimisation may end up performing two reads instead + * of one, so skip it. + */ + if (!buffer_uptodate(bitmap_bh)) { + brelse(bitmap_bh); + goto make_io; + } + for (i = start; i < start + inodes_per_block; i++) { + if (i == inode_offset) + continue; + if (ext4_test_bit(i, bitmap_bh->b_data)) + break; + } + brelse(bitmap_bh); + if (i == start + inodes_per_block) { + struct ext4_inode *raw_inode = + (struct ext4_inode *) (bh->b_data + iloc->offset); + + /* all other inodes are free, so skip I/O */ + memset(bh->b_data, 0, bh->b_size); + if (!ext4_test_inode_state(inode, EXT4_STATE_NEW)) + ext4_fill_raw_inode(inode, raw_inode); + set_buffer_uptodate(bh); + unlock_buffer(bh); + goto has_buffer; + } + } + +make_io: + /* + * If we need to do any I/O, try to pre-readahead extra + * blocks from the inode table. + */ + blk_start_plug(&plug); + if (EXT4_SB(sb)->s_inode_readahead_blks) { + ext4_fsblk_t b, end, table; + unsigned num; + __u32 ra_blks = EXT4_SB(sb)->s_inode_readahead_blks; + + table = ext4_inode_table(sb, gdp); + /* s_inode_readahead_blks is always a power of 2 */ + b = block & ~((ext4_fsblk_t) ra_blks - 1); + if (table > b) + b = table; + end = b + ra_blks; + num = EXT4_INODES_PER_GROUP(sb); + if (ext4_has_group_desc_csum(sb)) + num -= ext4_itable_unused_count(sb, gdp); + table += num / inodes_per_block; + if (end > table) + end = table; + while (b <= end) + ext4_sb_breadahead_unmovable(sb, b++); + } + + /* + * There are other valid inodes in the buffer, this inode + * has in-inode xattrs, or we don't have this inode in memory. + * Read the block from disk. + */ + trace_ext4_load_inode(sb, ino); + ext4_read_bh_nowait(bh, REQ_META | REQ_PRIO, NULL, + ext4_simulate_fail(sb, EXT4_SIM_INODE_EIO)); + blk_finish_plug(&plug); + wait_on_buffer(bh); + if (!buffer_uptodate(bh)) { + if (ret_block) + *ret_block = block; + brelse(bh); + return -EIO; + } +has_buffer: + iloc->bh = bh; + return 0; +} + +static int __ext4_get_inode_loc_noinmem(struct inode *inode, + struct ext4_iloc *iloc) +{ + ext4_fsblk_t err_blk = 0; + int ret; + + ret = __ext4_get_inode_loc(inode->i_sb, inode->i_ino, NULL, iloc, + &err_blk); + + if (ret == -EIO) + ext4_error_inode_block(inode, err_blk, EIO, + "unable to read itable block"); + + return ret; +} + +int ext4_get_inode_loc(struct inode *inode, struct ext4_iloc *iloc) +{ + ext4_fsblk_t err_blk = 0; + int ret; + + ret = __ext4_get_inode_loc(inode->i_sb, inode->i_ino, inode, iloc, + &err_blk); + + if (ret == -EIO) + ext4_error_inode_block(inode, err_blk, EIO, + "unable to read itable block"); + + return ret; +} + +/* + * ext4_get_inode_loc_noio() is a best-effort variant of ext4_get_inode_loc(). + * It looks up the inode table block in the buffer cache and returns -EAGAIN if + * the block is not present or not uptodate, without starting any I/O. + */ +int ext4_get_inode_loc_noio(struct inode *inode, struct ext4_iloc *iloc) +{ + struct super_block *sb = inode->i_sb; + struct ext4_group_desc *gdp; + struct buffer_head *bh; + ext4_fsblk_t block; + int inodes_per_block, inode_offset; + unsigned long ino = inode->i_ino; + + iloc->bh = NULL; + if (ino < EXT4_ROOT_INO || + ino > le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count)) + return -EFSCORRUPTED; + + iloc->block_group = (ino - 1) / EXT4_INODES_PER_GROUP(sb); + gdp = ext4_get_group_desc(sb, iloc->block_group, NULL); + if (!gdp) + return -EIO; + + /* Figure out the offset within the block group inode table. */ + inodes_per_block = EXT4_SB(sb)->s_inodes_per_block; + inode_offset = ((ino - 1) % EXT4_INODES_PER_GROUP(sb)); + iloc->offset = (inode_offset % inodes_per_block) * EXT4_INODE_SIZE(sb); + + block = ext4_inode_table(sb, gdp); + if (block <= le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block) || + block >= ext4_blocks_count(EXT4_SB(sb)->s_es)) { + ext4_error(sb, + "Invalid inode table block %llu in block_group %u", + block, iloc->block_group); + return -EFSCORRUPTED; + } + block += inode_offset / inodes_per_block; + + bh = sb_find_get_block(sb, block); + if (!bh) + return -EAGAIN; + if (!ext4_buffer_uptodate(bh)) { + brelse(bh); + return -EAGAIN; + } + + iloc->bh = bh; + return 0; +} + + +int ext4_get_fc_inode_loc(struct super_block *sb, unsigned long ino, + struct ext4_iloc *iloc) +{ + return __ext4_get_inode_loc(sb, ino, NULL, iloc, NULL); +} + +static bool ext4_should_enable_dax(struct inode *inode) +{ + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + + if (test_opt2(inode->i_sb, DAX_NEVER)) + return false; + if (!S_ISREG(inode->i_mode)) + return false; + if (ext4_should_journal_data(inode)) + return false; + if (ext4_has_inline_data(inode)) + return false; + if (ext4_test_inode_flag(inode, EXT4_INODE_ENCRYPT)) + return false; + if (ext4_test_inode_flag(inode, EXT4_INODE_VERITY)) + return false; + if (!test_bit(EXT4_FLAGS_BDEV_IS_DAX, &sbi->s_ext4_flags)) + return false; + if (test_opt(inode->i_sb, DAX_ALWAYS)) + return true; + + return ext4_test_inode_flag(inode, EXT4_INODE_DAX); +} + +void ext4_set_inode_flags(struct inode *inode, bool init) +{ + unsigned int flags = EXT4_I(inode)->i_flags; + unsigned int new_fl = 0; + + WARN_ON_ONCE(IS_DAX(inode) && init); + + if (flags & EXT4_SYNC_FL) + new_fl |= S_SYNC; + if (flags & EXT4_APPEND_FL) + new_fl |= S_APPEND; + if (flags & EXT4_IMMUTABLE_FL) + new_fl |= S_IMMUTABLE; + if (flags & EXT4_NOATIME_FL) + new_fl |= S_NOATIME; + if (flags & EXT4_DIRSYNC_FL) + new_fl |= S_DIRSYNC; + + /* Because of the way inode_set_flags() works we must preserve S_DAX + * here if already set. */ + new_fl |= (inode->i_flags & S_DAX); + if (init && ext4_should_enable_dax(inode)) + new_fl |= S_DAX; + + if (flags & EXT4_ENCRYPT_FL) + new_fl |= S_ENCRYPTED; + if (flags & EXT4_CASEFOLD_FL) + new_fl |= S_CASEFOLD; + if (flags & EXT4_VERITY_FL) + new_fl |= S_VERITY; + inode_set_flags(inode, new_fl, + S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC|S_DAX| + S_ENCRYPTED|S_CASEFOLD|S_VERITY); +} + +static blkcnt_t ext4_inode_blocks(struct ext4_inode *raw_inode, + struct ext4_inode_info *ei) +{ + blkcnt_t i_blocks ; + struct inode *inode = &(ei->vfs_inode); + struct super_block *sb = inode->i_sb; + + if (ext4_has_feature_huge_file(sb)) { + /* we are using combined 48 bit field */ + i_blocks = ((u64)le16_to_cpu(raw_inode->i_blocks_high)) << 32 | + le32_to_cpu(raw_inode->i_blocks_lo); + if (ext4_test_inode_flag(inode, EXT4_INODE_HUGE_FILE)) { + /* i_blocks represent file system block size */ + return i_blocks << (inode->i_blkbits - 9); + } else { + return i_blocks; + } + } else { + return le32_to_cpu(raw_inode->i_blocks_lo); + } +} + +static inline int ext4_iget_extra_inode(struct inode *inode, + struct ext4_inode *raw_inode, + struct ext4_inode_info *ei) +{ + __le32 *magic = (void *)raw_inode + + EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize; + + if (EXT4_INODE_HAS_XATTR_SPACE(inode) && + *magic == cpu_to_le32(EXT4_XATTR_MAGIC)) { + int err; + + err = xattr_check_inode(inode, IHDR(inode, raw_inode), + ITAIL(inode, raw_inode)); + if (err) + return err; + + ext4_set_inode_state(inode, EXT4_STATE_XATTR); + err = ext4_find_inline_data_nolock(inode); + if (!err && ext4_has_inline_data(inode)) + ext4_set_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); + return err; + } else + EXT4_I(inode)->i_inline_off = 0; + return 0; +} + +int ext4_get_projid(struct inode *inode, kprojid_t *projid) +{ + if (!ext4_has_feature_project(inode->i_sb)) + return -EOPNOTSUPP; + *projid = EXT4_I(inode)->i_projid; + return 0; +} + +/* + * ext4 has self-managed i_version for ea inodes, it stores the lower 32bit of + * refcount in i_version, so use raw values if inode has EXT4_EA_INODE_FL flag + * set. + */ +static inline void ext4_inode_set_iversion_queried(struct inode *inode, u64 val) +{ + if (unlikely(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) + inode_set_iversion_raw(inode, val); + else + inode_set_iversion_queried(inode, val); +} + +static int check_igot_inode(struct inode *inode, ext4_iget_flags flags, + const char *function, unsigned int line) +{ + const char *err_str; + + if (flags & EXT4_IGET_EA_INODE) { + if (!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) { + err_str = "missing EA_INODE flag"; + goto error; + } + if (ext4_test_inode_state(inode, EXT4_STATE_XATTR) || + EXT4_I(inode)->i_file_acl) { + err_str = "ea_inode with extended attributes"; + goto error; + } + } else { + if ((EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) { + /* + * open_by_handle_at() could provide an old inode number + * that has since been reused for an ea_inode; this does + * not indicate filesystem corruption + */ + if (flags & EXT4_IGET_HANDLE) + return -ESTALE; + err_str = "unexpected EA_INODE flag"; + goto error; + } + } + if (is_bad_inode(inode) && !(flags & EXT4_IGET_BAD)) { + err_str = "unexpected bad inode w/o EXT4_IGET_BAD"; + goto error; + } + return 0; + +error: + ext4_error_inode(inode, function, line, 0, "%s", err_str); + return -EFSCORRUPTED; +} + +void ext4_set_inode_mapping_order(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + u16 min_order, max_order; + + max_order = EXT4_SB(sb)->s_max_folio_order; + if (!max_order) + return; + + min_order = EXT4_SB(sb)->s_min_folio_order; + if (!min_order && !S_ISREG(inode->i_mode)) + return; + + if (ext4_test_inode_flag(inode, EXT4_INODE_JOURNAL_DATA)) + max_order = min_order; + + mapping_set_folio_order_range(inode->i_mapping, min_order, max_order); +} + +static int ext4_iget_match(struct inode *inode, u64 ino, void *data) +{ + if (inode->i_ino != ino) + return 0; + spin_lock(&inode->i_lock); + if (inode_state_read(inode) & (I_FREEING | I_WILL_FREE | I_CREATING)) { + spin_unlock(&inode->i_lock); + return -1; + } + __iget(inode); + spin_unlock(&inode->i_lock); + return 1; +} + +struct inode *__ext4_iget(struct super_block *sb, unsigned long ino, + ext4_iget_flags flags, const char *function, + unsigned int line) +{ + struct ext4_iloc iloc; + struct ext4_inode *raw_inode; + struct ext4_inode_info *ei; + struct ext4_super_block *es = EXT4_SB(sb)->s_es; + struct inode *inode; + journal_t *journal = EXT4_SB(sb)->s_journal; + long ret; + loff_t size; + int block; + uid_t i_uid; + gid_t i_gid; + projid_t i_projid; + + if ((!(flags & EXT4_IGET_SPECIAL) && is_special_ino(sb, ino)) || + (ino < EXT4_ROOT_INO) || + (ino > le32_to_cpu(es->s_inodes_count))) { + if (flags & EXT4_IGET_HANDLE) + return ERR_PTR(-ESTALE); + __ext4_error(sb, function, line, false, EFSCORRUPTED, 0, + "inode #%lu: comm %s: iget: illegal inode #", + ino, current->comm); + return ERR_PTR(-EFSCORRUPTED); + } + + if (flags & EXT4_IGET_NOWAIT) { + inode = find_inode_nowait(sb, ino, ext4_iget_match, NULL); + if (!inode) + return ERR_PTR(-ENOENT); + + if (inode_state_read_once(inode) & I_NEW) + wait_on_new_inode(inode); + + if (unlikely(inode_unhashed(inode))) { + iput(inode); + return ERR_PTR(-ENOENT); + } + } else { + inode = iget_locked(sb, ino); + if (!inode) + return ERR_PTR(-ENOMEM); + } + + if (!(inode_state_read_once(inode) & I_NEW)) { + ret = check_igot_inode(inode, flags, function, line); + if (ret) { + iput(inode); + return ERR_PTR(ret); + } + return inode; + } + + ei = EXT4_I(inode); + iloc.bh = NULL; + + ret = __ext4_get_inode_loc_noinmem(inode, &iloc); + if (ret < 0) + goto bad_inode; + raw_inode = ext4_raw_inode(&iloc); + + if ((flags & EXT4_IGET_HANDLE) && + (raw_inode->i_links_count == 0) && (raw_inode->i_mode == 0)) { + ret = -ESTALE; + goto bad_inode; + } + + if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) { + ei->i_extra_isize = le16_to_cpu(raw_inode->i_extra_isize); + if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize > + EXT4_INODE_SIZE(inode->i_sb) || + (ei->i_extra_isize & 3)) { + ext4_error_inode(inode, function, line, 0, + "iget: bad extra_isize %u " + "(inode size %u)", + ei->i_extra_isize, + EXT4_INODE_SIZE(inode->i_sb)); + ret = -EFSCORRUPTED; + goto bad_inode; + } + } else + ei->i_extra_isize = 0; + + /* Precompute checksum seed for inode metadata */ + if (ext4_has_feature_metadata_csum(sb)) { + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + __u32 csum; + __le32 inum = cpu_to_le32(inode->i_ino); + __le32 gen = raw_inode->i_generation; + csum = ext4_chksum(sbi->s_csum_seed, (__u8 *)&inum, + sizeof(inum)); + ei->i_csum_seed = ext4_chksum(csum, (__u8 *)&gen, sizeof(gen)); + } + + if ((!ext4_inode_csum_verify(inode, raw_inode, ei) || + ext4_simulate_fail(sb, EXT4_SIM_INODE_CRC)) && + (!(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY))) { + ext4_error_inode_err(inode, function, line, 0, + EFSBADCRC, "iget: checksum invalid"); + ret = -EFSBADCRC; + goto bad_inode; + } + + inode->i_mode = le16_to_cpu(raw_inode->i_mode); + i_uid = (uid_t)le16_to_cpu(raw_inode->i_uid_low); + i_gid = (gid_t)le16_to_cpu(raw_inode->i_gid_low); + if (ext4_has_feature_project(sb) && + EXT4_INODE_SIZE(sb) > EXT4_GOOD_OLD_INODE_SIZE && + EXT4_FITS_IN_INODE(raw_inode, ei, i_projid)) + i_projid = (projid_t)le32_to_cpu(raw_inode->i_projid); + else + i_projid = EXT4_DEF_PROJID; + + if (!(test_opt(inode->i_sb, NO_UID32))) { + i_uid |= le16_to_cpu(raw_inode->i_uid_high) << 16; + i_gid |= le16_to_cpu(raw_inode->i_gid_high) << 16; + } + i_uid_write(inode, i_uid); + i_gid_write(inode, i_gid); + ei->i_projid = make_kprojid(&init_user_ns, i_projid); + set_nlink(inode, le16_to_cpu(raw_inode->i_links_count)); + + ei->i_inline_off = 0; + ei->i_dir_start_lookup = 0; + ei->i_dtime = le32_to_cpu(raw_inode->i_dtime); + /* We now have enough fields to check if the inode was active or not. + * This is needed because nfsd might try to access dead inodes + * the test is that same one that e2fsck uses + * NeilBrown 1999oct15 + */ + if (inode->i_nlink == 0) { + if ((inode->i_mode == 0 || flags & EXT4_IGET_SPECIAL || + !(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_ORPHAN_FS)) && + ino != EXT4_BOOT_LOADER_INO) { + /* this inode is deleted or unallocated */ + if (flags & EXT4_IGET_SPECIAL) { + ext4_error_inode(inode, function, line, 0, + "iget: special inode unallocated"); + ret = -EFSCORRUPTED; + } else + ret = -ESTALE; + goto bad_inode; + } + /* The only unlinked inodes we let through here have + * valid i_mode and are being read by the orphan + * recovery code: that's fine, we're about to complete + * the process of deleting those. + * OR it is the EXT4_BOOT_LOADER_INO which is + * not initialized on a new filesystem. */ + } + ei->i_flags = le32_to_cpu(raw_inode->i_flags); + ext4_set_inode_flags(inode, true); + /* Detect invalid flag combination - can't have both inline data and extents */ + if (ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA) && + ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) { + ext4_error_inode(inode, function, line, 0, + "inode has both inline data and extents flags"); + ret = -EFSCORRUPTED; + goto bad_inode; + } + inode->i_blocks = ext4_inode_blocks(raw_inode, ei); + ei->i_file_acl = le32_to_cpu(raw_inode->i_file_acl_lo); + if (ext4_has_feature_64bit(sb)) + ei->i_file_acl |= + ((__u64)le16_to_cpu(raw_inode->i_file_acl_high)) << 32; + inode->i_size = ext4_isize(sb, raw_inode); + size = i_size_read(inode); + if (size < 0 || size > ext4_get_maxbytes(inode)) { + ext4_error_inode(inode, function, line, 0, + "iget: bad i_size value: %lld", size); + ret = -EFSCORRUPTED; + goto bad_inode; + } + /* + * If dir_index is not enabled but there's dir with INDEX flag set, + * we'd normally treat htree data as empty space. But with metadata + * checksumming that corrupts checksums so forbid that. + */ + if (!ext4_has_feature_dir_index(sb) && + ext4_has_feature_metadata_csum(sb) && + ext4_test_inode_flag(inode, EXT4_INODE_INDEX)) { + ext4_error_inode(inode, function, line, 0, + "iget: Dir with htree data on filesystem without dir_index feature."); + ret = -EFSCORRUPTED; + goto bad_inode; + } + ei->i_disksize = inode->i_size; +#ifdef CONFIG_QUOTA + ei->i_reserved_quota = 0; +#endif + inode->i_generation = le32_to_cpu(raw_inode->i_generation); + ei->i_block_group = iloc.block_group; + ei->i_last_alloc_group = ~0; + /* + * NOTE! The in-memory inode i_data array is in little-endian order + * even on big-endian machines: we do NOT byteswap the block numbers! + */ + for (block = 0; block < EXT4_N_BLOCKS; block++) + ei->i_data[block] = raw_inode->i_block[block]; + INIT_LIST_HEAD(&ei->i_orphan); + ext4_fc_init_inode(&ei->vfs_inode); + + /* + * Set transaction id's of transactions that have to be committed + * to finish f[data]sync. We set them to currently running transaction + * as we cannot be sure that the inode or some of its metadata isn't + * part of the transaction - the inode could have been reclaimed and + * now it is reread from disk. + */ + if (journal) { + transaction_t *transaction; + tid_t tid; + + read_lock(&journal->j_state_lock); + if (journal->j_running_transaction) + transaction = journal->j_running_transaction; + else + transaction = journal->j_committing_transaction; + if (transaction) + tid = transaction->t_tid; + else + tid = journal->j_commit_sequence; + read_unlock(&journal->j_state_lock); + ei->i_sync_tid = tid; + ei->i_datasync_tid = tid; + } + + if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) { + if (ei->i_extra_isize == 0) { + /* The extra space is currently unused. Use it. */ + BUILD_BUG_ON(sizeof(struct ext4_inode) & 3); + ei->i_extra_isize = sizeof(struct ext4_inode) - + EXT4_GOOD_OLD_INODE_SIZE; + } else { + ret = ext4_iget_extra_inode(inode, raw_inode, ei); + if (ret) + goto bad_inode; + } + } + + EXT4_INODE_GET_CTIME(inode, raw_inode); + EXT4_INODE_GET_ATIME(inode, raw_inode); + EXT4_INODE_GET_MTIME(inode, raw_inode); + EXT4_EINODE_GET_XTIME(i_crtime, ei, raw_inode); + + if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) { + u64 ivers = le32_to_cpu(raw_inode->i_disk_version); + + if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) { + if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi)) + ivers |= + (__u64)(le32_to_cpu(raw_inode->i_version_hi)) << 32; + } + ext4_inode_set_iversion_queried(inode, ivers); + } + + ret = 0; + if (ei->i_file_acl && + !ext4_inode_block_valid(inode, ei->i_file_acl, 1)) { + ext4_error_inode(inode, function, line, 0, + "iget: bad extended attribute block %llu", + ei->i_file_acl); + ret = -EFSCORRUPTED; + goto bad_inode; + } else if (!ext4_has_inline_data(inode)) { + /* validate the block references in the inode */ + if (!(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY) && + (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) || + (S_ISLNK(inode->i_mode) && + !ext4_inode_is_fast_symlink(inode)))) { + if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) + ret = ext4_ext_check_inode(inode); + else + ret = ext4_ind_check_inode(inode); + } + } + if (ret) + goto bad_inode; + + if (S_ISREG(inode->i_mode)) { + inode->i_op = &ext4_file_inode_operations; + inode->i_fop = &ext4_file_operations; + ext4_set_aops(inode); + } else if (S_ISDIR(inode->i_mode)) { + inode->i_op = &ext4_dir_inode_operations; + inode->i_fop = &ext4_dir_operations; + } else if (S_ISLNK(inode->i_mode)) { + /* VFS does not allow setting these so must be corruption */ + if (IS_APPEND(inode) || IS_IMMUTABLE(inode)) { + ext4_error_inode(inode, function, line, 0, + "iget: immutable or append flags " + "not allowed on symlinks"); + ret = -EFSCORRUPTED; + goto bad_inode; + } + if (IS_ENCRYPTED(inode)) { + inode->i_op = &ext4_encrypted_symlink_inode_operations; + } else if (ext4_inode_is_fast_symlink(inode)) { + inode->i_op = &ext4_fast_symlink_inode_operations; + + /* + * Orphan cleanup can see inodes with i_size == 0 + * and i_data uninitialized. Skip size checks in + * that case. This is safe because the first thing + * ext4_evict_inode() does for fast symlinks is + * clearing of i_data and i_size. + */ + if ((EXT4_SB(sb)->s_mount_state & EXT4_ORPHAN_FS)) { + if (inode->i_nlink != 0) { + ext4_error_inode(inode, function, line, 0, + "invalid orphan symlink nlink %d", + inode->i_nlink); + ret = -EFSCORRUPTED; + goto bad_inode; + } + } else { + if (inode->i_size == 0 || + inode->i_size >= sizeof(ei->i_data) || + strnlen((char *)ei->i_data, inode->i_size + 1) != + inode->i_size) { + ext4_error_inode(inode, function, line, 0, + "invalid fast symlink length %llu", + (unsigned long long)inode->i_size); + ret = -EFSCORRUPTED; + goto bad_inode; + } + inode_set_cached_link(inode, (char *)ei->i_data, + inode->i_size); + } + } else { + inode->i_op = &ext4_symlink_inode_operations; + } + } else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) || + S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) { + inode->i_op = &ext4_special_inode_operations; + if (raw_inode->i_block[0]) + init_special_inode(inode, inode->i_mode, + old_decode_dev(le32_to_cpu(raw_inode->i_block[0]))); + else + init_special_inode(inode, inode->i_mode, + new_decode_dev(le32_to_cpu(raw_inode->i_block[1]))); + } else if (ino == EXT4_BOOT_LOADER_INO) { + make_bad_inode(inode); + } else { + ret = -EFSCORRUPTED; + ext4_error_inode(inode, function, line, 0, + "iget: bogus i_mode (%o)", inode->i_mode); + goto bad_inode; + } + if (IS_CASEFOLDED(inode) && !ext4_has_feature_casefold(inode->i_sb)) { + ext4_error_inode(inode, function, line, 0, + "casefold flag without casefold feature"); + ret = -EFSCORRUPTED; + goto bad_inode; + } + + ext4_set_inode_mapping_order(inode); + + ret = check_igot_inode(inode, flags, function, line); + /* + * -ESTALE here means there is nothing inherently wrong with the inode, + * it's just not an inode we can return for an fhandle lookup. + */ + if (ret == -ESTALE) { + brelse(iloc.bh); + unlock_new_inode(inode); + iput(inode); + return ERR_PTR(-ESTALE); + } + if (ret) + goto bad_inode; + brelse(iloc.bh); + /* Initialize the "no ACL's" state for the simple cases */ + if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR) && !ei->i_file_acl) + cache_no_acl(inode); + unlock_new_inode(inode); + return inode; + +bad_inode: + brelse(iloc.bh); + iget_failed(inode); + return ERR_PTR(ret); +} + +static void __ext4_update_other_inode_time(struct super_block *sb, + unsigned long orig_ino, + unsigned long ino, + struct ext4_inode *raw_inode) +{ + struct inode *inode; + + inode = find_inode_by_ino_rcu(sb, ino); + if (!inode) + return; + + if (!inode_is_dirtytime_only(inode)) + return; + + spin_lock(&inode->i_lock); + if (inode_is_dirtytime_only(inode)) { + struct ext4_inode_info *ei = EXT4_I(inode); + + inode_state_clear(inode, I_DIRTY_TIME); + spin_unlock(&inode->i_lock); + + spin_lock(&ei->i_raw_lock); + EXT4_INODE_SET_CTIME(inode, raw_inode); + EXT4_INODE_SET_MTIME(inode, raw_inode); + EXT4_INODE_SET_ATIME(inode, raw_inode); + ext4_inode_csum_set(inode, raw_inode, ei); + spin_unlock(&ei->i_raw_lock); + trace_ext4_other_inode_update_time(inode, orig_ino); + return; + } + spin_unlock(&inode->i_lock); +} + +/* + * Opportunistically update the other time fields for other inodes in + * the same inode table block. + */ +static void ext4_update_other_inodes_time(struct super_block *sb, + unsigned long orig_ino, char *buf) +{ + unsigned long ino; + int i, inodes_per_block = EXT4_SB(sb)->s_inodes_per_block; + int inode_size = EXT4_INODE_SIZE(sb); + + /* + * Calculate the first inode in the inode table block. Inode + * numbers are one-based. That is, the first inode in a block + * (assuming 4k blocks and 256 byte inodes) is (n*16 + 1). + */ + ino = ((orig_ino - 1) & ~(inodes_per_block - 1)) + 1; + rcu_read_lock(); + for (i = 0; i < inodes_per_block; i++, ino++, buf += inode_size) { + if (ino == orig_ino) + continue; + __ext4_update_other_inode_time(sb, orig_ino, ino, + (struct ext4_inode *)buf); + } + rcu_read_unlock(); +} + +/* + * Post the struct inode info into an on-disk inode location in the + * buffer-cache. This gobbles the caller's reference to the + * buffer_head in the inode location struct. + * + * The caller must have write access to iloc->bh. + */ +static int ext4_do_update_inode(handle_t *handle, + struct inode *inode, + struct ext4_iloc *iloc) +{ + struct ext4_inode *raw_inode = ext4_raw_inode(iloc); + struct ext4_inode_info *ei = EXT4_I(inode); + struct buffer_head *bh = iloc->bh; + struct super_block *sb = inode->i_sb; + int err; + int need_datasync = 0, set_large_file = 0; + + spin_lock(&ei->i_raw_lock); + + /* + * For fields not tracked in the in-memory inode, initialise them + * to zero for new inodes. + */ + if (ext4_test_inode_state(inode, EXT4_STATE_NEW)) + memset(raw_inode, 0, EXT4_SB(inode->i_sb)->s_inode_size); + + if (READ_ONCE(ei->i_disksize) != ext4_isize(inode->i_sb, raw_inode)) + need_datasync = 1; + if (ei->i_disksize > 0x7fffffffULL) { + if (!ext4_has_feature_large_file(sb) || + EXT4_SB(sb)->s_es->s_rev_level == cpu_to_le32(EXT4_GOOD_OLD_REV)) + set_large_file = 1; + } + + err = ext4_fill_raw_inode(inode, raw_inode); + spin_unlock(&ei->i_raw_lock); + if (err) { + EXT4_ERROR_INODE(inode, "corrupted inode contents"); + goto out_brelse; + } + + if (inode->i_sb->s_flags & SB_LAZYTIME) + ext4_update_other_inodes_time(inode->i_sb, inode->i_ino, + bh->b_data); + + BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata"); + err = ext4_handle_dirty_metadata(handle, NULL, bh); + if (err) + goto out_error; + ext4_clear_inode_state(inode, EXT4_STATE_NEW); + if (set_large_file) { + BUFFER_TRACE(EXT4_SB(sb)->s_sbh, "get write access"); + err = ext4_journal_get_write_access(handle, sb, + EXT4_SB(sb)->s_sbh, + EXT4_JTR_NONE); + if (err) + goto out_error; + lock_buffer(EXT4_SB(sb)->s_sbh); + ext4_set_feature_large_file(sb); + ext4_superblock_csum_set(sb); + unlock_buffer(EXT4_SB(sb)->s_sbh); + ext4_handle_sync(handle); + err = ext4_handle_dirty_metadata(handle, NULL, + EXT4_SB(sb)->s_sbh); + } + ext4_update_inode_fsync_trans(handle, inode, need_datasync); +out_error: + ext4_std_error(inode->i_sb, err); +out_brelse: + brelse(bh); + return err; +} + +/* + * ext4_write_inode() + * + * We are called from a few places: + * + * - Within generic_file_aio_write() -> generic_write_sync() for O_SYNC files. + * Here, there will be no transaction running. We wait for any running + * transaction to commit. + * + * - Within flush work (sys_sync(), kupdate and such). + * We wait on commit, if told to. + * + * - Within iput_final() -> write_inode_now() + * We wait on commit, if told to. + * + * In all cases it is actually safe for us to return without doing anything, + * because the inode has been copied into a raw inode buffer in + * ext4_mark_inode_dirty(). This is a correctness thing for WB_SYNC_ALL + * writeback. + * + * For nojournal mode all the work is done in ext4_sync_inode_metadata() + * because inode content is already copied into raw inode buffer and inode + * is marked with I_METADATA_WRITEBACK. + * + * Note that we are absolutely dependent upon all inode dirtiers doing the + * right thing: they *must* call mark_inode_dirty() after dirtying info in + * which we are interested. + * + * It would be a bug for them to not do this. The code: + * + * mark_inode_dirty(inode) + * stuff(); + * inode->i_size = expr; + * + * is in error because write_inode() could occur while `stuff()' is running, + * and the new i_size will be lost. Plus the inode will no longer be on the + * superblock's dirty inode list. + */ +int ext4_write_inode(struct inode *inode, struct writeback_control *wbc) +{ + int err; + + if (WARN_ON_ONCE(current->flags & PF_MEMALLOC)) + return 0; + + err = ext4_emergency_state(inode->i_sb); + if (unlikely(err)) + return err; + + if (!EXT4_SB(inode->i_sb)->s_journal) + return 0; + + if (ext4_journal_current_handle()) { + ext4_debug("called recursively, non-PF_MEMALLOC!\n"); + dump_stack(); + return -EIO; + } + + /* + * No need to force transaction in WB_SYNC_NONE mode. Also + * ext4_sync_fs() will force the commit after everything is + * written. + */ + if (wbc->sync_mode != WB_SYNC_ALL || wbc->for_sync) + return 0; + + return ext4_fc_commit(EXT4_SB(inode->i_sb)->s_journal, + EXT4_I(inode)->i_sync_tid); +} + +int ext4_sync_inode_metadata(struct inode *inode, struct writeback_control *wbc) +{ + struct ext4_iloc iloc; + struct mapping_metadata_bhs *mmb; + int err; + + /* We should only get here in nojournal mode */ + if (WARN_ON_ONCE(EXT4_SB(inode->i_sb)->s_journal)) + return -EFSCORRUPTED; + + err = __ext4_get_inode_loc_noinmem(inode, &iloc); + if (err) + return err; + mmb = READ_ONCE(EXT4_I(inode)->i_metadata_bhs); + if (mmb) { + err = mmb_sync(mmb); + if (err) + goto out; + } + sync_dirty_buffer(iloc.bh); + if (buffer_write_io_error(iloc.bh)) { + ext4_error_inode_block(inode, iloc.bh->b_blocknr, EIO, + "IO error syncing inode"); + err = -EIO; + } +out: + brelse(iloc.bh); + return err; +} + +/* + * In data=journal mode ext4_journalled_invalidate_folio() may fail to invalidate + * buffers that are attached to a folio straddling i_size and are undergoing + * commit. In that case we have to wait for commit to finish and try again. + */ +static void ext4_wait_for_tail_page_commit(struct inode *inode) +{ + unsigned offset; + journal_t *journal = EXT4_SB(inode->i_sb)->s_journal; + tid_t commit_tid; + int ret; + bool has_transaction; + + offset = inode->i_size & (PAGE_SIZE - 1); + /* + * If the folio is fully truncated, we don't need to wait for any commit + * (and we even should not as __ext4_journalled_invalidate_folio() may + * strip all buffers from the folio but keep the folio dirty which can then + * confuse e.g. concurrent ext4_writepages() seeing dirty folio without + * buffers). Also we don't need to wait for any commit if all buffers in + * the folio remain valid. This is most beneficial for the common case of + * blocksize == PAGESIZE. + */ + if (!offset || offset > (PAGE_SIZE - i_blocksize(inode))) + return; + while (1) { + struct folio *folio = filemap_lock_folio(inode->i_mapping, + inode->i_size >> PAGE_SHIFT); + if (IS_ERR(folio)) + return; + ret = __ext4_journalled_invalidate_folio(folio, offset, + folio_size(folio) - offset); + folio_unlock(folio); + folio_put(folio); + if (ret != -EBUSY) + return; + has_transaction = false; + read_lock(&journal->j_state_lock); + if (journal->j_committing_transaction) { + commit_tid = journal->j_committing_transaction->t_tid; + has_transaction = true; + } + read_unlock(&journal->j_state_lock); + if (has_transaction) + jbd2_log_wait_commit(journal, commit_tid); + } +} + +/* + * ext4_setattr() + * + * Called from notify_change. + * + * We want to trap VFS attempts to truncate the file as soon as + * possible. In particular, we want to make sure that when the VFS + * shrinks i_size, we put the inode on the orphan list and modify + * i_disksize immediately, so that during the subsequent flushing of + * dirty pages and freeing of disk blocks, we can guarantee that any + * commit will leave the blocks being flushed in an unused state on + * disk. (On recovery, the inode will get truncated and the blocks will + * be freed, so we have a strong guarantee that no future commit will + * leave these blocks visible to the user.) + * + * Another thing we have to assure is that if we are in ordered mode + * and inode is still attached to the committing transaction, we must + * we start writeout of all the dirty pages which are being truncated. + * This way we are sure that all the data written in the previous + * transaction are already on disk (truncate waits for pages under + * writeback). + * + * Called with inode->i_rwsem down. + */ +int ext4_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +{ + struct inode *inode = d_inode(dentry); + int error, rc = 0; + int orphan = 0; + const unsigned int ia_valid = attr->ia_valid; + bool inc_ivers = true; + + error = ext4_emergency_state(inode->i_sb); + if (unlikely(error)) + return error; + + if (unlikely(IS_IMMUTABLE(inode))) + return -EPERM; + + if (unlikely(IS_APPEND(inode) && + (ia_valid & (ATTR_MODE | ATTR_UID | + ATTR_GID | ATTR_TIMES_SET)))) + return -EPERM; + + error = setattr_prepare(idmap, dentry, attr); + if (error) + return error; + + error = fscrypt_prepare_setattr(dentry, attr); + if (error) + return error; + + if (is_quota_modification(idmap, inode, attr)) { + error = dquot_initialize(inode); + if (error) + return error; + } + + if (i_uid_needs_update(idmap, attr, inode) || + i_gid_needs_update(idmap, attr, inode)) { + handle_t *handle; + + /* (user+group)*(old+new) structure, inode write (sb, + * inode block, ? - but truncate inode update has it) */ + handle = ext4_journal_start(inode, EXT4_HT_QUOTA, + (EXT4_MAXQUOTAS_INIT_BLOCKS(inode->i_sb) + + EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb)) + 3); + if (IS_ERR(handle)) { + error = PTR_ERR(handle); + goto err_out; + } + + /* dquot_transfer() calls back ext4_get_inode_usage() which + * counts xattr inode references. + */ + down_read(&EXT4_I(inode)->xattr_sem); + error = dquot_transfer(idmap, inode, attr); + up_read(&EXT4_I(inode)->xattr_sem); + + if (error) { + ext4_journal_stop(handle); + return error; + } + /* Update corresponding info in inode so that everything is in + * one transaction */ + i_uid_update(idmap, attr, inode); + i_gid_update(idmap, attr, inode); + error = ext4_mark_inode_dirty(handle, inode); + ext4_journal_stop(handle); + if (unlikely(error)) { + return error; + } + } + + if (attr->ia_valid & ATTR_SIZE) { + handle_t *handle; + loff_t oldsize = inode->i_size; + loff_t old_disksize; + int shrink = (attr->ia_size < inode->i_size); + + if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) { + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + + if (attr->ia_size > sbi->s_bitmap_maxbytes) { + return -EFBIG; + } + } + if (!S_ISREG(inode->i_mode)) { + return -EINVAL; + } + + if (attr->ia_size == inode->i_size) + inc_ivers = false; + + /* + * If file has inline data but new size exceeds inline capacity, + * convert to extent-based storage first to prevent inconsistent + * state (inline flag set but size exceeds inline capacity). + */ + if (ext4_has_inline_data(inode) && + attr->ia_size > EXT4_I(inode)->i_inline_size) { + error = ext4_convert_inline_data(inode); + if (error) + goto err_out; + } + + if (shrink) { + if (ext4_should_order_data(inode)) { + error = ext4_begin_ordered_truncate(inode, + attr->ia_size); + if (error) + goto err_out; + } + /* + * Blocks are going to be removed from the inode. Wait + * for dio in flight. + */ + inode_dio_wait(inode); + } + + filemap_invalidate_lock(inode->i_mapping); + + rc = ext4_break_layouts(inode); + if (rc) { + filemap_invalidate_unlock(inode->i_mapping); + goto err_out; + } + + if (attr->ia_size != inode->i_size) { + /* attach jbd2 jinode for EOF folio tail zeroing */ + if (attr->ia_size & (inode->i_sb->s_blocksize - 1) || + oldsize & (inode->i_sb->s_blocksize - 1)) { + error = ext4_inode_attach_jinode(inode); + if (error) + goto out_mmap_sem; + } + + /* + * Update c/mtime and tail zero the EOF folio on + * truncate up. ext4_truncate() handles the shrink case + * below. + */ + if (!shrink) { + inode_set_mtime_to_ts(inode, + inode_set_ctime_current(inode)); + if (oldsize & (inode->i_sb->s_blocksize - 1)) { + error = ext4_block_zero_eof(inode, + oldsize, LLONG_MAX); + if (error) + goto out_mmap_sem; + } + } + + handle = ext4_journal_start(inode, EXT4_HT_INODE, 3); + if (IS_ERR(handle)) { + error = PTR_ERR(handle); + goto out_mmap_sem; + } + if (ext4_handle_valid(handle) && shrink) { + error = ext4_orphan_add(handle, inode); + orphan = 1; + } + + if (shrink) + ext4_fc_track_range(handle, inode, + (attr->ia_size > 0 ? attr->ia_size - 1 : 0) >> + inode->i_sb->s_blocksize_bits, + EXT_MAX_BLOCKS - 1); + else + ext4_fc_track_range( + handle, inode, + (oldsize > 0 ? oldsize - 1 : oldsize) >> + inode->i_sb->s_blocksize_bits, + (attr->ia_size > 0 ? attr->ia_size - 1 : 0) >> + inode->i_sb->s_blocksize_bits); + + down_write(&EXT4_I(inode)->i_data_sem); + old_disksize = EXT4_I(inode)->i_disksize; + EXT4_I(inode)->i_disksize = attr->ia_size; + + /* + * We have to update i_size under i_data_sem together + * with i_disksize to avoid races with writeback code + * running ext4_wb_update_i_disksize(). + */ + if (!error) + i_size_write(inode, attr->ia_size); + else + EXT4_I(inode)->i_disksize = old_disksize; + up_write(&EXT4_I(inode)->i_data_sem); + rc = ext4_mark_inode_dirty(handle, inode); + if (!error) + error = rc; + ext4_journal_stop(handle); + if (error) + goto out_mmap_sem; + if (!shrink) { + pagecache_isize_extended(inode, oldsize, + inode->i_size); + } else if (ext4_should_journal_data(inode)) { + ext4_wait_for_tail_page_commit(inode); + } + } + + /* + * Truncate pagecache after we've waited for commit + * in data=journal mode to make pages freeable. + */ + truncate_pagecache(inode, inode->i_size); + /* + * Call ext4_truncate() even if i_size didn't change to + * truncate possible preallocated blocks. + */ + if (attr->ia_size <= oldsize) { + rc = ext4_truncate(inode); + if (rc) + error = rc; + } +out_mmap_sem: + filemap_invalidate_unlock(inode->i_mapping); + } + + if (!error) { + if (inc_ivers) + inode_inc_iversion(inode); + setattr_copy(idmap, inode, attr); + mark_inode_dirty(inode); + } + + /* + * If the call to ext4_truncate failed to get a transaction handle at + * all, we need to clean up the in-core orphan list manually. + */ + if (orphan && inode->i_nlink) + ext4_orphan_del(NULL, inode); + + if (!error && (ia_valid & ATTR_MODE)) + rc = posix_acl_chmod(idmap, dentry, inode->i_mode); + +err_out: + if (error) + ext4_std_error(inode->i_sb, error); + if (!error) + error = rc; + return error; +} + +u32 ext4_dio_alignment(struct inode *inode) +{ + if (fsverity_active(inode)) + return 0; + if (ext4_should_journal_data(inode)) + return 0; + if (ext4_has_inline_data(inode)) + return 0; + if (IS_ENCRYPTED(inode)) + return i_blocksize(inode); + return 1; /* use the iomap defaults */ +} + +int ext4_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int query_flags) +{ + struct inode *inode = d_inode(path->dentry); + struct ext4_inode *raw_inode; + struct ext4_inode_info *ei = EXT4_I(inode); + unsigned int flags; + + if ((request_mask & STATX_BTIME) && + EXT4_FITS_IN_INODE(raw_inode, ei, i_crtime)) { + stat->result_mask |= STATX_BTIME; + stat->btime.tv_sec = ei->i_crtime.tv_sec; + stat->btime.tv_nsec = ei->i_crtime.tv_nsec; + } + + /* Return the DIO alignment restrictions if requested. */ + if ((request_mask & STATX_DIOALIGN) && S_ISREG(inode->i_mode)) { + u32 dio_align = ext4_dio_alignment(inode); + + stat->result_mask |= STATX_DIOALIGN; + if (dio_align == 1) { + struct block_device *bdev = inode->i_sb->s_bdev; + + /* iomap defaults */ + stat->dio_mem_align = bdev_dma_alignment(bdev) + 1; + stat->dio_offset_align = bdev_logical_block_size(bdev); + } else { + stat->dio_mem_align = dio_align; + stat->dio_offset_align = dio_align; + } + } + + if ((request_mask & STATX_WRITE_ATOMIC) && S_ISREG(inode->i_mode)) { + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + unsigned int awu_min = 0, awu_max = 0; + + if (ext4_inode_can_atomic_write(inode)) { + awu_min = sbi->s_awu_min; + awu_max = sbi->s_awu_max; + } + + generic_fill_statx_atomic_writes(stat, awu_min, awu_max, 0); + } + + flags = ei->i_flags & EXT4_FL_USER_VISIBLE; + if (flags & EXT4_APPEND_FL) + stat->attributes |= STATX_ATTR_APPEND; + if (flags & EXT4_COMPR_FL) + stat->attributes |= STATX_ATTR_COMPRESSED; + if (flags & EXT4_ENCRYPT_FL) + stat->attributes |= STATX_ATTR_ENCRYPTED; + if (flags & EXT4_IMMUTABLE_FL) + stat->attributes |= STATX_ATTR_IMMUTABLE; + if (flags & EXT4_NODUMP_FL) + stat->attributes |= STATX_ATTR_NODUMP; + if (flags & EXT4_VERITY_FL) + stat->attributes |= STATX_ATTR_VERITY; + + stat->attributes_mask |= (STATX_ATTR_APPEND | + STATX_ATTR_COMPRESSED | + STATX_ATTR_ENCRYPTED | + STATX_ATTR_IMMUTABLE | + STATX_ATTR_NODUMP | + STATX_ATTR_VERITY); + + generic_fillattr(idmap, request_mask, inode, stat); + return 0; +} + +int ext4_file_getattr(struct mnt_idmap *idmap, + const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int query_flags) +{ + struct inode *inode = d_inode(path->dentry); + u64 delalloc_blocks; + + ext4_getattr(idmap, path, stat, request_mask, query_flags); + + /* + * If there is inline data in the inode, the inode will normally not + * have data blocks allocated (it may have an external xattr block). + * Report at least one sector for such files, so tools like tar, rsync, + * others don't incorrectly think the file is completely sparse. + */ + if (unlikely(ext4_has_inline_data(inode))) + stat->blocks += (stat->size + 511) >> 9; + + /* + * We can't update i_blocks if the block allocation is delayed + * otherwise in the case of system crash before the real block + * allocation is done, we will have i_blocks inconsistent with + * on-disk file blocks. + * We always keep i_blocks updated together with real + * allocation. But to not confuse with user, stat + * will return the blocks that include the delayed allocation + * blocks for this file. + */ + delalloc_blocks = EXT4_C2B(EXT4_SB(inode->i_sb), + EXT4_I(inode)->i_reserved_data_blocks); + stat->blocks += delalloc_blocks << (inode->i_sb->s_blocksize_bits - 9); + return 0; +} + +static int ext4_index_trans_blocks(struct inode *inode, int lblocks, + int pextents) +{ + if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) + return ext4_ind_trans_blocks(inode, lblocks); + return ext4_ext_index_trans_blocks(inode, pextents); +} + +/* + * Calculate number of credits needed in a transaction to: + * * Allocate data blocks from @alloc_extents different groups - note that + * with flexbg a single physical extent can span multiple groups but + * single mballoc request only returns extent within one group. + * * Allocate metatadata (extent tree blocks, indirect blocks) to store + * pointers to @pextents data extents having @lblocks in total. + * * Modify extent tree / indirect block tree, inode, superblock, quota + * tracking, xattr blocks + */ +int ext4_meta_trans_blocks(struct inode *inode, int lblocks, int pextents, + int alloc_extents) +{ + ext4_group_t groups, ngroups = ext4_get_groups_count(inode->i_sb); + int gdpblocks; + int idxblocks; + int ret; + + /* + * How many index and leaf blocks need to touch to map @lblocks + * logical blocks to @pextents physical extents? + */ + idxblocks = ext4_index_trans_blocks(inode, lblocks, pextents); + + /* + * Now let's see how many group bitmaps and group descriptors need + * to account + */ + groups = idxblocks + alloc_extents; + gdpblocks = groups; + if (groups > ngroups) + groups = ngroups; + if (groups > EXT4_SB(inode->i_sb)->s_gdb_count) + gdpblocks = EXT4_SB(inode->i_sb)->s_gdb_count; + + /* bitmaps and block group descriptor blocks */ + ret = idxblocks + groups + gdpblocks; + + /* Blocks for super block, inode, quota and xattr blocks */ + ret += EXT4_META_TRANS_BLOCKS(inode->i_sb); + + return ret; +} + +/* + * Calculate the journal credits for modifying the number of blocks + * in a single extent within one transaction. 'nrblocks' is used only + * for non-extent inodes. For extent type inodes, 'nrblocks' can be + * zero if the exact number of blocks is unknown. + */ +int ext4_chunk_trans_extent(struct inode *inode, int nrblocks) +{ + int ret; + + ret = ext4_meta_trans_blocks(inode, nrblocks, 1, 1); + /* Account for data blocks for journalled mode */ + if (ext4_should_journal_data(inode)) + ret += nrblocks; + return ret; +} + +/* + * Calculate the journal credits for a chunk of data modification. + * + * This is called from DIO, fallocate or whoever calling + * ext4_map_blocks() to map/allocate a chunk of contiguous disk blocks. + * + * journal buffers for data blocks are not included here, as DIO + * and fallocate do no need to journal data buffers. + */ +int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks) +{ + return ext4_meta_trans_blocks(inode, nrblocks, 1, 1); +} + +/* + * The caller must have previously called ext4_reserve_inode_write(). + * Give this, we know that the caller already has write access to iloc->bh. + */ +int ext4_mark_iloc_dirty(handle_t *handle, + struct inode *inode, struct ext4_iloc *iloc) +{ + int err = 0; + + err = ext4_emergency_state(inode->i_sb); + if (unlikely(err)) { + put_bh(iloc->bh); + return err; + } + ext4_fc_track_inode(handle, inode); + + /* the do_update_inode consumes one bh->b_count */ + get_bh(iloc->bh); + + /* ext4_do_update_inode() does jbd2_journal_dirty_metadata */ + err = ext4_do_update_inode(handle, inode, iloc); + put_bh(iloc->bh); + /* + * Mark that there's metadata writeout pending for the inode so that it + * gets properly flushed on fsync(2) and similar. + */ + if (!EXT4_SB(inode->i_sb)->s_journal) { + /* + * Inode didn't need to go through dirtying, make sure it is + * attached to wb so that writeback can handle it. + */ + spin_lock(&inode->i_lock); + inode_attach_wb(inode, NULL); + spin_unlock(&inode->i_lock); + set_inode_metadata_writeback(inode); + } + return err; +} + +/* + * On success, We end up with an outstanding reference count against + * iloc->bh. This _must_ be cleaned up later. + */ + +int +ext4_reserve_inode_write(handle_t *handle, struct inode *inode, + struct ext4_iloc *iloc) +{ + int err; + + err = ext4_emergency_state(inode->i_sb); + if (unlikely(err)) + return err; + + err = ext4_get_inode_loc(inode, iloc); + if (!err) { + BUFFER_TRACE(iloc->bh, "get_write_access"); + err = ext4_journal_get_write_access(handle, inode->i_sb, + iloc->bh, EXT4_JTR_NONE); + if (err) { + brelse(iloc->bh); + iloc->bh = NULL; + } + ext4_fc_track_inode(handle, inode); + } + ext4_std_error(inode->i_sb, err); + return err; +} + +static int __ext4_expand_extra_isize(struct inode *inode, + unsigned int new_extra_isize, + struct ext4_iloc *iloc, + handle_t *handle, int *no_expand) +{ + struct ext4_inode *raw_inode; + struct ext4_xattr_ibody_header *header; + unsigned int inode_size = EXT4_INODE_SIZE(inode->i_sb); + struct ext4_inode_info *ei = EXT4_I(inode); + int error; + + /* this was checked at iget time, but double check for good measure */ + if ((EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize > inode_size) || + (ei->i_extra_isize & 3)) { + EXT4_ERROR_INODE(inode, "bad extra_isize %u (inode size %u)", + ei->i_extra_isize, + EXT4_INODE_SIZE(inode->i_sb)); + return -EFSCORRUPTED; + } + if ((new_extra_isize < ei->i_extra_isize) || + (new_extra_isize < 4) || + (new_extra_isize > inode_size - EXT4_GOOD_OLD_INODE_SIZE)) + return -EINVAL; /* Should never happen */ + + raw_inode = ext4_raw_inode(iloc); + + header = IHDR(inode, raw_inode); + + /* No extended attributes present */ + if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR) || + header->h_magic != cpu_to_le32(EXT4_XATTR_MAGIC)) { + memset((void *)raw_inode + EXT4_GOOD_OLD_INODE_SIZE + + EXT4_I(inode)->i_extra_isize, 0, + new_extra_isize - EXT4_I(inode)->i_extra_isize); + EXT4_I(inode)->i_extra_isize = new_extra_isize; + return 0; + } + + /* + * We may need to allocate external xattr block so we need quotas + * initialized. Here we can be called with various locks held so we + * cannot affort to initialize quotas ourselves. So just bail. + */ + if (dquot_initialize_needed(inode)) + return -EAGAIN; + + /* try to expand with EAs present */ + error = ext4_expand_extra_isize_ea(inode, new_extra_isize, + raw_inode, handle); + if (error) { + /* + * Inode size expansion failed; don't try again + */ + *no_expand = 1; + } + + return error; +} + +/* + * Expand an inode by new_extra_isize bytes. + * Returns 0 on success or negative error number on failure. + */ +static int ext4_try_to_expand_extra_isize(struct inode *inode, + unsigned int new_extra_isize, + struct ext4_iloc iloc, + handle_t *handle) +{ + int no_expand; + int error; + + if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND)) + return -EOVERFLOW; + + /* + * Skip expansion during mount (!SB_ACTIVE). Expanding extra isize + * may move xattrs to external blocks and release ea_inodes via iput. + * When !SB_ACTIVE, iput triggers write_inode_now() which acquires + * s_writepages_rwsem, causing a deadlock with the caller's active + * jbd2 handle (lock order: s_writepages_rwsem -> jbd2_handle). + */ + if (unlikely(!(inode->i_sb->s_flags & SB_ACTIVE))) + return -EBUSY; + + /* + * In nojournal mode, we can immediately attempt to expand + * the inode. When journaled, we first need to obtain extra + * buffer credits since we may write into the EA block + * with this same handle. If journal_extend fails, then it will + * only result in a minor loss of functionality for that inode. + * If this is felt to be critical, then e2fsck should be run to + * force a large enough s_min_extra_isize. + */ + if (ext4_journal_extend(handle, + EXT4_DATA_TRANS_BLOCKS(inode->i_sb), 0) != 0) + return -ENOSPC; + + if (ext4_write_trylock_xattr(inode, &no_expand) == 0) + return -EBUSY; + + error = __ext4_expand_extra_isize(inode, new_extra_isize, &iloc, + handle, &no_expand); + ext4_write_unlock_xattr(inode, &no_expand); + + return error; +} + +int ext4_expand_extra_isize(struct inode *inode, + unsigned int new_extra_isize, + struct ext4_iloc *iloc) +{ + handle_t *handle; + int no_expand; + int error, rc; + + if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND)) { + brelse(iloc->bh); + return -EOVERFLOW; + } + + handle = ext4_journal_start(inode, EXT4_HT_INODE, + EXT4_DATA_TRANS_BLOCKS(inode->i_sb)); + if (IS_ERR(handle)) { + error = PTR_ERR(handle); + brelse(iloc->bh); + return error; + } + + ext4_write_lock_xattr(inode, &no_expand); + + BUFFER_TRACE(iloc->bh, "get_write_access"); + error = ext4_journal_get_write_access(handle, inode->i_sb, iloc->bh, + EXT4_JTR_NONE); + if (error) { + brelse(iloc->bh); + goto out_unlock; + } + + error = __ext4_expand_extra_isize(inode, new_extra_isize, iloc, + handle, &no_expand); + + rc = ext4_mark_iloc_dirty(handle, inode, iloc); + if (!error) + error = rc; + +out_unlock: + ext4_write_unlock_xattr(inode, &no_expand); + ext4_journal_stop(handle); + return error; +} + +/* + * What we do here is to mark the in-core inode as clean with respect to inode + * dirtiness (it may still be data-dirty). + * This means that the in-core inode may be reaped by prune_icache + * without having to perform any I/O. This is a very good thing, + * because *any* task may call prune_icache - even ones which + * have a transaction open against a different journal. + * + * Is this cheating? Not really. Sure, we haven't written the + * inode out, but prune_icache isn't a user-visible syncing function. + * Whenever the user wants stuff synced (sys_sync, sys_msync, sys_fsync) + * we start and wait on commits. + */ +int __ext4_mark_inode_dirty(handle_t *handle, struct inode *inode, + const char *func, unsigned int line) +{ + struct ext4_iloc iloc; + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + int err; + + might_sleep(); + trace_ext4_mark_inode_dirty(inode, _RET_IP_); + err = ext4_reserve_inode_write(handle, inode, &iloc); + if (err) + goto out; + + if (EXT4_I(inode)->i_extra_isize < sbi->s_want_extra_isize) + ext4_try_to_expand_extra_isize(inode, sbi->s_want_extra_isize, + iloc, handle); + + err = ext4_mark_iloc_dirty(handle, inode, &iloc); +out: + if (unlikely(err)) + ext4_error_inode_err(inode, func, line, 0, err, + "mark_inode_dirty error"); + return err; +} + +/* + * ext4_dirty_inode() is called from __mark_inode_dirty() + * + * We're really interested in the case where a file is being extended. + * i_size has been changed by generic_commit_write() and we thus need + * to include the updated inode in the current transaction. + * + * Also, dquot_alloc_block() will always dirty the inode when blocks + * are allocated to the file. + * + * If the inode is marked synchronous, we don't honour that here - doing + * so would cause a commit on atime updates, which we don't bother doing. + * We handle synchronous inodes at the highest possible level. + */ +void ext4_dirty_inode(struct inode *inode, int flags) +{ + handle_t *handle; + + handle = ext4_journal_start(inode, EXT4_HT_INODE, 2); + if (IS_ERR(handle)) + return; + ext4_mark_inode_dirty(handle, inode); + ext4_journal_stop(handle); +} + +int ext4_change_inode_journal_flag(struct inode *inode, int val) +{ + journal_t *journal; + handle_t *handle; + int err; + int alloc_ctx; + + /* + * We have to be very careful here: changing a data block's + * journaling status dynamically is dangerous. If we write a + * data block to the journal, change the status and then delete + * that block, we risk forgetting to revoke the old log record + * from the journal and so a subsequent replay can corrupt data. + * So, first we make sure that the journal is empty and that + * nobody is changing anything. + */ + + journal = EXT4_JOURNAL(inode); + if (!journal) + return 0; + if (is_journal_aborted(journal)) + return -EROFS; + + /* Wait for all existing dio workers */ + inode_dio_wait(inode); + + /* + * Before flushing the journal and switching inode's aops, we have + * to flush all dirty data the inode has. There can be outstanding + * delayed allocations, there can be unwritten extents created by + * fallocate or buffered writes in dioread_nolock mode covered by + * dirty data which can be converted only after flushing the dirty + * data (and journalled aops don't know how to handle these cases). + */ + filemap_invalidate_lock(inode->i_mapping); + err = filemap_write_and_wait(inode->i_mapping); + if (err < 0) { + filemap_invalidate_unlock(inode->i_mapping); + return err; + } + /* Before switch the inode journalling mode evict all the page cache. */ + truncate_pagecache(inode, 0); + + alloc_ctx = ext4_writepages_down_write(inode->i_sb); + jbd2_journal_lock_updates(journal); + + /* + * OK, there are no updates running now, and all cached data is + * synced to disk. We are now in a completely consistent state + * which doesn't have anything in the journal, and we know that + * no filesystem updates are running, so it is safe to modify + * the inode's in-core data-journaling state flag now. + */ + + if (val) + ext4_set_inode_flag(inode, EXT4_INODE_JOURNAL_DATA); + else { + err = jbd2_journal_flush(journal, 0); + if (err < 0) { + jbd2_journal_unlock_updates(journal); + ext4_writepages_up_write(inode->i_sb, alloc_ctx); + filemap_invalidate_unlock(inode->i_mapping); + return err; + } + ext4_clear_inode_flag(inode, EXT4_INODE_JOURNAL_DATA); + } + ext4_set_aops(inode); + ext4_set_inode_mapping_order(inode); + + jbd2_journal_unlock_updates(journal); + ext4_writepages_up_write(inode->i_sb, alloc_ctx); + filemap_invalidate_unlock(inode->i_mapping); + + /* Finally we can mark the inode as dirty. */ + + handle = ext4_journal_start(inode, EXT4_HT_INODE, 1); + if (IS_ERR(handle)) + return PTR_ERR(handle); + + ext4_fc_mark_ineligible(inode->i_sb, + EXT4_FC_REASON_JOURNAL_FLAG_CHANGE, handle); + err = ext4_mark_inode_dirty(handle, inode); + ext4_handle_sync(handle); + ext4_journal_stop(handle); + ext4_std_error(inode->i_sb, err); + + return err; +} + +static int ext4_bh_unmapped(handle_t *handle, struct inode *inode, + struct buffer_head *bh) +{ + return !buffer_mapped(bh); +} + +static int ext4_block_page_mkwrite(struct inode *inode, struct folio *folio, + get_block_t get_block) +{ + handle_t *handle; + loff_t size; + unsigned long len; + int credits; + int ret; + + credits = ext4_chunk_trans_extent(inode, + ext4_journal_blocks_per_folio(inode)); + handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, credits); + if (IS_ERR(handle)) + return PTR_ERR(handle); + + folio_lock(folio); + size = i_size_read(inode); + /* Page got truncated from under us? */ + if (folio->mapping != inode->i_mapping || folio_pos(folio) > size) { + ret = -EFAULT; + goto out_error; + } + + len = folio_size(folio); + if (folio_pos(folio) + len > size) + len = size - folio_pos(folio); + + ret = ext4_block_write_begin(handle, folio, 0, len, get_block); + if (ret) + goto out_error; + + if (!ext4_should_journal_data(inode)) { + block_commit_write(folio, 0, len); + folio_mark_dirty(folio); + } else { + ret = ext4_journal_folio_buffers(handle, folio, len); + if (ret) + goto out_error; + } + ext4_journal_stop(handle); + folio_wait_stable(folio); + return ret; + +out_error: + folio_unlock(folio); + ext4_journal_stop(handle); + return ret; +} + +vm_fault_t ext4_page_mkwrite(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct folio *folio = page_folio(vmf->page); + loff_t size; + unsigned long len; + int err; + vm_fault_t ret; + struct file *file = vma->vm_file; + struct inode *inode = file_inode(file); + struct address_space *mapping = inode->i_mapping; + get_block_t *get_block = ext4_get_block; + int retries = 0; + + if (unlikely(IS_IMMUTABLE(inode))) + return VM_FAULT_SIGBUS; + + sb_start_pagefault(inode->i_sb); + file_update_time(vma->vm_file); + + filemap_invalidate_lock_shared(mapping); + + err = ext4_convert_inline_data(inode); + if (err) + goto out_ret; + + /* + * On data journalling we skip straight to the transaction handle: + * there's no delalloc; page truncated will be checked later; the + * early return w/ all buffers mapped (calculates size/len) can't + * be used; and there's no dioread_nolock, so only ext4_get_block. + */ + if (ext4_should_journal_data(inode)) + goto retry_alloc; + + /* Delalloc case is easy... */ + if (test_opt(inode->i_sb, DELALLOC) && + !ext4_nonda_switch(inode->i_sb)) { + do { + err = block_page_mkwrite(vma, vmf, + ext4_da_get_block_prep); + } while (err == -ENOSPC && + ext4_should_retry_alloc(inode->i_sb, &retries)); + goto out_ret; + } + + folio_lock(folio); + size = i_size_read(inode); + /* Page got truncated from under us? */ + if (folio->mapping != mapping || folio_pos(folio) > size) { + folio_unlock(folio); + ret = VM_FAULT_NOPAGE; + goto out; + } + + len = folio_size(folio); + if (folio_pos(folio) + len > size) + len = size - folio_pos(folio); + /* + * Return if we have all the buffers mapped. This avoids the need to do + * journal_start/journal_stop which can block and take a long time + * + * This cannot be done for data journalling, as we have to add the + * inode to the transaction's list to writeprotect pages on commit. + */ + if (folio_buffers(folio)) { + if (!ext4_walk_page_buffers(NULL, inode, folio_buffers(folio), + 0, len, NULL, + ext4_bh_unmapped)) { + /* Wait so that we don't change page under IO */ + folio_wait_stable(folio); + ret = VM_FAULT_LOCKED; + goto out; + } + } + folio_unlock(folio); + /* OK, we need to fill the hole... */ + if (ext4_should_dioread_nolock(inode)) + get_block = ext4_get_block_unwritten; +retry_alloc: + /* Start journal and allocate blocks */ + err = ext4_block_page_mkwrite(inode, folio, get_block); + if (err == -EAGAIN || + (err == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))) + goto retry_alloc; +out_ret: + ret = vmf_fs_error(err); +out: + filemap_invalidate_unlock_shared(mapping); + sb_end_pagefault(inode->i_sb); + return ret; +} diff --git a/fs/ext4/ioctl.c b/fs/ext4/ioctl.c new file mode 100644 index 000000000..c8387e6a2 --- /dev/null +++ b/fs/ext4/ioctl.c @@ -0,0 +1,2028 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext4/ioctl.c + * + * Copyright (C) 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ext4_jbd2.h" +#include "ext4.h" +#include +#include "fsmap.h" +#include +#include + +typedef void ext4_update_sb_callback(struct ext4_sb_info *sbi, + struct ext4_super_block *es, + const void *arg); + +/* + * Superblock modification callback function for changing file system + * label + */ +static void ext4_sb_setlabel(struct ext4_sb_info *sbi, + struct ext4_super_block *es, const void *arg) +{ + /* Sanity check, this should never happen */ + BUILD_BUG_ON(sizeof(es->s_volume_name) < EXT4_LABEL_MAX); + + memcpy(es->s_volume_name, (char *)arg, EXT4_LABEL_MAX); +} + +/* + * Superblock modification callback function for changing file system + * UUID. + */ +static void ext4_sb_setuuid(struct ext4_sb_info *sbi, + struct ext4_super_block *es, const void *arg) +{ + memcpy(es->s_uuid, (__u8 *)arg, UUID_SIZE); +} + +static +int ext4_update_primary_sb(struct super_block *sb, handle_t *handle, + ext4_update_sb_callback func, + const void *arg) +{ + int err = 0; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct buffer_head *bh = sbi->s_sbh; + struct ext4_super_block *es = sbi->s_es; + + trace_ext4_update_sb(sb, bh->b_blocknr, 1); + + BUFFER_TRACE(bh, "get_write_access"); + err = ext4_journal_get_write_access(handle, sb, + bh, + EXT4_JTR_NONE); + if (err) + goto out_err; + + lock_buffer(bh); + func(sbi, es, arg); + ext4_superblock_csum_set(sb); + unlock_buffer(bh); + + if (buffer_write_io_error(bh) || !buffer_uptodate(bh)) { + ext4_msg(sbi->s_sb, KERN_ERR, "previous I/O error to " + "superblock detected"); + clear_buffer_write_io_error(bh); + set_buffer_uptodate(bh); + } + + err = ext4_handle_dirty_metadata(handle, NULL, bh); + if (err) + goto out_err; + err = sync_dirty_buffer(bh); +out_err: + ext4_std_error(sb, err); + return err; +} + +/* + * Update one backup superblock in the group 'grp' using the callback + * function 'func' and argument 'arg'. If the handle is NULL the + * modification is not journalled. + * + * Returns: 0 when no modification was done (no superblock in the group) + * 1 when the modification was successful + * <0 on error + */ +static int ext4_update_backup_sb(struct super_block *sb, + handle_t *handle, ext4_group_t grp, + ext4_update_sb_callback func, const void *arg) +{ + int err = 0; + ext4_fsblk_t sb_block; + struct buffer_head *bh; + unsigned long offset = 0; + struct ext4_super_block *es; + + if (!ext4_bg_has_super(sb, grp)) + return 0; + + /* + * For the group 0 there is always 1k padding, so we have + * either adjust offset, or sb_block depending on blocksize + */ + if (grp == 0) { + sb_block = 1 * EXT4_MIN_BLOCK_SIZE; + offset = do_div(sb_block, sb->s_blocksize); + } else { + sb_block = ext4_group_first_block_no(sb, grp); + offset = 0; + } + + trace_ext4_update_sb(sb, sb_block, handle ? 1 : 0); + + bh = ext4_sb_bread(sb, sb_block, 0); + if (IS_ERR(bh)) + return PTR_ERR(bh); + + if (handle) { + BUFFER_TRACE(bh, "get_write_access"); + err = ext4_journal_get_write_access(handle, sb, + bh, + EXT4_JTR_NONE); + if (err) + goto out_bh; + } + + es = (struct ext4_super_block *) (bh->b_data + offset); + lock_buffer(bh); + if (ext4_has_feature_metadata_csum(sb) && + es->s_checksum != ext4_superblock_csum(es)) { + ext4_msg(sb, KERN_ERR, "Invalid checksum for backup " + "superblock %llu", sb_block); + unlock_buffer(bh); + goto out_bh; + } + func(EXT4_SB(sb), es, arg); + if (ext4_has_feature_metadata_csum(sb)) + es->s_checksum = ext4_superblock_csum(es); + set_buffer_uptodate(bh); + unlock_buffer(bh); + + if (handle) { + err = ext4_handle_dirty_metadata(handle, NULL, bh); + if (err) + goto out_bh; + } else { + BUFFER_TRACE(bh, "marking dirty"); + mark_buffer_dirty(bh); + } + err = sync_dirty_buffer(bh); + +out_bh: + brelse(bh); + ext4_std_error(sb, err); + return (err) ? err : 1; +} + +/* + * Update primary and backup superblocks using the provided function + * func and argument arg. + * + * Only the primary superblock and at most two backup superblock + * modifications are journalled; the rest is modified without journal. + * This is safe because e2fsck will re-write them if there is a problem, + * and we're very unlikely to ever need more than two backups. + */ +static +int ext4_update_superblocks_fn(struct super_block *sb, + ext4_update_sb_callback func, + const void *arg) +{ + handle_t *handle; + ext4_group_t ngroups; + unsigned int three = 1; + unsigned int five = 5; + unsigned int seven = 7; + int err = 0, ret, i; + ext4_group_t grp, primary_grp; + struct ext4_sb_info *sbi = EXT4_SB(sb); + + /* + * We can't update superblocks while the online resize is running + */ + if (test_and_set_bit_lock(EXT4_FLAGS_RESIZING, + &sbi->s_ext4_flags)) { + ext4_msg(sb, KERN_ERR, "Can't modify superblock while" + "performing online resize"); + return -EBUSY; + } + + /* + * We're only going to update primary superblock and two + * backup superblocks in this transaction. + */ + handle = ext4_journal_start_sb(sb, EXT4_HT_MISC, 3); + if (IS_ERR(handle)) { + err = PTR_ERR(handle); + goto out; + } + + /* Update primary superblock */ + err = ext4_update_primary_sb(sb, handle, func, arg); + if (err) { + ext4_msg(sb, KERN_ERR, "Failed to update primary " + "superblock"); + goto out_journal; + } + + primary_grp = ext4_get_group_number(sb, sbi->s_sbh->b_blocknr); + ngroups = ext4_get_groups_count(sb); + + /* + * Update backup superblocks. We have to start from group 0 + * because it might not be where the primary superblock is + * if the fs is mounted with -o sb= + */ + i = 0; + grp = 0; + while (grp < ngroups) { + /* Skip primary superblock */ + if (grp == primary_grp) + goto next_grp; + + ret = ext4_update_backup_sb(sb, handle, grp, func, arg); + if (ret < 0) { + /* Ignore bad checksum; try to update next sb */ + if (ret == -EFSBADCRC) + goto next_grp; + err = ret; + goto out_journal; + } + + i += ret; + if (handle && i > 1) { + /* + * We're only journalling primary superblock and + * two backup superblocks; the rest is not + * journalled. + */ + err = ext4_journal_stop(handle); + if (err) + goto out; + handle = NULL; + } +next_grp: + grp = ext4_list_backups(sb, &three, &five, &seven); + } + +out_journal: + if (handle) { + ret = ext4_journal_stop(handle); + if (ret && !err) + err = ret; + } +out: + clear_bit_unlock(EXT4_FLAGS_RESIZING, &sbi->s_ext4_flags); + smp_mb__after_atomic(); + return err ? err : 0; +} + +/* + * Swap memory between @a and @b for @len bytes. + * + * @a: pointer to first memory area + * @b: pointer to second memory area + * @len: number of bytes to swap + * + */ +static void memswap(void *a, void *b, size_t len) +{ + unsigned char *ap, *bp; + + ap = (unsigned char *)a; + bp = (unsigned char *)b; + while (len-- > 0) { + swap(*ap, *bp); + ap++; + bp++; + } +} + +/* + * Swap i_data and associated attributes between @inode1 and @inode2. + * This function is used for the primary swap between inode1 and inode2 + * and also to revert this primary swap in case of errors. + * + * Therefore you have to make sure, that calling this method twice + * will revert all changes. + * + * @inode1: pointer to first inode + * @inode2: pointer to second inode + */ +static void swap_inode_data(struct inode *inode1, struct inode *inode2) +{ + loff_t isize; + struct ext4_inode_info *ei1; + struct ext4_inode_info *ei2; + unsigned long tmp; + struct timespec64 ts1, ts2; + + ei1 = EXT4_I(inode1); + ei2 = EXT4_I(inode2); + + swap(inode1->i_version, inode2->i_version); + + ts1 = inode_get_atime(inode1); + ts2 = inode_get_atime(inode2); + inode_set_atime_to_ts(inode1, ts2); + inode_set_atime_to_ts(inode2, ts1); + + ts1 = inode_get_mtime(inode1); + ts2 = inode_get_mtime(inode2); + inode_set_mtime_to_ts(inode1, ts2); + inode_set_mtime_to_ts(inode2, ts1); + + memswap(ei1->i_data, ei2->i_data, sizeof(ei1->i_data)); + tmp = ei1->i_flags & EXT4_FL_SHOULD_SWAP; + ei1->i_flags = (ei2->i_flags & EXT4_FL_SHOULD_SWAP) | + (ei1->i_flags & ~EXT4_FL_SHOULD_SWAP); + ei2->i_flags = tmp | (ei2->i_flags & ~EXT4_FL_SHOULD_SWAP); + swap(ei1->i_disksize, ei2->i_disksize); + ext4_es_remove_extent(inode1, 0, EXT_MAX_BLOCKS); + ext4_es_remove_extent(inode2, 0, EXT_MAX_BLOCKS); + + isize = i_size_read(inode1); + i_size_write(inode1, i_size_read(inode2)); + i_size_write(inode2, isize); +} + +void ext4_reset_inode_seed(struct inode *inode) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + __le32 inum = cpu_to_le32(inode->i_ino); + __le32 gen = cpu_to_le32(inode->i_generation); + __u32 csum; + + if (!ext4_has_feature_metadata_csum(inode->i_sb)) + return; + + csum = ext4_chksum(sbi->s_csum_seed, (__u8 *)&inum, sizeof(inum)); + ei->i_csum_seed = ext4_chksum(csum, (__u8 *)&gen, sizeof(gen)); +} + +/* + * Swap the information from the given @inode and the inode + * EXT4_BOOT_LOADER_INO. It will basically swap i_data and all other + * important fields of the inodes. + * + * @sb: the super block of the filesystem + * @idmap: idmap of the mount the inode was found from + * @inode: the inode to swap with EXT4_BOOT_LOADER_INO + * + */ +static long swap_inode_boot_loader(struct super_block *sb, + struct mnt_idmap *idmap, + struct inode *inode) +{ + handle_t *handle; + int err; + struct inode *inode_bl; + struct ext4_inode_info *ei_bl; + qsize_t size, size_bl, diff; + blkcnt_t blocks; + unsigned short bytes; + + inode_bl = ext4_iget(sb, EXT4_BOOT_LOADER_INO, + EXT4_IGET_SPECIAL | EXT4_IGET_BAD); + if (IS_ERR(inode_bl)) + return PTR_ERR(inode_bl); + ei_bl = EXT4_I(inode_bl); + + /* Protect orig inodes against a truncate and make sure, + * that only 1 swap_inode_boot_loader is running. */ + lock_two_nondirectories(inode, inode_bl); + + if (inode->i_nlink != 1 || !S_ISREG(inode->i_mode) || + IS_SWAPFILE(inode) || IS_ENCRYPTED(inode) || + (EXT4_I(inode)->i_flags & EXT4_JOURNAL_DATA_FL) || + ext4_has_inline_data(inode)) { + err = -EINVAL; + goto journal_err_out; + } + + if (IS_RDONLY(inode) || IS_APPEND(inode) || IS_IMMUTABLE(inode) || + !inode_owner_or_capable(idmap, inode) || + !capable(CAP_SYS_ADMIN)) { + err = -EPERM; + goto journal_err_out; + } + + filemap_invalidate_lock(inode->i_mapping); + err = filemap_write_and_wait(inode->i_mapping); + if (err) + goto err_out; + + err = filemap_write_and_wait(inode_bl->i_mapping); + if (err) + goto err_out; + + /* Wait for all existing dio workers */ + inode_dio_wait(inode); + inode_dio_wait(inode_bl); + + truncate_inode_pages(&inode->i_data, 0); + truncate_inode_pages(&inode_bl->i_data, 0); + + handle = ext4_journal_start(inode_bl, EXT4_HT_MOVE_EXTENTS, 2); + if (IS_ERR(handle)) { + err = -EINVAL; + goto err_out; + } + ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_SWAP_BOOT, handle); + + /* Protect extent tree against block allocations via delalloc */ + ext4_double_down_write_data_sem(inode, inode_bl); + + if (is_bad_inode(inode_bl) || !S_ISREG(inode_bl->i_mode)) { + /* this inode has never been used as a BOOT_LOADER */ + set_nlink(inode_bl, 1); + i_uid_write(inode_bl, 0); + i_gid_write(inode_bl, 0); + inode_bl->i_flags = 0; + ei_bl->i_flags = 0; + inode_set_iversion(inode_bl, 1); + i_size_write(inode_bl, 0); + EXT4_I(inode_bl)->i_disksize = inode_bl->i_size; + inode_bl->i_mode = S_IFREG; + if (ext4_has_feature_extents(sb)) { + ext4_set_inode_flag(inode_bl, EXT4_INODE_EXTENTS); + ext4_ext_tree_init(handle, inode_bl); + } else + memset(ei_bl->i_data, 0, sizeof(ei_bl->i_data)); + } + + err = dquot_initialize(inode); + if (err) + goto err_out1; + + size = (qsize_t)(inode->i_blocks) * (1 << 9) + inode->i_bytes; + size_bl = (qsize_t)(inode_bl->i_blocks) * (1 << 9) + inode_bl->i_bytes; + diff = size - size_bl; + swap_inode_data(inode, inode_bl); + + inode_set_ctime_current(inode); + inode_set_ctime_current(inode_bl); + inode_inc_iversion(inode); + + inode->i_generation = get_random_u32(); + inode_bl->i_generation = get_random_u32(); + ext4_reset_inode_seed(inode); + ext4_reset_inode_seed(inode_bl); + + ext4_discard_preallocations(inode); + + err = ext4_mark_inode_dirty(handle, inode); + if (err < 0) { + /* No need to update quota information. */ + ext4_warning(inode->i_sb, + "couldn't mark inode #%llu dirty (err %d)", + inode->i_ino, err); + /* Revert all changes: */ + swap_inode_data(inode, inode_bl); + ext4_mark_inode_dirty(handle, inode); + goto err_out1; + } + + blocks = inode_bl->i_blocks; + bytes = inode_bl->i_bytes; + inode_bl->i_blocks = inode->i_blocks; + inode_bl->i_bytes = inode->i_bytes; + err = ext4_mark_inode_dirty(handle, inode_bl); + if (err < 0) { + /* No need to update quota information. */ + ext4_warning(inode_bl->i_sb, + "couldn't mark inode #%llu dirty (err %d)", + inode_bl->i_ino, err); + goto revert; + } + + /* Bootloader inode should not be counted into quota information. */ + if (diff > 0) + dquot_free_space(inode, diff); + else + err = dquot_alloc_space(inode, -1 * diff); + + if (err < 0) { +revert: + /* Revert all changes: */ + inode_bl->i_blocks = blocks; + inode_bl->i_bytes = bytes; + swap_inode_data(inode, inode_bl); + ext4_mark_inode_dirty(handle, inode); + ext4_mark_inode_dirty(handle, inode_bl); + } + +err_out1: + ext4_journal_stop(handle); + ext4_double_up_write_data_sem(inode, inode_bl); + +err_out: + filemap_invalidate_unlock(inode->i_mapping); +journal_err_out: + unlock_two_nondirectories(inode, inode_bl); + iput(inode_bl); + return err; +} + +/* + * If immutable is set and we are not clearing it, we're not allowed to change + * anything else in the inode. Don't error out if we're only trying to set + * immutable on an immutable file. + */ +static int ext4_ioctl_check_immutable(struct inode *inode, __u32 new_projid, + unsigned int flags) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + unsigned int oldflags = ei->i_flags; + + if (!(oldflags & EXT4_IMMUTABLE_FL) || !(flags & EXT4_IMMUTABLE_FL)) + return 0; + + if ((oldflags & ~EXT4_IMMUTABLE_FL) != (flags & ~EXT4_IMMUTABLE_FL)) + return -EPERM; + if (ext4_has_feature_project(inode->i_sb) && + __kprojid_val(ei->i_projid) != new_projid) + return -EPERM; + + return 0; +} + +static void ext4_dax_dontcache(struct inode *inode, unsigned int flags) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + + if (S_ISDIR(inode->i_mode)) + return; + + if (test_opt2(inode->i_sb, DAX_NEVER) || + test_opt(inode->i_sb, DAX_ALWAYS)) + return; + + if ((ei->i_flags ^ flags) & EXT4_DAX_FL) + d_mark_dontcache(inode); +} + +static bool dax_compatible(struct inode *inode, unsigned int oldflags, + unsigned int flags) +{ + /* Allow the DAX flag to be changed on inline directories */ + if (S_ISDIR(inode->i_mode)) { + flags &= ~EXT4_INLINE_DATA_FL; + oldflags &= ~EXT4_INLINE_DATA_FL; + } + + if (flags & EXT4_DAX_FL) { + if ((oldflags & EXT4_DAX_MUT_EXCL) || + ext4_test_inode_state(inode, + EXT4_STATE_VERITY_IN_PROGRESS)) { + return false; + } + } + + if ((flags & EXT4_DAX_MUT_EXCL) && (oldflags & EXT4_DAX_FL)) + return false; + + return true; +} + +static int ext4_ioctl_setflags(struct inode *inode, + unsigned int flags) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + handle_t *handle = NULL; + int err = -EPERM, migrate = 0; + struct ext4_iloc iloc; + unsigned int oldflags, mask, i; + struct super_block *sb = inode->i_sb; + + /* Is it quota file? Do not allow user to mess with it */ + if (ext4_is_quota_file(inode)) + goto flags_out; + + oldflags = ei->i_flags; + /* + * The JOURNAL_DATA flag can only be changed by + * the relevant capability. + */ + if ((flags ^ oldflags) & (EXT4_JOURNAL_DATA_FL)) { + if (!capable(CAP_SYS_RESOURCE)) + goto flags_out; + } + + if (!dax_compatible(inode, oldflags, flags)) { + err = -EOPNOTSUPP; + goto flags_out; + } + + if ((flags ^ oldflags) & EXT4_EXTENTS_FL) + migrate = 1; + + if ((flags ^ oldflags) & EXT4_CASEFOLD_FL) { + if (!ext4_has_feature_casefold(sb)) { + err = -EOPNOTSUPP; + goto flags_out; + } + + if (!S_ISDIR(inode->i_mode)) { + err = -ENOTDIR; + goto flags_out; + } + + if (!ext4_empty_dir(inode)) { + err = -ENOTEMPTY; + goto flags_out; + } + } + + /* + * Wait for all pending directio and then flush all the dirty pages + * for this file. The flush marks all the pages readonly, so any + * subsequent attempt to write to the file (particularly mmap pages) + * will come through the filesystem and fail. + */ + if (S_ISREG(inode->i_mode) && !IS_IMMUTABLE(inode) && + (flags & EXT4_IMMUTABLE_FL)) { + inode_dio_wait(inode); + err = filemap_write_and_wait(inode->i_mapping); + if (err) + goto flags_out; + } + + handle = ext4_journal_start(inode, EXT4_HT_INODE, 1); + if (IS_ERR(handle)) { + err = PTR_ERR(handle); + goto flags_out; + } + if (IS_SYNC(inode)) + ext4_handle_sync(handle); + err = ext4_reserve_inode_write(handle, inode, &iloc); + if (err) + goto flags_err; + + ext4_dax_dontcache(inode, flags); + + for (i = 0, mask = 1; i < 32; i++, mask <<= 1) { + if (!(mask & EXT4_FL_USER_MODIFIABLE)) + continue; + /* These flags get special treatment later */ + if (mask == EXT4_JOURNAL_DATA_FL || mask == EXT4_EXTENTS_FL) + continue; + if (mask & flags) + ext4_set_inode_flag(inode, i); + else + ext4_clear_inode_flag(inode, i); + } + + ext4_set_inode_flags(inode, false); + + inode_set_ctime_current(inode); + inode_inc_iversion(inode); + + err = ext4_mark_iloc_dirty(handle, inode, &iloc); +flags_err: + ext4_journal_stop(handle); + if (err) + goto flags_out; + + if ((flags ^ oldflags) & (EXT4_JOURNAL_DATA_FL)) { + /* + * Changes to the journaling mode can cause unsafe changes to + * S_DAX if the inode is DAX + */ + if (IS_DAX(inode)) { + err = -EBUSY; + goto flags_out; + } + + err = ext4_change_inode_journal_flag(inode, + flags & EXT4_JOURNAL_DATA_FL); + if (err) + goto flags_out; + } + if (migrate) { + if (flags & EXT4_EXTENTS_FL) + err = ext4_ext_migrate(inode); + else + err = ext4_ind_migrate(inode); + } + +flags_out: + return err; +} + +#ifdef CONFIG_QUOTA +static int ext4_ioctl_setproject(struct inode *inode, __u32 projid) +{ + struct super_block *sb = inode->i_sb; + struct ext4_inode_info *ei = EXT4_I(inode); + int err, rc; + handle_t *handle; + kprojid_t kprojid; + struct ext4_iloc iloc; + struct ext4_inode *raw_inode; + struct dquot *transfer_to[MAXQUOTAS] = { }; + + if (!ext4_has_feature_project(sb)) { + if (projid != EXT4_DEF_PROJID) + return -EOPNOTSUPP; + else + return 0; + } + + if (EXT4_INODE_SIZE(sb) <= EXT4_GOOD_OLD_INODE_SIZE) + return -EOPNOTSUPP; + + kprojid = make_kprojid(&init_user_ns, (projid_t)projid); + + if (projid_eq(kprojid, EXT4_I(inode)->i_projid)) + return 0; + + err = -EPERM; + /* Is it quota file? Do not allow user to mess with it */ + if (ext4_is_quota_file(inode)) + return err; + + err = dquot_initialize(inode); + if (err) + return err; + + err = ext4_get_inode_loc(inode, &iloc); + if (err) + return err; + + raw_inode = ext4_raw_inode(&iloc); + if (!EXT4_FITS_IN_INODE(raw_inode, ei, i_projid)) { + err = ext4_expand_extra_isize(inode, + EXT4_SB(sb)->s_want_extra_isize, + &iloc); + if (err) + return err; + } else { + brelse(iloc.bh); + } + + handle = ext4_journal_start(inode, EXT4_HT_QUOTA, + EXT4_QUOTA_INIT_BLOCKS(sb) + + EXT4_QUOTA_DEL_BLOCKS(sb) + 3); + if (IS_ERR(handle)) + return PTR_ERR(handle); + + err = ext4_reserve_inode_write(handle, inode, &iloc); + if (err) + goto out_stop; + + transfer_to[PRJQUOTA] = dqget(sb, make_kqid_projid(kprojid)); + if (!IS_ERR(transfer_to[PRJQUOTA])) { + + /* __dquot_transfer() calls back ext4_get_inode_usage() which + * counts xattr inode references. + */ + down_read(&EXT4_I(inode)->xattr_sem); + err = __dquot_transfer(inode, transfer_to); + up_read(&EXT4_I(inode)->xattr_sem); + dqput(transfer_to[PRJQUOTA]); + if (err) + goto out_dirty; + } + + EXT4_I(inode)->i_projid = kprojid; + inode_set_ctime_current(inode); + inode_inc_iversion(inode); +out_dirty: + rc = ext4_mark_iloc_dirty(handle, inode, &iloc); + if (!err) + err = rc; +out_stop: + ext4_journal_stop(handle); + return err; +} +#else +static int ext4_ioctl_setproject(struct inode *inode, __u32 projid) +{ + if (projid != EXT4_DEF_PROJID) + return -EOPNOTSUPP; + return 0; +} +#endif + +int ext4_force_shutdown(struct super_block *sb, u32 flags) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + int ret; + + if (flags > EXT4_GOING_FLAGS_NOLOGFLUSH) + return -EINVAL; + + if (ext4_forced_shutdown(sb)) + return 0; + + ext4_msg(sb, KERN_ALERT, "shut down requested (%d)", flags); + trace_ext4_shutdown(sb, flags); + + switch (flags) { + case EXT4_GOING_FLAGS_DEFAULT: + ret = bdev_freeze(sb->s_bdev); + if (ret) + return ret; + set_bit(EXT4_FLAGS_SHUTDOWN, &sbi->s_ext4_flags); + bdev_thaw(sb->s_bdev); + break; + case EXT4_GOING_FLAGS_LOGFLUSH: + /* + * Call ext4_force_commit() before setting EXT4_FLAGS_SHUTDOWN. + * This is because in data=ordered mode, journal commit + * triggers data writeback which fails if shutdown is already + * set, causing the journal to be aborted prematurely before + * the commit succeeds. + */ + (void) ext4_force_commit(sb); + set_bit(EXT4_FLAGS_SHUTDOWN, &sbi->s_ext4_flags); + if (sbi->s_journal && !is_journal_aborted(sbi->s_journal)) + jbd2_journal_abort(sbi->s_journal, -ESHUTDOWN); + break; + case EXT4_GOING_FLAGS_NOLOGFLUSH: + set_bit(EXT4_FLAGS_SHUTDOWN, &sbi->s_ext4_flags); + if (sbi->s_journal && !is_journal_aborted(sbi->s_journal)) + jbd2_journal_abort(sbi->s_journal, -ESHUTDOWN); + break; + default: + return -EINVAL; + } + clear_opt(sb, DISCARD); + fserror_report_shutdown(sb, GFP_KERNEL); + return 0; +} + +static int ext4_ioctl_shutdown(struct super_block *sb, unsigned long arg) +{ + u32 flags; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (get_user(flags, (__u32 __user *)arg)) + return -EFAULT; + + return ext4_force_shutdown(sb, flags); +} + +struct getfsmap_info { + struct super_block *gi_sb; + struct fsmap_head __user *gi_data; + unsigned int gi_idx; + __u32 gi_last_flags; +}; + +static int ext4_getfsmap_format(struct ext4_fsmap *xfm, void *priv) +{ + struct getfsmap_info *info = priv; + struct fsmap fm; + + trace_ext4_getfsmap_mapping(info->gi_sb, xfm); + + info->gi_last_flags = xfm->fmr_flags; + ext4_fsmap_from_internal(info->gi_sb, &fm, xfm); + if (copy_to_user(&info->gi_data->fmh_recs[info->gi_idx++], &fm, + sizeof(struct fsmap))) + return -EFAULT; + + return 0; +} + +static int ext4_ioc_getfsmap(struct super_block *sb, + struct fsmap_head __user *arg) +{ + struct getfsmap_info info = { NULL }; + struct ext4_fsmap_head xhead = {0}; + struct fsmap_head head; + bool aborted = false; + int error; + + if (copy_from_user(&head, arg, sizeof(struct fsmap_head))) + return -EFAULT; + if (memchr_inv(head.fmh_reserved, 0, sizeof(head.fmh_reserved)) || + memchr_inv(head.fmh_keys[0].fmr_reserved, 0, + sizeof(head.fmh_keys[0].fmr_reserved)) || + memchr_inv(head.fmh_keys[1].fmr_reserved, 0, + sizeof(head.fmh_keys[1].fmr_reserved))) + return -EINVAL; + /* + * ext4 doesn't report file extents at all, so the only valid + * file offsets are the magic ones (all zeroes or all ones). + */ + if (head.fmh_keys[0].fmr_offset || + (head.fmh_keys[1].fmr_offset != 0 && + head.fmh_keys[1].fmr_offset != -1ULL)) + return -EINVAL; + + xhead.fmh_iflags = head.fmh_iflags; + xhead.fmh_count = head.fmh_count; + ext4_fsmap_to_internal(sb, &xhead.fmh_keys[0], &head.fmh_keys[0]); + ext4_fsmap_to_internal(sb, &xhead.fmh_keys[1], &head.fmh_keys[1]); + + trace_ext4_getfsmap_low_key(sb, &xhead.fmh_keys[0]); + trace_ext4_getfsmap_high_key(sb, &xhead.fmh_keys[1]); + + info.gi_sb = sb; + info.gi_data = arg; + error = ext4_getfsmap(sb, &xhead, ext4_getfsmap_format, &info); + if (error == EXT4_QUERY_RANGE_ABORT) + aborted = true; + else if (error) + return error; + + /* If we didn't abort, set the "last" flag in the last fmx */ + if (!aborted && info.gi_idx) { + info.gi_last_flags |= FMR_OF_LAST; + if (copy_to_user(&info.gi_data->fmh_recs[info.gi_idx - 1].fmr_flags, + &info.gi_last_flags, + sizeof(info.gi_last_flags))) + return -EFAULT; + } + + /* copy back header */ + head.fmh_entries = xhead.fmh_entries; + head.fmh_oflags = xhead.fmh_oflags; + if (copy_to_user(arg, &head, sizeof(struct fsmap_head))) + return -EFAULT; + + return 0; +} + +static long ext4_ioctl_group_add(struct file *file, + struct ext4_new_group_data *input) +{ + struct super_block *sb = file_inode(file)->i_sb; + int err, err2=0; + + err = ext4_resize_begin(sb); + if (err) + return err; + + if (ext4_has_feature_bigalloc(sb)) { + ext4_msg(sb, KERN_ERR, + "Online resizing not supported with bigalloc"); + err = -EOPNOTSUPP; + goto group_add_out; + } + + err = mnt_want_write_file(file); + if (err) + goto group_add_out; + + err = ext4_group_add(sb, input); + if (EXT4_SB(sb)->s_journal) { + ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_RESIZE, NULL); + jbd2_journal_lock_updates(EXT4_SB(sb)->s_journal); + err2 = jbd2_journal_flush(EXT4_SB(sb)->s_journal, 0); + jbd2_journal_unlock_updates(EXT4_SB(sb)->s_journal); + } + if (err == 0) + err = err2; + mnt_drop_write_file(file); + if (!err && ext4_has_group_desc_csum(sb) && + test_opt(sb, INIT_INODE_TABLE)) + err = ext4_register_li_request(sb, input->group); +group_add_out: + err2 = ext4_resize_end(sb, false); + if (err == 0) + err = err2; + return err; +} + +int ext4_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + struct ext4_inode_info *ei = EXT4_I(inode); + u32 flags = ei->i_flags & EXT4_FL_USER_VISIBLE; + + if (S_ISREG(inode->i_mode)) + flags &= ~FS_PROJINHERIT_FL; + + fileattr_fill_flags(fa, flags); + if (ext4_has_feature_project(inode->i_sb)) + fa->fsx_projid = from_kprojid(&init_user_ns, ei->i_projid); + + return 0; +} + +int ext4_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + u32 flags = fa->flags; + int err = -EOPNOTSUPP; + + if (flags & ~EXT4_FL_USER_VISIBLE) + goto out; + + /* + * chattr(1) grabs flags via GETFLAGS, modifies the result and + * passes that to SETFLAGS. So we cannot easily make SETFLAGS + * more restrictive than just silently masking off visible but + * not settable flags as we always did. + */ + flags &= EXT4_FL_USER_MODIFIABLE; + if (ext4_mask_flags(inode->i_mode, flags) != flags) + goto out; + err = ext4_ioctl_check_immutable(inode, fa->fsx_projid, flags); + if (err) + goto out; + err = ext4_ioctl_setflags(inode, flags); + if (err) + goto out; + err = ext4_ioctl_setproject(inode, fa->fsx_projid); +out: + return err; +} + +/* So that the fiemap access checks can't overflow on 32 bit machines. */ +#define FIEMAP_MAX_EXTENTS (UINT_MAX / sizeof(struct fiemap_extent)) + +static int ext4_ioctl_get_es_cache(struct file *filp, unsigned long arg) +{ + struct fiemap fiemap; + struct fiemap __user *ufiemap = (struct fiemap __user *) arg; + struct fiemap_extent_info fieinfo = { 0, }; + struct inode *inode = file_inode(filp); + int error; + + if (copy_from_user(&fiemap, ufiemap, sizeof(fiemap))) + return -EFAULT; + + if (fiemap.fm_extent_count > FIEMAP_MAX_EXTENTS) + return -EINVAL; + + fieinfo.fi_flags = fiemap.fm_flags; + fieinfo.fi_extents_max = fiemap.fm_extent_count; + fieinfo.fi_extents_start = ufiemap->fm_extents; + + error = ext4_get_es_cache(inode, &fieinfo, fiemap.fm_start, + fiemap.fm_length); + fiemap.fm_flags = fieinfo.fi_flags; + fiemap.fm_mapped_extents = fieinfo.fi_extents_mapped; + if (copy_to_user(ufiemap, &fiemap, sizeof(fiemap))) + error = -EFAULT; + + return error; +} + +static int ext4_ioctl_checkpoint(struct file *filp, unsigned long arg) +{ + int err = 0; + __u32 flags = 0; + unsigned int flush_flags = 0; + struct super_block *sb = file_inode(filp)->i_sb; + + if (copy_from_user(&flags, (__u32 __user *)arg, + sizeof(__u32))) + return -EFAULT; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + /* check for invalid bits set */ + if ((flags & ~EXT4_IOC_CHECKPOINT_FLAG_VALID) || + ((flags & JBD2_JOURNAL_FLUSH_DISCARD) && + (flags & JBD2_JOURNAL_FLUSH_ZEROOUT))) + return -EINVAL; + + if (!EXT4_SB(sb)->s_journal) + return -ENODEV; + + if ((flags & JBD2_JOURNAL_FLUSH_DISCARD) && + !bdev_max_discard_sectors(EXT4_SB(sb)->s_journal->j_dev)) + return -EOPNOTSUPP; + + if (flags & EXT4_IOC_CHECKPOINT_FLAG_DRY_RUN) + return 0; + + if (flags & EXT4_IOC_CHECKPOINT_FLAG_DISCARD) + flush_flags |= JBD2_JOURNAL_FLUSH_DISCARD; + + if (flags & EXT4_IOC_CHECKPOINT_FLAG_ZEROOUT) { + flush_flags |= JBD2_JOURNAL_FLUSH_ZEROOUT; + pr_info_ratelimited("warning: checkpointing journal with EXT4_IOC_CHECKPOINT_FLAG_ZEROOUT can be slow"); + } + + jbd2_journal_lock_updates(EXT4_SB(sb)->s_journal); + err = jbd2_journal_flush(EXT4_SB(sb)->s_journal, flush_flags); + jbd2_journal_unlock_updates(EXT4_SB(sb)->s_journal); + + return err; +} + +static int ext4_ioctl_setlabel(struct file *filp, const char __user *user_label) +{ + size_t len; + int ret = 0; + char new_label[EXT4_LABEL_MAX + 1]; + struct super_block *sb = file_inode(filp)->i_sb; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + /* + * Copy the maximum length allowed for ext4 label with one more to + * find the required terminating null byte in order to test the + * label length. The on disk label doesn't need to be null terminated. + */ + if (copy_from_user(new_label, user_label, EXT4_LABEL_MAX + 1)) + return -EFAULT; + + len = strnlen(new_label, EXT4_LABEL_MAX + 1); + if (len > EXT4_LABEL_MAX) + return -EINVAL; + + /* + * Clear the buffer after the new label + */ + memset(new_label + len, 0, EXT4_LABEL_MAX - len); + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + + ret = ext4_update_superblocks_fn(sb, ext4_sb_setlabel, new_label); + + mnt_drop_write_file(filp); + return ret; +} + +static int ext4_ioctl_getlabel(struct ext4_sb_info *sbi, char __user *user_label) +{ + char label[EXT4_LABEL_MAX + 1]; + + /* + * EXT4_LABEL_MAX must always be smaller than FSLABEL_MAX because + * FSLABEL_MAX must include terminating null byte, while s_volume_name + * does not have to. + */ + BUILD_BUG_ON(EXT4_LABEL_MAX >= FSLABEL_MAX); + + lock_buffer(sbi->s_sbh); + memtostr_pad(label, sbi->s_es->s_volume_name); + unlock_buffer(sbi->s_sbh); + + if (copy_to_user(user_label, label, sizeof(label))) + return -EFAULT; + return 0; +} + +static int ext4_ioctl_getuuid(struct ext4_sb_info *sbi, + struct fsuuid __user *ufsuuid) +{ + struct fsuuid fsuuid; + __u8 uuid[UUID_SIZE]; + + if (copy_from_user(&fsuuid, ufsuuid, sizeof(fsuuid))) + return -EFAULT; + + if (fsuuid.fsu_len == 0) { + fsuuid.fsu_len = UUID_SIZE; + if (copy_to_user(&ufsuuid->fsu_len, &fsuuid.fsu_len, + sizeof(fsuuid.fsu_len))) + return -EFAULT; + return 0; + } + + if (fsuuid.fsu_len < UUID_SIZE || fsuuid.fsu_flags != 0) + return -EINVAL; + + lock_buffer(sbi->s_sbh); + memcpy(uuid, sbi->s_es->s_uuid, UUID_SIZE); + unlock_buffer(sbi->s_sbh); + + fsuuid.fsu_len = UUID_SIZE; + if (copy_to_user(ufsuuid, &fsuuid, sizeof(fsuuid)) || + copy_to_user(&ufsuuid->fsu_uuid[0], uuid, UUID_SIZE)) + return -EFAULT; + return 0; +} + +static int ext4_ioctl_setuuid(struct file *filp, + const struct fsuuid __user *ufsuuid) +{ + int ret = 0; + struct super_block *sb = file_inode(filp)->i_sb; + struct fsuuid fsuuid; + __u8 uuid[UUID_SIZE]; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + /* + * If any checksums (group descriptors or metadata) are being used + * then the checksum seed feature is required to change the UUID. + */ + if (((ext4_has_feature_gdt_csum(sb) || + ext4_has_feature_metadata_csum(sb)) + && !ext4_has_feature_csum_seed(sb)) + || ext4_has_feature_stable_inodes(sb)) + return -EOPNOTSUPP; + + if (copy_from_user(&fsuuid, ufsuuid, sizeof(fsuuid))) + return -EFAULT; + + if (fsuuid.fsu_len != UUID_SIZE || fsuuid.fsu_flags != 0) + return -EINVAL; + + if (copy_from_user(uuid, &ufsuuid->fsu_uuid[0], UUID_SIZE)) + return -EFAULT; + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + + ret = ext4_update_superblocks_fn(sb, ext4_sb_setuuid, &uuid); + mnt_drop_write_file(filp); + + return ret; +} + + +#define TUNE_OPS_SUPPORTED (EXT4_TUNE_FL_ERRORS_BEHAVIOR | \ + EXT4_TUNE_FL_MNT_COUNT | EXT4_TUNE_FL_MAX_MNT_COUNT | \ + EXT4_TUNE_FL_CHECKINTRVAL | EXT4_TUNE_FL_LAST_CHECK_TIME | \ + EXT4_TUNE_FL_RESERVED_BLOCKS | EXT4_TUNE_FL_RESERVED_UID | \ + EXT4_TUNE_FL_RESERVED_GID | EXT4_TUNE_FL_DEFAULT_MNT_OPTS | \ + EXT4_TUNE_FL_DEF_HASH_ALG | EXT4_TUNE_FL_RAID_STRIDE | \ + EXT4_TUNE_FL_RAID_STRIPE_WIDTH | EXT4_TUNE_FL_MOUNT_OPTS | \ + EXT4_TUNE_FL_FEATURES | EXT4_TUNE_FL_EDIT_FEATURES | \ + EXT4_TUNE_FL_FORCE_FSCK | EXT4_TUNE_FL_ENCODING | \ + EXT4_TUNE_FL_ENCODING_FLAGS) + +#define EXT4_TUNE_SET_COMPAT_SUPP \ + (EXT4_FEATURE_COMPAT_DIR_INDEX | \ + EXT4_FEATURE_COMPAT_STABLE_INODES) +#define EXT4_TUNE_SET_INCOMPAT_SUPP \ + (EXT4_FEATURE_INCOMPAT_EXTENTS | \ + EXT4_FEATURE_INCOMPAT_EA_INODE | \ + EXT4_FEATURE_INCOMPAT_ENCRYPT | \ + EXT4_FEATURE_INCOMPAT_CSUM_SEED | \ + EXT4_FEATURE_INCOMPAT_LARGEDIR | \ + EXT4_FEATURE_INCOMPAT_CASEFOLD) +#define EXT4_TUNE_SET_RO_COMPAT_SUPP \ + (EXT4_FEATURE_RO_COMPAT_LARGE_FILE | \ + EXT4_FEATURE_RO_COMPAT_DIR_NLINK | \ + EXT4_FEATURE_RO_COMPAT_EXTRA_ISIZE | \ + EXT4_FEATURE_RO_COMPAT_PROJECT | \ + EXT4_FEATURE_RO_COMPAT_VERITY) + +#define EXT4_TUNE_CLEAR_COMPAT_SUPP (0) +#define EXT4_TUNE_CLEAR_INCOMPAT_SUPP (0) +#define EXT4_TUNE_CLEAR_RO_COMPAT_SUPP (0) + +#define SB_ENC_SUPP_MASK (SB_ENC_STRICT_MODE_FL | \ + SB_ENC_NO_COMPAT_FALLBACK_FL) + +static int ext4_ioctl_get_tune_sb(struct ext4_sb_info *sbi, + struct ext4_tune_sb_params __user *params) +{ + struct ext4_tune_sb_params ret; + struct ext4_super_block *es = sbi->s_es; + + memset(&ret, 0, sizeof(ret)); + ret.set_flags = TUNE_OPS_SUPPORTED; + ret.errors_behavior = le16_to_cpu(es->s_errors); + ret.mnt_count = le16_to_cpu(es->s_mnt_count); + ret.max_mnt_count = le16_to_cpu(es->s_max_mnt_count); + ret.checkinterval = le32_to_cpu(es->s_checkinterval); + ret.last_check_time = le32_to_cpu(es->s_lastcheck); + ret.reserved_blocks = ext4_r_blocks_count(es); + ret.blocks_count = ext4_blocks_count(es); + ret.reserved_uid = ext4_get_resuid(es); + ret.reserved_gid = ext4_get_resgid(es); + ret.default_mnt_opts = le32_to_cpu(es->s_default_mount_opts); + ret.def_hash_alg = es->s_def_hash_version; + ret.raid_stride = le16_to_cpu(es->s_raid_stride); + ret.raid_stripe_width = le32_to_cpu(es->s_raid_stripe_width); + ret.encoding = le16_to_cpu(es->s_encoding); + ret.encoding_flags = le16_to_cpu(es->s_encoding_flags); + strscpy_pad(ret.mount_opts, es->s_mount_opts); + ret.feature_compat = le32_to_cpu(es->s_feature_compat); + ret.feature_incompat = le32_to_cpu(es->s_feature_incompat); + ret.feature_ro_compat = le32_to_cpu(es->s_feature_ro_compat); + ret.set_feature_compat_mask = EXT4_TUNE_SET_COMPAT_SUPP; + ret.set_feature_incompat_mask = EXT4_TUNE_SET_INCOMPAT_SUPP; + ret.set_feature_ro_compat_mask = EXT4_TUNE_SET_RO_COMPAT_SUPP; + ret.clear_feature_compat_mask = EXT4_TUNE_CLEAR_COMPAT_SUPP; + ret.clear_feature_incompat_mask = EXT4_TUNE_CLEAR_INCOMPAT_SUPP; + ret.clear_feature_ro_compat_mask = EXT4_TUNE_CLEAR_RO_COMPAT_SUPP; + if (copy_to_user(params, &ret, sizeof(ret))) + return -EFAULT; + return 0; +} + +static void ext4_sb_setparams(struct ext4_sb_info *sbi, + struct ext4_super_block *es, const void *arg) +{ + const struct ext4_tune_sb_params *params = arg; + + if (params->set_flags & EXT4_TUNE_FL_ERRORS_BEHAVIOR) + es->s_errors = cpu_to_le16(params->errors_behavior); + if (params->set_flags & EXT4_TUNE_FL_MNT_COUNT) + es->s_mnt_count = cpu_to_le16(params->mnt_count); + if (params->set_flags & EXT4_TUNE_FL_MAX_MNT_COUNT) + es->s_max_mnt_count = cpu_to_le16(params->max_mnt_count); + if (params->set_flags & EXT4_TUNE_FL_CHECKINTRVAL) + es->s_checkinterval = cpu_to_le32(params->checkinterval); + if (params->set_flags & EXT4_TUNE_FL_LAST_CHECK_TIME) + es->s_lastcheck = cpu_to_le32(params->last_check_time); + if (params->set_flags & EXT4_TUNE_FL_RESERVED_BLOCKS) { + ext4_fsblk_t blk = params->reserved_blocks; + + es->s_r_blocks_count_lo = cpu_to_le32((u32)blk); + es->s_r_blocks_count_hi = cpu_to_le32(blk >> 32); + } + if (params->set_flags & EXT4_TUNE_FL_RESERVED_UID) { + int uid = params->reserved_uid; + + es->s_def_resuid = cpu_to_le16(uid & 0xFFFF); + es->s_def_resuid_hi = cpu_to_le16(uid >> 16); + } + if (params->set_flags & EXT4_TUNE_FL_RESERVED_GID) { + int gid = params->reserved_gid; + + es->s_def_resgid = cpu_to_le16(gid & 0xFFFF); + es->s_def_resgid_hi = cpu_to_le16(gid >> 16); + } + if (params->set_flags & EXT4_TUNE_FL_DEFAULT_MNT_OPTS) + es->s_default_mount_opts = cpu_to_le32(params->default_mnt_opts); + if (params->set_flags & EXT4_TUNE_FL_DEF_HASH_ALG) + es->s_def_hash_version = params->def_hash_alg; + if (params->set_flags & EXT4_TUNE_FL_RAID_STRIDE) + es->s_raid_stride = cpu_to_le16(params->raid_stride); + if (params->set_flags & EXT4_TUNE_FL_RAID_STRIPE_WIDTH) + es->s_raid_stripe_width = + cpu_to_le32(params->raid_stripe_width); + if (params->set_flags & EXT4_TUNE_FL_ENCODING) + es->s_encoding = cpu_to_le16(params->encoding); + if (params->set_flags & EXT4_TUNE_FL_ENCODING_FLAGS) + es->s_encoding_flags = cpu_to_le16(params->encoding_flags); + strscpy_pad(es->s_mount_opts, params->mount_opts); + if (params->set_flags & EXT4_TUNE_FL_EDIT_FEATURES) { + es->s_feature_compat |= + cpu_to_le32(params->set_feature_compat_mask); + es->s_feature_incompat |= + cpu_to_le32(params->set_feature_incompat_mask); + es->s_feature_ro_compat |= + cpu_to_le32(params->set_feature_ro_compat_mask); + es->s_feature_compat &= + ~cpu_to_le32(params->clear_feature_compat_mask); + es->s_feature_incompat &= + ~cpu_to_le32(params->clear_feature_incompat_mask); + es->s_feature_ro_compat &= + ~cpu_to_le32(params->clear_feature_ro_compat_mask); + if (params->set_feature_compat_mask & + EXT4_FEATURE_COMPAT_DIR_INDEX) + es->s_def_hash_version = sbi->s_def_hash_version; + if (params->set_feature_incompat_mask & + EXT4_FEATURE_INCOMPAT_CSUM_SEED) + es->s_checksum_seed = cpu_to_le32(sbi->s_csum_seed); + } + if (params->set_flags & EXT4_TUNE_FL_FORCE_FSCK) + es->s_state |= cpu_to_le16(EXT4_ERROR_FS); +} + +static int ext4_ioctl_set_tune_sb(struct file *filp, + struct ext4_tune_sb_params __user *in) +{ + struct ext4_tune_sb_params params; + struct super_block *sb = file_inode(filp)->i_sb; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_super_block *es = sbi->s_es; + int enabling_casefold = 0; + int ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (copy_from_user(¶ms, in, sizeof(params))) + return -EFAULT; + + if (strnlen(params.mount_opts, sizeof(params.mount_opts)) == + sizeof(params.mount_opts)) + return -E2BIG; + + if ((params.set_flags & ~TUNE_OPS_SUPPORTED) != 0) + return -EOPNOTSUPP; + + if ((params.set_flags & EXT4_TUNE_FL_ERRORS_BEHAVIOR) && + (params.errors_behavior > EXT4_ERRORS_PANIC)) + return -EINVAL; + + if ((params.set_flags & EXT4_TUNE_FL_RESERVED_BLOCKS) && + (params.reserved_blocks > ext4_blocks_count(sbi->s_es) / 2)) + return -EINVAL; + if ((params.set_flags & EXT4_TUNE_FL_DEF_HASH_ALG) && + ((params.def_hash_alg > DX_HASH_LAST) || + (params.def_hash_alg == DX_HASH_SIPHASH))) + return -EINVAL; + if ((params.set_flags & EXT4_TUNE_FL_FEATURES) && + (params.set_flags & EXT4_TUNE_FL_EDIT_FEATURES)) + return -EINVAL; + + if (params.set_flags & EXT4_TUNE_FL_FEATURES) { + params.set_feature_compat_mask = + params.feature_compat & + ~le32_to_cpu(es->s_feature_compat); + params.set_feature_incompat_mask = + params.feature_incompat & + ~le32_to_cpu(es->s_feature_incompat); + params.set_feature_ro_compat_mask = + params.feature_ro_compat & + ~le32_to_cpu(es->s_feature_ro_compat); + params.clear_feature_compat_mask = + ~params.feature_compat & + le32_to_cpu(es->s_feature_compat); + params.clear_feature_incompat_mask = + ~params.feature_incompat & + le32_to_cpu(es->s_feature_incompat); + params.clear_feature_ro_compat_mask = + ~params.feature_ro_compat & + le32_to_cpu(es->s_feature_ro_compat); + params.set_flags |= EXT4_TUNE_FL_EDIT_FEATURES; + } + if (params.set_flags & EXT4_TUNE_FL_EDIT_FEATURES) { + if ((params.set_feature_compat_mask & + ~EXT4_TUNE_SET_COMPAT_SUPP) || + (params.set_feature_incompat_mask & + ~EXT4_TUNE_SET_INCOMPAT_SUPP) || + (params.set_feature_ro_compat_mask & + ~EXT4_TUNE_SET_RO_COMPAT_SUPP) || + (params.clear_feature_compat_mask & + ~EXT4_TUNE_CLEAR_COMPAT_SUPP) || + (params.clear_feature_incompat_mask & + ~EXT4_TUNE_CLEAR_INCOMPAT_SUPP) || + (params.clear_feature_ro_compat_mask & + ~EXT4_TUNE_CLEAR_RO_COMPAT_SUPP)) + return -EOPNOTSUPP; + + /* + * Filter out the features that are already set from + * the set_mask. + */ + params.set_feature_compat_mask &= + ~le32_to_cpu(es->s_feature_compat); + params.set_feature_incompat_mask &= + ~le32_to_cpu(es->s_feature_incompat); + params.set_feature_ro_compat_mask &= + ~le32_to_cpu(es->s_feature_ro_compat); + if ((params.set_feature_incompat_mask & + EXT4_FEATURE_INCOMPAT_CASEFOLD)) { + enabling_casefold = 1; + if (!(params.set_flags & EXT4_TUNE_FL_ENCODING)) { + params.encoding = EXT4_ENC_UTF8_12_1; + params.set_flags |= EXT4_TUNE_FL_ENCODING; + } + if (!(params.set_flags & EXT4_TUNE_FL_ENCODING_FLAGS)) { + params.encoding_flags = 0; + params.set_flags |= EXT4_TUNE_FL_ENCODING_FLAGS; + } + } + if ((params.set_feature_compat_mask & + EXT4_FEATURE_COMPAT_DIR_INDEX)) { + uuid_t uu; + + memcpy(&uu, sbi->s_hash_seed, UUID_SIZE); + if (uuid_is_null(&uu)) + generate_random_uuid((char *) + &sbi->s_hash_seed); + if (params.set_flags & EXT4_TUNE_FL_DEF_HASH_ALG) + sbi->s_def_hash_version = params.def_hash_alg; + else if (sbi->s_def_hash_version == 0) + sbi->s_def_hash_version = DX_HASH_HALF_MD4; + if (!(es->s_flags & + cpu_to_le32(EXT2_FLAGS_UNSIGNED_HASH)) && + !(es->s_flags & + cpu_to_le32(EXT2_FLAGS_SIGNED_HASH))) { +#ifdef __CHAR_UNSIGNED__ + sbi->s_hash_unsigned = 3; +#else + sbi->s_hash_unsigned = 0; +#endif + } + } + } + if (params.set_flags & EXT4_TUNE_FL_ENCODING) { + if (!enabling_casefold) + return -EINVAL; + if (params.encoding == 0) + params.encoding = EXT4_ENC_UTF8_12_1; + else if (params.encoding != EXT4_ENC_UTF8_12_1) + return -EINVAL; + } + if (params.set_flags & EXT4_TUNE_FL_ENCODING_FLAGS) { + if (!enabling_casefold) + return -EINVAL; + if (params.encoding_flags & ~SB_ENC_SUPP_MASK) + return -EINVAL; + } + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + + ret = ext4_update_superblocks_fn(sb, ext4_sb_setparams, ¶ms); + mnt_drop_write_file(filp); + + if (params.set_flags & EXT4_TUNE_FL_DEF_HASH_ALG) + sbi->s_def_hash_version = params.def_hash_alg; + + return ret; +} + +static long __ext4_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct super_block *sb = inode->i_sb; + struct mnt_idmap *idmap = file_mnt_idmap(filp); + + ext4_debug("cmd = %u, arg = %lu\n", cmd, arg); + + switch (cmd) { + case FS_IOC_GETFSMAP: + return ext4_ioc_getfsmap(sb, (void __user *)arg); + case EXT4_IOC_GETVERSION: + case EXT4_IOC_GETVERSION_OLD: + return put_user(inode->i_generation, (int __user *) arg); + case EXT4_IOC_SETVERSION: + case EXT4_IOC_SETVERSION_OLD: { + handle_t *handle; + struct ext4_iloc iloc; + __u32 generation; + int err; + + if (!inode_owner_or_capable(idmap, inode)) + return -EPERM; + + if (ext4_has_feature_metadata_csum(inode->i_sb)) { + ext4_warning(sb, "Setting inode version is not " + "supported with metadata_csum enabled."); + return -ENOTTY; + } + + err = mnt_want_write_file(filp); + if (err) + return err; + if (get_user(generation, (int __user *) arg)) { + err = -EFAULT; + goto setversion_out; + } + + inode_lock(inode); + handle = ext4_journal_start(inode, EXT4_HT_INODE, 1); + if (IS_ERR(handle)) { + err = PTR_ERR(handle); + goto unlock_out; + } + err = ext4_reserve_inode_write(handle, inode, &iloc); + if (err == 0) { + inode_set_ctime_current(inode); + inode_inc_iversion(inode); + inode->i_generation = generation; + err = ext4_mark_iloc_dirty(handle, inode, &iloc); + } + ext4_journal_stop(handle); + +unlock_out: + inode_unlock(inode); +setversion_out: + mnt_drop_write_file(filp); + return err; + } + case EXT4_IOC_GROUP_EXTEND: { + ext4_fsblk_t n_blocks_count; + int err, err2=0; + + err = ext4_resize_begin(sb); + if (err) + return err; + + if (get_user(n_blocks_count, (__u32 __user *)arg)) { + err = -EFAULT; + goto group_extend_out; + } + + if (ext4_has_feature_bigalloc(sb)) { + ext4_msg(sb, KERN_ERR, + "Online resizing not supported with bigalloc"); + err = -EOPNOTSUPP; + goto group_extend_out; + } + + err = mnt_want_write_file(filp); + if (err) + goto group_extend_out; + + err = ext4_group_extend(sb, EXT4_SB(sb)->s_es, n_blocks_count); + if (EXT4_SB(sb)->s_journal) { + ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_RESIZE, + NULL); + jbd2_journal_lock_updates(EXT4_SB(sb)->s_journal); + err2 = jbd2_journal_flush(EXT4_SB(sb)->s_journal, 0); + jbd2_journal_unlock_updates(EXT4_SB(sb)->s_journal); + } + if (err == 0) + err = err2; + mnt_drop_write_file(filp); +group_extend_out: + err2 = ext4_resize_end(sb, false); + if (err == 0) + err = err2; + return err; + } + + case EXT4_IOC_MOVE_EXT: { + struct move_extent me; + int err; + + if (!(filp->f_mode & FMODE_READ) || + !(filp->f_mode & FMODE_WRITE)) + return -EBADF; + + if (copy_from_user(&me, + (struct move_extent __user *)arg, sizeof(me))) + return -EFAULT; + me.moved_len = 0; + + CLASS(fd, donor)(me.donor_fd); + if (fd_empty(donor)) + return -EBADF; + + if (!(fd_file(donor)->f_mode & FMODE_WRITE)) + return -EBADF; + + if (file_inode(filp)->i_sb != file_inode(fd_file(donor))->i_sb) + return -EXDEV; + + err = mnt_want_write_file(filp); + if (err) + return err; + + err = ext4_move_extents(filp, fd_file(donor), me.orig_start, + me.donor_start, me.len, &me.moved_len); + mnt_drop_write_file(filp); + + if (copy_to_user((struct move_extent __user *)arg, + &me, sizeof(me))) + err = -EFAULT; + return err; + } + + case EXT4_IOC_GROUP_ADD: { + struct ext4_new_group_data input; + + if (copy_from_user(&input, (struct ext4_new_group_input __user *)arg, + sizeof(input))) + return -EFAULT; + + return ext4_ioctl_group_add(filp, &input); + } + + case EXT4_IOC_MIGRATE: + { + int err; + if (!inode_owner_or_capable(idmap, inode)) + return -EACCES; + + err = mnt_want_write_file(filp); + if (err) + return err; + /* + * inode_mutex prevent write and truncate on the file. + * Read still goes through. We take i_data_sem in + * ext4_ext_swap_inode_data before we switch the + * inode format to prevent read. + */ + inode_lock((inode)); + err = ext4_ext_migrate(inode); + inode_unlock((inode)); + mnt_drop_write_file(filp); + return err; + } + + case EXT4_IOC_ALLOC_DA_BLKS: + { + int err; + if (!inode_owner_or_capable(idmap, inode)) + return -EACCES; + + err = mnt_want_write_file(filp); + if (err) + return err; + err = ext4_alloc_da_blocks(inode); + mnt_drop_write_file(filp); + return err; + } + + case EXT4_IOC_SWAP_BOOT: + { + int err; + if (!(filp->f_mode & FMODE_WRITE)) + return -EBADF; + err = mnt_want_write_file(filp); + if (err) + return err; + err = swap_inode_boot_loader(sb, idmap, inode); + mnt_drop_write_file(filp); + return err; + } + + case EXT4_IOC_RESIZE_FS: { + ext4_fsblk_t n_blocks_count; + int err = 0, err2 = 0; + ext4_group_t o_group = EXT4_SB(sb)->s_groups_count; + + if (copy_from_user(&n_blocks_count, (__u64 __user *)arg, + sizeof(__u64))) { + return -EFAULT; + } + + err = ext4_resize_begin(sb); + if (err) + return err; + + err = mnt_want_write_file(filp); + if (err) + goto resizefs_out; + + err = ext4_resize_fs(sb, n_blocks_count); + if (EXT4_SB(sb)->s_journal) { + ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_RESIZE, NULL); + jbd2_journal_lock_updates(EXT4_SB(sb)->s_journal); + err2 = jbd2_journal_flush(EXT4_SB(sb)->s_journal, 0); + jbd2_journal_unlock_updates(EXT4_SB(sb)->s_journal); + } + if (err == 0) + err = err2; + mnt_drop_write_file(filp); + if (!err && (o_group < EXT4_SB(sb)->s_groups_count) && + ext4_has_group_desc_csum(sb) && + test_opt(sb, INIT_INODE_TABLE)) + err = ext4_register_li_request(sb, o_group); + +resizefs_out: + err2 = ext4_resize_end(sb, true); + if (err == 0) + err = err2; + return err; + } + + case FITRIM: + { + struct fstrim_range range; + int ret = 0; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (!bdev_max_discard_sectors(sb->s_bdev)) + return -EOPNOTSUPP; + + /* + * We haven't replayed the journal, so we cannot use our + * block-bitmap-guided storage zapping commands. + */ + if (test_opt(sb, NOLOAD) && ext4_has_feature_journal(sb)) + return -EROFS; + + if (copy_from_user(&range, (struct fstrim_range __user *)arg, + sizeof(range))) + return -EFAULT; + + ret = ext4_trim_fs(sb, &range); + if (ret < 0) + return ret; + + if (copy_to_user((struct fstrim_range __user *)arg, &range, + sizeof(range))) + return -EFAULT; + + return 0; + } + case EXT4_IOC_PRECACHE_EXTENTS: + { + int ret; + + inode_lock_shared(inode); + ret = ext4_ext_precache(inode); + inode_unlock_shared(inode); + return ret; + } + case FS_IOC_SET_ENCRYPTION_POLICY: + if (!ext4_has_feature_encrypt(sb)) + return -EOPNOTSUPP; + return fscrypt_ioctl_set_policy(filp, (const void __user *)arg); + + case FS_IOC_GET_ENCRYPTION_PWSALT: + return ext4_ioctl_get_encryption_pwsalt(filp, (void __user *)arg); + + case FS_IOC_GET_ENCRYPTION_POLICY: + if (!ext4_has_feature_encrypt(sb)) + return -EOPNOTSUPP; + return fscrypt_ioctl_get_policy(filp, (void __user *)arg); + + case FS_IOC_GET_ENCRYPTION_POLICY_EX: + if (!ext4_has_feature_encrypt(sb)) + return -EOPNOTSUPP; + return fscrypt_ioctl_get_policy_ex(filp, (void __user *)arg); + + case FS_IOC_ADD_ENCRYPTION_KEY: + if (!ext4_has_feature_encrypt(sb)) + return -EOPNOTSUPP; + return fscrypt_ioctl_add_key(filp, (void __user *)arg); + + case FS_IOC_REMOVE_ENCRYPTION_KEY: + if (!ext4_has_feature_encrypt(sb)) + return -EOPNOTSUPP; + return fscrypt_ioctl_remove_key(filp, (void __user *)arg); + + case FS_IOC_REMOVE_ENCRYPTION_KEY_ALL_USERS: + if (!ext4_has_feature_encrypt(sb)) + return -EOPNOTSUPP; + return fscrypt_ioctl_remove_key_all_users(filp, + (void __user *)arg); + case FS_IOC_GET_ENCRYPTION_KEY_STATUS: + if (!ext4_has_feature_encrypt(sb)) + return -EOPNOTSUPP; + return fscrypt_ioctl_get_key_status(filp, (void __user *)arg); + + case FS_IOC_GET_ENCRYPTION_NONCE: + if (!ext4_has_feature_encrypt(sb)) + return -EOPNOTSUPP; + return fscrypt_ioctl_get_nonce(filp, (void __user *)arg); + + case EXT4_IOC_CLEAR_ES_CACHE: + { + if (!inode_owner_or_capable(idmap, inode)) + return -EACCES; + ext4_clear_inode_es(inode); + return 0; + } + + case EXT4_IOC_GETSTATE: + { + __u32 state = 0; + + if (ext4_test_inode_state(inode, EXT4_STATE_EXT_PRECACHED)) + state |= EXT4_STATE_FLAG_EXT_PRECACHED; + if (ext4_test_inode_state(inode, EXT4_STATE_NEW)) + state |= EXT4_STATE_FLAG_NEW; + if (ext4_test_inode_state(inode, EXT4_STATE_NEWENTRY)) + state |= EXT4_STATE_FLAG_NEWENTRY; + if (ext4_test_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE)) + state |= EXT4_STATE_FLAG_DA_ALLOC_CLOSE; + + return put_user(state, (__u32 __user *) arg); + } + + case EXT4_IOC_GET_ES_CACHE: + return ext4_ioctl_get_es_cache(filp, arg); + + case EXT4_IOC_SHUTDOWN: + return ext4_ioctl_shutdown(sb, arg); + + case FS_IOC_ENABLE_VERITY: + if (!ext4_has_feature_verity(sb)) + return -EOPNOTSUPP; + return fsverity_ioctl_enable(filp, (const void __user *)arg); + + case FS_IOC_MEASURE_VERITY: + if (!ext4_has_feature_verity(sb)) + return -EOPNOTSUPP; + return fsverity_ioctl_measure(filp, (void __user *)arg); + + case FS_IOC_READ_VERITY_METADATA: + if (!ext4_has_feature_verity(sb)) + return -EOPNOTSUPP; + return fsverity_ioctl_read_metadata(filp, + (const void __user *)arg); + + case EXT4_IOC_CHECKPOINT: + return ext4_ioctl_checkpoint(filp, arg); + + case FS_IOC_GETFSLABEL: + return ext4_ioctl_getlabel(EXT4_SB(sb), (void __user *)arg); + + case FS_IOC_SETFSLABEL: + return ext4_ioctl_setlabel(filp, + (const void __user *)arg); + + case EXT4_IOC_GETFSUUID: + return ext4_ioctl_getuuid(EXT4_SB(sb), (void __user *)arg); + case EXT4_IOC_SETFSUUID: + return ext4_ioctl_setuuid(filp, (const void __user *)arg); + case EXT4_IOC_GET_TUNE_SB_PARAM: + return ext4_ioctl_get_tune_sb(EXT4_SB(sb), + (void __user *)arg); + case EXT4_IOC_SET_TUNE_SB_PARAM: + return ext4_ioctl_set_tune_sb(filp, (void __user *)arg); + default: + return -ENOTTY; + } +} + +long ext4_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + return __ext4_ioctl(filp, cmd, arg); +} + +#ifdef CONFIG_COMPAT +long ext4_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + /* These are just misnamed, they actually get/put from/to user an int */ + switch (cmd) { + case EXT4_IOC32_GETVERSION: + cmd = EXT4_IOC_GETVERSION; + break; + case EXT4_IOC32_SETVERSION: + cmd = EXT4_IOC_SETVERSION; + break; + case EXT4_IOC32_GROUP_EXTEND: + cmd = EXT4_IOC_GROUP_EXTEND; + break; + case EXT4_IOC32_GETVERSION_OLD: + cmd = EXT4_IOC_GETVERSION_OLD; + break; + case EXT4_IOC32_SETVERSION_OLD: + cmd = EXT4_IOC_SETVERSION_OLD; + break; + case EXT4_IOC32_GETRSVSZ: + cmd = EXT4_IOC_GETRSVSZ; + break; + case EXT4_IOC32_SETRSVSZ: + cmd = EXT4_IOC_SETRSVSZ; + break; + case EXT4_IOC32_GROUP_ADD: { + struct compat_ext4_new_group_input __user *uinput; + struct ext4_new_group_data input; + int err; + + uinput = compat_ptr(arg); + err = get_user(input.group, &uinput->group); + err |= get_user(input.block_bitmap, &uinput->block_bitmap); + err |= get_user(input.inode_bitmap, &uinput->inode_bitmap); + err |= get_user(input.inode_table, &uinput->inode_table); + err |= get_user(input.blocks_count, &uinput->blocks_count); + err |= get_user(input.reserved_blocks, + &uinput->reserved_blocks); + if (err) + return -EFAULT; + return ext4_ioctl_group_add(file, &input); + } + case EXT4_IOC_MOVE_EXT: + case EXT4_IOC_RESIZE_FS: + case FITRIM: + case EXT4_IOC_PRECACHE_EXTENTS: + case FS_IOC_SET_ENCRYPTION_POLICY: + case FS_IOC_GET_ENCRYPTION_PWSALT: + case FS_IOC_GET_ENCRYPTION_POLICY: + case FS_IOC_GET_ENCRYPTION_POLICY_EX: + case FS_IOC_ADD_ENCRYPTION_KEY: + case FS_IOC_REMOVE_ENCRYPTION_KEY: + case FS_IOC_REMOVE_ENCRYPTION_KEY_ALL_USERS: + case FS_IOC_GET_ENCRYPTION_KEY_STATUS: + case FS_IOC_GET_ENCRYPTION_NONCE: + case EXT4_IOC_SHUTDOWN: + case FS_IOC_GETFSMAP: + case FS_IOC_ENABLE_VERITY: + case FS_IOC_MEASURE_VERITY: + case FS_IOC_READ_VERITY_METADATA: + case EXT4_IOC_CLEAR_ES_CACHE: + case EXT4_IOC_GETSTATE: + case EXT4_IOC_GET_ES_CACHE: + case EXT4_IOC_CHECKPOINT: + case FS_IOC_GETFSLABEL: + case FS_IOC_SETFSLABEL: + case EXT4_IOC_GETFSUUID: + case EXT4_IOC_SETFSUUID: + break; + default: + return -ENOIOCTLCMD; + } + return ext4_ioctl(file, cmd, (unsigned long) compat_ptr(arg)); +} +#endif + +static void set_overhead(struct ext4_sb_info *sbi, + struct ext4_super_block *es, const void *arg) +{ + es->s_overhead_clusters = cpu_to_le32(*((unsigned long *) arg)); +} + +int ext4_update_overhead(struct super_block *sb, bool force) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + + if (ext4_emergency_state(sb) || sb_rdonly(sb)) + return 0; + if (!force && + (sbi->s_overhead == 0 || + sbi->s_overhead == le32_to_cpu(sbi->s_es->s_overhead_clusters))) + return 0; + return ext4_update_superblocks_fn(sb, set_overhead, &sbi->s_overhead); +} diff --git a/fs/ext4/mballoc-test.c b/fs/ext4/mballoc-test.c new file mode 100644 index 000000000..d31780075 --- /dev/null +++ b/fs/ext4/mballoc-test.c @@ -0,0 +1,1008 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit test of ext4 multiblocks allocation. + */ + +#include +#include +#include +#include + +#include "ext4.h" +#include "mballoc.h" + +struct mbt_grp_ctx { + struct buffer_head bitmap_bh; + /* desc and gd_bh are just the place holders for now */ + struct ext4_group_desc desc; + struct buffer_head gd_bh; +}; + +struct mbt_ctx { + struct mbt_grp_ctx *grp_ctx; +}; + +struct mbt_ext4_super_block { + struct ext4_super_block es; + struct ext4_sb_info sbi; + struct mbt_ctx mbt_ctx; +}; + +#define MBT_SB(_sb) (container_of((_sb)->s_fs_info, struct mbt_ext4_super_block, sbi)) +#define MBT_CTX(_sb) (&MBT_SB(_sb)->mbt_ctx) +#define MBT_GRP_CTX(_sb, _group) (&MBT_CTX(_sb)->grp_ctx[_group]) + +static struct inode *mbt_alloc_inode(struct super_block *sb) +{ + struct ext4_inode_info *ei; + + ei = kmalloc_obj(struct ext4_inode_info); + if (!ei) + return NULL; + + INIT_LIST_HEAD(&ei->i_orphan); + init_rwsem(&ei->xattr_sem); + init_rwsem(&ei->i_data_sem); + inode_init_once(&ei->vfs_inode); + ext4_fc_init_inode(&ei->vfs_inode); + + return &ei->vfs_inode; +} + +static void mbt_free_inode(struct inode *inode) +{ + kfree(EXT4_I(inode)); +} + +static const struct super_operations mbt_sops = { + .alloc_inode = mbt_alloc_inode, + .free_inode = mbt_free_inode, +}; + +static int mbt_init_fs_context(struct fs_context *fc) +{ + return 0; +} + +static struct file_system_type mbt_fs_type = { + .name = "mballoc test", + .init_fs_context = mbt_init_fs_context, + .kill_sb = kill_anon_super, +}; + +static int mbt_mb_init(struct super_block *sb) +{ + ext4_fsblk_t block; + int ret; + + /* needed by ext4_mb_init->bdev_rot(sb->s_bdev) */ + sb->s_bdev = kzalloc_obj(*sb->s_bdev); + if (sb->s_bdev == NULL) + return -ENOMEM; + + sb->s_bdev->bd_queue = kzalloc_obj(struct request_queue); + if (sb->s_bdev->bd_queue == NULL) { + kfree(sb->s_bdev); + return -ENOMEM; + } + + /* + * needed by ext4_mb_init->ext4_mb_init_backend-> sbi->s_buddy_cache = + * new_inode(sb); + */ + INIT_LIST_HEAD(&sb->s_inodes); + sb->s_op = &mbt_sops; + + ret = ext4_mb_init(sb); + if (ret != 0) + goto err_out; + + block = ext4_count_free_clusters(sb); + ret = percpu_counter_init(&EXT4_SB(sb)->s_freeclusters_counter, block, + GFP_KERNEL); + if (ret != 0) + goto err_mb_release; + + ret = percpu_counter_init(&EXT4_SB(sb)->s_dirtyclusters_counter, 0, + GFP_KERNEL); + if (ret != 0) + goto err_freeclusters; + + return 0; + +err_freeclusters: + percpu_counter_destroy(&EXT4_SB(sb)->s_freeclusters_counter); +err_mb_release: + ext4_mb_release(sb); +err_out: + kfree(sb->s_bdev->bd_queue); + kfree(sb->s_bdev); + return ret; +} + +static void mbt_mb_release(struct super_block *sb) +{ + percpu_counter_destroy(&EXT4_SB(sb)->s_dirtyclusters_counter); + percpu_counter_destroy(&EXT4_SB(sb)->s_freeclusters_counter); + ext4_mb_release(sb); + kfree(sb->s_bdev->bd_queue); + kfree(sb->s_bdev); +} + +static int mbt_set(struct super_block *sb, struct fs_context *fc) +{ + return set_anon_super_fc(sb, fc); +} + +static struct super_block *mbt_ext4_alloc_super_block(void) +{ + struct mbt_ext4_super_block *fsb; + struct super_block *sb; + struct fs_context *fc; + struct ext4_sb_info *sbi; + + fsb = kzalloc_obj(*fsb); + if (fsb == NULL) + return NULL; + + fc = fs_context_for_mount(&mbt_fs_type, 0); + if (IS_ERR(fc)) + goto out; + + sb = sget_fc(fc, NULL, mbt_set); + put_fs_context(fc); + if (IS_ERR(sb)) + goto out; + + sbi = &fsb->sbi; + + sbi->s_blockgroup_lock = + kzalloc_obj(struct blockgroup_lock); + if (!sbi->s_blockgroup_lock) + goto out_deactivate; + + bgl_lock_init(sbi->s_blockgroup_lock); + + sbi->s_es = &fsb->es; + sbi->s_sb = sb; + sb->s_fs_info = sbi; + + up_write(&sb->s_umount); + return sb; + +out_deactivate: + deactivate_locked_super(sb); +out: + kfree(fsb); + return NULL; +} + +static void mbt_ext4_free_super_block(struct super_block *sb) +{ + struct mbt_ext4_super_block *fsb = MBT_SB(sb); + struct ext4_sb_info *sbi = EXT4_SB(sb); + + kfree(sbi->s_blockgroup_lock); + deactivate_super(sb); + kfree(fsb); +} + +struct mbt_ext4_block_layout { + unsigned char blocksize_bits; + unsigned int cluster_bits; + uint32_t blocks_per_group; + ext4_group_t group_count; + uint16_t desc_size; +}; + +static void mbt_init_sb_layout(struct super_block *sb, + struct mbt_ext4_block_layout *layout) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_super_block *es = sbi->s_es; + + sb->s_blocksize = 1UL << layout->blocksize_bits; + sb->s_blocksize_bits = layout->blocksize_bits; + + sbi->s_groups_count = layout->group_count; + sbi->s_blocks_per_group = layout->blocks_per_group; + sbi->s_cluster_bits = layout->cluster_bits; + sbi->s_cluster_ratio = 1U << layout->cluster_bits; + sbi->s_clusters_per_group = layout->blocks_per_group >> + layout->cluster_bits; + sbi->s_desc_size = layout->desc_size; + sbi->s_desc_per_block_bits = + sb->s_blocksize_bits - (fls(layout->desc_size) - 1); + sbi->s_desc_per_block = 1 << sbi->s_desc_per_block_bits; + + es->s_first_data_block = cpu_to_le32(0); + es->s_blocks_count_lo = cpu_to_le32(layout->blocks_per_group * + layout->group_count); +} + +static int mbt_grp_ctx_init(struct super_block *sb, + struct mbt_grp_ctx *grp_ctx) +{ + ext4_grpblk_t max = EXT4_CLUSTERS_PER_GROUP(sb); + + grp_ctx->bitmap_bh.b_data = kzalloc(EXT4_BLOCK_SIZE(sb), GFP_KERNEL); + if (grp_ctx->bitmap_bh.b_data == NULL) + return -ENOMEM; + mb_set_bits(grp_ctx->bitmap_bh.b_data, max, sb->s_blocksize * 8 - max); + ext4_free_group_clusters_set(sb, &grp_ctx->desc, max); + + return 0; +} + +static void mbt_grp_ctx_release(struct mbt_grp_ctx *grp_ctx) +{ + kfree(grp_ctx->bitmap_bh.b_data); + grp_ctx->bitmap_bh.b_data = NULL; +} + +static void mbt_ctx_mark_used(struct super_block *sb, ext4_group_t group, + unsigned int start, unsigned int len) +{ + struct mbt_grp_ctx *grp_ctx = MBT_GRP_CTX(sb, group); + + mb_set_bits(grp_ctx->bitmap_bh.b_data, start, len); +} + +static void *mbt_ctx_bitmap(struct super_block *sb, ext4_group_t group) +{ + struct mbt_grp_ctx *grp_ctx = MBT_GRP_CTX(sb, group); + + return grp_ctx->bitmap_bh.b_data; +} + +/* called after mbt_init_sb_layout */ +static int mbt_ctx_init(struct super_block *sb) +{ + struct mbt_ctx *ctx = MBT_CTX(sb); + ext4_group_t i, ngroups = ext4_get_groups_count(sb); + + ctx->grp_ctx = kzalloc_objs(struct mbt_grp_ctx, ngroups); + if (ctx->grp_ctx == NULL) + return -ENOMEM; + + for (i = 0; i < ngroups; i++) + if (mbt_grp_ctx_init(sb, &ctx->grp_ctx[i])) + goto out; + + /* + * first data block(first cluster in first group) is used by + * metadata, mark it used to avoid to alloc data block at first + * block which will fail ext4_sb_block_valid check. + */ + mb_set_bits(ctx->grp_ctx[0].bitmap_bh.b_data, 0, 1); + ext4_free_group_clusters_set(sb, &ctx->grp_ctx[0].desc, + EXT4_CLUSTERS_PER_GROUP(sb) - 1); + + return 0; +out: + while (i-- > 0) + mbt_grp_ctx_release(&ctx->grp_ctx[i]); + kfree(ctx->grp_ctx); + return -ENOMEM; +} + +static void mbt_ctx_release(struct super_block *sb) +{ + struct mbt_ctx *ctx = MBT_CTX(sb); + ext4_group_t i, ngroups = ext4_get_groups_count(sb); + + for (i = 0; i < ngroups; i++) + mbt_grp_ctx_release(&ctx->grp_ctx[i]); + kfree(ctx->grp_ctx); +} + +static struct buffer_head * +ext4_read_block_bitmap_nowait_stub(struct super_block *sb, ext4_group_t block_group, + bool ignore_locked) +{ + struct mbt_grp_ctx *grp_ctx = MBT_GRP_CTX(sb, block_group); + + /* paired with brelse from caller of ext4_read_block_bitmap_nowait */ + get_bh(&grp_ctx->bitmap_bh); + return &grp_ctx->bitmap_bh; +} + +static int ext4_wait_block_bitmap_stub(struct super_block *sb, + ext4_group_t block_group, + struct buffer_head *bh) +{ + /* + * real ext4_wait_block_bitmap will set these flags and + * functions like ext4_mb_init_cache will verify the flags. + */ + set_buffer_uptodate(bh); + set_bitmap_uptodate(bh); + set_buffer_verified(bh); + return 0; +} + +static struct ext4_group_desc * +ext4_get_group_desc_stub(struct super_block *sb, ext4_group_t block_group, + struct buffer_head **bh) +{ + struct mbt_grp_ctx *grp_ctx = MBT_GRP_CTX(sb, block_group); + + if (bh != NULL) + *bh = &grp_ctx->gd_bh; + + return &grp_ctx->desc; +} + +static int +ext4_mb_mark_context_stub(handle_t *handle, struct super_block *sb, bool state, + ext4_group_t group, ext4_grpblk_t blkoff, + ext4_grpblk_t len, int flags, + ext4_grpblk_t *ret_changed) +{ + struct mbt_grp_ctx *grp_ctx = MBT_GRP_CTX(sb, group); + struct buffer_head *bitmap_bh = &grp_ctx->bitmap_bh; + + if (state) + mb_set_bits(bitmap_bh->b_data, blkoff, len); + else + mb_clear_bits_test(bitmap_bh->b_data, blkoff, len); + + return 0; +} + +#define TEST_GOAL_GROUP 1 +static int mbt_kunit_init(struct kunit *test) +{ + struct mbt_ext4_block_layout *layout = + (struct mbt_ext4_block_layout *)(test->param_value); + struct super_block *sb; + int ret; + + sb = mbt_ext4_alloc_super_block(); + if (sb == NULL) + return -ENOMEM; + + mbt_init_sb_layout(sb, layout); + + ret = mbt_ctx_init(sb); + if (ret != 0) { + mbt_ext4_free_super_block(sb); + return ret; + } + + kunit_activate_static_stub(test, + ext4_read_block_bitmap_nowait, + ext4_read_block_bitmap_nowait_stub); + kunit_activate_static_stub(test, + ext4_wait_block_bitmap, + ext4_wait_block_bitmap_stub); + kunit_activate_static_stub(test, + ext4_get_group_desc, + ext4_get_group_desc_stub); + kunit_activate_static_stub(test, + ext4_mb_mark_context, + ext4_mb_mark_context_stub); + + /* stub function will be called in mbt_mb_init->ext4_mb_init */ + if (mbt_mb_init(sb) != 0) { + mbt_ctx_release(sb); + mbt_ext4_free_super_block(sb); + return -ENOMEM; + } + + test->priv = sb; + + return 0; +} + +static void mbt_kunit_exit(struct kunit *test) +{ + struct super_block *sb = (struct super_block *)test->priv; + + if (!sb) + return; + + mbt_mb_release(sb); + mbt_ctx_release(sb); + mbt_ext4_free_super_block(sb); +} + +static void test_new_blocks_simple(struct kunit *test) +{ + struct super_block *sb = (struct super_block *)test->priv; + struct inode *inode; + struct ext4_allocation_request ar; + ext4_group_t i, goal_group = TEST_GOAL_GROUP; + int err = 0; + ext4_fsblk_t found; + struct ext4_sb_info *sbi = EXT4_SB(sb); + + inode = kunit_kzalloc(test, sizeof(*inode), GFP_KERNEL); + if (!inode) + return; + + inode->i_sb = sb; + ar.inode = inode; + + /* get block at goal */ + ar.goal = ext4_group_first_block_no(sb, goal_group); + found = ext4_mb_new_blocks_simple_test(&ar, &err); + KUNIT_ASSERT_EQ_MSG(test, ar.goal, found, + "failed to alloc block at goal, expected %llu found %llu", + ar.goal, found); + + /* get block after goal in goal group */ + ar.goal = ext4_group_first_block_no(sb, goal_group); + found = ext4_mb_new_blocks_simple_test(&ar, &err); + KUNIT_ASSERT_EQ_MSG(test, ar.goal + EXT4_C2B(sbi, 1), found, + "failed to alloc block after goal in goal group, expected %llu found %llu", + ar.goal + 1, found); + + /* get block after goal group */ + mbt_ctx_mark_used(sb, goal_group, 0, EXT4_CLUSTERS_PER_GROUP(sb)); + ar.goal = ext4_group_first_block_no(sb, goal_group); + found = ext4_mb_new_blocks_simple_test(&ar, &err); + KUNIT_ASSERT_EQ_MSG(test, + ext4_group_first_block_no(sb, goal_group + 1), found, + "failed to alloc block after goal group, expected %llu found %llu", + ext4_group_first_block_no(sb, goal_group + 1), found); + + /* get block before goal group */ + for (i = goal_group; i < ext4_get_groups_count(sb); i++) + mbt_ctx_mark_used(sb, i, 0, EXT4_CLUSTERS_PER_GROUP(sb)); + ar.goal = ext4_group_first_block_no(sb, goal_group); + found = ext4_mb_new_blocks_simple_test(&ar, &err); + KUNIT_ASSERT_EQ_MSG(test, + ext4_group_first_block_no(sb, 0) + EXT4_C2B(sbi, 1), found, + "failed to alloc block before goal group, expected %llu found %llu", + ext4_group_first_block_no(sb, 0 + EXT4_C2B(sbi, 1)), found); + + /* no block available, fail to allocate block */ + for (i = 0; i < ext4_get_groups_count(sb); i++) + mbt_ctx_mark_used(sb, i, 0, EXT4_CLUSTERS_PER_GROUP(sb)); + ar.goal = ext4_group_first_block_no(sb, goal_group); + found = ext4_mb_new_blocks_simple_test(&ar, &err); + KUNIT_ASSERT_NE_MSG(test, err, 0, + "unexpectedly get block when no block is available"); +} + +#define TEST_RANGE_COUNT 8 + +struct test_range { + ext4_grpblk_t start; + ext4_grpblk_t len; +}; + +static void +mbt_generate_test_ranges(struct super_block *sb, struct test_range *ranges, + int count) +{ + ext4_grpblk_t start, len, max; + int i; + + max = EXT4_CLUSTERS_PER_GROUP(sb) / count; + for (i = 0; i < count; i++) { + start = get_random_u32() % max; + len = get_random_u32() % max; + len = min(len, max - start); + + ranges[i].start = start + i * max; + ranges[i].len = len; + } +} + +static void +validate_free_blocks_simple(struct kunit *test, struct super_block *sb, + ext4_group_t goal_group, ext4_grpblk_t start, + ext4_grpblk_t len) +{ + void *bitmap; + ext4_grpblk_t bit, max = EXT4_CLUSTERS_PER_GROUP(sb); + ext4_group_t i; + + for (i = 0; i < ext4_get_groups_count(sb); i++) { + if (i == goal_group) + continue; + + bitmap = mbt_ctx_bitmap(sb, i); + bit = mb_find_next_zero_bit_test(bitmap, max, 0); + KUNIT_ASSERT_EQ_MSG(test, bit, max, + "free block on unexpected group %d", i); + } + + bitmap = mbt_ctx_bitmap(sb, goal_group); + bit = mb_find_next_zero_bit_test(bitmap, max, 0); + KUNIT_ASSERT_EQ(test, bit, start); + + bit = mb_find_next_bit_test(bitmap, max, bit + 1); + KUNIT_ASSERT_EQ(test, bit, start + len); +} + +static void +test_free_blocks_simple_range(struct kunit *test, ext4_group_t goal_group, + ext4_grpblk_t start, ext4_grpblk_t len) +{ + struct super_block *sb = (struct super_block *)test->priv; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct inode *inode; + ext4_fsblk_t block; + + inode = kunit_kzalloc(test, sizeof(*inode), GFP_KERNEL); + if (!inode) + return; + inode->i_sb = sb; + + if (len == 0) + return; + + block = ext4_group_first_block_no(sb, goal_group) + + EXT4_C2B(sbi, start); + ext4_free_blocks_simple_test(inode, block, len); + validate_free_blocks_simple(test, sb, goal_group, start, len); + mbt_ctx_mark_used(sb, goal_group, 0, EXT4_CLUSTERS_PER_GROUP(sb)); +} + +static void test_free_blocks_simple(struct kunit *test) +{ + struct super_block *sb = (struct super_block *)test->priv; + ext4_grpblk_t max = EXT4_CLUSTERS_PER_GROUP(sb); + ext4_group_t i; + struct test_range ranges[TEST_RANGE_COUNT]; + + for (i = 0; i < ext4_get_groups_count(sb); i++) + mbt_ctx_mark_used(sb, i, 0, max); + + mbt_generate_test_ranges(sb, ranges, TEST_RANGE_COUNT); + for (i = 0; i < TEST_RANGE_COUNT; i++) + test_free_blocks_simple_range(test, TEST_GOAL_GROUP, + ranges[i].start, ranges[i].len); +} + +static void +test_mark_diskspace_used_range(struct kunit *test, + struct ext4_allocation_context *ac, + ext4_grpblk_t start, + ext4_grpblk_t len) +{ + struct super_block *sb = (struct super_block *)test->priv; + int ret; + void *bitmap; + ext4_grpblk_t i, max; + + /* ext4_mb_mark_diskspace_used will BUG if len is 0 */ + if (len == 0) + return; + + ac->ac_b_ex.fe_group = TEST_GOAL_GROUP; + ac->ac_b_ex.fe_start = start; + ac->ac_b_ex.fe_len = len; + + bitmap = mbt_ctx_bitmap(sb, TEST_GOAL_GROUP); + memset(bitmap, 0, sb->s_blocksize); + ret = ext4_mb_mark_diskspace_used_test(ac, NULL); + KUNIT_ASSERT_EQ(test, ret, 0); + + max = EXT4_CLUSTERS_PER_GROUP(sb); + i = mb_find_next_bit_test(bitmap, max, 0); + KUNIT_ASSERT_EQ(test, i, start); + i = mb_find_next_zero_bit_test(bitmap, max, i + 1); + KUNIT_ASSERT_EQ(test, i, start + len); + i = mb_find_next_bit_test(bitmap, max, i + 1); + KUNIT_ASSERT_EQ(test, max, i); +} + +static void test_mark_diskspace_used(struct kunit *test) +{ + struct super_block *sb = (struct super_block *)test->priv; + struct inode *inode; + struct ext4_allocation_context ac; + struct test_range ranges[TEST_RANGE_COUNT]; + int i; + + mbt_generate_test_ranges(sb, ranges, TEST_RANGE_COUNT); + + inode = kunit_kzalloc(test, sizeof(*inode), GFP_KERNEL); + if (!inode) + return; + inode->i_sb = sb; + + ac.ac_status = AC_STATUS_FOUND; + ac.ac_sb = sb; + ac.ac_inode = inode; + for (i = 0; i < TEST_RANGE_COUNT; i++) + test_mark_diskspace_used_range(test, &ac, ranges[i].start, + ranges[i].len); +} + +static void mbt_generate_buddy(struct super_block *sb, void *buddy, + void *bitmap, struct ext4_group_info *grp) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + uint32_t order, off; + void *bb, *bb_h; + int max; + + memset(buddy, 0xff, sb->s_blocksize); + memset(grp, 0, offsetof(struct ext4_group_info, + bb_counters[MB_NUM_ORDERS(sb)])); + + bb = bitmap; + max = EXT4_CLUSTERS_PER_GROUP(sb); + bb_h = buddy + sbi->s_mb_offsets[1]; + + off = mb_find_next_zero_bit_test(bb, max, 0); + grp->bb_first_free = off; + while (off < max) { + grp->bb_counters[0]++; + grp->bb_free++; + + if (!(off & 1) && !mb_test_bit_test(off + 1, bb)) { + grp->bb_free++; + grp->bb_counters[0]--; + mb_clear_bit_test(off >> 1, bb_h); + grp->bb_counters[1]++; + grp->bb_largest_free_order = 1; + off++; + } + + off = mb_find_next_zero_bit_test(bb, max, off + 1); + } + + for (order = 1; order < MB_NUM_ORDERS(sb) - 1; order++) { + bb = buddy + sbi->s_mb_offsets[order]; + bb_h = buddy + sbi->s_mb_offsets[order + 1]; + max = max >> 1; + off = mb_find_next_zero_bit_test(bb, max, 0); + + while (off < max) { + if (!(off & 1) && !mb_test_bit_test(off + 1, bb)) { + mb_set_bits(bb, off, 2); + grp->bb_counters[order] -= 2; + mb_clear_bit_test(off >> 1, bb_h); + grp->bb_counters[order + 1]++; + grp->bb_largest_free_order = order + 1; + off++; + } + + off = mb_find_next_zero_bit_test(bb, max, off + 1); + } + } + + max = EXT4_CLUSTERS_PER_GROUP(sb); + off = mb_find_next_zero_bit_test(bitmap, max, 0); + while (off < max) { + grp->bb_fragments++; + + off = mb_find_next_bit_test(bitmap, max, off + 1); + if (off + 1 >= max) + break; + + off = mb_find_next_zero_bit_test(bitmap, max, off + 1); + } +} + +static void +mbt_validate_group_info(struct kunit *test, struct ext4_group_info *grp1, + struct ext4_group_info *grp2) +{ + struct super_block *sb = (struct super_block *)test->priv; + int i; + + KUNIT_ASSERT_EQ(test, grp1->bb_first_free, + grp2->bb_first_free); + KUNIT_ASSERT_EQ(test, grp1->bb_fragments, + grp2->bb_fragments); + KUNIT_ASSERT_EQ(test, grp1->bb_free, grp2->bb_free); + KUNIT_ASSERT_EQ(test, grp1->bb_largest_free_order, + grp2->bb_largest_free_order); + + for (i = 1; i < MB_NUM_ORDERS(sb); i++) { + KUNIT_ASSERT_EQ_MSG(test, grp1->bb_counters[i], + grp2->bb_counters[i], + "bb_counters[%d] diffs, expected %d, generated %d", + i, grp1->bb_counters[i], + grp2->bb_counters[i]); + } +} + +static void +do_test_generate_buddy(struct kunit *test, struct super_block *sb, void *bitmap, + void *mbt_buddy, struct ext4_group_info *mbt_grp, + void *ext4_buddy, struct ext4_group_info *ext4_grp) +{ + int i; + + mbt_generate_buddy(sb, mbt_buddy, bitmap, mbt_grp); + + for (i = 0; i < MB_NUM_ORDERS(sb); i++) + ext4_grp->bb_counters[i] = 0; + /* needed by validation in ext4_mb_generate_buddy */ + ext4_grp->bb_free = mbt_grp->bb_free; + memset(ext4_buddy, 0xff, sb->s_blocksize); + ext4_mb_generate_buddy_test(sb, ext4_buddy, bitmap, TEST_GOAL_GROUP, + ext4_grp); + + KUNIT_ASSERT_MEMEQ(test, mbt_buddy, ext4_buddy, sb->s_blocksize); + mbt_validate_group_info(test, mbt_grp, ext4_grp); +} + +static void test_mb_generate_buddy(struct kunit *test) +{ + struct super_block *sb = (struct super_block *)test->priv; + void *bitmap, *expected_bb, *generate_bb; + struct ext4_group_info *expected_grp, *generate_grp; + struct test_range ranges[TEST_RANGE_COUNT]; + int i; + + bitmap = kunit_kzalloc(test, sb->s_blocksize, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bitmap); + expected_bb = kunit_kzalloc(test, sb->s_blocksize, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, expected_bb); + generate_bb = kunit_kzalloc(test, sb->s_blocksize, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, generate_bb); + expected_grp = kunit_kzalloc(test, offsetof(struct ext4_group_info, + bb_counters[MB_NUM_ORDERS(sb)]), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, expected_grp); + generate_grp = ext4_get_group_info(sb, TEST_GOAL_GROUP); + KUNIT_ASSERT_NOT_NULL(test, generate_grp); + + mbt_generate_test_ranges(sb, ranges, TEST_RANGE_COUNT); + for (i = 0; i < TEST_RANGE_COUNT; i++) { + mb_set_bits(bitmap, ranges[i].start, ranges[i].len); + do_test_generate_buddy(test, sb, bitmap, expected_bb, + expected_grp, generate_bb, generate_grp); + } +} + +static void +test_mb_mark_used_range(struct kunit *test, struct ext4_buddy *e4b, + ext4_grpblk_t start, ext4_grpblk_t len, void *bitmap, + void *buddy, struct ext4_group_info *grp) +{ + struct super_block *sb = (struct super_block *)test->priv; + struct ext4_free_extent ex; + int i; + + /* mb_mark_used only accepts non-zero len */ + if (len == 0) + return; + + ex.fe_start = start; + ex.fe_len = len; + ex.fe_group = TEST_GOAL_GROUP; + + ext4_lock_group(sb, TEST_GOAL_GROUP); + mb_mark_used_test(e4b, &ex); + ext4_unlock_group(sb, TEST_GOAL_GROUP); + + mb_set_bits(bitmap, start, len); + /* bypass bb_free validatoin in ext4_mb_generate_buddy */ + grp->bb_free -= len; + memset(buddy, 0xff, sb->s_blocksize); + for (i = 0; i < MB_NUM_ORDERS(sb); i++) + grp->bb_counters[i] = 0; + ext4_mb_generate_buddy_test(sb, buddy, bitmap, 0, grp); + + KUNIT_ASSERT_MEMEQ(test, buddy, e4b->bd_buddy, sb->s_blocksize); + mbt_validate_group_info(test, grp, e4b->bd_info); +} + +static void test_mb_mark_used(struct kunit *test) +{ + struct ext4_buddy e4b; + struct super_block *sb = (struct super_block *)test->priv; + void *bitmap, *buddy; + struct ext4_group_info *grp; + int ret; + struct test_range ranges[TEST_RANGE_COUNT]; + int i; + + /* buddy cache assumes that each page contains at least one block */ + if (sb->s_blocksize > PAGE_SIZE) + kunit_skip(test, "blocksize exceeds pagesize"); + + bitmap = kunit_kzalloc(test, sb->s_blocksize, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bitmap); + buddy = kunit_kzalloc(test, sb->s_blocksize, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buddy); + grp = kunit_kzalloc(test, offsetof(struct ext4_group_info, + bb_counters[MB_NUM_ORDERS(sb)]), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, grp); + + ret = ext4_mb_load_buddy_test(sb, TEST_GOAL_GROUP, &e4b); + KUNIT_ASSERT_EQ(test, ret, 0); + + grp->bb_free = EXT4_CLUSTERS_PER_GROUP(sb); + grp->bb_largest_free_order = -1; + grp->bb_avg_fragment_size_order = -1; + mbt_generate_test_ranges(sb, ranges, TEST_RANGE_COUNT); + for (i = 0; i < TEST_RANGE_COUNT; i++) + test_mb_mark_used_range(test, &e4b, ranges[i].start, + ranges[i].len, bitmap, buddy, grp); + + ext4_mb_unload_buddy_test(&e4b); +} + +static void +test_mb_free_blocks_range(struct kunit *test, struct ext4_buddy *e4b, + ext4_grpblk_t start, ext4_grpblk_t len, void *bitmap, + void *buddy, struct ext4_group_info *grp) +{ + struct super_block *sb = (struct super_block *)test->priv; + int i; + + /* mb_free_blocks will WARN if len is 0 */ + if (len == 0) + return; + + ext4_lock_group(sb, e4b->bd_group); + mb_free_blocks_test(NULL, e4b, start, len); + ext4_unlock_group(sb, e4b->bd_group); + + mb_clear_bits_test(bitmap, start, len); + /* bypass bb_free validatoin in ext4_mb_generate_buddy */ + grp->bb_free += len; + memset(buddy, 0xff, sb->s_blocksize); + for (i = 0; i < MB_NUM_ORDERS(sb); i++) + grp->bb_counters[i] = 0; + ext4_mb_generate_buddy_test(sb, buddy, bitmap, 0, grp); + + KUNIT_ASSERT_MEMEQ(test, buddy, e4b->bd_buddy, sb->s_blocksize); + mbt_validate_group_info(test, grp, e4b->bd_info); + +} + +static void test_mb_free_blocks(struct kunit *test) +{ + struct ext4_buddy e4b; + struct super_block *sb = (struct super_block *)test->priv; + void *bitmap, *buddy; + struct ext4_group_info *grp; + struct ext4_free_extent ex; + int ret; + int i; + struct test_range ranges[TEST_RANGE_COUNT]; + + /* buddy cache assumes that each page contains at least one block */ + if (sb->s_blocksize > PAGE_SIZE) + kunit_skip(test, "blocksize exceeds pagesize"); + + bitmap = kunit_kzalloc(test, sb->s_blocksize, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bitmap); + buddy = kunit_kzalloc(test, sb->s_blocksize, GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buddy); + grp = kunit_kzalloc(test, offsetof(struct ext4_group_info, + bb_counters[MB_NUM_ORDERS(sb)]), GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, grp); + + ret = ext4_mb_load_buddy_test(sb, TEST_GOAL_GROUP, &e4b); + KUNIT_ASSERT_EQ(test, ret, 0); + + ex.fe_start = 0; + ex.fe_len = EXT4_CLUSTERS_PER_GROUP(sb); + ex.fe_group = TEST_GOAL_GROUP; + + ext4_lock_group(sb, TEST_GOAL_GROUP); + mb_mark_used_test(&e4b, &ex); + ext4_unlock_group(sb, TEST_GOAL_GROUP); + + grp->bb_free = 0; + grp->bb_largest_free_order = -1; + grp->bb_avg_fragment_size_order = -1; + memset(bitmap, 0xff, sb->s_blocksize); + + mbt_generate_test_ranges(sb, ranges, TEST_RANGE_COUNT); + for (i = 0; i < TEST_RANGE_COUNT; i++) + test_mb_free_blocks_range(test, &e4b, ranges[i].start, + ranges[i].len, bitmap, buddy, grp); + + ext4_mb_unload_buddy_test(&e4b); +} + +#define COUNT_FOR_ESTIMATE 100000 +static void test_mb_mark_used_cost(struct kunit *test) +{ + struct ext4_buddy e4b; + struct super_block *sb = (struct super_block *)test->priv; + struct ext4_free_extent ex; + int ret; + struct test_range ranges[TEST_RANGE_COUNT]; + int i, j; + unsigned long start, end, all = 0; + + /* buddy cache assumes that each page contains at least one block */ + if (sb->s_blocksize > PAGE_SIZE) + kunit_skip(test, "blocksize exceeds pagesize"); + + ret = ext4_mb_load_buddy_test(sb, TEST_GOAL_GROUP, &e4b); + KUNIT_ASSERT_EQ(test, ret, 0); + + ex.fe_group = TEST_GOAL_GROUP; + for (j = 0; j < COUNT_FOR_ESTIMATE; j++) { + mbt_generate_test_ranges(sb, ranges, TEST_RANGE_COUNT); + start = jiffies; + for (i = 0; i < TEST_RANGE_COUNT; i++) { + if (ranges[i].len == 0) + continue; + + ex.fe_start = ranges[i].start; + ex.fe_len = ranges[i].len; + ext4_lock_group(sb, TEST_GOAL_GROUP); + mb_mark_used_test(&e4b, &ex); + ext4_unlock_group(sb, TEST_GOAL_GROUP); + } + end = jiffies; + all += (end - start); + + for (i = 0; i < TEST_RANGE_COUNT; i++) { + if (ranges[i].len == 0) + continue; + + ext4_lock_group(sb, TEST_GOAL_GROUP); + mb_free_blocks_test(NULL, &e4b, ranges[i].start, + ranges[i].len); + ext4_unlock_group(sb, TEST_GOAL_GROUP); + } + } + + kunit_info(test, "costed jiffies %lu\n", all); + ext4_mb_unload_buddy_test(&e4b); +} + +static const struct mbt_ext4_block_layout mbt_test_layouts[] = { + { + .blocksize_bits = 10, + .cluster_bits = 3, + .blocks_per_group = 8192, + .group_count = 4, + .desc_size = 64, + }, + { + .blocksize_bits = 12, + .cluster_bits = 3, + .blocks_per_group = 8192, + .group_count = 4, + .desc_size = 64, + }, + { + .blocksize_bits = 16, + .cluster_bits = 3, + .blocks_per_group = 8192, + .group_count = 4, + .desc_size = 64, + }, +}; + +static void mbt_show_layout(const struct mbt_ext4_block_layout *layout, + char *desc) +{ + snprintf(desc, KUNIT_PARAM_DESC_SIZE, "block_bits=%d cluster_bits=%d " + "blocks_per_group=%d group_count=%d desc_size=%d\n", + layout->blocksize_bits, layout->cluster_bits, + layout->blocks_per_group, layout->group_count, + layout->desc_size); +} +KUNIT_ARRAY_PARAM(mbt_layouts, mbt_test_layouts, mbt_show_layout); + +static struct kunit_case mbt_test_cases[] = { + KUNIT_CASE_PARAM(test_new_blocks_simple, mbt_layouts_gen_params), + KUNIT_CASE_PARAM(test_free_blocks_simple, mbt_layouts_gen_params), + KUNIT_CASE_PARAM(test_mb_generate_buddy, mbt_layouts_gen_params), + KUNIT_CASE_PARAM(test_mb_mark_used, mbt_layouts_gen_params), + KUNIT_CASE_PARAM(test_mb_free_blocks, mbt_layouts_gen_params), + KUNIT_CASE_PARAM(test_mark_diskspace_used, mbt_layouts_gen_params), + KUNIT_CASE_PARAM_ATTR(test_mb_mark_used_cost, mbt_layouts_gen_params, + { .speed = KUNIT_SPEED_SLOW }), + {} +}; + +static struct kunit_suite mbt_test_suite = { + .name = "ext4_mballoc_test", + .init = mbt_kunit_init, + .exit = mbt_kunit_exit, + .test_cases = mbt_test_cases, +}; + +kunit_test_suites(&mbt_test_suite); + +MODULE_LICENSE("GPL"); diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c new file mode 100644 index 000000000..06171a11d --- /dev/null +++ b/fs/ext4/mballoc.c @@ -0,0 +1,7284 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2003-2006, Cluster File Systems, Inc, info@clusterfs.com + * Written by Alex Tomas + */ + + +/* + * mballoc.c contains the multiblocks allocation routines + */ + +#include "ext4_jbd2.h" +#include "mballoc.h" +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * MUSTDO: + * - test ext4_ext_search_left() and ext4_ext_search_right() + * - search for metadata in few groups + * + * TODO v4: + * - normalization should take into account whether file is still open + * - discard preallocations if no free space left (policy?) + * - don't normalize tails + * - quota + * - reservation for superuser + * + * TODO v3: + * - bitmap read-ahead (proposed by Oleg Drokin aka green) + * - track min/max extents in each group for better group selection + * - mb_mark_used() may allocate chunk right after splitting buddy + * - tree of groups sorted by number of free blocks + * - error handling + */ + +/* + * The allocation request involve request for multiple number of blocks + * near to the goal(block) value specified. + * + * During initialization phase of the allocator we decide to use the + * group preallocation or inode preallocation depending on the size of + * the file. The size of the file could be the resulting file size we + * would have after allocation, or the current file size, which ever + * is larger. If the size is less than sbi->s_mb_stream_request we + * select to use the group preallocation. The default value of + * s_mb_stream_request is 16 blocks. This can also be tuned via + * /sys/fs/ext4//mb_stream_req. The value is represented in + * terms of number of blocks. + * + * The main motivation for having small file use group preallocation is to + * ensure that we have small files closer together on the disk. + * + * First stage the allocator looks at the inode prealloc list, + * ext4_inode_info->i_prealloc_list, which contains list of prealloc + * spaces for this particular inode. The inode prealloc space is + * represented as: + * + * pa_lstart -> the logical start block for this prealloc space + * pa_pstart -> the physical start block for this prealloc space + * pa_len -> length for this prealloc space (in clusters) + * pa_free -> free space available in this prealloc space (in clusters) + * + * The inode preallocation space is used looking at the _logical_ start + * block. If only the logical file block falls within the range of prealloc + * space we will consume the particular prealloc space. This makes sure that + * we have contiguous physical blocks representing the file blocks + * + * The important thing to be noted in case of inode prealloc space is that + * we don't modify the values associated to inode prealloc space except + * pa_free. + * + * If we are not able to find blocks in the inode prealloc space and if we + * have the group allocation flag set then we look at the locality group + * prealloc space. These are per CPU prealloc list represented as + * + * ext4_sb_info.s_locality_groups[smp_processor_id()] + * + * The reason for having a per cpu locality group is to reduce the contention + * between CPUs. It is possible to get scheduled at this point. + * + * The locality group prealloc space is used looking at whether we have + * enough free space (pa_free) within the prealloc space. + * + * If we can't allocate blocks via inode prealloc or/and locality group + * prealloc then we look at the buddy cache. The buddy cache is represented + * by ext4_sb_info.s_buddy_cache (struct inode) whose file offset gets + * mapped to the buddy and bitmap information regarding different + * groups. The buddy information is attached to buddy cache inode so that + * we can access them through the page cache. The information regarding + * each group is loaded via ext4_mb_load_buddy. The information involve + * block bitmap and buddy information. The information are stored in the + * inode as: + * + * { folio } + * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]... + * + * + * one block each for bitmap and buddy information. So for each group we + * take up 2 blocks. A folio can contain blocks_per_folio (folio_size / + * blocksize) blocks. So it can have information regarding groups_per_folio + * which is blocks_per_folio/2 + * + * The buddy cache inode is not stored on disk. The inode is thrown + * away when the filesystem is unmounted. + * + * We look for count number of blocks in the buddy cache. If we were able + * to locate that many free blocks we return with additional information + * regarding rest of the contiguous physical block available + * + * Before allocating blocks via buddy cache we normalize the request + * blocks. This ensure we ask for more blocks that we needed. The extra + * blocks that we get after allocation is added to the respective prealloc + * list. In case of inode preallocation we follow a list of heuristics + * based on file size. This can be found in ext4_mb_normalize_request. If + * we are doing a group prealloc we try to normalize the request to + * sbi->s_mb_group_prealloc. The default value of s_mb_group_prealloc is + * dependent on the cluster size; for non-bigalloc file systems, it is + * 512 blocks. This can be tuned via + * /sys/fs/ext4//mb_group_prealloc. The value is represented in + * terms of number of blocks. If we have mounted the file system with -O + * stripe= option the group prealloc request is normalized to the + * smallest multiple of the stripe value (sbi->s_stripe) which is + * greater than the default mb_group_prealloc. + * + * If "mb_optimize_scan" mount option is set, we maintain in memory group info + * structures in two data structures: + * + * 1) Array of largest free order xarrays (sbi->s_mb_largest_free_orders) + * + * Locking: Writers use xa_lock, readers use rcu_read_lock. + * + * This is an array of xarrays where the index in the array represents the + * largest free order in the buddy bitmap of the participating group infos of + * that xarray. So, there are exactly MB_NUM_ORDERS(sb) (which means total + * number of buddy bitmap orders possible) number of xarrays. Group-infos are + * placed in appropriate xarrays. + * + * 2) Average fragment size xarrays (sbi->s_mb_avg_fragment_size) + * + * Locking: Writers use xa_lock, readers use rcu_read_lock. + * + * This is an array of xarrays where in the i-th xarray there are groups with + * average fragment size >= 2^i and < 2^(i+1). The average fragment size + * is computed as ext4_group_info->bb_free / ext4_group_info->bb_fragments. + * Note that we don't bother with a special xarray for completely empty + * groups so we only have MB_NUM_ORDERS(sb) xarrays. Group-infos are placed + * in appropriate xarrays. + * + * In xarray, the index is the block group number, the value is the block group + * information, and a non-empty value indicates the block group is present in + * the current xarray. + * + * When "mb_optimize_scan" mount option is set, mballoc consults the above data + * structures to decide the order in which groups are to be traversed for + * fulfilling an allocation request. + * + * At CR_POWER2_ALIGNED , we look for groups which have the largest_free_order + * >= the order of the request. We directly look at the largest free order list + * in the data structure (1) above where largest_free_order = order of the + * request. If that list is empty, we look at remaining list in the increasing + * order of largest_free_order. This allows us to perform CR_POWER2_ALIGNED + * lookup in O(1) time. + * + * At CR_GOAL_LEN_FAST, we only consider groups where + * average fragment size > request size. So, we lookup a group which has average + * fragment size just above or equal to request size using our average fragment + * size group lists (data structure 2) in O(1) time. + * + * At CR_BEST_AVAIL_LEN, we aim to optimize allocations which can't be satisfied + * in CR_GOAL_LEN_FAST. The fact that we couldn't find a group in + * CR_GOAL_LEN_FAST suggests that there is no BG that has avg + * fragment size > goal length. So before falling to the slower + * CR_GOAL_LEN_SLOW, in CR_BEST_AVAIL_LEN we proactively trim goal length and + * then use the same fragment lists as CR_GOAL_LEN_FAST to find a BG with a big + * enough average fragment size. This increases the chances of finding a + * suitable block group in O(1) time and results in faster allocation at the + * cost of reduced size of allocation. + * + * If "mb_optimize_scan" mount option is not set, mballoc traverses groups in + * linear order which requires O(N) search time for each CR_POWER2_ALIGNED and + * CR_GOAL_LEN_FAST phase. + * + * The regular allocator (using the buddy cache) supports a few tunables. + * + * /sys/fs/ext4//mb_min_to_scan + * /sys/fs/ext4//mb_max_to_scan + * /sys/fs/ext4//mb_order2_req + * /sys/fs/ext4//mb_max_linear_groups + * + * The regular allocator uses buddy scan only if the request len is power of + * 2 blocks and the order of allocation is >= sbi->s_mb_order2_reqs. The + * value of s_mb_order2_reqs can be tuned via + * /sys/fs/ext4//mb_order2_req. If the request len is equal to + * stripe size (sbi->s_stripe), we try to search for contiguous block in + * stripe size. This should result in better allocation on RAID setups. If + * not, we search in the specific group using bitmap for best extents. The + * tunable min_to_scan and max_to_scan control the behaviour here. + * min_to_scan indicate how long the mballoc __must__ look for a best + * extent and max_to_scan indicates how long the mballoc __can__ look for a + * best extent in the found extents. Searching for the blocks starts with + * the group specified as the goal value in allocation context via + * ac_g_ex. Each group is first checked based on the criteria whether it + * can be used for allocation. ext4_mb_good_group explains how the groups are + * checked. + * + * When "mb_optimize_scan" is turned on, as mentioned above, the groups may not + * get traversed linearly. That may result in subsequent allocations being not + * close to each other. And so, the underlying device may get filled up in a + * non-linear fashion. While that may not matter on non-rotational devices, for + * rotational devices that may result in higher seek times. "mb_max_linear_groups" + * tells mballoc how many groups mballoc should search linearly before + * performing consulting above data structures for more efficient lookups. For + * non rotational devices, this value defaults to 0 and for rotational devices + * this is set to MB_DEFAULT_LINEAR_LIMIT. + * + * Both the prealloc space are getting populated as above. So for the first + * request we will hit the buddy cache which will result in this prealloc + * space getting filled. The prealloc space is then later used for the + * subsequent request. + */ + +/* + * mballoc operates on the following data: + * - on-disk bitmap + * - in-core buddy (actually includes buddy and bitmap) + * - preallocation descriptors (PAs) + * + * there are two types of preallocations: + * - inode + * assiged to specific inode and can be used for this inode only. + * it describes part of inode's space preallocated to specific + * physical blocks. any block from that preallocated can be used + * independent. the descriptor just tracks number of blocks left + * unused. so, before taking some block from descriptor, one must + * make sure corresponded logical block isn't allocated yet. this + * also means that freeing any block within descriptor's range + * must discard all preallocated blocks. + * - locality group + * assigned to specific locality group which does not translate to + * permanent set of inodes: inode can join and leave group. space + * from this type of preallocation can be used for any inode. thus + * it's consumed from the beginning to the end. + * + * relation between them can be expressed as: + * in-core buddy = on-disk bitmap + preallocation descriptors + * + * this mean blocks mballoc considers used are: + * - allocated blocks (persistent) + * - preallocated blocks (non-persistent) + * + * consistency in mballoc world means that at any time a block is either + * free or used in ALL structures. notice: "any time" should not be read + * literally -- time is discrete and delimited by locks. + * + * to keep it simple, we don't use block numbers, instead we count number of + * blocks: how many blocks marked used/free in on-disk bitmap, buddy and PA. + * + * all operations can be expressed as: + * - init buddy: buddy = on-disk + PAs + * - new PA: buddy += N; PA = N + * - use inode PA: on-disk += N; PA -= N + * - discard inode PA buddy -= on-disk - PA; PA = 0 + * - use locality group PA on-disk += N; PA -= N + * - discard locality group PA buddy -= PA; PA = 0 + * note: 'buddy -= on-disk - PA' is used to show that on-disk bitmap + * is used in real operation because we can't know actual used + * bits from PA, only from on-disk bitmap + * + * if we follow this strict logic, then all operations above should be atomic. + * given some of them can block, we'd have to use something like semaphores + * killing performance on high-end SMP hardware. let's try to relax it using + * the following knowledge: + * 1) if buddy is referenced, it's already initialized + * 2) while block is used in buddy and the buddy is referenced, + * nobody can re-allocate that block + * 3) we work on bitmaps and '+' actually means 'set bits'. if on-disk has + * bit set and PA claims same block, it's OK. IOW, one can set bit in + * on-disk bitmap if buddy has same bit set or/and PA covers corresponded + * block + * + * so, now we're building a concurrency table: + * - init buddy vs. + * - new PA + * blocks for PA are allocated in the buddy, buddy must be referenced + * until PA is linked to allocation group to avoid concurrent buddy init + * - use inode PA + * we need to make sure that either on-disk bitmap or PA has uptodate data + * given (3) we care that PA-=N operation doesn't interfere with init + * - discard inode PA + * the simplest way would be to have buddy initialized by the discard + * - use locality group PA + * again PA-=N must be serialized with init + * - discard locality group PA + * the simplest way would be to have buddy initialized by the discard + * - new PA vs. + * - use inode PA + * i_data_sem serializes them + * - discard inode PA + * discard process must wait until PA isn't used by another process + * - use locality group PA + * some mutex should serialize them + * - discard locality group PA + * discard process must wait until PA isn't used by another process + * - use inode PA + * - use inode PA + * i_data_sem or another mutex should serializes them + * - discard inode PA + * discard process must wait until PA isn't used by another process + * - use locality group PA + * nothing wrong here -- they're different PAs covering different blocks + * - discard locality group PA + * discard process must wait until PA isn't used by another process + * + * now we're ready to make few consequences: + * - PA is referenced and while it is no discard is possible + * - PA is referenced until block isn't marked in on-disk bitmap + * - PA changes only after on-disk bitmap + * - discard must not compete with init. either init is done before + * any discard or they're serialized somehow + * - buddy init as sum of on-disk bitmap and PAs is done atomically + * + * a special case when we've used PA to emptiness. no need to modify buddy + * in this case, but we should care about concurrent init + * + */ + + /* + * Logic in few words: + * + * - allocation: + * load group + * find blocks + * mark bits in on-disk bitmap + * release group + * + * - use preallocation: + * find proper PA (per-inode or group) + * load group + * mark bits in on-disk bitmap + * release group + * release PA + * + * - free: + * load group + * mark bits in on-disk bitmap + * release group + * + * - discard preallocations in group: + * mark PAs deleted + * move them onto local list + * load on-disk bitmap + * load group + * remove PA from object (inode or locality group) + * mark free blocks in-core + * + * - discard inode's preallocations: + */ + +/* + * Locking rules + * + * Locks: + * - bitlock on a group (group) + * - object (inode/locality) (object) + * - per-pa lock (pa) + * - cr_power2_aligned lists lock (cr_power2_aligned) + * - cr_goal_len_fast lists lock (cr_goal_len_fast) + * + * Paths: + * - new pa + * object + * group + * + * - find and use pa: + * pa + * + * - release consumed pa: + * pa + * group + * object + * + * - generate in-core bitmap: + * group + * pa + * + * - discard all for given object (inode, locality group): + * object + * pa + * group + * + * - discard all for given group: + * group + * pa + * group + * object + * + * - allocation path (ext4_mb_regular_allocator) + * group + * cr_power2_aligned/cr_goal_len_fast + */ +static struct kmem_cache *ext4_pspace_cachep; +static struct kmem_cache *ext4_ac_cachep; +static struct kmem_cache *ext4_free_data_cachep; + +/* We create slab caches for groupinfo data structures based on the + * superblock block size. There will be one per mounted filesystem for + * each unique s_blocksize_bits */ +#define NR_GRPINFO_CACHES 8 +static struct kmem_cache *ext4_groupinfo_caches[NR_GRPINFO_CACHES]; + +static const char * const ext4_groupinfo_slab_names[NR_GRPINFO_CACHES] = { + "ext4_groupinfo_1k", "ext4_groupinfo_2k", "ext4_groupinfo_4k", + "ext4_groupinfo_8k", "ext4_groupinfo_16k", "ext4_groupinfo_32k", + "ext4_groupinfo_64k", "ext4_groupinfo_128k" +}; + +static void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap, + ext4_group_t group); +static void ext4_mb_new_preallocation(struct ext4_allocation_context *ac); + +static int ext4_mb_scan_group(struct ext4_allocation_context *ac, + ext4_group_t group); + +static int ext4_try_to_trim_range(struct super_block *sb, + struct ext4_buddy *e4b, ext4_grpblk_t start, + ext4_grpblk_t max, ext4_grpblk_t minblocks); + +/* + * The algorithm using this percpu seq counter goes below: + * 1. We sample the percpu discard_pa_seq counter before trying for block + * allocation in ext4_mb_new_blocks(). + * 2. We increment this percpu discard_pa_seq counter when we either allocate + * or free these blocks i.e. while marking those blocks as used/free in + * mb_mark_used()/mb_free_blocks(). + * 3. We also increment this percpu seq counter when we successfully identify + * that the bb_prealloc_list is not empty and hence proceed for discarding + * of those PAs inside ext4_mb_discard_group_preallocations(). + * + * Now to make sure that the regular fast path of block allocation is not + * affected, as a small optimization we only sample the percpu seq counter + * on that cpu. Only when the block allocation fails and when freed blocks + * found were 0, that is when we sample percpu seq counter for all cpus using + * below function ext4_get_discard_pa_seq_sum(). This happens after making + * sure that all the PAs on grp->bb_prealloc_list got freed or if it's empty. + */ +static DEFINE_PER_CPU(u64, discard_pa_seq); +static inline u64 ext4_get_discard_pa_seq_sum(void) +{ + int __cpu; + u64 __seq = 0; + + for_each_possible_cpu(__cpu) + __seq += per_cpu(discard_pa_seq, __cpu); + return __seq; +} + +static inline void *mb_correct_addr_and_bit(int *bit, void *addr) +{ +#if BITS_PER_LONG == 64 + *bit += ((unsigned long) addr & 7UL) << 3; + addr = (void *) ((unsigned long) addr & ~7UL); +#elif BITS_PER_LONG == 32 + *bit += ((unsigned long) addr & 3UL) << 3; + addr = (void *) ((unsigned long) addr & ~3UL); +#else +#error "how many bits you are?!" +#endif + return addr; +} + +static inline int mb_test_bit(int bit, void *addr) +{ + /* + * ext4_test_bit on architecture like powerpc + * needs unsigned long aligned address + */ + addr = mb_correct_addr_and_bit(&bit, addr); + return ext4_test_bit(bit, addr); +} + +static inline void mb_set_bit(int bit, void *addr) +{ + addr = mb_correct_addr_and_bit(&bit, addr); + ext4_set_bit(bit, addr); +} + +static inline void mb_clear_bit(int bit, void *addr) +{ + addr = mb_correct_addr_and_bit(&bit, addr); + ext4_clear_bit(bit, addr); +} + +static inline int mb_test_and_clear_bit(int bit, void *addr) +{ + addr = mb_correct_addr_and_bit(&bit, addr); + return ext4_test_and_clear_bit(bit, addr); +} + +static inline int mb_find_next_zero_bit(void *addr, int max, int start) +{ + int fix = 0, ret, tmpmax; + addr = mb_correct_addr_and_bit(&fix, addr); + tmpmax = max + fix; + start += fix; + + ret = ext4_find_next_zero_bit(addr, tmpmax, start) - fix; + if (ret > max) + return max; + return ret; +} + +static inline int mb_find_next_bit(void *addr, int max, int start) +{ + int fix = 0, ret, tmpmax; + addr = mb_correct_addr_and_bit(&fix, addr); + tmpmax = max + fix; + start += fix; + + ret = ext4_find_next_bit(addr, tmpmax, start) - fix; + if (ret > max) + return max; + return ret; +} + +static void *mb_find_buddy(struct ext4_buddy *e4b, int order, int *max) +{ + char *bb; + + BUG_ON(e4b->bd_bitmap == e4b->bd_buddy); + BUG_ON(max == NULL); + + if (order > e4b->bd_blkbits + 1) { + *max = 0; + return NULL; + } + + /* at order 0 we see each particular block */ + if (order == 0) { + *max = 1 << (e4b->bd_blkbits + 3); + return e4b->bd_bitmap; + } + + bb = e4b->bd_buddy + EXT4_SB(e4b->bd_sb)->s_mb_offsets[order]; + *max = EXT4_SB(e4b->bd_sb)->s_mb_maxs[order]; + + return bb; +} + +#ifdef DOUBLE_CHECK +static void mb_free_blocks_double(struct inode *inode, struct ext4_buddy *e4b, + int first, int count) +{ + int i; + struct super_block *sb = e4b->bd_sb; + + if (unlikely(e4b->bd_info->bb_bitmap == NULL)) + return; + assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group)); + for (i = 0; i < count; i++) { + if (!mb_test_bit(first + i, e4b->bd_info->bb_bitmap)) { + ext4_fsblk_t blocknr; + + blocknr = ext4_group_first_block_no(sb, e4b->bd_group); + blocknr += EXT4_C2B(EXT4_SB(sb), first + i); + ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group, + EXT4_GROUP_INFO_BBITMAP_CORRUPT); + ext4_grp_locked_error(sb, e4b->bd_group, + inode ? inode->i_ino : 0, + blocknr, + "freeing block already freed " + "(bit %u)", + first + i); + } + mb_clear_bit(first + i, e4b->bd_info->bb_bitmap); + } +} + +static void mb_mark_used_double(struct ext4_buddy *e4b, int first, int count) +{ + int i; + + if (unlikely(e4b->bd_info->bb_bitmap == NULL)) + return; + assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); + for (i = 0; i < count; i++) { + BUG_ON(mb_test_bit(first + i, e4b->bd_info->bb_bitmap)); + mb_set_bit(first + i, e4b->bd_info->bb_bitmap); + } +} + +static void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap) +{ + if (unlikely(e4b->bd_info->bb_bitmap == NULL)) + return; + if (memcmp(e4b->bd_info->bb_bitmap, bitmap, e4b->bd_sb->s_blocksize)) { + unsigned char *b1, *b2; + int i; + b1 = (unsigned char *) e4b->bd_info->bb_bitmap; + b2 = (unsigned char *) bitmap; + for (i = 0; i < e4b->bd_sb->s_blocksize; i++) { + if (b1[i] != b2[i]) { + ext4_msg(e4b->bd_sb, KERN_ERR, + "corruption in group %u " + "at byte %u(%u): %x in copy != %x " + "on disk/prealloc", + e4b->bd_group, i, i * 8, b1[i], b2[i]); + BUG(); + } + } + } +} + +static void mb_group_bb_bitmap_alloc(struct super_block *sb, + struct ext4_group_info *grp, ext4_group_t group) +{ + struct buffer_head *bh; + + grp->bb_bitmap = kmalloc(sb->s_blocksize, GFP_NOFS); + if (!grp->bb_bitmap) + return; + + bh = ext4_read_block_bitmap(sb, group); + if (IS_ERR_OR_NULL(bh)) { + kfree(grp->bb_bitmap); + grp->bb_bitmap = NULL; + return; + } + + memcpy(grp->bb_bitmap, bh->b_data, sb->s_blocksize); + put_bh(bh); +} + +static void mb_group_bb_bitmap_free(struct ext4_group_info *grp) +{ + kfree(grp->bb_bitmap); +} + +#else +static inline void mb_free_blocks_double(struct inode *inode, + struct ext4_buddy *e4b, int first, int count) +{ + return; +} +static inline void mb_mark_used_double(struct ext4_buddy *e4b, + int first, int count) +{ + return; +} +static inline void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap) +{ + return; +} + +static inline void mb_group_bb_bitmap_alloc(struct super_block *sb, + struct ext4_group_info *grp, ext4_group_t group) +{ + return; +} + +static inline void mb_group_bb_bitmap_free(struct ext4_group_info *grp) +{ + return; +} +#endif + +#ifdef AGGRESSIVE_CHECK + +#define MB_CHECK_ASSERT(assert) \ +do { \ + if (!(assert)) { \ + printk(KERN_EMERG \ + "Assertion failure in %s() at %s:%d: \"%s\"\n", \ + function, file, line, # assert); \ + BUG(); \ + } \ +} while (0) + +/* + * Perform buddy integrity check with the following steps: + * + * 1. Top-down validation (from highest order down to order 1, excluding order-0 bitmap): + * For each pair of adjacent orders, if a higher-order bit is set (indicating a free block), + * at most one of the two corresponding lower-order bits may be clear (free). + * + * 2. Order-0 (bitmap) validation, performed on bit pairs: + * - If either bit in a pair is set (1, allocated), then all corresponding higher-order bits + * must not be free (0). + * - If both bits in a pair are clear (0, free), then exactly one of the corresponding + * higher-order bits must be free (0). + * + * 3. Preallocation (pa) list validation: + * For each preallocated block (pa) in the group: + * - Verify that pa_pstart falls within the bounds of this block group. + * - Ensure the corresponding bit(s) in the order-0 bitmap are marked as allocated (1). + */ +static void __mb_check_buddy(struct ext4_buddy *e4b, char *file, + const char *function, int line) +{ + struct super_block *sb = e4b->bd_sb; + int order = e4b->bd_blkbits + 1; + int max; + int max2; + int i; + int j; + int k; + int count; + struct ext4_group_info *grp; + int fragments = 0; + int fstart; + struct list_head *cur; + void *buddy; + void *buddy2; + + if (e4b->bd_info->bb_check_counter++ % 10) + return; + + while (order > 1) { + buddy = mb_find_buddy(e4b, order, &max); + MB_CHECK_ASSERT(buddy); + buddy2 = mb_find_buddy(e4b, order - 1, &max2); + MB_CHECK_ASSERT(buddy2); + MB_CHECK_ASSERT(buddy != buddy2); + MB_CHECK_ASSERT(max * 2 == max2); + + count = 0; + for (i = 0; i < max; i++) { + + if (mb_test_bit(i, buddy)) { + /* only single bit in buddy2 may be 0 */ + if (!mb_test_bit(i << 1, buddy2)) { + MB_CHECK_ASSERT( + mb_test_bit((i<<1)+1, buddy2)); + } + continue; + } + + count++; + } + MB_CHECK_ASSERT(e4b->bd_info->bb_counters[order] == count); + order--; + } + + fstart = -1; + buddy = mb_find_buddy(e4b, 0, &max); + for (i = 0; i < max; i++) { + if (!mb_test_bit(i, buddy)) { + MB_CHECK_ASSERT(i >= e4b->bd_info->bb_first_free); + if (fstart == -1) { + fragments++; + fstart = i; + } + } else { + fstart = -1; + } + if (!(i & 1)) { + int in_use, zero_bit_count = 0; + + in_use = mb_test_bit(i, buddy) || mb_test_bit(i + 1, buddy); + for (j = 1; j < e4b->bd_blkbits + 2; j++) { + buddy2 = mb_find_buddy(e4b, j, &max2); + k = i >> j; + MB_CHECK_ASSERT(k < max2); + if (!mb_test_bit(k, buddy2)) + zero_bit_count++; + } + MB_CHECK_ASSERT(zero_bit_count == !in_use); + } + } + MB_CHECK_ASSERT(!EXT4_MB_GRP_NEED_INIT(e4b->bd_info)); + MB_CHECK_ASSERT(e4b->bd_info->bb_fragments == fragments); + + grp = ext4_get_group_info(sb, e4b->bd_group); + if (!grp) + return; + list_for_each(cur, &grp->bb_prealloc_list) { + ext4_group_t groupnr; + struct ext4_prealloc_space *pa; + pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); + if (!pa->pa_len) + continue; + ext4_get_group_no_and_offset(sb, pa->pa_pstart, &groupnr, &k); + MB_CHECK_ASSERT(groupnr == e4b->bd_group); + for (i = 0; i < pa->pa_len; i++) + MB_CHECK_ASSERT(mb_test_bit(k + i, buddy)); + } +} +#undef MB_CHECK_ASSERT +#define mb_check_buddy(e4b) __mb_check_buddy(e4b, \ + __FILE__, __func__, __LINE__) +#else +#define mb_check_buddy(e4b) +#endif + +/* + * Divide blocks started from @first with length @len into + * smaller chunks with power of 2 blocks. + * Clear the bits in bitmap which the blocks of the chunk(s) covered, + * then increase bb_counters[] for corresponded chunk size. + */ +static void ext4_mb_mark_free_simple(struct super_block *sb, + void *buddy, ext4_grpblk_t first, ext4_grpblk_t len, + struct ext4_group_info *grp) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + ext4_grpblk_t min; + ext4_grpblk_t max; + ext4_grpblk_t chunk; + unsigned int border; + + BUG_ON(len > EXT4_CLUSTERS_PER_GROUP(sb)); + + border = 2 << sb->s_blocksize_bits; + + while (len > 0) { + /* find how many blocks can be covered since this position */ + max = ffs(first | border) - 1; + + /* find how many blocks of power 2 we need to mark */ + min = fls(len) - 1; + + if (max < min) + min = max; + chunk = 1 << min; + + /* mark multiblock chunks only */ + grp->bb_counters[min]++; + if (min > 0) + mb_clear_bit(first >> min, + buddy + sbi->s_mb_offsets[min]); + + len -= chunk; + first += chunk; + } +} + +static int mb_avg_fragment_size_order(struct super_block *sb, ext4_grpblk_t len) +{ + int order; + + /* + * We don't bother with a special lists groups with only 1 block free + * extents and for completely empty groups. + */ + order = fls(len) - 2; + if (order < 0) + return 0; + if (order == MB_NUM_ORDERS(sb)) + order--; + if (WARN_ON_ONCE(order > MB_NUM_ORDERS(sb))) + order = MB_NUM_ORDERS(sb) - 1; + return order; +} + +/* Move group to appropriate avg_fragment_size list */ +static void +mb_update_avg_fragment_size(struct super_block *sb, struct ext4_group_info *grp) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + int new, old; + + if (!test_opt2(sb, MB_OPTIMIZE_SCAN)) + return; + + old = grp->bb_avg_fragment_size_order; + new = grp->bb_fragments == 0 ? -1 : + mb_avg_fragment_size_order(sb, grp->bb_free / grp->bb_fragments); + if (new == old) + return; + + if (old >= 0) + xa_erase(&sbi->s_mb_avg_fragment_size[old], grp->bb_group); + + grp->bb_avg_fragment_size_order = new; + if (new >= 0) { + /* + * Cannot use __GFP_NOFAIL because we hold the group lock. + * Although allocation for insertion may fails, it's not fatal + * as we have linear traversal to fall back on. + */ + int err = xa_insert(&sbi->s_mb_avg_fragment_size[new], + grp->bb_group, grp, GFP_ATOMIC); + if (err) + mb_debug(sb, "insert group: %u to s_mb_avg_fragment_size[%d] failed, err %d", + grp->bb_group, new, err); + } +} + +static ext4_group_t ext4_get_allocation_groups_count( + struct ext4_allocation_context *ac) +{ + ext4_group_t ngroups = ext4_get_groups_count(ac->ac_sb); + + /* non-extent files are limited to low blocks/groups */ + if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))) + ngroups = EXT4_SB(ac->ac_sb)->s_blockfile_groups; + + /* Pairs with smp_wmb() in ext4_update_super() */ + smp_rmb(); + + return ngroups; +} + +static int ext4_mb_scan_groups_xa_range(struct ext4_allocation_context *ac, + struct xarray *xa, + ext4_group_t start, ext4_group_t end) +{ + struct super_block *sb = ac->ac_sb; + struct ext4_sb_info *sbi = EXT4_SB(sb); + enum criteria cr = ac->ac_criteria; + ext4_group_t ngroups = ext4_get_allocation_groups_count(ac); + unsigned long group = start; + struct ext4_group_info *grp; + + if (WARN_ON_ONCE(end > ngroups || start >= end)) + return 0; + + xa_for_each_range(xa, group, grp, start, end - 1) { + int err; + + if (sbi->s_mb_stats) + atomic64_inc(&sbi->s_bal_cX_groups_considered[cr]); + + err = ext4_mb_scan_group(ac, grp->bb_group); + if (err || ac->ac_status != AC_STATUS_CONTINUE) + return err; + + cond_resched(); + } + + return 0; +} + +/* + * Find a suitable group of given order from the largest free orders xarray. + */ +static inline int +ext4_mb_scan_groups_largest_free_order_range(struct ext4_allocation_context *ac, + int order, ext4_group_t start, + ext4_group_t end) +{ + struct xarray *xa = &EXT4_SB(ac->ac_sb)->s_mb_largest_free_orders[order]; + + if (xa_empty(xa)) + return 0; + + return ext4_mb_scan_groups_xa_range(ac, xa, start, end); +} + +/* + * Choose next group by traversing largest_free_order lists. Updates *new_cr if + * cr level needs an update. + */ +static int ext4_mb_scan_groups_p2_aligned(struct ext4_allocation_context *ac, + ext4_group_t group) +{ + struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); + int i; + int ret = 0; + ext4_group_t start, end; + + start = group; + end = ext4_get_allocation_groups_count(ac); +wrap_around: + for (i = ac->ac_2order; i < MB_NUM_ORDERS(ac->ac_sb); i++) { + ret = ext4_mb_scan_groups_largest_free_order_range(ac, i, + start, end); + if (ret || ac->ac_status != AC_STATUS_CONTINUE) + return ret; + } + if (start) { + end = start; + start = 0; + goto wrap_around; + } + + if (sbi->s_mb_stats) + atomic64_inc(&sbi->s_bal_cX_failed[ac->ac_criteria]); + + /* Increment cr and search again if no group is found */ + ac->ac_criteria = CR_GOAL_LEN_FAST; + return ret; +} + +/* + * Find a suitable group of given order from the average fragments xarray. + */ +static int +ext4_mb_scan_groups_avg_frag_order_range(struct ext4_allocation_context *ac, + int order, ext4_group_t start, + ext4_group_t end) +{ + struct xarray *xa = &EXT4_SB(ac->ac_sb)->s_mb_avg_fragment_size[order]; + + if (xa_empty(xa)) + return 0; + + return ext4_mb_scan_groups_xa_range(ac, xa, start, end); +} + +/* + * Choose next group by traversing average fragment size list of suitable + * order. Updates *new_cr if cr level needs an update. + */ +static int ext4_mb_scan_groups_goal_fast(struct ext4_allocation_context *ac, + ext4_group_t group) +{ + struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); + int i, ret = 0; + ext4_group_t start, end; + + start = group; + end = ext4_get_allocation_groups_count(ac); +wrap_around: + i = mb_avg_fragment_size_order(ac->ac_sb, ac->ac_g_ex.fe_len); + for (; i < MB_NUM_ORDERS(ac->ac_sb); i++) { + ret = ext4_mb_scan_groups_avg_frag_order_range(ac, i, + start, end); + if (ret || ac->ac_status != AC_STATUS_CONTINUE) + return ret; + } + if (start) { + end = start; + start = 0; + goto wrap_around; + } + + if (sbi->s_mb_stats) + atomic64_inc(&sbi->s_bal_cX_failed[ac->ac_criteria]); + /* + * CR_BEST_AVAIL_LEN works based on the concept that we have + * a larger normalized goal len request which can be trimmed to + * a smaller goal len such that it can still satisfy original + * request len. However, allocation request for non-regular + * files never gets normalized. + * See function ext4_mb_normalize_request() (EXT4_MB_HINT_DATA). + */ + if (ac->ac_flags & EXT4_MB_HINT_DATA) + ac->ac_criteria = CR_BEST_AVAIL_LEN; + else + ac->ac_criteria = CR_GOAL_LEN_SLOW; + + return ret; +} + +/* + * We couldn't find a group in CR_GOAL_LEN_FAST so try to find the highest free fragment + * order we have and proactively trim the goal request length to that order to + * find a suitable group faster. + * + * This optimizes allocation speed at the cost of slightly reduced + * preallocations. However, we make sure that we don't trim the request too + * much and fall to CR_GOAL_LEN_SLOW in that case. + */ +static int ext4_mb_scan_groups_best_avail(struct ext4_allocation_context *ac, + ext4_group_t group) +{ + int ret = 0; + struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); + int i, order, min_order; + unsigned long num_stripe_clusters = 0; + ext4_group_t start, end; + + /* + * mb_avg_fragment_size_order() returns order in a way that makes + * retrieving back the length using (1 << order) inaccurate. Hence, use + * fls() instead since we need to know the actual length while modifying + * goal length. + */ + order = fls(ac->ac_g_ex.fe_len) - 1; + if (WARN_ON_ONCE(order - 1 > MB_NUM_ORDERS(ac->ac_sb))) + order = MB_NUM_ORDERS(ac->ac_sb); + min_order = order - sbi->s_mb_best_avail_max_trim_order; + if (min_order < 0) + min_order = 0; + + if (sbi->s_stripe > 0) { + /* + * We are assuming that stripe size is always a multiple of + * cluster ratio otherwise __ext4_fill_super exists early. + */ + num_stripe_clusters = EXT4_NUM_B2C(sbi, sbi->s_stripe); + if (1 << min_order < num_stripe_clusters) + /* + * We consider 1 order less because later we round + * up the goal len to num_stripe_clusters + */ + min_order = fls(num_stripe_clusters) - 1; + } + + if (1 << min_order < ac->ac_o_ex.fe_len) + min_order = fls(ac->ac_o_ex.fe_len); + + start = group; + end = ext4_get_allocation_groups_count(ac); +wrap_around: + for (i = order; i >= min_order; i--) { + int frag_order; + /* + * Scale down goal len to make sure we find something + * in the free fragments list. Basically, reduce + * preallocations. + */ + ac->ac_g_ex.fe_len = 1 << i; + + if (num_stripe_clusters > 0) { + /* + * Try to round up the adjusted goal length to + * stripe size (in cluster units) multiple for + * efficiency. + */ + ac->ac_g_ex.fe_len = roundup(ac->ac_g_ex.fe_len, + num_stripe_clusters); + } + + frag_order = mb_avg_fragment_size_order(ac->ac_sb, + ac->ac_g_ex.fe_len); + + ret = ext4_mb_scan_groups_avg_frag_order_range(ac, frag_order, + start, end); + if (ret || ac->ac_status != AC_STATUS_CONTINUE) + return ret; + } + if (start) { + end = start; + start = 0; + goto wrap_around; + } + + /* Reset goal length to original goal length before falling into CR_GOAL_LEN_SLOW */ + ac->ac_g_ex.fe_len = ac->ac_orig_goal_len; + if (sbi->s_mb_stats) + atomic64_inc(&sbi->s_bal_cX_failed[ac->ac_criteria]); + ac->ac_criteria = CR_GOAL_LEN_SLOW; + + return ret; +} + +static inline int should_optimize_scan(struct ext4_allocation_context *ac) +{ + if (unlikely(!test_opt2(ac->ac_sb, MB_OPTIMIZE_SCAN))) + return 0; + if (ac->ac_criteria >= CR_GOAL_LEN_SLOW) + return 0; + return 1; +} + +/* + * next linear group for allocation. + */ +static void next_linear_group(ext4_group_t *group, ext4_group_t ngroups) +{ + /* + * Artificially restricted ngroups for non-extent + * files makes group > ngroups possible on first loop. + */ + *group = *group + 1 >= ngroups ? 0 : *group + 1; +} + +static int ext4_mb_scan_groups_linear(struct ext4_allocation_context *ac, + ext4_group_t ngroups, ext4_group_t *start, ext4_group_t count) +{ + int ret, i; + enum criteria cr = ac->ac_criteria; + struct super_block *sb = ac->ac_sb; + struct ext4_sb_info *sbi = EXT4_SB(sb); + ext4_group_t group = *start; + + for (i = 0; i < count; i++, next_linear_group(&group, ngroups)) { + ret = ext4_mb_scan_group(ac, group); + if (ret || ac->ac_status != AC_STATUS_CONTINUE) + return ret; + cond_resched(); + } + + *start = group; + if (count == ngroups) + ac->ac_criteria++; + + /* Processed all groups and haven't found blocks */ + if (sbi->s_mb_stats && i == ngroups) + atomic64_inc(&sbi->s_bal_cX_failed[cr]); + + return 0; +} + +static int ext4_mb_scan_groups(struct ext4_allocation_context *ac) +{ + int ret = 0; + ext4_group_t start; + struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); + ext4_group_t ngroups = ext4_get_allocation_groups_count(ac); + + /* searching for the right group start from the goal value specified */ + start = ac->ac_g_ex.fe_group; + if (start >= ngroups) + start = 0; + ac->ac_prefetch_grp = start; + ac->ac_prefetch_nr = 0; + + if (!should_optimize_scan(ac)) + return ext4_mb_scan_groups_linear(ac, ngroups, &start, ngroups); + + /* + * Optimized scanning can return non adjacent groups which can cause + * seek overhead for rotational disks. So try few linear groups before + * trying optimized scan. + */ + if (sbi->s_mb_max_linear_groups) + ret = ext4_mb_scan_groups_linear(ac, ngroups, &start, + sbi->s_mb_max_linear_groups); + if (ret || ac->ac_status != AC_STATUS_CONTINUE) + return ret; + + switch (ac->ac_criteria) { + case CR_POWER2_ALIGNED: + return ext4_mb_scan_groups_p2_aligned(ac, start); + case CR_GOAL_LEN_FAST: + return ext4_mb_scan_groups_goal_fast(ac, start); + case CR_BEST_AVAIL_LEN: + return ext4_mb_scan_groups_best_avail(ac, start); + default: + /* + * TODO: For CR_GOAL_LEN_SLOW, we can arrange groups in an + * rb tree sorted by bb_free. But until that happens, we should + * never come here. + */ + WARN_ON(1); + } + + return 0; +} + +/* + * Cache the order of the largest free extent we have available in this block + * group. + */ +static void +mb_set_largest_free_order(struct super_block *sb, struct ext4_group_info *grp) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + int new, old = grp->bb_largest_free_order; + + for (new = MB_NUM_ORDERS(sb) - 1; new >= 0; new--) + if (grp->bb_counters[new] > 0) + break; + + /* No need to move between order lists? */ + if (new == old) + return; + + if (old >= 0) { + struct xarray *xa = &sbi->s_mb_largest_free_orders[old]; + + if (!xa_empty(xa) && xa_load(xa, grp->bb_group)) + xa_erase(xa, grp->bb_group); + } + + grp->bb_largest_free_order = new; + if (test_opt2(sb, MB_OPTIMIZE_SCAN) && new >= 0 && grp->bb_free) { + /* + * Cannot use __GFP_NOFAIL because we hold the group lock. + * Although allocation for insertion may fails, it's not fatal + * as we have linear traversal to fall back on. + */ + int err = xa_insert(&sbi->s_mb_largest_free_orders[new], + grp->bb_group, grp, GFP_ATOMIC); + if (err) + mb_debug(sb, "insert group: %u to s_mb_largest_free_orders[%d] failed, err %d", + grp->bb_group, new, err); + } +} + +static noinline_for_stack +void ext4_mb_generate_buddy(struct super_block *sb, + void *buddy, void *bitmap, ext4_group_t group, + struct ext4_group_info *grp) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + ext4_grpblk_t max = EXT4_CLUSTERS_PER_GROUP(sb); + ext4_grpblk_t i = 0; + ext4_grpblk_t first; + ext4_grpblk_t len; + unsigned free = 0; + unsigned fragments = 0; + unsigned long long period = get_cycles(); + + /* initialize buddy from bitmap which is aggregation + * of on-disk bitmap and preallocations */ + i = mb_find_next_zero_bit(bitmap, max, 0); + grp->bb_first_free = i; + while (i < max) { + fragments++; + first = i; + i = mb_find_next_bit(bitmap, max, i); + len = i - first; + free += len; + if (len > 1) + ext4_mb_mark_free_simple(sb, buddy, first, len, grp); + else + grp->bb_counters[0]++; + if (i < max) + i = mb_find_next_zero_bit(bitmap, max, i); + } + grp->bb_fragments = fragments; + + if (free != grp->bb_free) { + ext4_grp_locked_error(sb, group, 0, 0, + "block bitmap and bg descriptor " + "inconsistent: %u vs %u free clusters", + free, grp->bb_free); + /* + * If we intend to continue, we consider group descriptor + * corrupt and update bb_free using bitmap value + */ + grp->bb_free = free; + ext4_mark_group_bitmap_corrupted(sb, group, + EXT4_GROUP_INFO_BBITMAP_CORRUPT); + } + mb_set_largest_free_order(sb, grp); + mb_update_avg_fragment_size(sb, grp); + + clear_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &(grp->bb_state)); + + period = get_cycles() - period; + atomic_inc(&sbi->s_mb_buddies_generated); + atomic64_add(period, &sbi->s_mb_generation_time); +} + +static void mb_regenerate_buddy(struct ext4_buddy *e4b) +{ + int count; + int order = 1; + void *buddy; + + while ((buddy = mb_find_buddy(e4b, order++, &count))) + mb_set_bits(buddy, 0, count); + + e4b->bd_info->bb_fragments = 0; + memset(e4b->bd_info->bb_counters, 0, + sizeof(*e4b->bd_info->bb_counters) * + (e4b->bd_sb->s_blocksize_bits + 2)); + + ext4_mb_generate_buddy(e4b->bd_sb, e4b->bd_buddy, + e4b->bd_bitmap, e4b->bd_group, e4b->bd_info); +} + +/* The buddy information is attached the buddy cache inode + * for convenience. The information regarding each group + * is loaded via ext4_mb_load_buddy. The information involve + * block bitmap and buddy information. The information are + * stored in the inode as + * + * { folio } + * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]... + * + * + * one block each for bitmap and buddy information. + * So for each group we take up 2 blocks. A folio can + * contain blocks_per_folio (folio_size / blocksize) blocks. + * So it can have information regarding groups_per_folio which + * is blocks_per_folio/2 + * + * Locking note: This routine takes the block group lock of all groups + * for this folio; do not hold this lock when calling this routine! + */ +static int ext4_mb_init_cache(struct folio *folio, char *incore, gfp_t gfp) +{ + ext4_group_t ngroups; + unsigned int blocksize; + int blocks_per_folio; + int groups_per_folio; + int err = 0; + int i; + ext4_group_t first_group, group; + int first_block; + struct super_block *sb; + struct buffer_head *bhs; + struct buffer_head **bh = NULL; + struct inode *inode; + char *data; + char *bitmap; + struct ext4_group_info *grinfo; + + inode = folio->mapping->host; + sb = inode->i_sb; + ngroups = ext4_get_groups_count(sb); + blocksize = i_blocksize(inode); + blocks_per_folio = folio_size(folio) / blocksize; + WARN_ON_ONCE(!blocks_per_folio); + groups_per_folio = DIV_ROUND_UP(blocks_per_folio, 2); + + mb_debug(sb, "init folio %lu\n", folio->index); + + /* allocate buffer_heads to read bitmaps */ + if (groups_per_folio > 1) { + i = sizeof(struct buffer_head *) * groups_per_folio; + bh = kzalloc(i, gfp); + if (bh == NULL) + return -ENOMEM; + } else + bh = &bhs; + + /* read all groups the folio covers into the cache */ + first_group = EXT4_PG_TO_LBLK(inode, folio->index) / 2; + for (i = 0, group = first_group; i < groups_per_folio; i++, group++) { + if (group >= ngroups) + break; + + grinfo = ext4_get_group_info(sb, group); + if (!grinfo) + continue; + /* + * If folio is uptodate then we came here after online resize + * which added some new uninitialized group info structs, so + * we must skip all initialized uptodate buddies on the folio, + * which may be currently in use by an allocating task. + */ + if (folio_test_uptodate(folio) && + !EXT4_MB_GRP_NEED_INIT(grinfo)) { + bh[i] = NULL; + continue; + } + bh[i] = ext4_read_block_bitmap_nowait(sb, group, false); + if (IS_ERR(bh[i])) { + err = PTR_ERR(bh[i]); + bh[i] = NULL; + goto out; + } + mb_debug(sb, "read bitmap for group %u\n", group); + } + + /* wait for I/O completion */ + for (i = 0, group = first_group; i < groups_per_folio; i++, group++) { + int err2; + + if (!bh[i]) + continue; + err2 = ext4_wait_block_bitmap(sb, group, bh[i]); + if (!err) + err = err2; + } + + first_block = EXT4_PG_TO_LBLK(inode, folio->index); + for (i = 0; i < blocks_per_folio; i++) { + group = (first_block + i) >> 1; + if (group >= ngroups) + break; + + if (!bh[group - first_group]) + /* skip initialized uptodate buddy */ + continue; + + if (!buffer_verified(bh[group - first_group])) + /* Skip faulty bitmaps */ + continue; + err = 0; + + /* + * data carry information regarding this + * particular group in the format specified + * above + * + */ + data = folio_address(folio) + (i * blocksize); + bitmap = bh[group - first_group]->b_data; + + /* + * We place the buddy block and bitmap block + * close together + */ + grinfo = ext4_get_group_info(sb, group); + if (!grinfo) { + err = -EFSCORRUPTED; + goto out; + } + if ((first_block + i) & 1) { + /* this is block of buddy */ + BUG_ON(incore == NULL); + mb_debug(sb, "put buddy for group %u in folio %lu/%x\n", + group, folio->index, i * blocksize); + trace_ext4_mb_buddy_bitmap_load(sb, group); + grinfo->bb_fragments = 0; + memset(grinfo->bb_counters, 0, + sizeof(*grinfo->bb_counters) * + (MB_NUM_ORDERS(sb))); + /* + * incore got set to the group block bitmap below + */ + ext4_lock_group(sb, group); + /* init the buddy */ + memset(data, 0xff, blocksize); + ext4_mb_generate_buddy(sb, data, incore, group, grinfo); + ext4_unlock_group(sb, group); + incore = NULL; + } else { + /* this is block of bitmap */ + BUG_ON(incore != NULL); + mb_debug(sb, "put bitmap for group %u in folio %lu/%x\n", + group, folio->index, i * blocksize); + trace_ext4_mb_bitmap_load(sb, group); + + /* see comments in ext4_mb_put_pa() */ + ext4_lock_group(sb, group); + memcpy(data, bitmap, blocksize); + + /* mark all preallocated blks used in in-core bitmap */ + ext4_mb_generate_from_pa(sb, data, group); + WARN_ON_ONCE(!RB_EMPTY_ROOT(&grinfo->bb_free_root)); + ext4_unlock_group(sb, group); + + /* set incore so that the buddy information can be + * generated using this + */ + incore = data; + } + } + folio_mark_uptodate(folio); + +out: + if (bh) { + for (i = 0; i < groups_per_folio; i++) + brelse(bh[i]); + if (bh != &bhs) + kfree(bh); + } + return err; +} + +/* + * Lock the buddy and bitmap folios. This makes sure other parallel init_group + * on the same buddy folio doesn't happen while holding the buddy folio lock. + * Return locked buddy and bitmap folios on e4b struct. If buddy and bitmap + * are on the same folio e4b->bd_buddy_folio is NULL and return value is 0. + */ +static int ext4_mb_get_buddy_folio_lock(struct super_block *sb, + ext4_group_t group, struct ext4_buddy *e4b, gfp_t gfp) +{ + struct inode *inode = EXT4_SB(sb)->s_buddy_cache; + int block, pnum; + struct folio *folio; + + e4b->bd_buddy_folio = NULL; + e4b->bd_bitmap_folio = NULL; + + /* + * the buddy cache inode stores the block bitmap + * and buddy information in consecutive blocks. + * So for each group we need two blocks. + */ + block = group * 2; + pnum = EXT4_LBLK_TO_PG(inode, block); + folio = __filemap_get_folio(inode->i_mapping, pnum, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp); + if (IS_ERR(folio)) + return PTR_ERR(folio); + BUG_ON(folio->mapping != inode->i_mapping); + WARN_ON_ONCE(folio_size(folio) < sb->s_blocksize); + e4b->bd_bitmap_folio = folio; + e4b->bd_bitmap = folio_address(folio) + + offset_in_folio(folio, EXT4_LBLK_TO_B(inode, block)); + + block++; + pnum = EXT4_LBLK_TO_PG(inode, block); + if (folio_contains(folio, pnum)) { + /* buddy and bitmap are on the same folio */ + return 0; + } + + /* we need another folio for the buddy */ + folio = __filemap_get_folio(inode->i_mapping, pnum, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp); + if (IS_ERR(folio)) + return PTR_ERR(folio); + BUG_ON(folio->mapping != inode->i_mapping); + WARN_ON_ONCE(folio_size(folio) < sb->s_blocksize); + e4b->bd_buddy_folio = folio; + return 0; +} + +static void ext4_mb_put_buddy_folio_lock(struct ext4_buddy *e4b) +{ + if (e4b->bd_bitmap_folio) { + folio_unlock(e4b->bd_bitmap_folio); + folio_put(e4b->bd_bitmap_folio); + } + if (e4b->bd_buddy_folio) { + folio_unlock(e4b->bd_buddy_folio); + folio_put(e4b->bd_buddy_folio); + } +} + +/* + * Locking note: This routine calls ext4_mb_init_cache(), which takes the + * block group lock of all groups for this folio; do not hold the BG lock when + * calling this routine! + */ +static noinline_for_stack +int ext4_mb_init_group(struct super_block *sb, ext4_group_t group, gfp_t gfp) +{ + + struct ext4_group_info *this_grp; + struct ext4_buddy e4b; + struct folio *folio; + int ret = 0; + + might_sleep(); + mb_debug(sb, "init group %u\n", group); + this_grp = ext4_get_group_info(sb, group); + if (!this_grp) + return -EFSCORRUPTED; + + /* + * This ensures that we don't reinit the buddy cache + * folio which map to the group from which we are already + * allocating. If we are looking at the buddy cache we would + * have taken a reference using ext4_mb_load_buddy and that + * would have pinned buddy folio to page cache. + * The call to ext4_mb_get_buddy_folio_lock will mark the + * folio accessed. + */ + ret = ext4_mb_get_buddy_folio_lock(sb, group, &e4b, gfp); + if (ret || !EXT4_MB_GRP_NEED_INIT(this_grp)) { + /* + * somebody initialized the group + * return without doing anything + */ + goto err; + } + + folio = e4b.bd_bitmap_folio; + ret = ext4_mb_init_cache(folio, NULL, gfp); + if (ret) + goto err; + if (!folio_test_uptodate(folio)) { + ret = -EIO; + goto err; + } + + if (e4b.bd_buddy_folio == NULL) { + /* + * If both the bitmap and buddy are in + * the same folio we don't need to force + * init the buddy + */ + ret = 0; + goto err; + } + /* init buddy cache */ + folio = e4b.bd_buddy_folio; + ret = ext4_mb_init_cache(folio, e4b.bd_bitmap, gfp); + if (ret) + goto err; + if (!folio_test_uptodate(folio)) { + ret = -EIO; + goto err; + } +err: + ext4_mb_put_buddy_folio_lock(&e4b); + return ret; +} + +/* + * Locking note: This routine calls ext4_mb_init_cache(), which takes the + * block group lock of all groups for this folio; do not hold the BG lock when + * calling this routine! + */ +static noinline_for_stack int +ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group, + struct ext4_buddy *e4b, gfp_t gfp) +{ + int block; + int pnum; + struct folio *folio; + int ret; + struct ext4_group_info *grp; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct inode *inode = sbi->s_buddy_cache; + + might_sleep(); + mb_debug(sb, "load group %u\n", group); + + grp = ext4_get_group_info(sb, group); + if (!grp) + return -EFSCORRUPTED; + + e4b->bd_blkbits = sb->s_blocksize_bits; + e4b->bd_info = grp; + e4b->bd_sb = sb; + e4b->bd_group = group; + e4b->bd_buddy_folio = NULL; + e4b->bd_bitmap_folio = NULL; + + if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) { + /* + * we need full data about the group + * to make a good selection + */ + ret = ext4_mb_init_group(sb, group, gfp); + if (ret) + return ret; + } + + /* + * the buddy cache inode stores the block bitmap + * and buddy information in consecutive blocks. + * So for each group we need two blocks. + */ + block = group * 2; + pnum = EXT4_LBLK_TO_PG(inode, block); + + /* Avoid locking the folio in the fast path ... */ + folio = __filemap_get_folio(inode->i_mapping, pnum, FGP_ACCESSED, 0); + if (IS_ERR(folio) || !folio_test_uptodate(folio) || folio_test_locked(folio)) { + /* + * folio_test_locked is employed to detect ongoing folio + * migrations, since concurrent migrations can lead to + * bitmap inconsistency. And if we are not uptodate that + * implies somebody just created the folio but is yet to + * initialize it. We can drop the folio reference and + * try to get the folio with lock in both cases to avoid + * concurrency. + */ + if (!IS_ERR(folio)) + folio_put(folio); + folio = __filemap_get_folio(inode->i_mapping, pnum, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp); + if (!IS_ERR(folio)) { + if (WARN_RATELIMIT(folio->mapping != inode->i_mapping, + "ext4: bitmap's mapping != inode->i_mapping\n")) { + /* should never happen */ + folio_unlock(folio); + ret = -EINVAL; + goto err; + } + if (!folio_test_uptodate(folio)) { + ret = ext4_mb_init_cache(folio, NULL, gfp); + if (ret) { + folio_unlock(folio); + goto err; + } + mb_cmp_bitmaps(e4b, folio_address(folio) + + offset_in_folio(folio, + EXT4_LBLK_TO_B(inode, block))); + } + folio_unlock(folio); + } + } + if (IS_ERR(folio)) { + ret = PTR_ERR(folio); + goto err; + } + if (!folio_test_uptodate(folio)) { + ret = -EIO; + goto err; + } + + /* Folios marked accessed already */ + e4b->bd_bitmap_folio = folio; + e4b->bd_bitmap = folio_address(folio) + + offset_in_folio(folio, EXT4_LBLK_TO_B(inode, block)); + + block++; + pnum = EXT4_LBLK_TO_PG(inode, block); + /* buddy and bitmap are on the same folio? */ + if (folio_contains(folio, pnum)) { + folio_get(folio); + goto update_buddy; + } + + /* we need another folio for the buddy */ + folio = __filemap_get_folio(inode->i_mapping, pnum, FGP_ACCESSED, 0); + if (IS_ERR(folio) || !folio_test_uptodate(folio) || folio_test_locked(folio)) { + if (!IS_ERR(folio)) + folio_put(folio); + folio = __filemap_get_folio(inode->i_mapping, pnum, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp); + if (!IS_ERR(folio)) { + if (WARN_RATELIMIT(folio->mapping != inode->i_mapping, + "ext4: buddy bitmap's mapping != inode->i_mapping\n")) { + /* should never happen */ + folio_unlock(folio); + ret = -EINVAL; + goto err; + } + if (!folio_test_uptodate(folio)) { + ret = ext4_mb_init_cache(folio, e4b->bd_bitmap, + gfp); + if (ret) { + folio_unlock(folio); + goto err; + } + } + folio_unlock(folio); + } + } + if (IS_ERR(folio)) { + ret = PTR_ERR(folio); + goto err; + } + if (!folio_test_uptodate(folio)) { + ret = -EIO; + goto err; + } + +update_buddy: + /* Folios marked accessed already */ + e4b->bd_buddy_folio = folio; + e4b->bd_buddy = folio_address(folio) + + offset_in_folio(folio, EXT4_LBLK_TO_B(inode, block)); + + return 0; + +err: + if (!IS_ERR_OR_NULL(folio)) + folio_put(folio); + if (e4b->bd_bitmap_folio) + folio_put(e4b->bd_bitmap_folio); + + e4b->bd_buddy = NULL; + e4b->bd_bitmap = NULL; + return ret; +} + +static int ext4_mb_load_buddy(struct super_block *sb, ext4_group_t group, + struct ext4_buddy *e4b) +{ + return ext4_mb_load_buddy_gfp(sb, group, e4b, GFP_NOFS); +} + +static void ext4_mb_unload_buddy(struct ext4_buddy *e4b) +{ + if (e4b->bd_bitmap_folio) + folio_put(e4b->bd_bitmap_folio); + if (e4b->bd_buddy_folio) + folio_put(e4b->bd_buddy_folio); +} + + +static int mb_find_order_for_block(struct ext4_buddy *e4b, int block) +{ + int order = 1, max; + void *bb; + + BUG_ON(e4b->bd_bitmap == e4b->bd_buddy); + BUG_ON(block >= (1 << (e4b->bd_blkbits + 3))); + + while (order <= e4b->bd_blkbits + 1) { + bb = mb_find_buddy(e4b, order, &max); + if (!mb_test_bit(block >> order, bb)) { + /* this block is part of buddy of order 'order' */ + return order; + } + order++; + } + return 0; +} + +static void mb_clear_bits(void *bm, int cur, int len) +{ + __u32 *addr; + + len = cur + len; + while (cur < len) { + if ((cur & 31) == 0 && (len - cur) >= 32) { + /* fast path: clear whole word at once */ + addr = bm + (cur >> 3); + *addr = 0; + cur += 32; + continue; + } + mb_clear_bit(cur, bm); + cur++; + } +} + +/* clear bits in given range + * will return first found zero bit if any, -1 otherwise + */ +static int mb_test_and_clear_bits(void *bm, int cur, int len) +{ + __u32 *addr; + int zero_bit = -1; + + len = cur + len; + while (cur < len) { + if ((cur & 31) == 0 && (len - cur) >= 32) { + /* fast path: clear whole word at once */ + addr = bm + (cur >> 3); + if (*addr != (__u32)(-1) && zero_bit == -1) + zero_bit = cur + mb_find_next_zero_bit(addr, 32, 0); + *addr = 0; + cur += 32; + continue; + } + if (!mb_test_and_clear_bit(cur, bm) && zero_bit == -1) + zero_bit = cur; + cur++; + } + + return zero_bit; +} + +void mb_set_bits(void *bm, int cur, int len) +{ + __u32 *addr; + + len = cur + len; + while (cur < len) { + if ((cur & 31) == 0 && (len - cur) >= 32) { + /* fast path: set whole word at once */ + addr = bm + (cur >> 3); + *addr = 0xffffffff; + cur += 32; + continue; + } + mb_set_bit(cur, bm); + cur++; + } +} + +static inline int mb_buddy_adjust_border(int* bit, void* bitmap, int side) +{ + if (mb_test_bit(*bit + side, bitmap)) { + mb_clear_bit(*bit, bitmap); + (*bit) -= side; + return 1; + } + else { + (*bit) += side; + mb_set_bit(*bit, bitmap); + return -1; + } +} + +static void mb_buddy_mark_free(struct ext4_buddy *e4b, int first, int last) +{ + int max; + int order = 1; + void *buddy = mb_find_buddy(e4b, order, &max); + + while (buddy) { + void *buddy2; + + /* Bits in range [first; last] are known to be set since + * corresponding blocks were allocated. Bits in range + * (first; last) will stay set because they form buddies on + * upper layer. We just deal with borders if they don't + * align with upper layer and then go up. + * Releasing entire group is all about clearing + * single bit of highest order buddy. + */ + + /* Example: + * --------------------------------- + * | 1 | 1 | 1 | 1 | + * --------------------------------- + * | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | + * --------------------------------- + * 0 1 2 3 4 5 6 7 + * \_____________________/ + * + * Neither [1] nor [6] is aligned to above layer. + * Left neighbour [0] is free, so mark it busy, + * decrease bb_counters and extend range to + * [0; 6] + * Right neighbour [7] is busy. It can't be coaleasced with [6], so + * mark [6] free, increase bb_counters and shrink range to + * [0; 5]. + * Then shift range to [0; 2], go up and do the same. + */ + + + if (first & 1) + e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&first, buddy, -1); + if (!(last & 1)) + e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&last, buddy, 1); + if (first > last) + break; + order++; + + buddy2 = mb_find_buddy(e4b, order, &max); + if (!buddy2) { + mb_clear_bits(buddy, first, last - first + 1); + e4b->bd_info->bb_counters[order - 1] += last - first + 1; + break; + } + first >>= 1; + last >>= 1; + buddy = buddy2; + } +} + +static void mb_free_blocks(struct inode *inode, struct ext4_buddy *e4b, + int first, int count) +{ + int left_is_free = 0; + int right_is_free = 0; + int block; + int last = first + count - 1; + struct super_block *sb = e4b->bd_sb; + + if (WARN_ON(count == 0)) + return; + BUG_ON(last >= (sb->s_blocksize << 3)); + assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group)); + /* Don't bother if the block group is corrupt. */ + if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info))) + return; + + mb_check_buddy(e4b); + mb_free_blocks_double(inode, e4b, first, count); + + /* access memory sequentially: check left neighbour, + * clear range and then check right neighbour + */ + if (first != 0) + left_is_free = !mb_test_bit(first - 1, e4b->bd_bitmap); + block = mb_test_and_clear_bits(e4b->bd_bitmap, first, count); + if (last + 1 < EXT4_SB(sb)->s_mb_maxs[0]) + right_is_free = !mb_test_bit(last + 1, e4b->bd_bitmap); + + if (unlikely(block != -1)) { + struct ext4_sb_info *sbi = EXT4_SB(sb); + ext4_fsblk_t blocknr; + + /* + * Fastcommit replay can free already freed blocks which + * corrupts allocation info. Regenerate it. + */ + if (sbi->s_mount_state & EXT4_FC_REPLAY) { + mb_regenerate_buddy(e4b); + goto check; + } + + blocknr = ext4_group_first_block_no(sb, e4b->bd_group); + blocknr += EXT4_C2B(sbi, block); + ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group, + EXT4_GROUP_INFO_BBITMAP_CORRUPT); + ext4_grp_locked_error(sb, e4b->bd_group, + inode ? inode->i_ino : 0, blocknr, + "freeing already freed block (bit %u); block bitmap corrupt.", + block); + return; + } + + this_cpu_inc(discard_pa_seq); + e4b->bd_info->bb_free += count; + if (first < e4b->bd_info->bb_first_free) + e4b->bd_info->bb_first_free = first; + + /* let's maintain fragments counter */ + if (left_is_free && right_is_free) + e4b->bd_info->bb_fragments--; + else if (!left_is_free && !right_is_free) + e4b->bd_info->bb_fragments++; + + /* buddy[0] == bd_bitmap is a special case, so handle + * it right away and let mb_buddy_mark_free stay free of + * zero order checks. + * Check if neighbours are to be coaleasced, + * adjust bitmap bb_counters and borders appropriately. + */ + if (first & 1) { + first += !left_is_free; + e4b->bd_info->bb_counters[0] += left_is_free ? -1 : 1; + } + if (!(last & 1)) { + last -= !right_is_free; + e4b->bd_info->bb_counters[0] += right_is_free ? -1 : 1; + } + + if (first <= last) + mb_buddy_mark_free(e4b, first >> 1, last >> 1); + + mb_set_largest_free_order(sb, e4b->bd_info); + mb_update_avg_fragment_size(sb, e4b->bd_info); +check: + mb_check_buddy(e4b); +} + +static int mb_find_extent(struct ext4_buddy *e4b, int block, + int needed, struct ext4_free_extent *ex) +{ + int max, order, next; + void *buddy; + + assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); + BUG_ON(ex == NULL); + + buddy = mb_find_buddy(e4b, 0, &max); + BUG_ON(buddy == NULL); + BUG_ON(block >= max); + if (mb_test_bit(block, buddy)) { + ex->fe_len = 0; + ex->fe_start = 0; + ex->fe_group = 0; + return 0; + } + + /* find actual order */ + order = mb_find_order_for_block(e4b, block); + + ex->fe_len = (1 << order) - (block & ((1 << order) - 1)); + ex->fe_start = block; + ex->fe_group = e4b->bd_group; + + block = block >> order; + + while (needed > ex->fe_len && + mb_find_buddy(e4b, order, &max)) { + + if (block + 1 >= max) + break; + + next = (block + 1) * (1 << order); + if (mb_test_bit(next, e4b->bd_bitmap)) + break; + + order = mb_find_order_for_block(e4b, next); + + block = next >> order; + ex->fe_len += 1 << order; + } + + if (ex->fe_start + ex->fe_len > EXT4_CLUSTERS_PER_GROUP(e4b->bd_sb)) { + /* Should never happen! (but apparently sometimes does?!?) */ + WARN_ON(1); + ext4_grp_locked_error(e4b->bd_sb, e4b->bd_group, 0, 0, + "corruption or bug in mb_find_extent " + "block=%d, order=%d needed=%d ex=%u/%d/%d@%u", + block, order, needed, ex->fe_group, ex->fe_start, + ex->fe_len, ex->fe_logical); + ex->fe_len = 0; + ex->fe_start = 0; + ex->fe_group = 0; + } + return ex->fe_len; +} + +static int mb_mark_used(struct ext4_buddy *e4b, struct ext4_free_extent *ex) +{ + int ord; + int mlen = 0; + int max = 0; + int start = ex->fe_start; + int len = ex->fe_len; + unsigned ret = 0; + int len0 = len; + void *buddy; + int ord_start, ord_end; + + BUG_ON(start + len > (e4b->bd_sb->s_blocksize << 3)); + BUG_ON(e4b->bd_group != ex->fe_group); + assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); + mb_check_buddy(e4b); + mb_mark_used_double(e4b, start, len); + + this_cpu_inc(discard_pa_seq); + e4b->bd_info->bb_free -= len; + if (e4b->bd_info->bb_first_free == start) + e4b->bd_info->bb_first_free += len; + + /* let's maintain fragments counter */ + if (start != 0) + mlen = !mb_test_bit(start - 1, e4b->bd_bitmap); + if (start + len < EXT4_SB(e4b->bd_sb)->s_mb_maxs[0]) + max = !mb_test_bit(start + len, e4b->bd_bitmap); + if (mlen && max) + e4b->bd_info->bb_fragments++; + else if (!mlen && !max) + e4b->bd_info->bb_fragments--; + + /* let's maintain buddy itself */ + while (len) { + ord = mb_find_order_for_block(e4b, start); + + if (((start >> ord) << ord) == start && len >= (1 << ord)) { + /* the whole chunk may be allocated at once! */ + mlen = 1 << ord; + buddy = mb_find_buddy(e4b, ord, &max); + BUG_ON((start >> ord) >= max); + mb_set_bit(start >> ord, buddy); + e4b->bd_info->bb_counters[ord]--; + start += mlen; + len -= mlen; + BUG_ON(len < 0); + continue; + } + + /* store for history */ + if (ret == 0) + ret = len | (ord << 16); + + BUG_ON(ord <= 0); + buddy = mb_find_buddy(e4b, ord, &max); + mb_set_bit(start >> ord, buddy); + e4b->bd_info->bb_counters[ord]--; + + ord_start = (start >> ord) << ord; + ord_end = ord_start + (1 << ord); + /* first chunk */ + if (start > ord_start) + ext4_mb_mark_free_simple(e4b->bd_sb, e4b->bd_buddy, + ord_start, start - ord_start, + e4b->bd_info); + + /* last chunk */ + if (start + len < ord_end) { + ext4_mb_mark_free_simple(e4b->bd_sb, e4b->bd_buddy, + start + len, + ord_end - (start + len), + e4b->bd_info); + break; + } + len = start + len - ord_end; + start = ord_end; + } + mb_set_largest_free_order(e4b->bd_sb, e4b->bd_info); + + mb_update_avg_fragment_size(e4b->bd_sb, e4b->bd_info); + mb_set_bits(e4b->bd_bitmap, ex->fe_start, len0); + mb_check_buddy(e4b); + + return ret; +} + +/* + * Must be called under group lock! + */ +static void ext4_mb_use_best_found(struct ext4_allocation_context *ac, + struct ext4_buddy *e4b) +{ + struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); + int ret; + + BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group); + BUG_ON(ac->ac_status == AC_STATUS_FOUND); + + ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len); + ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical; + ret = mb_mark_used(e4b, &ac->ac_b_ex); + + /* preallocation can change ac_b_ex, thus we store actually + * allocated blocks for history */ + ac->ac_f_ex = ac->ac_b_ex; + + ac->ac_status = AC_STATUS_FOUND; + ac->ac_tail = ret & 0xffff; + ac->ac_buddy = ret >> 16; + + /* + * take the folio reference. We want the folio to be pinned + * so that we don't get a ext4_mb_init_cache_call for this + * group until we update the bitmap. That would mean we + * double allocate blocks. The reference is dropped + * in ext4_mb_release_context + */ + ac->ac_bitmap_folio = e4b->bd_bitmap_folio; + folio_get(ac->ac_bitmap_folio); + ac->ac_buddy_folio = e4b->bd_buddy_folio; + folio_get(ac->ac_buddy_folio); + /* store last allocated for subsequent stream allocation */ + if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { + int hash = (unsigned int)ac->ac_inode->i_ino % sbi->s_mb_nr_global_goals; + + WRITE_ONCE(sbi->s_mb_last_groups[hash], ac->ac_f_ex.fe_group); + } + + /* + * As we've just preallocated more space than + * user requested originally, we store allocated + * space in a special descriptor. + */ + if (ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len) + ext4_mb_new_preallocation(ac); + +} + +static void ext4_mb_check_limits(struct ext4_allocation_context *ac, + struct ext4_buddy *e4b, + int finish_group) +{ + struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); + struct ext4_free_extent *bex = &ac->ac_b_ex; + struct ext4_free_extent *gex = &ac->ac_g_ex; + + if (ac->ac_status == AC_STATUS_FOUND) + return; + /* + * We don't want to scan for a whole year + */ + if (ac->ac_found > sbi->s_mb_max_to_scan && + !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { + ac->ac_status = AC_STATUS_BREAK; + return; + } + + /* + * Haven't found good chunk so far, let's continue + */ + if (bex->fe_len < gex->fe_len) + return; + + if (finish_group || ac->ac_found > sbi->s_mb_min_to_scan) + ext4_mb_use_best_found(ac, e4b); +} + +/* + * The routine checks whether found extent is good enough. If it is, + * then the extent gets marked used and flag is set to the context + * to stop scanning. Otherwise, the extent is compared with the + * previous found extent and if new one is better, then it's stored + * in the context. Later, the best found extent will be used, if + * mballoc can't find good enough extent. + * + * The algorithm used is roughly as follows: + * + * * If free extent found is exactly as big as goal, then + * stop the scan and use it immediately + * + * * If free extent found is smaller than goal, then keep retrying + * upto a max of sbi->s_mb_max_to_scan times (default 200). After + * that stop scanning and use whatever we have. + * + * * If free extent found is bigger than goal, then keep retrying + * upto a max of sbi->s_mb_min_to_scan times (default 10) before + * stopping the scan and using the extent. + * + * + * FIXME: real allocation policy is to be designed yet! + */ +static void ext4_mb_measure_extent(struct ext4_allocation_context *ac, + struct ext4_free_extent *ex, + struct ext4_buddy *e4b) +{ + struct ext4_free_extent *bex = &ac->ac_b_ex; + struct ext4_free_extent *gex = &ac->ac_g_ex; + + BUG_ON(ex->fe_len <= 0); + BUG_ON(ex->fe_len > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); + BUG_ON(ex->fe_start >= EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); + BUG_ON(ac->ac_status != AC_STATUS_CONTINUE); + + ac->ac_found++; + ac->ac_cX_found[ac->ac_criteria]++; + + /* + * The special case - take what you catch first + */ + if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) { + *bex = *ex; + ext4_mb_use_best_found(ac, e4b); + return; + } + + /* + * Let's check whether the chuck is good enough + */ + if (ex->fe_len == gex->fe_len) { + *bex = *ex; + ext4_mb_use_best_found(ac, e4b); + return; + } + + /* + * If this is first found extent, just store it in the context + */ + if (bex->fe_len == 0) { + *bex = *ex; + return; + } + + /* + * If new found extent is better, store it in the context + */ + if (bex->fe_len < gex->fe_len) { + /* if the request isn't satisfied, any found extent + * larger than previous best one is better */ + if (ex->fe_len > bex->fe_len) + *bex = *ex; + } else if (ex->fe_len > gex->fe_len) { + /* if the request is satisfied, then we try to find + * an extent that still satisfy the request, but is + * smaller than previous one */ + if (ex->fe_len < bex->fe_len) + *bex = *ex; + } + + ext4_mb_check_limits(ac, e4b, 0); +} + +static noinline_for_stack +void ext4_mb_try_best_found(struct ext4_allocation_context *ac, + struct ext4_buddy *e4b) +{ + struct ext4_free_extent ex = ac->ac_b_ex; + ext4_group_t group = ex.fe_group; + int max; + int err; + + BUG_ON(ex.fe_len <= 0); + err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); + if (err) + return; + + ext4_lock_group(ac->ac_sb, group); + if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info))) + goto out; + + max = mb_find_extent(e4b, ex.fe_start, ex.fe_len, &ex); + + if (max > 0) { + ac->ac_b_ex = ex; + ext4_mb_use_best_found(ac, e4b); + } + +out: + ext4_unlock_group(ac->ac_sb, group); + ext4_mb_unload_buddy(e4b); +} + +static noinline_for_stack +int ext4_mb_find_by_goal(struct ext4_allocation_context *ac, + struct ext4_buddy *e4b) +{ + ext4_group_t group = ac->ac_g_ex.fe_group; + int max; + int err; + struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); + struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); + struct ext4_free_extent ex; + + if (!grp) + return -EFSCORRUPTED; + if (!(ac->ac_flags & (EXT4_MB_HINT_TRY_GOAL | EXT4_MB_HINT_GOAL_ONLY))) + return 0; + if (grp->bb_free == 0) + return 0; + + err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); + if (err) { + if (EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info) && + !(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) + return 0; + return err; + } + + ext4_lock_group(ac->ac_sb, group); + if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info))) + goto out; + + max = mb_find_extent(e4b, ac->ac_g_ex.fe_start, + ac->ac_g_ex.fe_len, &ex); + ex.fe_logical = 0xDEADFA11; /* debug value */ + + if (max >= ac->ac_g_ex.fe_len && + ac->ac_g_ex.fe_len == EXT4_NUM_B2C(sbi, sbi->s_stripe)) { + ext4_fsblk_t start; + + start = ext4_grp_offs_to_block(ac->ac_sb, &ex); + /* use do_div to get remainder (would be 64-bit modulo) */ + if (do_div(start, sbi->s_stripe) == 0) { + ac->ac_found++; + ac->ac_b_ex = ex; + ext4_mb_use_best_found(ac, e4b); + } + } else if (max >= ac->ac_g_ex.fe_len) { + BUG_ON(ex.fe_len <= 0); + BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); + BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); + ac->ac_found++; + ac->ac_b_ex = ex; + ext4_mb_use_best_found(ac, e4b); + } else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) { + /* Sometimes, caller may want to merge even small + * number of blocks to an existing extent */ + BUG_ON(ex.fe_len <= 0); + BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); + BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); + ac->ac_found++; + ac->ac_b_ex = ex; + ext4_mb_use_best_found(ac, e4b); + } +out: + ext4_unlock_group(ac->ac_sb, group); + ext4_mb_unload_buddy(e4b); + + return 0; +} + +/* + * The routine scans buddy structures (not bitmap!) from given order + * to max order and tries to find big enough chunk to satisfy the req + */ +static noinline_for_stack +void ext4_mb_simple_scan_group(struct ext4_allocation_context *ac, + struct ext4_buddy *e4b) +{ + struct super_block *sb = ac->ac_sb; + struct ext4_group_info *grp = e4b->bd_info; + void *buddy; + int i; + int k; + int max; + + BUG_ON(ac->ac_2order <= 0); + for (i = ac->ac_2order; i < MB_NUM_ORDERS(sb); i++) { + if (grp->bb_counters[i] == 0) + continue; + + buddy = mb_find_buddy(e4b, i, &max); + if (WARN_RATELIMIT(buddy == NULL, + "ext4: mb_simple_scan_group: mb_find_buddy failed, (%d)\n", i)) + continue; + + k = mb_find_next_zero_bit(buddy, max, 0); + if (k >= max) { + ext4_mark_group_bitmap_corrupted(ac->ac_sb, + e4b->bd_group, + EXT4_GROUP_INFO_BBITMAP_CORRUPT); + ext4_grp_locked_error(ac->ac_sb, e4b->bd_group, 0, 0, + "%d free clusters of order %d. But found 0", + grp->bb_counters[i], i); + break; + } + ac->ac_found++; + ac->ac_cX_found[ac->ac_criteria]++; + + ac->ac_b_ex.fe_len = 1 << i; + ac->ac_b_ex.fe_start = k << i; + ac->ac_b_ex.fe_group = e4b->bd_group; + + ext4_mb_use_best_found(ac, e4b); + + BUG_ON(ac->ac_f_ex.fe_len != ac->ac_g_ex.fe_len); + + if (EXT4_SB(sb)->s_mb_stats) + atomic_inc(&EXT4_SB(sb)->s_bal_2orders); + + break; + } +} + +/* + * The routine scans the group and measures all found extents. + * In order to optimize scanning, caller must pass number of + * free blocks in the group, so the routine can know upper limit. + */ +static noinline_for_stack +void ext4_mb_complex_scan_group(struct ext4_allocation_context *ac, + struct ext4_buddy *e4b) +{ + struct super_block *sb = ac->ac_sb; + void *bitmap = e4b->bd_bitmap; + struct ext4_free_extent ex; + int i, j, freelen; + int free; + + free = e4b->bd_info->bb_free; + if (WARN_ON(free <= 0)) + return; + + i = e4b->bd_info->bb_first_free; + + while (free && ac->ac_status == AC_STATUS_CONTINUE) { + i = mb_find_next_zero_bit(bitmap, + EXT4_CLUSTERS_PER_GROUP(sb), i); + if (i >= EXT4_CLUSTERS_PER_GROUP(sb)) { + /* + * IF we have corrupt bitmap, we won't find any + * free blocks even though group info says we + * have free blocks + */ + ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group, + EXT4_GROUP_INFO_BBITMAP_CORRUPT); + ext4_grp_locked_error(sb, e4b->bd_group, 0, 0, + "%d free clusters as per " + "group info. But bitmap says 0", + free); + break; + } + + if (!ext4_mb_cr_expensive(ac->ac_criteria)) { + /* + * In CR_GOAL_LEN_FAST and CR_BEST_AVAIL_LEN, we are + * sure that this group will have a large enough + * continuous free extent, so skip over the smaller free + * extents + */ + j = mb_find_next_bit(bitmap, + EXT4_CLUSTERS_PER_GROUP(sb), i); + freelen = j - i; + + if (freelen < ac->ac_g_ex.fe_len) { + i = j; + free -= freelen; + continue; + } + } + + mb_find_extent(e4b, i, ac->ac_g_ex.fe_len, &ex); + if (WARN_ON(ex.fe_len <= 0)) + break; + if (free < ex.fe_len) { + ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group, + EXT4_GROUP_INFO_BBITMAP_CORRUPT); + ext4_grp_locked_error(sb, e4b->bd_group, 0, 0, + "%d free clusters as per " + "group info. But got %d blocks", + free, ex.fe_len); + /* + * The number of free blocks differs. This mostly + * indicate that the bitmap is corrupt. So exit + * without claiming the space. + */ + break; + } + ex.fe_logical = 0xDEADC0DE; /* debug value */ + ext4_mb_measure_extent(ac, &ex, e4b); + + i += ex.fe_len; + free -= ex.fe_len; + } + + ext4_mb_check_limits(ac, e4b, 1); +} + +/* + * This is a special case for storages like raid5 + * we try to find stripe-aligned chunks for stripe-size-multiple requests + */ +static noinline_for_stack +void ext4_mb_scan_aligned(struct ext4_allocation_context *ac, + struct ext4_buddy *e4b) +{ + struct super_block *sb = ac->ac_sb; + struct ext4_sb_info *sbi = EXT4_SB(sb); + void *bitmap = e4b->bd_bitmap; + struct ext4_free_extent ex; + ext4_fsblk_t first_group_block; + ext4_fsblk_t a; + ext4_grpblk_t i, stripe; + int max; + + BUG_ON(sbi->s_stripe == 0); + + /* find first stripe-aligned block in group */ + first_group_block = ext4_group_first_block_no(sb, e4b->bd_group); + + a = first_group_block + sbi->s_stripe - 1; + do_div(a, sbi->s_stripe); + i = (a * sbi->s_stripe) - first_group_block; + + stripe = EXT4_NUM_B2C(sbi, sbi->s_stripe); + i = EXT4_B2C(sbi, i); + while (i < EXT4_CLUSTERS_PER_GROUP(sb)) { + if (!mb_test_bit(i, bitmap)) { + max = mb_find_extent(e4b, i, stripe, &ex); + if (max >= stripe) { + ac->ac_found++; + ac->ac_cX_found[ac->ac_criteria]++; + ex.fe_logical = 0xDEADF00D; /* debug value */ + ac->ac_b_ex = ex; + ext4_mb_use_best_found(ac, e4b); + break; + } + } + i += stripe; + } +} + +static void __ext4_mb_scan_group(struct ext4_allocation_context *ac) +{ + bool is_stripe_aligned; + struct ext4_sb_info *sbi; + enum criteria cr = ac->ac_criteria; + + ac->ac_groups_scanned++; + if (cr == CR_POWER2_ALIGNED) + return ext4_mb_simple_scan_group(ac, ac->ac_e4b); + + sbi = EXT4_SB(ac->ac_sb); + is_stripe_aligned = false; + if ((sbi->s_stripe >= sbi->s_cluster_ratio) && + !(ac->ac_g_ex.fe_len % EXT4_NUM_B2C(sbi, sbi->s_stripe))) + is_stripe_aligned = true; + + if ((cr == CR_GOAL_LEN_FAST || cr == CR_BEST_AVAIL_LEN) && + is_stripe_aligned) + ext4_mb_scan_aligned(ac, ac->ac_e4b); + + if (ac->ac_status == AC_STATUS_CONTINUE) + ext4_mb_complex_scan_group(ac, ac->ac_e4b); +} + +/* + * This is also called BEFORE we load the buddy bitmap. + * Returns either 1 or 0 indicating that the group is either suitable + * for the allocation or not. + */ +static bool ext4_mb_good_group(struct ext4_allocation_context *ac, + ext4_group_t group, enum criteria cr) +{ + ext4_grpblk_t free, fragments; + int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb)); + struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); + + BUG_ON(cr < CR_POWER2_ALIGNED || cr >= EXT4_MB_NUM_CRS); + + if (unlikely(!grp || EXT4_MB_GRP_BBITMAP_CORRUPT(grp))) + return false; + + free = grp->bb_free; + if (free == 0) + return false; + + fragments = grp->bb_fragments; + if (fragments == 0) + return false; + + switch (cr) { + case CR_POWER2_ALIGNED: + BUG_ON(ac->ac_2order == 0); + + /* Avoid using the first bg of a flexgroup for data files */ + if ((ac->ac_flags & EXT4_MB_HINT_DATA) && + (flex_size >= EXT4_FLEX_SIZE_DIR_ALLOC_SCHEME) && + ((group % flex_size) == 0)) + return false; + + if (free < ac->ac_g_ex.fe_len) + return false; + + if (ac->ac_2order >= MB_NUM_ORDERS(ac->ac_sb)) + return true; + + if (grp->bb_largest_free_order < ac->ac_2order) + return false; + + return true; + case CR_GOAL_LEN_FAST: + case CR_BEST_AVAIL_LEN: + if ((free / fragments) >= ac->ac_g_ex.fe_len) + return true; + break; + case CR_GOAL_LEN_SLOW: + if (free >= ac->ac_g_ex.fe_len) + return true; + break; + case CR_ANY_FREE: + return true; + default: + BUG(); + } + + return false; +} + +/* + * This could return negative error code if something goes wrong + * during ext4_mb_init_group(). This should not be called with + * ext4_lock_group() held. + * + * Note: because we are conditionally operating with the group lock in + * the EXT4_MB_STRICT_CHECK case, we need to fake out sparse in this + * function using __acquire and __release. This means we need to be + * super careful before messing with the error path handling via "goto + * out"! + */ +static int ext4_mb_good_group_nolock(struct ext4_allocation_context *ac, + ext4_group_t group, enum criteria cr) +{ + struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); + struct super_block *sb = ac->ac_sb; + struct ext4_sb_info *sbi = EXT4_SB(sb); + bool should_lock = ac->ac_flags & EXT4_MB_STRICT_CHECK; + ext4_grpblk_t free; + int ret = 0; + + if (!grp) + return -EFSCORRUPTED; + if (sbi->s_mb_stats) + atomic64_inc(&sbi->s_bal_cX_groups_considered[ac->ac_criteria]); + if (should_lock) { + ext4_lock_group(sb, group); + __release(ext4_group_lock_ptr(sb, group)); + } + free = grp->bb_free; + if (free == 0) + goto out; + /* + * In all criterias except CR_ANY_FREE we try to avoid groups that + * can't possibly satisfy the full goal request due to insufficient + * free blocks. + */ + if (cr < CR_ANY_FREE && free < ac->ac_g_ex.fe_len) + goto out; + if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(grp))) + goto out; + if (should_lock) { + __acquire(ext4_group_lock_ptr(sb, group)); + ext4_unlock_group(sb, group); + } + + /* We only do this if the grp has never been initialized */ + if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) { + struct ext4_group_desc *gdp = + ext4_get_group_desc(sb, group, NULL); + int ret; + + /* + * CR_POWER2_ALIGNED/CR_GOAL_LEN_FAST is a very optimistic + * search to find large good chunks almost for free. If buddy + * data is not ready, then this optimization makes no sense. But + * we never skip the first block group in a flex_bg, since this + * gets used for metadata block allocation, and we want to make + * sure we locate metadata blocks in the first block group in + * the flex_bg if possible. + */ + if (!ext4_mb_cr_expensive(cr) && + (!sbi->s_log_groups_per_flex || + ((group & ((1 << sbi->s_log_groups_per_flex) - 1)) != 0)) && + !(ext4_has_group_desc_csum(sb) && + (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)))) + return 0; + ret = ext4_mb_init_group(sb, group, GFP_NOFS); + if (ret) + return ret; + } + + if (should_lock) { + ext4_lock_group(sb, group); + __release(ext4_group_lock_ptr(sb, group)); + } + ret = ext4_mb_good_group(ac, group, cr); +out: + if (should_lock) { + __acquire(ext4_group_lock_ptr(sb, group)); + ext4_unlock_group(sb, group); + } + return ret; +} + +/* + * Start prefetching @nr block bitmaps starting at @group. + * Return the next group which needs to be prefetched. + */ +ext4_group_t ext4_mb_prefetch(struct super_block *sb, ext4_group_t group, + unsigned int nr, int *cnt) +{ + ext4_group_t ngroups = ext4_get_groups_count(sb); + struct buffer_head *bh; + struct blk_plug plug; + + blk_start_plug(&plug); + while (nr-- > 0) { + struct ext4_group_info *grp = ext4_get_group_info(sb, group); + + /* + * Prefetch block groups with free blocks; but don't + * bother if it is marked uninitialized on disk, since + * it won't require I/O to read. Also only try to + * prefetch once, so we avoid getblk() call, which can + * be expensive. + */ + if (grp && !EXT4_MB_GRP_TEST_AND_SET_READ(grp) && + EXT4_MB_GRP_NEED_INIT(grp)) { + struct ext4_group_desc *gdp = ext4_get_group_desc(sb, group, NULL); + + if (gdp && ext4_free_group_clusters(sb, gdp) > 0) { + bh = ext4_read_block_bitmap_nowait(sb, group, true); + if (!IS_ERR_OR_NULL(bh)) { + if (!buffer_uptodate(bh) && cnt) + (*cnt)++; + brelse(bh); + } + } + } + if (++group >= ngroups) + group = 0; + } + blk_finish_plug(&plug); + return group; +} + +/* + * Batch reads of the block allocation bitmaps to get + * multiple READs in flight; limit prefetching at inexpensive + * CR, otherwise mballoc can spend a lot of time loading + * imperfect groups + */ +static void ext4_mb_might_prefetch(struct ext4_allocation_context *ac, + ext4_group_t group) +{ + struct ext4_sb_info *sbi; + + if (ac->ac_prefetch_grp != group) + return; + + sbi = EXT4_SB(ac->ac_sb); + if (ext4_mb_cr_expensive(ac->ac_criteria) || + ac->ac_prefetch_ios < sbi->s_mb_prefetch_limit) { + unsigned int nr = sbi->s_mb_prefetch; + + if (ext4_has_feature_flex_bg(ac->ac_sb)) { + nr = 1 << sbi->s_log_groups_per_flex; + nr -= group & (nr - 1); + nr = umin(nr, sbi->s_mb_prefetch); + } + + ac->ac_prefetch_nr = nr; + ac->ac_prefetch_grp = ext4_mb_prefetch(ac->ac_sb, group, nr, + &ac->ac_prefetch_ios); + } +} + +/* + * Prefetching reads the block bitmap into the buffer cache; but we + * need to make sure that the buddy bitmap in the page cache has been + * initialized. Note that ext4_mb_init_group() will block if the I/O + * is not yet completed, or indeed if it was not initiated by + * ext4_mb_prefetch did not start the I/O. + * + * TODO: We should actually kick off the buddy bitmap setup in a work + * queue when the buffer I/O is completed, so that we don't block + * waiting for the block allocation bitmap read to finish when + * ext4_mb_prefetch_fini is called from ext4_mb_regular_allocator(). + */ +void ext4_mb_prefetch_fini(struct super_block *sb, ext4_group_t group, + unsigned int nr) +{ + struct ext4_group_desc *gdp; + struct ext4_group_info *grp; + + while (nr-- > 0) { + if (!group) + group = ext4_get_groups_count(sb); + group--; + gdp = ext4_get_group_desc(sb, group, NULL); + grp = ext4_get_group_info(sb, group); + + if (grp && gdp && EXT4_MB_GRP_NEED_INIT(grp) && + ext4_free_group_clusters(sb, gdp) > 0) { + if (ext4_mb_init_group(sb, group, GFP_NOFS)) + break; + } + } +} + +static int ext4_mb_scan_group(struct ext4_allocation_context *ac, + ext4_group_t group) +{ + int ret; + struct super_block *sb = ac->ac_sb; + enum criteria cr = ac->ac_criteria; + + ext4_mb_might_prefetch(ac, group); + + /* prevent unnecessary buddy loading. */ + if (cr < CR_ANY_FREE && spin_is_locked(ext4_group_lock_ptr(sb, group))) + return 0; + + /* This now checks without needing the buddy folio */ + ret = ext4_mb_good_group_nolock(ac, group, cr); + if (ret <= 0) { + if (!ac->ac_first_err) + ac->ac_first_err = ret; + return 0; + } + + ret = ext4_mb_load_buddy(sb, group, ac->ac_e4b); + if (ret) + return ret; + + /* skip busy group */ + if (cr >= CR_ANY_FREE) + ext4_lock_group(sb, group); + else if (!ext4_try_lock_group(sb, group)) + goto out_unload; + + /* We need to check again after locking the block group. */ + if (unlikely(!ext4_mb_good_group(ac, group, cr))) + goto out_unlock; + + __ext4_mb_scan_group(ac); + +out_unlock: + ext4_unlock_group(sb, group); +out_unload: + ext4_mb_unload_buddy(ac->ac_e4b); + return ret; +} + +static noinline_for_stack int +ext4_mb_regular_allocator(struct ext4_allocation_context *ac) +{ + ext4_group_t i; + int err = 0; + struct super_block *sb = ac->ac_sb; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_buddy e4b; + + BUG_ON(ac->ac_status == AC_STATUS_FOUND); + + /* first, try the goal */ + err = ext4_mb_find_by_goal(ac, &e4b); + if (err || ac->ac_status == AC_STATUS_FOUND) + goto out; + + if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) + goto out; + + /* + * ac->ac_2order is set only if the fe_len is a power of 2 + * if ac->ac_2order is set we also set criteria to CR_POWER2_ALIGNED + * so that we try exact allocation using buddy. + */ + i = fls(ac->ac_g_ex.fe_len); + ac->ac_2order = 0; + /* + * We search using buddy data only if the order of the request + * is greater than equal to the sbi_s_mb_order2_reqs + * You can tune it via /sys/fs/ext4//mb_order2_req + * We also support searching for power-of-two requests only for + * requests upto maximum buddy size we have constructed. + */ + if (i >= sbi->s_mb_order2_reqs && i <= MB_NUM_ORDERS(sb)) { + if (is_power_of_2(ac->ac_g_ex.fe_len)) + ac->ac_2order = array_index_nospec(i - 1, + MB_NUM_ORDERS(sb)); + } + + /* if stream allocation is enabled, use global goal */ + if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { + int hash = (unsigned int)ac->ac_inode->i_ino % sbi->s_mb_nr_global_goals; + + ac->ac_g_ex.fe_group = READ_ONCE(sbi->s_mb_last_groups[hash]); + ac->ac_g_ex.fe_start = -1; + ac->ac_flags &= ~EXT4_MB_HINT_TRY_GOAL; + } + + /* + * Let's just scan groups to find more-less suitable blocks We + * start with CR_GOAL_LEN_FAST, unless it is power of 2 + * aligned, in which case let's do that faster approach first. + */ + ac->ac_criteria = CR_GOAL_LEN_FAST; + if (ac->ac_2order) + ac->ac_criteria = CR_POWER2_ALIGNED; + + ac->ac_e4b = &e4b; + ac->ac_prefetch_ios = 0; + ac->ac_first_err = 0; +repeat: + while (ac->ac_criteria < EXT4_MB_NUM_CRS) { + err = ext4_mb_scan_groups(ac); + if (err) + goto out; + + if (ac->ac_status != AC_STATUS_CONTINUE) + break; + } + + if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND && + !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { + /* + * We've been searching too long. Let's try to allocate + * the best chunk we've found so far + */ + ext4_mb_try_best_found(ac, &e4b); + if (ac->ac_status != AC_STATUS_FOUND) { + int lost; + + /* + * Someone more lucky has already allocated it. + * The only thing we can do is just take first + * found block(s) + */ + lost = atomic_inc_return(&sbi->s_mb_lost_chunks); + mb_debug(sb, "lost chunk, group: %u, start: %d, len: %d, lost: %d\n", + ac->ac_b_ex.fe_group, ac->ac_b_ex.fe_start, + ac->ac_b_ex.fe_len, lost); + + ac->ac_b_ex.fe_group = 0; + ac->ac_b_ex.fe_start = 0; + ac->ac_b_ex.fe_len = 0; + ac->ac_status = AC_STATUS_CONTINUE; + ac->ac_flags |= EXT4_MB_HINT_FIRST; + ac->ac_criteria = CR_ANY_FREE; + goto repeat; + } + } + + if (sbi->s_mb_stats && ac->ac_status == AC_STATUS_FOUND) { + atomic64_inc(&sbi->s_bal_cX_hits[ac->ac_criteria]); + if (ac->ac_flags & EXT4_MB_STREAM_ALLOC && + ac->ac_b_ex.fe_group == ac->ac_g_ex.fe_group) + atomic_inc(&sbi->s_bal_stream_goals); + } +out: + if (!err && ac->ac_status != AC_STATUS_FOUND && ac->ac_first_err) + err = ac->ac_first_err; + + mb_debug(sb, "Best len %d, origin len %d, ac_status %u, ac_flags 0x%x, cr %d ret %d\n", + ac->ac_b_ex.fe_len, ac->ac_o_ex.fe_len, ac->ac_status, + ac->ac_flags, ac->ac_criteria, err); + + if (ac->ac_prefetch_nr) + ext4_mb_prefetch_fini(sb, ac->ac_prefetch_grp, ac->ac_prefetch_nr); + + return err; +} + +static void *ext4_mb_seq_groups_start(struct seq_file *seq, loff_t *pos) +{ + struct super_block *sb = pde_data(file_inode(seq->file)); + ext4_group_t group; + + if (*pos < 0 || *pos >= ext4_get_groups_count(sb)) + return NULL; + group = *pos + 1; + return (void *) ((unsigned long) group); +} + +static void *ext4_mb_seq_groups_next(struct seq_file *seq, void *v, loff_t *pos) +{ + struct super_block *sb = pde_data(file_inode(seq->file)); + ext4_group_t group; + + ++*pos; + if (*pos < 0 || *pos >= ext4_get_groups_count(sb)) + return NULL; + group = *pos + 1; + return (void *) ((unsigned long) group); +} + +static int ext4_mb_seq_groups_show(struct seq_file *seq, void *v) +{ + struct super_block *sb = pde_data(file_inode(seq->file)); + ext4_group_t group = (ext4_group_t) ((unsigned long) v); + int i, err; + char nbuf[16]; + struct ext4_buddy e4b; + struct ext4_group_info *grinfo; + unsigned char blocksize_bits = min_t(unsigned char, + sb->s_blocksize_bits, + EXT4_MAX_BLOCK_LOG_SIZE); + DEFINE_RAW_FLEX(struct ext4_group_info, sg, bb_counters, + EXT4_MAX_BLOCK_LOG_SIZE + 2); + + group--; + if (group == 0) + seq_puts(seq, "#group: free frags first [" + " 2^0 2^1 2^2 2^3 2^4 2^5 2^6 " + " 2^7 2^8 2^9 2^10 2^11 2^12 2^13 ]\n"); + + i = (blocksize_bits + 2) * sizeof(sg->bb_counters[0]) + + sizeof(struct ext4_group_info); + + grinfo = ext4_get_group_info(sb, group); + if (!grinfo) + return 0; + /* Load the group info in memory only if not already loaded. */ + if (unlikely(EXT4_MB_GRP_NEED_INIT(grinfo))) { + err = ext4_mb_load_buddy(sb, group, &e4b); + if (err) { + seq_printf(seq, "#%-5u: %s\n", group, ext4_decode_error(NULL, err, nbuf)); + return 0; + } + ext4_mb_unload_buddy(&e4b); + } + + /* + * We care only about free space counters in the group info and + * these are safe to access even after the buddy has been unloaded + */ + memcpy(sg, grinfo, i); + seq_printf(seq, "#%-5u: %-5u %-5u %-5u [", group, sg->bb_free, + sg->bb_fragments, sg->bb_first_free); + for (i = 0; i <= 13; i++) + seq_printf(seq, " %-5u", i <= blocksize_bits + 1 ? + sg->bb_counters[i] : 0); + seq_puts(seq, " ]"); + if (EXT4_MB_GRP_BBITMAP_CORRUPT(sg)) + seq_puts(seq, " Block bitmap corrupted!"); + seq_putc(seq, '\n'); + return 0; +} + +static void ext4_mb_seq_groups_stop(struct seq_file *seq, void *v) +{ +} + +const struct seq_operations ext4_mb_seq_groups_ops = { + .start = ext4_mb_seq_groups_start, + .next = ext4_mb_seq_groups_next, + .stop = ext4_mb_seq_groups_stop, + .show = ext4_mb_seq_groups_show, +}; + +int ext4_seq_mb_stats_show(struct seq_file *seq, void *offset) +{ + struct super_block *sb = seq->private; + struct ext4_sb_info *sbi = EXT4_SB(sb); + + seq_puts(seq, "mballoc:\n"); + if (!sbi->s_mb_stats) { + seq_puts(seq, "\tmb stats collection turned off.\n"); + seq_puts( + seq, + "\tTo enable, please write \"1\" to sysfs file mb_stats.\n"); + return 0; + } + seq_printf(seq, "\treqs: %u\n", atomic_read(&sbi->s_bal_reqs)); + seq_printf(seq, "\tsuccess: %u\n", atomic_read(&sbi->s_bal_success)); + + seq_printf(seq, "\tgroups_scanned: %u\n", + atomic_read(&sbi->s_bal_groups_scanned)); + + /* CR_POWER2_ALIGNED stats */ + seq_puts(seq, "\tcr_p2_aligned_stats:\n"); + seq_printf(seq, "\t\thits: %llu\n", + atomic64_read(&sbi->s_bal_cX_hits[CR_POWER2_ALIGNED])); + seq_printf( + seq, "\t\tgroups_considered: %llu\n", + atomic64_read( + &sbi->s_bal_cX_groups_considered[CR_POWER2_ALIGNED])); + seq_printf(seq, "\t\textents_scanned: %u\n", + atomic_read(&sbi->s_bal_cX_ex_scanned[CR_POWER2_ALIGNED])); + seq_printf(seq, "\t\tuseless_loops: %llu\n", + atomic64_read(&sbi->s_bal_cX_failed[CR_POWER2_ALIGNED])); + + /* CR_GOAL_LEN_FAST stats */ + seq_puts(seq, "\tcr_goal_fast_stats:\n"); + seq_printf(seq, "\t\thits: %llu\n", + atomic64_read(&sbi->s_bal_cX_hits[CR_GOAL_LEN_FAST])); + seq_printf(seq, "\t\tgroups_considered: %llu\n", + atomic64_read( + &sbi->s_bal_cX_groups_considered[CR_GOAL_LEN_FAST])); + seq_printf(seq, "\t\textents_scanned: %u\n", + atomic_read(&sbi->s_bal_cX_ex_scanned[CR_GOAL_LEN_FAST])); + seq_printf(seq, "\t\tuseless_loops: %llu\n", + atomic64_read(&sbi->s_bal_cX_failed[CR_GOAL_LEN_FAST])); + + /* CR_BEST_AVAIL_LEN stats */ + seq_puts(seq, "\tcr_best_avail_stats:\n"); + seq_printf(seq, "\t\thits: %llu\n", + atomic64_read(&sbi->s_bal_cX_hits[CR_BEST_AVAIL_LEN])); + seq_printf( + seq, "\t\tgroups_considered: %llu\n", + atomic64_read( + &sbi->s_bal_cX_groups_considered[CR_BEST_AVAIL_LEN])); + seq_printf(seq, "\t\textents_scanned: %u\n", + atomic_read(&sbi->s_bal_cX_ex_scanned[CR_BEST_AVAIL_LEN])); + seq_printf(seq, "\t\tuseless_loops: %llu\n", + atomic64_read(&sbi->s_bal_cX_failed[CR_BEST_AVAIL_LEN])); + + /* CR_GOAL_LEN_SLOW stats */ + seq_puts(seq, "\tcr_goal_slow_stats:\n"); + seq_printf(seq, "\t\thits: %llu\n", + atomic64_read(&sbi->s_bal_cX_hits[CR_GOAL_LEN_SLOW])); + seq_printf(seq, "\t\tgroups_considered: %llu\n", + atomic64_read( + &sbi->s_bal_cX_groups_considered[CR_GOAL_LEN_SLOW])); + seq_printf(seq, "\t\textents_scanned: %u\n", + atomic_read(&sbi->s_bal_cX_ex_scanned[CR_GOAL_LEN_SLOW])); + seq_printf(seq, "\t\tuseless_loops: %llu\n", + atomic64_read(&sbi->s_bal_cX_failed[CR_GOAL_LEN_SLOW])); + + /* CR_ANY_FREE stats */ + seq_puts(seq, "\tcr_any_free_stats:\n"); + seq_printf(seq, "\t\thits: %llu\n", + atomic64_read(&sbi->s_bal_cX_hits[CR_ANY_FREE])); + seq_printf( + seq, "\t\tgroups_considered: %llu\n", + atomic64_read(&sbi->s_bal_cX_groups_considered[CR_ANY_FREE])); + seq_printf(seq, "\t\textents_scanned: %u\n", + atomic_read(&sbi->s_bal_cX_ex_scanned[CR_ANY_FREE])); + seq_printf(seq, "\t\tuseless_loops: %llu\n", + atomic64_read(&sbi->s_bal_cX_failed[CR_ANY_FREE])); + + /* Aggregates */ + seq_printf(seq, "\textents_scanned: %u\n", + atomic_read(&sbi->s_bal_ex_scanned)); + seq_printf(seq, "\t\tgoal_hits: %u\n", atomic_read(&sbi->s_bal_goals)); + seq_printf(seq, "\t\tstream_goal_hits: %u\n", + atomic_read(&sbi->s_bal_stream_goals)); + seq_printf(seq, "\t\tlen_goal_hits: %u\n", + atomic_read(&sbi->s_bal_len_goals)); + seq_printf(seq, "\t\t2^n_hits: %u\n", atomic_read(&sbi->s_bal_2orders)); + seq_printf(seq, "\t\tbreaks: %u\n", atomic_read(&sbi->s_bal_breaks)); + seq_printf(seq, "\t\tlost: %u\n", atomic_read(&sbi->s_mb_lost_chunks)); + seq_printf(seq, "\tbuddies_generated: %u/%u\n", + atomic_read(&sbi->s_mb_buddies_generated), + ext4_get_groups_count(sb)); + seq_printf(seq, "\tbuddies_time_used: %llu\n", + atomic64_read(&sbi->s_mb_generation_time)); + seq_printf(seq, "\tpreallocated: %u\n", + atomic_read(&sbi->s_mb_preallocated)); + seq_printf(seq, "\tdiscarded: %u\n", atomic_read(&sbi->s_mb_discarded)); + return 0; +} + +static void *ext4_mb_seq_structs_summary_start(struct seq_file *seq, loff_t *pos) +{ + struct super_block *sb = pde_data(file_inode(seq->file)); + unsigned long position; + + if (*pos < 0 || *pos >= 2*MB_NUM_ORDERS(sb)) + return NULL; + position = *pos + 1; + return (void *) ((unsigned long) position); +} + +static void *ext4_mb_seq_structs_summary_next(struct seq_file *seq, void *v, loff_t *pos) +{ + struct super_block *sb = pde_data(file_inode(seq->file)); + unsigned long position; + + ++*pos; + if (*pos < 0 || *pos >= 2*MB_NUM_ORDERS(sb)) + return NULL; + position = *pos + 1; + return (void *) ((unsigned long) position); +} + +static int ext4_mb_seq_structs_summary_show(struct seq_file *seq, void *v) +{ + struct super_block *sb = pde_data(file_inode(seq->file)); + struct ext4_sb_info *sbi = EXT4_SB(sb); + unsigned long position = ((unsigned long) v); + struct ext4_group_info *grp; + unsigned int count; + unsigned long idx; + + position--; + if (position >= MB_NUM_ORDERS(sb)) { + position -= MB_NUM_ORDERS(sb); + if (position == 0) + seq_puts(seq, "avg_fragment_size_lists:\n"); + + count = 0; + xa_for_each(&sbi->s_mb_avg_fragment_size[position], idx, grp) + count++; + seq_printf(seq, "\tlist_order_%u_groups: %u\n", + (unsigned int)position, count); + return 0; + } + + if (position == 0) { + seq_printf(seq, "optimize_scan: %d\n", + test_opt2(sb, MB_OPTIMIZE_SCAN) ? 1 : 0); + seq_puts(seq, "max_free_order_lists:\n"); + } + count = 0; + xa_for_each(&sbi->s_mb_largest_free_orders[position], idx, grp) + count++; + seq_printf(seq, "\tlist_order_%u_groups: %u\n", + (unsigned int)position, count); + + return 0; +} + +static void ext4_mb_seq_structs_summary_stop(struct seq_file *seq, void *v) +{ +} + +const struct seq_operations ext4_mb_seq_structs_summary_ops = { + .start = ext4_mb_seq_structs_summary_start, + .next = ext4_mb_seq_structs_summary_next, + .stop = ext4_mb_seq_structs_summary_stop, + .show = ext4_mb_seq_structs_summary_show, +}; + +static struct kmem_cache *get_groupinfo_cache(int blocksize_bits) +{ + int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE; + struct kmem_cache *cachep = ext4_groupinfo_caches[cache_index]; + + BUG_ON(!cachep); + return cachep; +} + +/* + * Allocate the top-level s_group_info array for the specified number + * of groups + */ +int ext4_mb_alloc_groupinfo(struct super_block *sb, ext4_group_t ngroups) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + unsigned size; + struct ext4_group_info ***old_groupinfo, ***new_groupinfo; + + size = (ngroups + EXT4_DESC_PER_BLOCK(sb) - 1) >> + EXT4_DESC_PER_BLOCK_BITS(sb); + if (size <= sbi->s_group_info_size) + return 0; + + size = roundup_pow_of_two(sizeof(*sbi->s_group_info) * size); + new_groupinfo = kvzalloc(size, GFP_KERNEL); + if (!new_groupinfo) { + ext4_msg(sb, KERN_ERR, "can't allocate buddy meta group"); + return -ENOMEM; + } + rcu_read_lock(); + old_groupinfo = rcu_dereference(sbi->s_group_info); + if (old_groupinfo) + memcpy(new_groupinfo, old_groupinfo, + sbi->s_group_info_size * sizeof(*sbi->s_group_info)); + rcu_read_unlock(); + rcu_assign_pointer(sbi->s_group_info, new_groupinfo); + sbi->s_group_info_size = size / sizeof(*sbi->s_group_info); + if (old_groupinfo) + ext4_kvfree_array_rcu(old_groupinfo); + ext4_debug("allocated s_groupinfo array for %d meta_bg's\n", + sbi->s_group_info_size); + return 0; +} + +/* Create and initialize ext4_group_info data for the given group. */ +int ext4_mb_add_groupinfo(struct super_block *sb, ext4_group_t group, + struct ext4_group_desc *desc) +{ + int i; + int metalen = 0; + int idx = group >> EXT4_DESC_PER_BLOCK_BITS(sb); + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_group_info **meta_group_info; + struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits); + + /* + * First check if this group is the first of a reserved block. + * If it's true, we have to allocate a new table of pointers + * to ext4_group_info structures + */ + if (group % EXT4_DESC_PER_BLOCK(sb) == 0) { + metalen = sizeof(*meta_group_info) << + EXT4_DESC_PER_BLOCK_BITS(sb); + meta_group_info = kmalloc(metalen, GFP_NOFS); + if (meta_group_info == NULL) { + ext4_msg(sb, KERN_ERR, "can't allocate mem " + "for a buddy group"); + return -ENOMEM; + } + rcu_read_lock(); + rcu_dereference(sbi->s_group_info)[idx] = meta_group_info; + rcu_read_unlock(); + } + + meta_group_info = sbi_array_rcu_deref(sbi, s_group_info, idx); + i = group & (EXT4_DESC_PER_BLOCK(sb) - 1); + + meta_group_info[i] = kmem_cache_zalloc(cachep, GFP_NOFS); + if (meta_group_info[i] == NULL) { + ext4_msg(sb, KERN_ERR, "can't allocate buddy mem"); + goto exit_group_info; + } + set_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, + &(meta_group_info[i]->bb_state)); + + /* + * initialize bb_free to be able to skip + * empty groups without initialization + */ + if (ext4_has_group_desc_csum(sb) && + (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) { + meta_group_info[i]->bb_free = + ext4_free_clusters_after_init(sb, group, desc); + } else { + meta_group_info[i]->bb_free = + ext4_free_group_clusters(sb, desc); + } + + INIT_LIST_HEAD(&meta_group_info[i]->bb_prealloc_list); + init_rwsem(&meta_group_info[i]->alloc_sem); + meta_group_info[i]->bb_free_root = RB_ROOT; + meta_group_info[i]->bb_largest_free_order = -1; /* uninit */ + meta_group_info[i]->bb_avg_fragment_size_order = -1; /* uninit */ + meta_group_info[i]->bb_group = group; + + mb_group_bb_bitmap_alloc(sb, meta_group_info[i], group); + return 0; + +exit_group_info: + /* If a meta_group_info table has been allocated, release it now */ + if (group % EXT4_DESC_PER_BLOCK(sb) == 0) { + struct ext4_group_info ***group_info; + + rcu_read_lock(); + group_info = rcu_dereference(sbi->s_group_info); + kfree(group_info[idx]); + group_info[idx] = NULL; + rcu_read_unlock(); + } + return -ENOMEM; +} /* ext4_mb_add_groupinfo */ + +static int ext4_mb_init_backend(struct super_block *sb) +{ + ext4_group_t ngroups = ext4_get_groups_count(sb); + ext4_group_t i; + struct ext4_sb_info *sbi = EXT4_SB(sb); + int err; + struct ext4_group_desc *desc; + struct ext4_group_info ***group_info; + struct kmem_cache *cachep; + + err = ext4_mb_alloc_groupinfo(sb, ngroups); + if (err) + return err; + + sbi->s_buddy_cache = new_inode(sb); + if (sbi->s_buddy_cache == NULL) { + ext4_msg(sb, KERN_ERR, "can't get new inode"); + goto err_freesgi; + } + /* To avoid potentially colliding with an valid on-disk inode number, + * use EXT4_BAD_INO for the buddy cache inode number. This inode is + * not in the inode hash, so it should never be found by iget(), but + * this will avoid confusion if it ever shows up during debugging. */ + sbi->s_buddy_cache->i_ino = EXT4_BAD_INO; + EXT4_I(sbi->s_buddy_cache)->i_disksize = 0; + ext4_set_inode_mapping_order(sbi->s_buddy_cache); + + for (i = 0; i < ngroups; i++) { + cond_resched(); + desc = ext4_get_group_desc(sb, i, NULL); + if (desc == NULL) { + ext4_msg(sb, KERN_ERR, "can't read descriptor %u", i); + goto err_freebuddy; + } + if (ext4_mb_add_groupinfo(sb, i, desc) != 0) + goto err_freebuddy; + } + + if (ext4_has_feature_flex_bg(sb)) { + /* a single flex group is supposed to be read by a single IO. + * 2 ^ s_log_groups_per_flex != UINT_MAX as s_mb_prefetch is + * unsigned integer, so the maximum shift is 32. + */ + if (sbi->s_es->s_log_groups_per_flex >= 32) { + ext4_msg(sb, KERN_ERR, "too many log groups per flexible block group"); + goto err_freebuddy; + } + sbi->s_mb_prefetch = min_t(uint, 1 << sbi->s_es->s_log_groups_per_flex, + BLK_MAX_SEGMENT_SIZE >> (sb->s_blocksize_bits - 9)); + sbi->s_mb_prefetch *= 8; /* 8 prefetch IOs in flight at most */ + } else { + sbi->s_mb_prefetch = 32; + } + if (sbi->s_mb_prefetch > ext4_get_groups_count(sb)) + sbi->s_mb_prefetch = ext4_get_groups_count(sb); + /* + * now many real IOs to prefetch within a single allocation at + * CR_POWER2_ALIGNED. Given CR_POWER2_ALIGNED is an CPU-related + * optimization we shouldn't try to load too many groups, at some point + * we should start to use what we've got in memory. + * with an average random access time 5ms, it'd take a second to get + * 200 groups (* N with flex_bg), so let's make this limit 4 + */ + sbi->s_mb_prefetch_limit = sbi->s_mb_prefetch * 4; + if (sbi->s_mb_prefetch_limit > ext4_get_groups_count(sb)) + sbi->s_mb_prefetch_limit = ext4_get_groups_count(sb); + + return 0; + +err_freebuddy: + cachep = get_groupinfo_cache(sb->s_blocksize_bits); + while (i-- > 0) { + struct ext4_group_info *grp = ext4_get_group_info(sb, i); + + if (grp) + kmem_cache_free(cachep, grp); + } + i = sbi->s_group_info_size; + rcu_read_lock(); + group_info = rcu_dereference(sbi->s_group_info); + while (i-- > 0) + kfree(group_info[i]); + rcu_read_unlock(); + iput(sbi->s_buddy_cache); +err_freesgi: + kvfree(rcu_access_pointer(sbi->s_group_info)); + return -ENOMEM; +} + +static void ext4_groupinfo_destroy_slabs(void) +{ + int i; + + for (i = 0; i < NR_GRPINFO_CACHES; i++) { + kmem_cache_destroy(ext4_groupinfo_caches[i]); + ext4_groupinfo_caches[i] = NULL; + } +} + +static int ext4_groupinfo_create_slab(size_t size) +{ + static DEFINE_MUTEX(ext4_grpinfo_slab_create_mutex); + int slab_size; + int blocksize_bits = order_base_2(size); + int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE; + struct kmem_cache *cachep; + + if (cache_index >= NR_GRPINFO_CACHES) + return -EINVAL; + + if (unlikely(cache_index < 0)) + cache_index = 0; + + mutex_lock(&ext4_grpinfo_slab_create_mutex); + if (ext4_groupinfo_caches[cache_index]) { + mutex_unlock(&ext4_grpinfo_slab_create_mutex); + return 0; /* Already created */ + } + + slab_size = offsetof(struct ext4_group_info, + bb_counters[blocksize_bits + 2]); + + cachep = kmem_cache_create(ext4_groupinfo_slab_names[cache_index], + slab_size, 0, SLAB_RECLAIM_ACCOUNT, + NULL); + + ext4_groupinfo_caches[cache_index] = cachep; + + mutex_unlock(&ext4_grpinfo_slab_create_mutex); + if (!cachep) { + printk(KERN_EMERG + "EXT4-fs: no memory for groupinfo slab cache\n"); + return -ENOMEM; + } + + return 0; +} + +static void ext4_discard_work(struct work_struct *work) +{ + struct ext4_sb_info *sbi = container_of(work, + struct ext4_sb_info, s_discard_work); + struct super_block *sb = sbi->s_sb; + struct ext4_free_data *fd, *nfd; + struct ext4_buddy e4b; + LIST_HEAD(discard_list); + ext4_group_t grp, load_grp; + int err = 0; + + spin_lock(&sbi->s_md_lock); + list_splice_init(&sbi->s_discard_list, &discard_list); + spin_unlock(&sbi->s_md_lock); + + load_grp = UINT_MAX; + list_for_each_entry_safe(fd, nfd, &discard_list, efd_list) { + /* + * If filesystem is umounting or no memory or suffering + * from no space, give up the discard + */ + if ((sb->s_flags & SB_ACTIVE) && !err && + !atomic_read(&sbi->s_retry_alloc_pending)) { + grp = fd->efd_group; + if (grp != load_grp) { + if (load_grp != UINT_MAX) + ext4_mb_unload_buddy(&e4b); + + err = ext4_mb_load_buddy(sb, grp, &e4b); + if (err) { + kmem_cache_free(ext4_free_data_cachep, fd); + load_grp = UINT_MAX; + continue; + } else { + load_grp = grp; + } + } + + ext4_lock_group(sb, grp); + ext4_try_to_trim_range(sb, &e4b, fd->efd_start_cluster, + fd->efd_start_cluster + fd->efd_count - 1, 1); + ext4_unlock_group(sb, grp); + } + kmem_cache_free(ext4_free_data_cachep, fd); + } + + if (load_grp != UINT_MAX) + ext4_mb_unload_buddy(&e4b); +} + +static inline void ext4_mb_avg_fragment_size_destroy(struct ext4_sb_info *sbi) +{ + if (!sbi->s_mb_avg_fragment_size) + return; + + for (int i = 0; i < MB_NUM_ORDERS(sbi->s_sb); i++) + xa_destroy(&sbi->s_mb_avg_fragment_size[i]); + + kfree(sbi->s_mb_avg_fragment_size); + sbi->s_mb_avg_fragment_size = NULL; +} + +static inline void ext4_mb_largest_free_orders_destroy(struct ext4_sb_info *sbi) +{ + if (!sbi->s_mb_largest_free_orders) + return; + + for (int i = 0; i < MB_NUM_ORDERS(sbi->s_sb); i++) + xa_destroy(&sbi->s_mb_largest_free_orders[i]); + + kfree(sbi->s_mb_largest_free_orders); + sbi->s_mb_largest_free_orders = NULL; +} + +int ext4_mb_init(struct super_block *sb) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + unsigned i, j; + unsigned offset, offset_incr; + unsigned max; + int ret; + + i = MB_NUM_ORDERS(sb) * sizeof(*sbi->s_mb_offsets); + + sbi->s_mb_offsets = kmalloc(i, GFP_KERNEL); + if (sbi->s_mb_offsets == NULL) { + ret = -ENOMEM; + goto out; + } + + i = MB_NUM_ORDERS(sb) * sizeof(*sbi->s_mb_maxs); + sbi->s_mb_maxs = kmalloc(i, GFP_KERNEL); + if (sbi->s_mb_maxs == NULL) { + ret = -ENOMEM; + goto out; + } + + ret = ext4_groupinfo_create_slab(sb->s_blocksize); + if (ret < 0) + goto out; + + /* order 0 is regular bitmap */ + sbi->s_mb_maxs[0] = sb->s_blocksize << 3; + sbi->s_mb_offsets[0] = 0; + + i = 1; + offset = 0; + offset_incr = 1 << (sb->s_blocksize_bits - 1); + max = sb->s_blocksize << 2; + do { + sbi->s_mb_offsets[i] = offset; + sbi->s_mb_maxs[i] = max; + offset += offset_incr; + offset_incr = offset_incr >> 1; + max = max >> 1; + i++; + } while (i < MB_NUM_ORDERS(sb)); + + sbi->s_mb_avg_fragment_size = + kmalloc_objs(struct xarray, MB_NUM_ORDERS(sb)); + if (!sbi->s_mb_avg_fragment_size) { + ret = -ENOMEM; + goto out; + } + for (i = 0; i < MB_NUM_ORDERS(sb); i++) + xa_init(&sbi->s_mb_avg_fragment_size[i]); + + sbi->s_mb_largest_free_orders = + kmalloc_objs(struct xarray, MB_NUM_ORDERS(sb)); + if (!sbi->s_mb_largest_free_orders) { + ret = -ENOMEM; + goto out; + } + for (i = 0; i < MB_NUM_ORDERS(sb); i++) + xa_init(&sbi->s_mb_largest_free_orders[i]); + + spin_lock_init(&sbi->s_md_lock); + atomic_set(&sbi->s_mb_free_pending, 0); + INIT_LIST_HEAD(&sbi->s_freed_data_list[0]); + INIT_LIST_HEAD(&sbi->s_freed_data_list[1]); + INIT_LIST_HEAD(&sbi->s_discard_list); + INIT_WORK(&sbi->s_discard_work, ext4_discard_work); + atomic_set(&sbi->s_retry_alloc_pending, 0); + + sbi->s_mb_max_to_scan = MB_DEFAULT_MAX_TO_SCAN; + sbi->s_mb_min_to_scan = MB_DEFAULT_MIN_TO_SCAN; + sbi->s_mb_stats = MB_DEFAULT_STATS; + sbi->s_mb_stream_request = MB_DEFAULT_STREAM_THRESHOLD; + sbi->s_mb_order2_reqs = MB_DEFAULT_ORDER2_REQS; + sbi->s_mb_best_avail_max_trim_order = MB_DEFAULT_BEST_AVAIL_TRIM_ORDER; + + /* + * The default group preallocation is 512, which for 4k block + * sizes translates to 2 megabytes. However for bigalloc file + * systems, this is probably too big (i.e, if the cluster size + * is 1 megabyte, then group preallocation size becomes half a + * gigabyte!). As a default, we will keep a two megabyte + * group pralloc size for cluster sizes up to 64k, and after + * that, we will force a minimum group preallocation size of + * 32 clusters. This translates to 8 megs when the cluster + * size is 256k, and 32 megs when the cluster size is 1 meg, + * which seems reasonable as a default. + */ + sbi->s_mb_group_prealloc = max(MB_DEFAULT_GROUP_PREALLOC >> + sbi->s_cluster_bits, 32); + /* + * If there is a s_stripe > 1, then we set the s_mb_group_prealloc + * to the lowest multiple of s_stripe which is bigger than + * the s_mb_group_prealloc as determined above. We want + * the preallocation size to be an exact multiple of the + * RAID stripe size so that preallocations don't fragment + * the stripes. + */ + if (sbi->s_stripe > 1) { + sbi->s_mb_group_prealloc = roundup( + sbi->s_mb_group_prealloc, EXT4_NUM_B2C(sbi, sbi->s_stripe)); + } + + sbi->s_mb_nr_global_goals = umin(num_possible_cpus(), + DIV_ROUND_UP(sbi->s_groups_count, 4)); + sbi->s_mb_last_groups = kzalloc_objs(ext4_group_t, + sbi->s_mb_nr_global_goals); + if (sbi->s_mb_last_groups == NULL) { + ret = -ENOMEM; + goto out; + } + + sbi->s_locality_groups = alloc_percpu(struct ext4_locality_group); + if (sbi->s_locality_groups == NULL) { + ret = -ENOMEM; + goto out_free_last_groups; + } + for_each_possible_cpu(i) { + struct ext4_locality_group *lg; + lg = per_cpu_ptr(sbi->s_locality_groups, i); + mutex_init(&lg->lg_mutex); + for (j = 0; j < PREALLOC_TB_SIZE; j++) + INIT_LIST_HEAD(&lg->lg_prealloc_list[j]); + spin_lock_init(&lg->lg_prealloc_lock); + } + + if (!bdev_rot(sb->s_bdev)) + sbi->s_mb_max_linear_groups = 0; + else + sbi->s_mb_max_linear_groups = MB_DEFAULT_LINEAR_LIMIT; + /* init file for buddy data */ + ret = ext4_mb_init_backend(sb); + if (ret != 0) + goto out_free_locality_groups; + + return 0; + +out_free_locality_groups: + free_percpu(sbi->s_locality_groups); + sbi->s_locality_groups = NULL; +out_free_last_groups: + kfree(sbi->s_mb_last_groups); + sbi->s_mb_last_groups = NULL; +out: + ext4_mb_avg_fragment_size_destroy(sbi); + ext4_mb_largest_free_orders_destroy(sbi); + kfree(sbi->s_mb_offsets); + sbi->s_mb_offsets = NULL; + kfree(sbi->s_mb_maxs); + sbi->s_mb_maxs = NULL; + return ret; +} + +/* need to called with the ext4 group lock held */ +static int ext4_mb_cleanup_pa(struct ext4_group_info *grp) +{ + struct ext4_prealloc_space *pa; + struct list_head *cur, *tmp; + int count = 0; + + list_for_each_safe(cur, tmp, &grp->bb_prealloc_list) { + pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); + list_del(&pa->pa_group_list); + count++; + kmem_cache_free(ext4_pspace_cachep, pa); + } + return count; +} + +void ext4_mb_release(struct super_block *sb) +{ + ext4_group_t ngroups = ext4_get_groups_count(sb); + ext4_group_t i; + int num_meta_group_infos; + struct ext4_group_info *grinfo, ***group_info; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits); + int count; + + /* + * wait the discard work to drain all of ext4_free_data + */ + flush_work(&sbi->s_discard_work); + WARN_ON_ONCE(!list_empty(&sbi->s_discard_list)); + + group_info = rcu_access_pointer(sbi->s_group_info); + if (group_info) { + for (i = 0; i < ngroups; i++) { + cond_resched(); + grinfo = ext4_get_group_info(sb, i); + if (!grinfo) + continue; + mb_group_bb_bitmap_free(grinfo); + ext4_lock_group(sb, i); + count = ext4_mb_cleanup_pa(grinfo); + if (count) + mb_debug(sb, "mballoc: %d PAs left\n", + count); + ext4_unlock_group(sb, i); + kmem_cache_free(cachep, grinfo); + } + num_meta_group_infos = (ngroups + + EXT4_DESC_PER_BLOCK(sb) - 1) >> + EXT4_DESC_PER_BLOCK_BITS(sb); + for (i = 0; i < num_meta_group_infos; i++) + kfree(group_info[i]); + kvfree(group_info); + } + ext4_mb_avg_fragment_size_destroy(sbi); + ext4_mb_largest_free_orders_destroy(sbi); + kfree(sbi->s_mb_offsets); + kfree(sbi->s_mb_maxs); + iput(sbi->s_buddy_cache); + if (sbi->s_mb_stats) { + ext4_msg(sb, KERN_INFO, + "mballoc: %u blocks %u reqs (%u success)", + atomic_read(&sbi->s_bal_allocated), + atomic_read(&sbi->s_bal_reqs), + atomic_read(&sbi->s_bal_success)); + ext4_msg(sb, KERN_INFO, + "mballoc: %u extents scanned, %u groups scanned, %u goal hits, " + "%u 2^N hits, %u breaks, %u lost", + atomic_read(&sbi->s_bal_ex_scanned), + atomic_read(&sbi->s_bal_groups_scanned), + atomic_read(&sbi->s_bal_goals), + atomic_read(&sbi->s_bal_2orders), + atomic_read(&sbi->s_bal_breaks), + atomic_read(&sbi->s_mb_lost_chunks)); + ext4_msg(sb, KERN_INFO, + "mballoc: %u generated and it took %llu", + atomic_read(&sbi->s_mb_buddies_generated), + atomic64_read(&sbi->s_mb_generation_time)); + ext4_msg(sb, KERN_INFO, + "mballoc: %u preallocated, %u discarded", + atomic_read(&sbi->s_mb_preallocated), + atomic_read(&sbi->s_mb_discarded)); + } + + free_percpu(sbi->s_locality_groups); + kfree(sbi->s_mb_last_groups); +} + +static inline int ext4_issue_discard(struct super_block *sb, + ext4_group_t block_group, ext4_grpblk_t cluster, int count) +{ + ext4_fsblk_t discard_block; + + discard_block = (EXT4_C2B(EXT4_SB(sb), cluster) + + ext4_group_first_block_no(sb, block_group)); + count = EXT4_C2B(EXT4_SB(sb), count); + trace_ext4_discard_blocks(sb, + (unsigned long long) discard_block, count); + + return sb_issue_discard(sb, discard_block, count, GFP_NOFS, 0); +} + +static void ext4_free_data_in_buddy(struct super_block *sb, + struct ext4_free_data *entry) +{ + struct ext4_buddy e4b; + struct ext4_group_info *db; + int err, count = 0; + + mb_debug(sb, "gonna free %u blocks in group %u (0x%p):", + entry->efd_count, entry->efd_group, entry); + + err = ext4_mb_load_buddy(sb, entry->efd_group, &e4b); + /* we expect to find existing buddy because it's pinned */ + BUG_ON(err != 0); + + atomic_sub(entry->efd_count, &EXT4_SB(sb)->s_mb_free_pending); + db = e4b.bd_info; + /* there are blocks to put in buddy to make them really free */ + count += entry->efd_count; + ext4_lock_group(sb, entry->efd_group); + /* Take it out of per group rb tree */ + rb_erase(&entry->efd_node, &(db->bb_free_root)); + mb_free_blocks(NULL, &e4b, entry->efd_start_cluster, entry->efd_count); + + /* + * Clear the trimmed flag for the group so that the next + * ext4_trim_fs can trim it. + */ + EXT4_MB_GRP_CLEAR_TRIMMED(db); + + if (!db->bb_free_root.rb_node) { + /* No more items in the per group rb tree + * balance refcounts from ext4_mb_free_metadata() + */ + folio_put(e4b.bd_buddy_folio); + folio_put(e4b.bd_bitmap_folio); + } + ext4_unlock_group(sb, entry->efd_group); + ext4_mb_unload_buddy(&e4b); + + mb_debug(sb, "freed %d blocks in 1 structures\n", count); +} + +/* + * This function is called by the jbd2 layer once the commit has finished, + * so we know we can free the blocks that were released with that commit. + */ +void ext4_process_freed_data(struct super_block *sb, tid_t commit_tid) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_free_data *entry, *tmp; + LIST_HEAD(freed_data_list); + struct list_head *s_freed_head = &sbi->s_freed_data_list[commit_tid & 1]; + bool wake; + + list_replace_init(s_freed_head, &freed_data_list); + + list_for_each_entry(entry, &freed_data_list, efd_list) + ext4_free_data_in_buddy(sb, entry); + + if (test_opt(sb, DISCARD)) { + spin_lock(&sbi->s_md_lock); + wake = list_empty(&sbi->s_discard_list); + list_splice_tail(&freed_data_list, &sbi->s_discard_list); + spin_unlock(&sbi->s_md_lock); + if (wake) + queue_work(system_dfl_wq, &sbi->s_discard_work); + } else { + list_for_each_entry_safe(entry, tmp, &freed_data_list, efd_list) + kmem_cache_free(ext4_free_data_cachep, entry); + } +} + +int __init ext4_init_mballoc(void) +{ + ext4_pspace_cachep = KMEM_CACHE(ext4_prealloc_space, + SLAB_RECLAIM_ACCOUNT); + if (ext4_pspace_cachep == NULL) + goto out; + + ext4_ac_cachep = KMEM_CACHE(ext4_allocation_context, + SLAB_RECLAIM_ACCOUNT); + if (ext4_ac_cachep == NULL) + goto out_pa_free; + + ext4_free_data_cachep = KMEM_CACHE(ext4_free_data, + SLAB_RECLAIM_ACCOUNT); + if (ext4_free_data_cachep == NULL) + goto out_ac_free; + + return 0; + +out_ac_free: + kmem_cache_destroy(ext4_ac_cachep); +out_pa_free: + kmem_cache_destroy(ext4_pspace_cachep); +out: + return -ENOMEM; +} + +void ext4_exit_mballoc(void) +{ + /* + * Wait for completion of call_rcu()'s on ext4_pspace_cachep + * before destroying the slab cache. + */ + rcu_barrier(); + kmem_cache_destroy(ext4_pspace_cachep); + kmem_cache_destroy(ext4_ac_cachep); + kmem_cache_destroy(ext4_free_data_cachep); + ext4_groupinfo_destroy_slabs(); +} + +#define EXT4_MB_BITMAP_MARKED_CHECK 0x0001 +#define EXT4_MB_SYNC_UPDATE 0x0002 +int +ext4_mb_mark_context(handle_t *handle, struct super_block *sb, bool state, + ext4_group_t group, ext4_grpblk_t blkoff, + ext4_grpblk_t len, int flags, ext4_grpblk_t *ret_changed) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct buffer_head *bitmap_bh = NULL; + struct ext4_group_desc *gdp; + struct buffer_head *gdp_bh; + int err; + unsigned int i, already, changed = len; + + KUNIT_STATIC_STUB_REDIRECT(ext4_mb_mark_context, + handle, sb, state, group, blkoff, len, + flags, ret_changed); + + if (ret_changed) + *ret_changed = 0; + bitmap_bh = ext4_read_block_bitmap(sb, group); + if (IS_ERR(bitmap_bh)) + return PTR_ERR(bitmap_bh); + + if (handle) { + BUFFER_TRACE(bitmap_bh, "getting write access"); + err = ext4_journal_get_write_access(handle, sb, bitmap_bh, + EXT4_JTR_NONE); + if (err) + goto out_err; + } + + err = -EIO; + gdp = ext4_get_group_desc(sb, group, &gdp_bh); + if (!gdp) + goto out_err; + + if (handle) { + BUFFER_TRACE(gdp_bh, "get_write_access"); + err = ext4_journal_get_write_access(handle, sb, gdp_bh, + EXT4_JTR_NONE); + if (err) + goto out_err; + } + + ext4_lock_group(sb, group); + if (ext4_has_group_desc_csum(sb) && + (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) { + gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT); + ext4_free_group_clusters_set(sb, gdp, + ext4_free_clusters_after_init(sb, group, gdp)); + } + + if (flags & EXT4_MB_BITMAP_MARKED_CHECK) { + already = 0; + for (i = 0; i < len; i++) + if (mb_test_bit(blkoff + i, bitmap_bh->b_data) == + state) + already++; + changed = len - already; + } + + if (state) { + mb_set_bits(bitmap_bh->b_data, blkoff, len); + ext4_free_group_clusters_set(sb, gdp, + ext4_free_group_clusters(sb, gdp) - changed); + } else { + mb_clear_bits(bitmap_bh->b_data, blkoff, len); + ext4_free_group_clusters_set(sb, gdp, + ext4_free_group_clusters(sb, gdp) + changed); + } + + ext4_block_bitmap_csum_set(sb, gdp, bitmap_bh); + ext4_group_desc_csum_set(sb, group, gdp); + ext4_unlock_group(sb, group); + if (ret_changed) + *ret_changed = changed; + + if (sbi->s_log_groups_per_flex) { + ext4_group_t flex_group = ext4_flex_group(sbi, group); + struct flex_groups *fg = sbi_array_rcu_deref(sbi, + s_flex_groups, flex_group); + + if (state) + atomic64_sub(changed, &fg->free_clusters); + else + atomic64_add(changed, &fg->free_clusters); + } + + err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); + if (err) + goto out_err; + err = ext4_handle_dirty_metadata(handle, NULL, gdp_bh); + if (err) + goto out_err; + + if (flags & EXT4_MB_SYNC_UPDATE) { + sync_dirty_buffer(bitmap_bh); + sync_dirty_buffer(gdp_bh); + } + +out_err: + brelse(bitmap_bh); + return err; +} + +/* + * Check quota and mark chosen space (ac->ac_b_ex) non-free in bitmaps + * Returns 0 if success or error code + */ +static noinline_for_stack int +ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, handle_t *handle) +{ + struct ext4_group_desc *gdp; + struct ext4_sb_info *sbi; + struct super_block *sb; + ext4_fsblk_t block; + int err, len; + int flags = 0; + ext4_grpblk_t changed; + + BUG_ON(ac->ac_status != AC_STATUS_FOUND); + BUG_ON(ac->ac_b_ex.fe_len <= 0); + + sb = ac->ac_sb; + sbi = EXT4_SB(sb); + + gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, NULL); + if (!gdp) + return -EIO; + ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group, + ext4_free_group_clusters(sb, gdp)); + + block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); + len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len); + if (!ext4_inode_block_valid(ac->ac_inode, block, len)) { + ext4_error(sb, "Allocating blocks %llu-%llu which overlap " + "fs metadata", block, block+len); + /* File system mounted not to panic on error + * Fix the bitmap and return EFSCORRUPTED + * We leak some of the blocks here. + */ + err = ext4_mb_mark_context(handle, sb, true, + ac->ac_b_ex.fe_group, + ac->ac_b_ex.fe_start, + ac->ac_b_ex.fe_len, + 0, NULL); + if (!err) + err = -EFSCORRUPTED; + return err; + } + +#ifdef AGGRESSIVE_CHECK + flags |= EXT4_MB_BITMAP_MARKED_CHECK; +#endif + err = ext4_mb_mark_context(handle, sb, true, ac->ac_b_ex.fe_group, + ac->ac_b_ex.fe_start, ac->ac_b_ex.fe_len, + flags, &changed); + + if (err && changed == 0) + return err; + +#ifdef AGGRESSIVE_CHECK + BUG_ON(changed != ac->ac_b_ex.fe_len); +#endif + percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len); + + return err; +} + +/* + * Idempotent helper for Ext4 fast commit replay path to set the state of + * blocks in bitmaps and update counters. + */ +void ext4_mb_mark_bb(struct super_block *sb, ext4_fsblk_t block, + int len, bool state) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + ext4_group_t group; + ext4_grpblk_t blkoff; + int err = 0; + unsigned int clen, thisgrp_len; + + while (len > 0) { + ext4_get_group_no_and_offset(sb, block, &group, &blkoff); + + /* + * Check to see if we are freeing blocks across a group + * boundary. + * In case of flex_bg, this can happen that (block, len) may + * span across more than one group. In that case we need to + * get the corresponding group metadata to work with. + * For this we have goto again loop. + */ + thisgrp_len = min(len, EXT4_BLOCKS_PER_GROUP(sb) - EXT4_C2B(sbi, blkoff)); + clen = EXT4_NUM_B2C(sbi, thisgrp_len); + + if (!ext4_sb_block_valid(sb, NULL, block, thisgrp_len)) { + ext4_error(sb, "Marking blocks in system zone - " + "Block = %llu, len = %u", + block, thisgrp_len); + break; + } + + err = ext4_mb_mark_context(NULL, sb, state, + group, blkoff, clen, + EXT4_MB_BITMAP_MARKED_CHECK | + EXT4_MB_SYNC_UPDATE, + NULL); + if (err) + break; + + block += thisgrp_len; + len -= thisgrp_len; + BUG_ON(len < 0); + } +} + +/* + * here we normalize request for locality group + * Group request are normalized to s_mb_group_prealloc, which goes to + * s_strip if we set the same via mount option. + * s_mb_group_prealloc can be configured via + * /sys/fs/ext4//mb_group_prealloc + * + * XXX: should we try to preallocate more than the group has now? + */ +static void ext4_mb_normalize_group_request(struct ext4_allocation_context *ac) +{ + struct super_block *sb = ac->ac_sb; + struct ext4_locality_group *lg = ac->ac_lg; + + BUG_ON(lg == NULL); + ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc; + mb_debug(sb, "goal %u blocks for locality group\n", ac->ac_g_ex.fe_len); +} + +/* + * This function returns the next element to look at during inode + * PA rbtree walk. We assume that we have held the inode PA rbtree lock + * (ei->i_prealloc_lock) + * + * new_start The start of the range we want to compare + * cur_start The existing start that we are comparing against + * node The node of the rb_tree + */ +static inline struct rb_node* +ext4_mb_pa_rb_next_iter(ext4_lblk_t new_start, ext4_lblk_t cur_start, struct rb_node *node) +{ + if (new_start < cur_start) + return node->rb_left; + else + return node->rb_right; +} + +static inline void +ext4_mb_pa_assert_overlap(struct ext4_allocation_context *ac, + ext4_lblk_t start, loff_t end) +{ + struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); + struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); + struct ext4_prealloc_space *tmp_pa; + ext4_lblk_t tmp_pa_start; + loff_t tmp_pa_end; + struct rb_node *iter; + + read_lock(&ei->i_prealloc_lock); + for (iter = ei->i_prealloc_node.rb_node; iter; + iter = ext4_mb_pa_rb_next_iter(start, tmp_pa_start, iter)) { + tmp_pa = rb_entry(iter, struct ext4_prealloc_space, + pa_node.inode_node); + tmp_pa_start = tmp_pa->pa_lstart; + tmp_pa_end = pa_logical_end(sbi, tmp_pa); + + spin_lock(&tmp_pa->pa_lock); + if (tmp_pa->pa_deleted == 0) + BUG_ON(!(start >= tmp_pa_end || end <= tmp_pa_start)); + spin_unlock(&tmp_pa->pa_lock); + } + read_unlock(&ei->i_prealloc_lock); +} + +/* + * Given an allocation context "ac" and a range "start", "end", check + * and adjust boundaries if the range overlaps with any of the existing + * preallocatoins stored in the corresponding inode of the allocation context. + * + * Parameters: + * ac allocation context + * start start of the new range + * end end of the new range + */ +static inline void +ext4_mb_pa_adjust_overlap(struct ext4_allocation_context *ac, + ext4_lblk_t *start, loff_t *end) +{ + struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); + struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); + struct ext4_prealloc_space *tmp_pa = NULL, *left_pa = NULL, *right_pa = NULL; + struct rb_node *iter; + ext4_lblk_t new_start, tmp_pa_start, right_pa_start = -1; + loff_t new_end, tmp_pa_end, left_pa_end = -1; + + new_start = *start; + new_end = *end; + + /* + * Adjust the normalized range so that it doesn't overlap with any + * existing preallocated blocks(PAs). Make sure to hold the rbtree lock + * so it doesn't change underneath us. + */ + read_lock(&ei->i_prealloc_lock); + + /* Step 1: find any one immediate neighboring PA of the normalized range */ + for (iter = ei->i_prealloc_node.rb_node; iter; + iter = ext4_mb_pa_rb_next_iter(ac->ac_o_ex.fe_logical, + tmp_pa_start, iter)) { + tmp_pa = rb_entry(iter, struct ext4_prealloc_space, + pa_node.inode_node); + tmp_pa_start = tmp_pa->pa_lstart; + tmp_pa_end = pa_logical_end(sbi, tmp_pa); + + /* PA must not overlap original request */ + spin_lock(&tmp_pa->pa_lock); + if (tmp_pa->pa_deleted == 0) + BUG_ON(!(ac->ac_o_ex.fe_logical >= tmp_pa_end || + ac->ac_o_ex.fe_logical < tmp_pa_start)); + spin_unlock(&tmp_pa->pa_lock); + } + + /* + * Step 2: check if the found PA is left or right neighbor and + * get the other neighbor + */ + if (tmp_pa) { + if (tmp_pa->pa_lstart < ac->ac_o_ex.fe_logical) { + struct rb_node *tmp; + + left_pa = tmp_pa; + tmp = rb_next(&left_pa->pa_node.inode_node); + if (tmp) { + right_pa = rb_entry(tmp, + struct ext4_prealloc_space, + pa_node.inode_node); + } + } else { + struct rb_node *tmp; + + right_pa = tmp_pa; + tmp = rb_prev(&right_pa->pa_node.inode_node); + if (tmp) { + left_pa = rb_entry(tmp, + struct ext4_prealloc_space, + pa_node.inode_node); + } + } + } + + /* Step 3: get the non deleted neighbors */ + if (left_pa) { + for (iter = &left_pa->pa_node.inode_node;; + iter = rb_prev(iter)) { + if (!iter) { + left_pa = NULL; + break; + } + + tmp_pa = rb_entry(iter, struct ext4_prealloc_space, + pa_node.inode_node); + left_pa = tmp_pa; + spin_lock(&tmp_pa->pa_lock); + if (tmp_pa->pa_deleted == 0) { + spin_unlock(&tmp_pa->pa_lock); + break; + } + spin_unlock(&tmp_pa->pa_lock); + } + } + + if (right_pa) { + for (iter = &right_pa->pa_node.inode_node;; + iter = rb_next(iter)) { + if (!iter) { + right_pa = NULL; + break; + } + + tmp_pa = rb_entry(iter, struct ext4_prealloc_space, + pa_node.inode_node); + right_pa = tmp_pa; + spin_lock(&tmp_pa->pa_lock); + if (tmp_pa->pa_deleted == 0) { + spin_unlock(&tmp_pa->pa_lock); + break; + } + spin_unlock(&tmp_pa->pa_lock); + } + } + + if (left_pa) { + left_pa_end = pa_logical_end(sbi, left_pa); + BUG_ON(left_pa_end > ac->ac_o_ex.fe_logical); + } + + if (right_pa) { + right_pa_start = right_pa->pa_lstart; + BUG_ON(right_pa_start <= ac->ac_o_ex.fe_logical); + } + + /* Step 4: trim our normalized range to not overlap with the neighbors */ + if (left_pa) { + if (left_pa_end > new_start) + new_start = left_pa_end; + } + + if (right_pa) { + if (right_pa_start < new_end) + new_end = right_pa_start; + } + read_unlock(&ei->i_prealloc_lock); + + /* XXX: extra loop to check we really don't overlap preallocations */ + ext4_mb_pa_assert_overlap(ac, new_start, new_end); + + *start = new_start; + *end = new_end; +} + +/* + * Normalization means making request better in terms of + * size and alignment + */ +static noinline_for_stack void +ext4_mb_normalize_request(struct ext4_allocation_context *ac, + struct ext4_allocation_request *ar) +{ + struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); + struct ext4_super_block *es = sbi->s_es; + int bsbits, max; + loff_t size, start_off, end; + loff_t orig_size __maybe_unused; + ext4_lblk_t start; + + /* do normalize only data requests, metadata requests + do not need preallocation */ + if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) + return; + + /* sometime caller may want exact blocks */ + if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) + return; + + /* caller may indicate that preallocation isn't + * required (it's a tail, for example) */ + if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC) + return; + + if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) { + ext4_mb_normalize_group_request(ac); + return ; + } + + bsbits = ac->ac_sb->s_blocksize_bits; + + /* first, let's learn actual file size + * given current request is allocated */ + size = extent_logical_end(sbi, &ac->ac_o_ex); + size = size << bsbits; + if (size < i_size_read(ac->ac_inode)) + size = i_size_read(ac->ac_inode); + orig_size = size; + + /* max size of free chunks */ + max = 2 << bsbits; + +#define NRL_CHECK_SIZE(req, size, max, chunk_size) \ + (req <= (size) || max <= (chunk_size)) + + /* first, try to predict filesize */ + start_off = 0; + if (size <= SZ_1M) { + /* + * For files up to 1MB, round up the preallocation size to + * the next power of two, with a minimum of 16KB. + */ + if (size <= (unsigned long)SZ_16K) + size = SZ_16K; + else + size = roundup_pow_of_two(size); + } else if (NRL_CHECK_SIZE(size, 4 * 1024 * 1024, max, 2 * 1024)) { + start_off = ((loff_t)ac->ac_o_ex.fe_logical >> + (21 - bsbits)) << 21; + size = 2 * 1024 * 1024; + } else if (NRL_CHECK_SIZE(size, 8 * 1024 * 1024, max, 4 * 1024)) { + start_off = ((loff_t)ac->ac_o_ex.fe_logical >> + (22 - bsbits)) << 22; + size = 4 * 1024 * 1024; + } else if (NRL_CHECK_SIZE(EXT4_C2B(sbi, ac->ac_o_ex.fe_len), + (8<<20)>>bsbits, max, 8 * 1024)) { + start_off = ((loff_t)ac->ac_o_ex.fe_logical >> + (23 - bsbits)) << 23; + size = 8 * 1024 * 1024; + } else { + start_off = (loff_t) ac->ac_o_ex.fe_logical << bsbits; + size = (loff_t) EXT4_C2B(sbi, + ac->ac_o_ex.fe_len) << bsbits; + } + size = size >> bsbits; + start = start_off >> bsbits; + + /* + * For tiny groups (smaller than 8MB) the chosen allocation + * alignment may be larger than group size. Make sure the + * alignment does not move allocation to a different group which + * makes mballoc fail assertions later. + */ + start = max(start, rounddown(ac->ac_o_ex.fe_logical, + (ext4_lblk_t)EXT4_BLOCKS_PER_GROUP(ac->ac_sb))); + + /* avoid unnecessary preallocation that may trigger assertions */ + if (start + size > EXT_MAX_BLOCKS) + size = EXT_MAX_BLOCKS - start; + + /* don't cover already allocated blocks in selected range */ + if (ar->pleft && start <= ar->lleft) { + size -= ar->lleft + 1 - start; + start = ar->lleft + 1; + } + if (ar->pright && start + size - 1 >= ar->lright) + size -= start + size - ar->lright; + + /* + * Trim allocation request for filesystems with artificially small + * groups. + */ + if (size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)) + size = EXT4_BLOCKS_PER_GROUP(ac->ac_sb); + + end = start + size; + + ext4_mb_pa_adjust_overlap(ac, &start, &end); + + size = end - start; + + /* + * In this function "start" and "size" are normalized for better + * alignment and length such that we could preallocate more blocks. + * This normalization is done such that original request of + * ac->ac_o_ex.fe_logical & fe_len should always lie within "start" and + * "size" boundaries. + * (Note fe_len can be relaxed since FS block allocation API does not + * provide gurantee on number of contiguous blocks allocation since that + * depends upon free space left, etc). + * In case of inode pa, later we use the allocated blocks + * [pa_pstart + fe_logical - pa_lstart, fe_len/size] from the preallocated + * range of goal/best blocks [start, size] to put it at the + * ac_o_ex.fe_logical extent of this inode. + * (See ext4_mb_use_inode_pa() for more details) + */ + if (start + size <= ac->ac_o_ex.fe_logical || + start > ac->ac_o_ex.fe_logical) { + ext4_msg(ac->ac_sb, KERN_ERR, + "start %lu, size %lu, fe_logical %lu", + (unsigned long) start, (unsigned long) size, + (unsigned long) ac->ac_o_ex.fe_logical); + BUG(); + } + BUG_ON(size <= 0 || size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)); + + /* now prepare goal request */ + + /* XXX: is it better to align blocks WRT to logical + * placement or satisfy big request as is */ + ac->ac_g_ex.fe_logical = start; + ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size); + ac->ac_orig_goal_len = ac->ac_g_ex.fe_len; + + /* define goal start in order to merge */ + if (ar->pright && (ar->lright == (start + size)) && + ar->pright >= size && + ar->pright - size >= le32_to_cpu(es->s_first_data_block)) { + /* merge to the right */ + ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size, + &ac->ac_g_ex.fe_group, + &ac->ac_g_ex.fe_start); + ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; + } + if (ar->pleft && (ar->lleft + 1 == start) && + ar->pleft + 1 < ext4_blocks_count(es)) { + /* merge to the left */ + ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1, + &ac->ac_g_ex.fe_group, + &ac->ac_g_ex.fe_start); + ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; + } + + mb_debug(ac->ac_sb, "goal: %lld(was %lld) blocks at %u\n", size, + orig_size, start); +} + +static void ext4_mb_collect_stats(struct ext4_allocation_context *ac) +{ + struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); + + if (sbi->s_mb_stats && ac->ac_g_ex.fe_len >= 1) { + atomic_inc(&sbi->s_bal_reqs); + atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated); + if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len) + atomic_inc(&sbi->s_bal_success); + + atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned); + for (int i=0; iac_cX_found[i], &sbi->s_bal_cX_ex_scanned[i]); + } + + atomic_add(ac->ac_groups_scanned, &sbi->s_bal_groups_scanned); + if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start && + ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group) + atomic_inc(&sbi->s_bal_goals); + /* did we allocate as much as normalizer originally wanted? */ + if (ac->ac_f_ex.fe_len == ac->ac_orig_goal_len) + atomic_inc(&sbi->s_bal_len_goals); + + if (ac->ac_found > sbi->s_mb_max_to_scan) + atomic_inc(&sbi->s_bal_breaks); + } + + if (ac->ac_op == EXT4_MB_HISTORY_ALLOC) + trace_ext4_mballoc_alloc(ac); + else + trace_ext4_mballoc_prealloc(ac); +} + +/* + * Called on failure; free up any blocks from the inode PA for this + * context. We don't need this for MB_GROUP_PA because we only change + * pa_free in ext4_mb_release_context(), but on failure, we've already + * zeroed out ac->ac_b_ex.fe_len, so group_pa->pa_free is not changed. + */ +static void ext4_discard_allocated_blocks(struct ext4_allocation_context *ac) +{ + struct ext4_prealloc_space *pa = ac->ac_pa; + struct ext4_buddy e4b; + int err; + + if (pa == NULL) { + if (ac->ac_f_ex.fe_len == 0) + return; + err = ext4_mb_load_buddy(ac->ac_sb, ac->ac_f_ex.fe_group, &e4b); + if (WARN_RATELIMIT(err, + "ext4: mb_load_buddy failed (%d)", err)) + /* + * This should never happen since we pin the + * folios in the ext4_allocation_context so + * ext4_mb_load_buddy() should never fail. + */ + return; + ext4_lock_group(ac->ac_sb, ac->ac_f_ex.fe_group); + mb_free_blocks(ac->ac_inode, &e4b, ac->ac_f_ex.fe_start, + ac->ac_f_ex.fe_len); + ext4_unlock_group(ac->ac_sb, ac->ac_f_ex.fe_group); + ext4_mb_unload_buddy(&e4b); + return; + } + if (pa->pa_type == MB_INODE_PA) { + spin_lock(&pa->pa_lock); + pa->pa_free += ac->ac_b_ex.fe_len; + spin_unlock(&pa->pa_lock); + } +} + +/* + * use blocks preallocated to inode + */ +static void ext4_mb_use_inode_pa(struct ext4_allocation_context *ac, + struct ext4_prealloc_space *pa) +{ + struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); + ext4_fsblk_t start; + ext4_fsblk_t end; + int len; + + /* found preallocated blocks, use them */ + start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart); + end = min(pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len), + start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len)); + len = EXT4_NUM_B2C(sbi, end - start); + ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group, + &ac->ac_b_ex.fe_start); + ac->ac_b_ex.fe_len = len; + ac->ac_status = AC_STATUS_FOUND; + ac->ac_pa = pa; + + BUG_ON(start < pa->pa_pstart); + BUG_ON(end > pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len)); + BUG_ON(pa->pa_free < len); + BUG_ON(ac->ac_b_ex.fe_len <= 0); + pa->pa_free -= len; + + mb_debug(ac->ac_sb, "use %llu/%d from inode pa %p\n", start, len, pa); +} + +/* + * use blocks preallocated to locality group + */ +static void ext4_mb_use_group_pa(struct ext4_allocation_context *ac, + struct ext4_prealloc_space *pa) +{ + unsigned int len = ac->ac_o_ex.fe_len; + + ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart, + &ac->ac_b_ex.fe_group, + &ac->ac_b_ex.fe_start); + ac->ac_b_ex.fe_len = len; + ac->ac_status = AC_STATUS_FOUND; + ac->ac_pa = pa; + + /* we don't correct pa_pstart or pa_len here to avoid + * possible race when the group is being loaded concurrently + * instead we correct pa later, after blocks are marked + * in on-disk bitmap -- see ext4_mb_release_context() + * Other CPUs are prevented from allocating from this pa by lg_mutex + */ + mb_debug(ac->ac_sb, "use %u/%u from group pa %p\n", + pa->pa_lstart, len, pa); +} + +/* + * Return the prealloc space that have minimal distance + * from the goal block. @cpa is the prealloc + * space that is having currently known minimal distance + * from the goal block. + */ +static struct ext4_prealloc_space * +ext4_mb_check_group_pa(ext4_fsblk_t goal_block, + struct ext4_prealloc_space *pa, + struct ext4_prealloc_space *cpa) +{ + ext4_fsblk_t cur_distance, new_distance; + + if (cpa == NULL) { + atomic_inc(&pa->pa_count); + return pa; + } + cur_distance = abs(goal_block - cpa->pa_pstart); + new_distance = abs(goal_block - pa->pa_pstart); + + if (cur_distance <= new_distance) + return cpa; + + /* drop the previous reference */ + atomic_dec(&cpa->pa_count); + atomic_inc(&pa->pa_count); + return pa; +} + +/* + * check if found pa meets EXT4_MB_HINT_GOAL_ONLY + */ +static bool +ext4_mb_pa_goal_check(struct ext4_allocation_context *ac, + struct ext4_prealloc_space *pa) +{ + struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); + ext4_fsblk_t start; + + if (likely(!(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))) + return true; + + /* + * If EXT4_MB_HINT_GOAL_ONLY is set, ac_g_ex will not be adjusted + * in ext4_mb_normalize_request and will keep same with ac_o_ex + * from ext4_mb_initialize_context. Choose ac_g_ex here to keep + * consistent with ext4_mb_find_by_goal. + */ + start = pa->pa_pstart + + (ac->ac_g_ex.fe_logical - pa->pa_lstart); + if (ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex) != start) + return false; + + if (ac->ac_g_ex.fe_len > pa->pa_len - + EXT4_B2C(sbi, ac->ac_g_ex.fe_logical - pa->pa_lstart)) + return false; + + return true; +} + +/* + * search goal blocks in preallocated space + */ +static noinline_for_stack bool +ext4_mb_use_preallocated(struct ext4_allocation_context *ac) +{ + struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); + int order, i; + struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); + struct ext4_locality_group *lg; + struct ext4_prealloc_space *tmp_pa = NULL, *cpa = NULL; + struct rb_node *iter; + ext4_fsblk_t goal_block; + + /* only data can be preallocated */ + if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) + return false; + + /* + * first, try per-file preallocation by searching the inode pa rbtree. + * + * Here, we can't do a direct traversal of the tree because + * ext4_mb_discard_group_preallocation() can paralelly mark the pa + * deleted and that can cause direct traversal to skip some entries. + */ + read_lock(&ei->i_prealloc_lock); + + if (RB_EMPTY_ROOT(&ei->i_prealloc_node)) { + goto try_group_pa; + } + + /* + * Step 1: Find a pa with logical start immediately adjacent to the + * original logical start. This could be on the left or right. + * + * (tmp_pa->pa_lstart never changes so we can skip locking for it). + */ + for (iter = ei->i_prealloc_node.rb_node; iter; + iter = ext4_mb_pa_rb_next_iter(ac->ac_o_ex.fe_logical, + tmp_pa->pa_lstart, iter)) { + tmp_pa = rb_entry(iter, struct ext4_prealloc_space, + pa_node.inode_node); + } + + /* + * Step 2: The adjacent pa might be to the right of logical start, find + * the left adjacent pa. After this step we'd have a valid tmp_pa whose + * logical start is towards the left of original request's logical start + */ + if (tmp_pa->pa_lstart > ac->ac_o_ex.fe_logical) { + struct rb_node *tmp; + tmp = rb_prev(&tmp_pa->pa_node.inode_node); + + if (tmp) { + tmp_pa = rb_entry(tmp, struct ext4_prealloc_space, + pa_node.inode_node); + } else { + /* + * If there is no adjacent pa to the left then finding + * an overlapping pa is not possible hence stop searching + * inode pa tree + */ + goto try_group_pa; + } + } + + BUG_ON(!(tmp_pa && tmp_pa->pa_lstart <= ac->ac_o_ex.fe_logical)); + + /* + * Step 3: If the left adjacent pa is deleted, keep moving left to find + * the first non deleted adjacent pa. After this step we should have a + * valid tmp_pa which is guaranteed to be non deleted. + */ + for (iter = &tmp_pa->pa_node.inode_node;; iter = rb_prev(iter)) { + if (!iter) { + /* + * no non deleted left adjacent pa, so stop searching + * inode pa tree + */ + goto try_group_pa; + } + tmp_pa = rb_entry(iter, struct ext4_prealloc_space, + pa_node.inode_node); + spin_lock(&tmp_pa->pa_lock); + if (tmp_pa->pa_deleted == 0) { + /* + * We will keep holding the pa_lock from + * this point on because we don't want group discard + * to delete this pa underneath us. Since group + * discard is anyways an ENOSPC operation it + * should be okay for it to wait a few more cycles. + */ + break; + } else { + spin_unlock(&tmp_pa->pa_lock); + } + } + + BUG_ON(!(tmp_pa && tmp_pa->pa_lstart <= ac->ac_o_ex.fe_logical)); + BUG_ON(tmp_pa->pa_deleted == 1); + + /* + * Step 4: We now have the non deleted left adjacent pa. Only this + * pa can possibly satisfy the request hence check if it overlaps + * original logical start and stop searching if it doesn't. + */ + if (ac->ac_o_ex.fe_logical >= pa_logical_end(sbi, tmp_pa)) { + spin_unlock(&tmp_pa->pa_lock); + goto try_group_pa; + } + + /* non-extent files can't have physical blocks past 2^32 */ + if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) && + (tmp_pa->pa_pstart + EXT4_C2B(sbi, tmp_pa->pa_len) > + EXT4_MAX_BLOCK_FILE_PHYS)) { + /* + * Since PAs don't overlap, we won't find any other PA to + * satisfy this. + */ + spin_unlock(&tmp_pa->pa_lock); + goto try_group_pa; + } + + if (tmp_pa->pa_free && likely(ext4_mb_pa_goal_check(ac, tmp_pa))) { + atomic_inc(&tmp_pa->pa_count); + ext4_mb_use_inode_pa(ac, tmp_pa); + spin_unlock(&tmp_pa->pa_lock); + read_unlock(&ei->i_prealloc_lock); + return true; + } else { + /* + * We found a valid overlapping pa but couldn't use it because + * it had no free blocks. This should ideally never happen + * because: + * + * 1. When a new inode pa is added to rbtree it must have + * pa_free > 0 since otherwise we won't actually need + * preallocation. + * + * 2. An inode pa that is in the rbtree can only have it's + * pa_free become zero when another thread calls: + * ext4_mb_new_blocks + * ext4_mb_use_preallocated + * ext4_mb_use_inode_pa + * + * 3. Further, after the above calls make pa_free == 0, we will + * immediately remove it from the rbtree in: + * ext4_mb_new_blocks + * ext4_mb_release_context + * ext4_mb_put_pa + * + * 4. Since the pa_free becoming 0 and pa_free getting removed + * from tree both happen in ext4_mb_new_blocks, which is always + * called with i_data_sem held for data allocations, we can be + * sure that another process will never see a pa in rbtree with + * pa_free == 0. + */ + WARN_ON_ONCE(tmp_pa->pa_free == 0); + } + spin_unlock(&tmp_pa->pa_lock); +try_group_pa: + read_unlock(&ei->i_prealloc_lock); + + /* can we use group allocation? */ + if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)) + return false; + + /* inode may have no locality group for some reason */ + lg = ac->ac_lg; + if (lg == NULL) + return false; + order = fls(ac->ac_o_ex.fe_len) - 1; + if (order > PREALLOC_TB_SIZE - 1) + /* The max size of hash table is PREALLOC_TB_SIZE */ + order = PREALLOC_TB_SIZE - 1; + + goal_block = ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex); + /* + * search for the prealloc space that is having + * minimal distance from the goal block. + */ + for (i = order; i < PREALLOC_TB_SIZE; i++) { + rcu_read_lock(); + list_for_each_entry_rcu(tmp_pa, &lg->lg_prealloc_list[i], + pa_node.lg_list) { + spin_lock(&tmp_pa->pa_lock); + if (tmp_pa->pa_deleted == 0 && + tmp_pa->pa_free >= ac->ac_o_ex.fe_len) { + + cpa = ext4_mb_check_group_pa(goal_block, + tmp_pa, cpa); + } + spin_unlock(&tmp_pa->pa_lock); + } + rcu_read_unlock(); + } + if (cpa) { + ext4_mb_use_group_pa(ac, cpa); + return true; + } + return false; +} + +/* + * the function goes through all preallocation in this group and marks them + * used in in-core bitmap. buddy must be generated from this bitmap + * Need to be called with ext4 group lock held + */ +static noinline_for_stack +void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap, + ext4_group_t group) +{ + struct ext4_group_info *grp = ext4_get_group_info(sb, group); + struct ext4_prealloc_space *pa; + struct list_head *cur; + ext4_group_t groupnr; + ext4_grpblk_t start; + int preallocated = 0; + int len; + + if (!grp) + return; + + /* all form of preallocation discards first load group, + * so the only competing code is preallocation use. + * we don't need any locking here + * notice we do NOT ignore preallocations with pa_deleted + * otherwise we could leave used blocks available for + * allocation in buddy when concurrent ext4_mb_put_pa() + * is dropping preallocation + */ + list_for_each(cur, &grp->bb_prealloc_list) { + pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); + spin_lock(&pa->pa_lock); + ext4_get_group_no_and_offset(sb, pa->pa_pstart, + &groupnr, &start); + len = pa->pa_len; + spin_unlock(&pa->pa_lock); + if (unlikely(len == 0)) + continue; + BUG_ON(groupnr != group); + mb_set_bits(bitmap, start, len); + preallocated += len; + } + mb_debug(sb, "preallocated %d for group %u\n", preallocated, group); +} + +static void ext4_mb_mark_pa_deleted(struct super_block *sb, + struct ext4_prealloc_space *pa) +{ + struct ext4_inode_info *ei; + + if (pa->pa_deleted) { + ext4_warning(sb, "deleted pa, type:%d, pblk:%llu, lblk:%u, len:%d\n", + pa->pa_type, pa->pa_pstart, pa->pa_lstart, + pa->pa_len); + return; + } + + pa->pa_deleted = 1; + + if (pa->pa_type == MB_INODE_PA) { + ei = EXT4_I(pa->pa_inode); + atomic_dec(&ei->i_prealloc_active); + } +} + +static inline void ext4_mb_pa_free(struct ext4_prealloc_space *pa) +{ + BUG_ON(!pa); + BUG_ON(atomic_read(&pa->pa_count)); + BUG_ON(pa->pa_deleted == 0); + kmem_cache_free(ext4_pspace_cachep, pa); +} + +static void ext4_mb_pa_callback(struct rcu_head *head) +{ + struct ext4_prealloc_space *pa; + + pa = container_of(head, struct ext4_prealloc_space, u.pa_rcu); + ext4_mb_pa_free(pa); +} + +/* + * drops a reference to preallocated space descriptor + * if this was the last reference and the space is consumed + */ +static void ext4_mb_put_pa(struct ext4_allocation_context *ac, + struct super_block *sb, struct ext4_prealloc_space *pa) +{ + ext4_group_t grp; + ext4_fsblk_t grp_blk; + struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); + + /* in this short window concurrent discard can set pa_deleted */ + spin_lock(&pa->pa_lock); + if (!atomic_dec_and_test(&pa->pa_count) || pa->pa_free != 0) { + spin_unlock(&pa->pa_lock); + return; + } + + if (pa->pa_deleted == 1) { + spin_unlock(&pa->pa_lock); + return; + } + + ext4_mb_mark_pa_deleted(sb, pa); + spin_unlock(&pa->pa_lock); + + grp_blk = pa->pa_pstart; + /* + * If doing group-based preallocation, pa_pstart may be in the + * next group when pa is used up + */ + if (pa->pa_type == MB_GROUP_PA) + grp_blk--; + + grp = ext4_get_group_number(sb, grp_blk); + + /* + * possible race: + * + * P1 (buddy init) P2 (regular allocation) + * find block B in PA + * copy on-disk bitmap to buddy + * mark B in on-disk bitmap + * drop PA from group + * mark all PAs in buddy + * + * thus, P1 initializes buddy with B available. to prevent this + * we make "copy" and "mark all PAs" atomic and serialize "drop PA" + * against that pair + */ + ext4_lock_group(sb, grp); + list_del(&pa->pa_group_list); + ext4_unlock_group(sb, grp); + + if (pa->pa_type == MB_INODE_PA) { + write_lock(pa->pa_node_lock.inode_lock); + rb_erase(&pa->pa_node.inode_node, &ei->i_prealloc_node); + write_unlock(pa->pa_node_lock.inode_lock); + ext4_mb_pa_free(pa); + } else { + spin_lock(pa->pa_node_lock.lg_lock); + list_del_rcu(&pa->pa_node.lg_list); + spin_unlock(pa->pa_node_lock.lg_lock); + call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); + } +} + +static void ext4_mb_pa_rb_insert(struct rb_root *root, struct rb_node *new) +{ + struct rb_node **iter = &root->rb_node, *parent = NULL; + struct ext4_prealloc_space *iter_pa, *new_pa; + ext4_lblk_t iter_start, new_start; + + while (*iter) { + iter_pa = rb_entry(*iter, struct ext4_prealloc_space, + pa_node.inode_node); + new_pa = rb_entry(new, struct ext4_prealloc_space, + pa_node.inode_node); + iter_start = iter_pa->pa_lstart; + new_start = new_pa->pa_lstart; + + parent = *iter; + if (new_start < iter_start) + iter = &((*iter)->rb_left); + else + iter = &((*iter)->rb_right); + } + + rb_link_node(new, parent, iter); + rb_insert_color(new, root); +} + +/* + * creates new preallocated space for given inode + */ +static noinline_for_stack void +ext4_mb_new_inode_pa(struct ext4_allocation_context *ac) +{ + struct super_block *sb = ac->ac_sb; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_prealloc_space *pa; + struct ext4_group_info *grp; + struct ext4_inode_info *ei; + + /* preallocate only when found space is larger then requested */ + BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); + BUG_ON(ac->ac_status != AC_STATUS_FOUND); + BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); + BUG_ON(ac->ac_pa == NULL); + + pa = ac->ac_pa; + + if (ac->ac_b_ex.fe_len < ac->ac_orig_goal_len) { + struct ext4_free_extent ex = { + .fe_logical = ac->ac_g_ex.fe_logical, + .fe_len = ac->ac_orig_goal_len, + }; + loff_t orig_goal_end = extent_logical_end(sbi, &ex); + loff_t o_ex_end = extent_logical_end(sbi, &ac->ac_o_ex); + + /* + * We can't allocate as much as normalizer wants, so we try + * to get proper lstart to cover the original request, except + * when the goal doesn't cover the original request as below: + * + * orig_ex:2045/2055(10), isize:8417280 -> normalized:0/2048 + * best_ex:0/200(200) -> adjusted: 1848/2048(200) + */ + BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical); + BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len); + + /* + * Use the below logic for adjusting best extent as it keeps + * fragmentation in check while ensuring logical range of best + * extent doesn't overflow out of goal extent: + * + * 1. Check if best ex can be kept at end of goal (before + * cr_best_avail trimmed it) and still cover original start + * 2. Else, check if best ex can be kept at start of goal and + * still cover original end + * 3. Else, keep the best ex at start of original request. + */ + ex.fe_len = ac->ac_b_ex.fe_len; + + ex.fe_logical = orig_goal_end - EXT4_C2B(sbi, ex.fe_len); + if (ac->ac_o_ex.fe_logical >= ex.fe_logical) + goto adjust_bex; + + ex.fe_logical = ac->ac_g_ex.fe_logical; + if (o_ex_end <= extent_logical_end(sbi, &ex)) + goto adjust_bex; + + ex.fe_logical = ac->ac_o_ex.fe_logical; +adjust_bex: + ac->ac_b_ex.fe_logical = ex.fe_logical; + + BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical); + BUG_ON(extent_logical_end(sbi, &ex) > orig_goal_end); + } + + pa->pa_lstart = ac->ac_b_ex.fe_logical; + pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); + pa->pa_len = ac->ac_b_ex.fe_len; + pa->pa_free = pa->pa_len; + spin_lock_init(&pa->pa_lock); + INIT_LIST_HEAD(&pa->pa_group_list); + pa->pa_deleted = 0; + pa->pa_type = MB_INODE_PA; + + mb_debug(sb, "new inode pa %p: %llu/%d for %u\n", pa, pa->pa_pstart, + pa->pa_len, pa->pa_lstart); + trace_ext4_mb_new_inode_pa(ac, pa); + + atomic_add(pa->pa_free, &sbi->s_mb_preallocated); + ext4_mb_use_inode_pa(ac, pa); + + ei = EXT4_I(ac->ac_inode); + grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); + if (!grp) + return; + + pa->pa_node_lock.inode_lock = &ei->i_prealloc_lock; + pa->pa_inode = ac->ac_inode; + + list_add(&pa->pa_group_list, &grp->bb_prealloc_list); + + write_lock(pa->pa_node_lock.inode_lock); + ext4_mb_pa_rb_insert(&ei->i_prealloc_node, &pa->pa_node.inode_node); + write_unlock(pa->pa_node_lock.inode_lock); + atomic_inc(&ei->i_prealloc_active); +} + +/* + * creates new preallocated space for locality group inodes belongs to + */ +static noinline_for_stack void +ext4_mb_new_group_pa(struct ext4_allocation_context *ac) +{ + struct super_block *sb = ac->ac_sb; + struct ext4_locality_group *lg; + struct ext4_prealloc_space *pa; + struct ext4_group_info *grp; + + /* preallocate only when found space is larger then requested */ + BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); + BUG_ON(ac->ac_status != AC_STATUS_FOUND); + BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); + BUG_ON(ac->ac_pa == NULL); + + pa = ac->ac_pa; + + pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); + pa->pa_lstart = pa->pa_pstart; + pa->pa_len = ac->ac_b_ex.fe_len; + pa->pa_free = pa->pa_len; + spin_lock_init(&pa->pa_lock); + INIT_LIST_HEAD(&pa->pa_node.lg_list); + INIT_LIST_HEAD(&pa->pa_group_list); + pa->pa_deleted = 0; + pa->pa_type = MB_GROUP_PA; + + mb_debug(sb, "new group pa %p: %llu/%d for %u\n", pa, pa->pa_pstart, + pa->pa_len, pa->pa_lstart); + trace_ext4_mb_new_group_pa(ac, pa); + + ext4_mb_use_group_pa(ac, pa); + atomic_add(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated); + + grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); + if (!grp) + return; + lg = ac->ac_lg; + BUG_ON(lg == NULL); + + pa->pa_node_lock.lg_lock = &lg->lg_prealloc_lock; + pa->pa_inode = NULL; + + list_add(&pa->pa_group_list, &grp->bb_prealloc_list); + + /* + * We will later add the new pa to the right bucket + * after updating the pa_free in ext4_mb_release_context + */ +} + +static void ext4_mb_new_preallocation(struct ext4_allocation_context *ac) +{ + if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) + ext4_mb_new_group_pa(ac); + else + ext4_mb_new_inode_pa(ac); +} + +/* + * finds all unused blocks in on-disk bitmap, frees them in + * in-core bitmap and buddy. + * @pa must be unlinked from inode and group lists, so that + * nobody else can find/use it. + * the caller MUST hold group/inode locks. + * TODO: optimize the case when there are no in-core structures yet + */ +static noinline_for_stack void +ext4_mb_release_inode_pa(struct ext4_buddy *e4b, struct buffer_head *bitmap_bh, + struct ext4_prealloc_space *pa) +{ + struct super_block *sb = e4b->bd_sb; + struct ext4_sb_info *sbi = EXT4_SB(sb); + unsigned int end; + unsigned int next; + ext4_group_t group; + ext4_grpblk_t bit; + unsigned long long grp_blk_start; + int free = 0; + + BUG_ON(pa->pa_deleted == 0); + ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit); + grp_blk_start = pa->pa_pstart - EXT4_C2B(sbi, bit); + BUG_ON(group != e4b->bd_group && pa->pa_len != 0); + end = bit + pa->pa_len; + + while (bit < end) { + bit = mb_find_next_zero_bit(bitmap_bh->b_data, end, bit); + if (bit >= end) + break; + next = mb_find_next_bit(bitmap_bh->b_data, end, bit); + mb_debug(sb, "free preallocated %u/%u in group %u\n", + (unsigned) ext4_group_first_block_no(sb, group) + bit, + (unsigned) next - bit, (unsigned) group); + free += next - bit; + + trace_ext4_mballoc_discard(sb, NULL, group, bit, next - bit); + trace_ext4_mb_release_inode_pa(pa, (grp_blk_start + + EXT4_C2B(sbi, bit)), + next - bit); + mb_free_blocks(pa->pa_inode, e4b, bit, next - bit); + bit = next + 1; + } + if (free != pa->pa_free) { + ext4_msg(e4b->bd_sb, KERN_CRIT, + "pa %p: logic %lu, phys. %lu, len %d", + pa, (unsigned long) pa->pa_lstart, + (unsigned long) pa->pa_pstart, + pa->pa_len); + ext4_grp_locked_error(sb, group, 0, 0, "free %u, pa_free %u", + free, pa->pa_free); + /* + * pa is already deleted so we use the value obtained + * from the bitmap and continue. + */ + } + atomic_add(free, &sbi->s_mb_discarded); +} + +static noinline_for_stack void +ext4_mb_release_group_pa(struct ext4_buddy *e4b, + struct ext4_prealloc_space *pa) +{ + struct super_block *sb = e4b->bd_sb; + ext4_group_t group; + ext4_grpblk_t bit; + + trace_ext4_mb_release_group_pa(sb, pa); + BUG_ON(pa->pa_deleted == 0); + ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit); + if (unlikely(group != e4b->bd_group && pa->pa_len != 0)) { + ext4_warning(sb, "bad group: expected %u, group %u, pa_start %llu", + e4b->bd_group, group, pa->pa_pstart); + return; + } + mb_free_blocks(pa->pa_inode, e4b, bit, pa->pa_len); + atomic_add(pa->pa_len, &EXT4_SB(sb)->s_mb_discarded); + trace_ext4_mballoc_discard(sb, NULL, group, bit, pa->pa_len); +} + +/* + * releases all preallocations in given group + * + * first, we need to decide discard policy: + * - when do we discard + * 1) ENOSPC + * - how many do we discard + * 1) how many requested + */ +static noinline_for_stack int +ext4_mb_discard_group_preallocations(struct super_block *sb, + ext4_group_t group, int *busy) +{ + struct ext4_group_info *grp = ext4_get_group_info(sb, group); + struct buffer_head *bitmap_bh = NULL; + struct ext4_prealloc_space *pa, *tmp; + LIST_HEAD(list); + struct ext4_buddy e4b; + struct ext4_inode_info *ei; + int err; + int free = 0; + + if (!grp) + return 0; + mb_debug(sb, "discard preallocation for group %u\n", group); + if (list_empty(&grp->bb_prealloc_list)) + goto out_dbg; + + bitmap_bh = ext4_read_block_bitmap(sb, group); + if (IS_ERR(bitmap_bh)) { + err = PTR_ERR(bitmap_bh); + ext4_error_err(sb, -err, + "Error %d reading block bitmap for %u", + err, group); + goto out_dbg; + } + + err = ext4_mb_load_buddy(sb, group, &e4b); + if (err) { + ext4_warning(sb, "Error %d loading buddy information for %u", + err, group); + put_bh(bitmap_bh); + goto out_dbg; + } + + ext4_lock_group(sb, group); + list_for_each_entry_safe(pa, tmp, + &grp->bb_prealloc_list, pa_group_list) { + spin_lock(&pa->pa_lock); + if (atomic_read(&pa->pa_count)) { + spin_unlock(&pa->pa_lock); + *busy = 1; + continue; + } + if (pa->pa_deleted) { + spin_unlock(&pa->pa_lock); + continue; + } + + /* seems this one can be freed ... */ + ext4_mb_mark_pa_deleted(sb, pa); + + if (!free) + this_cpu_inc(discard_pa_seq); + + /* we can trust pa_free ... */ + free += pa->pa_free; + + spin_unlock(&pa->pa_lock); + + list_del(&pa->pa_group_list); + list_add(&pa->u.pa_tmp_list, &list); + } + + /* now free all selected PAs */ + list_for_each_entry_safe(pa, tmp, &list, u.pa_tmp_list) { + + /* remove from object (inode or locality group) */ + if (pa->pa_type == MB_GROUP_PA) { + spin_lock(pa->pa_node_lock.lg_lock); + list_del_rcu(&pa->pa_node.lg_list); + spin_unlock(pa->pa_node_lock.lg_lock); + } else { + write_lock(pa->pa_node_lock.inode_lock); + ei = EXT4_I(pa->pa_inode); + rb_erase(&pa->pa_node.inode_node, &ei->i_prealloc_node); + write_unlock(pa->pa_node_lock.inode_lock); + } + + list_del(&pa->u.pa_tmp_list); + + if (pa->pa_type == MB_GROUP_PA) { + ext4_mb_release_group_pa(&e4b, pa); + call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); + } else { + ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa); + ext4_mb_pa_free(pa); + } + } + + ext4_unlock_group(sb, group); + ext4_mb_unload_buddy(&e4b); + put_bh(bitmap_bh); +out_dbg: + mb_debug(sb, "discarded (%d) blocks preallocated for group %u bb_free (%d)\n", + free, group, grp->bb_free); + return free; +} + +/* + * releases all non-used preallocated blocks for given inode + * + * It's important to discard preallocations under i_data_sem + * We don't want another block to be served from the prealloc + * space when we are discarding the inode prealloc space. + * + * FIXME!! Make sure it is valid at all the call sites + */ +void ext4_discard_preallocations(struct inode *inode) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + struct super_block *sb = inode->i_sb; + struct buffer_head *bitmap_bh = NULL; + struct ext4_prealloc_space *pa, *tmp; + ext4_group_t group = 0; + LIST_HEAD(list); + struct ext4_buddy e4b; + struct rb_node *iter; + int err; + + if (!S_ISREG(inode->i_mode)) + return; + + if (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY) + return; + + mb_debug(sb, "discard preallocation for inode %llu\n", + inode->i_ino); + trace_ext4_discard_preallocations(inode, + atomic_read(&ei->i_prealloc_active)); + +repeat: + /* first, collect all pa's in the inode */ + write_lock(&ei->i_prealloc_lock); + for (iter = rb_first(&ei->i_prealloc_node); iter; + iter = rb_next(iter)) { + pa = rb_entry(iter, struct ext4_prealloc_space, + pa_node.inode_node); + BUG_ON(pa->pa_node_lock.inode_lock != &ei->i_prealloc_lock); + + spin_lock(&pa->pa_lock); + if (atomic_read(&pa->pa_count)) { + /* this shouldn't happen often - nobody should + * use preallocation while we're discarding it */ + spin_unlock(&pa->pa_lock); + write_unlock(&ei->i_prealloc_lock); + ext4_msg(sb, KERN_ERR, + "uh-oh! used pa while discarding"); + WARN_ON(1); + schedule_timeout_uninterruptible(HZ); + goto repeat; + + } + if (pa->pa_deleted == 0) { + ext4_mb_mark_pa_deleted(sb, pa); + spin_unlock(&pa->pa_lock); + rb_erase(&pa->pa_node.inode_node, &ei->i_prealloc_node); + list_add(&pa->u.pa_tmp_list, &list); + continue; + } + + /* someone is deleting pa right now */ + spin_unlock(&pa->pa_lock); + write_unlock(&ei->i_prealloc_lock); + + /* we have to wait here because pa_deleted + * doesn't mean pa is already unlinked from + * the list. as we might be called from + * ->clear_inode() the inode will get freed + * and concurrent thread which is unlinking + * pa from inode's list may access already + * freed memory, bad-bad-bad */ + + /* XXX: if this happens too often, we can + * add a flag to force wait only in case + * of ->clear_inode(), but not in case of + * regular truncate */ + schedule_timeout_uninterruptible(HZ); + goto repeat; + } + write_unlock(&ei->i_prealloc_lock); + + list_for_each_entry_safe(pa, tmp, &list, u.pa_tmp_list) { + BUG_ON(pa->pa_type != MB_INODE_PA); + group = ext4_get_group_number(sb, pa->pa_pstart); + + err = ext4_mb_load_buddy_gfp(sb, group, &e4b, + GFP_NOFS|__GFP_NOFAIL); + if (err) { + ext4_error_err(sb, -err, "Error %d loading buddy information for %u", + err, group); + continue; + } + + bitmap_bh = ext4_read_block_bitmap(sb, group); + if (IS_ERR(bitmap_bh)) { + err = PTR_ERR(bitmap_bh); + ext4_error_err(sb, -err, "Error %d reading block bitmap for %u", + err, group); + ext4_mb_unload_buddy(&e4b); + continue; + } + + ext4_lock_group(sb, group); + list_del(&pa->pa_group_list); + ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa); + ext4_unlock_group(sb, group); + + ext4_mb_unload_buddy(&e4b); + put_bh(bitmap_bh); + + list_del(&pa->u.pa_tmp_list); + ext4_mb_pa_free(pa); + } +} + +static int ext4_mb_pa_alloc(struct ext4_allocation_context *ac) +{ + struct ext4_prealloc_space *pa; + + BUG_ON(ext4_pspace_cachep == NULL); + pa = kmem_cache_zalloc(ext4_pspace_cachep, GFP_NOFS); + if (!pa) + return -ENOMEM; + atomic_set(&pa->pa_count, 1); + ac->ac_pa = pa; + return 0; +} + +static void ext4_mb_pa_put_free(struct ext4_allocation_context *ac) +{ + struct ext4_prealloc_space *pa = ac->ac_pa; + + BUG_ON(!pa); + ac->ac_pa = NULL; + WARN_ON(!atomic_dec_and_test(&pa->pa_count)); + /* + * current function is only called due to an error or due to + * len of found blocks < len of requested blocks hence the PA has not + * been added to grp->bb_prealloc_list. So we don't need to lock it + */ + pa->pa_deleted = 1; + ext4_mb_pa_free(pa); +} + +#ifdef CONFIG_EXT4_DEBUG +static inline void ext4_mb_show_pa(struct super_block *sb) +{ + ext4_group_t i, ngroups; + + if (ext4_emergency_state(sb)) + return; + + ngroups = ext4_get_groups_count(sb); + mb_debug(sb, "groups: "); + for (i = 0; i < ngroups; i++) { + struct ext4_group_info *grp = ext4_get_group_info(sb, i); + struct ext4_prealloc_space *pa; + ext4_grpblk_t start; + struct list_head *cur; + + if (!grp) + continue; + ext4_lock_group(sb, i); + list_for_each(cur, &grp->bb_prealloc_list) { + pa = list_entry(cur, struct ext4_prealloc_space, + pa_group_list); + spin_lock(&pa->pa_lock); + ext4_get_group_no_and_offset(sb, pa->pa_pstart, + NULL, &start); + spin_unlock(&pa->pa_lock); + mb_debug(sb, "PA:%u:%d:%d\n", i, start, + pa->pa_len); + } + ext4_unlock_group(sb, i); + mb_debug(sb, "%u: %d/%d\n", i, grp->bb_free, + grp->bb_fragments); + } +} + +static void ext4_mb_show_ac(struct ext4_allocation_context *ac) +{ + struct super_block *sb = ac->ac_sb; + + if (ext4_emergency_state(sb)) + return; + + mb_debug(sb, "Can't allocate:" + " Allocation context details:"); + mb_debug(sb, "status %u flags 0x%x", + ac->ac_status, ac->ac_flags); + mb_debug(sb, "orig %lu/%lu/%lu@%lu, " + "goal %lu/%lu/%lu@%lu, " + "best %lu/%lu/%lu@%lu cr %d", + (unsigned long)ac->ac_o_ex.fe_group, + (unsigned long)ac->ac_o_ex.fe_start, + (unsigned long)ac->ac_o_ex.fe_len, + (unsigned long)ac->ac_o_ex.fe_logical, + (unsigned long)ac->ac_g_ex.fe_group, + (unsigned long)ac->ac_g_ex.fe_start, + (unsigned long)ac->ac_g_ex.fe_len, + (unsigned long)ac->ac_g_ex.fe_logical, + (unsigned long)ac->ac_b_ex.fe_group, + (unsigned long)ac->ac_b_ex.fe_start, + (unsigned long)ac->ac_b_ex.fe_len, + (unsigned long)ac->ac_b_ex.fe_logical, + (int)ac->ac_criteria); + mb_debug(sb, "%u found", ac->ac_found); + mb_debug(sb, "used pa: %s, ", str_yes_no(ac->ac_pa)); + if (ac->ac_pa) + mb_debug(sb, "pa_type %s\n", ac->ac_pa->pa_type == MB_GROUP_PA ? + "group pa" : "inode pa"); + ext4_mb_show_pa(sb); +} +#else +static inline void ext4_mb_show_pa(struct super_block *sb) +{ +} +static inline void ext4_mb_show_ac(struct ext4_allocation_context *ac) +{ + ext4_mb_show_pa(ac->ac_sb); +} +#endif + +/* + * We use locality group preallocation for small size file. The size of the + * file is determined by the current size or the resulting size after + * allocation which ever is larger + * + * One can tune this size via /sys/fs/ext4//mb_stream_req + */ +static void ext4_mb_group_or_file(struct ext4_allocation_context *ac) +{ + struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); + int bsbits = ac->ac_sb->s_blocksize_bits; + loff_t size, isize; + bool inode_pa_eligible, group_pa_eligible; + + if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) + return; + + if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) + return; + + group_pa_eligible = sbi->s_mb_group_prealloc > 0; + inode_pa_eligible = true; + size = extent_logical_end(sbi, &ac->ac_o_ex); + isize = (i_size_read(ac->ac_inode) + ac->ac_sb->s_blocksize - 1) + >> bsbits; + + /* No point in using inode preallocation for closed files */ + if ((size == isize) && !ext4_fs_is_busy(sbi) && + !inode_is_open_for_write(ac->ac_inode)) + inode_pa_eligible = false; + + size = max(size, isize); + /* Don't use group allocation for large files */ + if (size > sbi->s_mb_stream_request) + group_pa_eligible = false; + + if (!group_pa_eligible) { + if (inode_pa_eligible) + ac->ac_flags |= EXT4_MB_STREAM_ALLOC; + else + ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC; + return; + } + + BUG_ON(ac->ac_lg != NULL); + /* + * locality group prealloc space are per cpu. The reason for having + * per cpu locality group is to reduce the contention between block + * request from multiple CPUs. + */ + ac->ac_lg = raw_cpu_ptr(sbi->s_locality_groups); + + /* we're going to use group allocation */ + ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC; + + /* serialize all allocations in the group */ + mutex_lock(&ac->ac_lg->lg_mutex); +} + +static noinline_for_stack void +ext4_mb_initialize_context(struct ext4_allocation_context *ac, + struct ext4_allocation_request *ar) +{ + struct super_block *sb = ar->inode->i_sb; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_super_block *es = sbi->s_es; + ext4_group_t group; + unsigned int len; + ext4_fsblk_t goal; + ext4_grpblk_t block; + + /* we can't allocate > group size */ + len = ar->len; + + /* just a dirty hack to filter too big requests */ + if (len >= EXT4_CLUSTERS_PER_GROUP(sb)) + len = EXT4_CLUSTERS_PER_GROUP(sb); + + /* start searching from the goal */ + goal = ar->goal; + if (goal < le32_to_cpu(es->s_first_data_block) || + goal >= ext4_blocks_count(es)) + goal = le32_to_cpu(es->s_first_data_block); + ext4_get_group_no_and_offset(sb, goal, &group, &block); + + /* set up allocation goals */ + ac->ac_b_ex.fe_logical = EXT4_LBLK_CMASK(sbi, ar->logical); + ac->ac_status = AC_STATUS_CONTINUE; + ac->ac_sb = sb; + ac->ac_inode = ar->inode; + ac->ac_o_ex.fe_logical = ac->ac_b_ex.fe_logical; + ac->ac_o_ex.fe_group = group; + ac->ac_o_ex.fe_start = block; + ac->ac_o_ex.fe_len = len; + ac->ac_g_ex = ac->ac_o_ex; + ac->ac_orig_goal_len = ac->ac_g_ex.fe_len; + ac->ac_flags = ar->flags; + + /* we have to define context: we'll work with a file or + * locality group. this is a policy, actually */ + ext4_mb_group_or_file(ac); + + mb_debug(sb, "init ac: %u blocks @ %u, goal %u, flags 0x%x, 2^%d, " + "left: %u/%u, right %u/%u to %swritable\n", + (unsigned) ar->len, (unsigned) ar->logical, + (unsigned) ar->goal, ac->ac_flags, ac->ac_2order, + (unsigned) ar->lleft, (unsigned) ar->pleft, + (unsigned) ar->lright, (unsigned) ar->pright, + inode_is_open_for_write(ar->inode) ? "" : "non-"); +} + +static noinline_for_stack void +ext4_mb_discard_lg_preallocations(struct super_block *sb, + struct ext4_locality_group *lg, + int order, int total_entries) +{ + ext4_group_t group = 0; + struct ext4_buddy e4b; + LIST_HEAD(discard_list); + struct ext4_prealloc_space *pa, *tmp; + + mb_debug(sb, "discard locality group preallocation\n"); + + spin_lock(&lg->lg_prealloc_lock); + list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[order], + pa_node.lg_list, + lockdep_is_held(&lg->lg_prealloc_lock)) { + spin_lock(&pa->pa_lock); + if (atomic_read(&pa->pa_count)) { + /* + * This is the pa that we just used + * for block allocation. So don't + * free that + */ + spin_unlock(&pa->pa_lock); + continue; + } + if (pa->pa_deleted) { + spin_unlock(&pa->pa_lock); + continue; + } + /* only lg prealloc space */ + BUG_ON(pa->pa_type != MB_GROUP_PA); + + /* seems this one can be freed ... */ + ext4_mb_mark_pa_deleted(sb, pa); + spin_unlock(&pa->pa_lock); + + list_del_rcu(&pa->pa_node.lg_list); + list_add(&pa->u.pa_tmp_list, &discard_list); + + total_entries--; + if (total_entries <= 5) { + /* + * we want to keep only 5 entries + * allowing it to grow to 8. This + * mak sure we don't call discard + * soon for this list. + */ + break; + } + } + spin_unlock(&lg->lg_prealloc_lock); + + list_for_each_entry_safe(pa, tmp, &discard_list, u.pa_tmp_list) { + int err; + + group = ext4_get_group_number(sb, pa->pa_pstart); + err = ext4_mb_load_buddy_gfp(sb, group, &e4b, + GFP_NOFS|__GFP_NOFAIL); + if (err) { + ext4_error_err(sb, -err, "Error %d loading buddy information for %u", + err, group); + continue; + } + ext4_lock_group(sb, group); + list_del(&pa->pa_group_list); + ext4_mb_release_group_pa(&e4b, pa); + ext4_unlock_group(sb, group); + + ext4_mb_unload_buddy(&e4b); + list_del(&pa->u.pa_tmp_list); + call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); + } +} + +/* + * We have incremented pa_count. So it cannot be freed at this + * point. Also we hold lg_mutex. So no parallel allocation is + * possible from this lg. That means pa_free cannot be updated. + * + * A parallel ext4_mb_discard_group_preallocations is possible. + * which can cause the lg_prealloc_list to be updated. + */ + +static void ext4_mb_add_n_trim(struct ext4_allocation_context *ac) +{ + int order, added = 0, lg_prealloc_count = 1; + struct super_block *sb = ac->ac_sb; + struct ext4_locality_group *lg = ac->ac_lg; + struct ext4_prealloc_space *tmp_pa, *pa = ac->ac_pa; + + order = fls(pa->pa_free) - 1; + if (order > PREALLOC_TB_SIZE - 1) + /* The max size of hash table is PREALLOC_TB_SIZE */ + order = PREALLOC_TB_SIZE - 1; + /* Add the prealloc space to lg */ + spin_lock(&lg->lg_prealloc_lock); + list_for_each_entry_rcu(tmp_pa, &lg->lg_prealloc_list[order], + pa_node.lg_list, + lockdep_is_held(&lg->lg_prealloc_lock)) { + spin_lock(&tmp_pa->pa_lock); + if (tmp_pa->pa_deleted) { + spin_unlock(&tmp_pa->pa_lock); + continue; + } + if (!added && pa->pa_free < tmp_pa->pa_free) { + /* Add to the tail of the previous entry */ + list_add_tail_rcu(&pa->pa_node.lg_list, + &tmp_pa->pa_node.lg_list); + added = 1; + /* + * we want to count the total + * number of entries in the list + */ + } + spin_unlock(&tmp_pa->pa_lock); + lg_prealloc_count++; + } + if (!added) + list_add_tail_rcu(&pa->pa_node.lg_list, + &lg->lg_prealloc_list[order]); + spin_unlock(&lg->lg_prealloc_lock); + + /* Now trim the list to be not more than 8 elements */ + if (lg_prealloc_count > 8) + ext4_mb_discard_lg_preallocations(sb, lg, + order, lg_prealloc_count); +} + +/* + * release all resource we used in allocation + */ +static void ext4_mb_release_context(struct ext4_allocation_context *ac) +{ + struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); + struct ext4_prealloc_space *pa = ac->ac_pa; + if (pa) { + if (pa->pa_type == MB_GROUP_PA) { + /* see comment in ext4_mb_use_group_pa() */ + spin_lock(&pa->pa_lock); + pa->pa_pstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); + pa->pa_lstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); + pa->pa_free -= ac->ac_b_ex.fe_len; + pa->pa_len -= ac->ac_b_ex.fe_len; + spin_unlock(&pa->pa_lock); + + /* + * We want to add the pa to the right bucket. + * Remove it from the list and while adding + * make sure the list to which we are adding + * doesn't grow big. + */ + if (likely(pa->pa_free)) { + spin_lock(pa->pa_node_lock.lg_lock); + list_del_rcu(&pa->pa_node.lg_list); + spin_unlock(pa->pa_node_lock.lg_lock); + ext4_mb_add_n_trim(ac); + } + } + + ext4_mb_put_pa(ac, ac->ac_sb, pa); + } + if (ac->ac_bitmap_folio) + folio_put(ac->ac_bitmap_folio); + if (ac->ac_buddy_folio) + folio_put(ac->ac_buddy_folio); + if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) + mutex_unlock(&ac->ac_lg->lg_mutex); + ext4_mb_collect_stats(ac); +} + +static int ext4_mb_discard_preallocations(struct super_block *sb, int needed) +{ + ext4_group_t i, ngroups = ext4_get_groups_count(sb); + int ret; + int freed = 0, busy = 0; + int retry = 0; + + trace_ext4_mb_discard_preallocations(sb, needed); + + if (needed == 0) + needed = EXT4_CLUSTERS_PER_GROUP(sb) + 1; + repeat: + for (i = 0; i < ngroups && needed > 0; i++) { + ret = ext4_mb_discard_group_preallocations(sb, i, &busy); + freed += ret; + needed -= ret; + cond_resched(); + } + + if (needed > 0 && busy && ++retry < 3) { + busy = 0; + goto repeat; + } + + return freed; +} + +static bool ext4_mb_discard_preallocations_should_retry(struct super_block *sb, + struct ext4_allocation_context *ac, u64 *seq) +{ + int freed; + u64 seq_retry = 0; + bool ret = false; + + freed = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len); + if (freed) { + ret = true; + goto out_dbg; + } + seq_retry = ext4_get_discard_pa_seq_sum(); + if (!(ac->ac_flags & EXT4_MB_STRICT_CHECK) || seq_retry != *seq) { + ac->ac_flags |= EXT4_MB_STRICT_CHECK; + *seq = seq_retry; + ret = true; + } + +out_dbg: + mb_debug(sb, "freed %d, retry ? %s\n", freed, str_yes_no(ret)); + return ret; +} + +/* + * Simple allocator for Ext4 fast commit replay path. It searches for blocks + * linearly starting at the goal block and also excludes the blocks which + * are going to be in use after fast commit replay. + */ +static ext4_fsblk_t +ext4_mb_new_blocks_simple(struct ext4_allocation_request *ar, int *errp) +{ + struct buffer_head *bitmap_bh; + struct super_block *sb = ar->inode->i_sb; + struct ext4_sb_info *sbi = EXT4_SB(sb); + ext4_group_t group, nr; + ext4_grpblk_t blkoff; + ext4_grpblk_t max = EXT4_CLUSTERS_PER_GROUP(sb); + ext4_grpblk_t i = 0; + ext4_fsblk_t goal, block; + struct ext4_super_block *es = sbi->s_es; + + goal = ar->goal; + if (goal < le32_to_cpu(es->s_first_data_block) || + goal >= ext4_blocks_count(es)) + goal = le32_to_cpu(es->s_first_data_block); + + ar->len = 0; + ext4_get_group_no_and_offset(sb, goal, &group, &blkoff); + for (nr = ext4_get_groups_count(sb); nr > 0; nr--) { + bitmap_bh = ext4_read_block_bitmap(sb, group); + if (IS_ERR(bitmap_bh)) { + *errp = PTR_ERR(bitmap_bh); + pr_warn("Failed to read block bitmap\n"); + return 0; + } + + while (1) { + i = mb_find_next_zero_bit(bitmap_bh->b_data, max, + blkoff); + if (i >= max) + break; + if (ext4_fc_replay_check_excluded(sb, + ext4_group_first_block_no(sb, group) + + EXT4_C2B(sbi, i))) { + blkoff = i + 1; + } else + break; + } + brelse(bitmap_bh); + if (i < max) + break; + + if (++group >= ext4_get_groups_count(sb)) + group = 0; + + blkoff = 0; + } + + if (i >= max) { + *errp = -ENOSPC; + return 0; + } + + block = ext4_group_first_block_no(sb, group) + EXT4_C2B(sbi, i); + ext4_mb_mark_bb(sb, block, 1, true); + ar->len = 1; + + *errp = 0; + return block; +} + +/* + * Main entry point into mballoc to allocate blocks + * it tries to use preallocation first, then falls back + * to usual allocation + */ +ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle, + struct ext4_allocation_request *ar, int *errp) +{ + struct ext4_allocation_context *ac = NULL; + struct ext4_sb_info *sbi; + struct super_block *sb; + ext4_fsblk_t block = 0; + unsigned int inquota = 0; + unsigned int reserv_clstrs = 0; + int retries = 0; + u64 seq; + + might_sleep(); + sb = ar->inode->i_sb; + sbi = EXT4_SB(sb); + + trace_ext4_request_blocks(ar); + if (sbi->s_mount_state & EXT4_FC_REPLAY) + return ext4_mb_new_blocks_simple(ar, errp); + + /* Allow to use superuser reservation for quota file */ + if (ext4_is_quota_file(ar->inode)) + ar->flags |= EXT4_MB_USE_ROOT_BLOCKS; + + if ((ar->flags & EXT4_MB_DELALLOC_RESERVED) == 0) { + /* Without delayed allocation we need to verify + * there is enough free blocks to do block allocation + * and verify allocation doesn't exceed the quota limits. + */ + while (ar->len && + ext4_claim_free_clusters(sbi, ar->len, ar->flags)) { + + /* let others to free the space */ + cond_resched(); + ar->len = ar->len >> 1; + } + if (!ar->len) { + ext4_mb_show_pa(sb); + *errp = -ENOSPC; + return 0; + } + reserv_clstrs = ar->len; + if (ar->flags & EXT4_MB_USE_ROOT_BLOCKS) { + dquot_alloc_block_nofail(ar->inode, + EXT4_C2B(sbi, ar->len)); + } else { + while (ar->len && + dquot_alloc_block(ar->inode, + EXT4_C2B(sbi, ar->len))) { + + ar->flags |= EXT4_MB_HINT_NOPREALLOC; + ar->len--; + } + } + inquota = ar->len; + if (ar->len == 0) { + *errp = -EDQUOT; + goto out; + } + } + + ac = kmem_cache_zalloc(ext4_ac_cachep, GFP_NOFS); + if (!ac) { + ar->len = 0; + *errp = -ENOMEM; + goto out; + } + + ext4_mb_initialize_context(ac, ar); + + ac->ac_op = EXT4_MB_HISTORY_PREALLOC; + seq = this_cpu_read(discard_pa_seq); + if (!ext4_mb_use_preallocated(ac)) { + ac->ac_op = EXT4_MB_HISTORY_ALLOC; + ext4_mb_normalize_request(ac, ar); + + *errp = ext4_mb_pa_alloc(ac); + if (*errp) + goto errout; +repeat: + /* allocate space in core */ + *errp = ext4_mb_regular_allocator(ac); + /* + * pa allocated above is added to grp->bb_prealloc_list only + * when we were able to allocate some block i.e. when + * ac->ac_status == AC_STATUS_FOUND. + * And error from above mean ac->ac_status != AC_STATUS_FOUND + * So we have to free this pa here itself. + */ + if (*errp) { + ext4_mb_pa_put_free(ac); + ext4_discard_allocated_blocks(ac); + goto errout; + } + if (ac->ac_status == AC_STATUS_FOUND && + ac->ac_o_ex.fe_len >= ac->ac_f_ex.fe_len) + ext4_mb_pa_put_free(ac); + } + if (likely(ac->ac_status == AC_STATUS_FOUND)) { + *errp = ext4_mb_mark_diskspace_used(ac, handle); + if (*errp) { + ext4_discard_allocated_blocks(ac); + goto errout; + } else { + block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); + ar->len = ac->ac_b_ex.fe_len; + } + } else { + if (++retries < 3 && + ext4_mb_discard_preallocations_should_retry(sb, ac, &seq)) + goto repeat; + /* + * If block allocation fails then the pa allocated above + * needs to be freed here itself. + */ + ext4_mb_pa_put_free(ac); + *errp = -ENOSPC; + } + + if (*errp) { +errout: + ac->ac_b_ex.fe_len = 0; + ar->len = 0; + ext4_mb_show_ac(ac); + } + ext4_mb_release_context(ac); + kmem_cache_free(ext4_ac_cachep, ac); +out: + if (inquota && ar->len < inquota) + dquot_free_block(ar->inode, EXT4_C2B(sbi, inquota - ar->len)); + /* release any reserved blocks */ + if (reserv_clstrs) + percpu_counter_sub(&sbi->s_dirtyclusters_counter, reserv_clstrs); + + trace_ext4_allocate_blocks(ar, (unsigned long long)block); + + return block; +} + +/* + * We can merge two free data extents only if the physical blocks + * are contiguous, AND the extents were freed by the same transaction, + * AND the blocks are associated with the same group. + */ +static inline bool +ext4_freed_extents_can_be_merged(struct ext4_free_data *entry1, + struct ext4_free_data *entry2) +{ + if (entry1->efd_tid != entry2->efd_tid) + return false; + if (entry1->efd_start_cluster + entry1->efd_count != + entry2->efd_start_cluster) + return false; + if (WARN_ON_ONCE(entry1->efd_group != entry2->efd_group)) + return false; + return true; +} + +static inline void +ext4_merge_freed_extents(struct ext4_sb_info *sbi, struct rb_root *root, + struct ext4_free_data *entry1, + struct ext4_free_data *entry2) +{ + entry1->efd_count += entry2->efd_count; + spin_lock(&sbi->s_md_lock); + list_del(&entry2->efd_list); + spin_unlock(&sbi->s_md_lock); + rb_erase(&entry2->efd_node, root); + kmem_cache_free(ext4_free_data_cachep, entry2); +} + +static inline void +ext4_try_merge_freed_extent_prev(struct ext4_sb_info *sbi, struct rb_root *root, + struct ext4_free_data *entry) +{ + struct ext4_free_data *prev; + struct rb_node *node; + + node = rb_prev(&entry->efd_node); + if (!node) + return; + + prev = rb_entry(node, struct ext4_free_data, efd_node); + if (ext4_freed_extents_can_be_merged(prev, entry)) + ext4_merge_freed_extents(sbi, root, prev, entry); +} + +static inline void +ext4_try_merge_freed_extent_next(struct ext4_sb_info *sbi, struct rb_root *root, + struct ext4_free_data *entry) +{ + struct ext4_free_data *next; + struct rb_node *node; + + node = rb_next(&entry->efd_node); + if (!node) + return; + + next = rb_entry(node, struct ext4_free_data, efd_node); + if (ext4_freed_extents_can_be_merged(entry, next)) + ext4_merge_freed_extents(sbi, root, entry, next); +} + +static noinline_for_stack void +ext4_mb_free_metadata(handle_t *handle, struct ext4_buddy *e4b, + struct ext4_free_data *new_entry) +{ + ext4_group_t group = e4b->bd_group; + ext4_grpblk_t cluster; + ext4_grpblk_t clusters = new_entry->efd_count; + struct ext4_free_data *entry = NULL; + struct ext4_group_info *db = e4b->bd_info; + struct super_block *sb = e4b->bd_sb; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct rb_root *root = &db->bb_free_root; + struct rb_node **n = &root->rb_node; + struct rb_node *parent = NULL, *new_node; + + BUG_ON(!ext4_handle_valid(handle)); + BUG_ON(e4b->bd_bitmap_folio == NULL); + BUG_ON(e4b->bd_buddy_folio == NULL); + + new_node = &new_entry->efd_node; + cluster = new_entry->efd_start_cluster; + + if (!*n) { + /* first free block exent. We need to + protect buddy cache from being freed, + * otherwise we'll refresh it from + * on-disk bitmap and lose not-yet-available + * blocks */ + folio_get(e4b->bd_buddy_folio); + folio_get(e4b->bd_bitmap_folio); + } + while (*n) { + parent = *n; + entry = rb_entry(parent, struct ext4_free_data, efd_node); + if (cluster < entry->efd_start_cluster) + n = &(*n)->rb_left; + else if (cluster >= (entry->efd_start_cluster + entry->efd_count)) + n = &(*n)->rb_right; + else { + ext4_grp_locked_error(sb, group, 0, + ext4_group_first_block_no(sb, group) + + EXT4_C2B(sbi, cluster), + "Block already on to-be-freed list"); + kmem_cache_free(ext4_free_data_cachep, new_entry); + return; + } + } + + atomic_add(clusters, &sbi->s_mb_free_pending); + if (!entry) + goto insert; + + /* Now try to see the extent can be merged to prev and next */ + if (ext4_freed_extents_can_be_merged(new_entry, entry)) { + entry->efd_start_cluster = cluster; + entry->efd_count += new_entry->efd_count; + kmem_cache_free(ext4_free_data_cachep, new_entry); + ext4_try_merge_freed_extent_prev(sbi, root, entry); + return; + } + if (ext4_freed_extents_can_be_merged(entry, new_entry)) { + entry->efd_count += new_entry->efd_count; + kmem_cache_free(ext4_free_data_cachep, new_entry); + ext4_try_merge_freed_extent_next(sbi, root, entry); + return; + } +insert: + rb_link_node(new_node, parent, n); + rb_insert_color(new_node, root); + + spin_lock(&sbi->s_md_lock); + list_add_tail(&new_entry->efd_list, &sbi->s_freed_data_list[new_entry->efd_tid & 1]); + spin_unlock(&sbi->s_md_lock); +} + +static void ext4_free_blocks_simple(struct inode *inode, ext4_fsblk_t block, + unsigned long count) +{ + struct super_block *sb = inode->i_sb; + ext4_group_t group; + ext4_grpblk_t blkoff; + + ext4_get_group_no_and_offset(sb, block, &group, &blkoff); + ext4_mb_mark_context(NULL, sb, false, group, blkoff, count, + EXT4_MB_BITMAP_MARKED_CHECK | + EXT4_MB_SYNC_UPDATE, + NULL); +} + +/** + * ext4_mb_clear_bb() -- helper function for freeing blocks. + * Used by ext4_free_blocks() + * @handle: handle for this transaction + * @inode: inode + * @block: starting physical block to be freed + * @count: number of blocks to be freed + * @flags: flags used by ext4_free_blocks + */ +static void ext4_mb_clear_bb(handle_t *handle, struct inode *inode, + ext4_fsblk_t block, unsigned long count, + int flags) +{ + struct super_block *sb = inode->i_sb; + struct ext4_group_info *grp; + unsigned int overflow; + ext4_grpblk_t bit; + ext4_group_t block_group; + struct ext4_sb_info *sbi; + struct ext4_buddy e4b; + unsigned int count_clusters; + int err = 0; + int mark_flags = 0; + ext4_grpblk_t changed; + + sbi = EXT4_SB(sb); + + if (!(flags & EXT4_FREE_BLOCKS_VALIDATED) && + !ext4_inode_block_valid(inode, block, count)) { + ext4_error(sb, "Freeing blocks in system zone - " + "Block = %llu, count = %lu", block, count); + /* err = 0. ext4_std_error should be a no op */ + goto error_out; + } + flags |= EXT4_FREE_BLOCKS_VALIDATED; + +do_more: + overflow = 0; + ext4_get_group_no_and_offset(sb, block, &block_group, &bit); + + grp = ext4_get_group_info(sb, block_group); + if (unlikely(!grp || EXT4_MB_GRP_BBITMAP_CORRUPT(grp))) + return; + + /* + * Check to see if we are freeing blocks across a group + * boundary. + */ + if (EXT4_C2B(sbi, bit) + count > EXT4_BLOCKS_PER_GROUP(sb)) { + overflow = EXT4_C2B(sbi, bit) + count - + EXT4_BLOCKS_PER_GROUP(sb); + count -= overflow; + /* The range changed so it's no longer validated */ + flags &= ~EXT4_FREE_BLOCKS_VALIDATED; + } + count_clusters = EXT4_NUM_B2C(sbi, count); + trace_ext4_mballoc_free(sb, inode, block_group, bit, count_clusters); + + /* __GFP_NOFAIL: retry infinitely, ignore TIF_MEMDIE and memcg limit. */ + err = ext4_mb_load_buddy_gfp(sb, block_group, &e4b, + GFP_NOFS|__GFP_NOFAIL); + if (err) + goto error_out; + + if (!(flags & EXT4_FREE_BLOCKS_VALIDATED) && + !ext4_inode_block_valid(inode, block, count)) { + ext4_error(sb, "Freeing blocks in system zone - " + "Block = %llu, count = %lu", block, count); + /* err = 0. ext4_std_error should be a no op */ + goto error_clean; + } + +#ifdef AGGRESSIVE_CHECK + mark_flags |= EXT4_MB_BITMAP_MARKED_CHECK; +#endif + err = ext4_mb_mark_context(handle, sb, false, block_group, bit, + count_clusters, mark_flags, &changed); + + + if (err && changed == 0) + goto error_clean; + +#ifdef AGGRESSIVE_CHECK + BUG_ON(changed != count_clusters); +#endif + + /* + * We need to make sure we don't reuse the freed block until after the + * transaction is committed. We make an exception if the inode is to be + * written in writeback mode since writeback mode has weak data + * consistency guarantees. + */ + if (ext4_handle_valid(handle) && + ((flags & EXT4_FREE_BLOCKS_METADATA) || + !ext4_should_writeback_data(inode))) { + struct ext4_free_data *new_entry; + /* + * We use __GFP_NOFAIL because ext4_free_blocks() is not allowed + * to fail. + */ + new_entry = kmem_cache_alloc(ext4_free_data_cachep, + GFP_NOFS|__GFP_NOFAIL); + new_entry->efd_start_cluster = bit; + new_entry->efd_group = block_group; + new_entry->efd_count = count_clusters; + new_entry->efd_tid = handle->h_transaction->t_tid; + + ext4_lock_group(sb, block_group); + ext4_mb_free_metadata(handle, &e4b, new_entry); + } else { + if (test_opt(sb, DISCARD)) { + err = ext4_issue_discard(sb, block_group, bit, + count_clusters); + /* + * Ignore EOPNOTSUPP error. This is consistent with + * what happens when using journal. + */ + if (err == -EOPNOTSUPP) + err = 0; + if (err) + ext4_msg(sb, KERN_WARNING, "discard request in" + " group:%u block:%d count:%lu failed" + " with %d", block_group, bit, count, + err); + } + + EXT4_MB_GRP_CLEAR_TRIMMED(e4b.bd_info); + + ext4_lock_group(sb, block_group); + mb_free_blocks(inode, &e4b, bit, count_clusters); + } + + ext4_unlock_group(sb, block_group); + + /* + * on a bigalloc file system, defer the s_freeclusters_counter + * update to the caller (ext4_remove_space and friends) so they + * can determine if a cluster freed here should be rereserved + */ + if (!(flags & EXT4_FREE_BLOCKS_RERESERVE_CLUSTER)) { + if (!(flags & EXT4_FREE_BLOCKS_NO_QUOT_UPDATE)) + dquot_free_block(inode, EXT4_C2B(sbi, count_clusters)); + percpu_counter_add(&sbi->s_freeclusters_counter, + count_clusters); + } + + if (overflow && !err) { + block += count; + count = overflow; + ext4_mb_unload_buddy(&e4b); + /* The range changed so it's no longer validated */ + flags &= ~EXT4_FREE_BLOCKS_VALIDATED; + goto do_more; + } + +error_clean: + ext4_mb_unload_buddy(&e4b); +error_out: + ext4_std_error(sb, err); +} + +/** + * ext4_free_blocks() -- Free given blocks and update quota + * @handle: handle for this transaction + * @inode: inode + * @bh: optional buffer of the block to be freed + * @block: starting physical block to be freed + * @count: number of blocks to be freed + * @flags: flags used by ext4_free_blocks + */ +void ext4_free_blocks(handle_t *handle, struct inode *inode, + struct buffer_head *bh, ext4_fsblk_t block, + unsigned long count, int flags) +{ + struct super_block *sb = inode->i_sb; + unsigned int overflow; + struct ext4_sb_info *sbi; + + sbi = EXT4_SB(sb); + + if (bh) { + if (block) + BUG_ON(block != bh->b_blocknr); + else + block = bh->b_blocknr; + } + + if (sbi->s_mount_state & EXT4_FC_REPLAY) { + ext4_free_blocks_simple(inode, block, EXT4_NUM_B2C(sbi, count)); + return; + } + + might_sleep(); + + if (!(flags & EXT4_FREE_BLOCKS_VALIDATED) && + !ext4_inode_block_valid(inode, block, count)) { + ext4_error(sb, "Freeing blocks not in datazone - " + "block = %llu, count = %lu", block, count); + return; + } + flags |= EXT4_FREE_BLOCKS_VALIDATED; + + ext4_debug("freeing block %llu\n", block); + trace_ext4_free_blocks(inode, block, count, flags); + + if (bh && (flags & EXT4_FREE_BLOCKS_FORGET)) { + BUG_ON(count > 1); + + ext4_forget(handle, flags & EXT4_FREE_BLOCKS_METADATA, + inode, bh, block); + } + + /* + * If the extent to be freed does not begin on a cluster + * boundary, we need to deal with partial clusters at the + * beginning and end of the extent. Normally we will free + * blocks at the beginning or the end unless we are explicitly + * requested to avoid doing so. + */ + overflow = EXT4_PBLK_COFF(sbi, block); + if (overflow) { + if (flags & EXT4_FREE_BLOCKS_NOFREE_FIRST_CLUSTER) { + overflow = sbi->s_cluster_ratio - overflow; + block += overflow; + if (count > overflow) + count -= overflow; + else + return; + } else { + block -= overflow; + count += overflow; + } + /* The range changed so it's no longer validated */ + flags &= ~EXT4_FREE_BLOCKS_VALIDATED; + } + overflow = EXT4_LBLK_COFF(sbi, count); + if (overflow) { + if (flags & EXT4_FREE_BLOCKS_NOFREE_LAST_CLUSTER) { + if (count > overflow) + count -= overflow; + else + return; + } else + count += sbi->s_cluster_ratio - overflow; + /* The range changed so it's no longer validated */ + flags &= ~EXT4_FREE_BLOCKS_VALIDATED; + } + + if (!bh && (flags & EXT4_FREE_BLOCKS_FORGET)) { + int i; + int is_metadata = flags & EXT4_FREE_BLOCKS_METADATA; + + for (i = 0; i < count; i++) { + cond_resched(); + if (is_metadata) + bh = sb_find_get_block_nonatomic(inode->i_sb, + block + i); + ext4_forget(handle, is_metadata, inode, bh, block + i); + } + } + + ext4_mb_clear_bb(handle, inode, block, count, flags); +} + +/** + * ext4_group_add_blocks() -- Add given blocks to an existing group + * @handle: handle to this transaction + * @sb: super block + * @block: start physical block to add to the block group + * @count: number of blocks to free + * + * This marks the blocks as free in the bitmap and buddy. + */ +int ext4_group_add_blocks(handle_t *handle, struct super_block *sb, + ext4_fsblk_t block, unsigned long count) +{ + ext4_group_t block_group; + ext4_grpblk_t bit; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_buddy e4b; + int err = 0; + ext4_fsblk_t first_cluster = EXT4_B2C(sbi, block); + ext4_fsblk_t last_cluster = EXT4_B2C(sbi, block + count - 1); + unsigned long cluster_count = last_cluster - first_cluster + 1; + ext4_grpblk_t changed; + + ext4_debug("Adding block(s) %llu-%llu\n", block, block + count - 1); + + if (cluster_count == 0) + return 0; + + ext4_get_group_no_and_offset(sb, block, &block_group, &bit); + /* + * Check to see if we are freeing blocks across a group + * boundary. + */ + if (bit + cluster_count > EXT4_CLUSTERS_PER_GROUP(sb)) { + ext4_warning(sb, "too many blocks added to group %u", + block_group); + err = -EINVAL; + goto error_out; + } + + err = ext4_mb_load_buddy(sb, block_group, &e4b); + if (err) + goto error_out; + + if (!ext4_sb_block_valid(sb, NULL, block, count)) { + ext4_error(sb, "Adding blocks in system zones - " + "Block = %llu, count = %lu", + block, count); + err = -EINVAL; + goto error_clean; + } + + err = ext4_mb_mark_context(handle, sb, false, block_group, bit, + cluster_count, EXT4_MB_BITMAP_MARKED_CHECK, + &changed); + if (err && changed == 0) + goto error_clean; + + if (changed != cluster_count) + ext4_error(sb, "bit already cleared in group %u", block_group); + + ext4_lock_group(sb, block_group); + mb_free_blocks(NULL, &e4b, bit, cluster_count); + ext4_unlock_group(sb, block_group); + percpu_counter_add(&sbi->s_freeclusters_counter, + changed); + +error_clean: + ext4_mb_unload_buddy(&e4b); +error_out: + ext4_std_error(sb, err); + return err; +} + +/** + * ext4_trim_extent -- function to TRIM one single free extent in the group + * @sb: super block for the file system + * @start: starting block of the free extent in the alloc. group + * @count: number of blocks to TRIM + * @e4b: ext4 buddy for the group + * + * Trim "count" blocks starting at "start" in the "group". To assure that no + * one will allocate those blocks, mark it as used in buddy bitmap. This must + * be called with under the group lock. + */ +static int ext4_trim_extent(struct super_block *sb, + int start, int count, struct ext4_buddy *e4b) +__releases(bitlock) +__acquires(bitlock) +{ + struct ext4_free_extent ex; + ext4_group_t group = e4b->bd_group; + int ret = 0; + + trace_ext4_trim_extent(sb, group, start, count); + + assert_spin_locked(ext4_group_lock_ptr(sb, group)); + + ex.fe_start = start; + ex.fe_group = group; + ex.fe_len = count; + + /* + * Mark blocks used, so no one can reuse them while + * being trimmed. + */ + mb_mark_used(e4b, &ex); + ext4_unlock_group(sb, group); + ret = ext4_issue_discard(sb, group, start, count); + ext4_lock_group(sb, group); + mb_free_blocks(NULL, e4b, start, ex.fe_len); + return ret; +} + +static ext4_grpblk_t ext4_last_grp_cluster(struct super_block *sb, + ext4_group_t grp) +{ + unsigned long nr_clusters_in_group; + + if (grp < (ext4_get_groups_count(sb) - 1)) + nr_clusters_in_group = EXT4_CLUSTERS_PER_GROUP(sb); + else + nr_clusters_in_group = (ext4_blocks_count(EXT4_SB(sb)->s_es) - + ext4_group_first_block_no(sb, grp)) + >> EXT4_CLUSTER_BITS(sb); + + return nr_clusters_in_group - 1; +} + +static bool ext4_trim_interrupted(void) +{ + return fatal_signal_pending(current) || freezing(current); +} + +static int ext4_try_to_trim_range(struct super_block *sb, + struct ext4_buddy *e4b, ext4_grpblk_t start, + ext4_grpblk_t max, ext4_grpblk_t minblocks) +__acquires(ext4_group_lock_ptr(sb, e4b->bd_group)) +__releases(ext4_group_lock_ptr(sb, e4b->bd_group)) +{ + ext4_grpblk_t next, count, free_count, last, origin_start; + bool set_trimmed = false; + void *bitmap; + + if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info))) + return 0; + + last = ext4_last_grp_cluster(sb, e4b->bd_group); + bitmap = e4b->bd_bitmap; + if (start == 0 && max >= last) + set_trimmed = true; + origin_start = start; + start = max(e4b->bd_info->bb_first_free, start); + count = 0; + free_count = 0; + + while (start <= max) { + start = mb_find_next_zero_bit(bitmap, max + 1, start); + if (start > max) + break; + + next = mb_find_next_bit(bitmap, last + 1, start); + if (origin_start == 0 && next >= last) + set_trimmed = true; + + if ((next - start) >= minblocks) { + int ret = ext4_trim_extent(sb, start, next - start, e4b); + + if (ret && ret != -EOPNOTSUPP) + return count; + count += next - start; + } + free_count += next - start; + start = next + 1; + + if (ext4_trim_interrupted()) + return count; + + if (need_resched()) { + ext4_unlock_group(sb, e4b->bd_group); + cond_resched(); + ext4_lock_group(sb, e4b->bd_group); + } + + if ((e4b->bd_info->bb_free - free_count) < minblocks) + break; + } + + if (set_trimmed) + EXT4_MB_GRP_SET_TRIMMED(e4b->bd_info); + + return count; +} + +/** + * ext4_trim_all_free -- function to trim all free space in alloc. group + * @sb: super block for file system + * @group: group to be trimmed + * @start: first group block to examine + * @max: last group block to examine + * @minblocks: minimum extent block count + * + * ext4_trim_all_free walks through group's block bitmap searching for free + * extents. When the free extent is found, mark it as used in group buddy + * bitmap. Then issue a TRIM command on this extent and free the extent in + * the group buddy bitmap. + */ +static ext4_grpblk_t +ext4_trim_all_free(struct super_block *sb, ext4_group_t group, + ext4_grpblk_t start, ext4_grpblk_t max, + ext4_grpblk_t minblocks) +{ + struct ext4_buddy e4b; + int ret; + + trace_ext4_trim_all_free(sb, group, start, max); + + ret = ext4_mb_load_buddy(sb, group, &e4b); + if (ret) { + ext4_warning(sb, "Error %d loading buddy information for %u", + ret, group); + return ret; + } + + ext4_lock_group(sb, group); + + if (!EXT4_MB_GRP_WAS_TRIMMED(e4b.bd_info) || + minblocks < EXT4_SB(sb)->s_last_trim_minblks) + ret = ext4_try_to_trim_range(sb, &e4b, start, max, minblocks); + else + ret = 0; + + ext4_unlock_group(sb, group); + ext4_mb_unload_buddy(&e4b); + + ext4_debug("trimmed %d blocks in the group %d\n", + ret, group); + + return ret; +} + +/** + * ext4_trim_fs() -- trim ioctl handle function + * @sb: superblock for filesystem + * @range: fstrim_range structure + * + * start: First Byte to trim + * len: number of Bytes to trim from start + * minlen: minimum extent length in Bytes + * ext4_trim_fs goes through all allocation groups containing Bytes from + * start to start+len. For each such a group ext4_trim_all_free function + * is invoked to trim all free space. + */ +int ext4_trim_fs(struct super_block *sb, struct fstrim_range *range) +{ + unsigned int discard_granularity = bdev_discard_granularity(sb->s_bdev); + struct ext4_group_info *grp; + ext4_group_t group, first_group, last_group; + ext4_grpblk_t cnt = 0, first_cluster, last_cluster; + uint64_t start, end, minlen, trimmed = 0; + ext4_fsblk_t first_data_blk = + le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block); + ext4_fsblk_t max_blks = ext4_blocks_count(EXT4_SB(sb)->s_es); + int ret = 0; + + start = range->start >> sb->s_blocksize_bits; + end = start + (range->len >> sb->s_blocksize_bits) - 1; + minlen = EXT4_NUM_B2C(EXT4_SB(sb), + range->minlen >> sb->s_blocksize_bits); + + if (minlen > EXT4_CLUSTERS_PER_GROUP(sb) || + start >= max_blks || + range->len < sb->s_blocksize) + return -EINVAL; + /* No point to try to trim less than discard granularity */ + if (range->minlen < discard_granularity) { + minlen = EXT4_NUM_B2C(EXT4_SB(sb), + discard_granularity >> sb->s_blocksize_bits); + if (minlen > EXT4_CLUSTERS_PER_GROUP(sb)) + goto out; + } + if (end >= max_blks - 1) + end = max_blks - 1; + if (end <= first_data_blk) + goto out; + if (start < first_data_blk) + start = first_data_blk; + + /* Determine first and last group to examine based on start and end */ + ext4_get_group_no_and_offset(sb, (ext4_fsblk_t) start, + &first_group, &first_cluster); + ext4_get_group_no_and_offset(sb, (ext4_fsblk_t) end, + &last_group, &last_cluster); + + /* end now represents the last cluster to discard in this group */ + end = EXT4_CLUSTERS_PER_GROUP(sb) - 1; + + for (group = first_group; group <= last_group; group++) { + if (ext4_trim_interrupted()) + break; + grp = ext4_get_group_info(sb, group); + if (!grp) + continue; + /* We only do this if the grp has never been initialized */ + if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) { + ret = ext4_mb_init_group(sb, group, GFP_NOFS); + if (ret) + break; + } + + /* + * For all the groups except the last one, last cluster will + * always be EXT4_CLUSTERS_PER_GROUP(sb)-1, so we only need to + * change it for the last group, note that last_cluster is + * already computed earlier by ext4_get_group_no_and_offset() + */ + if (group == last_group) + end = last_cluster; + if (grp->bb_free >= minlen) { + cnt = ext4_trim_all_free(sb, group, first_cluster, + end, minlen); + if (cnt < 0) { + ret = cnt; + break; + } + trimmed += cnt; + } + + /* + * For every group except the first one, we are sure + * that the first cluster to discard will be cluster #0. + */ + first_cluster = 0; + } + + if (!ret) + EXT4_SB(sb)->s_last_trim_minblks = minlen; + +out: + range->len = EXT4_C2B(EXT4_SB(sb), trimmed) << sb->s_blocksize_bits; + return ret; +} + +/* Iterate all the free extents in the group. */ +int +ext4_mballoc_query_range( + struct super_block *sb, + ext4_group_t group, + ext4_grpblk_t first, + ext4_grpblk_t end, + ext4_mballoc_query_range_fn meta_formatter, + ext4_mballoc_query_range_fn formatter, + void *priv) +{ + void *bitmap; + ext4_grpblk_t start, next; + struct ext4_buddy e4b; + int error; + + error = ext4_mb_load_buddy(sb, group, &e4b); + if (error) + return error; + bitmap = e4b.bd_bitmap; + + ext4_lock_group(sb, group); + + start = max(e4b.bd_info->bb_first_free, first); + if (end >= EXT4_CLUSTERS_PER_GROUP(sb)) + end = EXT4_CLUSTERS_PER_GROUP(sb) - 1; + if (meta_formatter && start != first) { + if (start > end) + start = end; + ext4_unlock_group(sb, group); + error = meta_formatter(sb, group, first, start - first, + priv); + if (error) + goto out_unload; + ext4_lock_group(sb, group); + } + while (start <= end) { + start = mb_find_next_zero_bit(bitmap, end + 1, start); + if (start > end) + break; + next = mb_find_next_bit(bitmap, end + 1, start); + + ext4_unlock_group(sb, group); + error = formatter(sb, group, start, next - start, priv); + if (error) + goto out_unload; + ext4_lock_group(sb, group); + + start = next + 1; + } + + ext4_unlock_group(sb, group); +out_unload: + ext4_mb_unload_buddy(&e4b); + + return error; +} + +#if IS_ENABLED(CONFIG_EXT4_KUNIT_TESTS) +void mb_clear_bits_test(void *bm, int cur, int len) +{ + mb_clear_bits(bm, cur, len); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_clear_bits_test); + +ext4_fsblk_t +ext4_mb_new_blocks_simple_test(struct ext4_allocation_request *ar, + int *errp) +{ + return ext4_mb_new_blocks_simple(ar, errp); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_new_blocks_simple_test); + +int mb_find_next_zero_bit_test(void *addr, int max, int start) +{ + return mb_find_next_zero_bit(addr, max, start); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_find_next_zero_bit_test); + +int mb_find_next_bit_test(void *addr, int max, int start) +{ + return mb_find_next_bit(addr, max, start); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_find_next_bit_test); + +void mb_clear_bit_test(int bit, void *addr) +{ + mb_clear_bit(bit, addr); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_clear_bit_test); + +int mb_test_bit_test(int bit, void *addr) +{ + return mb_test_bit(bit, addr); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_test_bit_test); + +int ext4_mb_mark_diskspace_used_test(struct ext4_allocation_context *ac, + handle_t *handle) +{ + return ext4_mb_mark_diskspace_used(ac, handle); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_mark_diskspace_used_test); + +int mb_mark_used_test(struct ext4_buddy *e4b, struct ext4_free_extent *ex) +{ + return mb_mark_used(e4b, ex); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_mark_used_test); + +void ext4_mb_generate_buddy_test(struct super_block *sb, void *buddy, + void *bitmap, ext4_group_t group, + struct ext4_group_info *grp) +{ + ext4_mb_generate_buddy(sb, buddy, bitmap, group, grp); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_generate_buddy_test); + +int ext4_mb_load_buddy_test(struct super_block *sb, ext4_group_t group, + struct ext4_buddy *e4b) +{ + return ext4_mb_load_buddy(sb, group, e4b); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_load_buddy_test); + +void ext4_mb_unload_buddy_test(struct ext4_buddy *e4b) +{ + ext4_mb_unload_buddy(e4b); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_unload_buddy_test); + +void mb_free_blocks_test(struct inode *inode, struct ext4_buddy *e4b, + int first, int count) +{ + mb_free_blocks(inode, e4b, first, count); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_free_blocks_test); + +void ext4_free_blocks_simple_test(struct inode *inode, ext4_fsblk_t block, + unsigned long count) +{ + return ext4_free_blocks_simple(inode, block, count); +} +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_free_blocks_simple_test); + +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_wait_block_bitmap); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_init); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_get_group_desc); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_count_free_clusters); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_get_group_info); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_free_group_clusters_set); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_release); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_read_block_bitmap_nowait); +EXPORT_SYMBOL_FOR_EXT4_TEST(mb_set_bits); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_fc_init_inode); +EXPORT_SYMBOL_FOR_EXT4_TEST(ext4_mb_mark_context); +#endif diff --git a/fs/ext4/mballoc.h b/fs/ext4/mballoc.h new file mode 100644 index 000000000..39333ce72 --- /dev/null +++ b/fs/ext4/mballoc.h @@ -0,0 +1,303 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/ext4/mballoc.h + * + * Written by: Alex Tomas + * + */ +#ifndef _EXT4_MBALLOC_H +#define _EXT4_MBALLOC_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ext4_jbd2.h" +#include "ext4.h" + +/* + * mb_debug() dynamic printk msgs could be used to debug mballoc code. + */ +#ifdef CONFIG_EXT4_DEBUG +#define mb_debug(sb, fmt, ...) \ + pr_debug("[%s/%d] EXT4-fs (%s): (%s, %d): %s: " fmt, \ + current->comm, task_pid_nr(current), sb->s_id, \ + __FILE__, __LINE__, __func__, ##__VA_ARGS__) +#else +#define mb_debug(sb, fmt, ...) no_printk(fmt, ##__VA_ARGS__) +#endif + +#define EXT4_MB_HISTORY_ALLOC 1 /* allocation */ +#define EXT4_MB_HISTORY_PREALLOC 2 /* preallocated blocks used */ + +/* + * How long mballoc can look for a best extent (in found extents) + */ +#define MB_DEFAULT_MAX_TO_SCAN 200 + +/* + * How long mballoc must look for a best extent + */ +#define MB_DEFAULT_MIN_TO_SCAN 10 + +/* + * with 's_mb_stats' allocator will collect stats that will be + * shown at umount. The collecting costs though! + */ +#define MB_DEFAULT_STATS 0 + +/* + * files smaller than MB_DEFAULT_STREAM_THRESHOLD are served + * by the stream allocator, which purpose is to pack requests + * as close each to other as possible to produce smooth I/O traffic + * We use locality group prealloc space for stream request. + * We can tune the same via /proc/fs/ext4//stream_req + */ +#define MB_DEFAULT_STREAM_THRESHOLD 16 /* 64K */ + +/* + * for which requests use 2^N search using buddies + */ +#define MB_DEFAULT_ORDER2_REQS 2 + +/* + * default group prealloc size 512 blocks + */ +#define MB_DEFAULT_GROUP_PREALLOC 512 + +/* + * Number of groups to search linearly before performing group scanning + * optimization. + */ +#define MB_DEFAULT_LINEAR_LIMIT 4 + +/* + * Minimum number of groups that should be present in the file system to perform + * group scanning optimizations. + */ +#define MB_DEFAULT_LINEAR_SCAN_THRESHOLD 16 + +/* + * The maximum order upto which CR_BEST_AVAIL_LEN can trim a particular + * allocation request. Example, if we have an order 7 request and max trim order + * of 3, we can trim this request upto order 4. + */ +#define MB_DEFAULT_BEST_AVAIL_TRIM_ORDER 3 + +/* + * Number of valid buddy orders + */ +#define MB_NUM_ORDERS(sb) ((sb)->s_blocksize_bits + 2) + +struct ext4_free_data { + /* this links the free block information from sb_info */ + struct list_head efd_list; + + /* this links the free block information from group_info */ + struct rb_node efd_node; + + /* group which free block extent belongs */ + ext4_group_t efd_group; + + /* free block extent */ + ext4_grpblk_t efd_start_cluster; + ext4_grpblk_t efd_count; + + /* transaction which freed this extent */ + tid_t efd_tid; +}; + +struct ext4_prealloc_space { + union { + struct rb_node inode_node; /* for inode PA rbtree */ + struct list_head lg_list; /* for lg PAs */ + } pa_node; + struct list_head pa_group_list; + union { + struct list_head pa_tmp_list; + struct rcu_head pa_rcu; + } u; + spinlock_t pa_lock; + atomic_t pa_count; + unsigned pa_deleted; + ext4_fsblk_t pa_pstart; /* phys. block */ + ext4_lblk_t pa_lstart; /* log. block */ + ext4_grpblk_t pa_len; /* len of preallocated chunk */ + ext4_grpblk_t pa_free; /* how many blocks are free */ + unsigned short pa_type; /* pa type. inode or group */ + union { + rwlock_t *inode_lock; /* locks the rbtree holding this PA */ + spinlock_t *lg_lock; /* locks the lg list holding this PA */ + } pa_node_lock; + struct inode *pa_inode; /* used to get the inode during group discard */ +}; + +enum { + MB_INODE_PA = 0, + MB_GROUP_PA = 1 +}; + +struct ext4_free_extent { + ext4_lblk_t fe_logical; + ext4_grpblk_t fe_start; /* In cluster units */ + ext4_group_t fe_group; + ext4_grpblk_t fe_len; /* In cluster units */ +}; + +/* + * Locality group: + * we try to group all related changes together + * so that writeback can flush/allocate them together as well + * Size of lg_prealloc_list hash is determined by MB_DEFAULT_GROUP_PREALLOC + * (512). We store prealloc space into the hash based on the pa_free blocks + * order value.ie, fls(pa_free)-1; + */ +#define PREALLOC_TB_SIZE 10 +struct ext4_locality_group { + /* for allocator */ + /* to serialize allocates */ + struct mutex lg_mutex; + /* list of preallocations */ + struct list_head lg_prealloc_list[PREALLOC_TB_SIZE]; + spinlock_t lg_prealloc_lock; +}; + +struct ext4_allocation_context { + struct inode *ac_inode; + struct super_block *ac_sb; + + /* original request */ + struct ext4_free_extent ac_o_ex; + + /* goal request (normalized ac_o_ex) */ + struct ext4_free_extent ac_g_ex; + + /* the best found extent */ + struct ext4_free_extent ac_b_ex; + + /* copy of the best found extent taken before preallocation efforts */ + struct ext4_free_extent ac_f_ex; + + /* + * goal len can change in CR_BEST_AVAIL_LEN, so save the original len. + * This is used while adjusting the PA window and for accounting. + */ + ext4_grpblk_t ac_orig_goal_len; + + ext4_group_t ac_prefetch_grp; + unsigned int ac_prefetch_ios; + unsigned int ac_prefetch_nr; + + int ac_first_err; + + __u32 ac_flags; /* allocation hints */ + __u16 ac_groups_scanned; + __u16 ac_found; + __u16 ac_cX_found[EXT4_MB_NUM_CRS]; + __u16 ac_tail; + __u16 ac_buddy; + __u8 ac_status; + __u8 ac_criteria; + __u8 ac_2order; /* if request is to allocate 2^N blocks and + * N > 0, the field stores N, otherwise 0 */ + __u8 ac_op; /* operation, for history only */ + + struct ext4_buddy *ac_e4b; + struct folio *ac_bitmap_folio; + struct folio *ac_buddy_folio; + struct ext4_prealloc_space *ac_pa; + struct ext4_locality_group *ac_lg; +}; + +#define AC_STATUS_CONTINUE 1 +#define AC_STATUS_FOUND 2 +#define AC_STATUS_BREAK 3 + +struct ext4_buddy { + struct folio *bd_buddy_folio; + void *bd_buddy; + struct folio *bd_bitmap_folio; + void *bd_bitmap; + struct ext4_group_info *bd_info; + struct super_block *bd_sb; + __u16 bd_blkbits; + ext4_group_t bd_group; +}; + +static inline ext4_fsblk_t ext4_grp_offs_to_block(struct super_block *sb, + struct ext4_free_extent *fex) +{ + return ext4_group_first_block_no(sb, fex->fe_group) + + (fex->fe_start << EXT4_SB(sb)->s_cluster_bits); +} + +static inline loff_t extent_logical_end(struct ext4_sb_info *sbi, + struct ext4_free_extent *fex) +{ + /* Use loff_t to avoid end exceeding ext4_lblk_t max. */ + return (loff_t)fex->fe_logical + EXT4_C2B(sbi, fex->fe_len); +} + +static inline loff_t pa_logical_end(struct ext4_sb_info *sbi, + struct ext4_prealloc_space *pa) +{ + /* Use loff_t to avoid end exceeding ext4_lblk_t max. */ + return (loff_t)pa->pa_lstart + EXT4_C2B(sbi, pa->pa_len); +} + +typedef int (*ext4_mballoc_query_range_fn)( + struct super_block *sb, + ext4_group_t agno, + ext4_grpblk_t start, + ext4_grpblk_t len, + void *priv); + +int +ext4_mballoc_query_range( + struct super_block *sb, + ext4_group_t agno, + ext4_grpblk_t start, + ext4_grpblk_t end, + ext4_mballoc_query_range_fn meta_formatter, + ext4_mballoc_query_range_fn formatter, + void *priv); + +extern int ext4_mb_mark_context(handle_t *handle, + struct super_block *sb, bool state, + ext4_group_t group, ext4_grpblk_t blkoff, + ext4_grpblk_t len, int flags, + ext4_grpblk_t *ret_changed); +#if IS_ENABLED(CONFIG_EXT4_KUNIT_TESTS) +extern void mb_clear_bits_test(void *bm, int cur, int len); +extern ext4_fsblk_t +ext4_mb_new_blocks_simple_test(struct ext4_allocation_request *ar, + int *errp); +extern int mb_find_next_zero_bit_test(void *addr, int max, int start); +extern int mb_find_next_bit_test(void *addr, int max, int start); +extern void mb_clear_bit_test(int bit, void *addr); +extern int mb_test_bit_test(int bit, void *addr); +extern int +ext4_mb_mark_diskspace_used_test(struct ext4_allocation_context *ac, + handle_t *handle); +extern int mb_mark_used_test(struct ext4_buddy *e4b, + struct ext4_free_extent *ex); +extern void ext4_mb_generate_buddy_test(struct super_block *sb, + void *buddy, void *bitmap, ext4_group_t group, + struct ext4_group_info *grp); +extern int ext4_mb_load_buddy_test(struct super_block *sb, + ext4_group_t group, struct ext4_buddy *e4b); +extern void ext4_mb_unload_buddy_test(struct ext4_buddy *e4b); +extern void mb_free_blocks_test(struct inode *inode, + struct ext4_buddy *e4b, int first, int count); +extern void ext4_free_blocks_simple_test(struct inode *inode, + ext4_fsblk_t block, unsigned long count); +#endif +#endif diff --git a/fs/ext4/migrate.c b/fs/ext4/migrate.c new file mode 100644 index 000000000..5d60ef10f --- /dev/null +++ b/fs/ext4/migrate.c @@ -0,0 +1,685 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * Copyright IBM Corporation, 2007 + * Author Aneesh Kumar K.V + * + */ + +#include +#include "ext4_jbd2.h" +#include "ext4_extents.h" + +/* + * The contiguous blocks details which can be + * represented by a single extent + */ +struct migrate_struct { + ext4_lblk_t first_block, last_block, curr_block; + ext4_fsblk_t first_pblock, last_pblock; +}; + +static int finish_range(handle_t *handle, struct inode *inode, + struct migrate_struct *lb) + +{ + int retval = 0, needed; + struct ext4_extent newext; + struct ext4_ext_path *path; + if (lb->first_pblock == 0) + return 0; + + /* Add the extent to temp inode*/ + newext.ee_block = cpu_to_le32(lb->first_block); + newext.ee_len = cpu_to_le16(lb->last_block - lb->first_block + 1); + ext4_ext_store_pblock(&newext, lb->first_pblock); + /* Locking only for convenience since we are operating on temp inode */ + down_write(&EXT4_I(inode)->i_data_sem); + path = ext4_find_extent(inode, lb->first_block, NULL, 0); + if (IS_ERR(path)) { + retval = PTR_ERR(path); + goto err_out; + } + + /* + * Calculate the credit needed to inserting this extent + * Since we are doing this in loop we may accumulate extra + * credit. But below we try to not accumulate too much + * of them by restarting the journal. + */ + needed = ext4_ext_calc_credits_for_single_extent(inode, + lb->last_block - lb->first_block + 1, path); + + retval = ext4_datasem_ensure_credits(handle, inode, needed, needed, 0); + if (retval < 0) + goto err_out; + path = ext4_ext_insert_extent(handle, inode, path, &newext, 0); + if (IS_ERR(path)) + retval = PTR_ERR(path); +err_out: + up_write((&EXT4_I(inode)->i_data_sem)); + ext4_free_ext_path(path); + lb->first_pblock = 0; + return retval; +} + +static int update_extent_range(handle_t *handle, struct inode *inode, + ext4_fsblk_t pblock, struct migrate_struct *lb) +{ + int retval; + /* + * See if we can add on to the existing range (if it exists) + */ + if (lb->first_pblock && + (lb->last_pblock+1 == pblock) && + (lb->last_block+1 == lb->curr_block)) { + lb->last_pblock = pblock; + lb->last_block = lb->curr_block; + lb->curr_block++; + return 0; + } + /* + * Start a new range. + */ + retval = finish_range(handle, inode, lb); + lb->first_pblock = lb->last_pblock = pblock; + lb->first_block = lb->last_block = lb->curr_block; + lb->curr_block++; + return retval; +} + +static int update_ind_extent_range(handle_t *handle, struct inode *inode, + ext4_fsblk_t pblock, + struct migrate_struct *lb) +{ + struct buffer_head *bh; + __le32 *i_data; + int i, retval = 0; + unsigned long max_entries = inode->i_sb->s_blocksize >> 2; + + bh = ext4_sb_bread(inode->i_sb, pblock, 0); + if (IS_ERR(bh)) + return PTR_ERR(bh); + + i_data = (__le32 *)bh->b_data; + for (i = 0; i < max_entries; i++) { + if (i_data[i]) { + retval = update_extent_range(handle, inode, + le32_to_cpu(i_data[i]), lb); + if (retval) + break; + } else { + lb->curr_block++; + } + } + put_bh(bh); + return retval; + +} + +static int update_dind_extent_range(handle_t *handle, struct inode *inode, + ext4_fsblk_t pblock, + struct migrate_struct *lb) +{ + struct buffer_head *bh; + __le32 *i_data; + int i, retval = 0; + unsigned long max_entries = inode->i_sb->s_blocksize >> 2; + + bh = ext4_sb_bread(inode->i_sb, pblock, 0); + if (IS_ERR(bh)) + return PTR_ERR(bh); + + i_data = (__le32 *)bh->b_data; + for (i = 0; i < max_entries; i++) { + if (i_data[i]) { + retval = update_ind_extent_range(handle, inode, + le32_to_cpu(i_data[i]), lb); + if (retval) + break; + } else { + /* Only update the file block number */ + lb->curr_block += max_entries; + } + } + put_bh(bh); + return retval; + +} + +static int update_tind_extent_range(handle_t *handle, struct inode *inode, + ext4_fsblk_t pblock, + struct migrate_struct *lb) +{ + struct buffer_head *bh; + __le32 *i_data; + int i, retval = 0; + unsigned long max_entries = inode->i_sb->s_blocksize >> 2; + + bh = ext4_sb_bread(inode->i_sb, pblock, 0); + if (IS_ERR(bh)) + return PTR_ERR(bh); + + i_data = (__le32 *)bh->b_data; + for (i = 0; i < max_entries; i++) { + if (i_data[i]) { + retval = update_dind_extent_range(handle, inode, + le32_to_cpu(i_data[i]), lb); + if (retval) + break; + } else { + /* Only update the file block number */ + lb->curr_block += max_entries * max_entries; + } + } + put_bh(bh); + return retval; + +} + +static int free_dind_blocks(handle_t *handle, + struct inode *inode, __le32 i_data) +{ + int i; + __le32 *tmp_idata; + struct buffer_head *bh; + struct super_block *sb = inode->i_sb; + unsigned long max_entries = inode->i_sb->s_blocksize >> 2; + int err; + + bh = ext4_sb_bread(sb, le32_to_cpu(i_data), 0); + if (IS_ERR(bh)) + return PTR_ERR(bh); + + tmp_idata = (__le32 *)bh->b_data; + for (i = 0; i < max_entries; i++) { + if (tmp_idata[i]) { + err = ext4_journal_ensure_credits(handle, + EXT4_RESERVE_TRANS_BLOCKS, + ext4_free_metadata_revoke_credits(sb, 1)); + if (err < 0) { + put_bh(bh); + return err; + } + ext4_free_blocks(handle, inode, NULL, + le32_to_cpu(tmp_idata[i]), 1, + EXT4_FREE_BLOCKS_METADATA | + EXT4_FREE_BLOCKS_FORGET); + } + } + put_bh(bh); + err = ext4_journal_ensure_credits(handle, EXT4_RESERVE_TRANS_BLOCKS, + ext4_free_metadata_revoke_credits(sb, 1)); + if (err < 0) + return err; + ext4_free_blocks(handle, inode, NULL, le32_to_cpu(i_data), 1, + EXT4_FREE_BLOCKS_METADATA | + EXT4_FREE_BLOCKS_FORGET); + return 0; +} + +static int free_tind_blocks(handle_t *handle, + struct inode *inode, __le32 i_data) +{ + int i, retval = 0; + __le32 *tmp_idata; + struct buffer_head *bh; + unsigned long max_entries = inode->i_sb->s_blocksize >> 2; + + bh = ext4_sb_bread(inode->i_sb, le32_to_cpu(i_data), 0); + if (IS_ERR(bh)) + return PTR_ERR(bh); + + tmp_idata = (__le32 *)bh->b_data; + for (i = 0; i < max_entries; i++) { + if (tmp_idata[i]) { + retval = free_dind_blocks(handle, + inode, tmp_idata[i]); + if (retval) { + put_bh(bh); + return retval; + } + } + } + put_bh(bh); + retval = ext4_journal_ensure_credits(handle, EXT4_RESERVE_TRANS_BLOCKS, + ext4_free_metadata_revoke_credits(inode->i_sb, 1)); + if (retval < 0) + return retval; + ext4_free_blocks(handle, inode, NULL, le32_to_cpu(i_data), 1, + EXT4_FREE_BLOCKS_METADATA | + EXT4_FREE_BLOCKS_FORGET); + return 0; +} + +static int free_ind_block(handle_t *handle, struct inode *inode, __le32 *i_data) +{ + int retval; + + /* ei->i_data[EXT4_IND_BLOCK] */ + if (i_data[0]) { + retval = ext4_journal_ensure_credits(handle, + EXT4_RESERVE_TRANS_BLOCKS, + ext4_free_metadata_revoke_credits(inode->i_sb, 1)); + if (retval < 0) + return retval; + ext4_free_blocks(handle, inode, NULL, + le32_to_cpu(i_data[0]), 1, + EXT4_FREE_BLOCKS_METADATA | + EXT4_FREE_BLOCKS_FORGET); + } + + /* ei->i_data[EXT4_DIND_BLOCK] */ + if (i_data[1]) { + retval = free_dind_blocks(handle, inode, i_data[1]); + if (retval) + return retval; + } + + /* ei->i_data[EXT4_TIND_BLOCK] */ + if (i_data[2]) { + retval = free_tind_blocks(handle, inode, i_data[2]); + if (retval) + return retval; + } + return 0; +} + +static int ext4_ext_swap_inode_data(handle_t *handle, struct inode *inode, + struct inode *tmp_inode) +{ + int retval, retval2 = 0; + __le32 i_data[3]; + struct ext4_inode_info *ei = EXT4_I(inode); + struct ext4_inode_info *tmp_ei = EXT4_I(tmp_inode); + + /* + * One credit accounted for writing the + * i_data field of the original inode + */ + retval = ext4_journal_ensure_credits(handle, 1, 0); + if (retval < 0) + goto err_out; + + i_data[0] = ei->i_data[EXT4_IND_BLOCK]; + i_data[1] = ei->i_data[EXT4_DIND_BLOCK]; + i_data[2] = ei->i_data[EXT4_TIND_BLOCK]; + + down_write(&EXT4_I(inode)->i_data_sem); + /* + * if EXT4_STATE_EXT_MIGRATE is cleared a block allocation + * happened after we started the migrate. We need to + * fail the migrate + */ + if (!ext4_test_inode_state(inode, EXT4_STATE_EXT_MIGRATE)) { + retval = -EAGAIN; + up_write(&EXT4_I(inode)->i_data_sem); + goto err_out; + } else + ext4_clear_inode_state(inode, EXT4_STATE_EXT_MIGRATE); + /* + * We have the extent map build with the tmp inode. + * Now copy the i_data across + */ + ext4_set_inode_flag(inode, EXT4_INODE_EXTENTS); + memcpy(ei->i_data, tmp_ei->i_data, sizeof(ei->i_data)); + + /* + * Update i_blocks with the new blocks that got + * allocated while adding extents for extent index + * blocks. + * + * While converting to extents we need not + * update the original inode i_blocks for extent blocks + * via quota APIs. The quota update happened via tmp_inode already. + */ + spin_lock(&inode->i_lock); + inode->i_blocks += tmp_inode->i_blocks; + spin_unlock(&inode->i_lock); + up_write(&EXT4_I(inode)->i_data_sem); + + /* + * We mark the inode dirty after, because we decrement the + * i_blocks when freeing the indirect meta-data blocks + */ + retval = free_ind_block(handle, inode, i_data); + retval2 = ext4_mark_inode_dirty(handle, inode); + if (unlikely(retval2 && !retval)) + retval = retval2; + +err_out: + return retval; +} + +static int free_ext_idx(handle_t *handle, struct inode *inode, + struct ext4_extent_idx *ix) +{ + int i, retval = 0; + ext4_fsblk_t block; + struct buffer_head *bh; + struct ext4_extent_header *eh; + + block = ext4_idx_pblock(ix); + bh = ext4_sb_bread(inode->i_sb, block, 0); + if (IS_ERR(bh)) + return PTR_ERR(bh); + + eh = (struct ext4_extent_header *)bh->b_data; + if (eh->eh_depth != 0) { + ix = EXT_FIRST_INDEX(eh); + for (i = 0; i < le16_to_cpu(eh->eh_entries); i++, ix++) { + retval = free_ext_idx(handle, inode, ix); + if (retval) { + put_bh(bh); + return retval; + } + } + } + put_bh(bh); + retval = ext4_journal_ensure_credits(handle, EXT4_RESERVE_TRANS_BLOCKS, + ext4_free_metadata_revoke_credits(inode->i_sb, 1)); + if (retval < 0) + return retval; + ext4_free_blocks(handle, inode, NULL, block, 1, + EXT4_FREE_BLOCKS_METADATA | EXT4_FREE_BLOCKS_FORGET); + return 0; +} + +/* + * Free the extent meta data blocks only + */ +static int free_ext_block(handle_t *handle, struct inode *inode) +{ + int i, retval = 0; + struct ext4_inode_info *ei = EXT4_I(inode); + struct ext4_extent_header *eh = (struct ext4_extent_header *)ei->i_data; + struct ext4_extent_idx *ix; + if (eh->eh_depth == 0) + /* + * No extra blocks allocated for extent meta data + */ + return 0; + ix = EXT_FIRST_INDEX(eh); + for (i = 0; i < le16_to_cpu(eh->eh_entries); i++, ix++) { + retval = free_ext_idx(handle, inode, ix); + if (retval) + return retval; + } + return retval; +} + +int ext4_ext_migrate(struct inode *inode) +{ + handle_t *handle; + int retval = 0, i; + __le32 *i_data; + struct ext4_inode_info *ei; + struct inode *tmp_inode = NULL; + struct migrate_struct lb; + unsigned long max_entries; + __u32 goal, tmp_csum_seed; + uid_t owner[2]; + int alloc_ctx; + + /* + * If the filesystem does not support extents, or the inode + * already is extent-based, error out. + */ + if (!ext4_has_feature_extents(inode->i_sb) || + ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS) || + ext4_has_inline_data(inode)) + return -EINVAL; + + if (S_ISLNK(inode->i_mode) && inode->i_blocks == 0) + /* + * don't migrate fast symlink + */ + return retval; + + alloc_ctx = ext4_writepages_down_write(inode->i_sb); + + /* + * Worst case we can touch the allocation bitmaps and a block + * group descriptor block. We do need to worry about + * credits for modifying the quota inode. + */ + handle = ext4_journal_start(inode, EXT4_HT_MIGRATE, + 3 + EXT4_MAXQUOTAS_TRANS_BLOCKS(inode->i_sb)); + + if (IS_ERR(handle)) { + retval = PTR_ERR(handle); + goto out_unlock; + } + /* + * This operation rewrites the inode's block mapping layout + * (indirect to extents) and is not tracked in the fast commit + * log, so disable fast commits for this transaction. + */ + ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_MIGRATE, handle); + goal = ((((u32)inode->i_ino - 1) / EXT4_INODES_PER_GROUP(inode->i_sb)) * + EXT4_INODES_PER_GROUP(inode->i_sb)) + 1; + owner[0] = i_uid_read(inode); + owner[1] = i_gid_read(inode); + tmp_inode = ext4_new_inode(handle, d_inode(inode->i_sb->s_root), + S_IFREG, NULL, goal, owner, 0); + if (IS_ERR(tmp_inode)) { + retval = PTR_ERR(tmp_inode); + ext4_journal_stop(handle); + tmp_inode = NULL; + goto out_unlock; + } + /* + * Use the correct seed for checksum (i.e. the seed from 'inode'). This + * is so that the metadata blocks will have the correct checksum after + * the migration. + */ + ei = EXT4_I(inode); + tmp_csum_seed = EXT4_I(tmp_inode)->i_csum_seed; + EXT4_I(tmp_inode)->i_csum_seed = ei->i_csum_seed; + i_size_write(tmp_inode, i_size_read(inode)); + /* + * Set the i_nlink to zero so it will be deleted later + * when we drop inode reference. + */ + clear_nlink(tmp_inode); + + ext4_ext_tree_init(handle, tmp_inode); + ext4_journal_stop(handle); + + /* + * start with one credit accounted for + * superblock modification. + * + * For the tmp_inode we already have committed the + * transaction that created the inode. Later as and + * when we add extents we extent the journal + */ + /* + * Even though we take i_rwsem we can still cause block + * allocation via mmap write to holes. If we have allocated + * new blocks we fail migrate. New block allocation will + * clear EXT4_STATE_EXT_MIGRATE flag. The flag is updated + * with i_data_sem held to prevent racing with block + * allocation. + */ + down_read(&EXT4_I(inode)->i_data_sem); + ext4_set_inode_state(inode, EXT4_STATE_EXT_MIGRATE); + up_read((&EXT4_I(inode)->i_data_sem)); + + handle = ext4_journal_start(inode, EXT4_HT_MIGRATE, 1); + if (IS_ERR(handle)) { + retval = PTR_ERR(handle); + goto out_tmp_inode; + } + + i_data = ei->i_data; + memset(&lb, 0, sizeof(lb)); + + /* 32 bit block address 4 bytes */ + max_entries = inode->i_sb->s_blocksize >> 2; + for (i = 0; i < EXT4_NDIR_BLOCKS; i++) { + if (i_data[i]) { + retval = update_extent_range(handle, tmp_inode, + le32_to_cpu(i_data[i]), &lb); + if (retval) + goto err_out; + } else + lb.curr_block++; + } + if (i_data[EXT4_IND_BLOCK]) { + retval = update_ind_extent_range(handle, tmp_inode, + le32_to_cpu(i_data[EXT4_IND_BLOCK]), &lb); + if (retval) + goto err_out; + } else + lb.curr_block += max_entries; + if (i_data[EXT4_DIND_BLOCK]) { + retval = update_dind_extent_range(handle, tmp_inode, + le32_to_cpu(i_data[EXT4_DIND_BLOCK]), &lb); + if (retval) + goto err_out; + } else + lb.curr_block += max_entries * max_entries; + if (i_data[EXT4_TIND_BLOCK]) { + retval = update_tind_extent_range(handle, tmp_inode, + le32_to_cpu(i_data[EXT4_TIND_BLOCK]), &lb); + if (retval) + goto err_out; + } + /* + * Build the last extent + */ + retval = finish_range(handle, tmp_inode, &lb); +err_out: + if (retval) + /* + * Failure case delete the extent information with the + * tmp_inode + */ + free_ext_block(handle, tmp_inode); + else { + retval = ext4_ext_swap_inode_data(handle, inode, tmp_inode); + if (retval) + /* + * if we fail to swap inode data free the extent + * details of the tmp inode + */ + free_ext_block(handle, tmp_inode); + } + + /* We mark the tmp_inode dirty via ext4_ext_tree_init. */ + retval = ext4_journal_ensure_credits(handle, 1, 0); + if (retval < 0) + goto out_stop; + /* + * Mark the tmp_inode as of size zero + */ + i_size_write(tmp_inode, 0); + + /* + * set the i_blocks count to zero + * so that the ext4_evict_inode() does the + * right job + * + * We don't need to take the i_lock because + * the inode is not visible to user space. + */ + tmp_inode->i_blocks = 0; + EXT4_I(tmp_inode)->i_csum_seed = tmp_csum_seed; + + /* Reset the extent details */ + ext4_ext_tree_init(handle, tmp_inode); +out_stop: + ext4_journal_stop(handle); +out_tmp_inode: + unlock_new_inode(tmp_inode); +out_unlock: + ext4_writepages_up_write(inode->i_sb, alloc_ctx); + iput(tmp_inode); + return retval; +} + +/* + * Migrate a simple extent-based inode to use the i_blocks[] array + */ +int ext4_ind_migrate(struct inode *inode) +{ + struct ext4_extent_header *eh; + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + struct ext4_super_block *es = sbi->s_es; + struct ext4_inode_info *ei = EXT4_I(inode); + struct ext4_extent *ex; + unsigned int i, len; + ext4_lblk_t start, end; + ext4_fsblk_t blk; + handle_t *handle; + int ret, ret2 = 0; + int alloc_ctx; + + if (!ext4_has_feature_extents(inode->i_sb) || + (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) + return -EINVAL; + + if (ext4_has_feature_bigalloc(inode->i_sb)) + return -EOPNOTSUPP; + + /* + * In order to get correct extent info, force all delayed allocation + * blocks to be allocated, otherwise delayed allocation blocks may not + * be reflected and bypass the checks on extent header. + */ + if (test_opt(inode->i_sb, DELALLOC)) + ext4_alloc_da_blocks(inode); + + alloc_ctx = ext4_writepages_down_write(inode->i_sb); + + handle = ext4_journal_start(inode, EXT4_HT_MIGRATE, 1); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + goto out_unlock; + } + /* + * This operation rewrites the inode's block mapping layout + * (extents to indirect blocks) and is not tracked in the fast + * commit log, so disable fast commits for this transaction. + */ + ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_MIGRATE, handle); + + down_write(&EXT4_I(inode)->i_data_sem); + ret = ext4_ext_check_inode(inode); + if (ret) + goto errout; + + eh = ext_inode_hdr(inode); + ex = EXT_FIRST_EXTENT(eh); + if (ext4_blocks_count(es) > EXT4_MAX_BLOCK_FILE_PHYS || + eh->eh_depth != 0 || le16_to_cpu(eh->eh_entries) > 1) { + ret = -EOPNOTSUPP; + goto errout; + } + if (eh->eh_entries == 0) + blk = len = start = end = 0; + else { + len = le16_to_cpu(ex->ee_len); + blk = ext4_ext_pblock(ex); + start = le32_to_cpu(ex->ee_block); + end = start + len - 1; + if (end >= EXT4_NDIR_BLOCKS) { + ret = -EOPNOTSUPP; + goto errout; + } + } + + ext4_clear_inode_flag(inode, EXT4_INODE_EXTENTS); + memset(ei->i_data, 0, sizeof(ei->i_data)); + for (i = start; i <= end; i++) + ei->i_data[i] = cpu_to_le32(blk++); + ret2 = ext4_mark_inode_dirty(handle, inode); + if (unlikely(ret2 && !ret)) + ret = ret2; +errout: + up_write(&EXT4_I(inode)->i_data_sem); + ext4_journal_stop(handle); +out_unlock: + ext4_writepages_up_write(inode->i_sb, alloc_ctx); + return ret; +} diff --git a/fs/ext4/mmp.c b/fs/ext4/mmp.c new file mode 100644 index 000000000..7ce361484 --- /dev/null +++ b/fs/ext4/mmp.c @@ -0,0 +1,403 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include + +#include "ext4.h" + +/* Checksumming functions */ +static __le32 ext4_mmp_csum(struct super_block *sb, struct mmp_struct *mmp) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + int offset = offsetof(struct mmp_struct, mmp_checksum); + __u32 csum; + + csum = ext4_chksum(sbi->s_csum_seed, (char *)mmp, offset); + + return cpu_to_le32(csum); +} + +static int ext4_mmp_csum_verify(struct super_block *sb, struct mmp_struct *mmp) +{ + if (!ext4_has_feature_metadata_csum(sb)) + return 1; + + return mmp->mmp_checksum == ext4_mmp_csum(sb, mmp); +} + +static void ext4_mmp_csum_set(struct super_block *sb, struct mmp_struct *mmp) +{ + if (!ext4_has_feature_metadata_csum(sb)) + return; + + mmp->mmp_checksum = ext4_mmp_csum(sb, mmp); +} + +/* + * Write the MMP block using REQ_SYNC to try to get the block on-disk + * faster. + */ +static int write_mmp_block_thawed(struct super_block *sb, + struct buffer_head *bh) +{ + struct mmp_struct *mmp = (struct mmp_struct *)(bh->b_data); + + ext4_mmp_csum_set(sb, mmp); + lock_buffer(bh); + bh_submit(bh, REQ_OP_WRITE | REQ_SYNC | REQ_META | REQ_PRIO, + bh_end_write); + wait_on_buffer(bh); + if (unlikely(!buffer_uptodate(bh))) + return -EIO; + return 0; +} + +static int write_mmp_block(struct super_block *sb, struct buffer_head *bh) +{ + /* + * We protect against freezing so that we don't create dirty buffers + * on frozen filesystem. + */ + scoped_guard(super_write, sb) + return write_mmp_block_thawed(sb, bh); +} + +/* + * Read the MMP block. It _must_ be read from disk and hence we clear the + * uptodate flag on the buffer. + */ +static int read_mmp_block(struct super_block *sb, struct buffer_head **bh, + ext4_fsblk_t mmp_block) +{ + struct mmp_struct *mmp; + int ret; + + if (*bh) + clear_buffer_uptodate(*bh); + + /* This would be sb_bread(sb, mmp_block), except we need to be sure + * that the MD RAID device cache has been bypassed, and that the read + * is not blocked in the elevator. */ + if (!*bh) { + *bh = sb_getblk(sb, mmp_block); + if (!*bh) { + ret = -ENOMEM; + goto warn_exit; + } + } + + lock_buffer(*bh); + ret = ext4_read_bh(*bh, REQ_META | REQ_PRIO, NULL, false); + if (ret) + goto warn_exit; + + mmp = (struct mmp_struct *)((*bh)->b_data); + if (le32_to_cpu(mmp->mmp_magic) != EXT4_MMP_MAGIC) { + ret = -EFSCORRUPTED; + goto warn_exit; + } + if (!ext4_mmp_csum_verify(sb, mmp)) { + ret = -EFSBADCRC; + goto warn_exit; + } + return 0; +warn_exit: + brelse(*bh); + *bh = NULL; + ext4_warning(sb, "Error %d while reading MMP block %llu", + ret, mmp_block); + return ret; +} + +/* + * Dump as much information as possible to help the admin. + */ +void __dump_mmp_msg(struct super_block *sb, struct mmp_struct *mmp, + const char *function, unsigned int line, const char *msg) +{ + __ext4_warning(sb, function, line, "%s", msg); + __ext4_warning(sb, function, line, + "MMP failure info: last update time: %llu, last update node: %.*s, last update device: %.*s", + (unsigned long long)le64_to_cpu(mmp->mmp_time), + (int)sizeof(mmp->mmp_nodename), mmp->mmp_nodename, + (int)sizeof(mmp->mmp_bdevname), mmp->mmp_bdevname); +} + +/* + * kmmpd will update the MMP sequence every s_mmp_update_interval seconds + */ +static int kmmpd(void *data) +{ + struct super_block *sb = data; + struct ext4_super_block *es = EXT4_SB(sb)->s_es; + struct buffer_head *bh = EXT4_SB(sb)->s_mmp_bh; + struct mmp_struct *mmp; + ext4_fsblk_t mmp_block; + u32 seq = 0; + unsigned long failed_writes = 0; + int mmp_update_interval = le16_to_cpu(es->s_mmp_update_interval); + unsigned mmp_check_interval; + unsigned long last_update_time; + unsigned long diff; + int retval = 0; + + mmp_block = le64_to_cpu(es->s_mmp_block); + mmp = (struct mmp_struct *)(bh->b_data); + mmp->mmp_time = cpu_to_le64(ktime_get_real_seconds()); + /* + * Start with the higher mmp_check_interval and reduce it if + * the MMP block is being updated on time. + */ + mmp_check_interval = max(EXT4_MMP_CHECK_MULT * mmp_update_interval, + EXT4_MMP_MIN_CHECK_INTERVAL); + mmp->mmp_check_interval = cpu_to_le16(mmp_check_interval); + + memcpy(mmp->mmp_nodename, init_utsname()->nodename, + sizeof(mmp->mmp_nodename)); + + while (!kthread_should_stop() && !ext4_emergency_state(sb)) { + if (!ext4_has_feature_mmp(sb)) { + ext4_warning(sb, "kmmpd being stopped since MMP feature" + " has been disabled."); + goto wait_to_exit; + } + if (++seq > EXT4_MMP_SEQ_MAX) + seq = 1; + + mmp->mmp_seq = cpu_to_le32(seq); + mmp->mmp_time = cpu_to_le64(ktime_get_real_seconds()); + last_update_time = jiffies; + + retval = write_mmp_block(sb, bh); + /* + * Don't spew too many error messages. Print one every + * (s_mmp_update_interval * 60) seconds. + */ + if (retval) { + if ((failed_writes % 60) == 0) { + ext4_error_err(sb, -retval, + "Error writing to MMP block"); + } + failed_writes++; + } + + diff = jiffies - last_update_time; + if (diff < mmp_update_interval * HZ) + schedule_timeout_interruptible(mmp_update_interval * + HZ - diff); + + /* + * We need to make sure that more than mmp_check_interval + * seconds have not passed since writing. If that has happened + * we need to check if the MMP block is as we left it. + */ + diff = jiffies - last_update_time; + if (diff > mmp_check_interval * HZ) { + struct buffer_head *bh_check = NULL; + struct mmp_struct *mmp_check; + + retval = read_mmp_block(sb, &bh_check, mmp_block); + if (retval) { + ext4_error_err(sb, -retval, + "error reading MMP data: %d", + retval); + goto wait_to_exit; + } + + mmp_check = (struct mmp_struct *)(bh_check->b_data); + if (mmp->mmp_seq != mmp_check->mmp_seq || + memcmp(mmp->mmp_nodename, mmp_check->mmp_nodename, + sizeof(mmp->mmp_nodename))) { + dump_mmp_msg(sb, mmp_check, + "Error while updating MMP info. " + "The filesystem seems to have been" + " multiply mounted."); + ext4_error_err(sb, EBUSY, "abort"); + put_bh(bh_check); + retval = -EBUSY; + goto wait_to_exit; + } + put_bh(bh_check); + } + + /* + * Adjust the mmp_check_interval depending on how much time + * it took for the MMP block to be written. + */ + mmp_check_interval = clamp(EXT4_MMP_CHECK_MULT * diff / HZ, + EXT4_MMP_MIN_CHECK_INTERVAL, + EXT4_MMP_MAX_CHECK_INTERVAL); + mmp->mmp_check_interval = cpu_to_le16(mmp_check_interval); + } + + /* + * Unmount seems to be clean. + */ + mmp->mmp_seq = cpu_to_le32(EXT4_MMP_SEQ_CLEAN); + mmp->mmp_time = cpu_to_le64(ktime_get_real_seconds()); + + retval = write_mmp_block(sb, bh); + +wait_to_exit: + while (!kthread_should_stop()) { + set_current_state(TASK_INTERRUPTIBLE); + if (!kthread_should_stop()) + schedule(); + } + set_current_state(TASK_RUNNING); + return retval; +} + +void ext4_stop_mmpd(struct ext4_sb_info *sbi) +{ + if (sbi->s_mmp_tsk) { + kthread_stop(sbi->s_mmp_tsk); + brelse(sbi->s_mmp_bh); + sbi->s_mmp_tsk = NULL; + } +} + +/* + * Get a random new sequence number but make sure it is not greater than + * EXT4_MMP_SEQ_MAX. + */ +static unsigned int mmp_new_seq(void) +{ + return get_random_u32_below(EXT4_MMP_SEQ_MAX + 1); +} + +/* + * Protect the filesystem from being mounted more than once. + */ +int ext4_multi_mount_protect(struct super_block *sb, + ext4_fsblk_t mmp_block) +{ + struct ext4_super_block *es = EXT4_SB(sb)->s_es; + struct buffer_head *bh = NULL; + struct mmp_struct *mmp = NULL; + u32 seq; + unsigned int mmp_check_interval = le16_to_cpu(es->s_mmp_update_interval); + unsigned int wait_time = 0; + int retval; + + if (mmp_block < le32_to_cpu(es->s_first_data_block) || + mmp_block >= ext4_blocks_count(es)) { + ext4_warning(sb, "Invalid MMP block in superblock"); + retval = -EINVAL; + goto failed; + } + + retval = read_mmp_block(sb, &bh, mmp_block); + if (retval) + goto failed; + + mmp = (struct mmp_struct *)(bh->b_data); + + if (mmp_check_interval < EXT4_MMP_MIN_CHECK_INTERVAL) + mmp_check_interval = EXT4_MMP_MIN_CHECK_INTERVAL; + + /* + * If check_interval in MMP block is larger, use that instead of + * update_interval from the superblock. + */ + if (le16_to_cpu(mmp->mmp_check_interval) > mmp_check_interval) + mmp_check_interval = le16_to_cpu(mmp->mmp_check_interval); + + seq = le32_to_cpu(mmp->mmp_seq); + if (seq == EXT4_MMP_SEQ_CLEAN) + goto skip; + + if (seq == EXT4_MMP_SEQ_FSCK) { + dump_mmp_msg(sb, mmp, "fsck is running on the filesystem"); + retval = -EBUSY; + goto failed; + } + + wait_time = min(mmp_check_interval * 2 + 1, + mmp_check_interval + 60); + + /* Print MMP interval if more than 20 secs. */ + if (wait_time > EXT4_MMP_MIN_CHECK_INTERVAL * 4) + ext4_warning(sb, "MMP interval %u higher than expected, please" + " wait.\n", wait_time * 2); + + if (schedule_timeout_interruptible(HZ * wait_time) != 0) { + ext4_warning(sb, "MMP startup interrupted, failing mount\n"); + retval = -ETIMEDOUT; + goto failed; + } + + retval = read_mmp_block(sb, &bh, mmp_block); + if (retval) + goto failed; + mmp = (struct mmp_struct *)(bh->b_data); + if (seq != le32_to_cpu(mmp->mmp_seq)) { + dump_mmp_msg(sb, mmp, + "Device is already active on another node."); + retval = -EBUSY; + goto failed; + } + +skip: + /* + * write a new random sequence number. + */ + seq = mmp_new_seq(); + mmp->mmp_seq = cpu_to_le32(seq); + + /* + * On mount / remount we are protected against fs freezing (by s_umount + * semaphore) and grabbing freeze protection upsets lockdep + */ + retval = write_mmp_block_thawed(sb, bh); + if (retval) + goto failed; + + /* + * wait for MMP interval and check mmp_seq. + */ + if (schedule_timeout_interruptible(HZ * wait_time) != 0) { + ext4_warning(sb, "MMP startup interrupted, failing mount"); + retval = -ETIMEDOUT; + goto failed; + } + + retval = read_mmp_block(sb, &bh, mmp_block); + if (retval) + goto failed; + mmp = (struct mmp_struct *)(bh->b_data); + if (seq != le32_to_cpu(mmp->mmp_seq)) { + dump_mmp_msg(sb, mmp, + "Device is already active on another node."); + retval = -EBUSY; + goto failed; + } + + EXT4_SB(sb)->s_mmp_bh = bh; + + BUILD_BUG_ON(sizeof(mmp->mmp_bdevname) < BDEVNAME_SIZE); + snprintf(mmp->mmp_bdevname, sizeof(mmp->mmp_bdevname), + "%pg", bh->b_bdev); + + /* + * Start a kernel thread to update the MMP block periodically. + */ + EXT4_SB(sb)->s_mmp_tsk = kthread_run(kmmpd, sb, "kmmpd-%.*s", + (int)sizeof(mmp->mmp_bdevname), + mmp->mmp_bdevname); + if (IS_ERR(EXT4_SB(sb)->s_mmp_tsk)) { + EXT4_SB(sb)->s_mmp_tsk = NULL; + ext4_warning(sb, "Unable to create kmmpd thread for %s.", + sb->s_id); + retval = -ENOMEM; + goto failed; + } + + return 0; + +failed: + brelse(bh); + return retval; +} diff --git a/fs/ext4/move_extent.c b/fs/ext4/move_extent.c new file mode 100644 index 000000000..3329b7ad5 --- /dev/null +++ b/fs/ext4/move_extent.c @@ -0,0 +1,658 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * Copyright (c) 2008,2009 NEC Software Tohoku, Ltd. + * Written by Takashi Sato + * Akira Fujita + */ + +#include +#include +#include +#include +#include "ext4_jbd2.h" +#include "ext4.h" +#include "ext4_extents.h" + +#include + +struct mext_data { + struct inode *orig_inode; /* Origin file inode */ + struct inode *donor_inode; /* Donor file inode */ + struct ext4_map_blocks orig_map;/* Origin file's move mapping */ + ext4_lblk_t donor_lblk; /* Start block of the donor file */ +}; + +/** + * ext4_double_down_write_data_sem() - write lock two inodes's i_data_sem + * @first: inode to be locked + * @second: inode to be locked + * + * Acquire write lock of i_data_sem of the two inodes + */ +void +ext4_double_down_write_data_sem(struct inode *first, struct inode *second) +{ + if (first < second) { + down_write(&EXT4_I(first)->i_data_sem); + down_write_nested(&EXT4_I(second)->i_data_sem, I_DATA_SEM_OTHER); + } else { + down_write(&EXT4_I(second)->i_data_sem); + down_write_nested(&EXT4_I(first)->i_data_sem, I_DATA_SEM_OTHER); + } +} + +/** + * ext4_double_up_write_data_sem - Release two inodes' write lock of i_data_sem + * + * @orig_inode: original inode structure to be released its lock first + * @donor_inode: donor inode structure to be released its lock second + * Release write lock of i_data_sem of two inodes (orig and donor). + */ +void +ext4_double_up_write_data_sem(struct inode *orig_inode, + struct inode *donor_inode) +{ + up_write(&EXT4_I(orig_inode)->i_data_sem); + up_write(&EXT4_I(donor_inode)->i_data_sem); +} + +/* Grab and lock folio on both @inode1 and @inode2 by inode order. */ +static int mext_folio_double_lock(struct inode *inode1, struct inode *inode2, + pgoff_t index1, pgoff_t index2, size_t len, + struct folio *folio[2]) +{ + struct address_space *mapping[2]; + unsigned int flags; + fgf_t fgp_flags = FGP_WRITEBEGIN; + + BUG_ON(!inode1 || !inode2); + if (inode1 < inode2) { + mapping[0] = inode1->i_mapping; + mapping[1] = inode2->i_mapping; + } else { + swap(index1, index2); + mapping[0] = inode2->i_mapping; + mapping[1] = inode1->i_mapping; + } + + flags = memalloc_nofs_save(); + fgp_flags |= fgf_set_order(len); + folio[0] = __filemap_get_folio(mapping[0], index1, fgp_flags, + mapping_gfp_mask(mapping[0])); + if (IS_ERR(folio[0])) { + memalloc_nofs_restore(flags); + return PTR_ERR(folio[0]); + } + + folio[1] = __filemap_get_folio(mapping[1], index2, fgp_flags, + mapping_gfp_mask(mapping[1])); + memalloc_nofs_restore(flags); + if (IS_ERR(folio[1])) { + folio_unlock(folio[0]); + folio_put(folio[0]); + return PTR_ERR(folio[1]); + } + /* + * __filemap_get_folio() may not wait on folio's writeback if + * BDI not demand that. But it is reasonable to be very conservative + * here and explicitly wait on folio's writeback + */ + folio_wait_writeback(folio[0]); + folio_wait_writeback(folio[1]); + if (inode1 > inode2) + swap(folio[0], folio[1]); + + return 0; +} + +static void mext_folio_double_unlock(struct folio *folio[2]) +{ + folio_unlock(folio[0]); + folio_put(folio[0]); + folio_unlock(folio[1]); + folio_put(folio[1]); +} + +/* Force folio buffers uptodate w/o dropping folio's lock */ +static int mext_folio_mkuptodate(struct folio *folio, size_t from, size_t to) +{ + struct inode *inode = folio->mapping->host; + sector_t block; + struct buffer_head *bh, *head; + unsigned int blocksize, block_start, block_end; + int nr = 0; + bool partial = false; + + BUG_ON(!folio_test_locked(folio)); + BUG_ON(folio_test_writeback(folio)); + + if (folio_test_uptodate(folio)) + return 0; + + blocksize = i_blocksize(inode); + head = folio_buffers(folio); + if (!head) + head = create_empty_buffers(folio, blocksize, 0); + + block = folio_pos(folio) >> inode->i_blkbits; + block_end = 0; + bh = head; + do { + block_start = block_end; + block_end = block_start + blocksize; + if (block_end <= from || block_start >= to) { + if (!buffer_uptodate(bh)) + partial = true; + continue; + } + if (buffer_uptodate(bh)) + continue; + if (!buffer_mapped(bh)) { + int err = ext4_get_block(inode, block, bh, 0); + if (err) + return err; + if (!buffer_mapped(bh)) { + folio_zero_range(folio, block_start, blocksize); + set_buffer_uptodate(bh); + continue; + } + } + lock_buffer(bh); + if (buffer_uptodate(bh)) { + unlock_buffer(bh); + continue; + } + ext4_read_bh_nowait(bh, 0, NULL, false); + nr++; + } while (block++, (bh = bh->b_this_page) != head); + + /* No io required */ + if (!nr) + goto out; + + bh = head; + do { + if (bh_offset(bh) + blocksize <= from) + continue; + if (bh_offset(bh) >= to) + break; + wait_on_buffer(bh); + if (buffer_uptodate(bh)) + continue; + return -EIO; + } while ((bh = bh->b_this_page) != head); +out: + if (!partial) + folio_mark_uptodate(folio); + return 0; +} + +enum mext_move_type {MEXT_SKIP_EXTENT, MEXT_MOVE_EXTENT, MEXT_COPY_DATA}; + +/* + * Start to move extent between the origin inode and the donor inode, + * hold one folio for each inode and check the candidate moving extent + * mapping status again. + */ +static int mext_move_begin(struct mext_data *mext, struct folio *folio[2], + enum mext_move_type *move_type) +{ + struct inode *orig_inode = mext->orig_inode; + struct inode *donor_inode = mext->donor_inode; + unsigned int blkbits = orig_inode->i_blkbits; + struct ext4_map_blocks donor_map = {0}; + loff_t orig_pos, donor_pos; + size_t move_len; + int ret; + + orig_pos = ((loff_t)mext->orig_map.m_lblk) << blkbits; + donor_pos = ((loff_t)mext->donor_lblk) << blkbits; + ret = mext_folio_double_lock(orig_inode, donor_inode, + orig_pos >> PAGE_SHIFT, donor_pos >> PAGE_SHIFT, + ((size_t)mext->orig_map.m_len) << blkbits, folio); + if (ret) + return ret; + + /* + * Check the origin inode's mapping information again under the + * folio lock, as we do not hold the i_data_sem at all times, and + * it may change during the concurrent write-back operation. + */ + if (mext->orig_map.m_seq != READ_ONCE(EXT4_I(orig_inode)->i_es_seq)) { + ret = -ESTALE; + goto error; + } + + /* Adjust the moving length according to the length of shorter folio. */ + move_len = umin(folio_next_pos(folio[0]) - orig_pos, + folio_next_pos(folio[1]) - donor_pos); + move_len >>= blkbits; + if (move_len < mext->orig_map.m_len) + mext->orig_map.m_len = move_len; + + donor_map.m_lblk = mext->donor_lblk; + donor_map.m_len = mext->orig_map.m_len; + donor_map.m_flags = 0; + ret = ext4_map_blocks(NULL, donor_inode, &donor_map, 0); + if (ret < 0) + goto error; + + /* Adjust the moving length according to the donor mapping length. */ + mext->orig_map.m_len = donor_map.m_len; + + /* Skip moving if the donor range is a hole or a delalloc extent. */ + if (!(donor_map.m_flags & (EXT4_MAP_MAPPED | EXT4_MAP_UNWRITTEN))) + *move_type = MEXT_SKIP_EXTENT; + /* If both mapping ranges are unwritten, no need to copy data. */ + else if ((mext->orig_map.m_flags & EXT4_MAP_UNWRITTEN) && + (donor_map.m_flags & EXT4_MAP_UNWRITTEN)) + *move_type = MEXT_MOVE_EXTENT; + else + *move_type = MEXT_COPY_DATA; + + return 0; +error: + mext_folio_double_unlock(folio); + return ret; +} + +/* + * Re-create the new moved mapping buffers of the original inode and commit + * the entire written range. + */ +static int mext_folio_mkwrite(struct inode *inode, struct folio *folio, + size_t from, size_t to) +{ + unsigned int blocksize = i_blocksize(inode); + struct buffer_head *bh, *head; + size_t block_start, block_end; + sector_t block; + int ret; + + head = folio_buffers(folio); + if (!head) + head = create_empty_buffers(folio, blocksize, 0); + + block = folio_pos(folio) >> inode->i_blkbits; + block_end = 0; + bh = head; + do { + block_start = block_end; + block_end = block_start + blocksize; + if (block_end <= from || block_start >= to) + continue; + + ret = ext4_get_block(inode, block, bh, 0); + if (ret) + return ret; + } while (block++, (bh = bh->b_this_page) != head); + + block_commit_write(folio, from, to); + return 0; +} + +/* + * Save the data in original inode extent blocks and replace one folio size + * aligned original inode extent with one or one partial donor inode extent, + * and then write out the saved data in new original inode blocks. Pass out + * the replaced block count through m_len. Return 0 on success, and an error + * code otherwise. + */ +static int mext_move_extent(struct mext_data *mext, u64 *m_len) +{ + struct inode *orig_inode = mext->orig_inode; + struct inode *donor_inode = mext->donor_inode; + struct ext4_map_blocks *orig_map = &mext->orig_map; + unsigned int blkbits = orig_inode->i_blkbits; + struct folio *folio[2] = {NULL, NULL}; + loff_t from, length; + enum mext_move_type move_type = 0; + handle_t *handle; + u64 r_len = 0; + unsigned int credits; + int ret, ret2; + + *m_len = 0; + trace_ext4_move_extent_enter(orig_inode, orig_map, donor_inode, + mext->donor_lblk); + credits = ext4_chunk_trans_extent(orig_inode, 0) * 2; + handle = ext4_journal_start(orig_inode, EXT4_HT_MOVE_EXTENTS, credits); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + goto out; + } + ext4_fc_mark_ineligible(orig_inode->i_sb, EXT4_FC_REASON_MOVE_EXT, + handle); + + ret = mext_move_begin(mext, folio, &move_type); + if (ret) + goto stop_handle; + + if (move_type == MEXT_SKIP_EXTENT) + goto unlock; + + /* + * Copy the data. First, read the original inode data into the page + * cache. Then, release the existing mapping relationships and swap + * the extent. Finally, re-establish the new mapping relationships + * and dirty the page cache. + */ + if (move_type == MEXT_COPY_DATA) { + from = offset_in_folio(folio[0], + ((loff_t)orig_map->m_lblk) << blkbits); + length = ((loff_t)orig_map->m_len) << blkbits; + + ret = mext_folio_mkuptodate(folio[0], from, from + length); + if (ret) + goto unlock; + } + + if (!filemap_release_folio(folio[0], 0) || + !filemap_release_folio(folio[1], 0)) { + ret = -EBUSY; + goto unlock; + } + + /* Move extent */ + ext4_double_down_write_data_sem(orig_inode, donor_inode); + *m_len = ext4_swap_extents(handle, orig_inode, donor_inode, + orig_map->m_lblk, mext->donor_lblk, + orig_map->m_len, 1, &ret); + ext4_double_up_write_data_sem(orig_inode, donor_inode); + + /* A short-length swap cannot occur after a successful swap extent. */ + if (WARN_ON_ONCE(!ret && (*m_len != orig_map->m_len))) + ret = -EIO; + + if (!(*m_len) || (move_type == MEXT_MOVE_EXTENT)) + goto unlock; + + /* Copy data */ + length = (*m_len) << blkbits; + ret2 = mext_folio_mkwrite(orig_inode, folio[0], from, from + length); + if (ret2) { + if (!ret) + ret = ret2; + goto repair_branches; + } + /* + * Even in case of data=writeback it is reasonable to pin + * inode to transaction, to prevent unexpected data loss. + */ + ret2 = ext4_jbd2_inode_add_write(handle, orig_inode, + ((loff_t)orig_map->m_lblk) << blkbits, length); + if (!ret) + ret = ret2; +unlock: + mext_folio_double_unlock(folio); +stop_handle: + ext4_journal_stop(handle); +out: + trace_ext4_move_extent_exit(orig_inode, orig_map->m_lblk, donor_inode, + mext->donor_lblk, orig_map->m_len, *m_len, + move_type, ret); + return ret; + +repair_branches: + ret2 = 0; + ext4_double_down_write_data_sem(orig_inode, donor_inode); + r_len = ext4_swap_extents(handle, donor_inode, orig_inode, + mext->donor_lblk, orig_map->m_lblk, + *m_len, 0, &ret2); + ext4_double_up_write_data_sem(orig_inode, donor_inode); + if (ret2 || r_len != *m_len) { + ext4_error_inode_block(orig_inode, (sector_t)(orig_map->m_lblk), + EIO, "Unable to copy data block, data will be lost!"); + ret = -EIO; + } + *m_len = 0; + goto unlock; +} + +/* + * Check the validity of the basic filesystem environment and the + * inodes' support status. + */ +static int mext_check_validity(struct inode *orig_inode, + struct inode *donor_inode) +{ + struct super_block *sb = orig_inode->i_sb; + + /* origin and donor should be different inodes */ + if (orig_inode == donor_inode) { + ext4_debug("ext4 move extent: The argument files should not be same inode [ino:orig %llu, donor %llu]\n", + orig_inode->i_ino, donor_inode->i_ino); + return -EINVAL; + } + + /* origin and donor should belone to the same filesystem */ + if (orig_inode->i_sb != donor_inode->i_sb) { + ext4_debug("ext4 move extent: The argument files should be in same FS [ino:orig %llu, donor %llu]\n", + orig_inode->i_ino, donor_inode->i_ino); + return -EINVAL; + } + + /* Regular file check */ + if (!S_ISREG(orig_inode->i_mode) || !S_ISREG(donor_inode->i_mode)) { + ext4_debug("ext4 move extent: The argument files should be regular file [ino:orig %llu, donor %llu]\n", + orig_inode->i_ino, donor_inode->i_ino); + return -EINVAL; + } + + if (ext4_has_feature_bigalloc(sb)) { + ext4_msg(sb, KERN_ERR, + "Online defrag not supported with bigalloc"); + return -EOPNOTSUPP; + } + + if (IS_DAX(orig_inode)) { + ext4_msg(sb, KERN_ERR, + "Online defrag not supported with DAX"); + return -EOPNOTSUPP; + } + + /* + * TODO: it's not obvious how to swap blocks for inodes with full + * journaling enabled. + */ + if (ext4_should_journal_data(orig_inode) || + ext4_should_journal_data(donor_inode)) { + ext4_msg(sb, KERN_ERR, + "Online defrag not supported with data journaling"); + return -EOPNOTSUPP; + } + + if (IS_ENCRYPTED(orig_inode) || IS_ENCRYPTED(donor_inode)) { + ext4_msg(sb, KERN_ERR, + "Online defrag not supported for encrypted files"); + return -EOPNOTSUPP; + } + + /* Ext4 move extent supports only extent based file */ + if (!(ext4_test_inode_flag(orig_inode, EXT4_INODE_EXTENTS)) || + !(ext4_test_inode_flag(donor_inode, EXT4_INODE_EXTENTS))) { + ext4_msg(sb, KERN_ERR, + "Online defrag not supported for non-extent files"); + return -EOPNOTSUPP; + } + + if (donor_inode->i_mode & (S_ISUID|S_ISGID)) { + ext4_debug("ext4 move extent: suid or sgid is set to donor file [ino:orig %llu, donor %llu]\n", + orig_inode->i_ino, donor_inode->i_ino); + return -EINVAL; + } + + if (IS_IMMUTABLE(donor_inode) || IS_APPEND(donor_inode)) { + ext4_debug("ext4 move extent: donor should not be immutable or append file [ino:orig %llu, donor %llu]\n", + orig_inode->i_ino, donor_inode->i_ino); + return -EPERM; + } + + /* Ext4 move extent does not support swap files */ + if (IS_SWAPFILE(orig_inode) || IS_SWAPFILE(donor_inode)) { + ext4_debug("ext4 move extent: The argument files should not be swap files [ino:orig %llu, donor %llu]\n", + orig_inode->i_ino, donor_inode->i_ino); + return -ETXTBSY; + } + + if (ext4_is_quota_file(orig_inode) || ext4_is_quota_file(donor_inode)) { + ext4_debug("ext4 move extent: The argument files should not be quota files [ino:orig %llu, donor %llu]\n", + orig_inode->i_ino, donor_inode->i_ino); + return -EOPNOTSUPP; + } + + if ((!orig_inode->i_size) || (!donor_inode->i_size)) { + ext4_debug("ext4 move extent: File size is 0 byte\n"); + return -EINVAL; + } + + return 0; +} + +/* + * Check the moving range of ext4_move_extents() whether the files can be + * exchanged with each other, and adjust the length to fit within the file + * size. Return 0 on success, or a negative error value on failure. + */ +static int mext_check_adjust_range(struct inode *orig_inode, + struct inode *donor_inode, __u64 orig_start, + __u64 donor_start, __u64 *len) +{ + __u64 orig_eof, donor_eof; + + /* Start offset should be same */ + if ((orig_start & ~(PAGE_MASK >> orig_inode->i_blkbits)) != + (donor_start & ~(PAGE_MASK >> orig_inode->i_blkbits))) { + ext4_debug("ext4 move extent: orig and donor's start offsets are not aligned [ino:orig %llu, donor %llu]\n", + orig_inode->i_ino, donor_inode->i_ino); + return -EINVAL; + } + + if ((orig_start >= EXT_MAX_BLOCKS) || + (donor_start >= EXT_MAX_BLOCKS) || + (*len > EXT_MAX_BLOCKS) || + (donor_start + *len >= EXT_MAX_BLOCKS) || + (orig_start + *len >= EXT_MAX_BLOCKS)) { + ext4_debug("ext4 move extent: Can't handle over [%u] blocks [ino:orig %llu, donor %llu]\n", + EXT_MAX_BLOCKS, + orig_inode->i_ino, donor_inode->i_ino); + return -EINVAL; + } + + orig_eof = EXT4_B_TO_LBLK(orig_inode, i_size_read(orig_inode)); + donor_eof = EXT4_B_TO_LBLK(donor_inode, i_size_read(donor_inode)); + if (orig_eof <= orig_start) + *len = 0; + else if (orig_eof < orig_start + *len - 1) + *len = orig_eof - orig_start; + if (donor_eof <= donor_start) + *len = 0; + else if (donor_eof < donor_start + *len - 1) + *len = donor_eof - donor_start; + if (!*len) { + ext4_debug("ext4 move extent: len should not be 0 [ino:orig %llu, donor %llu]\n", + orig_inode->i_ino, donor_inode->i_ino); + return -EINVAL; + } + + return 0; +} + +/** + * ext4_move_extents - Exchange the specified range of a file + * + * @o_filp: file structure of the original file + * @d_filp: file structure of the donor file + * @orig_blk: start offset in block for orig + * @donor_blk: start offset in block for donor + * @len: the number of blocks to be moved + * @moved_len: moved block length + * + * This function returns 0 and moved block length is set in moved_len + * if succeed, otherwise returns error value. + */ +int ext4_move_extents(struct file *o_filp, struct file *d_filp, __u64 orig_blk, + __u64 donor_blk, __u64 len, __u64 *moved_len) +{ + struct inode *orig_inode = file_inode(o_filp); + struct inode *donor_inode = file_inode(d_filp); + struct mext_data mext; + struct super_block *sb = orig_inode->i_sb; + struct ext4_sb_info *sbi = EXT4_SB(sb); + int retries = 0; + u64 m_len; + int ret; + + *moved_len = 0; + + /* Protect orig and donor inodes against a truncate */ + lock_two_nondirectories(orig_inode, donor_inode); + + ret = mext_check_validity(orig_inode, donor_inode); + if (ret) + goto out; + + /* Wait for all existing dio workers */ + inode_dio_wait(orig_inode); + inode_dio_wait(donor_inode); + + /* Check and adjust the specified move_extent range. */ + ret = mext_check_adjust_range(orig_inode, donor_inode, orig_blk, + donor_blk, &len); + if (ret) + goto out; + + mext.orig_inode = orig_inode; + mext.donor_inode = donor_inode; + while (len) { + mext.orig_map.m_lblk = orig_blk; + mext.orig_map.m_len = len; + mext.orig_map.m_flags = 0; + mext.donor_lblk = donor_blk; + + ret = ext4_map_blocks(NULL, orig_inode, &mext.orig_map, 0); + if (ret < 0) + goto out; + + /* Skip moving if it is a hole or a delalloc extent. */ + if (mext.orig_map.m_flags & + (EXT4_MAP_MAPPED | EXT4_MAP_UNWRITTEN)) { + ret = mext_move_extent(&mext, &m_len); + *moved_len += m_len; + if (!ret) + goto next; + + /* Move failed or partially failed. */ + if (m_len) { + orig_blk += m_len; + donor_blk += m_len; + len -= m_len; + } + if (ret == -ESTALE) + continue; + if (ret == -ENOSPC && + ext4_should_retry_alloc(sb, &retries)) + continue; + if (ret == -EBUSY && + sbi->s_journal && retries++ < 4 && + jbd2_journal_force_commit_nested(sbi->s_journal)) + continue; + + goto out; + } +next: + orig_blk += mext.orig_map.m_len; + donor_blk += mext.orig_map.m_len; + len -= mext.orig_map.m_len; + retries = 0; + } + +out: + if (*moved_len) { + ext4_discard_preallocations(orig_inode); + ext4_discard_preallocations(donor_inode); + } + + unlock_two_nondirectories(orig_inode, donor_inode); + return ret; +} diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c new file mode 100644 index 000000000..a6386c1d2 --- /dev/null +++ b/fs/ext4/namei.c @@ -0,0 +1,4251 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext4/namei.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * from + * + * linux/fs/minix/namei.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * Big-endian to little-endian byte-swapping/bitmaps by + * David S. Miller (davem@caip.rutgers.edu), 1995 + * Directory entry file type support and forward compatibility hooks + * for B-tree directories by Theodore Ts'o (tytso@mit.edu), 1998 + * Hash Tree Directory indexing (c) + * Daniel Phillips, 2001 + * Hash Tree Directory indexing porting + * Christopher Li, 2002 + * Hash Tree Directory indexing cleanup + * Theodore Ts'o, 2002 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ext4.h" +#include "ext4_jbd2.h" + +#include "xattr.h" +#include "acl.h" + +#include +/* + * define how far ahead to read directories while searching them. + */ +#define NAMEI_RA_CHUNKS 2 +#define NAMEI_RA_BLOCKS 4 +#define NAMEI_RA_SIZE (NAMEI_RA_CHUNKS * NAMEI_RA_BLOCKS) + +static struct buffer_head *ext4_append(handle_t *handle, + struct inode *inode, + ext4_lblk_t *block) +{ + struct ext4_map_blocks map; + struct buffer_head *bh; + int err; + + if (unlikely(EXT4_SB(inode->i_sb)->s_max_dir_size_kb && + ((inode->i_size >> 10) >= + EXT4_SB(inode->i_sb)->s_max_dir_size_kb))) + return ERR_PTR(-ENOSPC); + + *block = inode->i_size >> inode->i_sb->s_blocksize_bits; + map.m_lblk = *block; + map.m_len = 1; + + /* + * We're appending new directory block. Make sure the block is not + * allocated yet, otherwise we will end up corrupting the + * directory. + */ + err = ext4_map_blocks(NULL, inode, &map, 0); + if (err < 0) + return ERR_PTR(err); + if (err) { + EXT4_ERROR_INODE(inode, "Logical block already allocated"); + return ERR_PTR(-EFSCORRUPTED); + } + + bh = ext4_bread(handle, inode, *block, EXT4_GET_BLOCKS_CREATE); + if (IS_ERR(bh)) + return bh; + inode->i_size += inode->i_sb->s_blocksize; + EXT4_I(inode)->i_disksize = inode->i_size; + err = ext4_mark_inode_dirty(handle, inode); + if (err) + goto out; + BUFFER_TRACE(bh, "get_write_access"); + err = ext4_journal_get_write_access(handle, inode->i_sb, bh, + EXT4_JTR_NONE); + if (err) + goto out; + return bh; + +out: + brelse(bh); + ext4_std_error(inode->i_sb, err); + return ERR_PTR(err); +} + +static int ext4_dx_csum_verify(struct inode *inode, + struct ext4_dir_entry *dirent); + +/* + * Hints to ext4_read_dirblock regarding whether we expect a directory + * block being read to be an index block, or a block containing + * directory entries (and if the latter, whether it was found via a + * logical block in an htree index block). This is used to control + * what sort of sanity checkinig ext4_read_dirblock() will do on the + * directory block read from the storage device. EITHER will means + * the caller doesn't know what kind of directory block will be read, + * so no specific verification will be done. + */ +typedef enum { + EITHER, INDEX, DIRENT, DIRENT_HTREE +} dirblock_type_t; + +#define ext4_read_dirblock(inode, block, type) \ + __ext4_read_dirblock((inode), (block), (type), __func__, __LINE__) + +static struct buffer_head *__ext4_read_dirblock(struct inode *inode, + ext4_lblk_t block, + dirblock_type_t type, + const char *func, + unsigned int line) +{ + struct buffer_head *bh; + struct ext4_dir_entry *dirent; + int is_dx_block = 0; + + if (block >= inode->i_size >> inode->i_blkbits) { + ext4_error_inode(inode, func, line, block, + "Attempting to read directory block (%u) that is past i_size (%llu)", + block, inode->i_size); + return ERR_PTR(-EFSCORRUPTED); + } + + if (ext4_simulate_fail(inode->i_sb, EXT4_SIM_DIRBLOCK_EIO)) + bh = ERR_PTR(-EIO); + else + bh = ext4_bread(NULL, inode, block, 0); + if (IS_ERR(bh)) { + __ext4_warning(inode->i_sb, func, line, + "inode #%llu: lblock %lu: comm %s: " + "error %pe reading directory block", + inode->i_ino, (unsigned long)block, + current->comm, bh); + + return bh; + } + /* The first directory block must not be a hole. */ + if (!bh && (type == INDEX || type == DIRENT_HTREE || block == 0)) { + ext4_error_inode(inode, func, line, block, + "Directory hole found for htree %s block %u", + (type == INDEX) ? "index" : "leaf", block); + return ERR_PTR(-EFSCORRUPTED); + } + if (!bh) + return NULL; + dirent = (struct ext4_dir_entry *) bh->b_data; + /* Determine whether or not we have an index block */ + if (is_dx(inode)) { + if (block == 0) + is_dx_block = 1; + else if (ext4_rec_len_from_disk(dirent->rec_len, + inode->i_sb->s_blocksize) == + inode->i_sb->s_blocksize) + is_dx_block = 1; + } + if (!is_dx_block && type == INDEX) { + ext4_error_inode(inode, func, line, block, + "directory leaf block found instead of index block"); + brelse(bh); + return ERR_PTR(-EFSCORRUPTED); + } + if (!ext4_has_feature_metadata_csum(inode->i_sb) || + buffer_verified(bh)) + return bh; + + /* + * An empty leaf block can get mistaken for a index block; for + * this reason, we can only check the index checksum when the + * caller is sure it should be an index block. + */ + if (is_dx_block && type == INDEX) { + if (ext4_dx_csum_verify(inode, dirent) && + !ext4_simulate_fail(inode->i_sb, EXT4_SIM_DIRBLOCK_CRC)) + set_buffer_verified(bh); + else { + ext4_error_inode_err(inode, func, line, block, + EFSBADCRC, + "Directory index failed checksum"); + brelse(bh); + return ERR_PTR(-EFSBADCRC); + } + } + if (!is_dx_block) { + if (ext4_dirblock_csum_verify(inode, bh) && + !ext4_simulate_fail(inode->i_sb, EXT4_SIM_DIRBLOCK_CRC)) + set_buffer_verified(bh); + else { + ext4_error_inode_err(inode, func, line, block, + EFSBADCRC, + "Directory block failed checksum"); + brelse(bh); + return ERR_PTR(-EFSBADCRC); + } + } + return bh; +} + +#ifdef DX_DEBUG +#define dxtrace(command) command +#else +#define dxtrace(command) +#endif + +struct fake_dirent +{ + __le32 inode; + __le16 rec_len; + u8 name_len; + u8 file_type; +}; + +struct dx_countlimit +{ + __le16 limit; + __le16 count; +}; + +struct dx_entry +{ + __le32 hash; + __le32 block; +}; + +/* + * dx_root_info is laid out so that if it should somehow get overlaid by a + * dirent the two low bits of the hash version will be zero. Therefore, the + * hash version mod 4 should never be 0. Sincerely, the paranoia department. + */ + +struct dx_root +{ + struct fake_dirent dot; + char dot_name[4]; + struct fake_dirent dotdot; + char dotdot_name[4]; + struct dx_root_info + { + __le32 reserved_zero; + u8 hash_version; + u8 info_length; /* 8 */ + u8 indirect_levels; + u8 unused_flags; + } + info; + struct dx_entry entries[]; +}; + +struct dx_node +{ + struct fake_dirent fake; + struct dx_entry entries[]; +}; + + +struct dx_frame +{ + struct buffer_head *bh; + struct dx_entry *entries; + struct dx_entry *at; +}; + +struct dx_map_entry +{ + u32 hash; + u16 offs; + u16 size; +}; + +/* + * This goes at the end of each htree block. + */ +struct dx_tail { + u32 dt_reserved; + __le32 dt_checksum; /* crc32c(uuid+inum+dirblock) */ +}; + +static struct buffer_head * ext4_dx_find_entry(struct inode *dir, + struct ext4_filename *fname, + struct ext4_dir_entry_2 **res_dir); +static int ext4_dx_add_entry(handle_t *handle, struct ext4_filename *fname, + struct inode *dir, struct inode *inode); + +/* checksumming functions */ +void ext4_initialize_dirent_tail(struct buffer_head *bh, + unsigned int blocksize) +{ + struct ext4_dir_entry_tail *t = EXT4_DIRENT_TAIL(bh->b_data, blocksize); + + memset(t, 0, sizeof(struct ext4_dir_entry_tail)); + t->det_rec_len = ext4_rec_len_to_disk( + sizeof(struct ext4_dir_entry_tail), blocksize); + t->det_reserved_ft = EXT4_FT_DIR_CSUM; +} + +/* Walk through a dirent block to find a checksum "dirent" at the tail */ +static struct ext4_dir_entry_tail *get_dirent_tail(struct inode *inode, + struct buffer_head *bh) +{ + struct ext4_dir_entry_tail *t; + int blocksize = EXT4_BLOCK_SIZE(inode->i_sb); + +#ifdef PARANOID + struct ext4_dir_entry *d, *top; + + d = (struct ext4_dir_entry *)bh->b_data; + top = (struct ext4_dir_entry *)(bh->b_data + + (blocksize - sizeof(struct ext4_dir_entry_tail))); + while (d < top && ext4_rec_len_from_disk(d->rec_len, blocksize)) + d = (struct ext4_dir_entry *)(((void *)d) + + ext4_rec_len_from_disk(d->rec_len, blocksize)); + + if (d != top) + return NULL; + + t = (struct ext4_dir_entry_tail *)d; +#else + t = EXT4_DIRENT_TAIL(bh->b_data, EXT4_BLOCK_SIZE(inode->i_sb)); +#endif + + if (t->det_reserved_zero1 || + (ext4_rec_len_from_disk(t->det_rec_len, blocksize) != + sizeof(struct ext4_dir_entry_tail)) || + t->det_reserved_zero2 || + t->det_reserved_ft != EXT4_FT_DIR_CSUM) + return NULL; + + return t; +} + +static __le32 ext4_dirblock_csum(struct inode *inode, void *dirent, int size) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + __u32 csum; + + csum = ext4_chksum(ei->i_csum_seed, (__u8 *)dirent, size); + return cpu_to_le32(csum); +} + +#define warn_no_space_for_csum(inode) \ + __warn_no_space_for_csum((inode), __func__, __LINE__) + +static void __warn_no_space_for_csum(struct inode *inode, const char *func, + unsigned int line) +{ + __ext4_warning_inode(inode, func, line, + "No space for directory leaf checksum. Please run e2fsck -D."); +} + +int ext4_dirblock_csum_verify(struct inode *inode, struct buffer_head *bh) +{ + struct ext4_dir_entry_tail *t; + + if (!ext4_has_feature_metadata_csum(inode->i_sb)) + return 1; + + t = get_dirent_tail(inode, bh); + if (!t) { + warn_no_space_for_csum(inode); + return 0; + } + + if (t->det_checksum != ext4_dirblock_csum(inode, bh->b_data, + (char *)t - bh->b_data)) + return 0; + + return 1; +} + +static void ext4_dirblock_csum_set(struct inode *inode, + struct buffer_head *bh) +{ + struct ext4_dir_entry_tail *t; + + if (!ext4_has_feature_metadata_csum(inode->i_sb)) + return; + + t = get_dirent_tail(inode, bh); + if (!t) { + warn_no_space_for_csum(inode); + return; + } + + t->det_checksum = ext4_dirblock_csum(inode, bh->b_data, + (char *)t - bh->b_data); +} + +int ext4_handle_dirty_dirblock(handle_t *handle, + struct inode *inode, + struct buffer_head *bh) +{ + ext4_dirblock_csum_set(inode, bh); + return ext4_handle_dirty_metadata(handle, inode, bh); +} + +static struct dx_countlimit *get_dx_countlimit(struct inode *inode, + struct ext4_dir_entry *dirent, + int *offset) +{ + struct ext4_dir_entry *dp; + struct dx_root_info *root; + int count_offset; + int blocksize = EXT4_BLOCK_SIZE(inode->i_sb); + unsigned int rlen = ext4_rec_len_from_disk(dirent->rec_len, blocksize); + + if (rlen == blocksize) + count_offset = 8; + else if (rlen == 12) { + dp = (struct ext4_dir_entry *)(((void *)dirent) + 12); + if (ext4_rec_len_from_disk(dp->rec_len, blocksize) != blocksize - 12) + return NULL; + root = (struct dx_root_info *)(((void *)dp + 12)); + if (root->reserved_zero || + root->info_length != sizeof(struct dx_root_info)) + return NULL; + count_offset = 32; + } else + return NULL; + + if (offset) + *offset = count_offset; + return (struct dx_countlimit *)(((void *)dirent) + count_offset); +} + +static __le32 ext4_dx_csum(struct inode *inode, struct ext4_dir_entry *dirent, + int count_offset, int count, struct dx_tail *t) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + __u32 csum; + int size; + __u32 dummy_csum = 0; + int offset = offsetof(struct dx_tail, dt_checksum); + + size = count_offset + (count * sizeof(struct dx_entry)); + csum = ext4_chksum(ei->i_csum_seed, (__u8 *)dirent, size); + csum = ext4_chksum(csum, (__u8 *)t, offset); + csum = ext4_chksum(csum, (__u8 *)&dummy_csum, sizeof(dummy_csum)); + + return cpu_to_le32(csum); +} + +static int ext4_dx_csum_verify(struct inode *inode, + struct ext4_dir_entry *dirent) +{ + struct dx_countlimit *c; + struct dx_tail *t; + int count_offset, limit, count; + + if (!ext4_has_feature_metadata_csum(inode->i_sb)) + return 1; + + c = get_dx_countlimit(inode, dirent, &count_offset); + if (!c) { + EXT4_ERROR_INODE(inode, "dir seems corrupt? Run e2fsck -D."); + return 0; + } + limit = le16_to_cpu(c->limit); + count = le16_to_cpu(c->count); + if (count_offset + (limit * sizeof(struct dx_entry)) > + EXT4_BLOCK_SIZE(inode->i_sb) - sizeof(struct dx_tail)) { + warn_no_space_for_csum(inode); + return 0; + } + t = (struct dx_tail *)(((struct dx_entry *)c) + limit); + + if (t->dt_checksum != ext4_dx_csum(inode, dirent, count_offset, + count, t)) + return 0; + return 1; +} + +static void ext4_dx_csum_set(struct inode *inode, struct ext4_dir_entry *dirent) +{ + struct dx_countlimit *c; + struct dx_tail *t; + int count_offset, limit, count; + + if (!ext4_has_feature_metadata_csum(inode->i_sb)) + return; + + c = get_dx_countlimit(inode, dirent, &count_offset); + if (!c) { + EXT4_ERROR_INODE(inode, "dir seems corrupt? Run e2fsck -D."); + return; + } + limit = le16_to_cpu(c->limit); + count = le16_to_cpu(c->count); + if (count_offset + (limit * sizeof(struct dx_entry)) > + EXT4_BLOCK_SIZE(inode->i_sb) - sizeof(struct dx_tail)) { + warn_no_space_for_csum(inode); + return; + } + t = (struct dx_tail *)(((struct dx_entry *)c) + limit); + + t->dt_checksum = ext4_dx_csum(inode, dirent, count_offset, count, t); +} + +static inline int ext4_handle_dirty_dx_node(handle_t *handle, + struct inode *inode, + struct buffer_head *bh) +{ + ext4_dx_csum_set(inode, (struct ext4_dir_entry *)bh->b_data); + return ext4_handle_dirty_metadata(handle, inode, bh); +} + +/* + * p is at least 6 bytes before the end of page + */ +static inline struct ext4_dir_entry_2 * +ext4_next_entry(struct ext4_dir_entry_2 *p, unsigned long blocksize) +{ + return (struct ext4_dir_entry_2 *)((char *)p + + ext4_rec_len_from_disk(p->rec_len, blocksize)); +} + +/* + * Future: use high four bits of block for coalesce-on-delete flags + * Mask them off for now. + */ + +static inline ext4_lblk_t dx_get_block(struct dx_entry *entry) +{ + return le32_to_cpu(entry->block) & 0x0fffffff; +} + +static inline void dx_set_block(struct dx_entry *entry, ext4_lblk_t value) +{ + entry->block = cpu_to_le32(value); +} + +static inline unsigned dx_get_hash(struct dx_entry *entry) +{ + return le32_to_cpu(entry->hash); +} + +static inline void dx_set_hash(struct dx_entry *entry, unsigned value) +{ + entry->hash = cpu_to_le32(value); +} + +static inline unsigned dx_get_count(struct dx_entry *entries) +{ + return le16_to_cpu(((struct dx_countlimit *) entries)->count); +} + +static inline unsigned dx_get_limit(struct dx_entry *entries) +{ + return le16_to_cpu(((struct dx_countlimit *) entries)->limit); +} + +static inline void dx_set_count(struct dx_entry *entries, unsigned value) +{ + ((struct dx_countlimit *) entries)->count = cpu_to_le16(value); +} + +static inline void dx_set_limit(struct dx_entry *entries, unsigned value) +{ + ((struct dx_countlimit *) entries)->limit = cpu_to_le16(value); +} + +static inline unsigned dx_root_limit(struct inode *dir, unsigned infosize) +{ + unsigned int entry_space = dir->i_sb->s_blocksize - + ext4_dir_rec_len(1, NULL) - + ext4_dir_rec_len(2, NULL) - infosize; + + if (ext4_has_feature_metadata_csum(dir->i_sb)) + entry_space -= sizeof(struct dx_tail); + return entry_space / sizeof(struct dx_entry); +} + +static inline unsigned dx_node_limit(struct inode *dir) +{ + unsigned int entry_space = dir->i_sb->s_blocksize - + ext4_dir_rec_len(0, dir); + + if (ext4_has_feature_metadata_csum(dir->i_sb)) + entry_space -= sizeof(struct dx_tail); + return entry_space / sizeof(struct dx_entry); +} + +/* + * Debug + */ +#ifdef DX_DEBUG +static void dx_show_index(char * label, struct dx_entry *entries) +{ + int i, n = dx_get_count (entries); + printk(KERN_DEBUG "%s index", label); + for (i = 0; i < n; i++) { + printk(KERN_CONT " %x->%lu", + i ? dx_get_hash(entries + i) : 0, + (unsigned long)dx_get_block(entries + i)); + } + printk(KERN_CONT "\n"); +} + +struct stats +{ + unsigned names; + unsigned space; + unsigned bcount; +}; + +static struct stats dx_show_leaf(struct inode *dir, + struct dx_hash_info *hinfo, + struct ext4_dir_entry_2 *de, + int size, int show_names) +{ + unsigned names = 0, space = 0; + char *base = (char *) de; + struct dx_hash_info h = *hinfo; + + printk("names: "); + while ((char *) de < base + size) + { + if (de->inode) + { + if (show_names) + { +#ifdef CONFIG_FS_ENCRYPTION + int len; + char *name; + struct fscrypt_str fname_crypto_str = + FSTR_INIT(NULL, 0); + int res = 0; + + name = de->name; + len = de->name_len; + if (!IS_ENCRYPTED(dir)) { + /* Directory is not encrypted */ + (void) ext4fs_dirhash(dir, de->name, + de->name_len, &h); + printk("%.*s:(U)%x.%u ", len, + name, h.hash, + (unsigned) ((char *) de + - base)); + } else { + struct fscrypt_str de_name = + FSTR_INIT(name, len); + + /* Directory is encrypted */ + res = fscrypt_fname_alloc_buffer( + len, &fname_crypto_str); + if (res) + printk(KERN_WARNING "Error " + "allocating crypto " + "buffer--skipping " + "crypto\n"); + res = fscrypt_fname_disk_to_usr(dir, + 0, 0, &de_name, + &fname_crypto_str); + if (res) { + printk(KERN_WARNING "Error " + "converting filename " + "from disk to usr" + "\n"); + name = "??"; + len = 2; + } else { + name = fname_crypto_str.name; + len = fname_crypto_str.len; + } + if (IS_CASEFOLDED(dir)) + h.hash = EXT4_DIRENT_HASH(de); + else + (void) ext4fs_dirhash(dir, + de->name, + de->name_len, &h); + printk("%.*s:(E)%x.%u ", len, name, + h.hash, (unsigned) ((char *) de + - base)); + fscrypt_fname_free_buffer( + &fname_crypto_str); + } +#else + int len = de->name_len; + char *name = de->name; + (void) ext4fs_dirhash(dir, de->name, + de->name_len, &h); + printk("%.*s:%x.%u ", len, name, h.hash, + (unsigned) ((char *) de - base)); +#endif + } + space += ext4_dir_rec_len(de->name_len, dir); + names++; + } + de = ext4_next_entry(de, size); + } + printk(KERN_CONT "(%i)\n", names); + return (struct stats) { names, space, 1 }; +} + +struct stats dx_show_entries(struct dx_hash_info *hinfo, struct inode *dir, + struct dx_entry *entries, int levels) +{ + unsigned blocksize = dir->i_sb->s_blocksize; + unsigned count = dx_get_count(entries), names = 0, space = 0, i; + unsigned bcount = 0; + struct buffer_head *bh; + printk("%i indexed blocks...\n", count); + for (i = 0; i < count; i++, entries++) + { + ext4_lblk_t block = dx_get_block(entries); + ext4_lblk_t hash = i ? dx_get_hash(entries): 0; + u32 range = i < count - 1? (dx_get_hash(entries + 1) - hash): ~hash; + struct stats stats; + printk("%s%3u:%03u hash %8x/%8x ",levels?"":" ", i, block, hash, range); + bh = ext4_bread(NULL,dir, block, 0); + if (IS_ERR_OR_NULL(bh)) + continue; + stats = levels? + dx_show_entries(hinfo, dir, ((struct dx_node *) bh->b_data)->entries, levels - 1): + dx_show_leaf(dir, hinfo, (struct ext4_dir_entry_2 *) + bh->b_data, blocksize, 0); + names += stats.names; + space += stats.space; + bcount += stats.bcount; + brelse(bh); + } + if (bcount) + printk(KERN_DEBUG "%snames %u, fullness %u (%u%%)\n", + levels ? "" : " ", names, space/bcount, + (space/bcount)*100/blocksize); + return (struct stats) { names, space, bcount}; +} + +/* + * Linear search cross check + */ +static inline void htree_rep_invariant_check(struct dx_entry *at, + struct dx_entry *target, + u32 hash, unsigned int n) +{ + while (n--) { + dxtrace(printk(KERN_CONT ",")); + if (dx_get_hash(++at) > hash) { + at--; + break; + } + } + ASSERT(at == target - 1); +} +#else /* DX_DEBUG */ +static inline void htree_rep_invariant_check(struct dx_entry *at, + struct dx_entry *target, + u32 hash, unsigned int n) +{ +} +#endif /* DX_DEBUG */ + +/* + * Probe for a directory leaf block to search. + * + * dx_probe can return ERR_BAD_DX_DIR, which means there was a format + * error in the directory index, and the caller should fall back to + * searching the directory normally. The callers of dx_probe **MUST** + * check for this error code, and make sure it never gets reflected + * back to userspace. + */ +static struct dx_frame * +dx_probe(struct ext4_filename *fname, struct inode *dir, + struct dx_hash_info *hinfo, struct dx_frame *frame_in) +{ + unsigned count, indirect, level, i; + struct dx_entry *at, *entries, *p, *q, *m; + struct dx_root *root; + struct dx_frame *frame = frame_in; + struct dx_frame *ret_err = ERR_PTR(ERR_BAD_DX_DIR); + u32 hash; + ext4_lblk_t block; + ext4_lblk_t blocks[EXT4_HTREE_LEVEL]; + + memset(frame_in, 0, EXT4_HTREE_LEVEL * sizeof(frame_in[0])); + frame->bh = ext4_read_dirblock(dir, 0, INDEX); + if (IS_ERR(frame->bh)) + return (struct dx_frame *) frame->bh; + + root = (struct dx_root *) frame->bh->b_data; + if (root->info.hash_version != DX_HASH_TEA && + root->info.hash_version != DX_HASH_HALF_MD4 && + root->info.hash_version != DX_HASH_LEGACY && + root->info.hash_version != DX_HASH_SIPHASH) { + ext4_warning_inode(dir, "Unrecognised inode hash code %u", + root->info.hash_version); + goto fail; + } + if (ext4_hash_in_dirent(dir)) { + if (root->info.hash_version != DX_HASH_SIPHASH) { + ext4_warning_inode(dir, + "Hash in dirent, but hash is not SIPHASH"); + goto fail; + } + } else { + if (root->info.hash_version == DX_HASH_SIPHASH) { + ext4_warning_inode(dir, + "Hash code is SIPHASH, but hash not in dirent"); + goto fail; + } + } + if (fname) + hinfo = &fname->hinfo; + hinfo->hash_version = root->info.hash_version; + if (hinfo->hash_version <= DX_HASH_TEA) + hinfo->hash_version += EXT4_SB(dir->i_sb)->s_hash_unsigned; + hinfo->seed = EXT4_SB(dir->i_sb)->s_hash_seed; + /* hash is already computed for encrypted casefolded directory */ + if (fname && fname_name(fname) && + !(IS_ENCRYPTED(dir) && IS_CASEFOLDED(dir))) { + int ret = ext4fs_dirhash(dir, fname_name(fname), + fname_len(fname), hinfo); + if (ret < 0) { + ret_err = ERR_PTR(ret); + goto fail; + } + } + hash = hinfo->hash; + + if (root->info.unused_flags & 1) { + ext4_warning_inode(dir, "Unimplemented hash flags: %#06x", + root->info.unused_flags); + goto fail; + } + + indirect = root->info.indirect_levels; + if (indirect >= ext4_dir_htree_level(dir->i_sb)) { + ext4_warning(dir->i_sb, + "Directory (ino: %llu) htree depth %#06x exceed" + "supported value", dir->i_ino, + ext4_dir_htree_level(dir->i_sb)); + if (ext4_dir_htree_level(dir->i_sb) < EXT4_HTREE_LEVEL) { + ext4_warning(dir->i_sb, "Enable large directory " + "feature to access it"); + } + goto fail; + } + + entries = (struct dx_entry *)(((char *)&root->info) + + root->info.info_length); + + if (dx_get_limit(entries) != dx_root_limit(dir, + root->info.info_length)) { + ext4_warning_inode(dir, "dx entry: limit %u != root limit %u", + dx_get_limit(entries), + dx_root_limit(dir, root->info.info_length)); + goto fail; + } + + dxtrace(printk("Look up %x", hash)); + level = 0; + blocks[0] = 0; + while (1) { + count = dx_get_count(entries); + if (!count || count > dx_get_limit(entries)) { + ext4_warning_inode(dir, + "dx entry: count %u beyond limit %u", + count, dx_get_limit(entries)); + goto fail; + } + + p = entries + 1; + q = entries + count - 1; + while (p <= q) { + m = p + (q - p) / 2; + dxtrace(printk(KERN_CONT ".")); + if (dx_get_hash(m) > hash) + q = m - 1; + else + p = m + 1; + } + + htree_rep_invariant_check(entries, p, hash, count - 1); + + at = p - 1; + dxtrace(printk(KERN_CONT " %x->%u\n", + at == entries ? 0 : dx_get_hash(at), + dx_get_block(at))); + frame->entries = entries; + frame->at = at; + + block = dx_get_block(at); + for (i = 0; i <= level; i++) { + if (blocks[i] == block) { + ext4_warning_inode(dir, + "dx entry: tree cycle block %u points back to block %u", + blocks[level], block); + goto fail; + } + } + if (++level > indirect) + return frame; + blocks[level] = block; + frame++; + frame->bh = ext4_read_dirblock(dir, block, INDEX); + if (IS_ERR(frame->bh)) { + ret_err = (struct dx_frame *) frame->bh; + frame->bh = NULL; + goto fail; + } + + entries = ((struct dx_node *) frame->bh->b_data)->entries; + + if (dx_get_limit(entries) != dx_node_limit(dir)) { + ext4_warning_inode(dir, + "dx entry: limit %u != node limit %u", + dx_get_limit(entries), dx_node_limit(dir)); + goto fail; + } + } +fail: + while (frame >= frame_in) { + brelse(frame->bh); + frame--; + } + + if (ret_err == ERR_PTR(ERR_BAD_DX_DIR)) + ext4_warning_inode(dir, + "Corrupt directory, running e2fsck is recommended"); + return ret_err; +} + +static void dx_release(struct dx_frame *frames) +{ + struct dx_root_info *info; + int i; + unsigned int indirect_levels; + + if (frames[0].bh == NULL) + return; + + info = &((struct dx_root *)frames[0].bh->b_data)->info; + /* save local copy, "info" may be freed after brelse() */ + indirect_levels = info->indirect_levels; + for (i = 0; i <= indirect_levels; i++) { + if (frames[i].bh == NULL) + break; + brelse(frames[i].bh); + frames[i].bh = NULL; + } +} + +/* + * This function increments the frame pointer to search the next leaf + * block, and reads in the necessary intervening nodes if the search + * should be necessary. Whether or not the search is necessary is + * controlled by the hash parameter. If the hash value is even, then + * the search is only continued if the next block starts with that + * hash value. This is used if we are searching for a specific file. + * + * If the hash value is HASH_NB_ALWAYS, then always go to the next block. + * + * This function returns 1 if the caller should continue to search, + * or 0 if it should not. If there is an error reading one of the + * index blocks, it will a negative error code. + * + * If start_hash is non-null, it will be filled in with the starting + * hash of the next page. + */ +static int ext4_htree_next_block(struct inode *dir, __u32 hash, + struct dx_frame *frame, + struct dx_frame *frames, + __u32 *start_hash) +{ + struct dx_frame *p; + struct buffer_head *bh; + int num_frames = 0; + __u32 bhash; + + p = frame; + /* + * Find the next leaf page by incrementing the frame pointer. + * If we run out of entries in the interior node, loop around and + * increment pointer in the parent node. When we break out of + * this loop, num_frames indicates the number of interior + * nodes need to be read. + */ + while (1) { + if (++(p->at) < p->entries + dx_get_count(p->entries)) + break; + if (p == frames) + return 0; + num_frames++; + p--; + } + + /* + * If the hash is 1, then continue only if the next page has a + * continuation hash of any value. This is used for readdir + * handling. Otherwise, check to see if the hash matches the + * desired continuation hash. If it doesn't, return since + * there's no point to read in the successive index pages. + */ + bhash = dx_get_hash(p->at); + if (start_hash) + *start_hash = bhash; + if ((hash & 1) == 0) { + if ((bhash & ~1) != hash) + return 0; + } + /* + * If the hash is HASH_NB_ALWAYS, we always go to the next + * block so no check is necessary + */ + while (num_frames--) { + bh = ext4_read_dirblock(dir, dx_get_block(p->at), INDEX); + if (IS_ERR(bh)) + return PTR_ERR(bh); + p++; + brelse(p->bh); + p->bh = bh; + p->at = p->entries = ((struct dx_node *) bh->b_data)->entries; + } + return 1; +} + + +/* + * This function fills a red-black tree with information from a + * directory block. It returns the number directory entries loaded + * into the tree. If there is an error it is returned in err. + */ +static int htree_dirblock_to_tree(struct file *dir_file, + struct inode *dir, ext4_lblk_t block, + struct dx_hash_info *hinfo, + __u32 start_hash, __u32 start_minor_hash) +{ + struct buffer_head *bh; + struct ext4_dir_entry_2 *de, *top; + int err = 0, count = 0; + struct fscrypt_str fname_crypto_str = FSTR_INIT(NULL, 0), tmp_str; + int csum = ext4_has_feature_metadata_csum(dir->i_sb); + + dxtrace(printk(KERN_INFO "In htree dirblock_to_tree: block %lu\n", + (unsigned long)block)); + bh = ext4_read_dirblock(dir, block, DIRENT_HTREE); + if (IS_ERR(bh)) + return PTR_ERR(bh); + + de = (struct ext4_dir_entry_2 *) bh->b_data; + /* csum entries are not larger in the casefolded encrypted case */ + top = (struct ext4_dir_entry_2 *) ((char *) de + + dir->i_sb->s_blocksize - + ext4_dir_rec_len(0, + csum ? NULL : dir)); + /* Check if the directory is encrypted */ + if (IS_ENCRYPTED(dir)) { + err = fscrypt_prepare_readdir(dir); + if (err < 0) { + brelse(bh); + return err; + } + err = fscrypt_fname_alloc_buffer(EXT4_NAME_LEN, + &fname_crypto_str); + if (err < 0) { + brelse(bh); + return err; + } + } + + for (; de < top; de = ext4_next_entry(de, dir->i_sb->s_blocksize)) { + if (ext4_check_dir_entry(dir, NULL, de, bh, + bh->b_data, bh->b_size, + EXT4_LBLK_TO_B(dir, block) + + ((char *)de - bh->b_data))) { + /* silently ignore the rest of the block */ + break; + } + if (ext4_hash_in_dirent(dir)) { + if (de->name_len && de->inode) { + hinfo->hash = EXT4_DIRENT_HASH(de); + hinfo->minor_hash = EXT4_DIRENT_MINOR_HASH(de); + } else { + hinfo->hash = 0; + hinfo->minor_hash = 0; + } + } else { + err = ext4fs_dirhash(dir, de->name, + de->name_len, hinfo); + if (err < 0) { + count = err; + goto errout; + } + } + if ((hinfo->hash < start_hash) || + ((hinfo->hash == start_hash) && + (hinfo->minor_hash < start_minor_hash))) + continue; + if (de->inode == 0) + continue; + if (!IS_ENCRYPTED(dir)) { + tmp_str.name = de->name; + tmp_str.len = de->name_len; + err = ext4_htree_store_dirent(dir_file, + hinfo->hash, hinfo->minor_hash, de, + &tmp_str); + } else { + int save_len = fname_crypto_str.len; + struct fscrypt_str de_name = FSTR_INIT(de->name, + de->name_len); + + /* Directory is encrypted */ + err = fscrypt_fname_disk_to_usr(dir, hinfo->hash, + hinfo->minor_hash, &de_name, + &fname_crypto_str); + if (err) { + count = err; + goto errout; + } + err = ext4_htree_store_dirent(dir_file, + hinfo->hash, hinfo->minor_hash, de, + &fname_crypto_str); + fname_crypto_str.len = save_len; + } + if (err != 0) { + count = err; + goto errout; + } + count++; + } +errout: + brelse(bh); + fscrypt_fname_free_buffer(&fname_crypto_str); + return count; +} + + +/* + * This function fills a red-black tree with information from a + * directory. We start scanning the directory in hash order, starting + * at start_hash and start_minor_hash. + * + * This function returns the number of entries inserted into the tree, + * or a negative error code. + */ +int ext4_htree_fill_tree(struct file *dir_file, __u32 start_hash, + __u32 start_minor_hash, __u32 *next_hash) +{ + struct dx_hash_info hinfo; + struct ext4_dir_entry_2 *de; + struct dx_frame frames[EXT4_HTREE_LEVEL], *frame; + struct inode *dir; + ext4_lblk_t block; + int count = 0; + int ret, err; + __u32 hashval; + struct fscrypt_str tmp_str; + + dxtrace(printk(KERN_DEBUG "In htree_fill_tree, start hash: %x:%x\n", + start_hash, start_minor_hash)); + dir = file_inode(dir_file); + if (!(ext4_test_inode_flag(dir, EXT4_INODE_INDEX))) { + if (ext4_hash_in_dirent(dir)) + hinfo.hash_version = DX_HASH_SIPHASH; + else + hinfo.hash_version = + EXT4_SB(dir->i_sb)->s_def_hash_version; + if (hinfo.hash_version <= DX_HASH_TEA) + hinfo.hash_version += + EXT4_SB(dir->i_sb)->s_hash_unsigned; + hinfo.seed = EXT4_SB(dir->i_sb)->s_hash_seed; + if (ext4_has_inline_data(dir)) { + int has_inline_data = 1; + count = ext4_inlinedir_to_tree(dir_file, dir, 0, + &hinfo, start_hash, + start_minor_hash, + &has_inline_data); + if (has_inline_data) { + *next_hash = ~0; + return count; + } + } + count = htree_dirblock_to_tree(dir_file, dir, 0, &hinfo, + start_hash, start_minor_hash); + *next_hash = ~0; + return count; + } + hinfo.hash = start_hash; + hinfo.minor_hash = 0; + frame = dx_probe(NULL, dir, &hinfo, frames); + if (IS_ERR(frame)) + return PTR_ERR(frame); + + /* Add '.' and '..' from the htree header */ + if (!start_hash && !start_minor_hash) { + de = (struct ext4_dir_entry_2 *) frames[0].bh->b_data; + tmp_str.name = de->name; + tmp_str.len = de->name_len; + err = ext4_htree_store_dirent(dir_file, 0, 0, + de, &tmp_str); + if (err != 0) + goto errout; + count++; + } + if (start_hash < 2 || (start_hash ==2 && start_minor_hash==0)) { + de = (struct ext4_dir_entry_2 *) frames[0].bh->b_data; + de = ext4_next_entry(de, dir->i_sb->s_blocksize); + tmp_str.name = de->name; + tmp_str.len = de->name_len; + err = ext4_htree_store_dirent(dir_file, 2, 0, + de, &tmp_str); + if (err != 0) + goto errout; + count++; + } + + while (1) { + if (fatal_signal_pending(current)) { + err = -ERESTARTSYS; + goto errout; + } + cond_resched(); + block = dx_get_block(frame->at); + ret = htree_dirblock_to_tree(dir_file, dir, block, &hinfo, + start_hash, start_minor_hash); + if (ret < 0) { + err = ret; + goto errout; + } + count += ret; + hashval = ~0; + ret = ext4_htree_next_block(dir, HASH_NB_ALWAYS, + frame, frames, &hashval); + *next_hash = hashval; + if (ret < 0) { + err = ret; + goto errout; + } + /* + * Stop if: (a) there are no more entries, or + * (b) we have inserted at least one entry and the + * next hash value is not a continuation + */ + if ((ret == 0) || + (count && ((hashval & 1) == 0))) + break; + } + dx_release(frames); + dxtrace(printk(KERN_DEBUG "Fill tree: returned %d entries, " + "next hash: %x\n", count, *next_hash)); + return count; +errout: + dx_release(frames); + return (err); +} + +static inline int search_dirblock(struct buffer_head *bh, + struct inode *dir, + struct ext4_filename *fname, + unsigned int offset, + struct ext4_dir_entry_2 **res_dir) +{ + return ext4_search_dir(bh, bh->b_data, dir->i_sb->s_blocksize, dir, + fname, offset, res_dir); +} + +/* + * Directory block splitting, compacting + */ + +/* + * Create map of hash values, offsets, and sizes, stored at end of block. + * Returns number of entries mapped. + */ +static int dx_make_map(struct inode *dir, struct buffer_head *bh, + struct dx_hash_info *hinfo, + struct dx_map_entry *map_tail) +{ + int count = 0; + struct ext4_dir_entry_2 *de = (struct ext4_dir_entry_2 *)bh->b_data; + unsigned int buflen = bh->b_size; + char *base = bh->b_data; + struct dx_hash_info h = *hinfo; + int blocksize = EXT4_BLOCK_SIZE(dir->i_sb); + + if (ext4_has_feature_metadata_csum(dir->i_sb)) + buflen -= sizeof(struct ext4_dir_entry_tail); + + while ((char *) de < base + buflen) { + if (ext4_check_dir_entry(dir, NULL, de, bh, base, buflen, + ((char *)de) - base)) + return -EFSCORRUPTED; + if (de->name_len && de->inode) { + if (ext4_hash_in_dirent(dir)) + h.hash = EXT4_DIRENT_HASH(de); + else { + int err = ext4fs_dirhash(dir, de->name, + de->name_len, &h); + if (err < 0) + return err; + } + map_tail--; + map_tail->hash = h.hash; + map_tail->offs = ((char *) de - base)>>2; + map_tail->size = ext4_rec_len_from_disk(de->rec_len, + blocksize); + count++; + cond_resched(); + } + de = ext4_next_entry(de, blocksize); + } + return count; +} + +/* Sort map by hash value */ +static void dx_sort_map (struct dx_map_entry *map, unsigned count) +{ + struct dx_map_entry *p, *q, *top = map + count - 1; + int more; + /* Combsort until bubble sort doesn't suck */ + while (count > 2) { + count = count*10/13; + if (count - 9 < 2) /* 9, 10 -> 11 */ + count = 11; + for (p = top, q = p - count; q >= map; p--, q--) + if (p->hash < q->hash) + swap(*p, *q); + } + /* Garden variety bubble sort */ + do { + more = 0; + q = top; + while (q-- > map) { + if (q[1].hash >= q[0].hash) + continue; + swap(*(q+1), *q); + more = 1; + } + } while(more); +} + +static void dx_insert_block(struct dx_frame *frame, u32 hash, ext4_lblk_t block) +{ + struct dx_entry *entries = frame->entries; + struct dx_entry *old = frame->at, *new = old + 1; + int count = dx_get_count(entries); + + ASSERT(count < dx_get_limit(entries)); + ASSERT(old < entries + count); + memmove(new + 1, new, (char *)(entries + count) - (char *)(new)); + dx_set_hash(new, hash); + dx_set_block(new, block); + dx_set_count(entries, count + 1); +} + +#if IS_ENABLED(CONFIG_UNICODE) +int ext4_fname_setup_ci_filename(struct inode *dir, const struct qstr *iname, + struct ext4_filename *name) +{ + struct qstr *cf_name = &name->cf_name; + unsigned char *buf; + struct dx_hash_info *hinfo = &name->hinfo; + int len; + + if (!IS_CASEFOLDED(dir) || + (IS_ENCRYPTED(dir) && !fscrypt_has_encryption_key(dir))) { + cf_name->name = NULL; + return 0; + } + + buf = kmalloc(EXT4_NAME_LEN, GFP_NOFS); + if (!buf) + return -ENOMEM; + + len = utf8_casefold(dir->i_sb->s_encoding, iname, buf, EXT4_NAME_LEN); + if (len <= 0) { + kfree(buf); + buf = NULL; + } + cf_name->name = buf; + cf_name->len = (unsigned) len; + + if (!IS_ENCRYPTED(dir)) + return 0; + + hinfo->hash_version = DX_HASH_SIPHASH; + hinfo->seed = NULL; + if (cf_name->name) + return ext4fs_dirhash(dir, cf_name->name, cf_name->len, hinfo); + else + return ext4fs_dirhash(dir, iname->name, iname->len, hinfo); +} +#endif + +/* + * Test whether a directory entry matches the filename being searched for. + * + * Return: %true if the directory entry matches, otherwise %false. + */ +static bool ext4_match(struct inode *parent, + const struct ext4_filename *fname, + struct ext4_dir_entry_2 *de) +{ + struct fscrypt_name f; + + if (!de->inode) + return false; + + f.usr_fname = fname->usr_fname; + f.disk_name = fname->disk_name; +#ifdef CONFIG_FS_ENCRYPTION + f.crypto_buf = fname->crypto_buf; +#endif + +#if IS_ENABLED(CONFIG_UNICODE) + if (IS_CASEFOLDED(parent) && + (!IS_ENCRYPTED(parent) || fscrypt_has_encryption_key(parent))) { + /* + * Just checking IS_ENCRYPTED(parent) below is not + * sufficient to decide whether one can use the hash for + * skipping the string comparison, because the key might + * have been added right after + * ext4_fname_setup_ci_filename(). In this case, a hash + * mismatch will be a false negative. Therefore, make + * sure cf_name was properly initialized before + * considering the calculated hash. + */ + if (sb_no_casefold_compat_fallback(parent->i_sb) && + IS_ENCRYPTED(parent) && fname->cf_name.name && + (fname->hinfo.hash != EXT4_DIRENT_HASH(de) || + fname->hinfo.minor_hash != EXT4_DIRENT_MINOR_HASH(de))) + return false; + /* + * Treat comparison errors as not a match. The + * only case where it happens is on a disk + * corruption or ENOMEM. + */ + + return generic_ci_match(parent, fname->usr_fname, + &fname->cf_name, de->name, + de->name_len) > 0; + } +#endif + + return fscrypt_match_name(&f, de->name, de->name_len); +} + +/* + * Returns 0 if not found, -EFSCORRUPTED on failure, and 1 on success + */ +int ext4_search_dir(struct buffer_head *bh, char *search_buf, int buf_size, + struct inode *dir, struct ext4_filename *fname, + unsigned int offset, struct ext4_dir_entry_2 **res_dir) +{ + struct ext4_dir_entry_2 * de; + char * dlimit; + int de_len; + + de = (struct ext4_dir_entry_2 *)search_buf; + dlimit = search_buf + buf_size; + while ((char *) de < dlimit - EXT4_BASE_DIR_LEN) { + /* this code is executed quadratically often */ + /* do minimal checking `by hand' */ + if (de->name + de->name_len <= dlimit && + (!ext4_hash_in_dirent(dir) || + (char *)de + ext4_dir_rec_len(de->name_len, dir) <= dlimit) && + ext4_match(dir, fname, de)) { + /* found a match - just to be sure, do + * a full check */ + if (ext4_check_dir_entry(dir, NULL, de, bh, search_buf, + buf_size, offset)) + return -EFSCORRUPTED; + *res_dir = de; + return 1; + } + /* prevent looping on a bad block */ + de_len = ext4_rec_len_from_disk(de->rec_len, + dir->i_sb->s_blocksize); + if (de_len <= 0) + return -EFSCORRUPTED; + offset += de_len; + de = (struct ext4_dir_entry_2 *) ((char *) de + de_len); + } + return 0; +} + +static int is_dx_internal_node(struct inode *dir, ext4_lblk_t block, + struct ext4_dir_entry *de) +{ + struct super_block *sb = dir->i_sb; + + if (!is_dx(dir)) + return 0; + if (block == 0) + return 1; + if (de->inode == 0 && + ext4_rec_len_from_disk(de->rec_len, sb->s_blocksize) == + sb->s_blocksize) + return 1; + return 0; +} + +/* + * __ext4_find_entry() + * + * finds an entry in the specified directory with the wanted name. It + * returns the cache buffer in which the entry was found, and the entry + * itself (as a parameter - res_dir). It does NOT read the inode of the + * entry - you'll have to do that yourself if you want to. + * + * The returned buffer_head has ->b_count elevated. The caller is expected + * to brelse() it when appropriate. + */ +static struct buffer_head *__ext4_find_entry(struct inode *dir, + struct ext4_filename *fname, + struct ext4_dir_entry_2 **res_dir, + int *inlined) +{ + struct super_block *sb; + struct buffer_head *bh_use[NAMEI_RA_SIZE]; + struct buffer_head *bh, *ret = NULL; + ext4_lblk_t start, block; + const u8 *name = fname->usr_fname->name; + size_t ra_max = 0; /* Number of bh's in the readahead + buffer, bh_use[] */ + size_t ra_ptr = 0; /* Current index into readahead + buffer */ + ext4_lblk_t nblocks; + int i, namelen, retval; + + *res_dir = NULL; + sb = dir->i_sb; + namelen = fname->usr_fname->len; + if (namelen > EXT4_NAME_LEN) + return NULL; + + if (ext4_has_inline_data(dir)) { + int has_inline_data = 1; + ret = ext4_find_inline_entry(dir, fname, res_dir, + &has_inline_data); + if (inlined) + *inlined = has_inline_data; + if (has_inline_data || IS_ERR(ret)) + goto cleanup_and_exit; + } + + if ((namelen <= 2) && (name[0] == '.') && + (name[1] == '.' || name[1] == '\0')) { + /* + * "." or ".." will only be in the first block + * NFS may look up ".."; "." should be handled by the VFS + */ + block = start = 0; + nblocks = 1; + goto restart; + } + if (is_dx(dir)) { + ret = ext4_dx_find_entry(dir, fname, res_dir); + /* + * On success, or if the error was file not found, + * return. Otherwise, fall back to doing a search the + * old fashioned way. + */ + if (IS_ERR(ret) && PTR_ERR(ret) == ERR_BAD_DX_DIR) + dxtrace(printk(KERN_DEBUG "ext4_find_entry: dx failed, " + "falling back\n")); + else if (!sb_no_casefold_compat_fallback(dir->i_sb) && + *res_dir == NULL && IS_CASEFOLDED(dir)) + dxtrace(printk(KERN_DEBUG "ext4_find_entry: casefold " + "failed, falling back\n")); + else + goto cleanup_and_exit; + ret = NULL; + } + nblocks = dir->i_size >> EXT4_BLOCK_SIZE_BITS(sb); + if (!nblocks) { + ret = NULL; + goto cleanup_and_exit; + } + start = EXT4_I(dir)->i_dir_start_lookup; + if (start >= nblocks) + start = 0; + block = start; +restart: + do { + /* + * We deal with the read-ahead logic here. + */ + cond_resched(); + if (ra_ptr >= ra_max) { + /* Refill the readahead buffer */ + ra_ptr = 0; + if (block < start) + ra_max = start - block; + else + ra_max = nblocks - block; + ra_max = min(ra_max, ARRAY_SIZE(bh_use)); + retval = ext4_bread_batch(dir, block, ra_max, + false /* wait */, bh_use); + if (retval) { + ret = ERR_PTR(retval); + ra_max = 0; + goto cleanup_and_exit; + } + } + if ((bh = bh_use[ra_ptr++]) == NULL) + goto next; + wait_on_buffer(bh); + if (!buffer_uptodate(bh)) { + EXT4_ERROR_INODE_ERR(dir, EIO, + "reading directory lblock %lu", + (unsigned long) block); + brelse(bh); + ret = ERR_PTR(-EIO); + goto cleanup_and_exit; + } + if (!buffer_verified(bh) && + !is_dx_internal_node(dir, block, + (struct ext4_dir_entry *)bh->b_data) && + !ext4_dirblock_csum_verify(dir, bh)) { + EXT4_ERROR_INODE_ERR(dir, EFSBADCRC, + "checksumming directory " + "block %lu", (unsigned long)block); + brelse(bh); + ret = ERR_PTR(-EFSBADCRC); + goto cleanup_and_exit; + } + set_buffer_verified(bh); + i = search_dirblock(bh, dir, fname, + EXT4_LBLK_TO_B(dir, block), res_dir); + if (i == 1) { + EXT4_I(dir)->i_dir_start_lookup = block; + ret = bh; + goto cleanup_and_exit; + } else { + brelse(bh); + if (i < 0) { + ret = ERR_PTR(i); + goto cleanup_and_exit; + } + } + next: + if (++block >= nblocks) + block = 0; + } while (block != start); + + /* + * If the directory has grown while we were searching, then + * search the last part of the directory before giving up. + */ + block = nblocks; + nblocks = dir->i_size >> EXT4_BLOCK_SIZE_BITS(sb); + if (block < nblocks) { + start = 0; + goto restart; + } + +cleanup_and_exit: + /* Clean up the read-ahead blocks */ + for (; ra_ptr < ra_max; ra_ptr++) + brelse(bh_use[ra_ptr]); + return ret; +} + +static struct buffer_head *ext4_find_entry(struct inode *dir, + const struct qstr *d_name, + struct ext4_dir_entry_2 **res_dir, + int *inlined) +{ + int err; + struct ext4_filename fname; + struct buffer_head *bh; + + err = ext4_fname_setup_filename(dir, d_name, 1, &fname); + if (err == -ENOENT) + return NULL; + if (err) + return ERR_PTR(err); + + bh = __ext4_find_entry(dir, &fname, res_dir, inlined); + + ext4_fname_free_filename(&fname); + return bh; +} + +static struct buffer_head *ext4_lookup_entry(struct inode *dir, + struct dentry *dentry, + struct ext4_dir_entry_2 **res_dir) +{ + int err; + struct ext4_filename fname; + struct buffer_head *bh; + + err = ext4_fname_prepare_lookup(dir, dentry, &fname); + if (err == -ENOENT) + return NULL; + if (err) + return ERR_PTR(err); + + bh = __ext4_find_entry(dir, &fname, res_dir, NULL); + + ext4_fname_free_filename(&fname); + return bh; +} + +static struct buffer_head * ext4_dx_find_entry(struct inode *dir, + struct ext4_filename *fname, + struct ext4_dir_entry_2 **res_dir) +{ + struct dx_frame frames[EXT4_HTREE_LEVEL], *frame; + struct buffer_head *bh; + ext4_lblk_t block; + int retval; + +#ifdef CONFIG_FS_ENCRYPTION + *res_dir = NULL; +#endif + frame = dx_probe(fname, dir, NULL, frames); + if (IS_ERR(frame)) + return ERR_CAST(frame); + do { + block = dx_get_block(frame->at); + bh = ext4_read_dirblock(dir, block, DIRENT_HTREE); + if (IS_ERR(bh)) + goto errout; + + retval = search_dirblock(bh, dir, fname, + EXT4_LBLK_TO_B(dir, block), res_dir); + if (retval == 1) + goto success; + brelse(bh); + if (retval < 0) { + bh = ERR_PTR(ERR_BAD_DX_DIR); + goto errout; + } + + /* Check to see if we should continue to search */ + retval = ext4_htree_next_block(dir, fname->hinfo.hash, frame, + frames, NULL); + if (retval < 0) { + ext4_warning_inode(dir, + "error %d reading directory index block", + retval); + bh = ERR_PTR(retval); + goto errout; + } + } while (retval == 1); + + bh = NULL; +errout: + dxtrace(printk(KERN_DEBUG "%s not found\n", fname->usr_fname->name)); +success: + dx_release(frames); + return bh; +} + +static struct dentry *ext4_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) +{ + struct inode *inode; + struct ext4_dir_entry_2 *de = NULL; + struct buffer_head *bh; + + if (dentry->d_name.len > EXT4_NAME_LEN) + return ERR_PTR(-ENAMETOOLONG); + + bh = ext4_lookup_entry(dir, dentry, &de); + if (IS_ERR(bh)) + return ERR_CAST(bh); + inode = NULL; + if (bh) { + __u32 ino = le32_to_cpu(de->inode); + brelse(bh); + if (!ext4_valid_inum(dir->i_sb, ino)) { + EXT4_ERROR_INODE(dir, "bad inode number: %u", ino); + return ERR_PTR(-EFSCORRUPTED); + } + if (unlikely(ino == dir->i_ino)) { + EXT4_ERROR_INODE(dir, "'%pd' linked to parent dir", + dentry); + return ERR_PTR(-EFSCORRUPTED); + } + inode = ext4_iget(dir->i_sb, ino, EXT4_IGET_NORMAL); + if (inode == ERR_PTR(-ESTALE)) { + EXT4_ERROR_INODE(dir, + "deleted inode referenced: %u", + ino); + return ERR_PTR(-EFSCORRUPTED); + } + if (!IS_ERR(inode) && IS_ENCRYPTED(dir) && + (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode)) && + !fscrypt_has_permitted_context(dir, inode)) { + ext4_warning(inode->i_sb, + "Inconsistent encryption contexts: %llu/%llu", + dir->i_ino, inode->i_ino); + iput(inode); + return ERR_PTR(-EPERM); + } + } + + if (IS_ENABLED(CONFIG_UNICODE) && !inode && IS_CASEFOLDED(dir)) { + /* Eventually we want to call d_add_ci(dentry, NULL) + * for negative dentries in the encoding case as + * well. For now, prevent the negative dentry + * from being cached. + */ + return NULL; + } + + return d_splice_alias(inode, dentry); +} + + +struct dentry *ext4_get_parent(struct dentry *child) +{ + __u32 ino; + struct ext4_dir_entry_2 * de = NULL; + struct buffer_head *bh; + + bh = ext4_find_entry(d_inode(child), &dotdot_name, &de, NULL); + if (IS_ERR(bh)) + return ERR_CAST(bh); + if (!bh) + return ERR_PTR(-ENOENT); + ino = le32_to_cpu(de->inode); + brelse(bh); + + if (!ext4_valid_inum(child->d_sb, ino)) { + EXT4_ERROR_INODE(d_inode(child), + "bad parent inode number: %u", ino); + return ERR_PTR(-EFSCORRUPTED); + } + + return d_obtain_alias(ext4_iget(child->d_sb, ino, EXT4_IGET_NORMAL)); +} + +/* + * Move count entries from end of map between two memory locations. + * Returns pointer to last entry moved. + */ +static struct ext4_dir_entry_2 * +dx_move_dirents(struct inode *dir, char *from, char *to, + struct dx_map_entry *map, int count, + unsigned blocksize) +{ + unsigned rec_len = 0; + + while (count--) { + struct ext4_dir_entry_2 *de = (struct ext4_dir_entry_2 *) + (from + (map->offs<<2)); + rec_len = ext4_dir_rec_len(de->name_len, dir); + + memcpy (to, de, rec_len); + ((struct ext4_dir_entry_2 *) to)->rec_len = + ext4_rec_len_to_disk(rec_len, blocksize); + + /* wipe dir_entry excluding the rec_len field */ + de->inode = 0; + memset(&de->name_len, 0, ext4_rec_len_from_disk(de->rec_len, + blocksize) - + offsetof(struct ext4_dir_entry_2, + name_len)); + + map++; + to += rec_len; + } + return (struct ext4_dir_entry_2 *) (to - rec_len); +} + +/* + * Compact each dir entry in the range to the minimal rec_len. + * Returns pointer to last entry in range. + */ +static struct ext4_dir_entry_2 *dx_pack_dirents(struct inode *dir, char *base, + unsigned int blocksize) +{ + struct ext4_dir_entry_2 *next, *to, *prev, *de = (struct ext4_dir_entry_2 *) base; + unsigned rec_len = 0; + + prev = to = de; + while ((char*)de < base + blocksize) { + next = ext4_next_entry(de, blocksize); + if (de->inode && de->name_len) { + rec_len = ext4_dir_rec_len(de->name_len, dir); + if (de > to) + memmove(to, de, rec_len); + to->rec_len = ext4_rec_len_to_disk(rec_len, blocksize); + prev = to; + to = (struct ext4_dir_entry_2 *) (((char *) to) + rec_len); + } + de = next; + } + return prev; +} + +/* + * Split a full leaf block to make room for a new dir entry. + * Allocate a new block, and move entries so that they are approx. equally full. + * Returns pointer to de in block into which the new entry will be inserted. + */ +static struct ext4_dir_entry_2 *do_split(handle_t *handle, struct inode *dir, + struct buffer_head **bh,struct dx_frame *frame, + struct dx_hash_info *hinfo) +{ + unsigned blocksize = dir->i_sb->s_blocksize; + unsigned continued; + int count; + struct buffer_head *bh2; + ext4_lblk_t newblock; + u32 hash2; + struct dx_map_entry *map; + char *data1 = (*bh)->b_data, *data2; + unsigned split, move, size; + struct ext4_dir_entry_2 *de = NULL, *de2; + int csum_size = 0; + int err = 0, i; + + if (ext4_has_feature_metadata_csum(dir->i_sb)) + csum_size = sizeof(struct ext4_dir_entry_tail); + + bh2 = ext4_append(handle, dir, &newblock); + if (IS_ERR(bh2)) { + brelse(*bh); + *bh = NULL; + return ERR_CAST(bh2); + } + + BUFFER_TRACE(*bh, "get_write_access"); + err = ext4_journal_get_write_access(handle, dir->i_sb, *bh, + EXT4_JTR_NONE); + if (err) + goto journal_error; + + BUFFER_TRACE(frame->bh, "get_write_access"); + err = ext4_journal_get_write_access(handle, dir->i_sb, frame->bh, + EXT4_JTR_NONE); + if (err) + goto journal_error; + + data2 = bh2->b_data; + + /* create map in the end of data2 block */ + map = (struct dx_map_entry *) (data2 + blocksize); + count = dx_make_map(dir, *bh, hinfo, map); + if (count < 0) { + err = count; + goto journal_error; + } + map -= count; + dx_sort_map(map, count); + /* Ensure that neither split block is over half full */ + size = 0; + move = 0; + for (i = count-1; i >= 0; i--) { + /* is more than half of this entry in 2nd half of the block? */ + if (size + map[i].size/2 > blocksize/2) + break; + size += map[i].size; + move++; + } + /* + * map index at which we will split + * + * If the sum of active entries didn't exceed half the block size, just + * split it in half by count; each resulting block will have at least + * half the space free. + */ + if (i >= 0) + split = count - move; + else + split = count/2; + + if (WARN_ON_ONCE(split == 0)) { + /* Should never happen, but avoid out-of-bounds access below */ + ext4_error_inode_block(dir, (*bh)->b_blocknr, 0, + "bad indexed directory? hash=%08x:%08x count=%d move=%u", + hinfo->hash, hinfo->minor_hash, count, move); + err = -EFSCORRUPTED; + goto out; + } + + hash2 = map[split].hash; + continued = hash2 == map[split - 1].hash; + dxtrace(printk(KERN_INFO "Split block %lu at %x, %i/%i\n", + (unsigned long)dx_get_block(frame->at), + hash2, split, count-split)); + + /* Fancy dance to stay within two buffers */ + de2 = dx_move_dirents(dir, data1, data2, map + split, count - split, + blocksize); + de = dx_pack_dirents(dir, data1, blocksize); + de->rec_len = ext4_rec_len_to_disk(data1 + (blocksize - csum_size) - + (char *) de, + blocksize); + de2->rec_len = ext4_rec_len_to_disk(data2 + (blocksize - csum_size) - + (char *) de2, + blocksize); + if (csum_size) { + ext4_initialize_dirent_tail(*bh, blocksize); + ext4_initialize_dirent_tail(bh2, blocksize); + } + + dxtrace(dx_show_leaf(dir, hinfo, (struct ext4_dir_entry_2 *) data1, + blocksize, 1)); + dxtrace(dx_show_leaf(dir, hinfo, (struct ext4_dir_entry_2 *) data2, + blocksize, 1)); + + /* Which block gets the new entry? */ + if (hinfo->hash >= hash2) { + swap(*bh, bh2); + de = de2; + } + dx_insert_block(frame, hash2 + continued, newblock); + err = ext4_handle_dirty_dirblock(handle, dir, bh2); + if (err) + goto journal_error; + err = ext4_handle_dirty_dx_node(handle, dir, frame->bh); + if (err) + goto journal_error; + brelse(bh2); + dxtrace(dx_show_index("frame", frame->entries)); + return de; + +journal_error: + ext4_std_error(dir->i_sb, err); +out: + brelse(*bh); + brelse(bh2); + *bh = NULL; + return ERR_PTR(err); +} + +int ext4_find_dest_de(struct inode *dir, struct buffer_head *bh, + void *buf, int buf_size, + struct ext4_filename *fname, + struct ext4_dir_entry_2 **dest_de) +{ + struct ext4_dir_entry_2 *de; + unsigned short reclen = ext4_dir_rec_len(fname_len(fname), dir); + int nlen, rlen; + unsigned int offset = 0; + char *top; + + de = buf; + top = buf + buf_size - reclen; + while ((char *) de <= top) { + if (ext4_check_dir_entry(dir, NULL, de, bh, + buf, buf_size, offset)) + return -EFSCORRUPTED; + if (ext4_match(dir, fname, de)) + return -EEXIST; + nlen = ext4_dir_rec_len(de->name_len, dir); + rlen = ext4_rec_len_from_disk(de->rec_len, buf_size); + if ((de->inode ? rlen - nlen : rlen) >= reclen) + break; + de = (struct ext4_dir_entry_2 *)((char *)de + rlen); + offset += rlen; + } + if ((char *) de > top) + return -ENOSPC; + + *dest_de = de; + return 0; +} + +void ext4_insert_dentry(struct inode *dir, + struct inode *inode, + struct ext4_dir_entry_2 *de, + int buf_size, + struct ext4_filename *fname) +{ + + int nlen, rlen; + + nlen = ext4_dir_rec_len(de->name_len, dir); + rlen = ext4_rec_len_from_disk(de->rec_len, buf_size); + if (de->inode) { + struct ext4_dir_entry_2 *de1 = + (struct ext4_dir_entry_2 *)((char *)de + nlen); + de1->rec_len = ext4_rec_len_to_disk(rlen - nlen, buf_size); + de->rec_len = ext4_rec_len_to_disk(nlen, buf_size); + de = de1; + } + de->file_type = EXT4_FT_UNKNOWN; + de->inode = cpu_to_le32(inode->i_ino); + ext4_set_de_type(inode->i_sb, de, inode->i_mode); + de->name_len = fname_len(fname); + memcpy(de->name, fname_name(fname), fname_len(fname)); + if (ext4_hash_in_dirent(dir)) { + struct dx_hash_info *hinfo = &fname->hinfo; + + EXT4_DIRENT_HASHES(de)->hash = cpu_to_le32(hinfo->hash); + EXT4_DIRENT_HASHES(de)->minor_hash = + cpu_to_le32(hinfo->minor_hash); + } +} + +/* + * Add a new entry into a directory (leaf) block. If de is non-NULL, + * it points to a directory entry which is guaranteed to be large + * enough for new directory entry. If de is NULL, then + * add_dirent_to_buf will attempt search the directory block for + * space. It will return -ENOSPC if no space is available, and -EIO + * and -EEXIST if directory entry already exists. + */ +static int add_dirent_to_buf(handle_t *handle, struct ext4_filename *fname, + struct inode *dir, + struct inode *inode, struct ext4_dir_entry_2 *de, + struct buffer_head *bh) +{ + unsigned int blocksize = dir->i_sb->s_blocksize; + int csum_size = 0; + int err, err2; + + if (ext4_has_feature_metadata_csum(inode->i_sb)) + csum_size = sizeof(struct ext4_dir_entry_tail); + + if (!de) { + err = ext4_find_dest_de(dir, bh, bh->b_data, + blocksize - csum_size, fname, &de); + if (err) + return err; + } + BUFFER_TRACE(bh, "get_write_access"); + err = ext4_journal_get_write_access(handle, dir->i_sb, bh, + EXT4_JTR_NONE); + if (err) { + ext4_std_error(dir->i_sb, err); + return err; + } + + /* By now the buffer is marked for journaling */ + ext4_insert_dentry(dir, inode, de, blocksize, fname); + + /* + * XXX shouldn't update any times until successful + * completion of syscall, but too many callers depend + * on this. + * + * XXX similarly, too many callers depend on + * ext4_new_inode() setting the times, but error + * recovery deletes the inode, so the worst that can + * happen is that the times are slightly out of date + * and/or different from the directory change time. + */ + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + ext4_update_dx_flag(dir); + inode_inc_iversion(dir); + err2 = ext4_mark_inode_dirty(handle, dir); + BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata"); + err = ext4_handle_dirty_dirblock(handle, dir, bh); + if (err) + ext4_std_error(dir->i_sb, err); + return err ? err : err2; +} + +static bool ext4_check_dx_root(struct inode *dir, struct dx_root *root) +{ + struct fake_dirent *fde; + const char *error_msg; + unsigned int rlen; + unsigned int blocksize = dir->i_sb->s_blocksize; + char *blockend = (char *)root + dir->i_sb->s_blocksize; + + fde = &root->dot; + if (unlikely(fde->name_len != 1)) { + error_msg = "invalid name_len for '.'"; + goto corrupted; + } + if (unlikely(strncmp(root->dot_name, ".", fde->name_len))) { + error_msg = "invalid name for '.'"; + goto corrupted; + } + rlen = ext4_rec_len_from_disk(fde->rec_len, blocksize); + if (unlikely((char *)fde + rlen >= blockend)) { + error_msg = "invalid rec_len for '.'"; + goto corrupted; + } + + fde = &root->dotdot; + if (unlikely(fde->name_len != 2)) { + error_msg = "invalid name_len for '..'"; + goto corrupted; + } + if (unlikely(strncmp(root->dotdot_name, "..", fde->name_len))) { + error_msg = "invalid name for '..'"; + goto corrupted; + } + rlen = ext4_rec_len_from_disk(fde->rec_len, blocksize); + if (unlikely((char *)fde + rlen >= blockend)) { + error_msg = "invalid rec_len for '..'"; + goto corrupted; + } + + return true; + +corrupted: + EXT4_ERROR_INODE(dir, "Corrupt dir, %s, running e2fsck is recommended", + error_msg); + return false; +} + +/* + * This converts a one block unindexed directory to a 3 block indexed + * directory, and adds the dentry to the indexed directory. + */ +static int make_indexed_dir(handle_t *handle, struct ext4_filename *fname, + struct inode *dir, + struct inode *inode, struct buffer_head *bh) +{ + struct buffer_head *bh2; + struct dx_root *root; + struct dx_frame frames[EXT4_HTREE_LEVEL], *frame; + struct dx_entry *entries; + struct ext4_dir_entry_2 *de, *de2; + char *data2, *top; + unsigned len; + int retval; + unsigned blocksize; + ext4_lblk_t block; + struct fake_dirent *fde; + int csum_size = 0; + + if (ext4_has_feature_metadata_csum(inode->i_sb)) + csum_size = sizeof(struct ext4_dir_entry_tail); + + blocksize = dir->i_sb->s_blocksize; + dxtrace(printk(KERN_DEBUG "Creating index: inode %llu\n", dir->i_ino)); + BUFFER_TRACE(bh, "get_write_access"); + retval = ext4_journal_get_write_access(handle, dir->i_sb, bh, + EXT4_JTR_NONE); + if (retval) { + ext4_std_error(dir->i_sb, retval); + brelse(bh); + return retval; + } + + root = (struct dx_root *) bh->b_data; + if (!ext4_check_dx_root(dir, root)) { + brelse(bh); + return -EFSCORRUPTED; + } + + /* The 0th block becomes the root, move the dirents out */ + fde = &root->dotdot; + de = (struct ext4_dir_entry_2 *)((char *)fde + + ext4_rec_len_from_disk(fde->rec_len, blocksize)); + len = ((char *) root) + (blocksize - csum_size) - (char *) de; + + /* Allocate new block for the 0th block's dirents */ + bh2 = ext4_append(handle, dir, &block); + if (IS_ERR(bh2)) { + brelse(bh); + return PTR_ERR(bh2); + } + ext4_set_inode_flag(dir, EXT4_INODE_INDEX); + data2 = bh2->b_data; + + memcpy(data2, de, len); + memset(de, 0, len); /* wipe old data */ + de = (struct ext4_dir_entry_2 *) data2; + top = data2 + len; + while ((char *)(de2 = ext4_next_entry(de, blocksize)) < top) { + if (ext4_check_dir_entry(dir, NULL, de, bh2, data2, len, + (char *)de - data2)) { + brelse(bh2); + brelse(bh); + return -EFSCORRUPTED; + } + de = de2; + } + de->rec_len = ext4_rec_len_to_disk(data2 + (blocksize - csum_size) - + (char *) de, blocksize); + + if (csum_size) + ext4_initialize_dirent_tail(bh2, blocksize); + + /* Initialize the root; the dot dirents already exist */ + de = (struct ext4_dir_entry_2 *) (&root->dotdot); + de->rec_len = ext4_rec_len_to_disk( + blocksize - ext4_dir_rec_len(2, NULL), blocksize); + memset (&root->info, 0, sizeof(root->info)); + root->info.info_length = sizeof(root->info); + if (ext4_hash_in_dirent(dir)) + root->info.hash_version = DX_HASH_SIPHASH; + else + root->info.hash_version = + EXT4_SB(dir->i_sb)->s_def_hash_version; + + entries = root->entries; + dx_set_block(entries, 1); + dx_set_count(entries, 1); + dx_set_limit(entries, dx_root_limit(dir, sizeof(root->info))); + + /* Initialize as for dx_probe */ + fname->hinfo.hash_version = root->info.hash_version; + if (fname->hinfo.hash_version <= DX_HASH_TEA) + fname->hinfo.hash_version += EXT4_SB(dir->i_sb)->s_hash_unsigned; + fname->hinfo.seed = EXT4_SB(dir->i_sb)->s_hash_seed; + + /* casefolded encrypted hashes are computed on fname setup */ + if (!ext4_hash_in_dirent(dir)) { + int err = ext4fs_dirhash(dir, fname_name(fname), + fname_len(fname), &fname->hinfo); + if (err < 0) { + brelse(bh2); + brelse(bh); + return err; + } + } + memset(frames, 0, sizeof(frames)); + frame = frames; + frame->entries = entries; + frame->at = entries; + frame->bh = bh; + + retval = ext4_handle_dirty_dx_node(handle, dir, frame->bh); + if (retval) + goto out_frames; + retval = ext4_handle_dirty_dirblock(handle, dir, bh2); + if (retval) + goto out_frames; + + de = do_split(handle,dir, &bh2, frame, &fname->hinfo); + if (IS_ERR(de)) { + retval = PTR_ERR(de); + goto out_frames; + } + + retval = add_dirent_to_buf(handle, fname, dir, inode, de, bh2); +out_frames: + /* + * Even if the block split failed, we have to properly write + * out all the changes we did so far. Otherwise we can end up + * with corrupted filesystem. + */ + if (retval) + ext4_mark_inode_dirty(handle, dir); + dx_release(frames); + brelse(bh2); + return retval; +} + +/* + * ext4_add_entry() + * + * adds a file entry to the specified directory, using the same + * semantics as ext4_find_entry(). It returns NULL if it failed. + * + * NOTE!! The inode part of 'de' is left at 0 - which means you + * may not sleep between calling this and putting something into + * the entry, as someone else might have used it while you slept. + */ +static int __ext4_add_entry(handle_t *handle, struct inode *dir, + const struct qstr *d_name, + struct inode *inode) +{ + struct buffer_head *bh = NULL; + struct ext4_dir_entry_2 *de; + struct super_block *sb; + struct ext4_filename fname; + int retval; + int dx_fallback=0; + unsigned blocksize; + ext4_lblk_t block, blocks; + int csum_size = 0; + + if (ext4_has_feature_metadata_csum(inode->i_sb)) + csum_size = sizeof(struct ext4_dir_entry_tail); + + sb = dir->i_sb; + blocksize = sb->s_blocksize; + + if (!generic_ci_validate_strict_name(dir, d_name)) + return -EINVAL; + + retval = ext4_fname_setup_filename(dir, d_name, 0, &fname); + if (retval) + return retval; + + if (ext4_has_inline_data(dir)) { + retval = ext4_try_add_inline_entry(handle, &fname, dir, inode); + if (retval < 0) + goto out; + if (retval == 1) { + retval = 0; + goto out; + } + } + + if (is_dx(dir)) { + retval = ext4_dx_add_entry(handle, &fname, dir, inode); + if (!retval || (retval != ERR_BAD_DX_DIR)) + goto out; + /* Can we just ignore htree data? */ + if (ext4_has_feature_metadata_csum(sb)) { + EXT4_ERROR_INODE(dir, + "Directory has corrupted htree index."); + retval = -EFSCORRUPTED; + goto out; + } + ext4_clear_inode_flag(dir, EXT4_INODE_INDEX); + dx_fallback++; + retval = ext4_mark_inode_dirty(handle, dir); + if (unlikely(retval)) + goto out; + } + blocks = dir->i_size >> sb->s_blocksize_bits; + for (block = 0; block < blocks; block++) { + bh = ext4_read_dirblock(dir, block, DIRENT); + if (bh == NULL) { + bh = ext4_bread(handle, dir, block, + EXT4_GET_BLOCKS_CREATE); + goto add_to_new_block; + } + if (IS_ERR(bh)) { + retval = PTR_ERR(bh); + bh = NULL; + goto out; + } + retval = add_dirent_to_buf(handle, &fname, dir, inode, + NULL, bh); + if (retval != -ENOSPC) + goto out; + + if (blocks == 1 && !dx_fallback && + ext4_has_feature_dir_index(sb)) { + retval = make_indexed_dir(handle, &fname, dir, + inode, bh); + bh = NULL; /* make_indexed_dir releases bh */ + goto out; + } + brelse(bh); + } + bh = ext4_append(handle, dir, &block); +add_to_new_block: + if (IS_ERR(bh)) { + retval = PTR_ERR(bh); + bh = NULL; + goto out; + } + de = (struct ext4_dir_entry_2 *) bh->b_data; + de->inode = 0; + de->rec_len = ext4_rec_len_to_disk(blocksize - csum_size, blocksize); + + if (csum_size) + ext4_initialize_dirent_tail(bh, blocksize); + + retval = add_dirent_to_buf(handle, &fname, dir, inode, de, bh); +out: + ext4_fname_free_filename(&fname); + brelse(bh); + if (retval == 0) + ext4_set_inode_state(inode, EXT4_STATE_NEWENTRY); + return retval; +} + +static int ext4_add_entry(handle_t *handle, struct dentry *dentry, + struct inode *inode) +{ + struct inode *dir = d_inode(dentry->d_parent); + + if (fscrypt_is_nokey_name(dentry)) + return -ENOKEY; + return __ext4_add_entry(handle, dir, &dentry->d_name, inode); +} + +/* + * Returns 0 for success, or a negative error value + */ +static int ext4_dx_add_entry(handle_t *handle, struct ext4_filename *fname, + struct inode *dir, struct inode *inode) +{ + struct dx_frame frames[EXT4_HTREE_LEVEL], *frame; + struct dx_entry *entries, *at; + struct buffer_head *bh; + struct super_block *sb = dir->i_sb; + struct ext4_dir_entry_2 *de; + int restart; + int err; + +again: + restart = 0; + frame = dx_probe(fname, dir, NULL, frames); + if (IS_ERR(frame)) + return PTR_ERR(frame); + entries = frame->entries; + at = frame->at; + bh = ext4_read_dirblock(dir, dx_get_block(frame->at), DIRENT_HTREE); + if (IS_ERR(bh)) { + err = PTR_ERR(bh); + bh = NULL; + goto cleanup; + } + + BUFFER_TRACE(bh, "get_write_access"); + err = ext4_journal_get_write_access(handle, sb, bh, EXT4_JTR_NONE); + if (err) + goto journal_error; + + err = add_dirent_to_buf(handle, fname, dir, inode, NULL, bh); + if (err != -ENOSPC) + goto cleanup; + + err = 0; + /* Block full, should compress but for now just split */ + dxtrace(printk(KERN_DEBUG "using %u of %u node entries\n", + dx_get_count(entries), dx_get_limit(entries))); + /* Need to split index? */ + if (dx_get_count(entries) == dx_get_limit(entries)) { + ext4_lblk_t newblock; + int levels = frame - frames + 1; + unsigned int icount; + int add_level = 1; + struct dx_entry *entries2; + struct dx_node *node2; + struct buffer_head *bh2; + + while (frame > frames) { + if (dx_get_count((frame - 1)->entries) < + dx_get_limit((frame - 1)->entries)) { + add_level = 0; + break; + } + frame--; /* split higher index block */ + at = frame->at; + entries = frame->entries; + restart = 1; + } + if (add_level && levels == ext4_dir_htree_level(sb)) { + ext4_warning(sb, "Directory (ino: %llu) index full, " + "reach max htree level :%d", + dir->i_ino, levels); + if (ext4_dir_htree_level(sb) < EXT4_HTREE_LEVEL) { + ext4_warning(sb, "Large directory feature is " + "not enabled on this " + "filesystem"); + } + err = -ENOSPC; + goto cleanup; + } + icount = dx_get_count(entries); + bh2 = ext4_append(handle, dir, &newblock); + if (IS_ERR(bh2)) { + err = PTR_ERR(bh2); + goto cleanup; + } + node2 = (struct dx_node *)(bh2->b_data); + entries2 = node2->entries; + memset(&node2->fake, 0, sizeof(struct fake_dirent)); + node2->fake.rec_len = ext4_rec_len_to_disk(sb->s_blocksize, + sb->s_blocksize); + BUFFER_TRACE(frame->bh, "get_write_access"); + err = ext4_journal_get_write_access(handle, sb, frame->bh, + EXT4_JTR_NONE); + if (err) { + brelse(bh2); + goto journal_error; + } + if (!add_level) { + unsigned icount1 = icount/2, icount2 = icount - icount1; + unsigned hash2 = dx_get_hash(entries + icount1); + dxtrace(printk(KERN_DEBUG "Split index %i/%i\n", + icount1, icount2)); + + BUFFER_TRACE(frame->bh, "get_write_access"); /* index root */ + err = ext4_journal_get_write_access(handle, sb, + (frame - 1)->bh, + EXT4_JTR_NONE); + if (err) { + brelse(bh2); + goto journal_error; + } + + memcpy((char *) entries2, (char *) (entries + icount1), + icount2 * sizeof(struct dx_entry)); + dx_set_count(entries, icount1); + dx_set_count(entries2, icount2); + dx_set_limit(entries2, dx_node_limit(dir)); + + /* Which index block gets the new entry? */ + if (at - entries >= icount1) { + frame->at = at - entries - icount1 + entries2; + frame->entries = entries = entries2; + swap(frame->bh, bh2); + } + dx_insert_block((frame - 1), hash2, newblock); + dxtrace(dx_show_index("node", frame->entries)); + dxtrace(dx_show_index("node", + ((struct dx_node *) bh2->b_data)->entries)); + err = ext4_handle_dirty_dx_node(handle, dir, bh2); + if (err) { + brelse(bh2); + goto journal_error; + } + brelse (bh2); + err = ext4_handle_dirty_dx_node(handle, dir, + (frame - 1)->bh); + if (err) + goto journal_error; + err = ext4_handle_dirty_dx_node(handle, dir, + frame->bh); + if (restart || err) + goto journal_error; + } else { + struct dx_root *dxroot; + memcpy((char *) entries2, (char *) entries, + icount * sizeof(struct dx_entry)); + dx_set_limit(entries2, dx_node_limit(dir)); + + /* Set up root */ + dx_set_count(entries, 1); + dx_set_block(entries + 0, newblock); + dxroot = (struct dx_root *)frames[0].bh->b_data; + dxroot->info.indirect_levels += 1; + dxtrace(printk(KERN_DEBUG + "Creating %d level index...\n", + dxroot->info.indirect_levels)); + err = ext4_handle_dirty_dx_node(handle, dir, frame->bh); + if (err) { + brelse(bh2); + goto journal_error; + } + err = ext4_handle_dirty_dx_node(handle, dir, bh2); + brelse(bh2); + restart = 1; + goto journal_error; + } + } + de = do_split(handle, dir, &bh, frame, &fname->hinfo); + if (IS_ERR(de)) { + err = PTR_ERR(de); + goto cleanup; + } + err = add_dirent_to_buf(handle, fname, dir, inode, de, bh); + goto cleanup; + +journal_error: + ext4_std_error(dir->i_sb, err); /* this is a no-op if err == 0 */ +cleanup: + brelse(bh); + dx_release(frames); + /* @restart is true means htree-path has been changed, we need to + * repeat dx_probe() to find out valid htree-path + */ + if (restart && err == 0) + goto again; + return err; +} + +/* + * ext4_generic_delete_entry deletes a directory entry by merging it + * with the previous entry + */ +int ext4_generic_delete_entry(struct inode *dir, + struct ext4_dir_entry_2 *de_del, + struct buffer_head *bh, + void *entry_buf, + int buf_size, + int csum_size) +{ + struct ext4_dir_entry_2 *de, *pde; + unsigned int blocksize = dir->i_sb->s_blocksize; + int i; + + i = 0; + pde = NULL; + de = entry_buf; + while (i < buf_size - csum_size) { + if (ext4_check_dir_entry(dir, NULL, de, bh, + entry_buf, buf_size, i)) + return -EFSCORRUPTED; + if (de == de_del) { + if (pde) { + pde->rec_len = ext4_rec_len_to_disk( + ext4_rec_len_from_disk(pde->rec_len, + blocksize) + + ext4_rec_len_from_disk(de->rec_len, + blocksize), + blocksize); + + /* wipe entire dir_entry */ + memset(de, 0, ext4_rec_len_from_disk(de->rec_len, + blocksize)); + } else { + /* wipe dir_entry excluding the rec_len field */ + de->inode = 0; + memset(&de->name_len, 0, + ext4_rec_len_from_disk(de->rec_len, + blocksize) - + offsetof(struct ext4_dir_entry_2, + name_len)); + } + + inode_inc_iversion(dir); + return 0; + } + i += ext4_rec_len_from_disk(de->rec_len, blocksize); + pde = de; + de = ext4_next_entry(de, blocksize); + } + return -ENOENT; +} + +static int ext4_delete_entry(handle_t *handle, + struct inode *dir, + struct ext4_dir_entry_2 *de_del, + struct buffer_head *bh) +{ + int err, csum_size = 0; + + if (ext4_has_inline_data(dir)) { + int has_inline_data = 1; + err = ext4_delete_inline_entry(handle, dir, de_del, bh, + &has_inline_data); + if (has_inline_data) + return err; + } + + if (ext4_has_feature_metadata_csum(dir->i_sb)) + csum_size = sizeof(struct ext4_dir_entry_tail); + + BUFFER_TRACE(bh, "get_write_access"); + err = ext4_journal_get_write_access(handle, dir->i_sb, bh, + EXT4_JTR_NONE); + if (unlikely(err)) + goto out; + + err = ext4_generic_delete_entry(dir, de_del, bh, bh->b_data, + dir->i_sb->s_blocksize, csum_size); + if (err) + goto out; + + BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata"); + err = ext4_handle_dirty_dirblock(handle, dir, bh); + if (unlikely(err)) + goto out; + + return 0; +out: + if (err != -ENOENT) + ext4_std_error(dir->i_sb, err); + return err; +} + +/* + * Set directory link count to 1 if nlinks > EXT4_LINK_MAX, or if nlinks == 2 + * since this indicates that nlinks count was previously 1 to avoid overflowing + * the 16-bit i_links_count field on disk. Directories with i_nlink == 1 mean + * that subdirectory link counts are not being maintained accurately. + * + * The caller has already checked for i_nlink overflow in case the DIR_LINK + * feature is not enabled and returned -EMLINK. The is_dx() check is a proxy + * for checking S_ISDIR(inode) (since the INODE_INDEX feature will not be set + * on regular files) and to avoid creating huge/slow non-HTREE directories. + */ +static void ext4_inc_count(struct inode *inode) +{ + inc_nlink(inode); + if (is_dx(inode) && + (inode->i_nlink > EXT4_LINK_MAX || inode->i_nlink == 2)) + set_nlink(inode, 1); +} + +/* + * If a directory had nlink == 1, then we should let it be 1. This indicates + * directory has >EXT4_LINK_MAX subdirs. + */ +static void ext4_dec_count(struct inode *inode) +{ + if (!S_ISDIR(inode->i_mode) || inode->i_nlink > 2) + drop_nlink(inode); +} + + +/* + * Add non-directory inode to a directory. On success, the inode reference is + * consumed by dentry is instantiation. This is also indicated by clearing of + * *inodep pointer. On failure, the caller is responsible for dropping the + * inode reference in the safe context. + */ +static int ext4_add_nondir(handle_t *handle, + struct dentry *dentry, struct inode **inodep) +{ + struct inode *dir = d_inode(dentry->d_parent); + struct inode *inode = *inodep; + int err = ext4_add_entry(handle, dentry, inode); + if (!err) { + err = ext4_mark_inode_dirty(handle, inode); + if (IS_DIRSYNC(dir)) + ext4_handle_sync(handle); + d_instantiate_new(dentry, inode); + *inodep = NULL; + return err; + } + drop_nlink(inode); + ext4_mark_inode_dirty(handle, inode); + ext4_orphan_add(handle, inode); + unlock_new_inode(inode); + return err; +} + +/* + * By the time this is called, we already have created + * the directory cache entry for the new file, but it + * is so far negative - it has no inode. + * + * If the create succeeds, we fill in the inode information + * with d_instantiate(). + */ +static int ext4_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + handle_t *handle; + struct inode *inode; + int err, credits, retries = 0; + + err = dquot_initialize(dir); + if (err) + return err; + + credits = (EXT4_DATA_TRANS_BLOCKS(dir->i_sb) + + EXT4_INDEX_EXTRA_TRANS_BLOCKS + 3); +retry: + inode = ext4_new_inode_start_handle(idmap, dir, mode, &dentry->d_name, + 0, NULL, EXT4_HT_DIR, credits); + handle = ext4_journal_current_handle(); + err = PTR_ERR(inode); + if (!IS_ERR(inode)) { + inode->i_op = &ext4_file_inode_operations; + inode->i_fop = &ext4_file_operations; + ext4_set_aops(inode); + err = ext4_add_nondir(handle, dentry, &inode); + if (!err) + ext4_fc_track_create(handle, dentry); + } + if (handle) + ext4_journal_stop(handle); + if (!IS_ERR_OR_NULL(inode)) + iput(inode); + if (err == -ENOSPC && ext4_should_retry_alloc(dir->i_sb, &retries)) + goto retry; + return err; +} + +static int ext4_mknod(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) +{ + handle_t *handle; + struct inode *inode; + int err, credits, retries = 0; + + err = dquot_initialize(dir); + if (err) + return err; + + credits = (EXT4_DATA_TRANS_BLOCKS(dir->i_sb) + + EXT4_INDEX_EXTRA_TRANS_BLOCKS + 3); +retry: + inode = ext4_new_inode_start_handle(idmap, dir, mode, &dentry->d_name, + 0, NULL, EXT4_HT_DIR, credits); + handle = ext4_journal_current_handle(); + err = PTR_ERR(inode); + if (!IS_ERR(inode)) { + init_special_inode(inode, inode->i_mode, rdev); + inode->i_op = &ext4_special_inode_operations; + err = ext4_add_nondir(handle, dentry, &inode); + if (!err) + ext4_fc_track_create(handle, dentry); + } + if (handle) + ext4_journal_stop(handle); + if (!IS_ERR_OR_NULL(inode)) + iput(inode); + if (err == -ENOSPC && ext4_should_retry_alloc(dir->i_sb, &retries)) + goto retry; + return err; +} + +static int ext4_tmpfile(struct mnt_idmap *idmap, struct inode *dir, + struct file *file, umode_t mode) +{ + handle_t *handle; + struct inode *inode; + int err, retries = 0; + + err = dquot_initialize(dir); + if (err) + return err; + +retry: + inode = ext4_new_inode_start_handle(idmap, dir, mode, + NULL, 0, NULL, + EXT4_HT_DIR, + EXT4_MAXQUOTAS_TRANS_BLOCKS(dir->i_sb) + + 4 + EXT4_XATTR_TRANS_BLOCKS); + handle = ext4_journal_current_handle(); + err = PTR_ERR(inode); + if (!IS_ERR(inode)) { + inode->i_op = &ext4_file_inode_operations; + inode->i_fop = &ext4_file_operations; + ext4_set_aops(inode); + d_tmpfile(file, inode); + err = ext4_orphan_add(handle, inode); + if (err) + goto err_unlock_inode; + mark_inode_dirty(inode); + unlock_new_inode(inode); + } + if (handle) + ext4_journal_stop(handle); + if (err == -ENOSPC && ext4_should_retry_alloc(dir->i_sb, &retries)) + goto retry; + return finish_open_simple(file, err); +err_unlock_inode: + ext4_journal_stop(handle); + unlock_new_inode(inode); + return err; +} + +int ext4_init_dirblock(handle_t *handle, struct inode *inode, + struct buffer_head *bh, unsigned int parent_ino, + void *inline_buf, int inline_size) +{ + struct ext4_dir_entry_2 *de = (struct ext4_dir_entry_2 *) bh->b_data; + size_t blocksize = bh->b_size; + int csum_size = 0, header_size; + + if (ext4_has_feature_metadata_csum(inode->i_sb)) + csum_size = sizeof(struct ext4_dir_entry_tail); + + de->inode = cpu_to_le32(inode->i_ino); + de->name_len = 1; + de->rec_len = ext4_rec_len_to_disk(ext4_dir_rec_len(de->name_len, NULL), + blocksize); + memcpy(de->name, ".", 2); + ext4_set_de_type(inode->i_sb, de, S_IFDIR); + + de = ext4_next_entry(de, blocksize); + de->inode = cpu_to_le32(parent_ino); + de->name_len = 2; + memcpy(de->name, "..", 3); + ext4_set_de_type(inode->i_sb, de, S_IFDIR); + if (inline_buf) { + de->rec_len = ext4_rec_len_to_disk( + ext4_dir_rec_len(de->name_len, NULL), + blocksize); + de = ext4_next_entry(de, blocksize); + header_size = (char *)de - bh->b_data; + memcpy((void *)de, inline_buf, inline_size); + ext4_update_final_de(bh->b_data, inline_size + header_size, + blocksize - csum_size); + } else { + de->rec_len = ext4_rec_len_to_disk(blocksize - + (csum_size + ext4_dir_rec_len(1, NULL)), + blocksize); + } + + if (csum_size) + ext4_initialize_dirent_tail(bh, blocksize); + BUFFER_TRACE(dir_block, "call ext4_handle_dirty_metadata"); + set_buffer_uptodate(bh); + set_buffer_verified(bh); + return ext4_handle_dirty_dirblock(handle, inode, bh); +} + +int ext4_init_new_dir(handle_t *handle, struct inode *dir, + struct inode *inode) +{ + struct buffer_head *dir_block = NULL; + ext4_lblk_t block = 0; + int err; + + if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) { + err = ext4_try_create_inline_dir(handle, dir, inode); + if (err < 0 && err != -ENOSPC) + goto out; + if (!err) + goto out; + } + + set_nlink(inode, 2); + inode->i_size = 0; + dir_block = ext4_append(handle, inode, &block); + if (IS_ERR(dir_block)) + return PTR_ERR(dir_block); + err = ext4_init_dirblock(handle, inode, dir_block, dir->i_ino, NULL, 0); +out: + brelse(dir_block); + return err; +} + +static struct dentry *ext4_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + handle_t *handle; + struct inode *inode; + int err, err2 = 0, credits, retries = 0; + + if (EXT4_DIR_LINK_MAX(dir)) + return ERR_PTR(-EMLINK); + + err = dquot_initialize(dir); + if (err) + return ERR_PTR(err); + + credits = (EXT4_DATA_TRANS_BLOCKS(dir->i_sb) + + EXT4_INDEX_EXTRA_TRANS_BLOCKS + 3); +retry: + inode = ext4_new_inode_start_handle(idmap, dir, mode, + &dentry->d_name, + 0, NULL, EXT4_HT_DIR, credits); + handle = ext4_journal_current_handle(); + err = PTR_ERR(inode); + if (IS_ERR(inode)) + goto out_stop; + + inode->i_op = &ext4_dir_inode_operations; + inode->i_fop = &ext4_dir_operations; + err = ext4_init_new_dir(handle, dir, inode); + if (err) + goto out_clear_inode; + err = ext4_mark_inode_dirty(handle, inode); + if (!err) + err = ext4_add_entry(handle, dentry, inode); + if (err) { +out_clear_inode: + clear_nlink(inode); + ext4_orphan_add(handle, inode); + unlock_new_inode(inode); + err2 = ext4_mark_inode_dirty(handle, inode); + if (unlikely(err2)) + err = err2; + ext4_journal_stop(handle); + iput(inode); + goto out_retry; + } + ext4_inc_count(dir); + + ext4_update_dx_flag(dir); + err = ext4_mark_inode_dirty(handle, dir); + if (err) + goto out_clear_inode; + d_instantiate_new(dentry, inode); + ext4_fc_track_create(handle, dentry); + if (IS_DIRSYNC(dir)) + ext4_handle_sync(handle); + +out_stop: + if (handle) + ext4_journal_stop(handle); +out_retry: + if (err == -ENOSPC && ext4_should_retry_alloc(dir->i_sb, &retries)) + goto retry; + return err ? ERR_PTR(err) : NULL; +} + +/* + * routine to check that the specified directory is empty (for rmdir) + */ +bool ext4_empty_dir(struct inode *inode) +{ + unsigned int offset; + struct buffer_head *bh; + struct ext4_dir_entry_2 *de; + struct super_block *sb; + + if (ext4_has_inline_data(inode)) { + int has_inline_data = 1; + int ret; + + ret = empty_inline_dir(inode, &has_inline_data); + if (has_inline_data) + return ret; + } + + sb = inode->i_sb; + if (inode->i_size < ext4_dir_rec_len(1, NULL) + + ext4_dir_rec_len(2, NULL)) { + EXT4_ERROR_INODE(inode, "invalid size"); + return false; + } + bh = ext4_read_dirblock(inode, 0, EITHER); + if (IS_ERR(bh)) + return false; + + de = (struct ext4_dir_entry_2 *) bh->b_data; + if (ext4_check_dir_entry(inode, NULL, de, bh, bh->b_data, bh->b_size, + 0) || + le32_to_cpu(de->inode) != inode->i_ino || de->name_len != 1 || + de->name[0] != '.') { + ext4_warning_inode(inode, "directory missing '.'"); + brelse(bh); + return false; + } + offset = ext4_rec_len_from_disk(de->rec_len, sb->s_blocksize); + de = ext4_next_entry(de, sb->s_blocksize); + if (ext4_check_dir_entry(inode, NULL, de, bh, bh->b_data, bh->b_size, + offset) || + le32_to_cpu(de->inode) == 0 || de->name_len != 2 || + de->name[0] != '.' || de->name[1] != '.') { + ext4_warning_inode(inode, "directory missing '..'"); + brelse(bh); + return false; + } + offset += ext4_rec_len_from_disk(de->rec_len, sb->s_blocksize); + while (offset < inode->i_size) { + if (!(offset & (sb->s_blocksize - 1))) { + unsigned int lblock; + brelse(bh); + lblock = offset >> EXT4_BLOCK_SIZE_BITS(sb); + bh = ext4_read_dirblock(inode, lblock, EITHER); + if (bh == NULL) { + offset += sb->s_blocksize; + continue; + } + if (IS_ERR(bh)) + return false; + } + de = (struct ext4_dir_entry_2 *) (bh->b_data + + (offset & (sb->s_blocksize - 1))); + if (ext4_check_dir_entry(inode, NULL, de, bh, + bh->b_data, bh->b_size, offset) || + le32_to_cpu(de->inode)) { + brelse(bh); + return false; + } + offset += ext4_rec_len_from_disk(de->rec_len, sb->s_blocksize); + } + brelse(bh); + return true; +} + +static int ext4_rmdir(struct inode *dir, struct dentry *dentry) +{ + int retval; + struct inode *inode; + struct buffer_head *bh; + struct ext4_dir_entry_2 *de = NULL; + handle_t *handle = NULL; + + retval = ext4_emergency_state(dir->i_sb); + if (unlikely(retval)) + return retval; + + /* Initialize quotas before so that eventual writes go in + * separate transaction */ + retval = dquot_initialize(dir); + if (retval) + return retval; + retval = dquot_initialize(d_inode(dentry)); + if (retval) + return retval; + + retval = -ENOENT; + bh = ext4_find_entry(dir, &dentry->d_name, &de, NULL); + if (IS_ERR(bh)) + return PTR_ERR(bh); + if (!bh) + goto end_rmdir; + + inode = d_inode(dentry); + + retval = -EFSCORRUPTED; + if (le32_to_cpu(de->inode) != inode->i_ino) + goto end_rmdir; + + retval = -ENOTEMPTY; + if (!ext4_empty_dir(inode)) + goto end_rmdir; + + handle = ext4_journal_start(dir, EXT4_HT_DIR, + EXT4_DATA_TRANS_BLOCKS(dir->i_sb)); + if (IS_ERR(handle)) { + retval = PTR_ERR(handle); + handle = NULL; + goto end_rmdir; + } + + if (IS_DIRSYNC(dir)) + ext4_handle_sync(handle); + + retval = ext4_delete_entry(handle, dir, de, bh); + if (retval) + goto end_rmdir; + if (!EXT4_DIR_LINK_EMPTY(inode)) + ext4_warning_inode(inode, + "empty directory '%.*s' has too many links (%u)", + dentry->d_name.len, dentry->d_name.name, + inode->i_nlink); + inode_inc_iversion(inode); + clear_nlink(inode); + /* There's no need to set i_disksize: the fact that i_nlink is + * zero will ensure that the right thing happens during any + * recovery. */ + inode->i_size = 0; + ext4_orphan_add(handle, inode); + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + inode_set_ctime_current(inode); + retval = ext4_mark_inode_dirty(handle, inode); + if (retval) + goto end_rmdir; + ext4_dec_count(dir); + ext4_update_dx_flag(dir); + ext4_fc_track_unlink(handle, dentry); + retval = ext4_mark_inode_dirty(handle, dir); + + /* VFS negative dentries are incompatible with Encoding and + * Case-insensitiveness. Eventually we'll want avoid + * invalidating the dentries here, alongside with returning the + * negative dentries at ext4_lookup(), when it is better + * supported by the VFS for the CI case. + */ + if (IS_ENABLED(CONFIG_UNICODE) && IS_CASEFOLDED(dir)) + d_invalidate(dentry); + +end_rmdir: + brelse(bh); + if (handle) + ext4_journal_stop(handle); + return retval; +} + +int __ext4_unlink(struct inode *dir, const struct qstr *d_name, + struct inode *inode, + struct dentry *dentry /* NULL during fast_commit recovery */) +{ + int retval = -ENOENT; + struct buffer_head *bh; + struct ext4_dir_entry_2 *de = NULL; + handle_t *handle; + int skip_remove_dentry = 0; + + /* + * Keep this outside the transaction; it may have to set up the + * directory's encryption key, which isn't GFP_NOFS-safe. + */ + bh = ext4_find_entry(dir, d_name, &de, NULL); + if (IS_ERR(bh)) + return PTR_ERR(bh); + + if (!bh) + return -ENOENT; + + if (le32_to_cpu(de->inode) != inode->i_ino) { + /* + * It's okay if we find dont find dentry which matches + * the inode. That's because it might have gotten + * renamed to a different inode number + */ + if (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY) + skip_remove_dentry = 1; + else + goto out_bh; + } + + handle = ext4_journal_start(dir, EXT4_HT_DIR, + EXT4_DATA_TRANS_BLOCKS(dir->i_sb)); + if (IS_ERR(handle)) { + retval = PTR_ERR(handle); + goto out_bh; + } + + if (IS_DIRSYNC(dir)) + ext4_handle_sync(handle); + + if (!skip_remove_dentry) { + retval = ext4_delete_entry(handle, dir, de, bh); + if (retval) + goto out_handle; + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + ext4_update_dx_flag(dir); + retval = ext4_mark_inode_dirty(handle, dir); + if (retval) + goto out_handle; + } else { + retval = 0; + } + if (inode->i_nlink == 0) + ext4_warning_inode(inode, "Deleting file '%.*s' with no links", + d_name->len, d_name->name); + else + drop_nlink(inode); + if (!inode->i_nlink) + ext4_orphan_add(handle, inode); + inode_set_ctime_current(inode); + retval = ext4_mark_inode_dirty(handle, inode); + if (dentry && !retval) + ext4_fc_track_unlink(handle, dentry); +out_handle: + ext4_journal_stop(handle); +out_bh: + brelse(bh); + return retval; +} + +static int ext4_unlink(struct inode *dir, struct dentry *dentry) +{ + int retval; + + retval = ext4_emergency_state(dir->i_sb); + if (unlikely(retval)) + return retval; + + trace_ext4_unlink_enter(dir, dentry); + /* + * Initialize quotas before so that eventual writes go + * in separate transaction + */ + retval = dquot_initialize(dir); + if (retval) + goto out_trace; + retval = dquot_initialize(d_inode(dentry)); + if (retval) + goto out_trace; + + retval = __ext4_unlink(dir, &dentry->d_name, d_inode(dentry), dentry); + + /* VFS negative dentries are incompatible with Encoding and + * Case-insensitiveness. Eventually we'll want avoid + * invalidating the dentries here, alongside with returning the + * negative dentries at ext4_lookup(), when it is better + * supported by the VFS for the CI case. + */ + if (IS_ENABLED(CONFIG_UNICODE) && IS_CASEFOLDED(dir)) + d_invalidate(dentry); + +out_trace: + trace_ext4_unlink_exit(dentry, retval); + return retval; +} + +static int ext4_init_symlink_block(handle_t *handle, struct inode *inode, + struct fscrypt_str *disk_link) +{ + struct buffer_head *bh; + char *kaddr; + int err = 0; + + bh = ext4_bread(handle, inode, 0, EXT4_GET_BLOCKS_CREATE); + if (IS_ERR(bh)) + return PTR_ERR(bh); + + BUFFER_TRACE(bh, "get_write_access"); + err = ext4_journal_get_write_access(handle, inode->i_sb, bh, EXT4_JTR_NONE); + if (err) + goto out; + + kaddr = (char *)bh->b_data; + memcpy(kaddr, disk_link->name, disk_link->len); + inode->i_size = disk_link->len - 1; + EXT4_I(inode)->i_disksize = inode->i_size; + err = ext4_handle_dirty_metadata(handle, inode, bh); +out: + brelse(bh); + return err; +} + +static int ext4_symlink(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const char *symname) +{ + handle_t *handle; + struct inode *inode; + int err, len = strlen(symname); + int credits; + struct fscrypt_str disk_link; + int retries = 0; + + err = ext4_emergency_state(dir->i_sb); + if (unlikely(err)) + return err; + + err = fscrypt_prepare_symlink(dir, symname, len, dir->i_sb->s_blocksize, + &disk_link); + if (err) + return err; + + err = dquot_initialize(dir); + if (err) + return err; + + /* + * EXT4_INDEX_EXTRA_TRANS_BLOCKS for addition of entry into the + * directory. +3 for inode, inode bitmap, group descriptor allocation. + * EXT4_DATA_TRANS_BLOCKS for the data block allocation and + * modification. + */ + credits = EXT4_DATA_TRANS_BLOCKS(dir->i_sb) + + EXT4_INDEX_EXTRA_TRANS_BLOCKS + 3; +retry: + inode = ext4_new_inode_start_handle(idmap, dir, S_IFLNK|S_IRWXUGO, + &dentry->d_name, 0, NULL, + EXT4_HT_DIR, credits); + handle = ext4_journal_current_handle(); + if (IS_ERR(inode)) { + if (handle) + ext4_journal_stop(handle); + err = PTR_ERR(inode); + goto out_retry; + } + + if (IS_ENCRYPTED(inode)) { + err = fscrypt_encrypt_symlink(inode, symname, len, &disk_link); + if (err) + goto err_drop_inode; + inode->i_op = &ext4_encrypted_symlink_inode_operations; + } else { + if ((disk_link.len > EXT4_N_BLOCKS * 4)) { + inode->i_op = &ext4_symlink_inode_operations; + } else { + inode->i_op = &ext4_fast_symlink_inode_operations; + } + } + + if ((disk_link.len > EXT4_N_BLOCKS * 4)) { + /* alloc symlink block and fill it */ + err = ext4_init_symlink_block(handle, inode, &disk_link); + if (err) + goto err_drop_inode; + } else { + /* clear the extent format for fast symlink */ + ext4_clear_inode_flag(inode, EXT4_INODE_EXTENTS); + memcpy((char *)&EXT4_I(inode)->i_data, disk_link.name, + disk_link.len); + inode->i_size = disk_link.len - 1; + EXT4_I(inode)->i_disksize = inode->i_size; + if (!IS_ENCRYPTED(inode)) + inode_set_cached_link(inode, (char *)&EXT4_I(inode)->i_data, + inode->i_size); + } + err = ext4_add_nondir(handle, dentry, &inode); + if (handle) + ext4_journal_stop(handle); + iput(inode); + goto out_retry; + +err_drop_inode: + clear_nlink(inode); + ext4_mark_inode_dirty(handle, inode); + ext4_orphan_add(handle, inode); + unlock_new_inode(inode); + if (handle) + ext4_journal_stop(handle); + iput(inode); +out_retry: + if (err == -ENOSPC && ext4_should_retry_alloc(dir->i_sb, &retries)) + goto retry; + if (disk_link.name != (unsigned char *)symname) + kfree(disk_link.name); + return err; +} + +int __ext4_link(struct inode *dir, struct inode *inode, + const struct qstr *d_name, struct dentry *dentry) +{ + handle_t *handle; + int err, retries = 0; +retry: + handle = ext4_journal_start(dir, EXT4_HT_DIR, + (EXT4_DATA_TRANS_BLOCKS(dir->i_sb) + + EXT4_INDEX_EXTRA_TRANS_BLOCKS) + 1); + if (IS_ERR(handle)) + return PTR_ERR(handle); + + if (IS_DIRSYNC(dir)) + ext4_handle_sync(handle); + + inode_set_ctime_current(inode); + ext4_inc_count(inode); + + err = __ext4_add_entry(handle, dir, d_name, inode); + if (!err) { + err = ext4_mark_inode_dirty(handle, inode); + /* this can happen only for tmpfile being + * linked the first time + */ + if (inode->i_nlink == 1) + ext4_orphan_del(handle, inode); + if (dentry) + ext4_fc_track_link(handle, inode, dentry); + } else { + drop_nlink(inode); + } + ext4_journal_stop(handle); + if (err == -ENOSPC && ext4_should_retry_alloc(dir->i_sb, &retries)) + goto retry; + return err; +} + +static int ext4_link(struct dentry *old_dentry, + struct inode *dir, struct dentry *dentry) +{ + struct inode *inode = d_inode(old_dentry); + int err; + + if (inode->i_nlink >= EXT4_LINK_MAX) + return -EMLINK; + + err = fscrypt_prepare_link(old_dentry, dir, dentry); + if (err) + return err; + + if ((ext4_test_inode_flag(dir, EXT4_INODE_PROJINHERIT)) && + (!projid_eq(EXT4_I(dir)->i_projid, + EXT4_I(old_dentry->d_inode)->i_projid))) + return -EXDEV; + + err = dquot_initialize(dir); + if (err) + return err; + err = __ext4_link(dir, inode, &dentry->d_name, dentry); + if (!err) { + ihold(inode); + d_instantiate(dentry, inode); + } + return err; +} +/* + * Try to find buffer head where contains the parent block. + * It should be the inode block if it is inlined or the 1st block + * if it is a normal dir. + */ +static struct buffer_head *ext4_get_first_dir_block(handle_t *handle, + struct inode *inode, + int *retval, + struct ext4_dir_entry_2 **parent_de, + int *inlined) +{ + struct buffer_head *bh; + + if (!ext4_has_inline_data(inode)) { + struct ext4_dir_entry_2 *de; + unsigned int offset; + + bh = ext4_read_dirblock(inode, 0, EITHER); + if (IS_ERR(bh)) { + *retval = PTR_ERR(bh); + return NULL; + } + + de = (struct ext4_dir_entry_2 *) bh->b_data; + if (ext4_check_dir_entry(inode, NULL, de, bh, bh->b_data, + bh->b_size, 0) || + le32_to_cpu(de->inode) != inode->i_ino || + de->name_len != 1 || de->name[0] != '.') { + EXT4_ERROR_INODE(inode, "directory missing '.'"); + brelse(bh); + *retval = -EFSCORRUPTED; + return NULL; + } + offset = ext4_rec_len_from_disk(de->rec_len, + inode->i_sb->s_blocksize); + de = ext4_next_entry(de, inode->i_sb->s_blocksize); + if (ext4_check_dir_entry(inode, NULL, de, bh, bh->b_data, + bh->b_size, offset) || + le32_to_cpu(de->inode) == 0 || de->name_len != 2 || + de->name[0] != '.' || de->name[1] != '.') { + EXT4_ERROR_INODE(inode, "directory missing '..'"); + brelse(bh); + *retval = -EFSCORRUPTED; + return NULL; + } + *parent_de = de; + + return bh; + } + + *inlined = 1; + return ext4_get_first_inline_block(inode, parent_de, retval); +} + +struct ext4_renament { + struct inode *dir; + struct dentry *dentry; + struct inode *inode; + bool is_dir; + int dir_nlink_delta; + + /* entry for "dentry" */ + struct buffer_head *bh; + struct ext4_dir_entry_2 *de; + int inlined; + + /* entry for ".." in inode if it's a directory */ + struct buffer_head *dir_bh; + struct ext4_dir_entry_2 *parent_de; + int dir_inlined; +}; + +static int ext4_rename_dir_prepare(handle_t *handle, struct ext4_renament *ent, bool is_cross) +{ + int retval; + + ent->is_dir = true; + if (!is_cross) + return 0; + + ent->dir_bh = ext4_get_first_dir_block(handle, ent->inode, + &retval, &ent->parent_de, + &ent->dir_inlined); + if (!ent->dir_bh) + return retval; + if (le32_to_cpu(ent->parent_de->inode) != ent->dir->i_ino) + return -EFSCORRUPTED; + BUFFER_TRACE(ent->dir_bh, "get_write_access"); + return ext4_journal_get_write_access(handle, ent->dir->i_sb, + ent->dir_bh, EXT4_JTR_NONE); +} + +static int ext4_rename_dir_finish(handle_t *handle, struct ext4_renament *ent, + unsigned dir_ino) +{ + int retval; + + if (!ent->dir_bh) + return 0; + + ent->parent_de->inode = cpu_to_le32(dir_ino); + BUFFER_TRACE(ent->dir_bh, "call ext4_handle_dirty_metadata"); + if (!ent->dir_inlined) { + if (is_dx(ent->inode)) { + retval = ext4_handle_dirty_dx_node(handle, + ent->inode, + ent->dir_bh); + } else { + retval = ext4_handle_dirty_dirblock(handle, ent->inode, + ent->dir_bh); + } + } else { + retval = ext4_mark_inode_dirty(handle, ent->inode); + } + if (retval) { + ext4_std_error(ent->dir->i_sb, retval); + return retval; + } + return 0; +} + +static int ext4_setent(handle_t *handle, struct ext4_renament *ent, + unsigned ino, unsigned file_type) +{ + int retval, retval2; + + BUFFER_TRACE(ent->bh, "get write access"); + retval = ext4_journal_get_write_access(handle, ent->dir->i_sb, ent->bh, + EXT4_JTR_NONE); + if (retval) + return retval; + ent->de->inode = cpu_to_le32(ino); + if (ext4_has_feature_filetype(ent->dir->i_sb)) + ent->de->file_type = file_type; + inode_inc_iversion(ent->dir); + inode_set_mtime_to_ts(ent->dir, inode_set_ctime_current(ent->dir)); + retval = ext4_mark_inode_dirty(handle, ent->dir); + BUFFER_TRACE(ent->bh, "call ext4_handle_dirty_metadata"); + if (!ent->inlined) { + retval2 = ext4_handle_dirty_dirblock(handle, ent->dir, ent->bh); + if (unlikely(retval2)) { + ext4_std_error(ent->dir->i_sb, retval2); + return retval2; + } + } + return retval; +} + +static void ext4_resetent(handle_t *handle, struct ext4_renament *ent, + unsigned ino, unsigned file_type) +{ + struct ext4_renament old = *ent; + int retval = 0; + + /* + * old->de could have moved from under us during make indexed dir, + * so the old->de may no longer valid and need to find it again + * before reset old inode info. + */ + old.bh = ext4_find_entry(old.dir, &old.dentry->d_name, &old.de, + &old.inlined); + if (IS_ERR(old.bh)) + retval = PTR_ERR(old.bh); + if (!old.bh) + retval = -ENOENT; + if (retval) { + ext4_std_error(old.dir->i_sb, retval); + return; + } + + ext4_setent(handle, &old, ino, file_type); + brelse(old.bh); +} + +static int ext4_find_delete_entry(handle_t *handle, struct inode *dir, + const struct qstr *d_name) +{ + int retval = -ENOENT; + struct buffer_head *bh; + struct ext4_dir_entry_2 *de = NULL; + + bh = ext4_find_entry(dir, d_name, &de, NULL); + if (IS_ERR(bh)) + return PTR_ERR(bh); + if (bh) { + retval = ext4_delete_entry(handle, dir, de, bh); + brelse(bh); + } + return retval; +} + +static void ext4_rename_delete(handle_t *handle, struct ext4_renament *ent, + int force_reread) +{ + int retval; + /* + * ent->de could have moved from under us during htree split, so make + * sure that we are deleting the right entry. We might also be pointing + * to a stale entry in the unused part of ent->bh so just checking inum + * and the name isn't enough. + */ + if (le32_to_cpu(ent->de->inode) != ent->inode->i_ino || + ent->de->name_len != ent->dentry->d_name.len || + strncmp(ent->de->name, ent->dentry->d_name.name, + ent->de->name_len) || + force_reread) { + retval = ext4_find_delete_entry(handle, ent->dir, + &ent->dentry->d_name); + } else { + retval = ext4_delete_entry(handle, ent->dir, ent->de, ent->bh); + if (retval == -ENOENT) { + retval = ext4_find_delete_entry(handle, ent->dir, + &ent->dentry->d_name); + } + } + + if (retval) { + ext4_warning_inode(ent->dir, + "Deleting old file: nlink %d, error=%d", + ent->dir->i_nlink, retval); + } +} + +static void ext4_update_dir_count(handle_t *handle, struct ext4_renament *ent) +{ + if (ent->dir_nlink_delta) { + if (ent->dir_nlink_delta == -1) + ext4_dec_count(ent->dir); + else + ext4_inc_count(ent->dir); + ext4_mark_inode_dirty(handle, ent->dir); + } +} + +static struct inode *ext4_whiteout_for_rename(struct mnt_idmap *idmap, + struct ext4_renament *ent, + int credits, handle_t **h) +{ + struct inode *wh; + handle_t *handle; + int retries = 0; + + /* + * for inode block, sb block, group summaries, + * and inode bitmap + */ + credits += (EXT4_MAXQUOTAS_TRANS_BLOCKS(ent->dir->i_sb) + + EXT4_XATTR_TRANS_BLOCKS + 4); +retry: + wh = ext4_new_inode_start_handle(idmap, ent->dir, + S_IFCHR | WHITEOUT_MODE, + &ent->dentry->d_name, 0, NULL, + EXT4_HT_DIR, credits); + + handle = ext4_journal_current_handle(); + if (IS_ERR(wh)) { + if (handle) + ext4_journal_stop(handle); + if (PTR_ERR(wh) == -ENOSPC && + ext4_should_retry_alloc(ent->dir->i_sb, &retries)) + goto retry; + } else { + *h = handle; + init_special_inode(wh, wh->i_mode, WHITEOUT_DEV); + wh->i_op = &ext4_special_inode_operations; + } + return wh; +} + +/* + * Anybody can rename anything with this: the permission checks are left to the + * higher-level routines. + * + * n.b. old_{dentry,inode) refers to the source dentry/inode + * while new_{dentry,inode) refers to the destination dentry/inode + * This comes from rename(const char *oldpath, const char *newpath) + */ +static int ext4_rename(struct mnt_idmap *idmap, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) +{ + handle_t *handle = NULL; + struct ext4_renament old = { + .dir = old_dir, + .dentry = old_dentry, + .inode = d_inode(old_dentry), + }; + struct ext4_renament new = { + .dir = new_dir, + .dentry = new_dentry, + .inode = d_inode(new_dentry), + }; + int force_reread; + int retval; + struct inode *whiteout = NULL; + int credits; + u8 old_file_type; + + if (new.inode && new.inode->i_nlink == 0) { + EXT4_ERROR_INODE(new.inode, + "target of rename is already freed"); + return -EFSCORRUPTED; + } + + if ((ext4_test_inode_flag(new_dir, EXT4_INODE_PROJINHERIT)) && + (!projid_eq(EXT4_I(new_dir)->i_projid, + EXT4_I(old_dentry->d_inode)->i_projid))) + return -EXDEV; + + retval = dquot_initialize(old.dir); + if (retval) + return retval; + retval = dquot_initialize(old.inode); + if (retval) + return retval; + retval = dquot_initialize(new.dir); + if (retval) + return retval; + + /* Initialize quotas before so that eventual writes go + * in separate transaction */ + if (new.inode) { + retval = dquot_initialize(new.inode); + if (retval) + return retval; + } + + old.bh = ext4_find_entry(old.dir, &old.dentry->d_name, &old.de, + &old.inlined); + if (IS_ERR(old.bh)) + return PTR_ERR(old.bh); + + /* + * Check for inode number is _not_ due to possible IO errors. + * We might rmdir the source, keep it as pwd of some process + * and merrily kill the link to whatever was created under the + * same name. Goodbye sticky bit ;-< + */ + retval = -ENOENT; + if (!old.bh || le32_to_cpu(old.de->inode) != old.inode->i_ino) + goto release_bh; + + new.bh = ext4_find_entry(new.dir, &new.dentry->d_name, + &new.de, &new.inlined); + if (IS_ERR(new.bh)) { + retval = PTR_ERR(new.bh); + new.bh = NULL; + goto release_bh; + } + if (new.bh) { + if (!new.inode) { + brelse(new.bh); + new.bh = NULL; + } + } + if (new.inode && !test_opt(new.dir->i_sb, NO_AUTO_DA_ALLOC)) + ext4_alloc_da_blocks(old.inode); + + credits = (2 * EXT4_DATA_TRANS_BLOCKS(old.dir->i_sb) + + EXT4_INDEX_EXTRA_TRANS_BLOCKS + 2); + if (!(flags & RENAME_WHITEOUT)) { + handle = ext4_journal_start(old.dir, EXT4_HT_DIR, credits); + if (IS_ERR(handle)) { + retval = PTR_ERR(handle); + goto release_bh; + } + } else { + whiteout = ext4_whiteout_for_rename(idmap, &old, credits, &handle); + if (IS_ERR(whiteout)) { + retval = PTR_ERR(whiteout); + goto release_bh; + } + } + + old_file_type = old.de->file_type; + if (IS_DIRSYNC(old.dir) || IS_DIRSYNC(new.dir)) + ext4_handle_sync(handle); + + if (S_ISDIR(old.inode->i_mode)) { + if (new.inode) { + retval = -ENOTEMPTY; + if (!ext4_empty_dir(new.inode)) + goto end_rename; + } else { + retval = -EMLINK; + if (new.dir != old.dir && EXT4_DIR_LINK_MAX(new.dir)) + goto end_rename; + } + retval = ext4_rename_dir_prepare(handle, &old, new.dir != old.dir); + if (retval) + goto end_rename; + } + /* + * If we're renaming a file within an inline_data dir and adding or + * setting the new dirent causes a conversion from inline_data to + * extents/blockmap, we need to force the dirent delete code to + * re-read the directory, or else we end up trying to delete a dirent + * from what is now the extent tree root (or a block map). + */ + force_reread = (new.dir->i_ino == old.dir->i_ino && + ext4_test_inode_flag(new.dir, EXT4_INODE_INLINE_DATA)); + + if (whiteout) { + /* + * Do this before adding a new entry, so the old entry is sure + * to be still pointing to the valid old entry. + */ + retval = ext4_setent(handle, &old, whiteout->i_ino, + EXT4_FT_CHRDEV); + if (retval) + goto end_rename; + retval = ext4_mark_inode_dirty(handle, whiteout); + if (unlikely(retval)) + goto end_rename; + + } + if (!new.bh) { + retval = ext4_add_entry(handle, new.dentry, old.inode); + if (retval) + goto end_rename; + } else { + retval = ext4_setent(handle, &new, + old.inode->i_ino, old_file_type); + if (retval) + goto end_rename; + } + if (force_reread) + force_reread = !ext4_test_inode_flag(new.dir, + EXT4_INODE_INLINE_DATA); + + /* + * Like most other Unix systems, set the ctime for inodes on a + * rename. + */ + inode_set_ctime_current(old.inode); + retval = ext4_mark_inode_dirty(handle, old.inode); + if (unlikely(retval)) + goto end_rename; + + if (!whiteout) { + /* + * ok, that's it + */ + ext4_rename_delete(handle, &old, force_reread); + } + + if (new.inode) { + ext4_dec_count(new.inode); + inode_set_ctime_current(new.inode); + } + inode_set_mtime_to_ts(old.dir, inode_set_ctime_current(old.dir)); + ext4_update_dx_flag(old.dir); + if (old.is_dir) { + retval = ext4_rename_dir_finish(handle, &old, new.dir->i_ino); + if (retval) + goto end_rename; + + ext4_dec_count(old.dir); + if (new.inode) { + /* checked ext4_empty_dir above, can't have another + * parent, ext4_dec_count() won't work for many-linked + * dirs */ + clear_nlink(new.inode); + } else { + ext4_inc_count(new.dir); + ext4_update_dx_flag(new.dir); + retval = ext4_mark_inode_dirty(handle, new.dir); + if (unlikely(retval)) + goto end_rename; + } + } + retval = ext4_mark_inode_dirty(handle, old.dir); + if (unlikely(retval)) + goto end_rename; + + if (old.is_dir) { + /* + * We disable fast commits here that's because the + * replay code is not yet capable of changing dot dot + * dirents in directories. + */ + ext4_fc_mark_ineligible(old.inode->i_sb, + EXT4_FC_REASON_RENAME_DIR, handle); + } else { + struct super_block *sb = old.inode->i_sb; + + if (new.inode) + ext4_fc_track_unlink(handle, new.dentry); + if (test_opt2(sb, JOURNAL_FAST_COMMIT) && + !(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY) && + !(ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE))) { + __ext4_fc_track_link(handle, old.inode, new.dentry); + __ext4_fc_track_unlink(handle, old.inode, old.dentry); + if (whiteout) + __ext4_fc_track_create(handle, whiteout, + old.dentry); + } + } + + if (new.inode) { + retval = ext4_mark_inode_dirty(handle, new.inode); + if (unlikely(retval)) + goto end_rename; + if (!new.inode->i_nlink) + ext4_orphan_add(handle, new.inode); + } + retval = 0; + +end_rename: + if (whiteout) { + if (retval) { + ext4_resetent(handle, &old, + old.inode->i_ino, old_file_type); + drop_nlink(whiteout); + ext4_mark_inode_dirty(handle, whiteout); + ext4_orphan_add(handle, whiteout); + } + unlock_new_inode(whiteout); + ext4_journal_stop(handle); + iput(whiteout); + } else { + ext4_journal_stop(handle); + } +release_bh: + brelse(old.dir_bh); + brelse(old.bh); + brelse(new.bh); + + return retval; +} + +static int ext4_cross_rename(struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry) +{ + handle_t *handle = NULL; + struct ext4_renament old = { + .dir = old_dir, + .dentry = old_dentry, + .inode = d_inode(old_dentry), + }; + struct ext4_renament new = { + .dir = new_dir, + .dentry = new_dentry, + .inode = d_inode(new_dentry), + }; + u8 new_file_type; + int retval; + + if ((ext4_test_inode_flag(new_dir, EXT4_INODE_PROJINHERIT) && + !projid_eq(EXT4_I(new_dir)->i_projid, + EXT4_I(old_dentry->d_inode)->i_projid)) || + (ext4_test_inode_flag(old_dir, EXT4_INODE_PROJINHERIT) && + !projid_eq(EXT4_I(old_dir)->i_projid, + EXT4_I(new_dentry->d_inode)->i_projid))) + return -EXDEV; + + retval = dquot_initialize(old.dir); + if (retval) + return retval; + retval = dquot_initialize(new.dir); + if (retval) + return retval; + + old.bh = ext4_find_entry(old.dir, &old.dentry->d_name, + &old.de, &old.inlined); + if (IS_ERR(old.bh)) + return PTR_ERR(old.bh); + /* + * Check for inode number is _not_ due to possible IO errors. + * We might rmdir the source, keep it as pwd of some process + * and merrily kill the link to whatever was created under the + * same name. Goodbye sticky bit ;-< + */ + retval = -ENOENT; + if (!old.bh || le32_to_cpu(old.de->inode) != old.inode->i_ino) + goto end_rename; + + new.bh = ext4_find_entry(new.dir, &new.dentry->d_name, + &new.de, &new.inlined); + if (IS_ERR(new.bh)) { + retval = PTR_ERR(new.bh); + new.bh = NULL; + goto end_rename; + } + + /* RENAME_EXCHANGE case: old *and* new must both exist */ + if (!new.bh || le32_to_cpu(new.de->inode) != new.inode->i_ino) + goto end_rename; + + handle = ext4_journal_start(old.dir, EXT4_HT_DIR, + (2 * EXT4_DATA_TRANS_BLOCKS(old.dir->i_sb) + + 2 * EXT4_INDEX_EXTRA_TRANS_BLOCKS + 2)); + if (IS_ERR(handle)) { + retval = PTR_ERR(handle); + handle = NULL; + goto end_rename; + } + + if (IS_DIRSYNC(old.dir) || IS_DIRSYNC(new.dir)) + ext4_handle_sync(handle); + + if (S_ISDIR(old.inode->i_mode)) { + retval = ext4_rename_dir_prepare(handle, &old, new.dir != old.dir); + if (retval) + goto end_rename; + } + if (S_ISDIR(new.inode->i_mode)) { + retval = ext4_rename_dir_prepare(handle, &new, new.dir != old.dir); + if (retval) + goto end_rename; + } + + /* + * Other than the special case of overwriting a directory, parents' + * nlink only needs to be modified if this is a cross directory rename. + */ + if (old.dir != new.dir && old.is_dir != new.is_dir) { + old.dir_nlink_delta = old.is_dir ? -1 : 1; + new.dir_nlink_delta = -old.dir_nlink_delta; + retval = -EMLINK; + if ((old.dir_nlink_delta > 0 && EXT4_DIR_LINK_MAX(old.dir)) || + (new.dir_nlink_delta > 0 && EXT4_DIR_LINK_MAX(new.dir))) + goto end_rename; + } + + new_file_type = new.de->file_type; + retval = ext4_setent(handle, &new, old.inode->i_ino, old.de->file_type); + if (retval) + goto end_rename; + + retval = ext4_setent(handle, &old, new.inode->i_ino, new_file_type); + if (retval) + goto end_rename; + + /* + * Like most other Unix systems, set the ctime for inodes on a + * rename. + */ + inode_set_ctime_current(old.inode); + inode_set_ctime_current(new.inode); + retval = ext4_mark_inode_dirty(handle, old.inode); + if (unlikely(retval)) + goto end_rename; + retval = ext4_mark_inode_dirty(handle, new.inode); + if (unlikely(retval)) + goto end_rename; + ext4_fc_mark_ineligible(new.inode->i_sb, + EXT4_FC_REASON_CROSS_RENAME, handle); + if (old.dir_bh) { + retval = ext4_rename_dir_finish(handle, &old, new.dir->i_ino); + if (retval) + goto end_rename; + } + if (new.dir_bh) { + retval = ext4_rename_dir_finish(handle, &new, old.dir->i_ino); + if (retval) + goto end_rename; + } + ext4_update_dir_count(handle, &old); + ext4_update_dir_count(handle, &new); + retval = 0; + +end_rename: + brelse(old.dir_bh); + brelse(new.dir_bh); + brelse(old.bh); + brelse(new.bh); + if (handle) + ext4_journal_stop(handle); + return retval; +} + +static int ext4_rename2(struct mnt_idmap *idmap, + struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry, + unsigned int flags) +{ + int err; + + err = ext4_emergency_state(old_dir->i_sb); + if (unlikely(err)) + return err; + + if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) + return -EINVAL; + + err = fscrypt_prepare_rename(old_dir, old_dentry, new_dir, new_dentry, + flags); + if (err) + return err; + + if (flags & RENAME_EXCHANGE) { + return ext4_cross_rename(old_dir, old_dentry, + new_dir, new_dentry); + } + + return ext4_rename(idmap, old_dir, old_dentry, new_dir, new_dentry, flags); +} + +/* + * directories can handle most operations... + */ +const struct inode_operations ext4_dir_inode_operations = { + .create = ext4_create, + .lookup = ext4_lookup, + .link = ext4_link, + .unlink = ext4_unlink, + .symlink = ext4_symlink, + .mkdir = ext4_mkdir, + .rmdir = ext4_rmdir, + .mknod = ext4_mknod, + .tmpfile = ext4_tmpfile, + .rename = ext4_rename2, + .setattr = ext4_setattr, + .getattr = ext4_getattr, + .listxattr = ext4_listxattr, + .get_inode_acl = ext4_get_acl, + .set_acl = ext4_set_acl, + .fiemap = ext4_fiemap, + .fileattr_get = ext4_fileattr_get, + .fileattr_set = ext4_fileattr_set, +}; + +const struct inode_operations ext4_special_inode_operations = { + .setattr = ext4_setattr, + .getattr = ext4_getattr, + .listxattr = ext4_listxattr, + .get_inode_acl = ext4_get_acl, + .set_acl = ext4_set_acl, +}; diff --git a/fs/ext4/orphan.c b/fs/ext4/orphan.c new file mode 100644 index 000000000..b4675aa7e --- /dev/null +++ b/fs/ext4/orphan.c @@ -0,0 +1,662 @@ +/* + * Ext4 orphan inode handling + */ +#include +#include +#include +#include + +#include "ext4.h" +#include "ext4_jbd2.h" + +#define EXT4_MAX_ORPHAN_FILE_BLOCKS 512 + +static int ext4_orphan_file_add(handle_t *handle, struct inode *inode) +{ + int i, j, start; + struct ext4_orphan_info *oi = &EXT4_SB(inode->i_sb)->s_orphan_info; + int ret = 0; + bool found = false; + __le32 *bdata; + int inodes_per_ob = ext4_inodes_per_orphan_block(inode->i_sb); + int looped = 0; + + /* + * Find block with free orphan entry. Use CPU number for a naive hash + * for a search start in the orphan file + */ + start = raw_smp_processor_id()*13 % oi->of_blocks; + i = start; + do { + if (atomic_dec_if_positive(&oi->of_binfo[i].ob_free_entries) + >= 0) { + found = true; + break; + } + if (++i >= oi->of_blocks) + i = 0; + } while (i != start); + + if (!found) { + /* + * For now we don't grow or shrink orphan file. We just use + * whatever was allocated at mke2fs time. The additional + * credits we would have to reserve for each orphan inode + * operation just don't seem worth it. + */ + return -ENOSPC; + } + + ret = ext4_journal_get_write_access(handle, inode->i_sb, + oi->of_binfo[i].ob_bh, EXT4_JTR_ORPHAN_FILE); + if (ret) { + atomic_inc(&oi->of_binfo[i].ob_free_entries); + return ret; + } + + bdata = (__le32 *)(oi->of_binfo[i].ob_bh->b_data); + /* Find empty slot in a block */ + j = 0; + do { + if (looped) { + /* + * Did we walk through the block several times without + * finding free entry? It is theoretically possible + * if entries get constantly allocated and freed or + * if the block is corrupted. Avoid indefinite looping + * and bail. We'll use orphan list instead. + */ + if (looped > 3) { + atomic_inc(&oi->of_binfo[i].ob_free_entries); + return -ENOSPC; + } + cond_resched(); + } + while (bdata[j]) { + if (++j >= inodes_per_ob) { + j = 0; + looped++; + } + } + } while (cmpxchg(&bdata[j], (__le32)0, cpu_to_le32(inode->i_ino)) != + (__le32)0); + + EXT4_I(inode)->i_orphan_idx = i * inodes_per_ob + j; + ext4_set_inode_state(inode, EXT4_STATE_ORPHAN_FILE); + + return ext4_handle_dirty_metadata(handle, NULL, oi->of_binfo[i].ob_bh); +} + +/* + * ext4_orphan_add() links an unlinked or truncated inode into a list of + * such inodes, starting at the superblock, in case we crash before the + * file is closed/deleted, or in case the inode truncate spans multiple + * transactions and the last transaction is not recovered after a crash. + * + * At filesystem recovery time, we walk this list deleting unlinked + * inodes and truncating linked inodes in ext4_orphan_cleanup(). + * + * Orphan list manipulation functions must be called under i_rwsem unless + * we are just creating the inode or deleting it. + */ +int ext4_orphan_add(handle_t *handle, struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_iloc iloc; + int err = 0, rc; + bool dirty = false; + + if (!sbi->s_journal || is_bad_inode(inode)) + return 0; + + WARN_ON_ONCE(!(inode_state_read_once(inode) & (I_NEW | I_FREEING)) && + !inode_is_locked(inode)); + if (ext4_inode_orphan_tracked(inode)) + return 0; + + /* + * Orphan handling is only valid for files with data blocks + * being truncated, or files being unlinked. Note that we either + * hold i_rwsem, or the inode can not be referenced from outside, + * so i_nlink should not be bumped due to race + */ + ASSERT((S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) || + S_ISLNK(inode->i_mode)) || inode->i_nlink == 0); + + if (sbi->s_orphan_info.of_blocks) { + err = ext4_orphan_file_add(handle, inode); + /* + * Fallback to normal orphan list of orphan file is + * out of space + */ + if (err != -ENOSPC) + return err; + } + + BUFFER_TRACE(sbi->s_sbh, "get_write_access"); + err = ext4_journal_get_write_access(handle, sb, sbi->s_sbh, + EXT4_JTR_NONE); + if (err) + goto out; + + err = ext4_reserve_inode_write(handle, inode, &iloc); + if (err) + goto out; + + mutex_lock(&sbi->s_orphan_lock); + /* + * Due to previous errors inode may be already a part of on-disk + * orphan list. If so skip on-disk list modification. + */ + if (!NEXT_ORPHAN(inode) || NEXT_ORPHAN(inode) > + (le32_to_cpu(sbi->s_es->s_inodes_count))) { + /* Insert this inode at the head of the on-disk orphan list */ + NEXT_ORPHAN(inode) = le32_to_cpu(sbi->s_es->s_last_orphan); + lock_buffer(sbi->s_sbh); + sbi->s_es->s_last_orphan = cpu_to_le32(inode->i_ino); + ext4_superblock_csum_set(sb); + unlock_buffer(sbi->s_sbh); + dirty = true; + } + list_add(&EXT4_I(inode)->i_orphan, &sbi->s_orphan); + mutex_unlock(&sbi->s_orphan_lock); + + if (dirty) { + err = ext4_handle_dirty_metadata(handle, NULL, sbi->s_sbh); + rc = ext4_mark_iloc_dirty(handle, inode, &iloc); + if (!err) + err = rc; + if (err) { + /* + * We have to remove inode from in-memory list if + * addition to on disk orphan list failed. Stray orphan + * list entries can cause panics at unmount time. + */ + mutex_lock(&sbi->s_orphan_lock); + list_del_init(&EXT4_I(inode)->i_orphan); + mutex_unlock(&sbi->s_orphan_lock); + } + } else + brelse(iloc.bh); + + ext4_debug("superblock will point to %llu\n", inode->i_ino); + ext4_debug("orphan inode %llu will point to %d\n", + inode->i_ino, NEXT_ORPHAN(inode)); +out: + ext4_std_error(sb, err); + return err; +} + +static int ext4_orphan_file_del(handle_t *handle, struct inode *inode) +{ + struct ext4_orphan_info *oi = &EXT4_SB(inode->i_sb)->s_orphan_info; + __le32 *bdata; + int blk, off; + int inodes_per_ob = ext4_inodes_per_orphan_block(inode->i_sb); + int ret = 0; + + if (!handle) + goto out; + blk = EXT4_I(inode)->i_orphan_idx / inodes_per_ob; + off = EXT4_I(inode)->i_orphan_idx % inodes_per_ob; + if (WARN_ON_ONCE(blk >= oi->of_blocks)) + goto out; + + ret = ext4_journal_get_write_access(handle, inode->i_sb, + oi->of_binfo[blk].ob_bh, EXT4_JTR_ORPHAN_FILE); + if (ret) + goto out; + + bdata = (__le32 *)(oi->of_binfo[blk].ob_bh->b_data); + bdata[off] = 0; + atomic_inc(&oi->of_binfo[blk].ob_free_entries); + ret = ext4_handle_dirty_metadata(handle, NULL, oi->of_binfo[blk].ob_bh); +out: + ext4_clear_inode_state(inode, EXT4_STATE_ORPHAN_FILE); + INIT_LIST_HEAD(&EXT4_I(inode)->i_orphan); + + return ret; +} + +/* + * ext4_orphan_del() removes an unlinked or truncated inode from the list + * of such inodes stored on disk, because it is finally being cleaned up. + */ +int ext4_orphan_del(handle_t *handle, struct inode *inode) +{ + struct list_head *prev; + struct ext4_inode_info *ei = EXT4_I(inode); + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + __u32 ino_next; + struct ext4_iloc iloc; + int err = 0; + + if (!sbi->s_journal && !(sbi->s_mount_state & EXT4_ORPHAN_FS)) + return 0; + + WARN_ON_ONCE(!(inode_state_read_once(inode) & (I_NEW | I_FREEING)) && + !inode_is_locked(inode)); + if (ext4_test_inode_state(inode, EXT4_STATE_ORPHAN_FILE)) + return ext4_orphan_file_del(handle, inode); + + /* Do this quick check before taking global s_orphan_lock. */ + if (list_empty(&ei->i_orphan)) + return 0; + + if (handle) { + /* Grab inode buffer early before taking global s_orphan_lock */ + err = ext4_reserve_inode_write(handle, inode, &iloc); + } + + mutex_lock(&sbi->s_orphan_lock); + ext4_debug("remove inode %llu from orphan list\n", inode->i_ino); + + prev = ei->i_orphan.prev; + list_del_init(&ei->i_orphan); + + /* If we're on an error path, we may not have a valid + * transaction handle with which to update the orphan list on + * disk, but we still need to remove the inode from the linked + * list in memory. */ + if (!handle || err) { + mutex_unlock(&sbi->s_orphan_lock); + goto out_err; + } + + ino_next = NEXT_ORPHAN(inode); + if (prev == &sbi->s_orphan) { + ext4_debug("superblock will point to %u\n", ino_next); + BUFFER_TRACE(sbi->s_sbh, "get_write_access"); + err = ext4_journal_get_write_access(handle, inode->i_sb, + sbi->s_sbh, EXT4_JTR_NONE); + if (err) { + mutex_unlock(&sbi->s_orphan_lock); + goto out_brelse; + } + lock_buffer(sbi->s_sbh); + sbi->s_es->s_last_orphan = cpu_to_le32(ino_next); + ext4_superblock_csum_set(inode->i_sb); + unlock_buffer(sbi->s_sbh); + mutex_unlock(&sbi->s_orphan_lock); + err = ext4_handle_dirty_metadata(handle, NULL, sbi->s_sbh); + } else { + struct ext4_iloc iloc2; + struct inode *i_prev = + &list_entry(prev, struct ext4_inode_info, i_orphan)->vfs_inode; + + ext4_debug("orphan inode %llu will point to %u\n", + i_prev->i_ino, ino_next); + err = ext4_reserve_inode_write(handle, i_prev, &iloc2); + if (err) { + mutex_unlock(&sbi->s_orphan_lock); + goto out_brelse; + } + NEXT_ORPHAN(i_prev) = ino_next; + err = ext4_mark_iloc_dirty(handle, i_prev, &iloc2); + mutex_unlock(&sbi->s_orphan_lock); + } + if (err) + goto out_brelse; + NEXT_ORPHAN(inode) = 0; + err = ext4_mark_iloc_dirty(handle, inode, &iloc); +out_err: + ext4_std_error(inode->i_sb, err); + return err; + +out_brelse: + brelse(iloc.bh); + goto out_err; +} + +#ifdef CONFIG_QUOTA +static int ext4_quota_on_mount(struct super_block *sb, int type) +{ + return dquot_quota_on_mount(sb, + rcu_dereference_protected(EXT4_SB(sb)->s_qf_names[type], + lockdep_is_held(&sb->s_umount)), + EXT4_SB(sb)->s_jquota_fmt, type); +} +#endif + +static void ext4_process_orphan(struct inode *inode, + int *nr_truncates, int *nr_orphans) +{ + struct super_block *sb = inode->i_sb; + int ret; + + dquot_initialize(inode); + if (inode->i_nlink) { + if (test_opt(sb, DEBUG)) + ext4_msg(sb, KERN_DEBUG, + "%s: truncating inode %llu to %lld bytes", + __func__, inode->i_ino, inode->i_size); + ext4_debug("truncating inode %llu to %lld bytes\n", + inode->i_ino, inode->i_size); + inode_lock(inode); + truncate_inode_pages(inode->i_mapping, inode->i_size); + ret = ext4_truncate(inode); + if (ret) { + /* + * We need to clean up the in-core orphan list + * manually if ext4_truncate() failed to get a + * transaction handle. + */ + ext4_orphan_del(NULL, inode); + ext4_std_error(inode->i_sb, ret); + } + inode_unlock(inode); + (*nr_truncates)++; + } else { + if (test_opt(sb, DEBUG)) + ext4_msg(sb, KERN_DEBUG, + "%s: deleting unreferenced inode %llu", + __func__, inode->i_ino); + ext4_debug("deleting unreferenced inode %llu\n", + inode->i_ino); + (*nr_orphans)++; + } + iput(inode); /* The delete magic happens here! */ +} + +/* ext4_orphan_cleanup() walks a singly-linked list of inodes (starting at + * the superblock) which were deleted from all directories, but held open by + * a process at the time of a crash. We walk the list and try to delete these + * inodes at recovery time (only with a read-write filesystem). + * + * In order to keep the orphan inode chain consistent during traversal (in + * case of crash during recovery), we link each inode into the superblock + * orphan list_head and handle it the same way as an inode deletion during + * normal operation (which journals the operations for us). + * + * We only do an iget() and an iput() on each inode, which is very safe if we + * accidentally point at an in-use or already deleted inode. The worst that + * can happen in this case is that we get a "bit already cleared" message from + * ext4_free_inode(). The only reason we would point at a wrong inode is if + * e2fsck was run on this filesystem, and it must have already done the orphan + * inode cleanup for us, so we can safely abort without any further action. + */ +void ext4_orphan_cleanup(struct super_block *sb, struct ext4_super_block *es) +{ + unsigned int s_flags = sb->s_flags; + int nr_orphans = 0, nr_truncates = 0; + struct inode *inode; + int i, j; +#ifdef CONFIG_QUOTA + int quota_update = 0; +#endif + __le32 *bdata; + struct ext4_orphan_info *oi = &EXT4_SB(sb)->s_orphan_info; + int inodes_per_ob = ext4_inodes_per_orphan_block(sb); + + if (!es->s_last_orphan && ext4_orphan_file_empty(sb)) { + ext4_debug("no orphan inodes to clean up\n"); + return; + } + + if (bdev_read_only(sb->s_bdev)) { + ext4_msg(sb, KERN_ERR, "write access " + "unavailable, skipping orphan cleanup"); + return; + } + + /* Check if feature set would not allow a r/w mount */ + if (!ext4_feature_set_ok(sb, 0)) { + ext4_msg(sb, KERN_INFO, "Skipping orphan cleanup due to " + "unknown ROCOMPAT features"); + return; + } + + if (EXT4_SB(sb)->s_mount_state & EXT4_ERROR_FS) { + /* don't clear list on RO mount w/ errors */ + if (es->s_last_orphan && !(s_flags & SB_RDONLY)) { + ext4_msg(sb, KERN_INFO, "Errors on filesystem, " + "clearing orphan list."); + es->s_last_orphan = 0; + } + ext4_debug("Skipping orphan recovery on fs with errors.\n"); + return; + } + + if (s_flags & SB_RDONLY) { + ext4_msg(sb, KERN_INFO, "orphan cleanup on readonly fs"); + sb->s_flags &= ~SB_RDONLY; + } +#ifdef CONFIG_QUOTA + /* + * Turn on quotas which were not enabled for read-only mounts if + * filesystem has quota feature, so that they are updated correctly. + */ + if (ext4_has_feature_quota(sb) && (s_flags & SB_RDONLY)) { + int ret = ext4_enable_quotas(sb); + + if (!ret) + quota_update = 1; + else + ext4_msg(sb, KERN_ERR, + "Cannot turn on quotas: error %d", ret); + } + + /* Turn on journaled quotas used for old sytle */ + for (i = 0; i < EXT4_MAXQUOTAS; i++) { + if (EXT4_SB(sb)->s_qf_names[i]) { + int ret = ext4_quota_on_mount(sb, i); + + if (!ret) + quota_update = 1; + else + ext4_msg(sb, KERN_ERR, + "Cannot turn on journaled " + "quota: type %d: error %d", i, ret); + } + } +#endif + + while (es->s_last_orphan) { + /* + * We may have encountered an error during cleanup; if + * so, skip the rest. + */ + if (EXT4_SB(sb)->s_mount_state & EXT4_ERROR_FS) { + ext4_debug("Skipping orphan recovery on fs with errors.\n"); + es->s_last_orphan = 0; + break; + } + + inode = ext4_orphan_get(sb, le32_to_cpu(es->s_last_orphan)); + if (IS_ERR(inode)) { + es->s_last_orphan = 0; + break; + } + + list_add(&EXT4_I(inode)->i_orphan, &EXT4_SB(sb)->s_orphan); + ext4_process_orphan(inode, &nr_truncates, &nr_orphans); + } + + for (i = 0; i < oi->of_blocks; i++) { + bdata = (__le32 *)(oi->of_binfo[i].ob_bh->b_data); + for (j = 0; j < inodes_per_ob; j++) { + if (!bdata[j]) + continue; + inode = ext4_orphan_get(sb, le32_to_cpu(bdata[j])); + if (IS_ERR(inode)) + continue; + ext4_set_inode_state(inode, EXT4_STATE_ORPHAN_FILE); + EXT4_I(inode)->i_orphan_idx = i * inodes_per_ob + j; + ext4_process_orphan(inode, &nr_truncates, &nr_orphans); + } + } + + if (nr_orphans) + ext4_msg(sb, KERN_INFO, "%d orphan inode%s deleted", + nr_orphans, str_plural(nr_orphans)); + if (nr_truncates) + ext4_msg(sb, KERN_INFO, "%d truncate%s cleaned up", + nr_truncates, str_plural(nr_truncates)); +#ifdef CONFIG_QUOTA + /* Turn off quotas if they were enabled for orphan cleanup */ + if (quota_update) { + for (i = 0; i < EXT4_MAXQUOTAS; i++) { + if (sb_dqopt(sb)->files[i]) + dquot_quota_off(sb, i); + } + } +#endif + sb->s_flags = s_flags; /* Restore SB_RDONLY status */ +} + +void ext4_release_orphan_info(struct super_block *sb) +{ + int i; + struct ext4_orphan_info *oi = &EXT4_SB(sb)->s_orphan_info; + + if (!oi->of_blocks) + return; + for (i = 0; i < oi->of_blocks; i++) + brelse(oi->of_binfo[i].ob_bh); + kvfree(oi->of_binfo); +} + +static struct ext4_orphan_block_tail *ext4_orphan_block_tail( + struct super_block *sb, + struct buffer_head *bh) +{ + return (struct ext4_orphan_block_tail *)(bh->b_data + sb->s_blocksize - + sizeof(struct ext4_orphan_block_tail)); +} + +static int ext4_orphan_file_block_csum_verify(struct super_block *sb, + struct buffer_head *bh) +{ + __u32 calculated; + int inodes_per_ob = ext4_inodes_per_orphan_block(sb); + struct ext4_orphan_info *oi = &EXT4_SB(sb)->s_orphan_info; + struct ext4_orphan_block_tail *ot; + __le64 dsk_block_nr = cpu_to_le64(bh->b_blocknr); + + if (!ext4_has_feature_metadata_csum(sb)) + return 1; + + ot = ext4_orphan_block_tail(sb, bh); + calculated = ext4_chksum(oi->of_csum_seed, (__u8 *)&dsk_block_nr, + sizeof(dsk_block_nr)); + calculated = ext4_chksum(calculated, (__u8 *)bh->b_data, + inodes_per_ob * sizeof(__u32)); + return le32_to_cpu(ot->ob_checksum) == calculated; +} + +/* This gets called only when checksumming is enabled */ +void ext4_orphan_file_block_trigger(struct jbd2_buffer_trigger_type *triggers, + struct buffer_head *bh, + void *data, size_t size) +{ + struct super_block *sb = EXT4_TRIGGER(triggers)->sb; + __u32 csum; + int inodes_per_ob = ext4_inodes_per_orphan_block(sb); + struct ext4_orphan_info *oi = &EXT4_SB(sb)->s_orphan_info; + struct ext4_orphan_block_tail *ot; + __le64 dsk_block_nr = cpu_to_le64(bh->b_blocknr); + + csum = ext4_chksum(oi->of_csum_seed, (__u8 *)&dsk_block_nr, + sizeof(dsk_block_nr)); + csum = ext4_chksum(csum, (__u8 *)data, inodes_per_ob * sizeof(__u32)); + ot = ext4_orphan_block_tail(sb, bh); + ot->ob_checksum = cpu_to_le32(csum); +} + +int ext4_init_orphan_info(struct super_block *sb) +{ + struct ext4_orphan_info *oi = &EXT4_SB(sb)->s_orphan_info; + struct inode *inode; + int i, j; + int ret; + int free; + int loaded = 0; + __le32 *bdata; + int inodes_per_ob = ext4_inodes_per_orphan_block(sb); + struct ext4_orphan_block_tail *ot; + ino_t orphan_ino = le32_to_cpu(EXT4_SB(sb)->s_es->s_orphan_file_inum); + + if (!ext4_has_feature_orphan_file(sb)) + return 0; + + inode = ext4_iget(sb, orphan_ino, EXT4_IGET_SPECIAL); + if (IS_ERR(inode)) { + ext4_msg(sb, KERN_ERR, "get orphan inode failed"); + return PTR_ERR(inode); + } + /* + * This is just an artificial limit to prevent corrupted fs from + * consuming absurd amounts of memory when pinning blocks of orphan + * file in memory. + */ + if (inode->i_size > (EXT4_MAX_ORPHAN_FILE_BLOCKS << inode->i_blkbits)) { + ext4_msg(sb, KERN_ERR, "orphan file too big: %llu", + (unsigned long long)inode->i_size); + ret = -EFSCORRUPTED; + goto out_put; + } + oi->of_blocks = inode->i_size >> sb->s_blocksize_bits; + oi->of_csum_seed = EXT4_I(inode)->i_csum_seed; + oi->of_binfo = kvmalloc_objs(struct ext4_orphan_block, oi->of_blocks); + if (!oi->of_binfo) { + ret = -ENOMEM; + goto out_put; + } + for (i = 0; i < oi->of_blocks; i++) { + oi->of_binfo[i].ob_bh = ext4_bread(NULL, inode, i, 0); + if (IS_ERR(oi->of_binfo[i].ob_bh)) { + ret = PTR_ERR(oi->of_binfo[i].ob_bh); + goto out_free; + } + if (!oi->of_binfo[i].ob_bh) { + ret = -EIO; + goto out_free; + } + loaded++; + ot = ext4_orphan_block_tail(sb, oi->of_binfo[i].ob_bh); + if (le32_to_cpu(ot->ob_magic) != EXT4_ORPHAN_BLOCK_MAGIC) { + ext4_error(sb, "orphan file block %d: bad magic", i); + ret = -EIO; + goto out_free; + } + if (!ext4_orphan_file_block_csum_verify(sb, + oi->of_binfo[i].ob_bh)) { + ext4_error(sb, "orphan file block %d: bad checksum", i); + ret = -EIO; + goto out_free; + } + bdata = (__le32 *)(oi->of_binfo[i].ob_bh->b_data); + free = 0; + for (j = 0; j < inodes_per_ob; j++) + if (bdata[j] == 0) + free++; + atomic_set(&oi->of_binfo[i].ob_free_entries, free); + } + iput(inode); + return 0; +out_free: + while (loaded > 0) { + loaded--; + brelse(oi->of_binfo[loaded].ob_bh); + } + kvfree(oi->of_binfo); +out_put: + iput(inode); + return ret; +} + +int ext4_orphan_file_empty(struct super_block *sb) +{ + struct ext4_orphan_info *oi = &EXT4_SB(sb)->s_orphan_info; + int i; + int inodes_per_ob = ext4_inodes_per_orphan_block(sb); + + if (!ext4_has_feature_orphan_file(sb)) + return 1; + for (i = 0; i < oi->of_blocks; i++) + if (atomic_read(&oi->of_binfo[i].ob_free_entries) != + inodes_per_ob) + return 0; + return 1; +} diff --git a/fs/ext4/page-io.c b/fs/ext4/page-io.c new file mode 100644 index 000000000..0236b6b97 --- /dev/null +++ b/fs/ext4/page-io.c @@ -0,0 +1,549 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext4/page-io.c + * + * This contains the new page_io functions for ext4 + * + * Written by Theodore Ts'o, 2010. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ext4_jbd2.h" +#include "xattr.h" +#include "acl.h" + +static struct kmem_cache *io_end_cachep; +static struct kmem_cache *io_end_vec_cachep; + +int __init ext4_init_pageio(void) +{ + io_end_cachep = KMEM_CACHE(ext4_io_end, SLAB_RECLAIM_ACCOUNT); + if (io_end_cachep == NULL) + return -ENOMEM; + + io_end_vec_cachep = KMEM_CACHE(ext4_io_end_vec, 0); + if (io_end_vec_cachep == NULL) { + kmem_cache_destroy(io_end_cachep); + return -ENOMEM; + } + return 0; +} + +void ext4_exit_pageio(void) +{ + kmem_cache_destroy(io_end_cachep); + kmem_cache_destroy(io_end_vec_cachep); +} + +struct ext4_io_end_vec *ext4_alloc_io_end_vec(ext4_io_end_t *io_end) +{ + struct ext4_io_end_vec *io_end_vec; + + io_end_vec = kmem_cache_zalloc(io_end_vec_cachep, GFP_NOFS); + if (!io_end_vec) + return ERR_PTR(-ENOMEM); + INIT_LIST_HEAD(&io_end_vec->list); + list_add_tail(&io_end_vec->list, &io_end->list_vec); + return io_end_vec; +} + +static void ext4_free_io_end_vec(ext4_io_end_t *io_end) +{ + struct ext4_io_end_vec *io_end_vec, *tmp; + + if (list_empty(&io_end->list_vec)) + return; + list_for_each_entry_safe(io_end_vec, tmp, &io_end->list_vec, list) { + list_del(&io_end_vec->list); + kmem_cache_free(io_end_vec_cachep, io_end_vec); + } +} + +struct ext4_io_end_vec *ext4_last_io_end_vec(ext4_io_end_t *io_end) +{ + BUG_ON(list_empty(&io_end->list_vec)); + return list_last_entry(&io_end->list_vec, struct ext4_io_end_vec, list); +} + +/* + * Print an buffer I/O error compatible with the fs/buffer.c. This + * provides compatibility with dmesg scrapers that look for a specific + * buffer I/O error message. We really need a unified error reporting + * structure to userspace ala Digital Unix's uerf system, but it's + * probably not going to happen in my lifetime, due to LKML politics... + */ +static void buffer_io_error(struct buffer_head *bh) +{ + printk_ratelimited(KERN_ERR "Buffer I/O error on device %pg, logical block %llu\n", + bh->b_bdev, + (unsigned long long)bh->b_blocknr); +} + +static void ext4_finish_bio(struct bio *bio) +{ + struct folio_iter fi; + + bio_for_each_folio_all(fi, bio) { + struct folio *folio = fi.folio; + struct buffer_head *bh, *head; + size_t bio_start = fi.offset; + size_t bio_end = bio_start + fi.length; + unsigned under_io = 0; + unsigned long flags; + + if (bio->bi_status) { + int err = blk_status_to_errno(bio->bi_status); + mapping_set_error(folio->mapping, err); + } + bh = head = folio_buffers(folio); + /* + * We check all buffers in the folio under b_uptodate_lock + * to avoid races with other end io clearing async_write flags + */ + spin_lock_irqsave(&head->b_uptodate_lock, flags); + do { + if (bh_offset(bh) < bio_start || + bh_offset(bh) + bh->b_size > bio_end) { + if (buffer_async_write(bh)) + under_io++; + continue; + } + clear_buffer_async_write(bh); + if (bio->bi_status) { + set_buffer_write_io_error(bh); + buffer_io_error(bh); + } + } while ((bh = bh->b_this_page) != head); + spin_unlock_irqrestore(&head->b_uptodate_lock, flags); + if (!under_io) + folio_end_writeback(folio); + } +} + +static void ext4_release_io_end(ext4_io_end_t *io_end) +{ + struct bio *bio, *next_bio; + + BUG_ON(!list_empty(&io_end->list)); + BUG_ON(io_end->flag & EXT4_IO_END_UNWRITTEN); + WARN_ON(io_end->handle); + + for (bio = io_end->bio; bio; bio = next_bio) { + next_bio = bio->bi_private; + ext4_finish_bio(bio); + bio_put(bio); + } + ext4_free_io_end_vec(io_end); + kmem_cache_free(io_end_cachep, io_end); +} + +/* + * On successful IO, check a range of space and convert unwritten extents to + * written. On IO failure, check if journal abort is needed. Note that + * we are protected from truncate touching same part of extent tree by the + * fact that truncate code waits for all DIO to finish (thus exclusion from + * direct IO is achieved) and also waits for writeback to complete. Thus we + * cannot get to ext4_ext_truncate() before all IOs overlapping that range are + * completed (happens from ext4_free_ioend()). + */ +static int ext4_end_io_end(ext4_io_end_t *io_end) +{ + struct inode *inode = io_end->inode; + handle_t *handle = io_end->handle; + struct super_block *sb = inode->i_sb; + int ret = 0; + + ext4_debug("ext4_end_io_nolock: io_end 0x%p from inode %llu,list->next 0x%p," + "list->prev 0x%p\n", + io_end, inode->i_ino, io_end->list.next, io_end->list.prev); + + /* + * Do not convert the unwritten extents if data writeback fails, + * or stale data may be exposed. + */ + io_end->handle = NULL; /* Following call will use up the handle */ + if (unlikely(io_end->flag & EXT4_IO_END_FAILED)) { + ret = -EIO; + if (handle) + jbd2_journal_free_reserved(handle); + + if (test_opt(sb, DATA_ERR_ABORT)) + jbd2_journal_abort(EXT4_SB(sb)->s_journal, ret); + } else { + ret = ext4_convert_unwritten_io_end_vec(handle, io_end); + } + if (ret < 0 && !ext4_emergency_state(sb) && + io_end->flag & EXT4_IO_END_UNWRITTEN) { + ext4_msg(sb, KERN_EMERG, + "failed to convert unwritten extents to written " + "extents -- potential data loss! " + "(inode %llu, error %d)", inode->i_ino, ret); + } + + ext4_clear_io_unwritten_flag(io_end); + ext4_release_io_end(io_end); + return ret; +} + +static void dump_completed_IO(struct inode *inode, struct list_head *head) +{ +#ifdef EXT4FS_DEBUG + struct list_head *cur, *before, *after; + ext4_io_end_t *io_end, *io_end0, *io_end1; + + if (list_empty(head)) + return; + + ext4_debug("Dump inode %llu completed io list\n", inode->i_ino); + list_for_each_entry(io_end, head, list) { + cur = &io_end->list; + before = cur->prev; + io_end0 = container_of(before, ext4_io_end_t, list); + after = cur->next; + io_end1 = container_of(after, ext4_io_end_t, list); + + ext4_debug("io 0x%p from inode %llu,prev 0x%p,next 0x%p\n", + io_end, inode->i_ino, io_end0, io_end1); + } +#endif +} + +static bool ext4_io_end_defer_completion(ext4_io_end_t *io_end) +{ + if (io_end->flag & EXT4_IO_END_UNWRITTEN && + !list_empty(&io_end->list_vec)) + return true; + if (test_opt(io_end->inode->i_sb, DATA_ERR_ABORT) && + io_end->flag & EXT4_IO_END_FAILED && + !ext4_emergency_state(io_end->inode->i_sb)) + return true; + return false; +} + +/* Add the io_end to per-inode completed end_io list. */ +static void ext4_add_complete_io(ext4_io_end_t *io_end) +{ + struct ext4_inode_info *ei = EXT4_I(io_end->inode); + struct ext4_sb_info *sbi = EXT4_SB(io_end->inode->i_sb); + struct workqueue_struct *wq; + unsigned long flags; + + /* Only reserved conversions or pending IO errors will enter here. */ + WARN_ON(!(io_end->flag & EXT4_IO_END_DEFER_COMPLETION)); + WARN_ON(io_end->flag & EXT4_IO_END_UNWRITTEN && + !io_end->handle && sbi->s_journal); + WARN_ON(!io_end->bio); + + spin_lock_irqsave(&ei->i_completed_io_lock, flags); + wq = sbi->rsv_conversion_wq; + if (list_empty(&ei->i_rsv_conversion_list)) + queue_work(wq, &ei->i_rsv_conversion_work); + list_add_tail(&io_end->list, &ei->i_rsv_conversion_list); + spin_unlock_irqrestore(&ei->i_completed_io_lock, flags); +} + +static int ext4_do_flush_completed_IO(struct inode *inode, + struct list_head *head) +{ + ext4_io_end_t *io_end; + struct list_head unwritten; + unsigned long flags; + struct ext4_inode_info *ei = EXT4_I(inode); + int err, ret = 0; + + spin_lock_irqsave(&ei->i_completed_io_lock, flags); + dump_completed_IO(inode, head); + list_replace_init(head, &unwritten); + spin_unlock_irqrestore(&ei->i_completed_io_lock, flags); + + while (!list_empty(&unwritten)) { + io_end = list_entry(unwritten.next, ext4_io_end_t, list); + BUG_ON(!(io_end->flag & EXT4_IO_END_DEFER_COMPLETION)); + list_del_init(&io_end->list); + + err = ext4_end_io_end(io_end); + if (unlikely(!ret && err)) + ret = err; + } + return ret; +} + +/* + * Used to convert unwritten extents to written extents upon IO completion, + * or used to abort the journal upon IO errors. + */ +void ext4_end_io_rsv_work(struct work_struct *work) +{ + struct ext4_inode_info *ei = container_of(work, struct ext4_inode_info, + i_rsv_conversion_work); + ext4_do_flush_completed_IO(&ei->vfs_inode, &ei->i_rsv_conversion_list); +} + +ext4_io_end_t *ext4_init_io_end(struct inode *inode, gfp_t flags) +{ + ext4_io_end_t *io_end = kmem_cache_zalloc(io_end_cachep, flags); + + if (io_end) { + io_end->inode = inode; + INIT_LIST_HEAD(&io_end->list); + INIT_LIST_HEAD(&io_end->list_vec); + refcount_set(&io_end->count, 1); + } + return io_end; +} + +void ext4_put_io_end_defer(ext4_io_end_t *io_end) +{ + if (refcount_dec_and_test(&io_end->count)) { + if (ext4_io_end_defer_completion(io_end)) + return ext4_add_complete_io(io_end); + + ext4_release_io_end(io_end); + } +} + +int ext4_put_io_end(ext4_io_end_t *io_end) +{ + if (refcount_dec_and_test(&io_end->count)) { + if (ext4_io_end_defer_completion(io_end)) + return ext4_end_io_end(io_end); + + ext4_release_io_end(io_end); + } + return 0; +} + +ext4_io_end_t *ext4_get_io_end(ext4_io_end_t *io_end) +{ + refcount_inc(&io_end->count); + return io_end; +} + +/* BIO completion function for page writeback */ +static void ext4_end_bio(struct bio *bio) +{ + ext4_io_end_t *io_end = bio->bi_private; + sector_t bi_sector = bio->bi_iter.bi_sector; + + if (WARN_ONCE(!io_end, "io_end is NULL: %pg: sector %Lu len %u err %d\n", + bio->bi_bdev, + (long long) bio->bi_iter.bi_sector, + (unsigned) bio_sectors(bio), + bio->bi_status)) { + ext4_finish_bio(bio); + bio_put(bio); + return; + } + bio->bi_end_io = NULL; + + if (bio->bi_status) { + struct inode *inode = io_end->inode; + + ext4_warning(inode->i_sb, "I/O error %d writing to inode %llu " + "starting block %llu)", + bio->bi_status, inode->i_ino, + (unsigned long long) + bi_sector >> (inode->i_blkbits - 9)); + io_end->flag |= EXT4_IO_END_FAILED; + mapping_set_error(inode->i_mapping, + blk_status_to_errno(bio->bi_status)); + } + + if (ext4_io_end_defer_completion(io_end)) { + /* + * Link bio into list hanging from io_end. We have to do it + * atomically as bio completions can be racing against each + * other. + */ + bio->bi_private = xchg(&io_end->bio, bio); + ext4_put_io_end_defer(io_end); + } else { + /* + * Drop io_end reference early. Inode can get freed once + * we finish the bio. + */ + ext4_put_io_end_defer(io_end); + ext4_finish_bio(bio); + bio_put(bio); + } +} + +void ext4_io_submit(struct ext4_io_submit *io) +{ + struct bio *bio = io->io_bio; + + if (bio) { + if (io->io_wbc->sync_mode == WB_SYNC_ALL) + io->io_bio->bi_opf |= REQ_SYNC; + blk_crypto_submit_bio(io->io_bio); + } + io->io_bio = NULL; +} + +void ext4_io_submit_init(struct ext4_io_submit *io, + struct writeback_control *wbc) +{ + io->io_wbc = wbc; + io->io_bio = NULL; + io->io_end = NULL; +} + +static void io_submit_init_bio(struct ext4_io_submit *io, + struct inode *inode, + struct folio *folio, + struct buffer_head *bh) +{ + struct bio *bio; + + /* + * bio_alloc will _always_ be able to allocate a bio if + * __GFP_DIRECT_RECLAIM is set, see comments for bio_alloc_bioset(). + */ + bio = bio_alloc(bh->b_bdev, BIO_MAX_VECS, REQ_OP_WRITE, GFP_NOIO); + fscrypt_set_bio_crypt_ctx(bio, inode, folio_pos(folio) + bh_offset(bh), + GFP_NOIO); + bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9); + bio->bi_end_io = ext4_end_bio; + bio->bi_private = ext4_get_io_end(io->io_end); + bio->bi_write_hint = inode->i_write_hint; + io->io_bio = bio; + io->io_next_block = bh->b_blocknr; + wbc_init_bio(io->io_wbc, bio); +} + +static bool io_submit_need_new_bio(struct ext4_io_submit *io, + struct inode *inode, + struct folio *folio, + struct buffer_head *bh) +{ + if (bh->b_blocknr != io->io_next_block) + return true; + if (!fscrypt_mergeable_bio(io->io_bio, inode, + folio_pos(folio) + bh_offset(bh))) + return true; + return false; +} + +static void io_submit_add_bh(struct ext4_io_submit *io, + struct inode *inode, + struct folio *folio, + struct buffer_head *bh) +{ + if (io->io_bio && io_submit_need_new_bio(io, inode, folio, bh)) { +submit_and_retry: + ext4_io_submit(io); + } + if (io->io_bio == NULL) + io_submit_init_bio(io, inode, folio, bh); + if (!bio_add_folio(io->io_bio, folio, bh->b_size, bh_offset(bh))) + goto submit_and_retry; + wbc_account_cgroup_owner(io->io_wbc, folio, bh->b_size); + io->io_next_block++; +} + +void ext4_bio_write_folio(struct ext4_io_submit *io, struct folio *folio, + size_t len) +{ + struct inode *inode = folio->mapping->host; + unsigned block_start; + struct buffer_head *bh, *head; + int nr_to_submit = 0; + struct writeback_control *wbc = io->io_wbc; + bool keep_towrite = false; + + BUG_ON(!folio_test_locked(folio)); + BUG_ON(folio_test_writeback(folio)); + + /* + * Comments copied from block_write_full_folio: + * + * The folio straddles i_size. It must be zeroed out on each and every + * writepage invocation because it may be mmapped. "A file is mapped + * in multiples of the page size. For a file that is not a multiple of + * the page size, the remaining memory is zeroed when mapped, and + * writes to that region are not written out to the file." + */ + if (len < folio_size(folio)) + folio_zero_segment(folio, len, folio_size(folio)); + /* + * In the first loop we prepare and mark buffers to submit. We have to + * mark all buffers in the folio before submitting so that + * folio_end_writeback() cannot be called from ext4_end_bio() when IO + * on the first buffer finishes and we are still working on submitting + * the second buffer. + */ + bh = head = folio_buffers(folio); + do { + block_start = bh_offset(bh); + if (block_start >= len) { + clear_buffer_dirty(bh); + set_buffer_uptodate(bh); + continue; + } + if (!buffer_dirty(bh) || buffer_delay(bh) || + !buffer_mapped(bh) || buffer_unwritten(bh)) { + /* A hole? We can safely clear the dirty bit */ + if (!buffer_mapped(bh)) + clear_buffer_dirty(bh); + /* + * Keeping dirty some buffer we cannot write? Make sure + * to redirty the folio and keep TOWRITE tag so that + * racing WB_SYNC_ALL writeback does not skip the folio. + * This happens e.g. when doing writeout for + * transaction commit or when journalled data is not + * yet committed. + */ + if (buffer_dirty(bh) || + (buffer_jbd(bh) && buffer_jbddirty(bh))) { + if (!folio_test_dirty(folio)) + folio_redirty_for_writepage(wbc, folio); + keep_towrite = true; + } + continue; + } + if (buffer_new(bh)) + clear_buffer_new(bh); + set_buffer_async_write(bh); + clear_buffer_dirty(bh); + nr_to_submit++; + } while ((bh = bh->b_this_page) != head); + + if (!nr_to_submit) { + /* + * We have nothing to submit. Just cycle the folio through + * writeback state to properly update xarray tags. + */ + __folio_start_writeback(folio, keep_towrite); + folio_end_writeback(folio); + return; + } + + bh = head = folio_buffers(folio); + + __folio_start_writeback(folio, keep_towrite); + + /* Now submit buffers to write */ + do { + if (!buffer_async_write(bh)) + continue; + io_submit_add_bh(io, inode, folio, bh); + } while ((bh = bh->b_this_page) != head); +} diff --git a/fs/ext4/readpage.c b/fs/ext4/readpage.c new file mode 100644 index 000000000..c7b6cdb2e --- /dev/null +++ b/fs/ext4/readpage.c @@ -0,0 +1,386 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext4/readpage.c + * + * Copyright (C) 2002, Linus Torvalds. + * Copyright (C) 2015, Google, Inc. + * + * This was originally taken from fs/mpage.c + * + * The ext4_mpage_readpages() function here is intended to + * replace mpage_readahead() in the general case, not just for + * encrypted files. It has some limitations (see below), where it + * will fall back to read_block_full_page(), but these limitations + * should only be hit when page_size != block_size. + * + * This will allow us to attach a callback function to support ext4 + * encryption. + * + * If anything unusual happens, such as: + * + * - encountering a page which has buffers + * - encountering a page which has a non-hole after a hole + * - encountering a page with non-contiguous blocks + * + * then this code just gives up and calls the buffer_head-based read function. + * It does handle a page which has holes at the end - that is a common case: + * the end-of-file on blocksize < PAGE_SIZE setups. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ext4.h" +#include + +#define NUM_VERITY_WORKS 128 + +static struct kmem_cache *ext4_verity_work_cache; +static mempool_t *ext4_verity_work_pool; + +struct ext4_verity_work { + struct bio *bio; + struct fsverity_info *vi; + struct work_struct work; +}; + +static void __read_end_io(struct bio *bio) +{ + struct folio_iter fi; + + bio_for_each_folio_all(fi, bio) + folio_end_read(fi.folio, bio->bi_status == 0); + if (bio->bi_private) + mempool_free(bio->bi_private, ext4_verity_work_pool); + bio_put(bio); +} + +static void verity_work(struct work_struct *work) +{ + struct ext4_verity_work *ctx = + container_of(work, struct ext4_verity_work, work); + struct bio *bio = ctx->bio; + struct fsverity_info *vi = ctx->vi; + + /* + * Free the ext4_verity_work right away, since it's no longer needed. + * This relieves the pressure on the mempool as much as possible. + */ + mempool_free(ctx, ext4_verity_work_pool); + bio->bi_private = NULL; + + fsverity_verify_bio(vi, bio); + + __read_end_io(bio); +} + +/* + * I/O completion handler for multipage BIOs. + * + * The mpage code never puts partial pages into a BIO (except for end-of-file). + * If a page does not map to a contiguous run of blocks then it simply falls + * back to block_read_full_folio(). + * + * Why is this? If a page's completion depends on a number of different BIOs + * which can complete in any order (or at the same time) then determining the + * status of that page is hard. See end_buffer_async_read() for the details. + * There is no point in duplicating all that complexity. + */ +static void mpage_end_io(struct bio *bio) +{ + if (IS_ENABLED(CONFIG_FS_VERITY) && bio->bi_private && + !bio->bi_status) { + struct ext4_verity_work *ctx = bio->bi_private; + + INIT_WORK(&ctx->work, verity_work); + fsverity_enqueue_verify_work(&ctx->work); + return; + } + __read_end_io(bio); +} + +static void ext4_set_verity_work(struct bio *bio, struct fsverity_info *vi) +{ + if (vi) { + /* Due to the mempool, this never fails. */ + struct ext4_verity_work *ctx = + mempool_alloc(ext4_verity_work_pool, GFP_NOFS); + + ctx->bio = bio; + ctx->vi = vi; + bio->bi_private = ctx; + } +} + +static inline loff_t ext4_readpage_limit(struct inode *inode) +{ + if (IS_ENABLED(CONFIG_FS_VERITY) && IS_VERITY(inode)) + return inode->i_sb->s_maxbytes; + + return i_size_read(inode); +} + +static int ext4_mpage_readpages(struct inode *inode, struct fsverity_info *vi, + struct readahead_control *rac, struct folio *folio) +{ + struct bio *bio = NULL; + sector_t last_block_in_bio = 0; + const unsigned blkbits = inode->i_blkbits; + const unsigned blocksize = 1 << blkbits; + sector_t block_in_file; + sector_t last_block; + sector_t last_block_in_file; + sector_t first_block; + loff_t pos; + unsigned page_block; + struct block_device *bdev = inode->i_sb->s_bdev; + int length; + unsigned relative_block = 0; + struct ext4_map_blocks map; + unsigned int nr_pages, folio_pages; + + map.m_pblk = 0; + map.m_lblk = 0; + map.m_len = 0; + map.m_flags = 0; + + nr_pages = rac ? readahead_count(rac) : folio_nr_pages(folio); + for (; nr_pages; nr_pages -= folio_pages) { + int fully_mapped = 1; + unsigned int first_hole; + unsigned int blocks_per_folio; + + if (rac) + folio = readahead_folio(rac); + + folio_pages = folio_nr_pages(folio); + prefetchw(&folio->flags); + + if (folio_buffers(folio)) + goto confused; + + blocks_per_folio = folio_size(folio) >> blkbits; + first_hole = blocks_per_folio; + pos = folio_pos(folio); + block_in_file = pos >> blkbits; + last_block = EXT4_PG_TO_LBLK(inode, folio->index + nr_pages); + last_block_in_file = (ext4_readpage_limit(inode) + + blocksize - 1) >> blkbits; + if (last_block > last_block_in_file) + last_block = last_block_in_file; + page_block = 0; + + /* + * Map blocks using the previous result first. + */ + if ((map.m_flags & EXT4_MAP_MAPPED) && + block_in_file > map.m_lblk && + block_in_file < (map.m_lblk + map.m_len)) { + unsigned map_offset = block_in_file - map.m_lblk; + unsigned last = map.m_len - map_offset; + + first_block = map.m_pblk + map_offset; + for (relative_block = 0; ; relative_block++) { + if (relative_block == last) { + /* needed? */ + map.m_flags &= ~EXT4_MAP_MAPPED; + break; + } + if (page_block == blocks_per_folio) + break; + page_block++; + block_in_file++; + } + } + + /* + * Then do more ext4_map_blocks() calls until we are + * done with this folio. + */ + while (page_block < blocks_per_folio) { + if (block_in_file < last_block) { + map.m_lblk = block_in_file; + map.m_len = last_block - block_in_file; + + if (ext4_map_blocks(NULL, inode, &map, 0) < 0) { + set_error_page: + folio_zero_segment(folio, 0, + folio_size(folio)); + folio_unlock(folio); + goto next_page; + } + } + if ((map.m_flags & EXT4_MAP_MAPPED) == 0) { + fully_mapped = 0; + if (first_hole == blocks_per_folio) + first_hole = page_block; + page_block++; + block_in_file++; + continue; + } + if (first_hole != blocks_per_folio) + goto confused; /* hole -> non-hole */ + + /* Contiguous blocks? */ + if (!page_block) + first_block = map.m_pblk; + else if (first_block + page_block != map.m_pblk) + goto confused; + for (relative_block = 0; ; relative_block++) { + if (relative_block == map.m_len) { + /* needed? */ + map.m_flags &= ~EXT4_MAP_MAPPED; + break; + } else if (page_block == blocks_per_folio) + break; + page_block++; + block_in_file++; + } + } + if (first_hole != blocks_per_folio) { + folio_zero_segment(folio, first_hole << blkbits, + folio_size(folio)); + if (first_hole == 0) { + if (vi && !fsverity_verify_folio(vi, folio)) + goto set_error_page; + folio_end_read(folio, true); + continue; + } + } else if (fully_mapped) { + folio_set_mappedtodisk(folio); + } + + /* + * This folio will go to BIO. Do we need to send this + * BIO off first? + */ + if (bio && (last_block_in_bio != first_block - 1 || + !fscrypt_mergeable_bio(bio, inode, pos))) { + submit_and_realloc: + blk_crypto_submit_bio(bio); + bio = NULL; + } + if (bio == NULL) { + /* + * bio_alloc will _always_ be able to allocate a bio if + * __GFP_DIRECT_RECLAIM is set, see bio_alloc_bioset(). + */ + bio = bio_alloc(bdev, bio_max_segs(nr_pages), + REQ_OP_READ, GFP_KERNEL); + fscrypt_set_bio_crypt_ctx(bio, inode, pos, GFP_KERNEL); + ext4_set_verity_work(bio, vi); + bio->bi_iter.bi_sector = first_block << (blkbits - 9); + bio->bi_end_io = mpage_end_io; + if (rac) + bio->bi_opf |= REQ_RAHEAD; + } + + length = first_hole << blkbits; + if (!bio_add_folio(bio, folio, length, 0)) + goto submit_and_realloc; + + if (((map.m_flags & EXT4_MAP_BOUNDARY) && + (relative_block == map.m_len)) || + (first_hole != blocks_per_folio)) { + blk_crypto_submit_bio(bio); + bio = NULL; + } else + last_block_in_bio = first_block + blocks_per_folio - 1; + continue; + confused: + if (bio) { + blk_crypto_submit_bio(bio); + bio = NULL; + } + if (!folio_test_uptodate(folio)) + block_read_full_folio(folio, ext4_get_block); + else + folio_unlock(folio); +next_page: + ; /* A label shall be followed by a statement until C23 */ + } + if (bio) + blk_crypto_submit_bio(bio); + return 0; +} + +int ext4_read_folio(struct file *file, struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + struct fsverity_info *vi = NULL; + int ret; + + trace_ext4_read_folio(inode, folio); + + if (ext4_has_inline_data(inode)) { + ret = ext4_readpage_inline(inode, folio); + if (ret != -EAGAIN) + return ret; + } + + if (folio->index < DIV_ROUND_UP(inode->i_size, PAGE_SIZE)) + vi = fsverity_get_info(inode); + if (vi) + fsverity_readahead(vi, folio->index, folio_nr_pages(folio)); + return ext4_mpage_readpages(inode, vi, NULL, folio); +} + +void ext4_readahead(struct readahead_control *rac) +{ + struct inode *inode = rac->mapping->host; + struct fsverity_info *vi = NULL; + + /* If the file has inline data, no need to do readahead. */ + if (ext4_has_inline_data(inode)) + return; + + if (readahead_index(rac) < DIV_ROUND_UP(inode->i_size, PAGE_SIZE)) + vi = fsverity_get_info(inode); + if (vi) + fsverity_readahead(vi, readahead_index(rac), + readahead_count(rac)); + ext4_mpage_readpages(inode, vi, rac, NULL); +} + +int __init ext4_init_verity_caches(void) +{ + if (!IS_ENABLED(CONFIG_FS_VERITY)) + return 0; + ext4_verity_work_cache = + KMEM_CACHE(ext4_verity_work, SLAB_RECLAIM_ACCOUNT); + + if (!ext4_verity_work_cache) + goto fail; + ext4_verity_work_pool = mempool_create_slab_pool( + NUM_VERITY_WORKS, ext4_verity_work_cache); + if (!ext4_verity_work_pool) + goto fail_free_cache; + return 0; + +fail_free_cache: + kmem_cache_destroy(ext4_verity_work_cache); +fail: + return -ENOMEM; +} + +void ext4_exit_verity_caches(void) +{ + if (!IS_ENABLED(CONFIG_FS_VERITY)) + return; + mempool_destroy(ext4_verity_work_pool); + kmem_cache_destroy(ext4_verity_work_cache); +} diff --git a/fs/ext4/resize.c b/fs/ext4/resize.c new file mode 100644 index 000000000..2c5b851c5 --- /dev/null +++ b/fs/ext4/resize.c @@ -0,0 +1,2192 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext4/resize.c + * + * Support for resizing an ext4 filesystem while it is mounted. + * + * Copyright (C) 2001, 2002 Andreas Dilger + * + * This could probably be made into a module, because it is not often in use. + */ + + +#include +#include +#include + +#include "ext4_jbd2.h" + +struct ext4_rcu_ptr { + struct rcu_head rcu; + void *ptr; +}; + +static void ext4_rcu_ptr_callback(struct rcu_head *head) +{ + struct ext4_rcu_ptr *ptr; + + ptr = container_of(head, struct ext4_rcu_ptr, rcu); + kvfree(ptr->ptr); + kfree(ptr); +} + +void ext4_kvfree_array_rcu(void *to_free) +{ + struct ext4_rcu_ptr *ptr = kzalloc_obj(*ptr); + + if (ptr) { + ptr->ptr = to_free; + call_rcu(&ptr->rcu, ext4_rcu_ptr_callback); + return; + } + synchronize_rcu(); + kvfree(to_free); +} + +int ext4_resize_begin(struct super_block *sb) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + int ret = 0; + + if (!capable(CAP_SYS_RESOURCE)) + return -EPERM; + + /* + * If the reserved GDT blocks is non-zero, the resize_inode feature + * should always be set. + */ + if (sbi->s_es->s_reserved_gdt_blocks && + !ext4_has_feature_resize_inode(sb)) { + ext4_error(sb, "resize_inode disabled but reserved GDT blocks non-zero"); + return -EFSCORRUPTED; + } + + /* + * If we are not using the primary superblock/GDT copy don't resize, + * because the user tools have no way of handling this. Probably a + * bad time to do it anyways. + */ + if (EXT4_B2C(sbi, sbi->s_sbh->b_blocknr) != + le32_to_cpu(sbi->s_es->s_first_data_block)) { + ext4_warning(sb, "won't resize using backup superblock at %llu", + (unsigned long long)sbi->s_sbh->b_blocknr); + return -EPERM; + } + + /* + * We are not allowed to do online-resizing on a filesystem mounted + * with error, because it can destroy the filesystem easily. + */ + if (sbi->s_mount_state & EXT4_ERROR_FS) { + ext4_warning(sb, "There are errors in the filesystem, " + "so online resizing is not allowed"); + return -EPERM; + } + + if (ext4_has_feature_sparse_super2(sb)) { + ext4_msg(sb, KERN_ERR, "Online resizing not supported with sparse_super2"); + return -EOPNOTSUPP; + } + + if (test_and_set_bit_lock(EXT4_FLAGS_RESIZING, + &sbi->s_ext4_flags)) + ret = -EBUSY; + + return ret; +} + +int ext4_resize_end(struct super_block *sb, bool update_backups) +{ + clear_bit_unlock(EXT4_FLAGS_RESIZING, &EXT4_SB(sb)->s_ext4_flags); + smp_mb__after_atomic(); + if (update_backups) + return ext4_update_overhead(sb, true); + return 0; +} + +static ext4_grpblk_t ext4_group_overhead_blocks(struct super_block *sb, + ext4_group_t group) { + ext4_grpblk_t overhead; + overhead = ext4_bg_num_gdb(sb, group); + if (ext4_bg_has_super(sb, group)) + overhead += 1 + + le16_to_cpu(EXT4_SB(sb)->s_es->s_reserved_gdt_blocks); + return overhead; +} + +#define outside(b, first, last) ((b) < (first) || (b) >= (last)) +#define inside(b, first, last) ((b) >= (first) && (b) < (last)) + +static int verify_group_input(struct super_block *sb, + struct ext4_new_group_data *input) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_super_block *es = sbi->s_es; + ext4_fsblk_t start = ext4_blocks_count(es); + ext4_fsblk_t end = start + input->blocks_count; + ext4_group_t group = input->group; + ext4_fsblk_t itend = input->inode_table + sbi->s_itb_per_group; + unsigned overhead; + ext4_fsblk_t metaend; + struct buffer_head *bh = NULL; + ext4_grpblk_t free_blocks_count, offset; + int err = -EINVAL; + + if (group != sbi->s_groups_count) { + ext4_warning(sb, "Cannot add at group %u (only %u groups)", + input->group, sbi->s_groups_count); + return -EINVAL; + } + + overhead = ext4_group_overhead_blocks(sb, group); + metaend = start + overhead; + free_blocks_count = input->blocks_count - 2 - overhead - + sbi->s_itb_per_group; + input->free_clusters_count = EXT4_B2C(sbi, free_blocks_count); + + if (test_opt(sb, DEBUG)) + printk(KERN_DEBUG "EXT4-fs: adding %s group %u: %u blocks " + "(%d free, %u reserved)\n", + ext4_bg_has_super(sb, input->group) ? "normal" : + "no-super", input->group, input->blocks_count, + free_blocks_count, input->reserved_blocks); + + ext4_get_group_no_and_offset(sb, start, NULL, &offset); + if (offset != 0) + ext4_warning(sb, "Last group not full"); + else if (input->reserved_blocks > input->blocks_count / 5) + ext4_warning(sb, "Reserved blocks too high (%u)", + input->reserved_blocks); + else if (free_blocks_count < 0) + ext4_warning(sb, "Bad blocks count %u", + input->blocks_count); + else if (IS_ERR(bh = ext4_sb_bread(sb, end - 1, 0))) { + err = PTR_ERR(bh); + bh = NULL; + ext4_warning(sb, "Cannot read last block (%llu)", + end - 1); + } else if (outside(input->block_bitmap, start, end)) + ext4_warning(sb, "Block bitmap not in group (block %llu)", + (unsigned long long)input->block_bitmap); + else if (outside(input->inode_bitmap, start, end)) + ext4_warning(sb, "Inode bitmap not in group (block %llu)", + (unsigned long long)input->inode_bitmap); + else if (outside(input->inode_table, start, end) || + outside(itend - 1, start, end)) + ext4_warning(sb, "Inode table not in group (blocks %llu-%llu)", + (unsigned long long)input->inode_table, itend - 1); + else if (input->inode_bitmap == input->block_bitmap) + ext4_warning(sb, "Block bitmap same as inode bitmap (%llu)", + (unsigned long long)input->block_bitmap); + else if (inside(input->block_bitmap, input->inode_table, itend)) + ext4_warning(sb, "Block bitmap (%llu) in inode table " + "(%llu-%llu)", + (unsigned long long)input->block_bitmap, + (unsigned long long)input->inode_table, itend - 1); + else if (inside(input->inode_bitmap, input->inode_table, itend)) + ext4_warning(sb, "Inode bitmap (%llu) in inode table " + "(%llu-%llu)", + (unsigned long long)input->inode_bitmap, + (unsigned long long)input->inode_table, itend - 1); + else if (inside(input->block_bitmap, start, metaend)) + ext4_warning(sb, "Block bitmap (%llu) in GDT table (%llu-%llu)", + (unsigned long long)input->block_bitmap, + start, metaend - 1); + else if (inside(input->inode_bitmap, start, metaend)) + ext4_warning(sb, "Inode bitmap (%llu) in GDT table (%llu-%llu)", + (unsigned long long)input->inode_bitmap, + start, metaend - 1); + else if (inside(input->inode_table, start, metaend) || + inside(itend - 1, start, metaend)) + ext4_warning(sb, "Inode table (%llu-%llu) overlaps GDT table " + "(%llu-%llu)", + (unsigned long long)input->inode_table, + itend - 1, start, metaend - 1); + else + err = 0; + brelse(bh); + + return err; +} + +/* + * ext4_new_flex_group_data is used by 64bit-resize interface to add a flex + * group each time. + */ +struct ext4_new_flex_group_data { + struct ext4_new_group_data *groups; /* new_group_data for groups + in the flex group */ + __u16 *bg_flags; /* block group flags of groups + in @groups */ + ext4_group_t resize_bg; /* number of allocated + new_group_data */ + ext4_group_t count; /* number of groups in @groups + */ +}; + +/* + * Avoiding memory allocation failures due to too many groups added each time. + */ +#define MAX_RESIZE_BG 16384 + +/* + * alloc_flex_gd() allocates an ext4_new_flex_group_data that satisfies the + * resizing from @o_group to @n_group, its size is typically @flexbg_size. + * + * Returns NULL on failure otherwise address of the allocated structure. + */ +static struct ext4_new_flex_group_data *alloc_flex_gd(unsigned int flexbg_size, + ext4_group_t o_group, ext4_group_t n_group) +{ + ext4_group_t last_group; + unsigned int max_resize_bg; + struct ext4_new_flex_group_data *flex_gd; + + flex_gd = kmalloc_obj(*flex_gd, GFP_NOFS); + if (flex_gd == NULL) + goto out3; + + max_resize_bg = umin(flexbg_size, MAX_RESIZE_BG); + flex_gd->resize_bg = max_resize_bg; + + /* Avoid allocating large 'groups' array if not needed */ + last_group = o_group | (flex_gd->resize_bg - 1); + if (n_group <= last_group) + flex_gd->resize_bg = 1 << fls(n_group - o_group); + else if (n_group - last_group < flex_gd->resize_bg) + flex_gd->resize_bg = 1 << max(fls(last_group - o_group), + fls(n_group - last_group)); + + if (WARN_ON_ONCE(flex_gd->resize_bg > max_resize_bg)) + flex_gd->resize_bg = max_resize_bg; + + flex_gd->groups = kmalloc_objs(struct ext4_new_group_data, + flex_gd->resize_bg, GFP_NOFS); + if (flex_gd->groups == NULL) + goto out2; + + flex_gd->bg_flags = kmalloc_array(flex_gd->resize_bg, sizeof(__u16), + GFP_NOFS); + if (flex_gd->bg_flags == NULL) + goto out1; + + return flex_gd; + +out1: + kfree(flex_gd->groups); +out2: + kfree(flex_gd); +out3: + return NULL; +} + +static void free_flex_gd(struct ext4_new_flex_group_data *flex_gd) +{ + kfree(flex_gd->bg_flags); + kfree(flex_gd->groups); + kfree(flex_gd); +} + +/* + * ext4_alloc_group_tables() allocates block bitmaps, inode bitmaps + * and inode tables for a flex group. + * + * This function is used by 64bit-resize. Note that this function allocates + * group tables from the 1st group of groups contained by @flexgd, which may + * be a partial of a flex group. + * + * @sb: super block of fs to which the groups belongs + * + * Returns 0 on a successful allocation of the metadata blocks in the + * block group. + */ +static int ext4_alloc_group_tables(struct super_block *sb, + struct ext4_new_flex_group_data *flex_gd, + unsigned int flexbg_size) +{ + struct ext4_new_group_data *group_data = flex_gd->groups; + ext4_fsblk_t start_blk; + ext4_fsblk_t last_blk; + ext4_group_t src_group; + ext4_group_t bb_index = 0; + ext4_group_t ib_index = 0; + ext4_group_t it_index = 0; + ext4_group_t group; + ext4_group_t last_group; + unsigned overhead; + __u16 uninit_mask = (flexbg_size > 1) ? ~EXT4_BG_BLOCK_UNINIT : ~0; + int i; + + BUG_ON(flex_gd->count == 0 || group_data == NULL); + + src_group = group_data[0].group; + last_group = src_group + flex_gd->count - 1; + + BUG_ON((flexbg_size > 1) && ((src_group & ~(flexbg_size - 1)) != + (last_group & ~(flexbg_size - 1)))); +next_group: + group = group_data[0].group; + if (src_group >= group_data[0].group + flex_gd->count) + return -ENOSPC; + start_blk = ext4_group_first_block_no(sb, src_group); + last_blk = start_blk + group_data[src_group - group].blocks_count; + + overhead = ext4_group_overhead_blocks(sb, src_group); + + start_blk += overhead; + + /* We collect contiguous blocks as much as possible. */ + src_group++; + for (; src_group <= last_group; src_group++) { + overhead = ext4_group_overhead_blocks(sb, src_group); + if (overhead == 0) + last_blk += group_data[src_group - group].blocks_count; + else + break; + } + + /* Allocate block bitmaps */ + for (; bb_index < flex_gd->count; bb_index++) { + if (start_blk >= last_blk) + goto next_group; + group_data[bb_index].block_bitmap = start_blk++; + group = ext4_get_group_number(sb, start_blk - 1); + group -= group_data[0].group; + group_data[group].mdata_blocks++; + flex_gd->bg_flags[group] &= uninit_mask; + } + + /* Allocate inode bitmaps */ + for (; ib_index < flex_gd->count; ib_index++) { + if (start_blk >= last_blk) + goto next_group; + group_data[ib_index].inode_bitmap = start_blk++; + group = ext4_get_group_number(sb, start_blk - 1); + group -= group_data[0].group; + group_data[group].mdata_blocks++; + flex_gd->bg_flags[group] &= uninit_mask; + } + + /* Allocate inode tables */ + for (; it_index < flex_gd->count; it_index++) { + unsigned int itb = EXT4_SB(sb)->s_itb_per_group; + ext4_fsblk_t next_group_start; + + if (start_blk + itb > last_blk) + goto next_group; + group_data[it_index].inode_table = start_blk; + group = ext4_get_group_number(sb, start_blk); + next_group_start = ext4_group_first_block_no(sb, group + 1); + group -= group_data[0].group; + + if (start_blk + itb > next_group_start) { + flex_gd->bg_flags[group + 1] &= uninit_mask; + overhead = start_blk + itb - next_group_start; + group_data[group + 1].mdata_blocks += overhead; + itb -= overhead; + } + + group_data[group].mdata_blocks += itb; + flex_gd->bg_flags[group] &= uninit_mask; + start_blk += EXT4_SB(sb)->s_itb_per_group; + } + + /* Update free clusters count to exclude metadata blocks */ + for (i = 0; i < flex_gd->count; i++) { + group_data[i].free_clusters_count -= + EXT4_NUM_B2C(EXT4_SB(sb), + group_data[i].mdata_blocks); + } + + if (test_opt(sb, DEBUG)) { + int i; + group = group_data[0].group; + + printk(KERN_DEBUG "EXT4-fs: adding a flex group with " + "%u groups, flexbg size is %u:\n", flex_gd->count, + flexbg_size); + + for (i = 0; i < flex_gd->count; i++) { + ext4_debug( + "adding %s group %u: %u blocks (%u free, %u mdata blocks)\n", + ext4_bg_has_super(sb, group + i) ? "normal" : + "no-super", group + i, + group_data[i].blocks_count, + group_data[i].free_clusters_count, + group_data[i].mdata_blocks); + } + } + return 0; +} + +static struct buffer_head *bclean(handle_t *handle, struct super_block *sb, + ext4_fsblk_t blk) +{ + struct buffer_head *bh; + int err; + + bh = sb_getblk(sb, blk); + if (unlikely(!bh)) + return ERR_PTR(-ENOMEM); + BUFFER_TRACE(bh, "get_write_access"); + err = ext4_journal_get_write_access(handle, sb, bh, EXT4_JTR_NONE); + if (err) { + brelse(bh); + bh = ERR_PTR(err); + } else { + memset(bh->b_data, 0, sb->s_blocksize); + set_buffer_uptodate(bh); + } + + return bh; +} + +static int ext4_resize_ensure_credits_batch(handle_t *handle, int credits) +{ + return ext4_journal_ensure_credits_fn(handle, credits, + EXT4_MAX_TRANS_DATA, 0, 0); +} + +/* + * set_flexbg_block_bitmap() mark clusters [@first_cluster, @last_cluster] used. + * + * Helper function for ext4_setup_new_group_blocks() which set . + * + * @sb: super block + * @handle: journal handle + * @flex_gd: flex group data + */ +static int set_flexbg_block_bitmap(struct super_block *sb, handle_t *handle, + struct ext4_new_flex_group_data *flex_gd, + ext4_fsblk_t first_cluster, ext4_fsblk_t last_cluster) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + ext4_group_t count = last_cluster - first_cluster + 1; + ext4_group_t count2; + + ext4_debug("mark clusters [%llu-%llu] used\n", first_cluster, + last_cluster); + for (; count > 0; count -= count2, first_cluster += count2) { + ext4_fsblk_t start; + struct buffer_head *bh; + ext4_group_t group; + int err; + + group = ext4_get_group_number(sb, EXT4_C2B(sbi, first_cluster)); + start = EXT4_B2C(sbi, ext4_group_first_block_no(sb, group)); + group -= flex_gd->groups[0].group; + + count2 = EXT4_CLUSTERS_PER_GROUP(sb) - (first_cluster - start); + if (count2 > count) + count2 = count; + + if (flex_gd->bg_flags[group] & EXT4_BG_BLOCK_UNINIT) { + BUG_ON(flex_gd->count > 1); + continue; + } + + err = ext4_resize_ensure_credits_batch(handle, 1); + if (err < 0) + return err; + + bh = sb_getblk(sb, flex_gd->groups[group].block_bitmap); + if (unlikely(!bh)) + return -ENOMEM; + + BUFFER_TRACE(bh, "get_write_access"); + err = ext4_journal_get_write_access(handle, sb, bh, + EXT4_JTR_NONE); + if (err) { + brelse(bh); + return err; + } + ext4_debug("mark block bitmap %#04llx (+%llu/%u)\n", + first_cluster, first_cluster - start, count2); + mb_set_bits(bh->b_data, first_cluster - start, count2); + + err = ext4_handle_dirty_metadata(handle, NULL, bh); + brelse(bh); + if (unlikely(err)) + return err; + } + + return 0; +} + +/* + * Set up the block and inode bitmaps, and the inode table for the new groups. + * This doesn't need to be part of the main transaction, since we are only + * changing blocks outside the actual filesystem. We still do journaling to + * ensure the recovery is correct in case of a failure just after resize. + * If any part of this fails, we simply abort the resize. + * + * setup_new_flex_group_blocks handles a flex group as follow: + * 1. copy super block and GDT, and initialize group tables if necessary. + * In this step, we only set bits in blocks bitmaps for blocks taken by + * super block and GDT. + * 2. allocate group tables in block bitmaps, that is, set bits in block + * bitmap for blocks taken by group tables. + */ +static int setup_new_flex_group_blocks(struct super_block *sb, + struct ext4_new_flex_group_data *flex_gd) +{ + int group_table_count[] = {1, 1, EXT4_SB(sb)->s_itb_per_group}; + ext4_fsblk_t start; + ext4_fsblk_t block; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_super_block *es = sbi->s_es; + struct ext4_new_group_data *group_data = flex_gd->groups; + __u16 *bg_flags = flex_gd->bg_flags; + handle_t *handle; + ext4_group_t group, count; + struct buffer_head *bh = NULL; + int reserved_gdb, i, j, err = 0, err2; + int meta_bg; + + BUG_ON(!flex_gd->count || !group_data || + group_data[0].group != sbi->s_groups_count); + + reserved_gdb = le16_to_cpu(es->s_reserved_gdt_blocks); + meta_bg = ext4_has_feature_meta_bg(sb); + + /* This transaction may be extended/restarted along the way */ + handle = ext4_journal_start_sb(sb, EXT4_HT_RESIZE, EXT4_MAX_TRANS_DATA); + if (IS_ERR(handle)) + return PTR_ERR(handle); + + group = group_data[0].group; + for (i = 0; i < flex_gd->count; i++, group++) { + unsigned long gdblocks; + ext4_grpblk_t overhead; + + gdblocks = ext4_bg_num_gdb(sb, group); + start = ext4_group_first_block_no(sb, group); + + if (meta_bg == 0 && !ext4_bg_has_super(sb, group)) + goto handle_itb; + + if (meta_bg == 1) + goto handle_itb; + + block = start + ext4_bg_has_super(sb, group); + /* Copy all of the GDT blocks into the backup in this group */ + for (j = 0; j < gdblocks; j++, block++) { + struct buffer_head *gdb; + + ext4_debug("update backup group %#04llx\n", block); + err = ext4_resize_ensure_credits_batch(handle, 1); + if (err < 0) + goto out; + + gdb = sb_getblk(sb, block); + if (unlikely(!gdb)) { + err = -ENOMEM; + goto out; + } + + BUFFER_TRACE(gdb, "get_write_access"); + err = ext4_journal_get_write_access(handle, sb, gdb, + EXT4_JTR_NONE); + if (err) { + brelse(gdb); + goto out; + } + memcpy(gdb->b_data, sbi_array_rcu_deref(sbi, + s_group_desc, j)->b_data, gdb->b_size); + set_buffer_uptodate(gdb); + + err = ext4_handle_dirty_metadata(handle, NULL, gdb); + if (unlikely(err)) { + brelse(gdb); + goto out; + } + brelse(gdb); + } + + /* Zero out all of the reserved backup group descriptor + * table blocks + */ + if (ext4_bg_has_super(sb, group)) { + err = sb_issue_zeroout(sb, gdblocks + start + 1, + reserved_gdb, GFP_NOFS); + if (err) + goto out; + } + +handle_itb: + /* Initialize group tables of the group @group */ + if (!(bg_flags[i] & EXT4_BG_INODE_ZEROED)) + goto handle_bb; + + /* Zero out all of the inode table blocks */ + block = group_data[i].inode_table; + ext4_debug("clear inode table blocks %#04llx -> %#04lx\n", + block, sbi->s_itb_per_group); + err = sb_issue_zeroout(sb, block, sbi->s_itb_per_group, + GFP_NOFS); + if (err) + goto out; + +handle_bb: + if (bg_flags[i] & EXT4_BG_BLOCK_UNINIT) + goto handle_ib; + + /* Initialize block bitmap of the @group */ + block = group_data[i].block_bitmap; + err = ext4_resize_ensure_credits_batch(handle, 1); + if (err < 0) + goto out; + + bh = bclean(handle, sb, block); + if (IS_ERR(bh)) { + err = PTR_ERR(bh); + goto out; + } + overhead = ext4_group_overhead_blocks(sb, group); + if (overhead != 0) { + ext4_debug("mark backup superblock %#04llx (+0)\n", + start); + mb_set_bits(bh->b_data, 0, + EXT4_NUM_B2C(sbi, overhead)); + } + ext4_mark_bitmap_end(EXT4_B2C(sbi, group_data[i].blocks_count), + sb->s_blocksize * 8, bh->b_data); + err = ext4_handle_dirty_metadata(handle, NULL, bh); + brelse(bh); + if (err) + goto out; + +handle_ib: + if (bg_flags[i] & EXT4_BG_INODE_UNINIT) + continue; + + /* Initialize inode bitmap of the @group */ + block = group_data[i].inode_bitmap; + err = ext4_resize_ensure_credits_batch(handle, 1); + if (err < 0) + goto out; + /* Mark unused entries in inode bitmap used */ + bh = bclean(handle, sb, block); + if (IS_ERR(bh)) { + err = PTR_ERR(bh); + goto out; + } + + ext4_mark_bitmap_end(EXT4_INODES_PER_GROUP(sb), + sb->s_blocksize * 8, bh->b_data); + err = ext4_handle_dirty_metadata(handle, NULL, bh); + brelse(bh); + if (err) + goto out; + } + + /* Mark group tables in block bitmap */ + for (j = 0; j < GROUP_TABLE_COUNT; j++) { + count = group_table_count[j]; + start = (&group_data[0].block_bitmap)[j]; + block = start; + for (i = 1; i < flex_gd->count; i++) { + block += group_table_count[j]; + if (block == (&group_data[i].block_bitmap)[j]) { + count += group_table_count[j]; + continue; + } + err = set_flexbg_block_bitmap(sb, handle, + flex_gd, + EXT4_B2C(sbi, start), + EXT4_B2C(sbi, + start + count + - 1)); + if (err) + goto out; + count = group_table_count[j]; + start = (&group_data[i].block_bitmap)[j]; + block = start; + } + + err = set_flexbg_block_bitmap(sb, handle, + flex_gd, + EXT4_B2C(sbi, start), + EXT4_B2C(sbi, + start + count + - 1)); + if (err) + goto out; + } + +out: + err2 = ext4_journal_stop(handle); + if (err2 && !err) + err = err2; + + return err; +} + +/* + * Iterate through the groups which hold BACKUP superblock/GDT copies in an + * ext4 filesystem. The counters should be initialized to 1, 5, and 7 before + * calling this for the first time. In a sparse filesystem it will be the + * sequence of powers of 3, 5, and 7: 1, 3, 5, 7, 9, 25, 27, 49, 81, ... + * For a non-sparse filesystem it will be every group: 1, 2, 3, 4, ... + */ +unsigned int ext4_list_backups(struct super_block *sb, unsigned int *three, + unsigned int *five, unsigned int *seven) +{ + struct ext4_super_block *es = EXT4_SB(sb)->s_es; + unsigned int *min = three; + int mult = 3; + unsigned int ret; + + if (ext4_has_feature_sparse_super2(sb)) { + do { + if (*min > 2) + return UINT_MAX; + ret = le32_to_cpu(es->s_backup_bgs[*min - 1]); + *min += 1; + } while (!ret); + return ret; + } + + if (!ext4_has_feature_sparse_super(sb)) { + ret = *min; + *min += 1; + return ret; + } + + if (*five < *min) { + min = five; + mult = 5; + } + if (*seven < *min) { + min = seven; + mult = 7; + } + + ret = *min; + *min *= mult; + + return ret; +} + +/* + * Check that all of the backup GDT blocks are held in the primary GDT block. + * It is assumed that they are stored in group order. Returns the number of + * groups in current filesystem that have BACKUPS, or -ve error code. + */ +static int verify_reserved_gdb(struct super_block *sb, + ext4_group_t end, + struct buffer_head *primary) +{ + const ext4_fsblk_t blk = primary->b_blocknr; + unsigned three = 1; + unsigned five = 5; + unsigned seven = 7; + unsigned grp; + __le32 *p = (__le32 *)primary->b_data; + int gdbackups = 0; + + while ((grp = ext4_list_backups(sb, &three, &five, &seven)) < end) { + if (le32_to_cpu(*p++) != + grp * EXT4_BLOCKS_PER_GROUP(sb) + blk){ + ext4_warning(sb, "reserved GDT %llu" + " missing grp %d (%llu)", + blk, grp, + grp * + (ext4_fsblk_t)EXT4_BLOCKS_PER_GROUP(sb) + + blk); + return -EINVAL; + } + if (++gdbackups > EXT4_ADDR_PER_BLOCK(sb)) + return -EFBIG; + } + + return gdbackups; +} + +/* + * Called when we need to bring a reserved group descriptor table block into + * use from the resize inode. The primary copy of the new GDT block currently + * is an indirect block (under the double indirect block in the resize inode). + * The new backup GDT blocks will be stored as leaf blocks in this indirect + * block, in group order. Even though we know all the block numbers we need, + * we check to ensure that the resize inode has actually reserved these blocks. + * + * Don't need to update the block bitmaps because the blocks are still in use. + * + * We get all of the error cases out of the way, so that we are sure to not + * fail once we start modifying the data on disk, because JBD has no rollback. + */ +static int add_new_gdb(handle_t *handle, struct inode *inode, + ext4_group_t group) +{ + struct super_block *sb = inode->i_sb; + struct ext4_super_block *es = EXT4_SB(sb)->s_es; + unsigned long gdb_num = group / EXT4_DESC_PER_BLOCK(sb); + ext4_fsblk_t gdblock = EXT4_SB(sb)->s_sbh->b_blocknr + 1 + gdb_num; + struct buffer_head **o_group_desc, **n_group_desc = NULL; + struct buffer_head *dind = NULL; + struct buffer_head *gdb_bh = NULL; + int gdbackups; + struct ext4_iloc iloc = { .bh = NULL }; + __le32 *data; + int err; + + if (test_opt(sb, DEBUG)) + printk(KERN_DEBUG + "EXT4-fs: ext4_add_new_gdb: adding group block %lu\n", + gdb_num); + + gdb_bh = ext4_sb_bread(sb, gdblock, 0); + if (IS_ERR(gdb_bh)) + return PTR_ERR(gdb_bh); + + gdbackups = verify_reserved_gdb(sb, group, gdb_bh); + if (gdbackups < 0) { + err = gdbackups; + goto errout; + } + + data = EXT4_I(inode)->i_data + EXT4_DIND_BLOCK; + dind = ext4_sb_bread(sb, le32_to_cpu(*data), 0); + if (IS_ERR(dind)) { + err = PTR_ERR(dind); + dind = NULL; + goto errout; + } + + data = (__le32 *)dind->b_data; + if (le32_to_cpu(data[gdb_num % EXT4_ADDR_PER_BLOCK(sb)]) != gdblock) { + ext4_warning(sb, "new group %u GDT block %llu not reserved", + group, gdblock); + err = -EINVAL; + goto errout; + } + + BUFFER_TRACE(EXT4_SB(sb)->s_sbh, "get_write_access"); + err = ext4_journal_get_write_access(handle, sb, EXT4_SB(sb)->s_sbh, + EXT4_JTR_NONE); + if (unlikely(err)) + goto errout; + + BUFFER_TRACE(gdb_bh, "get_write_access"); + err = ext4_journal_get_write_access(handle, sb, gdb_bh, EXT4_JTR_NONE); + if (unlikely(err)) + goto errout; + + BUFFER_TRACE(dind, "get_write_access"); + err = ext4_journal_get_write_access(handle, sb, dind, EXT4_JTR_NONE); + if (unlikely(err)) { + ext4_std_error(sb, err); + goto errout; + } + + /* ext4_reserve_inode_write() gets a reference on the iloc */ + err = ext4_reserve_inode_write(handle, inode, &iloc); + if (unlikely(err)) + goto errout; + + n_group_desc = kvmalloc((gdb_num + 1) * sizeof(struct buffer_head *), + GFP_KERNEL); + if (!n_group_desc) { + err = -ENOMEM; + ext4_warning(sb, "not enough memory for %lu groups", + gdb_num + 1); + goto errout; + } + + /* + * Finally, we have all of the possible failures behind us... + * + * Remove new GDT block from inode double-indirect block and clear out + * the new GDT block for use (which also "frees" the backup GDT blocks + * from the reserved inode). We don't need to change the bitmaps for + * these blocks, because they are marked as in-use from being in the + * reserved inode, and will become GDT blocks (primary and backup). + */ + data[gdb_num % EXT4_ADDR_PER_BLOCK(sb)] = 0; + err = ext4_handle_dirty_metadata(handle, NULL, dind); + if (unlikely(err)) { + ext4_std_error(sb, err); + goto errout; + } + inode->i_blocks -= (gdbackups + 1) * sb->s_blocksize >> + (9 - EXT4_SB(sb)->s_cluster_bits); + ext4_mark_iloc_dirty(handle, inode, &iloc); + memset(gdb_bh->b_data, 0, sb->s_blocksize); + err = ext4_handle_dirty_metadata(handle, NULL, gdb_bh); + if (unlikely(err)) { + ext4_std_error(sb, err); + iloc.bh = NULL; + goto errout; + } + brelse(dind); + + rcu_read_lock(); + o_group_desc = rcu_dereference(EXT4_SB(sb)->s_group_desc); + memcpy(n_group_desc, o_group_desc, + EXT4_SB(sb)->s_gdb_count * sizeof(struct buffer_head *)); + rcu_read_unlock(); + n_group_desc[gdb_num] = gdb_bh; + rcu_assign_pointer(EXT4_SB(sb)->s_group_desc, n_group_desc); + EXT4_SB(sb)->s_gdb_count++; + ext4_kvfree_array_rcu(o_group_desc); + + lock_buffer(EXT4_SB(sb)->s_sbh); + le16_add_cpu(&es->s_reserved_gdt_blocks, -1); + ext4_superblock_csum_set(sb); + unlock_buffer(EXT4_SB(sb)->s_sbh); + err = ext4_handle_dirty_metadata(handle, NULL, EXT4_SB(sb)->s_sbh); + if (err) + ext4_std_error(sb, err); + return err; +errout: + kvfree(n_group_desc); + brelse(iloc.bh); + brelse(dind); + brelse(gdb_bh); + + ext4_debug("leaving with error %d\n", err); + return err; +} + +/* + * If there is no available space in the existing block group descriptors for + * the new block group and there are no reserved block group descriptors, then + * the meta_bg feature will get enabled, and es->s_first_meta_bg will get set + * to the first block group that is managed using meta_bg and s_first_meta_bg + * must be a multiple of EXT4_DESC_PER_BLOCK(sb). + * This function will be called when first group of meta_bg is added to bring + * new group descriptors block of new added meta_bg. + */ +static int add_new_gdb_meta_bg(struct super_block *sb, + handle_t *handle, ext4_group_t group) { + ext4_fsblk_t gdblock; + struct buffer_head *gdb_bh; + struct buffer_head **o_group_desc, **n_group_desc; + unsigned long gdb_num = group / EXT4_DESC_PER_BLOCK(sb); + int err; + + gdblock = ext4_group_first_block_no(sb, group) + + ext4_bg_has_super(sb, group); + gdb_bh = ext4_sb_bread(sb, gdblock, 0); + if (IS_ERR(gdb_bh)) + return PTR_ERR(gdb_bh); + n_group_desc = kvmalloc((gdb_num + 1) * sizeof(struct buffer_head *), + GFP_KERNEL); + if (!n_group_desc) { + brelse(gdb_bh); + err = -ENOMEM; + ext4_warning(sb, "not enough memory for %lu groups", + gdb_num + 1); + return err; + } + + rcu_read_lock(); + o_group_desc = rcu_dereference(EXT4_SB(sb)->s_group_desc); + memcpy(n_group_desc, o_group_desc, + EXT4_SB(sb)->s_gdb_count * sizeof(struct buffer_head *)); + rcu_read_unlock(); + n_group_desc[gdb_num] = gdb_bh; + + BUFFER_TRACE(gdb_bh, "get_write_access"); + err = ext4_journal_get_write_access(handle, sb, gdb_bh, EXT4_JTR_NONE); + if (err) { + kvfree(n_group_desc); + brelse(gdb_bh); + return err; + } + + rcu_assign_pointer(EXT4_SB(sb)->s_group_desc, n_group_desc); + EXT4_SB(sb)->s_gdb_count++; + ext4_kvfree_array_rcu(o_group_desc); + return err; +} + +/* + * Called when we are adding a new group which has a backup copy of each of + * the GDT blocks (i.e. sparse group) and there are reserved GDT blocks. + * We need to add these reserved backup GDT blocks to the resize inode, so + * that they are kept for future resizing and not allocated to files. + * + * Each reserved backup GDT block will go into a different indirect block. + * The indirect blocks are actually the primary reserved GDT blocks, + * so we know in advance what their block numbers are. We only get the + * double-indirect block to verify it is pointing to the primary reserved + * GDT blocks so we don't overwrite a data block by accident. The reserved + * backup GDT blocks are stored in their reserved primary GDT block. + */ +static int reserve_backup_gdb(handle_t *handle, struct inode *inode, + ext4_group_t group) +{ + struct super_block *sb = inode->i_sb; + int reserved_gdb =le16_to_cpu(EXT4_SB(sb)->s_es->s_reserved_gdt_blocks); + int cluster_bits = EXT4_SB(sb)->s_cluster_bits; + struct buffer_head **primary; + struct buffer_head *dind; + struct ext4_iloc iloc; + ext4_fsblk_t blk; + __le32 *data, *end; + int gdbackups = 0; + int res, i; + int err; + + primary = kmalloc_objs(*primary, reserved_gdb, GFP_NOFS); + if (!primary) + return -ENOMEM; + + data = EXT4_I(inode)->i_data + EXT4_DIND_BLOCK; + dind = ext4_sb_bread(sb, le32_to_cpu(*data), 0); + if (IS_ERR(dind)) { + err = PTR_ERR(dind); + dind = NULL; + goto exit_free; + } + + blk = EXT4_SB(sb)->s_sbh->b_blocknr + 1 + EXT4_SB(sb)->s_gdb_count; + data = (__le32 *)dind->b_data + (EXT4_SB(sb)->s_gdb_count % + EXT4_ADDR_PER_BLOCK(sb)); + end = (__le32 *)dind->b_data + EXT4_ADDR_PER_BLOCK(sb); + + /* Get each reserved primary GDT block and verify it holds backups */ + for (res = 0; res < reserved_gdb; res++, blk++) { + if (le32_to_cpu(*data) != blk) { + ext4_warning(sb, "reserved block %llu" + " not at offset %ld", + blk, + (long)(data - (__le32 *)dind->b_data)); + err = -EINVAL; + goto exit_bh; + } + primary[res] = ext4_sb_bread(sb, blk, 0); + if (IS_ERR(primary[res])) { + err = PTR_ERR(primary[res]); + primary[res] = NULL; + goto exit_bh; + } + gdbackups = verify_reserved_gdb(sb, group, primary[res]); + if (gdbackups < 0) { + brelse(primary[res]); + err = gdbackups; + goto exit_bh; + } + if (++data >= end) + data = (__le32 *)dind->b_data; + } + + for (i = 0; i < reserved_gdb; i++) { + BUFFER_TRACE(primary[i], "get_write_access"); + if ((err = ext4_journal_get_write_access(handle, sb, primary[i], + EXT4_JTR_NONE))) + goto exit_bh; + } + + if ((err = ext4_reserve_inode_write(handle, inode, &iloc))) + goto exit_bh; + + /* + * Finally we can add each of the reserved backup GDT blocks from + * the new group to its reserved primary GDT block. + */ + blk = group * EXT4_BLOCKS_PER_GROUP(sb); + for (i = 0; i < reserved_gdb; i++) { + int err2; + data = (__le32 *)primary[i]->b_data; + data[gdbackups] = cpu_to_le32(blk + primary[i]->b_blocknr); + err2 = ext4_handle_dirty_metadata(handle, NULL, primary[i]); + if (!err) + err = err2; + } + + inode->i_blocks += reserved_gdb * sb->s_blocksize >> (9 - cluster_bits); + ext4_mark_iloc_dirty(handle, inode, &iloc); + +exit_bh: + while (--res >= 0) + brelse(primary[res]); + brelse(dind); + +exit_free: + kfree(primary); + + return err; +} + +static inline void ext4_set_block_group_nr(struct super_block *sb, char *data, + ext4_group_t group) +{ + struct ext4_super_block *es = (struct ext4_super_block *) data; + + es->s_block_group_nr = cpu_to_le16(group); + if (ext4_has_feature_metadata_csum(sb)) + es->s_checksum = ext4_superblock_csum(es); +} + +/* + * Update the backup copies of the ext4 metadata. These don't need to be part + * of the main resize transaction, because e2fsck will re-write them if there + * is a problem (basically only OOM will cause a problem). However, we + * _should_ update the backups if possible, in case the primary gets trashed + * for some reason and we need to run e2fsck from a backup superblock. The + * important part is that the new block and inode counts are in the backup + * superblocks, and the location of the new group metadata in the GDT backups. + * + * We do not need take the s_resize_lock for this, because these + * blocks are not otherwise touched by the filesystem code when it is + * mounted. We don't need to worry about last changing from + * sbi->s_groups_count, because the worst that can happen is that we + * do not copy the full number of backups at this time. The resize + * which changed s_groups_count will backup again. + */ +static void update_backups(struct super_block *sb, sector_t blk_off, char *data, + int size, int meta_bg) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + ext4_group_t last; + const int bpg = EXT4_BLOCKS_PER_GROUP(sb); + unsigned three = 1; + unsigned five = 5; + unsigned seven = 7; + ext4_group_t group = 0; + int rest = sb->s_blocksize - size; + handle_t *handle; + int err = 0, err2; + + handle = ext4_journal_start_sb(sb, EXT4_HT_RESIZE, EXT4_MAX_TRANS_DATA); + if (IS_ERR(handle)) { + group = 1; + err = PTR_ERR(handle); + goto exit_err; + } + + if (meta_bg == 0) { + group = ext4_list_backups(sb, &three, &five, &seven); + last = sbi->s_groups_count; + } else { + group = ext4_get_group_number(sb, blk_off) + 1; + last = (ext4_group_t)(group + EXT4_DESC_PER_BLOCK(sb) - 2); + } + + while (group < sbi->s_groups_count) { + struct buffer_head *bh; + ext4_fsblk_t backup_block; + int has_super = ext4_bg_has_super(sb, group); + ext4_fsblk_t first_block = ext4_group_first_block_no(sb, group); + + /* Out of journal space, and can't get more - abort - so sad */ + err = ext4_resize_ensure_credits_batch(handle, 1); + if (err < 0) + break; + + if (meta_bg == 0) + backup_block = ((ext4_fsblk_t)group) * bpg + blk_off; + else + backup_block = first_block + has_super; + + bh = sb_getblk(sb, backup_block); + if (unlikely(!bh)) { + err = -ENOMEM; + break; + } + ext4_debug("update metadata backup %llu(+%llu)\n", + backup_block, backup_block - + ext4_group_first_block_no(sb, group)); + BUFFER_TRACE(bh, "get_write_access"); + if ((err = ext4_journal_get_write_access(handle, sb, bh, + EXT4_JTR_NONE))) { + brelse(bh); + break; + } + lock_buffer(bh); + memcpy(bh->b_data, data, size); + if (rest) + memset(bh->b_data + size, 0, rest); + if (has_super && (backup_block == first_block)) + ext4_set_block_group_nr(sb, bh->b_data, group); + set_buffer_uptodate(bh); + unlock_buffer(bh); + err = ext4_handle_dirty_metadata(handle, NULL, bh); + if (unlikely(err)) + ext4_std_error(sb, err); + brelse(bh); + + if (meta_bg == 0) + group = ext4_list_backups(sb, &three, &five, &seven); + else if (group == last) + break; + else + group = last; + } + if ((err2 = ext4_journal_stop(handle)) && !err) + err = err2; + + /* + * Ugh! Need to have e2fsck write the backup copies. It is too + * late to revert the resize, we shouldn't fail just because of + * the backup copies (they are only needed in case of corruption). + * + * However, if we got here we have a journal problem too, so we + * can't really start a transaction to mark the superblock. + * Chicken out and just set the flag on the hope it will be written + * to disk, and if not - we will simply wait until next fsck. + */ +exit_err: + if (err) { + ext4_warning(sb, "can't update backup for group %u (err %d), " + "forcing fsck on next reboot", group, err); + sbi->s_mount_state &= ~EXT4_VALID_FS; + sbi->s_es->s_state &= cpu_to_le16(~EXT4_VALID_FS); + mark_buffer_dirty(sbi->s_sbh); + } +} + +/* + * ext4_add_new_descs() adds @count group descriptor of groups + * starting at @group + * + * @handle: journal handle + * @sb: super block + * @group: the group no. of the first group desc to be added + * @resize_inode: the resize inode + * @count: number of group descriptors to be added + */ +static int ext4_add_new_descs(handle_t *handle, struct super_block *sb, + ext4_group_t group, struct inode *resize_inode, + ext4_group_t count) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_super_block *es = sbi->s_es; + struct buffer_head *gdb_bh; + int i, gdb_off, gdb_num, err = 0; + int meta_bg; + + meta_bg = ext4_has_feature_meta_bg(sb); + for (i = 0; i < count; i++, group++) { + int reserved_gdb = ext4_bg_has_super(sb, group) ? + le16_to_cpu(es->s_reserved_gdt_blocks) : 0; + + gdb_off = group % EXT4_DESC_PER_BLOCK(sb); + gdb_num = group / EXT4_DESC_PER_BLOCK(sb); + + /* + * We will only either add reserved group blocks to a backup group + * or remove reserved blocks for the first group in a new group block. + * Doing both would be mean more complex code, and sane people don't + * use non-sparse filesystems anymore. This is already checked above. + */ + if (gdb_off) { + gdb_bh = sbi_array_rcu_deref(sbi, s_group_desc, + gdb_num); + BUFFER_TRACE(gdb_bh, "get_write_access"); + err = ext4_journal_get_write_access(handle, sb, gdb_bh, + EXT4_JTR_NONE); + + if (!err && reserved_gdb && ext4_bg_num_gdb(sb, group)) + err = reserve_backup_gdb(handle, resize_inode, group); + } else if (meta_bg != 0) { + err = add_new_gdb_meta_bg(sb, handle, group); + } else { + err = add_new_gdb(handle, resize_inode, group); + } + if (err) + break; + } + return err; +} + +static struct buffer_head *ext4_get_bitmap(struct super_block *sb, __u64 block) +{ + struct buffer_head *bh = sb_getblk(sb, block); + if (unlikely(!bh)) + return NULL; + if (!bh_uptodate_or_lock(bh)) { + if (ext4_read_bh(bh, 0, NULL, false) < 0) { + brelse(bh); + return NULL; + } + } + + return bh; +} + +static int ext4_set_bitmap_checksums(struct super_block *sb, + struct ext4_group_desc *gdp, + struct ext4_new_group_data *group_data) +{ + struct buffer_head *bh; + + if (!ext4_has_feature_metadata_csum(sb)) + return 0; + + bh = ext4_get_bitmap(sb, group_data->inode_bitmap); + if (!bh) + return -EIO; + ext4_inode_bitmap_csum_set(sb, gdp, bh); + brelse(bh); + + bh = ext4_get_bitmap(sb, group_data->block_bitmap); + if (!bh) + return -EIO; + ext4_block_bitmap_csum_set(sb, gdp, bh); + brelse(bh); + + return 0; +} + +/* + * ext4_setup_new_descs() will set up the group descriptor descriptors of a flex bg + */ +static int ext4_setup_new_descs(handle_t *handle, struct super_block *sb, + struct ext4_new_flex_group_data *flex_gd) +{ + struct ext4_new_group_data *group_data = flex_gd->groups; + struct ext4_group_desc *gdp; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct buffer_head *gdb_bh; + ext4_group_t group; + __u16 *bg_flags = flex_gd->bg_flags; + int i, gdb_off, gdb_num, err = 0; + + + for (i = 0; i < flex_gd->count; i++, group_data++, bg_flags++) { + group = group_data->group; + + gdb_off = group % EXT4_DESC_PER_BLOCK(sb); + gdb_num = group / EXT4_DESC_PER_BLOCK(sb); + + /* + * get_write_access() has been called on gdb_bh by ext4_add_new_desc(). + */ + gdb_bh = sbi_array_rcu_deref(sbi, s_group_desc, gdb_num); + /* Update group descriptor block for new group */ + gdp = (struct ext4_group_desc *)(gdb_bh->b_data + + gdb_off * EXT4_DESC_SIZE(sb)); + + memset(gdp, 0, EXT4_DESC_SIZE(sb)); + ext4_block_bitmap_set(sb, gdp, group_data->block_bitmap); + ext4_inode_bitmap_set(sb, gdp, group_data->inode_bitmap); + err = ext4_set_bitmap_checksums(sb, gdp, group_data); + if (err) { + ext4_std_error(sb, err); + break; + } + + ext4_inode_table_set(sb, gdp, group_data->inode_table); + ext4_free_group_clusters_set(sb, gdp, + group_data->free_clusters_count); + ext4_free_inodes_set(sb, gdp, EXT4_INODES_PER_GROUP(sb)); + if (ext4_has_group_desc_csum(sb)) + ext4_itable_unused_set(sb, gdp, + EXT4_INODES_PER_GROUP(sb)); + gdp->bg_flags = cpu_to_le16(*bg_flags); + ext4_group_desc_csum_set(sb, group, gdp); + + err = ext4_handle_dirty_metadata(handle, NULL, gdb_bh); + if (unlikely(err)) { + ext4_std_error(sb, err); + break; + } + + /* + * We can allocate memory for mb_alloc based on the new group + * descriptor + */ + err = ext4_mb_add_groupinfo(sb, group, gdp); + if (err) + break; + } + return err; +} + +static void ext4_add_overhead(struct super_block *sb, + const ext4_fsblk_t overhead) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_super_block *es = sbi->s_es; + + sbi->s_overhead += overhead; + es->s_overhead_clusters = cpu_to_le32(sbi->s_overhead); + smp_wmb(); +} + +/* + * ext4_update_super() updates the super block so that the newly added + * groups can be seen by the filesystem. + * + * @sb: super block + * @flex_gd: new added groups + */ +static void ext4_update_super(struct super_block *sb, + struct ext4_new_flex_group_data *flex_gd) +{ + ext4_fsblk_t blocks_count = 0; + ext4_fsblk_t free_blocks = 0; + ext4_fsblk_t reserved_blocks = 0; + struct ext4_new_group_data *group_data = flex_gd->groups; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_super_block *es = sbi->s_es; + int i; + + BUG_ON(flex_gd->count == 0 || group_data == NULL); + /* + * Make the new blocks and inodes valid next. We do this before + * increasing the group count so that once the group is enabled, + * all of its blocks and inodes are already valid. + * + * We always allocate group-by-group, then block-by-block or + * inode-by-inode within a group, so enabling these + * blocks/inodes before the group is live won't actually let us + * allocate the new space yet. + */ + for (i = 0; i < flex_gd->count; i++) { + blocks_count += group_data[i].blocks_count; + free_blocks += EXT4_C2B(sbi, group_data[i].free_clusters_count); + } + + reserved_blocks = ext4_r_blocks_count(es) * 100; + reserved_blocks = div64_u64(reserved_blocks, ext4_blocks_count(es)); + reserved_blocks *= blocks_count; + do_div(reserved_blocks, 100); + + lock_buffer(sbi->s_sbh); + ext4_blocks_count_set(es, ext4_blocks_count(es) + blocks_count); + ext4_free_blocks_count_set(es, ext4_free_blocks_count(es) + free_blocks); + le32_add_cpu(&es->s_inodes_count, EXT4_INODES_PER_GROUP(sb) * + flex_gd->count); + le32_add_cpu(&es->s_free_inodes_count, EXT4_INODES_PER_GROUP(sb) * + flex_gd->count); + + ext4_debug("free blocks count %llu", ext4_free_blocks_count(es)); + /* + * We need to protect s_groups_count against other CPUs seeing + * inconsistent state in the superblock. + * + * The precise rules we use are: + * + * * Writers must perform a smp_wmb() after updating all + * dependent data and before modifying the groups count + * + * * Readers must perform an smp_rmb() after reading the groups + * count and before reading any dependent data. + * + * NB. These rules can be relaxed when checking the group count + * while freeing data, as we can only allocate from a block + * group after serialising against the group count, and we can + * only then free after serialising in turn against that + * allocation. + */ + smp_wmb(); + + /* Update the global fs size fields */ + sbi->s_groups_count += flex_gd->count; + sbi->s_blockfile_groups = min(sbi->s_groups_count, + (EXT4_MAX_BLOCK_FILE_PHYS / EXT4_BLOCKS_PER_GROUP(sb))); + + /* Update the reserved block counts only once the new group is + * active. */ + ext4_r_blocks_count_set(es, ext4_r_blocks_count(es) + + reserved_blocks); + + /* Update the free space counts */ + percpu_counter_add(&sbi->s_freeclusters_counter, + EXT4_NUM_B2C(sbi, free_blocks)); + percpu_counter_add(&sbi->s_freeinodes_counter, + EXT4_INODES_PER_GROUP(sb) * flex_gd->count); + + ext4_debug("free blocks count %llu", + percpu_counter_read(&sbi->s_freeclusters_counter)); + if (ext4_has_feature_flex_bg(sb) && sbi->s_log_groups_per_flex) { + ext4_group_t flex_group; + struct flex_groups *fg; + + flex_group = ext4_flex_group(sbi, group_data[0].group); + fg = sbi_array_rcu_deref(sbi, s_flex_groups, flex_group); + atomic64_add(EXT4_NUM_B2C(sbi, free_blocks), + &fg->free_clusters); + atomic_add(EXT4_INODES_PER_GROUP(sb) * flex_gd->count, + &fg->free_inodes); + } + + /* + * Update the fs overhead information. + * + * For bigalloc, if the superblock already has a properly calculated + * overhead, update it with a value based on numbers already computed + * above for the newly allocated capacity. + */ + if (ext4_has_feature_bigalloc(sb) && (sbi->s_overhead != 0)) + ext4_add_overhead(sb, + EXT4_NUM_B2C(sbi, blocks_count - free_blocks)); + else + ext4_calculate_overhead(sb); + es->s_overhead_clusters = cpu_to_le32(sbi->s_overhead); + + ext4_superblock_csum_set(sb); + unlock_buffer(sbi->s_sbh); + if (test_opt(sb, DEBUG)) + printk(KERN_DEBUG "EXT4-fs: added group %u:" + "%llu blocks(%llu free %llu reserved)\n", flex_gd->count, + blocks_count, free_blocks, reserved_blocks); +} + +/* Add a flex group to an fs. Ensure we handle all possible error conditions + * _before_ we start modifying the filesystem, because we cannot abort the + * transaction and not have it write the data to disk. + */ +static int ext4_flex_group_add(struct super_block *sb, + struct inode *resize_inode, + struct ext4_new_flex_group_data *flex_gd) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_super_block *es = sbi->s_es; + ext4_fsblk_t o_blocks_count; + ext4_grpblk_t last; + ext4_group_t group; + handle_t *handle; + unsigned reserved_gdb; + int err = 0, err2 = 0, credit; + + BUG_ON(!flex_gd->count || !flex_gd->groups || !flex_gd->bg_flags); + + reserved_gdb = le16_to_cpu(es->s_reserved_gdt_blocks); + o_blocks_count = ext4_blocks_count(es); + ext4_get_group_no_and_offset(sb, o_blocks_count, &group, &last); + BUG_ON(last); + + err = setup_new_flex_group_blocks(sb, flex_gd); + if (err) + goto exit; + /* + * We will always be modifying at least the superblock and GDT + * blocks. If we are adding a group past the last current GDT block, + * we will also modify the inode and the dindirect block. If we + * are adding a group with superblock/GDT backups we will also + * modify each of the reserved GDT dindirect blocks. + */ + credit = 3; /* sb, resize inode, resize inode dindirect */ + /* GDT blocks */ + credit += 1 + DIV_ROUND_UP(flex_gd->count, EXT4_DESC_PER_BLOCK(sb)); + credit += reserved_gdb; /* Reserved GDT dindirect blocks */ + handle = ext4_journal_start_sb(sb, EXT4_HT_RESIZE, credit); + if (IS_ERR(handle)) { + err = PTR_ERR(handle); + goto exit; + } + + BUFFER_TRACE(sbi->s_sbh, "get_write_access"); + err = ext4_journal_get_write_access(handle, sb, sbi->s_sbh, + EXT4_JTR_NONE); + if (err) + goto exit_journal; + + group = flex_gd->groups[0].group; + BUG_ON(group != sbi->s_groups_count); + err = ext4_add_new_descs(handle, sb, group, + resize_inode, flex_gd->count); + if (err) + goto exit_journal; + + err = ext4_setup_new_descs(handle, sb, flex_gd); + if (err) + goto exit_journal; + + ext4_update_super(sb, flex_gd); + + err = ext4_handle_dirty_metadata(handle, NULL, sbi->s_sbh); + +exit_journal: + err2 = ext4_journal_stop(handle); + if (!err) + err = err2; + + if (!err) { + int gdb_num = group / EXT4_DESC_PER_BLOCK(sb); + int gdb_num_end = ((group + flex_gd->count - 1) / + EXT4_DESC_PER_BLOCK(sb)); + int meta_bg = ext4_has_feature_meta_bg(sb) && + gdb_num >= le32_to_cpu(es->s_first_meta_bg); + sector_t padding_blocks = meta_bg ? 0 : sbi->s_sbh->b_blocknr - + ext4_group_first_block_no(sb, 0); + + update_backups(sb, ext4_group_first_block_no(sb, 0), + (char *)es, sizeof(struct ext4_super_block), 0); + for (; gdb_num <= gdb_num_end; gdb_num++) { + struct buffer_head *gdb_bh; + + gdb_bh = sbi_array_rcu_deref(sbi, s_group_desc, + gdb_num); + update_backups(sb, gdb_bh->b_blocknr - padding_blocks, + gdb_bh->b_data, gdb_bh->b_size, meta_bg); + } + } +exit: + return err; +} + +static int ext4_setup_next_flex_gd(struct super_block *sb, + struct ext4_new_flex_group_data *flex_gd, + ext4_fsblk_t n_blocks_count) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_super_block *es = sbi->s_es; + struct ext4_new_group_data *group_data = flex_gd->groups; + ext4_fsblk_t o_blocks_count; + ext4_group_t n_group; + ext4_group_t group; + ext4_group_t last_group; + ext4_grpblk_t last; + ext4_grpblk_t clusters_per_group; + unsigned long i; + + clusters_per_group = EXT4_CLUSTERS_PER_GROUP(sb); + + o_blocks_count = ext4_blocks_count(es); + + if (o_blocks_count == n_blocks_count) + return 0; + + ext4_get_group_no_and_offset(sb, o_blocks_count, &group, &last); + BUG_ON(last); + ext4_get_group_no_and_offset(sb, n_blocks_count - 1, &n_group, &last); + + last_group = group | (flex_gd->resize_bg - 1); + if (last_group > n_group) + last_group = n_group; + + flex_gd->count = last_group - group + 1; + + for (i = 0; i < flex_gd->count; i++) { + int overhead; + + group_data[i].group = group + i; + group_data[i].blocks_count = EXT4_BLOCKS_PER_GROUP(sb); + overhead = ext4_group_overhead_blocks(sb, group + i); + group_data[i].mdata_blocks = overhead; + group_data[i].free_clusters_count = EXT4_CLUSTERS_PER_GROUP(sb); + if (ext4_has_group_desc_csum(sb)) { + flex_gd->bg_flags[i] = EXT4_BG_BLOCK_UNINIT | + EXT4_BG_INODE_UNINIT; + if (!test_opt(sb, INIT_INODE_TABLE)) + flex_gd->bg_flags[i] |= EXT4_BG_INODE_ZEROED; + } else + flex_gd->bg_flags[i] = EXT4_BG_INODE_ZEROED; + } + + if (last_group == n_group && ext4_has_group_desc_csum(sb)) + /* We need to initialize block bitmap of last group. */ + flex_gd->bg_flags[i - 1] &= ~EXT4_BG_BLOCK_UNINIT; + + if ((last_group == n_group) && (last != clusters_per_group - 1)) { + group_data[i - 1].blocks_count = EXT4_C2B(sbi, last + 1); + group_data[i - 1].free_clusters_count -= clusters_per_group - + last - 1; + } + + return 1; +} + +/* Add group descriptor data to an existing or new group descriptor block. + * Ensure we handle all possible error conditions _before_ we start modifying + * the filesystem, because we cannot abort the transaction and not have it + * write the data to disk. + * + * If we are on a GDT block boundary, we need to get the reserved GDT block. + * Otherwise, we may need to add backup GDT blocks for a sparse group. + * + * We only need to hold the superblock lock while we are actually adding + * in the new group's counts to the superblock. Prior to that we have + * not really "added" the group at all. We re-check that we are still + * adding in the last group in case things have changed since verifying. + */ +int ext4_group_add(struct super_block *sb, struct ext4_new_group_data *input) +{ + struct ext4_new_flex_group_data flex_gd; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_super_block *es = sbi->s_es; + int reserved_gdb = ext4_bg_has_super(sb, input->group) ? + le16_to_cpu(es->s_reserved_gdt_blocks) : 0; + struct inode *inode = NULL; + int gdb_off; + int err; + __u16 bg_flags = 0; + + gdb_off = input->group % EXT4_DESC_PER_BLOCK(sb); + + if (gdb_off == 0 && !ext4_has_feature_sparse_super(sb)) { + ext4_warning(sb, "Can't resize non-sparse filesystem further"); + return -EPERM; + } + + if (ext4_blocks_count(es) + input->blocks_count < + ext4_blocks_count(es)) { + ext4_warning(sb, "blocks_count overflow"); + return -EINVAL; + } + + if (le32_to_cpu(es->s_inodes_count) + EXT4_INODES_PER_GROUP(sb) < + le32_to_cpu(es->s_inodes_count)) { + ext4_warning(sb, "inodes_count overflow"); + return -EINVAL; + } + + if (reserved_gdb || gdb_off == 0) { + if (!ext4_has_feature_resize_inode(sb) || + !le16_to_cpu(es->s_reserved_gdt_blocks)) { + ext4_warning(sb, + "No reserved GDT blocks, can't resize"); + return -EPERM; + } + inode = ext4_iget(sb, EXT4_RESIZE_INO, EXT4_IGET_SPECIAL); + if (IS_ERR(inode)) { + ext4_warning(sb, "Error opening resize inode"); + return PTR_ERR(inode); + } + } + + + err = verify_group_input(sb, input); + if (err) + goto out; + + err = ext4_alloc_flex_bg_array(sb, input->group + 1); + if (err) + goto out; + + err = ext4_mb_alloc_groupinfo(sb, input->group + 1); + if (err) + goto out; + + flex_gd.count = 1; + flex_gd.groups = input; + flex_gd.bg_flags = &bg_flags; + err = ext4_flex_group_add(sb, inode, &flex_gd); +out: + iput(inode); + return err; +} /* ext4_group_add */ + +/* + * extend a group without checking assuming that checking has been done. + */ +static int ext4_group_extend_no_check(struct super_block *sb, + ext4_fsblk_t o_blocks_count, ext4_grpblk_t add) +{ + struct ext4_super_block *es = EXT4_SB(sb)->s_es; + handle_t *handle; + int err = 0, err2; + + /* We will update the superblock, one block bitmap, and + * one group descriptor via ext4_group_add_blocks(). + */ + handle = ext4_journal_start_sb(sb, EXT4_HT_RESIZE, 3); + if (IS_ERR(handle)) { + err = PTR_ERR(handle); + ext4_warning(sb, "error %d on journal start", err); + return err; + } + + BUFFER_TRACE(EXT4_SB(sb)->s_sbh, "get_write_access"); + err = ext4_journal_get_write_access(handle, sb, EXT4_SB(sb)->s_sbh, + EXT4_JTR_NONE); + if (err) { + ext4_warning(sb, "error %d on journal write access", err); + goto errout; + } + + lock_buffer(EXT4_SB(sb)->s_sbh); + ext4_blocks_count_set(es, o_blocks_count + add); + ext4_free_blocks_count_set(es, ext4_free_blocks_count(es) + add); + ext4_superblock_csum_set(sb); + unlock_buffer(EXT4_SB(sb)->s_sbh); + ext4_debug("freeing blocks %llu through %llu\n", o_blocks_count, + o_blocks_count + add); + /* We add the blocks to the bitmap and set the group need init bit */ + err = ext4_group_add_blocks(handle, sb, o_blocks_count, add); + if (err) + goto errout; + ext4_handle_dirty_metadata(handle, NULL, EXT4_SB(sb)->s_sbh); + ext4_debug("freed blocks %llu through %llu\n", o_blocks_count, + o_blocks_count + add); +errout: + err2 = ext4_journal_stop(handle); + if (err2 && !err) + err = err2; + + if (!err) { + if (test_opt(sb, DEBUG)) + printk(KERN_DEBUG "EXT4-fs: extended group to %llu " + "blocks\n", ext4_blocks_count(es)); + update_backups(sb, ext4_group_first_block_no(sb, 0), + (char *)es, sizeof(struct ext4_super_block), 0); + } + return err; +} + +/* + * Extend the filesystem to the new number of blocks specified. This entry + * point is only used to extend the current filesystem to the end of the last + * existing group. It can be accessed via ioctl, or by "remount,resize=" + * for emergencies (because it has no dependencies on reserved blocks). + * + * If we _really_ wanted, we could use default values to call ext4_group_add() + * allow the "remount" trick to work for arbitrary resizing, assuming enough + * GDT blocks are reserved to grow to the desired size. + */ +int ext4_group_extend(struct super_block *sb, struct ext4_super_block *es, + ext4_fsblk_t n_blocks_count) +{ + ext4_fsblk_t o_blocks_count; + ext4_grpblk_t last; + ext4_grpblk_t add; + struct buffer_head *bh; + ext4_group_t group; + + o_blocks_count = ext4_blocks_count(es); + + if (test_opt(sb, DEBUG)) + ext4_msg(sb, KERN_DEBUG, + "extending last group from %llu to %llu blocks", + o_blocks_count, n_blocks_count); + + if (n_blocks_count == 0 || n_blocks_count == o_blocks_count) + return 0; + + if (n_blocks_count > (sector_t)(~0ULL) >> (sb->s_blocksize_bits - 9)) { + ext4_msg(sb, KERN_ERR, + "filesystem too large to resize to %llu blocks safely", + n_blocks_count); + return -EINVAL; + } + + if (n_blocks_count < o_blocks_count) { + ext4_warning(sb, "can't shrink FS - resize aborted"); + return -EINVAL; + } + + /* Handle the remaining blocks in the last group only. */ + ext4_get_group_no_and_offset(sb, o_blocks_count, &group, &last); + + if (last == 0) { + ext4_warning(sb, "need to use ext2online to resize further"); + return -EPERM; + } + + add = EXT4_BLOCKS_PER_GROUP(sb) - last; + + if (o_blocks_count + add < o_blocks_count) { + ext4_warning(sb, "blocks_count overflow"); + return -EINVAL; + } + + if (o_blocks_count + add > n_blocks_count) + add = n_blocks_count - o_blocks_count; + + if (o_blocks_count + add < n_blocks_count) + ext4_warning(sb, "will only finish group (%llu blocks, %u new)", + o_blocks_count + add, add); + + /* See if the device is actually as big as what was requested */ + bh = ext4_sb_bread(sb, o_blocks_count + add - 1, 0); + if (IS_ERR(bh)) { + ext4_warning(sb, "can't read last block, resize aborted"); + return -ENOSPC; + } + brelse(bh); + + return ext4_group_extend_no_check(sb, o_blocks_count, add); +} /* ext4_group_extend */ + + +static int num_desc_blocks(struct super_block *sb, ext4_group_t groups) +{ + return (groups + EXT4_DESC_PER_BLOCK(sb) - 1) / EXT4_DESC_PER_BLOCK(sb); +} + +/* + * Release the resize inode and drop the resize_inode feature if there + * are no more reserved gdt blocks, and then convert the file system + * to enable meta_bg + */ +static int ext4_convert_meta_bg(struct super_block *sb, struct inode *inode) +{ + handle_t *handle; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_super_block *es = sbi->s_es; + struct ext4_inode_info *ei = EXT4_I(inode); + ext4_fsblk_t nr; + int i, ret, err = 0; + int credits = 1; + + ext4_msg(sb, KERN_INFO, "Converting file system to meta_bg"); + if (inode) { + if (es->s_reserved_gdt_blocks) { + ext4_error(sb, "Unexpected non-zero " + "s_reserved_gdt_blocks"); + return -EPERM; + } + + /* Do a quick sanity check of the resize inode */ + if (inode->i_blocks != 1 << (inode->i_blkbits - + (9 - sbi->s_cluster_bits))) + goto invalid_resize_inode; + for (i = 0; i < EXT4_N_BLOCKS; i++) { + if (i == EXT4_DIND_BLOCK) { + if (ei->i_data[i]) + continue; + else + goto invalid_resize_inode; + } + if (ei->i_data[i]) + goto invalid_resize_inode; + } + credits += 3; /* block bitmap, bg descriptor, resize inode */ + } + + handle = ext4_journal_start_sb(sb, EXT4_HT_RESIZE, credits); + if (IS_ERR(handle)) + return PTR_ERR(handle); + + BUFFER_TRACE(sbi->s_sbh, "get_write_access"); + err = ext4_journal_get_write_access(handle, sb, sbi->s_sbh, + EXT4_JTR_NONE); + if (err) + goto errout; + + lock_buffer(sbi->s_sbh); + ext4_clear_feature_resize_inode(sb); + ext4_set_feature_meta_bg(sb); + sbi->s_es->s_first_meta_bg = + cpu_to_le32(num_desc_blocks(sb, sbi->s_groups_count)); + ext4_superblock_csum_set(sb); + unlock_buffer(sbi->s_sbh); + + err = ext4_handle_dirty_metadata(handle, NULL, sbi->s_sbh); + if (err) { + ext4_std_error(sb, err); + goto errout; + } + + if (inode) { + nr = le32_to_cpu(ei->i_data[EXT4_DIND_BLOCK]); + ext4_free_blocks(handle, inode, NULL, nr, 1, + EXT4_FREE_BLOCKS_METADATA | + EXT4_FREE_BLOCKS_FORGET); + ei->i_data[EXT4_DIND_BLOCK] = 0; + inode->i_blocks = 0; + + err = ext4_mark_inode_dirty(handle, inode); + if (err) + ext4_std_error(sb, err); + } + +errout: + ret = ext4_journal_stop(handle); + return err ? err : ret; + +invalid_resize_inode: + ext4_error(sb, "corrupted/inconsistent resize inode"); + return -EINVAL; +} + +/* + * ext4_resize_fs() resizes a fs to new size specified by @n_blocks_count + * + * @sb: super block of the fs to be resized + * @n_blocks_count: the number of blocks resides in the resized fs + */ +int ext4_resize_fs(struct super_block *sb, ext4_fsblk_t n_blocks_count) +{ + struct ext4_new_flex_group_data *flex_gd = NULL; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_super_block *es = sbi->s_es; + struct buffer_head *bh; + struct inode *resize_inode = NULL; + ext4_grpblk_t add, offset; + unsigned long n_desc_blocks; + unsigned long o_desc_blocks; + ext4_group_t o_group; + ext4_group_t n_group; + ext4_fsblk_t o_blocks_count; + ext4_fsblk_t n_blocks_count_retry = 0; + unsigned long last_update_time = 0; + int err = 0; + int meta_bg; + unsigned int flexbg_size = ext4_flex_bg_size(sbi); + + /* See if the device is actually as big as what was requested */ + bh = ext4_sb_bread(sb, n_blocks_count - 1, 0); + if (IS_ERR(bh)) { + ext4_warning(sb, "can't read last block, resize aborted"); + return -ENOSPC; + } + brelse(bh); + + /* + * For bigalloc, trim the requested size to the nearest cluster + * boundary to avoid creating an unusable filesystem. We do this + * silently, instead of returning an error, to avoid breaking + * callers that blindly resize the filesystem to the full size of + * the underlying block device. + */ + if (ext4_has_feature_bigalloc(sb)) + n_blocks_count &= ~((1 << EXT4_CLUSTER_BITS(sb)) - 1); + +retry: + o_blocks_count = ext4_blocks_count(es); + + ext4_msg(sb, KERN_INFO, "resizing filesystem from %llu " + "to %llu blocks", o_blocks_count, n_blocks_count); + + if (n_blocks_count < o_blocks_count) { + /* On-line shrinking not supported */ + ext4_warning(sb, "can't shrink FS - resize aborted"); + return -EINVAL; + } + + if (n_blocks_count == o_blocks_count) + /* Nothing need to do */ + return 0; + + n_group = ext4_get_group_number(sb, n_blocks_count - 1); + if (n_group >= (0xFFFFFFFFUL / EXT4_INODES_PER_GROUP(sb))) { + ext4_warning(sb, "resize would cause inodes_count overflow"); + return -EINVAL; + } + ext4_get_group_no_and_offset(sb, o_blocks_count - 1, &o_group, &offset); + + n_desc_blocks = num_desc_blocks(sb, n_group + 1); + o_desc_blocks = num_desc_blocks(sb, sbi->s_groups_count); + + meta_bg = ext4_has_feature_meta_bg(sb); + + if (ext4_has_feature_resize_inode(sb)) { + if (meta_bg) { + ext4_error(sb, "resize_inode and meta_bg enabled " + "simultaneously"); + return -EINVAL; + } + if (n_desc_blocks > o_desc_blocks + + le16_to_cpu(es->s_reserved_gdt_blocks)) { + n_blocks_count_retry = n_blocks_count; + n_desc_blocks = o_desc_blocks + + le16_to_cpu(es->s_reserved_gdt_blocks); + n_group = n_desc_blocks * EXT4_DESC_PER_BLOCK(sb); + n_blocks_count = (ext4_fsblk_t)n_group * + EXT4_BLOCKS_PER_GROUP(sb) + + le32_to_cpu(es->s_first_data_block); + n_group--; /* set to last group number */ + } + + if (!resize_inode) + resize_inode = ext4_iget(sb, EXT4_RESIZE_INO, + EXT4_IGET_SPECIAL); + if (IS_ERR(resize_inode)) { + ext4_warning(sb, "Error opening resize inode"); + return PTR_ERR(resize_inode); + } + } + + if ((!resize_inode && !meta_bg && n_desc_blocks > o_desc_blocks) || n_blocks_count == o_blocks_count) { + err = ext4_convert_meta_bg(sb, resize_inode); + if (err) + goto out; + if (resize_inode) { + iput(resize_inode); + resize_inode = NULL; + } + if (n_blocks_count_retry) { + n_blocks_count = n_blocks_count_retry; + n_blocks_count_retry = 0; + goto retry; + } + } + + /* + * Make sure the last group has enough space so that it's + * guaranteed to have enough space for all metadata blocks + * that it might need to hold. (We might not need to store + * the inode table blocks in the last block group, but there + * will be cases where this might be needed.) + */ + if ((ext4_group_first_block_no(sb, n_group) + + ext4_group_overhead_blocks(sb, n_group) + 2 + + sbi->s_itb_per_group + sbi->s_cluster_ratio) >= n_blocks_count) { + n_blocks_count = ext4_group_first_block_no(sb, n_group); + n_group--; + n_blocks_count_retry = 0; + if (resize_inode) { + iput(resize_inode); + resize_inode = NULL; + } + goto retry; + } + + /* extend the last group */ + if (n_group == o_group) + add = n_blocks_count - o_blocks_count; + else + add = EXT4_C2B(sbi, EXT4_CLUSTERS_PER_GROUP(sb) - (offset + 1)); + if (add > 0) { + err = ext4_group_extend_no_check(sb, o_blocks_count, add); + if (err) + goto out; + } + + if (ext4_blocks_count(es) == n_blocks_count && n_blocks_count_retry == 0) + goto out; + + err = ext4_alloc_flex_bg_array(sb, n_group + 1); + if (err) + goto out; + + err = ext4_mb_alloc_groupinfo(sb, n_group + 1); + if (err) + goto out; + + flex_gd = alloc_flex_gd(flexbg_size, o_group, n_group); + if (flex_gd == NULL) { + err = -ENOMEM; + goto out; + } + + /* Add flex groups. Note that a regular group is a + * flex group with 1 group. + */ + while (ext4_setup_next_flex_gd(sb, flex_gd, n_blocks_count)) { + if (time_is_before_jiffies(last_update_time + HZ * 10)) { + if (last_update_time) + ext4_msg(sb, KERN_INFO, + "resized to %llu blocks", + ext4_blocks_count(es)); + last_update_time = jiffies; + } + if (ext4_alloc_group_tables(sb, flex_gd, flexbg_size) != 0) + break; + err = ext4_flex_group_add(sb, resize_inode, flex_gd); + if (unlikely(err)) + break; + } + + if (!err && n_blocks_count_retry) { + n_blocks_count = n_blocks_count_retry; + n_blocks_count_retry = 0; + free_flex_gd(flex_gd); + flex_gd = NULL; + if (resize_inode) { + iput(resize_inode); + resize_inode = NULL; + } + goto retry; + } + +out: + if (flex_gd) + free_flex_gd(flex_gd); + if (resize_inode != NULL) + iput(resize_inode); + if (err) + ext4_warning(sb, "error (%d) occurred during " + "file system resize", err); + ext4_msg(sb, KERN_INFO, "resized filesystem to %llu", + ext4_blocks_count(es)); + return err; +} diff --git a/fs/ext4/super.c b/fs/ext4/super.c new file mode 100644 index 000000000..bca0dc87d --- /dev/null +++ b/fs/ext4/super.c @@ -0,0 +1,7641 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext4/super.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * from + * + * linux/fs/minix/inode.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * Big-endian to little-endian byte-swapping/bitmaps by + * David S. Miller (davem@caip.rutgers.edu), 1995 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ext4.h" +#include "ext4_extents.h" /* Needed for trace points definition */ +#include "ext4_jbd2.h" +#include "xattr.h" +#include "acl.h" +#include "mballoc.h" +#include "fsmap.h" + +#define CREATE_TRACE_POINTS +#include + +static struct ext4_lazy_init *ext4_li_info; +static DEFINE_MUTEX(ext4_li_mtx); +static struct ratelimit_state ext4_mount_msg_ratelimit; + +static int ext4_load_journal(struct super_block *, struct ext4_super_block *, + unsigned long journal_devnum); +static int ext4_show_options(struct seq_file *seq, struct dentry *root); +static void ext4_update_super(struct super_block *sb); +static int ext4_commit_super(struct super_block *sb); +static int ext4_mark_recovery_complete(struct super_block *sb, + struct ext4_super_block *es); +static int ext4_clear_journal_err(struct super_block *sb, + struct ext4_super_block *es); +static int ext4_sync_fs(struct super_block *sb, int wait); +static int ext4_statfs(struct dentry *dentry, struct kstatfs *buf); +static int ext4_unfreeze(struct super_block *sb); +static int ext4_freeze(struct super_block *sb); +static inline int ext2_feature_set_ok(struct super_block *sb); +static inline int ext3_feature_set_ok(struct super_block *sb); +static void ext4_unregister_li_request(struct super_block *sb); +static void ext4_clear_request_list(void); +static struct inode *ext4_get_journal_inode(struct super_block *sb, + unsigned int journal_inum); +static int ext4_validate_options(struct fs_context *fc); +static int ext4_check_opt_consistency(struct fs_context *fc, + struct super_block *sb); +static void ext4_apply_options(struct fs_context *fc, struct super_block *sb); +static int ext4_parse_param(struct fs_context *fc, struct fs_parameter *param); +static int ext4_get_tree(struct fs_context *fc); +static int ext4_reconfigure(struct fs_context *fc); +static void ext4_fc_free(struct fs_context *fc); +static int ext4_init_fs_context(struct fs_context *fc); +static void ext4_kill_sb(struct super_block *sb); +static const struct fs_parameter_spec ext4_param_specs[]; + +/* + * Lock ordering + * + * page fault path: + * mmap_lock -> sb_start_pagefault -> invalidate_lock (r) -> transaction start + * -> page lock -> i_data_sem (rw) + * + * buffered write path: + * sb_start_write -> i_mutex -> mmap_lock + * sb_start_write -> i_mutex -> transaction start -> page lock -> + * i_data_sem (rw) + * + * truncate: + * sb_start_write -> i_mutex -> invalidate_lock (w) -> i_mmap_rwsem (w) -> + * page lock + * sb_start_write -> i_mutex -> invalidate_lock (w) -> transaction start -> + * i_data_sem (rw) + * + * direct IO: + * sb_start_write -> i_mutex -> mmap_lock + * sb_start_write -> i_mutex -> transaction start -> i_data_sem (rw) + * + * writepages: + * transaction start -> page lock(s) -> i_data_sem (rw) + */ + +static const struct fs_context_operations ext4_context_ops = { + .parse_param = ext4_parse_param, + .get_tree = ext4_get_tree, + .reconfigure = ext4_reconfigure, + .free = ext4_fc_free, +}; + + +#if !defined(CONFIG_EXT2_FS) && !defined(CONFIG_EXT2_FS_MODULE) && defined(CONFIG_EXT4_USE_FOR_EXT2) +static struct file_system_type ext2_fs_type = { + .owner = THIS_MODULE, + .name = "ext2", + .init_fs_context = ext4_init_fs_context, + .parameters = ext4_param_specs, + .kill_sb = ext4_kill_sb, + .fs_flags = FS_REQUIRES_DEV, +}; +MODULE_ALIAS_FS("ext2"); +MODULE_ALIAS("ext2"); +#define IS_EXT2_SB(sb) ((sb)->s_type == &ext2_fs_type) +#else +#define IS_EXT2_SB(sb) (0) +#endif + + +static struct file_system_type ext3_fs_type = { + .owner = THIS_MODULE, + .name = "ext3", + .init_fs_context = ext4_init_fs_context, + .parameters = ext4_param_specs, + .kill_sb = ext4_kill_sb, + .fs_flags = FS_REQUIRES_DEV, +}; +MODULE_ALIAS_FS("ext3"); +MODULE_ALIAS("ext3"); +#define IS_EXT3_SB(sb) ((sb)->s_type == &ext3_fs_type) + + +static inline void __ext4_read_bh(struct buffer_head *bh, blk_opf_t op_flags, + bio_end_io_t end_io, bool simu_fail) +{ + if (simu_fail) { + clear_buffer_uptodate(bh); + unlock_buffer(bh); + return; + } + + /* + * buffer's verified bit is no longer valid after reading from + * disk again due to write out error, clear it to make sure we + * recheck the buffer contents. + */ + clear_buffer_verified(bh); + + if (!end_io) + end_io = bh_end_read; + bh_submit(bh, REQ_OP_READ | op_flags, end_io); +} + +void ext4_read_bh_nowait(struct buffer_head *bh, blk_opf_t op_flags, + bio_end_io_t end_io, bool simu_fail) +{ + BUG_ON(!buffer_locked(bh)); + + if (ext4_buffer_uptodate(bh)) { + unlock_buffer(bh); + return; + } + __ext4_read_bh(bh, op_flags, end_io, simu_fail); +} + +int ext4_read_bh(struct buffer_head *bh, blk_opf_t op_flags, + bio_end_io_t end_io, bool simu_fail) +{ + BUG_ON(!buffer_locked(bh)); + + if (ext4_buffer_uptodate(bh)) { + unlock_buffer(bh); + return 0; + } + + __ext4_read_bh(bh, op_flags, end_io, simu_fail); + + wait_on_buffer(bh); + if (buffer_uptodate(bh)) + return 0; + return -EIO; +} + +int ext4_read_bh_lock(struct buffer_head *bh, blk_opf_t op_flags, bool wait) +{ + lock_buffer(bh); + if (!wait) { + ext4_read_bh_nowait(bh, op_flags, NULL, false); + return 0; + } + return ext4_read_bh(bh, op_flags, NULL, false); +} + +/* + * This works like __bread_gfp() except it uses ERR_PTR for error + * returns. Currently with sb_bread it's impossible to distinguish + * between ENOMEM and EIO situations (since both result in a NULL + * return. + */ +static struct buffer_head *__ext4_sb_bread_gfp(struct super_block *sb, + sector_t block, + blk_opf_t op_flags, gfp_t gfp) +{ + struct buffer_head *bh; + int ret; + + bh = sb_getblk_gfp(sb, block, gfp); + if (bh == NULL) + return ERR_PTR(-ENOMEM); + if (ext4_buffer_uptodate(bh)) + return bh; + + ret = ext4_read_bh_lock(bh, REQ_META | op_flags, true); + if (ret) { + put_bh(bh); + return ERR_PTR(ret); + } + return bh; +} + +struct buffer_head *ext4_sb_bread(struct super_block *sb, sector_t block, + blk_opf_t op_flags) +{ + gfp_t gfp = mapping_gfp_constraint(sb->s_bdev->bd_mapping, + ~__GFP_FS) | __GFP_MOVABLE; + + return __ext4_sb_bread_gfp(sb, block, op_flags, gfp); +} + +struct buffer_head *ext4_sb_bread_unmovable(struct super_block *sb, + sector_t block) +{ + gfp_t gfp = mapping_gfp_constraint(sb->s_bdev->bd_mapping, + ~__GFP_FS); + + return __ext4_sb_bread_gfp(sb, block, 0, gfp); +} + +struct buffer_head *ext4_sb_bread_nofail(struct super_block *sb, + sector_t block) +{ + gfp_t gfp = mapping_gfp_constraint(sb->s_bdev->bd_mapping, + ~__GFP_FS) | __GFP_MOVABLE | __GFP_NOFAIL; + + return __ext4_sb_bread_gfp(sb, block, 0, gfp); +} + +void ext4_sb_breadahead_unmovable(struct super_block *sb, sector_t block) +{ + struct buffer_head *bh = bdev_getblk(sb->s_bdev, block, + sb->s_blocksize, GFP_NOWAIT); + + if (likely(bh)) { + if (trylock_buffer(bh)) + ext4_read_bh_nowait(bh, REQ_RAHEAD, NULL, false); + brelse(bh); + } +} + +static int ext4_verify_csum_type(struct super_block *sb, + struct ext4_super_block *es) +{ + if (!ext4_has_feature_metadata_csum(sb)) + return 1; + + return es->s_checksum_type == EXT4_CRC32C_CHKSUM; +} + +__le32 ext4_superblock_csum(struct ext4_super_block *es) +{ + int offset = offsetof(struct ext4_super_block, s_checksum); + __u32 csum; + + csum = ext4_chksum(~0, (char *)es, offset); + + return cpu_to_le32(csum); +} + +static int ext4_superblock_csum_verify(struct super_block *sb, + struct ext4_super_block *es) +{ + if (!ext4_has_feature_metadata_csum(sb)) + return 1; + + return es->s_checksum == ext4_superblock_csum(es); +} + +void ext4_superblock_csum_set(struct super_block *sb) +{ + struct ext4_super_block *es = EXT4_SB(sb)->s_es; + + if (!ext4_has_feature_metadata_csum(sb)) + return; + + es->s_checksum = ext4_superblock_csum(es); +} + +ext4_fsblk_t ext4_block_bitmap(struct super_block *sb, + struct ext4_group_desc *bg) +{ + return le32_to_cpu(bg->bg_block_bitmap_lo) | + (EXT4_DESC_SIZE(sb) >= EXT4_MIN_DESC_SIZE_64BIT ? + (ext4_fsblk_t)le32_to_cpu(bg->bg_block_bitmap_hi) << 32 : 0); +} + +ext4_fsblk_t ext4_inode_bitmap(struct super_block *sb, + struct ext4_group_desc *bg) +{ + return le32_to_cpu(bg->bg_inode_bitmap_lo) | + (EXT4_DESC_SIZE(sb) >= EXT4_MIN_DESC_SIZE_64BIT ? + (ext4_fsblk_t)le32_to_cpu(bg->bg_inode_bitmap_hi) << 32 : 0); +} + +ext4_fsblk_t ext4_inode_table(struct super_block *sb, + struct ext4_group_desc *bg) +{ + return le32_to_cpu(bg->bg_inode_table_lo) | + (EXT4_DESC_SIZE(sb) >= EXT4_MIN_DESC_SIZE_64BIT ? + (ext4_fsblk_t)le32_to_cpu(bg->bg_inode_table_hi) << 32 : 0); +} + +__u32 ext4_free_group_clusters(struct super_block *sb, + struct ext4_group_desc *bg) +{ + return le16_to_cpu(bg->bg_free_blocks_count_lo) | + (EXT4_DESC_SIZE(sb) >= EXT4_MIN_DESC_SIZE_64BIT ? + (__u32)le16_to_cpu(bg->bg_free_blocks_count_hi) << 16 : 0); +} + +__u32 ext4_free_inodes_count(struct super_block *sb, + struct ext4_group_desc *bg) +{ + return le16_to_cpu(READ_ONCE(bg->bg_free_inodes_count_lo)) | + (EXT4_DESC_SIZE(sb) >= EXT4_MIN_DESC_SIZE_64BIT ? + (__u32)le16_to_cpu(READ_ONCE(bg->bg_free_inodes_count_hi)) << 16 : 0); +} + +__u32 ext4_used_dirs_count(struct super_block *sb, + struct ext4_group_desc *bg) +{ + return le16_to_cpu(bg->bg_used_dirs_count_lo) | + (EXT4_DESC_SIZE(sb) >= EXT4_MIN_DESC_SIZE_64BIT ? + (__u32)le16_to_cpu(bg->bg_used_dirs_count_hi) << 16 : 0); +} + +__u32 ext4_itable_unused_count(struct super_block *sb, + struct ext4_group_desc *bg) +{ + return le16_to_cpu(bg->bg_itable_unused_lo) | + (EXT4_DESC_SIZE(sb) >= EXT4_MIN_DESC_SIZE_64BIT ? + (__u32)le16_to_cpu(bg->bg_itable_unused_hi) << 16 : 0); +} + +void ext4_block_bitmap_set(struct super_block *sb, + struct ext4_group_desc *bg, ext4_fsblk_t blk) +{ + bg->bg_block_bitmap_lo = cpu_to_le32((u32)blk); + if (EXT4_DESC_SIZE(sb) >= EXT4_MIN_DESC_SIZE_64BIT) + bg->bg_block_bitmap_hi = cpu_to_le32(blk >> 32); +} + +void ext4_inode_bitmap_set(struct super_block *sb, + struct ext4_group_desc *bg, ext4_fsblk_t blk) +{ + bg->bg_inode_bitmap_lo = cpu_to_le32((u32)blk); + if (EXT4_DESC_SIZE(sb) >= EXT4_MIN_DESC_SIZE_64BIT) + bg->bg_inode_bitmap_hi = cpu_to_le32(blk >> 32); +} + +void ext4_inode_table_set(struct super_block *sb, + struct ext4_group_desc *bg, ext4_fsblk_t blk) +{ + bg->bg_inode_table_lo = cpu_to_le32((u32)blk); + if (EXT4_DESC_SIZE(sb) >= EXT4_MIN_DESC_SIZE_64BIT) + bg->bg_inode_table_hi = cpu_to_le32(blk >> 32); +} + +void ext4_free_group_clusters_set(struct super_block *sb, + struct ext4_group_desc *bg, __u32 count) +{ + bg->bg_free_blocks_count_lo = cpu_to_le16((__u16)count); + if (EXT4_DESC_SIZE(sb) >= EXT4_MIN_DESC_SIZE_64BIT) + bg->bg_free_blocks_count_hi = cpu_to_le16(count >> 16); +} + +void ext4_free_inodes_set(struct super_block *sb, + struct ext4_group_desc *bg, __u32 count) +{ + WRITE_ONCE(bg->bg_free_inodes_count_lo, cpu_to_le16((__u16)count)); + if (EXT4_DESC_SIZE(sb) >= EXT4_MIN_DESC_SIZE_64BIT) + WRITE_ONCE(bg->bg_free_inodes_count_hi, cpu_to_le16(count >> 16)); +} + +void ext4_used_dirs_set(struct super_block *sb, + struct ext4_group_desc *bg, __u32 count) +{ + bg->bg_used_dirs_count_lo = cpu_to_le16((__u16)count); + if (EXT4_DESC_SIZE(sb) >= EXT4_MIN_DESC_SIZE_64BIT) + bg->bg_used_dirs_count_hi = cpu_to_le16(count >> 16); +} + +void ext4_itable_unused_set(struct super_block *sb, + struct ext4_group_desc *bg, __u32 count) +{ + bg->bg_itable_unused_lo = cpu_to_le16((__u16)count); + if (EXT4_DESC_SIZE(sb) >= EXT4_MIN_DESC_SIZE_64BIT) + bg->bg_itable_unused_hi = cpu_to_le16(count >> 16); +} + +static void __ext4_update_tstamp(__le32 *lo, __u8 *hi, time64_t now) +{ + now = clamp_val(now, 0, (1ull << 40) - 1); + + *lo = cpu_to_le32(lower_32_bits(now)); + *hi = upper_32_bits(now); +} + +static time64_t __ext4_get_tstamp(__le32 *lo, __u8 *hi) +{ + return ((time64_t)(*hi) << 32) + le32_to_cpu(*lo); +} +#define ext4_update_tstamp(es, tstamp) \ + __ext4_update_tstamp(&(es)->tstamp, &(es)->tstamp ## _hi, \ + ktime_get_real_seconds()) +#define ext4_get_tstamp(es, tstamp) \ + __ext4_get_tstamp(&(es)->tstamp, &(es)->tstamp ## _hi) + +/* + * The ext4_maybe_update_superblock() function checks and updates the + * superblock if needed. + * + * This function is designed to update the on-disk superblock only under + * certain conditions to prevent excessive disk writes and unnecessary + * waking of the disk from sleep. The superblock will be updated if: + * 1. More than sbi->s_sb_update_sec (def: 1 hour) has passed since the last + * superblock update + * 2. More than sbi->s_sb_update_kb (def: 16MB) kbs have been written since the + * last superblock update. + * + * @sb: The superblock + */ +static void ext4_maybe_update_superblock(struct super_block *sb) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_super_block *es = sbi->s_es; + journal_t *journal = sbi->s_journal; + time64_t now; + __u64 last_update; + __u64 lifetime_write_kbytes; + __u64 diff_size; + + if (ext4_emergency_state(sb) || sb_rdonly(sb) || + !(sb->s_flags & SB_ACTIVE) || !journal || + journal->j_flags & JBD2_UNMOUNT) + return; + + now = ktime_get_real_seconds(); + last_update = ext4_get_tstamp(es, s_wtime); + + if (likely(now - last_update < sbi->s_sb_update_sec)) + return; + + lifetime_write_kbytes = sbi->s_kbytes_written + + ((part_stat_read(sb->s_bdev, sectors[STAT_WRITE]) - + sbi->s_sectors_written_start) >> 1); + + /* Get the number of kilobytes not written to disk to account + * for statistics and compare with a multiple of 16 MB. This + * is used to determine when the next superblock commit should + * occur (i.e. not more often than once per 16MB if there was + * less written in an hour). + */ + diff_size = lifetime_write_kbytes - le64_to_cpu(es->s_kbytes_written); + + if (diff_size > sbi->s_sb_update_kb) + schedule_work(&EXT4_SB(sb)->s_sb_upd_work); +} + +static void ext4_journal_commit_callback(journal_t *journal, transaction_t *txn) +{ + struct super_block *sb = journal->j_private; + + BUG_ON(txn->t_state == T_FINISHED); + + ext4_process_freed_data(sb, txn->t_tid); + ext4_maybe_update_superblock(sb); +} + +static bool ext4_journalled_writepage_needs_redirty(struct jbd2_inode *jinode, + struct folio *folio) +{ + struct buffer_head *bh, *head; + struct journal_head *jh; + transaction_t *trans = READ_ONCE(jinode->i_transaction); + + bh = head = folio_buffers(folio); + do { + /* + * We have to redirty a page in these cases: + * 1) If buffer is dirty, it means the page was dirty because it + * contains a buffer that needs checkpointing. So the dirty bit + * needs to be preserved so that checkpointing writes the buffer + * properly. + * 2) If buffer is not part of the committing transaction + * (we may have just accidentally come across this buffer because + * inode range tracking is not exact) or if the currently running + * transaction already contains this buffer as well, dirty bit + * needs to be preserved so that the buffer gets writeprotected + * properly on running transaction's commit. + */ + jh = bh2jh(bh); + if (buffer_dirty(bh) || + (jh && (jh->b_transaction != trans || + jh->b_next_transaction))) + return true; + } while ((bh = bh->b_this_page) != head); + + return false; +} + +static int ext4_journalled_submit_inode_data_buffers(struct jbd2_inode *jinode) +{ + struct address_space *mapping = jinode->i_vfs_inode->i_mapping; + loff_t range_start, range_end; + struct writeback_control wbc = { + .sync_mode = WB_SYNC_ALL, + .nr_to_write = LONG_MAX, + }; + struct folio *folio = NULL; + int error; + + if (!jbd2_jinode_get_dirty_range(jinode, &range_start, &range_end)) + return 0; + + wbc.range_start = range_start; + wbc.range_end = range_end; + + /* + * writeback_iter() already checks for dirty pages and calls + * folio_clear_dirty_for_io(), which we want to write protect the + * folios. + * + * However, we may have to redirty a folio sometimes. + */ + while ((folio = writeback_iter(mapping, &wbc, folio, &error))) { + if (ext4_journalled_writepage_needs_redirty(jinode, folio)) + folio_redirty_for_writepage(&wbc, folio); + folio_unlock(folio); + } + + return error; +} + +static int ext4_journal_submit_inode_data_buffers(struct jbd2_inode *jinode) +{ + int ret; + + if (ext4_should_journal_data(jinode->i_vfs_inode)) + ret = ext4_journalled_submit_inode_data_buffers(jinode); + else + ret = ext4_normal_submit_inode_data_buffers(jinode); + return ret; +} + +static int ext4_journal_finish_inode_data_buffers(struct jbd2_inode *jinode) +{ + int ret = 0; + + if (!ext4_should_journal_data(jinode->i_vfs_inode)) + ret = jbd2_journal_finish_inode_data_buffers(jinode); + + return ret; +} + +static bool system_going_down(void) +{ + return system_state == SYSTEM_HALT || system_state == SYSTEM_POWER_OFF + || system_state == SYSTEM_RESTART; +} + +struct ext4_err_translation { + int code; + int errno; +}; + +#define EXT4_ERR_TRANSLATE(err) { .code = EXT4_ERR_##err, .errno = err } + +static struct ext4_err_translation err_translation[] = { + EXT4_ERR_TRANSLATE(EIO), + EXT4_ERR_TRANSLATE(ENOMEM), + EXT4_ERR_TRANSLATE(EFSBADCRC), + EXT4_ERR_TRANSLATE(EFSCORRUPTED), + EXT4_ERR_TRANSLATE(ENOSPC), + EXT4_ERR_TRANSLATE(ENOKEY), + EXT4_ERR_TRANSLATE(EROFS), + EXT4_ERR_TRANSLATE(EFBIG), + EXT4_ERR_TRANSLATE(EEXIST), + EXT4_ERR_TRANSLATE(ERANGE), + EXT4_ERR_TRANSLATE(EOVERFLOW), + EXT4_ERR_TRANSLATE(EBUSY), + EXT4_ERR_TRANSLATE(ENOTDIR), + EXT4_ERR_TRANSLATE(ENOTEMPTY), + EXT4_ERR_TRANSLATE(ESHUTDOWN), + EXT4_ERR_TRANSLATE(EFAULT), +}; + +static int ext4_errno_to_code(int errno) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(err_translation); i++) + if (err_translation[i].errno == errno) + return err_translation[i].code; + return EXT4_ERR_UNKNOWN; +} + +static void save_error_info(struct super_block *sb, int error, + __u32 ino, __u64 block, + const char *func, unsigned int line) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + + /* We default to EFSCORRUPTED error... */ + if (error == 0) + error = EFSCORRUPTED; + + spin_lock(&sbi->s_error_lock); + sbi->s_add_error_count++; + sbi->s_last_error_code = error; + sbi->s_last_error_line = line; + sbi->s_last_error_ino = ino; + sbi->s_last_error_block = block; + sbi->s_last_error_func = func; + sbi->s_last_error_time = ktime_get_real_seconds(); + if (!sbi->s_first_error_time) { + sbi->s_first_error_code = error; + sbi->s_first_error_line = line; + sbi->s_first_error_ino = ino; + sbi->s_first_error_block = block; + sbi->s_first_error_func = func; + sbi->s_first_error_time = sbi->s_last_error_time; + } + spin_unlock(&sbi->s_error_lock); +} + +/* Deal with the reporting of failure conditions on a filesystem such as + * inconsistencies detected or read IO failures. + * + * On ext2, we can store the error state of the filesystem in the + * superblock. That is not possible on ext4, because we may have other + * write ordering constraints on the superblock which prevent us from + * writing it out straight away; and given that the journal is about to + * be aborted, we can't rely on the current, or future, transactions to + * write out the superblock safely. + * + * We'll just use the jbd2_journal_abort() error code to record an error in + * the journal instead. On recovery, the journal will complain about + * that error until we've noted it down and cleared it. + * + * If force_ro is set, we unconditionally force the filesystem into an + * ABORT|READONLY state, unless the error response on the fs has been set to + * panic in which case we take the easy way out and panic immediately. This is + * used to deal with unrecoverable failures such as journal IO errors or ENOMEM + * at a critical moment in log management. + */ +static void ext4_handle_error(struct super_block *sb, bool force_ro, int error, + __u32 ino, __u64 block, + const char *func, unsigned int line) +{ + journal_t *journal = EXT4_SB(sb)->s_journal; + bool continue_fs = !force_ro && test_opt(sb, ERRORS_CONT); + + EXT4_SB(sb)->s_mount_state |= EXT4_ERROR_FS; + if (test_opt(sb, WARN_ON_ERROR)) + WARN_ON_ONCE(1); + + if (!continue_fs && !ext4_emergency_ro(sb) && journal) + jbd2_journal_abort(journal, -error); + + if (!bdev_read_only(sb->s_bdev)) { + save_error_info(sb, error, ino, block, func, line); + /* + * In case the fs should keep running, we need to writeout + * superblock through the journal. Due to lock ordering + * constraints, it may not be safe to do it right here so we + * defer superblock flushing to a workqueue. We just need to be + * careful when the journal is already shutting down. If we get + * here in that case, just update the sb directly as the last + * transaction won't commit anyway. + */ + if (continue_fs && journal && + !ext4_test_mount_flag(sb, EXT4_MF_JOURNAL_DESTROY)) + schedule_work(&EXT4_SB(sb)->s_sb_upd_work); + else + ext4_commit_super(sb); + } + + /* + * We force ERRORS_RO behavior when system is rebooting. Otherwise we + * could panic during 'reboot -f' as the underlying device got already + * disabled. + */ + if (test_opt(sb, ERRORS_PANIC) && !system_going_down()) { + panic("EXT4-fs (device %s): panic forced after error\n", + sb->s_id); + } + + if (ext4_emergency_ro(sb) || continue_fs) + return; + + ext4_msg(sb, KERN_CRIT, "Remounting filesystem read-only"); + /* + * We don't set SB_RDONLY because that requires sb->s_umount + * semaphore and setting it without proper remount procedure is + * confusing code such as freeze_super() leading to deadlocks + * and other problems. + */ + set_bit(EXT4_FLAGS_EMERGENCY_RO, &EXT4_SB(sb)->s_ext4_flags); +} + +static void update_super_work(struct work_struct *work) +{ + struct ext4_sb_info *sbi = container_of(work, struct ext4_sb_info, + s_sb_upd_work); + journal_t *journal = sbi->s_journal; + handle_t *handle; + + /* + * If the journal is still running, we have to write out superblock + * through the journal to avoid collisions of other journalled sb + * updates. + * + * We use directly jbd2 functions here to avoid recursing back into + * ext4 error handling code during handling of previous errors. + */ + if (!ext4_emergency_state(sbi->s_sb) && + !sb_rdonly(sbi->s_sb) && journal) { + struct buffer_head *sbh = sbi->s_sbh; + bool call_notify_err = false; + + handle = jbd2_journal_start(journal, 1); + if (IS_ERR(handle)) + goto write_directly; + if (jbd2_journal_get_write_access(handle, sbh)) { + jbd2_journal_stop(handle); + goto write_directly; + } + + if (sbi->s_add_error_count > 0) + call_notify_err = true; + + ext4_update_super(sbi->s_sb); + if (buffer_write_io_error(sbh) || !buffer_uptodate(sbh)) { + ext4_msg(sbi->s_sb, KERN_ERR, "previous I/O error to " + "superblock detected"); + clear_buffer_write_io_error(sbh); + set_buffer_uptodate(sbh); + } + + if (jbd2_journal_dirty_metadata(handle, sbh)) { + jbd2_journal_stop(handle); + goto write_directly; + } + jbd2_journal_stop(handle); + + if (call_notify_err) + ext4_notify_error_sysfs(sbi); + + return; + } +write_directly: + /* + * Write through journal failed. Write sb directly to get error info + * out and hope for the best. + */ + ext4_commit_super(sbi->s_sb); + ext4_notify_error_sysfs(sbi); +} + +#define ext4_error_ratelimit(sb) \ + ___ratelimit(&(EXT4_SB(sb)->s_err_ratelimit_state), \ + "EXT4-fs error") + +void __ext4_error(struct super_block *sb, const char *function, + unsigned int line, bool force_ro, int error, __u64 block, + const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + if (unlikely(ext4_emergency_state(sb))) + return; + + trace_ext4_error(sb, function, line); + if (ext4_error_ratelimit(sb)) { + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + printk(KERN_CRIT + "EXT4-fs error (device %s): %s:%d: comm %s: %pV\n", + sb->s_id, function, line, current->comm, &vaf); + va_end(args); + } + fserror_report_metadata(sb, error ? -abs(error) : -EFSCORRUPTED, + GFP_ATOMIC); + + ext4_handle_error(sb, force_ro, error, 0, block, function, line); +} + +void __ext4_error_inode(struct inode *inode, const char *function, + unsigned int line, ext4_fsblk_t block, int error, + const char *fmt, ...) +{ + va_list args; + struct va_format vaf; + + if (unlikely(ext4_emergency_state(inode->i_sb))) + return; + + trace_ext4_error(inode->i_sb, function, line); + if (ext4_error_ratelimit(inode->i_sb)) { + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + if (block) + printk(KERN_CRIT "EXT4-fs error (device %s): %s:%d: " + "inode #%llu: block %llu: comm %s: %pV\n", + inode->i_sb->s_id, function, line, inode->i_ino, + block, current->comm, &vaf); + else + printk(KERN_CRIT "EXT4-fs error (device %s): %s:%d: " + "inode #%llu: comm %s: %pV\n", + inode->i_sb->s_id, function, line, inode->i_ino, + current->comm, &vaf); + va_end(args); + } + fserror_report_file_metadata(inode, + error ? -abs(error) : -EFSCORRUPTED, + GFP_ATOMIC); + + ext4_handle_error(inode->i_sb, false, error, inode->i_ino, block, + function, line); +} + +void __ext4_error_file(struct file *file, const char *function, + unsigned int line, ext4_fsblk_t block, + const char *fmt, ...) +{ + va_list args; + struct va_format vaf; + struct inode *inode = file_inode(file); + char pathname[80], *path; + + if (unlikely(ext4_emergency_state(inode->i_sb))) + return; + + trace_ext4_error(inode->i_sb, function, line); + if (ext4_error_ratelimit(inode->i_sb)) { + path = file_path(file, pathname, sizeof(pathname)); + if (IS_ERR(path)) + path = "(unknown)"; + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + if (block) + printk(KERN_CRIT + "EXT4-fs error (device %s): %s:%d: inode #%llu: " + "block %llu: comm %s: path %s: %pV\n", + inode->i_sb->s_id, function, line, inode->i_ino, + block, current->comm, path, &vaf); + else + printk(KERN_CRIT + "EXT4-fs error (device %s): %s:%d: inode #%llu: " + "comm %s: path %s: %pV\n", + inode->i_sb->s_id, function, line, inode->i_ino, + current->comm, path, &vaf); + va_end(args); + } + fserror_report_file_metadata(inode, -EFSCORRUPTED, GFP_ATOMIC); + + ext4_handle_error(inode->i_sb, false, EFSCORRUPTED, inode->i_ino, block, + function, line); +} + +const char *ext4_decode_error(struct super_block *sb, int errno, + char nbuf[16]) +{ + char *errstr = NULL; + + switch (errno) { + case -EFSCORRUPTED: + errstr = "Corrupt filesystem"; + break; + case -EFSBADCRC: + errstr = "Filesystem failed CRC"; + break; + case -EIO: + errstr = "IO failure"; + break; + case -ENOMEM: + errstr = "Out of memory"; + break; + case -EROFS: + if (!sb || (EXT4_SB(sb)->s_journal && + EXT4_SB(sb)->s_journal->j_flags & JBD2_ABORT)) + errstr = "Journal has aborted"; + else + errstr = "Readonly filesystem"; + break; + default: + /* If the caller passed in an extra buffer for unknown + * errors, textualise them now. Else we just return + * NULL. */ + if (nbuf) { + /* Check for truncated error codes... */ + if (snprintf(nbuf, 16, "error %d", -errno) >= 0) + errstr = nbuf; + } + break; + } + + return errstr; +} + +/* __ext4_std_error decodes expected errors from journaling functions + * automatically and invokes the appropriate error response. */ + +void __ext4_std_error(struct super_block *sb, const char *function, + unsigned int line, int errno) +{ + char nbuf[16]; + const char *errstr; + + if (unlikely(ext4_emergency_state(sb))) + return; + + /* Special case: if the error is EROFS, and we're not already + * inside a transaction, then there's really no point in logging + * an error. */ + if (errno == -EROFS && journal_current_handle() == NULL && sb_rdonly(sb)) + return; + + if (ext4_error_ratelimit(sb)) { + errstr = ext4_decode_error(sb, errno, nbuf); + printk(KERN_CRIT "EXT4-fs error (device %s) in %s:%d: %s\n", + sb->s_id, function, line, errstr); + } + fserror_report_metadata(sb, errno ? -abs(errno) : -EFSCORRUPTED, + GFP_ATOMIC); + + ext4_handle_error(sb, false, -errno, 0, 0, function, line); +} + +void __ext4_msg(struct super_block *sb, + const char *prefix, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + if (sb) { + atomic_inc(&EXT4_SB(sb)->s_msg_count); + if (!___ratelimit(&(EXT4_SB(sb)->s_msg_ratelimit_state), + "EXT4-fs")) + return; + } + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + if (sb) + printk("%sEXT4-fs (%s): %pV\n", prefix, sb->s_id, &vaf); + else + printk("%sEXT4-fs: %pV\n", prefix, &vaf); + va_end(args); +} + +static int ext4_warning_ratelimit(struct super_block *sb) +{ + atomic_inc(&EXT4_SB(sb)->s_warning_count); + return ___ratelimit(&(EXT4_SB(sb)->s_warning_ratelimit_state), + "EXT4-fs warning"); +} + +void __ext4_warning(struct super_block *sb, const char *function, + unsigned int line, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + if (!ext4_warning_ratelimit(sb)) + return; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + printk(KERN_WARNING "EXT4-fs warning (device %s): %s:%d: %pV\n", + sb->s_id, function, line, &vaf); + va_end(args); +} + +void __ext4_warning_inode(const struct inode *inode, const char *function, + unsigned int line, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + if (!ext4_warning_ratelimit(inode->i_sb)) + return; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + printk(KERN_WARNING "EXT4-fs warning (device %s): %s:%d: " + "inode #%llu: comm %s: %pV\n", inode->i_sb->s_id, + function, line, inode->i_ino, current->comm, &vaf); + va_end(args); +} + +void __ext4_grp_locked_error(const char *function, unsigned int line, + struct super_block *sb, ext4_group_t grp, + u64 ino, ext4_fsblk_t block, + const char *fmt, ...) +__releases(bitlock) +__acquires(bitlock) +{ + struct va_format vaf; + va_list args; + + if (unlikely(ext4_emergency_state(sb))) + return; + + trace_ext4_error(sb, function, line); + if (ext4_error_ratelimit(sb)) { + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + printk(KERN_CRIT "EXT4-fs error (device %s): %s:%d: group %u, ", + sb->s_id, function, line, grp); + if (ino) + printk(KERN_CONT "inode %llu: ", ino); + if (block) + printk(KERN_CONT "block %llu:", + (unsigned long long) block); + printk(KERN_CONT "%pV\n", &vaf); + va_end(args); + } + + if (test_opt(sb, ERRORS_CONT)) { + if (test_opt(sb, WARN_ON_ERROR)) + WARN_ON_ONCE(1); + EXT4_SB(sb)->s_mount_state |= EXT4_ERROR_FS; + if (!bdev_read_only(sb->s_bdev)) { + save_error_info(sb, EFSCORRUPTED, ino, block, function, + line); + schedule_work(&EXT4_SB(sb)->s_sb_upd_work); + } + return; + } + ext4_unlock_group(sb, grp); + ext4_handle_error(sb, false, EFSCORRUPTED, ino, block, function, line); + /* + * We only get here in the ERRORS_RO case; relocking the group + * may be dangerous, but nothing bad will happen since the + * filesystem will have already been marked read/only and the + * journal has been aborted. We return 1 as a hint to callers + * who might what to use the return value from + * ext4_grp_locked_error() to distinguish between the + * ERRORS_CONT and ERRORS_RO case, and perhaps return more + * aggressively from the ext4 function in question, with a + * more appropriate error code. + */ + ext4_lock_group(sb, grp); + return; +} + +void ext4_mark_group_bitmap_corrupted(struct super_block *sb, + ext4_group_t group, + unsigned int flags) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_group_info *grp = ext4_get_group_info(sb, group); + struct ext4_group_desc *gdp = ext4_get_group_desc(sb, group, NULL); + int ret; + + if (!grp || !gdp) + return; + if (flags & EXT4_GROUP_INFO_BBITMAP_CORRUPT) { + ret = ext4_test_and_set_bit(EXT4_GROUP_INFO_BBITMAP_CORRUPT_BIT, + &grp->bb_state); + if (!ret) + percpu_counter_sub(&sbi->s_freeclusters_counter, + grp->bb_free); + } + + if (flags & EXT4_GROUP_INFO_IBITMAP_CORRUPT) { + ret = ext4_test_and_set_bit(EXT4_GROUP_INFO_IBITMAP_CORRUPT_BIT, + &grp->bb_state); + if (!ret && gdp) { + int count; + + count = ext4_free_inodes_count(sb, gdp); + percpu_counter_sub(&sbi->s_freeinodes_counter, + count); + } + } +} + +void ext4_update_dynamic_rev(struct super_block *sb) +{ + struct ext4_super_block *es = EXT4_SB(sb)->s_es; + + if (le32_to_cpu(es->s_rev_level) > EXT4_GOOD_OLD_REV) + return; + + ext4_warning(sb, + "updating to rev %d because of new feature flag, " + "running e2fsck is recommended", + EXT4_DYNAMIC_REV); + + es->s_first_ino = cpu_to_le32(EXT4_GOOD_OLD_FIRST_INO); + es->s_inode_size = cpu_to_le16(EXT4_GOOD_OLD_INODE_SIZE); + es->s_rev_level = cpu_to_le32(EXT4_DYNAMIC_REV); + /* leave es->s_feature_*compat flags alone */ + /* es->s_uuid will be set by e2fsck if empty */ + + /* + * The rest of the superblock fields should be zero, and if not it + * means they are likely already in use, so leave them alone. We + * can leave it up to e2fsck to clean up any inconsistencies there. + */ +} + +static inline struct inode *orphan_list_entry(struct list_head *l) +{ + return &list_entry(l, struct ext4_inode_info, i_orphan)->vfs_inode; +} + +static void dump_orphan_list(struct super_block *sb, struct ext4_sb_info *sbi) +{ + struct list_head *l; + + ext4_msg(sb, KERN_ERR, "sb orphan head is %d", + le32_to_cpu(sbi->s_es->s_last_orphan)); + + printk(KERN_ERR "sb_info orphan list:\n"); + list_for_each(l, &sbi->s_orphan) { + struct inode *inode = orphan_list_entry(l); + printk(KERN_ERR " " + "inode %s:%llu at %p: mode %o, nlink %d, next %d\n", + inode->i_sb->s_id, inode->i_ino, inode, + inode->i_mode, inode->i_nlink, + NEXT_ORPHAN(inode)); + } +} + +#ifdef CONFIG_QUOTA +static int ext4_quota_off(struct super_block *sb, int type); + +static inline void ext4_quotas_off(struct super_block *sb, int type) +{ + BUG_ON(type > EXT4_MAXQUOTAS); + + /* Use our quota_off function to clear inode flags etc. */ + for (type--; type >= 0; type--) + ext4_quota_off(sb, type); +} + +/* + * This is a helper function which is used in the mount/remount + * codepaths (which holds s_umount) to fetch the quota file name. + */ +static inline char *get_qf_name(struct super_block *sb, + struct ext4_sb_info *sbi, + int type) +{ + return rcu_dereference_protected(sbi->s_qf_names[type], + lockdep_is_held(&sb->s_umount)); +} +#else +static inline void ext4_quotas_off(struct super_block *sb, int type) +{ +} +#endif + +static int ext4_percpu_param_init(struct ext4_sb_info *sbi) +{ + ext4_fsblk_t block; + int err; + + block = ext4_count_free_clusters(sbi->s_sb); + ext4_free_blocks_count_set(sbi->s_es, EXT4_C2B(sbi, block)); + err = percpu_counter_init(&sbi->s_freeclusters_counter, block, + GFP_KERNEL); + if (!err) { + unsigned long freei = ext4_count_free_inodes(sbi->s_sb); + sbi->s_es->s_free_inodes_count = cpu_to_le32(freei); + err = percpu_counter_init(&sbi->s_freeinodes_counter, freei, + GFP_KERNEL); + } + if (!err) + err = percpu_counter_init(&sbi->s_dirs_counter, + ext4_count_dirs(sbi->s_sb), GFP_KERNEL); + if (!err) + err = percpu_counter_init(&sbi->s_dirtyclusters_counter, 0, + GFP_KERNEL); + if (!err) + err = percpu_counter_init(&sbi->s_sra_exceeded_retry_limit, 0, + GFP_KERNEL); + if (!err) + err = percpu_init_rwsem(&sbi->s_writepages_rwsem); + + if (err) + ext4_msg(sbi->s_sb, KERN_ERR, "insufficient memory"); + + return err; +} + +static void ext4_percpu_param_destroy(struct ext4_sb_info *sbi) +{ + percpu_counter_destroy(&sbi->s_freeclusters_counter); + percpu_counter_destroy(&sbi->s_freeinodes_counter); + percpu_counter_destroy(&sbi->s_dirs_counter); + percpu_counter_destroy(&sbi->s_dirtyclusters_counter); + percpu_counter_destroy(&sbi->s_sra_exceeded_retry_limit); + percpu_free_rwsem(&sbi->s_writepages_rwsem); +} + +static void ext4_group_desc_free(struct ext4_sb_info *sbi) +{ + struct buffer_head **group_desc; + int i; + + group_desc = rcu_access_pointer(sbi->s_group_desc); + for (i = 0; i < sbi->s_gdb_count; i++) + brelse(group_desc[i]); + kvfree(group_desc); +} + +static void ext4_flex_groups_free(struct ext4_sb_info *sbi) +{ + struct flex_groups **flex_groups; + int i; + + flex_groups = rcu_access_pointer(sbi->s_flex_groups); + if (flex_groups) { + for (i = 0; i < sbi->s_flex_groups_allocated; i++) + kvfree(flex_groups[i]); + kvfree(flex_groups); + } +} + +static void ext4_put_super(struct super_block *sb) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_super_block *es = sbi->s_es; + int aborted = 0; + int err; + + /* + * Unregister sysfs before destroying jbd2 journal. + * Since we could still access attr_journal_task attribute via sysfs + * path which could have sbi->s_journal->j_task as NULL + * Unregister sysfs before flush sbi->s_sb_upd_work. + * Since user may read /proc/fs/ext4/xx/mb_groups during umount, If + * read metadata verify failed then will queue error work. + * update_super_work will call start_this_handle may trigger + * BUG_ON. + */ + ext4_unregister_sysfs(sb); + + if (___ratelimit(&ext4_mount_msg_ratelimit, "EXT4-fs unmount")) + ext4_msg(sb, KERN_INFO, "unmounting filesystem %pU.", + &sb->s_uuid); + + ext4_unregister_li_request(sb); + /* Drain deferred EA inode iputs while quota is still active. */ + flush_delayed_work(&sbi->s_ea_inode_work); + ext4_quotas_off(sb, EXT4_MAXQUOTAS); + + destroy_workqueue(sbi->rsv_conversion_wq); + ext4_release_orphan_info(sb); + + if (sbi->s_journal) { + aborted = is_journal_aborted(sbi->s_journal); + err = ext4_journal_destroy(sbi, sbi->s_journal); + if ((err < 0) && !aborted) { + ext4_abort(sb, -err, "Couldn't clean up the journal"); + } + } else + flush_work(&sbi->s_sb_upd_work); + + ext4_es_unregister_shrinker(sbi); + timer_shutdown_sync(&sbi->s_err_report); + ext4_release_system_zone(sb); + ext4_mb_release(sb); + ext4_ext_release(sb); + + if (!ext4_emergency_state(sb) && !sb_rdonly(sb)) { + if (!aborted) { + ext4_clear_feature_journal_needs_recovery(sb); + ext4_clear_feature_orphan_present(sb); + es->s_state = cpu_to_le16(sbi->s_mount_state); + } + ext4_commit_super(sb); + } + + ext4_group_desc_free(sbi); + ext4_flex_groups_free(sbi); + + WARN_ON_ONCE(!(sbi->s_mount_state & EXT4_ERROR_FS) && + percpu_counter_sum(&sbi->s_dirtyclusters_counter)); + ext4_percpu_param_destroy(sbi); +#ifdef CONFIG_QUOTA + for (int i = 0; i < EXT4_MAXQUOTAS; i++) + kfree(get_qf_name(sb, sbi, i)); +#endif + + /* Debugging code just in case the in-memory inode orphan list + * isn't empty. The on-disk one can be non-empty if we've + * detected an error and taken the fs readonly, but the + * in-memory list had better be clean by this point. */ + if (!list_empty(&sbi->s_orphan)) + dump_orphan_list(sb, sbi); + ASSERT(list_empty(&sbi->s_orphan)); + + sync_blockdev(sb->s_bdev); + invalidate_bdev(sb->s_bdev); + if (sbi->s_journal_bdev_file) { + /* + * Invalidate the journal device's buffers. We don't want them + * floating about in memory - the physical journal device may + * hotswapped, and it breaks the `ro-after' testing code. + */ + sync_blockdev(file_bdev(sbi->s_journal_bdev_file)); + invalidate_bdev(file_bdev(sbi->s_journal_bdev_file)); + } + + ext4_xattr_destroy_cache(sbi->s_ea_inode_cache); + sbi->s_ea_inode_cache = NULL; + + ext4_xattr_destroy_cache(sbi->s_ea_block_cache); + sbi->s_ea_block_cache = NULL; + + ext4_stop_mmpd(sbi); + + brelse(sbi->s_sbh); + sb->s_fs_info = NULL; + /* + * Now that we are completely done shutting down the + * superblock, we need to actually destroy the kobject. + */ + kobject_put(&sbi->s_kobj); + wait_for_completion(&sbi->s_kobj_unregister); + kfree(sbi->s_blockgroup_lock); + fs_put_dax(sbi->s_daxdev, NULL); + fscrypt_free_dummy_policy(&sbi->s_dummy_enc_policy); +#if IS_ENABLED(CONFIG_UNICODE) + utf8_unload(sb->s_encoding); +#endif + kfree(sbi); +} + +static struct kmem_cache *ext4_inode_cachep; + +/* + * Called inside transaction, so use GFP_NOFS + */ +static struct inode *ext4_alloc_inode(struct super_block *sb) +{ + struct ext4_inode_info *ei; + + ei = alloc_inode_sb(sb, ext4_inode_cachep, GFP_NOFS); + if (!ei) + return NULL; + + inode_set_iversion(&ei->vfs_inode, 1); + ei->i_flags = 0; + ext4_clear_state_flags(ei); /* Only relevant on 32-bit archs */ + spin_lock_init(&ei->i_raw_lock); + ei->i_prealloc_node = RB_ROOT; + atomic_set(&ei->i_prealloc_active, 0); + rwlock_init(&ei->i_prealloc_lock); + ext4_es_init_tree(&ei->i_es_tree); + rwlock_init(&ei->i_es_lock); + INIT_LIST_HEAD(&ei->i_es_list); + ei->i_es_all_nr = 0; + ei->i_es_shk_nr = 0; + ei->i_es_shrink_lblk = 0; + ei->i_es_seq = 0; + ei->i_reserved_data_blocks = 0; + spin_lock_init(&(ei->i_block_reservation_lock)); + ext4_init_pending_tree(&ei->i_pending_tree); +#ifdef CONFIG_QUOTA + ei->i_reserved_quota = 0; + memset(&ei->i_dquot, 0, sizeof(ei->i_dquot)); +#endif + ei->jinode = NULL; + /* + * Reinitialize xattr_sem every allocation because EA inodes + * share this space with i_ea_iput_node (via union) which may + * have overwritten the semaphore when the slab object was + * previously used as an EA inode. + */ + init_rwsem(&ei->xattr_sem); + INIT_LIST_HEAD(&ei->i_rsv_conversion_list); + spin_lock_init(&ei->i_completed_io_lock); + ei->i_sync_tid = 0; + ei->i_datasync_tid = 0; + INIT_WORK(&ei->i_rsv_conversion_work, ext4_end_io_rsv_work); + ext4_fc_init_inode(&ei->vfs_inode); + spin_lock_init(&ei->i_fc_lock); + ei->i_metadata_bhs = NULL; +#ifdef CONFIG_LOCKDEP + lockdep_set_subclass(&ei->i_data_sem, I_DATA_SEM_NORMAL); +#endif + return &ei->vfs_inode; +} + +static int ext4_drop_inode(struct inode *inode) +{ + int drop = inode_generic_drop(inode); + + if (!drop) + drop = fscrypt_drop_inode(inode); + + trace_ext4_drop_inode(inode, drop); + return drop; +} + +static void ext4_free_in_core_inode(struct inode *inode) +{ + fscrypt_free_inode(inode); + kfree(ext4_i_metadata_bhs(inode)); + if (!list_empty(&(EXT4_I(inode)->i_fc_list))) { + pr_warn("%s: inode %llu still in fc list", + __func__, inode->i_ino); + } + kmem_cache_free(ext4_inode_cachep, EXT4_I(inode)); +} + +static void ext4_destroy_inode(struct inode *inode) +{ + if (ext4_inode_orphan_tracked(inode)) { + ext4_msg(inode->i_sb, KERN_ERR, + "Inode %llu (%p): inode tracked as orphan!", + inode->i_ino, EXT4_I(inode)); + print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS, 16, 4, + EXT4_I(inode), sizeof(struct ext4_inode_info), + true); + dump_stack(); + } + + if (!(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_ERROR_FS) && + WARN_ON_ONCE(EXT4_I(inode)->i_reserved_data_blocks)) + ext4_msg(inode->i_sb, KERN_ERR, + "Inode %llu (%p): i_reserved_data_blocks (%u) not cleared!", + inode->i_ino, EXT4_I(inode), + EXT4_I(inode)->i_reserved_data_blocks); +} + +static void ext4_shutdown(struct super_block *sb) +{ + ext4_force_shutdown(sb, EXT4_GOING_FLAGS_NOLOGFLUSH); +} + +static void init_once(void *foo) +{ + struct ext4_inode_info *ei = foo; + + INIT_LIST_HEAD(&ei->i_orphan); + init_rwsem(&ei->i_data_sem); + inode_init_once(&ei->vfs_inode); + ext4_fc_init_inode(&ei->vfs_inode); +#ifdef CONFIG_FS_ENCRYPTION + ei->i_crypt_info = NULL; +#endif +} + +static int __init init_inodecache(void) +{ + struct kmem_cache_args args = { + .useroffset = offsetof(struct ext4_inode_info, i_data), + .usersize = sizeof_field(struct ext4_inode_info, i_data), + .use_freeptr_offset = true, + .freeptr_offset = offsetof(struct ext4_inode_info, i_flags), + .ctor = init_once, + }; + + ext4_inode_cachep = kmem_cache_create("ext4_inode_cache", + sizeof(struct ext4_inode_info), + &args, + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT); + + if (ext4_inode_cachep == NULL) + return -ENOMEM; + return 0; +} + +static void destroy_inodecache(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(ext4_inode_cachep); +} + +void ext4_clear_inode(struct inode *inode) +{ + struct mapping_metadata_bhs *mmb = ext4_i_metadata_bhs(inode); + + ext4_fc_del(inode); + if (mmb) + mmb_invalidate(mmb); + clear_inode(inode); + ext4_discard_preallocations(inode); + /* + * We must remove the inode from the hash before ext4_free_inode() + * clears the bit in inode bitmap as otherwise another process reusing + * the inode will block in insert_inode_hash() waiting for inode + * eviction to complete while holding transaction handle open, but + * ext4_evict_inode() still running for that inode could block waiting + * for transaction commit if the inode is marked as IS_SYNC => deadlock. + * + * Removing the inode from the hash here is safe. There are two cases + * to consider: + * 1) The inode still has references to it (i_nlink > 0). In that case + * we are keeping the inode and once we remove the inode from the hash, + * iget() can create the new inode structure for the same inode number + * and we are fine with that as all IO on behalf of the inode is + * finished. + * 2) We are deleting the inode (i_nlink == 0). In that case inode + * number cannot be reused until ext4_free_inode() clears the bit in + * the inode bitmap, at which point all IO is done and reuse is fine + * again. + */ + remove_inode_hash(inode); + ext4_es_remove_extent(inode, 0, EXT_MAX_BLOCKS); + dquot_drop(inode); + if (EXT4_I(inode)->jinode) { + jbd2_journal_release_jbd_inode(EXT4_JOURNAL(inode), + EXT4_I(inode)->jinode); + jbd2_free_inode(EXT4_I(inode)->jinode); + EXT4_I(inode)->jinode = NULL; + } + fscrypt_put_encryption_info(inode); +} + +static struct inode *ext4_nfs_get_inode(struct super_block *sb, + u64 ino, u32 generation) +{ + struct inode *inode; + + /* + * Currently we don't know the generation for parent directory, so + * a generation of 0 means "accept any" + */ + inode = ext4_iget(sb, ino, EXT4_IGET_HANDLE); + if (IS_ERR(inode)) + return ERR_CAST(inode); + if (generation && inode->i_generation != generation) { + iput(inode); + return ERR_PTR(-ESTALE); + } + + return inode; +} + +static struct dentry *ext4_fh_to_dentry(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + return generic_fh_to_dentry(sb, fid, fh_len, fh_type, + ext4_nfs_get_inode); +} + +static struct dentry *ext4_fh_to_parent(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + return generic_fh_to_parent(sb, fid, fh_len, fh_type, + ext4_nfs_get_inode); +} + +static int ext4_nfs_commit_metadata(struct inode *inode) +{ + struct writeback_control wbc = { + .sync_mode = WB_SYNC_ALL + }; + int ret; + + trace_ext4_nfs_commit_metadata(inode); + ret = ext4_write_inode(inode, &wbc); + if (!ret && inode_state_read_once(inode) & I_METADATA_WRITEBACK) + ret = ext4_sync_inode_metadata(inode, &wbc); + return ret; +} + +#ifdef CONFIG_QUOTA +static const char * const quotatypes[] = INITQFNAMES; +#define QTYPE2NAME(t) (quotatypes[t]) + +static int ext4_write_dquot(struct dquot *dquot); +static int ext4_acquire_dquot(struct dquot *dquot); +static int ext4_release_dquot(struct dquot *dquot); +static int ext4_mark_dquot_dirty(struct dquot *dquot); +static int ext4_write_info(struct super_block *sb, int type); +static int ext4_quota_on(struct super_block *sb, int type, int format_id, + const struct path *path); +static ssize_t ext4_quota_read(struct super_block *sb, int type, char *data, + size_t len, loff_t off); +static ssize_t ext4_quota_write(struct super_block *sb, int type, + const char *data, size_t len, loff_t off); +static int ext4_quota_enable(struct super_block *sb, int type, int format_id, + unsigned int flags); + +static struct dquot __rcu **ext4_get_dquots(struct inode *inode) +{ + return EXT4_I(inode)->i_dquot; +} + +static const struct dquot_operations ext4_quota_operations = { + .get_reserved_space = ext4_get_reserved_space, + .write_dquot = ext4_write_dquot, + .acquire_dquot = ext4_acquire_dquot, + .release_dquot = ext4_release_dquot, + .mark_dirty = ext4_mark_dquot_dirty, + .write_info = ext4_write_info, + .alloc_dquot = dquot_alloc, + .destroy_dquot = dquot_destroy, + .get_projid = ext4_get_projid, + .get_inode_usage = ext4_get_inode_usage, + .get_next_id = dquot_get_next_id, +}; + +static const struct quotactl_ops ext4_qctl_operations = { + .quota_on = ext4_quota_on, + .quota_off = ext4_quota_off, + .quota_sync = dquot_quota_sync, + .get_state = dquot_get_state, + .set_info = dquot_set_dqinfo, + .get_dqblk = dquot_get_dqblk, + .set_dqblk = dquot_set_dqblk, + .get_nextdqblk = dquot_get_next_dqblk, +}; +#endif + +static const struct super_operations ext4_sops = { + .alloc_inode = ext4_alloc_inode, + .free_inode = ext4_free_in_core_inode, + .destroy_inode = ext4_destroy_inode, + .write_inode = ext4_write_inode, + .sync_inode_metadata = ext4_sync_inode_metadata, + .dirty_inode = ext4_dirty_inode, + .drop_inode = ext4_drop_inode, + .evict_inode = ext4_evict_inode, + .put_super = ext4_put_super, + .sync_fs = ext4_sync_fs, + .freeze_fs = ext4_freeze, + .unfreeze_fs = ext4_unfreeze, + .statfs = ext4_statfs, + .show_options = ext4_show_options, + .shutdown = ext4_shutdown, +#ifdef CONFIG_QUOTA + .quota_read = ext4_quota_read, + .quota_write = ext4_quota_write, + .get_dquots = ext4_get_dquots, +#endif +}; + +static const struct export_operations ext4_export_ops = { + .encode_fh = generic_encode_ino32_fh, + .fh_to_dentry = ext4_fh_to_dentry, + .fh_to_parent = ext4_fh_to_parent, + .get_parent = ext4_get_parent, + .commit_metadata = ext4_nfs_commit_metadata, +}; + +enum { + Opt_bsd_df, Opt_minix_df, Opt_grpid, Opt_nogrpid, + Opt_resgid, Opt_resuid, Opt_sb, + Opt_nouid32, Opt_debug, Opt_removed, + Opt_user_xattr, Opt_acl, + Opt_auto_da_alloc, Opt_noauto_da_alloc, Opt_noload, + Opt_commit, Opt_min_batch_time, Opt_max_batch_time, Opt_journal_dev, + Opt_journal_path, Opt_journal_checksum, Opt_journal_async_commit, + Opt_abort, Opt_data_journal, Opt_data_ordered, Opt_data_writeback, + Opt_data_err_abort, Opt_data_err_ignore, Opt_test_dummy_encryption, + Opt_inlinecrypt, + Opt_usrjquota, Opt_grpjquota, Opt_quota, + Opt_noquota, Opt_barrier, Opt_nobarrier, Opt_err, + Opt_usrquota, Opt_grpquota, Opt_prjquota, + Opt_dax, Opt_dax_always, Opt_dax_inode, Opt_dax_never, + Opt_stripe, Opt_delalloc, Opt_nodelalloc, Opt_warn_on_error, + Opt_nowarn_on_error, Opt_mblk_io_submit, Opt_debug_want_extra_isize, + Opt_nomblk_io_submit, Opt_block_validity, Opt_noblock_validity, + Opt_inode_readahead_blks, Opt_journal_ioprio, + Opt_dioread_nolock, Opt_dioread_lock, + Opt_discard, Opt_nodiscard, Opt_init_itable, Opt_noinit_itable, + Opt_max_dir_size_kb, Opt_nojournal_checksum, Opt_nombcache, + Opt_no_prefetch_block_bitmaps, Opt_mb_optimize_scan, + Opt_errors, Opt_data, Opt_data_err, Opt_jqfmt, Opt_dax_type, +#ifdef CONFIG_EXT4_DEBUG + Opt_fc_debug_max_replay, Opt_fc_debug_force +#endif +}; + +static const struct constant_table ext4_param_errors[] = { + {"continue", EXT4_MOUNT_ERRORS_CONT}, + {"panic", EXT4_MOUNT_ERRORS_PANIC}, + {"remount-ro", EXT4_MOUNT_ERRORS_RO}, + {} +}; + +static const struct constant_table ext4_param_data[] = { + {"journal", EXT4_MOUNT_JOURNAL_DATA}, + {"ordered", EXT4_MOUNT_ORDERED_DATA}, + {"writeback", EXT4_MOUNT_WRITEBACK_DATA}, + {} +}; + +static const struct constant_table ext4_param_data_err[] = { + {"abort", Opt_data_err_abort}, + {"ignore", Opt_data_err_ignore}, + {} +}; + +static const struct constant_table ext4_param_jqfmt[] = { + {"vfsold", QFMT_VFS_OLD}, + {"vfsv0", QFMT_VFS_V0}, + {"vfsv1", QFMT_VFS_V1}, + {} +}; + +static const struct constant_table ext4_param_dax[] = { + {"always", Opt_dax_always}, + {"inode", Opt_dax_inode}, + {"never", Opt_dax_never}, + {} +}; + +/* + * Mount option specification + * We don't use fsparam_flag_no because of the way we set the + * options and the way we show them in _ext4_show_options(). To + * keep the changes to a minimum, let's keep the negative options + * separate for now. + */ +static const struct fs_parameter_spec ext4_param_specs[] = { + fsparam_flag ("bsddf", Opt_bsd_df), + fsparam_flag ("minixdf", Opt_minix_df), + fsparam_flag ("grpid", Opt_grpid), + fsparam_flag ("bsdgroups", Opt_grpid), + fsparam_flag ("nogrpid", Opt_nogrpid), + fsparam_flag ("sysvgroups", Opt_nogrpid), + fsparam_gid ("resgid", Opt_resgid), + fsparam_uid ("resuid", Opt_resuid), + fsparam_u32 ("sb", Opt_sb), + fsparam_enum ("errors", Opt_errors, ext4_param_errors), + fsparam_flag ("nouid32", Opt_nouid32), + fsparam_flag ("debug", Opt_debug), + fsparam_flag ("oldalloc", Opt_removed), + fsparam_flag ("orlov", Opt_removed), + fsparam_flag ("user_xattr", Opt_user_xattr), + fsparam_flag ("acl", Opt_acl), + fsparam_flag ("norecovery", Opt_noload), + fsparam_flag ("noload", Opt_noload), + fsparam_flag ("bh", Opt_removed), + fsparam_flag ("nobh", Opt_removed), + fsparam_u32 ("commit", Opt_commit), + fsparam_u32 ("min_batch_time", Opt_min_batch_time), + fsparam_u32 ("max_batch_time", Opt_max_batch_time), + fsparam_u32 ("journal_dev", Opt_journal_dev), + fsparam_bdev ("journal_path", Opt_journal_path), + fsparam_flag ("journal_checksum", Opt_journal_checksum), + fsparam_flag ("nojournal_checksum", Opt_nojournal_checksum), + fsparam_flag ("journal_async_commit",Opt_journal_async_commit), + fsparam_flag ("abort", Opt_abort), + fsparam_enum ("data", Opt_data, ext4_param_data), + fsparam_enum ("data_err", Opt_data_err, + ext4_param_data_err), + fsparam_string_empty + ("usrjquota", Opt_usrjquota), + fsparam_string_empty + ("grpjquota", Opt_grpjquota), + fsparam_enum ("jqfmt", Opt_jqfmt, ext4_param_jqfmt), + fsparam_flag ("grpquota", Opt_grpquota), + fsparam_flag ("quota", Opt_quota), + fsparam_flag ("noquota", Opt_noquota), + fsparam_flag ("usrquota", Opt_usrquota), + fsparam_flag ("prjquota", Opt_prjquota), + fsparam_flag ("barrier", Opt_barrier), + fsparam_u32 ("barrier", Opt_barrier), + fsparam_flag ("nobarrier", Opt_nobarrier), + fsparam_flag ("i_version", Opt_removed), + fsparam_flag ("dax", Opt_dax), + fsparam_enum ("dax", Opt_dax_type, ext4_param_dax), + fsparam_u32 ("stripe", Opt_stripe), + fsparam_flag ("delalloc", Opt_delalloc), + fsparam_flag ("nodelalloc", Opt_nodelalloc), + fsparam_flag ("warn_on_error", Opt_warn_on_error), + fsparam_flag ("nowarn_on_error", Opt_nowarn_on_error), + fsparam_u32 ("debug_want_extra_isize", + Opt_debug_want_extra_isize), + fsparam_flag ("mblk_io_submit", Opt_removed), + fsparam_flag ("nomblk_io_submit", Opt_removed), + fsparam_flag ("block_validity", Opt_block_validity), + fsparam_flag ("noblock_validity", Opt_noblock_validity), + fsparam_u32 ("inode_readahead_blks", + Opt_inode_readahead_blks), + fsparam_u32 ("journal_ioprio", Opt_journal_ioprio), + fsparam_u32 ("auto_da_alloc", Opt_auto_da_alloc), + fsparam_flag ("auto_da_alloc", Opt_auto_da_alloc), + fsparam_flag ("noauto_da_alloc", Opt_noauto_da_alloc), + fsparam_flag ("dioread_nolock", Opt_dioread_nolock), + fsparam_flag ("nodioread_nolock", Opt_dioread_lock), + fsparam_flag ("dioread_lock", Opt_dioread_lock), + fsparam_flag ("discard", Opt_discard), + fsparam_flag ("nodiscard", Opt_nodiscard), + fsparam_u32 ("init_itable", Opt_init_itable), + fsparam_flag ("init_itable", Opt_init_itable), + fsparam_flag ("noinit_itable", Opt_noinit_itable), +#ifdef CONFIG_EXT4_DEBUG + fsparam_flag ("fc_debug_force", Opt_fc_debug_force), + fsparam_u32 ("fc_debug_max_replay", Opt_fc_debug_max_replay), +#endif + fsparam_u32 ("max_dir_size_kb", Opt_max_dir_size_kb), + fsparam_flag ("test_dummy_encryption", + Opt_test_dummy_encryption), + fsparam_string ("test_dummy_encryption", + Opt_test_dummy_encryption), + fsparam_flag ("inlinecrypt", Opt_inlinecrypt), + fsparam_flag ("nombcache", Opt_nombcache), + fsparam_flag ("no_mbcache", Opt_nombcache), /* for backward compatibility */ + fsparam_flag ("prefetch_block_bitmaps", + Opt_removed), + fsparam_flag ("no_prefetch_block_bitmaps", + Opt_no_prefetch_block_bitmaps), + fsparam_s32 ("mb_optimize_scan", Opt_mb_optimize_scan), + fsparam_string ("check", Opt_removed), /* mount option from ext2/3 */ + fsparam_flag ("nocheck", Opt_removed), /* mount option from ext2/3 */ + fsparam_flag ("reservation", Opt_removed), /* mount option from ext2/3 */ + fsparam_flag ("noreservation", Opt_removed), /* mount option from ext2/3 */ + fsparam_u32 ("journal", Opt_removed), /* mount option from ext2/3 */ + {} +}; + + +#define MOPT_SET 0x0001 +#define MOPT_CLEAR 0x0002 +#define MOPT_NOSUPPORT 0x0004 +#define MOPT_EXPLICIT 0x0008 +#ifdef CONFIG_QUOTA +#define MOPT_Q 0 +#define MOPT_QFMT 0x0010 +#else +#define MOPT_Q MOPT_NOSUPPORT +#define MOPT_QFMT MOPT_NOSUPPORT +#endif +#define MOPT_NO_EXT2 0x0020 +#define MOPT_NO_EXT3 0x0040 +#define MOPT_EXT4_ONLY (MOPT_NO_EXT2 | MOPT_NO_EXT3) +#define MOPT_SKIP 0x0080 +#define MOPT_2 0x0100 + +static const struct mount_opts { + int token; + int mount_opt; + int flags; +} ext4_mount_opts[] = { + {Opt_minix_df, EXT4_MOUNT_MINIX_DF, MOPT_SET}, + {Opt_bsd_df, EXT4_MOUNT_MINIX_DF, MOPT_CLEAR}, + {Opt_grpid, EXT4_MOUNT_GRPID, MOPT_SET}, + {Opt_nogrpid, EXT4_MOUNT_GRPID, MOPT_CLEAR}, + {Opt_block_validity, EXT4_MOUNT_BLOCK_VALIDITY, MOPT_SET}, + {Opt_noblock_validity, EXT4_MOUNT_BLOCK_VALIDITY, MOPT_CLEAR}, + {Opt_dioread_nolock, EXT4_MOUNT_DIOREAD_NOLOCK, + MOPT_EXT4_ONLY | MOPT_SET}, + {Opt_dioread_lock, EXT4_MOUNT_DIOREAD_NOLOCK, + MOPT_EXT4_ONLY | MOPT_CLEAR}, + {Opt_discard, EXT4_MOUNT_DISCARD, MOPT_SET}, + {Opt_nodiscard, EXT4_MOUNT_DISCARD, MOPT_CLEAR}, + {Opt_delalloc, EXT4_MOUNT_DELALLOC, + MOPT_EXT4_ONLY | MOPT_SET | MOPT_EXPLICIT}, + {Opt_nodelalloc, EXT4_MOUNT_DELALLOC, + MOPT_EXT4_ONLY | MOPT_CLEAR}, + {Opt_warn_on_error, EXT4_MOUNT_WARN_ON_ERROR, MOPT_SET}, + {Opt_nowarn_on_error, EXT4_MOUNT_WARN_ON_ERROR, MOPT_CLEAR}, + {Opt_commit, 0, MOPT_NO_EXT2}, + {Opt_nojournal_checksum, EXT4_MOUNT_JOURNAL_CHECKSUM, + MOPT_EXT4_ONLY | MOPT_CLEAR}, + {Opt_journal_checksum, EXT4_MOUNT_JOURNAL_CHECKSUM, + MOPT_EXT4_ONLY | MOPT_SET | MOPT_EXPLICIT}, + {Opt_journal_async_commit, (EXT4_MOUNT_JOURNAL_ASYNC_COMMIT | + EXT4_MOUNT_JOURNAL_CHECKSUM), + MOPT_EXT4_ONLY | MOPT_SET | MOPT_EXPLICIT}, + {Opt_noload, EXT4_MOUNT_NOLOAD, MOPT_NO_EXT2 | MOPT_SET}, + {Opt_data_err, EXT4_MOUNT_DATA_ERR_ABORT, MOPT_NO_EXT2}, + {Opt_barrier, EXT4_MOUNT_BARRIER, MOPT_SET}, + {Opt_nobarrier, EXT4_MOUNT_BARRIER, MOPT_CLEAR}, + {Opt_noauto_da_alloc, EXT4_MOUNT_NO_AUTO_DA_ALLOC, MOPT_SET}, + {Opt_auto_da_alloc, EXT4_MOUNT_NO_AUTO_DA_ALLOC, MOPT_CLEAR}, + {Opt_noinit_itable, EXT4_MOUNT_INIT_INODE_TABLE, MOPT_CLEAR}, + {Opt_dax_type, 0, MOPT_EXT4_ONLY}, + {Opt_journal_dev, 0, MOPT_NO_EXT2}, + {Opt_journal_path, 0, MOPT_NO_EXT2}, + {Opt_journal_ioprio, 0, MOPT_NO_EXT2}, + {Opt_data, 0, MOPT_NO_EXT2}, + {Opt_user_xattr, EXT4_MOUNT_XATTR_USER, MOPT_SET}, +#ifdef CONFIG_EXT4_FS_POSIX_ACL + {Opt_acl, EXT4_MOUNT_POSIX_ACL, MOPT_SET}, +#else + {Opt_acl, 0, MOPT_NOSUPPORT}, +#endif + {Opt_nouid32, EXT4_MOUNT_NO_UID32, MOPT_SET}, + {Opt_debug, EXT4_MOUNT_DEBUG, MOPT_SET}, + {Opt_quota, EXT4_MOUNT_QUOTA | EXT4_MOUNT_USRQUOTA, MOPT_SET | MOPT_Q}, + {Opt_usrquota, EXT4_MOUNT_QUOTA | EXT4_MOUNT_USRQUOTA, + MOPT_SET | MOPT_Q}, + {Opt_grpquota, EXT4_MOUNT_QUOTA | EXT4_MOUNT_GRPQUOTA, + MOPT_SET | MOPT_Q}, + {Opt_prjquota, EXT4_MOUNT_QUOTA | EXT4_MOUNT_PRJQUOTA, + MOPT_SET | MOPT_Q}, + {Opt_noquota, (EXT4_MOUNT_QUOTA | EXT4_MOUNT_USRQUOTA | + EXT4_MOUNT_GRPQUOTA | EXT4_MOUNT_PRJQUOTA), + MOPT_CLEAR | MOPT_Q}, + {Opt_usrjquota, 0, MOPT_Q}, + {Opt_grpjquota, 0, MOPT_Q}, + {Opt_jqfmt, 0, MOPT_QFMT}, + {Opt_nombcache, EXT4_MOUNT_NO_MBCACHE, MOPT_SET}, + {Opt_no_prefetch_block_bitmaps, EXT4_MOUNT_NO_PREFETCH_BLOCK_BITMAPS, + MOPT_SET}, +#ifdef CONFIG_EXT4_DEBUG + {Opt_fc_debug_force, EXT4_MOUNT2_JOURNAL_FAST_COMMIT, + MOPT_SET | MOPT_2 | MOPT_EXT4_ONLY}, +#endif + {Opt_abort, EXT4_MOUNT2_ABORT, MOPT_SET | MOPT_2}, + {Opt_err, 0, 0} +}; + +#if IS_ENABLED(CONFIG_UNICODE) +static const struct ext4_sb_encodings { + __u16 magic; + char *name; + unsigned int version; +} ext4_sb_encoding_map[] = { + {EXT4_ENC_UTF8_12_1, "utf8", UNICODE_AGE(12, 1, 0)}, +}; + +static const struct ext4_sb_encodings * +ext4_sb_read_encoding(const struct ext4_super_block *es) +{ + __u16 magic = le16_to_cpu(es->s_encoding); + int i; + + for (i = 0; i < ARRAY_SIZE(ext4_sb_encoding_map); i++) + if (magic == ext4_sb_encoding_map[i].magic) + return &ext4_sb_encoding_map[i]; + + return NULL; +} +#endif + +#define EXT4_SPEC_JQUOTA (1 << 0) +#define EXT4_SPEC_JQFMT (1 << 1) +#define EXT4_SPEC_DATAJ (1 << 2) +#define EXT4_SPEC_SB_BLOCK (1 << 3) +#define EXT4_SPEC_JOURNAL_DEV (1 << 4) +#define EXT4_SPEC_JOURNAL_IOPRIO (1 << 5) +#define EXT4_SPEC_s_want_extra_isize (1 << 7) +#define EXT4_SPEC_s_max_batch_time (1 << 8) +#define EXT4_SPEC_s_min_batch_time (1 << 9) +#define EXT4_SPEC_s_inode_readahead_blks (1 << 10) +#define EXT4_SPEC_s_li_wait_mult (1 << 11) +#define EXT4_SPEC_s_max_dir_size_kb (1 << 12) +#define EXT4_SPEC_s_stripe (1 << 13) +#define EXT4_SPEC_s_resuid (1 << 14) +#define EXT4_SPEC_s_resgid (1 << 15) +#define EXT4_SPEC_s_commit_interval (1 << 16) +#define EXT4_SPEC_s_fc_debug_max_replay (1 << 17) +#define EXT4_SPEC_s_sb_block (1 << 18) +#define EXT4_SPEC_mb_optimize_scan (1 << 19) + +struct ext4_fs_context { + char *s_qf_names[EXT4_MAXQUOTAS]; + struct fscrypt_dummy_policy dummy_enc_policy; + int s_jquota_fmt; /* Format of quota to use */ +#ifdef CONFIG_EXT4_DEBUG + int s_fc_debug_max_replay; +#endif + unsigned short qname_spec; + unsigned long vals_s_flags; /* Bits to set in s_flags */ + unsigned long mask_s_flags; /* Bits changed in s_flags */ + unsigned long journal_devnum; + unsigned long s_commit_interval; + unsigned long s_stripe; + unsigned int s_inode_readahead_blks; + unsigned int s_want_extra_isize; + unsigned int s_li_wait_mult; + unsigned int s_max_dir_size_kb; + unsigned int journal_ioprio; + unsigned int vals_s_mount_opt; + unsigned int mask_s_mount_opt; + unsigned int vals_s_mount_opt2; + unsigned int mask_s_mount_opt2; + unsigned int opt_flags; /* MOPT flags */ + unsigned int spec; + u32 s_max_batch_time; + u32 s_min_batch_time; + kuid_t s_resuid; + kgid_t s_resgid; + ext4_fsblk_t s_sb_block; +}; + +static void ext4_fc_free(struct fs_context *fc) +{ + struct ext4_fs_context *ctx = fc->fs_private; + int i; + + if (!ctx) + return; + + for (i = 0; i < EXT4_MAXQUOTAS; i++) + kfree(ctx->s_qf_names[i]); + + fscrypt_free_dummy_policy(&ctx->dummy_enc_policy); + kfree(ctx); +} + +int ext4_init_fs_context(struct fs_context *fc) +{ + struct ext4_fs_context *ctx; + + ctx = kzalloc_obj(struct ext4_fs_context); + if (!ctx) + return -ENOMEM; + + fc->fs_private = ctx; + fc->ops = &ext4_context_ops; + + /* i_version is always enabled now */ + fc->sb_flags |= SB_I_VERSION; + + return 0; +} + +#ifdef CONFIG_QUOTA +/* + * Note the name of the specified quota file. + */ +static int note_qf_name(struct fs_context *fc, int qtype, + struct fs_parameter *param) +{ + struct ext4_fs_context *ctx = fc->fs_private; + char *qname; + + if (param->size < 1) { + ext4_msg(NULL, KERN_ERR, "Missing quota name"); + return -EINVAL; + } + if (strchr(param->string, '/')) { + ext4_msg(NULL, KERN_ERR, + "quotafile must be on filesystem root"); + return -EINVAL; + } + if (ctx->s_qf_names[qtype]) { + if (strcmp(ctx->s_qf_names[qtype], param->string) != 0) { + ext4_msg(NULL, KERN_ERR, + "%s quota file already specified", + QTYPE2NAME(qtype)); + return -EINVAL; + } + return 0; + } + + qname = kmemdup_nul(param->string, param->size, GFP_KERNEL); + if (!qname) { + ext4_msg(NULL, KERN_ERR, + "Not enough memory for storing quotafile name"); + return -ENOMEM; + } + ctx->s_qf_names[qtype] = qname; + ctx->qname_spec |= 1 << qtype; + ctx->spec |= EXT4_SPEC_JQUOTA; + return 0; +} + +/* + * Clear the name of the specified quota file. + */ +static int unnote_qf_name(struct fs_context *fc, int qtype) +{ + struct ext4_fs_context *ctx = fc->fs_private; + + kfree(ctx->s_qf_names[qtype]); + + ctx->s_qf_names[qtype] = NULL; + ctx->qname_spec |= 1 << qtype; + ctx->spec |= EXT4_SPEC_JQUOTA; + return 0; +} +#endif + +static int ext4_parse_test_dummy_encryption(const struct fs_parameter *param, + struct ext4_fs_context *ctx) +{ + int err; + + if (!IS_ENABLED(CONFIG_FS_ENCRYPTION)) { + ext4_msg(NULL, KERN_WARNING, + "test_dummy_encryption option not supported"); + return -EINVAL; + } + err = fscrypt_parse_test_dummy_encryption(param, + &ctx->dummy_enc_policy); + if (err == -EINVAL) { + ext4_msg(NULL, KERN_WARNING, + "Value of option \"%s\" is unrecognized", param->key); + } else if (err == -EEXIST) { + ext4_msg(NULL, KERN_WARNING, + "Conflicting test_dummy_encryption options"); + return -EINVAL; + } + return err; +} + +#define EXT4_SET_CTX(name) \ +static inline __maybe_unused \ +void ctx_set_##name(struct ext4_fs_context *ctx, unsigned long flag) \ +{ \ + ctx->mask_s_##name |= flag; \ + ctx->vals_s_##name |= flag; \ +} + +#define EXT4_CLEAR_CTX(name) \ +static inline __maybe_unused \ +void ctx_clear_##name(struct ext4_fs_context *ctx, unsigned long flag) \ +{ \ + ctx->mask_s_##name |= flag; \ + ctx->vals_s_##name &= ~flag; \ +} + +#define EXT4_TEST_CTX(name) \ +static inline unsigned long \ +ctx_test_##name(struct ext4_fs_context *ctx, unsigned long flag) \ +{ \ + return (ctx->vals_s_##name & flag); \ +} + +EXT4_SET_CTX(flags); /* set only */ +EXT4_SET_CTX(mount_opt); +EXT4_CLEAR_CTX(mount_opt); +EXT4_TEST_CTX(mount_opt); +EXT4_SET_CTX(mount_opt2); +EXT4_CLEAR_CTX(mount_opt2); +EXT4_TEST_CTX(mount_opt2); + +static int ext4_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct ext4_fs_context *ctx = fc->fs_private; + struct fs_parse_result result; + const struct mount_opts *m; + int is_remount; + int token; + + token = fs_parse(fc, ext4_param_specs, param, &result); + if (token < 0) + return token; + is_remount = fc->purpose == FS_CONTEXT_FOR_RECONFIGURE; + + for (m = ext4_mount_opts; m->token != Opt_err; m++) + if (token == m->token) + break; + + ctx->opt_flags |= m->flags; + + if (m->flags & MOPT_EXPLICIT) { + if (m->mount_opt & EXT4_MOUNT_DELALLOC) { + ctx_set_mount_opt2(ctx, EXT4_MOUNT2_EXPLICIT_DELALLOC); + } else if (m->mount_opt & EXT4_MOUNT_JOURNAL_CHECKSUM) { + ctx_set_mount_opt2(ctx, + EXT4_MOUNT2_EXPLICIT_JOURNAL_CHECKSUM); + } else + return -EINVAL; + } + + if (m->flags & MOPT_NOSUPPORT) { + ext4_msg(NULL, KERN_ERR, "%s option not supported", + param->key); + return 0; + } + + switch (token) { +#ifdef CONFIG_QUOTA + case Opt_usrjquota: + if (!*param->string) + return unnote_qf_name(fc, USRQUOTA); + else + return note_qf_name(fc, USRQUOTA, param); + case Opt_grpjquota: + if (!*param->string) + return unnote_qf_name(fc, GRPQUOTA); + else + return note_qf_name(fc, GRPQUOTA, param); +#endif + case Opt_sb: + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE) { + ext4_msg(NULL, KERN_WARNING, + "Ignoring %s option on remount", param->key); + } else { + ctx->s_sb_block = result.uint_32; + ctx->spec |= EXT4_SPEC_s_sb_block; + } + return 0; + case Opt_removed: + ext4_msg(NULL, KERN_WARNING, "Ignoring removed %s option", + param->key); + return 0; + case Opt_inlinecrypt: +#ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT + ctx_set_flags(ctx, SB_INLINECRYPT); +#else + ext4_msg(NULL, KERN_ERR, "inline encryption not supported"); +#endif + return 0; + case Opt_errors: + ctx_clear_mount_opt(ctx, EXT4_MOUNT_ERRORS_MASK); + ctx_set_mount_opt(ctx, result.uint_32); + return 0; +#ifdef CONFIG_QUOTA + case Opt_jqfmt: + ctx->s_jquota_fmt = result.uint_32; + ctx->spec |= EXT4_SPEC_JQFMT; + return 0; +#endif + case Opt_data: + ctx_clear_mount_opt(ctx, EXT4_MOUNT_DATA_FLAGS); + ctx_set_mount_opt(ctx, result.uint_32); + ctx->spec |= EXT4_SPEC_DATAJ; + return 0; + case Opt_commit: + if (result.uint_32 == 0) + result.uint_32 = JBD2_DEFAULT_MAX_COMMIT_AGE; + else if (result.uint_32 > INT_MAX / HZ) { + ext4_msg(NULL, KERN_ERR, + "Invalid commit interval %d, " + "must be smaller than %d", + result.uint_32, INT_MAX / HZ); + return -EINVAL; + } + ctx->s_commit_interval = HZ * result.uint_32; + ctx->spec |= EXT4_SPEC_s_commit_interval; + return 0; + case Opt_debug_want_extra_isize: + if ((result.uint_32 & 1) || (result.uint_32 < 4)) { + ext4_msg(NULL, KERN_ERR, + "Invalid want_extra_isize %d", result.uint_32); + return -EINVAL; + } + ctx->s_want_extra_isize = result.uint_32; + ctx->spec |= EXT4_SPEC_s_want_extra_isize; + return 0; + case Opt_max_batch_time: + ctx->s_max_batch_time = result.uint_32; + ctx->spec |= EXT4_SPEC_s_max_batch_time; + return 0; + case Opt_min_batch_time: + ctx->s_min_batch_time = result.uint_32; + ctx->spec |= EXT4_SPEC_s_min_batch_time; + return 0; + case Opt_inode_readahead_blks: + if (result.uint_32 && + (result.uint_32 > (1 << 30) || + !is_power_of_2(result.uint_32))) { + ext4_msg(NULL, KERN_ERR, + "EXT4-fs: inode_readahead_blks must be " + "0 or a power of 2 smaller than 2^31"); + return -EINVAL; + } + ctx->s_inode_readahead_blks = result.uint_32; + ctx->spec |= EXT4_SPEC_s_inode_readahead_blks; + return 0; + case Opt_init_itable: + ctx_set_mount_opt(ctx, EXT4_MOUNT_INIT_INODE_TABLE); + ctx->s_li_wait_mult = EXT4_DEF_LI_WAIT_MULT; + if (param->type == fs_value_is_string) + ctx->s_li_wait_mult = result.uint_32; + ctx->spec |= EXT4_SPEC_s_li_wait_mult; + return 0; + case Opt_max_dir_size_kb: + ctx->s_max_dir_size_kb = result.uint_32; + ctx->spec |= EXT4_SPEC_s_max_dir_size_kb; + return 0; +#ifdef CONFIG_EXT4_DEBUG + case Opt_fc_debug_max_replay: + ctx->s_fc_debug_max_replay = result.uint_32; + ctx->spec |= EXT4_SPEC_s_fc_debug_max_replay; + return 0; +#endif + case Opt_stripe: + ctx->s_stripe = result.uint_32; + ctx->spec |= EXT4_SPEC_s_stripe; + return 0; + case Opt_resuid: + ctx->s_resuid = result.uid; + ctx->spec |= EXT4_SPEC_s_resuid; + return 0; + case Opt_resgid: + ctx->s_resgid = result.gid; + ctx->spec |= EXT4_SPEC_s_resgid; + return 0; + case Opt_journal_dev: + if (is_remount) { + ext4_msg(NULL, KERN_ERR, + "Cannot specify journal on remount"); + return -EINVAL; + } + ctx->journal_devnum = result.uint_32; + ctx->spec |= EXT4_SPEC_JOURNAL_DEV; + return 0; + case Opt_journal_path: + { + struct inode *journal_inode; + struct path path; + int error; + + if (is_remount) { + ext4_msg(NULL, KERN_ERR, + "Cannot specify journal on remount"); + return -EINVAL; + } + + error = fs_lookup_param(fc, param, 1, LOOKUP_FOLLOW, &path); + if (error) { + ext4_msg(NULL, KERN_ERR, "error: could not find " + "journal device path"); + return -EINVAL; + } + + journal_inode = d_inode(path.dentry); + ctx->journal_devnum = new_encode_dev(journal_inode->i_rdev); + ctx->spec |= EXT4_SPEC_JOURNAL_DEV; + path_put(&path); + return 0; + } + case Opt_journal_ioprio: + if (result.uint_32 > 7) { + ext4_msg(NULL, KERN_ERR, "Invalid journal IO priority" + " (must be 0-7)"); + return -EINVAL; + } + ctx->journal_ioprio = + IOPRIO_PRIO_VALUE(IOPRIO_CLASS_BE, result.uint_32); + ctx->spec |= EXT4_SPEC_JOURNAL_IOPRIO; + return 0; + case Opt_test_dummy_encryption: + return ext4_parse_test_dummy_encryption(param, ctx); + case Opt_dax: + case Opt_dax_type: +#ifdef CONFIG_FS_DAX + { + int type = (token == Opt_dax) ? + Opt_dax : result.uint_32; + + switch (type) { + case Opt_dax: + case Opt_dax_always: + ctx_set_mount_opt(ctx, EXT4_MOUNT_DAX_ALWAYS); + ctx_clear_mount_opt2(ctx, EXT4_MOUNT2_DAX_NEVER); + break; + case Opt_dax_never: + ctx_set_mount_opt2(ctx, EXT4_MOUNT2_DAX_NEVER); + ctx_clear_mount_opt(ctx, EXT4_MOUNT_DAX_ALWAYS); + break; + case Opt_dax_inode: + ctx_clear_mount_opt(ctx, EXT4_MOUNT_DAX_ALWAYS); + ctx_clear_mount_opt2(ctx, EXT4_MOUNT2_DAX_NEVER); + /* Strictly for printing options */ + ctx_set_mount_opt2(ctx, EXT4_MOUNT2_DAX_INODE); + break; + } + return 0; + } +#else + ext4_msg(NULL, KERN_INFO, "dax option not supported"); + return -EINVAL; +#endif + case Opt_data_err: + if (result.uint_32 == Opt_data_err_abort) + ctx_set_mount_opt(ctx, m->mount_opt); + else if (result.uint_32 == Opt_data_err_ignore) + ctx_clear_mount_opt(ctx, m->mount_opt); + return 0; + case Opt_mb_optimize_scan: + if (result.int_32 == 1) { + ctx_set_mount_opt2(ctx, EXT4_MOUNT2_MB_OPTIMIZE_SCAN); + ctx->spec |= EXT4_SPEC_mb_optimize_scan; + } else if (result.int_32 == 0) { + ctx_clear_mount_opt2(ctx, EXT4_MOUNT2_MB_OPTIMIZE_SCAN); + ctx->spec |= EXT4_SPEC_mb_optimize_scan; + } else { + ext4_msg(NULL, KERN_WARNING, + "mb_optimize_scan should be set to 0 or 1."); + return -EINVAL; + } + return 0; + } + + /* + * At this point we should only be getting options requiring MOPT_SET, + * or MOPT_CLEAR. Anything else is a bug + */ + if (m->token == Opt_err) { + ext4_msg(NULL, KERN_WARNING, "buggy handling of option %s", + param->key); + WARN_ON(1); + return -EINVAL; + } + + else { + unsigned int set = 0; + + if ((param->type == fs_value_is_flag) || + result.uint_32 > 0) + set = 1; + + if (m->flags & MOPT_CLEAR) + set = !set; + else if (unlikely(!(m->flags & MOPT_SET))) { + ext4_msg(NULL, KERN_WARNING, + "buggy handling of option %s", + param->key); + WARN_ON(1); + return -EINVAL; + } + if (m->flags & MOPT_2) { + if (set != 0) + ctx_set_mount_opt2(ctx, m->mount_opt); + else + ctx_clear_mount_opt2(ctx, m->mount_opt); + } else { + if (set != 0) + ctx_set_mount_opt(ctx, m->mount_opt); + else + ctx_clear_mount_opt(ctx, m->mount_opt); + } + } + + return 0; +} + +static int parse_options(struct fs_context *fc, char *options) +{ + struct fs_parameter param; + int ret; + char *key; + + if (!options) + return 0; + + while ((key = strsep(&options, ",")) != NULL) { + if (*key) { + size_t v_len = 0; + char *value = strchr(key, '='); + + param.type = fs_value_is_flag; + param.string = NULL; + + if (value) { + if (value == key) + continue; + + *value++ = 0; + v_len = strlen(value); + param.string = kmemdup_nul(value, v_len, + GFP_KERNEL); + if (!param.string) + return -ENOMEM; + param.type = fs_value_is_string; + } + + param.key = key; + param.size = v_len; + + ret = ext4_parse_param(fc, ¶m); + kfree(param.string); + if (ret < 0) + return ret; + } + } + + ret = ext4_validate_options(fc); + if (ret < 0) + return ret; + + return 0; +} + +static int parse_apply_sb_mount_options(struct super_block *sb, + struct ext4_fs_context *m_ctx) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + char s_mount_opts[64]; + struct ext4_fs_context *s_ctx = NULL; + struct fs_context *fc = NULL; + int ret = -ENOMEM; + + if (!sbi->s_es->s_mount_opts[0]) + return 0; + + if (strscpy_pad(s_mount_opts, sbi->s_es->s_mount_opts) < 0) + return -E2BIG; + + fc = kzalloc_obj(struct fs_context); + if (!fc) + return -ENOMEM; + + s_ctx = kzalloc_obj(struct ext4_fs_context); + if (!s_ctx) + goto out_free; + + fc->fs_private = s_ctx; + fc->s_fs_info = sbi; + + ret = parse_options(fc, s_mount_opts); + if (ret < 0) + goto parse_failed; + + ret = ext4_check_opt_consistency(fc, sb); + if (ret < 0) { +parse_failed: + ext4_msg(sb, KERN_WARNING, + "failed to parse options in superblock: %s", + s_mount_opts); + ret = 0; + goto out_free; + } + + if (s_ctx->spec & EXT4_SPEC_JOURNAL_DEV) + m_ctx->journal_devnum = s_ctx->journal_devnum; + if (s_ctx->spec & EXT4_SPEC_JOURNAL_IOPRIO) + m_ctx->journal_ioprio = s_ctx->journal_ioprio; + + ext4_apply_options(fc, sb); + ret = 0; + +out_free: + ext4_fc_free(fc); + kfree(fc); + return ret; +} + +static void ext4_apply_quota_options(struct fs_context *fc, + struct super_block *sb) +{ +#ifdef CONFIG_QUOTA + bool quota_feature = ext4_has_feature_quota(sb); + struct ext4_fs_context *ctx = fc->fs_private; + struct ext4_sb_info *sbi = EXT4_SB(sb); + char *qname; + int i; + + if (quota_feature) + return; + + if (ctx->spec & EXT4_SPEC_JQUOTA) { + for (i = 0; i < EXT4_MAXQUOTAS; i++) { + if (!(ctx->qname_spec & (1 << i))) + continue; + + qname = ctx->s_qf_names[i]; /* May be NULL */ + if (qname) + set_opt(sb, QUOTA); + ctx->s_qf_names[i] = NULL; + qname = rcu_replace_pointer(sbi->s_qf_names[i], qname, + lockdep_is_held(&sb->s_umount)); + if (qname) + kfree_rcu_mightsleep(qname); + } + } + + if (ctx->spec & EXT4_SPEC_JQFMT) + sbi->s_jquota_fmt = ctx->s_jquota_fmt; +#endif +} + +/* + * Check quota settings consistency. + */ +static int ext4_check_quota_consistency(struct fs_context *fc, + struct super_block *sb) +{ +#ifdef CONFIG_QUOTA + struct ext4_fs_context *ctx = fc->fs_private; + struct ext4_sb_info *sbi = EXT4_SB(sb); + bool quota_feature = ext4_has_feature_quota(sb); + bool quota_loaded = sb_any_quota_loaded(sb); + bool usr_qf_name, grp_qf_name, usrquota, grpquota; + int quota_flags, i; + + /* + * We do the test below only for project quotas. 'usrquota' and + * 'grpquota' mount options are allowed even without quota feature + * to support legacy quotas in quota files. + */ + if (ctx_test_mount_opt(ctx, EXT4_MOUNT_PRJQUOTA) && + !ext4_has_feature_project(sb)) { + ext4_msg(NULL, KERN_ERR, "Project quota feature not enabled. " + "Cannot enable project quota enforcement."); + return -EINVAL; + } + + quota_flags = EXT4_MOUNT_QUOTA | EXT4_MOUNT_USRQUOTA | + EXT4_MOUNT_GRPQUOTA | EXT4_MOUNT_PRJQUOTA; + if (quota_loaded && + ctx->mask_s_mount_opt & quota_flags && + !ctx_test_mount_opt(ctx, quota_flags)) + goto err_quota_change; + + if (ctx->spec & EXT4_SPEC_JQUOTA) { + + for (i = 0; i < EXT4_MAXQUOTAS; i++) { + if (!(ctx->qname_spec & (1 << i))) + continue; + + if (quota_loaded && + !!sbi->s_qf_names[i] != !!ctx->s_qf_names[i]) + goto err_jquota_change; + + if (sbi->s_qf_names[i] && ctx->s_qf_names[i] && + strcmp(get_qf_name(sb, sbi, i), + ctx->s_qf_names[i]) != 0) + goto err_jquota_specified; + } + + if (quota_feature) { + ext4_msg(NULL, KERN_INFO, + "Journaled quota options ignored when " + "QUOTA feature is enabled"); + return 0; + } + } + + if (ctx->spec & EXT4_SPEC_JQFMT) { + if (sbi->s_jquota_fmt != ctx->s_jquota_fmt && quota_loaded) + goto err_jquota_change; + if (quota_feature) { + ext4_msg(NULL, KERN_INFO, "Quota format mount options " + "ignored when QUOTA feature is enabled"); + return 0; + } + } + + /* Make sure we don't mix old and new quota format */ + usr_qf_name = (get_qf_name(sb, sbi, USRQUOTA) || + ctx->s_qf_names[USRQUOTA]); + grp_qf_name = (get_qf_name(sb, sbi, GRPQUOTA) || + ctx->s_qf_names[GRPQUOTA]); + + usrquota = (ctx_test_mount_opt(ctx, EXT4_MOUNT_USRQUOTA) || + test_opt(sb, USRQUOTA)); + + grpquota = (ctx_test_mount_opt(ctx, EXT4_MOUNT_GRPQUOTA) || + test_opt(sb, GRPQUOTA)); + + if (usr_qf_name) { + ctx_clear_mount_opt(ctx, EXT4_MOUNT_USRQUOTA); + usrquota = false; + } + if (grp_qf_name) { + ctx_clear_mount_opt(ctx, EXT4_MOUNT_GRPQUOTA); + grpquota = false; + } + + if (usr_qf_name || grp_qf_name) { + if (usrquota || grpquota) { + ext4_msg(NULL, KERN_ERR, "old and new quota " + "format mixing"); + return -EINVAL; + } + + if (!(ctx->spec & EXT4_SPEC_JQFMT || sbi->s_jquota_fmt)) { + ext4_msg(NULL, KERN_ERR, "journaled quota format " + "not specified"); + return -EINVAL; + } + } + + return 0; + +err_quota_change: + ext4_msg(NULL, KERN_ERR, + "Cannot change quota options when quota turned on"); + return -EINVAL; +err_jquota_change: + ext4_msg(NULL, KERN_ERR, "Cannot change journaled quota " + "options when quota turned on"); + return -EINVAL; +err_jquota_specified: + ext4_msg(NULL, KERN_ERR, "%s quota file already specified", + QTYPE2NAME(i)); + return -EINVAL; +#else + return 0; +#endif +} + +static int ext4_check_test_dummy_encryption(const struct fs_context *fc, + struct super_block *sb) +{ + const struct ext4_fs_context *ctx = fc->fs_private; + const struct ext4_sb_info *sbi = EXT4_SB(sb); + + if (!fscrypt_is_dummy_policy_set(&ctx->dummy_enc_policy)) + return 0; + + if (!ext4_has_feature_encrypt(sb)) { + ext4_msg(NULL, KERN_WARNING, + "test_dummy_encryption requires encrypt feature"); + return -EINVAL; + } + /* + * This mount option is just for testing, and it's not worthwhile to + * implement the extra complexity (e.g. RCU protection) that would be + * needed to allow it to be set or changed during remount. We do allow + * it to be specified during remount, but only if there is no change. + */ + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE) { + if (fscrypt_dummy_policies_equal(&sbi->s_dummy_enc_policy, + &ctx->dummy_enc_policy)) + return 0; + ext4_msg(NULL, KERN_WARNING, + "Can't set or change test_dummy_encryption on remount"); + return -EINVAL; + } + /* Also make sure s_mount_opts didn't contain a conflicting value. */ + if (fscrypt_is_dummy_policy_set(&sbi->s_dummy_enc_policy)) { + if (fscrypt_dummy_policies_equal(&sbi->s_dummy_enc_policy, + &ctx->dummy_enc_policy)) + return 0; + ext4_msg(NULL, KERN_WARNING, + "Conflicting test_dummy_encryption options"); + return -EINVAL; + } + return 0; +} + +static void ext4_apply_test_dummy_encryption(struct ext4_fs_context *ctx, + struct super_block *sb) +{ + if (!fscrypt_is_dummy_policy_set(&ctx->dummy_enc_policy) || + /* if already set, it was already verified to be the same */ + fscrypt_is_dummy_policy_set(&EXT4_SB(sb)->s_dummy_enc_policy)) + return; + EXT4_SB(sb)->s_dummy_enc_policy = ctx->dummy_enc_policy; + memset(&ctx->dummy_enc_policy, 0, sizeof(ctx->dummy_enc_policy)); + ext4_msg(sb, KERN_WARNING, "Test dummy encryption mode enabled"); +} + +static int ext4_check_opt_consistency(struct fs_context *fc, + struct super_block *sb) +{ + struct ext4_fs_context *ctx = fc->fs_private; + struct ext4_sb_info *sbi = fc->s_fs_info; + int is_remount = fc->purpose == FS_CONTEXT_FOR_RECONFIGURE; + int err; + + if ((ctx->opt_flags & MOPT_NO_EXT2) && IS_EXT2_SB(sb)) { + ext4_msg(NULL, KERN_ERR, + "Mount option(s) incompatible with ext2"); + return -EINVAL; + } + if ((ctx->opt_flags & MOPT_NO_EXT3) && IS_EXT3_SB(sb)) { + ext4_msg(NULL, KERN_ERR, + "Mount option(s) incompatible with ext3"); + return -EINVAL; + } + + if (ctx->s_want_extra_isize > + (sbi->s_inode_size - EXT4_GOOD_OLD_INODE_SIZE)) { + ext4_msg(NULL, KERN_ERR, + "Invalid want_extra_isize %d", + ctx->s_want_extra_isize); + return -EINVAL; + } + + err = ext4_check_test_dummy_encryption(fc, sb); + if (err) + return err; + + if ((ctx->spec & EXT4_SPEC_DATAJ) && is_remount) { + if (!sbi->s_journal) { + ext4_msg(NULL, KERN_WARNING, + "Remounting file system with no journal " + "so ignoring journalled data option"); + ctx_clear_mount_opt(ctx, EXT4_MOUNT_DATA_FLAGS); + } else if (ctx_test_mount_opt(ctx, EXT4_MOUNT_DATA_FLAGS) != + test_opt(sb, DATA_FLAGS)) { + ext4_msg(NULL, KERN_ERR, "Cannot change data mode " + "on remount"); + return -EINVAL; + } + } + + if (is_remount) { + if (!sbi->s_journal && + ctx_test_mount_opt(ctx, EXT4_MOUNT_DATA_ERR_ABORT)) { + ext4_msg(NULL, KERN_WARNING, + "Remounting fs w/o journal so ignoring data_err option"); + ctx_clear_mount_opt(ctx, EXT4_MOUNT_DATA_ERR_ABORT); + } + + if (ctx_test_mount_opt(ctx, EXT4_MOUNT_DAX_ALWAYS) && + (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_JOURNAL_DATA)) { + ext4_msg(NULL, KERN_ERR, "can't mount with " + "both data=journal and dax"); + return -EINVAL; + } + + if (ctx_test_mount_opt(ctx, EXT4_MOUNT_DAX_ALWAYS) && + (!(sbi->s_mount_opt & EXT4_MOUNT_DAX_ALWAYS) || + (sbi->s_mount_opt2 & EXT4_MOUNT2_DAX_NEVER))) { +fail_dax_change_remount: + ext4_msg(NULL, KERN_ERR, "can't change " + "dax mount option while remounting"); + return -EINVAL; + } else if (ctx_test_mount_opt2(ctx, EXT4_MOUNT2_DAX_NEVER) && + (!(sbi->s_mount_opt2 & EXT4_MOUNT2_DAX_NEVER) || + (sbi->s_mount_opt & EXT4_MOUNT_DAX_ALWAYS))) { + goto fail_dax_change_remount; + } else if (ctx_test_mount_opt2(ctx, EXT4_MOUNT2_DAX_INODE) && + ((sbi->s_mount_opt & EXT4_MOUNT_DAX_ALWAYS) || + (sbi->s_mount_opt2 & EXT4_MOUNT2_DAX_NEVER) || + !(sbi->s_mount_opt2 & EXT4_MOUNT2_DAX_INODE))) { + goto fail_dax_change_remount; + } + } + + return ext4_check_quota_consistency(fc, sb); +} + +static void ext4_apply_options(struct fs_context *fc, struct super_block *sb) +{ + struct ext4_fs_context *ctx = fc->fs_private; + struct ext4_sb_info *sbi = fc->s_fs_info; + + sbi->s_mount_opt &= ~ctx->mask_s_mount_opt; + sbi->s_mount_opt |= ctx->vals_s_mount_opt; + sbi->s_mount_opt2 &= ~ctx->mask_s_mount_opt2; + sbi->s_mount_opt2 |= ctx->vals_s_mount_opt2; + sb->s_flags &= ~ctx->mask_s_flags; + sb->s_flags |= ctx->vals_s_flags; + +#define APPLY(X) ({ if (ctx->spec & EXT4_SPEC_##X) sbi->X = ctx->X; }) + APPLY(s_commit_interval); + APPLY(s_stripe); + APPLY(s_max_batch_time); + APPLY(s_min_batch_time); + APPLY(s_want_extra_isize); + APPLY(s_inode_readahead_blks); + APPLY(s_max_dir_size_kb); + APPLY(s_li_wait_mult); + APPLY(s_resgid); + APPLY(s_resuid); + +#ifdef CONFIG_EXT4_DEBUG + APPLY(s_fc_debug_max_replay); +#endif + + ext4_apply_quota_options(fc, sb); + ext4_apply_test_dummy_encryption(ctx, sb); +} + + +static int ext4_validate_options(struct fs_context *fc) +{ +#ifdef CONFIG_QUOTA + struct ext4_fs_context *ctx = fc->fs_private; + char *usr_qf_name, *grp_qf_name; + + usr_qf_name = ctx->s_qf_names[USRQUOTA]; + grp_qf_name = ctx->s_qf_names[GRPQUOTA]; + + if (usr_qf_name || grp_qf_name) { + if (ctx_test_mount_opt(ctx, EXT4_MOUNT_USRQUOTA) && usr_qf_name) + ctx_clear_mount_opt(ctx, EXT4_MOUNT_USRQUOTA); + + if (ctx_test_mount_opt(ctx, EXT4_MOUNT_GRPQUOTA) && grp_qf_name) + ctx_clear_mount_opt(ctx, EXT4_MOUNT_GRPQUOTA); + + if (ctx_test_mount_opt(ctx, EXT4_MOUNT_USRQUOTA) || + ctx_test_mount_opt(ctx, EXT4_MOUNT_GRPQUOTA)) { + ext4_msg(NULL, KERN_ERR, "old and new quota " + "format mixing"); + return -EINVAL; + } + } +#endif + return 1; +} + +static inline void ext4_show_quota_options(struct seq_file *seq, + struct super_block *sb) +{ +#if defined(CONFIG_QUOTA) + struct ext4_sb_info *sbi = EXT4_SB(sb); + char *usr_qf_name, *grp_qf_name; + + if (sbi->s_jquota_fmt) { + char *fmtname = ""; + + switch (sbi->s_jquota_fmt) { + case QFMT_VFS_OLD: + fmtname = "vfsold"; + break; + case QFMT_VFS_V0: + fmtname = "vfsv0"; + break; + case QFMT_VFS_V1: + fmtname = "vfsv1"; + break; + } + seq_printf(seq, ",jqfmt=%s", fmtname); + } + + rcu_read_lock(); + usr_qf_name = rcu_dereference(sbi->s_qf_names[USRQUOTA]); + grp_qf_name = rcu_dereference(sbi->s_qf_names[GRPQUOTA]); + if (usr_qf_name) + seq_show_option(seq, "usrjquota", usr_qf_name); + if (grp_qf_name) + seq_show_option(seq, "grpjquota", grp_qf_name); + rcu_read_unlock(); +#endif +} + +static const char *token2str(int token) +{ + const struct fs_parameter_spec *spec; + + for (spec = ext4_param_specs; spec->name != NULL; spec++) + if (spec->opt == token && !spec->type) + break; + return spec->name; +} + +/* + * Show an option if + * - it's set to a non-default value OR + * - if the per-sb default is different from the global default + */ +static int _ext4_show_options(struct seq_file *seq, struct super_block *sb, + int nodefs) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_super_block *es = sbi->s_es; + int def_errors; + const struct mount_opts *m; + char sep = nodefs ? '\n' : ','; + +#define SEQ_OPTS_PUTS(str) seq_printf(seq, "%c" str, sep) +#define SEQ_OPTS_PRINT(str, arg) seq_printf(seq, "%c" str, sep, arg) + + if (sbi->s_sb_block != 1) + SEQ_OPTS_PRINT("sb=%llu", sbi->s_sb_block); + + for (m = ext4_mount_opts; m->token != Opt_err; m++) { + int want_set = m->flags & MOPT_SET; + int opt_2 = m->flags & MOPT_2; + unsigned int mount_opt, def_mount_opt; + + if (((m->flags & (MOPT_SET|MOPT_CLEAR)) == 0) || + m->flags & MOPT_SKIP) + continue; + + if (opt_2) { + mount_opt = sbi->s_mount_opt2; + def_mount_opt = sbi->s_def_mount_opt2; + } else { + mount_opt = sbi->s_mount_opt; + def_mount_opt = sbi->s_def_mount_opt; + } + /* skip if same as the default */ + if (!nodefs && !(m->mount_opt & (mount_opt ^ def_mount_opt))) + continue; + /* select Opt_noFoo vs Opt_Foo */ + if ((want_set && + (mount_opt & m->mount_opt) != m->mount_opt) || + (!want_set && (mount_opt & m->mount_opt))) + continue; + SEQ_OPTS_PRINT("%s", token2str(m->token)); + } + + if (nodefs || !uid_eq(sbi->s_resuid, make_kuid(&init_user_ns, EXT4_DEF_RESUID)) || + ext4_get_resuid(es) != EXT4_DEF_RESUID) + SEQ_OPTS_PRINT("resuid=%u", + from_kuid_munged(&init_user_ns, sbi->s_resuid)); + if (nodefs || !gid_eq(sbi->s_resgid, make_kgid(&init_user_ns, EXT4_DEF_RESGID)) || + ext4_get_resgid(es) != EXT4_DEF_RESGID) + SEQ_OPTS_PRINT("resgid=%u", + from_kgid_munged(&init_user_ns, sbi->s_resgid)); + def_errors = nodefs ? -1 : le16_to_cpu(es->s_errors); + if (test_opt(sb, ERRORS_RO) && def_errors != EXT4_ERRORS_RO) + SEQ_OPTS_PUTS("errors=remount-ro"); + if (test_opt(sb, ERRORS_CONT) && def_errors != EXT4_ERRORS_CONTINUE) + SEQ_OPTS_PUTS("errors=continue"); + if (test_opt(sb, ERRORS_PANIC) && def_errors != EXT4_ERRORS_PANIC) + SEQ_OPTS_PUTS("errors=panic"); + if (nodefs || sbi->s_commit_interval != JBD2_DEFAULT_MAX_COMMIT_AGE*HZ) + SEQ_OPTS_PRINT("commit=%lu", sbi->s_commit_interval / HZ); + if (nodefs || sbi->s_min_batch_time != EXT4_DEF_MIN_BATCH_TIME) + SEQ_OPTS_PRINT("min_batch_time=%u", sbi->s_min_batch_time); + if (nodefs || sbi->s_max_batch_time != EXT4_DEF_MAX_BATCH_TIME) + SEQ_OPTS_PRINT("max_batch_time=%u", sbi->s_max_batch_time); + if (nodefs && sb->s_flags & SB_I_VERSION) + SEQ_OPTS_PUTS("i_version"); + if (nodefs || sbi->s_stripe) + SEQ_OPTS_PRINT("stripe=%lu", sbi->s_stripe); + if (nodefs || EXT4_MOUNT_DATA_FLAGS & + (sbi->s_mount_opt ^ sbi->s_def_mount_opt)) { + if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_JOURNAL_DATA) + SEQ_OPTS_PUTS("data=journal"); + else if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_ORDERED_DATA) + SEQ_OPTS_PUTS("data=ordered"); + else if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_WRITEBACK_DATA) + SEQ_OPTS_PUTS("data=writeback"); + } + if (nodefs || + sbi->s_inode_readahead_blks != EXT4_DEF_INODE_READAHEAD_BLKS) + SEQ_OPTS_PRINT("inode_readahead_blks=%u", + sbi->s_inode_readahead_blks); + + if (test_opt(sb, INIT_INODE_TABLE) && (nodefs || + (sbi->s_li_wait_mult != EXT4_DEF_LI_WAIT_MULT))) + SEQ_OPTS_PRINT("init_itable=%u", sbi->s_li_wait_mult); + if (nodefs || sbi->s_max_dir_size_kb) + SEQ_OPTS_PRINT("max_dir_size_kb=%u", sbi->s_max_dir_size_kb); + if (test_opt(sb, DATA_ERR_ABORT)) + SEQ_OPTS_PUTS("data_err=abort"); + + fscrypt_show_test_dummy_encryption(seq, sep, sb); + + if (sb->s_flags & SB_INLINECRYPT) + SEQ_OPTS_PUTS("inlinecrypt"); + + if (test_opt(sb, DAX_ALWAYS)) { + if (IS_EXT2_SB(sb)) + SEQ_OPTS_PUTS("dax"); + else + SEQ_OPTS_PUTS("dax=always"); + } else if (test_opt2(sb, DAX_NEVER)) { + SEQ_OPTS_PUTS("dax=never"); + } else if (test_opt2(sb, DAX_INODE)) { + SEQ_OPTS_PUTS("dax=inode"); + } + + if (sbi->s_groups_count >= MB_DEFAULT_LINEAR_SCAN_THRESHOLD && + !test_opt2(sb, MB_OPTIMIZE_SCAN)) { + SEQ_OPTS_PUTS("mb_optimize_scan=0"); + } else if (sbi->s_groups_count < MB_DEFAULT_LINEAR_SCAN_THRESHOLD && + test_opt2(sb, MB_OPTIMIZE_SCAN)) { + SEQ_OPTS_PUTS("mb_optimize_scan=1"); + } + + if (nodefs && !test_opt(sb, NO_PREFETCH_BLOCK_BITMAPS)) + SEQ_OPTS_PUTS("prefetch_block_bitmaps"); + + if (ext4_emergency_ro(sb)) + SEQ_OPTS_PUTS("emergency_ro"); + + if (ext4_forced_shutdown(sb)) + SEQ_OPTS_PUTS("shutdown"); + + ext4_show_quota_options(seq, sb); + return 0; +} + +static int ext4_show_options(struct seq_file *seq, struct dentry *root) +{ + return _ext4_show_options(seq, root->d_sb, 0); +} + +int ext4_seq_options_show(struct seq_file *seq, void *offset) +{ + struct super_block *sb = seq->private; + int rc; + + seq_puts(seq, sb_rdonly(sb) ? "ro" : "rw"); + rc = _ext4_show_options(seq, sb, 1); + seq_putc(seq, '\n'); + return rc; +} + +static int ext4_setup_super(struct super_block *sb, struct ext4_super_block *es, + int read_only) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + int err = 0; + + if (le32_to_cpu(es->s_rev_level) > EXT4_MAX_SUPP_REV) { + ext4_msg(sb, KERN_ERR, "revision level too high, " + "forcing read-only mode"); + err = -EROFS; + goto done; + } + if (read_only) + goto done; + if (!(sbi->s_mount_state & EXT4_VALID_FS)) + ext4_msg(sb, KERN_WARNING, "warning: mounting unchecked fs, " + "running e2fsck is recommended"); + else if (sbi->s_mount_state & EXT4_ERROR_FS) + ext4_msg(sb, KERN_WARNING, + "warning: mounting fs with errors, " + "running e2fsck is recommended"); + else if ((__s16) le16_to_cpu(es->s_max_mnt_count) > 0 && + le16_to_cpu(es->s_mnt_count) >= + (unsigned short) (__s16) le16_to_cpu(es->s_max_mnt_count)) + ext4_msg(sb, KERN_WARNING, + "warning: maximal mount count reached, " + "running e2fsck is recommended"); + else if (le32_to_cpu(es->s_checkinterval) && + (ext4_get_tstamp(es, s_lastcheck) + + le32_to_cpu(es->s_checkinterval) <= ktime_get_real_seconds())) + ext4_msg(sb, KERN_WARNING, + "warning: checktime reached, " + "running e2fsck is recommended"); + if (!sbi->s_journal) + es->s_state &= cpu_to_le16(~EXT4_VALID_FS); + if (!(__s16) le16_to_cpu(es->s_max_mnt_count)) + es->s_max_mnt_count = cpu_to_le16(EXT4_DFL_MAX_MNT_COUNT); + le16_add_cpu(&es->s_mnt_count, 1); + ext4_update_tstamp(es, s_mtime); + if (sbi->s_journal) { + ext4_set_feature_journal_needs_recovery(sb); + if (ext4_has_feature_orphan_file(sb)) + ext4_set_feature_orphan_present(sb); + } + + err = ext4_commit_super(sb); +done: + if (test_opt(sb, DEBUG)) + printk(KERN_INFO "[EXT4 FS bs=%lu, gc=%u, " + "bpg=%lu, ipg=%lu, mo=%04x, mo2=%04x]\n", + sb->s_blocksize, + sbi->s_groups_count, + EXT4_BLOCKS_PER_GROUP(sb), + EXT4_INODES_PER_GROUP(sb), + sbi->s_mount_opt, sbi->s_mount_opt2); + return err; +} + +int ext4_alloc_flex_bg_array(struct super_block *sb, ext4_group_t ngroup) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct flex_groups **old_groups, **new_groups; + int size, i, j; + + if (!sbi->s_log_groups_per_flex) + return 0; + + size = ext4_flex_group(sbi, ngroup - 1) + 1; + if (size <= sbi->s_flex_groups_allocated) + return 0; + + new_groups = kvzalloc(roundup_pow_of_two(size * + sizeof(*sbi->s_flex_groups)), GFP_KERNEL); + if (!new_groups) { + ext4_msg(sb, KERN_ERR, + "not enough memory for %d flex group pointers", size); + return -ENOMEM; + } + for (i = sbi->s_flex_groups_allocated; i < size; i++) { + new_groups[i] = kvzalloc(roundup_pow_of_two( + sizeof(struct flex_groups)), + GFP_KERNEL); + if (!new_groups[i]) { + for (j = sbi->s_flex_groups_allocated; j < i; j++) + kvfree(new_groups[j]); + kvfree(new_groups); + ext4_msg(sb, KERN_ERR, + "not enough memory for %d flex groups", size); + return -ENOMEM; + } + } + rcu_read_lock(); + old_groups = rcu_dereference(sbi->s_flex_groups); + if (old_groups) + memcpy(new_groups, old_groups, + (sbi->s_flex_groups_allocated * + sizeof(struct flex_groups *))); + rcu_read_unlock(); + rcu_assign_pointer(sbi->s_flex_groups, new_groups); + sbi->s_flex_groups_allocated = size; + if (old_groups) + ext4_kvfree_array_rcu(old_groups); + return 0; +} + +static int ext4_fill_flex_info(struct super_block *sb) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_group_desc *gdp = NULL; + struct flex_groups *fg; + ext4_group_t flex_group; + int i, err; + + sbi->s_log_groups_per_flex = sbi->s_es->s_log_groups_per_flex; + if (sbi->s_log_groups_per_flex < 1 || sbi->s_log_groups_per_flex > 31) { + sbi->s_log_groups_per_flex = 0; + return 1; + } + + err = ext4_alloc_flex_bg_array(sb, sbi->s_groups_count); + if (err) + goto failed; + + for (i = 0; i < sbi->s_groups_count; i++) { + gdp = ext4_get_group_desc(sb, i, NULL); + + flex_group = ext4_flex_group(sbi, i); + fg = sbi_array_rcu_deref(sbi, s_flex_groups, flex_group); + atomic_add(ext4_free_inodes_count(sb, gdp), &fg->free_inodes); + atomic64_add(ext4_free_group_clusters(sb, gdp), + &fg->free_clusters); + atomic_add(ext4_used_dirs_count(sb, gdp), &fg->used_dirs); + } + + return 1; +failed: + return 0; +} + +static __le16 ext4_group_desc_csum(struct super_block *sb, __u32 block_group, + struct ext4_group_desc *gdp) +{ + int offset = offsetof(struct ext4_group_desc, bg_checksum); + __u16 crc = 0; + __le32 le_group = cpu_to_le32(block_group); + struct ext4_sb_info *sbi = EXT4_SB(sb); + + if (ext4_has_feature_metadata_csum(sbi->s_sb)) { + /* Use new metadata_csum algorithm */ + __u32 csum32; + __u16 dummy_csum = 0; + + csum32 = ext4_chksum(sbi->s_csum_seed, (__u8 *)&le_group, + sizeof(le_group)); + csum32 = ext4_chksum(csum32, (__u8 *)gdp, offset); + csum32 = ext4_chksum(csum32, (__u8 *)&dummy_csum, + sizeof(dummy_csum)); + offset += sizeof(dummy_csum); + if (offset < sbi->s_desc_size) + csum32 = ext4_chksum(csum32, (__u8 *)gdp + offset, + sbi->s_desc_size - offset); + + crc = csum32 & 0xFFFF; + goto out; + } + + /* old crc16 code */ + if (!ext4_has_feature_gdt_csum(sb)) + return 0; + + crc = crc16(~0, sbi->s_es->s_uuid, sizeof(sbi->s_es->s_uuid)); + crc = crc16(crc, (__u8 *)&le_group, sizeof(le_group)); + crc = crc16(crc, (__u8 *)gdp, offset); + offset += sizeof(gdp->bg_checksum); /* skip checksum */ + /* for checksum of struct ext4_group_desc do the rest...*/ + if (ext4_has_feature_64bit(sb) && offset < sbi->s_desc_size) + crc = crc16(crc, (__u8 *)gdp + offset, + sbi->s_desc_size - offset); + +out: + return cpu_to_le16(crc); +} + +int ext4_group_desc_csum_verify(struct super_block *sb, __u32 block_group, + struct ext4_group_desc *gdp) +{ + if (ext4_has_group_desc_csum(sb) && + (gdp->bg_checksum != ext4_group_desc_csum(sb, block_group, gdp))) + return 0; + + return 1; +} + +void ext4_group_desc_csum_set(struct super_block *sb, __u32 block_group, + struct ext4_group_desc *gdp) +{ + if (!ext4_has_group_desc_csum(sb)) + return; + gdp->bg_checksum = ext4_group_desc_csum(sb, block_group, gdp); +} + +/* Called at mount-time, super-block is locked */ +static int ext4_check_descriptors(struct super_block *sb, + ext4_fsblk_t sb_block, + ext4_group_t *first_not_zeroed) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + ext4_fsblk_t first_block = le32_to_cpu(sbi->s_es->s_first_data_block); + ext4_fsblk_t last_block; + ext4_fsblk_t last_bg_block = sb_block + ext4_bg_num_gdb(sb, 0); + ext4_fsblk_t block_bitmap; + ext4_fsblk_t inode_bitmap; + ext4_fsblk_t inode_table; + int flexbg_flag = 0; + ext4_group_t i, grp = sbi->s_groups_count; + + if (ext4_has_feature_flex_bg(sb)) + flexbg_flag = 1; + + ext4_debug("Checking group descriptors"); + + for (i = 0; i < sbi->s_groups_count; i++) { + struct ext4_group_desc *gdp = ext4_get_group_desc(sb, i, NULL); + + if (i == sbi->s_groups_count - 1 || flexbg_flag) + last_block = ext4_blocks_count(sbi->s_es) - 1; + else + last_block = first_block + + (EXT4_BLOCKS_PER_GROUP(sb) - 1); + + if ((grp == sbi->s_groups_count) && + !(gdp->bg_flags & cpu_to_le16(EXT4_BG_INODE_ZEROED))) + grp = i; + + block_bitmap = ext4_block_bitmap(sb, gdp); + if (block_bitmap == sb_block) { + ext4_msg(sb, KERN_ERR, "ext4_check_descriptors: " + "Block bitmap for group %u overlaps " + "superblock", i); + if (!sb_rdonly(sb)) + return 0; + } + if (block_bitmap >= sb_block + 1 && + block_bitmap <= last_bg_block) { + ext4_msg(sb, KERN_ERR, "ext4_check_descriptors: " + "Block bitmap for group %u overlaps " + "block group descriptors", i); + if (!sb_rdonly(sb)) + return 0; + } + if (block_bitmap < first_block || block_bitmap > last_block) { + ext4_msg(sb, KERN_ERR, "ext4_check_descriptors: " + "Block bitmap for group %u not in group " + "(block %llu)!", i, block_bitmap); + return 0; + } + inode_bitmap = ext4_inode_bitmap(sb, gdp); + if (inode_bitmap == sb_block) { + ext4_msg(sb, KERN_ERR, "ext4_check_descriptors: " + "Inode bitmap for group %u overlaps " + "superblock", i); + if (!sb_rdonly(sb)) + return 0; + } + if (inode_bitmap >= sb_block + 1 && + inode_bitmap <= last_bg_block) { + ext4_msg(sb, KERN_ERR, "ext4_check_descriptors: " + "Inode bitmap for group %u overlaps " + "block group descriptors", i); + if (!sb_rdonly(sb)) + return 0; + } + if (inode_bitmap < first_block || inode_bitmap > last_block) { + ext4_msg(sb, KERN_ERR, "ext4_check_descriptors: " + "Inode bitmap for group %u not in group " + "(block %llu)!", i, inode_bitmap); + return 0; + } + inode_table = ext4_inode_table(sb, gdp); + if (inode_table == sb_block) { + ext4_msg(sb, KERN_ERR, "ext4_check_descriptors: " + "Inode table for group %u overlaps " + "superblock", i); + if (!sb_rdonly(sb)) + return 0; + } + if (inode_table >= sb_block + 1 && + inode_table <= last_bg_block) { + ext4_msg(sb, KERN_ERR, "ext4_check_descriptors: " + "Inode table for group %u overlaps " + "block group descriptors", i); + if (!sb_rdonly(sb)) + return 0; + } + if (inode_table < first_block || + inode_table + sbi->s_itb_per_group - 1 > last_block) { + ext4_msg(sb, KERN_ERR, "ext4_check_descriptors: " + "Inode table for group %u not in group " + "(block %llu)!", i, inode_table); + return 0; + } + ext4_lock_group(sb, i); + if (!ext4_group_desc_csum_verify(sb, i, gdp)) { + ext4_msg(sb, KERN_ERR, "ext4_check_descriptors: " + "Checksum for group %u failed (%u!=%u)", + i, le16_to_cpu(ext4_group_desc_csum(sb, i, + gdp)), le16_to_cpu(gdp->bg_checksum)); + if (!sb_rdonly(sb)) { + ext4_unlock_group(sb, i); + return 0; + } + } + ext4_unlock_group(sb, i); + if (!flexbg_flag) + first_block += EXT4_BLOCKS_PER_GROUP(sb); + } + if (NULL != first_not_zeroed) + *first_not_zeroed = grp; + return 1; +} + +/* + * Maximal extent format file size. + * Resulting logical blkno at s_maxbytes must fit in our on-disk + * extent format containers, within a sector_t, and within i_blocks + * in the vfs. ext4 inode has 48 bits of i_block in fsblock units, + * so that won't be a limiting factor. + * + * However there is other limiting factor. We do store extents in the form + * of starting block and length, hence the resulting length of the extent + * covering maximum file size must fit into on-disk format containers as + * well. Given that length is always by 1 unit bigger than max unit (because + * we count 0 as well) we have to lower the s_maxbytes by one fs block. + * + * Note, this does *not* consider any metadata overhead for vfs i_blocks. + */ +static loff_t ext4_max_size(int blkbits, int has_huge_files) +{ + loff_t res; + loff_t upper_limit = MAX_LFS_FILESIZE; + + BUILD_BUG_ON(sizeof(blkcnt_t) < sizeof(u64)); + + if (!has_huge_files) { + upper_limit = (1LL << 32) - 1; + + /* total blocks in file system block size */ + upper_limit >>= (blkbits - 9); + upper_limit <<= blkbits; + } + + /* + * 32-bit extent-start container, ee_block. We lower the maxbytes + * by one fs block, so ee_len can cover the extent of maximum file + * size + */ + res = (1LL << 32) - 1; + res <<= blkbits; + + /* Sanity check against vm- & vfs- imposed limits */ + if (res > upper_limit) + res = upper_limit; + + return res; +} + +/* + * Maximal bitmap file size. There is a direct, and {,double-,triple-}indirect + * block limit, and also a limit of (2^48 - 1) 512-byte sectors in i_blocks. + * We need to be 1 filesystem block less than the 2^48 sector limit. + */ +static loff_t ext4_max_bitmap_size(int bits, int has_huge_files) +{ + loff_t upper_limit, res = EXT4_NDIR_BLOCKS; + int meta_blocks; + unsigned int ppb = 1 << (bits - 2); + + /* + * This is calculated to be the largest file size for a dense, block + * mapped file such that the file's total number of 512-byte sectors, + * including data and all indirect blocks, does not exceed (2^48 - 1). + * + * __u32 i_blocks_lo and _u16 i_blocks_high represent the total + * number of 512-byte sectors of the file. + */ + if (!has_huge_files) { + /* + * !has_huge_files or implies that the inode i_block field + * represents total file blocks in 2^32 512-byte sectors == + * size of vfs inode i_blocks * 8 + */ + upper_limit = (1LL << 32) - 1; + + /* total blocks in file system block size */ + upper_limit >>= (bits - 9); + + } else { + /* + * We use 48 bit ext4_inode i_blocks + * With EXT4_HUGE_FILE_FL set the i_blocks + * represent total number of blocks in + * file system block size + */ + upper_limit = (1LL << 48) - 1; + + } + + /* Compute how many blocks we can address by block tree */ + res += ppb; + res += ppb * ppb; + res += ((loff_t)ppb) * ppb * ppb; + /* Compute how many metadata blocks are needed */ + meta_blocks = 1; + meta_blocks += 1 + ppb; + meta_blocks += 1 + ppb + ppb * ppb; + /* Does block tree limit file size? */ + if (res + meta_blocks <= upper_limit) + goto check_lfs; + + res = upper_limit; + /* How many metadata blocks are needed for addressing upper_limit? */ + upper_limit -= EXT4_NDIR_BLOCKS; + /* indirect blocks */ + meta_blocks = 1; + upper_limit -= ppb; + /* double indirect blocks */ + if (upper_limit < ppb * ppb) { + meta_blocks += 1 + DIV_ROUND_UP_ULL(upper_limit, ppb); + res -= meta_blocks; + goto check_lfs; + } + meta_blocks += 1 + ppb; + upper_limit -= ppb * ppb; + /* tripple indirect blocks for the rest */ + meta_blocks += 1 + DIV_ROUND_UP_ULL(upper_limit, ppb) + + DIV_ROUND_UP_ULL(upper_limit, ppb*ppb); + res -= meta_blocks; +check_lfs: + res <<= bits; + if (res > MAX_LFS_FILESIZE) + res = MAX_LFS_FILESIZE; + + return res; +} + +static ext4_fsblk_t descriptor_loc(struct super_block *sb, + ext4_fsblk_t logical_sb_block, int nr) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + ext4_group_t bg, first_meta_bg; + int has_super = 0; + + first_meta_bg = le32_to_cpu(sbi->s_es->s_first_meta_bg); + + if (!ext4_has_feature_meta_bg(sb) || nr < first_meta_bg) + return logical_sb_block + nr + 1; + bg = sbi->s_desc_per_block * nr; + if (ext4_bg_has_super(sb, bg)) + has_super = 1; + + /* + * If we have a meta_bg fs with 1k blocks, group 0's GDT is at + * block 2, not 1. If s_first_data_block == 0 (bigalloc is enabled + * on modern mke2fs or blksize > 1k on older mke2fs) then we must + * compensate. + */ + if (sb->s_blocksize == 1024 && nr == 0 && + le32_to_cpu(sbi->s_es->s_first_data_block) == 0) + has_super++; + + return (has_super + ext4_group_first_block_no(sb, bg)); +} + +/** + * ext4_get_stripe_size: Get the stripe size. + * @sbi: In memory super block info + * + * If we have specified it via mount option, then + * use the mount option value. If the value specified at mount time is + * greater than the blocks per group use the super block value. + * If the super block value is greater than blocks per group return 0. + * Allocator needs it be less than blocks per group. + * + */ +static unsigned long ext4_get_stripe_size(struct ext4_sb_info *sbi) +{ + unsigned long stride = le16_to_cpu(sbi->s_es->s_raid_stride); + unsigned long stripe_width = + le32_to_cpu(sbi->s_es->s_raid_stripe_width); + int ret; + + if (sbi->s_stripe && sbi->s_stripe <= sbi->s_blocks_per_group) + ret = sbi->s_stripe; + else if (stripe_width && stripe_width <= sbi->s_blocks_per_group) + ret = stripe_width; + else if (stride && stride <= sbi->s_blocks_per_group) + ret = stride; + else + ret = 0; + + /* + * If the stripe width is 1, this makes no sense and + * we set it to 0 to turn off stripe handling code. + */ + if (ret <= 1) + ret = 0; + + return ret; +} + +/* + * Check whether this filesystem can be mounted based on + * the features present and the RDONLY/RDWR mount requested. + * Returns 1 if this filesystem can be mounted as requested, + * 0 if it cannot be. + */ +int ext4_feature_set_ok(struct super_block *sb, int readonly) +{ + if (ext4_has_unknown_ext4_incompat_features(sb)) { + ext4_msg(sb, KERN_ERR, + "Couldn't mount because of " + "unsupported optional features (%x)", + (le32_to_cpu(EXT4_SB(sb)->s_es->s_feature_incompat) & + ~EXT4_FEATURE_INCOMPAT_SUPP)); + return 0; + } + + if (!IS_ENABLED(CONFIG_UNICODE) && ext4_has_feature_casefold(sb)) { + ext4_msg(sb, KERN_ERR, + "Filesystem with casefold feature cannot be " + "mounted without CONFIG_UNICODE"); + return 0; + } + + if (readonly) + return 1; + + if (ext4_has_feature_readonly(sb)) { + ext4_msg(sb, KERN_INFO, "filesystem is read-only"); + sb->s_flags |= SB_RDONLY; + return 1; + } + + /* Check that feature set is OK for a read-write mount */ + if (ext4_has_unknown_ext4_ro_compat_features(sb)) { + ext4_msg(sb, KERN_ERR, "couldn't mount RDWR because of " + "unsupported optional features (%x)", + (le32_to_cpu(EXT4_SB(sb)->s_es->s_feature_ro_compat) & + ~EXT4_FEATURE_RO_COMPAT_SUPP)); + return 0; + } + if (ext4_has_feature_bigalloc(sb) && !ext4_has_feature_extents(sb)) { + ext4_msg(sb, KERN_ERR, + "Can't support bigalloc feature without " + "extents feature\n"); + return 0; + } + if (ext4_has_feature_bigalloc(sb) && + le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block)) { + ext4_msg(sb, KERN_WARNING, + "bad geometry: bigalloc file system with non-zero " + "first_data_block\n"); + return 0; + } + +#if !IS_ENABLED(CONFIG_QUOTA) || !IS_ENABLED(CONFIG_QFMT_V2) + if (!readonly && (ext4_has_feature_quota(sb) || + ext4_has_feature_project(sb))) { + ext4_msg(sb, KERN_ERR, + "The kernel was not built with CONFIG_QUOTA and CONFIG_QFMT_V2"); + return 0; + } +#endif /* CONFIG_QUOTA */ + return 1; +} + +/* + * This function is called once a day by default if we have errors logged + * on the file system. + * Use the err_report_sec sysfs attribute to disable or adjust its call + * freequency. + */ +void print_daily_error_info(struct timer_list *t) +{ + struct ext4_sb_info *sbi = timer_container_of(sbi, t, s_err_report); + struct super_block *sb = sbi->s_sb; + struct ext4_super_block *es = sbi->s_es; + + if (es->s_error_count) + /* fsck newer than v1.41.13 is needed to clean this condition. */ + ext4_msg(sb, KERN_NOTICE, "error count since last fsck: %u", + le32_to_cpu(es->s_error_count)); + if (es->s_first_error_time) { + printk(KERN_NOTICE "EXT4-fs (%s): initial error at time %llu: %.*s:%d", + sb->s_id, + ext4_get_tstamp(es, s_first_error_time), + (int) sizeof(es->s_first_error_func), + es->s_first_error_func, + le32_to_cpu(es->s_first_error_line)); + if (es->s_first_error_ino) + printk(KERN_CONT ": inode %u", + le32_to_cpu(es->s_first_error_ino)); + if (es->s_first_error_block) + printk(KERN_CONT ": block %llu", (unsigned long long) + le64_to_cpu(es->s_first_error_block)); + printk(KERN_CONT "\n"); + } + if (es->s_last_error_time) { + printk(KERN_NOTICE "EXT4-fs (%s): last error at time %llu: %.*s:%d", + sb->s_id, + ext4_get_tstamp(es, s_last_error_time), + (int) sizeof(es->s_last_error_func), + es->s_last_error_func, + le32_to_cpu(es->s_last_error_line)); + if (es->s_last_error_ino) + printk(KERN_CONT ": inode %u", + le32_to_cpu(es->s_last_error_ino)); + if (es->s_last_error_block) + printk(KERN_CONT ": block %llu", (unsigned long long) + le64_to_cpu(es->s_last_error_block)); + printk(KERN_CONT "\n"); + } + + if (sbi->s_err_report_sec) + mod_timer(&sbi->s_err_report, jiffies + secs_to_jiffies(sbi->s_err_report_sec)); +} + +/* Find next suitable group and run ext4_init_inode_table */ +static int ext4_run_li_request(struct ext4_li_request *elr) +{ + struct ext4_group_desc *gdp = NULL; + struct super_block *sb = elr->lr_super; + ext4_group_t ngroups = EXT4_SB(sb)->s_groups_count; + ext4_group_t group = elr->lr_next_group; + unsigned int prefetch_ios = 0; + int ret = 0; + int nr = EXT4_SB(sb)->s_mb_prefetch; + u64 start_time; + + if (elr->lr_mode == EXT4_LI_MODE_PREFETCH_BBITMAP) { + elr->lr_next_group = ext4_mb_prefetch(sb, group, nr, &prefetch_ios); + ext4_mb_prefetch_fini(sb, elr->lr_next_group, nr); + trace_ext4_prefetch_bitmaps(sb, group, elr->lr_next_group, nr); + if (group >= elr->lr_next_group) { + ret = 1; + if (elr->lr_first_not_zeroed != ngroups && + !ext4_emergency_state(sb) && !sb_rdonly(sb) && + test_opt(sb, INIT_INODE_TABLE)) { + elr->lr_next_group = elr->lr_first_not_zeroed; + elr->lr_mode = EXT4_LI_MODE_ITABLE; + ret = 0; + } + } + return ret; + } + + for (; group < ngroups; group++) { + gdp = ext4_get_group_desc(sb, group, NULL); + if (!gdp) { + ret = 1; + break; + } + + if (!(gdp->bg_flags & cpu_to_le16(EXT4_BG_INODE_ZEROED))) + break; + } + + if (group >= ngroups) + ret = 1; + + if (!ret) { + start_time = ktime_get_ns(); + ret = ext4_init_inode_table(sb, group, + elr->lr_timeout ? 0 : 1); + trace_ext4_lazy_itable_init(sb, group); + if (elr->lr_timeout == 0) { + elr->lr_timeout = nsecs_to_jiffies((ktime_get_ns() - start_time) * + EXT4_SB(elr->lr_super)->s_li_wait_mult); + } + elr->lr_next_sched = jiffies + elr->lr_timeout; + elr->lr_next_group = group + 1; + } + return ret; +} + +/* + * Remove lr_request from the list_request and free the + * request structure. Should be called with li_list_mtx held + */ +static void ext4_remove_li_request(struct ext4_li_request *elr) +{ + if (!elr) + return; + + list_del(&elr->lr_request); + EXT4_SB(elr->lr_super)->s_li_request = NULL; + kfree(elr); +} + +static void ext4_unregister_li_request(struct super_block *sb) +{ + mutex_lock(&ext4_li_mtx); + if (!ext4_li_info) { + mutex_unlock(&ext4_li_mtx); + return; + } + + mutex_lock(&ext4_li_info->li_list_mtx); + ext4_remove_li_request(EXT4_SB(sb)->s_li_request); + mutex_unlock(&ext4_li_info->li_list_mtx); + mutex_unlock(&ext4_li_mtx); +} + +static struct task_struct *ext4_lazyinit_task; + +/* + * This is the function where ext4lazyinit thread lives. It walks + * through the request list searching for next scheduled filesystem. + * When such a fs is found, run the lazy initialization request + * (ext4_rn_li_request) and keep track of the time spend in this + * function. Based on that time we compute next schedule time of + * the request. When walking through the list is complete, compute + * next waking time and put itself into sleep. + */ +static int ext4_lazyinit_thread(void *arg) +{ + struct ext4_lazy_init *eli = arg; + struct list_head *pos, *n; + struct ext4_li_request *elr; + unsigned long next_wakeup, cur; + + BUG_ON(NULL == eli); + set_freezable(); + +cont_thread: + while (true) { + bool next_wakeup_initialized = false; + + next_wakeup = 0; + mutex_lock(&eli->li_list_mtx); + if (list_empty(&eli->li_request_list)) { + mutex_unlock(&eli->li_list_mtx); + goto exit_thread; + } + list_for_each_safe(pos, n, &eli->li_request_list) { + int err = 0; + int progress = 0; + elr = list_entry(pos, struct ext4_li_request, + lr_request); + + if (time_before(jiffies, elr->lr_next_sched)) { + if (!next_wakeup_initialized || + time_before(elr->lr_next_sched, next_wakeup)) { + next_wakeup = elr->lr_next_sched; + next_wakeup_initialized = true; + } + continue; + } + if (down_read_trylock(&elr->lr_super->s_umount)) { + if (sb_start_write_trylock(elr->lr_super)) { + progress = 1; + /* + * We hold sb->s_umount, sb can not + * be removed from the list, it is + * now safe to drop li_list_mtx + */ + mutex_unlock(&eli->li_list_mtx); + err = ext4_run_li_request(elr); + sb_end_write(elr->lr_super); + mutex_lock(&eli->li_list_mtx); + n = pos->next; + } + up_read((&elr->lr_super->s_umount)); + } + /* error, remove the lazy_init job */ + if (err) { + ext4_remove_li_request(elr); + continue; + } + if (!progress) { + elr->lr_next_sched = jiffies + + get_random_u32_below(EXT4_DEF_LI_MAX_START_DELAY * HZ); + } + if (!next_wakeup_initialized || + time_before(elr->lr_next_sched, next_wakeup)) { + next_wakeup = elr->lr_next_sched; + next_wakeup_initialized = true; + } + } + mutex_unlock(&eli->li_list_mtx); + + try_to_freeze(); + + cur = jiffies; + if (!next_wakeup_initialized || time_after_eq(cur, next_wakeup)) { + cond_resched(); + continue; + } + + schedule_timeout_interruptible(next_wakeup - cur); + + if (kthread_should_stop()) { + ext4_clear_request_list(); + goto exit_thread; + } + } + +exit_thread: + /* + * It looks like the request list is empty, but we need + * to check it under the li_list_mtx lock, to prevent any + * additions into it, and of course we should lock ext4_li_mtx + * to atomically free the list and ext4_li_info, because at + * this point another ext4 filesystem could be registering + * new one. + */ + mutex_lock(&ext4_li_mtx); + mutex_lock(&eli->li_list_mtx); + if (!list_empty(&eli->li_request_list)) { + mutex_unlock(&eli->li_list_mtx); + mutex_unlock(&ext4_li_mtx); + goto cont_thread; + } + mutex_unlock(&eli->li_list_mtx); + kfree(ext4_li_info); + ext4_li_info = NULL; + mutex_unlock(&ext4_li_mtx); + + return 0; +} + +static void ext4_clear_request_list(void) +{ + struct list_head *pos, *n; + struct ext4_li_request *elr; + + mutex_lock(&ext4_li_info->li_list_mtx); + list_for_each_safe(pos, n, &ext4_li_info->li_request_list) { + elr = list_entry(pos, struct ext4_li_request, + lr_request); + ext4_remove_li_request(elr); + } + mutex_unlock(&ext4_li_info->li_list_mtx); +} + +static int ext4_run_lazyinit_thread(void) +{ + ext4_lazyinit_task = kthread_run(ext4_lazyinit_thread, + ext4_li_info, "ext4lazyinit"); + if (IS_ERR(ext4_lazyinit_task)) { + int err = PTR_ERR(ext4_lazyinit_task); + ext4_clear_request_list(); + kfree(ext4_li_info); + ext4_li_info = NULL; + printk(KERN_CRIT "EXT4-fs: error %d creating inode table " + "initialization thread\n", + err); + return err; + } + ext4_li_info->li_state |= EXT4_LAZYINIT_RUNNING; + return 0; +} + +/* + * Check whether it make sense to run itable init. thread or not. + * If there is at least one uninitialized inode table, return + * corresponding group number, else the loop goes through all + * groups and return total number of groups. + */ +static ext4_group_t ext4_has_uninit_itable(struct super_block *sb) +{ + ext4_group_t group, ngroups = EXT4_SB(sb)->s_groups_count; + struct ext4_group_desc *gdp = NULL; + + if (!ext4_has_group_desc_csum(sb)) + return ngroups; + + for (group = 0; group < ngroups; group++) { + gdp = ext4_get_group_desc(sb, group, NULL); + if (!gdp) + continue; + + if (!(gdp->bg_flags & cpu_to_le16(EXT4_BG_INODE_ZEROED))) + break; + } + + return group; +} + +static int ext4_li_info_new(void) +{ + struct ext4_lazy_init *eli = NULL; + + eli = kzalloc_obj(*eli); + if (!eli) + return -ENOMEM; + + INIT_LIST_HEAD(&eli->li_request_list); + mutex_init(&eli->li_list_mtx); + + eli->li_state |= EXT4_LAZYINIT_QUIT; + + ext4_li_info = eli; + + return 0; +} + +static struct ext4_li_request *ext4_li_request_new(struct super_block *sb, + ext4_group_t start) +{ + struct ext4_li_request *elr; + + elr = kzalloc_obj(*elr); + if (!elr) + return NULL; + + elr->lr_super = sb; + elr->lr_first_not_zeroed = start; + if (test_opt(sb, NO_PREFETCH_BLOCK_BITMAPS)) { + elr->lr_mode = EXT4_LI_MODE_ITABLE; + elr->lr_next_group = start; + } else { + elr->lr_mode = EXT4_LI_MODE_PREFETCH_BBITMAP; + } + + /* + * Randomize first schedule time of the request to + * spread the inode table initialization requests + * better. + */ + elr->lr_next_sched = jiffies + get_random_u32_below(EXT4_DEF_LI_MAX_START_DELAY * HZ); + return elr; +} + +int ext4_register_li_request(struct super_block *sb, + ext4_group_t first_not_zeroed) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_li_request *elr = NULL; + ext4_group_t ngroups = sbi->s_groups_count; + int ret = 0; + + mutex_lock(&ext4_li_mtx); + if (sbi->s_li_request != NULL) { + /* + * Reset timeout so it can be computed again, because + * s_li_wait_mult might have changed. + */ + sbi->s_li_request->lr_timeout = 0; + goto out; + } + + if (ext4_emergency_state(sb) || sb_rdonly(sb) || + (test_opt(sb, NO_PREFETCH_BLOCK_BITMAPS) && + (first_not_zeroed == ngroups || !test_opt(sb, INIT_INODE_TABLE)))) + goto out; + + elr = ext4_li_request_new(sb, first_not_zeroed); + if (!elr) { + ret = -ENOMEM; + goto out; + } + + if (NULL == ext4_li_info) { + ret = ext4_li_info_new(); + if (ret) + goto out; + } + + mutex_lock(&ext4_li_info->li_list_mtx); + list_add(&elr->lr_request, &ext4_li_info->li_request_list); + mutex_unlock(&ext4_li_info->li_list_mtx); + + sbi->s_li_request = elr; + /* + * set elr to NULL here since it has been inserted to + * the request_list and the removal and free of it is + * handled by ext4_clear_request_list from now on. + */ + elr = NULL; + + if (!(ext4_li_info->li_state & EXT4_LAZYINIT_RUNNING)) { + ret = ext4_run_lazyinit_thread(); + if (ret) + goto out; + } +out: + mutex_unlock(&ext4_li_mtx); + if (ret) + kfree(elr); + return ret; +} + +/* + * We do not need to lock anything since this is called on + * module unload. + */ +static void ext4_destroy_lazyinit_thread(void) +{ + /* + * If thread exited earlier + * there's nothing to be done. + */ + if (!ext4_li_info || !ext4_lazyinit_task) + return; + + kthread_stop(ext4_lazyinit_task); +} + +static int set_journal_csum_feature_set(struct super_block *sb) +{ + int ret = 1; + int compat, incompat; + struct ext4_sb_info *sbi = EXT4_SB(sb); + + if (ext4_has_feature_metadata_csum(sb)) { + /* journal checksum v3 */ + compat = 0; + incompat = JBD2_FEATURE_INCOMPAT_CSUM_V3; + } else { + /* journal checksum v1 */ + compat = JBD2_FEATURE_COMPAT_CHECKSUM; + incompat = 0; + } + + jbd2_journal_clear_features(sbi->s_journal, + JBD2_FEATURE_COMPAT_CHECKSUM, 0, + JBD2_FEATURE_INCOMPAT_CSUM_V3 | + JBD2_FEATURE_INCOMPAT_CSUM_V2); + if (test_opt(sb, JOURNAL_ASYNC_COMMIT)) { + ret = jbd2_journal_set_features(sbi->s_journal, + compat, 0, + JBD2_FEATURE_INCOMPAT_ASYNC_COMMIT | + incompat); + } else if (test_opt(sb, JOURNAL_CHECKSUM)) { + ret = jbd2_journal_set_features(sbi->s_journal, + compat, 0, + incompat); + jbd2_journal_clear_features(sbi->s_journal, 0, 0, + JBD2_FEATURE_INCOMPAT_ASYNC_COMMIT); + } else { + jbd2_journal_clear_features(sbi->s_journal, 0, 0, + JBD2_FEATURE_INCOMPAT_ASYNC_COMMIT); + } + + return ret; +} + +/* + * Note: calculating the overhead so we can be compatible with + * historical BSD practice is quite difficult in the face of + * clusters/bigalloc. This is because multiple metadata blocks from + * different block group can end up in the same allocation cluster. + * Calculating the exact overhead in the face of clustered allocation + * requires either O(all block bitmaps) in memory or O(number of block + * groups**2) in time. We will still calculate the superblock for + * older file systems --- and if we come across with a bigalloc file + * system with zero in s_overhead_clusters the estimate will be close to + * correct especially for very large cluster sizes --- but for newer + * file systems, it's better to calculate this figure once at mkfs + * time, and store it in the superblock. If the superblock value is + * present (even for non-bigalloc file systems), we will use it. + */ +static int count_overhead(struct super_block *sb, ext4_group_t grp, + char *buf) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_group_desc *gdp; + ext4_fsblk_t first_block, last_block, b; + ext4_group_t i, ngroups = ext4_get_groups_count(sb); + int s, j, count = 0; + int has_super = ext4_bg_has_super(sb, grp); + + if (!ext4_has_feature_bigalloc(sb)) + return (has_super + ext4_bg_num_gdb(sb, grp) + + (has_super ? le16_to_cpu(sbi->s_es->s_reserved_gdt_blocks) : 0) + + sbi->s_itb_per_group + 2); + + first_block = le32_to_cpu(sbi->s_es->s_first_data_block) + + (grp * EXT4_BLOCKS_PER_GROUP(sb)); + last_block = first_block + EXT4_BLOCKS_PER_GROUP(sb) - 1; + for (i = 0; i < ngroups; i++) { + gdp = ext4_get_group_desc(sb, i, NULL); + b = ext4_block_bitmap(sb, gdp); + if (b >= first_block && b <= last_block) { + ext4_set_bit(EXT4_B2C(sbi, b - first_block), buf); + count++; + } + b = ext4_inode_bitmap(sb, gdp); + if (b >= first_block && b <= last_block) { + ext4_set_bit(EXT4_B2C(sbi, b - first_block), buf); + count++; + } + b = ext4_inode_table(sb, gdp); + if (b >= first_block && b + sbi->s_itb_per_group <= last_block) + for (j = 0; j < sbi->s_itb_per_group; j++, b++) { + int c = EXT4_B2C(sbi, b - first_block); + ext4_set_bit(c, buf); + count++; + } + if (i != grp) + continue; + s = 0; + if (ext4_bg_has_super(sb, grp)) { + ext4_set_bit(s++, buf); + count++; + } + j = ext4_bg_num_gdb(sb, grp); + if (s + j > EXT4_BLOCKS_PER_GROUP(sb)) { + ext4_error(sb, "Invalid number of block group " + "descriptor blocks: %d", j); + j = EXT4_BLOCKS_PER_GROUP(sb) - s; + } + count += j; + for (; j > 0; j--) + ext4_set_bit(EXT4_B2C(sbi, s++), buf); + } + if (!count) + return 0; + return EXT4_CLUSTERS_PER_GROUP(sb) - + ext4_count_free(buf, EXT4_CLUSTERS_PER_GROUP(sb) / 8); +} + +/* + * Compute the overhead and stash it in sbi->s_overhead + */ +int ext4_calculate_overhead(struct super_block *sb) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_super_block *es = sbi->s_es; + struct inode *j_inode; + unsigned int j_blocks, j_inum = le32_to_cpu(es->s_journal_inum); + ext4_group_t i, ngroups = ext4_get_groups_count(sb); + ext4_fsblk_t overhead = 0; + char *buf = kvmalloc(sb->s_blocksize, GFP_NOFS | __GFP_ZERO); + + if (!buf) + return -ENOMEM; + + /* + * Compute the overhead (FS structures). This is constant + * for a given filesystem unless the number of block groups + * changes so we cache the previous value until it does. + */ + + /* + * All of the blocks before first_data_block are overhead + */ + overhead = EXT4_B2C(sbi, le32_to_cpu(es->s_first_data_block)); + + /* + * Add the overhead found in each block group + */ + for (i = 0; i < ngroups; i++) { + int blks; + + blks = count_overhead(sb, i, buf); + overhead += blks; + if (blks) + memset(buf, 0, sb->s_blocksize); + cond_resched(); + } + + /* + * Add the internal journal blocks whether the journal has been + * loaded or not + */ + if (sbi->s_journal && !sbi->s_journal_bdev_file) + overhead += EXT4_NUM_B2C(sbi, sbi->s_journal->j_total_len); + else if (ext4_has_feature_journal(sb) && !sbi->s_journal && j_inum) { + /* j_inum for internal journal is non-zero */ + j_inode = ext4_get_journal_inode(sb, j_inum); + if (!IS_ERR(j_inode)) { + j_blocks = j_inode->i_size >> sb->s_blocksize_bits; + overhead += EXT4_NUM_B2C(sbi, j_blocks); + iput(j_inode); + } else { + ext4_msg(sb, KERN_ERR, "can't get journal size"); + } + } + sbi->s_overhead = overhead; + smp_wmb(); + kvfree(buf); + return 0; +} + +static void ext4_set_resv_clusters(struct super_block *sb) +{ + ext4_fsblk_t resv_clusters; + struct ext4_sb_info *sbi = EXT4_SB(sb); + + /* + * There's no need to reserve anything when we aren't using extents. + * The space estimates are exact, there are no unwritten extents, + * hole punching doesn't need new metadata... This is needed especially + * to keep ext2/3 backward compatibility. + */ + if (!ext4_has_feature_extents(sb)) + return; + /* + * By default we reserve 2% or 4096 clusters, whichever is smaller. + * This should cover the situations where we can not afford to run + * out of space like for example punch hole, or converting + * unwritten extents in delalloc path. In most cases such + * allocation would require 1, or 2 blocks, higher numbers are + * very rare. + */ + resv_clusters = (ext4_blocks_count(sbi->s_es) >> + sbi->s_cluster_bits); + + do_div(resv_clusters, 50); + resv_clusters = min_t(ext4_fsblk_t, resv_clusters, 4096); + + atomic64_set(&sbi->s_resv_clusters, resv_clusters); +} + +static const char *ext4_quota_mode(struct super_block *sb) +{ +#ifdef CONFIG_QUOTA + if (!ext4_quota_capable(sb)) + return "none"; + + if (EXT4_SB(sb)->s_journal && ext4_is_quota_journalled(sb)) + return "journalled"; + else + return "writeback"; +#else + return "disabled"; +#endif +} + +static void ext4_setup_csum_trigger(struct super_block *sb, + enum ext4_journal_trigger_type type, + void (*trigger)( + struct jbd2_buffer_trigger_type *type, + struct buffer_head *bh, + void *mapped_data, + size_t size)) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + + sbi->s_journal_triggers[type].sb = sb; + sbi->s_journal_triggers[type].tr_triggers.t_frozen = trigger; +} + +static void ext4_free_sbi(struct ext4_sb_info *sbi) +{ + if (!sbi) + return; + + kfree(sbi->s_blockgroup_lock); + fs_put_dax(sbi->s_daxdev, NULL); + kfree(sbi); +} + +static struct ext4_sb_info *ext4_alloc_sbi(struct super_block *sb) +{ + struct ext4_sb_info *sbi; + + sbi = kzalloc_obj(*sbi); + if (!sbi) + return NULL; + + sbi->s_daxdev = fs_dax_get_by_bdev(sb->s_bdev, &sbi->s_dax_part_off, + NULL, NULL); + + sbi->s_blockgroup_lock = + kzalloc_obj(struct blockgroup_lock); + + if (!sbi->s_blockgroup_lock) + goto err_out; + + sb->s_fs_info = sbi; + sbi->s_sb = sb; + return sbi; +err_out: + fs_put_dax(sbi->s_daxdev, NULL); + kfree(sbi); + return NULL; +} + +static void ext4_set_def_opts(struct super_block *sb, + struct ext4_super_block *es) +{ + unsigned long def_mount_opts; + + /* Set defaults before we parse the mount options */ + def_mount_opts = le32_to_cpu(es->s_default_mount_opts); + set_opt(sb, INIT_INODE_TABLE); + if (def_mount_opts & EXT4_DEFM_DEBUG) + set_opt(sb, DEBUG); + if (def_mount_opts & EXT4_DEFM_BSDGROUPS) + set_opt(sb, GRPID); + if (def_mount_opts & EXT4_DEFM_UID16) + set_opt(sb, NO_UID32); + /* xattr user namespace & acls are now defaulted on */ + set_opt(sb, XATTR_USER); +#ifdef CONFIG_EXT4_FS_POSIX_ACL + set_opt(sb, POSIX_ACL); +#endif + if (ext4_has_feature_fast_commit(sb)) + set_opt2(sb, JOURNAL_FAST_COMMIT); + /* don't forget to enable journal_csum when metadata_csum is enabled. */ + if (ext4_has_feature_metadata_csum(sb)) + set_opt(sb, JOURNAL_CHECKSUM); + + if ((def_mount_opts & EXT4_DEFM_JMODE) == EXT4_DEFM_JMODE_DATA) + set_opt(sb, JOURNAL_DATA); + else if ((def_mount_opts & EXT4_DEFM_JMODE) == EXT4_DEFM_JMODE_ORDERED) + set_opt(sb, ORDERED_DATA); + else if ((def_mount_opts & EXT4_DEFM_JMODE) == EXT4_DEFM_JMODE_WBACK) + set_opt(sb, WRITEBACK_DATA); + + if (le16_to_cpu(es->s_errors) == EXT4_ERRORS_PANIC) + set_opt(sb, ERRORS_PANIC); + else if (le16_to_cpu(es->s_errors) == EXT4_ERRORS_CONTINUE) + set_opt(sb, ERRORS_CONT); + else + set_opt(sb, ERRORS_RO); + /* block_validity enabled by default; disable with noblock_validity */ + set_opt(sb, BLOCK_VALIDITY); + if (def_mount_opts & EXT4_DEFM_DISCARD) + set_opt(sb, DISCARD); + + if ((def_mount_opts & EXT4_DEFM_NOBARRIER) == 0) + set_opt(sb, BARRIER); + + /* + * enable delayed allocation by default + * Use -o nodelalloc to turn it off + */ + if (!IS_EXT3_SB(sb) && !IS_EXT2_SB(sb) && + ((def_mount_opts & EXT4_DEFM_NODELALLOC) == 0)) + set_opt(sb, DELALLOC); + + set_opt(sb, DIOREAD_NOLOCK); +} + +static int ext4_handle_clustersize(struct super_block *sb) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_super_block *es = sbi->s_es; + int clustersize; + + /* Handle clustersize */ + clustersize = BLOCK_SIZE << le32_to_cpu(es->s_log_cluster_size); + if (ext4_has_feature_bigalloc(sb)) { + if (clustersize < sb->s_blocksize) { + ext4_msg(sb, KERN_ERR, + "cluster size (%d) smaller than " + "block size (%lu)", clustersize, sb->s_blocksize); + return -EINVAL; + } + sbi->s_cluster_bits = le32_to_cpu(es->s_log_cluster_size) - + le32_to_cpu(es->s_log_block_size); + } else { + if (clustersize != sb->s_blocksize) { + ext4_msg(sb, KERN_ERR, + "fragment/cluster size (%d) != " + "block size (%lu)", clustersize, sb->s_blocksize); + return -EINVAL; + } + if (sbi->s_blocks_per_group > sb->s_blocksize * 8) { + ext4_msg(sb, KERN_ERR, + "#blocks per group too big: %lu", + sbi->s_blocks_per_group); + return -EINVAL; + } + sbi->s_cluster_bits = 0; + } + sbi->s_clusters_per_group = le32_to_cpu(es->s_clusters_per_group); + if (sbi->s_clusters_per_group > sb->s_blocksize * 8 || + sbi->s_clusters_per_group & 7) { + ext4_msg(sb, KERN_ERR, "invalid #clusters per group: %lu", + sbi->s_clusters_per_group); + return -EINVAL; + } + if (sbi->s_blocks_per_group != + (sbi->s_clusters_per_group * (clustersize / sb->s_blocksize))) { + ext4_msg(sb, KERN_ERR, + "blocks per group (%lu) and clusters per group (%lu) inconsistent", + sbi->s_blocks_per_group, sbi->s_clusters_per_group); + return -EINVAL; + } + sbi->s_cluster_ratio = clustersize / sb->s_blocksize; + + /* Do we have standard group size of clustersize * 8 blocks ? */ + if (sbi->s_blocks_per_group == clustersize << 3) + set_opt2(sb, STD_GROUP_SIZE); + + return 0; +} + +/* + * ext4_atomic_write_init: Initializes filesystem min & max atomic write units. + * With non-bigalloc filesystem awu will be based upon filesystem blocksize + * & bdev awu units. + * With bigalloc it will be based upon bigalloc cluster size & bdev awu units. + * @sb: super block + */ +static void ext4_atomic_write_init(struct super_block *sb) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct block_device *bdev = sb->s_bdev; + unsigned int clustersize = EXT4_CLUSTER_SIZE(sb); + + if (!bdev_can_atomic_write(bdev)) + return; + + if (!ext4_has_feature_extents(sb)) + return; + + sbi->s_awu_min = max(sb->s_blocksize, + bdev_atomic_write_unit_min_bytes(bdev)); + sbi->s_awu_max = min(clustersize, + bdev_atomic_write_unit_max_bytes(bdev)); + if (sbi->s_awu_min && sbi->s_awu_max && + sbi->s_awu_min <= sbi->s_awu_max) { + ext4_msg(sb, KERN_NOTICE, "Supports (experimental) DIO atomic writes awu_min: %u, awu_max: %u", + sbi->s_awu_min, sbi->s_awu_max); + } else { + sbi->s_awu_min = 0; + sbi->s_awu_max = 0; + } +} + +static void ext4_fast_commit_init(struct super_block *sb) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + + /* Initialize fast commit stuff */ + atomic_set(&sbi->s_fc_subtid, 0); + INIT_LIST_HEAD(&sbi->s_fc_q[FC_Q_MAIN]); + INIT_LIST_HEAD(&sbi->s_fc_q[FC_Q_STAGING]); + INIT_LIST_HEAD(&sbi->s_fc_dentry_q[FC_Q_MAIN]); + INIT_LIST_HEAD(&sbi->s_fc_dentry_q[FC_Q_STAGING]); + sbi->s_fc_bytes = 0; + ext4_clear_mount_flag(sb, EXT4_MF_FC_INELIGIBLE); + sbi->s_fc_ineligible_tid = 0; + mutex_init(&sbi->s_fc_lock); + memset(&sbi->s_fc_stats, 0, sizeof(sbi->s_fc_stats)); + memset(&sbi->s_fc_snap_stats, 0, sizeof(sbi->s_fc_snap_stats)); + sbi->s_fc_replay_state.fc_regions = NULL; + sbi->s_fc_replay_state.fc_regions_size = 0; + sbi->s_fc_replay_state.fc_regions_used = 0; + sbi->s_fc_replay_state.fc_regions_valid = 0; + sbi->s_fc_replay_state.fc_modified_inodes = NULL; + sbi->s_fc_replay_state.fc_modified_inodes_size = 0; + sbi->s_fc_replay_state.fc_modified_inodes_used = 0; +} + +static int ext4_inode_info_init(struct super_block *sb, + struct ext4_super_block *es) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + + if (le32_to_cpu(es->s_rev_level) == EXT4_GOOD_OLD_REV) { + sbi->s_inode_size = EXT4_GOOD_OLD_INODE_SIZE; + sbi->s_first_ino = EXT4_GOOD_OLD_FIRST_INO; + } else { + sbi->s_inode_size = le16_to_cpu(es->s_inode_size); + sbi->s_first_ino = le32_to_cpu(es->s_first_ino); + if (sbi->s_first_ino < EXT4_GOOD_OLD_FIRST_INO) { + ext4_msg(sb, KERN_ERR, "invalid first ino: %u", + sbi->s_first_ino); + return -EINVAL; + } + if ((sbi->s_inode_size < EXT4_GOOD_OLD_INODE_SIZE) || + (!is_power_of_2(sbi->s_inode_size)) || + (sbi->s_inode_size > sb->s_blocksize)) { + ext4_msg(sb, KERN_ERR, + "unsupported inode size: %d", + sbi->s_inode_size); + ext4_msg(sb, KERN_ERR, "blocksize: %lu", sb->s_blocksize); + return -EINVAL; + } + /* + * i_atime_extra is the last extra field available for + * [acm]times in struct ext4_inode. Checking for that + * field should suffice to ensure we have extra space + * for all three. + */ + if (sbi->s_inode_size >= offsetof(struct ext4_inode, i_atime_extra) + + sizeof(((struct ext4_inode *)0)->i_atime_extra)) { + sb->s_time_gran = 1; + sb->s_time_max = EXT4_EXTRA_TIMESTAMP_MAX; + } else { + sb->s_time_gran = NSEC_PER_SEC; + sb->s_time_max = EXT4_NON_EXTRA_TIMESTAMP_MAX; + } + sb->s_time_min = EXT4_TIMESTAMP_MIN; + } + + if (sbi->s_inode_size > EXT4_GOOD_OLD_INODE_SIZE) { + sbi->s_want_extra_isize = sizeof(struct ext4_inode) - + EXT4_GOOD_OLD_INODE_SIZE; + if (ext4_has_feature_extra_isize(sb)) { + unsigned v, max = (sbi->s_inode_size - + EXT4_GOOD_OLD_INODE_SIZE); + + v = le16_to_cpu(es->s_want_extra_isize); + if (v > max) { + ext4_msg(sb, KERN_ERR, + "bad s_want_extra_isize: %d", v); + return -EINVAL; + } + if (sbi->s_want_extra_isize < v) + sbi->s_want_extra_isize = v; + + v = le16_to_cpu(es->s_min_extra_isize); + if (v > max) { + ext4_msg(sb, KERN_ERR, + "bad s_min_extra_isize: %d", v); + return -EINVAL; + } + if (sbi->s_want_extra_isize < v) + sbi->s_want_extra_isize = v; + } + } + + return 0; +} + +#if IS_ENABLED(CONFIG_UNICODE) +static int ext4_encoding_init(struct super_block *sb, struct ext4_super_block *es) +{ + const struct ext4_sb_encodings *encoding_info; + struct unicode_map *encoding; + __u16 encoding_flags = le16_to_cpu(es->s_encoding_flags); + + if (!ext4_has_feature_casefold(sb) || sb->s_encoding) + return 0; + + encoding_info = ext4_sb_read_encoding(es); + if (!encoding_info) { + ext4_msg(sb, KERN_ERR, + "Encoding requested by superblock is unknown"); + return -EINVAL; + } + + encoding = utf8_load(encoding_info->version); + if (IS_ERR(encoding)) { + ext4_msg(sb, KERN_ERR, + "can't mount with superblock charset: %s-%u.%u.%u " + "not supported by the kernel. flags: 0x%x.", + encoding_info->name, + unicode_major(encoding_info->version), + unicode_minor(encoding_info->version), + unicode_rev(encoding_info->version), + encoding_flags); + return -EINVAL; + } + ext4_msg(sb, KERN_INFO,"Using encoding defined by superblock: " + "%s-%u.%u.%u with flags 0x%hx", encoding_info->name, + unicode_major(encoding_info->version), + unicode_minor(encoding_info->version), + unicode_rev(encoding_info->version), + encoding_flags); + + sb->s_encoding = encoding; + sb->s_encoding_flags = encoding_flags; + + return 0; +} +#else +static inline int ext4_encoding_init(struct super_block *sb, struct ext4_super_block *es) +{ + return 0; +} +#endif + +static int ext4_init_metadata_csum(struct super_block *sb, struct ext4_super_block *es) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + + /* Warn if metadata_csum and gdt_csum are both set. */ + if (ext4_has_feature_metadata_csum(sb) && + ext4_has_feature_gdt_csum(sb)) + ext4_warning(sb, "metadata_csum and uninit_bg are " + "redundant flags; please run fsck."); + + /* Check for a known checksum algorithm */ + if (!ext4_verify_csum_type(sb, es)) { + ext4_msg(sb, KERN_ERR, "VFS: Found ext4 filesystem with " + "unknown checksum algorithm."); + return -EINVAL; + } + ext4_setup_csum_trigger(sb, EXT4_JTR_ORPHAN_FILE, + ext4_orphan_file_block_trigger); + + /* Check superblock checksum */ + if (!ext4_superblock_csum_verify(sb, es)) { + ext4_msg(sb, KERN_ERR, "VFS: Found ext4 filesystem with " + "invalid superblock checksum. Run e2fsck?"); + return -EFSBADCRC; + } + + /* Precompute checksum seed for all metadata */ + if (ext4_has_feature_csum_seed(sb)) + sbi->s_csum_seed = le32_to_cpu(es->s_checksum_seed); + else if (ext4_has_feature_metadata_csum(sb) || + ext4_has_feature_ea_inode(sb)) + sbi->s_csum_seed = ext4_chksum(~0, es->s_uuid, + sizeof(es->s_uuid)); + return 0; +} + +static int ext4_check_feature_compatibility(struct super_block *sb, + struct ext4_super_block *es, + int silent) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + + if (le32_to_cpu(es->s_rev_level) == EXT4_GOOD_OLD_REV && + (ext4_has_compat_features(sb) || + ext4_has_ro_compat_features(sb) || + ext4_has_incompat_features(sb))) + ext4_msg(sb, KERN_WARNING, + "feature flags set on rev 0 fs, " + "running e2fsck is recommended"); + + if (es->s_creator_os == cpu_to_le32(EXT4_OS_HURD)) { + set_opt2(sb, HURD_COMPAT); + if (ext4_has_feature_64bit(sb)) { + ext4_msg(sb, KERN_ERR, + "The Hurd can't support 64-bit file systems"); + return -EINVAL; + } + + /* + * ea_inode feature uses l_i_version field which is not + * available in HURD_COMPAT mode. + */ + if (ext4_has_feature_ea_inode(sb)) { + ext4_msg(sb, KERN_ERR, + "ea_inode feature is not supported for Hurd"); + return -EINVAL; + } + } + + if (IS_EXT2_SB(sb)) { + if (ext2_feature_set_ok(sb)) + ext4_msg(sb, KERN_INFO, "mounting ext2 file system " + "using the ext4 subsystem"); + else { + /* + * If we're probing be silent, if this looks like + * it's actually an ext[34] filesystem. + */ + if (silent && ext4_feature_set_ok(sb, sb_rdonly(sb))) + return -EINVAL; + ext4_msg(sb, KERN_ERR, "couldn't mount as ext2 due " + "to feature incompatibilities"); + return -EINVAL; + } + } + + if (IS_EXT3_SB(sb)) { + if (ext3_feature_set_ok(sb)) + ext4_msg(sb, KERN_INFO, "mounting ext3 file system " + "using the ext4 subsystem"); + else { + /* + * If we're probing be silent, if this looks like + * it's actually an ext4 filesystem. + */ + if (silent && ext4_feature_set_ok(sb, sb_rdonly(sb))) + return -EINVAL; + ext4_msg(sb, KERN_ERR, "couldn't mount as ext3 due " + "to feature incompatibilities"); + return -EINVAL; + } + } + + /* + * Check feature flags regardless of the revision level, since we + * previously didn't change the revision level when setting the flags, + * so there is a chance incompat flags are set on a rev 0 filesystem. + */ + if (!ext4_feature_set_ok(sb, (sb_rdonly(sb)))) + return -EINVAL; + + if (sbi->s_daxdev) { + if (sb->s_blocksize == PAGE_SIZE) + set_bit(EXT4_FLAGS_BDEV_IS_DAX, &sbi->s_ext4_flags); + else + ext4_msg(sb, KERN_ERR, "unsupported blocksize for DAX\n"); + } + + if (sbi->s_mount_opt & EXT4_MOUNT_DAX_ALWAYS) { + if (ext4_has_feature_inline_data(sb)) { + ext4_msg(sb, KERN_ERR, "Cannot use DAX on a filesystem" + " that may contain inline data"); + return -EINVAL; + } + if (!test_bit(EXT4_FLAGS_BDEV_IS_DAX, &sbi->s_ext4_flags)) { + ext4_msg(sb, KERN_ERR, + "DAX unsupported by block device."); + return -EINVAL; + } + } + + if (ext4_has_feature_encrypt(sb) && es->s_encryption_level) { + ext4_msg(sb, KERN_ERR, "Unsupported encryption level %d", + es->s_encryption_level); + return -EINVAL; + } + + return 0; +} + +static int ext4_check_geometry(struct super_block *sb, + struct ext4_super_block *es) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + __u64 blocks_count; + int err; + + if (le16_to_cpu(sbi->s_es->s_reserved_gdt_blocks) > (sb->s_blocksize / 4)) { + ext4_msg(sb, KERN_ERR, + "Number of reserved GDT blocks insanely large: %d", + le16_to_cpu(sbi->s_es->s_reserved_gdt_blocks)); + return -EINVAL; + } + /* + * Test whether we have more sectors than will fit in sector_t, + * and whether the max offset is addressable by the page cache. + */ + err = generic_check_addressable(sb->s_blocksize_bits, + ext4_blocks_count(es)); + if (err) { + ext4_msg(sb, KERN_ERR, "filesystem" + " too large to mount safely on this system"); + return err; + } + + /* check blocks count against device size */ + blocks_count = sb_bdev_nr_blocks(sb); + if (blocks_count && ext4_blocks_count(es) > blocks_count) { + ext4_msg(sb, KERN_WARNING, "bad geometry: block count %llu " + "exceeds size of device (%llu blocks)", + ext4_blocks_count(es), blocks_count); + return -EINVAL; + } + + /* + * It makes no sense for the first data block to be beyond the end + * of the filesystem. + */ + if (le32_to_cpu(es->s_first_data_block) >= ext4_blocks_count(es)) { + ext4_msg(sb, KERN_WARNING, "bad geometry: first data " + "block %u is beyond end of filesystem (%llu)", + le32_to_cpu(es->s_first_data_block), + ext4_blocks_count(es)); + return -EINVAL; + } + if ((es->s_first_data_block == 0) && (es->s_log_block_size == 0) && + (sbi->s_cluster_ratio == 1)) { + ext4_msg(sb, KERN_WARNING, "bad geometry: first data " + "block is 0 with a 1k block and cluster size"); + return -EINVAL; + } + + blocks_count = (ext4_blocks_count(es) - + le32_to_cpu(es->s_first_data_block) + + EXT4_BLOCKS_PER_GROUP(sb) - 1); + do_div(blocks_count, EXT4_BLOCKS_PER_GROUP(sb)); + if (blocks_count > ((uint64_t)1<<32) - EXT4_DESC_PER_BLOCK(sb)) { + ext4_msg(sb, KERN_WARNING, "groups count too large: %llu " + "(block count %llu, first data block %u, " + "blocks per group %lu)", blocks_count, + ext4_blocks_count(es), + le32_to_cpu(es->s_first_data_block), + EXT4_BLOCKS_PER_GROUP(sb)); + return -EINVAL; + } + sbi->s_groups_count = blocks_count; + sbi->s_blockfile_groups = min(sbi->s_groups_count, + (EXT4_MAX_BLOCK_FILE_PHYS / EXT4_BLOCKS_PER_GROUP(sb))); + if (((u64)sbi->s_groups_count * sbi->s_inodes_per_group) != + le32_to_cpu(es->s_inodes_count)) { + ext4_msg(sb, KERN_ERR, "inodes count not valid: %u vs %llu", + le32_to_cpu(es->s_inodes_count), + ((u64)sbi->s_groups_count * sbi->s_inodes_per_group)); + return -EINVAL; + } + + return 0; +} + +static int ext4_group_desc_init(struct super_block *sb, + struct ext4_super_block *es, + ext4_fsblk_t logical_sb_block, + ext4_group_t *first_not_zeroed) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + unsigned int db_count; + ext4_fsblk_t block; + int i; + + db_count = (sbi->s_groups_count + EXT4_DESC_PER_BLOCK(sb) - 1) / + EXT4_DESC_PER_BLOCK(sb); + if (ext4_has_feature_meta_bg(sb)) { + if (le32_to_cpu(es->s_first_meta_bg) > db_count) { + ext4_msg(sb, KERN_WARNING, + "first meta block group too large: %u " + "(group descriptor block count %u)", + le32_to_cpu(es->s_first_meta_bg), db_count); + return -EINVAL; + } + } + rcu_assign_pointer(sbi->s_group_desc, + kvmalloc_objs(struct buffer_head *, db_count)); + if (sbi->s_group_desc == NULL) { + ext4_msg(sb, KERN_ERR, "not enough memory"); + return -ENOMEM; + } + + bgl_lock_init(sbi->s_blockgroup_lock); + + /* Pre-read the descriptors into the buffer cache */ + for (i = 0; i < db_count; i++) { + block = descriptor_loc(sb, logical_sb_block, i); + ext4_sb_breadahead_unmovable(sb, block); + } + + for (i = 0; i < db_count; i++) { + struct buffer_head *bh; + + block = descriptor_loc(sb, logical_sb_block, i); + bh = ext4_sb_bread_unmovable(sb, block); + if (IS_ERR(bh)) { + ext4_msg(sb, KERN_ERR, + "can't read group descriptor %d", i); + sbi->s_gdb_count = i; + return PTR_ERR(bh); + } + rcu_read_lock(); + rcu_dereference(sbi->s_group_desc)[i] = bh; + rcu_read_unlock(); + } + sbi->s_gdb_count = db_count; + if (!ext4_check_descriptors(sb, logical_sb_block, first_not_zeroed)) { + ext4_msg(sb, KERN_ERR, "group descriptors corrupted!"); + return -EFSCORRUPTED; + } + + return 0; +} + +static int ext4_load_and_init_journal(struct super_block *sb, + struct ext4_super_block *es, + struct ext4_fs_context *ctx) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + int err; + + err = ext4_load_journal(sb, es, ctx->journal_devnum); + if (err) + return err; + + if (ext4_has_feature_64bit(sb) && + !jbd2_journal_set_features(EXT4_SB(sb)->s_journal, 0, 0, + JBD2_FEATURE_INCOMPAT_64BIT)) { + ext4_msg(sb, KERN_ERR, "Failed to set 64-bit journal feature"); + goto out; + } + + if (!set_journal_csum_feature_set(sb)) { + ext4_msg(sb, KERN_ERR, "Failed to set journal checksum " + "feature set"); + goto out; + } + + if (test_opt2(sb, JOURNAL_FAST_COMMIT) && + !jbd2_journal_set_features(EXT4_SB(sb)->s_journal, 0, 0, + JBD2_FEATURE_INCOMPAT_FAST_COMMIT)) { + ext4_msg(sb, KERN_ERR, + "Failed to set fast commit journal feature"); + goto out; + } + + /* We have now updated the journal if required, so we can + * validate the data journaling mode. */ + switch (test_opt(sb, DATA_FLAGS)) { + case 0: + /* No mode set, assume a default based on the journal + * capabilities: ORDERED_DATA if the journal can + * cope, else JOURNAL_DATA + */ + if (jbd2_journal_check_available_features + (sbi->s_journal, 0, 0, JBD2_FEATURE_INCOMPAT_REVOKE)) { + set_opt(sb, ORDERED_DATA); + sbi->s_def_mount_opt |= EXT4_MOUNT_ORDERED_DATA; + } else { + set_opt(sb, JOURNAL_DATA); + sbi->s_def_mount_opt |= EXT4_MOUNT_JOURNAL_DATA; + } + break; + + case EXT4_MOUNT_ORDERED_DATA: + case EXT4_MOUNT_WRITEBACK_DATA: + if (!jbd2_journal_check_available_features + (sbi->s_journal, 0, 0, JBD2_FEATURE_INCOMPAT_REVOKE)) { + ext4_msg(sb, KERN_ERR, "Journal does not support " + "requested data journaling mode"); + goto out; + } + break; + default: + break; + } + + if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_ORDERED_DATA && + test_opt(sb, JOURNAL_ASYNC_COMMIT)) { + ext4_msg(sb, KERN_ERR, "can't mount with " + "journal_async_commit in data=ordered mode"); + goto out; + } + + set_task_ioprio(sbi->s_journal->j_task, ctx->journal_ioprio); + + sbi->s_journal->j_submit_inode_data_buffers = + ext4_journal_submit_inode_data_buffers; + sbi->s_journal->j_finish_inode_data_buffers = + ext4_journal_finish_inode_data_buffers; + + return 0; + +out: + ext4_journal_destroy(sbi, sbi->s_journal); + return -EINVAL; +} + +static int ext4_check_journal_data_mode(struct super_block *sb) +{ + if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_JOURNAL_DATA) { + printk_once(KERN_WARNING "EXT4-fs: Warning: mounting with " + "data=journal disables delayed allocation, " + "dioread_nolock, O_DIRECT and fast_commit support!\n"); + /* can't mount with both data=journal and dioread_nolock. */ + clear_opt(sb, DIOREAD_NOLOCK); + clear_opt2(sb, JOURNAL_FAST_COMMIT); + if (test_opt2(sb, EXPLICIT_DELALLOC)) { + ext4_msg(sb, KERN_ERR, "can't mount with " + "both data=journal and delalloc"); + return -EINVAL; + } + if (test_opt(sb, DAX_ALWAYS)) { + ext4_msg(sb, KERN_ERR, "can't mount with " + "both data=journal and dax"); + return -EINVAL; + } + if (ext4_has_feature_encrypt(sb)) { + ext4_msg(sb, KERN_WARNING, + "encrypted files will use data=ordered " + "instead of data journaling mode"); + } + if (test_opt(sb, DELALLOC)) + clear_opt(sb, DELALLOC); + } else { + sb->s_iflags |= SB_I_CGROUPWB; + } + + return 0; +} + +static const char *ext4_has_journal_option(struct super_block *sb) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + + if (test_opt(sb, JOURNAL_ASYNC_COMMIT)) + return "journal_async_commit"; + if (test_opt2(sb, EXPLICIT_JOURNAL_CHECKSUM)) + return "journal_checksum"; + if (sbi->s_commit_interval != JBD2_DEFAULT_MAX_COMMIT_AGE*HZ) + return "commit="; + if (EXT4_MOUNT_DATA_FLAGS & + (sbi->s_mount_opt ^ sbi->s_def_mount_opt)) + return "data="; + if (test_opt(sb, DATA_ERR_ABORT)) + return "data_err=abort"; + return NULL; +} + +/* + * Limit the maximum folio order to 2048 blocks to prevent overestimation + * of reserve handle credits during the folio writeback in environments + * where the PAGE_SIZE exceeds 4KB. + */ +#define EXT4_MAX_PAGECACHE_ORDER(sb) \ + umin(MAX_PAGECACHE_ORDER, (11 + (sb)->s_blocksize_bits - PAGE_SHIFT)) +static void ext4_set_max_mapping_order(struct super_block *sb) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + + if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_JOURNAL_DATA) + sbi->s_max_folio_order = sbi->s_min_folio_order; + else + sbi->s_max_folio_order = EXT4_MAX_PAGECACHE_ORDER(sb); +} + +static int ext4_check_large_folio(struct super_block *sb) +{ + const char *err_str = NULL; + + if (ext4_has_feature_encrypt(sb)) + err_str = "encrypt"; + + if (!err_str) { + ext4_set_max_mapping_order(sb); + } else if (sb->s_blocksize > PAGE_SIZE) { + ext4_msg(sb, KERN_ERR, "bs(%lu) > ps(%lu) unsupported for %s", + sb->s_blocksize, PAGE_SIZE, err_str); + return -EINVAL; + } + + return 0; +} + +static int ext4_load_super(struct super_block *sb, ext4_fsblk_t *lsb, + int silent) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_super_block *es; + ext4_fsblk_t logical_sb_block; + unsigned long offset = 0; + struct buffer_head *bh; + int ret = -EINVAL; + int blocksize; + + blocksize = sb_min_blocksize(sb, EXT4_MIN_BLOCK_SIZE); + if (!blocksize) { + ext4_msg(sb, KERN_ERR, "unable to set blocksize"); + return -EINVAL; + } + + /* + * The ext4 superblock will not be buffer aligned for other than 1kB + * block sizes. We need to calculate the offset from buffer start. + */ + if (blocksize != EXT4_MIN_BLOCK_SIZE) { + logical_sb_block = sbi->s_sb_block * EXT4_MIN_BLOCK_SIZE; + offset = do_div(logical_sb_block, blocksize); + } else { + logical_sb_block = sbi->s_sb_block; + } + + bh = ext4_sb_bread_unmovable(sb, logical_sb_block); + if (IS_ERR(bh)) { + ext4_msg(sb, KERN_ERR, "unable to read superblock"); + return PTR_ERR(bh); + } + /* + * Note: s_es must be initialized as soon as possible because + * some ext4 macro-instructions depend on its value + */ + es = (struct ext4_super_block *) (bh->b_data + offset); + sbi->s_es = es; + sb->s_magic = le16_to_cpu(es->s_magic); + if (sb->s_magic != EXT4_SUPER_MAGIC) { + if (!silent) + ext4_msg(sb, KERN_ERR, "VFS: Can't find ext4 filesystem"); + goto out; + } + + if (le32_to_cpu(es->s_log_block_size) > + (EXT4_MAX_BLOCK_LOG_SIZE - EXT4_MIN_BLOCK_LOG_SIZE)) { + ext4_msg(sb, KERN_ERR, + "Invalid log block size: %u", + le32_to_cpu(es->s_log_block_size)); + goto out; + } + if (le32_to_cpu(es->s_log_cluster_size) > + (EXT4_MAX_CLUSTER_LOG_SIZE - EXT4_MIN_BLOCK_LOG_SIZE)) { + ext4_msg(sb, KERN_ERR, + "Invalid log cluster size: %u", + le32_to_cpu(es->s_log_cluster_size)); + goto out; + } + + blocksize = EXT4_MIN_BLOCK_SIZE << le32_to_cpu(es->s_log_block_size); + + /* + * If the default block size is not the same as the real block size, + * we need to reload it. + */ + if (sb->s_blocksize == blocksize) + goto success; + + /* + * bh must be released before kill_bdev(), otherwise + * it won't be freed and its page also. kill_bdev() + * is called by sb_set_blocksize(). + */ + brelse(bh); + /* Validate the filesystem blocksize */ + if (!sb_set_blocksize(sb, blocksize)) { + ext4_msg(sb, KERN_ERR, "bad block size %d", + blocksize); + bh = NULL; + goto out; + } + + logical_sb_block = sbi->s_sb_block * EXT4_MIN_BLOCK_SIZE; + offset = do_div(logical_sb_block, blocksize); + bh = ext4_sb_bread_unmovable(sb, logical_sb_block); + if (IS_ERR(bh)) { + ext4_msg(sb, KERN_ERR, "Can't read superblock on 2nd try"); + ret = PTR_ERR(bh); + bh = NULL; + goto out; + } + es = (struct ext4_super_block *)(bh->b_data + offset); + sbi->s_es = es; + if (es->s_magic != cpu_to_le16(EXT4_SUPER_MAGIC)) { + ext4_msg(sb, KERN_ERR, "Magic mismatch, very weird!"); + goto out; + } + +success: + sbi->s_min_folio_order = get_order(blocksize); + *lsb = logical_sb_block; + sbi->s_sbh = bh; + return 0; +out: + brelse(bh); + return ret; +} + +static int ext4_hash_info_init(struct super_block *sb) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_super_block *es = sbi->s_es; + unsigned int i; + + sbi->s_def_hash_version = es->s_def_hash_version; + + if (sbi->s_def_hash_version > DX_HASH_LAST) { + ext4_msg(sb, KERN_ERR, + "Invalid default hash set in the superblock"); + return -EINVAL; + } else if (sbi->s_def_hash_version == DX_HASH_SIPHASH) { + ext4_msg(sb, KERN_ERR, + "SIPHASH is not a valid default hash value"); + return -EINVAL; + } + + for (i = 0; i < 4; i++) + sbi->s_hash_seed[i] = le32_to_cpu(es->s_hash_seed[i]); + + if (ext4_has_feature_dir_index(sb)) { + i = le32_to_cpu(es->s_flags); + if (i & EXT2_FLAGS_UNSIGNED_HASH) + sbi->s_hash_unsigned = 3; + else if ((i & EXT2_FLAGS_SIGNED_HASH) == 0) { +#ifdef __CHAR_UNSIGNED__ + if (!sb_rdonly(sb)) + es->s_flags |= + cpu_to_le32(EXT2_FLAGS_UNSIGNED_HASH); + sbi->s_hash_unsigned = 3; +#else + if (!sb_rdonly(sb)) + es->s_flags |= + cpu_to_le32(EXT2_FLAGS_SIGNED_HASH); +#endif + } + } + return 0; +} + +static int ext4_block_group_meta_init(struct super_block *sb, int silent) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_super_block *es = sbi->s_es; + int has_huge_files; + + has_huge_files = ext4_has_feature_huge_file(sb); + sbi->s_bitmap_maxbytes = ext4_max_bitmap_size(sb->s_blocksize_bits, + has_huge_files); + sb->s_maxbytes = ext4_max_size(sb->s_blocksize_bits, has_huge_files); + + sbi->s_desc_size = le16_to_cpu(es->s_desc_size); + if (ext4_has_feature_64bit(sb)) { + if (sbi->s_desc_size < EXT4_MIN_DESC_SIZE_64BIT || + sbi->s_desc_size > EXT4_MAX_DESC_SIZE || + !is_power_of_2(sbi->s_desc_size)) { + ext4_msg(sb, KERN_ERR, + "unsupported descriptor size %lu", + sbi->s_desc_size); + return -EINVAL; + } + } else + sbi->s_desc_size = EXT4_MIN_DESC_SIZE; + + sbi->s_blocks_per_group = le32_to_cpu(es->s_blocks_per_group); + sbi->s_inodes_per_group = le32_to_cpu(es->s_inodes_per_group); + + sbi->s_inodes_per_block = sb->s_blocksize / EXT4_INODE_SIZE(sb); + if (sbi->s_inodes_per_block == 0 || sbi->s_blocks_per_group == 0) { + if (!silent) + ext4_msg(sb, KERN_ERR, "VFS: Can't find ext4 filesystem"); + return -EINVAL; + } + if (sbi->s_inodes_per_group < sbi->s_inodes_per_block || + sbi->s_inodes_per_group > sb->s_blocksize * 8 || + sbi->s_inodes_per_group & 7 || + sbi->s_inodes_per_group % sbi->s_inodes_per_block) { + ext4_msg(sb, KERN_ERR, "invalid inodes per group: %lu", + sbi->s_inodes_per_group); + return -EINVAL; + } + sbi->s_itb_per_group = sbi->s_inodes_per_group / + sbi->s_inodes_per_block; + sbi->s_desc_per_block = sb->s_blocksize / EXT4_DESC_SIZE(sb); + sbi->s_mount_state = le16_to_cpu(es->s_state) & ~EXT4_FC_REPLAY; + sbi->s_addr_per_block_bits = ilog2(EXT4_ADDR_PER_BLOCK(sb)); + sbi->s_desc_per_block_bits = ilog2(EXT4_DESC_PER_BLOCK(sb)); + + return 0; +} + +/* + * It's hard to get stripe aligned blocks if stripe is not aligned with + * cluster, just disable stripe and alert user to simplify code and avoid + * stripe aligned allocation which will rarely succeed. + */ +static bool ext4_is_stripe_incompatible(struct super_block *sb, unsigned long stripe) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + return (stripe > 0 && sbi->s_cluster_ratio > 1 && + stripe % sbi->s_cluster_ratio != 0); +} + +static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) +{ + struct ext4_super_block *es = NULL; + struct ext4_sb_info *sbi = EXT4_SB(sb); + ext4_fsblk_t logical_sb_block; + struct inode *root; + int needs_recovery; + int err; + ext4_group_t first_not_zeroed; + struct ext4_fs_context *ctx = fc->fs_private; + int silent = fc->sb_flags & SB_SILENT; + + /* Set defaults for the variables that will be set during parsing */ + if (!(ctx->spec & EXT4_SPEC_JOURNAL_IOPRIO)) + ctx->journal_ioprio = EXT4_DEF_JOURNAL_IOPRIO; + + sbi->s_inode_readahead_blks = EXT4_DEF_INODE_READAHEAD_BLKS; + sbi->s_sectors_written_start = + part_stat_read(sb->s_bdev, sectors[STAT_WRITE]); + + err = ext4_load_super(sb, &logical_sb_block, silent); + if (err) + goto out_fail; + + es = sbi->s_es; + sbi->s_kbytes_written = le64_to_cpu(es->s_kbytes_written); + + err = ext4_init_metadata_csum(sb, es); + if (err) + goto failed_mount; + + ext4_set_def_opts(sb, es); + + sbi->s_resuid = make_kuid(&init_user_ns, ext4_get_resuid(es)); + sbi->s_resgid = make_kgid(&init_user_ns, ext4_get_resgid(es)); + sbi->s_commit_interval = JBD2_DEFAULT_MAX_COMMIT_AGE * HZ; + sbi->s_min_batch_time = EXT4_DEF_MIN_BATCH_TIME; + sbi->s_max_batch_time = EXT4_DEF_MAX_BATCH_TIME; + sbi->s_sb_update_kb = EXT4_DEF_SB_UPDATE_INTERVAL_KB; + sbi->s_sb_update_sec = EXT4_DEF_SB_UPDATE_INTERVAL_SEC; + + /* + * set default s_li_wait_mult for lazyinit, for the case there is + * no mount option specified. + */ + sbi->s_li_wait_mult = EXT4_DEF_LI_WAIT_MULT; + + err = ext4_inode_info_init(sb, es); + if (err) + goto failed_mount; + + err = parse_apply_sb_mount_options(sb, ctx); + if (err < 0) + goto failed_mount; + + sbi->s_def_mount_opt = sbi->s_mount_opt; + sbi->s_def_mount_opt2 = sbi->s_mount_opt2; + + err = ext4_check_opt_consistency(fc, sb); + if (err < 0) + goto failed_mount; + + ext4_apply_options(fc, sb); + + err = ext4_check_large_folio(sb); + if (err < 0) + goto failed_mount; + + err = ext4_encoding_init(sb, es); + if (err) + goto failed_mount; + + err = ext4_check_journal_data_mode(sb); + if (err) + goto failed_mount; + + sb->s_flags = (sb->s_flags & ~SB_POSIXACL) | + (test_opt(sb, POSIX_ACL) ? SB_POSIXACL : 0); + + /* HSM events are allowed by default. */ + sb->s_iflags |= SB_I_ALLOW_HSM; + + err = ext4_check_feature_compatibility(sb, es, silent); + if (err) + goto failed_mount; + + err = ext4_block_group_meta_init(sb, silent); + if (err) + goto failed_mount; + + err = ext4_hash_info_init(sb); + if (err) + goto failed_mount; + + err = ext4_handle_clustersize(sb); + if (err) + goto failed_mount; + + err = ext4_check_geometry(sb, es); + if (err) + goto failed_mount; + + timer_setup(&sbi->s_err_report, print_daily_error_info, 0); + spin_lock_init(&sbi->s_error_lock); + mutex_init(&sbi->s_error_notify_mutex); + INIT_WORK(&sbi->s_sb_upd_work, update_super_work); + + err = ext4_group_desc_init(sb, es, logical_sb_block, &first_not_zeroed); + if (err) + goto failed_mount3; + + err = ext4_es_register_shrinker(sbi); + if (err) + goto failed_mount3; + + sbi->s_stripe = ext4_get_stripe_size(sbi); + if (ext4_is_stripe_incompatible(sb, sbi->s_stripe)) { + ext4_msg(sb, KERN_WARNING, + "stripe (%lu) is not aligned with cluster size (%u), " + "stripe is disabled", + sbi->s_stripe, sbi->s_cluster_ratio); + sbi->s_stripe = 0; + } + sbi->s_extent_max_zeroout_kb = 32; + + /* + * set up enough so that it can read an inode + */ + sb->s_op = &ext4_sops; + sb->s_export_op = &ext4_export_ops; + sb->s_xattr = ext4_xattr_handlers; +#ifdef CONFIG_FS_ENCRYPTION + sb->s_cop = &ext4_cryptops; +#endif +#ifdef CONFIG_FS_VERITY + sb->s_vop = &ext4_verityops; +#endif +#ifdef CONFIG_QUOTA + sb->dq_op = &ext4_quota_operations; + if (ext4_has_feature_quota(sb)) + sb->s_qcop = &dquot_quotactl_sysfile_ops; + else + sb->s_qcop = &ext4_qctl_operations; + sb->s_quota_types = QTYPE_MASK_USR | QTYPE_MASK_GRP | QTYPE_MASK_PRJ; +#endif + super_set_uuid(sb, es->s_uuid, sizeof(es->s_uuid)); + super_set_sysfs_name_bdev(sb); + + INIT_LIST_HEAD(&sbi->s_orphan); /* unlinked but open files */ + mutex_init(&sbi->s_orphan_lock); + + spin_lock_init(&sbi->s_bdev_wb_lock); + + ext4_atomic_write_init(sb); + ext4_fast_commit_init(sb); + + sb->s_root = NULL; + + needs_recovery = (es->s_last_orphan != 0 || + ext4_has_feature_orphan_present(sb) || + ext4_has_feature_journal_needs_recovery(sb)); + + ext4_init_ea_inode_work(sbi); + + if (ext4_has_feature_mmp(sb) && !sb_rdonly(sb)) { + err = ext4_multi_mount_protect(sb, le64_to_cpu(es->s_mmp_block)); + if (err) + goto failed_mount3a; + } + + err = -EINVAL; + /* + * The first inode we look at is the journal inode. Don't try + * root first: it may be modified in the journal! + */ + if (!test_opt(sb, NOLOAD) && ext4_has_feature_journal(sb)) { + err = ext4_load_and_init_journal(sb, es, ctx); + if (err) + goto failed_mount3a; + if (bdev_read_only(sb->s_bdev)) + needs_recovery = 0; + } else if (test_opt(sb, NOLOAD) && !sb_rdonly(sb) && + ext4_has_feature_journal_needs_recovery(sb)) { + ext4_msg(sb, KERN_ERR, "required journal recovery " + "suppressed and not mounted read-only"); + goto failed_mount3a; + } else { + const char *journal_option; + + /* Nojournal mode, all journal mount options are illegal */ + journal_option = ext4_has_journal_option(sb); + if (journal_option != NULL) { + ext4_msg(sb, KERN_ERR, + "can't mount with %s, fs mounted w/o journal", + journal_option); + goto failed_mount3a; + } + + sbi->s_def_mount_opt &= ~EXT4_MOUNT_JOURNAL_CHECKSUM; + clear_opt(sb, JOURNAL_CHECKSUM); + clear_opt(sb, DATA_FLAGS); + clear_opt2(sb, JOURNAL_FAST_COMMIT); + sbi->s_journal = NULL; + needs_recovery = 0; + } + + if (!test_opt(sb, NO_MBCACHE)) { + sbi->s_ea_block_cache = ext4_xattr_create_cache(); + if (!sbi->s_ea_block_cache) { + ext4_msg(sb, KERN_ERR, + "Failed to create ea_block_cache"); + err = -EINVAL; + goto failed_mount_wq; + } + + if (ext4_has_feature_ea_inode(sb)) { + sbi->s_ea_inode_cache = ext4_xattr_create_cache(); + if (!sbi->s_ea_inode_cache) { + ext4_msg(sb, KERN_ERR, + "Failed to create ea_inode_cache"); + err = -EINVAL; + goto failed_mount_wq; + } + } + } + + /* + * Get the # of file system overhead blocks from the + * superblock if present. + */ + sbi->s_overhead = le32_to_cpu(es->s_overhead_clusters); + /* ignore the precalculated value if it is ridiculous */ + if (sbi->s_overhead > ext4_blocks_count(es)) + sbi->s_overhead = 0; + /* + * If the bigalloc feature is not enabled recalculating the + * overhead doesn't take long, so we might as well just redo + * it to make sure we are using the correct value. + */ + if (!ext4_has_feature_bigalloc(sb)) + sbi->s_overhead = 0; + if (sbi->s_overhead == 0) { + err = ext4_calculate_overhead(sb); + if (err) + goto failed_mount_wq; + } + + /* + * The maximum number of concurrent works can be high and + * concurrency isn't really necessary. Limit it to 1. + */ + EXT4_SB(sb)->rsv_conversion_wq = + alloc_workqueue("ext4-rsv-conversion", WQ_MEM_RECLAIM | WQ_UNBOUND, 1); + if (!EXT4_SB(sb)->rsv_conversion_wq) { + printk(KERN_ERR "EXT4-fs: failed to create workqueue\n"); + err = -ENOMEM; + goto failed_mount4; + } + + /* + * The jbd2_journal_load will have done any necessary log recovery, + * so we can safely mount the rest of the filesystem now. + */ + + root = ext4_iget(sb, EXT4_ROOT_INO, EXT4_IGET_SPECIAL); + if (IS_ERR(root)) { + ext4_msg(sb, KERN_ERR, "get root inode failed"); + err = PTR_ERR(root); + root = NULL; + goto failed_mount4; + } + if (!S_ISDIR(root->i_mode) || !root->i_blocks || !root->i_size) { + ext4_msg(sb, KERN_ERR, "corrupt root inode, run e2fsck"); + iput(root); + err = -EFSCORRUPTED; + goto failed_mount4; + } + + generic_set_sb_d_ops(sb); + sb->s_root = d_make_root(root); + if (!sb->s_root) { + ext4_msg(sb, KERN_ERR, "get root dentry failed"); + err = -ENOMEM; + goto failed_mount4; + } + + err = ext4_setup_super(sb, es, sb_rdonly(sb)); + if (err == -EROFS) { + sb->s_flags |= SB_RDONLY; + } else if (err) + goto failed_mount4a; + + ext4_set_resv_clusters(sb); + + if (test_opt(sb, BLOCK_VALIDITY)) { + err = ext4_setup_system_zone(sb); + if (err) { + ext4_msg(sb, KERN_ERR, "failed to initialize system " + "zone (%d)", err); + goto failed_mount4a; + } + } + ext4_fc_replay_cleanup(sb); + + ext4_ext_init(sb); + + /* + * Enable optimize_scan if number of groups is > threshold. This can be + * turned off by passing "mb_optimize_scan=0". This can also be + * turned on forcefully by passing "mb_optimize_scan=1". + */ + if (!(ctx->spec & EXT4_SPEC_mb_optimize_scan)) { + if (sbi->s_groups_count >= MB_DEFAULT_LINEAR_SCAN_THRESHOLD) + set_opt2(sb, MB_OPTIMIZE_SCAN); + else + clear_opt2(sb, MB_OPTIMIZE_SCAN); + } + + err = ext4_percpu_param_init(sbi); + if (err) + goto failed_mount5; + + err = ext4_mb_init(sb); + if (err) { + ext4_msg(sb, KERN_ERR, "failed to initialize mballoc (%d)", + err); + goto failed_mount5; + } + + /* + * We can only set up the journal commit callback once + * mballoc is initialized + */ + if (sbi->s_journal) + sbi->s_journal->j_commit_callback = + ext4_journal_commit_callback; + + if (ext4_has_feature_flex_bg(sb)) + if (!ext4_fill_flex_info(sb)) { + ext4_msg(sb, KERN_ERR, + "unable to initialize " + "flex_bg meta info!"); + err = -ENOMEM; + goto failed_mount6; + } + + err = ext4_register_li_request(sb, first_not_zeroed); + if (err) + goto failed_mount6; + + err = ext4_init_orphan_info(sb); + if (err) + goto failed_mount7; +#ifdef CONFIG_QUOTA + /* Enable quota usage during mount. */ + if (ext4_has_feature_quota(sb) && !sb_rdonly(sb)) { + err = ext4_enable_quotas(sb); + if (err) + goto failed_mount8; + } +#endif /* CONFIG_QUOTA */ + + /* + * Save the original bdev mapping's wb_err value which could be + * used to detect the metadata async write error. + */ + errseq_check_and_advance(&sb->s_bdev->bd_mapping->wb_err, + &sbi->s_bdev_wb_err); + EXT4_SB(sb)->s_mount_state |= EXT4_ORPHAN_FS; + ext4_orphan_cleanup(sb, es); + EXT4_SB(sb)->s_mount_state &= ~EXT4_ORPHAN_FS; + /* + * Update the checksum after updating free space/inode counters and + * ext4_orphan_cleanup. Otherwise the superblock can have an incorrect + * checksum in the buffer cache until it is written out and + * e2fsprogs programs trying to open a file system immediately + * after it is mounted can fail. + */ + ext4_superblock_csum_set(sb); + if (needs_recovery) { + ext4_msg(sb, KERN_INFO, "recovery complete"); + err = ext4_mark_recovery_complete(sb, es); + if (err) + goto failed_mount9; + } + + if (test_opt(sb, DISCARD) && !bdev_max_discard_sectors(sb->s_bdev)) { + ext4_msg(sb, KERN_WARNING, + "mounting with \"discard\" option, but the device does not support discard"); + clear_opt(sb, DISCARD); + } + + if (es->s_error_count) { + sbi->s_err_report_sec = 5*60; /* first time 5 minutes */ + mod_timer(&sbi->s_err_report, + jiffies + secs_to_jiffies(sbi->s_err_report_sec)); + } + sbi->s_err_report_sec = 24*60*60; /* Once a day */ + + /* Enable message ratelimiting. Default is 10 messages per 5 secs. */ + ratelimit_state_init(&sbi->s_err_ratelimit_state, 5 * HZ, 10); + ratelimit_state_init(&sbi->s_warning_ratelimit_state, 5 * HZ, 10); + ratelimit_state_init(&sbi->s_msg_ratelimit_state, 5 * HZ, 10); + atomic_set(&sbi->s_warning_count, 0); + atomic_set(&sbi->s_msg_count, 0); + + /* Register sysfs after all initializations are complete. */ + err = ext4_register_sysfs(sb); + if (err) + goto failed_mount9; + + return 0; + +failed_mount9: + /* Drain deferred EA inode iputs before quota shutdown */ + flush_delayed_work(&sbi->s_ea_inode_work); + ext4_quotas_off(sb, EXT4_MAXQUOTAS); +failed_mount8: __maybe_unused + ext4_release_orphan_info(sb); +failed_mount7: + ext4_unregister_li_request(sb); +failed_mount6: + ext4_mb_release(sb); + ext4_flex_groups_free(sbi); +failed_mount5: + ext4_percpu_param_destroy(sbi); + ext4_ext_release(sb); + ext4_release_system_zone(sb); +failed_mount4a: + dput(sb->s_root); + sb->s_root = NULL; +failed_mount4: + ext4_msg(sb, KERN_ERR, "mount failed"); + if (EXT4_SB(sb)->rsv_conversion_wq) + destroy_workqueue(EXT4_SB(sb)->rsv_conversion_wq); +failed_mount_wq: + /* Drain deferred EA inode iputs before freeing structures */ + flush_delayed_work(&sbi->s_ea_inode_work); + ext4_xattr_destroy_cache(sbi->s_ea_inode_cache); + sbi->s_ea_inode_cache = NULL; + + ext4_xattr_destroy_cache(sbi->s_ea_block_cache); + sbi->s_ea_block_cache = NULL; + + if (sbi->s_journal) { + ext4_journal_destroy(sbi, sbi->s_journal); + } +failed_mount3a: + /* Drain deferred EA inode iputs from journal replay */ + flush_delayed_work(&sbi->s_ea_inode_work); + ext4_es_unregister_shrinker(sbi); +failed_mount3: + /* flush s_sb_upd_work before sbi destroy */ + flush_work(&sbi->s_sb_upd_work); + ext4_stop_mmpd(sbi); + timer_delete_sync(&sbi->s_err_report); + ext4_group_desc_free(sbi); +failed_mount: +#if IS_ENABLED(CONFIG_UNICODE) + utf8_unload(sb->s_encoding); +#endif + +#ifdef CONFIG_QUOTA + for (unsigned int i = 0; i < EXT4_MAXQUOTAS; i++) + kfree(get_qf_name(sb, sbi, i)); +#endif + fscrypt_free_dummy_policy(&sbi->s_dummy_enc_policy); + brelse(sbi->s_sbh); + if (sbi->s_journal_bdev_file) { + invalidate_bdev(file_bdev(sbi->s_journal_bdev_file)); + fs_bdev_file_release(sbi->s_journal_bdev_file, sb); + } +out_fail: + invalidate_bdev(sb->s_bdev); + sb->s_fs_info = NULL; + return err; +} + +static int ext4_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct ext4_fs_context *ctx = fc->fs_private; + struct ext4_sb_info *sbi; + const char *descr; + int ret; + + sbi = ext4_alloc_sbi(sb); + if (!sbi) + return -ENOMEM; + + fc->s_fs_info = sbi; + + /* Cleanup superblock name */ + strreplace(sb->s_id, '/', '!'); + + sbi->s_sb_block = 1; /* Default super block location */ + if (ctx->spec & EXT4_SPEC_s_sb_block) + sbi->s_sb_block = ctx->s_sb_block; + + ret = __ext4_fill_super(fc, sb); + if (ret < 0) + goto free_sbi; + + if (sbi->s_journal) { + if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_JOURNAL_DATA) + descr = " journalled data mode"; + else if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_ORDERED_DATA) + descr = " ordered data mode"; + else + descr = " writeback data mode"; + } else + descr = "out journal"; + + if (___ratelimit(&ext4_mount_msg_ratelimit, "EXT4-fs mount")) + ext4_msg(sb, KERN_INFO, "mounted filesystem %pU %s with%s. " + "Quota mode: %s.", &sb->s_uuid, + sb_rdonly(sb) ? "ro" : "r/w", descr, + ext4_quota_mode(sb)); + + /* Update the s_overhead_clusters if necessary */ + ext4_update_overhead(sb, false); + return 0; + +free_sbi: + ext4_free_sbi(sbi); + fc->s_fs_info = NULL; + return ret; +} + +static int ext4_get_tree(struct fs_context *fc) +{ + return get_tree_bdev(fc, ext4_fill_super); +} + +/* + * Setup any per-fs journal parameters now. We'll do this both on + * initial mount, once the journal has been initialised but before we've + * done any recovery; and again on any subsequent remount. + */ +static void ext4_init_journal_params(struct super_block *sb, journal_t *journal) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + + journal->j_commit_interval = sbi->s_commit_interval; + journal->j_min_batch_time = sbi->s_min_batch_time; + journal->j_max_batch_time = sbi->s_max_batch_time; + ext4_fc_init(sb, journal); + + write_lock(&journal->j_state_lock); + if (test_opt(sb, BARRIER)) + journal->j_flags |= JBD2_BARRIER; + else + journal->j_flags &= ~JBD2_BARRIER; + /* + * Always enable journal cycle record option, letting the journal + * records log transactions continuously between each mount. + */ + journal->j_flags |= JBD2_CYCLE_RECORD; + write_unlock(&journal->j_state_lock); +} + +static struct inode *ext4_get_journal_inode(struct super_block *sb, + unsigned int journal_inum) +{ + struct inode *journal_inode; + + /* + * Test for the existence of a valid inode on disk. Bad things + * happen if we iget() an unused inode, as the subsequent iput() + * will try to delete it. + */ + journal_inode = ext4_iget(sb, journal_inum, EXT4_IGET_SPECIAL); + if (IS_ERR(journal_inode)) { + ext4_msg(sb, KERN_ERR, "no journal found"); + return ERR_CAST(journal_inode); + } + if (!journal_inode->i_nlink) { + make_bad_inode(journal_inode); + iput(journal_inode); + ext4_msg(sb, KERN_ERR, "journal inode is deleted"); + return ERR_PTR(-EFSCORRUPTED); + } + if (!S_ISREG(journal_inode->i_mode) || IS_ENCRYPTED(journal_inode)) { + ext4_msg(sb, KERN_ERR, "invalid journal inode"); + iput(journal_inode); + return ERR_PTR(-EFSCORRUPTED); + } + +#ifdef CONFIG_LOCKDEP + lockdep_set_subclass(&EXT4_I(journal_inode)->i_data_sem, + I_DATA_SEM_JOURNAL); +#endif + + ext4_debug("Journal inode found at %p: %lld bytes\n", + journal_inode, journal_inode->i_size); + return journal_inode; +} + +static int ext4_journal_bmap(journal_t *journal, sector_t *block) +{ + struct ext4_map_blocks map; + int ret; + + if (journal->j_inode == NULL) + return 0; + + map.m_lblk = *block; + map.m_len = 1; + ret = ext4_map_blocks(NULL, journal->j_inode, &map, 0); + if (ret <= 0) { + ext4_msg(journal->j_inode->i_sb, KERN_CRIT, + "journal bmap failed: block %llu ret %d\n", + *block, ret); + jbd2_journal_abort(journal, ret ? ret : -EFSCORRUPTED); + return ret; + } + *block = map.m_pblk; + return 0; +} + +static journal_t *ext4_open_inode_journal(struct super_block *sb, + unsigned int journal_inum) +{ + struct inode *journal_inode; + journal_t *journal; + + journal_inode = ext4_get_journal_inode(sb, journal_inum); + if (IS_ERR(journal_inode)) + return ERR_CAST(journal_inode); + + journal = jbd2_journal_init_inode(journal_inode); + if (IS_ERR(journal)) { + ext4_msg(sb, KERN_ERR, "Could not load journal inode"); + iput(journal_inode); + return ERR_CAST(journal); + } + journal->j_private = sb; + journal->j_bmap = ext4_journal_bmap; + ext4_init_journal_params(sb, journal); + return journal; +} + +static struct file *ext4_get_journal_blkdev(struct super_block *sb, + dev_t j_dev, ext4_fsblk_t *j_start, + ext4_fsblk_t *j_len) +{ + struct buffer_head *bh; + struct block_device *bdev; + struct file *bdev_file; + int hblock, blocksize; + ext4_fsblk_t sb_block; + unsigned long offset; + struct ext4_super_block *es; + int errno; + + bdev_file = fs_bdev_file_open_by_dev(j_dev, + BLK_OPEN_READ | BLK_OPEN_WRITE | BLK_OPEN_RESTRICT_WRITES, + sb, sb); + if (IS_ERR(bdev_file)) { + ext4_msg(sb, KERN_ERR, + "failed to open journal device unknown-block(%u,%u) %pe", + MAJOR(j_dev), MINOR(j_dev), bdev_file); + return bdev_file; + } + + bdev = file_bdev(bdev_file); + blocksize = sb->s_blocksize; + hblock = bdev_logical_block_size(bdev); + if (blocksize < hblock) { + ext4_msg(sb, KERN_ERR, + "blocksize too small for journal device"); + errno = -EINVAL; + goto out_bdev; + } + + sb_block = EXT4_MIN_BLOCK_SIZE / blocksize; + offset = EXT4_MIN_BLOCK_SIZE % blocksize; + set_blocksize(bdev_file, blocksize); + bh = __bread(bdev, sb_block, blocksize); + if (!bh) { + ext4_msg(sb, KERN_ERR, "couldn't read superblock of " + "external journal"); + errno = -EINVAL; + goto out_bdev; + } + + es = (struct ext4_super_block *) (bh->b_data + offset); + if ((le16_to_cpu(es->s_magic) != EXT4_SUPER_MAGIC) || + !(le32_to_cpu(es->s_feature_incompat) & + EXT4_FEATURE_INCOMPAT_JOURNAL_DEV)) { + ext4_msg(sb, KERN_ERR, "external journal has bad superblock"); + errno = -EFSCORRUPTED; + goto out_bh; + } + + if ((le32_to_cpu(es->s_feature_ro_compat) & + EXT4_FEATURE_RO_COMPAT_METADATA_CSUM) && + es->s_checksum != ext4_superblock_csum(es)) { + ext4_msg(sb, KERN_ERR, "external journal has corrupt superblock"); + errno = -EFSCORRUPTED; + goto out_bh; + } + + if (memcmp(EXT4_SB(sb)->s_es->s_journal_uuid, es->s_uuid, 16)) { + ext4_msg(sb, KERN_ERR, "journal UUID does not match"); + errno = -EFSCORRUPTED; + goto out_bh; + } + + *j_start = sb_block + 1; + *j_len = ext4_blocks_count(es); + brelse(bh); + return bdev_file; + +out_bh: + brelse(bh); +out_bdev: + fs_bdev_file_release(bdev_file, sb); + return ERR_PTR(errno); +} + +static journal_t *ext4_open_dev_journal(struct super_block *sb, + dev_t j_dev) +{ + journal_t *journal; + ext4_fsblk_t j_start; + ext4_fsblk_t j_len; + struct file *bdev_file; + int errno = 0; + + bdev_file = ext4_get_journal_blkdev(sb, j_dev, &j_start, &j_len); + if (IS_ERR(bdev_file)) + return ERR_CAST(bdev_file); + + journal = jbd2_journal_init_dev(file_bdev(bdev_file), sb->s_bdev, j_start, + j_len, sb->s_blocksize); + if (IS_ERR(journal)) { + ext4_msg(sb, KERN_ERR, "failed to create device journal"); + errno = PTR_ERR(journal); + goto out_bdev; + } + if (be32_to_cpu(journal->j_superblock->s_nr_users) != 1) { + ext4_msg(sb, KERN_ERR, "External journal has more than one " + "user (unsupported) - %d", + be32_to_cpu(journal->j_superblock->s_nr_users)); + errno = -EINVAL; + goto out_journal; + } + journal->j_private = sb; + EXT4_SB(sb)->s_journal_bdev_file = bdev_file; + ext4_init_journal_params(sb, journal); + return journal; + +out_journal: + ext4_journal_destroy(EXT4_SB(sb), journal); +out_bdev: + fs_bdev_file_release(bdev_file, sb); + return ERR_PTR(errno); +} + +static int ext4_load_journal(struct super_block *sb, + struct ext4_super_block *es, + unsigned long journal_devnum) +{ + journal_t *journal; + unsigned int journal_inum = le32_to_cpu(es->s_journal_inum); + dev_t journal_dev; + int err = 0; + int really_read_only; + int journal_dev_ro; + + if (WARN_ON_ONCE(!ext4_has_feature_journal(sb))) + return -EFSCORRUPTED; + + if (journal_devnum && + journal_devnum != le32_to_cpu(es->s_journal_dev)) { + ext4_msg(sb, KERN_INFO, "external journal device major/minor " + "numbers have changed"); + journal_dev = new_decode_dev(journal_devnum); + } else + journal_dev = new_decode_dev(le32_to_cpu(es->s_journal_dev)); + + if (journal_inum && journal_dev) { + ext4_msg(sb, KERN_ERR, + "filesystem has both journal inode and journal device!"); + return -EINVAL; + } + + if (journal_inum) { + journal = ext4_open_inode_journal(sb, journal_inum); + if (IS_ERR(journal)) + return PTR_ERR(journal); + } else { + journal = ext4_open_dev_journal(sb, journal_dev); + if (IS_ERR(journal)) + return PTR_ERR(journal); + } + + journal_dev_ro = bdev_read_only(journal->j_dev); + really_read_only = bdev_read_only(sb->s_bdev) | journal_dev_ro; + + if (journal_dev_ro && !sb_rdonly(sb)) { + ext4_msg(sb, KERN_ERR, + "journal device read-only, try mounting with '-o ro'"); + err = -EROFS; + goto err_out; + } + + /* + * Are we loading a blank journal or performing recovery after a + * crash? For recovery, we need to check in advance whether we + * can get read-write access to the device. + */ + if (ext4_has_feature_journal_needs_recovery(sb)) { + if (sb_rdonly(sb)) { + ext4_msg(sb, KERN_INFO, "INFO: recovery " + "required on readonly filesystem"); + if (really_read_only) { + ext4_msg(sb, KERN_ERR, "write access " + "unavailable, cannot proceed " + "(try mounting with noload)"); + err = -EROFS; + goto err_out; + } + ext4_msg(sb, KERN_INFO, "write access will " + "be enabled during recovery"); + } + } + + if (!(journal->j_flags & JBD2_BARRIER)) + ext4_msg(sb, KERN_INFO, "barriers disabled"); + + if (!ext4_has_feature_journal_needs_recovery(sb)) + err = jbd2_journal_wipe(journal, !really_read_only); + if (!err) { + char *save = kmalloc(EXT4_S_ERR_LEN, GFP_KERNEL); + __le16 orig_state; + bool changed = false; + + if (save) + memcpy(save, ((char *) es) + + EXT4_S_ERR_START, EXT4_S_ERR_LEN); + err = jbd2_journal_load(journal); + if (save && memcmp(((char *) es) + EXT4_S_ERR_START, + save, EXT4_S_ERR_LEN)) { + memcpy(((char *) es) + EXT4_S_ERR_START, + save, EXT4_S_ERR_LEN); + changed = true; + } + kfree(save); + orig_state = es->s_state; + es->s_state |= cpu_to_le16(EXT4_SB(sb)->s_mount_state & + EXT4_ERROR_FS); + if (orig_state != es->s_state) + changed = true; + /* Write out restored error information to the superblock */ + if (changed && !really_read_only) { + int err2; + err2 = ext4_commit_super(sb); + err = err ? : err2; + } + } + + if (err) { + ext4_msg(sb, KERN_ERR, "error loading journal"); + goto err_out; + } + + EXT4_SB(sb)->s_journal = journal; + err = ext4_clear_journal_err(sb, es); + if (err) { + ext4_journal_destroy(EXT4_SB(sb), journal); + return err; + } + + if (!really_read_only && journal_devnum && + journal_devnum != le32_to_cpu(es->s_journal_dev)) { + es->s_journal_dev = cpu_to_le32(journal_devnum); + ext4_commit_super(sb); + } + if (!really_read_only && journal_inum && + journal_inum != le32_to_cpu(es->s_journal_inum)) { + es->s_journal_inum = cpu_to_le32(journal_inum); + ext4_commit_super(sb); + } + + return 0; + +err_out: + ext4_journal_destroy(EXT4_SB(sb), journal); + return err; +} + +/* Copy state of EXT4_SB(sb) into buffer for on-disk superblock */ +static void ext4_update_super(struct super_block *sb) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_super_block *es = sbi->s_es; + struct buffer_head *sbh = sbi->s_sbh; + + lock_buffer(sbh); + /* + * If the file system is mounted read-only, don't update the + * superblock write time. This avoids updating the superblock + * write time when we are mounting the root file system + * read/only but we need to replay the journal; at that point, + * for people who are east of GMT and who make their clock + * tick in localtime for Windows bug-for-bug compatibility, + * the clock is set in the future, and this will cause e2fsck + * to complain and force a full file system check. + */ + if (!sb_rdonly(sb)) + ext4_update_tstamp(es, s_wtime); + es->s_kbytes_written = + cpu_to_le64(sbi->s_kbytes_written + + ((part_stat_read(sb->s_bdev, sectors[STAT_WRITE]) - + sbi->s_sectors_written_start) >> 1)); + if (percpu_counter_initialized(&sbi->s_freeclusters_counter)) + ext4_free_blocks_count_set(es, + EXT4_C2B(sbi, percpu_counter_sum_positive( + &sbi->s_freeclusters_counter))); + if (percpu_counter_initialized(&sbi->s_freeinodes_counter)) + es->s_free_inodes_count = + cpu_to_le32(percpu_counter_sum_positive( + &sbi->s_freeinodes_counter)); + /* Copy error information to the on-disk superblock */ + spin_lock(&sbi->s_error_lock); + if (sbi->s_add_error_count > 0) { + es->s_state |= cpu_to_le16(EXT4_ERROR_FS); + if (!es->s_first_error_time && !es->s_first_error_time_hi) { + __ext4_update_tstamp(&es->s_first_error_time, + &es->s_first_error_time_hi, + sbi->s_first_error_time); + strtomem_pad(es->s_first_error_func, + sbi->s_first_error_func, 0); + es->s_first_error_line = + cpu_to_le32(sbi->s_first_error_line); + es->s_first_error_ino = + cpu_to_le32(sbi->s_first_error_ino); + es->s_first_error_block = + cpu_to_le64(sbi->s_first_error_block); + es->s_first_error_errcode = + ext4_errno_to_code(sbi->s_first_error_code); + } + __ext4_update_tstamp(&es->s_last_error_time, + &es->s_last_error_time_hi, + sbi->s_last_error_time); + strtomem_pad(es->s_last_error_func, sbi->s_last_error_func, 0); + es->s_last_error_line = cpu_to_le32(sbi->s_last_error_line); + es->s_last_error_ino = cpu_to_le32(sbi->s_last_error_ino); + es->s_last_error_block = cpu_to_le64(sbi->s_last_error_block); + es->s_last_error_errcode = + ext4_errno_to_code(sbi->s_last_error_code); + /* + * Start the daily error reporting function if it hasn't been + * started already and sbi->s_err_report_sec is not zero + */ + if (!es->s_error_count && !sbi->s_err_report_sec) + mod_timer(&sbi->s_err_report, + jiffies + secs_to_jiffies(sbi->s_err_report_sec)); + le32_add_cpu(&es->s_error_count, sbi->s_add_error_count); + sbi->s_add_error_count = 0; + } + spin_unlock(&sbi->s_error_lock); + + ext4_superblock_csum_set(sb); + unlock_buffer(sbh); +} + +static int ext4_commit_super(struct super_block *sb) +{ + struct buffer_head *sbh = EXT4_SB(sb)->s_sbh; + + if (!sbh) + return -EINVAL; + + ext4_update_super(sb); + + lock_buffer(sbh); + /* Buffer got discarded which means block device got invalidated */ + if (!buffer_mapped(sbh)) { + unlock_buffer(sbh); + return -EIO; + } + + if (buffer_write_io_error(sbh) || !buffer_uptodate(sbh)) { + /* + * Oh, dear. A previous attempt to write the + * superblock failed. This could happen because the + * USB device was yanked out. Or it could happen to + * be a transient write error and maybe the block will + * be remapped. Nothing we can do but to retry the + * write and hope for the best. + */ + ext4_msg(sb, KERN_ERR, "previous I/O error to " + "superblock detected"); + clear_buffer_write_io_error(sbh); + set_buffer_uptodate(sbh); + } + /* Clear potential dirty bit if it was journalled update */ + clear_buffer_dirty(sbh); + bh_submit(sbh, REQ_OP_WRITE | REQ_SYNC | + (test_opt(sb, BARRIER) ? REQ_FUA : 0), bh_end_write); + wait_on_buffer(sbh); + if (buffer_write_io_error(sbh)) { + ext4_msg(sb, KERN_ERR, "I/O error while writing " + "superblock"); + clear_buffer_write_io_error(sbh); + set_buffer_uptodate(sbh); + return -EIO; + } + return 0; +} + +/* + * Have we just finished recovery? If so, and if we are mounting (or + * remounting) the filesystem readonly, then we will end up with a + * consistent fs on disk. Record that fact. + */ +static int ext4_mark_recovery_complete(struct super_block *sb, + struct ext4_super_block *es) +{ + int err; + journal_t *journal = EXT4_SB(sb)->s_journal; + + if (!ext4_has_feature_journal(sb)) { + if (journal != NULL) { + ext4_error(sb, "Journal got removed while the fs was " + "mounted!"); + return -EFSCORRUPTED; + } + return 0; + } + jbd2_journal_lock_updates(journal); + err = jbd2_journal_flush(journal, 0); + if (err < 0) + goto out; + + if (sb_rdonly(sb) && (ext4_has_feature_journal_needs_recovery(sb) || + ext4_has_feature_orphan_present(sb))) { + if (!ext4_orphan_file_empty(sb)) { + ext4_error(sb, "Orphan file not empty on read-only fs."); + err = -EFSCORRUPTED; + goto out; + } + ext4_clear_feature_journal_needs_recovery(sb); + ext4_clear_feature_orphan_present(sb); + ext4_commit_super(sb); + } +out: + jbd2_journal_unlock_updates(journal); + return err; +} + +/* + * If we are mounting (or read-write remounting) a filesystem whose journal + * has recorded an error from a previous lifetime, move that error to the + * main filesystem now. + */ +static int ext4_clear_journal_err(struct super_block *sb, + struct ext4_super_block *es) +{ + journal_t *journal; + int j_errno; + const char *errstr; + + if (!ext4_has_feature_journal(sb)) { + ext4_error(sb, "Journal got removed while the fs was mounted!"); + return -EFSCORRUPTED; + } + + journal = EXT4_SB(sb)->s_journal; + + /* + * Now check for any error status which may have been recorded in the + * journal by a prior ext4_error() or ext4_abort() + */ + + j_errno = jbd2_journal_errno(journal); + if (j_errno) { + char nbuf[16]; + + errstr = ext4_decode_error(sb, j_errno, nbuf); + ext4_warning(sb, "Filesystem error recorded " + "from previous mount: %s", errstr); + + EXT4_SB(sb)->s_mount_state |= EXT4_ERROR_FS; + es->s_state |= cpu_to_le16(EXT4_ERROR_FS); + j_errno = ext4_commit_super(sb); + if (j_errno) + return j_errno; + ext4_warning(sb, "Marked fs in need of filesystem check."); + + jbd2_journal_clear_err(journal); + jbd2_journal_update_sb_errno(journal); + } + return 0; +} + +/* + * Force the running and committing transactions to commit, + * and wait on the commit. + */ +int ext4_force_commit(struct super_block *sb) +{ + return ext4_journal_force_commit(EXT4_SB(sb)->s_journal); +} + +static int ext4_sync_fs(struct super_block *sb, int wait) +{ + int ret = 0; + tid_t target; + bool needs_barrier = false; + struct ext4_sb_info *sbi = EXT4_SB(sb); + + ret = ext4_emergency_state(sb); + if (unlikely(ret)) + return ret; + + trace_ext4_sync_fs(sb, wait); + flush_workqueue(sbi->rsv_conversion_wq); + flush_delayed_work(&sbi->s_ea_inode_work); + /* + * Writeback quota in non-journalled quota case - journalled quota has + * no dirty dquots + */ + dquot_writeback_dquots(sb, -1); + /* + * Data writeback is possible w/o journal transaction, so barrier must + * being sent at the end of the function. But we can skip it if + * transaction_commit will do it for us. + */ + if (sbi->s_journal) { + target = jbd2_get_latest_transaction(sbi->s_journal); + if (wait && sbi->s_journal->j_flags & JBD2_BARRIER && + !jbd2_trans_will_send_data_barrier(sbi->s_journal, target)) + needs_barrier = true; + + if (jbd2_journal_start_commit(sbi->s_journal, &target)) { + if (wait) + ret = jbd2_log_wait_commit(sbi->s_journal, + target); + } + } else if (wait && test_opt(sb, BARRIER)) + needs_barrier = true; + if (needs_barrier) { + int err; + err = blkdev_issue_flush(sb->s_bdev); + if (!ret) + ret = err; + } + + return ret; +} + +/* + * LVM calls this function before a (read-only) snapshot is created. This + * gives us a chance to flush the journal completely and mark the fs clean. + * + * Note that only this function cannot bring a filesystem to be in a clean + * state independently. It relies on upper layer to stop all data & metadata + * modifications. + */ +static int ext4_freeze(struct super_block *sb) +{ + int error = 0; + journal_t *journal = EXT4_SB(sb)->s_journal; + + if (journal) { + /* Now we set up the journal barrier. */ + jbd2_journal_lock_updates(journal); + + /* + * Don't clear the needs_recovery flag if we failed to + * flush the journal. + */ + error = jbd2_journal_flush(journal, 0); + if (error < 0) + goto out; + + /* Journal blocked and flushed, clear needs_recovery flag. */ + ext4_clear_feature_journal_needs_recovery(sb); + if (ext4_orphan_file_empty(sb)) + ext4_clear_feature_orphan_present(sb); + } + + error = ext4_commit_super(sb); +out: + if (journal) + /* we rely on upper layer to stop further updates */ + jbd2_journal_unlock_updates(journal); + return error; +} + +/* + * Called by LVM after the snapshot is done. We need to reset the RECOVER + * flag here, even though the filesystem is not technically dirty yet. + */ +static int ext4_unfreeze(struct super_block *sb) +{ + if (ext4_emergency_state(sb)) + return 0; + + if (EXT4_SB(sb)->s_journal) { + /* Reset the needs_recovery flag before the fs is unlocked. */ + ext4_set_feature_journal_needs_recovery(sb); + if (ext4_has_feature_orphan_file(sb)) + ext4_set_feature_orphan_present(sb); + } + + ext4_commit_super(sb); + return 0; +} + +/* + * Structure to save mount options for ext4_remount's benefit + */ +struct ext4_mount_options { + unsigned long s_mount_opt; + unsigned long s_mount_opt2; + kuid_t s_resuid; + kgid_t s_resgid; + unsigned long s_commit_interval; + u32 s_min_batch_time, s_max_batch_time; +#ifdef CONFIG_QUOTA + int s_jquota_fmt; + char *s_qf_names[EXT4_MAXQUOTAS]; +#endif +}; + +static int __ext4_remount(struct fs_context *fc, struct super_block *sb) +{ + struct ext4_fs_context *ctx = fc->fs_private; + struct ext4_super_block *es; + struct ext4_sb_info *sbi = EXT4_SB(sb); + unsigned long old_sb_flags; + struct ext4_mount_options old_opts; + ext4_group_t g; + int err = 0; + int alloc_ctx; +#ifdef CONFIG_QUOTA + int enable_quota = 0; + int i, j; + char *to_free[EXT4_MAXQUOTAS]; +#endif + + + /* Store the original options */ + old_sb_flags = sb->s_flags; + old_opts.s_mount_opt = sbi->s_mount_opt; + old_opts.s_mount_opt2 = sbi->s_mount_opt2; + old_opts.s_resuid = sbi->s_resuid; + old_opts.s_resgid = sbi->s_resgid; + old_opts.s_commit_interval = sbi->s_commit_interval; + old_opts.s_min_batch_time = sbi->s_min_batch_time; + old_opts.s_max_batch_time = sbi->s_max_batch_time; +#ifdef CONFIG_QUOTA + old_opts.s_jquota_fmt = sbi->s_jquota_fmt; + for (i = 0; i < EXT4_MAXQUOTAS; i++) + if (sbi->s_qf_names[i]) { + char *qf_name = get_qf_name(sb, sbi, i); + + old_opts.s_qf_names[i] = kstrdup(qf_name, GFP_KERNEL); + if (!old_opts.s_qf_names[i]) { + for (j = 0; j < i; j++) + kfree(old_opts.s_qf_names[j]); + return -ENOMEM; + } + } else + old_opts.s_qf_names[i] = NULL; +#endif + if (!(ctx->spec & EXT4_SPEC_JOURNAL_IOPRIO)) { + if (sbi->s_journal && sbi->s_journal->j_task->io_context) + ctx->journal_ioprio = + sbi->s_journal->j_task->io_context->ioprio; + else + ctx->journal_ioprio = EXT4_DEF_JOURNAL_IOPRIO; + + } + + if ((ctx->spec & EXT4_SPEC_s_stripe) && + ext4_is_stripe_incompatible(sb, ctx->s_stripe)) { + ext4_msg(sb, KERN_WARNING, + "stripe (%lu) is not aligned with cluster size (%u), " + "stripe is disabled", + ctx->s_stripe, sbi->s_cluster_ratio); + ctx->s_stripe = 0; + } + + /* + * Changing the DIOREAD_NOLOCK or DELALLOC mount options may cause + * two calls to ext4_should_dioread_nolock() to return inconsistent + * values, triggering WARN_ON in ext4_add_complete_io(). we grab + * here s_writepages_rwsem to avoid race between writepages ops and + * remount. + */ + alloc_ctx = ext4_writepages_down_write(sb); + ext4_apply_options(fc, sb); + ext4_writepages_up_write(sb, alloc_ctx); + + if ((old_opts.s_mount_opt & EXT4_MOUNT_JOURNAL_CHECKSUM) ^ + test_opt(sb, JOURNAL_CHECKSUM)) { + ext4_msg(sb, KERN_ERR, "changing journal_checksum " + "during remount not supported; ignoring"); + sbi->s_mount_opt ^= EXT4_MOUNT_JOURNAL_CHECKSUM; + } + + if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_JOURNAL_DATA) { + if (test_opt2(sb, EXPLICIT_DELALLOC)) { + ext4_msg(sb, KERN_ERR, "can't mount with " + "both data=journal and delalloc"); + err = -EINVAL; + goto restore_opts; + } + if (test_opt(sb, DIOREAD_NOLOCK)) { + ext4_msg(sb, KERN_ERR, "can't mount with " + "both data=journal and dioread_nolock"); + err = -EINVAL; + goto restore_opts; + } + } else if (test_opt(sb, DATA_FLAGS) == EXT4_MOUNT_ORDERED_DATA) { + if (test_opt(sb, JOURNAL_ASYNC_COMMIT)) { + ext4_msg(sb, KERN_ERR, "can't mount with " + "journal_async_commit in data=ordered mode"); + err = -EINVAL; + goto restore_opts; + } + } + + if ((sbi->s_mount_opt ^ old_opts.s_mount_opt) & EXT4_MOUNT_NO_MBCACHE) { + ext4_msg(sb, KERN_ERR, "can't enable nombcache during remount"); + err = -EINVAL; + goto restore_opts; + } + + if ((old_opts.s_mount_opt & EXT4_MOUNT_DELALLOC) && + !test_opt(sb, DELALLOC)) { + ext4_msg(sb, KERN_ERR, "can't disable delalloc during remount"); + err = -EINVAL; + goto restore_opts; + } + + sb->s_flags = (sb->s_flags & ~SB_POSIXACL) | + (test_opt(sb, POSIX_ACL) ? SB_POSIXACL : 0); + + es = sbi->s_es; + + if (sbi->s_journal) { + ext4_init_journal_params(sb, sbi->s_journal); + set_task_ioprio(sbi->s_journal->j_task, ctx->journal_ioprio); + } + + /* Flush outstanding errors before changing fs state */ + flush_work(&sbi->s_sb_upd_work); + + if ((bool)(fc->sb_flags & SB_RDONLY) != sb_rdonly(sb)) { + if (ext4_emergency_state(sb)) { + err = -EROFS; + goto restore_opts; + } + + if (fc->sb_flags & SB_RDONLY) { + err = sync_filesystem(sb); + if (err < 0) + goto restore_opts; + err = dquot_suspend(sb, -1); + if (err < 0) + goto restore_opts; + + /* + * First of all, the unconditional stuff we have to do + * to disable replay of the journal when we next remount + */ + sb->s_flags |= SB_RDONLY; + + /* + * OK, test if we are remounting a valid rw partition + * readonly, and if so set the rdonly flag and then + * mark the partition as valid again. + */ + if (!(es->s_state & cpu_to_le16(EXT4_VALID_FS)) && + (sbi->s_mount_state & EXT4_VALID_FS)) + es->s_state = cpu_to_le16(sbi->s_mount_state); + + if (sbi->s_journal) { + /* + * We let remount-ro finish even if marking fs + * as clean failed... + */ + ext4_mark_recovery_complete(sb, es); + } + } else { + /* Make sure we can mount this feature set readwrite */ + if (ext4_has_feature_readonly(sb) || + !ext4_feature_set_ok(sb, 0)) { + err = -EROFS; + goto restore_opts; + } + /* + * Make sure the group descriptor checksums + * are sane. If they aren't, refuse to remount r/w. + */ + for (g = 0; g < sbi->s_groups_count; g++) { + struct ext4_group_desc *gdp = + ext4_get_group_desc(sb, g, NULL); + + if (!ext4_group_desc_csum_verify(sb, g, gdp)) { + ext4_msg(sb, KERN_ERR, + "ext4_remount: Checksum for group %u failed (%u!=%u)", + g, le16_to_cpu(ext4_group_desc_csum(sb, g, gdp)), + le16_to_cpu(gdp->bg_checksum)); + err = -EFSBADCRC; + goto restore_opts; + } + } + + /* + * If we have an unprocessed orphan list hanging + * around from a previously readonly bdev mount, + * require a full umount/remount for now. + */ + if (es->s_last_orphan || !ext4_orphan_file_empty(sb)) { + ext4_msg(sb, KERN_WARNING, "Couldn't " + "remount RDWR because of unprocessed " + "orphan inode list. Please " + "umount/remount instead"); + err = -EINVAL; + goto restore_opts; + } + + /* + * Mounting a RDONLY partition read-write, so reread + * and store the current valid flag. (It may have + * been changed by e2fsck since we originally mounted + * the partition.) + */ + if (sbi->s_journal) { + err = ext4_clear_journal_err(sb, es); + if (err) + goto restore_opts; + } + sbi->s_mount_state = (le16_to_cpu(es->s_state) & + ~EXT4_FC_REPLAY); + + err = ext4_setup_super(sb, es, 0); + if (err) + goto restore_opts; + + sb->s_flags &= ~SB_RDONLY; + if (ext4_has_feature_mmp(sb)) { + err = ext4_multi_mount_protect(sb, + le64_to_cpu(es->s_mmp_block)); + if (err) + goto restore_opts; + } +#ifdef CONFIG_QUOTA + enable_quota = 1; +#endif + } + } + + /* + * Handle creation of system zone data early because it can fail. + * Releasing of existing data is done when we are sure remount will + * succeed. + */ + if (test_opt(sb, BLOCK_VALIDITY) && !sbi->s_system_blks) { + err = ext4_setup_system_zone(sb); + if (err) + goto restore_opts; + } + + if (sbi->s_journal == NULL && !(old_sb_flags & SB_RDONLY)) { + err = ext4_commit_super(sb); + if (err) + goto restore_opts; + } + +#ifdef CONFIG_QUOTA + if (enable_quota) { + if (sb_any_quota_suspended(sb)) + dquot_resume(sb, -1); + else if (ext4_has_feature_quota(sb)) { + err = ext4_enable_quotas(sb); + if (err) + goto restore_opts; + } + } + /* Release old quota file names */ + for (i = 0; i < EXT4_MAXQUOTAS; i++) + kfree(old_opts.s_qf_names[i]); +#endif + if (!test_opt(sb, BLOCK_VALIDITY) && sbi->s_system_blks) + ext4_release_system_zone(sb); + + /* + * Reinitialize lazy itable initialization thread based on + * current settings + */ + if (sb_rdonly(sb) || !test_opt(sb, INIT_INODE_TABLE)) + ext4_unregister_li_request(sb); + else { + ext4_group_t first_not_zeroed; + first_not_zeroed = ext4_has_uninit_itable(sb); + ext4_register_li_request(sb, first_not_zeroed); + } + + if (!ext4_has_feature_mmp(sb) || sb_rdonly(sb)) + ext4_stop_mmpd(sbi); + + /* + * Handle aborting the filesystem as the last thing during remount to + * avoid obsure errors during remount when some option changes fail to + * apply due to shutdown filesystem. + */ + if (test_opt2(sb, ABORT)) + ext4_abort(sb, ESHUTDOWN, "Abort forced by user"); + + return 0; + +restore_opts: + /* + * If there was a failing r/w to ro transition, we may need to + * re-enable quota + */ + if (sb_rdonly(sb) && !(old_sb_flags & SB_RDONLY) && + sb_any_quota_suspended(sb)) + dquot_resume(sb, -1); + + alloc_ctx = ext4_writepages_down_write(sb); + sb->s_flags = old_sb_flags; + sbi->s_mount_opt = old_opts.s_mount_opt; + sbi->s_mount_opt2 = old_opts.s_mount_opt2; + sbi->s_resuid = old_opts.s_resuid; + sbi->s_resgid = old_opts.s_resgid; + sbi->s_commit_interval = old_opts.s_commit_interval; + sbi->s_min_batch_time = old_opts.s_min_batch_time; + sbi->s_max_batch_time = old_opts.s_max_batch_time; + ext4_writepages_up_write(sb, alloc_ctx); + + if (!test_opt(sb, BLOCK_VALIDITY) && sbi->s_system_blks) + ext4_release_system_zone(sb); +#ifdef CONFIG_QUOTA + sbi->s_jquota_fmt = old_opts.s_jquota_fmt; + for (i = 0; i < EXT4_MAXQUOTAS; i++) { + to_free[i] = get_qf_name(sb, sbi, i); + rcu_assign_pointer(sbi->s_qf_names[i], old_opts.s_qf_names[i]); + } + synchronize_rcu(); + for (i = 0; i < EXT4_MAXQUOTAS; i++) + kfree(to_free[i]); +#endif + if (!ext4_has_feature_mmp(sb) || sb_rdonly(sb)) + ext4_stop_mmpd(sbi); + return err; +} + +static int ext4_reconfigure(struct fs_context *fc) +{ + struct super_block *sb = fc->root->d_sb; + int ret; + bool old_ro = sb_rdonly(sb); + + fc->s_fs_info = EXT4_SB(sb); + + ret = ext4_check_opt_consistency(fc, sb); + if (ret < 0) + return ret; + + ret = __ext4_remount(fc, sb); + if (ret < 0) + return ret; + + ext4_msg(sb, KERN_INFO, "re-mounted %pU%s.", + &sb->s_uuid, + (old_ro != sb_rdonly(sb)) ? (sb_rdonly(sb) ? " ro" : " r/w") : ""); + + return 0; +} + +#ifdef CONFIG_QUOTA +static int ext4_statfs_project(struct super_block *sb, + kprojid_t projid, struct kstatfs *buf) +{ + struct kqid qid; + struct dquot *dquot; + u64 limit; + u64 curblock; + + qid = make_kqid_projid(projid); + dquot = dqget(sb, qid); + if (IS_ERR(dquot)) + return PTR_ERR(dquot); + spin_lock(&dquot->dq_dqb_lock); + + limit = min_not_zero(dquot->dq_dqb.dqb_bsoftlimit, + dquot->dq_dqb.dqb_bhardlimit); + limit >>= sb->s_blocksize_bits; + + if (limit) { + uint64_t remaining = 0; + + curblock = (dquot->dq_dqb.dqb_curspace + + dquot->dq_dqb.dqb_rsvspace) >> sb->s_blocksize_bits; + if (limit > curblock) + remaining = limit - curblock; + + buf->f_blocks = min(buf->f_blocks, limit); + buf->f_bfree = min(buf->f_bfree, remaining); + buf->f_bavail = min(buf->f_bavail, remaining); + } + + limit = min_not_zero(dquot->dq_dqb.dqb_isoftlimit, + dquot->dq_dqb.dqb_ihardlimit); + if (limit) { + uint64_t remaining = 0; + + if (limit > dquot->dq_dqb.dqb_curinodes) + remaining = limit - dquot->dq_dqb.dqb_curinodes; + + buf->f_files = min(buf->f_files, limit); + buf->f_ffree = min(buf->f_ffree, remaining); + } + + spin_unlock(&dquot->dq_dqb_lock); + dqput(dquot); + return 0; +} +#endif + +static int ext4_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct ext4_super_block *es = sbi->s_es; + ext4_fsblk_t overhead = 0, resv_blocks; + s64 bfree; + resv_blocks = EXT4_C2B(sbi, atomic64_read(&sbi->s_resv_clusters)); + + if (!test_opt(sb, MINIX_DF)) + overhead = sbi->s_overhead; + + buf->f_type = EXT4_SUPER_MAGIC; + buf->f_bsize = sb->s_blocksize; + buf->f_blocks = ext4_blocks_count(es) - EXT4_C2B(sbi, overhead); + bfree = percpu_counter_sum_positive(&sbi->s_freeclusters_counter) - + percpu_counter_sum_positive(&sbi->s_dirtyclusters_counter); + /* prevent underflow in case that few free space is available */ + buf->f_bfree = EXT4_C2B(sbi, max_t(s64, bfree, 0)); + buf->f_bavail = buf->f_bfree - + (ext4_r_blocks_count(es) + resv_blocks); + if (buf->f_bfree < (ext4_r_blocks_count(es) + resv_blocks)) + buf->f_bavail = 0; + buf->f_files = le32_to_cpu(es->s_inodes_count); + buf->f_ffree = percpu_counter_sum_positive(&sbi->s_freeinodes_counter); + buf->f_namelen = EXT4_NAME_LEN; + buf->f_fsid = uuid_to_fsid(es->s_uuid); + +#ifdef CONFIG_QUOTA + if (ext4_test_inode_flag(dentry->d_inode, EXT4_INODE_PROJINHERIT) && + sb_has_quota_limits_enabled(sb, PRJQUOTA)) + ext4_statfs_project(sb, EXT4_I(dentry->d_inode)->i_projid, buf); +#endif + return 0; +} + + +#ifdef CONFIG_QUOTA + +/* + * Helper functions so that transaction is started before we acquire dqio_sem + * to keep correct lock ordering of transaction > dqio_sem + */ +static inline struct inode *dquot_to_inode(struct dquot *dquot) +{ + return sb_dqopt(dquot->dq_sb)->files[dquot->dq_id.type]; +} + +static int ext4_write_dquot(struct dquot *dquot) +{ + int ret, err; + handle_t *handle; + struct inode *inode; + + inode = dquot_to_inode(dquot); + handle = ext4_journal_start(inode, EXT4_HT_QUOTA, + EXT4_QUOTA_TRANS_BLOCKS(dquot->dq_sb)); + if (IS_ERR(handle)) + return PTR_ERR(handle); + ret = dquot_commit(dquot); + if (ret < 0) + ext4_error_err(dquot->dq_sb, -ret, + "Failed to commit dquot type %d", + dquot->dq_id.type); + err = ext4_journal_stop(handle); + if (!ret) + ret = err; + return ret; +} + +static int ext4_acquire_dquot(struct dquot *dquot) +{ + int ret, err; + handle_t *handle; + + handle = ext4_journal_start(dquot_to_inode(dquot), EXT4_HT_QUOTA, + EXT4_QUOTA_INIT_BLOCKS(dquot->dq_sb)); + if (IS_ERR(handle)) + return PTR_ERR(handle); + ret = dquot_acquire(dquot); + if (ret < 0) + ext4_error_err(dquot->dq_sb, -ret, + "Failed to acquire dquot type %d", + dquot->dq_id.type); + err = ext4_journal_stop(handle); + if (!ret) + ret = err; + return ret; +} + +static int ext4_release_dquot(struct dquot *dquot) +{ + int ret, err; + handle_t *handle; + bool freeze_protected = false; + + /* + * Trying to sb_start_intwrite() in a running transaction + * can result in a deadlock. Further, running transactions + * are already protected from freezing. + */ + if (!ext4_journal_current_handle()) { + sb_start_intwrite(dquot->dq_sb); + freeze_protected = true; + } + + handle = ext4_journal_start(dquot_to_inode(dquot), EXT4_HT_QUOTA, + EXT4_QUOTA_DEL_BLOCKS(dquot->dq_sb)); + if (IS_ERR(handle)) { + /* Release dquot anyway to avoid endless cycle in dqput() */ + dquot_release(dquot); + if (freeze_protected) + sb_end_intwrite(dquot->dq_sb); + return PTR_ERR(handle); + } + ret = dquot_release(dquot); + if (ret < 0) + ext4_error_err(dquot->dq_sb, -ret, + "Failed to release dquot type %d", + dquot->dq_id.type); + err = ext4_journal_stop(handle); + if (!ret) + ret = err; + + if (freeze_protected) + sb_end_intwrite(dquot->dq_sb); + + return ret; +} + +static int ext4_mark_dquot_dirty(struct dquot *dquot) +{ + struct super_block *sb = dquot->dq_sb; + + if (ext4_is_quota_journalled(sb)) { + dquot_mark_dquot_dirty(dquot); + return ext4_write_dquot(dquot); + } else { + return dquot_mark_dquot_dirty(dquot); + } +} + +static int ext4_write_info(struct super_block *sb, int type) +{ + int ret, err; + handle_t *handle; + + /* Data block + inode block */ + handle = ext4_journal_start_sb(sb, EXT4_HT_QUOTA, 2); + if (IS_ERR(handle)) + return PTR_ERR(handle); + ret = dquot_commit_info(sb, type); + err = ext4_journal_stop(handle); + if (!ret) + ret = err; + return ret; +} + +static void lockdep_set_quota_inode(struct inode *inode, int subclass) +{ + struct ext4_inode_info *ei = EXT4_I(inode); + + /* The first argument of lockdep_set_subclass has to be + * *exactly* the same as the argument to init_rwsem() --- in + * this case, in init_once() --- or lockdep gets unhappy + * because the name of the lock is set using the + * stringification of the argument to init_rwsem(). + */ + (void) ei; /* shut up clang warning if !CONFIG_LOCKDEP */ + lockdep_set_subclass(&ei->i_data_sem, subclass); +} + +/* + * Standard function to be called on quota_on + */ +static int ext4_quota_on(struct super_block *sb, int type, int format_id, + const struct path *path) +{ + int err; + + if (!test_opt(sb, QUOTA)) + return -EINVAL; + + /* Quotafile not on the same filesystem? */ + if (path->dentry->d_sb != sb) + return -EXDEV; + + /* Quota already enabled for this file? */ + if (IS_NOQUOTA(d_inode(path->dentry))) + return -EBUSY; + + /* Journaling quota? */ + if (EXT4_SB(sb)->s_qf_names[type]) { + /* Quotafile not in fs root? */ + if (path->dentry->d_parent != sb->s_root) + ext4_msg(sb, KERN_WARNING, + "Quota file not on filesystem root. " + "Journaled quota will not work"); + sb_dqopt(sb)->flags |= DQUOT_NOLIST_DIRTY; + } else { + /* + * Clear the flag just in case mount options changed since + * last time. + */ + sb_dqopt(sb)->flags &= ~DQUOT_NOLIST_DIRTY; + } + + lockdep_set_quota_inode(path->dentry->d_inode, I_DATA_SEM_QUOTA); + err = dquot_quota_on(sb, type, format_id, path); + if (!err) { + struct inode *inode = d_inode(path->dentry); + handle_t *handle; + + /* + * Set inode flags to prevent userspace from messing with quota + * files. If this fails, we return success anyway since quotas + * are already enabled and this is not a hard failure. + */ + inode_lock(inode); + handle = ext4_journal_start(inode, EXT4_HT_QUOTA, 1); + if (IS_ERR(handle)) + goto unlock_inode; + EXT4_I(inode)->i_flags |= EXT4_NOATIME_FL | EXT4_IMMUTABLE_FL; + inode_set_flags(inode, S_NOATIME | S_IMMUTABLE, + S_NOATIME | S_IMMUTABLE); + err = ext4_mark_inode_dirty(handle, inode); + ext4_journal_stop(handle); + unlock_inode: + inode_unlock(inode); + if (err) + dquot_quota_off(sb, type); + } + if (err) + lockdep_set_quota_inode(path->dentry->d_inode, + I_DATA_SEM_NORMAL); + return err; +} + +static inline bool ext4_check_quota_inum(int type, unsigned long qf_inum) +{ + switch (type) { + case USRQUOTA: + return qf_inum == EXT4_USR_QUOTA_INO; + case GRPQUOTA: + return qf_inum == EXT4_GRP_QUOTA_INO; + case PRJQUOTA: + return qf_inum >= EXT4_GOOD_OLD_FIRST_INO; + default: + BUG(); + } +} + +static int ext4_quota_enable(struct super_block *sb, int type, int format_id, + unsigned int flags) +{ + int err; + struct inode *qf_inode; + unsigned long qf_inums[EXT4_MAXQUOTAS] = { + le32_to_cpu(EXT4_SB(sb)->s_es->s_usr_quota_inum), + le32_to_cpu(EXT4_SB(sb)->s_es->s_grp_quota_inum), + le32_to_cpu(EXT4_SB(sb)->s_es->s_prj_quota_inum) + }; + + BUG_ON(!ext4_has_feature_quota(sb)); + + if (!qf_inums[type]) + return -EPERM; + + if (!ext4_check_quota_inum(type, qf_inums[type])) { + ext4_error(sb, "Bad quota inum: %lu, type: %d", + qf_inums[type], type); + return -EUCLEAN; + } + + qf_inode = ext4_iget(sb, qf_inums[type], EXT4_IGET_SPECIAL); + if (IS_ERR(qf_inode)) { + ext4_error(sb, "Bad quota inode: %lu, type: %d", + qf_inums[type], type); + return PTR_ERR(qf_inode); + } + + /* Don't account quota for quota files to avoid recursion */ + qf_inode->i_flags |= S_NOQUOTA; + lockdep_set_quota_inode(qf_inode, I_DATA_SEM_QUOTA); + err = dquot_load_quota_inode(qf_inode, type, format_id, flags); + if (err) + lockdep_set_quota_inode(qf_inode, I_DATA_SEM_NORMAL); + iput(qf_inode); + + return err; +} + +/* Enable usage tracking for all quota types. */ +int ext4_enable_quotas(struct super_block *sb) +{ + int type, err = 0; + unsigned long qf_inums[EXT4_MAXQUOTAS] = { + le32_to_cpu(EXT4_SB(sb)->s_es->s_usr_quota_inum), + le32_to_cpu(EXT4_SB(sb)->s_es->s_grp_quota_inum), + le32_to_cpu(EXT4_SB(sb)->s_es->s_prj_quota_inum) + }; + bool quota_mopt[EXT4_MAXQUOTAS] = { + test_opt(sb, USRQUOTA), + test_opt(sb, GRPQUOTA), + test_opt(sb, PRJQUOTA), + }; + + sb_dqopt(sb)->flags |= DQUOT_QUOTA_SYS_FILE | DQUOT_NOLIST_DIRTY; + for (type = 0; type < EXT4_MAXQUOTAS; type++) { + if (qf_inums[type]) { + err = ext4_quota_enable(sb, type, QFMT_VFS_V1, + DQUOT_USAGE_ENABLED | + (quota_mopt[type] ? DQUOT_LIMITS_ENABLED : 0)); + if (err) { + ext4_warning(sb, + "Failed to enable quota tracking " + "(type=%d, err=%d, ino=%lu). " + "Please run e2fsck to fix.", type, + err, qf_inums[type]); + + ext4_quotas_off(sb, type); + return err; + } + } + } + return 0; +} + +static int ext4_quota_off(struct super_block *sb, int type) +{ + struct inode *inode = sb_dqopt(sb)->files[type]; + handle_t *handle; + int err; + + /* Force all delayed allocation blocks to be allocated. + * Caller already holds s_umount sem */ + if (test_opt(sb, DELALLOC)) + sync_filesystem(sb); + + if (!inode || !igrab(inode)) + goto out; + + err = dquot_quota_off(sb, type); + if (err || ext4_has_feature_quota(sb)) + goto out_put; + /* + * When the filesystem was remounted read-only first, we cannot cleanup + * inode flags here. Bad luck but people should be using QUOTA feature + * these days anyway. + */ + if (sb_rdonly(sb)) + goto out_put; + + inode_lock(inode); + /* + * Update modification times of quota files when userspace can + * start looking at them. If we fail, we return success anyway since + * this is not a hard failure and quotas are already disabled. + */ + handle = ext4_journal_start(inode, EXT4_HT_QUOTA, 1); + if (IS_ERR(handle)) { + err = PTR_ERR(handle); + goto out_unlock; + } + EXT4_I(inode)->i_flags &= ~(EXT4_NOATIME_FL | EXT4_IMMUTABLE_FL); + inode_set_flags(inode, 0, S_NOATIME | S_IMMUTABLE); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + err = ext4_mark_inode_dirty(handle, inode); + ext4_journal_stop(handle); +out_unlock: + inode_unlock(inode); +out_put: + lockdep_set_quota_inode(inode, I_DATA_SEM_NORMAL); + iput(inode); + return err; +out: + return dquot_quota_off(sb, type); +} + +/* Read data from quotafile - avoid pagecache and such because we cannot afford + * acquiring the locks... As quota files are never truncated and quota code + * itself serializes the operations (and no one else should touch the files) + * we don't have to be afraid of races */ +static ssize_t ext4_quota_read(struct super_block *sb, int type, char *data, + size_t len, loff_t off) +{ + struct inode *inode = sb_dqopt(sb)->files[type]; + ext4_lblk_t blk = off >> EXT4_BLOCK_SIZE_BITS(sb); + int offset = off & (sb->s_blocksize - 1); + int tocopy; + size_t toread; + struct buffer_head *bh; + loff_t i_size = i_size_read(inode); + + if (off > i_size) + return 0; + if (off+len > i_size) + len = i_size-off; + toread = len; + while (toread > 0) { + tocopy = min_t(unsigned long, sb->s_blocksize - offset, toread); + bh = ext4_bread(NULL, inode, blk, 0); + if (IS_ERR(bh)) + return PTR_ERR(bh); + if (!bh) /* A hole? */ + memset(data, 0, tocopy); + else + memcpy(data, bh->b_data+offset, tocopy); + brelse(bh); + offset = 0; + toread -= tocopy; + data += tocopy; + blk++; + } + return len; +} + +/* Write to quotafile (we know the transaction is already started and has + * enough credits) */ +static ssize_t ext4_quota_write(struct super_block *sb, int type, + const char *data, size_t len, loff_t off) +{ + struct inode *inode = sb_dqopt(sb)->files[type]; + ext4_lblk_t blk = off >> EXT4_BLOCK_SIZE_BITS(sb); + int err = 0, err2 = 0, offset = off & (sb->s_blocksize - 1); + int retries = 0; + struct buffer_head *bh; + handle_t *handle = journal_current_handle(); + + if (!handle) { + ext4_msg(sb, KERN_WARNING, "Quota write (off=%llu, len=%llu)" + " cancelled because transaction is not started", + (unsigned long long)off, (unsigned long long)len); + return -EIO; + } + /* + * Since we account only one data block in transaction credits, + * then it is impossible to cross a block boundary. + */ + if (sb->s_blocksize - offset < len) { + ext4_msg(sb, KERN_WARNING, "Quota write (off=%llu, len=%llu)" + " cancelled because not block aligned", + (unsigned long long)off, (unsigned long long)len); + return -EIO; + } + + do { + bh = ext4_bread(handle, inode, blk, + EXT4_GET_BLOCKS_CREATE | + EXT4_GET_BLOCKS_METADATA_NOFAIL); + } while (PTR_ERR(bh) == -ENOSPC && + ext4_should_retry_alloc(inode->i_sb, &retries)); + if (IS_ERR(bh)) + return PTR_ERR(bh); + if (!bh) + goto out; + BUFFER_TRACE(bh, "get write access"); + err = ext4_journal_get_write_access(handle, sb, bh, EXT4_JTR_NONE); + if (err) { + brelse(bh); + return err; + } + lock_buffer(bh); + memcpy(bh->b_data+offset, data, len); + flush_dcache_folio(bh->b_folio); + unlock_buffer(bh); + err = ext4_handle_dirty_metadata(handle, NULL, bh); + brelse(bh); +out: + if (inode->i_size < off + len) { + i_size_write(inode, off + len); + EXT4_I(inode)->i_disksize = inode->i_size; + err2 = ext4_mark_inode_dirty(handle, inode); + if (unlikely(err2 && !err)) + err = err2; + } + return err ? err : len; +} +#endif + +#if !defined(CONFIG_EXT2_FS) && !defined(CONFIG_EXT2_FS_MODULE) && defined(CONFIG_EXT4_USE_FOR_EXT2) +static inline void register_as_ext2(void) +{ + int err = register_filesystem(&ext2_fs_type); + if (err) + printk(KERN_WARNING + "EXT4-fs: Unable to register as ext2 (%d)\n", err); +} + +static inline void unregister_as_ext2(void) +{ + unregister_filesystem(&ext2_fs_type); +} + +static inline int ext2_feature_set_ok(struct super_block *sb) +{ + if (ext4_has_unknown_ext2_incompat_features(sb)) + return 0; + if (sb_rdonly(sb)) + return 1; + if (ext4_has_unknown_ext2_ro_compat_features(sb)) + return 0; + return 1; +} +#else +static inline void register_as_ext2(void) { } +static inline void unregister_as_ext2(void) { } +static inline int ext2_feature_set_ok(struct super_block *sb) { return 0; } +#endif + +static inline void register_as_ext3(void) +{ + int err = register_filesystem(&ext3_fs_type); + if (err) + printk(KERN_WARNING + "EXT4-fs: Unable to register as ext3 (%d)\n", err); +} + +static inline void unregister_as_ext3(void) +{ + unregister_filesystem(&ext3_fs_type); +} + +static inline int ext3_feature_set_ok(struct super_block *sb) +{ + if (ext4_has_unknown_ext3_incompat_features(sb)) + return 0; + if (!ext4_has_feature_journal(sb)) + return 0; + if (sb_rdonly(sb)) + return 1; + if (ext4_has_unknown_ext3_ro_compat_features(sb)) + return 0; + return 1; +} + +static void ext4_kill_sb(struct super_block *sb) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + struct file *bdev_file = sbi ? sbi->s_journal_bdev_file : NULL; + + kill_block_super(sb); + + if (bdev_file) + fs_bdev_file_release(bdev_file, sb); +} + +static struct file_system_type ext4_fs_type = { + .owner = THIS_MODULE, + .name = "ext4", + .init_fs_context = ext4_init_fs_context, + .parameters = ext4_param_specs, + .kill_sb = ext4_kill_sb, + .fs_flags = FS_REQUIRES_DEV | FS_ALLOW_IDMAP | FS_MGTIME | + FS_LBS, +}; +MODULE_ALIAS_FS("ext4"); + +static int __init ext4_init_fs(void) +{ + int err; + + ratelimit_state_init(&ext4_mount_msg_ratelimit, 30 * HZ, 64); + ext4_li_info = NULL; + + /* Build-time check for flags consistency */ + ext4_check_flag_values(); + + err = ext4_init_es(); + if (err) + return err; + + err = ext4_init_pending(); + if (err) + goto out7; + + err = ext4_init_verity_caches(); + if (err) + goto out6; + + err = ext4_init_pageio(); + if (err) + goto out5; + + err = ext4_init_system_zone(); + if (err) + goto out4; + + err = ext4_init_sysfs(); + if (err) + goto out3; + + err = ext4_init_mballoc(); + if (err) + goto out2; + err = init_inodecache(); + if (err) + goto out1; + + err = ext4_fc_init_dentry_cache(); + if (err) + goto out05; + + register_as_ext3(); + register_as_ext2(); + err = register_filesystem(&ext4_fs_type); + if (err) + goto out; + + return 0; +out: + unregister_as_ext2(); + unregister_as_ext3(); + ext4_fc_destroy_dentry_cache(); +out05: + destroy_inodecache(); +out1: + ext4_exit_mballoc(); +out2: + ext4_exit_sysfs(); +out3: + ext4_exit_system_zone(); +out4: + ext4_exit_pageio(); +out5: + ext4_exit_verity_caches(); +out6: + ext4_exit_pending(); +out7: + ext4_exit_es(); + + return err; +} + +static void __exit ext4_exit_fs(void) +{ + ext4_destroy_lazyinit_thread(); + unregister_as_ext2(); + unregister_as_ext3(); + unregister_filesystem(&ext4_fs_type); + ext4_fc_destroy_dentry_cache(); + destroy_inodecache(); + ext4_exit_mballoc(); + ext4_exit_sysfs(); + ext4_exit_system_zone(); + ext4_exit_pageio(); + ext4_exit_verity_caches(); + ext4_exit_es(); + ext4_exit_pending(); +} + +MODULE_AUTHOR("Remy Card, Stephen Tweedie, Andrew Morton, Andreas Dilger, Theodore Ts'o and others"); +MODULE_DESCRIPTION("Fourth Extended Filesystem"); +MODULE_LICENSE("GPL"); +module_init(ext4_init_fs) +module_exit(ext4_exit_fs) diff --git a/fs/ext4/symlink.c b/fs/ext4/symlink.c new file mode 100644 index 000000000..b61226271 --- /dev/null +++ b/fs/ext4/symlink.c @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext4/symlink.c + * + * Only fast symlinks left here - the rest is done by generic code. AV, 1999 + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * from + * + * linux/fs/minix/symlink.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * ext4 symlink handling code + */ + +#include +#include +#include "ext4.h" +#include "xattr.h" + +static const char *ext4_encrypted_get_link(struct dentry *dentry, + struct inode *inode, + struct delayed_call *done) +{ + struct buffer_head *bh = NULL; + const void *caddr; + unsigned int max_size; + const char *paddr; + + if (!dentry) + return ERR_PTR(-ECHILD); + + if (ext4_inode_is_fast_symlink(inode)) { + caddr = EXT4_I(inode)->i_data; + max_size = sizeof(EXT4_I(inode)->i_data); + } else { + bh = ext4_bread(NULL, inode, 0, 0); + if (IS_ERR(bh)) + return ERR_CAST(bh); + if (!bh) { + EXT4_ERROR_INODE(inode, "bad symlink."); + return ERR_PTR(-EFSCORRUPTED); + } + caddr = bh->b_data; + max_size = inode->i_sb->s_blocksize; + } + + paddr = fscrypt_get_symlink(inode, caddr, max_size, done); + brelse(bh); + return paddr; +} + +static int ext4_encrypted_symlink_getattr(struct mnt_idmap *idmap, + const struct path *path, + struct kstat *stat, u32 request_mask, + unsigned int query_flags) +{ + ext4_getattr(idmap, path, stat, request_mask, query_flags); + + return fscrypt_symlink_getattr(path, stat); +} + +static void ext4_free_link(void *bh) +{ + brelse(bh); +} + +static const char *ext4_get_link(struct dentry *dentry, struct inode *inode, + struct delayed_call *callback) +{ + struct buffer_head *bh; + char *inline_link; + + /* + * Create a new inlined symlink is not supported, just provide a + * method to read the leftovers. + */ + if (ext4_has_inline_data(inode)) { + if (!dentry) + return ERR_PTR(-ECHILD); + + inline_link = ext4_read_inline_link(inode); + if (!IS_ERR(inline_link)) + set_delayed_call(callback, kfree_link, inline_link); + return inline_link; + } + + if (!dentry) { + bh = ext4_getblk(NULL, inode, 0, EXT4_GET_BLOCKS_CACHED_NOWAIT); + if (IS_ERR_OR_NULL(bh)) + return ERR_PTR(-ECHILD); + if (!ext4_buffer_uptodate(bh)) { + brelse(bh); + return ERR_PTR(-ECHILD); + } + } else { + bh = ext4_bread(NULL, inode, 0, 0); + if (IS_ERR(bh)) + return ERR_CAST(bh); + if (!bh) { + EXT4_ERROR_INODE(inode, "bad symlink."); + return ERR_PTR(-EFSCORRUPTED); + } + } + + set_delayed_call(callback, ext4_free_link, bh); + nd_terminate_link(bh->b_data, inode->i_size, + inode->i_sb->s_blocksize - 1); + return bh->b_data; +} + +const struct inode_operations ext4_encrypted_symlink_inode_operations = { + .get_link = ext4_encrypted_get_link, + .setattr = ext4_setattr, + .getattr = ext4_encrypted_symlink_getattr, + .listxattr = ext4_listxattr, +}; + +const struct inode_operations ext4_symlink_inode_operations = { + .get_link = ext4_get_link, + .setattr = ext4_setattr, + .getattr = ext4_getattr, + .listxattr = ext4_listxattr, +}; + +const struct inode_operations ext4_fast_symlink_inode_operations = { + .get_link = simple_get_link, + .setattr = ext4_setattr, + .getattr = ext4_getattr, + .listxattr = ext4_listxattr, +}; diff --git a/fs/ext4/sysfs.c b/fs/ext4/sysfs.c new file mode 100644 index 000000000..923b375e0 --- /dev/null +++ b/fs/ext4/sysfs.c @@ -0,0 +1,698 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext4/sysfs.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Theodore Ts'o (tytso@mit.edu) + * + */ + +#include +#include +#include +#include +#include +#include + +#include "ext4.h" +#include "ext4_jbd2.h" + +typedef enum { + attr_noop, + attr_delayed_allocation_blocks, + attr_session_write_kbytes, + attr_lifetime_write_kbytes, + attr_reserved_clusters, + attr_sra_exceeded_retry_limit, + attr_inode_readahead, + attr_trigger_test_error, + attr_first_error_time, + attr_last_error_time, + attr_clusters_in_group, + attr_mb_order, + attr_feature, + attr_pointer_pi, + attr_pointer_ui, + attr_pointer_ul, + attr_pointer_u64, + attr_pointer_u8, + attr_pointer_string, + attr_pointer_atomic, + attr_journal_task, + attr_err_report_sec, +} attr_id_t; + +typedef enum { + ptr_explicit, + ptr_ext4_sb_info_offset, + ptr_ext4_super_block_offset, +} attr_ptr_t; + +static const char proc_dirname[] = "fs/ext4"; +static struct proc_dir_entry *ext4_proc_root; + +struct ext4_attr { + struct attribute attr; + short attr_id; + short attr_ptr; + unsigned short attr_size; + union { + int offset; + void *explicit_ptr; + } u; +}; + +static ssize_t session_write_kbytes_show(struct ext4_sb_info *sbi, char *buf) +{ + struct super_block *sb = sbi->s_buddy_cache->i_sb; + + return sysfs_emit(buf, "%lu\n", + (part_stat_read(sb->s_bdev, sectors[STAT_WRITE]) - + sbi->s_sectors_written_start) >> 1); +} + +static ssize_t lifetime_write_kbytes_show(struct ext4_sb_info *sbi, char *buf) +{ + struct super_block *sb = sbi->s_buddy_cache->i_sb; + + return sysfs_emit(buf, "%llu\n", + (unsigned long long)(sbi->s_kbytes_written + + ((part_stat_read(sb->s_bdev, sectors[STAT_WRITE]) - + EXT4_SB(sb)->s_sectors_written_start) >> 1))); +} + +static ssize_t inode_readahead_blks_store(struct ext4_sb_info *sbi, + const char *buf, size_t count) +{ + unsigned long t; + int ret; + + ret = kstrtoul(skip_spaces(buf), 0, &t); + if (ret) + return ret; + + if (t && (!is_power_of_2(t) || t > 0x40000000)) + return -EINVAL; + + sbi->s_inode_readahead_blks = t; + return count; +} + +static ssize_t reserved_clusters_store(struct ext4_sb_info *sbi, + const char *buf, size_t count) +{ + unsigned long long val; + ext4_fsblk_t clusters = (ext4_blocks_count(sbi->s_es) >> + sbi->s_cluster_bits); + int ret; + + ret = kstrtoull(skip_spaces(buf), 0, &val); + if (ret || val >= clusters || (s64)val < 0) + return -EINVAL; + + atomic64_set(&sbi->s_resv_clusters, val); + return count; +} + +static ssize_t trigger_test_error(struct ext4_sb_info *sbi, + const char *buf, size_t count) +{ + int len = count; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (len && buf[len-1] == '\n') + len--; + + if (len) + ext4_error(sbi->s_sb, "%.*s", len, buf); + return count; +} + +static ssize_t err_report_sec_store(struct ext4_sb_info *sbi, + const char *buf, size_t count) +{ + unsigned long t; + int ret; + + ret = kstrtoul(skip_spaces(buf), 0, &t); + if (ret) + return ret; + + /*the maximum time interval must not exceed one year.*/ + if (t > (365*24*60*60)) + return -EINVAL; + + if (sbi->s_err_report_sec == t) /*nothing to do*/ + goto out; + else if (!sbi->s_err_report_sec && t) { + timer_setup(&sbi->s_err_report, print_daily_error_info, 0); + } else if (sbi->s_err_report_sec && !t) { + timer_delete_sync(&sbi->s_err_report); + goto out; + } + + sbi->s_err_report_sec = t; + mod_timer(&sbi->s_err_report, jiffies + secs_to_jiffies(sbi->s_err_report_sec)); + +out: + return count; +} + +static ssize_t journal_task_show(struct ext4_sb_info *sbi, char *buf) +{ + if (!sbi->s_journal) + return sysfs_emit(buf, "\n"); + return sysfs_emit(buf, "%d\n", + task_pid_vnr(sbi->s_journal->j_task)); +} + +#define EXT4_ATTR(_name,_mode,_id) \ +static struct ext4_attr ext4_attr_##_name = { \ + .attr = {.name = __stringify(_name), .mode = _mode }, \ + .attr_id = attr_##_id, \ +} + +#define EXT4_ATTR_FUNC(_name,_mode) EXT4_ATTR(_name,_mode,_name) + +#define EXT4_ATTR_FEATURE(_name) EXT4_ATTR(_name, 0444, feature) + +#define EXT4_ATTR_OFFSET(_name,_mode,_id,_struct,_elname) \ +static struct ext4_attr ext4_attr_##_name = { \ + .attr = {.name = __stringify(_name), .mode = _mode }, \ + .attr_id = attr_##_id, \ + .attr_ptr = ptr_##_struct##_offset, \ + .u = { \ + .offset = offsetof(struct _struct, _elname),\ + }, \ +} + +#define EXT4_ATTR_STRING(_name,_mode,_size,_struct,_elname) \ +static struct ext4_attr ext4_attr_##_name = { \ + .attr = {.name = __stringify(_name), .mode = _mode }, \ + .attr_id = attr_pointer_string, \ + .attr_size = _size, \ + .attr_ptr = ptr_##_struct##_offset, \ + .u = { \ + .offset = offsetof(struct _struct, _elname),\ + }, \ +} + +#define EXT4_RO_ATTR_ES_UI(_name,_elname) \ + EXT4_ATTR_OFFSET(_name, 0444, pointer_ui, ext4_super_block, _elname) + +#define EXT4_RO_ATTR_ES_U8(_name,_elname) \ + EXT4_ATTR_OFFSET(_name, 0444, pointer_u8, ext4_super_block, _elname) + +#define EXT4_RO_ATTR_ES_U64(_name,_elname) \ + EXT4_ATTR_OFFSET(_name, 0444, pointer_u64, ext4_super_block, _elname) + +#define EXT4_RO_ATTR_ES_STRING(_name,_elname,_size) \ + EXT4_ATTR_STRING(_name, 0444, _size, ext4_super_block, _elname) + +#define EXT4_RW_ATTR_SBI_PI(_name,_elname) \ + EXT4_ATTR_OFFSET(_name, 0644, pointer_pi, ext4_sb_info, _elname) + +#define EXT4_RW_ATTR_SBI_UI(_name,_elname) \ + EXT4_ATTR_OFFSET(_name, 0644, pointer_ui, ext4_sb_info, _elname) + +#define EXT4_RW_ATTR_SBI_UL(_name,_elname) \ + EXT4_ATTR_OFFSET(_name, 0644, pointer_ul, ext4_sb_info, _elname) + +#define EXT4_RO_ATTR_SBI_ATOMIC(_name,_elname) \ + EXT4_ATTR_OFFSET(_name, 0444, pointer_atomic, ext4_sb_info, _elname) + +#define EXT4_ATTR_PTR(_name,_mode,_id,_ptr) \ +static struct ext4_attr ext4_attr_##_name = { \ + .attr = {.name = __stringify(_name), .mode = _mode }, \ + .attr_id = attr_##_id, \ + .attr_ptr = ptr_explicit, \ + .u = { \ + .explicit_ptr = _ptr, \ + }, \ +} + +#define ATTR_LIST(name) &ext4_attr_##name.attr + +EXT4_ATTR_FUNC(delayed_allocation_blocks, 0444); +EXT4_ATTR_FUNC(session_write_kbytes, 0444); +EXT4_ATTR_FUNC(lifetime_write_kbytes, 0444); +EXT4_ATTR_FUNC(reserved_clusters, 0644); +EXT4_ATTR_FUNC(sra_exceeded_retry_limit, 0444); + +EXT4_ATTR_OFFSET(inode_readahead_blks, 0644, inode_readahead, + ext4_sb_info, s_inode_readahead_blks); +EXT4_ATTR_OFFSET(mb_group_prealloc, 0644, clusters_in_group, + ext4_sb_info, s_mb_group_prealloc); +EXT4_ATTR_OFFSET(mb_best_avail_max_trim_order, 0644, mb_order, + ext4_sb_info, s_mb_best_avail_max_trim_order); +EXT4_ATTR_OFFSET(err_report_sec, 0644, err_report_sec, ext4_sb_info, s_err_report_sec); +EXT4_RW_ATTR_SBI_UI(inode_goal, s_inode_goal); +EXT4_RW_ATTR_SBI_UI(mb_stats, s_mb_stats); +EXT4_RW_ATTR_SBI_UI(mb_max_to_scan, s_mb_max_to_scan); +EXT4_RW_ATTR_SBI_UI(mb_min_to_scan, s_mb_min_to_scan); +EXT4_RW_ATTR_SBI_UI(mb_order2_req, s_mb_order2_reqs); +EXT4_RW_ATTR_SBI_UI(mb_stream_req, s_mb_stream_request); +EXT4_RW_ATTR_SBI_UI(mb_max_linear_groups, s_mb_max_linear_groups); +EXT4_RW_ATTR_SBI_UI(extent_max_zeroout_kb, s_extent_max_zeroout_kb); +EXT4_ATTR(trigger_fs_error, 0200, trigger_test_error); +EXT4_RW_ATTR_SBI_PI(err_ratelimit_interval_ms, s_err_ratelimit_state.interval); +EXT4_RW_ATTR_SBI_PI(err_ratelimit_burst, s_err_ratelimit_state.burst); +EXT4_RW_ATTR_SBI_PI(warning_ratelimit_interval_ms, s_warning_ratelimit_state.interval); +EXT4_RW_ATTR_SBI_PI(warning_ratelimit_burst, s_warning_ratelimit_state.burst); +EXT4_RW_ATTR_SBI_PI(msg_ratelimit_interval_ms, s_msg_ratelimit_state.interval); +EXT4_RW_ATTR_SBI_PI(msg_ratelimit_burst, s_msg_ratelimit_state.burst); +#ifdef CONFIG_EXT4_DEBUG +EXT4_RW_ATTR_SBI_UL(simulate_fail, s_simulate_fail); +#endif +EXT4_RO_ATTR_SBI_ATOMIC(warning_count, s_warning_count); +EXT4_RO_ATTR_SBI_ATOMIC(msg_count, s_msg_count); +EXT4_RO_ATTR_ES_UI(errors_count, s_error_count); +EXT4_RO_ATTR_ES_U8(first_error_errcode, s_first_error_errcode); +EXT4_RO_ATTR_ES_U8(last_error_errcode, s_last_error_errcode); +EXT4_RO_ATTR_ES_UI(first_error_ino, s_first_error_ino); +EXT4_RO_ATTR_ES_UI(last_error_ino, s_last_error_ino); +EXT4_RO_ATTR_ES_U64(first_error_block, s_first_error_block); +EXT4_RO_ATTR_ES_U64(last_error_block, s_last_error_block); +EXT4_RO_ATTR_ES_UI(first_error_line, s_first_error_line); +EXT4_RO_ATTR_ES_UI(last_error_line, s_last_error_line); +EXT4_RO_ATTR_ES_STRING(first_error_func, s_first_error_func, 32); +EXT4_RO_ATTR_ES_STRING(last_error_func, s_last_error_func, 32); +EXT4_ATTR(first_error_time, 0444, first_error_time); +EXT4_ATTR(last_error_time, 0444, last_error_time); +EXT4_ATTR(journal_task, 0444, journal_task); +EXT4_RW_ATTR_SBI_UI(mb_prefetch, s_mb_prefetch); +EXT4_RW_ATTR_SBI_UI(mb_prefetch_limit, s_mb_prefetch_limit); +EXT4_RW_ATTR_SBI_UL(last_trim_minblks, s_last_trim_minblks); +EXT4_RW_ATTR_SBI_UI(sb_update_sec, s_sb_update_sec); +EXT4_RW_ATTR_SBI_UI(sb_update_kb, s_sb_update_kb); + +static unsigned int old_bump_val = 128; +EXT4_ATTR_PTR(max_writeback_mb_bump, 0444, pointer_ui, &old_bump_val); + +static struct attribute *ext4_attrs[] = { + ATTR_LIST(delayed_allocation_blocks), + ATTR_LIST(session_write_kbytes), + ATTR_LIST(lifetime_write_kbytes), + ATTR_LIST(reserved_clusters), + ATTR_LIST(sra_exceeded_retry_limit), + ATTR_LIST(inode_readahead_blks), + ATTR_LIST(inode_goal), + ATTR_LIST(mb_stats), + ATTR_LIST(mb_max_to_scan), + ATTR_LIST(mb_min_to_scan), + ATTR_LIST(mb_order2_req), + ATTR_LIST(mb_stream_req), + ATTR_LIST(mb_group_prealloc), + ATTR_LIST(mb_max_linear_groups), + ATTR_LIST(max_writeback_mb_bump), + ATTR_LIST(extent_max_zeroout_kb), + ATTR_LIST(trigger_fs_error), + ATTR_LIST(err_ratelimit_interval_ms), + ATTR_LIST(err_ratelimit_burst), + ATTR_LIST(warning_ratelimit_interval_ms), + ATTR_LIST(warning_ratelimit_burst), + ATTR_LIST(msg_ratelimit_interval_ms), + ATTR_LIST(msg_ratelimit_burst), + ATTR_LIST(mb_best_avail_max_trim_order), + ATTR_LIST(errors_count), + ATTR_LIST(warning_count), + ATTR_LIST(msg_count), + ATTR_LIST(first_error_ino), + ATTR_LIST(last_error_ino), + ATTR_LIST(first_error_block), + ATTR_LIST(last_error_block), + ATTR_LIST(first_error_line), + ATTR_LIST(last_error_line), + ATTR_LIST(first_error_func), + ATTR_LIST(last_error_func), + ATTR_LIST(first_error_errcode), + ATTR_LIST(last_error_errcode), + ATTR_LIST(first_error_time), + ATTR_LIST(last_error_time), + ATTR_LIST(journal_task), +#ifdef CONFIG_EXT4_DEBUG + ATTR_LIST(simulate_fail), +#endif + ATTR_LIST(mb_prefetch), + ATTR_LIST(mb_prefetch_limit), + ATTR_LIST(last_trim_minblks), + ATTR_LIST(sb_update_sec), + ATTR_LIST(sb_update_kb), + ATTR_LIST(err_report_sec), + NULL, +}; +ATTRIBUTE_GROUPS(ext4); + +/* Features this copy of ext4 supports */ +EXT4_ATTR_FEATURE(lazy_itable_init); +EXT4_ATTR_FEATURE(batched_discard); +EXT4_ATTR_FEATURE(meta_bg_resize); +#ifdef CONFIG_FS_ENCRYPTION +EXT4_ATTR_FEATURE(encryption); +EXT4_ATTR_FEATURE(test_dummy_encryption_v2); +#endif +#if IS_ENABLED(CONFIG_UNICODE) +EXT4_ATTR_FEATURE(casefold); +#endif +#ifdef CONFIG_FS_VERITY +EXT4_ATTR_FEATURE(verity); +#endif +EXT4_ATTR_FEATURE(metadata_csum_seed); +EXT4_ATTR_FEATURE(fast_commit); +#if IS_ENABLED(CONFIG_UNICODE) && defined(CONFIG_FS_ENCRYPTION) +EXT4_ATTR_FEATURE(encrypted_casefold); +#endif +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +EXT4_ATTR_FEATURE(blocksize_gt_pagesize); +#endif + +static struct attribute *ext4_feat_attrs[] = { + ATTR_LIST(lazy_itable_init), + ATTR_LIST(batched_discard), + ATTR_LIST(meta_bg_resize), +#ifdef CONFIG_FS_ENCRYPTION + ATTR_LIST(encryption), + ATTR_LIST(test_dummy_encryption_v2), +#endif +#if IS_ENABLED(CONFIG_UNICODE) + ATTR_LIST(casefold), +#endif +#ifdef CONFIG_FS_VERITY + ATTR_LIST(verity), +#endif + ATTR_LIST(metadata_csum_seed), + ATTR_LIST(fast_commit), +#if IS_ENABLED(CONFIG_UNICODE) && defined(CONFIG_FS_ENCRYPTION) + ATTR_LIST(encrypted_casefold), +#endif +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + ATTR_LIST(blocksize_gt_pagesize), +#endif + NULL, +}; +ATTRIBUTE_GROUPS(ext4_feat); + +static void *calc_ptr(struct ext4_attr *a, struct ext4_sb_info *sbi) +{ + switch (a->attr_ptr) { + case ptr_explicit: + return a->u.explicit_ptr; + case ptr_ext4_sb_info_offset: + return (void *) (((char *) sbi) + a->u.offset); + case ptr_ext4_super_block_offset: + return (void *) (((char *) sbi->s_es) + a->u.offset); + } + return NULL; +} + +static ssize_t __print_tstamp(char *buf, __le32 lo, __u8 hi) +{ + return sysfs_emit(buf, "%lld\n", + ((time64_t)hi << 32) + le32_to_cpu(lo)); +} + +#define print_tstamp(buf, es, tstamp) \ + __print_tstamp(buf, (es)->tstamp, (es)->tstamp ## _hi) + +static ssize_t ext4_generic_attr_show(struct ext4_attr *a, + struct ext4_sb_info *sbi, char *buf) +{ + void *ptr = calc_ptr(a, sbi); + + if (!ptr) + return 0; + + switch (a->attr_id) { + case attr_inode_readahead: + case attr_clusters_in_group: + case attr_mb_order: + case attr_pointer_pi: + case attr_pointer_ui: + if (a->attr_ptr == ptr_ext4_super_block_offset) + return sysfs_emit(buf, "%u\n", le32_to_cpup(ptr)); + return sysfs_emit(buf, "%u\n", *((unsigned int *) ptr)); + case attr_pointer_ul: + case attr_err_report_sec: + return sysfs_emit(buf, "%lu\n", *((unsigned long *) ptr)); + case attr_pointer_u8: + return sysfs_emit(buf, "%u\n", *((unsigned char *) ptr)); + case attr_pointer_u64: + if (a->attr_ptr == ptr_ext4_super_block_offset) + return sysfs_emit(buf, "%llu\n", le64_to_cpup(ptr)); + return sysfs_emit(buf, "%llu\n", *((unsigned long long *) ptr)); + case attr_pointer_string: + return sysfs_emit(buf, "%.*s\n", a->attr_size, (char *) ptr); + case attr_pointer_atomic: + return sysfs_emit(buf, "%d\n", atomic_read((atomic_t *) ptr)); + } + return 0; +} + +static ssize_t ext4_attr_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct ext4_sb_info *sbi = container_of(kobj, struct ext4_sb_info, + s_kobj); + struct ext4_attr *a = container_of(attr, struct ext4_attr, attr); + + switch (a->attr_id) { + case attr_delayed_allocation_blocks: + return sysfs_emit(buf, "%llu\n", + (s64) EXT4_C2B(sbi, + percpu_counter_sum(&sbi->s_dirtyclusters_counter))); + case attr_session_write_kbytes: + return session_write_kbytes_show(sbi, buf); + case attr_lifetime_write_kbytes: + return lifetime_write_kbytes_show(sbi, buf); + case attr_reserved_clusters: + return sysfs_emit(buf, "%llu\n", + (unsigned long long) + atomic64_read(&sbi->s_resv_clusters)); + case attr_sra_exceeded_retry_limit: + return sysfs_emit(buf, "%llu\n", + (unsigned long long) + percpu_counter_sum(&sbi->s_sra_exceeded_retry_limit)); + case attr_feature: + return sysfs_emit(buf, "supported\n"); + case attr_first_error_time: + return print_tstamp(buf, sbi->s_es, s_first_error_time); + case attr_last_error_time: + return print_tstamp(buf, sbi->s_es, s_last_error_time); + case attr_journal_task: + return journal_task_show(sbi, buf); + default: + return ext4_generic_attr_show(a, sbi, buf); + } +} + +static ssize_t ext4_generic_attr_store(struct ext4_attr *a, + struct ext4_sb_info *sbi, + const char *buf, size_t len) +{ + int ret; + unsigned int t; + unsigned long lt; + void *ptr = calc_ptr(a, sbi); + + if (!ptr) + return 0; + + switch (a->attr_id) { + case attr_pointer_pi: + ret = kstrtouint(skip_spaces(buf), 0, &t); + if (ret) + return ret; + if ((int)t < 0) + return -EINVAL; + *((unsigned int *) ptr) = t; + return len; + case attr_pointer_ui: + ret = kstrtouint(skip_spaces(buf), 0, &t); + if (ret) + return ret; + if (a->attr_ptr == ptr_ext4_super_block_offset) + *((__le32 *) ptr) = cpu_to_le32(t); + else + *((unsigned int *) ptr) = t; + return len; + case attr_mb_order: + ret = kstrtouint(skip_spaces(buf), 0, &t); + if (ret) + return ret; + if (t > 64) + return -EINVAL; + *((unsigned int *) ptr) = t; + return len; + case attr_clusters_in_group: + ret = kstrtouint(skip_spaces(buf), 0, &t); + if (ret) + return ret; + if (t > sbi->s_clusters_per_group) + return -EINVAL; + *((unsigned int *) ptr) = t; + return len; + case attr_pointer_ul: + ret = kstrtoul(skip_spaces(buf), 0, <); + if (ret) + return ret; + *((unsigned long *) ptr) = lt; + return len; + } + return 0; +} + +static ssize_t ext4_attr_store(struct kobject *kobj, + struct attribute *attr, + const char *buf, size_t len) +{ + struct ext4_sb_info *sbi = container_of(kobj, struct ext4_sb_info, + s_kobj); + struct ext4_attr *a = container_of(attr, struct ext4_attr, attr); + + switch (a->attr_id) { + case attr_reserved_clusters: + return reserved_clusters_store(sbi, buf, len); + case attr_inode_readahead: + return inode_readahead_blks_store(sbi, buf, len); + case attr_trigger_test_error: + return trigger_test_error(sbi, buf, len); + case attr_err_report_sec: + return err_report_sec_store(sbi, buf, len); + default: + return ext4_generic_attr_store(a, sbi, buf, len); + } +} + +static void ext4_sb_release(struct kobject *kobj) +{ + struct ext4_sb_info *sbi = container_of(kobj, struct ext4_sb_info, + s_kobj); + complete(&sbi->s_kobj_unregister); +} + +static void ext4_feat_release(struct kobject *kobj) +{ + kfree(kobj); +} + +static const struct sysfs_ops ext4_attr_ops = { + .show = ext4_attr_show, + .store = ext4_attr_store, +}; + +static const struct kobj_type ext4_sb_ktype = { + .default_groups = ext4_groups, + .sysfs_ops = &ext4_attr_ops, + .release = ext4_sb_release, +}; + +static const struct kobj_type ext4_feat_ktype = { + .default_groups = ext4_feat_groups, + .sysfs_ops = &ext4_attr_ops, + .release = ext4_feat_release, +}; + +void ext4_notify_error_sysfs(struct ext4_sb_info *sbi) +{ + mutex_lock(&sbi->s_error_notify_mutex); + if (sbi->s_kobj.state_in_sysfs) + sysfs_notify(&sbi->s_kobj, NULL, "errors_count"); + mutex_unlock(&sbi->s_error_notify_mutex); +} + +static struct kobject *ext4_root; + +static struct kobject *ext4_feat; + +int ext4_register_sysfs(struct super_block *sb) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + int err; + + init_completion(&sbi->s_kobj_unregister); + mutex_lock(&sbi->s_error_notify_mutex); + err = kobject_init_and_add(&sbi->s_kobj, &ext4_sb_ktype, ext4_root, + "%s", sb->s_id); + mutex_unlock(&sbi->s_error_notify_mutex); + if (err) { + kobject_put(&sbi->s_kobj); + wait_for_completion(&sbi->s_kobj_unregister); + return err; + } + + if (ext4_proc_root) + sbi->s_proc = proc_mkdir(sb->s_id, ext4_proc_root); + if (sbi->s_proc) { + proc_create_single_data("options", S_IRUGO, sbi->s_proc, + ext4_seq_options_show, sb); + proc_create_single_data("es_shrinker_info", S_IRUGO, + sbi->s_proc, ext4_seq_es_shrinker_info_show, + sb); + proc_create_single_data("fc_info", 0444, sbi->s_proc, + ext4_fc_info_show, sb); + proc_create_seq_data("mb_groups", S_IRUGO, sbi->s_proc, + &ext4_mb_seq_groups_ops, sb); + proc_create_single_data("mb_stats", 0444, sbi->s_proc, + ext4_seq_mb_stats_show, sb); + proc_create_seq_data("mb_structs_summary", 0444, sbi->s_proc, + &ext4_mb_seq_structs_summary_ops, sb); + } + return 0; +} + +void ext4_unregister_sysfs(struct super_block *sb) +{ + struct ext4_sb_info *sbi = EXT4_SB(sb); + + if (sbi->s_proc) + remove_proc_subtree(sb->s_id, ext4_proc_root); + + mutex_lock(&sbi->s_error_notify_mutex); + kobject_del(&sbi->s_kobj); + mutex_unlock(&sbi->s_error_notify_mutex); +} + +int __init ext4_init_sysfs(void) +{ + int ret; + + ext4_root = kobject_create_and_add("ext4", fs_kobj); + if (!ext4_root) + return -ENOMEM; + + ext4_feat = kzalloc_obj(*ext4_feat); + if (!ext4_feat) { + ret = -ENOMEM; + goto root_err; + } + + ret = kobject_init_and_add(ext4_feat, &ext4_feat_ktype, + ext4_root, "features"); + if (ret) + goto feat_err; + + ext4_proc_root = proc_mkdir(proc_dirname, NULL); + return ret; + +feat_err: + kobject_put(ext4_feat); + ext4_feat = NULL; +root_err: + kobject_put(ext4_root); + ext4_root = NULL; + return ret; +} + +void ext4_exit_sysfs(void) +{ + kobject_put(ext4_feat); + ext4_feat = NULL; + kobject_put(ext4_root); + ext4_root = NULL; + remove_proc_entry(proc_dirname, NULL); + ext4_proc_root = NULL; +} + diff --git a/fs/ext4/truncate.h b/fs/ext4/truncate.h new file mode 100644 index 000000000..ce84aa278 --- /dev/null +++ b/fs/ext4/truncate.h @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext4/truncate.h + * + * Common inline functions needed for truncate support + */ + +/* + * Truncate blocks that were not used by write. We have to truncate the + * pagecache as well so that corresponding buffers get properly unmapped. + */ +static inline void ext4_truncate_failed_write(struct inode *inode) +{ + struct address_space *mapping = inode->i_mapping; + + /* + * We don't need to call ext4_break_layouts() because the blocks we + * are truncating were never visible to userspace. + */ + filemap_invalidate_lock(mapping); + truncate_inode_pages(mapping, inode->i_size); + ext4_truncate(inode); + filemap_invalidate_unlock(mapping); +} + +/* + * Work out how many blocks we need to proceed with the next chunk of a + * truncate transaction. + */ +static inline unsigned long ext4_blocks_for_truncate(struct inode *inode) +{ + ext4_lblk_t needed; + + needed = inode->i_blocks >> (inode->i_sb->s_blocksize_bits - 9); + + /* Give ourselves just enough room to cope with inodes in which + * i_blocks is corrupt: we've seen disk corruptions in the past + * which resulted in random data in an inode which looked enough + * like a regular file for ext4 to try to delete it. Things + * will go a bit crazy if that happens, but at least we should + * try not to panic the whole kernel. */ + if (needed < 2) + needed = 2; + + /* But we need to bound the transaction so we don't overflow the + * journal. */ + if (needed > EXT4_MAX_TRANS_DATA) + needed = EXT4_MAX_TRANS_DATA; + + return EXT4_DATA_TRANS_BLOCKS(inode->i_sb) + needed; +} + diff --git a/fs/ext4/verity.c b/fs/ext4/verity.c new file mode 100644 index 000000000..ca61da53f --- /dev/null +++ b/fs/ext4/verity.c @@ -0,0 +1,391 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/ext4/verity.c: fs-verity support for ext4 + * + * Copyright 2019 Google LLC + */ + +/* + * Implementation of fsverity_operations for ext4. + * + * ext4 stores the verity metadata (Merkle tree and fsverity_descriptor) past + * the end of the file, starting at the first 64K boundary beyond i_size. This + * approach works because (a) verity files are readonly, and (b) pages fully + * beyond i_size aren't visible to userspace but can be read/written internally + * by ext4 with only some relatively small changes to ext4. This approach + * avoids having to depend on the EA_INODE feature and on rearchitecturing + * ext4's xattr support to support paging multi-gigabyte xattrs into memory, and + * to support encrypting xattrs. Note that the verity metadata *must* be + * encrypted when the file is, since it contains hashes of the plaintext data. + * + * Using a 64K boundary rather than a 4K one keeps things ready for + * architectures with 64K pages, and it doesn't necessarily waste space on-disk + * since there can be a hole between i_size and the start of the Merkle tree. + */ + +#include + +#include "ext4.h" +#include "ext4_extents.h" +#include "ext4_jbd2.h" + +static inline loff_t ext4_verity_metadata_pos(const struct inode *inode) +{ + return round_up(inode->i_size, 65536); +} + +/* + * Read some verity metadata from the inode. __vfs_read() can't be used because + * we need to read beyond i_size. + */ +static int pagecache_read(struct inode *inode, void *buf, size_t count, + loff_t pos) +{ + while (count) { + struct folio *folio; + size_t n; + + folio = read_mapping_folio(inode->i_mapping, pos >> PAGE_SHIFT, + NULL); + if (IS_ERR(folio)) + return PTR_ERR(folio); + + n = memcpy_from_file_folio(buf, folio, pos, count); + folio_put(folio); + + buf += n; + pos += n; + count -= n; + } + return 0; +} + +/* + * Write some verity metadata to the inode for FS_IOC_ENABLE_VERITY. + * kernel_write() can't be used because the file descriptor is readonly. + */ +static int pagecache_write(struct inode *inode, const void *buf, size_t count, + loff_t pos) +{ + struct address_space *mapping = inode->i_mapping; + const struct address_space_operations *aops = mapping->a_ops; + + if (pos + count > inode->i_sb->s_maxbytes) + return -EFBIG; + + while (count) { + size_t n = min_t(size_t, count, + PAGE_SIZE - offset_in_page(pos)); + struct folio *folio; + void *fsdata = NULL; + int res; + + res = aops->write_begin(NULL, mapping, pos, n, &folio, &fsdata); + if (res) + return res; + + memcpy_to_folio(folio, offset_in_folio(folio, pos), buf, n); + + res = aops->write_end(NULL, mapping, pos, n, n, folio, fsdata); + if (res < 0) + return res; + if (res != n) + return -EIO; + + buf += n; + pos += n; + count -= n; + } + return 0; +} + +static int ext4_begin_enable_verity(struct file *filp) +{ + struct inode *inode = file_inode(filp); + const int credits = 2; /* superblock and inode for ext4_orphan_add() */ + handle_t *handle; + int err; + + if (IS_DAX(inode) || ext4_test_inode_flag(inode, EXT4_INODE_DAX)) + return -EINVAL; + + if (ext4_verity_in_progress(inode)) + return -EBUSY; + + /* + * Since the file was opened readonly, we have to initialize the jbd + * inode and quotas here and not rely on ->open() doing it. This must + * be done before evicting the inline data. + */ + + err = ext4_inode_attach_jinode(inode); + if (err) + return err; + + err = dquot_initialize(inode); + if (err) + return err; + + err = ext4_convert_inline_data(inode); + if (err) + return err; + + if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) { + ext4_warning_inode(inode, + "verity is only allowed on extent-based files"); + return -EOPNOTSUPP; + } + + /* + * ext4 uses the last allocated block to find the verity descriptor, so + * we must remove any other blocks past EOF which might confuse things. + */ + err = ext4_truncate(inode); + if (err) + return err; + + handle = ext4_journal_start(inode, EXT4_HT_INODE, credits); + if (IS_ERR(handle)) + return PTR_ERR(handle); + + err = ext4_orphan_add(handle, inode); + if (err == 0) + ext4_set_inode_state(inode, EXT4_STATE_VERITY_IN_PROGRESS); + + ext4_journal_stop(handle); + return err; +} + +/* + * ext4 stores the verity descriptor beginning on the next filesystem block + * boundary after the Merkle tree. Then, the descriptor size is stored in the + * last 4 bytes of the last allocated filesystem block --- which is either the + * block in which the descriptor ends, or the next block after that if there + * weren't at least 4 bytes remaining. + * + * We can't simply store the descriptor in an xattr because it *must* be + * encrypted when ext4 encryption is used, but ext4 encryption doesn't encrypt + * xattrs. Also, if the descriptor includes a large signature blob it may be + * too large to store in an xattr without the EA_INODE feature. + */ +static int ext4_write_verity_descriptor(struct inode *inode, const void *desc, + size_t desc_size, u64 merkle_tree_size) +{ + const u64 desc_pos = round_up(ext4_verity_metadata_pos(inode) + + merkle_tree_size, i_blocksize(inode)); + const u64 desc_end = desc_pos + desc_size; + const __le32 desc_size_disk = cpu_to_le32(desc_size); + const u64 desc_size_pos = round_up(desc_end + sizeof(desc_size_disk), + i_blocksize(inode)) - + sizeof(desc_size_disk); + int err; + + err = pagecache_write(inode, desc, desc_size, desc_pos); + if (err) + return err; + + return pagecache_write(inode, &desc_size_disk, sizeof(desc_size_disk), + desc_size_pos); +} + +static int ext4_end_enable_verity(struct file *filp, const void *desc, + size_t desc_size, u64 merkle_tree_size) +{ + struct inode *inode = file_inode(filp); + const int credits = 2; /* superblock and inode for ext4_orphan_del() */ + handle_t *handle; + struct ext4_iloc iloc; + int err = 0; + + /* + * If an error already occurred (which fs/verity/ signals by passing + * desc == NULL), then only clean-up is needed. + */ + if (desc == NULL) + goto cleanup; + + /* Append the verity descriptor. */ + err = ext4_write_verity_descriptor(inode, desc, desc_size, + merkle_tree_size); + if (err) + goto cleanup; + + /* + * Write all pages (both data and verity metadata). Note that this must + * happen before clearing EXT4_STATE_VERITY_IN_PROGRESS; otherwise pages + * beyond i_size won't be written properly. For crash consistency, this + * also must happen before the verity inode flag gets persisted. + */ + err = filemap_write_and_wait(inode->i_mapping); + if (err) + goto cleanup; + + /* + * Finally, set the verity inode flag and remove the inode from the + * orphan list (in a single transaction). + */ + + handle = ext4_journal_start(inode, EXT4_HT_INODE, credits); + if (IS_ERR(handle)) { + err = PTR_ERR(handle); + goto cleanup; + } + + ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_VERITY, handle); + + err = ext4_orphan_del(handle, inode); + if (err) + goto stop_and_cleanup; + + err = ext4_reserve_inode_write(handle, inode, &iloc); + if (err) + goto stop_and_cleanup; + + ext4_set_inode_flag(inode, EXT4_INODE_VERITY); + ext4_set_inode_flags(inode, false); + err = ext4_mark_iloc_dirty(handle, inode, &iloc); + if (err) + goto stop_and_cleanup; + + ext4_journal_stop(handle); + + ext4_clear_inode_state(inode, EXT4_STATE_VERITY_IN_PROGRESS); + return 0; + +stop_and_cleanup: + ext4_journal_stop(handle); +cleanup: + /* + * Verity failed to be enabled, so clean up by truncating any verity + * metadata that was written beyond i_size (both from cache and from + * disk), removing the inode from the orphan list (if it wasn't done + * already), and clearing EXT4_STATE_VERITY_IN_PROGRESS. + */ + truncate_inode_pages(inode->i_mapping, inode->i_size); + ext4_truncate(inode); + ext4_orphan_del(NULL, inode); + ext4_clear_inode_state(inode, EXT4_STATE_VERITY_IN_PROGRESS); + return err; +} + +static int ext4_get_verity_descriptor_location(struct inode *inode, + size_t *desc_size_ret, + u64 *desc_pos_ret) +{ + struct ext4_ext_path *path; + struct ext4_extent *last_extent; + u32 end_lblk; + u64 desc_size_pos; + __le32 desc_size_disk; + u32 desc_size; + u64 desc_pos; + int err; + + /* + * Descriptor size is in last 4 bytes of last allocated block. + * See ext4_write_verity_descriptor(). + */ + + if (!ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) { + EXT4_ERROR_INODE(inode, "verity file doesn't use extents"); + return -EFSCORRUPTED; + } + + path = ext4_find_extent(inode, EXT_MAX_BLOCKS - 1, NULL, 0); + if (IS_ERR(path)) + return PTR_ERR(path); + + last_extent = path[path->p_depth].p_ext; + if (!last_extent) { + EXT4_ERROR_INODE(inode, "verity file has no extents"); + ext4_free_ext_path(path); + return -EFSCORRUPTED; + } + + end_lblk = le32_to_cpu(last_extent->ee_block) + + ext4_ext_get_actual_len(last_extent); + desc_size_pos = EXT4_LBLK_TO_B(inode, end_lblk); + ext4_free_ext_path(path); + + if (desc_size_pos < sizeof(desc_size_disk)) + goto bad; + desc_size_pos -= sizeof(desc_size_disk); + + err = pagecache_read(inode, &desc_size_disk, sizeof(desc_size_disk), + desc_size_pos); + if (err) + return err; + desc_size = le32_to_cpu(desc_size_disk); + + /* + * The descriptor is stored just before the desc_size_disk, but starting + * on a filesystem block boundary. + */ + + if (desc_size > INT_MAX || desc_size > desc_size_pos) + goto bad; + + desc_pos = round_down(desc_size_pos - desc_size, i_blocksize(inode)); + if (desc_pos < ext4_verity_metadata_pos(inode)) + goto bad; + + *desc_size_ret = desc_size; + *desc_pos_ret = desc_pos; + return 0; + +bad: + EXT4_ERROR_INODE(inode, "verity file corrupted; can't find descriptor"); + return -EFSCORRUPTED; +} + +static int ext4_get_verity_descriptor(struct inode *inode, void *buf, + size_t buf_size) +{ + size_t desc_size = 0; + u64 desc_pos = 0; + int err; + + err = ext4_get_verity_descriptor_location(inode, &desc_size, &desc_pos); + if (err) + return err; + + if (buf_size) { + if (desc_size > buf_size) + return -ERANGE; + err = pagecache_read(inode, buf, desc_size, desc_pos); + if (err) + return err; + } + return desc_size; +} + +static struct page *ext4_read_merkle_tree_page(struct inode *inode, + pgoff_t index) +{ + index += ext4_verity_metadata_pos(inode) >> PAGE_SHIFT; + return generic_read_merkle_tree_page(inode, index); +} + +static void ext4_readahead_merkle_tree(struct inode *inode, pgoff_t index, + unsigned long nr_pages) +{ + index += ext4_verity_metadata_pos(inode) >> PAGE_SHIFT; + generic_readahead_merkle_tree(inode, index, nr_pages); +} + +static int ext4_write_merkle_tree_block(struct file *file, const void *buf, + u64 pos, unsigned int size) +{ + pos += ext4_verity_metadata_pos(file_inode(file)); + + return pagecache_write(file_inode(file), buf, size, pos); +} + +const struct fsverity_operations ext4_verityops = { + .begin_enable_verity = ext4_begin_enable_verity, + .end_enable_verity = ext4_end_enable_verity, + .get_verity_descriptor = ext4_get_verity_descriptor, + .read_merkle_tree_page = ext4_read_merkle_tree_page, + .readahead_merkle_tree = ext4_readahead_merkle_tree, + .write_merkle_tree_block = ext4_write_merkle_tree_block, +}; diff --git a/fs/ext4/xattr.c b/fs/ext4/xattr.c new file mode 100644 index 000000000..5c310747b --- /dev/null +++ b/fs/ext4/xattr.c @@ -0,0 +1,3238 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext4/xattr.c + * + * Copyright (C) 2001-2003 Andreas Gruenbacher, + * + * Fix by Harrison Xing . + * Ext4 code with a lot of help from Eric Jarman . + * Extended attributes for symlinks and special files added per + * suggestion of Luka Renko . + * xattr consolidation Copyright (c) 2004 James Morris , + * Red Hat Inc. + * ea-in-inode support by Alex Tomas aka bzzz + * and Andreas Gruenbacher . + */ + +/* + * Extended attributes are stored directly in inodes (on file systems with + * inodes bigger than 128 bytes) and on additional disk blocks. The i_file_acl + * field contains the block number if an inode uses an additional block. All + * attributes must fit in the inode and one additional block. Blocks that + * contain the identical set of attributes may be shared among several inodes. + * Identical blocks are detected by keeping a cache of blocks that have + * recently been accessed. + * + * The attributes in inodes and on blocks have a different header; the entries + * are stored in the same format: + * + * +------------------+ + * | header | + * | entry 1 | | + * | entry 2 | | growing downwards + * | entry 3 | v + * | four null bytes | + * | . . . | + * | value 1 | ^ + * | value 3 | | growing upwards + * | value 2 | | + * +------------------+ + * + * The header is followed by multiple entry descriptors. In disk blocks, the + * entry descriptors are kept sorted. In inodes, they are unsorted. The + * attribute values are aligned to the end of the block in no specific order. + * + * Locking strategy + * ---------------- + * EXT4_I(inode)->i_file_acl is protected by EXT4_I(inode)->xattr_sem. + * EA blocks are only changed if they are exclusive to an inode, so + * holding xattr_sem also means that nothing but the EA block's reference + * count can change. Multiple writers to the same block are synchronized + * by the buffer lock. + */ + +#include +#include +#include +#include +#include +#include +#include "ext4_jbd2.h" +#include "ext4.h" +#include "xattr.h" +#include "acl.h" + +#ifdef EXT4_XATTR_DEBUG +# define ea_idebug(inode, fmt, ...) \ + printk(KERN_DEBUG "inode %s:%llu: " fmt "\n", \ + inode->i_sb->s_id, inode->i_ino, ##__VA_ARGS__) +# define ea_bdebug(bh, fmt, ...) \ + printk(KERN_DEBUG "block %pg:%lu: " fmt "\n", \ + bh->b_bdev, (unsigned long)bh->b_blocknr, ##__VA_ARGS__) +#else +# define ea_idebug(inode, fmt, ...) no_printk(fmt, ##__VA_ARGS__) +# define ea_bdebug(bh, fmt, ...) no_printk(fmt, ##__VA_ARGS__) +#endif + +static void ext4_xattr_block_cache_insert(struct mb_cache *, + struct buffer_head *); +static struct buffer_head * +ext4_xattr_block_cache_find(struct inode *, struct ext4_xattr_header *, + struct mb_cache_entry **); +static __le32 ext4_xattr_hash_entry(char *name, size_t name_len, __le32 *value, + size_t value_count); +static __le32 ext4_xattr_hash_entry_signed(char *name, size_t name_len, __le32 *value, + size_t value_count); +static void ext4_xattr_rehash(struct ext4_xattr_header *); + +static const struct xattr_handler * const ext4_xattr_handler_map[] = { + [EXT4_XATTR_INDEX_USER] = &ext4_xattr_user_handler, +#ifdef CONFIG_EXT4_FS_POSIX_ACL + [EXT4_XATTR_INDEX_POSIX_ACL_ACCESS] = &nop_posix_acl_access, + [EXT4_XATTR_INDEX_POSIX_ACL_DEFAULT] = &nop_posix_acl_default, +#endif + [EXT4_XATTR_INDEX_TRUSTED] = &ext4_xattr_trusted_handler, +#ifdef CONFIG_EXT4_FS_SECURITY + [EXT4_XATTR_INDEX_SECURITY] = &ext4_xattr_security_handler, +#endif + [EXT4_XATTR_INDEX_HURD] = &ext4_xattr_hurd_handler, +}; + +const struct xattr_handler * const ext4_xattr_handlers[] = { + &ext4_xattr_user_handler, + &ext4_xattr_trusted_handler, +#ifdef CONFIG_EXT4_FS_SECURITY + &ext4_xattr_security_handler, +#endif + &ext4_xattr_hurd_handler, + NULL +}; + +#define EA_BLOCK_CACHE(inode) (((struct ext4_sb_info *) \ + inode->i_sb->s_fs_info)->s_ea_block_cache) + +#define EA_INODE_CACHE(inode) (((struct ext4_sb_info *) \ + inode->i_sb->s_fs_info)->s_ea_inode_cache) + +#ifdef CONFIG_LOCKDEP +void ext4_xattr_inode_set_class(struct inode *ea_inode) +{ + struct ext4_inode_info *ei = EXT4_I(ea_inode); + + lockdep_set_subclass(&ea_inode->i_rwsem, 1); + (void) ei; /* shut up clang warning if !CONFIG_LOCKDEP */ + lockdep_set_subclass(&ei->i_data_sem, I_DATA_SEM_EA); +} +#endif + +static __le32 ext4_xattr_block_csum(struct inode *inode, + sector_t block_nr, + struct ext4_xattr_header *hdr) +{ + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + __u32 csum; + __le64 dsk_block_nr = cpu_to_le64(block_nr); + __u32 dummy_csum = 0; + int offset = offsetof(struct ext4_xattr_header, h_checksum); + + csum = ext4_chksum(sbi->s_csum_seed, (__u8 *)&dsk_block_nr, + sizeof(dsk_block_nr)); + csum = ext4_chksum(csum, (__u8 *)hdr, offset); + csum = ext4_chksum(csum, (__u8 *)&dummy_csum, sizeof(dummy_csum)); + offset += sizeof(dummy_csum); + csum = ext4_chksum(csum, (__u8 *)hdr + offset, + EXT4_BLOCK_SIZE(inode->i_sb) - offset); + + return cpu_to_le32(csum); +} + +static int ext4_xattr_block_csum_verify(struct inode *inode, + struct buffer_head *bh) +{ + struct ext4_xattr_header *hdr = BHDR(bh); + int ret = 1; + + if (ext4_has_feature_metadata_csum(inode->i_sb)) { + lock_buffer(bh); + ret = (hdr->h_checksum == ext4_xattr_block_csum(inode, + bh->b_blocknr, hdr)); + unlock_buffer(bh); + } + return ret; +} + +static void ext4_xattr_block_csum_set(struct inode *inode, + struct buffer_head *bh) +{ + if (ext4_has_feature_metadata_csum(inode->i_sb)) + BHDR(bh)->h_checksum = ext4_xattr_block_csum(inode, + bh->b_blocknr, BHDR(bh)); +} + +static inline const char *ext4_xattr_prefix(int name_index, + struct dentry *dentry) +{ + const struct xattr_handler *handler = NULL; + + if (name_index > 0 && name_index < ARRAY_SIZE(ext4_xattr_handler_map)) + handler = ext4_xattr_handler_map[name_index]; + + if (!xattr_handler_can_list(handler, dentry)) + return NULL; + + return xattr_prefix(handler); +} + +static int +check_xattrs(struct inode *inode, struct buffer_head *bh, + struct ext4_xattr_entry *entry, void *end, void *value_start, + const char *function, unsigned int line) +{ + struct ext4_xattr_entry *e = entry; + int err = -EFSCORRUPTED; + char *err_str; + + if (bh) { + if (BHDR(bh)->h_magic != cpu_to_le32(EXT4_XATTR_MAGIC) || + BHDR(bh)->h_blocks != cpu_to_le32(1)) { + err_str = "invalid header"; + goto errout; + } + if (buffer_verified(bh)) + return 0; + if (!ext4_xattr_block_csum_verify(inode, bh)) { + err = -EFSBADCRC; + err_str = "invalid checksum"; + goto errout; + } + } else { + struct ext4_xattr_ibody_header *header = value_start; + + header -= 1; + if (end - (void *)header < sizeof(*header) + sizeof(u32)) { + err_str = "in-inode xattr block too small"; + goto errout; + } + if (header->h_magic != cpu_to_le32(EXT4_XATTR_MAGIC)) { + err_str = "bad magic number in in-inode xattr"; + goto errout; + } + } + + /* Find the end of the names list */ + while (!IS_LAST_ENTRY(e)) { + struct ext4_xattr_entry *next = EXT4_XATTR_NEXT(e); + if ((void *)next + sizeof(u32) > end) { + err_str = "e_name out of bounds"; + goto errout; + } + if (strnlen(e->e_name, e->e_name_len) != e->e_name_len) { + err_str = "bad e_name length"; + goto errout; + } + e = next; + } + + /* Check the values */ + while (!IS_LAST_ENTRY(entry)) { + u32 size = le32_to_cpu(entry->e_value_size); + unsigned long ea_ino = le32_to_cpu(entry->e_value_inum); + + if (!ext4_has_feature_ea_inode(inode->i_sb) && ea_ino) { + err_str = "ea_inode specified without ea_inode feature enabled"; + goto errout; + } + if (ea_ino && ((ea_ino == EXT4_ROOT_INO) || + !ext4_valid_inum(inode->i_sb, ea_ino))) { + err_str = "invalid ea_ino"; + goto errout; + } + if (ea_ino && !size) { + err_str = "invalid size in ea xattr"; + goto errout; + } + if (size > EXT4_XATTR_SIZE_MAX) { + err_str = "e_value size too large"; + goto errout; + } + + if (size != 0 && entry->e_value_inum == 0) { + u16 offs = le16_to_cpu(entry->e_value_offs); + void *value; + + /* + * The value cannot overlap the names, and the value + * with padding cannot extend beyond 'end'. Check both + * the padded and unpadded sizes, since the size may + * overflow to 0 when adding padding. + */ + if (offs > end - value_start) { + err_str = "e_value out of bounds"; + goto errout; + } + value = value_start + offs; + if (value < (void *)e + sizeof(u32) || + size > end - value || + EXT4_XATTR_SIZE(size) > end - value) { + err_str = "overlapping e_value "; + goto errout; + } + } + entry = EXT4_XATTR_NEXT(entry); + } + if (bh) + set_buffer_verified(bh); + return 0; + +errout: + if (bh) + __ext4_error_inode(inode, function, line, 0, -err, + "corrupted xattr block %llu: %s", + (unsigned long long) bh->b_blocknr, + err_str); + else + __ext4_error_inode(inode, function, line, 0, -err, + "corrupted in-inode xattr: %s", err_str); + return err; +} + +static inline int +__ext4_xattr_check_block(struct inode *inode, struct buffer_head *bh, + const char *function, unsigned int line) +{ + return check_xattrs(inode, bh, BFIRST(bh), bh->b_data + bh->b_size, + bh->b_data, function, line); +} + +#define ext4_xattr_check_block(inode, bh) \ + __ext4_xattr_check_block((inode), (bh), __func__, __LINE__) + + +int +__xattr_check_inode(struct inode *inode, struct ext4_xattr_ibody_header *header, + void *end, const char *function, unsigned int line) +{ + return check_xattrs(inode, NULL, IFIRST(header), end, IFIRST(header), + function, line); +} + +static int +xattr_find_entry(struct inode *inode, struct ext4_xattr_entry **pentry, + void *end, int name_index, const char *name, int sorted) +{ + struct ext4_xattr_entry *entry, *next; + size_t name_len; + int cmp = 1; + + if (name == NULL) + return -EINVAL; + name_len = strlen(name); + for (entry = *pentry; !IS_LAST_ENTRY(entry); entry = next) { + next = EXT4_XATTR_NEXT(entry); + if ((void *) next >= end) { + EXT4_ERROR_INODE(inode, "corrupted xattr entries"); + return -EFSCORRUPTED; + } + cmp = name_index - entry->e_name_index; + if (!cmp) + cmp = name_len - entry->e_name_len; + if (!cmp) + cmp = memcmp(name, entry->e_name, name_len); + if (!cmp || (cmp < 0 && sorted)) + break; + } + *pentry = entry; + return cmp ? -ENODATA : 0; +} + +static u32 +ext4_xattr_inode_hash(struct ext4_sb_info *sbi, const void *buffer, size_t size) +{ + return ext4_chksum(sbi->s_csum_seed, buffer, size); +} + +static u64 ext4_xattr_inode_get_ref(struct inode *ea_inode) +{ + return ((u64) inode_get_ctime_sec(ea_inode) << 32) | + (u32) inode_peek_iversion_raw(ea_inode); +} + +static void ext4_xattr_inode_set_ref(struct inode *ea_inode, u64 ref_count) +{ + inode_set_ctime(ea_inode, (u32)(ref_count >> 32), 0); + inode_set_iversion_raw(ea_inode, ref_count & 0xffffffff); +} + +static u32 ext4_xattr_inode_get_hash(struct inode *ea_inode) +{ + return (u32) inode_get_atime_sec(ea_inode); +} + +static void ext4_xattr_inode_set_hash(struct inode *ea_inode, u32 hash) +{ + inode_set_atime(ea_inode, hash, 0); +} + +/* + * Read the EA value from an inode. + */ +static int ext4_xattr_inode_read(struct inode *ea_inode, void *buf, size_t size) +{ + int blocksize = 1 << ea_inode->i_blkbits; + int bh_count = (size + blocksize - 1) >> ea_inode->i_blkbits; + int tail_size = (size % blocksize) ?: blocksize; + struct buffer_head *bhs_inline[8]; + struct buffer_head **bhs = bhs_inline; + int i, ret; + + if (bh_count > ARRAY_SIZE(bhs_inline)) { + bhs = kmalloc_objs(*bhs, bh_count, GFP_NOFS); + if (!bhs) + return -ENOMEM; + } + + ret = ext4_bread_batch(ea_inode, 0 /* block */, bh_count, + true /* wait */, bhs); + if (ret) + goto free_bhs; + + for (i = 0; i < bh_count; i++) { + /* There shouldn't be any holes in ea_inode. */ + if (!bhs[i]) { + ret = -EFSCORRUPTED; + goto put_bhs; + } + memcpy((char *)buf + blocksize * i, bhs[i]->b_data, + i < bh_count - 1 ? blocksize : tail_size); + } + ret = 0; +put_bhs: + for (i = 0; i < bh_count; i++) + brelse(bhs[i]); +free_bhs: + if (bhs != bhs_inline) + kfree(bhs); + return ret; +} + +#define EXT4_XATTR_INODE_GET_PARENT(inode) ((__u32)(inode_get_mtime_sec(inode))) + +static int ext4_xattr_inode_iget(struct inode *parent, unsigned long ea_ino, + u32 ea_inode_hash, struct inode **ea_inode) +{ + struct inode *inode; + int err; + + /* + * We have to check for this corruption early as otherwise + * iget_locked() could wait indefinitely for the state of our + * parent inode. + */ + if (parent->i_ino == ea_ino) { + ext4_error(parent->i_sb, + "Parent and EA inode have the same ino %lu", ea_ino); + return -EFSCORRUPTED; + } + + inode = ext4_iget(parent->i_sb, ea_ino, EXT4_IGET_EA_INODE); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + ext4_error(parent->i_sb, + "error while reading EA inode %lu err=%d", ea_ino, + err); + return err; + } + ext4_xattr_inode_set_class(inode); + + /* + * Check whether this is an old Lustre-style xattr inode. Lustre + * implementation does not have hash validation, rather it has a + * backpointer from ea_inode to the parent inode. + */ + if (ea_inode_hash != ext4_xattr_inode_get_hash(inode) && + EXT4_XATTR_INODE_GET_PARENT(inode) == parent->i_ino && + inode->i_generation == parent->i_generation) { + ext4_set_inode_state(inode, EXT4_STATE_LUSTRE_EA_INODE); + ext4_xattr_inode_set_ref(inode, 1); + } else { + inode_lock_nested(inode, I_MUTEX_XATTR); + inode->i_flags |= S_NOQUOTA; + inode_unlock(inode); + } + + /* + * Since this function resolves references from active xattr entries, + * the EA inode must be in active state (i_nlink=1, ref_count>0). + * i_nlink > 1, i_nlink == 0 (dangling reference), or ref_count == 0 + * (inconsistent with an active entry) all indicate on-disk corruption. + */ + if (inode->i_nlink != 1 || !ext4_xattr_inode_get_ref(inode)) { + ext4_error(parent->i_sb, + "EA inode %lu has unexpected i_nlink=%u ref_count=%llu", + ea_ino, inode->i_nlink, + ext4_xattr_inode_get_ref(inode)); + ext4_put_ea_inode(inode); + return -EFSCORRUPTED; + } + + *ea_inode = inode; + return 0; +} + +/* Remove entry from mbcache when EA inode is getting evicted */ +void ext4_evict_ea_inode(struct inode *inode) +{ + struct mb_cache_entry *oe; + + if (!EA_INODE_CACHE(inode)) + return; + /* Wait for entry to get unused so that we can remove it */ + while ((oe = mb_cache_entry_delete_or_get(EA_INODE_CACHE(inode), + ext4_xattr_inode_get_hash(inode), inode->i_ino))) { + mb_cache_entry_wait_unused(oe); + mb_cache_entry_put(EA_INODE_CACHE(inode), oe); + } +} + +static int +ext4_xattr_inode_verify_hashes(struct inode *ea_inode, + struct ext4_xattr_entry *entry, void *buffer, + size_t size) +{ + u32 hash; + + /* Verify stored hash matches calculated hash. */ + hash = ext4_xattr_inode_hash(EXT4_SB(ea_inode->i_sb), buffer, size); + if (hash != ext4_xattr_inode_get_hash(ea_inode)) + return -EFSCORRUPTED; + + if (entry) { + __le32 e_hash, tmp_data; + + /* Verify entry hash. */ + tmp_data = cpu_to_le32(hash); + e_hash = ext4_xattr_hash_entry(entry->e_name, entry->e_name_len, + &tmp_data, 1); + /* All good? */ + if (e_hash == entry->e_hash) + return 0; + + /* + * Not good. Maybe the entry hash was calculated + * using the buggy signed char version? + */ + e_hash = ext4_xattr_hash_entry_signed(entry->e_name, entry->e_name_len, + &tmp_data, 1); + /* Still no match - bad */ + if (e_hash != entry->e_hash) + return -EFSCORRUPTED; + + /* Let people know about old hash */ + pr_warn_once("ext4: filesystem with signed xattr name hash"); + } + return 0; +} + +/* + * Read xattr value from the EA inode. + */ +static int +ext4_xattr_inode_get(struct inode *inode, struct ext4_xattr_entry *entry, + void *buffer, size_t size) +{ + struct mb_cache *ea_inode_cache = EA_INODE_CACHE(inode); + struct inode *ea_inode; + int err; + + err = ext4_xattr_inode_iget(inode, le32_to_cpu(entry->e_value_inum), + le32_to_cpu(entry->e_hash), &ea_inode); + if (err) { + ea_inode = NULL; + goto out; + } + + if (i_size_read(ea_inode) != size) { + ext4_warning_inode(ea_inode, + "ea_inode file size=%llu entry size=%zu", + i_size_read(ea_inode), size); + err = -EFSCORRUPTED; + goto out; + } + + err = ext4_xattr_inode_read(ea_inode, buffer, size); + if (err) + goto out; + + if (!ext4_test_inode_state(ea_inode, EXT4_STATE_LUSTRE_EA_INODE)) { + err = ext4_xattr_inode_verify_hashes(ea_inode, entry, buffer, + size); + if (err) { + ext4_warning_inode(ea_inode, + "EA inode hash validation failed"); + goto out; + } + + if (ea_inode_cache) + mb_cache_entry_create(ea_inode_cache, GFP_NOFS, + ext4_xattr_inode_get_hash(ea_inode), + ea_inode->i_ino, true /* reusable */); + } +out: + ext4_put_ea_inode(ea_inode); + return err; +} + +static int +ext4_xattr_block_get(struct inode *inode, int name_index, const char *name, + void *buffer, size_t buffer_size) +{ + struct buffer_head *bh = NULL; + struct ext4_xattr_entry *entry; + size_t size; + void *end; + int error; + struct mb_cache *ea_block_cache = EA_BLOCK_CACHE(inode); + + ea_idebug(inode, "name=%d.%s, buffer=%p, buffer_size=%ld", + name_index, name, buffer, (long)buffer_size); + + if (!EXT4_I(inode)->i_file_acl) + return -ENODATA; + ea_idebug(inode, "reading block %llu", + (unsigned long long)EXT4_I(inode)->i_file_acl); + bh = ext4_sb_bread(inode->i_sb, EXT4_I(inode)->i_file_acl, REQ_PRIO); + if (IS_ERR(bh)) + return PTR_ERR(bh); + ea_bdebug(bh, "b_count=%d, refcount=%d", + atomic_read(&(bh->b_count)), le32_to_cpu(BHDR(bh)->h_refcount)); + error = ext4_xattr_check_block(inode, bh); + if (error) + goto cleanup; + ext4_xattr_block_cache_insert(ea_block_cache, bh); + entry = BFIRST(bh); + end = bh->b_data + bh->b_size; + error = xattr_find_entry(inode, &entry, end, name_index, name, 1); + if (error) + goto cleanup; + size = le32_to_cpu(entry->e_value_size); + error = -ERANGE; + if (unlikely(size > EXT4_XATTR_SIZE_MAX)) + goto cleanup; + if (buffer) { + if (size > buffer_size) + goto cleanup; + if (entry->e_value_inum) { + error = ext4_xattr_inode_get(inode, entry, buffer, + size); + if (error) + goto cleanup; + } else { + u16 offset = le16_to_cpu(entry->e_value_offs); + void *p = bh->b_data + offset; + + if (unlikely(p + size > end)) + goto cleanup; + memcpy(buffer, p, size); + } + } + error = size; + +cleanup: + brelse(bh); + return error; +} + +int +ext4_xattr_ibody_get(struct inode *inode, int name_index, const char *name, + void *buffer, size_t buffer_size) +{ + struct ext4_xattr_ibody_header *header; + struct ext4_xattr_entry *entry; + struct ext4_inode *raw_inode; + struct ext4_iloc iloc; + size_t size; + void *end; + int error; + + if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR)) + return -ENODATA; + error = ext4_get_inode_loc(inode, &iloc); + if (error) + return error; + raw_inode = ext4_raw_inode(&iloc); + header = IHDR(inode, raw_inode); + end = ITAIL(inode, raw_inode); + entry = IFIRST(header); + error = xattr_find_entry(inode, &entry, end, name_index, name, 0); + if (error) + goto cleanup; + size = le32_to_cpu(entry->e_value_size); + error = -ERANGE; + if (unlikely(size > EXT4_XATTR_SIZE_MAX)) + goto cleanup; + if (buffer) { + if (size > buffer_size) + goto cleanup; + if (entry->e_value_inum) { + error = ext4_xattr_inode_get(inode, entry, buffer, + size); + if (error) + goto cleanup; + } else { + u16 offset = le16_to_cpu(entry->e_value_offs); + void *p = (void *)IFIRST(header) + offset; + + if (unlikely(p + size > end)) + goto cleanup; + memcpy(buffer, p, size); + } + } + error = size; + +cleanup: + brelse(iloc.bh); + return error; +} + +/* + * ext4_xattr_get() + * + * Copy an extended attribute into the buffer + * provided, or compute the buffer size required. + * Buffer is NULL to compute the size of the buffer required. + * + * Returns a negative error number on failure, or the number of bytes + * used / required on success. + */ +int +ext4_xattr_get(struct inode *inode, int name_index, const char *name, + void *buffer, size_t buffer_size) +{ + int error; + + if (unlikely(ext4_forced_shutdown(inode->i_sb))) + return -EIO; + + if (strlen(name) > 255) + return -ERANGE; + + down_read(&EXT4_I(inode)->xattr_sem); + error = ext4_xattr_ibody_get(inode, name_index, name, buffer, + buffer_size); + if (error == -ENODATA) + error = ext4_xattr_block_get(inode, name_index, name, buffer, + buffer_size); + up_read(&EXT4_I(inode)->xattr_sem); + return error; +} + +static int +ext4_xattr_list_entries(struct dentry *dentry, struct ext4_xattr_entry *entry, + char *buffer, size_t buffer_size) +{ + size_t rest = buffer_size; + + for (; !IS_LAST_ENTRY(entry); entry = EXT4_XATTR_NEXT(entry)) { + const char *prefix; + + prefix = ext4_xattr_prefix(entry->e_name_index, dentry); + if (prefix) { + size_t prefix_len = strlen(prefix); + size_t size = prefix_len + entry->e_name_len + 1; + + if (buffer) { + if (size > rest) + return -ERANGE; + memcpy(buffer, prefix, prefix_len); + buffer += prefix_len; + memcpy(buffer, entry->e_name, entry->e_name_len); + buffer += entry->e_name_len; + *buffer++ = 0; + } + rest -= size; + } + } + return buffer_size - rest; /* total size */ +} + +static int +ext4_xattr_block_list(struct dentry *dentry, char *buffer, size_t buffer_size) +{ + struct inode *inode = d_inode(dentry); + struct buffer_head *bh = NULL; + int error; + + ea_idebug(inode, "buffer=%p, buffer_size=%ld", + buffer, (long)buffer_size); + + if (!EXT4_I(inode)->i_file_acl) + return 0; + ea_idebug(inode, "reading block %llu", + (unsigned long long)EXT4_I(inode)->i_file_acl); + bh = ext4_sb_bread(inode->i_sb, EXT4_I(inode)->i_file_acl, REQ_PRIO); + if (IS_ERR(bh)) + return PTR_ERR(bh); + ea_bdebug(bh, "b_count=%d, refcount=%d", + atomic_read(&(bh->b_count)), le32_to_cpu(BHDR(bh)->h_refcount)); + error = ext4_xattr_check_block(inode, bh); + if (error) + goto cleanup; + ext4_xattr_block_cache_insert(EA_BLOCK_CACHE(inode), bh); + error = ext4_xattr_list_entries(dentry, BFIRST(bh), buffer, + buffer_size); +cleanup: + brelse(bh); + return error; +} + +static int +ext4_xattr_ibody_list(struct dentry *dentry, char *buffer, size_t buffer_size) +{ + struct inode *inode = d_inode(dentry); + struct ext4_xattr_ibody_header *header; + struct ext4_inode *raw_inode; + struct ext4_iloc iloc; + int error; + + if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR)) + return 0; + error = ext4_get_inode_loc(inode, &iloc); + if (error) + return error; + raw_inode = ext4_raw_inode(&iloc); + header = IHDR(inode, raw_inode); + error = ext4_xattr_list_entries(dentry, IFIRST(header), + buffer, buffer_size); + + brelse(iloc.bh); + return error; +} + +/* + * Inode operation listxattr() + * + * d_inode(dentry)->i_rwsem: don't care + * + * Copy a list of attribute names into the buffer + * provided, or compute the buffer size required. + * Buffer is NULL to compute the size of the buffer required. + * + * Returns a negative error number on failure, or the number of bytes + * used / required on success. + */ +ssize_t +ext4_listxattr(struct dentry *dentry, char *buffer, size_t buffer_size) +{ + int ret, ret2; + + down_read(&EXT4_I(d_inode(dentry))->xattr_sem); + ret = ret2 = ext4_xattr_ibody_list(dentry, buffer, buffer_size); + if (ret < 0) + goto errout; + if (buffer) { + buffer += ret; + buffer_size -= ret; + } + ret = ext4_xattr_block_list(dentry, buffer, buffer_size); + if (ret < 0) + goto errout; + ret += ret2; +errout: + up_read(&EXT4_I(d_inode(dentry))->xattr_sem); + return ret; +} + +/* + * If the EXT4_FEATURE_COMPAT_EXT_ATTR feature of this file system is + * not set, set it. + */ +static void ext4_xattr_update_super_block(handle_t *handle, + struct super_block *sb) +{ + if (ext4_has_feature_xattr(sb)) + return; + + BUFFER_TRACE(EXT4_SB(sb)->s_sbh, "get_write_access"); + if (ext4_journal_get_write_access(handle, sb, EXT4_SB(sb)->s_sbh, + EXT4_JTR_NONE) == 0) { + lock_buffer(EXT4_SB(sb)->s_sbh); + ext4_set_feature_xattr(sb); + ext4_superblock_csum_set(sb); + unlock_buffer(EXT4_SB(sb)->s_sbh); + ext4_handle_dirty_metadata(handle, NULL, EXT4_SB(sb)->s_sbh); + } +} + +int ext4_get_inode_usage(struct inode *inode, qsize_t *usage) +{ + struct ext4_iloc iloc = { .bh = NULL }; + struct buffer_head *bh = NULL; + struct ext4_inode *raw_inode; + struct ext4_xattr_ibody_header *header; + struct ext4_xattr_entry *entry; + qsize_t ea_inode_refs = 0; + int ret; + + lockdep_assert_held_read(&EXT4_I(inode)->xattr_sem); + + if (ext4_test_inode_state(inode, EXT4_STATE_XATTR)) { + ret = ext4_get_inode_loc(inode, &iloc); + if (ret) + goto out; + raw_inode = ext4_raw_inode(&iloc); + header = IHDR(inode, raw_inode); + + for (entry = IFIRST(header); !IS_LAST_ENTRY(entry); + entry = EXT4_XATTR_NEXT(entry)) + if (entry->e_value_inum) + ea_inode_refs++; + } + + if (EXT4_I(inode)->i_file_acl) { + bh = ext4_sb_bread(inode->i_sb, EXT4_I(inode)->i_file_acl, REQ_PRIO); + if (IS_ERR(bh)) { + ret = PTR_ERR(bh); + bh = NULL; + goto out; + } + + ret = ext4_xattr_check_block(inode, bh); + if (ret) + goto out; + + for (entry = BFIRST(bh); !IS_LAST_ENTRY(entry); + entry = EXT4_XATTR_NEXT(entry)) + if (entry->e_value_inum) + ea_inode_refs++; + } + *usage = ea_inode_refs + 1; + ret = 0; +out: + brelse(iloc.bh); + brelse(bh); + return ret; +} + +static inline size_t round_up_cluster(struct inode *inode, size_t length) +{ + struct super_block *sb = inode->i_sb; + size_t cluster_size = 1 << (EXT4_SB(sb)->s_cluster_bits + + inode->i_blkbits); + size_t mask = ~(cluster_size - 1); + + return (length + cluster_size - 1) & mask; +} + +static int ext4_xattr_inode_alloc_quota(struct inode *inode, size_t len) +{ + int err; + + err = dquot_alloc_inode(inode); + if (err) + return err; + err = dquot_alloc_space_nodirty(inode, round_up_cluster(inode, len)); + if (err) + dquot_free_inode(inode); + return err; +} + +static void ext4_xattr_inode_free_quota(struct inode *parent, + struct inode *ea_inode, + size_t len) +{ + if (ea_inode && + ext4_test_inode_state(ea_inode, EXT4_STATE_LUSTRE_EA_INODE)) + return; + dquot_free_space_nodirty(parent, round_up_cluster(parent, len)); + dquot_free_inode(parent); +} + +int __ext4_xattr_set_credits(struct super_block *sb, struct inode *inode, + struct buffer_head *block_bh, size_t value_len, + bool is_create) +{ + int credits; + int blocks; + + /* + * 1) Owner inode update + * 2) Ref count update on old xattr block + * 3) new xattr block + * 4) block bitmap update for new xattr block + * 5) group descriptor for new xattr block + * 6) block bitmap update for old xattr block + * 7) group descriptor for old block + * + * 6 & 7 can happen if we have two racing threads T_a and T_b + * which are each trying to set an xattr on inodes I_a and I_b + * which were both initially sharing an xattr block. + */ + credits = 7; + + /* Quota updates. */ + credits += EXT4_MAXQUOTAS_TRANS_BLOCKS(sb); + + /* + * In case of inline data, we may push out the data to a block, + * so we need to reserve credits for this eventuality + */ + if (inode && ext4_has_inline_data(inode)) + credits += ext4_chunk_trans_extent(inode, 1) + 1; + + /* We are done if ea_inode feature is not enabled. */ + if (!ext4_has_feature_ea_inode(sb)) + return credits; + + /* New ea_inode, inode map, block bitmap, group descriptor. */ + credits += 4; + + /* Data blocks. */ + blocks = (value_len + sb->s_blocksize - 1) >> sb->s_blocksize_bits; + + /* Indirection block or one level of extent tree. */ + blocks += 1; + + /* Block bitmap and group descriptor updates for each block. */ + credits += blocks * 2; + + /* Blocks themselves. */ + credits += blocks; + + if (!is_create) { + /* Dereference ea_inode holding old xattr value. + * Old ea_inode, inode map, block bitmap, group descriptor. + */ + credits += 4; + + /* Data blocks for old ea_inode. */ + blocks = XATTR_SIZE_MAX >> sb->s_blocksize_bits; + + /* Indirection block or one level of extent tree for old + * ea_inode. + */ + blocks += 1; + + /* Block bitmap and group descriptor updates for each block. */ + credits += blocks * 2; + } + + /* We may need to clone the existing xattr block in which case we need + * to increment ref counts for existing ea_inodes referenced by it. + */ + if (block_bh) { + struct ext4_xattr_entry *entry = BFIRST(block_bh); + + for (; !IS_LAST_ENTRY(entry); entry = EXT4_XATTR_NEXT(entry)) + if (entry->e_value_inum) + /* Ref count update on ea_inode. */ + credits += 1; + } + return credits; +} + +static int ext4_xattr_inode_update_ref(handle_t *handle, struct inode *ea_inode, + int ref_change) +{ + struct ext4_iloc iloc; + u64 ref_count; + int ret; + + inode_lock_nested(ea_inode, I_MUTEX_XATTR); + + ret = ext4_reserve_inode_write(handle, ea_inode, &iloc); + if (ret) + goto out; + + ref_count = ext4_xattr_inode_get_ref(ea_inode); + if ((ref_count == 0 && ref_change < 0) || (ref_count == U64_MAX && ref_change > 0)) { + ext4_error_inode(ea_inode, __func__, __LINE__, 0, + "EA inode %llu ref wraparound: ref_count=%lld ref_change=%d", + ea_inode->i_ino, ref_count, ref_change); + brelse(iloc.bh); + ret = -EFSCORRUPTED; + goto out; + } + ref_count += ref_change; + ext4_xattr_inode_set_ref(ea_inode, ref_count); + + if (ref_change > 0) { + if (ref_count == 1) { + WARN_ONCE(ea_inode->i_nlink, "EA inode %llu i_nlink=%u", + ea_inode->i_ino, ea_inode->i_nlink); + + set_nlink(ea_inode, 1); + ext4_orphan_del(handle, ea_inode); + } + } else { + if (ref_count == 0) { + WARN_ONCE(ea_inode->i_nlink != 1, + "EA inode %llu i_nlink=%u", + ea_inode->i_ino, ea_inode->i_nlink); + + clear_nlink(ea_inode); + ext4_orphan_add(handle, ea_inode); + } + } + + ret = ext4_mark_iloc_dirty(handle, ea_inode, &iloc); + if (ret) + ext4_warning_inode(ea_inode, + "ext4_mark_iloc_dirty() failed ret=%d", ret); +out: + inode_unlock(ea_inode); + return ret; +} + +static int ext4_xattr_inode_inc_ref(handle_t *handle, struct inode *ea_inode) +{ + return ext4_xattr_inode_update_ref(handle, ea_inode, 1); +} + +static int ext4_xattr_inode_dec_ref(handle_t *handle, struct inode *ea_inode) +{ + return ext4_xattr_inode_update_ref(handle, ea_inode, -1); +} + +static int ext4_xattr_inode_inc_ref_all(handle_t *handle, struct inode *parent, + struct ext4_xattr_entry *first) +{ + struct inode *ea_inode; + struct ext4_xattr_entry *entry; + struct ext4_xattr_entry *failed_entry; + unsigned int ea_ino; + int err, saved_err; + + for (entry = first; !IS_LAST_ENTRY(entry); + entry = EXT4_XATTR_NEXT(entry)) { + if (!entry->e_value_inum) + continue; + ea_ino = le32_to_cpu(entry->e_value_inum); + err = ext4_xattr_inode_iget(parent, ea_ino, + le32_to_cpu(entry->e_hash), + &ea_inode); + if (err) + goto cleanup; + err = ext4_xattr_inode_inc_ref(handle, ea_inode); + if (err) { + ext4_warning_inode(ea_inode, "inc ref error %d", err); + ext4_put_ea_inode(ea_inode); + goto cleanup; + } + ext4_put_ea_inode(ea_inode); + } + return 0; + +cleanup: + saved_err = err; + failed_entry = entry; + + for (entry = first; entry != failed_entry; + entry = EXT4_XATTR_NEXT(entry)) { + if (!entry->e_value_inum) + continue; + ea_ino = le32_to_cpu(entry->e_value_inum); + err = ext4_xattr_inode_iget(parent, ea_ino, + le32_to_cpu(entry->e_hash), + &ea_inode); + if (err) { + ext4_warning(parent->i_sb, + "cleanup ea_ino %u iget error %d", ea_ino, + err); + continue; + } + err = ext4_xattr_inode_dec_ref(handle, ea_inode); + if (err) + ext4_warning_inode(ea_inode, "cleanup dec ref error %d", + err); + ext4_put_ea_inode(ea_inode); + } + return saved_err; +} + +static int ext4_xattr_restart_fn(handle_t *handle, struct inode *inode, + struct buffer_head *bh, bool block_csum, bool dirty) +{ + int error; + + if (bh && dirty) { + if (block_csum) + ext4_xattr_block_csum_set(inode, bh); + error = ext4_handle_dirty_metadata(handle, NULL, bh); + if (error) { + ext4_warning(inode->i_sb, "Handle metadata (error %d)", + error); + return error; + } + } + return 0; +} + +static void +ext4_xattr_inode_dec_ref_all(handle_t *handle, struct inode *parent, + struct buffer_head *bh, + struct ext4_xattr_entry *first, bool block_csum, + int extra_credits, bool skip_quota) +{ + struct inode *ea_inode; + struct ext4_xattr_entry *entry; + struct ext4_iloc iloc = { .bh = NULL }; + bool dirty = false; + unsigned int ea_ino; + int err; + int credits; + void *end; + + if (block_csum) + end = (void *)bh->b_data + bh->b_size; + else { + err = ext4_get_inode_loc(parent, &iloc); + if (err) { + EXT4_ERROR_INODE(parent, "parent inode loc (error %d)", err); + return; + } + end = (void *)ext4_raw_inode(&iloc) + EXT4_SB(parent->i_sb)->s_inode_size; + } + + /* One credit for dec ref on ea_inode, one for orphan list addition, */ + credits = 2 + extra_credits; + + for (entry = first; (void *)entry < end && !IS_LAST_ENTRY(entry); + entry = EXT4_XATTR_NEXT(entry)) { + if (!entry->e_value_inum) + continue; + ea_ino = le32_to_cpu(entry->e_value_inum); + err = ext4_xattr_inode_iget(parent, ea_ino, + le32_to_cpu(entry->e_hash), + &ea_inode); + if (err) + continue; + + err = ext4_journal_ensure_credits_fn(handle, credits, credits, + ext4_free_metadata_revoke_credits(parent->i_sb, 1), + ext4_xattr_restart_fn(handle, parent, bh, block_csum, + dirty)); + if (err < 0) { + ext4_warning_inode(ea_inode, "Ensure credits err=%d", + err); + ext4_put_ea_inode(ea_inode); + continue; + } + if (err > 0) { + err = ext4_journal_get_write_access(handle, + parent->i_sb, bh, EXT4_JTR_NONE); + if (err) { + ext4_warning_inode(ea_inode, + "Re-get write access err=%d", + err); + ext4_put_ea_inode(ea_inode); + continue; + } + } + + err = ext4_xattr_inode_dec_ref(handle, ea_inode); + if (err) { + ext4_warning_inode(ea_inode, "ea_inode dec ref err=%d", + err); + ext4_put_ea_inode(ea_inode); + continue; + } + + if (!skip_quota) + ext4_xattr_inode_free_quota(parent, ea_inode, + le32_to_cpu(entry->e_value_size)); + + /* + * Forget about ea_inode within the same transaction that + * decrements the ref count. This avoids duplicate decrements in + * case the rest of the work spills over to subsequent + * transactions. + */ + entry->e_value_inum = 0; + entry->e_value_size = 0; + + ext4_put_ea_inode(ea_inode); + dirty = true; + } + + if (dirty) { + /* + * Note that we are deliberately skipping csum calculation for + * the final update because we do not expect any journal + * restarts until xattr block is freed. + */ + + err = ext4_handle_dirty_metadata(handle, NULL, bh); + if (err) + ext4_warning_inode(parent, + "handle dirty metadata err=%d", err); + } + + brelse(iloc.bh); +} + +/* + * Release the xattr block BH: If the reference count is > 1, decrement it; + * otherwise free the block. + */ +static void +ext4_xattr_release_block(handle_t *handle, struct inode *inode, + struct buffer_head *bh, + int extra_credits) +{ + struct mb_cache *ea_block_cache = EA_BLOCK_CACHE(inode); + u32 hash, ref; + int error = 0; + + BUFFER_TRACE(bh, "get_write_access"); + error = ext4_journal_get_write_access(handle, inode->i_sb, bh, + EXT4_JTR_NONE); + if (error) + goto out; + +retry_ref: + lock_buffer(bh); + hash = le32_to_cpu(BHDR(bh)->h_hash); + ref = le32_to_cpu(BHDR(bh)->h_refcount); + if (ref == 1) { + ea_bdebug(bh, "refcount now=0; freeing"); + /* + * This must happen under buffer lock for + * ext4_xattr_block_set() to reliably detect freed block + */ + if (ea_block_cache) { + struct mb_cache_entry *oe; + + oe = mb_cache_entry_delete_or_get(ea_block_cache, hash, + bh->b_blocknr); + if (oe) { + unlock_buffer(bh); + mb_cache_entry_wait_unused(oe); + mb_cache_entry_put(ea_block_cache, oe); + goto retry_ref; + } + } + get_bh(bh); + unlock_buffer(bh); + + if (ext4_has_feature_ea_inode(inode->i_sb)) + ext4_xattr_inode_dec_ref_all(handle, inode, bh, + BFIRST(bh), + true /* block_csum */, + extra_credits, + true /* skip_quota */); + ext4_free_blocks(handle, inode, bh, 0, 1, + EXT4_FREE_BLOCKS_METADATA | + EXT4_FREE_BLOCKS_FORGET); + } else { + ref--; + BHDR(bh)->h_refcount = cpu_to_le32(ref); + if (ref == EXT4_XATTR_REFCOUNT_MAX - 1) { + struct mb_cache_entry *ce; + + if (ea_block_cache) { + ce = mb_cache_entry_get(ea_block_cache, hash, + bh->b_blocknr); + if (ce) { + set_bit(MBE_REUSABLE_B, &ce->e_flags); + mb_cache_entry_put(ea_block_cache, ce); + } + } + } + + ext4_xattr_block_csum_set(inode, bh); + /* + * Beware of this ugliness: Releasing of xattr block references + * from different inodes can race and so we have to protect + * from a race where someone else frees the block (and releases + * its journal_head) before we are done dirtying the buffer. In + * nojournal mode this race is harmless and we actually cannot + * call ext4_handle_dirty_metadata() with locked buffer as + * that function can call sync_dirty_buffer() so for that case + * we handle the dirtying after unlocking the buffer. + */ + if (ext4_handle_valid(handle)) + error = ext4_handle_dirty_metadata(handle, inode, bh); + unlock_buffer(bh); + if (!ext4_handle_valid(handle)) + error = ext4_handle_dirty_metadata(handle, inode, bh); + if (IS_SYNC(inode)) + ext4_handle_sync(handle); + dquot_free_block(inode, EXT4_C2B(EXT4_SB(inode->i_sb), 1)); + ea_bdebug(bh, "refcount now=%d; releasing", + le32_to_cpu(BHDR(bh)->h_refcount)); + } +out: + ext4_std_error(inode->i_sb, error); + return; +} + +/* + * Find the available free space for EAs. This also returns the total number of + * bytes used by EA entries. + */ +static size_t ext4_xattr_free_space(struct ext4_xattr_entry *last, + size_t *min_offs, void *base, int *total) +{ + for (; !IS_LAST_ENTRY(last); last = EXT4_XATTR_NEXT(last)) { + if (!last->e_value_inum && last->e_value_size) { + size_t offs = le16_to_cpu(last->e_value_offs); + if (offs < *min_offs) + *min_offs = offs; + } + if (total) + *total += EXT4_XATTR_LEN(last->e_name_len); + } + return (*min_offs - ((void *)last - base) - sizeof(__u32)); +} + +/* + * Write the value of the EA in an inode. + */ +static int ext4_xattr_inode_write(handle_t *handle, struct inode *ea_inode, + const void *buf, int bufsize) +{ + struct buffer_head *bh = NULL; + unsigned long block = 0; + int blocksize = ea_inode->i_sb->s_blocksize; + int max_blocks = (bufsize + blocksize - 1) >> ea_inode->i_blkbits; + int csize, wsize = 0; + int ret = 0, ret2 = 0; + int retries = 0; + +retry: + while (ret >= 0 && ret < max_blocks) { + struct ext4_map_blocks map; + map.m_lblk = block += ret; + map.m_len = max_blocks -= ret; + + ret = ext4_map_blocks(handle, ea_inode, &map, + EXT4_GET_BLOCKS_CREATE); + if (ret <= 0) { + ext4_mark_inode_dirty(handle, ea_inode); + if (ret == -ENOSPC && + ext4_should_retry_alloc(ea_inode->i_sb, &retries)) { + ret = 0; + goto retry; + } + break; + } + } + + if (ret < 0) + return ret; + + block = 0; + while (wsize < bufsize) { + brelse(bh); + csize = (bufsize - wsize) > blocksize ? blocksize : + bufsize - wsize; + bh = ext4_getblk(handle, ea_inode, block, 0); + if (IS_ERR(bh)) + return PTR_ERR(bh); + if (!bh) { + WARN_ON_ONCE(1); + EXT4_ERROR_INODE(ea_inode, + "ext4_getblk() return bh = NULL"); + return -EFSCORRUPTED; + } + ret = ext4_journal_get_write_access(handle, ea_inode->i_sb, bh, + EXT4_JTR_NONE); + if (ret) + goto out; + + memcpy(bh->b_data, buf, csize); + /* + * Zero out block tail to avoid writing uninitialized memory + * to disk. + */ + if (csize < blocksize) + memset(bh->b_data + csize, 0, blocksize - csize); + set_buffer_uptodate(bh); + ext4_handle_dirty_metadata(handle, ea_inode, bh); + + buf += csize; + wsize += csize; + block += 1; + } + + inode_lock(ea_inode); + i_size_write(ea_inode, wsize); + ext4_update_i_disksize(ea_inode, wsize); + inode_unlock(ea_inode); + + ret2 = ext4_mark_inode_dirty(handle, ea_inode); + if (unlikely(ret2 && !ret)) + ret = ret2; + +out: + brelse(bh); + + return ret; +} + +/* + * Create an inode to store the value of a large EA. + */ +static struct inode *ext4_xattr_inode_create(handle_t *handle, + struct inode *inode, u32 hash) +{ + struct inode *ea_inode = NULL; + uid_t owner[2] = { i_uid_read(inode), i_gid_read(inode) }; + int err; + + if (inode->i_sb->s_root == NULL) { + ext4_warning(inode->i_sb, + "refuse to create EA inode when umounting"); + WARN_ON(1); + return ERR_PTR(-EINVAL); + } + + /* + * Let the next inode be the goal, so we try and allocate the EA inode + * in the same group, or nearby one. + */ + ea_inode = ext4_new_inode(handle, inode->i_sb->s_root->d_inode, + S_IFREG | 0600, NULL, inode->i_ino + 1, owner, + EXT4_EA_INODE_FL); + if (!IS_ERR(ea_inode)) { + ea_inode->i_op = &ext4_file_inode_operations; + ea_inode->i_fop = &ext4_file_operations; + ext4_set_aops(ea_inode); + ext4_xattr_inode_set_class(ea_inode); + unlock_new_inode(ea_inode); + ext4_xattr_inode_set_ref(ea_inode, 1); + ext4_xattr_inode_set_hash(ea_inode, hash); + err = ext4_mark_inode_dirty(handle, ea_inode); + if (!err) + err = ext4_inode_attach_jinode(ea_inode); + if (err) { + if (ext4_xattr_inode_dec_ref(handle, ea_inode)) + ext4_warning_inode(ea_inode, + "cleanup dec ref error %d", err); + ext4_put_ea_inode(ea_inode); + return ERR_PTR(err); + } + + /* + * Xattr inodes are shared therefore quota charging is performed + * at a higher level. + */ + dquot_free_inode(ea_inode); + dquot_drop(ea_inode); + inode_lock(ea_inode); + ea_inode->i_flags |= S_NOQUOTA; + inode_unlock(ea_inode); + } + + return ea_inode; +} + +static struct inode * +ext4_xattr_inode_cache_find(struct inode *inode, const void *value, + size_t value_len, u32 hash) +{ + struct inode *ea_inode; + struct mb_cache_entry *ce; + struct mb_cache *ea_inode_cache = EA_INODE_CACHE(inode); + void *ea_data; + + if (!ea_inode_cache) + return NULL; + + ce = mb_cache_entry_find_first(ea_inode_cache, hash); + if (!ce) + return NULL; + + WARN_ON_ONCE(ext4_handle_valid(journal_current_handle()) && + !(current->flags & PF_MEMALLOC_NOFS)); + + ea_data = kvmalloc(value_len, GFP_NOFS); + if (!ea_data) { + mb_cache_entry_put(ea_inode_cache, ce); + return NULL; + } + + while (ce) { + ea_inode = ext4_iget(inode->i_sb, ce->e_value, + EXT4_IGET_EA_INODE | EXT4_IGET_NOWAIT); + if (IS_ERR(ea_inode)) + goto next_entry; + ext4_xattr_inode_set_class(ea_inode); + if (i_size_read(ea_inode) == value_len && + !ext4_xattr_inode_read(ea_inode, ea_data, value_len) && + !ext4_xattr_inode_verify_hashes(ea_inode, NULL, ea_data, + value_len) && + !memcmp(value, ea_data, value_len)) { + mb_cache_entry_touch(ea_inode_cache, ce); + mb_cache_entry_put(ea_inode_cache, ce); + kvfree(ea_data); + return ea_inode; + } + ext4_put_ea_inode(ea_inode); + next_entry: + ce = mb_cache_entry_find_next(ea_inode_cache, ce); + } + kvfree(ea_data); + return NULL; +} + +/* + * Add value of the EA in an inode. + */ +static struct inode *ext4_xattr_inode_lookup_create(handle_t *handle, + struct inode *inode, const void *value, size_t value_len) +{ + struct inode *ea_inode; + u32 hash; + int err; + + /* Account inode & space to quota even if sharing... */ + err = ext4_xattr_inode_alloc_quota(inode, value_len); + if (err) + return ERR_PTR(err); + + hash = ext4_xattr_inode_hash(EXT4_SB(inode->i_sb), value, value_len); + ea_inode = ext4_xattr_inode_cache_find(inode, value, value_len, hash); + if (ea_inode) { + err = ext4_xattr_inode_inc_ref(handle, ea_inode); + if (err) + goto out_err; + return ea_inode; + } + + /* Create an inode for the EA value */ + ea_inode = ext4_xattr_inode_create(handle, inode, hash); + if (IS_ERR(ea_inode)) { + ext4_xattr_inode_free_quota(inode, NULL, value_len); + return ea_inode; + } + + err = ext4_xattr_inode_write(handle, ea_inode, value, value_len); + if (err) { + if (ext4_xattr_inode_dec_ref(handle, ea_inode)) + ext4_warning_inode(ea_inode, "cleanup dec ref error %d", err); + goto out_err; + } + + if (EA_INODE_CACHE(inode)) + mb_cache_entry_create(EA_INODE_CACHE(inode), GFP_NOFS, hash, + ea_inode->i_ino, true /* reusable */); + return ea_inode; +out_err: + ext4_put_ea_inode(ea_inode); + ext4_xattr_inode_free_quota(inode, NULL, value_len); + return ERR_PTR(err); +} + +/* + * Reserve min(block_size/8, 1024) bytes for xattr entries/names if ea_inode + * feature is enabled. + */ +#define EXT4_XATTR_BLOCK_RESERVE(inode) min(i_blocksize(inode)/8, 1024U) + +static int ext4_xattr_set_entry(struct ext4_xattr_info *i, + struct ext4_xattr_search *s, + handle_t *handle, struct inode *inode, + struct inode *new_ea_inode, + bool is_block) +{ + struct ext4_xattr_entry *last, *next; + struct ext4_xattr_entry *here = s->here; + size_t min_offs = s->end - s->base, name_len = strlen(i->name); + int in_inode = i->in_inode; + struct inode *old_ea_inode = NULL; + size_t old_size, new_size; + int ret; + + /* Space used by old and new values. */ + old_size = (!s->not_found && !here->e_value_inum) ? + EXT4_XATTR_SIZE(le32_to_cpu(here->e_value_size)) : 0; + new_size = (i->value && !in_inode) ? EXT4_XATTR_SIZE(i->value_len) : 0; + + /* + * Optimization for the simple case when old and new values have the + * same padded sizes. Not applicable if external inodes are involved. + */ + if (new_size && new_size == old_size) { + size_t offs = le16_to_cpu(here->e_value_offs); + void *val = s->base + offs; + + here->e_value_size = cpu_to_le32(i->value_len); + if (i->value == EXT4_ZERO_XATTR_VALUE) { + memset(val, 0, new_size); + } else { + memcpy(val, i->value, i->value_len); + /* Clear padding bytes. */ + memset(val + i->value_len, 0, new_size - i->value_len); + } + goto update_hash; + } + + /* Compute min_offs and last. */ + last = s->first; + for (; !IS_LAST_ENTRY(last); last = next) { + next = EXT4_XATTR_NEXT(last); + if ((void *)next >= s->end) { + EXT4_ERROR_INODE(inode, "corrupted xattr entries"); + ret = -EFSCORRUPTED; + goto out; + } + if (!last->e_value_inum && last->e_value_size) { + size_t offs = le16_to_cpu(last->e_value_offs); + if (offs < min_offs) + min_offs = offs; + } + } + + /* Check whether we have enough space. */ + if (i->value) { + size_t free; + + free = min_offs - ((void *)last - s->base) - sizeof(__u32); + if (!s->not_found) + free += EXT4_XATTR_LEN(name_len) + old_size; + + if (free < EXT4_XATTR_LEN(name_len) + new_size) { + ret = -ENOSPC; + goto out; + } + + /* + * If storing the value in an external inode is an option, + * reserve space for xattr entries/names in the external + * attribute block so that a long value does not occupy the + * whole space and prevent further entries being added. + */ + if (ext4_has_feature_ea_inode(inode->i_sb) && + new_size && is_block && + (min_offs + old_size - new_size) < + EXT4_XATTR_BLOCK_RESERVE(inode)) { + ret = -ENOSPC; + goto out; + } + } + + /* + * Getting access to old and new ea inodes is subject to failures. + * Finish that work before doing any modifications to the xattr data. + */ + if (!s->not_found && here->e_value_inum) { + ret = ext4_xattr_inode_iget(inode, + le32_to_cpu(here->e_value_inum), + le32_to_cpu(here->e_hash), + &old_ea_inode); + if (ret) { + old_ea_inode = NULL; + goto out; + } + + /* We are ready to release ref count on the old_ea_inode. */ + ret = ext4_xattr_inode_dec_ref(handle, old_ea_inode); + if (ret) + goto out; + + ext4_xattr_inode_free_quota(inode, old_ea_inode, + le32_to_cpu(here->e_value_size)); + } + + /* No failures allowed past this point. */ + + if (!s->not_found && here->e_value_size && !here->e_value_inum) { + /* Remove the old value. */ + void *first_val = s->base + min_offs; + size_t offs = le16_to_cpu(here->e_value_offs); + void *val = s->base + offs; + + memmove(first_val + old_size, first_val, val - first_val); + memset(first_val, 0, old_size); + min_offs += old_size; + + /* Adjust all value offsets. */ + last = s->first; + while (!IS_LAST_ENTRY(last)) { + size_t o = le16_to_cpu(last->e_value_offs); + + if (!last->e_value_inum && + last->e_value_size && o < offs) + last->e_value_offs = cpu_to_le16(o + old_size); + last = EXT4_XATTR_NEXT(last); + } + } + + if (!i->value) { + /* Remove old name. */ + size_t size = EXT4_XATTR_LEN(name_len); + + last = ENTRY((void *)last - size); + memmove(here, (void *)here + size, + (void *)last - (void *)here + sizeof(__u32)); + memset(last, 0, size); + + /* + * Update i_inline_off - moved ibody region might contain + * system.data attribute. Handling a failure here won't + * cause other complications for setting an xattr. + */ + if (!is_block && ext4_has_inline_data(inode)) { + ret = ext4_find_inline_data_nolock(inode); + if (ret) { + ext4_warning_inode(inode, + "unable to update i_inline_off"); + goto out; + } + } + } else if (s->not_found) { + /* Insert new name. */ + size_t size = EXT4_XATTR_LEN(name_len); + size_t rest = (void *)last - (void *)here + sizeof(__u32); + + memmove((void *)here + size, here, rest); + memset(here, 0, size); + here->e_name_index = i->name_index; + here->e_name_len = name_len; + memcpy(here->e_name, i->name, name_len); + } else { + /* This is an update, reset value info. */ + here->e_value_inum = 0; + here->e_value_offs = 0; + here->e_value_size = 0; + } + + if (i->value) { + /* Insert new value. */ + if (in_inode) { + here->e_value_inum = cpu_to_le32(new_ea_inode->i_ino); + } else if (i->value_len) { + void *val = s->base + min_offs - new_size; + + here->e_value_offs = cpu_to_le16(min_offs - new_size); + if (i->value == EXT4_ZERO_XATTR_VALUE) { + memset(val, 0, new_size); + } else { + memcpy(val, i->value, i->value_len); + /* Clear padding bytes. */ + memset(val + i->value_len, 0, + new_size - i->value_len); + } + } + here->e_value_size = cpu_to_le32(i->value_len); + } + +update_hash: + if (i->value) { + __le32 hash = 0; + + /* Entry hash calculation. */ + if (in_inode) { + __le32 crc32c_hash; + + /* + * Feed crc32c hash instead of the raw value for entry + * hash calculation. This is to avoid walking + * potentially long value buffer again. + */ + crc32c_hash = cpu_to_le32( + ext4_xattr_inode_get_hash(new_ea_inode)); + hash = ext4_xattr_hash_entry(here->e_name, + here->e_name_len, + &crc32c_hash, 1); + } else if (is_block) { + __le32 *value = s->base + le16_to_cpu( + here->e_value_offs); + + hash = ext4_xattr_hash_entry(here->e_name, + here->e_name_len, value, + new_size >> 2); + } + here->e_hash = hash; + } + + if (is_block) + ext4_xattr_rehash((struct ext4_xattr_header *)s->base); + + ret = 0; +out: + ext4_put_ea_inode(old_ea_inode); + return ret; +} + +struct ext4_xattr_block_find { + struct ext4_xattr_search s; + struct buffer_head *bh; +}; + +static int +ext4_xattr_block_find(struct inode *inode, struct ext4_xattr_info *i, + struct ext4_xattr_block_find *bs) +{ + struct super_block *sb = inode->i_sb; + int error; + + ea_idebug(inode, "name=%d.%s, value=%p, value_len=%ld", + i->name_index, i->name, i->value, (long)i->value_len); + + if (EXT4_I(inode)->i_file_acl) { + /* The inode already has an extended attribute block. */ + bs->bh = ext4_sb_bread(sb, EXT4_I(inode)->i_file_acl, REQ_PRIO); + if (IS_ERR(bs->bh)) { + error = PTR_ERR(bs->bh); + bs->bh = NULL; + return error; + } + ea_bdebug(bs->bh, "b_count=%d, refcount=%d", + atomic_read(&(bs->bh->b_count)), + le32_to_cpu(BHDR(bs->bh)->h_refcount)); + error = ext4_xattr_check_block(inode, bs->bh); + if (error) + return error; + /* Find the named attribute. */ + bs->s.base = BHDR(bs->bh); + bs->s.first = BFIRST(bs->bh); + bs->s.end = bs->bh->b_data + bs->bh->b_size; + bs->s.here = bs->s.first; + error = xattr_find_entry(inode, &bs->s.here, bs->s.end, + i->name_index, i->name, 1); + if (error && error != -ENODATA) + return error; + bs->s.not_found = error; + } + return 0; +} + +static int +ext4_xattr_block_set(handle_t *handle, struct inode *inode, + struct ext4_xattr_info *i, + struct ext4_xattr_block_find *bs) +{ + struct super_block *sb = inode->i_sb; + struct buffer_head *new_bh = NULL; + struct ext4_xattr_search s_copy = bs->s; + struct ext4_xattr_search *s = &s_copy; + struct mb_cache_entry *ce = NULL; + int error = 0; + struct mb_cache *ea_block_cache = EA_BLOCK_CACHE(inode); + struct inode *ea_inode = NULL, *tmp_inode; + size_t old_ea_inode_quota = 0; + unsigned int ea_ino; + +#define header(x) ((struct ext4_xattr_header *)(x)) + + /* If we need EA inode, prepare it before locking the buffer */ + if (i->value && i->in_inode) { + WARN_ON_ONCE(!i->value_len); + + ea_inode = ext4_xattr_inode_lookup_create(handle, inode, + i->value, i->value_len); + if (IS_ERR(ea_inode)) { + error = PTR_ERR(ea_inode); + ea_inode = NULL; + goto cleanup; + } + } + + if (s->base) { + int offset = (char *)s->here - bs->bh->b_data; + + BUFFER_TRACE(bs->bh, "get_write_access"); + error = ext4_journal_get_write_access(handle, sb, bs->bh, + EXT4_JTR_NONE); + if (error) + goto cleanup; + + lock_buffer(bs->bh); + + if (header(s->base)->h_refcount == cpu_to_le32(1)) { + __u32 hash = le32_to_cpu(BHDR(bs->bh)->h_hash); + + /* + * This must happen under buffer lock for + * ext4_xattr_block_set() to reliably detect modified + * block + */ + if (ea_block_cache) { + struct mb_cache_entry *oe; + + oe = mb_cache_entry_delete_or_get(ea_block_cache, + hash, bs->bh->b_blocknr); + if (oe) { + /* + * Xattr block is getting reused. Leave + * it alone. + */ + mb_cache_entry_put(ea_block_cache, oe); + goto clone_block; + } + } + ea_bdebug(bs->bh, "modifying in-place"); + error = ext4_xattr_set_entry(i, s, handle, inode, + ea_inode, true /* is_block */); + ext4_xattr_block_csum_set(inode, bs->bh); + unlock_buffer(bs->bh); + if (error == -EFSCORRUPTED) + goto bad_block; + if (!error) + error = ext4_handle_dirty_metadata(handle, + inode, + bs->bh); + if (error) + goto cleanup; + goto inserted; + } +clone_block: + unlock_buffer(bs->bh); + ea_bdebug(bs->bh, "cloning"); + s->base = kmemdup(BHDR(bs->bh), bs->bh->b_size, GFP_NOFS); + error = -ENOMEM; + if (s->base == NULL) + goto cleanup; + s->first = ENTRY(header(s->base)+1); + header(s->base)->h_refcount = cpu_to_le32(1); + s->here = ENTRY(s->base + offset); + s->end = s->base + bs->bh->b_size; + + /* + * If existing entry points to an xattr inode, we need + * to prevent ext4_xattr_set_entry() from decrementing + * ref count on it because the reference belongs to the + * original block. In this case, make the entry look + * like it has an empty value. + */ + if (!s->not_found && s->here->e_value_inum) { + ea_ino = le32_to_cpu(s->here->e_value_inum); + error = ext4_xattr_inode_iget(inode, ea_ino, + le32_to_cpu(s->here->e_hash), + &tmp_inode); + if (error) + goto cleanup; + + if (!ext4_test_inode_state(tmp_inode, + EXT4_STATE_LUSTRE_EA_INODE)) { + /* + * Defer quota free call for previous + * inode until success is guaranteed. + */ + old_ea_inode_quota = le32_to_cpu( + s->here->e_value_size); + } + ext4_put_ea_inode(tmp_inode); + + s->here->e_value_inum = 0; + s->here->e_value_size = 0; + } + } else { + /* Allocate a buffer where we construct the new block. */ + s->base = kzalloc(sb->s_blocksize, GFP_NOFS); + error = -ENOMEM; + if (s->base == NULL) + goto cleanup; + header(s->base)->h_magic = cpu_to_le32(EXT4_XATTR_MAGIC); + header(s->base)->h_blocks = cpu_to_le32(1); + header(s->base)->h_refcount = cpu_to_le32(1); + s->first = ENTRY(header(s->base)+1); + s->here = ENTRY(header(s->base)+1); + s->end = s->base + sb->s_blocksize; + } + + error = ext4_xattr_set_entry(i, s, handle, inode, ea_inode, + true /* is_block */); + if (error == -EFSCORRUPTED) + goto bad_block; + if (error) + goto cleanup; + +inserted: + if (!IS_LAST_ENTRY(s->first)) { + new_bh = ext4_xattr_block_cache_find(inode, header(s->base), &ce); + if (IS_ERR(new_bh)) { + error = PTR_ERR(new_bh); + new_bh = NULL; + goto cleanup; + } + + if (new_bh) { + /* We found an identical block in the cache. */ + if (new_bh == bs->bh) + ea_bdebug(new_bh, "keeping"); + else { + u32 ref; + +#ifdef EXT4_XATTR_DEBUG + WARN_ON_ONCE(dquot_initialize_needed(inode)); +#endif + /* The old block is released after updating + the inode. */ + error = dquot_alloc_block(inode, + EXT4_C2B(EXT4_SB(sb), 1)); + if (error) + goto cleanup; + BUFFER_TRACE(new_bh, "get_write_access"); + error = ext4_journal_get_write_access( + handle, sb, new_bh, + EXT4_JTR_NONE); + if (error) + goto cleanup_dquot; + lock_buffer(new_bh); + /* + * We have to be careful about races with + * adding references to xattr block. Once we + * hold buffer lock xattr block's state is + * stable so we can check the additional + * reference fits. + */ + ref = le32_to_cpu(BHDR(new_bh)->h_refcount); + if (ref >= EXT4_XATTR_REFCOUNT_MAX) { + /* + * Undo everything and check mbcache + * again. + */ + clear_bit(MBE_REUSABLE_B, &ce->e_flags); + unlock_buffer(new_bh); + dquot_free_block(inode, + EXT4_C2B(EXT4_SB(sb), + 1)); + brelse(new_bh); + mb_cache_entry_put(ea_block_cache, ce); + ce = NULL; + new_bh = NULL; + goto inserted; + } + ref++; + BHDR(new_bh)->h_refcount = cpu_to_le32(ref); + if (ref == EXT4_XATTR_REFCOUNT_MAX) + clear_bit(MBE_REUSABLE_B, &ce->e_flags); + ea_bdebug(new_bh, "reusing; refcount now=%d", + ref); + ext4_xattr_block_csum_set(inode, new_bh); + unlock_buffer(new_bh); + error = ext4_handle_dirty_metadata(handle, + inode, + new_bh); + if (error) + goto cleanup_dquot; + } + mb_cache_entry_touch(ea_block_cache, ce); + mb_cache_entry_put(ea_block_cache, ce); + ce = NULL; + } else if (bs->bh && s->base == bs->bh->b_data) { + /* We were modifying this block in-place. */ + ea_bdebug(bs->bh, "keeping this block"); + ext4_xattr_block_cache_insert(ea_block_cache, bs->bh); + new_bh = bs->bh; + get_bh(new_bh); + } else { + /* We need to allocate a new block */ + ext4_fsblk_t goal, block; + +#ifdef EXT4_XATTR_DEBUG + WARN_ON_ONCE(dquot_initialize_needed(inode)); +#endif + goal = ext4_group_first_block_no(sb, + EXT4_I(inode)->i_block_group); + block = ext4_new_meta_blocks(handle, inode, goal, 0, + NULL, &error); + if (error) + goto cleanup; + + ea_idebug(inode, "creating block %llu", + (unsigned long long)block); + + new_bh = sb_getblk(sb, block); + if (unlikely(!new_bh)) { + error = -ENOMEM; +getblk_failed: + ext4_free_blocks(handle, inode, NULL, block, 1, + EXT4_FREE_BLOCKS_METADATA); + goto cleanup; + } + error = ext4_xattr_inode_inc_ref_all(handle, inode, + ENTRY(header(s->base)+1)); + if (error) + goto getblk_failed; + if (ea_inode) { + /* Drop the extra ref on ea_inode. */ + error = ext4_xattr_inode_dec_ref(handle, + ea_inode); + if (error) + ext4_warning_inode(ea_inode, + "dec ref error=%d", + error); + ext4_put_ea_inode(ea_inode); + ea_inode = NULL; + } + + lock_buffer(new_bh); + error = ext4_journal_get_create_access(handle, sb, + new_bh, EXT4_JTR_NONE); + if (error) { + unlock_buffer(new_bh); + error = -EIO; + goto getblk_failed; + } + memcpy(new_bh->b_data, s->base, new_bh->b_size); + ext4_xattr_block_csum_set(inode, new_bh); + set_buffer_uptodate(new_bh); + unlock_buffer(new_bh); + ext4_xattr_block_cache_insert(ea_block_cache, new_bh); + error = ext4_handle_dirty_metadata(handle, inode, + new_bh); + if (error) + goto cleanup; + } + } + + if (old_ea_inode_quota) + ext4_xattr_inode_free_quota(inode, NULL, old_ea_inode_quota); + + /* Update the inode. */ + EXT4_I(inode)->i_file_acl = new_bh ? new_bh->b_blocknr : 0; + + /* Drop the previous xattr block. */ + if (bs->bh && bs->bh != new_bh) { + ext4_xattr_release_block(handle, inode, bs->bh, + 0 /* extra_credits */); + } + error = 0; + +cleanup: + if (ea_inode) { + if (error) { + int error2; + + error2 = ext4_xattr_inode_dec_ref(handle, ea_inode); + if (error2) + ext4_warning_inode(ea_inode, "dec ref error=%d", + error2); + ext4_xattr_inode_free_quota(inode, ea_inode, + i_size_read(ea_inode)); + } + ext4_put_ea_inode(ea_inode); + } + if (ce) + mb_cache_entry_put(ea_block_cache, ce); + brelse(new_bh); + if (!(bs->bh && s->base == bs->bh->b_data)) + kfree(s->base); + + return error; + +cleanup_dquot: + dquot_free_block(inode, EXT4_C2B(EXT4_SB(sb), 1)); + goto cleanup; + +bad_block: + EXT4_ERROR_INODE(inode, "bad block %llu", + EXT4_I(inode)->i_file_acl); + goto cleanup; + +#undef header +} + +int ext4_xattr_ibody_find(struct inode *inode, struct ext4_xattr_info *i, + struct ext4_xattr_ibody_find *is) +{ + struct ext4_xattr_ibody_header *header; + struct ext4_inode *raw_inode; + int error; + + if (!EXT4_INODE_HAS_XATTR_SPACE(inode)) + return 0; + + raw_inode = ext4_raw_inode(&is->iloc); + header = IHDR(inode, raw_inode); + is->s.base = is->s.first = IFIRST(header); + is->s.here = is->s.first; + is->s.end = ITAIL(inode, raw_inode); + if (ext4_test_inode_state(inode, EXT4_STATE_XATTR)) { + /* Find the named attribute. */ + error = xattr_find_entry(inode, &is->s.here, is->s.end, + i->name_index, i->name, 0); + if (error && error != -ENODATA) + return error; + is->s.not_found = error; + } + return 0; +} + +int ext4_xattr_ibody_set(handle_t *handle, struct inode *inode, + struct ext4_xattr_info *i, + struct ext4_xattr_ibody_find *is) +{ + struct ext4_xattr_ibody_header *header; + struct ext4_xattr_search *s = &is->s; + struct inode *ea_inode = NULL; + int error; + + if (!EXT4_INODE_HAS_XATTR_SPACE(inode)) + return -ENOSPC; + + /* If we need EA inode, prepare it before locking the buffer */ + if (i->value && i->in_inode) { + WARN_ON_ONCE(!i->value_len); + + ea_inode = ext4_xattr_inode_lookup_create(handle, inode, + i->value, i->value_len); + if (IS_ERR(ea_inode)) + return PTR_ERR(ea_inode); + } + error = ext4_xattr_set_entry(i, s, handle, inode, ea_inode, + false /* is_block */); + if (error) { + if (ea_inode) { + int error2; + + error2 = ext4_xattr_inode_dec_ref(handle, ea_inode); + if (error2) + ext4_warning_inode(ea_inode, "dec ref error=%d", + error2); + + ext4_xattr_inode_free_quota(inode, ea_inode, + i_size_read(ea_inode)); + ext4_put_ea_inode(ea_inode); + } + return error; + } + header = IHDR(inode, ext4_raw_inode(&is->iloc)); + if (!IS_LAST_ENTRY(s->first)) { + header->h_magic = cpu_to_le32(EXT4_XATTR_MAGIC); + ext4_set_inode_state(inode, EXT4_STATE_XATTR); + } else { + header->h_magic = cpu_to_le32(0); + ext4_clear_inode_state(inode, EXT4_STATE_XATTR); + } + ext4_put_ea_inode(ea_inode); + return 0; +} + +static int ext4_xattr_value_same(struct ext4_xattr_search *s, + struct ext4_xattr_info *i) +{ + void *value; + + /* When e_value_inum is set the value is stored externally. */ + if (s->here->e_value_inum) + return 0; + if (le32_to_cpu(s->here->e_value_size) != i->value_len) + return 0; + value = ((void *)s->base) + le16_to_cpu(s->here->e_value_offs); + return !memcmp(value, i->value, i->value_len); +} + +static struct buffer_head *ext4_xattr_get_block(struct inode *inode) +{ + struct buffer_head *bh; + int error; + + if (!EXT4_I(inode)->i_file_acl) + return NULL; + bh = ext4_sb_bread(inode->i_sb, EXT4_I(inode)->i_file_acl, REQ_PRIO); + if (IS_ERR(bh)) + return bh; + error = ext4_xattr_check_block(inode, bh); + if (error) { + brelse(bh); + return ERR_PTR(error); + } + return bh; +} + +/* + * ext4_xattr_set_handle() + * + * Create, replace or remove an extended attribute for this inode. Value + * is NULL to remove an existing extended attribute, and non-NULL to + * either replace an existing extended attribute, or create a new extended + * attribute. The flags XATTR_REPLACE and XATTR_CREATE + * specify that an extended attribute must exist and must not exist + * previous to the call, respectively. + * + * Returns 0, or a negative error number on failure. + */ +int +ext4_xattr_set_handle(handle_t *handle, struct inode *inode, int name_index, + const char *name, const void *value, size_t value_len, + int flags) +{ + struct ext4_xattr_info i = { + .name_index = name_index, + .name = name, + .value = value, + .value_len = value_len, + .in_inode = 0, + }; + struct ext4_xattr_ibody_find is = { + .s = { .not_found = -ENODATA, }, + }; + struct ext4_xattr_block_find bs = { + .s = { .not_found = -ENODATA, }, + }; + int no_expand; + int error; + + if (!name) + return -EINVAL; + if (strlen(name) > 255) + return -ERANGE; + + ext4_write_lock_xattr(inode, &no_expand); + + /* Check journal credits under write lock. */ + if (ext4_handle_valid(handle)) { + struct buffer_head *bh; + int credits; + + bh = ext4_xattr_get_block(inode); + if (IS_ERR(bh)) { + error = PTR_ERR(bh); + goto cleanup; + } + + credits = __ext4_xattr_set_credits(inode->i_sb, inode, bh, + value_len, + flags & XATTR_CREATE); + brelse(bh); + + if (jbd2_handle_buffer_credits(handle) < credits) { + error = -ENOSPC; + goto cleanup; + } + WARN_ON_ONCE(!(current->flags & PF_MEMALLOC_NOFS)); + } + + error = ext4_reserve_inode_write(handle, inode, &is.iloc); + if (error) + goto cleanup; + + if (ext4_test_inode_state(inode, EXT4_STATE_NEW)) { + struct ext4_inode *raw_inode = ext4_raw_inode(&is.iloc); + memset(raw_inode, 0, EXT4_SB(inode->i_sb)->s_inode_size); + ext4_clear_inode_state(inode, EXT4_STATE_NEW); + } + + error = ext4_xattr_ibody_find(inode, &i, &is); + if (error) + goto cleanup; + if (is.s.not_found) + error = ext4_xattr_block_find(inode, &i, &bs); + if (error) + goto cleanup; + if (is.s.not_found && bs.s.not_found) { + error = -ENODATA; + if (flags & XATTR_REPLACE) + goto cleanup; + error = 0; + if (!value) + goto cleanup; + } else { + error = -EEXIST; + if (flags & XATTR_CREATE) + goto cleanup; + } + + if (!value) { + if (!is.s.not_found) + error = ext4_xattr_ibody_set(handle, inode, &i, &is); + else if (!bs.s.not_found) + error = ext4_xattr_block_set(handle, inode, &i, &bs); + } else { + error = 0; + /* Xattr value did not change? Save us some work and bail out */ + if (!is.s.not_found && ext4_xattr_value_same(&is.s, &i)) + goto cleanup; + if (!bs.s.not_found && ext4_xattr_value_same(&bs.s, &i)) + goto cleanup; + + if (ext4_has_feature_ea_inode(inode->i_sb) && + (EXT4_XATTR_SIZE(i.value_len) > + EXT4_XATTR_MIN_LARGE_EA_SIZE(inode->i_sb->s_blocksize))) + i.in_inode = 1; +retry_inode: + error = ext4_xattr_ibody_set(handle, inode, &i, &is); + if (!error && !bs.s.not_found) { + i.value = NULL; + error = ext4_xattr_block_set(handle, inode, &i, &bs); + } else if (error == -ENOSPC) { + if (EXT4_I(inode)->i_file_acl && !bs.s.base) { + brelse(bs.bh); + bs.bh = NULL; + error = ext4_xattr_block_find(inode, &i, &bs); + if (error) + goto cleanup; + } + error = ext4_xattr_block_set(handle, inode, &i, &bs); + if (!error && !is.s.not_found) { + i.value = NULL; + error = ext4_xattr_ibody_set(handle, inode, &i, + &is); + } else if (error == -ENOSPC) { + /* + * Xattr does not fit in the block, store at + * external inode if possible. + */ + if (ext4_has_feature_ea_inode(inode->i_sb) && + i.value_len && !i.in_inode) { + i.in_inode = 1; + goto retry_inode; + } + } + } + } + if (!error) { + ext4_xattr_update_super_block(handle, inode->i_sb); + inode_set_ctime_current(inode); + inode_inc_iversion(inode); + if (!value) + no_expand = 0; + error = ext4_mark_iloc_dirty(handle, inode, &is.iloc); + /* + * The bh is consumed by ext4_mark_iloc_dirty, even with + * error != 0. + */ + is.iloc.bh = NULL; + if (IS_SYNC(inode)) + ext4_handle_sync(handle); + } + ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_XATTR, handle); + +cleanup: + brelse(is.iloc.bh); + brelse(bs.bh); + ext4_write_unlock_xattr(inode, &no_expand); + return error; +} + +int ext4_xattr_set_credits(struct inode *inode, size_t value_len, + bool is_create, int *credits) +{ + struct buffer_head *bh; + int err; + + *credits = 0; + + if (!EXT4_SB(inode->i_sb)->s_journal) + return 0; + + down_read(&EXT4_I(inode)->xattr_sem); + + bh = ext4_xattr_get_block(inode); + if (IS_ERR(bh)) { + err = PTR_ERR(bh); + } else { + *credits = __ext4_xattr_set_credits(inode->i_sb, inode, bh, + value_len, is_create); + brelse(bh); + err = 0; + } + + up_read(&EXT4_I(inode)->xattr_sem); + return err; +} + +/* + * ext4_xattr_set() + * + * Like ext4_xattr_set_handle, but start from an inode. This extended + * attribute modification is a filesystem transaction by itself. + * + * Returns 0, or a negative error number on failure. + */ +int +ext4_xattr_set(struct inode *inode, int name_index, const char *name, + const void *value, size_t value_len, int flags) +{ + handle_t *handle; + struct super_block *sb = inode->i_sb; + int error, retries = 0; + int credits; + + error = dquot_initialize(inode); + if (error) + return error; + +retry: + error = ext4_xattr_set_credits(inode, value_len, flags & XATTR_CREATE, + &credits); + if (error) + return error; + + handle = ext4_journal_start(inode, EXT4_HT_XATTR, credits); + if (IS_ERR(handle)) { + error = PTR_ERR(handle); + } else { + int error2; + + error = ext4_xattr_set_handle(handle, inode, name_index, name, + value, value_len, flags); + ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_XATTR, + handle); + error2 = ext4_journal_stop(handle); + if (error == -ENOSPC && + ext4_should_retry_alloc(sb, &retries)) + goto retry; + if (error == 0) + error = error2; + } + + return error; +} + +/* + * Shift the EA entries in the inode to create space for the increased + * i_extra_isize. + */ +static void ext4_xattr_shift_entries(struct ext4_xattr_entry *entry, + int value_offs_shift, void *to, + void *from, size_t n) +{ + struct ext4_xattr_entry *last = entry; + int new_offs; + + /* We always shift xattr headers further thus offsets get lower */ + BUG_ON(value_offs_shift > 0); + + /* Adjust the value offsets of the entries */ + for (; !IS_LAST_ENTRY(last); last = EXT4_XATTR_NEXT(last)) { + if (!last->e_value_inum && last->e_value_size) { + new_offs = le16_to_cpu(last->e_value_offs) + + value_offs_shift; + last->e_value_offs = cpu_to_le16(new_offs); + } + } + /* Shift the entries by n bytes */ + memmove(to, from, n); +} + +/* + * Move xattr pointed to by 'entry' from inode into external xattr block + */ +static int ext4_xattr_move_to_block(handle_t *handle, struct inode *inode, + struct ext4_inode *raw_inode, + struct ext4_xattr_entry *entry) +{ + struct ext4_xattr_ibody_find *is = NULL; + struct ext4_xattr_block_find *bs = NULL; + char *buffer = NULL, *b_entry_name = NULL; + size_t value_size = le32_to_cpu(entry->e_value_size); + struct ext4_xattr_info i = { + .value = NULL, + .value_len = 0, + .name_index = entry->e_name_index, + .in_inode = !!entry->e_value_inum, + }; + struct ext4_xattr_ibody_header *header = IHDR(inode, raw_inode); + int needs_kvfree = 0; + int error; + + is = kzalloc_obj(struct ext4_xattr_ibody_find, GFP_NOFS); + bs = kzalloc_obj(struct ext4_xattr_block_find, GFP_NOFS); + b_entry_name = kmalloc(entry->e_name_len + 1, GFP_NOFS); + if (!is || !bs || !b_entry_name) { + error = -ENOMEM; + goto out; + } + + is->s.not_found = -ENODATA; + bs->s.not_found = -ENODATA; + is->iloc.bh = NULL; + bs->bh = NULL; + + /* Save the entry name and the entry value */ + if (entry->e_value_inum) { + buffer = kvmalloc(value_size, GFP_NOFS); + if (!buffer) { + error = -ENOMEM; + goto out; + } + needs_kvfree = 1; + error = ext4_xattr_inode_get(inode, entry, buffer, value_size); + if (error) + goto out; + } else { + size_t value_offs = le16_to_cpu(entry->e_value_offs); + buffer = (void *)IFIRST(header) + value_offs; + } + + memcpy(b_entry_name, entry->e_name, entry->e_name_len); + b_entry_name[entry->e_name_len] = '\0'; + i.name = b_entry_name; + + error = ext4_get_inode_loc(inode, &is->iloc); + if (error) + goto out; + + error = ext4_xattr_ibody_find(inode, &i, is); + if (error) + goto out; + + i.value = buffer; + i.value_len = value_size; + error = ext4_xattr_block_find(inode, &i, bs); + if (error) + goto out; + + /* Move ea entry from the inode into the block */ + error = ext4_xattr_block_set(handle, inode, &i, bs); + if (error) + goto out; + + /* Remove the chosen entry from the inode */ + i.value = NULL; + i.value_len = 0; + error = ext4_xattr_ibody_set(handle, inode, &i, is); + +out: + kfree(b_entry_name); + if (needs_kvfree && buffer) + kvfree(buffer); + if (is) + brelse(is->iloc.bh); + if (bs) + brelse(bs->bh); + kfree(is); + kfree(bs); + + return error; +} + +static int ext4_xattr_make_inode_space(handle_t *handle, struct inode *inode, + struct ext4_inode *raw_inode, + int isize_diff, size_t ifree, + size_t bfree, int *total_ino) +{ + struct ext4_xattr_ibody_header *header = IHDR(inode, raw_inode); + struct ext4_xattr_entry *small_entry; + struct ext4_xattr_entry *entry; + struct ext4_xattr_entry *last; + unsigned int entry_size; /* EA entry size */ + unsigned int total_size; /* EA entry size + value size */ + unsigned int min_total_size; + int error; + + while (isize_diff > ifree) { + entry = NULL; + small_entry = NULL; + min_total_size = ~0U; + last = IFIRST(header); + /* Find the entry best suited to be pushed into EA block */ + for (; !IS_LAST_ENTRY(last); last = EXT4_XATTR_NEXT(last)) { + /* never move system.data out of the inode */ + if ((last->e_name_len == 4) && + (last->e_name_index == EXT4_XATTR_INDEX_SYSTEM) && + !memcmp(last->e_name, "data", 4)) + continue; + total_size = EXT4_XATTR_LEN(last->e_name_len); + if (!last->e_value_inum) + total_size += EXT4_XATTR_SIZE( + le32_to_cpu(last->e_value_size)); + if (total_size <= bfree && + total_size < min_total_size) { + if (total_size + ifree < isize_diff) { + small_entry = last; + } else { + entry = last; + min_total_size = total_size; + } + } + } + + if (entry == NULL) { + if (small_entry == NULL) + return -ENOSPC; + entry = small_entry; + } + + entry_size = EXT4_XATTR_LEN(entry->e_name_len); + total_size = entry_size; + if (!entry->e_value_inum) + total_size += EXT4_XATTR_SIZE( + le32_to_cpu(entry->e_value_size)); + error = ext4_xattr_move_to_block(handle, inode, raw_inode, + entry); + if (error) + return error; + + *total_ino -= entry_size; + ifree += total_size; + bfree -= total_size; + } + + return 0; +} + +/* + * Expand an inode by new_extra_isize bytes when EAs are present. + * Returns 0 on success or negative error number on failure. + */ +int ext4_expand_extra_isize_ea(struct inode *inode, int new_extra_isize, + struct ext4_inode *raw_inode, handle_t *handle) +{ + struct ext4_xattr_ibody_header *header; + struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); + static unsigned int mnt_count; + size_t min_offs; + size_t ifree, bfree; + int total_ino; + void *base, *end; + int error = 0, tried_min_extra_isize = 0; + int s_min_extra_isize = le16_to_cpu(sbi->s_es->s_min_extra_isize); + int isize_diff; /* How much do we need to grow i_extra_isize */ + +retry: + isize_diff = new_extra_isize - EXT4_I(inode)->i_extra_isize; + if (EXT4_I(inode)->i_extra_isize >= new_extra_isize) + return 0; + + header = IHDR(inode, raw_inode); + + /* + * Check if enough free space is available in the inode to shift the + * entries ahead by new_extra_isize. + */ + + base = IFIRST(header); + end = ITAIL(inode, raw_inode); + min_offs = end - base; + total_ino = sizeof(struct ext4_xattr_ibody_header) + sizeof(u32); + + ifree = ext4_xattr_free_space(base, &min_offs, base, &total_ino); + if (ifree >= isize_diff) + goto shift; + + /* + * Enough free space isn't available in the inode, check if + * EA block can hold new_extra_isize bytes. + */ + if (EXT4_I(inode)->i_file_acl) { + struct buffer_head *bh; + + bh = ext4_sb_bread(inode->i_sb, EXT4_I(inode)->i_file_acl, REQ_PRIO); + if (IS_ERR(bh)) { + error = PTR_ERR(bh); + goto cleanup; + } + error = ext4_xattr_check_block(inode, bh); + if (error) { + brelse(bh); + goto cleanup; + } + base = BHDR(bh); + end = bh->b_data + bh->b_size; + min_offs = end - base; + bfree = ext4_xattr_free_space(BFIRST(bh), &min_offs, base, + NULL); + brelse(bh); + if (bfree + ifree < isize_diff) { + if (!tried_min_extra_isize && s_min_extra_isize) { + tried_min_extra_isize++; + new_extra_isize = s_min_extra_isize; + goto retry; + } + error = -ENOSPC; + goto cleanup; + } + } else { + bfree = inode->i_sb->s_blocksize; + } + + error = ext4_xattr_make_inode_space(handle, inode, raw_inode, + isize_diff, ifree, bfree, + &total_ino); + if (error) { + if (error == -ENOSPC && !tried_min_extra_isize && + s_min_extra_isize) { + tried_min_extra_isize++; + new_extra_isize = s_min_extra_isize; + error = 0; + goto retry; + } + goto cleanup; + } +shift: + /* Adjust the offsets and shift the remaining entries ahead */ + ext4_xattr_shift_entries(IFIRST(header), EXT4_I(inode)->i_extra_isize + - new_extra_isize, (void *)raw_inode + + EXT4_GOOD_OLD_INODE_SIZE + new_extra_isize, + (void *)header, total_ino); + EXT4_I(inode)->i_extra_isize = new_extra_isize; + + if (ext4_has_inline_data(inode)) + error = ext4_find_inline_data_nolock(inode); + +cleanup: + if (error && (mnt_count != le16_to_cpu(sbi->s_es->s_mnt_count))) { + ext4_warning(inode->i_sb, "Unable to expand inode %llu. Delete some EAs or run e2fsck.", + inode->i_ino); + mnt_count = le16_to_cpu(sbi->s_es->s_mnt_count); + } + return error; +} + +/* + * ext4_xattr_delete_inode() + * + * Free extended attribute resources associated with this inode. Traverse + * all entries and decrement reference on any xattr inodes associated with this + * inode. This is called immediately before an inode is freed. We have exclusive + * access to the inode. If an orphan inode is deleted it will also release its + * references on xattr block and xattr inodes. + */ +int ext4_xattr_delete_inode(handle_t *handle, struct inode *inode, + int extra_credits) +{ + struct buffer_head *bh = NULL; + struct ext4_xattr_ibody_header *header; + struct ext4_iloc iloc = { .bh = NULL }; + struct ext4_xattr_entry *entry; + struct inode *ea_inode; + int error; + + error = ext4_journal_ensure_credits(handle, extra_credits, + ext4_free_metadata_revoke_credits(inode->i_sb, 1)); + if (error < 0) { + EXT4_ERROR_INODE(inode, "ensure credits (error %d)", error); + goto cleanup; + } + + if (ext4_has_feature_ea_inode(inode->i_sb) && + ext4_test_inode_state(inode, EXT4_STATE_XATTR)) { + + error = ext4_get_inode_loc(inode, &iloc); + if (error) { + EXT4_ERROR_INODE(inode, "inode loc (error %d)", error); + goto cleanup; + } + + error = ext4_journal_get_write_access(handle, inode->i_sb, + iloc.bh, EXT4_JTR_NONE); + if (error) { + EXT4_ERROR_INODE(inode, "write access (error %d)", + error); + goto cleanup; + } + + header = IHDR(inode, ext4_raw_inode(&iloc)); + if (header->h_magic == cpu_to_le32(EXT4_XATTR_MAGIC)) + ext4_xattr_inode_dec_ref_all(handle, inode, iloc.bh, + IFIRST(header), + false /* block_csum */, + extra_credits, + false /* skip_quota */); + } + + if (EXT4_I(inode)->i_file_acl) { + bh = ext4_sb_bread(inode->i_sb, EXT4_I(inode)->i_file_acl, REQ_PRIO); + if (IS_ERR(bh)) { + error = PTR_ERR(bh); + if (error == -EIO) { + EXT4_ERROR_INODE_ERR(inode, EIO, + "block %llu read error", + EXT4_I(inode)->i_file_acl); + } + bh = NULL; + goto cleanup; + } + error = ext4_xattr_check_block(inode, bh); + if (error) + goto cleanup; + + if (ext4_has_feature_ea_inode(inode->i_sb)) { + for (entry = BFIRST(bh); !IS_LAST_ENTRY(entry); + entry = EXT4_XATTR_NEXT(entry)) { + if (!entry->e_value_inum) + continue; + error = ext4_xattr_inode_iget(inode, + le32_to_cpu(entry->e_value_inum), + le32_to_cpu(entry->e_hash), + &ea_inode); + if (error) + continue; + ext4_xattr_inode_free_quota(inode, ea_inode, + le32_to_cpu(entry->e_value_size)); + ext4_put_ea_inode(ea_inode); + } + + } + + ext4_xattr_release_block(handle, inode, bh, + extra_credits); + /* + * Update i_file_acl value in the same transaction that releases + * block. + */ + EXT4_I(inode)->i_file_acl = 0; + error = ext4_mark_inode_dirty(handle, inode); + if (error) { + EXT4_ERROR_INODE(inode, "mark inode dirty (error %d)", + error); + goto cleanup; + } + ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_XATTR, handle); + } + error = 0; +cleanup: + brelse(iloc.bh); + brelse(bh); + return error; +} + +/* + * Worker function for deferred EA inode iput. Processes all inodes queued + * on s_ea_inode_to_free in a context free of xattr_sem/jbd2 handle locks. + */ +static void ext4_ea_inode_work(struct work_struct *work) +{ + struct ext4_sb_info *sbi = container_of(to_delayed_work(work), + struct ext4_sb_info, + s_ea_inode_work); + struct llist_node *node = llist_del_all(&sbi->s_ea_inode_to_free); + + while (node) { + struct ext4_inode_info *ei = container_of(node, + struct ext4_inode_info, i_ea_iput_node); + node = node->next; + iput(&ei->vfs_inode); + } +} + +/* + * Release a VFS reference on an EA inode. Must be used instead of iput() + * in any context where xattr_sem or a jbd2 handle is held. + * + * If this is not the last reference, drops it immediately via + * iput_if_not_last() with no further action needed. + * + * If this is the last reference, the inode is linked onto a per-sb + * llist via i_ea_iput_node (embedded in ext4_inode_info, sharing space + * with the unused xattr_sem) and a delayed worker performs the final + * iput() in a clean context. + * + * Note: while an inode is on s_ea_inode_to_free, the unconsumed i_count + * reference (still 1) keeps it in the inode cache, so any concurrent + * iget() bumps i_count to >= 2 and iput_if_not_last() will succeed. + * Nobody will add the inode a second time until ext4_ea_inode_work() + * drops that reference via iput(). + */ +void ext4_put_ea_inode(struct inode *inode) +{ + if (!inode) + return; + WARN_ON_ONCE(!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)); + if (iput_if_not_last(inode)) + return; + llist_add(&EXT4_I(inode)->i_ea_iput_node, + &EXT4_SB(inode->i_sb)->s_ea_inode_to_free); + /* + * Use a short delay to allow multiple EA inodes to accumulate, + * reducing workqueue wakeups when several are released together. + */ + schedule_delayed_work(&EXT4_SB(inode->i_sb)->s_ea_inode_work, 1); +} + +void ext4_init_ea_inode_work(struct ext4_sb_info *sbi) +{ + init_llist_head(&sbi->s_ea_inode_to_free); + INIT_DELAYED_WORK(&sbi->s_ea_inode_work, ext4_ea_inode_work); +} + +/* + * ext4_xattr_block_cache_insert() + * + * Create a new entry in the extended attribute block cache, and insert + * it unless such an entry is already in the cache. + */ +static void +ext4_xattr_block_cache_insert(struct mb_cache *ea_block_cache, + struct buffer_head *bh) +{ + struct ext4_xattr_header *header = BHDR(bh); + __u32 hash = le32_to_cpu(header->h_hash); + int reusable = le32_to_cpu(header->h_refcount) < + EXT4_XATTR_REFCOUNT_MAX; + int error; + + if (!ea_block_cache) + return; + error = mb_cache_entry_create(ea_block_cache, GFP_NOFS, hash, + bh->b_blocknr, reusable); + if (error) { + if (error == -EBUSY) + ea_bdebug(bh, "already in cache"); + } else + ea_bdebug(bh, "inserting [%x]", (int)hash); +} + +/* + * ext4_xattr_cmp() + * + * Compare two extended attribute blocks for equality. + * + * Returns 0 if the blocks are equal, 1 if they differ. + */ +static int +ext4_xattr_cmp(struct ext4_xattr_header *header1, + struct ext4_xattr_header *header2) +{ + struct ext4_xattr_entry *entry1, *entry2; + + entry1 = ENTRY(header1+1); + entry2 = ENTRY(header2+1); + while (!IS_LAST_ENTRY(entry1)) { + if (IS_LAST_ENTRY(entry2)) + return 1; + if (entry1->e_hash != entry2->e_hash || + entry1->e_name_index != entry2->e_name_index || + entry1->e_name_len != entry2->e_name_len || + entry1->e_value_size != entry2->e_value_size || + entry1->e_value_inum != entry2->e_value_inum || + memcmp(entry1->e_name, entry2->e_name, entry1->e_name_len)) + return 1; + if (!entry1->e_value_inum && + memcmp((char *)header1 + le16_to_cpu(entry1->e_value_offs), + (char *)header2 + le16_to_cpu(entry2->e_value_offs), + le32_to_cpu(entry1->e_value_size))) + return 1; + + entry1 = EXT4_XATTR_NEXT(entry1); + entry2 = EXT4_XATTR_NEXT(entry2); + } + if (!IS_LAST_ENTRY(entry2)) + return 1; + return 0; +} + +/* + * ext4_xattr_block_cache_find() + * + * Find an identical extended attribute block. + * + * Returns a pointer to the block found, or NULL if such a block was not + * found, or an error pointer if an error occurred while reading ea block. + */ +static struct buffer_head * +ext4_xattr_block_cache_find(struct inode *inode, + struct ext4_xattr_header *header, + struct mb_cache_entry **pce) +{ + __u32 hash = le32_to_cpu(header->h_hash); + struct mb_cache_entry *ce; + struct mb_cache *ea_block_cache = EA_BLOCK_CACHE(inode); + + if (!ea_block_cache) + return NULL; + if (!header->h_hash) + return NULL; /* never share */ + ea_idebug(inode, "looking for cached blocks [%x]", (int)hash); + ce = mb_cache_entry_find_first(ea_block_cache, hash); + while (ce) { + struct buffer_head *bh; + + bh = ext4_sb_bread(inode->i_sb, ce->e_value, REQ_PRIO); + if (IS_ERR(bh)) { + if (PTR_ERR(bh) != -ENOMEM) + EXT4_ERROR_INODE(inode, "block %lu read error", + (unsigned long)ce->e_value); + mb_cache_entry_put(ea_block_cache, ce); + return bh; + } else if (ext4_xattr_cmp(header, BHDR(bh)) == 0) { + *pce = ce; + return bh; + } + brelse(bh); + ce = mb_cache_entry_find_next(ea_block_cache, ce); + } + return NULL; +} + +#define NAME_HASH_SHIFT 5 +#define VALUE_HASH_SHIFT 16 + +/* + * ext4_xattr_hash_entry() + * + * Compute the hash of an extended attribute. + */ +static __le32 ext4_xattr_hash_entry(char *name, size_t name_len, __le32 *value, + size_t value_count) +{ + __u32 hash = 0; + + while (name_len--) { + hash = (hash << NAME_HASH_SHIFT) ^ + (hash >> (8*sizeof(hash) - NAME_HASH_SHIFT)) ^ + (unsigned char)*name++; + } + while (value_count--) { + hash = (hash << VALUE_HASH_SHIFT) ^ + (hash >> (8*sizeof(hash) - VALUE_HASH_SHIFT)) ^ + le32_to_cpu(*value++); + } + return cpu_to_le32(hash); +} + +/* + * ext4_xattr_hash_entry_signed() + * + * Compute the hash of an extended attribute incorrectly. + */ +static __le32 ext4_xattr_hash_entry_signed(char *name, size_t name_len, __le32 *value, size_t value_count) +{ + __u32 hash = 0; + + while (name_len--) { + hash = (hash << NAME_HASH_SHIFT) ^ + (hash >> (8*sizeof(hash) - NAME_HASH_SHIFT)) ^ + (signed char)*name++; + } + while (value_count--) { + hash = (hash << VALUE_HASH_SHIFT) ^ + (hash >> (8*sizeof(hash) - VALUE_HASH_SHIFT)) ^ + le32_to_cpu(*value++); + } + return cpu_to_le32(hash); +} + +#undef NAME_HASH_SHIFT +#undef VALUE_HASH_SHIFT + +#define BLOCK_HASH_SHIFT 16 + +/* + * ext4_xattr_rehash() + * + * Re-compute the extended attribute hash value after an entry has changed. + */ +static void ext4_xattr_rehash(struct ext4_xattr_header *header) +{ + struct ext4_xattr_entry *here; + __u32 hash = 0; + + here = ENTRY(header+1); + while (!IS_LAST_ENTRY(here)) { + if (!here->e_hash) { + /* Block is not shared if an entry's hash value == 0 */ + hash = 0; + break; + } + hash = (hash << BLOCK_HASH_SHIFT) ^ + (hash >> (8*sizeof(hash) - BLOCK_HASH_SHIFT)) ^ + le32_to_cpu(here->e_hash); + here = EXT4_XATTR_NEXT(here); + } + header->h_hash = cpu_to_le32(hash); +} + +#undef BLOCK_HASH_SHIFT + +#define HASH_BUCKET_BITS 10 + +struct mb_cache * +ext4_xattr_create_cache(void) +{ + return mb_cache_create(HASH_BUCKET_BITS); +} + +void ext4_xattr_destroy_cache(struct mb_cache *cache) +{ + if (cache) + mb_cache_destroy(cache); +} + diff --git a/fs/ext4/xattr.h b/fs/ext4/xattr.h new file mode 100644 index 000000000..821dc6a50 --- /dev/null +++ b/fs/ext4/xattr.h @@ -0,0 +1,231 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + File: fs/ext4/xattr.h + + On-disk format of extended attributes for the ext4 filesystem. + + (C) 2001 Andreas Gruenbacher, +*/ + +#include + +/* Magic value in attribute blocks */ +#define EXT4_XATTR_MAGIC 0xEA020000 + +/* Maximum number of references to one attribute block */ +#define EXT4_XATTR_REFCOUNT_MAX 1024 + +/* Name indexes */ +#define EXT4_XATTR_INDEX_USER 1 +#define EXT4_XATTR_INDEX_POSIX_ACL_ACCESS 2 +#define EXT4_XATTR_INDEX_POSIX_ACL_DEFAULT 3 +#define EXT4_XATTR_INDEX_TRUSTED 4 +#define EXT4_XATTR_INDEX_LUSTRE 5 +#define EXT4_XATTR_INDEX_SECURITY 6 +#define EXT4_XATTR_INDEX_SYSTEM 7 +#define EXT4_XATTR_INDEX_RICHACL 8 +#define EXT4_XATTR_INDEX_ENCRYPTION 9 +#define EXT4_XATTR_INDEX_HURD 10 /* Reserved for Hurd */ + +struct ext4_xattr_header { + __le32 h_magic; /* magic number for identification */ + __le32 h_refcount; /* reference count */ + __le32 h_blocks; /* number of disk blocks used */ + __le32 h_hash; /* hash value of all attributes */ + __le32 h_checksum; /* crc32c(uuid+blknum+xattrblock) */ + __u32 h_reserved[3]; /* zero right now */ +}; + +struct ext4_xattr_ibody_header { + __le32 h_magic; /* magic number for identification */ +}; + +struct ext4_xattr_entry { + __u8 e_name_len; /* length of name */ + __u8 e_name_index; /* attribute name index */ + __le16 e_value_offs; /* offset in disk block of value */ + __le32 e_value_inum; /* inode in which the value is stored */ + __le32 e_value_size; /* size of attribute value */ + __le32 e_hash; /* hash value of name and value */ + char e_name[]; /* attribute name */ +}; + +#define EXT4_XATTR_PAD_BITS 2 +#define EXT4_XATTR_PAD (1<e_name_len))) +#define EXT4_XATTR_SIZE(size) \ + (((size) + EXT4_XATTR_ROUND) & ~EXT4_XATTR_ROUND) + +#define IHDR(inode, raw_inode) \ + ((struct ext4_xattr_ibody_header *) \ + ((void *)raw_inode + \ + EXT4_GOOD_OLD_INODE_SIZE + \ + EXT4_I(inode)->i_extra_isize)) +#define ITAIL(inode, raw_inode) \ + ((void *)(raw_inode) + \ + EXT4_SB((inode)->i_sb)->s_inode_size) +#define IFIRST(hdr) ((struct ext4_xattr_entry *)((hdr)+1)) + +/* + * XATTR_SIZE_MAX is currently 64k, but for the purposes of checking + * for file system consistency errors, we use a somewhat bigger value. + * This allows XATTR_SIZE_MAX to grow in the future, but by using this + * instead of INT_MAX for certain consistency checks, we don't need to + * worry about arithmetic overflows. (Actually XATTR_SIZE_MAX is + * defined in include/uapi/linux/limits.h, so changing it is going + * not going to be trivial....) + */ +#define EXT4_XATTR_SIZE_MAX (1 << 24) + +/* + * The minimum size of EA value when you start storing it in an external inode + * size of block - size of header - size of 1 entry - 4 null bytes + */ +#define EXT4_XATTR_MIN_LARGE_EA_SIZE(b) \ + ((b) - EXT4_XATTR_LEN(3) - sizeof(struct ext4_xattr_header) - 4) + +#define BHDR(bh) ((struct ext4_xattr_header *)((bh)->b_data)) +#define ENTRY(ptr) ((struct ext4_xattr_entry *)(ptr)) +#define BFIRST(bh) ENTRY(BHDR(bh)+1) +#define IS_LAST_ENTRY(entry) (*(__u32 *)(entry) == 0) + +#define EXT4_ZERO_XATTR_VALUE ((void *)-1) + +/* + * If we want to add an xattr to the inode, we should make sure that + * i_extra_isize is not 0 and that the inode size is not less than + * EXT4_GOOD_OLD_INODE_SIZE + extra_isize + pad. + * EXT4_GOOD_OLD_INODE_SIZE extra_isize header entry pad data + * |--------------------------|------------|------|---------|---|-------| + */ +#define EXT4_INODE_HAS_XATTR_SPACE(inode) \ + ((EXT4_I(inode)->i_extra_isize != 0) && \ + (EXT4_GOOD_OLD_INODE_SIZE + EXT4_I(inode)->i_extra_isize + \ + sizeof(struct ext4_xattr_ibody_header) + EXT4_XATTR_PAD <= \ + EXT4_INODE_SIZE((inode)->i_sb))) + +struct ext4_xattr_info { + const char *name; + const void *value; + size_t value_len; + int name_index; + int in_inode; +}; + +struct ext4_xattr_search { + struct ext4_xattr_entry *first; + void *base; + void *end; + struct ext4_xattr_entry *here; + int not_found; +}; + +struct ext4_xattr_ibody_find { + struct ext4_xattr_search s; + struct ext4_iloc iloc; +}; + +extern const struct xattr_handler ext4_xattr_user_handler; +extern const struct xattr_handler ext4_xattr_trusted_handler; +extern const struct xattr_handler ext4_xattr_security_handler; +extern const struct xattr_handler ext4_xattr_hurd_handler; + +#define EXT4_XATTR_NAME_ENCRYPTION_CONTEXT "c" + +/* + * The EXT4_STATE_NO_EXPAND is overloaded and used for two purposes. + * The first is to signal that there the inline xattrs and data are + * taking up so much space that we might as well not keep trying to + * expand it. The second is that xattr_sem is taken for writing, so + * we shouldn't try to recurse into the inode expansion. For this + * second case, we need to make sure that we take save and restore the + * NO_EXPAND state flag appropriately. + */ +static inline void ext4_write_lock_xattr(struct inode *inode, int *save) +{ + down_write(&EXT4_I(inode)->xattr_sem); + *save = ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND); + ext4_set_inode_state(inode, EXT4_STATE_NO_EXPAND); +} + +static inline int ext4_write_trylock_xattr(struct inode *inode, int *save) +{ + if (down_write_trylock(&EXT4_I(inode)->xattr_sem) == 0) + return 0; + *save = ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND); + ext4_set_inode_state(inode, EXT4_STATE_NO_EXPAND); + return 1; +} + +static inline void ext4_write_unlock_xattr(struct inode *inode, int *save) +{ + if (*save == 0) + ext4_clear_inode_state(inode, EXT4_STATE_NO_EXPAND); + up_write(&EXT4_I(inode)->xattr_sem); +} + +extern ssize_t ext4_listxattr(struct dentry *, char *, size_t); + +extern int ext4_xattr_get(struct inode *, int, const char *, void *, size_t); +extern int ext4_xattr_set(struct inode *, int, const char *, const void *, size_t, int); +extern int ext4_xattr_set_handle(handle_t *, struct inode *, int, const char *, const void *, size_t, int); +extern int ext4_xattr_set_credits(struct inode *inode, size_t value_len, + bool is_create, int *credits); +extern int __ext4_xattr_set_credits(struct super_block *sb, struct inode *inode, + struct buffer_head *block_bh, size_t value_len, + bool is_create); + +extern int ext4_xattr_delete_inode(handle_t *handle, struct inode *inode, + int extra_credits); +extern void ext4_init_ea_inode_work(struct ext4_sb_info *sbi); +extern void ext4_put_ea_inode(struct inode *inode); + +extern int ext4_expand_extra_isize_ea(struct inode *inode, int new_extra_isize, + struct ext4_inode *raw_inode, handle_t *handle); +extern void ext4_evict_ea_inode(struct inode *inode); + +extern const struct xattr_handler * const ext4_xattr_handlers[]; + +extern int ext4_xattr_ibody_find(struct inode *inode, struct ext4_xattr_info *i, + struct ext4_xattr_ibody_find *is); +extern int ext4_xattr_ibody_get(struct inode *inode, int name_index, + const char *name, + void *buffer, size_t buffer_size); +extern int ext4_xattr_ibody_set(handle_t *handle, struct inode *inode, + struct ext4_xattr_info *i, + struct ext4_xattr_ibody_find *is); + +extern struct mb_cache *ext4_xattr_create_cache(void); +extern void ext4_xattr_destroy_cache(struct mb_cache *); + +extern int +__xattr_check_inode(struct inode *inode, struct ext4_xattr_ibody_header *header, + void *end, const char *function, unsigned int line); + +#define xattr_check_inode(inode, header, end) \ + __xattr_check_inode((inode), (header), (end), __func__, __LINE__) + +#ifdef CONFIG_EXT4_FS_SECURITY +extern int ext4_init_security(handle_t *handle, struct inode *inode, + struct inode *dir, const struct qstr *qstr); +#else +static inline int ext4_init_security(handle_t *handle, struct inode *inode, + struct inode *dir, const struct qstr *qstr) +{ + return 0; +} +#endif + +#ifdef CONFIG_LOCKDEP +extern void ext4_xattr_inode_set_class(struct inode *ea_inode); +#else +static inline void ext4_xattr_inode_set_class(struct inode *ea_inode) { } +#endif + +extern int ext4_get_inode_usage(struct inode *inode, qsize_t *usage); diff --git a/fs/ext4/xattr_hurd.c b/fs/ext4/xattr_hurd.c new file mode 100644 index 000000000..8a5842e4c --- /dev/null +++ b/fs/ext4/xattr_hurd.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext4/xattr_hurd.c + * Handler for extended gnu attributes for the Hurd. + * + * Copyright (C) 2001 by Andreas Gruenbacher, + * Copyright (C) 2020 by Jan (janneke) Nieuwenhuizen, + */ + +#include +#include +#include "ext4.h" +#include "xattr.h" + +static bool +ext4_xattr_hurd_list(struct dentry *dentry) +{ + return test_opt(dentry->d_sb, XATTR_USER); +} + +static int +ext4_xattr_hurd_get(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + if (!test_opt(inode->i_sb, XATTR_USER)) + return -EOPNOTSUPP; + + return ext4_xattr_get(inode, EXT4_XATTR_INDEX_HURD, + name, buffer, size); +} + +static int +ext4_xattr_hurd_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +{ + if (!test_opt(inode->i_sb, XATTR_USER)) + return -EOPNOTSUPP; + + return ext4_xattr_set(inode, EXT4_XATTR_INDEX_HURD, + name, value, size, flags); +} + +const struct xattr_handler ext4_xattr_hurd_handler = { + .prefix = XATTR_HURD_PREFIX, + .list = ext4_xattr_hurd_list, + .get = ext4_xattr_hurd_get, + .set = ext4_xattr_hurd_set, +}; diff --git a/fs/ext4/xattr_security.c b/fs/ext4/xattr_security.c new file mode 100644 index 000000000..776cf11d2 --- /dev/null +++ b/fs/ext4/xattr_security.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext4/xattr_security.c + * Handler for storing security labels as extended attributes. + */ + +#include +#include +#include +#include +#include "ext4_jbd2.h" +#include "ext4.h" +#include "xattr.h" + +static int +ext4_xattr_security_get(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + return ext4_xattr_get(inode, EXT4_XATTR_INDEX_SECURITY, + name, buffer, size); +} + +static int +ext4_xattr_security_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +{ + return ext4_xattr_set(inode, EXT4_XATTR_INDEX_SECURITY, + name, value, size, flags); +} + +static int +ext4_initxattrs(struct inode *inode, const struct xattr *xattr_array, + void *fs_info) +{ + const struct xattr *xattr; + handle_t *handle = fs_info; + int err = 0; + + for (xattr = xattr_array; xattr->name != NULL; xattr++) { + err = ext4_xattr_set_handle(handle, inode, + EXT4_XATTR_INDEX_SECURITY, + xattr->name, xattr->value, + xattr->value_len, XATTR_CREATE); + if (err < 0) + break; + } + return err; +} + +int +ext4_init_security(handle_t *handle, struct inode *inode, struct inode *dir, + const struct qstr *qstr) +{ + return security_inode_init_security(inode, dir, qstr, + &ext4_initxattrs, handle); +} + +const struct xattr_handler ext4_xattr_security_handler = { + .prefix = XATTR_SECURITY_PREFIX, + .get = ext4_xattr_security_get, + .set = ext4_xattr_security_set, +}; diff --git a/fs/ext4/xattr_trusted.c b/fs/ext4/xattr_trusted.c new file mode 100644 index 000000000..9811eb0ab --- /dev/null +++ b/fs/ext4/xattr_trusted.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext4/xattr_trusted.c + * Handler for trusted extended attributes. + * + * Copyright (C) 2003 by Andreas Gruenbacher, + */ + +#include +#include +#include +#include "ext4_jbd2.h" +#include "ext4.h" +#include "xattr.h" + +static bool +ext4_xattr_trusted_list(struct dentry *dentry) +{ + return capable(CAP_SYS_ADMIN); +} + +static int +ext4_xattr_trusted_get(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + return ext4_xattr_get(inode, EXT4_XATTR_INDEX_TRUSTED, + name, buffer, size); +} + +static int +ext4_xattr_trusted_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +{ + return ext4_xattr_set(inode, EXT4_XATTR_INDEX_TRUSTED, + name, value, size, flags); +} + +const struct xattr_handler ext4_xattr_trusted_handler = { + .prefix = XATTR_TRUSTED_PREFIX, + .list = ext4_xattr_trusted_list, + .get = ext4_xattr_trusted_get, + .set = ext4_xattr_trusted_set, +}; diff --git a/fs/ext4/xattr_user.c b/fs/ext4/xattr_user.c new file mode 100644 index 000000000..4b70bf4e7 --- /dev/null +++ b/fs/ext4/xattr_user.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ext4/xattr_user.c + * Handler for extended user attributes. + * + * Copyright (C) 2001 by Andreas Gruenbacher, + */ + +#include +#include +#include "ext4_jbd2.h" +#include "ext4.h" +#include "xattr.h" + +static bool +ext4_xattr_user_list(struct dentry *dentry) +{ + return test_opt(dentry->d_sb, XATTR_USER); +} + +static int +ext4_xattr_user_get(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + if (!test_opt(inode->i_sb, XATTR_USER)) + return -EOPNOTSUPP; + return ext4_xattr_get(inode, EXT4_XATTR_INDEX_USER, + name, buffer, size); +} + +static int +ext4_xattr_user_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +{ + if (!test_opt(inode->i_sb, XATTR_USER)) + return -EOPNOTSUPP; + return ext4_xattr_set(inode, EXT4_XATTR_INDEX_USER, + name, value, size, flags); +} + +const struct xattr_handler ext4_xattr_user_handler = { + .prefix = XATTR_USER_PREFIX, + .list = ext4_xattr_user_list, + .get = ext4_xattr_user_get, + .set = ext4_xattr_user_set, +}; diff --git a/fs/f2fs/Kconfig b/fs/f2fs/Kconfig new file mode 100644 index 000000000..5916a02fb --- /dev/null +++ b/fs/f2fs/Kconfig @@ -0,0 +1,152 @@ +# SPDX-License-Identifier: GPL-2.0-only +config F2FS_FS + tristate "F2FS filesystem support" + depends on BLOCK + select BUFFER_HEAD + select NLS + select CRC32 + select F2FS_FS_XATTR if FS_ENCRYPTION + select FS_ENCRYPTION_ALGS if FS_ENCRYPTION + select FS_IOMAP + select LZ4_COMPRESS if F2FS_FS_LZ4 + select LZ4_DECOMPRESS if F2FS_FS_LZ4 + select LZ4HC_COMPRESS if F2FS_FS_LZ4HC + select LZO_COMPRESS if F2FS_FS_LZO + select LZO_DECOMPRESS if F2FS_FS_LZO + select ZSTD_COMPRESS if F2FS_FS_ZSTD + select ZSTD_DECOMPRESS if F2FS_FS_ZSTD + help + F2FS is based on Log-structured File System (LFS), which supports + versatile "flash-friendly" features. The design has been focused on + addressing the fundamental issues in LFS, which are snowball effect + of wandering tree and high cleaning overhead. + + Since flash-based storages show different characteristics according to + the internal geometry or flash memory management schemes aka FTL, F2FS + and tools support various parameters not only for configuring on-disk + layout, but also for selecting allocation and cleaning algorithms. + + If unsure, say N. + +config F2FS_STAT_FS + bool "F2FS Status Information" + depends on F2FS_FS + default y + help + /sys/kernel/debug/f2fs/ contains information about all the partitions + mounted as f2fs. Each file shows the whole f2fs information. + + /sys/kernel/debug/f2fs/status includes: + - major filesystem information managed by f2fs currently + - average SIT information about whole segments + - current memory footprint consumed by f2fs. + +config F2FS_FS_XATTR + bool "F2FS extended attributes" + depends on F2FS_FS + default y + help + Extended attributes are name:value pairs associated with inodes by + the kernel or by users (see the attr(5) manual page for details). + + If unsure, say N. + +config F2FS_FS_POSIX_ACL + bool "F2FS Access Control Lists" + depends on F2FS_FS_XATTR + select FS_POSIX_ACL + default y + help + Posix Access Control Lists (ACLs) support permissions for users and + groups beyond the owner/group/world scheme. + + If you don't know what Access Control Lists are, say N + +config F2FS_FS_SECURITY + bool "F2FS Security Labels" + depends on F2FS_FS_XATTR + help + Security labels provide an access control facility to support Linux + Security Models (LSMs) accepted by AppArmor, SELinux, Smack and TOMOYO + Linux. This option enables an extended attribute handler for file + security labels in the f2fs filesystem, so that it requires enabling + the extended attribute support in advance. In particular you need this + option if you use the setcap command to assign initial process capabi- + lities to executables (the security.* extended attributes). + + If you are not using a security module, say N. + +config F2FS_CHECK_FS + bool "F2FS consistency checking feature" + depends on F2FS_FS + help + Enables BUG_ONs which check the filesystem consistency in runtime. + + If you want to improve the performance, say N. + +config F2FS_FAULT_INJECTION + bool "F2FS fault injection facility" + depends on F2FS_FS + help + Test F2FS to inject faults such as ENOMEM, ENOSPC, and so on. + + If unsure, say N. + +config F2FS_FS_COMPRESSION + bool "F2FS compression feature" + depends on F2FS_FS + help + Enable filesystem-level compression on f2fs regular files, + multiple back-end compression algorithms are supported. + +config F2FS_FS_LZO + bool "LZO compression support" + depends on F2FS_FS_COMPRESSION + default y + help + Support LZO compress algorithm, if unsure, say Y. + +config F2FS_FS_LZORLE + bool "LZO-RLE compression support" + depends on F2FS_FS_LZO + default y + help + Support LZO-RLE compress algorithm, if unsure, say Y. + +config F2FS_FS_LZ4 + bool "LZ4 compression support" + depends on F2FS_FS_COMPRESSION + default y + help + Support LZ4 compress algorithm, if unsure, say Y. + +config F2FS_FS_LZ4HC + bool "LZ4HC compression support" + depends on F2FS_FS_LZ4 + default y + help + Support LZ4HC compress algorithm, LZ4HC has compatible on-disk + layout with LZ4, if unsure, say Y. + +config F2FS_FS_ZSTD + bool "ZSTD compression support" + depends on F2FS_FS_COMPRESSION + default y + help + Support ZSTD compress algorithm, if unsure, say Y. + +config F2FS_IOSTAT + bool "F2FS IO statistics information" + depends on F2FS_FS + default y + help + Support getting IO statistics through sysfs and printing out periodic + IO statistics tracepoint events. You have to turn on "iostat_enable" + sysfs node to enable this feature. + +config F2FS_UNFAIR_RWSEM + bool "F2FS unfair rw_semaphore" + depends on F2FS_FS && BLK_CGROUP + help + Use unfair rw_semaphore, if system configured IO priority by block + cgroup. diff --git a/fs/f2fs/Makefile b/fs/f2fs/Makefile new file mode 100644 index 000000000..8a7322d22 --- /dev/null +++ b/fs/f2fs/Makefile @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_F2FS_FS) += f2fs.o + +f2fs-y := dir.o file.o inode.o namei.o hash.o super.o inline.o +f2fs-y += checkpoint.o gc.o data.o node.o segment.o recovery.o +f2fs-y += shrinker.o extent_cache.o sysfs.o +f2fs-$(CONFIG_F2FS_STAT_FS) += debug.o +f2fs-$(CONFIG_F2FS_FS_XATTR) += xattr.o +f2fs-$(CONFIG_F2FS_FS_POSIX_ACL) += acl.o +f2fs-$(CONFIG_FS_VERITY) += verity.o +f2fs-$(CONFIG_F2FS_FS_COMPRESSION) += compress.o +f2fs-$(CONFIG_F2FS_IOSTAT) += iostat.o diff --git a/fs/f2fs/acl.c b/fs/f2fs/acl.c new file mode 100644 index 000000000..d32535491 --- /dev/null +++ b/fs/f2fs/acl.c @@ -0,0 +1,455 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/f2fs/acl.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * + * Portions of this code from linux/fs/ext2/acl.c + * + * Copyright (C) 2001-2003 Andreas Gruenbacher, + */ +#include +#include +#include "f2fs.h" +#include "xattr.h" +#include "acl.h" + +static inline size_t f2fs_acl_size(int count) +{ + if (count <= 4) { + return sizeof(struct f2fs_acl_header) + + count * sizeof(struct f2fs_acl_entry_short); + } else { + return sizeof(struct f2fs_acl_header) + + 4 * sizeof(struct f2fs_acl_entry_short) + + (count - 4) * sizeof(struct f2fs_acl_entry); + } +} + +static inline int f2fs_acl_count(size_t size) +{ + ssize_t s; + + size -= sizeof(struct f2fs_acl_header); + s = size - 4 * sizeof(struct f2fs_acl_entry_short); + if (s < 0) { + if (size % sizeof(struct f2fs_acl_entry_short)) + return -1; + return size / sizeof(struct f2fs_acl_entry_short); + } else { + if (s % sizeof(struct f2fs_acl_entry)) + return -1; + return s / sizeof(struct f2fs_acl_entry) + 4; + } +} + +static struct posix_acl *f2fs_acl_from_disk(const char *value, size_t size) +{ + int i, count; + int err = -EINVAL; + struct posix_acl *acl; + struct f2fs_acl_header *hdr = (struct f2fs_acl_header *)value; + struct f2fs_acl_entry *entry = (struct f2fs_acl_entry *)(hdr + 1); + const char *end = value + size; + + if (size < sizeof(struct f2fs_acl_header)) + return ERR_PTR(-EINVAL); + + if (hdr->a_version != cpu_to_le32(F2FS_ACL_VERSION)) + return ERR_PTR(-EINVAL); + + count = f2fs_acl_count(size); + if (count < 0) + return ERR_PTR(-EINVAL); + if (count == 0) + return NULL; + + acl = posix_acl_alloc(count, GFP_NOFS); + if (!acl) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < count; i++) { + + if (unlikely((char *)entry + + sizeof(struct f2fs_acl_entry_short) > end)) { + err = -EFSCORRUPTED; + goto fail; + } + + acl->a_entries[i].e_tag = le16_to_cpu(entry->e_tag); + acl->a_entries[i].e_perm = le16_to_cpu(entry->e_perm); + + switch (acl->a_entries[i].e_tag) { + case ACL_USER_OBJ: + case ACL_GROUP_OBJ: + case ACL_MASK: + case ACL_OTHER: + entry = (struct f2fs_acl_entry *)((char *)entry + + sizeof(struct f2fs_acl_entry_short)); + break; + + case ACL_USER: + if (unlikely((char *)entry + + sizeof(struct f2fs_acl_entry) > end)) { + err = -EFSCORRUPTED; + goto fail; + } + acl->a_entries[i].e_uid = + make_kuid(&init_user_ns, + le32_to_cpu(entry->e_id)); + entry = (struct f2fs_acl_entry *)((char *)entry + + sizeof(struct f2fs_acl_entry)); + break; + case ACL_GROUP: + if (unlikely((char *)entry + + sizeof(struct f2fs_acl_entry) > end)) { + err = -EFSCORRUPTED; + goto fail; + } + acl->a_entries[i].e_gid = + make_kgid(&init_user_ns, + le32_to_cpu(entry->e_id)); + entry = (struct f2fs_acl_entry *)((char *)entry + + sizeof(struct f2fs_acl_entry)); + break; + default: + goto fail; + } + } + if ((char *)entry != end) + goto fail; + return acl; +fail: + posix_acl_release(acl); + return ERR_PTR(err); +} + +static void *f2fs_acl_to_disk(struct f2fs_sb_info *sbi, + const struct posix_acl *acl, size_t *size) +{ + struct f2fs_acl_header *f2fs_acl; + struct f2fs_acl_entry *entry; + int i; + + f2fs_acl = f2fs_kmalloc(sbi, sizeof(struct f2fs_acl_header) + + acl->a_count * sizeof(struct f2fs_acl_entry), + GFP_NOFS); + if (!f2fs_acl) + return ERR_PTR(-ENOMEM); + + f2fs_acl->a_version = cpu_to_le32(F2FS_ACL_VERSION); + entry = (struct f2fs_acl_entry *)(f2fs_acl + 1); + + for (i = 0; i < acl->a_count; i++) { + + entry->e_tag = cpu_to_le16(acl->a_entries[i].e_tag); + entry->e_perm = cpu_to_le16(acl->a_entries[i].e_perm); + + switch (acl->a_entries[i].e_tag) { + case ACL_USER: + entry->e_id = cpu_to_le32( + from_kuid(&init_user_ns, + acl->a_entries[i].e_uid)); + entry = (struct f2fs_acl_entry *)((char *)entry + + sizeof(struct f2fs_acl_entry)); + break; + case ACL_GROUP: + entry->e_id = cpu_to_le32( + from_kgid(&init_user_ns, + acl->a_entries[i].e_gid)); + entry = (struct f2fs_acl_entry *)((char *)entry + + sizeof(struct f2fs_acl_entry)); + break; + case ACL_USER_OBJ: + case ACL_GROUP_OBJ: + case ACL_MASK: + case ACL_OTHER: + entry = (struct f2fs_acl_entry *)((char *)entry + + sizeof(struct f2fs_acl_entry_short)); + break; + default: + goto fail; + } + } + *size = f2fs_acl_size(acl->a_count); + return (void *)f2fs_acl; + +fail: + kfree(f2fs_acl); + return ERR_PTR(-EINVAL); +} + +static struct posix_acl *__f2fs_get_acl(struct inode *inode, int type, + struct folio *dfolio) +{ + int name_index = F2FS_XATTR_INDEX_POSIX_ACL_DEFAULT; + void *value = NULL; + struct posix_acl *acl; + int retval; + + if (type == ACL_TYPE_ACCESS) + name_index = F2FS_XATTR_INDEX_POSIX_ACL_ACCESS; + + retval = f2fs_getxattr(inode, name_index, "", NULL, 0, dfolio); + if (retval > 0) { + value = f2fs_kmalloc(F2FS_I_SB(inode), retval, GFP_F2FS_ZERO); + if (!value) + return ERR_PTR(-ENOMEM); + retval = f2fs_getxattr(inode, name_index, "", value, + retval, dfolio); + } + + if (retval > 0) + acl = f2fs_acl_from_disk(value, retval); + else if (retval == -ENODATA) + acl = NULL; + else + acl = ERR_PTR(retval); + kfree(value); + + return acl; +} + +struct posix_acl *f2fs_get_acl(struct inode *inode, int type, bool rcu) +{ + if (rcu) + return ERR_PTR(-ECHILD); + + return __f2fs_get_acl(inode, type, NULL); +} + +static int f2fs_acl_update_mode(struct mnt_idmap *idmap, + struct inode *inode, umode_t *mode_p, + struct posix_acl **acl) +{ + umode_t mode = inode->i_mode; + int error; + + if (is_inode_flag_set(inode, FI_ACL_MODE)) + mode = F2FS_I(inode)->i_acl_mode; + + error = posix_acl_equiv_mode(*acl, &mode); + if (error < 0) + return error; + if (error == 0) + *acl = NULL; + if (!in_group_or_capable(idmap, inode, i_gid_into_vfsgid(idmap, inode))) + mode &= ~S_ISGID; + *mode_p = mode; + return 0; +} + +static int __f2fs_set_acl(struct mnt_idmap *idmap, + struct inode *inode, int type, + struct posix_acl *acl, struct folio *ifolio) +{ + int name_index; + void *value = NULL; + size_t size = 0; + int error; + umode_t mode = inode->i_mode; + + switch (type) { + case ACL_TYPE_ACCESS: + name_index = F2FS_XATTR_INDEX_POSIX_ACL_ACCESS; + if (acl && !ifolio) { + error = f2fs_acl_update_mode(idmap, inode, &mode, &acl); + if (error) + return error; + set_acl_inode(inode, mode); + } + break; + + case ACL_TYPE_DEFAULT: + name_index = F2FS_XATTR_INDEX_POSIX_ACL_DEFAULT; + if (!S_ISDIR(inode->i_mode)) + return acl ? -EACCES : 0; + break; + + default: + return -EINVAL; + } + + if (acl) { + value = f2fs_acl_to_disk(F2FS_I_SB(inode), acl, &size); + if (IS_ERR(value)) { + clear_inode_flag(inode, FI_ACL_MODE); + return PTR_ERR(value); + } + } + + error = f2fs_setxattr(inode, name_index, "", value, size, ifolio, 0); + + kfree(value); + if (!error) + set_cached_acl(inode, type, acl); + + clear_inode_flag(inode, FI_ACL_MODE); + return error; +} + +int f2fs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type) +{ + struct inode *inode = d_inode(dentry); + + if (unlikely(f2fs_cp_error(F2FS_I_SB(inode)))) + return -EIO; + + return __f2fs_set_acl(idmap, inode, type, acl, NULL); +} + +/* + * Most part of f2fs_acl_clone, f2fs_acl_create_masq, f2fs_acl_create + * are copied from posix_acl.c + */ +static struct posix_acl *f2fs_acl_clone(const struct posix_acl *acl, + gfp_t flags) +{ + struct posix_acl *clone = NULL; + + if (acl) { + clone = kmemdup(acl, struct_size(acl, a_entries, acl->a_count), + flags); + if (clone) + refcount_set(&clone->a_refcount, 1); + } + return clone; +} + +static int f2fs_acl_create_masq(struct posix_acl *acl, umode_t *mode_p) +{ + struct posix_acl_entry *pa, *pe; + struct posix_acl_entry *group_obj = NULL, *mask_obj = NULL; + umode_t mode = *mode_p; + int not_equiv = 0; + + /* assert(atomic_read(acl->a_refcount) == 1); */ + + FOREACH_ACL_ENTRY(pa, acl, pe) { + switch (pa->e_tag) { + case ACL_USER_OBJ: + pa->e_perm &= (mode >> 6) | ~S_IRWXO; + mode &= (pa->e_perm << 6) | ~S_IRWXU; + break; + + case ACL_USER: + case ACL_GROUP: + not_equiv = 1; + break; + + case ACL_GROUP_OBJ: + group_obj = pa; + break; + + case ACL_OTHER: + pa->e_perm &= mode | ~S_IRWXO; + mode &= pa->e_perm | ~S_IRWXO; + break; + + case ACL_MASK: + mask_obj = pa; + not_equiv = 1; + break; + + default: + return -EIO; + } + } + + if (mask_obj) { + mask_obj->e_perm &= (mode >> 3) | ~S_IRWXO; + mode &= (mask_obj->e_perm << 3) | ~S_IRWXG; + } else { + if (!group_obj) + return -EIO; + group_obj->e_perm &= (mode >> 3) | ~S_IRWXO; + mode &= (group_obj->e_perm << 3) | ~S_IRWXG; + } + + *mode_p = (*mode_p & ~S_IRWXUGO) | mode; + return not_equiv; +} + +static int f2fs_acl_create(struct inode *dir, umode_t *mode, + struct posix_acl **default_acl, struct posix_acl **acl, + struct folio *dfolio) +{ + struct posix_acl *p; + struct posix_acl *clone; + int ret; + + *acl = NULL; + *default_acl = NULL; + + if (S_ISLNK(*mode) || !IS_POSIXACL(dir)) + return 0; + + p = __f2fs_get_acl(dir, ACL_TYPE_DEFAULT, dfolio); + if (!p || p == ERR_PTR(-EOPNOTSUPP)) { + *mode &= ~current_umask(); + return 0; + } + if (IS_ERR(p)) + return PTR_ERR(p); + + clone = f2fs_acl_clone(p, GFP_NOFS); + if (!clone) { + ret = -ENOMEM; + goto release_acl; + } + + ret = f2fs_acl_create_masq(clone, mode); + if (ret < 0) + goto release_clone; + + if (ret == 0) + posix_acl_release(clone); + else + *acl = clone; + + if (!S_ISDIR(*mode)) + posix_acl_release(p); + else + *default_acl = p; + + return 0; + +release_clone: + posix_acl_release(clone); +release_acl: + posix_acl_release(p); + return ret; +} + +int f2fs_init_acl(struct inode *inode, struct inode *dir, struct folio *ifolio, + struct folio *dfolio) +{ + struct posix_acl *default_acl = NULL, *acl = NULL; + int error; + + error = f2fs_acl_create(dir, &inode->i_mode, &default_acl, &acl, dfolio); + if (error) + return error; + + f2fs_mark_inode_dirty_sync(inode, true); + + if (default_acl) { + error = __f2fs_set_acl(NULL, inode, ACL_TYPE_DEFAULT, + default_acl, ifolio); + posix_acl_release(default_acl); + } else { + inode->i_default_acl = NULL; + } + if (acl) { + if (!error) + error = __f2fs_set_acl(NULL, inode, ACL_TYPE_ACCESS, + acl, ifolio); + posix_acl_release(acl); + } else { + inode->i_acl = NULL; + } + + return error; +} diff --git a/fs/f2fs/acl.h b/fs/f2fs/acl.h new file mode 100644 index 000000000..20e87e63c --- /dev/null +++ b/fs/f2fs/acl.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * fs/f2fs/acl.h + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * + * Portions of this code from linux/fs/ext2/acl.h + * + * Copyright (C) 2001-2003 Andreas Gruenbacher, + */ +#ifndef __F2FS_ACL_H__ +#define __F2FS_ACL_H__ + +#include + +#define F2FS_ACL_VERSION 0x0001 + +struct f2fs_acl_entry { + __le16 e_tag; + __le16 e_perm; + __le32 e_id; +}; + +struct f2fs_acl_entry_short { + __le16 e_tag; + __le16 e_perm; +}; + +struct f2fs_acl_header { + __le32 a_version; +}; + +#ifdef CONFIG_F2FS_FS_POSIX_ACL + +struct posix_acl *f2fs_get_acl(struct inode *, int, bool); +int f2fs_set_acl(struct mnt_idmap *, struct dentry *, + struct posix_acl *, int); +int f2fs_init_acl(struct inode *, struct inode *, struct folio *ifolio, + struct folio *dfolio); +#else +#define f2fs_get_acl NULL +#define f2fs_set_acl NULL + +static inline int f2fs_init_acl(struct inode *inode, struct inode *dir, + struct folio *ifolio, struct folio *dfolio) +{ + return 0; +} +#endif +#endif /* __F2FS_ACL_H__ */ diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c new file mode 100644 index 000000000..064f5b537 --- /dev/null +++ b/fs/f2fs/checkpoint.c @@ -0,0 +1,2207 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/f2fs/checkpoint.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "f2fs.h" +#include "node.h" +#include "segment.h" +#include "iostat.h" +#include + +static inline void get_lock_elapsed_time(struct f2fs_time_stat *ts) +{ + ts->total_time = ktime_get(); +#ifdef CONFIG_64BIT + ts->running_time = current->se.sum_exec_runtime; +#endif +#if defined(CONFIG_SCHED_INFO) && defined(CONFIG_SCHEDSTATS) + ts->runnable_time = current->sched_info.run_delay; +#endif +#ifdef CONFIG_TASK_DELAY_ACCT + if (current->delays) + ts->io_sleep_time = current->delays->blkio_delay; +#endif +} + +static inline void trace_lock_elapsed_time_start(struct f2fs_rwsem *sem, + struct f2fs_lock_context *lc) +{ + lc->lock_trace = trace_f2fs_lock_elapsed_time_enabled(); + if (!lc->lock_trace) + return; + + get_lock_elapsed_time(&lc->ts); +} + +static inline void trace_lock_elapsed_time_end(struct f2fs_rwsem *sem, + struct f2fs_lock_context *lc, bool is_write) +{ + struct f2fs_time_stat tts; + unsigned long long total_time; + unsigned long long running_time = 0; + unsigned long long runnable_time = 0; + unsigned long long io_sleep_time = 0; + unsigned long long other_time = 0; + unsigned npm = NSEC_PER_MSEC; + + if (!lc->lock_trace) + return; + + if (time_to_inject(sem->sbi, FAULT_LOCK_TIMEOUT)) + f2fs_schedule_timeout_killable(DEFAULT_FAULT_TIMEOUT, true); + + get_lock_elapsed_time(&tts); + + total_time = div_u64(tts.total_time - lc->ts.total_time, npm); + if (total_time <= sem->sbi->max_lock_elapsed_time) + return; + +#ifdef CONFIG_64BIT + running_time = div_u64(tts.running_time - lc->ts.running_time, npm); +#endif +#if defined(CONFIG_SCHED_INFO) && defined(CONFIG_SCHEDSTATS) + runnable_time = div_u64(tts.runnable_time - lc->ts.runnable_time, npm); +#endif +#ifdef CONFIG_TASK_DELAY_ACCT + io_sleep_time = div_u64(tts.io_sleep_time - lc->ts.io_sleep_time, npm); +#endif + if (total_time > running_time + io_sleep_time + runnable_time) + other_time = total_time - running_time - + io_sleep_time - runnable_time; + + trace_f2fs_lock_elapsed_time(sem->sbi, sem->name, is_write, current, + get_current_ioprio(), total_time, running_time, + runnable_time, io_sleep_time, other_time); +} + +static bool need_uplift_priority(struct f2fs_rwsem *sem, bool is_write) +{ + if (!(sem->sbi->adjust_lock_priority & BIT(sem->name - 1))) + return false; + + switch (sem->name) { + /* + * writer is checkpoint which has high priority, let's just uplift + * priority for reader + */ + case LOCK_NAME_CP_RWSEM: + case LOCK_NAME_NODE_CHANGE: + case LOCK_NAME_NODE_WRITE: + return !is_write; + case LOCK_NAME_GC_LOCK: + case LOCK_NAME_CP_GLOBAL: + case LOCK_NAME_IO_RWSEM: + return true; + default: + f2fs_bug_on(sem->sbi, 1); + } + return false; +} + +static void uplift_priority(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc, + bool is_write) +{ + lc->need_restore = false; + if (!sem->sbi->adjust_lock_priority) + return; + if (rt_task(current)) + return; + if (!need_uplift_priority(sem, is_write)) + return; + lc->orig_nice = task_nice(current); + lc->new_nice = PRIO_TO_NICE(sem->sbi->lock_duration_priority); + if (lc->orig_nice <= lc->new_nice) + return; + set_user_nice(current, lc->new_nice); + lc->need_restore = true; + + trace_f2fs_priority_uplift(sem->sbi, sem->name, is_write, current, + NICE_TO_PRIO(lc->orig_nice), NICE_TO_PRIO(lc->new_nice)); +} + +static void restore_priority(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc, + bool is_write) +{ + if (!lc->need_restore) + return; + /* someone has updated the priority */ + if (task_nice(current) != lc->new_nice) + return; + set_user_nice(current, lc->orig_nice); + + trace_f2fs_priority_restore(sem->sbi, sem->name, is_write, current, + NICE_TO_PRIO(lc->orig_nice), NICE_TO_PRIO(lc->new_nice)); +} + +void f2fs_down_read_trace(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc) +{ + uplift_priority(sem, lc, false); + f2fs_down_read(sem); + trace_lock_elapsed_time_start(sem, lc); +} + +int f2fs_down_read_trylock_trace(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc) +{ + uplift_priority(sem, lc, false); + if (!f2fs_down_read_trylock(sem)) { + restore_priority(sem, lc, false); + return 0; + } + trace_lock_elapsed_time_start(sem, lc); + return 1; +} + +void f2fs_up_read_trace(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc) +{ + f2fs_up_read(sem); + restore_priority(sem, lc, false); + trace_lock_elapsed_time_end(sem, lc, false); +} + +void f2fs_down_write_trace(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc) +{ + uplift_priority(sem, lc, true); + f2fs_down_write(sem); + trace_lock_elapsed_time_start(sem, lc); +} + +int f2fs_down_write_trylock_trace(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc) +{ + uplift_priority(sem, lc, true); + if (!f2fs_down_write_trylock(sem)) { + restore_priority(sem, lc, true); + return 0; + } + trace_lock_elapsed_time_start(sem, lc); + return 1; +} + +void f2fs_up_write_trace(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc) +{ + f2fs_up_write(sem); + restore_priority(sem, lc, true); + trace_lock_elapsed_time_end(sem, lc, true); +} + +void f2fs_lock_op(struct f2fs_sb_info *sbi, struct f2fs_lock_context *lc) +{ + f2fs_down_read_trace(&sbi->cp_rwsem, lc); +} + +int f2fs_trylock_op(struct f2fs_sb_info *sbi, struct f2fs_lock_context *lc) +{ + if (time_to_inject(sbi, FAULT_LOCK_OP)) + return 0; + + return f2fs_down_read_trylock_trace(&sbi->cp_rwsem, lc); +} + +void f2fs_unlock_op(struct f2fs_sb_info *sbi, struct f2fs_lock_context *lc) +{ + f2fs_up_read_trace(&sbi->cp_rwsem, lc); +} + +static inline void f2fs_lock_all(struct f2fs_sb_info *sbi) +{ + f2fs_down_write(&sbi->cp_rwsem); +} + +static inline void f2fs_unlock_all(struct f2fs_sb_info *sbi) +{ + f2fs_up_write(&sbi->cp_rwsem); +} + +#define DEFAULT_CHECKPOINT_IOPRIO (IOPRIO_PRIO_VALUE(IOPRIO_CLASS_RT, 3)) + +static struct kmem_cache *ino_entry_slab; +struct kmem_cache *f2fs_inode_entry_slab; + +/* + * We guarantee no failure on the returned page. + */ +struct folio *f2fs_grab_meta_folio(struct f2fs_sb_info *sbi, pgoff_t index) +{ + struct address_space *mapping = META_MAPPING(sbi); + struct folio *folio; +repeat: + folio = f2fs_grab_cache_folio(mapping, index, false); + if (IS_ERR(folio)) { + cond_resched(); + goto repeat; + } + f2fs_folio_wait_writeback(folio, META, true, true); + if (!folio_test_uptodate(folio)) + folio_mark_uptodate(folio); + return folio; +} + +static struct folio *__get_meta_folio(struct f2fs_sb_info *sbi, pgoff_t index, + bool is_meta) +{ + struct address_space *mapping = META_MAPPING(sbi); + struct folio *folio; + struct f2fs_io_info fio = { + .sbi = sbi, + .type = META, + .op = REQ_OP_READ, + .op_flags = REQ_META | REQ_PRIO, + .old_blkaddr = index, + .new_blkaddr = index, + .encrypted_page = NULL, + .is_por = !is_meta ? 1 : 0, + }; + int err; + + if (unlikely(!is_meta)) + fio.op_flags &= ~REQ_META; +repeat: + folio = f2fs_grab_cache_folio(mapping, index, false); + if (IS_ERR(folio)) { + cond_resched(); + goto repeat; + } + if (folio_test_uptodate(folio)) + goto out; + + fio.folio = folio; + + err = f2fs_submit_page_bio(&fio); + if (err) { + f2fs_folio_put(folio, true); + return ERR_PTR(err); + } + + f2fs_update_iostat(sbi, NULL, FS_META_READ_IO, F2FS_BLKSIZE); + + folio_lock(folio); + if (unlikely(!is_meta_folio(folio))) { + f2fs_folio_put(folio, true); + goto repeat; + } + + if (unlikely(!folio_test_uptodate(folio))) { + f2fs_handle_page_eio(sbi, folio, META); + f2fs_folio_put(folio, true); + return ERR_PTR(-EIO); + } +out: + return folio; +} + +struct folio *f2fs_get_meta_folio(struct f2fs_sb_info *sbi, pgoff_t index) +{ + return __get_meta_folio(sbi, index, true); +} + +struct folio *f2fs_get_meta_folio_retry(struct f2fs_sb_info *sbi, pgoff_t index) +{ + struct folio *folio; + int count = 0; + +retry: + folio = __get_meta_folio(sbi, index, true); + if (IS_ERR(folio)) { + if (PTR_ERR(folio) == -EIO && + ++count <= DEFAULT_RETRY_IO_COUNT) + goto retry; + f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_META_PAGE); + } + return folio; +} + +/* for POR only */ +struct folio *f2fs_get_tmp_folio(struct f2fs_sb_info *sbi, pgoff_t index) +{ + return __get_meta_folio(sbi, index, false); +} + +static bool __is_bitmap_valid(struct f2fs_sb_info *sbi, block_t blkaddr, + int type) +{ + struct seg_entry *se; + unsigned int segno, offset; + bool exist; + + if (type == DATA_GENERIC) + return true; + + segno = GET_SEGNO(sbi, blkaddr); + offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); + se = get_seg_entry(sbi, segno); + + exist = f2fs_test_bit(offset, se->cur_valid_map); + + /* skip data, if we already have an error in checkpoint. */ + if (unlikely(f2fs_cp_error(sbi))) + return exist; + + if ((exist && type == DATA_GENERIC_ENHANCE_UPDATE) || + (!exist && type == DATA_GENERIC_ENHANCE)) + goto out_err; + if (!exist && type != DATA_GENERIC_ENHANCE_UPDATE) + goto out_handle; + return exist; + +out_err: + f2fs_err(sbi, "Inconsistent error blkaddr:%u, sit bitmap:%d", + blkaddr, exist); + set_sbi_flag(sbi, SBI_NEED_FSCK); + dump_stack(); +out_handle: + f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR); + return exist; +} + +static bool __f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi, + block_t blkaddr, int type) +{ + switch (type) { + case META_NAT: + break; + case META_SIT: + if (unlikely(blkaddr >= SIT_BLK_CNT(sbi))) + goto check_only; + break; + case META_SSA: + if (unlikely(blkaddr >= MAIN_BLKADDR(sbi) || + blkaddr < SM_I(sbi)->ssa_blkaddr)) + goto check_only; + break; + case META_CP: + if (unlikely(blkaddr >= SIT_I(sbi)->sit_base_addr || + blkaddr < __start_cp_addr(sbi))) + goto check_only; + break; + case META_POR: + if (unlikely(blkaddr >= MAX_BLKADDR(sbi) || + blkaddr < MAIN_BLKADDR(sbi))) + goto check_only; + break; + case DATA_GENERIC: + case DATA_GENERIC_ENHANCE: + case DATA_GENERIC_ENHANCE_READ: + case DATA_GENERIC_ENHANCE_UPDATE: + if (unlikely(blkaddr >= MAX_BLKADDR(sbi) || + blkaddr < MAIN_BLKADDR(sbi))) { + + /* Skip to emit an error message. */ + if (unlikely(f2fs_cp_error(sbi))) + return false; + + f2fs_warn(sbi, "access invalid blkaddr:%u", + blkaddr); + set_sbi_flag(sbi, SBI_NEED_FSCK); + dump_stack(); + goto err; + } else { + return __is_bitmap_valid(sbi, blkaddr, type); + } + break; + case META_GENERIC: + if (unlikely(blkaddr < SEG0_BLKADDR(sbi) || + blkaddr >= MAIN_BLKADDR(sbi))) + goto err; + break; + default: + BUG(); + } + + return true; +err: + f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR); +check_only: + return false; +} + +bool f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi, + block_t blkaddr, int type) +{ + if (time_to_inject(sbi, FAULT_BLKADDR_VALIDITY)) + return false; + return __f2fs_is_valid_blkaddr(sbi, blkaddr, type); +} + +bool f2fs_is_valid_blkaddr_raw(struct f2fs_sb_info *sbi, + block_t blkaddr, int type) +{ + return __f2fs_is_valid_blkaddr(sbi, blkaddr, type); +} + +/* + * Readahead CP/NAT/SIT/SSA/POR pages + */ +int f2fs_ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages, + int type, bool sync) +{ + block_t blkno = start; + struct f2fs_io_info fio = { + .sbi = sbi, + .type = META, + .op = REQ_OP_READ, + .op_flags = sync ? (REQ_META | REQ_PRIO) : REQ_RAHEAD, + .encrypted_page = NULL, + .in_list = 0, + .is_por = (type == META_POR) ? 1 : 0, + }; + struct blk_plug plug; + int err; + + if (unlikely(type == META_POR)) + fio.op_flags &= ~REQ_META; + + blk_start_plug(&plug); + for (; nrpages-- > 0; blkno++) { + struct folio *folio; + + if (!f2fs_is_valid_blkaddr(sbi, blkno, type)) + goto out; + + switch (type) { + case META_NAT: + if (unlikely(blkno >= + NAT_BLOCK_OFFSET(NM_I(sbi)->max_nid))) + blkno = 0; + /* get nat block addr */ + fio.new_blkaddr = current_nat_addr(sbi, + blkno * NAT_ENTRY_PER_BLOCK); + break; + case META_SIT: + if (unlikely(blkno >= TOTAL_SEGS(sbi))) + goto out; + /* get sit block addr */ + fio.new_blkaddr = current_sit_addr(sbi, + blkno * SIT_ENTRY_PER_BLOCK); + break; + case META_SSA: + case META_CP: + case META_POR: + fio.new_blkaddr = blkno; + break; + default: + BUG(); + } + + folio = f2fs_grab_cache_folio(META_MAPPING(sbi), + fio.new_blkaddr, false); + if (IS_ERR(folio)) + continue; + if (folio_test_uptodate(folio)) { + f2fs_folio_put(folio, true); + continue; + } + + fio.folio = folio; + err = f2fs_submit_page_bio(&fio); + f2fs_folio_put(folio, err ? true : false); + + if (!err) + f2fs_update_iostat(sbi, NULL, FS_META_READ_IO, + F2FS_BLKSIZE); + } +out: + blk_finish_plug(&plug); + return blkno - start; +} + +void f2fs_ra_meta_pages_cond(struct f2fs_sb_info *sbi, pgoff_t index, + unsigned int ra_blocks) +{ + struct folio *folio; + bool readahead = false; + + if (ra_blocks == RECOVERY_MIN_RA_BLOCKS) + return; + + folio = filemap_get_folio(META_MAPPING(sbi), index); + if (IS_ERR(folio) || !folio_test_uptodate(folio)) + readahead = true; + f2fs_folio_put(folio, false); + + if (readahead) + f2fs_ra_meta_pages(sbi, index, ra_blocks, META_POR, true); +} + +static bool __f2fs_write_meta_folio(struct folio *folio, + struct writeback_control *wbc, + enum iostat_type io_type) +{ + struct f2fs_sb_info *sbi = F2FS_F_SB(folio); + + trace_f2fs_writepage(folio, META); + + if (unlikely(f2fs_cp_error(sbi))) { + if (is_sbi_flag_set(sbi, SBI_IS_CLOSE)) { + folio_clear_uptodate(folio); + dec_page_count(sbi, F2FS_DIRTY_META); + folio_unlock(folio); + return true; + } + goto redirty_out; + } + if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) + goto redirty_out; + + f2fs_do_write_meta_page(sbi, folio, io_type); + dec_page_count(sbi, F2FS_DIRTY_META); + + folio_unlock(folio); + + if (unlikely(f2fs_cp_error(sbi))) + f2fs_submit_merged_write(sbi, META); + + return true; + +redirty_out: + folio_redirty_for_writepage(wbc, folio); + return false; +} + +static int f2fs_write_meta_pages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct f2fs_sb_info *sbi = F2FS_M_SB(mapping); + struct f2fs_lock_context lc; + long diff, written; + + if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) + goto skip_write; + + /* collect a number of dirty meta pages and write together */ + if (wbc->sync_mode != WB_SYNC_ALL && + get_pages(sbi, F2FS_DIRTY_META) < + nr_pages_to_skip(sbi, META)) + goto skip_write; + + /* if locked failed, cp will flush dirty pages instead */ + if (!f2fs_down_write_trylock_trace(&sbi->cp_global_sem, &lc)) + goto skip_write; + + trace_f2fs_writepages(mapping->host, wbc, META); + diff = nr_pages_to_write(sbi, META, wbc); + written = f2fs_sync_meta_pages(sbi, wbc->nr_to_write, FS_META_IO); + f2fs_up_write_trace(&sbi->cp_global_sem, &lc); + wbc->nr_to_write = max((long)0, wbc->nr_to_write - written - diff); + return 0; + +skip_write: + wbc->pages_skipped += get_pages(sbi, F2FS_DIRTY_META); + trace_f2fs_writepages(mapping->host, wbc, META); + return 0; +} + +long f2fs_sync_meta_pages(struct f2fs_sb_info *sbi, long nr_to_write, + enum iostat_type io_type) +{ + struct address_space *mapping = META_MAPPING(sbi); + pgoff_t index = 0, prev = ULONG_MAX; + struct folio_batch fbatch; + long nwritten = 0; + int nr_folios; + struct writeback_control wbc = {}; + struct blk_plug plug; + + folio_batch_init(&fbatch); + + blk_start_plug(&plug); + + while ((nr_folios = filemap_get_folios_tag(mapping, &index, + (pgoff_t)-1, + PAGECACHE_TAG_DIRTY, &fbatch))) { + int i; + + for (i = 0; i < nr_folios; i++) { + struct folio *folio = fbatch.folios[i]; + + if (nr_to_write != LONG_MAX && i != 0 && + folio->index != prev + + folio_nr_pages(fbatch.folios[i-1])) { + folio_batch_release(&fbatch); + goto stop; + } + + folio_lock(folio); + + if (unlikely(!is_meta_folio(folio))) { +continue_unlock: + folio_unlock(folio); + continue; + } + if (!folio_test_dirty(folio)) { + /* someone wrote it for us */ + goto continue_unlock; + } + + f2fs_folio_wait_writeback(folio, META, true, true); + + if (!folio_clear_dirty_for_io(folio)) + goto continue_unlock; + + if (!__f2fs_write_meta_folio(folio, &wbc, + io_type)) { + folio_unlock(folio); + break; + } + nwritten += folio_nr_pages(folio); + prev = folio->index; + if (unlikely(nwritten >= nr_to_write)) + break; + } + folio_batch_release(&fbatch); + cond_resched(); + } +stop: + if (nwritten) + f2fs_submit_merged_write(sbi, META); + + blk_finish_plug(&plug); + + return nwritten; +} + +static bool f2fs_dirty_meta_folio(struct address_space *mapping, + struct folio *folio) +{ + trace_f2fs_set_page_dirty(folio, META); + + if (!folio_test_uptodate(folio)) + folio_mark_uptodate(folio); + if (filemap_dirty_folio(mapping, folio)) { + inc_page_count(F2FS_M_SB(mapping), F2FS_DIRTY_META); + folio_set_f2fs_reference(folio); + return true; + } + return false; +} + +const struct address_space_operations f2fs_meta_aops = { + .writepages = f2fs_write_meta_pages, + .dirty_folio = f2fs_dirty_meta_folio, + .invalidate_folio = f2fs_invalidate_folio, + .release_folio = f2fs_release_folio, + .migrate_folio = filemap_migrate_folio, +}; + +static void __add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, + unsigned int devidx, int type) +{ + struct inode_management *im = &sbi->im[type]; + struct ino_entry *e = NULL, *new = NULL; + int ret; + + if (type == FLUSH_INO) { + rcu_read_lock(); + e = radix_tree_lookup(&im->ino_root, ino); + rcu_read_unlock(); + } + +retry: + if (!e) + new = f2fs_kmem_cache_alloc(ino_entry_slab, + GFP_NOFS, true, NULL); + + ret = radix_tree_preload(GFP_NOFS | __GFP_NOFAIL); + f2fs_bug_on(sbi, ret); + + spin_lock(&im->ino_lock); + e = radix_tree_lookup(&im->ino_root, ino); + if (!e) { + if (!new) { + spin_unlock(&im->ino_lock); + radix_tree_preload_end(); + goto retry; + } + e = new; + if (unlikely(radix_tree_insert(&im->ino_root, ino, e))) + f2fs_bug_on(sbi, 1); + + memset(e, 0, sizeof(struct ino_entry)); + e->ino = ino; + + list_add_tail(&e->list, &im->ino_list); + if (type != ORPHAN_INO) + im->ino_num++; + } + + if (type == FLUSH_INO) + f2fs_set_bit(devidx, (char *)&e->dirty_device); + + spin_unlock(&im->ino_lock); + radix_tree_preload_end(); + + if (new && e != new) + kmem_cache_free(ino_entry_slab, new); +} + +static void __remove_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type) +{ + struct inode_management *im = &sbi->im[type]; + struct ino_entry *e; + + spin_lock(&im->ino_lock); + e = radix_tree_lookup(&im->ino_root, ino); + if (e) { + list_del(&e->list); + radix_tree_delete(&im->ino_root, ino); + im->ino_num--; + spin_unlock(&im->ino_lock); + kmem_cache_free(ino_entry_slab, e); + return; + } + spin_unlock(&im->ino_lock); +} + +void f2fs_add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type) +{ + /* add new dirty ino entry into list */ + __add_ino_entry(sbi, ino, 0, type); +} + +void f2fs_remove_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type) +{ + /* remove dirty ino entry from list */ + __remove_ino_entry(sbi, ino, type); +} + +/* mode should be APPEND_INO, UPDATE_INO or TRANS_DIR_INO */ +bool f2fs_exist_written_data(struct f2fs_sb_info *sbi, nid_t ino, int mode) +{ + struct inode_management *im = &sbi->im[mode]; + struct ino_entry *e; + + spin_lock(&im->ino_lock); + e = radix_tree_lookup(&im->ino_root, ino); + spin_unlock(&im->ino_lock); + return e ? true : false; +} + +void f2fs_release_ino_entry(struct f2fs_sb_info *sbi, bool all) +{ + struct ino_entry *e, *tmp; + int i; + + for (i = all ? ORPHAN_INO : APPEND_INO; i < MAX_INO_ENTRY; i++) { + struct inode_management *im = &sbi->im[i]; + + spin_lock(&im->ino_lock); + list_for_each_entry_safe(e, tmp, &im->ino_list, list) { + list_del(&e->list); + radix_tree_delete(&im->ino_root, e->ino); + kmem_cache_free(ino_entry_slab, e); + im->ino_num--; + } + spin_unlock(&im->ino_lock); + } +} + +void f2fs_set_dirty_device(struct f2fs_sb_info *sbi, nid_t ino, + unsigned int devidx, int type) +{ + __add_ino_entry(sbi, ino, devidx, type); +} + +bool f2fs_is_dirty_device(struct f2fs_sb_info *sbi, nid_t ino, + unsigned int devidx, int type) +{ + struct inode_management *im = &sbi->im[type]; + struct ino_entry *e; + bool is_dirty = false; + + spin_lock(&im->ino_lock); + e = radix_tree_lookup(&im->ino_root, ino); + if (e && f2fs_test_bit(devidx, (char *)&e->dirty_device)) + is_dirty = true; + spin_unlock(&im->ino_lock); + return is_dirty; +} + +int f2fs_acquire_orphan_inode(struct f2fs_sb_info *sbi) +{ + struct inode_management *im = &sbi->im[ORPHAN_INO]; + int err = 0; + + spin_lock(&im->ino_lock); + + if (time_to_inject(sbi, FAULT_ORPHAN)) { + spin_unlock(&im->ino_lock); + return -ENOSPC; + } + + if (unlikely(im->ino_num >= sbi->max_orphans)) + err = -ENOSPC; + else + im->ino_num++; + spin_unlock(&im->ino_lock); + + return err; +} + +void f2fs_release_orphan_inode(struct f2fs_sb_info *sbi) +{ + struct inode_management *im = &sbi->im[ORPHAN_INO]; + + spin_lock(&im->ino_lock); + f2fs_bug_on(sbi, im->ino_num == 0); + im->ino_num--; + spin_unlock(&im->ino_lock); +} + +void f2fs_add_orphan_inode(struct inode *inode) +{ + /* add new orphan ino entry into list */ + __add_ino_entry(F2FS_I_SB(inode), inode->i_ino, 0, ORPHAN_INO); + f2fs_update_inode_page(inode); +} + +void f2fs_remove_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino) +{ + /* remove orphan entry from orphan list */ + __remove_ino_entry(sbi, ino, ORPHAN_INO); +} + +static int recover_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino) +{ + struct inode *inode; + struct node_info ni; + int err; + + inode = f2fs_iget_retry(sbi->sb, ino); + if (IS_ERR(inode)) { + /* + * there should be a bug that we can't find the entry + * to orphan inode. + */ + f2fs_bug_on(sbi, PTR_ERR(inode) == -ENOENT); + return PTR_ERR(inode); + } + + err = f2fs_dquot_initialize(inode); + if (err) { + iput(inode); + goto err_out; + } + + clear_nlink(inode); + + /* truncate all the data during iput */ + iput(inode); + + err = f2fs_get_node_info(sbi, ino, &ni, false); + if (err) + goto err_out; + + /* ENOMEM was fully retried in f2fs_evict_inode. */ + if (ni.blk_addr != NULL_ADDR) { + err = -EIO; + goto err_out; + } + return 0; + +err_out: + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_warn(sbi, "%s: orphan failed (ino=%x), run fsck to fix.", + __func__, ino); + return err; +} + +int f2fs_recover_orphan_inodes(struct f2fs_sb_info *sbi) +{ + block_t start_blk, orphan_blocks, i, j; + int err = 0; + + if (!is_set_ckpt_flags(sbi, CP_ORPHAN_PRESENT_FLAG)) + return 0; + + if (f2fs_hw_is_readonly(sbi)) { + f2fs_info(sbi, "write access unavailable, skipping orphan cleanup"); + return 0; + } + + if (is_sbi_flag_set(sbi, SBI_IS_WRITABLE)) + f2fs_info(sbi, "orphan cleanup on readonly fs"); + + start_blk = __start_cp_addr(sbi) + 1 + __cp_payload(sbi); + orphan_blocks = __start_sum_addr(sbi) - 1 - __cp_payload(sbi); + + f2fs_ra_meta_pages(sbi, start_blk, orphan_blocks, META_CP, true); + + for (i = 0; i < orphan_blocks; i++) { + struct folio *folio; + struct f2fs_orphan_block *orphan_blk; + unsigned int entry_count; + + folio = f2fs_get_meta_folio(sbi, start_blk + i); + if (IS_ERR(folio)) { + err = PTR_ERR(folio); + goto out; + } + + orphan_blk = folio_address(folio); + entry_count = le32_to_cpu(orphan_blk->entry_count); + if (entry_count > F2FS_ORPHANS_PER_BLOCK) { + f2fs_err(sbi, "invalid orphan inode entry count %u", + entry_count); + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_handle_error(sbi, ERROR_INCONSISTENT_ORPHAN); + err = -EFSCORRUPTED; + f2fs_folio_put(folio, true); + goto out; + } + + for (j = 0; j < entry_count; j++) { + nid_t ino = le32_to_cpu(orphan_blk->ino[j]); + + err = recover_orphan_inode(sbi, ino); + if (err) { + f2fs_folio_put(folio, true); + goto out; + } + } + f2fs_folio_put(folio, true); + } + /* clear Orphan Flag */ + clear_ckpt_flags(sbi, CP_ORPHAN_PRESENT_FLAG); +out: + set_sbi_flag(sbi, SBI_IS_RECOVERED); + + return err; +} + +static void write_orphan_inodes(struct f2fs_sb_info *sbi, block_t start_blk) +{ + struct list_head *head; + struct f2fs_orphan_block *orphan_blk = NULL; + unsigned int nentries = 0; + unsigned short index = 1; + unsigned short orphan_blocks; + struct folio *folio = NULL; + struct ino_entry *orphan = NULL; + struct inode_management *im = &sbi->im[ORPHAN_INO]; + + orphan_blocks = GET_ORPHAN_BLOCKS(im->ino_num); + + /* + * we don't need to do spin_lock(&im->ino_lock) here, since all the + * orphan inode operations are covered under f2fs_lock_op(). + * And, spin_lock should be avoided due to page operations below. + */ + head = &im->ino_list; + + /* loop for each orphan inode entry and write them in journal block */ + list_for_each_entry(orphan, head, list) { + if (!folio) { + folio = f2fs_grab_meta_folio(sbi, start_blk++); + orphan_blk = folio_address(folio); + memset(orphan_blk, 0, sizeof(*orphan_blk)); + } + + orphan_blk->ino[nentries++] = cpu_to_le32(orphan->ino); + + if (nentries == F2FS_ORPHANS_PER_BLOCK) { + /* + * an orphan block is full of 1020 entries, + * then we need to flush current orphan blocks + * and bring another one in memory + */ + orphan_blk->blk_addr = cpu_to_le16(index); + orphan_blk->blk_count = cpu_to_le16(orphan_blocks); + orphan_blk->entry_count = cpu_to_le32(nentries); + folio_mark_dirty(folio); + f2fs_folio_put(folio, true); + index++; + nentries = 0; + folio = NULL; + } + } + + if (folio) { + orphan_blk->blk_addr = cpu_to_le16(index); + orphan_blk->blk_count = cpu_to_le16(orphan_blocks); + orphan_blk->entry_count = cpu_to_le32(nentries); + folio_mark_dirty(folio); + f2fs_folio_put(folio, true); + } +} + +static __u32 f2fs_checkpoint_chksum(struct f2fs_checkpoint *ckpt) +{ + unsigned int chksum_ofs = le32_to_cpu(ckpt->checksum_offset); + __u32 chksum; + + chksum = f2fs_crc32(ckpt, chksum_ofs); + if (chksum_ofs < CP_CHKSUM_OFFSET) { + chksum_ofs += sizeof(chksum); + chksum = f2fs_chksum(chksum, (__u8 *)ckpt + chksum_ofs, + F2FS_BLKSIZE - chksum_ofs); + } + return chksum; +} + +static int get_checkpoint_version(struct f2fs_sb_info *sbi, block_t cp_addr, + struct f2fs_checkpoint **cp_block, struct folio **cp_folio, + unsigned long long *version) +{ + size_t crc_offset = 0; + __u32 crc; + + *cp_folio = f2fs_get_meta_folio(sbi, cp_addr); + if (IS_ERR(*cp_folio)) + return PTR_ERR(*cp_folio); + + *cp_block = folio_address(*cp_folio); + + crc_offset = le32_to_cpu((*cp_block)->checksum_offset); + if (crc_offset < CP_MIN_CHKSUM_OFFSET || + crc_offset > CP_CHKSUM_OFFSET) { + f2fs_folio_put(*cp_folio, true); + f2fs_warn(sbi, "invalid crc_offset: %zu", crc_offset); + return -EINVAL; + } + + crc = f2fs_checkpoint_chksum(*cp_block); + if (crc != cur_cp_crc(*cp_block)) { + f2fs_folio_put(*cp_folio, true); + f2fs_warn(sbi, "invalid crc value"); + return -EINVAL; + } + + *version = cur_cp_version(*cp_block); + return 0; +} + +static struct folio *validate_checkpoint(struct f2fs_sb_info *sbi, + block_t cp_addr, unsigned long long *version) +{ + struct folio *cp_folio_1 = NULL, *cp_folio_2 = NULL; + struct f2fs_checkpoint *cp_block = NULL; + unsigned long long cur_version = 0, pre_version = 0; + unsigned int cp_blocks; + int err; + + err = get_checkpoint_version(sbi, cp_addr, &cp_block, + &cp_folio_1, version); + if (err) + return NULL; + + cp_blocks = le32_to_cpu(cp_block->cp_pack_total_block_count); + + if (cp_blocks > BLKS_PER_SEG(sbi) || cp_blocks <= F2FS_CP_PACKS) { + f2fs_warn(sbi, "invalid cp_pack_total_block_count:%u", + le32_to_cpu(cp_block->cp_pack_total_block_count)); + goto invalid_cp; + } + pre_version = *version; + + cp_addr += cp_blocks - 1; + err = get_checkpoint_version(sbi, cp_addr, &cp_block, + &cp_folio_2, version); + if (err) + goto invalid_cp; + cur_version = *version; + + if (cur_version == pre_version) { + *version = cur_version; + f2fs_folio_put(cp_folio_2, true); + return cp_folio_1; + } + f2fs_folio_put(cp_folio_2, true); +invalid_cp: + f2fs_folio_put(cp_folio_1, true); + return NULL; +} + +int f2fs_get_valid_checkpoint(struct f2fs_sb_info *sbi) +{ + struct f2fs_checkpoint *cp_block; + struct f2fs_super_block *fsb = sbi->raw_super; + struct folio *cp1, *cp2, *cur_folio; + unsigned long blk_size = sbi->blocksize; + unsigned long long cp1_version = 0, cp2_version = 0; + unsigned long long cp_start_blk_no; + unsigned int cp_blks = 1 + __cp_payload(sbi); + block_t cp_blk_no; + int i; + int err; + + sbi->ckpt = f2fs_kvzalloc(sbi, array_size(blk_size, cp_blks), + GFP_KERNEL); + if (!sbi->ckpt) + return -ENOMEM; + /* + * Finding out valid cp block involves read both + * sets( cp pack 1 and cp pack 2) + */ + cp_start_blk_no = le32_to_cpu(fsb->cp_blkaddr); + cp1 = validate_checkpoint(sbi, cp_start_blk_no, &cp1_version); + + /* The second checkpoint pack should start at the next segment */ + cp_start_blk_no += ((unsigned long long)1) << + le32_to_cpu(fsb->log_blocks_per_seg); + cp2 = validate_checkpoint(sbi, cp_start_blk_no, &cp2_version); + + if (cp1 && cp2) { + if (ver_after(cp2_version, cp1_version)) + cur_folio = cp2; + else + cur_folio = cp1; + } else if (cp1) { + cur_folio = cp1; + } else if (cp2) { + cur_folio = cp2; + } else { + err = -EFSCORRUPTED; + goto fail_no_cp; + } + + cp_block = folio_address(cur_folio); + memcpy(sbi->ckpt, cp_block, blk_size); + + if (cur_folio == cp1) + sbi->cur_cp_pack = 1; + else + sbi->cur_cp_pack = 2; + + /* Sanity checking of checkpoint */ + if (f2fs_sanity_check_ckpt(sbi)) { + err = -EFSCORRUPTED; + goto free_fail_no_cp; + } + + if (cp_blks <= 1) + goto done; + + cp_blk_no = le32_to_cpu(fsb->cp_blkaddr); + if (cur_folio == cp2) + cp_blk_no += BIT(le32_to_cpu(fsb->log_blocks_per_seg)); + + for (i = 1; i < cp_blks; i++) { + void *sit_bitmap_ptr; + unsigned char *ckpt = (unsigned char *)sbi->ckpt; + + cur_folio = f2fs_get_meta_folio(sbi, cp_blk_no + i); + if (IS_ERR(cur_folio)) { + err = PTR_ERR(cur_folio); + goto free_fail_no_cp; + } + sit_bitmap_ptr = folio_address(cur_folio); + memcpy(ckpt + i * blk_size, sit_bitmap_ptr, blk_size); + f2fs_folio_put(cur_folio, true); + } +done: + f2fs_folio_put(cp1, true); + f2fs_folio_put(cp2, true); + return 0; + +free_fail_no_cp: + f2fs_folio_put(cp1, true); + f2fs_folio_put(cp2, true); +fail_no_cp: + kvfree(sbi->ckpt); + return err; +} + +static void __add_dirty_inode(struct inode *inode, enum inode_type type) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + int flag = (type == DIR_INODE) ? FI_DIRTY_DIR : FI_DIRTY_FILE; + + if (is_inode_flag_set(inode, flag)) + return; + + set_inode_flag(inode, flag); + list_add_tail(&F2FS_I(inode)->dirty_list, &sbi->inode_list[type]); + stat_inc_dirty_inode(sbi, type); +} + +static void __remove_dirty_inode(struct inode *inode, enum inode_type type) +{ + int flag = (type == DIR_INODE) ? FI_DIRTY_DIR : FI_DIRTY_FILE; + + if (get_dirty_pages(inode) || !is_inode_flag_set(inode, flag)) + return; + + list_del_init(&F2FS_I(inode)->dirty_list); + clear_inode_flag(inode, flag); + stat_dec_dirty_inode(F2FS_I_SB(inode), type); +} + +void f2fs_update_dirty_folio(struct inode *inode, struct folio *folio) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + enum inode_type type = S_ISDIR(inode->i_mode) ? DIR_INODE : FILE_INODE; + + if (!S_ISDIR(inode->i_mode) && !S_ISREG(inode->i_mode) && + !S_ISLNK(inode->i_mode)) + return; + + spin_lock(&sbi->inode_lock[type]); + if (type != FILE_INODE || test_opt(sbi, DATA_FLUSH)) + __add_dirty_inode(inode, type); + inode_inc_dirty_pages(inode); + spin_unlock(&sbi->inode_lock[type]); + + folio_set_f2fs_reference(folio); +} + +void f2fs_remove_dirty_inode(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + enum inode_type type = S_ISDIR(inode->i_mode) ? DIR_INODE : FILE_INODE; + + if (!S_ISDIR(inode->i_mode) && !S_ISREG(inode->i_mode) && + !S_ISLNK(inode->i_mode)) + return; + + if (type == FILE_INODE && !test_opt(sbi, DATA_FLUSH)) + return; + + spin_lock(&sbi->inode_lock[type]); + __remove_dirty_inode(inode, type); + spin_unlock(&sbi->inode_lock[type]); +} + +int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type, + bool from_cp) +{ + struct list_head *head; + struct inode *inode; + struct f2fs_inode_info *fi; + bool is_dir = (type == DIR_INODE); + unsigned long ino = 0; + + trace_f2fs_sync_dirty_inodes_enter(sbi->sb, is_dir, + get_pages(sbi, is_dir ? + F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA)); +retry: + if (unlikely(f2fs_cp_error(sbi))) { + trace_f2fs_sync_dirty_inodes_exit(sbi->sb, is_dir, + get_pages(sbi, is_dir ? + F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA)); + return -EIO; + } + + spin_lock(&sbi->inode_lock[type]); + + head = &sbi->inode_list[type]; + if (list_empty(head)) { + spin_unlock(&sbi->inode_lock[type]); + trace_f2fs_sync_dirty_inodes_exit(sbi->sb, is_dir, + get_pages(sbi, is_dir ? + F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA)); + return 0; + } + fi = list_first_entry(head, struct f2fs_inode_info, dirty_list); + inode = igrab(&fi->vfs_inode); + spin_unlock(&sbi->inode_lock[type]); + if (inode) { + unsigned long cur_ino = inode->i_ino; + + if (from_cp) + F2FS_I(inode)->cp_task = current; + F2FS_I(inode)->wb_task = current; + + filemap_fdatawrite(inode->i_mapping); + + F2FS_I(inode)->wb_task = NULL; + if (from_cp) + F2FS_I(inode)->cp_task = NULL; + + iput(inode); + /* We need to give cpu to another writers. */ + if (ino == cur_ino) + cond_resched(); + else + ino = cur_ino; + } else { + /* + * We should submit bio, since it exists several + * writebacking dentry pages in the freeing inode. + */ + f2fs_submit_merged_write(sbi, DATA); + cond_resched(); + } + goto retry; +} + +static int f2fs_sync_inode_meta(struct f2fs_sb_info *sbi) +{ + struct list_head *head = &sbi->inode_list[DIRTY_META]; + struct inode *inode; + struct f2fs_inode_info *fi; + s64 total = get_pages(sbi, F2FS_DIRTY_IMETA); + + while (total--) { + if (unlikely(f2fs_cp_error(sbi))) + return -EIO; + + spin_lock(&sbi->inode_lock[DIRTY_META]); + if (list_empty(head)) { + spin_unlock(&sbi->inode_lock[DIRTY_META]); + return 0; + } + fi = list_first_entry(head, struct f2fs_inode_info, + gdirty_list); + inode = igrab(&fi->vfs_inode); + spin_unlock(&sbi->inode_lock[DIRTY_META]); + if (inode) { + sync_inode_metadata(inode, 0); + + /* it's on eviction */ + if (is_inode_flag_set(inode, FI_DIRTY_INODE)) + f2fs_update_inode_page(inode); + iput(inode); + } + } + return 0; +} + +static void __prepare_cp_block(struct f2fs_sb_info *sbi) +{ + struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); + struct f2fs_nm_info *nm_i = NM_I(sbi); + nid_t last_nid = nm_i->next_scan_nid; + + next_free_nid(sbi, &last_nid); + ckpt->valid_block_count = cpu_to_le64(valid_user_blocks(sbi)); + ckpt->valid_node_count = cpu_to_le32(valid_node_count(sbi)); + ckpt->valid_inode_count = cpu_to_le32(valid_inode_count(sbi)); + ckpt->next_free_nid = cpu_to_le32(last_nid); + + /* update user_block_counts */ + sbi->last_valid_block_count = sbi->total_valid_block_count; + percpu_counter_set(&sbi->alloc_valid_block_count, 0); + percpu_counter_set(&sbi->rf_node_block_count, 0); +} + +static bool __need_flush_quota(struct f2fs_sb_info *sbi) +{ + bool ret = false; + + if (!is_journalled_quota(sbi)) + return false; + + if (!f2fs_down_write_trylock(&sbi->quota_sem)) + return true; + if (is_sbi_flag_set(sbi, SBI_QUOTA_SKIP_FLUSH)) { + ret = false; + } else if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_REPAIR)) { + ret = false; + } else if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_FLUSH)) { + clear_sbi_flag(sbi, SBI_QUOTA_NEED_FLUSH); + ret = true; + } else if (get_pages(sbi, F2FS_DIRTY_QDATA)) { + ret = true; + } + f2fs_up_write(&sbi->quota_sem); + return ret; +} + +/* + * Freeze all the FS-operations for checkpoint. + */ +static int block_operations(struct f2fs_sb_info *sbi) +{ + struct writeback_control wbc = { + .sync_mode = WB_SYNC_ALL, + .nr_to_write = LONG_MAX, + }; + int err = 0, cnt = 0; + + /* + * Let's flush inline_data in dirty node pages. + */ + f2fs_flush_inline_data(sbi); + +retry_flush_quotas: + f2fs_lock_all(sbi); + if (__need_flush_quota(sbi)) { + bool need_lock = sbi->umount_lock_holder != current; + + if (++cnt > DEFAULT_RETRY_QUOTA_FLUSH_COUNT) { + set_sbi_flag(sbi, SBI_QUOTA_SKIP_FLUSH); + set_sbi_flag(sbi, SBI_QUOTA_NEED_FLUSH); + goto retry_flush_dents; + } + f2fs_unlock_all(sbi); + + /* don't grab s_umount lock during mount/umount/remount/freeze/quotactl */ + if (!need_lock) { + f2fs_do_quota_sync(sbi->sb, -1); + } else if (down_read_trylock(&sbi->sb->s_umount)) { + f2fs_do_quota_sync(sbi->sb, -1); + up_read(&sbi->sb->s_umount); + } + cond_resched(); + goto retry_flush_quotas; + } + +retry_flush_dents: + /* write all the dirty dentry pages */ + if (get_pages(sbi, F2FS_DIRTY_DENTS)) { + f2fs_unlock_all(sbi); + err = f2fs_sync_dirty_inodes(sbi, DIR_INODE, true); + if (err) + return err; + cond_resched(); + goto retry_flush_quotas; + } + + /* + * POR: we should ensure that there are no dirty node pages + * until finishing nat/sit flush. inode->i_blocks can be updated. + */ + f2fs_down_write(&sbi->node_change); + + if (get_pages(sbi, F2FS_DIRTY_IMETA)) { + f2fs_up_write(&sbi->node_change); + f2fs_unlock_all(sbi); + err = f2fs_sync_inode_meta(sbi); + if (err) + return err; + cond_resched(); + goto retry_flush_quotas; + } + +retry_flush_nodes: + f2fs_down_write(&sbi->node_write); + + if (get_pages(sbi, F2FS_DIRTY_NODES)) { + f2fs_up_write(&sbi->node_write); + atomic_inc(&sbi->wb_sync_req[NODE]); + err = f2fs_sync_node_pages(sbi, &wbc, false, FS_CP_NODE_IO); + atomic_dec(&sbi->wb_sync_req[NODE]); + if (err) { + f2fs_up_write(&sbi->node_change); + f2fs_unlock_all(sbi); + return err; + } + cond_resched(); + goto retry_flush_nodes; + } + + /* + * sbi->node_change is used only for AIO write_begin path which produces + * dirty node blocks and some checkpoint values by block allocation. + */ + __prepare_cp_block(sbi); + f2fs_up_write(&sbi->node_change); + return err; +} + +static void unblock_operations(struct f2fs_sb_info *sbi) +{ + f2fs_up_write(&sbi->node_write); + f2fs_unlock_all(sbi); +} + +void f2fs_wait_on_all_pages(struct f2fs_sb_info *sbi, int type) +{ + DEFINE_WAIT(wait); + + for (;;) { + if (!get_pages(sbi, type)) + break; + + if (unlikely(f2fs_cp_error(sbi) && + !is_sbi_flag_set(sbi, SBI_IS_CLOSE))) + break; + + if (type == F2FS_DIRTY_META) + f2fs_sync_meta_pages(sbi, LONG_MAX, FS_CP_META_IO); + else if (type == F2FS_WB_CP_DATA) + f2fs_submit_merged_write(sbi, DATA); + + prepare_to_wait(&sbi->cp_wait, &wait, TASK_UNINTERRUPTIBLE); + io_schedule_timeout(DEFAULT_SCHEDULE_TIMEOUT); + } + finish_wait(&sbi->cp_wait, &wait); +} + +static void update_ckpt_flags(struct f2fs_sb_info *sbi, struct cp_control *cpc) +{ + unsigned long orphan_num = sbi->im[ORPHAN_INO].ino_num; + struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); + unsigned long flags; + + spin_lock_irqsave(&sbi->cp_lock, flags); + + if ((cpc->reason & CP_UMOUNT) && + le32_to_cpu(ckpt->cp_pack_total_block_count) > + sbi->blocks_per_seg - NM_I(sbi)->nat_bits_blocks) + disable_nat_bits(sbi, false); + + if (cpc->reason & CP_TRIMMED) + __set_ckpt_flags(ckpt, CP_TRIMMED_FLAG); + else + __clear_ckpt_flags(ckpt, CP_TRIMMED_FLAG); + + if (cpc->reason & CP_UMOUNT) + __set_ckpt_flags(ckpt, CP_UMOUNT_FLAG); + else + __clear_ckpt_flags(ckpt, CP_UMOUNT_FLAG); + + if (cpc->reason & CP_FASTBOOT) + __set_ckpt_flags(ckpt, CP_FASTBOOT_FLAG); + else + __clear_ckpt_flags(ckpt, CP_FASTBOOT_FLAG); + + if (orphan_num) + __set_ckpt_flags(ckpt, CP_ORPHAN_PRESENT_FLAG); + else + __clear_ckpt_flags(ckpt, CP_ORPHAN_PRESENT_FLAG); + + if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) + __set_ckpt_flags(ckpt, CP_FSCK_FLAG); + + if (is_sbi_flag_set(sbi, SBI_IS_RESIZEFS)) + __set_ckpt_flags(ckpt, CP_RESIZEFS_FLAG); + else + __clear_ckpt_flags(ckpt, CP_RESIZEFS_FLAG); + + if (is_sbi_flag_set(sbi, SBI_CP_DISABLED)) + __set_ckpt_flags(ckpt, CP_DISABLED_FLAG); + else + __clear_ckpt_flags(ckpt, CP_DISABLED_FLAG); + + if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK)) + __set_ckpt_flags(ckpt, CP_DISABLED_QUICK_FLAG); + else + __clear_ckpt_flags(ckpt, CP_DISABLED_QUICK_FLAG); + + if (is_sbi_flag_set(sbi, SBI_QUOTA_SKIP_FLUSH)) + __set_ckpt_flags(ckpt, CP_QUOTA_NEED_FSCK_FLAG); + else + __clear_ckpt_flags(ckpt, CP_QUOTA_NEED_FSCK_FLAG); + + if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_REPAIR)) + __set_ckpt_flags(ckpt, CP_QUOTA_NEED_FSCK_FLAG); + + /* set this flag to activate crc|cp_ver for recovery */ + __set_ckpt_flags(ckpt, CP_CRC_RECOVERY_FLAG); + __clear_ckpt_flags(ckpt, CP_NOCRC_RECOVERY_FLAG); + + spin_unlock_irqrestore(&sbi->cp_lock, flags); +} + +static void commit_checkpoint(struct f2fs_sb_info *sbi, + void *src, block_t blk_addr) +{ + struct writeback_control wbc = {}; + + /* + * filemap_get_folios_tag and folio_lock again will take + * some extra time. Therefore, f2fs_update_meta_pages and + * f2fs_sync_meta_pages are combined in this function. + */ + struct folio *folio = f2fs_grab_meta_folio(sbi, blk_addr); + + memcpy(folio_address(folio), src, PAGE_SIZE); + + folio_mark_dirty(folio); + if (unlikely(!folio_clear_dirty_for_io(folio))) + f2fs_bug_on(sbi, 1); + + /* writeout cp pack 2 page */ + if (unlikely(!__f2fs_write_meta_folio(folio, &wbc, FS_CP_META_IO))) { + if (f2fs_cp_error(sbi)) { + f2fs_folio_put(folio, true); + return; + } + f2fs_bug_on(sbi, true); + } + + f2fs_folio_put(folio, false); + + /* submit checkpoint (with barrier if NOBARRIER is not set) */ + f2fs_submit_merged_write(sbi, META_FLUSH); +} + +static inline u64 get_sectors_written(struct block_device *bdev) +{ + return (u64)part_stat_read(bdev, sectors[STAT_WRITE]); +} + +u64 f2fs_get_sectors_written(struct f2fs_sb_info *sbi) +{ + if (f2fs_is_multi_device(sbi)) { + u64 sectors = 0; + int i; + + for (i = 0; i < sbi->s_ndevs; i++) + sectors += get_sectors_written(FDEV(i).bdev); + + return sectors; + } + + return get_sectors_written(sbi->sb->s_bdev); +} + +static inline void stat_cp_time(struct cp_control *cpc, enum cp_time type) +{ + cpc->stats.times[type] = ktime_get(); +} + +static inline void check_cp_time(struct f2fs_sb_info *sbi, struct cp_control *cpc) +{ + unsigned long long sb_diff, cur_diff; + enum cp_time ct; + + sb_diff = (u64)ktime_ms_delta(sbi->cp_stats.times[CP_TIME_END], + sbi->cp_stats.times[CP_TIME_START]); + cur_diff = (u64)ktime_ms_delta(cpc->stats.times[CP_TIME_END], + cpc->stats.times[CP_TIME_START]); + + if (cur_diff > sb_diff) { + sbi->cp_stats = cpc->stats; + if (cur_diff < CP_LONG_LATENCY_THRESHOLD) + return; + + f2fs_warn(sbi, "checkpoint was blocked for %llu ms", cur_diff); + for (ct = CP_TIME_START; ct < CP_TIME_MAX - 1; ct++) + f2fs_warn(sbi, "Step#%d: %llu ms", ct, + (u64)ktime_ms_delta(cpc->stats.times[ct + 1], + cpc->stats.times[ct])); + } +} + +static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc) +{ + struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); + struct f2fs_nm_info *nm_i = NM_I(sbi); + unsigned long orphan_num = sbi->im[ORPHAN_INO].ino_num, flags; + block_t start_blk; + unsigned int data_sum_blocks, orphan_blocks; + __u32 crc32 = 0; + int i; + int cp_payload_blks = __cp_payload(sbi); + struct curseg_info *seg_i = CURSEG_I(sbi, CURSEG_HOT_NODE); + u64 kbytes_written; + int err; + + /* Flush all the NAT/SIT pages */ + f2fs_sync_meta_pages(sbi, LONG_MAX, FS_CP_META_IO); + + stat_cp_time(cpc, CP_TIME_SYNC_META); + + /* start to update checkpoint, cp ver is already updated previously */ + ckpt->elapsed_time = cpu_to_le64(get_mtime(sbi, true)); + ckpt->free_segment_count = cpu_to_le32(free_segments(sbi)); + for (i = 0; i < NR_CURSEG_NODE_TYPE; i++) { + struct curseg_info *curseg = CURSEG_I(sbi, i + CURSEG_HOT_NODE); + + ckpt->cur_node_segno[i] = cpu_to_le32(curseg->segno); + ckpt->cur_node_blkoff[i] = cpu_to_le16(curseg->next_blkoff); + ckpt->alloc_type[i + CURSEG_HOT_NODE] = curseg->alloc_type; + } + for (i = 0; i < NR_CURSEG_DATA_TYPE; i++) { + struct curseg_info *curseg = CURSEG_I(sbi, i + CURSEG_HOT_DATA); + + ckpt->cur_data_segno[i] = cpu_to_le32(curseg->segno); + ckpt->cur_data_blkoff[i] = cpu_to_le16(curseg->next_blkoff); + ckpt->alloc_type[i + CURSEG_HOT_DATA] = curseg->alloc_type; + } + + /* 2 cp + n data seg summary + orphan inode blocks */ + data_sum_blocks = f2fs_npages_for_summary_flush(sbi, false); + spin_lock_irqsave(&sbi->cp_lock, flags); + if (data_sum_blocks < NR_CURSEG_DATA_TYPE) + __set_ckpt_flags(ckpt, CP_COMPACT_SUM_FLAG); + else + __clear_ckpt_flags(ckpt, CP_COMPACT_SUM_FLAG); + spin_unlock_irqrestore(&sbi->cp_lock, flags); + + orphan_blocks = GET_ORPHAN_BLOCKS(orphan_num); + ckpt->cp_pack_start_sum = cpu_to_le32(1 + cp_payload_blks + + orphan_blocks); + + if (__remain_node_summaries(cpc->reason)) + ckpt->cp_pack_total_block_count = cpu_to_le32(F2FS_CP_PACKS + + cp_payload_blks + data_sum_blocks + + orphan_blocks + NR_CURSEG_NODE_TYPE); + else + ckpt->cp_pack_total_block_count = cpu_to_le32(F2FS_CP_PACKS + + cp_payload_blks + data_sum_blocks + + orphan_blocks); + + /* update ckpt flag for checkpoint */ + update_ckpt_flags(sbi, cpc); + + /* update SIT/NAT bitmap */ + get_sit_bitmap(sbi, __bitmap_ptr(sbi, SIT_BITMAP)); + get_nat_bitmap(sbi, __bitmap_ptr(sbi, NAT_BITMAP)); + + crc32 = f2fs_checkpoint_chksum(ckpt); + *((__le32 *)((unsigned char *)ckpt + + le32_to_cpu(ckpt->checksum_offset))) + = cpu_to_le32(crc32); + + start_blk = __start_cp_next_addr(sbi); + + /* write nat bits */ + if (enabled_nat_bits(sbi, cpc)) { + __u64 cp_ver = cur_cp_version(ckpt); + block_t blk; + + cp_ver |= ((__u64)crc32 << 32); + *(__le64 *)nm_i->nat_bits = cpu_to_le64(cp_ver); + + blk = start_blk + BLKS_PER_SEG(sbi) - nm_i->nat_bits_blocks; + for (i = 0; i < nm_i->nat_bits_blocks; i++) + f2fs_update_meta_page(sbi, nm_i->nat_bits + + F2FS_BLK_TO_BYTES(i), blk + i); + } + + /* write out checkpoint buffer at block 0 */ + f2fs_update_meta_page(sbi, ckpt, start_blk++); + + for (i = 1; i < 1 + cp_payload_blks; i++) + f2fs_update_meta_page(sbi, (char *)ckpt + i * F2FS_BLKSIZE, + start_blk++); + + if (orphan_num) { + write_orphan_inodes(sbi, start_blk); + start_blk += orphan_blocks; + } + + f2fs_write_data_summaries(sbi, start_blk); + start_blk += data_sum_blocks; + + /* Record write statistics in the hot node summary */ + kbytes_written = sbi->kbytes_written; + kbytes_written += (f2fs_get_sectors_written(sbi) - + sbi->sectors_written_start) >> 1; + seg_i->journal->info.kbytes_written = cpu_to_le64(kbytes_written); + + if (__remain_node_summaries(cpc->reason)) { + f2fs_write_node_summaries(sbi, start_blk); + start_blk += NR_CURSEG_NODE_TYPE; + } + + /* Here, we have one bio having CP pack except cp pack 2 page */ + f2fs_sync_meta_pages(sbi, LONG_MAX, FS_CP_META_IO); + stat_cp_time(cpc, CP_TIME_SYNC_CP_META); + + /* Wait for all dirty meta pages to be submitted for IO */ + f2fs_wait_on_all_pages(sbi, F2FS_DIRTY_META); + stat_cp_time(cpc, CP_TIME_WAIT_DIRTY_META); + + /* wait for previous submitted meta pages writeback */ + f2fs_wait_on_all_pages(sbi, F2FS_WB_CP_DATA); + stat_cp_time(cpc, CP_TIME_WAIT_CP_DATA); + + /* flush all device cache */ + err = f2fs_flush_device_cache(sbi); + if (err) + return err; + stat_cp_time(cpc, CP_TIME_FLUSH_DEVICE); + + /* barrier and flush checkpoint cp pack 2 page if it can */ + commit_checkpoint(sbi, ckpt, start_blk); + f2fs_wait_on_all_pages(sbi, F2FS_WB_CP_DATA); + stat_cp_time(cpc, CP_TIME_WAIT_LAST_CP); + + /* + * invalidate intermediate page cache borrowed from meta inode which are + * used for migration of encrypted, verity or compressed inode's blocks. + */ + if (f2fs_sb_has_encrypt(sbi) || f2fs_sb_has_verity(sbi) || + f2fs_sb_has_compression(sbi)) + f2fs_bug_on(sbi, + invalidate_inode_pages2_range(META_MAPPING(sbi), + MAIN_BLKADDR(sbi), MAX_BLKADDR(sbi) - 1)); + + f2fs_release_ino_entry(sbi, false); + + f2fs_reset_fsync_node_info(sbi); + + clear_sbi_flag(sbi, SBI_IS_DIRTY); + clear_sbi_flag(sbi, SBI_NEED_CP); + clear_sbi_flag(sbi, SBI_QUOTA_SKIP_FLUSH); + + spin_lock(&sbi->stat_lock); + sbi->unusable_block_count = 0; + spin_unlock(&sbi->stat_lock); + + __set_cp_next_pack(sbi); + + /* + * redirty superblock if metadata like node page or inode cache is + * updated during writing checkpoint. + */ + if (get_pages(sbi, F2FS_DIRTY_NODES) || + get_pages(sbi, F2FS_DIRTY_IMETA)) + set_sbi_flag(sbi, SBI_IS_DIRTY); + + f2fs_bug_on(sbi, get_pages(sbi, F2FS_DIRTY_DENTS)); + + return unlikely(f2fs_cp_error(sbi)) ? -EIO : 0; +} + +int f2fs_write_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc) +{ + struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); + struct f2fs_lock_context lc; + unsigned long long ckpt_ver; + int err = 0; + + stat_cp_time(cpc, CP_TIME_START); + + if (f2fs_readonly(sbi->sb) || f2fs_hw_is_readonly(sbi)) + return -EROFS; + + if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { + if (cpc->reason != CP_PAUSE) + return 0; + f2fs_warn(sbi, "Start checkpoint disabled!"); + } + if (cpc->reason != CP_RESIZE) + f2fs_down_write_trace(&sbi->cp_global_sem, &lc); + + stat_cp_time(cpc, CP_TIME_LOCK); + + if (!is_sbi_flag_set(sbi, SBI_IS_DIRTY) && + ((cpc->reason & CP_FASTBOOT) || (cpc->reason & CP_SYNC) || + ((cpc->reason & CP_DISCARD) && !sbi->discard_blks))) + goto out; + if (unlikely(f2fs_cp_error(sbi))) { + err = -EIO; + goto out; + } + + trace_f2fs_write_checkpoint(sbi->sb, cpc->reason, CP_PHASE_START_BLOCK_OPS); + + err = block_operations(sbi); + if (err) + goto out; + + stat_cp_time(cpc, CP_TIME_OP_LOCK); + + trace_f2fs_write_checkpoint(sbi->sb, cpc->reason, CP_PHASE_FINISH_BLOCK_OPS); + + f2fs_flush_merged_writes(sbi); + + /* this is the case of multiple fstrims without any changes */ + if (cpc->reason & CP_DISCARD) { + if (!f2fs_exist_trim_candidates(sbi, cpc)) { + unblock_operations(sbi); + goto out; + } + + if (NM_I(sbi)->nat_cnt[DIRTY_NAT] == 0 && + SIT_I(sbi)->dirty_sentries == 0 && + prefree_segments(sbi) == 0) { + f2fs_flush_sit_entries(sbi, cpc); + f2fs_clear_prefree_segments(sbi, cpc); + unblock_operations(sbi); + goto out; + } + } + stat_cp_time(cpc, CP_TIME_MERGE_WRITE); + + /* + * update checkpoint pack index + * Increase the version number so that + * SIT entries and seg summaries are written at correct place + */ + ckpt_ver = cur_cp_version(ckpt); + ckpt->checkpoint_ver = cpu_to_le64(++ckpt_ver); + + /* write cached NAT/SIT entries to NAT/SIT area */ + err = f2fs_flush_nat_entries(sbi, cpc); + if (err) { + f2fs_err(sbi, "f2fs_flush_nat_entries failed err:%d, stop checkpoint", err); + f2fs_bug_on(sbi, !f2fs_cp_error(sbi)); + goto stop; + } + stat_cp_time(cpc, CP_TIME_FLUSH_NAT); + + f2fs_flush_sit_entries(sbi, cpc); + + stat_cp_time(cpc, CP_TIME_FLUSH_SIT); + + /* save inmem log status */ + f2fs_save_inmem_curseg(sbi); + + err = do_checkpoint(sbi, cpc); + if (err) { + f2fs_err(sbi, "do_checkpoint failed err:%d, stop checkpoint", err); + f2fs_bug_on(sbi, !f2fs_cp_error(sbi)); + f2fs_release_discard_addrs(sbi); + } else { + f2fs_clear_prefree_segments(sbi, cpc); + } + + f2fs_restore_inmem_curseg(sbi); + f2fs_reinit_atgc_curseg(sbi); + stat_inc_cp_count(sbi); +stop: + unblock_operations(sbi); + stat_cp_time(cpc, CP_TIME_END); + check_cp_time(sbi, cpc); + + if (cpc->reason & CP_RECOVERY) + f2fs_notice(sbi, "checkpoint: version = %llx", ckpt_ver); + + /* update CP_TIME to trigger checkpoint periodically */ + f2fs_update_time(sbi, CP_TIME); + trace_f2fs_write_checkpoint(sbi->sb, cpc->reason, CP_PHASE_FINISH_CHECKPOINT); +out: + if (cpc->reason != CP_RESIZE) + f2fs_up_write_trace(&sbi->cp_global_sem, &lc); + return err; +} + +void f2fs_init_ino_entry_info(struct f2fs_sb_info *sbi) +{ + int i; + + for (i = 0; i < MAX_INO_ENTRY; i++) { + struct inode_management *im = &sbi->im[i]; + + INIT_RADIX_TREE(&im->ino_root, GFP_ATOMIC); + spin_lock_init(&im->ino_lock); + INIT_LIST_HEAD(&im->ino_list); + im->ino_num = 0; + } + + sbi->max_orphans = (BLKS_PER_SEG(sbi) - F2FS_CP_PACKS - + NR_CURSEG_PERSIST_TYPE - __cp_payload(sbi)) * + F2FS_ORPHANS_PER_BLOCK; +} + +int __init f2fs_create_checkpoint_caches(void) +{ + ino_entry_slab = f2fs_kmem_cache_create("f2fs_ino_entry", + sizeof(struct ino_entry)); + if (!ino_entry_slab) + return -ENOMEM; + f2fs_inode_entry_slab = f2fs_kmem_cache_create("f2fs_inode_entry", + sizeof(struct inode_entry)); + if (!f2fs_inode_entry_slab) { + kmem_cache_destroy(ino_entry_slab); + return -ENOMEM; + } + return 0; +} + +void f2fs_destroy_checkpoint_caches(void) +{ + kmem_cache_destroy(ino_entry_slab); + kmem_cache_destroy(f2fs_inode_entry_slab); +} + +static int __write_checkpoint_sync(struct f2fs_sb_info *sbi) +{ + struct cp_control cpc = { .reason = CP_SYNC, }; + struct f2fs_lock_context lc; + int err; + + f2fs_down_write_trace(&sbi->gc_lock, &lc); + err = f2fs_write_checkpoint(sbi, &cpc); + f2fs_up_write_trace(&sbi->gc_lock, &lc); + + return err; +} + +static void __checkpoint_and_complete_reqs(struct f2fs_sb_info *sbi) +{ + struct ckpt_req_control *cprc = &sbi->cprc_info; + struct ckpt_req *req, *next; + struct llist_node *dispatch_list; + u64 sum_diff = 0, diff, count = 0; + int ret; + + dispatch_list = llist_del_all(&cprc->issue_list); + if (!dispatch_list) + return; + dispatch_list = llist_reverse_order(dispatch_list); + + ret = __write_checkpoint_sync(sbi); + atomic_inc(&cprc->issued_ckpt); + + llist_for_each_entry_safe(req, next, dispatch_list, llnode) { + diff = (u64)ktime_ms_delta(ktime_get(), req->queue_time); + req->ret = ret; + req->delta_time = diff; + complete(&req->wait); + + sum_diff += diff; + count++; + } + atomic_sub(count, &cprc->queued_ckpt); + atomic_add(count, &cprc->total_ckpt); + + spin_lock(&cprc->stat_lock); + cprc->cur_time = (unsigned int)div64_u64(sum_diff, count); + if (cprc->peak_time < cprc->cur_time) + cprc->peak_time = cprc->cur_time; + spin_unlock(&cprc->stat_lock); +} + +static int issue_checkpoint_thread(void *data) +{ + struct f2fs_sb_info *sbi = data; + struct ckpt_req_control *cprc = &sbi->cprc_info; + wait_queue_head_t *q = &cprc->ckpt_wait_queue; +repeat: + if (kthread_should_stop()) + return 0; + + if (!llist_empty(&cprc->issue_list)) + __checkpoint_and_complete_reqs(sbi); + + wait_event_interruptible(*q, + kthread_should_stop() || !llist_empty(&cprc->issue_list)); + goto repeat; +} + +static void flush_remained_ckpt_reqs(struct f2fs_sb_info *sbi, + struct ckpt_req *wait_req) +{ + struct ckpt_req_control *cprc = &sbi->cprc_info; + + if (!llist_empty(&cprc->issue_list)) { + __checkpoint_and_complete_reqs(sbi); + } else { + /* already dispatched by issue_checkpoint_thread */ + if (wait_req) + wait_for_completion(&wait_req->wait); + } +} + +static void init_ckpt_req(struct ckpt_req *req) +{ + memset(req, 0, sizeof(struct ckpt_req)); + + init_completion(&req->wait); + req->queue_time = ktime_get(); +} + +int f2fs_issue_checkpoint(struct f2fs_sb_info *sbi) +{ + struct ckpt_req_control *cprc = &sbi->cprc_info; + struct ckpt_req req; + struct cp_control cpc; + + cpc.reason = __get_cp_reason(sbi); + if (!test_opt(sbi, MERGE_CHECKPOINT) || cpc.reason != CP_SYNC || + sbi->umount_lock_holder == current) { + struct f2fs_lock_context lc; + int ret; + + f2fs_down_write_trace(&sbi->gc_lock, &lc); + ret = f2fs_write_checkpoint(sbi, &cpc); + f2fs_up_write_trace(&sbi->gc_lock, &lc); + + return ret; + } + + if (!cprc->f2fs_issue_ckpt) + return __write_checkpoint_sync(sbi); + + init_ckpt_req(&req); + + llist_add(&req.llnode, &cprc->issue_list); + atomic_inc(&cprc->queued_ckpt); + + /* + * update issue_list before we wake up issue_checkpoint thread, + * this smp_mb() pairs with another barrier in ___wait_event(), + * see more details in comments of waitqueue_active(). + */ + smp_mb(); + + if (waitqueue_active(&cprc->ckpt_wait_queue)) + wake_up(&cprc->ckpt_wait_queue); + + if (cprc->f2fs_issue_ckpt) + wait_for_completion(&req.wait); + else + flush_remained_ckpt_reqs(sbi, &req); + + if (unlikely(req.delta_time >= CP_LONG_LATENCY_THRESHOLD)) { + f2fs_warn_ratelimited(sbi, + "blocked on checkpoint for %u ms", cprc->peak_time); + dump_stack(); + } + + return req.ret; +} + +int f2fs_start_ckpt_thread(struct f2fs_sb_info *sbi) +{ + dev_t dev = sbi->sb->s_bdev->bd_dev; + struct ckpt_req_control *cprc = &sbi->cprc_info; + + if (cprc->f2fs_issue_ckpt) + return 0; + + cprc->f2fs_issue_ckpt = kthread_run(issue_checkpoint_thread, sbi, + "f2fs_ckpt-%u:%u", MAJOR(dev), MINOR(dev)); + if (IS_ERR(cprc->f2fs_issue_ckpt)) { + int err = PTR_ERR(cprc->f2fs_issue_ckpt); + + cprc->f2fs_issue_ckpt = NULL; + return err; + } + + set_task_ioprio(cprc->f2fs_issue_ckpt, cprc->ckpt_thread_ioprio); + set_user_nice(cprc->f2fs_issue_ckpt, + PRIO_TO_NICE(sbi->critical_task_priority)); + + return 0; +} + +void f2fs_stop_ckpt_thread(struct f2fs_sb_info *sbi) +{ + struct ckpt_req_control *cprc = &sbi->cprc_info; + struct task_struct *ckpt_task; + + if (!cprc->f2fs_issue_ckpt) + return; + + ckpt_task = cprc->f2fs_issue_ckpt; + cprc->f2fs_issue_ckpt = NULL; + kthread_stop(ckpt_task); + + f2fs_flush_ckpt_thread(sbi); +} + +void f2fs_flush_ckpt_thread(struct f2fs_sb_info *sbi) +{ + struct ckpt_req_control *cprc = &sbi->cprc_info; + + flush_remained_ckpt_reqs(sbi, NULL); + + /* Let's wait for the previous dispatched checkpoint. */ + while (atomic_read(&cprc->queued_ckpt)) + io_schedule_timeout(DEFAULT_SCHEDULE_TIMEOUT); +} + +void f2fs_init_ckpt_req_control(struct f2fs_sb_info *sbi) +{ + struct ckpt_req_control *cprc = &sbi->cprc_info; + + atomic_set(&cprc->issued_ckpt, 0); + atomic_set(&cprc->total_ckpt, 0); + atomic_set(&cprc->queued_ckpt, 0); + cprc->ckpt_thread_ioprio = DEFAULT_CHECKPOINT_IOPRIO; + init_waitqueue_head(&cprc->ckpt_wait_queue); + init_llist_head(&cprc->issue_list); + spin_lock_init(&cprc->stat_lock); +} diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c new file mode 100644 index 000000000..ce88092d9 --- /dev/null +++ b/fs/f2fs/compress.c @@ -0,0 +1,2090 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * f2fs compress support + * + * Copyright (c) 2019 Chao Yu + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "f2fs.h" +#include "node.h" +#include "segment.h" +#include + +static struct kmem_cache *cic_entry_slab; +static struct kmem_cache *dic_entry_slab; + +static void *page_array_alloc(struct f2fs_sb_info *sbi, int nr) +{ + unsigned int size = sizeof(struct page *) * nr; + + if (likely(size <= sbi->page_array_slab_size)) + return f2fs_kmem_cache_alloc(sbi->page_array_slab, + GFP_F2FS_ZERO, false, sbi); + return f2fs_kzalloc(sbi, size, GFP_NOFS); +} + +static void page_array_free(struct f2fs_sb_info *sbi, void *pages, int nr) +{ + unsigned int size = sizeof(struct page *) * nr; + + if (!pages) + return; + + if (likely(size <= sbi->page_array_slab_size)) + kmem_cache_free(sbi->page_array_slab, pages); + else + kfree(pages); +} + +struct f2fs_compress_ops { + int (*init_compress_ctx)(struct compress_ctx *cc); + void (*destroy_compress_ctx)(struct compress_ctx *cc); + int (*compress_pages)(struct compress_ctx *cc); + int (*init_decompress_ctx)(struct decompress_io_ctx *dic); + void (*destroy_decompress_ctx)(struct decompress_io_ctx *dic); + int (*decompress_pages)(struct decompress_io_ctx *dic); + bool (*is_level_valid)(int level); +}; + +static unsigned int offset_in_cluster(struct compress_ctx *cc, pgoff_t index) +{ + return index & (cc->cluster_size - 1); +} + +static pgoff_t cluster_idx(struct compress_ctx *cc, pgoff_t index) +{ + return index >> cc->log_cluster_size; +} + +static pgoff_t start_idx_of_cluster(struct compress_ctx *cc) +{ + return cc->cluster_idx << cc->log_cluster_size; +} + +bool f2fs_is_compressed_page(struct folio *folio) +{ + if (!folio->private) + return false; + if (folio_test_f2fs_nonpointer(folio)) + return false; + + f2fs_bug_on(F2FS_F_SB(folio), + *((u32 *)folio->private) != F2FS_COMPRESSED_PAGE_MAGIC); + return true; +} + +static void f2fs_set_compressed_page(struct page *page, + struct inode *inode, pgoff_t index, void *data) +{ + struct folio *folio = page_folio(page); + + folio_attach_private(folio, (void *)data); + + /* i_crypto_info and iv index */ + folio->index = index; + folio->mapping = inode->i_mapping; +} + +static void f2fs_drop_rpages(struct compress_ctx *cc, int len, bool unlock) +{ + int i; + + for (i = 0; i < len; i++) { + if (!cc->rpages[i]) + continue; + if (unlock) + unlock_page(cc->rpages[i]); + else + put_page(cc->rpages[i]); + } +} + +static void f2fs_put_rpages(struct compress_ctx *cc) +{ + f2fs_drop_rpages(cc, cc->cluster_size, false); +} + +static void f2fs_unlock_rpages(struct compress_ctx *cc, int len) +{ + f2fs_drop_rpages(cc, len, true); +} + +static void f2fs_put_rpages_wbc(struct compress_ctx *cc, + struct writeback_control *wbc, bool redirty, bool unlock) +{ + unsigned int i; + + for (i = 0; i < cc->cluster_size; i++) { + if (!cc->rpages[i]) + continue; + if (redirty) + redirty_page_for_writepage(wbc, cc->rpages[i]); + f2fs_put_page(cc->rpages[i], unlock); + } +} + +struct folio *f2fs_compress_control_folio(struct folio *folio) +{ + struct compress_io_ctx *ctx = folio->private; + + return page_folio(ctx->rpages[0]); +} + +int f2fs_init_compress_ctx(struct compress_ctx *cc) +{ + if (cc->rpages) + return 0; + + cc->rpages = page_array_alloc(F2FS_I_SB(cc->inode), cc->cluster_size); + return cc->rpages ? 0 : -ENOMEM; +} + +void f2fs_destroy_compress_ctx(struct compress_ctx *cc, bool reuse) +{ + page_array_free(F2FS_I_SB(cc->inode), cc->rpages, cc->cluster_size); + cc->rpages = NULL; + cc->nr_rpages = 0; + cc->nr_cpages = 0; + cc->valid_nr_cpages = 0; + if (!reuse) + cc->cluster_idx = NULL_CLUSTER; +} + +void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct folio *folio) +{ + unsigned int cluster_ofs; + + if (!f2fs_cluster_can_merge_page(cc, folio->index)) + f2fs_bug_on(F2FS_I_SB(cc->inode), 1); + + cluster_ofs = offset_in_cluster(cc, folio->index); + cc->rpages[cluster_ofs] = folio_page(folio, 0); + cc->nr_rpages++; + cc->cluster_idx = cluster_idx(cc, folio->index); +} + +#ifdef CONFIG_F2FS_FS_LZO +static int lzo_init_compress_ctx(struct compress_ctx *cc) +{ + cc->private = f2fs_vmalloc(F2FS_I_SB(cc->inode), + LZO1X_MEM_COMPRESS); + if (!cc->private) + return -ENOMEM; + + cc->clen = lzo1x_worst_compress(PAGE_SIZE << cc->log_cluster_size); + return 0; +} + +static void lzo_destroy_compress_ctx(struct compress_ctx *cc) +{ + vfree(cc->private); + cc->private = NULL; +} + +static int lzo_compress_pages(struct compress_ctx *cc) +{ + int ret; + + ret = lzo1x_1_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata, + &cc->clen, cc->private); + if (ret != LZO_E_OK) { + f2fs_err_ratelimited(F2FS_I_SB(cc->inode), + "lzo compress failed, ret:%d", ret); + return -EIO; + } + return 0; +} + +static int lzo_decompress_pages(struct decompress_io_ctx *dic) +{ + int ret; + + ret = lzo1x_decompress_safe(dic->cbuf->cdata, dic->clen, + dic->rbuf, &dic->rlen); + if (ret != LZO_E_OK) { + f2fs_err_ratelimited(dic->sbi, + "lzo decompress failed, ret:%d", ret); + return -EIO; + } + + if (dic->rlen != PAGE_SIZE << dic->log_cluster_size) { + f2fs_err_ratelimited(dic->sbi, + "lzo invalid rlen:%zu, expected:%lu", + dic->rlen, PAGE_SIZE << dic->log_cluster_size); + return -EIO; + } + return 0; +} + +static const struct f2fs_compress_ops f2fs_lzo_ops = { + .init_compress_ctx = lzo_init_compress_ctx, + .destroy_compress_ctx = lzo_destroy_compress_ctx, + .compress_pages = lzo_compress_pages, + .decompress_pages = lzo_decompress_pages, +}; +#endif + +#ifdef CONFIG_F2FS_FS_LZ4 +static int lz4_init_compress_ctx(struct compress_ctx *cc) +{ + unsigned int size = LZ4_MEM_COMPRESS; + +#ifdef CONFIG_F2FS_FS_LZ4HC + if (F2FS_I(cc->inode)->i_compress_level) + size = LZ4HC_MEM_COMPRESS; +#endif + + cc->private = f2fs_vmalloc(F2FS_I_SB(cc->inode), size); + if (!cc->private) + return -ENOMEM; + + /* + * we do not change cc->clen to LZ4_compressBound(inputsize) to + * adapt worst compress case, because lz4 compressor can handle + * output budget properly. + */ + cc->clen = cc->rlen - PAGE_SIZE - COMPRESS_HEADER_SIZE; + return 0; +} + +static void lz4_destroy_compress_ctx(struct compress_ctx *cc) +{ + vfree(cc->private); + cc->private = NULL; +} + +static int lz4_compress_pages(struct compress_ctx *cc) +{ + int len = -EINVAL; + unsigned char level = F2FS_I(cc->inode)->i_compress_level; + + if (!level) + len = LZ4_compress_default(cc->rbuf, cc->cbuf->cdata, cc->rlen, + cc->clen, cc->private); +#ifdef CONFIG_F2FS_FS_LZ4HC + else + len = LZ4_compress_HC(cc->rbuf, cc->cbuf->cdata, cc->rlen, + cc->clen, level, cc->private); +#endif + if (len < 0) + return len; + if (!len) + return -EAGAIN; + + cc->clen = len; + return 0; +} + +static int lz4_decompress_pages(struct decompress_io_ctx *dic) +{ + int ret; + + ret = LZ4_decompress_safe(dic->cbuf->cdata, dic->rbuf, + dic->clen, dic->rlen); + if (ret < 0) { + f2fs_err_ratelimited(dic->sbi, + "lz4 decompress failed, ret:%d", ret); + return -EIO; + } + + if (ret != PAGE_SIZE << dic->log_cluster_size) { + f2fs_err_ratelimited(dic->sbi, + "lz4 invalid ret:%d, expected:%lu", + ret, PAGE_SIZE << dic->log_cluster_size); + return -EIO; + } + return 0; +} + +static bool lz4_is_level_valid(int lvl) +{ +#ifdef CONFIG_F2FS_FS_LZ4HC + return !lvl || (lvl >= LZ4HC_MIN_CLEVEL && lvl <= LZ4HC_MAX_CLEVEL); +#else + return lvl == 0; +#endif +} + +static const struct f2fs_compress_ops f2fs_lz4_ops = { + .init_compress_ctx = lz4_init_compress_ctx, + .destroy_compress_ctx = lz4_destroy_compress_ctx, + .compress_pages = lz4_compress_pages, + .decompress_pages = lz4_decompress_pages, + .is_level_valid = lz4_is_level_valid, +}; +#endif + +#ifdef CONFIG_F2FS_FS_ZSTD +static int zstd_init_compress_ctx(struct compress_ctx *cc) +{ + zstd_parameters params; + zstd_cstream *stream; + void *workspace; + unsigned int workspace_size; + unsigned char level = F2FS_I(cc->inode)->i_compress_level; + + /* Need to remain this for backward compatibility */ + if (!level) + level = F2FS_ZSTD_DEFAULT_CLEVEL; + + params = zstd_get_params(level, cc->rlen); + workspace_size = zstd_cstream_workspace_bound(¶ms.cParams); + + workspace = f2fs_vmalloc(F2FS_I_SB(cc->inode), workspace_size); + if (!workspace) + return -ENOMEM; + + stream = zstd_init_cstream(¶ms, 0, workspace, workspace_size); + if (!stream) { + f2fs_err_ratelimited(F2FS_I_SB(cc->inode), + "%s zstd_init_cstream failed", __func__); + vfree(workspace); + return -EIO; + } + + cc->private = workspace; + cc->private2 = stream; + + cc->clen = cc->rlen - PAGE_SIZE - COMPRESS_HEADER_SIZE; + return 0; +} + +static void zstd_destroy_compress_ctx(struct compress_ctx *cc) +{ + vfree(cc->private); + cc->private = NULL; + cc->private2 = NULL; +} + +static int zstd_compress_pages(struct compress_ctx *cc) +{ + zstd_cstream *stream = cc->private2; + zstd_in_buffer inbuf; + zstd_out_buffer outbuf; + int src_size = cc->rlen; + int dst_size = src_size - PAGE_SIZE - COMPRESS_HEADER_SIZE; + int ret; + + inbuf.pos = 0; + inbuf.src = cc->rbuf; + inbuf.size = src_size; + + outbuf.pos = 0; + outbuf.dst = cc->cbuf->cdata; + outbuf.size = dst_size; + + ret = zstd_compress_stream(stream, &outbuf, &inbuf); + if (zstd_is_error(ret)) { + f2fs_err_ratelimited(F2FS_I_SB(cc->inode), + "%s zstd_compress_stream failed, ret: %d", + __func__, zstd_get_error_code(ret)); + return -EIO; + } + + ret = zstd_end_stream(stream, &outbuf); + if (zstd_is_error(ret)) { + f2fs_err_ratelimited(F2FS_I_SB(cc->inode), + "%s zstd_end_stream returned %d", + __func__, zstd_get_error_code(ret)); + return -EIO; + } + + /* + * there is compressed data remained in intermediate buffer due to + * no more space in cbuf.cdata + */ + if (ret) + return -EAGAIN; + + cc->clen = outbuf.pos; + return 0; +} + +static int zstd_init_decompress_ctx(struct decompress_io_ctx *dic) +{ + zstd_dstream *stream; + void *workspace; + unsigned int workspace_size; + unsigned int max_window_size = + MAX_COMPRESS_WINDOW_SIZE(dic->log_cluster_size); + + workspace_size = zstd_dstream_workspace_bound(max_window_size); + + workspace = f2fs_vmalloc(dic->sbi, workspace_size); + if (!workspace) + return -ENOMEM; + + stream = zstd_init_dstream(max_window_size, workspace, workspace_size); + if (!stream) { + f2fs_err_ratelimited(dic->sbi, + "%s zstd_init_dstream failed", __func__); + vfree(workspace); + return -EIO; + } + + dic->private = workspace; + dic->private2 = stream; + + return 0; +} + +static void zstd_destroy_decompress_ctx(struct decompress_io_ctx *dic) +{ + vfree(dic->private); + dic->private = NULL; + dic->private2 = NULL; +} + +static int zstd_decompress_pages(struct decompress_io_ctx *dic) +{ + zstd_dstream *stream = dic->private2; + zstd_in_buffer inbuf; + zstd_out_buffer outbuf; + int ret; + + inbuf.pos = 0; + inbuf.src = dic->cbuf->cdata; + inbuf.size = dic->clen; + + outbuf.pos = 0; + outbuf.dst = dic->rbuf; + outbuf.size = dic->rlen; + + ret = zstd_decompress_stream(stream, &outbuf, &inbuf); + if (zstd_is_error(ret)) { + f2fs_err_ratelimited(dic->sbi, + "%s zstd_decompress_stream failed, ret: %d", + __func__, zstd_get_error_code(ret)); + return -EIO; + } + + if (dic->rlen != outbuf.pos) { + f2fs_err_ratelimited(dic->sbi, + "%s ZSTD invalid rlen:%zu, expected:%lu", + __func__, dic->rlen, + PAGE_SIZE << dic->log_cluster_size); + return -EIO; + } + + return 0; +} + +static bool zstd_is_level_valid(int lvl) +{ + return lvl >= zstd_min_clevel() && lvl <= zstd_max_clevel(); +} + +static const struct f2fs_compress_ops f2fs_zstd_ops = { + .init_compress_ctx = zstd_init_compress_ctx, + .destroy_compress_ctx = zstd_destroy_compress_ctx, + .compress_pages = zstd_compress_pages, + .init_decompress_ctx = zstd_init_decompress_ctx, + .destroy_decompress_ctx = zstd_destroy_decompress_ctx, + .decompress_pages = zstd_decompress_pages, + .is_level_valid = zstd_is_level_valid, +}; +#endif + +#ifdef CONFIG_F2FS_FS_LZO +#ifdef CONFIG_F2FS_FS_LZORLE +static int lzorle_compress_pages(struct compress_ctx *cc) +{ + int ret; + + ret = lzorle1x_1_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata, + &cc->clen, cc->private); + if (ret != LZO_E_OK) { + f2fs_err_ratelimited(F2FS_I_SB(cc->inode), + "lzo-rle compress failed, ret:%d", ret); + return -EIO; + } + return 0; +} + +static const struct f2fs_compress_ops f2fs_lzorle_ops = { + .init_compress_ctx = lzo_init_compress_ctx, + .destroy_compress_ctx = lzo_destroy_compress_ctx, + .compress_pages = lzorle_compress_pages, + .decompress_pages = lzo_decompress_pages, +}; +#endif +#endif + +static const struct f2fs_compress_ops *f2fs_cops[COMPRESS_MAX] = { +#ifdef CONFIG_F2FS_FS_LZO + &f2fs_lzo_ops, +#else + NULL, +#endif +#ifdef CONFIG_F2FS_FS_LZ4 + &f2fs_lz4_ops, +#else + NULL, +#endif +#ifdef CONFIG_F2FS_FS_ZSTD + &f2fs_zstd_ops, +#else + NULL, +#endif +#if defined(CONFIG_F2FS_FS_LZO) && defined(CONFIG_F2FS_FS_LZORLE) + &f2fs_lzorle_ops, +#else + NULL, +#endif +}; + +bool f2fs_is_compress_backend_ready(struct inode *inode) +{ + if (!f2fs_compressed_file(inode)) + return true; + return f2fs_cops[F2FS_I(inode)->i_compress_algorithm]; +} + +bool f2fs_is_compress_level_valid(int alg, int lvl) +{ + const struct f2fs_compress_ops *cops = f2fs_cops[alg]; + + if (cops->is_level_valid) + return cops->is_level_valid(lvl); + + return lvl == 0; +} + +static mempool_t *compress_page_pool; +static int num_compress_pages = 512; +module_param(num_compress_pages, uint, 0444); +MODULE_PARM_DESC(num_compress_pages, + "Number of intermediate compress pages to preallocate"); + +int __init f2fs_init_compress_mempool(void) +{ + compress_page_pool = mempool_create_page_pool(num_compress_pages, 0); + return compress_page_pool ? 0 : -ENOMEM; +} + +void f2fs_destroy_compress_mempool(void) +{ + mempool_destroy(compress_page_pool); +} + +static struct page *f2fs_compress_alloc_page(void) +{ + struct page *page; + + page = mempool_alloc(compress_page_pool, GFP_NOFS); + lock_page(page); + + return page; +} + +static void f2fs_compress_free_page(struct page *page) +{ + struct folio *folio; + + if (!page) + return; + folio = page_folio(page); + folio_detach_private(folio); + folio->mapping = NULL; + folio_unlock(folio); + mempool_free(page, compress_page_pool); +} + +#define MAX_VMAP_RETRIES 3 + +static void *f2fs_vmap(struct page **pages, unsigned int count) +{ + int i; + void *buf = NULL; + + for (i = 0; i < MAX_VMAP_RETRIES; i++) { + buf = vm_map_ram(pages, count, -1); + if (buf) + break; + vm_unmap_aliases(); + } + return buf; +} + +static int f2fs_compress_pages(struct compress_ctx *cc) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode); + struct f2fs_inode_info *fi = F2FS_I(cc->inode); + const struct f2fs_compress_ops *cops = + f2fs_cops[fi->i_compress_algorithm]; + unsigned int max_len, new_nr_cpages; + u32 chksum = 0; + int i, ret; + + trace_f2fs_compress_pages_start(cc->inode, cc->cluster_idx, + cc->cluster_size, fi->i_compress_algorithm); + + if (cops->init_compress_ctx) { + ret = cops->init_compress_ctx(cc); + if (ret) + goto out; + } + + max_len = COMPRESS_HEADER_SIZE + cc->clen; + cc->nr_cpages = DIV_ROUND_UP(max_len, PAGE_SIZE); + cc->valid_nr_cpages = cc->nr_cpages; + + cc->cpages = page_array_alloc(sbi, cc->nr_cpages); + if (!cc->cpages) { + ret = -ENOMEM; + goto destroy_compress_ctx; + } + + for (i = 0; i < cc->nr_cpages; i++) + cc->cpages[i] = f2fs_compress_alloc_page(); + + cc->rbuf = f2fs_vmap(cc->rpages, cc->cluster_size); + if (!cc->rbuf) { + ret = -ENOMEM; + goto out_free_cpages; + } + + cc->cbuf = f2fs_vmap(cc->cpages, cc->nr_cpages); + if (!cc->cbuf) { + ret = -ENOMEM; + goto out_vunmap_rbuf; + } + + ret = cops->compress_pages(cc); + if (ret) + goto out_vunmap_cbuf; + + max_len = PAGE_SIZE * (cc->cluster_size - 1) - COMPRESS_HEADER_SIZE; + + if (cc->clen > max_len) { + ret = -EAGAIN; + goto out_vunmap_cbuf; + } + + cc->cbuf->clen = cpu_to_le32(cc->clen); + + if (fi->i_compress_flag & BIT(COMPRESS_CHKSUM)) + chksum = f2fs_crc32(cc->cbuf->cdata, cc->clen); + cc->cbuf->chksum = cpu_to_le32(chksum); + + for (i = 0; i < COMPRESS_DATA_RESERVED_SIZE; i++) + cc->cbuf->reserved[i] = cpu_to_le32(0); + + new_nr_cpages = DIV_ROUND_UP(cc->clen + COMPRESS_HEADER_SIZE, PAGE_SIZE); + + /* zero out any unused part of the last page */ + memset(&cc->cbuf->cdata[cc->clen], 0, + (new_nr_cpages * PAGE_SIZE) - + (cc->clen + COMPRESS_HEADER_SIZE)); + + vm_unmap_ram(cc->cbuf, cc->nr_cpages); + vm_unmap_ram(cc->rbuf, cc->cluster_size); + + for (i = new_nr_cpages; i < cc->nr_cpages; i++) { + f2fs_compress_free_page(cc->cpages[i]); + cc->cpages[i] = NULL; + } + + if (cops->destroy_compress_ctx) + cops->destroy_compress_ctx(cc); + + cc->valid_nr_cpages = new_nr_cpages; + + trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx, + cc->clen, ret); + return 0; + +out_vunmap_cbuf: + vm_unmap_ram(cc->cbuf, cc->nr_cpages); +out_vunmap_rbuf: + vm_unmap_ram(cc->rbuf, cc->cluster_size); +out_free_cpages: + for (i = 0; i < cc->nr_cpages; i++) { + if (cc->cpages[i]) + f2fs_compress_free_page(cc->cpages[i]); + } + page_array_free(sbi, cc->cpages, cc->nr_cpages); + cc->cpages = NULL; +destroy_compress_ctx: + if (cops->destroy_compress_ctx) + cops->destroy_compress_ctx(cc); +out: + trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx, + cc->clen, ret); + return ret; +} + +static int f2fs_prepare_decomp_mem(struct decompress_io_ctx *dic, + bool pre_alloc); +static void f2fs_release_decomp_mem(struct decompress_io_ctx *dic, + bool bypass_destroy_callback, bool pre_alloc); + +void f2fs_decompress_cluster(struct decompress_io_ctx *dic, bool in_task) +{ + struct f2fs_sb_info *sbi = dic->sbi; + struct f2fs_inode_info *fi = F2FS_I(dic->inode); + const struct f2fs_compress_ops *cops = + f2fs_cops[fi->i_compress_algorithm]; + bool bypass_callback = false; + int ret; + + trace_f2fs_decompress_pages_start(dic->inode, dic->cluster_idx, + dic->cluster_size, fi->i_compress_algorithm); + + if (dic->failed) { + ret = -EIO; + goto out_end_io; + } + + ret = f2fs_prepare_decomp_mem(dic, false); + if (ret) { + bypass_callback = true; + goto out_release; + } + + dic->clen = le32_to_cpu(dic->cbuf->clen); + dic->rlen = PAGE_SIZE << dic->log_cluster_size; + + if (dic->clen > PAGE_SIZE * dic->nr_cpages - COMPRESS_HEADER_SIZE) { + ret = -EFSCORRUPTED; + + /* Avoid f2fs_commit_super in irq context */ + f2fs_handle_error(sbi, ERROR_FAIL_DECOMPRESSION); + fserror_report_file_metadata(dic->inode, ret, GFP_NOFS); + goto out_release; + } + + ret = cops->decompress_pages(dic); + + if (!ret && (fi->i_compress_flag & BIT(COMPRESS_CHKSUM))) { + u32 provided = le32_to_cpu(dic->cbuf->chksum); + u32 calculated = f2fs_crc32(dic->cbuf->cdata, dic->clen); + + if (provided != calculated) { + if (!is_inode_flag_set(dic->inode, FI_COMPRESS_CORRUPT)) { + set_inode_flag(dic->inode, FI_COMPRESS_CORRUPT); + f2fs_info_ratelimited(sbi, + "checksum invalid, nid = %llu, %x vs %x", + dic->inode->i_ino, + provided, calculated); + } + set_sbi_flag(sbi, SBI_NEED_FSCK); + } + } + +out_release: + f2fs_release_decomp_mem(dic, bypass_callback, false); + +out_end_io: + trace_f2fs_decompress_pages_end(dic->inode, dic->cluster_idx, + dic->clen, ret); + f2fs_decompress_end_io(dic, ret, in_task); +} + +static void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi, + struct folio *folio, nid_t ino, block_t blkaddr); + +/* + * This is called when a page of a compressed cluster has been read from disk + * (or failed to be read from disk). It checks whether this page was the last + * page being waited on in the cluster, and if so, it decompresses the cluster + * (or in the case of a failure, cleans up without actually decompressing). + */ +void f2fs_end_read_compressed_page(struct folio *folio, bool failed, + block_t blkaddr, bool in_task) +{ + struct decompress_io_ctx *dic = folio->private; + struct f2fs_sb_info *sbi = dic->sbi; + + dec_page_count(sbi, F2FS_RD_DATA); + + if (failed) + WRITE_ONCE(dic->failed, true); + else if (blkaddr && in_task) + f2fs_cache_compressed_page(sbi, folio, + dic->inode->i_ino, blkaddr); + + if (atomic_dec_and_test(&dic->remaining_pages)) + f2fs_decompress_cluster(dic, in_task); +} + +static bool is_page_in_cluster(struct compress_ctx *cc, pgoff_t index) +{ + if (cc->cluster_idx == NULL_CLUSTER) + return true; + return cc->cluster_idx == cluster_idx(cc, index); +} + +bool f2fs_cluster_is_empty(struct compress_ctx *cc) +{ + return cc->nr_rpages == 0; +} + +static bool f2fs_cluster_is_full(struct compress_ctx *cc) +{ + return cc->cluster_size == cc->nr_rpages; +} + +bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index) +{ + if (f2fs_cluster_is_empty(cc)) + return true; + return is_page_in_cluster(cc, index); +} + +bool f2fs_all_cluster_page_ready(struct compress_ctx *cc, struct page **pages, + int index, int nr_pages, bool uptodate) +{ + unsigned long pgidx = page_folio(pages[index])->index; + int i = uptodate ? 0 : 1; + + /* + * when uptodate set to true, try to check all pages in cluster is + * uptodate or not. + */ + if (uptodate && (pgidx % cc->cluster_size)) + return false; + + if (nr_pages - index < cc->cluster_size) + return false; + + for (; i < cc->cluster_size; i++) { + struct folio *folio = page_folio(pages[index + i]); + + if (folio->index != pgidx + i) + return false; + if (uptodate && !folio_test_uptodate(folio)) + return false; + } + + return true; +} + +static bool cluster_has_invalid_data(struct compress_ctx *cc) +{ + loff_t i_size = i_size_read(cc->inode); + unsigned nr_pages = DIV_ROUND_UP(i_size, PAGE_SIZE); + int i; + + for (i = 0; i < cc->cluster_size; i++) { + struct page *page = cc->rpages[i]; + + f2fs_bug_on(F2FS_I_SB(cc->inode), !page); + + /* beyond EOF */ + if (page_folio(page)->index >= nr_pages) + return true; + } + return false; +} + +bool f2fs_sanity_check_cluster(struct dnode_of_data *dn) +{ +#ifdef CONFIG_F2FS_CHECK_FS + struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); + unsigned int cluster_size = F2FS_I(dn->inode)->i_cluster_size; + int cluster_end = 0; + unsigned int count; + int i; + char *reason = ""; + + if (dn->data_blkaddr != COMPRESS_ADDR) + return false; + + /* [..., COMPR_ADDR, ...] */ + if (dn->ofs_in_node % cluster_size) { + reason = "[*|C|*|*]"; + goto out; + } + + for (i = 1, count = 1; i < cluster_size; i++, count++) { + block_t blkaddr = data_blkaddr(dn->inode, dn->node_folio, + dn->ofs_in_node + i); + + /* [COMPR_ADDR, ..., COMPR_ADDR] */ + if (blkaddr == COMPRESS_ADDR) { + reason = "[C|*|C|*]"; + goto out; + } + if (!__is_valid_data_blkaddr(blkaddr)) { + if (!cluster_end) + cluster_end = i; + continue; + } + /* [COMPR_ADDR, NULL_ADDR or NEW_ADDR, valid_blkaddr] */ + if (cluster_end) { + reason = "[C|N|N|V]"; + goto out; + } + } + + f2fs_bug_on(F2FS_I_SB(dn->inode), count != cluster_size && + !is_inode_flag_set(dn->inode, FI_COMPRESS_RELEASED)); + + return false; +out: + f2fs_warn(sbi, "access invalid cluster, ino:%llu, nid:%u, ofs_in_node:%u, reason:%s", + dn->inode->i_ino, dn->nid, dn->ofs_in_node, reason); + set_sbi_flag(sbi, SBI_NEED_FSCK); + return true; +#else + return false; +#endif +} + +static int __f2fs_get_cluster_blocks(struct inode *inode, + struct dnode_of_data *dn) +{ + unsigned int cluster_size = F2FS_I(inode)->i_cluster_size; + int count, i; + + for (i = 0, count = 0; i < cluster_size; i++) { + block_t blkaddr = data_blkaddr(dn->inode, dn->node_folio, + dn->ofs_in_node + i); + + if (__is_valid_data_blkaddr(blkaddr)) + count++; + } + + return count; +} + +static int __f2fs_cluster_blocks(struct inode *inode, unsigned int cluster_idx, + enum cluster_check_type type) +{ + struct dnode_of_data dn; + unsigned int start_idx = cluster_idx << + F2FS_I(inode)->i_log_cluster_size; + int ret; + + set_new_dnode(&dn, inode, NULL, NULL, 0); + ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE); + if (ret) { + if (ret == -ENOENT) + ret = 0; + goto fail; + } + + if (f2fs_sanity_check_cluster(&dn)) { + ret = -EFSCORRUPTED; + goto fail; + } + + if (dn.data_blkaddr == COMPRESS_ADDR) { + if (type == CLUSTER_COMPR_BLKS) + ret = 1 + __f2fs_get_cluster_blocks(inode, &dn); + else if (type == CLUSTER_IS_COMPR) + ret = 1; + } else if (type == CLUSTER_RAW_BLKS) { + ret = __f2fs_get_cluster_blocks(inode, &dn); + } +fail: + f2fs_put_dnode(&dn); + return ret; +} + +/* return # of compressed blocks in compressed cluster */ +static int f2fs_compressed_blocks(struct compress_ctx *cc) +{ + return __f2fs_cluster_blocks(cc->inode, cc->cluster_idx, + CLUSTER_COMPR_BLKS); +} + +/* return # of raw blocks in non-compressed cluster */ +static int f2fs_decompressed_blocks(struct inode *inode, + unsigned int cluster_idx) +{ + return __f2fs_cluster_blocks(inode, cluster_idx, + CLUSTER_RAW_BLKS); +} + +/* return whether cluster is compressed one or not */ +int f2fs_is_compressed_cluster(struct inode *inode, pgoff_t index) +{ + return __f2fs_cluster_blocks(inode, + index >> F2FS_I(inode)->i_log_cluster_size, + CLUSTER_IS_COMPR); +} + +/* return whether cluster contains non raw blocks or not */ +bool f2fs_is_sparse_cluster(struct inode *inode, pgoff_t index) +{ + unsigned int cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size; + + return f2fs_decompressed_blocks(inode, cluster_idx) != + F2FS_I(inode)->i_cluster_size; +} + +static bool cluster_may_compress(struct compress_ctx *cc) +{ + if (!f2fs_need_compress_data(cc->inode)) + return false; + if (f2fs_is_atomic_file(cc->inode)) + return false; + if (!f2fs_cluster_is_full(cc)) + return false; + if (unlikely(f2fs_cp_error(F2FS_I_SB(cc->inode)))) + return false; + return !cluster_has_invalid_data(cc); +} + +static void set_cluster_writeback(struct compress_ctx *cc) +{ + int i; + + for (i = 0; i < cc->cluster_size; i++) { + if (cc->rpages[i]) + set_page_writeback(cc->rpages[i]); + } +} + +static void cancel_cluster_writeback(struct compress_ctx *cc, + struct compress_io_ctx *cic, int submitted) +{ + int i; + + /* Wait for submitted IOs. */ + if (submitted > 1) { + f2fs_submit_merged_write(F2FS_I_SB(cc->inode), DATA); + while (atomic_read(&cic->pending_pages) != + (cc->valid_nr_cpages - submitted + 1)) + f2fs_io_schedule_timeout(DEFAULT_SCHEDULE_TIMEOUT); + } + + /* Cancel writeback and stay locked. */ + for (i = 0; i < cc->cluster_size; i++) { + if (i < submitted) { + inode_inc_dirty_pages(cc->inode); + lock_page(cc->rpages[i]); + } + clear_page_private_gcing(cc->rpages[i]); + if (folio_test_writeback(page_folio(cc->rpages[i]))) + end_page_writeback(cc->rpages[i]); + } +} + +static void set_cluster_dirty(struct compress_ctx *cc) +{ + int i; + + for (i = 0; i < cc->cluster_size; i++) + if (cc->rpages[i]) { + set_page_dirty(cc->rpages[i]); + set_page_private_gcing(cc->rpages[i]); + } +} + +static int prepare_compress_overwrite(struct compress_ctx *cc, + struct page **pagep, pgoff_t index, void **fsdata) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode); + struct address_space *mapping = cc->inode->i_mapping; + struct folio *folio; + sector_t last_block_in_bio; + fgf_t fgp_flag = FGP_LOCK | FGP_WRITE | FGP_CREAT; + pgoff_t start_idx = start_idx_of_cluster(cc); + int i, ret; + +retry: + ret = f2fs_is_compressed_cluster(cc->inode, start_idx); + if (ret <= 0) + return ret; + + ret = f2fs_init_compress_ctx(cc); + if (ret) + return ret; + + /* keep folio reference to avoid page reclaim */ + for (i = 0; i < cc->cluster_size; i++) { + folio = f2fs_filemap_get_folio(mapping, start_idx + i, + fgp_flag, GFP_NOFS); + if (IS_ERR(folio)) { + ret = PTR_ERR(folio); + goto unlock_pages; + } + + if (folio_test_uptodate(folio)) + f2fs_folio_put(folio, true); + else + f2fs_compress_ctx_add_page(cc, folio); + } + + if (!f2fs_cluster_is_empty(cc)) { + struct bio *bio = NULL; + + ret = f2fs_read_multi_pages(cc, &bio, cc->cluster_size, + &last_block_in_bio, NULL, true); + f2fs_put_rpages(cc); + f2fs_destroy_compress_ctx(cc, true); + if (ret) + goto out; + if (bio) + f2fs_submit_read_bio(sbi, bio, DATA); + + ret = f2fs_init_compress_ctx(cc); + if (ret) + goto out; + } + + for (i = 0; i < cc->cluster_size; i++) { + f2fs_bug_on(sbi, cc->rpages[i]); + + folio = filemap_lock_folio(mapping, start_idx + i); + if (IS_ERR(folio)) { + /* folio could be truncated */ + goto release_and_retry; + } + + f2fs_folio_wait_writeback(folio, DATA, true, true); + f2fs_compress_ctx_add_page(cc, folio); + + if (!folio_test_uptodate(folio)) { + f2fs_handle_page_eio(sbi, folio, DATA); +release_and_retry: + f2fs_put_rpages(cc); + f2fs_unlock_rpages(cc, i + 1); + f2fs_destroy_compress_ctx(cc, true); + goto retry; + } + } + + if (likely(!ret)) { + *fsdata = cc->rpages; + *pagep = cc->rpages[offset_in_cluster(cc, index)]; + return cc->cluster_size; + } + +unlock_pages: + f2fs_put_rpages(cc); + f2fs_unlock_rpages(cc, i); + f2fs_destroy_compress_ctx(cc, true); +out: + return ret; +} + +int f2fs_prepare_compress_overwrite(struct inode *inode, + struct page **pagep, pgoff_t index, void **fsdata) +{ + struct compress_ctx cc = { + .inode = inode, + .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, + .cluster_size = F2FS_I(inode)->i_cluster_size, + .cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size, + .rpages = NULL, + .nr_rpages = 0, + .vi = NULL, /* can't write to fsverity files */ + }; + + return prepare_compress_overwrite(&cc, pagep, index, fsdata); +} + +bool f2fs_compress_write_end(struct inode *inode, void *fsdata, + pgoff_t index, unsigned copied) + +{ + struct compress_ctx cc = { + .inode = inode, + .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, + .cluster_size = F2FS_I(inode)->i_cluster_size, + .rpages = fsdata, + }; + struct folio *folio = page_folio(cc.rpages[0]); + bool first_index = (index == folio->index); + + if (copied) + set_cluster_dirty(&cc); + + f2fs_put_rpages_wbc(&cc, NULL, false, true); + f2fs_destroy_compress_ctx(&cc, false); + + return first_index; +} + +int f2fs_truncate_partial_cluster(struct inode *inode, u64 from, bool lock) +{ + void *fsdata = NULL; + struct page *pagep; + struct page **rpages; + int log_cluster_size = F2FS_I(inode)->i_log_cluster_size; + pgoff_t start_idx = from >> (PAGE_SHIFT + log_cluster_size) << + log_cluster_size; + int i; + int err; + + err = f2fs_is_compressed_cluster(inode, start_idx); + if (err < 0) + return err; + + /* truncate normal cluster */ + if (!err) + return f2fs_do_truncate_blocks(inode, from, lock); + + /* truncate compressed cluster */ + err = f2fs_prepare_compress_overwrite(inode, &pagep, + start_idx, &fsdata); + + /* should not be a normal cluster */ + f2fs_bug_on(F2FS_I_SB(inode), err == 0); + + if (err <= 0) + return err; + + rpages = fsdata; + + for (i = (1 << log_cluster_size) - 1; i >= 0; i--) { + struct folio *folio = page_folio(rpages[i]); + loff_t start = (loff_t)folio->index << PAGE_SHIFT; + loff_t offset = from > start ? from - start : 0; + + folio_zero_segment(folio, offset, folio_size(folio)); + + if (from >= start) + break; + } + + f2fs_compress_write_end(inode, fsdata, start_idx, true); + + err = filemap_write_and_wait_range(inode->i_mapping, + round_down(from, 1 << log_cluster_size << PAGE_SHIFT), + LLONG_MAX); + if (err) + return err; + + truncate_pagecache(inode, from); + + return f2fs_do_truncate_blocks(inode, round_up(from, PAGE_SIZE), lock); +} + +static int f2fs_write_compressed_pages(struct compress_ctx *cc, + int *submitted, + struct writeback_control *wbc, + enum iostat_type io_type) +{ + struct inode *inode = cc->inode; + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_inode_info *fi = F2FS_I(inode); + struct f2fs_io_info fio = { + .sbi = sbi, + .ino = cc->inode->i_ino, + .type = DATA, + .op = REQ_OP_WRITE, + .op_flags = wbc_to_write_flags(wbc), + .old_blkaddr = NEW_ADDR, + .page = NULL, + .encrypted_page = NULL, + .compressed_page = NULL, + .io_type = io_type, + .io_wbc = wbc, + }; + struct folio *folio; + struct dnode_of_data dn; + struct node_info ni; + struct compress_io_ctx *cic; + struct f2fs_lock_context lc; + pgoff_t start_idx = start_idx_of_cluster(cc); + unsigned int last_index = cc->cluster_size - 1; + loff_t psize; + int i, err; + bool quota_inode = IS_NOQUOTA(inode); + + /* we should bypass data pages to proceed the kworker jobs */ + if (unlikely(f2fs_cp_error(sbi))) { + mapping_set_error(inode->i_mapping, -EIO); + goto out_free; + } + + if (quota_inode) { + /* + * We need to wait for node_write to avoid block allocation during + * checkpoint. This can only happen to quota writes which can cause + * the below discard race condition. + */ + f2fs_down_read_trace(&sbi->node_write, &lc); + } else if (!f2fs_trylock_op(sbi, &lc)) { + goto out_free; + } + + set_new_dnode(&dn, cc->inode, NULL, NULL, 0); + + err = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE); + if (err) + goto out_unlock_op; + + for (i = 0; i < cc->cluster_size; i++) { + if (data_blkaddr(dn.inode, dn.node_folio, + dn.ofs_in_node + i) == NULL_ADDR) + goto out_put_dnode; + } + + folio = page_folio(cc->rpages[last_index]); + psize = folio_next_pos(folio); + + err = f2fs_get_node_info(fio.sbi, dn.nid, &ni, false); + if (err) + goto out_put_dnode; + + fio.version = ni.version; + + cic = f2fs_kmem_cache_alloc(cic_entry_slab, GFP_F2FS_ZERO, false, sbi); + if (!cic) + goto out_put_dnode; + + cic->magic = F2FS_COMPRESSED_PAGE_MAGIC; + cic->inode = inode; + atomic_set(&cic->pending_pages, cc->valid_nr_cpages); + cic->rpages = page_array_alloc(sbi, cc->cluster_size); + if (!cic->rpages) + goto out_put_cic; + + cic->nr_rpages = cc->cluster_size; + + for (i = 0; i < cc->valid_nr_cpages; i++) { + f2fs_set_compressed_page(cc->cpages[i], inode, + page_folio(cc->rpages[i + 1])->index, cic); + fio.compressed_page = cc->cpages[i]; + + fio.old_blkaddr = data_blkaddr(dn.inode, dn.node_folio, + dn.ofs_in_node + i + 1); + + /* wait for GCed page writeback via META_MAPPING */ + f2fs_wait_on_block_writeback(inode, fio.old_blkaddr); + } + + set_cluster_writeback(cc); + + for (i = 0; i < cc->cluster_size; i++) + cic->rpages[i] = cc->rpages[i]; + + for (i = 0; i < cc->cluster_size; i++, dn.ofs_in_node++) { + block_t blkaddr; + + blkaddr = f2fs_data_blkaddr(&dn); + fio.page = cc->rpages[i]; + fio.old_blkaddr = blkaddr; + + /* cluster header */ + if (i == 0) { + if (blkaddr == COMPRESS_ADDR) + fio.compr_blocks++; + if (__is_valid_data_blkaddr(blkaddr)) + f2fs_invalidate_blocks(sbi, blkaddr, 1); + f2fs_update_data_blkaddr(&dn, COMPRESS_ADDR); + goto unlock_continue; + } + + if (fio.compr_blocks && __is_valid_data_blkaddr(blkaddr)) + fio.compr_blocks++; + + if (i > cc->valid_nr_cpages) { + if (__is_valid_data_blkaddr(blkaddr)) { + f2fs_invalidate_blocks(sbi, blkaddr, 1); + f2fs_update_data_blkaddr(&dn, NEW_ADDR); + } + goto unlock_continue; + } + + f2fs_bug_on(fio.sbi, blkaddr == NULL_ADDR); + + fio.compressed_page = cc->cpages[i - 1]; + + cc->cpages[i - 1] = NULL; + fio.submitted = 0; + f2fs_outplace_write_data(&dn, &fio); + if (unlikely(!fio.submitted)) { + cancel_cluster_writeback(cc, cic, i); + *submitted = 0; + goto out_free_page_array; + } + (*submitted)++; +unlock_continue: + inode_dec_dirty_pages(cc->inode); + folio_unlock(fio.folio); + } + + if (fio.compr_blocks) + f2fs_i_compr_blocks_update(inode, fio.compr_blocks - 1, false); + f2fs_i_compr_blocks_update(inode, cc->valid_nr_cpages, true); + add_compr_block_stat(inode, cc->valid_nr_cpages); + + set_inode_flag(cc->inode, FI_APPEND_WRITE); + + f2fs_put_dnode(&dn); + if (quota_inode) + f2fs_up_read_trace(&sbi->node_write, &lc); + else + f2fs_unlock_op(sbi, &lc); + + spin_lock(&fi->i_size_lock); + if (fi->last_disk_size < psize) + fi->last_disk_size = psize; + spin_unlock(&fi->i_size_lock); + + f2fs_put_rpages(cc); + page_array_free(sbi, cc->cpages, cc->nr_cpages); + cc->cpages = NULL; + f2fs_destroy_compress_ctx(cc, false); + return 0; + +out_free_page_array: + page_array_free(sbi, cic->rpages, cc->cluster_size); +out_put_cic: + kmem_cache_free(cic_entry_slab, cic); +out_put_dnode: + f2fs_put_dnode(&dn); +out_unlock_op: + if (quota_inode) + f2fs_up_read_trace(&sbi->node_write, &lc); + else + f2fs_unlock_op(sbi, &lc); +out_free: + for (i = 0; i < cc->valid_nr_cpages; i++) { + f2fs_compress_free_page(cc->cpages[i]); + cc->cpages[i] = NULL; + } + page_array_free(sbi, cc->cpages, cc->nr_cpages); + cc->cpages = NULL; + return -EAGAIN; +} + +void f2fs_compress_write_end_io(struct bio *bio, struct folio *folio) +{ + struct page *page = &folio->page; + struct f2fs_sb_info *sbi = bio->bi_private; + struct compress_io_ctx *cic = folio->private; + enum count_type type = WB_DATA_TYPE(folio, true); + int i; + + if (unlikely(bio->bi_status != BLK_STS_OK)) + mapping_set_error(cic->inode->i_mapping, -EIO); + + f2fs_compress_free_page(page); + + if (atomic_dec_return(&cic->pending_pages)) { + dec_page_count(sbi, type); + return; + } + + for (i = 0; i < cic->nr_rpages; i++) { + WARN_ON(!cic->rpages[i]); + clear_page_private_gcing(cic->rpages[i]); + end_page_writeback(cic->rpages[i]); + } + + page_array_free(sbi, cic->rpages, cic->nr_rpages); + kmem_cache_free(cic_entry_slab, cic); + + /* + * Make sure dec_page_count() is the last access to sbi. + * Once it drops the F2FS_WB_CP_DATA counter to zero, the + * unmount thread can proceed to destroy sbi and + * sbi->page_array_slab. + */ + dec_page_count(sbi, type); +} + +static int f2fs_write_raw_pages(struct compress_ctx *cc, + int *submitted_p, + struct writeback_control *wbc, + enum iostat_type io_type) +{ + struct address_space *mapping = cc->inode->i_mapping; + struct f2fs_sb_info *sbi = F2FS_M_SB(mapping); + struct f2fs_lock_context lc; + int submitted, compr_blocks, i; + int ret = 0; + + compr_blocks = f2fs_compressed_blocks(cc); + + for (i = 0; i < cc->cluster_size; i++) { + if (!cc->rpages[i]) + continue; + + redirty_page_for_writepage(wbc, cc->rpages[i]); + unlock_page(cc->rpages[i]); + } + + if (compr_blocks < 0) + return compr_blocks; + + /* overwrite compressed cluster w/ normal cluster */ + if (compr_blocks > 0) + f2fs_lock_op(sbi, &lc); + + for (i = 0; i < cc->cluster_size; i++) { + struct folio *folio; + + if (!cc->rpages[i]) + continue; + folio = page_folio(cc->rpages[i]); +retry_write: + folio_lock(folio); + + if (folio->mapping != mapping) { +continue_unlock: + folio_unlock(folio); + continue; + } + + if (!folio_test_dirty(folio)) + goto continue_unlock; + + if (folio_test_writeback(folio)) { + if (wbc->sync_mode == WB_SYNC_NONE) + goto continue_unlock; + f2fs_folio_wait_writeback(folio, DATA, true, true); + } + + if (!folio_clear_dirty_for_io(folio)) + goto continue_unlock; + + submitted = 0; + ret = f2fs_write_single_data_page(folio, &submitted, + NULL, NULL, wbc, io_type, + compr_blocks, false); + if (ret) { + if (ret == 1) { + ret = 0; + } else if (ret == -EAGAIN) { + ret = 0; + /* + * for quota file, just redirty left pages to + * avoid deadlock caused by cluster update race + * from foreground operation. + */ + if (IS_NOQUOTA(cc->inode)) + goto out; + f2fs_schedule_timeout(DEFAULT_SCHEDULE_TIMEOUT); + goto retry_write; + } + goto out; + } + + *submitted_p += submitted; + } + +out: + if (compr_blocks > 0) + f2fs_unlock_op(sbi, &lc); + + f2fs_balance_fs(sbi, true); + return ret; +} + +int f2fs_write_multi_pages(struct compress_ctx *cc, + int *submitted, + struct writeback_control *wbc, + enum iostat_type io_type) +{ + int err; + + *submitted = 0; + if (cluster_may_compress(cc)) { + err = f2fs_compress_pages(cc); + if (err == -EAGAIN) { + add_compr_block_stat(cc->inode, cc->cluster_size); + goto write; + } else if (err) { + f2fs_put_rpages_wbc(cc, wbc, true, true); + goto destroy_out; + } + + err = f2fs_write_compressed_pages(cc, submitted, + wbc, io_type); + if (!err) + return 0; + f2fs_bug_on(F2FS_I_SB(cc->inode), err != -EAGAIN); + } +write: + f2fs_bug_on(F2FS_I_SB(cc->inode), *submitted); + + err = f2fs_write_raw_pages(cc, submitted, wbc, io_type); + f2fs_put_rpages_wbc(cc, wbc, false, false); +destroy_out: + f2fs_destroy_compress_ctx(cc, false); + return err; +} + +static inline bool allow_memalloc_for_decomp(struct f2fs_sb_info *sbi, + bool pre_alloc) +{ + return pre_alloc ^ f2fs_low_mem_mode(sbi); +} + +static int f2fs_prepare_decomp_mem(struct decompress_io_ctx *dic, + bool pre_alloc) +{ + const struct f2fs_compress_ops *cops = f2fs_cops[dic->compress_algorithm]; + int i; + + if (!allow_memalloc_for_decomp(dic->sbi, pre_alloc)) + return 0; + + dic->tpages = page_array_alloc(dic->sbi, dic->cluster_size); + if (!dic->tpages) + return -ENOMEM; + + for (i = 0; i < dic->cluster_size; i++) { + if (dic->rpages[i]) { + dic->tpages[i] = dic->rpages[i]; + continue; + } + + dic->tpages[i] = f2fs_compress_alloc_page(); + } + + dic->rbuf = f2fs_vmap(dic->tpages, dic->cluster_size); + if (!dic->rbuf) + return -ENOMEM; + + dic->cbuf = f2fs_vmap(dic->cpages, dic->nr_cpages); + if (!dic->cbuf) + return -ENOMEM; + + if (cops->init_decompress_ctx) + return cops->init_decompress_ctx(dic); + + return 0; +} + +static void f2fs_release_decomp_mem(struct decompress_io_ctx *dic, + bool bypass_destroy_callback, bool pre_alloc) +{ + const struct f2fs_compress_ops *cops = f2fs_cops[dic->compress_algorithm]; + + if (!allow_memalloc_for_decomp(dic->sbi, pre_alloc)) + return; + + if (!bypass_destroy_callback && cops->destroy_decompress_ctx) + cops->destroy_decompress_ctx(dic); + + if (dic->cbuf) + vm_unmap_ram(dic->cbuf, dic->nr_cpages); + + if (dic->rbuf) + vm_unmap_ram(dic->rbuf, dic->cluster_size); +} + +static void f2fs_free_dic(struct decompress_io_ctx *dic, + bool bypass_destroy_callback); + +struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc) +{ + struct decompress_io_ctx *dic; + pgoff_t start_idx = start_idx_of_cluster(cc); + struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode); + int i, ret; + + dic = f2fs_kmem_cache_alloc(dic_entry_slab, GFP_F2FS_ZERO, false, sbi); + if (!dic) + return ERR_PTR(-ENOMEM); + + dic->rpages = page_array_alloc(sbi, cc->cluster_size); + if (!dic->rpages) { + kmem_cache_free(dic_entry_slab, dic); + return ERR_PTR(-ENOMEM); + } + + dic->magic = F2FS_COMPRESSED_PAGE_MAGIC; + dic->inode = cc->inode; + dic->sbi = sbi; + dic->compress_algorithm = F2FS_I(cc->inode)->i_compress_algorithm; + atomic_set(&dic->remaining_pages, cc->nr_cpages); + dic->cluster_idx = cc->cluster_idx; + dic->cluster_size = cc->cluster_size; + dic->log_cluster_size = cc->log_cluster_size; + dic->nr_cpages = cc->nr_cpages; + refcount_set(&dic->refcnt, 1); + dic->failed = false; + dic->vi = cc->vi; + + for (i = 0; i < dic->cluster_size; i++) + dic->rpages[i] = cc->rpages[i]; + dic->nr_rpages = cc->cluster_size; + + dic->cpages = page_array_alloc(sbi, dic->nr_cpages); + if (!dic->cpages) { + ret = -ENOMEM; + goto out_free; + } + + for (i = 0; i < dic->nr_cpages; i++) { + struct page *page; + + page = f2fs_compress_alloc_page(); + f2fs_set_compressed_page(page, cc->inode, + start_idx + i + 1, dic); + dic->cpages[i] = page; + } + + ret = f2fs_prepare_decomp_mem(dic, true); + if (ret) + goto out_free; + + return dic; + +out_free: + f2fs_free_dic(dic, true); + return ERR_PTR(ret); +} + +static void f2fs_free_dic(struct decompress_io_ctx *dic, + bool bypass_destroy_callback) +{ + int i; + /* use sbi in dic to avoid UFA of dic->inode*/ + struct f2fs_sb_info *sbi = dic->sbi; + + f2fs_release_decomp_mem(dic, bypass_destroy_callback, true); + + if (dic->tpages) { + for (i = 0; i < dic->cluster_size; i++) { + if (dic->rpages[i]) + continue; + if (!dic->tpages[i]) + continue; + f2fs_compress_free_page(dic->tpages[i]); + } + page_array_free(sbi, dic->tpages, dic->cluster_size); + } + + if (dic->cpages) { + for (i = 0; i < dic->nr_cpages; i++) { + if (!dic->cpages[i]) + continue; + f2fs_compress_free_page(dic->cpages[i]); + } + page_array_free(sbi, dic->cpages, dic->nr_cpages); + } + + page_array_free(sbi, dic->rpages, dic->nr_rpages); + kmem_cache_free(dic_entry_slab, dic); +} + +static void f2fs_late_free_dic(struct work_struct *work) +{ + struct decompress_io_ctx *dic = + container_of(work, struct decompress_io_ctx, free_work); + + f2fs_free_dic(dic, false); +} + +static void f2fs_put_dic(struct decompress_io_ctx *dic, bool in_task) +{ + if (refcount_dec_and_test(&dic->refcnt)) { + if (in_task) { + f2fs_free_dic(dic, false); + } else { + INIT_WORK(&dic->free_work, f2fs_late_free_dic); + queue_work(dic->sbi->wq, &dic->free_work); + } + } +} + +static void f2fs_verify_cluster(struct work_struct *work) +{ + struct decompress_io_ctx *dic = + container_of(work, struct decompress_io_ctx, verity_work); + int i; + + /* Verify, update, and unlock the decompressed pages. */ + for (i = 0; i < dic->cluster_size; i++) { + struct page *rpage = dic->rpages[i]; + struct folio *rfolio; + + if (!rpage) + continue; + rfolio = page_folio(rpage); + if (fsverity_verify_folio(dic->vi, rfolio)) + folio_mark_uptodate(rfolio); + folio_unlock(rfolio); + } + + f2fs_put_dic(dic, true); +} + +/* + * This is called when a compressed cluster has been decompressed + * (or failed to be read and/or decompressed). + */ +void f2fs_decompress_end_io(struct decompress_io_ctx *dic, bool failed, + bool in_task) +{ + int i; + + if (IS_ENABLED(CONFIG_FS_VERITY) && !failed && dic->vi) { + /* + * Note that to avoid deadlocks, the verity work can't be done + * on the decompression workqueue. This is because verifying + * the data pages can involve reading metadata pages from the + * file, and these metadata pages may be compressed. + */ + INIT_WORK(&dic->verity_work, f2fs_verify_cluster); + fsverity_enqueue_verify_work(&dic->verity_work); + return; + } + + /* Update and unlock the cluster's pagecache pages. */ + for (i = 0; i < dic->cluster_size; i++) { + struct page *rpage = dic->rpages[i]; + + if (!rpage) + continue; + + if (failed) + ClearPageUptodate(rpage); + else + SetPageUptodate(rpage); + unlock_page(rpage); + } + + /* + * Release the reference to the decompress_io_ctx that was being held + * for I/O completion. + */ + f2fs_put_dic(dic, in_task); +} + +/* + * Put a reference to a compressed folio's decompress_io_ctx. + * + * This is called when the folio is no longer needed and can be freed. + */ +void f2fs_put_folio_dic(struct folio *folio, bool in_task) +{ + struct decompress_io_ctx *dic = folio->private; + + f2fs_put_dic(dic, in_task); +} + +/* + * check whether cluster blocks are contiguous, and add extent cache entry + * only if cluster blocks are logically and physically contiguous. + */ +unsigned int f2fs_cluster_blocks_are_contiguous(struct dnode_of_data *dn, + unsigned int ofs_in_node) +{ + bool compressed = data_blkaddr(dn->inode, dn->node_folio, + ofs_in_node) == COMPRESS_ADDR; + int i = compressed ? 1 : 0; + block_t first_blkaddr = data_blkaddr(dn->inode, dn->node_folio, + ofs_in_node + i); + + for (i += 1; i < F2FS_I(dn->inode)->i_cluster_size; i++) { + block_t blkaddr = data_blkaddr(dn->inode, dn->node_folio, + ofs_in_node + i); + + if (!__is_valid_data_blkaddr(blkaddr)) + break; + if (first_blkaddr + i - (compressed ? 1 : 0) != blkaddr) + return 0; + } + + return compressed ? i - 1 : i; +} + +const struct address_space_operations f2fs_compress_aops = { + .release_folio = f2fs_release_folio, + .invalidate_folio = f2fs_invalidate_folio, + .migrate_folio = filemap_migrate_folio, +}; + +struct address_space *COMPRESS_MAPPING(struct f2fs_sb_info *sbi) +{ + return sbi->compress_inode->i_mapping; +} + +void f2fs_invalidate_compress_pages_range(struct f2fs_sb_info *sbi, + block_t blkaddr, unsigned int len) +{ + if (!sbi->compress_inode) + return; + invalidate_mapping_pages(COMPRESS_MAPPING(sbi), blkaddr, blkaddr + len - 1); +} + +static void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi, + struct folio *folio, nid_t ino, block_t blkaddr) +{ + struct folio *cfolio; + int ret; + + if (!test_opt(sbi, COMPRESS_CACHE)) + return; + + if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE_READ)) + return; + + if (!f2fs_available_free_memory(sbi, COMPRESS_PAGE)) + return; + + cfolio = filemap_get_folio(COMPRESS_MAPPING(sbi), blkaddr); + if (!IS_ERR(cfolio)) { + f2fs_folio_put(cfolio, false); + return; + } + + cfolio = filemap_alloc_folio(__GFP_NOWARN | __GFP_IO, 0, NULL); + if (!cfolio) + return; + + ret = filemap_add_folio(COMPRESS_MAPPING(sbi), cfolio, + blkaddr, GFP_NOFS); + if (ret) { + f2fs_folio_put(cfolio, false); + return; + } + + folio_set_f2fs_data(cfolio, ino); + + memcpy(folio_address(cfolio), folio_address(folio), PAGE_SIZE); + folio_mark_uptodate(cfolio); + f2fs_folio_put(cfolio, true); +} + +bool f2fs_load_compressed_folio(struct f2fs_sb_info *sbi, struct folio *folio, + block_t blkaddr) +{ + struct folio *cfolio; + bool hitted = false; + + if (!test_opt(sbi, COMPRESS_CACHE)) + return false; + + cfolio = f2fs_filemap_get_folio(COMPRESS_MAPPING(sbi), + blkaddr, FGP_LOCK | FGP_NOWAIT, GFP_NOFS); + if (!IS_ERR(cfolio)) { + if (folio_test_uptodate(cfolio)) { + atomic_inc(&sbi->compress_page_hit); + memcpy(folio_address(folio), + folio_address(cfolio), folio_size(folio)); + hitted = true; + } + f2fs_folio_put(cfolio, true); + } + + return hitted; +} + +void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi, nid_t ino) +{ + struct address_space *mapping = COMPRESS_MAPPING(sbi); + struct folio_batch fbatch; + pgoff_t index = 0; + pgoff_t end = MAX_BLKADDR(sbi); + + if (!mapping->nrpages) + return; + + folio_batch_init(&fbatch); + + do { + unsigned int nr, i; + + nr = filemap_get_folios(mapping, &index, end - 1, &fbatch); + if (!nr) + break; + + for (i = 0; i < nr; i++) { + struct folio *folio = fbatch.folios[i]; + + folio_lock(folio); + if (folio->mapping != mapping) { + folio_unlock(folio); + continue; + } + + if (ino != folio_get_f2fs_data(folio)) { + folio_unlock(folio); + continue; + } + + generic_error_remove_folio(mapping, folio); + folio_unlock(folio); + } + folio_batch_release(&fbatch); + cond_resched(); + } while (index < end); +} + +int f2fs_init_compress_inode(struct f2fs_sb_info *sbi) +{ + struct inode *inode; + + if (!test_opt(sbi, COMPRESS_CACHE)) + return 0; + + inode = f2fs_iget(sbi->sb, F2FS_COMPRESS_INO(sbi)); + if (IS_ERR(inode)) + return PTR_ERR(inode); + sbi->compress_inode = inode; + + sbi->compress_percent = COMPRESS_PERCENT; + sbi->compress_watermark = COMPRESS_WATERMARK; + + atomic_set(&sbi->compress_page_hit, 0); + + return 0; +} + +void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi) +{ + if (!sbi->compress_inode) + return; + iput(sbi->compress_inode); + sbi->compress_inode = NULL; +} + +int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi) +{ + dev_t dev = sbi->sb->s_bdev->bd_dev; + char slab_name[35]; + + if (!f2fs_sb_has_compression(sbi)) + return 0; + + sprintf(slab_name, "f2fs_page_array_entry-%u:%u", MAJOR(dev), MINOR(dev)); + + sbi->page_array_slab_size = sizeof(struct page *) << + F2FS_OPTION(sbi).compress_log_size; + + sbi->page_array_slab = f2fs_kmem_cache_create(slab_name, + sbi->page_array_slab_size); + return sbi->page_array_slab ? 0 : -ENOMEM; +} + +void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi) +{ + kmem_cache_destroy(sbi->page_array_slab); +} + +int __init f2fs_init_compress_cache(void) +{ + cic_entry_slab = f2fs_kmem_cache_create("f2fs_cic_entry", + sizeof(struct compress_io_ctx)); + if (!cic_entry_slab) + return -ENOMEM; + dic_entry_slab = f2fs_kmem_cache_create("f2fs_dic_entry", + sizeof(struct decompress_io_ctx)); + if (!dic_entry_slab) + goto free_cic; + return 0; +free_cic: + kmem_cache_destroy(cic_entry_slab); + return -ENOMEM; +} + +void f2fs_destroy_compress_cache(void) +{ + kmem_cache_destroy(dic_entry_slab); + kmem_cache_destroy(cic_entry_slab); +} diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c new file mode 100644 index 000000000..dad5c2bd5 --- /dev/null +++ b/fs/f2fs/data.c @@ -0,0 +1,4585 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/f2fs/data.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "f2fs.h" +#include "node.h" +#include "segment.h" +#include "iostat.h" +#include + +#define NUM_PREALLOC_POST_READ_CTXS 128 + +static struct kmem_cache *bio_post_read_ctx_cache; +static struct kmem_cache *bio_entry_slab; +static struct kmem_cache *ffs_entry_slab; +static mempool_t *bio_post_read_ctx_pool; +static struct bio_set f2fs_bioset; + +struct f2fs_folio_state { + spinlock_t state_lock; + unsigned int read_pages_pending; +}; + +struct f2fs_bio { + struct work_struct work; + struct bio bio; +}; + +#define F2FS_BIO_POOL_SIZE NR_CURSEG_TYPE + +int __init f2fs_init_bioset(void) +{ + return bioset_init(&f2fs_bioset, F2FS_BIO_POOL_SIZE, + offsetof(struct f2fs_bio, bio), BIOSET_NEED_BVECS); +} + +void f2fs_destroy_bioset(void) +{ + bioset_exit(&f2fs_bioset); +} + +bool f2fs_is_cp_guaranteed(const struct folio *folio) +{ + struct address_space *mapping = folio->mapping; + struct inode *inode; + struct f2fs_sb_info *sbi; + + inode = mapping->host; + sbi = F2FS_I_SB(inode); + + if (inode->i_ino == F2FS_META_INO(sbi) || + inode->i_ino == F2FS_NODE_INO(sbi) || + S_ISDIR(inode->i_mode)) + return true; + + if ((S_ISREG(inode->i_mode) && IS_NOQUOTA(inode)) || + folio_test_f2fs_gcing(folio)) + return true; + return false; +} + +static enum count_type __read_io_type(struct folio *folio) +{ + struct address_space *mapping = folio->mapping; + + if (mapping) { + struct inode *inode = mapping->host; + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + + if (inode->i_ino == F2FS_META_INO(sbi)) + return F2FS_RD_META; + + if (inode->i_ino == F2FS_NODE_INO(sbi)) + return F2FS_RD_NODE; + } + return F2FS_RD_DATA; +} + +/* postprocessing steps for read bios */ +enum bio_post_read_step { +#ifdef CONFIG_F2FS_FS_COMPRESSION + STEP_DECOMPRESS = BIT(1), +#else + STEP_DECOMPRESS = 0, /* compile out the decompression-related code */ +#endif +#ifdef CONFIG_FS_VERITY + STEP_VERITY = BIT(2), +#else + STEP_VERITY = 0, /* compile out the verity-related code */ +#endif +}; + +struct bio_post_read_ctx { + struct bio *bio; + struct f2fs_sb_info *sbi; + struct fsverity_info *vi; + struct work_struct work; + unsigned int enabled_steps; + /* + * decompression_attempted keeps track of whether + * f2fs_end_read_compressed_page() has been called on the pages in the + * bio that belong to a compressed cluster yet. + */ + bool decompression_attempted; + block_t fs_blkaddr; +}; + +/* + * Update and unlock a bio's pages, and free the bio. + * + * This marks pages up-to-date only if there was no error in the bio (I/O error, + * decryption error, or verity error), as indicated by bio->bi_status. + * + * "Compressed pages" (pagecache pages backed by a compressed cluster on-disk) + * aren't marked up-to-date here, as decompression is done on a per-compression- + * cluster basis rather than a per-bio basis. Instead, we only must do two + * things for each compressed page here: call f2fs_end_read_compressed_page() + * with failed=true if an error occurred before it would have normally gotten + * called (i.e., I/O error or decryption error, but *not* verity error), and + * release the bio's reference to the decompress_io_ctx of the page's cluster. + */ +static void f2fs_finish_read_bio(struct bio *bio, bool in_task) +{ + struct folio_iter fi; + struct bio_post_read_ctx *ctx = bio->bi_private; + unsigned long flags; + + bio_for_each_folio_all(fi, bio) { + struct folio *folio = fi.folio; + unsigned nr_pages = fi.length >> PAGE_SHIFT; + bool finished = true; + + if (!folio_test_large(folio) && + f2fs_is_compressed_page(folio)) { + if (ctx && !ctx->decompression_attempted) + f2fs_end_read_compressed_page(folio, true, 0, + in_task); + f2fs_put_folio_dic(folio, in_task); + continue; + } + + if (folio_test_large(folio)) { + struct f2fs_folio_state *ffs = folio->private; + + spin_lock_irqsave(&ffs->state_lock, flags); + ffs->read_pages_pending -= nr_pages; + finished = !ffs->read_pages_pending; + spin_unlock_irqrestore(&ffs->state_lock, flags); + } + + while (nr_pages--) + dec_page_count(F2FS_F_SB(folio), __read_io_type(folio)); + + if (bio->bi_status == BLK_STS_OK && + F2FS_F_SB(folio)->node_inode && is_node_folio(folio) && + f2fs_sanity_check_node_footer(F2FS_F_SB(folio), + folio, folio->index, NODE_TYPE_REGULAR, true)) + bio->bi_status = BLK_STS_IOERR; + + if (finished) + folio_end_read(folio, bio->bi_status == BLK_STS_OK); + } + + if (ctx) + mempool_free(ctx, bio_post_read_ctx_pool); + bio_put(bio); +} + +static void f2fs_verify_bio(struct work_struct *work) +{ + struct bio_post_read_ctx *ctx = + container_of(work, struct bio_post_read_ctx, work); + struct bio *bio = ctx->bio; + bool may_have_compressed_pages = (ctx->enabled_steps & STEP_DECOMPRESS); + struct fsverity_info *vi = ctx->vi; + + /* + * fsverity_verify_bio() may call readahead() again, and while verity + * will be disabled for this, decryption and/or decompression may still + * be needed, resulting in another bio_post_read_ctx being allocated. + * So to prevent deadlocks we need to release the current ctx to the + * mempool first. This assumes that verity is the last post-read step. + */ + mempool_free(ctx, bio_post_read_ctx_pool); + bio->bi_private = NULL; + + /* + * Verify the bio's pages with fs-verity. Exclude compressed pages, + * as those were handled separately by f2fs_end_read_compressed_page(). + */ + if (may_have_compressed_pages) { + struct folio_iter fi; + + bio_for_each_folio_all(fi, bio) { + struct folio *folio = fi.folio; + + if (!f2fs_is_compressed_page(folio) && + !fsverity_verify_folio(vi, folio)) { + bio->bi_status = BLK_STS_IOERR; + break; + } + } + } else { + fsverity_verify_bio(vi, bio); + } + + f2fs_finish_read_bio(bio, true); +} + +/* + * If the bio's data needs to be verified with fs-verity, then enqueue the + * verity work for the bio. Otherwise finish the bio now. + * + * Note that to avoid deadlocks, the verity work can't be done on the + * decryption/decompression workqueue. This is because verifying the data pages + * can involve reading verity metadata pages from the file, and these verity + * metadata pages may be encrypted and/or compressed. + */ +static void f2fs_verify_and_finish_bio(struct bio *bio, bool in_task) +{ + struct bio_post_read_ctx *ctx = bio->bi_private; + + if (ctx && (ctx->enabled_steps & STEP_VERITY)) { + INIT_WORK(&ctx->work, f2fs_verify_bio); + fsverity_enqueue_verify_work(&ctx->work); + } else { + f2fs_finish_read_bio(bio, in_task); + } +} + +/* + * Handle STEP_DECOMPRESS by decompressing any compressed clusters whose last + * remaining page was read by @ctx->bio. + * + * Note that a bio may span clusters (even a mix of compressed and uncompressed + * clusters) or be for just part of a cluster. STEP_DECOMPRESS just indicates + * that the bio includes at least one compressed page. The actual decompression + * is done on a per-cluster basis, not a per-bio basis. + */ +static void f2fs_handle_step_decompress(struct bio_post_read_ctx *ctx, + bool in_task) +{ + struct folio_iter fi; + bool all_compressed = true; + block_t blkaddr = ctx->fs_blkaddr; + + bio_for_each_folio_all(fi, ctx->bio) { + struct folio *folio = fi.folio; + + if (f2fs_is_compressed_page(folio)) + f2fs_end_read_compressed_page(folio, false, blkaddr, + in_task); + else + all_compressed = false; + + blkaddr++; + } + + ctx->decompression_attempted = true; + + /* + * Optimization: if all the bio's pages are compressed, then scheduling + * the per-bio verity work is unnecessary, as verity will be fully + * handled at the compression cluster level. + */ + if (all_compressed) + ctx->enabled_steps &= ~STEP_VERITY; +} + +static void f2fs_post_read_work(struct work_struct *work) +{ + struct bio_post_read_ctx *ctx = + container_of(work, struct bio_post_read_ctx, work); + struct bio *bio = ctx->bio; + + if (ctx->enabled_steps & STEP_DECOMPRESS) + f2fs_handle_step_decompress(ctx, true); + + f2fs_verify_and_finish_bio(bio, true); +} + +static void f2fs_read_end_io(struct bio *bio) +{ + struct f2fs_sb_info *sbi = F2FS_F_SB(bio_first_folio_all(bio)); + struct bio_post_read_ctx *ctx; + bool intask = in_task() && !irqs_disabled(); + + iostat_update_and_unbind_ctx(bio); + ctx = bio->bi_private; + + if (time_to_inject(sbi, FAULT_READ_IO)) + bio->bi_status = BLK_STS_IOERR; + + if (bio->bi_status != BLK_STS_OK) { + f2fs_finish_read_bio(bio, intask); + return; + } + + if (ctx && (ctx->enabled_steps & STEP_DECOMPRESS)) { + if (!f2fs_low_mem_mode(sbi)) { + /* Decompress inline. */ + f2fs_handle_step_decompress(ctx, intask); + } else { + INIT_WORK(&ctx->work, f2fs_post_read_work); + queue_work(ctx->sbi->wq, &ctx->work); + return; + } + } + + f2fs_verify_and_finish_bio(bio, intask); +} + +static void f2fs_write_end_bio(struct bio *bio) +{ + struct f2fs_sb_info *sbi = bio->bi_private; + struct folio_iter fi; + + if (time_to_inject(sbi, FAULT_WRITE_IO)) + bio->bi_status = BLK_STS_IOERR; + + bio_for_each_folio_all(fi, bio) { + struct folio *folio = fi.folio; + enum count_type type; + +#ifdef CONFIG_F2FS_FS_COMPRESSION + if (f2fs_is_compressed_page(folio)) { + f2fs_compress_write_end_io(bio, folio); + continue; + } +#endif + + type = WB_DATA_TYPE(folio, false); + + if (unlikely(bio->bi_status != BLK_STS_OK)) { + mapping_set_error(folio->mapping, -EIO); + if (type == F2FS_WB_CP_DATA) { + f2fs_stop_checkpoint(sbi, true, + STOP_CP_REASON_WRITE_FAIL); + } + } + + if (is_node_folio(folio)) { + f2fs_sanity_check_node_footer(sbi, folio, + folio->index, NODE_TYPE_REGULAR, true); + f2fs_bug_on(sbi, folio->index != nid_of_node(folio)); + } + if (f2fs_in_warm_node_list(folio)) + f2fs_del_fsync_node_entry(sbi, folio); + + dec_page_count(sbi, type); + + /* + * we should access sbi before folio_end_writeback() to + * avoid racing w/ kill_f2fs_super() + */ + if (type == F2FS_WB_CP_DATA && !get_pages(sbi, type) && + wq_has_sleeper(&sbi->cp_wait)) + wake_up(&sbi->cp_wait); + + folio_clear_f2fs_gcing(folio); + folio_end_writeback(folio); + } + + bio_put(bio); +} + +static void f2fs_write_end_io(struct bio *bio) +{ + iostat_update_and_unbind_ctx(bio); + + f2fs_write_end_bio(bio); +} + +#ifdef CONFIG_BLK_DEV_ZONED +static void f2fs_zone_write_end_io(struct bio *bio) +{ + struct f2fs_bio_info *io = (struct f2fs_bio_info *)bio->bi_private; + + bio->bi_private = io->bi_private; + complete(&io->zone_wait); + f2fs_write_end_io(bio); +} +#endif + +struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi, + block_t blk_addr, sector_t *sector) +{ + struct block_device *bdev = sbi->sb->s_bdev; + int i; + + if (f2fs_is_multi_device(sbi)) { + for (i = 0; i < sbi->s_ndevs; i++) { + if (FDEV(i).start_blk <= blk_addr && + FDEV(i).end_blk >= blk_addr) { + blk_addr -= FDEV(i).start_blk; + bdev = FDEV(i).bdev; + break; + } + } + } + + if (sector) + *sector = SECTOR_FROM_BLOCK(blk_addr); + return bdev; +} + +int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr) +{ + int i; + + if (!f2fs_is_multi_device(sbi)) + return 0; + + for (i = 0; i < sbi->s_ndevs; i++) + if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr) + return i; + return 0; +} + +static blk_opf_t f2fs_io_flags(struct f2fs_io_info *fio) +{ + unsigned int temp_mask = GENMASK(NR_TEMP_TYPE - 1, 0); + unsigned int fua_flag, meta_flag, io_flag; + blk_opf_t op_flags = 0; + + if (fio->op != REQ_OP_WRITE) + return 0; + if (fio->type == DATA) + io_flag = fio->sbi->data_io_flag; + else if (fio->type == NODE) + io_flag = fio->sbi->node_io_flag; + else + return 0; + + fua_flag = io_flag & temp_mask; + meta_flag = (io_flag >> NR_TEMP_TYPE) & temp_mask; + + /* + * data/node io flag bits per temp: + * REQ_META | REQ_FUA | + * 5 | 4 | 3 | 2 | 1 | 0 | + * Cold | Warm | Hot | Cold | Warm | Hot | + */ + if (BIT(fio->temp) & meta_flag) + op_flags |= REQ_META; + if (BIT(fio->temp) & fua_flag) + op_flags |= REQ_FUA; + + if (fio->type == DATA && + F2FS_I(fio->folio->mapping->host)->ioprio_hint == F2FS_IOPRIO_WRITE) + op_flags |= REQ_PRIO; + + return op_flags; +} + +static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages) +{ + struct f2fs_sb_info *sbi = fio->sbi; + struct block_device *bdev; + sector_t sector; + struct bio *bio; + + bdev = f2fs_target_device(sbi, fio->new_blkaddr, §or); + bio = bio_alloc_bioset(bdev, npages, + fio->op | fio->op_flags | f2fs_io_flags(fio), + GFP_NOIO, &f2fs_bioset); + bio->bi_iter.bi_sector = sector; + if (is_read_io(fio->op)) { + bio->bi_end_io = f2fs_read_end_io; + bio->bi_private = NULL; + } else { + bio->bi_end_io = f2fs_write_end_io; + bio->bi_private = sbi; + bio->bi_write_hint = f2fs_io_type_to_rw_hint(sbi, + fio->type, fio->temp); + bio->bi_write_stream = f2fs_io_type_to_write_stream(bdev, fio->type, + fio->temp); + } + iostat_alloc_and_bind_ctx(sbi, bio, NULL); + + if (fio->io_wbc) + wbc_init_bio(fio->io_wbc, bio); + + return bio; +} + +static void f2fs_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, + pgoff_t first_idx, + const struct f2fs_io_info *fio, + gfp_t gfp_mask) +{ + /* + * The f2fs garbage collector sets ->encrypted_page when it wants to + * read/write raw data without encryption. + */ + if (!fio || !fio->encrypted_page) + fscrypt_set_bio_crypt_ctx(bio, inode, + (loff_t)first_idx << inode->i_blkbits, + gfp_mask); +} + +static bool f2fs_crypt_mergeable_bio(struct bio *bio, const struct inode *inode, + pgoff_t next_idx, + const struct f2fs_io_info *fio) +{ + /* + * The f2fs garbage collector sets ->encrypted_page when it wants to + * read/write raw data without encryption. + */ + if (fio && fio->encrypted_page) + return !bio_has_crypt_ctx(bio); + + return fscrypt_mergeable_bio(bio, inode, + (loff_t)next_idx << inode->i_blkbits); +} + +void f2fs_submit_read_bio(struct f2fs_sb_info *sbi, struct bio *bio, + enum page_type type) +{ + if (!bio) + return; + + WARN_ON_ONCE(!is_read_io(bio_op(bio))); + trace_f2fs_submit_read_bio(sbi->sb, type, bio); + + iostat_update_submit_ctx(bio, type); + blk_crypto_submit_bio(bio); +} + +static void f2fs_submit_write_bio(struct f2fs_sb_info *sbi, struct bio *bio, + enum page_type type) +{ + WARN_ON_ONCE(is_read_io(bio_op(bio))); + trace_f2fs_submit_write_bio(sbi->sb, type, bio); + iostat_update_submit_ctx(bio, type); + blk_crypto_submit_bio(bio); +} + +static void __submit_merged_bio(struct f2fs_bio_info *io) +{ + struct f2fs_io_info *fio = &io->fio; + + if (!io->bio) + return; + + if (is_read_io(fio->op)) { + trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio); + f2fs_submit_read_bio(io->sbi, io->bio, fio->type); + } else { + trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio); + f2fs_submit_write_bio(io->sbi, io->bio, fio->type); + } + io->bio = NULL; +} + +static bool __has_merged_page(struct bio *bio, struct inode *inode, + struct folio *folio, nid_t ino) +{ + struct folio_iter fi; + + if (!bio) + return false; + + if (!inode && !folio && !ino) + return true; + + bio_for_each_folio_all(fi, bio) { + struct folio *target = fi.folio; + + if (f2fs_is_compressed_page(target)) { + target = f2fs_compress_control_folio(target); + if (IS_ERR(target)) + continue; + } + + if (inode && inode == target->mapping->host) + return true; + if (folio && folio == target) + return true; + if (ino && ino == ino_of_node(target)) + return true; + } + + return false; +} + +int f2fs_init_write_merge_io(struct f2fs_sb_info *sbi) +{ + int i; + + for (i = 0; i < NR_PAGE_TYPE; i++) { + int n = (i == META) ? 1 : NR_TEMP_TYPE; + int j; + + sbi->write_io[i] = f2fs_kmalloc(sbi, + array_size(n, sizeof(struct f2fs_bio_info)), + GFP_KERNEL); + if (!sbi->write_io[i]) + return -ENOMEM; + + for (j = HOT; j < n; j++) { + struct f2fs_bio_info *io = &sbi->write_io[i][j]; + + init_f2fs_rwsem_trace(&io->io_rwsem, sbi, + LOCK_NAME_IO_RWSEM); + io->sbi = sbi; + io->bio = NULL; + io->last_block_in_bio = 0; + spin_lock_init(&io->io_lock); + INIT_LIST_HEAD(&io->io_list); + INIT_LIST_HEAD(&io->bio_list); + init_f2fs_rwsem(&io->bio_list_lock); +#ifdef CONFIG_BLK_DEV_ZONED + init_completion(&io->zone_wait); + io->zone_pending_bio = NULL; + io->bi_private = NULL; +#endif + } + } + + return 0; +} + +static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi, + enum page_type type, enum temp_type temp) +{ + enum page_type btype = PAGE_TYPE_OF_BIO(type); + struct f2fs_bio_info *io = sbi->write_io[btype] + temp; + struct f2fs_lock_context lc; + + f2fs_down_write_trace(&io->io_rwsem, &lc); + + if (!io->bio) + goto unlock_out; + + /* change META to META_FLUSH in the checkpoint procedure */ + if (type >= META_FLUSH) { + io->fio.type = META_FLUSH; + io->bio->bi_opf |= REQ_META | REQ_PRIO | REQ_SYNC; + if (!test_opt(sbi, NOBARRIER)) + io->bio->bi_opf |= REQ_PREFLUSH | REQ_FUA; + } + __submit_merged_bio(io); +unlock_out: + f2fs_up_write_trace(&io->io_rwsem, &lc); +} + +static void __submit_merged_write_cond(struct f2fs_sb_info *sbi, + struct inode *inode, struct folio *folio, + nid_t ino, enum page_type type, bool writeback) +{ + enum temp_type temp; + bool ret = true; + bool force = !inode && !folio && !ino; + + for (temp = HOT; temp < NR_TEMP_TYPE; temp++) { + if (!force) { + enum page_type btype = PAGE_TYPE_OF_BIO(type); + struct f2fs_bio_info *io = sbi->write_io[btype] + temp; + struct f2fs_lock_context lc; + + f2fs_down_read_trace(&io->io_rwsem, &lc); + ret = __has_merged_page(io->bio, inode, folio, ino); + f2fs_up_read_trace(&io->io_rwsem, &lc); + } + if (ret) { + __f2fs_submit_merged_write(sbi, type, temp); + /* + * For waitting writebck case, if the bio owned by the + * folio is already submitted, we do not need to submit + * other types of bios. + */ + if (writeback) + break; + } + + /* TODO: use HOT temp only for meta pages now. */ + if (type >= META) + break; + } +} + +void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type) +{ + __submit_merged_write_cond(sbi, NULL, NULL, 0, type, false); +} + +void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi, + struct inode *inode, struct folio *folio, + nid_t ino, enum page_type type) +{ + __submit_merged_write_cond(sbi, inode, folio, ino, type, false); +} + +void f2fs_submit_merged_write_folio(struct f2fs_sb_info *sbi, + struct folio *folio, enum page_type type) +{ + __submit_merged_write_cond(sbi, NULL, folio, 0, type, true); +} + +void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi) +{ + f2fs_submit_merged_write(sbi, DATA); + f2fs_submit_merged_write(sbi, NODE); + f2fs_submit_merged_write(sbi, META); +} + +/* + * Fill the locked page with data located in the block address. + * A caller needs to unlock the page on failure. + */ +int f2fs_submit_page_bio(struct f2fs_io_info *fio) +{ + struct bio *bio; + struct folio *fio_folio = fio->folio; + struct folio *data_folio = fio->encrypted_page ? + page_folio(fio->encrypted_page) : fio_folio; + + if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr, + fio->is_por ? META_POR : (__is_meta_io(fio) ? + META_GENERIC : DATA_GENERIC_ENHANCE))) + return -EFSCORRUPTED; + + trace_f2fs_submit_folio_bio(data_folio, fio); + + /* Allocate a new bio */ + bio = __bio_alloc(fio, 1); + + f2fs_set_bio_crypt_ctx(bio, fio_folio->mapping->host, + fio_folio->index, fio, GFP_NOIO); + bio_add_folio_nofail(bio, data_folio, folio_size(data_folio), 0); + + if (fio->io_wbc && !is_read_io(fio->op)) + wbc_account_cgroup_owner(fio->io_wbc, fio_folio, PAGE_SIZE); + + inc_page_count(fio->sbi, is_read_io(fio->op) ? + __read_io_type(data_folio) : WB_DATA_TYPE(fio->folio, false)); + + if (is_read_io(bio_op(bio))) + f2fs_submit_read_bio(fio->sbi, bio, fio->type); + else + f2fs_submit_write_bio(fio->sbi, bio, fio->type); + return 0; +} + +static bool page_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio, + block_t last_blkaddr, block_t cur_blkaddr) +{ + if (unlikely(sbi->max_io_bytes && + bio->bi_iter.bi_size >= sbi->max_io_bytes)) + return false; + if (last_blkaddr + 1 != cur_blkaddr) + return false; + return bio->bi_bdev == f2fs_target_device(sbi, cur_blkaddr, NULL); +} + +static bool io_type_is_mergeable(struct f2fs_bio_info *io, + struct f2fs_io_info *fio) +{ + blk_opf_t mask = ~(REQ_PREFLUSH | REQ_FUA); + + if (io->fio.op != fio->op) + return false; + return (io->fio.op_flags & mask) == (fio->op_flags & mask); +} + +static bool io_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio, + struct f2fs_bio_info *io, + struct f2fs_io_info *fio, + block_t last_blkaddr, + block_t cur_blkaddr) +{ + if (!page_is_mergeable(sbi, bio, last_blkaddr, cur_blkaddr)) + return false; + return io_type_is_mergeable(io, fio); +} + +static void add_bio_entry(struct f2fs_sb_info *sbi, struct bio *bio, + struct folio *folio, enum temp_type temp) +{ + struct f2fs_bio_info *io = sbi->write_io[DATA] + temp; + struct bio_entry *be; + + be = f2fs_kmem_cache_alloc(bio_entry_slab, GFP_NOFS, true, NULL); + be->bio = bio; + bio_get(bio); + + bio_add_folio_nofail(bio, folio, folio_size(folio), 0); + + f2fs_down_write(&io->bio_list_lock); + list_add_tail(&be->list, &io->bio_list); + f2fs_up_write(&io->bio_list_lock); +} + +static void del_bio_entry(struct bio_entry *be) +{ + list_del(&be->list); + kmem_cache_free(bio_entry_slab, be); +} + +static int add_ipu_page(struct f2fs_io_info *fio, struct bio **bio, + struct folio *folio) +{ + struct folio *fio_folio = fio->folio; + struct f2fs_sb_info *sbi = fio->sbi; + enum temp_type temp; + bool found = false; + int ret = -EAGAIN; + + for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) { + struct f2fs_bio_info *io = sbi->write_io[DATA] + temp; + struct list_head *head = &io->bio_list; + struct bio_entry *be; + + f2fs_down_write(&io->bio_list_lock); + list_for_each_entry(be, head, list) { + if (be->bio != *bio) + continue; + + found = true; + + f2fs_bug_on(sbi, !page_is_mergeable(sbi, *bio, + *fio->last_block, + fio->new_blkaddr)); + if (f2fs_crypt_mergeable_bio(*bio, + fio_folio->mapping->host, + fio_folio->index, fio) && + bio_add_folio(*bio, folio, folio_size(folio), 0)) { + ret = 0; + break; + } + + /* page can't be merged into bio; submit the bio */ + del_bio_entry(be); + f2fs_submit_write_bio(sbi, *bio, DATA); + break; + } + f2fs_up_write(&io->bio_list_lock); + } + + if (ret) { + bio_put(*bio); + *bio = NULL; + } + + return ret; +} + +void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi, + struct bio **bio, struct folio *folio) +{ + enum temp_type temp; + bool found = false; + struct bio *target = bio ? *bio : NULL; + + f2fs_bug_on(sbi, !target && !folio); + + for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) { + struct f2fs_bio_info *io = sbi->write_io[DATA] + temp; + struct list_head *head = &io->bio_list; + struct bio_entry *be; + + if (list_empty(head)) + continue; + + f2fs_down_read(&io->bio_list_lock); + list_for_each_entry(be, head, list) { + if (target) + found = (target == be->bio); + else + found = __has_merged_page(be->bio, NULL, + folio, 0); + if (found) + break; + } + f2fs_up_read(&io->bio_list_lock); + + if (!found) + continue; + + found = false; + + f2fs_down_write(&io->bio_list_lock); + list_for_each_entry(be, head, list) { + if (target) + found = (target == be->bio); + else + found = __has_merged_page(be->bio, NULL, + folio, 0); + if (found) { + target = be->bio; + del_bio_entry(be); + break; + } + } + f2fs_up_write(&io->bio_list_lock); + } + + if (found) + f2fs_submit_write_bio(sbi, target, DATA); + if (bio && *bio) { + bio_put(*bio); + *bio = NULL; + } +} + +void f2fs_submit_all_merged_ipu_writes(struct f2fs_sb_info *sbi) +{ + struct bio_entry *be, *tmp; + struct f2fs_bio_info *io; + enum temp_type temp; + + for (temp = HOT; temp < NR_TEMP_TYPE; temp++) { + LIST_HEAD(list); + + io = sbi->write_io[DATA] + temp; + + /* A lockless list_empty() check is safe here: any bios from + * other kworkers that we miss will be submitted by those + * kworkers accordingly. + */ + if (list_empty(&io->bio_list)) + continue; + + f2fs_down_write(&io->bio_list_lock); + list_splice_init(&io->bio_list, &list); + f2fs_up_write(&io->bio_list_lock); + + list_for_each_entry_safe(be, tmp, &list, list) { + f2fs_submit_write_bio(sbi, be->bio, DATA); + del_bio_entry(be); + } + } +} + +int f2fs_merge_page_bio(struct f2fs_io_info *fio) +{ + struct bio *bio = *fio->bio; + struct folio *data_folio = fio->encrypted_page ? + page_folio(fio->encrypted_page) : fio->folio; + struct folio *folio = fio->folio; + + if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr, + __is_meta_io(fio) ? META_GENERIC : DATA_GENERIC)) + return -EFSCORRUPTED; + + trace_f2fs_submit_folio_bio(data_folio, fio); + + if (bio && !page_is_mergeable(fio->sbi, bio, *fio->last_block, + fio->new_blkaddr)) + f2fs_submit_merged_ipu_write(fio->sbi, &bio, NULL); +alloc_new: + if (!bio) { + bio = __bio_alloc(fio, BIO_MAX_VECS); + f2fs_set_bio_crypt_ctx(bio, folio->mapping->host, + folio->index, fio, GFP_NOIO); + + add_bio_entry(fio->sbi, bio, data_folio, fio->temp); + } else { + if (add_ipu_page(fio, &bio, data_folio)) + goto alloc_new; + } + + if (fio->io_wbc) + wbc_account_cgroup_owner(fio->io_wbc, folio, folio_size(folio)); + + inc_page_count(fio->sbi, WB_DATA_TYPE(folio, false)); + + *fio->last_block = fio->new_blkaddr; + *fio->bio = bio; + + return 0; +} + +#ifdef CONFIG_BLK_DEV_ZONED +static bool is_end_zone_blkaddr(struct f2fs_sb_info *sbi, block_t blkaddr) +{ + struct block_device *bdev = sbi->sb->s_bdev; + int devi = 0; + + if (f2fs_is_multi_device(sbi)) { + devi = f2fs_target_device_index(sbi, blkaddr); + if (blkaddr < FDEV(devi).start_blk || + blkaddr > FDEV(devi).end_blk) { + f2fs_err(sbi, "Invalid block %x", blkaddr); + return false; + } + blkaddr -= FDEV(devi).start_blk; + bdev = FDEV(devi).bdev; + } + return bdev_is_zoned(bdev) && + f2fs_blkz_is_seq(sbi, devi, blkaddr) && + (blkaddr % sbi->blocks_per_blkz == sbi->blocks_per_blkz - 1); +} +#endif + +void f2fs_submit_page_write(struct f2fs_io_info *fio) +{ + struct f2fs_sb_info *sbi = fio->sbi; + enum page_type btype = PAGE_TYPE_OF_BIO(fio->type); + struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp; + struct folio *bio_folio; + struct f2fs_lock_context lc; + enum count_type type; + + f2fs_bug_on(sbi, is_read_io(fio->op)); + + f2fs_down_write_trace(&io->io_rwsem, &lc); +next: +#ifdef CONFIG_BLK_DEV_ZONED + if (f2fs_sb_has_blkzoned(sbi) && btype < META && io->zone_pending_bio) { + wait_for_completion_io(&io->zone_wait); + bio_put(io->zone_pending_bio); + io->zone_pending_bio = NULL; + io->bi_private = NULL; + } +#endif + + if (fio->in_list) { + spin_lock(&io->io_lock); + if (list_empty(&io->io_list)) { + spin_unlock(&io->io_lock); + goto out; + } + fio = list_first_entry(&io->io_list, + struct f2fs_io_info, list); + list_del(&fio->list); + spin_unlock(&io->io_lock); + } + + verify_fio_blkaddr(fio); + + if (fio->encrypted_page) + bio_folio = page_folio(fio->encrypted_page); + else if (fio->compressed_page) + bio_folio = page_folio(fio->compressed_page); + else + bio_folio = fio->folio; + + /* set submitted = true as a return value */ + fio->submitted = 1; + + type = WB_DATA_TYPE(bio_folio, fio->compressed_page); + inc_page_count(sbi, type); + + if (io->bio && + (!io_is_mergeable(sbi, io->bio, io, fio, io->last_block_in_bio, + fio->new_blkaddr) || + !f2fs_crypt_mergeable_bio(io->bio, fio_inode(fio), + bio_folio->index, fio))) + __submit_merged_bio(io); +alloc_new: + if (io->bio == NULL) { + io->bio = __bio_alloc(fio, BIO_MAX_VECS); + f2fs_set_bio_crypt_ctx(io->bio, fio_inode(fio), + bio_folio->index, fio, GFP_NOIO); + io->fio = *fio; + } + + if (!bio_add_folio(io->bio, bio_folio, folio_size(bio_folio), 0)) { + __submit_merged_bio(io); + goto alloc_new; + } + + if (fio->io_wbc) + wbc_account_cgroup_owner(fio->io_wbc, fio->folio, + folio_size(fio->folio)); + + io->last_block_in_bio = fio->new_blkaddr; + + trace_f2fs_submit_folio_write(fio->folio, fio); +#ifdef CONFIG_BLK_DEV_ZONED + if (f2fs_sb_has_blkzoned(sbi) && btype < META && + is_end_zone_blkaddr(sbi, fio->new_blkaddr)) { + bio_get(io->bio); + reinit_completion(&io->zone_wait); + io->bi_private = io->bio->bi_private; + io->bio->bi_private = io; + io->bio->bi_end_io = f2fs_zone_write_end_io; + io->zone_pending_bio = io->bio; + __submit_merged_bio(io); + } +#endif + if (fio->in_list) + goto next; +out: + if (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN) || + !f2fs_is_checkpoint_ready(sbi)) + __submit_merged_bio(io); + f2fs_up_write_trace(&io->io_rwsem, &lc); +} + +static struct bio *f2fs_grab_read_bio(struct inode *inode, + struct fsverity_info *vi, block_t blkaddr, + unsigned nr_pages, blk_opf_t op_flag, + pgoff_t first_idx, bool for_write) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct bio *bio; + struct bio_post_read_ctx *ctx = NULL; + unsigned int post_read_steps = 0; + sector_t sector; + struct block_device *bdev = f2fs_target_device(sbi, blkaddr, §or); + + bio = bio_alloc_bioset(bdev, bio_max_segs(nr_pages), + REQ_OP_READ | op_flag, + for_write ? GFP_NOIO : GFP_KERNEL, &f2fs_bioset); + bio->bi_iter.bi_sector = sector; + f2fs_set_bio_crypt_ctx(bio, inode, first_idx, NULL, GFP_NOFS); + bio->bi_end_io = f2fs_read_end_io; + + if (vi) + post_read_steps |= STEP_VERITY; + + /* + * STEP_DECOMPRESS is handled specially, since a compressed file might + * contain both compressed and uncompressed clusters. We'll allocate a + * bio_post_read_ctx if the file is compressed, but the caller is + * responsible for enabling STEP_DECOMPRESS if it's actually needed. + */ + + if (post_read_steps || f2fs_compressed_file(inode)) { + /* Due to the mempool, this never fails. */ + ctx = mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS); + ctx->bio = bio; + ctx->sbi = sbi; + ctx->vi = vi; + ctx->enabled_steps = post_read_steps; + ctx->fs_blkaddr = blkaddr; + ctx->decompression_attempted = false; + bio->bi_private = ctx; + } + iostat_alloc_and_bind_ctx(sbi, bio, ctx); + + return bio; +} + +/* This can handle encryption stuffs */ +static void f2fs_submit_page_read(struct inode *inode, struct fsverity_info *vi, + struct folio *folio, block_t blkaddr, + blk_opf_t op_flags, bool for_write) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct bio *bio; + + bio = f2fs_grab_read_bio(inode, vi, blkaddr, 1, op_flags, folio->index, + for_write); + + /* wait for GCed page writeback via META_MAPPING */ + f2fs_wait_on_block_writeback(inode, blkaddr); + + if (!bio_add_folio(bio, folio, PAGE_SIZE, 0)) + f2fs_bug_on(sbi, 1); + + inc_page_count(sbi, F2FS_RD_DATA); + f2fs_update_iostat(sbi, NULL, FS_DATA_READ_IO, F2FS_BLKSIZE); + f2fs_submit_read_bio(sbi, bio, DATA); +} + +static void __set_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr) +{ + __le32 *addr = get_dnode_addr(dn->inode, dn->node_folio); + + dn->data_blkaddr = blkaddr; + addr[dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr); +} + +/* + * Lock ordering for the change of data block address: + * ->data_page + * ->node_folio + * update block addresses in the node page + */ +void f2fs_set_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr) +{ + f2fs_folio_wait_writeback(dn->node_folio, NODE, true, true); + __set_data_blkaddr(dn, blkaddr); + if (folio_mark_dirty(dn->node_folio)) + dn->node_changed = true; +} + +void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr) +{ + f2fs_set_data_blkaddr(dn, blkaddr); + f2fs_update_read_extent_cache(dn); +} + +/* dn->ofs_in_node will be returned with up-to-date last block pointer */ +int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); + int err; + + if (!count) + return 0; + + if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC))) + return -EPERM; + err = inc_valid_block_count(sbi, dn->inode, &count, true); + if (unlikely(err)) + return err; + + trace_f2fs_reserve_new_blocks(dn->inode, dn->nid, + dn->ofs_in_node, count); + + f2fs_folio_wait_writeback(dn->node_folio, NODE, true, true); + + for (; count > 0; dn->ofs_in_node++) { + block_t blkaddr = f2fs_data_blkaddr(dn); + + if (blkaddr == NULL_ADDR) { + __set_data_blkaddr(dn, NEW_ADDR); + count--; + } + } + + if (folio_mark_dirty(dn->node_folio)) + dn->node_changed = true; + return 0; +} + +/* Should keep dn->ofs_in_node unchanged */ +int f2fs_reserve_new_block(struct dnode_of_data *dn) +{ + unsigned int ofs_in_node = dn->ofs_in_node; + int ret; + + ret = f2fs_reserve_new_blocks(dn, 1); + dn->ofs_in_node = ofs_in_node; + return ret; +} + +int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index) +{ + bool need_put = dn->inode_folio ? false : true; + int err; + + err = f2fs_get_dnode_of_data(dn, index, ALLOC_NODE); + if (err) + return err; + + if (dn->data_blkaddr == NULL_ADDR) + err = f2fs_reserve_new_block(dn); + if (err || need_put) + f2fs_put_dnode(dn); + return err; +} + +static inline struct fsverity_info *f2fs_need_verity(const struct inode *inode, + pgoff_t idx) +{ + if (idx < DIV_ROUND_UP(inode->i_size, PAGE_SIZE)) + return fsverity_get_info(inode); + return NULL; +} + +struct folio *f2fs_get_read_data_folio(struct inode *inode, pgoff_t index, + blk_opf_t op_flags, bool for_write, pgoff_t *next_pgofs) +{ + struct address_space *mapping = inode->i_mapping; + struct dnode_of_data dn; + struct folio *folio; + int err; +retry: + folio = f2fs_grab_cache_folio(mapping, index, for_write); + if (IS_ERR(folio)) + return folio; + + if (folio_test_large(folio)) { + pgoff_t folio_index = mapping_align_index(mapping, index); + + f2fs_folio_put(folio, true); + invalidate_inode_pages2_range(mapping, folio_index, + folio_index + folio_nr_pages(folio) - 1); + f2fs_schedule_timeout(DEFAULT_SCHEDULE_TIMEOUT); + goto retry; + } + + if (f2fs_lookup_read_extent_cache_block(inode, index, + &dn.data_blkaddr)) { + if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), dn.data_blkaddr, + DATA_GENERIC_ENHANCE_READ)) { + err = -EFSCORRUPTED; + goto put_err; + } + goto got_it; + } + + set_new_dnode(&dn, inode, NULL, NULL, 0); + err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE); + if (err) { + if (err == -ENOENT && next_pgofs) + *next_pgofs = f2fs_get_next_page_offset(&dn, index); + goto put_err; + } + f2fs_put_dnode(&dn); + + if (unlikely(dn.data_blkaddr == NULL_ADDR)) { + err = -ENOENT; + if (next_pgofs) + *next_pgofs = index + 1; + goto put_err; + } + if (dn.data_blkaddr != NEW_ADDR && + !f2fs_is_valid_blkaddr(F2FS_I_SB(inode), + dn.data_blkaddr, + DATA_GENERIC_ENHANCE)) { + err = -EFSCORRUPTED; + goto put_err; + } +got_it: + if (folio_test_uptodate(folio)) { + folio_unlock(folio); + return folio; + } + + /* + * A new dentry page is allocated but not able to be written, since its + * new inode page couldn't be allocated due to -ENOSPC. + * In such the case, its blkaddr can be remained as NEW_ADDR. + * see, f2fs_add_link -> f2fs_get_new_data_folio -> + * f2fs_init_inode_metadata. + */ + if (dn.data_blkaddr == NEW_ADDR) { + folio_zero_segment(folio, 0, folio_size(folio)); + if (!folio_test_uptodate(folio)) + folio_mark_uptodate(folio); + folio_unlock(folio); + return folio; + } + + f2fs_submit_page_read(inode, f2fs_need_verity(inode, folio->index), + folio, dn.data_blkaddr, op_flags, for_write); + return folio; + +put_err: + f2fs_folio_put(folio, true); + return ERR_PTR(err); +} + +struct folio *f2fs_find_data_folio(struct inode *inode, pgoff_t index, + pgoff_t *next_pgofs) +{ + struct address_space *mapping = inode->i_mapping; + struct folio *folio; + + folio = f2fs_filemap_get_folio(mapping, index, FGP_ACCESSED, 0); + if (IS_ERR(folio)) + goto read; + if (folio_test_uptodate(folio)) + return folio; + f2fs_folio_put(folio, false); + +read: + folio = f2fs_get_read_data_folio(inode, index, 0, false, next_pgofs); + if (IS_ERR(folio)) + return folio; + + if (folio_test_uptodate(folio)) + return folio; + + folio_wait_locked(folio); + if (unlikely(!folio_test_uptodate(folio))) { + f2fs_folio_put(folio, false); + return ERR_PTR(-EIO); + } + return folio; +} + +/* + * If it tries to access a hole, return an error. + * Because, the callers, functions in dir.c and GC, should be able to know + * whether this page exists or not. + */ +struct folio *f2fs_get_lock_data_folio(struct inode *inode, pgoff_t index, + bool for_write) +{ + struct address_space *mapping = inode->i_mapping; + struct folio *folio; + + folio = f2fs_get_read_data_folio(inode, index, 0, for_write, NULL); + if (IS_ERR(folio)) + return folio; + + /* wait for read completion */ + folio_lock(folio); + if (unlikely(folio->mapping != mapping || !folio_test_uptodate(folio))) { + f2fs_folio_put(folio, true); + return ERR_PTR(-EIO); + } + return folio; +} + +/* + * Caller ensures that this data page is never allocated. + * A new zero-filled data page is allocated in the page cache. + * + * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and + * f2fs_unlock_op(). + * Note that, ifolio is set only by make_empty_dir, and if any error occur, + * ifolio should be released by this function. + */ +struct folio *f2fs_get_new_data_folio(struct inode *inode, + struct folio *ifolio, pgoff_t index, bool new_i_size) +{ + struct address_space *mapping = inode->i_mapping; + struct folio *folio; + struct dnode_of_data dn; + int err; + + folio = f2fs_grab_cache_folio(mapping, index, true); + if (IS_ERR(folio)) { + /* + * before exiting, we should make sure ifolio will be released + * if any error occur. + */ + f2fs_folio_put(ifolio, true); + return ERR_PTR(-ENOMEM); + } + + set_new_dnode(&dn, inode, ifolio, NULL, 0); + err = f2fs_reserve_block(&dn, index); + if (err) { + f2fs_folio_put(folio, true); + return ERR_PTR(err); + } + if (!ifolio) + f2fs_put_dnode(&dn); + + if (folio_test_uptodate(folio)) + goto got_it; + + if (dn.data_blkaddr == NEW_ADDR) { + folio_zero_segment(folio, 0, folio_size(folio)); + if (!folio_test_uptodate(folio)) + folio_mark_uptodate(folio); + } else { + f2fs_folio_put(folio, true); + + /* if ifolio exists, blkaddr should be NEW_ADDR */ + f2fs_bug_on(F2FS_I_SB(inode), ifolio); + folio = f2fs_get_lock_data_folio(inode, index, true); + if (IS_ERR(folio)) + return folio; + } +got_it: + if (new_i_size && i_size_read(inode) < + ((loff_t)(index + 1) << PAGE_SHIFT)) + f2fs_i_size_write(inode, ((loff_t)(index + 1) << PAGE_SHIFT)); + return folio; +} + +static int __allocate_data_block(struct dnode_of_data *dn, int seg_type) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); + struct f2fs_summary sum; + struct node_info ni; + block_t old_blkaddr; + blkcnt_t count = 1; + int err; + + if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC))) + return -EPERM; + + err = f2fs_get_node_info(sbi, dn->nid, &ni, false); + if (err) + return err; + + dn->data_blkaddr = f2fs_data_blkaddr(dn); + if (dn->data_blkaddr == NULL_ADDR) { + err = inc_valid_block_count(sbi, dn->inode, &count, true); + if (unlikely(err)) + return err; + } + + set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version); + old_blkaddr = dn->data_blkaddr; + err = f2fs_allocate_data_block(sbi, NULL, old_blkaddr, + &dn->data_blkaddr, &sum, seg_type, NULL); + if (err) + return err; + + if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) + f2fs_invalidate_internal_cache(sbi, old_blkaddr, 1); + + f2fs_update_data_blkaddr(dn, dn->data_blkaddr); + return 0; +} + +static void f2fs_map_lock(struct f2fs_sb_info *sbi, + struct f2fs_lock_context *lc, + int flag) +{ + if (flag == F2FS_GET_BLOCK_PRE_AIO) + f2fs_down_read_trace(&sbi->node_change, lc); + else + f2fs_lock_op(sbi, lc); +} + +static void f2fs_map_unlock(struct f2fs_sb_info *sbi, + struct f2fs_lock_context *lc, + int flag) +{ + if (flag == F2FS_GET_BLOCK_PRE_AIO) + f2fs_up_read_trace(&sbi->node_change, lc); + else + f2fs_unlock_op(sbi, lc); +} + +int f2fs_get_block_locked(struct dnode_of_data *dn, pgoff_t index) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); + struct f2fs_lock_context lc; + int err = 0; + + f2fs_map_lock(sbi, &lc, F2FS_GET_BLOCK_PRE_AIO); + if (!f2fs_lookup_read_extent_cache_block(dn->inode, index, + &dn->data_blkaddr)) + err = f2fs_reserve_block(dn, index); + f2fs_map_unlock(sbi, &lc, F2FS_GET_BLOCK_PRE_AIO); + + return err; +} + +static int f2fs_map_no_dnode(struct inode *inode, + struct f2fs_map_blocks *map, struct dnode_of_data *dn, + pgoff_t pgoff) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + + /* + * There is one exceptional case that read_node_page() may return + * -ENOENT due to filesystem has been shutdown or cp_error, return + * -EIO in that case. + */ + if (map->m_may_create && + (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN) || f2fs_cp_error(sbi))) + return -EIO; + + if (map->m_next_pgofs) + *map->m_next_pgofs = f2fs_get_next_page_offset(dn, pgoff); + if (map->m_next_extent) + *map->m_next_extent = f2fs_get_next_page_offset(dn, pgoff); + return 0; +} + +static bool f2fs_map_blocks_cached(struct inode *inode, + struct f2fs_map_blocks *map, int flag) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + unsigned int maxblocks = map->m_len; + pgoff_t pgoff = (pgoff_t)map->m_lblk; + struct extent_info ei = {}; + + if (!f2fs_lookup_read_extent_cache(inode, pgoff, &ei)) + return false; + + map->m_pblk = ei.blk + pgoff - ei.fofs; + map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgoff); + map->m_flags = F2FS_MAP_MAPPED; + if (map->m_next_extent) + *map->m_next_extent = pgoff + map->m_len; + + /* for hardware encryption, but to avoid potential issue in future */ + if (flag == F2FS_GET_BLOCK_DIO) + f2fs_wait_on_block_writeback_range(inode, + map->m_pblk, map->m_len); + + map->m_multidev_dio = f2fs_allow_multi_device_dio(sbi, flag); + if (map->m_multidev_dio) { + int bidx = f2fs_target_device_index(sbi, map->m_pblk); + struct f2fs_dev_info *dev = &sbi->devs[bidx]; + + map->m_bdev = dev->bdev; + map->m_len = min(map->m_len, dev->end_blk + 1 - map->m_pblk); + map->m_pblk -= dev->start_blk; + } else { + map->m_bdev = inode->i_sb->s_bdev; + } + return true; +} + +static bool map_is_mergeable(struct f2fs_sb_info *sbi, + struct f2fs_map_blocks *map, + block_t blkaddr, int flag, int bidx, + int ofs) +{ + if (map->m_multidev_dio && map->m_bdev != FDEV(bidx).bdev) + return false; + if (map->m_pblk != NEW_ADDR && blkaddr == (map->m_pblk + ofs)) + return true; + if (map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR) + return true; + if (flag == F2FS_GET_BLOCK_PRE_DIO) + return true; + if (flag == F2FS_GET_BLOCK_DIO && + map->m_pblk == NULL_ADDR && blkaddr == NULL_ADDR) + return true; + return false; +} + +/* + * f2fs_map_blocks() tries to find or build mapping relationship which + * maps continuous logical blocks to physical blocks, and return such + * info via f2fs_map_blocks structure. + */ +int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag) +{ + unsigned int maxblocks = map->m_len; + struct dnode_of_data dn; + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_lock_context lc; + int mode = map->m_may_create ? ALLOC_NODE : LOOKUP_NODE; + pgoff_t pgofs, end_offset, end; + int err = 0, ofs = 1; + unsigned int ofs_in_node, last_ofs_in_node; + blkcnt_t prealloc; + block_t blkaddr; + unsigned int start_pgofs; + int bidx = 0; + bool is_hole; + bool lfs_dio_write; + + if (!maxblocks) + return 0; + + lfs_dio_write = (flag == F2FS_GET_BLOCK_DIO && f2fs_lfs_mode(sbi) && + map->m_may_create); + + if (!map->m_may_create && f2fs_map_blocks_cached(inode, map, flag)) { + struct extent_info ei; + + /* + * 1. If map->m_multidev_dio is true, map->m_pblk cannot be + * waitted by f2fs_wait_on_block_writeback_range() and are not + * mergeable. + * 2. If pgofs hits the read extent cache, it means the mapping + * is already cached in the extent cache, but it is not + * mergeable, and there is no need to query the mapping again + * via f2fs_get_dnode_of_data(). + */ + pgofs = (pgoff_t)map->m_lblk + map->m_len; + if (map->m_len == maxblocks || + map->m_multidev_dio || + f2fs_lookup_read_extent_cache(inode, pgofs, &ei)) + goto out; + ofs = map->m_len; + goto map_more; + } + + map->m_bdev = inode->i_sb->s_bdev; + map->m_multidev_dio = + f2fs_allow_multi_device_dio(F2FS_I_SB(inode), flag); + + map->m_len = 0; + map->m_flags = 0; + + /* it only supports block size == page size */ + pgofs = (pgoff_t)map->m_lblk; +map_more: + end = (pgoff_t)map->m_lblk + maxblocks; + + if (flag == F2FS_GET_BLOCK_PRECACHE) + mode = LOOKUP_NODE_RA; + +next_dnode: + if (map->m_may_create) { + if (f2fs_lfs_mode(sbi)) + f2fs_balance_fs(sbi, true); + f2fs_map_lock(sbi, &lc, flag); + } + + /* When reading holes, we need its node page */ + set_new_dnode(&dn, inode, NULL, NULL, 0); + err = f2fs_get_dnode_of_data(&dn, pgofs, mode); + if (err) { + if (flag == F2FS_GET_BLOCK_BMAP) + map->m_pblk = 0; + if (err == -ENOENT) + err = f2fs_map_no_dnode(inode, map, &dn, pgofs); + goto unlock_out; + } + + start_pgofs = pgofs; + prealloc = 0; + last_ofs_in_node = ofs_in_node = dn.ofs_in_node; + end_offset = ADDRS_PER_PAGE(dn.node_folio, inode); + +next_block: + blkaddr = f2fs_data_blkaddr(&dn); + is_hole = !__is_valid_data_blkaddr(blkaddr); + if (!is_hole && + !f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE)) { + err = -EFSCORRUPTED; + goto sync_out; + } + + /* use out-place-update for direct IO under LFS mode */ + if (map->m_may_create && (is_hole || + (flag == F2FS_GET_BLOCK_DIO && f2fs_lfs_mode(sbi) && + !f2fs_is_pinned_file(inode) && map->m_last_pblk != blkaddr))) { + if (unlikely(f2fs_cp_error(sbi))) { + err = -EIO; + goto sync_out; + } + + switch (flag) { + case F2FS_GET_BLOCK_PRE_AIO: + if (blkaddr == NULL_ADDR) { + prealloc++; + last_ofs_in_node = dn.ofs_in_node; + } + break; + case F2FS_GET_BLOCK_PRE_DIO: + case F2FS_GET_BLOCK_DIO: + err = __allocate_data_block(&dn, map->m_seg_type); + if (err) + goto sync_out; + if (flag == F2FS_GET_BLOCK_PRE_DIO) + file_need_truncate(inode); + set_inode_flag(inode, FI_APPEND_WRITE); + break; + default: + WARN_ON_ONCE(1); + err = -EIO; + goto sync_out; + } + + blkaddr = dn.data_blkaddr; + if (is_hole) + map->m_flags |= F2FS_MAP_NEW; + } else if (is_hole) { + if (f2fs_compressed_file(inode) && + f2fs_sanity_check_cluster(&dn)) { + err = -EFSCORRUPTED; + f2fs_handle_error(sbi, + ERROR_CORRUPTED_CLUSTER); + fserror_report_file_metadata(inode, err, GFP_NOFS); + goto sync_out; + } + + switch (flag) { + case F2FS_GET_BLOCK_PRECACHE: + goto sync_out; + case F2FS_GET_BLOCK_BMAP: + map->m_pblk = 0; + goto sync_out; + case F2FS_GET_BLOCK_FIEMAP: + if (blkaddr == NULL_ADDR) { + if (map->m_next_pgofs) + *map->m_next_pgofs = pgofs + 1; + goto sync_out; + } + break; + case F2FS_GET_BLOCK_DIO: + if (map->m_next_pgofs) + *map->m_next_pgofs = pgofs + 1; + break; + default: + /* for defragment case */ + if (map->m_next_pgofs) + *map->m_next_pgofs = pgofs + 1; + goto sync_out; + } + } + + if (flag == F2FS_GET_BLOCK_PRE_AIO) + goto skip; + + if (map->m_multidev_dio) + bidx = f2fs_target_device_index(sbi, blkaddr); + + if (map->m_len == 0) { + /* reserved delalloc block should be mapped for fiemap. */ + if (blkaddr == NEW_ADDR) + map->m_flags |= F2FS_MAP_DELALLOC; + /* DIO READ and hole case, should not map the blocks. */ + if (!(flag == F2FS_GET_BLOCK_DIO && is_hole && !map->m_may_create)) + map->m_flags |= F2FS_MAP_MAPPED; + + map->m_pblk = blkaddr; + map->m_len = 1; + + if (map->m_multidev_dio) + map->m_bdev = FDEV(bidx).bdev; + + if (lfs_dio_write) + map->m_last_pblk = NULL_ADDR; + } else if (map_is_mergeable(sbi, map, blkaddr, flag, bidx, ofs)) { + ofs++; + map->m_len++; + } else { + if (lfs_dio_write && !f2fs_is_pinned_file(inode)) + map->m_last_pblk = blkaddr; + goto sync_out; + } + +skip: + dn.ofs_in_node++; + pgofs++; + + /* preallocate blocks in batch for one dnode page */ + if (flag == F2FS_GET_BLOCK_PRE_AIO && + (pgofs == end || dn.ofs_in_node == end_offset)) { + + dn.ofs_in_node = ofs_in_node; + err = f2fs_reserve_new_blocks(&dn, prealloc); + if (err) + goto sync_out; + + map->m_len += dn.ofs_in_node - ofs_in_node; + if (prealloc && dn.ofs_in_node != last_ofs_in_node + 1) { + err = -ENOSPC; + goto sync_out; + } + dn.ofs_in_node = end_offset; + } + + if (pgofs >= end) + goto sync_out; + else if (dn.ofs_in_node < end_offset) + goto next_block; + + if (flag == F2FS_GET_BLOCK_PRECACHE) { + if (map->m_flags & F2FS_MAP_MAPPED) { + unsigned int ofs = start_pgofs - map->m_lblk; + + f2fs_update_read_extent_cache_range(&dn, + start_pgofs, map->m_pblk + ofs, + map->m_len - ofs); + } + } + + f2fs_put_dnode(&dn); + + if (map->m_may_create) { + f2fs_map_unlock(sbi, &lc, flag); + f2fs_balance_fs(sbi, dn.node_changed); + } + goto next_dnode; + +sync_out: + + if (flag == F2FS_GET_BLOCK_DIO && map->m_flags & F2FS_MAP_MAPPED) { + /* + * for hardware encryption, but to avoid potential issue + * in future + */ + f2fs_wait_on_block_writeback_range(inode, + map->m_pblk, map->m_len); + + if (map->m_multidev_dio) { + block_t blk_addr = map->m_pblk; + + bidx = f2fs_target_device_index(sbi, map->m_pblk); + + map->m_bdev = FDEV(bidx).bdev; + map->m_pblk -= FDEV(bidx).start_blk; + + if (map->m_may_create) + f2fs_update_device_state(sbi, inode->i_ino, + blk_addr, map->m_len); + + f2fs_bug_on(sbi, blk_addr + map->m_len > + FDEV(bidx).end_blk + 1); + } + } + + if (flag == F2FS_GET_BLOCK_PRECACHE) { + if (map->m_flags & F2FS_MAP_MAPPED) { + unsigned int ofs = start_pgofs - map->m_lblk; + + if (map->m_len > ofs) + f2fs_update_read_extent_cache_range(&dn, + start_pgofs, map->m_pblk + ofs, + map->m_len - ofs); + } + if (map->m_next_extent) + *map->m_next_extent = is_hole ? pgofs + 1 : pgofs; + } + f2fs_put_dnode(&dn); +unlock_out: + if (map->m_may_create) { + f2fs_map_unlock(sbi, &lc, flag); + f2fs_balance_fs(sbi, dn.node_changed); + } +out: + trace_f2fs_map_blocks(inode, map, flag, err); + return err; +} + +static bool __f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len, + bool check_first) +{ + struct f2fs_map_blocks map; + block_t last_lblk; + int err; + + if (pos + len > i_size_read(inode)) + return false; + + map.m_lblk = F2FS_BYTES_TO_BLK(pos); + map.m_next_pgofs = NULL; + map.m_next_extent = NULL; + map.m_seg_type = NO_CHECK_TYPE; + map.m_may_create = false; + last_lblk = F2FS_BLK_ALIGN(pos + len); + + while (map.m_lblk < last_lblk) { + map.m_len = last_lblk - map.m_lblk; + err = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_DEFAULT); + if (err || map.m_len == 0) + return false; + map.m_lblk += map.m_len; + if (check_first) + break; + } + return true; +} + +bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len) +{ + return __f2fs_overwrite_io(inode, pos, len, false); +} + +static int f2fs_xattr_fiemap(struct inode *inode, + struct fiemap_extent_info *fieinfo) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct node_info ni; + __u64 phys = 0, len; + __u32 flags; + nid_t xnid = F2FS_I(inode)->i_xattr_nid; + int err = 0; + + if (f2fs_has_inline_xattr(inode)) { + int offset; + struct folio *folio = f2fs_grab_cache_folio(NODE_MAPPING(sbi), + inode->i_ino, false); + + if (IS_ERR(folio)) + return PTR_ERR(folio); + + err = f2fs_get_node_info(sbi, inode->i_ino, &ni, false); + if (err) { + f2fs_folio_put(folio, true); + return err; + } + + phys = F2FS_BLK_TO_BYTES(ni.blk_addr); + offset = offsetof(struct f2fs_inode, i_addr) + + sizeof(__le32) * (DEF_ADDRS_PER_INODE - + get_inline_xattr_addrs(inode)); + + phys += offset; + len = inline_xattr_size(inode); + + f2fs_folio_put(folio, true); + + flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED; + + if (!xnid) + flags |= FIEMAP_EXTENT_LAST; + + err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags); + trace_f2fs_fiemap(inode, 0, phys, len, flags, err); + if (err) + return err; + } + + if (xnid) { + struct folio *folio = f2fs_grab_cache_folio(NODE_MAPPING(sbi), + xnid, false); + + if (IS_ERR(folio)) + return PTR_ERR(folio); + + err = f2fs_get_node_info(sbi, xnid, &ni, false); + if (err) { + f2fs_folio_put(folio, true); + return err; + } + + phys = F2FS_BLK_TO_BYTES(ni.blk_addr); + len = inode->i_sb->s_blocksize; + + f2fs_folio_put(folio, true); + + flags = FIEMAP_EXTENT_LAST; + } + + if (phys) { + err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags); + trace_f2fs_fiemap(inode, 0, phys, len, flags, err); + } + + return (err < 0 ? err : 0); +} + +int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + u64 start, u64 len) +{ + struct f2fs_map_blocks map; + sector_t start_blk, last_blk, blk_len, max_len; + pgoff_t next_pgofs; + u64 logical = 0, phys = 0, size = 0; + u32 flags = 0; + int ret = 0; + bool compr_cluster = false, compr_appended; + unsigned int cluster_size = F2FS_I(inode)->i_cluster_size; + unsigned int count_in_cluster = 0; + loff_t maxbytes; + + if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) { + ret = f2fs_precache_extents(inode); + if (ret) + return ret; + } + + ret = fiemap_prep(inode, fieinfo, start, &len, FIEMAP_FLAG_XATTR); + if (ret) + return ret; + + inode_lock_shared(inode); + + maxbytes = F2FS_BLK_TO_BYTES(max_file_blocks(inode)); + if (start > maxbytes) { + ret = -EFBIG; + goto out; + } + + if (len > maxbytes || (maxbytes - len) < start) + len = maxbytes - start; + + if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) { + ret = f2fs_xattr_fiemap(inode, fieinfo); + goto out; + } + + if (f2fs_has_inline_data(inode) || f2fs_has_inline_dentry(inode)) { + ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len); + if (ret != -EAGAIN) + goto out; + } + + start_blk = F2FS_BYTES_TO_BLK(start); + last_blk = F2FS_BYTES_TO_BLK(start + len - 1); + blk_len = last_blk - start_blk + 1; + max_len = F2FS_BYTES_TO_BLK(maxbytes) - start_blk; + +next: + memset(&map, 0, sizeof(map)); + map.m_lblk = start_blk; + map.m_len = blk_len; + map.m_next_pgofs = &next_pgofs; + map.m_seg_type = NO_CHECK_TYPE; + + if (compr_cluster) { + map.m_lblk += 1; + map.m_len = cluster_size - count_in_cluster; + } + + ret = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_FIEMAP); + if (ret) + goto out; + + /* HOLE */ + if (!compr_cluster && !(map.m_flags & F2FS_MAP_FLAGS)) { + start_blk = next_pgofs; + + if (F2FS_BLK_TO_BYTES(start_blk) < maxbytes) + goto prep_next; + + flags |= FIEMAP_EXTENT_LAST; + } + + /* + * current extent may cross boundary of inquiry, increase len to + * requery. + */ + if (!compr_cluster && (map.m_flags & F2FS_MAP_MAPPED) && + map.m_lblk + map.m_len - 1 == last_blk && + blk_len != max_len) { + blk_len = max_len; + goto next; + } + + compr_appended = false; + /* In a case of compressed cluster, append this to the last extent */ + if (compr_cluster && ((map.m_flags & F2FS_MAP_DELALLOC) || + !(map.m_flags & F2FS_MAP_FLAGS))) { + compr_appended = true; + goto skip_fill; + } + + if (size) { + flags |= FIEMAP_EXTENT_MERGED; + if (IS_ENCRYPTED(inode)) + flags |= FIEMAP_EXTENT_DATA_ENCRYPTED; + + ret = fiemap_fill_next_extent(fieinfo, logical, + phys, size, flags); + trace_f2fs_fiemap(inode, logical, phys, size, flags, ret); + if (ret) + goto out; + size = 0; + } + + if (start_blk > last_blk) + goto out; + +skip_fill: + if (map.m_pblk == COMPRESS_ADDR) { + compr_cluster = true; + count_in_cluster = 1; + } else if (compr_appended) { + unsigned int appended_blks = cluster_size - + count_in_cluster + 1; + size += F2FS_BLK_TO_BYTES(appended_blks); + start_blk += appended_blks; + compr_cluster = false; + } else { + logical = F2FS_BLK_TO_BYTES(start_blk); + phys = __is_valid_data_blkaddr(map.m_pblk) ? + F2FS_BLK_TO_BYTES(map.m_pblk) : 0; + size = F2FS_BLK_TO_BYTES(map.m_len); + flags = 0; + + if (compr_cluster) { + flags = FIEMAP_EXTENT_ENCODED; + count_in_cluster += map.m_len; + if (count_in_cluster == cluster_size) { + compr_cluster = false; + size += F2FS_BLKSIZE; + } + } else if (map.m_flags & F2FS_MAP_DELALLOC) { + flags = FIEMAP_EXTENT_UNWRITTEN; + } + + start_blk += F2FS_BYTES_TO_BLK(size); + } + +prep_next: + cond_resched(); + if (fatal_signal_pending(current)) + ret = -EINTR; + else + goto next; +out: + if (ret == 1) + ret = 0; + + inode_unlock_shared(inode); + return ret; +} + +static inline loff_t f2fs_readpage_limit(struct inode *inode) +{ + if (IS_ENABLED(CONFIG_FS_VERITY) && IS_VERITY(inode)) + return F2FS_BLK_TO_BYTES(max_file_blocks(inode)); + + return i_size_read(inode); +} + +static inline blk_opf_t f2fs_ra_op_flags(struct readahead_control *rac) +{ + return rac ? REQ_RAHEAD : 0; +} + +static int f2fs_read_single_page(struct inode *inode, struct fsverity_info *vi, + struct folio *folio, unsigned int nr_pages, + struct f2fs_map_blocks *map, + struct bio **bio_ret, + sector_t *last_block_in_bio, + struct readahead_control *rac) +{ + struct bio *bio = *bio_ret; + const unsigned int blocksize = F2FS_BLKSIZE; + sector_t block_in_file; + sector_t last_block; + sector_t last_block_in_file; + sector_t block_nr; + pgoff_t index = folio->index; + int ret = 0; + + block_in_file = (sector_t)index; + last_block = block_in_file + nr_pages; + last_block_in_file = F2FS_BYTES_TO_BLK(f2fs_readpage_limit(inode) + + blocksize - 1); + if (last_block > last_block_in_file) + last_block = last_block_in_file; + + /* just zeroing out page which is beyond EOF */ + if (block_in_file >= last_block) + goto zero_out; + /* + * Map blocks using the previous result first. + */ + if (map->m_flags & F2FS_MAP_MAPPED) { + if (block_in_file > map->m_lblk && + block_in_file < (map->m_lblk + map->m_len)) + goto got_it; + } else if (block_in_file < *map->m_next_pgofs) { + goto got_it; + } + + /* + * Then do more f2fs_map_blocks() calls until we are + * done with this page. + */ + map->m_lblk = block_in_file; + map->m_len = last_block - block_in_file; + + ret = f2fs_map_blocks(inode, map, F2FS_GET_BLOCK_DEFAULT); + if (ret) + goto out; +got_it: + if ((map->m_flags & F2FS_MAP_MAPPED)) { + block_nr = map->m_pblk + block_in_file - map->m_lblk; + folio_set_mappedtodisk(folio); + + if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), block_nr, + DATA_GENERIC_ENHANCE_READ)) { + ret = -EFSCORRUPTED; + goto out; + } + } else { +zero_out: + folio_zero_segment(folio, 0, folio_size(folio)); + if (vi && !fsverity_verify_folio(vi, folio)) { + ret = -EIO; + goto out; + } + if (!folio_test_uptodate(folio)) + folio_mark_uptodate(folio); + folio_unlock(folio); + goto out; + } + + /* + * This page will go to BIO. Do we need to send this + * BIO off first? + */ + if (bio && (!page_is_mergeable(F2FS_I_SB(inode), bio, + *last_block_in_bio, block_nr) || + !f2fs_crypt_mergeable_bio(bio, inode, index, NULL))) { +submit_and_realloc: + f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA); + bio = NULL; + } + if (bio == NULL) + bio = f2fs_grab_read_bio(inode, vi, block_nr, nr_pages, + f2fs_ra_op_flags(rac), index, false); + + /* + * If the page is under writeback, we need to wait for + * its completion to see the correct decrypted data. + */ + f2fs_wait_on_block_writeback(inode, block_nr); + + if (!bio_add_folio(bio, folio, blocksize, 0)) + goto submit_and_realloc; + + inc_page_count(F2FS_I_SB(inode), F2FS_RD_DATA); + f2fs_update_iostat(F2FS_I_SB(inode), NULL, FS_DATA_READ_IO, + F2FS_BLKSIZE); + *last_block_in_bio = block_nr; +out: + *bio_ret = bio; + return ret; +} + +#ifdef CONFIG_F2FS_FS_COMPRESSION +int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret, + unsigned nr_pages, sector_t *last_block_in_bio, + struct readahead_control *rac, bool for_write) +{ + struct dnode_of_data dn; + struct inode *inode = cc->inode; + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct bio *bio = *bio_ret; + unsigned int start_idx = cc->cluster_idx << cc->log_cluster_size; + sector_t last_block_in_file; + const unsigned int blocksize = F2FS_BLKSIZE; + struct decompress_io_ctx *dic = NULL; + struct extent_info ei = {}; + bool from_dnode = true; + int i; + int ret = 0; + + if (unlikely(f2fs_cp_error(sbi))) { + ret = -EIO; + from_dnode = false; + goto out_put_dnode; + } + + f2fs_bug_on(sbi, f2fs_cluster_is_empty(cc)); + + last_block_in_file = F2FS_BYTES_TO_BLK(f2fs_readpage_limit(inode) + + blocksize - 1); + + /* get rid of pages beyond EOF */ + for (i = 0; i < cc->cluster_size; i++) { + struct page *page = cc->rpages[i]; + struct folio *folio; + + if (!page) + continue; + + folio = page_folio(page); + if ((sector_t)folio->index >= last_block_in_file) { + folio_zero_segment(folio, 0, folio_size(folio)); + if (!folio_test_uptodate(folio)) + folio_mark_uptodate(folio); + } else if (!folio_test_uptodate(folio)) { + continue; + } + folio_unlock(folio); + if (for_write) + folio_put(folio); + cc->rpages[i] = NULL; + cc->nr_rpages--; + } + + /* we are done since all pages are beyond EOF */ + if (f2fs_cluster_is_empty(cc)) + goto out; + + if (f2fs_lookup_read_extent_cache(inode, start_idx, &ei)) + from_dnode = false; + + if (!from_dnode) + goto skip_reading_dnode; + + set_new_dnode(&dn, inode, NULL, NULL, 0); + ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE); + if (ret) + goto out; + + f2fs_bug_on(sbi, dn.data_blkaddr != COMPRESS_ADDR); + +skip_reading_dnode: + for (i = 1; i < cc->cluster_size; i++) { + block_t blkaddr; + + blkaddr = from_dnode ? data_blkaddr(dn.inode, dn.node_folio, + dn.ofs_in_node + i) : + ei.blk + i - 1; + + if (!__is_valid_data_blkaddr(blkaddr)) + break; + + if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC)) { + ret = -EFAULT; + goto out_put_dnode; + } + cc->nr_cpages++; + + if (!from_dnode && i >= ei.c_len) + break; + } + + /* nothing to decompress */ + if (cc->nr_cpages == 0) { + ret = 0; + goto out_put_dnode; + } + + dic = f2fs_alloc_dic(cc); + if (IS_ERR(dic)) { + ret = PTR_ERR(dic); + goto out_put_dnode; + } + + for (i = 0; i < cc->nr_cpages; i++) { + struct folio *folio = page_folio(dic->cpages[i]); + block_t blkaddr; + struct bio_post_read_ctx *ctx; + + blkaddr = from_dnode ? data_blkaddr(dn.inode, dn.node_folio, + dn.ofs_in_node + i + 1) : + ei.blk + i; + + f2fs_wait_on_block_writeback(inode, blkaddr); + + if (f2fs_load_compressed_folio(sbi, folio, blkaddr)) { + if (atomic_dec_and_test(&dic->remaining_pages)) { + f2fs_decompress_cluster(dic, true); + break; + } + continue; + } + + if (bio && (!page_is_mergeable(sbi, bio, + *last_block_in_bio, blkaddr) || + !f2fs_crypt_mergeable_bio(bio, inode, folio->index, NULL))) { +submit_and_realloc: + f2fs_submit_read_bio(sbi, bio, DATA); + bio = NULL; + } + + if (!bio) + bio = f2fs_grab_read_bio(inode, cc->vi, blkaddr, + nr_pages - i, + f2fs_ra_op_flags(rac), + folio->index, for_write); + + if (!bio_add_folio(bio, folio, blocksize, 0)) + goto submit_and_realloc; + + ctx = get_post_read_ctx(bio); + ctx->enabled_steps |= STEP_DECOMPRESS; + refcount_inc(&dic->refcnt); + + inc_page_count(sbi, F2FS_RD_DATA); + f2fs_update_iostat(sbi, inode, FS_DATA_READ_IO, F2FS_BLKSIZE); + *last_block_in_bio = blkaddr; + } + + if (from_dnode) + f2fs_put_dnode(&dn); + + *bio_ret = bio; + return 0; + +out_put_dnode: + if (from_dnode) + f2fs_put_dnode(&dn); +out: + for (i = 0; i < cc->cluster_size; i++) { + if (cc->rpages[i]) { + ClearPageUptodate(cc->rpages[i]); + unlock_page(cc->rpages[i]); + } + } + *bio_ret = bio; + return ret; +} +#endif + +static struct f2fs_folio_state *ffs_find_or_alloc(struct folio *folio) +{ + struct f2fs_folio_state *ffs = folio->private; + + if (ffs) + return ffs; + + ffs = f2fs_kmem_cache_alloc(ffs_entry_slab, + GFP_NOIO | __GFP_ZERO, true, NULL); + + spin_lock_init(&ffs->state_lock); + folio_attach_private(folio, ffs); + return ffs; +} + +static void ffs_detach_free(struct folio *folio) +{ + struct f2fs_folio_state *ffs; + + if (!folio_test_large(folio)) { + folio_detach_private(folio); + return; + } + + ffs = folio_detach_private(folio); + if (!ffs) + return; + + WARN_ON_ONCE(ffs->read_pages_pending != 0); + kmem_cache_free(ffs_entry_slab, ffs); +} + +static int f2fs_read_data_large_folio(struct inode *inode, + struct fsverity_info *vi, + struct readahead_control *rac, struct folio *folio) +{ + struct bio *bio = NULL; + sector_t last_block_in_bio = 0; + struct f2fs_map_blocks map = {0, }; + pgoff_t index, offset, next_pgofs = 0; + unsigned max_nr_pages = rac ? readahead_count(rac) : + folio_nr_pages(folio); + unsigned nrpages; + struct f2fs_folio_state *ffs; + int ret = 0; + bool folio_in_bio = false; + + if (!IS_IMMUTABLE(inode) || f2fs_compressed_file(inode)) { + if (folio) + folio_unlock(folio); + return -EOPNOTSUPP; + } + + map.m_seg_type = NO_CHECK_TYPE; + + if (rac) + folio = readahead_folio(rac); +next_folio: + if (!folio) + goto out; + + f2fs_update_read_folio_count(F2FS_I_SB(inode), folio); + + folio_in_bio = false; + index = folio->index; + offset = 0; + ffs = NULL; + nrpages = folio_nr_pages(folio); + + for (; nrpages; nrpages--, max_nr_pages--, index++, offset++) { + sector_t block_nr; + /* + * Map blocks using the previous result first. + */ + if (map.m_flags & F2FS_MAP_MAPPED) { + if (index > map.m_lblk && + index < (map.m_lblk + map.m_len)) + goto got_it; + } else if (index < next_pgofs) { + /* hole case */ + goto got_it; + } + + /* + * Then do more f2fs_map_blocks() calls until we are + * done with this page. + */ + memset(&map, 0, sizeof(map)); + map.m_next_pgofs = &next_pgofs; + map.m_seg_type = NO_CHECK_TYPE; + map.m_lblk = index; + map.m_len = max_nr_pages; + + ret = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_DEFAULT); + if (ret) + goto err_out; +got_it: + if ((map.m_flags & F2FS_MAP_MAPPED)) { + block_nr = map.m_pblk + index - map.m_lblk; + if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), block_nr, + DATA_GENERIC_ENHANCE_READ)) { + ret = -EFSCORRUPTED; + goto err_out; + } + } else { + size_t page_offset = offset << PAGE_SHIFT; + folio_zero_range(folio, page_offset, PAGE_SIZE); + if (vi && !fsverity_verify_blocks(vi, folio, PAGE_SIZE, page_offset)) { + ret = -EIO; + goto err_out; + } + continue; + } + + /* We must increment read_pages_pending before possible BIOs submitting + * to prevent from premature folio_end_read() call on folio + */ + if (folio_test_large(folio)) { + ffs = ffs_find_or_alloc(folio); + + /* set the bitmap to wait */ + spin_lock_irq(&ffs->state_lock); + ffs->read_pages_pending++; + spin_unlock_irq(&ffs->state_lock); + } + + /* + * This page will go to BIO. Do we need to send this + * BIO off first? + */ + if (bio && (!page_is_mergeable(F2FS_I_SB(inode), bio, + last_block_in_bio, block_nr) || + !f2fs_crypt_mergeable_bio(bio, inode, index, NULL))) { +submit_and_realloc: + f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA); + bio = NULL; + } + if (bio == NULL) + bio = f2fs_grab_read_bio(inode, vi, + block_nr, max_nr_pages, + f2fs_ra_op_flags(rac), + index, false); + + /* + * If the page is under writeback, we need to wait for + * its completion to see the correct decrypted data. + */ + f2fs_wait_on_block_writeback(inode, block_nr); + + if (!bio_add_folio(bio, folio, F2FS_BLKSIZE, + offset << PAGE_SHIFT)) + goto submit_and_realloc; + + folio_in_bio = true; + inc_page_count(F2FS_I_SB(inode), F2FS_RD_DATA); + f2fs_update_iostat(F2FS_I_SB(inode), NULL, FS_DATA_READ_IO, + F2FS_BLKSIZE); + last_block_in_bio = block_nr; + } + trace_f2fs_read_folio(folio, DATA); +err_out: + if (!folio_in_bio) + folio_end_read(folio, !ret); + if (ret) + goto out; + if (rac) { + folio = readahead_folio(rac); + goto next_folio; + } +out: + f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA); + if (ret && folio_in_bio) { + /* Wait bios and clear uptodate. */ + folio_lock(folio); + folio_clear_uptodate(folio); + folio_unlock(folio); + } + return ret; +} + +/* + * This function was originally taken from fs/mpage.c, and customized for f2fs. + * Major change was from block_size == page_size in f2fs by default. + */ +static int f2fs_mpage_readpages(struct inode *inode, struct fsverity_info *vi, + struct readahead_control *rac, struct folio *folio) +{ + struct bio *bio = NULL; + sector_t last_block_in_bio = 0; + struct f2fs_map_blocks map; +#ifdef CONFIG_F2FS_FS_COMPRESSION + struct compress_ctx cc = { + .inode = inode, + .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, + .cluster_size = F2FS_I(inode)->i_cluster_size, + .cluster_idx = NULL_CLUSTER, + .rpages = NULL, + .cpages = NULL, + .nr_rpages = 0, + .nr_cpages = 0, + }; + pgoff_t nc_cluster_idx = NULL_CLUSTER; + pgoff_t index; +#endif + pgoff_t next_pgofs = 0; + unsigned nr_pages = rac ? readahead_count(rac) : 1; + struct address_space *mapping = rac ? rac->mapping : folio->mapping; + unsigned max_nr_pages = nr_pages; + int ret = 0; + + if (mapping_large_folio_support(mapping)) + return f2fs_read_data_large_folio(inode, vi, rac, folio); + +#ifdef CONFIG_F2FS_FS_COMPRESSION + if (f2fs_compressed_file(inode)) { + index = rac ? readahead_index(rac) : folio->index; + max_nr_pages = round_up(index + nr_pages, cc.cluster_size) - + round_down(index, cc.cluster_size); + } +#endif + + map.m_pblk = 0; + map.m_lblk = 0; + map.m_len = 0; + map.m_flags = 0; + map.m_next_pgofs = &next_pgofs; + map.m_next_extent = NULL; + map.m_seg_type = NO_CHECK_TYPE; + map.m_may_create = false; + + for (; nr_pages; nr_pages--) { + if (rac) { + folio = readahead_folio(rac); + prefetchw(&folio->flags); + } + + f2fs_update_read_folio_count(F2FS_I_SB(inode), folio); + +#ifdef CONFIG_F2FS_FS_COMPRESSION + index = folio->index; + + if (!f2fs_compressed_file(inode)) + goto read_single_page; + + /* there are remained compressed pages, submit them */ + if (!f2fs_cluster_can_merge_page(&cc, index)) { + cc.vi = vi; + ret = f2fs_read_multi_pages(&cc, &bio, + max_nr_pages, + &last_block_in_bio, + rac, false); + f2fs_destroy_compress_ctx(&cc, false); + if (ret) + goto set_error_page; + } + if (cc.cluster_idx == NULL_CLUSTER) { + if (nc_cluster_idx == index >> cc.log_cluster_size) + goto read_single_page; + + ret = f2fs_is_compressed_cluster(inode, index); + if (ret < 0) + goto set_error_page; + else if (!ret) { + nc_cluster_idx = + index >> cc.log_cluster_size; + goto read_single_page; + } + + nc_cluster_idx = NULL_CLUSTER; + } + ret = f2fs_init_compress_ctx(&cc); + if (ret) + goto set_error_page; + + f2fs_compress_ctx_add_page(&cc, folio); + + goto next_page; +read_single_page: +#endif + + ret = f2fs_read_single_page(inode, vi, folio, max_nr_pages, + &map, &bio, &last_block_in_bio, + rac); + if (ret) { +#ifdef CONFIG_F2FS_FS_COMPRESSION +set_error_page: +#endif + folio_zero_segment(folio, 0, folio_size(folio)); + folio_unlock(folio); + } +#ifdef CONFIG_F2FS_FS_COMPRESSION +next_page: +#endif + +#ifdef CONFIG_F2FS_FS_COMPRESSION + if (f2fs_compressed_file(inode)) { + /* last page */ + if (nr_pages == 1 && !f2fs_cluster_is_empty(&cc)) { + cc.vi = vi; + ret = f2fs_read_multi_pages(&cc, &bio, + max_nr_pages, + &last_block_in_bio, + rac, false); + f2fs_destroy_compress_ctx(&cc, false); + } + } +#endif + } + f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA); + return ret; +} + +static int f2fs_read_data_folio(struct file *file, struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + struct fsverity_info *vi = NULL; + int ret; + + trace_f2fs_readpage(folio, DATA); + + if (!f2fs_is_compress_backend_ready(inode)) { + folio_unlock(folio); + return -EOPNOTSUPP; + } + + /* If the file has inline data, try to read it directly */ + if (f2fs_has_inline_data(inode)) { + ret = f2fs_read_inline_data(inode, folio); + if (ret != -EAGAIN) + return ret; + } + + vi = f2fs_need_verity(inode, folio->index); + if (vi) + fsverity_readahead(vi, folio->index, folio_nr_pages(folio)); + return f2fs_mpage_readpages(inode, vi, NULL, folio); +} + +static void f2fs_readahead(struct readahead_control *rac) +{ + struct inode *inode = rac->mapping->host; + struct fsverity_info *vi = NULL; + + trace_f2fs_readpages(inode, readahead_index(rac), readahead_count(rac)); + + if (!f2fs_is_compress_backend_ready(inode)) + return; + + /* If the file has inline data, skip readahead */ + if (f2fs_has_inline_data(inode)) + return; + + vi = f2fs_need_verity(inode, readahead_index(rac)); + if (vi) + fsverity_readahead(vi, readahead_index(rac), + readahead_count(rac)); + f2fs_mpage_readpages(inode, vi, rac, NULL); +} + +static inline bool check_inplace_update_policy(struct inode *inode, + struct f2fs_io_info *fio) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + + if (IS_F2FS_IPU_HONOR_OPU_WRITE(sbi) && + is_inode_flag_set(inode, FI_OPU_WRITE)) + return false; + if (IS_F2FS_IPU_FORCE(sbi)) + return true; + if (IS_F2FS_IPU_SSR(sbi) && f2fs_need_SSR(sbi)) + return true; + if (IS_F2FS_IPU_UTIL(sbi) && utilization(sbi) > SM_I(sbi)->min_ipu_util) + return true; + if (IS_F2FS_IPU_SSR_UTIL(sbi) && f2fs_need_SSR(sbi) && + utilization(sbi) > SM_I(sbi)->min_ipu_util) + return true; + + /* + * IPU for rewrite async pages + */ + if (IS_F2FS_IPU_ASYNC(sbi) && fio && fio->op == REQ_OP_WRITE && + !(fio->op_flags & REQ_SYNC) && !IS_ENCRYPTED(inode)) + return true; + + /* this is only set during fdatasync */ + if (IS_F2FS_IPU_FSYNC(sbi) && is_inode_flag_set(inode, FI_NEED_IPU)) + return true; + + if (unlikely(fio && is_sbi_flag_set(sbi, SBI_CP_DISABLED) && + !f2fs_is_checkpointed_data(sbi, fio->old_blkaddr))) + return true; + + return false; +} + +bool f2fs_should_update_inplace(struct inode *inode, struct f2fs_io_info *fio) +{ + /* swap file is migrating in aligned write mode */ + if (is_inode_flag_set(inode, FI_ALIGNED_WRITE)) + return false; + + if (f2fs_is_pinned_file(inode)) + return true; + + /* if this is cold file, we should overwrite to avoid fragmentation */ + if (file_is_cold(inode) && !is_inode_flag_set(inode, FI_OPU_WRITE)) + return true; + + return check_inplace_update_policy(inode, fio); +} + +bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + + /* The below cases were checked when setting it. */ + if (f2fs_is_pinned_file(inode)) + return false; + if (fio && is_sbi_flag_set(sbi, SBI_NEED_FSCK)) + return true; + if (f2fs_lfs_mode(sbi)) + return true; + if (S_ISDIR(inode->i_mode)) + return true; + if (IS_NOQUOTA(inode)) + return true; + if (f2fs_used_in_atomic_write(inode)) + return true; + /* rewrite low ratio compress data w/ OPU mode to avoid fragmentation */ + if (f2fs_compressed_file(inode) && + F2FS_OPTION(sbi).compress_mode == COMPR_MODE_USER && + is_inode_flag_set(inode, FI_ENABLE_COMPRESS)) + return true; + + /* swap file is migrating in aligned write mode */ + if (is_inode_flag_set(inode, FI_ALIGNED_WRITE)) + return true; + + if (is_inode_flag_set(inode, FI_OPU_WRITE)) + return true; + + if (fio) { + if (page_private_gcing(fio->page)) + return true; + if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED) && + f2fs_is_checkpointed_data(sbi, fio->old_blkaddr))) + return true; + } + return false; +} + +static inline bool need_inplace_update(struct f2fs_io_info *fio) +{ + struct inode *inode = fio_inode(fio); + + if (f2fs_should_update_outplace(inode, fio)) + return false; + + return f2fs_should_update_inplace(inode, fio); +} + +int f2fs_do_write_data_page(struct f2fs_io_info *fio) +{ + struct folio *folio = fio->folio; + struct inode *inode = folio->mapping->host; + struct dnode_of_data dn; + struct node_info ni; + struct f2fs_lock_context lc; + bool ipu_force = false; + bool atomic_commit; + int err = 0; + + /* Use COW inode to make dnode_of_data for atomic write */ + atomic_commit = f2fs_is_atomic_file(inode) && + folio_test_f2fs_atomic(folio); + if (atomic_commit) + set_new_dnode(&dn, F2FS_I(inode)->cow_inode, NULL, NULL, 0); + else + set_new_dnode(&dn, inode, NULL, NULL, 0); + + if (need_inplace_update(fio) && + f2fs_lookup_read_extent_cache_block(inode, folio->index, + &fio->old_blkaddr)) { + if (!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr, + DATA_GENERIC_ENHANCE)) + return -EFSCORRUPTED; + + ipu_force = true; + fio->need_lock = LOCK_DONE; + goto got_it; + } + + if (is_sbi_flag_set(fio->sbi, SBI_ENABLE_CHECKPOINT) && + time_to_inject(fio->sbi, FAULT_SKIP_WRITE)) + return -EINVAL; + + /* Deadlock due to between page->lock and f2fs_lock_op */ + if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi, &lc)) + return -EAGAIN; + + err = f2fs_get_dnode_of_data(&dn, folio->index, LOOKUP_NODE); + if (err) + goto out; + + fio->old_blkaddr = dn.data_blkaddr; + + /* This page is already truncated */ + if (fio->old_blkaddr == NULL_ADDR) { + folio_clear_uptodate(folio); + folio_clear_f2fs_gcing(folio); + goto out_writepage; + } +got_it: + if (__is_valid_data_blkaddr(fio->old_blkaddr) && + !f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr, + DATA_GENERIC_ENHANCE)) { + err = -EFSCORRUPTED; + goto out_writepage; + } + + /* wait for GCed page writeback via META_MAPPING */ + if (fio->meta_gc) + f2fs_wait_on_block_writeback(inode, fio->old_blkaddr); + + /* + * If current allocation needs SSR, + * it had better in-place writes for updated data. + */ + if (ipu_force || + (__is_valid_data_blkaddr(fio->old_blkaddr) && + need_inplace_update(fio))) { + folio_start_writeback(folio); + f2fs_put_dnode(&dn); + if (fio->need_lock == LOCK_REQ) + f2fs_unlock_op(fio->sbi, &lc); + err = f2fs_inplace_write_data(fio); + if (err) + folio_end_writeback(folio); + else + set_inode_flag(inode, FI_UPDATE_WRITE); + trace_f2fs_do_write_data_page(folio, IPU); + return err; + } + + if (fio->need_lock == LOCK_RETRY) { + if (!f2fs_trylock_op(fio->sbi, &lc)) { + err = -EAGAIN; + goto out_writepage; + } + fio->need_lock = LOCK_REQ; + } + + err = f2fs_get_node_info(fio->sbi, dn.nid, &ni, false); + if (err) + goto out_writepage; + + fio->version = ni.version; + + folio_start_writeback(folio); + + if (fio->compr_blocks && fio->old_blkaddr == COMPRESS_ADDR) + f2fs_i_compr_blocks_update(inode, fio->compr_blocks - 1, false); + + /* LFS mode write path */ + f2fs_outplace_write_data(&dn, fio); + trace_f2fs_do_write_data_page(folio, OPU); + set_inode_flag(inode, FI_APPEND_WRITE); + if (atomic_commit) + folio_clear_f2fs_atomic(folio); +out_writepage: + f2fs_put_dnode(&dn); +out: + if (fio->need_lock == LOCK_REQ) + f2fs_unlock_op(fio->sbi, &lc); + return err; +} + +int f2fs_write_single_data_page(struct folio *folio, int *submitted, + struct bio **bio, + sector_t *last_block, + struct writeback_control *wbc, + enum iostat_type io_type, + int compr_blocks, + bool allow_balance) +{ + struct inode *inode = folio->mapping->host; + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + loff_t i_size = i_size_read(inode); + const pgoff_t end_index = ((unsigned long long)i_size) + >> PAGE_SHIFT; + loff_t psize = (loff_t)(folio->index + 1) << PAGE_SHIFT; + unsigned offset = 0; + bool need_balance_fs = false; + bool quota_inode = IS_NOQUOTA(inode); + int err = 0; + struct f2fs_io_info fio = { + .sbi = sbi, + .ino = inode->i_ino, + .type = DATA, + .op = REQ_OP_WRITE, + .op_flags = wbc_to_write_flags(wbc), + .old_blkaddr = NULL_ADDR, + .folio = folio, + .encrypted_page = NULL, + .submitted = 0, + .compr_blocks = compr_blocks, + .need_lock = compr_blocks ? LOCK_DONE : LOCK_RETRY, + .meta_gc = f2fs_meta_inode_gc_required(inode) ? 1 : 0, + .io_type = io_type, + .io_wbc = wbc, + .bio = bio, + .last_block = last_block, + }; + + trace_f2fs_writepage(folio, DATA); + + /* we should bypass data pages to proceed the kworker jobs */ + if (unlikely(f2fs_cp_error(sbi))) { + mapping_set_error(folio->mapping, -EIO); + /* + * don't drop any dirty dentry pages for keeping lastest + * directory structure. + */ + if (S_ISDIR(inode->i_mode) && + !is_sbi_flag_set(sbi, SBI_IS_CLOSE)) + goto redirty_out; + + /* keep data pages in remount-ro mode */ + if (F2FS_OPTION(sbi).errors == MOUNT_ERRORS_READONLY) + goto redirty_out; + goto out; + } + + if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) + goto redirty_out; + + if (folio->index < end_index || + f2fs_verity_in_progress(inode) || + compr_blocks) + goto write; + + /* + * If the offset is out-of-range of file size, + * this page does not have to be written to disk. + */ + offset = i_size & (PAGE_SIZE - 1); + if ((folio->index >= end_index + 1) || !offset) + goto out; + + folio_zero_segment(folio, offset, folio_size(folio)); +write: + /* Dentry/quota blocks are controlled by checkpoint */ + if (S_ISDIR(inode->i_mode) || quota_inode) { + struct f2fs_lock_context lc; + + /* + * We need to wait for node_write to avoid block allocation during + * checkpoint. This can only happen to quota writes which can cause + * the below discard race condition. + */ + if (quota_inode) + f2fs_down_read_trace(&sbi->node_write, &lc); + + fio.need_lock = LOCK_DONE; + err = f2fs_do_write_data_page(&fio); + + if (quota_inode) + f2fs_up_read_trace(&sbi->node_write, &lc); + + goto done; + } + + need_balance_fs = true; + err = -EAGAIN; + if (f2fs_has_inline_data(inode)) { + err = f2fs_write_inline_data(inode, folio); + if (!err) + goto out; + } + + if (err == -EAGAIN) { + err = f2fs_do_write_data_page(&fio); + if (err == -EAGAIN) { + f2fs_bug_on(sbi, compr_blocks); + fio.need_lock = LOCK_REQ; + err = f2fs_do_write_data_page(&fio); + } + } + + if (err) { + file_set_keep_isize(inode); + } else { + spin_lock(&F2FS_I(inode)->i_size_lock); + if (F2FS_I(inode)->last_disk_size < psize) + F2FS_I(inode)->last_disk_size = psize; + spin_unlock(&F2FS_I(inode)->i_size_lock); + } + +done: + if (err && err != -ENOENT) + goto redirty_out; + +out: + inode_dec_dirty_pages(inode); + if (err) { + folio_clear_uptodate(folio); + folio_clear_f2fs_gcing(folio); + } + folio_unlock(folio); + if (!S_ISDIR(inode->i_mode) && !IS_NOQUOTA(inode) && + !F2FS_I(inode)->wb_task && allow_balance) + f2fs_balance_fs(sbi, need_balance_fs); + + if (unlikely(f2fs_cp_error(sbi))) { + f2fs_submit_merged_write(sbi, DATA); + if (bio && *bio) + f2fs_submit_merged_ipu_write(sbi, bio, NULL); + submitted = NULL; + } + + if (submitted) + *submitted = fio.submitted; + + return 0; + +redirty_out: + folio_redirty_for_writepage(wbc, folio); + /* + * pageout() in MM translates EAGAIN, so calls handle_write_error() + * -> mapping_set_error() -> set_bit(AS_EIO, ...). + * file_write_and_wait_range() will see EIO error, which is critical + * to return value of fsync() followed by atomic_write failure to user. + */ + folio_unlock(folio); + if (!err) + return 1; + return err; +} + +/* + * This function was copied from write_cache_pages from mm/page-writeback.c. + * The major change is making write step of cold data page separately from + * warm/hot data page. + */ +static int f2fs_write_cache_pages(struct address_space *mapping, + struct writeback_control *wbc, + enum iostat_type io_type) +{ + int ret = 0; + int done = 0, retry = 0; + struct page *pages_local[F2FS_ONSTACK_PAGES]; + struct page **pages = pages_local; + struct folio_batch fbatch; + struct f2fs_sb_info *sbi = F2FS_M_SB(mapping); + struct bio *bio = NULL; + sector_t last_block; +#ifdef CONFIG_F2FS_FS_COMPRESSION + struct inode *inode = mapping->host; + struct compress_ctx cc = { + .inode = inode, + .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, + .cluster_size = F2FS_I(inode)->i_cluster_size, + .cluster_idx = NULL_CLUSTER, + .rpages = NULL, + .nr_rpages = 0, + .cpages = NULL, + .valid_nr_cpages = 0, + .rbuf = NULL, + .cbuf = NULL, + .rlen = PAGE_SIZE * F2FS_I(inode)->i_cluster_size, + .private = NULL, + }; +#endif + int nr_folios, p, idx; + int nr_pages; + unsigned int max_pages = F2FS_ONSTACK_PAGES; + pgoff_t index; + pgoff_t end; /* Inclusive */ + pgoff_t done_index; + int range_whole = 0; + xa_mark_t tag; + int nwritten = 0; + int submitted = 0; + int i; + +#ifdef CONFIG_F2FS_FS_COMPRESSION + if (f2fs_compressed_file(inode) && + 1 << cc.log_cluster_size > F2FS_ONSTACK_PAGES) { + pages = f2fs_kzalloc(sbi, sizeof(struct page *) << + cc.log_cluster_size, GFP_NOFS | __GFP_NOFAIL); + max_pages = 1 << cc.log_cluster_size; + } +#endif + + folio_batch_init(&fbatch); + + if (get_dirty_pages(mapping->host) <= + SM_I(F2FS_M_SB(mapping))->min_hot_blocks) + set_inode_flag(mapping->host, FI_HOT_DATA); + else + clear_inode_flag(mapping->host, FI_HOT_DATA); + + if (wbc->range_cyclic) { + index = mapping->writeback_index; /* prev offset */ + end = -1; + } else { + index = wbc->range_start >> PAGE_SHIFT; + end = wbc->range_end >> PAGE_SHIFT; + if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) + range_whole = 1; + } + tag = wbc_to_tag(wbc); +retry: + retry = 0; + if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) + tag_pages_for_writeback(mapping, index, end); + done_index = index; + while (!done && !retry && (index <= end)) { + nr_pages = 0; +again: + nr_folios = filemap_get_folios_tag(mapping, &index, end, + tag, &fbatch); + if (nr_folios == 0) { + if (nr_pages) + goto write; + break; + } + + for (i = 0; i < nr_folios; i++) { + struct folio *folio = fbatch.folios[i]; + + idx = 0; + p = folio_nr_pages(folio); +add_more: + pages[nr_pages] = folio_page(folio, idx); + folio_get(folio); + if (++nr_pages == max_pages) { + index = folio->index + idx + 1; + folio_batch_release(&fbatch); + goto write; + } + if (++idx < p) + goto add_more; + } + folio_batch_release(&fbatch); + goto again; +write: + for (i = 0; i < nr_pages; i++) { + struct page *page = pages[i]; + struct folio *folio = page_folio(page); + bool need_readd; +readd: + need_readd = false; +#ifdef CONFIG_F2FS_FS_COMPRESSION + if (f2fs_compressed_file(inode)) { + void *fsdata = NULL; + struct page *pagep; + int ret2; + + ret = f2fs_init_compress_ctx(&cc); + if (ret) { + done = 1; + break; + } + + if (!f2fs_cluster_can_merge_page(&cc, + folio->index)) { + ret = f2fs_write_multi_pages(&cc, + &submitted, wbc, io_type); + if (!ret) + need_readd = true; + goto result; + } + + if (unlikely(f2fs_cp_error(sbi))) + goto lock_folio; + + if (!f2fs_cluster_is_empty(&cc)) + goto lock_folio; + + if (f2fs_all_cluster_page_ready(&cc, + pages, i, nr_pages, true)) + goto lock_folio; + + ret2 = f2fs_prepare_compress_overwrite( + inode, &pagep, + folio->index, &fsdata); + if (ret2 < 0) { + ret = ret2; + done = 1; + break; + } else if (ret2 && + (!f2fs_compress_write_end(inode, + fsdata, folio->index, 1) || + !f2fs_all_cluster_page_ready(&cc, + pages, i, nr_pages, + false))) { + retry = 1; + break; + } + } +#endif + /* give a priority to WB_SYNC threads */ + if (atomic_read(&sbi->wb_sync_req[DATA]) && + wbc->sync_mode == WB_SYNC_NONE) { + done = 1; + break; + } +#ifdef CONFIG_F2FS_FS_COMPRESSION +lock_folio: +#endif + done_index = folio->index; +retry_write: + folio_lock(folio); + + if (unlikely(folio->mapping != mapping)) { +continue_unlock: + folio_unlock(folio); + continue; + } + + if (!folio_test_dirty(folio)) { + /* someone wrote it for us */ + goto continue_unlock; + } + + if (folio_test_writeback(folio)) { + if (wbc->sync_mode == WB_SYNC_NONE) + goto continue_unlock; + f2fs_folio_wait_writeback(folio, DATA, true, true); + } + + if (!folio_clear_dirty_for_io(folio)) + goto continue_unlock; + +#ifdef CONFIG_F2FS_FS_COMPRESSION + if (f2fs_compressed_file(inode)) { + folio_get(folio); + f2fs_compress_ctx_add_page(&cc, folio); + continue; + } +#endif + submitted = 0; + ret = f2fs_write_single_data_page(folio, + &submitted, &bio, &last_block, + wbc, io_type, 0, true); +#ifdef CONFIG_F2FS_FS_COMPRESSION +result: +#endif + nwritten += submitted; + wbc->nr_to_write -= submitted; + + if (unlikely(ret)) { + /* + * keep nr_to_write, since vfs uses this to + * get # of written pages. + */ + if (ret == 1) { + ret = 0; + goto next; + } else if (ret == -EAGAIN) { + ret = 0; + if (wbc->sync_mode == WB_SYNC_ALL) { + f2fs_schedule_timeout( + DEFAULT_SCHEDULE_TIMEOUT); + goto retry_write; + } + goto next; + } + done_index = folio_next_index(folio); + done = 1; + break; + } + + if (wbc->nr_to_write <= 0 && + wbc->sync_mode == WB_SYNC_NONE) { + done = 1; + break; + } +next: + if (need_readd) + goto readd; + } + release_pages(pages, nr_pages); + cond_resched(); + } +#ifdef CONFIG_F2FS_FS_COMPRESSION + /* flush remained pages in compress cluster */ + if (f2fs_compressed_file(inode) && !f2fs_cluster_is_empty(&cc)) { + ret = f2fs_write_multi_pages(&cc, &submitted, wbc, io_type); + nwritten += submitted; + wbc->nr_to_write -= submitted; + if (ret) { + done = 1; + retry = 0; + } + } + if (f2fs_compressed_file(inode)) + f2fs_destroy_compress_ctx(&cc, false); +#endif + if (retry) { + index = 0; + end = -1; + goto retry; + } + if (wbc->range_cyclic && !done) + done_index = 0; + if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) + mapping->writeback_index = done_index; + + if (nwritten) + f2fs_submit_merged_write_cond(F2FS_M_SB(mapping), mapping->host, + NULL, 0, DATA); + /* submit cached bio of IPU write */ + if (bio) + f2fs_submit_merged_ipu_write(sbi, &bio, NULL); + +#ifdef CONFIG_F2FS_FS_COMPRESSION + if (pages != pages_local) + kfree(pages); +#endif + + return ret; +} + +static inline bool __should_serialize_io(struct inode *inode, + struct writeback_control *wbc) +{ + /* to avoid deadlock in path of data flush */ + if (F2FS_I(inode)->wb_task) + return false; + + if (!S_ISREG(inode->i_mode)) + return false; + if (IS_NOQUOTA(inode)) + return false; + + if (f2fs_is_pinned_file(inode)) + return false; + if (f2fs_need_compress_data(inode)) + return true; + if (wbc->sync_mode != WB_SYNC_ALL) + return true; + if (get_dirty_pages(inode) >= SM_I(F2FS_I_SB(inode))->min_seq_blocks) + return true; + return false; +} + +static inline void account_writeback(struct inode *inode, bool inc) +{ + if (!f2fs_sb_has_compression(F2FS_I_SB(inode))) + return; + + f2fs_down_read(&F2FS_I(inode)->i_sem); + if (inc) + atomic_inc(&F2FS_I(inode)->writeback); + else + atomic_dec(&F2FS_I(inode)->writeback); + f2fs_up_read(&F2FS_I(inode)->i_sem); +} + +static inline void update_skipped_write(struct f2fs_sb_info *sbi, + struct writeback_control *wbc) +{ + long skipped = wbc->pages_skipped; + + if (is_sbi_flag_set(sbi, SBI_ENABLE_CHECKPOINT) && skipped && + wbc->sync_mode == WB_SYNC_ALL) + atomic_add(skipped, &sbi->nr_pages[F2FS_SKIPPED_WRITE]); +} + +static int __f2fs_write_data_pages(struct address_space *mapping, + struct writeback_control *wbc, + enum iostat_type io_type) +{ + struct inode *inode = mapping->host; + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct blk_plug plug; + int ret; + bool locked = false; + + /* skip writing if there is no dirty page in this inode */ + if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE) + return 0; + + /* during POR, we don't need to trigger writepage at all. */ + if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) + goto skip_write; + + if ((S_ISDIR(inode->i_mode) || IS_NOQUOTA(inode)) && + wbc->sync_mode == WB_SYNC_NONE && + get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) && + f2fs_available_free_memory(sbi, DIRTY_DENTS)) + goto skip_write; + + /* skip writing in file defragment preparing stage */ + if (is_inode_flag_set(inode, FI_SKIP_WRITES)) + goto skip_write; + + trace_f2fs_writepages(mapping->host, wbc, DATA); + + /* to avoid spliting IOs due to mixed WB_SYNC_ALL and WB_SYNC_NONE */ + if (wbc->sync_mode == WB_SYNC_ALL) + atomic_inc(&sbi->wb_sync_req[DATA]); + else if (atomic_read(&sbi->wb_sync_req[DATA])) { + /* to avoid potential deadlock */ + if (current->plug) + blk_finish_plug(current->plug); + goto skip_write; + } + + if (__should_serialize_io(inode, wbc)) { + mutex_lock(&sbi->writepages); + locked = true; + } + + account_writeback(inode, true); + + blk_start_plug(&plug); + ret = f2fs_write_cache_pages(mapping, wbc, io_type); + blk_finish_plug(&plug); + + account_writeback(inode, false); + + if (locked) + mutex_unlock(&sbi->writepages); + + if (wbc->sync_mode == WB_SYNC_ALL) + atomic_dec(&sbi->wb_sync_req[DATA]); + /* + * if some pages were truncated, we cannot guarantee its mapping->host + * to detect pending bios. + */ + + f2fs_remove_dirty_inode(inode); + + /* + * f2fs_write_cache_pages() has retry logic for EAGAIN case which is + * common when racing w/ checkpoint, so only update skipped write + * when ret is non-zero. + */ + if (ret) + update_skipped_write(sbi, wbc); + return ret; + +skip_write: + wbc->pages_skipped += get_dirty_pages(inode); + update_skipped_write(sbi, wbc); + trace_f2fs_writepages(mapping->host, wbc, DATA); + return 0; +} + +static int f2fs_write_data_pages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct inode *inode = mapping->host; + + return __f2fs_write_data_pages(mapping, wbc, + F2FS_I(inode)->cp_task == current ? + FS_CP_DATA_IO : FS_DATA_IO); +} + +void f2fs_write_failed(struct inode *inode, loff_t to) +{ + loff_t i_size = i_size_read(inode); + + if (IS_NOQUOTA(inode)) + return; + + /* In the fs-verity case, f2fs_end_enable_verity() does the truncate */ + if (to > i_size && !f2fs_verity_in_progress(inode)) { + f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); + filemap_invalidate_lock(inode->i_mapping); + + truncate_pagecache(inode, i_size); + f2fs_truncate_blocks(inode, i_size, true); + + filemap_invalidate_unlock(inode->i_mapping); + f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); + } +} + +static int prepare_write_begin(struct f2fs_sb_info *sbi, + struct folio *folio, loff_t pos, unsigned int len, + block_t *blk_addr, bool *node_changed) +{ + struct inode *inode = folio->mapping->host; + pgoff_t index = folio->index; + struct dnode_of_data dn; + struct f2fs_lock_context lc; + struct folio *ifolio; + bool locked = false; + int flag = F2FS_GET_BLOCK_PRE_AIO; + int err = 0; + + if (!f2fs_has_inline_data(inode) && !f2fs_compressed_file(inode) && + (pos & PAGE_MASK) < i_size_read(inode) && + f2fs_lookup_read_extent_cache_block(inode, index, blk_addr)) + return 0; + + /* + * If a whole page is being written and we already preallocated all the + * blocks, then there is no need to get a block address now. + */ + if (len == PAGE_SIZE && is_inode_flag_set(inode, FI_PREALLOCATED_ALL)) + return 0; + + /* f2fs_lock_op avoids race between write CP and convert_inline_page */ + if (f2fs_has_inline_data(inode)) { + if (pos + len > MAX_INLINE_DATA(inode)) + flag = F2FS_GET_BLOCK_DEFAULT; + f2fs_map_lock(sbi, &lc, flag); + locked = true; + } else if ((pos & PAGE_MASK) >= i_size_read(inode)) { + f2fs_map_lock(sbi, &lc, flag); + locked = true; + } + +restart: + /* check inline_data */ + ifolio = f2fs_get_inode_folio(sbi, inode->i_ino); + if (IS_ERR(ifolio)) { + err = PTR_ERR(ifolio); + goto unlock_out; + } + + set_new_dnode(&dn, inode, ifolio, ifolio, 0); + + if (f2fs_has_inline_data(inode)) { + if (pos + len <= MAX_INLINE_DATA(inode)) { + f2fs_do_read_inline_data(folio, ifolio); + set_inode_flag(inode, FI_DATA_EXIST); + if (inode->i_nlink) + folio_set_f2fs_inline(ifolio); + goto out; + } + err = f2fs_convert_inline_folio(&dn, folio); + if (err || dn.data_blkaddr != NULL_ADDR) + goto out; + } + + if (!f2fs_lookup_read_extent_cache_block(inode, index, + &dn.data_blkaddr)) { + if (IS_DEVICE_ALIASING(inode)) { + err = -ENODATA; + goto out; + } + + if (locked) { + err = f2fs_reserve_block(&dn, index); + goto out; + } + + /* hole case */ + err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE); + if (!err && dn.data_blkaddr != NULL_ADDR) + goto out; + f2fs_put_dnode(&dn); + f2fs_map_lock(sbi, &lc, F2FS_GET_BLOCK_PRE_AIO); + WARN_ON(flag != F2FS_GET_BLOCK_PRE_AIO); + locked = true; + goto restart; + } +out: + if (!err) { + /* convert_inline_page can make node_changed */ + *blk_addr = dn.data_blkaddr; + *node_changed = dn.node_changed; + } + f2fs_put_dnode(&dn); +unlock_out: + if (locked) + f2fs_map_unlock(sbi, &lc, flag); + return err; +} + +static int __find_data_block(struct inode *inode, pgoff_t index, + block_t *blk_addr) +{ + struct dnode_of_data dn; + struct folio *ifolio; + int err = 0; + + ifolio = f2fs_get_inode_folio(F2FS_I_SB(inode), inode->i_ino); + if (IS_ERR(ifolio)) + return PTR_ERR(ifolio); + + set_new_dnode(&dn, inode, ifolio, ifolio, 0); + + if (!f2fs_lookup_read_extent_cache_block(inode, index, + &dn.data_blkaddr)) { + /* hole case */ + err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE); + if (err) { + dn.data_blkaddr = NULL_ADDR; + err = 0; + } + } + *blk_addr = dn.data_blkaddr; + f2fs_put_dnode(&dn); + return err; +} + +static int __reserve_data_block(struct inode *inode, pgoff_t index, + block_t *blk_addr, bool *node_changed) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct dnode_of_data dn; + struct f2fs_lock_context lc; + struct folio *ifolio; + int err = 0; + + f2fs_map_lock(sbi, &lc, F2FS_GET_BLOCK_PRE_AIO); + + ifolio = f2fs_get_inode_folio(sbi, inode->i_ino); + if (IS_ERR(ifolio)) { + err = PTR_ERR(ifolio); + goto unlock_out; + } + set_new_dnode(&dn, inode, ifolio, ifolio, 0); + + if (!f2fs_lookup_read_extent_cache_block(dn.inode, index, + &dn.data_blkaddr)) + err = f2fs_reserve_block(&dn, index); + + *blk_addr = dn.data_blkaddr; + *node_changed = dn.node_changed; + f2fs_put_dnode(&dn); + +unlock_out: + f2fs_map_unlock(sbi, &lc, F2FS_GET_BLOCK_PRE_AIO); + return err; +} + +static int prepare_atomic_write_begin(struct f2fs_sb_info *sbi, + struct folio *folio, loff_t pos, unsigned int len, + block_t *blk_addr, bool *node_changed) +{ + struct inode *inode = folio->mapping->host; + struct inode *cow_inode = F2FS_I(inode)->cow_inode; + pgoff_t index = folio->index; + int err = 0; + block_t ori_blk_addr = NULL_ADDR; + bool cow_has_reserved_block = false; + + /* If pos is beyond the end of file, reserve a new block in COW inode */ + if ((pos & PAGE_MASK) >= i_size_read(inode)) + goto reserve_block; + + /* Look for the block in COW inode first */ + err = __find_data_block(cow_inode, index, blk_addr); + if (err) + return err; + + if (__is_valid_data_blkaddr(*blk_addr)) + return 0; + + if (*blk_addr == NEW_ADDR) + cow_has_reserved_block = true; + + if (is_inode_flag_set(inode, FI_ATOMIC_REPLACE)) + goto reserve_block; + + /* Look for the block in the original inode */ + err = __find_data_block(inode, index, &ori_blk_addr); + if (err) + return err; + +reserve_block: + /* Finally, we should reserve a new block in COW inode for the update */ + if (!cow_has_reserved_block) { + err = __reserve_data_block(cow_inode, index, blk_addr, + node_changed); + if (err) + return err; + inc_atomic_write_cnt(inode); + } + + if (ori_blk_addr != NULL_ADDR) + *blk_addr = ori_blk_addr; + return 0; +} + +static int f2fs_write_begin(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, struct folio **foliop, + void **fsdata) +{ + struct inode *inode = mapping->host; + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct folio *folio; + pgoff_t index = pos >> PAGE_SHIFT; + bool need_balance = false; + block_t blkaddr = NULL_ADDR; + int err = 0; + + trace_f2fs_write_begin(inode, pos, len); + + if (!f2fs_is_checkpoint_ready(sbi)) { + err = -ENOSPC; + goto fail; + } + + /* + * We should check this at this moment to avoid deadlock on inode page + * and #0 page. The locking rule for inline_data conversion should be: + * folio_lock(folio #0) -> folio_lock(inode_page) + */ + if (index != 0) { + err = f2fs_convert_inline_inode(inode); + if (err) + goto fail; + } + +#ifdef CONFIG_F2FS_FS_COMPRESSION + if (f2fs_compressed_file(inode)) { + int ret; + struct page *page; + + *fsdata = NULL; + + if (len == PAGE_SIZE && !(f2fs_is_atomic_file(inode))) + goto repeat; + + ret = f2fs_prepare_compress_overwrite(inode, &page, + index, fsdata); + if (ret < 0) { + err = ret; + goto fail; + } else if (ret) { + *foliop = page_folio(page); + return 0; + } + } +#endif + +repeat: + /* + * Do not use FGP_STABLE to avoid deadlock. + * Will wait that below with our IO control. + */ + folio = f2fs_filemap_get_folio(mapping, index, + FGP_LOCK | FGP_WRITE | FGP_CREAT | FGP_NOFS, + mapping_gfp_mask(mapping)); + if (IS_ERR(folio)) { + err = PTR_ERR(folio); + goto fail; + } + + /* TODO: cluster can be compressed due to race with .writepage */ + + *foliop = folio; + + if (f2fs_is_atomic_file(inode)) + err = prepare_atomic_write_begin(sbi, folio, pos, len, + &blkaddr, &need_balance); + else + err = prepare_write_begin(sbi, folio, pos, len, + &blkaddr, &need_balance); + if (err) + goto put_folio; + + if (need_balance && !IS_NOQUOTA(inode) && + has_not_enough_free_secs(sbi, 0, 0)) { + folio_unlock(folio); + f2fs_balance_fs(sbi, true); + folio_lock(folio); + if (folio->mapping != mapping) { + /* The folio got truncated from under us */ + folio_unlock(folio); + folio_put(folio); + goto repeat; + } + } + + f2fs_folio_wait_writeback(folio, DATA, false, true); + + if (len == folio_size(folio) || folio_test_uptodate(folio)) + return 0; + + if (!(pos & (PAGE_SIZE - 1)) && (pos + len) >= i_size_read(inode) && + !f2fs_verity_in_progress(inode)) { + folio_zero_segment(folio, len, folio_size(folio)); + return 0; + } + + if (blkaddr == NEW_ADDR) { + folio_zero_segment(folio, 0, folio_size(folio)); + folio_mark_uptodate(folio); + } else { + if (!f2fs_is_valid_blkaddr(sbi, blkaddr, + DATA_GENERIC_ENHANCE_READ)) { + err = -EFSCORRUPTED; + goto put_folio; + } + /* + * Although the block may be stored in the COW inode, the folio + * belongs to @inode and its data was encrypted (or not) using + * @inode's context (see f2fs_set_bio_crypt_ctx()). Read with + * @inode so the post-read decryption decision matches the + * folio's owner; otherwise an unencrypted @inode whose COW inode + * is encrypted hits a NULL ->i_crypt_info on decryption. + */ + f2fs_submit_page_read(inode, + NULL, /* can't write to fsverity files */ + folio, blkaddr, 0, true); + + folio_lock(folio); + if (unlikely(folio->mapping != mapping)) { + folio_unlock(folio); + folio_put(folio); + goto repeat; + } + if (unlikely(!folio_test_uptodate(folio))) { + err = -EIO; + goto put_folio; + } + } + return 0; + +put_folio: + f2fs_folio_put(folio, true); +fail: + f2fs_write_failed(inode, pos + len); + return err; +} + +static int f2fs_write_end(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct folio *folio, void *fsdata) +{ + struct inode *inode = folio->mapping->host; + + trace_f2fs_write_end(inode, pos, len, copied); + + /* + * This should be come from len == PAGE_SIZE, and we expect copied + * should be PAGE_SIZE. Otherwise, we treat it with zero copied and + * let generic_perform_write() try to copy data again through copied=0. + */ + if (!folio_test_uptodate(folio)) { + if (unlikely(copied != len)) + copied = 0; + else + folio_mark_uptodate(folio); + } + +#ifdef CONFIG_F2FS_FS_COMPRESSION + /* overwrite compressed file */ + if (f2fs_compressed_file(inode) && fsdata) { + f2fs_compress_write_end(inode, fsdata, folio->index, copied); + f2fs_update_time(F2FS_I_SB(inode), REQ_TIME); + + if (pos + copied > i_size_read(inode) && + !f2fs_verity_in_progress(inode)) + f2fs_i_size_write(inode, pos + copied); + return copied; + } +#endif + + if (!copied) + goto unlock_out; + + folio_mark_dirty(folio); + + if (f2fs_is_atomic_file(inode)) + folio_set_f2fs_atomic(folio); + + if (pos + copied > i_size_read(inode) && + !f2fs_verity_in_progress(inode)) { + f2fs_i_size_write(inode, pos + copied); + if (f2fs_is_atomic_file(inode)) + f2fs_i_size_write(F2FS_I(inode)->cow_inode, + pos + copied); + } +unlock_out: + f2fs_folio_put(folio, true); + f2fs_update_time(F2FS_I_SB(inode), REQ_TIME); + return copied; +} + +void f2fs_invalidate_folio(struct folio *folio, size_t offset, size_t length) +{ + struct inode *inode = folio->mapping->host; + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + + if (inode->i_ino >= F2FS_ROOT_INO(sbi) && + (offset || length != folio_size(folio))) + return; + + if (folio_test_dirty(folio)) { + if (inode->i_ino == F2FS_META_INO(sbi)) { + dec_page_count(sbi, F2FS_DIRTY_META); + } else if (inode->i_ino == F2FS_NODE_INO(sbi)) { + dec_page_count(sbi, F2FS_DIRTY_NODES); + } else { + inode_dec_dirty_pages(inode); + f2fs_remove_dirty_inode(inode); + } + } + + if (offset || length != folio_size(folio)) + return; + + folio_cancel_dirty(folio); + ffs_detach_free(folio); +} + +bool f2fs_release_folio(struct folio *folio, gfp_t wait) +{ + /* If this is dirty folio, keep private data */ + if (folio_test_dirty(folio)) + return false; + + ffs_detach_free(folio); + return true; +} + +static bool f2fs_dirty_data_folio(struct address_space *mapping, + struct folio *folio) +{ + struct inode *inode = mapping->host; + + trace_f2fs_set_page_dirty(folio, DATA); + + if (!folio_test_uptodate(folio)) + folio_mark_uptodate(folio); + BUG_ON(folio_test_swapcache(folio)); + + if (filemap_dirty_folio(mapping, folio)) { + f2fs_update_dirty_folio(inode, folio); + return true; + } + return false; +} + + +static sector_t f2fs_bmap_compress(struct inode *inode, sector_t block) +{ +#ifdef CONFIG_F2FS_FS_COMPRESSION + struct dnode_of_data dn; + sector_t start_idx, blknr = 0; + int ret; + + start_idx = round_down(block, F2FS_I(inode)->i_cluster_size); + + set_new_dnode(&dn, inode, NULL, NULL, 0); + ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE); + if (ret) + return 0; + + if (dn.data_blkaddr != COMPRESS_ADDR) { + dn.ofs_in_node += block - start_idx; + blknr = f2fs_data_blkaddr(&dn); + if (!__is_valid_data_blkaddr(blknr)) + blknr = 0; + } + + f2fs_put_dnode(&dn); + return blknr; +#else + return 0; +#endif +} + + +static sector_t f2fs_bmap(struct address_space *mapping, sector_t block) +{ + struct inode *inode = mapping->host; + sector_t blknr = 0; + + if (f2fs_has_inline_data(inode)) + goto out; + + /* make sure allocating whole blocks */ + if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) + filemap_write_and_wait(mapping); + + /* Block number less than F2FS MAX BLOCKS */ + if (unlikely(block >= max_file_blocks(inode))) + goto out; + + if (f2fs_compressed_file(inode)) { + blknr = f2fs_bmap_compress(inode, block); + } else { + struct f2fs_map_blocks map; + + memset(&map, 0, sizeof(map)); + map.m_lblk = block; + map.m_len = 1; + map.m_next_pgofs = NULL; + map.m_seg_type = NO_CHECK_TYPE; + + if (!f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_BMAP)) + blknr = map.m_pblk; + } +out: + trace_f2fs_bmap(inode, block, blknr); + return blknr; +} + +#ifdef CONFIG_SWAP +static int f2fs_migrate_blocks(struct inode *inode, block_t start_blk, + unsigned int blkcnt) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + unsigned int blkofs; + unsigned int blk_per_sec = BLKS_PER_SEC(sbi); + unsigned int end_blk = start_blk + blkcnt - 1; + unsigned int secidx = start_blk / blk_per_sec; + unsigned int end_sec; + int ret = 0; + + if (!blkcnt) + return 0; + end_sec = end_blk / blk_per_sec; + + f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); + filemap_invalidate_lock(inode->i_mapping); + + set_inode_flag(inode, FI_ALIGNED_WRITE); + set_inode_flag(inode, FI_OPU_WRITE); + + for (; secidx <= end_sec; secidx++) { + unsigned int blkofs_end = secidx == end_sec ? + end_blk % blk_per_sec : blk_per_sec - 1; + + f2fs_down_write(&sbi->pin_sem); + + ret = f2fs_allocate_pinning_section(sbi); + if (ret) { + f2fs_up_write(&sbi->pin_sem); + break; + } + + set_inode_flag(inode, FI_SKIP_WRITES); + + for (blkofs = 0; blkofs <= blkofs_end; blkofs++) { + struct folio *folio; + unsigned int blkidx = secidx * blk_per_sec + blkofs; + + folio = f2fs_get_lock_data_folio(inode, blkidx, true); + if (IS_ERR(folio)) { + f2fs_up_write(&sbi->pin_sem); + ret = PTR_ERR(folio); + goto done; + } + + folio_mark_dirty(folio); + f2fs_folio_put(folio, true); + } + + clear_inode_flag(inode, FI_SKIP_WRITES); + + ret = filemap_fdatawrite(inode->i_mapping); + + f2fs_up_write(&sbi->pin_sem); + + if (ret) + break; + } + +done: + clear_inode_flag(inode, FI_SKIP_WRITES); + clear_inode_flag(inode, FI_OPU_WRITE); + clear_inode_flag(inode, FI_ALIGNED_WRITE); + + filemap_invalidate_unlock(inode->i_mapping); + f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); + + return ret; +} + +static int check_swap_activate(struct swap_info_struct *sis, + struct file *swap_file, sector_t *span) +{ + struct address_space *mapping = swap_file->f_mapping; + struct inode *inode = mapping->host; + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + block_t cur_lblock; + block_t last_lblock; + block_t pblock; + block_t lowest_pblock = -1; + block_t highest_pblock = 0; + int nr_extents = 0; + unsigned int nr_pblocks; + unsigned int blks_per_sec = BLKS_PER_SEC(sbi); + unsigned int not_aligned = 0; + int ret = 0; + + /* + * Map all the blocks into the extent list. This code doesn't try + * to be very smart. + */ + cur_lblock = 0; + last_lblock = F2FS_BYTES_TO_BLK(i_size_read(inode)); + + while (cur_lblock < last_lblock && cur_lblock < sis->max) { + struct f2fs_map_blocks map; + bool last_extent = false; +retry: + cond_resched(); + + memset(&map, 0, sizeof(map)); + map.m_lblk = cur_lblock; + map.m_len = last_lblock - cur_lblock; + map.m_next_pgofs = NULL; + map.m_next_extent = NULL; + map.m_seg_type = NO_CHECK_TYPE; + map.m_may_create = false; + + ret = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_FIEMAP); + if (ret) + goto out; + + /* hole */ + if (!(map.m_flags & F2FS_MAP_FLAGS)) { + f2fs_err(sbi, "Swapfile has holes"); + ret = -EINVAL; + goto out; + } + + pblock = map.m_pblk; + nr_pblocks = map.m_len; + + if (!last_extent && + ((pblock - SM_I(sbi)->main_blkaddr) % blks_per_sec || + nr_pblocks % blks_per_sec || + f2fs_is_sequential_zone_area(sbi, pblock))) { + not_aligned++; + + nr_pblocks = roundup(nr_pblocks, blks_per_sec); + if (cur_lblock + nr_pblocks > sis->max) + nr_pblocks -= blks_per_sec; + + /* this extent is last one */ + if (!nr_pblocks) { + nr_pblocks = last_lblock - cur_lblock; + last_extent = true; + } + + ret = f2fs_migrate_blocks(inode, cur_lblock, + nr_pblocks); + if (ret) { + if (ret == -ENOENT) + ret = -EINVAL; + goto out; + } + + /* lookup block mapping info after block migration */ + goto retry; + } + + if (cur_lblock + nr_pblocks >= sis->max) + nr_pblocks = sis->max - cur_lblock; + + if (cur_lblock) { /* exclude the header page */ + if (pblock < lowest_pblock) + lowest_pblock = pblock; + if (pblock + nr_pblocks - 1 > highest_pblock) + highest_pblock = pblock + nr_pblocks - 1; + } + + /* + * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks + */ + ret = add_swap_extent(sis, cur_lblock, nr_pblocks, pblock); + if (ret < 0) + goto out; + nr_extents += ret; + cur_lblock += nr_pblocks; + } + ret = nr_extents; + *span = 1 + highest_pblock - lowest_pblock; + if (cur_lblock == 0) + cur_lblock = 1; /* force Empty message */ + sis->max = cur_lblock; + sis->pages = cur_lblock - 1; +out: + if (not_aligned) + f2fs_warn(sbi, "Swapfile (%u) is not align to section: 1) creat(), 2) ioctl(F2FS_IOC_SET_PIN_FILE), 3) fallocate(%lu * N)", + not_aligned, blks_per_sec * F2FS_BLKSIZE); + return ret; +} + +static int f2fs_swap_activate(struct swap_info_struct *sis, struct file *file, + sector_t *span) +{ + struct inode *inode = file_inode(file); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + int ret; + + if (!S_ISREG(inode->i_mode)) + return -EINVAL; + + if (f2fs_readonly(sbi->sb)) + return -EROFS; + + if (f2fs_lfs_mode(sbi) && !f2fs_sb_has_blkzoned(sbi)) { + f2fs_err(sbi, "Swapfile not supported in LFS mode"); + return -EINVAL; + } + + ret = f2fs_convert_inline_inode(inode); + if (ret) + return ret; + + if (!f2fs_disable_compressed_file(inode)) + return -EINVAL; + + ret = filemap_fdatawrite(inode->i_mapping); + if (ret < 0) + return ret; + + f2fs_precache_extents(inode); + + ret = check_swap_activate(sis, file, span); + if (ret < 0) + return ret; + + stat_inc_swapfile_inode(inode); + set_inode_flag(inode, FI_PIN_FILE); + f2fs_update_time(sbi, REQ_TIME); + return ret; +} + +static void f2fs_swap_deactivate(struct file *file) +{ + struct inode *inode = file_inode(file); + + stat_dec_swapfile_inode(inode); + clear_inode_flag(inode, FI_PIN_FILE); +} +#else +static int f2fs_swap_activate(struct swap_info_struct *sis, struct file *file, + sector_t *span) +{ + return -EOPNOTSUPP; +} + +static void f2fs_swap_deactivate(struct file *file) +{ +} +#endif + +const struct address_space_operations f2fs_dblock_aops = { + .read_folio = f2fs_read_data_folio, + .readahead = f2fs_readahead, + .writepages = f2fs_write_data_pages, + .write_begin = f2fs_write_begin, + .write_end = f2fs_write_end, + .dirty_folio = f2fs_dirty_data_folio, + .migrate_folio = filemap_migrate_folio, + .invalidate_folio = f2fs_invalidate_folio, + .release_folio = f2fs_release_folio, + .bmap = f2fs_bmap, + .swap_activate = f2fs_swap_activate, + .swap_deactivate = f2fs_swap_deactivate, +}; + +void f2fs_clear_page_cache_dirty_tag(struct folio *folio) +{ + struct address_space *mapping = folio->mapping; + unsigned long flags; + + xa_lock_irqsave(&mapping->i_pages, flags); + __xa_clear_mark(&mapping->i_pages, folio->index, + PAGECACHE_TAG_DIRTY); + xa_unlock_irqrestore(&mapping->i_pages, flags); +} + +int __init f2fs_init_post_read_processing(void) +{ + bio_post_read_ctx_cache = + kmem_cache_create("f2fs_bio_post_read_ctx", + sizeof(struct bio_post_read_ctx), 0, 0, NULL); + if (!bio_post_read_ctx_cache) + goto fail; + bio_post_read_ctx_pool = + mempool_create_slab_pool(NUM_PREALLOC_POST_READ_CTXS, + bio_post_read_ctx_cache); + if (!bio_post_read_ctx_pool) + goto fail_free_cache; + return 0; + +fail_free_cache: + kmem_cache_destroy(bio_post_read_ctx_cache); +fail: + return -ENOMEM; +} + +void f2fs_destroy_post_read_processing(void) +{ + mempool_destroy(bio_post_read_ctx_pool); + kmem_cache_destroy(bio_post_read_ctx_cache); +} + +int f2fs_init_wq(struct f2fs_sb_info *sbi) +{ + sbi->wq = alloc_workqueue("f2fs_wq", WQ_UNBOUND | WQ_HIGHPRI, + num_online_cpus()); + return sbi->wq ? 0 : -ENOMEM; +} + +void f2fs_destroy_wq(struct f2fs_sb_info *sbi) +{ + if (sbi->wq) + destroy_workqueue(sbi->wq); +} + +int __init f2fs_init_bio_entry_cache(void) +{ + bio_entry_slab = f2fs_kmem_cache_create("f2fs_bio_entry_slab", + sizeof(struct bio_entry)); + + if (!bio_entry_slab) + return -ENOMEM; + + ffs_entry_slab = f2fs_kmem_cache_create("f2fs_ffs_slab", + sizeof(struct f2fs_folio_state)); + + if (!ffs_entry_slab) { + kmem_cache_destroy(bio_entry_slab); + return -ENOMEM; + } + + return 0; +} + +void f2fs_destroy_bio_entry_cache(void) +{ + kmem_cache_destroy(bio_entry_slab); + kmem_cache_destroy(ffs_entry_slab); +} + +static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, + unsigned int flags, struct iomap *iomap, + struct iomap *srcmap) +{ + struct f2fs_map_blocks map = { NULL, }; + pgoff_t next_pgofs = 0; + int err; + + map.m_lblk = F2FS_BYTES_TO_BLK(offset); + map.m_len = F2FS_BYTES_TO_BLK(offset + length - 1) - map.m_lblk + 1; + map.m_next_pgofs = &next_pgofs; + map.m_seg_type = f2fs_rw_hint_to_seg_type(F2FS_I_SB(inode), + inode->i_write_hint); + if (flags & IOMAP_WRITE && iomap->private) { + map.m_last_pblk = (unsigned long)iomap->private; + iomap->private = NULL; + } + + /* + * If the blocks being overwritten are already allocated, + * f2fs_map_lock and f2fs_balance_fs are not necessary. + */ + if ((flags & IOMAP_WRITE) && + !__f2fs_overwrite_io(inode, offset, length, true)) + map.m_may_create = true; + + err = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_DIO); + if (err) + return err; + + iomap->offset = F2FS_BLK_TO_BYTES(map.m_lblk); + + /* + * Sometimes I/O to an encrypted file has to be broken up to guarantee + * DUN contiguity. Handle this by limiting the length of the mapping + * returned. + */ + map.m_len = fscrypt_limit_io_blocks(inode, map.m_lblk, map.m_len); + + /* + * We should never see delalloc or compressed extents here based on + * prior flushing and checks. + */ + if (WARN_ON_ONCE(map.m_pblk == COMPRESS_ADDR)) + return -EINVAL; + + if (map.m_flags & F2FS_MAP_MAPPED) { + if (WARN_ON_ONCE(map.m_pblk == NEW_ADDR)) + return -EINVAL; + + iomap->length = F2FS_BLK_TO_BYTES(map.m_len); + iomap->type = IOMAP_MAPPED; + iomap->flags |= IOMAP_F_MERGED; + iomap->bdev = map.m_bdev; + iomap->addr = F2FS_BLK_TO_BYTES(map.m_pblk); + + if (flags & IOMAP_WRITE && map.m_last_pblk) + iomap->private = (void *)map.m_last_pblk; + } else { + if (flags & IOMAP_WRITE) + return -ENOTBLK; + + if (map.m_pblk == NULL_ADDR) { + iomap->length = F2FS_BLK_TO_BYTES(next_pgofs) - + iomap->offset; + iomap->type = IOMAP_HOLE; + } else if (map.m_pblk == NEW_ADDR) { + iomap->length = F2FS_BLK_TO_BYTES(map.m_len); + iomap->type = IOMAP_UNWRITTEN; + } else { + f2fs_bug_on(F2FS_I_SB(inode), 1); + } + iomap->addr = IOMAP_NULL_ADDR; + } + + if (map.m_flags & F2FS_MAP_NEW) + iomap->flags |= IOMAP_F_NEW; + if ((inode_state_read_once(inode) & I_DIRTY_DATASYNC) || + offset + length > i_size_read(inode)) + iomap->flags |= IOMAP_F_DIRTY; + + return 0; +} + +static DEFINE_IOMAP_ITER_NEXT(f2fs_iomap_next, f2fs_iomap_begin); + +const struct iomap_ops f2fs_iomap_ops = { + .iomap_next = f2fs_iomap_next, +}; diff --git a/fs/f2fs/debug.c b/fs/f2fs/debug.c new file mode 100644 index 000000000..af88db8fd --- /dev/null +++ b/fs/f2fs/debug.c @@ -0,0 +1,864 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * f2fs debugging statistics + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * Copyright (c) 2012 Linux Foundation + * Copyright (c) 2012 Greg Kroah-Hartman + */ + +#include +#include +#include +#include +#include +#include + +#include "f2fs.h" +#include "node.h" +#include "segment.h" +#include "gc.h" + +static LIST_HEAD(f2fs_stat_list); +static DEFINE_SPINLOCK(f2fs_stat_lock); +#ifdef CONFIG_DEBUG_FS +static struct dentry *f2fs_debugfs_root; +#endif + +/* + * This function calculates BDF of every segments + */ +void f2fs_update_sit_info(struct f2fs_sb_info *sbi) +{ + struct f2fs_stat_info *si = F2FS_STAT(sbi); + unsigned long long blks_per_sec, hblks_per_sec, total_vblocks; + unsigned long long bimodal, dist; + unsigned int segno, vblocks; + int ndirty = 0; + + bimodal = 0; + total_vblocks = 0; + blks_per_sec = CAP_BLKS_PER_SEC(sbi); + hblks_per_sec = blks_per_sec / 2; + for (segno = 0; segno < MAIN_SEGS(sbi); segno += SEGS_PER_SEC(sbi)) { + vblocks = get_valid_blocks(sbi, segno, true); + dist = abs(vblocks - hblks_per_sec); + bimodal += dist * dist; + + if (vblocks > 0 && vblocks < blks_per_sec) { + total_vblocks += vblocks; + ndirty++; + } + } + dist = div_u64(MAIN_SECS(sbi) * hblks_per_sec * hblks_per_sec, 100); + si->bimodal = div64_u64(bimodal, dist); + if (si->dirty_count) + si->avg_vblocks = div_u64(total_vblocks, ndirty); + else + si->avg_vblocks = 0; +} + +#ifdef CONFIG_DEBUG_FS +static void update_multidevice_stats(struct f2fs_sb_info *sbi) +{ + struct f2fs_stat_info *si = F2FS_STAT(sbi); + struct f2fs_dev_stats *dev_stats = si->dev_stats; + int i, j; + + if (!f2fs_is_multi_device(sbi)) + return; + + memset(dev_stats, 0, sizeof(struct f2fs_dev_stats) * sbi->s_ndevs); + for (i = 0; i < sbi->s_ndevs; i++) { + unsigned int start_segno, end_segno; + block_t start_blk, end_blk; + + if (i == 0) { + start_blk = MAIN_BLKADDR(sbi); + end_blk = FDEV(i).end_blk + 1 - SEG0_BLKADDR(sbi); + } else { + start_blk = FDEV(i).start_blk; + end_blk = FDEV(i).end_blk + 1; + } + + start_segno = GET_SEGNO(sbi, start_blk); + end_segno = GET_SEGNO(sbi, end_blk); + + for (j = start_segno; j < end_segno; j++) { + unsigned int seg_blks, sec_blks; + + seg_blks = get_seg_entry(sbi, j)->valid_blocks; + + /* update segment stats */ + if (is_curseg(sbi, j)) + dev_stats[i].devstats[0][DEVSTAT_INUSE]++; + else if (seg_blks == BLKS_PER_SEG(sbi)) + dev_stats[i].devstats[0][DEVSTAT_FULL]++; + else if (seg_blks != 0) + dev_stats[i].devstats[0][DEVSTAT_DIRTY]++; + else if (!test_bit(j, FREE_I(sbi)->free_segmap)) + dev_stats[i].devstats[0][DEVSTAT_FREE]++; + else + dev_stats[i].devstats[0][DEVSTAT_PREFREE]++; + + if (!__is_large_section(sbi) || + (j % SEGS_PER_SEC(sbi)) != 0) + continue; + + sec_blks = get_sec_entry(sbi, j)->valid_blocks; + + /* update section stats */ + if (is_cursec(sbi, GET_SEC_FROM_SEG(sbi, j))) + dev_stats[i].devstats[1][DEVSTAT_INUSE]++; + else if (sec_blks == BLKS_PER_SEC(sbi)) + dev_stats[i].devstats[1][DEVSTAT_FULL]++; + else if (sec_blks != 0) + dev_stats[i].devstats[1][DEVSTAT_DIRTY]++; + else if (!test_bit(GET_SEC_FROM_SEG(sbi, j), + FREE_I(sbi)->free_secmap)) + dev_stats[i].devstats[1][DEVSTAT_FREE]++; + else + dev_stats[i].devstats[1][DEVSTAT_PREFREE]++; + } + } +} + +static void update_general_status(struct f2fs_sb_info *sbi) +{ + struct f2fs_stat_info *si = F2FS_STAT(sbi); + struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); + int i; + + /* these will be changed if online resize is done */ + si->main_area_segs = le32_to_cpu(raw_super->segment_count_main); + si->main_area_sections = le32_to_cpu(raw_super->section_count); + si->main_area_zones = si->main_area_sections / + le32_to_cpu(raw_super->secs_per_zone); + + /* general extent cache stats */ + for (i = 0; i < NR_EXTENT_CACHES; i++) { + struct extent_tree_info *eti = &sbi->extent_tree[i]; + + si->hit_cached[i] = atomic64_read(&sbi->read_hit_cached[i]); + si->hit_rbtree[i] = atomic64_read(&sbi->read_hit_rbtree[i]); + si->total_ext[i] = atomic64_read(&sbi->total_hit_ext[i]); + si->hit_total[i] = si->hit_cached[i] + si->hit_rbtree[i]; + si->ext_tree[i] = atomic_read(&eti->total_ext_tree); + si->zombie_tree[i] = atomic_read(&eti->total_zombie_tree); + si->ext_node[i] = atomic_read(&eti->total_ext_node); + } + /* read extent_cache only */ + si->hit_largest = atomic64_read(&sbi->read_hit_largest); + si->hit_total[EX_READ] += si->hit_largest; + + /* block age extent_cache only */ + si->allocated_data_blocks = atomic64_read(&sbi->allocated_data_blocks); + + /* validation check of the segment numbers */ + si->ndirty_node = get_pages(sbi, F2FS_DIRTY_NODES); + si->ndirty_dent = get_pages(sbi, F2FS_DIRTY_DENTS); + si->ndirty_meta = get_pages(sbi, F2FS_DIRTY_META); + si->ndirty_data = get_pages(sbi, F2FS_DIRTY_DATA); + si->ndirty_qdata = get_pages(sbi, F2FS_DIRTY_QDATA); + si->ndirty_imeta = get_pages(sbi, F2FS_DIRTY_IMETA); + si->ndirty_dirs = sbi->ndirty_inode[DIR_INODE]; + si->ndirty_files = sbi->ndirty_inode[FILE_INODE]; + si->ndonate_files = sbi->donate_files; + si->nquota_files = sbi->nquota_files; + si->ndirty_all = sbi->ndirty_inode[DIRTY_META]; + si->aw_cnt = atomic_read(&sbi->atomic_files); + si->max_aw_cnt = atomic_read(&sbi->max_aw_cnt); + si->nr_dio_read = get_pages(sbi, F2FS_DIO_READ); + si->nr_dio_write = get_pages(sbi, F2FS_DIO_WRITE); + si->nr_wb_cp_data = get_pages(sbi, F2FS_WB_CP_DATA); + si->nr_wb_data = get_pages(sbi, F2FS_WB_DATA); + si->nr_rd_data = get_pages(sbi, F2FS_RD_DATA); + si->nr_rd_node = get_pages(sbi, F2FS_RD_NODE); + si->nr_rd_meta = get_pages(sbi, F2FS_RD_META); + if (SM_I(sbi)->fcc_info) { + si->nr_flushed = + atomic_read(&SM_I(sbi)->fcc_info->issued_flush); + si->nr_flushing = + atomic_read(&SM_I(sbi)->fcc_info->queued_flush); + si->flush_list_empty = + llist_empty(&SM_I(sbi)->fcc_info->issue_list); + } + if (SM_I(sbi)->dcc_info) { + si->nr_discarded = + atomic_read(&SM_I(sbi)->dcc_info->issued_discard); + si->nr_discarding = + atomic_read(&SM_I(sbi)->dcc_info->queued_discard); + si->nr_discard_cmd = + atomic_read(&SM_I(sbi)->dcc_info->discard_cmd_cnt); + si->undiscard_blks = SM_I(sbi)->dcc_info->undiscard_blks; + } + si->nr_issued_ckpt = atomic_read(&sbi->cprc_info.issued_ckpt); + si->nr_total_ckpt = atomic_read(&sbi->cprc_info.total_ckpt); + si->nr_queued_ckpt = atomic_read(&sbi->cprc_info.queued_ckpt); + spin_lock(&sbi->cprc_info.stat_lock); + si->cur_ckpt_time = sbi->cprc_info.cur_time; + si->peak_ckpt_time = sbi->cprc_info.peak_time; + spin_unlock(&sbi->cprc_info.stat_lock); + si->total_count = BLKS_TO_SEGS(sbi, (int)sbi->user_block_count); + si->rsvd_segs = reserved_segments(sbi); + si->overp_segs = overprovision_segments(sbi); + si->valid_count = valid_user_blocks(sbi); + si->discard_blks = discard_blocks(sbi); + si->valid_node_count = valid_node_count(sbi); + si->valid_inode_count = valid_inode_count(sbi); + si->inline_xattr = atomic_read(&sbi->inline_xattr); + si->inline_inode = atomic_read(&sbi->inline_inode); + si->inline_dir = atomic_read(&sbi->inline_dir); + si->compr_inode = atomic_read(&sbi->compr_inode); + si->swapfile_inode = atomic_read(&sbi->swapfile_inode); + si->compr_blocks = atomic64_read(&sbi->compr_blocks); + si->append = sbi->im[APPEND_INO].ino_num; + si->update = sbi->im[UPDATE_INO].ino_num; + si->orphans = sbi->im[ORPHAN_INO].ino_num; + si->utilization = utilization(sbi); + + si->free_segs = free_segments(sbi); + si->free_secs = free_sections(sbi); + si->prefree_count = prefree_segments(sbi); + si->dirty_count = dirty_segments(sbi); + if (sbi->node_inode) + si->node_pages = NODE_MAPPING(sbi)->nrpages; + if (sbi->meta_inode) + si->meta_pages = META_MAPPING(sbi)->nrpages; +#ifdef CONFIG_F2FS_FS_COMPRESSION + if (sbi->compress_inode) { + si->compress_pages = COMPRESS_MAPPING(sbi)->nrpages; + si->compress_page_hit = atomic_read(&sbi->compress_page_hit); + } +#endif + si->nats = NM_I(sbi)->nat_cnt[TOTAL_NAT]; + si->dirty_nats = NM_I(sbi)->nat_cnt[DIRTY_NAT]; + si->sits = MAIN_SEGS(sbi); + si->dirty_sits = SIT_I(sbi)->dirty_sentries; + si->free_nids = NM_I(sbi)->nid_cnt[FREE_NID]; + si->avail_nids = NM_I(sbi)->available_nids; + si->alloc_nids = NM_I(sbi)->nid_cnt[PREALLOC_NID]; + si->io_skip_bggc = sbi->io_skip_bggc; + si->other_skip_bggc = sbi->other_skip_bggc; + si->util_free = (int)(BLKS_TO_SEGS(sbi, free_user_blocks(sbi))) + * 100 / (int)(sbi->user_block_count >> sbi->log_blocks_per_seg) + / 2; + si->util_valid = (int)(BLKS_TO_SEGS(sbi, written_block_count(sbi))) + * 100 / (int)(sbi->user_block_count >> sbi->log_blocks_per_seg) + / 2; + si->util_invalid = 50 - si->util_free - si->util_valid; + for (i = CURSEG_HOT_DATA; i < NO_CHECK_TYPE; i++) { + struct curseg_info *curseg = CURSEG_I(sbi, i); + + si->blkoff[i] = curseg->next_blkoff; + si->curseg[i] = curseg->segno; + si->cursec[i] = GET_SEC_FROM_SEG(sbi, curseg->segno); + si->curzone[i] = GET_ZONE_FROM_SEC(sbi, si->cursec[i]); + } + + for (i = META_CP; i < META_MAX; i++) + si->meta_count[i] = atomic_read(&sbi->meta_count[i]); + + for (i = 0; i < NO_CHECK_TYPE; i++) { + si->dirty_seg[i] = 0; + si->full_seg[i] = 0; + si->valid_blks[i] = 0; + } + + for (i = 0; i < MAIN_SEGS(sbi); i++) { + int blks = get_seg_entry(sbi, i)->valid_blocks; + int type = get_seg_entry(sbi, i)->type; + + if (!blks) + continue; + + if (blks == BLKS_PER_SEG(sbi)) + si->full_seg[type]++; + else + si->dirty_seg[type]++; + si->valid_blks[type] += blks; + } + + update_multidevice_stats(sbi); + + for (i = 0; i < MAX_CALL_TYPE; i++) + si->cp_call_count[i] = atomic_read(&sbi->cp_call_count[i]); + + for (i = 0; i < 2; i++) { + si->segment_count[i] = sbi->segment_count[i]; + si->block_count[i] = sbi->block_count[i]; + } + + si->inplace_count = atomic_read(&sbi->inplace_count); +} + +/* + * This function calculates memory footprint. + */ +static void update_mem_info(struct f2fs_sb_info *sbi) +{ + struct f2fs_stat_info *si = F2FS_STAT(sbi); + int i; + + if (si->base_mem) + goto get_cache; + + /* build stat */ + si->base_mem = sizeof(struct f2fs_stat_info); + + /* build superblock */ + si->base_mem += sizeof(struct f2fs_sb_info) + sbi->sb->s_blocksize; + si->base_mem += 2 * sizeof(struct f2fs_inode_info); + si->base_mem += sizeof(*sbi->ckpt); + + /* build sm */ + si->base_mem += sizeof(struct f2fs_sm_info); + + /* build sit */ + si->base_mem += sizeof(struct sit_info); + si->base_mem += MAIN_SEGS(sbi) * sizeof(struct seg_entry); + si->base_mem += f2fs_bitmap_size(MAIN_SEGS(sbi)); + si->base_mem += 2 * SIT_VBLOCK_MAP_SIZE * MAIN_SEGS(sbi); + si->base_mem += SIT_VBLOCK_MAP_SIZE * MAIN_SEGS(sbi); + si->base_mem += SIT_VBLOCK_MAP_SIZE; + if (__is_large_section(sbi)) + si->base_mem += MAIN_SECS(sbi) * sizeof(struct sec_entry); + si->base_mem += __bitmap_size(sbi, SIT_BITMAP); + + /* build free segmap */ + si->base_mem += sizeof(struct free_segmap_info); + si->base_mem += f2fs_bitmap_size(MAIN_SEGS(sbi)); + si->base_mem += f2fs_bitmap_size(MAIN_SECS(sbi)); + + /* build curseg */ + si->base_mem += sizeof(struct curseg_info) * NR_CURSEG_TYPE; + si->base_mem += PAGE_SIZE * NR_CURSEG_TYPE; + + /* build dirty segmap */ + si->base_mem += sizeof(struct dirty_seglist_info); + si->base_mem += NR_DIRTY_TYPE * f2fs_bitmap_size(MAIN_SEGS(sbi)); + si->base_mem += f2fs_bitmap_size(MAIN_SECS(sbi)); + + /* build nm */ + si->base_mem += sizeof(struct f2fs_nm_info); + si->base_mem += __bitmap_size(sbi, NAT_BITMAP); + si->base_mem += F2FS_BLK_TO_BYTES(NM_I(sbi)->nat_bits_blocks); + si->base_mem += NM_I(sbi)->nat_blocks * + f2fs_bitmap_size(NAT_ENTRY_PER_BLOCK); + si->base_mem += NM_I(sbi)->nat_blocks / 8; + si->base_mem += NM_I(sbi)->nat_blocks * sizeof(unsigned short); + +get_cache: + si->cache_mem = 0; + + /* build gc */ + if (sbi->gc_thread) + si->cache_mem += sizeof(struct f2fs_gc_kthread); + + /* build merge flush thread */ + if (SM_I(sbi)->fcc_info) + si->cache_mem += sizeof(struct flush_cmd_control); + if (SM_I(sbi)->dcc_info) { + si->cache_mem += sizeof(struct discard_cmd_control); + si->cache_mem += sizeof(struct discard_cmd) * + atomic_read(&SM_I(sbi)->dcc_info->discard_cmd_cnt); + } + + /* free nids */ + si->cache_mem += (NM_I(sbi)->nid_cnt[FREE_NID] + + NM_I(sbi)->nid_cnt[PREALLOC_NID]) * + sizeof(struct free_nid); + si->cache_mem += NM_I(sbi)->nat_cnt[TOTAL_NAT] * + sizeof(struct nat_entry); + si->cache_mem += NM_I(sbi)->nat_cnt[DIRTY_NAT] * + sizeof(struct nat_entry_set); + for (i = 0; i < MAX_INO_ENTRY; i++) + si->cache_mem += sbi->im[i].ino_num * sizeof(struct ino_entry); + + for (i = 0; i < NR_EXTENT_CACHES; i++) { + struct extent_tree_info *eti = &sbi->extent_tree[i]; + + si->ext_mem[i] = atomic_read(&eti->total_ext_tree) * + sizeof(struct extent_tree); + si->ext_mem[i] += atomic_read(&eti->total_ext_node) * + sizeof(struct extent_node); + si->cache_mem += si->ext_mem[i]; + } + + si->page_mem = 0; + if (sbi->node_inode) { + unsigned long npages = NODE_MAPPING(sbi)->nrpages; + + si->page_mem += (unsigned long long)npages << PAGE_SHIFT; + } + if (sbi->meta_inode) { + unsigned long npages = META_MAPPING(sbi)->nrpages; + + si->page_mem += (unsigned long long)npages << PAGE_SHIFT; + } +#ifdef CONFIG_F2FS_FS_COMPRESSION + if (sbi->compress_inode) { + unsigned long npages = COMPRESS_MAPPING(sbi)->nrpages; + + si->page_mem += (unsigned long long)npages << PAGE_SHIFT; + } +#endif +} + +static const char *s_flag[MAX_SBI_FLAG] = { + [SBI_IS_DIRTY] = "fs_dirty", + [SBI_IS_CLOSE] = "closing", + [SBI_NEED_FSCK] = "need_fsck", + [SBI_POR_DOING] = "recovering", + [SBI_NEED_SB_WRITE] = "sb_dirty", + [SBI_NEED_CP] = "need_cp", + [SBI_IS_SHUTDOWN] = "shutdown", + [SBI_IS_RECOVERED] = "recovered", + [SBI_CP_DISABLED] = "cp_disabled", + [SBI_CP_DISABLED_QUICK] = "cp_disabled_quick", + [SBI_QUOTA_NEED_FLUSH] = "quota_need_flush", + [SBI_QUOTA_SKIP_FLUSH] = "quota_skip_flush", + [SBI_QUOTA_NEED_REPAIR] = "quota_need_repair", + [SBI_IS_RESIZEFS] = "resizefs", + [SBI_IS_FREEZING] = "freezefs", + [SBI_IS_WRITABLE] = "writable", + [SBI_ENABLE_CHECKPOINT] = "enable_checkpoint", +}; + +static const char *ipu_mode_names[F2FS_IPU_MAX] = { + [F2FS_IPU_FORCE] = "FORCE", + [F2FS_IPU_SSR] = "SSR", + [F2FS_IPU_UTIL] = "UTIL", + [F2FS_IPU_SSR_UTIL] = "SSR_UTIL", + [F2FS_IPU_FSYNC] = "FSYNC", + [F2FS_IPU_ASYNC] = "ASYNC", + [F2FS_IPU_NOCACHE] = "NOCACHE", + [F2FS_IPU_HONOR_OPU_WRITE] = "HONOR_OPU_WRITE", +}; + +static int stat_show(struct seq_file *s, void *v) +{ + struct f2fs_stat_info *si; + int i = 0, j = 0; + + spin_lock(&f2fs_stat_lock); + list_for_each_entry(si, &f2fs_stat_list, stat_list) { + struct f2fs_sb_info *sbi = si->sbi; + + update_general_status(sbi); + + seq_printf(s, "\n=====[ partition info(%pg). #%d, %s, CP: %s]=====\n", + sbi->sb->s_bdev, i++, + f2fs_readonly(sbi->sb) ? "RO" : "RW", + is_set_ckpt_flags(sbi, CP_DISABLED_FLAG) ? + "Disabled" : (f2fs_cp_error(sbi) ? "Error" : "Good")); + if (sbi->s_flag) { + seq_puts(s, "[SBI:"); + for_each_set_bit(j, &sbi->s_flag, MAX_SBI_FLAG) + seq_printf(s, " %s", s_flag[j]); + seq_puts(s, "]\n"); + } + seq_printf(s, "[SB: 1] [CP: 2] [SIT: %d] [NAT: %d] ", + si->sit_area_segs, si->nat_area_segs); + seq_printf(s, "[SSA: %d] [MAIN: %d", + si->ssa_area_segs, si->main_area_segs); + seq_printf(s, "(OverProv:%d Resv:%d)]\n\n", + si->overp_segs, si->rsvd_segs); + seq_printf(s, "Current Time Sec: %llu / Mounted Time Sec: %llu\n\n", + ktime_get_boottime_seconds(), + SIT_I(sbi)->mounted_time); + + seq_puts(s, "Policy:\n"); + seq_puts(s, " - IPU: ["); + if (IS_F2FS_IPU_DISABLE(sbi)) { + seq_puts(s, " DISABLE"); + } else { + unsigned long policy = SM_I(sbi)->ipu_policy; + + for_each_set_bit(j, &policy, F2FS_IPU_MAX) + seq_printf(s, " %s", ipu_mode_names[j]); + } + seq_puts(s, " ]\n\n"); + + if (test_opt(sbi, DISCARD)) + seq_printf(s, "Utilization: %u%% (%u valid blocks, %u discard blocks)\n", + si->utilization, si->valid_count, si->discard_blks); + else + seq_printf(s, "Utilization: %u%% (%u valid blocks)\n", + si->utilization, si->valid_count); + + seq_printf(s, " - Node: %u (Inode: %u, ", + si->valid_node_count, si->valid_inode_count); + seq_printf(s, "Other: %u)\n - Data: %u\n", + si->valid_node_count - si->valid_inode_count, + si->valid_count - si->valid_node_count); + seq_printf(s, " - Inline_xattr Inode: %u\n", + si->inline_xattr); + seq_printf(s, " - Inline_data Inode: %u\n", + si->inline_inode); + seq_printf(s, " - Inline_dentry Inode: %u\n", + si->inline_dir); + seq_printf(s, " - Compressed Inode: %u, Blocks: %llu\n", + si->compr_inode, si->compr_blocks); + seq_printf(s, " - Swapfile Inode: %u\n", + si->swapfile_inode); + seq_printf(s, " - Donate Inode: %u\n", + si->ndonate_files); + seq_printf(s, " - Orphan/Append/Update Inode: %u, %u, %u\n", + si->orphans, si->append, si->update); + seq_printf(s, "\nMain area: %d segs, %d secs %d zones\n", + si->main_area_segs, si->main_area_sections, + si->main_area_zones); + seq_printf(s, " TYPE %8s %8s %8s %8s %10s %10s %10s\n", + "blkoff", "segno", "secno", "zoneno", "dirty_seg", "full_seg", "valid_blk"); + seq_printf(s, " - COLD data: %8d %8d %8d %8d %10u %10u %10u\n", + si->blkoff[CURSEG_COLD_DATA], + si->curseg[CURSEG_COLD_DATA], + si->cursec[CURSEG_COLD_DATA], + si->curzone[CURSEG_COLD_DATA], + si->dirty_seg[CURSEG_COLD_DATA], + si->full_seg[CURSEG_COLD_DATA], + si->valid_blks[CURSEG_COLD_DATA]); + seq_printf(s, " - WARM data: %8d %8d %8d %8d %10u %10u %10u\n", + si->blkoff[CURSEG_WARM_DATA], + si->curseg[CURSEG_WARM_DATA], + si->cursec[CURSEG_WARM_DATA], + si->curzone[CURSEG_WARM_DATA], + si->dirty_seg[CURSEG_WARM_DATA], + si->full_seg[CURSEG_WARM_DATA], + si->valid_blks[CURSEG_WARM_DATA]); + seq_printf(s, " - HOT data: %8d %8d %8d %8d %10u %10u %10u\n", + si->blkoff[CURSEG_HOT_DATA], + si->curseg[CURSEG_HOT_DATA], + si->cursec[CURSEG_HOT_DATA], + si->curzone[CURSEG_HOT_DATA], + si->dirty_seg[CURSEG_HOT_DATA], + si->full_seg[CURSEG_HOT_DATA], + si->valid_blks[CURSEG_HOT_DATA]); + seq_printf(s, " - Dir dnode: %8d %8d %8d %8d %10u %10u %10u\n", + si->blkoff[CURSEG_HOT_NODE], + si->curseg[CURSEG_HOT_NODE], + si->cursec[CURSEG_HOT_NODE], + si->curzone[CURSEG_HOT_NODE], + si->dirty_seg[CURSEG_HOT_NODE], + si->full_seg[CURSEG_HOT_NODE], + si->valid_blks[CURSEG_HOT_NODE]); + seq_printf(s, " - File dnode: %8d %8d %8d %8d %10u %10u %10u\n", + si->blkoff[CURSEG_WARM_NODE], + si->curseg[CURSEG_WARM_NODE], + si->cursec[CURSEG_WARM_NODE], + si->curzone[CURSEG_WARM_NODE], + si->dirty_seg[CURSEG_WARM_NODE], + si->full_seg[CURSEG_WARM_NODE], + si->valid_blks[CURSEG_WARM_NODE]); + seq_printf(s, " - Indir nodes: %8d %8d %8d %8d %10u %10u %10u\n", + si->blkoff[CURSEG_COLD_NODE], + si->curseg[CURSEG_COLD_NODE], + si->cursec[CURSEG_COLD_NODE], + si->curzone[CURSEG_COLD_NODE], + si->dirty_seg[CURSEG_COLD_NODE], + si->full_seg[CURSEG_COLD_NODE], + si->valid_blks[CURSEG_COLD_NODE]); + seq_printf(s, " - Pinned file: %8d %8d %8d %8d\n", + si->blkoff[CURSEG_COLD_DATA_PINNED], + si->curseg[CURSEG_COLD_DATA_PINNED], + si->cursec[CURSEG_COLD_DATA_PINNED], + si->curzone[CURSEG_COLD_DATA_PINNED]); + seq_printf(s, " - ATGC data: %8d %8d %8d %8d\n", + si->blkoff[CURSEG_ALL_DATA_ATGC], + si->curseg[CURSEG_ALL_DATA_ATGC], + si->cursec[CURSEG_ALL_DATA_ATGC], + si->curzone[CURSEG_ALL_DATA_ATGC]); + seq_printf(s, "\n - Valid: %d\n - Dirty: %d\n", + si->main_area_segs - si->dirty_count - + si->prefree_count - si->free_segs, + si->dirty_count); + seq_printf(s, " - Prefree: %d\n - Free: %d (%d)\n\n", + si->prefree_count, si->free_segs, si->free_secs); + if (f2fs_is_multi_device(sbi)) { + seq_puts(s, "Multidevice stats:\n"); + seq_printf(s, " [seg: %8s %8s %8s %8s %8s]", + "inuse", "dirty", "full", "free", "prefree"); + if (__is_large_section(sbi)) + seq_printf(s, " [sec: %8s %8s %8s %8s %8s]\n", + "inuse", "dirty", "full", "free", "prefree"); + else + seq_puts(s, "\n"); + + for (i = 0; i < sbi->s_ndevs; i++) { + seq_printf(s, " #%-2d %8u %8u %8u %8u %8u", i, + si->dev_stats[i].devstats[0][DEVSTAT_INUSE], + si->dev_stats[i].devstats[0][DEVSTAT_DIRTY], + si->dev_stats[i].devstats[0][DEVSTAT_FULL], + si->dev_stats[i].devstats[0][DEVSTAT_FREE], + si->dev_stats[i].devstats[0][DEVSTAT_PREFREE]); + if (!__is_large_section(sbi)) { + seq_puts(s, "\n"); + continue; + } + seq_printf(s, " %8u %8u %8u %8u %8u\n", + si->dev_stats[i].devstats[1][DEVSTAT_INUSE], + si->dev_stats[i].devstats[1][DEVSTAT_DIRTY], + si->dev_stats[i].devstats[1][DEVSTAT_FULL], + si->dev_stats[i].devstats[1][DEVSTAT_FREE], + si->dev_stats[i].devstats[1][DEVSTAT_PREFREE]); + } + seq_puts(s, "\n"); + } + seq_printf(s, "CP calls: %d (BG: %d)\n", + si->cp_call_count[TOTAL_CALL], + si->cp_call_count[BACKGROUND]); + seq_printf(s, "CP count: %d\n", si->cp_count); + seq_printf(s, " - cp blocks : %u\n", si->meta_count[META_CP]); + seq_printf(s, " - sit blocks : %u\n", + si->meta_count[META_SIT]); + seq_printf(s, " - nat blocks : %u\n", + si->meta_count[META_NAT]); + seq_printf(s, " - ssa blocks : %u\n", + si->meta_count[META_SSA]); + seq_puts(s, "CP merge:\n"); + seq_printf(s, " - Queued : %4d\n", si->nr_queued_ckpt); + seq_printf(s, " - Issued : %4d\n", si->nr_issued_ckpt); + seq_printf(s, " - Total : %4d\n", si->nr_total_ckpt); + seq_printf(s, " - Cur time : %4d(ms)\n", si->cur_ckpt_time); + seq_printf(s, " - Peak time : %4d(ms)\n", si->peak_ckpt_time); + seq_printf(s, "GC calls: %d (gc_thread: %d)\n", + si->gc_call_count[BACKGROUND] + + si->gc_call_count[FOREGROUND], + si->gc_call_count[BACKGROUND]); + if (__is_large_section(sbi)) { + seq_printf(s, " - data sections : %d (BG: %d)\n", + si->gc_secs[DATA][BG_GC] + si->gc_secs[DATA][FG_GC], + si->gc_secs[DATA][BG_GC]); + seq_printf(s, " - node sections : %d (BG: %d)\n", + si->gc_secs[NODE][BG_GC] + si->gc_secs[NODE][FG_GC], + si->gc_secs[NODE][BG_GC]); + } + seq_printf(s, " - data segments : %d (BG: %d)\n", + si->gc_segs[DATA][BG_GC] + si->gc_segs[DATA][FG_GC], + si->gc_segs[DATA][BG_GC]); + seq_printf(s, " - node segments : %d (BG: %d)\n", + si->gc_segs[NODE][BG_GC] + si->gc_segs[NODE][FG_GC], + si->gc_segs[NODE][BG_GC]); + seq_puts(s, " - Reclaimed segs :\n"); + seq_printf(s, " - Normal : %d\n", sbi->gc_reclaimed_segs[GC_NORMAL]); + seq_printf(s, " - Idle CB : %d\n", sbi->gc_reclaimed_segs[GC_IDLE_CB]); + seq_printf(s, " - Idle Greedy : %d\n", + sbi->gc_reclaimed_segs[GC_IDLE_GREEDY]); + seq_printf(s, " - Idle AT : %d\n", sbi->gc_reclaimed_segs[GC_IDLE_AT]); + seq_printf(s, " - Urgent High : %d\n", + sbi->gc_reclaimed_segs[GC_URGENT_HIGH]); + seq_printf(s, " - Urgent Mid : %d\n", sbi->gc_reclaimed_segs[GC_URGENT_MID]); + seq_printf(s, " - Urgent Low : %d\n", sbi->gc_reclaimed_segs[GC_URGENT_LOW]); + seq_printf(s, "Try to move %d blocks (BG: %d)\n", si->tot_blks, + si->bg_data_blks + si->bg_node_blks); + seq_printf(s, " - data blocks : %d (%d)\n", si->data_blks, + si->bg_data_blks); + seq_printf(s, " - node blocks : %d (%d)\n", si->node_blks, + si->bg_node_blks); + seq_printf(s, "BG skip : IO: %u, Other: %u\n", + si->io_skip_bggc, si->other_skip_bggc); + seq_printf(s, "defrag blocks : %u\n", si->defrag_blks); + seq_puts(s, "\nExtent Cache (Read):\n"); + seq_printf(s, " - Hit Count: L1-1:%llu L1-2:%llu L2:%llu\n", + si->hit_largest, si->hit_cached[EX_READ], + si->hit_rbtree[EX_READ]); + seq_printf(s, " - Hit Ratio: %llu%% (%llu / %llu)\n", + !si->total_ext[EX_READ] ? 0 : + div64_u64(si->hit_total[EX_READ] * 100, + si->total_ext[EX_READ]), + si->hit_total[EX_READ], si->total_ext[EX_READ]); + seq_printf(s, " - Inner Struct Count: tree: %d(%d), node: %d\n", + si->ext_tree[EX_READ], si->zombie_tree[EX_READ], + si->ext_node[EX_READ]); + seq_puts(s, "\nExtent Cache (Block Age):\n"); + seq_printf(s, " - Allocated Data Blocks: %llu\n", + si->allocated_data_blocks); + seq_printf(s, " - Hit Count: L1:%llu L2:%llu\n", + si->hit_cached[EX_BLOCK_AGE], + si->hit_rbtree[EX_BLOCK_AGE]); + seq_printf(s, " - Hit Ratio: %llu%% (%llu / %llu)\n", + !si->total_ext[EX_BLOCK_AGE] ? 0 : + div64_u64(si->hit_total[EX_BLOCK_AGE] * 100, + si->total_ext[EX_BLOCK_AGE]), + si->hit_total[EX_BLOCK_AGE], + si->total_ext[EX_BLOCK_AGE]); + seq_printf(s, " - Inner Struct Count: tree: %d(%d), node: %d\n", + si->ext_tree[EX_BLOCK_AGE], + si->zombie_tree[EX_BLOCK_AGE], + si->ext_node[EX_BLOCK_AGE]); + seq_puts(s, "\nBalancing F2FS Async:\n"); + seq_printf(s, " - DIO (R: %4d, W: %4d)\n", + si->nr_dio_read, si->nr_dio_write); + seq_printf(s, " - IO_R (Data: %4d, Node: %4d, Meta: %4d\n", + si->nr_rd_data, si->nr_rd_node, si->nr_rd_meta); + seq_printf(s, " - IO_W (CP: %4d, Data: %4d, Flush: (%4d %4d %4d), ", + si->nr_wb_cp_data, si->nr_wb_data, + si->nr_flushing, si->nr_flushed, + si->flush_list_empty); + seq_printf(s, "Discard: (%4d %4d)) cmd: %4d undiscard:%4u\n", + si->nr_discarding, si->nr_discarded, + si->nr_discard_cmd, si->undiscard_blks); + seq_printf(s, " - atomic IO: %4d (Max. %4d)\n", + si->aw_cnt, si->max_aw_cnt); + seq_printf(s, " - compress: %4d, hit:%8d\n", si->compress_pages, si->compress_page_hit); + seq_printf(s, " - nodes: %4d in %4d\n", + si->ndirty_node, si->node_pages); + seq_printf(s, " - dents: %4d in dirs:%4d (%4d)\n", + si->ndirty_dent, si->ndirty_dirs, si->ndirty_all); + seq_printf(s, " - data: %4d in files:%4d\n", + si->ndirty_data, si->ndirty_files); + seq_printf(s, " - quota data: %4d in quota files:%4d\n", + si->ndirty_qdata, si->nquota_files); + seq_printf(s, " - meta: %4d in %4d\n", + si->ndirty_meta, si->meta_pages); + seq_printf(s, " - imeta: %4d\n", + si->ndirty_imeta); + seq_printf(s, " - fsync mark: %4lld\n", + percpu_counter_sum_positive( + &sbi->rf_node_block_count)); + seq_printf(s, " - NATs: %9d/%9d\n - SITs: %9d/%9d\n", + si->dirty_nats, si->nats, si->dirty_sits, si->sits); + seq_printf(s, " - free_nids: %9d/%9d\n - alloc_nids: %9d\n", + si->free_nids, si->avail_nids, si->alloc_nids); + seq_puts(s, "\nDistribution of User Blocks:"); + seq_puts(s, " [ valid | invalid | free ]\n"); + seq_puts(s, " ["); + + for (j = 0; j < si->util_valid; j++) + seq_putc(s, '-'); + seq_putc(s, '|'); + + for (j = 0; j < si->util_invalid; j++) + seq_putc(s, '-'); + seq_putc(s, '|'); + + for (j = 0; j < si->util_free; j++) + seq_putc(s, '-'); + seq_puts(s, "]\n\n"); + seq_printf(s, "IPU: %u blocks\n", si->inplace_count); + seq_printf(s, "SSR: %u blocks in %u segments\n", + si->block_count[SSR], si->segment_count[SSR]); + seq_printf(s, "LFS: %u blocks in %u segments\n", + si->block_count[LFS], si->segment_count[LFS]); + + /* segment usage info */ + f2fs_update_sit_info(sbi); + seq_printf(s, "\nBDF: %u, avg. vblocks: %u\n", + si->bimodal, si->avg_vblocks); + + /* memory footprint */ + update_mem_info(sbi); + seq_printf(s, "\nMemory: %llu KB\n", + (si->base_mem + si->cache_mem + si->page_mem) >> 10); + seq_printf(s, " - static: %llu KB\n", + si->base_mem >> 10); + seq_printf(s, " - cached all: %llu KB\n", + si->cache_mem >> 10); + seq_printf(s, " - read extent cache: %llu KB\n", + si->ext_mem[EX_READ] >> 10); + seq_printf(s, " - block age extent cache: %llu KB\n", + si->ext_mem[EX_BLOCK_AGE] >> 10); + seq_printf(s, " - paged : %llu KB\n", + si->page_mem >> 10); + } + spin_unlock(&f2fs_stat_lock); + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(stat); +#endif + +int f2fs_build_stats(struct f2fs_sb_info *sbi) +{ + struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); + struct f2fs_stat_info *si; + struct f2fs_dev_stats *dev_stats; + int i; + + si = f2fs_kzalloc(sbi, sizeof(struct f2fs_stat_info), GFP_KERNEL); + if (!si) + return -ENOMEM; + + dev_stats = f2fs_kzalloc(sbi, sizeof(struct f2fs_dev_stats) * + sbi->s_ndevs, GFP_KERNEL); + if (!dev_stats) { + kfree(si); + return -ENOMEM; + } + + si->dev_stats = dev_stats; + + si->all_area_segs = le32_to_cpu(raw_super->segment_count); + si->sit_area_segs = le32_to_cpu(raw_super->segment_count_sit); + si->nat_area_segs = le32_to_cpu(raw_super->segment_count_nat); + si->ssa_area_segs = le32_to_cpu(raw_super->segment_count_ssa); + si->main_area_segs = le32_to_cpu(raw_super->segment_count_main); + si->main_area_sections = le32_to_cpu(raw_super->section_count); + si->main_area_zones = si->main_area_sections / + le32_to_cpu(raw_super->secs_per_zone); + si->sbi = sbi; + sbi->stat_info = si; + + /* general extent cache stats */ + for (i = 0; i < NR_EXTENT_CACHES; i++) { + atomic64_set(&sbi->total_hit_ext[i], 0); + atomic64_set(&sbi->read_hit_rbtree[i], 0); + atomic64_set(&sbi->read_hit_cached[i], 0); + } + + /* read extent_cache only */ + atomic64_set(&sbi->read_hit_largest, 0); + + atomic_set(&sbi->inline_xattr, 0); + atomic_set(&sbi->inline_inode, 0); + atomic_set(&sbi->inline_dir, 0); + atomic_set(&sbi->compr_inode, 0); + atomic64_set(&sbi->compr_blocks, 0); + atomic_set(&sbi->swapfile_inode, 0); + atomic_set(&sbi->atomic_files, 0); + atomic_set(&sbi->inplace_count, 0); + for (i = META_CP; i < META_MAX; i++) + atomic_set(&sbi->meta_count[i], 0); + for (i = 0; i < MAX_CALL_TYPE; i++) + atomic_set(&sbi->cp_call_count[i], 0); + + atomic_set(&sbi->max_aw_cnt, 0); + + spin_lock(&f2fs_stat_lock); + list_add_tail(&si->stat_list, &f2fs_stat_list); + spin_unlock(&f2fs_stat_lock); + + return 0; +} + +void f2fs_destroy_stats(struct f2fs_sb_info *sbi) +{ + struct f2fs_stat_info *si = F2FS_STAT(sbi); + + spin_lock(&f2fs_stat_lock); + list_del(&si->stat_list); + spin_unlock(&f2fs_stat_lock); + + kfree(si->dev_stats); + kfree(si); +} + +void __init f2fs_create_root_stats(void) +{ +#ifdef CONFIG_DEBUG_FS + f2fs_debugfs_root = debugfs_create_dir("f2fs", NULL); + + debugfs_create_file("status", 0444, f2fs_debugfs_root, NULL, + &stat_fops); +#endif +} + +void f2fs_destroy_root_stats(void) +{ +#ifdef CONFIG_DEBUG_FS + debugfs_remove_recursive(f2fs_debugfs_root); + f2fs_debugfs_root = NULL; +#endif +} diff --git a/fs/f2fs/dir.c b/fs/f2fs/dir.c new file mode 100644 index 000000000..a9563f7fc --- /dev/null +++ b/fs/f2fs/dir.c @@ -0,0 +1,1148 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/f2fs/dir.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + */ +#include +#include +#include +#include +#include +#include +#include +#include "f2fs.h" +#include "node.h" +#include "acl.h" +#include "xattr.h" +#include + +static inline bool f2fs_should_fallback_to_linear(struct inode *dir) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + + switch (F2FS_OPTION(sbi).lookup_mode) { + case LOOKUP_PERF: + return false; + case LOOKUP_COMPAT: + return true; + case LOOKUP_AUTO: + return !sb_no_casefold_compat_fallback(sbi->sb); + } + return false; +} + +#if IS_ENABLED(CONFIG_UNICODE) +extern struct kmem_cache *f2fs_cf_name_slab; +#endif + +static unsigned long dir_blocks(struct inode *inode) +{ + return ((unsigned long long) (i_size_read(inode) + PAGE_SIZE - 1)) + >> PAGE_SHIFT; +} + +static unsigned int dir_buckets(unsigned int level, int dir_level) +{ + if (level + dir_level < MAX_DIR_HASH_DEPTH / 2) + return BIT(level + dir_level); + else + return MAX_DIR_BUCKETS; +} + +static unsigned int bucket_blocks(unsigned int level) +{ + if (level < MAX_DIR_HASH_DEPTH / 2) + return 2; + else + return 4; +} + +#if IS_ENABLED(CONFIG_UNICODE) +/* If @dir is casefolded, initialize @fname->cf_name from @fname->usr_fname. */ +int f2fs_init_casefolded_name(const struct inode *dir, + struct f2fs_filename *fname) +{ + struct super_block *sb = dir->i_sb; + unsigned char *buf; + int len; + + if (IS_CASEFOLDED(dir) && + !name_is_dot_dotdot(fname->usr_fname->name, fname->usr_fname->len)) { + buf = f2fs_kmem_cache_alloc(f2fs_cf_name_slab, + GFP_NOFS, false, F2FS_SB(sb)); + if (!buf) + return -ENOMEM; + + len = utf8_casefold(sb->s_encoding, fname->usr_fname, + buf, F2FS_NAME_LEN); + if (len <= 0) { + kmem_cache_free(f2fs_cf_name_slab, buf); + if (sb_has_strict_encoding(sb)) + return -EINVAL; + /* fall back to treating name as opaque byte sequence */ + return 0; + } + fname->cf_name.name = buf; + fname->cf_name.len = len; + } + + return 0; +} + +void f2fs_free_casefolded_name(struct f2fs_filename *fname) +{ + unsigned char *buf = (unsigned char *)fname->cf_name.name; + + if (buf) { + kmem_cache_free(f2fs_cf_name_slab, buf); + fname->cf_name.name = NULL; + } +} +#endif /* CONFIG_UNICODE */ + +static int __f2fs_setup_filename(const struct inode *dir, + const struct fscrypt_name *crypt_name, + struct f2fs_filename *fname) +{ + int err; + + memset(fname, 0, sizeof(*fname)); + + fname->usr_fname = crypt_name->usr_fname; + fname->disk_name = crypt_name->disk_name; +#ifdef CONFIG_FS_ENCRYPTION + fname->crypto_buf = crypt_name->crypto_buf; +#endif + if (crypt_name->is_nokey_name) { + /* hash was decoded from the no-key name */ + fname->hash = cpu_to_le32(crypt_name->hash); + } else { + err = f2fs_init_casefolded_name(dir, fname); + if (err) { + f2fs_free_filename(fname); + return err; + } + f2fs_hash_filename(dir, fname); + } + return 0; +} + +/* + * Prepare to search for @iname in @dir. This is similar to + * fscrypt_setup_filename(), but this also handles computing the casefolded name + * and the f2fs dirhash if needed, then packing all the information about this + * filename up into a 'struct f2fs_filename'. + */ +int f2fs_setup_filename(struct inode *dir, const struct qstr *iname, + int lookup, struct f2fs_filename *fname) +{ + struct fscrypt_name crypt_name; + int err; + + err = fscrypt_setup_filename(dir, iname, lookup, &crypt_name); + if (err) + return err; + + return __f2fs_setup_filename(dir, &crypt_name, fname); +} + +/* + * Prepare to look up @dentry in @dir. This is similar to + * fscrypt_prepare_lookup(), but this also handles computing the casefolded name + * and the f2fs dirhash if needed, then packing all the information about this + * filename up into a 'struct f2fs_filename'. + */ +int f2fs_prepare_lookup(struct inode *dir, struct dentry *dentry, + struct f2fs_filename *fname) +{ + struct fscrypt_name crypt_name; + int err; + + err = fscrypt_prepare_lookup(dir, dentry, &crypt_name); + if (err) + return err; + + return __f2fs_setup_filename(dir, &crypt_name, fname); +} + +void f2fs_free_filename(struct f2fs_filename *fname) +{ +#ifdef CONFIG_FS_ENCRYPTION + kfree(fname->crypto_buf.name); + fname->crypto_buf.name = NULL; +#endif + f2fs_free_casefolded_name(fname); +} + +static unsigned long dir_block_index(unsigned int level, + int dir_level, unsigned int idx) +{ + unsigned long i; + unsigned long bidx = 0; + + for (i = 0; i < level; i++) + bidx += mul_u32_u32(dir_buckets(i, dir_level), + bucket_blocks(i)); + bidx += idx * bucket_blocks(level); + return bidx; +} + +static struct f2fs_dir_entry *find_in_block(struct inode *dir, + struct folio *dentry_folio, + const struct f2fs_filename *fname, + int *max_slots, + bool use_hash) +{ + struct f2fs_dentry_block *dentry_blk; + struct f2fs_dentry_ptr d; + + dentry_blk = folio_address(dentry_folio); + + make_dentry_ptr_block(dir, &d, dentry_blk); + return f2fs_find_target_dentry(&d, fname, max_slots, use_hash); +} + +static inline int f2fs_match_name(const struct inode *dir, + const struct f2fs_filename *fname, + const u8 *de_name, u32 de_name_len) +{ + struct fscrypt_name f; + +#if IS_ENABLED(CONFIG_UNICODE) + if (fname->cf_name.name) + return generic_ci_match(dir, fname->usr_fname, + &fname->cf_name, + de_name, de_name_len); + +#endif + f.usr_fname = fname->usr_fname; + f.disk_name = fname->disk_name; +#ifdef CONFIG_FS_ENCRYPTION + f.crypto_buf = fname->crypto_buf; +#endif + return fscrypt_match_name(&f, de_name, de_name_len); +} + +struct f2fs_dir_entry *f2fs_find_target_dentry(const struct f2fs_dentry_ptr *d, + const struct f2fs_filename *fname, int *max_slots, + bool use_hash) +{ + struct f2fs_dir_entry *de; + unsigned long bit_pos = 0; + int max_len = 0; + int res = 0; + + if (max_slots) + *max_slots = 0; + while (bit_pos < d->max) { + if (!test_bit_le(bit_pos, d->bitmap)) { + bit_pos++; + max_len++; + continue; + } + + de = &d->dentry[bit_pos]; + + if (unlikely(!de->name_len)) { + bit_pos++; + continue; + } + + if (unlikely(le16_to_cpu(de->name_len) > F2FS_NAME_LEN || + bit_pos + GET_DENTRY_SLOTS(le16_to_cpu(de->name_len)) > + d->max)) + return ERR_PTR(-EFSCORRUPTED); + + if (!use_hash || de->hash_code == fname->hash) { + res = f2fs_match_name(d->inode, fname, + d->filename[bit_pos], + le16_to_cpu(de->name_len)); + if (res < 0) + return ERR_PTR(res); + if (res) + goto found; + } + + if (max_slots && max_len > *max_slots) + *max_slots = max_len; + max_len = 0; + + bit_pos += GET_DENTRY_SLOTS(le16_to_cpu(de->name_len)); + } + + de = NULL; +found: + if (max_slots && max_len > *max_slots) + *max_slots = max_len; + return de; +} + +static struct f2fs_dir_entry *find_in_level(struct inode *dir, + unsigned int level, + const struct f2fs_filename *fname, + struct folio **res_folio, + bool use_hash) +{ + int s = GET_DENTRY_SLOTS(fname->disk_name.len); + unsigned int nbucket, nblock; + unsigned int bidx, end_block, bucket_no; + struct f2fs_dir_entry *de = NULL; + pgoff_t next_pgofs; + bool room = false; + int max_slots; + + nbucket = dir_buckets(level, F2FS_I(dir)->i_dir_level); + nblock = bucket_blocks(level); + + bucket_no = use_hash ? le32_to_cpu(fname->hash) % nbucket : 0; + +start_find_bucket: + bidx = dir_block_index(level, F2FS_I(dir)->i_dir_level, + bucket_no); + end_block = bidx + nblock; + + while (bidx < end_block) { + /* no need to allocate new dentry pages to all the indices */ + struct folio *dentry_folio; + dentry_folio = f2fs_find_data_folio(dir, bidx, &next_pgofs); + if (IS_ERR(dentry_folio)) { + if (PTR_ERR(dentry_folio) == -ENOENT) { + room = true; + bidx = next_pgofs; + continue; + } else { + *res_folio = dentry_folio; + break; + } + } + + de = find_in_block(dir, dentry_folio, fname, &max_slots, use_hash); + if (IS_ERR(de)) { + *res_folio = ERR_CAST(de); + de = NULL; + break; + } else if (de) { + *res_folio = dentry_folio; + break; + } + + if (max_slots >= s) + room = true; + f2fs_folio_put(dentry_folio, false); + + bidx++; + } + + if (de) + return de; + + if (likely(use_hash)) { + if (room && F2FS_I(dir)->chash != fname->hash) { + F2FS_I(dir)->chash = fname->hash; + F2FS_I(dir)->clevel = level; + } + } else if (++bucket_no < nbucket) { + goto start_find_bucket; + } + return NULL; +} + +struct f2fs_dir_entry *__f2fs_find_entry(struct inode *dir, + const struct f2fs_filename *fname, + struct folio **res_folio) +{ + unsigned long npages = dir_blocks(dir); + struct f2fs_dir_entry *de = NULL; + unsigned int max_depth; + unsigned int level; + bool use_hash = true; + + *res_folio = NULL; + +#if IS_ENABLED(CONFIG_UNICODE) +start_find_entry: +#endif + if (f2fs_has_inline_dentry(dir)) { + de = f2fs_find_in_inline_dir(dir, fname, res_folio, use_hash); + goto out; + } + + if (npages == 0) + goto out; + + max_depth = F2FS_I(dir)->i_current_depth; + if (unlikely(max_depth > MAX_DIR_HASH_DEPTH)) { + f2fs_warn(F2FS_I_SB(dir), "Corrupted max_depth of %llu: %u", + dir->i_ino, max_depth); + max_depth = MAX_DIR_HASH_DEPTH; + f2fs_i_depth_write(dir, max_depth); + } + + for (level = 0; level < max_depth; level++) { + de = find_in_level(dir, level, fname, res_folio, use_hash); + if (de || IS_ERR(*res_folio)) + break; + } + +out: +#if IS_ENABLED(CONFIG_UNICODE) + if (f2fs_should_fallback_to_linear(dir) && + IS_CASEFOLDED(dir) && !de && use_hash) { + use_hash = false; + goto start_find_entry; + } +#endif + /* This is to increase the speed of f2fs_create */ + if (!de) + F2FS_I(dir)->task = current; + return de; +} + +/* + * Find an entry in the specified directory with the wanted name. + * It returns the page where the entry was found (as a parameter - res_page), + * and the entry itself. Page is returned mapped and unlocked. + * Entry is guaranteed to be valid. + */ +struct f2fs_dir_entry *f2fs_find_entry(struct inode *dir, + const struct qstr *child, struct folio **res_folio) +{ + struct f2fs_dir_entry *de = NULL; + struct f2fs_filename fname; + int err; + + err = f2fs_setup_filename(dir, child, 1, &fname); + if (err) { + if (err == -ENOENT) + *res_folio = NULL; + else + *res_folio = ERR_PTR(err); + return NULL; + } + + de = __f2fs_find_entry(dir, &fname, res_folio); + + f2fs_free_filename(&fname); + return de; +} + +struct f2fs_dir_entry *f2fs_parent_dir(struct inode *dir, struct folio **f) +{ + return f2fs_find_entry(dir, &dotdot_name, f); +} + +ino_t f2fs_inode_by_name(struct inode *dir, const struct qstr *qstr, + struct folio **folio) +{ + ino_t res = 0; + struct f2fs_dir_entry *de; + + de = f2fs_find_entry(dir, qstr, folio); + if (de) { + res = le32_to_cpu(de->ino); + f2fs_folio_put(*folio, false); + } + + return res; +} + +void f2fs_set_link(struct inode *dir, struct f2fs_dir_entry *de, + struct folio *folio, struct inode *inode) +{ + enum page_type type = f2fs_has_inline_dentry(dir) ? NODE : DATA; + + folio_lock(folio); + f2fs_folio_wait_writeback(folio, type, true, true); + de->ino = cpu_to_le32(inode->i_ino); + de->file_type = fs_umode_to_ftype(inode->i_mode); + folio_mark_dirty(folio); + + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + f2fs_mark_inode_dirty_sync(dir, false); + f2fs_folio_put(folio, true); +} + +static void init_dent_inode(struct inode *dir, struct inode *inode, + const struct f2fs_filename *fname, + struct folio *ifolio) +{ + struct f2fs_inode *ri; + + if (!fname) /* tmpfile case? */ + return; + + f2fs_folio_wait_writeback(ifolio, NODE, true, true); + + /* copy name info. to this inode folio */ + ri = F2FS_INODE(ifolio); + ri->i_namelen = cpu_to_le32(fname->disk_name.len); + memcpy(ri->i_name, fname->disk_name.name, fname->disk_name.len); + if (IS_ENCRYPTED(dir)) { + file_set_enc_name(inode); + /* + * Roll-forward recovery doesn't have encryption keys available, + * so it can't compute the dirhash for encrypted+casefolded + * filenames. Append it to i_name if possible. Else, disable + * roll-forward recovery of the dentry (i.e., make fsync'ing the + * file force a checkpoint) by setting LOST_PINO. + */ + if (IS_CASEFOLDED(dir)) { + if (fname->disk_name.len + sizeof(f2fs_hash_t) <= + F2FS_NAME_LEN) + put_unaligned(fname->hash, (f2fs_hash_t *) + &ri->i_name[fname->disk_name.len]); + else + file_lost_pino(inode); + } + } + folio_mark_dirty(ifolio); +} + +void f2fs_do_make_empty_dir(struct inode *inode, struct inode *parent, + struct f2fs_dentry_ptr *d) +{ + struct fscrypt_str dot = FSTR_INIT(".", 1); + struct fscrypt_str dotdot = FSTR_INIT("..", 2); + + /* update dirent of "." */ + f2fs_update_dentry(inode->i_ino, inode->i_mode, d, &dot, 0, 0); + + /* update dirent of ".." */ + f2fs_update_dentry(parent->i_ino, parent->i_mode, d, &dotdot, 0, 1); +} + +static int make_empty_dir(struct inode *inode, + struct inode *parent, struct folio *folio) +{ + struct folio *dentry_folio; + struct f2fs_dentry_block *dentry_blk; + struct f2fs_dentry_ptr d; + + if (f2fs_has_inline_dentry(inode)) + return f2fs_make_empty_inline_dir(inode, parent, folio); + + dentry_folio = f2fs_get_new_data_folio(inode, folio, 0, true); + if (IS_ERR(dentry_folio)) + return PTR_ERR(dentry_folio); + + dentry_blk = folio_address(dentry_folio); + + make_dentry_ptr_block(NULL, &d, dentry_blk); + f2fs_do_make_empty_dir(inode, parent, &d); + + folio_mark_dirty(dentry_folio); + f2fs_folio_put(dentry_folio, true); + return 0; +} + +struct folio *f2fs_init_inode_metadata(struct inode *inode, struct inode *dir, + const struct f2fs_filename *fname, struct folio *dfolio) +{ + struct folio *folio; + int err; + + if (is_inode_flag_set(inode, FI_NEW_INODE)) { + folio = f2fs_new_inode_folio(inode); + if (IS_ERR(folio)) + return folio; + + if (S_ISDIR(inode->i_mode)) { + /* in order to handle error case */ + folio_get(folio); + err = make_empty_dir(inode, dir, folio); + if (err) { + folio_lock(folio); + goto put_error; + } + folio_put(folio); + } + + err = f2fs_init_acl(inode, dir, folio, dfolio); + if (err) + goto put_error; + + err = f2fs_init_security(inode, dir, + fname ? fname->usr_fname : NULL, + folio); + if (err) + goto put_error; + + if (IS_ENCRYPTED(inode)) { + err = fscrypt_set_context(inode, folio); + if (err) + goto put_error; + } + } else { + folio = f2fs_get_inode_folio(F2FS_I_SB(dir), inode->i_ino); + if (IS_ERR(folio)) + return folio; + } + + init_dent_inode(dir, inode, fname, folio); + + /* + * This file should be checkpointed during fsync. + * We lost i_pino from now on. + */ + if (is_inode_flag_set(inode, FI_INC_LINK)) { + if (!S_ISDIR(inode->i_mode)) + file_lost_pino(inode); + /* + * If link the tmpfile to alias through linkat path, + * we should remove this inode from orphan list. + */ + if (inode->i_nlink == 0) + f2fs_remove_orphan_inode(F2FS_I_SB(dir), inode->i_ino); + f2fs_i_links_write(inode, true); + } + return folio; + +put_error: + clear_nlink(inode); + f2fs_update_inode(inode, folio); + f2fs_folio_put(folio, true); + return ERR_PTR(err); +} + +void f2fs_update_parent_metadata(struct inode *dir, struct inode *inode, + unsigned int current_depth) +{ + if (inode && is_inode_flag_set(inode, FI_NEW_INODE)) { + if (S_ISDIR(inode->i_mode)) + f2fs_i_links_write(dir, true); + clear_inode_flag(inode, FI_NEW_INODE); + } + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + f2fs_mark_inode_dirty_sync(dir, false); + + if (F2FS_I(dir)->i_current_depth != current_depth) + f2fs_i_depth_write(dir, current_depth); + + if (inode && is_inode_flag_set(inode, FI_INC_LINK)) + clear_inode_flag(inode, FI_INC_LINK); +} + +int f2fs_room_for_filename(const void *bitmap, int slots, int max_slots) +{ + int bit_start = 0; + int zero_start, zero_end; +next: + zero_start = find_next_zero_bit_le(bitmap, max_slots, bit_start); + if (zero_start >= max_slots) + return max_slots; + + zero_end = find_next_bit_le(bitmap, max_slots, zero_start); + if (zero_end - zero_start >= slots) + return zero_start; + + bit_start = zero_end + 1; + + if (zero_end + 1 >= max_slots) + return max_slots; + goto next; +} + +bool f2fs_has_enough_room(struct inode *dir, struct folio *ifolio, + const struct f2fs_filename *fname) +{ + struct f2fs_dentry_ptr d; + unsigned int bit_pos; + int slots = GET_DENTRY_SLOTS(fname->disk_name.len); + + make_dentry_ptr_inline(dir, &d, inline_data_addr(dir, ifolio)); + + bit_pos = f2fs_room_for_filename(d.bitmap, slots, d.max); + + return bit_pos < d.max; +} + +void f2fs_update_dentry(nid_t ino, umode_t mode, struct f2fs_dentry_ptr *d, + const struct fscrypt_str *name, f2fs_hash_t name_hash, + unsigned int bit_pos) +{ + struct f2fs_dir_entry *de; + int slots = GET_DENTRY_SLOTS(name->len); + int i; + + de = &d->dentry[bit_pos]; + de->hash_code = name_hash; + de->name_len = cpu_to_le16(name->len); + memcpy(d->filename[bit_pos], name->name, name->len); + de->ino = cpu_to_le32(ino); + de->file_type = fs_umode_to_ftype(mode); + for (i = 0; i < slots; i++) { + __set_bit_le(bit_pos + i, (void *)d->bitmap); + /* avoid wrong garbage data for readdir */ + if (i) + (de + i)->name_len = 0; + } +} + +int f2fs_add_regular_entry(struct inode *dir, const struct f2fs_filename *fname, + struct inode *inode, nid_t ino, umode_t mode) +{ + unsigned int bit_pos; + unsigned int level; + unsigned int current_depth; + unsigned long bidx, block; + unsigned int nbucket, nblock; + struct folio *dentry_folio = NULL; + struct f2fs_dentry_block *dentry_blk = NULL; + struct f2fs_dentry_ptr d; + struct folio *folio = NULL; + int slots, err = 0; + + level = 0; + slots = GET_DENTRY_SLOTS(fname->disk_name.len); + + current_depth = F2FS_I(dir)->i_current_depth; + if (F2FS_I(dir)->chash == fname->hash) { + level = F2FS_I(dir)->clevel; + F2FS_I(dir)->chash = 0; + } + +start: + if (time_to_inject(F2FS_I_SB(dir), FAULT_DIR_DEPTH)) + return -ENOSPC; + + if (unlikely(current_depth == MAX_DIR_HASH_DEPTH)) + return -ENOSPC; + + /* Increase the depth, if required */ + if (level == current_depth) + ++current_depth; + + nbucket = dir_buckets(level, F2FS_I(dir)->i_dir_level); + nblock = bucket_blocks(level); + + bidx = dir_block_index(level, F2FS_I(dir)->i_dir_level, + (le32_to_cpu(fname->hash) % nbucket)); + + for (block = bidx; block <= (bidx + nblock - 1); block++) { + dentry_folio = f2fs_get_new_data_folio(dir, NULL, block, true); + if (IS_ERR(dentry_folio)) + return PTR_ERR(dentry_folio); + + dentry_blk = folio_address(dentry_folio); + bit_pos = f2fs_room_for_filename(&dentry_blk->dentry_bitmap, + slots, NR_DENTRY_IN_BLOCK); + if (bit_pos < NR_DENTRY_IN_BLOCK) + goto add_dentry; + + f2fs_folio_put(dentry_folio, true); + } + + /* Move to next level to find the empty slot for new dentry */ + ++level; + goto start; +add_dentry: + f2fs_folio_wait_writeback(dentry_folio, DATA, true, true); + + if (inode) { + f2fs_down_write(&F2FS_I(inode)->i_sem); + folio = f2fs_init_inode_metadata(inode, dir, fname, NULL); + if (IS_ERR(folio)) { + err = PTR_ERR(folio); + goto fail; + } + } + + make_dentry_ptr_block(NULL, &d, dentry_blk); + f2fs_update_dentry(ino, mode, &d, &fname->disk_name, fname->hash, + bit_pos); + + folio_mark_dirty(dentry_folio); + + if (inode) { + f2fs_i_pino_write(inode, dir->i_ino); + + /* synchronize inode page's data from inode cache */ + if (is_inode_flag_set(inode, FI_NEW_INODE)) + f2fs_update_inode(inode, folio); + + f2fs_folio_put(folio, true); + } + + f2fs_update_parent_metadata(dir, inode, current_depth); +fail: + if (inode) + f2fs_up_write(&F2FS_I(inode)->i_sem); + + f2fs_folio_put(dentry_folio, true); + + return err; +} + +int f2fs_add_dentry(struct inode *dir, const struct f2fs_filename *fname, + struct inode *inode, nid_t ino, umode_t mode) +{ + int err = -EAGAIN; + + if (f2fs_has_inline_dentry(dir)) { + /* + * Should get i_xattr_sem to keep the lock order: + * i_xattr_sem -> inode_page lock used by f2fs_setxattr. + */ + f2fs_down_read(&F2FS_I(dir)->i_xattr_sem); + err = f2fs_add_inline_entry(dir, fname, inode, ino, mode); + f2fs_up_read(&F2FS_I(dir)->i_xattr_sem); + } + if (err == -EAGAIN) + err = f2fs_add_regular_entry(dir, fname, inode, ino, mode); + + f2fs_update_time(F2FS_I_SB(dir), REQ_TIME); + return err; +} + +/* + * Caller should grab and release a rwsem by calling f2fs_lock_op() and + * f2fs_unlock_op(). + */ +int f2fs_do_add_link(struct inode *dir, const struct qstr *name, + struct inode *inode, nid_t ino, umode_t mode) +{ + struct f2fs_filename fname; + struct folio *folio = NULL; + struct f2fs_dir_entry *de = NULL; + int err; + + err = f2fs_setup_filename(dir, name, 0, &fname); + if (err) + return err; + + /* + * An immature stackable filesystem shows a race condition between lookup + * and create. If we have same task when doing lookup and create, it's + * definitely fine as expected by VFS normally. Otherwise, let's just + * verify on-disk dentry one more time, which guarantees filesystem + * consistency more. + */ + if (current != F2FS_I(dir)->task) { + de = __f2fs_find_entry(dir, &fname, &folio); + F2FS_I(dir)->task = NULL; + } + if (de) { + f2fs_folio_put(folio, false); + err = -EEXIST; + } else if (IS_ERR(folio)) { + err = PTR_ERR(folio); + } else { + err = f2fs_add_dentry(dir, &fname, inode, ino, mode); + } + f2fs_free_filename(&fname); + return err; +} + +int f2fs_do_tmpfile(struct inode *inode, struct inode *dir, + struct f2fs_filename *fname) +{ + struct folio *folio; + int err = 0; + + f2fs_down_write(&F2FS_I(inode)->i_sem); + folio = f2fs_init_inode_metadata(inode, dir, fname, NULL); + if (IS_ERR(folio)) { + err = PTR_ERR(folio); + goto fail; + } + f2fs_folio_put(folio, true); + + clear_inode_flag(inode, FI_NEW_INODE); + f2fs_update_time(F2FS_I_SB(inode), REQ_TIME); +fail: + f2fs_up_write(&F2FS_I(inode)->i_sem); + return err; +} + +void f2fs_drop_nlink(struct inode *dir, struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + + f2fs_down_write(&F2FS_I(inode)->i_sem); + + if (S_ISDIR(inode->i_mode)) + f2fs_i_links_write(dir, false); + inode_set_ctime_current(inode); + + f2fs_i_links_write(inode, false); + if (S_ISDIR(inode->i_mode)) { + f2fs_i_links_write(inode, false); + f2fs_i_size_write(inode, 0); + } + f2fs_up_write(&F2FS_I(inode)->i_sem); + + if (inode->i_nlink == 0) + f2fs_add_orphan_inode(inode); + else + f2fs_release_orphan_inode(sbi); +} + +/* + * It only removes the dentry from the dentry page, corresponding name + * entry in name page does not need to be touched during deletion. + */ +void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct folio *folio, + struct inode *dir, struct inode *inode) +{ + struct f2fs_dentry_block *dentry_blk; + unsigned int bit_pos; + int slots = GET_DENTRY_SLOTS(le16_to_cpu(dentry->name_len)); + pgoff_t index = folio->index; + int i; + + f2fs_update_time(F2FS_I_SB(dir), REQ_TIME); + + if (F2FS_OPTION(F2FS_I_SB(dir)).fsync_mode == FSYNC_MODE_STRICT) + f2fs_add_ino_entry(F2FS_I_SB(dir), dir->i_ino, TRANS_DIR_INO); + + if (f2fs_has_inline_dentry(dir)) + return f2fs_delete_inline_entry(dentry, folio, dir, inode); + + folio_lock(folio); + f2fs_folio_wait_writeback(folio, DATA, true, true); + + dentry_blk = folio_address(folio); + bit_pos = dentry - dentry_blk->dentry; + for (i = 0; i < slots; i++) + __clear_bit_le(bit_pos + i, &dentry_blk->dentry_bitmap); + + /* Let's check and deallocate this dentry page */ + bit_pos = find_next_bit_le(&dentry_blk->dentry_bitmap, + NR_DENTRY_IN_BLOCK, + 0); + folio_mark_dirty(folio); + + if (bit_pos == NR_DENTRY_IN_BLOCK && + !f2fs_truncate_hole(dir, index, index + 1)) { + f2fs_clear_page_cache_dirty_tag(folio); + folio_clear_dirty_for_io(folio); + folio_clear_uptodate(folio); + folio_detach_private(folio); + + inode_dec_dirty_pages(dir); + f2fs_remove_dirty_inode(dir); + } + f2fs_folio_put(folio, true); + + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + f2fs_mark_inode_dirty_sync(dir, false); + + if (inode) + f2fs_drop_nlink(dir, inode); +} + +bool f2fs_empty_dir(struct inode *dir) +{ + unsigned long bidx = 0; + unsigned int bit_pos; + struct f2fs_dentry_block *dentry_blk; + unsigned long nblock = dir_blocks(dir); + + if (f2fs_has_inline_dentry(dir)) + return f2fs_empty_inline_dir(dir); + + while (bidx < nblock) { + pgoff_t next_pgofs; + struct folio *dentry_folio; + + dentry_folio = f2fs_find_data_folio(dir, bidx, &next_pgofs); + if (IS_ERR(dentry_folio)) { + if (PTR_ERR(dentry_folio) == -ENOENT) { + bidx = next_pgofs; + continue; + } else { + return false; + } + } + + dentry_blk = folio_address(dentry_folio); + if (bidx == 0) + bit_pos = 2; + else + bit_pos = 0; + bit_pos = find_next_bit_le(&dentry_blk->dentry_bitmap, + NR_DENTRY_IN_BLOCK, + bit_pos); + + f2fs_folio_put(dentry_folio, false); + + if (bit_pos < NR_DENTRY_IN_BLOCK) + return false; + + bidx++; + } + return true; +} + +int f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d, + unsigned int start_pos, struct fscrypt_str *fstr) +{ + unsigned char d_type = DT_UNKNOWN; + unsigned int bit_pos; + struct f2fs_dir_entry *de = NULL; + struct fscrypt_str de_name = FSTR_INIT(NULL, 0); + struct f2fs_sb_info *sbi = F2FS_I_SB(d->inode); + struct blk_plug plug; + bool readdir_ra = sbi->readdir_ra; + bool found_valid_dirent = false; + int err = 0; + + bit_pos = ((unsigned long)ctx->pos % d->max); + + if (readdir_ra) + blk_start_plug(&plug); + + while (bit_pos < d->max) { + bit_pos = find_next_bit_le(d->bitmap, d->max, bit_pos); + if (bit_pos >= d->max) + break; + + de = &d->dentry[bit_pos]; + if (de->name_len == 0) { + if (found_valid_dirent || !bit_pos) { + f2fs_warn_ratelimited(sbi, + "invalid namelen(0), ino:%u, run fsck to fix.", + le32_to_cpu(de->ino)); + set_sbi_flag(sbi, SBI_NEED_FSCK); + } + bit_pos++; + ctx->pos = start_pos + bit_pos; + continue; + } + + d_type = fs_ftype_to_dtype(de->file_type); + + de_name.name = d->filename[bit_pos]; + de_name.len = le16_to_cpu(de->name_len); + + /* check memory boundary before moving forward */ + bit_pos += GET_DENTRY_SLOTS(le16_to_cpu(de->name_len)); + if (unlikely(bit_pos > d->max || + le16_to_cpu(de->name_len) > F2FS_NAME_LEN)) { + f2fs_warn(sbi, "%s: corrupted namelen=%d, run fsck to fix.", + __func__, le16_to_cpu(de->name_len)); + set_sbi_flag(sbi, SBI_NEED_FSCK); + err = -EFSCORRUPTED; + f2fs_handle_error(sbi, ERROR_CORRUPTED_DIRENT); + fserror_report_file_metadata(d->inode, err, GFP_NOFS); + goto out; + } + + if (IS_ENCRYPTED(d->inode)) { + int save_len = fstr->len; + + err = fscrypt_fname_disk_to_usr(d->inode, + (u32)le32_to_cpu(de->hash_code), + 0, &de_name, fstr); + if (err) + goto out; + + de_name = *fstr; + fstr->len = save_len; + } + + if (!dir_emit(ctx, de_name.name, de_name.len, + le32_to_cpu(de->ino), d_type)) { + err = 1; + goto out; + } + + if (readdir_ra) + f2fs_ra_node_page(sbi, le32_to_cpu(de->ino)); + + ctx->pos = start_pos + bit_pos; + found_valid_dirent = true; + } +out: + if (readdir_ra) + blk_finish_plug(&plug); + return err; +} + +static int f2fs_readdir(struct file *file, struct dir_context *ctx) +{ + struct inode *inode = file_inode(file); + unsigned long npages = dir_blocks(inode); + struct f2fs_dentry_block *dentry_blk = NULL; + struct file_ra_state *ra = &file->f_ra; + loff_t start_pos = ctx->pos; + unsigned int n = ((unsigned long)ctx->pos / NR_DENTRY_IN_BLOCK); + struct f2fs_dentry_ptr d; + struct fscrypt_str fstr = FSTR_INIT(NULL, 0); + int err = 0; + + if (IS_ENCRYPTED(inode)) { + err = fscrypt_prepare_readdir(inode); + if (err) + goto out; + + err = fscrypt_fname_alloc_buffer(F2FS_NAME_LEN, &fstr); + if (err < 0) + goto out; + } + + if (f2fs_has_inline_dentry(inode)) { + err = f2fs_read_inline_dir(file, ctx, &fstr); + goto out_free; + } + + for (; n < npages; ctx->pos = n * NR_DENTRY_IN_BLOCK) { + struct folio *dentry_folio; + pgoff_t next_pgofs; + + /* allow readdir() to be interrupted */ + if (fatal_signal_pending(current)) { + err = -ERESTARTSYS; + goto out_free; + } + cond_resched(); + + /* readahead for multi pages of dir */ + if (npages - n > 1 && !ra_has_index(ra, n)) + page_cache_sync_readahead(inode->i_mapping, ra, file, n, + min(npages - n, (pgoff_t)MAX_DIR_RA_PAGES)); + + dentry_folio = f2fs_find_data_folio(inode, n, &next_pgofs); + if (IS_ERR(dentry_folio)) { + err = PTR_ERR(dentry_folio); + if (err == -ENOENT) { + err = 0; + n = next_pgofs; + continue; + } else { + goto out_free; + } + } + + dentry_blk = folio_address(dentry_folio); + + make_dentry_ptr_block(inode, &d, dentry_blk); + + err = f2fs_fill_dentries(ctx, &d, + n * NR_DENTRY_IN_BLOCK, &fstr); + f2fs_folio_put(dentry_folio, false); + if (err) + break; + + n++; + } +out_free: + fscrypt_fname_free_buffer(&fstr); +out: + trace_f2fs_readdir(inode, start_pos, ctx->pos, err); + return err < 0 ? err : 0; +} + +const struct file_operations f2fs_dir_operations = { + .llseek = generic_file_llseek, + .read = generic_read_dir, + .iterate_shared = f2fs_readdir, + .fsync = f2fs_sync_file, + .unlocked_ioctl = f2fs_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = f2fs_compat_ioctl, +#endif + .setlease = generic_setlease, +}; diff --git a/fs/f2fs/extent_cache.c b/fs/f2fs/extent_cache.c new file mode 100644 index 000000000..61f6b9714 --- /dev/null +++ b/fs/f2fs/extent_cache.c @@ -0,0 +1,1253 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * f2fs extent cache support + * + * Copyright (c) 2015 Motorola Mobility + * Copyright (c) 2015 Samsung Electronics + * Authors: Jaegeuk Kim + * Chao Yu + * + * block_age-based extent cache added by: + * Copyright (c) 2022 xiaomi Co., Ltd. + * http://www.xiaomi.com/ + */ + +#include +#include + +#include "f2fs.h" +#include "node.h" +#include + +bool sanity_check_extent_cache(struct inode *inode, struct folio *ifolio) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_extent *i_ext = &F2FS_INODE(ifolio)->i_ext; + struct extent_info ei; + int devi; + + get_read_extent_info(&ei, i_ext); + + if (!ei.len) + return true; + + if (!f2fs_is_valid_blkaddr(sbi, ei.blk, DATA_GENERIC_ENHANCE) || + !f2fs_is_valid_blkaddr(sbi, ei.blk + ei.len - 1, + DATA_GENERIC_ENHANCE)) { + f2fs_warn(sbi, "%s: inode (ino=%llx) extent info [%u, %u, %u] is incorrect, run fsck to fix", + __func__, inode->i_ino, + ei.blk, ei.fofs, ei.len); + return false; + } + + if (!IS_DEVICE_ALIASING(inode)) + return true; + + for (devi = 0; devi < sbi->s_ndevs; devi++) { + if (FDEV(devi).start_blk != ei.blk || + FDEV(devi).end_blk != ei.blk + ei.len - 1) + continue; + + if (devi == 0) { + f2fs_warn(sbi, + "%s: inode (ino=%llx) is an alias of meta device", + __func__, inode->i_ino); + return false; + } + + if (bdev_is_zoned(FDEV(devi).bdev)) { + f2fs_warn(sbi, + "%s: device alias inode (ino=%llx)'s extent info " + "[%u, %u, %u] maps to zoned block device", + __func__, inode->i_ino, ei.blk, ei.fofs, ei.len); + return false; + } + return true; + } + + f2fs_warn(sbi, "%s: device alias inode (ino=%llx)'s extent info " + "[%u, %u, %u] is inconsistent w/ any devices", + __func__, inode->i_ino, ei.blk, ei.fofs, ei.len); + return false; +} + +static void __set_extent_info(struct extent_info *ei, + unsigned int fofs, unsigned int len, + block_t blk, bool keep_clen, + unsigned long age, unsigned long last_blocks, + enum extent_type type) +{ + ei->fofs = fofs; + ei->len = len; + + if (type == EX_READ) { + ei->blk = blk; + if (keep_clen) + return; +#ifdef CONFIG_F2FS_FS_COMPRESSION + ei->c_len = 0; +#endif + } else if (type == EX_BLOCK_AGE) { + ei->age = age; + ei->last_blocks = last_blocks; + } +} + +static bool __init_may_extent_tree(struct inode *inode, enum extent_type type) +{ + if (type == EX_READ) + return test_opt(F2FS_I_SB(inode), READ_EXTENT_CACHE) && + S_ISREG(inode->i_mode); + if (type == EX_BLOCK_AGE) + return test_opt(F2FS_I_SB(inode), AGE_EXTENT_CACHE) && + (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode)); + return false; +} + +static bool __may_extent_tree(struct inode *inode, enum extent_type type) +{ + if (IS_DEVICE_ALIASING(inode) && type == EX_READ) + return true; + + /* + * for recovered files during mount do not create extents + * if shrinker is not registered. + */ + if (list_empty(&F2FS_I_SB(inode)->s_list)) + return false; + + if (!__init_may_extent_tree(inode, type)) + return false; + + if (type == EX_READ) { + if (is_inode_flag_set(inode, FI_NO_EXTENT)) + return false; + if (is_inode_flag_set(inode, FI_COMPRESSED_FILE) && + !f2fs_sb_has_readonly(F2FS_I_SB(inode))) + return false; + } else if (type == EX_BLOCK_AGE) { + if (is_inode_flag_set(inode, FI_COMPRESSED_FILE)) + return false; + if (file_is_cold(inode)) + return false; + } + return true; +} + +static void __try_update_largest_extent(struct extent_tree *et, + struct extent_node *en) +{ + if (et->type != EX_READ) + return; + if (en->ei.len <= et->largest.len) + return; + + et->largest = en->ei; + et->largest_updated = true; +} + +static bool __is_extent_mergeable(struct extent_info *back, + struct extent_info *front, enum extent_type type) +{ + if (type == EX_READ) { +#ifdef CONFIG_F2FS_FS_COMPRESSION + if (back->c_len && back->len != back->c_len) + return false; + if (front->c_len && front->len != front->c_len) + return false; +#endif + return (back->fofs + back->len == front->fofs && + back->blk + back->len == front->blk); + } else if (type == EX_BLOCK_AGE) { + return (back->fofs + back->len == front->fofs && + abs(back->age - front->age) <= SAME_AGE_REGION && + abs(back->last_blocks - front->last_blocks) <= + SAME_AGE_REGION); + } + return false; +} + +static bool __is_back_mergeable(struct extent_info *cur, + struct extent_info *back, enum extent_type type) +{ + return __is_extent_mergeable(back, cur, type); +} + +static bool __is_front_mergeable(struct extent_info *cur, + struct extent_info *front, enum extent_type type) +{ + return __is_extent_mergeable(cur, front, type); +} + +static struct extent_node *__lookup_extent_node(struct rb_root_cached *root, + struct extent_node *cached_en, unsigned int fofs) +{ + struct rb_node *node = root->rb_root.rb_node; + struct extent_node *en; + + /* check a cached entry */ + if (cached_en && cached_en->ei.fofs <= fofs && + cached_en->ei.fofs + cached_en->ei.len > fofs) + return cached_en; + + /* check rb_tree */ + while (node) { + en = rb_entry(node, struct extent_node, rb_node); + + if (fofs < en->ei.fofs) + node = node->rb_left; + else if (fofs >= en->ei.fofs + en->ei.len) + node = node->rb_right; + else + return en; + } + return NULL; +} + +/* + * lookup rb entry in position of @fofs in rb-tree, + * if hit, return the entry, otherwise, return NULL + * @prev_ex: extent before fofs + * @next_ex: extent after fofs + * @insert_p: insert point for new extent at fofs + * in order to simplify the insertion after. + * tree must stay unchanged between lookup and insertion. + */ +static struct extent_node *__lookup_extent_node_ret(struct rb_root_cached *root, + struct extent_node *cached_en, + unsigned int fofs, + struct extent_node **prev_entry, + struct extent_node **next_entry, + struct rb_node ***insert_p, + struct rb_node **insert_parent, + bool *leftmost) +{ + struct rb_node **pnode = &root->rb_root.rb_node; + struct rb_node *parent = NULL, *tmp_node; + struct extent_node *en = cached_en; + + *insert_p = NULL; + *insert_parent = NULL; + *prev_entry = NULL; + *next_entry = NULL; + + if (RB_EMPTY_ROOT(&root->rb_root)) + return NULL; + + if (en && en->ei.fofs <= fofs && en->ei.fofs + en->ei.len > fofs) + goto lookup_neighbors; + + *leftmost = true; + + while (*pnode) { + parent = *pnode; + en = rb_entry(*pnode, struct extent_node, rb_node); + + if (fofs < en->ei.fofs) { + pnode = &(*pnode)->rb_left; + } else if (fofs >= en->ei.fofs + en->ei.len) { + pnode = &(*pnode)->rb_right; + *leftmost = false; + } else { + goto lookup_neighbors; + } + } + + *insert_p = pnode; + *insert_parent = parent; + + en = rb_entry(parent, struct extent_node, rb_node); + tmp_node = parent; + if (parent && fofs > en->ei.fofs) + tmp_node = rb_next(parent); + *next_entry = rb_entry_safe(tmp_node, struct extent_node, rb_node); + + tmp_node = parent; + if (parent && fofs < en->ei.fofs) + tmp_node = rb_prev(parent); + *prev_entry = rb_entry_safe(tmp_node, struct extent_node, rb_node); + return NULL; + +lookup_neighbors: + if (fofs == en->ei.fofs) { + /* lookup prev node for merging backward later */ + tmp_node = rb_prev(&en->rb_node); + *prev_entry = rb_entry_safe(tmp_node, + struct extent_node, rb_node); + } + if (fofs == en->ei.fofs + en->ei.len - 1) { + /* lookup next node for merging frontward later */ + tmp_node = rb_next(&en->rb_node); + *next_entry = rb_entry_safe(tmp_node, + struct extent_node, rb_node); + } + return en; +} + +static struct kmem_cache *extent_tree_slab; +static struct kmem_cache *extent_node_slab; + +static struct extent_node *__attach_extent_node(struct f2fs_sb_info *sbi, + struct extent_tree *et, struct extent_info *ei, + struct rb_node *parent, struct rb_node **p, + bool leftmost) +{ + struct extent_tree_info *eti = &sbi->extent_tree[et->type]; + struct extent_node *en; + + en = f2fs_kmem_cache_alloc(extent_node_slab, GFP_ATOMIC, false, sbi); + if (!en) + return NULL; + + en->ei = *ei; + INIT_LIST_HEAD(&en->list); + en->et = et; + + rb_link_node(&en->rb_node, parent, p); + rb_insert_color_cached(&en->rb_node, &et->root, leftmost); + atomic_inc(&et->node_cnt); + atomic_inc(&eti->total_ext_node); + return en; +} + +static void __detach_extent_node(struct f2fs_sb_info *sbi, + struct extent_tree *et, struct extent_node *en) +{ + struct extent_tree_info *eti = &sbi->extent_tree[et->type]; + + rb_erase_cached(&en->rb_node, &et->root); + atomic_dec(&et->node_cnt); + atomic_dec(&eti->total_ext_node); + + if (et->cached_en == en) + et->cached_en = NULL; + kmem_cache_free(extent_node_slab, en); +} + +/* + * Flow to release an extent_node: + * 1. list_del_init + * 2. __detach_extent_node + * 3. kmem_cache_free. + */ +static void __release_extent_node(struct f2fs_sb_info *sbi, + struct extent_tree *et, struct extent_node *en) +{ + struct extent_tree_info *eti = &sbi->extent_tree[et->type]; + + spin_lock(&eti->extent_lock); + f2fs_bug_on(sbi, list_empty(&en->list)); + list_del_init(&en->list); + spin_unlock(&eti->extent_lock); + + __detach_extent_node(sbi, et, en); +} + +static struct extent_tree *__grab_extent_tree(struct inode *inode, + enum extent_type type) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct extent_tree_info *eti = &sbi->extent_tree[type]; + struct extent_tree *et; + nid_t ino = inode->i_ino; + + mutex_lock(&eti->extent_tree_lock); + et = radix_tree_lookup(&eti->extent_tree_root, ino); + if (!et) { + et = f2fs_kmem_cache_alloc(extent_tree_slab, + GFP_NOFS, true, NULL); + f2fs_radix_tree_insert(&eti->extent_tree_root, ino, et); + memset(et, 0, sizeof(struct extent_tree)); + et->ino = ino; + et->type = type; + et->root = RB_ROOT_CACHED; + et->cached_en = NULL; + rwlock_init(&et->lock); + INIT_LIST_HEAD(&et->list); + atomic_set(&et->node_cnt, 0); + atomic_inc(&eti->total_ext_tree); + } else { + atomic_dec(&eti->total_zombie_tree); + list_del_init(&et->list); + } + mutex_unlock(&eti->extent_tree_lock); + + /* never died until evict_inode */ + F2FS_I(inode)->extent_tree[type] = et; + + return et; +} + +static unsigned int __free_extent_tree(struct f2fs_sb_info *sbi, + struct extent_tree *et, unsigned int nr_shrink) +{ + struct rb_node *node, *next; + struct extent_node *en; + unsigned int count; + + node = rb_first_cached(&et->root); + + for (count = 0; node && count < nr_shrink; count++) { + next = rb_next(node); + en = rb_entry(node, struct extent_node, rb_node); + __release_extent_node(sbi, et, en); + node = next; + } + + return count; +} + +static void __drop_largest_extent(struct extent_tree *et, + pgoff_t fofs, unsigned int len) +{ + if (fofs < (pgoff_t)et->largest.fofs + et->largest.len && + fofs + len > et->largest.fofs) { + et->largest.len = 0; + et->largest_updated = true; + } +} + +void f2fs_init_read_extent_tree(struct inode *inode, struct folio *ifolio) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct extent_tree_info *eti = &sbi->extent_tree[EX_READ]; + struct f2fs_extent *i_ext = &F2FS_INODE(ifolio)->i_ext; + struct extent_tree *et; + struct extent_node *en; + struct extent_info ei = {0}; + + if (!__may_extent_tree(inode, EX_READ)) { + /* drop largest read extent */ + if (i_ext->len) { + f2fs_folio_wait_writeback(ifolio, NODE, true, true); + i_ext->len = 0; + folio_mark_dirty(ifolio); + } + set_inode_flag(inode, FI_NO_EXTENT); + return; + } + + et = __grab_extent_tree(inode, EX_READ); + + get_read_extent_info(&ei, i_ext); + + write_lock(&et->lock); + if (atomic_read(&et->node_cnt) || !ei.len) + goto skip; + + if (IS_DEVICE_ALIASING(inode)) { + et->largest = ei; + goto skip; + } + + en = __attach_extent_node(sbi, et, &ei, NULL, + &et->root.rb_root.rb_node, true); + if (en) { + et->largest = en->ei; + et->cached_en = en; + + spin_lock(&eti->extent_lock); + list_add_tail(&en->list, &eti->extent_list); + spin_unlock(&eti->extent_lock); + } +skip: + /* Let's drop, if checkpoint got corrupted. */ + if (f2fs_cp_error(sbi)) { + et->largest.len = 0; + et->largest_updated = true; + } + write_unlock(&et->lock); +} + +void f2fs_init_age_extent_tree(struct inode *inode) +{ + if (!__init_may_extent_tree(inode, EX_BLOCK_AGE)) + return; + __grab_extent_tree(inode, EX_BLOCK_AGE); +} + +void f2fs_init_extent_tree(struct inode *inode) +{ + /* initialize read cache */ + if (__init_may_extent_tree(inode, EX_READ)) + __grab_extent_tree(inode, EX_READ); + + /* initialize block age cache */ + if (__init_may_extent_tree(inode, EX_BLOCK_AGE)) + __grab_extent_tree(inode, EX_BLOCK_AGE); +} + +static bool __lookup_extent_tree(struct inode *inode, pgoff_t pgofs, + struct extent_info *ei, enum extent_type type) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct extent_tree_info *eti = &sbi->extent_tree[type]; + struct extent_tree *et = F2FS_I(inode)->extent_tree[type]; + struct extent_node *en; + bool ret = false; + + if (!et) + return false; + + trace_f2fs_lookup_extent_tree_start(inode, pgofs, type); + + read_lock(&et->lock); + + if (type == EX_READ && + et->largest.fofs <= pgofs && + (pgoff_t)et->largest.fofs + et->largest.len > pgofs) { + *ei = et->largest; + ret = true; + stat_inc_largest_node_hit(sbi); + goto out; + } + + if (IS_DEVICE_ALIASING(inode)) { + ret = false; + goto out; + } + + en = __lookup_extent_node(&et->root, et->cached_en, pgofs); + if (!en) + goto out; + + if (en == et->cached_en) + stat_inc_cached_node_hit(sbi, type); + else + stat_inc_rbtree_node_hit(sbi, type); + + *ei = en->ei; + spin_lock(&eti->extent_lock); + if (!list_empty(&en->list)) { + list_move_tail(&en->list, &eti->extent_list); + et->cached_en = en; + } + spin_unlock(&eti->extent_lock); + ret = true; +out: + stat_inc_total_hit(sbi, type); + read_unlock(&et->lock); + + if (type == EX_READ) + trace_f2fs_lookup_read_extent_tree_end(inode, pgofs, ei); + else if (type == EX_BLOCK_AGE) + trace_f2fs_lookup_age_extent_tree_end(inode, pgofs, ei); + return ret; +} + +static struct extent_node *__try_merge_extent_node(struct f2fs_sb_info *sbi, + struct extent_tree *et, struct extent_info *ei, + struct extent_node *prev_ex, + struct extent_node *next_ex) +{ + struct extent_tree_info *eti = &sbi->extent_tree[et->type]; + struct extent_node *en = NULL; + + if (prev_ex && __is_back_mergeable(ei, &prev_ex->ei, et->type)) { + prev_ex->ei.len += ei->len; + ei = &prev_ex->ei; + en = prev_ex; + } + + if (next_ex && __is_front_mergeable(ei, &next_ex->ei, et->type)) { + next_ex->ei.fofs = ei->fofs; + next_ex->ei.len += ei->len; + if (et->type == EX_READ) + next_ex->ei.blk = ei->blk; + if (en) + __release_extent_node(sbi, et, prev_ex); + + en = next_ex; + } + + if (!en) + return NULL; + + __try_update_largest_extent(et, en); + + spin_lock(&eti->extent_lock); + if (!list_empty(&en->list)) { + list_move_tail(&en->list, &eti->extent_list); + et->cached_en = en; + } + spin_unlock(&eti->extent_lock); + return en; +} + +static struct extent_node *__insert_extent_tree(struct f2fs_sb_info *sbi, + struct extent_tree *et, struct extent_info *ei, + struct rb_node **insert_p, + struct rb_node *insert_parent, + bool leftmost) +{ + struct extent_tree_info *eti = &sbi->extent_tree[et->type]; + struct rb_node **p = &et->root.rb_root.rb_node; + struct rb_node *parent = NULL; + struct extent_node *en = NULL; + + if (insert_p && insert_parent) { + parent = insert_parent; + p = insert_p; + goto do_insert; + } + + leftmost = true; + + /* look up extent_node in the rb tree */ + while (*p) { + parent = *p; + en = rb_entry(parent, struct extent_node, rb_node); + + if (ei->fofs < en->ei.fofs) { + p = &(*p)->rb_left; + } else if (ei->fofs >= en->ei.fofs + en->ei.len) { + p = &(*p)->rb_right; + leftmost = false; + } else { + f2fs_err_ratelimited(sbi, "%s: corrupted extent, type: %d, " + "extent node in rb tree [%u, %u, %u], age [%llu, %llu], " + "extent node to insert [%u, %u, %u], age [%llu, %llu]", + __func__, et->type, en->ei.fofs, en->ei.blk, en->ei.len, en->ei.age, + en->ei.last_blocks, ei->fofs, ei->blk, ei->len, ei->age, ei->last_blocks); + f2fs_bug_on(sbi, 1); + return NULL; + } + } + +do_insert: + en = __attach_extent_node(sbi, et, ei, parent, p, leftmost); + if (!en) + return NULL; + + __try_update_largest_extent(et, en); + + /* update in global extent list */ + spin_lock(&eti->extent_lock); + list_add_tail(&en->list, &eti->extent_list); + et->cached_en = en; + spin_unlock(&eti->extent_lock); + return en; +} + +static unsigned int __destroy_extent_node(struct inode *inode, + enum extent_type type) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct extent_tree *et = F2FS_I(inode)->extent_tree[type]; + unsigned int nr_shrink = type == EX_READ ? + READ_EXTENT_CACHE_SHRINK_NUMBER : + AGE_EXTENT_CACHE_SHRINK_NUMBER; + unsigned int node_cnt = 0; + + if (!et || !atomic_read(&et->node_cnt)) + return 0; + + while (atomic_read(&et->node_cnt)) { + write_lock(&et->lock); + node_cnt += __free_extent_tree(sbi, et, nr_shrink); + write_unlock(&et->lock); + } + + return node_cnt; +} + +static void __update_extent_tree_range(struct inode *inode, + struct extent_info *tei, enum extent_type type) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct extent_tree *et = F2FS_I(inode)->extent_tree[type]; + struct extent_node *en = NULL, *en1 = NULL; + struct extent_node *prev_en = NULL, *next_en = NULL; + struct extent_info ei, dei, prev; + struct rb_node **insert_p = NULL, *insert_parent = NULL; + unsigned int fofs = tei->fofs, len = tei->len; + unsigned int end = fofs + len; + bool updated = false; + bool leftmost = false; + + if (!et) + return; + + if (unlikely(len == 0)) { + f2fs_err_ratelimited(sbi, "%s: extent len is zero, type: %d, " + "extent [%u, %u, %u], age [%llu, %llu]", + __func__, type, tei->fofs, tei->blk, tei->len, + tei->age, tei->last_blocks); + f2fs_bug_on(sbi, 1); + return; + } + + if (type == EX_READ) + trace_f2fs_update_read_extent_tree_range(inode, fofs, len, + tei->blk, 0); + else if (type == EX_BLOCK_AGE) + trace_f2fs_update_age_extent_tree_range(inode, fofs, len, + tei->age, tei->last_blocks); + + write_lock(&et->lock); + + if (type == EX_READ) { + if (is_inode_flag_set(inode, FI_NO_EXTENT)) { + write_unlock(&et->lock); + return; + } + + prev = et->largest; + dei.len = 0; + + /* + * drop largest extent before lookup, in case it's already + * been shrunk from extent tree + */ + __drop_largest_extent(et, fofs, len); + } + + /* 1. lookup first extent node in range [fofs, fofs + len - 1] */ + en = __lookup_extent_node_ret(&et->root, + et->cached_en, fofs, + &prev_en, &next_en, + &insert_p, &insert_parent, + &leftmost); + if (!en) + en = next_en; + + /* 2. invalidate all extent nodes in range [fofs, fofs + len - 1] */ + while (en && en->ei.fofs < end) { + unsigned int org_end; + int parts = 0; /* # of parts current extent split into */ + + next_en = en1 = NULL; + + dei = en->ei; + org_end = dei.fofs + dei.len; + f2fs_bug_on(sbi, fofs >= org_end); + + if (fofs > dei.fofs && (type != EX_READ || + fofs - dei.fofs >= F2FS_MIN_EXTENT_LEN)) { + en->ei.len = fofs - en->ei.fofs; + prev_en = en; + parts = 1; + } + + if (end < org_end && (type != EX_READ || + (org_end - end >= F2FS_MIN_EXTENT_LEN && + atomic_read(&et->node_cnt) < + sbi->max_read_extent_count))) { + if (parts) { + __set_extent_info(&ei, + end, org_end - end, + end - dei.fofs + dei.blk, false, + dei.age, dei.last_blocks, + type); + en1 = __insert_extent_tree(sbi, et, &ei, + NULL, NULL, true); + next_en = en1; + } else { + __set_extent_info(&en->ei, + end, en->ei.len - (end - dei.fofs), + en->ei.blk + (end - dei.fofs), true, + dei.age, dei.last_blocks, + type); + next_en = en; + } + parts++; + } + + if (!next_en) { + struct rb_node *node = rb_next(&en->rb_node); + + next_en = rb_entry_safe(node, struct extent_node, + rb_node); + } + + if (parts) + __try_update_largest_extent(et, en); + else + __release_extent_node(sbi, et, en); + + /* + * if original extent is split into zero or two parts, extent + * tree has been altered by deletion or insertion, therefore + * invalidate pointers regard to tree. + */ + if (parts != 1) { + insert_p = NULL; + insert_parent = NULL; + } + en = next_en; + } + + if (type == EX_BLOCK_AGE) + goto update_age_extent_cache; + + /* 3. update extent in read extent cache */ + BUG_ON(type != EX_READ); + + if (tei->blk) { + __set_extent_info(&ei, fofs, len, tei->blk, false, + 0, 0, EX_READ); + if (!__try_merge_extent_node(sbi, et, &ei, prev_en, next_en)) + __insert_extent_tree(sbi, et, &ei, + insert_p, insert_parent, leftmost); + + /* give up extent_cache, if split and small updates happen */ + if (dei.len >= 1 && + prev.len < F2FS_MIN_EXTENT_LEN && + et->largest.len < F2FS_MIN_EXTENT_LEN) { + et->largest.len = 0; + et->largest_updated = true; + set_inode_flag(inode, FI_NO_EXTENT); + } + } + + if (et->largest_updated) { + et->largest_updated = false; + updated = true; + } + goto out_read_extent_cache; +update_age_extent_cache: + if (tei->last_blocks == F2FS_EXTENT_AGE_INVALID) + goto out_read_extent_cache; + + __set_extent_info(&ei, fofs, len, 0, false, + tei->age, tei->last_blocks, EX_BLOCK_AGE); + if (!__try_merge_extent_node(sbi, et, &ei, prev_en, next_en)) + __insert_extent_tree(sbi, et, &ei, + insert_p, insert_parent, leftmost); +out_read_extent_cache: + write_unlock(&et->lock); + + if (is_inode_flag_set(inode, FI_NO_EXTENT)) + __destroy_extent_node(inode, EX_READ); + + if (updated) + f2fs_mark_inode_dirty_sync(inode, true); +} + +#ifdef CONFIG_F2FS_FS_COMPRESSION +void f2fs_update_read_extent_tree_range_compressed(struct inode *inode, + pgoff_t fofs, block_t blkaddr, unsigned int llen, + unsigned int c_len) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct extent_tree *et = F2FS_I(inode)->extent_tree[EX_READ]; + struct extent_node *en = NULL; + struct extent_node *prev_en = NULL, *next_en = NULL; + struct extent_info ei; + struct rb_node **insert_p = NULL, *insert_parent = NULL; + bool leftmost = false; + + trace_f2fs_update_read_extent_tree_range(inode, fofs, llen, + blkaddr, c_len); + + /* it is safe here to check FI_NO_EXTENT w/o et->lock in ro image */ + if (is_inode_flag_set(inode, FI_NO_EXTENT)) + return; + + write_lock(&et->lock); + + en = __lookup_extent_node_ret(&et->root, + et->cached_en, fofs, + &prev_en, &next_en, + &insert_p, &insert_parent, + &leftmost); + if (en) + goto unlock_out; + + __set_extent_info(&ei, fofs, llen, blkaddr, true, 0, 0, EX_READ); + ei.c_len = c_len; + + if (!__try_merge_extent_node(sbi, et, &ei, prev_en, next_en)) + __insert_extent_tree(sbi, et, &ei, + insert_p, insert_parent, leftmost); +unlock_out: + write_unlock(&et->lock); +} +#endif + +static unsigned long long __calculate_block_age(struct f2fs_sb_info *sbi, + unsigned long long new, + unsigned long long old) +{ + unsigned int rem_old, rem_new; + unsigned long long res; + unsigned int weight = sbi->last_age_weight; + + res = div_u64_rem(new, 100, &rem_new) * (100 - weight) + + div_u64_rem(old, 100, &rem_old) * weight; + + if (rem_new) + res += rem_new * (100 - weight) / 100; + if (rem_old) + res += rem_old * weight / 100; + + return res; +} + +/* This returns a new age and allocated blocks in ei */ +static int __get_new_block_age(struct inode *inode, struct extent_info *ei, + block_t blkaddr) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + loff_t f_size = i_size_read(inode); + unsigned long long cur_blocks = + atomic64_read(&sbi->allocated_data_blocks); + struct extent_info tei = *ei; /* only fofs and len are valid */ + + /* + * When I/O is not aligned to a PAGE_SIZE, update will happen to the last + * file block even in seq write. So don't record age for newly last file + * block here. + */ + if ((f_size >> PAGE_SHIFT) == ei->fofs && f_size & (PAGE_SIZE - 1) && + blkaddr == NEW_ADDR) + return -EINVAL; + + if (__lookup_extent_tree(inode, ei->fofs, &tei, EX_BLOCK_AGE)) { + unsigned long long cur_age; + + if (cur_blocks >= tei.last_blocks) + cur_age = cur_blocks - tei.last_blocks; + else + /* allocated_data_blocks overflow */ + cur_age = (ULLONG_MAX - 1) - tei.last_blocks + cur_blocks; + + if (tei.age) + ei->age = __calculate_block_age(sbi, cur_age, tei.age); + else + ei->age = cur_age; + ei->last_blocks = cur_blocks; + WARN_ON(ei->age > cur_blocks); + return 0; + } + + f2fs_bug_on(sbi, blkaddr == NULL_ADDR); + + /* the data block was allocated for the first time */ + if (blkaddr == NEW_ADDR) + goto out; + + if (__is_valid_data_blkaddr(blkaddr) && + !f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE)) + return -EINVAL; +out: + /* + * init block age with zero, this can happen when the block age extent + * was reclaimed due to memory constraint or system reboot + */ + ei->age = 0; + ei->last_blocks = cur_blocks; + return 0; +} + +static void __update_extent_cache(struct dnode_of_data *dn, enum extent_type type) +{ + struct extent_info ei = {}; + + if (!__may_extent_tree(dn->inode, type)) + return; + + ei.fofs = f2fs_start_bidx_of_node(ofs_of_node(dn->node_folio), dn->inode) + + dn->ofs_in_node; + ei.len = 1; + + if (type == EX_READ) { + if (dn->data_blkaddr == NEW_ADDR) + ei.blk = NULL_ADDR; + else + ei.blk = dn->data_blkaddr; + } else if (type == EX_BLOCK_AGE) { + if (__get_new_block_age(dn->inode, &ei, dn->data_blkaddr)) + return; + } + __update_extent_tree_range(dn->inode, &ei, type); +} + +static unsigned int __shrink_extent_tree(struct f2fs_sb_info *sbi, int nr_shrink, + enum extent_type type) +{ + struct extent_tree_info *eti = &sbi->extent_tree[type]; + struct extent_tree *et, *next; + struct extent_node *en; + unsigned int node_cnt = 0, tree_cnt = 0; + int remained; + + if (!atomic_read(&eti->total_zombie_tree)) + goto free_node; + + if (!mutex_trylock(&eti->extent_tree_lock)) + goto out; + + /* 1. remove unreferenced extent tree */ + list_for_each_entry_safe(et, next, &eti->zombie_list, list) { + if (atomic_read(&et->node_cnt)) { + write_lock(&et->lock); + node_cnt += __free_extent_tree(sbi, et, + nr_shrink - node_cnt - tree_cnt); + write_unlock(&et->lock); + } + + if (atomic_read(&et->node_cnt)) + goto unlock_out; + + list_del_init(&et->list); + radix_tree_delete(&eti->extent_tree_root, et->ino); + kmem_cache_free(extent_tree_slab, et); + atomic_dec(&eti->total_ext_tree); + atomic_dec(&eti->total_zombie_tree); + tree_cnt++; + + if (node_cnt + tree_cnt >= nr_shrink) + goto unlock_out; + cond_resched(); + } + mutex_unlock(&eti->extent_tree_lock); + +free_node: + /* 2. remove LRU extent entries */ + if (!mutex_trylock(&eti->extent_tree_lock)) + goto out; + + remained = nr_shrink - (node_cnt + tree_cnt); + + spin_lock(&eti->extent_lock); + for (; remained > 0; remained--) { + if (list_empty(&eti->extent_list)) + break; + en = list_first_entry(&eti->extent_list, + struct extent_node, list); + et = en->et; + if (!write_trylock(&et->lock)) { + /* refresh this extent node's position in extent list */ + list_move_tail(&en->list, &eti->extent_list); + continue; + } + + list_del_init(&en->list); + spin_unlock(&eti->extent_lock); + + __detach_extent_node(sbi, et, en); + + write_unlock(&et->lock); + node_cnt++; + spin_lock(&eti->extent_lock); + } + spin_unlock(&eti->extent_lock); + +unlock_out: + mutex_unlock(&eti->extent_tree_lock); +out: + trace_f2fs_shrink_extent_tree(sbi, node_cnt, tree_cnt, type); + + return node_cnt + tree_cnt; +} + +/* read extent cache operations */ +bool f2fs_lookup_read_extent_cache(struct inode *inode, pgoff_t pgofs, + struct extent_info *ei) +{ + if (!__may_extent_tree(inode, EX_READ)) + return false; + + return __lookup_extent_tree(inode, pgofs, ei, EX_READ); +} + +bool f2fs_lookup_read_extent_cache_block(struct inode *inode, pgoff_t index, + block_t *blkaddr) +{ + struct extent_info ei = {}; + + if (!f2fs_lookup_read_extent_cache(inode, index, &ei)) + return false; + *blkaddr = ei.blk + index - ei.fofs; + return true; +} + +void f2fs_update_read_extent_cache(struct dnode_of_data *dn) +{ + return __update_extent_cache(dn, EX_READ); +} + +void f2fs_update_read_extent_cache_range(struct dnode_of_data *dn, + pgoff_t fofs, block_t blkaddr, unsigned int len) +{ + struct extent_info ei = { + .fofs = fofs, + .len = len, + .blk = blkaddr, + }; + + if (!__may_extent_tree(dn->inode, EX_READ)) + return; + + __update_extent_tree_range(dn->inode, &ei, EX_READ); +} + +unsigned int f2fs_shrink_read_extent_tree(struct f2fs_sb_info *sbi, int nr_shrink) +{ + if (!test_opt(sbi, READ_EXTENT_CACHE)) + return 0; + + return __shrink_extent_tree(sbi, nr_shrink, EX_READ); +} + +/* block age extent cache operations */ +bool f2fs_lookup_age_extent_cache(struct inode *inode, pgoff_t pgofs, + struct extent_info *ei) +{ + if (!__may_extent_tree(inode, EX_BLOCK_AGE)) + return false; + + return __lookup_extent_tree(inode, pgofs, ei, EX_BLOCK_AGE); +} + +void f2fs_update_age_extent_cache(struct dnode_of_data *dn) +{ + return __update_extent_cache(dn, EX_BLOCK_AGE); +} + +void f2fs_update_age_extent_cache_range(struct dnode_of_data *dn, + pgoff_t fofs, unsigned int len) +{ + struct extent_info ei = { + .fofs = fofs, + .len = len, + .last_blocks = F2FS_EXTENT_AGE_INVALID, + }; + + if (!__may_extent_tree(dn->inode, EX_BLOCK_AGE)) + return; + + __update_extent_tree_range(dn->inode, &ei, EX_BLOCK_AGE); +} + +unsigned int f2fs_shrink_age_extent_tree(struct f2fs_sb_info *sbi, int nr_shrink) +{ + if (!test_opt(sbi, AGE_EXTENT_CACHE)) + return 0; + + return __shrink_extent_tree(sbi, nr_shrink, EX_BLOCK_AGE); +} + +void f2fs_destroy_extent_node(struct inode *inode) +{ + __destroy_extent_node(inode, EX_READ); + __destroy_extent_node(inode, EX_BLOCK_AGE); +} + +static void __drop_extent_tree(struct inode *inode, enum extent_type type) +{ + struct extent_tree *et = F2FS_I(inode)->extent_tree[type]; + bool updated = false; + + if (!__may_extent_tree(inode, type)) + return; + + write_lock(&et->lock); + if (type == EX_READ) { + set_inode_flag(inode, FI_NO_EXTENT); + if (et->largest.len) { + et->largest.len = 0; + updated = true; + } + } + write_unlock(&et->lock); + + __destroy_extent_node(inode, type); + + if (updated) + f2fs_mark_inode_dirty_sync(inode, true); +} + +void f2fs_drop_extent_tree(struct inode *inode) +{ + __drop_extent_tree(inode, EX_READ); + __drop_extent_tree(inode, EX_BLOCK_AGE); +} + +static void __destroy_extent_tree(struct inode *inode, enum extent_type type) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct extent_tree_info *eti = &sbi->extent_tree[type]; + struct extent_tree *et = F2FS_I(inode)->extent_tree[type]; + unsigned int node_cnt = 0; + + if (!et) + return; + + if (inode->i_nlink && !is_bad_inode(inode) && + atomic_read(&et->node_cnt)) { + mutex_lock(&eti->extent_tree_lock); + list_add_tail(&et->list, &eti->zombie_list); + atomic_inc(&eti->total_zombie_tree); + mutex_unlock(&eti->extent_tree_lock); + return; + } + + /* free all extent info belong to this extent tree */ + node_cnt = __destroy_extent_node(inode, type); + + /* delete extent tree entry in radix tree */ + mutex_lock(&eti->extent_tree_lock); + f2fs_bug_on(sbi, atomic_read(&et->node_cnt)); + radix_tree_delete(&eti->extent_tree_root, inode->i_ino); + kmem_cache_free(extent_tree_slab, et); + atomic_dec(&eti->total_ext_tree); + mutex_unlock(&eti->extent_tree_lock); + + F2FS_I(inode)->extent_tree[type] = NULL; + + trace_f2fs_destroy_extent_tree(inode, node_cnt, type); +} + +void f2fs_destroy_extent_tree(struct inode *inode) +{ + __destroy_extent_tree(inode, EX_READ); + __destroy_extent_tree(inode, EX_BLOCK_AGE); +} + +static void __init_extent_tree_info(struct extent_tree_info *eti) +{ + INIT_RADIX_TREE(&eti->extent_tree_root, GFP_NOIO); + mutex_init(&eti->extent_tree_lock); + INIT_LIST_HEAD(&eti->extent_list); + spin_lock_init(&eti->extent_lock); + atomic_set(&eti->total_ext_tree, 0); + INIT_LIST_HEAD(&eti->zombie_list); + atomic_set(&eti->total_zombie_tree, 0); + atomic_set(&eti->total_ext_node, 0); +} + +void f2fs_init_extent_cache_info(struct f2fs_sb_info *sbi) +{ + __init_extent_tree_info(&sbi->extent_tree[EX_READ]); + __init_extent_tree_info(&sbi->extent_tree[EX_BLOCK_AGE]); + + /* initialize for block age extents */ + atomic64_set(&sbi->allocated_data_blocks, 0); + sbi->hot_data_age_threshold = DEF_HOT_DATA_AGE_THRESHOLD; + sbi->warm_data_age_threshold = DEF_WARM_DATA_AGE_THRESHOLD; + sbi->last_age_weight = LAST_AGE_WEIGHT; + sbi->max_read_extent_count = DEF_MAX_READ_EXTENT_COUNT; +} + +int __init f2fs_create_extent_cache(void) +{ + extent_tree_slab = f2fs_kmem_cache_create("f2fs_extent_tree", + sizeof(struct extent_tree)); + if (!extent_tree_slab) + return -ENOMEM; + extent_node_slab = f2fs_kmem_cache_create("f2fs_extent_node", + sizeof(struct extent_node)); + if (!extent_node_slab) { + kmem_cache_destroy(extent_tree_slab); + return -ENOMEM; + } + return 0; +} + +void f2fs_destroy_extent_cache(void) +{ + kmem_cache_destroy(extent_node_slab); + kmem_cache_destroy(extent_tree_slab); +} diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h new file mode 100644 index 000000000..8376bbe58 --- /dev/null +++ b/fs/f2fs/f2fs.h @@ -0,0 +1,5160 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * fs/f2fs/f2fs.h + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + */ +#ifndef _LINUX_F2FS_H +#define _LINUX_F2FS_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#ifdef CONFIG_F2FS_CHECK_FS +#define f2fs_bug_on(sbi, condition) BUG_ON(condition) +#else +#define f2fs_bug_on(sbi, condition) \ + do { \ + if (WARN_ON(condition)) \ + set_sbi_flag(sbi, SBI_NEED_FSCK); \ + } while (0) +#endif + +enum { + FAULT_KMALLOC, + FAULT_KVMALLOC, + FAULT_PAGE_ALLOC, + FAULT_PAGE_GET, + FAULT_ALLOC_BIO, /* it's obsolete due to bio_alloc() will never fail */ + FAULT_ALLOC_NID, + FAULT_ORPHAN, + FAULT_BLOCK, + FAULT_DIR_DEPTH, + FAULT_EVICT_INODE, + FAULT_TRUNCATE, + FAULT_READ_IO, + FAULT_CHECKPOINT, + FAULT_DISCARD, /* it's obsolete due to __blkdev_issue_discard() will never fail */ + FAULT_WRITE_IO, + FAULT_SLAB_ALLOC, + FAULT_DQUOT_INIT, + FAULT_LOCK_OP, + FAULT_BLKADDR_VALIDITY, + FAULT_BLKADDR_CONSISTENCE, + FAULT_NO_SEGMENT, + FAULT_INCONSISTENT_FOOTER, + FAULT_ATOMIC_TIMEOUT, + FAULT_VMALLOC, + FAULT_LOCK_TIMEOUT, + FAULT_SKIP_WRITE, + FAULT_MAX, +}; + +/* indicate which option to update */ +enum fault_option { + FAULT_RATE = 1, /* only update fault rate */ + FAULT_TYPE = 2, /* only update fault type */ + FAULT_TIMEOUT = 4, /* only update fault timeout type */ + FAULT_ALL = 8, /* reset all fault injection options/stats */ +}; + +#ifdef CONFIG_F2FS_FAULT_INJECTION +struct f2fs_fault_info { + atomic_t inject_ops; + int inject_rate; + unsigned int inject_type; + /* Used to account total count of injection for each type */ + unsigned int inject_count[FAULT_MAX]; + unsigned int inject_lock_timeout; /* inject lock timeout */ +}; + +extern const char *f2fs_fault_name[FAULT_MAX]; +#define IS_FAULT_SET(fi, type) ((fi)->inject_type & BIT(type)) + +/* maximum retry count for injected failure */ +#define DEFAULT_FAILURE_RETRY_COUNT 8 +#else +#define DEFAULT_FAILURE_RETRY_COUNT 1 +#endif + +enum { + REPORT_FAULT_NEED_FSCK, + REPORT_FAULT_STOP_CP, + REPORT_FAULT_MAX, +}; + +void f2fs_fault_report(struct super_block *sb, unsigned int err_code, + const char *func, unsigned int data); + +/* + * For mount options + */ +enum f2fs_mount_opt { + F2FS_MOUNT_DISABLE_ROLL_FORWARD, + F2FS_MOUNT_DISCARD, + F2FS_MOUNT_NOHEAP, + F2FS_MOUNT_XATTR_USER, + F2FS_MOUNT_POSIX_ACL, + F2FS_MOUNT_DISABLE_EXT_IDENTIFY, + F2FS_MOUNT_INLINE_XATTR, + F2FS_MOUNT_INLINE_DATA, + F2FS_MOUNT_INLINE_DENTRY, + F2FS_MOUNT_FLUSH_MERGE, + F2FS_MOUNT_NOBARRIER, + F2FS_MOUNT_FASTBOOT, + F2FS_MOUNT_READ_EXTENT_CACHE, + F2FS_MOUNT_DATA_FLUSH, + F2FS_MOUNT_FAULT_INJECTION, + F2FS_MOUNT_USRQUOTA, + F2FS_MOUNT_GRPQUOTA, + F2FS_MOUNT_PRJQUOTA, + F2FS_MOUNT_QUOTA, + F2FS_MOUNT_INLINE_XATTR_SIZE, + F2FS_MOUNT_RESERVE_ROOT, + F2FS_MOUNT_DISABLE_CHECKPOINT, + F2FS_MOUNT_NORECOVERY, + F2FS_MOUNT_ATGC, + F2FS_MOUNT_MERGE_CHECKPOINT, + F2FS_MOUNT_GC_MERGE, + F2FS_MOUNT_COMPRESS_CACHE, + F2FS_MOUNT_AGE_EXTENT_CACHE, + F2FS_MOUNT_NAT_BITS, + F2FS_MOUNT_INLINECRYPT, + /* + * Some f2fs environments expect to be able to pass the "lazytime" option + * string rather than using the MS_LAZYTIME flag, so this must remain. + */ + F2FS_MOUNT_LAZYTIME, + F2FS_MOUNT_RESERVE_NODE, +}; + +#define F2FS_OPTION(sbi) ((sbi)->mount_opt) +#define clear_opt(sbi, option) \ + (F2FS_OPTION(sbi).opt &= ~BIT(F2FS_MOUNT_##option)) +#define set_opt(sbi, option) \ + (F2FS_OPTION(sbi).opt |= BIT(F2FS_MOUNT_##option)) +#define test_opt(sbi, option) \ + (F2FS_OPTION(sbi).opt & BIT(F2FS_MOUNT_##option)) + +#define ver_after(a, b) (typecheck(unsigned long long, a) && \ + typecheck(unsigned long long, b) && \ + ((long long)((a) - (b)) > 0)) + +typedef u32 block_t; /* + * should not change u32, since it is the on-disk block + * address format, __le32. + */ +typedef u32 nid_t; + +#define COMPRESS_EXT_NUM 16 + +enum blkzone_allocation_policy { + BLKZONE_ALLOC_PRIOR_SEQ, /* Prioritize writing to sequential zones */ + BLKZONE_ALLOC_ONLY_SEQ, /* Only allow writing to sequential zones */ + BLKZONE_ALLOC_PRIOR_CONV, /* Prioritize writing to conventional zones */ +}; + +enum bggc_io_aware_policy { + AWARE_ALL_IO, /* skip background GC if there is any kind of pending IO */ + AWARE_READ_IO, /* skip background GC if there is pending read IO */ + AWARE_NONE, /* don't aware IO for background GC */ +}; + +enum device_allocation_policy { + ALLOCATE_FORWARD_NOHINT, + ALLOCATE_FORWARD_WITHIN_HINT, + ALLOCATE_FORWARD_FROM_HINT, +}; + +enum f2fs_lock_name { + LOCK_NAME_NONE, + LOCK_NAME_CP_RWSEM, + LOCK_NAME_NODE_CHANGE, + LOCK_NAME_NODE_WRITE, + LOCK_NAME_GC_LOCK, + LOCK_NAME_CP_GLOBAL, + LOCK_NAME_IO_RWSEM, + LOCK_NAME_MAX, +}; + +enum f2fs_timeout_type { + TIMEOUT_TYPE_NONE, + TIMEOUT_TYPE_RUNNING, + TIMEOUT_TYPE_IO_SLEEP, + TIMEOUT_TYPE_NONIO_SLEEP, + TIMEOUT_TYPE_RUNNABLE, + TIMEOUT_TYPE_MAX, +}; + +/* + * An implementation of an rwsem that is explicitly unfair to readers. This + * prevents priority inversion when a low-priority reader acquires the read lock + * while sleeping on the write lock but the write lock is needed by + * higher-priority clients. + */ + +struct f2fs_rwsem { + struct f2fs_sb_info *sbi; + enum f2fs_lock_name name; + struct rw_semaphore internal_rwsem; +#ifdef CONFIG_F2FS_UNFAIR_RWSEM + wait_queue_head_t read_waiters; +#endif +}; + +struct f2fs_mount_info { + unsigned long long opt; + block_t root_reserved_blocks; /* root reserved blocks */ + block_t root_reserved_nodes; /* root reserved nodes */ + kuid_t s_resuid; /* reserved blocks for uid */ + kgid_t s_resgid; /* reserved blocks for gid */ + int active_logs; /* # of active logs */ + int inline_xattr_size; /* inline xattr size */ +#ifdef CONFIG_F2FS_FAULT_INJECTION + struct f2fs_fault_info fault_info; /* For fault injection */ +#endif +#ifdef CONFIG_QUOTA + /* Names of quota files with journalled quota */ + char *s_qf_names[MAXQUOTAS]; + int s_jquota_fmt; /* Format of quota to use */ +#endif + /* For which write hints are passed down to block layer */ + int alloc_mode; /* segment allocation policy */ + int fsync_mode; /* fsync policy */ + int fs_mode; /* fs mode: LFS or ADAPTIVE */ + int bggc_mode; /* bggc mode: off, on or sync */ + int memory_mode; /* memory mode */ + int errors; /* errors parameter */ + int discard_unit; /* + * discard command's offset/size should + * be aligned to this unit: block, + * segment or section + */ + struct fscrypt_dummy_policy dummy_enc_policy; /* test dummy encryption */ + block_t unusable_cap_perc; /* percentage for cap */ + block_t unusable_cap; /* Amount of space allowed to be + * unusable when disabling checkpoint + */ + + /* For compression */ + unsigned char compress_algorithm; /* algorithm type */ + unsigned char compress_log_size; /* cluster log size */ + unsigned char compress_level; /* compress level */ + bool compress_chksum; /* compressed data chksum */ + unsigned char compress_ext_cnt; /* extension count */ + unsigned char nocompress_ext_cnt; /* nocompress extension count */ + int compress_mode; /* compression mode */ + unsigned char extensions[COMPRESS_EXT_NUM][F2FS_EXTENSION_LEN]; /* extensions */ + unsigned char noextensions[COMPRESS_EXT_NUM][F2FS_EXTENSION_LEN]; /* extensions */ + unsigned int lookup_mode; +}; + +#define F2FS_FEATURE_ENCRYPT 0x00000001 +#define F2FS_FEATURE_BLKZONED 0x00000002 +#define F2FS_FEATURE_ATOMIC_WRITE 0x00000004 +#define F2FS_FEATURE_EXTRA_ATTR 0x00000008 +#define F2FS_FEATURE_PRJQUOTA 0x00000010 +#define F2FS_FEATURE_INODE_CHKSUM 0x00000020 +#define F2FS_FEATURE_FLEXIBLE_INLINE_XATTR 0x00000040 +#define F2FS_FEATURE_QUOTA_INO 0x00000080 +#define F2FS_FEATURE_INODE_CRTIME 0x00000100 +#define F2FS_FEATURE_LOST_FOUND 0x00000200 +#define F2FS_FEATURE_VERITY 0x00000400 +#define F2FS_FEATURE_SB_CHKSUM 0x00000800 +#define F2FS_FEATURE_CASEFOLD 0x00001000 +#define F2FS_FEATURE_COMPRESSION 0x00002000 +#define F2FS_FEATURE_RO 0x00004000 +#define F2FS_FEATURE_DEVICE_ALIAS 0x00008000 +#define F2FS_FEATURE_PACKED_SSA 0x00010000 + +#define __F2FS_HAS_FEATURE(raw_super, mask) \ + ((raw_super->feature & cpu_to_le32(mask)) != 0) +#define F2FS_HAS_FEATURE(sbi, mask) __F2FS_HAS_FEATURE(sbi->raw_super, mask) + +/* + * Default values for user and/or group using reserved blocks + */ +#define F2FS_DEF_RESUID 0 +#define F2FS_DEF_RESGID 0 + +/* + * For checkpoint manager + */ +enum { + NAT_BITMAP, + SIT_BITMAP +}; + +#define CP_UMOUNT 0x00000001 +#define CP_FASTBOOT 0x00000002 +#define CP_SYNC 0x00000004 +#define CP_RECOVERY 0x00000008 +#define CP_DISCARD 0x00000010 +#define CP_TRIMMED 0x00000020 +#define CP_PAUSE 0x00000040 +#define CP_RESIZE 0x00000080 + +#define DEF_MAX_DISCARD_REQUEST 8 /* issue 8 discards per round */ +#define DEF_MIN_DISCARD_ISSUE_TIME 50 /* 50 ms, if exists */ +#define DEF_MID_DISCARD_ISSUE_TIME 500 /* 500 ms, if device busy */ +#define DEF_MAX_DISCARD_ISSUE_TIME 60000 /* 60 s, if no candidates */ +#define DEF_DISCARD_URGENT_UTIL 80 /* do more discard over 80% */ +#define DEF_CP_INTERVAL 60 /* 60 secs */ +#define DEF_IDLE_INTERVAL 5 /* 5 secs */ +#define DEF_DISABLE_INTERVAL 5 /* 5 secs */ +#define DEF_DISABLE_QUICK_INTERVAL 1 /* 1 secs */ +#define DEF_UMOUNT_DISCARD_TIMEOUT 5 /* 5 secs */ + +enum cp_time { + CP_TIME_START, /* begin */ + CP_TIME_LOCK, /* after cp_global_sem */ + CP_TIME_OP_LOCK, /* after block_operation */ + CP_TIME_MERGE_WRITE, /* after flush DATA/NODE/META */ + CP_TIME_FLUSH_NAT, /* after flush nat */ + CP_TIME_FLUSH_SIT, /* after flush sit */ + CP_TIME_SYNC_META, /* after sync_meta_pages */ + CP_TIME_SYNC_CP_META, /* after sync cp meta pages */ + CP_TIME_WAIT_DIRTY_META,/* after wait on dirty meta */ + CP_TIME_WAIT_CP_DATA, /* after wait on cp data */ + CP_TIME_FLUSH_DEVICE, /* after flush device cache */ + CP_TIME_WAIT_LAST_CP, /* after wait on last cp pack */ + CP_TIME_END, /* after unblock_operation */ + CP_TIME_MAX, +}; + +/* time cost stats of checkpoint */ +struct cp_stats { + ktime_t times[CP_TIME_MAX]; +}; + +struct cp_control { + int reason; + __u64 trim_start; + __u64 trim_end; + __u64 trim_minlen; + struct cp_stats stats; +}; + +enum f2fs_cp_phase { + CP_PHASE_START_BLOCK_OPS, + CP_PHASE_FINISH_BLOCK_OPS, + CP_PHASE_FINISH_CHECKPOINT, +}; + +/* + * indicate meta/data type + */ +enum { + META_CP, + META_NAT, + META_SIT, + META_SSA, + META_MAX, + META_POR, + DATA_GENERIC, /* check range only */ + DATA_GENERIC_ENHANCE, /* strong check on range and segment bitmap */ + DATA_GENERIC_ENHANCE_READ, /* + * strong check on range and segment + * bitmap but no warning due to race + * condition of read on truncated area + * by extent_cache + */ + DATA_GENERIC_ENHANCE_UPDATE, /* + * strong check on range and segment + * bitmap for update case + */ + META_GENERIC, +}; + +/* for the list of ino */ +enum { + ORPHAN_INO, /* for orphan ino list */ + APPEND_INO, /* for append ino list */ + UPDATE_INO, /* for update ino list */ + TRANS_DIR_INO, /* for transactions dir ino list */ + XATTR_DIR_INO, /* for xattr updated dir ino list */ + FLUSH_INO, /* for multiple device flushing */ + MAX_INO_ENTRY, /* max. list */ +}; + +struct ino_entry { + struct list_head list; /* list head */ + nid_t ino; /* inode number */ + unsigned int dirty_device; /* dirty device bitmap */ +}; + +/* for the list of inodes to be GCed */ +struct inode_entry { + struct list_head list; /* list head */ + struct inode *inode; /* vfs inode pointer */ +}; + +struct fsync_node_entry { + struct list_head list; /* list head */ + struct folio *folio; /* warm node folio pointer */ + unsigned int seq_id; /* sequence id */ +}; + +struct ckpt_req { + struct completion wait; /* completion for checkpoint done */ + struct llist_node llnode; /* llist_node to be linked in wait queue */ + int ret; /* return code of checkpoint */ + union { + ktime_t queue_time; /* request queued time */ + ktime_t delta_time; /* time in queue */ + }; +}; + +struct ckpt_req_control { + struct task_struct *f2fs_issue_ckpt; /* checkpoint task */ + int ckpt_thread_ioprio; /* checkpoint merge thread ioprio */ + wait_queue_head_t ckpt_wait_queue; /* waiting queue for wake-up */ + atomic_t issued_ckpt; /* # of actually issued ckpts */ + atomic_t total_ckpt; /* # of total ckpts */ + atomic_t queued_ckpt; /* # of queued ckpts */ + struct llist_head issue_list; /* list for command issue */ + spinlock_t stat_lock; /* lock for below checkpoint time stats */ + unsigned int cur_time; /* cur wait time in msec for currently issued checkpoint */ + unsigned int peak_time; /* peak wait time in msec until now */ +}; + +/* a time threshold that checkpoint was blocked for, unit: ms */ +#define CP_LONG_LATENCY_THRESHOLD 5000 + +/* for the bitmap indicate blocks to be discarded */ +struct discard_entry { + struct list_head list; /* list head */ + block_t start_blkaddr; /* start blockaddr of current segment */ + unsigned char discard_map[SIT_VBLOCK_MAP_SIZE]; /* segment discard bitmap */ +}; + +/* minimum discard granularity, unit: block count */ +#define MIN_DISCARD_GRANULARITY 1 +/* default discard granularity of inner discard thread, unit: block count */ +#define DEFAULT_DISCARD_GRANULARITY 16 +/* default maximum discard granularity of ordered discard, unit: block count */ +#define DEFAULT_MAX_ORDERED_DISCARD_GRANULARITY 16 +/* default interval of periodical discard submission */ +#define DEFAULT_DISCARD_INTERVAL (msecs_to_jiffies(20)) + +/* max discard pend list number */ +#define MAX_PLIST_NUM 512 +#define plist_idx(blk_num) ((blk_num) >= MAX_PLIST_NUM ? \ + (MAX_PLIST_NUM - 1) : ((blk_num) - 1)) + +enum { + D_PREP, /* initial */ + D_PARTIAL, /* partially submitted */ + D_SUBMIT, /* all submitted */ + D_DONE, /* finished */ +}; + +struct discard_info { + block_t lstart; /* logical start address */ + block_t len; /* length */ + block_t start; /* actual start address in dev */ +}; + +struct discard_cmd { + struct rb_node rb_node; /* rb node located in rb-tree */ + struct discard_info di; /* discard info */ + struct list_head list; /* command list */ + struct completion wait; /* completion */ + struct block_device *bdev; /* bdev */ + unsigned short ref; /* reference count */ + unsigned char state; /* state */ + unsigned char queued; /* queued discard */ + int error; /* bio error */ + spinlock_t lock; /* for state/bio_ref updating */ + unsigned short bio_ref; /* bio reference count */ +}; + +enum { + DPOLICY_BG, + DPOLICY_FORCE, + DPOLICY_FSTRIM, + DPOLICY_UMOUNT, + MAX_DPOLICY, +}; + +enum { + DPOLICY_IO_AWARE_DISABLE, /* force to not be aware of IO */ + DPOLICY_IO_AWARE_ENABLE, /* force to be aware of IO */ + DPOLICY_IO_AWARE_MAX, +}; + +struct discard_policy { + int type; /* type of discard */ + unsigned int min_interval; /* used for candidates exist */ + unsigned int mid_interval; /* used for device busy */ + unsigned int max_interval; /* used for candidates not exist */ + unsigned int max_requests; /* # of discards issued per round */ + unsigned int io_aware_gran; /* minimum granularity discard not be aware of I/O */ + bool io_aware; /* issue discard in idle time */ + bool sync; /* submit discard with REQ_SYNC flag */ + bool ordered; /* issue discard by lba order */ + bool timeout; /* discard timeout for put_super */ + unsigned int granularity; /* discard granularity */ +}; + +struct discard_cmd_control { + struct task_struct *f2fs_issue_discard; /* discard thread */ + struct list_head entry_list; /* 4KB discard entry list */ + struct list_head pend_list[MAX_PLIST_NUM];/* store pending entries */ + struct list_head wait_list; /* store on-flushing entries */ + struct list_head fstrim_list; /* in-flight discard from fstrim */ + wait_queue_head_t discard_wait_queue; /* waiting queue for wake-up */ + struct mutex cmd_lock; + unsigned int nr_discards; /* # of discards in the list */ + unsigned int max_discards; /* max. discards to be issued */ + unsigned int max_discard_request; /* max. discard request per round */ + unsigned int min_discard_issue_time; /* min. interval between discard issue */ + unsigned int mid_discard_issue_time; /* mid. interval between discard issue */ + unsigned int max_discard_issue_time; /* max. interval between discard issue */ + unsigned int discard_io_aware_gran; /* minimum discard granularity not be aware of I/O */ + unsigned int discard_urgent_util; /* utilization which issue discard proactively */ + unsigned int discard_granularity; /* discard granularity */ + unsigned int max_ordered_discard; /* maximum discard granularity issued by lba order */ + unsigned int discard_io_aware; /* io_aware policy */ + unsigned int undiscard_blks; /* # of undiscard blocks */ + unsigned int next_pos; /* next discard position */ + atomic_t issued_discard; /* # of issued discard */ + atomic_t queued_discard; /* # of queued discard */ + atomic_t discard_cmd_cnt; /* # of cached cmd count */ + struct rb_root_cached root; /* root of discard rb-tree */ + bool rbtree_check; /* config for consistence check */ + bool discard_wake; /* to wake up discard thread */ +}; + +/* for the list of fsync inodes, used only during recovery */ +struct fsync_inode_entry { + struct list_head list; /* list head */ + struct inode *inode; /* vfs inode pointer */ + block_t blkaddr; /* block address locating the last fsync */ + block_t last_dentry; /* block address locating the last dentry */ +}; + +#define nats_in_cursum(jnl) (le16_to_cpu((jnl)->n_nats)) +#define sits_in_cursum(jnl) (le16_to_cpu((jnl)->n_sits)) + +#define nat_in_journal(jnl, i) \ + (((struct nat_journal_entry *)(jnl)->nat_j.entries)[i].ne) +#define nid_in_journal(jnl, i) \ + (((struct nat_journal_entry *)(jnl)->nat_j.entries)[i].nid) +#define sit_in_journal(jnl, i) \ + (((struct sit_journal_entry *)(jnl)->sit_j.entries)[i].se) +#define segno_in_journal(jnl, i) \ + (((struct sit_journal_entry *)(jnl)->sit_j.entries)[i].segno) + +#define sum_entries(sum) ((struct f2fs_summary *)(sum)) +#define sum_journal(sbi, sum) \ + ((struct f2fs_journal *)((char *)(sum) + \ + ((sbi)->entries_in_sum * sizeof(struct f2fs_summary)))) +#define sum_footer(sbi, sum) \ + ((struct summary_footer *)((char *)(sum) + (sbi)->sum_blocksize - \ + sizeof(struct summary_footer))) + +#define MAX_NAT_JENTRIES(sbi, jnl) ((sbi)->nat_journal_entries - nats_in_cursum(jnl)) +#define MAX_SIT_JENTRIES(sbi, jnl) ((sbi)->sit_journal_entries - sits_in_cursum(jnl)) + +static inline int update_nats_in_cursum(struct f2fs_journal *journal, int i) +{ + int before = nats_in_cursum(journal); + + journal->n_nats = cpu_to_le16(before + i); + return before; +} + +static inline int update_sits_in_cursum(struct f2fs_journal *journal, int i) +{ + int before = sits_in_cursum(journal); + + journal->n_sits = cpu_to_le16(before + i); + return before; +} + +/* for inline stuff */ +#define DEF_INLINE_RESERVED_SIZE 1 +static inline int get_extra_isize(struct inode *inode); +static inline int get_inline_xattr_addrs(struct inode *inode); +#define MAX_INLINE_DATA(inode) (sizeof(__le32) * \ + (CUR_ADDRS_PER_INODE(inode) - \ + get_inline_xattr_addrs(inode) - \ + DEF_INLINE_RESERVED_SIZE)) + +/* for inline dir */ +#define NR_INLINE_DENTRY(inode) (MAX_INLINE_DATA(inode) * BITS_PER_BYTE / \ + ((SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * \ + BITS_PER_BYTE + 1)) +#define INLINE_DENTRY_BITMAP_SIZE(inode) \ + DIV_ROUND_UP(NR_INLINE_DENTRY(inode), BITS_PER_BYTE) +#define INLINE_RESERVED_SIZE(inode) (MAX_INLINE_DATA(inode) - \ + ((SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * \ + NR_INLINE_DENTRY(inode) + \ + INLINE_DENTRY_BITMAP_SIZE(inode))) + +/* + * For INODE and NODE manager + */ +/* for directory operations */ + +struct f2fs_filename { + /* + * The filename the user specified. This is NULL for some + * filesystem-internal operations, e.g. converting an inline directory + * to a non-inline one, or roll-forward recovering an encrypted dentry. + */ + const struct qstr *usr_fname; + + /* + * The on-disk filename. For encrypted directories, this is encrypted. + * This may be NULL for lookups in an encrypted dir without the key. + */ + struct fscrypt_str disk_name; + + /* The dirhash of this filename */ + f2fs_hash_t hash; + +#ifdef CONFIG_FS_ENCRYPTION + /* + * For lookups in encrypted directories: either the buffer backing + * disk_name, or a buffer that holds the decoded no-key name. + */ + struct fscrypt_str crypto_buf; +#endif +#if IS_ENABLED(CONFIG_UNICODE) + /* + * For casefolded directories: the casefolded name, but it's left NULL + * if the original name is not valid Unicode, if the original name is + * "." or "..", if the directory is both casefolded and encrypted and + * its encryption key is unavailable, or if the filesystem is doing an + * internal operation where usr_fname is also NULL. In all these cases + * we fall back to treating the name as an opaque byte sequence. + */ + struct qstr cf_name; +#endif +}; + +struct f2fs_dentry_ptr { + struct inode *inode; + void *bitmap; + struct f2fs_dir_entry *dentry; + __u8 (*filename)[F2FS_SLOT_LEN]; + int max; + int nr_bitmap; +}; + +static inline void make_dentry_ptr_block(struct inode *inode, + struct f2fs_dentry_ptr *d, struct f2fs_dentry_block *t) +{ + d->inode = inode; + d->max = NR_DENTRY_IN_BLOCK; + d->nr_bitmap = SIZE_OF_DENTRY_BITMAP; + d->bitmap = t->dentry_bitmap; + d->dentry = t->dentry; + d->filename = t->filename; +} + +static inline void make_dentry_ptr_inline(struct inode *inode, + struct f2fs_dentry_ptr *d, void *t) +{ + int entry_cnt = NR_INLINE_DENTRY(inode); + int bitmap_size = INLINE_DENTRY_BITMAP_SIZE(inode); + int reserved_size = INLINE_RESERVED_SIZE(inode); + + d->inode = inode; + d->max = entry_cnt; + d->nr_bitmap = bitmap_size; + d->bitmap = t; + d->dentry = t + bitmap_size + reserved_size; + d->filename = t + bitmap_size + reserved_size + + SIZE_OF_DIR_ENTRY * entry_cnt; +} + +/* + * XATTR_NODE_OFFSET stores xattrs to one node block per file keeping -1 + * as its node offset to distinguish from index node blocks. + * But some bits are used to mark the node block. + */ +#define XATTR_NODE_OFFSET ((((unsigned int)-1) << OFFSET_BIT_SHIFT) \ + >> OFFSET_BIT_SHIFT) +enum { + ALLOC_NODE, /* allocate a new node page if needed */ + LOOKUP_NODE, /* look up a node without readahead */ + LOOKUP_NODE_RA, /* + * look up a node with readahead called + * by get_data_block. + */ +}; + +#define DEFAULT_RETRY_IO_COUNT 8 /* maximum retry read IO or flush count */ + +#define MAX_FLUSH_RETRY_COUNT 3 /* maximum flush retry count in f2fs_enable_checkpoint() */ + +/* IO/non-IO congestion wait timeout value, default: 1 jiffies */ +#define DEFAULT_SCHEDULE_TIMEOUT 1 + +/* timeout value injected, default: 1000ms */ +#define DEFAULT_FAULT_TIMEOUT (msecs_to_jiffies(1000)) + +/* maximum retry quota flush count */ +#define DEFAULT_RETRY_QUOTA_FLUSH_COUNT 8 + +/* maximum retry of EIO'ed page */ +#define MAX_RETRY_PAGE_EIO 100 + +#define F2FS_LINK_MAX 0xffffffff /* maximum link count per file */ + +#define MAX_DIR_RA_PAGES 4 /* maximum ra pages of dir */ + +/* dirty segments threshold for triggering CP */ +#define DEFAULT_DIRTY_THRESHOLD 4 + +#define RECOVERY_MAX_RA_BLOCKS BIO_MAX_VECS +#define RECOVERY_MIN_RA_BLOCKS 1 + +#define F2FS_ONSTACK_PAGES 16 /* nr of onstack pages */ + +/* for in-memory extent cache entry */ +#define F2FS_MIN_EXTENT_LEN 64 /* minimum extent length */ + +/* number of extent info in extent cache we try to shrink */ +#define READ_EXTENT_CACHE_SHRINK_NUMBER 128 + +/* number of age extent info in extent cache we try to shrink */ +#define AGE_EXTENT_CACHE_SHRINK_NUMBER 128 +#define LAST_AGE_WEIGHT 30 +#define SAME_AGE_REGION 1024 + +/* + * Define data block with age less than 1GB as hot data + * define data block with age less than 10GB but more than 1GB as warm data + */ +#define DEF_HOT_DATA_AGE_THRESHOLD 262144 +#define DEF_WARM_DATA_AGE_THRESHOLD 2621440 + +/* default max read extent count per inode */ +#define DEF_MAX_READ_EXTENT_COUNT 10240 + +/* extent cache type */ +enum extent_type { + EX_READ, + EX_BLOCK_AGE, + NR_EXTENT_CACHES, +}; + +/* + * Reserved value to mark invalid age extents, hence valid block range + * from 0 to ULLONG_MAX-1 + */ +#define F2FS_EXTENT_AGE_INVALID ULLONG_MAX + +struct extent_info { + unsigned int fofs; /* start offset in a file */ + unsigned int len; /* length of the extent */ + union { + /* read extent_cache */ + struct { + /* start block address of the extent */ + block_t blk; +#ifdef CONFIG_F2FS_FS_COMPRESSION + /* physical extent length of compressed blocks */ + unsigned int c_len; +#endif + }; + /* block age extent_cache */ + struct { + /* block age of the extent */ + unsigned long long age; + /* last total blocks allocated */ + unsigned long long last_blocks; + }; + }; +}; + +struct extent_node { + struct rb_node rb_node; /* rb node located in rb-tree */ + struct extent_info ei; /* extent info */ + struct list_head list; /* node in global extent list of sbi */ + struct extent_tree *et; /* extent tree pointer */ +}; + +struct extent_tree { + nid_t ino; /* inode number */ + enum extent_type type; /* keep the extent tree type */ + struct rb_root_cached root; /* root of extent info rb-tree */ + struct extent_node *cached_en; /* recently accessed extent node */ + struct list_head list; /* to be used by sbi->zombie_list */ + rwlock_t lock; /* protect extent info rb-tree */ + atomic_t node_cnt; /* # of extent node in rb-tree*/ + bool largest_updated; /* largest extent updated */ + struct extent_info largest; /* largest cached extent for EX_READ */ +}; + +struct extent_tree_info { + struct radix_tree_root extent_tree_root;/* cache extent cache entries */ + struct mutex extent_tree_lock; /* locking extent radix tree */ + struct list_head extent_list; /* lru list for shrinker */ + spinlock_t extent_lock; /* locking extent lru list */ + atomic_t total_ext_tree; /* extent tree count */ + struct list_head zombie_list; /* extent zombie tree list */ + atomic_t total_zombie_tree; /* extent zombie tree count */ + atomic_t total_ext_node; /* extent info count */ +}; + +/* + * State of block returned by f2fs_map_blocks. + */ +#define F2FS_MAP_NEW (1U << 0) +#define F2FS_MAP_MAPPED (1U << 1) +#define F2FS_MAP_DELALLOC (1U << 2) +#define F2FS_MAP_FLAGS (F2FS_MAP_NEW | F2FS_MAP_MAPPED |\ + F2FS_MAP_DELALLOC) + +struct f2fs_map_blocks { + struct block_device *m_bdev; /* for multi-device dio */ + block_t m_pblk; + block_t m_lblk; + unsigned int m_len; + unsigned int m_flags; + unsigned long m_last_pblk; /* last allocated block, only used for DIO in LFS mode */ + pgoff_t *m_next_pgofs; /* point next possible non-hole pgofs */ + pgoff_t *m_next_extent; /* point to next possible extent */ + int m_seg_type; + bool m_may_create; /* indicate it is from write path */ + bool m_multidev_dio; /* indicate it allows multi-device dio */ +}; + +/* for flag in get_data_block */ +enum { + F2FS_GET_BLOCK_DEFAULT, + F2FS_GET_BLOCK_FIEMAP, + F2FS_GET_BLOCK_BMAP, + F2FS_GET_BLOCK_DIO, + F2FS_GET_BLOCK_PRE_DIO, + F2FS_GET_BLOCK_PRE_AIO, + F2FS_GET_BLOCK_PRECACHE, +}; + +/* + * i_advise uses FADVISE_XXX_BIT. We can add additional hints later. + */ +#define FADVISE_COLD_BIT 0x01 +#define FADVISE_LOST_PINO_BIT 0x02 +#define FADVISE_ENCRYPT_BIT 0x04 +#define FADVISE_ENC_NAME_BIT 0x08 +#define FADVISE_KEEP_SIZE_BIT 0x10 +#define FADVISE_HOT_BIT 0x20 +#define FADVISE_VERITY_BIT 0x40 +#define FADVISE_TRUNC_BIT 0x80 + +#define FADVISE_MODIFIABLE_BITS (FADVISE_COLD_BIT | FADVISE_HOT_BIT) + +#define file_is_cold(inode) is_file(inode, FADVISE_COLD_BIT) +#define file_set_cold(inode) set_file(inode, FADVISE_COLD_BIT) +#define file_clear_cold(inode) clear_file(inode, FADVISE_COLD_BIT) + +#define file_wrong_pino(inode) is_file(inode, FADVISE_LOST_PINO_BIT) +#define file_lost_pino(inode) set_file(inode, FADVISE_LOST_PINO_BIT) +#define file_got_pino(inode) clear_file(inode, FADVISE_LOST_PINO_BIT) + +#define file_is_encrypt(inode) is_file(inode, FADVISE_ENCRYPT_BIT) +#define file_set_encrypt(inode) set_file(inode, FADVISE_ENCRYPT_BIT) + +#define file_enc_name(inode) is_file(inode, FADVISE_ENC_NAME_BIT) +#define file_set_enc_name(inode) set_file(inode, FADVISE_ENC_NAME_BIT) + +#define file_keep_isize(inode) is_file(inode, FADVISE_KEEP_SIZE_BIT) +#define file_set_keep_isize(inode) set_file(inode, FADVISE_KEEP_SIZE_BIT) + +#define file_is_hot(inode) is_file(inode, FADVISE_HOT_BIT) +#define file_set_hot(inode) set_file(inode, FADVISE_HOT_BIT) +#define file_clear_hot(inode) clear_file(inode, FADVISE_HOT_BIT) + +#define file_is_verity(inode) is_file(inode, FADVISE_VERITY_BIT) +#define file_set_verity(inode) set_file(inode, FADVISE_VERITY_BIT) + +#define file_should_truncate(inode) is_file(inode, FADVISE_TRUNC_BIT) +#define file_need_truncate(inode) set_file(inode, FADVISE_TRUNC_BIT) +#define file_dont_truncate(inode) clear_file(inode, FADVISE_TRUNC_BIT) + +#define DEF_DIR_LEVEL 0 + +/* used for f2fs_inode_info->flags */ +enum { + FI_NEW_INODE, /* indicate newly allocated inode */ + FI_DIRTY_INODE, /* indicate inode is dirty or not */ + FI_AUTO_RECOVER, /* indicate inode is recoverable */ + FI_DIRTY_DIR, /* indicate directory has dirty pages */ + FI_INC_LINK, /* need to increment i_nlink */ + FI_ACL_MODE, /* indicate acl mode */ + FI_NO_ALLOC, /* should not allocate any blocks */ + FI_FREE_NID, /* free allocated nide */ + FI_NO_EXTENT, /* not to use the extent cache */ + FI_INLINE_XATTR, /* used for inline xattr */ + FI_INLINE_DATA, /* used for inline data*/ + FI_INLINE_DENTRY, /* used for inline dentry */ + FI_APPEND_WRITE, /* inode has appended data */ + FI_UPDATE_WRITE, /* inode has in-place-update data */ + FI_NEED_IPU, /* used for ipu per file */ + FI_ATOMIC_FILE, /* indicate atomic file */ + FI_DATA_EXIST, /* indicate data exists */ + FI_SKIP_WRITES, /* should skip data page writeback */ + FI_OPU_WRITE, /* used for opu per file */ + FI_DIRTY_FILE, /* indicate regular/symlink has dirty pages */ + FI_PREALLOCATED_ALL, /* all blocks for write were preallocated */ + FI_HOT_DATA, /* indicate file is hot */ + FI_EXTRA_ATTR, /* indicate file has extra attribute */ + FI_PROJ_INHERIT, /* indicate file inherits projectid */ + FI_PIN_FILE, /* indicate file should not be gced */ + FI_VERITY_IN_PROGRESS, /* building fs-verity Merkle tree */ + FI_COMPRESSED_FILE, /* indicate file's data can be compressed */ + FI_COMPRESS_CORRUPT, /* indicate compressed cluster is corrupted */ + FI_MMAP_FILE, /* indicate file was mmapped */ + FI_ENABLE_COMPRESS, /* enable compression in "user" compression mode */ + FI_COMPRESS_RELEASED, /* compressed blocks were released */ + FI_ALIGNED_WRITE, /* enable aligned write */ + FI_COW_FILE, /* indicate COW file */ + FI_ATOMIC_COMMITTED, /* indicate atomic commit completed except disk sync */ + FI_ATOMIC_DIRTIED, /* indicate atomic file is dirtied */ + FI_ATOMIC_REPLACE, /* indicate atomic replace */ + FI_OPENED_FILE, /* indicate file has been opened */ + FI_DONATE_FINISHED, /* indicate page donation of file has been finished */ + FI_MAX, /* max flag, never be used */ +}; + +struct f2fs_inode_info { + struct inode vfs_inode; /* serve a vfs inode */ + unsigned long i_flags; /* keep an inode flags for ioctl */ + unsigned char i_advise; /* use to give file attribute hints */ + unsigned char i_dir_level; /* use for dentry level for large dir */ + union { + unsigned int i_current_depth; /* only for directory depth */ + unsigned short i_gc_failures; /* for gc failure statistic */ + }; + unsigned int i_pino; /* parent inode number */ + umode_t i_acl_mode; /* keep file acl mode temporarily */ + + /* Use below internally in f2fs*/ + unsigned long flags[BITS_TO_LONGS(FI_MAX)]; /* use to pass per-file flags */ + unsigned int ioprio_hint; /* hint for IO priority */ + struct f2fs_rwsem i_sem; /* protect fi info */ + atomic_t dirty_pages; /* # of dirty pages */ + f2fs_hash_t chash; /* hash value of given file name */ + unsigned int clevel; /* maximum level of given file name */ + struct task_struct *task; /* lookup and create consistency */ + struct task_struct *cp_task; /* separate cp/wb IO stats*/ + struct task_struct *wb_task; /* indicate inode is in context of writeback */ + nid_t i_xattr_nid; /* node id that contains xattrs */ + loff_t last_disk_size; /* lastly written file size */ + spinlock_t i_size_lock; /* protect last_disk_size */ + +#ifdef CONFIG_QUOTA + struct dquot __rcu *i_dquot[MAXQUOTAS]; + + /* quota space reservation, managed internally by quota code */ + qsize_t i_reserved_quota; +#endif + struct list_head dirty_list; /* dirty list for dirs and files */ + struct list_head gdirty_list; /* linked in global dirty list */ + + /* linked in global inode list for cache donation */ + struct list_head gdonate_list; + pgoff_t donate_start, donate_end; /* inclusive */ + atomic_t open_count; /* # of open files */ + + struct task_struct *atomic_write_task; /* store atomic write task */ + struct extent_tree *extent_tree[NR_EXTENT_CACHES]; + /* cached extent_tree entry */ + union { + struct inode *cow_inode; /* copy-on-write inode for atomic write */ + struct inode *atomic_inode; + /* point to atomic_inode, available only for cow_inode */ + }; + + /* avoid racing between foreground op and gc */ + struct f2fs_rwsem i_gc_rwsem[2]; + struct f2fs_rwsem i_xattr_sem; /* avoid racing between reading and changing EAs */ + + int i_extra_isize; /* size of extra space located in i_addr */ + kprojid_t i_projid; /* id for project quota */ + int i_inline_xattr_size; /* inline xattr size */ + struct timespec64 i_crtime; /* inode creation time */ + struct timespec64 i_disk_time[3];/* inode disk times */ + + /* for file compress */ + atomic_t i_compr_blocks; /* # of compressed blocks */ + unsigned char i_compress_algorithm; /* algorithm type */ + unsigned char i_log_cluster_size; /* log of cluster size */ + unsigned char i_compress_level; /* compress level (lz4hc,zstd) */ + unsigned char i_compress_flag; /* compress flag */ + unsigned int i_cluster_size; /* cluster size */ + atomic_t writeback; /* count # of writeback thread */ + + unsigned int atomic_write_cnt; + loff_t original_i_size; /* original i_size before atomic write */ +#ifdef CONFIG_FS_ENCRYPTION + struct fscrypt_inode_info *i_crypt_info; /* filesystem encryption info */ +#endif +}; + +static inline void get_read_extent_info(struct extent_info *ext, + struct f2fs_extent *i_ext) +{ + ext->fofs = le32_to_cpu(i_ext->fofs); + ext->blk = le32_to_cpu(i_ext->blk); + ext->len = le32_to_cpu(i_ext->len); +} + +static inline void set_raw_read_extent(struct extent_info *ext, + struct f2fs_extent *i_ext) +{ + i_ext->fofs = cpu_to_le32(ext->fofs); + i_ext->blk = cpu_to_le32(ext->blk); + i_ext->len = cpu_to_le32(ext->len); +} + +static inline bool __is_discard_mergeable(struct discard_info *back, + struct discard_info *front, unsigned int max_len) +{ + return (back->lstart + back->len == front->lstart) && + (back->len + front->len <= max_len); +} + +static inline bool __is_discard_back_mergeable(struct discard_info *cur, + struct discard_info *back, unsigned int max_len) +{ + return __is_discard_mergeable(back, cur, max_len); +} + +static inline bool __is_discard_front_mergeable(struct discard_info *cur, + struct discard_info *front, unsigned int max_len) +{ + return __is_discard_mergeable(cur, front, max_len); +} + +/* + * For free nid management + */ +enum nid_state { + FREE_NID, /* newly added to free nid list */ + PREALLOC_NID, /* it is preallocated */ + MAX_NID_STATE, +}; + +enum nat_state { + TOTAL_NAT, + DIRTY_NAT, + RECLAIMABLE_NAT, + MAX_NAT_STATE, +}; + +struct f2fs_nm_info { + block_t nat_blkaddr; /* base disk address of NAT */ + nid_t max_nid; /* maximum possible node ids */ + nid_t available_nids; /* # of available node ids */ + nid_t next_scan_nid; /* the next nid to be scanned */ + nid_t max_rf_node_blocks; /* max # of nodes for recovery */ + unsigned int ram_thresh; /* control the memory footprint */ + unsigned int ra_nid_pages; /* # of nid pages to be readaheaded */ + unsigned int dirty_nats_ratio; /* control dirty nats ratio threshold */ + + /* NAT cache management */ + struct radix_tree_root nat_root;/* root of the nat entry cache */ + struct radix_tree_root nat_set_root;/* root of the nat set cache */ + struct f2fs_rwsem nat_tree_lock; /* protect nat entry tree */ + struct list_head nat_entries; /* cached nat entry list (clean) */ + spinlock_t nat_list_lock; /* protect clean nat entry list */ + unsigned int nat_cnt[MAX_NAT_STATE]; /* the # of cached nat entries */ + unsigned int nat_blocks; /* # of nat blocks */ + + /* free node ids management */ + struct radix_tree_root free_nid_root;/* root of the free_nid cache */ + struct list_head free_nid_list; /* list for free nids excluding preallocated nids */ + unsigned int nid_cnt[MAX_NID_STATE]; /* the number of free node id */ + spinlock_t nid_list_lock; /* protect nid lists ops */ + struct mutex build_lock; /* lock for build free nids */ + unsigned char **free_nid_bitmap; + unsigned char *nat_block_bitmap; + unsigned short *free_nid_count; /* free nid count of NAT block */ + + /* for checkpoint */ + char *nat_bitmap; /* NAT bitmap pointer */ + + unsigned int nat_bits_blocks; /* # of nat bits blocks */ + unsigned char *nat_bits; /* NAT bits blocks */ + unsigned char *full_nat_bits; /* full NAT pages */ + unsigned char *empty_nat_bits; /* empty NAT pages */ +#ifdef CONFIG_F2FS_CHECK_FS + char *nat_bitmap_mir; /* NAT bitmap mirror */ +#endif + int bitmap_size; /* bitmap size */ +}; + +/* + * this structure is used as one of function parameters. + * all the information are dedicated to a given direct node block determined + * by the data offset in a file. + */ +struct dnode_of_data { + struct inode *inode; /* vfs inode pointer */ + struct folio *inode_folio; /* its inode folio, NULL is possible */ + struct folio *node_folio; /* cached direct node folio */ + nid_t nid; /* node id of the direct node block */ + unsigned int ofs_in_node; /* data offset in the node page */ + bool inode_folio_locked; /* inode folio is locked or not */ + bool node_changed; /* is node block changed */ + char cur_level; /* level of hole node page */ + char max_level; /* level of current page located */ + block_t data_blkaddr; /* block address of the node block */ +}; + +static inline void set_new_dnode(struct dnode_of_data *dn, struct inode *inode, + struct folio *ifolio, struct folio *nfolio, nid_t nid) +{ + memset(dn, 0, sizeof(*dn)); + dn->inode = inode; + dn->inode_folio = ifolio; + dn->node_folio = nfolio; + dn->nid = nid; +} + +/* + * For SIT manager + * + * By default, there are 6 active log areas across the whole main area. + * When considering hot and cold data separation to reduce cleaning overhead, + * we split 3 for data logs and 3 for node logs as hot, warm, and cold types, + * respectively. + * In the current design, you should not change the numbers intentionally. + * Instead, as a mount option such as active_logs=x, you can use 2, 4, and 6 + * logs individually according to the underlying devices. (default: 6) + * Just in case, on-disk layout covers maximum 16 logs that consist of 8 for + * data and 8 for node logs. + */ +#define NR_CURSEG_DATA_TYPE (3) +#define NR_CURSEG_NODE_TYPE (3) +#define NR_CURSEG_INMEM_TYPE (2) +#define NR_CURSEG_RO_TYPE (2) +#define NR_CURSEG_PERSIST_TYPE (NR_CURSEG_DATA_TYPE + NR_CURSEG_NODE_TYPE) +#define NR_CURSEG_TYPE (NR_CURSEG_INMEM_TYPE + NR_CURSEG_PERSIST_TYPE) + +enum log_type { + CURSEG_HOT_DATA = 0, /* directory entry blocks */ + CURSEG_WARM_DATA, /* data blocks */ + CURSEG_COLD_DATA, /* multimedia or GCed data blocks */ + CURSEG_HOT_NODE, /* direct node blocks of directory files */ + CURSEG_WARM_NODE, /* direct node blocks of normal files */ + CURSEG_COLD_NODE, /* indirect node blocks */ + NR_PERSISTENT_LOG, /* number of persistent log */ + CURSEG_COLD_DATA_PINNED = NR_PERSISTENT_LOG, + /* pinned file that needs consecutive block address */ + CURSEG_ALL_DATA_ATGC, /* SSR alloctor in hot/warm/cold data area */ + NO_CHECK_TYPE, /* number of persistent & inmem log */ +}; + +struct flush_cmd { + struct completion wait; + struct llist_node llnode; + nid_t ino; + int ret; +}; + +struct flush_cmd_control { + struct task_struct *f2fs_issue_flush; /* flush thread */ + wait_queue_head_t flush_wait_queue; /* waiting queue for wake-up */ + atomic_t issued_flush; /* # of issued flushes */ + atomic_t queued_flush; /* # of queued flushes */ + struct llist_head issue_list; /* list for command issue */ + struct llist_node *dispatch_list; /* list for command dispatch */ +}; + +struct f2fs_sm_info { + struct sit_info *sit_info; /* whole segment information */ + struct free_segmap_info *free_info; /* free segment information */ + struct dirty_seglist_info *dirty_info; /* dirty segment information */ + struct curseg_info *curseg_array; /* active segment information */ + + struct f2fs_rwsem curseg_lock; /* for preventing curseg change */ + + block_t seg0_blkaddr; /* block address of 0'th segment */ + block_t main_blkaddr; /* start block address of main area */ + block_t ssa_blkaddr; /* start block address of SSA area */ + + unsigned int segment_count; /* total # of segments */ + unsigned int main_segments; /* # of segments in main area */ + unsigned int reserved_segments; /* # of reserved segments */ + unsigned int ovp_segments; /* # of overprovision segments */ + + /* a threshold to reclaim prefree segments */ + unsigned int rec_prefree_segments; + + struct list_head sit_entry_set; /* sit entry set list */ + + unsigned int ipu_policy; /* in-place-update policy */ + unsigned int min_ipu_util; /* in-place-update threshold */ + unsigned int min_fsync_blocks; /* threshold for fsync */ + unsigned int min_seq_blocks; /* threshold for sequential blocks */ + unsigned int min_hot_blocks; /* threshold for hot block allocation */ + unsigned int min_ssr_sections; /* threshold to trigger SSR allocation */ + + /* for flush command control */ + struct flush_cmd_control *fcc_info; + + /* for discard command control */ + struct discard_cmd_control *dcc_info; +}; + +/* + * For superblock + */ +/* + * COUNT_TYPE for monitoring + * + * f2fs monitors the number of several block types such as on-writeback, + * dirty dentry blocks, dirty node blocks, and dirty meta blocks. + */ +#define WB_DATA_TYPE(folio, f) \ + (f || f2fs_is_cp_guaranteed(folio) ? F2FS_WB_CP_DATA : F2FS_WB_DATA) +enum count_type { + F2FS_DIRTY_DENTS, + F2FS_DIRTY_DATA, + F2FS_DIRTY_QDATA, + F2FS_DIRTY_NODES, + F2FS_DIRTY_META, + F2FS_DIRTY_IMETA, + F2FS_WB_CP_DATA, + F2FS_WB_DATA, + F2FS_RD_DATA, + F2FS_RD_NODE, + F2FS_RD_META, + F2FS_DIO_WRITE, + F2FS_DIO_READ, + F2FS_SKIPPED_WRITE, /* skip or fail during f2fs_enable_checkpoint() */ + NR_COUNT_TYPE, +}; + +/* + * The below are the page types of bios used in submit_bio(). + * The available types are: + * DATA User data pages. It operates as async mode. + * NODE Node pages. It operates as async mode. + * META FS metadata pages such as SIT, NAT, CP. + * NR_PAGE_TYPE The number of page types. + * META_FLUSH Make sure the previous pages are written + * with waiting the bio's completion + * ... Only can be used with META. + */ +#define PAGE_TYPE_OF_BIO(type) ((type) > META ? META : (type)) +#define PAGE_TYPE_ON_MAIN(type) ((type) == DATA || (type) == NODE) +enum page_type { + DATA = 0, + NODE = 1, /* should not change this */ + META, + NR_PAGE_TYPE, + META_FLUSH, + IPU, /* the below types are used by tracepoints only. */ + OPU, +}; + +enum temp_type { + HOT = 0, /* must be zero for meta bio */ + WARM, + COLD, + NR_TEMP_TYPE, +}; + +enum need_lock_type { + LOCK_REQ = 0, + LOCK_DONE, + LOCK_RETRY, +}; + +enum cp_reason_type { + CP_NO_NEEDED, + CP_NON_REGULAR, + CP_COMPRESSED, + CP_HARDLINK, + CP_SB_NEED_CP, + CP_WRONG_PINO, + CP_NO_SPC_ROLL, + CP_NODE_NEED_CP, + CP_FASTBOOT_MODE, + CP_SPEC_LOG_NUM, + CP_RECOVER_DIR, + CP_XATTR_DIR, +}; + +enum iostat_type { + /* WRITE IO */ + APP_DIRECT_IO, /* app direct write IOs */ + APP_BUFFERED_IO, /* app buffered write IOs */ + APP_WRITE_IO, /* app write IOs */ + APP_MAPPED_IO, /* app mapped IOs */ + APP_BUFFERED_CDATA_IO, /* app buffered write IOs on compressed file */ + APP_MAPPED_CDATA_IO, /* app mapped write IOs on compressed file */ + FS_DATA_IO, /* data IOs from kworker/fsync/reclaimer */ + FS_CDATA_IO, /* data IOs from kworker/fsync/reclaimer on compressed file */ + FS_NODE_IO, /* node IOs from kworker/fsync/reclaimer */ + FS_META_IO, /* meta IOs from kworker/reclaimer */ + FS_GC_DATA_IO, /* data IOs from forground gc */ + FS_GC_NODE_IO, /* node IOs from forground gc */ + FS_CP_DATA_IO, /* data IOs from checkpoint */ + FS_CP_NODE_IO, /* node IOs from checkpoint */ + FS_CP_META_IO, /* meta IOs from checkpoint */ + + /* READ IO */ + APP_DIRECT_READ_IO, /* app direct read IOs */ + APP_BUFFERED_READ_IO, /* app buffered read IOs */ + APP_READ_IO, /* app read IOs */ + APP_MAPPED_READ_IO, /* app mapped read IOs */ + APP_BUFFERED_CDATA_READ_IO, /* app buffered read IOs on compressed file */ + APP_MAPPED_CDATA_READ_IO, /* app mapped read IOs on compressed file */ + FS_DATA_READ_IO, /* data read IOs */ + FS_GDATA_READ_IO, /* data read IOs from background gc */ + FS_CDATA_READ_IO, /* compressed data read IOs */ + FS_NODE_READ_IO, /* node read IOs */ + FS_META_READ_IO, /* meta read IOs */ + + /* other */ + FS_DISCARD_IO, /* discard */ + FS_FLUSH_IO, /* flush */ + FS_ZONE_RESET_IO, /* zone reset */ + NR_IO_TYPE, +}; + +struct f2fs_io_info { + struct f2fs_sb_info *sbi; /* f2fs_sb_info pointer */ + nid_t ino; /* inode number */ + enum page_type type; /* contains DATA/NODE/META/META_FLUSH */ + enum temp_type temp; /* contains HOT/WARM/COLD */ + enum req_op op; /* contains REQ_OP_ */ + blk_opf_t op_flags; /* req_flag_bits */ + block_t new_blkaddr; /* new block address to be written */ + block_t old_blkaddr; /* old block address before Cow */ + union { + struct page *page; /* page to be written */ + struct folio *folio; + }; + struct page *encrypted_page; /* encrypted page */ + struct page *compressed_page; /* compressed page */ + struct list_head list; /* serialize IOs */ + unsigned int compr_blocks; /* # of compressed block addresses */ + unsigned int need_lock:8; /* indicate we need to lock cp_rwsem */ + unsigned int version:8; /* version of the node */ + unsigned int submitted:1; /* indicate IO submission */ + unsigned int in_list:1; /* indicate fio is in io_list */ + unsigned int is_por:1; /* indicate IO is from recovery or not */ + unsigned int meta_gc:1; /* require meta inode GC */ + enum iostat_type io_type; /* io type */ + struct writeback_control *io_wbc; /* writeback control */ + struct bio **bio; /* bio for ipu */ + sector_t *last_block; /* last block number in bio */ +}; + +struct bio_entry { + struct bio *bio; + struct list_head list; +}; + +#define is_read_io(rw) ((rw) == READ) +struct f2fs_bio_info { + struct f2fs_sb_info *sbi; /* f2fs superblock */ + struct bio *bio; /* bios to merge */ + sector_t last_block_in_bio; /* last block number */ + struct f2fs_io_info fio; /* store buffered io info. */ +#ifdef CONFIG_BLK_DEV_ZONED + struct completion zone_wait; /* condition value for the previous open zone to close */ + struct bio *zone_pending_bio; /* pending bio for the previous zone */ + void *bi_private; /* previous bi_private for pending bio */ +#endif + struct f2fs_rwsem io_rwsem; /* blocking op for bio */ + spinlock_t io_lock; /* serialize DATA/NODE IOs */ + struct list_head io_list; /* track fios */ + struct list_head bio_list; /* bio entry list head */ + struct f2fs_rwsem bio_list_lock; /* lock to protect bio entry list */ +}; + +#define FDEV(i) (sbi->devs[i]) +#define RDEV(i) (raw_super->devs[i]) +struct f2fs_dev_info { + struct file *bdev_file; + struct block_device *bdev; + char path[MAX_PATH_LEN + 1]; + unsigned int total_segments; + block_t start_blk; + block_t end_blk; +#ifdef CONFIG_BLK_DEV_ZONED + unsigned int nr_blkz; /* Total number of zones */ + unsigned long *blkz_seq; /* Bitmap indicating sequential zones */ +#endif +}; + +enum inode_type { + DIR_INODE, /* for dirty dir inode */ + FILE_INODE, /* for dirty regular/symlink inode */ + DIRTY_META, /* for all dirtied inode metadata */ + DONATE_INODE, /* for all inode to donate pages */ + NR_INODE_TYPE, +}; + +/* for inner inode cache management */ +struct inode_management { + struct radix_tree_root ino_root; /* ino entry array */ + spinlock_t ino_lock; /* for ino entry lock */ + struct list_head ino_list; /* inode list head */ + unsigned long ino_num; /* number of entries */ +}; + +/* for GC_AT */ +struct atgc_management { + bool atgc_enabled; /* ATGC is enabled or not */ + struct rb_root_cached root; /* root of victim rb-tree */ + struct list_head victim_list; /* linked with all victim entries */ + unsigned int victim_count; /* victim count in rb-tree */ + unsigned int candidate_ratio; /* candidate ratio */ + unsigned int max_candidate_count; /* max candidate count */ + unsigned int age_weight; /* age weight, vblock_weight = 100 - age_weight */ + unsigned long long age_threshold; /* age threshold */ +}; + +struct f2fs_time_stat { + unsigned long long total_time; /* total wall clock time */ +#ifdef CONFIG_64BIT + unsigned long long running_time; /* running time */ +#endif +#if defined(CONFIG_SCHED_INFO) && defined(CONFIG_SCHEDSTATS) + unsigned long long runnable_time; /* runnable(including preempted) time */ +#endif +#ifdef CONFIG_TASK_DELAY_ACCT + unsigned long long io_sleep_time; /* IO sleep time */ +#endif +}; + +struct f2fs_lock_context { + struct f2fs_time_stat ts; + int orig_nice; + int new_nice; + bool lock_trace; + bool need_restore; +}; + +struct f2fs_gc_control { + unsigned int victim_segno; /* target victim segment number */ + int init_gc_type; /* FG_GC or BG_GC */ + bool no_bg_gc; /* check the space and stop bg_gc */ + bool should_migrate_blocks; /* should migrate blocks */ + bool err_gc_skipped; /* return EAGAIN if GC skipped */ + bool one_time; /* require one time GC in one migration unit */ + unsigned int nr_free_secs; /* # of free sections to do GC */ + struct f2fs_lock_context lc; /* lock context for gc_lock */ +}; + +/* + * For s_flag in struct f2fs_sb_info + * Modification on enum should be synchronized with s_flag array + */ +enum { + SBI_IS_DIRTY, /* dirty flag for checkpoint */ + SBI_IS_CLOSE, /* specify unmounting */ + SBI_NEED_FSCK, /* need fsck.f2fs to fix */ + SBI_POR_DOING, /* recovery is doing or not */ + SBI_NEED_SB_WRITE, /* need to recover superblock */ + SBI_NEED_CP, /* need to checkpoint */ + SBI_IS_SHUTDOWN, /* shutdown by ioctl */ + SBI_IS_RECOVERED, /* recovered orphan/data */ + SBI_CP_DISABLED, /* CP was disabled last mount */ + SBI_CP_DISABLED_QUICK, /* CP was disabled quickly */ + SBI_QUOTA_NEED_FLUSH, /* need to flush quota info in CP */ + SBI_QUOTA_SKIP_FLUSH, /* skip flushing quota in current CP */ + SBI_QUOTA_NEED_REPAIR, /* quota file may be corrupted */ + SBI_IS_RESIZEFS, /* resizefs is in process */ + SBI_IS_FREEZING, /* freezefs is in process */ + SBI_IS_WRITABLE, /* remove ro mountoption transiently */ + SBI_ENABLE_CHECKPOINT, /* indicate it's during f2fs_enable_checkpoint() */ + MAX_SBI_FLAG, +}; + +enum { + CP_TIME, + REQ_TIME, + DISCARD_TIME, + GC_TIME, + DISABLE_TIME, + UMOUNT_DISCARD_TIMEOUT, + MAX_TIME, +}; + +/* Note that you need to keep synchronization with this gc_mode_names array */ +enum { + GC_NORMAL, + GC_IDLE_CB, + GC_IDLE_GREEDY, + GC_IDLE_AT, + GC_URGENT_HIGH, + GC_URGENT_LOW, + GC_URGENT_MID, + MAX_GC_MODE, +}; + +enum { + BGGC_MODE_ON, /* background gc is on */ + BGGC_MODE_OFF, /* background gc is off */ + BGGC_MODE_SYNC, /* + * background gc is on, migrating blocks + * like foreground gc + */ +}; + +enum { + FS_MODE_ADAPTIVE, /* use both lfs/ssr allocation */ + FS_MODE_LFS, /* use lfs allocation only */ + FS_MODE_FRAGMENT_SEG, /* segment fragmentation mode */ + FS_MODE_FRAGMENT_BLK, /* block fragmentation mode */ +}; + +enum { + ALLOC_MODE_DEFAULT, /* stay default */ + ALLOC_MODE_REUSE, /* reuse segments as much as possible */ +}; + +enum fsync_mode { + FSYNC_MODE_POSIX, /* fsync follows posix semantics */ + FSYNC_MODE_STRICT, /* fsync behaves in line with ext4 */ + FSYNC_MODE_NOBARRIER, /* fsync behaves nobarrier based on posix */ +}; + +enum { + COMPR_MODE_FS, /* + * automatically compress compression + * enabled files + */ + COMPR_MODE_USER, /* + * automatical compression is disabled. + * user can control the file compression + * using ioctls + */ +}; + +enum { + DISCARD_UNIT_BLOCK, /* basic discard unit is block */ + DISCARD_UNIT_SEGMENT, /* basic discard unit is segment */ + DISCARD_UNIT_SECTION, /* basic discard unit is section */ +}; + +enum { + MEMORY_MODE_NORMAL, /* memory mode for normal devices */ + MEMORY_MODE_LOW, /* memory mode for low memory devices */ +}; + +enum errors_option { + MOUNT_ERRORS_READONLY, /* remount fs ro on errors */ + MOUNT_ERRORS_CONTINUE, /* continue on errors */ + MOUNT_ERRORS_PANIC, /* panic on errors */ +}; + +enum { + BACKGROUND, + FOREGROUND, + MAX_CALL_TYPE, + TOTAL_CALL = FOREGROUND, +}; + +enum f2fs_lookup_mode { + LOOKUP_PERF, + LOOKUP_COMPAT, + LOOKUP_AUTO, +}; + +/* For node type in __get_node_folio() */ +enum node_type { + NODE_TYPE_REGULAR, + NODE_TYPE_INODE, + NODE_TYPE_XATTR, + NODE_TYPE_NON_INODE, + NODE_TYPE_NON_IXNODE, /* non inode and xnode */ +}; + +/* a threshold of maximum elapsed time in critical region to print tracepoint */ +#define MAX_LOCK_ELAPSED_TIME 500 + +#define F2FS_DEFAULT_TASK_PRIORITY (DEFAULT_PRIO) +#define F2FS_CRITICAL_TASK_PRIORITY NICE_TO_PRIO(0) + +static inline int f2fs_test_bit(unsigned int nr, char *addr); +static inline void f2fs_set_bit(unsigned int nr, char *addr); +static inline void f2fs_clear_bit(unsigned int nr, char *addr); + +/* + * Layout of f2fs page.private: + * + * Layout A: lowest bit should be 1 + * | bit0 = 1 | bit1 | bit2 | ... | bit MAX | private data .... | + * bit 0 PAGE_PRIVATE_NOT_POINTER + * bit 1 PAGE_PRIVATE_ONGOING_MIGRATION + * bit 2 PAGE_PRIVATE_INLINE_INODE + * bit 3 PAGE_PRIVATE_REF_RESOURCE + * bit 4 PAGE_PRIVATE_ATOMIC_WRITE + * bit 5- f2fs private data + * + * Layout B: lowest bit should be 0 + * page.private is a wrapped pointer. + */ +enum { + PAGE_PRIVATE_NOT_POINTER, /* private contains non-pointer data */ + PAGE_PRIVATE_ONGOING_MIGRATION, /* data page which is on-going migrating */ + PAGE_PRIVATE_INLINE_INODE, /* inode page contains inline data */ + PAGE_PRIVATE_REF_RESOURCE, /* dirty page has referenced resources */ + PAGE_PRIVATE_ATOMIC_WRITE, /* data page from atomic write path */ + PAGE_PRIVATE_MAX +}; + +/* For compression */ +enum compress_algorithm_type { + COMPRESS_LZO, + COMPRESS_LZ4, + COMPRESS_ZSTD, + COMPRESS_LZORLE, + COMPRESS_MAX, +}; + +enum compress_flag { + COMPRESS_CHKSUM, + COMPRESS_MAX_FLAG, +}; + +#define COMPRESS_WATERMARK 20 +#define COMPRESS_PERCENT 20 + +#define COMPRESS_DATA_RESERVED_SIZE 4 +struct compress_data { + __le32 clen; /* compressed data size */ + __le32 chksum; /* compressed data checksum */ + __le32 reserved[COMPRESS_DATA_RESERVED_SIZE]; /* reserved */ + u8 cdata[]; /* compressed data */ +}; + +#define COMPRESS_HEADER_SIZE (sizeof(struct compress_data)) + +#define F2FS_COMPRESSED_PAGE_MAGIC 0xF5F2C000 + +#define F2FS_ZSTD_DEFAULT_CLEVEL 1 + +#define COMPRESS_LEVEL_OFFSET 8 + +/* compress context */ +struct compress_ctx { + struct inode *inode; /* inode the context belong to */ + pgoff_t cluster_idx; /* cluster index number */ + unsigned int cluster_size; /* page count in cluster */ + unsigned int log_cluster_size; /* log of cluster size */ + struct page **rpages; /* pages store raw data in cluster */ + unsigned int nr_rpages; /* total page number in rpages */ + struct page **cpages; /* pages store compressed data in cluster */ + unsigned int nr_cpages; /* total page number in cpages */ + unsigned int valid_nr_cpages; /* valid page number in cpages */ + void *rbuf; /* virtual mapped address on rpages */ + struct compress_data *cbuf; /* virtual mapped address on cpages */ + size_t rlen; /* valid data length in rbuf */ + size_t clen; /* valid data length in cbuf */ + void *private; /* payload buffer for specified compression algorithm */ + void *private2; /* extra payload buffer */ + struct fsverity_info *vi; /* verity info if needed */ +}; + +/* compress context for write IO path */ +struct compress_io_ctx { + u32 magic; /* magic number to indicate page is compressed */ + struct inode *inode; /* inode the context belong to */ + struct page **rpages; /* pages store raw data in cluster */ + unsigned int nr_rpages; /* total page number in rpages */ + atomic_t pending_pages; /* in-flight compressed page count */ +}; + +/* Context for decompressing one cluster on the read IO path */ +struct decompress_io_ctx { + u32 magic; /* magic number to indicate page is compressed */ + struct inode *inode; /* inode the context belong to */ + struct f2fs_sb_info *sbi; /* f2fs_sb_info pointer */ + pgoff_t cluster_idx; /* cluster index number */ + unsigned int cluster_size; /* page count in cluster */ + unsigned int log_cluster_size; /* log of cluster size */ + struct page **rpages; /* pages store raw data in cluster */ + unsigned int nr_rpages; /* total page number in rpages */ + struct page **cpages; /* pages store compressed data in cluster */ + unsigned int nr_cpages; /* total page number in cpages */ + struct page **tpages; /* temp pages to pad holes in cluster */ + void *rbuf; /* virtual mapped address on rpages */ + struct compress_data *cbuf; /* virtual mapped address on cpages */ + size_t rlen; /* valid data length in rbuf */ + size_t clen; /* valid data length in cbuf */ + + /* + * The number of compressed pages remaining to be read in this cluster. + * This is initially nr_cpages. It is decremented by 1 each time a page + * has been read (or failed to be read). When it reaches 0, the cluster + * is decompressed (or an error is reported). + * + * If an error occurs before all the pages have been submitted for I/O, + * then this will never reach 0. In this case the I/O submitter is + * responsible for calling f2fs_decompress_end_io() instead. + */ + atomic_t remaining_pages; + + /* + * Number of references to this decompress_io_ctx. + * + * One reference is held for I/O completion. This reference is dropped + * after the pagecache pages are updated and unlocked -- either after + * decompression (and verity if enabled), or after an error. + * + * In addition, each compressed page holds a reference while it is in a + * bio. These references are necessary prevent compressed pages from + * being freed while they are still in a bio. + */ + refcount_t refcnt; + + bool failed; /* IO error occurred before decompression? */ + struct fsverity_info *vi; /* fs-verity context if needed */ + unsigned char compress_algorithm; /* backup algorithm type */ + void *private; /* payload buffer for specified decompression algorithm */ + void *private2; /* extra payload buffer */ + struct work_struct verity_work; /* work to verify the decompressed pages */ + struct work_struct free_work; /* work for late free this structure itself */ +}; + +#define NULL_CLUSTER ((unsigned int)(~0)) +#define MIN_COMPRESS_LOG_SIZE 2 +#define MAX_COMPRESS_LOG_SIZE 8 +#define MAX_COMPRESS_WINDOW_SIZE(log_size) ((PAGE_SIZE) << (log_size)) + +struct f2fs_sb_info { + struct super_block *sb; /* pointer to VFS super block */ + struct proc_dir_entry *s_proc; /* proc entry */ + struct f2fs_super_block *raw_super; /* raw super block pointer */ + struct f2fs_rwsem sb_lock; /* lock for raw super block */ + int valid_super_block; /* valid super block no */ + unsigned long s_flag; /* flags for sbi */ + struct mutex writepages; /* mutex for writepages() */ + +#ifdef CONFIG_BLK_DEV_ZONED + unsigned int blocks_per_blkz; /* F2FS blocks per zone */ + unsigned int unusable_blocks_per_sec; /* unusable blocks per section */ + unsigned int max_open_zones; /* max open zone resources of the zoned device */ + /* For adjust the priority writing position of data in zone UFS */ + unsigned int blkzone_alloc_policy; +#endif + + /* for node-related operations */ + struct f2fs_nm_info *nm_info; /* node manager */ + struct inode *node_inode; /* cache node blocks */ + + /* for segment-related operations */ + struct f2fs_sm_info *sm_info; /* segment manager */ + + /* for bio operations */ + struct f2fs_bio_info *write_io[NR_PAGE_TYPE]; /* for write bios */ + /* keep migration IO order for LFS mode */ + struct f2fs_rwsem io_order_lock; + pgoff_t page_eio_ofs[NR_PAGE_TYPE]; /* EIO page offset */ + int page_eio_cnt[NR_PAGE_TYPE]; /* EIO count */ + + /* for checkpoint */ + struct f2fs_checkpoint *ckpt; /* raw checkpoint pointer */ + int cur_cp_pack; /* remain current cp pack */ + spinlock_t cp_lock; /* for flag in ckpt */ + struct inode *meta_inode; /* cache meta blocks */ + struct f2fs_rwsem cp_global_sem; /* checkpoint procedure lock */ + struct f2fs_rwsem cp_rwsem; /* blocking FS operations */ + struct f2fs_rwsem node_write; /* locking node writes */ + struct f2fs_rwsem node_change; /* locking node change */ + wait_queue_head_t cp_wait; + unsigned long last_time[MAX_TIME]; /* to store time in jiffies */ + long interval_time[MAX_TIME]; /* to store thresholds */ + struct ckpt_req_control cprc_info; /* for checkpoint request control */ + struct cp_stats cp_stats; /* for time stat of checkpoint */ + + struct inode_management im[MAX_INO_ENTRY]; /* manage inode cache */ + + spinlock_t fsync_node_lock; /* for node entry lock */ + struct list_head fsync_node_list; /* node list head */ + unsigned int fsync_seg_id; /* sequence id */ + unsigned int fsync_node_num; /* number of node entries */ + + /* for orphan inode, use 0'th array */ + unsigned int max_orphans; /* max orphan inodes */ + + /* for inode management */ + struct list_head inode_list[NR_INODE_TYPE]; /* dirty inode list */ + spinlock_t inode_lock[NR_INODE_TYPE]; /* for dirty inode list lock */ + struct mutex flush_lock; /* for flush exclusion */ + + /* for extent tree cache */ + struct extent_tree_info extent_tree[NR_EXTENT_CACHES]; + atomic64_t allocated_data_blocks; /* for block age extent_cache */ + unsigned int max_read_extent_count; /* max read extent count per inode */ + + /* The threshold used for hot and warm data seperation*/ + unsigned int hot_data_age_threshold; + unsigned int warm_data_age_threshold; + unsigned int last_age_weight; + + /* control donate caches */ + unsigned int donate_files; + + /* basic filesystem units */ + unsigned int log_sectors_per_block; /* log2 sectors per block */ + unsigned int log_blocksize; /* log2 block size */ + unsigned int blocksize; /* block size */ + unsigned int root_ino_num; /* root inode number*/ + unsigned int node_ino_num; /* node inode number*/ + unsigned int meta_ino_num; /* meta inode number*/ + unsigned int log_blocks_per_seg; /* log2 blocks per segment */ + unsigned int blocks_per_seg; /* blocks per segment */ + unsigned int segs_per_sec; /* segments per section */ + unsigned int secs_per_zone; /* sections per zone */ + unsigned int total_sections; /* total section count */ + unsigned int total_node_count; /* total node block count */ + unsigned int total_valid_node_count; /* valid node block count */ + int dir_level; /* directory level */ + bool readdir_ra; /* readahead inode in readdir */ + unsigned int max_io_bytes; /* max io bytes to merge IOs */ + + /* variable summary block units */ + unsigned int sum_blocksize; /* sum block size */ + unsigned int sums_per_block; /* sum block count per block */ + unsigned int entries_in_sum; /* entry count in sum block */ + unsigned int sum_entry_size; /* total entry size in sum block */ + unsigned int sum_journal_size; /* journal size in sum block */ + unsigned int nat_journal_entries; /* nat journal entry count in the journal */ + unsigned int sit_journal_entries; /* sit journal entry count in the journal */ + + block_t user_block_count; /* # of user blocks */ + block_t total_valid_block_count; /* # of valid blocks */ + block_t discard_blks; /* discard command candidats */ + block_t last_valid_block_count; /* for recovery */ + block_t reserved_blocks; /* configurable reserved blocks */ + block_t current_reserved_blocks; /* current reserved blocks */ + + /* Additional tracking for no checkpoint mode */ + block_t unusable_block_count; /* # of blocks saved by last cp */ + + unsigned int nquota_files; /* # of quota sysfile */ + struct f2fs_rwsem quota_sem; /* blocking cp for flags */ + struct task_struct *umount_lock_holder; /* s_umount lock holder */ + + /* # of pages, see count_type */ + atomic_t nr_pages[NR_COUNT_TYPE]; + /* # of allocated blocks */ + struct percpu_counter alloc_valid_block_count; + /* # of node block writes as roll forward recovery */ + struct percpu_counter rf_node_block_count; + + /* writeback control */ + atomic_t wb_sync_req[META]; /* count # of WB_SYNC threads */ + + /* valid inode count */ + struct percpu_counter total_valid_inode_count; + + struct f2fs_mount_info mount_opt; /* mount options */ + + /* for cleaning operations */ + struct f2fs_rwsem gc_lock; /* + * semaphore for GC, avoid + * race between GC and GC or CP + */ + struct f2fs_gc_kthread *gc_thread; /* GC thread */ + struct atgc_management am; /* atgc management */ + unsigned int cur_victim_sec; /* current victim section num */ + unsigned int gc_mode; /* current GC state */ + unsigned int next_victim_seg[2]; /* next segment in victim section */ + spinlock_t gc_remaining_trials_lock; + /* remaining trial count for GC_URGENT_* and GC_IDLE_* */ + unsigned int gc_remaining_trials; + + /* for skip statistic */ + unsigned long long skipped_gc_rwsem; /* FG_GC only */ + + /* free sections reserved for pinned file */ + unsigned int reserved_pin_section; + + /* threshold for gc trials on pinned files */ + unsigned short gc_pin_file_threshold; + struct f2fs_rwsem pin_sem; + + /* maximum # of trials to find a victim segment for SSR and GC */ + unsigned int max_victim_search; + /* migration granularity of garbage collection, unit: segment */ + unsigned int migration_granularity; + /* migration window granularity of garbage collection, unit: segment */ + unsigned int migration_window_granularity; + + /* + * for stat information. + * one is for the LFS mode, and the other is for the SSR mode. + */ +#ifdef CONFIG_F2FS_STAT_FS + struct f2fs_stat_info *stat_info; /* FS status information */ + atomic_t meta_count[META_MAX]; /* # of meta blocks */ + unsigned int segment_count[2]; /* # of allocated segments */ + unsigned int block_count[2]; /* # of allocated blocks */ + atomic_t inplace_count; /* # of inplace update */ + /* # of lookup extent cache */ + atomic64_t total_hit_ext[NR_EXTENT_CACHES]; + /* # of hit rbtree extent node */ + atomic64_t read_hit_rbtree[NR_EXTENT_CACHES]; + /* # of hit cached extent node */ + atomic64_t read_hit_cached[NR_EXTENT_CACHES]; + /* # of hit largest extent node in read extent cache */ + atomic64_t read_hit_largest; + atomic_t inline_xattr; /* # of inline_xattr inodes */ + atomic_t inline_inode; /* # of inline_data inodes */ + atomic_t inline_dir; /* # of inline_dentry inodes */ + atomic_t compr_inode; /* # of compressed inodes */ + atomic64_t compr_blocks; /* # of compressed blocks */ + atomic_t swapfile_inode; /* # of swapfile inodes */ + atomic_t atomic_files; /* # of opened atomic file */ + atomic_t max_aw_cnt; /* max # of atomic writes */ + unsigned int io_skip_bggc; /* skip background gc for in-flight IO */ + unsigned int other_skip_bggc; /* skip background gc for other reasons */ + unsigned int ndirty_inode[NR_INODE_TYPE]; /* # of dirty inodes */ + atomic_t cp_call_count[MAX_CALL_TYPE]; /* # of cp call */ +#endif + spinlock_t stat_lock; /* lock for stat operations */ + + /* to attach REQ_META|REQ_FUA flags */ + unsigned int data_io_flag; + unsigned int node_io_flag; + + /* For sysfs support */ + struct kobject s_kobj; /* /sys/fs/f2fs/ */ + struct completion s_kobj_unregister; + + struct kobject s_stat_kobj; /* /sys/fs/f2fs//stat */ + struct completion s_stat_kobj_unregister; + + struct kobject s_feature_list_kobj; /* /sys/fs/f2fs//feature_list */ + struct completion s_feature_list_kobj_unregister; + + /* For shrinker support */ + struct list_head s_list; + struct mutex umount_mutex; + unsigned int shrinker_run_no; + + /* For multi devices */ + int s_ndevs; /* number of devices */ + struct f2fs_dev_info *devs; /* for device list */ + unsigned int dirty_device; /* for checkpoint data flush */ + spinlock_t dev_lock; /* protect dirty_device */ + bool aligned_blksize; /* all devices has the same logical blksize */ + unsigned int first_seq_zone_segno; /* first segno in sequential zone */ + unsigned int bggc_io_aware; /* For adjust the BG_GC priority when pending IO */ + unsigned int allocate_section_hint; /* the boundary position between devices */ + unsigned int allocate_section_policy; /* determine the section writing priority */ + + /* For write statistics */ + u64 sectors_written_start; + u64 kbytes_written; + + /* Precomputed FS UUID checksum for seeding other checksums */ + __u32 s_chksum_seed; + + struct workqueue_struct *wq; /* bio completion workqueue */ + + /* + * If we are in irq context, let's update error information into + * on-disk superblock in the work. + */ + struct work_struct s_error_work; + unsigned char errors[MAX_F2FS_ERRORS]; /* error flags */ + unsigned char stop_reason[MAX_STOP_REASON]; /* stop reason */ + spinlock_t error_lock; /* protect errors/stop_reason array */ + bool error_dirty; /* errors of sb is dirty */ + bool stop_reason_dirty; /* stop reason of sb is dirty */ + + /* For reclaimed segs statistics per each GC mode */ + unsigned int gc_segment_mode; /* GC state for reclaimed segments */ + unsigned int gc_reclaimed_segs[MAX_GC_MODE]; /* Reclaimed segs for each mode */ + + unsigned int seq_file_ra_mul; /* multiplier for ra_pages of seq. files in fadvise */ + + int max_fragment_chunk; /* max chunk size for block fragmentation mode */ + int max_fragment_hole; /* max hole size for block fragmentation mode */ + + /* For atomic write statistics */ + atomic64_t current_atomic_write; + s64 peak_atomic_write; + u64 committed_atomic_block; + u64 revoked_atomic_block; + + /* carve out reserved_blocks from total blocks */ + bool carve_out; + + /* max elapsed time threshold in critical region that lock covered */ + unsigned long long max_lock_elapsed_time; + + /* enable/disable to adjust task priority in critical region covered by lock */ + unsigned int adjust_lock_priority; + + /* adjust priority for task which is in critical region covered by lock */ + unsigned int lock_duration_priority; + + /* priority for critical task, e.g. f2fs_ckpt, f2fs_gc threads */ + long critical_task_priority; + +#ifdef CONFIG_F2FS_FS_COMPRESSION + struct kmem_cache *page_array_slab; /* page array entry */ + unsigned int page_array_slab_size; /* default page array slab size */ + + /* For runtime compression statistics */ + u64 compr_written_block; + u64 compr_saved_block; + u32 compr_new_inode; + + /* For compressed block cache */ + struct inode *compress_inode; /* cache compressed blocks */ + unsigned int compress_percent; /* cache page percentage */ + unsigned int compress_watermark; /* cache page watermark */ + atomic_t compress_page_hit; /* cache hit count */ +#endif + +#ifdef CONFIG_F2FS_IOSTAT + /* For app/fs IO statistics */ + spinlock_t iostat_lock; + unsigned long long iostat_count[NR_IO_TYPE]; + unsigned long long iostat_bytes[NR_IO_TYPE]; + unsigned long long prev_iostat_bytes[NR_IO_TYPE]; + unsigned long long iostat_read_folio_count[NR_PAGE_ORDERS]; + unsigned long long prev_iostat_read_folio_count[NR_PAGE_ORDERS]; + bool iostat_enable; + unsigned long iostat_next_period; + unsigned int iostat_period_ms; + + /* For io latency related statistics info in one iostat period */ + spinlock_t iostat_lat_lock; + struct iostat_lat_info *iostat_io_lat; +#endif +#ifdef CONFIG_DEBUG_LOCK_ALLOC + struct lock_class_key cp_global_sem_key; +#endif +}; + +/* Definitions to access f2fs_sb_info */ +#define SEGS_TO_BLKS(sbi, segs) \ + ((segs) << (sbi)->log_blocks_per_seg) +#define BLKS_TO_SEGS(sbi, blks) \ + ((blks) >> (sbi)->log_blocks_per_seg) + +#define BLKS_PER_SEG(sbi) ((sbi)->blocks_per_seg) +#define BLKS_PER_SEC(sbi) (SEGS_TO_BLKS(sbi, (sbi)->segs_per_sec)) +#define SEGS_PER_SEC(sbi) ((sbi)->segs_per_sec) + +__printf(3, 4) +void f2fs_printk(struct f2fs_sb_info *sbi, bool limit_rate, const char *fmt, ...); + +#define f2fs_err(sbi, fmt, ...) \ + f2fs_printk(sbi, false, KERN_ERR fmt, ##__VA_ARGS__) +#define f2fs_warn(sbi, fmt, ...) \ + f2fs_printk(sbi, false, KERN_WARNING fmt, ##__VA_ARGS__) +#define f2fs_notice(sbi, fmt, ...) \ + f2fs_printk(sbi, false, KERN_NOTICE fmt, ##__VA_ARGS__) +#define f2fs_info(sbi, fmt, ...) \ + f2fs_printk(sbi, false, KERN_INFO fmt, ##__VA_ARGS__) +#define f2fs_debug(sbi, fmt, ...) \ + f2fs_printk(sbi, false, KERN_DEBUG fmt, ##__VA_ARGS__) + +#define f2fs_err_ratelimited(sbi, fmt, ...) \ + f2fs_printk(sbi, true, KERN_ERR fmt, ##__VA_ARGS__) +#define f2fs_warn_ratelimited(sbi, fmt, ...) \ + f2fs_printk(sbi, true, KERN_WARNING fmt, ##__VA_ARGS__) +#define f2fs_info_ratelimited(sbi, fmt, ...) \ + f2fs_printk(sbi, true, KERN_INFO fmt, ##__VA_ARGS__) + +#ifdef CONFIG_F2FS_FAULT_INJECTION +#define time_to_inject(sbi, type) __time_to_inject(sbi, type, __func__, \ + __builtin_return_address(0)) +static inline bool __time_to_inject(struct f2fs_sb_info *sbi, int type, + const char *func, const char *parent_func) +{ + struct f2fs_fault_info *ffi = &F2FS_OPTION(sbi).fault_info; + + if (!ffi->inject_rate) + return false; + + if (!IS_FAULT_SET(ffi, type)) + return false; + + atomic_inc(&ffi->inject_ops); + if (atomic_read(&ffi->inject_ops) >= ffi->inject_rate) { + atomic_set(&ffi->inject_ops, 0); + ffi->inject_count[type]++; + f2fs_info_ratelimited(sbi, "inject %s in %s of %pS", + f2fs_fault_name[type], func, parent_func); + return true; + } + return false; +} +#else +static inline bool time_to_inject(struct f2fs_sb_info *sbi, int type) +{ + return false; +} +#endif + +/* + * Test if the mounted volume is a multi-device volume. + * - For a single regular disk volume, sbi->s_ndevs is 0. + * - For a single zoned disk volume, sbi->s_ndevs is 1. + * - For a multi-device volume, sbi->s_ndevs is always 2 or more. + */ +static inline bool f2fs_is_multi_device(struct f2fs_sb_info *sbi) +{ + return sbi->s_ndevs > 1; +} + +static inline void f2fs_update_time(struct f2fs_sb_info *sbi, int type) +{ + unsigned long now = jiffies; + + WRITE_ONCE(sbi->last_time[type], now); + + /* DISCARD_TIME and GC_TIME are based on REQ_TIME */ + if (type == REQ_TIME) { + WRITE_ONCE(sbi->last_time[DISCARD_TIME], now); + WRITE_ONCE(sbi->last_time[GC_TIME], now); + } +} + +static inline bool f2fs_time_over(struct f2fs_sb_info *sbi, int type) +{ + unsigned long interval = sbi->interval_time[type] * HZ; + + return time_after(jiffies, READ_ONCE(sbi->last_time[type]) + interval); +} + +static inline unsigned int f2fs_time_to_wait(struct f2fs_sb_info *sbi, + int type) +{ + unsigned long interval = sbi->interval_time[type] * HZ; + unsigned int wait_ms = 0; + long delta; + + delta = (READ_ONCE(sbi->last_time[type]) + interval) - jiffies; + if (delta > 0) + wait_ms = jiffies_to_msecs(delta); + + return wait_ms; +} + +/* + * Inline functions + */ +static inline u32 __f2fs_crc32(u32 crc, const void *address, + unsigned int length) +{ + return crc32(crc, address, length); +} + +static inline u32 f2fs_crc32(const void *address, unsigned int length) +{ + return __f2fs_crc32(F2FS_SUPER_MAGIC, address, length); +} + +static inline u32 f2fs_chksum(u32 crc, const void *address, unsigned int length) +{ + return __f2fs_crc32(crc, address, length); +} + +static inline struct f2fs_inode_info *F2FS_I(struct inode *inode) +{ + return container_of(inode, struct f2fs_inode_info, vfs_inode); +} + +static inline struct f2fs_sb_info *F2FS_SB(struct super_block *sb) +{ + return sb->s_fs_info; +} + +static inline struct f2fs_sb_info *F2FS_I_SB(struct inode *inode) +{ + return F2FS_SB(inode->i_sb); +} + +static inline struct f2fs_sb_info *F2FS_M_SB(struct address_space *mapping) +{ + return F2FS_I_SB(mapping->host); +} + +static inline struct f2fs_sb_info *F2FS_F_SB(const struct folio *folio) +{ + return F2FS_M_SB(folio->mapping); +} + +static inline struct f2fs_super_block *F2FS_RAW_SUPER(struct f2fs_sb_info *sbi) +{ + return (struct f2fs_super_block *)(sbi->raw_super); +} + +static inline struct f2fs_super_block *F2FS_SUPER_BLOCK(struct folio *folio, + pgoff_t index) +{ + pgoff_t idx_in_folio = index % folio_nr_pages(folio); + + return (struct f2fs_super_block *) + (page_address(folio_page(folio, idx_in_folio)) + + F2FS_SUPER_OFFSET); +} + +static inline struct f2fs_checkpoint *F2FS_CKPT(struct f2fs_sb_info *sbi) +{ + return (struct f2fs_checkpoint *)(sbi->ckpt); +} + +static inline struct f2fs_node *F2FS_NODE(const struct folio *folio) +{ + return (struct f2fs_node *)folio_address(folio); +} + +static inline struct f2fs_inode *F2FS_INODE(const struct folio *folio) +{ + return &((struct f2fs_node *)folio_address(folio))->i; +} + +static inline struct f2fs_nm_info *NM_I(struct f2fs_sb_info *sbi) +{ + return (struct f2fs_nm_info *)(sbi->nm_info); +} + +static inline struct f2fs_sm_info *SM_I(struct f2fs_sb_info *sbi) +{ + return (struct f2fs_sm_info *)(sbi->sm_info); +} + +static inline struct sit_info *SIT_I(struct f2fs_sb_info *sbi) +{ + return (struct sit_info *)(SM_I(sbi)->sit_info); +} + +static inline struct free_segmap_info *FREE_I(struct f2fs_sb_info *sbi) +{ + return (struct free_segmap_info *)(SM_I(sbi)->free_info); +} + +static inline struct dirty_seglist_info *DIRTY_I(struct f2fs_sb_info *sbi) +{ + return (struct dirty_seglist_info *)(SM_I(sbi)->dirty_info); +} + +static inline struct address_space *META_MAPPING(struct f2fs_sb_info *sbi) +{ + return sbi->meta_inode->i_mapping; +} + +static inline struct address_space *NODE_MAPPING(struct f2fs_sb_info *sbi) +{ + return sbi->node_inode->i_mapping; +} + +static inline bool is_meta_folio(struct folio *folio) +{ + return folio->mapping == META_MAPPING(F2FS_F_SB(folio)); +} + +static inline bool is_node_folio(struct folio *folio) +{ + return folio->mapping == NODE_MAPPING(F2FS_F_SB(folio)); +} + +static inline bool is_sbi_flag_set(struct f2fs_sb_info *sbi, unsigned int type) +{ + return test_bit(type, &sbi->s_flag); +} + +static inline void __set_sbi_flag(struct f2fs_sb_info *sbi, unsigned int type) +{ + set_bit(type, &sbi->s_flag); +} + +#define set_sbi_flag(sbi, type) \ +do { \ + __set_sbi_flag(sbi, type); \ + if ((type) == SBI_NEED_FSCK) \ + f2fs_fault_report(sbi->sb, REPORT_FAULT_NEED_FSCK, __func__, __LINE__); \ +} while (0) + +static inline void clear_sbi_flag(struct f2fs_sb_info *sbi, unsigned int type) +{ + clear_bit(type, &sbi->s_flag); +} + +static inline unsigned long long cur_cp_version(struct f2fs_checkpoint *cp) +{ + return le64_to_cpu(cp->checkpoint_ver); +} + +static inline unsigned long f2fs_qf_ino(struct super_block *sb, int type) +{ + if (type < F2FS_MAX_QUOTAS) + return le32_to_cpu(F2FS_SB(sb)->raw_super->qf_ino[type]); + return 0; +} + +static inline __u64 cur_cp_crc(struct f2fs_checkpoint *cp) +{ + size_t crc_offset = le32_to_cpu(cp->checksum_offset); + return le32_to_cpu(*((__le32 *)((unsigned char *)cp + crc_offset))); +} + +static inline bool __is_set_ckpt_flags(struct f2fs_checkpoint *cp, unsigned int f) +{ + unsigned int ckpt_flags = le32_to_cpu(cp->ckpt_flags); + + return ckpt_flags & f; +} + +static inline bool is_set_ckpt_flags(struct f2fs_sb_info *sbi, unsigned int f) +{ + return __is_set_ckpt_flags(F2FS_CKPT(sbi), f); +} + +static inline void __set_ckpt_flags(struct f2fs_checkpoint *cp, unsigned int f) +{ + unsigned int ckpt_flags; + + ckpt_flags = le32_to_cpu(cp->ckpt_flags); + ckpt_flags |= f; + cp->ckpt_flags = cpu_to_le32(ckpt_flags); +} + +static inline void set_ckpt_flags(struct f2fs_sb_info *sbi, unsigned int f) +{ + unsigned long flags; + + spin_lock_irqsave(&sbi->cp_lock, flags); + __set_ckpt_flags(F2FS_CKPT(sbi), f); + spin_unlock_irqrestore(&sbi->cp_lock, flags); +} + +static inline void __clear_ckpt_flags(struct f2fs_checkpoint *cp, unsigned int f) +{ + unsigned int ckpt_flags; + + ckpt_flags = le32_to_cpu(cp->ckpt_flags); + ckpt_flags &= (~f); + cp->ckpt_flags = cpu_to_le32(ckpt_flags); +} + +static inline void clear_ckpt_flags(struct f2fs_sb_info *sbi, unsigned int f) +{ + unsigned long flags; + + spin_lock_irqsave(&sbi->cp_lock, flags); + __clear_ckpt_flags(F2FS_CKPT(sbi), f); + spin_unlock_irqrestore(&sbi->cp_lock, flags); +} + +#define init_f2fs_rwsem(sem) __init_f2fs_rwsem(sem, NULL, LOCK_NAME_NONE) +#define init_f2fs_rwsem_trace __init_f2fs_rwsem + +#define __init_f2fs_rwsem(sem, sbi, name) \ +do { \ + static struct lock_class_key __key; \ + \ + do_init_f2fs_rwsem((sem), #sem, &__key, sbi, name); \ +} while (0) + +static inline void do_init_f2fs_rwsem(struct f2fs_rwsem *sem, + const char *sem_name, struct lock_class_key *key, + struct f2fs_sb_info *sbi, enum f2fs_lock_name name) +{ + sem->sbi = sbi; + sem->name = name; + __init_rwsem(&sem->internal_rwsem, sem_name, key); +#ifdef CONFIG_F2FS_UNFAIR_RWSEM + init_waitqueue_head(&sem->read_waiters); +#endif +} + +static inline int f2fs_rwsem_is_locked(struct f2fs_rwsem *sem) +{ + return rwsem_is_locked(&sem->internal_rwsem); +} + +static inline int f2fs_rwsem_is_contended(struct f2fs_rwsem *sem) +{ + return rwsem_is_contended(&sem->internal_rwsem); +} + +static inline void f2fs_down_read(struct f2fs_rwsem *sem) +{ +#ifdef CONFIG_F2FS_UNFAIR_RWSEM + wait_event(sem->read_waiters, down_read_trylock(&sem->internal_rwsem)); +#else + down_read(&sem->internal_rwsem); +#endif +} + +static inline int f2fs_down_read_trylock(struct f2fs_rwsem *sem) +{ + return down_read_trylock(&sem->internal_rwsem); +} + +static inline void f2fs_up_read(struct f2fs_rwsem *sem) +{ + up_read(&sem->internal_rwsem); +} + +static inline void f2fs_down_write(struct f2fs_rwsem *sem) +{ + down_write(&sem->internal_rwsem); +} + +#ifdef CONFIG_DEBUG_LOCK_ALLOC +static inline void f2fs_down_read_nested(struct f2fs_rwsem *sem, int subclass) +{ + down_read_nested(&sem->internal_rwsem, subclass); +} + +static inline void f2fs_down_write_nested(struct f2fs_rwsem *sem, int subclass) +{ + down_write_nested(&sem->internal_rwsem, subclass); +} +#else +#define f2fs_down_read_nested(sem, subclass) f2fs_down_read(sem) +#define f2fs_down_write_nested(sem, subclass) f2fs_down_write(sem) +#endif + +static inline int f2fs_down_write_trylock(struct f2fs_rwsem *sem) +{ + return down_write_trylock(&sem->internal_rwsem); +} + +static inline void f2fs_up_write(struct f2fs_rwsem *sem) +{ + up_write(&sem->internal_rwsem); +#ifdef CONFIG_F2FS_UNFAIR_RWSEM + wake_up_all(&sem->read_waiters); +#endif +} + +void f2fs_down_read_trace(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc); +int f2fs_down_read_trylock_trace(struct f2fs_rwsem *sem, + struct f2fs_lock_context *lc); +void f2fs_up_read_trace(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc); +void f2fs_down_write_trace(struct f2fs_rwsem *sem, + struct f2fs_lock_context *lc); +int f2fs_down_write_trylock_trace(struct f2fs_rwsem *sem, + struct f2fs_lock_context *lc); +void f2fs_up_write_trace(struct f2fs_rwsem *sem, struct f2fs_lock_context *lc); + +static inline void disable_nat_bits(struct f2fs_sb_info *sbi, bool lock) +{ + unsigned long flags; + unsigned char *nat_bits; + + /* + * In order to re-enable nat_bits we need to call fsck.f2fs by + * set_sbi_flag(sbi, SBI_NEED_FSCK). But it may give huge cost, + * so let's rely on regular fsck or unclean shutdown. + */ + + if (lock) + spin_lock_irqsave(&sbi->cp_lock, flags); + __clear_ckpt_flags(F2FS_CKPT(sbi), CP_NAT_BITS_FLAG); + nat_bits = NM_I(sbi)->nat_bits; + NM_I(sbi)->nat_bits = NULL; + if (lock) + spin_unlock_irqrestore(&sbi->cp_lock, flags); + + kvfree(nat_bits); +} + +static inline bool enabled_nat_bits(struct f2fs_sb_info *sbi, + struct cp_control *cpc) +{ + bool set = is_set_ckpt_flags(sbi, CP_NAT_BITS_FLAG); + + return (cpc) ? (cpc->reason & CP_UMOUNT) && set : set; +} + +static inline int __get_cp_reason(struct f2fs_sb_info *sbi) +{ + int reason = CP_SYNC; + + if (test_opt(sbi, FASTBOOT)) + reason = CP_FASTBOOT; + if (is_sbi_flag_set(sbi, SBI_IS_CLOSE)) + reason = CP_UMOUNT; + return reason; +} + +static inline bool __remain_node_summaries(int reason) +{ + return (reason & (CP_UMOUNT | CP_FASTBOOT)); +} + +static inline bool __exist_node_summaries(struct f2fs_sb_info *sbi) +{ + return (is_set_ckpt_flags(sbi, CP_UMOUNT_FLAG) || + is_set_ckpt_flags(sbi, CP_FASTBOOT_FLAG)); +} + +/* + * Check whether the inode has blocks or not + */ +static inline int F2FS_HAS_BLOCKS(struct inode *inode) +{ + block_t xattr_block = F2FS_I(inode)->i_xattr_nid ? 1 : 0; + + return (inode->i_blocks >> F2FS_LOG_SECTORS_PER_BLOCK) > xattr_block; +} + +static inline bool f2fs_has_xattr_block(unsigned int ofs) +{ + return ofs == XATTR_NODE_OFFSET; +} + +static inline bool __allow_reserved_root(struct f2fs_sb_info *sbi, + struct inode *inode, bool cap) +{ + if (!inode) + return true; + if (IS_NOQUOTA(inode)) + return true; + if (uid_eq(F2FS_OPTION(sbi).s_resuid, current_fsuid())) + return true; + if (!gid_eq(F2FS_OPTION(sbi).s_resgid, GLOBAL_ROOT_GID) && + in_group_p(F2FS_OPTION(sbi).s_resgid)) + return true; + if (cap && capable(CAP_SYS_RESOURCE)) + return true; + return false; +} + +static inline unsigned int get_available_block_count(struct f2fs_sb_info *sbi, + struct inode *inode, bool cap) +{ + block_t avail_user_block_count; + + avail_user_block_count = sbi->user_block_count - + sbi->current_reserved_blocks; + + if (test_opt(sbi, RESERVE_ROOT) && !__allow_reserved_root(sbi, inode, cap)) + avail_user_block_count -= F2FS_OPTION(sbi).root_reserved_blocks; + + if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { + if (avail_user_block_count > sbi->unusable_block_count) + avail_user_block_count -= sbi->unusable_block_count; + else + avail_user_block_count = 0; + } + + return avail_user_block_count; +} + +static inline void f2fs_i_blocks_write(struct inode *, block_t, bool, bool); +static inline int inc_valid_block_count(struct f2fs_sb_info *sbi, + struct inode *inode, blkcnt_t *count, bool partial) +{ + long long diff = 0, release = 0; + block_t avail_user_block_count; + int ret; + + ret = dquot_reserve_block(inode, *count); + if (ret) + return ret; + + if (time_to_inject(sbi, FAULT_BLOCK)) { + release = *count; + goto release_quota; + } + + /* + * let's increase this in prior to actual block count change in order + * for f2fs_sync_file to avoid data races when deciding checkpoint. + */ + percpu_counter_add(&sbi->alloc_valid_block_count, (*count)); + + spin_lock(&sbi->stat_lock); + + avail_user_block_count = get_available_block_count(sbi, inode, true); + diff = (long long)sbi->total_valid_block_count + *count - + avail_user_block_count; + if (unlikely(diff > 0)) { + if (!partial) { + spin_unlock(&sbi->stat_lock); + release = *count; + goto enospc; + } + if (diff > *count) + diff = *count; + *count -= diff; + release = diff; + if (!*count) { + spin_unlock(&sbi->stat_lock); + goto enospc; + } + } + sbi->total_valid_block_count += (block_t)(*count); + + spin_unlock(&sbi->stat_lock); + + if (unlikely(release)) { + percpu_counter_sub(&sbi->alloc_valid_block_count, release); + dquot_release_reservation_block(inode, release); + } + f2fs_i_blocks_write(inode, *count, true, true); + return 0; + +enospc: + percpu_counter_sub(&sbi->alloc_valid_block_count, release); +release_quota: + dquot_release_reservation_block(inode, release); + return -ENOSPC; +} + +#define PAGE_PRIVATE_GET_FUNC(name, flagname) \ +static inline bool folio_test_f2fs_##name(const struct folio *folio) \ +{ \ + unsigned long priv = (unsigned long)folio->private; \ + unsigned long v = (1UL << PAGE_PRIVATE_NOT_POINTER) | \ + (1UL << PAGE_PRIVATE_##flagname); \ + return (priv & v) == v; \ +} \ +static inline bool page_private_##name(struct page *page) \ +{ \ + return PagePrivate(page) && \ + test_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page)) && \ + test_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \ +} + +#define PAGE_PRIVATE_SET_FUNC(name, flagname) \ +static inline void folio_set_f2fs_##name(struct folio *folio) \ +{ \ + unsigned long v = (1UL << PAGE_PRIVATE_NOT_POINTER) | \ + (1UL << PAGE_PRIVATE_##flagname); \ + if (!folio->private) \ + folio_attach_private(folio, (void *)v); \ + else { \ + v |= (unsigned long)folio->private; \ + folio->private = (void *)v; \ + } \ +} \ +static inline void set_page_private_##name(struct page *page) \ +{ \ + if (!PagePrivate(page)) \ + attach_page_private(page, (void *)0); \ + set_bit(PAGE_PRIVATE_NOT_POINTER, &page_private(page)); \ + set_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \ +} + +#define PAGE_PRIVATE_CLEAR_FUNC(name, flagname) \ +static inline void folio_clear_f2fs_##name(struct folio *folio) \ +{ \ + unsigned long v = (unsigned long)folio->private; \ + \ + v &= ~(1UL << PAGE_PRIVATE_##flagname); \ + if (v == (1UL << PAGE_PRIVATE_NOT_POINTER)) \ + folio_detach_private(folio); \ + else \ + folio->private = (void *)v; \ +} \ +static inline void clear_page_private_##name(struct page *page) \ +{ \ + clear_bit(PAGE_PRIVATE_##flagname, &page_private(page)); \ + if (page_private(page) == BIT(PAGE_PRIVATE_NOT_POINTER)) \ + detach_page_private(page); \ +} + +PAGE_PRIVATE_GET_FUNC(nonpointer, NOT_POINTER); +PAGE_PRIVATE_GET_FUNC(inline, INLINE_INODE); +PAGE_PRIVATE_GET_FUNC(gcing, ONGOING_MIGRATION); +PAGE_PRIVATE_GET_FUNC(atomic, ATOMIC_WRITE); + +PAGE_PRIVATE_SET_FUNC(reference, REF_RESOURCE); +PAGE_PRIVATE_SET_FUNC(inline, INLINE_INODE); +PAGE_PRIVATE_SET_FUNC(gcing, ONGOING_MIGRATION); +PAGE_PRIVATE_SET_FUNC(atomic, ATOMIC_WRITE); + +PAGE_PRIVATE_CLEAR_FUNC(reference, REF_RESOURCE); +PAGE_PRIVATE_CLEAR_FUNC(inline, INLINE_INODE); +PAGE_PRIVATE_CLEAR_FUNC(gcing, ONGOING_MIGRATION); +PAGE_PRIVATE_CLEAR_FUNC(atomic, ATOMIC_WRITE); + +static inline unsigned long folio_get_f2fs_data(struct folio *folio) +{ + unsigned long data = (unsigned long)folio->private; + + if (!test_bit(PAGE_PRIVATE_NOT_POINTER, &data)) + return 0; + return data >> PAGE_PRIVATE_MAX; +} + +static inline void folio_set_f2fs_data(struct folio *folio, unsigned long data) +{ + data = (1UL << PAGE_PRIVATE_NOT_POINTER) | (data << PAGE_PRIVATE_MAX); + + if (!folio_test_private(folio)) + folio_attach_private(folio, (void *)data); + else + folio->private = (void *)((unsigned long)folio->private | data); +} + +static inline void dec_valid_block_count(struct f2fs_sb_info *sbi, + struct inode *inode, + block_t count) +{ + blkcnt_t sectors = count << F2FS_LOG_SECTORS_PER_BLOCK; + + spin_lock(&sbi->stat_lock); + if (unlikely(sbi->total_valid_block_count < count)) { + f2fs_warn(sbi, "Inconsistent total_valid_block_count:%u, ino:%llu, count:%u", + sbi->total_valid_block_count, inode->i_ino, count); + sbi->total_valid_block_count = 0; + set_sbi_flag(sbi, SBI_NEED_FSCK); + } else { + sbi->total_valid_block_count -= count; + } + if (sbi->reserved_blocks && + sbi->current_reserved_blocks < sbi->reserved_blocks) + sbi->current_reserved_blocks = min(sbi->reserved_blocks, + sbi->current_reserved_blocks + count); + spin_unlock(&sbi->stat_lock); + if (unlikely(inode->i_blocks < sectors)) { + f2fs_warn(sbi, "Inconsistent i_blocks, ino:%llu, iblocks:%llu, sectors:%llu", + inode->i_ino, + (unsigned long long)inode->i_blocks, + (unsigned long long)sectors); + set_sbi_flag(sbi, SBI_NEED_FSCK); + return; + } + f2fs_i_blocks_write(inode, count, false, true); +} + +static inline void inc_page_count(struct f2fs_sb_info *sbi, int count_type) +{ + atomic_inc(&sbi->nr_pages[count_type]); + + if (count_type == F2FS_DIRTY_DENTS || + count_type == F2FS_DIRTY_NODES || + count_type == F2FS_DIRTY_META || + count_type == F2FS_DIRTY_QDATA || + count_type == F2FS_DIRTY_IMETA) + set_sbi_flag(sbi, SBI_IS_DIRTY); +} + +static inline void inode_inc_dirty_pages(struct inode *inode) +{ + atomic_inc(&F2FS_I(inode)->dirty_pages); + inc_page_count(F2FS_I_SB(inode), S_ISDIR(inode->i_mode) ? + F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA); + if (IS_NOQUOTA(inode)) + inc_page_count(F2FS_I_SB(inode), F2FS_DIRTY_QDATA); +} + +static inline void dec_page_count(struct f2fs_sb_info *sbi, int count_type) +{ + atomic_dec(&sbi->nr_pages[count_type]); +} + +static inline void inode_dec_dirty_pages(struct inode *inode) +{ + if (!S_ISDIR(inode->i_mode) && !S_ISREG(inode->i_mode) && + !S_ISLNK(inode->i_mode)) + return; + + atomic_dec(&F2FS_I(inode)->dirty_pages); + dec_page_count(F2FS_I_SB(inode), S_ISDIR(inode->i_mode) ? + F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA); + if (IS_NOQUOTA(inode)) + dec_page_count(F2FS_I_SB(inode), F2FS_DIRTY_QDATA); +} + +static inline void inc_atomic_write_cnt(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_inode_info *fi = F2FS_I(inode); + u64 current_write; + + fi->atomic_write_cnt++; + atomic64_inc(&sbi->current_atomic_write); + current_write = atomic64_read(&sbi->current_atomic_write); + if (current_write > sbi->peak_atomic_write) + sbi->peak_atomic_write = current_write; +} + +static inline void release_atomic_write_cnt(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_inode_info *fi = F2FS_I(inode); + + atomic64_sub(fi->atomic_write_cnt, &sbi->current_atomic_write); + fi->atomic_write_cnt = 0; +} + +static inline s64 get_pages(struct f2fs_sb_info *sbi, int count_type) +{ + return atomic_read(&sbi->nr_pages[count_type]); +} + +static inline int get_dirty_pages(struct inode *inode) +{ + return atomic_read(&F2FS_I(inode)->dirty_pages); +} + +static inline int get_blocktype_secs(struct f2fs_sb_info *sbi, int block_type) +{ + return div_u64(get_pages(sbi, block_type) + BLKS_PER_SEC(sbi) - 1, + BLKS_PER_SEC(sbi)); +} + +static inline block_t valid_user_blocks(struct f2fs_sb_info *sbi) +{ + return sbi->total_valid_block_count; +} + +static inline block_t discard_blocks(struct f2fs_sb_info *sbi) +{ + return sbi->discard_blks; +} + +static inline unsigned long __bitmap_size(struct f2fs_sb_info *sbi, int flag) +{ + struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); + + /* return NAT or SIT bitmap */ + if (flag == NAT_BITMAP) + return le32_to_cpu(ckpt->nat_ver_bitmap_bytesize); + else if (flag == SIT_BITMAP) + return le32_to_cpu(ckpt->sit_ver_bitmap_bytesize); + + return 0; +} + +static inline block_t __cp_payload(struct f2fs_sb_info *sbi) +{ + return le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_payload); +} + +static inline void *__bitmap_ptr(struct f2fs_sb_info *sbi, int flag) +{ + struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); + void *tmp_ptr = &ckpt->sit_nat_version_bitmap; + int offset; + + if (is_set_ckpt_flags(sbi, CP_LARGE_NAT_BITMAP_FLAG)) { + offset = (flag == SIT_BITMAP) ? + le32_to_cpu(ckpt->nat_ver_bitmap_bytesize) : 0; + /* + * if large_nat_bitmap feature is enabled, leave checksum + * protection for all nat/sit bitmaps. + */ + return tmp_ptr + offset + sizeof(__le32); + } + + if (__cp_payload(sbi) > 0) { + if (flag == NAT_BITMAP) + return tmp_ptr; + else + return (unsigned char *)ckpt + F2FS_BLKSIZE; + } else { + offset = (flag == NAT_BITMAP) ? + le32_to_cpu(ckpt->sit_ver_bitmap_bytesize) : 0; + return tmp_ptr + offset; + } +} + +static inline block_t __start_cp_addr(struct f2fs_sb_info *sbi) +{ + block_t start_addr = le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_blkaddr); + + if (sbi->cur_cp_pack == 2) + start_addr += BLKS_PER_SEG(sbi); + return start_addr; +} + +static inline block_t __start_cp_next_addr(struct f2fs_sb_info *sbi) +{ + block_t start_addr = le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_blkaddr); + + if (sbi->cur_cp_pack == 1) + start_addr += BLKS_PER_SEG(sbi); + return start_addr; +} + +static inline void __set_cp_next_pack(struct f2fs_sb_info *sbi) +{ + sbi->cur_cp_pack = (sbi->cur_cp_pack == 1) ? 2 : 1; +} + +static inline block_t __start_sum_addr(struct f2fs_sb_info *sbi) +{ + return le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_start_sum); +} + +static inline bool __has_cursum_space(struct f2fs_sb_info *sbi, + struct f2fs_journal *journal, unsigned int size, int type) +{ + if (type == NAT_JOURNAL) + return size <= MAX_NAT_JENTRIES(sbi, journal); + return size <= MAX_SIT_JENTRIES(sbi, journal); +} + +extern void f2fs_mark_inode_dirty_sync(struct inode *inode, bool sync); +static inline int inc_valid_node_count(struct f2fs_sb_info *sbi, + struct inode *inode, bool is_inode) +{ + block_t valid_block_count; + unsigned int valid_node_count, avail_user_node_count; + unsigned int avail_user_block_count; + int err; + + if (is_inode) { + if (inode) { + err = dquot_alloc_inode(inode); + if (err) + return err; + } + } else { + err = dquot_reserve_block(inode, 1); + if (err) + return err; + } + + if (time_to_inject(sbi, FAULT_BLOCK)) + goto enospc; + + spin_lock(&sbi->stat_lock); + + valid_block_count = sbi->total_valid_block_count + 1; + avail_user_block_count = get_available_block_count(sbi, inode, + test_opt(sbi, RESERVE_NODE)); + + if (unlikely(valid_block_count > avail_user_block_count)) { + spin_unlock(&sbi->stat_lock); + goto enospc; + } + + avail_user_node_count = sbi->total_node_count - F2FS_RESERVED_NODE_NUM; + if (test_opt(sbi, RESERVE_NODE) && + !__allow_reserved_root(sbi, inode, true)) + avail_user_node_count -= F2FS_OPTION(sbi).root_reserved_nodes; + valid_node_count = sbi->total_valid_node_count + 1; + if (unlikely(valid_node_count > avail_user_node_count)) { + spin_unlock(&sbi->stat_lock); + goto enospc; + } + + sbi->total_valid_node_count++; + sbi->total_valid_block_count++; + spin_unlock(&sbi->stat_lock); + + if (inode) { + if (is_inode) + f2fs_mark_inode_dirty_sync(inode, true); + else + f2fs_i_blocks_write(inode, 1, true, true); + } + + percpu_counter_inc(&sbi->alloc_valid_block_count); + return 0; + +enospc: + if (is_inode) { + if (inode) + dquot_free_inode(inode); + } else { + dquot_release_reservation_block(inode, 1); + } + return -ENOSPC; +} + +static inline void dec_valid_node_count(struct f2fs_sb_info *sbi, + struct inode *inode, bool is_inode) +{ + spin_lock(&sbi->stat_lock); + + if (unlikely(!sbi->total_valid_block_count || + !sbi->total_valid_node_count)) { + f2fs_warn(sbi, "dec_valid_node_count: inconsistent block counts, total_valid_block:%u, total_valid_node:%u", + sbi->total_valid_block_count, + sbi->total_valid_node_count); + set_sbi_flag(sbi, SBI_NEED_FSCK); + } else { + sbi->total_valid_block_count--; + sbi->total_valid_node_count--; + } + + if (sbi->reserved_blocks && + sbi->current_reserved_blocks < sbi->reserved_blocks) + sbi->current_reserved_blocks++; + + spin_unlock(&sbi->stat_lock); + + if (is_inode) { + dquot_free_inode(inode); + } else { + if (unlikely(inode->i_blocks == 0)) { + f2fs_warn(sbi, "dec_valid_node_count: inconsistent i_blocks, ino:%llu, iblocks:%llu", + inode->i_ino, + (unsigned long long)inode->i_blocks); + set_sbi_flag(sbi, SBI_NEED_FSCK); + return; + } + f2fs_i_blocks_write(inode, 1, false, true); + } +} + +static inline unsigned int valid_node_count(struct f2fs_sb_info *sbi) +{ + return sbi->total_valid_node_count; +} + +static inline void inc_valid_inode_count(struct f2fs_sb_info *sbi) +{ + percpu_counter_inc(&sbi->total_valid_inode_count); +} + +static inline void dec_valid_inode_count(struct f2fs_sb_info *sbi) +{ + percpu_counter_dec(&sbi->total_valid_inode_count); +} + +static inline s64 valid_inode_count(struct f2fs_sb_info *sbi) +{ + return percpu_counter_sum_positive(&sbi->total_valid_inode_count); +} + +static inline struct folio *f2fs_grab_cache_folio(struct address_space *mapping, + pgoff_t index, bool for_write) +{ + struct folio *folio; + unsigned int flags; + + if (IS_ENABLED(CONFIG_F2FS_FAULT_INJECTION)) { + fgf_t fgf_flags; + + if (!for_write) + fgf_flags = FGP_LOCK | FGP_ACCESSED; + else + fgf_flags = FGP_LOCK; + folio = __filemap_get_folio(mapping, index, fgf_flags, 0); + if (!IS_ERR(folio)) + return folio; + + if (time_to_inject(F2FS_M_SB(mapping), FAULT_PAGE_ALLOC)) + return ERR_PTR(-ENOMEM); + } + + if (!for_write) + return filemap_grab_folio(mapping, index); + + flags = memalloc_nofs_save(); + folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN, + mapping_gfp_mask(mapping)); + memalloc_nofs_restore(flags); + + return folio; +} + +static inline struct folio *f2fs_filemap_get_folio( + struct address_space *mapping, pgoff_t index, + fgf_t fgp_flags, gfp_t gfp_mask) +{ + if (time_to_inject(F2FS_M_SB(mapping), FAULT_PAGE_GET)) + return ERR_PTR(-ENOMEM); + + return __filemap_get_folio(mapping, index, fgp_flags, gfp_mask); +} + +static inline void f2fs_folio_put(struct folio *folio, bool unlock) +{ + if (IS_ERR_OR_NULL(folio)) + return; + + if (unlock) { + f2fs_bug_on(F2FS_F_SB(folio), !folio_test_locked(folio)); + folio_unlock(folio); + } + folio_put(folio); +} + +static inline void f2fs_put_page(struct page *page, bool unlock) +{ + if (!page) + return; + f2fs_folio_put(page_folio(page), unlock); +} + +static inline void f2fs_put_dnode(struct dnode_of_data *dn) +{ + if (dn->node_folio) + f2fs_folio_put(dn->node_folio, true); + if (dn->inode_folio && dn->node_folio != dn->inode_folio) + f2fs_folio_put(dn->inode_folio, false); + dn->node_folio = NULL; + dn->inode_folio = NULL; +} + +static inline struct kmem_cache *f2fs_kmem_cache_create(const char *name, + size_t size) +{ + return kmem_cache_create(name, size, 0, SLAB_RECLAIM_ACCOUNT, NULL); +} + +static inline void *f2fs_kmem_cache_alloc_nofail(struct kmem_cache *cachep, + gfp_t flags) +{ + void *entry; + + entry = kmem_cache_alloc(cachep, flags); + if (!entry) + entry = kmem_cache_alloc(cachep, flags | __GFP_NOFAIL); + return entry; +} + +static inline void *f2fs_kmem_cache_alloc(struct kmem_cache *cachep, + gfp_t flags, bool nofail, struct f2fs_sb_info *sbi) +{ + if (nofail) + return f2fs_kmem_cache_alloc_nofail(cachep, flags); + + if (time_to_inject(sbi, FAULT_SLAB_ALLOC)) + return NULL; + + return kmem_cache_alloc(cachep, flags); +} + +static inline bool is_inflight_io(struct f2fs_sb_info *sbi, int type) +{ + if (get_pages(sbi, F2FS_RD_DATA) || get_pages(sbi, F2FS_RD_NODE) || + get_pages(sbi, F2FS_RD_META) || get_pages(sbi, F2FS_WB_DATA) || + get_pages(sbi, F2FS_WB_CP_DATA) || + get_pages(sbi, F2FS_DIO_READ) || + get_pages(sbi, F2FS_DIO_WRITE)) + return true; + + if (type != DISCARD_TIME && SM_I(sbi) && SM_I(sbi)->dcc_info && + atomic_read(&SM_I(sbi)->dcc_info->queued_discard)) + return true; + + if (SM_I(sbi) && SM_I(sbi)->fcc_info && + atomic_read(&SM_I(sbi)->fcc_info->queued_flush)) + return true; + return false; +} + +static inline bool is_inflight_read_io(struct f2fs_sb_info *sbi) +{ + return get_pages(sbi, F2FS_RD_DATA) || get_pages(sbi, F2FS_DIO_READ); +} + +static inline bool is_idle(struct f2fs_sb_info *sbi, int type) +{ + bool zoned_gc = (type == GC_TIME && + F2FS_HAS_FEATURE(sbi, F2FS_FEATURE_BLKZONED)); + + if (sbi->gc_mode == GC_URGENT_HIGH) + return true; + + if (sbi->bggc_io_aware == AWARE_READ_IO && is_inflight_read_io(sbi)) + return false; + if (sbi->bggc_io_aware == AWARE_ALL_IO && is_inflight_io(sbi, type)) + return false; + + if (sbi->gc_mode == GC_URGENT_MID) + return true; + + if (sbi->gc_mode == GC_URGENT_LOW && + (type == DISCARD_TIME || type == GC_TIME)) + return true; + + if (zoned_gc) + return true; + + return f2fs_time_over(sbi, type); +} + +static inline void f2fs_radix_tree_insert(struct radix_tree_root *root, + unsigned long index, void *item) +{ + while (radix_tree_insert(root, index, item)) + cond_resched(); +} + +#define RAW_IS_INODE(p) ((p)->footer.nid == (p)->footer.ino) + +static inline bool IS_INODE(const struct folio *folio) +{ + struct f2fs_node *p = F2FS_NODE(folio); + + return RAW_IS_INODE(p); +} + +static inline int offset_in_addr(struct f2fs_inode *i) +{ + return (i->i_inline & F2FS_EXTRA_ATTR) ? + (le16_to_cpu(i->i_extra_isize) / sizeof(__le32)) : 0; +} + +static inline __le32 *blkaddr_in_node(struct f2fs_node *node) +{ + return RAW_IS_INODE(node) ? node->i.i_addr : node->dn.addr; +} + +static inline int f2fs_has_extra_attr(struct inode *inode); +static inline unsigned int get_dnode_base(struct inode *inode, + struct folio *node_folio) +{ + if (!IS_INODE(node_folio)) + return 0; + + return inode ? get_extra_isize(inode) : + offset_in_addr(&F2FS_NODE(node_folio)->i); +} + +static inline __le32 *get_dnode_addr(struct inode *inode, + struct folio *node_folio) +{ + return blkaddr_in_node(F2FS_NODE(node_folio)) + + get_dnode_base(inode, node_folio); +} + +static inline block_t data_blkaddr(struct inode *inode, + struct folio *node_folio, unsigned int offset) +{ + return le32_to_cpu(*(get_dnode_addr(inode, node_folio) + offset)); +} + +static inline block_t f2fs_data_blkaddr(struct dnode_of_data *dn) +{ + return data_blkaddr(dn->inode, dn->node_folio, dn->ofs_in_node); +} + +static inline int f2fs_test_bit(unsigned int nr, char *addr) +{ + int mask; + + addr += (nr >> 3); + mask = BIT(7 - (nr & 0x07)); + return mask & *addr; +} + +static inline void f2fs_set_bit(unsigned int nr, char *addr) +{ + int mask; + + addr += (nr >> 3); + mask = BIT(7 - (nr & 0x07)); + *addr |= mask; +} + +static inline void f2fs_clear_bit(unsigned int nr, char *addr) +{ + int mask; + + addr += (nr >> 3); + mask = BIT(7 - (nr & 0x07)); + *addr &= ~mask; +} + +static inline int f2fs_test_and_set_bit(unsigned int nr, char *addr) +{ + int mask; + int ret; + + addr += (nr >> 3); + mask = BIT(7 - (nr & 0x07)); + ret = mask & *addr; + *addr |= mask; + return ret; +} + +static inline int f2fs_test_and_clear_bit(unsigned int nr, char *addr) +{ + int mask; + int ret; + + addr += (nr >> 3); + mask = BIT(7 - (nr & 0x07)); + ret = mask & *addr; + *addr &= ~mask; + return ret; +} + +static inline void f2fs_change_bit(unsigned int nr, char *addr) +{ + int mask; + + addr += (nr >> 3); + mask = BIT(7 - (nr & 0x07)); + *addr ^= mask; +} + +/* + * On-disk inode flags (f2fs_inode::i_flags) + */ +#define F2FS_COMPR_FL 0x00000004 /* Compress file */ +#define F2FS_SYNC_FL 0x00000008 /* Synchronous updates */ +#define F2FS_IMMUTABLE_FL 0x00000010 /* Immutable file */ +#define F2FS_APPEND_FL 0x00000020 /* writes to file may only append */ +#define F2FS_NODUMP_FL 0x00000040 /* do not dump file */ +#define F2FS_NOATIME_FL 0x00000080 /* do not update atime */ +#define F2FS_NOCOMP_FL 0x00000400 /* Don't compress */ +#define F2FS_INDEX_FL 0x00001000 /* hash-indexed directory */ +#define F2FS_DIRSYNC_FL 0x00010000 /* dirsync behaviour (directories only) */ +#define F2FS_PROJINHERIT_FL 0x20000000 /* Create with parents projid */ +#define F2FS_CASEFOLD_FL 0x40000000 /* Casefolded file */ +#define F2FS_DEVICE_ALIAS_FL 0x80000000 /* File for aliasing a device */ + +#define F2FS_QUOTA_DEFAULT_FL (F2FS_NOATIME_FL | F2FS_IMMUTABLE_FL) + +/* Flags that should be inherited by new inodes from their parent. */ +#define F2FS_FL_INHERITED (F2FS_SYNC_FL | F2FS_NODUMP_FL | F2FS_NOATIME_FL | \ + F2FS_DIRSYNC_FL | F2FS_PROJINHERIT_FL | \ + F2FS_CASEFOLD_FL) + +/* Flags that are appropriate for regular files (all but dir-specific ones). */ +#define F2FS_REG_FLMASK (~(F2FS_DIRSYNC_FL | F2FS_PROJINHERIT_FL | \ + F2FS_CASEFOLD_FL)) + +/* Flags that are appropriate for non-directories/regular files. */ +#define F2FS_OTHER_FLMASK (F2FS_NODUMP_FL | F2FS_NOATIME_FL) + +#define IS_DEVICE_ALIASING(inode) (F2FS_I(inode)->i_flags & F2FS_DEVICE_ALIAS_FL) + +static inline __u32 f2fs_mask_flags(umode_t mode, __u32 flags) +{ + if (S_ISDIR(mode)) + return flags; + else if (S_ISREG(mode)) + return flags & F2FS_REG_FLMASK; + else + return flags & F2FS_OTHER_FLMASK; +} + +static inline void __mark_inode_dirty_flag(struct inode *inode, + int flag, bool set) +{ + switch (flag) { + case FI_INLINE_XATTR: + case FI_INLINE_DATA: + case FI_INLINE_DENTRY: + case FI_NEW_INODE: + if (set) + return; + fallthrough; + case FI_DATA_EXIST: + case FI_PIN_FILE: + case FI_COMPRESS_RELEASED: + f2fs_mark_inode_dirty_sync(inode, true); + } +} + +static inline void set_inode_flag(struct inode *inode, int flag) +{ + set_bit(flag, F2FS_I(inode)->flags); + __mark_inode_dirty_flag(inode, flag, true); +} + +static inline int is_inode_flag_set(struct inode *inode, int flag) +{ + return test_bit(flag, F2FS_I(inode)->flags); +} + +static inline void clear_inode_flag(struct inode *inode, int flag) +{ + clear_bit(flag, F2FS_I(inode)->flags); + __mark_inode_dirty_flag(inode, flag, false); +} + +static inline bool f2fs_verity_in_progress(struct inode *inode) +{ + return IS_ENABLED(CONFIG_FS_VERITY) && + is_inode_flag_set(inode, FI_VERITY_IN_PROGRESS); +} + +static inline void set_acl_inode(struct inode *inode, umode_t mode) +{ + F2FS_I(inode)->i_acl_mode = mode; + set_inode_flag(inode, FI_ACL_MODE); + f2fs_mark_inode_dirty_sync(inode, false); +} + +static inline void f2fs_i_links_write(struct inode *inode, bool inc) +{ + if (inc) + inc_nlink(inode); + else + drop_nlink(inode); + f2fs_mark_inode_dirty_sync(inode, true); +} + +static inline void f2fs_i_blocks_write(struct inode *inode, + block_t diff, bool add, bool claim) +{ + bool clean = !is_inode_flag_set(inode, FI_DIRTY_INODE); + bool recover = is_inode_flag_set(inode, FI_AUTO_RECOVER); + + /* add = 1, claim = 1 should be dquot_reserve_block in pair */ + if (add) { + if (claim) + dquot_claim_block(inode, diff); + else + dquot_alloc_block_nofail(inode, diff); + } else { + dquot_free_block(inode, diff); + } + + f2fs_mark_inode_dirty_sync(inode, true); + if (clean || recover) + set_inode_flag(inode, FI_AUTO_RECOVER); +} + +static inline bool f2fs_is_atomic_file(struct inode *inode); + +static inline void f2fs_i_size_write(struct inode *inode, loff_t i_size) +{ + bool clean = !is_inode_flag_set(inode, FI_DIRTY_INODE); + bool recover = is_inode_flag_set(inode, FI_AUTO_RECOVER); + + if (i_size_read(inode) == i_size) + return; + + i_size_write(inode, i_size); + + if (f2fs_is_atomic_file(inode)) + return; + + f2fs_mark_inode_dirty_sync(inode, true); + if (clean || recover) + set_inode_flag(inode, FI_AUTO_RECOVER); +} + +static inline void f2fs_i_depth_write(struct inode *inode, unsigned int depth) +{ + F2FS_I(inode)->i_current_depth = depth; + f2fs_mark_inode_dirty_sync(inode, true); +} + +static inline void f2fs_i_gc_failures_write(struct inode *inode, + unsigned int count) +{ + F2FS_I(inode)->i_gc_failures = count; + f2fs_mark_inode_dirty_sync(inode, true); +} + +static inline void f2fs_i_xnid_write(struct inode *inode, nid_t xnid) +{ + F2FS_I(inode)->i_xattr_nid = xnid; + f2fs_mark_inode_dirty_sync(inode, true); +} + +static inline void f2fs_i_pino_write(struct inode *inode, nid_t pino) +{ + F2FS_I(inode)->i_pino = pino; + f2fs_mark_inode_dirty_sync(inode, true); +} + +static inline void get_inline_info(struct inode *inode, struct f2fs_inode *ri) +{ + struct f2fs_inode_info *fi = F2FS_I(inode); + + if (ri->i_inline & F2FS_INLINE_XATTR) + set_bit(FI_INLINE_XATTR, fi->flags); + if (ri->i_inline & F2FS_INLINE_DATA) + set_bit(FI_INLINE_DATA, fi->flags); + if (ri->i_inline & F2FS_INLINE_DENTRY) + set_bit(FI_INLINE_DENTRY, fi->flags); + if (ri->i_inline & F2FS_DATA_EXIST) + set_bit(FI_DATA_EXIST, fi->flags); + if (ri->i_inline & F2FS_EXTRA_ATTR) + set_bit(FI_EXTRA_ATTR, fi->flags); + if (ri->i_inline & F2FS_PIN_FILE) + set_bit(FI_PIN_FILE, fi->flags); + if (ri->i_inline & F2FS_COMPRESS_RELEASED) + set_bit(FI_COMPRESS_RELEASED, fi->flags); +} + +static inline void set_raw_inline(struct inode *inode, struct f2fs_inode *ri) +{ + ri->i_inline = 0; + + if (is_inode_flag_set(inode, FI_INLINE_XATTR)) + ri->i_inline |= F2FS_INLINE_XATTR; + if (is_inode_flag_set(inode, FI_INLINE_DATA)) + ri->i_inline |= F2FS_INLINE_DATA; + if (is_inode_flag_set(inode, FI_INLINE_DENTRY)) + ri->i_inline |= F2FS_INLINE_DENTRY; + if (is_inode_flag_set(inode, FI_DATA_EXIST)) + ri->i_inline |= F2FS_DATA_EXIST; + if (is_inode_flag_set(inode, FI_EXTRA_ATTR)) + ri->i_inline |= F2FS_EXTRA_ATTR; + if (is_inode_flag_set(inode, FI_PIN_FILE)) + ri->i_inline |= F2FS_PIN_FILE; + if (is_inode_flag_set(inode, FI_COMPRESS_RELEASED)) + ri->i_inline |= F2FS_COMPRESS_RELEASED; +} + +static inline int f2fs_has_extra_attr(struct inode *inode) +{ + return is_inode_flag_set(inode, FI_EXTRA_ATTR); +} + +static inline int f2fs_has_inline_xattr(struct inode *inode) +{ + return is_inode_flag_set(inode, FI_INLINE_XATTR); +} + +static inline int f2fs_compressed_file(struct inode *inode) +{ + return S_ISREG(inode->i_mode) && + is_inode_flag_set(inode, FI_COMPRESSED_FILE); +} + +static inline bool f2fs_need_compress_data(struct inode *inode) +{ + int compress_mode = F2FS_OPTION(F2FS_I_SB(inode)).compress_mode; + + if (!f2fs_compressed_file(inode)) + return false; + + if (compress_mode == COMPR_MODE_FS) + return true; + else if (compress_mode == COMPR_MODE_USER && + is_inode_flag_set(inode, FI_ENABLE_COMPRESS)) + return true; + + return false; +} + +static inline unsigned int addrs_per_page(struct inode *inode, + bool is_inode) +{ + unsigned int addrs = is_inode ? (CUR_ADDRS_PER_INODE(inode) - + get_inline_xattr_addrs(inode)) : DEF_ADDRS_PER_BLOCK; + + if (f2fs_compressed_file(inode)) + return ALIGN_DOWN(addrs, F2FS_I(inode)->i_cluster_size); + return addrs; +} + +static inline +void *inline_xattr_addr(struct inode *inode, const struct folio *folio) +{ + struct f2fs_inode *ri = F2FS_INODE(folio); + + return (void *)&(ri->i_addr[DEF_ADDRS_PER_INODE - + get_inline_xattr_addrs(inode)]); +} + +static inline int inline_xattr_size(struct inode *inode) +{ + if (f2fs_has_inline_xattr(inode)) + return get_inline_xattr_addrs(inode) * sizeof(__le32); + return 0; +} + +/* + * Notice: check inline_data flag without inode page lock is unsafe. + * It could change at any time by f2fs_convert_inline_folio(). + */ +static inline int f2fs_has_inline_data(struct inode *inode) +{ + return is_inode_flag_set(inode, FI_INLINE_DATA); +} + +static inline int f2fs_exist_data(struct inode *inode) +{ + return is_inode_flag_set(inode, FI_DATA_EXIST); +} + +static inline int f2fs_is_mmap_file(struct inode *inode) +{ + return is_inode_flag_set(inode, FI_MMAP_FILE); +} + +static inline bool f2fs_is_pinned_file(struct inode *inode) +{ + return is_inode_flag_set(inode, FI_PIN_FILE); +} + +static inline bool f2fs_is_atomic_file(struct inode *inode) +{ + return is_inode_flag_set(inode, FI_ATOMIC_FILE); +} + +static inline bool f2fs_is_cow_file(struct inode *inode) +{ + return is_inode_flag_set(inode, FI_COW_FILE); +} + +static inline void *inline_data_addr(struct inode *inode, struct folio *folio) +{ + __le32 *addr = get_dnode_addr(inode, folio); + + return (void *)(addr + DEF_INLINE_RESERVED_SIZE); +} + +static inline int f2fs_has_inline_dentry(struct inode *inode) +{ + return is_inode_flag_set(inode, FI_INLINE_DENTRY); +} + +static inline int is_file(struct inode *inode, int type) +{ + return F2FS_I(inode)->i_advise & type; +} + +static inline void set_file(struct inode *inode, int type) +{ + if (is_file(inode, type)) + return; + F2FS_I(inode)->i_advise |= type; + f2fs_mark_inode_dirty_sync(inode, true); +} + +static inline void clear_file(struct inode *inode, int type) +{ + if (!is_file(inode, type)) + return; + F2FS_I(inode)->i_advise &= ~type; + f2fs_mark_inode_dirty_sync(inode, true); +} + +static inline bool f2fs_is_time_consistent(struct inode *inode) +{ + struct timespec64 ts = inode_get_atime(inode); + + if (!timespec64_equal(F2FS_I(inode)->i_disk_time, &ts)) + return false; + ts = inode_get_ctime(inode); + if (!timespec64_equal(F2FS_I(inode)->i_disk_time + 1, &ts)) + return false; + ts = inode_get_mtime(inode); + if (!timespec64_equal(F2FS_I(inode)->i_disk_time + 2, &ts)) + return false; + return true; +} + +static inline bool f2fs_skip_inode_update(struct inode *inode, int dsync) +{ + bool ret; + + if (dsync) { + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + + spin_lock(&sbi->inode_lock[DIRTY_META]); + ret = list_empty(&F2FS_I(inode)->gdirty_list); + spin_unlock(&sbi->inode_lock[DIRTY_META]); + return ret; + } + if (!is_inode_flag_set(inode, FI_AUTO_RECOVER) || + file_keep_isize(inode) || + i_size_read(inode) & ~PAGE_MASK) + return false; + + if (!f2fs_is_time_consistent(inode)) + return false; + + spin_lock(&F2FS_I(inode)->i_size_lock); + ret = F2FS_I(inode)->last_disk_size == i_size_read(inode); + spin_unlock(&F2FS_I(inode)->i_size_lock); + + return ret; +} + +static inline bool f2fs_readonly(struct super_block *sb) +{ + return sb_rdonly(sb); +} + +static inline bool f2fs_cp_error(struct f2fs_sb_info *sbi) +{ + return is_set_ckpt_flags(sbi, CP_ERROR_FLAG); +} + +static inline void *f2fs_kmalloc(struct f2fs_sb_info *sbi, + size_t size, gfp_t flags) +{ + if (time_to_inject(sbi, FAULT_KMALLOC)) + return NULL; + + return kmalloc(size, flags); +} + +static inline void *f2fs_getname(struct f2fs_sb_info *sbi) +{ + if (time_to_inject(sbi, FAULT_KMALLOC)) + return NULL; + + return __getname(); +} + +static inline void f2fs_putname(char *buf) +{ + __putname(buf); +} + +static inline void *f2fs_kzalloc(struct f2fs_sb_info *sbi, + size_t size, gfp_t flags) +{ + return f2fs_kmalloc(sbi, size, flags | __GFP_ZERO); +} + +static inline void *f2fs_kvmalloc(struct f2fs_sb_info *sbi, + size_t size, gfp_t flags) +{ + if (time_to_inject(sbi, FAULT_KVMALLOC)) + return NULL; + + return kvmalloc(size, flags); +} + +static inline void *f2fs_kvzalloc(struct f2fs_sb_info *sbi, + size_t size, gfp_t flags) +{ + return f2fs_kvmalloc(sbi, size, flags | __GFP_ZERO); +} + +static inline void *f2fs_vmalloc(struct f2fs_sb_info *sbi, size_t size) +{ + if (time_to_inject(sbi, FAULT_VMALLOC)) + return NULL; + + return vmalloc(size); +} + +static inline int get_extra_isize(struct inode *inode) +{ + return F2FS_I(inode)->i_extra_isize / sizeof(__le32); +} + +static inline int get_inline_xattr_addrs(struct inode *inode) +{ + return F2FS_I(inode)->i_inline_xattr_size; +} + +#define f2fs_get_inode_mode(i) \ + ((is_inode_flag_set(i, FI_ACL_MODE)) ? \ + (F2FS_I(i)->i_acl_mode) : ((i)->i_mode)) + +#define F2FS_MIN_EXTRA_ATTR_SIZE (sizeof(__le32)) + +#define F2FS_TOTAL_EXTRA_ATTR_SIZE \ + (offsetof(struct f2fs_inode, i_extra_end) - \ + offsetof(struct f2fs_inode, i_extra_isize)) \ + +#define F2FS_OLD_ATTRIBUTE_SIZE (offsetof(struct f2fs_inode, i_addr)) +#define F2FS_FITS_IN_INODE(f2fs_inode, extra_isize, field) \ + ((offsetof(typeof(*(f2fs_inode)), field) + \ + sizeof((f2fs_inode)->field)) \ + <= (F2FS_OLD_ATTRIBUTE_SIZE + (extra_isize))) \ + +#define __is_large_section(sbi) (SEGS_PER_SEC(sbi) > 1) + +#define __is_meta_io(fio) (PAGE_TYPE_OF_BIO((fio)->type) == META) + +bool f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi, + block_t blkaddr, int type); +static inline void verify_blkaddr(struct f2fs_sb_info *sbi, + block_t blkaddr, int type) +{ + if (!f2fs_is_valid_blkaddr(sbi, blkaddr, type)) + f2fs_err(sbi, "invalid blkaddr: %u, type: %d, run fsck to fix.", + blkaddr, type); +} + +static inline bool __is_valid_data_blkaddr(block_t blkaddr) +{ + if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR || + blkaddr == COMPRESS_ADDR) + return false; + return true; +} + +/* + * file.c + */ +int f2fs_sync_file(struct file *file, loff_t start, loff_t end, int datasync); +int f2fs_do_truncate_blocks(struct inode *inode, u64 from, bool lock); +int f2fs_truncate_blocks(struct inode *inode, u64 from, bool lock); +int f2fs_truncate(struct inode *inode); +int f2fs_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags); +int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr); +int f2fs_truncate_hole(struct inode *inode, pgoff_t pg_start, pgoff_t pg_end); +void f2fs_truncate_data_blocks_range(struct dnode_of_data *dn, int count); +int f2fs_do_shutdown(struct f2fs_sb_info *sbi, unsigned int flag, + bool readonly, bool need_lock); +int f2fs_precache_extents(struct inode *inode); +int f2fs_fileattr_get(struct dentry *dentry, struct file_kattr *fa); +int f2fs_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa); +long f2fs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); +long f2fs_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg); +int f2fs_transfer_project_quota(struct inode *inode, kprojid_t kprojid); +int f2fs_pin_file_control(struct inode *inode, bool inc); + +/* + * inode.c + */ +void f2fs_set_inode_flags(struct inode *inode); +bool f2fs_inode_chksum_verify(struct f2fs_sb_info *sbi, struct folio *folio); +void f2fs_inode_chksum_set(struct f2fs_sb_info *sbi, struct folio *folio); +struct inode *f2fs_iget(struct super_block *sb, unsigned long ino); +struct inode *f2fs_iget_retry(struct super_block *sb, unsigned long ino); +int f2fs_try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink); +void f2fs_update_inode(struct inode *inode, struct folio *node_folio); +void f2fs_update_inode_page(struct inode *inode); +int f2fs_write_inode(struct inode *inode, struct writeback_control *wbc); +void f2fs_remove_donate_inode(struct inode *inode); +void f2fs_evict_inode(struct inode *inode); +void f2fs_handle_failed_inode(struct inode *inode, struct f2fs_lock_context *lc); + +/* + * namei.c + */ +int f2fs_update_extension_list(struct f2fs_sb_info *sbi, const char *name, + bool hot, bool set); +struct dentry *f2fs_get_parent(struct dentry *child); +int f2fs_get_tmpfile(struct mnt_idmap *idmap, struct inode *dir, + struct inode **new_inode); + +/* + * dir.c + */ +#if IS_ENABLED(CONFIG_UNICODE) +int f2fs_init_casefolded_name(const struct inode *dir, + struct f2fs_filename *fname); +void f2fs_free_casefolded_name(struct f2fs_filename *fname); +#else +static inline int f2fs_init_casefolded_name(const struct inode *dir, + struct f2fs_filename *fname) +{ + return 0; +} + +static inline void f2fs_free_casefolded_name(struct f2fs_filename *fname) +{ +} +#endif /* CONFIG_UNICODE */ + +int f2fs_setup_filename(struct inode *dir, const struct qstr *iname, + int lookup, struct f2fs_filename *fname); +int f2fs_prepare_lookup(struct inode *dir, struct dentry *dentry, + struct f2fs_filename *fname); +void f2fs_free_filename(struct f2fs_filename *fname); +struct f2fs_dir_entry *f2fs_find_target_dentry(const struct f2fs_dentry_ptr *d, + const struct f2fs_filename *fname, int *max_slots, + bool use_hash); +int f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d, + unsigned int start_pos, struct fscrypt_str *fstr); +void f2fs_do_make_empty_dir(struct inode *inode, struct inode *parent, + struct f2fs_dentry_ptr *d); +struct folio *f2fs_init_inode_metadata(struct inode *inode, struct inode *dir, + const struct f2fs_filename *fname, struct folio *dfolio); +void f2fs_update_parent_metadata(struct inode *dir, struct inode *inode, + unsigned int current_depth); +int f2fs_room_for_filename(const void *bitmap, int slots, int max_slots); +void f2fs_drop_nlink(struct inode *dir, struct inode *inode); +struct f2fs_dir_entry *__f2fs_find_entry(struct inode *dir, + const struct f2fs_filename *fname, struct folio **res_folio); +struct f2fs_dir_entry *f2fs_find_entry(struct inode *dir, + const struct qstr *child, struct folio **res_folio); +struct f2fs_dir_entry *f2fs_parent_dir(struct inode *dir, struct folio **f); +ino_t f2fs_inode_by_name(struct inode *dir, const struct qstr *qstr, + struct folio **folio); +void f2fs_set_link(struct inode *dir, struct f2fs_dir_entry *de, + struct folio *folio, struct inode *inode); +bool f2fs_has_enough_room(struct inode *dir, struct folio *ifolio, + const struct f2fs_filename *fname); +void f2fs_update_dentry(nid_t ino, umode_t mode, struct f2fs_dentry_ptr *d, + const struct fscrypt_str *name, f2fs_hash_t name_hash, + unsigned int bit_pos); +int f2fs_add_regular_entry(struct inode *dir, const struct f2fs_filename *fname, + struct inode *inode, nid_t ino, umode_t mode); +int f2fs_add_dentry(struct inode *dir, const struct f2fs_filename *fname, + struct inode *inode, nid_t ino, umode_t mode); +int f2fs_do_add_link(struct inode *dir, const struct qstr *name, + struct inode *inode, nid_t ino, umode_t mode); +void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct folio *folio, + struct inode *dir, struct inode *inode); +int f2fs_do_tmpfile(struct inode *inode, struct inode *dir, + struct f2fs_filename *fname); +bool f2fs_empty_dir(struct inode *dir); + +static inline int f2fs_add_link(struct dentry *dentry, struct inode *inode) +{ + if (fscrypt_is_nokey_name(dentry)) + return -ENOKEY; + return f2fs_do_add_link(d_inode(dentry->d_parent), &dentry->d_name, + inode, inode->i_ino, inode->i_mode); +} + +/* + * super.c + */ +int f2fs_inode_dirtied(struct inode *inode, bool sync); +void f2fs_inode_synced(struct inode *inode); +int f2fs_dquot_initialize(struct inode *inode); +int f2fs_enable_quota_files(struct f2fs_sb_info *sbi, bool rdonly); +int f2fs_do_quota_sync(struct super_block *sb, int type); +loff_t max_file_blocks(struct inode *inode); +void f2fs_quota_off_umount(struct super_block *sb); +void f2fs_save_errors(struct f2fs_sb_info *sbi, unsigned char flag); +void f2fs_handle_error(struct f2fs_sb_info *sbi, unsigned char error); +int f2fs_commit_super(struct f2fs_sb_info *sbi, bool recover); +int f2fs_sync_fs(struct super_block *sb, int sync); +int f2fs_sanity_check_ckpt(struct f2fs_sb_info *sbi); + +/* + * hash.c + */ +void f2fs_hash_filename(const struct inode *dir, struct f2fs_filename *fname); + +/* + * node.c + */ +struct node_info; +enum node_type; + +int f2fs_check_nid_range(struct f2fs_sb_info *sbi, nid_t nid); +bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type); +bool f2fs_in_warm_node_list(struct folio *folio); +void f2fs_init_fsync_node_info(struct f2fs_sb_info *sbi); +void f2fs_del_fsync_node_entry(struct f2fs_sb_info *sbi, struct folio *folio); +void f2fs_reset_fsync_node_info(struct f2fs_sb_info *sbi); +bool f2fs_need_dentry_mark(struct f2fs_sb_info *sbi, nid_t nid); +bool f2fs_is_checkpointed_node(struct f2fs_sb_info *sbi, nid_t nid); +bool f2fs_need_inode_block_update(struct f2fs_sb_info *sbi, nid_t ino); +int f2fs_get_node_info(struct f2fs_sb_info *sbi, nid_t nid, + struct node_info *ni, bool checkpoint_context); +pgoff_t f2fs_get_next_page_offset(struct dnode_of_data *dn, pgoff_t pgofs); +int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode); +int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from); +int f2fs_truncate_xattr_node(struct inode *inode); +int f2fs_wait_on_node_pages_writeback(struct f2fs_sb_info *sbi, + unsigned int seq_id); +int f2fs_remove_inode_page(struct inode *inode); +struct folio *f2fs_new_inode_folio(struct inode *inode); +struct folio *f2fs_new_node_folio(struct dnode_of_data *dn, unsigned int ofs); +void f2fs_ra_node_page(struct f2fs_sb_info *sbi, nid_t nid); +struct folio *f2fs_get_node_folio(struct f2fs_sb_info *sbi, pgoff_t nid, + enum node_type node_type); +int f2fs_sanity_check_node_footer(struct f2fs_sb_info *sbi, + struct folio *folio, pgoff_t nid, + enum node_type ntype, bool in_irq); +struct folio *f2fs_get_inode_folio(struct f2fs_sb_info *sbi, pgoff_t ino); +struct folio *f2fs_get_xnode_folio(struct f2fs_sb_info *sbi, pgoff_t xnid); +int f2fs_write_single_node_folio(struct folio *node_folio, int sync_mode, + bool mark_dirty, enum iostat_type io_type); +int f2fs_move_node_folio(struct folio *node_folio, int gc_type); +void f2fs_flush_inline_data(struct f2fs_sb_info *sbi); +int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode, + struct writeback_control *wbc, bool atomic, + unsigned int *seq_id); +int f2fs_sync_node_pages(struct f2fs_sb_info *sbi, + struct writeback_control *wbc, + bool do_balance, enum iostat_type io_type); +int f2fs_build_free_nids(struct f2fs_sb_info *sbi, bool sync, bool mount); +bool f2fs_alloc_nid(struct f2fs_sb_info *sbi, nid_t *nid); +void f2fs_alloc_nid_done(struct f2fs_sb_info *sbi, nid_t nid); +void f2fs_alloc_nid_failed(struct f2fs_sb_info *sbi, nid_t nid); +int f2fs_try_to_free_nids(struct f2fs_sb_info *sbi, int nr_shrink); +int f2fs_recover_inline_xattr(struct inode *inode, struct folio *folio); +int f2fs_recover_xattr_data(struct inode *inode, struct folio *folio); +int f2fs_recover_inode_page(struct f2fs_sb_info *sbi, struct folio *folio); +int f2fs_restore_node_summary(struct f2fs_sb_info *sbi, + unsigned int segno, struct f2fs_summary_block *sum); +int f2fs_flush_nat_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc); +int f2fs_build_node_manager(struct f2fs_sb_info *sbi); +void f2fs_destroy_node_manager(struct f2fs_sb_info *sbi); +int __init f2fs_create_node_manager_caches(void); +void f2fs_destroy_node_manager_caches(void); + +/* + * segment.c + */ +bool f2fs_need_SSR(struct f2fs_sb_info *sbi); +int f2fs_commit_atomic_write(struct inode *inode); +void f2fs_abort_atomic_write(struct inode *inode, bool clean); +void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need); +void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg); +int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino); +int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi); +int f2fs_flush_device_cache(struct f2fs_sb_info *sbi); +void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free); +void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr, + unsigned int len); +bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr); +int f2fs_start_discard_thread(struct f2fs_sb_info *sbi); +void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi); +void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi); +bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi, bool need_check); +void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi, + struct cp_control *cpc); +void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi); +block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi); +int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable); +void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi); +int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra); +bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno); +int f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi); +int f2fs_reinit_atgc_curseg(struct f2fs_sb_info *sbi); +void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi); +void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi); +int f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type, + unsigned int start, unsigned int end); +int f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force); +int f2fs_allocate_pinning_section(struct f2fs_sb_info *sbi); +int f2fs_allocate_new_segments(struct f2fs_sb_info *sbi); +int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range); +bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi, + struct cp_control *cpc); +struct folio *f2fs_get_sum_folio(struct f2fs_sb_info *sbi, unsigned int segno); +void f2fs_update_meta_page(struct f2fs_sb_info *sbi, void *src, + block_t blk_addr); +void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct folio *folio, + enum iostat_type io_type); +void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio); +void f2fs_outplace_write_data(struct dnode_of_data *dn, + struct f2fs_io_info *fio); +int f2fs_inplace_write_data(struct f2fs_io_info *fio); +void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, + block_t old_blkaddr, block_t new_blkaddr, + bool recover_curseg, bool recover_newaddr, + bool from_gc); +void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn, + block_t old_addr, block_t new_addr, + unsigned char version, bool recover_curseg, + bool recover_newaddr); +enum temp_type f2fs_get_segment_temp(struct f2fs_sb_info *sbi, + enum log_type seg_type); +int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct folio *folio, + block_t old_blkaddr, block_t *new_blkaddr, + struct f2fs_summary *sum, int type, + struct f2fs_io_info *fio); +void f2fs_update_device_state(struct f2fs_sb_info *sbi, nid_t ino, + block_t blkaddr, unsigned int blkcnt); +void f2fs_folio_wait_writeback(struct folio *folio, enum page_type type, + bool ordered, bool locked); +#define f2fs_wait_on_page_writeback(page, type, ordered, locked) \ + f2fs_folio_wait_writeback(page_folio(page), type, ordered, locked) +void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr); +void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr, + block_t len); +void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk); +void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk); +int f2fs_lookup_journal_in_cursum(struct f2fs_sb_info *sbi, + struct f2fs_journal *journal, int type, + unsigned int val, int alloc); +void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc); +int f2fs_check_and_fix_write_pointer(struct f2fs_sb_info *sbi); +int f2fs_build_segment_manager(struct f2fs_sb_info *sbi); +void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi); +int __init f2fs_create_segment_manager_caches(void); +void f2fs_destroy_segment_manager_caches(void); +int f2fs_rw_hint_to_seg_type(struct f2fs_sb_info *sbi, enum rw_hint hint); +enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi, + enum page_type type, enum temp_type temp); +u8 f2fs_io_type_to_write_stream(struct block_device *bdev, + enum page_type type, enum temp_type temp); +unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi); +unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi, + unsigned int segno); +unsigned long long f2fs_get_section_mtime(struct f2fs_sb_info *sbi, + unsigned int segno); + +static inline struct inode *fio_inode(struct f2fs_io_info *fio) +{ + return fio->folio->mapping->host; +} + +#define DEF_FRAGMENT_SIZE 4 +#define MIN_FRAGMENT_SIZE 1 +#define MAX_FRAGMENT_SIZE 512 + +static inline bool f2fs_need_rand_seg(struct f2fs_sb_info *sbi) +{ + return F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_SEG || + F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK; +} + +/* + * checkpoint.c + */ +void f2fs_lock_op(struct f2fs_sb_info *sbi, struct f2fs_lock_context *lc); +int f2fs_trylock_op(struct f2fs_sb_info *sbi, struct f2fs_lock_context *lc); +void f2fs_unlock_op(struct f2fs_sb_info *sbi, struct f2fs_lock_context *lc); +void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io, + unsigned char reason); +void f2fs_flush_ckpt_thread(struct f2fs_sb_info *sbi); +struct folio *f2fs_grab_meta_folio(struct f2fs_sb_info *sbi, pgoff_t index); +struct folio *f2fs_get_meta_folio(struct f2fs_sb_info *sbi, pgoff_t index); +struct folio *f2fs_get_meta_folio_retry(struct f2fs_sb_info *sbi, pgoff_t index); +struct folio *f2fs_get_tmp_folio(struct f2fs_sb_info *sbi, pgoff_t index); +bool f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi, + block_t blkaddr, int type); +bool f2fs_is_valid_blkaddr_raw(struct f2fs_sb_info *sbi, + block_t blkaddr, int type); +int f2fs_ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages, + int type, bool sync); +void f2fs_ra_meta_pages_cond(struct f2fs_sb_info *sbi, pgoff_t index, + unsigned int ra_blocks); +long f2fs_sync_meta_pages(struct f2fs_sb_info *sbi, long nr_to_write, + enum iostat_type io_type); +void f2fs_add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type); +void f2fs_remove_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type); +void f2fs_release_ino_entry(struct f2fs_sb_info *sbi, bool all); +bool f2fs_exist_written_data(struct f2fs_sb_info *sbi, nid_t ino, int mode); +void f2fs_set_dirty_device(struct f2fs_sb_info *sbi, nid_t ino, + unsigned int devidx, int type); +bool f2fs_is_dirty_device(struct f2fs_sb_info *sbi, nid_t ino, + unsigned int devidx, int type); +int f2fs_acquire_orphan_inode(struct f2fs_sb_info *sbi); +void f2fs_release_orphan_inode(struct f2fs_sb_info *sbi); +void f2fs_add_orphan_inode(struct inode *inode); +void f2fs_remove_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino); +int f2fs_recover_orphan_inodes(struct f2fs_sb_info *sbi); +int f2fs_get_valid_checkpoint(struct f2fs_sb_info *sbi); +void f2fs_update_dirty_folio(struct inode *inode, struct folio *folio); +void f2fs_remove_dirty_inode(struct inode *inode); +int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type, + bool from_cp); +void f2fs_wait_on_all_pages(struct f2fs_sb_info *sbi, int type); +u64 f2fs_get_sectors_written(struct f2fs_sb_info *sbi); +int f2fs_write_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc); +void f2fs_init_ino_entry_info(struct f2fs_sb_info *sbi); +int __init f2fs_create_checkpoint_caches(void); +void f2fs_destroy_checkpoint_caches(void); +int f2fs_issue_checkpoint(struct f2fs_sb_info *sbi); +int f2fs_start_ckpt_thread(struct f2fs_sb_info *sbi); +void f2fs_stop_ckpt_thread(struct f2fs_sb_info *sbi); +void f2fs_init_ckpt_req_control(struct f2fs_sb_info *sbi); + +/* + * data.c + */ +int __init f2fs_init_bioset(void); +void f2fs_destroy_bioset(void); +bool f2fs_is_cp_guaranteed(const struct folio *folio); +int f2fs_init_bio_entry_cache(void); +void f2fs_destroy_bio_entry_cache(void); +void f2fs_submit_read_bio(struct f2fs_sb_info *sbi, struct bio *bio, + enum page_type type); +int f2fs_init_write_merge_io(struct f2fs_sb_info *sbi); +void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type); +void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi, + struct inode *inode, struct folio *folio, + nid_t ino, enum page_type type); +void f2fs_submit_merged_write_folio(struct f2fs_sb_info *sbi, + struct folio *folio, enum page_type type); +void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi, + struct bio **bio, struct folio *folio); +void f2fs_submit_all_merged_ipu_writes(struct f2fs_sb_info *sbi); +void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi); +int f2fs_submit_page_bio(struct f2fs_io_info *fio); +int f2fs_merge_page_bio(struct f2fs_io_info *fio); +void f2fs_submit_page_write(struct f2fs_io_info *fio); +struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi, + block_t blk_addr, sector_t *sector); +int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr); +void f2fs_set_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr); +void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr); +int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count); +int f2fs_reserve_new_block(struct dnode_of_data *dn); +int f2fs_get_block_locked(struct dnode_of_data *dn, pgoff_t index); +int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index); +struct folio *f2fs_get_read_data_folio(struct inode *inode, pgoff_t index, + blk_opf_t op_flags, bool for_write, pgoff_t *next_pgofs); +struct folio *f2fs_find_data_folio(struct inode *inode, pgoff_t index, + pgoff_t *next_pgofs); +struct folio *f2fs_get_lock_data_folio(struct inode *inode, pgoff_t index, + bool for_write); +struct folio *f2fs_get_new_data_folio(struct inode *inode, + struct folio *ifolio, pgoff_t index, bool new_i_size); +int f2fs_do_write_data_page(struct f2fs_io_info *fio); +int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag); +int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + u64 start, u64 len); +bool f2fs_should_update_inplace(struct inode *inode, struct f2fs_io_info *fio); +bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio); +int f2fs_write_single_data_page(struct folio *folio, int *submitted, + struct bio **bio, sector_t *last_block, + struct writeback_control *wbc, + enum iostat_type io_type, + int compr_blocks, bool allow_balance); +void f2fs_write_failed(struct inode *inode, loff_t to); +void f2fs_invalidate_folio(struct folio *folio, size_t offset, size_t length); +bool f2fs_release_folio(struct folio *folio, gfp_t wait); +bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len); +void f2fs_clear_page_cache_dirty_tag(struct folio *folio); +int f2fs_init_post_read_processing(void); +void f2fs_destroy_post_read_processing(void); +int f2fs_init_wq(struct f2fs_sb_info *sbi); +void f2fs_destroy_wq(struct f2fs_sb_info *sbi); +extern const struct iomap_ops f2fs_iomap_ops; + +/* + * gc.c + */ +int f2fs_start_gc_thread(struct f2fs_sb_info *sbi); +void f2fs_stop_gc_thread(struct f2fs_sb_info *sbi); +block_t f2fs_start_bidx_of_node(unsigned int node_ofs, struct inode *inode); +int f2fs_gc(struct f2fs_sb_info *sbi, struct f2fs_gc_control *gc_control); +void f2fs_build_gc_manager(struct f2fs_sb_info *sbi); +int f2fs_gc_range(struct f2fs_sb_info *sbi, + unsigned int start_seg, unsigned int end_seg, + bool dry_run, unsigned int dry_run_sections); +int f2fs_resize_fs(struct file *filp, __u64 block_count); +int __init f2fs_create_garbage_collection_cache(void); +void f2fs_destroy_garbage_collection_cache(void); +/* victim selection function for cleaning and SSR */ +int f2fs_get_victim(struct f2fs_sb_info *sbi, unsigned int *result, + int gc_type, int type, char alloc_mode, + unsigned long long age, bool one_time); + +/* + * recovery.c + */ +int f2fs_recover_fsync_data(struct f2fs_sb_info *sbi, bool check_only); +bool f2fs_space_for_roll_forward(struct f2fs_sb_info *sbi); +int __init f2fs_create_recovery_cache(void); +void f2fs_destroy_recovery_cache(void); + +/* + * debug.c + */ +#ifdef CONFIG_F2FS_STAT_FS +enum { + DEVSTAT_INUSE, + DEVSTAT_DIRTY, + DEVSTAT_FULL, + DEVSTAT_FREE, + DEVSTAT_PREFREE, + DEVSTAT_MAX, +}; + +struct f2fs_dev_stats { + unsigned int devstats[2][DEVSTAT_MAX]; /* 0: segs, 1: secs */ +}; + +struct f2fs_stat_info { + struct list_head stat_list; + struct f2fs_sb_info *sbi; + int all_area_segs, sit_area_segs, nat_area_segs, ssa_area_segs; + int main_area_segs, main_area_sections, main_area_zones; + unsigned long long hit_cached[NR_EXTENT_CACHES]; + unsigned long long hit_rbtree[NR_EXTENT_CACHES]; + unsigned long long total_ext[NR_EXTENT_CACHES]; + unsigned long long hit_total[NR_EXTENT_CACHES]; + int ext_tree[NR_EXTENT_CACHES]; + int zombie_tree[NR_EXTENT_CACHES]; + int ext_node[NR_EXTENT_CACHES]; + /* to count memory footprint */ + unsigned long long ext_mem[NR_EXTENT_CACHES]; + /* for read extent cache */ + unsigned long long hit_largest; + /* for block age extent cache */ + unsigned long long allocated_data_blocks; + int ndirty_node, ndirty_dent, ndirty_meta, ndirty_imeta; + int ndirty_data, ndirty_qdata; + unsigned int ndirty_dirs, ndirty_files, ndirty_all; + unsigned int nquota_files, ndonate_files; + int nats, dirty_nats, sits, dirty_sits; + int free_nids, avail_nids, alloc_nids; + int total_count, utilization; + int nr_wb_cp_data, nr_wb_data; + int nr_rd_data, nr_rd_node, nr_rd_meta; + int nr_dio_read, nr_dio_write; + unsigned int io_skip_bggc, other_skip_bggc; + int nr_flushing, nr_flushed, flush_list_empty; + int nr_discarding, nr_discarded; + int nr_discard_cmd; + unsigned int undiscard_blks; + int nr_issued_ckpt, nr_total_ckpt, nr_queued_ckpt; + unsigned int cur_ckpt_time, peak_ckpt_time; + int inline_xattr, inline_inode, inline_dir, append, update, orphans; + int compr_inode, swapfile_inode; + unsigned long long compr_blocks; + int aw_cnt, max_aw_cnt; + unsigned int valid_count, valid_node_count, valid_inode_count, discard_blks; + unsigned int bimodal, avg_vblocks; + int util_free, util_valid, util_invalid; + int rsvd_segs, overp_segs; + int dirty_count, node_pages, meta_pages, compress_pages; + int compress_page_hit; + int prefree_count, free_segs, free_secs; + int cp_call_count[MAX_CALL_TYPE], cp_count; + int gc_call_count[MAX_CALL_TYPE]; + int gc_segs[2][2]; + int gc_secs[2][2]; + int tot_blks, data_blks, node_blks; + int bg_data_blks, bg_node_blks; + unsigned int defrag_blks; + int blkoff[NR_CURSEG_TYPE]; + int curseg[NR_CURSEG_TYPE]; + int cursec[NR_CURSEG_TYPE]; + int curzone[NR_CURSEG_TYPE]; + unsigned int dirty_seg[NR_CURSEG_TYPE]; + unsigned int full_seg[NR_CURSEG_TYPE]; + unsigned int valid_blks[NR_CURSEG_TYPE]; + + unsigned int meta_count[META_MAX]; + unsigned int segment_count[2]; + unsigned int block_count[2]; + unsigned int inplace_count; + unsigned long long base_mem, cache_mem, page_mem; + struct f2fs_dev_stats *dev_stats; +}; + +static inline struct f2fs_stat_info *F2FS_STAT(struct f2fs_sb_info *sbi) +{ + return (struct f2fs_stat_info *)sbi->stat_info; +} + +#define stat_inc_cp_call_count(sbi, foreground) \ + atomic_inc(&sbi->cp_call_count[(foreground)]) +#define stat_inc_cp_count(sbi) (F2FS_STAT(sbi)->cp_count++) +#define stat_io_skip_bggc_count(sbi) ((sbi)->io_skip_bggc++) +#define stat_other_skip_bggc_count(sbi) ((sbi)->other_skip_bggc++) +#define stat_inc_dirty_inode(sbi, type) ((sbi)->ndirty_inode[type]++) +#define stat_dec_dirty_inode(sbi, type) ((sbi)->ndirty_inode[type]--) +#define stat_inc_total_hit(sbi, type) (atomic64_inc(&(sbi)->total_hit_ext[type])) +#define stat_inc_rbtree_node_hit(sbi, type) (atomic64_inc(&(sbi)->read_hit_rbtree[type])) +#define stat_inc_largest_node_hit(sbi) (atomic64_inc(&(sbi)->read_hit_largest)) +#define stat_inc_cached_node_hit(sbi, type) (atomic64_inc(&(sbi)->read_hit_cached[type])) +#define stat_inc_inline_xattr(inode) \ + do { \ + if (f2fs_has_inline_xattr(inode)) \ + (atomic_inc(&F2FS_I_SB(inode)->inline_xattr)); \ + } while (0) +#define stat_dec_inline_xattr(inode) \ + do { \ + if (f2fs_has_inline_xattr(inode)) \ + (atomic_dec(&F2FS_I_SB(inode)->inline_xattr)); \ + } while (0) +#define stat_inc_inline_inode(inode) \ + do { \ + if (f2fs_has_inline_data(inode)) \ + (atomic_inc(&F2FS_I_SB(inode)->inline_inode)); \ + } while (0) +#define stat_dec_inline_inode(inode) \ + do { \ + if (f2fs_has_inline_data(inode)) \ + (atomic_dec(&F2FS_I_SB(inode)->inline_inode)); \ + } while (0) +#define stat_inc_inline_dir(inode) \ + do { \ + if (f2fs_has_inline_dentry(inode)) \ + (atomic_inc(&F2FS_I_SB(inode)->inline_dir)); \ + } while (0) +#define stat_dec_inline_dir(inode) \ + do { \ + if (f2fs_has_inline_dentry(inode)) \ + (atomic_dec(&F2FS_I_SB(inode)->inline_dir)); \ + } while (0) +#define stat_inc_compr_inode(inode) \ + do { \ + if (f2fs_compressed_file(inode)) \ + (atomic_inc(&F2FS_I_SB(inode)->compr_inode)); \ + } while (0) +#define stat_dec_compr_inode(inode) \ + do { \ + if (f2fs_compressed_file(inode)) \ + (atomic_dec(&F2FS_I_SB(inode)->compr_inode)); \ + } while (0) +#define stat_add_compr_blocks(inode, blocks) \ + (atomic64_add(blocks, &F2FS_I_SB(inode)->compr_blocks)) +#define stat_sub_compr_blocks(inode, blocks) \ + (atomic64_sub(blocks, &F2FS_I_SB(inode)->compr_blocks)) +#define stat_inc_swapfile_inode(inode) \ + (atomic_inc(&F2FS_I_SB(inode)->swapfile_inode)) +#define stat_dec_swapfile_inode(inode) \ + (atomic_dec(&F2FS_I_SB(inode)->swapfile_inode)) +#define stat_inc_atomic_inode(inode) \ + (atomic_inc(&F2FS_I_SB(inode)->atomic_files)) +#define stat_dec_atomic_inode(inode) \ + (atomic_dec(&F2FS_I_SB(inode)->atomic_files)) +#define stat_inc_meta_count(sbi, blkaddr) \ + do { \ + if (blkaddr < SIT_I(sbi)->sit_base_addr) \ + atomic_inc(&(sbi)->meta_count[META_CP]); \ + else if (blkaddr < NM_I(sbi)->nat_blkaddr) \ + atomic_inc(&(sbi)->meta_count[META_SIT]); \ + else if (blkaddr < SM_I(sbi)->ssa_blkaddr) \ + atomic_inc(&(sbi)->meta_count[META_NAT]); \ + else if (blkaddr < SM_I(sbi)->main_blkaddr) \ + atomic_inc(&(sbi)->meta_count[META_SSA]); \ + } while (0) +#define stat_inc_seg_type(sbi, curseg) \ + ((sbi)->segment_count[(curseg)->alloc_type]++) +#define stat_inc_block_count(sbi, curseg) \ + ((sbi)->block_count[(curseg)->alloc_type]++) +#define stat_inc_inplace_blocks(sbi) \ + (atomic_inc(&(sbi)->inplace_count)) +#define stat_update_max_atomic_write(inode) \ + do { \ + int cur = atomic_read(&F2FS_I_SB(inode)->atomic_files); \ + int max = atomic_read(&F2FS_I_SB(inode)->max_aw_cnt); \ + if (cur > max) \ + atomic_set(&F2FS_I_SB(inode)->max_aw_cnt, cur); \ + } while (0) +#define stat_inc_gc_call_count(sbi, foreground) \ + (F2FS_STAT(sbi)->gc_call_count[(foreground)]++) +#define stat_inc_gc_sec_count(sbi, type, gc_type) \ + (F2FS_STAT(sbi)->gc_secs[(type)][(gc_type)]++) +#define stat_inc_gc_seg_count(sbi, type, gc_type) \ + (F2FS_STAT(sbi)->gc_segs[(type)][(gc_type)]++) + +#define stat_inc_tot_blk_count(si, blks) \ + ((si)->tot_blks += (blks)) + +#define stat_inc_data_blk_count(sbi, blks, gc_type) \ + do { \ + struct f2fs_stat_info *si = F2FS_STAT(sbi); \ + stat_inc_tot_blk_count(si, blks); \ + si->data_blks += (blks); \ + si->bg_data_blks += ((gc_type) == BG_GC) ? (blks) : 0; \ + } while (0) + +#define stat_inc_node_blk_count(sbi, blks, gc_type) \ + do { \ + struct f2fs_stat_info *si = F2FS_STAT(sbi); \ + stat_inc_tot_blk_count(si, blks); \ + si->node_blks += (blks); \ + si->bg_node_blks += ((gc_type) == BG_GC) ? (blks) : 0; \ + } while (0) + +#define stat_inc_defrag_blk_count(sbi, blks) \ + (F2FS_STAT(sbi)->defrag_blks += (blks)) + +int f2fs_build_stats(struct f2fs_sb_info *sbi); +void f2fs_destroy_stats(struct f2fs_sb_info *sbi); +void __init f2fs_create_root_stats(void); +void f2fs_destroy_root_stats(void); +void f2fs_update_sit_info(struct f2fs_sb_info *sbi); +#else +#define stat_inc_cp_call_count(sbi, foreground) do { } while (0) +#define stat_inc_cp_count(sbi) do { } while (0) +#define stat_io_skip_bggc_count(sbi) do { } while (0) +#define stat_other_skip_bggc_count(sbi) do { } while (0) +#define stat_inc_dirty_inode(sbi, type) do { } while (0) +#define stat_dec_dirty_inode(sbi, type) do { } while (0) +#define stat_inc_total_hit(sbi, type) do { } while (0) +#define stat_inc_rbtree_node_hit(sbi, type) do { } while (0) +#define stat_inc_largest_node_hit(sbi) do { } while (0) +#define stat_inc_cached_node_hit(sbi, type) do { } while (0) +#define stat_inc_inline_xattr(inode) do { } while (0) +#define stat_dec_inline_xattr(inode) do { } while (0) +#define stat_inc_inline_inode(inode) do { } while (0) +#define stat_dec_inline_inode(inode) do { } while (0) +#define stat_inc_inline_dir(inode) do { } while (0) +#define stat_dec_inline_dir(inode) do { } while (0) +#define stat_inc_compr_inode(inode) do { } while (0) +#define stat_dec_compr_inode(inode) do { } while (0) +#define stat_add_compr_blocks(inode, blocks) do { } while (0) +#define stat_sub_compr_blocks(inode, blocks) do { } while (0) +#define stat_inc_swapfile_inode(inode) do { } while (0) +#define stat_dec_swapfile_inode(inode) do { } while (0) +#define stat_inc_atomic_inode(inode) do { } while (0) +#define stat_dec_atomic_inode(inode) do { } while (0) +#define stat_update_max_atomic_write(inode) do { } while (0) +#define stat_inc_meta_count(sbi, blkaddr) do { } while (0) +#define stat_inc_seg_type(sbi, curseg) do { } while (0) +#define stat_inc_block_count(sbi, curseg) do { } while (0) +#define stat_inc_inplace_blocks(sbi) do { } while (0) +#define stat_inc_gc_call_count(sbi, foreground) do { } while (0) +#define stat_inc_gc_sec_count(sbi, type, gc_type) do { } while (0) +#define stat_inc_gc_seg_count(sbi, type, gc_type) do { } while (0) +#define stat_inc_tot_blk_count(si, blks) do { } while (0) +#define stat_inc_data_blk_count(sbi, blks, gc_type) do { } while (0) +#define stat_inc_node_blk_count(sbi, blks, gc_type) do { } while (0) +#define stat_inc_defrag_blk_count(sbi, blks) do { } while (0) + +static inline int f2fs_build_stats(struct f2fs_sb_info *sbi) { return 0; } +static inline void f2fs_destroy_stats(struct f2fs_sb_info *sbi) { } +static inline void __init f2fs_create_root_stats(void) { } +static inline void f2fs_destroy_root_stats(void) { } +static inline void f2fs_update_sit_info(struct f2fs_sb_info *sbi) {} +#endif + +extern const struct file_operations f2fs_dir_operations; +extern const struct file_operations f2fs_file_operations; +extern const struct inode_operations f2fs_file_inode_operations; +extern const struct address_space_operations f2fs_dblock_aops; +extern const struct address_space_operations f2fs_node_aops; +extern const struct address_space_operations f2fs_meta_aops; +extern const struct inode_operations f2fs_dir_inode_operations; +extern const struct inode_operations f2fs_symlink_inode_operations; +extern const struct inode_operations f2fs_encrypted_symlink_inode_operations; +extern const struct inode_operations f2fs_special_inode_operations; +extern struct kmem_cache *f2fs_inode_entry_slab; + +/* + * inline.c + */ +bool f2fs_may_inline_data(struct inode *inode); +bool f2fs_sanity_check_inline_data(struct inode *inode, struct folio *ifolio); +bool f2fs_may_inline_dentry(struct inode *inode); +void f2fs_do_read_inline_data(struct folio *folio, struct folio *ifolio); +void f2fs_truncate_inline_inode(struct inode *inode, struct folio *ifolio, + u64 from); +int f2fs_read_inline_data(struct inode *inode, struct folio *folio); +int f2fs_convert_inline_folio(struct dnode_of_data *dn, struct folio *folio); +int f2fs_convert_inline_inode(struct inode *inode); +int f2fs_try_convert_inline_dir(struct inode *dir, struct dentry *dentry); +int f2fs_write_inline_data(struct inode *inode, struct folio *folio); +int f2fs_recover_inline_data(struct inode *inode, struct folio *nfolio); +struct f2fs_dir_entry *f2fs_find_in_inline_dir(struct inode *dir, + const struct f2fs_filename *fname, struct folio **res_folio, + bool use_hash); +int f2fs_make_empty_inline_dir(struct inode *inode, struct inode *parent, + struct folio *ifolio); +int f2fs_add_inline_entry(struct inode *dir, const struct f2fs_filename *fname, + struct inode *inode, nid_t ino, umode_t mode); +void f2fs_delete_inline_entry(struct f2fs_dir_entry *dentry, + struct folio *folio, struct inode *dir, struct inode *inode); +bool f2fs_empty_inline_dir(struct inode *dir); +int f2fs_read_inline_dir(struct file *file, struct dir_context *ctx, + struct fscrypt_str *fstr); +int f2fs_inline_data_fiemap(struct inode *inode, + struct fiemap_extent_info *fieinfo, + __u64 start, __u64 len); + +/* + * shrinker.c + */ +unsigned long f2fs_shrink_count(struct shrinker *shrink, + struct shrink_control *sc); +unsigned long f2fs_shrink_scan(struct shrinker *shrink, + struct shrink_control *sc); +unsigned int f2fs_donate_files(void); +void f2fs_reclaim_caches(unsigned int reclaim_caches_kb); +void f2fs_join_shrinker(struct f2fs_sb_info *sbi); +void f2fs_leave_shrinker(struct f2fs_sb_info *sbi); + +/* + * extent_cache.c + */ +bool sanity_check_extent_cache(struct inode *inode, struct folio *ifolio); +void f2fs_init_extent_tree(struct inode *inode); +void f2fs_drop_extent_tree(struct inode *inode); +void f2fs_destroy_extent_node(struct inode *inode); +void f2fs_destroy_extent_tree(struct inode *inode); +void f2fs_init_extent_cache_info(struct f2fs_sb_info *sbi); +int __init f2fs_create_extent_cache(void); +void f2fs_destroy_extent_cache(void); + +/* read extent cache ops */ +void f2fs_init_read_extent_tree(struct inode *inode, struct folio *ifolio); +bool f2fs_lookup_read_extent_cache(struct inode *inode, pgoff_t pgofs, + struct extent_info *ei); +bool f2fs_lookup_read_extent_cache_block(struct inode *inode, pgoff_t index, + block_t *blkaddr); +void f2fs_update_read_extent_cache(struct dnode_of_data *dn); +void f2fs_update_read_extent_cache_range(struct dnode_of_data *dn, + pgoff_t fofs, block_t blkaddr, unsigned int len); +unsigned int f2fs_shrink_read_extent_tree(struct f2fs_sb_info *sbi, + int nr_shrink); + +/* block age extent cache ops */ +void f2fs_init_age_extent_tree(struct inode *inode); +bool f2fs_lookup_age_extent_cache(struct inode *inode, pgoff_t pgofs, + struct extent_info *ei); +void f2fs_update_age_extent_cache(struct dnode_of_data *dn); +void f2fs_update_age_extent_cache_range(struct dnode_of_data *dn, + pgoff_t fofs, unsigned int len); +unsigned int f2fs_shrink_age_extent_tree(struct f2fs_sb_info *sbi, + int nr_shrink); + +/* + * sysfs.c + */ +#define MIN_RA_MUL 2 +#define MAX_RA_MUL 256 + +int __init f2fs_init_sysfs(void); +void f2fs_exit_sysfs(void); +int f2fs_register_sysfs(struct f2fs_sb_info *sbi); +void f2fs_unregister_sysfs(struct f2fs_sb_info *sbi); + +/* verity.c */ +extern const struct fsverity_operations f2fs_verityops; + +/* + * crypto support + */ +static inline bool f2fs_encrypted_file(struct inode *inode) +{ + return IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode); +} + +static inline void f2fs_set_encrypted_inode(struct inode *inode) +{ +#ifdef CONFIG_FS_ENCRYPTION + file_set_encrypt(inode); + f2fs_set_inode_flags(inode); +#endif +} + +/* + * Returns true if the reads of the inode's data need to undergo some + * postprocessing step, like decryption or authenticity verification. + */ +static inline bool f2fs_post_read_required(struct inode *inode) +{ + return f2fs_encrypted_file(inode) || fsverity_active(inode) || + f2fs_compressed_file(inode); +} + +static inline bool f2fs_used_in_atomic_write(struct inode *inode) +{ + return f2fs_is_atomic_file(inode) || f2fs_is_cow_file(inode); +} + +static inline bool f2fs_meta_inode_gc_required(struct inode *inode) +{ + return f2fs_post_read_required(inode) || f2fs_used_in_atomic_write(inode); +} + +/* + * compress.c + */ +#ifdef CONFIG_F2FS_FS_COMPRESSION +enum cluster_check_type { + CLUSTER_IS_COMPR, /* check only if compressed cluster */ + CLUSTER_COMPR_BLKS, /* return # of compressed blocks in a cluster */ + CLUSTER_RAW_BLKS /* return # of raw blocks in a cluster */ +}; +bool f2fs_is_compressed_page(struct folio *folio); +struct folio *f2fs_compress_control_folio(struct folio *folio); +int f2fs_prepare_compress_overwrite(struct inode *inode, + struct page **pagep, pgoff_t index, void **fsdata); +bool f2fs_compress_write_end(struct inode *inode, void *fsdata, + pgoff_t index, unsigned copied); +int f2fs_truncate_partial_cluster(struct inode *inode, u64 from, bool lock); +void f2fs_compress_write_end_io(struct bio *bio, struct folio *folio); +bool f2fs_is_compress_backend_ready(struct inode *inode); +bool f2fs_is_compress_level_valid(int alg, int lvl); +int __init f2fs_init_compress_mempool(void); +void f2fs_destroy_compress_mempool(void); +void f2fs_decompress_cluster(struct decompress_io_ctx *dic, bool in_task); +void f2fs_end_read_compressed_page(struct folio *folio, bool failed, + block_t blkaddr, bool in_task); +bool f2fs_cluster_is_empty(struct compress_ctx *cc); +bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index); +bool f2fs_all_cluster_page_ready(struct compress_ctx *cc, struct page **pages, + int index, int nr_pages, bool uptodate); +bool f2fs_sanity_check_cluster(struct dnode_of_data *dn); +void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct folio *folio); +int f2fs_write_multi_pages(struct compress_ctx *cc, + int *submitted, + struct writeback_control *wbc, + enum iostat_type io_type); +int f2fs_is_compressed_cluster(struct inode *inode, pgoff_t index); +bool f2fs_is_sparse_cluster(struct inode *inode, pgoff_t index); +void f2fs_update_read_extent_tree_range_compressed(struct inode *inode, + pgoff_t fofs, block_t blkaddr, + unsigned int llen, unsigned int c_len); +int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret, + unsigned nr_pages, sector_t *last_block_in_bio, + struct readahead_control *rac, bool for_write); +struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc); +void f2fs_decompress_end_io(struct decompress_io_ctx *dic, bool failed, + bool in_task); +void f2fs_put_folio_dic(struct folio *folio, bool in_task); +unsigned int f2fs_cluster_blocks_are_contiguous(struct dnode_of_data *dn, + unsigned int ofs_in_node); +int f2fs_init_compress_ctx(struct compress_ctx *cc); +void f2fs_destroy_compress_ctx(struct compress_ctx *cc, bool reuse); +void f2fs_init_compress_info(struct f2fs_sb_info *sbi); +int f2fs_init_compress_inode(struct f2fs_sb_info *sbi); +void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi); +int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi); +void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi); +int __init f2fs_init_compress_cache(void); +void f2fs_destroy_compress_cache(void); +struct address_space *COMPRESS_MAPPING(struct f2fs_sb_info *sbi); +void f2fs_invalidate_compress_pages_range(struct f2fs_sb_info *sbi, + block_t blkaddr, unsigned int len); +bool f2fs_load_compressed_folio(struct f2fs_sb_info *sbi, struct folio *folio, + block_t blkaddr); +void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi, nid_t ino); +#define inc_compr_inode_stat(inode) \ + do { \ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); \ + sbi->compr_new_inode++; \ + } while (0) +#define add_compr_block_stat(inode, blocks) \ + do { \ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); \ + int diff = F2FS_I(inode)->i_cluster_size - blocks; \ + sbi->compr_written_block += blocks; \ + sbi->compr_saved_block += diff; \ + } while (0) +#else +static inline bool f2fs_is_compressed_page(struct folio *folio) { return false; } +static inline bool f2fs_is_compress_backend_ready(struct inode *inode) +{ + if (!f2fs_compressed_file(inode)) + return true; + /* not support compression */ + return false; +} +static inline bool f2fs_is_compress_level_valid(int alg, int lvl) { return false; } +static inline struct folio *f2fs_compress_control_folio(struct folio *folio) +{ + WARN_ON_ONCE(1); + return ERR_PTR(-EINVAL); +} +static inline int __init f2fs_init_compress_mempool(void) { return 0; } +static inline void f2fs_destroy_compress_mempool(void) { } +static inline void f2fs_decompress_cluster(struct decompress_io_ctx *dic, + bool in_task) { } +static inline void f2fs_end_read_compressed_page(struct folio *folio, + bool failed, block_t blkaddr, bool in_task) +{ + WARN_ON_ONCE(1); +} +static inline void f2fs_put_folio_dic(struct folio *folio, bool in_task) +{ + WARN_ON_ONCE(1); +} +static inline unsigned int f2fs_cluster_blocks_are_contiguous( + struct dnode_of_data *dn, unsigned int ofs_in_node) { return 0; } +static inline bool f2fs_sanity_check_cluster(struct dnode_of_data *dn) { return false; } +static inline int f2fs_init_compress_inode(struct f2fs_sb_info *sbi) { return 0; } +static inline void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi) { } +static inline int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi) { return 0; } +static inline void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi) { } +static inline int __init f2fs_init_compress_cache(void) { return 0; } +static inline void f2fs_destroy_compress_cache(void) { } +static inline void f2fs_invalidate_compress_pages_range(struct f2fs_sb_info *sbi, + block_t blkaddr, unsigned int len) { } +static inline bool f2fs_load_compressed_folio(struct f2fs_sb_info *sbi, + struct folio *folio, block_t blkaddr) { return false; } +static inline void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi, + nid_t ino) { } +#define inc_compr_inode_stat(inode) do { } while (0) +static inline int f2fs_is_compressed_cluster( + struct inode *inode, + pgoff_t index) { return 0; } +static inline bool f2fs_is_sparse_cluster( + struct inode *inode, + pgoff_t index) { return true; } +static inline void f2fs_update_read_extent_tree_range_compressed( + struct inode *inode, + pgoff_t fofs, block_t blkaddr, + unsigned int llen, unsigned int c_len) { } +#endif + +static inline int set_compress_context(struct inode *inode) +{ +#ifdef CONFIG_F2FS_FS_COMPRESSION + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_inode_info *fi = F2FS_I(inode); + + fi->i_compress_algorithm = F2FS_OPTION(sbi).compress_algorithm; + fi->i_log_cluster_size = F2FS_OPTION(sbi).compress_log_size; + fi->i_compress_flag = F2FS_OPTION(sbi).compress_chksum ? + BIT(COMPRESS_CHKSUM) : 0; + fi->i_cluster_size = BIT(fi->i_log_cluster_size); + if ((fi->i_compress_algorithm == COMPRESS_LZ4 || + fi->i_compress_algorithm == COMPRESS_ZSTD) && + F2FS_OPTION(sbi).compress_level) + fi->i_compress_level = F2FS_OPTION(sbi).compress_level; + fi->i_flags |= F2FS_COMPR_FL; + set_inode_flag(inode, FI_COMPRESSED_FILE); + stat_inc_compr_inode(inode); + inc_compr_inode_stat(inode); + f2fs_mark_inode_dirty_sync(inode, true); + return 0; +#else + return -EOPNOTSUPP; +#endif +} + +static inline bool f2fs_disable_compressed_file(struct inode *inode) +{ + struct f2fs_inode_info *fi = F2FS_I(inode); + + f2fs_down_write(&fi->i_sem); + + if (!f2fs_compressed_file(inode)) { + f2fs_up_write(&fi->i_sem); + return true; + } + if (f2fs_is_mmap_file(inode) || atomic_read(&fi->writeback) || + (S_ISREG(inode->i_mode) && F2FS_HAS_BLOCKS(inode))) { + f2fs_up_write(&fi->i_sem); + return false; + } + + fi->i_flags &= ~F2FS_COMPR_FL; + stat_dec_compr_inode(inode); + clear_inode_flag(inode, FI_COMPRESSED_FILE); + f2fs_mark_inode_dirty_sync(inode, true); + + f2fs_up_write(&fi->i_sem); + return true; +} + +#define F2FS_FEATURE_FUNCS(name, flagname) \ +static inline bool f2fs_sb_has_##name(struct f2fs_sb_info *sbi) \ +{ \ + return F2FS_HAS_FEATURE(sbi, F2FS_FEATURE_##flagname); \ +} + +F2FS_FEATURE_FUNCS(encrypt, ENCRYPT); +F2FS_FEATURE_FUNCS(blkzoned, BLKZONED); +F2FS_FEATURE_FUNCS(extra_attr, EXTRA_ATTR); +F2FS_FEATURE_FUNCS(project_quota, PRJQUOTA); +F2FS_FEATURE_FUNCS(inode_chksum, INODE_CHKSUM); +F2FS_FEATURE_FUNCS(flexible_inline_xattr, FLEXIBLE_INLINE_XATTR); +F2FS_FEATURE_FUNCS(quota_ino, QUOTA_INO); +F2FS_FEATURE_FUNCS(inode_crtime, INODE_CRTIME); +F2FS_FEATURE_FUNCS(lost_found, LOST_FOUND); +F2FS_FEATURE_FUNCS(verity, VERITY); +F2FS_FEATURE_FUNCS(sb_chksum, SB_CHKSUM); +F2FS_FEATURE_FUNCS(casefold, CASEFOLD); +F2FS_FEATURE_FUNCS(compression, COMPRESSION); +F2FS_FEATURE_FUNCS(readonly, RO); +F2FS_FEATURE_FUNCS(device_alias, DEVICE_ALIAS); +F2FS_FEATURE_FUNCS(packed_ssa, PACKED_SSA); + +#ifdef CONFIG_BLK_DEV_ZONED +static inline bool f2fs_zone_is_seq(struct f2fs_sb_info *sbi, int devi, + unsigned int zone) +{ + return test_bit(zone, FDEV(devi).blkz_seq); +} + +static inline bool f2fs_blkz_is_seq(struct f2fs_sb_info *sbi, int devi, + block_t blkaddr) +{ + return f2fs_zone_is_seq(sbi, devi, blkaddr / sbi->blocks_per_blkz); +} +#endif + +static inline int f2fs_bdev_index(struct f2fs_sb_info *sbi, + struct block_device *bdev) +{ + int i; + + if (!f2fs_is_multi_device(sbi)) + return 0; + + for (i = 0; i < sbi->s_ndevs; i++) + if (FDEV(i).bdev == bdev) + return i; + + WARN_ON(1); + return -1; +} + +static inline bool f2fs_hw_should_discard(struct f2fs_sb_info *sbi) +{ + return f2fs_sb_has_blkzoned(sbi); +} + +static inline bool f2fs_bdev_support_discard(struct block_device *bdev) +{ + return bdev_max_discard_sectors(bdev) || bdev_is_zoned(bdev); +} + +static inline bool f2fs_hw_support_discard(struct f2fs_sb_info *sbi) +{ + int i; + + if (!f2fs_is_multi_device(sbi)) + return f2fs_bdev_support_discard(sbi->sb->s_bdev); + + for (i = 0; i < sbi->s_ndevs; i++) + if (f2fs_bdev_support_discard(FDEV(i).bdev)) + return true; + return false; +} + +static inline unsigned int f2fs_hw_discard_granularity(struct f2fs_sb_info *sbi) +{ + int i = 1; + unsigned int discard_granularity = bdev_discard_granularity(sbi->sb->s_bdev); + + if (f2fs_is_multi_device(sbi)) + for (; i < sbi->s_ndevs && !bdev_is_zoned(FDEV(i).bdev); i++) + discard_granularity = max_t(unsigned int, discard_granularity, + bdev_discard_granularity(FDEV(i).bdev)); + return discard_granularity; +} + +static inline bool f2fs_realtime_discard_enable(struct f2fs_sb_info *sbi) +{ + return (test_opt(sbi, DISCARD) && f2fs_hw_support_discard(sbi)) || + f2fs_hw_should_discard(sbi); +} + +static inline bool f2fs_hw_is_readonly(struct f2fs_sb_info *sbi) +{ + int i; + + if (!f2fs_is_multi_device(sbi)) + return bdev_read_only(sbi->sb->s_bdev); + + for (i = 0; i < sbi->s_ndevs; i++) + if (bdev_read_only(FDEV(i).bdev)) + return true; + return false; +} + +static inline bool f2fs_dev_is_readonly(struct f2fs_sb_info *sbi) +{ + return f2fs_sb_has_readonly(sbi) || f2fs_hw_is_readonly(sbi); +} + +static inline bool f2fs_lfs_mode(struct f2fs_sb_info *sbi) +{ + return F2FS_OPTION(sbi).fs_mode == FS_MODE_LFS; +} + +static inline bool f2fs_is_sequential_zone_area(struct f2fs_sb_info *sbi, + block_t blkaddr) +{ + if (f2fs_sb_has_blkzoned(sbi)) { +#ifdef CONFIG_BLK_DEV_ZONED + int devi = f2fs_target_device_index(sbi, blkaddr); + + if (!bdev_is_zoned(FDEV(devi).bdev)) + return false; + + if (f2fs_is_multi_device(sbi)) { + if (blkaddr < FDEV(devi).start_blk || + blkaddr > FDEV(devi).end_blk) { + f2fs_err(sbi, "Invalid block %x", blkaddr); + return false; + } + blkaddr -= FDEV(devi).start_blk; + } + + return f2fs_blkz_is_seq(sbi, devi, blkaddr); +#else + return false; +#endif + } + return false; +} + +static inline bool f2fs_low_mem_mode(struct f2fs_sb_info *sbi) +{ + return F2FS_OPTION(sbi).memory_mode == MEMORY_MODE_LOW; +} + +static inline bool f2fs_may_compress(struct inode *inode) +{ + if (IS_SWAPFILE(inode) || f2fs_is_pinned_file(inode) || + f2fs_is_atomic_file(inode) || f2fs_has_inline_data(inode) || + f2fs_is_mmap_file(inode)) + return false; + return S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode); +} + +static inline void f2fs_i_compr_blocks_update(struct inode *inode, + u64 blocks, bool add) +{ + struct f2fs_inode_info *fi = F2FS_I(inode); + int diff = fi->i_cluster_size - blocks; + + /* don't update i_compr_blocks if saved blocks were released */ + if (!add && !atomic_read(&fi->i_compr_blocks)) + return; + + if (add) { + atomic_add(diff, &fi->i_compr_blocks); + stat_add_compr_blocks(inode, diff); + } else { + atomic_sub(diff, &fi->i_compr_blocks); + stat_sub_compr_blocks(inode, diff); + } + f2fs_mark_inode_dirty_sync(inode, true); +} + +static inline bool f2fs_allow_multi_device_dio(struct f2fs_sb_info *sbi, + int flag) +{ + if (!f2fs_is_multi_device(sbi)) + return false; + if (flag != F2FS_GET_BLOCK_DIO) + return false; + return sbi->aligned_blksize; +} + +#ifdef CONFIG_F2FS_FAULT_INJECTION +extern int f2fs_build_fault_attr(struct f2fs_sb_info *sbi, unsigned long rate, + unsigned long type, enum fault_option fo); +extern void f2fs_simulate_lock_timeout(struct f2fs_sb_info *sbi); +#else +static inline int f2fs_build_fault_attr(struct f2fs_sb_info *sbi, + unsigned long rate, unsigned long type, + enum fault_option fo) +{ + return 0; +} +static inline void f2fs_simulate_lock_timeout(struct f2fs_sb_info *sbi) +{ + return; +} +#endif + +static inline bool is_journalled_quota(struct f2fs_sb_info *sbi) +{ +#ifdef CONFIG_QUOTA + if (f2fs_sb_has_quota_ino(sbi)) + return true; + if (F2FS_OPTION(sbi).s_qf_names[USRQUOTA] || + F2FS_OPTION(sbi).s_qf_names[GRPQUOTA] || + F2FS_OPTION(sbi).s_qf_names[PRJQUOTA]) + return true; +#endif + return false; +} + +static inline bool f2fs_quota_file(struct f2fs_sb_info *sbi, nid_t ino) +{ +#ifdef CONFIG_QUOTA + int i; + + if (!f2fs_sb_has_quota_ino(sbi)) + return false; + + for (i = 0; i < MAXQUOTAS; i++) { + if (f2fs_qf_ino(sbi->sb, i) == ino) + return true; + } +#endif + return false; +} + +static inline bool f2fs_block_unit_discard(struct f2fs_sb_info *sbi) +{ + return F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_BLOCK; +} + +static inline void __f2fs_schedule_timeout(long timeout, bool io) +{ + set_current_state(TASK_UNINTERRUPTIBLE); + if (io) + io_schedule_timeout(timeout); + else + schedule_timeout(timeout); +} + +#define f2fs_io_schedule_timeout(timeout) \ + __f2fs_schedule_timeout(timeout, true) +#define f2fs_schedule_timeout(timeout) \ + __f2fs_schedule_timeout(timeout, false) + +static inline void f2fs_schedule_timeout_killable(long timeout, bool io) +{ + unsigned long last_time = jiffies + timeout; + + while (jiffies < last_time) { + if (fatal_signal_pending(current)) + return; + __f2fs_schedule_timeout(DEFAULT_SCHEDULE_TIMEOUT, io); + } +} + +static inline void f2fs_handle_page_eio(struct f2fs_sb_info *sbi, + struct folio *folio, enum page_type type) +{ + pgoff_t ofs = folio->index; + + if (unlikely(f2fs_cp_error(sbi))) + return; + + if (ofs == sbi->page_eio_ofs[type]) { + if (sbi->page_eio_cnt[type]++ == MAX_RETRY_PAGE_EIO) { + enum stop_cp_reason stop_reason; + + switch (type) { + case META: + stop_reason = STOP_CP_REASON_READ_META; + break; + case NODE: + stop_reason = STOP_CP_REASON_READ_NODE; + break; + case DATA: + stop_reason = STOP_CP_REASON_READ_DATA; + break; + default: + f2fs_bug_on(sbi, 1); + return; + } + f2fs_stop_checkpoint(sbi, false, stop_reason); + } + } else { + sbi->page_eio_ofs[type] = ofs; + sbi->page_eio_cnt[type] = 0; + } +} + +static inline bool f2fs_is_readonly(struct f2fs_sb_info *sbi) +{ + return f2fs_sb_has_readonly(sbi) || f2fs_readonly(sbi->sb); +} + +static inline void f2fs_truncate_meta_inode_pages(struct f2fs_sb_info *sbi, + block_t blkaddr, unsigned int cnt) +{ + bool need_submit = false; + int i = 0; + + do { + struct folio *folio; + + folio = filemap_get_folio(META_MAPPING(sbi), blkaddr + i); + if (!IS_ERR(folio)) { + if (folio_test_writeback(folio)) + need_submit = true; + f2fs_folio_put(folio, false); + } + } while (++i < cnt && !need_submit); + + if (need_submit) + f2fs_submit_merged_write_cond(sbi, sbi->meta_inode, + NULL, 0, DATA); + + truncate_inode_pages_range(META_MAPPING(sbi), + F2FS_BLK_TO_BYTES((loff_t)blkaddr), + F2FS_BLK_END_BYTES((loff_t)(blkaddr + cnt - 1))); +} + +static inline void f2fs_invalidate_internal_cache(struct f2fs_sb_info *sbi, + block_t blkaddr, unsigned int len) +{ + f2fs_truncate_meta_inode_pages(sbi, blkaddr, len); + f2fs_invalidate_compress_pages_range(sbi, blkaddr, len); +} + +#endif /* _LINUX_F2FS_H */ diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c new file mode 100644 index 000000000..089759366 --- /dev/null +++ b/fs/f2fs/file.c @@ -0,0 +1,5558 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/f2fs/file.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "f2fs.h" +#include "node.h" +#include "segment.h" +#include "xattr.h" +#include "acl.h" +#include "gc.h" +#include "iostat.h" +#include +#include + +static void f2fs_zero_post_eof_page(struct inode *inode, + loff_t new_size, bool lock) +{ + loff_t old_size = i_size_read(inode); + + if (old_size >= new_size) + return; + + if (mapping_empty(inode->i_mapping)) + return; + + if (lock) + filemap_invalidate_lock(inode->i_mapping); + /* zero or drop pages only in range of [old_size, new_size] */ + truncate_inode_pages_range(inode->i_mapping, old_size, new_size); + if (lock) + filemap_invalidate_unlock(inode->i_mapping); +} + +static vm_fault_t f2fs_filemap_fault(struct vm_fault *vmf) +{ + struct inode *inode = file_inode(vmf->vma->vm_file); + vm_flags_t flags = vmf->vma->vm_flags; + vm_fault_t ret; + + ret = filemap_fault(vmf); + if (ret & VM_FAULT_LOCKED) + f2fs_update_iostat(F2FS_I_SB(inode), inode, + APP_MAPPED_READ_IO, F2FS_BLKSIZE); + + trace_f2fs_filemap_fault(inode, vmf->pgoff, flags, ret); + + return ret; +} + +static vm_fault_t f2fs_vm_page_mkwrite(struct vm_fault *vmf) +{ + struct folio *folio = page_folio(vmf->page); + struct inode *inode = file_inode(vmf->vma->vm_file); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct dnode_of_data dn; + bool need_alloc = !f2fs_is_pinned_file(inode); + int err = 0; + vm_fault_t ret; + + /* + * We only support large folio on the read case. + * Don't make any dirty pages. + */ + if (unlikely(IS_IMMUTABLE(inode)) || + mapping_large_folio_support(inode->i_mapping)) { + f2fs_err(sbi, "Not expected: immutable: %d large_folio: %d", + IS_IMMUTABLE(inode), + mapping_large_folio_support(inode->i_mapping)); + return VM_FAULT_SIGBUS; + } + + if (is_inode_flag_set(inode, FI_COMPRESS_RELEASED)) { + err = -EIO; + goto out; + } + + if (unlikely(f2fs_cp_error(sbi))) { + err = -EIO; + goto out; + } + + if (!f2fs_is_checkpoint_ready(sbi)) { + err = -ENOSPC; + goto out; + } + + err = f2fs_convert_inline_inode(inode); + if (err) + goto out; + +#ifdef CONFIG_F2FS_FS_COMPRESSION + if (f2fs_compressed_file(inode)) { + int ret = f2fs_is_compressed_cluster(inode, folio->index); + + if (ret < 0) { + err = ret; + goto out; + } else if (ret) { + need_alloc = false; + } + } +#endif + /* should do out of any locked page */ + if (need_alloc) + f2fs_balance_fs(sbi, true); + + sb_start_pagefault(inode->i_sb); + + f2fs_bug_on(sbi, f2fs_has_inline_data(inode)); + + f2fs_zero_post_eof_page(inode, (folio->index + 1) << PAGE_SHIFT, true); + + file_update_time(vmf->vma->vm_file); + filemap_invalidate_lock_shared(inode->i_mapping); + + folio_lock(folio); + if (unlikely(folio->mapping != inode->i_mapping || + folio_pos(folio) > i_size_read(inode) || + !folio_test_uptodate(folio))) { + folio_unlock(folio); + err = -EFAULT; + goto out_sem; + } + + set_new_dnode(&dn, inode, NULL, NULL, 0); + if (need_alloc) { + /* block allocation */ + err = f2fs_get_block_locked(&dn, folio->index); + } else { + err = f2fs_get_dnode_of_data(&dn, folio->index, LOOKUP_NODE); + f2fs_put_dnode(&dn); + if (f2fs_is_pinned_file(inode) && + !__is_valid_data_blkaddr(dn.data_blkaddr)) + err = -EIO; + } + + if (err) { + folio_unlock(folio); + goto out_sem; + } + + f2fs_folio_wait_writeback(folio, DATA, false, true); + + /* wait for GCed page writeback via META_MAPPING */ + f2fs_wait_on_block_writeback(inode, dn.data_blkaddr); + + /* + * check to see if the page is mapped already (no holes) + */ + if (folio_test_mappedtodisk(folio)) + goto out_sem; + + /* page is wholly or partially inside EOF */ + if (((loff_t)(folio->index + 1) << PAGE_SHIFT) > + i_size_read(inode)) { + loff_t offset; + + offset = i_size_read(inode) & ~PAGE_MASK; + folio_zero_segment(folio, offset, folio_size(folio)); + } + folio_mark_dirty(folio); + + f2fs_update_iostat(sbi, inode, APP_MAPPED_IO, F2FS_BLKSIZE); + f2fs_update_time(sbi, REQ_TIME); + +out_sem: + filemap_invalidate_unlock_shared(inode->i_mapping); + + sb_end_pagefault(inode->i_sb); +out: + ret = vmf_fs_error(err); + + trace_f2fs_vm_page_mkwrite(inode, folio->index, vmf->vma->vm_flags, ret); + return ret; +} + +static const struct vm_operations_struct f2fs_file_vm_ops = { + .fault = f2fs_filemap_fault, + .map_pages = filemap_map_pages, + .page_mkwrite = f2fs_vm_page_mkwrite, +}; + +static int get_parent_ino(struct inode *inode, nid_t *pino) +{ + struct dentry *dentry; + + /* + * Make sure to get the non-deleted alias. The alias associated with + * the open file descriptor being fsync()'ed may be deleted already. + */ + dentry = d_find_alias(inode); + if (!dentry) + return 0; + + *pino = d_parent_ino(dentry); + dput(dentry); + return 1; +} + +static inline enum cp_reason_type need_do_checkpoint(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + enum cp_reason_type cp_reason = CP_NO_NEEDED; + + if (!S_ISREG(inode->i_mode)) + cp_reason = CP_NON_REGULAR; + else if (f2fs_compressed_file(inode)) + cp_reason = CP_COMPRESSED; + else if (inode->i_nlink != 1) + cp_reason = CP_HARDLINK; + else if (is_sbi_flag_set(sbi, SBI_NEED_CP)) + cp_reason = CP_SB_NEED_CP; + else if (file_wrong_pino(inode)) + cp_reason = CP_WRONG_PINO; + else if (!f2fs_space_for_roll_forward(sbi)) + cp_reason = CP_NO_SPC_ROLL; + else if (!f2fs_is_checkpointed_node(sbi, F2FS_I(inode)->i_pino)) + cp_reason = CP_NODE_NEED_CP; + else if (test_opt(sbi, FASTBOOT)) + cp_reason = CP_FASTBOOT_MODE; + else if (F2FS_OPTION(sbi).active_logs == 2) + cp_reason = CP_SPEC_LOG_NUM; + else if (F2FS_OPTION(sbi).fsync_mode == FSYNC_MODE_STRICT && + f2fs_need_dentry_mark(sbi, inode->i_ino) && + f2fs_exist_written_data(sbi, F2FS_I(inode)->i_pino, + TRANS_DIR_INO)) + cp_reason = CP_RECOVER_DIR; + else if (f2fs_exist_written_data(sbi, F2FS_I(inode)->i_pino, + XATTR_DIR_INO)) + cp_reason = CP_XATTR_DIR; + + return cp_reason; +} + +static bool need_inode_page_update(struct f2fs_sb_info *sbi, nid_t ino) +{ + struct folio *i = filemap_get_folio(NODE_MAPPING(sbi), ino); + bool ret = false; + /* But we need to avoid that there are some inode updates */ + if ((!IS_ERR(i) && folio_test_dirty(i)) || + f2fs_need_inode_block_update(sbi, ino)) + ret = true; + f2fs_folio_put(i, false); + return ret; +} + +static void try_to_fix_pino(struct inode *inode) +{ + struct f2fs_inode_info *fi = F2FS_I(inode); + nid_t pino; + + f2fs_down_write(&fi->i_sem); + if (file_wrong_pino(inode) && inode->i_nlink == 1 && + get_parent_ino(inode, &pino)) { + f2fs_i_pino_write(inode, pino); + file_got_pino(inode); + } + f2fs_up_write(&fi->i_sem); +} + +static int f2fs_do_sync_file(struct file *file, loff_t start, loff_t end, + int datasync, bool atomic) +{ + struct inode *inode = file->f_mapping->host; + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + nid_t ino = inode->i_ino; + int ret = 0; + enum cp_reason_type cp_reason = 0; + struct writeback_control wbc = { + .sync_mode = WB_SYNC_ALL, + .nr_to_write = LONG_MAX, + }; + unsigned int seq_id = 0; + + if (unlikely(f2fs_readonly(inode->i_sb))) + return 0; + + trace_f2fs_sync_file_enter(inode); + + if (S_ISDIR(inode->i_mode)) + goto go_write; + + /* if fdatasync is triggered, let's do in-place-update */ + if (datasync || get_dirty_pages(inode) <= SM_I(sbi)->min_fsync_blocks) + set_inode_flag(inode, FI_NEED_IPU); + ret = file_write_and_wait_range(file, start, end); + clear_inode_flag(inode, FI_NEED_IPU); + + if (ret || is_sbi_flag_set(sbi, SBI_CP_DISABLED)) { + trace_f2fs_sync_file_exit(inode, cp_reason, datasync, ret); + return ret; + } + + /* if the inode is dirty, let's recover all the time */ + if (!f2fs_skip_inode_update(inode, datasync)) { + f2fs_write_inode(inode, NULL); + goto go_write; + } + + /* + * if there is no written data, don't waste time to write recovery info. + */ + if (!is_inode_flag_set(inode, FI_APPEND_WRITE) && + !f2fs_exist_written_data(sbi, ino, APPEND_INO)) { + + /* it may call write_inode just prior to fsync */ + if (need_inode_page_update(sbi, ino)) + goto go_write; + + if (is_inode_flag_set(inode, FI_UPDATE_WRITE) || + f2fs_exist_written_data(sbi, ino, UPDATE_INO)) + goto flush_out; + goto out; + } else { + /* + * for OPU case, during fsync(), node can be persisted before + * data when lower device doesn't support write barrier, result + * in data corruption after SPO. + * So for strict fsync mode, force to use atomic write semantics + * to keep write order in between data/node and last node to + * avoid potential data corruption. + */ + if (F2FS_OPTION(sbi).fsync_mode == + FSYNC_MODE_STRICT && !atomic) + atomic = true; + } +go_write: + /* + * Both of fdatasync() and fsync() are able to be recovered from + * sudden-power-off. + */ + f2fs_down_read(&F2FS_I(inode)->i_sem); + cp_reason = need_do_checkpoint(inode); + f2fs_up_read(&F2FS_I(inode)->i_sem); + + if (cp_reason) { + /* all the dirty node pages should be flushed for POR */ + ret = f2fs_sync_fs(inode->i_sb, 1); + + /* + * We've secured consistency through sync_fs. Following pino + * will be used only for fsynced inodes after checkpoint. + */ + try_to_fix_pino(inode); + clear_inode_flag(inode, FI_APPEND_WRITE); + clear_inode_flag(inode, FI_UPDATE_WRITE); + goto out; + } +sync_nodes: + atomic_inc(&sbi->wb_sync_req[NODE]); + ret = f2fs_fsync_node_pages(sbi, inode, &wbc, atomic, &seq_id); + atomic_dec(&sbi->wb_sync_req[NODE]); + if (ret) + goto out; + + /* if cp_error was enabled, we should avoid infinite loop */ + if (unlikely(f2fs_cp_error(sbi))) { + ret = -EIO; + goto out; + } + + if (f2fs_need_inode_block_update(sbi, ino)) { + f2fs_mark_inode_dirty_sync(inode, true); + f2fs_write_inode(inode, NULL); + goto sync_nodes; + } + + /* + * If it's atomic_write, it's just fine to keep write ordering. So + * here we don't need to wait for node write completion, since we use + * node chain which serializes node blocks. If one of node writes are + * reordered, we can see simply broken chain, resulting in stopping + * roll-forward recovery. It means we'll recover all or none node blocks + * given fsync mark. + */ + if (!atomic) { + ret = f2fs_wait_on_node_pages_writeback(sbi, seq_id); + if (ret) + goto out; + } + + /* once recovery info is written, don't need to tack this */ + f2fs_remove_ino_entry(sbi, ino, APPEND_INO); + clear_inode_flag(inode, FI_APPEND_WRITE); +flush_out: + if (!atomic && F2FS_OPTION(sbi).fsync_mode != FSYNC_MODE_NOBARRIER) + ret = f2fs_issue_flush(sbi, inode->i_ino); + if (!ret) { + f2fs_remove_ino_entry(sbi, ino, UPDATE_INO); + clear_inode_flag(inode, FI_UPDATE_WRITE); + f2fs_remove_ino_entry(sbi, ino, FLUSH_INO); + } + f2fs_update_time(sbi, REQ_TIME); +out: + trace_f2fs_sync_file_exit(inode, cp_reason, datasync, ret); + return ret; +} + +int f2fs_sync_file(struct file *file, loff_t start, loff_t end, int datasync) +{ + if (unlikely(f2fs_cp_error(F2FS_I_SB(file_inode(file))))) + return -EIO; + return f2fs_do_sync_file(file, start, end, datasync, false); +} + +static bool __found_offset(struct address_space *mapping, + struct dnode_of_data *dn, pgoff_t index, int whence) +{ + block_t blkaddr = f2fs_data_blkaddr(dn); + struct inode *inode = mapping->host; + bool compressed_cluster = false; + + if (f2fs_compressed_file(inode)) { + block_t first_blkaddr = data_blkaddr(dn->inode, dn->node_folio, + ALIGN_DOWN(dn->ofs_in_node, F2FS_I(inode)->i_cluster_size)); + + compressed_cluster = first_blkaddr == COMPRESS_ADDR; + } + + switch (whence) { + case SEEK_DATA: + if (__is_valid_data_blkaddr(blkaddr)) + return true; + if (blkaddr == NEW_ADDR && + xa_get_mark(&mapping->i_pages, index, PAGECACHE_TAG_DIRTY)) + return true; + if (compressed_cluster) + return true; + break; + case SEEK_HOLE: + if (compressed_cluster) + return false; + if (blkaddr == NULL_ADDR) + return true; + break; + } + return false; +} + +static loff_t f2fs_seek_block(struct file *file, loff_t offset, int whence) +{ + struct inode *inode = file->f_mapping->host; + loff_t maxbytes = F2FS_BLK_TO_BYTES(max_file_blocks(inode)); + struct dnode_of_data dn; + pgoff_t pgofs, end_offset; + loff_t data_ofs = offset; + loff_t isize; + int err = 0; + + inode_lock_shared(inode); + + isize = i_size_read(inode); + if (offset >= isize) + goto fail; + + /* handle inline data case */ + if (f2fs_has_inline_data(inode)) { + if (whence == SEEK_HOLE) { + data_ofs = isize; + goto found; + } else if (whence == SEEK_DATA) { + data_ofs = offset; + goto found; + } + } + + pgofs = (pgoff_t)(offset >> PAGE_SHIFT); + + for (; data_ofs < isize; data_ofs = (loff_t)pgofs << PAGE_SHIFT) { + set_new_dnode(&dn, inode, NULL, NULL, 0); + err = f2fs_get_dnode_of_data(&dn, pgofs, LOOKUP_NODE); + if (err && err != -ENOENT) { + goto fail; + } else if (err == -ENOENT) { + /* direct node does not exists */ + if (whence == SEEK_DATA) { + pgofs = f2fs_get_next_page_offset(&dn, pgofs); + continue; + } else { + goto found; + } + } + + end_offset = ADDRS_PER_PAGE(dn.node_folio, inode); + + /* find data/hole in dnode block */ + for (; dn.ofs_in_node < end_offset; + dn.ofs_in_node++, pgofs++, + data_ofs = (loff_t)pgofs << PAGE_SHIFT) { + block_t blkaddr; + + blkaddr = f2fs_data_blkaddr(&dn); + + if (__is_valid_data_blkaddr(blkaddr) && + !f2fs_is_valid_blkaddr(F2FS_I_SB(inode), + blkaddr, DATA_GENERIC_ENHANCE)) { + f2fs_put_dnode(&dn); + goto fail; + } + + if (__found_offset(file->f_mapping, &dn, + pgofs, whence)) { + f2fs_put_dnode(&dn); + goto found; + } + } + f2fs_put_dnode(&dn); + } + + if (whence == SEEK_DATA) + goto fail; +found: + if (whence == SEEK_HOLE && data_ofs > isize) + data_ofs = isize; + inode_unlock_shared(inode); + return vfs_setpos(file, data_ofs, maxbytes); +fail: + inode_unlock_shared(inode); + return -ENXIO; +} + +static loff_t f2fs_llseek(struct file *file, loff_t offset, int whence) +{ + struct inode *inode = file->f_mapping->host; + loff_t maxbytes = F2FS_BLK_TO_BYTES(max_file_blocks(inode)); + + switch (whence) { + case SEEK_SET: + case SEEK_CUR: + case SEEK_END: + return generic_file_llseek_size(file, offset, whence, + maxbytes, i_size_read(inode)); + case SEEK_DATA: + case SEEK_HOLE: + if (offset < 0) + return -ENXIO; + return f2fs_seek_block(file, offset, whence); + } + + return -EINVAL; +} + +static int f2fs_file_mmap_prepare(struct vm_area_desc *desc) +{ + struct file *file = desc->file; + struct inode *inode = file_inode(file); + + if (unlikely(f2fs_cp_error(F2FS_I_SB(inode)))) + return -EIO; + + if (!f2fs_is_compress_backend_ready(inode)) + return -EOPNOTSUPP; + + file_accessed(file); + desc->vm_ops = &f2fs_file_vm_ops; + + f2fs_down_read(&F2FS_I(inode)->i_sem); + set_inode_flag(inode, FI_MMAP_FILE); + f2fs_up_read(&F2FS_I(inode)->i_sem); + + return 0; +} + +static int finish_preallocate_blocks(struct inode *inode) +{ + int ret = 0; + bool opened; + + f2fs_down_read(&F2FS_I(inode)->i_sem); + opened = is_inode_flag_set(inode, FI_OPENED_FILE); + f2fs_up_read(&F2FS_I(inode)->i_sem); + if (opened) + return 0; + + inode_lock(inode); + if (is_inode_flag_set(inode, FI_OPENED_FILE)) + goto out_unlock; + + if (!file_should_truncate(inode)) + goto out_update; + + f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); + filemap_invalidate_lock(inode->i_mapping); + + truncate_setsize(inode, i_size_read(inode)); + ret = f2fs_truncate(inode); + + filemap_invalidate_unlock(inode->i_mapping); + f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); + if (ret) + goto out_unlock; + + file_dont_truncate(inode); +out_update: + f2fs_down_write(&F2FS_I(inode)->i_sem); + set_inode_flag(inode, FI_OPENED_FILE); + f2fs_up_write(&F2FS_I(inode)->i_sem); +out_unlock: + inode_unlock(inode); + return ret; +} + +static int f2fs_file_open(struct inode *inode, struct file *filp) +{ + int err = fscrypt_file_open(inode, filp); + + if (err) + return err; + + if (!f2fs_is_compress_backend_ready(inode)) + return -EOPNOTSUPP; + + if (mapping_large_folio_support(inode->i_mapping) && + filp->f_mode & FMODE_WRITE) + return -EOPNOTSUPP; + + err = fsverity_file_open(inode, filp); + if (err) + return err; + + filp->f_mode |= FMODE_NOWAIT; + filp->f_mode |= FMODE_CAN_ODIRECT; + + err = dquot_file_open(inode, filp); + if (err) + return err; + + err = finish_preallocate_blocks(inode); + if (!err) + atomic_inc(&F2FS_I(inode)->open_count); + return err; +} + +void f2fs_truncate_data_blocks_range(struct dnode_of_data *dn, int count) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); + int nr_free = 0, ofs = dn->ofs_in_node, len = count; + __le32 *addr; + bool compressed_cluster = false; + int cluster_index = 0, valid_blocks = 0; + int cluster_size = F2FS_I(dn->inode)->i_cluster_size; + bool released = !atomic_read(&F2FS_I(dn->inode)->i_compr_blocks); + block_t blkstart; + int blklen = 0; + + addr = get_dnode_addr(dn->inode, dn->node_folio) + ofs; + blkstart = le32_to_cpu(*addr); + + /* Assumption: truncation starts with cluster */ + for (; count > 0; count--, addr++, dn->ofs_in_node++, cluster_index++) { + block_t blkaddr = le32_to_cpu(*addr); + + if (f2fs_compressed_file(dn->inode) && + !(cluster_index & (cluster_size - 1))) { + if (compressed_cluster) + f2fs_i_compr_blocks_update(dn->inode, + valid_blocks, false); + compressed_cluster = (blkaddr == COMPRESS_ADDR); + valid_blocks = 0; + } + + if (blkaddr == NULL_ADDR) + goto next; + + f2fs_set_data_blkaddr(dn, NULL_ADDR); + + if (__is_valid_data_blkaddr(blkaddr)) { + if (time_to_inject(sbi, FAULT_BLKADDR_CONSISTENCE)) + goto next; + if (!f2fs_is_valid_blkaddr_raw(sbi, blkaddr, + DATA_GENERIC_ENHANCE)) + goto next; + if (compressed_cluster) + valid_blocks++; + } + + if (blkstart + blklen == blkaddr) { + blklen++; + } else { + f2fs_invalidate_blocks(sbi, blkstart, blklen); + blkstart = blkaddr; + blklen = 1; + } + + if (!released || blkaddr != COMPRESS_ADDR) + nr_free++; + + continue; + +next: + if (blklen) + f2fs_invalidate_blocks(sbi, blkstart, blklen); + + blkstart = le32_to_cpu(*(addr + 1)); + blklen = 0; + } + + if (blklen) + f2fs_invalidate_blocks(sbi, blkstart, blklen); + + if (compressed_cluster) + f2fs_i_compr_blocks_update(dn->inode, valid_blocks, false); + + if (nr_free) { + pgoff_t fofs; + /* + * once we invalidate valid blkaddr in range [ofs, ofs + count], + * we will invalidate all blkaddr in the whole range. + */ + fofs = f2fs_start_bidx_of_node(ofs_of_node(dn->node_folio), + dn->inode) + ofs; + f2fs_update_read_extent_cache_range(dn, fofs, 0, len); + f2fs_update_age_extent_cache_range(dn, fofs, len); + dec_valid_block_count(sbi, dn->inode, nr_free); + } + dn->ofs_in_node = ofs; + + f2fs_update_time(sbi, REQ_TIME); + trace_f2fs_truncate_data_blocks_range(dn->inode, dn->nid, + dn->ofs_in_node, nr_free); +} + +static int truncate_partial_data_page(struct inode *inode, u64 from, + bool cache_only) +{ + loff_t offset = from & (PAGE_SIZE - 1); + pgoff_t index = from >> PAGE_SHIFT; + struct address_space *mapping = inode->i_mapping; + struct folio *folio; + + if (!offset && !cache_only) + return 0; + + if (cache_only) { + folio = filemap_lock_folio(mapping, index); + if (IS_ERR(folio)) + return 0; + if (folio_test_uptodate(folio)) + goto truncate_out; + f2fs_folio_put(folio, true); + return 0; + } + + folio = f2fs_get_lock_data_folio(inode, index, true); + if (IS_ERR(folio)) + return PTR_ERR(folio) == -ENOENT ? 0 : PTR_ERR(folio); +truncate_out: + f2fs_folio_wait_writeback(folio, DATA, true, true); + folio_zero_segment(folio, offset, folio_size(folio)); + + /* An encrypted inode should have a key and truncate the last page. */ + f2fs_bug_on(F2FS_I_SB(inode), cache_only && IS_ENCRYPTED(inode)); + if (!cache_only) + folio_mark_dirty(folio); + f2fs_folio_put(folio, true); + return 0; +} + +int f2fs_do_truncate_blocks(struct inode *inode, u64 from, bool lock) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct dnode_of_data dn; + struct f2fs_lock_context lc; + pgoff_t free_from; + int count = 0, err = 0; + struct folio *ifolio; + bool truncate_page = false; + + trace_f2fs_truncate_blocks_enter(inode, from); + + if (IS_DEVICE_ALIASING(inode) && from) { + err = -EINVAL; + goto out_err; + } + + free_from = (pgoff_t)F2FS_BLK_ALIGN(from); + + if (free_from >= max_file_blocks(inode)) + goto free_partial; + + if (lock) + f2fs_lock_op(sbi, &lc); + + ifolio = f2fs_get_inode_folio(sbi, inode->i_ino); + if (IS_ERR(ifolio)) { + err = PTR_ERR(ifolio); + goto out; + } + + if (IS_DEVICE_ALIASING(inode)) { + struct extent_tree *et = F2FS_I(inode)->extent_tree[EX_READ]; + struct extent_info ei = et->largest; + + f2fs_invalidate_blocks(sbi, ei.blk, ei.len); + + dec_valid_block_count(sbi, inode, ei.len); + f2fs_update_time(sbi, REQ_TIME); + + f2fs_folio_put(ifolio, true); + goto out; + } + + if (f2fs_has_inline_data(inode)) { + f2fs_truncate_inline_inode(inode, ifolio, from); + f2fs_folio_put(ifolio, true); + truncate_page = true; + goto out; + } + + set_new_dnode(&dn, inode, ifolio, NULL, 0); + err = f2fs_get_dnode_of_data(&dn, free_from, LOOKUP_NODE_RA); + if (err) { + if (err == -ENOENT) + goto free_next; + goto out; + } + + count = ADDRS_PER_PAGE(dn.node_folio, inode); + + count -= dn.ofs_in_node; + f2fs_bug_on(sbi, count < 0); + + if (dn.ofs_in_node || IS_INODE(dn.node_folio)) { + f2fs_truncate_data_blocks_range(&dn, count); + free_from += count; + } + + f2fs_put_dnode(&dn); +free_next: + err = f2fs_truncate_inode_blocks(inode, free_from); +out: + if (lock) + f2fs_unlock_op(sbi, &lc); +free_partial: + /* lastly zero out the first data page */ + if (!err) + err = truncate_partial_data_page(inode, from, truncate_page); +out_err: + trace_f2fs_truncate_blocks_exit(inode, err); + return err; +} + +int f2fs_truncate_blocks(struct inode *inode, u64 from, bool lock) +{ + u64 free_from = from; + int err; + +#ifdef CONFIG_F2FS_FS_COMPRESSION + /* + * for compressed file, only support cluster size + * aligned truncation. + */ + if (f2fs_compressed_file(inode)) + free_from = round_up(from, + F2FS_I(inode)->i_cluster_size << PAGE_SHIFT); +#endif + + err = f2fs_do_truncate_blocks(inode, free_from, lock); + if (err) + return err; + +#ifdef CONFIG_F2FS_FS_COMPRESSION + /* + * For compressed file, after release compress blocks, don't allow write + * direct, but we should allow write direct after truncate to zero. + */ + if (f2fs_compressed_file(inode) && !free_from + && is_inode_flag_set(inode, FI_COMPRESS_RELEASED)) + clear_inode_flag(inode, FI_COMPRESS_RELEASED); + + if (from != free_from) { + err = f2fs_truncate_partial_cluster(inode, from, lock); + if (err) + return err; + } +#endif + + return 0; +} + +int f2fs_truncate(struct inode *inode) +{ + int err; + + if (unlikely(f2fs_cp_error(F2FS_I_SB(inode)))) + return -EIO; + + if (!(S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) || + S_ISLNK(inode->i_mode))) + return 0; + + trace_f2fs_truncate(inode); + + if (time_to_inject(F2FS_I_SB(inode), FAULT_TRUNCATE)) + return -EIO; + + err = f2fs_dquot_initialize(inode); + if (err) + return err; + + /* we should check inline_data size */ + if (!f2fs_may_inline_data(inode)) { + err = f2fs_convert_inline_inode(inode); + if (err) { + /* + * Always truncate page #0 to avoid page cache + * leak in evict() path. + */ + truncate_inode_pages_range(inode->i_mapping, + F2FS_BLK_TO_BYTES(0), + F2FS_BLK_END_BYTES(0)); + return err; + } + } + + err = f2fs_truncate_blocks(inode, i_size_read(inode), true); + if (err) + return err; + + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + f2fs_mark_inode_dirty_sync(inode, false); + return 0; +} + +static bool f2fs_force_buffered_io(struct inode *inode, int rw) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + + if (fsverity_active(inode)) + return true; + if (f2fs_compressed_file(inode)) + return true; + /* + * only force direct read to use buffered IO, for direct write, + * it expects inline data conversion before committing IO. + */ + if (f2fs_has_inline_data(inode) && rw == READ) + return true; + + /* disallow direct IO if any of devices has unaligned blksize */ + if (f2fs_is_multi_device(sbi) && !sbi->aligned_blksize) + return true; + /* + * for blkzoned device, fallback direct IO to buffered IO, so + * all IOs can be serialized by log-structured write. + */ + if (f2fs_sb_has_blkzoned(sbi) && (rw == WRITE) && + !f2fs_is_pinned_file(inode)) + return true; + if (is_sbi_flag_set(sbi, SBI_CP_DISABLED)) + return true; + + return false; +} + +int f2fs_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int query_flags) +{ + struct inode *inode = d_inode(path->dentry); + struct f2fs_inode_info *fi = F2FS_I(inode); + struct f2fs_inode *ri = NULL; + unsigned int flags; + + if (f2fs_has_extra_attr(inode) && + f2fs_sb_has_inode_crtime(F2FS_I_SB(inode)) && + F2FS_FITS_IN_INODE(ri, fi->i_extra_isize, i_crtime)) { + stat->result_mask |= STATX_BTIME; + stat->btime.tv_sec = fi->i_crtime.tv_sec; + stat->btime.tv_nsec = fi->i_crtime.tv_nsec; + } + + /* + * Return the DIO alignment restrictions if requested. + * + * f2fs sometimes supports DIO reads but not DIO writes. STATX_DIOALIGN + * cannot represent that, so in that case we report no DIO support. + */ + if ((request_mask & STATX_DIOALIGN) && S_ISREG(inode->i_mode)) { + unsigned int bsize = i_blocksize(inode); + + stat->result_mask |= STATX_DIOALIGN; + if (!f2fs_force_buffered_io(inode, WRITE)) { + stat->dio_mem_align = bsize; + stat->dio_offset_align = bsize; + } + } + + flags = fi->i_flags; + if (flags & F2FS_COMPR_FL) + stat->attributes |= STATX_ATTR_COMPRESSED; + if (flags & F2FS_APPEND_FL) + stat->attributes |= STATX_ATTR_APPEND; + if (IS_ENCRYPTED(inode)) + stat->attributes |= STATX_ATTR_ENCRYPTED; + if (flags & F2FS_IMMUTABLE_FL) + stat->attributes |= STATX_ATTR_IMMUTABLE; + if (flags & F2FS_NODUMP_FL) + stat->attributes |= STATX_ATTR_NODUMP; + if (IS_VERITY(inode)) + stat->attributes |= STATX_ATTR_VERITY; + + stat->attributes_mask |= (STATX_ATTR_COMPRESSED | + STATX_ATTR_APPEND | + STATX_ATTR_ENCRYPTED | + STATX_ATTR_IMMUTABLE | + STATX_ATTR_NODUMP | + STATX_ATTR_VERITY); + + generic_fillattr(idmap, request_mask, inode, stat); + + /* we need to show initial sectors used for inline_data/dentries */ + if ((S_ISREG(inode->i_mode) && f2fs_has_inline_data(inode)) || + f2fs_has_inline_dentry(inode)) + stat->blocks += (stat->size + 511) >> 9; + + return 0; +} + +#ifdef CONFIG_F2FS_FS_POSIX_ACL +static void __setattr_copy(struct mnt_idmap *idmap, + struct inode *inode, const struct iattr *attr) +{ + unsigned int ia_valid = attr->ia_valid; + + i_uid_update(idmap, attr, inode); + i_gid_update(idmap, attr, inode); + if (ia_valid & ATTR_ATIME) + inode_set_atime_to_ts(inode, attr->ia_atime); + if (ia_valid & ATTR_MTIME) + inode_set_mtime_to_ts(inode, attr->ia_mtime); + if (ia_valid & ATTR_CTIME) + inode_set_ctime_to_ts(inode, attr->ia_ctime); + if (ia_valid & ATTR_MODE) { + umode_t mode = attr->ia_mode; + + if (!in_group_or_capable(idmap, inode, i_gid_into_vfsgid(idmap, inode))) + mode &= ~S_ISGID; + set_acl_inode(inode, mode); + } +} +#else +#define __setattr_copy setattr_copy +#endif + +int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +{ + struct inode *inode = d_inode(dentry); + struct f2fs_inode_info *fi = F2FS_I(inode); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + int err; + + if (unlikely(f2fs_cp_error(sbi))) + return -EIO; + + err = setattr_prepare(idmap, dentry, attr); + if (err) + return err; + + err = fscrypt_prepare_setattr(dentry, attr); + if (err) + return err; + + if (unlikely(IS_IMMUTABLE(inode))) + return -EPERM; + + if (unlikely(IS_APPEND(inode) && + (attr->ia_valid & (ATTR_MODE | ATTR_UID | + ATTR_GID | ATTR_TIMES_SET)))) + return -EPERM; + + if ((attr->ia_valid & ATTR_SIZE)) { + if (mapping_large_folio_support(inode->i_mapping)) + return -EOPNOTSUPP; + if (!f2fs_is_compress_backend_ready(inode) || + IS_DEVICE_ALIASING(inode)) + return -EOPNOTSUPP; + if (is_inode_flag_set(inode, FI_COMPRESS_RELEASED) && + !IS_ALIGNED(attr->ia_size, + F2FS_BLK_TO_BYTES(fi->i_cluster_size))) + return -EINVAL; + /* + * To prevent scattered pin block generation, we don't allow + * smaller/equal size unaligned truncation for pinned file. + * We only support overwrite IO to pinned file, so don't + * care about larger size truncation. + */ + if (f2fs_is_pinned_file(inode) && + attr->ia_size <= i_size_read(inode) && + !IS_ALIGNED(attr->ia_size, + F2FS_BLK_TO_BYTES(CAP_BLKS_PER_SEC(sbi)))) + return -EINVAL; + } + + if (is_quota_modification(idmap, inode, attr)) { + err = f2fs_dquot_initialize(inode); + if (err) + return err; + } + if (i_uid_needs_update(idmap, attr, inode) || + i_gid_needs_update(idmap, attr, inode)) { + struct f2fs_lock_context lc; + + f2fs_lock_op(sbi, &lc); + err = dquot_transfer(idmap, inode, attr); + if (err) { + set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR); + f2fs_unlock_op(sbi, &lc); + return err; + } + /* + * update uid/gid under lock_op(), so that dquot and inode can + * be updated atomically. + */ + i_uid_update(idmap, attr, inode); + i_gid_update(idmap, attr, inode); + f2fs_mark_inode_dirty_sync(inode, true); + f2fs_unlock_op(sbi, &lc); + } + + if (attr->ia_valid & ATTR_SIZE) { + loff_t old_size = i_size_read(inode); + + if (attr->ia_size > MAX_INLINE_DATA(inode)) { + /* + * should convert inline inode before i_size_write to + * keep smaller than inline_data size with inline flag. + */ + err = f2fs_convert_inline_inode(inode); + if (err) + return err; + } + + /* + * wait for inflight dio, blocks should be removed after + * IO completion. + */ + if (attr->ia_size < old_size) + inode_dio_wait(inode); + + f2fs_down_write(&fi->i_gc_rwsem[WRITE]); + filemap_invalidate_lock(inode->i_mapping); + + if (attr->ia_size > old_size) + f2fs_zero_post_eof_page(inode, attr->ia_size, false); + truncate_setsize(inode, attr->ia_size); + + if (attr->ia_size <= old_size) + err = f2fs_truncate(inode); + /* + * do not trim all blocks after i_size if target size is + * larger than i_size. + */ + filemap_invalidate_unlock(inode->i_mapping); + f2fs_up_write(&fi->i_gc_rwsem[WRITE]); + if (err) + return err; + + spin_lock(&fi->i_size_lock); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + fi->last_disk_size = i_size_read(inode); + spin_unlock(&fi->i_size_lock); + } + + __setattr_copy(idmap, inode, attr); + + if (attr->ia_valid & ATTR_MODE) { + err = posix_acl_chmod(idmap, dentry, f2fs_get_inode_mode(inode)); + + if (is_inode_flag_set(inode, FI_ACL_MODE)) { + if (!err) + inode->i_mode = fi->i_acl_mode; + clear_inode_flag(inode, FI_ACL_MODE); + } + } + + /* file size may changed here */ + f2fs_mark_inode_dirty_sync(inode, true); + + /* inode change will produce dirty node pages flushed by checkpoint */ + f2fs_balance_fs(sbi, true); + + return err; +} + +const struct inode_operations f2fs_file_inode_operations = { + .getattr = f2fs_getattr, + .setattr = f2fs_setattr, + .get_inode_acl = f2fs_get_acl, + .set_acl = f2fs_set_acl, + .listxattr = f2fs_listxattr, + .fiemap = f2fs_fiemap, + .fileattr_get = f2fs_fileattr_get, + .fileattr_set = f2fs_fileattr_set, +}; + +static int fill_zero(struct inode *inode, pgoff_t index, + loff_t start, loff_t len) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct folio *folio; + struct f2fs_lock_context lc; + + if (!len) + return 0; + + f2fs_balance_fs(sbi, true); + + f2fs_lock_op(sbi, &lc); + folio = f2fs_get_new_data_folio(inode, NULL, index, false); + f2fs_unlock_op(sbi, &lc); + + if (IS_ERR(folio)) + return PTR_ERR(folio); + + f2fs_folio_wait_writeback(folio, DATA, true, true); + folio_zero_range(folio, start, len); + folio_mark_dirty(folio); + f2fs_folio_put(folio, true); + return 0; +} + +int f2fs_truncate_hole(struct inode *inode, pgoff_t pg_start, pgoff_t pg_end) +{ + int err; + + while (pg_start < pg_end) { + struct dnode_of_data dn; + pgoff_t end_offset, count; + + set_new_dnode(&dn, inode, NULL, NULL, 0); + err = f2fs_get_dnode_of_data(&dn, pg_start, LOOKUP_NODE); + if (err) { + if (err == -ENOENT) { + pg_start = f2fs_get_next_page_offset(&dn, + pg_start); + continue; + } + return err; + } + + end_offset = ADDRS_PER_PAGE(dn.node_folio, inode); + count = min(end_offset - dn.ofs_in_node, pg_end - pg_start); + + f2fs_bug_on(F2FS_I_SB(inode), count == 0 || count > end_offset); + + f2fs_truncate_data_blocks_range(&dn, count); + f2fs_put_dnode(&dn); + + pg_start += count; + } + return 0; +} + +static int f2fs_punch_hole(struct inode *inode, loff_t offset, loff_t len) +{ + pgoff_t pg_start, pg_end; + loff_t off_start, off_end; + int ret; + + ret = f2fs_convert_inline_inode(inode); + if (ret) + return ret; + + f2fs_zero_post_eof_page(inode, offset + len, true); + + pg_start = ((unsigned long long) offset) >> PAGE_SHIFT; + pg_end = ((unsigned long long) offset + len) >> PAGE_SHIFT; + + off_start = offset & (PAGE_SIZE - 1); + off_end = (offset + len) & (PAGE_SIZE - 1); + + if (pg_start == pg_end) { + ret = fill_zero(inode, pg_start, off_start, + off_end - off_start); + if (ret) + return ret; + } else { + if (off_start) { + ret = fill_zero(inode, pg_start++, off_start, + PAGE_SIZE - off_start); + if (ret) + return ret; + } + if (off_end) { + ret = fill_zero(inode, pg_end, 0, off_end); + if (ret) + return ret; + } + + if (pg_start < pg_end) { + loff_t blk_start, blk_end; + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_lock_context lc; + + f2fs_balance_fs(sbi, true); + + blk_start = (loff_t)pg_start << PAGE_SHIFT; + blk_end = (loff_t)pg_end << PAGE_SHIFT; + + f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); + filemap_invalidate_lock(inode->i_mapping); + + truncate_pagecache_range(inode, blk_start, blk_end - 1); + + f2fs_lock_op(sbi, &lc); + ret = f2fs_truncate_hole(inode, pg_start, pg_end); + f2fs_unlock_op(sbi, &lc); + + filemap_invalidate_unlock(inode->i_mapping); + f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); + } + } + + return ret; +} + +static int __read_out_blkaddrs(struct inode *inode, block_t *blkaddr, + int *do_replace, pgoff_t off, pgoff_t len) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct dnode_of_data dn; + int ret, done, i; + +next_dnode: + set_new_dnode(&dn, inode, NULL, NULL, 0); + ret = f2fs_get_dnode_of_data(&dn, off, LOOKUP_NODE_RA); + if (ret && ret != -ENOENT) { + return ret; + } else if (ret == -ENOENT) { + if (dn.max_level == 0) + return -ENOENT; + done = min((pgoff_t)ADDRS_PER_BLOCK(inode) - + dn.ofs_in_node, len); + blkaddr += done; + do_replace += done; + goto next; + } + + done = min((pgoff_t)ADDRS_PER_PAGE(dn.node_folio, inode) - + dn.ofs_in_node, len); + for (i = 0; i < done; i++, blkaddr++, do_replace++, dn.ofs_in_node++) { + *blkaddr = f2fs_data_blkaddr(&dn); + + if (__is_valid_data_blkaddr(*blkaddr) && + !f2fs_is_valid_blkaddr(sbi, *blkaddr, + DATA_GENERIC_ENHANCE)) { + f2fs_put_dnode(&dn); + return -EFSCORRUPTED; + } + + if (!f2fs_is_checkpointed_data(sbi, *blkaddr)) { + + if (f2fs_lfs_mode(sbi)) { + f2fs_put_dnode(&dn); + return -EOPNOTSUPP; + } + + /* do not invalidate this block address */ + f2fs_update_data_blkaddr(&dn, NULL_ADDR); + *do_replace = 1; + } + } + f2fs_put_dnode(&dn); +next: + len -= done; + off += done; + if (len) + goto next_dnode; + return 0; +} + +static int __roll_back_blkaddrs(struct inode *inode, block_t *blkaddr, + int *do_replace, pgoff_t off, int len) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct dnode_of_data dn; + int ret, i; + + for (i = 0; i < len; i++, do_replace++, blkaddr++) { + if (*do_replace == 0) + continue; + + set_new_dnode(&dn, inode, NULL, NULL, 0); + ret = f2fs_get_dnode_of_data(&dn, off + i, LOOKUP_NODE_RA); + if (ret) { + dec_valid_block_count(sbi, inode, 1); + f2fs_invalidate_blocks(sbi, *blkaddr, 1); + } else { + f2fs_update_data_blkaddr(&dn, *blkaddr); + } + f2fs_put_dnode(&dn); + } + return 0; +} + +static int __clone_blkaddrs(struct inode *src_inode, struct inode *dst_inode, + block_t *blkaddr, int *do_replace, + pgoff_t src, pgoff_t dst, pgoff_t len, bool full) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(src_inode); + pgoff_t i = 0; + int ret; + + while (i < len) { + if (blkaddr[i] == NULL_ADDR && !full) { + i++; + continue; + } + + if (do_replace[i] || blkaddr[i] == NULL_ADDR) { + struct dnode_of_data dn; + struct node_info ni; + size_t new_size; + pgoff_t ilen; + + set_new_dnode(&dn, dst_inode, NULL, NULL, 0); + ret = f2fs_get_dnode_of_data(&dn, dst + i, ALLOC_NODE); + if (ret) + return ret; + + ret = f2fs_get_node_info(sbi, dn.nid, &ni, false); + if (ret) { + f2fs_put_dnode(&dn); + return ret; + } + + ilen = min((pgoff_t) + ADDRS_PER_PAGE(dn.node_folio, dst_inode) - + dn.ofs_in_node, len - i); + do { + dn.data_blkaddr = f2fs_data_blkaddr(&dn); + f2fs_truncate_data_blocks_range(&dn, 1); + + if (do_replace[i]) { + f2fs_i_blocks_write(src_inode, + 1, false, false); + f2fs_i_blocks_write(dst_inode, + 1, true, false); + f2fs_replace_block(sbi, &dn, dn.data_blkaddr, + blkaddr[i], ni.version, true, false); + + do_replace[i] = 0; + } + dn.ofs_in_node++; + i++; + new_size = (loff_t)(dst + i) << PAGE_SHIFT; + if (dst_inode->i_size < new_size) + f2fs_i_size_write(dst_inode, new_size); + } while (--ilen && (do_replace[i] || blkaddr[i] == NULL_ADDR)); + + f2fs_put_dnode(&dn); + } else { + struct folio *fsrc, *fdst; + + fsrc = f2fs_get_lock_data_folio(src_inode, + src + i, true); + if (IS_ERR(fsrc)) + return PTR_ERR(fsrc); + fdst = f2fs_get_new_data_folio(dst_inode, NULL, dst + i, + true); + if (IS_ERR(fdst)) { + f2fs_folio_put(fsrc, true); + return PTR_ERR(fdst); + } + + f2fs_folio_wait_writeback(fdst, DATA, true, true); + + memcpy_folio(fdst, 0, fsrc, 0, PAGE_SIZE); + folio_mark_dirty(fdst); + folio_set_f2fs_gcing(fdst); + f2fs_folio_put(fdst, true); + f2fs_folio_put(fsrc, true); + + ret = f2fs_truncate_hole(src_inode, + src + i, src + i + 1); + if (ret) + return ret; + i++; + } + } + return 0; +} + +static int __exchange_data_block(struct inode *src_inode, + struct inode *dst_inode, pgoff_t src, pgoff_t dst, + pgoff_t len, bool full) +{ + block_t *src_blkaddr; + int *do_replace; + pgoff_t olen; + int ret; + + while (len) { + olen = min((pgoff_t)4 * ADDRS_PER_BLOCK(src_inode), len); + + src_blkaddr = f2fs_kvzalloc(F2FS_I_SB(src_inode), + array_size(olen, sizeof(block_t)), + GFP_NOFS); + if (!src_blkaddr) + return -ENOMEM; + + do_replace = f2fs_kvzalloc(F2FS_I_SB(src_inode), + array_size(olen, sizeof(int)), + GFP_NOFS); + if (!do_replace) { + kvfree(src_blkaddr); + return -ENOMEM; + } + + ret = __read_out_blkaddrs(src_inode, src_blkaddr, + do_replace, src, olen); + if (ret) + goto roll_back; + + ret = __clone_blkaddrs(src_inode, dst_inode, src_blkaddr, + do_replace, src, dst, olen, full); + if (ret) + goto roll_back; + + src += olen; + dst += olen; + len -= olen; + + kvfree(src_blkaddr); + kvfree(do_replace); + } + return 0; + +roll_back: + __roll_back_blkaddrs(src_inode, src_blkaddr, do_replace, src, olen); + kvfree(src_blkaddr); + kvfree(do_replace); + return ret; +} + +static int f2fs_do_collapse(struct inode *inode, loff_t offset, loff_t len) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_lock_context lc; + pgoff_t nrpages = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); + pgoff_t start = offset >> PAGE_SHIFT; + pgoff_t end = (offset + len) >> PAGE_SHIFT; + int ret; + + f2fs_balance_fs(sbi, true); + + /* avoid gc operation during block exchange */ + f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); + filemap_invalidate_lock(inode->i_mapping); + + f2fs_zero_post_eof_page(inode, offset + len, false); + + f2fs_lock_op(sbi, &lc); + f2fs_drop_extent_tree(inode); + truncate_pagecache(inode, offset); + ret = __exchange_data_block(inode, inode, end, start, nrpages - end, true); + f2fs_unlock_op(sbi, &lc); + + filemap_invalidate_unlock(inode->i_mapping); + f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); + return ret; +} + +static int f2fs_collapse_range(struct inode *inode, loff_t offset, loff_t len) +{ + loff_t new_size; + int ret; + + if (offset + len >= i_size_read(inode)) + return -EINVAL; + + /* collapse range should be aligned to block size of f2fs. */ + if (offset & (F2FS_BLKSIZE - 1) || len & (F2FS_BLKSIZE - 1)) + return -EINVAL; + + ret = f2fs_convert_inline_inode(inode); + if (ret) + return ret; + + /* write out all dirty pages from offset */ + ret = filemap_write_and_wait_range(inode->i_mapping, offset, LLONG_MAX); + if (ret) + return ret; + + ret = f2fs_do_collapse(inode, offset, len); + if (ret) + return ret; + + /* write out all moved pages, if possible */ + filemap_invalidate_lock(inode->i_mapping); + filemap_write_and_wait_range(inode->i_mapping, offset, LLONG_MAX); + truncate_pagecache(inode, offset); + + new_size = i_size_read(inode) - len; + ret = f2fs_truncate_blocks(inode, new_size, true); + filemap_invalidate_unlock(inode->i_mapping); + if (!ret) + f2fs_i_size_write(inode, new_size); + return ret; +} + +static int f2fs_do_zero_range(struct dnode_of_data *dn, pgoff_t start, + pgoff_t end) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); + pgoff_t index = start; + unsigned int ofs_in_node = dn->ofs_in_node; + blkcnt_t count = 0; + int ret; + + for (; index < end; index++, dn->ofs_in_node++) { + if (f2fs_data_blkaddr(dn) == NULL_ADDR) + count++; + } + + dn->ofs_in_node = ofs_in_node; + ret = f2fs_reserve_new_blocks(dn, count); + if (ret) + return ret; + + dn->ofs_in_node = ofs_in_node; + for (index = start; index < end; index++, dn->ofs_in_node++) { + dn->data_blkaddr = f2fs_data_blkaddr(dn); + /* + * f2fs_reserve_new_blocks will not guarantee entire block + * allocation. + */ + if (dn->data_blkaddr == NULL_ADDR) { + ret = -ENOSPC; + break; + } + + if (dn->data_blkaddr == NEW_ADDR) + continue; + + if (!f2fs_is_valid_blkaddr(sbi, dn->data_blkaddr, + DATA_GENERIC_ENHANCE)) { + ret = -EFSCORRUPTED; + break; + } + + f2fs_invalidate_blocks(sbi, dn->data_blkaddr, 1); + f2fs_set_data_blkaddr(dn, NEW_ADDR); + } + + if (index > start) { + f2fs_update_read_extent_cache_range(dn, start, 0, + index - start); + f2fs_update_age_extent_cache_range(dn, start, index - start); + } + + return ret; +} + +static int f2fs_zero_range(struct inode *inode, loff_t offset, loff_t len, + int mode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct address_space *mapping = inode->i_mapping; + pgoff_t index, pg_start, pg_end; + loff_t new_size = i_size_read(inode); + loff_t off_start, off_end; + int ret = 0; + + ret = inode_newsize_ok(inode, (len + offset)); + if (ret) + return ret; + + ret = f2fs_convert_inline_inode(inode); + if (ret) + return ret; + + ret = filemap_write_and_wait_range(mapping, offset, offset + len - 1); + if (ret) + return ret; + + f2fs_zero_post_eof_page(inode, offset + len, true); + + pg_start = ((unsigned long long) offset) >> PAGE_SHIFT; + pg_end = ((unsigned long long) offset + len) >> PAGE_SHIFT; + + off_start = offset & (PAGE_SIZE - 1); + off_end = (offset + len) & (PAGE_SIZE - 1); + + if (pg_start == pg_end) { + ret = fill_zero(inode, pg_start, off_start, + off_end - off_start); + if (ret) + return ret; + + new_size = max_t(loff_t, new_size, offset + len); + } else { + if (off_start) { + ret = fill_zero(inode, pg_start++, off_start, + PAGE_SIZE - off_start); + if (ret) + return ret; + + new_size = max_t(loff_t, new_size, + (loff_t)pg_start << PAGE_SHIFT); + } + + for (index = pg_start; index < pg_end;) { + struct dnode_of_data dn; + struct f2fs_lock_context lc; + unsigned int end_offset; + pgoff_t end; + + f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); + filemap_invalidate_lock(mapping); + + truncate_pagecache_range(inode, + (loff_t)index << PAGE_SHIFT, + ((loff_t)pg_end << PAGE_SHIFT) - 1); + + f2fs_lock_op(sbi, &lc); + + set_new_dnode(&dn, inode, NULL, NULL, 0); + ret = f2fs_get_dnode_of_data(&dn, index, ALLOC_NODE); + if (ret) { + f2fs_unlock_op(sbi, &lc); + filemap_invalidate_unlock(mapping); + f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); + goto out; + } + + end_offset = ADDRS_PER_PAGE(dn.node_folio, inode); + end = min(pg_end, end_offset - dn.ofs_in_node + index); + + ret = f2fs_do_zero_range(&dn, index, end); + f2fs_put_dnode(&dn); + + f2fs_unlock_op(sbi, &lc); + filemap_invalidate_unlock(mapping); + f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); + + f2fs_balance_fs(sbi, dn.node_changed); + + if (ret) + goto out; + + index = end; + new_size = max_t(loff_t, new_size, + (loff_t)index << PAGE_SHIFT); + } + + if (off_end) { + ret = fill_zero(inode, pg_end, 0, off_end); + if (ret) + goto out; + + new_size = max_t(loff_t, new_size, offset + len); + } + } + +out: + if (new_size > i_size_read(inode)) { + if (mode & FALLOC_FL_KEEP_SIZE) + file_set_keep_isize(inode); + else + f2fs_i_size_write(inode, new_size); + } + return ret; +} + +static int f2fs_insert_range(struct inode *inode, loff_t offset, loff_t len) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct address_space *mapping = inode->i_mapping; + pgoff_t nr, pg_start, pg_end, delta, idx; + loff_t new_size; + int ret = 0; + + new_size = i_size_read(inode) + len; + ret = inode_newsize_ok(inode, new_size); + if (ret) + return ret; + + if (offset >= i_size_read(inode)) + return -EINVAL; + + /* insert range should be aligned to block size of f2fs. */ + if (offset & (F2FS_BLKSIZE - 1) || len & (F2FS_BLKSIZE - 1)) + return -EINVAL; + + ret = f2fs_convert_inline_inode(inode); + if (ret) + return ret; + + f2fs_balance_fs(sbi, true); + + filemap_invalidate_lock(mapping); + ret = f2fs_truncate_blocks(inode, i_size_read(inode), true); + filemap_invalidate_unlock(mapping); + if (ret) + return ret; + + /* write out all dirty pages from offset */ + ret = filemap_write_and_wait_range(mapping, offset, LLONG_MAX); + if (ret) + return ret; + + pg_start = offset >> PAGE_SHIFT; + pg_end = (offset + len) >> PAGE_SHIFT; + delta = pg_end - pg_start; + idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); + + /* avoid gc operation during block exchange */ + f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); + filemap_invalidate_lock(mapping); + + f2fs_zero_post_eof_page(inode, offset + len, false); + truncate_pagecache(inode, offset); + + while (!ret && idx > pg_start) { + struct f2fs_lock_context lc; + + nr = idx - pg_start; + if (nr > delta) + nr = delta; + idx -= nr; + + f2fs_lock_op(sbi, &lc); + f2fs_drop_extent_tree(inode); + + ret = __exchange_data_block(inode, inode, idx, + idx + delta, nr, false); + f2fs_unlock_op(sbi, &lc); + } + filemap_invalidate_unlock(mapping); + f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); + if (ret) + return ret; + + /* write out all moved pages, if possible */ + filemap_invalidate_lock(mapping); + ret = filemap_write_and_wait_range(mapping, offset, LLONG_MAX); + truncate_pagecache(inode, offset); + filemap_invalidate_unlock(mapping); + + if (!ret) + f2fs_i_size_write(inode, new_size); + return ret; +} + +static int f2fs_expand_inode_data(struct inode *inode, loff_t offset, + loff_t len, int mode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_map_blocks map = { .m_next_pgofs = NULL, + .m_next_extent = NULL, .m_seg_type = NO_CHECK_TYPE, + .m_may_create = true }; + struct f2fs_gc_control gc_control = { .victim_segno = NULL_SEGNO, + .init_gc_type = FG_GC, + .should_migrate_blocks = false, + .err_gc_skipped = true, + .nr_free_secs = 0 }; + pgoff_t pg_start, pg_end; + loff_t new_size; + loff_t off_end; + block_t expanded = 0; + int err; + + err = inode_newsize_ok(inode, (len + offset)); + if (err) + return err; + + err = f2fs_convert_inline_inode(inode); + if (err) + return err; + + f2fs_zero_post_eof_page(inode, offset + len, true); + + f2fs_balance_fs(sbi, true); + + pg_start = ((unsigned long long)offset) >> PAGE_SHIFT; + pg_end = ((unsigned long long)offset + len) >> PAGE_SHIFT; + off_end = (offset + len) & (PAGE_SIZE - 1); + + map.m_lblk = pg_start; + map.m_len = pg_end - pg_start; + if (off_end) + map.m_len++; + + if (!map.m_len) + return 0; + + if (f2fs_is_pinned_file(inode)) { + block_t sec_blks = CAP_BLKS_PER_SEC(sbi); + block_t sec_len; + + if (map.m_lblk % sec_blks) { + map.m_lblk = rounddown(map.m_lblk, sec_blks); + map.m_len = pg_end - map.m_lblk; + if (off_end) + map.m_len++; + } + sec_len = roundup(map.m_len, sec_blks); + map.m_len = sec_blks; +next_alloc: + f2fs_down_write(&sbi->pin_sem); + + if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { + if (has_not_enough_free_secs(sbi, 0, 0)) { + f2fs_up_write(&sbi->pin_sem); + err = -ENOSPC; + f2fs_warn_ratelimited(sbi, + "ino:%llu, start:%lu, end:%lu, need to trigger GC to " + "reclaim enough free segment when checkpoint is enabled", + inode->i_ino, pg_start, pg_end); + goto out_err; + } + } + + if (has_not_enough_free_secs(sbi, 0, + sbi->reserved_pin_section)) { + f2fs_down_write_trace(&sbi->gc_lock, &gc_control.lc); + stat_inc_gc_call_count(sbi, FOREGROUND); + err = f2fs_gc(sbi, &gc_control); + if (err && err != -ENODATA) { + f2fs_up_write(&sbi->pin_sem); + goto out_err; + } + } + + err = f2fs_allocate_pinning_section(sbi); + if (err) { + f2fs_up_write(&sbi->pin_sem); + goto out_err; + } + + map.m_seg_type = CURSEG_COLD_DATA_PINNED; + err = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_PRE_DIO); + file_dont_truncate(inode); + + f2fs_up_write(&sbi->pin_sem); + + expanded += map.m_len; + sec_len -= map.m_len; + map.m_lblk += map.m_len; + if (!err && sec_len) + goto next_alloc; + + map.m_len = expanded; + } else { + err = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_PRE_AIO); + expanded = map.m_len; + } +out_err: + if (err) { + pgoff_t last_off; + + if (!expanded) + return err; + + last_off = pg_start + expanded - 1; + + /* update new size to the failed position */ + new_size = (last_off == pg_end) ? offset + len : + (loff_t)(last_off + 1) << PAGE_SHIFT; + } else { + new_size = ((loff_t)pg_end << PAGE_SHIFT) + off_end; + } + + if (new_size > i_size_read(inode)) { + if (mode & FALLOC_FL_KEEP_SIZE) + file_set_keep_isize(inode); + else + f2fs_i_size_write(inode, new_size); + } + + return err; +} + +static long f2fs_fallocate(struct file *file, int mode, + loff_t offset, loff_t len) +{ + struct inode *inode = file_inode(file); + long ret = 0; + + if (unlikely(f2fs_cp_error(F2FS_I_SB(inode)))) + return -EIO; + if (!f2fs_is_checkpoint_ready(F2FS_I_SB(inode))) + return -ENOSPC; + if (!f2fs_is_compress_backend_ready(inode) || IS_DEVICE_ALIASING(inode)) + return -EOPNOTSUPP; + + /* f2fs only support ->fallocate for regular file */ + if (!S_ISREG(inode->i_mode)) + return -EINVAL; + + if (IS_ENCRYPTED(inode) && + (mode & (FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_INSERT_RANGE))) + return -EOPNOTSUPP; + + if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE | + FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_ZERO_RANGE | + FALLOC_FL_INSERT_RANGE)) + return -EOPNOTSUPP; + + inode_lock(inode); + + /* + * Pinned file should not support partial truncation since the block + * can be used by applications. + */ + if ((f2fs_compressed_file(inode) || f2fs_is_pinned_file(inode)) && + (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_COLLAPSE_RANGE | + FALLOC_FL_ZERO_RANGE | FALLOC_FL_INSERT_RANGE))) { + ret = -EOPNOTSUPP; + goto out; + } + + ret = file_modified(file); + if (ret) + goto out; + + /* + * wait for inflight dio, blocks should be removed after IO + * completion. + */ + inode_dio_wait(inode); + + if (mode & FALLOC_FL_PUNCH_HOLE) { + if (offset >= inode->i_size) + goto out; + + ret = f2fs_punch_hole(inode, offset, len); + } else if (mode & FALLOC_FL_COLLAPSE_RANGE) { + ret = f2fs_collapse_range(inode, offset, len); + } else if (mode & FALLOC_FL_ZERO_RANGE) { + ret = f2fs_zero_range(inode, offset, len, mode); + } else if (mode & FALLOC_FL_INSERT_RANGE) { + ret = f2fs_insert_range(inode, offset, len); + } else { + ret = f2fs_expand_inode_data(inode, offset, len, mode); + } + + if (!ret) { + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + f2fs_mark_inode_dirty_sync(inode, false); + f2fs_update_time(F2FS_I_SB(inode), REQ_TIME); + } + +out: + inode_unlock(inode); + + trace_f2fs_fallocate(inode, mode, offset, len, ret); + return ret; +} + +static int f2fs_release_file(struct inode *inode, struct file *filp) +{ + if (atomic_dec_and_test(&F2FS_I(inode)->open_count)) + f2fs_remove_donate_inode(inode); + + /* + * f2fs_release_file is called at every close calls. So we should + * not drop any inmemory pages by close called by other process. + */ + if (!(filp->f_mode & FMODE_WRITE) || + atomic_read(&inode->i_writecount) != 1) + return 0; + + inode_lock(inode); + f2fs_abort_atomic_write(inode, true); + inode_unlock(inode); + + return 0; +} + +static int f2fs_file_flush(struct file *file, fl_owner_t id) +{ + struct inode *inode = file_inode(file); + + /* + * If the process doing a transaction is crashed, we should do + * roll-back. Otherwise, other reader/write can see corrupted database + * until all the writers close its file. Since this should be done + * before dropping file lock, it needs to do in ->flush. + */ + if (F2FS_I(inode)->atomic_write_task == current && + (current->flags & PF_EXITING)) { + inode_lock(inode); + f2fs_abort_atomic_write(inode, true); + inode_unlock(inode); + } + + return 0; +} + +static int f2fs_setflags_common(struct inode *inode, u32 iflags, u32 mask) +{ + struct f2fs_inode_info *fi = F2FS_I(inode); + u32 masked_flags = fi->i_flags & mask; + + /* mask can be shrunk by flags_valid selector */ + iflags &= mask; + + /* Is it quota file? Do not allow user to mess with it */ + if (IS_NOQUOTA(inode)) + return -EPERM; + + if ((iflags ^ masked_flags) & F2FS_CASEFOLD_FL) { + if (!f2fs_sb_has_casefold(F2FS_I_SB(inode))) + return -EOPNOTSUPP; + if (!f2fs_empty_dir(inode)) + return -ENOTEMPTY; + } + + if (iflags & (F2FS_COMPR_FL | F2FS_NOCOMP_FL)) { + if (!f2fs_sb_has_compression(F2FS_I_SB(inode))) + return -EOPNOTSUPP; + if ((iflags & F2FS_COMPR_FL) && (iflags & F2FS_NOCOMP_FL)) + return -EINVAL; + } + + if ((iflags ^ masked_flags) & F2FS_COMPR_FL) { + if (masked_flags & F2FS_COMPR_FL) { + if (!f2fs_disable_compressed_file(inode)) + return -EINVAL; + } else { + /* try to convert inline_data to support compression */ + int err = f2fs_convert_inline_inode(inode); + if (err) + return err; + + f2fs_down_write(&fi->i_sem); + if (!f2fs_may_compress(inode) || + atomic_read(&fi->writeback) || + (S_ISREG(inode->i_mode) && + F2FS_HAS_BLOCKS(inode))) { + f2fs_up_write(&fi->i_sem); + return -EINVAL; + } + err = set_compress_context(inode); + f2fs_up_write(&fi->i_sem); + + if (err) + return err; + } + } + + fi->i_flags = iflags | (fi->i_flags & ~mask); + f2fs_bug_on(F2FS_I_SB(inode), (fi->i_flags & F2FS_COMPR_FL) && + (fi->i_flags & F2FS_NOCOMP_FL)); + + if (fi->i_flags & F2FS_PROJINHERIT_FL) + set_inode_flag(inode, FI_PROJ_INHERIT); + else + clear_inode_flag(inode, FI_PROJ_INHERIT); + + inode_set_ctime_current(inode); + f2fs_set_inode_flags(inode); + f2fs_mark_inode_dirty_sync(inode, true); + return 0; +} + +/* FS_IOC_[GS]ETFLAGS and FS_IOC_FS[GS]ETXATTR support */ + +/* + * To make a new on-disk f2fs i_flag gettable via FS_IOC_GETFLAGS, add an entry + * for it to f2fs_fsflags_map[], and add its FS_*_FL equivalent to + * F2FS_GETTABLE_FS_FL. To also make it settable via FS_IOC_SETFLAGS, also add + * its FS_*_FL equivalent to F2FS_SETTABLE_FS_FL. + * + * Translating flags to fsx_flags value used by FS_IOC_FSGETXATTR and + * FS_IOC_FSSETXATTR is done by the VFS. + */ + +static const struct { + u32 iflag; + u32 fsflag; +} f2fs_fsflags_map[] = { + { F2FS_COMPR_FL, FS_COMPR_FL }, + { F2FS_SYNC_FL, FS_SYNC_FL }, + { F2FS_IMMUTABLE_FL, FS_IMMUTABLE_FL }, + { F2FS_APPEND_FL, FS_APPEND_FL }, + { F2FS_NODUMP_FL, FS_NODUMP_FL }, + { F2FS_NOATIME_FL, FS_NOATIME_FL }, + { F2FS_NOCOMP_FL, FS_NOCOMP_FL }, + { F2FS_INDEX_FL, FS_INDEX_FL }, + { F2FS_DIRSYNC_FL, FS_DIRSYNC_FL }, + { F2FS_PROJINHERIT_FL, FS_PROJINHERIT_FL }, + { F2FS_CASEFOLD_FL, FS_CASEFOLD_FL }, +}; + +#define F2FS_GETTABLE_FS_FL ( \ + FS_COMPR_FL | \ + FS_SYNC_FL | \ + FS_IMMUTABLE_FL | \ + FS_APPEND_FL | \ + FS_NODUMP_FL | \ + FS_NOATIME_FL | \ + FS_NOCOMP_FL | \ + FS_INDEX_FL | \ + FS_DIRSYNC_FL | \ + FS_PROJINHERIT_FL | \ + FS_ENCRYPT_FL | \ + FS_INLINE_DATA_FL | \ + FS_NOCOW_FL | \ + FS_VERITY_FL | \ + FS_CASEFOLD_FL) + +#define F2FS_SETTABLE_FS_FL ( \ + FS_COMPR_FL | \ + FS_SYNC_FL | \ + FS_IMMUTABLE_FL | \ + FS_APPEND_FL | \ + FS_NODUMP_FL | \ + FS_NOATIME_FL | \ + FS_NOCOMP_FL | \ + FS_DIRSYNC_FL | \ + FS_PROJINHERIT_FL | \ + FS_CASEFOLD_FL) + +/* Convert f2fs on-disk i_flags to FS_IOC_{GET,SET}FLAGS flags */ +static inline u32 f2fs_iflags_to_fsflags(u32 iflags) +{ + u32 fsflags = 0; + int i; + + for (i = 0; i < ARRAY_SIZE(f2fs_fsflags_map); i++) + if (iflags & f2fs_fsflags_map[i].iflag) + fsflags |= f2fs_fsflags_map[i].fsflag; + + return fsflags; +} + +/* Convert FS_IOC_{GET,SET}FLAGS flags to f2fs on-disk i_flags */ +static inline u32 f2fs_fsflags_to_iflags(u32 fsflags) +{ + u32 iflags = 0; + int i; + + for (i = 0; i < ARRAY_SIZE(f2fs_fsflags_map); i++) + if (fsflags & f2fs_fsflags_map[i].fsflag) + iflags |= f2fs_fsflags_map[i].iflag; + + return iflags; +} + +static int f2fs_ioc_getversion(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + + return put_user(inode->i_generation, (int __user *)arg); +} + +static int f2fs_ioc_start_atomic_write(struct file *filp, bool truncate) +{ + struct inode *inode = file_inode(filp); + struct mnt_idmap *idmap = file_mnt_idmap(filp); + struct f2fs_inode_info *fi = F2FS_I(inode); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + loff_t isize; + int ret; + + if (!(filp->f_mode & FMODE_WRITE)) + return -EBADF; + + if (!inode_owner_or_capable(idmap, inode)) + return -EACCES; + + if (!S_ISREG(inode->i_mode)) + return -EINVAL; + + if (filp->f_flags & O_DIRECT) + return -EINVAL; + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + + inode_lock(inode); + + if (!f2fs_disable_compressed_file(inode) || + f2fs_is_pinned_file(inode)) { + ret = -EINVAL; + goto out; + } + + if (f2fs_is_atomic_file(inode)) + goto out; + + ret = f2fs_convert_inline_inode(inode); + if (ret) + goto out; + + f2fs_down_write(&fi->i_gc_rwsem[WRITE]); + f2fs_down_write(&fi->i_gc_rwsem[READ]); + + /* + * Should wait end_io to count F2FS_WB_CP_DATA correctly by + * f2fs_is_atomic_file. + */ + if (get_dirty_pages(inode)) + f2fs_warn(sbi, "Unexpected flush for atomic writes: ino=%llu, npages=%u", + inode->i_ino, get_dirty_pages(inode)); + ret = filemap_write_and_wait_range(inode->i_mapping, 0, LLONG_MAX); + if (ret) + goto out_unlock; + + /* Check if the inode already has a COW inode */ + if (fi->cow_inode == NULL) { + /* Create a COW inode for atomic write */ + struct dentry *dentry = file_dentry(filp); + struct inode *dir = d_inode(dentry->d_parent); + + ret = f2fs_get_tmpfile(idmap, dir, &fi->cow_inode); + if (ret) + goto out_unlock; + + set_inode_flag(fi->cow_inode, FI_COW_FILE); + clear_inode_flag(fi->cow_inode, FI_INLINE_DATA); + + /* Set the COW inode's atomic_inode to the atomic inode */ + F2FS_I(fi->cow_inode)->atomic_inode = inode; + } else { + /* Reuse the already created COW inode */ + f2fs_bug_on(sbi, get_dirty_pages(fi->cow_inode)); + + invalidate_mapping_pages(fi->cow_inode->i_mapping, 0, -1); + + ret = f2fs_do_truncate_blocks(fi->cow_inode, 0, true); + if (ret) + goto out_unlock; + } + + f2fs_write_inode(inode, NULL); + + stat_inc_atomic_inode(inode); + + set_inode_flag(inode, FI_ATOMIC_FILE); + + isize = i_size_read(inode); + fi->original_i_size = isize; + if (truncate) { + set_inode_flag(inode, FI_ATOMIC_REPLACE); + truncate_inode_pages_final(inode->i_mapping); + f2fs_i_size_write(inode, 0); + isize = 0; + } + f2fs_i_size_write(fi->cow_inode, isize); + +out_unlock: + f2fs_up_write(&fi->i_gc_rwsem[READ]); + f2fs_up_write(&fi->i_gc_rwsem[WRITE]); + if (ret) + goto out; + + f2fs_update_time(sbi, REQ_TIME); + fi->atomic_write_task = current; + stat_update_max_atomic_write(inode); + fi->atomic_write_cnt = 0; +out: + inode_unlock(inode); + mnt_drop_write_file(filp); + return ret; +} + +static int f2fs_ioc_commit_atomic_write(struct file *filp) +{ + struct inode *inode = file_inode(filp); + struct mnt_idmap *idmap = file_mnt_idmap(filp); + int ret; + + if (!(filp->f_mode & FMODE_WRITE)) + return -EBADF; + + if (!inode_owner_or_capable(idmap, inode)) + return -EACCES; + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + + f2fs_balance_fs(F2FS_I_SB(inode), true); + + inode_lock(inode); + + if (f2fs_is_atomic_file(inode)) { + ret = f2fs_commit_atomic_write(inode); + if (!ret) + ret = f2fs_do_sync_file(filp, 0, LLONG_MAX, 0, true); + + f2fs_abort_atomic_write(inode, ret); + } else { + ret = f2fs_do_sync_file(filp, 0, LLONG_MAX, 1, false); + } + + inode_unlock(inode); + mnt_drop_write_file(filp); + return ret; +} + +static int f2fs_ioc_abort_atomic_write(struct file *filp) +{ + struct inode *inode = file_inode(filp); + struct mnt_idmap *idmap = file_mnt_idmap(filp); + int ret; + + if (!(filp->f_mode & FMODE_WRITE)) + return -EBADF; + + if (!inode_owner_or_capable(idmap, inode)) + return -EACCES; + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + + inode_lock(inode); + + f2fs_abort_atomic_write(inode, true); + + inode_unlock(inode); + + mnt_drop_write_file(filp); + f2fs_update_time(F2FS_I_SB(inode), REQ_TIME); + return ret; +} + +int f2fs_do_shutdown(struct f2fs_sb_info *sbi, unsigned int flag, + bool readonly, bool need_lock) +{ + struct super_block *sb = sbi->sb; + int ret = 0; + + switch (flag) { + case F2FS_GOING_DOWN_FULLSYNC: + ret = bdev_freeze(sb->s_bdev); + if (ret) + goto out; + f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_SHUTDOWN); + bdev_thaw(sb->s_bdev); + break; + case F2FS_GOING_DOWN_METASYNC: + /* do checkpoint only */ + ret = f2fs_sync_fs(sb, 1); + if (ret) { + if (ret == -EIO) + ret = 0; + goto out; + } + f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_SHUTDOWN); + break; + case F2FS_GOING_DOWN_NOSYNC: + f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_SHUTDOWN); + break; + case F2FS_GOING_DOWN_METAFLUSH: + f2fs_sync_meta_pages(sbi, LONG_MAX, FS_META_IO); + f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_SHUTDOWN); + break; + case F2FS_GOING_DOWN_NEED_FSCK: + set_sbi_flag(sbi, SBI_NEED_FSCK); + set_sbi_flag(sbi, SBI_CP_DISABLED_QUICK); + set_sbi_flag(sbi, SBI_IS_DIRTY); + /* do checkpoint only */ + ret = f2fs_sync_fs(sb, 1); + if (ret == -EIO) + ret = 0; + goto out; + default: + ret = -EINVAL; + goto out; + } + + if (readonly) + goto out; + + /* + * grab sb->s_umount to avoid racing w/ remount() and other shutdown + * paths. + */ + if (need_lock) + down_write(&sbi->sb->s_umount); + + f2fs_stop_gc_thread(sbi); + f2fs_stop_discard_thread(sbi); + + f2fs_drop_discard_cmd(sbi); + clear_opt(sbi, DISCARD); + + if (need_lock) + up_write(&sbi->sb->s_umount); + + f2fs_update_time(sbi, REQ_TIME); +out: + + trace_f2fs_shutdown(sbi, flag, ret); + + return ret; +} + +static int f2fs_ioc_shutdown(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + __u32 in; + int ret; + bool need_drop = false, readonly = false; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (get_user(in, (__u32 __user *)arg)) + return -EFAULT; + + if (in != F2FS_GOING_DOWN_FULLSYNC) { + ret = mnt_want_write_file(filp); + if (ret) { + if (ret != -EROFS) + return ret; + + /* fallback to nosync shutdown for readonly fs */ + in = F2FS_GOING_DOWN_NOSYNC; + readonly = true; + } else { + need_drop = true; + } + } + + ret = f2fs_do_shutdown(sbi, in, readonly, true); + + if (need_drop) + mnt_drop_write_file(filp); + + return ret; +} + +static int f2fs_keep_noreuse_range(struct inode *inode, + loff_t offset, loff_t len) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + u64 max_bytes = F2FS_BLK_TO_BYTES(max_file_blocks(inode)); + u64 start, end; + int ret = 0; + + if (!S_ISREG(inode->i_mode)) + return 0; + + if (offset >= max_bytes || len > max_bytes || + (offset + len) > max_bytes) + return 0; + + start = offset >> PAGE_SHIFT; + end = DIV_ROUND_UP(offset + len, PAGE_SIZE); + + inode_lock(inode); + if (f2fs_is_atomic_file(inode)) { + inode_unlock(inode); + return 0; + } + + spin_lock(&sbi->inode_lock[DONATE_INODE]); + /* let's remove the range, if len = 0 */ + if (!len) { + if (!list_empty(&F2FS_I(inode)->gdonate_list)) { + list_del_init(&F2FS_I(inode)->gdonate_list); + sbi->donate_files--; + if (is_inode_flag_set(inode, FI_DONATE_FINISHED)) + ret = -EALREADY; + else + set_inode_flag(inode, FI_DONATE_FINISHED); + } else + ret = -ENOENT; + } else { + if (list_empty(&F2FS_I(inode)->gdonate_list)) { + list_add_tail(&F2FS_I(inode)->gdonate_list, + &sbi->inode_list[DONATE_INODE]); + sbi->donate_files++; + } else { + list_move_tail(&F2FS_I(inode)->gdonate_list, + &sbi->inode_list[DONATE_INODE]); + } + F2FS_I(inode)->donate_start = start; + F2FS_I(inode)->donate_end = end - 1; + clear_inode_flag(inode, FI_DONATE_FINISHED); + } + spin_unlock(&sbi->inode_lock[DONATE_INODE]); + inode_unlock(inode); + + return ret; +} + +static int f2fs_ioc_fitrim(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct fstrim_range range; + int ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (!f2fs_hw_support_discard(sbi)) + return -EOPNOTSUPP; + + if (copy_from_user(&range, (struct fstrim_range __user *)arg, + sizeof(range))) + return -EFAULT; + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + + range.minlen = max_t(unsigned int, range.minlen, + f2fs_hw_discard_granularity(sbi)); + ret = f2fs_trim_fs(sbi, &range); + mnt_drop_write_file(filp); + if (ret < 0) + return ret; + + if (copy_to_user((struct fstrim_range __user *)arg, &range, + sizeof(range))) + return -EFAULT; + f2fs_update_time(sbi, REQ_TIME); + return 0; +} + +static bool uuid_is_nonzero(__u8 u[16]) +{ + int i; + + for (i = 0; i < 16; i++) + if (u[i]) + return true; + return false; +} + +static int f2fs_ioc_set_encryption_policy(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + int ret; + + if (!f2fs_sb_has_encrypt(F2FS_I_SB(inode))) + return -EOPNOTSUPP; + + ret = fscrypt_ioctl_set_policy(filp, (const void __user *)arg); + f2fs_update_time(F2FS_I_SB(inode), REQ_TIME); + return ret; +} + +static int f2fs_ioc_get_encryption_policy(struct file *filp, unsigned long arg) +{ + if (!f2fs_sb_has_encrypt(F2FS_I_SB(file_inode(filp)))) + return -EOPNOTSUPP; + return fscrypt_ioctl_get_policy(filp, (void __user *)arg); +} + +static int f2fs_ioc_get_encryption_pwsalt(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + u8 encrypt_pw_salt[16]; + int err; + + if (!f2fs_sb_has_encrypt(sbi)) + return -EOPNOTSUPP; + + err = mnt_want_write_file(filp); + if (err) + return err; + + f2fs_down_write(&sbi->sb_lock); + + if (uuid_is_nonzero(sbi->raw_super->encrypt_pw_salt)) + goto got_it; + + /* update superblock with uuid */ + generate_random_uuid(sbi->raw_super->encrypt_pw_salt); + + err = f2fs_commit_super(sbi, false); + if (err) { + /* undo new data */ + memset(sbi->raw_super->encrypt_pw_salt, 0, 16); + goto out_err; + } +got_it: + memcpy(encrypt_pw_salt, sbi->raw_super->encrypt_pw_salt, 16); +out_err: + f2fs_up_write(&sbi->sb_lock); + mnt_drop_write_file(filp); + + if (!err && copy_to_user((__u8 __user *)arg, encrypt_pw_salt, 16)) + err = -EFAULT; + + return err; +} + +static int f2fs_ioc_get_encryption_policy_ex(struct file *filp, + unsigned long arg) +{ + if (!f2fs_sb_has_encrypt(F2FS_I_SB(file_inode(filp)))) + return -EOPNOTSUPP; + + return fscrypt_ioctl_get_policy_ex(filp, (void __user *)arg); +} + +static int f2fs_ioc_add_encryption_key(struct file *filp, unsigned long arg) +{ + if (!f2fs_sb_has_encrypt(F2FS_I_SB(file_inode(filp)))) + return -EOPNOTSUPP; + + return fscrypt_ioctl_add_key(filp, (void __user *)arg); +} + +static int f2fs_ioc_remove_encryption_key(struct file *filp, unsigned long arg) +{ + if (!f2fs_sb_has_encrypt(F2FS_I_SB(file_inode(filp)))) + return -EOPNOTSUPP; + + return fscrypt_ioctl_remove_key(filp, (void __user *)arg); +} + +static int f2fs_ioc_remove_encryption_key_all_users(struct file *filp, + unsigned long arg) +{ + if (!f2fs_sb_has_encrypt(F2FS_I_SB(file_inode(filp)))) + return -EOPNOTSUPP; + + return fscrypt_ioctl_remove_key_all_users(filp, (void __user *)arg); +} + +static int f2fs_ioc_get_encryption_key_status(struct file *filp, + unsigned long arg) +{ + if (!f2fs_sb_has_encrypt(F2FS_I_SB(file_inode(filp)))) + return -EOPNOTSUPP; + + return fscrypt_ioctl_get_key_status(filp, (void __user *)arg); +} + +static int f2fs_ioc_get_encryption_nonce(struct file *filp, unsigned long arg) +{ + if (!f2fs_sb_has_encrypt(F2FS_I_SB(file_inode(filp)))) + return -EOPNOTSUPP; + + return fscrypt_ioctl_get_nonce(filp, (void __user *)arg); +} + +static int f2fs_ioc_gc(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_gc_control gc_control = { .victim_segno = NULL_SEGNO, + .no_bg_gc = false, + .should_migrate_blocks = false, + .nr_free_secs = 0 }; + __u32 sync; + int ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (get_user(sync, (__u32 __user *)arg)) + return -EFAULT; + + if (f2fs_readonly(sbi->sb)) + return -EROFS; + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + + if (!sync) { + if (!f2fs_down_write_trylock_trace(&sbi->gc_lock, + &gc_control.lc)) { + ret = -EBUSY; + goto out; + } + } else { + f2fs_down_write_trace(&sbi->gc_lock, &gc_control.lc); + } + + gc_control.init_gc_type = sync ? FG_GC : BG_GC; + gc_control.err_gc_skipped = sync; + stat_inc_gc_call_count(sbi, FOREGROUND); + ret = f2fs_gc(sbi, &gc_control); +out: + mnt_drop_write_file(filp); + return ret; +} + +static int __f2fs_ioc_gc_range(struct file *filp, struct f2fs_gc_range *range) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(file_inode(filp)); + struct f2fs_gc_control gc_control = { + .init_gc_type = range->sync ? FG_GC : BG_GC, + .no_bg_gc = false, + .should_migrate_blocks = false, + .err_gc_skipped = range->sync, + .nr_free_secs = 0 }; + u64 end; + int ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (f2fs_readonly(sbi->sb)) + return -EROFS; + + end = range->start + range->len; + if (end < range->start || range->start < MAIN_BLKADDR(sbi) || + end >= MAX_BLKADDR(sbi)) + return -EINVAL; + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + +do_more: + if (!range->sync) { + if (!f2fs_down_write_trylock_trace(&sbi->gc_lock, &gc_control.lc)) { + ret = -EBUSY; + goto out; + } + } else { + f2fs_down_write_trace(&sbi->gc_lock, &gc_control.lc); + } + + gc_control.victim_segno = GET_SEGNO(sbi, range->start); + stat_inc_gc_call_count(sbi, FOREGROUND); + ret = f2fs_gc(sbi, &gc_control); + if (ret) { + if (ret == -EBUSY) + ret = -EAGAIN; + goto out; + } + range->start += CAP_BLKS_PER_SEC(sbi); + if (range->start <= end) + goto do_more; +out: + mnt_drop_write_file(filp); + return ret; +} + +static int f2fs_ioc_gc_range(struct file *filp, unsigned long arg) +{ + struct f2fs_gc_range range; + + if (copy_from_user(&range, (struct f2fs_gc_range __user *)arg, + sizeof(range))) + return -EFAULT; + return __f2fs_ioc_gc_range(filp, &range); +} + +static int f2fs_ioc_write_checkpoint(struct file *filp) +{ + struct inode *inode = file_inode(filp); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + int ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (f2fs_readonly(sbi->sb)) + return -EROFS; + + if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { + f2fs_info(sbi, "Skipping Checkpoint. Checkpoints currently disabled."); + return -EINVAL; + } + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + + ret = f2fs_sync_fs(sbi->sb, 1); + + mnt_drop_write_file(filp); + return ret; +} + +static int f2fs_defragment_range(struct f2fs_sb_info *sbi, + struct file *filp, + struct f2fs_defragment *range) +{ + struct inode *inode = file_inode(filp); + struct f2fs_map_blocks map = { .m_next_extent = NULL, + .m_seg_type = NO_CHECK_TYPE, + .m_may_create = false }; + struct extent_info ei = {}; + pgoff_t pg_start, pg_end, next_pgofs; + unsigned int total = 0, sec_num; + block_t blk_end = 0; + bool fragmented = false; + int err; + + f2fs_balance_fs(sbi, true); + + inode_lock(inode); + pg_start = range->start >> PAGE_SHIFT; + pg_end = min_t(pgoff_t, + (range->start + range->len) >> PAGE_SHIFT, + DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE)); + + if (is_inode_flag_set(inode, FI_COMPRESS_RELEASED) || + f2fs_is_atomic_file(inode)) { + err = -EINVAL; + goto unlock_out; + } + + /* if in-place-update policy is enabled, don't waste time here */ + set_inode_flag(inode, FI_OPU_WRITE); + if (f2fs_should_update_inplace(inode, NULL)) { + err = -EINVAL; + goto out; + } + + /* writeback all dirty pages in the range */ + err = filemap_write_and_wait_range(inode->i_mapping, + pg_start << PAGE_SHIFT, + (pg_end << PAGE_SHIFT) - 1); + if (err) + goto out; + + /* + * lookup mapping info in extent cache, skip defragmenting if physical + * block addresses are continuous. + */ + if (f2fs_lookup_read_extent_cache(inode, pg_start, &ei)) { + if ((pgoff_t)ei.fofs + ei.len >= pg_end) + goto out; + } + + map.m_lblk = pg_start; + map.m_next_pgofs = &next_pgofs; + + /* + * lookup mapping info in dnode page cache, skip defragmenting if all + * physical block addresses are continuous even if there are hole(s) + * in logical blocks. + */ + while (map.m_lblk < pg_end) { + map.m_len = pg_end - map.m_lblk; + err = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_DEFAULT); + if (err) + goto out; + + if (!(map.m_flags & F2FS_MAP_FLAGS)) { + map.m_lblk = next_pgofs; + continue; + } + + if (blk_end && blk_end != map.m_pblk) + fragmented = true; + + /* record total count of block that we're going to move */ + total += map.m_len; + + blk_end = map.m_pblk + map.m_len; + + map.m_lblk += map.m_len; + } + + if (!fragmented) { + total = 0; + goto out; + } + + sec_num = DIV_ROUND_UP(total, CAP_BLKS_PER_SEC(sbi)); + + /* + * make sure there are enough free section for LFS allocation, this can + * avoid defragment running in SSR mode when free section are allocated + * intensively + */ + if (has_not_enough_free_secs(sbi, 0, sec_num)) { + err = -EAGAIN; + goto out; + } + + map.m_lblk = pg_start; + map.m_len = pg_end - pg_start; + total = 0; + + while (map.m_lblk < pg_end) { + pgoff_t idx; + int cnt = 0; + +do_map: + map.m_len = pg_end - map.m_lblk; + err = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_DEFAULT); + if (err) + goto clear_out; + + if (!(map.m_flags & F2FS_MAP_FLAGS)) { + map.m_lblk = next_pgofs; + goto check; + } + + set_inode_flag(inode, FI_SKIP_WRITES); + + idx = map.m_lblk; + while (idx < map.m_lblk + map.m_len && + cnt < BLKS_PER_SEG(sbi)) { + struct folio *folio; + + folio = f2fs_get_lock_data_folio(inode, idx, true); + if (IS_ERR(folio)) { + err = PTR_ERR(folio); + goto clear_out; + } + + f2fs_folio_wait_writeback(folio, DATA, true, true); + + folio_mark_dirty(folio); + folio_set_f2fs_gcing(folio); + f2fs_folio_put(folio, true); + + idx++; + cnt++; + total++; + } + + map.m_lblk = idx; +check: + if (map.m_lblk < pg_end && cnt < BLKS_PER_SEG(sbi)) + goto do_map; + + clear_inode_flag(inode, FI_SKIP_WRITES); + + err = filemap_fdatawrite(inode->i_mapping); + if (err) + goto out; + } +clear_out: + clear_inode_flag(inode, FI_SKIP_WRITES); +out: + clear_inode_flag(inode, FI_OPU_WRITE); +unlock_out: + inode_unlock(inode); + if (!err) { + range->len = (u64)total << PAGE_SHIFT; + stat_inc_defrag_blk_count(sbi, total); + } + return err; +} + +static int f2fs_ioc_defragment(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_defragment range; + int err; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (!S_ISREG(inode->i_mode)) + return -EINVAL; + + if (f2fs_readonly(sbi->sb)) + return -EROFS; + + if (copy_from_user(&range, (struct f2fs_defragment __user *)arg, + sizeof(range))) + return -EFAULT; + + /* verify alignment of offset & size */ + if (range.start & (F2FS_BLKSIZE - 1) || range.len & (F2FS_BLKSIZE - 1)) + return -EINVAL; + + if (unlikely((range.start + range.len) >> PAGE_SHIFT > + max_file_blocks(inode))) + return -EINVAL; + + err = mnt_want_write_file(filp); + if (err) + return err; + + err = f2fs_defragment_range(sbi, filp, &range); + mnt_drop_write_file(filp); + + if (range.len) + f2fs_update_time(sbi, REQ_TIME); + if (err < 0) + return err; + + if (copy_to_user((struct f2fs_defragment __user *)arg, &range, + sizeof(range))) + return -EFAULT; + + return 0; +} + +static int f2fs_move_file_range(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, size_t len) +{ + struct inode *src = file_inode(file_in); + struct inode *dst = file_inode(file_out); + struct f2fs_sb_info *sbi = F2FS_I_SB(src); + struct f2fs_lock_context lc; + size_t olen = len, dst_max_i_size = 0; + size_t dst_osize; + int ret; + + if (file_in->f_path.mnt != file_out->f_path.mnt || + src->i_sb != dst->i_sb) + return -EXDEV; + + if (unlikely(f2fs_readonly(src->i_sb))) + return -EROFS; + + if (!S_ISREG(src->i_mode) || !S_ISREG(dst->i_mode)) + return -EINVAL; + + if (IS_ENCRYPTED(src) || IS_ENCRYPTED(dst)) + return -EOPNOTSUPP; + + if (pos_out < 0 || pos_in < 0) + return -EINVAL; + + if (src == dst) { + if (pos_in == pos_out) + return 0; + if (pos_out > pos_in && pos_out < pos_in + len) + return -EINVAL; + } + + inode_lock(src); + if (src != dst) { + ret = -EBUSY; + if (!inode_trylock(dst)) + goto out; + } + + if (f2fs_compressed_file(src) || f2fs_compressed_file(dst) || + f2fs_is_pinned_file(src) || f2fs_is_pinned_file(dst)) { + ret = -EOPNOTSUPP; + goto out_unlock; + } + + if (f2fs_is_atomic_file(src) || f2fs_is_atomic_file(dst)) { + ret = -EINVAL; + goto out_unlock; + } + + ret = -EINVAL; + if (pos_in + len > src->i_size || pos_in + len < pos_in) + goto out_unlock; + if (len == 0) + olen = len = src->i_size - pos_in; + if (pos_in + len == src->i_size) + len = ALIGN(src->i_size, F2FS_BLKSIZE) - pos_in; + if (len == 0) { + ret = 0; + goto out_unlock; + } + + dst_osize = dst->i_size; + if (pos_out + olen > dst->i_size) + dst_max_i_size = pos_out + olen; + + /* verify the end result is block aligned */ + if (!IS_ALIGNED(pos_in, F2FS_BLKSIZE) || + !IS_ALIGNED(pos_in + len, F2FS_BLKSIZE) || + !IS_ALIGNED(pos_out, F2FS_BLKSIZE)) + goto out_unlock; + + ret = f2fs_convert_inline_inode(src); + if (ret) + goto out_unlock; + + ret = f2fs_convert_inline_inode(dst); + if (ret) + goto out_unlock; + + /* write out all dirty pages from offset */ + ret = filemap_write_and_wait_range(src->i_mapping, + pos_in, pos_in + len); + if (ret) + goto out_unlock; + + ret = filemap_write_and_wait_range(dst->i_mapping, + pos_out, pos_out + len); + if (ret) + goto out_unlock; + + f2fs_balance_fs(sbi, true); + + f2fs_down_write(&F2FS_I(src)->i_gc_rwsem[WRITE]); + if (src != dst) { + ret = -EBUSY; + if (!f2fs_down_write_trylock(&F2FS_I(dst)->i_gc_rwsem[WRITE])) + goto out_src; + } + + f2fs_lock_op(sbi, &lc); + ret = __exchange_data_block(src, dst, F2FS_BYTES_TO_BLK(pos_in), + F2FS_BYTES_TO_BLK(pos_out), + F2FS_BYTES_TO_BLK(len), false); + + if (!ret) { + if (dst_max_i_size) + f2fs_i_size_write(dst, dst_max_i_size); + else if (dst_osize != dst->i_size) + f2fs_i_size_write(dst, dst_osize); + } + f2fs_unlock_op(sbi, &lc); + + if (src != dst) + f2fs_up_write(&F2FS_I(dst)->i_gc_rwsem[WRITE]); +out_src: + f2fs_up_write(&F2FS_I(src)->i_gc_rwsem[WRITE]); + if (ret) + goto out_unlock; + + inode_set_mtime_to_ts(src, inode_set_ctime_current(src)); + f2fs_mark_inode_dirty_sync(src, false); + if (src != dst) { + inode_set_mtime_to_ts(dst, inode_set_ctime_current(dst)); + f2fs_mark_inode_dirty_sync(dst, false); + } + f2fs_update_time(sbi, REQ_TIME); + +out_unlock: + if (src != dst) + inode_unlock(dst); +out: + inode_unlock(src); + return ret; +} + +static int __f2fs_ioc_move_range(struct file *filp, + struct f2fs_move_range *range) +{ + int err; + + if (!(filp->f_mode & FMODE_READ) || + !(filp->f_mode & FMODE_WRITE)) + return -EBADF; + + CLASS(fd, dst)(range->dst_fd); + if (fd_empty(dst)) + return -EBADF; + + if (!(fd_file(dst)->f_mode & FMODE_WRITE)) + return -EBADF; + + err = mnt_want_write_file(filp); + if (err) + return err; + + err = f2fs_move_file_range(filp, range->pos_in, fd_file(dst), + range->pos_out, range->len); + + mnt_drop_write_file(filp); + return err; +} + +static int f2fs_ioc_move_range(struct file *filp, unsigned long arg) +{ + struct f2fs_move_range range; + + if (copy_from_user(&range, (struct f2fs_move_range __user *)arg, + sizeof(range))) + return -EFAULT; + return __f2fs_ioc_move_range(filp, &range); +} + +static int f2fs_ioc_flush_device(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct sit_info *sm = SIT_I(sbi); + unsigned int start_segno = 0, end_segno = 0; + unsigned int dev_start_segno = 0, dev_end_segno = 0; + struct f2fs_flush_device range; + struct f2fs_gc_control gc_control = { + .init_gc_type = FG_GC, + .should_migrate_blocks = true, + .err_gc_skipped = true, + .nr_free_secs = 0 }; + int ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (f2fs_readonly(sbi->sb)) + return -EROFS; + + if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) + return -EINVAL; + + if (copy_from_user(&range, (struct f2fs_flush_device __user *)arg, + sizeof(range))) + return -EFAULT; + + if (!f2fs_is_multi_device(sbi) || sbi->s_ndevs - 1 <= range.dev_num || + __is_large_section(sbi)) { + f2fs_warn(sbi, "Can't flush %u in %d for SEGS_PER_SEC %u != 1", + range.dev_num, sbi->s_ndevs, SEGS_PER_SEC(sbi)); + return -EINVAL; + } + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + + if (range.dev_num != 0) + dev_start_segno = GET_SEGNO(sbi, FDEV(range.dev_num).start_blk); + dev_end_segno = GET_SEGNO(sbi, FDEV(range.dev_num).end_blk); + + start_segno = sm->last_victim[FLUSH_DEVICE]; + if (start_segno < dev_start_segno || start_segno >= dev_end_segno) + start_segno = dev_start_segno; + end_segno = min(start_segno + range.segments, dev_end_segno); + + while (start_segno < end_segno) { + if (!f2fs_down_write_trylock_trace(&sbi->gc_lock, &gc_control.lc)) { + ret = -EBUSY; + goto out; + } + sm->last_victim[GC_CB] = end_segno + 1; + sm->last_victim[GC_GREEDY] = end_segno + 1; + sm->last_victim[ALLOC_NEXT] = end_segno + 1; + + gc_control.victim_segno = start_segno; + stat_inc_gc_call_count(sbi, FOREGROUND); + ret = f2fs_gc(sbi, &gc_control); + if (ret == -EAGAIN) + ret = 0; + else if (ret < 0) + break; + start_segno++; + } +out: + mnt_drop_write_file(filp); + return ret; +} + +static int f2fs_ioc_get_features(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + u32 sb_feature = le32_to_cpu(F2FS_I_SB(inode)->raw_super->feature); + + /* Must validate to set it with SQLite behavior in Android. */ + sb_feature |= F2FS_FEATURE_ATOMIC_WRITE; + + return put_user(sb_feature, (u32 __user *)arg); +} + +#ifdef CONFIG_QUOTA +int f2fs_transfer_project_quota(struct inode *inode, kprojid_t kprojid) +{ + struct dquot *transfer_to[MAXQUOTAS] = {}; + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct super_block *sb = sbi->sb; + int err; + + transfer_to[PRJQUOTA] = dqget(sb, make_kqid_projid(kprojid)); + if (IS_ERR(transfer_to[PRJQUOTA])) + return PTR_ERR(transfer_to[PRJQUOTA]); + + err = __dquot_transfer(inode, transfer_to); + if (err) + set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR); + dqput(transfer_to[PRJQUOTA]); + return err; +} + +static int f2fs_ioc_setproject(struct inode *inode, __u32 projid) +{ + struct f2fs_inode_info *fi = F2FS_I(inode); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_inode *ri = NULL; + struct f2fs_lock_context lc; + kprojid_t kprojid; + int err; + + if (!f2fs_sb_has_project_quota(sbi)) { + if (projid != F2FS_DEF_PROJID) + return -EOPNOTSUPP; + else + return 0; + } + + if (!f2fs_has_extra_attr(inode)) + return -EOPNOTSUPP; + + kprojid = make_kprojid(&init_user_ns, (projid_t)projid); + + if (projid_eq(kprojid, fi->i_projid)) + return 0; + + err = -EPERM; + /* Is it quota file? Do not allow user to mess with it */ + if (IS_NOQUOTA(inode)) + return err; + + if (!F2FS_FITS_IN_INODE(ri, fi->i_extra_isize, i_projid)) + return -EOVERFLOW; + + err = f2fs_dquot_initialize(inode); + if (err) + return err; + + f2fs_lock_op(sbi, &lc); + err = f2fs_transfer_project_quota(inode, kprojid); + if (err) + goto out_unlock; + + fi->i_projid = kprojid; + inode_set_ctime_current(inode); + f2fs_mark_inode_dirty_sync(inode, true); +out_unlock: + f2fs_unlock_op(sbi, &lc); + return err; +} +#else +int f2fs_transfer_project_quota(struct inode *inode, kprojid_t kprojid) +{ + return 0; +} + +static int f2fs_ioc_setproject(struct inode *inode, __u32 projid) +{ + if (projid != F2FS_DEF_PROJID) + return -EOPNOTSUPP; + return 0; +} +#endif + +int f2fs_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + struct f2fs_inode_info *fi = F2FS_I(inode); + u32 fsflags = f2fs_iflags_to_fsflags(fi->i_flags); + + if (IS_ENCRYPTED(inode)) + fsflags |= FS_ENCRYPT_FL; + if (IS_VERITY(inode)) + fsflags |= FS_VERITY_FL; + if (f2fs_has_inline_data(inode) || f2fs_has_inline_dentry(inode)) + fsflags |= FS_INLINE_DATA_FL; + if (is_inode_flag_set(inode, FI_PIN_FILE)) + fsflags |= FS_NOCOW_FL; + + fileattr_fill_flags(fa, fsflags & F2FS_GETTABLE_FS_FL); + + if (f2fs_sb_has_project_quota(F2FS_I_SB(inode))) + fa->fsx_projid = from_kprojid(&init_user_ns, fi->i_projid); + + return 0; +} + +int f2fs_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + u32 fsflags = fa->flags, mask = F2FS_SETTABLE_FS_FL; + u32 iflags; + int err; + + if (unlikely(f2fs_cp_error(F2FS_I_SB(inode)))) + return -EIO; + if (!f2fs_is_checkpoint_ready(F2FS_I_SB(inode))) + return -ENOSPC; + if (fsflags & ~F2FS_GETTABLE_FS_FL) + return -EOPNOTSUPP; + fsflags &= F2FS_SETTABLE_FS_FL; + if (!fa->flags_valid) + mask &= FS_COMMON_FL; + + iflags = f2fs_fsflags_to_iflags(fsflags); + if (f2fs_mask_flags(inode->i_mode, iflags) != iflags) + return -EOPNOTSUPP; + + err = f2fs_setflags_common(inode, iflags, f2fs_fsflags_to_iflags(mask)); + if (!err) + err = f2fs_ioc_setproject(inode, fa->fsx_projid); + + return err; +} + +int f2fs_pin_file_control(struct inode *inode, bool inc) +{ + struct f2fs_inode_info *fi = F2FS_I(inode); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + + if (IS_DEVICE_ALIASING(inode)) + return -EINVAL; + + if (fi->i_gc_failures >= sbi->gc_pin_file_threshold) { + f2fs_warn(sbi, "%s: Enable GC = ino %llx after %x GC trials", + __func__, inode->i_ino, fi->i_gc_failures); + clear_inode_flag(inode, FI_PIN_FILE); + return -EAGAIN; + } + + /* Use i_gc_failures for normal file as a risk signal. */ + if (inc) + f2fs_i_gc_failures_write(inode, fi->i_gc_failures + 1); + + return 0; +} + +static int f2fs_ioc_set_pin_file(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + __u32 pin; + int ret = 0; + + if (get_user(pin, (__u32 __user *)arg)) + return -EFAULT; + + if (!S_ISREG(inode->i_mode)) + return -EINVAL; + + if (f2fs_readonly(sbi->sb)) + return -EROFS; + + if (!pin && IS_DEVICE_ALIASING(inode)) + return -EOPNOTSUPP; + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + + inode_lock(inode); + + if (f2fs_is_atomic_file(inode)) { + ret = -EINVAL; + goto out; + } + + if (!pin) { + clear_inode_flag(inode, FI_PIN_FILE); + f2fs_i_gc_failures_write(inode, 0); + goto done; + } else if (f2fs_is_pinned_file(inode)) { + goto done; + } + + if (F2FS_HAS_BLOCKS(inode)) { + ret = -EFBIG; + goto out; + } + + /* Let's allow file pinning on zoned device. */ + if (!f2fs_sb_has_blkzoned(sbi) && + f2fs_should_update_outplace(inode, NULL)) { + ret = -EINVAL; + goto out; + } + + if (f2fs_pin_file_control(inode, false)) { + ret = -EAGAIN; + goto out; + } + + ret = f2fs_convert_inline_inode(inode); + if (ret) + goto out; + + if (!f2fs_disable_compressed_file(inode)) { + ret = -EOPNOTSUPP; + goto out; + } + + set_inode_flag(inode, FI_PIN_FILE); + ret = F2FS_I(inode)->i_gc_failures; +done: + f2fs_update_time(sbi, REQ_TIME); +out: + inode_unlock(inode); + mnt_drop_write_file(filp); + return ret; +} + +static int f2fs_ioc_get_pin_file(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + __u32 pin = 0; + + if (is_inode_flag_set(inode, FI_PIN_FILE)) + pin = F2FS_I(inode)->i_gc_failures; + return put_user(pin, (u32 __user *)arg); +} + +static int f2fs_ioc_get_dev_alias_file(struct file *filp, unsigned long arg) +{ + return put_user(IS_DEVICE_ALIASING(file_inode(filp)) ? 1 : 0, + (u32 __user *)arg); +} + +static int f2fs_ioc_io_prio(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + __u32 level; + + if (get_user(level, (__u32 __user *)arg)) + return -EFAULT; + + if (!S_ISREG(inode->i_mode) || level >= F2FS_IOPRIO_MAX) + return -EINVAL; + + inode_lock(inode); + F2FS_I(inode)->ioprio_hint = level; + inode_unlock(inode); + return 0; +} + +int f2fs_precache_extents(struct inode *inode) +{ + struct f2fs_inode_info *fi = F2FS_I(inode); + struct f2fs_map_blocks map; + pgoff_t m_next_extent; + loff_t end; + int err; + + if (is_inode_flag_set(inode, FI_NO_EXTENT)) + return -EOPNOTSUPP; + + map.m_lblk = 0; + map.m_pblk = 0; + map.m_next_pgofs = NULL; + map.m_next_extent = &m_next_extent; + map.m_seg_type = NO_CHECK_TYPE; + map.m_may_create = false; + end = F2FS_BLK_ALIGN(i_size_read(inode)); + + while (map.m_lblk < end) { + map.m_len = end - map.m_lblk; + + f2fs_down_write(&fi->i_gc_rwsem[WRITE]); + err = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_PRECACHE); + f2fs_up_write(&fi->i_gc_rwsem[WRITE]); + if (err || !map.m_len) + return err; + + map.m_lblk = m_next_extent; + } + + return 0; +} + +static int f2fs_ioc_precache_extents(struct file *filp) +{ + return f2fs_precache_extents(file_inode(filp)); +} + +static int f2fs_ioc_resize_fs(struct file *filp, unsigned long arg) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(file_inode(filp)); + __u64 block_count; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (f2fs_readonly(sbi->sb)) + return -EROFS; + + if (copy_from_user(&block_count, (void __user *)arg, + sizeof(block_count))) + return -EFAULT; + + return f2fs_resize_fs(filp, block_count); +} + +static int f2fs_ioc_enable_verity(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + + f2fs_update_time(F2FS_I_SB(inode), REQ_TIME); + + if (!f2fs_sb_has_verity(F2FS_I_SB(inode))) { + f2fs_warn(F2FS_I_SB(inode), + "Can't enable fs-verity on inode %llu: the verity feature is not enabled on this filesystem", + inode->i_ino); + return -EOPNOTSUPP; + } + + return fsverity_ioctl_enable(filp, (const void __user *)arg); +} + +static int f2fs_ioc_measure_verity(struct file *filp, unsigned long arg) +{ + if (!f2fs_sb_has_verity(F2FS_I_SB(file_inode(filp)))) + return -EOPNOTSUPP; + + return fsverity_ioctl_measure(filp, (void __user *)arg); +} + +static int f2fs_ioc_read_verity_metadata(struct file *filp, unsigned long arg) +{ + if (!f2fs_sb_has_verity(F2FS_I_SB(file_inode(filp)))) + return -EOPNOTSUPP; + + return fsverity_ioctl_read_metadata(filp, (const void __user *)arg); +} + +static int f2fs_ioc_getfslabel(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + char *vbuf; + int count; + int err = 0; + + vbuf = f2fs_kzalloc(sbi, MAX_VOLUME_NAME, GFP_KERNEL); + if (!vbuf) + return -ENOMEM; + + f2fs_down_read(&sbi->sb_lock); + count = utf16s_to_utf8s(sbi->raw_super->volume_name, + ARRAY_SIZE(sbi->raw_super->volume_name), + UTF16_LITTLE_ENDIAN, vbuf, MAX_VOLUME_NAME); + f2fs_up_read(&sbi->sb_lock); + + if (copy_to_user((char __user *)arg, vbuf, + min(FSLABEL_MAX, count))) + err = -EFAULT; + + kfree(vbuf); + return err; +} + +static int f2fs_ioc_setfslabel(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + char *vbuf; + int err = 0; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + vbuf = strndup_user((const char __user *)arg, FSLABEL_MAX); + if (IS_ERR(vbuf)) + return PTR_ERR(vbuf); + + err = mnt_want_write_file(filp); + if (err) + goto out; + + f2fs_down_write(&sbi->sb_lock); + + memset(sbi->raw_super->volume_name, 0, + sizeof(sbi->raw_super->volume_name)); + utf8s_to_utf16s(vbuf, strlen(vbuf), UTF16_LITTLE_ENDIAN, + sbi->raw_super->volume_name, + ARRAY_SIZE(sbi->raw_super->volume_name)); + + err = f2fs_commit_super(sbi, false); + + f2fs_up_write(&sbi->sb_lock); + + mnt_drop_write_file(filp); +out: + kfree(vbuf); + return err; +} + +static int f2fs_get_compress_blocks(struct inode *inode, __u64 *blocks) +{ + if (!f2fs_sb_has_compression(F2FS_I_SB(inode))) + return -EOPNOTSUPP; + + if (!f2fs_compressed_file(inode)) + return -EINVAL; + + *blocks = atomic_read(&F2FS_I(inode)->i_compr_blocks); + + return 0; +} + +static int f2fs_ioc_get_compress_blocks(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + __u64 blocks; + int ret; + + ret = f2fs_get_compress_blocks(inode, &blocks); + if (ret < 0) + return ret; + + return put_user(blocks, (u64 __user *)arg); +} + +static int release_compress_blocks(struct dnode_of_data *dn, pgoff_t count) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); + unsigned int released_blocks = 0; + int cluster_size = F2FS_I(dn->inode)->i_cluster_size; + block_t blkaddr; + int i; + + for (i = 0; i < count; i++) { + blkaddr = data_blkaddr(dn->inode, dn->node_folio, + dn->ofs_in_node + i); + + if (!__is_valid_data_blkaddr(blkaddr)) + continue; + if (unlikely(!f2fs_is_valid_blkaddr(sbi, blkaddr, + DATA_GENERIC_ENHANCE))) + return -EFSCORRUPTED; + } + + while (count) { + int compr_blocks = 0; + + for (i = 0; i < cluster_size; i++, dn->ofs_in_node++) { + blkaddr = f2fs_data_blkaddr(dn); + + if (i == 0) { + if (blkaddr == COMPRESS_ADDR) + continue; + dn->ofs_in_node += cluster_size; + goto next; + } + + if (__is_valid_data_blkaddr(blkaddr)) + compr_blocks++; + + if (blkaddr != NEW_ADDR) + continue; + + f2fs_set_data_blkaddr(dn, NULL_ADDR); + } + + f2fs_i_compr_blocks_update(dn->inode, compr_blocks, false); + dec_valid_block_count(sbi, dn->inode, + cluster_size - compr_blocks); + + released_blocks += cluster_size - compr_blocks; +next: + count -= cluster_size; + } + + return released_blocks; +} + +static int f2fs_release_compress_blocks(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct f2fs_inode_info *fi = F2FS_I(inode); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_lock_context lc; + pgoff_t page_idx = 0, last_idx; + unsigned int released_blocks = 0; + int ret; + int writecount; + + if (!f2fs_sb_has_compression(sbi)) + return -EOPNOTSUPP; + + if (f2fs_readonly(sbi->sb)) + return -EROFS; + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + + f2fs_balance_fs(sbi, true); + + inode_lock(inode); + + writecount = atomic_read(&inode->i_writecount); + if ((filp->f_mode & FMODE_WRITE && writecount != 1) || + (!(filp->f_mode & FMODE_WRITE) && writecount)) { + ret = -EBUSY; + goto out; + } + + if (!f2fs_compressed_file(inode) || + is_inode_flag_set(inode, FI_COMPRESS_RELEASED)) { + ret = -EINVAL; + goto out; + } + + ret = filemap_write_and_wait_range(inode->i_mapping, 0, LLONG_MAX); + if (ret) + goto out; + + if (!atomic_read(&fi->i_compr_blocks)) { + ret = -EPERM; + goto out; + } + + set_inode_flag(inode, FI_COMPRESS_RELEASED); + inode_set_ctime_current(inode); + f2fs_mark_inode_dirty_sync(inode, true); + + f2fs_down_write(&fi->i_gc_rwsem[WRITE]); + filemap_invalidate_lock(inode->i_mapping); + + last_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); + + while (page_idx < last_idx) { + struct dnode_of_data dn; + pgoff_t end_offset, count; + + f2fs_lock_op(sbi, &lc); + + set_new_dnode(&dn, inode, NULL, NULL, 0); + ret = f2fs_get_dnode_of_data(&dn, page_idx, LOOKUP_NODE); + if (ret) { + f2fs_unlock_op(sbi, &lc); + if (ret == -ENOENT) { + page_idx = f2fs_get_next_page_offset(&dn, + page_idx); + ret = 0; + continue; + } + break; + } + + end_offset = ADDRS_PER_PAGE(dn.node_folio, inode); + count = min(end_offset - dn.ofs_in_node, last_idx - page_idx); + count = round_up(count, fi->i_cluster_size); + + ret = release_compress_blocks(&dn, count); + + f2fs_put_dnode(&dn); + + f2fs_unlock_op(sbi, &lc); + + if (ret < 0) + break; + + page_idx += count; + released_blocks += ret; + } + + filemap_invalidate_unlock(inode->i_mapping); + f2fs_up_write(&fi->i_gc_rwsem[WRITE]); +out: + if (released_blocks) + f2fs_update_time(sbi, REQ_TIME); + inode_unlock(inode); + + mnt_drop_write_file(filp); + + if (ret >= 0) { + ret = put_user(released_blocks, (u64 __user *)arg); + } else if (released_blocks && + atomic_read(&fi->i_compr_blocks)) { + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_warn(sbi, "%s: partial blocks were released i_ino=%llx " + "iblocks=%llu, released=%u, compr_blocks=%u, " + "run fsck to fix.", + __func__, inode->i_ino, inode->i_blocks, + released_blocks, + atomic_read(&fi->i_compr_blocks)); + } + + return ret; +} + +static int reserve_compress_blocks(struct dnode_of_data *dn, pgoff_t count, + unsigned int *reserved_blocks) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); + int cluster_size = F2FS_I(dn->inode)->i_cluster_size; + block_t blkaddr; + int i; + + for (i = 0; i < count; i++) { + blkaddr = data_blkaddr(dn->inode, dn->node_folio, + dn->ofs_in_node + i); + + if (!__is_valid_data_blkaddr(blkaddr)) + continue; + if (unlikely(!f2fs_is_valid_blkaddr(sbi, blkaddr, + DATA_GENERIC_ENHANCE))) + return -EFSCORRUPTED; + } + + while (count) { + int compr_blocks = 0; + blkcnt_t reserved = 0; + blkcnt_t to_reserved; + int ret; + + for (i = 0; i < cluster_size; i++) { + blkaddr = data_blkaddr(dn->inode, dn->node_folio, + dn->ofs_in_node + i); + + if (i == 0) { + if (blkaddr != COMPRESS_ADDR) { + dn->ofs_in_node += cluster_size; + goto next; + } + continue; + } + + /* + * compressed cluster was not released due to it + * fails in release_compress_blocks(), so NEW_ADDR + * is a possible case. + */ + if (blkaddr == NEW_ADDR) { + reserved++; + continue; + } + if (__is_valid_data_blkaddr(blkaddr)) { + compr_blocks++; + continue; + } + } + + to_reserved = cluster_size - compr_blocks - reserved; + + /* for the case all blocks in cluster were reserved */ + if (reserved && to_reserved == 1) { + dn->ofs_in_node += cluster_size; + goto next; + } + + ret = inc_valid_block_count(sbi, dn->inode, + &to_reserved, false); + if (unlikely(ret)) + return ret; + + for (i = 0; i < cluster_size; i++, dn->ofs_in_node++) { + if (f2fs_data_blkaddr(dn) == NULL_ADDR) + f2fs_set_data_blkaddr(dn, NEW_ADDR); + } + + f2fs_i_compr_blocks_update(dn->inode, compr_blocks, true); + + *reserved_blocks += to_reserved; +next: + count -= cluster_size; + } + + return 0; +} + +static int f2fs_reserve_compress_blocks(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct f2fs_inode_info *fi = F2FS_I(inode); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + pgoff_t page_idx = 0, last_idx; + unsigned int reserved_blocks = 0; + int ret; + + if (!f2fs_sb_has_compression(sbi)) + return -EOPNOTSUPP; + + if (f2fs_readonly(sbi->sb)) + return -EROFS; + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + + f2fs_balance_fs(sbi, true); + + inode_lock(inode); + + if (!f2fs_compressed_file(inode) || + !is_inode_flag_set(inode, FI_COMPRESS_RELEASED)) { + ret = -EINVAL; + goto unlock_inode; + } + + if (atomic_read(&fi->i_compr_blocks)) + goto unlock_inode; + + f2fs_down_write(&fi->i_gc_rwsem[WRITE]); + filemap_invalidate_lock(inode->i_mapping); + + last_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); + + while (page_idx < last_idx) { + struct dnode_of_data dn; + struct f2fs_lock_context lc; + pgoff_t end_offset, count; + + f2fs_lock_op(sbi, &lc); + + set_new_dnode(&dn, inode, NULL, NULL, 0); + ret = f2fs_get_dnode_of_data(&dn, page_idx, LOOKUP_NODE); + if (ret) { + f2fs_unlock_op(sbi, &lc); + if (ret == -ENOENT) { + page_idx = f2fs_get_next_page_offset(&dn, + page_idx); + ret = 0; + continue; + } + break; + } + + end_offset = ADDRS_PER_PAGE(dn.node_folio, inode); + count = min(end_offset - dn.ofs_in_node, last_idx - page_idx); + count = round_up(count, fi->i_cluster_size); + + ret = reserve_compress_blocks(&dn, count, &reserved_blocks); + + f2fs_put_dnode(&dn); + + f2fs_unlock_op(sbi, &lc); + + if (ret < 0) + break; + + page_idx += count; + } + + filemap_invalidate_unlock(inode->i_mapping); + f2fs_up_write(&fi->i_gc_rwsem[WRITE]); + + if (!ret) { + clear_inode_flag(inode, FI_COMPRESS_RELEASED); + inode_set_ctime_current(inode); + f2fs_mark_inode_dirty_sync(inode, true); + } +unlock_inode: + if (reserved_blocks) + f2fs_update_time(sbi, REQ_TIME); + inode_unlock(inode); + mnt_drop_write_file(filp); + + if (!ret) { + ret = put_user(reserved_blocks, (u64 __user *)arg); + } else if (reserved_blocks && + atomic_read(&fi->i_compr_blocks)) { + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_warn(sbi, "%s: partial blocks were reserved i_ino=%llx " + "iblocks=%llu, reserved=%u, compr_blocks=%u, " + "run fsck to fix.", + __func__, inode->i_ino, inode->i_blocks, + reserved_blocks, + atomic_read(&fi->i_compr_blocks)); + } + + return ret; +} + +static int f2fs_secure_erase(struct block_device *bdev, struct inode *inode, + pgoff_t off, block_t block, block_t len, u32 flags) +{ + sector_t sector = SECTOR_FROM_BLOCK(block); + sector_t nr_sects = SECTOR_FROM_BLOCK(len); + int ret = 0; + + if (flags & F2FS_TRIM_FILE_DISCARD) { + if (bdev_max_secure_erase_sectors(bdev)) + ret = blkdev_issue_secure_erase(bdev, sector, nr_sects, + GFP_NOFS); + else + ret = blkdev_issue_discard(bdev, sector, nr_sects, + GFP_NOFS); + } + + if (!ret && (flags & F2FS_TRIM_FILE_ZEROOUT)) { + if (IS_ENCRYPTED(inode)) + ret = fscrypt_zeroout_range(inode, + (loff_t)off << inode->i_blkbits, sector, + (u64)len << inode->i_blkbits); + else + ret = blkdev_issue_zeroout(bdev, sector, nr_sects, + GFP_NOFS, 0); + } + + return ret; +} + +static int f2fs_sec_trim_file(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct address_space *mapping = inode->i_mapping; + struct block_device *prev_bdev = NULL; + struct f2fs_sectrim_range range; + pgoff_t index, pg_end, prev_index = 0; + block_t prev_block = 0, len = 0; + loff_t end_addr; + bool to_end = false; + int ret = 0; + + if (!(filp->f_mode & FMODE_WRITE)) + return -EBADF; + + if (copy_from_user(&range, (struct f2fs_sectrim_range __user *)arg, + sizeof(range))) + return -EFAULT; + + if (range.flags == 0 || (range.flags & ~F2FS_TRIM_FILE_MASK) || + !S_ISREG(inode->i_mode)) + return -EINVAL; + + if (((range.flags & F2FS_TRIM_FILE_DISCARD) && + !f2fs_hw_support_discard(sbi)) || + ((range.flags & F2FS_TRIM_FILE_ZEROOUT) && + IS_ENCRYPTED(inode) && f2fs_is_multi_device(sbi))) + return -EOPNOTSUPP; + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + inode_lock(inode); + + if (f2fs_is_atomic_file(inode) || f2fs_compressed_file(inode) || + range.start >= inode->i_size) { + ret = -EINVAL; + goto err; + } + + if (range.len == 0) + goto err; + + if (inode->i_size - range.start > range.len) { + end_addr = range.start + range.len; + } else { + end_addr = range.len == (u64)-1 ? + sbi->sb->s_maxbytes : inode->i_size; + to_end = true; + } + + if (!IS_ALIGNED(range.start, F2FS_BLKSIZE) || + (!to_end && !IS_ALIGNED(end_addr, F2FS_BLKSIZE))) { + ret = -EINVAL; + goto err; + } + + index = F2FS_BYTES_TO_BLK(range.start); + pg_end = DIV_ROUND_UP(end_addr, F2FS_BLKSIZE); + + ret = f2fs_convert_inline_inode(inode); + if (ret) + goto err; + + f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); + filemap_invalidate_lock(mapping); + + ret = filemap_write_and_wait_range(mapping, range.start, + to_end ? LLONG_MAX : end_addr - 1); + if (ret) + goto out; + + truncate_inode_pages_range(mapping, range.start, + to_end ? -1 : end_addr - 1); + + while (index < pg_end) { + struct dnode_of_data dn; + pgoff_t end_offset, count; + int i; + + set_new_dnode(&dn, inode, NULL, NULL, 0); + ret = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE); + if (ret) { + if (ret == -ENOENT) { + index = f2fs_get_next_page_offset(&dn, index); + continue; + } + goto out; + } + + end_offset = ADDRS_PER_PAGE(dn.node_folio, inode); + count = min(end_offset - dn.ofs_in_node, pg_end - index); + for (i = 0; i < count; i++, index++, dn.ofs_in_node++) { + struct block_device *cur_bdev; + block_t blkaddr = f2fs_data_blkaddr(&dn); + + if (!__is_valid_data_blkaddr(blkaddr)) + continue; + + if (!f2fs_is_valid_blkaddr(sbi, blkaddr, + DATA_GENERIC_ENHANCE)) { + ret = -EFSCORRUPTED; + f2fs_put_dnode(&dn); + goto out; + } + + cur_bdev = f2fs_target_device(sbi, blkaddr, NULL); + if (f2fs_is_multi_device(sbi)) { + int di = f2fs_target_device_index(sbi, blkaddr); + + blkaddr -= FDEV(di).start_blk; + } + + if (len) { + if (prev_bdev == cur_bdev && + index == prev_index + len && + blkaddr == prev_block + len) { + len++; + } else { + ret = f2fs_secure_erase(prev_bdev, + inode, prev_index, prev_block, + len, range.flags); + if (ret) { + f2fs_put_dnode(&dn); + goto out; + } + + len = 0; + } + } + + if (!len) { + prev_bdev = cur_bdev; + prev_index = index; + prev_block = blkaddr; + len = 1; + } + } + + f2fs_put_dnode(&dn); + + if (fatal_signal_pending(current)) { + ret = -EINTR; + goto out; + } + cond_resched(); + } + + if (len) + ret = f2fs_secure_erase(prev_bdev, inode, prev_index, + prev_block, len, range.flags); + f2fs_update_time(sbi, REQ_TIME); +out: + filemap_invalidate_unlock(mapping); + f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); +err: + inode_unlock(inode); + mnt_drop_write_file(filp); + + return ret; +} + +static int f2fs_ioc_get_compress_option(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct f2fs_comp_option option; + + if (!f2fs_sb_has_compression(F2FS_I_SB(inode))) + return -EOPNOTSUPP; + + inode_lock_shared(inode); + + if (!f2fs_compressed_file(inode)) { + inode_unlock_shared(inode); + return -ENODATA; + } + + option.algorithm = F2FS_I(inode)->i_compress_algorithm; + option.log_cluster_size = F2FS_I(inode)->i_log_cluster_size; + + inode_unlock_shared(inode); + + if (copy_to_user((struct f2fs_comp_option __user *)arg, &option, + sizeof(option))) + return -EFAULT; + + return 0; +} + +static int f2fs_ioc_set_compress_option(struct file *filp, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct f2fs_inode_info *fi = F2FS_I(inode); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_comp_option option; + int ret = 0; + + if (!f2fs_sb_has_compression(sbi)) + return -EOPNOTSUPP; + + if (!(filp->f_mode & FMODE_WRITE)) + return -EBADF; + + if (copy_from_user(&option, (struct f2fs_comp_option __user *)arg, + sizeof(option))) + return -EFAULT; + + if (option.log_cluster_size < MIN_COMPRESS_LOG_SIZE || + option.log_cluster_size > MAX_COMPRESS_LOG_SIZE || + option.algorithm >= COMPRESS_MAX) + return -EINVAL; + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + inode_lock(inode); + + f2fs_down_write(&F2FS_I(inode)->i_sem); + if (!f2fs_compressed_file(inode)) { + ret = -EINVAL; + goto out; + } + + if (f2fs_is_mmap_file(inode) || get_dirty_pages(inode)) { + ret = -EBUSY; + goto out; + } + + if (F2FS_HAS_BLOCKS(inode)) { + ret = -EFBIG; + goto out; + } + + fi->i_compress_algorithm = option.algorithm; + fi->i_log_cluster_size = option.log_cluster_size; + fi->i_cluster_size = BIT(option.log_cluster_size); + /* Set default level */ + if (fi->i_compress_algorithm == COMPRESS_ZSTD) + fi->i_compress_level = F2FS_ZSTD_DEFAULT_CLEVEL; + else + fi->i_compress_level = 0; + /* Adjust mount option level */ + if (option.algorithm == F2FS_OPTION(sbi).compress_algorithm && + F2FS_OPTION(sbi).compress_level) + fi->i_compress_level = F2FS_OPTION(sbi).compress_level; + f2fs_mark_inode_dirty_sync(inode, true); + + if (!f2fs_is_compress_backend_ready(inode)) + f2fs_warn(sbi, "compression algorithm is successfully set, " + "but current kernel doesn't support this algorithm."); +out: + f2fs_up_write(&fi->i_sem); + inode_unlock(inode); + mnt_drop_write_file(filp); + + return ret; +} + +static int redirty_blocks(struct inode *inode, pgoff_t page_idx, int len) +{ + DEFINE_READAHEAD(ractl, NULL, NULL, inode->i_mapping, page_idx); + struct address_space *mapping = inode->i_mapping; + struct folio *folio; + pgoff_t redirty_idx = page_idx; + int page_len = 0, ret = 0; + + filemap_invalidate_lock_shared(mapping); + page_cache_ra_unbounded(&ractl, len, 0); + filemap_invalidate_unlock_shared(mapping); + + do { + folio = read_cache_folio(mapping, page_idx, NULL, NULL); + if (IS_ERR(folio)) { + ret = PTR_ERR(folio); + break; + } + page_len += folio_nr_pages(folio) - (page_idx - folio->index); + page_idx = folio_next_index(folio); + } while (page_len < len); + + do { + folio = filemap_lock_folio(mapping, redirty_idx); + + /* It will never fail, when folio has pinned above */ + f2fs_bug_on(F2FS_I_SB(inode), IS_ERR(folio)); + + f2fs_folio_wait_writeback(folio, DATA, true, true); + + folio_mark_dirty(folio); + folio_set_f2fs_gcing(folio); + redirty_idx = folio_next_index(folio); + folio_unlock(folio); + folio_put_refs(folio, 2); + } while (redirty_idx < page_idx); + + return ret; +} + +static int f2fs_ioc_decompress_file(struct file *filp) +{ + struct inode *inode = file_inode(filp); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_inode_info *fi = F2FS_I(inode); + pgoff_t page_idx = 0, last_idx, cluster_idx; + int ret; + + if (!f2fs_sb_has_compression(sbi) || + F2FS_OPTION(sbi).compress_mode != COMPR_MODE_USER) + return -EOPNOTSUPP; + + if (!(filp->f_mode & FMODE_WRITE)) + return -EBADF; + + f2fs_balance_fs(sbi, true); + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + inode_lock(inode); + + if (!f2fs_is_compress_backend_ready(inode)) { + ret = -EOPNOTSUPP; + goto out; + } + + if (!f2fs_compressed_file(inode) || + is_inode_flag_set(inode, FI_COMPRESS_RELEASED)) { + ret = -EINVAL; + goto out; + } + + ret = filemap_write_and_wait_range(inode->i_mapping, 0, LLONG_MAX); + if (ret) + goto out; + + if (!atomic_read(&fi->i_compr_blocks)) + goto out; + + last_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); + last_idx >>= fi->i_log_cluster_size; + + for (cluster_idx = 0; cluster_idx < last_idx; cluster_idx++) { + page_idx = cluster_idx << fi->i_log_cluster_size; + + if (!f2fs_is_compressed_cluster(inode, page_idx)) + continue; + + ret = redirty_blocks(inode, page_idx, fi->i_cluster_size); + if (ret < 0) + break; + + if (get_dirty_pages(inode) >= BLKS_PER_SEG(sbi)) { + ret = filemap_fdatawrite(inode->i_mapping); + if (ret < 0) + break; + } + + cond_resched(); + if (fatal_signal_pending(current)) { + ret = -EINTR; + break; + } + } + + if (!ret) + ret = filemap_write_and_wait_range(inode->i_mapping, 0, + LLONG_MAX); + + if (ret) + f2fs_warn(sbi, "%s: The file might be partially decompressed (errno=%d). Please delete the file.", + __func__, ret); + f2fs_update_time(sbi, REQ_TIME); +out: + inode_unlock(inode); + mnt_drop_write_file(filp); + + return ret; +} + +static int f2fs_ioc_compress_file(struct file *filp) +{ + struct inode *inode = file_inode(filp); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_inode_info *fi = F2FS_I(inode); + pgoff_t page_idx = 0, last_idx, cluster_idx; + int ret; + + if (!f2fs_sb_has_compression(sbi) || + F2FS_OPTION(sbi).compress_mode != COMPR_MODE_USER) + return -EOPNOTSUPP; + + if (!(filp->f_mode & FMODE_WRITE)) + return -EBADF; + + f2fs_balance_fs(sbi, true); + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + inode_lock(inode); + + if (!f2fs_is_compress_backend_ready(inode)) { + ret = -EOPNOTSUPP; + goto out; + } + + if (!f2fs_compressed_file(inode) || + is_inode_flag_set(inode, FI_COMPRESS_RELEASED)) { + ret = -EINVAL; + goto out; + } + + ret = filemap_write_and_wait_range(inode->i_mapping, 0, LLONG_MAX); + if (ret) + goto out; + + set_inode_flag(inode, FI_ENABLE_COMPRESS); + + last_idx = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); + last_idx >>= fi->i_log_cluster_size; + + for (cluster_idx = 0; cluster_idx < last_idx; cluster_idx++) { + page_idx = cluster_idx << fi->i_log_cluster_size; + + if (f2fs_is_sparse_cluster(inode, page_idx)) + continue; + + ret = redirty_blocks(inode, page_idx, fi->i_cluster_size); + if (ret < 0) + break; + + if (get_dirty_pages(inode) >= BLKS_PER_SEG(sbi)) { + ret = filemap_fdatawrite(inode->i_mapping); + if (ret < 0) + break; + } + + cond_resched(); + if (fatal_signal_pending(current)) { + ret = -EINTR; + break; + } + } + + if (!ret) + ret = filemap_write_and_wait_range(inode->i_mapping, 0, + LLONG_MAX); + + clear_inode_flag(inode, FI_ENABLE_COMPRESS); + + if (ret) + f2fs_warn(sbi, "%s: The file might be partially compressed (errno=%d). Please delete the file.", + __func__, ret); + f2fs_update_time(sbi, REQ_TIME); +out: + inode_unlock(inode); + mnt_drop_write_file(filp); + + return ret; +} + +static long __f2fs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + switch (cmd) { + case FS_IOC_GETVERSION: + return f2fs_ioc_getversion(filp, arg); + case F2FS_IOC_START_ATOMIC_WRITE: + return f2fs_ioc_start_atomic_write(filp, false); + case F2FS_IOC_START_ATOMIC_REPLACE: + return f2fs_ioc_start_atomic_write(filp, true); + case F2FS_IOC_COMMIT_ATOMIC_WRITE: + return f2fs_ioc_commit_atomic_write(filp); + case F2FS_IOC_ABORT_ATOMIC_WRITE: + return f2fs_ioc_abort_atomic_write(filp); + case F2FS_IOC_START_VOLATILE_WRITE: + case F2FS_IOC_RELEASE_VOLATILE_WRITE: + return -EOPNOTSUPP; + case F2FS_IOC_SHUTDOWN: + return f2fs_ioc_shutdown(filp, arg); + case FITRIM: + return f2fs_ioc_fitrim(filp, arg); + case FS_IOC_SET_ENCRYPTION_POLICY: + return f2fs_ioc_set_encryption_policy(filp, arg); + case FS_IOC_GET_ENCRYPTION_POLICY: + return f2fs_ioc_get_encryption_policy(filp, arg); + case FS_IOC_GET_ENCRYPTION_PWSALT: + return f2fs_ioc_get_encryption_pwsalt(filp, arg); + case FS_IOC_GET_ENCRYPTION_POLICY_EX: + return f2fs_ioc_get_encryption_policy_ex(filp, arg); + case FS_IOC_ADD_ENCRYPTION_KEY: + return f2fs_ioc_add_encryption_key(filp, arg); + case FS_IOC_REMOVE_ENCRYPTION_KEY: + return f2fs_ioc_remove_encryption_key(filp, arg); + case FS_IOC_REMOVE_ENCRYPTION_KEY_ALL_USERS: + return f2fs_ioc_remove_encryption_key_all_users(filp, arg); + case FS_IOC_GET_ENCRYPTION_KEY_STATUS: + return f2fs_ioc_get_encryption_key_status(filp, arg); + case FS_IOC_GET_ENCRYPTION_NONCE: + return f2fs_ioc_get_encryption_nonce(filp, arg); + case F2FS_IOC_GARBAGE_COLLECT: + return f2fs_ioc_gc(filp, arg); + case F2FS_IOC_GARBAGE_COLLECT_RANGE: + return f2fs_ioc_gc_range(filp, arg); + case F2FS_IOC_WRITE_CHECKPOINT: + return f2fs_ioc_write_checkpoint(filp); + case F2FS_IOC_DEFRAGMENT: + return f2fs_ioc_defragment(filp, arg); + case F2FS_IOC_MOVE_RANGE: + return f2fs_ioc_move_range(filp, arg); + case F2FS_IOC_FLUSH_DEVICE: + return f2fs_ioc_flush_device(filp, arg); + case F2FS_IOC_GET_FEATURES: + return f2fs_ioc_get_features(filp, arg); + case F2FS_IOC_GET_PIN_FILE: + return f2fs_ioc_get_pin_file(filp, arg); + case F2FS_IOC_SET_PIN_FILE: + return f2fs_ioc_set_pin_file(filp, arg); + case F2FS_IOC_PRECACHE_EXTENTS: + return f2fs_ioc_precache_extents(filp); + case F2FS_IOC_RESIZE_FS: + return f2fs_ioc_resize_fs(filp, arg); + case FS_IOC_ENABLE_VERITY: + return f2fs_ioc_enable_verity(filp, arg); + case FS_IOC_MEASURE_VERITY: + return f2fs_ioc_measure_verity(filp, arg); + case FS_IOC_READ_VERITY_METADATA: + return f2fs_ioc_read_verity_metadata(filp, arg); + case FS_IOC_GETFSLABEL: + return f2fs_ioc_getfslabel(filp, arg); + case FS_IOC_SETFSLABEL: + return f2fs_ioc_setfslabel(filp, arg); + case F2FS_IOC_GET_COMPRESS_BLOCKS: + return f2fs_ioc_get_compress_blocks(filp, arg); + case F2FS_IOC_RELEASE_COMPRESS_BLOCKS: + return f2fs_release_compress_blocks(filp, arg); + case F2FS_IOC_RESERVE_COMPRESS_BLOCKS: + return f2fs_reserve_compress_blocks(filp, arg); + case F2FS_IOC_SEC_TRIM_FILE: + return f2fs_sec_trim_file(filp, arg); + case F2FS_IOC_GET_COMPRESS_OPTION: + return f2fs_ioc_get_compress_option(filp, arg); + case F2FS_IOC_SET_COMPRESS_OPTION: + return f2fs_ioc_set_compress_option(filp, arg); + case F2FS_IOC_DECOMPRESS_FILE: + return f2fs_ioc_decompress_file(filp); + case F2FS_IOC_COMPRESS_FILE: + return f2fs_ioc_compress_file(filp); + case F2FS_IOC_GET_DEV_ALIAS_FILE: + return f2fs_ioc_get_dev_alias_file(filp, arg); + case F2FS_IOC_IO_PRIO: + return f2fs_ioc_io_prio(filp, arg); + default: + return -ENOTTY; + } +} + +long f2fs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + if (unlikely(f2fs_cp_error(F2FS_I_SB(file_inode(filp))))) + return -EIO; + if (!f2fs_is_checkpoint_ready(F2FS_I_SB(file_inode(filp)))) + return -ENOSPC; + + return __f2fs_ioctl(filp, cmd, arg); +} + +/* + * Return %true if the given read or write request should use direct I/O, or + * %false if it should use buffered I/O. + */ +static bool f2fs_should_use_dio(struct inode *inode, struct kiocb *iocb, + struct iov_iter *iter) +{ + unsigned int align; + + if (!(iocb->ki_flags & IOCB_DIRECT)) + return false; + + if (f2fs_force_buffered_io(inode, iov_iter_rw(iter))) + return false; + + /* + * Direct I/O not aligned to the disk's logical_block_size will be + * attempted, but will fail with -EINVAL. + * + * f2fs additionally requires that direct I/O be aligned to the + * filesystem block size, which is often a stricter requirement. + * However, f2fs traditionally falls back to buffered I/O on requests + * that are logical_block_size-aligned but not fs-block aligned. + * + * The below logic implements this behavior. + */ + align = iocb->ki_pos | iov_iter_alignment(iter); + if (!IS_ALIGNED(align, i_blocksize(inode)) && + IS_ALIGNED(align, bdev_logical_block_size(inode->i_sb->s_bdev))) + return false; + + return true; +} + +#ifdef CONFIG_F2FS_IOSTAT +static void f2fs_dio_end_bio(struct bio *bio) +{ + struct bio_iostat_ctx *iostat_ctx = bio->bi_private; + void *orig_bi_private = iostat_ctx->post_read_ctx; + + iostat_update_and_unbind_ctx(bio); + bio->bi_private = orig_bi_private; + iomap_dio_bio_end_io(bio); +} + +static void f2fs_dio_iostat_start(struct f2fs_sb_info *sbi, struct bio *bio) +{ + void *bi_private = bio->bi_private; + + if (!sbi->iostat_enable) + return; + + iostat_alloc_and_bind_ctx(sbi, bio, bi_private); + iostat_update_submit_ctx(bio, DATA); + bio->bi_end_io = f2fs_dio_end_bio; +} +#else +static inline void f2fs_dio_iostat_start(struct f2fs_sb_info *sbi, + struct bio *bio) {} +#endif + +static int f2fs_dio_read_end_io(struct kiocb *iocb, ssize_t size, int error, + unsigned int flags) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(file_inode(iocb->ki_filp)); + + dec_page_count(sbi, F2FS_DIO_READ); + if (error) + return error; + f2fs_update_iostat(sbi, NULL, APP_DIRECT_READ_IO, size); + return 0; +} + +static void f2fs_dio_read_submit_io(const struct iomap_iter *iter, + struct bio *bio, loff_t file_offset) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(iter->inode); + + f2fs_dio_iostat_start(sbi, bio); + blk_crypto_submit_bio(bio); +} + +static const struct iomap_dio_ops f2fs_iomap_dio_read_ops = { + .end_io = f2fs_dio_read_end_io, + .submit_io = f2fs_dio_read_submit_io, +}; + +static ssize_t f2fs_dio_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_inode_info *fi = F2FS_I(inode); + const loff_t pos = iocb->ki_pos; + const size_t count = iov_iter_count(to); + struct iomap_dio *dio; + ssize_t ret; + + if (count == 0) + return 0; /* skip atime update */ + + trace_f2fs_direct_IO_enter(inode, iocb, count, READ); + + if (iocb->ki_flags & IOCB_NOWAIT) { + if (!f2fs_down_read_trylock(&fi->i_gc_rwsem[READ])) { + ret = -EAGAIN; + goto out; + } + } else { + f2fs_down_read(&fi->i_gc_rwsem[READ]); + } + + /* dio is not compatible w/ atomic file */ + if (f2fs_is_atomic_file(inode)) { + f2fs_up_read(&fi->i_gc_rwsem[READ]); + ret = -EOPNOTSUPP; + goto out; + } + + /* + * We have to use __iomap_dio_rw() and iomap_dio_complete() instead of + * the higher-level function iomap_dio_rw() in order to ensure that the + * F2FS_DIO_READ counter will be decremented correctly in all cases. + */ + inc_page_count(sbi, F2FS_DIO_READ); + dio = __iomap_dio_rw(iocb, to, &f2fs_iomap_ops, + &f2fs_iomap_dio_read_ops, 0, NULL, 0); + if (IS_ERR_OR_NULL(dio)) { + ret = PTR_ERR_OR_ZERO(dio); + if (ret != -EIOCBQUEUED) + dec_page_count(sbi, F2FS_DIO_READ); + } else { + ret = iomap_dio_complete(dio); + } + + f2fs_up_read(&fi->i_gc_rwsem[READ]); + + file_accessed(file); +out: + trace_f2fs_direct_IO_exit(inode, pos, count, READ, ret); + return ret; +} + +static void f2fs_trace_rw_file_path(struct file *file, loff_t pos, size_t count, + int rw) +{ + struct inode *inode = file_inode(file); + char *buf, *path; + + buf = f2fs_getname(F2FS_I_SB(inode)); + if (!buf) + return; + path = dentry_path_raw(file_dentry(file), buf, PATH_MAX); + if (IS_ERR(path)) + goto free_buf; + if (rw == WRITE) + trace_f2fs_datawrite_start(inode, pos, count, + current->pid, path, current->comm); + else + trace_f2fs_dataread_start(inode, pos, count, + current->pid, path, current->comm); +free_buf: + f2fs_putname(buf); +} + +static ssize_t f2fs_file_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct inode *inode = file_inode(iocb->ki_filp); + const loff_t pos = iocb->ki_pos; + ssize_t ret; + bool dio; + + if (!f2fs_is_compress_backend_ready(inode)) + return -EOPNOTSUPP; + + if (trace_f2fs_dataread_start_enabled()) + f2fs_trace_rw_file_path(iocb->ki_filp, iocb->ki_pos, + iov_iter_count(to), READ); + + dio = f2fs_should_use_dio(inode, iocb, to); + + /* In LFS mode, if there is inflight dio, wait for its completion */ + if (f2fs_lfs_mode(F2FS_I_SB(inode)) && + get_pages(F2FS_I_SB(inode), F2FS_DIO_WRITE) && + (!f2fs_is_pinned_file(inode) || !dio)) + inode_dio_wait(inode); + + if (dio) { + ret = f2fs_dio_read_iter(iocb, to); + } else { + ret = filemap_read(iocb, to, 0); + if (ret > 0) + f2fs_update_iostat(F2FS_I_SB(inode), inode, + APP_BUFFERED_READ_IO, ret); + } + trace_f2fs_dataread_end(inode, pos, ret); + return ret; +} + +static ssize_t f2fs_file_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, + size_t len, unsigned int flags) +{ + struct inode *inode = file_inode(in); + const loff_t pos = *ppos; + ssize_t ret; + + if (!f2fs_is_compress_backend_ready(inode)) + return -EOPNOTSUPP; + + if (trace_f2fs_dataread_start_enabled()) + f2fs_trace_rw_file_path(in, pos, len, READ); + + ret = filemap_splice_read(in, ppos, pipe, len, flags); + if (ret > 0) + f2fs_update_iostat(F2FS_I_SB(inode), inode, + APP_BUFFERED_READ_IO, ret); + + trace_f2fs_dataread_end(inode, pos, ret); + return ret; +} + +static ssize_t f2fs_write_checks(struct kiocb *iocb, struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + ssize_t count; + int err; + + if (IS_IMMUTABLE(inode)) + return -EPERM; + + if (is_inode_flag_set(inode, FI_COMPRESS_RELEASED)) + return -EPERM; + + count = generic_write_checks(iocb, from); + if (count <= 0) + return count; + + err = file_modified(file); + if (err) + return err; + + f2fs_zero_post_eof_page(inode, + iocb->ki_pos + iov_iter_count(from), true); + return count; +} + +/* + * Preallocate blocks for a write request, if it is possible and helpful to do + * so. Returns a positive number if blocks may have been preallocated, 0 if no + * blocks were preallocated, or a negative errno value if something went + * seriously wrong. Also sets FI_PREALLOCATED_ALL on the inode if *all* the + * requested blocks (not just some of them) have been allocated. + */ +static int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *iter, + bool dio) +{ + struct inode *inode = file_inode(iocb->ki_filp); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + const loff_t pos = iocb->ki_pos; + const size_t count = iov_iter_count(iter); + struct f2fs_map_blocks map = {}; + int flag; + int ret; + + /* If it will be an out-of-place direct write, don't bother. */ + if (dio && f2fs_lfs_mode(sbi)) + return 0; + /* + * Don't preallocate holes aligned to DIO_SKIP_HOLES which turns into + * buffered IO, if DIO meets any holes. + */ + if (dio && i_size_read(inode) && + (F2FS_BYTES_TO_BLK(pos) < F2FS_BLK_ALIGN(i_size_read(inode)))) + return 0; + + /* No-wait I/O can't allocate blocks. */ + if (iocb->ki_flags & IOCB_NOWAIT) + return 0; + + /* If it will be a short write, don't bother. */ + if (fault_in_iov_iter_readable(iter, count)) + return 0; + + if (f2fs_has_inline_data(inode)) { + /* If the data will fit inline, don't bother. */ + if (pos + count <= MAX_INLINE_DATA(inode)) + return 0; + ret = f2fs_convert_inline_inode(inode); + if (ret) + return ret; + } + + /* Do not preallocate blocks that will be written partially in 4KB. */ + map.m_lblk = F2FS_BLK_ALIGN(pos); + map.m_len = F2FS_BYTES_TO_BLK(pos + count); + if (map.m_len > map.m_lblk) + map.m_len -= map.m_lblk; + else + return 0; + + if (!IS_DEVICE_ALIASING(inode)) + map.m_may_create = true; + if (dio) { + map.m_seg_type = f2fs_rw_hint_to_seg_type(sbi, + inode->i_write_hint); + flag = F2FS_GET_BLOCK_PRE_DIO; + } else { + map.m_seg_type = NO_CHECK_TYPE; + flag = F2FS_GET_BLOCK_PRE_AIO; + } + + ret = f2fs_map_blocks(inode, &map, flag); + /* -ENOSPC|-EDQUOT are fine to report the number of allocated blocks. */ + if (ret < 0 && !((ret == -ENOSPC || ret == -EDQUOT) && map.m_len > 0)) + return ret; + if (ret == 0) + set_inode_flag(inode, FI_PREALLOCATED_ALL); + return map.m_len; +} + +static ssize_t f2fs_buffered_write_iter(struct kiocb *iocb, + struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + ssize_t ret; + + if (iocb->ki_flags & IOCB_NOWAIT) + return -EOPNOTSUPP; + + ret = generic_perform_write(iocb, from); + + if (ret > 0) { + f2fs_update_iostat(F2FS_I_SB(inode), inode, + APP_BUFFERED_IO, ret); + } + return ret; +} + +static int f2fs_dio_write_end_io(struct kiocb *iocb, ssize_t size, int error, + unsigned int flags) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(file_inode(iocb->ki_filp)); + + dec_page_count(sbi, F2FS_DIO_WRITE); + if (error) + return error; + f2fs_update_time(sbi, REQ_TIME); + f2fs_update_iostat(sbi, NULL, APP_DIRECT_IO, size); + return 0; +} + +static bool f2fs_valid_write_stream(struct f2fs_sb_info *sbi, u8 write_stream) +{ + int i; + + if (!write_stream) + return true; + if (!f2fs_is_multi_device(sbi)) + return write_stream <= bdev_max_write_streams(sbi->sb->s_bdev); + + for (i = 0; i < sbi->s_ndevs; i++) + if (write_stream > bdev_max_write_streams(FDEV(i).bdev)) + return false; + return true; +} + +static void f2fs_dio_write_submit_io(const struct iomap_iter *iter, + struct bio *bio, loff_t file_offset) +{ + struct inode *inode = iter->inode; + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct kiocb *iocb = iter->private; + enum log_type type = f2fs_rw_hint_to_seg_type(sbi, inode->i_write_hint); + enum temp_type temp = f2fs_get_segment_temp(sbi, type); + + bio->bi_write_hint = f2fs_io_type_to_rw_hint(sbi, DATA, temp); + bio->bi_write_stream = + iocb->ki_write_stream ? iocb->ki_write_stream : + f2fs_io_type_to_write_stream(bio->bi_bdev, DATA, temp); + f2fs_dio_iostat_start(sbi, bio); + blk_crypto_submit_bio(bio); +} + +static const struct iomap_dio_ops f2fs_iomap_dio_write_ops = { + .end_io = f2fs_dio_write_end_io, + .submit_io = f2fs_dio_write_submit_io, +}; + +static void f2fs_flush_buffered_write(struct address_space *mapping, + loff_t start_pos, loff_t end_pos) +{ + int ret; + + ret = filemap_write_and_wait_range(mapping, start_pos, end_pos); + if (ret < 0) + return; + invalidate_mapping_pages(mapping, + start_pos >> PAGE_SHIFT, + end_pos >> PAGE_SHIFT); +} + +static ssize_t f2fs_dio_write_iter(struct kiocb *iocb, struct iov_iter *from, + bool *may_need_sync) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct f2fs_inode_info *fi = F2FS_I(inode); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + const bool do_opu = f2fs_lfs_mode(sbi); + const loff_t pos = iocb->ki_pos; + const ssize_t count = iov_iter_count(from); + unsigned int dio_flags; + struct iomap_dio *dio; + ssize_t ret; + + trace_f2fs_direct_IO_enter(inode, iocb, count, WRITE); + + if (!f2fs_valid_write_stream(sbi, iocb->ki_write_stream)) { + ret = -EINVAL; + goto out; + } + + if (iocb->ki_flags & IOCB_NOWAIT) { + /* f2fs_convert_inline_inode() and block allocation can block */ + if (f2fs_has_inline_data(inode) || + !f2fs_overwrite_io(inode, pos, count)) { + ret = -EAGAIN; + goto out; + } + + if (!f2fs_down_read_trylock(&fi->i_gc_rwsem[WRITE])) { + ret = -EAGAIN; + goto out; + } + if (do_opu && !f2fs_down_read_trylock(&fi->i_gc_rwsem[READ])) { + f2fs_up_read(&fi->i_gc_rwsem[WRITE]); + ret = -EAGAIN; + goto out; + } + } else { + ret = f2fs_convert_inline_inode(inode); + if (ret) + goto out; + + f2fs_down_read(&fi->i_gc_rwsem[WRITE]); + if (do_opu) + f2fs_down_read(&fi->i_gc_rwsem[READ]); + } + + /* + * We have to use __iomap_dio_rw() and iomap_dio_complete() instead of + * the higher-level function iomap_dio_rw() in order to ensure that the + * F2FS_DIO_WRITE counter will be decremented correctly in all cases. + */ + inc_page_count(sbi, F2FS_DIO_WRITE); + dio_flags = 0; + if (pos + count > inode->i_size) + dio_flags |= IOMAP_DIO_FORCE_WAIT; + dio = __iomap_dio_rw(iocb, from, &f2fs_iomap_ops, + &f2fs_iomap_dio_write_ops, dio_flags, iocb, 0); + if (IS_ERR_OR_NULL(dio)) { + ret = PTR_ERR_OR_ZERO(dio); + if (ret == -ENOTBLK) + ret = 0; + if (ret != -EIOCBQUEUED) + dec_page_count(sbi, F2FS_DIO_WRITE); + } else { + ret = iomap_dio_complete(dio); + } + + if (do_opu) + f2fs_up_read(&fi->i_gc_rwsem[READ]); + f2fs_up_read(&fi->i_gc_rwsem[WRITE]); + + if (ret < 0) + goto out; + if (pos + ret > inode->i_size) + f2fs_i_size_write(inode, pos + ret); + if (!do_opu) + set_inode_flag(inode, FI_UPDATE_WRITE); + + if (iov_iter_count(from)) { + ssize_t ret2; + loff_t bufio_start_pos = iocb->ki_pos; + + /* + * The direct write was partial, so we need to fall back to a + * buffered write for the remainder. + */ + + ret2 = f2fs_buffered_write_iter(iocb, from); + if (iov_iter_count(from)) + f2fs_write_failed(inode, iocb->ki_pos); + if (ret2 < 0) + goto out; + + /* + * Ensure that the pagecache pages are written to disk and + * invalidated to preserve the expected O_DIRECT semantics. + */ + if (ret2 > 0) { + loff_t bufio_end_pos = bufio_start_pos + ret2 - 1; + + ret += ret2; + + f2fs_flush_buffered_write(file->f_mapping, + bufio_start_pos, + bufio_end_pos); + } + } else { + /* iomap_dio_rw() already handled the generic_write_sync(). */ + *may_need_sync = false; + } +out: + trace_f2fs_direct_IO_exit(inode, pos, count, WRITE, ret); + return ret; +} + +static ssize_t f2fs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + struct inode *inode = file_inode(iocb->ki_filp); + const loff_t orig_pos = iocb->ki_pos; + const size_t orig_count = iov_iter_count(from); + loff_t target_size; + bool dio; + bool may_need_sync = true; + int preallocated; + const loff_t pos = iocb->ki_pos; + const ssize_t count = iov_iter_count(from); + ssize_t ret; + + if (unlikely(f2fs_cp_error(F2FS_I_SB(inode)))) { + ret = -EIO; + goto out; + } + + if (!f2fs_is_compress_backend_ready(inode)) { + ret = -EOPNOTSUPP; + goto out; + } + + if (iocb->ki_flags & IOCB_NOWAIT) { + if (!inode_trylock(inode)) { + ret = -EAGAIN; + goto out; + } + } else { + inode_lock(inode); + } + + if (f2fs_is_pinned_file(inode) && + !f2fs_overwrite_io(inode, pos, count)) { + ret = -EIO; + goto out_unlock; + } + + ret = f2fs_write_checks(iocb, from); + if (ret <= 0) + goto out_unlock; + + /* Determine whether we will do a direct write or a buffered write. */ + dio = f2fs_should_use_dio(inode, iocb, from); + + /* dio is not compatible w/ atomic write */ + if (dio && f2fs_is_atomic_file(inode)) { + ret = -EOPNOTSUPP; + goto out_unlock; + } + + /* Possibly preallocate the blocks for the write. */ + target_size = iocb->ki_pos + iov_iter_count(from); + preallocated = f2fs_preallocate_blocks(iocb, from, dio); + if (preallocated < 0) { + ret = preallocated; + } else { + if (trace_f2fs_datawrite_start_enabled()) + f2fs_trace_rw_file_path(iocb->ki_filp, iocb->ki_pos, + orig_count, WRITE); + + /* Do the actual write. */ + ret = dio ? + f2fs_dio_write_iter(iocb, from, &may_need_sync) : + f2fs_buffered_write_iter(iocb, from); + + trace_f2fs_datawrite_end(inode, orig_pos, ret); + } + + /* Don't leave any preallocated blocks around past i_size. */ + if (preallocated && i_size_read(inode) < target_size) { + f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); + filemap_invalidate_lock(inode->i_mapping); + if (!f2fs_truncate(inode)) + file_dont_truncate(inode); + filemap_invalidate_unlock(inode->i_mapping); + f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); + } else { + file_dont_truncate(inode); + } + + clear_inode_flag(inode, FI_PREALLOCATED_ALL); +out_unlock: + inode_unlock(inode); +out: + trace_f2fs_file_write_iter(inode, orig_pos, orig_count, ret); + + if (ret > 0 && may_need_sync) + ret = generic_write_sync(iocb, ret); + + /* If buffered IO was forced, flush and drop the data from + * the page cache to preserve O_DIRECT semantics + */ + if (ret > 0 && !dio && (iocb->ki_flags & IOCB_DIRECT)) + f2fs_flush_buffered_write(iocb->ki_filp->f_mapping, + orig_pos, + orig_pos + ret - 1); + + return ret; +} + +static int f2fs_file_fadvise(struct file *filp, loff_t offset, loff_t len, + int advice) +{ + struct address_space *mapping; + struct backing_dev_info *bdi; + struct inode *inode = file_inode(filp); + int err; + + trace_f2fs_fadvise(inode, offset, len, advice); + + if (advice == POSIX_FADV_SEQUENTIAL) { + if (S_ISFIFO(inode->i_mode)) + return -ESPIPE; + + mapping = filp->f_mapping; + if (!mapping || len < 0) + return -EINVAL; + + bdi = inode_to_bdi(mapping->host); + filp->f_ra.ra_pages = bdi->ra_pages * + F2FS_I_SB(inode)->seq_file_ra_mul; + spin_lock(&filp->f_lock); + filp->f_mode &= ~FMODE_RANDOM; + spin_unlock(&filp->f_lock); + return 0; + } else if (advice == POSIX_FADV_WILLNEED && offset == 0) { + /* Load extent cache at the first readahead. */ + f2fs_precache_extents(inode); + } + + err = generic_fadvise(filp, offset, len, advice); + if (err) + return err; + + if (advice == POSIX_FADV_DONTNEED && + (test_opt(F2FS_I_SB(inode), COMPRESS_CACHE) && + f2fs_compressed_file(inode))) + f2fs_invalidate_compress_pages(F2FS_I_SB(inode), inode->i_ino); + else if (advice == POSIX_FADV_NOREUSE) + err = f2fs_keep_noreuse_range(inode, offset, len); + return err; +} + +#ifdef CONFIG_COMPAT +struct compat_f2fs_gc_range { + u32 sync; + compat_u64 start; + compat_u64 len; +}; +#define F2FS_IOC32_GARBAGE_COLLECT_RANGE _IOW(F2FS_IOCTL_MAGIC, 11,\ + struct compat_f2fs_gc_range) + +static int f2fs_compat_ioc_gc_range(struct file *file, unsigned long arg) +{ + struct compat_f2fs_gc_range __user *urange; + struct f2fs_gc_range range; + int err; + + urange = compat_ptr(arg); + err = get_user(range.sync, &urange->sync); + err |= get_user(range.start, &urange->start); + err |= get_user(range.len, &urange->len); + if (err) + return -EFAULT; + + return __f2fs_ioc_gc_range(file, &range); +} + +struct compat_f2fs_move_range { + u32 dst_fd; + compat_u64 pos_in; + compat_u64 pos_out; + compat_u64 len; +}; +#define F2FS_IOC32_MOVE_RANGE _IOWR(F2FS_IOCTL_MAGIC, 9, \ + struct compat_f2fs_move_range) + +static int f2fs_compat_ioc_move_range(struct file *file, unsigned long arg) +{ + struct compat_f2fs_move_range __user *urange; + struct f2fs_move_range range; + int err; + + urange = compat_ptr(arg); + err = get_user(range.dst_fd, &urange->dst_fd); + err |= get_user(range.pos_in, &urange->pos_in); + err |= get_user(range.pos_out, &urange->pos_out); + err |= get_user(range.len, &urange->len); + if (err) + return -EFAULT; + + return __f2fs_ioc_move_range(file, &range); +} + +long f2fs_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + if (unlikely(f2fs_cp_error(F2FS_I_SB(file_inode(file))))) + return -EIO; + if (!f2fs_is_checkpoint_ready(F2FS_I_SB(file_inode(file)))) + return -ENOSPC; + + switch (cmd) { + case FS_IOC32_GETVERSION: + cmd = FS_IOC_GETVERSION; + break; + case F2FS_IOC32_GARBAGE_COLLECT_RANGE: + return f2fs_compat_ioc_gc_range(file, arg); + case F2FS_IOC32_MOVE_RANGE: + return f2fs_compat_ioc_move_range(file, arg); + case F2FS_IOC_START_ATOMIC_WRITE: + case F2FS_IOC_START_ATOMIC_REPLACE: + case F2FS_IOC_COMMIT_ATOMIC_WRITE: + case F2FS_IOC_START_VOLATILE_WRITE: + case F2FS_IOC_RELEASE_VOLATILE_WRITE: + case F2FS_IOC_ABORT_ATOMIC_WRITE: + case F2FS_IOC_SHUTDOWN: + case FITRIM: + case FS_IOC_SET_ENCRYPTION_POLICY: + case FS_IOC_GET_ENCRYPTION_PWSALT: + case FS_IOC_GET_ENCRYPTION_POLICY: + case FS_IOC_GET_ENCRYPTION_POLICY_EX: + case FS_IOC_ADD_ENCRYPTION_KEY: + case FS_IOC_REMOVE_ENCRYPTION_KEY: + case FS_IOC_REMOVE_ENCRYPTION_KEY_ALL_USERS: + case FS_IOC_GET_ENCRYPTION_KEY_STATUS: + case FS_IOC_GET_ENCRYPTION_NONCE: + case F2FS_IOC_GARBAGE_COLLECT: + case F2FS_IOC_WRITE_CHECKPOINT: + case F2FS_IOC_DEFRAGMENT: + case F2FS_IOC_FLUSH_DEVICE: + case F2FS_IOC_GET_FEATURES: + case F2FS_IOC_GET_PIN_FILE: + case F2FS_IOC_SET_PIN_FILE: + case F2FS_IOC_PRECACHE_EXTENTS: + case F2FS_IOC_RESIZE_FS: + case FS_IOC_ENABLE_VERITY: + case FS_IOC_MEASURE_VERITY: + case FS_IOC_READ_VERITY_METADATA: + case FS_IOC_GETFSLABEL: + case FS_IOC_SETFSLABEL: + case F2FS_IOC_GET_COMPRESS_BLOCKS: + case F2FS_IOC_RELEASE_COMPRESS_BLOCKS: + case F2FS_IOC_RESERVE_COMPRESS_BLOCKS: + case F2FS_IOC_SEC_TRIM_FILE: + case F2FS_IOC_GET_COMPRESS_OPTION: + case F2FS_IOC_SET_COMPRESS_OPTION: + case F2FS_IOC_DECOMPRESS_FILE: + case F2FS_IOC_COMPRESS_FILE: + case F2FS_IOC_GET_DEV_ALIAS_FILE: + case F2FS_IOC_IO_PRIO: + break; + default: + return -ENOIOCTLCMD; + } + return __f2fs_ioctl(file, cmd, (unsigned long) compat_ptr(arg)); +} +#endif + +const struct file_operations f2fs_file_operations = { + .llseek = f2fs_llseek, + .read_iter = f2fs_file_read_iter, + .write_iter = f2fs_file_write_iter, + .iopoll = iocb_bio_iopoll, + .open = f2fs_file_open, + .release = f2fs_release_file, + .mmap_prepare = f2fs_file_mmap_prepare, + .flush = f2fs_file_flush, + .fsync = f2fs_sync_file, + .fallocate = f2fs_fallocate, + .unlocked_ioctl = f2fs_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = f2fs_compat_ioctl, +#endif + .splice_read = f2fs_file_splice_read, + .splice_write = iter_file_splice_write, + .fadvise = f2fs_file_fadvise, + .fop_flags = FOP_BUFFER_RASYNC, + .setlease = generic_setlease, +}; diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c new file mode 100644 index 000000000..ffaa7ba76 --- /dev/null +++ b/fs/f2fs/gc.c @@ -0,0 +1,2461 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/f2fs/gc.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "f2fs.h" +#include "node.h" +#include "segment.h" +#include "gc.h" +#include "iostat.h" +#include + +static struct kmem_cache *victim_entry_slab; + +static unsigned int count_bits(const unsigned long *addr, + unsigned int offset, unsigned int len); + +static int gc_thread_func(void *data) +{ + struct f2fs_sb_info *sbi = data; + struct f2fs_gc_kthread *gc_th = sbi->gc_thread; + wait_queue_head_t *wq = &sbi->gc_thread->gc_wait_queue_head; + wait_queue_head_t *fggc_wq = &sbi->gc_thread->fggc_wq; + unsigned int wait_ms; + struct f2fs_gc_control gc_control = { + .victim_segno = NULL_SEGNO, + .should_migrate_blocks = false, + .err_gc_skipped = false, + .one_time = false }; + + wait_ms = gc_th->min_sleep_time; + + set_freezable(); + do { + bool sync_mode, foreground = false, gc_boost = false; + + wait_event_freezable_timeout(*wq, + kthread_should_stop() || + waitqueue_active(fggc_wq) || + gc_th->gc_wake, + msecs_to_jiffies(wait_ms)); + + if (test_opt(sbi, GC_MERGE) && waitqueue_active(fggc_wq)) { + foreground = true; + gc_control.one_time = false; + } else if (f2fs_sb_has_blkzoned(sbi)) { + gc_control.one_time = true; + } + + /* give it a try one time */ + if (gc_th->gc_wake) + gc_th->gc_wake = false; + + if (f2fs_readonly(sbi->sb)) { + stat_other_skip_bggc_count(sbi); + continue; + } + if (kthread_should_stop()) + break; + + if (sbi->sb->s_writers.frozen >= SB_FREEZE_WRITE) { + increase_sleep_time(gc_th, &wait_ms); + stat_other_skip_bggc_count(sbi); + continue; + } + + if (time_to_inject(sbi, FAULT_CHECKPOINT)) + f2fs_stop_checkpoint(sbi, false, + STOP_CP_REASON_FAULT_INJECT); + + if (!sb_start_write_trylock(sbi->sb)) { + stat_other_skip_bggc_count(sbi); + continue; + } + + /* + * [GC triggering condition] + * 0. GC is not conducted currently. + * 1. There are enough dirty segments. + * 2. IO subsystem is idle by checking the # of writeback pages. + * 3. IO subsystem is idle by checking the # of requests in + * bdev's request list. + * + * Note) We have to avoid triggering GCs frequently. + * Because it is possible that some segments can be + * invalidated soon after by user update or deletion. + * So, I'd like to wait some time to collect dirty segments. + */ + if (sbi->gc_mode == GC_URGENT_HIGH || + sbi->gc_mode == GC_URGENT_MID) { + wait_ms = gc_th->urgent_sleep_time; + f2fs_down_write_trace(&sbi->gc_lock, &gc_control.lc); + goto do_gc; + } + + if (foreground) { + f2fs_down_write_trace(&sbi->gc_lock, &gc_control.lc); + goto do_gc; + } else if (!f2fs_down_write_trylock_trace(&sbi->gc_lock, + &gc_control.lc)) { + stat_other_skip_bggc_count(sbi); + goto next; + } + + if (!is_idle(sbi, GC_TIME)) { + increase_sleep_time(gc_th, &wait_ms); + f2fs_up_write_trace(&sbi->gc_lock, &gc_control.lc); + stat_io_skip_bggc_count(sbi); + goto next; + } + + if (f2fs_sb_has_blkzoned(sbi)) { + if (has_enough_free_blocks(sbi, + gc_th->no_zoned_gc_percent)) { + wait_ms = gc_th->no_gc_sleep_time; + f2fs_up_write_trace(&sbi->gc_lock, + &gc_control.lc); + goto next; + } + if (wait_ms == gc_th->no_gc_sleep_time) + wait_ms = gc_th->max_sleep_time; + } + + if (need_to_boost_gc(sbi)) { + decrease_sleep_time(gc_th, &wait_ms); + if (f2fs_sb_has_blkzoned(sbi)) + gc_boost = true; + } else { + increase_sleep_time(gc_th, &wait_ms); + } +do_gc: + stat_inc_gc_call_count(sbi, foreground ? + FOREGROUND : BACKGROUND); + + sync_mode = (F2FS_OPTION(sbi).bggc_mode == BGGC_MODE_SYNC) || + (gc_boost && gc_th->boost_gc_greedy); + + /* foreground GC was been triggered via f2fs_balance_fs() */ + if (foreground && !f2fs_sb_has_blkzoned(sbi)) + sync_mode = false; + + gc_control.init_gc_type = sync_mode ? FG_GC : BG_GC; + gc_control.no_bg_gc = foreground; + gc_control.nr_free_secs = foreground ? 1 : 0; + + /* if return value is not zero, no victim was selected */ + if (f2fs_gc(sbi, &gc_control)) { + /* don't bother wait_ms by foreground gc */ + if (!foreground) + wait_ms = gc_th->no_gc_sleep_time; + } else { + /* reset wait_ms to default sleep time */ + if (wait_ms == gc_th->no_gc_sleep_time) + wait_ms = gc_th->min_sleep_time; + } + + if (foreground) + wake_up_all(&gc_th->fggc_wq); + + trace_f2fs_background_gc(sbi->sb, wait_ms, + prefree_segments(sbi), free_segments(sbi)); + + /* balancing f2fs's metadata periodically */ + f2fs_balance_fs_bg(sbi, true); +next: + if (sbi->gc_mode != GC_NORMAL) { + spin_lock(&sbi->gc_remaining_trials_lock); + if (sbi->gc_remaining_trials) { + sbi->gc_remaining_trials--; + if (!sbi->gc_remaining_trials) + sbi->gc_mode = GC_NORMAL; + } + spin_unlock(&sbi->gc_remaining_trials_lock); + } + sb_end_write(sbi->sb); + + } while (!kthread_should_stop()); + return 0; +} + +int f2fs_start_gc_thread(struct f2fs_sb_info *sbi) +{ + struct f2fs_gc_kthread *gc_th; + dev_t dev = sbi->sb->s_bdev->bd_dev; + + gc_th = f2fs_kmalloc(sbi, sizeof(struct f2fs_gc_kthread), GFP_KERNEL); + if (!gc_th) + return -ENOMEM; + + gc_th->urgent_sleep_time = DEF_GC_THREAD_URGENT_SLEEP_TIME; + gc_th->valid_thresh_ratio = DEF_GC_THREAD_VALID_THRESH_RATIO; + gc_th->boost_gc_multiple = BOOST_GC_MULTIPLE; + gc_th->boost_gc_greedy = GC_GREEDY; + + if (f2fs_sb_has_blkzoned(sbi)) { + gc_th->min_sleep_time = DEF_GC_THREAD_MIN_SLEEP_TIME_ZONED; + gc_th->max_sleep_time = DEF_GC_THREAD_MAX_SLEEP_TIME_ZONED; + gc_th->no_gc_sleep_time = DEF_GC_THREAD_NOGC_SLEEP_TIME_ZONED; + gc_th->no_zoned_gc_percent = LIMIT_NO_ZONED_GC; + gc_th->boost_zoned_gc_percent = LIMIT_BOOST_ZONED_GC; + } else { + gc_th->min_sleep_time = DEF_GC_THREAD_MIN_SLEEP_TIME; + gc_th->max_sleep_time = DEF_GC_THREAD_MAX_SLEEP_TIME; + gc_th->no_gc_sleep_time = DEF_GC_THREAD_NOGC_SLEEP_TIME; + gc_th->no_zoned_gc_percent = 0; + gc_th->boost_zoned_gc_percent = 0; + } + + gc_th->gc_wake = false; + + sbi->gc_thread = gc_th; + init_waitqueue_head(&sbi->gc_thread->gc_wait_queue_head); + init_waitqueue_head(&sbi->gc_thread->fggc_wq); + sbi->gc_thread->f2fs_gc_task = kthread_run(gc_thread_func, sbi, + "f2fs_gc-%u:%u", MAJOR(dev), MINOR(dev)); + if (IS_ERR(gc_th->f2fs_gc_task)) { + int err = PTR_ERR(gc_th->f2fs_gc_task); + + kfree(gc_th); + sbi->gc_thread = NULL; + return err; + } + + set_user_nice(gc_th->f2fs_gc_task, + PRIO_TO_NICE(sbi->critical_task_priority)); + return 0; +} + +void f2fs_stop_gc_thread(struct f2fs_sb_info *sbi) +{ + struct f2fs_gc_kthread *gc_th = sbi->gc_thread; + + if (!gc_th) + return; + kthread_stop(gc_th->f2fs_gc_task); + wake_up_all(&gc_th->fggc_wq); + kfree(gc_th); + sbi->gc_thread = NULL; +} + +static int select_gc_type(struct f2fs_sb_info *sbi, int gc_type) +{ + int gc_mode; + + if (gc_type == BG_GC) { + if (sbi->am.atgc_enabled) + gc_mode = GC_AT; + else + gc_mode = GC_CB; + } else { + gc_mode = GC_GREEDY; + } + + switch (sbi->gc_mode) { + case GC_IDLE_CB: + case GC_URGENT_LOW: + case GC_URGENT_MID: + gc_mode = GC_CB; + break; + case GC_IDLE_GREEDY: + case GC_URGENT_HIGH: + gc_mode = GC_GREEDY; + break; + case GC_IDLE_AT: + gc_mode = GC_AT; + break; + } + + return gc_mode; +} + +static void select_policy(struct f2fs_sb_info *sbi, int gc_type, + int type, struct victim_sel_policy *p) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + + if (p->alloc_mode == SSR || p->alloc_mode == AT_SSR) { + p->gc_mode = GC_GREEDY; + p->dirty_bitmap = dirty_i->dirty_segmap[type]; + p->max_search = dirty_i->nr_dirty[type]; + p->ofs_unit = 1; + } else { + p->gc_mode = select_gc_type(sbi, gc_type); + p->ofs_unit = SEGS_PER_SEC(sbi); + if (__is_large_section(sbi)) { + p->dirty_bitmap = dirty_i->dirty_secmap; + p->max_search = count_bits(p->dirty_bitmap, + 0, MAIN_SECS(sbi)); + } else { + p->dirty_bitmap = dirty_i->dirty_segmap[DIRTY]; + p->max_search = dirty_i->nr_dirty[DIRTY]; + } + } + + /* + * adjust candidates range, should select all dirty segments for + * foreground GC and urgent GC cases. + */ + if (gc_type != FG_GC && + (sbi->gc_mode != GC_URGENT_HIGH) && + (p->gc_mode != GC_AT && p->alloc_mode != AT_SSR) && + p->max_search > sbi->max_victim_search) + p->max_search = sbi->max_victim_search; + + /* let's select beginning hot/small space first. */ + if (f2fs_need_rand_seg(sbi)) { + p->offset = get_random_u32_below(MAIN_SECS(sbi) * + SEGS_PER_SEC(sbi)); + SIT_I(sbi)->last_victim[p->gc_mode] = p->offset; + } else if (type == CURSEG_HOT_DATA || IS_NODESEG(type)) + p->offset = 0; + else + p->offset = SIT_I(sbi)->last_victim[p->gc_mode]; +} + +static unsigned int get_max_cost(struct f2fs_sb_info *sbi, + struct victim_sel_policy *p) +{ + /* SSR allocates in a segment unit */ + if (p->alloc_mode == SSR) + return BLKS_PER_SEG(sbi); + else if (p->alloc_mode == AT_SSR) + return UINT_MAX; + + /* LFS */ + if (p->gc_mode == GC_GREEDY) + return SEGS_TO_BLKS(sbi, 2 * p->ofs_unit); + else if (p->gc_mode == GC_CB) + return UINT_MAX; + else if (p->gc_mode == GC_AT) + return UINT_MAX; + else /* No other gc_mode */ + return 0; +} + +static unsigned int check_bg_victims(struct f2fs_sb_info *sbi) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + unsigned int secno; + + /* + * If the gc_type is FG_GC, we can select victim segments + * selected by background GC before. + * Those segments guarantee they have small valid blocks. + */ + for_each_set_bit(secno, dirty_i->victim_secmap, MAIN_SECS(sbi)) { + if (sec_usage_check(sbi, secno)) + continue; + clear_bit(secno, dirty_i->victim_secmap); + return GET_SEG_FROM_SEC(sbi, secno); + } + return NULL_SEGNO; +} + +static unsigned int get_cb_cost(struct f2fs_sb_info *sbi, unsigned int segno) +{ + struct sit_info *sit_i = SIT_I(sbi); + unsigned long long mtime = 0; + unsigned int vblocks; + unsigned char age = 0; + unsigned char u; + unsigned int usable_segs_per_sec = f2fs_usable_segs_in_sec(sbi); + + mtime = f2fs_get_section_mtime(sbi, segno); + f2fs_bug_on(sbi, mtime == INVALID_MTIME); + vblocks = get_valid_blocks(sbi, segno, true); + vblocks = div_u64(vblocks, usable_segs_per_sec); + + u = BLKS_TO_SEGS(sbi, vblocks * 100); + + /* Handle if the system time has changed by the user */ + if (mtime < sit_i->min_mtime) + sit_i->min_mtime = mtime; + if (mtime > sit_i->max_mtime) + sit_i->max_mtime = mtime; + if (sit_i->max_mtime != sit_i->min_mtime) + age = 100 - div64_u64(100 * (mtime - sit_i->min_mtime), + sit_i->max_mtime - sit_i->min_mtime); + + return UINT_MAX - ((100 * (100 - u) * age) / (100 + u)); +} + +static inline unsigned int get_gc_cost(struct f2fs_sb_info *sbi, + unsigned int segno, struct victim_sel_policy *p, + unsigned int valid_thresh_ratio) +{ + if (p->alloc_mode == SSR) + return get_seg_entry(sbi, segno)->ckpt_valid_blocks; + + if (p->one_time_gc && (valid_thresh_ratio < 100) && + (get_valid_blocks(sbi, segno, true) >= + CAP_BLKS_PER_SEC(sbi) * valid_thresh_ratio / 100)) + return UINT_MAX; + + /* alloc_mode == LFS */ + if (p->gc_mode == GC_GREEDY) + return get_valid_blocks(sbi, segno, true); + else if (p->gc_mode == GC_CB) + return get_cb_cost(sbi, segno); + + f2fs_bug_on(sbi, 1); + return 0; +} + +static unsigned int count_bits(const unsigned long *addr, + unsigned int offset, unsigned int len) +{ + unsigned int end = offset + len, sum = 0; + + while (offset < end) { + if (test_bit(offset++, addr)) + ++sum; + } + return sum; +} + +static bool f2fs_check_victim_tree(struct f2fs_sb_info *sbi, + struct rb_root_cached *root) +{ +#ifdef CONFIG_F2FS_CHECK_FS + struct rb_node *cur = rb_first_cached(root), *next; + struct victim_entry *cur_ve, *next_ve; + + while (cur) { + next = rb_next(cur); + if (!next) + return true; + + cur_ve = rb_entry(cur, struct victim_entry, rb_node); + next_ve = rb_entry(next, struct victim_entry, rb_node); + + if (cur_ve->mtime > next_ve->mtime) { + f2fs_info(sbi, "broken victim_rbtree, " + "cur_mtime(%llu) next_mtime(%llu)", + cur_ve->mtime, next_ve->mtime); + return false; + } + cur = next; + } +#endif + return true; +} + +static struct victim_entry *__lookup_victim_entry(struct f2fs_sb_info *sbi, + unsigned long long mtime) +{ + struct atgc_management *am = &sbi->am; + struct rb_node *node = am->root.rb_root.rb_node; + struct victim_entry *ve = NULL; + + while (node) { + ve = rb_entry(node, struct victim_entry, rb_node); + + if (mtime < ve->mtime) + node = node->rb_left; + else + node = node->rb_right; + } + return ve; +} + +static struct victim_entry *__create_victim_entry(struct f2fs_sb_info *sbi, + unsigned long long mtime, unsigned int segno) +{ + struct atgc_management *am = &sbi->am; + struct victim_entry *ve; + + ve = f2fs_kmem_cache_alloc(victim_entry_slab, GFP_NOFS, true, NULL); + + ve->mtime = mtime; + ve->segno = segno; + + list_add_tail(&ve->list, &am->victim_list); + am->victim_count++; + + return ve; +} + +static void __insert_victim_entry(struct f2fs_sb_info *sbi, + unsigned long long mtime, unsigned int segno) +{ + struct atgc_management *am = &sbi->am; + struct rb_root_cached *root = &am->root; + struct rb_node **p = &root->rb_root.rb_node; + struct rb_node *parent = NULL; + struct victim_entry *ve; + bool left_most = true; + + /* look up rb tree to find parent node */ + while (*p) { + parent = *p; + ve = rb_entry(parent, struct victim_entry, rb_node); + + if (mtime < ve->mtime) { + p = &(*p)->rb_left; + } else { + p = &(*p)->rb_right; + left_most = false; + } + } + + ve = __create_victim_entry(sbi, mtime, segno); + + rb_link_node(&ve->rb_node, parent, p); + rb_insert_color_cached(&ve->rb_node, root, left_most); +} + +static void add_victim_entry(struct f2fs_sb_info *sbi, + struct victim_sel_policy *p, unsigned int segno) +{ + struct sit_info *sit_i = SIT_I(sbi); + unsigned long long mtime = 0; + + if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { + if (p->gc_mode == GC_AT && + get_valid_blocks(sbi, segno, true) == 0) + return; + } + + mtime = f2fs_get_section_mtime(sbi, segno); + f2fs_bug_on(sbi, mtime == INVALID_MTIME); + + /* Handle if the system time has changed by the user */ + if (mtime < sit_i->min_mtime) + sit_i->min_mtime = mtime; + if (mtime > sit_i->max_mtime) + sit_i->max_mtime = mtime; + if (mtime < sit_i->dirty_min_mtime) + sit_i->dirty_min_mtime = mtime; + if (mtime > sit_i->dirty_max_mtime) + sit_i->dirty_max_mtime = mtime; + + /* don't choose young section as candidate */ + if (sit_i->dirty_max_mtime - mtime < p->age_threshold) + return; + + __insert_victim_entry(sbi, mtime, segno); +} + +static void atgc_lookup_victim(struct f2fs_sb_info *sbi, + struct victim_sel_policy *p) +{ + struct sit_info *sit_i = SIT_I(sbi); + struct atgc_management *am = &sbi->am; + struct rb_root_cached *root = &am->root; + struct rb_node *node; + struct victim_entry *ve; + unsigned long long total_time; + unsigned long long age, u, accu; + unsigned long long max_mtime = sit_i->dirty_max_mtime; + unsigned long long min_mtime = sit_i->dirty_min_mtime; + unsigned int sec_blocks = CAP_BLKS_PER_SEC(sbi); + unsigned int vblocks; + unsigned int dirty_threshold = max(am->max_candidate_count, + am->candidate_ratio * + am->victim_count / 100); + unsigned int age_weight = am->age_weight; + unsigned int cost; + unsigned int iter = 0; + + if (max_mtime < min_mtime) + return; + + max_mtime += 1; + total_time = max_mtime - min_mtime; + + accu = div64_u64(ULLONG_MAX, total_time); + accu = min_t(unsigned long long, div_u64(accu, 100), + DEFAULT_ACCURACY_CLASS); + + node = rb_first_cached(root); +next: + ve = rb_entry_safe(node, struct victim_entry, rb_node); + if (!ve) + return; + + if (ve->mtime >= max_mtime || ve->mtime < min_mtime) + goto skip; + + /* age = 10000 * x% * 60 */ + age = div64_u64(accu * (max_mtime - ve->mtime), total_time) * + age_weight; + + vblocks = get_valid_blocks(sbi, ve->segno, true); + f2fs_bug_on(sbi, !vblocks || vblocks == sec_blocks); + + /* u = 10000 * x% * 40 */ + u = div64_u64(accu * (sec_blocks - vblocks), sec_blocks) * + (100 - age_weight); + + f2fs_bug_on(sbi, age + u >= UINT_MAX); + + cost = UINT_MAX - (age + u); + iter++; + + if (cost < p->min_cost || + (cost == p->min_cost && age > p->oldest_age)) { + p->min_cost = cost; + p->oldest_age = age; + p->min_segno = ve->segno; + } +skip: + if (iter < dirty_threshold) { + node = rb_next(node); + goto next; + } +} + +/* + * select candidates around source section in range of + * [target - dirty_threshold, target + dirty_threshold] + */ +static void atssr_lookup_victim(struct f2fs_sb_info *sbi, + struct victim_sel_policy *p) +{ + struct sit_info *sit_i = SIT_I(sbi); + struct atgc_management *am = &sbi->am; + struct victim_entry *ve; + unsigned long long age; + unsigned long long max_mtime = sit_i->dirty_max_mtime; + unsigned long long min_mtime = sit_i->dirty_min_mtime; + unsigned int vblocks; + unsigned int dirty_threshold = max(am->max_candidate_count, + am->candidate_ratio * + am->victim_count / 100); + unsigned int cost, iter; + int stage = 0; + + if (max_mtime < min_mtime) + return; + max_mtime += 1; +next_stage: + iter = 0; + ve = __lookup_victim_entry(sbi, p->age); +next_node: + if (!ve) { + if (stage++ == 0) + goto next_stage; + return; + } + + if (ve->mtime >= max_mtime || ve->mtime < min_mtime) + goto skip_node; + + age = max_mtime - ve->mtime; + + vblocks = get_seg_entry(sbi, ve->segno)->ckpt_valid_blocks; + f2fs_bug_on(sbi, !vblocks); + + /* rare case */ + if (vblocks == BLKS_PER_SEG(sbi)) + goto skip_node; + + iter++; + + age = max_mtime - abs(p->age - age); + cost = UINT_MAX - vblocks; + + if (cost < p->min_cost || + (cost == p->min_cost && age > p->oldest_age)) { + p->min_cost = cost; + p->oldest_age = age; + p->min_segno = ve->segno; + } +skip_node: + if (iter < dirty_threshold) { + ve = rb_entry(stage == 0 ? rb_prev(&ve->rb_node) : + rb_next(&ve->rb_node), + struct victim_entry, rb_node); + goto next_node; + } + + if (stage++ == 0) + goto next_stage; +} + +static void lookup_victim_by_age(struct f2fs_sb_info *sbi, + struct victim_sel_policy *p) +{ + f2fs_bug_on(sbi, !f2fs_check_victim_tree(sbi, &sbi->am.root)); + + if (p->gc_mode == GC_AT) + atgc_lookup_victim(sbi, p); + else if (p->alloc_mode == AT_SSR) + atssr_lookup_victim(sbi, p); + else + f2fs_bug_on(sbi, 1); +} + +static void release_victim_entry(struct f2fs_sb_info *sbi) +{ + struct atgc_management *am = &sbi->am; + struct victim_entry *ve, *tmp; + + list_for_each_entry_safe(ve, tmp, &am->victim_list, list) { + list_del(&ve->list); + kmem_cache_free(victim_entry_slab, ve); + am->victim_count--; + } + + am->root = RB_ROOT_CACHED; + + f2fs_bug_on(sbi, am->victim_count); + f2fs_bug_on(sbi, !list_empty(&am->victim_list)); +} + +static bool f2fs_pin_section(struct f2fs_sb_info *sbi, unsigned int segno) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + unsigned int secno = GET_SEC_FROM_SEG(sbi, segno); + + if (!dirty_i->enable_pin_section) + return false; + if (!test_and_set_bit(secno, dirty_i->pinned_secmap)) + dirty_i->pinned_secmap_cnt++; + return true; +} + +static bool f2fs_pinned_section_exists(struct dirty_seglist_info *dirty_i) +{ + return dirty_i->pinned_secmap_cnt; +} + +static bool f2fs_section_is_pinned(struct dirty_seglist_info *dirty_i, + unsigned int secno) +{ + return dirty_i->enable_pin_section && + f2fs_pinned_section_exists(dirty_i) && + test_bit(secno, dirty_i->pinned_secmap); +} + +static void f2fs_unpin_all_sections(struct f2fs_sb_info *sbi, bool enable) +{ + unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); + + if (f2fs_pinned_section_exists(DIRTY_I(sbi))) { + memset(DIRTY_I(sbi)->pinned_secmap, 0, bitmap_size); + DIRTY_I(sbi)->pinned_secmap_cnt = 0; + } + DIRTY_I(sbi)->enable_pin_section = enable; +} + +static int f2fs_gc_pinned_control(struct inode *inode, int gc_type, + unsigned int segno) +{ + if (!f2fs_is_pinned_file(inode)) + return 0; + if (gc_type != FG_GC) + return -EBUSY; + if (!f2fs_pin_section(F2FS_I_SB(inode), segno)) + f2fs_pin_file_control(inode, true); + return -EAGAIN; +} + +/* + * This function is called from two paths. + * One is garbage collection and the other is SSR segment selection. + * When it is called during GC, it just gets a victim segment + * and it does not remove it from dirty seglist. + * When it is called from SSR segment selection, it finds a segment + * which has minimum valid blocks and removes it from dirty seglist. + */ +int f2fs_get_victim(struct f2fs_sb_info *sbi, unsigned int *result, + int gc_type, int type, char alloc_mode, + unsigned long long age, bool one_time) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + struct sit_info *sm = SIT_I(sbi); + struct victim_sel_policy p = {0}; + unsigned int secno, last_victim; + unsigned int last_segment; + unsigned int nsearched; + unsigned int valid_thresh_ratio = 100; + bool is_atgc; + int ret = 0; + + mutex_lock(&dirty_i->seglist_lock); + last_segment = MAIN_SECS(sbi) * SEGS_PER_SEC(sbi); + + p.alloc_mode = alloc_mode; + p.age = age; + p.age_threshold = sbi->am.age_threshold; + if (one_time) { + p.one_time_gc = one_time; + if (has_enough_free_secs(sbi, 0, NR_PERSISTENT_LOG)) + valid_thresh_ratio = sbi->gc_thread->valid_thresh_ratio; + } + +retry: + select_policy(sbi, gc_type, type, &p); + p.min_segno = NULL_SEGNO; + p.oldest_age = 0; + p.min_cost = get_max_cost(sbi, &p); + + is_atgc = (p.gc_mode == GC_AT || p.alloc_mode == AT_SSR); + nsearched = 0; + + if (is_atgc) + SIT_I(sbi)->dirty_min_mtime = ULLONG_MAX; + + if (*result != NULL_SEGNO) { + if (!get_valid_blocks(sbi, *result, false)) { + ret = -ENODATA; + goto out; + } + + if (sec_usage_check(sbi, GET_SEC_FROM_SEG(sbi, *result))) { + ret = -EBUSY; + goto out; + } + if (gc_type == FG_GC) + clear_bit(GET_SEC_FROM_SEG(sbi, *result), dirty_i->victim_secmap); + p.min_segno = *result; + goto got_result; + } + + ret = -ENODATA; + if (p.max_search == 0) + goto out; + + if (__is_large_section(sbi) && p.alloc_mode == LFS) { + if (sbi->next_victim_seg[BG_GC] != NULL_SEGNO) { + p.min_segno = sbi->next_victim_seg[BG_GC]; + *result = p.min_segno; + sbi->next_victim_seg[BG_GC] = NULL_SEGNO; + goto got_result; + } + if (gc_type == FG_GC && + sbi->next_victim_seg[FG_GC] != NULL_SEGNO) { + p.min_segno = sbi->next_victim_seg[FG_GC]; + *result = p.min_segno; + sbi->next_victim_seg[FG_GC] = NULL_SEGNO; + goto got_result; + } + } + + last_victim = sm->last_victim[p.gc_mode]; + if (p.alloc_mode == LFS && gc_type == FG_GC) { + p.min_segno = check_bg_victims(sbi); + if (p.min_segno != NULL_SEGNO) + goto got_it; + } + + while (1) { + unsigned long cost, *dirty_bitmap; + unsigned int unit_no, segno; + + dirty_bitmap = p.dirty_bitmap; + unit_no = find_next_bit(dirty_bitmap, + last_segment / p.ofs_unit, + p.offset / p.ofs_unit); + segno = unit_no * p.ofs_unit; + if (segno >= last_segment) { + if (sm->last_victim[p.gc_mode]) { + last_segment = + sm->last_victim[p.gc_mode]; + sm->last_victim[p.gc_mode] = 0; + p.offset = 0; + continue; + } + break; + } + + p.offset = segno + p.ofs_unit; + nsearched++; + +#ifdef CONFIG_F2FS_CHECK_FS + /* + * skip selecting the invalid segno (that is failed due to block + * validity check failure during GC) to avoid endless GC loop in + * such cases. + */ + if (test_bit(segno, sm->invalid_segmap)) + goto next; +#endif + + secno = GET_SEC_FROM_SEG(sbi, segno); + + if (sec_usage_check(sbi, secno)) + goto next; + + /* Don't touch checkpointed data */ + if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { + if (p.alloc_mode == LFS) { + /* + * LFS is set to find source section during GC. + * The victim should have no checkpointed data. + */ + if (get_ckpt_valid_blocks(sbi, segno, true)) + goto next; + } else { + /* + * SSR | AT_SSR are set to find target segment + * for writes which can be full by checkpointed + * and newly written blocks. + */ + if (!f2fs_segment_has_free_slot(sbi, segno)) + goto next; + } + + if (!get_valid_blocks(sbi, segno, true)) + goto next; + } + + if (gc_type == BG_GC && test_bit(secno, dirty_i->victim_secmap)) + goto next; + + if (gc_type == FG_GC && f2fs_section_is_pinned(dirty_i, secno)) + goto next; + + if (is_atgc) { + add_victim_entry(sbi, &p, segno); + goto next; + } + + cost = get_gc_cost(sbi, segno, &p, valid_thresh_ratio); + + if (p.min_cost > cost) { + p.min_segno = segno; + p.min_cost = cost; + } +next: + if (nsearched >= p.max_search) { + if (!sm->last_victim[p.gc_mode] && segno <= last_victim) + sm->last_victim[p.gc_mode] = + last_victim + p.ofs_unit; + else + sm->last_victim[p.gc_mode] = segno + p.ofs_unit; + sm->last_victim[p.gc_mode] %= + (MAIN_SECS(sbi) * SEGS_PER_SEC(sbi)); + break; + } + } + + /* get victim for GC_AT/AT_SSR */ + if (is_atgc) { + lookup_victim_by_age(sbi, &p); + release_victim_entry(sbi); + } + + if (is_atgc && p.min_segno == NULL_SEGNO && + sm->elapsed_time < p.age_threshold) { + p.age_threshold = 0; + goto retry; + } + + if (p.min_segno != NULL_SEGNO) { +got_it: + *result = (p.min_segno / p.ofs_unit) * p.ofs_unit; +got_result: + if (p.alloc_mode == LFS) { + secno = GET_SEC_FROM_SEG(sbi, p.min_segno); + if (gc_type == FG_GC) + sbi->cur_victim_sec = secno; + else + set_bit(secno, dirty_i->victim_secmap); + } + ret = 0; + + } +out: + if (p.min_segno != NULL_SEGNO) + trace_f2fs_get_victim(sbi->sb, type, gc_type, &p, + sbi->cur_victim_sec, + prefree_segments(sbi), free_segments(sbi)); + mutex_unlock(&dirty_i->seglist_lock); + + return ret; +} + +static struct inode *find_gc_inode(struct gc_inode_list *gc_list, nid_t ino) +{ + struct inode_entry *ie; + + ie = radix_tree_lookup(&gc_list->iroot, ino); + if (ie) + return ie->inode; + return NULL; +} + +static void add_gc_inode(struct gc_inode_list *gc_list, struct inode *inode) +{ + struct inode_entry *new_ie; + + if (inode == find_gc_inode(gc_list, inode->i_ino)) { + iput(inode); + return; + } + new_ie = f2fs_kmem_cache_alloc(f2fs_inode_entry_slab, + GFP_NOFS, true, NULL); + new_ie->inode = inode; + + f2fs_radix_tree_insert(&gc_list->iroot, inode->i_ino, new_ie); + list_add_tail(&new_ie->list, &gc_list->ilist); +} + +static void put_gc_inode(struct gc_inode_list *gc_list) +{ + struct inode_entry *ie, *next_ie; + + list_for_each_entry_safe(ie, next_ie, &gc_list->ilist, list) { + radix_tree_delete(&gc_list->iroot, ie->inode->i_ino); + iput(ie->inode); + list_del(&ie->list); + kmem_cache_free(f2fs_inode_entry_slab, ie); + } +} + +static int check_valid_map(struct f2fs_sb_info *sbi, + unsigned int segno, int offset) +{ + struct sit_info *sit_i = SIT_I(sbi); + struct seg_entry *sentry; + int ret; + + down_read(&sit_i->sentry_lock); + sentry = get_seg_entry(sbi, segno); + ret = f2fs_test_bit(offset, sentry->cur_valid_map); + up_read(&sit_i->sentry_lock); + return ret; +} + +/* + * This function compares node address got in summary with that in NAT. + * On validity, copy that node with cold status, otherwise (invalid node) + * ignore that. + */ +static int gc_node_segment(struct f2fs_sb_info *sbi, + struct f2fs_summary *sum, unsigned int segno, int gc_type, + struct blk_plug *plug) +{ + struct f2fs_summary *entry; + block_t start_addr; + int off; + int phase = 0; + bool fggc = (gc_type == FG_GC); + int submitted = 0; + unsigned int usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno); + + start_addr = START_BLOCK(sbi, segno); + +next_step: + entry = sum; + + if (fggc && phase == 2) + atomic_inc(&sbi->wb_sync_req[NODE]); + + for (off = 0; off < usable_blks_in_seg; off++, entry++) { + nid_t nid = le32_to_cpu(entry->nid); + struct folio *node_folio; + struct node_info ni; + int err; + + /* stop BG_GC if there is not enough free sections. */ + if (gc_type == BG_GC && has_not_enough_free_secs(sbi, 0, 0)) + return submitted; + + if (check_valid_map(sbi, segno, off) == 0) + continue; + + if (phase == 0) { + f2fs_ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nid), 1, + META_NAT, true); + continue; + } + + if (phase == 1) { + f2fs_ra_node_page(sbi, nid); + continue; + } + + /* phase == 2 */ + node_folio = f2fs_get_node_folio(sbi, nid, NODE_TYPE_REGULAR); + if (IS_ERR(node_folio)) + continue; + + /* block may become invalid during f2fs_get_node_folio */ + if (check_valid_map(sbi, segno, off) == 0) { + f2fs_folio_put(node_folio, true); + continue; + } + + if (f2fs_get_node_info(sbi, nid, &ni, false)) { + f2fs_folio_put(node_folio, true); + continue; + } + + if (ni.blk_addr != start_addr + off) { + f2fs_folio_put(node_folio, true); + continue; + } + + err = f2fs_move_node_folio(node_folio, gc_type); + if (!err && gc_type == FG_GC) + submitted++; + stat_inc_node_blk_count(sbi, 1, gc_type); + } + + if (++phase < 3) { + blk_finish_plug(plug); + blk_start_plug(plug); + goto next_step; + } + + if (fggc) + atomic_dec(&sbi->wb_sync_req[NODE]); + return submitted; +} + +/* + * Calculate start block index indicating the given node offset. + * Be careful, caller should give this node offset only indicating direct node + * blocks. If any node offsets, which point the other types of node blocks such + * as indirect or double indirect node blocks, are given, it must be a caller's + * bug. + */ +block_t f2fs_start_bidx_of_node(unsigned int node_ofs, struct inode *inode) +{ + unsigned int indirect_blks = 2 * NIDS_PER_BLOCK + 4; + unsigned int bidx; + + if (node_ofs == 0) + return 0; + + if (node_ofs <= 2) { + bidx = node_ofs - 1; + } else if (node_ofs <= indirect_blks) { + int dec = (node_ofs - 4) / (NIDS_PER_BLOCK + 1); + + bidx = node_ofs - 2 - dec; + } else { + int dec = (node_ofs - indirect_blks - 3) / (NIDS_PER_BLOCK + 1); + + bidx = node_ofs - 5 - dec; + } + return bidx * ADDRS_PER_BLOCK(inode) + ADDRS_PER_INODE(inode); +} + +static bool is_alive(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, + struct node_info *dni, block_t blkaddr, unsigned int *nofs) +{ + struct folio *node_folio; + nid_t nid; + unsigned int ofs_in_node, max_addrs, base; + block_t source_blkaddr; + + nid = le32_to_cpu(sum->nid); + ofs_in_node = le16_to_cpu(sum->ofs_in_node); + + node_folio = f2fs_get_node_folio(sbi, nid, NODE_TYPE_REGULAR); + if (IS_ERR(node_folio)) + return false; + + if (f2fs_get_node_info(sbi, nid, dni, false)) { + f2fs_folio_put(node_folio, true); + return false; + } + + if (sum->version != dni->version) { + f2fs_warn(sbi, "%s: valid data with mismatched node version.", + __func__); + set_sbi_flag(sbi, SBI_NEED_FSCK); + } + + if (f2fs_check_nid_range(sbi, dni->ino)) { + f2fs_folio_put(node_folio, true); + return false; + } + + if (IS_INODE(node_folio)) { + base = offset_in_addr(F2FS_INODE(node_folio)); + max_addrs = DEF_ADDRS_PER_INODE; + } else { + base = 0; + max_addrs = DEF_ADDRS_PER_BLOCK; + } + + if (base + ofs_in_node >= max_addrs) { + f2fs_err(sbi, "Inconsistent blkaddr offset: base:%u, ofs_in_node:%u, max:%u, ino:%u, nid:%u", + base, ofs_in_node, max_addrs, dni->ino, dni->nid); + f2fs_folio_put(node_folio, true); + return false; + } + + *nofs = ofs_of_node(node_folio); + source_blkaddr = data_blkaddr(NULL, node_folio, ofs_in_node); + f2fs_folio_put(node_folio, true); + + if (source_blkaddr != blkaddr) { +#ifdef CONFIG_F2FS_CHECK_FS + unsigned int segno = GET_SEGNO(sbi, blkaddr); + unsigned long offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); + + if (unlikely(check_valid_map(sbi, segno, offset))) { + if (!test_and_set_bit(segno, SIT_I(sbi)->invalid_segmap)) { + f2fs_err(sbi, "mismatched blkaddr %u (source_blkaddr %u) in seg %u", + blkaddr, source_blkaddr, segno); + set_sbi_flag(sbi, SBI_NEED_FSCK); + } + } +#endif + return false; + } + return true; +} + +static int ra_data_block(struct inode *inode, pgoff_t index) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct address_space *mapping = inode->i_mapping; + struct inode *atomic_inode = NULL; + struct dnode_of_data dn; + struct folio *folio, *efolio; + struct f2fs_io_info fio = { + .sbi = sbi, + .ino = inode->i_ino, + .type = DATA, + .temp = COLD, + .op = REQ_OP_READ, + .op_flags = 0, + .encrypted_page = NULL, + .in_list = 0, + }; + int err = 0; + + f2fs_down_read(&F2FS_I(inode)->i_sem); + if (f2fs_is_cow_file(inode)) { + atomic_inode = igrab(F2FS_I(inode)->atomic_inode); + if (!atomic_inode) { + f2fs_up_read(&F2FS_I(inode)->i_sem); + return -EBUSY; + } + mapping = atomic_inode->i_mapping; + } + f2fs_up_read(&F2FS_I(inode)->i_sem); + + folio = f2fs_grab_cache_folio(mapping, index, true); + if (IS_ERR(folio)) { + err = PTR_ERR(folio); + goto out_iput; + } + + if (f2fs_lookup_read_extent_cache_block(inode, index, + &dn.data_blkaddr)) { + if (unlikely(!f2fs_is_valid_blkaddr(sbi, dn.data_blkaddr, + DATA_GENERIC_ENHANCE_READ))) { + err = -EFSCORRUPTED; + goto put_folio; + } + goto got_it; + } + + set_new_dnode(&dn, inode, NULL, NULL, 0); + err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE); + if (err) + goto put_folio; + f2fs_put_dnode(&dn); + + if (!__is_valid_data_blkaddr(dn.data_blkaddr)) { + err = -ENOENT; + goto put_folio; + } + if (unlikely(!f2fs_is_valid_blkaddr(sbi, dn.data_blkaddr, + DATA_GENERIC_ENHANCE))) { + err = -EFSCORRUPTED; + goto put_folio; + } +got_it: + /* read folio */ + fio.folio = folio; + fio.new_blkaddr = fio.old_blkaddr = dn.data_blkaddr; + + /* + * don't cache encrypted data into meta inode until previous dirty + * data were writebacked to avoid racing between GC and flush. + */ + f2fs_folio_wait_writeback(folio, DATA, true, true); + + f2fs_wait_on_block_writeback(inode, dn.data_blkaddr); + + efolio = f2fs_filemap_get_folio(META_MAPPING(sbi), dn.data_blkaddr, + FGP_LOCK | FGP_CREAT, GFP_NOFS); + if (IS_ERR(efolio)) { + err = PTR_ERR(efolio); + goto put_folio; + } + + fio.encrypted_page = &efolio->page; + + if (folio_test_uptodate(efolio)) + goto put_encrypted_page; + + err = f2fs_submit_page_bio(&fio); + if (err) + goto put_encrypted_page; + f2fs_put_page(fio.encrypted_page, false); + f2fs_folio_put(folio, true); + + f2fs_update_iostat(sbi, inode, FS_DATA_READ_IO, F2FS_BLKSIZE); + f2fs_update_iostat(sbi, NULL, FS_GDATA_READ_IO, F2FS_BLKSIZE); + + if (atomic_inode) + iput(atomic_inode); + return 0; +put_encrypted_page: + f2fs_put_page(fio.encrypted_page, true); +put_folio: + f2fs_folio_put(folio, true); +out_iput: + if (atomic_inode) + iput(atomic_inode); + return err; +} + +/* + * Move data block via META_MAPPING while keeping locked data page. + * This can be used to move blocks, aka LBAs, directly on disk. + */ +static int move_data_block(struct inode *inode, block_t bidx, + int gc_type, unsigned int segno, int off) +{ + struct address_space *mapping = inode->i_mapping; + struct inode *atomic_inode = NULL; + struct f2fs_io_info fio = { + .sbi = F2FS_I_SB(inode), + .ino = inode->i_ino, + .type = DATA, + .temp = COLD, + .op = REQ_OP_READ, + .op_flags = 0, + .encrypted_page = NULL, + .in_list = 0, + }; + struct dnode_of_data dn; + struct f2fs_summary sum; + struct node_info ni; + struct folio *folio, *mfolio, *efolio; + block_t newaddr; + int err = 0; + bool lfs_mode = f2fs_lfs_mode(fio.sbi); + int type = fio.sbi->am.atgc_enabled && (gc_type == BG_GC) && + (fio.sbi->gc_mode != GC_URGENT_HIGH) ? + CURSEG_ALL_DATA_ATGC : CURSEG_COLD_DATA; + + f2fs_down_read(&F2FS_I(inode)->i_sem); + if (f2fs_is_cow_file(inode)) { + atomic_inode = igrab(F2FS_I(inode)->atomic_inode); + if (!atomic_inode) { + f2fs_up_read(&F2FS_I(inode)->i_sem); + return -EBUSY; + } + mapping = atomic_inode->i_mapping; + } + f2fs_up_read(&F2FS_I(inode)->i_sem); + + /* do not read out */ + folio = f2fs_grab_cache_folio(mapping, bidx, false); + if (IS_ERR(folio)) { + err = PTR_ERR(folio); + goto out_iput; + } + + if (!check_valid_map(F2FS_I_SB(inode), segno, off)) { + err = -ENOENT; + goto out; + } + + err = f2fs_gc_pinned_control(inode, gc_type, segno); + if (err) + goto out; + + set_new_dnode(&dn, inode, NULL, NULL, 0); + err = f2fs_get_dnode_of_data(&dn, bidx, LOOKUP_NODE); + if (err) + goto out; + + if (unlikely(dn.data_blkaddr == NULL_ADDR)) { + folio_clear_uptodate(folio); + err = -ENOENT; + goto put_out; + } + + /* + * don't cache encrypted data into meta inode until previous dirty + * data were writebacked to avoid racing between GC and flush. + */ + f2fs_folio_wait_writeback(folio, DATA, true, true); + + f2fs_wait_on_block_writeback(inode, dn.data_blkaddr); + + err = f2fs_get_node_info(fio.sbi, dn.nid, &ni, false); + if (err) + goto put_out; + + /* read page */ + fio.folio = folio; + fio.new_blkaddr = fio.old_blkaddr = dn.data_blkaddr; + + if (lfs_mode) + f2fs_down_write(&fio.sbi->io_order_lock); + + mfolio = f2fs_grab_cache_folio(META_MAPPING(fio.sbi), + fio.old_blkaddr, false); + if (IS_ERR(mfolio)) { + err = PTR_ERR(mfolio); + goto up_out; + } + + fio.encrypted_page = folio_file_page(mfolio, fio.old_blkaddr); + + /* read source block in mfolio */ + if (!folio_test_uptodate(mfolio)) { + err = f2fs_submit_page_bio(&fio); + if (err) { + f2fs_folio_put(mfolio, true); + goto up_out; + } + + f2fs_update_iostat(fio.sbi, inode, FS_DATA_READ_IO, + F2FS_BLKSIZE); + f2fs_update_iostat(fio.sbi, NULL, FS_GDATA_READ_IO, + F2FS_BLKSIZE); + + folio_lock(mfolio); + if (unlikely(!is_meta_folio(mfolio) || + !folio_test_uptodate(mfolio))) { + err = -EIO; + f2fs_folio_put(mfolio, true); + goto up_out; + } + } + + set_summary(&sum, dn.nid, dn.ofs_in_node, ni.version); + + /* allocate block address */ + err = f2fs_allocate_data_block(fio.sbi, NULL, fio.old_blkaddr, &newaddr, + &sum, type, NULL); + if (err) { + f2fs_folio_put(mfolio, true); + /* filesystem should shutdown, no need to recovery block */ + goto up_out; + } + + efolio = f2fs_filemap_get_folio(META_MAPPING(fio.sbi), newaddr, + FGP_LOCK | FGP_CREAT, GFP_NOFS); + if (IS_ERR(efolio)) { + err = PTR_ERR(efolio); + f2fs_folio_put(mfolio, true); + goto recover_block; + } + + fio.encrypted_page = &efolio->page; + + /* write target block */ + f2fs_wait_on_page_writeback(fio.encrypted_page, DATA, true, true); + memcpy(page_address(fio.encrypted_page), + folio_address(mfolio), PAGE_SIZE); + f2fs_folio_put(mfolio, true); + + f2fs_invalidate_internal_cache(fio.sbi, fio.old_blkaddr, 1); + + set_page_dirty(fio.encrypted_page); + if (clear_page_dirty_for_io(fio.encrypted_page)) + dec_page_count(fio.sbi, F2FS_DIRTY_META); + + set_page_writeback(fio.encrypted_page); + + fio.op = REQ_OP_WRITE; + fio.op_flags = REQ_SYNC; + fio.new_blkaddr = newaddr; + f2fs_submit_page_write(&fio); + + f2fs_update_iostat(fio.sbi, NULL, FS_GC_DATA_IO, F2FS_BLKSIZE); + + f2fs_update_data_blkaddr(&dn, newaddr); + set_inode_flag(inode, FI_APPEND_WRITE); + + f2fs_put_page(fio.encrypted_page, true); +recover_block: + if (err) + f2fs_do_replace_block(fio.sbi, &sum, newaddr, fio.old_blkaddr, + true, true, true); +up_out: + if (lfs_mode) + f2fs_up_write(&fio.sbi->io_order_lock); +put_out: + f2fs_put_dnode(&dn); +out: + f2fs_folio_put(folio, true); +out_iput: + if (atomic_inode) + iput(atomic_inode); + return err; +} + +static int move_data_page(struct inode *inode, block_t bidx, int gc_type, + unsigned int segno, int off) +{ + struct folio *folio; + int err = 0; + + folio = f2fs_get_lock_data_folio(inode, bidx, true); + if (IS_ERR(folio)) + return PTR_ERR(folio); + + if (!check_valid_map(F2FS_I_SB(inode), segno, off)) { + err = -ENOENT; + goto out; + } + + err = f2fs_gc_pinned_control(inode, gc_type, segno); + if (err) + goto out; + + if (gc_type == BG_GC) { + if (folio_test_writeback(folio)) { + err = -EAGAIN; + goto out; + } + folio_mark_dirty(folio); + folio_set_f2fs_gcing(folio); + } else { + struct f2fs_io_info fio = { + .sbi = F2FS_I_SB(inode), + .ino = inode->i_ino, + .type = DATA, + .temp = COLD, + .op = REQ_OP_WRITE, + .op_flags = REQ_SYNC, + .old_blkaddr = NULL_ADDR, + .folio = folio, + .encrypted_page = NULL, + .need_lock = LOCK_REQ, + .io_type = FS_GC_DATA_IO, + }; + bool is_dirty = folio_test_dirty(folio); + +retry: + f2fs_folio_wait_writeback(folio, DATA, true, true); + + folio_mark_dirty(folio); + if (folio_clear_dirty_for_io(folio)) { + inode_dec_dirty_pages(inode); + f2fs_remove_dirty_inode(inode); + } + + folio_set_f2fs_gcing(folio); + + err = f2fs_do_write_data_page(&fio); + if (err) { + folio_clear_f2fs_gcing(folio); + if (err == -ENOMEM) { + memalloc_retry_wait(GFP_NOFS); + goto retry; + } + if (is_dirty) + folio_mark_dirty(folio); + } + } +out: + f2fs_folio_put(folio, true); + return err; +} + +/* + * This function tries to get parent node of victim data block, and identifies + * data block validity. If the block is valid, copy that with cold status and + * modify parent node. + * If the parent node is not valid or the data block address is different, + * the victim data block is ignored. + */ +static int gc_data_segment(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, + struct gc_inode_list *gc_list, unsigned int segno, int gc_type, + bool force_migrate, struct blk_plug *plug) +{ + struct super_block *sb = sbi->sb; + struct f2fs_summary *entry; + block_t start_addr; + int off; + int phase = 0; + int submitted = 0; + unsigned int usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno); + + start_addr = START_BLOCK(sbi, segno); + +next_step: + entry = sum; + + for (off = 0; off < usable_blks_in_seg; off++, entry++) { + struct inode *inode; + struct node_info dni; /* dnode info for the data */ + unsigned int ofs_in_node, nofs; + block_t start_bidx; + nid_t nid = le32_to_cpu(entry->nid); + + /* + * stop BG_GC if there is not enough free sections. + * Or, stop GC if the segment becomes fully valid caused by + * race condition along with SSR block allocation. + */ + if ((gc_type == BG_GC && has_not_enough_free_secs(sbi, 0, 0)) || + (!force_migrate && get_valid_blocks(sbi, segno, true) == + CAP_BLKS_PER_SEC(sbi))) + return submitted; + + if (check_valid_map(sbi, segno, off) == 0) + continue; + + if (phase == 0) { + f2fs_ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nid), 1, + META_NAT, true); + continue; + } + + if (phase == 1) { + f2fs_ra_node_page(sbi, nid); + continue; + } + + /* Get an inode by ino with checking validity */ + if (!is_alive(sbi, entry, &dni, start_addr + off, &nofs)) + continue; + + if (phase == 2) { + f2fs_ra_node_page(sbi, dni.ino); + continue; + } + + ofs_in_node = le16_to_cpu(entry->ofs_in_node); + + if (phase == 3) { + struct folio *data_folio; + int err; + + inode = f2fs_iget(sb, dni.ino); + if (IS_ERR(inode)) + continue; + + if (is_bad_inode(inode) || + special_file(inode->i_mode)) { + iput(inode); + continue; + } + + if (f2fs_has_inline_data(inode)) { + iput(inode); + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_err_ratelimited(sbi, + "inode %llu has both inline_data flag and " + "data block, nid=%u, ofs_in_node=%u", + inode->i_ino, dni.nid, ofs_in_node); + continue; + } + + err = f2fs_gc_pinned_control(inode, gc_type, segno); + if (err == -EAGAIN) { + iput(inode); + return submitted; + } + + if (!f2fs_down_write_trylock( + &F2FS_I(inode)->i_gc_rwsem[WRITE])) { + iput(inode); + sbi->skipped_gc_rwsem++; + continue; + } + + start_bidx = f2fs_start_bidx_of_node(nofs, inode) + + ofs_in_node; + + if (f2fs_meta_inode_gc_required(inode)) { + int err = ra_data_block(inode, start_bidx); + + f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); + if (err) { + iput(inode); + continue; + } + add_gc_inode(gc_list, inode); + continue; + } + + data_folio = f2fs_get_read_data_folio(inode, start_bidx, + REQ_RAHEAD, true, NULL); + f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); + if (IS_ERR(data_folio)) { + iput(inode); + continue; + } + + f2fs_folio_put(data_folio, false); + add_gc_inode(gc_list, inode); + continue; + } + + /* phase 4 */ + inode = find_gc_inode(gc_list, dni.ino); + if (inode) { + struct f2fs_inode_info *fi = F2FS_I(inode); + bool locked = false; + int err; + + if (S_ISREG(inode->i_mode)) { + if (!f2fs_down_write_trylock(&fi->i_gc_rwsem[WRITE])) { + sbi->skipped_gc_rwsem++; + continue; + } + if (!f2fs_down_write_trylock( + &fi->i_gc_rwsem[READ])) { + sbi->skipped_gc_rwsem++; + f2fs_up_write(&fi->i_gc_rwsem[WRITE]); + continue; + } + locked = true; + + /* wait for all inflight aio data */ + inode_dio_wait(inode); + } + + start_bidx = f2fs_start_bidx_of_node(nofs, inode) + + ofs_in_node; + if (f2fs_meta_inode_gc_required(inode)) + err = move_data_block(inode, start_bidx, + gc_type, segno, off); + else + err = move_data_page(inode, start_bidx, gc_type, + segno, off); + + if (!err && (gc_type == FG_GC || + f2fs_meta_inode_gc_required(inode))) + submitted++; + + if (locked) { + f2fs_up_write(&fi->i_gc_rwsem[READ]); + f2fs_up_write(&fi->i_gc_rwsem[WRITE]); + } + + stat_inc_data_blk_count(sbi, 1, gc_type); + } + } + + if (++phase < 5) { + blk_finish_plug(plug); + blk_start_plug(plug); + goto next_step; + } + + return submitted; +} + +static int __get_victim(struct f2fs_sb_info *sbi, unsigned int *victim, + int gc_type, bool one_time) +{ + struct sit_info *sit_i = SIT_I(sbi); + int ret; + + down_write(&sit_i->sentry_lock); + ret = f2fs_get_victim(sbi, victim, gc_type, NO_CHECK_TYPE, + LFS, 0, one_time); + up_write(&sit_i->sentry_lock); + return ret; +} + +static int do_garbage_collect(struct f2fs_sb_info *sbi, + unsigned int start_segno, + struct gc_inode_list *gc_list, int gc_type, + bool force_migrate, bool one_time) +{ + struct blk_plug plug; + unsigned int segno = start_segno; + unsigned int end_segno = start_segno + SEGS_PER_SEC(sbi); + unsigned int sec_end_segno; + int seg_freed = 0, migrated = 0; + unsigned char type; + unsigned char data_type; + int submitted = 0, sum_blk_cnt; + + if (__is_large_section(sbi)) { + sec_end_segno = rounddown(end_segno, SEGS_PER_SEC(sbi)); + + /* + * zone-capacity can be less than zone-size in zoned devices, + * resulting in less than expected usable segments in the zone, + * calculate the end segno in the zone which can be garbage + * collected + */ + if (f2fs_sb_has_blkzoned(sbi)) + sec_end_segno -= SEGS_PER_SEC(sbi) - + f2fs_usable_segs_in_sec(sbi); + + if (gc_type == BG_GC || one_time) { + unsigned int window_granularity = + sbi->migration_window_granularity; + + if (f2fs_sb_has_blkzoned(sbi) && + !has_enough_free_blocks(sbi, + sbi->gc_thread->boost_zoned_gc_percent)) + window_granularity *= + sbi->gc_thread->boost_gc_multiple; + + end_segno = start_segno + window_granularity; + } + + if (end_segno > sec_end_segno) + end_segno = sec_end_segno; + } + + sanity_check_seg_type(sbi, get_seg_entry(sbi, segno)->type); + + segno = rounddown(segno, sbi->sums_per_block); + sum_blk_cnt = DIV_ROUND_UP(end_segno - segno, sbi->sums_per_block); + /* readahead multi ssa blocks those have contiguous address */ + if (__is_large_section(sbi)) + f2fs_ra_meta_pages(sbi, GET_SUM_BLOCK(sbi, segno), + sum_blk_cnt, META_SSA, true); + + /* reference all summary page */ + while (segno < end_segno) { + struct folio *sum_folio = f2fs_get_sum_folio(sbi, segno); + + segno += sbi->sums_per_block; + if (IS_ERR(sum_folio)) { + int err = PTR_ERR(sum_folio); + + end_segno = segno - sbi->sums_per_block; + segno = rounddown(start_segno, sbi->sums_per_block); + while (segno < end_segno) { + sum_folio = filemap_get_folio(META_MAPPING(sbi), + GET_SUM_BLOCK(sbi, segno)); + folio_put_refs(sum_folio, 2); + segno += sbi->sums_per_block; + } + return err; + } + folio_unlock(sum_folio); + } + + blk_start_plug(&plug); + + segno = start_segno; + while (segno < end_segno) { + unsigned int cur_segno; + + /* find segment summary of victim */ + struct folio *sum_folio = filemap_get_folio(META_MAPPING(sbi), + GET_SUM_BLOCK(sbi, segno)); + unsigned int block_end_segno = rounddown(segno, sbi->sums_per_block) + + sbi->sums_per_block; + + if (block_end_segno > end_segno) + block_end_segno = end_segno; + + if (is_cursec(sbi, GET_SEC_FROM_SEG(sbi, segno))) { + f2fs_err(sbi, "%s: segment %u is used by log", + __func__, segno); + f2fs_bug_on(sbi, 1); + goto next_block; + } + + if (!folio_test_uptodate(sum_folio) || + unlikely(f2fs_cp_error(sbi))) + goto next_block; + + for (cur_segno = segno; cur_segno < block_end_segno; + cur_segno++) { + struct f2fs_summary_block *sum; + + if (get_valid_blocks(sbi, cur_segno, false) == 0) + goto freed; + if (gc_type == BG_GC && __is_large_section(sbi) && + migrated >= sbi->migration_granularity) + continue; + + if (migrated == 0) { + type = IS_DATASEG(get_seg_entry(sbi, cur_segno)->type) ? + SUM_TYPE_DATA : SUM_TYPE_NODE; + data_type = (type == SUM_TYPE_DATA) ? DATA : NODE; + } + + sum = SUM_BLK_PAGE_ADDR(sbi, sum_folio, cur_segno); + if (type != GET_SUM_TYPE(sum_footer(sbi, sum))) { + f2fs_err(sbi, "Inconsistent segment (%u) type " + "[%d, %d] in SIT and SSA", + cur_segno, type, + GET_SUM_TYPE( + sum_footer(sbi, sum))); + f2fs_stop_checkpoint(sbi, false, + STOP_CP_REASON_CORRUPTED_SUMMARY); + continue; + } + + /* + * this is to avoid deadlock: + * - lock_page(sum_page) - f2fs_replace_block + * - check_valid_map() - down_write(sentry_lock) + * - down_read(sentry_lock) - change_curseg() + * - lock_page(sum_page) + */ + if (type == SUM_TYPE_NODE) + submitted += gc_node_segment(sbi, sum->entries, + cur_segno, gc_type, &plug); + else + submitted += gc_data_segment(sbi, sum->entries, + gc_list, cur_segno, + gc_type, force_migrate, &plug); + + stat_inc_gc_seg_count(sbi, data_type, gc_type); + sbi->gc_reclaimed_segs[sbi->gc_mode]++; + migrated++; + +freed: + if (gc_type == FG_GC && + get_valid_blocks(sbi, cur_segno, false) == 0) + seg_freed++; + + if (__is_large_section(sbi)) + sbi->next_victim_seg[gc_type] = + (cur_segno + 1 < sec_end_segno) ? + cur_segno + 1 : NULL_SEGNO; + + if (unlikely(freezing(current))) { + folio_put_refs(sum_folio, 2); + goto stop; + } + } +next_block: + folio_put_refs(sum_folio, 2); + segno = block_end_segno; + } + +stop: + if (submitted) + f2fs_submit_merged_write(sbi, data_type); + + blk_finish_plug(&plug); + + if (migrated) + stat_inc_gc_sec_count(sbi, data_type, gc_type); + + return seg_freed; +} + +int f2fs_gc(struct f2fs_sb_info *sbi, struct f2fs_gc_control *gc_control) +{ + int gc_type = gc_control->init_gc_type; + unsigned int segno = gc_control->victim_segno; + int sec_freed = 0, seg_freed = 0, total_freed = 0, total_sec_freed = 0; + int ret = 0; + struct cp_control cpc; + struct gc_inode_list gc_list = { + .ilist = LIST_HEAD_INIT(gc_list.ilist), + .iroot = RADIX_TREE_INIT(gc_list.iroot, GFP_NOFS), + }; + unsigned int skipped_round = 0, round = 0; + unsigned int upper_secs; + + trace_f2fs_gc_begin(sbi->sb, gc_type, gc_control->no_bg_gc, + gc_control->nr_free_secs, + get_pages(sbi, F2FS_DIRTY_NODES), + get_pages(sbi, F2FS_DIRTY_DENTS), + get_pages(sbi, F2FS_DIRTY_IMETA), + free_sections(sbi), + free_segments(sbi), + reserved_segments(sbi), + prefree_segments(sbi)); + + cpc.reason = __get_cp_reason(sbi); +gc_more: + sbi->skipped_gc_rwsem = 0; + if (unlikely(!(sbi->sb->s_flags & SB_ACTIVE))) { + ret = -EINVAL; + goto stop; + } + if (unlikely(f2fs_cp_error(sbi))) { + ret = -EIO; + goto stop; + } + + /* Let's run FG_GC, if we don't have enough space. */ + if (has_not_enough_free_secs(sbi, 0, 0)) { + gc_type = FG_GC; + gc_control->one_time = false; + + /* + * For example, if there are many prefree_segments below given + * threshold, we can make them free by checkpoint. Then, we + * secure free segments which doesn't need fggc any more. + */ + if (prefree_segments(sbi)) { + stat_inc_cp_call_count(sbi, TOTAL_CALL); + ret = f2fs_write_checkpoint(sbi, &cpc); + if (ret) + goto stop; + /* Reset due to checkpoint */ + sec_freed = 0; + } + } + + /* f2fs_balance_fs doesn't need to do BG_GC in critical path. */ + if (gc_type == BG_GC && gc_control->no_bg_gc) { + ret = -EINVAL; + goto stop; + } +retry: + if (unlikely(freezing(current))) { + ret = 0; + goto stop; + } + ret = __get_victim(sbi, &segno, gc_type, gc_control->one_time); + if (ret) { + /* allow to search victim from sections has pinned data */ + if (ret == -ENODATA && gc_type == FG_GC && + f2fs_pinned_section_exists(DIRTY_I(sbi))) { + f2fs_unpin_all_sections(sbi, false); + goto retry; + } + goto stop; + } + + seg_freed = do_garbage_collect(sbi, segno, &gc_list, gc_type, + gc_control->should_migrate_blocks, + gc_control->one_time); + if (seg_freed < 0) + goto stop; + + total_freed += seg_freed; + + if (seg_freed == f2fs_usable_segs_in_sec(sbi)) { + sec_freed++; + total_sec_freed++; + } + + if (gc_control->one_time) + goto stop; + + if (gc_type == FG_GC) { + sbi->cur_victim_sec = NULL_SEGNO; + + if (has_enough_free_secs(sbi, sec_freed, 0)) { + if (!gc_control->no_bg_gc && + total_sec_freed < gc_control->nr_free_secs) + goto go_gc_more; + goto stop; + } + if (sbi->skipped_gc_rwsem) + skipped_round++; + round++; + if (skipped_round > MAX_SKIP_GC_COUNT && + skipped_round * 2 >= round) { + stat_inc_cp_call_count(sbi, TOTAL_CALL); + ret = f2fs_write_checkpoint(sbi, &cpc); + goto stop; + } + } else if (has_enough_free_secs(sbi, 0, 0)) { + goto stop; + } + + upper_secs = __get_secs_required(sbi); + + /* + * Write checkpoint to reclaim prefree segments. + * We need more three extra sections for writer's data/node/dentry. + */ + if (free_sections(sbi) <= upper_secs + NR_GC_CHECKPOINT_SECS && + prefree_segments(sbi)) { + stat_inc_cp_call_count(sbi, TOTAL_CALL); + ret = f2fs_write_checkpoint(sbi, &cpc); + if (ret) + goto stop; + /* Reset due to checkpoint */ + sec_freed = 0; + } +go_gc_more: + segno = NULL_SEGNO; + goto gc_more; + +stop: + SIT_I(sbi)->last_victim[ALLOC_NEXT] = 0; + SIT_I(sbi)->last_victim[FLUSH_DEVICE] = gc_control->victim_segno; + + if (gc_type == FG_GC) + f2fs_unpin_all_sections(sbi, true); + + trace_f2fs_gc_end(sbi->sb, ret, total_freed, total_sec_freed, + get_pages(sbi, F2FS_DIRTY_NODES), + get_pages(sbi, F2FS_DIRTY_DENTS), + get_pages(sbi, F2FS_DIRTY_IMETA), + free_sections(sbi), + free_segments(sbi), + reserved_segments(sbi), + prefree_segments(sbi)); + + f2fs_up_write_trace(&sbi->gc_lock, &gc_control->lc); + + put_gc_inode(&gc_list); + + if (gc_control->err_gc_skipped && !ret) + ret = total_sec_freed ? 0 : -EAGAIN; + return ret; +} + +int __init f2fs_create_garbage_collection_cache(void) +{ + victim_entry_slab = f2fs_kmem_cache_create("f2fs_victim_entry", + sizeof(struct victim_entry)); + return victim_entry_slab ? 0 : -ENOMEM; +} + +void f2fs_destroy_garbage_collection_cache(void) +{ + kmem_cache_destroy(victim_entry_slab); +} + +static void init_atgc_management(struct f2fs_sb_info *sbi) +{ + struct atgc_management *am = &sbi->am; + + if (test_opt(sbi, ATGC) && + SIT_I(sbi)->elapsed_time >= DEF_GC_THREAD_AGE_THRESHOLD) + am->atgc_enabled = true; + + am->root = RB_ROOT_CACHED; + INIT_LIST_HEAD(&am->victim_list); + am->victim_count = 0; + + am->candidate_ratio = DEF_GC_THREAD_CANDIDATE_RATIO; + am->max_candidate_count = DEF_GC_THREAD_MAX_CANDIDATE_COUNT; + am->age_weight = DEF_GC_THREAD_AGE_WEIGHT; + am->age_threshold = DEF_GC_THREAD_AGE_THRESHOLD; +} + +void f2fs_build_gc_manager(struct f2fs_sb_info *sbi) +{ + sbi->gc_pin_file_threshold = DEF_GC_FAILED_PINNED_FILES; + + /* give warm/cold data area from slower device */ + if (f2fs_is_multi_device(sbi) && !__is_large_section(sbi)) + SIT_I(sbi)->last_victim[ALLOC_NEXT] = + GET_SEGNO(sbi, FDEV(0).end_blk) + 1; + + init_atgc_management(sbi); +} + +int f2fs_gc_range(struct f2fs_sb_info *sbi, + unsigned int start_seg, unsigned int end_seg, + bool dry_run, unsigned int dry_run_sections) +{ + unsigned int segno; + unsigned int gc_secs = dry_run_sections; + + if (unlikely(f2fs_cp_error(sbi))) + return -EIO; + + stat_inc_gc_call_count(sbi, FOREGROUND); + for (segno = start_seg; segno <= end_seg; segno += SEGS_PER_SEC(sbi)) { + struct gc_inode_list gc_list = { + .ilist = LIST_HEAD_INIT(gc_list.ilist), + .iroot = RADIX_TREE_INIT(gc_list.iroot, GFP_NOFS), + }; + + /* + * avoid migrating empty section, as it can be allocated by + * log in parallel. + */ + if (!get_valid_blocks(sbi, segno, true)) + continue; + + if (is_cursec(sbi, GET_SEC_FROM_SEG(sbi, segno))) + continue; + + do_garbage_collect(sbi, segno, &gc_list, FG_GC, true, false); + put_gc_inode(&gc_list); + + if (!dry_run && get_valid_blocks(sbi, segno, true)) + return -EAGAIN; + if (dry_run && dry_run_sections && + !get_valid_blocks(sbi, segno, true) && --gc_secs == 0) + break; + + if (fatal_signal_pending(current)) + return -ERESTARTSYS; + } + + return 0; +} + +static int free_segment_range(struct f2fs_sb_info *sbi, + unsigned int secs, bool dry_run) +{ + unsigned int next_inuse, start, end; + struct cp_control cpc = { CP_RESIZE, 0, 0, 0 }; + int gc_mode, gc_type; + int err = 0; + int type; + + /* Force block allocation for GC */ + MAIN_SECS(sbi) -= secs; + start = MAIN_SECS(sbi) * SEGS_PER_SEC(sbi); + end = MAIN_SEGS(sbi) - 1; + + mutex_lock(&DIRTY_I(sbi)->seglist_lock); + for (gc_mode = 0; gc_mode < MAX_GC_POLICY; gc_mode++) + if (SIT_I(sbi)->last_victim[gc_mode] >= start) + SIT_I(sbi)->last_victim[gc_mode] = 0; + + for (gc_type = BG_GC; gc_type <= FG_GC; gc_type++) + if (sbi->next_victim_seg[gc_type] >= start) + sbi->next_victim_seg[gc_type] = NULL_SEGNO; + mutex_unlock(&DIRTY_I(sbi)->seglist_lock); + + /* Move out cursegs from the target range */ + for (type = CURSEG_HOT_DATA; type < NR_CURSEG_PERSIST_TYPE; type++) { + err = f2fs_allocate_segment_for_resize(sbi, type, start, end); + if (err) + goto out; + } + + /* do GC to move out valid blocks in the range */ + err = f2fs_gc_range(sbi, start, end, dry_run, 0); + if (err || dry_run) + goto out; + + stat_inc_cp_call_count(sbi, TOTAL_CALL); + err = f2fs_write_checkpoint(sbi, &cpc); + if (err) + goto out; + + next_inuse = find_next_inuse(FREE_I(sbi), end + 1, start); + if (next_inuse <= end) { + f2fs_err(sbi, "segno %u should be free but still inuse!", + next_inuse); + f2fs_bug_on(sbi, 1); + } +out: + MAIN_SECS(sbi) += secs; + return err; +} + +static void update_sb_metadata(struct f2fs_sb_info *sbi, int secs) +{ + struct f2fs_super_block *raw_sb = F2FS_RAW_SUPER(sbi); + int section_count; + int segment_count; + int segment_count_main; + long long block_count; + int segs = secs * SEGS_PER_SEC(sbi); + + f2fs_down_write(&sbi->sb_lock); + + section_count = le32_to_cpu(raw_sb->section_count); + segment_count = le32_to_cpu(raw_sb->segment_count); + segment_count_main = le32_to_cpu(raw_sb->segment_count_main); + block_count = le64_to_cpu(raw_sb->block_count); + + raw_sb->section_count = cpu_to_le32(section_count + secs); + raw_sb->segment_count = cpu_to_le32(segment_count + segs); + raw_sb->segment_count_main = cpu_to_le32(segment_count_main + segs); + raw_sb->block_count = cpu_to_le64(block_count + + (long long)SEGS_TO_BLKS(sbi, segs)); + if (f2fs_is_multi_device(sbi)) { + int last_dev = sbi->s_ndevs - 1; + int dev_segs = + le32_to_cpu(raw_sb->devs[last_dev].total_segments); + + raw_sb->devs[last_dev].total_segments = + cpu_to_le32(dev_segs + segs); + } + + f2fs_up_write(&sbi->sb_lock); +} + +static void update_fs_metadata(struct f2fs_sb_info *sbi, int secs) +{ + int segs = secs * SEGS_PER_SEC(sbi); + long long blks = SEGS_TO_BLKS(sbi, segs); + long long user_block_count = + le64_to_cpu(F2FS_CKPT(sbi)->user_block_count); + + SM_I(sbi)->segment_count = (int)SM_I(sbi)->segment_count + segs; + MAIN_SEGS(sbi) = (int)MAIN_SEGS(sbi) + segs; + MAIN_SECS(sbi) += secs; + if (sbi->allocate_section_hint > MAIN_SECS(sbi)) + sbi->allocate_section_hint = MAIN_SECS(sbi); + FREE_I(sbi)->free_sections = (int)FREE_I(sbi)->free_sections + secs; + FREE_I(sbi)->free_segments = (int)FREE_I(sbi)->free_segments + segs; + F2FS_CKPT(sbi)->user_block_count = cpu_to_le64(user_block_count + blks); + + if (f2fs_is_multi_device(sbi)) { + int last_dev = sbi->s_ndevs - 1; + + sbi->allocate_section_hint = FDEV(0).total_segments / + SEGS_PER_SEC(sbi); + + FDEV(last_dev).total_segments = + (int)FDEV(last_dev).total_segments + segs; + FDEV(last_dev).end_blk = + (long long)FDEV(last_dev).end_blk + blks; +#ifdef CONFIG_BLK_DEV_ZONED + FDEV(last_dev).nr_blkz = FDEV(last_dev).nr_blkz + + div_u64(blks, sbi->blocks_per_blkz); +#endif + } +} + +int f2fs_resize_fs(struct file *filp, __u64 block_count) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(file_inode(filp)); + __u64 old_block_count, shrunk_blocks; + struct cp_control cpc = { CP_RESIZE, 0, 0, 0 }; + struct f2fs_lock_context lc; + struct f2fs_lock_context glc; + struct f2fs_lock_context clc; + unsigned int secs; + int err = 0; + __u32 rem; + + old_block_count = le64_to_cpu(F2FS_RAW_SUPER(sbi)->block_count); + if (block_count > old_block_count) + return -EINVAL; + + if (f2fs_is_multi_device(sbi)) { + int last_dev = sbi->s_ndevs - 1; + __u64 last_segs = FDEV(last_dev).total_segments; + + if (block_count + SEGS_TO_BLKS(sbi, last_segs) <= + old_block_count) + return -EINVAL; + } + + /* new fs size should align to section size */ + div_u64_rem(block_count, BLKS_PER_SEC(sbi), &rem); + if (rem) + return -EINVAL; + + if (block_count == old_block_count) + return 0; + + if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) { + f2fs_err(sbi, "Should run fsck to repair first."); + return -EFSCORRUPTED; + } + + if (test_opt(sbi, DISABLE_CHECKPOINT)) { + f2fs_err(sbi, "Checkpoint should be enabled."); + return -EINVAL; + } + + err = mnt_want_write_file(filp); + if (err) + return err; + + shrunk_blocks = old_block_count - block_count; + secs = div_u64(shrunk_blocks, BLKS_PER_SEC(sbi)); + + /* stop other GC */ + if (!f2fs_down_write_trylock_trace(&sbi->gc_lock, &glc)) { + err = -EAGAIN; + goto out_drop_write; + } + + /* stop CP to protect MAIN_SEC in free_segment_range */ + f2fs_lock_op(sbi, &lc); + + spin_lock(&sbi->stat_lock); + if (shrunk_blocks + valid_user_blocks(sbi) + + sbi->current_reserved_blocks + sbi->unusable_block_count + + F2FS_OPTION(sbi).root_reserved_blocks > sbi->user_block_count) + err = -ENOSPC; + spin_unlock(&sbi->stat_lock); + + if (err) + goto out_unlock; + + err = free_segment_range(sbi, secs, true); + +out_unlock: + f2fs_unlock_op(sbi, &lc); + f2fs_up_write_trace(&sbi->gc_lock, &glc); +out_drop_write: + mnt_drop_write_file(filp); + if (err) + return err; + + err = freeze_super(sbi->sb, FREEZE_HOLDER_KERNEL, NULL); + if (err) + return err; + + if (f2fs_readonly(sbi->sb)) { + err = thaw_super(sbi->sb, FREEZE_HOLDER_KERNEL, NULL); + if (err) + return err; + return -EROFS; + } + + f2fs_down_write_trace(&sbi->gc_lock, &glc); + f2fs_down_write_trace(&sbi->cp_global_sem, &clc); + + spin_lock(&sbi->stat_lock); + if (shrunk_blocks + valid_user_blocks(sbi) + + sbi->current_reserved_blocks + sbi->unusable_block_count + + F2FS_OPTION(sbi).root_reserved_blocks > sbi->user_block_count) + err = -ENOSPC; + else + sbi->user_block_count -= shrunk_blocks; + spin_unlock(&sbi->stat_lock); + if (err) + goto out_err; + + set_sbi_flag(sbi, SBI_IS_RESIZEFS); + err = free_segment_range(sbi, secs, false); + if (err) + goto recover_out; + + update_sb_metadata(sbi, -secs); + + err = f2fs_commit_super(sbi, false); + if (err) { + update_sb_metadata(sbi, secs); + goto recover_out; + } + + update_fs_metadata(sbi, -secs); + clear_sbi_flag(sbi, SBI_IS_RESIZEFS); + set_sbi_flag(sbi, SBI_IS_DIRTY); + + stat_inc_cp_call_count(sbi, TOTAL_CALL); + err = f2fs_write_checkpoint(sbi, &cpc); + if (err) { + update_fs_metadata(sbi, secs); + update_sb_metadata(sbi, secs); + f2fs_commit_super(sbi, false); + } +recover_out: + clear_sbi_flag(sbi, SBI_IS_RESIZEFS); + if (err) { + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_err(sbi, "resize_fs failed, should run fsck to repair!"); + + spin_lock(&sbi->stat_lock); + sbi->user_block_count += shrunk_blocks; + spin_unlock(&sbi->stat_lock); + } +out_err: + f2fs_up_write_trace(&sbi->cp_global_sem, &clc); + f2fs_up_write_trace(&sbi->gc_lock, &glc); + thaw_super(sbi->sb, FREEZE_HOLDER_KERNEL, NULL); + return err; +} diff --git a/fs/f2fs/gc.h b/fs/f2fs/gc.h new file mode 100644 index 000000000..6c4d45675 --- /dev/null +++ b/fs/f2fs/gc.h @@ -0,0 +1,202 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * fs/f2fs/gc.h + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + */ +#define GC_THREAD_MIN_WB_PAGES 1 /* + * a threshold to determine + * whether IO subsystem is idle + * or not + */ +#define DEF_GC_THREAD_URGENT_SLEEP_TIME 500 /* 500 ms */ +#define DEF_GC_THREAD_MIN_SLEEP_TIME 30000 /* milliseconds */ +#define DEF_GC_THREAD_MAX_SLEEP_TIME 60000 +#define DEF_GC_THREAD_NOGC_SLEEP_TIME 300000 /* wait 5 min */ + +/* GC sleep parameters for zoned deivces */ +#define DEF_GC_THREAD_MIN_SLEEP_TIME_ZONED 10 +#define DEF_GC_THREAD_MAX_SLEEP_TIME_ZONED 20 +#define DEF_GC_THREAD_NOGC_SLEEP_TIME_ZONED 60000 + +/* choose candidates from sections which has age of more than 7 days */ +#define DEF_GC_THREAD_AGE_THRESHOLD (60 * 60 * 24 * 7) +#define DEF_GC_THREAD_CANDIDATE_RATIO 20 /* select 20% oldest sections as candidates */ +#define DEF_GC_THREAD_MAX_CANDIDATE_COUNT 10 /* select at most 10 sections as candidates */ +#define DEF_GC_THREAD_AGE_WEIGHT 60 /* age weight */ +#define DEF_GC_THREAD_VALID_THRESH_RATIO 80 /* do not GC over 80% valid block ratio for one time GC */ +#define DEFAULT_ACCURACY_CLASS 10000 /* accuracy class */ + +#define LIMIT_INVALID_BLOCK 40 /* percentage over total user space */ +#define LIMIT_FREE_BLOCK 40 /* percentage over invalid + free space */ + +#define LIMIT_NO_ZONED_GC 60 /* percentage over total user space of no gc for zoned devices */ +#define LIMIT_BOOST_ZONED_GC 25 /* percentage over total user space of boosted gc for zoned devices */ +#define DEF_MIGRATION_WINDOW_GRANULARITY_ZONED 3 +#define BOOST_GC_MULTIPLE 5 +#define ZONED_PIN_SEC_REQUIRED_COUNT 1 + +#define DEF_GC_FAILED_PINNED_FILES 2048 +#define MAX_GC_FAILED_PINNED_FILES USHRT_MAX + +/* Search max. number of dirty segments to select a victim segment */ +#define DEF_MAX_VICTIM_SEARCH 4096 /* covers 8GB */ + +#define NR_GC_CHECKPOINT_SECS (3) /* data/node/dentry sections */ + +struct f2fs_gc_kthread { + struct task_struct *f2fs_gc_task; + wait_queue_head_t gc_wait_queue_head; + + /* for gc sleep time */ + unsigned int urgent_sleep_time; + unsigned int min_sleep_time; + unsigned int max_sleep_time; + unsigned int no_gc_sleep_time; + + /* for changing gc mode */ + bool gc_wake; + + /* for GC_MERGE mount option */ + wait_queue_head_t fggc_wq; /* + * caller of f2fs_balance_fs() + * will wait on this wait queue. + */ + + /* for gc control for zoned devices */ + unsigned int no_zoned_gc_percent; + unsigned int boost_zoned_gc_percent; + unsigned int valid_thresh_ratio; + unsigned int boost_gc_multiple; + unsigned int boost_gc_greedy; +}; + +struct gc_inode_list { + struct list_head ilist; + struct radix_tree_root iroot; +}; + +struct victim_entry { + struct rb_node rb_node; /* rb node located in rb-tree */ + unsigned long long mtime; /* mtime of section */ + unsigned int segno; /* segment No. */ + struct list_head list; +}; + +/* + * inline functions + */ + +/* + * On a Zoned device zone-capacity can be less than zone-size and if + * zone-capacity is not aligned to f2fs segment size(2MB), then the segment + * starting just before zone-capacity has some blocks spanning across the + * zone-capacity, these blocks are not usable. + * Such spanning segments can be in free list so calculate the sum of usable + * blocks in currently free segments including normal and spanning segments. + */ +static inline block_t free_segs_blk_count_zoned(struct f2fs_sb_info *sbi) +{ + block_t free_seg_blks = 0; + struct free_segmap_info *free_i = FREE_I(sbi); + int j; + + spin_lock(&free_i->segmap_lock); + for (j = 0; j < MAIN_SEGS(sbi); j++) + if (!test_bit(j, free_i->free_segmap)) + free_seg_blks += f2fs_usable_blks_in_seg(sbi, j); + spin_unlock(&free_i->segmap_lock); + + return free_seg_blks; +} + +static inline block_t free_segs_blk_count(struct f2fs_sb_info *sbi) +{ + if (f2fs_sb_has_blkzoned(sbi)) + return free_segs_blk_count_zoned(sbi); + + return SEGS_TO_BLKS(sbi, free_segments(sbi)); +} + +static inline block_t free_user_blocks(struct f2fs_sb_info *sbi) +{ + block_t free_blks, ovp_blks; + + free_blks = free_segs_blk_count(sbi); + ovp_blks = SEGS_TO_BLKS(sbi, overprovision_segments(sbi)); + + if (free_blks < ovp_blks) + return 0; + + return free_blks - ovp_blks; +} + +static inline block_t limit_invalid_user_blocks(block_t user_block_count) +{ + return (long)(user_block_count * LIMIT_INVALID_BLOCK) / 100; +} + +static inline block_t limit_free_user_blocks(block_t reclaimable_user_blocks) +{ + return (long)(reclaimable_user_blocks * LIMIT_FREE_BLOCK) / 100; +} + +static inline void increase_sleep_time(struct f2fs_gc_kthread *gc_th, + unsigned int *wait) +{ + unsigned int min_time = gc_th->min_sleep_time; + unsigned int max_time = gc_th->max_sleep_time; + + if (*wait == gc_th->no_gc_sleep_time) + return; + + if ((long long)*wait + (long long)min_time > (long long)max_time) + *wait = max_time; + else + *wait += min_time; +} + +static inline void decrease_sleep_time(struct f2fs_gc_kthread *gc_th, + unsigned int *wait) +{ + unsigned int min_time = gc_th->min_sleep_time; + + if (*wait == gc_th->no_gc_sleep_time) + *wait = gc_th->max_sleep_time; + + if ((long long)*wait - (long long)min_time < (long long)min_time) + *wait = min_time; + else + *wait -= min_time; +} + +static inline bool has_enough_free_blocks(struct f2fs_sb_info *sbi, + unsigned int limit_perc) +{ + return free_sections(sbi) > ((sbi->total_sections * limit_perc) / 100); +} + +static inline bool has_enough_invalid_blocks(struct f2fs_sb_info *sbi) +{ + block_t user_block_count = sbi->user_block_count; + block_t invalid_user_blocks = user_block_count - + written_block_count(sbi); + /* + * Background GC is triggered with the following conditions. + * 1. There are a number of invalid blocks. + * 2. There is not enough free space. + */ + return (invalid_user_blocks > + limit_invalid_user_blocks(user_block_count) && + free_user_blocks(sbi) < + limit_free_user_blocks(invalid_user_blocks)); +} + +static inline bool need_to_boost_gc(struct f2fs_sb_info *sbi) +{ + if (f2fs_sb_has_blkzoned(sbi)) + return !has_enough_free_blocks(sbi, + sbi->gc_thread->boost_zoned_gc_percent); + return has_enough_invalid_blocks(sbi); +} diff --git a/fs/f2fs/hash.c b/fs/f2fs/hash.c new file mode 100644 index 000000000..14082fe5e --- /dev/null +++ b/fs/f2fs/hash.c @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/f2fs/hash.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * + * Portions of this code from linux/fs/ext3/hash.c + * + * Copyright (C) 2002 by Theodore Ts'o + */ +#include +#include +#include +#include +#include + +#include "f2fs.h" + +/* + * Hashing code copied from ext3 + */ +#define DELTA 0x9E3779B9 + +static void TEA_transform(unsigned int buf[4], unsigned int const in[]) +{ + __u32 sum = 0; + __u32 b0 = buf[0], b1 = buf[1]; + __u32 a = in[0], b = in[1], c = in[2], d = in[3]; + int n = 16; + + do { + sum += DELTA; + b0 += ((b1 << 4)+a) ^ (b1+sum) ^ ((b1 >> 5)+b); + b1 += ((b0 << 4)+c) ^ (b0+sum) ^ ((b0 >> 5)+d); + } while (--n); + + buf[0] += b0; + buf[1] += b1; +} + +static void str2hashbuf(const unsigned char *msg, size_t len, + unsigned int *buf, int num) +{ + unsigned pad, val; + int i; + + pad = (__u32)len | ((__u32)len << 8); + pad |= pad << 16; + + val = pad; + if (len > num * 4) + len = num * 4; + for (i = 0; i < len; i++) { + if ((i % 4) == 0) + val = pad; + val = msg[i] + (val << 8); + if ((i % 4) == 3) { + *buf++ = val; + val = pad; + num--; + } + } + if (--num >= 0) + *buf++ = val; + while (--num >= 0) + *buf++ = pad; +} + +static u32 TEA_hash_name(const u8 *p, size_t len) +{ + __u32 in[8], buf[4]; + + /* Initialize the default seed for the hash checksum functions */ + buf[0] = 0x67452301; + buf[1] = 0xefcdab89; + buf[2] = 0x98badcfe; + buf[3] = 0x10325476; + + while (1) { + str2hashbuf(p, len, in, 4); + TEA_transform(buf, in); + p += 16; + if (len <= 16) + break; + len -= 16; + } + return buf[0] & ~F2FS_HASH_COL_BIT; +} + +/* + * Compute @fname->hash. For all directories, @fname->disk_name must be set. + * For casefolded directories, @fname->usr_fname must be set, and also + * @fname->cf_name if the filename is valid Unicode and is not "." or "..". + */ +void f2fs_hash_filename(const struct inode *dir, struct f2fs_filename *fname) +{ + const u8 *name = fname->disk_name.name; + size_t len = fname->disk_name.len; + + WARN_ON_ONCE(!name); + + if (name_is_dot_dotdot(name, len)) { + fname->hash = 0; + return; + } + +#if IS_ENABLED(CONFIG_UNICODE) + if (IS_CASEFOLDED(dir)) { + /* + * If the casefolded name is provided, hash it instead of the + * on-disk name. If the casefolded name is *not* provided, that + * should only be because the name wasn't valid Unicode or was + * "." or "..", so fall back to treating the name as an opaque + * byte sequence. Note that to handle encrypted directories, + * the fallback must use usr_fname (plaintext) rather than + * disk_name (ciphertext). + */ + WARN_ON_ONCE(!fname->usr_fname->name); + if (fname->cf_name.name) { + name = fname->cf_name.name; + len = fname->cf_name.len; + } else { + name = fname->usr_fname->name; + len = fname->usr_fname->len; + } + if (IS_ENCRYPTED(dir)) { + struct qstr tmp = QSTR_INIT(name, len); + + fname->hash = + cpu_to_le32(fscrypt_fname_siphash(dir, &tmp)); + return; + } + } +#endif + fname->hash = cpu_to_le32(TEA_hash_name(name, len)); +} diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c new file mode 100644 index 000000000..e2f7bedf1 --- /dev/null +++ b/fs/f2fs/inline.c @@ -0,0 +1,860 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/f2fs/inline.c + * Copyright (c) 2013, Intel Corporation + * Authors: Huajun Li + * Haicheng Li + */ + +#include +#include +#include +#include + +#include "f2fs.h" +#include "node.h" +#include + +static bool support_inline_data(struct inode *inode) +{ + if (f2fs_used_in_atomic_write(inode)) + return false; + if (!S_ISREG(inode->i_mode) && !S_ISLNK(inode->i_mode)) + return false; + if (i_size_read(inode) > MAX_INLINE_DATA(inode)) + return false; + return true; +} + +bool f2fs_may_inline_data(struct inode *inode) +{ + if (!support_inline_data(inode)) + return false; + + return !f2fs_post_read_required(inode); +} + +static bool inode_has_blocks(struct inode *inode, struct folio *ifolio) +{ + struct f2fs_inode *ri = F2FS_INODE(ifolio); + int i; + + if (F2FS_HAS_BLOCKS(inode)) + return true; + + for (i = 0; i < DEF_NIDS_PER_INODE; i++) { + if (ri->i_nid[i]) + return true; + } + return false; +} + +bool f2fs_sanity_check_inline_data(struct inode *inode, struct folio *ifolio) +{ + if (!f2fs_has_inline_data(inode)) + return false; + + if (inode_has_blocks(inode, ifolio)) + return false; + + if (!support_inline_data(inode)) + return true; + + /* + * used by sanity_check_inode(), when disk layout fields has not + * been synchronized to inmem fields. + */ + return (S_ISREG(inode->i_mode) && + (file_is_encrypt(inode) || file_is_verity(inode) || + (F2FS_I(inode)->i_flags & F2FS_COMPR_FL))); +} + +bool f2fs_may_inline_dentry(struct inode *inode) +{ + if (!test_opt(F2FS_I_SB(inode), INLINE_DENTRY)) + return false; + + if (!S_ISDIR(inode->i_mode)) + return false; + + return true; +} + +void f2fs_do_read_inline_data(struct folio *folio, struct folio *ifolio) +{ + struct inode *inode = folio->mapping->host; + + if (folio_test_uptodate(folio)) + return; + + f2fs_bug_on(F2FS_I_SB(inode), folio->index); + + folio_zero_segment(folio, MAX_INLINE_DATA(inode), folio_size(folio)); + + /* Copy the whole inline data block */ + memcpy_to_folio(folio, 0, inline_data_addr(inode, ifolio), + MAX_INLINE_DATA(inode)); + if (!folio_test_uptodate(folio)) + folio_mark_uptodate(folio); +} + +void f2fs_truncate_inline_inode(struct inode *inode, struct folio *ifolio, + u64 from) +{ + void *addr; + + if (from >= MAX_INLINE_DATA(inode)) + return; + + addr = inline_data_addr(inode, ifolio); + + f2fs_folio_wait_writeback(ifolio, NODE, true, true); + memset(addr + from, 0, MAX_INLINE_DATA(inode) - from); + folio_mark_dirty(ifolio); + + if (from == 0) + clear_inode_flag(inode, FI_DATA_EXIST); +} + +int f2fs_read_inline_data(struct inode *inode, struct folio *folio) +{ + struct folio *ifolio; + + ifolio = f2fs_get_inode_folio(F2FS_I_SB(inode), inode->i_ino); + if (IS_ERR(ifolio)) { + folio_unlock(folio); + return PTR_ERR(ifolio); + } + + if (!f2fs_has_inline_data(inode)) { + f2fs_folio_put(ifolio, true); + return -EAGAIN; + } + + if (folio->index) + folio_zero_segment(folio, 0, folio_size(folio)); + else + f2fs_do_read_inline_data(folio, ifolio); + + if (!folio_test_uptodate(folio)) + folio_mark_uptodate(folio); + f2fs_folio_put(ifolio, true); + folio_unlock(folio); + return 0; +} + +int f2fs_convert_inline_folio(struct dnode_of_data *dn, struct folio *folio) +{ + struct f2fs_io_info fio = { + .sbi = F2FS_I_SB(dn->inode), + .ino = dn->inode->i_ino, + .type = DATA, + .op = REQ_OP_WRITE, + .op_flags = REQ_SYNC | REQ_PRIO, + .folio = folio, + .encrypted_page = NULL, + .io_type = FS_DATA_IO, + }; + struct node_info ni; + int dirty, err; + + if (!f2fs_exist_data(dn->inode)) + goto clear_out; + + err = f2fs_reserve_block(dn, 0); + if (err) + return err; + + err = f2fs_get_node_info(fio.sbi, dn->nid, &ni, false); + if (err) { + f2fs_truncate_data_blocks_range(dn, 1); + f2fs_put_dnode(dn); + return err; + } + + fio.version = ni.version; + + if (unlikely(dn->data_blkaddr != NEW_ADDR)) { + f2fs_put_dnode(dn); + set_sbi_flag(fio.sbi, SBI_NEED_FSCK); + f2fs_warn(fio.sbi, "%s: corrupted inline inode ino=%llu, i_addr[0]:0x%x, run fsck to fix.", + __func__, dn->inode->i_ino, dn->data_blkaddr); + f2fs_handle_error(fio.sbi, ERROR_INVALID_BLKADDR); + fserror_report_file_metadata(dn->inode, -EFSCORRUPTED, GFP_NOFS); + return -EFSCORRUPTED; + } + + f2fs_bug_on(F2FS_F_SB(folio), folio_test_writeback(folio)); + + f2fs_do_read_inline_data(folio, dn->inode_folio); + folio_mark_dirty(folio); + + /* clear dirty state */ + dirty = folio_clear_dirty_for_io(folio); + + /* write data page to try to make data consistent */ + folio_start_writeback(folio); + fio.old_blkaddr = dn->data_blkaddr; + set_inode_flag(dn->inode, FI_HOT_DATA); + f2fs_outplace_write_data(dn, &fio); + f2fs_folio_wait_writeback(folio, DATA, true, true); + if (dirty) { + inode_dec_dirty_pages(dn->inode); + f2fs_remove_dirty_inode(dn->inode); + } + + /* this converted inline_data should be recovered. */ + set_inode_flag(dn->inode, FI_APPEND_WRITE); + + /* clear inline data and flag after data writeback */ + f2fs_truncate_inline_inode(dn->inode, dn->inode_folio, 0); + folio_clear_f2fs_inline(dn->inode_folio); +clear_out: + stat_dec_inline_inode(dn->inode); + clear_inode_flag(dn->inode, FI_INLINE_DATA); + f2fs_put_dnode(dn); + return 0; +} + +int f2fs_convert_inline_inode(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct dnode_of_data dn; + struct f2fs_lock_context lc; + struct folio *ifolio, *folio; + int err = 0; + + if (f2fs_hw_is_readonly(sbi) || f2fs_readonly(sbi->sb)) + return -EROFS; + + if (!f2fs_has_inline_data(inode)) + return 0; + + err = f2fs_dquot_initialize(inode); + if (err) + return err; + + folio = f2fs_grab_cache_folio(inode->i_mapping, 0, false); + if (IS_ERR(folio)) + return PTR_ERR(folio); + + f2fs_lock_op(sbi, &lc); + + ifolio = f2fs_get_inode_folio(sbi, inode->i_ino); + if (IS_ERR(ifolio)) { + err = PTR_ERR(ifolio); + goto out; + } + + set_new_dnode(&dn, inode, ifolio, ifolio, 0); + + if (f2fs_has_inline_data(inode)) + err = f2fs_convert_inline_folio(&dn, folio); + + f2fs_put_dnode(&dn); +out: + f2fs_unlock_op(sbi, &lc); + + f2fs_folio_put(folio, true); + + if (!err) + f2fs_balance_fs(sbi, dn.node_changed); + + return err; +} + +int f2fs_write_inline_data(struct inode *inode, struct folio *folio) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct folio *ifolio; + + ifolio = f2fs_get_inode_folio(sbi, inode->i_ino); + if (IS_ERR(ifolio)) + return PTR_ERR(ifolio); + + if (!f2fs_has_inline_data(inode)) { + f2fs_folio_put(ifolio, true); + return -EAGAIN; + } + + f2fs_bug_on(F2FS_I_SB(inode), folio->index); + + f2fs_folio_wait_writeback(ifolio, NODE, true, true); + memcpy_from_folio(inline_data_addr(inode, ifolio), + folio, 0, MAX_INLINE_DATA(inode)); + folio_mark_dirty(ifolio); + + f2fs_clear_page_cache_dirty_tag(folio); + + set_inode_flag(inode, FI_APPEND_WRITE); + set_inode_flag(inode, FI_DATA_EXIST); + + folio_clear_f2fs_inline(ifolio); + f2fs_folio_put(ifolio, true); + return 0; +} + +int f2fs_recover_inline_data(struct inode *inode, struct folio *nfolio) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_inode *ri = NULL; + void *src_addr, *dst_addr; + + /* + * The inline_data recovery policy is as follows. + * [prev.] [next] of inline_data flag + * o o -> recover inline_data + * o x -> remove inline_data, and then recover data blocks + * x o -> remove data blocks, and then recover inline_data + * x x -> recover data blocks + */ + if (IS_INODE(nfolio)) + ri = F2FS_INODE(nfolio); + + if (f2fs_has_inline_data(inode) && + ri && (ri->i_inline & F2FS_INLINE_DATA)) { + struct folio *ifolio; +process_inline: + ifolio = f2fs_get_inode_folio(sbi, inode->i_ino); + if (IS_ERR(ifolio)) + return PTR_ERR(ifolio); + + f2fs_folio_wait_writeback(ifolio, NODE, true, true); + + src_addr = inline_data_addr(inode, nfolio); + dst_addr = inline_data_addr(inode, ifolio); + memcpy(dst_addr, src_addr, MAX_INLINE_DATA(inode)); + + set_inode_flag(inode, FI_INLINE_DATA); + set_inode_flag(inode, FI_DATA_EXIST); + + folio_mark_dirty(ifolio); + f2fs_folio_put(ifolio, true); + return 1; + } + + if (f2fs_has_inline_data(inode)) { + struct folio *ifolio = f2fs_get_inode_folio(sbi, inode->i_ino); + if (IS_ERR(ifolio)) + return PTR_ERR(ifolio); + f2fs_truncate_inline_inode(inode, ifolio, 0); + stat_dec_inline_inode(inode); + clear_inode_flag(inode, FI_INLINE_DATA); + f2fs_folio_put(ifolio, true); + } else if (ri && (ri->i_inline & F2FS_INLINE_DATA)) { + int ret; + + ret = f2fs_truncate_blocks(inode, 0, false); + if (ret) + return ret; + stat_inc_inline_inode(inode); + goto process_inline; + } + return 0; +} + +struct f2fs_dir_entry *f2fs_find_in_inline_dir(struct inode *dir, + const struct f2fs_filename *fname, + struct folio **res_folio, + bool use_hash) +{ + struct f2fs_sb_info *sbi = F2FS_SB(dir->i_sb); + struct f2fs_dir_entry *de; + struct f2fs_dentry_ptr d; + struct folio *ifolio; + void *inline_dentry; + + ifolio = f2fs_get_inode_folio(sbi, dir->i_ino); + if (IS_ERR(ifolio)) { + *res_folio = ifolio; + return NULL; + } + + inline_dentry = inline_data_addr(dir, ifolio); + + make_dentry_ptr_inline(dir, &d, inline_dentry); + de = f2fs_find_target_dentry(&d, fname, NULL, use_hash); + folio_unlock(ifolio); + if (IS_ERR(de)) { + *res_folio = ERR_CAST(de); + de = NULL; + } + if (de) + *res_folio = ifolio; + else + f2fs_folio_put(ifolio, false); + + return de; +} + +int f2fs_make_empty_inline_dir(struct inode *inode, struct inode *parent, + struct folio *ifolio) +{ + struct f2fs_dentry_ptr d; + void *inline_dentry; + + inline_dentry = inline_data_addr(inode, ifolio); + + make_dentry_ptr_inline(inode, &d, inline_dentry); + f2fs_do_make_empty_dir(inode, parent, &d); + + folio_mark_dirty(ifolio); + + /* update i_size to MAX_INLINE_DATA */ + if (i_size_read(inode) < MAX_INLINE_DATA(inode)) + f2fs_i_size_write(inode, MAX_INLINE_DATA(inode)); + return 0; +} + +/* + * NOTE: ipage is grabbed by caller, but if any error occurs, we should + * release ipage in this function. + */ +static int f2fs_move_inline_dirents(struct inode *dir, struct folio *ifolio, + void *inline_dentry) +{ + struct folio *folio; + struct dnode_of_data dn; + struct f2fs_dentry_block *dentry_blk; + struct f2fs_dentry_ptr src, dst; + int err; + + folio = f2fs_grab_cache_folio(dir->i_mapping, 0, true); + if (IS_ERR(folio)) { + f2fs_folio_put(ifolio, true); + return PTR_ERR(folio); + } + + set_new_dnode(&dn, dir, ifolio, NULL, 0); + err = f2fs_reserve_block(&dn, 0); + if (err) + goto out; + + if (unlikely(dn.data_blkaddr != NEW_ADDR)) { + f2fs_put_dnode(&dn); + set_sbi_flag(F2FS_F_SB(folio), SBI_NEED_FSCK); + f2fs_warn(F2FS_F_SB(folio), "%s: corrupted inline inode ino=%llu, i_addr[0]:0x%x, run fsck to fix.", + __func__, dir->i_ino, dn.data_blkaddr); + f2fs_handle_error(F2FS_F_SB(folio), ERROR_INVALID_BLKADDR); + err = -EFSCORRUPTED; + fserror_report_file_metadata(dn.inode, err, GFP_NOFS); + goto out; + } + + f2fs_folio_wait_writeback(folio, DATA, true, true); + + dentry_blk = folio_address(folio); + + /* + * Start by zeroing the full block, to ensure that all unused space is + * zeroed and no uninitialized memory is leaked to disk. + */ + memset(dentry_blk, 0, F2FS_BLKSIZE); + + make_dentry_ptr_inline(dir, &src, inline_dentry); + make_dentry_ptr_block(dir, &dst, dentry_blk); + + /* copy data from inline dentry block to new dentry block */ + memcpy(dst.bitmap, src.bitmap, src.nr_bitmap); + memcpy(dst.dentry, src.dentry, SIZE_OF_DIR_ENTRY * src.max); + memcpy(dst.filename, src.filename, src.max * F2FS_SLOT_LEN); + + if (!folio_test_uptodate(folio)) + folio_mark_uptodate(folio); + folio_mark_dirty(folio); + + /* clear inline dir and flag after data writeback */ + f2fs_truncate_inline_inode(dir, ifolio, 0); + + stat_dec_inline_dir(dir); + clear_inode_flag(dir, FI_INLINE_DENTRY); + + /* + * should retrieve reserved space which was used to keep + * inline_dentry's structure for backward compatibility. + */ + if (!f2fs_sb_has_flexible_inline_xattr(F2FS_I_SB(dir)) && + !f2fs_has_inline_xattr(dir)) + F2FS_I(dir)->i_inline_xattr_size = 0; + + f2fs_i_depth_write(dir, 1); + if (i_size_read(dir) < PAGE_SIZE) + f2fs_i_size_write(dir, PAGE_SIZE); +out: + f2fs_folio_put(folio, true); + return err; +} + +static int f2fs_add_inline_entries(struct inode *dir, void *inline_dentry) +{ + struct f2fs_dentry_ptr d; + unsigned long bit_pos = 0; + int err = 0; + + make_dentry_ptr_inline(dir, &d, inline_dentry); + + while (bit_pos < d.max) { + struct f2fs_dir_entry *de; + struct f2fs_filename fname; + nid_t ino; + umode_t fake_mode; + + if (!test_bit_le(bit_pos, d.bitmap)) { + bit_pos++; + continue; + } + + de = &d.dentry[bit_pos]; + + if (unlikely(!de->name_len)) { + bit_pos++; + continue; + } + if (unlikely(le16_to_cpu(de->name_len) > F2FS_NAME_LEN || + bit_pos + GET_DENTRY_SLOTS(le16_to_cpu(de->name_len)) > + d.max)) { + err = -EFSCORRUPTED; + goto punch_dentry_pages; + } + + /* + * We only need the disk_name and hash to move the dentry. + * We don't need the original or casefolded filenames. + */ + memset(&fname, 0, sizeof(fname)); + fname.disk_name.name = d.filename[bit_pos]; + fname.disk_name.len = le16_to_cpu(de->name_len); + fname.hash = de->hash_code; + + ino = le32_to_cpu(de->ino); + fake_mode = fs_ftype_to_dtype(de->file_type) << S_DT_SHIFT; + + err = f2fs_add_regular_entry(dir, &fname, NULL, ino, fake_mode); + if (err) + goto punch_dentry_pages; + + bit_pos += GET_DENTRY_SLOTS(le16_to_cpu(de->name_len)); + } + return 0; + +punch_dentry_pages: + truncate_inode_pages(&dir->i_data, 0); + f2fs_truncate_blocks(dir, 0, false); + f2fs_remove_dirty_inode(dir); + return err; +} + +static int f2fs_move_rehashed_dirents(struct inode *dir, struct folio *ifolio, + void *inline_dentry) +{ + void *backup_dentry; + int err; + + backup_dentry = f2fs_kmalloc(F2FS_I_SB(dir), + MAX_INLINE_DATA(dir), GFP_F2FS_ZERO); + if (!backup_dentry) { + f2fs_folio_put(ifolio, true); + return -ENOMEM; + } + + memcpy(backup_dentry, inline_dentry, MAX_INLINE_DATA(dir)); + f2fs_truncate_inline_inode(dir, ifolio, 0); + + folio_unlock(ifolio); + + err = f2fs_add_inline_entries(dir, backup_dentry); + if (err) + goto recover; + + folio_lock(ifolio); + + stat_dec_inline_dir(dir); + clear_inode_flag(dir, FI_INLINE_DENTRY); + + /* + * should retrieve reserved space which was used to keep + * inline_dentry's structure for backward compatibility. + */ + if (!f2fs_sb_has_flexible_inline_xattr(F2FS_I_SB(dir)) && + !f2fs_has_inline_xattr(dir)) + F2FS_I(dir)->i_inline_xattr_size = 0; + + kfree(backup_dentry); + return 0; +recover: + folio_lock(ifolio); + f2fs_folio_wait_writeback(ifolio, NODE, true, true); + memcpy(inline_dentry, backup_dentry, MAX_INLINE_DATA(dir)); + f2fs_i_depth_write(dir, 0); + f2fs_i_size_write(dir, MAX_INLINE_DATA(dir)); + folio_mark_dirty(ifolio); + f2fs_folio_put(ifolio, true); + + kfree(backup_dentry); + return err; +} + +static int do_convert_inline_dir(struct inode *dir, struct folio *ifolio, + void *inline_dentry) +{ + if (!F2FS_I(dir)->i_dir_level) + return f2fs_move_inline_dirents(dir, ifolio, inline_dentry); + else + return f2fs_move_rehashed_dirents(dir, ifolio, inline_dentry); +} + +int f2fs_try_convert_inline_dir(struct inode *dir, struct dentry *dentry) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + struct folio *ifolio; + struct f2fs_filename fname; + struct f2fs_lock_context lc; + void *inline_dentry = NULL; + int err = 0; + + if (!f2fs_has_inline_dentry(dir)) + return 0; + + f2fs_lock_op(sbi, &lc); + + err = f2fs_setup_filename(dir, &dentry->d_name, 0, &fname); + if (err) + goto out; + + ifolio = f2fs_get_inode_folio(sbi, dir->i_ino); + if (IS_ERR(ifolio)) { + err = PTR_ERR(ifolio); + goto out_fname; + } + + if (f2fs_has_enough_room(dir, ifolio, &fname)) { + f2fs_folio_put(ifolio, true); + goto out_fname; + } + + inline_dentry = inline_data_addr(dir, ifolio); + + err = do_convert_inline_dir(dir, ifolio, inline_dentry); + if (!err) + f2fs_folio_put(ifolio, true); +out_fname: + f2fs_free_filename(&fname); +out: + f2fs_unlock_op(sbi, &lc); + return err; +} + +int f2fs_add_inline_entry(struct inode *dir, const struct f2fs_filename *fname, + struct inode *inode, nid_t ino, umode_t mode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + struct folio *ifolio; + unsigned int bit_pos; + void *inline_dentry = NULL; + struct f2fs_dentry_ptr d; + int slots = GET_DENTRY_SLOTS(fname->disk_name.len); + struct folio *folio = NULL; + int err = 0; + + ifolio = f2fs_get_inode_folio(sbi, dir->i_ino); + if (IS_ERR(ifolio)) + return PTR_ERR(ifolio); + + inline_dentry = inline_data_addr(dir, ifolio); + make_dentry_ptr_inline(dir, &d, inline_dentry); + + bit_pos = f2fs_room_for_filename(d.bitmap, slots, d.max); + if (bit_pos >= d.max) { + err = do_convert_inline_dir(dir, ifolio, inline_dentry); + if (err) + return err; + err = -EAGAIN; + goto out; + } + + if (inode) { + f2fs_down_write_nested(&F2FS_I(inode)->i_sem, + SINGLE_DEPTH_NESTING); + folio = f2fs_init_inode_metadata(inode, dir, fname, ifolio); + if (IS_ERR(folio)) { + err = PTR_ERR(folio); + goto fail; + } + } + + f2fs_folio_wait_writeback(ifolio, NODE, true, true); + + f2fs_update_dentry(ino, mode, &d, &fname->disk_name, fname->hash, + bit_pos); + + folio_mark_dirty(ifolio); + + /* we don't need to mark_inode_dirty now */ + if (inode) { + f2fs_i_pino_write(inode, dir->i_ino); + + /* synchronize inode page's data from inode cache */ + if (is_inode_flag_set(inode, FI_NEW_INODE)) + f2fs_update_inode(inode, folio); + + f2fs_folio_put(folio, true); + } + + f2fs_update_parent_metadata(dir, inode, 0); +fail: + if (inode) + f2fs_up_write(&F2FS_I(inode)->i_sem); +out: + f2fs_folio_put(ifolio, true); + return err; +} + +void f2fs_delete_inline_entry(struct f2fs_dir_entry *dentry, + struct folio *folio, struct inode *dir, struct inode *inode) +{ + struct f2fs_dentry_ptr d; + void *inline_dentry; + int slots = GET_DENTRY_SLOTS(le16_to_cpu(dentry->name_len)); + unsigned int bit_pos; + int i; + + folio_lock(folio); + f2fs_folio_wait_writeback(folio, NODE, true, true); + + inline_dentry = inline_data_addr(dir, folio); + make_dentry_ptr_inline(dir, &d, inline_dentry); + + bit_pos = dentry - d.dentry; + for (i = 0; i < slots; i++) + __clear_bit_le(bit_pos + i, d.bitmap); + + folio_mark_dirty(folio); + f2fs_folio_put(folio, true); + + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + f2fs_mark_inode_dirty_sync(dir, false); + + if (inode) + f2fs_drop_nlink(dir, inode); +} + +bool f2fs_empty_inline_dir(struct inode *dir) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + struct folio *ifolio; + unsigned int bit_pos = 2; + void *inline_dentry; + struct f2fs_dentry_ptr d; + + ifolio = f2fs_get_inode_folio(sbi, dir->i_ino); + if (IS_ERR(ifolio)) + return false; + + inline_dentry = inline_data_addr(dir, ifolio); + make_dentry_ptr_inline(dir, &d, inline_dentry); + + bit_pos = find_next_bit_le(d.bitmap, d.max, bit_pos); + + f2fs_folio_put(ifolio, true); + + if (bit_pos < d.max) + return false; + + return true; +} + +int f2fs_read_inline_dir(struct file *file, struct dir_context *ctx, + struct fscrypt_str *fstr) +{ + struct inode *inode = file_inode(file); + struct folio *ifolio = NULL; + struct f2fs_dentry_ptr d; + void *inline_dentry = NULL; + int err; + + make_dentry_ptr_inline(inode, &d, inline_dentry); + + if (ctx->pos == d.max) + return 0; + + ifolio = f2fs_get_inode_folio(F2FS_I_SB(inode), inode->i_ino); + if (IS_ERR(ifolio)) + return PTR_ERR(ifolio); + + /* + * f2fs_readdir was protected by inode.i_rwsem, it is safe to access + * ipage without page's lock held. + */ + folio_unlock(ifolio); + + inline_dentry = inline_data_addr(inode, ifolio); + + make_dentry_ptr_inline(inode, &d, inline_dentry); + + err = f2fs_fill_dentries(ctx, &d, 0, fstr); + if (!err) + ctx->pos = d.max; + + f2fs_folio_put(ifolio, false); + return err < 0 ? err : 0; +} + +int f2fs_inline_data_fiemap(struct inode *inode, + struct fiemap_extent_info *fieinfo, __u64 start, __u64 len) +{ + __u64 byteaddr = 0, ilen; + __u32 flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED | + FIEMAP_EXTENT_LAST; + struct node_info ni; + struct folio *ifolio; + int err = 0; + + ifolio = f2fs_get_inode_folio(F2FS_I_SB(inode), inode->i_ino); + if (IS_ERR(ifolio)) + return PTR_ERR(ifolio); + + if ((S_ISREG(inode->i_mode) || S_ISLNK(inode->i_mode)) && + !f2fs_has_inline_data(inode)) { + err = -EAGAIN; + goto out; + } + + if (S_ISDIR(inode->i_mode) && !f2fs_has_inline_dentry(inode)) { + err = -EAGAIN; + goto out; + } + + if (fieinfo->fi_flags & FIEMAP_FLAG_SYNC) { + err = f2fs_write_single_node_folio(ifolio, true, false, FS_NODE_IO); + if (err) + return err; + ifolio = f2fs_get_inode_folio(F2FS_I_SB(inode), inode->i_ino); + if (IS_ERR(ifolio)) + return PTR_ERR(ifolio); + f2fs_folio_wait_writeback(ifolio, NODE, true, true); + } + ilen = min_t(size_t, MAX_INLINE_DATA(inode), i_size_read(inode)); + if (start >= ilen) + goto out; + if (start + len < ilen) + ilen = start + len; + ilen -= start; + + err = f2fs_get_node_info(F2FS_I_SB(inode), inode->i_ino, &ni, false); + if (err) + goto out; + + if (__is_valid_data_blkaddr(ni.blk_addr)) { + byteaddr = (__u64)ni.blk_addr << inode->i_sb->s_blocksize_bits; + byteaddr += (char *)inline_data_addr(inode, ifolio) - + (char *)F2FS_INODE(ifolio); + } else { + f2fs_bug_on(F2FS_I_SB(inode), ni.blk_addr != NEW_ADDR); + flags |= FIEMAP_EXTENT_DELALLOC | FIEMAP_EXTENT_UNKNOWN; + } + err = fiemap_fill_next_extent(fieinfo, start, byteaddr, ilen, flags); + trace_f2fs_fiemap(inode, start, byteaddr, ilen, flags, err); +out: + f2fs_folio_put(ifolio, true); + return err; +} diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c new file mode 100644 index 000000000..c95e0b126 --- /dev/null +++ b/fs/f2fs/inode.c @@ -0,0 +1,1081 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/f2fs/inode.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + */ +#include +#include +#include +#include +#include +#include +#include + +#include "f2fs.h" +#include "node.h" +#include "segment.h" +#include "xattr.h" + +#include + +#ifdef CONFIG_F2FS_FS_COMPRESSION +extern const struct address_space_operations f2fs_compress_aops; +#endif + +void f2fs_mark_inode_dirty_sync(struct inode *inode, bool sync) +{ + if (is_inode_flag_set(inode, FI_NEW_INODE)) + return; + + if (f2fs_readonly(F2FS_I_SB(inode)->sb)) + return; + + if (f2fs_inode_dirtied(inode, sync)) + return; + + /* only atomic file w/ FI_ATOMIC_COMMITTED can be set vfs dirty */ + if (f2fs_is_atomic_file(inode) && + !is_inode_flag_set(inode, FI_ATOMIC_COMMITTED)) + return; + + mark_inode_dirty_sync(inode); +} + +void f2fs_set_inode_flags(struct inode *inode) +{ + unsigned int flags = F2FS_I(inode)->i_flags; + unsigned int new_fl = 0; + + if (flags & F2FS_SYNC_FL) + new_fl |= S_SYNC; + if (flags & F2FS_APPEND_FL) + new_fl |= S_APPEND; + if (flags & F2FS_IMMUTABLE_FL) + new_fl |= S_IMMUTABLE; + if (flags & F2FS_NOATIME_FL) + new_fl |= S_NOATIME; + if (flags & F2FS_DIRSYNC_FL) + new_fl |= S_DIRSYNC; + if (file_is_encrypt(inode)) + new_fl |= S_ENCRYPTED; + if (file_is_verity(inode)) + new_fl |= S_VERITY; + if (flags & F2FS_CASEFOLD_FL) + new_fl |= S_CASEFOLD; + inode_set_flags(inode, new_fl, + S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC| + S_ENCRYPTED|S_VERITY|S_CASEFOLD); +} + +static void __get_inode_rdev(struct inode *inode, struct folio *node_folio) +{ + __le32 *addr = get_dnode_addr(inode, node_folio); + + if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) || + S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) { + if (addr[0]) + inode->i_rdev = old_decode_dev(le32_to_cpu(addr[0])); + else + inode->i_rdev = new_decode_dev(le32_to_cpu(addr[1])); + } +} + +static void __set_inode_rdev(struct inode *inode, struct folio *node_folio) +{ + __le32 *addr = get_dnode_addr(inode, node_folio); + + if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) { + if (old_valid_dev(inode->i_rdev)) { + addr[0] = cpu_to_le32(old_encode_dev(inode->i_rdev)); + addr[1] = 0; + } else { + addr[0] = 0; + addr[1] = cpu_to_le32(new_encode_dev(inode->i_rdev)); + addr[2] = 0; + } + } +} + +static void __recover_inline_status(struct inode *inode, struct folio *ifolio) +{ + void *inline_data = inline_data_addr(inode, ifolio); + __le32 *start = inline_data; + __le32 *end = start + MAX_INLINE_DATA(inode) / sizeof(__le32); + + while (start < end) { + if (*start++) { + f2fs_folio_wait_writeback(ifolio, NODE, true, true); + + set_inode_flag(inode, FI_DATA_EXIST); + set_raw_inline(inode, F2FS_INODE(ifolio)); + folio_mark_dirty(ifolio); + return; + } + } + return; +} + +static +bool f2fs_enable_inode_chksum(struct f2fs_sb_info *sbi, struct folio *folio) +{ + struct f2fs_inode *ri = &F2FS_NODE(folio)->i; + + if (!f2fs_sb_has_inode_chksum(sbi)) + return false; + + if (!IS_INODE(folio) || !(ri->i_inline & F2FS_EXTRA_ATTR)) + return false; + + if (!F2FS_FITS_IN_INODE(ri, le16_to_cpu(ri->i_extra_isize), + i_inode_checksum)) + return false; + + return true; +} + +static __u32 f2fs_inode_chksum(struct f2fs_sb_info *sbi, struct folio *folio) +{ + struct f2fs_node *node = F2FS_NODE(folio); + struct f2fs_inode *ri = &node->i; + __le32 ino = node->footer.ino; + __le32 gen = ri->i_generation; + __u32 chksum, chksum_seed; + __u32 dummy_cs = 0; + unsigned int offset = offsetof(struct f2fs_inode, i_inode_checksum); + unsigned int cs_size = sizeof(dummy_cs); + + chksum = f2fs_chksum(sbi->s_chksum_seed, (__u8 *)&ino, sizeof(ino)); + chksum_seed = f2fs_chksum(chksum, (__u8 *)&gen, sizeof(gen)); + + chksum = f2fs_chksum(chksum_seed, (__u8 *)ri, offset); + chksum = f2fs_chksum(chksum, (__u8 *)&dummy_cs, cs_size); + offset += cs_size; + chksum = f2fs_chksum(chksum, (__u8 *)ri + offset, + F2FS_BLKSIZE - offset); + return chksum; +} + +bool f2fs_inode_chksum_verify(struct f2fs_sb_info *sbi, struct folio *folio) +{ + struct f2fs_inode *ri; + __u32 provided, calculated; + + if (unlikely(is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN))) + return true; + +#ifdef CONFIG_F2FS_CHECK_FS + if (!f2fs_enable_inode_chksum(sbi, folio)) +#else + if (!f2fs_enable_inode_chksum(sbi, folio) || + folio_test_dirty(folio) || + folio_test_writeback(folio)) +#endif + return true; + + ri = &F2FS_NODE(folio)->i; + provided = le32_to_cpu(ri->i_inode_checksum); + calculated = f2fs_inode_chksum(sbi, folio); + + if (provided != calculated) + f2fs_warn(sbi, "checksum invalid, nid = %lu, ino_of_node = %x, %x vs. %x", + folio->index, ino_of_node(folio), + provided, calculated); + + return provided == calculated; +} + +void f2fs_inode_chksum_set(struct f2fs_sb_info *sbi, struct folio *folio) +{ + struct f2fs_inode *ri = &F2FS_NODE(folio)->i; + + if (!f2fs_enable_inode_chksum(sbi, folio)) + return; + + ri->i_inode_checksum = cpu_to_le32(f2fs_inode_chksum(sbi, folio)); +} + +static bool sanity_check_compress_inode(struct inode *inode, + struct f2fs_inode *ri) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + unsigned char clevel; + + if (ri->i_compress_algorithm >= COMPRESS_MAX) { + f2fs_warn(sbi, + "%s: inode (ino=%llx) has unsupported compress algorithm: %u, run fsck to fix", + __func__, inode->i_ino, ri->i_compress_algorithm); + return false; + } + if (le64_to_cpu(ri->i_compr_blocks) > + SECTOR_TO_BLOCK(inode->i_blocks)) { + f2fs_warn(sbi, + "%s: inode (ino=%llx) has inconsistent i_compr_blocks:%llu, i_blocks:%llu, run fsck to fix", + __func__, inode->i_ino, le64_to_cpu(ri->i_compr_blocks), + SECTOR_TO_BLOCK(inode->i_blocks)); + return false; + } + if (ri->i_log_cluster_size < MIN_COMPRESS_LOG_SIZE || + ri->i_log_cluster_size > MAX_COMPRESS_LOG_SIZE) { + f2fs_warn(sbi, + "%s: inode (ino=%llx) has unsupported log cluster size: %u, run fsck to fix", + __func__, inode->i_ino, ri->i_log_cluster_size); + return false; + } + + clevel = le16_to_cpu(ri->i_compress_flag) >> + COMPRESS_LEVEL_OFFSET; + switch (ri->i_compress_algorithm) { + case COMPRESS_LZO: +#ifdef CONFIG_F2FS_FS_LZO + if (clevel) + goto err_level; +#endif + break; + case COMPRESS_LZORLE: +#ifdef CONFIG_F2FS_FS_LZORLE + if (clevel) + goto err_level; +#endif + break; + case COMPRESS_LZ4: +#ifdef CONFIG_F2FS_FS_LZ4 +#ifdef CONFIG_F2FS_FS_LZ4HC + if (clevel && + (clevel < LZ4HC_MIN_CLEVEL || clevel > LZ4HC_MAX_CLEVEL)) + goto err_level; +#else + if (clevel) + goto err_level; +#endif +#endif + break; + case COMPRESS_ZSTD: +#ifdef CONFIG_F2FS_FS_ZSTD + if (clevel < zstd_min_clevel() || clevel > zstd_max_clevel()) + goto err_level; +#endif + break; + default: + goto err_level; + } + + return true; +err_level: + f2fs_warn(sbi, "%s: inode (ino=%llx) has unsupported compress level: %u, run fsck to fix", + __func__, inode->i_ino, clevel); + return false; +} + +static bool sanity_check_inode(struct inode *inode, struct folio *node_folio) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_inode_info *fi = F2FS_I(inode); + struct f2fs_inode *ri = F2FS_INODE(node_folio); + unsigned long long iblocks; + + iblocks = le64_to_cpu(F2FS_INODE(node_folio)->i_blocks); + if (!iblocks) { + f2fs_warn(sbi, "%s: corrupted inode i_blocks i_ino=%llx iblocks=%llu, run fsck to fix.", + __func__, inode->i_ino, iblocks); + return false; + } + + if (ino_of_node(node_folio) != nid_of_node(node_folio)) { + f2fs_warn(sbi, "%s: corrupted inode footer i_ino=%llx, ino,nid: [%u, %u] run fsck to fix.", + __func__, inode->i_ino, + ino_of_node(node_folio), nid_of_node(node_folio)); + return false; + } + + if (ino_of_node(node_folio) == fi->i_xattr_nid) { + f2fs_warn(sbi, "%s: corrupted inode i_ino=%llx, xnid=%x, run fsck to fix.", + __func__, inode->i_ino, fi->i_xattr_nid); + return false; + } + + if (S_ISDIR(inode->i_mode) && unlikely(inode->i_nlink == 1)) { + f2fs_warn(sbi, "%s: directory inode (ino=%llx) has a single i_nlink", + __func__, inode->i_ino); + return false; + } + + if (f2fs_has_extra_attr(inode)) { + if (!f2fs_sb_has_extra_attr(sbi)) { + f2fs_warn(sbi, "%s: inode (ino=%llx) is with extra_attr, but extra_attr feature is off", + __func__, inode->i_ino); + return false; + } + if (fi->i_extra_isize > F2FS_TOTAL_EXTRA_ATTR_SIZE || + fi->i_extra_isize < F2FS_MIN_EXTRA_ATTR_SIZE || + fi->i_extra_isize % sizeof(__le32)) { + f2fs_warn(sbi, "%s: inode (ino=%llx) has corrupted i_extra_isize: %d, max: %zu", + __func__, inode->i_ino, fi->i_extra_isize, + F2FS_TOTAL_EXTRA_ATTR_SIZE); + return false; + } + if (f2fs_sb_has_compression(sbi) && + fi->i_flags & F2FS_COMPR_FL && + F2FS_FITS_IN_INODE(ri, fi->i_extra_isize, + i_compress_flag)) { + if (!sanity_check_compress_inode(inode, ri)) + return false; + } + } + + if (f2fs_sb_has_flexible_inline_xattr(sbi) && + (fi->i_inline_xattr_size > MAX_INLINE_XATTR_SIZE || + (f2fs_has_inline_xattr(inode) && + fi->i_inline_xattr_size < MIN_INLINE_XATTR_SIZE))) { + f2fs_warn(sbi, "%s: inode (ino=%llx) has corrupted i_inline_xattr_size: %d, min: %zu, max: %lu", + __func__, inode->i_ino, fi->i_inline_xattr_size, + MIN_INLINE_XATTR_SIZE, MAX_INLINE_XATTR_SIZE); + return false; + } + + if (!f2fs_sb_has_extra_attr(sbi)) { + if (f2fs_sb_has_project_quota(sbi)) { + f2fs_warn(sbi, "%s: corrupted inode ino=%llx, wrong feature flag: %u, run fsck to fix.", + __func__, inode->i_ino, F2FS_FEATURE_PRJQUOTA); + return false; + } + if (f2fs_sb_has_inode_chksum(sbi)) { + f2fs_warn(sbi, "%s: corrupted inode ino=%llx, wrong feature flag: %u, run fsck to fix.", + __func__, inode->i_ino, F2FS_FEATURE_INODE_CHKSUM); + return false; + } + if (f2fs_sb_has_flexible_inline_xattr(sbi)) { + f2fs_warn(sbi, "%s: corrupted inode ino=%llx, wrong feature flag: %u, run fsck to fix.", + __func__, inode->i_ino, F2FS_FEATURE_FLEXIBLE_INLINE_XATTR); + return false; + } + if (f2fs_sb_has_inode_crtime(sbi)) { + f2fs_warn(sbi, "%s: corrupted inode ino=%llx, wrong feature flag: %u, run fsck to fix.", + __func__, inode->i_ino, F2FS_FEATURE_INODE_CRTIME); + return false; + } + if (f2fs_sb_has_compression(sbi)) { + f2fs_warn(sbi, "%s: corrupted inode ino=%llx, wrong feature flag: %u, run fsck to fix.", + __func__, inode->i_ino, F2FS_FEATURE_COMPRESSION); + return false; + } + } + + if (f2fs_sanity_check_inline_data(inode, node_folio)) { + f2fs_warn(sbi, "%s: inode (ino=%llx, mode=%u) should not have inline_data, run fsck to fix", + __func__, inode->i_ino, inode->i_mode); + return false; + } + + if (f2fs_has_inline_dentry(inode) && !S_ISDIR(inode->i_mode)) { + f2fs_warn(sbi, "%s: inode (ino=%llx, mode=%u) should not have inline_dentry, run fsck to fix", + __func__, inode->i_ino, inode->i_mode); + return false; + } + + if ((fi->i_flags & F2FS_CASEFOLD_FL) && !f2fs_sb_has_casefold(sbi)) { + f2fs_warn(sbi, "%s: inode (ino=%llx) has casefold flag, but casefold feature is off", + __func__, inode->i_ino); + return false; + } + + if (fi->i_xattr_nid && f2fs_check_nid_range(sbi, fi->i_xattr_nid)) { + f2fs_warn(sbi, "%s: inode (ino=%llx) has corrupted i_xattr_nid: %u, run fsck to fix.", + __func__, inode->i_ino, fi->i_xattr_nid); + return false; + } + + if (IS_DEVICE_ALIASING(inode)) { + if (!f2fs_sb_has_device_alias(sbi)) { + f2fs_warn(sbi, "%s: inode (ino=%llx) has device alias flag, but the feature is off", + __func__, inode->i_ino); + return false; + } + if (!f2fs_is_pinned_file(inode)) { + f2fs_warn(sbi, "%s: inode (ino=%llx) has device alias flag, but is not pinned", + __func__, inode->i_ino); + return false; + } + } + + return true; +} + +static void init_idisk_time(struct inode *inode) +{ + struct f2fs_inode_info *fi = F2FS_I(inode); + + fi->i_disk_time[0] = inode_get_atime(inode); + fi->i_disk_time[1] = inode_get_ctime(inode); + fi->i_disk_time[2] = inode_get_mtime(inode); +} + +static int do_read_inode(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_inode_info *fi = F2FS_I(inode); + struct folio *node_folio; + struct f2fs_inode *ri; + projid_t i_projid; + + /* Check if ino is within scope */ + if (f2fs_check_nid_range(sbi, inode->i_ino)) + return -EINVAL; + + node_folio = f2fs_get_inode_folio(sbi, inode->i_ino); + if (IS_ERR(node_folio)) + return PTR_ERR(node_folio); + + ri = F2FS_INODE(node_folio); + + inode->i_mode = le16_to_cpu(ri->i_mode); + i_uid_write(inode, le32_to_cpu(ri->i_uid)); + i_gid_write(inode, le32_to_cpu(ri->i_gid)); + set_nlink(inode, le32_to_cpu(ri->i_links)); + inode->i_size = le64_to_cpu(ri->i_size); + inode->i_blocks = SECTOR_FROM_BLOCK(le64_to_cpu(ri->i_blocks) - 1); + + inode_set_atime(inode, le64_to_cpu(ri->i_atime), + le32_to_cpu(ri->i_atime_nsec)); + inode_set_ctime(inode, le64_to_cpu(ri->i_ctime), + le32_to_cpu(ri->i_ctime_nsec)); + inode_set_mtime(inode, le64_to_cpu(ri->i_mtime), + le32_to_cpu(ri->i_mtime_nsec)); + inode->i_generation = le32_to_cpu(ri->i_generation); + if (S_ISDIR(inode->i_mode)) + fi->i_current_depth = le32_to_cpu(ri->i_current_depth); + else if (S_ISREG(inode->i_mode)) + fi->i_gc_failures = le16_to_cpu(ri->i_gc_failures); + fi->i_xattr_nid = le32_to_cpu(ri->i_xattr_nid); + fi->i_flags = le32_to_cpu(ri->i_flags); + if (S_ISREG(inode->i_mode)) + fi->i_flags &= ~F2FS_PROJINHERIT_FL; + bitmap_zero(fi->flags, FI_MAX); + fi->i_advise = ri->i_advise; + fi->i_pino = le32_to_cpu(ri->i_pino); + fi->i_dir_level = ri->i_dir_level; + + get_inline_info(inode, ri); + + fi->i_extra_isize = f2fs_has_extra_attr(inode) ? + le16_to_cpu(ri->i_extra_isize) : 0; + + if (f2fs_sb_has_flexible_inline_xattr(sbi)) { + fi->i_inline_xattr_size = le16_to_cpu(ri->i_inline_xattr_size); + } else if (f2fs_has_inline_xattr(inode) || + f2fs_has_inline_dentry(inode)) { + fi->i_inline_xattr_size = DEFAULT_INLINE_XATTR_ADDRS; + } else { + + /* + * Previous inline data or directory always reserved 200 bytes + * in inode layout, even if inline_xattr is disabled. In order + * to keep inline_dentry's structure for backward compatibility, + * we get the space back only from inline_data. + */ + fi->i_inline_xattr_size = 0; + } + + if (!sanity_check_inode(inode, node_folio)) { + f2fs_folio_put(node_folio, true); + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE); + fserror_report_file_metadata(inode, -EFSCORRUPTED, GFP_NOFS); + return -EFSCORRUPTED; + } + + /* check data exist */ + if (f2fs_has_inline_data(inode) && !f2fs_exist_data(inode)) + __recover_inline_status(inode, node_folio); + + /* try to recover cold bit for non-dir inode */ + if (!S_ISDIR(inode->i_mode) && !is_cold_node(node_folio)) { + f2fs_folio_wait_writeback(node_folio, NODE, true, true); + set_cold_node(node_folio, false); + folio_mark_dirty(node_folio); + } + + /* get rdev by using inline_info */ + __get_inode_rdev(inode, node_folio); + + if (!f2fs_need_inode_block_update(sbi, inode->i_ino)) + fi->last_disk_size = inode->i_size; + + if (fi->i_flags & F2FS_PROJINHERIT_FL) + set_inode_flag(inode, FI_PROJ_INHERIT); + + if (f2fs_has_extra_attr(inode) && f2fs_sb_has_project_quota(sbi) && + F2FS_FITS_IN_INODE(ri, fi->i_extra_isize, i_projid)) + i_projid = (projid_t)le32_to_cpu(ri->i_projid); + else + i_projid = F2FS_DEF_PROJID; + fi->i_projid = make_kprojid(&init_user_ns, i_projid); + + if (f2fs_has_extra_attr(inode) && f2fs_sb_has_inode_crtime(sbi) && + F2FS_FITS_IN_INODE(ri, fi->i_extra_isize, i_crtime)) { + fi->i_crtime.tv_sec = le64_to_cpu(ri->i_crtime); + fi->i_crtime.tv_nsec = le32_to_cpu(ri->i_crtime_nsec); + } + + if (f2fs_has_extra_attr(inode) && f2fs_sb_has_compression(sbi) && + (fi->i_flags & F2FS_COMPR_FL)) { + if (F2FS_FITS_IN_INODE(ri, fi->i_extra_isize, + i_compress_flag)) { + unsigned short compress_flag; + + atomic_set(&fi->i_compr_blocks, + le64_to_cpu(ri->i_compr_blocks)); + fi->i_compress_algorithm = ri->i_compress_algorithm; + fi->i_log_cluster_size = ri->i_log_cluster_size; + compress_flag = le16_to_cpu(ri->i_compress_flag); + fi->i_compress_level = compress_flag >> + COMPRESS_LEVEL_OFFSET; + fi->i_compress_flag = compress_flag & + GENMASK(COMPRESS_LEVEL_OFFSET - 1, 0); + fi->i_cluster_size = BIT(fi->i_log_cluster_size); + set_inode_flag(inode, FI_COMPRESSED_FILE); + } + } + + init_idisk_time(inode); + + if (!sanity_check_extent_cache(inode, node_folio)) { + f2fs_folio_put(node_folio, true); + f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE); + fserror_report_file_metadata(inode, -EFSCORRUPTED, GFP_NOFS); + return -EFSCORRUPTED; + } + + /* Need all the flag bits */ + f2fs_init_read_extent_tree(inode, node_folio); + f2fs_init_age_extent_tree(inode); + + f2fs_folio_put(node_folio, true); + + stat_inc_inline_xattr(inode); + stat_inc_inline_inode(inode); + stat_inc_inline_dir(inode); + stat_inc_compr_inode(inode); + stat_add_compr_blocks(inode, atomic_read(&fi->i_compr_blocks)); + + return 0; +} + +static bool is_meta_ino(struct f2fs_sb_info *sbi, unsigned int ino) +{ + if (ino == F2FS_NODE_INO(sbi) || ino == F2FS_META_INO(sbi)) + return true; +#ifdef CONFIG_F2FS_FS_COMPRESSION + if (test_opt(sbi, COMPRESS_CACHE) && ino == F2FS_COMPRESS_INO(sbi)) + return true; +#endif + return false; +} + +struct inode *f2fs_iget(struct super_block *sb, unsigned long ino) +{ + struct f2fs_sb_info *sbi = F2FS_SB(sb); + struct inode *inode; + int ret = 0; + + inode = iget_locked(sb, ino); + if (!inode) + return ERR_PTR(-ENOMEM); + + if (!(inode_state_read_once(inode) & I_NEW)) { + if (is_meta_ino(sbi, ino)) { + f2fs_err(sbi, "inaccessible inode: %lu, run fsck to repair", ino); + set_sbi_flag(sbi, SBI_NEED_FSCK); + ret = -EFSCORRUPTED; + trace_f2fs_iget_exit(inode, ret); + iput(inode); + f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE); + fserror_report_file_metadata(inode, ret, GFP_NOFS); + return ERR_PTR(ret); + } + + trace_f2fs_iget(inode); + return inode; + } + + if (is_meta_ino(sbi, ino)) + goto make_now; + + ret = do_read_inode(inode); + if (ret) + goto bad_inode; +make_now: + f2fs_set_inode_flags(inode); + + if (ino == F2FS_NODE_INO(sbi)) { + inode->i_mapping->a_ops = &f2fs_node_aops; + mapping_set_gfp_mask(inode->i_mapping, GFP_NOFS); + } else if (ino == F2FS_META_INO(sbi)) { + inode->i_mapping->a_ops = &f2fs_meta_aops; + mapping_set_gfp_mask(inode->i_mapping, GFP_NOFS); + } else if (ino == F2FS_COMPRESS_INO(sbi)) { +#ifdef CONFIG_F2FS_FS_COMPRESSION + inode->i_mapping->a_ops = &f2fs_compress_aops; + /* + * generic_error_remove_folio only truncates pages of regular + * inode + */ + inode->i_mode |= S_IFREG; +#endif + mapping_set_gfp_mask(inode->i_mapping, + GFP_NOFS | __GFP_HIGHMEM | __GFP_MOVABLE); + } else if (S_ISREG(inode->i_mode)) { + inode->i_op = &f2fs_file_inode_operations; + inode->i_fop = &f2fs_file_operations; + inode->i_mapping->a_ops = &f2fs_dblock_aops; + if (IS_IMMUTABLE(inode) && !f2fs_compressed_file(inode) && + !f2fs_quota_file(sbi, inode->i_ino)) + mapping_set_folio_min_order(inode->i_mapping, 0); + } else if (S_ISDIR(inode->i_mode)) { + inode->i_op = &f2fs_dir_inode_operations; + inode->i_fop = &f2fs_dir_operations; + inode->i_mapping->a_ops = &f2fs_dblock_aops; + mapping_set_gfp_mask(inode->i_mapping, GFP_NOFS); + } else if (S_ISLNK(inode->i_mode)) { + if (file_is_encrypt(inode)) + inode->i_op = &f2fs_encrypted_symlink_inode_operations; + else + inode->i_op = &f2fs_symlink_inode_operations; + inode_nohighmem(inode); + inode->i_mapping->a_ops = &f2fs_dblock_aops; + } else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) || + S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) { + inode->i_op = &f2fs_special_inode_operations; + init_special_inode(inode, inode->i_mode, inode->i_rdev); + } else { + ret = -EIO; + goto bad_inode; + } + + unlock_new_inode(inode); + trace_f2fs_iget(inode); + return inode; + +bad_inode: + f2fs_inode_synced(inode); + iget_failed(inode); + trace_f2fs_iget_exit(inode, ret); + return ERR_PTR(ret); +} + +struct inode *f2fs_iget_retry(struct super_block *sb, unsigned long ino) +{ + struct inode *inode; +retry: + inode = f2fs_iget(sb, ino); + if (IS_ERR(inode)) { + if (PTR_ERR(inode) == -ENOMEM) { + memalloc_retry_wait(GFP_NOFS); + goto retry; + } + } + return inode; +} + +void f2fs_update_inode(struct inode *inode, struct folio *node_folio) +{ + struct f2fs_inode_info *fi = F2FS_I(inode); + struct f2fs_inode *ri; + struct extent_tree *et = fi->extent_tree[EX_READ]; + + f2fs_folio_wait_writeback(node_folio, NODE, true, true); + folio_mark_dirty(node_folio); + + f2fs_inode_synced(inode); + + ri = F2FS_INODE(node_folio); + + ri->i_mode = cpu_to_le16(inode->i_mode); + ri->i_advise = fi->i_advise; + ri->i_uid = cpu_to_le32(i_uid_read(inode)); + ri->i_gid = cpu_to_le32(i_gid_read(inode)); + ri->i_links = cpu_to_le32(inode->i_nlink); + ri->i_blocks = cpu_to_le64(SECTOR_TO_BLOCK(READ_ONCE(inode->i_blocks)) + 1); + + if (!f2fs_is_atomic_file(inode) || + is_inode_flag_set(inode, FI_ATOMIC_COMMITTED)) + ri->i_size = cpu_to_le64(i_size_read(inode)); + + if (et) { + read_lock(&et->lock); + set_raw_read_extent(&et->largest, &ri->i_ext); + read_unlock(&et->lock); + } else { + memset(&ri->i_ext, 0, sizeof(ri->i_ext)); + } + set_raw_inline(inode, ri); + + ri->i_atime = cpu_to_le64(inode_get_atime_sec(inode)); + ri->i_ctime = cpu_to_le64(inode_get_ctime_sec(inode)); + ri->i_mtime = cpu_to_le64(inode_get_mtime_sec(inode)); + ri->i_atime_nsec = cpu_to_le32(inode_get_atime_nsec(inode)); + ri->i_ctime_nsec = cpu_to_le32(inode_get_ctime_nsec(inode)); + ri->i_mtime_nsec = cpu_to_le32(inode_get_mtime_nsec(inode)); + if (S_ISDIR(inode->i_mode)) + ri->i_current_depth = cpu_to_le32(fi->i_current_depth); + else if (S_ISREG(inode->i_mode)) + ri->i_gc_failures = cpu_to_le16(fi->i_gc_failures); + ri->i_xattr_nid = cpu_to_le32(fi->i_xattr_nid); + ri->i_flags = cpu_to_le32(fi->i_flags); + ri->i_pino = cpu_to_le32(fi->i_pino); + ri->i_generation = cpu_to_le32(inode->i_generation); + ri->i_dir_level = fi->i_dir_level; + + if (f2fs_has_extra_attr(inode)) { + ri->i_extra_isize = cpu_to_le16(fi->i_extra_isize); + + if (f2fs_sb_has_flexible_inline_xattr(F2FS_I_SB(inode))) + ri->i_inline_xattr_size = + cpu_to_le16(fi->i_inline_xattr_size); + + if (f2fs_sb_has_project_quota(F2FS_I_SB(inode)) && + F2FS_FITS_IN_INODE(ri, fi->i_extra_isize, i_projid)) { + projid_t i_projid; + + i_projid = from_kprojid(&init_user_ns, fi->i_projid); + ri->i_projid = cpu_to_le32(i_projid); + } + + if (f2fs_sb_has_inode_crtime(F2FS_I_SB(inode)) && + F2FS_FITS_IN_INODE(ri, fi->i_extra_isize, i_crtime)) { + ri->i_crtime = cpu_to_le64(fi->i_crtime.tv_sec); + ri->i_crtime_nsec = cpu_to_le32(fi->i_crtime.tv_nsec); + } + + if (f2fs_sb_has_compression(F2FS_I_SB(inode)) && + F2FS_FITS_IN_INODE(ri, fi->i_extra_isize, + i_compress_flag)) { + unsigned short compress_flag; + + ri->i_compr_blocks = cpu_to_le64( + atomic_read(&fi->i_compr_blocks)); + ri->i_compress_algorithm = fi->i_compress_algorithm; + compress_flag = fi->i_compress_flag | + fi->i_compress_level << + COMPRESS_LEVEL_OFFSET; + ri->i_compress_flag = cpu_to_le16(compress_flag); + ri->i_log_cluster_size = fi->i_log_cluster_size; + } + } + + __set_inode_rdev(inode, node_folio); + + /* deleted inode */ + if (inode->i_nlink == 0) + folio_clear_f2fs_inline(node_folio); + + init_idisk_time(inode); +#ifdef CONFIG_F2FS_CHECK_FS + f2fs_inode_chksum_set(F2FS_I_SB(inode), node_folio); +#endif +} + +void f2fs_update_inode_page(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct folio *node_folio; + int count = 0; +retry: + node_folio = f2fs_get_inode_folio(sbi, inode->i_ino); + if (IS_ERR(node_folio)) { + int err = PTR_ERR(node_folio); + + /* The node block was truncated. */ + if (err == -ENOENT) + return; + + if (err == -EFSCORRUPTED) + goto stop_checkpoint; + + if (err == -ENOMEM || ++count <= DEFAULT_RETRY_IO_COUNT) + goto retry; +stop_checkpoint: + fserror_report_file_metadata(inode, -EFSCORRUPTED, GFP_NOFS); + f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_UPDATE_INODE); + return; + } + f2fs_update_inode(inode, node_folio); + f2fs_folio_put(node_folio, true); +} + +int f2fs_write_inode(struct inode *inode, struct writeback_control *wbc) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + + if (inode->i_ino == F2FS_NODE_INO(sbi) || + inode->i_ino == F2FS_META_INO(sbi)) + return 0; + + /* + * atime could be updated without dirtying f2fs inode in lazytime mode + */ + if (f2fs_is_time_consistent(inode) && + !is_inode_flag_set(inode, FI_DIRTY_INODE)) + return 0; + + /* + * no need to update inode page, ultimately f2fs_evict_inode() will + * clear dirty status of inode. + */ + if (f2fs_cp_error(sbi)) + return -EIO; + + if (!f2fs_is_checkpoint_ready(sbi)) { + f2fs_mark_inode_dirty_sync(inode, true); + return -ENOSPC; + } + + /* + * We need to balance fs here to prevent from producing dirty node pages + * during the urgent cleaning time when running out of free sections. + */ + f2fs_update_inode_page(inode); + if (wbc && wbc->nr_to_write) + f2fs_balance_fs(sbi, true); + return 0; +} + +void f2fs_remove_donate_inode(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + + if (list_empty(&F2FS_I(inode)->gdonate_list)) + return; + + spin_lock(&sbi->inode_lock[DONATE_INODE]); + list_del_init(&F2FS_I(inode)->gdonate_list); + sbi->donate_files--; + spin_unlock(&sbi->inode_lock[DONATE_INODE]); +} + +/* + * Called at the last iput() if i_nlink is zero + */ +void f2fs_evict_inode(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_inode_info *fi = F2FS_I(inode); + nid_t xnid = fi->i_xattr_nid; + int err = 0; + bool freeze_protected = false; + + f2fs_abort_atomic_write(inode, true); + + if (fi->cow_inode && f2fs_is_cow_file(fi->cow_inode)) { + struct inode *cow_inode = fi->cow_inode; + + f2fs_down_write(&F2FS_I(cow_inode)->i_sem); + clear_inode_flag(cow_inode, FI_COW_FILE); + F2FS_I(cow_inode)->atomic_inode = NULL; + fi->cow_inode = NULL; + f2fs_up_write(&F2FS_I(cow_inode)->i_sem); + + iput(cow_inode); + } + + trace_f2fs_evict_inode(inode); + truncate_inode_pages_final(&inode->i_data); + + if ((inode->i_nlink || is_bad_inode(inode)) && + test_opt(sbi, COMPRESS_CACHE) && f2fs_compressed_file(inode)) + f2fs_invalidate_compress_pages(sbi, inode->i_ino); + + if (inode->i_ino == F2FS_NODE_INO(sbi) || + inode->i_ino == F2FS_META_INO(sbi) || + inode->i_ino == F2FS_COMPRESS_INO(sbi)) + goto out_clear; + + f2fs_bug_on(sbi, get_dirty_pages(inode)); + f2fs_remove_dirty_inode(inode); + f2fs_remove_donate_inode(inode); + + if (!IS_DEVICE_ALIASING(inode)) + f2fs_destroy_extent_tree(inode); + + if (inode->i_nlink || is_bad_inode(inode)) + goto no_delete; + + err = f2fs_dquot_initialize(inode); + if (err) { + err = 0; + set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR); + } + + f2fs_remove_ino_entry(sbi, inode->i_ino, APPEND_INO); + f2fs_remove_ino_entry(sbi, inode->i_ino, UPDATE_INO); + f2fs_remove_ino_entry(sbi, inode->i_ino, FLUSH_INO); + + if (!is_sbi_flag_set(sbi, SBI_IS_FREEZING)) { + sb_start_intwrite(inode->i_sb); + freeze_protected = true; + } + set_inode_flag(inode, FI_NO_ALLOC); + i_size_write(inode, 0); +retry: + if (F2FS_HAS_BLOCKS(inode)) + err = f2fs_truncate(inode); + + if (time_to_inject(sbi, FAULT_EVICT_INODE)) + err = -EIO; + + if (!err) { + struct f2fs_lock_context lc; + + f2fs_lock_op(sbi, &lc); + err = f2fs_remove_inode_page(inode); + f2fs_unlock_op(sbi, &lc); + if (err == -ENOENT) { + err = 0; + + /* + * in fuzzed image, another node may has the same + * block address as inode's, if it was truncated + * previously, truncation of inode node will fail. + */ + if (is_inode_flag_set(inode, FI_DIRTY_INODE)) { + f2fs_warn(F2FS_I_SB(inode), + "f2fs_evict_inode: inconsistent node id, ino:%llu", + inode->i_ino); + f2fs_inode_synced(inode); + set_sbi_flag(sbi, SBI_NEED_FSCK); + } + } + } + + /* give more chances, if ENOMEM case */ + if (err == -ENOMEM) { + err = 0; + goto retry; + } + + if (IS_DEVICE_ALIASING(inode)) + f2fs_destroy_extent_tree(inode); + + if (err) { + f2fs_update_inode_page(inode); + if (dquot_initialize_needed(inode)) + set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR); + + /* + * If both f2fs_truncate() and f2fs_update_inode_page() failed + * due to fuzzed corrupted inode, call f2fs_inode_synced() to + * avoid triggering later f2fs_bug_on(). + */ + if (is_inode_flag_set(inode, FI_DIRTY_INODE)) { + f2fs_warn(sbi, + "f2fs_evict_inode: inode is dirty, ino:%llu", + inode->i_ino); + f2fs_inode_synced(inode); + set_sbi_flag(sbi, SBI_NEED_FSCK); + } + } + if (freeze_protected) + sb_end_intwrite(inode->i_sb); +no_delete: + dquot_drop(inode); + + stat_dec_inline_xattr(inode); + stat_dec_inline_dir(inode); + stat_dec_inline_inode(inode); + stat_dec_compr_inode(inode); + stat_sub_compr_blocks(inode, + atomic_read(&fi->i_compr_blocks)); + + if (likely(!f2fs_cp_error(sbi) && + !is_sbi_flag_set(sbi, SBI_CP_DISABLED))) + f2fs_bug_on(sbi, is_inode_flag_set(inode, FI_DIRTY_INODE)); + + /* + * anyway, it needs to remove the inode from sbi->inode_list[DIRTY_META] + * list to avoid UAF in f2fs_sync_inode_meta() during checkpoint. + */ + f2fs_inode_synced(inode); + + /* for the case f2fs_new_inode() was failed, .i_ino is zero, skip it */ + if (inode->i_ino) + invalidate_mapping_pages(NODE_MAPPING(sbi), inode->i_ino, + inode->i_ino); + if (xnid) + invalidate_mapping_pages(NODE_MAPPING(sbi), xnid, xnid); + if (inode->i_nlink) { + if (is_inode_flag_set(inode, FI_APPEND_WRITE)) + f2fs_add_ino_entry(sbi, inode->i_ino, APPEND_INO); + if (is_inode_flag_set(inode, FI_UPDATE_WRITE)) + f2fs_add_ino_entry(sbi, inode->i_ino, UPDATE_INO); + } + if (is_inode_flag_set(inode, FI_FREE_NID)) { + f2fs_alloc_nid_failed(sbi, inode->i_ino); + clear_inode_flag(inode, FI_FREE_NID); + } else { + /* + * If xattr nid is corrupted, we can reach out error condition, + * err & !f2fs_exist_written_data(sbi, inode->i_ino, ORPHAN_INO)). + * In that case, f2fs_check_nid_range() is enough to give a clue. + */ + } +out_clear: + fscrypt_put_encryption_info(inode); + clear_inode(inode); +} + +/* caller should call f2fs_lock_op() */ +void f2fs_handle_failed_inode(struct inode *inode, struct f2fs_lock_context *lc) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct node_info ni; + int err; + + /* + * clear nlink of inode in order to release resource of inode + * immediately. + */ + clear_nlink(inode); + + /* + * we must call this to avoid inode being remained as dirty, resulting + * in a panic when flushing dirty inodes in gdirty_list. + */ + f2fs_update_inode_page(inode); + f2fs_inode_synced(inode); + + /* don't make bad inode, since it becomes a regular file. */ + unlock_new_inode(inode); + + /* + * Note: we should add inode to orphan list before f2fs_unlock_op() + * so we can prevent losing this orphan when encoutering checkpoint + * and following suddenly power-off. + */ + err = f2fs_get_node_info(sbi, inode->i_ino, &ni, false); + if (err) { + set_sbi_flag(sbi, SBI_NEED_FSCK); + set_inode_flag(inode, FI_FREE_NID); + f2fs_warn(sbi, "May loss orphan inode, run fsck to fix."); + goto out; + } + + if (ni.blk_addr != NULL_ADDR) { + err = f2fs_acquire_orphan_inode(sbi); + if (err) { + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_warn(sbi, "Too many orphan inodes, run fsck to fix."); + } else { + f2fs_add_orphan_inode(inode); + } + f2fs_alloc_nid_done(sbi, inode->i_ino); + } else { + set_inode_flag(inode, FI_FREE_NID); + } + +out: + f2fs_unlock_op(sbi, lc); + + /* iput will drop the inode object */ + iput(inode); +} diff --git a/fs/f2fs/iostat.c b/fs/f2fs/iostat.c new file mode 100644 index 000000000..ae265e3e9 --- /dev/null +++ b/fs/f2fs/iostat.c @@ -0,0 +1,351 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * f2fs iostat support + * + * Copyright 2021 Google LLC + * Author: Daeho Jeong + */ + +#include +#include +#include + +#include "f2fs.h" +#include "iostat.h" +#include + +static struct kmem_cache *bio_iostat_ctx_cache; +static mempool_t *bio_iostat_ctx_pool; + +static inline unsigned long long iostat_get_avg_bytes(struct f2fs_sb_info *sbi, + enum iostat_type type) +{ + return sbi->iostat_count[type] ? div64_u64(sbi->iostat_bytes[type], + sbi->iostat_count[type]) : 0; +} + +#define IOSTAT_INFO_SHOW(name, type) \ + seq_printf(seq, "%-23s %-16llu %-16llu %-16llu\n", \ + name":", sbi->iostat_bytes[type], \ + sbi->iostat_count[type], \ + iostat_get_avg_bytes(sbi, type)) + +int __maybe_unused iostat_info_seq_show(struct seq_file *seq, void *offset) +{ + struct super_block *sb = seq->private; + struct f2fs_sb_info *sbi = F2FS_SB(sb); + int i; + + if (!sbi->iostat_enable) + return 0; + + seq_printf(seq, "time: %-16llu\n", ktime_get_real_seconds()); + seq_printf(seq, "\t\t\t%-16s %-16s %-16s\n", + "io_bytes", "count", "avg_bytes"); + + /* print app write IOs */ + seq_puts(seq, "[WRITE]\n"); + IOSTAT_INFO_SHOW("app buffered data", APP_BUFFERED_IO); + IOSTAT_INFO_SHOW("app direct data", APP_DIRECT_IO); + IOSTAT_INFO_SHOW("app mapped data", APP_MAPPED_IO); + IOSTAT_INFO_SHOW("app buffered cdata", APP_BUFFERED_CDATA_IO); + IOSTAT_INFO_SHOW("app mapped cdata", APP_MAPPED_CDATA_IO); + + /* print fs write IOs */ + IOSTAT_INFO_SHOW("fs data", FS_DATA_IO); + IOSTAT_INFO_SHOW("fs cdata", FS_CDATA_IO); + IOSTAT_INFO_SHOW("fs node", FS_NODE_IO); + IOSTAT_INFO_SHOW("fs meta", FS_META_IO); + IOSTAT_INFO_SHOW("fs gc data", FS_GC_DATA_IO); + IOSTAT_INFO_SHOW("fs gc node", FS_GC_NODE_IO); + IOSTAT_INFO_SHOW("fs cp data", FS_CP_DATA_IO); + IOSTAT_INFO_SHOW("fs cp node", FS_CP_NODE_IO); + IOSTAT_INFO_SHOW("fs cp meta", FS_CP_META_IO); + + /* print app read IOs */ + seq_puts(seq, "[READ]\n"); + IOSTAT_INFO_SHOW("app buffered data", APP_BUFFERED_READ_IO); + IOSTAT_INFO_SHOW("app direct data", APP_DIRECT_READ_IO); + IOSTAT_INFO_SHOW("app mapped data", APP_MAPPED_READ_IO); + IOSTAT_INFO_SHOW("app buffered cdata", APP_BUFFERED_CDATA_READ_IO); + IOSTAT_INFO_SHOW("app mapped cdata", APP_MAPPED_CDATA_READ_IO); + + /* print fs read IOs */ + IOSTAT_INFO_SHOW("fs data", FS_DATA_READ_IO); + IOSTAT_INFO_SHOW("fs gc data", FS_GDATA_READ_IO); + IOSTAT_INFO_SHOW("fs cdata", FS_CDATA_READ_IO); + IOSTAT_INFO_SHOW("fs node", FS_NODE_READ_IO); + IOSTAT_INFO_SHOW("fs meta", FS_META_READ_IO); + + /* print read folio order stats */ + seq_printf(seq, "%-23s", "fs read folio order:"); + for (i = 0; i < NR_PAGE_ORDERS; i++) + seq_printf(seq, " %llu", sbi->iostat_read_folio_count[i]); + seq_putc(seq, '\n'); + + /* print other IOs */ + seq_puts(seq, "[OTHER]\n"); + IOSTAT_INFO_SHOW("fs discard", FS_DISCARD_IO); + IOSTAT_INFO_SHOW("fs flush", FS_FLUSH_IO); + IOSTAT_INFO_SHOW("fs zone reset", FS_ZONE_RESET_IO); + + return 0; +} + +static inline void __record_iostat_latency(struct f2fs_sb_info *sbi) +{ + int io, idx; + struct f2fs_iostat_latency iostat_lat[MAX_IO_TYPE][NR_PAGE_TYPE]; + struct iostat_lat_info *io_lat = sbi->iostat_io_lat; + unsigned long flags; + + spin_lock_irqsave(&sbi->iostat_lat_lock, flags); + for (idx = 0; idx < MAX_IO_TYPE; idx++) { + for (io = 0; io < NR_PAGE_TYPE; io++) { + iostat_lat[idx][io].peak_lat = + jiffies_to_msecs(io_lat->peak_lat[idx][io]); + iostat_lat[idx][io].cnt = io_lat->bio_cnt[idx][io]; + iostat_lat[idx][io].avg_lat = iostat_lat[idx][io].cnt ? + jiffies_to_msecs(io_lat->sum_lat[idx][io]) / iostat_lat[idx][io].cnt : 0; + io_lat->sum_lat[idx][io] = 0; + io_lat->peak_lat[idx][io] = 0; + io_lat->bio_cnt[idx][io] = 0; + } + } + spin_unlock_irqrestore(&sbi->iostat_lat_lock, flags); + + trace_f2fs_iostat_latency(sbi, iostat_lat); +} + +static inline void f2fs_record_iostat(struct f2fs_sb_info *sbi) +{ + unsigned long long iostat_diff[NR_IO_TYPE]; + unsigned long long read_folio_count_diff[NR_PAGE_ORDERS]; + int i; + unsigned long flags; + + if (time_is_after_jiffies(sbi->iostat_next_period)) + return; + + /* Need double check under the lock */ + spin_lock_irqsave(&sbi->iostat_lock, flags); + if (time_is_after_jiffies(sbi->iostat_next_period)) { + spin_unlock_irqrestore(&sbi->iostat_lock, flags); + return; + } + sbi->iostat_next_period = jiffies + + msecs_to_jiffies(sbi->iostat_period_ms); + + for (i = 0; i < NR_IO_TYPE; i++) { + iostat_diff[i] = sbi->iostat_bytes[i] - + sbi->prev_iostat_bytes[i]; + sbi->prev_iostat_bytes[i] = sbi->iostat_bytes[i]; + } + + for (i = 0; i < NR_PAGE_ORDERS; i++) { + read_folio_count_diff[i] = sbi->iostat_read_folio_count[i] - + sbi->prev_iostat_read_folio_count[i]; + sbi->prev_iostat_read_folio_count[i] = sbi->iostat_read_folio_count[i]; + } + spin_unlock_irqrestore(&sbi->iostat_lock, flags); + + trace_f2fs_iostat(sbi, iostat_diff, read_folio_count_diff); + + __record_iostat_latency(sbi); +} + +void f2fs_reset_iostat(struct f2fs_sb_info *sbi) +{ + struct iostat_lat_info *io_lat = sbi->iostat_io_lat; + int i; + + spin_lock_irq(&sbi->iostat_lock); + for (i = 0; i < NR_IO_TYPE; i++) { + sbi->iostat_count[i] = 0; + sbi->iostat_bytes[i] = 0; + sbi->prev_iostat_bytes[i] = 0; + } + for (i = 0; i < NR_PAGE_ORDERS; i++) { + sbi->iostat_read_folio_count[i] = 0; + sbi->prev_iostat_read_folio_count[i] = 0; + } + spin_unlock_irq(&sbi->iostat_lock); + + spin_lock_irq(&sbi->iostat_lat_lock); + memset(io_lat, 0, sizeof(struct iostat_lat_info)); + spin_unlock_irq(&sbi->iostat_lat_lock); +} + +static inline void __f2fs_update_iostat(struct f2fs_sb_info *sbi, + enum iostat_type type, unsigned long long io_bytes) +{ + sbi->iostat_bytes[type] += io_bytes; + sbi->iostat_count[type]++; +} + +void f2fs_update_read_folio_count(struct f2fs_sb_info *sbi, struct folio *folio) +{ + unsigned int order = folio_order(folio); + unsigned long flags; + + if (!sbi->iostat_enable) + return; + + if (order >= NR_PAGE_ORDERS) + order = NR_PAGE_ORDERS - 1; + + spin_lock_irqsave(&sbi->iostat_lock, flags); + sbi->iostat_read_folio_count[order]++; + spin_unlock_irqrestore(&sbi->iostat_lock, flags); + + f2fs_record_iostat(sbi); +} + +void f2fs_update_iostat(struct f2fs_sb_info *sbi, struct inode *inode, + enum iostat_type type, unsigned long long io_bytes) +{ + unsigned long flags; + + if (!sbi->iostat_enable) + return; + + spin_lock_irqsave(&sbi->iostat_lock, flags); + __f2fs_update_iostat(sbi, type, io_bytes); + + if (type == APP_BUFFERED_IO || type == APP_DIRECT_IO) + __f2fs_update_iostat(sbi, APP_WRITE_IO, io_bytes); + + if (type == APP_BUFFERED_READ_IO || type == APP_DIRECT_READ_IO) + __f2fs_update_iostat(sbi, APP_READ_IO, io_bytes); + +#ifdef CONFIG_F2FS_FS_COMPRESSION + if (inode && f2fs_compressed_file(inode)) { + if (type == APP_BUFFERED_IO) + __f2fs_update_iostat(sbi, APP_BUFFERED_CDATA_IO, io_bytes); + + if (type == APP_BUFFERED_READ_IO) + __f2fs_update_iostat(sbi, APP_BUFFERED_CDATA_READ_IO, io_bytes); + + if (type == APP_MAPPED_READ_IO) + __f2fs_update_iostat(sbi, APP_MAPPED_CDATA_READ_IO, io_bytes); + + if (type == APP_MAPPED_IO) + __f2fs_update_iostat(sbi, APP_MAPPED_CDATA_IO, io_bytes); + + if (type == FS_DATA_READ_IO) + __f2fs_update_iostat(sbi, FS_CDATA_READ_IO, io_bytes); + + if (type == FS_DATA_IO) + __f2fs_update_iostat(sbi, FS_CDATA_IO, io_bytes); + } +#endif + + spin_unlock_irqrestore(&sbi->iostat_lock, flags); + + f2fs_record_iostat(sbi); +} + +static inline void __update_iostat_latency(struct bio_iostat_ctx *iostat_ctx, + enum iostat_lat_type lat_type) +{ + unsigned long ts_diff; + unsigned int page_type = iostat_ctx->type; + struct f2fs_sb_info *sbi = iostat_ctx->sbi; + struct iostat_lat_info *io_lat = sbi->iostat_io_lat; + unsigned long flags; + + if (!sbi->iostat_enable) + return; + + ts_diff = jiffies - iostat_ctx->submit_ts; + if (page_type == META_FLUSH) { + page_type = META; + } else if (page_type >= NR_PAGE_TYPE) { + f2fs_warn(sbi, "%s: %d over NR_PAGE_TYPE", __func__, page_type); + return; + } + + spin_lock_irqsave(&sbi->iostat_lat_lock, flags); + io_lat->sum_lat[lat_type][page_type] += ts_diff; + io_lat->bio_cnt[lat_type][page_type]++; + if (ts_diff > io_lat->peak_lat[lat_type][page_type]) + io_lat->peak_lat[lat_type][page_type] = ts_diff; + spin_unlock_irqrestore(&sbi->iostat_lat_lock, flags); +} + +void iostat_update_and_unbind_ctx(struct bio *bio) +{ + struct bio_iostat_ctx *iostat_ctx = bio->bi_private; + enum iostat_lat_type lat_type; + + if (op_is_write(bio_op(bio))) { + lat_type = bio->bi_opf & REQ_SYNC ? + WRITE_SYNC_IO : WRITE_ASYNC_IO; + bio->bi_private = iostat_ctx->sbi; + } else { + lat_type = READ_IO; + bio->bi_private = iostat_ctx->post_read_ctx; + } + + __update_iostat_latency(iostat_ctx, lat_type); + mempool_free(iostat_ctx, bio_iostat_ctx_pool); +} + +void iostat_alloc_and_bind_ctx(struct f2fs_sb_info *sbi, + struct bio *bio, struct bio_post_read_ctx *ctx) +{ + struct bio_iostat_ctx *iostat_ctx; + /* Due to the mempool, this never fails. */ + iostat_ctx = mempool_alloc(bio_iostat_ctx_pool, GFP_NOFS); + iostat_ctx->sbi = sbi; + iostat_ctx->submit_ts = 0; + iostat_ctx->type = 0; + iostat_ctx->post_read_ctx = ctx; + bio->bi_private = iostat_ctx; +} + +int __init f2fs_init_iostat_processing(void) +{ + bio_iostat_ctx_cache = + kmem_cache_create("f2fs_bio_iostat_ctx", + sizeof(struct bio_iostat_ctx), 0, 0, NULL); + if (!bio_iostat_ctx_cache) + goto fail; + bio_iostat_ctx_pool = + mempool_create_slab_pool(NUM_PREALLOC_IOSTAT_CTXS, + bio_iostat_ctx_cache); + if (!bio_iostat_ctx_pool) + goto fail_free_cache; + return 0; + +fail_free_cache: + kmem_cache_destroy(bio_iostat_ctx_cache); +fail: + return -ENOMEM; +} + +void f2fs_destroy_iostat_processing(void) +{ + mempool_destroy(bio_iostat_ctx_pool); + kmem_cache_destroy(bio_iostat_ctx_cache); +} + +int f2fs_init_iostat(struct f2fs_sb_info *sbi) +{ + /* init iostat info */ + spin_lock_init(&sbi->iostat_lock); + spin_lock_init(&sbi->iostat_lat_lock); + sbi->iostat_enable = false; + sbi->iostat_period_ms = DEFAULT_IOSTAT_PERIOD_MS; + sbi->iostat_io_lat = f2fs_kzalloc(sbi, sizeof(struct iostat_lat_info), + GFP_KERNEL); + if (!sbi->iostat_io_lat) + return -ENOMEM; + + return 0; +} + +void f2fs_destroy_iostat(struct f2fs_sb_info *sbi) +{ + kfree(sbi->iostat_io_lat); +} diff --git a/fs/f2fs/iostat.h b/fs/f2fs/iostat.h new file mode 100644 index 000000000..2025225b5 --- /dev/null +++ b/fs/f2fs/iostat.h @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright 2021 Google LLC + * Author: Daeho Jeong + */ +#ifndef __F2FS_IOSTAT_H__ +#define __F2FS_IOSTAT_H__ + +struct bio_post_read_ctx; + +enum iostat_lat_type { + READ_IO = 0, + WRITE_SYNC_IO, + WRITE_ASYNC_IO, + MAX_IO_TYPE, +}; + +#ifdef CONFIG_F2FS_IOSTAT + +#define NUM_PREALLOC_IOSTAT_CTXS 128 +#define DEFAULT_IOSTAT_PERIOD_MS 3000 +#define MIN_IOSTAT_PERIOD_MS 100 +/* maximum period of iostat tracing is 1 day */ +#define MAX_IOSTAT_PERIOD_MS 8640000 + +struct iostat_lat_info { + unsigned long sum_lat[MAX_IO_TYPE][NR_PAGE_TYPE]; /* sum of io latencies */ + unsigned long peak_lat[MAX_IO_TYPE][NR_PAGE_TYPE]; /* peak io latency */ + unsigned int bio_cnt[MAX_IO_TYPE][NR_PAGE_TYPE]; /* bio count */ +}; + +extern int __maybe_unused iostat_info_seq_show(struct seq_file *seq, + void *offset); +extern void f2fs_reset_iostat(struct f2fs_sb_info *sbi); +extern void f2fs_update_iostat(struct f2fs_sb_info *sbi, struct inode *inode, + enum iostat_type type, unsigned long long io_bytes); +extern void f2fs_update_read_folio_count(struct f2fs_sb_info *sbi, + struct folio *folio); + +struct bio_iostat_ctx { + struct f2fs_sb_info *sbi; + unsigned long submit_ts; + enum page_type type; + struct bio_post_read_ctx *post_read_ctx; +}; + +static inline void iostat_update_submit_ctx(struct bio *bio, + enum page_type type) +{ + struct bio_iostat_ctx *iostat_ctx = bio->bi_private; + + iostat_ctx->submit_ts = jiffies; + iostat_ctx->type = type; +} + +static inline struct bio_post_read_ctx *get_post_read_ctx(struct bio *bio) +{ + struct bio_iostat_ctx *iostat_ctx = bio->bi_private; + + return iostat_ctx->post_read_ctx; +} + +extern void iostat_update_and_unbind_ctx(struct bio *bio); +extern void iostat_alloc_and_bind_ctx(struct f2fs_sb_info *sbi, + struct bio *bio, struct bio_post_read_ctx *ctx); +extern int f2fs_init_iostat_processing(void); +extern void f2fs_destroy_iostat_processing(void); +extern int f2fs_init_iostat(struct f2fs_sb_info *sbi); +extern void f2fs_destroy_iostat(struct f2fs_sb_info *sbi); +#else +static inline void f2fs_update_iostat(struct f2fs_sb_info *sbi, struct inode *inode, + enum iostat_type type, unsigned long long io_bytes) {} +static inline void f2fs_update_read_folio_count(struct f2fs_sb_info *sbi, + struct folio *folio) {} +static inline void iostat_update_and_unbind_ctx(struct bio *bio) {} +static inline void iostat_alloc_and_bind_ctx(struct f2fs_sb_info *sbi, + struct bio *bio, struct bio_post_read_ctx *ctx) {} +static inline void iostat_update_submit_ctx(struct bio *bio, + enum page_type type) {} +static inline struct bio_post_read_ctx *get_post_read_ctx(struct bio *bio) +{ + return bio->bi_private; +} +static inline int f2fs_init_iostat_processing(void) { return 0; } +static inline void f2fs_destroy_iostat_processing(void) {} +static inline int f2fs_init_iostat(struct f2fs_sb_info *sbi) { return 0; } +static inline void f2fs_destroy_iostat(struct f2fs_sb_info *sbi) {} +#endif +#endif /* __F2FS_IOSTAT_H__ */ diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c new file mode 100644 index 000000000..5ae647a35 --- /dev/null +++ b/fs/f2fs/namei.c @@ -0,0 +1,1402 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/f2fs/namei.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "f2fs.h" +#include "node.h" +#include "segment.h" +#include "xattr.h" +#include "acl.h" +#include + +static inline bool is_extension_exist(const unsigned char *s, const char *sub, + bool tmp_ext, bool tmp_dot) +{ + size_t slen = strlen(s); + size_t sublen = strlen(sub); + int i; + + if (sublen == 1 && *sub == '*') + return true; + + /* + * filename format of multimedia file should be defined as: + * "filename + '.' + extension + (optional: '.' + temp extension)". + */ + if (slen < sublen + 2) + return false; + + if (!tmp_ext) { + /* file has no temp extension */ + if (s[slen - sublen - 1] != '.') + return false; + return !strncasecmp(s + slen - sublen, sub, sublen); + } + + for (i = 1; i < slen - sublen; i++) { + if (s[i] != '.') + continue; + if (!strncasecmp(s + i + 1, sub, sublen)) { + if (!tmp_dot) + return true; + if (i == slen - sublen - 1 || s[i + 1 + sublen] == '.') + return true; + } + } + + return false; +} + +static inline bool is_temperature_extension(const unsigned char *s, const char *sub) +{ + return is_extension_exist(s, sub, true, false); +} + +static inline bool is_compress_extension(const unsigned char *s, const char *sub) +{ + return is_extension_exist(s, sub, true, true); +} + +int f2fs_update_extension_list(struct f2fs_sb_info *sbi, const char *name, + bool hot, bool set) +{ + __u8 (*extlist)[F2FS_EXTENSION_LEN] = sbi->raw_super->extension_list; + int cold_count = le32_to_cpu(sbi->raw_super->extension_count); + int hot_count = sbi->raw_super->hot_ext_count; + int total_count = cold_count + hot_count; + int start, count; + int i; + + if (set) { + if (total_count == F2FS_MAX_EXTENSION) + return -EINVAL; + + if (hot) { + start = 0; + count = cold_count; + } else { + start = cold_count; + count = total_count; + } + for (i = start; i < count; i++) { + if (!strcmp(name, extlist[i])) { + f2fs_warn(sbi, "extension '%s' already exists in %s list", + name, hot ? "cold" : "hot"); + return -EINVAL; + } + } + } else { + if (!hot && !cold_count) + return -EINVAL; + if (hot && !hot_count) + return -EINVAL; + } + + if (hot) { + start = cold_count; + count = total_count; + } else { + start = 0; + count = cold_count; + } + + for (i = start; i < count; i++) { + if (strcmp(name, extlist[i])) + continue; + + if (set) + return -EINVAL; + + memcpy(extlist[i], extlist[i + 1], + F2FS_EXTENSION_LEN * (total_count - i - 1)); + memset(extlist[total_count - 1], 0, F2FS_EXTENSION_LEN); + if (hot) + sbi->raw_super->hot_ext_count = hot_count - 1; + else + sbi->raw_super->extension_count = + cpu_to_le32(cold_count - 1); + return 0; + } + + if (!set) + return -EINVAL; + + if (hot) { + memcpy(extlist[count], name, strlen(name)); + sbi->raw_super->hot_ext_count = hot_count + 1; + } else { + char buf[F2FS_MAX_EXTENSION][F2FS_EXTENSION_LEN]; + + memcpy(buf, &extlist[cold_count], + F2FS_EXTENSION_LEN * hot_count); + memset(extlist[cold_count], 0, F2FS_EXTENSION_LEN); + memcpy(extlist[cold_count], name, strlen(name)); + memcpy(&extlist[cold_count + 1], buf, + F2FS_EXTENSION_LEN * hot_count); + sbi->raw_super->extension_count = cpu_to_le32(cold_count + 1); + } + return 0; +} + +static void set_compress_new_inode(struct f2fs_sb_info *sbi, struct inode *dir, + struct inode *inode, const unsigned char *name) +{ + __u8 (*extlist)[F2FS_EXTENSION_LEN] = sbi->raw_super->extension_list; + unsigned char (*noext)[F2FS_EXTENSION_LEN] = + F2FS_OPTION(sbi).noextensions; + unsigned char (*ext)[F2FS_EXTENSION_LEN] = F2FS_OPTION(sbi).extensions; + unsigned char ext_cnt = F2FS_OPTION(sbi).compress_ext_cnt; + unsigned char noext_cnt = F2FS_OPTION(sbi).nocompress_ext_cnt; + int i, cold_count, hot_count; + + if (!f2fs_sb_has_compression(sbi)) + return; + + if (S_ISDIR(inode->i_mode)) + goto inherit_comp; + + /* This name comes only from normal files. */ + if (!name) + return; + + /* Don't compress hot files. */ + f2fs_down_read(&sbi->sb_lock); + cold_count = le32_to_cpu(sbi->raw_super->extension_count); + hot_count = sbi->raw_super->hot_ext_count; + for (i = cold_count; i < cold_count + hot_count; i++) + if (is_temperature_extension(name, extlist[i])) + break; + f2fs_up_read(&sbi->sb_lock); + if (i < (cold_count + hot_count)) + return; + + /* Don't compress unallowed extension. */ + for (i = 0; i < noext_cnt; i++) + if (is_compress_extension(name, noext[i])) + return; + + /* Compress wanting extension. */ + for (i = 0; i < ext_cnt; i++) { + if (is_compress_extension(name, ext[i])) { + set_compress_context(inode); + return; + } + } +inherit_comp: + /* Inherit the {no-}compression flag in directory */ + if (F2FS_I(dir)->i_flags & F2FS_NOCOMP_FL) { + F2FS_I(inode)->i_flags |= F2FS_NOCOMP_FL; + f2fs_mark_inode_dirty_sync(inode, true); + } else if (F2FS_I(dir)->i_flags & F2FS_COMPR_FL) { + set_compress_context(inode); + } +} + +/* + * Set file's temperature for hot/cold data separation + */ +static void set_file_temperature(struct f2fs_sb_info *sbi, struct inode *inode, + const unsigned char *name) +{ + __u8 (*extlist)[F2FS_EXTENSION_LEN] = sbi->raw_super->extension_list; + int i, cold_count, hot_count; + + f2fs_down_read(&sbi->sb_lock); + cold_count = le32_to_cpu(sbi->raw_super->extension_count); + hot_count = sbi->raw_super->hot_ext_count; + for (i = 0; i < cold_count + hot_count; i++) + if (is_temperature_extension(name, extlist[i])) + break; + f2fs_up_read(&sbi->sb_lock); + + if (i == cold_count + hot_count) + return; + + if (i < cold_count) + file_set_cold(inode); + else + file_set_hot(inode); +} + +static struct inode *f2fs_new_inode(struct mnt_idmap *idmap, + struct inode *dir, umode_t mode, + const char *name) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + struct f2fs_inode_info *fi; + nid_t ino; + struct inode *inode; + bool nid_free = false; + bool encrypt = false; + int xattr_size = 0; + int err; + + inode = new_inode(dir->i_sb); + if (!inode) + return ERR_PTR(-ENOMEM); + + if (!f2fs_alloc_nid(sbi, &ino)) { + err = -ENOSPC; + goto fail; + } + + nid_free = true; + + inode_init_owner(idmap, inode, dir, mode); + + fi = F2FS_I(inode); + inode->i_ino = ino; + inode->i_blocks = 0; + simple_inode_init_ts(inode); + fi->i_crtime = inode_get_mtime(inode); + inode->i_generation = get_random_u32(); + + if (S_ISDIR(inode->i_mode)) + fi->i_current_depth = 1; + + err = insert_inode_locked(inode); + if (err) { + err = -EINVAL; + goto fail; + } + + if (f2fs_sb_has_project_quota(sbi) && + (F2FS_I(dir)->i_flags & F2FS_PROJINHERIT_FL)) + fi->i_projid = F2FS_I(dir)->i_projid; + else + fi->i_projid = make_kprojid(&init_user_ns, + F2FS_DEF_PROJID); + + err = fscrypt_prepare_new_inode(dir, inode, &encrypt); + if (err) + goto fail_drop; + + err = f2fs_dquot_initialize(inode); + if (err) + goto fail_drop; + + set_inode_flag(inode, FI_NEW_INODE); + + if (encrypt) + f2fs_set_encrypted_inode(inode); + + if (f2fs_sb_has_extra_attr(sbi)) { + set_inode_flag(inode, FI_EXTRA_ATTR); + fi->i_extra_isize = F2FS_TOTAL_EXTRA_ATTR_SIZE; + } + + if (test_opt(sbi, INLINE_XATTR)) + set_inode_flag(inode, FI_INLINE_XATTR); + + if (f2fs_may_inline_dentry(inode)) + set_inode_flag(inode, FI_INLINE_DENTRY); + + if (f2fs_sb_has_flexible_inline_xattr(sbi)) { + f2fs_bug_on(sbi, !f2fs_has_extra_attr(inode)); + if (f2fs_has_inline_xattr(inode)) + xattr_size = F2FS_OPTION(sbi).inline_xattr_size; + /* Otherwise, will be 0 */ + } else if (f2fs_has_inline_xattr(inode) || + f2fs_has_inline_dentry(inode)) { + xattr_size = DEFAULT_INLINE_XATTR_ADDRS; + } + fi->i_inline_xattr_size = xattr_size; + + fi->i_flags = + f2fs_mask_flags(mode, F2FS_I(dir)->i_flags & F2FS_FL_INHERITED); + + if (S_ISDIR(inode->i_mode)) + fi->i_flags |= F2FS_INDEX_FL; + + if (fi->i_flags & F2FS_PROJINHERIT_FL) + set_inode_flag(inode, FI_PROJ_INHERIT); + + /* Check compression first. */ + set_compress_new_inode(sbi, dir, inode, name); + + /* Should enable inline_data after compression set */ + if (test_opt(sbi, INLINE_DATA) && f2fs_may_inline_data(inode)) + set_inode_flag(inode, FI_INLINE_DATA); + + if (name && !test_opt(sbi, DISABLE_EXT_IDENTIFY)) + set_file_temperature(sbi, inode, name); + + stat_inc_inline_xattr(inode); + stat_inc_inline_inode(inode); + stat_inc_inline_dir(inode); + + f2fs_set_inode_flags(inode); + + f2fs_init_extent_tree(inode); + + trace_f2fs_new_inode(inode, 0); + return inode; + +fail: + trace_f2fs_new_inode(inode, err); + make_bad_inode(inode); + if (nid_free) + set_inode_flag(inode, FI_FREE_NID); + iput(inode); + return ERR_PTR(err); +fail_drop: + trace_f2fs_new_inode(inode, err); + dquot_drop(inode); + inode->i_flags |= S_NOQUOTA; + make_bad_inode(inode); + if (nid_free) + set_inode_flag(inode, FI_FREE_NID); + clear_nlink(inode); + unlock_new_inode(inode); + iput(inode); + return ERR_PTR(err); +} + +static int f2fs_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + struct f2fs_lock_context lc; + struct inode *inode; + nid_t ino = 0; + int err; + + if (unlikely(f2fs_cp_error(sbi))) + return -EIO; + if (!f2fs_is_checkpoint_ready(sbi)) + return -ENOSPC; + + err = f2fs_dquot_initialize(dir); + if (err) + return err; + + inode = f2fs_new_inode(idmap, dir, mode, dentry->d_name.name); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + inode->i_op = &f2fs_file_inode_operations; + inode->i_fop = &f2fs_file_operations; + inode->i_mapping->a_ops = &f2fs_dblock_aops; + ino = inode->i_ino; + + f2fs_lock_op(sbi, &lc); + err = f2fs_add_link(dentry, inode); + if (err) + goto out; + f2fs_unlock_op(sbi, &lc); + + f2fs_alloc_nid_done(sbi, ino); + + d_instantiate_new(dentry, inode); + + if (IS_DIRSYNC(dir)) + f2fs_sync_fs(sbi->sb, 1); + + f2fs_balance_fs(sbi, true); + return 0; +out: + f2fs_handle_failed_inode(inode, &lc); + return err; +} + +static int f2fs_link(struct dentry *old_dentry, struct inode *dir, + struct dentry *dentry) +{ + struct inode *inode = d_inode(old_dentry); + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + struct f2fs_lock_context lc; + int err; + + if (unlikely(f2fs_cp_error(sbi))) + return -EIO; + if (!f2fs_is_checkpoint_ready(sbi)) + return -ENOSPC; + + err = fscrypt_prepare_link(old_dentry, dir, dentry); + if (err) + return err; + + if (is_inode_flag_set(dir, FI_PROJ_INHERIT) && + (!projid_eq(F2FS_I(dir)->i_projid, + F2FS_I(inode)->i_projid))) + return -EXDEV; + + err = f2fs_dquot_initialize(dir); + if (err) + return err; + + f2fs_balance_fs(sbi, true); + + inode_set_ctime_current(inode); + ihold(inode); + + set_inode_flag(inode, FI_INC_LINK); + f2fs_lock_op(sbi, &lc); + err = f2fs_add_link(dentry, inode); + if (err) + goto out; + f2fs_unlock_op(sbi, &lc); + + d_instantiate(dentry, inode); + + if (IS_DIRSYNC(dir)) + f2fs_sync_fs(sbi->sb, 1); + return 0; +out: + clear_inode_flag(inode, FI_INC_LINK); + iput(inode); + f2fs_unlock_op(sbi, &lc); + return err; +} + +struct dentry *f2fs_get_parent(struct dentry *child) +{ + struct folio *folio; + unsigned long ino = f2fs_inode_by_name(d_inode(child), &dotdot_name, &folio); + + if (!ino) { + if (IS_ERR(folio)) + return ERR_CAST(folio); + return ERR_PTR(-ENOENT); + } + return d_obtain_alias(f2fs_iget(child->d_sb, ino)); +} + +static struct dentry *f2fs_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + struct inode *inode = NULL; + struct f2fs_dir_entry *de; + struct folio *folio; + struct dentry *new; + nid_t ino = -1; + int err = 0; + struct f2fs_filename fname; + + trace_f2fs_lookup_start(dir, dentry, flags); + + if (dentry->d_name.len > F2FS_NAME_LEN) { + err = -ENAMETOOLONG; + goto out; + } + + err = f2fs_prepare_lookup(dir, dentry, &fname); + if (err == -ENOENT) + goto out_splice; + if (err) + goto out; + de = __f2fs_find_entry(dir, &fname, &folio); + f2fs_free_filename(&fname); + + if (!de) { + if (IS_ERR(folio)) { + err = PTR_ERR(folio); + goto out; + } + err = -ENOENT; + goto out_splice; + } + + ino = le32_to_cpu(de->ino); + f2fs_folio_put(folio, false); + + inode = f2fs_iget(dir->i_sb, ino); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + goto out; + } + + if (inode->i_nlink == 0) { + f2fs_warn(F2FS_I_SB(inode), "%s: inode (ino=%llx) has zero i_nlink", + __func__, inode->i_ino); + err = -EFSCORRUPTED; + set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK); + goto out_iput; + } + + if (IS_ENCRYPTED(dir) && + (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode)) && + !fscrypt_has_permitted_context(dir, inode)) { + f2fs_warn(F2FS_I_SB(inode), "Inconsistent encryption contexts: %llu/%llu", + dir->i_ino, inode->i_ino); + err = -EPERM; + goto out_iput; + } +out_splice: + if (IS_ENABLED(CONFIG_UNICODE) && !inode && IS_CASEFOLDED(dir)) { + /* Eventually we want to call d_add_ci(dentry, NULL) + * for negative dentries in the encoding case as + * well. For now, prevent the negative dentry + * from being cached. + */ + trace_f2fs_lookup_end(dir, dentry, ino, err); + return NULL; + } + + new = d_splice_alias(inode, dentry); + trace_f2fs_lookup_end(dir, !IS_ERR_OR_NULL(new) ? new : dentry, + ino, IS_ERR(new) ? PTR_ERR(new) : err); + return new; +out_iput: + iput(inode); +out: + trace_f2fs_lookup_end(dir, dentry, ino, err); + return ERR_PTR(err); +} + +static int f2fs_unlink(struct inode *dir, struct dentry *dentry) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + struct inode *inode = d_inode(dentry); + struct f2fs_dir_entry *de; + struct f2fs_lock_context lc; + struct folio *folio; + int err; + + trace_f2fs_unlink_enter(dir, dentry); + + if (unlikely(f2fs_cp_error(sbi))) { + err = -EIO; + goto out; + } + + err = f2fs_dquot_initialize(dir); + if (err) + goto out; + err = f2fs_dquot_initialize(inode); + if (err) + goto out; + + de = f2fs_find_entry(dir, &dentry->d_name, &folio); + if (!de) { + if (IS_ERR(folio)) + err = PTR_ERR(folio); + goto out; + } + + if (unlikely(inode->i_nlink == 0)) { + f2fs_warn(sbi, "%s: inode (ino=%llx) has zero i_nlink", + __func__, inode->i_ino); + goto corrupted; + } else if (S_ISDIR(inode->i_mode) && unlikely(inode->i_nlink == 1)) { + f2fs_warn(sbi, "%s: directory inode (ino=%llx) has a single i_nlink", + __func__, inode->i_ino); + goto corrupted; + } + + f2fs_balance_fs(sbi, true); + + f2fs_lock_op(sbi, &lc); + err = f2fs_acquire_orphan_inode(sbi); + if (err) { + f2fs_unlock_op(sbi, &lc); + f2fs_folio_put(folio, false); + goto out; + } + f2fs_delete_entry(de, folio, dir, inode); + f2fs_unlock_op(sbi, &lc); + + /* VFS negative dentries are incompatible with Encoding and + * Case-insensitiveness. Eventually we'll want avoid + * invalidating the dentries here, alongside with returning the + * negative dentries at f2fs_lookup(), when it is better + * supported by the VFS for the CI case. + */ + if (IS_ENABLED(CONFIG_UNICODE) && IS_CASEFOLDED(dir)) + d_invalidate(dentry); + + if (IS_DIRSYNC(dir)) + f2fs_sync_fs(sbi->sb, 1); + + goto out; +corrupted: + err = -EFSCORRUPTED; + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_folio_put(folio, false); +out: + trace_f2fs_unlink_exit(inode, err); + return err; +} + +static const char *f2fs_get_link(struct dentry *dentry, + struct inode *inode, + struct delayed_call *done) +{ + const char *link = page_get_link(dentry, inode, done); + + if (!IS_ERR(link) && !*link) { + /* this is broken symlink case */ + do_delayed_call(done); + clear_delayed_call(done); + link = ERR_PTR(-ENOENT); + } + return link; +} + +static int f2fs_symlink(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const char *symname) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + struct f2fs_lock_context lc; + struct inode *inode; + size_t len = strlen(symname); + struct fscrypt_str disk_link; + int err; + + if (unlikely(f2fs_cp_error(sbi))) + return -EIO; + if (!f2fs_is_checkpoint_ready(sbi)) + return -ENOSPC; + + err = fscrypt_prepare_symlink(dir, symname, len, dir->i_sb->s_blocksize, + &disk_link); + if (err) + return err; + + err = f2fs_dquot_initialize(dir); + if (err) + return err; + + inode = f2fs_new_inode(idmap, dir, S_IFLNK | S_IRWXUGO, NULL); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + if (IS_ENCRYPTED(inode)) + inode->i_op = &f2fs_encrypted_symlink_inode_operations; + else + inode->i_op = &f2fs_symlink_inode_operations; + inode_nohighmem(inode); + inode->i_mapping->a_ops = &f2fs_dblock_aops; + + f2fs_lock_op(sbi, &lc); + err = f2fs_add_link(dentry, inode); + if (err) + goto out_f2fs_handle_failed_inode; + f2fs_unlock_op(sbi, &lc); + f2fs_alloc_nid_done(sbi, inode->i_ino); + + err = fscrypt_encrypt_symlink(inode, symname, len, &disk_link); + if (err) + goto err_out; + + err = page_symlink(inode, disk_link.name, disk_link.len); + +err_out: + d_instantiate_new(dentry, inode); + + /* + * Let's flush symlink data in order to avoid broken symlink as much as + * possible. Nevertheless, fsyncing is the best way, but there is no + * way to get a file descriptor in order to flush that. + * + * Note that, it needs to do dir->fsync to make this recoverable. + * If the symlink path is stored into inline_data, there is no + * performance regression. + */ + if (!err) { + filemap_write_and_wait_range(inode->i_mapping, 0, + disk_link.len - 1); + + if (IS_DIRSYNC(dir)) + f2fs_sync_fs(sbi->sb, 1); + } else { + f2fs_unlink(dir, dentry); + } + + f2fs_balance_fs(sbi, true); + goto out_free_encrypted_link; + +out_f2fs_handle_failed_inode: + f2fs_handle_failed_inode(inode, &lc); +out_free_encrypted_link: + if (disk_link.name != (unsigned char *)symname) + kfree(disk_link.name); + return err; +} + +static struct dentry *f2fs_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + struct f2fs_lock_context lc; + struct inode *inode; + int err; + + if (unlikely(f2fs_cp_error(sbi))) + return ERR_PTR(-EIO); + + err = f2fs_dquot_initialize(dir); + if (err) + return ERR_PTR(err); + + inode = f2fs_new_inode(idmap, dir, mode, NULL); + if (IS_ERR(inode)) + return ERR_CAST(inode); + + inode->i_op = &f2fs_dir_inode_operations; + inode->i_fop = &f2fs_dir_operations; + inode->i_mapping->a_ops = &f2fs_dblock_aops; + mapping_set_gfp_mask(inode->i_mapping, GFP_NOFS); + + set_inode_flag(inode, FI_INC_LINK); + f2fs_lock_op(sbi, &lc); + err = f2fs_add_link(dentry, inode); + if (err) + goto out_fail; + f2fs_unlock_op(sbi, &lc); + + f2fs_alloc_nid_done(sbi, inode->i_ino); + + d_instantiate_new(dentry, inode); + + if (IS_DIRSYNC(dir)) + f2fs_sync_fs(sbi->sb, 1); + + f2fs_balance_fs(sbi, true); + return NULL; + +out_fail: + clear_inode_flag(inode, FI_INC_LINK); + f2fs_handle_failed_inode(inode, &lc); + return ERR_PTR(err); +} + +static int f2fs_rmdir(struct inode *dir, struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + + if (f2fs_empty_dir(inode)) + return f2fs_unlink(dir, dentry); + return -ENOTEMPTY; +} + +static int f2fs_mknod(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + struct f2fs_lock_context lc; + struct inode *inode; + int err = 0; + + if (unlikely(f2fs_cp_error(sbi))) + return -EIO; + if (!f2fs_is_checkpoint_ready(sbi)) + return -ENOSPC; + + err = f2fs_dquot_initialize(dir); + if (err) + return err; + + inode = f2fs_new_inode(idmap, dir, mode, NULL); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + init_special_inode(inode, inode->i_mode, rdev); + inode->i_op = &f2fs_special_inode_operations; + + f2fs_lock_op(sbi, &lc); + err = f2fs_add_link(dentry, inode); + if (err) + goto out; + f2fs_unlock_op(sbi, &lc); + + f2fs_alloc_nid_done(sbi, inode->i_ino); + + d_instantiate_new(dentry, inode); + + if (IS_DIRSYNC(dir)) + f2fs_sync_fs(sbi->sb, 1); + + f2fs_balance_fs(sbi, true); + return 0; +out: + f2fs_handle_failed_inode(inode, &lc); + return err; +} + +static int __f2fs_tmpfile(struct mnt_idmap *idmap, struct inode *dir, + struct file *file, umode_t mode, bool is_whiteout, + struct inode **new_inode, struct f2fs_filename *fname) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + struct f2fs_lock_context lc; + struct inode *inode; + int err; + + err = f2fs_dquot_initialize(dir); + if (err) + return err; + + inode = f2fs_new_inode(idmap, dir, mode, NULL); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + if (is_whiteout) { + init_special_inode(inode, inode->i_mode, WHITEOUT_DEV); + inode->i_op = &f2fs_special_inode_operations; + } else { + inode->i_op = &f2fs_file_inode_operations; + inode->i_fop = &f2fs_file_operations; + inode->i_mapping->a_ops = &f2fs_dblock_aops; + } + + f2fs_lock_op(sbi, &lc); + err = f2fs_acquire_orphan_inode(sbi); + if (err) + goto out; + + err = f2fs_do_tmpfile(inode, dir, fname); + if (err) + goto release_out; + + /* + * add this non-linked tmpfile to orphan list, in this way we could + * remove all unused data of tmpfile after abnormal power-off. + */ + f2fs_add_orphan_inode(inode); + f2fs_alloc_nid_done(sbi, inode->i_ino); + + if (is_whiteout) { + f2fs_i_links_write(inode, false); + + spin_lock(&inode->i_lock); + inode_state_set(inode, I_LINKABLE); + spin_unlock(&inode->i_lock); + } else { + if (file) + d_tmpfile(file, inode); + else + f2fs_i_links_write(inode, false); + } + /* link_count was changed by d_tmpfile as well. */ + f2fs_unlock_op(sbi, &lc); + unlock_new_inode(inode); + + if (new_inode) + *new_inode = inode; + + f2fs_balance_fs(sbi, true); + return 0; + +release_out: + f2fs_release_orphan_inode(sbi); +out: + f2fs_handle_failed_inode(inode, &lc); + return err; +} + +static int f2fs_tmpfile(struct mnt_idmap *idmap, struct inode *dir, + struct file *file, umode_t mode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dir); + int err; + + if (unlikely(f2fs_cp_error(sbi))) + return -EIO; + if (!f2fs_is_checkpoint_ready(sbi)) + return -ENOSPC; + + err = __f2fs_tmpfile(idmap, dir, file, mode, false, NULL, NULL); + + return finish_open_simple(file, err); +} + +static int f2fs_create_whiteout(struct mnt_idmap *idmap, + struct inode *dir, struct inode **whiteout, + struct f2fs_filename *fname) +{ + return __f2fs_tmpfile(idmap, dir, NULL, S_IFCHR | WHITEOUT_MODE, + true, whiteout, fname); +} + +int f2fs_get_tmpfile(struct mnt_idmap *idmap, struct inode *dir, + struct inode **new_inode) +{ + return __f2fs_tmpfile(idmap, dir, NULL, S_IFREG, + false, new_inode, NULL); +} + +static int f2fs_rename(struct mnt_idmap *idmap, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(old_dir); + struct inode *old_inode = d_inode(old_dentry); + struct inode *new_inode = d_inode(new_dentry); + struct inode *whiteout = NULL; + struct folio *old_dir_folio = NULL; + struct folio *old_folio, *new_folio = NULL; + struct f2fs_dir_entry *old_dir_entry = NULL; + struct f2fs_dir_entry *old_entry; + struct f2fs_dir_entry *new_entry; + struct f2fs_lock_context lc; + bool old_is_dir = S_ISDIR(old_inode->i_mode); + int err; + + if (unlikely(f2fs_cp_error(sbi))) + return -EIO; + if (!f2fs_is_checkpoint_ready(sbi)) + return -ENOSPC; + + if (is_inode_flag_set(new_dir, FI_PROJ_INHERIT) && + (!projid_eq(F2FS_I(new_dir)->i_projid, + F2FS_I(old_inode)->i_projid))) + return -EXDEV; + + /* + * If new_inode is null, the below renaming flow will + * add a link in old_dir which can convert inline_dir. + * After then, if we failed to get the entry due to other + * reasons like ENOMEM, we had to remove the new entry. + * Instead of adding such the error handling routine, let's + * simply convert first here. + */ + if (old_dir == new_dir && !new_inode) { + err = f2fs_try_convert_inline_dir(old_dir, new_dentry); + if (err) + return err; + } + + if (flags & RENAME_WHITEOUT) { + struct f2fs_filename fname; + + err = f2fs_setup_filename(old_dir, &old_dentry->d_name, + 0, &fname); + if (err) + return err; + + err = f2fs_create_whiteout(idmap, old_dir, &whiteout, &fname); + f2fs_free_filename(&fname); + if (err) + return err; + } + + err = f2fs_dquot_initialize(old_dir); + if (err) + goto out; + + err = f2fs_dquot_initialize(new_dir); + if (err) + goto out; + + if (new_inode) { + err = f2fs_dquot_initialize(new_inode); + if (err) + goto out; + } + + err = -ENOENT; + old_entry = f2fs_find_entry(old_dir, &old_dentry->d_name, &old_folio); + if (!old_entry) { + if (IS_ERR(old_folio)) + err = PTR_ERR(old_folio); + goto out; + } + + if (old_is_dir && old_dir != new_dir) { + old_dir_entry = f2fs_parent_dir(old_inode, &old_dir_folio); + if (!old_dir_entry) { + if (IS_ERR(old_dir_folio)) + err = PTR_ERR(old_dir_folio); + goto out_old; + } + } + + if (new_inode) { + + err = -ENOTEMPTY; + if (old_is_dir && !f2fs_empty_dir(new_inode)) + goto out_dir; + + err = -ENOENT; + new_entry = f2fs_find_entry(new_dir, &new_dentry->d_name, + &new_folio); + if (!new_entry) { + if (IS_ERR(new_folio)) + err = PTR_ERR(new_folio); + goto out_dir; + } + + f2fs_balance_fs(sbi, true); + + f2fs_lock_op(sbi, &lc); + + err = f2fs_acquire_orphan_inode(sbi); + if (err) + goto put_out_dir; + + f2fs_set_link(new_dir, new_entry, new_folio, old_inode); + new_folio = NULL; + + inode_set_ctime_current(new_inode); + f2fs_down_write(&F2FS_I(new_inode)->i_sem); + if (old_is_dir) + f2fs_i_links_write(new_inode, false); + f2fs_i_links_write(new_inode, false); + f2fs_up_write(&F2FS_I(new_inode)->i_sem); + + if (!new_inode->i_nlink) + f2fs_add_orphan_inode(new_inode); + else + f2fs_release_orphan_inode(sbi); + } else { + f2fs_balance_fs(sbi, true); + + f2fs_lock_op(sbi, &lc); + + err = f2fs_add_link(new_dentry, old_inode); + if (err) { + f2fs_unlock_op(sbi, &lc); + goto out_dir; + } + + if (old_is_dir) + f2fs_i_links_write(new_dir, true); + } + + f2fs_down_write(&F2FS_I(old_inode)->i_sem); + if (!old_is_dir || whiteout) + file_lost_pino(old_inode); + else + /* adjust dir's i_pino to pass fsck check */ + f2fs_i_pino_write(old_inode, new_dir->i_ino); + f2fs_up_write(&F2FS_I(old_inode)->i_sem); + + inode_set_ctime_current(old_inode); + f2fs_mark_inode_dirty_sync(old_inode, false); + + f2fs_delete_entry(old_entry, old_folio, old_dir, NULL); + old_folio = NULL; + + if (whiteout) { + set_inode_flag(whiteout, FI_INC_LINK); + err = f2fs_add_link(old_dentry, whiteout); + if (err) { + d_invalidate(old_dentry); + d_invalidate(new_dentry); + goto put_out_dir; + } + spin_lock(&whiteout->i_lock); + inode_state_clear(whiteout, I_LINKABLE); + spin_unlock(&whiteout->i_lock); + + iput(whiteout); + } + + if (old_dir_entry) + f2fs_set_link(old_inode, old_dir_entry, old_dir_folio, new_dir); + if (old_is_dir) + f2fs_i_links_write(old_dir, false); + + if (F2FS_OPTION(sbi).fsync_mode == FSYNC_MODE_STRICT) { + f2fs_add_ino_entry(sbi, new_dir->i_ino, TRANS_DIR_INO); + if (S_ISDIR(old_inode->i_mode)) + f2fs_add_ino_entry(sbi, old_inode->i_ino, + TRANS_DIR_INO); + } + + f2fs_unlock_op(sbi, &lc); + + if (IS_DIRSYNC(old_dir) || IS_DIRSYNC(new_dir)) + f2fs_sync_fs(sbi->sb, 1); + + f2fs_update_time(sbi, REQ_TIME); + return 0; + +put_out_dir: + f2fs_unlock_op(sbi, &lc); + f2fs_folio_put(new_folio, false); +out_dir: + if (old_dir_entry) + f2fs_folio_put(old_dir_folio, false); +out_old: + f2fs_folio_put(old_folio, false); +out: + iput(whiteout); + return err; +} + +static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(old_dir); + struct inode *old_inode = d_inode(old_dentry); + struct inode *new_inode = d_inode(new_dentry); + struct folio *old_dir_folio, *new_dir_folio; + struct folio *old_folio, *new_folio; + struct f2fs_dir_entry *old_dir_entry = NULL, *new_dir_entry = NULL; + struct f2fs_dir_entry *old_entry, *new_entry; + struct f2fs_lock_context lc; + int old_nlink = 0, new_nlink = 0; + int err; + + if (unlikely(f2fs_cp_error(sbi))) + return -EIO; + if (!f2fs_is_checkpoint_ready(sbi)) + return -ENOSPC; + + if ((is_inode_flag_set(new_dir, FI_PROJ_INHERIT) && + !projid_eq(F2FS_I(new_dir)->i_projid, + F2FS_I(old_inode)->i_projid)) || + (is_inode_flag_set(old_dir, FI_PROJ_INHERIT) && + !projid_eq(F2FS_I(old_dir)->i_projid, + F2FS_I(new_inode)->i_projid))) + return -EXDEV; + + err = f2fs_dquot_initialize(old_dir); + if (err) + goto out; + + err = f2fs_dquot_initialize(new_dir); + if (err) + goto out; + + err = -ENOENT; + old_entry = f2fs_find_entry(old_dir, &old_dentry->d_name, &old_folio); + if (!old_entry) { + if (IS_ERR(old_folio)) + err = PTR_ERR(old_folio); + goto out; + } + + new_entry = f2fs_find_entry(new_dir, &new_dentry->d_name, &new_folio); + if (!new_entry) { + if (IS_ERR(new_folio)) + err = PTR_ERR(new_folio); + goto out_old; + } + + /* prepare for updating ".." directory entry info later */ + if (old_dir != new_dir) { + if (S_ISDIR(old_inode->i_mode)) { + old_dir_entry = f2fs_parent_dir(old_inode, + &old_dir_folio); + if (!old_dir_entry) { + if (IS_ERR(old_dir_folio)) + err = PTR_ERR(old_dir_folio); + goto out_new; + } + } + + if (S_ISDIR(new_inode->i_mode)) { + new_dir_entry = f2fs_parent_dir(new_inode, + &new_dir_folio); + if (!new_dir_entry) { + if (IS_ERR(new_dir_folio)) + err = PTR_ERR(new_dir_folio); + goto out_old_dir; + } + } + } + + /* + * If cross rename between file and directory those are not + * in the same directory, we will inc nlink of file's parent + * later, so we should check upper boundary of its nlink. + */ + if ((!old_dir_entry || !new_dir_entry) && + old_dir_entry != new_dir_entry) { + old_nlink = old_dir_entry ? -1 : 1; + new_nlink = -old_nlink; + err = -EMLINK; + if ((old_nlink > 0 && old_dir->i_nlink >= F2FS_LINK_MAX) || + (new_nlink > 0 && new_dir->i_nlink >= F2FS_LINK_MAX)) + goto out_new_dir; + } + + f2fs_balance_fs(sbi, true); + + f2fs_lock_op(sbi, &lc); + + /* update ".." directory entry info of old dentry */ + if (old_dir_entry) + f2fs_set_link(old_inode, old_dir_entry, old_dir_folio, new_dir); + + /* update ".." directory entry info of new dentry */ + if (new_dir_entry) + f2fs_set_link(new_inode, new_dir_entry, new_dir_folio, old_dir); + + /* update directory entry info of old dir inode */ + f2fs_set_link(old_dir, old_entry, old_folio, new_inode); + + f2fs_down_write(&F2FS_I(old_inode)->i_sem); + if (!old_dir_entry) + file_lost_pino(old_inode); + else + /* adjust dir's i_pino to pass fsck check */ + f2fs_i_pino_write(old_inode, new_dir->i_ino); + f2fs_up_write(&F2FS_I(old_inode)->i_sem); + + inode_set_ctime_current(old_dir); + if (old_nlink) { + f2fs_down_write(&F2FS_I(old_dir)->i_sem); + f2fs_i_links_write(old_dir, old_nlink > 0); + f2fs_up_write(&F2FS_I(old_dir)->i_sem); + } + f2fs_mark_inode_dirty_sync(old_dir, false); + + /* update directory entry info of new dir inode */ + f2fs_set_link(new_dir, new_entry, new_folio, old_inode); + + f2fs_down_write(&F2FS_I(new_inode)->i_sem); + if (!new_dir_entry) + file_lost_pino(new_inode); + else + /* adjust dir's i_pino to pass fsck check */ + f2fs_i_pino_write(new_inode, old_dir->i_ino); + f2fs_up_write(&F2FS_I(new_inode)->i_sem); + + inode_set_ctime_current(new_dir); + if (new_nlink) { + f2fs_down_write(&F2FS_I(new_dir)->i_sem); + f2fs_i_links_write(new_dir, new_nlink > 0); + f2fs_up_write(&F2FS_I(new_dir)->i_sem); + } + f2fs_mark_inode_dirty_sync(new_dir, false); + + if (F2FS_OPTION(sbi).fsync_mode == FSYNC_MODE_STRICT) { + f2fs_add_ino_entry(sbi, old_dir->i_ino, TRANS_DIR_INO); + f2fs_add_ino_entry(sbi, new_dir->i_ino, TRANS_DIR_INO); + } + + f2fs_unlock_op(sbi, &lc); + + if (IS_DIRSYNC(old_dir) || IS_DIRSYNC(new_dir)) + f2fs_sync_fs(sbi->sb, 1); + + f2fs_update_time(sbi, REQ_TIME); + return 0; +out_new_dir: + if (new_dir_entry) { + f2fs_folio_put(new_dir_folio, false); + } +out_old_dir: + if (old_dir_entry) { + f2fs_folio_put(old_dir_folio, false); + } +out_new: + f2fs_folio_put(new_folio, false); +out_old: + f2fs_folio_put(old_folio, false); +out: + return err; +} + +static int f2fs_rename2(struct mnt_idmap *idmap, + struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry, + unsigned int flags) +{ + int err; + + if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) + return -EINVAL; + + trace_f2fs_rename_start(old_dir, old_dentry, new_dir, new_dentry, + flags); + + err = fscrypt_prepare_rename(old_dir, old_dentry, new_dir, new_dentry, + flags); + if (err) + return err; + + if (flags & RENAME_EXCHANGE) + err = f2fs_cross_rename(old_dir, old_dentry, + new_dir, new_dentry); + else + /* + * VFS has already handled the new dentry existence case, + * here, we just deal with "RENAME_NOREPLACE" as regular rename. + */ + err = f2fs_rename(idmap, old_dir, old_dentry, + new_dir, new_dentry, flags); + + trace_f2fs_rename_end(old_dentry, new_dentry, flags, err); + return err; +} + +static const char *f2fs_encrypted_get_link(struct dentry *dentry, + struct inode *inode, + struct delayed_call *done) +{ + struct folio *folio; + const char *target; + + if (!dentry) + return ERR_PTR(-ECHILD); + + folio = read_mapping_folio(inode->i_mapping, 0, NULL); + if (IS_ERR(folio)) + return ERR_CAST(folio); + + target = fscrypt_get_symlink(inode, folio_address(folio), + inode->i_sb->s_blocksize, done); + folio_put(folio); + return target; +} + +static int f2fs_encrypted_symlink_getattr(struct mnt_idmap *idmap, + const struct path *path, + struct kstat *stat, u32 request_mask, + unsigned int query_flags) +{ + f2fs_getattr(idmap, path, stat, request_mask, query_flags); + + return fscrypt_symlink_getattr(path, stat); +} + +const struct inode_operations f2fs_encrypted_symlink_inode_operations = { + .get_link = f2fs_encrypted_get_link, + .getattr = f2fs_encrypted_symlink_getattr, + .setattr = f2fs_setattr, + .listxattr = f2fs_listxattr, +}; + +const struct inode_operations f2fs_dir_inode_operations = { + .create = f2fs_create, + .lookup = f2fs_lookup, + .link = f2fs_link, + .unlink = f2fs_unlink, + .symlink = f2fs_symlink, + .mkdir = f2fs_mkdir, + .rmdir = f2fs_rmdir, + .mknod = f2fs_mknod, + .rename = f2fs_rename2, + .tmpfile = f2fs_tmpfile, + .getattr = f2fs_getattr, + .setattr = f2fs_setattr, + .get_inode_acl = f2fs_get_acl, + .set_acl = f2fs_set_acl, + .listxattr = f2fs_listxattr, + .fiemap = f2fs_fiemap, + .fileattr_get = f2fs_fileattr_get, + .fileattr_set = f2fs_fileattr_set, +}; + +const struct inode_operations f2fs_symlink_inode_operations = { + .get_link = f2fs_get_link, + .getattr = f2fs_getattr, + .setattr = f2fs_setattr, + .listxattr = f2fs_listxattr, +}; + +const struct inode_operations f2fs_special_inode_operations = { + .getattr = f2fs_getattr, + .setattr = f2fs_setattr, + .get_inode_acl = f2fs_get_acl, + .set_acl = f2fs_set_acl, + .listxattr = f2fs_listxattr, +}; diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c new file mode 100644 index 000000000..38917e4a7 --- /dev/null +++ b/fs/f2fs/node.c @@ -0,0 +1,3583 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/f2fs/node.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "f2fs.h" +#include "node.h" +#include "segment.h" +#include "xattr.h" +#include "iostat.h" +#include + +#define on_f2fs_build_free_nids(nm_i) mutex_is_locked(&(nm_i)->build_lock) + +static struct kmem_cache *nat_entry_slab; +static struct kmem_cache *free_nid_slab; +static struct kmem_cache *nat_entry_set_slab; +static struct kmem_cache *fsync_node_entry_slab; + +static inline bool is_invalid_nid(struct f2fs_sb_info *sbi, nid_t nid) +{ + return nid < F2FS_ROOT_INO(sbi) || nid >= NM_I(sbi)->max_nid; +} + +/* + * Check whether the given nid is within node id range. + */ +int f2fs_check_nid_range(struct f2fs_sb_info *sbi, nid_t nid) +{ + if (unlikely(is_invalid_nid(sbi, nid))) { + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_warn(sbi, "%s: out-of-range nid=%x, run fsck to fix.", + __func__, nid); + f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE); + return -EFSCORRUPTED; + } + return 0; +} + +bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct sysinfo val; + unsigned long avail_ram; + unsigned long mem_size = 0; + bool res = false; + + if (!nm_i) + return true; + + si_meminfo(&val); + + /* only uses low memory */ + avail_ram = val.totalram - val.totalhigh; + + /* + * give 25%, 25%, 50%, 50%, 25%, 25% memory for each components respectively + */ + if (type == FREE_NIDS) { + mem_size = (nm_i->nid_cnt[FREE_NID] * + sizeof(struct free_nid)) >> PAGE_SHIFT; + res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2); + } else if (type == NAT_ENTRIES) { + /* + * nat_cnt[] is heuristic accounting. Sample it locklessly here + * to avoid taking nat_tree_lock in the balance path. + */ + mem_size = (data_race(READ_ONCE(nm_i->nat_cnt[TOTAL_NAT])) * + sizeof(struct nat_entry)) >> PAGE_SHIFT; + res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2); + if (excess_cached_nats(sbi)) + res = false; + } else if (type == DIRTY_DENTS) { + if (bdi_wb_dirty_exceeded(sbi->sb->s_bdi)) + return false; + mem_size = get_pages(sbi, F2FS_DIRTY_DENTS); + res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1); + } else if (type == INO_ENTRIES) { + int i; + + for (i = 0; i < MAX_INO_ENTRY; i++) + mem_size += sbi->im[i].ino_num * + sizeof(struct ino_entry); + mem_size >>= PAGE_SHIFT; + res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1); + } else if (type == READ_EXTENT_CACHE || type == AGE_EXTENT_CACHE) { + enum extent_type etype = type == READ_EXTENT_CACHE ? + EX_READ : EX_BLOCK_AGE; + struct extent_tree_info *eti = &sbi->extent_tree[etype]; + + mem_size = (atomic_read(&eti->total_ext_tree) * + sizeof(struct extent_tree) + + atomic_read(&eti->total_ext_node) * + sizeof(struct extent_node)) >> PAGE_SHIFT; + res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2); + } else if (type == DISCARD_CACHE) { + mem_size = (atomic_read(&dcc->discard_cmd_cnt) * + sizeof(struct discard_cmd)) >> PAGE_SHIFT; + res = mem_size < (avail_ram * nm_i->ram_thresh / 100); + } else if (type == COMPRESS_PAGE) { +#ifdef CONFIG_F2FS_FS_COMPRESSION + unsigned long free_ram = val.freeram; + + /* + * free memory is lower than watermark or cached page count + * exceed threshold, deny caching compress page. + */ + res = (free_ram > avail_ram * sbi->compress_watermark / 100) && + (COMPRESS_MAPPING(sbi)->nrpages < + free_ram * sbi->compress_percent / 100); +#else + res = false; +#endif + } else { + if (!bdi_wb_dirty_exceeded(sbi->sb->s_bdi)) + return true; + } + return res; +} + +static void clear_node_folio_dirty(struct folio *folio) +{ + if (folio_test_dirty(folio)) { + f2fs_clear_page_cache_dirty_tag(folio); + folio_clear_dirty_for_io(folio); + dec_page_count(F2FS_F_SB(folio), F2FS_DIRTY_NODES); + } + folio_clear_uptodate(folio); +} + +static struct folio *get_current_nat_folio(struct f2fs_sb_info *sbi, nid_t nid) +{ + return f2fs_get_meta_folio_retry(sbi, current_nat_addr(sbi, nid)); +} + +static struct folio *get_next_nat_folio(struct f2fs_sb_info *sbi, nid_t nid) +{ + struct folio *src_folio; + struct folio *dst_folio; + pgoff_t dst_off; + void *src_addr; + void *dst_addr; + struct f2fs_nm_info *nm_i = NM_I(sbi); + + dst_off = next_nat_addr(sbi, current_nat_addr(sbi, nid)); + + /* get current nat block page with lock */ + src_folio = get_current_nat_folio(sbi, nid); + if (IS_ERR(src_folio)) + return src_folio; + dst_folio = f2fs_grab_meta_folio(sbi, dst_off); + f2fs_bug_on(sbi, folio_test_dirty(src_folio)); + + src_addr = folio_address(src_folio); + dst_addr = folio_address(dst_folio); + memcpy(dst_addr, src_addr, PAGE_SIZE); + folio_mark_dirty(dst_folio); + f2fs_folio_put(src_folio, true); + + set_to_next_nat(nm_i, nid); + + return dst_folio; +} + +static struct nat_entry *__alloc_nat_entry(struct f2fs_sb_info *sbi, + nid_t nid, bool no_fail) +{ + struct nat_entry *new; + + new = f2fs_kmem_cache_alloc(nat_entry_slab, + GFP_F2FS_ZERO, no_fail, sbi); + if (new) { + nat_set_nid(new, nid); + nat_reset_flag(new); + } + return new; +} + +static void __free_nat_entry(struct nat_entry *e) +{ + kmem_cache_free(nat_entry_slab, e); +} + +/* must be locked by nat_tree_lock */ +static struct nat_entry *__init_nat_entry(struct f2fs_nm_info *nm_i, + struct nat_entry *ne, struct f2fs_nat_entry *raw_ne, bool no_fail, bool init_dirty) +{ + if (no_fail) + f2fs_radix_tree_insert(&nm_i->nat_root, nat_get_nid(ne), ne); + else if (radix_tree_insert(&nm_i->nat_root, nat_get_nid(ne), ne)) + return NULL; + + if (raw_ne) + node_info_from_raw_nat(&ne->ni, raw_ne); + + if (init_dirty) { + INIT_LIST_HEAD(&ne->list); + nm_i->nat_cnt[TOTAL_NAT]++; + return ne; + } + + spin_lock(&nm_i->nat_list_lock); + list_add_tail(&ne->list, &nm_i->nat_entries); + spin_unlock(&nm_i->nat_list_lock); + + nm_i->nat_cnt[TOTAL_NAT]++; + nm_i->nat_cnt[RECLAIMABLE_NAT]++; + return ne; +} + +static struct nat_entry *__lookup_nat_cache(struct f2fs_nm_info *nm_i, nid_t n, bool for_dirty) +{ + struct nat_entry *ne; + + ne = radix_tree_lookup(&nm_i->nat_root, n); + + /* + * for recent accessed nat entry which will not be dirtied soon + * later, move it to tail of lru list. + */ + if (ne && !get_nat_flag(ne, IS_DIRTY) && !for_dirty) { + spin_lock(&nm_i->nat_list_lock); + if (!list_empty(&ne->list)) + list_move_tail(&ne->list, &nm_i->nat_entries); + spin_unlock(&nm_i->nat_list_lock); + } + + return ne; +} + +static unsigned int __gang_lookup_nat_cache(struct f2fs_nm_info *nm_i, + nid_t start, unsigned int nr, struct nat_entry **ep) +{ + return radix_tree_gang_lookup(&nm_i->nat_root, (void **)ep, start, nr); +} + +static void __del_from_nat_cache(struct f2fs_nm_info *nm_i, struct nat_entry *e) +{ + radix_tree_delete(&nm_i->nat_root, nat_get_nid(e)); + nm_i->nat_cnt[TOTAL_NAT]--; + nm_i->nat_cnt[RECLAIMABLE_NAT]--; + __free_nat_entry(e); +} + +static struct nat_entry_set *__grab_nat_entry_set(struct f2fs_nm_info *nm_i, + struct nat_entry *ne) +{ + nid_t set = NAT_BLOCK_OFFSET(ne->ni.nid); + struct nat_entry_set *head; + + head = radix_tree_lookup(&nm_i->nat_set_root, set); + if (!head) { + head = f2fs_kmem_cache_alloc(nat_entry_set_slab, + GFP_NOFS, true, NULL); + + INIT_LIST_HEAD(&head->entry_list); + INIT_LIST_HEAD(&head->set_list); + head->set = set; + head->entry_cnt = 0; + f2fs_radix_tree_insert(&nm_i->nat_set_root, set, head); + } + return head; +} + +static void __set_nat_cache_dirty(struct f2fs_nm_info *nm_i, + struct nat_entry *ne, bool init_dirty) +{ + struct nat_entry_set *head; + bool new_ne = nat_get_blkaddr(ne) == NEW_ADDR; + + if (!new_ne) + head = __grab_nat_entry_set(nm_i, ne); + + /* + * update entry_cnt in below condition: + * 1. update NEW_ADDR to valid block address; + * 2. update old block address to new one; + */ + if (!new_ne && (get_nat_flag(ne, IS_PREALLOC) || + !get_nat_flag(ne, IS_DIRTY))) + head->entry_cnt++; + + set_nat_flag(ne, IS_PREALLOC, new_ne); + + if (get_nat_flag(ne, IS_DIRTY)) + goto refresh_list; + + nm_i->nat_cnt[DIRTY_NAT]++; + if (!init_dirty) + nm_i->nat_cnt[RECLAIMABLE_NAT]--; + set_nat_flag(ne, IS_DIRTY, true); +refresh_list: + spin_lock(&nm_i->nat_list_lock); + if (new_ne) + list_del_init(&ne->list); + else + list_move_tail(&ne->list, &head->entry_list); + spin_unlock(&nm_i->nat_list_lock); +} + +static void __clear_nat_cache_dirty(struct f2fs_nm_info *nm_i, + struct nat_entry_set *set, struct nat_entry *ne) +{ + spin_lock(&nm_i->nat_list_lock); + list_move_tail(&ne->list, &nm_i->nat_entries); + spin_unlock(&nm_i->nat_list_lock); + + set_nat_flag(ne, IS_DIRTY, false); + set->entry_cnt--; + nm_i->nat_cnt[DIRTY_NAT]--; + nm_i->nat_cnt[RECLAIMABLE_NAT]++; +} + +static unsigned int __gang_lookup_nat_set(struct f2fs_nm_info *nm_i, + nid_t start, unsigned int nr, struct nat_entry_set **ep) +{ + return radix_tree_gang_lookup(&nm_i->nat_set_root, (void **)ep, + start, nr); +} + +bool f2fs_in_warm_node_list(struct folio *folio) +{ + return is_node_folio(folio) && IS_DNODE(folio) && is_cold_node(folio); +} + +void f2fs_init_fsync_node_info(struct f2fs_sb_info *sbi) +{ + spin_lock_init(&sbi->fsync_node_lock); + INIT_LIST_HEAD(&sbi->fsync_node_list); + sbi->fsync_seg_id = 0; + sbi->fsync_node_num = 0; +} + +static unsigned int f2fs_add_fsync_node_entry(struct f2fs_sb_info *sbi, + struct folio *folio) +{ + struct fsync_node_entry *fn; + unsigned long flags; + unsigned int seq_id; + + fn = f2fs_kmem_cache_alloc(fsync_node_entry_slab, + GFP_NOFS, true, NULL); + + folio_get(folio); + fn->folio = folio; + INIT_LIST_HEAD(&fn->list); + + spin_lock_irqsave(&sbi->fsync_node_lock, flags); + list_add_tail(&fn->list, &sbi->fsync_node_list); + fn->seq_id = sbi->fsync_seg_id++; + seq_id = fn->seq_id; + sbi->fsync_node_num++; + spin_unlock_irqrestore(&sbi->fsync_node_lock, flags); + + return seq_id; +} + +void f2fs_del_fsync_node_entry(struct f2fs_sb_info *sbi, struct folio *folio) +{ + struct fsync_node_entry *fn; + unsigned long flags; + + spin_lock_irqsave(&sbi->fsync_node_lock, flags); + list_for_each_entry(fn, &sbi->fsync_node_list, list) { + if (fn->folio == folio) { + list_del(&fn->list); + sbi->fsync_node_num--; + spin_unlock_irqrestore(&sbi->fsync_node_lock, flags); + kmem_cache_free(fsync_node_entry_slab, fn); + folio_put(folio); + return; + } + } + spin_unlock_irqrestore(&sbi->fsync_node_lock, flags); + f2fs_bug_on(sbi, 1); +} + +void f2fs_reset_fsync_node_info(struct f2fs_sb_info *sbi) +{ + unsigned long flags; + + spin_lock_irqsave(&sbi->fsync_node_lock, flags); + sbi->fsync_seg_id = 0; + spin_unlock_irqrestore(&sbi->fsync_node_lock, flags); +} + +bool f2fs_need_dentry_mark(struct f2fs_sb_info *sbi, nid_t nid) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct nat_entry *e; + bool need = false; + + f2fs_down_read(&nm_i->nat_tree_lock); + e = __lookup_nat_cache(nm_i, nid, false); + if (e) { + if (!get_nat_flag(e, IS_CHECKPOINTED) && + !get_nat_flag(e, HAS_FSYNCED_INODE)) + need = true; + } + f2fs_up_read(&nm_i->nat_tree_lock); + return need; +} + +bool f2fs_is_checkpointed_node(struct f2fs_sb_info *sbi, nid_t nid) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct nat_entry *e; + bool is_cp = true; + + f2fs_down_read(&nm_i->nat_tree_lock); + e = __lookup_nat_cache(nm_i, nid, false); + if (e && !get_nat_flag(e, IS_CHECKPOINTED)) + is_cp = false; + f2fs_up_read(&nm_i->nat_tree_lock); + return is_cp; +} + +bool f2fs_need_inode_block_update(struct f2fs_sb_info *sbi, nid_t ino) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct nat_entry *e; + bool need_update = true; + struct f2fs_lock_context lc; + + f2fs_down_read_trace(&sbi->node_write, &lc); + f2fs_down_read(&nm_i->nat_tree_lock); + e = __lookup_nat_cache(nm_i, ino, false); + if (e && get_nat_flag(e, HAS_LAST_FSYNC) && + (get_nat_flag(e, IS_CHECKPOINTED) || + get_nat_flag(e, HAS_FSYNCED_INODE))) + need_update = false; + f2fs_up_read(&nm_i->nat_tree_lock); + f2fs_up_read_trace(&sbi->node_write, &lc); + return need_update; +} + +/* must be locked by nat_tree_lock */ +static void cache_nat_entry(struct f2fs_sb_info *sbi, nid_t nid, + struct f2fs_nat_entry *ne) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct nat_entry *new, *e; + + /* Let's mitigate lock contention of nat_tree_lock during checkpoint */ + if (f2fs_rwsem_is_locked(&sbi->cp_global_sem)) + return; + + new = __alloc_nat_entry(sbi, nid, false); + if (!new) + return; + + f2fs_down_write(&nm_i->nat_tree_lock); + e = __lookup_nat_cache(nm_i, nid, false); + if (!e) + e = __init_nat_entry(nm_i, new, ne, false, false); + else + f2fs_bug_on(sbi, nat_get_ino(e) != le32_to_cpu(ne->ino) || + nat_get_blkaddr(e) != + le32_to_cpu(ne->block_addr) || + nat_get_version(e) != ne->version); + f2fs_up_write(&nm_i->nat_tree_lock); + if (e != new) + __free_nat_entry(new); +} + +static void set_node_addr(struct f2fs_sb_info *sbi, struct node_info *ni, + block_t new_blkaddr, bool fsync_done) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct nat_entry *e; + struct nat_entry *new = __alloc_nat_entry(sbi, ni->nid, true); + bool init_dirty = false; + + f2fs_down_write(&nm_i->nat_tree_lock); + e = __lookup_nat_cache(nm_i, ni->nid, true); + if (!e) { + init_dirty = true; + e = __init_nat_entry(nm_i, new, NULL, true, true); + copy_node_info(&e->ni, ni); + f2fs_bug_on(sbi, ni->blk_addr == NEW_ADDR); + } else if (new_blkaddr == NEW_ADDR) { + /* + * when nid is reallocated, + * previous nat entry can be remained in nat cache. + * So, reinitialize it with new information. + */ + copy_node_info(&e->ni, ni); + f2fs_bug_on(sbi, ni->blk_addr != NULL_ADDR); + } + /* let's free early to reduce memory consumption */ + if (e != new) + __free_nat_entry(new); + + /* sanity check */ + f2fs_bug_on(sbi, nat_get_blkaddr(e) != ni->blk_addr); + f2fs_bug_on(sbi, nat_get_blkaddr(e) == NULL_ADDR && + new_blkaddr == NULL_ADDR); + f2fs_bug_on(sbi, nat_get_blkaddr(e) == NEW_ADDR && + new_blkaddr == NEW_ADDR); + f2fs_bug_on(sbi, __is_valid_data_blkaddr(nat_get_blkaddr(e)) && + new_blkaddr == NEW_ADDR); + + /* increment version no as node is removed */ + if (nat_get_blkaddr(e) != NEW_ADDR && new_blkaddr == NULL_ADDR) { + unsigned char version = nat_get_version(e); + + nat_set_version(e, inc_node_version(version)); + } + + /* change address */ + nat_set_blkaddr(e, new_blkaddr); + if (!__is_valid_data_blkaddr(new_blkaddr)) + set_nat_flag(e, IS_CHECKPOINTED, false); + __set_nat_cache_dirty(nm_i, e, init_dirty); + + /* update fsync_mark if its inode nat entry is still alive */ + if (ni->nid != ni->ino) + e = __lookup_nat_cache(nm_i, ni->ino, false); + if (e) { + if (fsync_done && ni->nid == ni->ino) + set_nat_flag(e, HAS_FSYNCED_INODE, true); + set_nat_flag(e, HAS_LAST_FSYNC, fsync_done); + } + f2fs_up_write(&nm_i->nat_tree_lock); +} + +int f2fs_try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + int nr = nr_shrink; + + if (!f2fs_down_write_trylock(&nm_i->nat_tree_lock)) + return 0; + + spin_lock(&nm_i->nat_list_lock); + while (nr_shrink) { + struct nat_entry *ne; + + if (list_empty(&nm_i->nat_entries)) + break; + + ne = list_first_entry(&nm_i->nat_entries, + struct nat_entry, list); + list_del(&ne->list); + spin_unlock(&nm_i->nat_list_lock); + + __del_from_nat_cache(nm_i, ne); + nr_shrink--; + + spin_lock(&nm_i->nat_list_lock); + } + spin_unlock(&nm_i->nat_list_lock); + + f2fs_up_write(&nm_i->nat_tree_lock); + return nr - nr_shrink; +} + +int f2fs_get_node_info(struct f2fs_sb_info *sbi, nid_t nid, + struct node_info *ni, bool checkpoint_context) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA); + struct f2fs_journal *journal = curseg->journal; + nid_t start_nid = START_NID(nid); + struct f2fs_nat_block *nat_blk; + struct folio *folio = NULL; + struct f2fs_nat_entry ne; + struct nat_entry *e; + pgoff_t index; + int i; + bool need_cache = true; + + ni->flag = 0; + ni->nid = nid; +retry: + /* Check nat cache */ + f2fs_down_read(&nm_i->nat_tree_lock); + e = __lookup_nat_cache(nm_i, nid, false); + if (e) { + ni->ino = nat_get_ino(e); + ni->blk_addr = nat_get_blkaddr(e); + ni->version = nat_get_version(e); + f2fs_up_read(&nm_i->nat_tree_lock); + if (IS_ENABLED(CONFIG_F2FS_CHECK_FS)) { + need_cache = false; + goto sanity_check; + } + return 0; + } + + /* + * Check current segment summary by trying to grab journal_rwsem first. + * This sem is on the critical path on the checkpoint requiring the above + * nat_tree_lock. Therefore, we should retry, if we failed to grab here + * while not bothering checkpoint. + */ + if (!f2fs_rwsem_is_locked(&sbi->cp_global_sem) || checkpoint_context) { + down_read(&curseg->journal_rwsem); + } else if (f2fs_rwsem_is_contended(&nm_i->nat_tree_lock) || + !down_read_trylock(&curseg->journal_rwsem)) { + f2fs_up_read(&nm_i->nat_tree_lock); + goto retry; + } + + i = f2fs_lookup_journal_in_cursum(sbi, journal, NAT_JOURNAL, nid, 0); + if (i >= 0) { + ne = nat_in_journal(journal, i); + node_info_from_raw_nat(ni, &ne); + } + up_read(&curseg->journal_rwsem); + if (i >= 0) { + f2fs_up_read(&nm_i->nat_tree_lock); + goto sanity_check; + } + + /* Fill node_info from nat page */ + index = current_nat_addr(sbi, nid); + f2fs_up_read(&nm_i->nat_tree_lock); + + folio = f2fs_get_meta_folio(sbi, index); + if (IS_ERR(folio)) + return PTR_ERR(folio); + + nat_blk = folio_address(folio); + ne = nat_blk->entries[nid - start_nid]; + node_info_from_raw_nat(ni, &ne); + f2fs_folio_put(folio, true); +sanity_check: + if (__is_valid_data_blkaddr(ni->blk_addr) && + !f2fs_is_valid_blkaddr(sbi, ni->blk_addr, + DATA_GENERIC_ENHANCE)) { + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_err_ratelimited(sbi, + "f2fs_get_node_info of %pS: inconsistent nat entry, " + "ino:%u, nid:%u, blkaddr:%u, ver:%u, flag:%u", + __builtin_return_address(0), + ni->ino, ni->nid, ni->blk_addr, ni->version, ni->flag); + f2fs_handle_error(sbi, ERROR_INCONSISTENT_NAT); + return -EFSCORRUPTED; + } + + if (unlikely(f2fs_quota_file(sbi, ni->nid) && + !__is_valid_data_blkaddr(ni->blk_addr))) { + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_err_ratelimited(sbi, + "f2fs_get_node_info of %pS: inconsistent nat entry from qf_ino, " + "ino:%u, nid:%u, blkaddr:%u, ver:%u, flag:%u", + __builtin_return_address(0), + ni->ino, ni->nid, ni->blk_addr, ni->version, ni->flag); + f2fs_handle_error(sbi, ERROR_INCONSISTENT_NAT); + } + + /* cache nat entry */ + if (need_cache) + cache_nat_entry(sbi, nid, &ne); + return 0; +} + +/* + * readahead MAX_RA_NODE number of node pages. + */ +static void f2fs_ra_node_pages(struct folio *parent, int start, int n) +{ + struct f2fs_sb_info *sbi = F2FS_F_SB(parent); + struct blk_plug plug; + int i, end; + nid_t nid; + + blk_start_plug(&plug); + + /* Then, try readahead for siblings of the desired node */ + end = start + n; + end = min(end, (int)NIDS_PER_BLOCK); + for (i = start; i < end; i++) { + nid = get_nid(parent, i, false); + f2fs_ra_node_page(sbi, nid); + } + + blk_finish_plug(&plug); +} + +pgoff_t f2fs_get_next_page_offset(struct dnode_of_data *dn, pgoff_t pgofs) +{ + const long direct_index = ADDRS_PER_INODE(dn->inode); + const long direct_blks = ADDRS_PER_BLOCK(dn->inode); + const long indirect_blks = ADDRS_PER_BLOCK(dn->inode) * NIDS_PER_BLOCK; + unsigned int skipped_unit = ADDRS_PER_BLOCK(dn->inode); + int cur_level = dn->cur_level; + int max_level = dn->max_level; + pgoff_t base = 0; + + if (!dn->max_level) + return pgofs + 1; + + while (max_level-- > cur_level) + skipped_unit *= NIDS_PER_BLOCK; + + switch (dn->max_level) { + case 3: + base += 2 * indirect_blks; + fallthrough; + case 2: + base += 2 * direct_blks; + fallthrough; + case 1: + base += direct_index; + break; + default: + f2fs_bug_on(F2FS_I_SB(dn->inode), 1); + } + + return ((pgofs - base) / skipped_unit + 1) * skipped_unit + base; +} + +/* + * The maximum depth is four. + * Offset[0] will have raw inode offset. + */ +static int get_node_path(struct inode *inode, long block, + int offset[4], unsigned int noffset[4]) +{ + const long direct_index = ADDRS_PER_INODE(inode); + const long direct_blks = ADDRS_PER_BLOCK(inode); + const long dptrs_per_blk = NIDS_PER_BLOCK; + const long indirect_blks = ADDRS_PER_BLOCK(inode) * NIDS_PER_BLOCK; + const long dindirect_blks = indirect_blks * NIDS_PER_BLOCK; + int n = 0; + int level = 0; + + noffset[0] = 0; + + if (block < direct_index) { + offset[n] = block; + goto got; + } + block -= direct_index; + if (block < direct_blks) { + offset[n++] = NODE_DIR1_BLOCK; + noffset[n] = 1; + offset[n] = block; + level = 1; + goto got; + } + block -= direct_blks; + if (block < direct_blks) { + offset[n++] = NODE_DIR2_BLOCK; + noffset[n] = 2; + offset[n] = block; + level = 1; + goto got; + } + block -= direct_blks; + if (block < indirect_blks) { + offset[n++] = NODE_IND1_BLOCK; + noffset[n] = 3; + offset[n++] = block / direct_blks; + noffset[n] = 4 + offset[n - 1]; + offset[n] = block % direct_blks; + level = 2; + goto got; + } + block -= indirect_blks; + if (block < indirect_blks) { + offset[n++] = NODE_IND2_BLOCK; + noffset[n] = 4 + dptrs_per_blk; + offset[n++] = block / direct_blks; + noffset[n] = 5 + dptrs_per_blk + offset[n - 1]; + offset[n] = block % direct_blks; + level = 2; + goto got; + } + block -= indirect_blks; + if (block < dindirect_blks) { + offset[n++] = NODE_DIND_BLOCK; + noffset[n] = 5 + (dptrs_per_blk * 2); + offset[n++] = block / indirect_blks; + noffset[n] = 6 + (dptrs_per_blk * 2) + + offset[n - 1] * (dptrs_per_blk + 1); + offset[n++] = (block / direct_blks) % dptrs_per_blk; + noffset[n] = 7 + (dptrs_per_blk * 2) + + offset[n - 2] * (dptrs_per_blk + 1) + + offset[n - 1]; + offset[n] = block % direct_blks; + level = 3; + goto got; + } else { + return -E2BIG; + } +got: + return level; +} + +static struct folio *f2fs_get_node_folio_ra(struct folio *parent, int start); + +/* + * Caller should call f2fs_put_dnode(dn). + * Also, it should grab and release a rwsem by calling f2fs_lock_op() and + * f2fs_unlock_op() only if mode is set with ALLOC_NODE. + */ +int f2fs_get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); + struct folio *nfolio[4]; + struct folio *parent = NULL; + int offset[4]; + unsigned int noffset[4]; + nid_t nids[4]; + int level, i = 0; + int err = 0; + + level = get_node_path(dn->inode, index, offset, noffset); + if (level < 0) + return level; + + nids[0] = dn->inode->i_ino; + + if (!dn->inode_folio) { + nfolio[0] = f2fs_get_inode_folio(sbi, nids[0]); + if (IS_ERR(nfolio[0])) + return PTR_ERR(nfolio[0]); + } else { + nfolio[0] = dn->inode_folio; + } + + /* if inline_data is set, should not report any block indices */ + if (f2fs_has_inline_data(dn->inode) && index) { + err = -ENOENT; + f2fs_folio_put(nfolio[0], true); + goto release_out; + } + + parent = nfolio[0]; + if (level != 0) + nids[1] = get_nid(parent, offset[0], true); + dn->inode_folio = nfolio[0]; + dn->inode_folio_locked = true; + + /* get indirect or direct nodes */ + for (i = 1; i <= level; i++) { + bool done = false; + + if (nids[i] && nids[i] == dn->inode->i_ino) { + err = -EFSCORRUPTED; + f2fs_err_ratelimited(sbi, + "inode mapping table is corrupted, run fsck to fix it, " + "ino:%llu, nid:%u, level:%d, offset:%d", + dn->inode->i_ino, nids[i], level, offset[level]); + set_sbi_flag(sbi, SBI_NEED_FSCK); + goto release_pages; + } + + if (!nids[i] && mode == ALLOC_NODE) { + /* alloc new node */ + if (!f2fs_alloc_nid(sbi, &(nids[i]))) { + err = -ENOSPC; + goto release_pages; + } + + dn->nid = nids[i]; + nfolio[i] = f2fs_new_node_folio(dn, noffset[i]); + if (IS_ERR(nfolio[i])) { + f2fs_alloc_nid_failed(sbi, nids[i]); + err = PTR_ERR(nfolio[i]); + goto release_pages; + } + + set_nid(parent, offset[i - 1], nids[i], i == 1); + f2fs_alloc_nid_done(sbi, nids[i]); + done = true; + } else if (mode == LOOKUP_NODE_RA && i == level && level > 1) { + nfolio[i] = f2fs_get_node_folio_ra(parent, offset[i - 1]); + if (IS_ERR(nfolio[i])) { + err = PTR_ERR(nfolio[i]); + goto release_pages; + } + done = true; + } + if (i == 1) { + dn->inode_folio_locked = false; + folio_unlock(parent); + } else { + f2fs_folio_put(parent, true); + } + + if (!done) { + nfolio[i] = f2fs_get_node_folio(sbi, nids[i], + NODE_TYPE_NON_INODE); + if (IS_ERR(nfolio[i])) { + err = PTR_ERR(nfolio[i]); + f2fs_folio_put(nfolio[0], false); + goto release_out; + } + } + if (i < level) { + parent = nfolio[i]; + nids[i + 1] = get_nid(parent, offset[i], false); + } + } + dn->nid = nids[level]; + dn->ofs_in_node = offset[level]; + dn->node_folio = nfolio[level]; + dn->data_blkaddr = f2fs_data_blkaddr(dn); + + if (is_inode_flag_set(dn->inode, FI_COMPRESSED_FILE) && + f2fs_sb_has_readonly(sbi)) { + unsigned int cluster_size = F2FS_I(dn->inode)->i_cluster_size; + unsigned int ofs_in_node = dn->ofs_in_node; + pgoff_t fofs = index; + unsigned int c_len; + block_t blkaddr; + + /* should align fofs and ofs_in_node to cluster_size */ + if (fofs % cluster_size) { + fofs = round_down(fofs, cluster_size); + ofs_in_node = round_down(ofs_in_node, cluster_size); + } + + c_len = f2fs_cluster_blocks_are_contiguous(dn, ofs_in_node); + if (!c_len) + goto out; + + blkaddr = data_blkaddr(dn->inode, dn->node_folio, ofs_in_node); + if (blkaddr == COMPRESS_ADDR) + blkaddr = data_blkaddr(dn->inode, dn->node_folio, + ofs_in_node + 1); + + f2fs_update_read_extent_tree_range_compressed(dn->inode, + fofs, blkaddr, cluster_size, c_len); + } +out: + return 0; + +release_pages: + f2fs_folio_put(parent, true); + if (i > 1) + f2fs_folio_put(nfolio[0], false); +release_out: + dn->inode_folio = NULL; + dn->node_folio = NULL; + if (err == -ENOENT) { + dn->cur_level = i; + dn->max_level = level; + dn->ofs_in_node = offset[level]; + } + return err; +} + +static int truncate_node(struct dnode_of_data *dn) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); + struct node_info ni; + int err; + pgoff_t index; + + err = f2fs_get_node_info(sbi, dn->nid, &ni, false); + if (err) + return err; + + if (ni.blk_addr != NEW_ADDR && + !f2fs_is_valid_blkaddr(sbi, ni.blk_addr, DATA_GENERIC_ENHANCE)) { + f2fs_err_ratelimited(sbi, + "nat entry is corrupted, run fsck to fix it, ino:%u, " + "nid:%u, blkaddr:%u", ni.ino, ni.nid, ni.blk_addr); + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_handle_error(sbi, ERROR_INCONSISTENT_NAT); + return -EFSCORRUPTED; + } + + /* Deallocate node address */ + f2fs_invalidate_blocks(sbi, ni.blk_addr, 1); + dec_valid_node_count(sbi, dn->inode, dn->nid == dn->inode->i_ino); + set_node_addr(sbi, &ni, NULL_ADDR, false); + + if (dn->nid == dn->inode->i_ino) { + f2fs_remove_orphan_inode(sbi, dn->nid); + dec_valid_inode_count(sbi); + f2fs_inode_synced(dn->inode); + } + + clear_node_folio_dirty(dn->node_folio); + set_sbi_flag(sbi, SBI_IS_DIRTY); + + index = dn->node_folio->index; + f2fs_folio_put(dn->node_folio, true); + + invalidate_mapping_pages(NODE_MAPPING(sbi), + index, index); + + dn->node_folio = NULL; + trace_f2fs_truncate_node(dn->inode, dn->nid, ni.blk_addr); + + return 0; +} + +static int truncate_dnode(struct dnode_of_data *dn) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); + struct folio *folio; + int err; + + if (dn->nid == 0) + return 1; + + /* get direct node */ + folio = f2fs_get_node_folio(sbi, dn->nid, NODE_TYPE_NON_INODE); + if (PTR_ERR(folio) == -ENOENT) + return 1; + else if (IS_ERR(folio)) + return PTR_ERR(folio); + + if (IS_INODE(folio) || ino_of_node(folio) != dn->inode->i_ino) { + f2fs_err(sbi, "incorrect node reference, ino: %llu, nid: %u, ino_of_node: %u", + dn->inode->i_ino, dn->nid, ino_of_node(folio)); + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_handle_error(sbi, ERROR_INVALID_NODE_REFERENCE); + f2fs_folio_put(folio, true); + return -EFSCORRUPTED; + } + + /* Make dnode_of_data for parameter */ + dn->node_folio = folio; + dn->ofs_in_node = 0; + f2fs_truncate_data_blocks_range(dn, ADDRS_PER_BLOCK(dn->inode)); + err = truncate_node(dn); + if (err) { + f2fs_folio_put(folio, true); + return err; + } + + return 1; +} + +static int truncate_nodes(struct dnode_of_data *dn, unsigned int nofs, + int ofs, int depth) +{ + struct dnode_of_data rdn = *dn; + struct folio *folio; + struct f2fs_node *rn; + nid_t child_nid; + unsigned int child_nofs; + int freed = 0; + int i, ret; + + if (dn->nid == 0) + return NIDS_PER_BLOCK + 1; + + trace_f2fs_truncate_nodes_enter(dn->inode, dn->nid, dn->data_blkaddr); + + folio = f2fs_get_node_folio(F2FS_I_SB(dn->inode), dn->nid, + NODE_TYPE_NON_INODE); + if (IS_ERR(folio)) { + trace_f2fs_truncate_nodes_exit(dn->inode, PTR_ERR(folio)); + return PTR_ERR(folio); + } + + f2fs_ra_node_pages(folio, ofs, NIDS_PER_BLOCK); + + rn = F2FS_NODE(folio); + if (depth < 3) { + for (i = ofs; i < NIDS_PER_BLOCK; i++, freed++) { + child_nid = le32_to_cpu(rn->in.nid[i]); + if (child_nid == 0) + continue; + rdn.nid = child_nid; + ret = truncate_dnode(&rdn); + if (ret < 0) + goto out_err; + if (set_nid(folio, i, 0, false)) + dn->node_changed = true; + } + } else { + child_nofs = nofs + ofs * (NIDS_PER_BLOCK + 1) + 1; + for (i = ofs; i < NIDS_PER_BLOCK; i++) { + child_nid = le32_to_cpu(rn->in.nid[i]); + if (child_nid == 0) { + child_nofs += NIDS_PER_BLOCK + 1; + continue; + } + rdn.nid = child_nid; + ret = truncate_nodes(&rdn, child_nofs, 0, depth - 1); + if (ret == (NIDS_PER_BLOCK + 1)) { + if (set_nid(folio, i, 0, false)) + dn->node_changed = true; + child_nofs += ret; + } else if (ret < 0 && ret != -ENOENT) { + goto out_err; + } + } + freed = child_nofs; + } + + if (!ofs) { + /* remove current indirect node */ + dn->node_folio = folio; + ret = truncate_node(dn); + if (ret) + goto out_err; + freed++; + } else { + f2fs_folio_put(folio, true); + } + trace_f2fs_truncate_nodes_exit(dn->inode, freed); + return freed; + +out_err: + f2fs_folio_put(folio, true); + trace_f2fs_truncate_nodes_exit(dn->inode, ret); + return ret; +} + +static int truncate_partial_nodes(struct dnode_of_data *dn, + int *offset, int depth) +{ + struct folio *folios[2]; + nid_t nid[3]; + nid_t child_nid; + int err = 0; + int i; + int idx = depth - 2; + + nid[0] = get_nid(dn->inode_folio, offset[0], true); + if (!nid[0]) + return 0; + + /* get indirect nodes in the path */ + for (i = 0; i < idx + 1; i++) { + /* reference count'll be increased */ + folios[i] = f2fs_get_node_folio(F2FS_I_SB(dn->inode), nid[i], + NODE_TYPE_NON_INODE); + if (IS_ERR(folios[i])) { + err = PTR_ERR(folios[i]); + idx = i - 1; + goto fail; + } + nid[i + 1] = get_nid(folios[i], offset[i + 1], false); + } + + f2fs_ra_node_pages(folios[idx], offset[idx + 1], NIDS_PER_BLOCK); + + /* free direct nodes linked to a partial indirect node */ + for (i = offset[idx + 1]; i < NIDS_PER_BLOCK; i++) { + child_nid = get_nid(folios[idx], i, false); + if (!child_nid) + continue; + dn->nid = child_nid; + err = truncate_dnode(dn); + if (err < 0) + goto fail; + if (set_nid(folios[idx], i, 0, false)) + dn->node_changed = true; + } + + if (offset[idx + 1] == 0) { + dn->node_folio = folios[idx]; + dn->nid = nid[idx]; + err = truncate_node(dn); + if (err) + goto fail; + } else { + f2fs_folio_put(folios[idx], true); + } + offset[idx]++; + offset[idx + 1] = 0; + idx--; +fail: + for (i = idx; i >= 0; i--) + f2fs_folio_put(folios[i], true); + + trace_f2fs_truncate_partial_nodes(dn->inode, nid, depth, err); + + return err; +} + +/* + * All the block addresses of data and nodes should be nullified. + */ +int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + int err = 0, cont = 1; + int level, offset[4], noffset[4]; + unsigned int nofs = 0; + struct dnode_of_data dn; + struct folio *folio; + + trace_f2fs_truncate_inode_blocks_enter(inode, from); + + level = get_node_path(inode, from, offset, noffset); + if (level <= 0) { + if (!level) { + level = -EFSCORRUPTED; + f2fs_err(sbi, "%s: inode ino=%llx has corrupted node block, from:%lu addrs:%u", + __func__, inode->i_ino, + from, ADDRS_PER_INODE(inode)); + set_sbi_flag(sbi, SBI_NEED_FSCK); + } + trace_f2fs_truncate_inode_blocks_exit(inode, level); + return level; + } + + folio = f2fs_get_inode_folio(sbi, inode->i_ino); + if (IS_ERR(folio)) { + trace_f2fs_truncate_inode_blocks_exit(inode, PTR_ERR(folio)); + return PTR_ERR(folio); + } + + set_new_dnode(&dn, inode, folio, NULL, 0); + folio_unlock(folio); + + switch (level) { + case 0: + case 1: + nofs = noffset[1]; + break; + case 2: + nofs = noffset[1]; + if (!offset[level - 1]) + goto skip_partial; + err = truncate_partial_nodes(&dn, offset, level); + if (err < 0 && err != -ENOENT) + goto fail; + nofs += 1 + NIDS_PER_BLOCK; + break; + case 3: + nofs = 5 + 2 * NIDS_PER_BLOCK; + if (!offset[level - 1]) + goto skip_partial; + err = truncate_partial_nodes(&dn, offset, level); + if (err < 0 && err != -ENOENT) + goto fail; + break; + default: + BUG(); + } + +skip_partial: + while (cont) { + dn.nid = get_nid(folio, offset[0], true); + switch (offset[0]) { + case NODE_DIR1_BLOCK: + case NODE_DIR2_BLOCK: + err = truncate_dnode(&dn); + break; + + case NODE_IND1_BLOCK: + case NODE_IND2_BLOCK: + err = truncate_nodes(&dn, nofs, offset[1], 2); + break; + + case NODE_DIND_BLOCK: + err = truncate_nodes(&dn, nofs, offset[1], 3); + cont = 0; + break; + + default: + BUG(); + } + if (err == -ENOENT) { + set_sbi_flag(F2FS_F_SB(folio), SBI_NEED_FSCK); + f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR); + fserror_report_file_metadata(dn.inode, -EFSCORRUPTED, + GFP_NOFS); + f2fs_err_ratelimited(sbi, + "truncate node fail, ino:%llu, nid:%u, " + "offset[0]:%d, offset[1]:%d, nofs:%d", + inode->i_ino, dn.nid, offset[0], + offset[1], nofs); + err = 0; + } + if (err < 0) + goto fail; + if (offset[1] == 0 && get_nid(folio, offset[0], true)) { + folio_lock(folio); + BUG_ON(!is_node_folio(folio)); + set_nid(folio, offset[0], 0, true); + folio_unlock(folio); + } + offset[1] = 0; + offset[0]++; + nofs += err; + } +fail: + f2fs_folio_put(folio, false); + trace_f2fs_truncate_inode_blocks_exit(inode, err); + return err > 0 ? 0 : err; +} + +/* caller must lock inode page */ +int f2fs_truncate_xattr_node(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + nid_t nid = F2FS_I(inode)->i_xattr_nid; + struct dnode_of_data dn; + struct folio *nfolio; + int err; + + if (!nid) + return 0; + + nfolio = f2fs_get_xnode_folio(sbi, nid); + if (IS_ERR(nfolio)) + return PTR_ERR(nfolio); + + set_new_dnode(&dn, inode, NULL, nfolio, nid); + err = truncate_node(&dn); + if (err) { + f2fs_folio_put(nfolio, true); + return err; + } + + f2fs_i_xnid_write(inode, 0); + + return 0; +} + +/* + * Caller should grab and release a rwsem by calling f2fs_lock_op() and + * f2fs_unlock_op(). + */ +int f2fs_remove_inode_page(struct inode *inode) +{ + struct dnode_of_data dn; + int err; + + set_new_dnode(&dn, inode, NULL, NULL, inode->i_ino); + err = f2fs_get_dnode_of_data(&dn, 0, LOOKUP_NODE); + if (err) + return err; + + err = f2fs_truncate_xattr_node(inode); + if (err) { + f2fs_put_dnode(&dn); + return err; + } + + /* remove potential inline_data blocks */ + if (!IS_DEVICE_ALIASING(inode) && + (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) || + S_ISLNK(inode->i_mode))) + f2fs_truncate_data_blocks_range(&dn, 1); + + /* 0 is possible, after f2fs_new_inode() has failed */ + if (unlikely(f2fs_cp_error(F2FS_I_SB(inode)))) { + f2fs_put_dnode(&dn); + return -EIO; + } + + if (unlikely(inode->i_blocks != 0 && inode->i_blocks != 8)) { + f2fs_warn(F2FS_I_SB(inode), + "f2fs_remove_inode_page: inconsistent i_blocks, ino:%llu, iblocks:%llu", + inode->i_ino, (unsigned long long)inode->i_blocks); + set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK); + } + + /* will put inode & node pages */ + err = truncate_node(&dn); + if (err) { + f2fs_put_dnode(&dn); + return err; + } + return 0; +} + +struct folio *f2fs_new_inode_folio(struct inode *inode) +{ + struct dnode_of_data dn; + + /* allocate inode page for new inode */ + set_new_dnode(&dn, inode, NULL, NULL, inode->i_ino); + + /* caller should f2fs_folio_put(folio, true); */ + return f2fs_new_node_folio(&dn, 0); +} + +struct folio *f2fs_new_node_folio(struct dnode_of_data *dn, unsigned int ofs) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); + struct node_info new_ni; + struct folio *folio; + int err; + + if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC))) + return ERR_PTR(-EPERM); + + folio = f2fs_grab_cache_folio(NODE_MAPPING(sbi), dn->nid, false); + if (IS_ERR(folio)) + return folio; + + if (unlikely((err = inc_valid_node_count(sbi, dn->inode, !ofs)))) + goto fail; + +#ifdef CONFIG_F2FS_CHECK_FS + err = f2fs_get_node_info(sbi, dn->nid, &new_ni, false); + if (err) { + dec_valid_node_count(sbi, dn->inode, !ofs); + goto fail; + } + if (unlikely(new_ni.blk_addr != NULL_ADDR)) { + err = -EFSCORRUPTED; + dec_valid_node_count(sbi, dn->inode, !ofs); + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_warn_ratelimited(sbi, + "f2fs_new_node_folio: inconsistent nat entry, " + "ino:%u, nid:%u, blkaddr:%u, ver:%u, flag:%u", + new_ni.ino, new_ni.nid, new_ni.blk_addr, + new_ni.version, new_ni.flag); + f2fs_handle_error(sbi, ERROR_INCONSISTENT_NAT); + goto fail; + } +#endif + new_ni.nid = dn->nid; + new_ni.ino = dn->inode->i_ino; + new_ni.blk_addr = NULL_ADDR; + new_ni.flag = 0; + new_ni.version = 0; + set_node_addr(sbi, &new_ni, NEW_ADDR, false); + + f2fs_folio_wait_writeback(folio, NODE, true, true); + fill_node_footer(folio, dn->nid, dn->inode->i_ino, ofs, true); + set_cold_node(folio, S_ISDIR(dn->inode->i_mode)); + if (!folio_test_uptodate(folio)) + folio_mark_uptodate(folio); + if (folio_mark_dirty(folio)) + dn->node_changed = true; + + if (f2fs_has_xattr_block(ofs)) + f2fs_i_xnid_write(dn->inode, dn->nid); + + if (ofs == 0) + inc_valid_inode_count(sbi); + return folio; +fail: + clear_node_folio_dirty(folio); + f2fs_folio_put(folio, true); + return ERR_PTR(err); +} + +/* + * Caller should do after getting the following values. + * 0: f2fs_folio_put(folio, false) + * LOCKED_PAGE or error: f2fs_folio_put(folio, true) + */ +static int read_node_folio(struct folio *folio, blk_opf_t op_flags) +{ + struct f2fs_sb_info *sbi = F2FS_F_SB(folio); + struct node_info ni; + struct f2fs_io_info fio = { + .sbi = sbi, + .type = NODE, + .op = REQ_OP_READ, + .op_flags = op_flags, + .folio = folio, + .encrypted_page = NULL, + }; + int err; + + if (folio_test_uptodate(folio)) { + if (!f2fs_inode_chksum_verify(sbi, folio)) { + folio_clear_uptodate(folio); + return -EFSBADCRC; + } + return LOCKED_PAGE; + } + + err = f2fs_get_node_info(sbi, folio->index, &ni, false); + if (err) + return err; + + /* NEW_ADDR can be seen, after cp_error drops some dirty node pages */ + if (unlikely(ni.blk_addr == NULL_ADDR || ni.blk_addr == NEW_ADDR)) { + folio_clear_uptodate(folio); + return -ENOENT; + } + + fio.new_blkaddr = fio.old_blkaddr = ni.blk_addr; + + err = f2fs_submit_page_bio(&fio); + + if (!err) + f2fs_update_iostat(sbi, NULL, FS_NODE_READ_IO, F2FS_BLKSIZE); + + return err; +} + +/* + * Readahead a node page + */ +void f2fs_ra_node_page(struct f2fs_sb_info *sbi, nid_t nid) +{ + struct folio *afolio; + int err; + + if (!nid) + return; + if (f2fs_check_nid_range(sbi, nid)) + return; + + afolio = xa_load(&NODE_MAPPING(sbi)->i_pages, nid); + if (afolio) + return; + + afolio = f2fs_grab_cache_folio(NODE_MAPPING(sbi), nid, false); + if (IS_ERR(afolio)) + return; + + err = read_node_folio(afolio, REQ_RAHEAD); + f2fs_folio_put(afolio, err ? true : false); +} + +int f2fs_sanity_check_node_footer(struct f2fs_sb_info *sbi, + struct folio *folio, pgoff_t nid, + enum node_type ntype, bool in_irq) +{ + bool is_inode, is_xnode; + + if (unlikely(nid != nid_of_node(folio))) + goto out_err; + + is_inode = IS_INODE(folio); + is_xnode = f2fs_has_xattr_block(ofs_of_node(folio)); + + switch (ntype) { + case NODE_TYPE_REGULAR: + if (is_inode && is_xnode) + goto out_err; + break; + case NODE_TYPE_INODE: + if (!is_inode || is_xnode) + goto out_err; + break; + case NODE_TYPE_XATTR: + if (is_inode || !is_xnode) + goto out_err; + break; + case NODE_TYPE_NON_INODE: + if (is_inode) + goto out_err; + break; + case NODE_TYPE_NON_IXNODE: + if (is_inode || is_xnode) + goto out_err; + break; + default: + break; + } + if (time_to_inject(sbi, FAULT_INCONSISTENT_FOOTER)) + goto out_err; + return 0; +out_err: + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_warn_ratelimited(sbi, "inconsistent node block, node_type:%d, nid:%lu, " + "node_footer[nid:%u,ino:%u,ofs:%u,cpver:%llu,blkaddr:%u]", + ntype, nid, nid_of_node(folio), ino_of_node(folio), + ofs_of_node(folio), cpver_of_node(folio), + next_blkaddr_of_node(folio)); + + f2fs_handle_error(sbi, ERROR_INCONSISTENT_FOOTER); + fserror_report_file_metadata(folio->mapping->host, + -EFSCORRUPTED, in_irq ? GFP_NOWAIT : GFP_NOFS); + return -EFSCORRUPTED; +} + +static struct folio *__get_node_folio(struct f2fs_sb_info *sbi, pgoff_t nid, + struct folio *parent, int start, enum node_type ntype) +{ + struct folio *folio; + int err; + + if (!nid) + return ERR_PTR(-ENOENT); + if (f2fs_check_nid_range(sbi, nid)) + return ERR_PTR(-EINVAL); +repeat: + folio = f2fs_grab_cache_folio(NODE_MAPPING(sbi), nid, false); + if (IS_ERR(folio)) + return folio; + + err = read_node_folio(folio, 0); + if (err < 0) + goto out_put_err; + if (err == LOCKED_PAGE) + goto page_hit; + + if (parent) + f2fs_ra_node_pages(parent, start + 1, MAX_RA_NODE); + + folio_lock(folio); + + if (unlikely(!is_node_folio(folio))) { + f2fs_folio_put(folio, true); + goto repeat; + } + + if (unlikely(!folio_test_uptodate(folio))) { + err = -EIO; + goto out_put_err; + } + + if (!f2fs_inode_chksum_verify(sbi, folio)) { + err = -EFSBADCRC; + goto out_err; + } +page_hit: + err = f2fs_sanity_check_node_footer(sbi, folio, nid, ntype, false); + if (!err) + return folio; +out_err: + folio_clear_uptodate(folio); +out_put_err: + /* ENOENT comes from read_node_folio which is not an error. */ + if (err != -ENOENT) + f2fs_handle_page_eio(sbi, folio, NODE); + f2fs_folio_put(folio, true); + return ERR_PTR(err); +} + +struct folio *f2fs_get_node_folio(struct f2fs_sb_info *sbi, pgoff_t nid, + enum node_type node_type) +{ + return __get_node_folio(sbi, nid, NULL, 0, node_type); +} + +struct folio *f2fs_get_inode_folio(struct f2fs_sb_info *sbi, pgoff_t ino) +{ + return __get_node_folio(sbi, ino, NULL, 0, NODE_TYPE_INODE); +} + +struct folio *f2fs_get_xnode_folio(struct f2fs_sb_info *sbi, pgoff_t xnid) +{ + return __get_node_folio(sbi, xnid, NULL, 0, NODE_TYPE_XATTR); +} + +static struct folio *f2fs_get_node_folio_ra(struct folio *parent, int start) +{ + struct f2fs_sb_info *sbi = F2FS_F_SB(parent); + nid_t nid = get_nid(parent, start, false); + + return __get_node_folio(sbi, nid, parent, start, NODE_TYPE_NON_IXNODE); +} + +static void flush_inline_data(struct f2fs_sb_info *sbi, nid_t ino) +{ + struct inode *inode; + struct folio *folio; + int ret; + + /* should flush inline_data before evict_inode */ + inode = ilookup(sbi->sb, ino); + if (!inode) + return; + + folio = f2fs_filemap_get_folio(inode->i_mapping, 0, + FGP_LOCK|FGP_NOWAIT, 0); + if (IS_ERR(folio)) + goto iput_out; + + if (!folio_test_uptodate(folio)) + goto folio_out; + + if (!folio_test_dirty(folio)) + goto folio_out; + + if (!folio_clear_dirty_for_io(folio)) + goto folio_out; + + ret = f2fs_write_inline_data(inode, folio); + inode_dec_dirty_pages(inode); + f2fs_remove_dirty_inode(inode); + if (ret) + folio_mark_dirty(folio); +folio_out: + f2fs_folio_put(folio, true); +iput_out: + iput(inode); +} + +static struct folio *last_fsync_dnode(struct f2fs_sb_info *sbi, nid_t ino) +{ + pgoff_t index; + struct folio_batch fbatch; + struct folio *last_folio = NULL; + int nr_folios; + + folio_batch_init(&fbatch); + index = 0; + + while ((nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi), &index, + (pgoff_t)-1, PAGECACHE_TAG_DIRTY, + &fbatch))) { + int i; + + for (i = 0; i < nr_folios; i++) { + struct folio *folio = fbatch.folios[i]; + + if (unlikely(f2fs_cp_error(sbi))) { + f2fs_folio_put(last_folio, false); + folio_batch_release(&fbatch); + return ERR_PTR(-EIO); + } + + if (!IS_DNODE(folio) || !is_cold_node(folio)) + continue; + if (ino_of_node(folio) != ino) + continue; + + folio_lock(folio); + + if (unlikely(!is_node_folio(folio))) { +continue_unlock: + folio_unlock(folio); + continue; + } + if (ino_of_node(folio) != ino) + goto continue_unlock; + + if (!folio_test_dirty(folio)) { + /* someone wrote it for us */ + goto continue_unlock; + } + + if (last_folio) + f2fs_folio_put(last_folio, false); + + folio_get(folio); + last_folio = folio; + folio_unlock(folio); + } + folio_batch_release(&fbatch); + cond_resched(); + } + return last_folio; +} + +static bool __write_node_folio(struct folio *folio, bool atomic, bool do_fsync, + bool *submitted, struct writeback_control *wbc, + bool do_balance, enum iostat_type io_type, + unsigned int *seq_id) +{ + struct f2fs_sb_info *sbi = F2FS_F_SB(folio); + nid_t nid; + struct node_info ni; + struct f2fs_io_info fio = { + .sbi = sbi, + .ino = ino_of_node(folio), + .type = NODE, + .op = REQ_OP_WRITE, + .op_flags = wbc_to_write_flags(wbc), + .folio = folio, + .encrypted_page = NULL, + .submitted = 0, + .io_type = io_type, + .io_wbc = wbc, + }; + struct f2fs_lock_context lc; + unsigned int seq; + + trace_f2fs_writepage(folio, NODE); + + if (unlikely(f2fs_cp_error(sbi))) { + /* keep node pages in remount-ro mode */ + if (F2FS_OPTION(sbi).errors == MOUNT_ERRORS_READONLY) + goto redirty_out; + folio_clear_uptodate(folio); + dec_page_count(sbi, F2FS_DIRTY_NODES); + folio_unlock(folio); + return true; + } + + if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) + goto redirty_out; + + if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) && + wbc->sync_mode == WB_SYNC_NONE && + IS_DNODE(folio) && is_cold_node(folio)) + goto redirty_out; + + /* get old block addr of this node page */ + nid = nid_of_node(folio); + + if (f2fs_sanity_check_node_footer(sbi, folio, nid, + NODE_TYPE_REGULAR, false)) { + fserror_report_metadata(sbi->sb, -EFSCORRUPTED, GFP_NOFS); + f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_CORRUPTED_NID); + goto redirty_out; + } + + if (f2fs_get_node_info(sbi, nid, &ni, !do_balance)) + goto redirty_out; + + f2fs_down_read_trace(&sbi->node_write, &lc); + + /* This page is already truncated */ + if (unlikely(ni.blk_addr == NULL_ADDR)) { + folio_clear_uptodate(folio); + dec_page_count(sbi, F2FS_DIRTY_NODES); + f2fs_up_read_trace(&sbi->node_write, &lc); + folio_unlock(folio); + return true; + } + + if (__is_valid_data_blkaddr(ni.blk_addr) && + !f2fs_is_valid_blkaddr(sbi, ni.blk_addr, + DATA_GENERIC_ENHANCE)) { + f2fs_up_read_trace(&sbi->node_write, &lc); + goto redirty_out; + } + + if (atomic && !test_opt(sbi, NOBARRIER)) + fio.op_flags |= REQ_PREFLUSH | REQ_FUA; + + set_dentry_mark(folio, false); + set_fsync_mark(folio, do_fsync); + if (IS_INODE(folio) && (atomic || is_fsync_dnode(folio))) + set_dentry_mark(folio, + f2fs_need_dentry_mark(sbi, ino_of_node(folio))); + + /* should add to global list before clearing PAGECACHE status */ + if (f2fs_in_warm_node_list(folio)) { + seq = f2fs_add_fsync_node_entry(sbi, folio); + if (seq_id) + *seq_id = seq; + } + + folio_start_writeback(folio); + + fio.old_blkaddr = ni.blk_addr; + f2fs_do_write_node_page(nid, &fio); + set_node_addr(sbi, &ni, fio.new_blkaddr, is_fsync_dnode(folio)); + dec_page_count(sbi, F2FS_DIRTY_NODES); + f2fs_up_read_trace(&sbi->node_write, &lc); + + folio_unlock(folio); + + if (unlikely(f2fs_cp_error(sbi))) { + f2fs_submit_merged_write(sbi, NODE); + submitted = NULL; + } + if (submitted) + *submitted = fio.submitted; + + if (do_balance) + f2fs_balance_fs(sbi, false); + return true; + +redirty_out: + folio_redirty_for_writepage(wbc, folio); + folio_unlock(folio); + return false; +} + +int f2fs_write_single_node_folio(struct folio *node_folio, int sync_mode, + bool mark_dirty, enum iostat_type io_type) +{ + int err = 0; + struct writeback_control wbc = { + .sync_mode = WB_SYNC_ALL, + .nr_to_write = 1, + }; + + if (!sync_mode) { + /* set page dirty and write it */ + if (!folio_test_writeback(node_folio)) + folio_mark_dirty(node_folio); + goto out_folio; + } + + f2fs_folio_wait_writeback(node_folio, NODE, true, true); + + if (mark_dirty) + folio_mark_dirty(node_folio); + else if (!folio_test_dirty(node_folio)) + goto out_folio; + + if (!folio_clear_dirty_for_io(node_folio)) { + err = -EAGAIN; + goto out_folio; + } + + if (!__write_node_folio(node_folio, false, false, NULL, + &wbc, false, io_type, NULL)) + err = -EAGAIN; + goto release_folio; +out_folio: + folio_unlock(node_folio); +release_folio: + f2fs_folio_put(node_folio, false); + return err; +} + +int f2fs_move_node_folio(struct folio *node_folio, int gc_type) +{ + return f2fs_write_single_node_folio(node_folio, gc_type == FG_GC, + true, FS_GC_NODE_IO); +} + +int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode, + struct writeback_control *wbc, bool atomic, + unsigned int *seq_id) +{ + pgoff_t index; + struct folio_batch fbatch; + int ret = 0; + struct folio *last_folio = NULL; + bool marked = false; + nid_t ino = inode->i_ino; + int nr_folios; + int nwritten = 0; + + if (atomic) { + last_folio = last_fsync_dnode(sbi, ino); + if (IS_ERR_OR_NULL(last_folio)) + return PTR_ERR_OR_ZERO(last_folio); + } +retry: + folio_batch_init(&fbatch); + index = 0; + + while ((nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi), &index, + (pgoff_t)-1, PAGECACHE_TAG_DIRTY, + &fbatch))) { + int i; + + for (i = 0; i < nr_folios; i++) { + struct folio *folio = fbatch.folios[i]; + bool submitted = false; + bool do_fsync = false; + + if (unlikely(f2fs_cp_error(sbi))) { + f2fs_folio_put(last_folio, false); + folio_batch_release(&fbatch); + ret = -EIO; + goto out; + } + + if (!IS_DNODE(folio) || !is_cold_node(folio)) + continue; + if (ino_of_node(folio) != ino) + continue; + + folio_lock(folio); + + if (unlikely(!is_node_folio(folio))) { +continue_unlock: + folio_unlock(folio); + continue; + } + if (ino_of_node(folio) != ino) + goto continue_unlock; + + if (!folio_test_dirty(folio) && folio != last_folio) { + /* someone wrote it for us */ + goto continue_unlock; + } + + f2fs_folio_wait_writeback(folio, NODE, true, true); + + if (!atomic || folio == last_folio) { + do_fsync = true; + percpu_counter_inc(&sbi->rf_node_block_count); + if (IS_INODE(folio)) { + if (is_inode_flag_set(inode, + FI_DIRTY_INODE)) + f2fs_update_inode(inode, folio); + } + /* may be written by other thread */ + if (!folio_test_dirty(folio)) + folio_mark_dirty(folio); + } + + if (!folio_clear_dirty_for_io(folio)) + goto continue_unlock; + + if (!__write_node_folio(folio, atomic && + folio == last_folio, + do_fsync, &submitted, + wbc, true, FS_NODE_IO, + seq_id)) { + f2fs_folio_put(last_folio, false); + folio_batch_release(&fbatch); + ret = -EIO; + goto out; + } + if (submitted) + nwritten++; + + if (folio == last_folio) { + f2fs_folio_put(folio, false); + folio_batch_release(&fbatch); + marked = true; + goto out; + } + } + folio_batch_release(&fbatch); + cond_resched(); + } + if (atomic && !marked) { + f2fs_debug(sbi, "Retry to write fsync mark: ino=%u, idx=%lx", + ino, last_folio->index); + folio_lock(last_folio); + f2fs_folio_wait_writeback(last_folio, NODE, true, true); + folio_mark_dirty(last_folio); + folio_unlock(last_folio); + goto retry; + } +out: + if (nwritten) + f2fs_submit_merged_write_cond(sbi, NULL, NULL, ino, NODE); + return ret; +} + +static int f2fs_match_ino(struct inode *inode, u64 ino, void *data) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + bool clean; + + if (inode->i_ino != ino) + return 0; + + if (!is_inode_flag_set(inode, FI_DIRTY_INODE)) + return 0; + + spin_lock(&sbi->inode_lock[DIRTY_META]); + clean = list_empty(&F2FS_I(inode)->gdirty_list); + spin_unlock(&sbi->inode_lock[DIRTY_META]); + + if (clean) + return 0; + + inode = igrab(inode); + if (!inode) + return 0; + return 1; +} + +static bool flush_dirty_inode(struct folio *folio) +{ + struct f2fs_sb_info *sbi = F2FS_F_SB(folio); + struct inode *inode; + nid_t ino = ino_of_node(folio); + + inode = find_inode_nowait(sbi->sb, ino, f2fs_match_ino, NULL); + if (!inode) + return false; + + f2fs_update_inode(inode, folio); + folio_unlock(folio); + + iput(inode); + return true; +} + +void f2fs_flush_inline_data(struct f2fs_sb_info *sbi) +{ + pgoff_t index = 0; + struct folio_batch fbatch; + int nr_folios; + + folio_batch_init(&fbatch); + + while ((nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi), &index, + (pgoff_t)-1, PAGECACHE_TAG_DIRTY, + &fbatch))) { + int i; + + for (i = 0; i < nr_folios; i++) { + struct folio *folio = fbatch.folios[i]; + + if (!IS_INODE(folio)) + continue; + + folio_lock(folio); + + if (unlikely(!is_node_folio(folio))) + goto unlock; + if (!folio_test_dirty(folio)) + goto unlock; + + /* flush inline_data, if it's async context. */ + if (folio_test_f2fs_inline(folio)) { + folio_clear_f2fs_inline(folio); + folio_unlock(folio); + flush_inline_data(sbi, ino_of_node(folio)); + continue; + } +unlock: + folio_unlock(folio); + } + folio_batch_release(&fbatch); + cond_resched(); + } +} + +int f2fs_sync_node_pages(struct f2fs_sb_info *sbi, + struct writeback_control *wbc, + bool do_balance, enum iostat_type io_type) +{ + pgoff_t index; + struct folio_batch fbatch; + int step = 0; + int nwritten = 0; + int ret = 0; + int nr_folios, done = 0; + + folio_batch_init(&fbatch); + +next_step: + index = 0; + + while (!done && (nr_folios = filemap_get_folios_tag(NODE_MAPPING(sbi), + &index, (pgoff_t)-1, PAGECACHE_TAG_DIRTY, + &fbatch))) { + int i; + + for (i = 0; i < nr_folios; i++) { + struct folio *folio = fbatch.folios[i]; + bool submitted = false; + + /* give a priority to WB_SYNC threads */ + if (atomic_read(&sbi->wb_sync_req[NODE]) && + wbc->sync_mode == WB_SYNC_NONE) { + done = 1; + break; + } + + /* + * flushing sequence with step: + * 0. indirect nodes + * 1. dentry dnodes + * 2. file dnodes + */ + if (step == 0 && IS_DNODE(folio)) + continue; + if (step == 1 && (!IS_DNODE(folio) || + is_cold_node(folio))) + continue; + if (step == 2 && (!IS_DNODE(folio) || + !is_cold_node(folio))) + continue; +lock_node: + if (wbc->sync_mode == WB_SYNC_ALL) + folio_lock(folio); + else if (!folio_trylock(folio)) + continue; + + if (unlikely(!is_node_folio(folio))) { +continue_unlock: + folio_unlock(folio); + continue; + } + + if (!folio_test_dirty(folio)) { + /* someone wrote it for us */ + goto continue_unlock; + } + + /* flush inline_data/inode, if it's async context. */ + if (!do_balance) + goto write_node; + + /* flush inline_data */ + if (folio_test_f2fs_inline(folio)) { + folio_clear_f2fs_inline(folio); + folio_unlock(folio); + flush_inline_data(sbi, ino_of_node(folio)); + goto lock_node; + } + + /* flush dirty inode */ + if (IS_INODE(folio) && flush_dirty_inode(folio)) + goto lock_node; +write_node: + f2fs_folio_wait_writeback(folio, NODE, true, true); + + if (!folio_clear_dirty_for_io(folio)) + goto continue_unlock; + + if (!__write_node_folio(folio, false, false, &submitted, + wbc, do_balance, io_type, NULL)) { + folio_batch_release(&fbatch); + ret = -EIO; + goto out; + } + if (submitted) + nwritten++; + + if (--wbc->nr_to_write == 0) + break; + } + folio_batch_release(&fbatch); + cond_resched(); + + if (wbc->nr_to_write == 0) { + step = 2; + break; + } + } + + if (step < 2) { + if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) && + wbc->sync_mode == WB_SYNC_NONE && step == 1) + goto out; + step++; + goto next_step; + } +out: + if (nwritten) + f2fs_submit_merged_write(sbi, NODE); + + if (unlikely(f2fs_cp_error(sbi))) + return -EIO; + return ret; +} + +int f2fs_wait_on_node_pages_writeback(struct f2fs_sb_info *sbi, + unsigned int seq_id) +{ + struct fsync_node_entry *fn; + struct list_head *head = &sbi->fsync_node_list; + unsigned long flags; + unsigned int cur_seq_id = 0; + + while (seq_id && cur_seq_id < seq_id) { + struct folio *folio; + + spin_lock_irqsave(&sbi->fsync_node_lock, flags); + if (list_empty(head)) { + spin_unlock_irqrestore(&sbi->fsync_node_lock, flags); + break; + } + fn = list_first_entry(head, struct fsync_node_entry, list); + if (fn->seq_id > seq_id) { + spin_unlock_irqrestore(&sbi->fsync_node_lock, flags); + break; + } + cur_seq_id = fn->seq_id; + folio = fn->folio; + folio_get(folio); + spin_unlock_irqrestore(&sbi->fsync_node_lock, flags); + + f2fs_folio_wait_writeback(folio, NODE, true, false); + + folio_put(folio); + } + + return filemap_check_errors(NODE_MAPPING(sbi)); +} + +static int f2fs_write_node_pages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct f2fs_sb_info *sbi = F2FS_M_SB(mapping); + struct blk_plug plug; + long diff; + + if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) + goto skip_write; + + /* balancing f2fs's metadata in background */ + f2fs_balance_fs_bg(sbi, true); + + /* collect a number of dirty node pages and write together */ + if (wbc->sync_mode != WB_SYNC_ALL && + get_pages(sbi, F2FS_DIRTY_NODES) < + nr_pages_to_skip(sbi, NODE)) + goto skip_write; + + if (wbc->sync_mode == WB_SYNC_ALL) + atomic_inc(&sbi->wb_sync_req[NODE]); + else if (atomic_read(&sbi->wb_sync_req[NODE])) { + /* to avoid potential deadlock */ + if (current->plug) + blk_finish_plug(current->plug); + goto skip_write; + } + + trace_f2fs_writepages(mapping->host, wbc, NODE); + + diff = nr_pages_to_write(sbi, NODE, wbc); + blk_start_plug(&plug); + f2fs_sync_node_pages(sbi, wbc, true, FS_NODE_IO); + blk_finish_plug(&plug); + wbc->nr_to_write = max((long)0, wbc->nr_to_write - diff); + + if (wbc->sync_mode == WB_SYNC_ALL) + atomic_dec(&sbi->wb_sync_req[NODE]); + return 0; + +skip_write: + wbc->pages_skipped += get_pages(sbi, F2FS_DIRTY_NODES); + trace_f2fs_writepages(mapping->host, wbc, NODE); + return 0; +} + +static bool f2fs_dirty_node_folio(struct address_space *mapping, + struct folio *folio) +{ + trace_f2fs_set_page_dirty(folio, NODE); + + if (!folio_test_uptodate(folio)) + folio_mark_uptodate(folio); +#ifdef CONFIG_F2FS_CHECK_FS + if (IS_INODE(folio)) + f2fs_inode_chksum_set(F2FS_M_SB(mapping), folio); +#endif + if (filemap_dirty_folio(mapping, folio)) { + inc_page_count(F2FS_M_SB(mapping), F2FS_DIRTY_NODES); + folio_set_f2fs_reference(folio); + return true; + } + return false; +} + +/* + * Structure of the f2fs node operations + */ +const struct address_space_operations f2fs_node_aops = { + .writepages = f2fs_write_node_pages, + .dirty_folio = f2fs_dirty_node_folio, + .invalidate_folio = f2fs_invalidate_folio, + .release_folio = f2fs_release_folio, + .migrate_folio = filemap_migrate_folio, +}; + +static struct free_nid *__lookup_free_nid_list(struct f2fs_nm_info *nm_i, + nid_t n) +{ + return radix_tree_lookup(&nm_i->free_nid_root, n); +} + +static int __insert_free_nid(struct f2fs_sb_info *sbi, + struct free_nid *i) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + int err = radix_tree_insert(&nm_i->free_nid_root, i->nid, i); + + if (err) + return err; + + nm_i->nid_cnt[FREE_NID]++; + list_add_tail(&i->list, &nm_i->free_nid_list); + return 0; +} + +static void __remove_free_nid(struct f2fs_sb_info *sbi, + struct free_nid *i, enum nid_state state) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + + f2fs_bug_on(sbi, state != i->state); + nm_i->nid_cnt[state]--; + if (state == FREE_NID) + list_del(&i->list); + radix_tree_delete(&nm_i->free_nid_root, i->nid); +} + +static void __move_free_nid(struct f2fs_sb_info *sbi, struct free_nid *i, + enum nid_state org_state, enum nid_state dst_state) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + + f2fs_bug_on(sbi, org_state != i->state); + i->state = dst_state; + nm_i->nid_cnt[org_state]--; + nm_i->nid_cnt[dst_state]++; + + switch (dst_state) { + case PREALLOC_NID: + list_del(&i->list); + break; + case FREE_NID: + list_add_tail(&i->list, &nm_i->free_nid_list); + break; + default: + BUG_ON(1); + } +} + +static void update_free_nid_bitmap(struct f2fs_sb_info *sbi, nid_t nid, + bool set, bool build) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + unsigned int nat_ofs = NAT_BLOCK_OFFSET(nid); + unsigned int nid_ofs = nid - START_NID(nid); + + if (!test_bit_le(nat_ofs, nm_i->nat_block_bitmap)) + return; + + if (set) { + if (test_bit_le(nid_ofs, nm_i->free_nid_bitmap[nat_ofs])) + return; + __set_bit_le(nid_ofs, nm_i->free_nid_bitmap[nat_ofs]); + nm_i->free_nid_count[nat_ofs]++; + } else { + if (!test_bit_le(nid_ofs, nm_i->free_nid_bitmap[nat_ofs])) + return; + __clear_bit_le(nid_ofs, nm_i->free_nid_bitmap[nat_ofs]); + if (!build) + nm_i->free_nid_count[nat_ofs]--; + } +} + +/* return if the nid is recognized as free */ +static bool add_free_nid(struct f2fs_sb_info *sbi, + nid_t nid, bool build, bool update) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct free_nid *i, *e; + struct nat_entry *ne; + int err; + bool ret = false; + + /* 0 nid should not be used */ + if (unlikely(nid == 0)) + return false; + + if (unlikely(f2fs_check_nid_range(sbi, nid))) + return false; + + i = f2fs_kmem_cache_alloc(free_nid_slab, GFP_NOFS, true, NULL); + i->nid = nid; + i->state = FREE_NID; + + err = radix_tree_preload(GFP_NOFS | __GFP_NOFAIL); + f2fs_bug_on(sbi, err); + + err = -EINVAL; + + spin_lock(&nm_i->nid_list_lock); + + if (build) { + /* + * Thread A Thread B + * - f2fs_create + * - f2fs_new_inode + * - f2fs_alloc_nid + * - __insert_nid_to_list(PREALLOC_NID) + * - f2fs_balance_fs_bg + * - f2fs_build_free_nids + * - __f2fs_build_free_nids + * - scan_nat_page + * - add_free_nid + * - __lookup_nat_cache + * - f2fs_add_link + * - f2fs_init_inode_metadata + * - f2fs_new_inode_folio + * - f2fs_new_node_folio + * - set_node_addr + * - f2fs_alloc_nid_done + * - __remove_nid_from_list(PREALLOC_NID) + * - __insert_nid_to_list(FREE_NID) + */ + ne = __lookup_nat_cache(nm_i, nid, false); + if (ne && (!get_nat_flag(ne, IS_CHECKPOINTED) || + nat_get_blkaddr(ne) != NULL_ADDR)) + goto err_out; + + e = __lookup_free_nid_list(nm_i, nid); + if (e) { + if (e->state == FREE_NID) + ret = true; + goto err_out; + } + } + ret = true; + err = __insert_free_nid(sbi, i); +err_out: + if (update) { + update_free_nid_bitmap(sbi, nid, ret, build); + if (!build) + nm_i->available_nids++; + } + spin_unlock(&nm_i->nid_list_lock); + radix_tree_preload_end(); + + if (err) + kmem_cache_free(free_nid_slab, i); + return ret; +} + +static void remove_free_nid(struct f2fs_sb_info *sbi, nid_t nid) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct free_nid *i; + bool need_free = false; + + spin_lock(&nm_i->nid_list_lock); + i = __lookup_free_nid_list(nm_i, nid); + if (i && i->state == FREE_NID) { + __remove_free_nid(sbi, i, FREE_NID); + need_free = true; + } + spin_unlock(&nm_i->nid_list_lock); + + if (need_free) + kmem_cache_free(free_nid_slab, i); +} + +static int scan_nat_page(struct f2fs_sb_info *sbi, + struct f2fs_nat_block *nat_blk, nid_t start_nid) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + block_t blk_addr; + unsigned int nat_ofs = NAT_BLOCK_OFFSET(start_nid); + int i; + + __set_bit_le(nat_ofs, nm_i->nat_block_bitmap); + + i = start_nid % NAT_ENTRY_PER_BLOCK; + + for (; i < NAT_ENTRY_PER_BLOCK; i++, start_nid++) { + if (unlikely(start_nid >= nm_i->max_nid)) + break; + + blk_addr = le32_to_cpu(nat_blk->entries[i].block_addr); + + if (blk_addr == NEW_ADDR) + return -EFSCORRUPTED; + + if (blk_addr == NULL_ADDR) { + add_free_nid(sbi, start_nid, true, true); + } else { + spin_lock(&NM_I(sbi)->nid_list_lock); + update_free_nid_bitmap(sbi, start_nid, false, true); + spin_unlock(&NM_I(sbi)->nid_list_lock); + } + } + + return 0; +} + +static void scan_curseg_cache(struct f2fs_sb_info *sbi) +{ + struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA); + struct f2fs_journal *journal = curseg->journal; + int i; + + down_read(&curseg->journal_rwsem); + for (i = 0; i < nats_in_cursum(journal); i++) { + block_t addr; + nid_t nid; + + addr = le32_to_cpu(nat_in_journal(journal, i).block_addr); + nid = le32_to_cpu(nid_in_journal(journal, i)); + if (addr == NULL_ADDR) + add_free_nid(sbi, nid, true, false); + else + remove_free_nid(sbi, nid); + } + up_read(&curseg->journal_rwsem); +} + +static void scan_free_nid_bits(struct f2fs_sb_info *sbi) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + unsigned int i, idx; + nid_t nid; + + f2fs_down_read(&nm_i->nat_tree_lock); + + for (i = 0; i < nm_i->nat_blocks; i++) { + if (!test_bit_le(i, nm_i->nat_block_bitmap)) + continue; + if (!nm_i->free_nid_count[i]) + continue; + for (idx = 0; idx < NAT_ENTRY_PER_BLOCK; idx++) { + idx = find_next_bit_le(nm_i->free_nid_bitmap[i], + NAT_ENTRY_PER_BLOCK, idx); + if (idx >= NAT_ENTRY_PER_BLOCK) + break; + + nid = i * NAT_ENTRY_PER_BLOCK + idx; + add_free_nid(sbi, nid, true, false); + + if (nm_i->nid_cnt[FREE_NID] >= MAX_FREE_NIDS) + goto out; + } + } +out: + scan_curseg_cache(sbi); + + f2fs_up_read(&nm_i->nat_tree_lock); +} + +static int __f2fs_build_free_nids(struct f2fs_sb_info *sbi, + bool sync, bool mount) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + int i = 0, ret; + nid_t nid = nm_i->next_scan_nid; + + if (unlikely(nid >= nm_i->max_nid)) + nid = 0; + + if (unlikely(nid % NAT_ENTRY_PER_BLOCK)) + nid = NAT_BLOCK_OFFSET(nid) * NAT_ENTRY_PER_BLOCK; + + /* Enough entries */ + if (nm_i->nid_cnt[FREE_NID] >= NAT_ENTRY_PER_BLOCK) + return 0; + + if (!sync && !f2fs_available_free_memory(sbi, FREE_NIDS)) + return 0; + + if (!mount) { + /* try to find free nids in free_nid_bitmap */ + scan_free_nid_bits(sbi); + + if (nm_i->nid_cnt[FREE_NID] >= NAT_ENTRY_PER_BLOCK) + return 0; + } + + /* readahead nat pages to be scanned */ + f2fs_ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nid), FREE_NID_PAGES, + META_NAT, true); + + f2fs_down_read(&nm_i->nat_tree_lock); + + while (1) { + if (!test_bit_le(NAT_BLOCK_OFFSET(nid), + nm_i->nat_block_bitmap)) { + struct folio *folio = get_current_nat_folio(sbi, nid); + + if (IS_ERR(folio)) { + ret = PTR_ERR(folio); + } else { + ret = scan_nat_page(sbi, folio_address(folio), + nid); + f2fs_folio_put(folio, true); + } + + if (ret) { + f2fs_up_read(&nm_i->nat_tree_lock); + + if (ret == -EFSCORRUPTED) { + f2fs_err(sbi, "NAT is corrupt, run fsck to fix it"); + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_handle_error(sbi, + ERROR_INCONSISTENT_NAT); + } + + return ret; + } + } + + nid += (NAT_ENTRY_PER_BLOCK - (nid % NAT_ENTRY_PER_BLOCK)); + if (unlikely(nid >= nm_i->max_nid)) + nid = 0; + + if (++i >= FREE_NID_PAGES) + break; + } + + /* go to the next free nat pages to find free nids abundantly */ + nm_i->next_scan_nid = nid; + + /* find free nids from current sum_pages */ + scan_curseg_cache(sbi); + + f2fs_up_read(&nm_i->nat_tree_lock); + + f2fs_ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nm_i->next_scan_nid), + nm_i->ra_nid_pages, META_NAT, false); + + return 0; +} + +int f2fs_build_free_nids(struct f2fs_sb_info *sbi, bool sync, bool mount) +{ + int ret; + + mutex_lock(&NM_I(sbi)->build_lock); + ret = __f2fs_build_free_nids(sbi, sync, mount); + mutex_unlock(&NM_I(sbi)->build_lock); + + return ret; +} + +/* + * If this function returns success, caller can obtain a new nid + * from second parameter of this function. + * The returned nid could be used ino as well as nid when inode is created. + */ +bool f2fs_alloc_nid(struct f2fs_sb_info *sbi, nid_t *nid) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct free_nid *i = NULL; +retry: + if (time_to_inject(sbi, FAULT_ALLOC_NID)) + return false; + + spin_lock(&nm_i->nid_list_lock); + + if (unlikely(nm_i->available_nids == 0)) { + spin_unlock(&nm_i->nid_list_lock); + return false; + } + + /* We should not use stale free nids created by f2fs_build_free_nids */ + if (nm_i->nid_cnt[FREE_NID] && !on_f2fs_build_free_nids(nm_i)) { + f2fs_bug_on(sbi, list_empty(&nm_i->free_nid_list)); + i = list_first_entry(&nm_i->free_nid_list, + struct free_nid, list); + + if (unlikely(is_invalid_nid(sbi, i->nid))) { + spin_unlock(&nm_i->nid_list_lock); + f2fs_err(sbi, "Corrupted nid %u in free_nid_list", + i->nid); + fserror_report_metadata(sbi->sb, -EFSCORRUPTED, + GFP_NOFS); + f2fs_stop_checkpoint(sbi, false, + STOP_CP_REASON_CORRUPTED_NID); + return false; + } + + *nid = i->nid; + + __move_free_nid(sbi, i, FREE_NID, PREALLOC_NID); + nm_i->available_nids--; + + update_free_nid_bitmap(sbi, *nid, false, false); + + spin_unlock(&nm_i->nid_list_lock); + return true; + } + spin_unlock(&nm_i->nid_list_lock); + + /* Let's scan nat pages and its caches to get free nids */ + if (!f2fs_build_free_nids(sbi, true, false)) + goto retry; + return false; +} + +/* + * f2fs_alloc_nid() should be called prior to this function. + */ +void f2fs_alloc_nid_done(struct f2fs_sb_info *sbi, nid_t nid) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct free_nid *i; + + spin_lock(&nm_i->nid_list_lock); + i = __lookup_free_nid_list(nm_i, nid); + f2fs_bug_on(sbi, !i); + __remove_free_nid(sbi, i, PREALLOC_NID); + spin_unlock(&nm_i->nid_list_lock); + + kmem_cache_free(free_nid_slab, i); +} + +/* + * f2fs_alloc_nid() should be called prior to this function. + */ +void f2fs_alloc_nid_failed(struct f2fs_sb_info *sbi, nid_t nid) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct free_nid *i; + bool need_free = false; + + if (!nid) + return; + + spin_lock(&nm_i->nid_list_lock); + i = __lookup_free_nid_list(nm_i, nid); + f2fs_bug_on(sbi, !i); + + if (!f2fs_available_free_memory(sbi, FREE_NIDS)) { + __remove_free_nid(sbi, i, PREALLOC_NID); + need_free = true; + } else { + __move_free_nid(sbi, i, PREALLOC_NID, FREE_NID); + } + + nm_i->available_nids++; + + update_free_nid_bitmap(sbi, nid, true, false); + + spin_unlock(&nm_i->nid_list_lock); + + if (need_free) + kmem_cache_free(free_nid_slab, i); +} + +int f2fs_try_to_free_nids(struct f2fs_sb_info *sbi, int nr_shrink) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + int nr = nr_shrink; + + if (nm_i->nid_cnt[FREE_NID] <= MAX_FREE_NIDS) + return 0; + + if (!mutex_trylock(&nm_i->build_lock)) + return 0; + + while (nr_shrink && nm_i->nid_cnt[FREE_NID] > MAX_FREE_NIDS) { + struct free_nid *i, *next; + unsigned int batch = SHRINK_NID_BATCH_SIZE; + + spin_lock(&nm_i->nid_list_lock); + list_for_each_entry_safe(i, next, &nm_i->free_nid_list, list) { + if (!nr_shrink || !batch || + nm_i->nid_cnt[FREE_NID] <= MAX_FREE_NIDS) + break; + __remove_free_nid(sbi, i, FREE_NID); + kmem_cache_free(free_nid_slab, i); + nr_shrink--; + batch--; + } + spin_unlock(&nm_i->nid_list_lock); + } + + mutex_unlock(&nm_i->build_lock); + + return nr - nr_shrink; +} + +int f2fs_recover_inline_xattr(struct inode *inode, struct folio *folio) +{ + void *src_addr, *dst_addr; + size_t inline_size; + struct folio *ifolio; + struct f2fs_inode *ri; + + ifolio = f2fs_get_inode_folio(F2FS_I_SB(inode), inode->i_ino); + if (IS_ERR(ifolio)) + return PTR_ERR(ifolio); + + ri = F2FS_INODE(folio); + if (ri->i_inline & F2FS_INLINE_XATTR) { + if (!f2fs_has_inline_xattr(inode)) { + set_inode_flag(inode, FI_INLINE_XATTR); + stat_inc_inline_xattr(inode); + } + } else { + if (f2fs_has_inline_xattr(inode)) { + stat_dec_inline_xattr(inode); + clear_inode_flag(inode, FI_INLINE_XATTR); + } + goto update_inode; + } + + dst_addr = inline_xattr_addr(inode, ifolio); + src_addr = inline_xattr_addr(inode, folio); + inline_size = inline_xattr_size(inode); + + f2fs_folio_wait_writeback(ifolio, NODE, true, true); + memcpy(dst_addr, src_addr, inline_size); +update_inode: + f2fs_update_inode(inode, ifolio); + f2fs_folio_put(ifolio, true); + return 0; +} + +int f2fs_recover_xattr_data(struct inode *inode, struct folio *folio) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + nid_t prev_xnid = F2FS_I(inode)->i_xattr_nid; + nid_t new_xnid; + struct dnode_of_data dn; + struct node_info ni; + struct folio *xfolio; + int err; + + if (!prev_xnid) + goto recover_xnid; + + /* 1: invalidate the previous xattr nid */ + err = f2fs_get_node_info(sbi, prev_xnid, &ni, false); + if (err) + return err; + + f2fs_invalidate_blocks(sbi, ni.blk_addr, 1); + dec_valid_node_count(sbi, inode, false); + set_node_addr(sbi, &ni, NULL_ADDR, false); + +recover_xnid: + /* 2: update xattr nid in inode */ + if (!f2fs_alloc_nid(sbi, &new_xnid)) + return -ENOSPC; + + set_new_dnode(&dn, inode, NULL, NULL, new_xnid); + xfolio = f2fs_new_node_folio(&dn, XATTR_NODE_OFFSET); + if (IS_ERR(xfolio)) { + f2fs_alloc_nid_failed(sbi, new_xnid); + return PTR_ERR(xfolio); + } + + f2fs_alloc_nid_done(sbi, new_xnid); + f2fs_update_inode_page(inode); + + /* 3: update and set xattr node page dirty */ + if (folio) { + memcpy(F2FS_NODE(xfolio), F2FS_NODE(folio), + VALID_XATTR_BLOCK_SIZE); + folio_mark_dirty(xfolio); + } + f2fs_folio_put(xfolio, true); + + return 0; +} + +int f2fs_recover_inode_page(struct f2fs_sb_info *sbi, struct folio *folio) +{ + struct f2fs_inode *src, *dst; + nid_t ino = ino_of_node(folio); + struct node_info old_ni, new_ni; + struct folio *ifolio; + int err; + + err = f2fs_get_node_info(sbi, ino, &old_ni, false); + if (err) + return err; + + if (unlikely(old_ni.blk_addr != NULL_ADDR)) + return -EINVAL; +retry: + ifolio = f2fs_grab_cache_folio(NODE_MAPPING(sbi), ino, false); + if (IS_ERR(ifolio)) { + memalloc_retry_wait(GFP_NOFS); + goto retry; + } + + /* Should not use this inode from free nid list */ + remove_free_nid(sbi, ino); + + if (!folio_test_uptodate(ifolio)) + folio_mark_uptodate(ifolio); + fill_node_footer(ifolio, ino, ino, 0, true); + set_cold_node(ifolio, false); + + src = F2FS_INODE(folio); + dst = F2FS_INODE(ifolio); + + memcpy(dst, src, offsetof(struct f2fs_inode, i_ext)); + dst->i_size = 0; + dst->i_blocks = cpu_to_le64(1); + dst->i_links = cpu_to_le32(1); + dst->i_xattr_nid = 0; + dst->i_inline = src->i_inline & (F2FS_INLINE_XATTR | F2FS_EXTRA_ATTR); + if (dst->i_inline & F2FS_EXTRA_ATTR) { + dst->i_extra_isize = src->i_extra_isize; + + if (f2fs_sb_has_flexible_inline_xattr(sbi) && + F2FS_FITS_IN_INODE(src, le16_to_cpu(src->i_extra_isize), + i_inline_xattr_size)) + dst->i_inline_xattr_size = src->i_inline_xattr_size; + + if (f2fs_sb_has_project_quota(sbi) && + F2FS_FITS_IN_INODE(src, le16_to_cpu(src->i_extra_isize), + i_projid)) + dst->i_projid = src->i_projid; + + if (f2fs_sb_has_inode_crtime(sbi) && + F2FS_FITS_IN_INODE(src, le16_to_cpu(src->i_extra_isize), + i_crtime_nsec)) { + dst->i_crtime = src->i_crtime; + dst->i_crtime_nsec = src->i_crtime_nsec; + } + } + + new_ni = old_ni; + new_ni.ino = ino; + + if (unlikely(inc_valid_node_count(sbi, NULL, true))) + WARN_ON(1); + set_node_addr(sbi, &new_ni, NEW_ADDR, false); + inc_valid_inode_count(sbi); + folio_mark_dirty(ifolio); + f2fs_folio_put(ifolio, true); + return 0; +} + +int f2fs_restore_node_summary(struct f2fs_sb_info *sbi, + unsigned int segno, struct f2fs_summary_block *sum) +{ + struct f2fs_node *rn; + struct f2fs_summary *sum_entry; + block_t addr; + int i, idx, last_offset, nrpages; + + /* scan the node segment */ + last_offset = BLKS_PER_SEG(sbi); + addr = START_BLOCK(sbi, segno); + sum_entry = sum_entries(sum); + + for (i = 0; i < last_offset; i += nrpages, addr += nrpages) { + nrpages = bio_max_segs(last_offset - i); + + /* readahead node pages */ + f2fs_ra_meta_pages(sbi, addr, nrpages, META_POR, true); + + for (idx = addr; idx < addr + nrpages; idx++) { + struct folio *folio = f2fs_get_tmp_folio(sbi, idx); + + if (IS_ERR(folio)) + return PTR_ERR(folio); + + rn = F2FS_NODE(folio); + sum_entry->nid = rn->footer.nid; + sum_entry->version = 0; + sum_entry->ofs_in_node = 0; + sum_entry++; + f2fs_folio_put(folio, true); + } + + invalidate_mapping_pages(META_MAPPING(sbi), addr, + addr + nrpages); + } + return 0; +} + +static void remove_nats_in_journal(struct f2fs_sb_info *sbi) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA); + struct f2fs_journal *journal = curseg->journal; + int i; + bool init_dirty; + + down_write(&curseg->journal_rwsem); + for (i = 0; i < nats_in_cursum(journal); i++) { + struct nat_entry *ne; + struct f2fs_nat_entry raw_ne; + nid_t nid = le32_to_cpu(nid_in_journal(journal, i)); + + if (f2fs_check_nid_range(sbi, nid)) + continue; + + init_dirty = false; + + raw_ne = nat_in_journal(journal, i); + + ne = __lookup_nat_cache(nm_i, nid, true); + if (!ne) { + init_dirty = true; + ne = __alloc_nat_entry(sbi, nid, true); + __init_nat_entry(nm_i, ne, &raw_ne, true, true); + } + + /* + * if a free nat in journal has not been used after last + * checkpoint, we should remove it from available nids, + * since later we will add it again. + */ + if (!get_nat_flag(ne, IS_DIRTY) && + le32_to_cpu(raw_ne.block_addr) == NULL_ADDR) { + spin_lock(&nm_i->nid_list_lock); + nm_i->available_nids--; + spin_unlock(&nm_i->nid_list_lock); + } + + __set_nat_cache_dirty(nm_i, ne, init_dirty); + } + update_nats_in_cursum(journal, -i); + up_write(&curseg->journal_rwsem); +} + +static void __adjust_nat_entry_set(struct nat_entry_set *nes, + struct list_head *head, int max) +{ + struct nat_entry_set *cur; + + if (nes->entry_cnt >= max) + goto add_out; + + list_for_each_entry(cur, head, set_list) { + if (cur->entry_cnt >= nes->entry_cnt) { + list_add(&nes->set_list, cur->set_list.prev); + return; + } + } +add_out: + list_add_tail(&nes->set_list, head); +} + +static void __update_nat_bits(struct f2fs_sb_info *sbi, nid_t start_nid, + const struct f2fs_nat_block *nat_blk) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + unsigned int nat_index = start_nid / NAT_ENTRY_PER_BLOCK; + int valid = 0; + int i = 0; + + if (!enabled_nat_bits(sbi, NULL)) + return; + + if (nat_index == 0) { + valid = 1; + i = 1; + } + for (; i < NAT_ENTRY_PER_BLOCK; i++) { + if (le32_to_cpu(nat_blk->entries[i].block_addr) != NULL_ADDR) + valid++; + } + if (valid == 0) { + __set_bit_le(nat_index, nm_i->empty_nat_bits); + __clear_bit_le(nat_index, nm_i->full_nat_bits); + return; + } + + __clear_bit_le(nat_index, nm_i->empty_nat_bits); + if (valid == NAT_ENTRY_PER_BLOCK) + __set_bit_le(nat_index, nm_i->full_nat_bits); + else + __clear_bit_le(nat_index, nm_i->full_nat_bits); +} + +static int __flush_nat_entry_set(struct f2fs_sb_info *sbi, + struct nat_entry_set *set, struct cp_control *cpc) +{ + struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA); + struct f2fs_journal *journal = curseg->journal; + nid_t start_nid = set->set * NAT_ENTRY_PER_BLOCK; + bool to_journal = true; + struct f2fs_nat_block *nat_blk; + struct nat_entry *ne, *cur; + struct folio *folio = NULL; + + /* + * there are two steps to flush nat entries: + * #1, flush nat entries to journal in current hot data summary block. + * #2, flush nat entries to nat page. + */ + if (enabled_nat_bits(sbi, cpc) || + !__has_cursum_space(sbi, journal, set->entry_cnt, NAT_JOURNAL)) + to_journal = false; + + if (to_journal) { + down_write(&curseg->journal_rwsem); + } else { + folio = get_next_nat_folio(sbi, start_nid); + if (IS_ERR(folio)) + return PTR_ERR(folio); + + nat_blk = folio_address(folio); + f2fs_bug_on(sbi, !nat_blk); + } + + /* flush dirty nats in nat entry set */ + list_for_each_entry_safe(ne, cur, &set->entry_list, list) { + struct f2fs_nat_entry *raw_ne; + nid_t nid = nat_get_nid(ne); + int offset; + + f2fs_bug_on(sbi, nat_get_blkaddr(ne) == NEW_ADDR); + + if (to_journal) { + offset = f2fs_lookup_journal_in_cursum(sbi, journal, + NAT_JOURNAL, nid, 1); + f2fs_bug_on(sbi, offset < 0); + raw_ne = &nat_in_journal(journal, offset); + nid_in_journal(journal, offset) = cpu_to_le32(nid); + } else { + raw_ne = &nat_blk->entries[nid - start_nid]; + } + raw_nat_from_node_info(raw_ne, &ne->ni); + nat_reset_flag(ne); + __clear_nat_cache_dirty(NM_I(sbi), set, ne); + if (nat_get_blkaddr(ne) == NULL_ADDR) { + add_free_nid(sbi, nid, false, true); + } else { + spin_lock(&NM_I(sbi)->nid_list_lock); + update_free_nid_bitmap(sbi, nid, false, false); + spin_unlock(&NM_I(sbi)->nid_list_lock); + } + } + + if (to_journal) { + up_write(&curseg->journal_rwsem); + } else { + __update_nat_bits(sbi, start_nid, nat_blk); + f2fs_folio_put(folio, true); + } + + /* Allow dirty nats by node block allocation in write_begin */ + if (!set->entry_cnt) { + radix_tree_delete(&NM_I(sbi)->nat_set_root, set->set); + kmem_cache_free(nat_entry_set_slab, set); + } + return 0; +} + +/* + * This function is called during the checkpointing process. + */ +int f2fs_flush_nat_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA); + struct f2fs_journal *journal = curseg->journal; + struct nat_entry_set *setvec[NAT_VEC_SIZE]; + struct nat_entry_set *set, *tmp; + unsigned int found, entry_count = 0; + nid_t set_idx = 0; + LIST_HEAD(sets); + int err = 0; + + /* + * during unmount, let's flush nat_bits before checking + * nat_cnt[DIRTY_NAT]. + */ + if (enabled_nat_bits(sbi, cpc)) { + f2fs_down_write(&nm_i->nat_tree_lock); + remove_nats_in_journal(sbi); + f2fs_up_write(&nm_i->nat_tree_lock); + } + + if (!nm_i->nat_cnt[DIRTY_NAT]) + return 0; + + f2fs_down_write(&nm_i->nat_tree_lock); + + /* + * if there are no enough space in journal to store dirty nat + * entries, remove all entries from journal and merge them + * into nat entry set. + */ + if (enabled_nat_bits(sbi, cpc) || + !__has_cursum_space(sbi, journal, + nm_i->nat_cnt[DIRTY_NAT], NAT_JOURNAL)) + remove_nats_in_journal(sbi); + + while ((found = __gang_lookup_nat_set(nm_i, + set_idx, NAT_VEC_SIZE, setvec))) { + unsigned idx; + + set_idx = setvec[found - 1]->set + 1; + for (idx = 0; idx < found; idx++) + __adjust_nat_entry_set(setvec[idx], &sets, + MAX_NAT_JENTRIES(sbi, journal)); + } + + /* + * Readahead the current NAT block to prevent read requests from + * being issued and waited on one by one. + */ + list_for_each_entry(set, &sets, set_list) { + entry_count += set->entry_cnt; + if (!enabled_nat_bits(sbi, cpc) && + __has_cursum_space(sbi, journal, + entry_count, NAT_JOURNAL)) + continue; + f2fs_ra_meta_pages(sbi, set->set, 1, META_NAT, true); + } + /* flush dirty nats in nat entry set */ + list_for_each_entry_safe(set, tmp, &sets, set_list) { + err = __flush_nat_entry_set(sbi, set, cpc); + if (err) + break; + } + + f2fs_up_write(&nm_i->nat_tree_lock); + /* Allow dirty nats by node block allocation in write_begin */ + + return err; +} + +static int __get_nat_bitmaps(struct f2fs_sb_info *sbi) +{ + struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); + struct f2fs_nm_info *nm_i = NM_I(sbi); + unsigned int nat_bits_bytes = nm_i->nat_blocks / BITS_PER_BYTE; + unsigned int i; + __u64 cp_ver = cur_cp_version(ckpt); + block_t nat_bits_addr; + + if (!enabled_nat_bits(sbi, NULL)) + return 0; + + nm_i->nat_bits_blocks = F2FS_BLK_ALIGN((nat_bits_bytes << 1) + 8); + nm_i->nat_bits = f2fs_kvzalloc(sbi, + F2FS_BLK_TO_BYTES(nm_i->nat_bits_blocks), GFP_KERNEL); + if (!nm_i->nat_bits) + return -ENOMEM; + + nat_bits_addr = __start_cp_addr(sbi) + BLKS_PER_SEG(sbi) - + nm_i->nat_bits_blocks; + for (i = 0; i < nm_i->nat_bits_blocks; i++) { + struct folio *folio; + + folio = f2fs_get_meta_folio(sbi, nat_bits_addr++); + if (IS_ERR(folio)) + return PTR_ERR(folio); + + memcpy(nm_i->nat_bits + F2FS_BLK_TO_BYTES(i), + folio_address(folio), F2FS_BLKSIZE); + f2fs_folio_put(folio, true); + } + + cp_ver |= (cur_cp_crc(ckpt) << 32); + if (cpu_to_le64(cp_ver) != *(__le64 *)nm_i->nat_bits) { + disable_nat_bits(sbi, true); + return 0; + } + + nm_i->full_nat_bits = nm_i->nat_bits + 8; + nm_i->empty_nat_bits = nm_i->full_nat_bits + nat_bits_bytes; + + f2fs_notice(sbi, "Found nat_bits in checkpoint"); + return 0; +} + +static inline void load_free_nid_bitmap(struct f2fs_sb_info *sbi) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + unsigned int i = 0; + nid_t nid, last_nid; + + if (!enabled_nat_bits(sbi, NULL)) + return; + + for (i = 0; i < nm_i->nat_blocks; i++) { + i = find_next_bit_le(nm_i->empty_nat_bits, nm_i->nat_blocks, i); + if (i >= nm_i->nat_blocks) + break; + + __set_bit_le(i, nm_i->nat_block_bitmap); + + nid = i * NAT_ENTRY_PER_BLOCK; + last_nid = nid + NAT_ENTRY_PER_BLOCK; + + spin_lock(&NM_I(sbi)->nid_list_lock); + for (; nid < last_nid; nid++) + update_free_nid_bitmap(sbi, nid, true, true); + spin_unlock(&NM_I(sbi)->nid_list_lock); + } + + for (i = 0; i < nm_i->nat_blocks; i++) { + i = find_next_bit_le(nm_i->full_nat_bits, nm_i->nat_blocks, i); + if (i >= nm_i->nat_blocks) + break; + + __set_bit_le(i, nm_i->nat_block_bitmap); + } +} + +static int init_node_manager(struct f2fs_sb_info *sbi) +{ + struct f2fs_super_block *sb_raw = F2FS_RAW_SUPER(sbi); + struct f2fs_nm_info *nm_i = NM_I(sbi); + unsigned char *version_bitmap; + unsigned int nat_segs; + int err; + + nm_i->nat_blkaddr = le32_to_cpu(sb_raw->nat_blkaddr); + + /* segment_count_nat includes pair segment so divide to 2. */ + nat_segs = le32_to_cpu(sb_raw->segment_count_nat) >> 1; + nm_i->nat_blocks = nat_segs << le32_to_cpu(sb_raw->log_blocks_per_seg); + nm_i->max_nid = NAT_ENTRY_PER_BLOCK * nm_i->nat_blocks; + + /* not used nids: 0, node, meta, (and root counted as valid node) */ + nm_i->available_nids = nm_i->max_nid - sbi->total_valid_node_count - + F2FS_RESERVED_NODE_NUM; + nm_i->nid_cnt[FREE_NID] = 0; + nm_i->nid_cnt[PREALLOC_NID] = 0; + nm_i->ram_thresh = DEF_RAM_THRESHOLD; + nm_i->ra_nid_pages = DEF_RA_NID_PAGES; + nm_i->dirty_nats_ratio = DEF_DIRTY_NAT_RATIO_THRESHOLD; + nm_i->max_rf_node_blocks = DEF_RF_NODE_BLOCKS; + + INIT_RADIX_TREE(&nm_i->free_nid_root, GFP_ATOMIC); + INIT_LIST_HEAD(&nm_i->free_nid_list); + INIT_RADIX_TREE(&nm_i->nat_root, GFP_NOIO); + INIT_RADIX_TREE(&nm_i->nat_set_root, GFP_NOIO); + INIT_LIST_HEAD(&nm_i->nat_entries); + spin_lock_init(&nm_i->nat_list_lock); + + mutex_init(&nm_i->build_lock); + spin_lock_init(&nm_i->nid_list_lock); + init_f2fs_rwsem(&nm_i->nat_tree_lock); + + nm_i->next_scan_nid = le32_to_cpu(sbi->ckpt->next_free_nid); + nm_i->bitmap_size = __bitmap_size(sbi, NAT_BITMAP); + version_bitmap = __bitmap_ptr(sbi, NAT_BITMAP); + nm_i->nat_bitmap = kmemdup(version_bitmap, nm_i->bitmap_size, + GFP_KERNEL); + if (!nm_i->nat_bitmap) + return -ENOMEM; + + if (!test_opt(sbi, NAT_BITS)) + disable_nat_bits(sbi, true); + + err = __get_nat_bitmaps(sbi); + if (err) + return err; + +#ifdef CONFIG_F2FS_CHECK_FS + nm_i->nat_bitmap_mir = kmemdup(version_bitmap, nm_i->bitmap_size, + GFP_KERNEL); + if (!nm_i->nat_bitmap_mir) + return -ENOMEM; +#endif + + return 0; +} + +static int init_free_nid_cache(struct f2fs_sb_info *sbi) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + int i; + + nm_i->free_nid_bitmap = + f2fs_kvzalloc(sbi, array_size(sizeof(unsigned char *), + nm_i->nat_blocks), + GFP_KERNEL); + if (!nm_i->free_nid_bitmap) + return -ENOMEM; + + for (i = 0; i < nm_i->nat_blocks; i++) { + nm_i->free_nid_bitmap[i] = f2fs_kvzalloc(sbi, + f2fs_bitmap_size(NAT_ENTRY_PER_BLOCK), GFP_KERNEL); + if (!nm_i->free_nid_bitmap[i]) + return -ENOMEM; + } + + nm_i->nat_block_bitmap = f2fs_kvzalloc(sbi, nm_i->nat_blocks / 8, + GFP_KERNEL); + if (!nm_i->nat_block_bitmap) + return -ENOMEM; + + nm_i->free_nid_count = + f2fs_kvzalloc(sbi, array_size(sizeof(unsigned short), + nm_i->nat_blocks), + GFP_KERNEL); + if (!nm_i->free_nid_count) + return -ENOMEM; + return 0; +} + +int f2fs_build_node_manager(struct f2fs_sb_info *sbi) +{ + int err; + + sbi->nm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_nm_info), + GFP_KERNEL); + if (!sbi->nm_info) + return -ENOMEM; + + err = init_node_manager(sbi); + if (err) + return err; + + err = init_free_nid_cache(sbi); + if (err) + return err; + + /* load free nid status from nat_bits table */ + load_free_nid_bitmap(sbi); + + return f2fs_build_free_nids(sbi, true, true); +} + +void f2fs_destroy_node_manager(struct f2fs_sb_info *sbi) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct free_nid *i, *next_i; + void *vec[NAT_VEC_SIZE]; + struct nat_entry **natvec = (struct nat_entry **)vec; + struct nat_entry_set **setvec = (struct nat_entry_set **)vec; + nid_t nid = 0; + unsigned int found; + + if (!nm_i) + return; + + /* destroy free nid list */ + spin_lock(&nm_i->nid_list_lock); + list_for_each_entry_safe(i, next_i, &nm_i->free_nid_list, list) { + __remove_free_nid(sbi, i, FREE_NID); + spin_unlock(&nm_i->nid_list_lock); + kmem_cache_free(free_nid_slab, i); + spin_lock(&nm_i->nid_list_lock); + } + f2fs_bug_on(sbi, nm_i->nid_cnt[FREE_NID]); + f2fs_bug_on(sbi, nm_i->nid_cnt[PREALLOC_NID]); + f2fs_bug_on(sbi, !list_empty(&nm_i->free_nid_list)); + spin_unlock(&nm_i->nid_list_lock); + + /* destroy nat cache */ + f2fs_down_write(&nm_i->nat_tree_lock); + while ((found = __gang_lookup_nat_cache(nm_i, + nid, NAT_VEC_SIZE, natvec))) { + unsigned idx; + + nid = nat_get_nid(natvec[found - 1]) + 1; + for (idx = 0; idx < found; idx++) { + spin_lock(&nm_i->nat_list_lock); + list_del(&natvec[idx]->list); + spin_unlock(&nm_i->nat_list_lock); + + __del_from_nat_cache(nm_i, natvec[idx]); + } + } + f2fs_bug_on(sbi, nm_i->nat_cnt[TOTAL_NAT]); + + /* destroy nat set cache */ + nid = 0; + memset(vec, 0, sizeof(void *) * NAT_VEC_SIZE); + while ((found = __gang_lookup_nat_set(nm_i, + nid, NAT_VEC_SIZE, setvec))) { + unsigned idx; + + nid = setvec[found - 1]->set + 1; + for (idx = 0; idx < found; idx++) { + /* entry_cnt is not zero, when cp_error was occurred */ + f2fs_bug_on(sbi, !list_empty(&setvec[idx]->entry_list)); + radix_tree_delete(&nm_i->nat_set_root, setvec[idx]->set); + kmem_cache_free(nat_entry_set_slab, setvec[idx]); + } + } + f2fs_up_write(&nm_i->nat_tree_lock); + + kvfree(nm_i->nat_block_bitmap); + if (nm_i->free_nid_bitmap) { + int i; + + for (i = 0; i < nm_i->nat_blocks; i++) + kvfree(nm_i->free_nid_bitmap[i]); + kvfree(nm_i->free_nid_bitmap); + } + kvfree(nm_i->free_nid_count); + + kfree(nm_i->nat_bitmap); + kvfree(nm_i->nat_bits); +#ifdef CONFIG_F2FS_CHECK_FS + kfree(nm_i->nat_bitmap_mir); +#endif + sbi->nm_info = NULL; + kfree(nm_i); +} + +int __init f2fs_create_node_manager_caches(void) +{ + nat_entry_slab = f2fs_kmem_cache_create("f2fs_nat_entry", + sizeof(struct nat_entry)); + if (!nat_entry_slab) + goto fail; + + free_nid_slab = f2fs_kmem_cache_create("f2fs_free_nid", + sizeof(struct free_nid)); + if (!free_nid_slab) + goto destroy_nat_entry; + + nat_entry_set_slab = f2fs_kmem_cache_create("f2fs_nat_entry_set", + sizeof(struct nat_entry_set)); + if (!nat_entry_set_slab) + goto destroy_free_nid; + + fsync_node_entry_slab = f2fs_kmem_cache_create("f2fs_fsync_node_entry", + sizeof(struct fsync_node_entry)); + if (!fsync_node_entry_slab) + goto destroy_nat_entry_set; + return 0; + +destroy_nat_entry_set: + kmem_cache_destroy(nat_entry_set_slab); +destroy_free_nid: + kmem_cache_destroy(free_nid_slab); +destroy_nat_entry: + kmem_cache_destroy(nat_entry_slab); +fail: + return -ENOMEM; +} + +void f2fs_destroy_node_manager_caches(void) +{ + kmem_cache_destroy(fsync_node_entry_slab); + kmem_cache_destroy(nat_entry_set_slab); + kmem_cache_destroy(free_nid_slab); + kmem_cache_destroy(nat_entry_slab); +} diff --git a/fs/f2fs/node.h b/fs/f2fs/node.h new file mode 100644 index 000000000..5e114f352 --- /dev/null +++ b/fs/f2fs/node.h @@ -0,0 +1,433 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * fs/f2fs/node.h + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + */ +/* start node id of a node block dedicated to the given node id */ +#define START_NID(nid) (((nid) / NAT_ENTRY_PER_BLOCK) * NAT_ENTRY_PER_BLOCK) + +/* node block offset on the NAT area dedicated to the given start node id */ +#define NAT_BLOCK_OFFSET(start_nid) ((start_nid) / NAT_ENTRY_PER_BLOCK) + +/* # of pages to perform synchronous readahead before building free nids */ +#define FREE_NID_PAGES 8 +#define MAX_FREE_NIDS (NAT_ENTRY_PER_BLOCK * FREE_NID_PAGES) + +/* size of free nid batch when shrinking */ +#define SHRINK_NID_BATCH_SIZE 8 + +#define DEF_RA_NID_PAGES 0 /* # of nid pages to be readaheaded */ + +/* maximum readahead size for node during getting data blocks */ +#define MAX_RA_NODE 128 + +/* control the memory footprint threshold (10MB per 1GB ram) */ +#define DEF_RAM_THRESHOLD 1 + +/* control dirty nats ratio threshold (default: 10% over max nid count) */ +#define DEF_DIRTY_NAT_RATIO_THRESHOLD 10 +/* control total # of nats */ +#define DEF_NAT_CACHE_THRESHOLD 100000 + +/* control total # of node writes used for roll-forward recovery */ +#define DEF_RF_NODE_BLOCKS 0 + +/* vector size for gang look-up from nat cache that consists of radix tree */ +#define NAT_VEC_SIZE 32 + +/* return value for read_node_page */ +#define LOCKED_PAGE 1 + +/* check pinned file's alignment status of physical blocks */ +#define FILE_NOT_ALIGNED 1 + +/* For flag in struct node_info */ +enum { + IS_CHECKPOINTED, /* is it checkpointed before? */ + HAS_FSYNCED_INODE, /* is the inode fsynced before? */ + HAS_LAST_FSYNC, /* has the latest node fsync mark? */ + IS_DIRTY, /* this nat entry is dirty? */ + IS_PREALLOC, /* nat entry is preallocated */ +}; + +/* + * For node information + */ +struct node_info { + nid_t nid; /* node id */ + nid_t ino; /* inode number of the node's owner */ + block_t blk_addr; /* block address of the node */ + unsigned char version; /* version of the node */ + unsigned char flag; /* for node information bits */ +}; + +struct nat_entry { + struct list_head list; /* for clean or dirty nat list */ + struct node_info ni; /* in-memory node information */ +}; + +#define nat_get_nid(nat) ((nat)->ni.nid) +#define nat_set_nid(nat, n) ((nat)->ni.nid = (n)) +#define nat_get_blkaddr(nat) ((nat)->ni.blk_addr) +#define nat_set_blkaddr(nat, b) ((nat)->ni.blk_addr = (b)) +#define nat_get_ino(nat) ((nat)->ni.ino) +#define nat_set_ino(nat, i) ((nat)->ni.ino = (i)) +#define nat_get_version(nat) ((nat)->ni.version) +#define nat_set_version(nat, v) ((nat)->ni.version = (v)) + +#define inc_node_version(version) (++(version)) + +static inline void copy_node_info(struct node_info *dst, + struct node_info *src) +{ + dst->nid = src->nid; + dst->ino = src->ino; + dst->blk_addr = src->blk_addr; + dst->version = src->version; + /* should not copy flag here */ +} + +static inline void set_nat_flag(struct nat_entry *ne, + unsigned int type, bool set) +{ + if (set) + ne->ni.flag |= BIT(type); + else + ne->ni.flag &= ~BIT(type); +} + +static inline bool get_nat_flag(struct nat_entry *ne, unsigned int type) +{ + return ne->ni.flag & BIT(type); +} + +static inline void nat_reset_flag(struct nat_entry *ne) +{ + /* these states can be set only after checkpoint was done */ + set_nat_flag(ne, IS_CHECKPOINTED, true); + set_nat_flag(ne, HAS_FSYNCED_INODE, false); + set_nat_flag(ne, HAS_LAST_FSYNC, true); +} + +static inline void node_info_from_raw_nat(struct node_info *ni, + struct f2fs_nat_entry *raw_ne) +{ + ni->ino = le32_to_cpu(raw_ne->ino); + ni->blk_addr = le32_to_cpu(raw_ne->block_addr); + ni->version = raw_ne->version; +} + +static inline void raw_nat_from_node_info(struct f2fs_nat_entry *raw_ne, + struct node_info *ni) +{ + raw_ne->ino = cpu_to_le32(ni->ino); + raw_ne->block_addr = cpu_to_le32(ni->blk_addr); + raw_ne->version = ni->version; +} + +static inline bool excess_dirty_nats(struct f2fs_sb_info *sbi) +{ + /* nat_cnt[] is heuristic accounting sampled locklessly here. */ + return data_race(READ_ONCE(NM_I(sbi)->nat_cnt[DIRTY_NAT])) >= + NM_I(sbi)->max_nid * + NM_I(sbi)->dirty_nats_ratio / 100; +} + +static inline bool excess_cached_nats(struct f2fs_sb_info *sbi) +{ + /* nat_cnt[] is heuristic accounting sampled locklessly here. */ + return data_race(READ_ONCE(NM_I(sbi)->nat_cnt[TOTAL_NAT])) >= + DEF_NAT_CACHE_THRESHOLD; +} + +enum mem_type { + FREE_NIDS, /* indicates the free nid list */ + NAT_ENTRIES, /* indicates the cached nat entry */ + DIRTY_DENTS, /* indicates dirty dentry pages */ + INO_ENTRIES, /* indicates inode entries */ + READ_EXTENT_CACHE, /* indicates read extent cache */ + AGE_EXTENT_CACHE, /* indicates age extent cache */ + DISCARD_CACHE, /* indicates memory of cached discard cmds */ + COMPRESS_PAGE, /* indicates memory of cached compressed pages */ + BASE_CHECK, /* check kernel status */ +}; + +struct nat_entry_set { + struct list_head set_list; /* link with other nat sets */ + struct list_head entry_list; /* link with dirty nat entries */ + nid_t set; /* set number*/ + unsigned int entry_cnt; /* the # of nat entries in set */ +}; + +struct free_nid { + struct list_head list; /* for free node id list */ + nid_t nid; /* node id */ + int state; /* in use or not: FREE_NID or PREALLOC_NID */ +}; + +static inline void next_free_nid(struct f2fs_sb_info *sbi, nid_t *nid) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + struct free_nid *fnid; + + spin_lock(&nm_i->nid_list_lock); + if (nm_i->nid_cnt[FREE_NID] <= 0) { + spin_unlock(&nm_i->nid_list_lock); + return; + } + fnid = list_first_entry(&nm_i->free_nid_list, struct free_nid, list); + *nid = fnid->nid; + spin_unlock(&nm_i->nid_list_lock); +} + +/* + * inline functions + */ +static inline void get_nat_bitmap(struct f2fs_sb_info *sbi, void *addr) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + +#ifdef CONFIG_F2FS_CHECK_FS + if (memcmp(nm_i->nat_bitmap, nm_i->nat_bitmap_mir, + nm_i->bitmap_size)) + f2fs_bug_on(sbi, 1); +#endif + memcpy(addr, nm_i->nat_bitmap, nm_i->bitmap_size); +} + +static inline pgoff_t current_nat_addr(struct f2fs_sb_info *sbi, nid_t start) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + pgoff_t block_off; + pgoff_t block_addr; + + /* + * block_off = segment_off * 512 + off_in_segment + * OLD = (segment_off * 512) * 2 + off_in_segment + * NEW = 2 * (segment_off * 512 + off_in_segment) - off_in_segment + */ + block_off = NAT_BLOCK_OFFSET(start); + + block_addr = (pgoff_t)(nm_i->nat_blkaddr + + (block_off << 1) - + (block_off & (BLKS_PER_SEG(sbi) - 1))); + + if (f2fs_test_bit(block_off, nm_i->nat_bitmap)) + block_addr += BLKS_PER_SEG(sbi); + + return block_addr; +} + +static inline pgoff_t next_nat_addr(struct f2fs_sb_info *sbi, + pgoff_t block_addr) +{ + struct f2fs_nm_info *nm_i = NM_I(sbi); + + block_addr -= nm_i->nat_blkaddr; + block_addr ^= BIT(sbi->log_blocks_per_seg); + return block_addr + nm_i->nat_blkaddr; +} + +static inline void set_to_next_nat(struct f2fs_nm_info *nm_i, nid_t start_nid) +{ + unsigned int block_off = NAT_BLOCK_OFFSET(start_nid); + + f2fs_change_bit(block_off, nm_i->nat_bitmap); +#ifdef CONFIG_F2FS_CHECK_FS + f2fs_change_bit(block_off, nm_i->nat_bitmap_mir); +#endif +} + +static inline nid_t ino_of_node(const struct folio *node_folio) +{ + struct f2fs_node *rn = F2FS_NODE(node_folio); + return le32_to_cpu(rn->footer.ino); +} + +static inline nid_t nid_of_node(const struct folio *node_folio) +{ + struct f2fs_node *rn = F2FS_NODE(node_folio); + return le32_to_cpu(rn->footer.nid); +} + +static inline unsigned int ofs_of_node(const struct folio *node_folio) +{ + struct f2fs_node *rn = F2FS_NODE(node_folio); + unsigned flag = le32_to_cpu(rn->footer.flag); + return flag >> OFFSET_BIT_SHIFT; +} + +static inline __u64 cpver_of_node(const struct folio *node_folio) +{ + struct f2fs_node *rn = F2FS_NODE(node_folio); + return le64_to_cpu(rn->footer.cp_ver); +} + +static inline block_t next_blkaddr_of_node(const struct folio *node_folio) +{ + struct f2fs_node *rn = F2FS_NODE(node_folio); + return le32_to_cpu(rn->footer.next_blkaddr); +} + +static inline void fill_node_footer(const struct folio *folio, nid_t nid, + nid_t ino, unsigned int ofs, bool reset) +{ + struct f2fs_node *rn = F2FS_NODE(folio); + unsigned int old_flag = 0; + + if (reset) + memset(rn, 0, sizeof(*rn)); + else + old_flag = le32_to_cpu(rn->footer.flag); + + rn->footer.nid = cpu_to_le32(nid); + rn->footer.ino = cpu_to_le32(ino); + + /* should remain old flag bits such as COLD_BIT_SHIFT */ + rn->footer.flag = cpu_to_le32((ofs << OFFSET_BIT_SHIFT) | + (old_flag & OFFSET_BIT_MASK)); +} + +static inline void copy_node_footer(const struct folio *dst, + const struct folio *src) +{ + struct f2fs_node *src_rn = F2FS_NODE(src); + struct f2fs_node *dst_rn = F2FS_NODE(dst); + memcpy(&dst_rn->footer, &src_rn->footer, sizeof(struct node_footer)); +} + +static inline void fill_node_footer_blkaddr(struct folio *folio, block_t blkaddr) +{ + struct f2fs_checkpoint *ckpt = F2FS_CKPT(F2FS_F_SB(folio)); + struct f2fs_node *rn = F2FS_NODE(folio); + __u64 cp_ver = cur_cp_version(ckpt); + + if (__is_set_ckpt_flags(ckpt, CP_CRC_RECOVERY_FLAG)) + cp_ver |= (cur_cp_crc(ckpt) << 32); + + rn->footer.cp_ver = cpu_to_le64(cp_ver); + rn->footer.next_blkaddr = cpu_to_le32(blkaddr); +} + +static inline bool is_recoverable_dnode(const struct folio *folio) +{ + struct f2fs_checkpoint *ckpt = F2FS_CKPT(F2FS_F_SB(folio)); + __u64 cp_ver = cur_cp_version(ckpt); + + /* Don't care crc part, if fsck.f2fs sets it. */ + if (__is_set_ckpt_flags(ckpt, CP_NOCRC_RECOVERY_FLAG)) + return (cp_ver << 32) == (cpver_of_node(folio) << 32); + + if (__is_set_ckpt_flags(ckpt, CP_CRC_RECOVERY_FLAG)) + cp_ver |= (cur_cp_crc(ckpt) << 32); + + return cp_ver == cpver_of_node(folio); +} + +/* + * f2fs assigns the following node offsets described as (num). + * N = NIDS_PER_BLOCK + * + * Inode block (0) + * |- direct node (1) + * |- direct node (2) + * |- indirect node (3) + * | `- direct node (4 => 4 + N - 1) + * |- indirect node (4 + N) + * | `- direct node (5 + N => 5 + 2N - 1) + * `- double indirect node (5 + 2N) + * `- indirect node (6 + 2N) + * `- direct node + * ...... + * `- indirect node ((6 + 2N) + x(N + 1)) + * `- direct node + * ...... + * `- indirect node ((6 + 2N) + (N - 1)(N + 1)) + * `- direct node + */ +static inline bool IS_DNODE(const struct folio *node_folio) +{ + unsigned int ofs = ofs_of_node(node_folio); + + if (f2fs_has_xattr_block(ofs)) + return true; + + if (ofs == 3 || ofs == 4 + NIDS_PER_BLOCK || + ofs == 5 + 2 * NIDS_PER_BLOCK) + return false; + if (ofs >= 6 + 2 * NIDS_PER_BLOCK) { + ofs -= 6 + 2 * NIDS_PER_BLOCK; + if (!((long int)ofs % (NIDS_PER_BLOCK + 1))) + return false; + } + return true; +} + +static inline int set_nid(struct folio *folio, int off, nid_t nid, bool i) +{ + struct f2fs_node *rn = F2FS_NODE(folio); + + f2fs_folio_wait_writeback(folio, NODE, true, true); + + if (i) + rn->i.i_nid[off - NODE_DIR1_BLOCK] = cpu_to_le32(nid); + else + rn->in.nid[off] = cpu_to_le32(nid); + return folio_mark_dirty(folio); +} + +static inline nid_t get_nid(const struct folio *folio, int off, bool i) +{ + struct f2fs_node *rn = F2FS_NODE(folio); + + if (i) + return le32_to_cpu(rn->i.i_nid[off - NODE_DIR1_BLOCK]); + return le32_to_cpu(rn->in.nid[off]); +} + +/* + * Coldness identification: + * - Mark cold files in f2fs_inode_info + * - Mark cold node blocks in their node footer + * - Mark cold data pages in page cache + */ + +static inline int is_node(const struct folio *folio, int type) +{ + struct f2fs_node *rn = F2FS_NODE(folio); + return le32_to_cpu(rn->footer.flag) & BIT(type); +} + +#define is_cold_node(folio) is_node(folio, COLD_BIT_SHIFT) +#define is_fsync_dnode(folio) is_node(folio, FSYNC_BIT_SHIFT) +#define is_dent_dnode(folio) is_node(folio, DENT_BIT_SHIFT) + +static inline void __set_mark(const struct folio *folio, bool mark, int type) +{ + struct f2fs_node *rn = F2FS_NODE(folio); + unsigned int flag = le32_to_cpu(rn->footer.flag); + + if (mark) + flag |= BIT(type); + else + flag &= ~BIT(type); + rn->footer.flag = cpu_to_le32(flag); +} + +static inline void set_cold_node(const struct folio *folio, bool is_dir) +{ + __set_mark(folio, !is_dir, COLD_BIT_SHIFT); +} + +static inline void set_mark(struct folio *folio, bool mark, int type) +{ + __set_mark(folio, mark, type); + +#ifdef CONFIG_F2FS_CHECK_FS + f2fs_inode_chksum_set(F2FS_F_SB(folio), folio); +#endif +} +#define set_dentry_mark(folio, mark) set_mark(folio, mark, DENT_BIT_SHIFT) +#define set_fsync_mark(folio, mark) set_mark(folio, mark, FSYNC_BIT_SHIFT) diff --git a/fs/f2fs/recovery.c b/fs/f2fs/recovery.c new file mode 100644 index 000000000..89af8407b --- /dev/null +++ b/fs/f2fs/recovery.c @@ -0,0 +1,970 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/f2fs/recovery.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + */ +#include +#include +#include +#include +#include +#include "f2fs.h" +#include "node.h" +#include "segment.h" + +/* + * Roll forward recovery scenarios. + * + * [Term] F: fsync_mark, D: dentry_mark + * + * 1. inode(x) | CP | inode(x) | dnode(F) + * -> Update the latest inode(x). + * + * 2. inode(x) | CP | inode(F) | dnode(F) + * -> No problem. + * + * 3. inode(x) | CP | dnode(F) | inode(x) + * -> Recover to the latest dnode(F), and drop the last inode(x) + * + * 4. inode(x) | CP | dnode(F) | inode(F) + * -> No problem. + * + * 5. CP | inode(x) | dnode(F) + * -> The inode(DF) was missing. Should drop this dnode(F). + * + * 6. CP | inode(DF) | dnode(F) + * -> No problem. + * + * 7. CP | dnode(F) | inode(DF) + * -> If f2fs_iget fails, then goto next to find inode(DF). + * + * 8. CP | dnode(F) | inode(x) + * -> If f2fs_iget fails, then goto next to find inode(DF). + * But it will fail due to no inode(DF). + */ + +static struct kmem_cache *fsync_entry_slab; + +bool f2fs_space_for_roll_forward(struct f2fs_sb_info *sbi) +{ + s64 nalloc = percpu_counter_sum_positive(&sbi->alloc_valid_block_count); + + if (sbi->last_valid_block_count + nalloc > sbi->user_block_count) + return false; + if (NM_I(sbi)->max_rf_node_blocks && + percpu_counter_sum_positive(&sbi->rf_node_block_count) >= + NM_I(sbi)->max_rf_node_blocks) + return false; + return true; +} + +static struct fsync_inode_entry *get_fsync_inode(struct list_head *head, + nid_t ino) +{ + struct fsync_inode_entry *entry; + + list_for_each_entry(entry, head, list) + if (entry->inode->i_ino == ino) + return entry; + + return NULL; +} + +static struct fsync_inode_entry *add_fsync_inode(struct f2fs_sb_info *sbi, + struct list_head *head, nid_t ino, bool quota_inode) +{ + struct inode *inode; + struct fsync_inode_entry *entry; + int err; + + inode = f2fs_iget_retry(sbi->sb, ino); + if (IS_ERR(inode)) + return ERR_CAST(inode); + + err = f2fs_dquot_initialize(inode); + if (err) + goto err_out; + + if (quota_inode) { + err = dquot_alloc_inode(inode); + if (err) + goto err_out; + } + + entry = f2fs_kmem_cache_alloc(fsync_entry_slab, + GFP_F2FS_ZERO, true, NULL); + entry->inode = inode; + list_add_tail(&entry->list, head); + + return entry; +err_out: + iput(inode); + return ERR_PTR(err); +} + +static void del_fsync_inode(struct fsync_inode_entry *entry, int drop) +{ + if (drop) { + /* inode should not be recovered, drop it */ + f2fs_inode_synced(entry->inode); + } + iput(entry->inode); + list_del(&entry->list); + kmem_cache_free(fsync_entry_slab, entry); +} + +static int init_recovered_filename(const struct inode *dir, + struct f2fs_inode *raw_inode, + struct f2fs_filename *fname, + struct qstr *usr_fname) +{ + int err; + + memset(fname, 0, sizeof(*fname)); + fname->disk_name.len = le32_to_cpu(raw_inode->i_namelen); + fname->disk_name.name = raw_inode->i_name; + + if (WARN_ON(fname->disk_name.len > F2FS_NAME_LEN)) + return -ENAMETOOLONG; + + if (!IS_ENCRYPTED(dir)) { + usr_fname->name = fname->disk_name.name; + usr_fname->len = fname->disk_name.len; + fname->usr_fname = usr_fname; + } + + /* Compute the hash of the filename */ + if (IS_ENCRYPTED(dir) && IS_CASEFOLDED(dir)) { + /* + * In this case the hash isn't computable without the key, so it + * was saved on-disk. + */ + if (fname->disk_name.len + sizeof(f2fs_hash_t) > F2FS_NAME_LEN) + return -EINVAL; + fname->hash = get_unaligned((f2fs_hash_t *) + &raw_inode->i_name[fname->disk_name.len]); + } else if (IS_CASEFOLDED(dir)) { + err = f2fs_init_casefolded_name(dir, fname); + if (err) + return err; + f2fs_hash_filename(dir, fname); + /* Case-sensitive match is fine for recovery */ + f2fs_free_casefolded_name(fname); + } else { + f2fs_hash_filename(dir, fname); + } + return 0; +} + +static int recover_dentry(struct inode *inode, struct folio *ifolio, + struct list_head *dir_list) +{ + struct f2fs_inode *raw_inode = F2FS_INODE(ifolio); + nid_t pino = le32_to_cpu(raw_inode->i_pino); + struct f2fs_dir_entry *de; + struct f2fs_filename fname; + struct qstr usr_fname; + struct folio *folio; + struct inode *dir, *einode; + struct fsync_inode_entry *entry; + int err = 0; + char *name; + + entry = get_fsync_inode(dir_list, pino); + if (!entry) { + entry = add_fsync_inode(F2FS_I_SB(inode), dir_list, + pino, false); + if (IS_ERR(entry)) { + dir = ERR_CAST(entry); + err = PTR_ERR(entry); + goto out; + } + } + + dir = entry->inode; + err = init_recovered_filename(dir, raw_inode, &fname, &usr_fname); + if (err) + goto out; +retry: + de = __f2fs_find_entry(dir, &fname, &folio); + if (de && inode->i_ino == le32_to_cpu(de->ino)) + goto out_put; + + if (de) { + einode = f2fs_iget_retry(inode->i_sb, le32_to_cpu(de->ino)); + if (IS_ERR(einode)) { + WARN_ON(1); + err = PTR_ERR(einode); + if (err == -ENOENT) + err = -EEXIST; + goto out_put; + } + + err = f2fs_dquot_initialize(einode); + if (err) { + iput(einode); + goto out_put; + } + + err = f2fs_acquire_orphan_inode(F2FS_I_SB(inode)); + if (err) { + iput(einode); + goto out_put; + } + f2fs_delete_entry(de, folio, dir, einode); + iput(einode); + goto retry; + } else if (IS_ERR(folio)) { + err = PTR_ERR(folio); + } else { + err = f2fs_add_dentry(dir, &fname, inode, + inode->i_ino, inode->i_mode); + } + if (err == -ENOMEM) + goto retry; + goto out; + +out_put: + f2fs_folio_put(folio, false); +out: + if (file_enc_name(inode)) + name = ""; + else + name = raw_inode->i_name; + f2fs_notice(F2FS_I_SB(inode), "%s: ino = %x, name = %s, dir = %llu, err = %d", + __func__, ino_of_node(ifolio), name, + IS_ERR(dir) ? 0 : dir->i_ino, err); + return err; +} + +static int recover_quota_data(struct inode *inode, struct folio *folio) +{ + struct f2fs_inode *raw = F2FS_INODE(folio); + struct iattr attr; + uid_t i_uid = le32_to_cpu(raw->i_uid); + gid_t i_gid = le32_to_cpu(raw->i_gid); + int err; + + memset(&attr, 0, sizeof(attr)); + + attr.ia_vfsuid = VFSUIDT_INIT(make_kuid(inode->i_sb->s_user_ns, i_uid)); + attr.ia_vfsgid = VFSGIDT_INIT(make_kgid(inode->i_sb->s_user_ns, i_gid)); + + if (!vfsuid_eq(attr.ia_vfsuid, i_uid_into_vfsuid(&nop_mnt_idmap, inode))) + attr.ia_valid |= ATTR_UID; + if (!vfsgid_eq(attr.ia_vfsgid, i_gid_into_vfsgid(&nop_mnt_idmap, inode))) + attr.ia_valid |= ATTR_GID; + + if (!attr.ia_valid) + return 0; + + err = dquot_transfer(&nop_mnt_idmap, inode, &attr); + if (err) + set_sbi_flag(F2FS_I_SB(inode), SBI_QUOTA_NEED_REPAIR); + return err; +} + +static void recover_inline_flags(struct inode *inode, struct f2fs_inode *ri) +{ + if (ri->i_inline & F2FS_PIN_FILE) + set_inode_flag(inode, FI_PIN_FILE); + else + clear_inode_flag(inode, FI_PIN_FILE); + if (ri->i_inline & F2FS_DATA_EXIST) + set_inode_flag(inode, FI_DATA_EXIST); + else + clear_inode_flag(inode, FI_DATA_EXIST); +} + +static int recover_inode(struct inode *inode, struct folio *folio) +{ + struct f2fs_inode *raw = F2FS_INODE(folio); + struct f2fs_inode_info *fi = F2FS_I(inode); + char *name; + int err; + + inode->i_mode = le16_to_cpu(raw->i_mode); + + err = recover_quota_data(inode, folio); + if (err) + return err; + + i_uid_write(inode, le32_to_cpu(raw->i_uid)); + i_gid_write(inode, le32_to_cpu(raw->i_gid)); + + if (raw->i_inline & F2FS_EXTRA_ATTR) { + if (f2fs_sb_has_project_quota(F2FS_I_SB(inode)) && + F2FS_FITS_IN_INODE(raw, le16_to_cpu(raw->i_extra_isize), + i_projid)) { + projid_t i_projid; + kprojid_t kprojid; + + i_projid = (projid_t)le32_to_cpu(raw->i_projid); + kprojid = make_kprojid(&init_user_ns, i_projid); + + if (!projid_eq(kprojid, fi->i_projid)) { + err = f2fs_transfer_project_quota(inode, + kprojid); + if (err) + return err; + fi->i_projid = kprojid; + } + } + } + + f2fs_i_size_write(inode, le64_to_cpu(raw->i_size)); + inode_set_atime(inode, le64_to_cpu(raw->i_atime), + le32_to_cpu(raw->i_atime_nsec)); + inode_set_ctime(inode, le64_to_cpu(raw->i_ctime), + le32_to_cpu(raw->i_ctime_nsec)); + inode_set_mtime(inode, le64_to_cpu(raw->i_mtime), + le32_to_cpu(raw->i_mtime_nsec)); + + fi->i_advise = raw->i_advise; + fi->i_flags = le32_to_cpu(raw->i_flags); + f2fs_set_inode_flags(inode); + fi->i_gc_failures = le16_to_cpu(raw->i_gc_failures); + + recover_inline_flags(inode, raw); + + f2fs_mark_inode_dirty_sync(inode, true); + + if (file_enc_name(inode)) + name = ""; + else + name = F2FS_INODE(folio)->i_name; + + f2fs_notice(F2FS_I_SB(inode), "recover_inode: ino = %x, name = %s, inline = %x", + ino_of_node(folio), name, raw->i_inline); + return 0; +} + +static unsigned int adjust_por_ra_blocks(struct f2fs_sb_info *sbi, + unsigned int ra_blocks, unsigned int blkaddr, + unsigned int next_blkaddr) +{ + if (blkaddr + 1 == next_blkaddr) + ra_blocks = min_t(unsigned int, RECOVERY_MAX_RA_BLOCKS, + ra_blocks * 2); + else if (next_blkaddr % BLKS_PER_SEG(sbi)) + ra_blocks = max_t(unsigned int, RECOVERY_MIN_RA_BLOCKS, + ra_blocks / 2); + return ra_blocks; +} + +/* Detect looped node chain with Floyd's cycle detection algorithm. */ +static int sanity_check_node_chain(struct f2fs_sb_info *sbi, block_t blkaddr, + block_t *blkaddr_fast, bool *is_detecting) +{ + unsigned int ra_blocks = RECOVERY_MAX_RA_BLOCKS; + int i; + + if (!*is_detecting) + return 0; + + for (i = 0; i < 2; i++) { + struct folio *folio; + + if (!f2fs_is_valid_blkaddr(sbi, *blkaddr_fast, META_POR)) { + *is_detecting = false; + return 0; + } + + folio = f2fs_get_tmp_folio(sbi, *blkaddr_fast); + if (IS_ERR(folio)) + return PTR_ERR(folio); + + if (!is_recoverable_dnode(folio)) { + f2fs_folio_put(folio, true); + *is_detecting = false; + return 0; + } + + ra_blocks = adjust_por_ra_blocks(sbi, ra_blocks, *blkaddr_fast, + next_blkaddr_of_node(folio)); + + *blkaddr_fast = next_blkaddr_of_node(folio); + f2fs_folio_put(folio, true); + + f2fs_ra_meta_pages_cond(sbi, *blkaddr_fast, ra_blocks); + } + + if (*blkaddr_fast == blkaddr) { + f2fs_notice(sbi, "%s: Detect looped node chain on blkaddr:%u." + " Run fsck to fix it.", __func__, blkaddr); + return -EINVAL; + } + return 0; +} + +static int find_fsync_dnodes(struct f2fs_sb_info *sbi, struct list_head *head, + bool check_only, bool *new_inode) +{ + struct curseg_info *curseg; + block_t blkaddr, blkaddr_fast; + bool is_detecting = true; + int err = 0; + + /* get node pages in the current segment */ + curseg = CURSEG_I(sbi, CURSEG_WARM_NODE); + blkaddr = NEXT_FREE_BLKADDR(sbi, curseg); + blkaddr_fast = blkaddr; + + while (1) { + struct fsync_inode_entry *entry; + struct folio *folio; + + if (!f2fs_is_valid_blkaddr(sbi, blkaddr, META_POR)) + return 0; + + folio = f2fs_get_tmp_folio(sbi, blkaddr); + if (IS_ERR(folio)) { + err = PTR_ERR(folio); + break; + } + + if (!is_recoverable_dnode(folio)) { + f2fs_folio_put(folio, true); + break; + } + + if (!is_fsync_dnode(folio)) + goto next; + + entry = get_fsync_inode(head, ino_of_node(folio)); + if (!entry) { + bool quota_inode = false; + + if (!check_only && + IS_INODE(folio) && + is_dent_dnode(folio)) { + err = f2fs_recover_inode_page(sbi, folio); + if (err) { + f2fs_folio_put(folio, true); + break; + } + quota_inode = true; + } + + entry = add_fsync_inode(sbi, head, ino_of_node(folio), + quota_inode); + if (IS_ERR(entry)) { + err = PTR_ERR(entry); + /* + * CP | dnode(F) | inode(DF) + * For this case, we should not give up now. + */ + if (err == -ENOENT) { + if (check_only) + *new_inode = true; + goto next; + } + f2fs_folio_put(folio, true); + break; + } + } + entry->blkaddr = blkaddr; + + if (IS_INODE(folio) && is_dent_dnode(folio)) + entry->last_dentry = blkaddr; +next: + /* check next segment */ + blkaddr = next_blkaddr_of_node(folio); + f2fs_folio_put(folio, true); + + err = sanity_check_node_chain(sbi, blkaddr, &blkaddr_fast, + &is_detecting); + if (err) + break; + } + return err; +} + +static void destroy_fsync_dnodes(struct list_head *head, int drop) +{ + struct fsync_inode_entry *entry, *tmp; + + list_for_each_entry_safe(entry, tmp, head, list) + del_fsync_inode(entry, drop); +} + +static int check_index_in_prev_nodes(struct f2fs_sb_info *sbi, + block_t blkaddr, struct dnode_of_data *dn) +{ + struct seg_entry *sentry; + unsigned int segno = GET_SEGNO(sbi, blkaddr); + unsigned short blkoff = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); + struct f2fs_summary_block *sum_node; + struct f2fs_summary sum; + struct folio *sum_folio, *node_folio; + struct dnode_of_data tdn = *dn; + nid_t ino, nid; + struct inode *inode; + unsigned int offset, ofs_in_node, max_addrs; + block_t bidx; + int i; + + sentry = get_seg_entry(sbi, segno); + if (!f2fs_test_bit(blkoff, sentry->cur_valid_map)) + return 0; + + /* Get the previous summary */ + for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { + struct curseg_info *curseg = CURSEG_I(sbi, i); + + if (curseg->segno == segno) { + sum = sum_entries(curseg->sum_blk)[blkoff]; + goto got_it; + } + } + + sum_folio = f2fs_get_sum_folio(sbi, segno); + if (IS_ERR(sum_folio)) + return PTR_ERR(sum_folio); + sum_node = SUM_BLK_PAGE_ADDR(sbi, sum_folio, segno); + sum = sum_entries(sum_node)[blkoff]; + f2fs_folio_put(sum_folio, true); +got_it: + /* Use the locked dnode page and inode */ + nid = le32_to_cpu(sum.nid); + ofs_in_node = le16_to_cpu(sum.ofs_in_node); + + max_addrs = ADDRS_PER_PAGE(dn->node_folio, dn->inode); + if (ofs_in_node >= max_addrs) { + f2fs_err(sbi, "Inconsistent ofs_in_node:%u in summary, ino:%llu, nid:%u, max:%u", + ofs_in_node, dn->inode->i_ino, nid, max_addrs); + f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUMMARY); + return -EFSCORRUPTED; + } + + if (dn->inode->i_ino == nid) { + tdn.nid = nid; + if (!dn->inode_folio_locked) + folio_lock(dn->inode_folio); + tdn.node_folio = dn->inode_folio; + tdn.ofs_in_node = ofs_in_node; + goto truncate_out; + } else if (dn->nid == nid) { + tdn.ofs_in_node = ofs_in_node; + goto truncate_out; + } + + /* Get the node page */ + node_folio = f2fs_get_node_folio(sbi, nid, NODE_TYPE_REGULAR); + if (IS_ERR(node_folio)) + return PTR_ERR(node_folio); + + offset = ofs_of_node(node_folio); + ino = ino_of_node(node_folio); + f2fs_folio_put(node_folio, true); + + if (ino != dn->inode->i_ino) { + int ret; + + /* Deallocate previous index in the node page */ + inode = f2fs_iget_retry(sbi->sb, ino); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + ret = f2fs_dquot_initialize(inode); + if (ret) { + iput(inode); + return ret; + } + } else { + inode = dn->inode; + } + + bidx = f2fs_start_bidx_of_node(offset, inode) + + le16_to_cpu(sum.ofs_in_node); + + /* + * if inode page is locked, unlock temporarily, but its reference + * count keeps alive. + */ + if (ino == dn->inode->i_ino && dn->inode_folio_locked) + folio_unlock(dn->inode_folio); + + set_new_dnode(&tdn, inode, NULL, NULL, 0); + if (f2fs_get_dnode_of_data(&tdn, bidx, LOOKUP_NODE)) + goto out; + + if (tdn.data_blkaddr == blkaddr) + f2fs_truncate_data_blocks_range(&tdn, 1); + + f2fs_put_dnode(&tdn); +out: + if (ino != dn->inode->i_ino) + iput(inode); + else if (dn->inode_folio_locked) + folio_lock(dn->inode_folio); + return 0; + +truncate_out: + if (f2fs_data_blkaddr(&tdn) == blkaddr) + f2fs_truncate_data_blocks_range(&tdn, 1); + if (dn->inode->i_ino == nid && !dn->inode_folio_locked) + folio_unlock(dn->inode_folio); + return 0; +} + +static int f2fs_reserve_new_block_retry(struct dnode_of_data *dn) +{ + int i, err = 0; + + for (i = DEFAULT_FAILURE_RETRY_COUNT; i > 0; i--) { + err = f2fs_reserve_new_block(dn); + if (!err) + break; + } + + return err; +} + +static int do_recover_data(struct f2fs_sb_info *sbi, struct inode *inode, + struct folio *folio) +{ + struct dnode_of_data dn; + struct node_info ni; + unsigned int start = 0, end = 0, index; + int err = 0, recovered = 0; + + /* step 1: recover xattr */ + if (IS_INODE(folio)) { + err = f2fs_recover_inline_xattr(inode, folio); + if (err) + goto out; + } else if (f2fs_has_xattr_block(ofs_of_node(folio))) { + err = f2fs_recover_xattr_data(inode, folio); + if (!err) + recovered++; + goto out; + } + + /* step 2: recover inline data */ + err = f2fs_recover_inline_data(inode, folio); + if (err) { + if (err == 1) + err = 0; + goto out; + } + + /* step 3: recover data indices */ + start = f2fs_start_bidx_of_node(ofs_of_node(folio), inode); + end = start + ADDRS_PER_PAGE(folio, inode); + + set_new_dnode(&dn, inode, NULL, NULL, 0); +retry_dn: + err = f2fs_get_dnode_of_data(&dn, start, ALLOC_NODE); + if (err) { + if (err == -ENOMEM) { + memalloc_retry_wait(GFP_NOFS); + goto retry_dn; + } + goto out; + } + + f2fs_folio_wait_writeback(dn.node_folio, NODE, true, true); + + err = f2fs_get_node_info(sbi, dn.nid, &ni, false); + if (err) + goto err; + + f2fs_bug_on(sbi, ni.ino != ino_of_node(folio)); + + if (ofs_of_node(dn.node_folio) != ofs_of_node(folio)) { + f2fs_warn(sbi, "Inconsistent ofs_of_node, ino:%llu, ofs:%u, %u", + inode->i_ino, ofs_of_node(dn.node_folio), + ofs_of_node(folio)); + err = -EFSCORRUPTED; + f2fs_handle_error(sbi, ERROR_INCONSISTENT_FOOTER); + fserror_report_file_metadata(dn.inode, err, GFP_NOFS); + goto err; + } + + for (index = start; index < end; index++, dn.ofs_in_node++) { + block_t src, dest; + + src = f2fs_data_blkaddr(&dn); + dest = data_blkaddr(dn.inode, folio, dn.ofs_in_node); + + if (__is_valid_data_blkaddr(src) && + !f2fs_is_valid_blkaddr(sbi, src, META_POR)) { + err = -EFSCORRUPTED; + goto err; + } + + if (__is_valid_data_blkaddr(dest) && + !f2fs_is_valid_blkaddr(sbi, dest, META_POR)) { + err = -EFSCORRUPTED; + goto err; + } + + /* skip recovering if dest is the same as src */ + if (src == dest) + continue; + + /* dest is invalid, just invalidate src block */ + if (dest == NULL_ADDR) { + f2fs_truncate_data_blocks_range(&dn, 1); + continue; + } + + if (!file_keep_isize(inode) && + (i_size_read(inode) <= ((loff_t)index << PAGE_SHIFT))) + f2fs_i_size_write(inode, + (loff_t)(index + 1) << PAGE_SHIFT); + + /* + * dest is reserved block, invalidate src block + * and then reserve one new block in dnode page. + */ + if (dest == NEW_ADDR) { + f2fs_truncate_data_blocks_range(&dn, 1); + + err = f2fs_reserve_new_block_retry(&dn); + if (err) + goto err; + continue; + } + + /* dest is valid block, try to recover from src to dest */ + if (f2fs_is_valid_blkaddr(sbi, dest, META_POR)) { + if (src == NULL_ADDR) { + err = f2fs_reserve_new_block_retry(&dn); + if (err) + goto err; + } +retry_prev: + /* Check the previous node page having this index */ + err = check_index_in_prev_nodes(sbi, dest, &dn); + if (err) { + if (err == -ENOMEM) { + memalloc_retry_wait(GFP_NOFS); + goto retry_prev; + } + goto err; + } + + if (f2fs_is_valid_blkaddr(sbi, dest, + DATA_GENERIC_ENHANCE_UPDATE)) { + f2fs_err(sbi, "Inconsistent dest blkaddr:%u, ino:%llu, ofs:%u", + dest, inode->i_ino, dn.ofs_in_node); + err = -EFSCORRUPTED; + goto err; + } + + /* write dummy data page */ + f2fs_replace_block(sbi, &dn, src, dest, + ni.version, false, false); + recovered++; + } + } + + copy_node_footer(dn.node_folio, folio); + fill_node_footer(dn.node_folio, dn.nid, ni.ino, + ofs_of_node(folio), false); + folio_mark_dirty(dn.node_folio); +err: + f2fs_put_dnode(&dn); +out: + f2fs_notice(sbi, "recover_data: ino = %llx, nid = %x (i_size: %s), " + "range (%u, %u), recovered = %d, err = %d", + inode->i_ino, nid_of_node(folio), + file_keep_isize(inode) ? "keep" : "recover", + start, end, recovered, err); + return err; +} + +static int recover_data(struct f2fs_sb_info *sbi, struct list_head *inode_list, + struct list_head *tmp_inode_list, struct list_head *dir_list) +{ + struct curseg_info *curseg; + int err = 0; + block_t blkaddr; + unsigned int ra_blocks = RECOVERY_MAX_RA_BLOCKS; + unsigned int recoverable_dnode = 0; + unsigned int fsynced_dnode = 0; + unsigned int total_dnode = 0; + unsigned int recovered_inode = 0; + unsigned int recovered_dentry = 0; + unsigned int recovered_dnode = 0; + + f2fs_notice(sbi, "do_recover_data: start to recover dnode"); + + /* get node pages in the current segment */ + curseg = CURSEG_I(sbi, CURSEG_WARM_NODE); + blkaddr = NEXT_FREE_BLKADDR(sbi, curseg); + + while (1) { + struct fsync_inode_entry *entry; + struct folio *folio; + + if (!f2fs_is_valid_blkaddr(sbi, blkaddr, META_POR)) + break; + + folio = f2fs_get_tmp_folio(sbi, blkaddr); + if (IS_ERR(folio)) { + err = PTR_ERR(folio); + break; + } + + if (!is_recoverable_dnode(folio)) { + f2fs_folio_put(folio, true); + break; + } + recoverable_dnode++; + + entry = get_fsync_inode(inode_list, ino_of_node(folio)); + if (!entry) + goto next; + fsynced_dnode++; + /* + * inode(x) | CP | inode(x) | dnode(F) + * In this case, we can lose the latest inode(x). + * So, call recover_inode for the inode update. + */ + if (IS_INODE(folio)) { + err = recover_inode(entry->inode, folio); + if (err) { + f2fs_folio_put(folio, true); + break; + } + recovered_inode++; + } + if (entry->last_dentry == blkaddr) { + err = recover_dentry(entry->inode, folio, dir_list); + if (err) { + f2fs_folio_put(folio, true); + break; + } + recovered_dentry++; + } + err = do_recover_data(sbi, entry->inode, folio); + if (err) { + f2fs_folio_put(folio, true); + break; + } + recovered_dnode++; + + if (entry->blkaddr == blkaddr) + list_move_tail(&entry->list, tmp_inode_list); +next: + ra_blocks = adjust_por_ra_blocks(sbi, ra_blocks, blkaddr, + next_blkaddr_of_node(folio)); + + /* check next segment */ + blkaddr = next_blkaddr_of_node(folio); + f2fs_folio_put(folio, true); + + f2fs_ra_meta_pages_cond(sbi, blkaddr, ra_blocks); + total_dnode++; + } + if (!err) + err = f2fs_allocate_new_segments(sbi); + + f2fs_notice(sbi, "do_recover_data: dnode: (recoverable: %u, fsynced: %u, " + "total: %u), recovered: (inode: %u, dentry: %u, dnode: %u), err: %d", + recoverable_dnode, fsynced_dnode, total_dnode, recovered_inode, + recovered_dentry, recovered_dnode, err); + return err; +} + +int f2fs_recover_fsync_data(struct f2fs_sb_info *sbi, bool check_only) +{ + LIST_HEAD(inode_list); + LIST_HEAD(tmp_inode_list); + LIST_HEAD(dir_list); + struct f2fs_lock_context lc; + int err; + int ret = 0; + unsigned long s_flags = sbi->sb->s_flags; + bool need_writecp = false; + bool new_inode = false; + + f2fs_notice(sbi, "f2fs_recover_fsync_data: recovery fsync data, " + "check_only: %d", check_only); + + if (is_sbi_flag_set(sbi, SBI_IS_WRITABLE)) + f2fs_info(sbi, "recover fsync data on readonly fs"); + + /* prevent checkpoint */ + f2fs_down_write_trace(&sbi->cp_global_sem, &lc); + + /* step #1: find fsynced inode numbers */ + err = find_fsync_dnodes(sbi, &inode_list, check_only, &new_inode); + if (err < 0 || (list_empty(&inode_list) && (!check_only || !new_inode))) + goto skip; + + if (check_only) { + ret = 1; + goto skip; + } + + need_writecp = true; + + /* step #2: recover data */ + err = recover_data(sbi, &inode_list, &tmp_inode_list, &dir_list); + if (!err) + f2fs_bug_on(sbi, !list_empty(&inode_list)); + else + f2fs_bug_on(sbi, sbi->sb->s_flags & SB_ACTIVE); +skip: + destroy_fsync_dnodes(&inode_list, err); + destroy_fsync_dnodes(&tmp_inode_list, err); + + /* truncate meta pages to be used by the recovery */ + truncate_inode_pages_range(META_MAPPING(sbi), + (loff_t)MAIN_BLKADDR(sbi) << PAGE_SHIFT, -1); + + if (err) { + truncate_inode_pages_final(NODE_MAPPING(sbi)); + truncate_inode_pages_final(META_MAPPING(sbi)); + } + + /* + * If fsync data succeeds or there is no fsync data to recover, + * and the f2fs is not read only, check and fix zoned block devices' + * write pointer consistency. + */ + if (!err) + err = f2fs_check_and_fix_write_pointer(sbi); + + if (!err) + clear_sbi_flag(sbi, SBI_POR_DOING); + + f2fs_up_write_trace(&sbi->cp_global_sem, &lc); + + /* let's drop all the directory inodes for clean checkpoint */ + destroy_fsync_dnodes(&dir_list, err); + + if (need_writecp) { + set_sbi_flag(sbi, SBI_IS_RECOVERED); + + if (!err) { + struct cp_control cpc = { + .reason = CP_RECOVERY, + }; + stat_inc_cp_call_count(sbi, TOTAL_CALL); + err = f2fs_write_checkpoint(sbi, &cpc); + } + } + + sbi->sb->s_flags = s_flags; /* Restore SB_RDONLY status */ + + return ret ? ret : err; +} + +int __init f2fs_create_recovery_cache(void) +{ + fsync_entry_slab = f2fs_kmem_cache_create("f2fs_fsync_inode_entry", + sizeof(struct fsync_inode_entry)); + return fsync_entry_slab ? 0 : -ENOMEM; +} + +void f2fs_destroy_recovery_cache(void) +{ + kmem_cache_destroy(fsync_entry_slab); +} diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c new file mode 100644 index 000000000..642668a18 --- /dev/null +++ b/fs/f2fs/segment.c @@ -0,0 +1,5901 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/f2fs/segment.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "f2fs.h" +#include "segment.h" +#include "node.h" +#include "gc.h" +#include "iostat.h" +#include + +#define __reverse_ffz(x) __reverse_ffs(~(x)) + +static struct kmem_cache *discard_entry_slab; +static struct kmem_cache *discard_cmd_slab; +static struct kmem_cache *sit_entry_set_slab; +static struct kmem_cache *revoke_entry_slab; + +static unsigned long __reverse_ulong(unsigned char *str) +{ + unsigned long tmp = 0; + int shift = 24, idx = 0; + +#if BITS_PER_LONG == 64 + shift = 56; +#endif + while (shift >= 0) { + tmp |= (unsigned long)str[idx++] << shift; + shift -= BITS_PER_BYTE; + } + return tmp; +} + +/* + * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since + * MSB and LSB are reversed in a byte by f2fs_set_bit. + */ +static inline unsigned long __reverse_ffs(unsigned long word) +{ + int num = 0; + +#if BITS_PER_LONG == 64 + if ((word & 0xffffffff00000000UL) == 0) + num += 32; + else + word >>= 32; +#endif + if ((word & 0xffff0000) == 0) + num += 16; + else + word >>= 16; + + if ((word & 0xff00) == 0) + num += 8; + else + word >>= 8; + + if ((word & 0xf0) == 0) + num += 4; + else + word >>= 4; + + if ((word & 0xc) == 0) + num += 2; + else + word >>= 2; + + if ((word & 0x2) == 0) + num += 1; + return num; +} + +/* + * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because + * f2fs_set_bit makes MSB and LSB reversed in a byte. + * @size must be integral times of unsigned long. + * Example: + * MSB <--> LSB + * f2fs_set_bit(0, bitmap) => 1000 0000 + * f2fs_set_bit(7, bitmap) => 0000 0001 + */ +static unsigned long __find_rev_next_bit(const unsigned long *addr, + unsigned long size, unsigned long offset) +{ + const unsigned long *p = addr + BIT_WORD(offset); + unsigned long result = size; + unsigned long tmp; + + if (offset >= size) + return size; + + size -= (offset & ~(BITS_PER_LONG - 1)); + offset %= BITS_PER_LONG; + + while (1) { + if (*p == 0) + goto pass; + + tmp = __reverse_ulong((unsigned char *)p); + + tmp &= ~0UL >> offset; + if (size < BITS_PER_LONG) + tmp &= (~0UL << (BITS_PER_LONG - size)); + if (tmp) + goto found; +pass: + if (size <= BITS_PER_LONG) + break; + size -= BITS_PER_LONG; + offset = 0; + p++; + } + return result; +found: + return result - size + __reverse_ffs(tmp); +} + +static unsigned long __find_rev_next_zero_bit(const unsigned long *addr, + unsigned long size, unsigned long offset) +{ + const unsigned long *p = addr + BIT_WORD(offset); + unsigned long result = size; + unsigned long tmp; + + if (offset >= size) + return size; + + size -= (offset & ~(BITS_PER_LONG - 1)); + offset %= BITS_PER_LONG; + + while (1) { + if (*p == ~0UL) + goto pass; + + tmp = __reverse_ulong((unsigned char *)p); + + if (offset) + tmp |= ~0UL << (BITS_PER_LONG - offset); + if (size < BITS_PER_LONG) + tmp |= ~0UL >> size; + if (tmp != ~0UL) + goto found; +pass: + if (size <= BITS_PER_LONG) + break; + size -= BITS_PER_LONG; + offset = 0; + p++; + } + return result; +found: + return result - size + __reverse_ffz(tmp); +} + +bool f2fs_need_SSR(struct f2fs_sb_info *sbi) +{ + int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES); + int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS); + int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA); + + if (f2fs_lfs_mode(sbi)) + return false; + if (sbi->gc_mode == GC_URGENT_HIGH) + return true; + if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) + return true; + + return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs + + SM_I(sbi)->min_ssr_sections + reserved_sections(sbi)); +} + +void f2fs_abort_atomic_write(struct inode *inode, bool clean) +{ + struct f2fs_inode_info *fi = F2FS_I(inode); + + if (!f2fs_is_atomic_file(inode)) + return; + + if (clean) + truncate_inode_pages_final(inode->i_mapping); + + release_atomic_write_cnt(inode); + clear_inode_flag(inode, FI_ATOMIC_COMMITTED); + clear_inode_flag(inode, FI_ATOMIC_REPLACE); + clear_inode_flag(inode, FI_ATOMIC_FILE); + if (is_inode_flag_set(inode, FI_ATOMIC_DIRTIED)) { + clear_inode_flag(inode, FI_ATOMIC_DIRTIED); + /* + * The vfs inode keeps clean during commit, but the f2fs inode + * doesn't. So clear the dirty state after commit and let + * f2fs_mark_inode_dirty_sync ensure a consistent dirty state. + */ + f2fs_inode_synced(inode); + f2fs_mark_inode_dirty_sync(inode, true); + } + stat_dec_atomic_inode(inode); + + F2FS_I(inode)->atomic_write_task = NULL; + + if (clean) { + f2fs_i_size_write(inode, fi->original_i_size); + fi->original_i_size = 0; + } + /* avoid stale dirty inode during eviction */ + sync_inode_metadata(inode, 0); +} + +static int __replace_atomic_write_block(struct inode *inode, pgoff_t index, + block_t new_addr, block_t *old_addr, bool recover) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct dnode_of_data dn; + struct node_info ni; + int err; + +retry: + set_new_dnode(&dn, inode, NULL, NULL, 0); + err = f2fs_get_dnode_of_data(&dn, index, ALLOC_NODE); + if (err) { + if (err == -ENOMEM) { + memalloc_retry_wait(GFP_NOFS); + goto retry; + } + return err; + } + + err = f2fs_get_node_info(sbi, dn.nid, &ni, false); + if (err) { + f2fs_put_dnode(&dn); + return err; + } + + if (recover) { + /* dn.data_blkaddr is always valid */ + if (!__is_valid_data_blkaddr(new_addr)) { + if (new_addr == NULL_ADDR) + dec_valid_block_count(sbi, inode, 1); + f2fs_invalidate_blocks(sbi, dn.data_blkaddr, 1); + f2fs_update_data_blkaddr(&dn, new_addr); + } else { + f2fs_replace_block(sbi, &dn, dn.data_blkaddr, + new_addr, ni.version, true, true); + } + } else { + blkcnt_t count = 1; + + err = inc_valid_block_count(sbi, inode, &count, true); + if (err) { + f2fs_put_dnode(&dn); + return err; + } + + *old_addr = dn.data_blkaddr; + f2fs_truncate_data_blocks_range(&dn, 1); + dec_valid_block_count(sbi, F2FS_I(inode)->cow_inode, count); + + f2fs_replace_block(sbi, &dn, dn.data_blkaddr, new_addr, + ni.version, true, false); + } + + f2fs_put_dnode(&dn); + + trace_f2fs_replace_atomic_write_block(inode, F2FS_I(inode)->cow_inode, + index, old_addr ? *old_addr : 0, new_addr, recover); + return 0; +} + +static void __complete_revoke_list(struct inode *inode, struct list_head *head, + bool revoke) +{ + struct revoke_entry *cur, *tmp; + pgoff_t start_index = 0; + bool truncate = is_inode_flag_set(inode, FI_ATOMIC_REPLACE); + + list_for_each_entry_safe(cur, tmp, head, list) { + if (revoke) { + __replace_atomic_write_block(inode, cur->index, + cur->old_addr, NULL, true); + } else if (truncate) { + f2fs_truncate_hole(inode, start_index, cur->index); + start_index = cur->index + 1; + } + + list_del(&cur->list); + kmem_cache_free(revoke_entry_slab, cur); + } + + if (!revoke && truncate) + f2fs_do_truncate_blocks(inode, start_index * PAGE_SIZE, false); +} + +static int __f2fs_commit_atomic_write(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_inode_info *fi = F2FS_I(inode); + struct inode *cow_inode = fi->cow_inode; + struct revoke_entry *new; + struct list_head revoke_list; + block_t blkaddr; + struct dnode_of_data dn; + pgoff_t len = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); + pgoff_t off = 0, blen, index; + int ret = 0, i; + + INIT_LIST_HEAD(&revoke_list); + + while (len) { + blen = min_t(pgoff_t, ADDRS_PER_BLOCK(cow_inode), len); + + set_new_dnode(&dn, cow_inode, NULL, NULL, 0); + ret = f2fs_get_dnode_of_data(&dn, off, LOOKUP_NODE_RA); + if (ret && ret != -ENOENT) { + goto out; + } else if (ret == -ENOENT) { + ret = 0; + if (dn.max_level == 0) + goto out; + goto next; + } + + blen = min((pgoff_t)ADDRS_PER_PAGE(dn.node_folio, cow_inode), + len); + index = off; + for (i = 0; i < blen; i++, dn.ofs_in_node++, index++) { + blkaddr = f2fs_data_blkaddr(&dn); + + if (!__is_valid_data_blkaddr(blkaddr)) { + continue; + } else if (!f2fs_is_valid_blkaddr(sbi, blkaddr, + DATA_GENERIC_ENHANCE)) { + f2fs_put_dnode(&dn); + ret = -EFSCORRUPTED; + goto out; + } + + new = f2fs_kmem_cache_alloc(revoke_entry_slab, GFP_NOFS, + true, NULL); + + ret = __replace_atomic_write_block(inode, index, blkaddr, + &new->old_addr, false); + if (ret) { + f2fs_put_dnode(&dn); + kmem_cache_free(revoke_entry_slab, new); + goto out; + } + + f2fs_update_data_blkaddr(&dn, NULL_ADDR); + new->index = index; + list_add_tail(&new->list, &revoke_list); + } + f2fs_put_dnode(&dn); +next: + off += blen; + len -= blen; + } + +out: + if (time_to_inject(sbi, FAULT_ATOMIC_TIMEOUT)) + f2fs_schedule_timeout_killable(DEFAULT_FAULT_TIMEOUT, true); + + if (ret) { + sbi->revoked_atomic_block += fi->atomic_write_cnt; + } else { + sbi->committed_atomic_block += fi->atomic_write_cnt; + set_inode_flag(inode, FI_ATOMIC_COMMITTED); + + /* + * inode may has no FI_ATOMIC_DIRTIED flag due to no write + * before commit. + */ + if (is_inode_flag_set(inode, FI_ATOMIC_DIRTIED)) { + /* clear atomic dirty status and set vfs dirty status */ + clear_inode_flag(inode, FI_ATOMIC_DIRTIED); + f2fs_mark_inode_dirty_sync(inode, true); + } + } + + __complete_revoke_list(inode, &revoke_list, ret ? true : false); + + return ret; +} + +int f2fs_commit_atomic_write(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_inode_info *fi = F2FS_I(inode); + struct f2fs_lock_context lc; + int err; + + err = filemap_write_and_wait_range(inode->i_mapping, 0, LLONG_MAX); + if (err) + return err; + + f2fs_down_write(&fi->i_gc_rwsem[WRITE]); + f2fs_lock_op(sbi, &lc); + + err = __f2fs_commit_atomic_write(inode); + + f2fs_unlock_op(sbi, &lc); + f2fs_up_write(&fi->i_gc_rwsem[WRITE]); + + return err; +} + +/* + * This function balances dirty node and dentry pages. + * In addition, it controls garbage collection. + */ +void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need) +{ + if (f2fs_cp_error(sbi)) + return; + + if (time_to_inject(sbi, FAULT_CHECKPOINT)) + f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_FAULT_INJECT); + + /* balance_fs_bg is able to be pending */ + if (need && excess_cached_nats(sbi)) + f2fs_balance_fs_bg(sbi, false); + + if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) + return; + + /* + * We should do GC or end up with checkpoint, if there are so many dirty + * dir/node pages without enough free segments. + */ + if (has_enough_free_secs(sbi, 0, 0)) + return; + + /* + * Submit all cached OPU/IPU DATA bios before triggering + * foreground GC to avoid potential deadlocks. + */ + f2fs_submit_merged_write(sbi, DATA); + f2fs_submit_all_merged_ipu_writes(sbi); + + if (test_opt(sbi, GC_MERGE) && sbi->gc_thread && + sbi->gc_thread->f2fs_gc_task) { + DEFINE_WAIT(wait); + + prepare_to_wait(&sbi->gc_thread->fggc_wq, &wait, + TASK_UNINTERRUPTIBLE); + wake_up(&sbi->gc_thread->gc_wait_queue_head); + io_schedule(); + finish_wait(&sbi->gc_thread->fggc_wq, &wait); + } else { + struct f2fs_gc_control gc_control = { + .victim_segno = NULL_SEGNO, + .init_gc_type = f2fs_sb_has_blkzoned(sbi) ? + FG_GC : BG_GC, + .no_bg_gc = true, + .should_migrate_blocks = false, + .err_gc_skipped = false, + .nr_free_secs = 1 }; + + f2fs_down_write_trace(&sbi->gc_lock, &gc_control.lc); + stat_inc_gc_call_count(sbi, FOREGROUND); + f2fs_gc(sbi, &gc_control); + } +} + +static inline bool excess_dirty_threshold(struct f2fs_sb_info *sbi) +{ + int factor = f2fs_rwsem_is_locked(&sbi->cp_rwsem) ? 3 : 2; + unsigned int dents = get_pages(sbi, F2FS_DIRTY_DENTS); + unsigned int qdata = get_pages(sbi, F2FS_DIRTY_QDATA); + unsigned int nodes = get_pages(sbi, F2FS_DIRTY_NODES); + unsigned int meta = get_pages(sbi, F2FS_DIRTY_META); + unsigned int imeta = get_pages(sbi, F2FS_DIRTY_IMETA); + unsigned int threshold = + SEGS_TO_BLKS(sbi, (factor * DEFAULT_DIRTY_THRESHOLD)); + unsigned int global_threshold = threshold * 3 / 2; + + if (dents >= threshold || qdata >= threshold || + nodes >= threshold || meta >= threshold || + imeta >= threshold) + return true; + return dents + qdata + nodes + meta + imeta > global_threshold; +} + +void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg) +{ + if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) + return; + + /* try to shrink extent cache when there is no enough memory */ + if (!f2fs_available_free_memory(sbi, READ_EXTENT_CACHE)) + f2fs_shrink_read_extent_tree(sbi, + READ_EXTENT_CACHE_SHRINK_NUMBER); + + /* try to shrink age extent cache when there is no enough memory */ + if (!f2fs_available_free_memory(sbi, AGE_EXTENT_CACHE)) + f2fs_shrink_age_extent_tree(sbi, + AGE_EXTENT_CACHE_SHRINK_NUMBER); + + /* check the # of cached NAT entries */ + if (!f2fs_available_free_memory(sbi, NAT_ENTRIES)) + f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK); + + if (!f2fs_available_free_memory(sbi, FREE_NIDS)) + f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS); + else + f2fs_build_free_nids(sbi, false, false); + + if (excess_dirty_nats(sbi) || excess_dirty_threshold(sbi) || + excess_prefree_segs(sbi) || !f2fs_space_for_roll_forward(sbi)) + goto do_sync; + + /* there is background inflight IO or foreground operation recently */ + if (is_inflight_io(sbi, REQ_TIME) || + (!f2fs_time_over(sbi, REQ_TIME) && f2fs_rwsem_is_locked(&sbi->cp_rwsem))) + return; + + /* exceed periodical checkpoint timeout threshold */ + if (f2fs_time_over(sbi, CP_TIME)) + goto do_sync; + + /* checkpoint is the only way to shrink partial cached entries */ + if (f2fs_available_free_memory(sbi, NAT_ENTRIES) && + f2fs_available_free_memory(sbi, INO_ENTRIES)) + return; + +do_sync: + if (test_opt(sbi, DATA_FLUSH) && from_bg) { + struct blk_plug plug; + + mutex_lock(&sbi->flush_lock); + + blk_start_plug(&plug); + f2fs_sync_dirty_inodes(sbi, FILE_INODE, false); + blk_finish_plug(&plug); + + mutex_unlock(&sbi->flush_lock); + } + stat_inc_cp_call_count(sbi, BACKGROUND); + f2fs_sync_fs(sbi->sb, 1); +} + +static int __submit_flush_wait(struct f2fs_sb_info *sbi, + struct block_device *bdev) +{ + int ret = blkdev_issue_flush(bdev); + + trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER), + test_opt(sbi, FLUSH_MERGE), ret); + if (!ret) + f2fs_update_iostat(sbi, NULL, FS_FLUSH_IO, 0); + return ret; +} + +static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino) +{ + int ret = 0; + int i; + + if (!f2fs_is_multi_device(sbi)) + return __submit_flush_wait(sbi, sbi->sb->s_bdev); + + for (i = 0; i < sbi->s_ndevs; i++) { + if (!f2fs_is_dirty_device(sbi, ino, i, FLUSH_INO)) + continue; + ret = __submit_flush_wait(sbi, FDEV(i).bdev); + if (ret) + break; + } + return ret; +} + +static int issue_flush_thread(void *data) +{ + struct f2fs_sb_info *sbi = data; + struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; + wait_queue_head_t *q = &fcc->flush_wait_queue; +repeat: + if (kthread_should_stop()) + return 0; + + if (!llist_empty(&fcc->issue_list)) { + struct flush_cmd *cmd, *next; + int ret; + + fcc->dispatch_list = llist_del_all(&fcc->issue_list); + fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list); + + cmd = llist_entry(fcc->dispatch_list, struct flush_cmd, llnode); + + ret = submit_flush_wait(sbi, cmd->ino); + atomic_inc(&fcc->issued_flush); + + llist_for_each_entry_safe(cmd, next, + fcc->dispatch_list, llnode) { + cmd->ret = ret; + complete(&cmd->wait); + } + fcc->dispatch_list = NULL; + } + + wait_event_interruptible(*q, + kthread_should_stop() || !llist_empty(&fcc->issue_list)); + goto repeat; +} + +int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino) +{ + struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; + struct flush_cmd cmd; + int ret; + + if (test_opt(sbi, NOBARRIER)) + return 0; + + if (!test_opt(sbi, FLUSH_MERGE)) { + atomic_inc(&fcc->queued_flush); + ret = submit_flush_wait(sbi, ino); + atomic_dec(&fcc->queued_flush); + atomic_inc(&fcc->issued_flush); + return ret; + } + + if (atomic_inc_return(&fcc->queued_flush) == 1 || + f2fs_is_multi_device(sbi)) { + ret = submit_flush_wait(sbi, ino); + atomic_dec(&fcc->queued_flush); + + atomic_inc(&fcc->issued_flush); + return ret; + } + + cmd.ino = ino; + init_completion(&cmd.wait); + + llist_add(&cmd.llnode, &fcc->issue_list); + + /* + * update issue_list before we wake up issue_flush thread, this + * smp_mb() pairs with another barrier in ___wait_event(), see + * more details in comments of waitqueue_active(). + */ + smp_mb(); + + if (waitqueue_active(&fcc->flush_wait_queue)) + wake_up(&fcc->flush_wait_queue); + + if (fcc->f2fs_issue_flush) { + wait_for_completion(&cmd.wait); + atomic_dec(&fcc->queued_flush); + } else { + struct llist_node *list; + + list = llist_del_all(&fcc->issue_list); + if (!list) { + wait_for_completion(&cmd.wait); + atomic_dec(&fcc->queued_flush); + } else { + struct flush_cmd *tmp, *next; + + ret = submit_flush_wait(sbi, ino); + + llist_for_each_entry_safe(tmp, next, list, llnode) { + if (tmp == &cmd) { + cmd.ret = ret; + atomic_dec(&fcc->queued_flush); + continue; + } + tmp->ret = ret; + complete(&tmp->wait); + } + } + } + + return cmd.ret; +} + +int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi) +{ + dev_t dev = sbi->sb->s_bdev->bd_dev; + struct flush_cmd_control *fcc; + + if (SM_I(sbi)->fcc_info) { + fcc = SM_I(sbi)->fcc_info; + if (fcc->f2fs_issue_flush) + return 0; + goto init_thread; + } + + fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL); + if (!fcc) + return -ENOMEM; + atomic_set(&fcc->issued_flush, 0); + atomic_set(&fcc->queued_flush, 0); + init_waitqueue_head(&fcc->flush_wait_queue); + init_llist_head(&fcc->issue_list); + SM_I(sbi)->fcc_info = fcc; + if (!test_opt(sbi, FLUSH_MERGE)) + return 0; + +init_thread: + fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi, + "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev)); + if (IS_ERR(fcc->f2fs_issue_flush)) { + int err = PTR_ERR(fcc->f2fs_issue_flush); + + fcc->f2fs_issue_flush = NULL; + return err; + } + + return 0; +} + +void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free) +{ + struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info; + + if (fcc && fcc->f2fs_issue_flush) { + struct task_struct *flush_thread = fcc->f2fs_issue_flush; + + fcc->f2fs_issue_flush = NULL; + kthread_stop(flush_thread); + } + if (free) { + kfree(fcc); + SM_I(sbi)->fcc_info = NULL; + } +} + +int f2fs_flush_device_cache(struct f2fs_sb_info *sbi) +{ + int ret = 0, i; + + if (!f2fs_is_multi_device(sbi)) + return 0; + + if (test_opt(sbi, NOBARRIER)) + return 0; + + for (i = 1; i < sbi->s_ndevs; i++) { + int count = DEFAULT_RETRY_IO_COUNT; + + if (!f2fs_test_bit(i, (char *)&sbi->dirty_device)) + continue; + + do { + ret = __submit_flush_wait(sbi, FDEV(i).bdev); + if (ret) + f2fs_schedule_timeout(DEFAULT_SCHEDULE_TIMEOUT); + } while (ret && --count); + + if (ret) { + f2fs_stop_checkpoint(sbi, false, + STOP_CP_REASON_FLUSH_FAIL); + break; + } + + spin_lock(&sbi->dev_lock); + f2fs_clear_bit(i, (char *)&sbi->dirty_device); + spin_unlock(&sbi->dev_lock); + } + + return ret; +} + +static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, + enum dirty_type dirty_type) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + + /* need not be added */ + if (is_curseg(sbi, segno)) + return; + + if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type])) + dirty_i->nr_dirty[dirty_type]++; + + if (dirty_type == DIRTY) { + struct seg_entry *sentry = get_seg_entry(sbi, segno); + enum dirty_type t = sentry->type; + + if (unlikely(t >= DIRTY)) { + f2fs_bug_on(sbi, 1); + return; + } + if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t])) + dirty_i->nr_dirty[t]++; + + if (__is_large_section(sbi)) { + unsigned int secno = GET_SEC_FROM_SEG(sbi, segno); + block_t valid_blocks = + get_valid_blocks(sbi, segno, true); + + f2fs_bug_on(sbi, + (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) && + !valid_blocks) || + valid_blocks == CAP_BLKS_PER_SEC(sbi)); + + if (!is_cursec(sbi, secno)) + set_bit(secno, dirty_i->dirty_secmap); + } + } +} + +static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno, + enum dirty_type dirty_type) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + block_t valid_blocks; + + if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type])) + dirty_i->nr_dirty[dirty_type]--; + + if (dirty_type == DIRTY) { + struct seg_entry *sentry = get_seg_entry(sbi, segno); + enum dirty_type t = sentry->type; + + if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t])) + dirty_i->nr_dirty[t]--; + + valid_blocks = get_valid_blocks(sbi, segno, true); + if (valid_blocks == 0) { + clear_bit(GET_SEC_FROM_SEG(sbi, segno), + dirty_i->victim_secmap); +#ifdef CONFIG_F2FS_CHECK_FS + clear_bit(segno, SIT_I(sbi)->invalid_segmap); +#endif + } + if (__is_large_section(sbi)) { + unsigned int secno = GET_SEC_FROM_SEG(sbi, segno); + + if (!valid_blocks || + valid_blocks == CAP_BLKS_PER_SEC(sbi)) { + clear_bit(secno, dirty_i->dirty_secmap); + return; + } + + if (!is_cursec(sbi, secno)) + set_bit(secno, dirty_i->dirty_secmap); + } + } +} + +/* + * Should not occur error such as -ENOMEM. + * Adding dirty entry into seglist is not critical operation. + * If a given segment is one of current working segments, it won't be added. + */ +static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + unsigned short valid_blocks, ckpt_valid_blocks; + unsigned int usable_blocks; + + if (segno == NULL_SEGNO || is_curseg(sbi, segno)) + return; + + usable_blocks = f2fs_usable_blks_in_seg(sbi, segno); + mutex_lock(&dirty_i->seglist_lock); + + valid_blocks = get_valid_blocks(sbi, segno, false); + ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno, false); + + if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) || + ckpt_valid_blocks == usable_blocks)) { + __locate_dirty_segment(sbi, segno, PRE); + __remove_dirty_segment(sbi, segno, DIRTY); + } else if (valid_blocks < usable_blocks) { + __locate_dirty_segment(sbi, segno, DIRTY); + } else { + /* Recovery routine with SSR needs this */ + __remove_dirty_segment(sbi, segno, DIRTY); + } + + mutex_unlock(&dirty_i->seglist_lock); +} + +/* This moves currently empty dirty blocks to prefree. Must hold seglist_lock */ +void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + unsigned int segno; + + mutex_lock(&dirty_i->seglist_lock); + for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) { + if (get_valid_blocks(sbi, segno, false)) + continue; + if (is_curseg(sbi, segno)) + continue; + __locate_dirty_segment(sbi, segno, PRE); + __remove_dirty_segment(sbi, segno, DIRTY); + } + mutex_unlock(&dirty_i->seglist_lock); +} + +block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi) +{ + int ovp_hole_segs = + (overprovision_segments(sbi) - reserved_segments(sbi)); + block_t ovp_holes = SEGS_TO_BLKS(sbi, ovp_hole_segs); + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + block_t holes[2] = {0, 0}; /* DATA and NODE */ + block_t unusable; + struct seg_entry *se; + unsigned int segno; + + mutex_lock(&dirty_i->seglist_lock); + for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) { + se = get_seg_entry(sbi, segno); + if (IS_NODESEG(se->type)) + holes[NODE] += f2fs_usable_blks_in_seg(sbi, segno) - + se->valid_blocks; + else + holes[DATA] += f2fs_usable_blks_in_seg(sbi, segno) - + se->valid_blocks; + } + mutex_unlock(&dirty_i->seglist_lock); + + unusable = max(holes[DATA], holes[NODE]); + if (unusable > ovp_holes) + return unusable - ovp_holes; + return 0; +} + +int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable) +{ + int ovp_hole_segs = + (overprovision_segments(sbi) - reserved_segments(sbi)); + + if (F2FS_OPTION(sbi).unusable_cap_perc == 100) + return 0; + if (unusable > F2FS_OPTION(sbi).unusable_cap) + return -EAGAIN; + if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK) && + dirty_segments(sbi) > ovp_hole_segs) + return -EAGAIN; + if (has_not_enough_free_secs(sbi, 0, 0)) + return -EAGAIN; + return 0; +} + +/* This is only used by SBI_CP_DISABLED */ +static unsigned int get_free_segment(struct f2fs_sb_info *sbi) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + unsigned int segno = 0; + + mutex_lock(&dirty_i->seglist_lock); + for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) { + if (get_valid_blocks(sbi, segno, false)) + continue; + if (get_ckpt_valid_blocks(sbi, segno, false)) + continue; + mutex_unlock(&dirty_i->seglist_lock); + return segno; + } + mutex_unlock(&dirty_i->seglist_lock); + return NULL_SEGNO; +} + +static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi, + struct block_device *bdev, block_t lstart, + block_t start, block_t len) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct list_head *pend_list; + struct discard_cmd *dc; + + f2fs_bug_on(sbi, !len); + + pend_list = &dcc->pend_list[plist_idx(len)]; + + dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS, true, NULL); + INIT_LIST_HEAD(&dc->list); + dc->bdev = bdev; + dc->di.lstart = lstart; + dc->di.start = start; + dc->di.len = len; + dc->ref = 0; + dc->state = D_PREP; + dc->queued = 0; + dc->error = 0; + init_completion(&dc->wait); + list_add_tail(&dc->list, pend_list); + spin_lock_init(&dc->lock); + dc->bio_ref = 0; + atomic_inc(&dcc->discard_cmd_cnt); + dcc->undiscard_blks += len; + + return dc; +} + +static bool f2fs_check_discard_tree(struct f2fs_sb_info *sbi) +{ +#ifdef CONFIG_F2FS_CHECK_FS + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct rb_node *cur = rb_first_cached(&dcc->root), *next; + struct discard_cmd *cur_dc, *next_dc; + + while (cur) { + next = rb_next(cur); + if (!next) + return true; + + cur_dc = rb_entry(cur, struct discard_cmd, rb_node); + next_dc = rb_entry(next, struct discard_cmd, rb_node); + + if (cur_dc->di.lstart + cur_dc->di.len > next_dc->di.lstart) { + f2fs_info(sbi, "broken discard_rbtree, " + "cur(%u, %u) next(%u, %u)", + cur_dc->di.lstart, cur_dc->di.len, + next_dc->di.lstart, next_dc->di.len); + return false; + } + cur = next; + } +#endif + return true; +} + +static struct discard_cmd *__lookup_discard_cmd(struct f2fs_sb_info *sbi, + block_t blkaddr) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct rb_node *node = dcc->root.rb_root.rb_node; + struct discard_cmd *dc; + + while (node) { + dc = rb_entry(node, struct discard_cmd, rb_node); + + if (blkaddr < dc->di.lstart) + node = node->rb_left; + else if (blkaddr >= dc->di.lstart + dc->di.len) + node = node->rb_right; + else + return dc; + } + return NULL; +} + +static struct discard_cmd *__lookup_discard_cmd_ret(struct rb_root_cached *root, + block_t blkaddr, + struct discard_cmd **prev_entry, + struct discard_cmd **next_entry, + struct rb_node ***insert_p, + struct rb_node **insert_parent) +{ + struct rb_node **pnode = &root->rb_root.rb_node; + struct rb_node *parent = NULL, *tmp_node; + struct discard_cmd *dc; + + *insert_p = NULL; + *insert_parent = NULL; + *prev_entry = NULL; + *next_entry = NULL; + + if (RB_EMPTY_ROOT(&root->rb_root)) + return NULL; + + while (*pnode) { + parent = *pnode; + dc = rb_entry(*pnode, struct discard_cmd, rb_node); + + if (blkaddr < dc->di.lstart) + pnode = &(*pnode)->rb_left; + else if (blkaddr >= dc->di.lstart + dc->di.len) + pnode = &(*pnode)->rb_right; + else + goto lookup_neighbors; + } + + *insert_p = pnode; + *insert_parent = parent; + + dc = rb_entry(parent, struct discard_cmd, rb_node); + tmp_node = parent; + if (parent && blkaddr > dc->di.lstart) + tmp_node = rb_next(parent); + *next_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node); + + tmp_node = parent; + if (parent && blkaddr < dc->di.lstart) + tmp_node = rb_prev(parent); + *prev_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node); + return NULL; + +lookup_neighbors: + /* lookup prev node for merging backward later */ + tmp_node = rb_prev(&dc->rb_node); + *prev_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node); + + /* lookup next node for merging frontward later */ + tmp_node = rb_next(&dc->rb_node); + *next_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node); + return dc; +} + +static void __detach_discard_cmd(struct discard_cmd_control *dcc, + struct discard_cmd *dc) +{ + if (dc->state == D_DONE) + atomic_sub(dc->queued, &dcc->queued_discard); + + list_del(&dc->list); + rb_erase_cached(&dc->rb_node, &dcc->root); + dcc->undiscard_blks -= dc->di.len; + + kmem_cache_free(discard_cmd_slab, dc); + + atomic_dec(&dcc->discard_cmd_cnt); +} + +static void __remove_discard_cmd(struct f2fs_sb_info *sbi, + struct discard_cmd *dc) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + unsigned long flags; + + trace_f2fs_remove_discard(dc->bdev, dc->di.start, dc->di.len); + + spin_lock_irqsave(&dc->lock, flags); + if (dc->bio_ref) { + spin_unlock_irqrestore(&dc->lock, flags); + return; + } + spin_unlock_irqrestore(&dc->lock, flags); + + f2fs_bug_on(sbi, dc->ref); + + if (dc->error == -EOPNOTSUPP) + dc->error = 0; + + if (dc->error) + f2fs_info_ratelimited(sbi, + "Issue discard(%u, %u, %u) failed, ret: %d", + dc->di.lstart, dc->di.start, dc->di.len, dc->error); + __detach_discard_cmd(dcc, dc); +} + +static void f2fs_submit_discard_endio(struct bio *bio) +{ + struct discard_cmd *dc = (struct discard_cmd *)bio->bi_private; + unsigned long flags; + + spin_lock_irqsave(&dc->lock, flags); + if (!dc->error) + dc->error = blk_status_to_errno(bio->bi_status); + dc->bio_ref--; + if (!dc->bio_ref && dc->state == D_SUBMIT) { + dc->state = D_DONE; + complete_all(&dc->wait); + } + spin_unlock_irqrestore(&dc->lock, flags); + bio_put(bio); +} + +static void __check_sit_bitmap(struct f2fs_sb_info *sbi, + block_t start, block_t end) +{ +#ifdef CONFIG_F2FS_CHECK_FS + struct seg_entry *sentry; + unsigned int segno; + block_t blk = start; + unsigned long offset, size, *map; + + while (blk < end) { + segno = GET_SEGNO(sbi, blk); + sentry = get_seg_entry(sbi, segno); + offset = GET_BLKOFF_FROM_SEG0(sbi, blk); + + if (end < START_BLOCK(sbi, segno + 1)) + size = GET_BLKOFF_FROM_SEG0(sbi, end); + else + size = BLKS_PER_SEG(sbi); + map = (unsigned long *)(sentry->cur_valid_map); + offset = __find_rev_next_bit(map, size, offset); + f2fs_bug_on(sbi, offset != size); + blk = START_BLOCK(sbi, segno + 1); + } +#endif +} + +static void __init_discard_policy(struct f2fs_sb_info *sbi, + struct discard_policy *dpolicy, + int discard_type, unsigned int granularity) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + + /* common policy */ + dpolicy->type = discard_type; + dpolicy->sync = true; + dpolicy->ordered = false; + dpolicy->granularity = granularity; + + dpolicy->max_requests = dcc->max_discard_request; + dpolicy->io_aware_gran = dcc->discard_io_aware_gran; + dpolicy->timeout = false; + + if (discard_type == DPOLICY_BG) { + dpolicy->min_interval = dcc->min_discard_issue_time; + dpolicy->mid_interval = dcc->mid_discard_issue_time; + dpolicy->max_interval = dcc->max_discard_issue_time; + if (dcc->discard_io_aware == DPOLICY_IO_AWARE_ENABLE) + dpolicy->io_aware = true; + else if (dcc->discard_io_aware == DPOLICY_IO_AWARE_DISABLE) + dpolicy->io_aware = false; + dpolicy->sync = false; + dpolicy->ordered = true; + if (utilization(sbi) > dcc->discard_urgent_util) { + dpolicy->granularity = MIN_DISCARD_GRANULARITY; + if (atomic_read(&dcc->discard_cmd_cnt)) + dpolicy->max_interval = + dcc->min_discard_issue_time; + } + } else if (discard_type == DPOLICY_FORCE) { + dpolicy->min_interval = dcc->min_discard_issue_time; + dpolicy->mid_interval = dcc->mid_discard_issue_time; + dpolicy->max_interval = dcc->max_discard_issue_time; + dpolicy->io_aware = false; + } else if (discard_type == DPOLICY_FSTRIM) { + dpolicy->io_aware = false; + } else if (discard_type == DPOLICY_UMOUNT) { + dpolicy->io_aware = false; + /* we need to issue all to keep CP_TRIMMED_FLAG */ + dpolicy->granularity = MIN_DISCARD_GRANULARITY; + dpolicy->timeout = true; + } +} + +static void __update_discard_tree_range(struct f2fs_sb_info *sbi, + struct block_device *bdev, block_t lstart, + block_t start, block_t len); + +#ifdef CONFIG_BLK_DEV_ZONED +static void __submit_zone_reset_cmd(struct f2fs_sb_info *sbi, + struct discard_cmd *dc, blk_opf_t flag, + struct list_head *wait_list, + unsigned int *issued) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct block_device *bdev = dc->bdev; + struct bio *bio = bio_alloc(bdev, 0, REQ_OP_ZONE_RESET | flag, GFP_NOFS); + unsigned long flags; + + trace_f2fs_issue_reset_zone(bdev, dc->di.start); + + spin_lock_irqsave(&dc->lock, flags); + dc->state = D_SUBMIT; + dc->bio_ref++; + spin_unlock_irqrestore(&dc->lock, flags); + + if (issued) + (*issued)++; + + atomic_inc(&dcc->queued_discard); + dc->queued++; + list_move_tail(&dc->list, wait_list); + + /* sanity check on discard range */ + __check_sit_bitmap(sbi, dc->di.lstart, dc->di.lstart + dc->di.len); + + bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(dc->di.start); + bio->bi_private = dc; + bio->bi_end_io = f2fs_submit_discard_endio; + submit_bio(bio); + + atomic_inc(&dcc->issued_discard); + f2fs_update_iostat(sbi, NULL, FS_ZONE_RESET_IO, dc->di.len * F2FS_BLKSIZE); +} +#endif + +/* this function is copied from blkdev_issue_discard from block/blk-lib.c */ +static int __submit_discard_cmd(struct f2fs_sb_info *sbi, + struct discard_policy *dpolicy, + struct discard_cmd *dc, int *issued) +{ + struct block_device *bdev = dc->bdev; + unsigned int max_discard_blocks = + SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev)); + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ? + &(dcc->fstrim_list) : &(dcc->wait_list); + blk_opf_t flag = dpolicy->sync ? REQ_SYNC : 0; + block_t lstart, start, len, total_len; + + if (dc->state != D_PREP) + return 0; + + if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) + return 0; + +#ifdef CONFIG_BLK_DEV_ZONED + if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev)) { + int devi = f2fs_bdev_index(sbi, bdev); + + if (devi < 0) + return -EINVAL; + + if (f2fs_blkz_is_seq(sbi, devi, dc->di.start)) { + __submit_zone_reset_cmd(sbi, dc, flag, + wait_list, issued); + return 0; + } + } +#endif + + /* + * stop issuing discard for any of below cases: + * 1. device is conventional zone, but it doesn't support discard. + * 2. device is regulare device, after snapshot it doesn't support + * discard. + */ + if (!bdev_max_discard_sectors(bdev)) + return -EOPNOTSUPP; + + trace_f2fs_issue_discard(bdev, dc->di.start, dc->di.len); + + lstart = dc->di.lstart; + start = dc->di.start; + len = dc->di.len; + total_len = len; + + dc->di.len = 0; + + while (total_len && *issued < dpolicy->max_requests) { + struct bio *bio = NULL; + unsigned long flags; + bool last = true; + + if (len > max_discard_blocks) { + len = max_discard_blocks; + last = false; + } + + (*issued)++; + if (*issued == dpolicy->max_requests) + last = true; + + dc->di.len += len; + + __blkdev_issue_discard(bdev, SECTOR_FROM_BLOCK(start), + SECTOR_FROM_BLOCK(len), GFP_NOFS, &bio); + f2fs_bug_on(sbi, !bio); + + /* + * should keep before submission to avoid D_DONE + * right away + */ + spin_lock_irqsave(&dc->lock, flags); + if (last) + dc->state = D_SUBMIT; + else + dc->state = D_PARTIAL; + dc->bio_ref++; + spin_unlock_irqrestore(&dc->lock, flags); + + atomic_inc(&dcc->queued_discard); + dc->queued++; + list_move_tail(&dc->list, wait_list); + + /* sanity check on discard range */ + __check_sit_bitmap(sbi, lstart, lstart + len); + + bio->bi_private = dc; + bio->bi_end_io = f2fs_submit_discard_endio; + bio->bi_opf |= flag; + submit_bio(bio); + + atomic_inc(&dcc->issued_discard); + + f2fs_update_iostat(sbi, NULL, FS_DISCARD_IO, len * F2FS_BLKSIZE); + + lstart += len; + start += len; + total_len -= len; + len = total_len; + } + + if (len) { + dcc->undiscard_blks -= len; + __update_discard_tree_range(sbi, bdev, lstart, start, len); + } + return 0; +} + +static void __insert_discard_cmd(struct f2fs_sb_info *sbi, + struct block_device *bdev, block_t lstart, + block_t start, block_t len) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct rb_node **p = &dcc->root.rb_root.rb_node; + struct rb_node *parent = NULL; + struct discard_cmd *dc; + bool leftmost = true; + + /* look up rb tree to find parent node */ + while (*p) { + parent = *p; + dc = rb_entry(parent, struct discard_cmd, rb_node); + + if (lstart < dc->di.lstart) { + p = &(*p)->rb_left; + } else if (lstart >= dc->di.lstart + dc->di.len) { + p = &(*p)->rb_right; + leftmost = false; + } else { + /* Let's skip to add, if exists */ + return; + } + } + + dc = __create_discard_cmd(sbi, bdev, lstart, start, len); + + rb_link_node(&dc->rb_node, parent, p); + rb_insert_color_cached(&dc->rb_node, &dcc->root, leftmost); +} + +static void __relocate_discard_cmd(struct discard_cmd_control *dcc, + struct discard_cmd *dc) +{ + list_move_tail(&dc->list, &dcc->pend_list[plist_idx(dc->di.len)]); +} + +static void __punch_discard_cmd(struct f2fs_sb_info *sbi, + struct discard_cmd *dc, block_t blkaddr) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct discard_info di = dc->di; + bool modified = false; + + if (dc->state == D_DONE || dc->di.len == 1) { + __remove_discard_cmd(sbi, dc); + return; + } + + dcc->undiscard_blks -= di.len; + + if (blkaddr > di.lstart) { + dc->di.len = blkaddr - dc->di.lstart; + dcc->undiscard_blks += dc->di.len; + __relocate_discard_cmd(dcc, dc); + modified = true; + } + + if (blkaddr < di.lstart + di.len - 1) { + if (modified) { + __insert_discard_cmd(sbi, dc->bdev, blkaddr + 1, + di.start + blkaddr + 1 - di.lstart, + di.lstart + di.len - 1 - blkaddr); + } else { + dc->di.lstart++; + dc->di.len--; + dc->di.start++; + dcc->undiscard_blks += dc->di.len; + __relocate_discard_cmd(dcc, dc); + } + } +} + +static void __update_discard_tree_range(struct f2fs_sb_info *sbi, + struct block_device *bdev, block_t lstart, + block_t start, block_t len) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct discard_cmd *prev_dc = NULL, *next_dc = NULL; + struct discard_cmd *dc; + struct discard_info di = {0}; + struct rb_node **insert_p = NULL, *insert_parent = NULL; + unsigned int max_discard_blocks = + SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev)); + block_t end = lstart + len; + + dc = __lookup_discard_cmd_ret(&dcc->root, lstart, + &prev_dc, &next_dc, &insert_p, &insert_parent); + if (dc) + prev_dc = dc; + + if (!prev_dc) { + di.lstart = lstart; + di.len = next_dc ? next_dc->di.lstart - lstart : len; + di.len = min(di.len, len); + di.start = start; + } + + while (1) { + struct rb_node *node; + bool merged = false; + struct discard_cmd *tdc = NULL; + + if (prev_dc) { + di.lstart = prev_dc->di.lstart + prev_dc->di.len; + if (di.lstart < lstart) + di.lstart = lstart; + if (di.lstart >= end) + break; + + if (!next_dc || next_dc->di.lstart > end) + di.len = end - di.lstart; + else + di.len = next_dc->di.lstart - di.lstart; + di.start = start + di.lstart - lstart; + } + + if (!di.len) + goto next; + + if (prev_dc && prev_dc->state == D_PREP && + prev_dc->bdev == bdev && + __is_discard_back_mergeable(&di, &prev_dc->di, + max_discard_blocks)) { + prev_dc->di.len += di.len; + dcc->undiscard_blks += di.len; + __relocate_discard_cmd(dcc, prev_dc); + di = prev_dc->di; + tdc = prev_dc; + merged = true; + } + + if (next_dc && next_dc->state == D_PREP && + next_dc->bdev == bdev && + __is_discard_front_mergeable(&di, &next_dc->di, + max_discard_blocks)) { + next_dc->di.lstart = di.lstart; + next_dc->di.len += di.len; + next_dc->di.start = di.start; + dcc->undiscard_blks += di.len; + __relocate_discard_cmd(dcc, next_dc); + if (tdc) + __remove_discard_cmd(sbi, tdc); + merged = true; + } + + if (!merged) + __insert_discard_cmd(sbi, bdev, + di.lstart, di.start, di.len); + next: + prev_dc = next_dc; + if (!prev_dc) + break; + + node = rb_next(&prev_dc->rb_node); + next_dc = rb_entry_safe(node, struct discard_cmd, rb_node); + } +} + +#ifdef CONFIG_BLK_DEV_ZONED +static void __queue_zone_reset_cmd(struct f2fs_sb_info *sbi, + struct block_device *bdev, block_t blkstart, block_t lblkstart, + block_t blklen) +{ + trace_f2fs_queue_reset_zone(bdev, blkstart); + + mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock); + __insert_discard_cmd(sbi, bdev, lblkstart, blkstart, blklen); + mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock); +} +#endif + +static void __queue_discard_cmd(struct f2fs_sb_info *sbi, + struct block_device *bdev, block_t blkstart, block_t blklen) +{ + block_t lblkstart = blkstart; + + if (!f2fs_bdev_support_discard(bdev)) + return; + + trace_f2fs_queue_discard(bdev, blkstart, blklen); + + if (f2fs_is_multi_device(sbi)) { + int devi = f2fs_target_device_index(sbi, blkstart); + + blkstart -= FDEV(devi).start_blk; + } + mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock); + __update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen); + mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock); +} + +static void __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi, + struct discard_policy *dpolicy, int *issued) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct discard_cmd *prev_dc = NULL, *next_dc = NULL; + struct rb_node **insert_p = NULL, *insert_parent = NULL; + struct discard_cmd *dc; + struct blk_plug plug; + bool io_interrupted = false; + + mutex_lock(&dcc->cmd_lock); + dc = __lookup_discard_cmd_ret(&dcc->root, dcc->next_pos, + &prev_dc, &next_dc, &insert_p, &insert_parent); + if (!dc) + dc = next_dc; + + blk_start_plug(&plug); + + while (dc) { + struct rb_node *node; + int err = 0; + + if (dc->state != D_PREP) + goto next; + + if (*issued > 0 && unlikely(freezing(current))) + break; + + if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) { + io_interrupted = true; + break; + } + + dcc->next_pos = dc->di.lstart + dc->di.len; + err = __submit_discard_cmd(sbi, dpolicy, dc, issued); + + if (*issued >= dpolicy->max_requests) + break; +next: + node = rb_next(&dc->rb_node); + if (err) + __remove_discard_cmd(sbi, dc); + dc = rb_entry_safe(node, struct discard_cmd, rb_node); + } + + blk_finish_plug(&plug); + + if (!dc) + dcc->next_pos = 0; + + mutex_unlock(&dcc->cmd_lock); + + if (!(*issued) && io_interrupted) + *issued = -1; +} +static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi, + struct discard_policy *dpolicy); + +static int __issue_discard_cmd(struct f2fs_sb_info *sbi, + struct discard_policy *dpolicy) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct list_head *pend_list; + struct discard_cmd *dc, *tmp; + struct blk_plug plug; + int i, issued; + bool io_interrupted = false; + bool suspended = false; + + if (dpolicy->timeout) + f2fs_update_time(sbi, UMOUNT_DISCARD_TIMEOUT); + +retry: + issued = 0; + for (i = MAX_PLIST_NUM - 1; i >= 0; i--) { + if (dpolicy->timeout && + f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT)) + break; + + if (i + 1 < dpolicy->granularity) + break; + + if (i + 1 < dcc->max_ordered_discard && dpolicy->ordered) { + __issue_discard_cmd_orderly(sbi, dpolicy, &issued); + return issued; + } + + pend_list = &dcc->pend_list[i]; + + mutex_lock(&dcc->cmd_lock); + if (list_empty(pend_list)) + goto next; + if (unlikely(dcc->rbtree_check)) + f2fs_bug_on(sbi, !f2fs_check_discard_tree(sbi)); + blk_start_plug(&plug); + list_for_each_entry_safe(dc, tmp, pend_list, list) { + f2fs_bug_on(sbi, dc->state != D_PREP); + + if (issued > 0 && unlikely(freezing(current))) { + suspended = true; + break; + } + + if (dpolicy->timeout && + f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT)) + break; + + if (dpolicy->io_aware && i < dpolicy->io_aware_gran && + !is_idle(sbi, DISCARD_TIME)) { + io_interrupted = true; + break; + } + + __submit_discard_cmd(sbi, dpolicy, dc, &issued); + + if (issued >= dpolicy->max_requests) + break; + } + blk_finish_plug(&plug); +next: + mutex_unlock(&dcc->cmd_lock); + + if (issued >= dpolicy->max_requests || io_interrupted || + suspended) + break; + } + + if (dpolicy->type == DPOLICY_UMOUNT && issued) { + __wait_all_discard_cmd(sbi, dpolicy); + goto retry; + } + + if (!issued && io_interrupted) + issued = -1; + + return issued; +} + +static bool __drop_discard_cmd(struct f2fs_sb_info *sbi) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct list_head *pend_list; + struct discard_cmd *dc, *tmp; + int i; + bool dropped = false; + + mutex_lock(&dcc->cmd_lock); + for (i = MAX_PLIST_NUM - 1; i >= 0; i--) { + pend_list = &dcc->pend_list[i]; + list_for_each_entry_safe(dc, tmp, pend_list, list) { + f2fs_bug_on(sbi, dc->state != D_PREP); + __remove_discard_cmd(sbi, dc); + dropped = true; + } + } + mutex_unlock(&dcc->cmd_lock); + + return dropped; +} + +void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi) +{ + __drop_discard_cmd(sbi); +} + +static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi, + struct discard_cmd *dc) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + unsigned int len = 0; + + wait_for_completion_io(&dc->wait); + mutex_lock(&dcc->cmd_lock); + f2fs_bug_on(sbi, dc->state != D_DONE); + dc->ref--; + if (!dc->ref) { + if (!dc->error) + len = dc->di.len; + __remove_discard_cmd(sbi, dc); + } + mutex_unlock(&dcc->cmd_lock); + + return len; +} + +static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi, + struct discard_policy *dpolicy, + block_t start, block_t end) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ? + &(dcc->fstrim_list) : &(dcc->wait_list); + struct discard_cmd *dc = NULL, *iter, *tmp; + unsigned int trimmed = 0; + +next: + dc = NULL; + + mutex_lock(&dcc->cmd_lock); + list_for_each_entry_safe(iter, tmp, wait_list, list) { + if (iter->di.lstart + iter->di.len <= start || + end <= iter->di.lstart) + continue; + if (iter->di.len < dpolicy->granularity) + continue; + if (iter->state == D_DONE && !iter->ref) { + wait_for_completion_io(&iter->wait); + if (!iter->error) + trimmed += iter->di.len; + __remove_discard_cmd(sbi, iter); + } else { + iter->ref++; + dc = iter; + break; + } + } + mutex_unlock(&dcc->cmd_lock); + + if (dc) { + trimmed += __wait_one_discard_bio(sbi, dc); + goto next; + } + + return trimmed; +} + +static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi, + struct discard_policy *dpolicy) +{ + struct discard_policy dp; + unsigned int discard_blks; + + if (dpolicy) + return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX); + + /* wait all */ + __init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, MIN_DISCARD_GRANULARITY); + discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX); + __init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, MIN_DISCARD_GRANULARITY); + discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX); + + return discard_blks; +} + +/* This should be covered by global mutex, &sit_i->sentry_lock */ +static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct discard_cmd *dc; + bool need_wait = false; + + mutex_lock(&dcc->cmd_lock); + dc = __lookup_discard_cmd(sbi, blkaddr); +#ifdef CONFIG_BLK_DEV_ZONED + if (dc && f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(dc->bdev)) { + int devi = f2fs_bdev_index(sbi, dc->bdev); + + if (devi < 0) { + mutex_unlock(&dcc->cmd_lock); + return; + } + + if (f2fs_blkz_is_seq(sbi, devi, dc->di.start)) { + /* force submit zone reset */ + if (dc->state == D_PREP) + __submit_zone_reset_cmd(sbi, dc, REQ_SYNC, + &dcc->wait_list, NULL); + dc->ref++; + mutex_unlock(&dcc->cmd_lock); + /* wait zone reset */ + __wait_one_discard_bio(sbi, dc); + return; + } + } +#endif + if (dc) { + if (dc->state == D_PREP) { + __punch_discard_cmd(sbi, dc, blkaddr); + } else { + dc->ref++; + need_wait = true; + } + } + mutex_unlock(&dcc->cmd_lock); + + if (need_wait) + __wait_one_discard_bio(sbi, dc); +} + +void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + + if (dcc && dcc->f2fs_issue_discard) { + struct task_struct *discard_thread = dcc->f2fs_issue_discard; + + dcc->f2fs_issue_discard = NULL; + kthread_stop(discard_thread); + } +} + +/** + * f2fs_issue_discard_timeout() - Issue all discard cmd within UMOUNT_DISCARD_TIMEOUT + * @sbi: the f2fs_sb_info data for discard cmd to issue + * + * When UMOUNT_DISCARD_TIMEOUT is exceeded, all remaining discard commands will be dropped + * + * Return true if issued all discard cmd or no discard cmd need issue, otherwise return false. + */ +bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi, bool need_check) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct discard_policy dpolicy; + bool dropped; + + if (!atomic_read(&dcc->discard_cmd_cnt)) + return true; + + __init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT, + dcc->discard_granularity); + __issue_discard_cmd(sbi, &dpolicy); + dropped = __drop_discard_cmd(sbi); + + /* just to make sure there is no pending discard commands */ + __wait_all_discard_cmd(sbi, NULL); + + f2fs_bug_on(sbi, need_check && atomic_read(&dcc->discard_cmd_cnt)); + return !dropped; +} + +static int issue_discard_thread(void *data) +{ + struct f2fs_sb_info *sbi = data; + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + wait_queue_head_t *q = &dcc->discard_wait_queue; + struct discard_policy dpolicy; + unsigned int wait_ms = dcc->min_discard_issue_time; + int issued; + + set_freezable(); + + do { + wait_event_freezable_timeout(*q, + kthread_should_stop() || dcc->discard_wake, + msecs_to_jiffies(wait_ms)); + + if (sbi->gc_mode == GC_URGENT_HIGH || + !f2fs_available_free_memory(sbi, DISCARD_CACHE)) + __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, + MIN_DISCARD_GRANULARITY); + else + __init_discard_policy(sbi, &dpolicy, DPOLICY_BG, + dcc->discard_granularity); + + if (dcc->discard_wake) + dcc->discard_wake = false; + + /* clean up pending candidates before going to sleep */ + if (atomic_read(&dcc->queued_discard)) + __wait_all_discard_cmd(sbi, NULL); + + if (f2fs_readonly(sbi->sb)) + continue; + if (kthread_should_stop()) + return 0; + if (is_sbi_flag_set(sbi, SBI_NEED_FSCK) || + !atomic_read(&dcc->discard_cmd_cnt)) { + wait_ms = dpolicy.max_interval; + continue; + } + + sb_start_intwrite(sbi->sb); + + issued = __issue_discard_cmd(sbi, &dpolicy); + if (issued > 0) { + __wait_all_discard_cmd(sbi, &dpolicy); + wait_ms = dpolicy.min_interval; + } else if (issued == -1) { + wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME); + if (!wait_ms) + wait_ms = dpolicy.mid_interval; + } else { + wait_ms = dpolicy.max_interval; + } + if (!atomic_read(&dcc->discard_cmd_cnt)) + wait_ms = dpolicy.max_interval; + + sb_end_intwrite(sbi->sb); + + } while (!kthread_should_stop()); + return 0; +} + +#ifdef CONFIG_BLK_DEV_ZONED +static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi, + struct block_device *bdev, block_t blkstart, block_t blklen) +{ + sector_t sector, nr_sects; + block_t lblkstart = blkstart; + int devi = 0; + u64 remainder = 0; + + if (f2fs_is_multi_device(sbi)) { + devi = f2fs_target_device_index(sbi, blkstart); + if (blkstart < FDEV(devi).start_blk || + blkstart > FDEV(devi).end_blk) { + f2fs_err(sbi, "Invalid block %x", blkstart); + return -EIO; + } + blkstart -= FDEV(devi).start_blk; + } + + /* For sequential zones, reset the zone write pointer */ + if (f2fs_blkz_is_seq(sbi, devi, blkstart)) { + sector = SECTOR_FROM_BLOCK(blkstart); + nr_sects = SECTOR_FROM_BLOCK(blklen); + div64_u64_rem(sector, bdev_zone_sectors(bdev), &remainder); + + if (remainder || nr_sects != bdev_zone_sectors(bdev)) { + f2fs_err(sbi, "(%d) %s: Unaligned zone reset attempted (block %x + %x)", + devi, sbi->s_ndevs ? FDEV(devi).path : "", + blkstart, blklen); + return -EIO; + } + + if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) { + unsigned int nofs_flags; + int ret; + + trace_f2fs_issue_reset_zone(bdev, blkstart); + nofs_flags = memalloc_nofs_save(); + ret = blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET, + sector, nr_sects); + memalloc_nofs_restore(nofs_flags); + return ret; + } + + __queue_zone_reset_cmd(sbi, bdev, blkstart, lblkstart, blklen); + return 0; + } + + /* For conventional zones, use regular discard if supported */ + __queue_discard_cmd(sbi, bdev, lblkstart, blklen); + return 0; +} +#endif + +static int __issue_discard_async(struct f2fs_sb_info *sbi, + struct block_device *bdev, block_t blkstart, block_t blklen) +{ +#ifdef CONFIG_BLK_DEV_ZONED + if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev)) + return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen); +#endif + __queue_discard_cmd(sbi, bdev, blkstart, blklen); + return 0; +} + +static int f2fs_issue_discard(struct f2fs_sb_info *sbi, + block_t blkstart, block_t blklen) +{ + sector_t start = blkstart, len = 0; + struct block_device *bdev; + struct seg_entry *se; + unsigned int offset; + block_t i; + int err = 0; + + bdev = f2fs_target_device(sbi, blkstart, NULL); + + for (i = blkstart; i < blkstart + blklen; i++, len++) { + if (i != start) { + struct block_device *bdev2 = + f2fs_target_device(sbi, i, NULL); + + if (bdev2 != bdev) { + err = __issue_discard_async(sbi, bdev, + start, len); + if (err) + return err; + bdev = bdev2; + start = i; + len = 0; + } + } + + se = get_seg_entry(sbi, GET_SEGNO(sbi, i)); + offset = GET_BLKOFF_FROM_SEG0(sbi, i); + + if (f2fs_block_unit_discard(sbi) && + !f2fs_test_and_set_bit(offset, se->discard_map)) + sbi->discard_blks--; + } + + if (len) + err = __issue_discard_async(sbi, bdev, start, len); + return err; +} + +static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc, + bool check_only) +{ + int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long); + struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start); + unsigned long *cur_map = (unsigned long *)se->cur_valid_map; + unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map; + unsigned long *discard_map = (unsigned long *)se->discard_map; + unsigned long *dmap = SIT_I(sbi)->tmp_map; + unsigned int start = 0, end = -1; + bool force = (cpc->reason & CP_DISCARD); + struct discard_entry *de = NULL; + struct list_head *head = &SM_I(sbi)->dcc_info->entry_list; + int i; + + if (se->valid_blocks == BLKS_PER_SEG(sbi) || + !f2fs_hw_support_discard(sbi) || + !f2fs_block_unit_discard(sbi)) + return false; + + if (!force) { + if (!f2fs_realtime_discard_enable(sbi) || + (!se->valid_blocks && + !is_curseg(sbi, cpc->trim_start)) || + SM_I(sbi)->dcc_info->nr_discards >= + SM_I(sbi)->dcc_info->max_discards) + return false; + } + + /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */ + for (i = 0; i < entries; i++) + dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] : + (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i]; + + while (force || SM_I(sbi)->dcc_info->nr_discards <= + SM_I(sbi)->dcc_info->max_discards) { + start = __find_rev_next_bit(dmap, BLKS_PER_SEG(sbi), end + 1); + if (start >= BLKS_PER_SEG(sbi)) + break; + + end = __find_rev_next_zero_bit(dmap, + BLKS_PER_SEG(sbi), start + 1); + if (force && start && end != BLKS_PER_SEG(sbi) && + (end - start) < cpc->trim_minlen) + continue; + + if (check_only) + return true; + + if (!de) { + de = f2fs_kmem_cache_alloc(discard_entry_slab, + GFP_F2FS_ZERO, true, NULL); + de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start); + list_add_tail(&de->list, head); + } + + for (i = start; i < end; i++) + __set_bit_le(i, (void *)de->discard_map); + + SM_I(sbi)->dcc_info->nr_discards += end - start; + } + return false; +} + +static void release_discard_addr(struct discard_entry *entry) +{ + list_del(&entry->list); + kmem_cache_free(discard_entry_slab, entry); +} + +void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi) +{ + struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list); + struct discard_entry *entry, *this; + + /* drop caches */ + list_for_each_entry_safe(entry, this, head, list) + release_discard_addr(entry); +} + +/* + * Should call f2fs_clear_prefree_segments after checkpoint is done. + */ +static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + unsigned int segno; + + mutex_lock(&dirty_i->seglist_lock); + for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi)) + __set_test_and_free(sbi, segno, false); + mutex_unlock(&dirty_i->seglist_lock); +} + +void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi, + struct cp_control *cpc) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct list_head *head = &dcc->entry_list; + struct discard_entry *entry, *this; + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + unsigned long *prefree_map = dirty_i->dirty_segmap[PRE]; + unsigned int start = 0, end = -1; + unsigned int secno, start_segno; + bool force = (cpc->reason & CP_DISCARD); + bool section_alignment = F2FS_OPTION(sbi).discard_unit == + DISCARD_UNIT_SECTION; + + if (f2fs_lfs_mode(sbi) && __is_large_section(sbi)) + section_alignment = true; + + mutex_lock(&dirty_i->seglist_lock); + + while (1) { + int i; + + if (section_alignment && end != -1) + end--; + start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1); + if (start >= MAIN_SEGS(sbi)) + break; + end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi), + start + 1); + + if (section_alignment) { + start = rounddown(start, SEGS_PER_SEC(sbi)); + end = roundup(end, SEGS_PER_SEC(sbi)); + } + + for (i = start; i < end; i++) { + if (test_and_clear_bit(i, prefree_map)) + dirty_i->nr_dirty[PRE]--; + } + + if (!f2fs_realtime_discard_enable(sbi)) + continue; + + if (force && start >= cpc->trim_start && + (end - 1) <= cpc->trim_end) + continue; + + /* Should cover 2MB zoned device for zone-based reset */ + if (!f2fs_sb_has_blkzoned(sbi) && + (!f2fs_lfs_mode(sbi) || !__is_large_section(sbi))) { + f2fs_issue_discard(sbi, START_BLOCK(sbi, start), + SEGS_TO_BLKS(sbi, end - start)); + continue; + } +next: + secno = GET_SEC_FROM_SEG(sbi, start); + start_segno = GET_SEG_FROM_SEC(sbi, secno); + if (!is_cursec(sbi, secno) && + !get_valid_blocks(sbi, start, true)) + f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno), + BLKS_PER_SEC(sbi)); + + start = start_segno + SEGS_PER_SEC(sbi); + if (start < end) + goto next; + else + end = start - 1; + } + mutex_unlock(&dirty_i->seglist_lock); + + if (!f2fs_block_unit_discard(sbi)) + goto wakeup; + + /* send small discards */ + list_for_each_entry_safe(entry, this, head, list) { + unsigned int cur_pos = 0, next_pos, len, total_len = 0; + bool is_valid = test_bit_le(0, entry->discard_map); + +find_next: + if (is_valid) { + next_pos = find_next_zero_bit_le(entry->discard_map, + BLKS_PER_SEG(sbi), cur_pos); + len = next_pos - cur_pos; + + if (f2fs_sb_has_blkzoned(sbi) || + (force && len < cpc->trim_minlen)) + goto skip; + + f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos, + len); + total_len += len; + } else { + next_pos = find_next_bit_le(entry->discard_map, + BLKS_PER_SEG(sbi), cur_pos); + } +skip: + cur_pos = next_pos; + is_valid = !is_valid; + + if (cur_pos < BLKS_PER_SEG(sbi)) + goto find_next; + + release_discard_addr(entry); + dcc->nr_discards -= total_len; + } + +wakeup: + wake_up_discard_thread(sbi, false); +} + +int f2fs_start_discard_thread(struct f2fs_sb_info *sbi) +{ + dev_t dev = sbi->sb->s_bdev->bd_dev; + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + int err = 0; + + if (f2fs_sb_has_readonly(sbi)) { + f2fs_info(sbi, + "Skip to start discard thread for readonly image"); + return 0; + } + + if (!f2fs_realtime_discard_enable(sbi)) + return 0; + + dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi, + "f2fs_discard-%u:%u", MAJOR(dev), MINOR(dev)); + if (IS_ERR(dcc->f2fs_issue_discard)) { + err = PTR_ERR(dcc->f2fs_issue_discard); + dcc->f2fs_issue_discard = NULL; + } + + return err; +} + +static int create_discard_cmd_control(struct f2fs_sb_info *sbi) +{ + struct discard_cmd_control *dcc; + int err = 0, i; + + if (SM_I(sbi)->dcc_info) { + dcc = SM_I(sbi)->dcc_info; + goto init_thread; + } + + dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL); + if (!dcc) + return -ENOMEM; + + dcc->discard_io_aware_gran = MAX_PLIST_NUM; + dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY; + dcc->max_ordered_discard = DEFAULT_MAX_ORDERED_DISCARD_GRANULARITY; + dcc->discard_io_aware = DPOLICY_IO_AWARE_ENABLE; + if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SEGMENT || + F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SECTION) + dcc->discard_granularity = BLKS_PER_SEG(sbi); + + INIT_LIST_HEAD(&dcc->entry_list); + for (i = 0; i < MAX_PLIST_NUM; i++) + INIT_LIST_HEAD(&dcc->pend_list[i]); + INIT_LIST_HEAD(&dcc->wait_list); + INIT_LIST_HEAD(&dcc->fstrim_list); + mutex_init(&dcc->cmd_lock); + atomic_set(&dcc->issued_discard, 0); + atomic_set(&dcc->queued_discard, 0); + atomic_set(&dcc->discard_cmd_cnt, 0); + dcc->nr_discards = 0; + dcc->max_discards = SEGS_TO_BLKS(sbi, MAIN_SEGS(sbi)); + dcc->max_discard_request = DEF_MAX_DISCARD_REQUEST; + dcc->min_discard_issue_time = DEF_MIN_DISCARD_ISSUE_TIME; + dcc->mid_discard_issue_time = DEF_MID_DISCARD_ISSUE_TIME; + dcc->max_discard_issue_time = DEF_MAX_DISCARD_ISSUE_TIME; + dcc->discard_urgent_util = DEF_DISCARD_URGENT_UTIL; + dcc->undiscard_blks = 0; + dcc->next_pos = 0; + dcc->root = RB_ROOT_CACHED; + dcc->rbtree_check = false; + + init_waitqueue_head(&dcc->discard_wait_queue); + SM_I(sbi)->dcc_info = dcc; +init_thread: + err = f2fs_start_discard_thread(sbi); + if (err) { + kfree(dcc); + SM_I(sbi)->dcc_info = NULL; + } + + return err; +} + +static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + + if (!dcc) + return; + + f2fs_stop_discard_thread(sbi); + + /* + * Recovery can cache discard commands, so in error path of + * fill_super(), it needs to give a chance to handle them. + */ + f2fs_issue_discard_timeout(sbi, true); + + kfree(dcc); + SM_I(sbi)->dcc_info = NULL; +} + +static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno) +{ + struct sit_info *sit_i = SIT_I(sbi); + + if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) { + sit_i->dirty_sentries++; + return false; + } + + return true; +} + +static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type, + unsigned int segno, int modified) +{ + struct seg_entry *se = get_seg_entry(sbi, segno); + + se->type = type; + if (modified) + __mark_sit_entry_dirty(sbi, segno); +} + +static inline unsigned long long get_segment_mtime(struct f2fs_sb_info *sbi, + block_t blkaddr) +{ + unsigned int segno = GET_SEGNO(sbi, blkaddr); + + if (segno == NULL_SEGNO) + return 0; + return get_seg_entry(sbi, segno)->mtime; +} + +static void update_segment_mtime(struct f2fs_sb_info *sbi, block_t blkaddr, + unsigned long long old_mtime) +{ + struct seg_entry *se; + unsigned int segno = GET_SEGNO(sbi, blkaddr); + unsigned long long ctime = get_mtime(sbi, false); + unsigned long long mtime = old_mtime ? old_mtime : ctime; + + if (segno == NULL_SEGNO) + return; + + se = get_seg_entry(sbi, segno); + + if (!se->mtime) + se->mtime = mtime; + else + se->mtime = div_u64(se->mtime * se->valid_blocks + mtime, + se->valid_blocks + 1); + + if (ctime > SIT_I(sbi)->max_mtime) + SIT_I(sbi)->max_mtime = ctime; +} + +/* + * NOTE: when updating multiple blocks at the same time, please ensure + * that the consecutive input blocks belong to the same segment. + */ +static int update_sit_entry_for_release(struct f2fs_sb_info *sbi, struct seg_entry *se, + unsigned int segno, block_t blkaddr, unsigned int offset, int del) +{ + bool exist; + int i; + int del_count = -del; + + f2fs_bug_on(sbi, GET_SEGNO(sbi, blkaddr) != GET_SEGNO(sbi, blkaddr + del_count - 1)); + + for (i = 0; i < del_count; i++) { + exist = f2fs_test_and_clear_bit(offset + i, se->cur_valid_map); + if (unlikely(!exist)) { + f2fs_err(sbi, "Bitmap was wrongly cleared, blk:%u", blkaddr + i); + f2fs_bug_on(sbi, 1); + se->valid_blocks++; + del += 1; + } else if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { + /* + * If checkpoints are off, we must not reuse data that + * was used in the previous checkpoint. If it was used + * before, we must track that to know how much space we + * really have. + */ + if (f2fs_test_bit(offset + i, se->ckpt_valid_map)) { + spin_lock(&sbi->stat_lock); + sbi->unusable_block_count++; + spin_unlock(&sbi->stat_lock); + } + } + + if (f2fs_block_unit_discard(sbi) && + f2fs_test_and_clear_bit(offset + i, se->discard_map)) + sbi->discard_blks++; + + if (!f2fs_test_bit(offset + i, se->ckpt_valid_map)) { + se->ckpt_valid_blocks -= 1; + if (__is_large_section(sbi)) + get_sec_entry(sbi, segno)->ckpt_valid_blocks -= 1; + } + } + + if (__is_large_section(sbi)) + sanity_check_valid_blocks(sbi, segno); + + return del; +} + +static int update_sit_entry_for_alloc(struct f2fs_sb_info *sbi, struct seg_entry *se, + unsigned int segno, block_t blkaddr, unsigned int offset, int del) +{ + bool exist; + + exist = f2fs_test_and_set_bit(offset, se->cur_valid_map); + if (unlikely(exist)) { + f2fs_err(sbi, "Bitmap was wrongly set, blk:%u", blkaddr); + f2fs_bug_on(sbi, 1); + se->valid_blocks--; + del = 0; + } + + if (f2fs_block_unit_discard(sbi) && + !f2fs_test_and_set_bit(offset, se->discard_map)) + sbi->discard_blks--; + + /* + * SSR should never reuse block which is checkpointed + * or newly invalidated. + */ + if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) { + if (!f2fs_test_and_set_bit(offset, se->ckpt_valid_map)) { + se->ckpt_valid_blocks++; + if (__is_large_section(sbi)) + get_sec_entry(sbi, segno)->ckpt_valid_blocks++; + } + } + + if (!f2fs_test_bit(offset, se->ckpt_valid_map)) { + se->ckpt_valid_blocks += del; + if (__is_large_section(sbi)) + get_sec_entry(sbi, segno)->ckpt_valid_blocks += del; + } + + if (__is_large_section(sbi)) + sanity_check_valid_blocks(sbi, segno); + + return del; +} + +/* + * If releasing blocks, this function supports updating multiple consecutive blocks + * at one time, but please note that these consecutive blocks need to belong to the + * same segment. + */ +static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del) +{ + struct seg_entry *se; + unsigned int segno, offset; + long int new_vblocks; + + segno = GET_SEGNO(sbi, blkaddr); + if (segno == NULL_SEGNO) + return; + + se = get_seg_entry(sbi, segno); + new_vblocks = se->valid_blocks + del; + offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); + + f2fs_bug_on(sbi, (new_vblocks < 0 || + (new_vblocks > f2fs_usable_blks_in_seg(sbi, segno)))); + + se->valid_blocks = new_vblocks; + + /* Update valid block bitmap */ + if (del > 0) { + del = update_sit_entry_for_alloc(sbi, se, segno, blkaddr, offset, del); + } else { + del = update_sit_entry_for_release(sbi, se, segno, blkaddr, offset, del); + } + + __mark_sit_entry_dirty(sbi, segno); + + /* update total number of valid blocks to be written in ckpt area */ + SIT_I(sbi)->written_valid_blocks += del; + + if (__is_large_section(sbi)) + get_sec_entry(sbi, segno)->valid_blocks += del; +} + +void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr, + unsigned int len) +{ + unsigned int segno = GET_SEGNO(sbi, addr); + struct sit_info *sit_i = SIT_I(sbi); + block_t addr_start = addr, addr_end = addr + len - 1; + unsigned int seg_num = GET_SEGNO(sbi, addr_end) - segno + 1; + unsigned int i = 1, max_blocks = sbi->blocks_per_seg, cnt; + + f2fs_bug_on(sbi, addr == NULL_ADDR); + if (addr == NEW_ADDR || addr == COMPRESS_ADDR) + return; + + f2fs_invalidate_internal_cache(sbi, addr, len); + + /* add it into sit main buffer */ + down_write(&sit_i->sentry_lock); + + if (seg_num == 1) + cnt = len; + else + cnt = max_blocks - GET_BLKOFF_FROM_SEG0(sbi, addr); + + do { + update_segment_mtime(sbi, addr_start, 0); + update_sit_entry(sbi, addr_start, -cnt); + + /* add it into dirty seglist */ + locate_dirty_segment(sbi, segno); + + /* update @addr_start and @cnt and @segno */ + addr_start = START_BLOCK(sbi, ++segno); + if (++i == seg_num) + cnt = GET_BLKOFF_FROM_SEG0(sbi, addr_end) + 1; + else + cnt = max_blocks; + } while (i <= seg_num); + + up_write(&sit_i->sentry_lock); +} + +bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr) +{ + struct sit_info *sit_i = SIT_I(sbi); + unsigned int segno, offset; + struct seg_entry *se; + bool is_cp = false; + + if (!__is_valid_data_blkaddr(blkaddr)) + return true; + + down_read(&sit_i->sentry_lock); + + segno = GET_SEGNO(sbi, blkaddr); + se = get_seg_entry(sbi, segno); + offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr); + + if (f2fs_test_bit(offset, se->ckpt_valid_map)) + is_cp = true; + + up_read(&sit_i->sentry_lock); + + return is_cp; +} + +static unsigned short f2fs_curseg_valid_blocks(struct f2fs_sb_info *sbi, int type) +{ + struct curseg_info *curseg = CURSEG_I(sbi, type); + + if (sbi->ckpt->alloc_type[type] == SSR) + return BLKS_PER_SEG(sbi); + return curseg->next_blkoff; +} + +/* + * Calculate the number of current summary pages for writing + */ +int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra) +{ + int valid_sum_count = 0; + int i, sum_in_page; + + for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { + if (sbi->ckpt->alloc_type[i] != SSR && for_ra) + valid_sum_count += + le16_to_cpu(F2FS_CKPT(sbi)->cur_data_blkoff[i]); + else + valid_sum_count += f2fs_curseg_valid_blocks(sbi, i); + } + + sum_in_page = (sbi->blocksize - 2 * sbi->sum_journal_size - + SUM_FOOTER_SIZE) / SUMMARY_SIZE; + if (valid_sum_count <= sum_in_page) + return 1; + else if ((valid_sum_count - sum_in_page) <= + (sbi->blocksize - SUM_FOOTER_SIZE) / SUMMARY_SIZE) + return 2; + return 3; +} + +/* + * Caller should put this summary folio + */ +struct folio *f2fs_get_sum_folio(struct f2fs_sb_info *sbi, unsigned int segno) +{ + if (unlikely(f2fs_cp_error(sbi))) + return ERR_PTR(-EIO); + return f2fs_get_meta_folio_retry(sbi, GET_SUM_BLOCK(sbi, segno)); +} + +void f2fs_update_meta_page(struct f2fs_sb_info *sbi, + void *src, block_t blk_addr) +{ + struct folio *folio; + + if (!f2fs_sb_has_packed_ssa(sbi)) + folio = f2fs_grab_meta_folio(sbi, blk_addr); + else + folio = f2fs_get_meta_folio_retry(sbi, blk_addr); + + if (IS_ERR(folio)) + return; + + memcpy(folio_address(folio), src, PAGE_SIZE); + folio_mark_dirty(folio); + f2fs_folio_put(folio, true); +} + +static void write_sum_page(struct f2fs_sb_info *sbi, + struct f2fs_summary_block *sum_blk, unsigned int segno) +{ + struct folio *folio; + + if (!f2fs_sb_has_packed_ssa(sbi)) + return f2fs_update_meta_page(sbi, (void *)sum_blk, + GET_SUM_BLOCK(sbi, segno)); + + folio = f2fs_get_sum_folio(sbi, segno); + if (IS_ERR(folio)) + return; + + memcpy(SUM_BLK_PAGE_ADDR(sbi, folio, segno), sum_blk, + sbi->sum_blocksize); + folio_mark_dirty(folio); + f2fs_folio_put(folio, true); +} + +static void write_current_sum_page(struct f2fs_sb_info *sbi, + int type, block_t blk_addr) +{ + struct curseg_info *curseg = CURSEG_I(sbi, type); + struct folio *folio = f2fs_grab_meta_folio(sbi, blk_addr); + struct f2fs_summary_block *src = curseg->sum_blk; + struct f2fs_summary_block *dst; + + dst = folio_address(folio); + memset(dst, 0, PAGE_SIZE); + + mutex_lock(&curseg->curseg_mutex); + + down_read(&curseg->journal_rwsem); + memcpy(sum_journal(sbi, dst), curseg->journal, sbi->sum_journal_size); + up_read(&curseg->journal_rwsem); + + memcpy(sum_entries(dst), sum_entries(src), sbi->sum_entry_size); + memcpy(sum_footer(sbi, dst), sum_footer(sbi, src), SUM_FOOTER_SIZE); + + mutex_unlock(&curseg->curseg_mutex); + + folio_mark_dirty(folio); + f2fs_folio_put(folio, true); +} + +static int is_next_segment_free(struct f2fs_sb_info *sbi, + struct curseg_info *curseg) +{ + unsigned int segno = curseg->segno + 1; + struct free_segmap_info *free_i = FREE_I(sbi); + + if (segno < MAIN_SEGS(sbi) && segno % SEGS_PER_SEC(sbi)) + return !test_bit(segno, free_i->free_segmap); + return 0; +} + +/* + * Find a new segment from the free segments bitmap to right order + * This function should be returned with success, otherwise BUG + */ +static int get_new_segment(struct f2fs_sb_info *sbi, + unsigned int *newseg, bool new_sec, bool pinning) +{ + struct free_segmap_info *free_i = FREE_I(sbi); + unsigned int segno, secno, zoneno; + unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone; + unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg); + unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg); + unsigned int alloc_policy = sbi->allocate_section_policy; + unsigned int alloc_hint = sbi->allocate_section_hint; + bool init = true; + int i; + int ret = 0; + + spin_lock(&free_i->segmap_lock); + + if (time_to_inject(sbi, FAULT_NO_SEGMENT)) { + ret = -ENOSPC; + goto out_unlock; + } + + if (!new_sec && ((*newseg + 1) % SEGS_PER_SEC(sbi))) { + segno = find_next_zero_bit(free_i->free_segmap, + GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1); + if (segno < GET_SEG_FROM_SEC(sbi, hint + 1)) + goto got_it; + } + +#ifdef CONFIG_BLK_DEV_ZONED + /* + * If we format f2fs on zoned storage, let's try to get pinned sections + * from beginning of the storage, which should be a conventional one. + */ + if (f2fs_sb_has_blkzoned(sbi)) { + /* Prioritize writing to conventional zones */ + if (sbi->blkzone_alloc_policy == BLKZONE_ALLOC_PRIOR_CONV || pinning) + segno = 0; + else + segno = max(sbi->first_seq_zone_segno, *newseg); + hint = GET_SEC_FROM_SEG(sbi, segno); + } +#endif + + /* + * Prevent allocate_section_hint from exceeding MAIN_SECS() + * due to desynchronization. + */ + if (alloc_policy != ALLOCATE_FORWARD_NOHINT && + alloc_hint > MAIN_SECS(sbi)) + alloc_hint = MAIN_SECS(sbi); + + if (alloc_policy == ALLOCATE_FORWARD_FROM_HINT && + hint < alloc_hint) + hint = alloc_hint; + else if (alloc_policy == ALLOCATE_FORWARD_WITHIN_HINT && + hint >= alloc_hint) + hint = 0; + +find_other_zone: + secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint); + +#ifdef CONFIG_BLK_DEV_ZONED + if (secno >= MAIN_SECS(sbi) && f2fs_sb_has_blkzoned(sbi)) { + /* Write only to sequential zones */ + if (sbi->blkzone_alloc_policy == BLKZONE_ALLOC_ONLY_SEQ) { + hint = GET_SEC_FROM_SEG(sbi, sbi->first_seq_zone_segno); + secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint); + } else + secno = find_first_zero_bit(free_i->free_secmap, + MAIN_SECS(sbi)); + if (secno >= MAIN_SECS(sbi)) { + ret = -ENOSPC; + f2fs_bug_on(sbi, 1); + goto out_unlock; + } + } +#endif + + if (secno >= MAIN_SECS(sbi)) { + secno = find_first_zero_bit(free_i->free_secmap, + MAIN_SECS(sbi)); + if (secno >= MAIN_SECS(sbi)) { + ret = -ENOSPC; + f2fs_bug_on(sbi, !pinning); + goto out_unlock; + } + } + segno = GET_SEG_FROM_SEC(sbi, secno); + zoneno = GET_ZONE_FROM_SEC(sbi, secno); + + /* give up on finding another zone */ + if (!init) + goto got_it; + if (sbi->secs_per_zone == 1) + goto got_it; + if (zoneno == old_zoneno) + goto got_it; + for (i = 0; i < NR_CURSEG_TYPE; i++) + if (CURSEG_I(sbi, i)->zone == zoneno) + break; + + if (i < NR_CURSEG_TYPE) { + /* zone is in user, try another */ + if (zoneno + 1 >= total_zones) + hint = 0; + else + hint = (zoneno + 1) * sbi->secs_per_zone; + init = false; + goto find_other_zone; + } +got_it: + /* set it as dirty segment in free segmap */ + if (test_bit(segno, free_i->free_segmap)) { + ret = -EFSCORRUPTED; + fserror_report_metadata(sbi->sb, -EFSCORRUPTED, GFP_NOFS); + f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_CORRUPTED_FREE_BITMAP); + goto out_unlock; + } + + /* no free section in conventional device or conventional zone */ + if (new_sec && pinning && + f2fs_is_sequential_zone_area(sbi, START_BLOCK(sbi, segno))) { + ret = -EAGAIN; + goto out_unlock; + } + __set_inuse(sbi, segno); + *newseg = segno; +out_unlock: + spin_unlock(&free_i->segmap_lock); + + if (ret == -ENOSPC && !pinning) + f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_NO_SEGMENT); + return ret; +} + +static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified) +{ + struct curseg_info *curseg = CURSEG_I(sbi, type); + struct summary_footer *sum_footer; + unsigned short seg_type = curseg->seg_type; + + /* only happen when get_new_segment() fails */ + if (curseg->next_segno == NULL_SEGNO) + return; + + curseg->inited = true; + curseg->segno = curseg->next_segno; + curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno); + curseg->next_blkoff = 0; + curseg->next_segno = NULL_SEGNO; + + sum_footer = sum_footer(sbi, curseg->sum_blk); + memset(sum_footer, 0, sizeof(struct summary_footer)); + + sanity_check_seg_type(sbi, seg_type); + + if (IS_DATASEG(seg_type)) + SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA); + if (IS_NODESEG(seg_type)) + SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE); + __set_sit_entry_type(sbi, seg_type, curseg->segno, modified); +} + +static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type) +{ + struct curseg_info *curseg = CURSEG_I(sbi, type); + unsigned short seg_type = curseg->seg_type; + + sanity_check_seg_type(sbi, seg_type); + if (__is_large_section(sbi)) { + if (f2fs_need_rand_seg(sbi)) { + unsigned int hint = GET_SEC_FROM_SEG(sbi, curseg->segno); + + if (GET_SEC_FROM_SEG(sbi, curseg->segno + 1) != hint) + return curseg->segno; + return get_random_u32_inclusive(curseg->segno + 1, + GET_SEG_FROM_SEC(sbi, hint + 1) - 1); + } + return curseg->segno; + } else if (f2fs_need_rand_seg(sbi)) { + return get_random_u32_below(MAIN_SECS(sbi) * SEGS_PER_SEC(sbi)); + } + + /* inmem log may not locate on any segment after mount */ + if (!curseg->inited) + return 0; + + if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) + return 0; + + if (seg_type == CURSEG_HOT_DATA || IS_NODESEG(seg_type)) + return 0; + + if (SIT_I(sbi)->last_victim[ALLOC_NEXT]) + return SIT_I(sbi)->last_victim[ALLOC_NEXT]; + + /* find segments from 0 to reuse freed segments */ + if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE) + return 0; + + return curseg->segno; +} + +static void reset_curseg_fields(struct curseg_info *curseg) +{ + curseg->inited = false; + curseg->segno = NULL_SEGNO; + curseg->next_segno = 0; +} + +/* + * Allocate a current working segment. + * This function always allocates a free segment in LFS manner. + */ +static int new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec) +{ + struct curseg_info *curseg = CURSEG_I(sbi, type); + unsigned int segno = curseg->segno; + bool pinning = type == CURSEG_COLD_DATA_PINNED; + int ret; + + if (curseg->inited) + write_sum_page(sbi, curseg->sum_blk, segno); + + segno = __get_next_segno(sbi, type); + ret = get_new_segment(sbi, &segno, new_sec, pinning); + if (ret) { + if (ret == -ENOSPC) + reset_curseg_fields(curseg); + return ret; + } + + curseg->next_segno = segno; + reset_curseg(sbi, type, 1); + curseg->alloc_type = LFS; + if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK) + curseg->fragment_remained_chunk = + get_random_u32_inclusive(1, sbi->max_fragment_chunk); + return 0; +} + +static int __next_free_blkoff(struct f2fs_sb_info *sbi, + int segno, block_t start) +{ + struct seg_entry *se = get_seg_entry(sbi, segno); + int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long); + unsigned long *target_map = SIT_I(sbi)->tmp_map; + unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map; + unsigned long *cur_map = (unsigned long *)se->cur_valid_map; + int i; + + for (i = 0; i < entries; i++) + target_map[i] = ckpt_map[i] | cur_map[i]; + + return __find_rev_next_zero_bit(target_map, BLKS_PER_SEG(sbi), start); +} + +static int f2fs_find_next_ssr_block(struct f2fs_sb_info *sbi, + struct curseg_info *seg) +{ + return __next_free_blkoff(sbi, seg->segno, seg->next_blkoff + 1); +} + +bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno) +{ + return __next_free_blkoff(sbi, segno, 0) < BLKS_PER_SEG(sbi); +} + +/* + * This function always allocates a used segment(from dirty seglist) by SSR + * manner, so it should recover the existing segment information of valid blocks + */ +static int change_curseg(struct f2fs_sb_info *sbi, int type) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + struct curseg_info *curseg = CURSEG_I(sbi, type); + unsigned int new_segno = curseg->next_segno; + struct f2fs_summary_block *sum_node; + struct folio *sum_folio; + + if (curseg->inited) + write_sum_page(sbi, curseg->sum_blk, curseg->segno); + + __set_test_and_inuse(sbi, new_segno); + + mutex_lock(&dirty_i->seglist_lock); + __remove_dirty_segment(sbi, new_segno, PRE); + __remove_dirty_segment(sbi, new_segno, DIRTY); + mutex_unlock(&dirty_i->seglist_lock); + + reset_curseg(sbi, type, 1); + curseg->alloc_type = SSR; + curseg->next_blkoff = __next_free_blkoff(sbi, curseg->segno, 0); + + sum_folio = f2fs_get_sum_folio(sbi, new_segno); + if (IS_ERR(sum_folio)) { + /* GC won't be able to use stale summary pages by cp_error */ + memset(curseg->sum_blk, 0, sbi->sum_entry_size); + return PTR_ERR(sum_folio); + } + sum_node = SUM_BLK_PAGE_ADDR(sbi, sum_folio, new_segno); + memcpy(curseg->sum_blk, sum_node, sbi->sum_entry_size); + f2fs_folio_put(sum_folio, true); + return 0; +} + +static int get_ssr_segment(struct f2fs_sb_info *sbi, int type, + int alloc_mode, unsigned long long age); + +static int get_atssr_segment(struct f2fs_sb_info *sbi, int type, + int target_type, int alloc_mode, + unsigned long long age) +{ + struct curseg_info *curseg = CURSEG_I(sbi, type); + int ret = 0; + + curseg->seg_type = target_type; + + if (get_ssr_segment(sbi, type, alloc_mode, age)) { + struct seg_entry *se = get_seg_entry(sbi, curseg->next_segno); + + curseg->seg_type = se->type; + ret = change_curseg(sbi, type); + } else { + /* allocate cold segment by default */ + curseg->seg_type = CURSEG_COLD_DATA; + ret = new_curseg(sbi, type, true); + } + stat_inc_seg_type(sbi, curseg); + return ret; +} + +static int __f2fs_init_atgc_curseg(struct f2fs_sb_info *sbi, bool force) +{ + struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_ALL_DATA_ATGC); + int ret = 0; + + if (!sbi->am.atgc_enabled && !force) + return 0; + + f2fs_down_read(&SM_I(sbi)->curseg_lock); + + mutex_lock(&curseg->curseg_mutex); + down_write(&SIT_I(sbi)->sentry_lock); + + ret = get_atssr_segment(sbi, CURSEG_ALL_DATA_ATGC, + CURSEG_COLD_DATA, SSR, 0); + + up_write(&SIT_I(sbi)->sentry_lock); + mutex_unlock(&curseg->curseg_mutex); + + f2fs_up_read(&SM_I(sbi)->curseg_lock); + return ret; +} + +int f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi) +{ + return __f2fs_init_atgc_curseg(sbi, false); +} + +int f2fs_reinit_atgc_curseg(struct f2fs_sb_info *sbi) +{ + int ret; + + if (!test_opt(sbi, ATGC)) + return 0; + if (sbi->am.atgc_enabled) + return 0; + if (le64_to_cpu(F2FS_CKPT(sbi)->elapsed_time) < + sbi->am.age_threshold) + return 0; + + ret = __f2fs_init_atgc_curseg(sbi, true); + if (!ret) { + sbi->am.atgc_enabled = true; + f2fs_info(sbi, "reenabled age threshold GC"); + } + return ret; +} + +static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type) +{ + struct curseg_info *curseg = CURSEG_I(sbi, type); + + mutex_lock(&curseg->curseg_mutex); + if (!curseg->inited) + goto out; + + if (get_valid_blocks(sbi, curseg->segno, false)) { + write_sum_page(sbi, curseg->sum_blk, curseg->segno); + } else { + mutex_lock(&DIRTY_I(sbi)->seglist_lock); + __set_test_and_free(sbi, curseg->segno, true); + mutex_unlock(&DIRTY_I(sbi)->seglist_lock); + } +out: + mutex_unlock(&curseg->curseg_mutex); +} + +void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi) +{ + __f2fs_save_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED); + + if (sbi->am.atgc_enabled) + __f2fs_save_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC); +} + +static void __f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi, int type) +{ + struct curseg_info *curseg = CURSEG_I(sbi, type); + + mutex_lock(&curseg->curseg_mutex); + if (!curseg->inited) + goto out; + if (get_valid_blocks(sbi, curseg->segno, false)) + goto out; + + mutex_lock(&DIRTY_I(sbi)->seglist_lock); + __set_test_and_inuse(sbi, curseg->segno); + mutex_unlock(&DIRTY_I(sbi)->seglist_lock); +out: + mutex_unlock(&curseg->curseg_mutex); +} + +void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi) +{ + __f2fs_restore_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED); + + if (sbi->am.atgc_enabled) + __f2fs_restore_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC); +} + +static int get_ssr_segment(struct f2fs_sb_info *sbi, int type, + int alloc_mode, unsigned long long age) +{ + struct curseg_info *curseg = CURSEG_I(sbi, type); + unsigned segno = NULL_SEGNO; + unsigned short seg_type = curseg->seg_type; + int i, cnt; + bool reversed = false; + + sanity_check_seg_type(sbi, seg_type); + + /* f2fs_need_SSR() already forces to do this */ + if (!f2fs_get_victim(sbi, &segno, BG_GC, seg_type, + alloc_mode, age, false)) { + curseg->next_segno = segno; + return 1; + } + + /* For node segments, let's do SSR more intensively */ + if (IS_NODESEG(seg_type)) { + if (seg_type >= CURSEG_WARM_NODE) { + reversed = true; + i = CURSEG_COLD_NODE; + } else { + i = CURSEG_HOT_NODE; + } + cnt = NR_CURSEG_NODE_TYPE; + } else { + if (seg_type >= CURSEG_WARM_DATA) { + reversed = true; + i = CURSEG_COLD_DATA; + } else { + i = CURSEG_HOT_DATA; + } + cnt = NR_CURSEG_DATA_TYPE; + } + + for (; cnt-- > 0; reversed ? i-- : i++) { + if (i == seg_type) + continue; + if (!f2fs_get_victim(sbi, &segno, BG_GC, i, + alloc_mode, age, false)) { + curseg->next_segno = segno; + return 1; + } + } + + /* find valid_blocks=0 in dirty list */ + if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { + segno = get_free_segment(sbi); + if (segno != NULL_SEGNO) { + curseg->next_segno = segno; + return 1; + } + } + return 0; +} + +static bool need_new_seg(struct f2fs_sb_info *sbi, int type) +{ + struct curseg_info *curseg = CURSEG_I(sbi, type); + + if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) && + curseg->seg_type == CURSEG_WARM_NODE) + return true; + if (curseg->alloc_type == LFS && is_next_segment_free(sbi, curseg) && + likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED))) + return true; + if (!f2fs_need_SSR(sbi) || !get_ssr_segment(sbi, type, SSR, 0)) + return true; + return false; +} + +int f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type, + unsigned int start, unsigned int end) +{ + struct curseg_info *curseg = CURSEG_I(sbi, type); + unsigned int segno; + int ret = 0; + + f2fs_down_read(&SM_I(sbi)->curseg_lock); + mutex_lock(&curseg->curseg_mutex); + down_write(&SIT_I(sbi)->sentry_lock); + + segno = CURSEG_I(sbi, type)->segno; + if (segno < start || segno > end) + goto unlock; + + if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type, SSR, 0)) + ret = change_curseg(sbi, type); + else + ret = new_curseg(sbi, type, true); + + stat_inc_seg_type(sbi, curseg); + + locate_dirty_segment(sbi, segno); +unlock: + up_write(&SIT_I(sbi)->sentry_lock); + + if (segno != curseg->segno) + f2fs_notice(sbi, "For resize: curseg of type %d: %u ==> %u", + type, segno, curseg->segno); + + mutex_unlock(&curseg->curseg_mutex); + f2fs_up_read(&SM_I(sbi)->curseg_lock); + return ret; +} + +static int __allocate_new_segment(struct f2fs_sb_info *sbi, int type, + bool new_sec, bool force) +{ + struct curseg_info *curseg = CURSEG_I(sbi, type); + unsigned int old_segno; + int err = 0; + + if (type == CURSEG_COLD_DATA_PINNED && !curseg->inited) + goto allocate; + + if (!force && curseg->inited && + !curseg->next_blkoff && + !get_valid_blocks(sbi, curseg->segno, new_sec) && + !get_ckpt_valid_blocks(sbi, curseg->segno, new_sec)) + return 0; + +allocate: + old_segno = curseg->segno; + err = new_curseg(sbi, type, true); + if (err) + return err; + stat_inc_seg_type(sbi, curseg); + locate_dirty_segment(sbi, old_segno); + return 0; +} + +int f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force) +{ + int ret; + + f2fs_down_read(&SM_I(sbi)->curseg_lock); + down_write(&SIT_I(sbi)->sentry_lock); + ret = __allocate_new_segment(sbi, type, true, force); + up_write(&SIT_I(sbi)->sentry_lock); + f2fs_up_read(&SM_I(sbi)->curseg_lock); + + return ret; +} + +int f2fs_allocate_pinning_section(struct f2fs_sb_info *sbi) +{ + struct f2fs_lock_context lc; + int err; + bool gc_required = true; + +retry: + f2fs_lock_op(sbi, &lc); + err = f2fs_allocate_new_section(sbi, CURSEG_COLD_DATA_PINNED, false); + f2fs_unlock_op(sbi, &lc); + + if (f2fs_sb_has_blkzoned(sbi) && err == -EAGAIN && gc_required) { + f2fs_down_write_trace(&sbi->gc_lock, &lc); + err = f2fs_gc_range(sbi, 0, sbi->first_seq_zone_segno - 1, + true, ZONED_PIN_SEC_REQUIRED_COUNT); + f2fs_up_write_trace(&sbi->gc_lock, &lc); + + gc_required = false; + if (!err) + goto retry; + } + + return err; +} + +int f2fs_allocate_new_segments(struct f2fs_sb_info *sbi) +{ + int i; + int err = 0; + + f2fs_down_read(&SM_I(sbi)->curseg_lock); + down_write(&SIT_I(sbi)->sentry_lock); + for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) + err += __allocate_new_segment(sbi, i, false, false); + up_write(&SIT_I(sbi)->sentry_lock); + f2fs_up_read(&SM_I(sbi)->curseg_lock); + + return err; +} + +bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi, + struct cp_control *cpc) +{ + __u64 trim_start = cpc->trim_start; + bool has_candidate = false; + + down_write(&SIT_I(sbi)->sentry_lock); + for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) { + if (add_discard_addrs(sbi, cpc, true)) { + has_candidate = true; + break; + } + } + up_write(&SIT_I(sbi)->sentry_lock); + + cpc->trim_start = trim_start; + return has_candidate; +} + +static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi, + struct discard_policy *dpolicy, + unsigned int start, unsigned int end) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + struct discard_cmd *prev_dc = NULL, *next_dc = NULL; + struct rb_node **insert_p = NULL, *insert_parent = NULL; + struct discard_cmd *dc; + struct blk_plug plug; + int issued; + unsigned int trimmed = 0; + +next: + issued = 0; + + mutex_lock(&dcc->cmd_lock); + if (unlikely(dcc->rbtree_check)) + f2fs_bug_on(sbi, !f2fs_check_discard_tree(sbi)); + + dc = __lookup_discard_cmd_ret(&dcc->root, start, + &prev_dc, &next_dc, &insert_p, &insert_parent); + if (!dc) + dc = next_dc; + + blk_start_plug(&plug); + + while (dc && dc->di.lstart <= end) { + struct rb_node *node; + int err = 0; + + if (dc->di.len < dpolicy->granularity) + goto skip; + + if (dc->state != D_PREP) { + list_move_tail(&dc->list, &dcc->fstrim_list); + goto skip; + } + + err = __submit_discard_cmd(sbi, dpolicy, dc, &issued); + + if (issued >= dpolicy->max_requests) { + start = dc->di.lstart + dc->di.len; + + if (err) + __remove_discard_cmd(sbi, dc); + + blk_finish_plug(&plug); + mutex_unlock(&dcc->cmd_lock); + trimmed += __wait_all_discard_cmd(sbi, NULL); + f2fs_schedule_timeout(DEFAULT_DISCARD_INTERVAL); + goto next; + } +skip: + node = rb_next(&dc->rb_node); + if (err) + __remove_discard_cmd(sbi, dc); + dc = rb_entry_safe(node, struct discard_cmd, rb_node); + + if (fatal_signal_pending(current)) + break; + } + + blk_finish_plug(&plug); + mutex_unlock(&dcc->cmd_lock); + + return trimmed; +} + +int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range) +{ + __u64 start = F2FS_BYTES_TO_BLK(range->start); + __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1; + unsigned int start_segno, end_segno; + block_t start_block, end_block; + struct cp_control cpc; + struct discard_policy dpolicy; + struct f2fs_lock_context lc; + unsigned long long trimmed = 0; + int err = 0; + bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi); + + if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize) + return -EINVAL; + + if (end < MAIN_BLKADDR(sbi)) + goto out; + + if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) { + f2fs_warn(sbi, "Found FS corruption, run fsck to fix."); + return -EFSCORRUPTED; + } + + /* start/end segment number in main_area */ + start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start); + end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 : + GET_SEGNO(sbi, end); + if (need_align) { + start_segno = rounddown(start_segno, SEGS_PER_SEC(sbi)); + end_segno = roundup(end_segno + 1, SEGS_PER_SEC(sbi)) - 1; + } + + cpc.reason = CP_DISCARD; + cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen)); + cpc.trim_start = start_segno; + cpc.trim_end = end_segno; + + if (sbi->discard_blks == 0) + goto out; + + f2fs_down_write_trace(&sbi->gc_lock, &lc); + stat_inc_cp_call_count(sbi, TOTAL_CALL); + err = f2fs_write_checkpoint(sbi, &cpc); + f2fs_up_write_trace(&sbi->gc_lock, &lc); + if (err) + goto out; + + /* + * We filed discard candidates, but actually we don't need to wait for + * all of them, since they'll be issued in idle time along with runtime + * discard option. User configuration looks like using runtime discard + * or periodic fstrim instead of it. + */ + if (f2fs_realtime_discard_enable(sbi)) + goto out; + + start_block = START_BLOCK(sbi, start_segno); + end_block = START_BLOCK(sbi, end_segno + 1); + + __init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen); + trimmed = __issue_discard_cmd_range(sbi, &dpolicy, + start_block, end_block); + + trimmed += __wait_discard_cmd_range(sbi, &dpolicy, + start_block, end_block); +out: + if (!err) + range->len = F2FS_BLK_TO_BYTES(trimmed); + return err; +} + +int f2fs_rw_hint_to_seg_type(struct f2fs_sb_info *sbi, enum rw_hint hint) +{ + if (F2FS_OPTION(sbi).active_logs == 2) + return CURSEG_HOT_DATA; + else if (F2FS_OPTION(sbi).active_logs == 4) + return CURSEG_COLD_DATA; + + /* active_log == 6 */ + switch (hint) { + case WRITE_LIFE_SHORT: + return CURSEG_HOT_DATA; + case WRITE_LIFE_EXTREME: + return CURSEG_COLD_DATA; + default: + return CURSEG_WARM_DATA; + } +} + +/* + * This returns write hints for each segment type. This hints will be + * passed down to block layer as below by default. + * + * User F2FS Block + * ---- ---- ----- + * META WRITE_LIFE_NONE|REQ_META + * HOT_NODE WRITE_LIFE_NONE + * WARM_NODE WRITE_LIFE_MEDIUM + * COLD_NODE WRITE_LIFE_LONG + * ioctl(COLD) COLD_DATA WRITE_LIFE_EXTREME + * extension list " " + * + * -- buffered io + * COLD_DATA WRITE_LIFE_EXTREME + * HOT_DATA WRITE_LIFE_SHORT + * WARM_DATA WRITE_LIFE_NOT_SET + * + * -- direct io + * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME + * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT + * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET + * WRITE_LIFE_NONE " WRITE_LIFE_NONE + * WRITE_LIFE_MEDIUM " WRITE_LIFE_MEDIUM + * WRITE_LIFE_LONG " WRITE_LIFE_LONG + */ +enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi, + enum page_type type, enum temp_type temp) +{ + switch (type) { + case DATA: + switch (temp) { + case WARM: + return WRITE_LIFE_NOT_SET; + case HOT: + return WRITE_LIFE_SHORT; + case COLD: + return WRITE_LIFE_EXTREME; + default: + return WRITE_LIFE_NONE; + } + case NODE: + switch (temp) { + case WARM: + return WRITE_LIFE_MEDIUM; + case HOT: + return WRITE_LIFE_NONE; + case COLD: + return WRITE_LIFE_LONG; + default: + return WRITE_LIFE_NONE; + } + case META: + return WRITE_LIFE_NONE; + default: + return WRITE_LIFE_NONE; + } +} + +u8 f2fs_io_type_to_write_stream(struct block_device *bdev, + enum page_type type, enum temp_type temp) +{ + unsigned short nr = bdev_max_write_streams(bdev); + + if (type != DATA || !nr) + return 0; + if (nr < NR_TEMP_TYPE) + return temp == COLD ? nr : HOT + 1; + + return temp + 1; +} + +static int __get_segment_type_2(struct f2fs_io_info *fio) +{ + if (fio->type == DATA) + return CURSEG_HOT_DATA; + else + return CURSEG_HOT_NODE; +} + +static int __get_segment_type_4(struct f2fs_io_info *fio) +{ + if (fio->type == DATA) { + struct inode *inode = fio_inode(fio); + + if (S_ISDIR(inode->i_mode)) + return CURSEG_HOT_DATA; + else + return CURSEG_COLD_DATA; + } else { + if (IS_DNODE(fio->folio) && is_cold_node(fio->folio)) + return CURSEG_WARM_NODE; + else + return CURSEG_COLD_NODE; + } +} + +static int __get_age_segment_type(struct inode *inode, pgoff_t pgofs) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct extent_info ei = {}; + + if (f2fs_lookup_age_extent_cache(inode, pgofs, &ei)) { + if (!ei.age) + return NO_CHECK_TYPE; + if (ei.age <= sbi->hot_data_age_threshold) + return CURSEG_HOT_DATA; + if (ei.age <= sbi->warm_data_age_threshold) + return CURSEG_WARM_DATA; + return CURSEG_COLD_DATA; + } + return NO_CHECK_TYPE; +} + +static int __get_segment_type_6(struct f2fs_io_info *fio) +{ + if (fio->type == DATA) { + struct inode *inode = fio_inode(fio); + int type; + + if (is_inode_flag_set(inode, FI_ALIGNED_WRITE)) + return CURSEG_COLD_DATA_PINNED; + + if (page_private_gcing(fio->page)) { + if (fio->sbi->am.atgc_enabled && + (fio->io_type == FS_DATA_IO) && + (fio->sbi->gc_mode != GC_URGENT_HIGH) && + __is_valid_data_blkaddr(fio->old_blkaddr) && + !is_inode_flag_set(inode, FI_OPU_WRITE)) + return CURSEG_ALL_DATA_ATGC; + else + return CURSEG_COLD_DATA; + } + if (file_is_cold(inode) || f2fs_need_compress_data(inode)) + return CURSEG_COLD_DATA; + + type = __get_age_segment_type(inode, fio->folio->index); + if (type != NO_CHECK_TYPE) + return type; + + if (file_is_hot(inode) || + is_inode_flag_set(inode, FI_HOT_DATA) || + f2fs_is_cow_file(inode) || + is_inode_flag_set(inode, FI_NEED_IPU)) + return CURSEG_HOT_DATA; + return f2fs_rw_hint_to_seg_type(F2FS_I_SB(inode), + inode->i_write_hint); + } else { + if (IS_DNODE(fio->folio)) + return is_cold_node(fio->folio) ? CURSEG_WARM_NODE : + CURSEG_HOT_NODE; + return CURSEG_COLD_NODE; + } +} + +enum temp_type f2fs_get_segment_temp(struct f2fs_sb_info *sbi, + enum log_type type) +{ + struct curseg_info *curseg = CURSEG_I(sbi, type); + enum temp_type temp = COLD; + + switch (curseg->seg_type) { + case CURSEG_HOT_NODE: + case CURSEG_HOT_DATA: + temp = HOT; + break; + case CURSEG_WARM_NODE: + case CURSEG_WARM_DATA: + temp = WARM; + break; + case CURSEG_COLD_NODE: + case CURSEG_COLD_DATA: + temp = COLD; + break; + default: + f2fs_bug_on(sbi, 1); + } + + return temp; +} + +static int __get_segment_type(struct f2fs_io_info *fio) +{ + enum log_type type = CURSEG_HOT_DATA; + + switch (F2FS_OPTION(fio->sbi).active_logs) { + case 2: + type = __get_segment_type_2(fio); + break; + case 4: + type = __get_segment_type_4(fio); + break; + case 6: + type = __get_segment_type_6(fio); + break; + default: + f2fs_bug_on(fio->sbi, true); + } + + fio->temp = f2fs_get_segment_temp(fio->sbi, type); + + return type; +} + +static void f2fs_randomize_chunk(struct f2fs_sb_info *sbi, + struct curseg_info *seg) +{ + /* To allocate block chunks in different sizes, use random number */ + if (--seg->fragment_remained_chunk > 0) + return; + + seg->fragment_remained_chunk = + get_random_u32_inclusive(1, sbi->max_fragment_chunk); + seg->next_blkoff += + get_random_u32_inclusive(1, sbi->max_fragment_hole); +} + +int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct folio *folio, + block_t old_blkaddr, block_t *new_blkaddr, + struct f2fs_summary *sum, int type, + struct f2fs_io_info *fio) +{ + struct sit_info *sit_i = SIT_I(sbi); + struct curseg_info *curseg = CURSEG_I(sbi, type); + unsigned long long old_mtime; + bool from_gc = (type == CURSEG_ALL_DATA_ATGC); + struct seg_entry *se = NULL; + bool segment_full = false; + int ret = 0; + + f2fs_down_read(&SM_I(sbi)->curseg_lock); + + mutex_lock(&curseg->curseg_mutex); + down_write(&sit_i->sentry_lock); + + if (curseg->segno == NULL_SEGNO) { + ret = -ENOSPC; + goto out_err; + } + + if (from_gc) { + f2fs_bug_on(sbi, GET_SEGNO(sbi, old_blkaddr) == NULL_SEGNO); + se = get_seg_entry(sbi, GET_SEGNO(sbi, old_blkaddr)); + sanity_check_seg_type(sbi, se->type); + f2fs_bug_on(sbi, IS_NODESEG(se->type)); + } + *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg); + + f2fs_bug_on(sbi, curseg->next_blkoff >= BLKS_PER_SEG(sbi)); + + f2fs_wait_discard_bio(sbi, *new_blkaddr); + + sum_entries(curseg->sum_blk)[curseg->next_blkoff] = *sum; + if (curseg->alloc_type == SSR) { + curseg->next_blkoff = f2fs_find_next_ssr_block(sbi, curseg); + } else { + curseg->next_blkoff++; + if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK) + f2fs_randomize_chunk(sbi, curseg); + } + if (curseg->next_blkoff >= f2fs_usable_blks_in_seg(sbi, curseg->segno)) + segment_full = true; + stat_inc_block_count(sbi, curseg); + + if (from_gc) { + old_mtime = get_segment_mtime(sbi, old_blkaddr); + } else { + update_segment_mtime(sbi, old_blkaddr, 0); + old_mtime = 0; + } + update_segment_mtime(sbi, *new_blkaddr, old_mtime); + + /* + * SIT information should be updated before segment allocation, + * since SSR needs latest valid block information. + */ + update_sit_entry(sbi, *new_blkaddr, 1); + update_sit_entry(sbi, old_blkaddr, -1); + + /* + * If the current segment is full, flush it out and replace it with a + * new segment. + */ + if (segment_full) { + if (type == CURSEG_COLD_DATA_PINNED && + !((curseg->segno + 1) % sbi->segs_per_sec)) { + write_sum_page(sbi, curseg->sum_blk, curseg->segno); + reset_curseg_fields(curseg); + goto skip_new_segment; + } + + if (from_gc) { + ret = get_atssr_segment(sbi, type, se->type, + AT_SSR, se->mtime); + } else { + if (need_new_seg(sbi, type)) + ret = new_curseg(sbi, type, false); + else + ret = change_curseg(sbi, type); + stat_inc_seg_type(sbi, curseg); + } + + if (ret) + goto out_err; + } + +skip_new_segment: + /* + * segment dirty status should be updated after segment allocation, + * so we just need to update status only one time after previous + * segment being closed. + */ + locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); + locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr)); + + if (IS_DATASEG(curseg->seg_type)) { + unsigned long long new_val; + + new_val = atomic64_inc_return(&sbi->allocated_data_blocks); + if (unlikely(new_val == ULLONG_MAX)) + atomic64_set(&sbi->allocated_data_blocks, 0); + } + + up_write(&sit_i->sentry_lock); + + if (folio && IS_NODESEG(curseg->seg_type)) { + fill_node_footer_blkaddr(folio, NEXT_FREE_BLKADDR(sbi, curseg)); + + f2fs_inode_chksum_set(sbi, folio); + } + + if (fio) { + struct f2fs_bio_info *io; + + INIT_LIST_HEAD(&fio->list); + fio->in_list = 1; + io = sbi->write_io[fio->type] + fio->temp; + spin_lock(&io->io_lock); + list_add_tail(&fio->list, &io->io_list); + spin_unlock(&io->io_lock); + } + + mutex_unlock(&curseg->curseg_mutex); + f2fs_up_read(&SM_I(sbi)->curseg_lock); + return 0; + +out_err: + *new_blkaddr = NULL_ADDR; + up_write(&sit_i->sentry_lock); + mutex_unlock(&curseg->curseg_mutex); + f2fs_up_read(&SM_I(sbi)->curseg_lock); + return ret; +} + +void f2fs_update_device_state(struct f2fs_sb_info *sbi, nid_t ino, + block_t blkaddr, unsigned int blkcnt) +{ + if (!f2fs_is_multi_device(sbi)) + return; + + while (1) { + unsigned int devidx = f2fs_target_device_index(sbi, blkaddr); + unsigned int blks = FDEV(devidx).end_blk - blkaddr + 1; + + /* update device state for fsync */ + f2fs_set_dirty_device(sbi, ino, devidx, FLUSH_INO); + + /* update device state for checkpoint */ + if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) { + spin_lock(&sbi->dev_lock); + f2fs_set_bit(devidx, (char *)&sbi->dirty_device); + spin_unlock(&sbi->dev_lock); + } + + if (blkcnt <= blks) + break; + blkcnt -= blks; + blkaddr += blks; + } +} + +static int log_type_to_seg_type(enum log_type type) +{ + int seg_type = CURSEG_COLD_DATA; + + switch (type) { + case CURSEG_HOT_DATA: + case CURSEG_WARM_DATA: + case CURSEG_COLD_DATA: + case CURSEG_HOT_NODE: + case CURSEG_WARM_NODE: + case CURSEG_COLD_NODE: + seg_type = (int)type; + break; + case CURSEG_COLD_DATA_PINNED: + case CURSEG_ALL_DATA_ATGC: + seg_type = CURSEG_COLD_DATA; + break; + default: + break; + } + return seg_type; +} + +static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio) +{ + struct folio *folio = fio->folio; + enum log_type type = __get_segment_type(fio); + int seg_type = log_type_to_seg_type(type); + bool keep_order = (f2fs_lfs_mode(fio->sbi) && + seg_type == CURSEG_COLD_DATA); + int err; + + if (keep_order) + f2fs_down_read(&fio->sbi->io_order_lock); + + err = f2fs_allocate_data_block(fio->sbi, folio, fio->old_blkaddr, + &fio->new_blkaddr, sum, type, fio); + if (unlikely(err)) { + f2fs_err_ratelimited(fio->sbi, + "%s Failed to allocate data block, ino:%u, index:%lu, type:%d, old_blkaddr:0x%x, new_blkaddr:0x%x, err:%d", + __func__, fio->ino, folio->index, type, + fio->old_blkaddr, fio->new_blkaddr, err); + folio_end_writeback(folio); + if (f2fs_in_warm_node_list(folio)) + f2fs_del_fsync_node_entry(fio->sbi, folio); + f2fs_bug_on(fio->sbi, !is_set_ckpt_flags(fio->sbi, + CP_ERROR_FLAG)); + goto out; + } + + f2fs_bug_on(fio->sbi, !f2fs_is_valid_blkaddr_raw(fio->sbi, + fio->new_blkaddr, DATA_GENERIC_ENHANCE)); + + if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO) + f2fs_invalidate_internal_cache(fio->sbi, fio->old_blkaddr, 1); + + /* writeout dirty page into bdev */ + f2fs_submit_page_write(fio); + + f2fs_update_device_state(fio->sbi, fio->ino, fio->new_blkaddr, 1); +out: + if (keep_order) + f2fs_up_read(&fio->sbi->io_order_lock); +} + +void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct folio *folio, + enum iostat_type io_type) +{ + struct f2fs_io_info fio = { + .sbi = sbi, + .type = META, + .temp = HOT, + .op = REQ_OP_WRITE, + .op_flags = REQ_SYNC | REQ_META | REQ_PRIO, + .old_blkaddr = folio->index, + .new_blkaddr = folio->index, + .folio = folio, + .encrypted_page = NULL, + .in_list = 0, + }; + + if (unlikely(folio->index >= MAIN_BLKADDR(sbi))) + fio.op_flags &= ~REQ_META; + + folio_start_writeback(folio); + f2fs_submit_page_write(&fio); + + stat_inc_meta_count(sbi, folio->index); + f2fs_update_iostat(sbi, NULL, io_type, F2FS_BLKSIZE); +} + +void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio) +{ + struct f2fs_summary sum; + + set_summary(&sum, nid, 0, 0); + do_write_page(&sum, fio); + + f2fs_update_iostat(fio->sbi, NULL, fio->io_type, F2FS_BLKSIZE); +} + +void f2fs_outplace_write_data(struct dnode_of_data *dn, + struct f2fs_io_info *fio) +{ + struct f2fs_sb_info *sbi = fio->sbi; + struct f2fs_summary sum; + + f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR); + if (fio->io_type == FS_DATA_IO || fio->io_type == FS_CP_DATA_IO) + f2fs_update_age_extent_cache(dn); + set_summary(&sum, dn->nid, dn->ofs_in_node, fio->version); + do_write_page(&sum, fio); + f2fs_update_data_blkaddr(dn, fio->new_blkaddr); + + f2fs_update_iostat(sbi, dn->inode, fio->io_type, F2FS_BLKSIZE); +} + +int f2fs_inplace_write_data(struct f2fs_io_info *fio) +{ + int err; + struct f2fs_sb_info *sbi = fio->sbi; + unsigned int segno; + + fio->new_blkaddr = fio->old_blkaddr; + /* i/o temperature is needed for passing down write hints */ + __get_segment_type(fio); + + segno = GET_SEGNO(sbi, fio->new_blkaddr); + + if (!IS_DATASEG(get_seg_entry(sbi, segno)->type)) { + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.", + __func__, segno); + err = -EFSCORRUPTED; + f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUM_TYPE); + goto drop_bio; + } + + if (f2fs_cp_error(sbi)) { + err = -EIO; + goto drop_bio; + } + + if (fio->meta_gc) + f2fs_truncate_meta_inode_pages(sbi, fio->new_blkaddr, 1); + + stat_inc_inplace_blocks(fio->sbi); + + if (fio->bio && !IS_F2FS_IPU_NOCACHE(sbi)) + err = f2fs_merge_page_bio(fio); + else + err = f2fs_submit_page_bio(fio); + if (!err) { + f2fs_update_device_state(fio->sbi, fio->ino, + fio->new_blkaddr, 1); + f2fs_update_iostat(fio->sbi, fio_inode(fio), + fio->io_type, F2FS_BLKSIZE); + } + + return err; +drop_bio: + if (fio->bio && *(fio->bio)) { + struct bio *bio = *(fio->bio); + + bio->bi_status = BLK_STS_IOERR; + bio_endio(bio); + *(fio->bio) = NULL; + } + return err; +} + +static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi, + unsigned int segno) +{ + int i; + + for (i = CURSEG_HOT_DATA; i < NO_CHECK_TYPE; i++) { + if (CURSEG_I(sbi, i)->segno == segno) + break; + } + return i; +} + +void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, + block_t old_blkaddr, block_t new_blkaddr, + bool recover_curseg, bool recover_newaddr, + bool from_gc) +{ + struct sit_info *sit_i = SIT_I(sbi); + struct curseg_info *curseg; + unsigned int segno, old_cursegno; + struct seg_entry *se; + int type; + unsigned short old_blkoff; + unsigned char old_alloc_type; + + segno = GET_SEGNO(sbi, new_blkaddr); + se = get_seg_entry(sbi, segno); + type = se->type; + + f2fs_down_write(&SM_I(sbi)->curseg_lock); + + if (!recover_curseg) { + /* for recovery flow */ + if (se->valid_blocks == 0 && !is_curseg(sbi, segno)) { + if (old_blkaddr == NULL_ADDR) + type = CURSEG_COLD_DATA; + else + type = CURSEG_WARM_DATA; + } + } else { + if (is_curseg(sbi, segno)) { + /* se->type is volatile as SSR allocation */ + type = __f2fs_get_curseg(sbi, segno); + f2fs_bug_on(sbi, type == NO_CHECK_TYPE); + } else { + type = CURSEG_WARM_DATA; + } + } + + curseg = CURSEG_I(sbi, type); + f2fs_bug_on(sbi, !IS_DATASEG(curseg->seg_type)); + + mutex_lock(&curseg->curseg_mutex); + down_write(&sit_i->sentry_lock); + + old_cursegno = curseg->segno; + old_blkoff = curseg->next_blkoff; + old_alloc_type = curseg->alloc_type; + + /* change the current segment */ + if (segno != curseg->segno) { + curseg->next_segno = segno; + if (change_curseg(sbi, type)) + goto out_unlock; + } + + curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr); + sum_entries(curseg->sum_blk)[curseg->next_blkoff] = *sum; + + if (!recover_curseg || recover_newaddr) { + if (!from_gc) + update_segment_mtime(sbi, new_blkaddr, 0); + update_sit_entry(sbi, new_blkaddr, 1); + } + if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) { + f2fs_invalidate_internal_cache(sbi, old_blkaddr, 1); + if (!from_gc) + update_segment_mtime(sbi, old_blkaddr, 0); + update_sit_entry(sbi, old_blkaddr, -1); + } + + locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr)); + locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr)); + + locate_dirty_segment(sbi, old_cursegno); + + if (recover_curseg) { + if (old_cursegno != curseg->segno) { + curseg->next_segno = old_cursegno; + if (change_curseg(sbi, type)) + goto out_unlock; + } + curseg->next_blkoff = old_blkoff; + curseg->alloc_type = old_alloc_type; + } + +out_unlock: + up_write(&sit_i->sentry_lock); + mutex_unlock(&curseg->curseg_mutex); + f2fs_up_write(&SM_I(sbi)->curseg_lock); +} + +void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn, + block_t old_addr, block_t new_addr, + unsigned char version, bool recover_curseg, + bool recover_newaddr) +{ + struct f2fs_summary sum; + + set_summary(&sum, dn->nid, dn->ofs_in_node, version); + + f2fs_do_replace_block(sbi, &sum, old_addr, new_addr, + recover_curseg, recover_newaddr, false); + + f2fs_update_data_blkaddr(dn, new_addr); +} + +void f2fs_folio_wait_writeback(struct folio *folio, enum page_type type, + bool ordered, bool locked) +{ + if (folio_test_writeback(folio)) { + struct f2fs_sb_info *sbi = F2FS_F_SB(folio); + + /* submit cached LFS IO */ + f2fs_submit_merged_write_folio(sbi, folio, type); + /* submit cached IPU IO */ + f2fs_submit_merged_ipu_write(sbi, NULL, folio); + if (ordered) { + folio_wait_writeback(folio); + f2fs_bug_on(sbi, locked && folio_test_writeback(folio)); + } else { + folio_wait_stable(folio); + } + } +} + +void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct folio *cfolio; + + if (!f2fs_meta_inode_gc_required(inode)) + return; + + if (!__is_valid_data_blkaddr(blkaddr)) + return; + + cfolio = filemap_lock_folio(META_MAPPING(sbi), blkaddr); + if (!IS_ERR(cfolio)) { + f2fs_folio_wait_writeback(cfolio, DATA, true, true); + f2fs_folio_put(cfolio, true); + } +} + +void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr, + block_t len) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + block_t i; + + if (!f2fs_meta_inode_gc_required(inode)) + return; + + for (i = 0; i < len; i++) + f2fs_wait_on_block_writeback(inode, blkaddr + i); + + f2fs_truncate_meta_inode_pages(sbi, blkaddr, len); +} + +static int read_compacted_summaries(struct f2fs_sb_info *sbi) +{ + struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); + struct curseg_info *seg_i; + unsigned char *kaddr; + struct folio *folio; + block_t start; + int i, j, offset; + + start = start_sum_block(sbi); + + folio = f2fs_get_meta_folio(sbi, start++); + if (IS_ERR(folio)) + return PTR_ERR(folio); + kaddr = folio_address(folio); + + /* Step 1: restore nat cache */ + seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); + memcpy(seg_i->journal, kaddr, sbi->sum_journal_size); + + /* Step 2: restore sit cache */ + seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); + memcpy(seg_i->journal, kaddr + sbi->sum_journal_size, sbi->sum_journal_size); + offset = 2 * sbi->sum_journal_size; + + /* Step 3: restore summary entries */ + for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { + unsigned short blk_off; + unsigned int segno; + + seg_i = CURSEG_I(sbi, i); + segno = le32_to_cpu(ckpt->cur_data_segno[i]); + blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]); + seg_i->next_segno = segno; + reset_curseg(sbi, i, 0); + seg_i->alloc_type = ckpt->alloc_type[i]; + seg_i->next_blkoff = blk_off; + + if (seg_i->alloc_type == SSR) + blk_off = BLKS_PER_SEG(sbi); + + for (j = 0; j < blk_off; j++) { + struct f2fs_summary *s; + + s = (struct f2fs_summary *)(kaddr + offset); + sum_entries(seg_i->sum_blk)[j] = *s; + offset += SUMMARY_SIZE; + if (offset + SUMMARY_SIZE <= sbi->blocksize - + SUM_FOOTER_SIZE) + continue; + + f2fs_folio_put(folio, true); + + folio = f2fs_get_meta_folio(sbi, start++); + if (IS_ERR(folio)) + return PTR_ERR(folio); + kaddr = folio_address(folio); + offset = 0; + } + } + f2fs_folio_put(folio, true); + return 0; +} + +static int read_normal_summaries(struct f2fs_sb_info *sbi, int type) +{ + struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); + struct f2fs_summary_block *sum; + struct curseg_info *curseg; + struct folio *new; + unsigned short blk_off; + unsigned int segno = 0; + block_t blk_addr = 0; + int err = 0; + + /* get segment number and block addr */ + if (IS_DATASEG(type)) { + segno = le32_to_cpu(ckpt->cur_data_segno[type]); + blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type - + CURSEG_HOT_DATA]); + if (__exist_node_summaries(sbi)) + blk_addr = sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type); + else + blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type); + } else { + segno = le32_to_cpu(ckpt->cur_node_segno[type - + CURSEG_HOT_NODE]); + blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type - + CURSEG_HOT_NODE]); + if (__exist_node_summaries(sbi)) + blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE, + type - CURSEG_HOT_NODE); + else + blk_addr = GET_SUM_BLOCK(sbi, segno); + } + + new = f2fs_get_meta_folio(sbi, blk_addr); + if (IS_ERR(new)) + return PTR_ERR(new); + sum = folio_address(new); + + if (IS_NODESEG(type)) { + if (__exist_node_summaries(sbi)) { + struct f2fs_summary *ns = sum_entries(sum); + int i; + + for (i = 0; i < BLKS_PER_SEG(sbi); i++, ns++) { + ns->version = 0; + ns->ofs_in_node = 0; + } + } else { + err = f2fs_restore_node_summary(sbi, segno, sum); + if (err) + goto out; + } + } + + /* set uncompleted segment to curseg */ + curseg = CURSEG_I(sbi, type); + mutex_lock(&curseg->curseg_mutex); + + /* update journal info */ + down_write(&curseg->journal_rwsem); + memcpy(curseg->journal, sum_journal(sbi, sum), sbi->sum_journal_size); + up_write(&curseg->journal_rwsem); + + memcpy(sum_entries(curseg->sum_blk), sum_entries(sum), + sbi->sum_entry_size); + memcpy(sum_footer(sbi, curseg->sum_blk), sum_footer(sbi, sum), + SUM_FOOTER_SIZE); + curseg->next_segno = segno; + reset_curseg(sbi, type, 0); + curseg->alloc_type = ckpt->alloc_type[type]; + curseg->next_blkoff = blk_off; + mutex_unlock(&curseg->curseg_mutex); +out: + f2fs_folio_put(new, true); + return err; +} + +static int restore_curseg_summaries(struct f2fs_sb_info *sbi) +{ + struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal; + struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal; + int type = CURSEG_HOT_DATA; + int err; + + if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) { + int npages = f2fs_npages_for_summary_flush(sbi, true); + + if (npages >= 2) + f2fs_ra_meta_pages(sbi, start_sum_block(sbi), npages, + META_CP, true); + + /* restore for compacted data summary */ + err = read_compacted_summaries(sbi); + if (err) + return err; + type = CURSEG_HOT_NODE; + } + + if (__exist_node_summaries(sbi)) + f2fs_ra_meta_pages(sbi, + sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type), + NR_CURSEG_PERSIST_TYPE - type, META_CP, true); + + for (; type <= CURSEG_COLD_NODE; type++) { + err = read_normal_summaries(sbi, type); + if (err) + return err; + } + + /* sanity check for summary blocks */ + if (nats_in_cursum(nat_j) > sbi->nat_journal_entries || + sits_in_cursum(sit_j) > sbi->sit_journal_entries) { + f2fs_err(sbi, "invalid journal entries nats %u sits %u", + nats_in_cursum(nat_j), sits_in_cursum(sit_j)); + return -EINVAL; + } + + return 0; +} + +static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr) +{ + struct folio *folio; + unsigned char *kaddr; + struct f2fs_summary *summary; + struct curseg_info *seg_i; + int written_size = 0; + int i, j; + + folio = f2fs_grab_meta_folio(sbi, blkaddr++); + kaddr = folio_address(folio); + memset(kaddr, 0, PAGE_SIZE); + + /* Step 1: write nat cache */ + seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA); + memcpy(kaddr, seg_i->journal, sbi->sum_journal_size); + written_size += sbi->sum_journal_size; + + /* Step 2: write sit cache */ + seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA); + memcpy(kaddr + written_size, seg_i->journal, sbi->sum_journal_size); + written_size += sbi->sum_journal_size; + + /* Step 3: write summary entries */ + for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) { + seg_i = CURSEG_I(sbi, i); + for (j = 0; j < f2fs_curseg_valid_blocks(sbi, i); j++) { + if (!folio) { + folio = f2fs_grab_meta_folio(sbi, blkaddr++); + kaddr = folio_address(folio); + memset(kaddr, 0, PAGE_SIZE); + written_size = 0; + } + summary = (struct f2fs_summary *)(kaddr + written_size); + *summary = sum_entries(seg_i->sum_blk)[j]; + written_size += SUMMARY_SIZE; + + if (written_size + SUMMARY_SIZE <= sbi->blocksize - + SUM_FOOTER_SIZE) + continue; + + folio_mark_dirty(folio); + f2fs_folio_put(folio, true); + folio = NULL; + } + } + if (folio) { + folio_mark_dirty(folio); + f2fs_folio_put(folio, true); + } +} + +static void write_normal_summaries(struct f2fs_sb_info *sbi, + block_t blkaddr, int type) +{ + int i, end; + + if (IS_DATASEG(type)) + end = type + NR_CURSEG_DATA_TYPE; + else + end = type + NR_CURSEG_NODE_TYPE; + + for (i = type; i < end; i++) + write_current_sum_page(sbi, i, blkaddr + (i - type)); +} + +void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk) +{ + if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) + write_compacted_summaries(sbi, start_blk); + else + write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA); +} + +void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk) +{ + write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE); +} + +int f2fs_lookup_journal_in_cursum(struct f2fs_sb_info *sbi, + struct f2fs_journal *journal, int type, + unsigned int val, int alloc) +{ + int i; + + if (type == NAT_JOURNAL) { + for (i = 0; i < nats_in_cursum(journal); i++) { + if (le32_to_cpu(nid_in_journal(journal, i)) == val) + return i; + } + if (alloc && __has_cursum_space(sbi, journal, 1, NAT_JOURNAL)) + return update_nats_in_cursum(journal, 1); + } else if (type == SIT_JOURNAL) { + for (i = 0; i < sits_in_cursum(journal); i++) + if (le32_to_cpu(segno_in_journal(journal, i)) == val) + return i; + if (alloc && __has_cursum_space(sbi, journal, 1, SIT_JOURNAL)) + return update_sits_in_cursum(journal, 1); + } + return -1; +} + +static struct folio *get_current_sit_folio(struct f2fs_sb_info *sbi, + unsigned int segno) +{ + return f2fs_get_meta_folio(sbi, current_sit_addr(sbi, segno)); +} + +static struct folio *get_next_sit_folio(struct f2fs_sb_info *sbi, + unsigned int start) +{ + struct sit_info *sit_i = SIT_I(sbi); + struct folio *folio; + pgoff_t src_off, dst_off; + + src_off = current_sit_addr(sbi, start); + dst_off = next_sit_addr(sbi, src_off); + + folio = f2fs_grab_meta_folio(sbi, dst_off); + seg_info_to_sit_folio(sbi, folio, start); + + folio_mark_dirty(folio); + set_to_next_sit(sit_i, start); + + return folio; +} + +static struct sit_entry_set *grab_sit_entry_set(void) +{ + struct sit_entry_set *ses = + f2fs_kmem_cache_alloc(sit_entry_set_slab, + GFP_NOFS, true, NULL); + + ses->entry_cnt = 0; + INIT_LIST_HEAD(&ses->set_list); + return ses; +} + +static void release_sit_entry_set(struct sit_entry_set *ses) +{ + list_del(&ses->set_list); + kmem_cache_free(sit_entry_set_slab, ses); +} + +static void adjust_sit_entry_set(struct sit_entry_set *ses, + struct list_head *head) +{ + struct sit_entry_set *next = ses; + + if (list_is_last(&ses->set_list, head)) + return; + + list_for_each_entry_continue(next, head, set_list) + if (ses->entry_cnt <= next->entry_cnt) { + list_move_tail(&ses->set_list, &next->set_list); + return; + } + + list_move_tail(&ses->set_list, head); +} + +static void add_sit_entry(unsigned int segno, struct list_head *head) +{ + struct sit_entry_set *ses; + unsigned int start_segno = START_SEGNO(segno); + + list_for_each_entry(ses, head, set_list) { + if (ses->start_segno == start_segno) { + ses->entry_cnt++; + adjust_sit_entry_set(ses, head); + return; + } + } + + ses = grab_sit_entry_set(); + + ses->start_segno = start_segno; + ses->entry_cnt++; + list_add(&ses->set_list, head); +} + +static void add_sits_in_set(struct f2fs_sb_info *sbi) +{ + struct f2fs_sm_info *sm_info = SM_I(sbi); + struct list_head *set_list = &sm_info->sit_entry_set; + unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap; + unsigned int segno; + + for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi)) + add_sit_entry(segno, set_list); +} + +static void remove_sits_in_journal(struct f2fs_sb_info *sbi) +{ + struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); + struct f2fs_journal *journal = curseg->journal; + int i; + + down_write(&curseg->journal_rwsem); + for (i = 0; i < sits_in_cursum(journal); i++) { + unsigned int segno; + bool dirtied; + + segno = le32_to_cpu(segno_in_journal(journal, i)); + dirtied = __mark_sit_entry_dirty(sbi, segno); + + if (!dirtied) + add_sit_entry(segno, &SM_I(sbi)->sit_entry_set); + } + update_sits_in_cursum(journal, -i); + up_write(&curseg->journal_rwsem); +} + +/* + * CP calls this function, which flushes SIT entries including sit_journal, + * and moves prefree segs to free segs. + */ +void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) +{ + struct sit_info *sit_i = SIT_I(sbi); + unsigned long *bitmap = sit_i->dirty_sentries_bitmap; + struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); + struct f2fs_journal *journal = curseg->journal; + struct sit_entry_set *ses, *tmp; + struct list_head *head = &SM_I(sbi)->sit_entry_set; + bool to_journal = !is_sbi_flag_set(sbi, SBI_IS_RESIZEFS); + struct seg_entry *se; + + down_write(&sit_i->sentry_lock); + + if (!sit_i->dirty_sentries) + goto out; + + /* + * add and account sit entries of dirty bitmap in sit entry + * set temporarily + */ + add_sits_in_set(sbi); + + /* + * if there are no enough space in journal to store dirty sit + * entries, remove all entries from journal and add and account + * them in sit entry set. + */ + if (!__has_cursum_space(sbi, journal, + sit_i->dirty_sentries, SIT_JOURNAL) || !to_journal) + remove_sits_in_journal(sbi); + + /* + * there are two steps to flush sit entries: + * #1, flush sit entries to journal in current cold data summary block. + * #2, flush sit entries to sit page. + */ + list_for_each_entry_safe(ses, tmp, head, set_list) { + struct folio *folio = NULL; + struct f2fs_sit_block *raw_sit = NULL; + unsigned int start_segno = ses->start_segno; + unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK, + (unsigned long)MAIN_SEGS(sbi)); + unsigned int segno = start_segno; + + if (to_journal && + !__has_cursum_space(sbi, journal, ses->entry_cnt, + SIT_JOURNAL)) + to_journal = false; + + if (to_journal) { + down_write(&curseg->journal_rwsem); + } else { + folio = get_next_sit_folio(sbi, start_segno); + raw_sit = folio_address(folio); + } + + /* flush dirty sit entries in region of current sit set */ + for_each_set_bit_from(segno, bitmap, end) { + int offset, sit_offset; + + se = get_seg_entry(sbi, segno); + + /* add discard candidates */ + if (!(cpc->reason & CP_DISCARD)) { + cpc->trim_start = segno; + add_discard_addrs(sbi, cpc, false); + } + + if (to_journal) { + offset = f2fs_lookup_journal_in_cursum(sbi, journal, + SIT_JOURNAL, segno, 1); + f2fs_bug_on(sbi, offset < 0); + segno_in_journal(journal, offset) = + cpu_to_le32(segno); + seg_info_to_raw_sit(se, + &sit_in_journal(journal, offset)); + check_block_count(sbi, segno, + &sit_in_journal(journal, offset)); + } else { + sit_offset = SIT_ENTRY_OFFSET(sit_i, segno); + seg_info_to_raw_sit(se, + &raw_sit->entries[sit_offset]); + check_block_count(sbi, segno, + &raw_sit->entries[sit_offset]); + } + + /* update ckpt_valid_block */ + if (__is_large_section(sbi)) + set_ckpt_valid_blocks(sbi, segno); + + __clear_bit(segno, bitmap); + sit_i->dirty_sentries--; + ses->entry_cnt--; + } + + if (to_journal) + up_write(&curseg->journal_rwsem); + else + f2fs_folio_put(folio, true); + + f2fs_bug_on(sbi, ses->entry_cnt); + release_sit_entry_set(ses); + } + + f2fs_bug_on(sbi, !list_empty(head)); + f2fs_bug_on(sbi, sit_i->dirty_sentries); +out: + if (cpc->reason & CP_DISCARD) { + __u64 trim_start = cpc->trim_start; + + for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) + add_discard_addrs(sbi, cpc, false); + + cpc->trim_start = trim_start; + } + up_write(&sit_i->sentry_lock); + + set_prefree_as_free_segments(sbi); +} + +static int build_sit_info(struct f2fs_sb_info *sbi) +{ + struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); + struct sit_info *sit_i; + unsigned int sit_segs, start; + char *src_bitmap, *bitmap; + unsigned int bitmap_size, main_bitmap_size, sit_bitmap_size; + unsigned int discard_map = f2fs_block_unit_discard(sbi) ? 1 : 0; + + /* allocate memory for SIT information */ + sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL); + if (!sit_i) + return -ENOMEM; + + SM_I(sbi)->sit_info = sit_i; + + sit_i->sentries = + f2fs_kvzalloc(sbi, array_size(sizeof(struct seg_entry), + MAIN_SEGS(sbi)), + GFP_KERNEL); + if (!sit_i->sentries) + return -ENOMEM; + + main_bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); + sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, main_bitmap_size, + GFP_KERNEL); + if (!sit_i->dirty_sentries_bitmap) + return -ENOMEM; + + bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (2 + discard_map); + sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL); + if (!sit_i->bitmap) + return -ENOMEM; + + bitmap = sit_i->bitmap; + + for (start = 0; start < MAIN_SEGS(sbi); start++) { + sit_i->sentries[start].cur_valid_map = bitmap; + bitmap += SIT_VBLOCK_MAP_SIZE; + + sit_i->sentries[start].ckpt_valid_map = bitmap; + bitmap += SIT_VBLOCK_MAP_SIZE; + + if (discard_map) { + sit_i->sentries[start].discard_map = bitmap; + bitmap += SIT_VBLOCK_MAP_SIZE; + } + } + + sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL); + if (!sit_i->tmp_map) + return -ENOMEM; + + if (__is_large_section(sbi)) { + sit_i->sec_entries = + f2fs_kvzalloc(sbi, array_size(sizeof(struct sec_entry), + MAIN_SECS(sbi)), + GFP_KERNEL); + if (!sit_i->sec_entries) + return -ENOMEM; + } + + /* get information related with SIT */ + sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1; + + /* setup SIT bitmap from ckeckpoint pack */ + sit_bitmap_size = __bitmap_size(sbi, SIT_BITMAP); + src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP); + + sit_i->sit_bitmap = kmemdup(src_bitmap, sit_bitmap_size, GFP_KERNEL); + if (!sit_i->sit_bitmap) + return -ENOMEM; + +#ifdef CONFIG_F2FS_CHECK_FS + sit_i->invalid_segmap = f2fs_kvzalloc(sbi, + main_bitmap_size, GFP_KERNEL); + if (!sit_i->invalid_segmap) + return -ENOMEM; +#endif + + sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr); + sit_i->sit_blocks = SEGS_TO_BLKS(sbi, sit_segs); + sit_i->written_valid_blocks = 0; + sit_i->bitmap_size = sit_bitmap_size; + sit_i->dirty_sentries = 0; + sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK; + sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time); + sit_i->mounted_time = ktime_get_boottime_seconds(); + init_rwsem(&sit_i->sentry_lock); + return 0; +} + +static int build_free_segmap(struct f2fs_sb_info *sbi) +{ + struct free_segmap_info *free_i; + unsigned int bitmap_size, sec_bitmap_size; + + /* allocate memory for free segmap information */ + free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL); + if (!free_i) + return -ENOMEM; + + SM_I(sbi)->free_info = free_i; + + bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); + free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL); + if (!free_i->free_segmap) + return -ENOMEM; + + sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); + free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL); + if (!free_i->free_secmap) + return -ENOMEM; + + /* set all segments as dirty temporarily */ + memset(free_i->free_segmap, 0xff, bitmap_size); + memset(free_i->free_secmap, 0xff, sec_bitmap_size); + + /* init free segmap information */ + free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi)); + free_i->free_segments = 0; + free_i->free_sections = 0; + spin_lock_init(&free_i->segmap_lock); + return 0; +} + +static int build_curseg(struct f2fs_sb_info *sbi) +{ + struct curseg_info *array; + int i; + + array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE, + sizeof(*array)), GFP_KERNEL); + if (!array) + return -ENOMEM; + + SM_I(sbi)->curseg_array = array; + + for (i = 0; i < NO_CHECK_TYPE; i++) { + mutex_init(&array[i].curseg_mutex); + array[i].sum_blk = f2fs_kzalloc(sbi, sbi->sum_blocksize, + GFP_KERNEL); + if (!array[i].sum_blk) + return -ENOMEM; + init_rwsem(&array[i].journal_rwsem); + array[i].journal = f2fs_kzalloc(sbi, + sbi->sum_journal_size, GFP_KERNEL); + if (!array[i].journal) + return -ENOMEM; + array[i].seg_type = log_type_to_seg_type(i); + reset_curseg_fields(&array[i]); + } + return restore_curseg_summaries(sbi); +} + +static int build_sit_entries(struct f2fs_sb_info *sbi) +{ + struct sit_info *sit_i = SIT_I(sbi); + struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA); + struct f2fs_journal *journal = curseg->journal; + struct seg_entry *se; + struct f2fs_sit_entry sit; + int sit_blk_cnt = SIT_BLK_CNT(sbi); + unsigned int i, start, end; + unsigned int readed, start_blk = 0; + int err = 0; + block_t sit_valid_blocks[2] = {0, 0}; + + do { + readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_VECS, + META_SIT, true); + + start = start_blk * sit_i->sents_per_block; + end = (start_blk + readed) * sit_i->sents_per_block; + + for (; start < end && start < MAIN_SEGS(sbi); start++) { + struct f2fs_sit_block *sit_blk; + struct folio *folio; + + se = &sit_i->sentries[start]; + folio = get_current_sit_folio(sbi, start); + if (IS_ERR(folio)) + return PTR_ERR(folio); + sit_blk = folio_address(folio); + sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)]; + f2fs_folio_put(folio, true); + + err = check_block_count(sbi, start, &sit); + if (err) + return err; + seg_info_from_raw_sit(se, &sit); + + if (se->type >= NR_PERSISTENT_LOG) { + f2fs_err(sbi, "Invalid segment type: %u, segno: %u", + se->type, start); + f2fs_handle_error(sbi, + ERROR_INCONSISTENT_SUM_TYPE); + return -EFSCORRUPTED; + } + + sit_valid_blocks[SE_PAGETYPE(se)] += se->valid_blocks; + + if (!f2fs_block_unit_discard(sbi)) + goto init_discard_map_done; + + /* build discard map only one time */ + if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) { + memset(se->discard_map, 0xff, + SIT_VBLOCK_MAP_SIZE); + goto init_discard_map_done; + } + memcpy(se->discard_map, se->cur_valid_map, + SIT_VBLOCK_MAP_SIZE); + sbi->discard_blks += BLKS_PER_SEG(sbi) - + se->valid_blocks; +init_discard_map_done: + if (__is_large_section(sbi)) + get_sec_entry(sbi, start)->valid_blocks += + se->valid_blocks; + } + start_blk += readed; + } while (start_blk < sit_blk_cnt); + + down_read(&curseg->journal_rwsem); + for (i = 0; i < sits_in_cursum(journal); i++) { + unsigned int old_valid_blocks; + + start = le32_to_cpu(segno_in_journal(journal, i)); + if (start >= MAIN_SEGS(sbi)) { + f2fs_err(sbi, "Wrong journal entry on segno %u", + start); + err = -EFSCORRUPTED; + f2fs_handle_error(sbi, ERROR_CORRUPTED_JOURNAL); + break; + } + + se = &sit_i->sentries[start]; + sit = sit_in_journal(journal, i); + + old_valid_blocks = se->valid_blocks; + + sit_valid_blocks[SE_PAGETYPE(se)] -= old_valid_blocks; + + err = check_block_count(sbi, start, &sit); + if (err) + break; + seg_info_from_raw_sit(se, &sit); + + if (se->type >= NR_PERSISTENT_LOG) { + f2fs_err(sbi, "Invalid segment type: %u, segno: %u", + se->type, start); + err = -EFSCORRUPTED; + f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUM_TYPE); + break; + } + + sit_valid_blocks[SE_PAGETYPE(se)] += se->valid_blocks; + + if (f2fs_block_unit_discard(sbi)) { + if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) { + memset(se->discard_map, 0xff, SIT_VBLOCK_MAP_SIZE); + } else { + memcpy(se->discard_map, se->cur_valid_map, + SIT_VBLOCK_MAP_SIZE); + sbi->discard_blks += old_valid_blocks; + sbi->discard_blks -= se->valid_blocks; + } + } + + if (__is_large_section(sbi)) { + get_sec_entry(sbi, start)->valid_blocks += + se->valid_blocks; + get_sec_entry(sbi, start)->valid_blocks -= + old_valid_blocks; + } + } + up_read(&curseg->journal_rwsem); + + /* update ckpt_valid_block */ + if (__is_large_section(sbi)) { + unsigned int segno; + + for (segno = 0; segno < MAIN_SEGS(sbi); segno += SEGS_PER_SEC(sbi)) + set_ckpt_valid_blocks(sbi, segno); + } + + if (err) + return err; + + if (sit_valid_blocks[NODE] != valid_node_count(sbi)) { + f2fs_err(sbi, "SIT is corrupted node# %u vs %u", + sit_valid_blocks[NODE], valid_node_count(sbi)); + f2fs_handle_error(sbi, ERROR_INCONSISTENT_NODE_COUNT); + return -EFSCORRUPTED; + } + + if (sit_valid_blocks[DATA] + sit_valid_blocks[NODE] > + valid_user_blocks(sbi)) { + f2fs_err(sbi, "SIT is corrupted data# %u %u vs %u", + sit_valid_blocks[DATA], sit_valid_blocks[NODE], + valid_user_blocks(sbi)); + f2fs_handle_error(sbi, ERROR_INCONSISTENT_BLOCK_COUNT); + return -EFSCORRUPTED; + } + + return 0; +} + +static void init_free_segmap(struct f2fs_sb_info *sbi) +{ + unsigned int start; + int type; + struct seg_entry *sentry; + + for (start = 0; start < MAIN_SEGS(sbi); start++) { + if (f2fs_usable_blks_in_seg(sbi, start) == 0) + continue; + sentry = get_seg_entry(sbi, start); + if (!sentry->valid_blocks) + __set_free(sbi, start); + else + SIT_I(sbi)->written_valid_blocks += + sentry->valid_blocks; + } + + /* set use the current segments */ + for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) { + struct curseg_info *curseg_t = CURSEG_I(sbi, type); + + __set_test_and_inuse(sbi, curseg_t->segno); + } +} + +static void init_dirty_segmap(struct f2fs_sb_info *sbi) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + struct free_segmap_info *free_i = FREE_I(sbi); + unsigned int segno = 0, offset = 0, secno; + block_t valid_blocks, usable_blks_in_seg; + + while (1) { + /* find dirty segment based on free segmap */ + segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset); + if (segno >= MAIN_SEGS(sbi)) + break; + offset = segno + 1; + valid_blocks = get_valid_blocks(sbi, segno, false); + usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno); + if (valid_blocks == usable_blks_in_seg || !valid_blocks) + continue; + if (valid_blocks > usable_blks_in_seg) { + f2fs_bug_on(sbi, 1); + continue; + } + mutex_lock(&dirty_i->seglist_lock); + __locate_dirty_segment(sbi, segno, DIRTY); + mutex_unlock(&dirty_i->seglist_lock); + } + + if (!__is_large_section(sbi)) + return; + + mutex_lock(&dirty_i->seglist_lock); + for (segno = 0; segno < MAIN_SEGS(sbi); segno += SEGS_PER_SEC(sbi)) { + valid_blocks = get_valid_blocks(sbi, segno, true); + secno = GET_SEC_FROM_SEG(sbi, segno); + + if (!valid_blocks || valid_blocks == CAP_BLKS_PER_SEC(sbi)) + continue; + if (is_cursec(sbi, secno)) + continue; + set_bit(secno, dirty_i->dirty_secmap); + } + mutex_unlock(&dirty_i->seglist_lock); +} + +static int init_victim_secmap(struct f2fs_sb_info *sbi) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); + + dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL); + if (!dirty_i->victim_secmap) + return -ENOMEM; + + dirty_i->pinned_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL); + if (!dirty_i->pinned_secmap) + return -ENOMEM; + + dirty_i->pinned_secmap_cnt = 0; + dirty_i->enable_pin_section = true; + return 0; +} + +static int build_dirty_segmap(struct f2fs_sb_info *sbi) +{ + struct dirty_seglist_info *dirty_i; + unsigned int bitmap_size, i; + + /* allocate memory for dirty segments list information */ + dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info), + GFP_KERNEL); + if (!dirty_i) + return -ENOMEM; + + SM_I(sbi)->dirty_info = dirty_i; + mutex_init(&dirty_i->seglist_lock); + + bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi)); + + for (i = 0; i < NR_DIRTY_TYPE; i++) { + dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size, + GFP_KERNEL); + if (!dirty_i->dirty_segmap[i]) + return -ENOMEM; + } + + if (__is_large_section(sbi)) { + bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi)); + dirty_i->dirty_secmap = f2fs_kvzalloc(sbi, + bitmap_size, GFP_KERNEL); + if (!dirty_i->dirty_secmap) + return -ENOMEM; + } + + init_dirty_segmap(sbi); + return init_victim_secmap(sbi); +} + +static int sanity_check_curseg(struct f2fs_sb_info *sbi) +{ + int i; + + /* + * In LFS/SSR curseg, .next_blkoff should point to an unused blkaddr; + * In LFS curseg, all blkaddr after .next_blkoff should be unused. + */ + for (i = 0; i < NR_PERSISTENT_LOG; i++) { + struct curseg_info *curseg = CURSEG_I(sbi, i); + struct seg_entry *se = get_seg_entry(sbi, curseg->segno); + unsigned int blkofs = curseg->next_blkoff; + + if (f2fs_sb_has_readonly(sbi) && + i != CURSEG_HOT_DATA && i != CURSEG_HOT_NODE) + continue; + + sanity_check_seg_type(sbi, curseg->seg_type); + + if (curseg->alloc_type != LFS && curseg->alloc_type != SSR) { + f2fs_err(sbi, + "Current segment has invalid alloc_type:%d", + curseg->alloc_type); + f2fs_handle_error(sbi, ERROR_INVALID_CURSEG); + return -EFSCORRUPTED; + } + + if (f2fs_test_bit(blkofs, se->cur_valid_map)) + goto out; + + if (curseg->alloc_type == SSR) + continue; + + for (blkofs += 1; blkofs < BLKS_PER_SEG(sbi); blkofs++) { + if (!f2fs_test_bit(blkofs, se->cur_valid_map)) + continue; +out: + f2fs_err(sbi, + "Current segment's next free block offset is inconsistent with bitmap, logtype:%u, segno:%u, type:%u, next_blkoff:%u, blkofs:%u", + i, curseg->segno, curseg->alloc_type, + curseg->next_blkoff, blkofs); + f2fs_handle_error(sbi, ERROR_INVALID_CURSEG); + return -EFSCORRUPTED; + } + } + return 0; +} + +#ifdef CONFIG_BLK_DEV_ZONED +static int check_zone_write_pointer(struct f2fs_sb_info *sbi, + struct f2fs_dev_info *fdev, + struct blk_zone *zone) +{ + unsigned int zone_segno; + block_t zone_block, valid_block_cnt; + unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT; + int ret; + unsigned int nofs_flags; + + if (zone->type != BLK_ZONE_TYPE_SEQWRITE_REQ) + return 0; + + zone_block = fdev->start_blk + (zone->start >> log_sectors_per_block); + zone_segno = GET_SEGNO(sbi, zone_block); + + /* + * Skip check of zones cursegs point to, since + * fix_curseg_write_pointer() checks them. + */ + if (zone_segno >= MAIN_SEGS(sbi)) + return 0; + + /* + * Get # of valid block of the zone. + */ + valid_block_cnt = get_valid_blocks(sbi, zone_segno, true); + if (is_cursec(sbi, GET_SEC_FROM_SEG(sbi, zone_segno))) { + f2fs_notice(sbi, "Open zones: valid block[0x%x,0x%x] cond[%s]", + zone_segno, valid_block_cnt, + blk_zone_cond_str(zone->cond)); + return 0; + } + + if ((!valid_block_cnt && zone->cond == BLK_ZONE_COND_EMPTY) || + (valid_block_cnt && zone->cond == BLK_ZONE_COND_FULL)) + return 0; + + if (!valid_block_cnt) { + f2fs_notice(sbi, "Zone without valid block has non-zero write " + "pointer. Reset the write pointer: cond[%s]", + blk_zone_cond_str(zone->cond)); + ret = __f2fs_issue_discard_zone(sbi, fdev->bdev, zone_block, + zone->len >> log_sectors_per_block); + if (ret) + f2fs_err(sbi, "Discard zone failed: %s (errno=%d)", + fdev->path, ret); + return ret; + } + + /* + * If there are valid blocks and the write pointer doesn't match + * with them, we need to report the inconsistency and fill + * the zone till the end to close the zone. This inconsistency + * does not cause write error because the zone will not be + * selected for write operation until it get discarded. + */ + f2fs_notice(sbi, "Valid blocks are not aligned with write " + "pointer: valid block[0x%x,0x%x] cond[%s]", + zone_segno, valid_block_cnt, blk_zone_cond_str(zone->cond)); + + nofs_flags = memalloc_nofs_save(); + ret = blkdev_zone_mgmt(fdev->bdev, REQ_OP_ZONE_FINISH, + zone->start, zone->len); + memalloc_nofs_restore(nofs_flags); + if (ret == -EOPNOTSUPP) { + ret = blkdev_issue_zeroout(fdev->bdev, zone->wp, + zone->len - (zone->wp - zone->start), + GFP_NOFS, 0); + if (ret) + f2fs_err(sbi, "Fill up zone failed: %s (errno=%d)", + fdev->path, ret); + } else if (ret) { + f2fs_err(sbi, "Finishing zone failed: %s (errno=%d)", + fdev->path, ret); + } + + return ret; +} + +static struct f2fs_dev_info *get_target_zoned_dev(struct f2fs_sb_info *sbi, + block_t zone_blkaddr) +{ + int i; + + for (i = 0; i < sbi->s_ndevs; i++) { + if (!bdev_is_zoned(FDEV(i).bdev)) + continue; + if (sbi->s_ndevs == 1 || (FDEV(i).start_blk <= zone_blkaddr && + zone_blkaddr <= FDEV(i).end_blk)) + return &FDEV(i); + } + + return NULL; +} + +static int report_one_zone_cb(struct blk_zone *zone, unsigned int idx, + void *data) +{ + memcpy(data, zone, sizeof(struct blk_zone)); + return 0; +} + +static int do_fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type) +{ + struct curseg_info *cs = CURSEG_I(sbi, type); + struct f2fs_dev_info *zbd; + struct blk_zone zone; + unsigned int cs_section, wp_segno, wp_blkoff, wp_sector_off; + block_t cs_zone_block, wp_block; + unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT; + sector_t zone_sector; + int err; + + cs_section = GET_SEC_FROM_SEG(sbi, cs->segno); + cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section)); + + zbd = get_target_zoned_dev(sbi, cs_zone_block); + if (!zbd) + return 0; + + /* report zone for the sector the curseg points to */ + zone_sector = (sector_t)(cs_zone_block - zbd->start_blk) + << log_sectors_per_block; + err = blkdev_report_zones(zbd->bdev, zone_sector, 1, + report_one_zone_cb, &zone); + if (err != 1) { + f2fs_err(sbi, "Report zone failed: %s errno=(%d)", + zbd->path, err); + return err; + } + + if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ) + return 0; + + /* + * When safely unmounted in the previous mount, we could use current + * segments. Otherwise, allocate new sections. + */ + if (is_set_ckpt_flags(sbi, CP_UMOUNT_FLAG)) { + wp_block = zbd->start_blk + (zone.wp >> log_sectors_per_block); + wp_segno = GET_SEGNO(sbi, wp_block); + wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno); + wp_sector_off = zone.wp & GENMASK(log_sectors_per_block - 1, 0); + + if (cs->segno == wp_segno && cs->next_blkoff == wp_blkoff && + wp_sector_off == 0) + return 0; + + f2fs_notice(sbi, "Unaligned curseg[%d] with write pointer: " + "curseg[0x%x,0x%x] wp[0x%x,0x%x]", type, cs->segno, + cs->next_blkoff, wp_segno, wp_blkoff); + } + + /* Allocate a new section if it's not new. */ + if (cs->next_blkoff || + cs->segno != GET_SEG_FROM_SEC(sbi, GET_ZONE_FROM_SEC(sbi, cs_section))) { + unsigned int old_segno = cs->segno, old_blkoff = cs->next_blkoff; + + f2fs_allocate_new_section(sbi, type, true); + f2fs_notice(sbi, "Assign new section to curseg[%d]: " + "[0x%x,0x%x] -> [0x%x,0x%x]", + type, old_segno, old_blkoff, + cs->segno, cs->next_blkoff); + } + + /* check consistency of the zone curseg pointed to */ + if (check_zone_write_pointer(sbi, zbd, &zone)) + return -EIO; + + /* check newly assigned zone */ + cs_section = GET_SEC_FROM_SEG(sbi, cs->segno); + cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section)); + + zbd = get_target_zoned_dev(sbi, cs_zone_block); + if (!zbd) + return 0; + + zone_sector = (sector_t)(cs_zone_block - zbd->start_blk) + << log_sectors_per_block; + err = blkdev_report_zones(zbd->bdev, zone_sector, 1, + report_one_zone_cb, &zone); + if (err != 1) { + f2fs_err(sbi, "Report zone failed: %s errno=(%d)", + zbd->path, err); + return err; + } + + if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ) + return 0; + + if (zone.wp != zone.start) { + f2fs_notice(sbi, + "New zone for curseg[%d] is not yet discarded. " + "Reset the zone: curseg[0x%x,0x%x]", + type, cs->segno, cs->next_blkoff); + err = __f2fs_issue_discard_zone(sbi, zbd->bdev, cs_zone_block, + zone.len >> log_sectors_per_block); + if (err) { + f2fs_err(sbi, "Discard zone failed: %s (errno=%d)", + zbd->path, err); + return err; + } + } + + return 0; +} + +static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi) +{ + int i, ret; + + for (i = 0; i < NR_PERSISTENT_LOG; i++) { + ret = do_fix_curseg_write_pointer(sbi, i); + if (ret) + return ret; + } + + return 0; +} + +struct check_zone_write_pointer_args { + struct f2fs_sb_info *sbi; + struct f2fs_dev_info *fdev; +}; + +static int check_zone_write_pointer_cb(struct blk_zone *zone, unsigned int idx, + void *data) +{ + struct check_zone_write_pointer_args *args; + + args = (struct check_zone_write_pointer_args *)data; + + return check_zone_write_pointer(args->sbi, args->fdev, zone); +} + +static int check_write_pointer(struct f2fs_sb_info *sbi) +{ + int i, ret; + struct check_zone_write_pointer_args args; + + for (i = 0; i < sbi->s_ndevs; i++) { + if (!bdev_is_zoned(FDEV(i).bdev)) + continue; + + args.sbi = sbi; + args.fdev = &FDEV(i); + ret = blkdev_report_zones(FDEV(i).bdev, 0, BLK_ALL_ZONES, + check_zone_write_pointer_cb, &args); + if (ret < 0) + return ret; + } + + return 0; +} + +int f2fs_check_and_fix_write_pointer(struct f2fs_sb_info *sbi) +{ + int ret; + + if (!f2fs_sb_has_blkzoned(sbi) || f2fs_readonly(sbi->sb) || + f2fs_hw_is_readonly(sbi)) + return 0; + + f2fs_notice(sbi, "Checking entire write pointers"); + ret = fix_curseg_write_pointer(sbi); + if (!ret) + ret = check_write_pointer(sbi); + return ret; +} + +/* + * Return the number of usable blocks in a segment. The number of blocks + * returned is always equal to the number of blocks in a segment for + * segments fully contained within a sequential zone capacity or a + * conventional zone. For segments partially contained in a sequential + * zone capacity, the number of usable blocks up to the zone capacity + * is returned. 0 is returned in all other cases. + */ +static inline unsigned int f2fs_usable_zone_blks_in_seg( + struct f2fs_sb_info *sbi, unsigned int segno) +{ + block_t seg_start, sec_start_blkaddr, sec_cap_blkaddr; + unsigned int secno; + + if (!sbi->unusable_blocks_per_sec) + return BLKS_PER_SEG(sbi); + + secno = GET_SEC_FROM_SEG(sbi, segno); + seg_start = START_BLOCK(sbi, segno); + sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno)); + sec_cap_blkaddr = sec_start_blkaddr + CAP_BLKS_PER_SEC(sbi); + + /* + * If segment starts before zone capacity and spans beyond + * zone capacity, then usable blocks are from seg start to + * zone capacity. If the segment starts after the zone capacity, + * then there are no usable blocks. + */ + if (seg_start >= sec_cap_blkaddr) + return 0; + if (seg_start + BLKS_PER_SEG(sbi) > sec_cap_blkaddr) + return sec_cap_blkaddr - seg_start; + + return BLKS_PER_SEG(sbi); +} +#else +int f2fs_check_and_fix_write_pointer(struct f2fs_sb_info *sbi) +{ + return 0; +} + +static inline unsigned int f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info *sbi, + unsigned int segno) +{ + return 0; +} + +#endif +unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi, + unsigned int segno) +{ + if (f2fs_sb_has_blkzoned(sbi)) + return f2fs_usable_zone_blks_in_seg(sbi, segno); + + return BLKS_PER_SEG(sbi); +} + +unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi) +{ + if (f2fs_sb_has_blkzoned(sbi)) + return CAP_SEGS_PER_SEC(sbi); + + return SEGS_PER_SEC(sbi); +} + +unsigned long long f2fs_get_section_mtime(struct f2fs_sb_info *sbi, + unsigned int segno) +{ + unsigned int usable_segs_per_sec = f2fs_usable_segs_in_sec(sbi); + unsigned int secno = 0, start = 0; + unsigned int total_valid_blocks = 0; + unsigned long long mtime = 0; + unsigned int i = 0; + + secno = GET_SEC_FROM_SEG(sbi, segno); + start = GET_SEG_FROM_SEC(sbi, secno); + + if (!__is_large_section(sbi)) { + mtime = get_seg_entry(sbi, start + i)->mtime; + goto out; + } + + for (i = 0; i < usable_segs_per_sec; i++) { + /* for large section, only check the mtime of valid segments */ + struct seg_entry *se = get_seg_entry(sbi, start+i); + + mtime += se->mtime * se->valid_blocks; + total_valid_blocks += se->valid_blocks; + } + + if (total_valid_blocks == 0) + return INVALID_MTIME; + + mtime = div_u64(mtime, total_valid_blocks); +out: + if (unlikely(mtime == INVALID_MTIME)) + mtime -= 1; + return mtime; +} + +/* + * Update min, max modified time for cost-benefit GC algorithm + */ +static void init_min_max_mtime(struct f2fs_sb_info *sbi) +{ + struct sit_info *sit_i = SIT_I(sbi); + unsigned int segno; + + down_write(&sit_i->sentry_lock); + + sit_i->min_mtime = ULLONG_MAX; + + for (segno = 0; segno < MAIN_SEGS(sbi); segno += SEGS_PER_SEC(sbi)) { + unsigned long long mtime = 0; + + mtime = f2fs_get_section_mtime(sbi, segno); + + if (sit_i->min_mtime > mtime) + sit_i->min_mtime = mtime; + } + sit_i->max_mtime = get_mtime(sbi, false); + sit_i->dirty_max_mtime = 0; + up_write(&sit_i->sentry_lock); +} + +int f2fs_build_segment_manager(struct f2fs_sb_info *sbi) +{ + struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); + struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); + struct f2fs_sm_info *sm_info; + int err; + + sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL); + if (!sm_info) + return -ENOMEM; + + /* init sm info */ + sbi->sm_info = sm_info; + sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr); + sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr); + sm_info->segment_count = le32_to_cpu(raw_super->segment_count); + sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count); + sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count); + sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main); + sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr); + sm_info->rec_prefree_segments = sm_info->main_segments * + DEF_RECLAIM_PREFREE_SEGMENTS / 100; + if (sm_info->rec_prefree_segments > DEF_MAX_RECLAIM_PREFREE_SEGMENTS) + sm_info->rec_prefree_segments = DEF_MAX_RECLAIM_PREFREE_SEGMENTS; + + if (!f2fs_lfs_mode(sbi)) + sm_info->ipu_policy = BIT(F2FS_IPU_FSYNC); + sm_info->min_ipu_util = DEF_MIN_IPU_UTIL; + sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS; + sm_info->min_seq_blocks = BLKS_PER_SEG(sbi); + sm_info->min_hot_blocks = DEF_MIN_HOT_BLOCKS; + sm_info->min_ssr_sections = reserved_sections(sbi); + + INIT_LIST_HEAD(&sm_info->sit_entry_set); + + init_f2fs_rwsem(&sm_info->curseg_lock); + + err = f2fs_create_flush_cmd_control(sbi); + if (err) + return err; + + err = create_discard_cmd_control(sbi); + if (err) + return err; + + err = build_sit_info(sbi); + if (err) + return err; + err = build_free_segmap(sbi); + if (err) + return err; + err = build_curseg(sbi); + if (err) + return err; + + /* reinit free segmap based on SIT */ + err = build_sit_entries(sbi); + if (err) + return err; + + init_free_segmap(sbi); + err = build_dirty_segmap(sbi); + if (err) + return err; + + err = sanity_check_curseg(sbi); + if (err) + return err; + + init_min_max_mtime(sbi); + return 0; +} + +static void discard_dirty_segmap(struct f2fs_sb_info *sbi, + enum dirty_type dirty_type) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + + mutex_lock(&dirty_i->seglist_lock); + kvfree(dirty_i->dirty_segmap[dirty_type]); + dirty_i->nr_dirty[dirty_type] = 0; + mutex_unlock(&dirty_i->seglist_lock); +} + +static void destroy_victim_secmap(struct f2fs_sb_info *sbi) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + + kvfree(dirty_i->pinned_secmap); + kvfree(dirty_i->victim_secmap); +} + +static void destroy_dirty_segmap(struct f2fs_sb_info *sbi) +{ + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + int i; + + if (!dirty_i) + return; + + /* discard pre-free/dirty segments list */ + for (i = 0; i < NR_DIRTY_TYPE; i++) + discard_dirty_segmap(sbi, i); + + if (__is_large_section(sbi)) { + mutex_lock(&dirty_i->seglist_lock); + kvfree(dirty_i->dirty_secmap); + mutex_unlock(&dirty_i->seglist_lock); + } + + destroy_victim_secmap(sbi); + SM_I(sbi)->dirty_info = NULL; + kfree(dirty_i); +} + +static void destroy_curseg(struct f2fs_sb_info *sbi) +{ + struct curseg_info *array = SM_I(sbi)->curseg_array; + int i; + + if (!array) + return; + SM_I(sbi)->curseg_array = NULL; + for (i = 0; i < NR_CURSEG_TYPE; i++) { + kfree(array[i].sum_blk); + kfree(array[i].journal); + } + kfree(array); +} + +static void destroy_free_segmap(struct f2fs_sb_info *sbi) +{ + struct free_segmap_info *free_i = SM_I(sbi)->free_info; + + if (!free_i) + return; + SM_I(sbi)->free_info = NULL; + kvfree(free_i->free_segmap); + kvfree(free_i->free_secmap); + kfree(free_i); +} + +static void destroy_sit_info(struct f2fs_sb_info *sbi) +{ + struct sit_info *sit_i = SIT_I(sbi); + + if (!sit_i) + return; + + if (sit_i->sentries) + kvfree(sit_i->bitmap); + kfree(sit_i->tmp_map); + + kvfree(sit_i->sentries); + kvfree(sit_i->sec_entries); + kvfree(sit_i->dirty_sentries_bitmap); + + SM_I(sbi)->sit_info = NULL; + kfree(sit_i->sit_bitmap); +#ifdef CONFIG_F2FS_CHECK_FS + kvfree(sit_i->invalid_segmap); +#endif + kfree(sit_i); +} + +void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi) +{ + struct f2fs_sm_info *sm_info = SM_I(sbi); + + if (!sm_info) + return; + f2fs_destroy_flush_cmd_control(sbi, true); + destroy_discard_cmd_control(sbi); + destroy_dirty_segmap(sbi); + destroy_curseg(sbi); + destroy_free_segmap(sbi); + destroy_sit_info(sbi); + sbi->sm_info = NULL; + kfree(sm_info); +} + +int __init f2fs_create_segment_manager_caches(void) +{ + discard_entry_slab = f2fs_kmem_cache_create("f2fs_discard_entry", + sizeof(struct discard_entry)); + if (!discard_entry_slab) + goto fail; + + discard_cmd_slab = f2fs_kmem_cache_create("f2fs_discard_cmd", + sizeof(struct discard_cmd)); + if (!discard_cmd_slab) + goto destroy_discard_entry; + + sit_entry_set_slab = f2fs_kmem_cache_create("f2fs_sit_entry_set", + sizeof(struct sit_entry_set)); + if (!sit_entry_set_slab) + goto destroy_discard_cmd; + + revoke_entry_slab = f2fs_kmem_cache_create("f2fs_revoke_entry", + sizeof(struct revoke_entry)); + if (!revoke_entry_slab) + goto destroy_sit_entry_set; + return 0; + +destroy_sit_entry_set: + kmem_cache_destroy(sit_entry_set_slab); +destroy_discard_cmd: + kmem_cache_destroy(discard_cmd_slab); +destroy_discard_entry: + kmem_cache_destroy(discard_entry_slab); +fail: + return -ENOMEM; +} + +void f2fs_destroy_segment_manager_caches(void) +{ + kmem_cache_destroy(sit_entry_set_slab); + kmem_cache_destroy(discard_cmd_slab); + kmem_cache_destroy(discard_entry_slab); + kmem_cache_destroy(revoke_entry_slab); +} diff --git a/fs/f2fs/segment.h b/fs/f2fs/segment.h new file mode 100644 index 000000000..b0c06b358 --- /dev/null +++ b/fs/f2fs/segment.h @@ -0,0 +1,1030 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * fs/f2fs/segment.h + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + */ +#include +#include + +/* constant macro */ +#define NULL_SEGNO ((unsigned int)(~0)) +#define NULL_SECNO ((unsigned int)(~0)) + +#define DEF_RECLAIM_PREFREE_SEGMENTS 5 /* 5% over total segments */ +#define DEF_MAX_RECLAIM_PREFREE_SEGMENTS 4096 /* 8GB in maximum */ + +#define F2FS_MIN_SEGMENTS 9 /* SB + 2 (CP + SIT + NAT) + SSA + MAIN */ +#define F2FS_MIN_META_SEGMENTS 8 /* SB + 2 (CP + SIT + NAT) + SSA */ + +#define INVALID_MTIME ULLONG_MAX /* no valid blocks in a segment/section */ + +/* L: Logical segment # in volume, R: Relative segment # in main area */ +#define GET_L2R_SEGNO(free_i, segno) ((segno) - (free_i)->start_segno) +#define GET_R2L_SEGNO(free_i, segno) ((segno) + (free_i)->start_segno) + +#define IS_DATASEG(t) ((t) <= CURSEG_COLD_DATA) +#define IS_NODESEG(t) ((t) >= CURSEG_HOT_NODE && (t) <= CURSEG_COLD_NODE) +#define SE_PAGETYPE(se) ((IS_NODESEG((se)->type) ? NODE : DATA)) + +static inline void sanity_check_seg_type(struct f2fs_sb_info *sbi, + unsigned short seg_type) +{ + f2fs_bug_on(sbi, seg_type >= NR_PERSISTENT_LOG); +} + +#define MAIN_BLKADDR(sbi) \ + (SM_I(sbi) ? SM_I(sbi)->main_blkaddr : \ + le32_to_cpu(F2FS_RAW_SUPER(sbi)->main_blkaddr)) +#define SEG0_BLKADDR(sbi) \ + (SM_I(sbi) ? SM_I(sbi)->seg0_blkaddr : \ + le32_to_cpu(F2FS_RAW_SUPER(sbi)->segment0_blkaddr)) + +#define MAIN_SEGS(sbi) (SM_I(sbi)->main_segments) +#define MAIN_SECS(sbi) ((sbi)->total_sections) + +#define TOTAL_SEGS(sbi) \ + (SM_I(sbi) ? SM_I(sbi)->segment_count : \ + le32_to_cpu(F2FS_RAW_SUPER(sbi)->segment_count)) +#define TOTAL_BLKS(sbi) (SEGS_TO_BLKS(sbi, TOTAL_SEGS(sbi))) + +#define MAX_BLKADDR(sbi) (SEG0_BLKADDR(sbi) + TOTAL_BLKS(sbi)) +#define SEGMENT_SIZE(sbi) (1ULL << ((sbi)->log_blocksize + \ + (sbi)->log_blocks_per_seg)) + +#define START_BLOCK(sbi, segno) (SEG0_BLKADDR(sbi) + \ + (SEGS_TO_BLKS(sbi, GET_R2L_SEGNO(FREE_I(sbi), segno)))) + +#define NEXT_FREE_BLKADDR(sbi, curseg) \ + (START_BLOCK(sbi, (curseg)->segno) + (curseg)->next_blkoff) + +#define GET_SEGOFF_FROM_SEG0(sbi, blk_addr) ((blk_addr) - SEG0_BLKADDR(sbi)) +#define GET_SEGNO_FROM_SEG0(sbi, blk_addr) \ + (BLKS_TO_SEGS(sbi, GET_SEGOFF_FROM_SEG0(sbi, blk_addr))) +#define GET_BLKOFF_FROM_SEG0(sbi, blk_addr) \ + (GET_SEGOFF_FROM_SEG0(sbi, blk_addr) & (BLKS_PER_SEG(sbi) - 1)) + +#define GET_SEGNO(sbi, blk_addr) \ + ((!__is_valid_data_blkaddr(blk_addr)) ? \ + NULL_SEGNO : GET_L2R_SEGNO(FREE_I(sbi), \ + GET_SEGNO_FROM_SEG0(sbi, blk_addr))) +#ifdef CONFIG_BLK_DEV_ZONED +#define CAP_BLKS_PER_SEC(sbi) \ + (BLKS_PER_SEC(sbi) - (sbi)->unusable_blocks_per_sec) +#define CAP_SEGS_PER_SEC(sbi) \ + (SEGS_PER_SEC(sbi) - \ + BLKS_TO_SEGS(sbi, (sbi)->unusable_blocks_per_sec)) +#else +#define CAP_BLKS_PER_SEC(sbi) BLKS_PER_SEC(sbi) +#define CAP_SEGS_PER_SEC(sbi) SEGS_PER_SEC(sbi) +#endif +#define GET_START_SEG_FROM_SEC(sbi, segno) \ + (rounddown(segno, SEGS_PER_SEC(sbi))) +#define GET_SEC_FROM_SEG(sbi, segno) \ + (((segno) == -1) ? -1 : (segno) / SEGS_PER_SEC(sbi)) +#define GET_SEG_FROM_SEC(sbi, secno) \ + ((secno) * SEGS_PER_SEC(sbi)) +#define GET_ZONE_FROM_SEC(sbi, secno) \ + (((secno) == -1) ? -1 : (secno) / (sbi)->secs_per_zone) +#define GET_ZONE_FROM_SEG(sbi, segno) \ + GET_ZONE_FROM_SEC(sbi, GET_SEC_FROM_SEG(sbi, segno)) + +#define GET_SUM_BLOCK(sbi, segno) \ + (SM_I(sbi)->ssa_blkaddr + (segno / (sbi)->sums_per_block)) +#define GET_SUM_BLKOFF(sbi, segno) (segno % (sbi)->sums_per_block) +#define SUM_BLK_PAGE_ADDR(sbi, folio, segno) \ + (folio_address(folio) + GET_SUM_BLKOFF(sbi, segno) * (sbi)->sum_blocksize) + +#define GET_SUM_TYPE(footer) ((footer)->entry_type) +#define SET_SUM_TYPE(footer, type) ((footer)->entry_type = (type)) + +#define SIT_ENTRY_OFFSET(sit_i, segno) \ + ((segno) % (sit_i)->sents_per_block) +#define SIT_BLOCK_OFFSET(segno) \ + ((segno) / SIT_ENTRY_PER_BLOCK) +#define START_SEGNO(segno) \ + (SIT_BLOCK_OFFSET(segno) * SIT_ENTRY_PER_BLOCK) +#define SIT_BLK_CNT(sbi) \ + DIV_ROUND_UP(MAIN_SEGS(sbi), SIT_ENTRY_PER_BLOCK) +#define f2fs_bitmap_size(nr) \ + (BITS_TO_LONGS(nr) * sizeof(unsigned long)) + +#define SECTOR_FROM_BLOCK(blk_addr) \ + (((sector_t)blk_addr) << F2FS_LOG_SECTORS_PER_BLOCK) +#define SECTOR_TO_BLOCK(sectors) \ + ((sectors) >> F2FS_LOG_SECTORS_PER_BLOCK) + +/* + * In the victim_sel_policy->alloc_mode, there are three block allocation modes. + * LFS writes data sequentially with cleaning operations. + * SSR (Slack Space Recycle) reuses obsolete space without cleaning operations. + * AT_SSR (Age Threshold based Slack Space Recycle) merges fragments into + * fragmented segment which has similar aging degree. + */ +enum { + LFS = 0, + SSR, + AT_SSR, +}; + +/* + * In the victim_sel_policy->gc_mode, there are three gc, aka cleaning, modes. + * GC_CB is based on cost-benefit algorithm. + * GC_GREEDY is based on greedy algorithm. + * GC_AT is based on age-threshold algorithm. + */ +enum { + GC_CB = 0, + GC_GREEDY, + GC_AT, + ALLOC_NEXT, + FLUSH_DEVICE, + MAX_GC_POLICY, +}; + +/* + * BG_GC means the background cleaning job. + * FG_GC means the on-demand cleaning job. + */ +enum { + BG_GC = 0, + FG_GC, +}; + +/* for a function parameter to select a victim segment */ +struct victim_sel_policy { + int alloc_mode; /* LFS or SSR */ + int gc_mode; /* GC_CB or GC_GREEDY */ + unsigned long *dirty_bitmap; /* dirty segment/section bitmap */ + unsigned int max_search; /* + * maximum # of segments/sections + * to search + */ + unsigned int offset; /* last scanned bitmap offset */ + unsigned int ofs_unit; /* bitmap search unit */ + unsigned int min_cost; /* minimum cost */ + unsigned long long oldest_age; /* oldest age of segments having the same min cost */ + unsigned int min_segno; /* segment # having min. cost */ + unsigned long long age; /* mtime of GCed section*/ + unsigned long long age_threshold;/* age threshold */ + bool one_time_gc; /* one time GC */ +}; + +struct seg_entry { + unsigned int type:6; /* segment type like CURSEG_XXX_TYPE */ + unsigned int valid_blocks:10; /* # of valid blocks */ + unsigned int ckpt_valid_blocks:10; /* # of valid blocks last cp */ + unsigned int padding:6; /* padding */ + unsigned char *cur_valid_map; /* validity bitmap of blocks */ + /* + * # of valid blocks and the validity bitmap stored in the last + * checkpoint pack. This information is used by the SSR mode. + */ + unsigned char *ckpt_valid_map; /* validity bitmap of blocks last cp */ + unsigned char *discard_map; + unsigned long long mtime; /* modification time of the segment */ +}; + +struct sec_entry { + unsigned int valid_blocks; /* # of valid blocks in a section */ + unsigned int ckpt_valid_blocks; /* # of valid blocks last cp in a section */ +}; + +#define MAX_SKIP_GC_COUNT 16 + +struct revoke_entry { + struct list_head list; + block_t old_addr; /* for revoking when fail to commit */ + pgoff_t index; +}; + +struct sit_info { + block_t sit_base_addr; /* start block address of SIT area */ + block_t sit_blocks; /* # of blocks used by SIT area */ + block_t written_valid_blocks; /* # of valid blocks in main area */ + char *bitmap; /* all bitmaps pointer */ + char *sit_bitmap; /* SIT bitmap pointer */ +#ifdef CONFIG_F2FS_CHECK_FS + /* bitmap of segments to be ignored by GC in case of errors */ + unsigned long *invalid_segmap; +#endif + unsigned int bitmap_size; /* SIT bitmap size */ + + unsigned long *tmp_map; /* bitmap for temporal use */ + unsigned long *dirty_sentries_bitmap; /* bitmap for dirty sentries */ + unsigned int dirty_sentries; /* # of dirty sentries */ + unsigned int sents_per_block; /* # of SIT entries per block */ + struct rw_semaphore sentry_lock; /* to protect SIT cache */ + struct seg_entry *sentries; /* SIT segment-level cache */ + struct sec_entry *sec_entries; /* SIT section-level cache */ + + /* for cost-benefit algorithm in cleaning procedure */ + unsigned long long elapsed_time; /* elapsed time after mount */ + unsigned long long mounted_time; /* mount time */ + unsigned long long min_mtime; /* min. modification time */ + unsigned long long max_mtime; /* max. modification time */ + unsigned long long dirty_min_mtime; /* rerange candidates in GC_AT */ + unsigned long long dirty_max_mtime; /* rerange candidates in GC_AT */ + + unsigned int last_victim[MAX_GC_POLICY]; /* last victim segment # */ +}; + +struct free_segmap_info { + unsigned int start_segno; /* start segment number logically */ + unsigned int free_segments; /* # of free segments */ + unsigned int free_sections; /* # of free sections */ + spinlock_t segmap_lock; /* free segmap lock */ + unsigned long *free_segmap; /* free segment bitmap */ + unsigned long *free_secmap; /* free section bitmap */ +}; + +/* Notice: The order of dirty type is same with CURSEG_XXX in f2fs.h */ +enum dirty_type { + DIRTY_HOT_DATA, /* dirty segments assigned as hot data logs */ + DIRTY_WARM_DATA, /* dirty segments assigned as warm data logs */ + DIRTY_COLD_DATA, /* dirty segments assigned as cold data logs */ + DIRTY_HOT_NODE, /* dirty segments assigned as hot node logs */ + DIRTY_WARM_NODE, /* dirty segments assigned as warm node logs */ + DIRTY_COLD_NODE, /* dirty segments assigned as cold node logs */ + DIRTY, /* to count # of dirty segments */ + PRE, /* to count # of entirely obsolete segments */ + NR_DIRTY_TYPE +}; + +struct dirty_seglist_info { + unsigned long *dirty_segmap[NR_DIRTY_TYPE]; + unsigned long *dirty_secmap; + struct mutex seglist_lock; /* lock for segment bitmaps */ + int nr_dirty[NR_DIRTY_TYPE]; /* # of dirty segments */ + unsigned long *victim_secmap; /* background GC victims */ + unsigned long *pinned_secmap; /* pinned victims from foreground GC */ + unsigned int pinned_secmap_cnt; /* count of victims which has pinned data */ + bool enable_pin_section; /* enable pinning section */ +}; + +/* for active log information */ +struct curseg_info { + struct mutex curseg_mutex; /* lock for consistency */ + struct f2fs_summary_block *sum_blk; /* cached summary block */ + struct rw_semaphore journal_rwsem; /* protect journal area */ + struct f2fs_journal *journal; /* cached journal info */ + unsigned char alloc_type; /* current allocation type */ + unsigned short seg_type; /* segment type like CURSEG_XXX_TYPE */ + unsigned int segno; /* current segment number */ + unsigned short next_blkoff; /* next block offset to write */ + unsigned int zone; /* current zone number */ + unsigned int next_segno; /* preallocated segment */ + int fragment_remained_chunk; /* remained block size in a chunk for block fragmentation mode */ + bool inited; /* indicate inmem log is inited */ +}; + +struct sit_entry_set { + struct list_head set_list; /* link with all sit sets */ + unsigned int start_segno; /* start segno of sits in set */ + unsigned int entry_cnt; /* the # of sit entries in set */ +}; + +/* + * inline functions + */ +static inline struct curseg_info *CURSEG_I(struct f2fs_sb_info *sbi, int type) +{ + return (struct curseg_info *)(SM_I(sbi)->curseg_array + type); +} + +static inline bool is_curseg(struct f2fs_sb_info *sbi, unsigned int segno) +{ + int i; + + for (i = CURSEG_HOT_DATA; i < NO_CHECK_TYPE; i++) { + if (segno == CURSEG_I(sbi, i)->segno) + return true; + } + return false; +} + +static inline bool is_cursec(struct f2fs_sb_info *sbi, unsigned int secno) +{ + int i; + + for (i = CURSEG_HOT_DATA; i < NO_CHECK_TYPE; i++) { + if (secno == GET_SEC_FROM_SEG(sbi, CURSEG_I(sbi, i)->segno)) + return true; + } + return false; +} + +static inline struct seg_entry *get_seg_entry(struct f2fs_sb_info *sbi, + unsigned int segno) +{ + struct sit_info *sit_i = SIT_I(sbi); + return &sit_i->sentries[segno]; +} + +static inline struct sec_entry *get_sec_entry(struct f2fs_sb_info *sbi, + unsigned int segno) +{ + struct sit_info *sit_i = SIT_I(sbi); + return &sit_i->sec_entries[GET_SEC_FROM_SEG(sbi, segno)]; +} + +static inline unsigned int get_valid_blocks(struct f2fs_sb_info *sbi, + unsigned int segno, bool use_section) +{ + /* + * In order to get # of valid blocks in a section instantly from many + * segments, f2fs manages two counting structures separately. + */ + if (use_section && __is_large_section(sbi)) + return get_sec_entry(sbi, segno)->valid_blocks; + else + return get_seg_entry(sbi, segno)->valid_blocks; +} + +static inline unsigned int get_ckpt_valid_blocks(struct f2fs_sb_info *sbi, + unsigned int segno, bool use_section) +{ + if (use_section && __is_large_section(sbi)) + return get_sec_entry(sbi, segno)->ckpt_valid_blocks; + else + return get_seg_entry(sbi, segno)->ckpt_valid_blocks; +} + +static inline void set_ckpt_valid_blocks(struct f2fs_sb_info *sbi, + unsigned int segno) +{ + unsigned int secno = GET_SEC_FROM_SEG(sbi, segno); + unsigned int start_segno = GET_SEG_FROM_SEC(sbi, secno); + unsigned int blocks = 0; + int i; + + for (i = 0; i < SEGS_PER_SEC(sbi); i++, start_segno++) { + struct seg_entry *se = get_seg_entry(sbi, start_segno); + + blocks += se->ckpt_valid_blocks; + } + get_sec_entry(sbi, segno)->ckpt_valid_blocks = blocks; +} + +#ifdef CONFIG_F2FS_CHECK_FS +static inline void sanity_check_valid_blocks(struct f2fs_sb_info *sbi, + unsigned int segno) +{ + unsigned int secno = GET_SEC_FROM_SEG(sbi, segno); + unsigned int start_segno = GET_SEG_FROM_SEC(sbi, secno); + unsigned int blocks = 0; + int i; + + for (i = 0; i < SEGS_PER_SEC(sbi); i++, start_segno++) { + struct seg_entry *se = get_seg_entry(sbi, start_segno); + + blocks += se->ckpt_valid_blocks; + } + + if (blocks != get_sec_entry(sbi, segno)->ckpt_valid_blocks) { + f2fs_err(sbi, + "Inconsistent ckpt valid blocks: " + "seg entry(%d) vs sec entry(%d) at secno %d", + blocks, get_sec_entry(sbi, segno)->ckpt_valid_blocks, secno); + f2fs_bug_on(sbi, 1); + } +} +#else +static inline void sanity_check_valid_blocks(struct f2fs_sb_info *sbi, + unsigned int segno) +{ +} +#endif +static inline void seg_info_from_raw_sit(struct seg_entry *se, + struct f2fs_sit_entry *rs) +{ + se->valid_blocks = GET_SIT_VBLOCKS(rs); + se->ckpt_valid_blocks = GET_SIT_VBLOCKS(rs); + memcpy(se->cur_valid_map, rs->valid_map, SIT_VBLOCK_MAP_SIZE); + memcpy(se->ckpt_valid_map, rs->valid_map, SIT_VBLOCK_MAP_SIZE); + se->type = GET_SIT_TYPE(rs); + se->mtime = le64_to_cpu(rs->mtime); +} + +static inline void __seg_info_to_raw_sit(struct seg_entry *se, + struct f2fs_sit_entry *rs) +{ + unsigned short raw_vblocks = (se->type << SIT_VBLOCKS_SHIFT) | + se->valid_blocks; + rs->vblocks = cpu_to_le16(raw_vblocks); + memcpy(rs->valid_map, se->cur_valid_map, SIT_VBLOCK_MAP_SIZE); + rs->mtime = cpu_to_le64(se->mtime); +} + +static inline void seg_info_to_sit_folio(struct f2fs_sb_info *sbi, + struct folio *folio, unsigned int start) +{ + struct f2fs_sit_block *raw_sit; + struct seg_entry *se; + struct f2fs_sit_entry *rs; + unsigned int end = min(start + SIT_ENTRY_PER_BLOCK, + (unsigned long)MAIN_SEGS(sbi)); + int i; + + raw_sit = folio_address(folio); + memset(raw_sit, 0, PAGE_SIZE); + for (i = 0; i < end - start; i++) { + rs = &raw_sit->entries[i]; + se = get_seg_entry(sbi, start + i); + __seg_info_to_raw_sit(se, rs); + } +} + +static inline void seg_info_to_raw_sit(struct seg_entry *se, + struct f2fs_sit_entry *rs) +{ + __seg_info_to_raw_sit(se, rs); + + memcpy(se->ckpt_valid_map, rs->valid_map, SIT_VBLOCK_MAP_SIZE); + se->ckpt_valid_blocks = se->valid_blocks; +} + +static inline unsigned int find_next_inuse(struct free_segmap_info *free_i, + unsigned int max, unsigned int segno) +{ + unsigned int ret; + spin_lock(&free_i->segmap_lock); + ret = find_next_bit(free_i->free_segmap, max, segno); + spin_unlock(&free_i->segmap_lock); + return ret; +} + +static inline void __set_free(struct f2fs_sb_info *sbi, unsigned int segno) +{ + struct free_segmap_info *free_i = FREE_I(sbi); + unsigned int secno = GET_SEC_FROM_SEG(sbi, segno); + unsigned int start_segno = GET_SEG_FROM_SEC(sbi, secno); + unsigned int next; + + spin_lock(&free_i->segmap_lock); + clear_bit(segno, free_i->free_segmap); + free_i->free_segments++; + + next = find_next_bit(free_i->free_segmap, + start_segno + SEGS_PER_SEC(sbi), start_segno); + if (next >= start_segno + f2fs_usable_segs_in_sec(sbi)) { + clear_bit(secno, free_i->free_secmap); + free_i->free_sections++; + } + spin_unlock(&free_i->segmap_lock); +} + +static inline void __set_inuse(struct f2fs_sb_info *sbi, + unsigned int segno) +{ + struct free_segmap_info *free_i = FREE_I(sbi); + unsigned int secno = GET_SEC_FROM_SEG(sbi, segno); + + set_bit(segno, free_i->free_segmap); + free_i->free_segments--; + if (!test_and_set_bit(secno, free_i->free_secmap)) + free_i->free_sections--; +} + +static inline void __set_test_and_free(struct f2fs_sb_info *sbi, + unsigned int segno, bool inmem) +{ + struct free_segmap_info *free_i = FREE_I(sbi); + unsigned int secno = GET_SEC_FROM_SEG(sbi, segno); + unsigned int start_segno = GET_SEG_FROM_SEC(sbi, secno); + unsigned int next; + bool ret; + + spin_lock(&free_i->segmap_lock); + ret = test_and_clear_bit(segno, free_i->free_segmap); + if (!ret) + goto unlock_out; + + free_i->free_segments++; + + if (!inmem && is_cursec(sbi, secno)) + goto unlock_out; + + /* check large section */ + next = find_next_bit(free_i->free_segmap, + start_segno + SEGS_PER_SEC(sbi), start_segno); + if (next < start_segno + f2fs_usable_segs_in_sec(sbi)) + goto unlock_out; + + ret = test_and_clear_bit(secno, free_i->free_secmap); + if (!ret) + goto unlock_out; + + free_i->free_sections++; + + if (GET_SEC_FROM_SEG(sbi, sbi->next_victim_seg[BG_GC]) == secno) + sbi->next_victim_seg[BG_GC] = NULL_SEGNO; + if (GET_SEC_FROM_SEG(sbi, sbi->next_victim_seg[FG_GC]) == secno) + sbi->next_victim_seg[FG_GC] = NULL_SEGNO; + +unlock_out: + spin_unlock(&free_i->segmap_lock); +} + +static inline void __set_test_and_inuse(struct f2fs_sb_info *sbi, + unsigned int segno) +{ + struct free_segmap_info *free_i = FREE_I(sbi); + unsigned int secno = GET_SEC_FROM_SEG(sbi, segno); + + spin_lock(&free_i->segmap_lock); + if (!test_and_set_bit(segno, free_i->free_segmap)) { + free_i->free_segments--; + if (!test_and_set_bit(secno, free_i->free_secmap)) + free_i->free_sections--; + } + spin_unlock(&free_i->segmap_lock); +} + +static inline void get_sit_bitmap(struct f2fs_sb_info *sbi, + void *dst_addr) +{ + struct sit_info *sit_i = SIT_I(sbi); + + memcpy(dst_addr, sit_i->sit_bitmap, sit_i->bitmap_size); +} + +static inline block_t written_block_count(struct f2fs_sb_info *sbi) +{ + return SIT_I(sbi)->written_valid_blocks; +} + +static inline unsigned int free_segments(struct f2fs_sb_info *sbi) +{ + return FREE_I(sbi)->free_segments; +} + +static inline unsigned int reserved_segments(struct f2fs_sb_info *sbi) +{ + return SM_I(sbi)->reserved_segments; +} + +static inline unsigned int free_sections(struct f2fs_sb_info *sbi) +{ + return FREE_I(sbi)->free_sections; +} + +static inline unsigned int prefree_segments(struct f2fs_sb_info *sbi) +{ + return DIRTY_I(sbi)->nr_dirty[PRE]; +} + +static inline unsigned int dirty_segments(struct f2fs_sb_info *sbi) +{ + return DIRTY_I(sbi)->nr_dirty[DIRTY_HOT_DATA] + + DIRTY_I(sbi)->nr_dirty[DIRTY_WARM_DATA] + + DIRTY_I(sbi)->nr_dirty[DIRTY_COLD_DATA] + + DIRTY_I(sbi)->nr_dirty[DIRTY_HOT_NODE] + + DIRTY_I(sbi)->nr_dirty[DIRTY_WARM_NODE] + + DIRTY_I(sbi)->nr_dirty[DIRTY_COLD_NODE]; +} + +static inline int overprovision_segments(struct f2fs_sb_info *sbi) +{ + return SM_I(sbi)->ovp_segments; +} + +static inline int reserved_sections(struct f2fs_sb_info *sbi) +{ + return GET_SEC_FROM_SEG(sbi, reserved_segments(sbi)); +} + +static inline unsigned int get_left_section_blocks(struct f2fs_sb_info *sbi, + enum log_type type, unsigned int segno) +{ + if (f2fs_lfs_mode(sbi)) { + unsigned int used_blocks = __is_large_section(sbi) ? SEGS_TO_BLKS(sbi, + (segno - GET_START_SEG_FROM_SEC(sbi, segno))) : 0; + return CAP_BLKS_PER_SEC(sbi) - used_blocks - + CURSEG_I(sbi, type)->next_blkoff; + } + return CAP_BLKS_PER_SEC(sbi) - get_ckpt_valid_blocks(sbi, segno, true); +} + +static inline void get_additional_blocks_required(struct f2fs_sb_info *sbi, + unsigned int *total_node_blocks, unsigned int *total_data_blocks, + unsigned int *total_dent_blocks, bool separate_dent) +{ + unsigned int segno, left_blocks; + int i; + unsigned int min_free_node_blocks = CAP_BLKS_PER_SEC(sbi); + unsigned int min_free_dent_blocks = CAP_BLKS_PER_SEC(sbi); + unsigned int min_free_data_blocks = CAP_BLKS_PER_SEC(sbi); + + /* check current data/node sections in the worst case. */ + for (i = CURSEG_HOT_DATA; i < NR_PERSISTENT_LOG; i++) { + segno = CURSEG_I(sbi, i)->segno; + + if (unlikely(segno == NULL_SEGNO)) + return; + + left_blocks = get_left_section_blocks(sbi, i, segno); + + if (i > CURSEG_COLD_DATA) + min_free_node_blocks = min(min_free_node_blocks, left_blocks); + else if (i == CURSEG_HOT_DATA && separate_dent) + min_free_dent_blocks = left_blocks; + else + min_free_data_blocks = min(min_free_data_blocks, left_blocks); + } + + *total_node_blocks = (*total_node_blocks > min_free_node_blocks) ? + *total_node_blocks - min_free_node_blocks : 0; + *total_dent_blocks = (*total_dent_blocks > min_free_dent_blocks) ? + *total_dent_blocks - min_free_dent_blocks : 0; + *total_data_blocks = (*total_data_blocks > min_free_data_blocks) ? + *total_data_blocks - min_free_data_blocks : 0; +} + +/* + * call get_additional_blocks_required to calculate dirty blocks + * needing to be placed in free sections, please note that, it + * needs to account dirty data as well in lfs mode when checkpoint + * is disabled. + */ +static inline int __get_secs_required(struct f2fs_sb_info *sbi) +{ + unsigned int total_node_blocks = get_pages(sbi, F2FS_DIRTY_NODES) + + get_pages(sbi, F2FS_DIRTY_DENTS) + + get_pages(sbi, F2FS_DIRTY_IMETA); + unsigned int total_dent_blocks = get_pages(sbi, F2FS_DIRTY_DENTS); + unsigned int total_data_blocks = 0; + bool separate_dent = true; + + if (f2fs_lfs_mode(sbi)) + total_data_blocks = get_pages(sbi, F2FS_DIRTY_DATA); + + /* + * When active_logs != 4, dentry blocks and data blocks can be + * mixed in the same logs, so check their space together. + */ + if (F2FS_OPTION(sbi).active_logs != 4) { + total_data_blocks += total_dent_blocks; + total_dent_blocks = 0; + separate_dent = false; + } + + get_additional_blocks_required(sbi, &total_node_blocks, &total_dent_blocks, + &total_data_blocks, separate_dent); + + return DIV_ROUND_UP(total_node_blocks, CAP_BLKS_PER_SEC(sbi)) + + DIV_ROUND_UP(total_dent_blocks, CAP_BLKS_PER_SEC(sbi)) + + DIV_ROUND_UP(total_data_blocks, CAP_BLKS_PER_SEC(sbi)); +} + +static inline bool has_not_enough_free_secs(struct f2fs_sb_info *sbi, + int freed, int needed) +{ + unsigned int free_secs, required_secs; + + if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) + return false; + + free_secs = free_sections(sbi) + freed; + required_secs = needed + reserved_sections(sbi) + + __get_secs_required(sbi); + + return free_secs < required_secs; +} + +static inline bool has_enough_free_secs(struct f2fs_sb_info *sbi, + int freed, int needed) +{ + return !has_not_enough_free_secs(sbi, freed, needed); +} + +static inline bool has_enough_free_blks(struct f2fs_sb_info *sbi) +{ + unsigned int total_free_blocks = 0; + unsigned int avail_user_block_count; + + spin_lock(&sbi->stat_lock); + + avail_user_block_count = get_available_block_count(sbi, NULL, true); + total_free_blocks = avail_user_block_count - (unsigned int)valid_user_blocks(sbi); + + spin_unlock(&sbi->stat_lock); + + return total_free_blocks > 0; +} + +static inline bool f2fs_is_checkpoint_ready(struct f2fs_sb_info *sbi) +{ + if (likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED))) + return true; + if (likely(has_enough_free_secs(sbi, 0, 0))) + return true; + if (!f2fs_lfs_mode(sbi) && + likely(has_enough_free_blks(sbi))) + return true; + return false; +} + +static inline bool excess_prefree_segs(struct f2fs_sb_info *sbi) +{ + return prefree_segments(sbi) > SM_I(sbi)->rec_prefree_segments; +} + +static inline int utilization(struct f2fs_sb_info *sbi) +{ + return div_u64((u64)valid_user_blocks(sbi) * 100, + sbi->user_block_count); +} + +/* + * Sometimes f2fs may be better to drop out-of-place update policy. + * And, users can control the policy through sysfs entries. + * There are five policies with triggering conditions as follows. + * F2FS_IPU_FORCE - all the time, + * F2FS_IPU_SSR - if SSR mode is activated, + * F2FS_IPU_UTIL - if FS utilization is over threashold, + * F2FS_IPU_SSR_UTIL - if SSR mode is activated and FS utilization is over + * threashold, + * F2FS_IPU_FSYNC - activated in fsync path only for high performance flash + * storages. IPU will be triggered only if the # of dirty + * pages over min_fsync_blocks. (=default option) + * F2FS_IPU_ASYNC - do IPU given by asynchronous write requests. + * F2FS_IPU_NOCACHE - disable IPU bio cache. + * F2FS_IPU_HONOR_OPU_WRITE - use OPU write prior to IPU write if inode has + * FI_OPU_WRITE flag. + * F2FS_IPU_DISABLE - disable IPU. (=default option in LFS mode) + */ +#define DEF_MIN_IPU_UTIL 70 +#define DEF_MIN_FSYNC_BLOCKS 8 +#define DEF_MIN_HOT_BLOCKS 16 + +#define SMALL_VOLUME_SEGMENTS (16 * 512) /* 16GB */ + +#define F2FS_IPU_DISABLE 0 + +/* Modification on enum should be synchronized with ipu_mode_names array */ +enum { + F2FS_IPU_FORCE, + F2FS_IPU_SSR, + F2FS_IPU_UTIL, + F2FS_IPU_SSR_UTIL, + F2FS_IPU_FSYNC, + F2FS_IPU_ASYNC, + F2FS_IPU_NOCACHE, + F2FS_IPU_HONOR_OPU_WRITE, + F2FS_IPU_MAX, +}; + +static inline bool IS_F2FS_IPU_DISABLE(struct f2fs_sb_info *sbi) +{ + return SM_I(sbi)->ipu_policy == F2FS_IPU_DISABLE; +} + +#define F2FS_IPU_POLICY(name) \ +static inline bool IS_##name(struct f2fs_sb_info *sbi) \ +{ \ + return SM_I(sbi)->ipu_policy & BIT(name); \ +} + +F2FS_IPU_POLICY(F2FS_IPU_FORCE); +F2FS_IPU_POLICY(F2FS_IPU_SSR); +F2FS_IPU_POLICY(F2FS_IPU_UTIL); +F2FS_IPU_POLICY(F2FS_IPU_SSR_UTIL); +F2FS_IPU_POLICY(F2FS_IPU_FSYNC); +F2FS_IPU_POLICY(F2FS_IPU_ASYNC); +F2FS_IPU_POLICY(F2FS_IPU_NOCACHE); +F2FS_IPU_POLICY(F2FS_IPU_HONOR_OPU_WRITE); + +static inline unsigned int curseg_segno(struct f2fs_sb_info *sbi, + int type) +{ + struct curseg_info *curseg = CURSEG_I(sbi, type); + return curseg->segno; +} + +static inline unsigned char curseg_alloc_type(struct f2fs_sb_info *sbi, + int type) +{ + struct curseg_info *curseg = CURSEG_I(sbi, type); + return curseg->alloc_type; +} + +static inline bool valid_main_segno(struct f2fs_sb_info *sbi, + unsigned int segno) +{ + return segno <= (MAIN_SEGS(sbi) - 1); +} + +static inline void verify_fio_blkaddr(struct f2fs_io_info *fio) +{ + struct f2fs_sb_info *sbi = fio->sbi; + + if (__is_valid_data_blkaddr(fio->old_blkaddr)) + verify_blkaddr(sbi, fio->old_blkaddr, __is_meta_io(fio) ? + META_GENERIC : DATA_GENERIC); + verify_blkaddr(sbi, fio->new_blkaddr, __is_meta_io(fio) ? + META_GENERIC : DATA_GENERIC_ENHANCE); +} + +/* + * Summary block is always treated as an invalid block + */ +static inline int check_block_count(struct f2fs_sb_info *sbi, + int segno, struct f2fs_sit_entry *raw_sit) +{ + bool is_valid = test_bit_le(0, raw_sit->valid_map) ? true : false; + int valid_blocks = 0; + int cur_pos = 0, next_pos; + unsigned int usable_blks_per_seg = f2fs_usable_blks_in_seg(sbi, segno); + + /* check bitmap with valid block count */ + do { + if (is_valid) { + next_pos = find_next_zero_bit_le(&raw_sit->valid_map, + usable_blks_per_seg, + cur_pos); + valid_blocks += next_pos - cur_pos; + } else + next_pos = find_next_bit_le(&raw_sit->valid_map, + usable_blks_per_seg, + cur_pos); + cur_pos = next_pos; + is_valid = !is_valid; + } while (cur_pos < usable_blks_per_seg); + + if (unlikely(GET_SIT_VBLOCKS(raw_sit) != valid_blocks)) { + f2fs_err(sbi, "Mismatch valid blocks %d vs. %d", + GET_SIT_VBLOCKS(raw_sit), valid_blocks); + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_handle_error(sbi, ERROR_INCONSISTENT_SIT); + return -EFSCORRUPTED; + } + + if (usable_blks_per_seg < BLKS_PER_SEG(sbi)) + f2fs_bug_on(sbi, find_next_bit_le(&raw_sit->valid_map, + BLKS_PER_SEG(sbi), + usable_blks_per_seg) != BLKS_PER_SEG(sbi)); + + /* check segment usage, and check boundary of a given segment number */ + if (unlikely(GET_SIT_VBLOCKS(raw_sit) > usable_blks_per_seg + || !valid_main_segno(sbi, segno))) { + f2fs_err(sbi, "Wrong valid blocks %d or segno %u", + GET_SIT_VBLOCKS(raw_sit), segno); + set_sbi_flag(sbi, SBI_NEED_FSCK); + f2fs_handle_error(sbi, ERROR_INCONSISTENT_SIT); + return -EFSCORRUPTED; + } + return 0; +} + +static inline pgoff_t current_sit_addr(struct f2fs_sb_info *sbi, + unsigned int start) +{ + struct sit_info *sit_i = SIT_I(sbi); + unsigned int offset = SIT_BLOCK_OFFSET(start); + block_t blk_addr = sit_i->sit_base_addr + offset; + + f2fs_bug_on(sbi, !valid_main_segno(sbi, start)); + + /* calculate sit block address */ + if (f2fs_test_bit(offset, sit_i->sit_bitmap)) + blk_addr += sit_i->sit_blocks; + + return blk_addr; +} + +static inline pgoff_t next_sit_addr(struct f2fs_sb_info *sbi, + pgoff_t block_addr) +{ + struct sit_info *sit_i = SIT_I(sbi); + block_addr -= sit_i->sit_base_addr; + if (block_addr < sit_i->sit_blocks) + block_addr += sit_i->sit_blocks; + else + block_addr -= sit_i->sit_blocks; + + return block_addr + sit_i->sit_base_addr; +} + +static inline void set_to_next_sit(struct sit_info *sit_i, unsigned int start) +{ + unsigned int block_off = SIT_BLOCK_OFFSET(start); + + f2fs_change_bit(block_off, sit_i->sit_bitmap); +} + +static inline unsigned long long get_mtime(struct f2fs_sb_info *sbi, + bool base_time) +{ + struct sit_info *sit_i = SIT_I(sbi); + time64_t diff, now = ktime_get_boottime_seconds(); + + if (now >= sit_i->mounted_time) + return sit_i->elapsed_time + now - sit_i->mounted_time; + + /* system time is set to the past */ + if (!base_time) { + diff = sit_i->mounted_time - now; + if (sit_i->elapsed_time >= diff) + return sit_i->elapsed_time - diff; + return 0; + } + return sit_i->elapsed_time; +} + +static inline void set_summary(struct f2fs_summary *sum, nid_t nid, + unsigned int ofs_in_node, unsigned char version) +{ + sum->nid = cpu_to_le32(nid); + sum->ofs_in_node = cpu_to_le16(ofs_in_node); + sum->version = version; +} + +static inline block_t start_sum_block(struct f2fs_sb_info *sbi) +{ + return __start_cp_addr(sbi) + + le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_start_sum); +} + +static inline block_t sum_blk_addr(struct f2fs_sb_info *sbi, int base, int type) +{ + return __start_cp_addr(sbi) + + le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_total_block_count) + - (base + 1) + type; +} + +static inline bool sec_usage_check(struct f2fs_sb_info *sbi, unsigned int secno) +{ + if (is_cursec(sbi, secno) || (sbi->cur_victim_sec == secno)) + return true; + return false; +} + +/* + * It is very important to gather dirty pages and write at once, so that we can + * submit a big bio without interfering other data writes. + * By default, 512 pages for directory data, + * 512 pages (2MB) * 8 for nodes, and + * 256 pages * 8 for meta are set. + */ +static inline int nr_pages_to_skip(struct f2fs_sb_info *sbi, int type) +{ + if (bdi_wb_dirty_exceeded(sbi->sb->s_bdi)) + return 0; + + if (type == DATA) + return BLKS_PER_SEG(sbi); + else if (type == NODE) + return SEGS_TO_BLKS(sbi, 8); + else if (type == META) + return 8 * BIO_MAX_VECS; + else + return 0; +} + +/* + * When writing pages, it'd better align nr_to_write for segment size. + */ +static inline long nr_pages_to_write(struct f2fs_sb_info *sbi, int type, + struct writeback_control *wbc) +{ + long nr_to_write, desired; + + if (wbc->sync_mode != WB_SYNC_NONE) + return 0; + + nr_to_write = wbc->nr_to_write; + desired = BIO_MAX_VECS; + if (type == NODE) + desired <<= 1; + + wbc->nr_to_write = desired; + return desired - nr_to_write; +} + +static inline void wake_up_discard_thread(struct f2fs_sb_info *sbi, bool force) +{ + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + bool wakeup = false; + int i; + + if (force) + goto wake_up; + + mutex_lock(&dcc->cmd_lock); + for (i = MAX_PLIST_NUM - 1; i >= 0; i--) { + if (i + 1 < dcc->discard_granularity) + break; + if (!list_empty(&dcc->pend_list[i])) { + wakeup = true; + break; + } + } + mutex_unlock(&dcc->cmd_lock); + if (!wakeup || !is_idle(sbi, DISCARD_TIME)) + return; +wake_up: + dcc->discard_wake = true; + wake_up_interruptible_all(&dcc->discard_wait_queue); +} diff --git a/fs/f2fs/shrinker.c b/fs/f2fs/shrinker.c new file mode 100644 index 000000000..b88babcf6 --- /dev/null +++ b/fs/f2fs/shrinker.c @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * f2fs shrinker support + * the basic infra was copied from fs/ubifs/shrinker.c + * + * Copyright (c) 2015 Motorola Mobility + * Copyright (c) 2015 Jaegeuk Kim + */ +#include +#include + +#include "f2fs.h" +#include "node.h" + +static LIST_HEAD(f2fs_list); +static DEFINE_SPINLOCK(f2fs_list_lock); +static unsigned int shrinker_run_no; + +static unsigned long __count_nat_entries(struct f2fs_sb_info *sbi) +{ + return NM_I(sbi)->nat_cnt[RECLAIMABLE_NAT]; +} + +static unsigned long __count_free_nids(struct f2fs_sb_info *sbi) +{ + long count = NM_I(sbi)->nid_cnt[FREE_NID] - MAX_FREE_NIDS; + + return count > 0 ? count : 0; +} + +static unsigned long __count_extent_cache(struct f2fs_sb_info *sbi, + enum extent_type type) +{ + struct extent_tree_info *eti = &sbi->extent_tree[type]; + + return atomic_read(&eti->total_zombie_tree) + + atomic_read(&eti->total_ext_node); +} + +unsigned long f2fs_shrink_count(struct shrinker *shrink, + struct shrink_control *sc) +{ + struct f2fs_sb_info *sbi; + struct list_head *p; + unsigned long count = 0; + + spin_lock(&f2fs_list_lock); + p = f2fs_list.next; + while (p != &f2fs_list) { + sbi = list_entry(p, struct f2fs_sb_info, s_list); + + /* stop f2fs_put_super */ + if (!mutex_trylock(&sbi->umount_mutex)) { + p = p->next; + continue; + } + spin_unlock(&f2fs_list_lock); + + /* count read extent cache entries */ + count += __count_extent_cache(sbi, EX_READ); + + /* count block age extent cache entries */ + count += __count_extent_cache(sbi, EX_BLOCK_AGE); + + /* count clean nat cache entries */ + count += __count_nat_entries(sbi); + + /* count free nids cache entries */ + count += __count_free_nids(sbi); + + spin_lock(&f2fs_list_lock); + p = p->next; + mutex_unlock(&sbi->umount_mutex); + } + spin_unlock(&f2fs_list_lock); + return count ?: SHRINK_EMPTY; +} + +unsigned long f2fs_shrink_scan(struct shrinker *shrink, + struct shrink_control *sc) +{ + unsigned long nr = sc->nr_to_scan; + struct f2fs_sb_info *sbi; + struct list_head *p; + unsigned int run_no; + unsigned long freed = 0; + + spin_lock(&f2fs_list_lock); + do { + run_no = ++shrinker_run_no; + } while (run_no == 0); + p = f2fs_list.next; + while (p != &f2fs_list) { + sbi = list_entry(p, struct f2fs_sb_info, s_list); + + if (sbi->shrinker_run_no == run_no) + break; + + /* stop f2fs_put_super */ + if (!mutex_trylock(&sbi->umount_mutex)) { + p = p->next; + continue; + } + spin_unlock(&f2fs_list_lock); + + sbi->shrinker_run_no = run_no; + + /* shrink extent cache entries */ + freed += f2fs_shrink_age_extent_tree(sbi, nr >> 2); + + /* shrink read extent cache entries */ + freed += f2fs_shrink_read_extent_tree(sbi, nr >> 2); + + /* shrink clean nat cache entries */ + if (freed < nr) + freed += f2fs_try_to_free_nats(sbi, nr - freed); + + /* shrink free nids cache entries */ + if (freed < nr) + freed += f2fs_try_to_free_nids(sbi, nr - freed); + + spin_lock(&f2fs_list_lock); + p = p->next; + list_move_tail(&sbi->s_list, &f2fs_list); + mutex_unlock(&sbi->umount_mutex); + if (freed >= nr) + break; + } + spin_unlock(&f2fs_list_lock); + return freed; +} + +unsigned int f2fs_donate_files(void) +{ + struct f2fs_sb_info *sbi; + struct list_head *p; + unsigned int donate_files = 0; + + spin_lock(&f2fs_list_lock); + p = f2fs_list.next; + while (p != &f2fs_list) { + sbi = list_entry(p, struct f2fs_sb_info, s_list); + + /* stop f2fs_put_super */ + if (!mutex_trylock(&sbi->umount_mutex)) { + p = p->next; + continue; + } + spin_unlock(&f2fs_list_lock); + + donate_files += sbi->donate_files; + + spin_lock(&f2fs_list_lock); + p = p->next; + mutex_unlock(&sbi->umount_mutex); + } + spin_unlock(&f2fs_list_lock); + + return donate_files; +} + +static unsigned int do_reclaim_caches(struct f2fs_sb_info *sbi, + unsigned int reclaim_caches_kb) +{ + struct inode *inode; + struct f2fs_inode_info *fi; + unsigned int nfiles = sbi->donate_files; + pgoff_t npages = reclaim_caches_kb >> (PAGE_SHIFT - 10); + + while (npages && nfiles--) { + pgoff_t len; + + spin_lock(&sbi->inode_lock[DONATE_INODE]); + if (list_empty(&sbi->inode_list[DONATE_INODE])) { + spin_unlock(&sbi->inode_lock[DONATE_INODE]); + break; + } + fi = list_first_entry(&sbi->inode_list[DONATE_INODE], + struct f2fs_inode_info, gdonate_list); + list_move_tail(&fi->gdonate_list, &sbi->inode_list[DONATE_INODE]); + inode = igrab(&fi->vfs_inode); + spin_unlock(&sbi->inode_lock[DONATE_INODE]); + + if (!inode) + continue; + + inode_lock(inode); + if (!is_inode_flag_set(inode, FI_DONATE_FINISHED)) { + len = fi->donate_end - fi->donate_start + 1; + npages = npages < len ? 0 : npages - len; + + invalidate_inode_pages2_range(inode->i_mapping, + fi->donate_start, fi->donate_end); + set_inode_flag(inode, FI_DONATE_FINISHED); + } + inode_unlock(inode); + + iput(inode); + cond_resched(); + } + return npages << (PAGE_SHIFT - 10); +} + +void f2fs_reclaim_caches(unsigned int reclaim_caches_kb) +{ + struct f2fs_sb_info *sbi; + struct list_head *p; + + spin_lock(&f2fs_list_lock); + p = f2fs_list.next; + while (p != &f2fs_list && reclaim_caches_kb) { + sbi = list_entry(p, struct f2fs_sb_info, s_list); + + /* stop f2fs_put_super */ + if (!mutex_trylock(&sbi->umount_mutex)) { + p = p->next; + continue; + } + spin_unlock(&f2fs_list_lock); + + reclaim_caches_kb = do_reclaim_caches(sbi, reclaim_caches_kb); + + spin_lock(&f2fs_list_lock); + p = p->next; + mutex_unlock(&sbi->umount_mutex); + } + spin_unlock(&f2fs_list_lock); +} + +void f2fs_join_shrinker(struct f2fs_sb_info *sbi) +{ + spin_lock(&f2fs_list_lock); + list_add_tail(&sbi->s_list, &f2fs_list); + spin_unlock(&f2fs_list_lock); +} + +void f2fs_leave_shrinker(struct f2fs_sb_info *sbi) +{ + f2fs_shrink_read_extent_tree(sbi, __count_extent_cache(sbi, EX_READ)); + f2fs_shrink_age_extent_tree(sbi, + __count_extent_cache(sbi, EX_BLOCK_AGE)); + + spin_lock(&f2fs_list_lock); + list_del_init(&sbi->s_list); + spin_unlock(&f2fs_list_lock); +} diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c new file mode 100644 index 000000000..be22c3101 --- /dev/null +++ b/fs/f2fs/super.c @@ -0,0 +1,5775 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/f2fs/super.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "f2fs.h" +#include "node.h" +#include "segment.h" +#include "xattr.h" +#include "gc.h" +#include "iostat.h" + +#define CREATE_TRACE_POINTS +#include + +static struct kmem_cache *f2fs_inode_cachep; + +#ifdef CONFIG_F2FS_FAULT_INJECTION + +const char *f2fs_fault_name[FAULT_MAX] = { + [FAULT_KMALLOC] = "kmalloc", + [FAULT_KVMALLOC] = "kvmalloc", + [FAULT_PAGE_ALLOC] = "page alloc", + [FAULT_PAGE_GET] = "page get", + [FAULT_ALLOC_BIO] = "alloc bio(obsolete)", + [FAULT_ALLOC_NID] = "alloc nid", + [FAULT_ORPHAN] = "orphan", + [FAULT_BLOCK] = "no more block", + [FAULT_DIR_DEPTH] = "too big dir depth", + [FAULT_EVICT_INODE] = "evict_inode fail", + [FAULT_TRUNCATE] = "truncate fail", + [FAULT_READ_IO] = "read IO error", + [FAULT_CHECKPOINT] = "checkpoint error", + [FAULT_DISCARD] = "discard error", + [FAULT_WRITE_IO] = "write IO error", + [FAULT_SLAB_ALLOC] = "slab alloc", + [FAULT_DQUOT_INIT] = "dquot initialize", + [FAULT_LOCK_OP] = "lock_op", + [FAULT_BLKADDR_VALIDITY] = "invalid blkaddr", + [FAULT_BLKADDR_CONSISTENCE] = "inconsistent blkaddr", + [FAULT_NO_SEGMENT] = "no free segment", + [FAULT_INCONSISTENT_FOOTER] = "inconsistent footer", + [FAULT_ATOMIC_TIMEOUT] = "atomic timeout", + [FAULT_VMALLOC] = "vmalloc", + [FAULT_LOCK_TIMEOUT] = "lock timeout", + [FAULT_SKIP_WRITE] = "skip write", +}; + +int f2fs_build_fault_attr(struct f2fs_sb_info *sbi, unsigned long rate, + unsigned long type, enum fault_option fo) +{ + struct f2fs_fault_info *ffi = &F2FS_OPTION(sbi).fault_info; + + if (fo & FAULT_ALL) { + memset(ffi, 0, sizeof(struct f2fs_fault_info)); + return 0; + } + + if (fo & FAULT_RATE) { + if (rate > INT_MAX) + return -EINVAL; + atomic_set(&ffi->inject_ops, 0); + ffi->inject_rate = (int)rate; + f2fs_info(sbi, "build fault injection rate: %lu", rate); + } + + if (fo & FAULT_TYPE) { + if (type >= BIT(FAULT_MAX)) + return -EINVAL; + ffi->inject_type = (unsigned int)type; + f2fs_info(sbi, "build fault injection type: 0x%lx", type); + } + + if (fo & FAULT_TIMEOUT) { + if (type >= TIMEOUT_TYPE_MAX) + return -EINVAL; + ffi->inject_lock_timeout = (unsigned int)type; + f2fs_info(sbi, "build fault timeout injection type: 0x%lx", type); + } + + return 0; +} + +static void inject_timeout(struct f2fs_sb_info *sbi) +{ + struct f2fs_fault_info *ffi = &F2FS_OPTION(sbi).fault_info; + enum f2fs_timeout_type type = ffi->inject_lock_timeout; + unsigned long start_time = jiffies; + unsigned long timeout = HZ; + + switch (type) { + case TIMEOUT_TYPE_RUNNING: + while (!time_after(jiffies, start_time + timeout)) { + if (fatal_signal_pending(current)) + return; + ; + } + break; + case TIMEOUT_TYPE_IO_SLEEP: + f2fs_schedule_timeout_killable(timeout, true); + break; + case TIMEOUT_TYPE_NONIO_SLEEP: + f2fs_schedule_timeout_killable(timeout, false); + break; + case TIMEOUT_TYPE_RUNNABLE: + while (!time_after(jiffies, start_time + timeout)) { + if (fatal_signal_pending(current)) + return; + schedule(); + } + break; + default: + return; + } +} + +void f2fs_simulate_lock_timeout(struct f2fs_sb_info *sbi) +{ + struct f2fs_lock_context lc; + + f2fs_lock_op(sbi, &lc); + inject_timeout(sbi); + f2fs_unlock_op(sbi, &lc); +} +#endif + +/* f2fs-wide shrinker description */ +static struct shrinker *f2fs_shrinker_info; + +static int __init f2fs_init_shrinker(void) +{ + f2fs_shrinker_info = shrinker_alloc(0, "f2fs-shrinker"); + if (!f2fs_shrinker_info) + return -ENOMEM; + + f2fs_shrinker_info->count_objects = f2fs_shrink_count; + f2fs_shrinker_info->scan_objects = f2fs_shrink_scan; + + shrinker_register(f2fs_shrinker_info); + + return 0; +} + +static void f2fs_exit_shrinker(void) +{ + shrinker_free(f2fs_shrinker_info); +} + +enum { + Opt_gc_background, + Opt_disable_roll_forward, + Opt_norecovery, + Opt_discard, + Opt_noheap, + Opt_heap, + Opt_user_xattr, + Opt_acl, + Opt_active_logs, + Opt_disable_ext_identify, + Opt_inline_xattr, + Opt_inline_xattr_size, + Opt_inline_data, + Opt_inline_dentry, + Opt_flush_merge, + Opt_barrier, + Opt_fastboot, + Opt_extent_cache, + Opt_data_flush, + Opt_reserve_root, + Opt_reserve_node, + Opt_resgid, + Opt_resuid, + Opt_mode, + Opt_fault_injection, + Opt_fault_type, + Opt_lazytime, + Opt_quota, + Opt_usrquota, + Opt_grpquota, + Opt_prjquota, + Opt_usrjquota, + Opt_grpjquota, + Opt_prjjquota, + Opt_alloc, + Opt_fsync, + Opt_test_dummy_encryption, + Opt_inlinecrypt, + Opt_checkpoint_disable, + Opt_checkpoint_disable_cap, + Opt_checkpoint_disable_cap_perc, + Opt_checkpoint_enable, + Opt_checkpoint_merge, + Opt_compress_algorithm, + Opt_compress_log_size, + Opt_nocompress_extension, + Opt_compress_extension, + Opt_compress_chksum, + Opt_compress_mode, + Opt_compress_cache, + Opt_atgc, + Opt_gc_merge, + Opt_discard_unit, + Opt_memory_mode, + Opt_age_extent_cache, + Opt_errors, + Opt_nat_bits, + Opt_jqfmt, + Opt_checkpoint, + Opt_lookup_mode, + Opt_err, +}; + +static const struct constant_table f2fs_param_background_gc[] = { + {"on", BGGC_MODE_ON}, + {"off", BGGC_MODE_OFF}, + {"sync", BGGC_MODE_SYNC}, + {} +}; + +static const struct constant_table f2fs_param_mode[] = { + {"adaptive", FS_MODE_ADAPTIVE}, + {"lfs", FS_MODE_LFS}, + {"fragment:segment", FS_MODE_FRAGMENT_SEG}, + {"fragment:block", FS_MODE_FRAGMENT_BLK}, + {} +}; + +static const struct constant_table f2fs_param_jqfmt[] = { + {"vfsold", QFMT_VFS_OLD}, + {"vfsv0", QFMT_VFS_V0}, + {"vfsv1", QFMT_VFS_V1}, + {} +}; + +static const struct constant_table f2fs_param_alloc_mode[] = { + {"default", ALLOC_MODE_DEFAULT}, + {"reuse", ALLOC_MODE_REUSE}, + {} +}; +static const struct constant_table f2fs_param_fsync_mode[] = { + {"posix", FSYNC_MODE_POSIX}, + {"strict", FSYNC_MODE_STRICT}, + {"nobarrier", FSYNC_MODE_NOBARRIER}, + {} +}; + +static const struct constant_table f2fs_param_compress_mode[] = { + {"fs", COMPR_MODE_FS}, + {"user", COMPR_MODE_USER}, + {} +}; + +static const struct constant_table f2fs_param_discard_unit[] = { + {"block", DISCARD_UNIT_BLOCK}, + {"segment", DISCARD_UNIT_SEGMENT}, + {"section", DISCARD_UNIT_SECTION}, + {} +}; + +static const struct constant_table f2fs_param_memory_mode[] = { + {"normal", MEMORY_MODE_NORMAL}, + {"low", MEMORY_MODE_LOW}, + {} +}; + +static const struct constant_table f2fs_param_errors[] = { + {"remount-ro", MOUNT_ERRORS_READONLY}, + {"continue", MOUNT_ERRORS_CONTINUE}, + {"panic", MOUNT_ERRORS_PANIC}, + {} +}; + +static const struct constant_table f2fs_param_lookup_mode[] = { + {"perf", LOOKUP_PERF}, + {"compat", LOOKUP_COMPAT}, + {"auto", LOOKUP_AUTO}, + {} +}; + +static const struct fs_parameter_spec f2fs_param_specs[] = { + fsparam_enum("background_gc", Opt_gc_background, f2fs_param_background_gc), + fsparam_flag("disable_roll_forward", Opt_disable_roll_forward), + fsparam_flag("norecovery", Opt_norecovery), + fsparam_flag_no("discard", Opt_discard), + fsparam_flag("no_heap", Opt_noheap), + fsparam_flag("heap", Opt_heap), + fsparam_flag_no("user_xattr", Opt_user_xattr), + fsparam_flag_no("acl", Opt_acl), + fsparam_s32("active_logs", Opt_active_logs), + fsparam_flag("disable_ext_identify", Opt_disable_ext_identify), + fsparam_flag_no("inline_xattr", Opt_inline_xattr), + fsparam_s32("inline_xattr_size", Opt_inline_xattr_size), + fsparam_flag_no("inline_data", Opt_inline_data), + fsparam_flag_no("inline_dentry", Opt_inline_dentry), + fsparam_flag_no("flush_merge", Opt_flush_merge), + fsparam_flag_no("barrier", Opt_barrier), + fsparam_flag("fastboot", Opt_fastboot), + fsparam_flag_no("extent_cache", Opt_extent_cache), + fsparam_flag("data_flush", Opt_data_flush), + fsparam_u32("reserve_root", Opt_reserve_root), + fsparam_u32("reserve_node", Opt_reserve_node), + fsparam_gid("resgid", Opt_resgid), + fsparam_uid("resuid", Opt_resuid), + fsparam_enum("mode", Opt_mode, f2fs_param_mode), + fsparam_s32("fault_injection", Opt_fault_injection), + fsparam_u32("fault_type", Opt_fault_type), + fsparam_flag_no("lazytime", Opt_lazytime), + fsparam_flag_no("quota", Opt_quota), + fsparam_flag("usrquota", Opt_usrquota), + fsparam_flag("grpquota", Opt_grpquota), + fsparam_flag("prjquota", Opt_prjquota), + fsparam_string("usrjquota", Opt_usrjquota), + fsparam_flag("usrjquota", Opt_usrjquota), + fsparam_string("grpjquota", Opt_grpjquota), + fsparam_flag("grpjquota", Opt_grpjquota), + fsparam_string("prjjquota", Opt_prjjquota), + fsparam_flag("prjjquota", Opt_prjjquota), + fsparam_flag("nat_bits", Opt_nat_bits), + fsparam_enum("jqfmt", Opt_jqfmt, f2fs_param_jqfmt), + fsparam_enum("alloc_mode", Opt_alloc, f2fs_param_alloc_mode), + fsparam_enum("fsync_mode", Opt_fsync, f2fs_param_fsync_mode), + fsparam_string("test_dummy_encryption", Opt_test_dummy_encryption), + fsparam_flag("test_dummy_encryption", Opt_test_dummy_encryption), + fsparam_flag("inlinecrypt", Opt_inlinecrypt), + fsparam_string("checkpoint", Opt_checkpoint), + fsparam_flag_no("checkpoint_merge", Opt_checkpoint_merge), + fsparam_string("compress_algorithm", Opt_compress_algorithm), + fsparam_u32("compress_log_size", Opt_compress_log_size), + fsparam_string("compress_extension", Opt_compress_extension), + fsparam_string("nocompress_extension", Opt_nocompress_extension), + fsparam_flag("compress_chksum", Opt_compress_chksum), + fsparam_enum("compress_mode", Opt_compress_mode, f2fs_param_compress_mode), + fsparam_flag("compress_cache", Opt_compress_cache), + fsparam_flag("atgc", Opt_atgc), + fsparam_flag_no("gc_merge", Opt_gc_merge), + fsparam_enum("discard_unit", Opt_discard_unit, f2fs_param_discard_unit), + fsparam_enum("memory", Opt_memory_mode, f2fs_param_memory_mode), + fsparam_flag("age_extent_cache", Opt_age_extent_cache), + fsparam_enum("errors", Opt_errors, f2fs_param_errors), + fsparam_enum("lookup_mode", Opt_lookup_mode, f2fs_param_lookup_mode), + {} +}; + +/* Resort to a match_table for this interestingly formatted option */ +static match_table_t f2fs_checkpoint_tokens = { + {Opt_checkpoint_disable, "disable"}, + {Opt_checkpoint_disable_cap, "disable:%u"}, + {Opt_checkpoint_disable_cap_perc, "disable:%u%%"}, + {Opt_checkpoint_enable, "enable"}, + {Opt_err, NULL}, +}; + +#define F2FS_SPEC_background_gc (1 << 0) +#define F2FS_SPEC_inline_xattr_size (1 << 1) +#define F2FS_SPEC_active_logs (1 << 2) +#define F2FS_SPEC_reserve_root (1 << 3) +#define F2FS_SPEC_resgid (1 << 4) +#define F2FS_SPEC_resuid (1 << 5) +#define F2FS_SPEC_mode (1 << 6) +#define F2FS_SPEC_fault_injection (1 << 7) +#define F2FS_SPEC_fault_type (1 << 8) +#define F2FS_SPEC_jqfmt (1 << 9) +#define F2FS_SPEC_alloc_mode (1 << 10) +#define F2FS_SPEC_fsync_mode (1 << 11) +#define F2FS_SPEC_checkpoint_disable_cap (1 << 12) +#define F2FS_SPEC_checkpoint_disable_cap_perc (1 << 13) +#define F2FS_SPEC_compress_level (1 << 14) +#define F2FS_SPEC_compress_algorithm (1 << 15) +#define F2FS_SPEC_compress_log_size (1 << 16) +#define F2FS_SPEC_compress_extension (1 << 17) +#define F2FS_SPEC_nocompress_extension (1 << 18) +#define F2FS_SPEC_compress_chksum (1 << 19) +#define F2FS_SPEC_compress_mode (1 << 20) +#define F2FS_SPEC_discard_unit (1 << 21) +#define F2FS_SPEC_memory_mode (1 << 22) +#define F2FS_SPEC_errors (1 << 23) +#define F2FS_SPEC_lookup_mode (1 << 24) +#define F2FS_SPEC_reserve_node (1 << 25) + +struct f2fs_fs_context { + struct f2fs_mount_info info; + unsigned long long opt_mask; /* Bits changed */ + unsigned int spec_mask; + unsigned short qname_mask; +}; + +#define F2FS_CTX_INFO(ctx) ((ctx)->info) + +static inline void ctx_set_opt(struct f2fs_fs_context *ctx, + enum f2fs_mount_opt flag) +{ + ctx->info.opt |= BIT(flag); + ctx->opt_mask |= BIT(flag); +} + +static inline void ctx_clear_opt(struct f2fs_fs_context *ctx, + enum f2fs_mount_opt flag) +{ + ctx->info.opt &= ~BIT(flag); + ctx->opt_mask |= BIT(flag); +} + +static inline bool ctx_test_opt(struct f2fs_fs_context *ctx, + enum f2fs_mount_opt flag) +{ + return ctx->info.opt & BIT(flag); +} + +void f2fs_printk(struct f2fs_sb_info *sbi, bool limit_rate, + const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + int level; + + va_start(args, fmt); + + level = printk_get_level(fmt); + vaf.fmt = printk_skip_level(fmt); + vaf.va = &args; + if (limit_rate) + if (sbi) + printk_ratelimited("%c%cF2FS-fs (%s): %pV\n", + KERN_SOH_ASCII, level, sbi->sb->s_id, &vaf); + else + printk_ratelimited("%c%cF2FS-fs: %pV\n", + KERN_SOH_ASCII, level, &vaf); + else + if (sbi) + printk("%c%cF2FS-fs (%s): %pV\n", + KERN_SOH_ASCII, level, sbi->sb->s_id, &vaf); + else + printk("%c%cF2FS-fs: %pV\n", + KERN_SOH_ASCII, level, &vaf); + + va_end(args); +} + +#if IS_ENABLED(CONFIG_UNICODE) +static const struct f2fs_sb_encodings { + __u16 magic; + char *name; + unsigned int version; +} f2fs_sb_encoding_map[] = { + {F2FS_ENC_UTF8_12_1, "utf8", UNICODE_AGE(12, 1, 0)}, +}; + +static const struct f2fs_sb_encodings * +f2fs_sb_read_encoding(const struct f2fs_super_block *sb) +{ + __u16 magic = le16_to_cpu(sb->s_encoding); + int i; + + for (i = 0; i < ARRAY_SIZE(f2fs_sb_encoding_map); i++) + if (magic == f2fs_sb_encoding_map[i].magic) + return &f2fs_sb_encoding_map[i]; + + return NULL; +} + +struct kmem_cache *f2fs_cf_name_slab; +static int __init f2fs_create_casefold_cache(void) +{ + f2fs_cf_name_slab = f2fs_kmem_cache_create("f2fs_casefolded_name", + F2FS_NAME_LEN); + return f2fs_cf_name_slab ? 0 : -ENOMEM; +} + +static void f2fs_destroy_casefold_cache(void) +{ + kmem_cache_destroy(f2fs_cf_name_slab); +} +#else +static int __init f2fs_create_casefold_cache(void) { return 0; } +static void f2fs_destroy_casefold_cache(void) { } +#endif + +static inline void limit_reserve_root(struct f2fs_sb_info *sbi) +{ + block_t block_limit = min((sbi->user_block_count >> 3), + sbi->user_block_count - sbi->reserved_blocks); + block_t node_limit = sbi->total_node_count >> 3; + + /* limit is 12.5% */ + if (test_opt(sbi, RESERVE_ROOT) && + F2FS_OPTION(sbi).root_reserved_blocks > block_limit) { + F2FS_OPTION(sbi).root_reserved_blocks = block_limit; + f2fs_info(sbi, "Reduce reserved blocks for root = %u", + F2FS_OPTION(sbi).root_reserved_blocks); + } + if (test_opt(sbi, RESERVE_NODE) && + F2FS_OPTION(sbi).root_reserved_nodes > node_limit) { + F2FS_OPTION(sbi).root_reserved_nodes = node_limit; + f2fs_info(sbi, "Reduce reserved nodes for root = %u", + F2FS_OPTION(sbi).root_reserved_nodes); + } + if (!test_opt(sbi, RESERVE_ROOT) && !test_opt(sbi, RESERVE_NODE) && + (!uid_eq(F2FS_OPTION(sbi).s_resuid, + make_kuid(&init_user_ns, F2FS_DEF_RESUID)) || + !gid_eq(F2FS_OPTION(sbi).s_resgid, + make_kgid(&init_user_ns, F2FS_DEF_RESGID)))) + f2fs_info(sbi, "Ignore s_resuid=%u, s_resgid=%u w/o reserve_root" + " and reserve_node", + from_kuid_munged(&init_user_ns, + F2FS_OPTION(sbi).s_resuid), + from_kgid_munged(&init_user_ns, + F2FS_OPTION(sbi).s_resgid)); +} + +static inline void adjust_unusable_cap_perc(struct f2fs_sb_info *sbi) +{ + if (!F2FS_OPTION(sbi).unusable_cap_perc) + return; + + if (F2FS_OPTION(sbi).unusable_cap_perc == 100) + F2FS_OPTION(sbi).unusable_cap = sbi->user_block_count; + else + F2FS_OPTION(sbi).unusable_cap = (sbi->user_block_count / 100) * + F2FS_OPTION(sbi).unusable_cap_perc; + + f2fs_info(sbi, "Adjust unusable cap for checkpoint=disable = %u / %u%%", + F2FS_OPTION(sbi).unusable_cap, + F2FS_OPTION(sbi).unusable_cap_perc); +} + +static void init_once(void *foo) +{ + struct f2fs_inode_info *fi = (struct f2fs_inode_info *) foo; + + inode_init_once(&fi->vfs_inode); +#ifdef CONFIG_FS_ENCRYPTION + fi->i_crypt_info = NULL; +#endif +} + +#ifdef CONFIG_QUOTA +static const char * const quotatypes[] = INITQFNAMES; +#define QTYPE2NAME(t) (quotatypes[t]) +/* + * Note the name of the specified quota file. + */ +static int f2fs_note_qf_name(struct fs_context *fc, int qtype, + struct fs_parameter *param) +{ + struct f2fs_fs_context *ctx = fc->fs_private; + char *qname; + + if (param->size < 1) { + f2fs_err(NULL, "Missing quota name"); + return -EINVAL; + } + if (strchr(param->string, '/')) { + f2fs_err(NULL, "quotafile must be on filesystem root"); + return -EINVAL; + } + if (ctx->info.s_qf_names[qtype]) { + if (strcmp(ctx->info.s_qf_names[qtype], param->string) != 0) { + f2fs_err(NULL, "Quota file already specified"); + return -EINVAL; + } + return 0; + } + + qname = kmemdup_nul(param->string, param->size, GFP_KERNEL); + if (!qname) { + f2fs_err(NULL, "Not enough memory for storing quotafile name"); + return -ENOMEM; + } + F2FS_CTX_INFO(ctx).s_qf_names[qtype] = qname; + ctx->qname_mask |= 1 << qtype; + return 0; +} + +/* + * Clear the name of the specified quota file. + */ +static int f2fs_unnote_qf_name(struct fs_context *fc, int qtype) +{ + struct f2fs_fs_context *ctx = fc->fs_private; + + kfree(ctx->info.s_qf_names[qtype]); + ctx->info.s_qf_names[qtype] = NULL; + ctx->qname_mask |= 1 << qtype; + return 0; +} + +static void f2fs_unnote_qf_name_all(struct fs_context *fc) +{ + int i; + + for (i = 0; i < MAXQUOTAS; i++) + f2fs_unnote_qf_name(fc, i); +} +#endif + +static int f2fs_parse_test_dummy_encryption(const struct fs_parameter *param, + struct f2fs_fs_context *ctx) +{ + int err; + + if (!IS_ENABLED(CONFIG_FS_ENCRYPTION)) { + f2fs_warn(NULL, "test_dummy_encryption option not supported"); + return -EINVAL; + } + err = fscrypt_parse_test_dummy_encryption(param, + &ctx->info.dummy_enc_policy); + if (err) { + if (err == -EINVAL) + f2fs_warn(NULL, "Value of option \"%s\" is unrecognized", + param->key); + else if (err == -EEXIST) + f2fs_warn(NULL, "Conflicting test_dummy_encryption options"); + else + f2fs_warn(NULL, "Error processing option \"%s\" [%d]", + param->key, err); + return -EINVAL; + } + return 0; +} + +#ifdef CONFIG_F2FS_FS_COMPRESSION +static bool is_compress_extension_exist(struct f2fs_mount_info *info, + const char *new_ext, bool is_ext) +{ + unsigned char (*ext)[F2FS_EXTENSION_LEN]; + int ext_cnt; + int i; + + if (is_ext) { + ext = info->extensions; + ext_cnt = info->compress_ext_cnt; + } else { + ext = info->noextensions; + ext_cnt = info->nocompress_ext_cnt; + } + + for (i = 0; i < ext_cnt; i++) { + if (!strcasecmp(new_ext, ext[i])) + return true; + } + + return false; +} + +/* + * 1. The same extension name cannot not appear in both compress and non-compress extension + * at the same time. + * 2. If the compress extension specifies all files, the types specified by the non-compress + * extension will be treated as special cases and will not be compressed. + * 3. Don't allow the non-compress extension specifies all files. + */ +static int f2fs_test_compress_extension(unsigned char (*noext)[F2FS_EXTENSION_LEN], + int noext_cnt, + unsigned char (*ext)[F2FS_EXTENSION_LEN], + int ext_cnt) +{ + int index = 0, no_index = 0; + + if (!noext_cnt) + return 0; + + for (no_index = 0; no_index < noext_cnt; no_index++) { + if (strlen(noext[no_index]) == 0) + continue; + if (!strcasecmp("*", noext[no_index])) { + f2fs_info(NULL, "Don't allow the nocompress extension specifies all files"); + return -EINVAL; + } + for (index = 0; index < ext_cnt; index++) { + if (strlen(ext[index]) == 0) + continue; + if (!strcasecmp(ext[index], noext[no_index])) { + f2fs_info(NULL, "Don't allow the same extension %s appear in both compress and nocompress extension", + ext[index]); + return -EINVAL; + } + } + } + return 0; +} + +#ifdef CONFIG_F2FS_FS_LZ4 +static int f2fs_set_lz4hc_level(struct f2fs_fs_context *ctx, const char *str) +{ +#ifdef CONFIG_F2FS_FS_LZ4HC + unsigned int level; + + if (strlen(str) == 3) { + F2FS_CTX_INFO(ctx).compress_level = 0; + ctx->spec_mask |= F2FS_SPEC_compress_level; + return 0; + } + + str += 3; + + if (str[0] != ':') { + f2fs_info(NULL, "wrong format, e.g. :"); + return -EINVAL; + } + if (kstrtouint(str + 1, 10, &level)) + return -EINVAL; + + if (!f2fs_is_compress_level_valid(COMPRESS_LZ4, level)) { + f2fs_info(NULL, "invalid lz4hc compress level: %d", level); + return -EINVAL; + } + + F2FS_CTX_INFO(ctx).compress_level = level; + ctx->spec_mask |= F2FS_SPEC_compress_level; + return 0; +#else + if (strlen(str) == 3) { + F2FS_CTX_INFO(ctx).compress_level = 0; + ctx->spec_mask |= F2FS_SPEC_compress_level; + return 0; + } + f2fs_info(NULL, "kernel doesn't support lz4hc compression"); + return -EINVAL; +#endif +} +#endif + +#ifdef CONFIG_F2FS_FS_ZSTD +static int f2fs_set_zstd_level(struct f2fs_fs_context *ctx, const char *str) +{ + int level; + int len = 4; + + if (strlen(str) == len) { + F2FS_CTX_INFO(ctx).compress_level = F2FS_ZSTD_DEFAULT_CLEVEL; + ctx->spec_mask |= F2FS_SPEC_compress_level; + return 0; + } + + str += len; + + if (str[0] != ':') { + f2fs_info(NULL, "wrong format, e.g. :"); + return -EINVAL; + } + if (kstrtoint(str + 1, 10, &level)) + return -EINVAL; + + /* f2fs does not support negative compress level now */ + if (level < 0) { + f2fs_info(NULL, "do not support negative compress level: %d", level); + return -ERANGE; + } + + if (!f2fs_is_compress_level_valid(COMPRESS_ZSTD, level)) { + f2fs_info(NULL, "invalid zstd compress level: %d", level); + return -EINVAL; + } + + F2FS_CTX_INFO(ctx).compress_level = level; + ctx->spec_mask |= F2FS_SPEC_compress_level; + return 0; +} +#endif +#endif + +static int f2fs_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct f2fs_fs_context *ctx = fc->fs_private; +#ifdef CONFIG_F2FS_FS_COMPRESSION + unsigned char (*ext)[F2FS_EXTENSION_LEN]; + unsigned char (*noext)[F2FS_EXTENSION_LEN]; + int ext_cnt, noext_cnt; + char *name; +#endif + substring_t args[MAX_OPT_ARGS]; + struct fs_parse_result result; + int token, ret, arg; + + token = fs_parse(fc, f2fs_param_specs, param, &result); + if (token < 0) + return token; + + switch (token) { + case Opt_gc_background: + F2FS_CTX_INFO(ctx).bggc_mode = result.uint_32; + ctx->spec_mask |= F2FS_SPEC_background_gc; + break; + case Opt_disable_roll_forward: + ctx_set_opt(ctx, F2FS_MOUNT_DISABLE_ROLL_FORWARD); + break; + case Opt_norecovery: + /* requires ro mount, checked in f2fs_validate_options */ + ctx_set_opt(ctx, F2FS_MOUNT_NORECOVERY); + break; + case Opt_discard: + if (result.negated) + ctx_clear_opt(ctx, F2FS_MOUNT_DISCARD); + else + ctx_set_opt(ctx, F2FS_MOUNT_DISCARD); + break; + case Opt_noheap: + case Opt_heap: + f2fs_warn(NULL, "heap/no_heap options were deprecated"); + break; +#ifdef CONFIG_F2FS_FS_XATTR + case Opt_user_xattr: + if (result.negated) + ctx_clear_opt(ctx, F2FS_MOUNT_XATTR_USER); + else + ctx_set_opt(ctx, F2FS_MOUNT_XATTR_USER); + break; + case Opt_inline_xattr: + if (result.negated) + ctx_clear_opt(ctx, F2FS_MOUNT_INLINE_XATTR); + else + ctx_set_opt(ctx, F2FS_MOUNT_INLINE_XATTR); + break; + case Opt_inline_xattr_size: + if (result.int_32 < MIN_INLINE_XATTR_SIZE || + result.int_32 > MAX_INLINE_XATTR_SIZE) { + f2fs_err(NULL, "inline xattr size is out of range: %u ~ %u", + (u32)MIN_INLINE_XATTR_SIZE, (u32)MAX_INLINE_XATTR_SIZE); + return -EINVAL; + } + ctx_set_opt(ctx, F2FS_MOUNT_INLINE_XATTR_SIZE); + F2FS_CTX_INFO(ctx).inline_xattr_size = result.int_32; + ctx->spec_mask |= F2FS_SPEC_inline_xattr_size; + break; +#else + case Opt_user_xattr: + case Opt_inline_xattr: + case Opt_inline_xattr_size: + f2fs_info(NULL, "%s options not supported", param->key); + break; +#endif +#ifdef CONFIG_F2FS_FS_POSIX_ACL + case Opt_acl: + if (result.negated) + ctx_clear_opt(ctx, F2FS_MOUNT_POSIX_ACL); + else + ctx_set_opt(ctx, F2FS_MOUNT_POSIX_ACL); + break; +#else + case Opt_acl: + f2fs_info(NULL, "%s options not supported", param->key); + break; +#endif + case Opt_active_logs: + if (result.int_32 != 2 && result.int_32 != 4 && + result.int_32 != NR_CURSEG_PERSIST_TYPE) + return -EINVAL; + ctx->spec_mask |= F2FS_SPEC_active_logs; + F2FS_CTX_INFO(ctx).active_logs = result.int_32; + break; + case Opt_disable_ext_identify: + ctx_set_opt(ctx, F2FS_MOUNT_DISABLE_EXT_IDENTIFY); + break; + case Opt_inline_data: + if (result.negated) + ctx_clear_opt(ctx, F2FS_MOUNT_INLINE_DATA); + else + ctx_set_opt(ctx, F2FS_MOUNT_INLINE_DATA); + break; + case Opt_inline_dentry: + if (result.negated) + ctx_clear_opt(ctx, F2FS_MOUNT_INLINE_DENTRY); + else + ctx_set_opt(ctx, F2FS_MOUNT_INLINE_DENTRY); + break; + case Opt_flush_merge: + if (result.negated) + ctx_clear_opt(ctx, F2FS_MOUNT_FLUSH_MERGE); + else + ctx_set_opt(ctx, F2FS_MOUNT_FLUSH_MERGE); + break; + case Opt_barrier: + if (result.negated) + ctx_set_opt(ctx, F2FS_MOUNT_NOBARRIER); + else + ctx_clear_opt(ctx, F2FS_MOUNT_NOBARRIER); + break; + case Opt_fastboot: + ctx_set_opt(ctx, F2FS_MOUNT_FASTBOOT); + break; + case Opt_extent_cache: + if (result.negated) + ctx_clear_opt(ctx, F2FS_MOUNT_READ_EXTENT_CACHE); + else + ctx_set_opt(ctx, F2FS_MOUNT_READ_EXTENT_CACHE); + break; + case Opt_data_flush: + ctx_set_opt(ctx, F2FS_MOUNT_DATA_FLUSH); + break; + case Opt_reserve_root: + ctx_set_opt(ctx, F2FS_MOUNT_RESERVE_ROOT); + F2FS_CTX_INFO(ctx).root_reserved_blocks = result.uint_32; + ctx->spec_mask |= F2FS_SPEC_reserve_root; + break; + case Opt_reserve_node: + ctx_set_opt(ctx, F2FS_MOUNT_RESERVE_NODE); + F2FS_CTX_INFO(ctx).root_reserved_nodes = result.uint_32; + ctx->spec_mask |= F2FS_SPEC_reserve_node; + break; + case Opt_resuid: + F2FS_CTX_INFO(ctx).s_resuid = result.uid; + ctx->spec_mask |= F2FS_SPEC_resuid; + break; + case Opt_resgid: + F2FS_CTX_INFO(ctx).s_resgid = result.gid; + ctx->spec_mask |= F2FS_SPEC_resgid; + break; + case Opt_mode: + F2FS_CTX_INFO(ctx).fs_mode = result.uint_32; + ctx->spec_mask |= F2FS_SPEC_mode; + break; +#ifdef CONFIG_F2FS_FAULT_INJECTION + case Opt_fault_injection: + F2FS_CTX_INFO(ctx).fault_info.inject_rate = result.int_32; + ctx->spec_mask |= F2FS_SPEC_fault_injection; + ctx_set_opt(ctx, F2FS_MOUNT_FAULT_INJECTION); + break; + + case Opt_fault_type: + if (result.uint_32 > BIT(FAULT_MAX)) + return -EINVAL; + F2FS_CTX_INFO(ctx).fault_info.inject_type = result.uint_32; + ctx->spec_mask |= F2FS_SPEC_fault_type; + ctx_set_opt(ctx, F2FS_MOUNT_FAULT_INJECTION); + break; +#else + case Opt_fault_injection: + case Opt_fault_type: + f2fs_info(NULL, "%s options not supported", param->key); + break; +#endif + case Opt_lazytime: + if (result.negated) + ctx_clear_opt(ctx, F2FS_MOUNT_LAZYTIME); + else + ctx_set_opt(ctx, F2FS_MOUNT_LAZYTIME); + break; +#ifdef CONFIG_QUOTA + case Opt_quota: + if (result.negated) { + ctx_clear_opt(ctx, F2FS_MOUNT_QUOTA); + ctx_clear_opt(ctx, F2FS_MOUNT_USRQUOTA); + ctx_clear_opt(ctx, F2FS_MOUNT_GRPQUOTA); + ctx_clear_opt(ctx, F2FS_MOUNT_PRJQUOTA); + } else + ctx_set_opt(ctx, F2FS_MOUNT_USRQUOTA); + break; + case Opt_usrquota: + ctx_set_opt(ctx, F2FS_MOUNT_USRQUOTA); + break; + case Opt_grpquota: + ctx_set_opt(ctx, F2FS_MOUNT_GRPQUOTA); + break; + case Opt_prjquota: + ctx_set_opt(ctx, F2FS_MOUNT_PRJQUOTA); + break; + case Opt_usrjquota: + if (param->type == fs_value_is_string && *param->string) + ret = f2fs_note_qf_name(fc, USRQUOTA, param); + else + ret = f2fs_unnote_qf_name(fc, USRQUOTA); + if (ret) + return ret; + break; + case Opt_grpjquota: + if (param->type == fs_value_is_string && *param->string) + ret = f2fs_note_qf_name(fc, GRPQUOTA, param); + else + ret = f2fs_unnote_qf_name(fc, GRPQUOTA); + if (ret) + return ret; + break; + case Opt_prjjquota: + if (param->type == fs_value_is_string && *param->string) + ret = f2fs_note_qf_name(fc, PRJQUOTA, param); + else + ret = f2fs_unnote_qf_name(fc, PRJQUOTA); + if (ret) + return ret; + break; + case Opt_jqfmt: + F2FS_CTX_INFO(ctx).s_jquota_fmt = result.int_32; + ctx->spec_mask |= F2FS_SPEC_jqfmt; + break; +#else + case Opt_quota: + case Opt_usrquota: + case Opt_grpquota: + case Opt_prjquota: + case Opt_usrjquota: + case Opt_grpjquota: + case Opt_prjjquota: + f2fs_info(NULL, "quota operations not supported"); + break; +#endif + case Opt_alloc: + F2FS_CTX_INFO(ctx).alloc_mode = result.uint_32; + ctx->spec_mask |= F2FS_SPEC_alloc_mode; + break; + case Opt_fsync: + F2FS_CTX_INFO(ctx).fsync_mode = result.uint_32; + ctx->spec_mask |= F2FS_SPEC_fsync_mode; + break; + case Opt_test_dummy_encryption: + ret = f2fs_parse_test_dummy_encryption(param, ctx); + if (ret) + return ret; + break; + case Opt_inlinecrypt: +#ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT + ctx_set_opt(ctx, F2FS_MOUNT_INLINECRYPT); +#else + f2fs_info(NULL, "inline encryption not supported"); +#endif + break; + case Opt_checkpoint: + /* + * Initialize args struct so we know whether arg was + * found; some options take optional arguments. + */ + args[0].from = args[0].to = NULL; + arg = 0; + + /* revert to match_table for checkpoint= options */ + token = match_token(param->string, f2fs_checkpoint_tokens, args); + switch (token) { + case Opt_checkpoint_disable_cap_perc: + if (args->from && match_int(args, &arg)) + return -EINVAL; + if (arg < 0 || arg > 100) + return -EINVAL; + F2FS_CTX_INFO(ctx).unusable_cap_perc = arg; + ctx->spec_mask |= F2FS_SPEC_checkpoint_disable_cap_perc; + ctx_set_opt(ctx, F2FS_MOUNT_DISABLE_CHECKPOINT); + break; + case Opt_checkpoint_disable_cap: + if (args->from && match_int(args, &arg)) + return -EINVAL; + F2FS_CTX_INFO(ctx).unusable_cap = arg; + ctx->spec_mask |= F2FS_SPEC_checkpoint_disable_cap; + ctx_set_opt(ctx, F2FS_MOUNT_DISABLE_CHECKPOINT); + break; + case Opt_checkpoint_disable: + ctx_set_opt(ctx, F2FS_MOUNT_DISABLE_CHECKPOINT); + break; + case Opt_checkpoint_enable: + F2FS_CTX_INFO(ctx).unusable_cap_perc = 0; + ctx->spec_mask |= F2FS_SPEC_checkpoint_disable_cap_perc; + F2FS_CTX_INFO(ctx).unusable_cap = 0; + ctx->spec_mask |= F2FS_SPEC_checkpoint_disable_cap; + ctx_clear_opt(ctx, F2FS_MOUNT_DISABLE_CHECKPOINT); + break; + default: + return -EINVAL; + } + break; + case Opt_checkpoint_merge: + if (result.negated) + ctx_clear_opt(ctx, F2FS_MOUNT_MERGE_CHECKPOINT); + else + ctx_set_opt(ctx, F2FS_MOUNT_MERGE_CHECKPOINT); + break; +#ifdef CONFIG_F2FS_FS_COMPRESSION + case Opt_compress_algorithm: + name = param->string; + if (!strcmp(name, "lzo")) { +#ifdef CONFIG_F2FS_FS_LZO + F2FS_CTX_INFO(ctx).compress_level = 0; + F2FS_CTX_INFO(ctx).compress_algorithm = COMPRESS_LZO; + ctx->spec_mask |= F2FS_SPEC_compress_level; + ctx->spec_mask |= F2FS_SPEC_compress_algorithm; +#else + f2fs_info(NULL, "kernel doesn't support lzo compression"); +#endif + } else if (!strncmp(name, "lz4", 3)) { +#ifdef CONFIG_F2FS_FS_LZ4 + ret = f2fs_set_lz4hc_level(ctx, name); + if (ret) + return -EINVAL; + F2FS_CTX_INFO(ctx).compress_algorithm = COMPRESS_LZ4; + ctx->spec_mask |= F2FS_SPEC_compress_algorithm; +#else + f2fs_info(NULL, "kernel doesn't support lz4 compression"); +#endif + } else if (!strncmp(name, "zstd", 4)) { +#ifdef CONFIG_F2FS_FS_ZSTD + ret = f2fs_set_zstd_level(ctx, name); + if (ret) + return -EINVAL; + F2FS_CTX_INFO(ctx).compress_algorithm = COMPRESS_ZSTD; + ctx->spec_mask |= F2FS_SPEC_compress_algorithm; +#else + f2fs_info(NULL, "kernel doesn't support zstd compression"); +#endif + } else if (!strcmp(name, "lzo-rle")) { +#ifdef CONFIG_F2FS_FS_LZORLE + F2FS_CTX_INFO(ctx).compress_level = 0; + F2FS_CTX_INFO(ctx).compress_algorithm = COMPRESS_LZORLE; + ctx->spec_mask |= F2FS_SPEC_compress_level; + ctx->spec_mask |= F2FS_SPEC_compress_algorithm; +#else + f2fs_info(NULL, "kernel doesn't support lzorle compression"); +#endif + } else + return -EINVAL; + break; + case Opt_compress_log_size: + if (result.uint_32 < MIN_COMPRESS_LOG_SIZE || + result.uint_32 > MAX_COMPRESS_LOG_SIZE) { + f2fs_err(NULL, + "Compress cluster log size is out of range"); + return -EINVAL; + } + F2FS_CTX_INFO(ctx).compress_log_size = result.uint_32; + ctx->spec_mask |= F2FS_SPEC_compress_log_size; + break; + case Opt_compress_extension: + name = param->string; + ext = F2FS_CTX_INFO(ctx).extensions; + ext_cnt = F2FS_CTX_INFO(ctx).compress_ext_cnt; + + if (strlen(name) >= F2FS_EXTENSION_LEN || + ext_cnt >= COMPRESS_EXT_NUM) { + f2fs_err(NULL, "invalid extension length/number"); + return -EINVAL; + } + + if (is_compress_extension_exist(&ctx->info, name, true)) + break; + + ret = strscpy(ext[ext_cnt], name, F2FS_EXTENSION_LEN); + if (ret < 0) + return ret; + F2FS_CTX_INFO(ctx).compress_ext_cnt++; + ctx->spec_mask |= F2FS_SPEC_compress_extension; + break; + case Opt_nocompress_extension: + name = param->string; + noext = F2FS_CTX_INFO(ctx).noextensions; + noext_cnt = F2FS_CTX_INFO(ctx).nocompress_ext_cnt; + + if (strlen(name) >= F2FS_EXTENSION_LEN || + noext_cnt >= COMPRESS_EXT_NUM) { + f2fs_err(NULL, "invalid extension length/number"); + return -EINVAL; + } + + if (is_compress_extension_exist(&ctx->info, name, false)) + break; + + ret = strscpy(noext[noext_cnt], name, F2FS_EXTENSION_LEN); + if (ret < 0) + return ret; + F2FS_CTX_INFO(ctx).nocompress_ext_cnt++; + ctx->spec_mask |= F2FS_SPEC_nocompress_extension; + break; + case Opt_compress_chksum: + F2FS_CTX_INFO(ctx).compress_chksum = true; + ctx->spec_mask |= F2FS_SPEC_compress_chksum; + break; + case Opt_compress_mode: + F2FS_CTX_INFO(ctx).compress_mode = result.uint_32; + ctx->spec_mask |= F2FS_SPEC_compress_mode; + break; + case Opt_compress_cache: + ctx_set_opt(ctx, F2FS_MOUNT_COMPRESS_CACHE); + break; +#else + case Opt_compress_algorithm: + case Opt_compress_log_size: + case Opt_compress_extension: + case Opt_nocompress_extension: + case Opt_compress_chksum: + case Opt_compress_mode: + case Opt_compress_cache: + f2fs_info(NULL, "compression options not supported"); + break; +#endif + case Opt_atgc: + ctx_set_opt(ctx, F2FS_MOUNT_ATGC); + break; + case Opt_gc_merge: + if (result.negated) + ctx_clear_opt(ctx, F2FS_MOUNT_GC_MERGE); + else + ctx_set_opt(ctx, F2FS_MOUNT_GC_MERGE); + break; + case Opt_discard_unit: + F2FS_CTX_INFO(ctx).discard_unit = result.uint_32; + ctx->spec_mask |= F2FS_SPEC_discard_unit; + break; + case Opt_memory_mode: + F2FS_CTX_INFO(ctx).memory_mode = result.uint_32; + ctx->spec_mask |= F2FS_SPEC_memory_mode; + break; + case Opt_age_extent_cache: + ctx_set_opt(ctx, F2FS_MOUNT_AGE_EXTENT_CACHE); + break; + case Opt_errors: + F2FS_CTX_INFO(ctx).errors = result.uint_32; + ctx->spec_mask |= F2FS_SPEC_errors; + break; + case Opt_nat_bits: + ctx_set_opt(ctx, F2FS_MOUNT_NAT_BITS); + break; + case Opt_lookup_mode: + F2FS_CTX_INFO(ctx).lookup_mode = result.uint_32; + ctx->spec_mask |= F2FS_SPEC_lookup_mode; + break; + } + return 0; +} + +/* + * Check quota settings consistency. + */ +static int f2fs_check_quota_consistency(struct fs_context *fc, + struct super_block *sb) +{ + struct f2fs_sb_info *sbi = F2FS_SB(sb); + #ifdef CONFIG_QUOTA + struct f2fs_fs_context *ctx = fc->fs_private; + bool quota_feature = f2fs_sb_has_quota_ino(sbi); + bool quota_turnon = sb_any_quota_loaded(sb); + char *old_qname, *new_qname; + bool usr_qf_name, grp_qf_name, prj_qf_name, usrquota, grpquota, prjquota; + int i; + + /* + * We do the test below only for project quotas. 'usrquota' and + * 'grpquota' mount options are allowed even without quota feature + * to support legacy quotas in quota files. + */ + if (ctx_test_opt(ctx, F2FS_MOUNT_PRJQUOTA) && + !f2fs_sb_has_project_quota(sbi)) { + f2fs_err(sbi, "Project quota feature not enabled. Cannot enable project quota enforcement."); + return -EINVAL; + } + + if (ctx->qname_mask) { + for (i = 0; i < MAXQUOTAS; i++) { + if (!(ctx->qname_mask & (1 << i))) + continue; + + old_qname = F2FS_OPTION(sbi).s_qf_names[i]; + new_qname = F2FS_CTX_INFO(ctx).s_qf_names[i]; + if (quota_turnon && + !!old_qname != !!new_qname) + goto err_jquota_change; + + if (old_qname) { + if (!new_qname) { + f2fs_info(sbi, "remove qf_name %s", + old_qname); + continue; + } else if (strcmp(old_qname, new_qname) == 0) { + ctx->qname_mask &= ~(1 << i); + continue; + } + goto err_jquota_specified; + } + + if (quota_feature) { + f2fs_info(sbi, "QUOTA feature is enabled, so ignore qf_name"); + ctx->qname_mask &= ~(1 << i); + kfree(F2FS_CTX_INFO(ctx).s_qf_names[i]); + F2FS_CTX_INFO(ctx).s_qf_names[i] = NULL; + } + } + } + + /* Make sure we don't mix old and new quota format */ + usr_qf_name = F2FS_OPTION(sbi).s_qf_names[USRQUOTA] || + F2FS_CTX_INFO(ctx).s_qf_names[USRQUOTA]; + grp_qf_name = F2FS_OPTION(sbi).s_qf_names[GRPQUOTA] || + F2FS_CTX_INFO(ctx).s_qf_names[GRPQUOTA]; + prj_qf_name = F2FS_OPTION(sbi).s_qf_names[PRJQUOTA] || + F2FS_CTX_INFO(ctx).s_qf_names[PRJQUOTA]; + usrquota = test_opt(sbi, USRQUOTA) || + ctx_test_opt(ctx, F2FS_MOUNT_USRQUOTA); + grpquota = test_opt(sbi, GRPQUOTA) || + ctx_test_opt(ctx, F2FS_MOUNT_GRPQUOTA); + prjquota = test_opt(sbi, PRJQUOTA) || + ctx_test_opt(ctx, F2FS_MOUNT_PRJQUOTA); + + if (usr_qf_name) { + ctx_clear_opt(ctx, F2FS_MOUNT_USRQUOTA); + usrquota = false; + } + if (grp_qf_name) { + ctx_clear_opt(ctx, F2FS_MOUNT_GRPQUOTA); + grpquota = false; + } + if (prj_qf_name) { + ctx_clear_opt(ctx, F2FS_MOUNT_PRJQUOTA); + prjquota = false; + } + if (usr_qf_name || grp_qf_name || prj_qf_name) { + if (grpquota || usrquota || prjquota) { + f2fs_err(sbi, "old and new quota format mixing"); + return -EINVAL; + } + if (!(ctx->spec_mask & F2FS_SPEC_jqfmt || + F2FS_OPTION(sbi).s_jquota_fmt)) { + f2fs_err(sbi, "journaled quota format not specified"); + return -EINVAL; + } + } + return 0; + +err_jquota_change: + f2fs_err(sbi, "Cannot change journaled quota options when quota turned on"); + return -EINVAL; +err_jquota_specified: + f2fs_err(sbi, "%s quota file already specified", + QTYPE2NAME(i)); + return -EINVAL; + +#else + if (f2fs_readonly(sbi->sb)) + return 0; + if (f2fs_sb_has_quota_ino(sbi)) { + f2fs_info(sbi, "Filesystem with quota feature cannot be mounted RDWR without CONFIG_QUOTA"); + return -EINVAL; + } + if (f2fs_sb_has_project_quota(sbi)) { + f2fs_err(sbi, "Filesystem with project quota feature cannot be mounted RDWR without CONFIG_QUOTA"); + return -EINVAL; + } + + return 0; +#endif +} + +static int f2fs_check_test_dummy_encryption(struct fs_context *fc, + struct super_block *sb) +{ + struct f2fs_fs_context *ctx = fc->fs_private; + struct f2fs_sb_info *sbi = F2FS_SB(sb); + + if (!fscrypt_is_dummy_policy_set(&F2FS_CTX_INFO(ctx).dummy_enc_policy)) + return 0; + + if (!f2fs_sb_has_encrypt(sbi)) { + f2fs_err(sbi, "Encrypt feature is off"); + return -EINVAL; + } + + /* + * This mount option is just for testing, and it's not worthwhile to + * implement the extra complexity (e.g. RCU protection) that would be + * needed to allow it to be set or changed during remount. We do allow + * it to be specified during remount, but only if there is no change. + */ + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE) { + if (fscrypt_dummy_policies_equal(&F2FS_OPTION(sbi).dummy_enc_policy, + &F2FS_CTX_INFO(ctx).dummy_enc_policy)) + return 0; + f2fs_warn(sbi, "Can't set or change test_dummy_encryption on remount"); + return -EINVAL; + } + return 0; +} + +static inline bool test_compression_spec(unsigned int mask) +{ + return mask & (F2FS_SPEC_compress_algorithm + | F2FS_SPEC_compress_log_size + | F2FS_SPEC_compress_extension + | F2FS_SPEC_nocompress_extension + | F2FS_SPEC_compress_chksum + | F2FS_SPEC_compress_mode); +} + +static inline void clear_compression_spec(struct f2fs_fs_context *ctx) +{ + ctx->spec_mask &= ~(F2FS_SPEC_compress_algorithm + | F2FS_SPEC_compress_log_size + | F2FS_SPEC_compress_extension + | F2FS_SPEC_nocompress_extension + | F2FS_SPEC_compress_chksum + | F2FS_SPEC_compress_mode); +} + +static int f2fs_check_compression(struct fs_context *fc, + struct super_block *sb) +{ +#ifdef CONFIG_F2FS_FS_COMPRESSION + struct f2fs_fs_context *ctx = fc->fs_private; + struct f2fs_sb_info *sbi = F2FS_SB(sb); + int i, cnt; + + if (!f2fs_sb_has_compression(sbi)) { + if (test_compression_spec(ctx->spec_mask) || + ctx_test_opt(ctx, F2FS_MOUNT_COMPRESS_CACHE)) + f2fs_info(sbi, "Image doesn't support compression"); + clear_compression_spec(ctx); + ctx->opt_mask &= ~BIT(F2FS_MOUNT_COMPRESS_CACHE); + return 0; + } + if (ctx->spec_mask & F2FS_SPEC_compress_extension) { + cnt = F2FS_CTX_INFO(ctx).compress_ext_cnt; + for (i = 0; i < F2FS_CTX_INFO(ctx).compress_ext_cnt; i++) { + if (is_compress_extension_exist(&F2FS_OPTION(sbi), + F2FS_CTX_INFO(ctx).extensions[i], true)) { + F2FS_CTX_INFO(ctx).extensions[i][0] = '\0'; + cnt--; + } + } + if (F2FS_OPTION(sbi).compress_ext_cnt + cnt > COMPRESS_EXT_NUM) { + f2fs_err(sbi, "invalid extension length/number"); + return -EINVAL; + } + } + if (ctx->spec_mask & F2FS_SPEC_nocompress_extension) { + cnt = F2FS_CTX_INFO(ctx).nocompress_ext_cnt; + for (i = 0; i < F2FS_CTX_INFO(ctx).nocompress_ext_cnt; i++) { + if (is_compress_extension_exist(&F2FS_OPTION(sbi), + F2FS_CTX_INFO(ctx).noextensions[i], false)) { + F2FS_CTX_INFO(ctx).noextensions[i][0] = '\0'; + cnt--; + } + } + if (F2FS_OPTION(sbi).nocompress_ext_cnt + cnt > COMPRESS_EXT_NUM) { + f2fs_err(sbi, "invalid noextension length/number"); + return -EINVAL; + } + } + + if (f2fs_test_compress_extension(F2FS_CTX_INFO(ctx).noextensions, + F2FS_CTX_INFO(ctx).nocompress_ext_cnt, + F2FS_CTX_INFO(ctx).extensions, + F2FS_CTX_INFO(ctx).compress_ext_cnt)) { + f2fs_err(sbi, "new noextensions conflicts with new extensions"); + return -EINVAL; + } + if (f2fs_test_compress_extension(F2FS_CTX_INFO(ctx).noextensions, + F2FS_CTX_INFO(ctx).nocompress_ext_cnt, + F2FS_OPTION(sbi).extensions, + F2FS_OPTION(sbi).compress_ext_cnt)) { + f2fs_err(sbi, "new noextensions conflicts with old extensions"); + return -EINVAL; + } + if (f2fs_test_compress_extension(F2FS_OPTION(sbi).noextensions, + F2FS_OPTION(sbi).nocompress_ext_cnt, + F2FS_CTX_INFO(ctx).extensions, + F2FS_CTX_INFO(ctx).compress_ext_cnt)) { + f2fs_err(sbi, "new extensions conflicts with old noextensions"); + return -EINVAL; + } +#endif + return 0; +} + +static int f2fs_check_opt_consistency(struct fs_context *fc, + struct super_block *sb) +{ + struct f2fs_fs_context *ctx = fc->fs_private; + struct f2fs_sb_info *sbi = F2FS_SB(sb); + int err; + + if (ctx_test_opt(ctx, F2FS_MOUNT_NORECOVERY) && !f2fs_readonly(sb)) + return -EINVAL; + + if (f2fs_hw_should_discard(sbi) && + (ctx->opt_mask & BIT(F2FS_MOUNT_DISCARD)) && + !ctx_test_opt(ctx, F2FS_MOUNT_DISCARD)) { + f2fs_warn(sbi, "discard is required for zoned block devices"); + return -EINVAL; + } + + if (!f2fs_hw_support_discard(sbi) && + (ctx->opt_mask & BIT(F2FS_MOUNT_DISCARD)) && + ctx_test_opt(ctx, F2FS_MOUNT_DISCARD)) { + f2fs_warn(sbi, "device does not support discard"); + ctx_clear_opt(ctx, F2FS_MOUNT_DISCARD); + ctx->opt_mask &= ~BIT(F2FS_MOUNT_DISCARD); + } + + if (f2fs_sb_has_device_alias(sbi) && + (ctx->opt_mask & BIT(F2FS_MOUNT_READ_EXTENT_CACHE)) && + !ctx_test_opt(ctx, F2FS_MOUNT_READ_EXTENT_CACHE)) { + f2fs_err(sbi, "device aliasing requires extent cache"); + return -EINVAL; + } + + if (test_opt(sbi, RESERVE_ROOT) && + (ctx->opt_mask & BIT(F2FS_MOUNT_RESERVE_ROOT)) && + ctx_test_opt(ctx, F2FS_MOUNT_RESERVE_ROOT)) { + f2fs_info(sbi, "Preserve previous reserve_root=%u", + F2FS_OPTION(sbi).root_reserved_blocks); + ctx_clear_opt(ctx, F2FS_MOUNT_RESERVE_ROOT); + ctx->opt_mask &= ~BIT(F2FS_MOUNT_RESERVE_ROOT); + ctx->spec_mask &= ~F2FS_SPEC_reserve_root; + } + if (test_opt(sbi, RESERVE_NODE) && + (ctx->opt_mask & BIT(F2FS_MOUNT_RESERVE_NODE)) && + ctx_test_opt(ctx, F2FS_MOUNT_RESERVE_NODE)) { + f2fs_info(sbi, "Preserve previous reserve_node=%u", + F2FS_OPTION(sbi).root_reserved_nodes); + ctx_clear_opt(ctx, F2FS_MOUNT_RESERVE_NODE); + ctx->opt_mask &= ~BIT(F2FS_MOUNT_RESERVE_NODE); + ctx->spec_mask &= ~F2FS_SPEC_reserve_node; + } + + err = f2fs_check_test_dummy_encryption(fc, sb); + if (err) + return err; + + err = f2fs_check_compression(fc, sb); + if (err) + return err; + + err = f2fs_check_quota_consistency(fc, sb); + if (err) + return err; + + if (!IS_ENABLED(CONFIG_UNICODE) && f2fs_sb_has_casefold(sbi)) { + f2fs_err(sbi, + "Filesystem with casefold feature cannot be mounted without CONFIG_UNICODE"); + return -EINVAL; + } + + /* + * The BLKZONED feature indicates that the drive was formatted with + * zone alignment optimization. This is optional for host-aware + * devices, but mandatory for host-managed zoned block devices. + */ + if (f2fs_sb_has_blkzoned(sbi)) { + if (F2FS_CTX_INFO(ctx).bggc_mode == BGGC_MODE_OFF) { + f2fs_warn(sbi, "zoned devices need bggc"); + return -EINVAL; + } +#ifdef CONFIG_BLK_DEV_ZONED + if ((ctx->spec_mask & F2FS_SPEC_discard_unit) && + F2FS_CTX_INFO(ctx).discard_unit != DISCARD_UNIT_SECTION) { + f2fs_info(sbi, "Zoned block device doesn't need small discard, set discard_unit=section by default"); + F2FS_CTX_INFO(ctx).discard_unit = DISCARD_UNIT_SECTION; + } + + if ((ctx->spec_mask & F2FS_SPEC_mode) && + F2FS_CTX_INFO(ctx).fs_mode != FS_MODE_LFS) { + f2fs_info(sbi, "Only lfs mode is allowed with zoned block device feature"); + return -EINVAL; + } +#else + f2fs_err(sbi, "Zoned block device support is not enabled"); + return -EINVAL; +#endif + } + + if (ctx_test_opt(ctx, F2FS_MOUNT_INLINE_XATTR_SIZE)) { + if (!f2fs_sb_has_extra_attr(sbi) || + !f2fs_sb_has_flexible_inline_xattr(sbi)) { + f2fs_err(sbi, "extra_attr or flexible_inline_xattr feature is off"); + return -EINVAL; + } + if (!ctx_test_opt(ctx, F2FS_MOUNT_INLINE_XATTR) && !test_opt(sbi, INLINE_XATTR)) { + f2fs_err(sbi, "inline_xattr_size option should be set with inline_xattr option"); + return -EINVAL; + } + } + + if (ctx_test_opt(ctx, F2FS_MOUNT_ATGC) && + F2FS_CTX_INFO(ctx).fs_mode == FS_MODE_LFS) { + f2fs_err(sbi, "LFS is not compatible with ATGC"); + return -EINVAL; + } + + if (f2fs_is_readonly(sbi) && ctx_test_opt(ctx, F2FS_MOUNT_FLUSH_MERGE)) { + f2fs_err(sbi, "FLUSH_MERGE not compatible with readonly mode"); + return -EINVAL; + } + + if (f2fs_sb_has_readonly(sbi) && !f2fs_readonly(sbi->sb)) { + f2fs_err(sbi, "Allow to mount readonly mode only"); + return -EROFS; + } + return 0; +} + +static void f2fs_apply_quota_options(struct fs_context *fc, + struct super_block *sb) +{ +#ifdef CONFIG_QUOTA + struct f2fs_fs_context *ctx = fc->fs_private; + struct f2fs_sb_info *sbi = F2FS_SB(sb); + bool quota_feature = f2fs_sb_has_quota_ino(sbi); + char *qname; + int i; + + if (quota_feature) + return; + + for (i = 0; i < MAXQUOTAS; i++) { + if (!(ctx->qname_mask & (1 << i))) + continue; + + qname = F2FS_CTX_INFO(ctx).s_qf_names[i]; + if (qname) { + qname = kstrdup(F2FS_CTX_INFO(ctx).s_qf_names[i], + GFP_KERNEL | __GFP_NOFAIL); + set_opt(sbi, QUOTA); + } + F2FS_OPTION(sbi).s_qf_names[i] = qname; + } + + if (ctx->spec_mask & F2FS_SPEC_jqfmt) + F2FS_OPTION(sbi).s_jquota_fmt = F2FS_CTX_INFO(ctx).s_jquota_fmt; + + if (quota_feature && F2FS_OPTION(sbi).s_jquota_fmt) { + f2fs_info(sbi, "QUOTA feature is enabled, so ignore jquota_fmt"); + F2FS_OPTION(sbi).s_jquota_fmt = 0; + } +#endif +} + +static void f2fs_apply_test_dummy_encryption(struct fs_context *fc, + struct super_block *sb) +{ + struct f2fs_fs_context *ctx = fc->fs_private; + struct f2fs_sb_info *sbi = F2FS_SB(sb); + + if (!fscrypt_is_dummy_policy_set(&F2FS_CTX_INFO(ctx).dummy_enc_policy) || + /* if already set, it was already verified to be the same */ + fscrypt_is_dummy_policy_set(&F2FS_OPTION(sbi).dummy_enc_policy)) + return; + swap(F2FS_OPTION(sbi).dummy_enc_policy, F2FS_CTX_INFO(ctx).dummy_enc_policy); + f2fs_warn(sbi, "Test dummy encryption mode enabled"); +} + +static void f2fs_apply_compression(struct fs_context *fc, + struct super_block *sb) +{ +#ifdef CONFIG_F2FS_FS_COMPRESSION + struct f2fs_fs_context *ctx = fc->fs_private; + struct f2fs_sb_info *sbi = F2FS_SB(sb); + unsigned char (*ctx_ext)[F2FS_EXTENSION_LEN]; + unsigned char (*sbi_ext)[F2FS_EXTENSION_LEN]; + int ctx_cnt, sbi_cnt, i; + + if (ctx->spec_mask & F2FS_SPEC_compress_level) + F2FS_OPTION(sbi).compress_level = + F2FS_CTX_INFO(ctx).compress_level; + if (ctx->spec_mask & F2FS_SPEC_compress_algorithm) + F2FS_OPTION(sbi).compress_algorithm = + F2FS_CTX_INFO(ctx).compress_algorithm; + if (ctx->spec_mask & F2FS_SPEC_compress_log_size) + F2FS_OPTION(sbi).compress_log_size = + F2FS_CTX_INFO(ctx).compress_log_size; + if (ctx->spec_mask & F2FS_SPEC_compress_chksum) + F2FS_OPTION(sbi).compress_chksum = + F2FS_CTX_INFO(ctx).compress_chksum; + if (ctx->spec_mask & F2FS_SPEC_compress_mode) + F2FS_OPTION(sbi).compress_mode = + F2FS_CTX_INFO(ctx).compress_mode; + if (ctx->spec_mask & F2FS_SPEC_compress_extension) { + ctx_ext = F2FS_CTX_INFO(ctx).extensions; + ctx_cnt = F2FS_CTX_INFO(ctx).compress_ext_cnt; + sbi_ext = F2FS_OPTION(sbi).extensions; + sbi_cnt = F2FS_OPTION(sbi).compress_ext_cnt; + for (i = 0; i < ctx_cnt; i++) { + if (strlen(ctx_ext[i]) == 0) + continue; + strscpy(sbi_ext[sbi_cnt], ctx_ext[i]); + sbi_cnt++; + } + F2FS_OPTION(sbi).compress_ext_cnt = sbi_cnt; + } + if (ctx->spec_mask & F2FS_SPEC_nocompress_extension) { + ctx_ext = F2FS_CTX_INFO(ctx).noextensions; + ctx_cnt = F2FS_CTX_INFO(ctx).nocompress_ext_cnt; + sbi_ext = F2FS_OPTION(sbi).noextensions; + sbi_cnt = F2FS_OPTION(sbi).nocompress_ext_cnt; + for (i = 0; i < ctx_cnt; i++) { + if (strlen(ctx_ext[i]) == 0) + continue; + strscpy(sbi_ext[sbi_cnt], ctx_ext[i]); + sbi_cnt++; + } + F2FS_OPTION(sbi).nocompress_ext_cnt = sbi_cnt; + } +#endif +} + +static void f2fs_apply_options(struct fs_context *fc, struct super_block *sb) +{ + struct f2fs_fs_context *ctx = fc->fs_private; + struct f2fs_sb_info *sbi = F2FS_SB(sb); + + F2FS_OPTION(sbi).opt &= ~ctx->opt_mask; + F2FS_OPTION(sbi).opt |= F2FS_CTX_INFO(ctx).opt; + + if (ctx->spec_mask & F2FS_SPEC_background_gc) + F2FS_OPTION(sbi).bggc_mode = F2FS_CTX_INFO(ctx).bggc_mode; + if (ctx->spec_mask & F2FS_SPEC_inline_xattr_size) + F2FS_OPTION(sbi).inline_xattr_size = + F2FS_CTX_INFO(ctx).inline_xattr_size; + if (ctx->spec_mask & F2FS_SPEC_active_logs) + F2FS_OPTION(sbi).active_logs = F2FS_CTX_INFO(ctx).active_logs; + if (ctx->spec_mask & F2FS_SPEC_reserve_root) + F2FS_OPTION(sbi).root_reserved_blocks = + F2FS_CTX_INFO(ctx).root_reserved_blocks; + if (ctx->spec_mask & F2FS_SPEC_reserve_node) + F2FS_OPTION(sbi).root_reserved_nodes = + F2FS_CTX_INFO(ctx).root_reserved_nodes; + if (ctx->spec_mask & F2FS_SPEC_resgid) + F2FS_OPTION(sbi).s_resgid = F2FS_CTX_INFO(ctx).s_resgid; + if (ctx->spec_mask & F2FS_SPEC_resuid) + F2FS_OPTION(sbi).s_resuid = F2FS_CTX_INFO(ctx).s_resuid; + if (ctx->spec_mask & F2FS_SPEC_mode) + F2FS_OPTION(sbi).fs_mode = F2FS_CTX_INFO(ctx).fs_mode; +#ifdef CONFIG_F2FS_FAULT_INJECTION + if (ctx->spec_mask & F2FS_SPEC_fault_injection) + (void)f2fs_build_fault_attr(sbi, + F2FS_CTX_INFO(ctx).fault_info.inject_rate, 0, FAULT_RATE); + if (ctx->spec_mask & F2FS_SPEC_fault_type) + (void)f2fs_build_fault_attr(sbi, 0, + F2FS_CTX_INFO(ctx).fault_info.inject_type, FAULT_TYPE); +#endif + if (ctx->spec_mask & F2FS_SPEC_alloc_mode) + F2FS_OPTION(sbi).alloc_mode = F2FS_CTX_INFO(ctx).alloc_mode; + if (ctx->spec_mask & F2FS_SPEC_fsync_mode) + F2FS_OPTION(sbi).fsync_mode = F2FS_CTX_INFO(ctx).fsync_mode; + if (ctx->spec_mask & F2FS_SPEC_checkpoint_disable_cap) + F2FS_OPTION(sbi).unusable_cap = F2FS_CTX_INFO(ctx).unusable_cap; + if (ctx->spec_mask & F2FS_SPEC_checkpoint_disable_cap_perc) + F2FS_OPTION(sbi).unusable_cap_perc = + F2FS_CTX_INFO(ctx).unusable_cap_perc; + if (ctx->spec_mask & F2FS_SPEC_discard_unit) + F2FS_OPTION(sbi).discard_unit = F2FS_CTX_INFO(ctx).discard_unit; + if (ctx->spec_mask & F2FS_SPEC_memory_mode) + F2FS_OPTION(sbi).memory_mode = F2FS_CTX_INFO(ctx).memory_mode; + if (ctx->spec_mask & F2FS_SPEC_errors) + F2FS_OPTION(sbi).errors = F2FS_CTX_INFO(ctx).errors; + if (ctx->spec_mask & F2FS_SPEC_lookup_mode) + F2FS_OPTION(sbi).lookup_mode = F2FS_CTX_INFO(ctx).lookup_mode; + + f2fs_apply_compression(fc, sb); + f2fs_apply_test_dummy_encryption(fc, sb); + f2fs_apply_quota_options(fc, sb); +} + +static int f2fs_sanity_check_options(struct f2fs_sb_info *sbi, bool remount) +{ + if (f2fs_sb_has_device_alias(sbi) && + !test_opt(sbi, READ_EXTENT_CACHE)) { + f2fs_err(sbi, "device aliasing requires extent cache"); + return -EINVAL; + } + + if (!remount) + return 0; + +#ifdef CONFIG_BLK_DEV_ZONED + if (f2fs_sb_has_blkzoned(sbi) && + sbi->max_open_zones < F2FS_OPTION(sbi).active_logs) { + f2fs_err(sbi, + "zoned: max open zones %u is too small, need at least %u open zones", + sbi->max_open_zones, F2FS_OPTION(sbi).active_logs); + return -EINVAL; + } +#endif + if (f2fs_lfs_mode(sbi) && !IS_F2FS_IPU_DISABLE(sbi)) { + f2fs_warn(sbi, "LFS is not compatible with IPU"); + return -EINVAL; + } + return 0; +} + +static struct inode *f2fs_alloc_inode(struct super_block *sb) +{ + struct f2fs_inode_info *fi; + + if (time_to_inject(F2FS_SB(sb), FAULT_SLAB_ALLOC)) + return NULL; + + fi = alloc_inode_sb(sb, f2fs_inode_cachep, GFP_F2FS_ZERO); + if (!fi) + return NULL; + + init_once((void *) fi); + + /* Initialize f2fs-specific inode info */ + atomic_set(&fi->dirty_pages, 0); + atomic_set(&fi->i_compr_blocks, 0); + atomic_set(&fi->open_count, 0); + atomic_set(&fi->writeback, 0); + init_f2fs_rwsem(&fi->i_sem); + spin_lock_init(&fi->i_size_lock); + INIT_LIST_HEAD(&fi->dirty_list); + INIT_LIST_HEAD(&fi->gdirty_list); + INIT_LIST_HEAD(&fi->gdonate_list); + init_f2fs_rwsem(&fi->i_gc_rwsem[READ]); + init_f2fs_rwsem(&fi->i_gc_rwsem[WRITE]); + init_f2fs_rwsem(&fi->i_xattr_sem); + + /* Will be used by directory only */ + fi->i_dir_level = F2FS_SB(sb)->dir_level; + + return &fi->vfs_inode; +} + +static int f2fs_drop_inode(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + int ret; + + /* + * during filesystem shutdown, if checkpoint is disabled, + * drop useless meta/node dirty pages. + */ + if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) { + if (inode->i_ino == F2FS_NODE_INO(sbi) || + inode->i_ino == F2FS_META_INO(sbi)) { + trace_f2fs_drop_inode(inode, 1); + return 1; + } + } + + /* + * This is to avoid a deadlock condition like below. + * writeback_single_inode(inode) + * - f2fs_write_data_page + * - f2fs_gc -> iput -> evict + * - inode_wait_for_writeback(inode) + */ + if ((!inode_unhashed(inode) && inode_state_read(inode) & I_SYNC)) { + if (!inode->i_nlink && !is_bad_inode(inode)) { + /* to avoid evict_inode call simultaneously */ + __iget(inode); + spin_unlock(&inode->i_lock); + + /* should remain fi->extent_tree for writepage */ + f2fs_destroy_extent_node(inode); + + sb_start_intwrite(inode->i_sb); + f2fs_i_size_write(inode, 0); + + f2fs_submit_merged_write_cond(F2FS_I_SB(inode), + inode, NULL, 0, DATA); + truncate_inode_pages_final(inode->i_mapping); + + if (F2FS_HAS_BLOCKS(inode)) + f2fs_truncate(inode); + + sb_end_intwrite(inode->i_sb); + + spin_lock(&inode->i_lock); + atomic_dec(&inode->i_count); + } + trace_f2fs_drop_inode(inode, 0); + return 0; + } + ret = inode_generic_drop(inode); + if (!ret) + ret = fscrypt_drop_inode(inode); + trace_f2fs_drop_inode(inode, ret); + return ret; +} + +int f2fs_inode_dirtied(struct inode *inode, bool sync) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + int ret = 0; + + spin_lock(&sbi->inode_lock[DIRTY_META]); + if (is_inode_flag_set(inode, FI_DIRTY_INODE)) { + ret = 1; + } else { + set_inode_flag(inode, FI_DIRTY_INODE); + stat_inc_dirty_inode(sbi, DIRTY_META); + } + if (sync && list_empty(&F2FS_I(inode)->gdirty_list)) { + list_add_tail(&F2FS_I(inode)->gdirty_list, + &sbi->inode_list[DIRTY_META]); + inc_page_count(sbi, F2FS_DIRTY_IMETA); + } + spin_unlock(&sbi->inode_lock[DIRTY_META]); + + /* if atomic write is not committed, set inode w/ atomic dirty */ + if (!ret && f2fs_is_atomic_file(inode) && + !is_inode_flag_set(inode, FI_ATOMIC_COMMITTED)) + set_inode_flag(inode, FI_ATOMIC_DIRTIED); + + return ret; +} + +void f2fs_inode_synced(struct inode *inode) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + + spin_lock(&sbi->inode_lock[DIRTY_META]); + if (!is_inode_flag_set(inode, FI_DIRTY_INODE)) { + spin_unlock(&sbi->inode_lock[DIRTY_META]); + return; + } + if (!list_empty(&F2FS_I(inode)->gdirty_list)) { + list_del_init(&F2FS_I(inode)->gdirty_list); + dec_page_count(sbi, F2FS_DIRTY_IMETA); + } + clear_inode_flag(inode, FI_DIRTY_INODE); + clear_inode_flag(inode, FI_AUTO_RECOVER); + stat_dec_dirty_inode(F2FS_I_SB(inode), DIRTY_META); + spin_unlock(&sbi->inode_lock[DIRTY_META]); +} + +/* + * f2fs_dirty_inode() is called from __mark_inode_dirty() + * + * We should call set_dirty_inode to write the dirty inode through write_inode. + */ +static void f2fs_dirty_inode(struct inode *inode, int flags) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + + if (inode->i_ino == F2FS_NODE_INO(sbi) || + inode->i_ino == F2FS_META_INO(sbi)) + return; + + if (is_inode_flag_set(inode, FI_AUTO_RECOVER)) + clear_inode_flag(inode, FI_AUTO_RECOVER); + + f2fs_inode_dirtied(inode, false); +} + +static void f2fs_free_inode(struct inode *inode) +{ + fscrypt_free_inode(inode); + kmem_cache_free(f2fs_inode_cachep, F2FS_I(inode)); +} + +static void destroy_percpu_info(struct f2fs_sb_info *sbi) +{ + percpu_counter_destroy(&sbi->total_valid_inode_count); + percpu_counter_destroy(&sbi->rf_node_block_count); + percpu_counter_destroy(&sbi->alloc_valid_block_count); +} + +static void destroy_device_list(struct f2fs_sb_info *sbi) +{ + int i; + + for (i = 0; i < sbi->s_ndevs; i++) { + if (i > 0) + fs_bdev_file_release(FDEV(i).bdev_file, sbi->sb); +#ifdef CONFIG_BLK_DEV_ZONED + kvfree(FDEV(i).blkz_seq); +#endif + } + kvfree(sbi->devs); +} + +static void f2fs_put_super(struct super_block *sb) +{ + struct f2fs_sb_info *sbi = F2FS_SB(sb); + int i; + int err = 0; + bool done; + + /* unregister procfs/sysfs entries in advance to avoid race case */ + f2fs_unregister_sysfs(sbi); + + f2fs_quota_off_umount(sb); + + /* prevent remaining shrinker jobs */ + mutex_lock(&sbi->umount_mutex); + + /* + * flush all issued checkpoints and stop checkpoint issue thread. + * after then, all checkpoints should be done by each process context. + */ + f2fs_stop_ckpt_thread(sbi); + + /* + * We don't need to do checkpoint when superblock is clean. + * But, the previous checkpoint was not done by umount, it needs to do + * clean checkpoint again. + */ + if ((is_sbi_flag_set(sbi, SBI_IS_DIRTY) || + !is_set_ckpt_flags(sbi, CP_UMOUNT_FLAG))) { + struct cp_control cpc = { + .reason = CP_UMOUNT, + }; + stat_inc_cp_call_count(sbi, TOTAL_CALL); + err = f2fs_write_checkpoint(sbi, &cpc); + } + + /* be sure to wait for any on-going discard commands */ + done = f2fs_issue_discard_timeout(sbi, true); + if (f2fs_realtime_discard_enable(sbi) && !sbi->discard_blks && done) { + struct cp_control cpc = { + .reason = CP_UMOUNT | CP_TRIMMED, + }; + stat_inc_cp_call_count(sbi, TOTAL_CALL); + err = f2fs_write_checkpoint(sbi, &cpc); + } + + /* + * normally superblock is clean, so we need to release this. + * In addition, EIO will skip do checkpoint, we need this as well. + */ + f2fs_release_ino_entry(sbi, true); + + f2fs_leave_shrinker(sbi); + mutex_unlock(&sbi->umount_mutex); + + /* our cp_error case, we can wait for any writeback page */ + f2fs_flush_merged_writes(sbi); + + f2fs_wait_on_all_pages(sbi, F2FS_WB_CP_DATA); + + if (err || f2fs_cp_error(sbi)) { + truncate_inode_pages_final(NODE_MAPPING(sbi)); + truncate_inode_pages_final(META_MAPPING(sbi)); + } + + f2fs_bug_on(sbi, sbi->fsync_node_num); + + f2fs_destroy_compress_inode(sbi); + + iput(sbi->node_inode); + sbi->node_inode = NULL; + + iput(sbi->meta_inode); + sbi->meta_inode = NULL; + + /* Should check the page counts after dropping all node/meta pages */ + for (i = 0; i < NR_COUNT_TYPE; i++) { + if (!get_pages(sbi, i)) + continue; + f2fs_err(sbi, "detect filesystem reference count leak during " + "umount, type: %d, count: %lld", i, get_pages(sbi, i)); + f2fs_bug_on(sbi, 1); + } + + /* + * iput() can update stat information, if f2fs_write_checkpoint() + * above failed with error. + */ + f2fs_destroy_stats(sbi); + + /* destroy f2fs internal modules */ + f2fs_destroy_node_manager(sbi); + f2fs_destroy_segment_manager(sbi); + + /* flush s_error_work before sbi destroy */ + flush_work(&sbi->s_error_work); + + f2fs_destroy_wq(sbi); + + kvfree(sbi->ckpt); + + kfree(sbi->raw_super); + + f2fs_destroy_page_array_cache(sbi); +#ifdef CONFIG_QUOTA + for (i = 0; i < MAXQUOTAS; i++) + kfree(F2FS_OPTION(sbi).s_qf_names[i]); +#endif + fscrypt_free_dummy_policy(&F2FS_OPTION(sbi).dummy_enc_policy); + destroy_percpu_info(sbi); + f2fs_destroy_iostat(sbi); + for (i = 0; i < NR_PAGE_TYPE; i++) + kfree(sbi->write_io[i]); +#if IS_ENABLED(CONFIG_UNICODE) + utf8_unload(sb->s_encoding); +#endif + sync_blockdev(sb->s_bdev); + invalidate_bdev(sb->s_bdev); + for (i = 1; i < sbi->s_ndevs; i++) { + sync_blockdev(FDEV(i).bdev); + invalidate_bdev(FDEV(i).bdev); + } +} + +int f2fs_sync_fs(struct super_block *sb, int sync) +{ + struct f2fs_sb_info *sbi = F2FS_SB(sb); + int err = 0; + + if (unlikely(f2fs_cp_error(sbi))) + return 0; + if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) + return 0; + + trace_f2fs_sync_fs(sb, sync); + + if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) + return -EAGAIN; + + if (sync) { + stat_inc_cp_call_count(sbi, TOTAL_CALL); + err = f2fs_issue_checkpoint(sbi); + } + + return err; +} + +static int f2fs_freeze(struct super_block *sb) +{ + struct f2fs_sb_info *sbi = F2FS_SB(sb); + + if (f2fs_readonly(sb)) + return 0; + + /* IO error happened before */ + if (unlikely(f2fs_cp_error(sbi))) + return -EIO; + + /* must be clean, since sync_filesystem() was already called */ + if (is_sbi_flag_set(sbi, SBI_IS_DIRTY)) + return -EINVAL; + + sbi->umount_lock_holder = current; + + /* Let's flush checkpoints and stop the thread. */ + f2fs_flush_ckpt_thread(sbi); + + sbi->umount_lock_holder = NULL; + + /* to avoid deadlock on f2fs_evict_inode->SB_FREEZE_FS */ + set_sbi_flag(sbi, SBI_IS_FREEZING); + return 0; +} + +static int f2fs_unfreeze(struct super_block *sb) +{ + struct f2fs_sb_info *sbi = F2FS_SB(sb); + + /* + * It will update discard_max_bytes of mounted lvm device to zero + * after creating snapshot on this lvm device, let's drop all + * remained discards. + * We don't need to disable real-time discard because discard_max_bytes + * will recover after removal of snapshot. + */ + if (test_opt(sbi, DISCARD) && !f2fs_hw_support_discard(sbi)) + f2fs_issue_discard_timeout(sbi, true); + + clear_sbi_flag(F2FS_SB(sb), SBI_IS_FREEZING); + return 0; +} + +#ifdef CONFIG_QUOTA +static int f2fs_statfs_project(struct super_block *sb, + kprojid_t projid, struct kstatfs *buf) +{ + struct kqid qid; + struct dquot *dquot; + u64 limit; + u64 curblock; + + qid = make_kqid_projid(projid); + dquot = dqget(sb, qid); + if (IS_ERR(dquot)) + return PTR_ERR(dquot); + spin_lock(&dquot->dq_dqb_lock); + + limit = min_not_zero(dquot->dq_dqb.dqb_bsoftlimit, + dquot->dq_dqb.dqb_bhardlimit); + limit >>= sb->s_blocksize_bits; + + if (limit) { + uint64_t remaining = 0; + + curblock = (dquot->dq_dqb.dqb_curspace + + dquot->dq_dqb.dqb_rsvspace) >> sb->s_blocksize_bits; + if (limit > curblock) + remaining = limit - curblock; + + buf->f_blocks = min(buf->f_blocks, limit); + buf->f_bfree = min(buf->f_bfree, remaining); + buf->f_bavail = min(buf->f_bavail, remaining); + } + + limit = min_not_zero(dquot->dq_dqb.dqb_isoftlimit, + dquot->dq_dqb.dqb_ihardlimit); + + if (limit) { + uint64_t remaining = 0; + + if (limit > dquot->dq_dqb.dqb_curinodes) + remaining = limit - dquot->dq_dqb.dqb_curinodes; + + buf->f_files = min(buf->f_files, limit); + buf->f_ffree = min(buf->f_ffree, remaining); + } + + spin_unlock(&dquot->dq_dqb_lock); + dqput(dquot); + return 0; +} +#endif + +static int f2fs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + struct f2fs_sb_info *sbi = F2FS_SB(sb); + u64 id = huge_encode_dev(sb->s_bdev->bd_dev); + block_t total_count, user_block_count, start_count; + u64 avail_node_count; + unsigned int total_valid_node_count; + + total_count = le64_to_cpu(sbi->raw_super->block_count); + start_count = le32_to_cpu(sbi->raw_super->segment0_blkaddr); + buf->f_type = F2FS_SUPER_MAGIC; + buf->f_bsize = sbi->blocksize; + + buf->f_blocks = total_count - start_count; + + spin_lock(&sbi->stat_lock); + if (sbi->carve_out) + buf->f_blocks -= sbi->current_reserved_blocks; + user_block_count = sbi->user_block_count; + total_valid_node_count = valid_node_count(sbi); + avail_node_count = sbi->total_node_count - F2FS_RESERVED_NODE_NUM; + buf->f_bfree = user_block_count - valid_user_blocks(sbi) - + sbi->current_reserved_blocks; + + if (unlikely(buf->f_bfree <= sbi->unusable_block_count)) + buf->f_bfree = 0; + else + buf->f_bfree -= sbi->unusable_block_count; + spin_unlock(&sbi->stat_lock); + + if (buf->f_bfree > F2FS_OPTION(sbi).root_reserved_blocks) + buf->f_bavail = buf->f_bfree - + F2FS_OPTION(sbi).root_reserved_blocks; + else + buf->f_bavail = 0; + + if (avail_node_count > user_block_count) { + buf->f_files = user_block_count; + buf->f_ffree = buf->f_bavail; + } else { + buf->f_files = avail_node_count; + buf->f_ffree = min(avail_node_count - total_valid_node_count, + buf->f_bavail); + } + + buf->f_namelen = F2FS_NAME_LEN; + buf->f_fsid = u64_to_fsid(id); + +#ifdef CONFIG_QUOTA + if (is_inode_flag_set(d_inode(dentry), FI_PROJ_INHERIT) && + sb_has_quota_limits_enabled(sb, PRJQUOTA)) { + f2fs_statfs_project(sb, F2FS_I(d_inode(dentry))->i_projid, buf); + } +#endif + return 0; +} + +static inline void f2fs_show_quota_options(struct seq_file *seq, + struct super_block *sb) +{ +#ifdef CONFIG_QUOTA + struct f2fs_sb_info *sbi = F2FS_SB(sb); + + if (F2FS_OPTION(sbi).s_jquota_fmt) { + char *fmtname = ""; + + switch (F2FS_OPTION(sbi).s_jquota_fmt) { + case QFMT_VFS_OLD: + fmtname = "vfsold"; + break; + case QFMT_VFS_V0: + fmtname = "vfsv0"; + break; + case QFMT_VFS_V1: + fmtname = "vfsv1"; + break; + } + seq_printf(seq, ",jqfmt=%s", fmtname); + } + + if (F2FS_OPTION(sbi).s_qf_names[USRQUOTA]) + seq_show_option(seq, "usrjquota", + F2FS_OPTION(sbi).s_qf_names[USRQUOTA]); + + if (F2FS_OPTION(sbi).s_qf_names[GRPQUOTA]) + seq_show_option(seq, "grpjquota", + F2FS_OPTION(sbi).s_qf_names[GRPQUOTA]); + + if (F2FS_OPTION(sbi).s_qf_names[PRJQUOTA]) + seq_show_option(seq, "prjjquota", + F2FS_OPTION(sbi).s_qf_names[PRJQUOTA]); +#endif +} + +#ifdef CONFIG_F2FS_FS_COMPRESSION +static inline void f2fs_show_compress_options(struct seq_file *seq, + struct super_block *sb) +{ + struct f2fs_sb_info *sbi = F2FS_SB(sb); + char *algtype = ""; + int i; + + if (!f2fs_sb_has_compression(sbi)) + return; + + switch (F2FS_OPTION(sbi).compress_algorithm) { + case COMPRESS_LZO: + algtype = "lzo"; + break; + case COMPRESS_LZ4: + algtype = "lz4"; + break; + case COMPRESS_ZSTD: + algtype = "zstd"; + break; + case COMPRESS_LZORLE: + algtype = "lzo-rle"; + break; + } + seq_printf(seq, ",compress_algorithm=%s", algtype); + + if (F2FS_OPTION(sbi).compress_level) + seq_printf(seq, ":%d", F2FS_OPTION(sbi).compress_level); + + seq_printf(seq, ",compress_log_size=%u", + F2FS_OPTION(sbi).compress_log_size); + + for (i = 0; i < F2FS_OPTION(sbi).compress_ext_cnt; i++) { + seq_printf(seq, ",compress_extension=%s", + F2FS_OPTION(sbi).extensions[i]); + } + + for (i = 0; i < F2FS_OPTION(sbi).nocompress_ext_cnt; i++) { + seq_printf(seq, ",nocompress_extension=%s", + F2FS_OPTION(sbi).noextensions[i]); + } + + if (F2FS_OPTION(sbi).compress_chksum) + seq_puts(seq, ",compress_chksum"); + + if (F2FS_OPTION(sbi).compress_mode == COMPR_MODE_FS) + seq_printf(seq, ",compress_mode=%s", "fs"); + else if (F2FS_OPTION(sbi).compress_mode == COMPR_MODE_USER) + seq_printf(seq, ",compress_mode=%s", "user"); + + if (test_opt(sbi, COMPRESS_CACHE)) + seq_puts(seq, ",compress_cache"); +} +#endif + +static int f2fs_show_options(struct seq_file *seq, struct dentry *root) +{ + struct f2fs_sb_info *sbi = F2FS_SB(root->d_sb); + + if (F2FS_OPTION(sbi).bggc_mode == BGGC_MODE_SYNC) + seq_printf(seq, ",background_gc=%s", "sync"); + else if (F2FS_OPTION(sbi).bggc_mode == BGGC_MODE_ON) + seq_printf(seq, ",background_gc=%s", "on"); + else if (F2FS_OPTION(sbi).bggc_mode == BGGC_MODE_OFF) + seq_printf(seq, ",background_gc=%s", "off"); + + if (test_opt(sbi, GC_MERGE)) + seq_puts(seq, ",gc_merge"); + else + seq_puts(seq, ",nogc_merge"); + + if (test_opt(sbi, DISABLE_ROLL_FORWARD)) + seq_puts(seq, ",disable_roll_forward"); + if (test_opt(sbi, NORECOVERY)) + seq_puts(seq, ",norecovery"); + if (test_opt(sbi, DISCARD)) { + seq_puts(seq, ",discard"); + if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_BLOCK) + seq_printf(seq, ",discard_unit=%s", "block"); + else if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SEGMENT) + seq_printf(seq, ",discard_unit=%s", "segment"); + else if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SECTION) + seq_printf(seq, ",discard_unit=%s", "section"); + } else { + seq_puts(seq, ",nodiscard"); + } +#ifdef CONFIG_F2FS_FS_XATTR + if (test_opt(sbi, XATTR_USER)) + seq_puts(seq, ",user_xattr"); + else + seq_puts(seq, ",nouser_xattr"); + if (test_opt(sbi, INLINE_XATTR)) + seq_puts(seq, ",inline_xattr"); + else + seq_puts(seq, ",noinline_xattr"); + if (test_opt(sbi, INLINE_XATTR_SIZE)) + seq_printf(seq, ",inline_xattr_size=%u", + F2FS_OPTION(sbi).inline_xattr_size); +#endif +#ifdef CONFIG_F2FS_FS_POSIX_ACL + if (test_opt(sbi, POSIX_ACL)) + seq_puts(seq, ",acl"); + else + seq_puts(seq, ",noacl"); +#endif + if (test_opt(sbi, DISABLE_EXT_IDENTIFY)) + seq_puts(seq, ",disable_ext_identify"); + if (test_opt(sbi, INLINE_DATA)) + seq_puts(seq, ",inline_data"); + else + seq_puts(seq, ",noinline_data"); + if (test_opt(sbi, INLINE_DENTRY)) + seq_puts(seq, ",inline_dentry"); + else + seq_puts(seq, ",noinline_dentry"); + if (test_opt(sbi, FLUSH_MERGE)) + seq_puts(seq, ",flush_merge"); + else + seq_puts(seq, ",noflush_merge"); + if (test_opt(sbi, NOBARRIER)) + seq_puts(seq, ",nobarrier"); + else + seq_puts(seq, ",barrier"); + if (test_opt(sbi, FASTBOOT)) + seq_puts(seq, ",fastboot"); + if (test_opt(sbi, READ_EXTENT_CACHE)) + seq_puts(seq, ",extent_cache"); + else + seq_puts(seq, ",noextent_cache"); + if (test_opt(sbi, AGE_EXTENT_CACHE)) + seq_puts(seq, ",age_extent_cache"); + if (test_opt(sbi, DATA_FLUSH)) + seq_puts(seq, ",data_flush"); + + seq_puts(seq, ",mode="); + if (F2FS_OPTION(sbi).fs_mode == FS_MODE_ADAPTIVE) + seq_puts(seq, "adaptive"); + else if (F2FS_OPTION(sbi).fs_mode == FS_MODE_LFS) + seq_puts(seq, "lfs"); + else if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_SEG) + seq_puts(seq, "fragment:segment"); + else if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK) + seq_puts(seq, "fragment:block"); + seq_printf(seq, ",active_logs=%u", F2FS_OPTION(sbi).active_logs); + if (test_opt(sbi, RESERVE_ROOT) || test_opt(sbi, RESERVE_NODE)) + seq_printf(seq, ",reserve_root=%u,reserve_node=%u,resuid=%u," + "resgid=%u", + F2FS_OPTION(sbi).root_reserved_blocks, + F2FS_OPTION(sbi).root_reserved_nodes, + from_kuid_munged(&init_user_ns, + F2FS_OPTION(sbi).s_resuid), + from_kgid_munged(&init_user_ns, + F2FS_OPTION(sbi).s_resgid)); +#ifdef CONFIG_F2FS_FAULT_INJECTION + if (test_opt(sbi, FAULT_INJECTION)) { + seq_printf(seq, ",fault_injection=%u", + F2FS_OPTION(sbi).fault_info.inject_rate); + seq_printf(seq, ",fault_type=%u", + F2FS_OPTION(sbi).fault_info.inject_type); + } +#endif +#ifdef CONFIG_QUOTA + if (test_opt(sbi, QUOTA)) + seq_puts(seq, ",quota"); + if (test_opt(sbi, USRQUOTA)) + seq_puts(seq, ",usrquota"); + if (test_opt(sbi, GRPQUOTA)) + seq_puts(seq, ",grpquota"); + if (test_opt(sbi, PRJQUOTA)) + seq_puts(seq, ",prjquota"); +#endif + f2fs_show_quota_options(seq, sbi->sb); + + fscrypt_show_test_dummy_encryption(seq, ',', sbi->sb); + + if (sbi->sb->s_flags & SB_INLINECRYPT) + seq_puts(seq, ",inlinecrypt"); + + if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_DEFAULT) + seq_printf(seq, ",alloc_mode=%s", "default"); + else if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE) + seq_printf(seq, ",alloc_mode=%s", "reuse"); + + if (test_opt(sbi, DISABLE_CHECKPOINT)) + seq_printf(seq, ",checkpoint=disable:%u", + F2FS_OPTION(sbi).unusable_cap); + if (test_opt(sbi, MERGE_CHECKPOINT)) + seq_puts(seq, ",checkpoint_merge"); + else + seq_puts(seq, ",nocheckpoint_merge"); + if (F2FS_OPTION(sbi).fsync_mode == FSYNC_MODE_POSIX) + seq_printf(seq, ",fsync_mode=%s", "posix"); + else if (F2FS_OPTION(sbi).fsync_mode == FSYNC_MODE_STRICT) + seq_printf(seq, ",fsync_mode=%s", "strict"); + else if (F2FS_OPTION(sbi).fsync_mode == FSYNC_MODE_NOBARRIER) + seq_printf(seq, ",fsync_mode=%s", "nobarrier"); + +#ifdef CONFIG_F2FS_FS_COMPRESSION + f2fs_show_compress_options(seq, sbi->sb); +#endif + + if (test_opt(sbi, ATGC)) + seq_puts(seq, ",atgc"); + + if (F2FS_OPTION(sbi).memory_mode == MEMORY_MODE_NORMAL) + seq_printf(seq, ",memory=%s", "normal"); + else if (F2FS_OPTION(sbi).memory_mode == MEMORY_MODE_LOW) + seq_printf(seq, ",memory=%s", "low"); + + if (F2FS_OPTION(sbi).errors == MOUNT_ERRORS_READONLY) + seq_printf(seq, ",errors=%s", "remount-ro"); + else if (F2FS_OPTION(sbi).errors == MOUNT_ERRORS_CONTINUE) + seq_printf(seq, ",errors=%s", "continue"); + else if (F2FS_OPTION(sbi).errors == MOUNT_ERRORS_PANIC) + seq_printf(seq, ",errors=%s", "panic"); + + if (test_opt(sbi, NAT_BITS)) + seq_puts(seq, ",nat_bits"); + + if (F2FS_OPTION(sbi).lookup_mode == LOOKUP_PERF) + seq_show_option(seq, "lookup_mode", "perf"); + else if (F2FS_OPTION(sbi).lookup_mode == LOOKUP_COMPAT) + seq_show_option(seq, "lookup_mode", "compat"); + else if (F2FS_OPTION(sbi).lookup_mode == LOOKUP_AUTO) + seq_show_option(seq, "lookup_mode", "auto"); + + return 0; +} + +static void default_options(struct f2fs_sb_info *sbi, bool remount) +{ + /* init some FS parameters */ + if (!remount) { + set_opt(sbi, READ_EXTENT_CACHE); + clear_opt(sbi, DISABLE_CHECKPOINT); + + if (f2fs_hw_support_discard(sbi) || f2fs_hw_should_discard(sbi)) + set_opt(sbi, DISCARD); + + if (f2fs_sb_has_blkzoned(sbi)) + F2FS_OPTION(sbi).discard_unit = DISCARD_UNIT_SECTION; + else + F2FS_OPTION(sbi).discard_unit = DISCARD_UNIT_BLOCK; + } + + if (f2fs_sb_has_readonly(sbi)) + F2FS_OPTION(sbi).active_logs = NR_CURSEG_RO_TYPE; + else + F2FS_OPTION(sbi).active_logs = NR_CURSEG_PERSIST_TYPE; + + F2FS_OPTION(sbi).inline_xattr_size = DEFAULT_INLINE_XATTR_ADDRS; + if (le32_to_cpu(F2FS_RAW_SUPER(sbi)->segment_count_main) <= + SMALL_VOLUME_SEGMENTS) + F2FS_OPTION(sbi).alloc_mode = ALLOC_MODE_REUSE; + else + F2FS_OPTION(sbi).alloc_mode = ALLOC_MODE_DEFAULT; + F2FS_OPTION(sbi).fsync_mode = FSYNC_MODE_POSIX; + F2FS_OPTION(sbi).s_resuid = make_kuid(&init_user_ns, F2FS_DEF_RESUID); + F2FS_OPTION(sbi).s_resgid = make_kgid(&init_user_ns, F2FS_DEF_RESGID); + if (f2fs_sb_has_compression(sbi)) { + F2FS_OPTION(sbi).compress_algorithm = COMPRESS_LZ4; + F2FS_OPTION(sbi).compress_log_size = MIN_COMPRESS_LOG_SIZE; + F2FS_OPTION(sbi).compress_ext_cnt = 0; + F2FS_OPTION(sbi).compress_mode = COMPR_MODE_FS; + } + F2FS_OPTION(sbi).bggc_mode = BGGC_MODE_ON; + F2FS_OPTION(sbi).memory_mode = MEMORY_MODE_NORMAL; + F2FS_OPTION(sbi).errors = MOUNT_ERRORS_CONTINUE; + + set_opt(sbi, INLINE_XATTR); + set_opt(sbi, INLINE_DATA); + set_opt(sbi, INLINE_DENTRY); + set_opt(sbi, MERGE_CHECKPOINT); + set_opt(sbi, LAZYTIME); + F2FS_OPTION(sbi).unusable_cap = 0; + if (!f2fs_is_readonly(sbi)) + set_opt(sbi, FLUSH_MERGE); + if (f2fs_sb_has_blkzoned(sbi)) + F2FS_OPTION(sbi).fs_mode = FS_MODE_LFS; + else + F2FS_OPTION(sbi).fs_mode = FS_MODE_ADAPTIVE; + +#ifdef CONFIG_F2FS_FS_XATTR + set_opt(sbi, XATTR_USER); +#endif +#ifdef CONFIG_F2FS_FS_POSIX_ACL + set_opt(sbi, POSIX_ACL); +#endif + + f2fs_build_fault_attr(sbi, 0, 0, FAULT_ALL); + + F2FS_OPTION(sbi).lookup_mode = LOOKUP_PERF; +} + +#ifdef CONFIG_QUOTA +static int f2fs_enable_quotas(struct super_block *sb); +#endif + +static int f2fs_disable_checkpoint(struct f2fs_sb_info *sbi) +{ + unsigned int s_flags = sbi->sb->s_flags; + struct cp_control cpc; + struct f2fs_lock_context lc; + unsigned int gc_mode = sbi->gc_mode; + int err = 0; + int ret; + block_t unusable; + + if (s_flags & SB_RDONLY) { + f2fs_err(sbi, "checkpoint=disable on readonly fs"); + return -EINVAL; + } + sbi->sb->s_flags |= SB_ACTIVE; + + /* check if we need more GC first */ + unusable = f2fs_get_unusable_blocks(sbi); + + f2fs_info(sbi, "%s starts, unusable: %u", __func__, unusable); + + if (!f2fs_disable_cp_again(sbi, unusable)) + goto skip_gc; + + f2fs_update_time(sbi, DISABLE_TIME); + + sbi->gc_mode = GC_URGENT_HIGH; + + f2fs_info(sbi, "%s: run f2fs_gc() to migrate blocks", __func__); + + while (!f2fs_time_over(sbi, DISABLE_TIME)) { + struct f2fs_gc_control gc_control = { + .victim_segno = NULL_SEGNO, + .init_gc_type = FG_GC, + .should_migrate_blocks = false, + .err_gc_skipped = true, + .no_bg_gc = true, + .nr_free_secs = 1 }; + + f2fs_down_write_trace(&sbi->gc_lock, &gc_control.lc); + stat_inc_gc_call_count(sbi, FOREGROUND); + err = f2fs_gc(sbi, &gc_control); + if (err == -ENODATA) { + err = 0; + break; + } + if (err && err != -EAGAIN) + break; + } + + f2fs_info(sbi, "%s: call sync_filesystem() to persist meta: %lld, node: %lld, data: %lld", + __func__, + get_pages(sbi, F2FS_DIRTY_META), + get_pages(sbi, F2FS_DIRTY_NODES), + get_pages(sbi, F2FS_DIRTY_DATA)); + + ret = sync_filesystem(sbi->sb); + if (ret || err) { + err = ret ? ret : err; + goto restore_flag; + } + + unusable = f2fs_get_unusable_blocks(sbi); + if (f2fs_disable_cp_again(sbi, unusable)) { + err = -EAGAIN; + goto restore_flag; + } + +skip_gc: + f2fs_info(sbi, "%s: call f2fs_write_checkpoint(), meta: %lld, node: %lld, data: %lld", + __func__, + get_pages(sbi, F2FS_DIRTY_META), + get_pages(sbi, F2FS_DIRTY_NODES), + get_pages(sbi, F2FS_DIRTY_DATA)); + + f2fs_down_write_trace(&sbi->gc_lock, &lc); + cpc.reason = CP_PAUSE; + set_sbi_flag(sbi, SBI_CP_DISABLED); + stat_inc_cp_call_count(sbi, TOTAL_CALL); + err = f2fs_write_checkpoint(sbi, &cpc); + if (err) + goto out_unlock; + + spin_lock(&sbi->stat_lock); + sbi->unusable_block_count = unusable; + spin_unlock(&sbi->stat_lock); + +out_unlock: + f2fs_up_write_trace(&sbi->gc_lock, &lc); +restore_flag: + sbi->gc_mode = gc_mode; + sbi->sb->s_flags = s_flags; /* Restore SB_RDONLY status */ + f2fs_info(sbi, "%s finishes, err:%d", __func__, err); + return err; +} + +static int f2fs_enable_checkpoint(struct f2fs_sb_info *sbi) +{ + int retry = MAX_FLUSH_RETRY_COUNT; + long long start, writeback, end; + int ret; + struct f2fs_lock_context lc; + long long skipped_write, dirty_data; + + f2fs_info(sbi, "f2fs_enable_checkpoint() starts, meta: %lld, node: %lld, data: %lld", + get_pages(sbi, F2FS_DIRTY_META), + get_pages(sbi, F2FS_DIRTY_NODES), + get_pages(sbi, F2FS_DIRTY_DATA)); + + start = ktime_get(); + + set_sbi_flag(sbi, SBI_ENABLE_CHECKPOINT); + + /* we should flush all the data to keep data consistency */ + do { + skipped_write = get_pages(sbi, F2FS_SKIPPED_WRITE); + dirty_data = get_pages(sbi, F2FS_DIRTY_DATA); + + sync_inodes_sb(sbi->sb); + f2fs_io_schedule_timeout(DEFAULT_SCHEDULE_TIMEOUT); + + f2fs_info(sbi, "sync_inode_sb done, dirty_data: %lld, %lld, " + "skipped write: %lld, %lld, retry: %d", + get_pages(sbi, F2FS_DIRTY_DATA), + dirty_data, + get_pages(sbi, F2FS_SKIPPED_WRITE), + skipped_write, retry); + + /* + * sync_inodes_sb() has retry logic, so let's check dirty_data + * in prior to skipped_write in case there is no dirty data. + */ + if (!get_pages(sbi, F2FS_DIRTY_DATA)) + break; + if (get_pages(sbi, F2FS_SKIPPED_WRITE) == skipped_write) + break; + } while (retry--); + + clear_sbi_flag(sbi, SBI_ENABLE_CHECKPOINT); + + writeback = ktime_get(); + + if (unlikely(get_pages(sbi, F2FS_DIRTY_DATA) || + get_pages(sbi, F2FS_SKIPPED_WRITE))) + f2fs_warn(sbi, "checkpoint=enable unwritten data: %lld, skipped data: %lld, retry: %d", + get_pages(sbi, F2FS_DIRTY_DATA), + get_pages(sbi, F2FS_SKIPPED_WRITE), retry); + + if (get_pages(sbi, F2FS_SKIPPED_WRITE)) + atomic_set(&sbi->nr_pages[F2FS_SKIPPED_WRITE], 0); + + f2fs_down_write_trace(&sbi->gc_lock, &lc); + f2fs_dirty_to_prefree(sbi); + + clear_sbi_flag(sbi, SBI_CP_DISABLED); + set_sbi_flag(sbi, SBI_IS_DIRTY); + f2fs_up_write_trace(&sbi->gc_lock, &lc); + + ret = f2fs_sync_fs(sbi->sb, 1); + if (ret) + f2fs_err(sbi, "%s sync_fs failed, ret: %d", __func__, ret); + + /* Let's ensure there's no pending checkpoint anymore */ + f2fs_flush_ckpt_thread(sbi); + + end = ktime_get(); + + f2fs_info(sbi, "f2fs_enable_checkpoint() finishes, writeback:%llu, sync:%llu", + ktime_ms_delta(writeback, start), + ktime_ms_delta(end, writeback)); + return ret; +} + +static int __f2fs_remount(struct fs_context *fc, struct super_block *sb) +{ + struct f2fs_sb_info *sbi = F2FS_SB(sb); + struct f2fs_mount_info org_mount_opt; + unsigned long old_sb_flags; + unsigned int flags = fc->sb_flags; + int err; + bool need_restart_gc = false, need_stop_gc = false; + bool need_restart_flush = false, need_stop_flush = false; + bool need_restart_discard = false, need_stop_discard = false; + bool need_enable_checkpoint = false, need_disable_checkpoint = false; + bool no_read_extent_cache = !test_opt(sbi, READ_EXTENT_CACHE); + bool no_age_extent_cache = !test_opt(sbi, AGE_EXTENT_CACHE); + bool enable_checkpoint = !test_opt(sbi, DISABLE_CHECKPOINT); + bool no_atgc = !test_opt(sbi, ATGC); + bool no_discard = !test_opt(sbi, DISCARD); + bool no_compress_cache = !test_opt(sbi, COMPRESS_CACHE); + bool block_unit_discard = f2fs_block_unit_discard(sbi); + bool no_nat_bits = !test_opt(sbi, NAT_BITS); +#ifdef CONFIG_QUOTA + int i, j; +#endif + + /* + * Save the old mount options in case we + * need to restore them. + */ + org_mount_opt = sbi->mount_opt; + old_sb_flags = sb->s_flags; + + sbi->umount_lock_holder = current; + +#ifdef CONFIG_QUOTA + org_mount_opt.s_jquota_fmt = F2FS_OPTION(sbi).s_jquota_fmt; + for (i = 0; i < MAXQUOTAS; i++) { + if (F2FS_OPTION(sbi).s_qf_names[i]) { + org_mount_opt.s_qf_names[i] = + kstrdup(F2FS_OPTION(sbi).s_qf_names[i], + GFP_KERNEL); + if (!org_mount_opt.s_qf_names[i]) { + for (j = 0; j < i; j++) + kfree(org_mount_opt.s_qf_names[j]); + return -ENOMEM; + } + } else { + org_mount_opt.s_qf_names[i] = NULL; + } + } +#endif + + /* recover superblocks we couldn't write due to previous RO mount */ + if (!(flags & SB_RDONLY) && is_sbi_flag_set(sbi, SBI_NEED_SB_WRITE)) { + err = f2fs_commit_super(sbi, false); + f2fs_info(sbi, "Try to recover all the superblocks, ret: %d", + err); + if (!err) + clear_sbi_flag(sbi, SBI_NEED_SB_WRITE); + } + + default_options(sbi, true); + + err = f2fs_check_opt_consistency(fc, sb); + if (err) + goto restore_opts; + + f2fs_apply_options(fc, sb); + + err = f2fs_sanity_check_options(sbi, true); + if (err) + goto restore_opts; + + /* flush outstanding errors before changing fs state */ + flush_work(&sbi->s_error_work); + + /* + * Previous and new state of filesystem is RO, + * so skip checking GC and FLUSH_MERGE conditions. + */ + if (f2fs_readonly(sb) && (flags & SB_RDONLY)) + goto skip; + + if (f2fs_dev_is_readonly(sbi) && !(flags & SB_RDONLY)) { + err = -EROFS; + goto restore_opts; + } + +#ifdef CONFIG_QUOTA + if (!f2fs_readonly(sb) && (flags & SB_RDONLY)) { + err = dquot_suspend(sb, -1); + if (err < 0) + goto restore_opts; + } else if (f2fs_readonly(sb) && !(flags & SB_RDONLY)) { + /* dquot_resume needs RW */ + sb->s_flags &= ~SB_RDONLY; + if (sb_any_quota_suspended(sb)) { + dquot_resume(sb, -1); + } else if (f2fs_sb_has_quota_ino(sbi)) { + err = f2fs_enable_quotas(sb); + if (err) + goto restore_opts; + } + } +#endif + /* disallow enable atgc dynamically */ + if (no_atgc == !!test_opt(sbi, ATGC)) { + err = -EINVAL; + f2fs_warn(sbi, "switch atgc option is not allowed"); + goto restore_opts; + } + + /* disallow enable/disable extent_cache dynamically */ + if (no_read_extent_cache == !!test_opt(sbi, READ_EXTENT_CACHE)) { + err = -EINVAL; + f2fs_warn(sbi, "switch extent_cache option is not allowed"); + goto restore_opts; + } + /* disallow enable/disable age extent_cache dynamically */ + if (no_age_extent_cache == !!test_opt(sbi, AGE_EXTENT_CACHE)) { + err = -EINVAL; + f2fs_warn(sbi, "switch age_extent_cache option is not allowed"); + goto restore_opts; + } + + if (no_compress_cache == !!test_opt(sbi, COMPRESS_CACHE)) { + err = -EINVAL; + f2fs_warn(sbi, "switch compress_cache option is not allowed"); + goto restore_opts; + } + + if (block_unit_discard != f2fs_block_unit_discard(sbi)) { + err = -EINVAL; + f2fs_warn(sbi, "switch discard_unit option is not allowed"); + goto restore_opts; + } + + if (no_nat_bits == !!test_opt(sbi, NAT_BITS)) { + err = -EINVAL; + f2fs_warn(sbi, "switch nat_bits option is not allowed"); + goto restore_opts; + } + + if ((flags & SB_RDONLY) && test_opt(sbi, DISABLE_CHECKPOINT)) { + err = -EINVAL; + f2fs_warn(sbi, "disabling checkpoint not compatible with read-only"); + goto restore_opts; + } + + /* + * We stop the GC thread if FS is mounted as RO + * or if background_gc = off is passed in mount + * option. Also sync the filesystem. + */ + if ((flags & SB_RDONLY) || + (F2FS_OPTION(sbi).bggc_mode == BGGC_MODE_OFF && + !test_opt(sbi, GC_MERGE))) { + if (sbi->gc_thread) { + f2fs_stop_gc_thread(sbi); + need_restart_gc = true; + } + } else if (!sbi->gc_thread) { + err = f2fs_start_gc_thread(sbi); + if (err) + goto restore_opts; + need_stop_gc = true; + } + + if (flags & SB_RDONLY) { + sync_inodes_sb(sb); + + set_sbi_flag(sbi, SBI_IS_DIRTY); + set_sbi_flag(sbi, SBI_IS_CLOSE); + f2fs_sync_fs(sb, 1); + clear_sbi_flag(sbi, SBI_IS_CLOSE); + } + + /* + * We stop issue flush thread if FS is mounted as RO + * or if flush_merge is not passed in mount option. + */ + if ((flags & SB_RDONLY) || !test_opt(sbi, FLUSH_MERGE)) { + clear_opt(sbi, FLUSH_MERGE); + f2fs_destroy_flush_cmd_control(sbi, false); + need_restart_flush = true; + } else { + err = f2fs_create_flush_cmd_control(sbi); + if (err) + goto restore_gc; + need_stop_flush = true; + } + + if (no_discard == !!test_opt(sbi, DISCARD)) { + if (test_opt(sbi, DISCARD)) { + err = f2fs_start_discard_thread(sbi); + if (err) + goto restore_flush; + need_stop_discard = true; + } else { + f2fs_stop_discard_thread(sbi); + /* + * f2fs_ioc_fitrim() won't race w/ "remount ro" + * so it's safe to check discard_cmd_cnt in + * f2fs_issue_discard_timeout(). + */ + f2fs_issue_discard_timeout(sbi, flags & SB_RDONLY); + need_restart_discard = true; + } + } + + adjust_unusable_cap_perc(sbi); + if (enable_checkpoint == !!test_opt(sbi, DISABLE_CHECKPOINT)) { + if (test_opt(sbi, DISABLE_CHECKPOINT)) { + err = f2fs_disable_checkpoint(sbi); + if (err) + goto restore_discard; + need_enable_checkpoint = true; + } else { + err = f2fs_enable_checkpoint(sbi); + if (err) + goto restore_discard; + need_disable_checkpoint = true; + } + } + + /* + * Place this routine at the end, since a new checkpoint would be + * triggered while remount and we need to take care of it before + * returning from remount. + */ + if ((flags & SB_RDONLY) || test_opt(sbi, DISABLE_CHECKPOINT) || + !test_opt(sbi, MERGE_CHECKPOINT)) { + f2fs_stop_ckpt_thread(sbi); + } else { + /* Flush if the previous checkpoint, if exists. */ + f2fs_flush_ckpt_thread(sbi); + + err = f2fs_start_ckpt_thread(sbi); + if (err) { + f2fs_err(sbi, + "Failed to start F2FS issue_checkpoint_thread (%d)", + err); + goto restore_checkpoint; + } + } + +skip: +#ifdef CONFIG_QUOTA + /* Release old quota file names */ + for (i = 0; i < MAXQUOTAS; i++) + kfree(org_mount_opt.s_qf_names[i]); +#endif + /* Update the POSIXACL Flag */ + sb->s_flags = (sb->s_flags & ~SB_POSIXACL) | + (test_opt(sbi, POSIX_ACL) ? SB_POSIXACL : 0); + + limit_reserve_root(sbi); + fc->sb_flags = (flags & ~SB_LAZYTIME) | (sb->s_flags & SB_LAZYTIME); + + sbi->umount_lock_holder = NULL; + return 0; +restore_checkpoint: + if (need_enable_checkpoint) { + if (f2fs_enable_checkpoint(sbi)) + f2fs_warn(sbi, "checkpoint has not been enabled"); + } else if (need_disable_checkpoint) { + if (f2fs_disable_checkpoint(sbi)) + f2fs_warn(sbi, "checkpoint has not been disabled"); + } +restore_discard: + if (need_restart_discard) { + if (f2fs_start_discard_thread(sbi)) + f2fs_warn(sbi, "discard has been stopped"); + } else if (need_stop_discard) { + f2fs_stop_discard_thread(sbi); + } +restore_flush: + if (need_restart_flush) { + if (f2fs_create_flush_cmd_control(sbi)) + f2fs_warn(sbi, "background flush thread has stopped"); + } else if (need_stop_flush) { + clear_opt(sbi, FLUSH_MERGE); + f2fs_destroy_flush_cmd_control(sbi, false); + } +restore_gc: + if (need_restart_gc) { + if (f2fs_start_gc_thread(sbi)) + f2fs_warn(sbi, "background gc thread has stopped"); + } else if (need_stop_gc) { + f2fs_stop_gc_thread(sbi); + } +restore_opts: +#ifdef CONFIG_QUOTA + F2FS_OPTION(sbi).s_jquota_fmt = org_mount_opt.s_jquota_fmt; + for (i = 0; i < MAXQUOTAS; i++) { + kfree(F2FS_OPTION(sbi).s_qf_names[i]); + F2FS_OPTION(sbi).s_qf_names[i] = org_mount_opt.s_qf_names[i]; + } +#endif + sbi->mount_opt = org_mount_opt; + sb->s_flags = old_sb_flags; + + sbi->umount_lock_holder = NULL; + return err; +} + +static void f2fs_shutdown(struct super_block *sb) +{ + f2fs_do_shutdown(F2FS_SB(sb), F2FS_GOING_DOWN_NOSYNC, false, false); +} + +#ifdef CONFIG_QUOTA +static bool f2fs_need_recovery(struct f2fs_sb_info *sbi) +{ + /* need to recovery orphan */ + if (is_set_ckpt_flags(sbi, CP_ORPHAN_PRESENT_FLAG)) + return true; + /* need to recovery data */ + if (test_opt(sbi, DISABLE_ROLL_FORWARD)) + return false; + if (test_opt(sbi, NORECOVERY)) + return false; + return !is_set_ckpt_flags(sbi, CP_UMOUNT_FLAG); +} + +static bool f2fs_recover_quota_begin(struct f2fs_sb_info *sbi) +{ + bool readonly = f2fs_readonly(sbi->sb); + + if (!f2fs_need_recovery(sbi)) + return false; + + /* it doesn't need to check f2fs_sb_has_readonly() */ + if (f2fs_hw_is_readonly(sbi)) + return false; + + if (readonly) { + sbi->sb->s_flags &= ~SB_RDONLY; + set_sbi_flag(sbi, SBI_IS_WRITABLE); + } + + /* + * Turn on quotas which were not enabled for read-only mounts if + * filesystem has quota feature, so that they are updated correctly. + */ + return f2fs_enable_quota_files(sbi, readonly); +} + +static void f2fs_recover_quota_end(struct f2fs_sb_info *sbi, + bool quota_enabled) +{ + if (quota_enabled) + f2fs_quota_off_umount(sbi->sb); + + if (is_sbi_flag_set(sbi, SBI_IS_WRITABLE)) { + clear_sbi_flag(sbi, SBI_IS_WRITABLE); + sbi->sb->s_flags |= SB_RDONLY; + } +} + +/* Read data from quotafile */ +static ssize_t f2fs_quota_read(struct super_block *sb, int type, char *data, + size_t len, loff_t off) +{ + struct inode *inode = sb_dqopt(sb)->files[type]; + struct address_space *mapping = inode->i_mapping; + int tocopy; + size_t toread; + loff_t i_size = i_size_read(inode); + + if (off > i_size) + return 0; + + if (off + len > i_size) + len = i_size - off; + toread = len; + while (toread > 0) { + struct folio *folio; + size_t offset; + +repeat: + folio = mapping_read_folio_gfp(mapping, off >> PAGE_SHIFT, + GFP_NOFS); + if (IS_ERR(folio)) { + if (PTR_ERR(folio) == -ENOMEM) { + memalloc_retry_wait(GFP_NOFS); + goto repeat; + } + set_sbi_flag(F2FS_SB(sb), SBI_QUOTA_NEED_REPAIR); + return PTR_ERR(folio); + } + offset = offset_in_folio(folio, off); + tocopy = min(folio_size(folio) - offset, toread); + + folio_lock(folio); + + if (unlikely(folio->mapping != mapping)) { + f2fs_folio_put(folio, true); + goto repeat; + } + + /* + * should never happen, just leave f2fs_bug_on() here to catch + * any potential bug. + */ + f2fs_bug_on(F2FS_SB(sb), !folio_test_uptodate(folio)); + + memcpy_from_folio(data, folio, offset, tocopy); + f2fs_folio_put(folio, true); + + toread -= tocopy; + data += tocopy; + off += tocopy; + } + return len; +} + +/* Write to quotafile */ +static ssize_t f2fs_quota_write(struct super_block *sb, int type, + const char *data, size_t len, loff_t off) +{ + struct inode *inode = sb_dqopt(sb)->files[type]; + struct address_space *mapping = inode->i_mapping; + const struct address_space_operations *a_ops = mapping->a_ops; + int offset = off & (sb->s_blocksize - 1); + size_t towrite = len; + struct folio *folio; + void *fsdata = NULL; + int err = 0; + int tocopy; + + while (towrite > 0) { + tocopy = min_t(unsigned long, sb->s_blocksize - offset, + towrite); +retry: + err = a_ops->write_begin(NULL, mapping, off, tocopy, + &folio, &fsdata); + if (unlikely(err)) { + if (err == -ENOMEM) { + memalloc_retry_wait(GFP_NOFS); + goto retry; + } + set_sbi_flag(F2FS_SB(sb), SBI_QUOTA_NEED_REPAIR); + break; + } + + memcpy_to_folio(folio, offset_in_folio(folio, off), data, tocopy); + + a_ops->write_end(NULL, mapping, off, tocopy, tocopy, + folio, fsdata); + offset = 0; + towrite -= tocopy; + off += tocopy; + data += tocopy; + cond_resched(); + } + + if (len == towrite) + return err; + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + f2fs_mark_inode_dirty_sync(inode, false); + return len - towrite; +} + +int f2fs_dquot_initialize(struct inode *inode) +{ + if (time_to_inject(F2FS_I_SB(inode), FAULT_DQUOT_INIT)) + return -ESRCH; + + return dquot_initialize(inode); +} + +static struct dquot __rcu **f2fs_get_dquots(struct inode *inode) +{ + return F2FS_I(inode)->i_dquot; +} + +static qsize_t *f2fs_get_reserved_space(struct inode *inode) +{ + return &F2FS_I(inode)->i_reserved_quota; +} + +static int f2fs_quota_on_mount(struct f2fs_sb_info *sbi, int type) +{ + if (is_set_ckpt_flags(sbi, CP_QUOTA_NEED_FSCK_FLAG)) { + f2fs_err(sbi, "quota sysfile may be corrupted, skip loading it"); + return 0; + } + + return dquot_quota_on_mount(sbi->sb, F2FS_OPTION(sbi).s_qf_names[type], + F2FS_OPTION(sbi).s_jquota_fmt, type); +} + +int f2fs_enable_quota_files(struct f2fs_sb_info *sbi, bool rdonly) +{ + int enabled = 0; + int i, err; + + if (f2fs_sb_has_quota_ino(sbi) && rdonly) { + err = f2fs_enable_quotas(sbi->sb); + if (err) { + f2fs_err(sbi, "Cannot turn on quota_ino: %d", err); + return 0; + } + return 1; + } + + for (i = 0; i < MAXQUOTAS; i++) { + if (F2FS_OPTION(sbi).s_qf_names[i]) { + err = f2fs_quota_on_mount(sbi, i); + if (!err) { + enabled = 1; + continue; + } + f2fs_err(sbi, "Cannot turn on quotas: %d on %d", + err, i); + } + } + return enabled; +} + +static int f2fs_quota_enable(struct super_block *sb, int type, int format_id, + unsigned int flags, unsigned long qf_inum) +{ + struct inode *qf_inode; + unsigned long qf_flag = F2FS_QUOTA_DEFAULT_FL; + int err; + + qf_inode = f2fs_iget(sb, qf_inum); + if (IS_ERR(qf_inode)) { + f2fs_err(F2FS_SB(sb), "Bad quota inode %u:%lu", type, qf_inum); + return PTR_ERR(qf_inode); + } + + /* Don't account quota for quota files to avoid recursion */ + inode_lock(qf_inode); + qf_inode->i_flags |= S_NOQUOTA; + + if ((F2FS_I(qf_inode)->i_flags & qf_flag) != qf_flag) { + F2FS_I(qf_inode)->i_flags |= qf_flag; + f2fs_set_inode_flags(qf_inode); + } + inode_unlock(qf_inode); + + err = dquot_load_quota_inode(qf_inode, type, format_id, flags); + iput(qf_inode); + return err; +} + +static int f2fs_enable_quotas(struct super_block *sb) +{ + struct f2fs_sb_info *sbi = F2FS_SB(sb); + int type, err = 0; + unsigned long qf_inum; + bool quota_mopt[MAXQUOTAS] = { + test_opt(sbi, USRQUOTA), + test_opt(sbi, GRPQUOTA), + test_opt(sbi, PRJQUOTA), + }; + + if (is_set_ckpt_flags(sbi, CP_QUOTA_NEED_FSCK_FLAG)) { + f2fs_err(sbi, "quota file may be corrupted, skip loading it"); + return 0; + } + + sb_dqopt(sb)->flags |= DQUOT_QUOTA_SYS_FILE; + + for (type = 0; type < MAXQUOTAS; type++) { + qf_inum = f2fs_qf_ino(sb, type); + if (qf_inum) { + err = f2fs_quota_enable(sb, type, QFMT_VFS_V1, + DQUOT_USAGE_ENABLED | + (quota_mopt[type] ? DQUOT_LIMITS_ENABLED : 0), qf_inum); + if (err) { + f2fs_err(sbi, "Failed to enable quota tracking (type=%d, err=%d). Please run fsck to fix.", + type, err); + for (type--; type >= 0; type--) + dquot_quota_off(sb, type); + set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR); + return err; + } + } + } + return 0; +} + +static int f2fs_quota_sync_file(struct f2fs_sb_info *sbi, int type) +{ + struct quota_info *dqopt = sb_dqopt(sbi->sb); + struct address_space *mapping = dqopt->files[type]->i_mapping; + int ret = 0; + + ret = dquot_writeback_dquots(sbi->sb, type); + if (ret) + goto out; + + ret = filemap_fdatawrite(mapping); + if (ret) + goto out; + + /* if we are using journalled quota */ + if (is_journalled_quota(sbi)) + goto out; + + ret = filemap_fdatawait(mapping); + + truncate_inode_pages(&dqopt->files[type]->i_data, 0); +out: + if (ret) + set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR); + return ret; +} + +int f2fs_do_quota_sync(struct super_block *sb, int type) +{ + struct f2fs_sb_info *sbi = F2FS_SB(sb); + struct quota_info *dqopt = sb_dqopt(sb); + int cnt; + int ret = 0; + + /* + * Now when everything is written we can discard the pagecache so + * that userspace sees the changes. + */ + for (cnt = 0; cnt < MAXQUOTAS; cnt++) { + struct f2fs_lock_context lc; + + if (type != -1 && cnt != type) + continue; + + if (!sb_has_quota_active(sb, cnt)) + continue; + + if (!f2fs_sb_has_quota_ino(sbi)) + inode_lock(dqopt->files[cnt]); + + /* + * do_quotactl + * f2fs_quota_sync + * f2fs_down_read(quota_sem) + * dquot_writeback_dquots() + * f2fs_dquot_commit + * block_operation + * f2fs_down_read(quota_sem) + */ + f2fs_lock_op(sbi, &lc); + f2fs_down_read(&sbi->quota_sem); + + ret = f2fs_quota_sync_file(sbi, cnt); + + f2fs_up_read(&sbi->quota_sem); + f2fs_unlock_op(sbi, &lc); + + if (!f2fs_sb_has_quota_ino(sbi)) + inode_unlock(dqopt->files[cnt]); + + if (ret) + break; + } + return ret; +} + +static int f2fs_quota_sync(struct super_block *sb, int type) +{ + int ret; + + F2FS_SB(sb)->umount_lock_holder = current; + ret = f2fs_do_quota_sync(sb, type); + F2FS_SB(sb)->umount_lock_holder = NULL; + return ret; +} + +static int f2fs_quota_on(struct super_block *sb, int type, int format_id, + const struct path *path) +{ + struct inode *inode; + int err = 0; + + /* if quota sysfile exists, deny enabling quota with specific file */ + if (f2fs_sb_has_quota_ino(F2FS_SB(sb))) { + f2fs_err(F2FS_SB(sb), "quota sysfile already exists"); + return -EBUSY; + } + + if (path->dentry->d_sb != sb) + return -EXDEV; + + F2FS_SB(sb)->umount_lock_holder = current; + + err = f2fs_do_quota_sync(sb, type); + if (err) + goto out; + + inode = d_inode(path->dentry); + + err = filemap_fdatawrite(inode->i_mapping); + if (err) + goto out; + + err = filemap_fdatawait(inode->i_mapping); + if (err) + goto out; + + err = dquot_quota_on(sb, type, format_id, path); + if (err) + goto out; + + inode_lock(inode); + F2FS_I(inode)->i_flags |= F2FS_QUOTA_DEFAULT_FL; + f2fs_set_inode_flags(inode); + inode_unlock(inode); + f2fs_mark_inode_dirty_sync(inode, false); +out: + F2FS_SB(sb)->umount_lock_holder = NULL; + return err; +} + +static int __f2fs_quota_off(struct super_block *sb, int type) +{ + struct inode *inode = sb_dqopt(sb)->files[type]; + int err; + + if (!inode || !igrab(inode)) + return dquot_quota_off(sb, type); + + err = f2fs_do_quota_sync(sb, type); + if (err) + goto out_put; + + err = dquot_quota_off(sb, type); + if (err || f2fs_sb_has_quota_ino(F2FS_SB(sb))) + goto out_put; + + inode_lock(inode); + F2FS_I(inode)->i_flags &= ~F2FS_QUOTA_DEFAULT_FL; + f2fs_set_inode_flags(inode); + inode_unlock(inode); + f2fs_mark_inode_dirty_sync(inode, false); +out_put: + iput(inode); + return err; +} + +static int f2fs_quota_off(struct super_block *sb, int type) +{ + struct f2fs_sb_info *sbi = F2FS_SB(sb); + int err; + + F2FS_SB(sb)->umount_lock_holder = current; + + err = __f2fs_quota_off(sb, type); + + /* + * quotactl can shutdown journalled quota, result in inconsistence + * between quota record and fs data by following updates, tag the + * flag to let fsck be aware of it. + */ + if (is_journalled_quota(sbi)) + set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR); + + F2FS_SB(sb)->umount_lock_holder = NULL; + + return err; +} + +void f2fs_quota_off_umount(struct super_block *sb) +{ + int type; + int err; + + for (type = 0; type < MAXQUOTAS; type++) { + err = __f2fs_quota_off(sb, type); + if (err) { + int ret = dquot_quota_off(sb, type); + + f2fs_err(F2FS_SB(sb), "Fail to turn off disk quota (type: %d, err: %d, ret:%d), Please run fsck to fix it.", + type, err, ret); + set_sbi_flag(F2FS_SB(sb), SBI_QUOTA_NEED_REPAIR); + } + } + /* + * In case of checkpoint=disable, we must flush quota blocks. + * This can cause NULL exception for node_inode in end_io, since + * put_super already dropped it. + */ + sync_filesystem(sb); +} + +static void f2fs_truncate_quota_inode_pages(struct super_block *sb) +{ + struct quota_info *dqopt = sb_dqopt(sb); + int type; + + for (type = 0; type < MAXQUOTAS; type++) { + if (!dqopt->files[type]) + continue; + f2fs_inode_synced(dqopt->files[type]); + } +} + +static int f2fs_dquot_commit(struct dquot *dquot) +{ + struct f2fs_sb_info *sbi = F2FS_SB(dquot->dq_sb); + int ret; + + f2fs_down_read_nested(&sbi->quota_sem, SINGLE_DEPTH_NESTING); + ret = dquot_commit(dquot); + if (ret < 0) + set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR); + f2fs_up_read(&sbi->quota_sem); + return ret; +} + +static int f2fs_dquot_acquire(struct dquot *dquot) +{ + struct f2fs_sb_info *sbi = F2FS_SB(dquot->dq_sb); + int ret; + + f2fs_down_read(&sbi->quota_sem); + ret = dquot_acquire(dquot); + if (ret < 0) + set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR); + f2fs_up_read(&sbi->quota_sem); + return ret; +} + +static int f2fs_dquot_release(struct dquot *dquot) +{ + struct f2fs_sb_info *sbi = F2FS_SB(dquot->dq_sb); + int ret = dquot_release(dquot); + + if (ret < 0) + set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR); + return ret; +} + +static int f2fs_dquot_mark_dquot_dirty(struct dquot *dquot) +{ + struct super_block *sb = dquot->dq_sb; + struct f2fs_sb_info *sbi = F2FS_SB(sb); + int ret = dquot_mark_dquot_dirty(dquot); + + /* if we are using journalled quota */ + if (is_journalled_quota(sbi)) + set_sbi_flag(sbi, SBI_QUOTA_NEED_FLUSH); + + return ret; +} + +static int f2fs_dquot_commit_info(struct super_block *sb, int type) +{ + struct f2fs_sb_info *sbi = F2FS_SB(sb); + int ret = dquot_commit_info(sb, type); + + if (ret < 0) + set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR); + return ret; +} + +static int f2fs_get_projid(struct inode *inode, kprojid_t *projid) +{ + *projid = F2FS_I(inode)->i_projid; + return 0; +} + +static const struct dquot_operations f2fs_quota_operations = { + .get_reserved_space = f2fs_get_reserved_space, + .write_dquot = f2fs_dquot_commit, + .acquire_dquot = f2fs_dquot_acquire, + .release_dquot = f2fs_dquot_release, + .mark_dirty = f2fs_dquot_mark_dquot_dirty, + .write_info = f2fs_dquot_commit_info, + .alloc_dquot = dquot_alloc, + .destroy_dquot = dquot_destroy, + .get_projid = f2fs_get_projid, + .get_next_id = dquot_get_next_id, +}; + +static const struct quotactl_ops f2fs_quotactl_ops = { + .quota_on = f2fs_quota_on, + .quota_off = f2fs_quota_off, + .quota_sync = f2fs_quota_sync, + .get_state = dquot_get_state, + .set_info = dquot_set_dqinfo, + .get_dqblk = dquot_get_dqblk, + .set_dqblk = dquot_set_dqblk, + .get_nextdqblk = dquot_get_next_dqblk, +}; +#else +int f2fs_dquot_initialize(struct inode *inode) +{ + return 0; +} + +int f2fs_do_quota_sync(struct super_block *sb, int type) +{ + return 0; +} + +void f2fs_quota_off_umount(struct super_block *sb) +{ +} +#endif + +static const struct super_operations f2fs_sops = { + .alloc_inode = f2fs_alloc_inode, + .free_inode = f2fs_free_inode, + .drop_inode = f2fs_drop_inode, + .write_inode = f2fs_write_inode, + .dirty_inode = f2fs_dirty_inode, + .show_options = f2fs_show_options, +#ifdef CONFIG_QUOTA + .quota_read = f2fs_quota_read, + .quota_write = f2fs_quota_write, + .get_dquots = f2fs_get_dquots, +#endif + .evict_inode = f2fs_evict_inode, + .put_super = f2fs_put_super, + .sync_fs = f2fs_sync_fs, + .freeze_fs = f2fs_freeze, + .unfreeze_fs = f2fs_unfreeze, + .statfs = f2fs_statfs, + .shutdown = f2fs_shutdown, +}; + +#ifdef CONFIG_FS_ENCRYPTION +static int f2fs_get_context(struct inode *inode, void *ctx, size_t len) +{ + return f2fs_getxattr(inode, F2FS_XATTR_INDEX_ENCRYPTION, + F2FS_XATTR_NAME_ENCRYPTION_CONTEXT, + ctx, len, NULL); +} + +static int f2fs_set_context(struct inode *inode, const void *ctx, size_t len, + void *fs_data) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + + /* + * Encrypting the root directory is not allowed because fsck + * expects lost+found directory to exist and remain unencrypted + * if LOST_FOUND feature is enabled. + * + */ + if (f2fs_sb_has_lost_found(sbi) && + inode->i_ino == F2FS_ROOT_INO(sbi)) + return -EPERM; + + return f2fs_setxattr(inode, F2FS_XATTR_INDEX_ENCRYPTION, + F2FS_XATTR_NAME_ENCRYPTION_CONTEXT, + ctx, len, fs_data, XATTR_CREATE); +} + +static const union fscrypt_policy *f2fs_get_dummy_policy(struct super_block *sb) +{ + return F2FS_OPTION(F2FS_SB(sb)).dummy_enc_policy.policy; +} + +static bool f2fs_has_stable_inodes(struct super_block *sb) +{ + return true; +} + +static unsigned int +f2fs_get_devices(struct super_block *sb, + struct block_device *devs[FSCRYPT_MAX_DEVICES]) +{ + struct f2fs_sb_info *sbi = F2FS_SB(sb); + int ndevs; + int i; + + static_assert(MAX_DEVICES <= FSCRYPT_MAX_DEVICES); + + if (!f2fs_is_multi_device(sbi)) { + devs[0] = sb->s_bdev; + return 1; + } + ndevs = sbi->s_ndevs; + if (WARN_ON_ONCE(ndevs > FSCRYPT_MAX_DEVICES)) + ndevs = FSCRYPT_MAX_DEVICES; + + for (i = 0; i < ndevs; i++) + devs[i] = FDEV(i).bdev; + return ndevs; +} + +static const struct fscrypt_operations f2fs_cryptops = { + .inode_info_offs = (int)offsetof(struct f2fs_inode_info, i_crypt_info) - + (int)offsetof(struct f2fs_inode_info, vfs_inode), + .is_block_based = 1, + .has_32bit_inodes = 1, + .supports_subblock_data_units = 1, + .legacy_key_prefix = "f2fs:", + .get_context = f2fs_get_context, + .set_context = f2fs_set_context, + .get_dummy_policy = f2fs_get_dummy_policy, + .empty_dir = f2fs_empty_dir, + .has_stable_inodes = f2fs_has_stable_inodes, + .get_devices = f2fs_get_devices, +}; +#endif /* CONFIG_FS_ENCRYPTION */ + +static struct inode *f2fs_nfs_get_inode(struct super_block *sb, + u64 ino, u32 generation) +{ + struct f2fs_sb_info *sbi = F2FS_SB(sb); + struct inode *inode; + + if (f2fs_check_nid_range(sbi, ino)) + return ERR_PTR(-ESTALE); + + /* + * f2fs_iget isn't quite right if the inode is currently unallocated! + * However f2fs_iget currently does appropriate checks to handle stale + * inodes so everything is OK. + */ + inode = f2fs_iget(sb, ino); + if (IS_ERR(inode)) + return ERR_CAST(inode); + if (unlikely(generation && inode->i_generation != generation)) { + /* we didn't find the right inode.. */ + iput(inode); + return ERR_PTR(-ESTALE); + } + return inode; +} + +static struct dentry *f2fs_fh_to_dentry(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + return generic_fh_to_dentry(sb, fid, fh_len, fh_type, + f2fs_nfs_get_inode); +} + +static struct dentry *f2fs_fh_to_parent(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + return generic_fh_to_parent(sb, fid, fh_len, fh_type, + f2fs_nfs_get_inode); +} + +static const struct export_operations f2fs_export_ops = { + .encode_fh = generic_encode_ino32_fh, + .fh_to_dentry = f2fs_fh_to_dentry, + .fh_to_parent = f2fs_fh_to_parent, + .get_parent = f2fs_get_parent, +}; + +loff_t max_file_blocks(struct inode *inode) +{ + loff_t result = 0; + loff_t leaf_count; + + /* + * note: previously, result is equal to (DEF_ADDRS_PER_INODE - + * DEFAULT_INLINE_XATTR_ADDRS), but now f2fs try to reserve more + * space in inode.i_addr, it will be more safe to reassign + * result as zero. + */ + + if (inode && f2fs_compressed_file(inode)) + leaf_count = ADDRS_PER_BLOCK(inode); + else + leaf_count = DEF_ADDRS_PER_BLOCK; + + /* two direct node blocks */ + result += (leaf_count * 2); + + /* two indirect node blocks */ + leaf_count *= NIDS_PER_BLOCK; + result += (leaf_count * 2); + + /* one double indirect node block */ + leaf_count *= NIDS_PER_BLOCK; + result += leaf_count; + + /* + * For compatibility with FSCRYPT_POLICY_FLAG_IV_INO_LBLK_{64,32} with + * a 4K crypto data unit, we must restrict the max filesize to what can + * fit within U32_MAX + 1 data units. + */ + + result = umin(result, F2FS_BYTES_TO_BLK(((loff_t)U32_MAX + 1) * 4096)); + + return result; +} + +static int __f2fs_commit_super(struct f2fs_sb_info *sbi, struct folio *folio, + pgoff_t index, bool update) +{ + struct bio *bio; + /* it's rare case, we can do fua all the time */ + blk_opf_t opf = REQ_OP_WRITE | REQ_SYNC | REQ_PREFLUSH | REQ_FUA; + int ret; + + folio_lock(folio); + folio_wait_writeback(folio); + if (update) + memcpy(F2FS_SUPER_BLOCK(folio, index), F2FS_RAW_SUPER(sbi), + sizeof(struct f2fs_super_block)); + folio_mark_dirty(folio); + folio_clear_dirty_for_io(folio); + folio_start_writeback(folio); + folio_unlock(folio); + + bio = bio_alloc(sbi->sb->s_bdev, 1, opf, GFP_NOFS); + + /* it doesn't need to set crypto context for superblock update */ + bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(folio->index); + + if (!bio_add_folio(bio, folio, folio_size(folio), 0)) + f2fs_bug_on(sbi, 1); + + ret = submit_bio_wait(bio); + bio_put(bio); + folio_end_writeback(folio); + + return ret; +} + +static inline bool sanity_check_area_boundary(struct f2fs_sb_info *sbi, + struct folio *folio, pgoff_t index) +{ + struct f2fs_super_block *raw_super = F2FS_SUPER_BLOCK(folio, index); + struct super_block *sb = sbi->sb; + u32 segment0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr); + u32 cp_blkaddr = le32_to_cpu(raw_super->cp_blkaddr); + u32 sit_blkaddr = le32_to_cpu(raw_super->sit_blkaddr); + u32 nat_blkaddr = le32_to_cpu(raw_super->nat_blkaddr); + u32 ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr); + u32 main_blkaddr = le32_to_cpu(raw_super->main_blkaddr); + u32 segment_count_ckpt = le32_to_cpu(raw_super->segment_count_ckpt); + u32 segment_count_sit = le32_to_cpu(raw_super->segment_count_sit); + u32 segment_count_nat = le32_to_cpu(raw_super->segment_count_nat); + u32 segment_count_ssa = le32_to_cpu(raw_super->segment_count_ssa); + u32 segment_count_main = le32_to_cpu(raw_super->segment_count_main); + u32 segment_count = le32_to_cpu(raw_super->segment_count); + u32 log_blocks_per_seg = le32_to_cpu(raw_super->log_blocks_per_seg); + u64 main_end_blkaddr = main_blkaddr + + ((u64)segment_count_main << log_blocks_per_seg); + u64 seg_end_blkaddr = segment0_blkaddr + + ((u64)segment_count << log_blocks_per_seg); + + if (segment0_blkaddr != cp_blkaddr) { + f2fs_info(sbi, "Mismatch start address, segment0(%u) cp_blkaddr(%u)", + segment0_blkaddr, cp_blkaddr); + return true; + } + + if (cp_blkaddr + (segment_count_ckpt << log_blocks_per_seg) != + sit_blkaddr) { + f2fs_info(sbi, "Wrong CP boundary, start(%u) end(%u) blocks(%u)", + cp_blkaddr, sit_blkaddr, + segment_count_ckpt << log_blocks_per_seg); + return true; + } + + if (sit_blkaddr + (segment_count_sit << log_blocks_per_seg) != + nat_blkaddr) { + f2fs_info(sbi, "Wrong SIT boundary, start(%u) end(%u) blocks(%u)", + sit_blkaddr, nat_blkaddr, + segment_count_sit << log_blocks_per_seg); + return true; + } + + if (nat_blkaddr + (segment_count_nat << log_blocks_per_seg) != + ssa_blkaddr) { + f2fs_info(sbi, "Wrong NAT boundary, start(%u) end(%u) blocks(%u)", + nat_blkaddr, ssa_blkaddr, + segment_count_nat << log_blocks_per_seg); + return true; + } + + if (ssa_blkaddr + (segment_count_ssa << log_blocks_per_seg) != + main_blkaddr) { + f2fs_info(sbi, "Wrong SSA boundary, start(%u) end(%u) blocks(%u)", + ssa_blkaddr, main_blkaddr, + segment_count_ssa << log_blocks_per_seg); + return true; + } + + if (main_end_blkaddr > seg_end_blkaddr) { + f2fs_info(sbi, "Wrong MAIN_AREA boundary, start(%u) end(%llu) block(%u)", + main_blkaddr, seg_end_blkaddr, + segment_count_main << log_blocks_per_seg); + return true; + } else if (main_end_blkaddr < seg_end_blkaddr) { + int err = 0; + char *res; + + /* fix in-memory information all the time */ + raw_super->segment_count = cpu_to_le32((main_end_blkaddr - + segment0_blkaddr) >> log_blocks_per_seg); + + if (f2fs_readonly(sb) || f2fs_hw_is_readonly(sbi)) { + set_sbi_flag(sbi, SBI_NEED_SB_WRITE); + res = "internally"; + } else { + err = __f2fs_commit_super(sbi, folio, index, false); + res = err ? "failed" : "done"; + } + f2fs_info(sbi, "Fix alignment : %s, start(%u) end(%llu) block(%u)", + res, main_blkaddr, seg_end_blkaddr, + segment_count_main << log_blocks_per_seg); + if (err) + return true; + } + return false; +} + +static int sanity_check_raw_super(struct f2fs_sb_info *sbi, + struct folio *folio, pgoff_t index) +{ + block_t segment_count, segs_per_sec, secs_per_zone, segment_count_main; + block_t total_sections, blocks_per_seg; + struct f2fs_super_block *raw_super = F2FS_SUPER_BLOCK(folio, index); + size_t crc_offset = 0; + __u32 crc = 0; + + if (le32_to_cpu(raw_super->magic) != F2FS_SUPER_MAGIC) { + f2fs_info(sbi, "Magic Mismatch, valid(0x%x) - read(0x%x)", + F2FS_SUPER_MAGIC, le32_to_cpu(raw_super->magic)); + return -EINVAL; + } + + /* Check checksum_offset and crc in superblock */ + if (__F2FS_HAS_FEATURE(raw_super, F2FS_FEATURE_SB_CHKSUM)) { + crc_offset = le32_to_cpu(raw_super->checksum_offset); + if (crc_offset != + offsetof(struct f2fs_super_block, crc)) { + f2fs_info(sbi, "Invalid SB checksum offset: %zu", + crc_offset); + return -EFSCORRUPTED; + } + crc = le32_to_cpu(raw_super->crc); + if (crc != f2fs_crc32(raw_super, crc_offset)) { + f2fs_info(sbi, "Invalid SB checksum value: %u", crc); + return -EFSCORRUPTED; + } + } + + /* only support block_size equals to PAGE_SIZE */ + if (le32_to_cpu(raw_super->log_blocksize) != F2FS_BLKSIZE_BITS) { + f2fs_info(sbi, "Invalid log_blocksize (%u), supports only %u", + le32_to_cpu(raw_super->log_blocksize), + F2FS_BLKSIZE_BITS); + return -EFSCORRUPTED; + } + + /* check log blocks per segment */ + if (le32_to_cpu(raw_super->log_blocks_per_seg) != 9) { + f2fs_info(sbi, "Invalid log blocks per segment (%u)", + le32_to_cpu(raw_super->log_blocks_per_seg)); + return -EFSCORRUPTED; + } + + /* Currently, support 512/1024/2048/4096/16K bytes sector size */ + if (le32_to_cpu(raw_super->log_sectorsize) > + F2FS_MAX_LOG_SECTOR_SIZE || + le32_to_cpu(raw_super->log_sectorsize) < + F2FS_MIN_LOG_SECTOR_SIZE) { + f2fs_info(sbi, "Invalid log sectorsize (%u)", + le32_to_cpu(raw_super->log_sectorsize)); + return -EFSCORRUPTED; + } + if (le32_to_cpu(raw_super->log_sectors_per_block) + + le32_to_cpu(raw_super->log_sectorsize) != + F2FS_MAX_LOG_SECTOR_SIZE) { + f2fs_info(sbi, "Invalid log sectors per block(%u) log sectorsize(%u)", + le32_to_cpu(raw_super->log_sectors_per_block), + le32_to_cpu(raw_super->log_sectorsize)); + return -EFSCORRUPTED; + } + + segment_count = le32_to_cpu(raw_super->segment_count); + segment_count_main = le32_to_cpu(raw_super->segment_count_main); + segs_per_sec = le32_to_cpu(raw_super->segs_per_sec); + secs_per_zone = le32_to_cpu(raw_super->secs_per_zone); + total_sections = le32_to_cpu(raw_super->section_count); + + /* blocks_per_seg should be 512, given the above check */ + blocks_per_seg = BIT(le32_to_cpu(raw_super->log_blocks_per_seg)); + + if (segment_count > F2FS_MAX_SEGMENT || + segment_count < F2FS_MIN_SEGMENTS) { + f2fs_info(sbi, "Invalid segment count (%u)", segment_count); + return -EFSCORRUPTED; + } + + if (total_sections > segment_count_main || total_sections < 1 || + segs_per_sec > segment_count || !segs_per_sec) { + f2fs_info(sbi, "Invalid segment/section count (%u, %u x %u)", + segment_count, total_sections, segs_per_sec); + return -EFSCORRUPTED; + } + + if (segment_count_main != total_sections * segs_per_sec) { + f2fs_info(sbi, "Invalid segment/section count (%u != %u * %u)", + segment_count_main, total_sections, segs_per_sec); + return -EFSCORRUPTED; + } + + if ((segment_count / segs_per_sec) < total_sections) { + f2fs_info(sbi, "Small segment_count (%u < %u * %u)", + segment_count, segs_per_sec, total_sections); + return -EFSCORRUPTED; + } + + if (segment_count > (le64_to_cpu(raw_super->block_count) >> 9)) { + f2fs_info(sbi, "Wrong segment_count / block_count (%u > %llu)", + segment_count, le64_to_cpu(raw_super->block_count)); + return -EFSCORRUPTED; + } + + if (RDEV(0).path[0]) { + block_t dev_seg_count = le32_to_cpu(RDEV(0).total_segments); + int i = 1; + + while (i < MAX_DEVICES && RDEV(i).path[0]) { + dev_seg_count += le32_to_cpu(RDEV(i).total_segments); + i++; + } + if (segment_count != dev_seg_count) { + f2fs_info(sbi, "Segment count (%u) mismatch with total segments from devices (%u)", + segment_count, dev_seg_count); + return -EFSCORRUPTED; + } + } else { + if (__F2FS_HAS_FEATURE(raw_super, F2FS_FEATURE_BLKZONED) && + !bdev_is_zoned(sbi->sb->s_bdev)) { + f2fs_info(sbi, "Zoned block device path is missing"); + return -EFSCORRUPTED; + } + } + + if (secs_per_zone > total_sections || !secs_per_zone) { + f2fs_info(sbi, "Wrong secs_per_zone / total_sections (%u, %u)", + secs_per_zone, total_sections); + return -EFSCORRUPTED; + } + if (le32_to_cpu(raw_super->extension_count) > F2FS_MAX_EXTENSION || + raw_super->hot_ext_count > F2FS_MAX_EXTENSION || + (le32_to_cpu(raw_super->extension_count) + + raw_super->hot_ext_count) > F2FS_MAX_EXTENSION) { + f2fs_info(sbi, "Corrupted extension count (%u + %u > %u)", + le32_to_cpu(raw_super->extension_count), + raw_super->hot_ext_count, + F2FS_MAX_EXTENSION); + return -EFSCORRUPTED; + } + + if (le32_to_cpu(raw_super->cp_payload) >= + (blocks_per_seg - F2FS_CP_PACKS - + NR_CURSEG_PERSIST_TYPE)) { + f2fs_info(sbi, "Insane cp_payload (%u >= %u)", + le32_to_cpu(raw_super->cp_payload), + blocks_per_seg - F2FS_CP_PACKS - + NR_CURSEG_PERSIST_TYPE); + return -EFSCORRUPTED; + } + + /* check reserved ino info */ + if (le32_to_cpu(raw_super->node_ino) != 1 || + le32_to_cpu(raw_super->meta_ino) != 2 || + le32_to_cpu(raw_super->root_ino) != 3) { + f2fs_info(sbi, "Invalid Fs Meta Ino: node(%u) meta(%u) root(%u)", + le32_to_cpu(raw_super->node_ino), + le32_to_cpu(raw_super->meta_ino), + le32_to_cpu(raw_super->root_ino)); + return -EFSCORRUPTED; + } + + /* check CP/SIT/NAT/SSA/MAIN_AREA area boundary */ + if (sanity_check_area_boundary(sbi, folio, index)) + return -EFSCORRUPTED; + + return 0; +} + +int f2fs_sanity_check_ckpt(struct f2fs_sb_info *sbi) +{ + unsigned int total, fsmeta; + struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); + struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); + unsigned int ovp_segments, reserved_segments; + unsigned int main_segs, blocks_per_seg; + unsigned int sit_segs, nat_segs; + unsigned int sit_bitmap_size, nat_bitmap_size; + unsigned int log_blocks_per_seg; + unsigned int segment_count_main; + unsigned int cp_pack_start_sum, cp_payload; + block_t user_block_count, valid_user_blocks; + block_t avail_node_count, valid_node_count; + unsigned int nat_blocks, nat_bits_bytes, nat_bits_blocks; + unsigned int sit_blk_cnt; + int i, j; + + total = le32_to_cpu(raw_super->segment_count); + fsmeta = le32_to_cpu(raw_super->segment_count_ckpt); + sit_segs = le32_to_cpu(raw_super->segment_count_sit); + fsmeta += sit_segs; + nat_segs = le32_to_cpu(raw_super->segment_count_nat); + fsmeta += nat_segs; + fsmeta += le32_to_cpu(ckpt->rsvd_segment_count); + fsmeta += le32_to_cpu(raw_super->segment_count_ssa); + + if (unlikely(fsmeta >= total)) + return 1; + + ovp_segments = le32_to_cpu(ckpt->overprov_segment_count); + reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count); + + if (!f2fs_sb_has_readonly(sbi) && + unlikely(fsmeta < F2FS_MIN_META_SEGMENTS || + ovp_segments == 0 || reserved_segments == 0)) { + f2fs_err(sbi, "Wrong layout: check mkfs.f2fs version"); + return 1; + } + user_block_count = le64_to_cpu(ckpt->user_block_count); + segment_count_main = le32_to_cpu(raw_super->segment_count_main) + + (f2fs_sb_has_readonly(sbi) ? 1 : 0); + log_blocks_per_seg = le32_to_cpu(raw_super->log_blocks_per_seg); + if (!user_block_count || user_block_count >= + segment_count_main << log_blocks_per_seg) { + f2fs_err(sbi, "Wrong user_block_count: %u", + user_block_count); + return 1; + } + + valid_user_blocks = le64_to_cpu(ckpt->valid_block_count); + if (valid_user_blocks > user_block_count) { + f2fs_err(sbi, "Wrong valid_user_blocks: %u, user_block_count: %u", + valid_user_blocks, user_block_count); + return 1; + } + + valid_node_count = le32_to_cpu(ckpt->valid_node_count); + avail_node_count = sbi->total_node_count - F2FS_RESERVED_NODE_NUM; + if (valid_node_count > avail_node_count) { + f2fs_err(sbi, "Wrong valid_node_count: %u, avail_node_count: %u", + valid_node_count, avail_node_count); + return 1; + } + + main_segs = le32_to_cpu(raw_super->segment_count_main); + blocks_per_seg = BLKS_PER_SEG(sbi); + + for (i = 0; i < NR_CURSEG_NODE_TYPE; i++) { + if (le32_to_cpu(ckpt->cur_node_segno[i]) >= main_segs || + le16_to_cpu(ckpt->cur_node_blkoff[i]) >= blocks_per_seg) + return 1; + + if (f2fs_sb_has_readonly(sbi)) + goto check_data; + + for (j = i + 1; j < NR_CURSEG_NODE_TYPE; j++) { + if (le32_to_cpu(ckpt->cur_node_segno[i]) == + le32_to_cpu(ckpt->cur_node_segno[j])) { + f2fs_err(sbi, "Node segment (%u, %u) has the same segno: %u", + i, j, + le32_to_cpu(ckpt->cur_node_segno[i])); + return 1; + } + } + } +check_data: + for (i = 0; i < NR_CURSEG_DATA_TYPE; i++) { + if (le32_to_cpu(ckpt->cur_data_segno[i]) >= main_segs || + le16_to_cpu(ckpt->cur_data_blkoff[i]) >= blocks_per_seg) + return 1; + + if (f2fs_sb_has_readonly(sbi)) + goto skip_cross; + + for (j = i + 1; j < NR_CURSEG_DATA_TYPE; j++) { + if (le32_to_cpu(ckpt->cur_data_segno[i]) == + le32_to_cpu(ckpt->cur_data_segno[j])) { + f2fs_err(sbi, "Data segment (%u, %u) has the same segno: %u", + i, j, + le32_to_cpu(ckpt->cur_data_segno[i])); + return 1; + } + } + } + for (i = 0; i < NR_CURSEG_NODE_TYPE; i++) { + for (j = 0; j < NR_CURSEG_DATA_TYPE; j++) { + if (le32_to_cpu(ckpt->cur_node_segno[i]) == + le32_to_cpu(ckpt->cur_data_segno[j])) { + f2fs_err(sbi, "Node segment (%u) and Data segment (%u) has the same segno: %u", + i, j, + le32_to_cpu(ckpt->cur_node_segno[i])); + return 1; + } + } + } +skip_cross: + sit_bitmap_size = le32_to_cpu(ckpt->sit_ver_bitmap_bytesize); + nat_bitmap_size = le32_to_cpu(ckpt->nat_ver_bitmap_bytesize); + + if (sit_bitmap_size != ((sit_segs / 2) << log_blocks_per_seg) / 8 || + nat_bitmap_size != ((nat_segs / 2) << log_blocks_per_seg) / 8) { + f2fs_err(sbi, "Wrong bitmap size: sit: %u, nat:%u", + sit_bitmap_size, nat_bitmap_size); + return 1; + } + + sit_blk_cnt = DIV_ROUND_UP(main_segs, SIT_ENTRY_PER_BLOCK); + if (sit_bitmap_size * 8 < sit_blk_cnt) { + f2fs_err(sbi, "Wrong bitmap size: sit: %u, sit_blk_cnt:%u", + sit_bitmap_size, sit_blk_cnt); + return 1; + } + + cp_pack_start_sum = __start_sum_addr(sbi); + cp_payload = __cp_payload(sbi); + if (cp_pack_start_sum < cp_payload + 1 || + cp_pack_start_sum > blocks_per_seg - 1 - + NR_CURSEG_PERSIST_TYPE) { + f2fs_err(sbi, "Wrong cp_pack_start_sum: %u", + cp_pack_start_sum); + return 1; + } + + if (__is_set_ckpt_flags(ckpt, CP_LARGE_NAT_BITMAP_FLAG) && + le32_to_cpu(ckpt->checksum_offset) != CP_MIN_CHKSUM_OFFSET) { + f2fs_warn(sbi, "using deprecated layout of large_nat_bitmap, " + "please run fsck v1.13.0 or higher to repair, chksum_offset: %u, " + "fixed with patch: \"f2fs-tools: relocate chksum_offset for large_nat_bitmap feature\"", + le32_to_cpu(ckpt->checksum_offset)); + return 1; + } + + nat_blocks = nat_segs << log_blocks_per_seg; + nat_bits_bytes = nat_blocks / BITS_PER_BYTE; + nat_bits_blocks = F2FS_BLK_ALIGN((nat_bits_bytes << 1) + 8); + if (__is_set_ckpt_flags(ckpt, CP_NAT_BITS_FLAG) && + (cp_payload + F2FS_CP_PACKS + + NR_CURSEG_PERSIST_TYPE + nat_bits_blocks >= blocks_per_seg)) { + f2fs_warn(sbi, "Insane cp_payload: %u, nat_bits_blocks: %u)", + cp_payload, nat_bits_blocks); + return 1; + } + + if (unlikely(f2fs_cp_error(sbi))) { + f2fs_err(sbi, "A bug case: need to run fsck"); + return 1; + } + return 0; +} + +static void init_sb_info(struct f2fs_sb_info *sbi) +{ + struct f2fs_super_block *raw_super = sbi->raw_super; + int i; + + sbi->log_sectors_per_block = + le32_to_cpu(raw_super->log_sectors_per_block); + sbi->log_blocksize = le32_to_cpu(raw_super->log_blocksize); + sbi->blocksize = BIT(sbi->log_blocksize); + sbi->log_blocks_per_seg = le32_to_cpu(raw_super->log_blocks_per_seg); + sbi->blocks_per_seg = BIT(sbi->log_blocks_per_seg); + sbi->segs_per_sec = le32_to_cpu(raw_super->segs_per_sec); + sbi->secs_per_zone = le32_to_cpu(raw_super->secs_per_zone); + sbi->total_sections = le32_to_cpu(raw_super->section_count); + sbi->total_node_count = SEGS_TO_BLKS(sbi, + ((le32_to_cpu(raw_super->segment_count_nat) / 2) * + NAT_ENTRY_PER_BLOCK)); + sbi->allocate_section_hint = le32_to_cpu(raw_super->section_count); + sbi->allocate_section_policy = ALLOCATE_FORWARD_NOHINT; + F2FS_ROOT_INO(sbi) = le32_to_cpu(raw_super->root_ino); + F2FS_NODE_INO(sbi) = le32_to_cpu(raw_super->node_ino); + F2FS_META_INO(sbi) = le32_to_cpu(raw_super->meta_ino); + sbi->cur_victim_sec = NULL_SECNO; + sbi->gc_mode = GC_NORMAL; + sbi->next_victim_seg[BG_GC] = NULL_SEGNO; + sbi->next_victim_seg[FG_GC] = NULL_SEGNO; + sbi->max_victim_search = DEF_MAX_VICTIM_SEARCH; + sbi->migration_granularity = SEGS_PER_SEC(sbi); + sbi->migration_window_granularity = f2fs_sb_has_blkzoned(sbi) ? + DEF_MIGRATION_WINDOW_GRANULARITY_ZONED : SEGS_PER_SEC(sbi); + sbi->seq_file_ra_mul = MIN_RA_MUL; + sbi->max_fragment_chunk = DEF_FRAGMENT_SIZE; + sbi->max_fragment_hole = DEF_FRAGMENT_SIZE; + spin_lock_init(&sbi->gc_remaining_trials_lock); + atomic64_set(&sbi->current_atomic_write, 0); + sbi->max_lock_elapsed_time = MAX_LOCK_ELAPSED_TIME; + sbi->adjust_lock_priority = 0; + sbi->lock_duration_priority = F2FS_DEFAULT_TASK_PRIORITY; + sbi->critical_task_priority = F2FS_CRITICAL_TASK_PRIORITY; + + sbi->sum_blocksize = f2fs_sb_has_packed_ssa(sbi) ? + 4096 : sbi->blocksize; + sbi->sums_per_block = sbi->blocksize / sbi->sum_blocksize; + sbi->entries_in_sum = sbi->sum_blocksize / 8; + sbi->sum_entry_size = SUMMARY_SIZE * sbi->entries_in_sum; + sbi->sum_journal_size = sbi->sum_blocksize - SUM_FOOTER_SIZE - + sbi->sum_entry_size; + sbi->nat_journal_entries = (sbi->sum_journal_size - 2) / + sizeof(struct nat_journal_entry); + sbi->sit_journal_entries = (sbi->sum_journal_size - 2) / + sizeof(struct sit_journal_entry); + + sbi->dir_level = DEF_DIR_LEVEL; + sbi->interval_time[CP_TIME] = DEF_CP_INTERVAL; + sbi->interval_time[REQ_TIME] = DEF_IDLE_INTERVAL; + sbi->interval_time[DISCARD_TIME] = DEF_IDLE_INTERVAL; + sbi->interval_time[GC_TIME] = DEF_IDLE_INTERVAL; + sbi->interval_time[DISABLE_TIME] = DEF_DISABLE_INTERVAL; + sbi->interval_time[UMOUNT_DISCARD_TIMEOUT] = + DEF_UMOUNT_DISCARD_TIMEOUT; + clear_sbi_flag(sbi, SBI_NEED_FSCK); + + for (i = 0; i < NR_COUNT_TYPE; i++) + atomic_set(&sbi->nr_pages[i], 0); + + for (i = 0; i < META; i++) + atomic_set(&sbi->wb_sync_req[i], 0); + + INIT_LIST_HEAD(&sbi->s_list); + mutex_init(&sbi->umount_mutex); + init_f2fs_rwsem(&sbi->io_order_lock); + spin_lock_init(&sbi->cp_lock); + + sbi->dirty_device = 0; + spin_lock_init(&sbi->dev_lock); + + init_f2fs_rwsem(&sbi->sb_lock); + init_f2fs_rwsem(&sbi->pin_sem); +} + +static int init_percpu_info(struct f2fs_sb_info *sbi) +{ + int err; + + err = percpu_counter_init(&sbi->alloc_valid_block_count, 0, GFP_KERNEL); + if (err) + return err; + + err = percpu_counter_init(&sbi->rf_node_block_count, 0, GFP_KERNEL); + if (err) + goto err_valid_block; + + err = percpu_counter_init(&sbi->total_valid_inode_count, 0, + GFP_KERNEL); + if (err) + goto err_node_block; + return 0; + +err_node_block: + percpu_counter_destroy(&sbi->rf_node_block_count); +err_valid_block: + percpu_counter_destroy(&sbi->alloc_valid_block_count); + return err; +} + +#ifdef CONFIG_BLK_DEV_ZONED + +struct f2fs_report_zones_args { + struct f2fs_sb_info *sbi; + struct f2fs_dev_info *dev; +}; + +static int f2fs_report_zone_cb(struct blk_zone *zone, unsigned int idx, + void *data) +{ + struct f2fs_report_zones_args *rz_args = data; + block_t unusable_blocks = (zone->len - zone->capacity) >> + F2FS_LOG_SECTORS_PER_BLOCK; + + if (zone->type == BLK_ZONE_TYPE_CONVENTIONAL) + return 0; + + set_bit(idx, rz_args->dev->blkz_seq); + if (!rz_args->sbi->unusable_blocks_per_sec) { + rz_args->sbi->unusable_blocks_per_sec = unusable_blocks; + return 0; + } + if (rz_args->sbi->unusable_blocks_per_sec != unusable_blocks) { + f2fs_err(rz_args->sbi, "F2FS supports single zone capacity\n"); + return -EINVAL; + } + return 0; +} + +static int init_blkz_info(struct f2fs_sb_info *sbi, int devi) +{ + struct block_device *bdev = FDEV(devi).bdev; + sector_t nr_sectors = bdev_nr_sectors(bdev); + struct f2fs_report_zones_args rep_zone_arg; + u64 zone_sectors; + unsigned int max_open_zones; + int ret; + + if (!f2fs_sb_has_blkzoned(sbi)) + return 0; + + if (bdev_is_zoned(FDEV(devi).bdev)) { + max_open_zones = bdev_max_open_zones(bdev); + if (max_open_zones && (max_open_zones < sbi->max_open_zones)) + sbi->max_open_zones = max_open_zones; + if (sbi->max_open_zones < F2FS_OPTION(sbi).active_logs) { + f2fs_err(sbi, + "zoned: max open zones %u is too small, need at least %u open zones", + sbi->max_open_zones, F2FS_OPTION(sbi).active_logs); + return -EINVAL; + } + } + + zone_sectors = bdev_zone_sectors(bdev); + if (sbi->blocks_per_blkz && sbi->blocks_per_blkz != + SECTOR_TO_BLOCK(zone_sectors)) + return -EINVAL; + sbi->blocks_per_blkz = SECTOR_TO_BLOCK(zone_sectors); + FDEV(devi).nr_blkz = div_u64(SECTOR_TO_BLOCK(nr_sectors), + sbi->blocks_per_blkz); + if (nr_sectors & (zone_sectors - 1)) + FDEV(devi).nr_blkz++; + + FDEV(devi).blkz_seq = f2fs_kvzalloc(sbi, + BITS_TO_LONGS(FDEV(devi).nr_blkz) + * sizeof(unsigned long), + GFP_KERNEL); + if (!FDEV(devi).blkz_seq) + return -ENOMEM; + + rep_zone_arg.sbi = sbi; + rep_zone_arg.dev = &FDEV(devi); + + ret = blkdev_report_zones(bdev, 0, BLK_ALL_ZONES, f2fs_report_zone_cb, + &rep_zone_arg); + if (ret < 0) + return ret; + return 0; +} +#endif + +/* + * Read f2fs raw super block. + * Because we have two copies of super block, so read both of them + * to get the first valid one. If any one of them is broken, we pass + * them recovery flag back to the caller. + */ +static int read_raw_super_block(struct f2fs_sb_info *sbi, + struct f2fs_super_block **raw_super, + int *valid_super_block, int *recovery) +{ + struct super_block *sb = sbi->sb; + int block; + struct folio *folio; + struct f2fs_super_block *super; + int err = 0; + + super = kzalloc_obj(struct f2fs_super_block); + if (!super) + return -ENOMEM; + + for (block = 0; block < 2; block++) { + folio = read_mapping_folio(sb->s_bdev->bd_mapping, block, NULL); + if (IS_ERR(folio)) { + f2fs_err(sbi, "Unable to read %dth superblock", + block + 1); + err = PTR_ERR(folio); + *recovery = 1; + continue; + } + + /* sanity checking of raw super */ + err = sanity_check_raw_super(sbi, folio, block); + if (err) { + f2fs_err(sbi, "Can't find valid F2FS filesystem in %dth superblock", + block + 1); + folio_put(folio); + *recovery = 1; + continue; + } + + if (!*raw_super) { + memcpy(super, F2FS_SUPER_BLOCK(folio, block), + sizeof(*super)); + *valid_super_block = block; + *raw_super = super; + } + folio_put(folio); + } + + /* No valid superblock */ + if (!*raw_super) + kfree(super); + else + err = 0; + + return err; +} + +int f2fs_commit_super(struct f2fs_sb_info *sbi, bool recover) +{ + struct folio *folio; + pgoff_t index; + __u32 crc = 0; + int err; + + if ((recover && f2fs_readonly(sbi->sb)) || + f2fs_hw_is_readonly(sbi)) { + set_sbi_flag(sbi, SBI_NEED_SB_WRITE); + return -EROFS; + } + + /* we should update superblock crc here */ + if (!recover && f2fs_sb_has_sb_chksum(sbi)) { + crc = f2fs_crc32(F2FS_RAW_SUPER(sbi), + offsetof(struct f2fs_super_block, crc)); + F2FS_RAW_SUPER(sbi)->crc = cpu_to_le32(crc); + } + + /* write back-up superblock first */ + index = sbi->valid_super_block ? 0 : 1; + folio = read_mapping_folio(sbi->sb->s_bdev->bd_mapping, index, NULL); + if (IS_ERR(folio)) + return PTR_ERR(folio); + err = __f2fs_commit_super(sbi, folio, index, true); + folio_put(folio); + + /* if we are in recovery path, skip writing valid superblock */ + if (recover || err) + return err; + + /* write current valid superblock */ + index = sbi->valid_super_block; + folio = read_mapping_folio(sbi->sb->s_bdev->bd_mapping, index, NULL); + if (IS_ERR(folio)) + return PTR_ERR(folio); + err = __f2fs_commit_super(sbi, folio, index, true); + folio_put(folio); + return err; +} + +static void save_stop_reason(struct f2fs_sb_info *sbi, unsigned char reason) +{ + unsigned long flags; + + spin_lock_irqsave(&sbi->error_lock, flags); + if (sbi->stop_reason[reason] < GENMASK(BITS_PER_BYTE - 1, 0)) + sbi->stop_reason[reason]++; + sbi->stop_reason_dirty = true; + spin_unlock_irqrestore(&sbi->error_lock, flags); +} + +static void f2fs_record_stop_reason(struct f2fs_sb_info *sbi) +{ + struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); + unsigned long flags; + bool report_shutdown = false; + int err; + + f2fs_down_write(&sbi->sb_lock); + + spin_lock_irqsave(&sbi->error_lock, flags); + if (sbi->error_dirty) { + memcpy(raw_super->s_errors, sbi->errors, MAX_F2FS_ERRORS); + sbi->error_dirty = false; + } + memcpy(raw_super->s_stop_reason, sbi->stop_reason, MAX_STOP_REASON); + if (sbi->stop_reason_dirty) { + report_shutdown = true; + sbi->stop_reason_dirty = false; + } + spin_unlock_irqrestore(&sbi->error_lock, flags); + + err = f2fs_commit_super(sbi, false); + + f2fs_up_write(&sbi->sb_lock); + if (err) + f2fs_err_ratelimited(sbi, + "f2fs_commit_super fails to record stop_reason, err:%d", + err); + + if (report_shutdown) + fserror_report_shutdown(sbi->sb, GFP_NOFS); +} + +void f2fs_save_errors(struct f2fs_sb_info *sbi, unsigned char flag) +{ + unsigned long flags; + + spin_lock_irqsave(&sbi->error_lock, flags); + if (!test_bit(flag, (unsigned long *)sbi->errors)) { + set_bit(flag, (unsigned long *)sbi->errors); + sbi->error_dirty = true; + } + spin_unlock_irqrestore(&sbi->error_lock, flags); +} + +static void f2fs_report_fserror(struct f2fs_sb_info *sbi, unsigned char error) +{ + switch (error) { + case ERROR_INVALID_BLKADDR: + case ERROR_CORRUPTED_INODE: + case ERROR_INCONSISTENT_SUMMARY: + case ERROR_INCONSISTENT_SUM_TYPE: + case ERROR_CORRUPTED_JOURNAL: + case ERROR_INCONSISTENT_NODE_COUNT: + case ERROR_INCONSISTENT_BLOCK_COUNT: + case ERROR_INVALID_CURSEG: + case ERROR_INCONSISTENT_SIT: + case ERROR_INVALID_NODE_REFERENCE: + case ERROR_INCONSISTENT_NAT: + fserror_report_metadata(sbi->sb, -EFSCORRUPTED, GFP_NOFS); + break; + default: + return; + } +} + +void f2fs_handle_error(struct f2fs_sb_info *sbi, unsigned char error) +{ + f2fs_save_errors(sbi, error); + + if (!sbi->error_dirty) + return; + if (!test_bit(error, (unsigned long *)sbi->errors)) + return; + schedule_work(&sbi->s_error_work); + + f2fs_report_fserror(sbi, error); +} + +static bool system_going_down(void) +{ + return system_state == SYSTEM_HALT || system_state == SYSTEM_POWER_OFF + || system_state == SYSTEM_RESTART; +} + +static void f2fs_handle_critical_error(struct f2fs_sb_info *sbi, + unsigned char reason) +{ + struct super_block *sb = sbi->sb; + bool shutdown = reason == STOP_CP_REASON_SHUTDOWN; + bool continue_fs = !shutdown && + F2FS_OPTION(sbi).errors == MOUNT_ERRORS_CONTINUE; + + set_ckpt_flags(sbi, CP_ERROR_FLAG); + + if (!f2fs_hw_is_readonly(sbi)) { + save_stop_reason(sbi, reason); + + /* + * always create an asynchronous task to record stop_reason + * in order to avoid potential deadlock when running into + * f2fs_record_stop_reason() synchronously. + */ + schedule_work(&sbi->s_error_work); + } + + /* + * We force ERRORS_RO behavior when system is rebooting. Otherwise we + * could panic during 'reboot -f' as the underlying device got already + * disabled. + */ + if (F2FS_OPTION(sbi).errors == MOUNT_ERRORS_PANIC && + !shutdown && !system_going_down() && + !is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN)) + panic("F2FS-fs (device %s): panic forced after error\n", + sb->s_id); + + if (shutdown) + set_sbi_flag(sbi, SBI_IS_SHUTDOWN); + else + dump_stack(); + + /* + * Continue filesystem operators if errors=continue. Should not set + * RO by shutdown, since RO bypasses thaw_super which can hang the + * system. + */ + if (continue_fs || f2fs_readonly(sb) || shutdown) { + f2fs_warn(sbi, "Stopped filesystem due to reason: %d", reason); + return; + } + + f2fs_warn(sbi, "Remounting filesystem read-only"); + + /* + * We have already set CP_ERROR_FLAG flag to stop all updates + * to filesystem, so it doesn't need to set SB_RDONLY flag here + * because the flag should be set covered w/ sb->s_umount semaphore + * via remount procedure, otherwise, it will confuse code like + * freeze_super() which will lead to deadlocks and other problems. + */ +} + +void f2fs_fault_report(struct super_block *sb, unsigned int err_code, + const char *func, unsigned int data) +{ + trace_f2fs_fault_report(sb, err_code, func, data); +} + +void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io, + unsigned char reason) +{ + if (reason != STOP_CP_REASON_SHUTDOWN) + f2fs_fault_report(sbi->sb, REPORT_FAULT_STOP_CP, __func__, reason); + + f2fs_build_fault_attr(sbi, 0, 0, FAULT_ALL); + if (!end_io) + f2fs_flush_merged_writes(sbi); + f2fs_handle_critical_error(sbi, reason); +} + + +static void f2fs_record_error_work(struct work_struct *work) +{ + struct f2fs_sb_info *sbi = container_of(work, + struct f2fs_sb_info, s_error_work); + + f2fs_record_stop_reason(sbi); +} + +static inline unsigned int get_first_seq_zone_segno(struct f2fs_sb_info *sbi) +{ +#ifdef CONFIG_BLK_DEV_ZONED + unsigned int zoneno, total_zones; + int devi; + + if (!f2fs_sb_has_blkzoned(sbi)) + return NULL_SEGNO; + + for (devi = 0; devi < sbi->s_ndevs; devi++) { + if (!bdev_is_zoned(FDEV(devi).bdev)) + continue; + + total_zones = GET_ZONE_FROM_SEG(sbi, FDEV(devi).total_segments); + + for (zoneno = 0; zoneno < total_zones; zoneno++) { + unsigned int segs, blks; + + if (!f2fs_zone_is_seq(sbi, devi, zoneno)) + continue; + + segs = GET_SEG_FROM_SEC(sbi, + zoneno * sbi->secs_per_zone); + blks = SEGS_TO_BLKS(sbi, segs); + return GET_SEGNO(sbi, FDEV(devi).start_blk + blks); + } + } +#endif + return NULL_SEGNO; +} + +static int f2fs_scan_devices(struct f2fs_sb_info *sbi) +{ + struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); + unsigned int max_devices = MAX_DEVICES; + unsigned int logical_blksize; + blk_mode_t mode = sb_open_mode(sbi->sb->s_flags); + int i; + + /* Initialize single device information */ + if (!RDEV(0).path[0]) { + if (!bdev_is_zoned(sbi->sb->s_bdev)) + return 0; + max_devices = 1; + } + + /* + * Initialize multiple devices information, or single + * zoned block device information. + */ + sbi->devs = f2fs_kzalloc(sbi, + array_size(max_devices, + sizeof(struct f2fs_dev_info)), + GFP_KERNEL); + if (!sbi->devs) + return -ENOMEM; + + logical_blksize = bdev_logical_block_size(sbi->sb->s_bdev); + sbi->aligned_blksize = true; + sbi->bggc_io_aware = AWARE_ALL_IO; +#ifdef CONFIG_BLK_DEV_ZONED + sbi->max_open_zones = UINT_MAX; + sbi->blkzone_alloc_policy = BLKZONE_ALLOC_PRIOR_SEQ; + sbi->bggc_io_aware = AWARE_READ_IO; +#endif + + for (i = 0; i < max_devices; i++) { + if (max_devices == 1) { + FDEV(i).total_segments = + le32_to_cpu(raw_super->segment_count_main); + FDEV(i).start_blk = 0; + FDEV(i).end_blk = FDEV(i).total_segments * + BLKS_PER_SEG(sbi); + } + + if (i == 0) + FDEV(0).bdev_file = sbi->sb->s_bdev_file; + else if (!RDEV(i).path[0]) + break; + + if (max_devices > 1) { + /* Multi-device mount */ + memcpy(FDEV(i).path, RDEV(i).path, MAX_PATH_LEN); + FDEV(i).total_segments = + le32_to_cpu(RDEV(i).total_segments); + if (i == 0) { + FDEV(i).start_blk = 0; + FDEV(i).end_blk = FDEV(i).start_blk + + SEGS_TO_BLKS(sbi, + FDEV(i).total_segments) - 1 + + le32_to_cpu(raw_super->segment0_blkaddr); + sbi->allocate_section_hint = FDEV(i).total_segments / + SEGS_PER_SEC(sbi); + } else { + FDEV(i).start_blk = FDEV(i - 1).end_blk + 1; + FDEV(i).end_blk = FDEV(i).start_blk + + SEGS_TO_BLKS(sbi, + FDEV(i).total_segments) - 1; + FDEV(i).bdev_file = fs_bdev_file_open_by_path( + FDEV(i).path, mode, sbi->sb, sbi->sb); + } + } + if (IS_ERR(FDEV(i).bdev_file)) + return PTR_ERR(FDEV(i).bdev_file); + + FDEV(i).bdev = file_bdev(FDEV(i).bdev_file); + /* to release errored devices */ + sbi->s_ndevs = i + 1; + + if (logical_blksize != bdev_logical_block_size(FDEV(i).bdev)) + sbi->aligned_blksize = false; + +#ifdef CONFIG_BLK_DEV_ZONED + if (bdev_is_zoned(FDEV(i).bdev)) { + if (!f2fs_sb_has_blkzoned(sbi)) { + f2fs_err(sbi, "Zoned block device feature not enabled"); + return -EINVAL; + } + if (init_blkz_info(sbi, i)) { + f2fs_err(sbi, "Failed to initialize F2FS blkzone information"); + return -EINVAL; + } + if (max_devices == 1) + break; + f2fs_info(sbi, "Mount Device [%2d]: %20s, %8u, %8x - %8x (zone: Host-managed)", + i, FDEV(i).path, + FDEV(i).total_segments, + FDEV(i).start_blk, FDEV(i).end_blk); + continue; + } +#endif + f2fs_info(sbi, "Mount Device [%2d]: %20s, %8u, %8x - %8x", + i, FDEV(i).path, + FDEV(i).total_segments, + FDEV(i).start_blk, FDEV(i).end_blk); + } + return 0; +} + +static int f2fs_setup_casefold(struct f2fs_sb_info *sbi) +{ +#if IS_ENABLED(CONFIG_UNICODE) + if (f2fs_sb_has_casefold(sbi) && !sbi->sb->s_encoding) { + const struct f2fs_sb_encodings *encoding_info; + struct unicode_map *encoding; + __u16 encoding_flags; + + encoding_info = f2fs_sb_read_encoding(sbi->raw_super); + if (!encoding_info) { + f2fs_err(sbi, + "Encoding requested by superblock is unknown"); + return -EINVAL; + } + + encoding_flags = le16_to_cpu(sbi->raw_super->s_encoding_flags); + encoding = utf8_load(encoding_info->version); + if (IS_ERR(encoding)) { + f2fs_err(sbi, + "can't mount with superblock charset: %s-%u.%u.%u " + "not supported by the kernel. flags: 0x%x.", + encoding_info->name, + unicode_major(encoding_info->version), + unicode_minor(encoding_info->version), + unicode_rev(encoding_info->version), + encoding_flags); + return PTR_ERR(encoding); + } + f2fs_info(sbi, "Using encoding defined by superblock: " + "%s-%u.%u.%u with flags 0x%hx", encoding_info->name, + unicode_major(encoding_info->version), + unicode_minor(encoding_info->version), + unicode_rev(encoding_info->version), + encoding_flags); + + sbi->sb->s_encoding = encoding; + sbi->sb->s_encoding_flags = encoding_flags; + } +#else + if (f2fs_sb_has_casefold(sbi)) { + f2fs_err(sbi, "Filesystem with casefold feature cannot be mounted without CONFIG_UNICODE"); + return -EINVAL; + } +#endif + return 0; +} + +static void f2fs_tuning_parameters(struct f2fs_sb_info *sbi) +{ + /* adjust parameters according to the volume size */ + if (MAIN_SEGS(sbi) <= SMALL_VOLUME_SEGMENTS) { + if (f2fs_block_unit_discard(sbi)) + SM_I(sbi)->dcc_info->discard_granularity = + MIN_DISCARD_GRANULARITY; + if (!f2fs_lfs_mode(sbi)) + SM_I(sbi)->ipu_policy = BIT(F2FS_IPU_FORCE) | + BIT(F2FS_IPU_HONOR_OPU_WRITE); + } + + sbi->readdir_ra = true; +} + +static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct f2fs_fs_context *ctx = fc->fs_private; + struct f2fs_sb_info *sbi; + struct f2fs_super_block *raw_super; + struct inode *root; + int err; + bool skip_recovery = false, need_fsck = false; + int recovery, i, valid_super_block; + struct curseg_info *seg_i; + int retry_cnt = 1; +#ifdef CONFIG_QUOTA + bool quota_enabled = false; +#endif + +try_onemore: + err = -EINVAL; + raw_super = NULL; + valid_super_block = -1; + recovery = 0; + + /* allocate memory for f2fs-specific super block info */ + sbi = kzalloc_obj(struct f2fs_sb_info); + if (!sbi) + return -ENOMEM; + + sbi->sb = sb; + + /* initialize locks within allocated memory */ + init_f2fs_rwsem_trace(&sbi->gc_lock, sbi, LOCK_NAME_GC_LOCK); + mutex_init(&sbi->writepages); + init_f2fs_rwsem_trace(&sbi->cp_global_sem, sbi, LOCK_NAME_CP_GLOBAL); +#ifdef CONFIG_DEBUG_LOCK_ALLOC + lockdep_register_key(&sbi->cp_global_sem_key); + lockdep_set_class(&sbi->cp_global_sem.internal_rwsem, + &sbi->cp_global_sem_key); +#endif + init_f2fs_rwsem_trace(&sbi->node_write, sbi, LOCK_NAME_NODE_WRITE); + init_f2fs_rwsem_trace(&sbi->node_change, sbi, LOCK_NAME_NODE_CHANGE); + spin_lock_init(&sbi->stat_lock); + init_f2fs_rwsem_trace(&sbi->cp_rwsem, sbi, LOCK_NAME_CP_RWSEM); + init_f2fs_rwsem(&sbi->quota_sem); + init_waitqueue_head(&sbi->cp_wait); + spin_lock_init(&sbi->error_lock); + + for (i = 0; i < NR_INODE_TYPE; i++) { + INIT_LIST_HEAD(&sbi->inode_list[i]); + spin_lock_init(&sbi->inode_lock[i]); + } + mutex_init(&sbi->flush_lock); + + /* set a block size */ + if (unlikely(!sb_set_blocksize(sb, F2FS_BLKSIZE))) { + f2fs_err(sbi, "unable to set blocksize"); + goto free_sbi; + } + + err = read_raw_super_block(sbi, &raw_super, &valid_super_block, + &recovery); + if (err) + goto free_sbi; + + sb->s_fs_info = sbi; + sbi->raw_super = raw_super; + + INIT_WORK(&sbi->s_error_work, f2fs_record_error_work); + memcpy(sbi->errors, raw_super->s_errors, MAX_F2FS_ERRORS); + memcpy(sbi->stop_reason, raw_super->s_stop_reason, MAX_STOP_REASON); + + /* precompute checksum seed for metadata */ + if (f2fs_sb_has_inode_chksum(sbi)) + sbi->s_chksum_seed = f2fs_chksum(~0, raw_super->uuid, + sizeof(raw_super->uuid)); + + default_options(sbi, false); + + err = f2fs_check_opt_consistency(fc, sb); + if (err) + goto free_sb_buf; + + f2fs_apply_options(fc, sb); + + err = f2fs_sanity_check_options(sbi, false); + if (err) + goto free_options; + + sb->s_maxbytes = max_file_blocks(NULL) << + le32_to_cpu(raw_super->log_blocksize); + sb->s_max_links = F2FS_LINK_MAX; + + err = f2fs_setup_casefold(sbi); + if (err) + goto free_options; + +#ifdef CONFIG_QUOTA + sb->dq_op = &f2fs_quota_operations; + sb->s_qcop = &f2fs_quotactl_ops; + sb->s_quota_types = QTYPE_MASK_USR | QTYPE_MASK_GRP | QTYPE_MASK_PRJ; + + if (f2fs_sb_has_quota_ino(sbi)) { + for (i = 0; i < MAXQUOTAS; i++) { + if (f2fs_qf_ino(sbi->sb, i)) + sbi->nquota_files++; + } + } +#endif + + sb->s_op = &f2fs_sops; +#ifdef CONFIG_FS_ENCRYPTION + sb->s_cop = &f2fs_cryptops; +#endif +#ifdef CONFIG_FS_VERITY + sb->s_vop = &f2fs_verityops; +#endif + sb->s_xattr = f2fs_xattr_handlers; + sb->s_export_op = &f2fs_export_ops; + sb->s_magic = F2FS_SUPER_MAGIC; + sb->s_time_gran = 1; + sb->s_flags = (sb->s_flags & ~SB_POSIXACL) | + (test_opt(sbi, POSIX_ACL) ? SB_POSIXACL : 0); + if (test_opt(sbi, INLINECRYPT)) + sb->s_flags |= SB_INLINECRYPT; + + if (test_opt(sbi, LAZYTIME)) + sb->s_flags |= SB_LAZYTIME; + else + sb->s_flags &= ~SB_LAZYTIME; + + super_set_uuid(sb, (void *) raw_super->uuid, sizeof(raw_super->uuid)); + super_set_sysfs_name_bdev(sb); + sb->s_iflags |= SB_I_CGROUPWB; + + /* init f2fs-specific super block info */ + sbi->valid_super_block = valid_super_block; + + /* disallow all the data/node/meta page writes */ + set_sbi_flag(sbi, SBI_POR_DOING); + + err = f2fs_init_write_merge_io(sbi); + if (err) + goto free_bio_info; + + init_sb_info(sbi); + + err = f2fs_init_iostat(sbi); + if (err) + goto free_bio_info; + + err = init_percpu_info(sbi); + if (err) + goto free_iostat; + + err = f2fs_init_page_array_cache(sbi); + if (err) + goto free_percpu; + + /* get an inode for meta space */ + sbi->meta_inode = f2fs_iget(sb, F2FS_META_INO(sbi)); + if (IS_ERR(sbi->meta_inode)) { + f2fs_err(sbi, "Failed to read F2FS meta data inode"); + err = PTR_ERR(sbi->meta_inode); + goto free_page_array_cache; + } + + err = f2fs_get_valid_checkpoint(sbi); + if (err) { + f2fs_err(sbi, "Failed to get valid F2FS checkpoint"); + goto free_meta_inode; + } + + if (__is_set_ckpt_flags(F2FS_CKPT(sbi), CP_QUOTA_NEED_FSCK_FLAG)) + set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR); + if (__is_set_ckpt_flags(F2FS_CKPT(sbi), CP_DISABLED_QUICK_FLAG)) { + set_sbi_flag(sbi, SBI_CP_DISABLED_QUICK); + sbi->interval_time[DISABLE_TIME] = DEF_DISABLE_QUICK_INTERVAL; + } + + if (__is_set_ckpt_flags(F2FS_CKPT(sbi), CP_FSCK_FLAG)) + set_sbi_flag(sbi, SBI_NEED_FSCK); + + /* Initialize device list */ + err = f2fs_scan_devices(sbi); + if (err) { + f2fs_err(sbi, "Failed to find devices"); + goto free_devices; + } + + err = f2fs_init_wq(sbi); + if (err) { + f2fs_err(sbi, "Failed to create workqueue"); + goto free_devices; + } + + sbi->total_valid_node_count = + le32_to_cpu(sbi->ckpt->valid_node_count); + percpu_counter_set(&sbi->total_valid_inode_count, + le32_to_cpu(sbi->ckpt->valid_inode_count)); + sbi->user_block_count = le64_to_cpu(sbi->ckpt->user_block_count); + sbi->total_valid_block_count = + le64_to_cpu(sbi->ckpt->valid_block_count); + sbi->last_valid_block_count = sbi->total_valid_block_count; + sbi->reserved_blocks = 0; + sbi->current_reserved_blocks = 0; + limit_reserve_root(sbi); + adjust_unusable_cap_perc(sbi); + + f2fs_init_extent_cache_info(sbi); + + f2fs_init_ino_entry_info(sbi); + + f2fs_init_fsync_node_info(sbi); + + /* setup checkpoint request control and start checkpoint issue thread */ + f2fs_init_ckpt_req_control(sbi); + if (!f2fs_readonly(sb) && !test_opt(sbi, DISABLE_CHECKPOINT) && + test_opt(sbi, MERGE_CHECKPOINT)) { + err = f2fs_start_ckpt_thread(sbi); + if (err) { + f2fs_err(sbi, + "Failed to start F2FS issue_checkpoint_thread (%d)", + err); + goto stop_ckpt_thread; + } + } + + /* setup f2fs internal modules */ + err = f2fs_build_segment_manager(sbi); + if (err) { + f2fs_err(sbi, "Failed to initialize F2FS segment manager (%d)", + err); + goto free_sm; + } + err = f2fs_build_node_manager(sbi); + if (err) { + f2fs_err(sbi, "Failed to initialize F2FS node manager (%d)", + err); + goto free_nm; + } + + /* For write statistics */ + sbi->sectors_written_start = f2fs_get_sectors_written(sbi); + + /* get segno of first zoned block device */ + sbi->first_seq_zone_segno = get_first_seq_zone_segno(sbi); + + sbi->reserved_pin_section = f2fs_sb_has_blkzoned(sbi) ? + ZONED_PIN_SEC_REQUIRED_COUNT : + GET_SEC_FROM_SEG(sbi, overprovision_segments(sbi)); + + /* Read accumulated write IO statistics if exists */ + seg_i = CURSEG_I(sbi, CURSEG_HOT_NODE); + if (__exist_node_summaries(sbi)) + sbi->kbytes_written = + le64_to_cpu(seg_i->journal->info.kbytes_written); + + f2fs_build_gc_manager(sbi); + + err = f2fs_build_stats(sbi); + if (err) + goto free_nm; + + /* get an inode for node space */ + sbi->node_inode = f2fs_iget(sb, F2FS_NODE_INO(sbi)); + if (IS_ERR(sbi->node_inode)) { + f2fs_err(sbi, "Failed to read node inode"); + err = PTR_ERR(sbi->node_inode); + goto free_stats; + } + + /* read root inode and dentry */ + root = f2fs_iget(sb, F2FS_ROOT_INO(sbi)); + if (IS_ERR(root)) { + f2fs_err(sbi, "Failed to read root inode"); + err = PTR_ERR(root); + goto free_node_inode; + } + if (!S_ISDIR(root->i_mode) || !root->i_blocks || + !root->i_size || !root->i_nlink) { + iput(root); + err = -EINVAL; + goto free_node_inode; + } + + generic_set_sb_d_ops(sb); + sb->s_root = d_make_root(root); /* allocate root dentry */ + if (!sb->s_root) { + err = -ENOMEM; + goto free_node_inode; + } + + err = f2fs_init_compress_inode(sbi); + if (err) + goto free_root_inode; + + err = f2fs_register_sysfs(sbi); + if (err) + goto free_compress_inode; + + sbi->umount_lock_holder = current; +#ifdef CONFIG_QUOTA + /* Enable quota usage during mount */ + if (f2fs_sb_has_quota_ino(sbi) && !f2fs_readonly(sb)) { + err = f2fs_enable_quotas(sb); + if (err) + f2fs_err(sbi, "Cannot turn on quotas: error %d", err); + } + + quota_enabled = f2fs_recover_quota_begin(sbi); +#endif + /* if there are any orphan inodes, free them */ + err = f2fs_recover_orphan_inodes(sbi); + if (err) + goto free_meta; + + if (unlikely(is_set_ckpt_flags(sbi, CP_DISABLED_FLAG))) { + skip_recovery = true; + goto reset_checkpoint; + } + + /* recover fsynced data */ + if (!test_opt(sbi, DISABLE_ROLL_FORWARD) && + !test_opt(sbi, NORECOVERY)) { + /* + * mount should be failed, when device has readonly mode, and + * previous checkpoint was not done by clean system shutdown. + */ + if (f2fs_hw_is_readonly(sbi)) { + if (!is_set_ckpt_flags(sbi, CP_UMOUNT_FLAG)) { + err = f2fs_recover_fsync_data(sbi, true); + if (err > 0) { + err = -EROFS; + f2fs_err(sbi, "Need to recover fsync data, but " + "write access unavailable, please try " + "mount w/ disable_roll_forward or norecovery"); + } + if (err < 0) + goto free_meta; + } + f2fs_info(sbi, "write access unavailable, skipping recovery"); + goto reset_checkpoint; + } + + if (need_fsck) + set_sbi_flag(sbi, SBI_NEED_FSCK); + + if (skip_recovery) + goto reset_checkpoint; + + err = f2fs_recover_fsync_data(sbi, false); + if (err < 0) { + if (err != -ENOMEM) + skip_recovery = true; + need_fsck = true; + f2fs_err(sbi, "Cannot recover all fsync data errno=%d", + err); + goto free_meta; + } + } else { + err = f2fs_recover_fsync_data(sbi, true); + if (err > 0) { + if (!f2fs_readonly(sb)) { + f2fs_err(sbi, "Need to recover fsync data"); + err = -EINVAL; + goto free_meta; + } else { + f2fs_info(sbi, "drop all fsynced data"); + err = 0; + } + } + } + +reset_checkpoint: +#ifdef CONFIG_QUOTA + f2fs_recover_quota_end(sbi, quota_enabled); +#endif + /* + * If the f2fs is not readonly and fsync data recovery succeeds, + * write pointer consistency of cursegs and other zones are already + * checked and fixed during recovery. However, if recovery fails, + * write pointers are left untouched, and retry-mount should check + * them here. + */ + if (skip_recovery) + err = f2fs_check_and_fix_write_pointer(sbi); + if (err) + goto free_meta; + + /* f2fs_recover_fsync_data() cleared this already */ + clear_sbi_flag(sbi, SBI_POR_DOING); + + err = f2fs_init_inmem_curseg(sbi); + if (err) + goto sync_free_meta; + + if (test_opt(sbi, DISABLE_CHECKPOINT)) + err = f2fs_disable_checkpoint(sbi); + else if (is_set_ckpt_flags(sbi, CP_DISABLED_FLAG)) + err = f2fs_enable_checkpoint(sbi); + if (err) + goto sync_free_meta; + + /* + * If filesystem is not mounted as read-only then + * do start the gc_thread. + */ + if ((F2FS_OPTION(sbi).bggc_mode != BGGC_MODE_OFF || + test_opt(sbi, GC_MERGE)) && !f2fs_readonly(sb)) { + /* After POR, we can run background GC thread.*/ + err = f2fs_start_gc_thread(sbi); + if (err) + goto sync_free_meta; + } + + /* recover broken superblock */ + if (recovery) { + err = f2fs_commit_super(sbi, true); + f2fs_info(sbi, "Try to recover %dth superblock, ret: %d", + sbi->valid_super_block ? 1 : 2, err); + } + + f2fs_join_shrinker(sbi); + + f2fs_tuning_parameters(sbi); + + f2fs_notice(sbi, "Mounted with checkpoint version = %llx", + cur_cp_version(F2FS_CKPT(sbi))); + f2fs_update_time(sbi, CP_TIME); + f2fs_update_time(sbi, REQ_TIME); + clear_sbi_flag(sbi, SBI_CP_DISABLED_QUICK); + + sbi->umount_lock_holder = NULL; + return 0; + +sync_free_meta: + /* safe to flush all the data */ + sync_filesystem(sbi->sb); + retry_cnt = 0; + +free_meta: +#ifdef CONFIG_QUOTA + f2fs_truncate_quota_inode_pages(sb); + if (f2fs_sb_has_quota_ino(sbi) && !f2fs_readonly(sb)) + f2fs_quota_off_umount(sbi->sb); +#endif + /* + * Some dirty meta pages can be produced by f2fs_recover_orphan_inodes() + * failed by EIO. Then, iput(node_inode) can trigger balance_fs_bg() + * followed by f2fs_write_checkpoint() through f2fs_write_node_pages(), which + * falls into an infinite loop in f2fs_sync_meta_pages(). + */ + truncate_inode_pages_final(META_MAPPING(sbi)); + /* evict some inodes being cached by GC */ + evict_inodes(sb); + f2fs_unregister_sysfs(sbi); +free_compress_inode: + f2fs_destroy_compress_inode(sbi); +free_root_inode: + dput(sb->s_root); + sb->s_root = NULL; +free_node_inode: + f2fs_release_ino_entry(sbi, true); + truncate_inode_pages_final(NODE_MAPPING(sbi)); + iput(sbi->node_inode); + sbi->node_inode = NULL; +free_stats: + f2fs_destroy_stats(sbi); +free_nm: + /* stop discard thread before destroying node manager */ + f2fs_stop_discard_thread(sbi); + f2fs_destroy_node_manager(sbi); +free_sm: + f2fs_destroy_segment_manager(sbi); +stop_ckpt_thread: + f2fs_stop_ckpt_thread(sbi); + /* flush s_error_work before sbi destroy */ + flush_work(&sbi->s_error_work); + f2fs_destroy_wq(sbi); +free_devices: + destroy_device_list(sbi); + kvfree(sbi->ckpt); +free_meta_inode: + make_bad_inode(sbi->meta_inode); + iput(sbi->meta_inode); + sbi->meta_inode = NULL; +free_page_array_cache: + f2fs_destroy_page_array_cache(sbi); +free_percpu: + destroy_percpu_info(sbi); +free_iostat: + f2fs_destroy_iostat(sbi); +free_bio_info: + for (i = 0; i < NR_PAGE_TYPE; i++) + kfree(sbi->write_io[i]); + +#if IS_ENABLED(CONFIG_UNICODE) + utf8_unload(sb->s_encoding); + sb->s_encoding = NULL; +#endif +free_options: +#ifdef CONFIG_QUOTA + for (i = 0; i < MAXQUOTAS; i++) + kfree(F2FS_OPTION(sbi).s_qf_names[i]); +#endif + /* no need to free dummy_enc_policy, we just keep it in ctx when failed */ + swap(F2FS_CTX_INFO(ctx).dummy_enc_policy, F2FS_OPTION(sbi).dummy_enc_policy); +free_sb_buf: + kfree(raw_super); +free_sbi: +#ifdef CONFIG_DEBUG_LOCK_ALLOC + lockdep_unregister_key(&sbi->cp_global_sem_key); +#endif + kfree(sbi); + sb->s_fs_info = NULL; + + /* give only one another chance */ + if (retry_cnt > 0 && skip_recovery) { + retry_cnt--; + shrink_dcache_sb(sb); + goto try_onemore; + } + return err; +} + +static int f2fs_get_tree(struct fs_context *fc) +{ + return get_tree_bdev(fc, f2fs_fill_super); +} + +static int f2fs_reconfigure(struct fs_context *fc) +{ + struct super_block *sb = fc->root->d_sb; + + return __f2fs_remount(fc, sb); +} + +static void f2fs_fc_free(struct fs_context *fc) +{ + struct f2fs_fs_context *ctx = fc->fs_private; + + if (!ctx) + return; + +#ifdef CONFIG_QUOTA + f2fs_unnote_qf_name_all(fc); +#endif + fscrypt_free_dummy_policy(&F2FS_CTX_INFO(ctx).dummy_enc_policy); + kfree(ctx); +} + +static const struct fs_context_operations f2fs_context_ops = { + .parse_param = f2fs_parse_param, + .get_tree = f2fs_get_tree, + .reconfigure = f2fs_reconfigure, + .free = f2fs_fc_free, +}; + +static void kill_f2fs_super(struct super_block *sb) +{ + struct f2fs_sb_info *sbi = F2FS_SB(sb); + + if (sb->s_root) { + sbi->umount_lock_holder = current; + + set_sbi_flag(sbi, SBI_IS_CLOSE); + f2fs_stop_gc_thread(sbi); + f2fs_stop_discard_thread(sbi); + +#ifdef CONFIG_F2FS_FS_COMPRESSION + /* + * latter evict_inode() can bypass checking and invalidating + * compress inode cache. + */ + if (test_opt(sbi, COMPRESS_CACHE)) + truncate_inode_pages_final(COMPRESS_MAPPING(sbi)); +#endif + + if (is_sbi_flag_set(sbi, SBI_IS_DIRTY) || + !is_set_ckpt_flags(sbi, CP_UMOUNT_FLAG)) { + struct cp_control cpc = { + .reason = CP_UMOUNT, + }; + stat_inc_cp_call_count(sbi, TOTAL_CALL); + f2fs_write_checkpoint(sbi, &cpc); + } + + if (is_sbi_flag_set(sbi, SBI_IS_RECOVERED) && f2fs_readonly(sb)) + sb->s_flags &= ~SB_RDONLY; + } + kill_block_super(sb); + /* Release block devices last, after fscrypt_destroy_keyring(). */ + if (sbi) { + destroy_device_list(sbi); +#ifdef CONFIG_DEBUG_LOCK_ALLOC + lockdep_unregister_key(&sbi->cp_global_sem_key); +#endif + kfree(sbi); + sb->s_fs_info = NULL; + } +} + +static int f2fs_init_fs_context(struct fs_context *fc) +{ + struct f2fs_fs_context *ctx; + + ctx = kzalloc_obj(struct f2fs_fs_context); + if (!ctx) + return -ENOMEM; + + fc->fs_private = ctx; + fc->ops = &f2fs_context_ops; + + return 0; +} + +static struct file_system_type f2fs_fs_type = { + .owner = THIS_MODULE, + .name = "f2fs", + .init_fs_context = f2fs_init_fs_context, + .kill_sb = kill_f2fs_super, + .fs_flags = FS_REQUIRES_DEV | FS_ALLOW_IDMAP, +}; +MODULE_ALIAS_FS("f2fs"); + +static int __init init_inodecache(void) +{ + f2fs_inode_cachep = kmem_cache_create("f2fs_inode_cache", + sizeof(struct f2fs_inode_info), 0, + SLAB_RECLAIM_ACCOUNT|SLAB_ACCOUNT, NULL); + return f2fs_inode_cachep ? 0 : -ENOMEM; +} + +static void destroy_inodecache(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(f2fs_inode_cachep); +} + +static int __init init_f2fs_fs(void) +{ + int err; + + err = init_inodecache(); + if (err) + goto fail; + err = f2fs_create_node_manager_caches(); + if (err) + goto free_inodecache; + err = f2fs_create_segment_manager_caches(); + if (err) + goto free_node_manager_caches; + err = f2fs_create_checkpoint_caches(); + if (err) + goto free_segment_manager_caches; + err = f2fs_create_recovery_cache(); + if (err) + goto free_checkpoint_caches; + err = f2fs_create_extent_cache(); + if (err) + goto free_recovery_cache; + err = f2fs_create_garbage_collection_cache(); + if (err) + goto free_extent_cache; + err = f2fs_init_sysfs(); + if (err) + goto free_garbage_collection_cache; + err = f2fs_init_shrinker(); + if (err) + goto free_sysfs; + f2fs_create_root_stats(); + err = f2fs_init_post_read_processing(); + if (err) + goto free_root_stats; + err = f2fs_init_iostat_processing(); + if (err) + goto free_post_read; + err = f2fs_init_bio_entry_cache(); + if (err) + goto free_iostat; + err = f2fs_init_bioset(); + if (err) + goto free_bio_entry_cache; + err = f2fs_init_compress_mempool(); + if (err) + goto free_bioset; + err = f2fs_init_compress_cache(); + if (err) + goto free_compress_mempool; + err = f2fs_create_casefold_cache(); + if (err) + goto free_compress_cache; + err = f2fs_init_xattr_cache(); + if (err) + goto free_casefold_cache; + err = register_filesystem(&f2fs_fs_type); + if (err) + goto free_xattr_cache; + return 0; +free_xattr_cache: + f2fs_destroy_xattr_cache(); +free_casefold_cache: + f2fs_destroy_casefold_cache(); +free_compress_cache: + f2fs_destroy_compress_cache(); +free_compress_mempool: + f2fs_destroy_compress_mempool(); +free_bioset: + f2fs_destroy_bioset(); +free_bio_entry_cache: + f2fs_destroy_bio_entry_cache(); +free_iostat: + f2fs_destroy_iostat_processing(); +free_post_read: + f2fs_destroy_post_read_processing(); +free_root_stats: + f2fs_destroy_root_stats(); + f2fs_exit_shrinker(); +free_sysfs: + f2fs_exit_sysfs(); +free_garbage_collection_cache: + f2fs_destroy_garbage_collection_cache(); +free_extent_cache: + f2fs_destroy_extent_cache(); +free_recovery_cache: + f2fs_destroy_recovery_cache(); +free_checkpoint_caches: + f2fs_destroy_checkpoint_caches(); +free_segment_manager_caches: + f2fs_destroy_segment_manager_caches(); +free_node_manager_caches: + f2fs_destroy_node_manager_caches(); +free_inodecache: + destroy_inodecache(); +fail: + return err; +} + +static void __exit exit_f2fs_fs(void) +{ + unregister_filesystem(&f2fs_fs_type); + f2fs_destroy_xattr_cache(); + f2fs_destroy_casefold_cache(); + f2fs_destroy_compress_cache(); + f2fs_destroy_compress_mempool(); + f2fs_destroy_bioset(); + f2fs_destroy_bio_entry_cache(); + f2fs_destroy_iostat_processing(); + f2fs_destroy_post_read_processing(); + f2fs_destroy_root_stats(); + f2fs_exit_shrinker(); + f2fs_exit_sysfs(); + f2fs_destroy_garbage_collection_cache(); + f2fs_destroy_extent_cache(); + f2fs_destroy_recovery_cache(); + f2fs_destroy_checkpoint_caches(); + f2fs_destroy_segment_manager_caches(); + f2fs_destroy_node_manager_caches(); + destroy_inodecache(); +} + +module_init(init_f2fs_fs) +module_exit(exit_f2fs_fs) + +MODULE_AUTHOR("Samsung Electronics's Praesto Team"); +MODULE_DESCRIPTION("Flash Friendly File System"); +MODULE_LICENSE("GPL"); diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c new file mode 100644 index 000000000..665687244 --- /dev/null +++ b/fs/f2fs/sysfs.c @@ -0,0 +1,2112 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * f2fs sysfs interface + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * Copyright (c) 2017 Chao Yu + */ +#include +#include +#include +#include +#include +#include +#include + +#include "f2fs.h" +#include "segment.h" +#include "gc.h" +#include "iostat.h" +#include + +static struct proc_dir_entry *f2fs_proc_root; + +/* Sysfs support for f2fs */ +enum { + GC_THREAD, /* struct f2fs_gc_thread */ + SM_INFO, /* struct f2fs_sm_info */ + DCC_INFO, /* struct discard_cmd_control */ + NM_INFO, /* struct f2fs_nm_info */ + F2FS_SBI, /* struct f2fs_sb_info */ +#ifdef CONFIG_F2FS_STAT_FS + STAT_INFO, /* struct f2fs_stat_info */ +#endif +#ifdef CONFIG_F2FS_FAULT_INJECTION + FAULT_INFO_RATE, /* struct f2fs_fault_info */ + FAULT_INFO_TYPE, /* struct f2fs_fault_info */ + FAULT_INFO_TIMEOUT, /* struct f2fs_fault_info */ +#endif + RESERVED_BLOCKS, /* struct f2fs_sb_info */ + CPRC_INFO, /* struct ckpt_req_control */ + ATGC_INFO, /* struct atgc_management */ +}; + +static const char *gc_mode_names[MAX_GC_MODE] = { + "GC_NORMAL", + "GC_IDLE_CB", + "GC_IDLE_GREEDY", + "GC_IDLE_AT", + "GC_URGENT_HIGH", + "GC_URGENT_LOW", + "GC_URGENT_MID" +}; + +struct f2fs_attr { + struct attribute attr; + ssize_t (*show)(struct f2fs_attr *a, struct f2fs_sb_info *sbi, char *buf); + ssize_t (*store)(struct f2fs_attr *a, struct f2fs_sb_info *sbi, + const char *buf, size_t len); + int struct_type; + int offset; + int size; + int id; +}; + +struct f2fs_base_attr { + struct attribute attr; + ssize_t (*show)(struct f2fs_base_attr *a, char *buf); + ssize_t (*store)(struct f2fs_base_attr *a, const char *buf, size_t len); +}; + +static ssize_t f2fs_sbi_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf); + +static unsigned char *__struct_ptr(struct f2fs_sb_info *sbi, int struct_type) +{ + if (struct_type == GC_THREAD) + return (unsigned char *)sbi->gc_thread; + else if (struct_type == SM_INFO) + return (unsigned char *)SM_I(sbi); + else if (struct_type == DCC_INFO) + return (unsigned char *)SM_I(sbi)->dcc_info; + else if (struct_type == NM_INFO) + return (unsigned char *)NM_I(sbi); + else if (struct_type == F2FS_SBI || struct_type == RESERVED_BLOCKS) + return (unsigned char *)sbi; +#ifdef CONFIG_F2FS_FAULT_INJECTION + else if (struct_type == FAULT_INFO_RATE || + struct_type == FAULT_INFO_TYPE || + struct_type == FAULT_INFO_TIMEOUT) + return (unsigned char *)&F2FS_OPTION(sbi).fault_info; +#endif +#ifdef CONFIG_F2FS_STAT_FS + else if (struct_type == STAT_INFO) + return (unsigned char *)F2FS_STAT(sbi); +#endif + else if (struct_type == CPRC_INFO) + return (unsigned char *)&sbi->cprc_info; + else if (struct_type == ATGC_INFO) + return (unsigned char *)&sbi->am; + return NULL; +} + +static ssize_t dirty_segments_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + return sysfs_emit(buf, "%llu\n", + (unsigned long long)(dirty_segments(sbi))); +} + +static ssize_t free_segments_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + return sysfs_emit(buf, "%llu\n", + (unsigned long long)(free_segments(sbi))); +} + +static ssize_t ovp_segments_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + return sysfs_emit(buf, "%llu\n", + (unsigned long long)(overprovision_segments(sbi))); +} + +static ssize_t lifetime_write_kbytes_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + return sysfs_emit(buf, "%llu\n", + (unsigned long long)(sbi->kbytes_written + + ((f2fs_get_sectors_written(sbi) - + sbi->sectors_written_start) >> 1))); +} + +static ssize_t sb_status_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + return sysfs_emit(buf, "%lx\n", sbi->s_flag); +} + +static ssize_t cp_status_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + return sysfs_emit(buf, "%x\n", le32_to_cpu(F2FS_CKPT(sbi)->ckpt_flags)); +} + +static ssize_t pending_discard_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + if (!SM_I(sbi)->dcc_info) + return -EINVAL; + return sysfs_emit(buf, "%llu\n", (unsigned long long)atomic_read( + &SM_I(sbi)->dcc_info->discard_cmd_cnt)); +} + +static ssize_t issued_discard_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + if (!SM_I(sbi)->dcc_info) + return -EINVAL; + return sysfs_emit(buf, "%llu\n", (unsigned long long)atomic_read( + &SM_I(sbi)->dcc_info->issued_discard)); +} + +static ssize_t queued_discard_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + if (!SM_I(sbi)->dcc_info) + return -EINVAL; + return sysfs_emit(buf, "%llu\n", (unsigned long long)atomic_read( + &SM_I(sbi)->dcc_info->queued_discard)); +} + +static ssize_t undiscard_blks_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + if (!SM_I(sbi)->dcc_info) + return -EINVAL; + return sysfs_emit(buf, "%u\n", + SM_I(sbi)->dcc_info->undiscard_blks); +} + +static ssize_t atgc_enabled_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + return sysfs_emit(buf, "%d\n", sbi->am.atgc_enabled ? 1 : 0); +} + +static ssize_t gc_mode_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + return sysfs_emit(buf, "%s\n", gc_mode_names[sbi->gc_mode]); +} + +static ssize_t features_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + int len = 0; + + if (f2fs_sb_has_encrypt(sbi)) + len += sysfs_emit_at(buf, len, "%s", + "encryption"); + if (f2fs_sb_has_blkzoned(sbi)) + len += sysfs_emit_at(buf, len, "%s%s", + len ? ", " : "", "blkzoned"); + if (f2fs_sb_has_extra_attr(sbi)) + len += sysfs_emit_at(buf, len, "%s%s", + len ? ", " : "", "extra_attr"); + if (f2fs_sb_has_project_quota(sbi)) + len += sysfs_emit_at(buf, len, "%s%s", + len ? ", " : "", "projquota"); + if (f2fs_sb_has_inode_chksum(sbi)) + len += sysfs_emit_at(buf, len, "%s%s", + len ? ", " : "", "inode_checksum"); + if (f2fs_sb_has_flexible_inline_xattr(sbi)) + len += sysfs_emit_at(buf, len, "%s%s", + len ? ", " : "", "flexible_inline_xattr"); + if (f2fs_sb_has_quota_ino(sbi)) + len += sysfs_emit_at(buf, len, "%s%s", + len ? ", " : "", "quota_ino"); + if (f2fs_sb_has_inode_crtime(sbi)) + len += sysfs_emit_at(buf, len, "%s%s", + len ? ", " : "", "inode_crtime"); + if (f2fs_sb_has_lost_found(sbi)) + len += sysfs_emit_at(buf, len, "%s%s", + len ? ", " : "", "lost_found"); + if (f2fs_sb_has_verity(sbi)) + len += sysfs_emit_at(buf, len, "%s%s", + len ? ", " : "", "verity"); + if (f2fs_sb_has_sb_chksum(sbi)) + len += sysfs_emit_at(buf, len, "%s%s", + len ? ", " : "", "sb_checksum"); + if (f2fs_sb_has_casefold(sbi)) + len += sysfs_emit_at(buf, len, "%s%s", + len ? ", " : "", "casefold"); + if (f2fs_sb_has_readonly(sbi)) + len += sysfs_emit_at(buf, len, "%s%s", + len ? ", " : "", "readonly"); + if (f2fs_sb_has_compression(sbi)) + len += sysfs_emit_at(buf, len, "%s%s", + len ? ", " : "", "compression"); + if (f2fs_sb_has_packed_ssa(sbi)) + len += sysfs_emit_at(buf, len, "%s%s", + len ? ", " : "", "packed_ssa"); + len += sysfs_emit_at(buf, len, "%s%s", + len ? ", " : "", "pin_file"); + len += sysfs_emit_at(buf, len, "\n"); + return len; +} + +static ssize_t current_reserved_blocks_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + return sysfs_emit(buf, "%u\n", sbi->current_reserved_blocks); +} + +static ssize_t unusable_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + block_t unusable; + + if (test_opt(sbi, DISABLE_CHECKPOINT)) + unusable = sbi->unusable_block_count; + else + unusable = f2fs_get_unusable_blocks(sbi); + return sysfs_emit(buf, "%llu\n", (unsigned long long)unusable); +} + +static ssize_t encoding_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ +#if IS_ENABLED(CONFIG_UNICODE) + struct super_block *sb = sbi->sb; + + if (f2fs_sb_has_casefold(sbi)) + return sysfs_emit(buf, "UTF-8 (%d.%d.%d)\n", + (sb->s_encoding->version >> 16) & 0xff, + (sb->s_encoding->version >> 8) & 0xff, + sb->s_encoding->version & 0xff); +#endif + return sysfs_emit(buf, "(none)\n"); +} + +static ssize_t encoding_flags_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + return sysfs_emit(buf, "%x\n", + le16_to_cpu(F2FS_RAW_SUPER(sbi)->s_encoding_flags)); +} + +static ssize_t effective_lookup_mode_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + switch (F2FS_OPTION(sbi).lookup_mode) { + case LOOKUP_PERF: + return sysfs_emit(buf, "perf\n"); + case LOOKUP_COMPAT: + return sysfs_emit(buf, "compat\n"); + case LOOKUP_AUTO: + if (sb_no_casefold_compat_fallback(sbi->sb)) + return sysfs_emit(buf, "auto:perf\n"); + return sysfs_emit(buf, "auto:compat\n"); + } + return 0; +} + +static ssize_t mounted_time_sec_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + return sysfs_emit(buf, "%llu\n", SIT_I(sbi)->mounted_time); +} + +#ifdef CONFIG_F2FS_STAT_FS +static ssize_t moved_blocks_foreground_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + struct f2fs_stat_info *si = F2FS_STAT(sbi); + + return sysfs_emit(buf, "%llu\n", + (unsigned long long)(si->tot_blks - + (si->bg_data_blks + si->bg_node_blks))); +} + +static ssize_t moved_blocks_background_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + struct f2fs_stat_info *si = F2FS_STAT(sbi); + + return sysfs_emit(buf, "%llu\n", + (unsigned long long)(si->bg_data_blks + si->bg_node_blks)); +} + +static ssize_t avg_vblocks_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + struct f2fs_stat_info *si = F2FS_STAT(sbi); + + si->dirty_count = dirty_segments(sbi); + f2fs_update_sit_info(sbi); + return sysfs_emit(buf, "%llu\n", (unsigned long long)(si->avg_vblocks)); +} + +static ssize_t defrag_blocks_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + struct f2fs_stat_info *si = F2FS_STAT(sbi); + + return sysfs_emit(buf, "%llu\n", (unsigned long long)(si->defrag_blks)); +} +#endif + +static ssize_t main_blkaddr_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + return sysfs_emit(buf, "%llu\n", + (unsigned long long)MAIN_BLKADDR(sbi)); +} + +static ssize_t __sbi_show_value(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf, + unsigned char *value) +{ + switch (a->size) { + case 1: + return sysfs_emit(buf, "%u\n", *(u8 *)value); + case 2: + return sysfs_emit(buf, "%u\n", *(u16 *)value); + case 4: + return sysfs_emit(buf, "%u\n", *(u32 *)value); + case 8: + return sysfs_emit(buf, "%llu\n", *(u64 *)value); + default: + f2fs_bug_on(sbi, 1); + return sysfs_emit(buf, + "show sysfs node value with wrong type\n"); + } +} + +static ssize_t f2fs_sbi_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + unsigned char *ptr = NULL; + + ptr = __struct_ptr(sbi, a->struct_type); + if (!ptr) + return -EINVAL; + + if (!strcmp(a->attr.name, "extension_list")) { + __u8 (*extlist)[F2FS_EXTENSION_LEN] = + sbi->raw_super->extension_list; + int cold_count, hot_count; + int len = 0, i; + + f2fs_down_read(&sbi->sb_lock); + cold_count = le32_to_cpu(sbi->raw_super->extension_count); + hot_count = sbi->raw_super->hot_ext_count; + len += sysfs_emit_at(buf, len, "cold file extension:\n"); + for (i = 0; i < cold_count; i++) + len += sysfs_emit_at(buf, len, "%s\n", extlist[i]); + + len += sysfs_emit_at(buf, len, "hot file extension:\n"); + for (i = cold_count; i < cold_count + hot_count; i++) + len += sysfs_emit_at(buf, len, "%s\n", extlist[i]); + f2fs_up_read(&sbi->sb_lock); + + return len; + } + + if (!strcmp(a->attr.name, "ckpt_thread_ioprio")) { + struct ckpt_req_control *cprc = &sbi->cprc_info; + int class = IOPRIO_PRIO_CLASS(cprc->ckpt_thread_ioprio); + int level = IOPRIO_PRIO_LEVEL(cprc->ckpt_thread_ioprio); + + if (class != IOPRIO_CLASS_RT && class != IOPRIO_CLASS_BE) + return -EINVAL; + + return sysfs_emit(buf, "%s,%d\n", + class == IOPRIO_CLASS_RT ? "rt" : "be", level); + } + +#ifdef CONFIG_F2FS_FS_COMPRESSION + if (!strcmp(a->attr.name, "compr_written_block")) + return sysfs_emit(buf, "%llu\n", sbi->compr_written_block); + + if (!strcmp(a->attr.name, "compr_saved_block")) + return sysfs_emit(buf, "%llu\n", sbi->compr_saved_block); + + if (!strcmp(a->attr.name, "compr_new_inode")) + return sysfs_emit(buf, "%u\n", sbi->compr_new_inode); +#endif + + if (!strcmp(a->attr.name, "gc_segment_mode")) + return sysfs_emit(buf, "%u\n", sbi->gc_segment_mode); + + if (!strcmp(a->attr.name, "gc_reclaimed_segments")) { + return sysfs_emit(buf, "%u\n", + sbi->gc_reclaimed_segs[sbi->gc_segment_mode]); + } + + if (!strcmp(a->attr.name, "current_atomic_write")) { + s64 current_write = atomic64_read(&sbi->current_atomic_write); + + return sysfs_emit(buf, "%lld\n", current_write); + } + + if (!strcmp(a->attr.name, "peak_atomic_write")) + return sysfs_emit(buf, "%lld\n", sbi->peak_atomic_write); + + if (!strcmp(a->attr.name, "committed_atomic_block")) + return sysfs_emit(buf, "%llu\n", sbi->committed_atomic_block); + + if (!strcmp(a->attr.name, "revoked_atomic_block")) + return sysfs_emit(buf, "%llu\n", sbi->revoked_atomic_block); + +#ifdef CONFIG_F2FS_STAT_FS + if (!strcmp(a->attr.name, "cp_foreground_calls")) + return sysfs_emit(buf, "%d\n", + atomic_read(&sbi->cp_call_count[TOTAL_CALL]) - + atomic_read(&sbi->cp_call_count[BACKGROUND])); + if (!strcmp(a->attr.name, "cp_background_calls")) + return sysfs_emit(buf, "%d\n", + atomic_read(&sbi->cp_call_count[BACKGROUND])); +#endif + + return __sbi_show_value(a, sbi, buf, ptr + a->offset); +} + +static void __sbi_store_value(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, + unsigned char *ui, unsigned long value) +{ + switch (a->size) { + case 1: + *(u8 *)ui = value; + break; + case 2: + *(u16 *)ui = value; + break; + case 4: + *(u32 *)ui = value; + break; + case 8: + *(u64 *)ui = value; + break; + default: + f2fs_bug_on(sbi, 1); + f2fs_err(sbi, "store sysfs node value with wrong type"); + } +} + +static ssize_t __sbi_store(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, + const char *buf, size_t count) +{ + unsigned char *ptr; + unsigned long t; + unsigned int *ui; + ssize_t ret; + + ptr = __struct_ptr(sbi, a->struct_type); + if (!ptr) + return -EINVAL; + + if (!strcmp(a->attr.name, "extension_list")) { + const char *name = strim((char *)buf); + bool set = true, hot; + + if (!strncmp(name, "[h]", 3)) + hot = true; + else if (!strncmp(name, "[c]", 3)) + hot = false; + else + return -EINVAL; + + name += 3; + + if (*name == '!') { + name++; + set = false; + } + + if (!strlen(name) || strlen(name) >= F2FS_EXTENSION_LEN) + return -EINVAL; + + f2fs_down_write(&sbi->sb_lock); + + ret = f2fs_update_extension_list(sbi, name, hot, set); + if (ret) + goto out; + + ret = f2fs_commit_super(sbi, false); + if (ret) + f2fs_update_extension_list(sbi, name, hot, !set); +out: + f2fs_up_write(&sbi->sb_lock); + return ret ? ret : count; + } + + if (!strcmp(a->attr.name, "ckpt_thread_ioprio")) { + const char *name = strim((char *)buf); + struct ckpt_req_control *cprc = &sbi->cprc_info; + int class; + long level; + int ret; + + if (!strncmp(name, "rt,", 3)) + class = IOPRIO_CLASS_RT; + else if (!strncmp(name, "be,", 3)) + class = IOPRIO_CLASS_BE; + else + return -EINVAL; + + name += 3; + ret = kstrtol(name, 10, &level); + if (ret) + return ret; + if (level >= IOPRIO_NR_LEVELS || level < 0) + return -EINVAL; + + cprc->ckpt_thread_ioprio = IOPRIO_PRIO_VALUE(class, level); + if (test_opt(sbi, MERGE_CHECKPOINT)) { + ret = set_task_ioprio(cprc->f2fs_issue_ckpt, + cprc->ckpt_thread_ioprio); + if (ret) + return ret; + } + + return count; + } + + ui = (unsigned int *)(ptr + a->offset); + + ret = kstrtoul(skip_spaces(buf), 0, &t); + if (ret < 0) + return ret; +#ifdef CONFIG_F2FS_FAULT_INJECTION + if (a->struct_type == FAULT_INFO_TYPE) { + if (f2fs_build_fault_attr(sbi, 0, t, FAULT_TYPE)) + return -EINVAL; + return count; + } + if (a->struct_type == FAULT_INFO_RATE) { + if (f2fs_build_fault_attr(sbi, t, 0, FAULT_RATE)) + return -EINVAL; + return count; + } + if (a->struct_type == FAULT_INFO_TIMEOUT) { + if (f2fs_build_fault_attr(sbi, 0, t, FAULT_TIMEOUT)) + return -EINVAL; + f2fs_simulate_lock_timeout(sbi); + return count; + } +#endif + if (a->struct_type == RESERVED_BLOCKS) { + spin_lock(&sbi->stat_lock); + if (t > (unsigned long)(sbi->user_block_count - + F2FS_OPTION(sbi).root_reserved_blocks)) { + spin_unlock(&sbi->stat_lock); + return -EINVAL; + } + *ui = t; + sbi->current_reserved_blocks = min(sbi->reserved_blocks, + sbi->user_block_count - valid_user_blocks(sbi)); + spin_unlock(&sbi->stat_lock); + return count; + } + + if (!strcmp(a->attr.name, "discard_io_aware_gran")) { + if (t > MAX_PLIST_NUM) + return -EINVAL; + if (!f2fs_block_unit_discard(sbi)) + return -EINVAL; + if (t == *ui) + return count; + *ui = t; + return count; + } + + if (!strcmp(a->attr.name, "discard_granularity")) { + if (t == 0 || t > MAX_PLIST_NUM) + return -EINVAL; + if (!f2fs_block_unit_discard(sbi)) + return -EINVAL; + if (t == *ui) + return count; + *ui = t; + return count; + } + + if (!strcmp(a->attr.name, "max_ordered_discard")) { + if (t == 0 || t > MAX_PLIST_NUM) + return -EINVAL; + if (!f2fs_block_unit_discard(sbi)) + return -EINVAL; + *ui = t; + return count; + } + + if (!strcmp(a->attr.name, "discard_urgent_util")) { + if (t > 100) + return -EINVAL; + *ui = t; + return count; + } + + if (!strcmp(a->attr.name, "discard_io_aware")) { + if (t >= DPOLICY_IO_AWARE_MAX) + return -EINVAL; + *ui = t; + return count; + } + + if (!strcmp(a->attr.name, "migration_granularity")) { + if (t == 0 || t > SEGS_PER_SEC(sbi)) + return -EINVAL; + } + + if (!strcmp(a->attr.name, "migration_window_granularity")) { + if (t == 0 || t > SEGS_PER_SEC(sbi)) + return -EINVAL; + } + + if (!strcmp(a->attr.name, "gc_urgent")) { + if (t == 0) { + sbi->gc_mode = GC_NORMAL; + } else if (t == 1) { + sbi->gc_mode = GC_URGENT_HIGH; + if (sbi->gc_thread) { + sbi->gc_thread->gc_wake = true; + wake_up_interruptible_all( + &sbi->gc_thread->gc_wait_queue_head); + wake_up_discard_thread(sbi, true); + } + } else if (t == 2) { + sbi->gc_mode = GC_URGENT_LOW; + } else if (t == 3) { + sbi->gc_mode = GC_URGENT_MID; + if (sbi->gc_thread) { + sbi->gc_thread->gc_wake = true; + wake_up_interruptible_all( + &sbi->gc_thread->gc_wait_queue_head); + } + } else { + return -EINVAL; + } + return count; + } + if (!strcmp(a->attr.name, "gc_idle")) { + if (t == GC_IDLE_CB) { + sbi->gc_mode = GC_IDLE_CB; + } else if (t == GC_IDLE_GREEDY) { + sbi->gc_mode = GC_IDLE_GREEDY; + } else if (t == GC_IDLE_AT) { + if (!sbi->am.atgc_enabled) + return -EINVAL; + sbi->gc_mode = GC_IDLE_AT; + } else { + sbi->gc_mode = GC_NORMAL; + } + return count; + } + + if (!strcmp(a->attr.name, "gc_remaining_trials")) { + spin_lock(&sbi->gc_remaining_trials_lock); + sbi->gc_remaining_trials = t; + spin_unlock(&sbi->gc_remaining_trials_lock); + + return count; + } + + if (!strcmp(a->attr.name, "gc_no_zoned_gc_percent")) { + if (t > 100) + return -EINVAL; + *ui = (unsigned int)t; + return count; + } + + if (!strcmp(a->attr.name, "gc_boost_zoned_gc_percent")) { + if (t > 100) + return -EINVAL; + *ui = (unsigned int)t; + return count; + } + + if (!strcmp(a->attr.name, "gc_valid_thresh_ratio")) { + if (t > 100) + return -EINVAL; + *ui = (unsigned int)t; + return count; + } + +#ifdef CONFIG_F2FS_IOSTAT + if (!strcmp(a->attr.name, "iostat_enable")) { + sbi->iostat_enable = !!t; + if (!sbi->iostat_enable) + f2fs_reset_iostat(sbi); + return count; + } + + if (!strcmp(a->attr.name, "iostat_period_ms")) { + if (t < MIN_IOSTAT_PERIOD_MS || t > MAX_IOSTAT_PERIOD_MS) + return -EINVAL; + spin_lock_irq(&sbi->iostat_lock); + sbi->iostat_period_ms = (unsigned int)t; + spin_unlock_irq(&sbi->iostat_lock); + return count; + } +#endif + +#ifdef CONFIG_BLK_DEV_ZONED + if (!strcmp(a->attr.name, "blkzone_alloc_policy")) { + if (t < BLKZONE_ALLOC_PRIOR_SEQ || t > BLKZONE_ALLOC_PRIOR_CONV) + return -EINVAL; + sbi->blkzone_alloc_policy = t; + return count; + } +#endif + +#ifdef CONFIG_F2FS_FS_COMPRESSION + if (!strcmp(a->attr.name, "compr_written_block") || + !strcmp(a->attr.name, "compr_saved_block")) { + if (t != 0) + return -EINVAL; + sbi->compr_written_block = 0; + sbi->compr_saved_block = 0; + return count; + } + + if (!strcmp(a->attr.name, "compr_new_inode")) { + if (t != 0) + return -EINVAL; + sbi->compr_new_inode = 0; + return count; + } + + if (!strcmp(a->attr.name, "compress_percent")) { + if (t == 0 || t > 100) + return -EINVAL; + *ui = t; + return count; + } + + if (!strcmp(a->attr.name, "compress_watermark")) { + if (t == 0 || t > 100) + return -EINVAL; + *ui = t; + return count; + } +#endif + + if (!strcmp(a->attr.name, "atgc_candidate_ratio")) { + if (t > 100) + return -EINVAL; + sbi->am.candidate_ratio = t; + return count; + } + + if (!strcmp(a->attr.name, "atgc_age_weight")) { + if (t > 100) + return -EINVAL; + sbi->am.age_weight = t; + return count; + } + + if (!strcmp(a->attr.name, "gc_segment_mode")) { + if (t < MAX_GC_MODE) + sbi->gc_segment_mode = t; + else + return -EINVAL; + return count; + } + + if (!strcmp(a->attr.name, "gc_pin_file_thresh")) { + if (t > MAX_GC_FAILED_PINNED_FILES) + return -EINVAL; + sbi->gc_pin_file_threshold = t; + return count; + } + + if (!strcmp(a->attr.name, "gc_reclaimed_segments")) { + if (t != 0) + return -EINVAL; + sbi->gc_reclaimed_segs[sbi->gc_segment_mode] = 0; + return count; + } + + if (!strcmp(a->attr.name, "seq_file_ra_mul")) { + if (t >= MIN_RA_MUL && t <= MAX_RA_MUL) + sbi->seq_file_ra_mul = t; + else + return -EINVAL; + return count; + } + + if (!strcmp(a->attr.name, "max_fragment_chunk")) { + if (t >= MIN_FRAGMENT_SIZE && t <= MAX_FRAGMENT_SIZE) + sbi->max_fragment_chunk = t; + else + return -EINVAL; + return count; + } + + if (!strcmp(a->attr.name, "max_fragment_hole")) { + if (t >= MIN_FRAGMENT_SIZE && t <= MAX_FRAGMENT_SIZE) + sbi->max_fragment_hole = t; + else + return -EINVAL; + return count; + } + + if (!strcmp(a->attr.name, "peak_atomic_write")) { + if (t != 0) + return -EINVAL; + sbi->peak_atomic_write = 0; + return count; + } + + if (!strcmp(a->attr.name, "committed_atomic_block")) { + if (t != 0) + return -EINVAL; + sbi->committed_atomic_block = 0; + return count; + } + + if (!strcmp(a->attr.name, "revoked_atomic_block")) { + if (t != 0) + return -EINVAL; + sbi->revoked_atomic_block = 0; + return count; + } + + if (!strcmp(a->attr.name, "readdir_ra")) { + sbi->readdir_ra = !!t; + return count; + } + + if (!strcmp(a->attr.name, "hot_data_age_threshold")) { + if (t == 0 || t >= sbi->warm_data_age_threshold) + return -EINVAL; + if (t == *ui) + return count; + *ui = (unsigned int)t; + return count; + } + + if (!strcmp(a->attr.name, "warm_data_age_threshold")) { + if (t <= sbi->hot_data_age_threshold) + return -EINVAL; + if (t == *ui) + return count; + *ui = (unsigned int)t; + return count; + } + + if (!strcmp(a->attr.name, "last_age_weight")) { + if (t > 100) + return -EINVAL; + if (t == *ui) + return count; + *ui = (unsigned int)t; + return count; + } + + if (!strcmp(a->attr.name, "max_read_extent_count")) { + if (t > UINT_MAX) + return -EINVAL; + *ui = (unsigned int)t; + return count; + } + + if (!strcmp(a->attr.name, "ipu_policy")) { + if (t >= BIT(F2FS_IPU_MAX)) + return -EINVAL; + /* allow F2FS_IPU_NOCACHE only for IPU in the pinned file */ + if (f2fs_lfs_mode(sbi) && (t & ~BIT(F2FS_IPU_NOCACHE))) + return -EINVAL; + SM_I(sbi)->ipu_policy = (unsigned int)t; + return count; + } + + if (!strcmp(a->attr.name, "dir_level")) { + if (t > MAX_DIR_HASH_DEPTH) + return -EINVAL; + sbi->dir_level = t; + return count; + } + + if (!strcmp(a->attr.name, "reserved_pin_section")) { + if (t > GET_SEC_FROM_SEG(sbi, overprovision_segments(sbi))) + return -EINVAL; + *ui = (unsigned int)t; + return count; + } + + if (!strcmp(a->attr.name, "gc_boost_gc_multiple")) { + if (t < 1 || t > SEGS_PER_SEC(sbi)) + return -EINVAL; + sbi->gc_thread->boost_gc_multiple = (unsigned int)t; + return count; + } + + if (!strcmp(a->attr.name, "gc_boost_gc_greedy")) { + if (t > GC_GREEDY) + return -EINVAL; + sbi->gc_thread->boost_gc_greedy = (unsigned int)t; + return count; + } + + if (!strcmp(a->attr.name, "bggc_io_aware")) { + if (t < AWARE_ALL_IO || t > AWARE_NONE) + return -EINVAL; + sbi->bggc_io_aware = t; + return count; + } + + if (!strcmp(a->attr.name, "allocate_section_hint")) { + if (t < 0 || t > MAIN_SECS(sbi)) + return -EINVAL; + sbi->allocate_section_hint = t; + return count; + } + + if (!strcmp(a->attr.name, "allocate_section_policy")) { + if (t < ALLOCATE_FORWARD_NOHINT || t > ALLOCATE_FORWARD_FROM_HINT) + return -EINVAL; + sbi->allocate_section_policy = t; + return count; + } + + if (!strcmp(a->attr.name, "adjust_lock_priority")) { + if (t >= BIT(LOCK_NAME_MAX - 1)) + return -EINVAL; + sbi->adjust_lock_priority = t; + return count; + } + + if (!strcmp(a->attr.name, "lock_duration_priority")) { + if (t < NICE_TO_PRIO(MIN_NICE) || t > NICE_TO_PRIO(MAX_NICE)) + return -EINVAL; + sbi->lock_duration_priority = t; + return count; + } + + if (!strcmp(a->attr.name, "critical_task_priority")) { + if (t < NICE_TO_PRIO(MIN_NICE) || t > NICE_TO_PRIO(MAX_NICE)) + return -EINVAL; + if (!capable(CAP_SYS_NICE)) + return -EPERM; + sbi->critical_task_priority = t; + if (sbi->cprc_info.f2fs_issue_ckpt) + set_user_nice(sbi->cprc_info.f2fs_issue_ckpt, + PRIO_TO_NICE(sbi->critical_task_priority)); + if (sbi->gc_thread && sbi->gc_thread->f2fs_gc_task) + set_user_nice(sbi->gc_thread->f2fs_gc_task, + PRIO_TO_NICE(sbi->critical_task_priority)); + return count; + } + + __sbi_store_value(a, sbi, ptr + a->offset, t); + + return count; +} + +static ssize_t f2fs_sbi_store(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, + const char *buf, size_t count) +{ + ssize_t ret; + bool gc_entry = (!strcmp(a->attr.name, "gc_urgent") || + a->struct_type == GC_THREAD); + + if (gc_entry) { + if (!down_read_trylock(&sbi->sb->s_umount)) + return -EAGAIN; + } + ret = __sbi_store(a, sbi, buf, count); + if (gc_entry) + up_read(&sbi->sb->s_umount); + + return ret; +} + +static ssize_t f2fs_attr_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct f2fs_sb_info *sbi = container_of(kobj, struct f2fs_sb_info, + s_kobj); + struct f2fs_attr *a = container_of(attr, struct f2fs_attr, attr); + + return a->show ? a->show(a, sbi, buf) : 0; +} + +static ssize_t f2fs_attr_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t len) +{ + struct f2fs_sb_info *sbi = container_of(kobj, struct f2fs_sb_info, + s_kobj); + struct f2fs_attr *a = container_of(attr, struct f2fs_attr, attr); + + return a->store ? a->store(a, sbi, buf, len) : 0; +} + +static void f2fs_sb_release(struct kobject *kobj) +{ + struct f2fs_sb_info *sbi = container_of(kobj, struct f2fs_sb_info, + s_kobj); + complete(&sbi->s_kobj_unregister); +} + +static ssize_t f2fs_base_attr_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct f2fs_base_attr *a = container_of(attr, + struct f2fs_base_attr, attr); + + return a->show ? a->show(a, buf) : 0; +} + +static ssize_t f2fs_base_attr_store(struct kobject *kobj, + struct attribute *attr, + const char *buf, size_t len) +{ + struct f2fs_base_attr *a = container_of(attr, + struct f2fs_base_attr, attr); + + return a->store ? a->store(a, buf, len) : 0; +} + +/* + * Note that there are three feature list entries: + * 1) /sys/fs/f2fs/features + * : shows runtime features supported by in-kernel f2fs along with Kconfig. + * - ref. F2FS_FEATURE_RO_ATTR() + * + * 2) /sys/fs/f2fs/$s_id/features + * : shows on-disk features enabled by mkfs.f2fs, used for old kernels. This + * won't add new feature anymore, and thus, users should check entries in 3) + * instead of this 2). + * + * 3) /sys/fs/f2fs/$s_id/feature_list + * : shows on-disk features enabled by mkfs.f2fs per instance, which follows + * sysfs entry rule where each entry should expose single value. + * This list covers old feature list provided by 2) and beyond. Therefore, + * please add new on-disk feature in this list only. + * - ref. F2FS_SB_FEATURE_RO_ATTR() + */ +static ssize_t f2fs_feature_show(struct f2fs_base_attr *a, char *buf) +{ + return sysfs_emit(buf, "supported\n"); +} + +#define F2FS_FEATURE_RO_ATTR(_name) \ +static struct f2fs_base_attr f2fs_base_attr_##_name = { \ + .attr = {.name = __stringify(_name), .mode = 0444 }, \ + .show = f2fs_feature_show, \ +} + +static ssize_t f2fs_tune_show(struct f2fs_base_attr *a, char *buf) +{ + unsigned int res = 0; + + if (!strcmp(a->attr.name, "reclaim_caches_kb")) + res = f2fs_donate_files(); + + return sysfs_emit(buf, "%u\n", res); +} + +static ssize_t f2fs_tune_store(struct f2fs_base_attr *a, + const char *buf, size_t count) +{ + unsigned long t; + int ret; + + ret = kstrtoul(skip_spaces(buf), 0, &t); + if (ret) + return ret; + + if (!strcmp(a->attr.name, "reclaim_caches_kb")) + f2fs_reclaim_caches(t); + + return count; +} + +#define F2FS_TUNE_RW_ATTR(_name) \ +static struct f2fs_base_attr f2fs_base_attr_##_name = { \ + .attr = {.name = __stringify(_name), .mode = 0644 }, \ + .show = f2fs_tune_show, \ + .store = f2fs_tune_store, \ +} + +static ssize_t f2fs_sb_feature_show(struct f2fs_attr *a, + struct f2fs_sb_info *sbi, char *buf) +{ + if (F2FS_HAS_FEATURE(sbi, a->id)) + return sysfs_emit(buf, "supported\n"); + return sysfs_emit(buf, "unsupported\n"); +} + +#define F2FS_SB_FEATURE_RO_ATTR(_name, _feat) \ +static struct f2fs_attr f2fs_attr_sb_##_name = { \ + .attr = {.name = __stringify(_name), .mode = 0444 }, \ + .show = f2fs_sb_feature_show, \ + .id = F2FS_FEATURE_##_feat, \ +} + +#define F2FS_ATTR_OFFSET(_struct_type, _name, _mode, _show, _store, _offset, _size) \ +static struct f2fs_attr f2fs_attr_##_name = { \ + .attr = {.name = __stringify(_name), .mode = _mode }, \ + .show = _show, \ + .store = _store, \ + .struct_type = _struct_type, \ + .offset = _offset, \ + .size = _size \ +} + +#define F2FS_RO_ATTR(struct_type, struct_name, name, elname) \ + F2FS_ATTR_OFFSET(struct_type, name, 0444, \ + f2fs_sbi_show, NULL, \ + offsetof(struct struct_name, elname), \ + sizeof_field(struct struct_name, elname)) + +#define F2FS_RW_ATTR(struct_type, struct_name, name, elname) \ + F2FS_ATTR_OFFSET(struct_type, name, 0644, \ + f2fs_sbi_show, f2fs_sbi_store, \ + offsetof(struct struct_name, elname), \ + sizeof_field(struct struct_name, elname)) + +#define F2FS_GENERAL_RO_ATTR(name) \ +static struct f2fs_attr f2fs_attr_##name = __ATTR(name, 0444, name##_show, NULL) + +#ifdef CONFIG_F2FS_STAT_FS +#define STAT_INFO_RO_ATTR(name, elname) \ + F2FS_RO_ATTR(STAT_INFO, f2fs_stat_info, name, elname) +#endif + +#define GC_THREAD_RW_ATTR(name, elname) \ + F2FS_RW_ATTR(GC_THREAD, f2fs_gc_kthread, name, elname) + +#define SM_INFO_RW_ATTR(name, elname) \ + F2FS_RW_ATTR(SM_INFO, f2fs_sm_info, name, elname) + +#define SM_INFO_GENERAL_RW_ATTR(elname) \ + SM_INFO_RW_ATTR(elname, elname) + +#define DCC_INFO_RW_ATTR(name, elname) \ + F2FS_RW_ATTR(DCC_INFO, discard_cmd_control, name, elname) + +#define DCC_INFO_GENERAL_RW_ATTR(elname) \ + DCC_INFO_RW_ATTR(elname, elname) + +#define NM_INFO_RW_ATTR(name, elname) \ + F2FS_RW_ATTR(NM_INFO, f2fs_nm_info, name, elname) + +#define NM_INFO_GENERAL_RW_ATTR(elname) \ + NM_INFO_RW_ATTR(elname, elname) + +#define F2FS_SBI_RW_ATTR(name, elname) \ + F2FS_RW_ATTR(F2FS_SBI, f2fs_sb_info, name, elname) + +#define F2FS_SBI_GENERAL_RW_ATTR(elname) \ + F2FS_SBI_RW_ATTR(elname, elname) + +#define F2FS_SBI_GENERAL_RO_ATTR(elname) \ + F2FS_RO_ATTR(F2FS_SBI, f2fs_sb_info, elname, elname) + +#ifdef CONFIG_F2FS_FAULT_INJECTION +#define FAULT_INFO_GENERAL_RW_ATTR(type, elname) \ + F2FS_RW_ATTR(type, f2fs_fault_info, elname, elname) +#endif + +#define RESERVED_BLOCKS_GENERAL_RW_ATTR(elname) \ + F2FS_RW_ATTR(RESERVED_BLOCKS, f2fs_sb_info, elname, elname) + +#define CPRC_INFO_GENERAL_RW_ATTR(elname) \ + F2FS_RW_ATTR(CPRC_INFO, ckpt_req_control, elname, elname) + +#define ATGC_INFO_RW_ATTR(name, elname) \ + F2FS_RW_ATTR(ATGC_INFO, atgc_management, name, elname) + +/* GC_THREAD ATTR */ +GC_THREAD_RW_ATTR(gc_urgent_sleep_time, urgent_sleep_time); +GC_THREAD_RW_ATTR(gc_min_sleep_time, min_sleep_time); +GC_THREAD_RW_ATTR(gc_max_sleep_time, max_sleep_time); +GC_THREAD_RW_ATTR(gc_no_gc_sleep_time, no_gc_sleep_time); +GC_THREAD_RW_ATTR(gc_no_zoned_gc_percent, no_zoned_gc_percent); +GC_THREAD_RW_ATTR(gc_boost_zoned_gc_percent, boost_zoned_gc_percent); +GC_THREAD_RW_ATTR(gc_valid_thresh_ratio, valid_thresh_ratio); +GC_THREAD_RW_ATTR(gc_boost_gc_multiple, boost_gc_multiple); +GC_THREAD_RW_ATTR(gc_boost_gc_greedy, boost_gc_greedy); + +/* SM_INFO ATTR */ +SM_INFO_RW_ATTR(reclaim_segments, rec_prefree_segments); +SM_INFO_GENERAL_RW_ATTR(ipu_policy); +SM_INFO_GENERAL_RW_ATTR(min_ipu_util); +SM_INFO_GENERAL_RW_ATTR(min_fsync_blocks); +SM_INFO_GENERAL_RW_ATTR(min_seq_blocks); +SM_INFO_GENERAL_RW_ATTR(min_hot_blocks); +SM_INFO_GENERAL_RW_ATTR(min_ssr_sections); +SM_INFO_GENERAL_RW_ATTR(reserved_segments); + +/* DCC_INFO ATTR */ +DCC_INFO_RW_ATTR(max_small_discards, max_discards); +DCC_INFO_GENERAL_RW_ATTR(max_discard_request); +DCC_INFO_GENERAL_RW_ATTR(min_discard_issue_time); +DCC_INFO_GENERAL_RW_ATTR(mid_discard_issue_time); +DCC_INFO_GENERAL_RW_ATTR(max_discard_issue_time); +DCC_INFO_GENERAL_RW_ATTR(discard_io_aware_gran); +DCC_INFO_GENERAL_RW_ATTR(discard_urgent_util); +DCC_INFO_GENERAL_RW_ATTR(discard_granularity); +DCC_INFO_GENERAL_RW_ATTR(max_ordered_discard); +DCC_INFO_GENERAL_RW_ATTR(discard_io_aware); + +/* NM_INFO ATTR */ +NM_INFO_RW_ATTR(max_roll_forward_node_blocks, max_rf_node_blocks); +NM_INFO_GENERAL_RW_ATTR(ram_thresh); +NM_INFO_GENERAL_RW_ATTR(ra_nid_pages); +NM_INFO_GENERAL_RW_ATTR(dirty_nats_ratio); + +/* F2FS_SBI ATTR */ +F2FS_RW_ATTR(F2FS_SBI, f2fs_super_block, extension_list, extension_list); +F2FS_SBI_RW_ATTR(gc_idle, gc_mode); +F2FS_SBI_RW_ATTR(gc_urgent, gc_mode); +F2FS_SBI_RW_ATTR(cp_interval, interval_time[CP_TIME]); +F2FS_SBI_RW_ATTR(idle_interval, interval_time[REQ_TIME]); +F2FS_SBI_RW_ATTR(discard_idle_interval, interval_time[DISCARD_TIME]); +F2FS_SBI_RW_ATTR(gc_idle_interval, interval_time[GC_TIME]); +F2FS_SBI_RW_ATTR(umount_discard_timeout, interval_time[UMOUNT_DISCARD_TIMEOUT]); +F2FS_SBI_RW_ATTR(gc_pin_file_thresh, gc_pin_file_threshold); +F2FS_SBI_RW_ATTR(gc_reclaimed_segments, gc_reclaimed_segs); +F2FS_SBI_GENERAL_RW_ATTR(max_victim_search); +F2FS_SBI_GENERAL_RW_ATTR(migration_granularity); +F2FS_SBI_GENERAL_RW_ATTR(migration_window_granularity); +F2FS_SBI_GENERAL_RW_ATTR(dir_level); +F2FS_SBI_GENERAL_RW_ATTR(allocate_section_hint); +F2FS_SBI_GENERAL_RW_ATTR(allocate_section_policy); +#ifdef CONFIG_F2FS_IOSTAT +F2FS_SBI_GENERAL_RW_ATTR(iostat_enable); +F2FS_SBI_GENERAL_RW_ATTR(iostat_period_ms); +#endif +F2FS_SBI_GENERAL_RW_ATTR(readdir_ra); +F2FS_SBI_GENERAL_RW_ATTR(max_io_bytes); +F2FS_SBI_GENERAL_RW_ATTR(data_io_flag); +F2FS_SBI_GENERAL_RW_ATTR(node_io_flag); +F2FS_SBI_GENERAL_RW_ATTR(gc_remaining_trials); +F2FS_SBI_GENERAL_RW_ATTR(seq_file_ra_mul); +F2FS_SBI_GENERAL_RW_ATTR(gc_segment_mode); +F2FS_SBI_GENERAL_RW_ATTR(max_fragment_chunk); +F2FS_SBI_GENERAL_RW_ATTR(max_fragment_hole); +#ifdef CONFIG_F2FS_FS_COMPRESSION +F2FS_SBI_GENERAL_RW_ATTR(compr_written_block); +F2FS_SBI_GENERAL_RW_ATTR(compr_saved_block); +F2FS_SBI_GENERAL_RW_ATTR(compr_new_inode); +F2FS_SBI_GENERAL_RW_ATTR(compress_percent); +F2FS_SBI_GENERAL_RW_ATTR(compress_watermark); +#endif +/* atomic write */ +F2FS_SBI_GENERAL_RO_ATTR(current_atomic_write); +F2FS_SBI_GENERAL_RW_ATTR(peak_atomic_write); +F2FS_SBI_GENERAL_RW_ATTR(committed_atomic_block); +F2FS_SBI_GENERAL_RW_ATTR(revoked_atomic_block); +/* block age extent cache */ +F2FS_SBI_GENERAL_RW_ATTR(hot_data_age_threshold); +F2FS_SBI_GENERAL_RW_ATTR(warm_data_age_threshold); +F2FS_SBI_GENERAL_RW_ATTR(last_age_weight); +/* read extent cache */ +F2FS_SBI_GENERAL_RW_ATTR(max_read_extent_count); +#ifdef CONFIG_BLK_DEV_ZONED +F2FS_SBI_GENERAL_RO_ATTR(unusable_blocks_per_sec); +F2FS_SBI_GENERAL_RO_ATTR(max_open_zones); +F2FS_SBI_GENERAL_RW_ATTR(blkzone_alloc_policy); +#endif +F2FS_SBI_GENERAL_RW_ATTR(carve_out); +F2FS_SBI_GENERAL_RW_ATTR(reserved_pin_section); +F2FS_SBI_GENERAL_RW_ATTR(bggc_io_aware); +F2FS_SBI_GENERAL_RW_ATTR(max_lock_elapsed_time); +F2FS_SBI_GENERAL_RW_ATTR(lock_duration_priority); +F2FS_SBI_GENERAL_RW_ATTR(adjust_lock_priority); +F2FS_SBI_GENERAL_RW_ATTR(critical_task_priority); + +/* STAT_INFO ATTR */ +#ifdef CONFIG_F2FS_STAT_FS +STAT_INFO_RO_ATTR(cp_foreground_calls, cp_call_count[FOREGROUND]); +STAT_INFO_RO_ATTR(cp_background_calls, cp_call_count[BACKGROUND]); +STAT_INFO_RO_ATTR(gc_foreground_calls, gc_call_count[FOREGROUND]); +STAT_INFO_RO_ATTR(gc_background_calls, gc_call_count[BACKGROUND]); +#endif + +/* FAULT_INFO ATTR */ +#ifdef CONFIG_F2FS_FAULT_INJECTION +FAULT_INFO_GENERAL_RW_ATTR(FAULT_INFO_RATE, inject_rate); +FAULT_INFO_GENERAL_RW_ATTR(FAULT_INFO_TYPE, inject_type); +FAULT_INFO_GENERAL_RW_ATTR(FAULT_INFO_TIMEOUT, inject_lock_timeout); +#endif + +/* RESERVED_BLOCKS ATTR */ +RESERVED_BLOCKS_GENERAL_RW_ATTR(reserved_blocks); + +/* CPRC_INFO ATTR */ +CPRC_INFO_GENERAL_RW_ATTR(ckpt_thread_ioprio); + +/* ATGC_INFO ATTR */ +ATGC_INFO_RW_ATTR(atgc_candidate_ratio, candidate_ratio); +ATGC_INFO_RW_ATTR(atgc_candidate_count, max_candidate_count); +ATGC_INFO_RW_ATTR(atgc_age_weight, age_weight); +ATGC_INFO_RW_ATTR(atgc_age_threshold, age_threshold); + +F2FS_GENERAL_RO_ATTR(dirty_segments); +F2FS_GENERAL_RO_ATTR(free_segments); +F2FS_GENERAL_RO_ATTR(ovp_segments); +F2FS_GENERAL_RO_ATTR(lifetime_write_kbytes); +F2FS_GENERAL_RO_ATTR(features); +F2FS_GENERAL_RO_ATTR(current_reserved_blocks); +F2FS_GENERAL_RO_ATTR(unusable); +F2FS_GENERAL_RO_ATTR(encoding); +F2FS_GENERAL_RO_ATTR(encoding_flags); +F2FS_GENERAL_RO_ATTR(effective_lookup_mode); +F2FS_GENERAL_RO_ATTR(mounted_time_sec); +F2FS_GENERAL_RO_ATTR(main_blkaddr); +F2FS_GENERAL_RO_ATTR(pending_discard); +F2FS_GENERAL_RO_ATTR(atgc_enabled); +F2FS_GENERAL_RO_ATTR(gc_mode); +#ifdef CONFIG_F2FS_STAT_FS +F2FS_GENERAL_RO_ATTR(moved_blocks_background); +F2FS_GENERAL_RO_ATTR(moved_blocks_foreground); +F2FS_GENERAL_RO_ATTR(avg_vblocks); +F2FS_GENERAL_RO_ATTR(defrag_blocks); +#endif + +#ifdef CONFIG_FS_ENCRYPTION +F2FS_FEATURE_RO_ATTR(encryption); +F2FS_FEATURE_RO_ATTR(test_dummy_encryption_v2); +#if IS_ENABLED(CONFIG_UNICODE) +F2FS_FEATURE_RO_ATTR(encrypted_casefold); +#endif +#endif /* CONFIG_FS_ENCRYPTION */ +#ifdef CONFIG_BLK_DEV_ZONED +F2FS_FEATURE_RO_ATTR(block_zoned); +#endif +F2FS_FEATURE_RO_ATTR(atomic_write); +F2FS_FEATURE_RO_ATTR(extra_attr); +F2FS_FEATURE_RO_ATTR(project_quota); +F2FS_FEATURE_RO_ATTR(inode_checksum); +F2FS_FEATURE_RO_ATTR(flexible_inline_xattr); +F2FS_FEATURE_RO_ATTR(quota_ino); +F2FS_FEATURE_RO_ATTR(inode_crtime); +F2FS_FEATURE_RO_ATTR(lost_found); +#ifdef CONFIG_FS_VERITY +F2FS_FEATURE_RO_ATTR(verity); +#endif +F2FS_FEATURE_RO_ATTR(sb_checksum); +#if IS_ENABLED(CONFIG_UNICODE) +F2FS_FEATURE_RO_ATTR(casefold); +#endif +F2FS_FEATURE_RO_ATTR(readonly); +#ifdef CONFIG_F2FS_FS_COMPRESSION +F2FS_FEATURE_RO_ATTR(compression); +#endif +F2FS_FEATURE_RO_ATTR(pin_file); +#ifdef CONFIG_UNICODE +F2FS_FEATURE_RO_ATTR(linear_lookup); +#endif +F2FS_FEATURE_RO_ATTR(packed_ssa); +F2FS_FEATURE_RO_ATTR(fserror); + +#define ATTR_LIST(name) (&f2fs_attr_##name.attr) +static struct attribute *f2fs_attrs[] = { + ATTR_LIST(gc_urgent_sleep_time), + ATTR_LIST(gc_min_sleep_time), + ATTR_LIST(gc_max_sleep_time), + ATTR_LIST(gc_no_gc_sleep_time), + ATTR_LIST(gc_no_zoned_gc_percent), + ATTR_LIST(gc_boost_zoned_gc_percent), + ATTR_LIST(gc_valid_thresh_ratio), + ATTR_LIST(gc_boost_gc_multiple), + ATTR_LIST(gc_boost_gc_greedy), + ATTR_LIST(gc_idle), + ATTR_LIST(gc_urgent), + ATTR_LIST(reclaim_segments), + ATTR_LIST(main_blkaddr), + ATTR_LIST(max_small_discards), + ATTR_LIST(max_discard_request), + ATTR_LIST(min_discard_issue_time), + ATTR_LIST(mid_discard_issue_time), + ATTR_LIST(max_discard_issue_time), + ATTR_LIST(discard_io_aware_gran), + ATTR_LIST(discard_urgent_util), + ATTR_LIST(discard_granularity), + ATTR_LIST(max_ordered_discard), + ATTR_LIST(discard_io_aware), + ATTR_LIST(pending_discard), + ATTR_LIST(gc_mode), + ATTR_LIST(ipu_policy), + ATTR_LIST(min_ipu_util), + ATTR_LIST(min_fsync_blocks), + ATTR_LIST(min_seq_blocks), + ATTR_LIST(min_hot_blocks), + ATTR_LIST(min_ssr_sections), + ATTR_LIST(reserved_segments), + ATTR_LIST(max_victim_search), + ATTR_LIST(migration_granularity), + ATTR_LIST(migration_window_granularity), + ATTR_LIST(dir_level), + ATTR_LIST(ram_thresh), + ATTR_LIST(ra_nid_pages), + ATTR_LIST(dirty_nats_ratio), + ATTR_LIST(max_roll_forward_node_blocks), + ATTR_LIST(cp_interval), + ATTR_LIST(idle_interval), + ATTR_LIST(discard_idle_interval), + ATTR_LIST(gc_idle_interval), + ATTR_LIST(umount_discard_timeout), + ATTR_LIST(bggc_io_aware), +#ifdef CONFIG_F2FS_IOSTAT + ATTR_LIST(iostat_enable), + ATTR_LIST(iostat_period_ms), +#endif + ATTR_LIST(readdir_ra), + ATTR_LIST(max_io_bytes), + ATTR_LIST(gc_pin_file_thresh), + ATTR_LIST(extension_list), +#ifdef CONFIG_F2FS_FAULT_INJECTION + ATTR_LIST(inject_rate), + ATTR_LIST(inject_type), + ATTR_LIST(inject_lock_timeout), +#endif + ATTR_LIST(data_io_flag), + ATTR_LIST(node_io_flag), + ATTR_LIST(gc_remaining_trials), + ATTR_LIST(ckpt_thread_ioprio), + ATTR_LIST(dirty_segments), + ATTR_LIST(free_segments), + ATTR_LIST(ovp_segments), + ATTR_LIST(unusable), + ATTR_LIST(lifetime_write_kbytes), + ATTR_LIST(features), + ATTR_LIST(reserved_blocks), + ATTR_LIST(current_reserved_blocks), + ATTR_LIST(encoding), + ATTR_LIST(encoding_flags), + ATTR_LIST(effective_lookup_mode), + ATTR_LIST(mounted_time_sec), +#ifdef CONFIG_F2FS_STAT_FS + ATTR_LIST(cp_foreground_calls), + ATTR_LIST(cp_background_calls), + ATTR_LIST(gc_foreground_calls), + ATTR_LIST(gc_background_calls), + ATTR_LIST(moved_blocks_foreground), + ATTR_LIST(moved_blocks_background), + ATTR_LIST(avg_vblocks), + ATTR_LIST(defrag_blocks), +#endif +#ifdef CONFIG_BLK_DEV_ZONED + ATTR_LIST(unusable_blocks_per_sec), + ATTR_LIST(max_open_zones), + ATTR_LIST(blkzone_alloc_policy), +#endif +#ifdef CONFIG_F2FS_FS_COMPRESSION + ATTR_LIST(compr_written_block), + ATTR_LIST(compr_saved_block), + ATTR_LIST(compr_new_inode), + ATTR_LIST(compress_percent), + ATTR_LIST(compress_watermark), +#endif + /* For ATGC */ + ATTR_LIST(atgc_candidate_ratio), + ATTR_LIST(atgc_candidate_count), + ATTR_LIST(atgc_age_weight), + ATTR_LIST(atgc_age_threshold), + ATTR_LIST(atgc_enabled), + ATTR_LIST(seq_file_ra_mul), + ATTR_LIST(gc_segment_mode), + ATTR_LIST(gc_reclaimed_segments), + ATTR_LIST(max_fragment_chunk), + ATTR_LIST(max_fragment_hole), + ATTR_LIST(current_atomic_write), + ATTR_LIST(peak_atomic_write), + ATTR_LIST(committed_atomic_block), + ATTR_LIST(revoked_atomic_block), + ATTR_LIST(hot_data_age_threshold), + ATTR_LIST(warm_data_age_threshold), + ATTR_LIST(last_age_weight), + ATTR_LIST(max_read_extent_count), + ATTR_LIST(carve_out), + ATTR_LIST(reserved_pin_section), + ATTR_LIST(allocate_section_hint), + ATTR_LIST(allocate_section_policy), + ATTR_LIST(max_lock_elapsed_time), + ATTR_LIST(lock_duration_priority), + ATTR_LIST(adjust_lock_priority), + ATTR_LIST(critical_task_priority), + NULL, +}; +ATTRIBUTE_GROUPS(f2fs); + +#define BASE_ATTR_LIST(name) (&f2fs_base_attr_##name.attr) +static struct attribute *f2fs_feat_attrs[] = { +#ifdef CONFIG_FS_ENCRYPTION + BASE_ATTR_LIST(encryption), + BASE_ATTR_LIST(test_dummy_encryption_v2), +#if IS_ENABLED(CONFIG_UNICODE) + BASE_ATTR_LIST(encrypted_casefold), +#endif +#endif /* CONFIG_FS_ENCRYPTION */ +#ifdef CONFIG_BLK_DEV_ZONED + BASE_ATTR_LIST(block_zoned), +#endif + BASE_ATTR_LIST(atomic_write), + BASE_ATTR_LIST(extra_attr), + BASE_ATTR_LIST(project_quota), + BASE_ATTR_LIST(inode_checksum), + BASE_ATTR_LIST(flexible_inline_xattr), + BASE_ATTR_LIST(quota_ino), + BASE_ATTR_LIST(inode_crtime), + BASE_ATTR_LIST(lost_found), +#ifdef CONFIG_FS_VERITY + BASE_ATTR_LIST(verity), +#endif + BASE_ATTR_LIST(sb_checksum), +#if IS_ENABLED(CONFIG_UNICODE) + BASE_ATTR_LIST(casefold), +#endif + BASE_ATTR_LIST(readonly), +#ifdef CONFIG_F2FS_FS_COMPRESSION + BASE_ATTR_LIST(compression), +#endif + BASE_ATTR_LIST(pin_file), +#ifdef CONFIG_UNICODE + BASE_ATTR_LIST(linear_lookup), +#endif + BASE_ATTR_LIST(packed_ssa), + BASE_ATTR_LIST(fserror), + NULL, +}; +ATTRIBUTE_GROUPS(f2fs_feat); + +F2FS_GENERAL_RO_ATTR(sb_status); +F2FS_GENERAL_RO_ATTR(cp_status); +F2FS_GENERAL_RO_ATTR(issued_discard); +F2FS_GENERAL_RO_ATTR(queued_discard); +F2FS_GENERAL_RO_ATTR(undiscard_blks); + +static struct attribute *f2fs_stat_attrs[] = { + ATTR_LIST(sb_status), + ATTR_LIST(cp_status), + ATTR_LIST(issued_discard), + ATTR_LIST(queued_discard), + ATTR_LIST(undiscard_blks), + NULL, +}; +ATTRIBUTE_GROUPS(f2fs_stat); + +F2FS_SB_FEATURE_RO_ATTR(encryption, ENCRYPT); +F2FS_SB_FEATURE_RO_ATTR(block_zoned, BLKZONED); +F2FS_SB_FEATURE_RO_ATTR(extra_attr, EXTRA_ATTR); +F2FS_SB_FEATURE_RO_ATTR(project_quota, PRJQUOTA); +F2FS_SB_FEATURE_RO_ATTR(inode_checksum, INODE_CHKSUM); +F2FS_SB_FEATURE_RO_ATTR(flexible_inline_xattr, FLEXIBLE_INLINE_XATTR); +F2FS_SB_FEATURE_RO_ATTR(quota_ino, QUOTA_INO); +F2FS_SB_FEATURE_RO_ATTR(inode_crtime, INODE_CRTIME); +F2FS_SB_FEATURE_RO_ATTR(lost_found, LOST_FOUND); +F2FS_SB_FEATURE_RO_ATTR(verity, VERITY); +F2FS_SB_FEATURE_RO_ATTR(sb_checksum, SB_CHKSUM); +F2FS_SB_FEATURE_RO_ATTR(casefold, CASEFOLD); +F2FS_SB_FEATURE_RO_ATTR(compression, COMPRESSION); +F2FS_SB_FEATURE_RO_ATTR(readonly, RO); +F2FS_SB_FEATURE_RO_ATTR(device_alias, DEVICE_ALIAS); +F2FS_SB_FEATURE_RO_ATTR(packed_ssa, PACKED_SSA); + +static struct attribute *f2fs_sb_feat_attrs[] = { + ATTR_LIST(sb_encryption), + ATTR_LIST(sb_block_zoned), + ATTR_LIST(sb_extra_attr), + ATTR_LIST(sb_project_quota), + ATTR_LIST(sb_inode_checksum), + ATTR_LIST(sb_flexible_inline_xattr), + ATTR_LIST(sb_quota_ino), + ATTR_LIST(sb_inode_crtime), + ATTR_LIST(sb_lost_found), + ATTR_LIST(sb_verity), + ATTR_LIST(sb_sb_checksum), + ATTR_LIST(sb_casefold), + ATTR_LIST(sb_compression), + ATTR_LIST(sb_readonly), + ATTR_LIST(sb_device_alias), + ATTR_LIST(sb_packed_ssa), + NULL, +}; +ATTRIBUTE_GROUPS(f2fs_sb_feat); + +F2FS_TUNE_RW_ATTR(reclaim_caches_kb); + +static struct attribute *f2fs_tune_attrs[] = { + BASE_ATTR_LIST(reclaim_caches_kb), + NULL, +}; +ATTRIBUTE_GROUPS(f2fs_tune); + +static const struct sysfs_ops f2fs_attr_ops = { + .show = f2fs_attr_show, + .store = f2fs_attr_store, +}; + +static const struct kobj_type f2fs_sb_ktype = { + .default_groups = f2fs_groups, + .sysfs_ops = &f2fs_attr_ops, + .release = f2fs_sb_release, +}; + +static const struct kobj_type f2fs_ktype = { + .sysfs_ops = &f2fs_attr_ops, +}; + +static struct kset f2fs_kset = { + .kobj = {.ktype = &f2fs_ktype}, +}; + +static const struct sysfs_ops f2fs_feat_attr_ops = { + .show = f2fs_base_attr_show, + .store = f2fs_base_attr_store, +}; + +static const struct kobj_type f2fs_feat_ktype = { + .default_groups = f2fs_feat_groups, + .sysfs_ops = &f2fs_feat_attr_ops, +}; + +static struct kobject f2fs_feat = { + .kset = &f2fs_kset, +}; + +static const struct sysfs_ops f2fs_tune_attr_ops = { + .show = f2fs_base_attr_show, + .store = f2fs_base_attr_store, +}; + +static const struct kobj_type f2fs_tune_ktype = { + .default_groups = f2fs_tune_groups, + .sysfs_ops = &f2fs_tune_attr_ops, +}; + +static struct kobject f2fs_tune = { + .kset = &f2fs_kset, +}; + +static ssize_t f2fs_stat_attr_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct f2fs_sb_info *sbi = container_of(kobj, struct f2fs_sb_info, + s_stat_kobj); + struct f2fs_attr *a = container_of(attr, struct f2fs_attr, attr); + + return a->show ? a->show(a, sbi, buf) : 0; +} + +static ssize_t f2fs_stat_attr_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t len) +{ + struct f2fs_sb_info *sbi = container_of(kobj, struct f2fs_sb_info, + s_stat_kobj); + struct f2fs_attr *a = container_of(attr, struct f2fs_attr, attr); + + return a->store ? a->store(a, sbi, buf, len) : 0; +} + +static void f2fs_stat_kobj_release(struct kobject *kobj) +{ + struct f2fs_sb_info *sbi = container_of(kobj, struct f2fs_sb_info, + s_stat_kobj); + complete(&sbi->s_stat_kobj_unregister); +} + +static const struct sysfs_ops f2fs_stat_attr_ops = { + .show = f2fs_stat_attr_show, + .store = f2fs_stat_attr_store, +}; + +static const struct kobj_type f2fs_stat_ktype = { + .default_groups = f2fs_stat_groups, + .sysfs_ops = &f2fs_stat_attr_ops, + .release = f2fs_stat_kobj_release, +}; + +static ssize_t f2fs_sb_feat_attr_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct f2fs_sb_info *sbi = container_of(kobj, struct f2fs_sb_info, + s_feature_list_kobj); + struct f2fs_attr *a = container_of(attr, struct f2fs_attr, attr); + + return a->show ? a->show(a, sbi, buf) : 0; +} + +static void f2fs_feature_list_kobj_release(struct kobject *kobj) +{ + struct f2fs_sb_info *sbi = container_of(kobj, struct f2fs_sb_info, + s_feature_list_kobj); + complete(&sbi->s_feature_list_kobj_unregister); +} + +static const struct sysfs_ops f2fs_feature_list_attr_ops = { + .show = f2fs_sb_feat_attr_show, +}; + +static const struct kobj_type f2fs_feature_list_ktype = { + .default_groups = f2fs_sb_feat_groups, + .sysfs_ops = &f2fs_feature_list_attr_ops, + .release = f2fs_feature_list_kobj_release, +}; + +static int __maybe_unused segment_info_seq_show(struct seq_file *seq, + void *offset) +{ + struct super_block *sb = seq->private; + struct f2fs_sb_info *sbi = F2FS_SB(sb); + unsigned int total_segs = + le32_to_cpu(sbi->raw_super->segment_count_main); + int i; + + seq_puts(seq, "format: segment_type|valid_blocks\n" + "segment_type(0:HD, 1:WD, 2:CD, 3:HN, 4:WN, 5:CN)\n"); + + for (i = 0; i < total_segs; i++) { + struct seg_entry *se = get_seg_entry(sbi, i); + + if ((i % 10) == 0) + seq_printf(seq, "%-10d", i); + seq_printf(seq, "%d|%-3u", se->type, se->valid_blocks); + if ((i % 10) == 9 || i == (total_segs - 1)) + seq_putc(seq, '\n'); + else + seq_putc(seq, ' '); + } + + return 0; +} + +static int __maybe_unused segment_bits_seq_show(struct seq_file *seq, + void *offset) +{ + struct super_block *sb = seq->private; + struct f2fs_sb_info *sbi = F2FS_SB(sb); + unsigned int total_segs = + le32_to_cpu(sbi->raw_super->segment_count_main); + int i, j; + + seq_puts(seq, "format: segment_type|valid_blocks|bitmaps|mtime\n" + "segment_type(0:HD, 1:WD, 2:CD, 3:HN, 4:WN, 5:CN)\n"); + + for (i = 0; i < total_segs; i++) { + struct seg_entry *se = get_seg_entry(sbi, i); + + seq_printf(seq, "%-10d", i); + seq_printf(seq, "%d|%-3u|", se->type, se->valid_blocks); + for (j = 0; j < SIT_VBLOCK_MAP_SIZE; j++) + seq_printf(seq, " %.2x", se->cur_valid_map[j]); + seq_printf(seq, "| %llx", se->mtime); + seq_putc(seq, '\n'); + } + return 0; +} + +static int __maybe_unused victim_bits_seq_show(struct seq_file *seq, + void *offset) +{ + struct super_block *sb = seq->private; + struct f2fs_sb_info *sbi = F2FS_SB(sb); + struct dirty_seglist_info *dirty_i = DIRTY_I(sbi); + int i; + + seq_puts(seq, "format: victim_secmap bitmaps\n"); + + for (i = 0; i < MAIN_SECS(sbi); i++) { + if ((i % 10) == 0) + seq_printf(seq, "%-10d", i); + seq_printf(seq, "%d", test_bit(i, dirty_i->victim_secmap) ? 1 : 0); + if ((i % 10) == 9 || i == (MAIN_SECS(sbi) - 1)) + seq_putc(seq, '\n'); + else + seq_putc(seq, ' '); + } + return 0; +} + +static int __maybe_unused discard_plist_seq_show(struct seq_file *seq, + void *offset) +{ + struct super_block *sb = seq->private; + struct f2fs_sb_info *sbi = F2FS_SB(sb); + struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info; + int i, count; + + seq_puts(seq, "Discard pend list(Show diacrd_cmd count on each entry, .:not exist):\n"); + if (!f2fs_realtime_discard_enable(sbi)) + return 0; + + if (dcc) { + mutex_lock(&dcc->cmd_lock); + for (i = 0; i < MAX_PLIST_NUM; i++) { + struct list_head *pend_list; + struct discard_cmd *dc, *tmp; + + if (i % 8 == 0) + seq_printf(seq, " %-3d", i); + count = 0; + pend_list = &dcc->pend_list[i]; + list_for_each_entry_safe(dc, tmp, pend_list, list) + count++; + if (count) + seq_printf(seq, " %7d", count); + else + seq_puts(seq, " ."); + if (i % 8 == 7) + seq_putc(seq, '\n'); + } + seq_putc(seq, '\n'); + mutex_unlock(&dcc->cmd_lock); + } + + return 0; +} + +static int __maybe_unused disk_map_seq_show(struct seq_file *seq, + void *offset) +{ + struct super_block *sb = seq->private; + struct f2fs_sb_info *sbi = F2FS_SB(sb); + int i; + + seq_printf(seq, "Address Layout : %5luB Block address (# of Segments)\n", + F2FS_BLKSIZE); + seq_printf(seq, " SB : %12s\n", "0/1024B"); + seq_printf(seq, " seg0_blkaddr : 0x%010x\n", SEG0_BLKADDR(sbi)); + seq_printf(seq, " Checkpoint : 0x%010x (%10d)\n", + le32_to_cpu(F2FS_RAW_SUPER(sbi)->cp_blkaddr), 2); + seq_printf(seq, " SIT : 0x%010x (%10d)\n", + SIT_I(sbi)->sit_base_addr, + le32_to_cpu(F2FS_RAW_SUPER(sbi)->segment_count_sit)); + seq_printf(seq, " NAT : 0x%010x (%10d)\n", + NM_I(sbi)->nat_blkaddr, + le32_to_cpu(F2FS_RAW_SUPER(sbi)->segment_count_nat)); + seq_printf(seq, " SSA : 0x%010x (%10d)\n", + SM_I(sbi)->ssa_blkaddr, + le32_to_cpu(F2FS_RAW_SUPER(sbi)->segment_count_ssa)); + seq_printf(seq, " Main : 0x%010x (%10d)\n", + SM_I(sbi)->main_blkaddr, + le32_to_cpu(F2FS_RAW_SUPER(sbi)->segment_count_main)); + seq_printf(seq, " Block size : %12lu KB\n", F2FS_BLKSIZE >> 10); + seq_printf(seq, " Segment size : %12d MB\n", + (BLKS_PER_SEG(sbi) << (F2FS_BLKSIZE_BITS - 10)) >> 10); + seq_printf(seq, " Segs/Sections : %12d\n", + SEGS_PER_SEC(sbi)); + seq_printf(seq, " Section size : %12d MB\n", + (BLKS_PER_SEC(sbi) << (F2FS_BLKSIZE_BITS - 10)) >> 10); + seq_printf(seq, " # of Sections : %12d\n", + le32_to_cpu(F2FS_RAW_SUPER(sbi)->section_count)); + + if (!f2fs_is_multi_device(sbi)) + return 0; + + seq_puts(seq, "\nDisk Map for multi devices:\n"); + for (i = 0; i < sbi->s_ndevs; i++) + seq_printf(seq, "Disk:%2d (zoned=%d): 0x%010x - 0x%010x on %s\n", + i, bdev_is_zoned(FDEV(i).bdev), + FDEV(i).start_blk, FDEV(i).end_blk, + FDEV(i).path); + return 0; +} + +static int __maybe_unused donation_list_seq_show(struct seq_file *seq, + void *offset) +{ + struct super_block *sb = seq->private; + struct f2fs_sb_info *sbi = F2FS_SB(sb); + struct inode *inode; + struct f2fs_inode_info *fi; + struct dentry *dentry; + char *buf, *path; + int i; + + buf = f2fs_getname(sbi); + if (!buf) + return 0; + + seq_printf(seq, "Donation List\n"); + seq_printf(seq, " # of files : %u\n", sbi->donate_files); + seq_printf(seq, " %-50s %10s %20s %20s %22s\n", + "File path", "Status", "Donation offset (kb)", + "Donation size (kb)", "File cached size (kb)"); + seq_printf(seq, "---\n"); + + for (i = 0; i < sbi->donate_files; i++) { + spin_lock(&sbi->inode_lock[DONATE_INODE]); + if (list_empty(&sbi->inode_list[DONATE_INODE])) { + spin_unlock(&sbi->inode_lock[DONATE_INODE]); + break; + } + fi = list_first_entry(&sbi->inode_list[DONATE_INODE], + struct f2fs_inode_info, gdonate_list); + list_move_tail(&fi->gdonate_list, &sbi->inode_list[DONATE_INODE]); + inode = igrab(&fi->vfs_inode); + spin_unlock(&sbi->inode_lock[DONATE_INODE]); + + if (!inode) + continue; + + inode_lock_shared(inode); + + dentry = d_find_alias(inode); + if (!dentry) { + path = NULL; + } else { + path = dentry_path_raw(dentry, buf, PATH_MAX); + if (IS_ERR(path)) + goto next; + } + seq_printf(seq, " %-50s %10s %20llu %20llu %22llu\n", + path ? path : "", + is_inode_flag_set(inode, FI_DONATE_FINISHED) ? + "Evicted" : "Donated", + (loff_t)fi->donate_start << (PAGE_SHIFT - 10), + (loff_t)(fi->donate_end + 1) << (PAGE_SHIFT - 10), + (loff_t)inode->i_mapping->nrpages << (PAGE_SHIFT - 10)); +next: + dput(dentry); + inode_unlock_shared(inode); + iput(inode); + } + f2fs_putname(buf); + return 0; +} + +#ifdef CONFIG_F2FS_FAULT_INJECTION +static int __maybe_unused inject_stats_seq_show(struct seq_file *seq, + void *offset) +{ + struct super_block *sb = seq->private; + struct f2fs_sb_info *sbi = F2FS_SB(sb); + struct f2fs_fault_info *ffi = &F2FS_OPTION(sbi).fault_info; + int i; + + seq_puts(seq, "fault_type injected_count\n"); + + for (i = 0; i < FAULT_MAX; i++) + seq_printf(seq, "%-24s%-10u\n", f2fs_fault_name[i], + ffi->inject_count[i]); + return 0; +} +#endif + +int __init f2fs_init_sysfs(void) +{ + int ret; + + kobject_set_name(&f2fs_kset.kobj, "f2fs"); + f2fs_kset.kobj.parent = fs_kobj; + ret = kset_register(&f2fs_kset); + if (ret) + return ret; + + ret = kobject_init_and_add(&f2fs_feat, &f2fs_feat_ktype, + NULL, "features"); + if (ret) + goto unregister_kset; + + ret = kobject_init_and_add(&f2fs_tune, &f2fs_tune_ktype, + NULL, "tuning"); + if (ret) + goto put_feat; + + f2fs_proc_root = proc_mkdir("fs/f2fs", NULL); + if (!f2fs_proc_root) { + ret = -ENOMEM; + goto put_tune; + } + + return 0; + +put_tune: + kobject_put(&f2fs_tune); +put_feat: + kobject_put(&f2fs_feat); +unregister_kset: + kset_unregister(&f2fs_kset); + return ret; +} + +void f2fs_exit_sysfs(void) +{ + kobject_put(&f2fs_tune); + kobject_put(&f2fs_feat); + kset_unregister(&f2fs_kset); + remove_proc_entry("fs/f2fs", NULL); + f2fs_proc_root = NULL; +} + +int f2fs_register_sysfs(struct f2fs_sb_info *sbi) +{ + struct super_block *sb = sbi->sb; + int err; + + sbi->s_kobj.kset = &f2fs_kset; + init_completion(&sbi->s_kobj_unregister); + err = kobject_init_and_add(&sbi->s_kobj, &f2fs_sb_ktype, NULL, + "%s", sb->s_id); + if (err) + goto put_sb_kobj; + + sbi->s_stat_kobj.kset = &f2fs_kset; + init_completion(&sbi->s_stat_kobj_unregister); + err = kobject_init_and_add(&sbi->s_stat_kobj, &f2fs_stat_ktype, + &sbi->s_kobj, "stat"); + if (err) + goto put_stat_kobj; + + sbi->s_feature_list_kobj.kset = &f2fs_kset; + init_completion(&sbi->s_feature_list_kobj_unregister); + err = kobject_init_and_add(&sbi->s_feature_list_kobj, + &f2fs_feature_list_ktype, + &sbi->s_kobj, "feature_list"); + if (err) + goto put_feature_list_kobj; + + sbi->s_proc = proc_mkdir(sb->s_id, f2fs_proc_root); + if (!sbi->s_proc) { + err = -ENOMEM; + goto put_feature_list_kobj; + } + + proc_create_single_data("segment_info", 0444, sbi->s_proc, + segment_info_seq_show, sb); + proc_create_single_data("segment_bits", 0444, sbi->s_proc, + segment_bits_seq_show, sb); +#ifdef CONFIG_F2FS_IOSTAT + proc_create_single_data("iostat_info", 0444, sbi->s_proc, + iostat_info_seq_show, sb); +#endif + proc_create_single_data("victim_bits", 0444, sbi->s_proc, + victim_bits_seq_show, sb); + proc_create_single_data("discard_plist_info", 0444, sbi->s_proc, + discard_plist_seq_show, sb); + proc_create_single_data("disk_map", 0444, sbi->s_proc, + disk_map_seq_show, sb); + proc_create_single_data("donation_list", 0444, sbi->s_proc, + donation_list_seq_show, sb); +#ifdef CONFIG_F2FS_FAULT_INJECTION + proc_create_single_data("inject_stats", 0444, sbi->s_proc, + inject_stats_seq_show, sb); +#endif + return 0; +put_feature_list_kobj: + kobject_put(&sbi->s_feature_list_kobj); + wait_for_completion(&sbi->s_feature_list_kobj_unregister); +put_stat_kobj: + kobject_put(&sbi->s_stat_kobj); + wait_for_completion(&sbi->s_stat_kobj_unregister); +put_sb_kobj: + kobject_put(&sbi->s_kobj); + wait_for_completion(&sbi->s_kobj_unregister); + return err; +} + +void f2fs_unregister_sysfs(struct f2fs_sb_info *sbi) +{ + remove_proc_subtree(sbi->sb->s_id, f2fs_proc_root); + + kobject_put(&sbi->s_stat_kobj); + wait_for_completion(&sbi->s_stat_kobj_unregister); + kobject_put(&sbi->s_feature_list_kobj); + wait_for_completion(&sbi->s_feature_list_kobj_unregister); + + kobject_put(&sbi->s_kobj); + wait_for_completion(&sbi->s_kobj_unregister); +} diff --git a/fs/f2fs/verity.c b/fs/f2fs/verity.c new file mode 100644 index 000000000..39f482515 --- /dev/null +++ b/fs/f2fs/verity.c @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/f2fs/verity.c: fs-verity support for f2fs + * + * Copyright 2019 Google LLC + */ + +/* + * Implementation of fsverity_operations for f2fs. + * + * Like ext4, f2fs stores the verity metadata (Merkle tree and + * fsverity_descriptor) past the end of the file, starting at the first 64K + * boundary beyond i_size. This approach works because (a) verity files are + * readonly, and (b) pages fully beyond i_size aren't visible to userspace but + * can be read/written internally by f2fs with only some relatively small + * changes to f2fs. Extended attributes cannot be used because (a) f2fs limits + * the total size of an inode's xattr entries to 4096 bytes, which wouldn't be + * enough for even a single Merkle tree block, and (b) f2fs encryption doesn't + * encrypt xattrs, yet the verity metadata *must* be encrypted when the file is + * because it contains hashes of the plaintext data. + * + * Using a 64K boundary rather than a 4K one keeps things ready for + * architectures with 64K pages, and it doesn't necessarily waste space on-disk + * since there can be a hole between i_size and the start of the Merkle tree. + */ + +#include +#include + +#include "f2fs.h" +#include "xattr.h" + +#define F2FS_VERIFY_VER (1) + +static inline loff_t f2fs_verity_metadata_pos(const struct inode *inode) +{ + return round_up(inode->i_size, 65536); +} + +/* + * Read some verity metadata from the inode. __vfs_read() can't be used because + * we need to read beyond i_size. + */ +static int pagecache_read(struct inode *inode, void *buf, size_t count, + loff_t pos) +{ + while (count) { + size_t n = min_t(size_t, count, + PAGE_SIZE - offset_in_page(pos)); + struct page *page; + + page = read_mapping_page(inode->i_mapping, pos >> PAGE_SHIFT, + NULL); + if (IS_ERR(page)) + return PTR_ERR(page); + + memcpy_from_page(buf, page, offset_in_page(pos), n); + + put_page(page); + + buf += n; + pos += n; + count -= n; + } + return 0; +} + +/* + * Write some verity metadata to the inode for FS_IOC_ENABLE_VERITY. + * kernel_write() can't be used because the file descriptor is readonly. + */ +static int pagecache_write(struct inode *inode, const void *buf, size_t count, + loff_t pos) +{ + struct address_space *mapping = inode->i_mapping; + const struct address_space_operations *aops = mapping->a_ops; + + if (pos + count > F2FS_BLK_TO_BYTES(max_file_blocks(inode))) + return -EFBIG; + + while (count) { + size_t n = min_t(size_t, count, + PAGE_SIZE - offset_in_page(pos)); + struct folio *folio; + void *fsdata = NULL; + int res; + + res = aops->write_begin(NULL, mapping, pos, n, &folio, &fsdata); + if (res) + return res; + + memcpy_to_folio(folio, offset_in_folio(folio, pos), buf, n); + + res = aops->write_end(NULL, mapping, pos, n, n, folio, fsdata); + if (res < 0) + return res; + if (res != n) + return -EIO; + + buf += n; + pos += n; + count -= n; + } + return 0; +} + +/* + * Format of f2fs verity xattr. This points to the location of the verity + * descriptor within the file data rather than containing it directly because + * the verity descriptor *must* be encrypted when f2fs encryption is used. But, + * f2fs encryption does not encrypt xattrs. + */ +struct fsverity_descriptor_location { + __le32 version; + __le32 size; + __le64 pos; +}; + +static int f2fs_begin_enable_verity(struct file *filp) +{ + struct inode *inode = file_inode(filp); + int err; + + if (f2fs_verity_in_progress(inode)) + return -EBUSY; + + if (f2fs_is_atomic_file(inode)) + return -EOPNOTSUPP; + + /* + * Since the file was opened readonly, we have to initialize the quotas + * here and not rely on ->open() doing it. This must be done before + * evicting the inline data. + */ + err = f2fs_dquot_initialize(inode); + if (err) + return err; + + err = f2fs_convert_inline_inode(inode); + if (err) + return err; + + set_inode_flag(inode, FI_VERITY_IN_PROGRESS); + return 0; +} + +static int f2fs_end_enable_verity(struct file *filp, const void *desc, + size_t desc_size, u64 merkle_tree_size) +{ + struct inode *inode = file_inode(filp); + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + u64 desc_pos = f2fs_verity_metadata_pos(inode) + merkle_tree_size; + struct fsverity_descriptor_location dloc = { + .version = cpu_to_le32(F2FS_VERIFY_VER), + .size = cpu_to_le32(desc_size), + .pos = cpu_to_le64(desc_pos), + }; + int err = 0, err2 = 0; + + /* + * If an error already occurred (which fs/verity/ signals by passing + * desc == NULL), then only clean-up is needed. + */ + if (desc == NULL) + goto cleanup; + + /* Append the verity descriptor. */ + err = pagecache_write(inode, desc, desc_size, desc_pos); + if (err) + goto cleanup; + + /* + * Write all pages (both data and verity metadata). Note that this must + * happen before clearing FI_VERITY_IN_PROGRESS; otherwise pages beyond + * i_size won't be written properly. For crash consistency, this also + * must happen before the verity inode flag gets persisted. + */ + err = filemap_write_and_wait(inode->i_mapping); + if (err) + goto cleanup; + + /* Set the verity xattr. */ + err = f2fs_setxattr(inode, F2FS_XATTR_INDEX_VERITY, + F2FS_XATTR_NAME_VERITY, &dloc, sizeof(dloc), + NULL, XATTR_CREATE); + if (err) + goto cleanup; + + /* Finally, set the verity inode flag. */ + file_set_verity(inode); + f2fs_set_inode_flags(inode); + f2fs_mark_inode_dirty_sync(inode, true); + + clear_inode_flag(inode, FI_VERITY_IN_PROGRESS); + return 0; + +cleanup: + /* + * Verity failed to be enabled, so clean up by truncating any verity + * metadata that was written beyond i_size (both from cache and from + * disk) and clearing FI_VERITY_IN_PROGRESS. + * + * Taking i_gc_rwsem[WRITE] is needed to stop f2fs garbage collection + * from re-instantiating cached pages we are truncating (since unlike + * normal file accesses, garbage collection isn't limited by i_size). + */ + f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); + truncate_inode_pages(inode->i_mapping, inode->i_size); + err2 = f2fs_truncate(inode); + if (err2) { + f2fs_err(sbi, "Truncating verity metadata failed (errno=%d)", + err2); + set_sbi_flag(sbi, SBI_NEED_FSCK); + } + f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); + clear_inode_flag(inode, FI_VERITY_IN_PROGRESS); + return err ?: err2; +} + +static int f2fs_get_verity_descriptor(struct inode *inode, void *buf, + size_t buf_size) +{ + struct fsverity_descriptor_location dloc; + int res; + u32 size; + u64 pos; + + /* Get the descriptor location */ + res = f2fs_getxattr(inode, F2FS_XATTR_INDEX_VERITY, + F2FS_XATTR_NAME_VERITY, &dloc, sizeof(dloc), NULL); + if (res < 0 && res != -ERANGE) + return res; + if (res != sizeof(dloc) || dloc.version != cpu_to_le32(F2FS_VERIFY_VER)) { + f2fs_warn(F2FS_I_SB(inode), "unknown verity xattr format"); + return -EINVAL; + } + size = le32_to_cpu(dloc.size); + pos = le64_to_cpu(dloc.pos); + + /* Get the descriptor */ + if (pos + size < pos || + pos + size > F2FS_BLK_TO_BYTES(max_file_blocks(inode)) || + pos < f2fs_verity_metadata_pos(inode) || size > INT_MAX) { + f2fs_warn(F2FS_I_SB(inode), "invalid verity xattr"); + f2fs_handle_error(F2FS_I_SB(inode), + ERROR_CORRUPTED_VERITY_XATTR); + fserror_report_file_metadata(inode, -EFSCORRUPTED, GFP_NOFS); + return -EFSCORRUPTED; + } + if (buf_size) { + if (size > buf_size) + return -ERANGE; + res = pagecache_read(inode, buf, size, pos); + if (res) + return res; + } + return size; +} + +static struct page *f2fs_read_merkle_tree_page(struct inode *inode, + pgoff_t index) +{ + index += f2fs_verity_metadata_pos(inode) >> PAGE_SHIFT; + return generic_read_merkle_tree_page(inode, index); +} + +static void f2fs_readahead_merkle_tree(struct inode *inode, pgoff_t index, + unsigned long nr_pages) +{ + index += f2fs_verity_metadata_pos(inode) >> PAGE_SHIFT; + generic_readahead_merkle_tree(inode, index, nr_pages); +} + +static int f2fs_write_merkle_tree_block(struct file *file, const void *buf, + u64 pos, unsigned int size) +{ + pos += f2fs_verity_metadata_pos(file_inode(file)); + + return pagecache_write(file_inode(file), buf, size, pos); +} + +const struct fsverity_operations f2fs_verityops = { + .begin_enable_verity = f2fs_begin_enable_verity, + .end_enable_verity = f2fs_end_enable_verity, + .get_verity_descriptor = f2fs_get_verity_descriptor, + .read_merkle_tree_page = f2fs_read_merkle_tree_page, + .readahead_merkle_tree = f2fs_readahead_merkle_tree, + .write_merkle_tree_block = f2fs_write_merkle_tree_block, +}; diff --git a/fs/f2fs/xattr.c b/fs/f2fs/xattr.c new file mode 100644 index 000000000..ed33e5110 --- /dev/null +++ b/fs/f2fs/xattr.c @@ -0,0 +1,851 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/f2fs/xattr.c + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * + * Portions of this code from linux/fs/ext2/xattr.c + * + * Copyright (C) 2001-2003 Andreas Gruenbacher + * + * Fix by Harrison Xing . + * Extended attributes for symlinks and special files added per + * suggestion of Luka Renko . + * xattr consolidation Copyright (c) 2004 James Morris , + * Red Hat Inc. + */ +#include +#include +#include +#include +#include +#include "f2fs.h" +#include "xattr.h" +#include "segment.h" + +static struct kmem_cache *inline_xattr_slab; +static void *xattr_alloc(struct f2fs_sb_info *sbi, int size, bool *is_inline) +{ + if (likely(size == DEFAULT_XATTR_SLAB_SIZE)) { + *is_inline = true; + return f2fs_kmem_cache_alloc(inline_xattr_slab, + GFP_F2FS_ZERO, false, sbi); + } + *is_inline = false; + return f2fs_kzalloc(sbi, size, GFP_NOFS); +} + +static void xattr_free(struct f2fs_sb_info *sbi, void *xattr_addr, + bool is_inline) +{ + if (is_inline) + kmem_cache_free(inline_xattr_slab, xattr_addr); + else + kfree(xattr_addr); +} + +static int f2fs_xattr_generic_get(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb); + + switch (handler->flags) { + case F2FS_XATTR_INDEX_USER: + if (!test_opt(sbi, XATTR_USER)) + return -EOPNOTSUPP; + break; + case F2FS_XATTR_INDEX_TRUSTED: + case F2FS_XATTR_INDEX_SECURITY: + break; + default: + return -EINVAL; + } + return f2fs_getxattr(inode, handler->flags, name, + buffer, size, NULL); +} + +static int f2fs_xattr_generic_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +{ + struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb); + + switch (handler->flags) { + case F2FS_XATTR_INDEX_USER: + if (!test_opt(sbi, XATTR_USER)) + return -EOPNOTSUPP; + break; + case F2FS_XATTR_INDEX_TRUSTED: + case F2FS_XATTR_INDEX_SECURITY: + break; + default: + return -EINVAL; + } + return f2fs_setxattr(inode, handler->flags, name, + value, size, NULL, flags); +} + +static bool f2fs_xattr_user_list(struct dentry *dentry) +{ + struct f2fs_sb_info *sbi = F2FS_SB(dentry->d_sb); + + return test_opt(sbi, XATTR_USER); +} + +static bool f2fs_xattr_trusted_list(struct dentry *dentry) +{ + return capable(CAP_SYS_ADMIN); +} + +static int f2fs_xattr_advise_get(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + if (buffer) + *((char *)buffer) = F2FS_I(inode)->i_advise; + return sizeof(char); +} + +static int f2fs_xattr_advise_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +{ + unsigned char old_advise = F2FS_I(inode)->i_advise; + unsigned char new_advise; + + if (!inode_owner_or_capable(&nop_mnt_idmap, inode)) + return -EPERM; + if (value == NULL) + return -EINVAL; + + new_advise = *(char *)value; + if (new_advise & ~FADVISE_MODIFIABLE_BITS) + return -EINVAL; + + new_advise = new_advise & FADVISE_MODIFIABLE_BITS; + new_advise |= old_advise & ~FADVISE_MODIFIABLE_BITS; + + F2FS_I(inode)->i_advise = new_advise; + f2fs_mark_inode_dirty_sync(inode, true); + return 0; +} + +#ifdef CONFIG_F2FS_FS_SECURITY +static int f2fs_initxattrs(struct inode *inode, const struct xattr *xattr_array, + void *folio) +{ + const struct xattr *xattr; + int err = 0; + + for (xattr = xattr_array; xattr->name != NULL; xattr++) { + err = f2fs_setxattr(inode, F2FS_XATTR_INDEX_SECURITY, + xattr->name, xattr->value, + xattr->value_len, folio, 0); + if (err < 0) + break; + } + return err; +} + +int f2fs_init_security(struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct folio *ifolio) +{ + return security_inode_init_security(inode, dir, qstr, + f2fs_initxattrs, ifolio); +} +#endif + +const struct xattr_handler f2fs_xattr_user_handler = { + .prefix = XATTR_USER_PREFIX, + .flags = F2FS_XATTR_INDEX_USER, + .list = f2fs_xattr_user_list, + .get = f2fs_xattr_generic_get, + .set = f2fs_xattr_generic_set, +}; + +const struct xattr_handler f2fs_xattr_trusted_handler = { + .prefix = XATTR_TRUSTED_PREFIX, + .flags = F2FS_XATTR_INDEX_TRUSTED, + .list = f2fs_xattr_trusted_list, + .get = f2fs_xattr_generic_get, + .set = f2fs_xattr_generic_set, +}; + +const struct xattr_handler f2fs_xattr_advise_handler = { + .name = F2FS_SYSTEM_ADVISE_NAME, + .flags = F2FS_XATTR_INDEX_ADVISE, + .get = f2fs_xattr_advise_get, + .set = f2fs_xattr_advise_set, +}; + +const struct xattr_handler f2fs_xattr_security_handler = { + .prefix = XATTR_SECURITY_PREFIX, + .flags = F2FS_XATTR_INDEX_SECURITY, + .get = f2fs_xattr_generic_get, + .set = f2fs_xattr_generic_set, +}; + +static const struct xattr_handler * const f2fs_xattr_handler_map[] = { + [F2FS_XATTR_INDEX_USER] = &f2fs_xattr_user_handler, +#ifdef CONFIG_F2FS_FS_POSIX_ACL + [F2FS_XATTR_INDEX_POSIX_ACL_ACCESS] = &nop_posix_acl_access, + [F2FS_XATTR_INDEX_POSIX_ACL_DEFAULT] = &nop_posix_acl_default, +#endif + [F2FS_XATTR_INDEX_TRUSTED] = &f2fs_xattr_trusted_handler, +#ifdef CONFIG_F2FS_FS_SECURITY + [F2FS_XATTR_INDEX_SECURITY] = &f2fs_xattr_security_handler, +#endif + [F2FS_XATTR_INDEX_ADVISE] = &f2fs_xattr_advise_handler, +}; + +const struct xattr_handler * const f2fs_xattr_handlers[] = { + &f2fs_xattr_user_handler, + &f2fs_xattr_trusted_handler, +#ifdef CONFIG_F2FS_FS_SECURITY + &f2fs_xattr_security_handler, +#endif + &f2fs_xattr_advise_handler, + NULL, +}; + +static inline const char *f2fs_xattr_prefix(int index, + struct dentry *dentry) +{ + const struct xattr_handler *handler = NULL; + + if (index > 0 && index < ARRAY_SIZE(f2fs_xattr_handler_map)) + handler = f2fs_xattr_handler_map[index]; + + if (!xattr_handler_can_list(handler, dentry)) + return NULL; + + return xattr_prefix(handler); +} + +static struct f2fs_xattr_entry *__find_xattr(void *base_addr, + void *last_base_addr, void **last_addr, + int index, size_t len, const char *name) +{ + struct f2fs_xattr_entry *entry; + + list_for_each_xattr(entry, base_addr) { + if ((void *)(entry) + sizeof(__u32) > last_base_addr || + (void *)XATTR_NEXT_ENTRY(entry) > last_base_addr) { + if (last_addr) + *last_addr = entry; + return NULL; + } + + if (entry->e_name_index != index) + continue; + if (entry->e_name_len != len) + continue; + if (!memcmp(entry->e_name, name, len)) + break; + } + return entry; +} + +static struct f2fs_xattr_entry *__find_inline_xattr(struct inode *inode, + void *base_addr, void **last_addr, int index, + size_t len, const char *name) +{ + struct f2fs_xattr_entry *entry; + unsigned int inline_size = inline_xattr_size(inode); + void *max_addr = base_addr + inline_size; + + entry = __find_xattr(base_addr, max_addr, last_addr, index, len, name); + if (!entry) + return NULL; + + /* inline xattr header or entry across max inline xattr size */ + if (IS_XATTR_LAST_ENTRY(entry) && + (void *)entry + sizeof(__u32) > max_addr) { + *last_addr = entry; + return NULL; + } + return entry; +} + +static int read_inline_xattr(struct inode *inode, struct folio *ifolio, + void *txattr_addr) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + unsigned int inline_size = inline_xattr_size(inode); + struct folio *folio = NULL; + void *inline_addr; + + if (ifolio) { + inline_addr = inline_xattr_addr(inode, ifolio); + } else { + folio = f2fs_get_inode_folio(sbi, inode->i_ino); + if (IS_ERR(folio)) + return PTR_ERR(folio); + + inline_addr = inline_xattr_addr(inode, folio); + } + memcpy(txattr_addr, inline_addr, inline_size); + f2fs_folio_put(folio, true); + + return 0; +} + +static int read_xattr_block(struct inode *inode, void *txattr_addr) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + nid_t xnid = F2FS_I(inode)->i_xattr_nid; + unsigned int inline_size = inline_xattr_size(inode); + struct folio *xfolio; + void *xattr_addr; + + /* The inode already has an extended attribute block. */ + xfolio = f2fs_get_xnode_folio(sbi, xnid); + if (IS_ERR(xfolio)) + return PTR_ERR(xfolio); + + xattr_addr = folio_address(xfolio); + memcpy(txattr_addr + inline_size, xattr_addr, VALID_XATTR_BLOCK_SIZE); + f2fs_folio_put(xfolio, true); + + return 0; +} + +static int lookup_all_xattrs(struct inode *inode, struct folio *ifolio, + unsigned int index, unsigned int len, + const char *name, struct f2fs_xattr_entry **xe, + void **base_addr, int *base_size, + bool *is_inline) +{ + void *cur_addr, *txattr_addr, *last_txattr_addr; + void *last_addr = NULL; + nid_t xnid = F2FS_I(inode)->i_xattr_nid; + unsigned int inline_size = inline_xattr_size(inode); + int err; + + if (!xnid && !inline_size) + return -ENODATA; + + *base_size = XATTR_SIZE(inode) + XATTR_PADDING_SIZE; + txattr_addr = xattr_alloc(F2FS_I_SB(inode), *base_size, is_inline); + if (!txattr_addr) + return -ENOMEM; + + last_txattr_addr = (void *)txattr_addr + XATTR_SIZE(inode); + + /* read from inline xattr */ + if (inline_size) { + err = read_inline_xattr(inode, ifolio, txattr_addr); + if (err) + goto out; + + *xe = __find_inline_xattr(inode, txattr_addr, &last_addr, + index, len, name); + if (*xe) { + *base_size = inline_size; + goto check; + } + } + + /* read from xattr node block */ + if (xnid) { + err = read_xattr_block(inode, txattr_addr); + if (err) + goto out; + } + + if (last_addr) + cur_addr = XATTR_HDR(last_addr) - 1; + else + cur_addr = txattr_addr; + + *xe = __find_xattr(cur_addr, last_txattr_addr, NULL, index, len, name); + if (!*xe) { + f2fs_err(F2FS_I_SB(inode), "lookup inode (%llu) has corrupted xattr", + inode->i_ino); + set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK); + err = -ENODATA; + f2fs_handle_error(F2FS_I_SB(inode), + ERROR_CORRUPTED_XATTR); + fserror_report_file_metadata(inode, err, GFP_NOFS); + goto out; + } +check: + if (IS_XATTR_LAST_ENTRY(*xe)) { + err = -ENODATA; + goto out; + } + + *base_addr = txattr_addr; + return 0; +out: + xattr_free(F2FS_I_SB(inode), txattr_addr, *is_inline); + return err; +} + +static int read_all_xattrs(struct inode *inode, struct folio *ifolio, + void **base_addr) +{ + struct f2fs_xattr_header *header; + nid_t xnid = F2FS_I(inode)->i_xattr_nid; + unsigned int size = VALID_XATTR_BLOCK_SIZE; + unsigned int inline_size = inline_xattr_size(inode); + void *txattr_addr; + int err; + + txattr_addr = f2fs_kzalloc(F2FS_I_SB(inode), + inline_size + size + XATTR_PADDING_SIZE, GFP_NOFS); + if (!txattr_addr) + return -ENOMEM; + + /* read from inline xattr */ + if (inline_size) { + err = read_inline_xattr(inode, ifolio, txattr_addr); + if (err) + goto fail; + } + + /* read from xattr node block */ + if (xnid) { + err = read_xattr_block(inode, txattr_addr); + if (err) + goto fail; + } + + header = XATTR_HDR(txattr_addr); + + /* never been allocated xattrs */ + if (le32_to_cpu(header->h_magic) != F2FS_XATTR_MAGIC) { + header->h_magic = cpu_to_le32(F2FS_XATTR_MAGIC); + header->h_refcount = cpu_to_le32(1); + } + *base_addr = txattr_addr; + return 0; +fail: + kfree(txattr_addr); + return err; +} + +static inline int write_all_xattrs(struct inode *inode, __u32 hsize, + void *txattr_addr, struct folio *ifolio) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + size_t inline_size = inline_xattr_size(inode); + struct folio *in_folio = NULL; + void *xattr_addr; + void *inline_addr = NULL; + struct folio *xfolio; + nid_t new_nid = 0; + int err = 0; + + if (hsize > inline_size && !F2FS_I(inode)->i_xattr_nid) + if (!f2fs_alloc_nid(sbi, &new_nid)) + return -ENOSPC; + + /* write to inline xattr */ + if (inline_size) { + if (ifolio) { + inline_addr = inline_xattr_addr(inode, ifolio); + } else { + in_folio = f2fs_get_inode_folio(sbi, inode->i_ino); + if (IS_ERR(in_folio)) { + f2fs_alloc_nid_failed(sbi, new_nid); + return PTR_ERR(in_folio); + } + inline_addr = inline_xattr_addr(inode, in_folio); + } + + f2fs_folio_wait_writeback(ifolio ? ifolio : in_folio, + NODE, true, true); + /* no need to use xattr node block */ + if (hsize <= inline_size) { + err = f2fs_truncate_xattr_node(inode); + f2fs_alloc_nid_failed(sbi, new_nid); + if (err) { + f2fs_folio_put(in_folio, true); + return err; + } + memcpy(inline_addr, txattr_addr, inline_size); + folio_mark_dirty(ifolio ? ifolio : in_folio); + goto in_page_out; + } + } + + /* write to xattr node block */ + if (F2FS_I(inode)->i_xattr_nid) { + xfolio = f2fs_get_xnode_folio(sbi, F2FS_I(inode)->i_xattr_nid); + if (IS_ERR(xfolio)) { + err = PTR_ERR(xfolio); + f2fs_alloc_nid_failed(sbi, new_nid); + goto in_page_out; + } + f2fs_bug_on(sbi, new_nid); + f2fs_folio_wait_writeback(xfolio, NODE, true, true); + } else { + struct dnode_of_data dn; + + set_new_dnode(&dn, inode, NULL, NULL, new_nid); + xfolio = f2fs_new_node_folio(&dn, XATTR_NODE_OFFSET); + if (IS_ERR(xfolio)) { + err = PTR_ERR(xfolio); + f2fs_alloc_nid_failed(sbi, new_nid); + goto in_page_out; + } + f2fs_alloc_nid_done(sbi, new_nid); + } + xattr_addr = folio_address(xfolio); + + if (inline_size) + memcpy(inline_addr, txattr_addr, inline_size); + memcpy(xattr_addr, txattr_addr + inline_size, VALID_XATTR_BLOCK_SIZE); + + if (inline_size) + folio_mark_dirty(ifolio ? ifolio : in_folio); + folio_mark_dirty(xfolio); + + f2fs_folio_put(xfolio, true); +in_page_out: + f2fs_folio_put(in_folio, true); + return err; +} + +int f2fs_getxattr(struct inode *inode, int index, const char *name, + void *buffer, size_t buffer_size, struct folio *ifolio) +{ + struct f2fs_xattr_entry *entry = NULL; + int error; + unsigned int size, len; + void *base_addr = NULL; + int base_size; + bool is_inline; + + if (name == NULL) + return -EINVAL; + + len = strlen(name); + if (len > F2FS_NAME_LEN) + return -ERANGE; + + if (!ifolio) + f2fs_down_read(&F2FS_I(inode)->i_xattr_sem); + error = lookup_all_xattrs(inode, ifolio, index, len, name, + &entry, &base_addr, &base_size, &is_inline); + if (!ifolio) + f2fs_up_read(&F2FS_I(inode)->i_xattr_sem); + if (error) + return error; + + size = le16_to_cpu(entry->e_value_size); + + if (buffer && size > buffer_size) { + error = -ERANGE; + goto out; + } + + if (buffer) { + char *pval = entry->e_name + entry->e_name_len; + + if (base_size - (pval - (char *)base_addr) < size) { + error = -ERANGE; + goto out; + } + memcpy(buffer, pval, size); + } + error = size; +out: + xattr_free(F2FS_I_SB(inode), base_addr, is_inline); + return error; +} + +ssize_t f2fs_listxattr(struct dentry *dentry, char *buffer, size_t buffer_size) +{ + struct inode *inode = d_inode(dentry); + struct f2fs_xattr_entry *entry; + void *base_addr, *last_base_addr; + int error; + size_t rest = buffer_size; + + f2fs_down_read(&F2FS_I(inode)->i_xattr_sem); + error = read_all_xattrs(inode, NULL, &base_addr); + f2fs_up_read(&F2FS_I(inode)->i_xattr_sem); + if (error) + return error; + + last_base_addr = (void *)base_addr + XATTR_SIZE(inode); + + list_for_each_xattr(entry, base_addr) { + const char *prefix; + size_t prefix_len; + size_t size; + + if ((void *)(entry) + sizeof(__u32) > last_base_addr || + (void *)XATTR_NEXT_ENTRY(entry) > last_base_addr) { + f2fs_err(F2FS_I_SB(inode), "list inode (%llu) has corrupted xattr", + inode->i_ino); + set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK); + f2fs_handle_error(F2FS_I_SB(inode), + ERROR_CORRUPTED_XATTR); + fserror_report_file_metadata(inode, + -EFSCORRUPTED, GFP_NOFS); + error = -EFSCORRUPTED; + goto cleanup; + } + + prefix = f2fs_xattr_prefix(entry->e_name_index, dentry); + if (!prefix) + continue; + + prefix_len = strlen(prefix); + size = prefix_len + entry->e_name_len + 1; + if (buffer) { + if (size > rest) { + error = -ERANGE; + goto cleanup; + } + memcpy(buffer, prefix, prefix_len); + buffer += prefix_len; + memcpy(buffer, entry->e_name, entry->e_name_len); + buffer += entry->e_name_len; + *buffer++ = 0; + } + rest -= size; + } + error = buffer_size - rest; +cleanup: + kfree(base_addr); + return error; +} + +static bool f2fs_xattr_value_same(struct f2fs_xattr_entry *entry, + const void *value, size_t size) +{ + void *pval = entry->e_name + entry->e_name_len; + + return (le16_to_cpu(entry->e_value_size) == size) && + !memcmp(pval, value, size); +} + +static int __f2fs_setxattr(struct inode *inode, int index, + const char *name, const void *value, size_t size, + struct folio *ifolio, int flags) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_xattr_entry *here, *last; + void *base_addr, *last_base_addr; + int found, newsize; + size_t len; + __u32 new_hsize; + int error; + + if (name == NULL) + return -EINVAL; + + if (value == NULL) + size = 0; + + len = strlen(name); + + if (len > F2FS_NAME_LEN) + return -ERANGE; + + if (size > MAX_VALUE_LEN(inode)) + return -E2BIG; +retry: + error = read_all_xattrs(inode, ifolio, &base_addr); + if (error) + return error; + + last_base_addr = (void *)base_addr + XATTR_SIZE(inode); + + /* find entry with wanted name. */ + here = __find_xattr(base_addr, last_base_addr, NULL, index, len, name); + if (!here) { + if (!F2FS_I(inode)->i_xattr_nid) { + error = f2fs_recover_xattr_data(inode, NULL); + f2fs_notice(F2FS_I_SB(inode), + "recover xattr in inode (%llu), error(%d)", + inode->i_ino, error); + if (!error) { + kfree(base_addr); + goto retry; + } + } + f2fs_err(F2FS_I_SB(inode), "set inode (%llu) has corrupted xattr", + inode->i_ino); + set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK); + error = -EFSCORRUPTED; + f2fs_handle_error(F2FS_I_SB(inode), + ERROR_CORRUPTED_XATTR); + fserror_report_file_metadata(inode, error, GFP_NOFS); + goto exit; + } + + found = IS_XATTR_LAST_ENTRY(here) ? 0 : 1; + + if (found) { + if ((flags & XATTR_CREATE)) { + error = -EEXIST; + goto exit; + } + + if (value && f2fs_xattr_value_same(here, value, size)) + goto same; + } else if ((flags & XATTR_REPLACE)) { + error = -ENODATA; + goto exit; + } + + last = here; + while (!IS_XATTR_LAST_ENTRY(last)) { + if ((void *)(last) + sizeof(__u32) > last_base_addr || + (void *)XATTR_NEXT_ENTRY(last) > last_base_addr) { + f2fs_err(F2FS_I_SB(inode), "inode (%llu) has invalid last xattr entry, entry_size: %zu", + inode->i_ino, ENTRY_SIZE(last)); + set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK); + error = -EFSCORRUPTED; + f2fs_handle_error(F2FS_I_SB(inode), + ERROR_CORRUPTED_XATTR); + fserror_report_file_metadata(inode, error, GFP_NOFS); + goto exit; + } + last = XATTR_NEXT_ENTRY(last); + } + + newsize = XATTR_ALIGN(sizeof(struct f2fs_xattr_entry) + len + size); + + /* 1. Check space */ + if (value) { + int free; + /* + * If value is NULL, it is remove operation. + * In case of update operation, we calculate free. + */ + free = MIN_OFFSET(inode) - ((char *)last - (char *)base_addr); + if (found) + free = free + ENTRY_SIZE(here); + + if (unlikely(free < newsize)) { + error = -E2BIG; + goto exit; + } + } + + /* 2. Remove old entry */ + if (found) { + /* + * If entry is found, remove old entry. + * If not found, remove operation is not needed. + */ + struct f2fs_xattr_entry *next = XATTR_NEXT_ENTRY(here); + int oldsize = ENTRY_SIZE(here); + + memmove(here, next, (char *)last - (char *)next); + last = (struct f2fs_xattr_entry *)((char *)last - oldsize); + memset(last, 0, oldsize); + } + + new_hsize = (char *)last - (char *)base_addr; + + /* 3. Write new entry */ + if (value) { + char *pval; + /* + * Before we come here, old entry is removed. + * We just write new entry. + */ + last->e_name_index = index; + last->e_name_len = len; + memcpy(last->e_name, name, len); + pval = last->e_name + len; + memcpy(pval, value, size); + last->e_value_size = cpu_to_le16(size); + new_hsize += newsize; + /* + * Explicitly add the null terminator. The unused xattr space + * is supposed to always be zeroed, which would make this + * unnecessary, but don't depend on that. + */ + *(u32 *)((u8 *)last + newsize) = 0; + } + + error = write_all_xattrs(inode, new_hsize, base_addr, ifolio); + if (error) + goto exit; + + if (index == F2FS_XATTR_INDEX_ENCRYPTION && + !strcmp(name, F2FS_XATTR_NAME_ENCRYPTION_CONTEXT)) + f2fs_set_encrypted_inode(inode); + + if (!S_ISDIR(inode->i_mode)) + goto same; + /* + * In restrict mode, fsync() always try to trigger checkpoint for all + * metadata consistency, in other mode, it triggers checkpoint when + * parent's xattr metadata was updated. + */ + if (F2FS_OPTION(sbi).fsync_mode == FSYNC_MODE_STRICT) + set_sbi_flag(sbi, SBI_NEED_CP); + else + f2fs_add_ino_entry(sbi, inode->i_ino, XATTR_DIR_INO); +same: + if (is_inode_flag_set(inode, FI_ACL_MODE)) { + inode->i_mode = F2FS_I(inode)->i_acl_mode; + clear_inode_flag(inode, FI_ACL_MODE); + } + + inode_set_ctime_current(inode); + f2fs_mark_inode_dirty_sync(inode, true); +exit: + kfree(base_addr); + return error; +} + +int f2fs_setxattr(struct inode *inode, int index, const char *name, + const void *value, size_t size, + struct folio *ifolio, int flags) +{ + struct f2fs_sb_info *sbi = F2FS_I_SB(inode); + struct f2fs_lock_context lc; + int err; + + if (unlikely(f2fs_cp_error(sbi))) + return -EIO; + if (!f2fs_is_checkpoint_ready(sbi)) + return -ENOSPC; + + err = f2fs_dquot_initialize(inode); + if (err) + return err; + + /* this case is only from f2fs_init_inode_metadata */ + if (ifolio) + return __f2fs_setxattr(inode, index, name, value, + size, ifolio, flags); + f2fs_balance_fs(sbi, true); + + f2fs_lock_op(sbi, &lc); + f2fs_down_write(&F2FS_I(inode)->i_xattr_sem); + err = __f2fs_setxattr(inode, index, name, value, size, NULL, flags); + f2fs_up_write(&F2FS_I(inode)->i_xattr_sem); + f2fs_unlock_op(sbi, &lc); + + f2fs_update_time(sbi, REQ_TIME); + return err; +} + +int __init f2fs_init_xattr_cache(void) +{ + inline_xattr_slab = f2fs_kmem_cache_create("f2fs_xattr_entry", + DEFAULT_XATTR_SLAB_SIZE); + return inline_xattr_slab ? 0 : -ENOMEM; +} + +void f2fs_destroy_xattr_cache(void) +{ + kmem_cache_destroy(inline_xattr_slab); +} \ No newline at end of file diff --git a/fs/f2fs/xattr.h b/fs/f2fs/xattr.h new file mode 100644 index 000000000..bce3d93e4 --- /dev/null +++ b/fs/f2fs/xattr.h @@ -0,0 +1,169 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * fs/f2fs/xattr.h + * + * Copyright (c) 2012 Samsung Electronics Co., Ltd. + * http://www.samsung.com/ + * + * Portions of this code from linux/fs/ext2/xattr.h + * + * On-disk format of extended attributes for the ext2 filesystem. + * + * (C) 2001 Andreas Gruenbacher, + */ +#ifndef __F2FS_XATTR_H__ +#define __F2FS_XATTR_H__ + +#include +#include + +/* Magic value in attribute blocks */ +#define F2FS_XATTR_MAGIC 0xF2F52011 + +/* Maximum number of references to one attribute block */ +#define F2FS_XATTR_REFCOUNT_MAX 1024 + +/* Name indexes */ +#define F2FS_SYSTEM_ADVISE_NAME "system.advise" +#define F2FS_XATTR_INDEX_USER 1 +#define F2FS_XATTR_INDEX_POSIX_ACL_ACCESS 2 +#define F2FS_XATTR_INDEX_POSIX_ACL_DEFAULT 3 +#define F2FS_XATTR_INDEX_TRUSTED 4 +#define F2FS_XATTR_INDEX_LUSTRE 5 +#define F2FS_XATTR_INDEX_SECURITY 6 +#define F2FS_XATTR_INDEX_ADVISE 7 +/* Should be same as EXT4_XATTR_INDEX_ENCRYPTION */ +#define F2FS_XATTR_INDEX_ENCRYPTION 9 +#define F2FS_XATTR_INDEX_VERITY 11 + +#define F2FS_XATTR_NAME_ENCRYPTION_CONTEXT "c" +#define F2FS_XATTR_NAME_VERITY "v" + +struct f2fs_xattr_header { + __le32 h_magic; /* magic number for identification */ + __le32 h_refcount; /* reference count */ + __u32 h_reserved[4]; /* zero right now */ +}; + +struct f2fs_xattr_entry { + __u8 e_name_index; + __u8 e_name_len; + __le16 e_value_size; /* size of attribute value */ + char e_name[]; /* attribute name */ +}; + +#define XATTR_HDR(ptr) ((struct f2fs_xattr_header *)(ptr)) +#define XATTR_ENTRY(ptr) ((struct f2fs_xattr_entry *)(ptr)) +#define XATTR_FIRST_ENTRY(ptr) (XATTR_ENTRY(XATTR_HDR(ptr) + 1)) +#define XATTR_ROUND (3) + +#define XATTR_ALIGN(size) (((size) + XATTR_ROUND) & ~XATTR_ROUND) + +#define ENTRY_SIZE(entry) (XATTR_ALIGN(sizeof(struct f2fs_xattr_entry) + \ + (entry)->e_name_len + le16_to_cpu((entry)->e_value_size))) + +#define XATTR_NEXT_ENTRY(entry) ((struct f2fs_xattr_entry *)((char *)(entry) +\ + ENTRY_SIZE(entry))) + +#define IS_XATTR_LAST_ENTRY(entry) (*(__u32 *)(entry) == 0) + +#define list_for_each_xattr(entry, addr) \ + for (entry = XATTR_FIRST_ENTRY(addr);\ + !IS_XATTR_LAST_ENTRY(entry);\ + entry = XATTR_NEXT_ENTRY(entry)) +#define VALID_XATTR_BLOCK_SIZE (PAGE_SIZE - sizeof(struct node_footer)) +#define XATTR_PADDING_SIZE (sizeof(__u32)) +#define XATTR_SIZE(i) ((F2FS_I(i)->i_xattr_nid ? \ + VALID_XATTR_BLOCK_SIZE : 0) + \ + (inline_xattr_size(i))) +#define MIN_OFFSET(i) XATTR_ALIGN(inline_xattr_size(i) + \ + VALID_XATTR_BLOCK_SIZE) + +#define MAX_VALUE_LEN(i) (MIN_OFFSET(i) - \ + sizeof(struct f2fs_xattr_header) - \ + sizeof(struct f2fs_xattr_entry)) + +#define MIN_INLINE_XATTR_SIZE (sizeof(struct f2fs_xattr_header) / sizeof(__le32)) +#define MAX_INLINE_XATTR_SIZE \ + (DEF_ADDRS_PER_INODE - \ + F2FS_TOTAL_EXTRA_ATTR_SIZE / sizeof(__le32) - \ + DEF_INLINE_RESERVED_SIZE - \ + MIN_INLINE_DENTRY_SIZE / sizeof(__le32)) +#define DEFAULT_XATTR_SLAB_SIZE (DEFAULT_INLINE_XATTR_ADDRS * \ + sizeof(__le32) + XATTR_PADDING_SIZE) + +/* + * On-disk structure of f2fs_xattr + * We use inline xattrs space + 1 block for xattr. + * + * +--------------------+ + * | f2fs_xattr_header | + * | | + * +--------------------+ + * | f2fs_xattr_entry | + * | .e_name_index = 1 | + * | .e_name_len = 3 | + * | .e_value_size = 14 | + * | .e_name = "foo" | + * | "value_of_xattr" |<- value_offs = e_name + e_name_len + * +--------------------+ + * | f2fs_xattr_entry | + * | .e_name_index = 4 | + * | .e_name = "bar" | + * +--------------------+ + * | | + * | Free | + * | | + * +--------------------+<- MIN_OFFSET + * | node_footer | + * | (nid, ino, offset) | + * +--------------------+ + * + **/ + +#ifdef CONFIG_F2FS_FS_XATTR +extern const struct xattr_handler f2fs_xattr_user_handler; +extern const struct xattr_handler f2fs_xattr_trusted_handler; +extern const struct xattr_handler f2fs_xattr_advise_handler; +extern const struct xattr_handler f2fs_xattr_security_handler; + +extern const struct xattr_handler * const f2fs_xattr_handlers[]; + +int f2fs_setxattr(struct inode *, int, const char *, const void *, + size_t, struct folio *, int); +int f2fs_getxattr(struct inode *, int, const char *, void *, + size_t, struct folio *); +ssize_t f2fs_listxattr(struct dentry *, char *, size_t); +int __init f2fs_init_xattr_cache(void); +void f2fs_destroy_xattr_cache(void); +#else + +#define f2fs_xattr_handlers NULL +#define f2fs_listxattr NULL +static inline int f2fs_setxattr(struct inode *inode, int index, + const char *name, const void *value, size_t size, + struct folio *folio, int flags) +{ + return -EOPNOTSUPP; +} +static inline int f2fs_getxattr(struct inode *inode, int index, + const char *name, void *buffer, + size_t buffer_size, struct folio *dfolio) +{ + return -EOPNOTSUPP; +} +static inline int __init f2fs_init_xattr_cache(void) { return 0; } +static inline void f2fs_destroy_xattr_cache(void) { } +#endif + +#ifdef CONFIG_F2FS_FS_SECURITY +int f2fs_init_security(struct inode *, struct inode *, + const struct qstr *, struct folio *); +#else +static inline int f2fs_init_security(struct inode *inode, struct inode *dir, + const struct qstr *qstr, struct folio *ifolio) +{ + return 0; +} +#endif +#endif /* __F2FS_XATTR_H__ */ diff --git a/fs/failfs.c b/fs/failfs.c new file mode 100644 index 000000000..66a36da3d --- /dev/null +++ b/fs/failfs.c @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2026 Christian Brauner */ +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +static struct path failfs_root_path = {}; + +void failfs_get_root(struct path *path) +{ + *path = failfs_root_path; + path_get(path); +} + +bool failfs_mnt(const struct vfsmount *mnt) +{ + return mnt->mnt_sb == failfs_root_path.mnt->mnt_sb; +} + +static int failfs_permission(struct mnt_idmap *idmap, struct inode *inode, + int mask) +{ + return -EOPNOTSUPP; +} + +static struct dentry *failfs_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + /* Unreachable: ->permission() already failed the walk. */ + return ERR_PTR(-EOPNOTSUPP); +} + +static int failfs_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, + unsigned int query_flags) +{ + return -EOPNOTSUPP; +} + +static const struct inode_operations failfs_dir_inode_operations = { + .permission = failfs_permission, + .lookup = failfs_lookup, + .getattr = failfs_getattr, +}; + +static const struct file_operations failfs_dir_operations = {}; + +static int failfs_d_weak_revalidate(struct dentry *dentry, unsigned int flags) +{ + /* + * The root is only ever reached as a path-walk terminal by jumping + * to it: as "/" when it is the caller's root, or through a + * /proc//{root,cwd} magic link. ->permission() already fails + * every walk of a component, but a jump lands on the root without + * one. Refuse here too so the root cannot be pinned by an O_PATH + * open or encoded into a file handle. + */ + return -EOPNOTSUPP; +} + +static char *failfs_dname(struct dentry *dentry, char *buffer, int buflen) +{ + return dynamic_dname(buffer, buflen, "failfs:/"); +} + +static const struct dentry_operations failfs_dentry_operations = { + .d_dname = failfs_dname, + .d_weak_revalidate = failfs_d_weak_revalidate, +}; + +static int failfs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + return -EOPNOTSUPP; +} + +static const struct super_operations failfs_super_operations = { + .statfs = failfs_statfs, +}; + +static int failfs_fill_super(struct super_block *s, struct fs_context *fc) +{ + struct inode *inode; + + s->s_maxbytes = MAX_LFS_FILESIZE; + s->s_blocksize = PAGE_SIZE; + s->s_blocksize_bits = PAGE_SHIFT; + s->s_magic = FAIL_FS_MAGIC; + s->s_op = &failfs_super_operations; + s->s_export_op = NULL; + s->s_xattr = NULL; + s->s_time_gran = 1; + s->s_d_flags = 0; + + inode = new_inode(s); + if (!inode) + return -ENOMEM; + + /* failfs supports no operations... */ + inode->i_mode = S_IFDIR; + set_nlink(inode, 2); + inode->i_op = &failfs_dir_inode_operations; + inode->i_fop = &failfs_dir_operations; + simple_inode_init_ts(inode); + inode->i_ino = 1; + /* ... and is immutable. */ + inode->i_flags |= S_IMMUTABLE; + + set_default_d_op(s, &failfs_dentry_operations); + s->s_root = d_make_root(inode); + if (!s->s_root) + return -ENOMEM; + + return 0; +} + +static int failfs_get_tree(struct fs_context *fc) +{ + return get_tree_single(fc, failfs_fill_super); +} + +static const struct fs_context_operations failfs_context_ops = { + .get_tree = failfs_get_tree, +}; + +static int failfs_init_fs_context(struct fs_context *fc) +{ + fc->ops = &failfs_context_ops; + fc->global = true; + fc->sb_flags |= SB_NOUSER; + fc->s_iflags |= SB_I_NOEXEC | SB_I_NODEV; + return 0; +} + +int failfs_current_chdir(void) +{ + struct path path; + + failfs_get_root(&path); + set_fs_pwd(current->fs, &path); + path_put(&path); + return 0; +} + +static struct file_system_type failfs_fs_type = { + .name = "failfs", + .init_fs_context = failfs_init_fs_context, + .kill_sb = kill_anon_super, +}; + +void __init failfs_init(void) +{ + struct vfsmount *mnt; + + /* A single instance that is member of no mount namespace. */ + mnt = kern_mount(&failfs_fs_type); + if (IS_ERR(mnt)) + panic("VFS: Failed to create failfs"); + + failfs_root_path.mnt = mnt; + failfs_root_path.dentry = mnt->mnt_root; +} diff --git a/fs/fat/.kunitconfig b/fs/fat/.kunitconfig new file mode 100644 index 000000000..0a6971dbe --- /dev/null +++ b/fs/fat/.kunitconfig @@ -0,0 +1,5 @@ +CONFIG_KUNIT=y +CONFIG_FAT_FS=y +CONFIG_MSDOS_FS=y +CONFIG_VFAT_FS=y +CONFIG_FAT_KUNIT_TEST=y diff --git a/fs/fat/Kconfig b/fs/fat/Kconfig new file mode 100644 index 000000000..25fae1c83 --- /dev/null +++ b/fs/fat/Kconfig @@ -0,0 +1,131 @@ +# SPDX-License-Identifier: GPL-2.0-only +config FAT_FS + tristate + select BUFFER_HEAD + select NLS + select LEGACY_DIRECT_IO + help + If you want to use one of the FAT-based file systems (the MS-DOS and + VFAT (Windows 95) file systems), then you must say Y or M here + to include FAT support. You will then be able to mount partitions or + diskettes with FAT-based file systems and transparently access the + files on them, i.e. MSDOS files will look and behave just like all + other Unix files. + + This FAT support is not a file system in itself, it only provides + the foundation for the other file systems. You will have to say Y or + M to at least one of "MSDOS fs support" or "VFAT fs support" in + order to make use of it. + + Another way to read and write MSDOS floppies and hard drive + partitions from within Linux (but not transparently) is with the + mtools ("man mtools") program suite. You don't need to say Y here in + order to do that. + + If you need to move large files on floppies between a DOS and a + Linux box, say Y here, mount the floppy under Linux with an MSDOS + file system and use GNU tar's M option. GNU tar is a program + available for Unix and DOS ("man tar" or "info tar"). + + The FAT support will enlarge your kernel by about 37 KB. If unsure, + say Y. + + To compile this as a module, choose M here: the module will be called + fat. Note that if you compile the FAT support as a module, you + cannot compile any of the FAT-based file systems into the kernel + -- they will have to be modules as well. + +config MSDOS_FS + tristate "MSDOS fs support" + select FAT_FS + help + This allows you to mount MSDOS partitions of your hard drive (unless + they are compressed; to access compressed MSDOS partitions under + Linux, you can either use the DOS emulator DOSEMU, described in the + DOSEMU-HOWTO, available from + , or try dmsdosfs in + . If you + intend to use dosemu with a non-compressed MSDOS partition, say Y + here) and MSDOS floppies. This means that file access becomes + transparent, i.e. the MSDOS files look and behave just like all + other Unix files. + + If you have Windows 95 or Windows NT installed on your MSDOS + partitions, you should use the VFAT file system (say Y to "VFAT fs + support" below), or you will not be able to see the long filenames + generated by Windows 95 / Windows NT. + + This option will enlarge your kernel by about 7 KB. If unsure, + answer Y. This will only work if you said Y to "DOS FAT fs support" + as well. To compile this as a module, choose M here: the module will + be called msdos. + +config VFAT_FS + tristate "VFAT (Windows-95) fs support" + select FAT_FS + help + This option provides support for normal Windows file systems with + long filenames. That includes non-compressed FAT-based file systems + used by Windows 95, Windows 98, Windows NT 4.0, and the Unix + programs from the mtools package. + + The VFAT support enlarges your kernel by about 10 KB and it only + works if you said Y to the "DOS FAT fs support" above. Please read + the file for details. If + unsure, say Y. + + To compile this as a module, choose M here: the module will be called + vfat. + +config FAT_DEFAULT_CODEPAGE + int "Default codepage for FAT" + depends on FAT_FS + default 437 + help + This option should be set to the codepage of your FAT filesystems. + It can be overridden with the "codepage" mount option. + See for more information. + +config FAT_DEFAULT_IOCHARSET + string "Default iocharset for FAT" + depends on VFAT_FS + default "iso8859-1" + help + Set this to the default input/output character set you'd + like FAT to use. It should probably match the character set + that most of your FAT filesystems use, and can be overridden + with the "iocharset" mount option for FAT filesystems. + Note that "utf8" is not recommended for FAT filesystems. + If unsure, you shouldn't set "utf8" here - select the next option + instead if you would like to use UTF-8 encoded file names by default. + See for more information. + + Enable any character sets you need in File Systems/Native Language + Support. + +config FAT_DEFAULT_UTF8 + bool "Enable FAT UTF-8 option by default" + depends on VFAT_FS + default n + help + Set this if you would like to have "utf8" mount option set + by default when mounting FAT filesystems. + + Even if you say Y here can always disable UTF-8 for + particular mount by adding "utf8=0" to mount options. + + Say Y if you use UTF-8 encoding for file names, N otherwise. + + See for more information. + +config FAT_KUNIT_TEST + tristate "Unit Tests for FAT filesystems" if !KUNIT_ALL_TESTS + depends on KUNIT && FAT_FS + default KUNIT_ALL_TESTS + help + This builds the FAT KUnit tests + + For more information on KUnit and unit tests in general, please refer + to the KUnit documentation in Documentation/dev-tools/kunit + + If unsure, say N diff --git a/fs/fat/Makefile b/fs/fat/Makefile new file mode 100644 index 000000000..2b0341126 --- /dev/null +++ b/fs/fat/Makefile @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the Linux fat filesystem support. +# + +obj-$(CONFIG_FAT_FS) += fat.o +obj-$(CONFIG_VFAT_FS) += vfat.o +obj-$(CONFIG_MSDOS_FS) += msdos.o + +fat-y := cache.o dir.o fatent.o file.o inode.o misc.o nfs.o +vfat-y := namei_vfat.o +msdos-y := namei_msdos.o + +obj-$(CONFIG_FAT_KUNIT_TEST) += fat_test.o diff --git a/fs/fat/cache.c b/fs/fat/cache.c new file mode 100644 index 000000000..1b87354e2 --- /dev/null +++ b/fs/fat/cache.c @@ -0,0 +1,381 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/fat/cache.c + * + * Written 1992,1993 by Werner Almesberger + * + * Mar 1999. AV. Changed cache, so that it uses the starting cluster instead + * of inode number. + * May 1999. AV. Fixed the bogosity with FAT32 (read "FAT28"). Fscking lusers. + */ + +#include +#include "fat.h" + +/* this must be > 0. */ +#define FAT_MAX_CACHE 8 + +struct fat_cache { + struct list_head cache_list; + int nr_contig; /* number of contiguous clusters */ + int fcluster; /* cluster number in the file. */ + int dcluster; /* cluster number on disk. */ +}; + +struct fat_cache_id { + unsigned int id; + int nr_contig; + int fcluster; + int dcluster; +}; + +static struct kmem_cache *fat_cache_cachep; + +static void init_once(void *foo) +{ + struct fat_cache *cache = (struct fat_cache *)foo; + + INIT_LIST_HEAD(&cache->cache_list); +} + +int __init fat_cache_init(void) +{ + fat_cache_cachep = kmem_cache_create("fat_cache", + sizeof(struct fat_cache), + 0, SLAB_RECLAIM_ACCOUNT, + init_once); + if (fat_cache_cachep == NULL) + return -ENOMEM; + return 0; +} + +void fat_cache_destroy(void) +{ + kmem_cache_destroy(fat_cache_cachep); +} + +static inline struct fat_cache *fat_cache_alloc(void) +{ + return kmem_cache_alloc(fat_cache_cachep, GFP_NOFS); +} + +static inline void fat_cache_free(struct fat_cache *cache) +{ + BUG_ON(!list_empty(&cache->cache_list)); + kmem_cache_free(fat_cache_cachep, cache); +} + +static inline void fat_cache_update_lru(struct inode *inode, + struct fat_cache *cache) +{ + if (MSDOS_I(inode)->cache_lru.next != &cache->cache_list) + list_move(&cache->cache_list, &MSDOS_I(inode)->cache_lru); +} + +static int fat_cache_lookup(struct inode *inode, int fclus, + struct fat_cache_id *cid, + int *cached_fclus, int *cached_dclus) +{ + static struct fat_cache nohit = { .fcluster = 0, }; + + struct fat_cache *hit = &nohit, *p; + int offset = -1; + + spin_lock(&MSDOS_I(inode)->cache_lru_lock); + list_for_each_entry(p, &MSDOS_I(inode)->cache_lru, cache_list) { + /* Find the cache of "fclus" or nearest cache. */ + if (p->fcluster <= fclus && hit->fcluster < p->fcluster) { + hit = p; + if ((hit->fcluster + hit->nr_contig) < fclus) { + offset = hit->nr_contig; + } else { + offset = fclus - hit->fcluster; + break; + } + } + } + if (hit != &nohit) { + fat_cache_update_lru(inode, hit); + + cid->id = MSDOS_I(inode)->cache_valid_id; + cid->nr_contig = hit->nr_contig; + cid->fcluster = hit->fcluster; + cid->dcluster = hit->dcluster; + *cached_fclus = cid->fcluster + offset; + *cached_dclus = cid->dcluster + offset; + } + spin_unlock(&MSDOS_I(inode)->cache_lru_lock); + + return offset; +} + +static struct fat_cache *fat_cache_merge(struct inode *inode, + struct fat_cache_id *new) +{ + struct fat_cache *p; + + list_for_each_entry(p, &MSDOS_I(inode)->cache_lru, cache_list) { + /* Find the same part as "new" in cluster-chain. */ + if (p->fcluster == new->fcluster) { + BUG_ON(p->dcluster != new->dcluster); + if (new->nr_contig > p->nr_contig) + p->nr_contig = new->nr_contig; + return p; + } + } + return NULL; +} + +static void fat_cache_add(struct inode *inode, struct fat_cache_id *new) +{ + struct fat_cache *cache, *tmp; + + if (new->fcluster == -1) /* dummy cache */ + return; + + spin_lock(&MSDOS_I(inode)->cache_lru_lock); + if (new->id != FAT_CACHE_VALID && + new->id != MSDOS_I(inode)->cache_valid_id) + goto out; /* this cache was invalidated */ + + cache = fat_cache_merge(inode, new); + if (cache == NULL) { + if (MSDOS_I(inode)->nr_caches < FAT_MAX_CACHE) { + MSDOS_I(inode)->nr_caches++; + spin_unlock(&MSDOS_I(inode)->cache_lru_lock); + + tmp = fat_cache_alloc(); + if (!tmp) { + spin_lock(&MSDOS_I(inode)->cache_lru_lock); + MSDOS_I(inode)->nr_caches--; + spin_unlock(&MSDOS_I(inode)->cache_lru_lock); + return; + } + + spin_lock(&MSDOS_I(inode)->cache_lru_lock); + cache = fat_cache_merge(inode, new); + if (cache != NULL) { + MSDOS_I(inode)->nr_caches--; + fat_cache_free(tmp); + goto out_update_lru; + } + cache = tmp; + } else { + struct list_head *p = MSDOS_I(inode)->cache_lru.prev; + cache = list_entry(p, struct fat_cache, cache_list); + } + cache->fcluster = new->fcluster; + cache->dcluster = new->dcluster; + cache->nr_contig = new->nr_contig; + } +out_update_lru: + fat_cache_update_lru(inode, cache); +out: + spin_unlock(&MSDOS_I(inode)->cache_lru_lock); +} + +/* + * Cache invalidation occurs rarely, thus the LRU chain is not updated. It + * fixes itself after a while. + */ +static void __fat_cache_inval_inode(struct inode *inode) +{ + struct msdos_inode_info *i = MSDOS_I(inode); + struct fat_cache *cache; + + while (!list_empty(&i->cache_lru)) { + cache = list_entry(i->cache_lru.next, + struct fat_cache, cache_list); + list_del_init(&cache->cache_list); + i->nr_caches--; + fat_cache_free(cache); + } + /* Update. The copy of caches before this id is discarded. */ + i->cache_valid_id++; + if (i->cache_valid_id == FAT_CACHE_VALID) + i->cache_valid_id++; +} + +void fat_cache_inval_inode(struct inode *inode) +{ + spin_lock(&MSDOS_I(inode)->cache_lru_lock); + __fat_cache_inval_inode(inode); + spin_unlock(&MSDOS_I(inode)->cache_lru_lock); +} + +static inline int cache_contiguous(struct fat_cache_id *cid, int dclus) +{ + cid->nr_contig++; + return ((cid->dcluster + cid->nr_contig) == dclus); +} + +static inline void cache_init(struct fat_cache_id *cid, int fclus, int dclus) +{ + cid->id = FAT_CACHE_VALID; + cid->fcluster = fclus; + cid->dcluster = dclus; + cid->nr_contig = 0; +} + +int fat_get_cluster(struct inode *inode, int cluster, int *fclus, int *dclus) +{ + struct super_block *sb = inode->i_sb; + struct msdos_sb_info *sbi = MSDOS_SB(sb); + const int limit = sb->s_maxbytes >> sbi->cluster_bits; + struct fat_entry fatent; + struct fat_cache_id cid; + int nr; + + BUG_ON(MSDOS_I(inode)->i_start == 0); + + *fclus = 0; + *dclus = MSDOS_I(inode)->i_start; + if (!fat_valid_entry(sbi, *dclus)) { + fat_fs_error_ratelimit(sb, + "%s: invalid start cluster (i_pos %lld, start %08x)", + __func__, MSDOS_I(inode)->i_pos, *dclus); + return -EIO; + } + if (cluster == 0) + return 0; + + if (fat_cache_lookup(inode, cluster, &cid, fclus, dclus) < 0) { + /* + * dummy, always not contiguous + * This is reinitialized by cache_init(), later. + */ + cache_init(&cid, -1, -1); + } + + fatent_init(&fatent); + while (*fclus < cluster) { + /* prevent the infinite loop of cluster chain */ + if (*fclus > limit) { + fat_fs_error_ratelimit(sb, + "%s: detected the cluster chain loop (i_pos %lld)", + __func__, MSDOS_I(inode)->i_pos); + nr = -EIO; + goto out; + } + + nr = fat_ent_read(inode, &fatent, *dclus); + if (nr < 0) + goto out; + else if (nr == FAT_ENT_FREE) { + fat_fs_error_ratelimit(sb, + "%s: invalid cluster chain (i_pos %lld)", + __func__, MSDOS_I(inode)->i_pos); + nr = -EIO; + goto out; + } else if (nr == FAT_ENT_EOF) { + fat_cache_add(inode, &cid); + goto out; + } + (*fclus)++; + *dclus = nr; + if (!cache_contiguous(&cid, *dclus)) + cache_init(&cid, *fclus, *dclus); + } + nr = 0; + fat_cache_add(inode, &cid); +out: + fatent_brelse(&fatent); + return nr; +} + +static int fat_bmap_cluster(struct inode *inode, int cluster) +{ + struct super_block *sb = inode->i_sb; + int ret, fclus, dclus; + + if (MSDOS_I(inode)->i_start == 0) + return 0; + + ret = fat_get_cluster(inode, cluster, &fclus, &dclus); + if (ret < 0) + return ret; + else if (ret == FAT_ENT_EOF) { + fat_fs_error(sb, "%s: request beyond EOF (i_pos %lld)", + __func__, MSDOS_I(inode)->i_pos); + return -EIO; + } + return dclus; +} + +int fat_get_mapped_cluster(struct inode *inode, sector_t sector, + sector_t last_block, + unsigned long *mapped_blocks, sector_t *bmap) +{ + struct super_block *sb = inode->i_sb; + struct msdos_sb_info *sbi = MSDOS_SB(sb); + int cluster, offset; + + cluster = sector >> (sbi->cluster_bits - sb->s_blocksize_bits); + offset = sector & (sbi->sec_per_clus - 1); + cluster = fat_bmap_cluster(inode, cluster); + if (cluster < 0) + return cluster; + else if (cluster) { + *bmap = fat_clus_to_blknr(sbi, cluster) + offset; + *mapped_blocks = sbi->sec_per_clus - offset; + if (*mapped_blocks > last_block - sector) + *mapped_blocks = last_block - sector; + } + + return 0; +} + +static int is_exceed_eof(struct inode *inode, sector_t sector, + sector_t *last_block, int create) +{ + struct super_block *sb = inode->i_sb; + const unsigned long blocksize = sb->s_blocksize; + const unsigned char blocksize_bits = sb->s_blocksize_bits; + + *last_block = (i_size_read(inode) + (blocksize - 1)) >> blocksize_bits; + if (sector >= *last_block) { + if (!create) + return 1; + + /* + * ->mmu_private can access on only allocation path. + * (caller must hold ->i_mutex) + */ + *last_block = (MSDOS_I(inode)->mmu_private + (blocksize - 1)) + >> blocksize_bits; + if (sector >= *last_block) + return 1; + } + + return 0; +} + +int fat_bmap(struct inode *inode, sector_t sector, sector_t *phys, + unsigned long *mapped_blocks, int create, bool from_bmap) +{ + struct msdos_sb_info *sbi = MSDOS_SB(inode->i_sb); + sector_t last_block; + + *phys = 0; + *mapped_blocks = 0; + if (!is_fat32(sbi) && (inode->i_ino == MSDOS_ROOT_INO)) { + if (sector < (sbi->dir_entries >> sbi->dir_per_block_bits)) { + *phys = sector + sbi->dir_start; + *mapped_blocks = 1; + } + return 0; + } + + if (!from_bmap) { + if (is_exceed_eof(inode, sector, &last_block, create)) + return 0; + } else { + last_block = inode->i_blocks >> + (inode->i_sb->s_blocksize_bits - 9); + if (sector >= last_block) + return 0; + } + + return fat_get_mapped_cluster(inode, sector, last_block, mapped_blocks, + phys); +} diff --git a/fs/fat/dir.c b/fs/fat/dir.c new file mode 100644 index 000000000..35bdb6294 --- /dev/null +++ b/fs/fat/dir.c @@ -0,0 +1,1467 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/fat/dir.c + * + * directory handling functions for fat-based filesystems + * + * Written 1992,1993 by Werner Almesberger + * + * Hidden files 1995 by Albert Cahalan + * + * VFAT extensions by Gordon Chaffee + * Merged with msdos fs by Henrik Storner + * Rewritten for constant inumbers. Plugged buffer overrun in readdir(). AV + * Short name translation 1999, 2001 by Wolfram Pienkoss + */ + +#include +#include +#include +#include +#include +#include +#include "fat.h" + +/* + * Maximum buffer size of short name. + * [(MSDOS_NAME + '.') * max one char + nul] + * For msdos style, ['.' (hidden) + MSDOS_NAME + '.' + nul] + */ +#define FAT_MAX_SHORT_SIZE ((MSDOS_NAME + 1) * NLS_MAX_CHARSET_SIZE + 1) +/* + * Maximum buffer size of unicode chars from slots. + * [(max longname slots * 13 (size in a slot) + nul) * sizeof(wchar_t)] + */ +#define FAT_MAX_UNI_CHARS ((MSDOS_SLOTS - 1) * 13 + 1) +#define FAT_MAX_UNI_SIZE (FAT_MAX_UNI_CHARS * sizeof(wchar_t)) + +static inline unsigned char fat_tolower(unsigned char c) +{ + return ((c >= 'A') && (c <= 'Z')) ? c+32 : c; +} + +static inline loff_t fat_make_i_pos(struct super_block *sb, + struct buffer_head *bh, + struct msdos_dir_entry *de) +{ + return ((loff_t)bh->b_blocknr << MSDOS_SB(sb)->dir_per_block_bits) + | (de - (struct msdos_dir_entry *)bh->b_data); +} + +static inline void fat_dir_readahead(struct inode *dir, sector_t iblock, + sector_t phys) +{ + struct super_block *sb = dir->i_sb; + struct msdos_sb_info *sbi = MSDOS_SB(sb); + struct buffer_head *bh; + int sec; + + /* This is not a first sector of cluster, or sec_per_clus == 1 */ + if ((iblock & (sbi->sec_per_clus - 1)) || sbi->sec_per_clus == 1) + return; + /* root dir of FAT12/FAT16 */ + if (!is_fat32(sbi) && (dir->i_ino == MSDOS_ROOT_INO)) + return; + + bh = sb_find_get_block(sb, phys); + if (bh == NULL || !buffer_uptodate(bh)) { + for (sec = 0; sec < sbi->sec_per_clus; sec++) + sb_breadahead(sb, phys + sec); + } + brelse(bh); +} + +/* Returns the inode number of the directory entry at offset pos. If bh is + non-NULL, it is brelse'd before. Pos is incremented. The buffer header is + returned in bh. + AV. Most often we do it item-by-item. Makes sense to optimize. + AV. OK, there we go: if both bh and de are non-NULL we assume that we just + AV. want the next entry (took one explicit de=NULL in vfat/namei.c). + AV. It's done in fat_get_entry() (inlined), here the slow case lives. + AV. Additionally, when we return -1 (i.e. reached the end of directory) + AV. we make bh NULL. + */ +static int fat__get_entry(struct inode *dir, loff_t *pos, + struct buffer_head **bh, struct msdos_dir_entry **de) +{ + struct super_block *sb = dir->i_sb; + sector_t phys, iblock; + unsigned long mapped_blocks; + int err, offset; + +next: + brelse(*bh); + *bh = NULL; + iblock = *pos >> sb->s_blocksize_bits; + err = fat_bmap(dir, iblock, &phys, &mapped_blocks, 0, false); + if (err || !phys) + return -1; /* beyond EOF or error */ + + fat_dir_readahead(dir, iblock, phys); + + *bh = sb_bread(sb, phys); + if (*bh == NULL) { + fat_msg_ratelimit(sb, KERN_ERR, + "Directory bread(block %llu) failed", (llu)phys); + /* skip this block */ + *pos = (iblock + 1) << sb->s_blocksize_bits; + goto next; + } + + offset = *pos & (sb->s_blocksize - 1); + *pos += sizeof(struct msdos_dir_entry); + *de = (struct msdos_dir_entry *)((*bh)->b_data + offset); + + return 0; +} + +static inline int fat_get_entry(struct inode *dir, loff_t *pos, + struct buffer_head **bh, + struct msdos_dir_entry **de) +{ + /* Fast stuff first */ + if (*bh && *de && + (*de - (struct msdos_dir_entry *)(*bh)->b_data) < + MSDOS_SB(dir->i_sb)->dir_per_block - 1) { + *pos += sizeof(struct msdos_dir_entry); + (*de)++; + return 0; + } + return fat__get_entry(dir, pos, bh, de); +} + +/* + * Like fat_get_entry(), but honour the FAT end-of-directory marker: + * a dirent whose first name byte is NUL terminates iteration per the + * spec, which also guarantees that every following slot is zeroed. + * Skip straight to the end of the directory so the next call returns + * -1 from fat_bmap() without re-reading the trailing zero slots, and + * so callers that persist *pos across invocations (e.g. readdir's + * ctx->pos) keep reporting EOD. Release *bh and set it to NULL to + * match fat_get_entry()'s contract that *bh is NULL on the -1 return. + */ +static int fat_get_entry_eod(struct inode *dir, loff_t *pos, + struct buffer_head **bh, + struct msdos_dir_entry **de) +{ + int err = fat_get_entry(dir, pos, bh, de); + + if (err == 0 && (*de)->name[0] == 0) { + brelse(*bh); + *bh = NULL; + *pos = dir->i_size; + return -1; + } + return err; +} + +/* + * Convert Unicode 16 to UTF-8, translated Unicode, or ASCII. + * If uni_xlate is enabled and we can't get a 1:1 conversion, use a + * colon as an escape character since it is normally invalid on the vfat + * filesystem. The following four characters are the hexadecimal digits + * of Unicode value. This lets us do a full dump and restore of Unicode + * filenames. We could get into some trouble with long Unicode names, + * but ignore that right now. + * Ahem... Stack smashing in ring 0 isn't fun. Fixed. + */ +static int uni16_to_x8(struct super_block *sb, unsigned char *ascii, + const wchar_t *uni, int len, struct nls_table *nls) +{ + int uni_xlate = MSDOS_SB(sb)->options.unicode_xlate; + const wchar_t *ip; + wchar_t ec; + unsigned char *op; + int charlen; + + ip = uni; + op = ascii; + + while (*ip && ((len - NLS_MAX_CHARSET_SIZE) > 0)) { + ec = *ip++; + charlen = nls->uni2char(ec, op, NLS_MAX_CHARSET_SIZE); + if (charlen > 0) { + op += charlen; + len -= charlen; + } else { + if (uni_xlate == 1) { + *op++ = ':'; + op = hex_byte_pack(op, ec >> 8); + op = hex_byte_pack(op, ec); + len -= 5; + } else { + *op++ = '?'; + len--; + } + } + } + + if (unlikely(*ip)) { + fat_msg(sb, KERN_WARNING, + "filename was truncated while converting."); + } + + *op = 0; + return op - ascii; +} + +static inline int fat_uni_to_x8(struct super_block *sb, const wchar_t *uni, + unsigned char *buf, int size) +{ + struct msdos_sb_info *sbi = MSDOS_SB(sb); + if (sbi->options.utf8) + return utf16s_to_utf8s(uni, FAT_MAX_UNI_CHARS, + UTF16_HOST_ENDIAN, buf, size); + else + return uni16_to_x8(sb, buf, uni, size, sbi->nls_io); +} + +static inline int +fat_short2uni(struct nls_table *t, unsigned char *c, int clen, wchar_t *uni) +{ + int charlen; + + charlen = t->char2uni(c, clen, uni); + if (charlen < 0) { + *uni = 0x003f; /* a question mark */ + charlen = 1; + } + return charlen; +} + +static inline int +fat_short2lower_uni(struct nls_table *t, unsigned char *c, + int clen, wchar_t *uni) +{ + int charlen; + wchar_t wc; + + charlen = t->char2uni(c, clen, &wc); + if (charlen < 0) { + *uni = 0x003f; /* a question mark */ + charlen = 1; + } else if (charlen <= 1) { + unsigned char nc = t->charset2lower[*c]; + + if (!nc) + nc = *c; + + charlen = t->char2uni(&nc, 1, uni); + if (charlen < 0) { + *uni = 0x003f; /* a question mark */ + charlen = 1; + } + } else + *uni = wc; + + return charlen; +} + +static inline int +fat_shortname2uni(struct nls_table *nls, unsigned char *buf, int buf_size, + wchar_t *uni_buf, unsigned short opt, int lower) +{ + int len = 0; + + if (opt & VFAT_SFN_DISPLAY_LOWER) + len = fat_short2lower_uni(nls, buf, buf_size, uni_buf); + else if (opt & VFAT_SFN_DISPLAY_WIN95) + len = fat_short2uni(nls, buf, buf_size, uni_buf); + else if (opt & VFAT_SFN_DISPLAY_WINNT) { + if (lower) + len = fat_short2lower_uni(nls, buf, buf_size, uni_buf); + else + len = fat_short2uni(nls, buf, buf_size, uni_buf); + } else + len = fat_short2uni(nls, buf, buf_size, uni_buf); + + return len; +} + +static inline int fat_name_match(struct msdos_sb_info *sbi, + const unsigned char *a, int a_len, + const unsigned char *b, int b_len) +{ + if (a_len != b_len) + return 0; + + if (sbi->options.name_check != 's') + return !nls_strnicmp(sbi->nls_io, a, b, a_len); + else + return !memcmp(a, b, a_len); +} + +enum { PARSE_INVALID = 1, PARSE_NOT_LONGNAME, PARSE_EOF, }; + +/** + * fat_parse_long - Parse extended directory entry. + * + * @dir: Pointer to the inode that represents the directory. + * @pos: On input, contains the starting position to read from. + * On output, updated with the new position. + * @bh: Pointer to the buffer head that may be used for reading directory + * entries. May be updated. + * @de: On input, points to the current directory entry. + * On output, points to the next directory entry. + * @unicode: Pointer to a buffer where the parsed Unicode long filename will be + * stored. + * @nr_slots: Pointer to a variable that will store the number of longname + * slots found. + * + * This function returns zero on success, negative value on error, or one of + * the following: + * + * %PARSE_INVALID - Directory entry is invalid. + * %PARSE_NOT_LONGNAME - Directory entry does not contain longname. + * %PARSE_EOF - Directory has no more entries. + */ +static int fat_parse_long(struct inode *dir, loff_t *pos, + struct buffer_head **bh, struct msdos_dir_entry **de, + wchar_t **unicode, unsigned char *nr_slots) +{ + struct msdos_dir_slot *ds; + unsigned char id, slot, slots, alias_checksum; + + if (!*unicode) { + *unicode = __getname(); + if (!*unicode) { + brelse(*bh); + return -ENOMEM; + } + } +parse_long: + ds = (struct msdos_dir_slot *)*de; + id = ds->id; + if (!(id & 0x40)) + return PARSE_INVALID; + slots = id & ~0x40; + if (slots > 20 || !slots) /* ceil(256 * 2 / 26) */ + return PARSE_INVALID; + *nr_slots = slots; + alias_checksum = ds->alias_checksum; + + slot = slots; + while (1) { + int offset; + + slot--; + offset = slot * 13; + fat16_towchar(*unicode + offset, ds->name0_4, 5); + fat16_towchar(*unicode + offset + 5, ds->name5_10, 6); + fat16_towchar(*unicode + offset + 11, ds->name11_12, 2); + + if (ds->id & 0x40) + (*unicode)[offset + 13] = 0; + if (fat_get_entry_eod(dir, pos, bh, de) < 0) + return PARSE_EOF; + if (slot == 0) + break; + ds = (struct msdos_dir_slot *)*de; + if (ds->attr != ATTR_EXT) + return PARSE_NOT_LONGNAME; + if ((ds->id & ~0x40) != slot) + goto parse_long; + if (ds->alias_checksum != alias_checksum) + goto parse_long; + } + if ((*de)->name[0] == DELETED_FLAG) + return PARSE_INVALID; + if ((*de)->attr == ATTR_EXT) + goto parse_long; + if (IS_FREE((*de)->name) || ((*de)->attr & ATTR_VOLUME)) + return PARSE_INVALID; + if (fat_checksum((*de)->name) != alias_checksum) + *nr_slots = 0; + + return 0; +} + +/** + * fat_parse_short - Parse MS-DOS (short) directory entry. + * @sb: superblock + * @de: directory entry to parse + * @name: FAT_MAX_SHORT_SIZE array in which to place extracted name + * @dot_hidden: Nonzero == prepend '.' to names with ATTR_HIDDEN + * + * Returns the number of characters extracted into 'name'. + */ +static int fat_parse_short(struct super_block *sb, + const struct msdos_dir_entry *de, + unsigned char *name, int dot_hidden) +{ + const struct msdos_sb_info *sbi = MSDOS_SB(sb); + int isvfat = sbi->options.isvfat; + int nocase = sbi->options.nocase; + unsigned short opt_shortname = sbi->options.shortname; + struct nls_table *nls_disk = sbi->nls_disk; + wchar_t uni_name[14]; + unsigned char c, work[MSDOS_NAME]; + unsigned char *ptname = name; + int chi, chl, i, j, k; + int dotoffset = 0; + int name_len = 0, uni_len = 0; + + if (!isvfat && dot_hidden && (de->attr & ATTR_HIDDEN)) { + *ptname++ = '.'; + dotoffset = 1; + } + + memcpy(work, de->name, sizeof(work)); + /* For an explanation of the special treatment of 0x05 in + * filenames, see msdos_format_name in namei_msdos.c + */ + if (work[0] == 0x05) + work[0] = 0xE5; + + /* Filename */ + for (i = 0, j = 0; i < 8;) { + c = work[i]; + if (!c) + break; + chl = fat_shortname2uni(nls_disk, &work[i], 8 - i, + &uni_name[j++], opt_shortname, + de->lcase & CASE_LOWER_BASE); + if (chl <= 1) { + if (!isvfat) + ptname[i] = nocase ? c : fat_tolower(c); + i++; + if (c != ' ') { + name_len = i; + uni_len = j; + } + } else { + uni_len = j; + if (isvfat) + i += min(chl, 8-i); + else { + for (chi = 0; chi < chl && i < 8; chi++, i++) + ptname[i] = work[i]; + } + if (chl) + name_len = i; + } + } + + i = name_len; + j = uni_len; + fat_short2uni(nls_disk, ".", 1, &uni_name[j++]); + if (!isvfat) + ptname[i] = '.'; + i++; + + /* Extension */ + for (k = 8; k < MSDOS_NAME;) { + c = work[k]; + if (!c) + break; + chl = fat_shortname2uni(nls_disk, &work[k], MSDOS_NAME - k, + &uni_name[j++], opt_shortname, + de->lcase & CASE_LOWER_EXT); + if (chl <= 1) { + k++; + if (!isvfat) + ptname[i] = nocase ? c : fat_tolower(c); + i++; + if (c != ' ') { + name_len = i; + uni_len = j; + } + } else { + uni_len = j; + if (isvfat) { + int offset = min(chl, MSDOS_NAME-k); + k += offset; + i += offset; + } else { + for (chi = 0; chi < chl && k < MSDOS_NAME; + chi++, i++, k++) { + ptname[i] = work[k]; + } + } + if (chl) + name_len = i; + } + } + + if (name_len > 0) { + name_len += dotoffset; + + if (sbi->options.isvfat) { + uni_name[uni_len] = 0x0000; + name_len = fat_uni_to_x8(sb, uni_name, name, + FAT_MAX_SHORT_SIZE); + } + } + + return name_len; +} + +/* + * Return values: negative -> error/not found, 0 -> found. + */ +int fat_search_long(struct inode *inode, const unsigned char *name, + int name_len, struct fat_slot_info *sinfo) +{ + struct super_block *sb = inode->i_sb; + struct msdos_sb_info *sbi = MSDOS_SB(sb); + struct buffer_head *bh = NULL; + struct msdos_dir_entry *de; + unsigned char nr_slots; + wchar_t *unicode = NULL; + unsigned char bufname[FAT_MAX_SHORT_SIZE]; + loff_t cpos = 0; + int err, len; + + err = -ENOENT; + while (1) { + if (fat_get_entry_eod(inode, &cpos, &bh, &de) == -1) + goto end_of_dir; +parse_record: + nr_slots = 0; + if (de->name[0] == DELETED_FLAG) + continue; + if (de->attr != ATTR_EXT && (de->attr & ATTR_VOLUME)) + continue; + if (de->attr != ATTR_EXT && IS_FREE(de->name)) + continue; + if (de->attr == ATTR_EXT) { + int status = fat_parse_long(inode, &cpos, &bh, &de, + &unicode, &nr_slots); + if (status < 0) { + err = status; + goto end_of_dir; + } else if (status == PARSE_INVALID) + continue; + else if (status == PARSE_NOT_LONGNAME) + goto parse_record; + else if (status == PARSE_EOF) + goto end_of_dir; + } + + /* Never prepend '.' to hidden files here. + * That is done only for msdos mounts (and only when + * 'dotsOK=yes'); if we are executing here, it is in the + * context of a vfat mount. + */ + len = fat_parse_short(sb, de, bufname, 0); + if (len == 0) + continue; + + /* Compare shortname */ + if (fat_name_match(sbi, name, name_len, bufname, len)) + goto found; + + if (nr_slots) { + void *longname = unicode + FAT_MAX_UNI_CHARS; + int size = PATH_MAX - FAT_MAX_UNI_SIZE; + + /* Compare longname */ + len = fat_uni_to_x8(sb, unicode, longname, size); + if (fat_name_match(sbi, name, name_len, longname, len)) + goto found; + } + } + +found: + nr_slots++; /* include the de */ + sinfo->slot_off = cpos - nr_slots * sizeof(*de); + sinfo->nr_slots = nr_slots; + sinfo->de = de; + sinfo->bh = bh; + sinfo->i_pos = fat_make_i_pos(sb, sinfo->bh, sinfo->de); + err = 0; +end_of_dir: + if (unicode) + __putname(unicode); + + return err; +} +EXPORT_SYMBOL_GPL(fat_search_long); + +struct fat_ioctl_filldir_callback { + struct dir_context ctx; + void __user *dirent; + int result; + /* for dir ioctl */ + const char *longname; + int long_len; + const char *shortname; + int short_len; +}; + +static int __fat_readdir(struct inode *inode, struct file *file, + struct dir_context *ctx, int short_only, + struct fat_ioctl_filldir_callback *both) +{ + struct super_block *sb = inode->i_sb; + struct msdos_sb_info *sbi = MSDOS_SB(sb); + struct buffer_head *bh; + struct msdos_dir_entry *de; + unsigned char nr_slots; + wchar_t *unicode = NULL; + unsigned char bufname[FAT_MAX_SHORT_SIZE]; + int isvfat = sbi->options.isvfat; + const char *fill_name = NULL; + int fake_offset = 0; + loff_t cpos; + int short_len = 0, fill_len = 0; + int ret = 0; + + mutex_lock(&sbi->s_lock); + + cpos = ctx->pos; + /* Fake . and .. for the root directory. */ + if (inode->i_ino == MSDOS_ROOT_INO) { + if (!dir_emit_dots(file, ctx)) + goto out; + if (ctx->pos == 2) { + fake_offset = 1; + cpos = 0; + } + } + if (cpos & (sizeof(struct msdos_dir_entry) - 1)) { + ret = -ENOENT; + goto out; + } + + bh = NULL; +get_new: + if (fat_get_entry_eod(inode, &cpos, &bh, &de) == -1) + goto end_of_dir; +parse_record: + nr_slots = 0; + /* + * Check for long filename entry, but if short_only, we don't + * need to parse long filename. + */ + if (isvfat && !short_only) { + if (de->name[0] == DELETED_FLAG) + goto record_end; + if (de->attr != ATTR_EXT && (de->attr & ATTR_VOLUME)) + goto record_end; + if (de->attr != ATTR_EXT && IS_FREE(de->name)) + goto record_end; + } else { + if ((de->attr & ATTR_VOLUME) || IS_FREE(de->name)) + goto record_end; + } + + if (isvfat && de->attr == ATTR_EXT) { + int status = fat_parse_long(inode, &cpos, &bh, &de, + &unicode, &nr_slots); + if (status < 0) { + bh = NULL; + ret = status; + goto end_of_dir; + } else if (status == PARSE_INVALID) + goto record_end; + else if (status == PARSE_NOT_LONGNAME) + goto parse_record; + else if (status == PARSE_EOF) + goto end_of_dir; + + if (nr_slots) { + void *longname = unicode + FAT_MAX_UNI_CHARS; + int size = PATH_MAX - FAT_MAX_UNI_SIZE; + int len = fat_uni_to_x8(sb, unicode, longname, size); + + fill_name = longname; + fill_len = len; + /* !both && !short_only, so we don't need shortname. */ + if (!both) + goto start_filldir; + + short_len = fat_parse_short(sb, de, bufname, + sbi->options.dotsOK); + if (short_len == 0) + goto record_end; + /* hack for fat_ioctl_filldir() */ + both->longname = fill_name; + both->long_len = fill_len; + both->shortname = bufname; + both->short_len = short_len; + fill_name = NULL; + fill_len = 0; + goto start_filldir; + } + } + + short_len = fat_parse_short(sb, de, bufname, sbi->options.dotsOK); + if (short_len == 0) + goto record_end; + + fill_name = bufname; + fill_len = short_len; + +start_filldir: + ctx->pos = cpos - (nr_slots + 1) * sizeof(struct msdos_dir_entry); + if (fake_offset && ctx->pos < 2) + ctx->pos = 2; + + if (!memcmp(de->name, MSDOS_DOT, MSDOS_NAME)) { + if (!dir_emit_dot(file, ctx)) + goto fill_failed; + } else if (!memcmp(de->name, MSDOS_DOTDOT, MSDOS_NAME)) { + if (!dir_emit_dotdot(file, ctx)) + goto fill_failed; + } else { + unsigned long inum; + loff_t i_pos = fat_make_i_pos(sb, bh, de); + struct inode *tmp = fat_iget(sb, i_pos); + if (tmp) { + inum = tmp->i_ino; + iput(tmp); + } else + inum = iunique(sb, MSDOS_ROOT_INO); + if (!dir_emit(ctx, fill_name, fill_len, inum, + (de->attr & ATTR_DIR) ? DT_DIR : DT_REG)) + goto fill_failed; + } + +record_end: + fake_offset = 0; + ctx->pos = cpos; + goto get_new; + +end_of_dir: + if (fake_offset && cpos < 2) + ctx->pos = 2; + else + ctx->pos = cpos; +fill_failed: + brelse(bh); + if (unicode) + __putname(unicode); +out: + mutex_unlock(&sbi->s_lock); + + return ret; +} + +static int fat_readdir(struct file *file, struct dir_context *ctx) +{ + return __fat_readdir(file_inode(file), file, ctx, 0, NULL); +} + +#define FAT_IOCTL_FILLDIR_FUNC(func, dirent_type) \ +static bool func(struct dir_context *ctx, const char *name, int name_len, \ + loff_t offset, u64 ino, unsigned int d_type) \ +{ \ + struct fat_ioctl_filldir_callback *buf = \ + container_of(ctx, struct fat_ioctl_filldir_callback, ctx); \ + struct dirent_type __user *d1 = buf->dirent; \ + struct dirent_type __user *d2 = d1 + 1; \ + \ + if (buf->result) \ + return false; \ + buf->result++; \ + \ + if (name != NULL) { \ + /* dirent has only short name */ \ + if (name_len >= sizeof(d1->d_name)) \ + name_len = sizeof(d1->d_name) - 1; \ + \ + if (put_user(0, &d2->d_name[0]) || \ + put_user(0, &d2->d_reclen) || \ + copy_to_user(d1->d_name, name, name_len) || \ + put_user(0, d1->d_name + name_len) || \ + put_user(name_len, &d1->d_reclen)) \ + goto efault; \ + } else { \ + /* dirent has short and long name */ \ + const char *longname = buf->longname; \ + int long_len = buf->long_len; \ + const char *shortname = buf->shortname; \ + int short_len = buf->short_len; \ + \ + if (long_len >= sizeof(d1->d_name)) \ + long_len = sizeof(d1->d_name) - 1; \ + if (short_len >= sizeof(d1->d_name)) \ + short_len = sizeof(d1->d_name) - 1; \ + \ + if (copy_to_user(d2->d_name, longname, long_len) || \ + put_user(0, d2->d_name + long_len) || \ + put_user(long_len, &d2->d_reclen) || \ + put_user(ino, &d2->d_ino) || \ + put_user(offset, &d2->d_off) || \ + copy_to_user(d1->d_name, shortname, short_len) || \ + put_user(0, d1->d_name + short_len) || \ + put_user(short_len, &d1->d_reclen)) \ + goto efault; \ + } \ + return true; \ +efault: \ + buf->result = -EFAULT; \ + return false; \ +} + +FAT_IOCTL_FILLDIR_FUNC(fat_ioctl_filldir, __fat_dirent) + +static int fat_ioctl_readdir(struct inode *inode, struct file *file, + void __user *dirent, filldir_t filldir, + int short_only, int both) +{ + struct fat_ioctl_filldir_callback buf = { + .ctx.actor = filldir, + .dirent = dirent + }; + int ret; + + buf.dirent = dirent; + buf.result = 0; + inode_lock_shared(inode); + buf.ctx.pos = file->f_pos; + ret = -ENOENT; + if (!IS_DEADDIR(inode)) { + ret = __fat_readdir(inode, file, &buf.ctx, + short_only, both ? &buf : NULL); + file->f_pos = buf.ctx.pos; + } + inode_unlock_shared(inode); + if (ret >= 0) + ret = buf.result; + return ret; +} + +static long fat_dir_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct __fat_dirent __user *d1 = (struct __fat_dirent __user *)arg; + int short_only, both; + + switch (cmd) { + case VFAT_IOCTL_READDIR_SHORT: + short_only = 1; + both = 0; + break; + case VFAT_IOCTL_READDIR_BOTH: + short_only = 0; + both = 1; + break; + default: + return fat_generic_ioctl(filp, cmd, arg); + } + + /* + * Yes, we don't need this put_user() absolutely. However old + * code didn't return the right value. So, app use this value, + * in order to check whether it is EOF. + */ + if (put_user(0, &d1->d_reclen)) + return -EFAULT; + + return fat_ioctl_readdir(inode, filp, d1, fat_ioctl_filldir, + short_only, both); +} + +#ifdef CONFIG_COMPAT +#define VFAT_IOCTL_READDIR_BOTH32 _IOR('r', 1, struct compat_dirent[2]) +#define VFAT_IOCTL_READDIR_SHORT32 _IOR('r', 2, struct compat_dirent[2]) + +FAT_IOCTL_FILLDIR_FUNC(fat_compat_ioctl_filldir, compat_dirent) + +static long fat_compat_dir_ioctl(struct file *filp, unsigned cmd, + unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct compat_dirent __user *d1 = compat_ptr(arg); + int short_only, both; + + switch (cmd) { + case VFAT_IOCTL_READDIR_SHORT32: + short_only = 1; + both = 0; + break; + case VFAT_IOCTL_READDIR_BOTH32: + short_only = 0; + both = 1; + break; + default: + return fat_generic_ioctl(filp, cmd, (unsigned long)arg); + } + + /* + * Yes, we don't need this put_user() absolutely. However old + * code didn't return the right value. So, app use this value, + * in order to check whether it is EOF. + */ + if (put_user(0, &d1->d_reclen)) + return -EFAULT; + + return fat_ioctl_readdir(inode, filp, d1, fat_compat_ioctl_filldir, + short_only, both); +} +#endif /* CONFIG_COMPAT */ + +const struct file_operations fat_dir_operations = { + .llseek = generic_file_llseek, + .read = generic_read_dir, + .iterate_shared = fat_readdir, + .unlocked_ioctl = fat_dir_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = fat_compat_dir_ioctl, +#endif + .fsync = fat_file_fsync, + .setlease = generic_setlease, +}; + +static int fat_get_short_entry(struct inode *dir, loff_t *pos, + struct buffer_head **bh, + struct msdos_dir_entry **de) +{ + while (fat_get_entry_eod(dir, pos, bh, de) >= 0) { + /* free entry or long name entry or volume label */ + if (!IS_FREE((*de)->name) && !((*de)->attr & ATTR_VOLUME)) + return 0; + } + return -ENOENT; +} + +/* + * The ".." entry can not provide the "struct fat_slot_info" information + * for inode, nor a usable i_pos. So, this function provides some information + * only. + * + * Since this function walks through the on-disk inodes within a directory, + * callers are responsible for taking any locks necessary to prevent the + * directory from changing. + */ +int fat_get_dotdot_entry(struct inode *dir, struct buffer_head **bh, + struct msdos_dir_entry **de) +{ + loff_t offset = 0; + + *de = NULL; + while (fat_get_short_entry(dir, &offset, bh, de) >= 0) { + if (!strncmp((*de)->name, MSDOS_DOTDOT, MSDOS_NAME)) + return 0; + } + return -ENOENT; +} +EXPORT_SYMBOL_GPL(fat_get_dotdot_entry); + +/* See if directory is empty */ +int fat_dir_empty(struct inode *dir) +{ + struct buffer_head *bh; + struct msdos_dir_entry *de; + loff_t cpos; + int result = 0; + + bh = NULL; + cpos = 0; + while (fat_get_short_entry(dir, &cpos, &bh, &de) >= 0) { + if (strncmp(de->name, MSDOS_DOT , MSDOS_NAME) && + strncmp(de->name, MSDOS_DOTDOT, MSDOS_NAME)) { + result = -ENOTEMPTY; + break; + } + } + brelse(bh); + return result; +} +EXPORT_SYMBOL_GPL(fat_dir_empty); + +/* + * fat_subdirs counts the number of sub-directories of dir. It can be run + * on directories being created. + */ +int fat_subdirs(struct inode *dir) +{ + struct buffer_head *bh; + struct msdos_dir_entry *de; + loff_t cpos; + int count = 0; + + bh = NULL; + cpos = 0; + while (fat_get_short_entry(dir, &cpos, &bh, &de) >= 0) { + if (de->attr & ATTR_DIR) + count++; + } + brelse(bh); + return count; +} + +/* + * Scans a directory for a given file (name points to its formatted name). + * Returns an error code or zero. + */ +int fat_scan(struct inode *dir, const unsigned char *name, + struct fat_slot_info *sinfo) +{ + struct super_block *sb = dir->i_sb; + + sinfo->slot_off = 0; + sinfo->bh = NULL; + while (fat_get_short_entry(dir, &sinfo->slot_off, &sinfo->bh, + &sinfo->de) >= 0) { + if (!strncmp(sinfo->de->name, name, MSDOS_NAME)) { + sinfo->slot_off -= sizeof(*sinfo->de); + sinfo->nr_slots = 1; + sinfo->i_pos = fat_make_i_pos(sb, sinfo->bh, sinfo->de); + return 0; + } + } + return -ENOENT; +} +EXPORT_SYMBOL_GPL(fat_scan); + +/* + * Scans a directory for a given logstart. + * Returns an error code or zero. + */ +int fat_scan_logstart(struct inode *dir, int i_logstart, + struct fat_slot_info *sinfo) +{ + struct super_block *sb = dir->i_sb; + + sinfo->slot_off = 0; + sinfo->bh = NULL; + while (fat_get_short_entry(dir, &sinfo->slot_off, &sinfo->bh, + &sinfo->de) >= 0) { + if (fat_get_start(MSDOS_SB(sb), sinfo->de) == i_logstart) { + sinfo->slot_off -= sizeof(*sinfo->de); + sinfo->nr_slots = 1; + sinfo->i_pos = fat_make_i_pos(sb, sinfo->bh, sinfo->de); + return 0; + } + } + return -ENOENT; +} + +static int __fat_remove_entries(struct inode *dir, loff_t pos, int nr_slots) +{ + struct super_block *sb = dir->i_sb; + struct buffer_head *bh; + struct msdos_dir_entry *de, *endp; + int err = 0, orig_slots; + + while (nr_slots) { + bh = NULL; + if (fat_get_entry(dir, &pos, &bh, &de) < 0) { + err = -EIO; + break; + } + + orig_slots = nr_slots; + endp = (struct msdos_dir_entry *)(bh->b_data + sb->s_blocksize); + while (nr_slots && de < endp) { + de->name[0] = DELETED_FLAG; + de++; + nr_slots--; + } + mmb_mark_buffer_dirty(bh, &MSDOS_I(dir)->i_metadata_bhs); + if (IS_DIRSYNC(dir)) + err = sync_dirty_buffer(bh); + brelse(bh); + if (err) + break; + + /* pos is *next* de's position, so this does `- sizeof(de)' */ + pos += ((orig_slots - nr_slots) * sizeof(*de)) - sizeof(*de); + } + + return err; +} + +int fat_remove_entries(struct inode *dir, struct fat_slot_info *sinfo) +{ + struct super_block *sb = dir->i_sb; + struct msdos_dir_entry *de; + struct buffer_head *bh; + int err = 0, nr_slots; + + /* + * First stage: Remove the shortname. By this, the directory + * entry is removed. + */ + nr_slots = sinfo->nr_slots; + de = sinfo->de; + sinfo->de = NULL; + bh = sinfo->bh; + sinfo->bh = NULL; + while (nr_slots && de >= (struct msdos_dir_entry *)bh->b_data) { + de->name[0] = DELETED_FLAG; + de--; + nr_slots--; + } + mmb_mark_buffer_dirty(bh, &MSDOS_I(dir)->i_metadata_bhs); + if (IS_DIRSYNC(dir)) + err = sync_dirty_buffer(bh); + brelse(bh); + if (err) + return err; + inode_inc_iversion(dir); + + if (nr_slots) { + /* + * Second stage: remove the remaining longname slots. + * (This directory entry is already removed, and so return + * the success) + */ + err = __fat_remove_entries(dir, sinfo->slot_off, nr_slots); + if (err) { + fat_msg(sb, KERN_WARNING, + "Couldn't remove the long name slots"); + } + } + + fat_truncate_time(dir, NULL, FAT_UPDATE_ATIME | FAT_UPDATE_CMTIME); + err = 0; + mark_inode_dirty(dir); + if (IS_DIRSYNC(dir)) + (void)sync_inode_metadata(dir, 1); + + return 0; +} +EXPORT_SYMBOL_GPL(fat_remove_entries); + +static int fat_zeroed_cluster(struct inode *dir, sector_t blknr, int nr_used, + struct buffer_head **bhs, int nr_bhs) +{ + struct super_block *sb = dir->i_sb; + sector_t last_blknr = blknr + MSDOS_SB(sb)->sec_per_clus; + int err, i, n; + + /* Zeroing the unused blocks on this cluster */ + blknr += nr_used; + n = nr_used; + while (blknr < last_blknr) { + bhs[n] = sb_getblk(sb, blknr); + if (!bhs[n]) { + err = -ENOMEM; + goto error; + } + /* Avoid race with userspace read via bdev */ + lock_buffer(bhs[n]); + memset(bhs[n]->b_data, 0, sb->s_blocksize); + set_buffer_uptodate(bhs[n]); + unlock_buffer(bhs[n]); + mmb_mark_buffer_dirty(bhs[n], &MSDOS_I(dir)->i_metadata_bhs); + + n++; + blknr++; + if (n == nr_bhs) { + if (IS_DIRSYNC(dir)) { + err = fat_sync_bhs(bhs, n); + if (err) + goto error; + } + for (i = 0; i < n; i++) + brelse(bhs[i]); + n = 0; + } + } + if (IS_DIRSYNC(dir)) { + err = fat_sync_bhs(bhs, n); + if (err) + goto error; + } + for (i = 0; i < n; i++) + brelse(bhs[i]); + + return 0; + +error: + for (i = 0; i < n; i++) + bforget(bhs[i]); + return err; +} + +int fat_alloc_new_dir(struct inode *dir, struct timespec64 *ts) +{ + struct super_block *sb = dir->i_sb; + struct msdos_sb_info *sbi = MSDOS_SB(sb); + struct buffer_head *bhs[MAX_BUF_PER_PAGE]; + struct msdos_dir_entry *de; + sector_t blknr; + __le16 date, time; + u8 time_cs; + int err, cluster; + + err = fat_alloc_clusters(dir, &cluster, 1); + if (err) + goto error; + + blknr = fat_clus_to_blknr(sbi, cluster); + bhs[0] = sb_getblk(sb, blknr); + if (!bhs[0]) { + err = -ENOMEM; + goto error_free; + } + + fat_time_unix2fat(sbi, ts, &time, &date, &time_cs); + + de = (struct msdos_dir_entry *)bhs[0]->b_data; + /* Avoid race with userspace read via bdev */ + lock_buffer(bhs[0]); + /* filling the new directory slots ("." and ".." entries) */ + memcpy(de[0].name, MSDOS_DOT, MSDOS_NAME); + memcpy(de[1].name, MSDOS_DOTDOT, MSDOS_NAME); + de->attr = de[1].attr = ATTR_DIR; + de[0].lcase = de[1].lcase = 0; + de[0].time = de[1].time = time; + de[0].date = de[1].date = date; + if (sbi->options.isvfat) { + /* extra timestamps */ + de[0].ctime = de[1].ctime = time; + de[0].ctime_cs = de[1].ctime_cs = time_cs; + de[0].adate = de[0].cdate = de[1].adate = de[1].cdate = date; + } else { + de[0].ctime = de[1].ctime = 0; + de[0].ctime_cs = de[1].ctime_cs = 0; + de[0].adate = de[0].cdate = de[1].adate = de[1].cdate = 0; + } + fat_set_start(&de[0], cluster); + fat_set_start(&de[1], MSDOS_I(dir)->i_logstart); + de[0].size = de[1].size = 0; + memset(de + 2, 0, sb->s_blocksize - 2 * sizeof(*de)); + set_buffer_uptodate(bhs[0]); + unlock_buffer(bhs[0]); + mmb_mark_buffer_dirty(bhs[0], &MSDOS_I(dir)->i_metadata_bhs); + + err = fat_zeroed_cluster(dir, blknr, 1, bhs, MAX_BUF_PER_PAGE); + if (err) + goto error_free; + + return cluster; + +error_free: + fat_free_clusters(dir, cluster); +error: + return err; +} +EXPORT_SYMBOL_GPL(fat_alloc_new_dir); + +static int fat_add_new_entries(struct inode *dir, void *slots, int nr_slots, + int *nr_cluster, struct msdos_dir_entry **de, + struct buffer_head **bh) +{ + struct super_block *sb = dir->i_sb; + struct msdos_sb_info *sbi = MSDOS_SB(sb); + struct buffer_head *bhs[MAX_BUF_PER_PAGE]; + sector_t blknr, start_blknr, last_blknr; + unsigned long size, copy; + int err, i, n, offset, cluster[2]; + + /* + * The minimum cluster size is 512bytes, and maximum entry + * size is 32*slots (672bytes). So, iff the cluster size is + * 512bytes, we may need two clusters. + */ + size = nr_slots * sizeof(struct msdos_dir_entry); + *nr_cluster = (size + (sbi->cluster_size - 1)) >> sbi->cluster_bits; + BUG_ON(*nr_cluster > 2); + + err = fat_alloc_clusters(dir, cluster, *nr_cluster); + if (err) + goto error; + + /* + * First stage: Fill the directory entry. NOTE: This cluster + * is not referenced from any inode yet, so updates order is + * not important. + */ + i = n = copy = 0; + do { + start_blknr = blknr = fat_clus_to_blknr(sbi, cluster[i]); + last_blknr = start_blknr + sbi->sec_per_clus; + while (blknr < last_blknr) { + bhs[n] = sb_getblk(sb, blknr); + if (!bhs[n]) { + err = -ENOMEM; + goto error_nomem; + } + + /* fill the directory entry */ + copy = min(size, sb->s_blocksize); + /* Avoid race with userspace read via bdev */ + lock_buffer(bhs[n]); + memcpy(bhs[n]->b_data, slots, copy); + set_buffer_uptodate(bhs[n]); + unlock_buffer(bhs[n]); + mmb_mark_buffer_dirty(bhs[n], + &MSDOS_I(dir)->i_metadata_bhs); + slots += copy; + size -= copy; + if (!size) + break; + n++; + blknr++; + } + } while (++i < *nr_cluster); + + memset(bhs[n]->b_data + copy, 0, sb->s_blocksize - copy); + offset = copy - sizeof(struct msdos_dir_entry); + get_bh(bhs[n]); + *bh = bhs[n]; + *de = (struct msdos_dir_entry *)((*bh)->b_data + offset); + + /* Second stage: clear the rest of cluster, and write outs */ + err = fat_zeroed_cluster(dir, start_blknr, ++n, bhs, MAX_BUF_PER_PAGE); + if (err) + goto error_free; + + return cluster[0]; + +error_free: + brelse(*bh); + *bh = NULL; + n = 0; +error_nomem: + for (i = 0; i < n; i++) + bforget(bhs[i]); + fat_free_clusters(dir, cluster[0]); +error: + return err; +} + +int fat_add_entries(struct inode *dir, void *slots, int nr_slots, + struct fat_slot_info *sinfo) +{ + struct super_block *sb = dir->i_sb; + struct msdos_sb_info *sbi = MSDOS_SB(sb); + struct buffer_head *bh, *prev, *bhs[3]; /* 32*slots (672bytes) */ + struct msdos_dir_entry *de; + int err, free_slots, i, nr_bhs; + loff_t pos; + bool saw_eod; + + sinfo->nr_slots = nr_slots; + + /* First stage: search free directory entries */ + free_slots = nr_bhs = 0; + bh = prev = NULL; + pos = 0; + err = -ENOSPC; + saw_eod = false; + while (fat_get_entry(dir, &pos, &bh, &de) > -1) { + /* check the maximum size of directory */ + if (pos >= FAT_MAX_DIR_SIZE) + goto error; + + if (IS_FREE(de->name)) { + if (de->name[0] == 0) + saw_eod = true; + if (prev != bh) { + get_bh(bh); + bhs[nr_bhs] = prev = bh; + nr_bhs++; + } + free_slots++; + if (free_slots == nr_slots) + goto found; + } else { + if (saw_eod) { + fat_fs_error_ratelimit(sb, + "allocated dir entry found after end-of-directory marker (i_pos %lld)", + MSDOS_I(dir)->i_pos); + err = -EIO; + goto error; + } + for (i = 0; i < nr_bhs; i++) + brelse(bhs[i]); + prev = NULL; + free_slots = nr_bhs = 0; + } + } + if (dir->i_ino == MSDOS_ROOT_INO) { + if (!is_fat32(sbi)) + goto error; + } else if (MSDOS_I(dir)->i_start == 0) { + fat_msg(sb, KERN_ERR, "Corrupted directory (i_pos %lld)", + MSDOS_I(dir)->i_pos); + err = -EIO; + goto error; + } + +found: + err = 0; + pos -= free_slots * sizeof(*de); + nr_slots -= free_slots; + if (free_slots) { + /* + * Second stage: filling the free entries with new entries. + * NOTE: If this slots has shortname, first, we write + * the long name slots, then write the short name. + */ + int size = free_slots * sizeof(*de); + int offset = pos & (sb->s_blocksize - 1); + int long_bhs = nr_bhs - (nr_slots == 0); + + /* Fill the long name slots. */ + for (i = 0; i < long_bhs; i++) { + int copy = umin(sb->s_blocksize - offset, size); + memcpy(bhs[i]->b_data + offset, slots, copy); + mmb_mark_buffer_dirty(bhs[i], + &MSDOS_I(dir)->i_metadata_bhs); + offset = 0; + slots += copy; + size -= copy; + } + if (long_bhs && IS_DIRSYNC(dir)) + err = fat_sync_bhs(bhs, long_bhs); + if (!err && i < nr_bhs) { + /* Fill the short name slot. */ + int copy = umin(sb->s_blocksize - offset, size); + memcpy(bhs[i]->b_data + offset, slots, copy); + mmb_mark_buffer_dirty(bhs[i], + &MSDOS_I(dir)->i_metadata_bhs); + if (IS_DIRSYNC(dir)) + err = sync_dirty_buffer(bhs[i]); + } + for (i = 0; i < nr_bhs; i++) + brelse(bhs[i]); + if (err) + goto error_remove; + } + + if (nr_slots) { + int cluster, nr_cluster; + + /* + * Third stage: allocate the cluster for new entries. + * And initialize the cluster with new entries, then + * add the cluster to dir. + */ + cluster = fat_add_new_entries(dir, slots, nr_slots, &nr_cluster, + &de, &bh); + if (cluster < 0) { + err = cluster; + goto error_remove; + } + err = fat_chain_add(dir, cluster, nr_cluster); + if (err) { + fat_free_clusters(dir, cluster); + goto error_remove; + } + if (dir->i_size & (sbi->cluster_size - 1)) { + fat_fs_error(sb, "Odd directory size"); + dir->i_size = (dir->i_size + sbi->cluster_size - 1) + & ~((loff_t)sbi->cluster_size - 1); + } + dir->i_size += nr_cluster << sbi->cluster_bits; + MSDOS_I(dir)->mmu_private += nr_cluster << sbi->cluster_bits; + } + sinfo->slot_off = pos; + sinfo->de = de; + sinfo->bh = bh; + sinfo->i_pos = fat_make_i_pos(sb, sinfo->bh, sinfo->de); + + return 0; + +error: + brelse(bh); + for (i = 0; i < nr_bhs; i++) + brelse(bhs[i]); + return err; + +error_remove: + brelse(bh); + if (free_slots) + __fat_remove_entries(dir, pos, free_slots); + return err; +} +EXPORT_SYMBOL_GPL(fat_add_entries); diff --git a/fs/fat/fat.h b/fs/fat/fat.h new file mode 100644 index 000000000..61338413d --- /dev/null +++ b/fs/fat/fat.h @@ -0,0 +1,492 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _FAT_H +#define _FAT_H + +#include +#include +#include +#include +#include +#include +#include + +struct file_kattr; + +/* + * vfat shortname flags + */ +#define VFAT_SFN_DISPLAY_LOWER 0x0001 /* convert to lowercase for display */ +#define VFAT_SFN_DISPLAY_WIN95 0x0002 /* emulate win95 rule for display */ +#define VFAT_SFN_DISPLAY_WINNT 0x0004 /* emulate winnt rule for display */ +#define VFAT_SFN_CREATE_WIN95 0x0100 /* emulate win95 rule for create */ +#define VFAT_SFN_CREATE_WINNT 0x0200 /* emulate winnt rule for create */ + +#define FAT_ERRORS_CONT 1 /* ignore error and continue */ +#define FAT_ERRORS_PANIC 2 /* panic on error */ +#define FAT_ERRORS_RO 3 /* remount r/o on error */ + +#define FAT_NFS_STALE_RW 1 /* NFS RW support, can cause ESTALE */ +#define FAT_NFS_NOSTALE_RO 2 /* NFS RO support, no ESTALE issue */ + +struct fat_mount_options { + kuid_t fs_uid; + kgid_t fs_gid; + unsigned short fs_fmask; + unsigned short fs_dmask; + unsigned short codepage; /* Codepage for shortname conversions */ + int time_offset; /* Offset of timestamps from UTC (in minutes) */ + char *iocharset; /* Charset used for filename input/display */ + unsigned short shortname; /* flags for shortname display/create rule */ + unsigned char name_check; /* r = relaxed, n = normal, s = strict */ + unsigned char errors; /* On error: continue, panic, remount-ro */ + unsigned char nfs; /* NFS support: nostale_ro, stale_rw */ + unsigned short allow_utime;/* permission for setting the [am]time */ + unsigned quiet:1, /* set = fake successful chmods and chowns */ + showexec:1, /* set = only set x bit for com/exe/bat */ + sys_immutable:1, /* set = system files are immutable */ + dotsOK:1, /* set = hidden and system files are named '.filename' */ + isvfat:1, /* 0=no vfat long filename support, 1=vfat support */ + utf8:1, /* Use of UTF-8 character set (Default) */ + unicode_xlate:1, /* create escape sequences for unhandled Unicode */ + numtail:1, /* Does first alias have a numeric '~1' type tail? */ + flush:1, /* write things quickly */ + nocase:1, /* Does this need case conversion? 0=need case conversion*/ + usefree:1, /* Use free_clusters for FAT32 */ + tz_set:1, /* Filesystem timestamps' offset set */ + rodir:1, /* allow ATTR_RO for directory */ + discard:1, /* Issue discard requests on deletions */ + dos1xfloppy:1, /* Assume default BPB for DOS 1.x floppies */ + debug:1; /* Not currently used */ +}; + +#define FAT_HASH_BITS 8 +#define FAT_HASH_SIZE (1UL << FAT_HASH_BITS) + +/* + * MS-DOS file system in-core superblock data + */ +struct msdos_sb_info { + unsigned short sec_per_clus; /* sectors/cluster */ + unsigned short cluster_bits; /* log2(cluster_size) */ + unsigned int cluster_size; /* cluster size */ + unsigned char fats, fat_bits; /* number of FATs, FAT bits (12,16 or 32) */ + unsigned short fat_start; + unsigned long fat_length; /* FAT start & length (sec.) */ + unsigned long dir_start; + unsigned short dir_entries; /* root dir start & entries */ + unsigned long data_start; /* first data sector */ + unsigned long max_cluster; /* maximum cluster number */ + unsigned long root_cluster; /* first cluster of the root directory */ + unsigned long fsinfo_sector; /* sector number of FAT32 fsinfo */ + struct mutex fat_lock; + struct mutex nfs_build_inode_lock; + struct mutex s_lock; + unsigned int prev_free; /* previously allocated cluster number */ + unsigned int free_clusters; /* -1 if undefined */ + unsigned int free_clus_valid; /* is free_clusters valid? */ + struct fat_mount_options options; + struct nls_table *nls_disk; /* Codepage used on disk */ + struct nls_table *nls_io; /* Charset used for input and display */ + const void *dir_ops; /* Opaque; default directory operations */ + int dir_per_block; /* dir entries per block */ + int dir_per_block_bits; /* log2(dir_per_block) */ + unsigned int vol_id; /*volume ID*/ + + int fatent_shift; + const struct fatent_operations *fatent_ops; + struct inode *fat_inode; + struct inode *fsinfo_inode; + + struct ratelimit_state ratelimit; + + spinlock_t inode_hash_lock; + struct hlist_head inode_hashtable[FAT_HASH_SIZE]; + + spinlock_t dir_hash_lock; + struct hlist_head dir_hashtable[FAT_HASH_SIZE]; + + unsigned int dirty; /* fs state before mount */ + struct rcu_head rcu; +}; + +#define FAT_CACHE_VALID 0 /* special case for valid cache */ + +/* + * MS-DOS file system inode data in memory + */ +struct msdos_inode_info { + spinlock_t cache_lru_lock; + struct list_head cache_lru; + int nr_caches; + /* for avoiding the race between fat_free() and fat_get_cluster() */ + unsigned int cache_valid_id; + + /* NOTE: mmu_private is 64bits, so must hold ->i_mutex to access */ + loff_t mmu_private; /* physically allocated size */ + + int i_start; /* first cluster or 0 */ + int i_logstart; /* logical first cluster */ + int i_attrs; /* unused attribute bits */ + loff_t i_pos; /* on-disk position of directory entry or 0 */ + struct hlist_node i_fat_hash; /* hash by i_location */ + struct hlist_node i_dir_hash; /* hash by i_logstart */ + struct rw_semaphore truncate_lock; /* protect bmap against truncate */ + struct timespec64 i_crtime; /* File creation (birth) time */ + struct mapping_metadata_bhs i_metadata_bhs; + struct inode vfs_inode; +}; + +struct fat_slot_info { + loff_t i_pos; /* on-disk position of directory entry */ + loff_t slot_off; /* offset for slot or de start */ + int nr_slots; /* number of slots + 1(de) in filename */ + struct msdos_dir_entry *de; + struct buffer_head *bh; +}; + +static inline struct msdos_sb_info *MSDOS_SB(struct super_block *sb) +{ + return sb->s_fs_info; +} + +/* + * Functions that determine the variant of the FAT file system (i.e., + * whether this is FAT12, FAT16 or FAT32. + */ +static inline bool is_fat12(const struct msdos_sb_info *sbi) +{ + return sbi->fat_bits == 12; +} + +static inline bool is_fat16(const struct msdos_sb_info *sbi) +{ + return sbi->fat_bits == 16; +} + +static inline bool is_fat32(const struct msdos_sb_info *sbi) +{ + return sbi->fat_bits == 32; +} + +/* Maximum number of clusters */ +static inline u32 max_fat(struct super_block *sb) +{ + struct msdos_sb_info *sbi = MSDOS_SB(sb); + + return is_fat32(sbi) ? MAX_FAT32 : + is_fat16(sbi) ? MAX_FAT16 : MAX_FAT12; +} + +static inline struct msdos_inode_info *MSDOS_I(struct inode *inode) +{ + return container_of(inode, struct msdos_inode_info, vfs_inode); +} + +/* + * If ->i_mode can't hold S_IWUGO (i.e. ATTR_RO), we use ->i_attrs to + * save ATTR_RO instead of ->i_mode. + * + * If it's directory and !sbi->options.rodir, ATTR_RO isn't read-only + * bit, it's just used as flag for app. + */ +static inline int fat_mode_can_hold_ro(struct inode *inode) +{ + struct msdos_sb_info *sbi = MSDOS_SB(inode->i_sb); + umode_t mask; + + if (S_ISDIR(inode->i_mode)) { + if (!sbi->options.rodir) + return 0; + mask = ~sbi->options.fs_dmask; + } else + mask = ~sbi->options.fs_fmask; + + if (!(mask & S_IWUGO)) + return 0; + return 1; +} + +/* Convert attribute bits and a mask to the UNIX mode. */ +static inline umode_t fat_make_mode(struct msdos_sb_info *sbi, + u8 attrs, umode_t mode) +{ + if (attrs & ATTR_RO && !((attrs & ATTR_DIR) && !sbi->options.rodir)) + mode &= ~S_IWUGO; + + if (attrs & ATTR_DIR) + return (mode & ~sbi->options.fs_dmask) | S_IFDIR; + else + return (mode & ~sbi->options.fs_fmask) | S_IFREG; +} + +/* Return the FAT attribute byte for this inode */ +static inline u8 fat_make_attrs(struct inode *inode) +{ + u8 attrs = MSDOS_I(inode)->i_attrs; + if (S_ISDIR(inode->i_mode)) + attrs |= ATTR_DIR; + if (fat_mode_can_hold_ro(inode) && !(inode->i_mode & S_IWUGO)) + attrs |= ATTR_RO; + return attrs; +} + +static inline void fat_save_attrs(struct inode *inode, u8 attrs) +{ + if (fat_mode_can_hold_ro(inode)) + MSDOS_I(inode)->i_attrs = attrs & ATTR_UNUSED; + else + MSDOS_I(inode)->i_attrs = attrs & (ATTR_UNUSED | ATTR_RO); +} + +static inline unsigned char fat_checksum(const __u8 *name) +{ + unsigned char s = name[0]; + s = (s<<7) + (s>>1) + name[1]; s = (s<<7) + (s>>1) + name[2]; + s = (s<<7) + (s>>1) + name[3]; s = (s<<7) + (s>>1) + name[4]; + s = (s<<7) + (s>>1) + name[5]; s = (s<<7) + (s>>1) + name[6]; + s = (s<<7) + (s>>1) + name[7]; s = (s<<7) + (s>>1) + name[8]; + s = (s<<7) + (s>>1) + name[9]; s = (s<<7) + (s>>1) + name[10]; + return s; +} + +static inline sector_t fat_clus_to_blknr(const struct msdos_sb_info *sbi, int clus) +{ + return ((sector_t)clus - FAT_START_ENT) * sbi->sec_per_clus + + sbi->data_start; +} + +static inline void fat_get_blknr_offset(const struct msdos_sb_info *sbi, + loff_t i_pos, sector_t *blknr, int *offset) +{ + *blknr = i_pos >> sbi->dir_per_block_bits; + *offset = i_pos & (sbi->dir_per_block - 1); +} + +static inline loff_t fat_i_pos_read(struct msdos_sb_info *sbi, + struct inode *inode) +{ + loff_t i_pos; +#if BITS_PER_LONG == 32 + spin_lock(&sbi->inode_hash_lock); +#endif + i_pos = MSDOS_I(inode)->i_pos; +#if BITS_PER_LONG == 32 + spin_unlock(&sbi->inode_hash_lock); +#endif + return i_pos; +} + +static inline void fat16_towchar(wchar_t *dst, const __u8 *src, size_t len) +{ +#ifdef __BIG_ENDIAN + while (len--) { + *dst++ = src[0] | (src[1] << 8); + src += 2; + } +#else + memcpy(dst, src, len * 2); +#endif +} + +static inline int fat_get_start(const struct msdos_sb_info *sbi, + const struct msdos_dir_entry *de) +{ + int cluster = le16_to_cpu(de->start); + if (is_fat32(sbi)) + cluster |= (le16_to_cpu(de->starthi) << 16); + return cluster; +} + +static inline void fat_set_start(struct msdos_dir_entry *de, int cluster) +{ + de->start = cpu_to_le16(cluster); + de->starthi = cpu_to_le16(cluster >> 16); +} + +static inline void fatwchar_to16(__u8 *dst, const wchar_t *src, size_t len) +{ +#ifdef __BIG_ENDIAN + while (len--) { + dst[0] = *src & 0x00FF; + dst[1] = (*src & 0xFF00) >> 8; + dst += 2; + src++; + } +#else + memcpy(dst, src, len * 2); +#endif +} + +/* fat/cache.c */ +extern void fat_cache_inval_inode(struct inode *inode); +extern int fat_get_cluster(struct inode *inode, int cluster, + int *fclus, int *dclus); +extern int fat_get_mapped_cluster(struct inode *inode, sector_t sector, + sector_t last_block, + unsigned long *mapped_blocks, sector_t *bmap); +extern int fat_bmap(struct inode *inode, sector_t sector, sector_t *phys, + unsigned long *mapped_blocks, int create, bool from_bmap); + +/* fat/dir.c */ +extern const struct file_operations fat_dir_operations; +extern int fat_search_long(struct inode *inode, const unsigned char *name, + int name_len, struct fat_slot_info *sinfo); +extern int fat_dir_empty(struct inode *dir); +extern int fat_subdirs(struct inode *dir); +extern int fat_scan(struct inode *dir, const unsigned char *name, + struct fat_slot_info *sinfo); +extern int fat_scan_logstart(struct inode *dir, int i_logstart, + struct fat_slot_info *sinfo); +extern int fat_get_dotdot_entry(struct inode *dir, struct buffer_head **bh, + struct msdos_dir_entry **de); +extern int fat_alloc_new_dir(struct inode *dir, struct timespec64 *ts); +extern int fat_add_entries(struct inode *dir, void *slots, int nr_slots, + struct fat_slot_info *sinfo); +extern int fat_remove_entries(struct inode *dir, struct fat_slot_info *sinfo); + +/* fat/fatent.c */ +struct fat_entry { + int entry; + union { + u8 *ent12_p[2]; + __le16 *ent16_p; + __le32 *ent32_p; + } u; + int nr_bhs; + struct buffer_head *bhs[2]; + struct inode *fat_inode; +}; + +static inline void fatent_init(struct fat_entry *fatent) +{ + fatent->nr_bhs = 0; + fatent->entry = 0; + fatent->u.ent32_p = NULL; + fatent->bhs[0] = fatent->bhs[1] = NULL; + fatent->fat_inode = NULL; +} + +static inline void fatent_set_entry(struct fat_entry *fatent, int entry) +{ + fatent->entry = entry; + fatent->u.ent32_p = NULL; +} + +static inline void fatent_brelse(struct fat_entry *fatent) +{ + int i; + fatent->u.ent32_p = NULL; + for (i = 0; i < fatent->nr_bhs; i++) + brelse(fatent->bhs[i]); + fatent->nr_bhs = 0; + fatent->bhs[0] = fatent->bhs[1] = NULL; + fatent->fat_inode = NULL; +} + +static inline bool fat_valid_entry(struct msdos_sb_info *sbi, int entry) +{ + return FAT_START_ENT <= entry && entry < sbi->max_cluster; +} + +extern void fat_ent_access_init(struct super_block *sb); +extern int fat_ent_read(struct inode *inode, struct fat_entry *fatent, + int entry); +extern int fat_ent_write(struct inode *inode, struct fat_entry *fatent, + int new, int wait); +extern int fat_alloc_clusters(struct inode *inode, int *cluster, + int nr_cluster); +extern int fat_free_clusters(struct inode *inode, int cluster); +extern int fat_count_free_clusters(struct super_block *sb); +extern int fat_trim_fs(struct inode *inode, struct fstrim_range *range); + +/* fat/file.c */ +extern long fat_generic_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg); +extern const struct file_operations fat_file_operations; +extern const struct inode_operations fat_file_inode_operations; +extern int fat_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr); +extern void fat_truncate_blocks(struct inode *inode, loff_t offset); +extern int fat_getattr(struct mnt_idmap *idmap, + const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int flags); +int fat_fileattr_get(struct dentry *dentry, struct file_kattr *fa); +extern int fat_file_fsync(struct file *file, loff_t start, loff_t end, + int datasync); + +/* fat/inode.c */ +extern int fat_block_truncate_page(struct inode *inode, loff_t from); +extern void fat_attach(struct inode *inode, loff_t i_pos); +extern void fat_detach(struct inode *inode); +extern struct inode *fat_iget(struct super_block *sb, loff_t i_pos); +extern struct inode *fat_build_inode(struct super_block *sb, + struct msdos_dir_entry *de, loff_t i_pos); +extern int fat_fill_super(struct super_block *sb, struct fs_context *fc, + void (*setup)(struct super_block *)); +extern int fat_fill_inode(struct inode *inode, struct msdos_dir_entry *de); + +extern int fat_flush_inodes(struct super_block *sb, struct inode *i1, + struct inode *i2); + +extern const struct fs_parameter_spec fat_param_spec[]; +int fat_init_fs_context(struct fs_context *fc, bool is_vfat); +void fat_free_fc(struct fs_context *fc); + +int fat_parse_param(struct fs_context *fc, struct fs_parameter *param, + bool is_vfat); +int fat_reconfigure(struct fs_context *fc); + +static inline unsigned long fat_dir_hash(int logstart) +{ + return hash_32(logstart, FAT_HASH_BITS); +} +extern int fat_add_cluster(struct inode *inode); + +/* fat/misc.c */ +extern __printf(3, 4) __cold +void __fat_fs_error(struct super_block *sb, int report, const char *fmt, ...); +#define fat_fs_error(sb, fmt, args...) \ + __fat_fs_error(sb, 1, fmt , ## args) +#define fat_fs_error_ratelimit(sb, fmt, args...) \ + __fat_fs_error(sb, __ratelimit(&MSDOS_SB(sb)->ratelimit), fmt , ## args) + +#define FAT_PRINTK_PREFIX "%sFAT-fs (%s): " +#define fat_msg(sb, level, fmt, args...) \ +do { \ + printk_index_subsys_emit(FAT_PRINTK_PREFIX, level, fmt, ##args);\ + _fat_msg(sb, level, fmt, ##args); \ +} while (0) +__printf(3, 4) __cold +void _fat_msg(struct super_block *sb, const char *level, const char *fmt, ...); +#define fat_msg_ratelimit(sb, level, fmt, args...) \ + do { \ + if (__ratelimit(&MSDOS_SB(sb)->ratelimit)) \ + fat_msg(sb, level, fmt, ## args); \ + } while (0) +extern int fat_clusters_flush(struct super_block *sb); +extern int fat_chain_add(struct inode *inode, int new_dclus, int nr_cluster); +extern void fat_time_fat2unix(struct msdos_sb_info *sbi, struct timespec64 *ts, + __le16 __time, __le16 __date, u8 time_cs); +extern void fat_time_unix2fat(struct msdos_sb_info *sbi, struct timespec64 *ts, + __le16 *time, __le16 *date, u8 *time_cs); +extern struct timespec64 fat_truncate_atime(const struct msdos_sb_info *sbi, + const struct timespec64 *ts); +#define FAT_UPDATE_ATIME (1u << 0) +#define FAT_UPDATE_CMTIME (1u << 1) +void fat_truncate_time(struct inode *inode, struct timespec64 *now, + unsigned int flags); +int fat_update_time(struct inode *inode, enum fs_update_time type, + unsigned int flags); +extern int fat_sync_bhs(struct buffer_head **bhs, int nr_bhs); + +int fat_cache_init(void); +void fat_cache_destroy(void); + +/* fat/nfs.c */ +extern const struct export_operations fat_export_ops; +extern const struct export_operations fat_export_ops_nostale; + +/* helper for printk */ +typedef unsigned long long llu; + +#endif /* !_FAT_H */ diff --git a/fs/fat/fat_test.c b/fs/fat/fat_test.c new file mode 100644 index 000000000..9583ce66d --- /dev/null +++ b/fs/fat/fat_test.c @@ -0,0 +1,395 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit tests for FAT filesystems. + * + * Copyright (C) 2020 Google LLC. + * Author: David Gow + */ + +#include + +#include "fat.h" + +static void fat_checksum_test(struct kunit *test) +{ + /* With no extension. */ + KUNIT_EXPECT_EQ(test, fat_checksum("VMLINUX "), (u8)44); + /* With 3-letter extension. */ + KUNIT_EXPECT_EQ(test, fat_checksum("README TXT"), (u8)115); + /* With short (1-letter) extension. */ + KUNIT_EXPECT_EQ(test, fat_checksum("ABCDEFGHA "), (u8)98); +} + +static void fat_clus_to_blknr_test(struct kunit *test) +{ + static const struct msdos_sb_info sbi = { + .sec_per_clus = 4, + .data_start = 100, + }; + + KUNIT_EXPECT_EQ(test, (sector_t)100, + fat_clus_to_blknr(&sbi, FAT_START_ENT)); + KUNIT_EXPECT_EQ(test, (sector_t)112, fat_clus_to_blknr(&sbi, 5)); +} + +static void fat_get_blknr_offset_test(struct kunit *test) +{ + static const struct msdos_sb_info sbi = { + .dir_per_block = 16, + .dir_per_block_bits = 4, + }; + + sector_t blknr; + int offset; + + fat_get_blknr_offset(&sbi, 0, &blknr, &offset); + KUNIT_EXPECT_EQ(test, (sector_t)0, blknr); + KUNIT_EXPECT_EQ(test, 0, offset); + + fat_get_blknr_offset(&sbi, (10 << 4) | 7, &blknr, &offset); + KUNIT_EXPECT_EQ(test, (sector_t)10, blknr); + KUNIT_EXPECT_EQ(test, 7, offset); +} + +struct fat_timestamp_testcase { + const char *name; + struct timespec64 ts; + __le16 time; + __le16 date; + u8 cs; + int time_offset; +}; + +struct fat_unix2fat_clamp_testcase { + const char *name; + struct timespec64 ts; + __le16 time; + __le16 date; + u8 cs; + int time_offset; +}; + +struct fat_truncate_atime_testcase { + const char *name; + struct timespec64 ts; + struct timespec64 expected; + int time_offset; +}; + +static struct fat_timestamp_testcase time_test_cases[] = { + { + .name = "Earliest possible UTC (1980-01-01 00:00:00)", + .ts = {.tv_sec = 315532800LL, .tv_nsec = 0L}, + .time = cpu_to_le16(0), + .date = cpu_to_le16(33), + .cs = 0, + .time_offset = 0, + }, + { + .name = "Latest possible UTC (2107-12-31 23:59:58)", + .ts = {.tv_sec = 4354819198LL, .tv_nsec = 0L}, + .time = cpu_to_le16(49021), + .date = cpu_to_le16(65439), + .cs = 0, + .time_offset = 0, + }, + { + .name = "Earliest possible (UTC-11) (== 1979-12-31 13:00:00 UTC)", + .ts = {.tv_sec = 315493200LL, .tv_nsec = 0L}, + .time = cpu_to_le16(0), + .date = cpu_to_le16(33), + .cs = 0, + .time_offset = 11 * 60, + }, + { + .name = "Latest possible (UTC+11) (== 2108-01-01 10:59:58 UTC)", + .ts = {.tv_sec = 4354858798LL, .tv_nsec = 0L}, + .time = cpu_to_le16(49021), + .date = cpu_to_le16(65439), + .cs = 0, + .time_offset = -11 * 60, + }, + { + .name = "Leap Day / Year (1996-02-29 00:00:00)", + .ts = {.tv_sec = 825552000LL, .tv_nsec = 0L}, + .time = cpu_to_le16(0), + .date = cpu_to_le16(8285), + .cs = 0, + .time_offset = 0, + }, + { + .name = "Year 2000 is leap year (2000-02-29 00:00:00)", + .ts = {.tv_sec = 951782400LL, .tv_nsec = 0L}, + .time = cpu_to_le16(0), + .date = cpu_to_le16(10333), + .cs = 0, + .time_offset = 0, + }, + { + .name = "Year 2100 not leap year (2100-03-01 00:00:00)", + .ts = {.tv_sec = 4107542400LL, .tv_nsec = 0L}, + .time = cpu_to_le16(0), + .date = cpu_to_le16(61537), + .cs = 0, + .time_offset = 0, + }, + { + .name = "Leap year + timezone UTC+1 (== 2004-02-29 00:30:00 UTC)", + .ts = {.tv_sec = 1078014600LL, .tv_nsec = 0L}, + .time = cpu_to_le16(48064), + .date = cpu_to_le16(12380), + .cs = 0, + .time_offset = -60, + }, + { + .name = "Leap year + timezone UTC-1 (== 2004-02-29 23:30:00 UTC)", + .ts = {.tv_sec = 1078097400LL, .tv_nsec = 0L}, + .time = cpu_to_le16(960), + .date = cpu_to_le16(12385), + .cs = 0, + .time_offset = 60, + }, + { + .name = "VFAT odd-second resolution (1999-12-31 23:59:59)", + .ts = {.tv_sec = 946684799LL, .tv_nsec = 0L}, + .time = cpu_to_le16(49021), + .date = cpu_to_le16(10143), + .cs = 100, + .time_offset = 0, + }, + { + .name = "VFAT 10ms resolution (1980-01-01 00:00:00:0010)", + .ts = {.tv_sec = 315532800LL, .tv_nsec = 10000000L}, + .time = cpu_to_le16(0), + .date = cpu_to_le16(33), + .cs = 1, + .time_offset = 0, + }, +}; + +static struct fat_unix2fat_clamp_testcase unix2fat_clamp_test_cases[] = { + { + .name = "Clamp to earliest FAT date for 1979-12-31 23:59:59 UTC", + .ts = {.tv_sec = 315532799LL, .tv_nsec = 0L}, + .time = cpu_to_le16(0), + .date = cpu_to_le16(33), + .cs = 0, + .time_offset = 0, + }, + { + .name = "Clamp after time_offset=-60 pushes 1980-01-01 00:30 UTC below 1980", + .ts = {.tv_sec = 315534600LL, .tv_nsec = 0L}, + .time = cpu_to_le16(0), + .date = cpu_to_le16(33), + .cs = 0, + .time_offset = -60, + }, + { + .name = "Clamp to latest FAT date for 2108-01-01 00:00:00 UTC", + .ts = {.tv_sec = 4354819200LL, .tv_nsec = 0L}, + .time = cpu_to_le16(49021), + .date = cpu_to_le16(65439), + .cs = 199, + .time_offset = 0, + }, + { + .name = "Clamp after time_offset=60 pushes 2107-12-31 23:30 UTC beyond 2107", + .ts = {.tv_sec = 4354817400LL, .tv_nsec = 0L}, + .time = cpu_to_le16(49021), + .date = cpu_to_le16(65439), + .cs = 199, + .time_offset = 60, + }, +}; + +static struct fat_truncate_atime_testcase truncate_atime_test_cases[] = { + { + .name = "UTC atime truncates to 2004-02-29 00:00:00", + .ts = {.tv_sec = 1078058096LL, .tv_nsec = 789000000L}, + .expected = {.tv_sec = 1078012800LL, .tv_nsec = 0L}, + .time_offset = 0, + }, + { + .name = "time_offset=-60 truncates 2004-02-29 00:30 UTC to previous local midnight", + .ts = {.tv_sec = 1078014645LL, .tv_nsec = 123000000L}, + .expected = {.tv_sec = 1077930000LL, .tv_nsec = 0L}, + .time_offset = -60, + }, + { + .name = "time_offset=60 truncates 2004-02-29 23:30 UTC to next local midnight", + .ts = {.tv_sec = 1078097445LL, .tv_nsec = 123000000L}, + .expected = {.tv_sec = 1078095600LL, .tv_nsec = 0L}, + .time_offset = 60, + }, +}; + +static void time_testcase_desc(struct fat_timestamp_testcase *t, + char *desc) +{ + strscpy(desc, t->name, KUNIT_PARAM_DESC_SIZE); +} + +static void unix2fat_clamp_testcase_desc(struct fat_unix2fat_clamp_testcase *t, + char *desc) +{ + strscpy(desc, t->name, KUNIT_PARAM_DESC_SIZE); +} + +static void truncate_atime_testcase_desc(struct fat_truncate_atime_testcase *t, + char *desc) +{ + strscpy(desc, t->name, KUNIT_PARAM_DESC_SIZE); +} + +KUNIT_ARRAY_PARAM(fat_time, time_test_cases, time_testcase_desc); +KUNIT_ARRAY_PARAM(fat_unix2fat_clamp, unix2fat_clamp_test_cases, + unix2fat_clamp_testcase_desc); +KUNIT_ARRAY_PARAM(fat_truncate_atime, truncate_atime_test_cases, + truncate_atime_testcase_desc); + +static void fat_test_set_time_offset(struct msdos_sb_info *sbi, int time_offset) +{ + memset(sbi, 0, sizeof(*sbi)); + sbi->options.tz_set = 1; + sbi->options.time_offset = time_offset; +} + +static void fat_time_fat2unix_test(struct kunit *test) +{ + static struct msdos_sb_info fake_sb; + struct timespec64 ts; + struct fat_timestamp_testcase *testcase = + (struct fat_timestamp_testcase *)test->param_value; + + fat_test_set_time_offset(&fake_sb, testcase->time_offset); + + fat_time_fat2unix(&fake_sb, &ts, + testcase->time, + testcase->date, + testcase->cs); + KUNIT_EXPECT_EQ_MSG(test, + testcase->ts.tv_sec, + ts.tv_sec, + "Timestamp mismatch (seconds)\n"); + KUNIT_EXPECT_EQ_MSG(test, + testcase->ts.tv_nsec, + ts.tv_nsec, + "Timestamp mismatch (nanoseconds)\n"); +} + +static void fat_time_unix2fat_test(struct kunit *test) +{ + static struct msdos_sb_info fake_sb; + __le16 date, time; + u8 cs; + struct fat_timestamp_testcase *testcase = + (struct fat_timestamp_testcase *)test->param_value; + + fat_test_set_time_offset(&fake_sb, testcase->time_offset); + + fat_time_unix2fat(&fake_sb, &testcase->ts, + &time, &date, &cs); + KUNIT_EXPECT_EQ_MSG(test, + testcase->time, + time, + "Time mismatch\n"); + KUNIT_EXPECT_EQ_MSG(test, + testcase->date, + date, + "Date mismatch\n"); + KUNIT_EXPECT_EQ_MSG(test, + testcase->cs, + cs, + "Centisecond mismatch\n"); +} + +static void fat_time_unix2fat_clamp_test(struct kunit *test) +{ + static struct msdos_sb_info fake_sb; + __le16 date, time; + u8 cs; + struct fat_unix2fat_clamp_testcase *testcase = + (struct fat_unix2fat_clamp_testcase *)test->param_value; + + fat_test_set_time_offset(&fake_sb, testcase->time_offset); + + fat_time_unix2fat(&fake_sb, &testcase->ts, &time, &date, &cs); + KUNIT_EXPECT_EQ_MSG(test, + testcase->time, + time, + "Clamped time mismatch\n"); + KUNIT_EXPECT_EQ_MSG(test, + testcase->date, + date, + "Clamped date mismatch\n"); + KUNIT_EXPECT_EQ_MSG(test, + testcase->cs, + cs, + "Clamped centisecond mismatch\n"); +} + +static void fat_time_unix2fat_no_csec_test(struct kunit *test) +{ + static struct msdos_sb_info fake_sb; + struct timespec64 ts = { + .tv_sec = 946684799LL, + .tv_nsec = 0L, + }; + __le16 date, time; + + fat_test_set_time_offset(&fake_sb, 0); + + fat_time_unix2fat(&fake_sb, &ts, &time, &date, NULL); + KUNIT_EXPECT_EQ_MSG(test, + 49021, + le16_to_cpu(time), + "Time mismatch without centiseconds\n"); + KUNIT_EXPECT_EQ_MSG(test, + 10143, + le16_to_cpu(date), + "Date mismatch without centiseconds\n"); +} + +static void fat_truncate_atime_test(struct kunit *test) +{ + static struct msdos_sb_info fake_sb; + struct timespec64 actual; + struct fat_truncate_atime_testcase *testcase = + (struct fat_truncate_atime_testcase *)test->param_value; + + fat_test_set_time_offset(&fake_sb, testcase->time_offset); + + actual = fat_truncate_atime(&fake_sb, &testcase->ts); + KUNIT_EXPECT_EQ_MSG(test, + testcase->expected.tv_sec, + actual.tv_sec, + "Atime truncation seconds mismatch\n"); + KUNIT_EXPECT_EQ_MSG(test, + testcase->expected.tv_nsec, + actual.tv_nsec, + "Atime truncation nanoseconds mismatch\n"); +} + +static struct kunit_case fat_test_cases[] = { + KUNIT_CASE(fat_checksum_test), + KUNIT_CASE(fat_clus_to_blknr_test), + KUNIT_CASE(fat_get_blknr_offset_test), + KUNIT_CASE_PARAM(fat_time_fat2unix_test, fat_time_gen_params), + KUNIT_CASE_PARAM(fat_time_unix2fat_test, fat_time_gen_params), + KUNIT_CASE_PARAM(fat_time_unix2fat_clamp_test, + fat_unix2fat_clamp_gen_params), + KUNIT_CASE(fat_time_unix2fat_no_csec_test), + KUNIT_CASE_PARAM(fat_truncate_atime_test, + fat_truncate_atime_gen_params), + {}, +}; + +static struct kunit_suite fat_test_suite = { + .name = "fat_test", + .test_cases = fat_test_cases, +}; + +kunit_test_suites(&fat_test_suite); + +MODULE_DESCRIPTION("KUnit tests for FAT filesystems"); +MODULE_LICENSE("GPL v2"); diff --git a/fs/fat/fatent.c b/fs/fat/fatent.c new file mode 100644 index 000000000..f0801d99d --- /dev/null +++ b/fs/fat/fatent.c @@ -0,0 +1,853 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2004, OGAWA Hirofumi + */ + +#include +#include +#include +#include "fat.h" + +struct fatent_operations { + void (*ent_blocknr)(struct super_block *, int, int *, sector_t *); + void (*ent_set_ptr)(struct fat_entry *, int); + int (*ent_bread)(struct super_block *, struct fat_entry *, + int, sector_t); + int (*ent_get)(struct fat_entry *); + void (*ent_put)(struct fat_entry *, int); + int (*ent_next)(struct fat_entry *); +}; + +static DEFINE_SPINLOCK(fat12_entry_lock); + +static void fat12_ent_blocknr(struct super_block *sb, int entry, + int *offset, sector_t *blocknr) +{ + struct msdos_sb_info *sbi = MSDOS_SB(sb); + int bytes = entry + (entry >> 1); + WARN_ON(!fat_valid_entry(sbi, entry)); + *offset = bytes & (sb->s_blocksize - 1); + *blocknr = sbi->fat_start + (bytes >> sb->s_blocksize_bits); +} + +static void fat_ent_blocknr(struct super_block *sb, int entry, + int *offset, sector_t *blocknr) +{ + struct msdos_sb_info *sbi = MSDOS_SB(sb); + int bytes = (entry << sbi->fatent_shift); + WARN_ON(!fat_valid_entry(sbi, entry)); + *offset = bytes & (sb->s_blocksize - 1); + *blocknr = sbi->fat_start + (bytes >> sb->s_blocksize_bits); +} + +static void fat12_ent_set_ptr(struct fat_entry *fatent, int offset) +{ + struct buffer_head **bhs = fatent->bhs; + if (fatent->nr_bhs == 1) { + WARN_ON(offset >= (bhs[0]->b_size - 1)); + fatent->u.ent12_p[0] = bhs[0]->b_data + offset; + fatent->u.ent12_p[1] = bhs[0]->b_data + (offset + 1); + } else { + WARN_ON(offset != (bhs[0]->b_size - 1)); + fatent->u.ent12_p[0] = bhs[0]->b_data + offset; + fatent->u.ent12_p[1] = bhs[1]->b_data; + } +} + +static void fat16_ent_set_ptr(struct fat_entry *fatent, int offset) +{ + WARN_ON(offset & (2 - 1)); + fatent->u.ent16_p = (__le16 *)(fatent->bhs[0]->b_data + offset); +} + +static void fat32_ent_set_ptr(struct fat_entry *fatent, int offset) +{ + WARN_ON(offset & (4 - 1)); + fatent->u.ent32_p = (__le32 *)(fatent->bhs[0]->b_data + offset); +} + +static int fat12_ent_bread(struct super_block *sb, struct fat_entry *fatent, + int offset, sector_t blocknr) +{ + struct buffer_head **bhs = fatent->bhs; + + WARN_ON(blocknr < MSDOS_SB(sb)->fat_start); + fatent->fat_inode = MSDOS_SB(sb)->fat_inode; + + bhs[0] = sb_bread(sb, blocknr); + if (!bhs[0]) + goto err; + + if ((offset + 1) < sb->s_blocksize) + fatent->nr_bhs = 1; + else { + /* This entry is block boundary, it needs the next block */ + blocknr++; + bhs[1] = sb_bread(sb, blocknr); + if (!bhs[1]) + goto err_brelse; + fatent->nr_bhs = 2; + } + fat12_ent_set_ptr(fatent, offset); + return 0; + +err_brelse: + brelse(bhs[0]); +err: + fat_msg_ratelimit(sb, KERN_ERR, "FAT read failed (blocknr %llu)", + (llu)blocknr); + return -EIO; +} + +static int fat_ent_bread(struct super_block *sb, struct fat_entry *fatent, + int offset, sector_t blocknr) +{ + const struct fatent_operations *ops = MSDOS_SB(sb)->fatent_ops; + + WARN_ON(blocknr < MSDOS_SB(sb)->fat_start); + fatent->fat_inode = MSDOS_SB(sb)->fat_inode; + fatent->bhs[0] = sb_bread(sb, blocknr); + if (!fatent->bhs[0]) { + fat_msg_ratelimit(sb, KERN_ERR, "FAT read failed (blocknr %llu)", + (llu)blocknr); + return -EIO; + } + fatent->nr_bhs = 1; + ops->ent_set_ptr(fatent, offset); + return 0; +} + +static int fat12_ent_get(struct fat_entry *fatent) +{ + u8 **ent12_p = fatent->u.ent12_p; + int next; + + spin_lock(&fat12_entry_lock); + if (fatent->entry & 1) + next = (*ent12_p[0] >> 4) | (*ent12_p[1] << 4); + else + next = (*ent12_p[1] << 8) | *ent12_p[0]; + spin_unlock(&fat12_entry_lock); + + next &= 0x0fff; + if (next >= BAD_FAT12) + next = FAT_ENT_EOF; + return next; +} + +static int fat16_ent_get(struct fat_entry *fatent) +{ + int next = le16_to_cpu(*fatent->u.ent16_p); + WARN_ON((unsigned long)fatent->u.ent16_p & (2 - 1)); + if (next >= BAD_FAT16) + next = FAT_ENT_EOF; + return next; +} + +static int fat32_ent_get(struct fat_entry *fatent) +{ + int next = le32_to_cpu(*fatent->u.ent32_p) & 0x0fffffff; + WARN_ON((unsigned long)fatent->u.ent32_p & (4 - 1)); + if (next >= BAD_FAT32) + next = FAT_ENT_EOF; + return next; +} + +static void fat12_ent_put(struct fat_entry *fatent, int new) +{ + u8 **ent12_p = fatent->u.ent12_p; + + if (new == FAT_ENT_EOF) + new = EOF_FAT12; + + spin_lock(&fat12_entry_lock); + if (fatent->entry & 1) { + *ent12_p[0] = (new << 4) | (*ent12_p[0] & 0x0f); + *ent12_p[1] = new >> 4; + } else { + *ent12_p[0] = new & 0xff; + *ent12_p[1] = (*ent12_p[1] & 0xf0) | (new >> 8); + } + spin_unlock(&fat12_entry_lock); + + mmb_mark_buffer_dirty(fatent->bhs[0], + &MSDOS_I(fatent->fat_inode)->i_metadata_bhs); + if (fatent->nr_bhs == 2) + mmb_mark_buffer_dirty(fatent->bhs[1], + &MSDOS_I(fatent->fat_inode)->i_metadata_bhs); +} + +static void fat16_ent_put(struct fat_entry *fatent, int new) +{ + if (new == FAT_ENT_EOF) + new = EOF_FAT16; + + *fatent->u.ent16_p = cpu_to_le16(new); + mmb_mark_buffer_dirty(fatent->bhs[0], + &MSDOS_I(fatent->fat_inode)->i_metadata_bhs); +} + +static void fat32_ent_put(struct fat_entry *fatent, int new) +{ + WARN_ON(new & 0xf0000000); + new |= le32_to_cpu(*fatent->u.ent32_p) & ~0x0fffffff; + *fatent->u.ent32_p = cpu_to_le32(new); + mmb_mark_buffer_dirty(fatent->bhs[0], + &MSDOS_I(fatent->fat_inode)->i_metadata_bhs); +} + +static int fat12_ent_next(struct fat_entry *fatent) +{ + u8 **ent12_p = fatent->u.ent12_p; + struct buffer_head **bhs = fatent->bhs; + u8 *nextp = ent12_p[1] + 1 + (fatent->entry & 1); + + fatent->entry++; + if (fatent->nr_bhs == 1) { + WARN_ON(ent12_p[0] > (u8 *)(bhs[0]->b_data + + (bhs[0]->b_size - 2))); + WARN_ON(ent12_p[1] > (u8 *)(bhs[0]->b_data + + (bhs[0]->b_size - 1))); + if (nextp < (u8 *)(bhs[0]->b_data + (bhs[0]->b_size - 1))) { + ent12_p[0] = nextp - 1; + ent12_p[1] = nextp; + return 1; + } + } else { + WARN_ON(ent12_p[0] != (u8 *)(bhs[0]->b_data + + (bhs[0]->b_size - 1))); + WARN_ON(ent12_p[1] != (u8 *)bhs[1]->b_data); + ent12_p[0] = nextp - 1; + ent12_p[1] = nextp; + brelse(bhs[0]); + bhs[0] = bhs[1]; + fatent->nr_bhs = 1; + return 1; + } + ent12_p[0] = NULL; + ent12_p[1] = NULL; + return 0; +} + +static int fat16_ent_next(struct fat_entry *fatent) +{ + const struct buffer_head *bh = fatent->bhs[0]; + fatent->entry++; + if (fatent->u.ent16_p < (__le16 *)(bh->b_data + (bh->b_size - 2))) { + fatent->u.ent16_p++; + return 1; + } + fatent->u.ent16_p = NULL; + return 0; +} + +static int fat32_ent_next(struct fat_entry *fatent) +{ + const struct buffer_head *bh = fatent->bhs[0]; + fatent->entry++; + if (fatent->u.ent32_p < (__le32 *)(bh->b_data + (bh->b_size - 4))) { + fatent->u.ent32_p++; + return 1; + } + fatent->u.ent32_p = NULL; + return 0; +} + +static const struct fatent_operations fat12_ops = { + .ent_blocknr = fat12_ent_blocknr, + .ent_set_ptr = fat12_ent_set_ptr, + .ent_bread = fat12_ent_bread, + .ent_get = fat12_ent_get, + .ent_put = fat12_ent_put, + .ent_next = fat12_ent_next, +}; + +static const struct fatent_operations fat16_ops = { + .ent_blocknr = fat_ent_blocknr, + .ent_set_ptr = fat16_ent_set_ptr, + .ent_bread = fat_ent_bread, + .ent_get = fat16_ent_get, + .ent_put = fat16_ent_put, + .ent_next = fat16_ent_next, +}; + +static const struct fatent_operations fat32_ops = { + .ent_blocknr = fat_ent_blocknr, + .ent_set_ptr = fat32_ent_set_ptr, + .ent_bread = fat_ent_bread, + .ent_get = fat32_ent_get, + .ent_put = fat32_ent_put, + .ent_next = fat32_ent_next, +}; + +static inline void lock_fat(struct msdos_sb_info *sbi) +{ + mutex_lock(&sbi->fat_lock); +} + +static inline void unlock_fat(struct msdos_sb_info *sbi) +{ + mutex_unlock(&sbi->fat_lock); +} + +void fat_ent_access_init(struct super_block *sb) +{ + struct msdos_sb_info *sbi = MSDOS_SB(sb); + + mutex_init(&sbi->fat_lock); + + if (is_fat32(sbi)) { + sbi->fatent_shift = 2; + sbi->fatent_ops = &fat32_ops; + } else if (is_fat16(sbi)) { + sbi->fatent_shift = 1; + sbi->fatent_ops = &fat16_ops; + } else if (is_fat12(sbi)) { + sbi->fatent_shift = -1; + sbi->fatent_ops = &fat12_ops; + } else { + fat_fs_error(sb, "invalid FAT variant, %u bits", sbi->fat_bits); + } +} + +static void mark_fsinfo_dirty(struct super_block *sb) +{ + struct msdos_sb_info *sbi = MSDOS_SB(sb); + + if (sb_rdonly(sb) || !is_fat32(sbi)) + return; + + __mark_inode_dirty(sbi->fsinfo_inode, I_DIRTY_SYNC); +} + +static inline int fat_ent_update_ptr(struct super_block *sb, + struct fat_entry *fatent, + int offset, sector_t blocknr) +{ + struct msdos_sb_info *sbi = MSDOS_SB(sb); + const struct fatent_operations *ops = sbi->fatent_ops; + struct buffer_head **bhs = fatent->bhs; + + /* Is this fatent's blocks including this entry? */ + if (!fatent->nr_bhs || bhs[0]->b_blocknr != blocknr) + return 0; + if (is_fat12(sbi)) { + if ((offset + 1) < sb->s_blocksize) { + /* This entry is on bhs[0]. */ + if (fatent->nr_bhs == 2) { + brelse(bhs[1]); + fatent->nr_bhs = 1; + } + } else { + /* This entry needs the next block. */ + if (fatent->nr_bhs != 2) + return 0; + if (bhs[1]->b_blocknr != (blocknr + 1)) + return 0; + } + } + ops->ent_set_ptr(fatent, offset); + return 1; +} + +int fat_ent_read(struct inode *inode, struct fat_entry *fatent, int entry) +{ + struct super_block *sb = inode->i_sb; + struct msdos_sb_info *sbi = MSDOS_SB(inode->i_sb); + const struct fatent_operations *ops = sbi->fatent_ops; + int err, offset; + sector_t blocknr; + + if (!fat_valid_entry(sbi, entry)) { + fatent_brelse(fatent); + fat_fs_error_ratelimit(sb, "invalid access to FAT (entry 0x%08x)", entry); + return -EIO; + } + + fatent_set_entry(fatent, entry); + ops->ent_blocknr(sb, entry, &offset, &blocknr); + + if (!fat_ent_update_ptr(sb, fatent, offset, blocknr)) { + fatent_brelse(fatent); + err = ops->ent_bread(sb, fatent, offset, blocknr); + if (err) + return err; + } + return ops->ent_get(fatent); +} + +/* FIXME: We can write the blocks as more big chunk. */ +static int fat_mirror_bhs(struct super_block *sb, struct buffer_head **bhs, + int nr_bhs) +{ + struct msdos_sb_info *sbi = MSDOS_SB(sb); + struct buffer_head *c_bh; + int err, n, copy; + + err = 0; + for (copy = 1; copy < sbi->fats; copy++) { + sector_t backup_fat = sbi->fat_length * copy; + + for (n = 0; n < nr_bhs; n++) { + c_bh = sb_getblk(sb, backup_fat + bhs[n]->b_blocknr); + if (!c_bh) { + err = -ENOMEM; + goto error; + } + /* Avoid race with userspace read via bdev */ + lock_buffer(c_bh); + memcpy(c_bh->b_data, bhs[n]->b_data, sb->s_blocksize); + set_buffer_uptodate(c_bh); + unlock_buffer(c_bh); + mmb_mark_buffer_dirty(c_bh, + &MSDOS_I(sbi->fat_inode)->i_metadata_bhs); + if (sb->s_flags & SB_SYNCHRONOUS) + err = sync_dirty_buffer(c_bh); + brelse(c_bh); + if (err) + goto error; + } + } +error: + return err; +} + +int fat_ent_write(struct inode *inode, struct fat_entry *fatent, + int new, int wait) +{ + struct super_block *sb = inode->i_sb; + const struct fatent_operations *ops = MSDOS_SB(sb)->fatent_ops; + int err; + + ops->ent_put(fatent, new); + if (wait) { + err = fat_sync_bhs(fatent->bhs, fatent->nr_bhs); + if (err) + return err; + } + return fat_mirror_bhs(sb, fatent->bhs, fatent->nr_bhs); +} + +static inline int fat_ent_next(struct msdos_sb_info *sbi, + struct fat_entry *fatent) +{ + if (sbi->fatent_ops->ent_next(fatent)) { + if (fatent->entry < sbi->max_cluster) + return 1; + } + return 0; +} + +static inline int fat_ent_read_block(struct super_block *sb, + struct fat_entry *fatent) +{ + const struct fatent_operations *ops = MSDOS_SB(sb)->fatent_ops; + sector_t blocknr; + int offset; + + fatent_brelse(fatent); + ops->ent_blocknr(sb, fatent->entry, &offset, &blocknr); + return ops->ent_bread(sb, fatent, offset, blocknr); +} + +static void fat_collect_bhs(struct buffer_head **bhs, int *nr_bhs, + struct fat_entry *fatent) +{ + int n, i; + + for (n = 0; n < fatent->nr_bhs; n++) { + for (i = 0; i < *nr_bhs; i++) { + if (fatent->bhs[n] == bhs[i]) + break; + } + if (i == *nr_bhs) { + get_bh(fatent->bhs[n]); + bhs[i] = fatent->bhs[n]; + (*nr_bhs)++; + } + } +} + +int fat_alloc_clusters(struct inode *inode, int *cluster, int nr_cluster) +{ + struct super_block *sb = inode->i_sb; + struct msdos_sb_info *sbi = MSDOS_SB(sb); + const struct fatent_operations *ops = sbi->fatent_ops; + struct fat_entry fatent, prev_ent; + struct buffer_head *bhs[MAX_BUF_PER_PAGE]; + int i, count, err, nr_bhs, idx_clus; + + BUG_ON(nr_cluster > (MAX_BUF_PER_PAGE / 2)); /* fixed limit */ + + lock_fat(sbi); + if (sbi->free_clusters != -1 && sbi->free_clus_valid && + sbi->free_clusters < nr_cluster) { + unlock_fat(sbi); + return -ENOSPC; + } + + err = nr_bhs = idx_clus = 0; + count = FAT_START_ENT; + fatent_init(&prev_ent); + fatent_init(&fatent); + fatent_set_entry(&fatent, sbi->prev_free + 1); + while (count < sbi->max_cluster) { + if (fatent.entry >= sbi->max_cluster) + fatent.entry = FAT_START_ENT; + fatent_set_entry(&fatent, fatent.entry); + err = fat_ent_read_block(sb, &fatent); + if (err) + goto out; + + /* Find the free entries in a block */ + do { + if (ops->ent_get(&fatent) == FAT_ENT_FREE) { + int entry = fatent.entry; + + /* make the cluster chain */ + ops->ent_put(&fatent, FAT_ENT_EOF); + if (prev_ent.nr_bhs) + ops->ent_put(&prev_ent, entry); + + fat_collect_bhs(bhs, &nr_bhs, &fatent); + + sbi->prev_free = entry; + if (sbi->free_clusters != -1) + sbi->free_clusters--; + + cluster[idx_clus] = entry; + idx_clus++; + if (idx_clus == nr_cluster) + goto out; + + /* + * fat_collect_bhs() gets ref-count of bhs, + * so we can still use the prev_ent. + */ + prev_ent = fatent; + } + count++; + if (count == sbi->max_cluster) + break; + } while (fat_ent_next(sbi, &fatent)); + } + + /* Couldn't allocate the free entries */ + sbi->free_clusters = 0; + sbi->free_clus_valid = 1; + err = -ENOSPC; + +out: + unlock_fat(sbi); + mark_fsinfo_dirty(sb); + fatent_brelse(&fatent); + if (!err) { + if (inode_needs_sync(inode)) + err = fat_sync_bhs(bhs, nr_bhs); + if (!err) + err = fat_mirror_bhs(sb, bhs, nr_bhs); + } + for (i = 0; i < nr_bhs; i++) + brelse(bhs[i]); + + if (err && idx_clus) + fat_free_clusters(inode, cluster[0]); + + return err; +} + +int fat_free_clusters(struct inode *inode, int cluster) +{ + struct super_block *sb = inode->i_sb; + struct msdos_sb_info *sbi = MSDOS_SB(sb); + const struct fatent_operations *ops = sbi->fatent_ops; + struct fat_entry fatent; + struct buffer_head *bhs[MAX_BUF_PER_PAGE]; + int i, err, nr_bhs; + int first_cl = cluster, dirty_fsinfo = 0; + + nr_bhs = 0; + fatent_init(&fatent); + lock_fat(sbi); + do { + cluster = fat_ent_read(inode, &fatent, cluster); + if (cluster < 0) { + err = cluster; + goto error; + } else if (cluster == FAT_ENT_FREE) { + fat_fs_error(sb, "%s: deleting FAT entry beyond EOF", + __func__); + err = -EIO; + goto error; + } + + if (sbi->options.discard) { + /* + * Issue discard for the sectors we no longer + * care about, batching contiguous clusters + * into one request + */ + if (cluster != fatent.entry + 1) { + int nr_clus = fatent.entry - first_cl + 1; + + sb_issue_discard(sb, + fat_clus_to_blknr(sbi, first_cl), + nr_clus * sbi->sec_per_clus, + GFP_NOFS, 0); + + first_cl = cluster; + } + } + + ops->ent_put(&fatent, FAT_ENT_FREE); + if (sbi->free_clusters != -1) { + sbi->free_clusters++; + dirty_fsinfo = 1; + } + + if (nr_bhs + fatent.nr_bhs > MAX_BUF_PER_PAGE) { + if (sb->s_flags & SB_SYNCHRONOUS) { + err = fat_sync_bhs(bhs, nr_bhs); + if (err) + goto error; + } + err = fat_mirror_bhs(sb, bhs, nr_bhs); + if (err) + goto error; + for (i = 0; i < nr_bhs; i++) + brelse(bhs[i]); + nr_bhs = 0; + } + fat_collect_bhs(bhs, &nr_bhs, &fatent); + } while (cluster != FAT_ENT_EOF); + + if (sb->s_flags & SB_SYNCHRONOUS) { + err = fat_sync_bhs(bhs, nr_bhs); + if (err) + goto error; + } + err = fat_mirror_bhs(sb, bhs, nr_bhs); +error: + fatent_brelse(&fatent); + for (i = 0; i < nr_bhs; i++) + brelse(bhs[i]); + unlock_fat(sbi); + if (dirty_fsinfo) + mark_fsinfo_dirty(sb); + + return err; +} +EXPORT_SYMBOL_GPL(fat_free_clusters); + +struct fatent_ra { + sector_t cur; + sector_t limit; + + unsigned int ra_blocks; + sector_t ra_advance; + sector_t ra_next; + sector_t ra_limit; +}; + +static void fat_ra_init(struct super_block *sb, struct fatent_ra *ra, + struct fat_entry *fatent, int ent_limit) +{ + struct msdos_sb_info *sbi = MSDOS_SB(sb); + const struct fatent_operations *ops = sbi->fatent_ops; + sector_t blocknr, block_end; + int offset; + /* + * This is the sequential read, so ra_pages * 2 (but try to + * align the optimal hardware IO size). + * [BTW, 128kb covers the whole sectors for FAT12 and FAT16] + */ + unsigned long ra_pages = sb->s_bdi->ra_pages; + unsigned int reada_blocks; + + if (fatent->entry >= ent_limit) + return; + + if (ra_pages > sb->s_bdi->io_pages) + ra_pages = rounddown(ra_pages, sb->s_bdi->io_pages); + reada_blocks = ra_pages << (PAGE_SHIFT - sb->s_blocksize_bits + 1); + + /* Initialize the range for sequential read */ + ops->ent_blocknr(sb, fatent->entry, &offset, &blocknr); + ops->ent_blocknr(sb, ent_limit - 1, &offset, &block_end); + ra->cur = 0; + ra->limit = (block_end + 1) - blocknr; + + /* Advancing the window at half size */ + ra->ra_blocks = reada_blocks >> 1; + ra->ra_advance = ra->cur; + ra->ra_next = ra->cur; + ra->ra_limit = ra->cur + min_t(sector_t, reada_blocks, ra->limit); +} + +/* Assuming to be called before reading a new block (increments ->cur). */ +static void fat_ent_reada(struct super_block *sb, struct fatent_ra *ra, + struct fat_entry *fatent) +{ + if (ra->ra_next >= ra->ra_limit) + return; + + if (ra->cur >= ra->ra_advance) { + struct msdos_sb_info *sbi = MSDOS_SB(sb); + const struct fatent_operations *ops = sbi->fatent_ops; + struct blk_plug plug; + sector_t blocknr, diff; + int offset; + + ops->ent_blocknr(sb, fatent->entry, &offset, &blocknr); + + diff = blocknr - ra->cur; + blk_start_plug(&plug); + /* + * FIXME: we would want to directly use the bio with + * pages to reduce the number of segments. + */ + for (; ra->ra_next < ra->ra_limit; ra->ra_next++) + sb_breadahead(sb, ra->ra_next + diff); + blk_finish_plug(&plug); + + /* Advance the readahead window */ + ra->ra_advance += ra->ra_blocks; + ra->ra_limit += min_t(sector_t, + ra->ra_blocks, ra->limit - ra->ra_limit); + } + ra->cur++; +} + +int fat_count_free_clusters(struct super_block *sb) +{ + struct msdos_sb_info *sbi = MSDOS_SB(sb); + const struct fatent_operations *ops = sbi->fatent_ops; + struct fat_entry fatent; + struct fatent_ra fatent_ra; + int err = 0, free; + + lock_fat(sbi); + if (sbi->free_clusters != -1 && sbi->free_clus_valid) + goto out; + + free = 0; + fatent_init(&fatent); + fatent_set_entry(&fatent, FAT_START_ENT); + fat_ra_init(sb, &fatent_ra, &fatent, sbi->max_cluster); + while (fatent.entry < sbi->max_cluster) { + /* readahead of fat blocks */ + fat_ent_reada(sb, &fatent_ra, &fatent); + + err = fat_ent_read_block(sb, &fatent); + if (err) + goto out; + + do { + if (ops->ent_get(&fatent) == FAT_ENT_FREE) + free++; + } while (fat_ent_next(sbi, &fatent)); + cond_resched(); + } + sbi->free_clusters = free; + sbi->free_clus_valid = 1; + mark_fsinfo_dirty(sb); + fatent_brelse(&fatent); +out: + unlock_fat(sbi); + return err; +} + +static int fat_trim_clusters(struct super_block *sb, u32 clus, u32 nr_clus) +{ + struct msdos_sb_info *sbi = MSDOS_SB(sb); + return sb_issue_discard(sb, fat_clus_to_blknr(sbi, clus), + nr_clus * sbi->sec_per_clus, GFP_NOFS, 0); +} + +int fat_trim_fs(struct inode *inode, struct fstrim_range *range) +{ + struct super_block *sb = inode->i_sb; + struct msdos_sb_info *sbi = MSDOS_SB(sb); + const struct fatent_operations *ops = sbi->fatent_ops; + struct fat_entry fatent; + struct fatent_ra fatent_ra; + u64 ent_start, ent_end, minlen, trimmed = 0; + u32 free = 0; + int err = 0; + + /* + * FAT data is organized as clusters, trim at the granulary of cluster. + * + * fstrim_range is in byte, convert values to cluster index. + * Treat sectors before data region as all used, not to trim them. + */ + ent_start = max_t(u64, range->start>>sbi->cluster_bits, FAT_START_ENT); + ent_end = ent_start + (range->len >> sbi->cluster_bits) - 1; + minlen = range->minlen >> sbi->cluster_bits; + + if (ent_start >= sbi->max_cluster || range->len < sbi->cluster_size) + return -EINVAL; + if (ent_end >= sbi->max_cluster) + ent_end = sbi->max_cluster - 1; + + fatent_init(&fatent); + lock_fat(sbi); + fatent_set_entry(&fatent, ent_start); + fat_ra_init(sb, &fatent_ra, &fatent, ent_end + 1); + while (fatent.entry <= ent_end) { + /* readahead of fat blocks */ + fat_ent_reada(sb, &fatent_ra, &fatent); + + err = fat_ent_read_block(sb, &fatent); + if (err) + goto error; + do { + if (ops->ent_get(&fatent) == FAT_ENT_FREE) { + free++; + } else if (free) { + if (free >= minlen) { + u32 clus = fatent.entry - free; + + err = fat_trim_clusters(sb, clus, free); + if (err && err != -EOPNOTSUPP) + goto error; + if (!err) + trimmed += free; + err = 0; + } + free = 0; + } + } while (fat_ent_next(sbi, &fatent) && fatent.entry <= ent_end); + + if (fatal_signal_pending(current)) { + err = -ERESTARTSYS; + goto error; + } + + if (need_resched()) { + fatent_brelse(&fatent); + unlock_fat(sbi); + cond_resched(); + lock_fat(sbi); + } + } + /* handle scenario when tail entries are all free */ + if (free && free >= minlen) { + u32 clus = fatent.entry - free; + + err = fat_trim_clusters(sb, clus, free); + if (err && err != -EOPNOTSUPP) + goto error; + if (!err) + trimmed += free; + err = 0; + } + +error: + fatent_brelse(&fatent); + unlock_fat(sbi); + + range->len = trimmed << sbi->cluster_bits; + + return err; +} diff --git a/fs/fat/file.c b/fs/fat/file.c new file mode 100644 index 000000000..1c835ca5f --- /dev/null +++ b/fs/fat/file.c @@ -0,0 +1,614 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/fat/file.c + * + * Written 1992,1993 by Werner Almesberger + * + * regular file handling primitives for fat-based filesystems + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fat.h" + +static long fat_fallocate(struct file *file, int mode, + loff_t offset, loff_t len); + +static int fat_ioctl_get_attributes(struct inode *inode, u32 __user *user_attr) +{ + u32 attr; + + inode_lock_shared(inode); + attr = fat_make_attrs(inode); + inode_unlock_shared(inode); + + return put_user(attr, user_attr); +} + +static int fat_ioctl_set_attributes(struct file *file, u32 __user *user_attr) +{ + struct inode *inode = file_inode(file); + struct msdos_sb_info *sbi = MSDOS_SB(inode->i_sb); + int is_dir = S_ISDIR(inode->i_mode); + u32 attr, oldattr; + struct iattr ia; + int err; + + err = get_user(attr, user_attr); + if (err) + goto out; + + err = mnt_want_write_file(file); + if (err) + goto out; + inode_lock(inode); + + /* + * ATTR_VOLUME and ATTR_DIR cannot be changed; this also + * prevents the user from turning us into a VFAT + * longname entry. Also, we obviously can't set + * any of the NTFS attributes in the high 24 bits. + */ + attr &= 0xff & ~(ATTR_VOLUME | ATTR_DIR); + /* Merge in ATTR_VOLUME and ATTR_DIR */ + attr |= (MSDOS_I(inode)->i_attrs & ATTR_VOLUME) | + (is_dir ? ATTR_DIR : 0); + oldattr = fat_make_attrs(inode); + + /* Equivalent to a chmod() */ + ia.ia_valid = ATTR_MODE | ATTR_CTIME; + ia.ia_ctime = current_time(inode); + if (is_dir) + ia.ia_mode = fat_make_mode(sbi, attr, S_IRWXUGO); + else { + ia.ia_mode = fat_make_mode(sbi, attr, + S_IRUGO | S_IWUGO | (inode->i_mode & S_IXUGO)); + } + + /* The root directory has no attributes */ + if (inode->i_ino == MSDOS_ROOT_INO && attr != ATTR_DIR) { + err = -EINVAL; + goto out_unlock_inode; + } + + if (sbi->options.sys_immutable && + ((attr | oldattr) & ATTR_SYS) && + !capable(CAP_LINUX_IMMUTABLE)) { + err = -EPERM; + goto out_unlock_inode; + } + + /* + * The security check is questionable... We single + * out the RO attribute for checking by the security + * module, just because it maps to a file mode. + */ + err = security_inode_setattr(file_mnt_idmap(file), + file->f_path.dentry, &ia); + if (err) + goto out_unlock_inode; + + /* This MUST be done before doing anything irreversible... */ + err = fat_setattr(file_mnt_idmap(file), file->f_path.dentry, &ia); + if (err) + goto out_unlock_inode; + + fsnotify_change(file->f_path.dentry, ia.ia_valid); + if (sbi->options.sys_immutable) { + if (attr & ATTR_SYS) + inode->i_flags |= S_IMMUTABLE; + else + inode->i_flags &= ~S_IMMUTABLE; + } + + fat_save_attrs(inode, attr); + mark_inode_dirty(inode); +out_unlock_inode: + inode_unlock(inode); + mnt_drop_write_file(file); +out: + return err; +} + +static int fat_ioctl_get_volume_id(struct inode *inode, u32 __user *user_attr) +{ + struct msdos_sb_info *sbi = MSDOS_SB(inode->i_sb); + return put_user(sbi->vol_id, user_attr); +} + +static int fat_ioctl_fitrim(struct inode *inode, unsigned long arg) +{ + struct super_block *sb = inode->i_sb; + struct fstrim_range __user *user_range; + struct fstrim_range range; + int err; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (!bdev_max_discard_sectors(sb->s_bdev)) + return -EOPNOTSUPP; + + user_range = (struct fstrim_range __user *)arg; + if (copy_from_user(&range, user_range, sizeof(range))) + return -EFAULT; + + range.minlen = max(range.minlen, bdev_discard_granularity(sb->s_bdev)); + + err = fat_trim_fs(inode, &range); + if (err < 0) + return err; + + if (copy_to_user(user_range, &range, sizeof(range))) + return -EFAULT; + + return 0; +} + +long fat_generic_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + u32 __user *user_attr = (u32 __user *)arg; + + switch (cmd) { + case FAT_IOCTL_GET_ATTRIBUTES: + return fat_ioctl_get_attributes(inode, user_attr); + case FAT_IOCTL_SET_ATTRIBUTES: + return fat_ioctl_set_attributes(filp, user_attr); + case FAT_IOCTL_GET_VOLUME_ID: + return fat_ioctl_get_volume_id(inode, user_attr); + case FITRIM: + return fat_ioctl_fitrim(inode, arg); + default: + return -ENOTTY; /* Inappropriate ioctl for device */ + } +} + +static int fat_file_release(struct inode *inode, struct file *filp) +{ + if ((filp->f_mode & FMODE_WRITE) && + MSDOS_SB(inode->i_sb)->options.flush) { + fat_flush_inodes(inode->i_sb, inode, NULL); + set_current_state(TASK_UNINTERRUPTIBLE); + io_schedule_timeout(HZ/10); + } + return 0; +} + +int fat_file_fsync(struct file *filp, loff_t start, loff_t end, int datasync) +{ + struct inode *inode = filp->f_mapping->host; + struct inode *fat_inode = MSDOS_SB(inode->i_sb)->fat_inode; + int err; + + err = simple_fsync_noflush(filp, start, end, datasync); + if (err) + return err; + + err = mmb_sync(&MSDOS_I(fat_inode)->i_metadata_bhs); + if (err) + return err; + + return blkdev_issue_flush(inode->i_sb->s_bdev); +} + + +const struct file_operations fat_file_operations = { + .llseek = generic_file_llseek, + .read_iter = generic_file_read_iter, + .write_iter = generic_file_write_iter, + .mmap_prepare = generic_file_mmap_prepare, + .release = fat_file_release, + .unlocked_ioctl = fat_generic_ioctl, + .compat_ioctl = compat_ptr_ioctl, + .fsync = fat_file_fsync, + .splice_read = filemap_splice_read, + .splice_write = iter_file_splice_write, + .fallocate = fat_fallocate, + .setlease = generic_setlease, +}; + +static int fat_cont_expand(struct inode *inode, loff_t size) +{ + struct address_space *mapping = inode->i_mapping; + loff_t start = inode->i_size, count = size - inode->i_size; + int err; + + err = generic_cont_expand_simple(inode, size); + if (err) + goto out; + + fat_truncate_time(inode, NULL, FAT_UPDATE_CMTIME); + mark_inode_dirty(inode); + if (IS_SYNC(inode)) { + int err2; + + /* + * Opencode syncing since we don't have a file open to use + * standard fsync path. + */ + err = filemap_fdatawrite_range(mapping, start, + start + count - 1); + err2 = mmb_sync(&MSDOS_I(inode)->i_metadata_bhs); + if (!err) + err = err2; + err2 = write_inode_now(inode, 1); + if (!err) + err = err2; + if (!err) { + err = filemap_fdatawait_range(mapping, start, + start + count - 1); + } + } +out: + return err; +} + +/* + * Preallocate space for a file. This implements fat's fallocate file + * operation, which gets called from sys_fallocate system call. User + * space requests len bytes at offset. If FALLOC_FL_KEEP_SIZE is set + * we just allocate clusters without zeroing them out. Otherwise we + * allocate and zero out clusters via an expanding truncate. + */ +static long fat_fallocate(struct file *file, int mode, + loff_t offset, loff_t len) +{ + int nr_cluster; /* Number of clusters to be allocated */ + loff_t mm_bytes; /* Number of bytes to be allocated for file */ + loff_t ondisksize; /* block aligned on-disk size in bytes*/ + struct inode *inode = file->f_mapping->host; + struct super_block *sb = inode->i_sb; + struct msdos_sb_info *sbi = MSDOS_SB(sb); + int err = 0; + + /* No support for hole punch or other fallocate flags. */ + if (mode & ~FALLOC_FL_KEEP_SIZE) + return -EOPNOTSUPP; + + /* No support for dir */ + if (!S_ISREG(inode->i_mode)) + return -EOPNOTSUPP; + + inode_lock(inode); + if (mode & FALLOC_FL_KEEP_SIZE) { + ondisksize = inode->i_blocks << 9; + if ((offset + len) <= ondisksize) + goto error; + + /* First compute the number of clusters to be allocated */ + mm_bytes = offset + len - ondisksize; + nr_cluster = (mm_bytes + (sbi->cluster_size - 1)) >> + sbi->cluster_bits; + + /* Start the allocation.We are not zeroing out the clusters */ + while (nr_cluster-- > 0) { + err = fat_add_cluster(inode); + if (err) + goto error; + } + } else { + if ((offset + len) <= i_size_read(inode)) + goto error; + + /* This is just an expanding truncate */ + err = fat_cont_expand(inode, (offset + len)); + } + +error: + inode_unlock(inode); + return err; +} + +/* Free all clusters after the skip'th cluster. */ +static int fat_free(struct inode *inode, int skip) +{ + struct super_block *sb = inode->i_sb; + int err, wait, free_start, i_start, i_logstart; + + if (MSDOS_I(inode)->i_start == 0) + return 0; + + fat_cache_inval_inode(inode); + + wait = IS_DIRSYNC(inode); + i_start = free_start = MSDOS_I(inode)->i_start; + i_logstart = MSDOS_I(inode)->i_logstart; + + /* First, we write the new file size. */ + if (!skip) { + MSDOS_I(inode)->i_start = 0; + MSDOS_I(inode)->i_logstart = 0; + } + MSDOS_I(inode)->i_attrs |= ATTR_ARCH; + fat_truncate_time(inode, NULL, FAT_UPDATE_CMTIME); + mark_inode_dirty(inode); + if (wait) { + err = sync_inode_metadata(inode, 1); + if (err) { + MSDOS_I(inode)->i_start = i_start; + MSDOS_I(inode)->i_logstart = i_logstart; + return err; + } + } + + /* Write a new EOF, and get the remaining cluster chain for freeing. */ + if (skip) { + struct fat_entry fatent; + int ret, fclus, dclus; + + ret = fat_get_cluster(inode, skip - 1, &fclus, &dclus); + if (ret < 0) + return ret; + else if (ret == FAT_ENT_EOF) + return 0; + + fatent_init(&fatent); + ret = fat_ent_read(inode, &fatent, dclus); + if (ret == FAT_ENT_EOF) { + fatent_brelse(&fatent); + return 0; + } else if (ret == FAT_ENT_FREE) { + fat_fs_error(sb, + "%s: invalid cluster chain (i_pos %lld)", + __func__, MSDOS_I(inode)->i_pos); + ret = -EIO; + } else if (ret > 0) { + err = fat_ent_write(inode, &fatent, FAT_ENT_EOF, wait); + if (err) + ret = err; + } + fatent_brelse(&fatent); + if (ret < 0) + return ret; + + free_start = ret; + } + inode->i_blocks = skip << (MSDOS_SB(sb)->cluster_bits - 9); + + /* Freeing the remained cluster chain */ + return fat_free_clusters(inode, free_start); +} + +void fat_truncate_blocks(struct inode *inode, loff_t offset) +{ + struct msdos_sb_info *sbi = MSDOS_SB(inode->i_sb); + const unsigned int cluster_size = sbi->cluster_size; + int nr_clusters; + + /* + * This protects against truncating a file bigger than it was then + * trying to write into the hole. + */ + if (MSDOS_I(inode)->mmu_private > offset) + MSDOS_I(inode)->mmu_private = offset; + + nr_clusters = (offset + (cluster_size - 1)) >> sbi->cluster_bits; + + fat_free(inode, nr_clusters); + fat_flush_inodes(inode->i_sb, inode, NULL); +} + +int fat_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + struct msdos_sb_info *sbi = MSDOS_SB(dentry->d_sb); + bool case_sensitive; + + /* + * FAT filesystems are case-insensitive by default. VFAT + * becomes case-sensitive when mounted with 'check=strict', + * which installs vfat_dentry_ops. MSDOS has no such option; + * its 'nocase' mount option selects case-sensitive matching. + * + * VFAT long filename entries preserve case. Without VFAT, only + * uppercased 8.3 short names are stored. MSDOS with 'nocase' + * also preserves case. + */ + if (sbi->options.isvfat) + case_sensitive = sbi->options.name_check == 's'; + else + case_sensitive = sbi->options.nocase; + + if (!case_sensitive) { + fa->fsx_xflags |= FS_XFLAG_CASEFOLD; + fa->flags |= FS_CASEFOLD_FL; + if (!sbi->options.isvfat) + fa->fsx_xflags |= FS_XFLAG_CASENONPRESERVING; + } + if (d_inode(dentry)->i_flags & S_IMMUTABLE) { + fa->fsx_xflags |= FS_XFLAG_IMMUTABLE; + fa->flags |= FS_IMMUTABLE_FL; + } + return 0; +} +EXPORT_SYMBOL_GPL(fat_fileattr_get); + +int fat_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags) +{ + struct inode *inode = d_inode(path->dentry); + struct msdos_sb_info *sbi = MSDOS_SB(inode->i_sb); + + generic_fillattr(idmap, request_mask, inode, stat); + stat->blksize = sbi->cluster_size; + + if (sbi->options.nfs == FAT_NFS_NOSTALE_RO) { + /* Use i_pos for ino. This is used as fileid of nfs. */ + stat->ino = fat_i_pos_read(sbi, inode); + } + + if (sbi->options.isvfat && request_mask & STATX_BTIME) { + stat->result_mask |= STATX_BTIME; + stat->btime = MSDOS_I(inode)->i_crtime; + } + + return 0; +} +EXPORT_SYMBOL_GPL(fat_getattr); + +static int fat_sanitize_mode(const struct msdos_sb_info *sbi, + struct inode *inode, umode_t *mode_ptr) +{ + umode_t mask, perm; + + /* + * Note, the basic check is already done by a caller of + * (attr->ia_mode & ~FAT_VALID_MODE) + */ + + if (S_ISREG(inode->i_mode)) + mask = sbi->options.fs_fmask; + else + mask = sbi->options.fs_dmask; + + perm = *mode_ptr & ~(S_IFMT | mask); + + /* + * Of the r and x bits, all (subject to umask) must be present. Of the + * w bits, either all (subject to umask) or none must be present. + * + * If fat_mode_can_hold_ro(inode) is false, can't change w bits. + */ + if ((perm & (S_IRUGO | S_IXUGO)) != (inode->i_mode & (S_IRUGO|S_IXUGO))) + return -EPERM; + if (fat_mode_can_hold_ro(inode)) { + if ((perm & S_IWUGO) && ((perm & S_IWUGO) != (S_IWUGO & ~mask))) + return -EPERM; + } else { + if ((perm & S_IWUGO) != (S_IWUGO & ~mask)) + return -EPERM; + } + + *mode_ptr &= S_IFMT | perm; + + return 0; +} + +static int fat_allow_set_time(struct mnt_idmap *idmap, + struct msdos_sb_info *sbi, struct inode *inode) +{ + umode_t allow_utime = sbi->options.allow_utime; + + if (!vfsuid_eq_kuid(i_uid_into_vfsuid(idmap, inode), + current_fsuid())) { + if (vfsgid_in_group_p(i_gid_into_vfsgid(idmap, inode))) + allow_utime >>= 3; + if (allow_utime & MAY_WRITE) + return 1; + } + + /* use a default check */ + return 0; +} + +#define TIMES_SET_FLAGS (ATTR_MTIME_SET | ATTR_ATIME_SET | ATTR_TIMES_SET) +/* valid file mode bits */ +#define FAT_VALID_MODE (S_IFREG | S_IFDIR | S_IRWXUGO) + +int fat_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +{ + struct msdos_sb_info *sbi = MSDOS_SB(dentry->d_sb); + struct inode *inode = d_inode(dentry); + unsigned int ia_valid; + int error; + + /* Check for setting the inode time. */ + ia_valid = attr->ia_valid; + if (ia_valid & TIMES_SET_FLAGS) { + if (fat_allow_set_time(idmap, sbi, inode)) + attr->ia_valid &= ~TIMES_SET_FLAGS; + } + + error = setattr_prepare(idmap, dentry, attr); + attr->ia_valid = ia_valid; + if (error) { + if (sbi->options.quiet) + error = 0; + goto out; + } + + /* + * Expand the file. Since inode_setattr() updates ->i_size + * before calling the ->truncate(), but FAT needs to fill the + * hole before it. XXX: this is no longer true with new truncate + * sequence. + */ + if (attr->ia_valid & ATTR_SIZE) { + inode_dio_wait(inode); + + if (attr->ia_size > inode->i_size) { + error = fat_cont_expand(inode, attr->ia_size); + if (error || attr->ia_valid == ATTR_SIZE) + goto out; + attr->ia_valid &= ~ATTR_SIZE; + } + } + + if (((attr->ia_valid & ATTR_UID) && + (!uid_eq(from_vfsuid(idmap, i_user_ns(inode), attr->ia_vfsuid), + sbi->options.fs_uid))) || + ((attr->ia_valid & ATTR_GID) && + (!gid_eq(from_vfsgid(idmap, i_user_ns(inode), attr->ia_vfsgid), + sbi->options.fs_gid))) || + ((attr->ia_valid & ATTR_MODE) && + (attr->ia_mode & ~FAT_VALID_MODE))) + error = -EPERM; + + if (error) { + if (sbi->options.quiet) + error = 0; + goto out; + } + + /* + * We don't return -EPERM here. Yes, strange, but this is too + * old behavior. + */ + if (attr->ia_valid & ATTR_MODE) { + if (fat_sanitize_mode(sbi, inode, &attr->ia_mode) < 0) + attr->ia_valid &= ~ATTR_MODE; + } + + if (attr->ia_valid & ATTR_SIZE) { + error = fat_block_truncate_page(inode, attr->ia_size); + if (error) + goto out; + down_write(&MSDOS_I(inode)->truncate_lock); + truncate_setsize(inode, attr->ia_size); + fat_truncate_blocks(inode, attr->ia_size); + up_write(&MSDOS_I(inode)->truncate_lock); + } + + /* + * setattr_copy can't truncate these appropriately, so we'll copy them + * ourselves. See fat_truncate_time for the c/mtime logic on fat. + */ + if (attr->ia_valid & ATTR_ATIME) + fat_truncate_time(inode, &attr->ia_atime, FAT_UPDATE_ATIME); + if (attr->ia_valid & ATTR_MTIME) + fat_truncate_time(inode, &attr->ia_mtime, FAT_UPDATE_CMTIME); + attr->ia_valid &= ~(ATTR_ATIME|ATTR_CTIME|ATTR_MTIME); + + setattr_copy(idmap, inode, attr); + mark_inode_dirty(inode); +out: + return error; +} +EXPORT_SYMBOL_GPL(fat_setattr); + +const struct inode_operations fat_file_inode_operations = { + .setattr = fat_setattr, + .getattr = fat_getattr, + .fileattr_get = fat_fileattr_get, + .update_time = fat_update_time, +}; diff --git a/fs/fat/inode.c b/fs/fat/inode.c new file mode 100644 index 000000000..f775a004c --- /dev/null +++ b/fs/fat/inode.c @@ -0,0 +1,2025 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/fat/inode.c + * + * Written 1992,1993 by Werner Almesberger + * VFAT extensions by Gordon Chaffee, merged with msdos fs by Henrik Storner + * Rewritten for the constant inumbers support by Al Viro + * + * Fixes: + * + * Max Cohan: Fixed invalid FSINFO offset when info_sector is 0 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fat.h" + +#ifndef CONFIG_FAT_DEFAULT_IOCHARSET +/* if user don't select VFAT, this is undefined. */ +#define CONFIG_FAT_DEFAULT_IOCHARSET "" +#endif + +#define KB_IN_SECTORS 2 + +/* DOS dates from 1980/1/1 through 2107/12/31 */ +#define FAT_DATE_MIN (0<<9 | 1<<5 | 1) +#define FAT_DATE_MAX (127<<9 | 12<<5 | 31) +#define FAT_TIME_MAX (23<<11 | 59<<5 | 29) + +/* + * A deserialized copy of the on-disk structure laid out in struct + * fat_boot_sector. + */ +struct fat_bios_param_block { + u16 fat_sector_size; + u8 fat_sec_per_clus; + u16 fat_reserved; + u8 fat_fats; + u16 fat_dir_entries; + u16 fat_sectors; + u16 fat_fat_length; + u32 fat_total_sect; + + u8 fat16_state; + u32 fat16_vol_id; + + u32 fat32_length; + u32 fat32_root_cluster; + u16 fat32_info_sector; + u8 fat32_state; + u32 fat32_vol_id; +}; + +static int fat_default_codepage = CONFIG_FAT_DEFAULT_CODEPAGE; +static char fat_default_iocharset[] = CONFIG_FAT_DEFAULT_IOCHARSET; + +static struct fat_floppy_defaults { + unsigned nr_sectors; + unsigned sec_per_clus; + unsigned dir_entries; + unsigned media; + unsigned fat_length; +} floppy_defaults[] = { +{ + .nr_sectors = 160 * KB_IN_SECTORS, + .sec_per_clus = 1, + .dir_entries = 64, + .media = 0xFE, + .fat_length = 1, +}, +{ + .nr_sectors = 180 * KB_IN_SECTORS, + .sec_per_clus = 1, + .dir_entries = 64, + .media = 0xFC, + .fat_length = 2, +}, +{ + .nr_sectors = 320 * KB_IN_SECTORS, + .sec_per_clus = 2, + .dir_entries = 112, + .media = 0xFF, + .fat_length = 1, +}, +{ + .nr_sectors = 360 * KB_IN_SECTORS, + .sec_per_clus = 2, + .dir_entries = 112, + .media = 0xFD, + .fat_length = 2, +}, +}; + +int fat_add_cluster(struct inode *inode) +{ + int err, cluster; + + err = fat_alloc_clusters(inode, &cluster, 1); + if (err) + return err; + /* FIXME: this cluster should be added after data of this + * cluster is writed */ + err = fat_chain_add(inode, cluster, 1); + if (err) + fat_free_clusters(inode, cluster); + return err; +} + +static inline int __fat_get_block(struct inode *inode, sector_t iblock, + unsigned long *max_blocks, + struct buffer_head *bh_result, int create) +{ + struct super_block *sb = inode->i_sb; + struct msdos_sb_info *sbi = MSDOS_SB(sb); + unsigned long mapped_blocks; + sector_t phys, last_block; + int err, offset; + + err = fat_bmap(inode, iblock, &phys, &mapped_blocks, create, false); + if (err) + return err; + if (phys) { + map_bh(bh_result, sb, phys); + *max_blocks = min(mapped_blocks, *max_blocks); + return 0; + } + if (!create) + return 0; + + if (iblock != MSDOS_I(inode)->mmu_private >> sb->s_blocksize_bits) { + fat_fs_error(sb, "corrupted file size (i_pos %lld, %lld)", + MSDOS_I(inode)->i_pos, MSDOS_I(inode)->mmu_private); + return -EIO; + } + + last_block = inode->i_blocks >> (sb->s_blocksize_bits - 9); + offset = (unsigned long)iblock & (sbi->sec_per_clus - 1); + /* + * allocate a cluster according to the following. + * 1) no more available blocks + * 2) not part of fallocate region + */ + if (!offset && !(iblock < last_block)) { + /* TODO: multiple cluster allocation would be desirable. */ + err = fat_add_cluster(inode); + if (err) + return err; + } + /* available blocks on this cluster */ + mapped_blocks = sbi->sec_per_clus - offset; + + *max_blocks = min(mapped_blocks, *max_blocks); + MSDOS_I(inode)->mmu_private += *max_blocks << sb->s_blocksize_bits; + + err = fat_bmap(inode, iblock, &phys, &mapped_blocks, create, false); + if (err) + return err; + if (!phys) { + fat_fs_error(sb, + "invalid FAT chain (i_pos %lld, last_block %llu)", + MSDOS_I(inode)->i_pos, + (unsigned long long)last_block); + return -EIO; + } + + BUG_ON(*max_blocks != mapped_blocks); + set_buffer_new(bh_result); + map_bh(bh_result, sb, phys); + + return 0; +} + +static int fat_get_block(struct inode *inode, sector_t iblock, + struct buffer_head *bh_result, int create) +{ + struct super_block *sb = inode->i_sb; + unsigned long max_blocks = bh_result->b_size >> inode->i_blkbits; + int err; + + err = __fat_get_block(inode, iblock, &max_blocks, bh_result, create); + if (err) + return err; + bh_result->b_size = max_blocks << sb->s_blocksize_bits; + return 0; +} + +static int fat_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + return mpage_writepages(mapping, wbc, fat_get_block); +} + +static int fat_read_folio(struct file *file, struct folio *folio) +{ + return mpage_read_folio(folio, fat_get_block); +} + +static void fat_readahead(struct readahead_control *rac) +{ + mpage_readahead(rac, fat_get_block); +} + +static void fat_write_failed(struct address_space *mapping, loff_t to) +{ + struct inode *inode = mapping->host; + + if (to > inode->i_size) { + truncate_pagecache(inode, inode->i_size); + fat_truncate_blocks(inode, inode->i_size); + } +} + +static int fat_write_begin(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, + struct folio **foliop, void **fsdata) +{ + int err; + + err = cont_write_begin(iocb, mapping, pos, len, + foliop, fsdata, fat_get_block, + &MSDOS_I(mapping->host)->mmu_private); + if (err < 0) + fat_write_failed(mapping, pos + len); + return err; +} + +static int fat_write_end(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct folio *folio, void *fsdata) +{ + struct inode *inode = mapping->host; + int err; + err = generic_write_end(iocb, mapping, pos, len, copied, folio, fsdata); + if (err < len) + fat_write_failed(mapping, pos + len); + if (!(err < 0) && !(MSDOS_I(inode)->i_attrs & ATTR_ARCH)) { + fat_truncate_time(inode, NULL, FAT_UPDATE_CMTIME); + MSDOS_I(inode)->i_attrs |= ATTR_ARCH; + mark_inode_dirty(inode); + } + return err; +} + +static ssize_t fat_direct_IO(struct kiocb *iocb, struct iov_iter *iter) +{ + struct file *file = iocb->ki_filp; + struct address_space *mapping = file->f_mapping; + struct inode *inode = mapping->host; + size_t count = iov_iter_count(iter); + loff_t offset = iocb->ki_pos; + ssize_t ret; + + if (iov_iter_rw(iter) == WRITE) { + /* + * FIXME: blockdev_direct_IO() doesn't use ->write_begin(), + * so we need to update the ->mmu_private to block boundary. + * + * But we must fill the remaining area or hole by nul for + * updating ->mmu_private. + * + * Return 0, and fallback to normal buffered write. + */ + loff_t size = offset + count; + if (MSDOS_I(inode)->mmu_private < size) + return 0; + } + + /* + * FAT need to use the DIO_LOCKING for avoiding the race + * condition of fat_get_block() and ->truncate(). + */ + ret = blockdev_direct_IO(iocb, inode, iter, fat_get_block); + if (ret < 0 && iov_iter_rw(iter) == WRITE) + fat_write_failed(mapping, offset + count); + + return ret; +} + +static int fat_get_block_bmap(struct inode *inode, sector_t iblock, + struct buffer_head *bh_result, int create) +{ + struct super_block *sb = inode->i_sb; + unsigned long max_blocks = bh_result->b_size >> inode->i_blkbits; + int err; + sector_t bmap; + unsigned long mapped_blocks; + + BUG_ON(create != 0); + + err = fat_bmap(inode, iblock, &bmap, &mapped_blocks, create, true); + if (err) + return err; + + if (bmap) { + map_bh(bh_result, sb, bmap); + max_blocks = min(mapped_blocks, max_blocks); + } + + bh_result->b_size = max_blocks << sb->s_blocksize_bits; + + return 0; +} + +static sector_t _fat_bmap(struct address_space *mapping, sector_t block) +{ + sector_t blocknr; + + /* fat_get_cluster() assumes the requested blocknr isn't truncated. */ + down_read(&MSDOS_I(mapping->host)->truncate_lock); + blocknr = generic_block_bmap(mapping, block, fat_get_block_bmap); + up_read(&MSDOS_I(mapping->host)->truncate_lock); + + return blocknr; +} + +/* + * fat_block_truncate_page() zeroes out a mapping from file offset `from' + * up to the end of the block which corresponds to `from'. + * This is required during truncate to physically zeroout the tail end + * of that block so it doesn't yield old data if the file is later grown. + * Also, avoid causing failure from fsx for cases of "data past EOF" + */ +int fat_block_truncate_page(struct inode *inode, loff_t from) +{ + return block_truncate_page(inode->i_mapping, from, fat_get_block); +} + +static const struct address_space_operations fat_aops = { + .dirty_folio = block_dirty_folio, + .invalidate_folio = block_invalidate_folio, + .read_folio = fat_read_folio, + .readahead = fat_readahead, + .writepages = fat_writepages, + .write_begin = fat_write_begin, + .write_end = fat_write_end, + .direct_IO = fat_direct_IO, + .bmap = _fat_bmap, + .migrate_folio = buffer_migrate_folio, +}; + +/* + * New FAT inode stuff. We do the following: + * a) i_ino is constant and has nothing with on-disk location. + * b) FAT manages its own cache of directory entries. + * c) *This* cache is indexed by on-disk location. + * d) inode has an associated directory entry, all right, but + * it may be unhashed. + * e) currently entries are stored within struct inode. That should + * change. + * f) we deal with races in the following way: + * 1. readdir() and lookup() do FAT-dir-cache lookup. + * 2. rename() unhashes the F-d-c entry and rehashes it in + * a new place. + * 3. unlink() and rmdir() unhash F-d-c entry. + * 4. fat_write_inode() checks whether the thing is unhashed. + * If it is we silently return. If it isn't we do bread(), + * check if the location is still valid and retry if it + * isn't. Otherwise we do changes. + * 5. Spinlock is used to protect hash/unhash/location check/lookup + * 6. fat_evict_inode() unhashes the F-d-c entry. + * 7. lookup() and readdir() do igrab() if they find a F-d-c entry + * and consider negative result as cache miss. + */ + +static void fat_hash_init(struct super_block *sb) +{ + struct msdos_sb_info *sbi = MSDOS_SB(sb); + int i; + + spin_lock_init(&sbi->inode_hash_lock); + for (i = 0; i < FAT_HASH_SIZE; i++) + INIT_HLIST_HEAD(&sbi->inode_hashtable[i]); +} + +static inline unsigned long fat_hash(loff_t i_pos) +{ + return hash_32(i_pos, FAT_HASH_BITS); +} + +static void dir_hash_init(struct super_block *sb) +{ + struct msdos_sb_info *sbi = MSDOS_SB(sb); + int i; + + spin_lock_init(&sbi->dir_hash_lock); + for (i = 0; i < FAT_HASH_SIZE; i++) + INIT_HLIST_HEAD(&sbi->dir_hashtable[i]); +} + +void fat_attach(struct inode *inode, loff_t i_pos) +{ + struct msdos_sb_info *sbi = MSDOS_SB(inode->i_sb); + + if (inode->i_ino != MSDOS_ROOT_INO) { + struct hlist_head *head = sbi->inode_hashtable + + fat_hash(i_pos); + + spin_lock(&sbi->inode_hash_lock); + MSDOS_I(inode)->i_pos = i_pos; + hlist_add_head(&MSDOS_I(inode)->i_fat_hash, head); + spin_unlock(&sbi->inode_hash_lock); + } + + /* If NFS support is enabled, cache the mapping of start cluster + * to directory inode. This is used during reconnection of + * dentries to the filesystem root. + */ + if (S_ISDIR(inode->i_mode) && sbi->options.nfs) { + struct hlist_head *d_head = sbi->dir_hashtable; + d_head += fat_dir_hash(MSDOS_I(inode)->i_logstart); + + spin_lock(&sbi->dir_hash_lock); + hlist_add_head(&MSDOS_I(inode)->i_dir_hash, d_head); + spin_unlock(&sbi->dir_hash_lock); + } +} +EXPORT_SYMBOL_GPL(fat_attach); + +void fat_detach(struct inode *inode) +{ + struct msdos_sb_info *sbi = MSDOS_SB(inode->i_sb); + spin_lock(&sbi->inode_hash_lock); + MSDOS_I(inode)->i_pos = 0; + hlist_del_init(&MSDOS_I(inode)->i_fat_hash); + spin_unlock(&sbi->inode_hash_lock); + + if (S_ISDIR(inode->i_mode) && sbi->options.nfs) { + spin_lock(&sbi->dir_hash_lock); + hlist_del_init(&MSDOS_I(inode)->i_dir_hash); + spin_unlock(&sbi->dir_hash_lock); + } +} +EXPORT_SYMBOL_GPL(fat_detach); + +struct inode *fat_iget(struct super_block *sb, loff_t i_pos) +{ + struct msdos_sb_info *sbi = MSDOS_SB(sb); + struct hlist_head *head = sbi->inode_hashtable + fat_hash(i_pos); + struct msdos_inode_info *i; + struct inode *inode = NULL; + + spin_lock(&sbi->inode_hash_lock); + hlist_for_each_entry(i, head, i_fat_hash) { + BUG_ON(i->vfs_inode.i_sb != sb); + if (i->i_pos != i_pos) + continue; + inode = igrab(&i->vfs_inode); + if (inode) + break; + } + spin_unlock(&sbi->inode_hash_lock); + return inode; +} + +static int is_exec(unsigned char *extension) +{ + unsigned char exe_extensions[] = "EXECOMBAT", *walk; + + for (walk = exe_extensions; *walk; walk += 3) + if (!strncmp(extension, walk, 3)) + return 1; + return 0; +} + +static int fat_calc_dir_size(struct inode *inode) +{ + struct msdos_sb_info *sbi = MSDOS_SB(inode->i_sb); + int ret, fclus, dclus; + + inode->i_size = 0; + if (MSDOS_I(inode)->i_start == 0) + return 0; + + ret = fat_get_cluster(inode, FAT_ENT_EOF, &fclus, &dclus); + if (ret < 0) + return ret; + inode->i_size = (fclus + 1) << sbi->cluster_bits; + + return 0; +} + +static int fat_validate_dir(struct inode *dir) +{ + struct super_block *sb = dir->i_sb; + + if (dir->i_nlink < 2) { + /* Directory should have "."/".." entries at least. */ + fat_fs_error(sb, "corrupted directory (invalid entries)"); + return -EIO; + } + if (MSDOS_I(dir)->i_start == 0 || + MSDOS_I(dir)->i_start == MSDOS_SB(sb)->root_cluster) { + /* Directory should point valid cluster. */ + fat_fs_error(sb, "corrupted directory (invalid i_start)"); + return -EIO; + } + return 0; +} + +/* doesn't deal with root inode */ +int fat_fill_inode(struct inode *inode, struct msdos_dir_entry *de) +{ + struct msdos_sb_info *sbi = MSDOS_SB(inode->i_sb); + struct timespec64 mtime; + int error; + + MSDOS_I(inode)->i_pos = 0; + inode->i_uid = sbi->options.fs_uid; + inode->i_gid = sbi->options.fs_gid; + inode_inc_iversion(inode); + inode->i_generation = get_random_u32(); + + if ((de->attr & ATTR_DIR) && !IS_FREE(de->name)) { + inode->i_generation &= ~1; + inode->i_mode = fat_make_mode(sbi, de->attr, S_IRWXUGO); + inode->i_op = sbi->dir_ops; + inode->i_fop = &fat_dir_operations; + + MSDOS_I(inode)->i_start = fat_get_start(sbi, de); + MSDOS_I(inode)->i_logstart = MSDOS_I(inode)->i_start; + error = fat_calc_dir_size(inode); + if (error < 0) + return error; + MSDOS_I(inode)->mmu_private = inode->i_size; + + set_nlink(inode, fat_subdirs(inode)); + + error = fat_validate_dir(inode); + if (error < 0) + return error; + } else { /* not a directory */ + inode->i_generation |= 1; + inode->i_mode = fat_make_mode(sbi, de->attr, + ((sbi->options.showexec && !is_exec(de->name + 8)) + ? S_IRUGO|S_IWUGO : S_IRWXUGO)); + MSDOS_I(inode)->i_start = fat_get_start(sbi, de); + + MSDOS_I(inode)->i_logstart = MSDOS_I(inode)->i_start; + inode->i_size = le32_to_cpu(de->size); + inode->i_op = &fat_file_inode_operations; + inode->i_fop = &fat_file_operations; + inode->i_mapping->a_ops = &fat_aops; + MSDOS_I(inode)->mmu_private = inode->i_size; + } + if (de->attr & ATTR_SYS) { + if (sbi->options.sys_immutable) + inode->i_flags |= S_IMMUTABLE; + } + fat_save_attrs(inode, de->attr); + + inode->i_blocks = ((inode->i_size + (sbi->cluster_size - 1)) + & ~((loff_t)sbi->cluster_size - 1)) >> 9; + + fat_time_fat2unix(sbi, &mtime, de->time, de->date, 0); + inode_set_mtime_to_ts(inode, mtime); + inode_set_ctime_to_ts(inode, mtime); + if (sbi->options.isvfat) { + struct timespec64 atime; + + fat_time_fat2unix(sbi, &atime, 0, de->adate, 0); + inode_set_atime_to_ts(inode, atime); + fat_time_fat2unix(sbi, &MSDOS_I(inode)->i_crtime, de->ctime, + de->cdate, de->ctime_cs); + } else + inode_set_atime_to_ts(inode, fat_truncate_atime(sbi, &mtime)); + + return 0; +} + +static inline void fat_lock_build_inode(struct msdos_sb_info *sbi) +{ + if (sbi->options.nfs == FAT_NFS_NOSTALE_RO) + mutex_lock(&sbi->nfs_build_inode_lock); +} + +static inline void fat_unlock_build_inode(struct msdos_sb_info *sbi) +{ + if (sbi->options.nfs == FAT_NFS_NOSTALE_RO) + mutex_unlock(&sbi->nfs_build_inode_lock); +} + +struct inode *fat_build_inode(struct super_block *sb, + struct msdos_dir_entry *de, loff_t i_pos) +{ + struct inode *inode; + int err; + + fat_lock_build_inode(MSDOS_SB(sb)); + inode = fat_iget(sb, i_pos); + if (inode) + goto out; + inode = new_inode(sb); + if (!inode) { + inode = ERR_PTR(-ENOMEM); + goto out; + } + inode->i_ino = iunique(sb, MSDOS_ROOT_INO); + inode_set_iversion(inode, 1); + err = fat_fill_inode(inode, de); + if (err) { + iput(inode); + inode = ERR_PTR(err); + goto out; + } + fat_attach(inode, i_pos); + insert_inode_hash(inode); +out: + fat_unlock_build_inode(MSDOS_SB(sb)); + return inode; +} + +EXPORT_SYMBOL_GPL(fat_build_inode); + +static int __fat_write_inode(struct inode *inode); + +static int fat_sync_inode_metadata(struct inode *inode, + struct writeback_control *wbc) +{ + struct msdos_sb_info *sbi = MSDOS_SB(inode->i_sb); + struct buffer_head *bh; + loff_t i_pos; + sector_t blocknr; + int offset; + + /* The root directory has no directory entry of its own. */ + if (inode->i_ino == MSDOS_ROOT_INO) + goto sync_bhs; + i_pos = fat_i_pos_read(sbi, inode); + if (!i_pos) + goto sync_bhs; + + fat_get_blknr_offset(sbi, i_pos, &blocknr, &offset); + bh = sb_find_get_block_nonatomic(inode->i_sb, blocknr); + /* + * Buffer present? We leave buffer_dirty check for sync_dirty_buffer() + * for proper synchronization with ongoing IO. + */ + if (bh && buffer_uptodate(bh)) { + sync_dirty_buffer(bh); + if (buffer_write_io_error(bh)) { + brelse(bh); + return -EIO; + } + } + brelse(bh); +sync_bhs: + return mmb_sync(&MSDOS_I(inode)->i_metadata_bhs); +} + +static void fat_free_eofblocks(struct inode *inode) +{ + /* Release unwritten fallocated blocks on inode eviction. */ + if ((inode->i_blocks << 9) > + round_up(MSDOS_I(inode)->mmu_private, + MSDOS_SB(inode->i_sb)->cluster_size)) { + int err; + + fat_truncate_blocks(inode, MSDOS_I(inode)->mmu_private); + /* Fallocate results in updating the i_start/iogstart + * for the zero byte file. So, make it return to + * original state during evict and commit it to avoid + * any corruption on the next access to the cluster + * chain for the file. + */ + err = sync_inode_metadata(inode, inode_needs_sync(inode)); + if (err) { + fat_msg(inode->i_sb, KERN_WARNING, "Failed to " + "update on disk inode for unused " + "fallocated blocks, inode could be " + "corrupted. Please run fsck"); + } + + } +} + +static void fat_evict_inode(struct inode *inode) +{ + truncate_inode_pages_final(&inode->i_data); + if (!inode->i_nlink) { + inode->i_size = 0; + fat_truncate_blocks(inode, 0); + } else { + mmb_sync(&MSDOS_I(inode)->i_metadata_bhs); + fat_free_eofblocks(inode); + } + + mmb_invalidate(&MSDOS_I(inode)->i_metadata_bhs); + clear_inode(inode); + fat_cache_inval_inode(inode); + fat_detach(inode); +} + +static void fat_set_state(struct super_block *sb, + unsigned int set, unsigned int force) +{ + struct buffer_head *bh; + struct fat_boot_sector *b; + struct msdos_sb_info *sbi = MSDOS_SB(sb); + + /* do not change any thing if mounted read only */ + if (sb_rdonly(sb) && !force) + return; + + /* do not change state if fs was dirty */ + if (sbi->dirty) { + /* warn only on set (mount). */ + if (set) + fat_msg(sb, KERN_WARNING, "Volume was not properly " + "unmounted. Some data may be corrupt. " + "Please run fsck."); + return; + } + + bh = sb_bread(sb, 0); + if (bh == NULL) { + fat_msg(sb, KERN_ERR, "unable to read boot sector " + "to mark fs as dirty"); + return; + } + + b = (struct fat_boot_sector *) bh->b_data; + + if (is_fat32(sbi)) { + if (set) + b->fat32.state |= FAT_STATE_DIRTY; + else + b->fat32.state &= ~FAT_STATE_DIRTY; + } else /* fat 16 and 12 */ { + if (set) + b->fat16.state |= FAT_STATE_DIRTY; + else + b->fat16.state &= ~FAT_STATE_DIRTY; + } + + mark_buffer_dirty(bh); + sync_dirty_buffer(bh); + brelse(bh); +} + +static void fat_reset_iocharset(struct fat_mount_options *opts) +{ + if (opts->iocharset != fat_default_iocharset) { + /* Note: opts->iocharset can be NULL here */ + kfree(opts->iocharset); + opts->iocharset = fat_default_iocharset; + } +} + +static void delayed_free(struct rcu_head *p) +{ + struct msdos_sb_info *sbi = container_of(p, struct msdos_sb_info, rcu); + unload_nls(sbi->nls_disk); + unload_nls(sbi->nls_io); + fat_reset_iocharset(&sbi->options); + kfree(sbi); +} + +static void fat_put_super(struct super_block *sb) +{ + struct msdos_sb_info *sbi = MSDOS_SB(sb); + + fat_set_state(sb, 0, 0); + + iput(sbi->fsinfo_inode); + iput(sbi->fat_inode); + + call_rcu(&sbi->rcu, delayed_free); +} + +static struct kmem_cache *fat_inode_cachep; + +static struct inode *fat_alloc_inode(struct super_block *sb) +{ + struct msdos_inode_info *ei; + ei = alloc_inode_sb(sb, fat_inode_cachep, GFP_NOFS); + if (!ei) + return NULL; + + init_rwsem(&ei->truncate_lock); + /* Zeroing to allow iput() even if partial initialized inode. */ + ei->mmu_private = 0; + ei->i_start = 0; + ei->i_logstart = 0; + ei->i_attrs = 0; + ei->i_pos = 0; + ei->i_crtime.tv_sec = 0; + ei->i_crtime.tv_nsec = 0; + mmb_init(&ei->i_metadata_bhs, &ei->vfs_inode.i_data); + + return &ei->vfs_inode; +} + +static void fat_free_inode(struct inode *inode) +{ + kmem_cache_free(fat_inode_cachep, MSDOS_I(inode)); +} + +static void init_once(void *foo) +{ + struct msdos_inode_info *ei = (struct msdos_inode_info *)foo; + + spin_lock_init(&ei->cache_lru_lock); + ei->nr_caches = 0; + ei->cache_valid_id = FAT_CACHE_VALID + 1; + INIT_LIST_HEAD(&ei->cache_lru); + INIT_HLIST_NODE(&ei->i_fat_hash); + INIT_HLIST_NODE(&ei->i_dir_hash); + inode_init_once(&ei->vfs_inode); +} + +static int __init fat_init_inodecache(void) +{ + fat_inode_cachep = kmem_cache_create("fat_inode_cache", + sizeof(struct msdos_inode_info), + 0, (SLAB_RECLAIM_ACCOUNT| + SLAB_ACCOUNT), + init_once); + if (fat_inode_cachep == NULL) + return -ENOMEM; + return 0; +} + +static void __exit fat_destroy_inodecache(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(fat_inode_cachep); +} + +int fat_reconfigure(struct fs_context *fc) +{ + bool new_rdonly; + struct super_block *sb = fc->root->d_sb; + struct msdos_sb_info *sbi = MSDOS_SB(sb); + fc->sb_flags |= SB_NODIRATIME | (sbi->options.isvfat ? 0 : SB_NOATIME); + + sync_filesystem(sb); + + /* make sure we update state on remount. */ + new_rdonly = fc->sb_flags & SB_RDONLY; + if (new_rdonly != sb_rdonly(sb)) { + if (new_rdonly) + fat_set_state(sb, 0, 0); + else + fat_set_state(sb, 1, 1); + } + return 0; +} +EXPORT_SYMBOL_GPL(fat_reconfigure); + +static int fat_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + struct msdos_sb_info *sbi = MSDOS_SB(sb); + u64 id = huge_encode_dev(sb->s_bdev->bd_dev); + + /* If the count of free cluster is still unknown, counts it here. */ + if (sbi->free_clusters == -1 || !sbi->free_clus_valid) { + int err = fat_count_free_clusters(dentry->d_sb); + if (err) + return err; + } + + buf->f_type = dentry->d_sb->s_magic; + buf->f_bsize = sbi->cluster_size; + buf->f_blocks = sbi->max_cluster - FAT_START_ENT; + buf->f_bfree = sbi->free_clusters; + buf->f_bavail = sbi->free_clusters; + buf->f_fsid = u64_to_fsid(id); + buf->f_namelen = + (sbi->options.isvfat ? FAT_LFN_LEN : 12) * NLS_MAX_CHARSET_SIZE; + + return 0; +} + +static int __fat_write_inode(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + struct msdos_sb_info *sbi = MSDOS_SB(sb); + struct buffer_head *bh; + struct msdos_dir_entry *raw_entry; + struct timespec64 mtime; + loff_t i_pos; + sector_t blocknr; + int offset; + + if (inode->i_ino == MSDOS_ROOT_INO) { + /* No entry to update but the metadata bh list may need syncing. */ + set_inode_metadata_writeback(inode); + return 0; + } + +retry: + i_pos = fat_i_pos_read(sbi, inode); + if (!i_pos) + return 0; + + fat_get_blknr_offset(sbi, i_pos, &blocknr, &offset); + bh = sb_bread(sb, blocknr); + if (!bh) { + fat_msg(sb, KERN_ERR, "unable to read inode block " + "for updating (i_pos %lld)", i_pos); + return -EIO; + } + spin_lock(&sbi->inode_hash_lock); + if (i_pos != MSDOS_I(inode)->i_pos) { + spin_unlock(&sbi->inode_hash_lock); + brelse(bh); + goto retry; + } + + raw_entry = &((struct msdos_dir_entry *) (bh->b_data))[offset]; + if (S_ISDIR(inode->i_mode)) + raw_entry->size = 0; + else + raw_entry->size = cpu_to_le32(inode->i_size); + raw_entry->attr = fat_make_attrs(inode); + fat_set_start(raw_entry, MSDOS_I(inode)->i_logstart); + mtime = inode_get_mtime(inode); + fat_time_unix2fat(sbi, &mtime, &raw_entry->time, + &raw_entry->date, NULL); + if (sbi->options.isvfat) { + struct timespec64 ts = inode_get_atime(inode); + __le16 atime; + + fat_time_unix2fat(sbi, &ts, &atime, &raw_entry->adate, NULL); + fat_time_unix2fat(sbi, &MSDOS_I(inode)->i_crtime, &raw_entry->ctime, + &raw_entry->cdate, &raw_entry->ctime_cs); + } + spin_unlock(&sbi->inode_hash_lock); + mark_buffer_dirty(bh); + brelse(bh); + set_inode_metadata_writeback(inode); + return 0; +} + +static int fat_write_inode(struct inode *inode, struct writeback_control *wbc) +{ + int err; + + if (inode->i_ino == MSDOS_FSINFO_INO) { + struct super_block *sb = inode->i_sb; + + mutex_lock(&MSDOS_SB(sb)->s_lock); + err = fat_clusters_flush(sb); + mutex_unlock(&MSDOS_SB(sb)->s_lock); + } else + err = __fat_write_inode(inode); + + return err; +} + +static int fat_show_options(struct seq_file *m, struct dentry *root); +static const struct super_operations fat_sops = { + .alloc_inode = fat_alloc_inode, + .free_inode = fat_free_inode, + .write_inode = fat_write_inode, + .sync_inode_metadata = fat_sync_inode_metadata, + .evict_inode = fat_evict_inode, + .put_super = fat_put_super, + .statfs = fat_statfs, + .show_options = fat_show_options, +}; + +static int fat_show_options(struct seq_file *m, struct dentry *root) +{ + struct msdos_sb_info *sbi = MSDOS_SB(root->d_sb); + struct fat_mount_options *opts = &sbi->options; + int isvfat = opts->isvfat; + + if (!uid_eq(opts->fs_uid, GLOBAL_ROOT_UID)) + seq_printf(m, ",uid=%u", + from_kuid_munged(&init_user_ns, opts->fs_uid)); + if (!gid_eq(opts->fs_gid, GLOBAL_ROOT_GID)) + seq_printf(m, ",gid=%u", + from_kgid_munged(&init_user_ns, opts->fs_gid)); + seq_printf(m, ",fmask=%04o", opts->fs_fmask); + seq_printf(m, ",dmask=%04o", opts->fs_dmask); + if (opts->allow_utime) + seq_printf(m, ",allow_utime=%04o", opts->allow_utime); + if (sbi->nls_disk) + /* strip "cp" prefix from displayed option */ + seq_printf(m, ",codepage=%s", &sbi->nls_disk->charset[2]); + if (isvfat) { + if (sbi->nls_io) + seq_printf(m, ",iocharset=%s", sbi->nls_io->charset); + + switch (opts->shortname) { + case VFAT_SFN_DISPLAY_WIN95 | VFAT_SFN_CREATE_WIN95: + seq_puts(m, ",shortname=win95"); + break; + case VFAT_SFN_DISPLAY_WINNT | VFAT_SFN_CREATE_WINNT: + seq_puts(m, ",shortname=winnt"); + break; + case VFAT_SFN_DISPLAY_WINNT | VFAT_SFN_CREATE_WIN95: + seq_puts(m, ",shortname=mixed"); + break; + case VFAT_SFN_DISPLAY_LOWER | VFAT_SFN_CREATE_WIN95: + seq_puts(m, ",shortname=lower"); + break; + default: + seq_puts(m, ",shortname=unknown"); + break; + } + } + if (opts->name_check != 'n') + seq_printf(m, ",check=%c", opts->name_check); + if (opts->usefree) + seq_puts(m, ",usefree"); + if (opts->quiet) + seq_puts(m, ",quiet"); + if (opts->showexec) + seq_puts(m, ",showexec"); + if (opts->sys_immutable) + seq_puts(m, ",sys_immutable"); + if (!isvfat) { + if (opts->dotsOK) + seq_puts(m, ",dotsOK=yes"); + if (opts->nocase) + seq_puts(m, ",nocase"); + } else { + if (opts->utf8) + seq_puts(m, ",utf8"); + if (opts->unicode_xlate) + seq_puts(m, ",uni_xlate"); + if (!opts->numtail) + seq_puts(m, ",nonumtail"); + if (opts->rodir) + seq_puts(m, ",rodir"); + } + if (opts->flush) + seq_puts(m, ",flush"); + if (opts->tz_set) { + if (opts->time_offset) + seq_printf(m, ",time_offset=%d", opts->time_offset); + else + seq_puts(m, ",tz=UTC"); + } + if (opts->errors == FAT_ERRORS_CONT) + seq_puts(m, ",errors=continue"); + else if (opts->errors == FAT_ERRORS_PANIC) + seq_puts(m, ",errors=panic"); + else + seq_puts(m, ",errors=remount-ro"); + if (opts->nfs == FAT_NFS_NOSTALE_RO) + seq_puts(m, ",nfs=nostale_ro"); + else if (opts->nfs) + seq_puts(m, ",nfs=stale_rw"); + if (opts->discard) + seq_puts(m, ",discard"); + if (opts->dos1xfloppy) + seq_puts(m, ",dos1xfloppy"); + + return 0; +} + +enum { + Opt_check, Opt_uid, Opt_gid, Opt_umask, Opt_dmask, Opt_fmask, + Opt_allow_utime, Opt_codepage, Opt_usefree, Opt_nocase, Opt_quiet, + Opt_showexec, Opt_debug, Opt_immutable, Opt_dots, Opt_dotsOK, + Opt_charset, Opt_shortname, Opt_utf8, Opt_utf8_bool, + Opt_uni_xl, Opt_uni_xl_bool, Opt_nonumtail, Opt_nonumtail_bool, + Opt_obsolete, Opt_flush, Opt_tz, Opt_rodir, Opt_errors, Opt_discard, + Opt_nfs, Opt_nfs_enum, Opt_time_offset, Opt_dos1xfloppy, +}; + +static const struct constant_table fat_param_check[] = { + {"relaxed", 'r'}, + {"r", 'r'}, + {"strict", 's'}, + {"s", 's'}, + {"normal", 'n'}, + {"n", 'n'}, + {} +}; + +static const struct constant_table fat_param_tz[] = { + {"UTC", 0}, + {} +}; + +static const struct constant_table fat_param_errors[] = { + {"continue", FAT_ERRORS_CONT}, + {"panic", FAT_ERRORS_PANIC}, + {"remount-ro", FAT_ERRORS_RO}, + {} +}; + + +static const struct constant_table fat_param_nfs[] = { + {"stale_rw", FAT_NFS_STALE_RW}, + {"nostale_ro", FAT_NFS_NOSTALE_RO}, + {} +}; + +/* + * These are all obsolete but we still reject invalid options. + * The corresponding values are therefore meaningless. + */ +static const struct constant_table fat_param_conv[] = { + {"binary", 0}, + {"text", 0}, + {"auto", 0}, + {"b", 0}, + {"t", 0}, + {"a", 0}, + {} +}; + +/* Core options. See below for vfat and msdos extras */ +const struct fs_parameter_spec fat_param_spec[] = { + fsparam_enum ("check", Opt_check, fat_param_check), + fsparam_uid ("uid", Opt_uid), + fsparam_gid ("gid", Opt_gid), + fsparam_u32oct ("umask", Opt_umask), + fsparam_u32oct ("dmask", Opt_dmask), + fsparam_u32oct ("fmask", Opt_fmask), + fsparam_u32oct ("allow_utime", Opt_allow_utime), + fsparam_u32 ("codepage", Opt_codepage), + fsparam_flag ("usefree", Opt_usefree), + fsparam_flag ("nocase", Opt_nocase), + fsparam_flag ("quiet", Opt_quiet), + fsparam_flag ("showexec", Opt_showexec), + fsparam_flag ("debug", Opt_debug), + fsparam_flag ("sys_immutable", Opt_immutable), + fsparam_flag ("flush", Opt_flush), + fsparam_enum ("tz", Opt_tz, fat_param_tz), + fsparam_s32 ("time_offset", Opt_time_offset), + fsparam_enum ("errors", Opt_errors, fat_param_errors), + fsparam_flag ("discard", Opt_discard), + fsparam_flag ("nfs", Opt_nfs), + fsparam_enum ("nfs", Opt_nfs_enum, fat_param_nfs), + fsparam_flag ("dos1xfloppy", Opt_dos1xfloppy), + __fsparam(fs_param_is_enum, "conv", + Opt_obsolete, fs_param_deprecated, fat_param_conv), + __fsparam(fs_param_is_u32, "fat", + Opt_obsolete, fs_param_deprecated, NULL), + __fsparam(fs_param_is_u32, "blocksize", + Opt_obsolete, fs_param_deprecated, NULL), + __fsparam(fs_param_is_string, "cvf_format", + Opt_obsolete, fs_param_deprecated, NULL), + __fsparam(fs_param_is_string, "cvf_options", + Opt_obsolete, fs_param_deprecated, NULL), + __fsparam(NULL, "posix", + Opt_obsolete, fs_param_deprecated, NULL), + {} +}; +EXPORT_SYMBOL_GPL(fat_param_spec); + +static const struct fs_parameter_spec msdos_param_spec[] = { + fsparam_flag_no ("dots", Opt_dots), + fsparam_bool ("dotsOK", Opt_dotsOK), + {} +}; + +static const struct constant_table fat_param_shortname[] = { + {"lower", VFAT_SFN_DISPLAY_LOWER | VFAT_SFN_CREATE_WIN95}, + {"win95", VFAT_SFN_DISPLAY_WIN95 | VFAT_SFN_CREATE_WIN95}, + {"winnt", VFAT_SFN_DISPLAY_WINNT | VFAT_SFN_CREATE_WINNT}, + {"mixed", VFAT_SFN_DISPLAY_WINNT | VFAT_SFN_CREATE_WIN95}, + {} +}; + +static const struct fs_parameter_spec vfat_param_spec[] = { + fsparam_string ("iocharset", Opt_charset), + fsparam_enum ("shortname", Opt_shortname, fat_param_shortname), + fsparam_flag ("utf8", Opt_utf8), + fsparam_bool ("utf8", Opt_utf8_bool), + fsparam_flag ("uni_xlate", Opt_uni_xl), + fsparam_bool ("uni_xlate", Opt_uni_xl_bool), + fsparam_flag ("nonumtail", Opt_nonumtail), + fsparam_bool ("nonumtail", Opt_nonumtail_bool), + fsparam_flag ("rodir", Opt_rodir), + {} +}; + +int fat_parse_param(struct fs_context *fc, struct fs_parameter *param, + bool is_vfat) +{ + struct fat_mount_options *opts = fc->fs_private; + struct fs_parse_result result; + int opt; + + /* remount options have traditionally been ignored */ + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE) + return 0; + + opt = fs_parse(fc, fat_param_spec, param, &result); + /* If option not found in fat_param_spec, try vfat/msdos options */ + if (opt == -ENOPARAM) { + if (is_vfat) + opt = fs_parse(fc, vfat_param_spec, param, &result); + else + opt = fs_parse(fc, msdos_param_spec, param, &result); + } + + if (opt < 0) + return opt; + + switch (opt) { + case Opt_check: + opts->name_check = result.uint_32; + break; + case Opt_usefree: + opts->usefree = 1; + break; + case Opt_nocase: + if (!is_vfat) + opts->nocase = 1; + else { + /* for backward compatibility */ + opts->shortname = VFAT_SFN_DISPLAY_WIN95 + | VFAT_SFN_CREATE_WIN95; + } + break; + case Opt_quiet: + opts->quiet = 1; + break; + case Opt_showexec: + opts->showexec = 1; + break; + case Opt_debug: + opts->debug = 1; + break; + case Opt_immutable: + opts->sys_immutable = 1; + break; + case Opt_uid: + opts->fs_uid = result.uid; + break; + case Opt_gid: + opts->fs_gid = result.gid; + break; + case Opt_umask: + opts->fs_fmask = opts->fs_dmask = result.uint_32; + break; + case Opt_dmask: + opts->fs_dmask = result.uint_32; + break; + case Opt_fmask: + opts->fs_fmask = result.uint_32; + break; + case Opt_allow_utime: + opts->allow_utime = result.uint_32 & (S_IWGRP | S_IWOTH); + break; + case Opt_codepage: + opts->codepage = result.uint_32; + break; + case Opt_flush: + opts->flush = 1; + break; + case Opt_time_offset: + /* + * GMT+-12 zones may have DST corrections so at least + * 13 hours difference is needed. Make the limit 24 + * just in case someone invents something unusual. + */ + if (result.int_32 < -24 * 60 || result.int_32 > 24 * 60) + return -EINVAL; + opts->tz_set = 1; + opts->time_offset = result.int_32; + break; + case Opt_tz: + opts->tz_set = 1; + opts->time_offset = result.uint_32; + break; + case Opt_errors: + opts->errors = result.uint_32; + break; + case Opt_nfs: + opts->nfs = FAT_NFS_STALE_RW; + break; + case Opt_nfs_enum: + opts->nfs = result.uint_32; + break; + case Opt_dos1xfloppy: + opts->dos1xfloppy = 1; + break; + + /* msdos specific */ + case Opt_dots: /* dots / nodots */ + opts->dotsOK = !result.negated; + break; + case Opt_dotsOK: /* dotsOK = yes/no */ + opts->dotsOK = result.boolean; + break; + + /* vfat specific */ + case Opt_charset: + fat_reset_iocharset(opts); + opts->iocharset = param->string; + param->string = NULL; /* Steal string */ + break; + case Opt_shortname: + opts->shortname = result.uint_32; + break; + case Opt_utf8: + opts->utf8 = 1; + break; + case Opt_utf8_bool: + opts->utf8 = result.boolean; + break; + case Opt_uni_xl: + opts->unicode_xlate = 1; + break; + case Opt_uni_xl_bool: + opts->unicode_xlate = result.boolean; + break; + case Opt_nonumtail: + opts->numtail = 0; /* negated option */ + break; + case Opt_nonumtail_bool: + opts->numtail = !result.boolean; /* negated option */ + break; + case Opt_rodir: + opts->rodir = 1; + break; + case Opt_discard: + opts->discard = 1; + break; + + /* obsolete mount options */ + case Opt_obsolete: + printk(KERN_INFO "FAT-fs: \"%s\" option is obsolete, " + "not supported now", param->key); + break; + default: + return -EINVAL; + } + + return 0; +} +EXPORT_SYMBOL_GPL(fat_parse_param); + +static int fat_read_root(struct inode *inode) +{ + struct msdos_sb_info *sbi = MSDOS_SB(inode->i_sb); + int error; + + MSDOS_I(inode)->i_pos = MSDOS_ROOT_INO; + inode->i_uid = sbi->options.fs_uid; + inode->i_gid = sbi->options.fs_gid; + inode_inc_iversion(inode); + inode->i_generation = 0; + inode->i_mode = fat_make_mode(sbi, ATTR_DIR, S_IRWXUGO); + inode->i_op = sbi->dir_ops; + inode->i_fop = &fat_dir_operations; + if (is_fat32(sbi)) { + MSDOS_I(inode)->i_start = sbi->root_cluster; + error = fat_calc_dir_size(inode); + if (error < 0) + return error; + } else { + MSDOS_I(inode)->i_start = 0; + inode->i_size = sbi->dir_entries * sizeof(struct msdos_dir_entry); + } + inode->i_blocks = ((inode->i_size + (sbi->cluster_size - 1)) + & ~((loff_t)sbi->cluster_size - 1)) >> 9; + MSDOS_I(inode)->i_logstart = 0; + MSDOS_I(inode)->mmu_private = inode->i_size; + + fat_save_attrs(inode, ATTR_DIR); + inode_set_mtime_to_ts(inode, + inode_set_atime_to_ts(inode, inode_set_ctime(inode, 0, 0))); + set_nlink(inode, fat_subdirs(inode)+2); + + return 0; +} + +static unsigned long calc_fat_clusters(struct super_block *sb) +{ + struct msdos_sb_info *sbi = MSDOS_SB(sb); + + /* Divide first to avoid overflow */ + if (!is_fat12(sbi)) { + unsigned long ent_per_sec = sb->s_blocksize * 8 / sbi->fat_bits; + return ent_per_sec * sbi->fat_length; + } + + return sbi->fat_length * sb->s_blocksize * 8 / sbi->fat_bits; +} + +static bool fat_bpb_is_zero(struct fat_boot_sector *b) +{ + if (get_unaligned_le16(&b->sector_size)) + return false; + if (b->sec_per_clus) + return false; + if (b->reserved) + return false; + if (b->fats) + return false; + if (get_unaligned_le16(&b->dir_entries)) + return false; + if (get_unaligned_le16(&b->sectors)) + return false; + if (b->media) + return false; + if (b->fat_length) + return false; + if (b->secs_track) + return false; + if (b->heads) + return false; + return true; +} + +static int fat_read_bpb(struct super_block *sb, struct fat_boot_sector *b, + int silent, struct fat_bios_param_block *bpb) +{ + int error = -EINVAL; + + /* Read in BPB ... */ + memset(bpb, 0, sizeof(*bpb)); + bpb->fat_sector_size = get_unaligned_le16(&b->sector_size); + bpb->fat_sec_per_clus = b->sec_per_clus; + bpb->fat_reserved = le16_to_cpu(b->reserved); + bpb->fat_fats = b->fats; + bpb->fat_dir_entries = get_unaligned_le16(&b->dir_entries); + bpb->fat_sectors = get_unaligned_le16(&b->sectors); + bpb->fat_fat_length = le16_to_cpu(b->fat_length); + bpb->fat_total_sect = le32_to_cpu(b->total_sect); + + bpb->fat16_state = b->fat16.state; + bpb->fat16_vol_id = get_unaligned_le32(b->fat16.vol_id); + + bpb->fat32_length = le32_to_cpu(b->fat32.length); + bpb->fat32_root_cluster = le32_to_cpu(b->fat32.root_cluster); + bpb->fat32_info_sector = le16_to_cpu(b->fat32.info_sector); + bpb->fat32_state = b->fat32.state; + bpb->fat32_vol_id = get_unaligned_le32(b->fat32.vol_id); + + /* Validate this looks like a FAT filesystem BPB */ + if (!bpb->fat_reserved) { + if (!silent) + fat_msg(sb, KERN_ERR, + "bogus number of reserved sectors"); + goto out; + } + if (!bpb->fat_fats) { + if (!silent) + fat_msg(sb, KERN_ERR, "bogus number of FAT structure"); + goto out; + } + + /* + * Earlier we checked here that b->secs_track and b->head are nonzero, + * but it turns out valid FAT filesystems can have zero there. + */ + + if (!fat_valid_media(b->media)) { + if (!silent) + fat_msg(sb, KERN_ERR, "invalid media value (0x%02x)", + (unsigned)b->media); + goto out; + } + + if (!is_power_of_2(bpb->fat_sector_size) + || (bpb->fat_sector_size < 512) + || (bpb->fat_sector_size > 4096)) { + if (!silent) + fat_msg(sb, KERN_ERR, "bogus logical sector size %u", + (unsigned)bpb->fat_sector_size); + goto out; + } + + if (!is_power_of_2(bpb->fat_sec_per_clus)) { + if (!silent) + fat_msg(sb, KERN_ERR, "bogus sectors per cluster %u", + (unsigned)bpb->fat_sec_per_clus); + goto out; + } + + if (bpb->fat_fat_length == 0 && bpb->fat32_length == 0) { + if (!silent) + fat_msg(sb, KERN_ERR, "bogus number of FAT sectors"); + goto out; + } + + error = 0; + +out: + return error; +} + +static int fat_read_static_bpb(struct super_block *sb, + struct fat_boot_sector *b, int silent, + struct fat_bios_param_block *bpb) +{ + static const char *notdos1x = "This doesn't look like a DOS 1.x volume"; + sector_t bd_sects = bdev_nr_sectors(sb->s_bdev); + struct fat_floppy_defaults *fdefaults = NULL; + int error = -EINVAL; + unsigned i; + + /* 16-bit DOS 1.x reliably wrote bootstrap short-jmp or near-jmp code */ + if ((b->ignored[0] != 0xeb || b->ignored[2] != 0x90) && + (b->ignored[0] != 0xe9)) { + if (!silent) + fat_msg(sb, KERN_ERR, + "%s; no bootstrapping code", notdos1x); + goto out; + } + + /* + * If any value in this region is non-zero, it isn't archaic + * DOS. + */ + if (!fat_bpb_is_zero(b)) { + if (!silent) + fat_msg(sb, KERN_ERR, + "%s; DOS 2.x BPB is non-zero", notdos1x); + goto out; + } + + for (i = 0; i < ARRAY_SIZE(floppy_defaults); i++) { + if (floppy_defaults[i].nr_sectors == bd_sects) { + fdefaults = &floppy_defaults[i]; + break; + } + } + + if (fdefaults == NULL) { + if (!silent) + fat_msg(sb, KERN_WARNING, + "This looks like a DOS 1.x volume, but isn't a recognized floppy size (%llu sectors)", + (u64)bd_sects); + goto out; + } + + if (!silent) + fat_msg(sb, KERN_INFO, + "This looks like a DOS 1.x volume; assuming default BPB values"); + + memset(bpb, 0, sizeof(*bpb)); + bpb->fat_sector_size = SECTOR_SIZE; + bpb->fat_sec_per_clus = fdefaults->sec_per_clus; + bpb->fat_reserved = 1; + bpb->fat_fats = 2; + bpb->fat_dir_entries = fdefaults->dir_entries; + bpb->fat_sectors = fdefaults->nr_sectors; + bpb->fat_fat_length = fdefaults->fat_length; + + error = 0; + +out: + return error; +} + +/* + * Read the super block of an MS-DOS FS. + */ +int fat_fill_super(struct super_block *sb, struct fs_context *fc, + void (*setup)(struct super_block *)) +{ + struct fat_mount_options *opts = fc->fs_private; + int silent = fc->sb_flags & SB_SILENT; + struct inode *root_inode = NULL, *fat_inode = NULL; + struct inode *fsinfo_inode = NULL; + struct buffer_head *bh; + struct fat_bios_param_block bpb; + struct msdos_sb_info *sbi; + u16 logical_sector_size; + u32 total_sectors, total_clusters, fat_clusters, rootdir_sectors; + long error; + char buf[50]; + struct timespec64 ts; + + /* + * GFP_KERNEL is ok here, because while we do hold the + * superblock lock, memory pressure can't call back into + * the filesystem, since we're only just about to mount + * it and have no inodes etc active! + */ + sbi = kzalloc_obj(struct msdos_sb_info); + if (!sbi) + return -ENOMEM; + sb->s_fs_info = sbi; + + sb->s_flags |= SB_NODIRATIME; + sb->s_magic = MSDOS_SUPER_MAGIC; + sb->s_op = &fat_sops; + sb->s_export_op = &fat_export_ops; + /* + * fat timestamps are complex and truncated by fat itself, so + * we set 1 here to be fast + */ + sb->s_time_gran = 1; + mutex_init(&sbi->nfs_build_inode_lock); + ratelimit_state_init(&sbi->ratelimit, DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + + /* UTF-8 doesn't provide FAT semantics */ + if (!strcmp(opts->iocharset, "utf8")) { + fat_msg(sb, KERN_WARNING, "utf8 is not a recommended IO charset" + " for FAT filesystems, filesystem will be" + " case sensitive!"); + } + + /* If user doesn't specify allow_utime, it's initialized from dmask. */ + if (opts->allow_utime == (unsigned short)-1) + opts->allow_utime = ~opts->fs_dmask & (S_IWGRP | S_IWOTH); + if (opts->unicode_xlate) + opts->utf8 = 0; + if (opts->nfs == FAT_NFS_NOSTALE_RO) { + sb->s_flags |= SB_RDONLY; + sb->s_export_op = &fat_export_ops_nostale; + } + + /* Apply parsed options to sbi (structure copy) */ + sbi->options = *opts; + /* Transfer ownership of iocharset to sbi->options */ + opts->iocharset = NULL; + + setup(sb); /* flavour-specific stuff that needs options */ + + error = -EINVAL; + if (!sb_min_blocksize(sb, 512)) { + fat_msg(sb, KERN_ERR, "unable to set blocksize"); + goto out_fail; + } + error = -EIO; + bh = sb_bread(sb, 0); + if (bh == NULL) { + fat_msg(sb, KERN_ERR, "unable to read boot sector"); + goto out_fail; + } + + error = fat_read_bpb(sb, (struct fat_boot_sector *)bh->b_data, silent, + &bpb); + if (error == -EINVAL && sbi->options.dos1xfloppy) + error = fat_read_static_bpb(sb, + (struct fat_boot_sector *)bh->b_data, silent, &bpb); + brelse(bh); + + if (error == -EINVAL) + goto out_invalid; + else if (error) + goto out_fail; + + logical_sector_size = bpb.fat_sector_size; + sbi->sec_per_clus = bpb.fat_sec_per_clus; + + error = -EIO; + if (logical_sector_size < sb->s_blocksize) { + fat_msg(sb, KERN_ERR, "logical sector size too small for device" + " (logical sector size = %u)", logical_sector_size); + goto out_fail; + } + + if (logical_sector_size > sb->s_blocksize) { + struct buffer_head *bh_resize; + + if (!sb_set_blocksize(sb, logical_sector_size)) { + fat_msg(sb, KERN_ERR, "unable to set blocksize %u", + logical_sector_size); + goto out_fail; + } + + /* Verify that the larger boot sector is fully readable */ + bh_resize = sb_bread(sb, 0); + if (bh_resize == NULL) { + fat_msg(sb, KERN_ERR, "unable to read boot sector" + " (logical sector size = %lu)", + sb->s_blocksize); + goto out_fail; + } + brelse(bh_resize); + } + + mutex_init(&sbi->s_lock); + sbi->cluster_size = sb->s_blocksize * sbi->sec_per_clus; + sbi->cluster_bits = ffs(sbi->cluster_size) - 1; + sbi->fats = bpb.fat_fats; + sbi->fat_bits = 0; /* Don't know yet */ + sbi->fat_start = bpb.fat_reserved; + sbi->fat_length = bpb.fat_fat_length; + sbi->root_cluster = 0; + sbi->free_clusters = -1; /* Don't know yet */ + sbi->free_clus_valid = 0; + sbi->prev_free = FAT_START_ENT; + sb->s_maxbytes = 0xffffffff; + fat_time_fat2unix(sbi, &ts, 0, cpu_to_le16(FAT_DATE_MIN), 0); + sb->s_time_min = ts.tv_sec; + + fat_time_fat2unix(sbi, &ts, cpu_to_le16(FAT_TIME_MAX), + cpu_to_le16(FAT_DATE_MAX), 0); + sb->s_time_max = ts.tv_sec; + + if (!sbi->fat_length && bpb.fat32_length) { + struct fat_boot_fsinfo *fsinfo; + struct buffer_head *fsinfo_bh; + + /* Must be FAT32 */ + sbi->fat_bits = 32; + sbi->fat_length = bpb.fat32_length; + sbi->root_cluster = bpb.fat32_root_cluster; + + /* MC - if info_sector is 0, don't multiply by 0 */ + sbi->fsinfo_sector = bpb.fat32_info_sector; + if (sbi->fsinfo_sector == 0) + sbi->fsinfo_sector = 1; + + fsinfo_bh = sb_bread(sb, sbi->fsinfo_sector); + if (fsinfo_bh == NULL) { + fat_msg(sb, KERN_ERR, "bread failed, FSINFO block" + " (sector = %lu)", sbi->fsinfo_sector); + goto out_fail; + } + + fsinfo = (struct fat_boot_fsinfo *)fsinfo_bh->b_data; + if (!IS_FSINFO(fsinfo)) { + fat_msg(sb, KERN_WARNING, "Invalid FSINFO signature: " + "0x%08x, 0x%08x (sector = %lu)", + le32_to_cpu(fsinfo->signature1), + le32_to_cpu(fsinfo->signature2), + sbi->fsinfo_sector); + } else { + if (sbi->options.usefree) + sbi->free_clus_valid = 1; + sbi->free_clusters = le32_to_cpu(fsinfo->free_clusters); + sbi->prev_free = le32_to_cpu(fsinfo->next_cluster); + } + + brelse(fsinfo_bh); + } + + /* interpret volume ID as a little endian 32 bit integer */ + if (is_fat32(sbi)) + sbi->vol_id = bpb.fat32_vol_id; + else /* fat 16 or 12 */ + sbi->vol_id = bpb.fat16_vol_id; + + __le32 vol_id_le = cpu_to_le32(sbi->vol_id); + super_set_uuid(sb, (void *) &vol_id_le, sizeof(vol_id_le)); + + sbi->dir_per_block = sb->s_blocksize / sizeof(struct msdos_dir_entry); + sbi->dir_per_block_bits = ffs(sbi->dir_per_block) - 1; + + sbi->dir_start = sbi->fat_start + sbi->fats * sbi->fat_length; + sbi->dir_entries = bpb.fat_dir_entries; + if (sbi->dir_entries & (sbi->dir_per_block - 1)) { + if (!silent) + fat_msg(sb, KERN_ERR, "bogus number of directory entries" + " (%u)", sbi->dir_entries); + goto out_invalid; + } + + rootdir_sectors = sbi->dir_entries + * sizeof(struct msdos_dir_entry) / sb->s_blocksize; + sbi->data_start = sbi->dir_start + rootdir_sectors; + total_sectors = bpb.fat_sectors; + if (total_sectors == 0) + total_sectors = bpb.fat_total_sect; + + if (total_sectors < sbi->data_start) { + if (!silent) + fat_msg(sb, KERN_ERR, + "data area starts beyond volume (%lu > %u)", + sbi->data_start, total_sectors); + goto out_invalid; + } + + total_clusters = (total_sectors - sbi->data_start) / sbi->sec_per_clus; + + if (!is_fat32(sbi)) + sbi->fat_bits = (total_clusters > MAX_FAT12) ? 16 : 12; + + /* some OSes set FAT_STATE_DIRTY and clean it on unmount. */ + if (is_fat32(sbi)) + sbi->dirty = bpb.fat32_state & FAT_STATE_DIRTY; + else /* fat 16 or 12 */ + sbi->dirty = bpb.fat16_state & FAT_STATE_DIRTY; + + /* check that FAT table does not overflow */ + fat_clusters = calc_fat_clusters(sb); + total_clusters = min(total_clusters, fat_clusters - FAT_START_ENT); + if (total_clusters > max_fat(sb)) { + if (!silent) + fat_msg(sb, KERN_ERR, "count of clusters too big (%u)", + total_clusters); + goto out_invalid; + } + + sbi->max_cluster = total_clusters + FAT_START_ENT; + /* check the free_clusters, it's not necessarily correct */ + if (sbi->free_clusters != -1 && sbi->free_clusters > total_clusters) + sbi->free_clusters = -1; + /* check the prev_free, it's not necessarily correct */ + sbi->prev_free %= sbi->max_cluster; + if (sbi->prev_free < FAT_START_ENT) + sbi->prev_free = FAT_START_ENT; + + /* set up enough so that it can read an inode */ + fat_hash_init(sb); + dir_hash_init(sb); + fat_ent_access_init(sb); + + /* + * The low byte of the first FAT entry must have the same value as + * the media field of the boot sector. But in real world, too many + * devices are writing wrong values. So, removed that validity check. + * + * The removed check compared the first FAT entry to a value dependent + * on the media field like this: + * == (0x0F00 | media), for FAT12 + * == (0XFF00 | media), for FAT16 + * == (0x0FFFFF | media), for FAT32 + */ + + error = -EINVAL; + scnprintf(buf, sizeof(buf), "cp%d", sbi->options.codepage); + sbi->nls_disk = load_nls(buf); + if (!sbi->nls_disk) { + fat_msg(sb, KERN_ERR, "codepage %s not found", buf); + goto out_fail; + } + + /* FIXME: utf8 is using iocharset for upper/lower conversion */ + if (sbi->options.isvfat) { + sbi->nls_io = load_nls(sbi->options.iocharset); + if (!sbi->nls_io) { + fat_msg(sb, KERN_ERR, "IO charset %s not found", + sbi->options.iocharset); + goto out_fail; + } + } + + error = -ENOMEM; + fat_inode = new_inode(sb); + if (!fat_inode) + goto out_fail; + sbi->fat_inode = fat_inode; + + fsinfo_inode = new_inode(sb); + if (!fsinfo_inode) + goto out_fail; + fsinfo_inode->i_ino = MSDOS_FSINFO_INO; + sbi->fsinfo_inode = fsinfo_inode; + insert_inode_hash(fsinfo_inode); + + root_inode = new_inode(sb); + if (!root_inode) + goto out_fail; + root_inode->i_ino = MSDOS_ROOT_INO; + inode_set_iversion(root_inode, 1); + error = fat_read_root(root_inode); + if (error < 0) { + iput(root_inode); + goto out_fail; + } + error = -ENOMEM; + insert_inode_hash(root_inode); + fat_attach(root_inode, 0); + sb->s_root = d_make_root(root_inode); + if (!sb->s_root) { + fat_msg(sb, KERN_ERR, "get root inode failed"); + goto out_fail; + } + + if (sbi->options.discard && !bdev_max_discard_sectors(sb->s_bdev)) + fat_msg(sb, KERN_WARNING, + "mounting with \"discard\" option, but the device does not support discard"); + + fat_set_state(sb, 1, 0); + return 0; + +out_invalid: + error = -EINVAL; + if (!silent) + fat_msg(sb, KERN_INFO, "Can't find a valid FAT filesystem"); + +out_fail: + iput(fsinfo_inode); + iput(fat_inode); + unload_nls(sbi->nls_io); + unload_nls(sbi->nls_disk); + fat_reset_iocharset(&sbi->options); + sb->s_fs_info = NULL; + kfree(sbi); + return error; +} + +EXPORT_SYMBOL_GPL(fat_fill_super); + +/* + * helper function for fat_flush_inodes. This writes both the inode + * and the file data blocks, waiting for in flight data blocks before + * the start of the call. It does not wait for any io started + * during the call + */ +static int writeback_inode(struct inode *inode) +{ + + int ret; + + /* if we used wait=1, sync_inode_metadata waits for the io for the + * inode to finish. So wait=0 is sent down to sync_inode_metadata + * and filemap_fdatawrite is used for the data blocks + */ + ret = sync_inode_metadata(inode, 0); + if (!ret) + ret = filemap_fdatawrite(inode->i_mapping); + return ret; +} + +/* + * write data and metadata corresponding to i1 and i2. The io is + * started but we do not wait for any of it to finish. + * + * filemap_flush is used for the block device, so if there is a dirty + * page for a block already in flight, we will not wait and start the + * io over again + */ +int fat_flush_inodes(struct super_block *sb, struct inode *i1, struct inode *i2) +{ + int ret = 0; + if (!MSDOS_SB(sb)->options.flush) + return 0; + if (i1) + ret = writeback_inode(i1); + if (!ret && i2) + ret = writeback_inode(i2); + if (!ret) + ret = sync_blockdev_nowait(sb->s_bdev); + return ret; +} +EXPORT_SYMBOL_GPL(fat_flush_inodes); + +int fat_init_fs_context(struct fs_context *fc, bool is_vfat) +{ + struct fat_mount_options *opts; + + opts = kzalloc_obj(*opts); + if (!opts) + return -ENOMEM; + + opts->isvfat = is_vfat; + opts->fs_uid = current_uid(); + opts->fs_gid = current_gid(); + opts->fs_fmask = opts->fs_dmask = current_umask(); + opts->allow_utime = -1; + opts->codepage = fat_default_codepage; + fat_reset_iocharset(opts); + if (is_vfat) { + opts->shortname = VFAT_SFN_DISPLAY_WINNT|VFAT_SFN_CREATE_WIN95; + opts->rodir = 0; + } else { + opts->shortname = 0; + opts->rodir = 1; + } + opts->name_check = 'n'; + opts->quiet = opts->showexec = opts->sys_immutable = opts->dotsOK = 0; + opts->unicode_xlate = 0; + opts->numtail = 1; + opts->usefree = opts->nocase = 0; + opts->tz_set = 0; + opts->nfs = 0; + opts->errors = FAT_ERRORS_RO; + opts->debug = 0; + + opts->utf8 = IS_ENABLED(CONFIG_FAT_DEFAULT_UTF8) && is_vfat; + + fc->fs_private = opts; + /* fc->ops assigned by caller */ + + return 0; +} +EXPORT_SYMBOL_GPL(fat_init_fs_context); + +void fat_free_fc(struct fs_context *fc) +{ + struct fat_mount_options *opts = fc->fs_private; + + if (opts->iocharset != fat_default_iocharset) + kfree(opts->iocharset); + kfree(fc->fs_private); +} +EXPORT_SYMBOL_GPL(fat_free_fc); + +static int __init init_fat_fs(void) +{ + int err; + + err = fat_cache_init(); + if (err) + return err; + + err = fat_init_inodecache(); + if (err) + goto failed; + + return 0; + +failed: + fat_cache_destroy(); + return err; +} + +static void __exit exit_fat_fs(void) +{ + fat_cache_destroy(); + fat_destroy_inodecache(); +} + +module_init(init_fat_fs) +module_exit(exit_fat_fs) + +MODULE_DESCRIPTION("Core FAT filesystem support"); +MODULE_LICENSE("GPL"); diff --git a/fs/fat/misc.c b/fs/fat/misc.c new file mode 100644 index 000000000..e79762cf1 --- /dev/null +++ b/fs/fat/misc.c @@ -0,0 +1,367 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/fat/misc.c + * + * Written 1992,1993 by Werner Almesberger + * 22/11/2000 - Fixed fat_date_unix2dos for dates earlier than 01/01/1980 + * and date_dos2unix for date==0 by Igor Zhbanov(bsg@uniyar.ac.ru) + */ + +#include "fat.h" +#include + +/* + * fat_fs_error reports a file system problem that might indicate fa data + * corruption/inconsistency. Depending on 'errors' mount option the + * panic() is called, or error message is printed FAT and nothing is done, + * or filesystem is remounted read-only (default behavior). + * In case the file system is remounted read-only, it can be made writable + * again by remounting it. + */ +void __fat_fs_error(struct super_block *sb, int report, const char *fmt, ...) +{ + struct fat_mount_options *opts = &MSDOS_SB(sb)->options; + va_list args; + struct va_format vaf; + + if (report) { + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + fat_msg(sb, KERN_ERR, "error, %pV", &vaf); + va_end(args); + } + + if (opts->errors == FAT_ERRORS_PANIC) + panic("FAT-fs (%s): fs panic from previous error\n", sb->s_id); + else if (opts->errors == FAT_ERRORS_RO && !sb_rdonly(sb)) { + sb->s_flags |= SB_RDONLY; + fat_msg(sb, KERN_ERR, "Filesystem has been set read-only"); + } +} +EXPORT_SYMBOL_GPL(__fat_fs_error); + +/** + * _fat_msg() - Print a preformatted FAT message based on a superblock. + * @sb: A pointer to a &struct super_block + * @level: A Kernel printk level constant + * @fmt: The printf-style format string to print. + * + * Everything that is not fat_fs_error() should be fat_msg(). + * + * fat_msg() wraps _fat_msg() for printk indexing. + */ +void _fat_msg(struct super_block *sb, const char *level, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + _printk(FAT_PRINTK_PREFIX "%pV\n", level, sb->s_id, &vaf); + va_end(args); +} + +/* Flushes the number of free clusters on FAT32 */ +/* XXX: Need to write one per FSINFO block. Currently only writes 1 */ +int fat_clusters_flush(struct super_block *sb) +{ + struct msdos_sb_info *sbi = MSDOS_SB(sb); + struct buffer_head *bh; + struct fat_boot_fsinfo *fsinfo; + + if (!is_fat32(sbi)) + return 0; + + bh = sb_bread(sb, sbi->fsinfo_sector); + if (bh == NULL) { + fat_msg(sb, KERN_ERR, "bread failed in fat_clusters_flush"); + return -EIO; + } + + fsinfo = (struct fat_boot_fsinfo *)bh->b_data; + /* Sanity check */ + if (!IS_FSINFO(fsinfo)) { + fat_msg(sb, KERN_ERR, "Invalid FSINFO signature: " + "0x%08x, 0x%08x (sector = %lu)", + le32_to_cpu(fsinfo->signature1), + le32_to_cpu(fsinfo->signature2), + sbi->fsinfo_sector); + } else { + if (sbi->free_clusters != -1) + fsinfo->free_clusters = cpu_to_le32(sbi->free_clusters); + if (sbi->prev_free != -1) + fsinfo->next_cluster = cpu_to_le32(sbi->prev_free); + mark_buffer_dirty(bh); + } + brelse(bh); + + return 0; +} + +/* + * fat_chain_add() adds a new cluster to the chain of clusters represented + * by inode. + */ +int fat_chain_add(struct inode *inode, int new_dclus, int nr_cluster) +{ + struct super_block *sb = inode->i_sb; + struct msdos_sb_info *sbi = MSDOS_SB(sb); + int ret, new_fclus, last; + + /* + * We must locate the last cluster of the file to add this new + * one (new_dclus) to the end of the link list (the FAT). + */ + last = new_fclus = 0; + if (MSDOS_I(inode)->i_start) { + int fclus, dclus; + + ret = fat_get_cluster(inode, FAT_ENT_EOF, &fclus, &dclus); + if (ret < 0) + return ret; + new_fclus = fclus + 1; + last = dclus; + } + + /* add new one to the last of the cluster chain */ + if (last) { + struct fat_entry fatent; + + fatent_init(&fatent); + ret = fat_ent_read(inode, &fatent, last); + if (ret >= 0) { + int wait = inode_needs_sync(inode); + int old = ret; + + ret = fat_ent_write(inode, &fatent, new_dclus, wait); + if (ret < 0) + fat_ent_write(inode, &fatent, old, wait); + fatent_brelse(&fatent); + } + if (ret < 0) + return ret; + /* + * FIXME:Although we can add this cache, fat_cache_add() is + * assuming to be called after linear search with fat_cache_id. + */ +// fat_cache_add(inode, new_fclus, new_dclus); + } else { + MSDOS_I(inode)->i_start = new_dclus; + MSDOS_I(inode)->i_logstart = new_dclus; + mark_inode_dirty(inode); + /* + * Since generic_write_sync() synchronizes regular files later, + * we sync here only directories. + */ + if (S_ISDIR(inode->i_mode) && IS_DIRSYNC(inode)) { + ret = sync_inode_metadata(inode, 1); + if (ret) + return ret; + } + + } + if (new_fclus != (inode->i_blocks >> (sbi->cluster_bits - 9))) { + fat_fs_error_ratelimit( + sb, "clusters badly computed (%d != %llu)", new_fclus, + (llu)(inode->i_blocks >> (sbi->cluster_bits - 9))); + fat_cache_inval_inode(inode); + } + inode->i_blocks += nr_cluster << (sbi->cluster_bits - 9); + + return 0; +} + +/* + * The epoch of FAT timestamp is 1980. + * : bits : value + * date: 0 - 4: day (1 - 31) + * date: 5 - 8: month (1 - 12) + * date: 9 - 15: year (0 - 127) from 1980 + * time: 0 - 4: sec (0 - 29) 2sec counts + * time: 5 - 10: min (0 - 59) + * time: 11 - 15: hour (0 - 23) + */ +#define SECS_PER_MIN 60 +#define SECS_PER_HOUR (60 * 60) +#define SECS_PER_DAY (SECS_PER_HOUR * 24) +/* days between 1.1.70 and 1.1.80 (2 leap days) */ +#define DAYS_DELTA (365 * 10 + 2) +/* 120 (2100 - 1980) isn't leap year */ +#define YEAR_2100 120 +#define IS_LEAP_YEAR(y) (!((y) & 3) && (y) != YEAR_2100) + +/* Linear day numbers of the respective 1sts in non-leap years. */ +static long days_in_year[] = { + /* Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec */ + 0, 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 0, 0, 0, +}; + +static inline int fat_tz_offset(const struct msdos_sb_info *sbi) +{ + return (sbi->options.tz_set ? + -sbi->options.time_offset : + sys_tz.tz_minuteswest) * SECS_PER_MIN; +} + +/* Convert a FAT time/date pair to a UNIX date (seconds since 1 1 70). */ +void fat_time_fat2unix(struct msdos_sb_info *sbi, struct timespec64 *ts, + __le16 __time, __le16 __date, u8 time_cs) +{ + u16 time = le16_to_cpu(__time), date = le16_to_cpu(__date); + time64_t second; + long day, leap_day, month, year; + + year = date >> 9; + month = max(1, (date >> 5) & 0xf); + day = max(1, date & 0x1f) - 1; + + leap_day = (year + 3) / 4; + if (year > YEAR_2100) /* 2100 isn't leap year */ + leap_day--; + if (IS_LEAP_YEAR(year) && month > 2) + leap_day++; + + second = (time & 0x1f) << 1; + second += ((time >> 5) & 0x3f) * SECS_PER_MIN; + second += (time >> 11) * SECS_PER_HOUR; + second += (time64_t)(year * 365 + leap_day + + days_in_year[month] + day + + DAYS_DELTA) * SECS_PER_DAY; + + second += fat_tz_offset(sbi); + + if (time_cs) { + ts->tv_sec = second + (time_cs / 100); + ts->tv_nsec = (time_cs % 100) * 10000000; + } else { + ts->tv_sec = second; + ts->tv_nsec = 0; + } +} + +/* Export fat_time_fat2unix() for the fat_test KUnit tests. */ +EXPORT_SYMBOL_GPL(fat_time_fat2unix); + +/* Convert linear UNIX date to a FAT time/date pair. */ +void fat_time_unix2fat(struct msdos_sb_info *sbi, struct timespec64 *ts, + __le16 *time, __le16 *date, u8 *time_cs) +{ + struct tm tm; + time64_to_tm(ts->tv_sec, -fat_tz_offset(sbi), &tm); + + /* FAT can only support year between 1980 to 2107 */ + if (tm.tm_year < 1980 - 1900) { + *time = 0; + *date = cpu_to_le16((0 << 9) | (1 << 5) | 1); + if (time_cs) + *time_cs = 0; + return; + } + if (tm.tm_year > 2107 - 1900) { + *time = cpu_to_le16((23 << 11) | (59 << 5) | 29); + *date = cpu_to_le16((127 << 9) | (12 << 5) | 31); + if (time_cs) + *time_cs = 199; + return; + } + + /* from 1900 -> from 1980 */ + tm.tm_year -= 80; + /* 0~11 -> 1~12 */ + tm.tm_mon++; + /* 0~59 -> 0~29(2sec counts) */ + tm.tm_sec >>= 1; + + *time = cpu_to_le16(tm.tm_hour << 11 | tm.tm_min << 5 | tm.tm_sec); + *date = cpu_to_le16(tm.tm_year << 9 | tm.tm_mon << 5 | tm.tm_mday); + if (time_cs) + *time_cs = (ts->tv_sec & 1) * 100 + ts->tv_nsec / 10000000; +} +EXPORT_SYMBOL_GPL(fat_time_unix2fat); + +static inline struct timespec64 fat_timespec64_trunc_2secs(struct timespec64 ts) +{ + return (struct timespec64){ ts.tv_sec & ~1ULL, 0 }; +} + +/* + * truncate atime to 24 hour granularity (00:00:00 in local timezone) + */ +struct timespec64 fat_truncate_atime(const struct msdos_sb_info *sbi, + const struct timespec64 *ts) +{ + /* to localtime */ + time64_t seconds = ts->tv_sec - fat_tz_offset(sbi); + s32 remainder; + + div_s64_rem(seconds, SECS_PER_DAY, &remainder); + /* to day boundary, and back to unix time */ + seconds = seconds + fat_tz_offset(sbi) - remainder; + + return (struct timespec64){ seconds, 0 }; +} +/* Export fat_truncate_atime() for the fat_test KUnit tests. */ +EXPORT_SYMBOL_GPL(fat_truncate_atime); + +/* + * Update the in-inode atime and/or mtime after truncating the timestamp to the + * granularity. All timestamps in root inode are always 0. + * + * ctime and mtime share the same on-disk field, and should be identical in + * memory. All mtime updates will be applied to ctime, but ctime updates are + * ignored. + */ +void fat_truncate_time(struct inode *inode, struct timespec64 *now, + unsigned int flags) +{ + struct msdos_sb_info *sbi = MSDOS_SB(inode->i_sb); + struct timespec64 ts; + + if (inode->i_ino == MSDOS_ROOT_INO) + return; + + if (now == NULL) { + now = &ts; + ts = current_time(inode); + } + + if (flags & FAT_UPDATE_ATIME) + inode_set_atime_to_ts(inode, fat_truncate_atime(sbi, now)); + if (flags & FAT_UPDATE_CMTIME) { + /* truncate mtime to 2 second granularity */ + struct timespec64 mtime = fat_timespec64_trunc_2secs(*now); + + inode_set_mtime_to_ts(inode, mtime); + inode_set_ctime_to_ts(inode, mtime); + } +} +EXPORT_SYMBOL_GPL(fat_truncate_time); + +int fat_update_time(struct inode *inode, enum fs_update_time type, + unsigned int flags) +{ + if (inode->i_ino != MSDOS_ROOT_INO) { + fat_truncate_time(inode, NULL, type == FS_UPD_ATIME ? + FAT_UPDATE_ATIME : FAT_UPDATE_CMTIME); + __mark_inode_dirty(inode, inode_time_dirty_flag(inode)); + } + return 0; +} +EXPORT_SYMBOL_GPL(fat_update_time); + +int fat_sync_bhs(struct buffer_head **bhs, int nr_bhs) +{ + int i, err = 0; + + for (i = 0; i < nr_bhs; i++) + write_dirty_buffer(bhs[i], 0); + + for (i = 0; i < nr_bhs; i++) { + wait_on_buffer(bhs[i]); + if (!err && !buffer_uptodate(bhs[i])) + err = -EIO; + } + return err; +} diff --git a/fs/fat/namei_msdos.c b/fs/fat/namei_msdos.c new file mode 100644 index 000000000..d46d1a385 --- /dev/null +++ b/fs/fat/namei_msdos.c @@ -0,0 +1,720 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/msdos/namei.c + * + * Written 1992,1993 by Werner Almesberger + * Hidden files 1995 by Albert Cahalan + * Rewritten for constant inumbers 1999 by Al Viro + */ + +#include +#include +#include "fat.h" + +/* Characters that are undesirable in an MS-DOS file name */ +static unsigned char bad_chars[] = "*?<>|\""; +static unsigned char bad_if_strict[] = "+=,; "; + +/***** Formats an MS-DOS file name. Rejects invalid names. */ +static int msdos_format_name(const unsigned char *name, int len, + unsigned char *res, struct fat_mount_options *opts) + /* + * name is the proposed name, len is its length, res is + * the resulting name, opts->name_check is either (r)elaxed, + * (n)ormal or (s)trict, opts->dotsOK allows dots at the + * beginning of name (for hidden files) + */ +{ + unsigned char *walk; + unsigned char c; + int space; + + if (len > NAME_MAX) + return -ENAMETOOLONG; + + if (name[0] == '.') { /* dotfile because . and .. already done */ + if (opts->dotsOK) { + /* Get rid of dot - test for it elsewhere */ + name++; + len--; + } else + return -EINVAL; + } + /* + * disallow names that _really_ start with a dot + */ + space = 1; + c = 0; + for (walk = res; len && walk - res < 8; walk++) { + c = *name++; + len--; + if (opts->name_check != 'r' && strchr(bad_chars, c)) + return -EINVAL; + if (opts->name_check == 's' && strchr(bad_if_strict, c)) + return -EINVAL; + if (c >= 'A' && c <= 'Z' && opts->name_check == 's') + return -EINVAL; + if (c < ' ' || c == ':' || c == '\\') + return -EINVAL; + /* + * 0xE5 is legal as a first character, but we must substitute + * 0x05 because 0xE5 marks deleted files. Yes, DOS really + * does this. + * It seems that Microsoft hacked DOS to support non-US + * characters after the 0xE5 character was already in use to + * mark deleted files. + */ + if ((res == walk) && (c == 0xE5)) + c = 0x05; + if (c == '.') + break; + space = (c == ' '); + *walk = (!opts->nocase && c >= 'a' && c <= 'z') ? c - 32 : c; + } + if (space) + return -EINVAL; + if (opts->name_check == 's' && len && c != '.') { + c = *name++; + len--; + if (c != '.') + return -EINVAL; + } + while (c != '.' && len--) + c = *name++; + if (c == '.') { + while (walk - res < 8) + *walk++ = ' '; + while (len > 0 && walk - res < MSDOS_NAME) { + c = *name++; + len--; + if (opts->name_check != 'r' && strchr(bad_chars, c)) + return -EINVAL; + if (opts->name_check == 's' && + strchr(bad_if_strict, c)) + return -EINVAL; + if (c < ' ' || c == ':' || c == '\\') + return -EINVAL; + if (c == '.') { + if (opts->name_check == 's') + return -EINVAL; + break; + } + if (c >= 'A' && c <= 'Z' && opts->name_check == 's') + return -EINVAL; + space = c == ' '; + if (!opts->nocase && c >= 'a' && c <= 'z') + *walk++ = c - 32; + else + *walk++ = c; + } + if (space) + return -EINVAL; + if (opts->name_check == 's' && len) + return -EINVAL; + } + while (walk - res < MSDOS_NAME) + *walk++ = ' '; + + return 0; +} + +/***** Locates a directory entry. Uses unformatted name. */ +static int msdos_find(struct inode *dir, const unsigned char *name, int len, + struct fat_slot_info *sinfo) +{ + struct msdos_sb_info *sbi = MSDOS_SB(dir->i_sb); + unsigned char msdos_name[MSDOS_NAME]; + int err; + + err = msdos_format_name(name, len, msdos_name, &sbi->options); + if (err) + return -ENOENT; + + err = fat_scan(dir, msdos_name, sinfo); + if (!err && sbi->options.dotsOK) { + if (name[0] == '.') { + if (!(sinfo->de->attr & ATTR_HIDDEN)) + err = -ENOENT; + } else { + if (sinfo->de->attr & ATTR_HIDDEN) + err = -ENOENT; + } + if (err) + brelse(sinfo->bh); + } + return err; +} + +/* + * Compute the hash for the msdos name corresponding to the dentry. + * Note: if the name is invalid, we leave the hash code unchanged so + * that the existing dentry can be used. The msdos fs routines will + * return ENOENT or EINVAL as appropriate. + */ +static int msdos_hash(const struct dentry *dentry, struct qstr *qstr) +{ + struct fat_mount_options *options = &MSDOS_SB(dentry->d_sb)->options; + unsigned char msdos_name[MSDOS_NAME]; + int error; + + error = msdos_format_name(qstr->name, qstr->len, msdos_name, options); + if (!error) + qstr->hash = full_name_hash(dentry, msdos_name, MSDOS_NAME); + return 0; +} + +/* + * Compare two msdos names. If either of the names are invalid, + * we fall back to doing the standard name comparison. + */ +static int msdos_cmp(const struct dentry *dentry, + unsigned int len, const char *str, const struct qstr *name) +{ + struct fat_mount_options *options = &MSDOS_SB(dentry->d_sb)->options; + unsigned char a_msdos_name[MSDOS_NAME], b_msdos_name[MSDOS_NAME]; + int error; + + error = msdos_format_name(name->name, name->len, a_msdos_name, options); + if (error) + goto old_compare; + error = msdos_format_name(str, len, b_msdos_name, options); + if (error) + goto old_compare; + error = memcmp(a_msdos_name, b_msdos_name, MSDOS_NAME); +out: + return error; + +old_compare: + error = 1; + if (name->len == len) + error = memcmp(name->name, str, len); + goto out; +} + +static const struct dentry_operations msdos_dentry_operations = { + .d_hash = msdos_hash, + .d_compare = msdos_cmp, +}; + +/* + * AV. Wrappers for FAT sb operations. Is it wise? + */ + +/***** Get inode using directory and name */ +static struct dentry *msdos_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + struct super_block *sb = dir->i_sb; + struct fat_slot_info sinfo; + struct inode *inode; + int err; + + mutex_lock(&MSDOS_SB(sb)->s_lock); + err = msdos_find(dir, dentry->d_name.name, dentry->d_name.len, &sinfo); + switch (err) { + case -ENOENT: + inode = NULL; + break; + case 0: + inode = fat_build_inode(sb, sinfo.de, sinfo.i_pos); + brelse(sinfo.bh); + break; + default: + inode = ERR_PTR(err); + } + mutex_unlock(&MSDOS_SB(sb)->s_lock); + return d_splice_alias(inode, dentry); +} + +/***** Creates a directory entry (name is already formatted). */ +static int msdos_add_entry(struct inode *dir, const unsigned char *name, + int is_dir, int is_hid, int cluster, + struct timespec64 *ts, struct fat_slot_info *sinfo) +{ + struct msdos_sb_info *sbi = MSDOS_SB(dir->i_sb); + struct msdos_dir_entry de; + __le16 time, date; + int err; + + memcpy(de.name, name, MSDOS_NAME); + de.attr = is_dir ? ATTR_DIR : ATTR_ARCH; + if (is_hid) + de.attr |= ATTR_HIDDEN; + de.lcase = 0; + fat_time_unix2fat(sbi, ts, &time, &date, NULL); + de.cdate = de.adate = 0; + de.ctime = 0; + de.ctime_cs = 0; + de.time = time; + de.date = date; + fat_set_start(&de, cluster); + de.size = 0; + + err = fat_add_entries(dir, &de, 1, sinfo); + if (err) + return err; + + fat_truncate_time(dir, ts, FAT_UPDATE_CMTIME); + mark_inode_dirty(dir); + if (IS_DIRSYNC(dir)) + (void)sync_inode_metadata(dir, 1); + + return 0; +} + +/***** Create a file */ +static int msdos_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct super_block *sb = dir->i_sb; + struct inode *inode = NULL; + struct fat_slot_info sinfo; + struct timespec64 ts; + unsigned char msdos_name[MSDOS_NAME]; + int err, is_hid; + + mutex_lock(&MSDOS_SB(sb)->s_lock); + + err = msdos_format_name(dentry->d_name.name, dentry->d_name.len, + msdos_name, &MSDOS_SB(sb)->options); + if (err) + goto out; + is_hid = (dentry->d_name.name[0] == '.') && (msdos_name[0] != '.'); + /* Have to do it due to foo vs. .foo conflicts */ + if (!fat_scan(dir, msdos_name, &sinfo)) { + brelse(sinfo.bh); + err = -EINVAL; + goto out; + } + + ts = current_time(dir); + err = msdos_add_entry(dir, msdos_name, 0, is_hid, 0, &ts, &sinfo); + if (err) + goto out; + inode = fat_build_inode(sb, sinfo.de, sinfo.i_pos); + brelse(sinfo.bh); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + goto out; + } + fat_truncate_time(inode, &ts, FAT_UPDATE_ATIME | FAT_UPDATE_CMTIME); + /* timestamp is already written, so mark_inode_dirty() is unneeded. */ + + d_instantiate(dentry, inode); +out: + mutex_unlock(&MSDOS_SB(sb)->s_lock); + if (!err) + err = fat_flush_inodes(sb, dir, inode); + return err; +} + +/***** Remove a directory */ +static int msdos_rmdir(struct inode *dir, struct dentry *dentry) +{ + struct super_block *sb = dir->i_sb; + struct inode *inode = d_inode(dentry); + struct fat_slot_info sinfo; + int err; + + mutex_lock(&MSDOS_SB(sb)->s_lock); + err = fat_dir_empty(inode); + if (err) + goto out; + err = msdos_find(dir, dentry->d_name.name, dentry->d_name.len, &sinfo); + if (err) + goto out; + + err = fat_remove_entries(dir, &sinfo); /* and releases bh */ + if (err) + goto out; + if (dir->i_nlink >= 3) + drop_nlink(dir); + else { + fat_fs_error(sb, "parent dir link count too low (%u)", + dir->i_nlink); + } + + clear_nlink(inode); + fat_detach(inode); +out: + mutex_unlock(&MSDOS_SB(sb)->s_lock); + if (!err) + err = fat_flush_inodes(sb, dir, inode); + + return err; +} + +/***** Make a directory */ +static struct dentry *msdos_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct super_block *sb = dir->i_sb; + struct fat_slot_info sinfo; + struct inode *inode; + unsigned char msdos_name[MSDOS_NAME]; + struct timespec64 ts; + int err, is_hid, cluster; + + mutex_lock(&MSDOS_SB(sb)->s_lock); + + err = msdos_format_name(dentry->d_name.name, dentry->d_name.len, + msdos_name, &MSDOS_SB(sb)->options); + if (err) + goto out; + is_hid = (dentry->d_name.name[0] == '.') && (msdos_name[0] != '.'); + /* foo vs .foo situation */ + if (!fat_scan(dir, msdos_name, &sinfo)) { + brelse(sinfo.bh); + err = -EINVAL; + goto out; + } + + ts = current_time(dir); + cluster = fat_alloc_new_dir(dir, &ts); + if (cluster < 0) { + err = cluster; + goto out; + } + err = msdos_add_entry(dir, msdos_name, 1, is_hid, cluster, &ts, &sinfo); + if (err) + goto out_free; + inc_nlink(dir); + + inode = fat_build_inode(sb, sinfo.de, sinfo.i_pos); + brelse(sinfo.bh); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + /* the directory was completed, just return a error */ + goto out; + } + set_nlink(inode, 2); + fat_truncate_time(inode, &ts, FAT_UPDATE_ATIME | FAT_UPDATE_CMTIME); + /* timestamp is already written, so mark_inode_dirty() is unneeded. */ + + d_instantiate(dentry, inode); + + mutex_unlock(&MSDOS_SB(sb)->s_lock); + fat_flush_inodes(sb, dir, inode); + return NULL; + +out_free: + fat_free_clusters(dir, cluster); +out: + mutex_unlock(&MSDOS_SB(sb)->s_lock); + return ERR_PTR(err); +} + +/***** Unlink a file */ +static int msdos_unlink(struct inode *dir, struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + struct super_block *sb = inode->i_sb; + struct fat_slot_info sinfo; + int err; + + mutex_lock(&MSDOS_SB(sb)->s_lock); + err = msdos_find(dir, dentry->d_name.name, dentry->d_name.len, &sinfo); + if (err) + goto out; + + err = fat_remove_entries(dir, &sinfo); /* and releases bh */ + if (err) + goto out; + clear_nlink(inode); + fat_detach(inode); +out: + mutex_unlock(&MSDOS_SB(sb)->s_lock); + if (!err) + err = fat_flush_inodes(sb, dir, inode); + + return err; +} + +static int do_msdos_rename(struct inode *old_dir, unsigned char *old_name, + struct dentry *old_dentry, + struct inode *new_dir, unsigned char *new_name, + struct dentry *new_dentry, int is_hid) +{ + struct buffer_head *dotdot_bh; + struct msdos_dir_entry *dotdot_de; + struct inode *old_inode, *new_inode; + struct fat_slot_info old_sinfo, sinfo; + struct timespec64 ts; + loff_t new_i_pos; + int err, old_attrs, is_dir, update_dotdot, corrupt = 0; + + old_sinfo.bh = sinfo.bh = dotdot_bh = NULL; + old_inode = d_inode(old_dentry); + new_inode = d_inode(new_dentry); + + err = fat_scan(old_dir, old_name, &old_sinfo); + if (err) { + err = -EIO; + goto out; + } + + is_dir = S_ISDIR(old_inode->i_mode); + update_dotdot = (is_dir && old_dir != new_dir); + if (update_dotdot) { + if (fat_get_dotdot_entry(old_inode, &dotdot_bh, &dotdot_de)) { + err = -EIO; + goto out; + } + } + + old_attrs = MSDOS_I(old_inode)->i_attrs; + err = fat_scan(new_dir, new_name, &sinfo); + if (!err) { + if (!new_inode) { + /* "foo" -> ".foo" case. just change the ATTR_HIDDEN */ + if (sinfo.de != old_sinfo.de) { + err = -EINVAL; + goto out; + } + if (is_hid) + MSDOS_I(old_inode)->i_attrs |= ATTR_HIDDEN; + else + MSDOS_I(old_inode)->i_attrs &= ~ATTR_HIDDEN; + mark_inode_dirty(old_inode); + if (IS_DIRSYNC(old_dir)) { + err = sync_inode_metadata(old_inode, 1); + if (err) { + MSDOS_I(old_inode)->i_attrs = old_attrs; + goto out; + } + } + + inode_inc_iversion(old_dir); + fat_truncate_time(old_dir, NULL, FAT_UPDATE_CMTIME); + mark_inode_dirty(old_dir); + if (IS_DIRSYNC(old_dir)) + (void)sync_inode_metadata(old_dir, 1); + goto out; + } + } + + ts = current_time(old_inode); + if (new_inode) { + if (err) + goto out; + if (is_dir) { + err = fat_dir_empty(new_inode); + if (err) + goto out; + } + new_i_pos = MSDOS_I(new_inode)->i_pos; + fat_detach(new_inode); + } else { + err = msdos_add_entry(new_dir, new_name, is_dir, is_hid, 0, + &ts, &sinfo); + if (err) + goto out; + new_i_pos = sinfo.i_pos; + } + inode_inc_iversion(new_dir); + + fat_detach(old_inode); + fat_attach(old_inode, new_i_pos); + if (is_hid) + MSDOS_I(old_inode)->i_attrs |= ATTR_HIDDEN; + else + MSDOS_I(old_inode)->i_attrs &= ~ATTR_HIDDEN; + mark_inode_dirty(old_inode); + if (IS_DIRSYNC(new_dir)) { + err = sync_inode_metadata(old_inode, 1); + if (err) + goto error_inode; + } + + if (update_dotdot) { + fat_set_start(dotdot_de, MSDOS_I(new_dir)->i_logstart); + mmb_mark_buffer_dirty(dotdot_bh, + &MSDOS_I(old_inode)->i_metadata_bhs); + if (IS_DIRSYNC(new_dir)) { + err = sync_dirty_buffer(dotdot_bh); + if (err) + goto error_dotdot; + } + drop_nlink(old_dir); + if (!new_inode) + inc_nlink(new_dir); + } + + err = fat_remove_entries(old_dir, &old_sinfo); /* and releases bh */ + old_sinfo.bh = NULL; + if (err) + goto error_dotdot; + inode_inc_iversion(old_dir); + fat_truncate_time(old_dir, &ts, FAT_UPDATE_CMTIME); + mark_inode_dirty(old_dir); + if (IS_DIRSYNC(old_dir)) + (void)sync_inode_metadata(old_dir, 1); + + if (new_inode) { + drop_nlink(new_inode); + if (is_dir) + drop_nlink(new_inode); + } +out: + brelse(sinfo.bh); + brelse(dotdot_bh); + brelse(old_sinfo.bh); + return err; + +error_dotdot: + /* data cluster is shared, serious corruption */ + corrupt = 1; + + if (update_dotdot) { + fat_set_start(dotdot_de, MSDOS_I(old_dir)->i_logstart); + mmb_mark_buffer_dirty(dotdot_bh, + &MSDOS_I(old_inode)->i_metadata_bhs); + corrupt |= sync_dirty_buffer(dotdot_bh); + } +error_inode: + fat_detach(old_inode); + fat_attach(old_inode, old_sinfo.i_pos); + MSDOS_I(old_inode)->i_attrs = old_attrs; + if (new_inode) { + fat_attach(new_inode, new_i_pos); + if (corrupt) { + mark_inode_dirty(new_inode); + corrupt |= sync_inode_metadata(new_inode, 1); + } + } else { + /* + * If new entry was not sharing the data cluster, it + * shouldn't be serious corruption. + */ + int err2 = fat_remove_entries(new_dir, &sinfo); + if (corrupt) + corrupt |= err2; + sinfo.bh = NULL; + } + if (corrupt < 0) { + fat_fs_error(new_dir->i_sb, + "%s: Filesystem corrupted (i_pos %lld)", + __func__, sinfo.i_pos); + } + goto out; +} + +/***** Rename, a wrapper for rename_same_dir & rename_diff_dir */ +static int msdos_rename(struct mnt_idmap *idmap, + struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry, + unsigned int flags) +{ + struct super_block *sb = old_dir->i_sb; + unsigned char old_msdos_name[MSDOS_NAME], new_msdos_name[MSDOS_NAME]; + int err, is_hid; + + if (flags & ~RENAME_NOREPLACE) + return -EINVAL; + + mutex_lock(&MSDOS_SB(sb)->s_lock); + + err = msdos_format_name(old_dentry->d_name.name, + old_dentry->d_name.len, old_msdos_name, + &MSDOS_SB(old_dir->i_sb)->options); + if (err) + goto out; + err = msdos_format_name(new_dentry->d_name.name, + new_dentry->d_name.len, new_msdos_name, + &MSDOS_SB(new_dir->i_sb)->options); + if (err) + goto out; + + is_hid = + (new_dentry->d_name.name[0] == '.') && (new_msdos_name[0] != '.'); + + err = do_msdos_rename(old_dir, old_msdos_name, old_dentry, + new_dir, new_msdos_name, new_dentry, is_hid); +out: + mutex_unlock(&MSDOS_SB(sb)->s_lock); + if (!err) + err = fat_flush_inodes(sb, old_dir, new_dir); + return err; +} + +static const struct inode_operations msdos_dir_inode_operations = { + .create = msdos_create, + .lookup = msdos_lookup, + .unlink = msdos_unlink, + .mkdir = msdos_mkdir, + .rmdir = msdos_rmdir, + .rename = msdos_rename, + .setattr = fat_setattr, + .getattr = fat_getattr, + .fileattr_get = fat_fileattr_get, + .update_time = fat_update_time, +}; + +static void setup(struct super_block *sb) +{ + MSDOS_SB(sb)->dir_ops = &msdos_dir_inode_operations; + set_default_d_op(sb, &msdos_dentry_operations); + sb->s_flags |= SB_NOATIME; +} + +static int msdos_fill_super(struct super_block *sb, struct fs_context *fc) +{ + return fat_fill_super(sb, fc, setup); +} + +static int msdos_get_tree(struct fs_context *fc) +{ + return get_tree_bdev(fc, msdos_fill_super); +} + +static int msdos_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + return fat_parse_param(fc, param, false); +} + +static const struct fs_context_operations msdos_context_ops = { + .parse_param = msdos_parse_param, + .get_tree = msdos_get_tree, + .reconfigure = fat_reconfigure, + .free = fat_free_fc, +}; + +static int msdos_init_fs_context(struct fs_context *fc) +{ + int err; + + /* Initialize with is_vfat == false */ + err = fat_init_fs_context(fc, false); + if (err) + return err; + + fc->ops = &msdos_context_ops; + return 0; +} + +static struct file_system_type msdos_fs_type = { + .owner = THIS_MODULE, + .name = "msdos", + .kill_sb = kill_block_super, + .fs_flags = FS_REQUIRES_DEV | FS_ALLOW_IDMAP, + .init_fs_context = msdos_init_fs_context, + .parameters = fat_param_spec, +}; +MODULE_ALIAS_FS("msdos"); + +static int __init init_msdos_fs(void) +{ + return register_filesystem(&msdos_fs_type); +} + +static void __exit exit_msdos_fs(void) +{ + unregister_filesystem(&msdos_fs_type); +} + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Werner Almesberger"); +MODULE_DESCRIPTION("MS-DOS filesystem support"); + +module_init(init_msdos_fs) +module_exit(exit_msdos_fs) diff --git a/fs/fat/namei_vfat.c b/fs/fat/namei_vfat.c new file mode 100644 index 000000000..da3e89c0b --- /dev/null +++ b/fs/fat/namei_vfat.c @@ -0,0 +1,1261 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/vfat/namei.c + * + * Written 1992,1993 by Werner Almesberger + * + * Windows95/Windows NT compatible extended MSDOS filesystem + * by Gordon Chaffee Copyright (C) 1995. Send bug reports for the + * VFAT filesystem to . Specify + * what file operation caused you trouble and if you can duplicate + * the problem, send a script that demonstrates it. + * + * Short name translation 1999, 2001 by Wolfram Pienkoss + * + * Support Multibyte characters and cleanup by + * OGAWA Hirofumi + */ + +#include +#include +#include +#include +#include +#include +#include +#include "fat.h" + +static inline unsigned long vfat_d_version(struct dentry *dentry) +{ + return (unsigned long) dentry->d_fsdata; +} + +static inline void vfat_d_version_set(struct dentry *dentry, + unsigned long version) +{ + dentry->d_fsdata = (void *) version; +} + +/* + * If new entry was created in the parent, it could create the 8.3 + * alias (the shortname of logname). So, the parent may have the + * negative-dentry which matches the created 8.3 alias. + * + * If it happened, the negative dentry isn't actually negative + * anymore. So, drop it. + */ +static bool vfat_revalidate_shortname(struct dentry *dentry, struct inode *dir) +{ + return inode_eq_iversion(dir, vfat_d_version(dentry)); +} + +static int vfat_revalidate(struct inode *dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + if (flags & LOOKUP_RCU) + return -ECHILD; + + /* This is not negative dentry. Always valid. */ + if (d_really_is_positive(dentry)) + return 1; + return vfat_revalidate_shortname(dentry, dir); +} + +static int vfat_revalidate_ci(struct inode *dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + if (flags & LOOKUP_RCU) + return -ECHILD; + + /* + * This is not negative dentry. Always valid. + * + * Note, rename() to existing directory entry will have ->d_inode, + * and will use existing name which isn't specified name by user. + * + * We may be able to drop this positive dentry here. But dropping + * positive dentry isn't good idea. So it's unsupported like + * rename("filename", "FILENAME") for now. + */ + if (d_really_is_positive(dentry)) + return 1; + + /* + * This may be nfsd (or something), anyway, we can't see the + * intent of this. So, since this can be for creation, drop it. + */ + if (!flags) + return 0; + + /* + * Drop the negative dentry, in order to make sure to use the + * case sensitive name which is specified by user if this is + * for creation. + */ + if (flags & (LOOKUP_CREATE | LOOKUP_RENAME_TARGET)) + return 0; + + return vfat_revalidate_shortname(dentry, dir); +} + +/* returns the length of a struct qstr, ignoring trailing dots */ +static unsigned int __vfat_striptail_len(unsigned int len, const char *name) +{ + while (len && name[len - 1] == '.') + len--; + return len; +} + +static unsigned int vfat_striptail_len(const struct qstr *qstr) +{ + return __vfat_striptail_len(qstr->len, qstr->name); +} + +/* + * Compute the hash for the vfat name corresponding to the dentry. + * Note: if the name is invalid, we leave the hash code unchanged so + * that the existing dentry can be used. The vfat fs routines will + * return ENOENT or EINVAL as appropriate. + */ +static int vfat_hash(const struct dentry *dentry, struct qstr *qstr) +{ + qstr->hash = full_name_hash(dentry, qstr->name, vfat_striptail_len(qstr)); + return 0; +} + +/* + * Compute the hash for the vfat name corresponding to the dentry. + * Note: if the name is invalid, we leave the hash code unchanged so + * that the existing dentry can be used. The vfat fs routines will + * return ENOENT or EINVAL as appropriate. + */ +static int vfat_hashi(const struct dentry *dentry, struct qstr *qstr) +{ + struct nls_table *t = MSDOS_SB(dentry->d_sb)->nls_io; + const unsigned char *name; + unsigned int len; + unsigned long hash; + + name = qstr->name; + len = vfat_striptail_len(qstr); + + hash = init_name_hash(dentry); + while (len--) + hash = partial_name_hash(nls_tolower(t, *name++), hash); + qstr->hash = end_name_hash(hash); + + return 0; +} + +/* + * Case insensitive compare of two vfat names. + */ +static int vfat_cmpi(const struct dentry *dentry, + unsigned int len, const char *str, const struct qstr *name) +{ + struct nls_table *t = MSDOS_SB(dentry->d_sb)->nls_io; + unsigned int alen, blen; + + /* A filename cannot end in '.' or we treat it like it has none */ + alen = vfat_striptail_len(name); + blen = __vfat_striptail_len(len, str); + if (alen == blen) { + if (nls_strnicmp(t, name->name, str, alen) == 0) + return 0; + } + return 1; +} + +/* + * Case sensitive compare of two vfat names. + */ +static int vfat_cmp(const struct dentry *dentry, + unsigned int len, const char *str, const struct qstr *name) +{ + unsigned int alen, blen; + + /* A filename cannot end in '.' or we treat it like it has none */ + alen = vfat_striptail_len(name); + blen = __vfat_striptail_len(len, str); + if (alen == blen) { + if (strncmp(name->name, str, alen) == 0) + return 0; + } + return 1; +} + +static const struct dentry_operations vfat_ci_dentry_ops = { + .d_revalidate = vfat_revalidate_ci, + .d_hash = vfat_hashi, + .d_compare = vfat_cmpi, +}; + +static const struct dentry_operations vfat_dentry_ops = { + .d_revalidate = vfat_revalidate, + .d_hash = vfat_hash, + .d_compare = vfat_cmp, +}; + +/* Characters that are undesirable in an MS-DOS file name */ + +static inline bool vfat_bad_char(wchar_t w) +{ + return (w < 0x0020) + || (w == '*') || (w == '?') || (w == '<') || (w == '>') + || (w == '|') || (w == '"') || (w == ':') || (w == '/') + || (w == '\\'); +} + +static inline bool vfat_replace_char(wchar_t w) +{ + return (w == '[') || (w == ']') || (w == ';') || (w == ',') + || (w == '+') || (w == '='); +} + +static wchar_t vfat_skip_char(wchar_t w) +{ + return (w == '.') || (w == ' '); +} + +static inline int vfat_is_used_badchars(const wchar_t *s, int len) +{ + int i; + + for (i = 0; i < len; i++) + if (vfat_bad_char(s[i])) + return -EINVAL; + + if (s[i - 1] == ' ') /* last character cannot be space */ + return -EINVAL; + + return 0; +} + +static int vfat_find_form(struct inode *dir, unsigned char *name) +{ + struct fat_slot_info sinfo; + int err = fat_scan(dir, name, &sinfo); + if (err) + return -ENOENT; + brelse(sinfo.bh); + return 0; +} + +/* + * 1) Valid characters for the 8.3 format alias are any combination of + * letters, uppercase alphabets, digits, any of the + * following special characters: + * $ % ' ` - @ { } ~ ! # ( ) & _ ^ + * In this case Longfilename is not stored in disk. + * + * WinNT's Extension: + * File name and extension name is contain uppercase/lowercase + * only. And it is expressed by CASE_LOWER_BASE and CASE_LOWER_EXT. + * + * 2) File name is 8.3 format, but it contain the uppercase and + * lowercase char, muliti bytes char, etc. In this case numtail is not + * added, but Longfilename is stored. + * + * 3) When the one except for the above, or the following special + * character are contained: + * . [ ] ; , + = + * numtail is added, and Longfilename must be stored in disk . + */ +struct shortname_info { + unsigned char lower:1, + upper:1, + valid:1; +}; +#define INIT_SHORTNAME_INFO(x) do { \ + (x)->lower = 1; \ + (x)->upper = 1; \ + (x)->valid = 1; \ +} while (0) + +static inline int to_shortname_char(struct nls_table *nls, + unsigned char *buf, int buf_size, + wchar_t *src, struct shortname_info *info) +{ + int len; + + if (vfat_skip_char(*src)) { + info->valid = 0; + return 0; + } + if (vfat_replace_char(*src)) { + info->valid = 0; + buf[0] = '_'; + return 1; + } + + len = nls->uni2char(*src, buf, buf_size); + if (len <= 0) { + info->valid = 0; + buf[0] = '_'; + len = 1; + } else if (len == 1) { + unsigned char prev = buf[0]; + + if (buf[0] >= 0x7F) { + info->lower = 0; + info->upper = 0; + } + + buf[0] = nls_toupper(nls, buf[0]); + if (isalpha(buf[0])) { + if (buf[0] == prev) + info->lower = 0; + else + info->upper = 0; + } + } else { + info->lower = 0; + info->upper = 0; + } + + return len; +} + +/* + * Given a valid longname, create a unique shortname. Make sure the + * shortname does not exist + * Returns negative number on error, 0 for a normal + * return, and 1 for valid shortname + */ +static int vfat_create_shortname(struct inode *dir, struct nls_table *nls, + wchar_t *uname, int ulen, + unsigned char *name_res, unsigned char *lcase) +{ + struct fat_mount_options *opts = &MSDOS_SB(dir->i_sb)->options; + wchar_t *ip, *ext_start, *end, *name_start; + unsigned char base[9], ext[4], buf[5], *p; + unsigned char charbuf[NLS_MAX_CHARSET_SIZE]; + int chl, chi; + int sz = 0, extlen, baselen, i, numtail_baselen, numtail2_baselen; + int is_shortname; + struct shortname_info base_info, ext_info; + + is_shortname = 1; + INIT_SHORTNAME_INFO(&base_info); + INIT_SHORTNAME_INFO(&ext_info); + + /* Now, we need to create a shortname from the long name */ + ext_start = end = &uname[ulen]; + while (--ext_start >= uname) { + if (*ext_start == 0x002E) { /* is `.' */ + if (ext_start == end - 1) { + sz = ulen; + ext_start = NULL; + } + break; + } + } + + if (ext_start == uname - 1) { + sz = ulen; + ext_start = NULL; + } else if (ext_start) { + /* + * Names which start with a dot could be just + * an extension eg. "...test". In this case Win95 + * uses the extension as the name and sets no extension. + */ + name_start = &uname[0]; + while (name_start < ext_start) { + if (!vfat_skip_char(*name_start)) + break; + name_start++; + } + if (name_start != ext_start) { + sz = ext_start - uname; + ext_start++; + } else { + sz = ulen; + ext_start = NULL; + } + } + + numtail_baselen = 6; + numtail2_baselen = 2; + for (baselen = i = 0, p = base, ip = uname; i < sz; i++, ip++) { + chl = to_shortname_char(nls, charbuf, sizeof(charbuf), + ip, &base_info); + if (chl == 0) + continue; + + if (baselen < 2 && (baselen + chl) > 2) + numtail2_baselen = baselen; + if (baselen < 6 && (baselen + chl) > 6) + numtail_baselen = baselen; + for (chi = 0; chi < chl; chi++) { + *p++ = charbuf[chi]; + baselen++; + if (baselen >= 8) + break; + } + if (baselen >= 8) { + if ((chi < chl - 1) || (ip + 1) - uname < sz) + is_shortname = 0; + break; + } + } + if (baselen == 0) { + return -EINVAL; + } + + extlen = 0; + if (ext_start) { + for (p = ext, ip = ext_start; extlen < 3 && ip < end; ip++) { + chl = to_shortname_char(nls, charbuf, sizeof(charbuf), + ip, &ext_info); + if (chl == 0) + continue; + + if ((extlen + chl) > 3) { + is_shortname = 0; + break; + } + for (chi = 0; chi < chl; chi++) { + *p++ = charbuf[chi]; + extlen++; + } + if (extlen >= 3) { + if (ip + 1 != end) + is_shortname = 0; + break; + } + } + } + ext[extlen] = '\0'; + base[baselen] = '\0'; + + /* Yes, it can happen. ".\xe5" would do it. */ + if (base[0] == DELETED_FLAG) + base[0] = 0x05; + + /* OK, at this point we know that base is not longer than 8 symbols, + * ext is not longer than 3, base is nonempty, both don't contain + * any bad symbols (lowercase transformed to uppercase). + */ + + memset(name_res, ' ', MSDOS_NAME); + memcpy(name_res, base, baselen); + memcpy(name_res + 8, ext, extlen); + *lcase = 0; + if (is_shortname && base_info.valid && ext_info.valid) { + if (vfat_find_form(dir, name_res) == 0) + return -EEXIST; + + if (opts->shortname & VFAT_SFN_CREATE_WIN95) { + return (base_info.upper && ext_info.upper); + } else if (opts->shortname & VFAT_SFN_CREATE_WINNT) { + if ((base_info.upper || base_info.lower) && + (ext_info.upper || ext_info.lower)) { + if (!base_info.upper && base_info.lower) + *lcase |= CASE_LOWER_BASE; + if (!ext_info.upper && ext_info.lower) + *lcase |= CASE_LOWER_EXT; + return 1; + } + return 0; + } else { + BUG(); + } + } + + if (opts->numtail == 0) + if (vfat_find_form(dir, name_res) < 0) + return 0; + + /* + * Try to find a unique extension. This used to + * iterate through all possibilities sequentially, + * but that gave extremely bad performance. Windows + * only tries a few cases before using random + * values for part of the base. + */ + + if (baselen > 6) { + baselen = numtail_baselen; + name_res[7] = ' '; + } + name_res[baselen] = '~'; + for (i = 1; i < 10; i++) { + name_res[baselen + 1] = i + '0'; + if (vfat_find_form(dir, name_res) < 0) + return 0; + } + + i = jiffies; + sz = (jiffies >> 16) & 0x7; + if (baselen > 2) { + baselen = numtail2_baselen; + name_res[7] = ' '; + } + name_res[baselen + 4] = '~'; + name_res[baselen + 5] = '1' + sz; + while (1) { + snprintf(buf, sizeof(buf), "%04X", i & 0xffff); + memcpy(&name_res[baselen], buf, 4); + if (vfat_find_form(dir, name_res) < 0) + break; + i -= 11; + } + return 0; +} + +/* Translate a string, including coded sequences into Unicode */ +static int +xlate_to_uni(const unsigned char *name, int len, unsigned char *outname, + int *longlen, int *outlen, int escape, int utf8, + struct nls_table *nls) +{ + const unsigned char *ip; + unsigned char *op; + int i, fill; + int charlen; + + if (utf8) { + *outlen = utf8s_to_utf16s(name, len, UTF16_HOST_ENDIAN, + (wchar_t *) outname, FAT_LFN_LEN + 2); + if (*outlen < 0) + return *outlen; + else if (*outlen > FAT_LFN_LEN) + return -ENAMETOOLONG; + + op = &outname[*outlen * sizeof(wchar_t)]; + } else { + for (i = 0, ip = name, op = outname, *outlen = 0; + i < len && *outlen < FAT_LFN_LEN; + *outlen += 1) { + if (escape && (*ip == ':')) { + u8 uc[2]; + + if (i > len - 5) + return -EINVAL; + + if (hex2bin(uc, ip + 1, 2) < 0) + return -EINVAL; + + *(wchar_t *)op = uc[0] << 8 | uc[1]; + + op += 2; + ip += 5; + i += 5; + } else { + charlen = nls->char2uni(ip, len - i, + (wchar_t *)op); + if (charlen < 0) + return -EINVAL; + ip += charlen; + i += charlen; + op += 2; + } + } + if (i < len) + return -ENAMETOOLONG; + } + + *longlen = *outlen; + if (*outlen % 13) { + *op++ = 0; + *op++ = 0; + *outlen += 1; + if (*outlen % 13) { + fill = 13 - (*outlen % 13); + for (i = 0; i < fill; i++) { + *op++ = 0xff; + *op++ = 0xff; + } + *outlen += fill; + } + } + + return 0; +} + +static int vfat_build_slots(struct inode *dir, const unsigned char *name, + int len, int is_dir, int cluster, + struct timespec64 *ts, + struct msdos_dir_slot *slots, int *nr_slots) +{ + struct msdos_sb_info *sbi = MSDOS_SB(dir->i_sb); + struct fat_mount_options *opts = &sbi->options; + struct msdos_dir_slot *ps; + struct msdos_dir_entry *de; + unsigned char cksum, lcase; + unsigned char msdos_name[MSDOS_NAME]; + wchar_t *uname; + __le16 time, date; + u8 time_cs; + int err, ulen, usize, i; + loff_t offset; + + *nr_slots = 0; + + uname = __getname(); + if (!uname) + return -ENOMEM; + + err = xlate_to_uni(name, len, (unsigned char *)uname, &ulen, &usize, + opts->unicode_xlate, opts->utf8, sbi->nls_io); + if (err) + goto out_free; + + err = vfat_is_used_badchars(uname, ulen); + if (err) + goto out_free; + + err = vfat_create_shortname(dir, sbi->nls_disk, uname, ulen, + msdos_name, &lcase); + if (err < 0) + goto out_free; + else if (err == 1) { + de = (struct msdos_dir_entry *)slots; + err = 0; + goto shortname; + } + + /* build the entry of long file name */ + cksum = fat_checksum(msdos_name); + + *nr_slots = usize / 13; + for (ps = slots, i = *nr_slots; i > 0; i--, ps++) { + ps->id = i; + ps->attr = ATTR_EXT; + ps->reserved = 0; + ps->alias_checksum = cksum; + ps->start = 0; + offset = (i - 1) * 13; + fatwchar_to16(ps->name0_4, uname + offset, 5); + fatwchar_to16(ps->name5_10, uname + offset + 5, 6); + fatwchar_to16(ps->name11_12, uname + offset + 11, 2); + } + slots[0].id |= 0x40; + de = (struct msdos_dir_entry *)ps; + +shortname: + /* build the entry of 8.3 alias name */ + (*nr_slots)++; + memcpy(de->name, msdos_name, MSDOS_NAME); + de->attr = is_dir ? ATTR_DIR : ATTR_ARCH; + de->lcase = lcase; + fat_time_unix2fat(sbi, ts, &time, &date, &time_cs); + de->time = de->ctime = time; + de->date = de->cdate = de->adate = date; + de->ctime_cs = time_cs; + fat_set_start(de, cluster); + de->size = 0; +out_free: + __putname(uname); + return err; +} + +static int vfat_add_entry(struct inode *dir, const struct qstr *qname, + int is_dir, int cluster, struct timespec64 *ts, + struct fat_slot_info *sinfo) +{ + struct msdos_dir_slot *slots; + unsigned int len; + int err, nr_slots; + + len = vfat_striptail_len(qname); + if (len == 0) + return -ENOENT; + + slots = kmalloc_objs(*slots, MSDOS_SLOTS, GFP_NOFS); + if (slots == NULL) + return -ENOMEM; + + err = vfat_build_slots(dir, qname->name, len, is_dir, cluster, ts, + slots, &nr_slots); + if (err) + goto cleanup; + + err = fat_add_entries(dir, slots, nr_slots, sinfo); + if (err) + goto cleanup; + + /* update timestamp */ + fat_truncate_time(dir, ts, FAT_UPDATE_CMTIME); + mark_inode_dirty(dir); + if (IS_DIRSYNC(dir)) + (void)sync_inode_metadata(dir, 1); +cleanup: + kfree(slots); + return err; +} + +static int vfat_find(struct inode *dir, const struct qstr *qname, + struct fat_slot_info *sinfo) +{ + unsigned int len = vfat_striptail_len(qname); + if (len == 0) + return -ENOENT; + return fat_search_long(dir, qname->name, len, sinfo); +} + +static struct dentry *vfat_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + struct super_block *sb = dir->i_sb; + struct fat_slot_info sinfo; + struct inode *inode; + struct dentry *alias; + int err; + + mutex_lock(&MSDOS_SB(sb)->s_lock); + + err = vfat_find(dir, &dentry->d_name, &sinfo); + if (err) { + if (err == -ENOENT) { + inode = NULL; + goto out; + } + goto error; + } + + inode = fat_build_inode(sb, sinfo.de, sinfo.i_pos); + brelse(sinfo.bh); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + goto error; + } + + alias = d_find_alias(inode); + /* + * Checking "alias->d_parent == dentry->d_parent" to make sure + * FS is not corrupted (especially double linked dir). + */ + if (alias && alias->d_parent == dentry->d_parent) { + /* + * This inode has non anonymous-DCACHE_DISCONNECTED + * dentry. This means, the user did ->lookup() by an + * another name (longname vs 8.3 alias of it) in past. + * + * Switch to new one for reason of locality if possible. + */ + if (!S_ISDIR(inode->i_mode)) + d_move(alias, dentry); + iput(inode); + mutex_unlock(&MSDOS_SB(sb)->s_lock); + return alias; + } else + dput(alias); + +out: + mutex_unlock(&MSDOS_SB(sb)->s_lock); + if (!inode) + vfat_d_version_set(dentry, inode_query_iversion(dir)); + return d_splice_alias(inode, dentry); +error: + mutex_unlock(&MSDOS_SB(sb)->s_lock); + return ERR_PTR(err); +} + +static int vfat_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct super_block *sb = dir->i_sb; + struct inode *inode; + struct fat_slot_info sinfo; + struct timespec64 ts; + int err; + + mutex_lock(&MSDOS_SB(sb)->s_lock); + + ts = current_time(dir); + err = vfat_add_entry(dir, &dentry->d_name, 0, 0, &ts, &sinfo); + if (err) + goto out; + inode_inc_iversion(dir); + + inode = fat_build_inode(sb, sinfo.de, sinfo.i_pos); + brelse(sinfo.bh); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + goto out; + } + inode_inc_iversion(inode); + + d_instantiate(dentry, inode); +out: + mutex_unlock(&MSDOS_SB(sb)->s_lock); + return err; +} + +static int vfat_rmdir(struct inode *dir, struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + struct super_block *sb = dir->i_sb; + struct fat_slot_info sinfo; + int err; + + mutex_lock(&MSDOS_SB(sb)->s_lock); + + err = fat_dir_empty(inode); + if (err) + goto out; + err = vfat_find(dir, &dentry->d_name, &sinfo); + if (err) + goto out; + + err = fat_remove_entries(dir, &sinfo); /* and releases bh */ + if (err) + goto out; + if (dir->i_nlink >= 3) + drop_nlink(dir); + else { + fat_fs_error(sb, "parent dir link count too low (%u)", + dir->i_nlink); + } + + clear_nlink(inode); + fat_truncate_time(inode, NULL, FAT_UPDATE_ATIME | FAT_UPDATE_CMTIME); + fat_detach(inode); + vfat_d_version_set(dentry, inode_query_iversion(dir)); +out: + mutex_unlock(&MSDOS_SB(sb)->s_lock); + + return err; +} + +static int vfat_unlink(struct inode *dir, struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + struct super_block *sb = dir->i_sb; + struct fat_slot_info sinfo; + int err; + + mutex_lock(&MSDOS_SB(sb)->s_lock); + + err = vfat_find(dir, &dentry->d_name, &sinfo); + if (err) + goto out; + + err = fat_remove_entries(dir, &sinfo); /* and releases bh */ + if (err) + goto out; + clear_nlink(inode); + fat_truncate_time(inode, NULL, FAT_UPDATE_ATIME | FAT_UPDATE_CMTIME); + fat_detach(inode); + vfat_d_version_set(dentry, inode_query_iversion(dir)); +out: + mutex_unlock(&MSDOS_SB(sb)->s_lock); + + return err; +} + +static struct dentry *vfat_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct super_block *sb = dir->i_sb; + struct inode *inode; + struct fat_slot_info sinfo; + struct timespec64 ts; + int err, cluster; + + mutex_lock(&MSDOS_SB(sb)->s_lock); + + ts = current_time(dir); + cluster = fat_alloc_new_dir(dir, &ts); + if (cluster < 0) { + err = cluster; + goto out; + } + err = vfat_add_entry(dir, &dentry->d_name, 1, cluster, &ts, &sinfo); + if (err) + goto out_free; + inode_inc_iversion(dir); + inc_nlink(dir); + + inode = fat_build_inode(sb, sinfo.de, sinfo.i_pos); + brelse(sinfo.bh); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + /* the directory was completed, just return a error */ + goto out; + } + inode_inc_iversion(inode); + set_nlink(inode, 2); + + d_instantiate(dentry, inode); + + mutex_unlock(&MSDOS_SB(sb)->s_lock); + return NULL; + +out_free: + fat_free_clusters(dir, cluster); +out: + mutex_unlock(&MSDOS_SB(sb)->s_lock); + return ERR_PTR(err); +} + +static int vfat_get_dotdot_de(struct inode *inode, struct buffer_head **bh, + struct msdos_dir_entry **de) +{ + if (S_ISDIR(inode->i_mode)) { + if (fat_get_dotdot_entry(inode, bh, de)) + return -EIO; + } + return 0; +} + +static int vfat_sync_ipos(struct inode *dir, struct inode *inode) +{ + mark_inode_dirty(inode); + if (IS_DIRSYNC(dir)) + return sync_inode_metadata(inode, 1); + return 0; +} + +static int vfat_update_dotdot_de(struct inode *dir, struct inode *inode, + struct buffer_head *dotdot_bh, + struct msdos_dir_entry *dotdot_de) +{ + fat_set_start(dotdot_de, MSDOS_I(dir)->i_logstart); + mmb_mark_buffer_dirty(dotdot_bh, &MSDOS_I(inode)->i_metadata_bhs); + if (IS_DIRSYNC(dir)) + return sync_dirty_buffer(dotdot_bh); + return 0; +} + +static void vfat_update_dir_metadata(struct inode *dir, struct timespec64 *ts) +{ + inode_inc_iversion(dir); + fat_truncate_time(dir, ts, FAT_UPDATE_CMTIME); + mark_inode_dirty(dir); + if (IS_DIRSYNC(dir)) + (void)sync_inode_metadata(dir, 1); +} + +static int vfat_rename(struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry) +{ + struct buffer_head *dotdot_bh; + struct msdos_dir_entry *dotdot_de = NULL; + struct inode *old_inode, *new_inode; + struct fat_slot_info old_sinfo, sinfo; + struct timespec64 ts; + loff_t new_i_pos; + int err, is_dir, corrupt = 0; + struct super_block *sb = old_dir->i_sb; + + old_sinfo.bh = sinfo.bh = dotdot_bh = NULL; + old_inode = d_inode(old_dentry); + new_inode = d_inode(new_dentry); + mutex_lock(&MSDOS_SB(sb)->s_lock); + err = vfat_find(old_dir, &old_dentry->d_name, &old_sinfo); + if (err) + goto out; + + if (old_dir != new_dir) { + err = vfat_get_dotdot_de(old_inode, &dotdot_bh, &dotdot_de); + if (err) + goto out; + } + + is_dir = S_ISDIR(old_inode->i_mode); + ts = current_time(old_dir); + if (new_inode) { + if (is_dir) { + err = fat_dir_empty(new_inode); + if (err) + goto out; + } + new_i_pos = MSDOS_I(new_inode)->i_pos; + fat_detach(new_inode); + } else { + err = vfat_add_entry(new_dir, &new_dentry->d_name, is_dir, 0, + &ts, &sinfo); + if (err) + goto out; + new_i_pos = sinfo.i_pos; + } + inode_inc_iversion(new_dir); + + fat_detach(old_inode); + fat_attach(old_inode, new_i_pos); + err = vfat_sync_ipos(new_dir, old_inode); + if (err) + goto error_inode; + + if (dotdot_de) { + err = vfat_update_dotdot_de(new_dir, old_inode, dotdot_bh, + dotdot_de); + if (err) + goto error_dotdot; + drop_nlink(old_dir); + if (!new_inode) + inc_nlink(new_dir); + } + + err = fat_remove_entries(old_dir, &old_sinfo); /* and releases bh */ + old_sinfo.bh = NULL; + if (err) + goto error_dotdot; + vfat_update_dir_metadata(old_dir, &ts); + + if (new_inode) { + drop_nlink(new_inode); + if (is_dir) + drop_nlink(new_inode); + } +out: + brelse(sinfo.bh); + brelse(dotdot_bh); + brelse(old_sinfo.bh); + mutex_unlock(&MSDOS_SB(sb)->s_lock); + + return err; + +error_dotdot: + /* data cluster is shared, serious corruption */ + corrupt = 1; + + if (dotdot_de) { + corrupt |= vfat_update_dotdot_de(old_dir, old_inode, dotdot_bh, + dotdot_de); + } +error_inode: + fat_detach(old_inode); + fat_attach(old_inode, old_sinfo.i_pos); + if (new_inode) { + fat_attach(new_inode, new_i_pos); + if (corrupt) { + mark_inode_dirty(new_inode); + corrupt |= sync_inode_metadata(new_inode, 1); + } + } else { + /* + * If new entry was not sharing the data cluster, it + * shouldn't be serious corruption. + */ + int err2 = fat_remove_entries(new_dir, &sinfo); + if (corrupt) + corrupt |= err2; + sinfo.bh = NULL; + } + if (corrupt < 0) { + fat_fs_error(new_dir->i_sb, + "%s: Filesystem corrupted (i_pos %lld)", + __func__, new_i_pos); + } + goto out; +} + +static void vfat_exchange_ipos(struct inode *old_inode, struct inode *new_inode, + loff_t old_i_pos, loff_t new_i_pos) +{ + fat_detach(old_inode); + fat_detach(new_inode); + fat_attach(old_inode, new_i_pos); + fat_attach(new_inode, old_i_pos); +} + +static void vfat_move_nlink(struct inode *src, struct inode *dst) +{ + drop_nlink(src); + inc_nlink(dst); +} + +static int vfat_rename_exchange(struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry) +{ + struct buffer_head *old_dotdot_bh = NULL, *new_dotdot_bh = NULL; + struct msdos_dir_entry *old_dotdot_de = NULL, *new_dotdot_de = NULL; + struct inode *old_inode, *new_inode; + struct timespec64 ts = current_time(old_dir); + loff_t old_i_pos, new_i_pos; + int err, corrupt = 0; + struct super_block *sb = old_dir->i_sb; + + old_inode = d_inode(old_dentry); + new_inode = d_inode(new_dentry); + + /* Acquire super block lock for the operation to be atomic */ + mutex_lock(&MSDOS_SB(sb)->s_lock); + + /* if directories are not the same, get ".." info to update */ + if (old_dir != new_dir) { + err = vfat_get_dotdot_de(old_inode, &old_dotdot_bh, + &old_dotdot_de); + if (err) + goto out; + + err = vfat_get_dotdot_de(new_inode, &new_dotdot_bh, + &new_dotdot_de); + if (err) + goto out; + } + + old_i_pos = MSDOS_I(old_inode)->i_pos; + new_i_pos = MSDOS_I(new_inode)->i_pos; + + vfat_exchange_ipos(old_inode, new_inode, old_i_pos, new_i_pos); + + err = vfat_sync_ipos(old_dir, new_inode); + if (err) + goto error_exchange; + err = vfat_sync_ipos(new_dir, old_inode); + if (err) + goto error_exchange; + + /* update ".." directory entry info */ + if (old_dotdot_de) { + err = vfat_update_dotdot_de(new_dir, old_inode, old_dotdot_bh, + old_dotdot_de); + if (err) + goto error_old_dotdot; + } + if (new_dotdot_de) { + err = vfat_update_dotdot_de(old_dir, new_inode, new_dotdot_bh, + new_dotdot_de); + if (err) + goto error_new_dotdot; + } + + /* if cross directory and only one is a directory, adjust nlink */ + if (!old_dotdot_de != !new_dotdot_de) { + if (old_dotdot_de) + vfat_move_nlink(old_dir, new_dir); + else + vfat_move_nlink(new_dir, old_dir); + } + + vfat_update_dir_metadata(old_dir, &ts); + /* if directories are not the same, update new_dir as well */ + if (old_dir != new_dir) + vfat_update_dir_metadata(new_dir, &ts); + +out: + brelse(old_dotdot_bh); + brelse(new_dotdot_bh); + mutex_unlock(&MSDOS_SB(sb)->s_lock); + + return err; + +error_new_dotdot: + if (new_dotdot_de) { + corrupt |= vfat_update_dotdot_de(new_dir, new_inode, + new_dotdot_bh, new_dotdot_de); + } + +error_old_dotdot: + if (old_dotdot_de) { + corrupt |= vfat_update_dotdot_de(old_dir, old_inode, + old_dotdot_bh, old_dotdot_de); + } + +error_exchange: + vfat_exchange_ipos(old_inode, new_inode, new_i_pos, old_i_pos); + corrupt |= vfat_sync_ipos(new_dir, new_inode); + corrupt |= vfat_sync_ipos(old_dir, old_inode); + + if (corrupt < 0) { + fat_fs_error(new_dir->i_sb, + "%s: Filesystem corrupted (i_pos %lld, %lld)", + __func__, old_i_pos, new_i_pos); + } + goto out; +} + +static int vfat_rename2(struct mnt_idmap *idmap, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) +{ + if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE)) + return -EINVAL; + + if (flags & RENAME_EXCHANGE) { + return vfat_rename_exchange(old_dir, old_dentry, + new_dir, new_dentry); + } + + /* VFS already handled RENAME_NOREPLACE, handle it as a normal rename */ + return vfat_rename(old_dir, old_dentry, new_dir, new_dentry); +} + +static const struct inode_operations vfat_dir_inode_operations = { + .create = vfat_create, + .lookup = vfat_lookup, + .unlink = vfat_unlink, + .mkdir = vfat_mkdir, + .rmdir = vfat_rmdir, + .rename = vfat_rename2, + .setattr = fat_setattr, + .getattr = fat_getattr, + .fileattr_get = fat_fileattr_get, + .update_time = fat_update_time, +}; + +static void setup(struct super_block *sb) +{ + MSDOS_SB(sb)->dir_ops = &vfat_dir_inode_operations; + if (MSDOS_SB(sb)->options.name_check != 's') + set_default_d_op(sb, &vfat_ci_dentry_ops); + else + set_default_d_op(sb, &vfat_dentry_ops); +} + +static int vfat_fill_super(struct super_block *sb, struct fs_context *fc) +{ + return fat_fill_super(sb, fc, setup); +} + +static int vfat_get_tree(struct fs_context *fc) +{ + return get_tree_bdev(fc, vfat_fill_super); +} + +static int vfat_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + return fat_parse_param(fc, param, true); +} + +static const struct fs_context_operations vfat_context_ops = { + .parse_param = vfat_parse_param, + .get_tree = vfat_get_tree, + .reconfigure = fat_reconfigure, + .free = fat_free_fc, +}; + +static int vfat_init_fs_context(struct fs_context *fc) +{ + int err; + + /* Initialize with is_vfat == true */ + err = fat_init_fs_context(fc, true); + if (err) + return err; + + fc->ops = &vfat_context_ops; + return 0; +} + +static struct file_system_type vfat_fs_type = { + .owner = THIS_MODULE, + .name = "vfat", + .kill_sb = kill_block_super, + .fs_flags = FS_REQUIRES_DEV | FS_ALLOW_IDMAP, + .init_fs_context = vfat_init_fs_context, + .parameters = fat_param_spec, +}; +MODULE_ALIAS_FS("vfat"); + +static int __init init_vfat_fs(void) +{ + return register_filesystem(&vfat_fs_type); +} + +static void __exit exit_vfat_fs(void) +{ + unregister_filesystem(&vfat_fs_type); +} + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("VFAT filesystem support"); +MODULE_AUTHOR("Gordon Chaffee"); + +module_init(init_vfat_fs) +module_exit(exit_vfat_fs) diff --git a/fs/fat/nfs.c b/fs/fat/nfs.c new file mode 100644 index 000000000..6e1b37171 --- /dev/null +++ b/fs/fat/nfs.c @@ -0,0 +1,301 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* fs/fat/nfs.c + */ + +#include +#include "fat.h" + +struct fat_fid { + u32 i_gen; + u32 i_pos_low; + u16 i_pos_hi; + u16 parent_i_pos_hi; + u32 parent_i_pos_low; + u32 parent_i_gen; +}; + +#define FAT_FID_SIZE_WITHOUT_PARENT 3 +#define FAT_FID_SIZE_WITH_PARENT (sizeof(struct fat_fid)/sizeof(u32)) + +/* + * Look up a directory inode given its starting cluster. + */ +static struct inode *fat_dget(struct super_block *sb, int i_logstart) +{ + struct msdos_sb_info *sbi = MSDOS_SB(sb); + struct hlist_head *head; + struct msdos_inode_info *i; + struct inode *inode = NULL; + + head = sbi->dir_hashtable + fat_dir_hash(i_logstart); + spin_lock(&sbi->dir_hash_lock); + hlist_for_each_entry(i, head, i_dir_hash) { + BUG_ON(i->vfs_inode.i_sb != sb); + if (i->i_logstart != i_logstart) + continue; + inode = igrab(&i->vfs_inode); + if (inode) + break; + } + spin_unlock(&sbi->dir_hash_lock); + return inode; +} + +static struct inode *fat_ilookup(struct super_block *sb, u64 ino, loff_t i_pos) +{ + if (MSDOS_SB(sb)->options.nfs == FAT_NFS_NOSTALE_RO) + return fat_iget(sb, i_pos); + + else { + if ((ino < MSDOS_ROOT_INO) || (ino == MSDOS_FSINFO_INO)) + return NULL; + return ilookup(sb, ino); + } +} + +static struct inode *__fat_nfs_get_inode(struct super_block *sb, + u64 ino, u32 generation, loff_t i_pos) +{ + struct inode *inode = fat_ilookup(sb, ino, i_pos); + + if (inode && generation && (inode->i_generation != generation)) { + iput(inode); + inode = NULL; + } + if (inode == NULL && MSDOS_SB(sb)->options.nfs == FAT_NFS_NOSTALE_RO) { + struct buffer_head *bh = NULL; + struct msdos_dir_entry *de ; + sector_t blocknr; + int offset; + fat_get_blknr_offset(MSDOS_SB(sb), i_pos, &blocknr, &offset); + bh = sb_bread(sb, blocknr); + if (!bh) { + fat_msg(sb, KERN_ERR, + "unable to read block(%llu) for building NFS inode", + (llu)blocknr); + return inode; + } + de = (struct msdos_dir_entry *)bh->b_data; + /* If a file is deleted on server and client is not updated + * yet, we must not build the inode upon a lookup call. + */ + if (IS_FREE(de[offset].name)) + inode = NULL; + else + inode = fat_build_inode(sb, &de[offset], i_pos); + brelse(bh); + } + + return inode; +} + +static struct inode *fat_nfs_get_inode(struct super_block *sb, + u64 ino, u32 generation) +{ + + return __fat_nfs_get_inode(sb, ino, generation, 0); +} + +static int +fat_encode_fh_nostale(struct inode *inode, __u32 *fh, int *lenp, + struct inode *parent) +{ + int len = *lenp; + struct msdos_sb_info *sbi = MSDOS_SB(inode->i_sb); + struct fat_fid *fid = (struct fat_fid *) fh; + loff_t i_pos; + int type = FILEID_FAT_WITHOUT_PARENT; + + if (parent) { + if (len < FAT_FID_SIZE_WITH_PARENT) { + *lenp = FAT_FID_SIZE_WITH_PARENT; + return FILEID_INVALID; + } + } else { + if (len < FAT_FID_SIZE_WITHOUT_PARENT) { + *lenp = FAT_FID_SIZE_WITHOUT_PARENT; + return FILEID_INVALID; + } + } + + i_pos = fat_i_pos_read(sbi, inode); + *lenp = FAT_FID_SIZE_WITHOUT_PARENT; + fid->i_gen = inode->i_generation; + fid->i_pos_low = i_pos & 0xFFFFFFFF; + fid->i_pos_hi = (i_pos >> 32) & 0xFFFF; + if (parent) { + i_pos = fat_i_pos_read(sbi, parent); + fid->parent_i_pos_hi = (i_pos >> 32) & 0xFFFF; + fid->parent_i_pos_low = i_pos & 0xFFFFFFFF; + fid->parent_i_gen = parent->i_generation; + type = FILEID_FAT_WITH_PARENT; + *lenp = FAT_FID_SIZE_WITH_PARENT; + } else { + /* + * We need to initialize this field because the fh is actually + * 12 bytes long + */ + fid->parent_i_pos_hi = 0; + } + + return type; +} + +/* + * Map a NFS file handle to a corresponding dentry. + * The dentry may or may not be connected to the filesystem root. + */ +static struct dentry *fat_fh_to_dentry(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + return generic_fh_to_dentry(sb, fid, fh_len, fh_type, + fat_nfs_get_inode); +} + +static struct dentry *fat_fh_to_dentry_nostale(struct super_block *sb, + struct fid *fh, int fh_len, + int fh_type) +{ + struct inode *inode = NULL; + struct fat_fid *fid = (struct fat_fid *)fh; + loff_t i_pos; + + switch (fh_type) { + case FILEID_FAT_WITHOUT_PARENT: + if (fh_len < FAT_FID_SIZE_WITHOUT_PARENT) + return NULL; + break; + case FILEID_FAT_WITH_PARENT: + if (fh_len < FAT_FID_SIZE_WITH_PARENT) + return NULL; + break; + default: + return NULL; + } + i_pos = fid->i_pos_hi; + i_pos = (i_pos << 32) | (fid->i_pos_low); + inode = __fat_nfs_get_inode(sb, 0, fid->i_gen, i_pos); + + return d_obtain_alias(inode); +} + +/* + * Find the parent for a file specified by NFS handle. + * This requires that the handle contain the i_ino of the parent. + */ +static struct dentry *fat_fh_to_parent(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + return generic_fh_to_parent(sb, fid, fh_len, fh_type, + fat_nfs_get_inode); +} + +static struct dentry *fat_fh_to_parent_nostale(struct super_block *sb, + struct fid *fh, int fh_len, + int fh_type) +{ + struct inode *inode = NULL; + struct fat_fid *fid = (struct fat_fid *)fh; + loff_t i_pos; + + if (fh_len < FAT_FID_SIZE_WITH_PARENT) + return NULL; + + switch (fh_type) { + case FILEID_FAT_WITH_PARENT: + i_pos = fid->parent_i_pos_hi; + i_pos = (i_pos << 32) | (fid->parent_i_pos_low); + inode = __fat_nfs_get_inode(sb, 0, fid->parent_i_gen, i_pos); + break; + } + + return d_obtain_alias(inode); +} + +/* + * Rebuild the parent for a directory that is not connected + * to the filesystem root + */ +static +struct inode *fat_rebuild_parent(struct super_block *sb, int parent_logstart) +{ + int search_clus, clus_to_match; + struct msdos_dir_entry *de; + struct inode *parent = NULL; + struct inode *dummy_grand_parent = NULL; + struct fat_slot_info sinfo; + struct msdos_sb_info *sbi = MSDOS_SB(sb); + sector_t blknr = fat_clus_to_blknr(sbi, parent_logstart); + struct buffer_head *parent_bh = sb_bread(sb, blknr); + if (!parent_bh) { + fat_msg(sb, KERN_ERR, + "unable to read cluster of parent directory"); + return NULL; + } + + de = (struct msdos_dir_entry *) parent_bh->b_data; + clus_to_match = fat_get_start(sbi, &de[0]); + search_clus = fat_get_start(sbi, &de[1]); + + dummy_grand_parent = fat_dget(sb, search_clus); + if (!dummy_grand_parent) { + dummy_grand_parent = new_inode(sb); + if (!dummy_grand_parent) { + brelse(parent_bh); + return parent; + } + + dummy_grand_parent->i_ino = iunique(sb, MSDOS_ROOT_INO); + fat_fill_inode(dummy_grand_parent, &de[1]); + MSDOS_I(dummy_grand_parent)->i_pos = -1; + } + + if (!fat_scan_logstart(dummy_grand_parent, clus_to_match, &sinfo)) { + parent = fat_build_inode(sb, sinfo.de, sinfo.i_pos); + brelse(sinfo.bh); + } + + brelse(parent_bh); + iput(dummy_grand_parent); + + return parent; +} + +/* + * Find the parent for a directory that is not currently connected to + * the filesystem root. + * + * On entry, the caller holds d_inode(child_dir)->i_mutex. + */ +static struct dentry *fat_get_parent(struct dentry *child_dir) +{ + struct super_block *sb = child_dir->d_sb; + struct buffer_head *bh = NULL; + struct msdos_dir_entry *de; + struct inode *parent_inode = NULL; + struct msdos_sb_info *sbi = MSDOS_SB(sb); + + if (!fat_get_dotdot_entry(d_inode(child_dir), &bh, &de)) { + int parent_logstart = fat_get_start(sbi, de); + parent_inode = fat_dget(sb, parent_logstart); + if (!parent_inode && sbi->options.nfs == FAT_NFS_NOSTALE_RO) + parent_inode = fat_rebuild_parent(sb, parent_logstart); + } + brelse(bh); + + return d_obtain_alias(parent_inode); +} + +const struct export_operations fat_export_ops = { + .encode_fh = generic_encode_ino32_fh, + .fh_to_dentry = fat_fh_to_dentry, + .fh_to_parent = fat_fh_to_parent, + .get_parent = fat_get_parent, +}; + +const struct export_operations fat_export_ops_nostale = { + .encode_fh = fat_encode_fh_nostale, + .fh_to_dentry = fat_fh_to_dentry_nostale, + .fh_to_parent = fat_fh_to_parent_nostale, + .get_parent = fat_get_parent, +}; diff --git a/fs/fcntl.c b/fs/fcntl.c new file mode 100644 index 000000000..c158f082f --- /dev/null +++ b/fs/fcntl.c @@ -0,0 +1,1183 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/fcntl.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "internal.h" + +#define SETFL_MASK (O_APPEND | O_NONBLOCK | O_NDELAY | O_DIRECT | O_NOATIME) + +static int setfl(int fd, struct file * filp, unsigned int arg) +{ + struct inode * inode = file_inode(filp); + int error = 0; + + /* + * O_APPEND cannot be cleared if the file is marked as append-only + * and the file is open for write. + */ + if (((arg ^ filp->f_flags) & O_APPEND) && IS_APPEND(inode)) + return -EPERM; + + /* O_NOATIME can only be set by the owner or superuser */ + if ((arg & O_NOATIME) && !(filp->f_flags & O_NOATIME)) + if (!inode_owner_or_capable(file_mnt_idmap(filp), inode)) + return -EPERM; + + /* required for strict SunOS emulation */ + if (O_NONBLOCK != O_NDELAY) + if (arg & O_NDELAY) + arg |= O_NONBLOCK; + + /* Pipe packetized mode is controlled by O_DIRECT flag */ + if (!S_ISFIFO(inode->i_mode) && + (arg & O_DIRECT) && + !(filp->f_mode & FMODE_CAN_ODIRECT)) + return -EINVAL; + + if (filp->f_op->check_flags) + error = filp->f_op->check_flags(arg); + if (error) + return error; + + /* + * ->fasync() is responsible for setting the FASYNC bit. + */ + if (((arg ^ filp->f_flags) & FASYNC) && filp->f_op->fasync) { + error = filp->f_op->fasync(fd, filp, (arg & FASYNC) != 0); + if (error < 0) + goto out; + if (error > 0) + error = 0; + } + spin_lock(&filp->f_lock); + filp->f_flags = (arg & SETFL_MASK) | (filp->f_flags & ~SETFL_MASK); + filp->f_iocb_flags = iocb_flags(filp); + spin_unlock(&filp->f_lock); + + out: + return error; +} + +/* + * Allocate an file->f_owner struct if it doesn't exist, handling racing + * allocations correctly. + */ +int file_f_owner_allocate(struct file *file) +{ + struct fown_struct *f_owner; + + f_owner = file_f_owner(file); + if (f_owner) + return 0; + + f_owner = kzalloc_obj(struct fown_struct); + if (!f_owner) + return -ENOMEM; + + rwlock_init(&f_owner->lock); + f_owner->file = file; + /* If someone else raced us, drop our allocation. */ + if (unlikely(cmpxchg(&file->f_owner, NULL, f_owner))) + kfree(f_owner); + return 0; +} +EXPORT_SYMBOL(file_f_owner_allocate); + +void file_f_owner_release(struct file *file) +{ + struct fown_struct *f_owner; + + f_owner = file_f_owner(file); + if (f_owner) { + put_pid(f_owner->pid); + kfree(f_owner); + } +} + +void __f_setown(struct file *filp, struct pid *pid, enum pid_type type, + int force) +{ + struct fown_struct *f_owner; + + f_owner = file_f_owner(filp); + if (WARN_ON_ONCE(!f_owner)) + return; + + write_lock_irq(&f_owner->lock); + if (force || !f_owner->pid) { + put_pid(f_owner->pid); + f_owner->pid = get_pid(pid); + f_owner->pid_type = type; + + if (pid) { + const struct cred *cred = current_cred(); + security_file_set_fowner(filp); + f_owner->uid = cred->uid; + f_owner->euid = cred->euid; + } + } + write_unlock_irq(&f_owner->lock); +} +EXPORT_SYMBOL(__f_setown); + +int f_setown(struct file *filp, int who, int force) +{ + enum pid_type type; + struct pid *pid = NULL; + int ret = 0; + + might_sleep(); + + type = PIDTYPE_TGID; + if (who < 0) { + /* avoid overflow below */ + if (who == INT_MIN) + return -EINVAL; + + type = PIDTYPE_PGID; + who = -who; + } + + ret = file_f_owner_allocate(filp); + if (ret) + return ret; + + rcu_read_lock(); + if (who) { + pid = find_vpid(who); + if (!pid) + ret = -ESRCH; + } + + if (!ret) + __f_setown(filp, pid, type, force); + rcu_read_unlock(); + + return ret; +} +EXPORT_SYMBOL(f_setown); + +void f_delown(struct file *filp) +{ + __f_setown(filp, NULL, PIDTYPE_TGID, 1); +} + +pid_t f_getown(struct file *filp) +{ + pid_t pid = 0; + struct fown_struct *f_owner; + + f_owner = file_f_owner(filp); + if (!f_owner) + return pid; + + read_lock_irq(&f_owner->lock); + rcu_read_lock(); + if (pid_task(f_owner->pid, f_owner->pid_type)) { + pid = pid_vnr(f_owner->pid); + if (f_owner->pid_type == PIDTYPE_PGID) + pid = -pid; + } + rcu_read_unlock(); + read_unlock_irq(&f_owner->lock); + return pid; +} + +static int f_setown_ex(struct file *filp, unsigned long arg) +{ + struct f_owner_ex __user *owner_p = (void __user *)arg; + struct f_owner_ex owner; + struct pid *pid; + int type; + int ret; + + ret = copy_from_user(&owner, owner_p, sizeof(owner)); + if (ret) + return -EFAULT; + + switch (owner.type) { + case F_OWNER_TID: + type = PIDTYPE_PID; + break; + + case F_OWNER_PID: + type = PIDTYPE_TGID; + break; + + case F_OWNER_PGRP: + type = PIDTYPE_PGID; + break; + + default: + return -EINVAL; + } + + ret = file_f_owner_allocate(filp); + if (ret) + return ret; + + rcu_read_lock(); + pid = find_vpid(owner.pid); + if (owner.pid && !pid) + ret = -ESRCH; + else + __f_setown(filp, pid, type, 1); + rcu_read_unlock(); + + return ret; +} + +static int f_getown_ex(struct file *filp, unsigned long arg) +{ + struct f_owner_ex __user *owner_p = (void __user *)arg; + struct f_owner_ex owner = {}; + int ret = 0; + struct fown_struct *f_owner; + enum pid_type pid_type = PIDTYPE_PID; + + f_owner = file_f_owner(filp); + if (f_owner) { + read_lock_irq(&f_owner->lock); + rcu_read_lock(); + if (pid_task(f_owner->pid, f_owner->pid_type)) + owner.pid = pid_vnr(f_owner->pid); + rcu_read_unlock(); + pid_type = f_owner->pid_type; + } + + switch (pid_type) { + case PIDTYPE_PID: + owner.type = F_OWNER_TID; + break; + + case PIDTYPE_TGID: + owner.type = F_OWNER_PID; + break; + + case PIDTYPE_PGID: + owner.type = F_OWNER_PGRP; + break; + + default: + WARN_ON(1); + ret = -EINVAL; + break; + } + if (f_owner) + read_unlock_irq(&f_owner->lock); + + if (!ret) { + ret = copy_to_user(owner_p, &owner, sizeof(owner)); + if (ret) + ret = -EFAULT; + } + return ret; +} + +#ifdef CONFIG_CHECKPOINT_RESTORE +static int f_getowner_uids(struct file *filp, unsigned long arg) +{ + struct user_namespace *user_ns = current_user_ns(); + struct fown_struct *f_owner; + uid_t __user *dst = (void __user *)arg; + uid_t src[2] = {0, 0}; + int err; + + f_owner = file_f_owner(filp); + if (f_owner) { + read_lock_irq(&f_owner->lock); + src[0] = from_kuid(user_ns, f_owner->uid); + src[1] = from_kuid(user_ns, f_owner->euid); + read_unlock_irq(&f_owner->lock); + } + + err = put_user(src[0], &dst[0]); + err |= put_user(src[1], &dst[1]); + + return err; +} +#else +static int f_getowner_uids(struct file *filp, unsigned long arg) +{ + return -EINVAL; +} +#endif + +static bool rw_hint_valid(u64 hint) +{ + BUILD_BUG_ON(WRITE_LIFE_NOT_SET != RWH_WRITE_LIFE_NOT_SET); + BUILD_BUG_ON(WRITE_LIFE_NONE != RWH_WRITE_LIFE_NONE); + BUILD_BUG_ON(WRITE_LIFE_SHORT != RWH_WRITE_LIFE_SHORT); + BUILD_BUG_ON(WRITE_LIFE_MEDIUM != RWH_WRITE_LIFE_MEDIUM); + BUILD_BUG_ON(WRITE_LIFE_LONG != RWH_WRITE_LIFE_LONG); + BUILD_BUG_ON(WRITE_LIFE_EXTREME != RWH_WRITE_LIFE_EXTREME); + + switch (hint) { + case RWH_WRITE_LIFE_NOT_SET: + case RWH_WRITE_LIFE_NONE: + case RWH_WRITE_LIFE_SHORT: + case RWH_WRITE_LIFE_MEDIUM: + case RWH_WRITE_LIFE_LONG: + case RWH_WRITE_LIFE_EXTREME: + return true; + default: + return false; + } +} + +static long fcntl_get_rw_hint(struct file *file, unsigned long arg) +{ + struct inode *inode = file_inode(file); + u64 __user *argp = (u64 __user *)arg; + u64 hint = READ_ONCE(inode->i_write_hint); + + if (copy_to_user(argp, &hint, sizeof(*argp))) + return -EFAULT; + return 0; +} + +static long fcntl_set_rw_hint(struct file *file, unsigned long arg) +{ + struct inode *inode = file_inode(file); + u64 __user *argp = (u64 __user *)arg; + u64 hint; + + if (!inode_owner_or_capable(file_mnt_idmap(file), inode)) + return -EPERM; + + if (copy_from_user(&hint, argp, sizeof(hint))) + return -EFAULT; + if (!rw_hint_valid(hint)) + return -EINVAL; + + WRITE_ONCE(inode->i_write_hint, hint); + + /* + * file->f_mapping->host may differ from inode. As an example, + * blkdev_open() modifies file->f_mapping. + */ + if (file->f_mapping->host != inode) + WRITE_ONCE(file->f_mapping->host->i_write_hint, hint); + + return 0; +} + +/* Is the file descriptor a dup of the file? */ +static long f_dupfd_query(int fd, struct file *filp) +{ + CLASS(fd_raw, f)(fd); + + if (fd_empty(f)) + return -EBADF; + + /* + * We can do the 'fdput()' immediately, as the only thing that + * matters is the pointer value which isn't changed by the fdput. + * + * Technically we didn't need a ref at all, and 'fdget()' was + * overkill, but given our lockless file pointer lookup, the + * alternatives are complicated. + */ + return fd_file(f) == filp; +} + +/* Let the caller figure out whether a given file was just created. */ +static long f_created_query(const struct file *filp) +{ + return !!(filp->f_mode & FMODE_CREATED); +} + +static int f_owner_sig(struct file *filp, int signum, bool setsig) +{ + int ret = 0; + struct fown_struct *f_owner; + + might_sleep(); + + if (setsig) { + if (!valid_signal(signum)) + return -EINVAL; + + ret = file_f_owner_allocate(filp); + if (ret) + return ret; + } + + f_owner = file_f_owner(filp); + if (setsig) + f_owner->signum = signum; + else if (f_owner) + ret = f_owner->signum; + return ret; +} + +static long do_fcntl(int fd, unsigned int cmd, unsigned long arg, + struct file *filp) +{ + void __user *argp = (void __user *)arg; + struct delegation deleg; + int argi = (int)arg; + struct flock flock; + long err = -EINVAL; + + switch (cmd) { + case F_CREATED_QUERY: + err = f_created_query(filp); + break; + case F_DUPFD: + err = f_dupfd(argi, filp, 0); + break; + case F_DUPFD_CLOEXEC: + err = f_dupfd(argi, filp, O_CLOEXEC); + break; + case F_DUPFD_QUERY: + err = f_dupfd_query(argi, filp); + break; + case F_GETFD: + err = get_close_on_exec(fd) ? FD_CLOEXEC : 0; + break; + case F_SETFD: + err = 0; + set_close_on_exec(fd, argi & FD_CLOEXEC); + break; + case F_GETFL: + err = filp->f_flags; + break; + case F_SETFL: + err = setfl(fd, filp, argi); + break; +#if BITS_PER_LONG != 32 + /* 32-bit arches must use fcntl64() */ + case F_OFD_GETLK: +#endif + case F_GETLK: + if (copy_from_user(&flock, argp, sizeof(flock))) + return -EFAULT; + err = fcntl_getlk(filp, cmd, &flock); + if (!err && copy_to_user(argp, &flock, sizeof(flock))) + return -EFAULT; + break; +#if BITS_PER_LONG != 32 + /* 32-bit arches must use fcntl64() */ + case F_OFD_SETLK: + case F_OFD_SETLKW: + fallthrough; +#endif + case F_SETLK: + case F_SETLKW: + if (copy_from_user(&flock, argp, sizeof(flock))) + return -EFAULT; + err = fcntl_setlk(fd, filp, cmd, &flock); + break; + case F_GETOWN: + /* + * XXX If f_owner is a process group, the + * negative return value will get converted + * into an error. Oops. If we keep the + * current syscall conventions, the only way + * to fix this will be in libc. + */ + err = f_getown(filp); + force_successful_syscall_return(); + break; + case F_SETOWN: + err = f_setown(filp, argi, 1); + break; + case F_GETOWN_EX: + err = f_getown_ex(filp, arg); + break; + case F_SETOWN_EX: + err = f_setown_ex(filp, arg); + break; + case F_GETOWNER_UIDS: + err = f_getowner_uids(filp, arg); + break; + case F_GETSIG: + err = f_owner_sig(filp, 0, false); + break; + case F_SETSIG: + err = f_owner_sig(filp, argi, true); + break; + case F_GETLEASE: + err = fcntl_getlease(filp); + break; + case F_SETLEASE: + err = fcntl_setlease(fd, filp, argi); + break; + case F_NOTIFY: + err = fcntl_dirnotify(fd, filp, argi); + break; + case F_SETPIPE_SZ: + case F_GETPIPE_SZ: + err = pipe_fcntl(filp, cmd, argi); + break; + case F_ADD_SEALS: + case F_GET_SEALS: + err = memfd_fcntl(filp, cmd, argi); + break; + case F_GET_RW_HINT: + err = fcntl_get_rw_hint(filp, arg); + break; + case F_SET_RW_HINT: + err = fcntl_set_rw_hint(filp, arg); + break; + case F_GETDELEG: + if (copy_from_user(&deleg, argp, sizeof(deleg))) + return -EFAULT; + err = fcntl_getdeleg(filp, &deleg); + if (!err && copy_to_user(argp, &deleg, sizeof(deleg))) + return -EFAULT; + break; + case F_SETDELEG: + if (copy_from_user(&deleg, argp, sizeof(deleg))) + return -EFAULT; + err = fcntl_setdeleg(fd, filp, &deleg); + break; + default: + break; + } + return err; +} + +static int check_fcntl_cmd(unsigned cmd) +{ + switch (cmd) { + case F_CREATED_QUERY: + case F_DUPFD: + case F_DUPFD_CLOEXEC: + case F_DUPFD_QUERY: + case F_GETFD: + case F_SETFD: + case F_GETFL: + return 1; + } + return 0; +} + +SYSCALL_DEFINE3(fcntl, unsigned int, fd, unsigned int, cmd, unsigned long, arg) +{ + CLASS(fd_raw, f)(fd); + long err; + + if (fd_empty(f)) + return -EBADF; + + if (unlikely(fd_file(f)->f_mode & FMODE_PATH)) { + if (!check_fcntl_cmd(cmd)) + return -EBADF; + } + + err = security_file_fcntl(fd_file(f), cmd, arg); + if (!err) + err = do_fcntl(fd, cmd, arg, fd_file(f)); + + return err; +} + +#if BITS_PER_LONG == 32 +SYSCALL_DEFINE3(fcntl64, unsigned int, fd, unsigned int, cmd, + unsigned long, arg) +{ + void __user *argp = (void __user *)arg; + CLASS(fd_raw, f)(fd); + struct flock64 flock; + long err; + + if (fd_empty(f)) + return -EBADF; + + if (unlikely(fd_file(f)->f_mode & FMODE_PATH)) { + if (!check_fcntl_cmd(cmd)) + return -EBADF; + } + + err = security_file_fcntl(fd_file(f), cmd, arg); + if (err) + return err; + + switch (cmd) { + case F_GETLK64: + case F_OFD_GETLK: + err = -EFAULT; + if (copy_from_user(&flock, argp, sizeof(flock))) + break; + err = fcntl_getlk64(fd_file(f), cmd, &flock); + if (!err && copy_to_user(argp, &flock, sizeof(flock))) + err = -EFAULT; + break; + case F_SETLK64: + case F_SETLKW64: + case F_OFD_SETLK: + case F_OFD_SETLKW: + err = -EFAULT; + if (copy_from_user(&flock, argp, sizeof(flock))) + break; + err = fcntl_setlk64(fd, fd_file(f), cmd, &flock); + break; + default: + err = do_fcntl(fd, cmd, arg, fd_file(f)); + break; + } + return err; +} +#endif + +#ifdef CONFIG_COMPAT +/* careful - don't use anywhere else */ +#define copy_flock_fields(dst, src) \ + (dst)->l_type = (src)->l_type; \ + (dst)->l_whence = (src)->l_whence; \ + (dst)->l_start = (src)->l_start; \ + (dst)->l_len = (src)->l_len; \ + (dst)->l_pid = (src)->l_pid; + +static int get_compat_flock(struct flock *kfl, const struct compat_flock __user *ufl) +{ + struct compat_flock fl; + + if (copy_from_user(&fl, ufl, sizeof(struct compat_flock))) + return -EFAULT; + copy_flock_fields(kfl, &fl); + return 0; +} + +static int get_compat_flock64(struct flock *kfl, const struct compat_flock64 __user *ufl) +{ + struct compat_flock64 fl; + + if (copy_from_user(&fl, ufl, sizeof(struct compat_flock64))) + return -EFAULT; + copy_flock_fields(kfl, &fl); + return 0; +} + +static int put_compat_flock(const struct flock *kfl, struct compat_flock __user *ufl) +{ + struct compat_flock fl; + + memset(&fl, 0, sizeof(struct compat_flock)); + copy_flock_fields(&fl, kfl); + if (copy_to_user(ufl, &fl, sizeof(struct compat_flock))) + return -EFAULT; + return 0; +} + +static int put_compat_flock64(const struct flock *kfl, struct compat_flock64 __user *ufl) +{ + struct compat_flock64 fl; + + BUILD_BUG_ON(sizeof(kfl->l_start) > sizeof(ufl->l_start)); + BUILD_BUG_ON(sizeof(kfl->l_len) > sizeof(ufl->l_len)); + + memset(&fl, 0, sizeof(struct compat_flock64)); + copy_flock_fields(&fl, kfl); + if (copy_to_user(ufl, &fl, sizeof(struct compat_flock64))) + return -EFAULT; + return 0; +} +#undef copy_flock_fields + +static unsigned int +convert_fcntl_cmd(unsigned int cmd) +{ + switch (cmd) { + case F_GETLK64: + return F_GETLK; + case F_SETLK64: + return F_SETLK; + case F_SETLKW64: + return F_SETLKW; + } + + return cmd; +} + +/* + * GETLK was successful and we need to return the data, but it needs to fit in + * the compat structure. + * l_start shouldn't be too big, unless the original start + end is greater than + * COMPAT_OFF_T_MAX, in which case the app was asking for trouble, so we return + * -EOVERFLOW in that case. l_len could be too big, in which case we just + * truncate it, and only allow the app to see that part of the conflicting lock + * that might make sense to it anyway + */ +static int fixup_compat_flock(struct flock *flock) +{ + if (flock->l_start > COMPAT_OFF_T_MAX) + return -EOVERFLOW; + if (flock->l_len > COMPAT_OFF_T_MAX) + flock->l_len = COMPAT_OFF_T_MAX; + return 0; +} + +static long do_compat_fcntl64(unsigned int fd, unsigned int cmd, + compat_ulong_t arg) +{ + CLASS(fd_raw, f)(fd); + struct flock flock; + long err; + + if (fd_empty(f)) + return -EBADF; + + if (unlikely(fd_file(f)->f_mode & FMODE_PATH)) { + if (!check_fcntl_cmd(cmd)) + return -EBADF; + } + + err = security_file_fcntl(fd_file(f), cmd, arg); + if (err) + return err; + + switch (cmd) { + case F_GETLK: + err = get_compat_flock(&flock, compat_ptr(arg)); + if (err) + break; + err = fcntl_getlk(fd_file(f), convert_fcntl_cmd(cmd), &flock); + if (err) + break; + err = fixup_compat_flock(&flock); + if (!err) + err = put_compat_flock(&flock, compat_ptr(arg)); + break; + case F_GETLK64: + case F_OFD_GETLK: + err = get_compat_flock64(&flock, compat_ptr(arg)); + if (err) + break; + err = fcntl_getlk(fd_file(f), convert_fcntl_cmd(cmd), &flock); + if (!err) + err = put_compat_flock64(&flock, compat_ptr(arg)); + break; + case F_SETLK: + case F_SETLKW: + err = get_compat_flock(&flock, compat_ptr(arg)); + if (err) + break; + err = fcntl_setlk(fd, fd_file(f), convert_fcntl_cmd(cmd), &flock); + break; + case F_SETLK64: + case F_SETLKW64: + case F_OFD_SETLK: + case F_OFD_SETLKW: + err = get_compat_flock64(&flock, compat_ptr(arg)); + if (err) + break; + err = fcntl_setlk(fd, fd_file(f), convert_fcntl_cmd(cmd), &flock); + break; + default: + err = do_fcntl(fd, cmd, arg, fd_file(f)); + break; + } + return err; +} + +COMPAT_SYSCALL_DEFINE3(fcntl64, unsigned int, fd, unsigned int, cmd, + compat_ulong_t, arg) +{ + return do_compat_fcntl64(fd, cmd, arg); +} + +COMPAT_SYSCALL_DEFINE3(fcntl, unsigned int, fd, unsigned int, cmd, + compat_ulong_t, arg) +{ + switch (cmd) { + case F_GETLK64: + case F_SETLK64: + case F_SETLKW64: + case F_OFD_GETLK: + case F_OFD_SETLK: + case F_OFD_SETLKW: + return -EINVAL; + } + return do_compat_fcntl64(fd, cmd, arg); +} +#endif + +/* Table to convert sigio signal codes into poll band bitmaps */ + +static const __poll_t band_table[NSIGPOLL] = { + EPOLLIN | EPOLLRDNORM, /* POLL_IN */ + EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND, /* POLL_OUT */ + EPOLLIN | EPOLLRDNORM | EPOLLMSG, /* POLL_MSG */ + EPOLLERR, /* POLL_ERR */ + EPOLLPRI | EPOLLRDBAND, /* POLL_PRI */ + EPOLLHUP | EPOLLERR /* POLL_HUP */ +}; + +static inline int sigio_perm(struct task_struct *p, + struct fown_struct *fown, int sig) +{ + const struct cred *cred; + int ret; + + rcu_read_lock(); + cred = __task_cred(p); + ret = ((uid_eq(fown->euid, GLOBAL_ROOT_UID) || + uid_eq(fown->euid, cred->suid) || uid_eq(fown->euid, cred->uid) || + uid_eq(fown->uid, cred->suid) || uid_eq(fown->uid, cred->uid)) && + !security_file_send_sigiotask(p, fown, sig)); + rcu_read_unlock(); + return ret; +} + +static void send_sigio_to_task(struct task_struct *p, + struct fown_struct *fown, + int fd, int reason, enum pid_type type) +{ + /* + * F_SETSIG can change ->signum lockless in parallel, make + * sure we read it once and use the same value throughout. + */ + int signum = READ_ONCE(fown->signum); + + if (!sigio_perm(p, fown, signum)) + return; + + switch (signum) { + default: { + kernel_siginfo_t si; + + /* Queue a rt signal with the appropriate fd as its + value. We use SI_SIGIO as the source, not + SI_KERNEL, since kernel signals always get + delivered even if we can't queue. Failure to + queue in this case _should_ be reported; we fall + back to SIGIO in that case. --sct */ + clear_siginfo(&si); + si.si_signo = signum; + si.si_errno = 0; + si.si_code = reason; + /* + * Posix definies POLL_IN and friends to be signal + * specific si_codes for SIG_POLL. Linux extended + * these si_codes to other signals in a way that is + * ambiguous if other signals also have signal + * specific si_codes. In that case use SI_SIGIO instead + * to remove the ambiguity. + */ + if ((signum != SIGPOLL) && sig_specific_sicodes(signum)) + si.si_code = SI_SIGIO; + + /* Make sure we are called with one of the POLL_* + reasons, otherwise we could leak kernel stack into + userspace. */ + BUG_ON((reason < POLL_IN) || ((reason - POLL_IN) >= NSIGPOLL)); + if (reason - POLL_IN >= NSIGPOLL) + si.si_band = ~0L; + else + si.si_band = mangle_poll(band_table[reason - POLL_IN]); + si.si_fd = fd; + if (!do_send_sig_info(signum, &si, p, type)) + break; + } + fallthrough; /* fall back on the old plain SIGIO signal */ + case 0: + do_send_sig_info(SIGIO, SEND_SIG_PRIV, p, type); + } +} + +void send_sigio(struct fown_struct *fown, int fd, int band) +{ + struct task_struct *p; + enum pid_type type; + unsigned long flags; + struct pid *pid; + + read_lock_irqsave(&fown->lock, flags); + + type = fown->pid_type; + pid = fown->pid; + if (!pid) + goto out_unlock_fown; + + if (type <= PIDTYPE_TGID) { + rcu_read_lock(); + p = pid_task(pid, PIDTYPE_PID); + if (p) + send_sigio_to_task(p, fown, fd, band, type); + rcu_read_unlock(); + } else { + rcu_read_lock(); + do_each_pid_task(pid, type, p) { + send_sigio_to_task(p, fown, fd, band, type); + } while_each_pid_task(pid, type, p); + rcu_read_unlock(); + } + out_unlock_fown: + read_unlock_irqrestore(&fown->lock, flags); +} + +static void send_sigurg_to_task(struct task_struct *p, + struct fown_struct *fown, enum pid_type type) +{ + if (sigio_perm(p, fown, SIGURG)) + do_send_sig_info(SIGURG, SEND_SIG_PRIV, p, type); +} + +int send_sigurg(struct file *file) +{ + struct fown_struct *fown; + struct task_struct *p; + enum pid_type type; + struct pid *pid; + unsigned long flags; + int ret = 0; + + fown = file_f_owner(file); + if (!fown) + return 0; + + read_lock_irqsave(&fown->lock, flags); + + type = fown->pid_type; + pid = fown->pid; + if (!pid) + goto out_unlock_fown; + + ret = 1; + + if (type <= PIDTYPE_TGID) { + rcu_read_lock(); + p = pid_task(pid, PIDTYPE_PID); + if (p) + send_sigurg_to_task(p, fown, type); + rcu_read_unlock(); + } else { + rcu_read_lock(); + do_each_pid_task(pid, type, p) { + send_sigurg_to_task(p, fown, type); + } while_each_pid_task(pid, type, p); + rcu_read_unlock(); + } + out_unlock_fown: + read_unlock_irqrestore(&fown->lock, flags); + return ret; +} + +static DEFINE_SPINLOCK(fasync_lock); +static struct kmem_cache *fasync_cache __ro_after_init; + +/* + * Remove a fasync entry. If successfully removed, return + * positive and clear the FASYNC flag. If no entry exists, + * do nothing and return 0. + * + * NOTE! It is very important that the FASYNC flag always + * match the state "is the filp on a fasync list". + * + */ +int fasync_remove_entry(struct file *filp, struct fasync_struct **fapp) +{ + struct fasync_struct *fa, **fp; + int result = 0; + + spin_lock(&filp->f_lock); + spin_lock(&fasync_lock); + for (fp = fapp; (fa = *fp) != NULL; fp = &fa->fa_next) { + if (fa->fa_file != filp) + continue; + + write_lock_irq(&fa->fa_lock); + fa->fa_file = NULL; + write_unlock_irq(&fa->fa_lock); + + *fp = fa->fa_next; + kfree_rcu(fa, fa_rcu); + filp->f_flags &= ~FASYNC; + result = 1; + break; + } + spin_unlock(&fasync_lock); + spin_unlock(&filp->f_lock); + return result; +} + +struct fasync_struct *fasync_alloc(void) +{ + return kmem_cache_alloc(fasync_cache, GFP_KERNEL); +} + +/* + * NOTE! This can be used only for unused fasync entries: + * entries that actually got inserted on the fasync list + * need to be released by rcu - see fasync_remove_entry. + */ +void fasync_free(struct fasync_struct *new) +{ + kmem_cache_free(fasync_cache, new); +} + +/* + * Insert a new entry into the fasync list. Return the pointer to the + * old one if we didn't use the new one. + * + * NOTE! It is very important that the FASYNC flag always + * match the state "is the filp on a fasync list". + */ +struct fasync_struct *fasync_insert_entry(int fd, struct file *filp, struct fasync_struct **fapp, struct fasync_struct *new) +{ + struct fasync_struct *fa, **fp; + + spin_lock(&filp->f_lock); + spin_lock(&fasync_lock); + for (fp = fapp; (fa = *fp) != NULL; fp = &fa->fa_next) { + if (fa->fa_file != filp) + continue; + + write_lock_irq(&fa->fa_lock); + fa->fa_fd = fd; + write_unlock_irq(&fa->fa_lock); + goto out; + } + + rwlock_init(&new->fa_lock); + new->magic = FASYNC_MAGIC; + new->fa_file = filp; + new->fa_fd = fd; + new->fa_next = *fapp; + rcu_assign_pointer(*fapp, new); + filp->f_flags |= FASYNC; + +out: + spin_unlock(&fasync_lock); + spin_unlock(&filp->f_lock); + return fa; +} + +/* + * Add a fasync entry. Return negative on error, positive if + * added, and zero if did nothing but change an existing one. + */ +static int fasync_add_entry(int fd, struct file *filp, struct fasync_struct **fapp) +{ + struct fasync_struct *new; + + new = fasync_alloc(); + if (!new) + return -ENOMEM; + + /* + * fasync_insert_entry() returns the old (update) entry if + * it existed. + * + * So free the (unused) new entry and return 0 to let the + * caller know that we didn't add any new fasync entries. + */ + if (fasync_insert_entry(fd, filp, fapp, new)) { + fasync_free(new); + return 0; + } + + return 1; +} + +/* + * fasync_helper() is used by almost all character device drivers + * to set up the fasync queue, and for regular files by the file + * lease code. It returns negative on error, 0 if it did no changes + * and positive if it added/deleted the entry. + */ +int fasync_helper(int fd, struct file * filp, int on, struct fasync_struct **fapp) +{ + if (!on) + return fasync_remove_entry(filp, fapp); + return fasync_add_entry(fd, filp, fapp); +} + +EXPORT_SYMBOL(fasync_helper); + +/* + * rcu_read_lock() is held + */ +static void kill_fasync_rcu(struct fasync_struct *fa, int sig, int band) +{ + while (fa) { + struct fown_struct *fown; + unsigned long flags; + + if (fa->magic != FASYNC_MAGIC) { + printk(KERN_ERR "kill_fasync: bad magic number in " + "fasync_struct!\n"); + return; + } + read_lock_irqsave(&fa->fa_lock, flags); + if (fa->fa_file) { + fown = file_f_owner(fa->fa_file); + if (!fown) + goto next; + /* Don't send SIGURG to processes which have not set a + queued signum: SIGURG has its own default signalling + mechanism. */ + if (!(sig == SIGURG && fown->signum == 0)) + send_sigio(fown, fa->fa_fd, band); + } +next: + read_unlock_irqrestore(&fa->fa_lock, flags); + fa = rcu_dereference(fa->fa_next); + } +} + +void kill_fasync(struct fasync_struct **fp, int sig, int band) +{ + /* First a quick test without locking: usually + * the list is empty. + */ + if (*fp) { + rcu_read_lock(); + kill_fasync_rcu(rcu_dereference(*fp), sig, band); + rcu_read_unlock(); + } +} +EXPORT_SYMBOL(kill_fasync); + +static int __init fcntl_init(void) +{ + /* + * Please add new bits here to ensure allocation uniqueness. + * Exceptions: O_NONBLOCK is a two bit define on parisc; O_NDELAY + * is defined as O_NONBLOCK on some platforms and not on others. + */ + BUILD_BUG_ON(22 - 1 /* for O_RDONLY being 0 */ != + HWEIGHT32( + (VALID_OPEN_FLAGS & ~(O_NONBLOCK | O_NDELAY)) | + __FMODE_EXEC | __O_REGULAR)); + + fasync_cache = kmem_cache_create("fasync_cache", + sizeof(struct fasync_struct), 0, + SLAB_PANIC | SLAB_ACCOUNT, NULL); + return 0; +} + +module_init(fcntl_init) diff --git a/fs/fhandle.c b/fs/fhandle.c new file mode 100644 index 000000000..f8829231e --- /dev/null +++ b/fs/fhandle.c @@ -0,0 +1,474 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" +#include "mount.h" + +static long do_sys_name_to_handle(const struct path *path, + struct file_handle __user *ufh, + void __user *mnt_id, bool unique_mntid, + int fh_flags) +{ + long retval; + struct file_handle f_handle; + int handle_dwords, handle_bytes; + struct file_handle *handle = NULL; + + /* + * We need to make sure whether the file system support decoding of + * the file handle if decodeable file handle was requested. + */ + if (!exportfs_can_encode_fh(path->dentry->d_sb->s_export_op, fh_flags)) + return -EOPNOTSUPP; + + /* + * A request to encode a connectable handle for a disconnected dentry + * is unexpected since AT_EMPTY_PATH is not allowed. + */ + if (fh_flags & EXPORT_FH_CONNECTABLE && + WARN_ON(path->dentry->d_flags & DCACHE_DISCONNECTED)) + return -EINVAL; + + if (copy_from_user(&f_handle, ufh, sizeof(struct file_handle))) + return -EFAULT; + + if (f_handle.handle_bytes > MAX_HANDLE_SZ) + return -EINVAL; + + handle = kzalloc_flex(*handle, f_handle, f_handle.handle_bytes); + if (!handle) + return -ENOMEM; + + /* convert handle size to multiple of sizeof(u32) */ + handle_dwords = f_handle.handle_bytes >> 2; + + /* Encode a possibly decodeable/connectable file handle */ + retval = exportfs_encode_fh(path->dentry, + (struct fid *)handle->f_handle, + &handle_dwords, fh_flags); + handle->handle_type = retval; + /* convert handle size to bytes */ + handle_bytes = handle_dwords * sizeof(u32); + handle->handle_bytes = handle_bytes; + if ((handle->handle_bytes > f_handle.handle_bytes) || + (retval == FILEID_INVALID) || (retval < 0)) { + /* As per old exportfs_encode_fh documentation + * we could return ENOSPC to indicate overflow + * But file system returned 255 always. So handle + * both the values + */ + if (retval == FILEID_INVALID || retval == -ENOSPC) + retval = -EOVERFLOW; + /* + * set the handle size to zero so we copy only + * non variable part of the file_handle + */ + handle_bytes = 0; + } else { + /* + * When asked to encode a connectable file handle, encode this + * property in the file handle itself, so that we later know + * how to decode it. + * For sanity, also encode in the file handle if the encoded + * object is a directory and verify this during decode, because + * decoding directory file handles is quite different than + * decoding connectable non-directory file handles. + */ + if (fh_flags & EXPORT_FH_CONNECTABLE) { + handle->handle_type |= FILEID_IS_CONNECTABLE; + if (d_is_dir(path->dentry)) + handle->handle_type |= FILEID_IS_DIR; + } + retval = 0; + } + /* copy the mount id */ + if (unique_mntid) { + if (put_user(real_mount(path->mnt)->mnt_id_unique, + (u64 __user *) mnt_id)) + retval = -EFAULT; + } else { + if (put_user(real_mount(path->mnt)->mnt_id, + (int __user *) mnt_id)) + retval = -EFAULT; + } + /* copy the handle */ + if (retval != -EFAULT && + copy_to_user(ufh, handle, + struct_size(handle, f_handle, handle_bytes))) + retval = -EFAULT; + kfree(handle); + return retval; +} + +/** + * sys_name_to_handle_at: convert name to handle + * @dfd: directory relative to which name is interpreted if not absolute + * @name: name that should be converted to handle. + * @handle: resulting file handle + * @mnt_id: mount id of the file system containing the file + * (u64 if AT_HANDLE_MNT_ID_UNIQUE, otherwise int) + * @flag: flag value to indicate whether to follow symlink or not + * and whether a decodable file handle is required. + * + * @handle->handle_size indicate the space available to store the + * variable part of the file handle in bytes. If there is not + * enough space, the field is updated to return the minimum + * value required. + */ +SYSCALL_DEFINE5(name_to_handle_at, int, dfd, const char __user *, name, + struct file_handle __user *, handle, void __user *, mnt_id, + int, flag) +{ + struct path path; + int lookup_flags; + int fh_flags = 0; + int err; + + if (flag & ~(AT_SYMLINK_FOLLOW | AT_EMPTY_PATH | AT_HANDLE_FID | + AT_HANDLE_MNT_ID_UNIQUE | AT_HANDLE_CONNECTABLE)) + return -EINVAL; + + /* + * AT_HANDLE_FID means there is no intention to decode file handle + * AT_HANDLE_CONNECTABLE means there is an intention to decode a + * connected fd (with known path), so these flags are conflicting. + * AT_EMPTY_PATH could be used along with a dfd that refers to a + * disconnected non-directory, which cannot be used to encode a + * connectable file handle, because its parent is unknown. + */ + if (flag & AT_HANDLE_CONNECTABLE && + flag & (AT_HANDLE_FID | AT_EMPTY_PATH)) + return -EINVAL; + else if (flag & AT_HANDLE_FID) + fh_flags |= EXPORT_FH_FID; + else if (flag & AT_HANDLE_CONNECTABLE) + fh_flags |= EXPORT_FH_CONNECTABLE; + + lookup_flags = (flag & AT_SYMLINK_FOLLOW) ? LOOKUP_FOLLOW : 0; + CLASS(filename_uflags, filename)(name, flag); + err = filename_lookup(dfd, filename, lookup_flags, &path, NULL); + if (!err) { + err = do_sys_name_to_handle(&path, handle, mnt_id, + flag & AT_HANDLE_MNT_ID_UNIQUE, + fh_flags); + path_put(&path); + } + return err; +} + +static int get_path_anchor(int fd, struct path *root) +{ + if (fd >= 0) { + CLASS(fd, f)(fd); + if (fd_empty(f)) + return -EBADF; + *root = fd_file(f)->f_path; + path_get(root); + return 0; + } + + if (fd == AT_FDCWD) { + get_fs_pwd(current->fs, root); + return 0; + } + + if (fd == FD_PIDFS_ROOT) { + pidfs_get_root(root); + return 0; + } + + if (fd == FD_NSFS_ROOT) { + nsfs_get_root(root); + return 0; + } + + return -EBADF; +} + +static int vfs_dentry_acceptable(void *context, struct dentry *dentry) +{ + struct handle_to_path_ctx *ctx = context; + struct user_namespace *user_ns = current_user_ns(); + struct dentry *d, *root = ctx->root.dentry; + struct mnt_idmap *idmap = mnt_idmap(ctx->root.mnt); + int retval = 0; + + if (!root) + return 1; + + /* Old permission model with global CAP_DAC_READ_SEARCH. */ + if (!ctx->flags) + return 1; + + /* + * Verify that the decoded dentry itself has a valid id mapping. + * In case the decoded dentry is the mountfd root itself, this + * verifies that the mountfd inode itself has a valid id mapping. + */ + if (!privileged_wrt_inode_uidgid(user_ns, idmap, d_inode(dentry))) + return 0; + + /* + * It's racy as we're not taking rename_lock but we're able to ignore + * permissions and we just need an approximation whether we were able + * to follow a path to the file. + * + * It's also potentially expensive on some filesystems especially if + * there is a deep path. + */ + d = dget(dentry); + while (d != root && !IS_ROOT(d)) { + struct dentry *parent = dget_parent(d); + + /* + * We know that we have the ability to override DAC permissions + * as we've verified this earlier via CAP_DAC_READ_SEARCH. But + * we also need to make sure that there aren't any unmapped + * inodes in the path that would prevent us from reaching the + * file. + */ + if (!privileged_wrt_inode_uidgid(user_ns, idmap, + d_inode(parent))) { + dput(d); + dput(parent); + return retval; + } + + dput(d); + d = parent; + } + + if (!(ctx->flags & HANDLE_CHECK_SUBTREE) || d == root) + retval = 1; + /* + * exportfs_decode_fh_raw() does not call acceptable() callback with + * a disconnected directory dentry, so we should have reached either + * mount fd directory or sb root. + */ + if (ctx->fh_flags & EXPORT_FH_DIR_ONLY) + WARN_ON_ONCE(d != root && d != root->d_sb->s_root); + dput(d); + return retval; +} + +static int do_handle_to_path(struct file_handle *handle, struct path *path, + struct handle_to_path_ctx *ctx) +{ + int handle_dwords; + struct vfsmount *mnt = ctx->root.mnt; + struct dentry *dentry; + + /* change the handle size to multiple of sizeof(u32) */ + handle_dwords = handle->handle_bytes >> 2; + dentry = exportfs_decode_fh_raw(mnt, (struct fid *)handle->f_handle, + handle_dwords, handle->handle_type, + ctx->fh_flags, vfs_dentry_acceptable, + ctx); + if (IS_ERR_OR_NULL(dentry)) { + if (dentry == ERR_PTR(-ENOMEM)) + return -ENOMEM; + return -ESTALE; + } + path->dentry = dentry; + path->mnt = mntget(mnt); + return 0; +} + +static bool capable_wrt_mount(struct mount *mount) +{ + struct mnt_namespace *mnt_ns; + + /* + * For ->mnt_ns access. + * The following READ_ONCE() is semantically rcu_dereference(). + */ + guard(rcu)(); + mnt_ns = READ_ONCE(mount->mnt_ns); + return mnt_ns && ns_capable(mnt_ns->user_ns, CAP_SYS_ADMIN); +} + +static inline int may_decode_fh(struct handle_to_path_ctx *ctx, + unsigned int o_flags) +{ + struct path *root = &ctx->root; + + if (capable(CAP_DAC_READ_SEARCH)) + return 0; + + /* + * Allow relaxed permissions of file handles if the caller has + * the ability to mount the filesystem or create a bind-mount of + * the provided @mountdirfd. + * + * In both cases the caller may be able to get an unobstructed + * way to the encoded file handle. If the caller is only able to + * create a bind-mount we need to verify that there are no + * locked mounts on top of it that could prevent us from getting + * to the encoded file. + * + * In principle, locked mounts can prevent the caller from + * mounting the filesystem but that only applies to procfs and + * sysfs neither of which support decoding file handles. + * + * Restrict to O_DIRECTORY to provide a deterministic API that + * avoids a confusing api in the face of disconnected non-dir + * dentries. + * + * There's only one dentry for each directory inode (VFS rule)... + */ + if (!(o_flags & O_DIRECTORY)) + return -EPERM; + + if (ns_capable(root->mnt->mnt_sb->s_user_ns, CAP_SYS_ADMIN)) + ctx->flags = HANDLE_CHECK_PERMS; + else if (is_mounted(root->mnt) && + capable_wrt_mount(real_mount(root->mnt)) && + !has_locked_children(real_mount(root->mnt), root->dentry)) + ctx->flags = HANDLE_CHECK_PERMS | HANDLE_CHECK_SUBTREE; + else + return -EPERM; + + /* Are we able to override DAC permissions? */ + if (!ns_capable(current_user_ns(), CAP_DAC_READ_SEARCH)) + return -EPERM; + + ctx->fh_flags = EXPORT_FH_DIR_ONLY; + return 0; +} + +static int handle_to_path(int mountdirfd, struct file_handle __user *ufh, + struct path *path, unsigned int o_flags) +{ + int retval = 0; + struct file_handle f_handle; + struct file_handle *handle __free(kfree) = NULL; + struct handle_to_path_ctx ctx = {}; + const struct export_operations *eops; + + if (copy_from_user(&f_handle, ufh, sizeof(struct file_handle))) + return -EFAULT; + + if ((f_handle.handle_bytes > MAX_HANDLE_SZ) || + (f_handle.handle_bytes == 0)) + return -EINVAL; + + if (f_handle.handle_type < 0 || + FILEID_USER_FLAGS(f_handle.handle_type) & ~FILEID_VALID_USER_FLAGS) + return -EINVAL; + + retval = get_path_anchor(mountdirfd, &ctx.root); + if (retval) + return retval; + + eops = ctx.root.mnt->mnt_sb->s_export_op; + if (eops && eops->permission) + retval = eops->permission(&ctx, o_flags); + else + retval = may_decode_fh(&ctx, o_flags); + if (retval) + goto out_path; + + handle = kmalloc_flex(*handle, f_handle, f_handle.handle_bytes); + if (!handle) { + retval = -ENOMEM; + goto out_path; + } + /* copy the full handle */ + *handle = f_handle; + if (copy_from_user(&handle->f_handle, + &ufh->f_handle, + f_handle.handle_bytes)) { + retval = -EFAULT; + goto out_path; + } + + /* + * If handle was encoded with AT_HANDLE_CONNECTABLE, verify that we + * are decoding an fd with connected path, which is accessible from + * the mount fd path. + */ + if (f_handle.handle_type & FILEID_IS_CONNECTABLE) { + ctx.fh_flags |= EXPORT_FH_CONNECTABLE; + ctx.flags |= HANDLE_CHECK_SUBTREE; + } + if (f_handle.handle_type & FILEID_IS_DIR) + ctx.fh_flags |= EXPORT_FH_DIR_ONLY; + /* Filesystem code should not be exposed to user flags */ + handle->handle_type &= ~FILEID_USER_FLAGS_MASK; + retval = do_handle_to_path(handle, path, &ctx); + +out_path: + path_put(&ctx.root); + return retval; +} + +static struct file *file_open_handle(struct path *path, int open_flag) +{ + const struct export_operations *eops; + + eops = path->mnt->mnt_sb->s_export_op; + if (eops->open) + return eops->open(path, open_flag); + + return file_open_root(path, "", open_flag, 0); +} + +static long do_handle_open(int mountdirfd, struct file_handle __user *ufh, + int open_flag) +{ + long retval; + struct path path __free(path_put) = {}; + + retval = handle_to_path(mountdirfd, ufh, &path, open_flag); + if (retval) + return retval; + + return FD_ADD(open_flag, file_open_handle(&path, open_flag)); +} + +/** + * sys_open_by_handle_at: Open the file handle + * @mountdirfd: directory file descriptor + * @handle: file handle to be opened + * @flags: open flags. + * + * @mountdirfd indicate the directory file descriptor + * of the mount point. file handle is decoded relative + * to the vfsmount pointed by the @mountdirfd. @flags + * value is same as the open(2) flags. + */ +SYSCALL_DEFINE3(open_by_handle_at, int, mountdirfd, + struct file_handle __user *, handle, + int, flags) +{ + long ret; + + if (force_o_largefile()) + flags |= O_LARGEFILE; + + ret = do_handle_open(mountdirfd, handle, flags); + return ret; +} + +#ifdef CONFIG_COMPAT +/* + * Exactly like fs/open.c:sys_open_by_handle_at(), except that it + * doesn't set the O_LARGEFILE flag. + */ +COMPAT_SYSCALL_DEFINE3(open_by_handle_at, int, mountdirfd, + struct file_handle __user *, handle, int, flags) +{ + return do_handle_open(mountdirfd, handle, flags); +} +#endif diff --git a/fs/file.c b/fs/file.c new file mode 100644 index 000000000..628ca07dc --- /dev/null +++ b/fs/file.c @@ -0,0 +1,1531 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/file.c + * + * Copyright (C) 1998-1999, Stephen Tweedie and Bill Hawes + * + * Manage the dynamic fd arrays in the process files_struct. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +static noinline bool __file_ref_put_badval(file_ref_t *ref, unsigned long cnt) +{ + /* + * If the reference count was already in the dead zone, then this + * put() operation is imbalanced. Warn, put the reference count back to + * DEAD and tell the caller to not deconstruct the object. + */ + if (WARN_ONCE(cnt >= FILE_REF_RELEASED, "imbalanced put on file reference count")) { + atomic_long_set(&ref->refcnt, FILE_REF_DEAD); + return false; + } + + /* + * This is a put() operation on a saturated refcount. Restore the + * mean saturation value and tell the caller to not deconstruct the + * object. + */ + if (cnt > FILE_REF_MAXREF) + atomic_long_set(&ref->refcnt, FILE_REF_SATURATED); + return false; +} + +/** + * __file_ref_put - Slowpath of file_ref_put() + * @ref: Pointer to the reference count + * @cnt: Current reference count + * + * Invoked when the reference count is outside of the valid zone. + * + * Return: + * True if this was the last reference with no future references + * possible. This signals the caller that it can safely schedule the + * object, which is protected by the reference counter, for + * deconstruction. + * + * False if there are still active references or the put() raced + * with a concurrent get()/put() pair. Caller is not allowed to + * deconstruct the protected object. + */ +bool __file_ref_put(file_ref_t *ref, unsigned long cnt) +{ + /* Did this drop the last reference? */ + if (likely(cnt == FILE_REF_NOREF)) { + /* + * Carefully try to set the reference count to FILE_REF_DEAD. + * + * This can fail if a concurrent get() operation has + * elevated it again or the corresponding put() even marked + * it dead already. Both are valid situations and do not + * require a retry. If this fails the caller is not + * allowed to deconstruct the object. + */ + if (!atomic_long_try_cmpxchg_release(&ref->refcnt, &cnt, FILE_REF_DEAD)) + return false; + + /* + * The caller can safely schedule the object for + * deconstruction. Provide acquire ordering. + */ + smp_acquire__after_ctrl_dep(); + return true; + } + + return __file_ref_put_badval(ref, cnt); +} +EXPORT_SYMBOL_GPL(__file_ref_put); + +unsigned int sysctl_nr_open __read_mostly = 1024*1024; +unsigned int sysctl_nr_open_min = BITS_PER_LONG; +/* our min() is unusable in constant expressions ;-/ */ +#define __const_min(x, y) ((x) < (y) ? (x) : (y)) +unsigned int sysctl_nr_open_max = + __const_min(INT_MAX, ~(size_t)0/sizeof(void *)) & -BITS_PER_LONG; + +static void __free_fdtable(struct fdtable *fdt) +{ + kvfree(fdt->fd); + kvfree(fdt->open_fds); + kfree(fdt); +} + +static void free_fdtable_rcu(struct rcu_head *rcu) +{ + __free_fdtable(container_of(rcu, struct fdtable, rcu)); +} + +#define BITBIT_NR(nr) BITS_TO_LONGS(BITS_TO_LONGS(nr)) +#define BITBIT_SIZE(nr) (BITBIT_NR(nr) * sizeof(long)) + +#define fdt_words(fdt) ((fdt)->max_fds / BITS_PER_LONG) // words in ->open_fds +/* + * Copy 'count' fd bits from the old table to the new table and clear the extra + * space if any. This does not copy the file pointers. Called with the files + * spinlock held for write. + */ +static inline void copy_fd_bitmaps(struct fdtable *nfdt, struct fdtable *ofdt, + unsigned int copy_words) +{ + unsigned int nwords = fdt_words(nfdt); + + bitmap_copy_and_extend(nfdt->open_fds, ofdt->open_fds, + copy_words * BITS_PER_LONG, nwords * BITS_PER_LONG); + bitmap_copy_and_extend(nfdt->close_on_exec, ofdt->close_on_exec, + copy_words * BITS_PER_LONG, nwords * BITS_PER_LONG); + bitmap_copy_and_extend(nfdt->full_fds_bits, ofdt->full_fds_bits, + copy_words, nwords); +} + +/* + * Copy all file descriptors from the old table to the new, expanded table and + * clear the extra space. Called with the files spinlock held for write. + */ +static void copy_fdtable(struct fdtable *nfdt, struct fdtable *ofdt) +{ + size_t cpy, set; + + BUG_ON(nfdt->max_fds < ofdt->max_fds); + + cpy = ofdt->max_fds * sizeof(struct file *); + set = (nfdt->max_fds - ofdt->max_fds) * sizeof(struct file *); + memcpy(nfdt->fd, ofdt->fd, cpy); + memset((char *)nfdt->fd + cpy, 0, set); + + copy_fd_bitmaps(nfdt, ofdt, fdt_words(ofdt)); +} + +/* + * Note how the fdtable bitmap allocations very much have to be a multiple of + * BITS_PER_LONG. This is not only because we walk those things in chunks of + * 'unsigned long' in some places, but simply because that is how the Linux + * kernel bitmaps are defined to work: they are not "bits in an array of bytes", + * they are very much "bits in an array of unsigned long". + */ +static struct fdtable *alloc_fdtable(unsigned int slots_wanted) +{ + struct fdtable *fdt; + unsigned int nr; + void *data; + + /* + * Figure out how many fds we actually want to support in this fdtable. + * Allocation steps are keyed to the size of the fdarray, since it + * grows far faster than any of the other dynamic data. We try to fit + * the fdarray into comfortable page-tuned chunks: starting at 1024B + * and growing in powers of two from there on. Since we called only + * with slots_wanted > BITS_PER_LONG (embedded instance in files->fdtab + * already gives BITS_PER_LONG slots), the above boils down to + * 1. use the smallest power of two large enough to give us that many + * slots. + * 2. on 32bit skip 64 and 128 - the minimal capacity we want there is + * 256 slots (i.e. 1Kb fd array). + * 3. on 64bit don't skip anything, 1Kb fd array means 128 slots there + * and we are never going to be asked for 64 or less. + */ + if (IS_ENABLED(CONFIG_32BIT) && slots_wanted < 256) + nr = 256; + else + nr = roundup_pow_of_two(slots_wanted); + /* + * Note that this can drive nr *below* what we had passed if sysctl_nr_open + * had been set lower between the check in expand_files() and here. + * + * We make sure that nr remains a multiple of BITS_PER_LONG - otherwise + * bitmaps handling below becomes unpleasant, to put it mildly... + */ + if (unlikely(nr > sysctl_nr_open)) { + nr = round_down(sysctl_nr_open, BITS_PER_LONG); + if (nr < slots_wanted) + return ERR_PTR(-EMFILE); + } + + /* + * Check if the allocation size would exceed INT_MAX. kvmalloc_array() + * and kvmalloc() will warn if the allocation size is greater than + * INT_MAX, as filp_cache objects are not __GFP_NOWARN. + * + * This can happen when sysctl_nr_open is set to a very high value and + * a process tries to use a file descriptor near that limit. For example, + * if sysctl_nr_open is set to 1073741816 (0x3ffffff8) - which is what + * systemd typically sets it to - then trying to use a file descriptor + * close to that value will require allocating a file descriptor table + * that exceeds 8GB in size. + */ + if (unlikely(nr > INT_MAX / sizeof(struct file *))) + return ERR_PTR(-EMFILE); + + fdt = kmalloc_obj(struct fdtable, GFP_KERNEL_ACCOUNT); + if (!fdt) + goto out; + fdt->max_fds = nr; + data = kvmalloc_objs(struct file *, nr, GFP_KERNEL_ACCOUNT); + if (!data) + goto out_fdt; + fdt->fd = data; + + data = kvmalloc(max_t(size_t, + 2 * nr / BITS_PER_BYTE + BITBIT_SIZE(nr), L1_CACHE_BYTES), + GFP_KERNEL_ACCOUNT); + if (!data) + goto out_arr; + fdt->open_fds = data; + data += nr / BITS_PER_BYTE; + fdt->close_on_exec = data; + data += nr / BITS_PER_BYTE; + fdt->full_fds_bits = data; + + return fdt; + +out_arr: + kvfree(fdt->fd); +out_fdt: + kfree(fdt); +out: + return ERR_PTR(-ENOMEM); +} + +/* + * Expand the file descriptor table. + * This function will allocate a new fdtable and both fd array and fdset, of + * the given size. + * Return <0 error code on error; 0 on successful completion. + * The files->file_lock should be held on entry, and will be held on exit. + */ +static int expand_fdtable(struct files_struct *files, unsigned int nr) + __releases(files->file_lock) + __acquires(files->file_lock) +{ + struct fdtable *new_fdt, *cur_fdt; + + spin_unlock(&files->file_lock); + new_fdt = alloc_fdtable(nr + 1); + + /* make sure all fd_install() have seen resize_in_progress + * or have finished their rcu_read_lock_sched() section. + */ + if (atomic_read(&files->count) > 1) + synchronize_rcu(); + + spin_lock(&files->file_lock); + if (IS_ERR(new_fdt)) + return PTR_ERR(new_fdt); + cur_fdt = files_fdtable(files); + BUG_ON(nr < cur_fdt->max_fds); + copy_fdtable(new_fdt, cur_fdt); + rcu_assign_pointer(files->fdt, new_fdt); + if (cur_fdt != &files->fdtab) + call_rcu(&cur_fdt->rcu, free_fdtable_rcu); + /* coupled with smp_rmb() in fd_install() */ + smp_wmb(); + return 0; +} + +/* + * Expand files. + * This function will expand the file structures, if the requested size exceeds + * the current capacity and there is room for expansion. + * Return <0 error code on error; 0 on success. + * The files->file_lock should be held on entry, and will be held on exit. + */ +static int expand_files(struct files_struct *files, unsigned int nr) + __releases(files->file_lock) + __acquires(files->file_lock) +{ + struct fdtable *fdt; + int error; + +repeat: + fdt = files_fdtable(files); + + /* Do we need to expand? */ + if (nr < fdt->max_fds) + return 0; + + if (unlikely(files->resize_in_progress)) { + spin_unlock(&files->file_lock); + wait_event(files->resize_wait, !files->resize_in_progress); + spin_lock(&files->file_lock); + goto repeat; + } + + /* Can we expand? */ + if (unlikely(nr >= sysctl_nr_open)) + return -EMFILE; + + /* All good, so we try */ + files->resize_in_progress = true; + error = expand_fdtable(files, nr); + files->resize_in_progress = false; + + wake_up_all(&files->resize_wait); + return error; +} + +static inline void __set_close_on_exec(unsigned int fd, struct fdtable *fdt, + bool set) +{ + if (set) { + __set_bit(fd, fdt->close_on_exec); + } else { + if (test_bit(fd, fdt->close_on_exec)) + __clear_bit(fd, fdt->close_on_exec); + } +} + +static inline void __set_open_fd(unsigned int fd, struct fdtable *fdt, bool set) +{ + __set_bit(fd, fdt->open_fds); + __set_close_on_exec(fd, fdt, set); + fd /= BITS_PER_LONG; + if (!~fdt->open_fds[fd]) + __set_bit(fd, fdt->full_fds_bits); +} + +static inline void __clear_open_fd(unsigned int fd, struct fdtable *fdt) +{ + __clear_bit(fd, fdt->open_fds); + fd /= BITS_PER_LONG; + if (test_bit(fd, fdt->full_fds_bits)) + __clear_bit(fd, fdt->full_fds_bits); +} + +static inline bool fd_is_open(unsigned int fd, const struct fdtable *fdt) +{ + return test_bit(fd, fdt->open_fds); +} + +/* + * Note that a sane fdtable size always has to be a multiple of + * BITS_PER_LONG, since we have bitmaps that are sized by this. + * + * punch_hole is optional - when close_range() is asked to unshare + * and close, we don't need to copy descriptors in that range, so + * a smaller cloned descriptor table might suffice if the last + * currently opened descriptor falls into that range. + */ +static unsigned int sane_fdtable_size(struct fdtable *fdt, struct fd_range *punch_hole) +{ + unsigned int last = find_last_bit(fdt->open_fds, fdt->max_fds); + + if (last == fdt->max_fds) + return NR_OPEN_DEFAULT; + if (punch_hole && punch_hole->to >= last && punch_hole->from <= last) { + last = find_last_bit(fdt->open_fds, punch_hole->from); + if (last == punch_hole->from) + return NR_OPEN_DEFAULT; + } + return ALIGN(last + 1, BITS_PER_LONG); +} + +/* + * Allocate a new descriptor table and copy contents from the passed in + * instance. Returns a pointer to cloned table on success, ERR_PTR() + * on failure. For 'punch_hole' see sane_fdtable_size(). + */ +struct files_struct *dup_fd(struct files_struct *oldf, struct fd_range *punch_hole) +{ + struct files_struct *newf; + struct file **old_fds, **new_fds; + unsigned int open_files, i; + struct fdtable *old_fdt, *new_fdt; + + newf = kmem_cache_alloc(files_cachep, GFP_KERNEL); + if (!newf) + return ERR_PTR(-ENOMEM); + + atomic_set(&newf->count, 1); + + spin_lock_init(&newf->file_lock); + newf->resize_in_progress = false; + init_waitqueue_head(&newf->resize_wait); + newf->next_fd = 0; + new_fdt = &newf->fdtab; + new_fdt->max_fds = NR_OPEN_DEFAULT; + new_fdt->close_on_exec = newf->close_on_exec_init; + new_fdt->open_fds = newf->open_fds_init; + new_fdt->full_fds_bits = newf->full_fds_bits_init; + new_fdt->fd = &newf->fd_array[0]; + + spin_lock(&oldf->file_lock); + old_fdt = files_fdtable(oldf); + open_files = sane_fdtable_size(old_fdt, punch_hole); + + /* + * Check whether we need to allocate a larger fd array and fd set. + */ + while (unlikely(open_files > new_fdt->max_fds)) { + spin_unlock(&oldf->file_lock); + + if (new_fdt != &newf->fdtab) + __free_fdtable(new_fdt); + + new_fdt = alloc_fdtable(open_files); + if (IS_ERR(new_fdt)) { + kmem_cache_free(files_cachep, newf); + return ERR_CAST(new_fdt); + } + + /* + * Reacquire the oldf lock and a pointer to its fd table + * who knows it may have a new bigger fd table. We need + * the latest pointer. + */ + spin_lock(&oldf->file_lock); + old_fdt = files_fdtable(oldf); + open_files = sane_fdtable_size(old_fdt, punch_hole); + } + + copy_fd_bitmaps(new_fdt, old_fdt, open_files / BITS_PER_LONG); + + old_fds = old_fdt->fd; + new_fds = new_fdt->fd; + + /* + * We may be racing against fd allocation from other threads using this + * files_struct, despite holding ->file_lock. + * + * alloc_fd() might have already claimed a slot, while fd_install() + * did not populate it yet. Note the latter operates locklessly, so + * the file can show up as we are walking the array below. + * + * At the same time we know no files will disappear as all other + * operations take the lock. + * + * Instead of trying to placate userspace racing with itself, we + * ref the file if we see it and mark the fd slot as unused otherwise. + */ + for (i = open_files; i != 0; i--) { + struct file *f = rcu_dereference_raw(*old_fds++); + if (f) { + get_file(f); + } else { + __clear_open_fd(open_files - i, new_fdt); + } + rcu_assign_pointer(*new_fds++, f); + } + spin_unlock(&oldf->file_lock); + + /* clear the remainder */ + memset(new_fds, 0, (new_fdt->max_fds - open_files) * sizeof(struct file *)); + + rcu_assign_pointer(newf->fdt, new_fdt); + + return newf; +} + +static struct fdtable *close_files(struct files_struct * files) +{ + /* + * It is safe to dereference the fd table without RCU or + * ->file_lock because this is the last reference to the + * files structure. + */ + struct fdtable *fdt = rcu_dereference_raw(files->fdt); + unsigned int i, j = 0; + + for (;;) { + unsigned long set; + i = j * BITS_PER_LONG; + if (i >= fdt->max_fds) + break; + set = fdt->open_fds[j++]; + while (set) { + if (set & 1) { + struct file *file = fdt->fd[i]; + if (file) { + filp_close(file, files); + cond_resched(); + } + } + i++; + set >>= 1; + } + } + + return fdt; +} + +void put_files_struct(struct files_struct *files) +{ + if (atomic_dec_and_test(&files->count)) { + struct fdtable *fdt = close_files(files); + + /* free the arrays if they are not embedded */ + if (fdt != &files->fdtab) + __free_fdtable(fdt); + kmem_cache_free(files_cachep, files); + } +} + +void exit_files(struct task_struct *tsk) +{ + struct files_struct * files = tsk->files; + + if (files) { + task_lock(tsk); + tsk->files = NULL; + task_unlock(tsk); + put_files_struct(files); + } +} + +struct files_struct init_files = { + .count = ATOMIC_INIT(1), + .fdt = &init_files.fdtab, + .fdtab = { + .max_fds = NR_OPEN_DEFAULT, + .fd = &init_files.fd_array[0], + .close_on_exec = init_files.close_on_exec_init, + .open_fds = init_files.open_fds_init, + .full_fds_bits = init_files.full_fds_bits_init, + }, + .file_lock = __SPIN_LOCK_UNLOCKED(init_files.file_lock), + .resize_wait = __WAIT_QUEUE_HEAD_INITIALIZER(init_files.resize_wait), +}; + +static unsigned int find_next_fd(struct fdtable *fdt, unsigned int start) +{ + unsigned int maxfd = fdt->max_fds; /* always multiple of BITS_PER_LONG */ + unsigned int max_fds_words = maxfd / BITS_PER_LONG; + unsigned int fds_word_idx = start / BITS_PER_LONG; + unsigned int bit; + + /* + * Try to avoid looking at the second level bitmap + */ + bit = find_next_zero_bit(&fdt->open_fds[fds_word_idx], BITS_PER_LONG, + start & (BITS_PER_LONG - 1)); + if (bit < BITS_PER_LONG) + return bit + (fds_word_idx * BITS_PER_LONG); + + bit = BITS_PER_LONG * + find_next_zero_bit(fdt->full_fds_bits, max_fds_words, fds_word_idx + 1); + if (bit >= maxfd) + return maxfd; + return find_next_zero_bit(fdt->open_fds, maxfd, bit); +} + +/* + * allocate a file descriptor, mark it busy. + */ +static int alloc_fd(unsigned start, unsigned end, unsigned flags) +{ + struct files_struct *files = current->files; + unsigned int fd; + int error; + struct fdtable *fdt; + + spin_lock(&files->file_lock); +repeat: + fdt = files_fdtable(files); + fd = start; + if (fd < files->next_fd) + fd = files->next_fd; + + if (likely(fd < fdt->max_fds)) + fd = find_next_fd(fdt, fd); + + /* + * N.B. For clone tasks sharing a files structure, this test + * will limit the total number of files that can be opened. + */ + error = -EMFILE; + if (unlikely(fd >= end)) + goto out; + + if (unlikely(fd >= fdt->max_fds)) { + error = expand_files(files, fd); + if (error < 0) + goto out; + + goto repeat; + } + + if (start <= files->next_fd) + files->next_fd = fd + 1; + + __set_open_fd(fd, fdt, flags & O_CLOEXEC); + error = fd; + VFS_BUG_ON(rcu_access_pointer(fdt->fd[fd]) != NULL); + +out: + spin_unlock(&files->file_lock); + return error; +} + +int __get_unused_fd_flags(unsigned flags, unsigned long nofile) +{ + return alloc_fd(0, nofile, flags); +} + +int get_unused_fd_flags(unsigned flags) +{ + return __get_unused_fd_flags(flags, rlimit(RLIMIT_NOFILE)); +} +EXPORT_SYMBOL(get_unused_fd_flags); + +static void __put_unused_fd(struct files_struct *files, unsigned int fd) +{ + struct fdtable *fdt = files_fdtable(files); + __clear_open_fd(fd, fdt); + if (fd < files->next_fd) + files->next_fd = fd; +} + +void put_unused_fd(unsigned int fd) +{ + struct files_struct *files = current->files; + spin_lock(&files->file_lock); + __put_unused_fd(files, fd); + spin_unlock(&files->file_lock); +} + +EXPORT_SYMBOL(put_unused_fd); + +/* + * Install a file pointer in the fd array while it is being resized. + * + * We need to make sure our update to the array does not get lost as the resizing + * thread can be copying the content as we modify it. + * + * We have two ways to do it: + * - go off CPU waiting for resize_in_progress to clear + * - take the spin lock + * + * The latter is trivial to implement and saves us from having to might_sleep() + * for debugging purposes. + * + * This is moved out of line from fd_install() to convince gcc to optimize that + * routine better. + */ +static void noinline fd_install_slowpath(unsigned int fd, struct file *file) +{ + struct files_struct *files = current->files; + struct fdtable *fdt; + + spin_lock(&files->file_lock); + fdt = files_fdtable(files); + VFS_BUG_ON(rcu_access_pointer(fdt->fd[fd]) != NULL); + rcu_assign_pointer(fdt->fd[fd], file); + spin_unlock(&files->file_lock); +} + +/** + * fd_install - install a file pointer in the fd array + * @fd: file descriptor to install the file in + * @file: the file to install + * + * This consumes the "file" refcount, so callers should treat it + * as if they had called fput(file). + */ +void fd_install(unsigned int fd, struct file *file) +{ + struct files_struct *files = current->files; + struct fdtable *fdt; + + if (WARN_ON_ONCE(unlikely(file->f_mode & FMODE_BACKING))) + return; + + rcu_read_lock_sched(); + if (unlikely(files->resize_in_progress)) { + rcu_read_unlock_sched(); + fd_install_slowpath(fd, file); + return; + } + /* coupled with smp_wmb() in expand_fdtable() */ + smp_rmb(); + fdt = rcu_dereference_sched(files->fdt); + VFS_BUG_ON(rcu_access_pointer(fdt->fd[fd]) != NULL); + rcu_assign_pointer(fdt->fd[fd], file); + rcu_read_unlock_sched(); +} + +EXPORT_SYMBOL(fd_install); + +/** + * file_close_fd_locked - return file associated with fd + * @files: file struct to retrieve file from + * @fd: file descriptor to retrieve file for + * + * Doesn't take a separate reference count. + * + * Context: files_lock must be held. + * + * Returns: The file associated with @fd (NULL if @fd is not open) + */ +struct file *file_close_fd_locked(struct files_struct *files, unsigned fd) +{ + struct fdtable *fdt = files_fdtable(files); + struct file *file; + + lockdep_assert_held(&files->file_lock); + + if (fd >= fdt->max_fds) + return NULL; + + fd = array_index_nospec(fd, fdt->max_fds); + file = rcu_dereference_raw(fdt->fd[fd]); + if (file) { + rcu_assign_pointer(fdt->fd[fd], NULL); + __put_unused_fd(files, fd); + } + return file; +} + +int close_fd(unsigned fd) +{ + struct files_struct *files = current->files; + struct file *file; + + spin_lock(&files->file_lock); + file = file_close_fd_locked(files, fd); + spin_unlock(&files->file_lock); + if (!file) + return -EBADF; + + return filp_close(file, files); +} +EXPORT_SYMBOL(close_fd); + +/** + * last_fd - return last valid index into fd table + * @fdt: File descriptor table. + * + * Context: Either rcu read lock or files_lock must be held. + * + * Returns: Last valid index into fdtable. + */ +static inline unsigned last_fd(struct fdtable *fdt) +{ + return fdt->max_fds - 1; +} + +static inline void __range_cloexec(struct files_struct *cur_fds, + unsigned int fd, unsigned int max_fd) +{ + struct fdtable *fdt; + + /* make sure we're using the correct maximum value */ + spin_lock(&cur_fds->file_lock); + fdt = files_fdtable(cur_fds); + max_fd = min(last_fd(fdt), max_fd); + if (fd <= max_fd) + bitmap_set(fdt->close_on_exec, fd, max_fd - fd + 1); + spin_unlock(&cur_fds->file_lock); +} + +static inline void __range_close(struct files_struct *files, unsigned int fd, + unsigned int max_fd) +{ + struct file *file; + struct fdtable *fdt; + unsigned n; + + spin_lock(&files->file_lock); + fdt = files_fdtable(files); + n = last_fd(fdt); + max_fd = min(max_fd, n); + + for (fd = find_next_bit(fdt->open_fds, max_fd + 1, fd); + fd <= max_fd; + fd = find_next_bit(fdt->open_fds, max_fd + 1, fd + 1)) { + file = file_close_fd_locked(files, fd); + if (file) { + spin_unlock(&files->file_lock); + filp_close(file, files); + cond_resched(); + spin_lock(&files->file_lock); + fdt = files_fdtable(files); + } else if (need_resched()) { + spin_unlock(&files->file_lock); + cond_resched(); + spin_lock(&files->file_lock); + fdt = files_fdtable(files); + } + } + spin_unlock(&files->file_lock); +} + +/** + * sys_close_range() - Close all file descriptors in a given range. + * + * @fd: starting file descriptor to close + * @max_fd: last file descriptor to close + * @flags: CLOSE_RANGE flags. + * + * This closes a range of file descriptors. All file descriptors + * from @fd up to and including @max_fd are closed. + * Currently, errors to close a given file descriptor are ignored. + */ +SYSCALL_DEFINE3(close_range, unsigned int, fd, unsigned int, max_fd, + unsigned int, flags) +{ + struct task_struct *me = current; + struct files_struct *cur_fds = me->files, *fds = NULL; + + if (flags & ~(CLOSE_RANGE_UNSHARE | CLOSE_RANGE_CLOEXEC)) + return -EINVAL; + + if (fd > max_fd) + return -EINVAL; + + if ((flags & CLOSE_RANGE_UNSHARE) && atomic_read(&cur_fds->count) > 1) { + struct fd_range range = {fd, max_fd}, *punch_hole = ⦥ + + /* + * If the caller requested all fds to be made cloexec we always + * copy all of the file descriptors since they still want to + * use them. + */ + if (flags & CLOSE_RANGE_CLOEXEC) + punch_hole = NULL; + + fds = dup_fd(cur_fds, punch_hole); + if (IS_ERR(fds)) + return PTR_ERR(fds); + /* + * We used to share our file descriptor table, and have now + * created a private one, make sure we're using it below. + */ + swap(cur_fds, fds); + } + + if (flags & CLOSE_RANGE_CLOEXEC) + __range_cloexec(cur_fds, fd, max_fd); + else + __range_close(cur_fds, fd, max_fd); + + if (fds) { + /* + * We're done closing the files we were supposed to. Time to install + * the new file descriptor table and drop the old one. + */ + task_lock(me); + me->files = cur_fds; + task_unlock(me); + put_files_struct(fds); + } + + return 0; +} + +/** + * file_close_fd - return file associated with fd + * @fd: file descriptor to retrieve file for + * + * Doesn't take a separate reference count. + * + * Returns: The file associated with @fd (NULL if @fd is not open) + */ +struct file *file_close_fd(unsigned int fd) +{ + struct files_struct *files = current->files; + struct file *file; + + spin_lock(&files->file_lock); + file = file_close_fd_locked(files, fd); + spin_unlock(&files->file_lock); + + return file; +} + +void do_close_on_exec(struct files_struct *files) +{ + unsigned i; + struct fdtable *fdt; + + /* exec unshares first */ + spin_lock(&files->file_lock); + for (i = 0; ; i++) { + unsigned long set; + unsigned fd = i * BITS_PER_LONG; + fdt = files_fdtable(files); + if (fd >= fdt->max_fds) + break; + set = fdt->close_on_exec[i]; + if (!set) + continue; + fdt->close_on_exec[i] = 0; + for ( ; set ; fd++, set >>= 1) { + struct file *file; + if (!(set & 1)) + continue; + file = fdt->fd[fd]; + if (!file) + continue; + rcu_assign_pointer(fdt->fd[fd], NULL); + __put_unused_fd(files, fd); + spin_unlock(&files->file_lock); + filp_close(file, files); + cond_resched(); + spin_lock(&files->file_lock); + } + + } + spin_unlock(&files->file_lock); +} + +static struct file *__get_file_rcu(struct file __rcu **f) +{ + struct file __rcu *file; + struct file __rcu *file_reloaded; + struct file __rcu *file_reloaded_cmp; + + file = rcu_dereference_raw(*f); + if (!file) + return NULL; + + if (unlikely(!file_ref_get(&file->f_ref))) + return ERR_PTR(-EAGAIN); + + file_reloaded = rcu_dereference_raw(*f); + + /* + * Ensure that all accesses have a dependency on the load from + * rcu_dereference_raw() above so we get correct ordering + * between reuse/allocation and the pointer check below. + */ + file_reloaded_cmp = file_reloaded; + OPTIMIZER_HIDE_VAR(file_reloaded_cmp); + + /* + * file_ref_get() above provided a full memory barrier when we + * acquired a reference. + * + * This is paired with the write barrier from assigning to the + * __rcu protected file pointer so that if that pointer still + * matches the current file, we know we have successfully + * acquired a reference to the right file. + * + * If the pointers don't match the file has been reallocated by + * SLAB_TYPESAFE_BY_RCU. + */ + if (file == file_reloaded_cmp) + return file_reloaded; + + fput(file); + return ERR_PTR(-EAGAIN); +} + +/** + * get_file_rcu - try go get a reference to a file under rcu + * @f: the file to get a reference on + * + * This function tries to get a reference on @f carefully verifying that + * @f hasn't been reused. + * + * This function should rarely have to be used and only by users who + * understand the implications of SLAB_TYPESAFE_BY_RCU. Try to avoid it. + * + * Return: Returns @f with the reference count increased or NULL. + */ +struct file *get_file_rcu(struct file __rcu **f) +{ + for (;;) { + struct file __rcu *file; + + file = __get_file_rcu(f); + if (!IS_ERR(file)) + return file; + } +} +EXPORT_SYMBOL_GPL(get_file_rcu); + +/** + * get_file_active - try go get a reference to a file + * @f: the file to get a reference on + * + * In contast to get_file_rcu() the pointer itself isn't part of the + * reference counting. + * + * This function should rarely have to be used and only by users who + * understand the implications of SLAB_TYPESAFE_BY_RCU. Try to avoid it. + * + * Return: Returns @f with the reference count increased or NULL. + */ +struct file *get_file_active(struct file **f) +{ + struct file __rcu *file; + + rcu_read_lock(); + file = __get_file_rcu(f); + rcu_read_unlock(); + if (IS_ERR(file)) + file = NULL; + return file; +} +EXPORT_SYMBOL_GPL(get_file_active); + +static inline struct file *__fget_files_rcu(struct files_struct *files, + unsigned int fd, fmode_t mask) +{ + for (;;) { + struct file *file; + struct fdtable *fdt = rcu_dereference_raw(files->fdt); + struct file __rcu **fdentry; + unsigned long nospec_mask; + + /* Mask is a 0 for invalid fd's, ~0 for valid ones */ + nospec_mask = array_index_mask_nospec(fd, fdt->max_fds); + + /* + * fdentry points to the 'fd' offset, or fdt->fd[0]. + * Loading from fdt->fd[0] is always safe, because the + * array always exists. + */ + fdentry = fdt->fd + (fd & nospec_mask); + + /* Do the load, then mask any invalid result */ + file = rcu_dereference_raw(*fdentry); + file = (void *)(nospec_mask & (unsigned long)file); + if (unlikely(!file)) + return NULL; + + /* + * Ok, we have a file pointer that was valid at + * some point, but it might have become stale since. + * + * We need to confirm it by incrementing the refcount + * and then check the lookup again. + * + * file_ref_get() gives us a full memory barrier. We + * only really need an 'acquire' one to protect the + * loads below, but we don't have that. + */ + if (unlikely(!file_ref_get(&file->f_ref))) + continue; + + /* + * Such a race can take two forms: + * + * (a) the file ref already went down to zero and the + * file hasn't been reused yet or the file count + * isn't zero but the file has already been reused. + * + * (b) the file table entry has changed under us. + * Note that we don't need to re-check the 'fdt->fd' + * pointer having changed, because it always goes + * hand-in-hand with 'fdt'. + * + * If so, we need to put our ref and try again. + */ + if (unlikely(file != rcu_dereference_raw(*fdentry)) || + unlikely(rcu_dereference_raw(files->fdt) != fdt)) { + fput(file); + continue; + } + + /* + * This isn't the file we're looking for or we're not + * allowed to get a reference to it. + */ + if (unlikely(file->f_mode & mask)) { + fput(file); + return NULL; + } + + /* + * Ok, we have a ref to the file, and checked that it + * still exists. + */ + return file; + } +} + +static struct file *__fget_files(struct files_struct *files, unsigned int fd, + fmode_t mask) +{ + struct file *file; + + rcu_read_lock(); + file = __fget_files_rcu(files, fd, mask); + rcu_read_unlock(); + + return file; +} + +static inline struct file *__fget(unsigned int fd, fmode_t mask) +{ + return __fget_files(current->files, fd, mask); +} + +struct file *fget(unsigned int fd) +{ + return __fget(fd, FMODE_PATH); +} +EXPORT_SYMBOL(fget); + +struct file *fget_raw(unsigned int fd) +{ + return __fget(fd, 0); +} +EXPORT_SYMBOL(fget_raw); + +struct file *fget_task(struct task_struct *task, unsigned int fd) +{ + struct file *file = NULL; + + task_lock(task); + if (task->files) + file = __fget_files(task->files, fd, 0); + task_unlock(task); + + return file; +} + +struct file *fget_task_next(struct task_struct *task, unsigned int *ret_fd) +{ + struct files_struct *files; + unsigned int fd = *ret_fd; + struct file *file = NULL; + + task_lock(task); + files = task->files; + if (files) { + rcu_read_lock(); + for (; fd < files_fdtable(files)->max_fds; fd++) { + file = __fget_files_rcu(files, fd, 0); + if (file) + break; + } + rcu_read_unlock(); + } + task_unlock(task); + *ret_fd = fd; + return file; +} +EXPORT_SYMBOL(fget_task_next); + +/* + * Lightweight file lookup - no refcnt increment if fd table isn't shared. + * + * You can use this instead of fget if you satisfy all of the following + * conditions: + * 1) You must call fput_light before exiting the syscall and returning control + * to userspace (i.e. you cannot remember the returned struct file * after + * returning to userspace). + * 2) You must not call filp_close on the returned struct file * in between + * calls to fget_light and fput_light. + * 3) You must not clone the current task in between the calls to fget_light + * and fput_light. + * + * The fput_needed flag returned by fget_light should be passed to the + * corresponding fput_light. + * + * (As an exception to rule 2, you can call filp_close between fget_light and + * fput_light provided that you capture a real refcount with get_file before + * the call to filp_close, and ensure that this real refcount is fput *after* + * the fput_light call.) + * + * See also the documentation in rust/kernel/file.rs. + */ +static inline struct fd __fget_light(unsigned int fd, fmode_t mask) +{ + struct files_struct *files = current->files; + struct file *file; + + /* + * If another thread is concurrently calling close_fd() followed + * by put_files_struct(), we must not observe the old table + * entry combined with the new refcount - otherwise we could + * return a file that is concurrently being freed. + * + * atomic_read_acquire() pairs with atomic_dec_and_test() in + * put_files_struct(). + */ + if (likely(atomic_read_acquire(&files->count) == 1)) { + file = files_lookup_fd_raw(files, fd); + if (!file || unlikely(file->f_mode & mask)) + return EMPTY_FD; + return BORROWED_FD(file); + } else { + file = __fget_files(files, fd, mask); + if (!file) + return EMPTY_FD; + return CLONED_FD(file); + } +} +struct fd fdget(unsigned int fd) +{ + return __fget_light(fd, FMODE_PATH); +} +EXPORT_SYMBOL(fdget); + +struct fd fdget_raw(unsigned int fd) +{ + return __fget_light(fd, 0); +} + +/* + * Try to avoid f_pos locking. We only need it if the + * file is marked for FMODE_ATOMIC_POS, and it can be + * accessed multiple ways. + * + * Always do it for directories, because pidfd_getfd() + * can make a file accessible even if it otherwise would + * not be, and for directories this is a correctness + * issue, not a "POSIX requirement". + */ +static inline bool file_needs_f_pos_lock(struct file *file) +{ + if (!(file->f_mode & FMODE_ATOMIC_POS)) + return false; + if (__file_ref_read_raw(&file->f_ref) != FILE_REF_ONEREF) + return true; + if (file->f_op->iterate_shared) + return true; + return false; +} + +bool file_seek_cur_needs_f_lock(struct file *file) +{ + if (!(file->f_mode & FMODE_ATOMIC_POS) && !file->f_op->iterate_shared) + return false; + + /* + * Note that we are not guaranteed to be called after fdget_pos() on + * this file obj, in which case the caller is expected to provide the + * appropriate locking. + */ + + return true; +} + +struct fd fdget_pos(unsigned int fd) +{ + struct fd f = fdget(fd); + struct file *file = fd_file(f); + + if (likely(file) && file_needs_f_pos_lock(file)) { + f.word |= FDPUT_POS_UNLOCK; + mutex_lock(&file->f_pos_lock); + } + return f; +} + +void __f_unlock_pos(struct file *f) +{ + mutex_unlock(&f->f_pos_lock); +} + +/* + * We only lock f_pos if we have threads or if the file might be + * shared with another process. In both cases we'll have an elevated + * file count (done either by fdget() or by fork()). + */ + +void set_close_on_exec(unsigned int fd, int flag) +{ + struct files_struct *files = current->files; + spin_lock(&files->file_lock); + __set_close_on_exec(fd, files_fdtable(files), flag); + spin_unlock(&files->file_lock); +} + +bool get_close_on_exec(unsigned int fd) +{ + bool res; + rcu_read_lock(); + res = close_on_exec(fd, current->files); + rcu_read_unlock(); + return res; +} + +static int do_dup2(struct files_struct *files, + struct file *file, unsigned fd, unsigned flags) +__releases(&files->file_lock) +{ + struct file *tofree; + struct fdtable *fdt; + + /* + * dup2() is expected to close the file installed in the target fd slot + * (if any). However, userspace hand-picking a fd may be racing against + * its own threads which happened to allocate it in open() et al but did + * not populate it yet. + * + * Broadly speaking we may be racing against the following: + * fd = get_unused_fd_flags(); // fd slot reserved, ->fd[fd] == NULL + * file = hard_work_goes_here(); + * fd_install(fd, file); // only now ->fd[fd] == file + * + * It is an invariant that a successfully allocated fd has a NULL entry + * in the array until the matching fd_install(). + * + * If we fit the window, we have the fd to populate, yet no target file + * to close. Trying to ignore it and install our new file would violate + * the invariant and make fd_install() overwrite our file. + * + * Things can be done(tm) to handle this. However, the issue does not + * concern legitimate programs and we only need to make sure the kernel + * does not trip over it. + * + * The simplest way out is to return an error if we find ourselves here. + * + * POSIX is silent on the issue, we return -EBUSY. + */ + fdt = files_fdtable(files); + fd = array_index_nospec(fd, fdt->max_fds); + tofree = rcu_dereference_raw(fdt->fd[fd]); + if (!tofree && fd_is_open(fd, fdt)) + goto Ebusy; + get_file(file); + rcu_assign_pointer(fdt->fd[fd], file); + __set_open_fd(fd, fdt, flags & O_CLOEXEC); + spin_unlock(&files->file_lock); + + if (tofree) + filp_close(tofree, files); + + return fd; + +Ebusy: + spin_unlock(&files->file_lock); + return -EBUSY; +} + +int replace_fd(unsigned fd, struct file *file, unsigned flags) +{ + int err; + struct files_struct *files = current->files; + + if (!file) + return close_fd(fd); + + if (fd >= rlimit(RLIMIT_NOFILE)) + return -EBADF; + + spin_lock(&files->file_lock); + err = expand_files(files, fd); + if (unlikely(err < 0)) + goto out_unlock; + err = do_dup2(files, file, fd, flags); + if (err < 0) + return err; + return 0; + +out_unlock: + spin_unlock(&files->file_lock); + return err; +} + +/** + * receive_fd() - Install received file into file descriptor table + * @file: struct file that was received from another process + * @ufd: __user pointer to write new fd number to + * @o_flags: the O_* flags to apply to the new fd entry + * + * Installs a received file into the file descriptor table, with appropriate + * checks and count updates. Optionally writes the fd number to userspace, if + * @ufd is non-NULL. + * + * This helper handles its own reference counting of the incoming + * struct file. + * + * Returns newly install fd or -ve on error. + */ +int receive_fd(struct file *file, int __user *ufd, unsigned int o_flags) +{ + int error; + + error = security_file_receive(file); + if (error) + return error; + + FD_PREPARE(fdf, o_flags, file); + if (fdf.err) + return fdf.err; + get_file(file); + + if (ufd) { + error = put_user(fd_prepare_fd(fdf), ufd); + if (error) + return error; + } + + __receive_sock(fd_prepare_file(fdf)); + return fd_publish(fdf); +} +EXPORT_SYMBOL_GPL(receive_fd); + +int receive_fd_replace(int new_fd, struct file *file, unsigned int o_flags) +{ + int error; + + error = security_file_receive(file); + if (error) + return error; + error = replace_fd(new_fd, file, o_flags); + if (error) + return error; + __receive_sock(file); + return new_fd; +} + +static int ksys_dup3(unsigned int oldfd, unsigned int newfd, int flags) +{ + int err = -EBADF; + struct file *file; + struct files_struct *files = current->files; + + if ((flags & ~O_CLOEXEC) != 0) + return -EINVAL; + + if (unlikely(oldfd == newfd)) + return -EINVAL; + + if (newfd >= rlimit(RLIMIT_NOFILE)) + return -EBADF; + + spin_lock(&files->file_lock); + err = expand_files(files, newfd); + file = files_lookup_fd_locked(files, oldfd); + if (unlikely(!file)) + goto Ebadf; + if (unlikely(err < 0)) { + if (err == -EMFILE) + goto Ebadf; + goto out_unlock; + } + return do_dup2(files, file, newfd, flags); + +Ebadf: + err = -EBADF; +out_unlock: + spin_unlock(&files->file_lock); + return err; +} + +SYSCALL_DEFINE3(dup3, unsigned int, oldfd, unsigned int, newfd, int, flags) +{ + return ksys_dup3(oldfd, newfd, flags); +} + +SYSCALL_DEFINE2(dup2, unsigned int, oldfd, unsigned int, newfd) +{ + if (unlikely(newfd == oldfd)) { /* corner case */ + struct files_struct *files = current->files; + struct file *f; + int retval = oldfd; + + rcu_read_lock(); + f = __fget_files_rcu(files, oldfd, 0); + if (!f) + retval = -EBADF; + rcu_read_unlock(); + if (f) + fput(f); + return retval; + } + return ksys_dup3(oldfd, newfd, 0); +} + +SYSCALL_DEFINE1(dup, unsigned int, fildes) +{ + int ret = -EBADF; + struct file *file = fget_raw(fildes); + + if (file) { + ret = get_unused_fd_flags(0); + if (ret >= 0) + fd_install(ret, file); + else + fput(file); + } + return ret; +} + +int f_dupfd(unsigned int from, struct file *file, unsigned flags) +{ + unsigned long nofile = rlimit(RLIMIT_NOFILE); + int err; + if (from >= nofile) + return -EINVAL; + err = alloc_fd(from, nofile, flags); + if (err >= 0) { + get_file(file); + fd_install(err, file); + } + return err; +} + +int iterate_fd(struct files_struct *files, unsigned n, + int (*f)(const void *, struct file *, unsigned), + const void *p) +{ + struct fdtable *fdt; + int res = 0; + if (!files) + return 0; + spin_lock(&files->file_lock); + for (fdt = files_fdtable(files); n < fdt->max_fds; n++) { + struct file *file; + file = rcu_dereference_check_fdtable(files, fdt->fd[n]); + if (!file) + continue; + res = f(p, file, n); + if (res) + break; + } + spin_unlock(&files->file_lock); + return res; +} +EXPORT_SYMBOL(iterate_fd); diff --git a/fs/file_attr.c b/fs/file_attr.c new file mode 100644 index 000000000..bfb00d256 --- /dev/null +++ b/fs/file_attr.c @@ -0,0 +1,486 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +/** + * fileattr_fill_xflags - initialize fileattr with xflags + * @fa: fileattr pointer + * @xflags: FS_XFLAG_* flags + * + * Set ->fsx_xflags, ->fsx_valid and ->flags (translated xflags). + */ +void fileattr_fill_xflags(struct file_kattr *fa, u32 xflags) +{ + fa->fsx_valid = true; + fa->fsx_xflags = xflags; + if (fa->fsx_xflags & FS_XFLAG_IMMUTABLE) + fa->flags |= FS_IMMUTABLE_FL; + if (fa->fsx_xflags & FS_XFLAG_APPEND) + fa->flags |= FS_APPEND_FL; + if (fa->fsx_xflags & FS_XFLAG_SYNC) + fa->flags |= FS_SYNC_FL; + if (fa->fsx_xflags & FS_XFLAG_NOATIME) + fa->flags |= FS_NOATIME_FL; + if (fa->fsx_xflags & FS_XFLAG_NODUMP) + fa->flags |= FS_NODUMP_FL; + if (fa->fsx_xflags & FS_XFLAG_DAX) + fa->flags |= FS_DAX_FL; + if (fa->fsx_xflags & FS_XFLAG_PROJINHERIT) + fa->flags |= FS_PROJINHERIT_FL; + if (fa->fsx_xflags & FS_XFLAG_VERITY) + fa->flags |= FS_VERITY_FL; + if (fa->fsx_xflags & FS_XFLAG_CASEFOLD) + fa->flags |= FS_CASEFOLD_FL; +} +EXPORT_SYMBOL(fileattr_fill_xflags); + +/** + * fileattr_fill_flags - initialize fileattr with flags + * @fa: fileattr pointer + * @flags: FS_*_FL flags + * + * Set ->flags, ->flags_valid and ->fsx_xflags (translated flags). + */ +void fileattr_fill_flags(struct file_kattr *fa, u32 flags) +{ + fa->flags_valid = true; + fa->flags = flags; + if (fa->flags & FS_SYNC_FL) + fa->fsx_xflags |= FS_XFLAG_SYNC; + if (fa->flags & FS_IMMUTABLE_FL) + fa->fsx_xflags |= FS_XFLAG_IMMUTABLE; + if (fa->flags & FS_APPEND_FL) + fa->fsx_xflags |= FS_XFLAG_APPEND; + if (fa->flags & FS_NODUMP_FL) + fa->fsx_xflags |= FS_XFLAG_NODUMP; + if (fa->flags & FS_NOATIME_FL) + fa->fsx_xflags |= FS_XFLAG_NOATIME; + if (fa->flags & FS_DAX_FL) + fa->fsx_xflags |= FS_XFLAG_DAX; + if (fa->flags & FS_PROJINHERIT_FL) + fa->fsx_xflags |= FS_XFLAG_PROJINHERIT; + if (fa->flags & FS_VERITY_FL) + fa->fsx_xflags |= FS_XFLAG_VERITY; + if (fa->flags & FS_CASEFOLD_FL) + fa->fsx_xflags |= FS_XFLAG_CASEFOLD; +} +EXPORT_SYMBOL(fileattr_fill_flags); + +/** + * vfs_fileattr_get - retrieve miscellaneous file attributes + * @dentry: the object to retrieve from + * @fa: fileattr pointer + * + * Call i_op->fileattr_get() callback, if exists. + * + * Return: 0 on success, or a negative error on failure. + */ +int vfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + int error; + + if (!inode->i_op->fileattr_get) + return -ENOIOCTLCMD; + + error = security_inode_file_getattr(dentry, fa); + if (error) + return error; + + return inode->i_op->fileattr_get(dentry, fa); +} +EXPORT_SYMBOL(vfs_fileattr_get); + +static void fileattr_to_file_attr(const struct file_kattr *fa, + struct file_attr *fattr) +{ + __u32 mask = FS_XFLAGS_MASK; + + memset(fattr, 0, sizeof(struct file_attr)); + fattr->fa_xflags = fa->fsx_xflags & mask; + fattr->fa_extsize = fa->fsx_extsize; + fattr->fa_nextents = fa->fsx_nextents; + fattr->fa_projid = fa->fsx_projid; + fattr->fa_cowextsize = fa->fsx_cowextsize; +} + +/** + * copy_fsxattr_to_user - copy fsxattr to userspace. + * @fa: fileattr pointer + * @ufa: fsxattr user pointer + * + * Return: 0 on success, or -EFAULT on failure. + */ +int copy_fsxattr_to_user(const struct file_kattr *fa, struct fsxattr __user *ufa) +{ + struct fsxattr xfa; + __u32 mask = FS_XFLAGS_MASK; + + memset(&xfa, 0, sizeof(xfa)); + xfa.fsx_xflags = fa->fsx_xflags & mask; + xfa.fsx_extsize = fa->fsx_extsize; + xfa.fsx_nextents = fa->fsx_nextents; + xfa.fsx_projid = fa->fsx_projid; + xfa.fsx_cowextsize = fa->fsx_cowextsize; + + if (copy_to_user(ufa, &xfa, sizeof(xfa))) + return -EFAULT; + + return 0; +} +EXPORT_SYMBOL(copy_fsxattr_to_user); + +static int file_attr_to_fileattr(const struct file_attr *fattr, + struct file_kattr *fa) +{ + __u64 mask = FS_XFLAGS_MASK; + + if (fattr->fa_xflags & ~mask) + return -EINVAL; + + fileattr_fill_xflags(fa, fattr->fa_xflags & ~FS_XFLAG_RDONLY_MASK); + fa->fsx_extsize = fattr->fa_extsize; + fa->fsx_projid = fattr->fa_projid; + fa->fsx_cowextsize = fattr->fa_cowextsize; + + return 0; +} + +static int copy_fsxattr_from_user(struct file_kattr *fa, + struct fsxattr __user *ufa) +{ + struct fsxattr xfa; + __u32 mask = FS_XFLAGS_MASK; + + if (copy_from_user(&xfa, ufa, sizeof(xfa))) + return -EFAULT; + + if (xfa.fsx_xflags & ~mask) + return -EOPNOTSUPP; + + fileattr_fill_xflags(fa, xfa.fsx_xflags & ~FS_XFLAG_RDONLY_MASK); + fa->fsx_extsize = xfa.fsx_extsize; + fa->fsx_nextents = xfa.fsx_nextents; + fa->fsx_projid = xfa.fsx_projid; + fa->fsx_cowextsize = xfa.fsx_cowextsize; + + return 0; +} + +/* + * Generic function to check FS_IOC_FSSETXATTR/FS_IOC_SETFLAGS values and reject + * any invalid configurations. + * + * Note: must be called with inode lock held. + */ +static int fileattr_set_prepare(struct inode *inode, + const struct file_kattr *old_ma, + struct file_kattr *fa) +{ + int err; + + /* + * The IMMUTABLE and APPEND_ONLY flags can only be changed by + * the relevant capability. + */ + if ((fa->flags ^ old_ma->flags) & (FS_APPEND_FL | FS_IMMUTABLE_FL) && + !capable(CAP_LINUX_IMMUTABLE)) + return -EPERM; + + err = fscrypt_prepare_setflags(inode, old_ma->flags, fa->flags); + if (err) + return err; + + /* + * Project Quota ID state is only allowed to change from within the init + * namespace. Enforce that restriction only if we are trying to change + * the quota ID state. Everything else is allowed in user namespaces. + */ + if (current_user_ns() != &init_user_ns) { + if (old_ma->fsx_projid != fa->fsx_projid) + return -EINVAL; + if ((old_ma->fsx_xflags ^ fa->fsx_xflags) & + FS_XFLAG_PROJINHERIT) + return -EINVAL; + } else { + /* + * Caller is allowed to change the project ID. If it is being + * changed, make sure that the new value is valid. + */ + if (old_ma->fsx_projid != fa->fsx_projid && + !projid_valid(make_kprojid(&init_user_ns, fa->fsx_projid))) + return -EINVAL; + } + + /* Check extent size hints. */ + if ((fa->fsx_xflags & FS_XFLAG_EXTSIZE) && !S_ISREG(inode->i_mode)) + return -EINVAL; + + if ((fa->fsx_xflags & FS_XFLAG_EXTSZINHERIT) && + !S_ISDIR(inode->i_mode)) + return -EINVAL; + + if ((fa->fsx_xflags & FS_XFLAG_COWEXTSIZE) && + !S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode)) + return -EINVAL; + + /* + * It is only valid to set the DAX flag on regular files and + * directories on filesystems. + */ + if ((fa->fsx_xflags & FS_XFLAG_DAX) && + !(S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode))) + return -EINVAL; + + /* Extent size hints of zero turn off the flags. */ + if (fa->fsx_extsize == 0) + fa->fsx_xflags &= ~(FS_XFLAG_EXTSIZE | FS_XFLAG_EXTSZINHERIT); + if (fa->fsx_cowextsize == 0) + fa->fsx_xflags &= ~FS_XFLAG_COWEXTSIZE; + + return 0; +} + +/** + * vfs_fileattr_set - change miscellaneous file attributes + * @idmap: idmap of the mount + * @dentry: the object to change + * @fa: fileattr pointer + * + * After verifying permissions, call i_op->fileattr_set() callback, if + * exists. + * + * Verifying attributes involves retrieving current attributes with + * i_op->fileattr_get(), this also allows initializing attributes that have + * not been set by the caller to current values. Inode lock is held + * thoughout to prevent racing with another instance. + * + * Return: 0 on success, or a negative error on failure. + */ +int vfs_fileattr_set(struct mnt_idmap *idmap, struct dentry *dentry, + struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + struct file_kattr old_ma = {}; + int err; + + if (!inode->i_op->fileattr_set) + return -ENOIOCTLCMD; + + if (!inode_owner_or_capable(idmap, inode)) + return -EPERM; + + inode_lock(inode); + err = vfs_fileattr_get(dentry, &old_ma); + if (!err) { + /* initialize missing bits from old_ma */ + if (fa->flags_valid) { + fa->fsx_xflags |= old_ma.fsx_xflags & ~FS_XFLAG_COMMON; + fa->fsx_extsize = old_ma.fsx_extsize; + fa->fsx_nextents = old_ma.fsx_nextents; + fa->fsx_projid = old_ma.fsx_projid; + fa->fsx_cowextsize = old_ma.fsx_cowextsize; + } else { + fa->flags |= old_ma.flags & ~FS_COMMON_FL; + } + + err = fileattr_set_prepare(inode, &old_ma, fa); + if (err) + goto out; + err = security_inode_file_setattr(dentry, fa); + if (err) + goto out; + err = inode->i_op->fileattr_set(idmap, dentry, fa); + if (err) + goto out; + fsnotify_xattr(dentry); + } + +out: + inode_unlock(inode); + return err; +} +EXPORT_SYMBOL(vfs_fileattr_set); + +int ioctl_getflags(struct file *file, unsigned int __user *argp) +{ + struct file_kattr fa = { .flags_valid = true }; /* hint only */ + int err; + + err = vfs_fileattr_get(file->f_path.dentry, &fa); + if (!err) + err = put_user(fa.flags, argp); + return err; +} + +int ioctl_setflags(struct file *file, unsigned int __user *argp) +{ + struct mnt_idmap *idmap = file_mnt_idmap(file); + struct dentry *dentry = file->f_path.dentry; + struct file_kattr fa = {}; + unsigned int flags; + int err; + + err = get_user(flags, argp); + if (!err) { + err = mnt_want_write_file(file); + if (!err) { + fileattr_fill_flags(&fa, flags); + err = vfs_fileattr_set(idmap, dentry, &fa); + mnt_drop_write_file(file); + } + } + return err; +} + +int ioctl_fsgetxattr(struct file *file, void __user *argp) +{ + struct file_kattr fa = { .fsx_valid = true }; /* hint only */ + int err; + + err = vfs_fileattr_get(file->f_path.dentry, &fa); + if (!err) + err = copy_fsxattr_to_user(&fa, argp); + + return err; +} + +int ioctl_fssetxattr(struct file *file, void __user *argp) +{ + struct mnt_idmap *idmap = file_mnt_idmap(file); + struct dentry *dentry = file->f_path.dentry; + struct file_kattr fa = {}; + int err; + + err = copy_fsxattr_from_user(&fa, argp); + if (!err) { + err = mnt_want_write_file(file); + if (!err) { + err = vfs_fileattr_set(idmap, dentry, &fa); + mnt_drop_write_file(file); + } + } + return err; +} + +SYSCALL_DEFINE5(file_getattr, int, dfd, const char __user *, filename, + struct file_attr __user *, ufattr, size_t, usize, + unsigned int, at_flags) +{ + struct path filepath __free(path_put) = {}; + unsigned int lookup_flags = 0; + struct file_attr fattr; + struct file_kattr fa = { .flags_valid = true }; /* hint only */ + int error; + + BUILD_BUG_ON(sizeof(struct file_attr) < FILE_ATTR_SIZE_VER0); + BUILD_BUG_ON(sizeof(struct file_attr) != FILE_ATTR_SIZE_LATEST); + + if ((at_flags & ~(AT_SYMLINK_NOFOLLOW | AT_EMPTY_PATH)) != 0) + return -EINVAL; + + if (!(at_flags & AT_SYMLINK_NOFOLLOW)) + lookup_flags |= LOOKUP_FOLLOW; + + if (usize > PAGE_SIZE) + return -E2BIG; + + if (usize < FILE_ATTR_SIZE_VER0) + return -EINVAL; + + CLASS(filename_maybe_null, name)(filename, at_flags); + if (!name && dfd >= 0) { + CLASS(fd, f)(dfd); + if (fd_empty(f)) + return -EBADF; + + filepath = fd_file(f)->f_path; + path_get(&filepath); + } else { + error = filename_lookup(dfd, name, lookup_flags, &filepath, + NULL); + if (error) + return error; + } + + error = vfs_fileattr_get(filepath.dentry, &fa); + if (error == -ENOIOCTLCMD || error == -ENOTTY) + error = -EOPNOTSUPP; + if (error) + return error; + + fileattr_to_file_attr(&fa, &fattr); + error = copy_struct_to_user(ufattr, usize, &fattr, + sizeof(struct file_attr), NULL); + + return error; +} + +SYSCALL_DEFINE5(file_setattr, int, dfd, const char __user *, filename, + struct file_attr __user *, ufattr, size_t, usize, + unsigned int, at_flags) +{ + struct path filepath __free(path_put) = {}; + unsigned int lookup_flags = 0; + struct file_attr fattr; + struct file_kattr fa = {}; + int error; + + BUILD_BUG_ON(sizeof(struct file_attr) < FILE_ATTR_SIZE_VER0); + BUILD_BUG_ON(sizeof(struct file_attr) != FILE_ATTR_SIZE_LATEST); + + if ((at_flags & ~(AT_SYMLINK_NOFOLLOW | AT_EMPTY_PATH)) != 0) + return -EINVAL; + + if (!(at_flags & AT_SYMLINK_NOFOLLOW)) + lookup_flags |= LOOKUP_FOLLOW; + + if (usize > PAGE_SIZE) + return -E2BIG; + + if (usize < FILE_ATTR_SIZE_VER0) + return -EINVAL; + + error = copy_struct_from_user(&fattr, sizeof(struct file_attr), ufattr, + usize); + if (error) + return error; + + error = file_attr_to_fileattr(&fattr, &fa); + if (error) + return error; + + CLASS(filename_maybe_null, name)(filename, at_flags); + if (!name && dfd >= 0) { + CLASS(fd, f)(dfd); + if (fd_empty(f)) + return -EBADF; + + filepath = fd_file(f)->f_path; + path_get(&filepath); + } else { + error = filename_lookup(dfd, name, lookup_flags, &filepath, + NULL); + if (error) + return error; + } + + error = mnt_want_write(filepath.mnt); + if (!error) { + error = vfs_fileattr_set(mnt_idmap(filepath.mnt), + filepath.dentry, &fa); + if (error == -ENOIOCTLCMD || error == -ENOTTY) + error = -EOPNOTSUPP; + mnt_drop_write(filepath.mnt); + } + + return error; +} diff --git a/fs/file_table.c b/fs/file_table.c new file mode 100644 index 000000000..c68b8c0a4 --- /dev/null +++ b/fs/file_table.c @@ -0,0 +1,667 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/file_table.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include "internal.h" + +/* sysctl tunables... */ +static struct files_stat_struct files_stat = { + .max_files = NR_FILE +}; + +/* SLAB cache for file structures */ +static struct kmem_cache *__filp_cache __ro_after_init; +#define filp_cache runtime_const_ptr(__filp_cache) +static struct kmem_cache *__bfilp_cache __ro_after_init; +#define bfilp_cache runtime_const_ptr(__bfilp_cache) + +static struct percpu_counter nr_files __cacheline_aligned_in_smp; + +/* Container for backing file with optional user path */ +struct backing_file { + struct file file; + union { + struct path user_path; + freeptr_t bf_freeptr; + }; +#ifdef CONFIG_SECURITY + void *security; +#endif +}; + +#define backing_file(f) container_of(f, struct backing_file, file) + +const struct path *backing_file_user_path(const struct file *f) +{ + return &backing_file(f)->user_path; +} +EXPORT_SYMBOL_GPL(backing_file_user_path); + +void backing_file_set_user_path(struct file *f, const struct path *path) +{ + backing_file(f)->user_path = *path; +} +EXPORT_SYMBOL_GPL(backing_file_set_user_path); + +#ifdef CONFIG_SECURITY +void *backing_file_security(const struct file *f) +{ + return backing_file(f)->security; +} + +void backing_file_set_security(struct file *f, void *security) +{ + backing_file(f)->security = security; +} +#endif /* CONFIG_SECURITY */ + +static inline void backing_file_free(struct backing_file *ff) +{ + security_backing_file_free(&ff->file); + path_put(&ff->user_path); + kmem_cache_free(bfilp_cache, ff); +} + +static inline void file_free(struct file *f) +{ + security_file_free(f); + if (likely(!(f->f_mode & FMODE_NOACCOUNT))) + percpu_counter_dec(&nr_files); + put_cred(f->f_cred); + if (unlikely(f->f_mode & FMODE_BACKING)) { + backing_file_free(backing_file(f)); + } else { + kmem_cache_free(filp_cache, f); + } +} + +/* + * Return the total number of open files in the system + */ +static long get_nr_files(void) +{ + return percpu_counter_read_positive(&nr_files); +} + +/* + * Return the maximum number of open files in the system + */ +unsigned long get_max_files(void) +{ + return files_stat.max_files; +} +EXPORT_SYMBOL_GPL(get_max_files); + +#if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS) + +/* + * Handle nr_files sysctl + */ +static int proc_nr_files(const struct ctl_table *table, int write, void *buffer, + size_t *lenp, loff_t *ppos) +{ + files_stat.nr_files = percpu_counter_sum_positive(&nr_files); + return proc_doulongvec_minmax(table, write, buffer, lenp, ppos); +} + +static const struct ctl_table fs_stat_sysctls[] = { + { + .procname = "file-nr", + .data = &files_stat, + .maxlen = sizeof(files_stat), + .mode = 0444, + .proc_handler = proc_nr_files, + }, + { + .procname = "file-max", + .data = &files_stat.max_files, + .maxlen = sizeof(files_stat.max_files), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + .extra1 = SYSCTL_LONG_ZERO, + .extra2 = SYSCTL_LONG_MAX, + }, + { + .procname = "nr_open", + .data = &sysctl_nr_open, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = &sysctl_nr_open_min, + .extra2 = &sysctl_nr_open_max, + }, +}; + +static int __init init_fs_stat_sysctls(void) +{ + register_sysctl_init("fs", fs_stat_sysctls); + if (IS_ENABLED(CONFIG_BINFMT_MISC)) { + struct ctl_table_header *hdr; + + hdr = register_sysctl_mount_point("fs/binfmt_misc"); + kmemleak_not_leak(hdr); + } + return 0; +} +fs_initcall(init_fs_stat_sysctls); +#endif + +static int init_file(struct file *f, int flags, const struct cred *cred) +{ + int error; + + f->f_cred = get_cred(cred); + error = security_file_alloc(f); + if (unlikely(error)) { + put_cred(f->f_cred); + return error; + } + + spin_lock_init(&f->f_lock); + /* + * Note that f_pos_lock is only used for files raising + * FMODE_ATOMIC_POS and directories. Other files such as pipes + * don't need it and since f_pos_lock is in a union may reuse + * the space for other purposes. They are expected to initialize + * the respective member when opening the file. + */ + mutex_init(&f->f_pos_lock); + memset(&f->__f_path, 0, sizeof(f->f_path)); + memset(&f->f_ra, 0, sizeof(f->f_ra)); + + f->f_flags = flags; + f->f_mode = OPEN_FMODE(flags); + /* + * Disable permission and pre-content events for all files by default. + * They may be enabled later by fsnotify_open_perm_and_set_mode(). + */ + file_set_fsnotify_mode(f, FMODE_NONOTIFY_PERM); + + f->f_op = NULL; + f->f_mapping = NULL; + f->private_data = NULL; + f->f_inode = NULL; + f->f_owner = NULL; +#ifdef CONFIG_EPOLL + f->f_ep = NULL; +#endif + + f->f_iocb_flags = 0; + f->f_pos = 0; + f->f_wb_err = 0; + f->f_sb_err = 0; + + /* + * We're SLAB_TYPESAFE_BY_RCU so initialize f_ref last. While + * fget-rcu pattern users need to be able to handle spurious + * refcount bumps we should reinitialize the reused file first. + */ + file_ref_init(&f->f_ref, 1); + return 0; +} + +/* Find an unused file structure and return a pointer to it. + * Returns an error pointer if some error happened, e.g., we exceed the file + * structures limit, run out of memory or operation is not permitted. + * + * Be very careful using this. You are responsible for + * getting write access to any mount that you might assign + * to this filp, if it is opened for write. If this is not + * done, the mount's writer count will be wrong + * and a warning at __fput() time. + */ +struct file *alloc_empty_file(int flags, const struct cred *cred) +{ + static long old_max; + struct file *f; + int error; + + /* + * Privileged users can go above max_files + */ + if (unlikely(get_nr_files() >= files_stat.max_files) && + !capable(CAP_SYS_ADMIN)) { + /* + * percpu_counters are inaccurate. Do an expensive check before + * we go and fail. + */ + if (percpu_counter_sum_positive(&nr_files) >= files_stat.max_files) + goto over; + } + + f = kmem_cache_alloc(filp_cache, GFP_KERNEL); + if (unlikely(!f)) + return ERR_PTR(-ENOMEM); + + error = init_file(f, flags, cred); + if (unlikely(error)) { + kmem_cache_free(filp_cache, f); + return ERR_PTR(error); + } + + percpu_counter_inc(&nr_files); + + return f; + +over: + /* Ran out of filps - report that */ + if (get_nr_files() > old_max) { + pr_info("VFS: file-max limit %lu reached\n", get_max_files()); + old_max = get_nr_files(); + } + return ERR_PTR(-ENFILE); +} + +/* + * Variant of alloc_empty_file() that doesn't check and modify nr_files. + * + * This is only for kernel internal use, and the allocate file must not be + * installed into file tables or such. + */ +struct file *alloc_empty_file_noaccount(int flags, const struct cred *cred) +{ + struct file *f; + int error; + + f = kmem_cache_alloc(filp_cache, GFP_KERNEL); + if (unlikely(!f)) + return ERR_PTR(-ENOMEM); + + error = init_file(f, flags, cred); + if (unlikely(error)) { + kmem_cache_free(filp_cache, f); + return ERR_PTR(error); + } + + f->f_mode |= FMODE_NOACCOUNT; + + return f; +} + +static int init_backing_file(struct backing_file *ff, + const struct file *user_file) +{ + memset(&ff->user_path, 0, sizeof(ff->user_path)); + backing_file_set_security(&ff->file, NULL); + return security_backing_file_alloc(&ff->file, user_file); +} + +/* + * Variant of alloc_empty_file() that allocates a backing_file container + * and doesn't check and modify nr_files. + * + * This is only for kernel internal use, and the allocate file must not be + * installed into file tables or such. + */ +struct file *alloc_empty_backing_file(int flags, const struct cred *cred, + const struct file *user_file) +{ + struct backing_file *ff; + int error; + + ff = kmem_cache_alloc(bfilp_cache, GFP_KERNEL); + if (unlikely(!ff)) + return ERR_PTR(-ENOMEM); + + error = init_file(&ff->file, flags, cred); + if (unlikely(error)) { + kmem_cache_free(bfilp_cache, ff); + return ERR_PTR(error); + } + + /* The f_mode flags must be set before fput(). */ + ff->file.f_mode |= FMODE_BACKING | FMODE_NOACCOUNT; + error = init_backing_file(ff, user_file); + if (unlikely(error)) { + fput(&ff->file); + return ERR_PTR(error); + } + + return &ff->file; +} +EXPORT_SYMBOL_GPL(alloc_empty_backing_file); + +/** + * file_init_path - initialize a 'struct file' based on path + * + * @file: the file to set up + * @path: the (dentry, vfsmount) pair for the new file + * @fop: the 'struct file_operations' for the new file + */ +static void file_init_path(struct file *file, const struct path *path, + const struct file_operations *fop) +{ + file->__f_path = *path; + file->f_inode = path->dentry->d_inode; + file->f_mapping = path->dentry->d_inode->i_mapping; + file->f_wb_err = filemap_sample_wb_err(file->f_mapping); + file->f_sb_err = file_sample_sb_err(file); + if (fop->llseek) + file->f_mode |= FMODE_LSEEK; + if ((file->f_mode & FMODE_READ) && + likely(fop->read || fop->read_iter)) + file->f_mode |= FMODE_CAN_READ; + if ((file->f_mode & FMODE_WRITE) && + likely(fop->write || fop->write_iter)) + file->f_mode |= FMODE_CAN_WRITE; + file->f_iocb_flags = iocb_flags(file); + file->f_mode |= FMODE_OPENED; + file->f_op = fop; + if ((file->f_mode & (FMODE_READ | FMODE_WRITE)) == FMODE_READ) + i_readcount_inc(path->dentry->d_inode); +} + +/** + * alloc_file - allocate and initialize a 'struct file' + * + * @path: the (dentry, vfsmount) pair for the new file + * @flags: O_... flags with which the new file will be opened + * @fop: the 'struct file_operations' for the new file + */ +static struct file *alloc_file(const struct path *path, int flags, + const struct file_operations *fop) +{ + struct file *file; + + file = alloc_empty_file(flags, current_cred()); + if (!IS_ERR(file)) + file_init_path(file, path, fop); + return file; +} + +static inline int alloc_path_pseudo(const char *name, struct inode *inode, + struct vfsmount *mnt, struct path *path) +{ + if (WARN_ON_ONCE(S_ISDIR(inode->i_mode))) + return -EINVAL; + path->dentry = d_alloc_pseudo(mnt->mnt_sb, &QSTR(name)); + if (!path->dentry) + return -ENOMEM; + path->mnt = mntget(mnt); + d_instantiate(path->dentry, inode); + return 0; +} + +struct file *alloc_file_pseudo(struct inode *inode, struct vfsmount *mnt, + const char *name, int flags, + const struct file_operations *fops) +{ + int ret; + struct path path; + struct file *file; + + ret = alloc_path_pseudo(name, inode, mnt, &path); + if (ret) + return ERR_PTR(ret); + + file = alloc_file(&path, flags, fops); + if (IS_ERR(file)) { + ihold(inode); + path_put(&path); + return file; + } + /* + * Disable all fsnotify events for pseudo files by default. + * They may be enabled by caller with file_set_fsnotify_mode(). + */ + file_set_fsnotify_mode(file, FMODE_NONOTIFY); + return file; +} +EXPORT_SYMBOL(alloc_file_pseudo); + +struct file *alloc_file_pseudo_noaccount(struct inode *inode, + struct vfsmount *mnt, const char *name, + int flags, + const struct file_operations *fops) +{ + int ret; + struct path path; + struct file *file; + + ret = alloc_path_pseudo(name, inode, mnt, &path); + if (ret) + return ERR_PTR(ret); + + file = alloc_empty_file_noaccount(flags, current_cred()); + if (IS_ERR(file)) { + ihold(inode); + path_put(&path); + return file; + } + file_init_path(file, &path, fops); + /* + * Disable all fsnotify events for pseudo files by default. + * They may be enabled by caller with file_set_fsnotify_mode(). + */ + file_set_fsnotify_mode(file, FMODE_NONOTIFY); + return file; +} +EXPORT_SYMBOL_GPL(alloc_file_pseudo_noaccount); + +struct file *alloc_file_clone(struct file *base, int flags, + const struct file_operations *fops) +{ + struct file *f; + + f = alloc_file(&base->f_path, flags, fops); + if (!IS_ERR(f)) { + path_get(&f->f_path); + f->f_mapping = base->f_mapping; + } + return f; +} + +/* the real guts of fput() - releasing the last reference to file + */ +static void __fput(struct file *file) +{ + struct dentry *dentry = file->f_path.dentry; + struct vfsmount *mnt = file->f_path.mnt; + struct inode *inode = file->f_inode; + fmode_t mode = file->f_mode; + + if (unlikely(!(file->f_mode & FMODE_OPENED))) + goto out; + + might_sleep(); + + fsnotify_close(file); + /* + * The function eventpoll_release() should be the first called + * in the file cleanup chain. + */ + eventpoll_release(file); + locks_remove_file(file); + + security_file_release(file); + if (unlikely(file->f_flags & FASYNC)) { + if (file->f_op->fasync) + file->f_op->fasync(-1, file, 0); + } + if (file->f_op->release) + file->f_op->release(inode, file); + if (unlikely(S_ISCHR(inode->i_mode) && inode->i_cdev != NULL && + !(mode & FMODE_PATH))) { + cdev_put(inode->i_cdev); + } + fops_put(file->f_op); + file_f_owner_release(file); + put_file_access(file); + dput(dentry); + if (unlikely(mode & FMODE_NEED_UNMOUNT)) + dissolve_on_fput(mnt); + mntput(mnt); +out: + file_free(file); +} + +static LLIST_HEAD(delayed_fput_list); +static void delayed_fput(struct work_struct *unused) +{ + struct llist_node *node = llist_del_all(&delayed_fput_list); + struct file *f, *t; + + llist_for_each_entry_safe(f, t, node, f_llist) + __fput(f); +} + +static void ____fput(struct callback_head *work) +{ + __fput(container_of(work, struct file, f_task_work)); +} + +static DECLARE_DELAYED_WORK(delayed_fput_work, delayed_fput); + +/* + * If kernel thread really needs to have the final fput() it has done + * to complete, call this. The only user right now is the boot - we + * *do* need to make sure our writes to binaries on initramfs has + * not left us with opened struct file waiting for __fput() - execve() + * won't work without that. Please, don't add more callers without + * very good reasons; in particular, never call that with locks + * held and never call that from a thread that might need to do + * some work on any kind of umount. + */ +void flush_delayed_fput(void) +{ + delayed_fput(NULL); + flush_delayed_work(&delayed_fput_work); +} +EXPORT_SYMBOL_GPL(flush_delayed_fput); + +static void __fput_deferred(struct file *file) +{ + struct task_struct *task = current; + + if (unlikely(!(file->f_mode & (FMODE_BACKING | FMODE_OPENED)))) { + file_free(file); + return; + } + + if (likely(!in_interrupt() && !(task->flags & PF_KTHREAD))) { + init_task_work(&file->f_task_work, ____fput); + if (!task_work_add(task, &file->f_task_work, TWA_RESUME)) + return; + /* + * After this task has run exit_task_work(), + * task_work_add() will fail. Fall through to delayed + * fput to avoid leaking *file. + */ + } + + if (llist_add(&file->f_llist, &delayed_fput_list)) + schedule_delayed_work(&delayed_fput_work, 1); +} + +void fput(struct file *file) +{ + if (unlikely(file_ref_put(&file->f_ref))) + __fput_deferred(file); +} +EXPORT_SYMBOL(fput); + +/* + * synchronous analog of fput(); for kernel threads that might be needed + * in some umount() (and thus can't use flush_delayed_fput() without + * risking deadlocks), need to wait for completion of __fput() and know + * for this specific struct file it won't involve anything that would + * need them. Use only if you really need it - at the very least, + * don't blindly convert fput() by kernel thread to that. + */ +void __fput_sync(struct file *file) +{ + if (file_ref_put(&file->f_ref)) + __fput(file); +} +EXPORT_SYMBOL(__fput_sync); + +/* + * Equivalent to __fput_sync(), but optimized for being called with the last + * reference. + * + * See file_ref_put_close() for details. + */ +void fput_close_sync(struct file *file) +{ + if (likely(file_ref_put_close(&file->f_ref))) + __fput(file); +} + +/* + * Equivalent to fput(), but optimized for being called with the last + * reference. + * + * See file_ref_put_close() for details. + */ +void fput_close(struct file *file) +{ + if (file_ref_put_close(&file->f_ref)) + __fput_deferred(file); +} + +void __init files_init(void) +{ + struct kmem_cache_args args = { + .use_freeptr_offset = true, + .freeptr_offset = offsetof(struct file, f_freeptr), + }; + + __filp_cache = kmem_cache_create("filp", sizeof(struct file), &args, + SLAB_HWCACHE_ALIGN | SLAB_PANIC | + SLAB_ACCOUNT | SLAB_TYPESAFE_BY_RCU); + runtime_const_init(ptr, __filp_cache); + + args.freeptr_offset = offsetof(struct backing_file, bf_freeptr); + __bfilp_cache = kmem_cache_create("bfilp", sizeof(struct backing_file), + &args, SLAB_HWCACHE_ALIGN | SLAB_PANIC | + SLAB_ACCOUNT | SLAB_TYPESAFE_BY_RCU); + runtime_const_init(ptr, __bfilp_cache); + + percpu_counter_init(&nr_files, 0, GFP_KERNEL); +} + +/* + * One file with associated inode and dcache is very roughly 1K. Per default + * do not use more than 10% of our memory for files. + */ +void __init files_maxfiles_init(void) +{ + unsigned long n; + unsigned long nr_pages = totalram_pages(); + unsigned long memreserve = (nr_pages - nr_free_pages()) * 3/2; + + memreserve = min(memreserve, nr_pages - 1); + n = ((nr_pages - memreserve) * (PAGE_SIZE / 1024)) / 10; + + files_stat.max_files = max_t(unsigned long, n, NR_FILE); +} diff --git a/fs/filesystems.c b/fs/filesystems.c new file mode 100644 index 000000000..3083d904d --- /dev/null +++ b/fs/filesystems.c @@ -0,0 +1,413 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/filesystems.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * table of configured filesystems + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Read-mostly filesystem drivers list. + * + * Readers walk under rcu_read_lock(); writers take file_systems_lock + * and publish via _rcu hlist primitives. unregister_filesystem() + * synchronize_rcu()s after unlock so the embedded file_system_type + * can't go away under a reader. To keep using a filesystem after + * the RCU section ends, take a module reference via try_module_get(). + */ +static HLIST_HEAD(file_systems); +static DEFINE_SPINLOCK(file_systems_lock); + +#ifdef CONFIG_PROC_FS +/* + * Cache a stringified version of the filesystem list. + * + * The fs list gets queried a lot by userspace because of libselinux, including + * rather surprising programs (would you guess *sed* is on the list?). In order + * to reduce the overhead we cache the resulting string, which normally hangs + * around below 512 bytes in size. + * + * As the list almost never changes, its creation is not particularly optimized + * to keep things simple. + * + * We sort it out on read in order to not introduce a failure point for fs + * registration (in principle we may be unable to alloc memory for the list). + */ +struct file_systems_string { + struct rcu_head rcu; + unsigned long gen; + size_t len; + char string[]; +}; + +static unsigned long file_systems_gen; +static struct file_systems_string __read_mostly __rcu *file_systems_string; + +static void invalidate_filesystems_string(void); +#else +static inline void invalidate_filesystems_string(void) { } +#endif + +/* WARNING: This can be used only if we _already_ own a reference */ +struct file_system_type *get_filesystem(struct file_system_type *fs) +{ + __module_get(fs->owner); + return fs; +} + +void put_filesystem(struct file_system_type *fs) +{ + module_put(fs->owner); +} + +static struct file_system_type *find_filesystem(const char *name, unsigned len) +{ + struct file_system_type *fs; + + hlist_for_each_entry_rcu(fs, &file_systems, list, + lockdep_is_held(&file_systems_lock)) + if (strncmp(fs->name, name, len) == 0 && !fs->name[len]) + return fs; + return NULL; +} + +/** + * register_filesystem - register a new filesystem + * @fs: the file system structure + * + * Adds the file system passed to the list of file systems the kernel + * is aware of for mount and other syscalls. Returns 0 on success, + * or a negative errno code on an error. + * + * The &struct file_system_type that is passed is linked into the kernel + * structures and must not be freed until the file system has been + * unregistered. + */ +int register_filesystem(struct file_system_type *fs) +{ + if (fs->parameters && + !fs_validate_description(fs->name, fs->parameters)) + return -EINVAL; + + BUG_ON(strchr(fs->name, '.')); + if (!hlist_unhashed_lockless(&fs->list)) + return -EBUSY; + + guard(spinlock)(&file_systems_lock); + if (find_filesystem(fs->name, strlen(fs->name))) + return -EBUSY; + hlist_add_tail_rcu(&fs->list, &file_systems); + invalidate_filesystems_string(); + return 0; +} +EXPORT_SYMBOL(register_filesystem); + +/** + * unregister_filesystem - unregister a file system + * @fs: filesystem to unregister + * + * Remove a file system that was previously successfully registered + * with the kernel. An error is returned if the file system is not found. + * Zero is returned on a success. + * + * Once this function has returned the &struct file_system_type structure + * may be freed or reused. + */ +int unregister_filesystem(struct file_system_type *fs) +{ + scoped_guard(spinlock, &file_systems_lock) { + if (hlist_unhashed(&fs->list)) + return -EINVAL; + hlist_del_init_rcu(&fs->list); + invalidate_filesystems_string(); + } + synchronize_rcu(); + return 0; +} +EXPORT_SYMBOL(unregister_filesystem); + +#ifdef CONFIG_SYSFS_SYSCALL +static int fs_index(const char __user *__name) +{ + struct file_system_type *p; + char *name __free(kfree) = strndup_user(__name, PATH_MAX); + int index = 0; + + if (IS_ERR(name)) + return PTR_ERR(name); + + guard(rcu)(); + hlist_for_each_entry_rcu(p, &file_systems, list) { + if (strcmp(p->name, name) == 0) + return index; + index++; + } + return -EINVAL; +} + +static int fs_name(unsigned int index, char __user *buf) +{ + struct file_system_type *p, *found = NULL; + int len, res; + + scoped_guard(rcu) { + hlist_for_each_entry_rcu(p, &file_systems, list) { + if (index--) + continue; + if (try_module_get(p->owner)) + found = p; + break; + } + } + if (!found) + return -EINVAL; + + /* OK, we got the reference, so we can safely block */ + len = strlen(found->name) + 1; + res = copy_to_user(buf, found->name, len) ? -EFAULT : 0; + put_filesystem(found); + return res; +} + +static int fs_maxindex(void) +{ + struct file_system_type *p; + int index = 0; + + guard(rcu)(); + hlist_for_each_entry_rcu(p, &file_systems, list) + index++; + return index; +} + +/* + * Whee.. Weird sysv syscall. + */ +SYSCALL_DEFINE3(sysfs, int, option, unsigned long, arg1, unsigned long, arg2) +{ + int retval = -EINVAL; + + switch (option) { + case 1: + retval = fs_index((const char __user *) arg1); + break; + + case 2: + retval = fs_name(arg1, (char __user *) arg2); + break; + + case 3: + retval = fs_maxindex(); + break; + } + return retval; +} +#endif + +int __init list_bdev_fs_names(char *buf, size_t size) +{ + struct file_system_type *p; + size_t len; + int count = 0; + + guard(rcu)(); + hlist_for_each_entry_rcu(p, &file_systems, list) { + if (!(p->fs_flags & FS_REQUIRES_DEV)) + continue; + len = strlen(p->name) + 1; + if (len > size) { + pr_warn("%s: truncating file system list\n", __func__); + break; + } + memcpy(buf, p->name, len); + buf += len; + size -= len; + count++; + } + return count; +} + +#ifdef CONFIG_PROC_FS +static void invalidate_filesystems_string(void) +{ + struct file_systems_string *old; + + lockdep_assert_held_write(&file_systems_lock); + file_systems_gen++; + old = rcu_replace_pointer(file_systems_string, NULL, + lockdep_is_held(&file_systems_lock)); + if (old) + kfree_rcu(old, rcu); +} + +static __cold noinline int regen_filesystems_string(void) +{ + struct file_system_type *p; + struct file_systems_string *old, *new; + size_t newlen, usedlen; + unsigned long gen; + +retry: + newlen = 0; + + /* pre-calc space for each fs */ + spin_lock(&file_systems_lock); + gen = file_systems_gen; + hlist_for_each_entry_rcu(p, &file_systems, list) { + if (!(p->fs_flags & FS_REQUIRES_DEV)) + newlen += strlen("nodev"); + newlen += strlen("\t") + strlen(p->name) + strlen("\n"); + } + spin_unlock(&file_systems_lock); + + new = kmalloc(offsetof(struct file_systems_string, string) + newlen + 1, + GFP_KERNEL); + if (!new) + return -ENOMEM; + + new->gen = gen; + new->len = newlen; + new->string[newlen] = '\0'; + + spin_lock(&file_systems_lock); + old = file_systems_string; + + /* + * Did someone beat us to it? + */ + if (old && old->gen == file_systems_gen) { + spin_unlock(&file_systems_lock); + kfree(new); + return 0; + } + + /* + * Did the list change in the meantime? + */ + if (gen != file_systems_gen) { + spin_unlock(&file_systems_lock); + kfree(new); + goto retry; + } + + /* + * Populate the string. + * + * We know we have just enough space because we calculated the right + * size the previous time we had the lock and confirmed the list has + * not changed after reacquiring it. + */ + usedlen = 0; + hlist_for_each_entry_rcu(p, &file_systems, list) { + usedlen += sprintf(&new->string[usedlen], "%s\t%s\n", + (p->fs_flags & FS_REQUIRES_DEV) ? "" : "nodev", + p->name); + } + + if (WARN_ON_ONCE(new->len != strlen(new->string))) { + /* + * Should never happen of course, keep this in case someone changes string + * generation above and messes it up. + */ + spin_unlock(&file_systems_lock); + kfree(new); + return -EINVAL; + } + + rcu_assign_pointer(file_systems_string, new); + spin_unlock(&file_systems_lock); + if (old) + kfree_rcu(old, rcu); + return 0; +} + +static __cold noinline int filesystems_proc_show_fallback(struct seq_file *m, void *v) +{ + struct file_system_type *p; + + guard(rcu)(); + hlist_for_each_entry_rcu(p, &file_systems, list) { + seq_printf(m, "%s\t%s\n", + (p->fs_flags & FS_REQUIRES_DEV) ? "" : "nodev", + p->name); + } + return 0; +} + +static int filesystems_proc_show(struct seq_file *m, void *v) +{ + struct file_systems_string *fss; + + for (;;) { + scoped_guard(rcu) { + fss = rcu_dereference(file_systems_string); + if (likely(fss)) { + seq_write(m, fss->string, fss->len); + return 0; + } + } + + int err = regen_filesystems_string(); + if (unlikely(err)) + return filesystems_proc_show_fallback(m, v); + } +} + +static int __init proc_filesystems_init(void) +{ + struct proc_dir_entry *pde; + + pde = proc_create_single("filesystems", 0, NULL, filesystems_proc_show); + if (!pde) + return -ENOMEM; + proc_make_permanent(pde); + return 0; +} +module_init(proc_filesystems_init); +#endif + +static struct file_system_type *__get_fs_type(const char *name, int len) +{ + struct file_system_type *fs; + + guard(rcu)(); + fs = find_filesystem(name, len); + if (fs && !try_module_get(fs->owner)) + fs = NULL; + return fs; +} + +struct file_system_type *get_fs_type(const char *name) +{ + struct file_system_type *fs; + const char *dot = strchr(name, '.'); + int len = dot ? dot - name : strlen(name); + + fs = __get_fs_type(name, len); + if (!fs && (request_module("fs-%.*s", len, name) == 0)) { + fs = __get_fs_type(name, len); + if (!fs) + pr_warn_once("request_module fs-%.*s succeeded, but still no fs?\n", + len, name); + } + + if (dot && fs && !(fs->fs_flags & FS_HAS_SUBTYPE)) { + put_filesystem(fs); + fs = NULL; + } + return fs; +} +EXPORT_SYMBOL(get_fs_type); diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c new file mode 100644 index 000000000..e744f9f9d --- /dev/null +++ b/fs/fs-writeback.c @@ -0,0 +1,3102 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * fs/fs-writeback.c + * + * Copyright (C) 2002, Linus Torvalds. + * + * Contains all the functions related to writing back and waiting + * upon dirty inodes against superblocks, and writing back dirty + * pages against inodes. ie: data writeback. Writeout of the + * inode itself is not handled here. + * + * 10Apr2002 Andrew Morton + * Split out of fs/inode.c + * Additions for address_space-based writeback + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +/* + * Passed into wb_writeback(), essentially a subset of writeback_control + */ +struct wb_writeback_work { + long nr_pages; + struct super_block *sb; + enum writeback_sync_modes sync_mode; + unsigned int tagged_writepages:1; + unsigned int for_kupdate:1; + unsigned int range_cyclic:1; + unsigned int for_background:1; + unsigned int for_sync:1; /* sync(2) WB_SYNC_ALL writeback */ + unsigned int auto_free:1; /* free on completion */ + enum wb_reason reason; /* why was writeback initiated? */ + + struct list_head list; /* pending work list */ + struct wb_completion *done; /* set if the caller waits */ +}; + +/* + * If an inode is constantly having its pages dirtied, but then the + * updates stop dirtytime_expire_interval seconds in the past, it's + * possible for the worst case time between when an inode has its + * timestamps updated and when they finally get written out to be two + * dirtytime_expire_intervals. We set the default to 12 hours (in + * seconds), which means most of the time inodes will have their + * timestamps written to disk after 12 hours, but in the worst case a + * few inodes might not their timestamps updated for 24 hours. + */ +static unsigned int dirtytime_expire_interval = 12 * 60 * 60; + +static inline struct inode *wb_inode(struct list_head *head) +{ + return list_entry(head, struct inode, i_io_list); +} + +/* + * Include the creation of the trace points after defining the + * wb_writeback_work structure and inline functions so that the definition + * remains local to this file. + */ +#define CREATE_TRACE_POINTS +#include + +EXPORT_TRACEPOINT_SYMBOL_GPL(wbc_writepage); + +static bool wb_io_lists_populated(struct bdi_writeback *wb) +{ + if (wb_has_dirty_io(wb)) { + return false; + } else { + set_bit(WB_has_dirty_io, &wb->state); + WARN_ON_ONCE(!wb->avg_write_bandwidth); + atomic_long_add(wb->avg_write_bandwidth, + &wb->bdi->tot_write_bandwidth); + return true; + } +} + +static void wb_io_lists_depopulated(struct bdi_writeback *wb) +{ + if (wb_has_dirty_io(wb) && list_empty(&wb->b_dirty) && + list_empty(&wb->b_io) && list_empty(&wb->b_more_io)) { + clear_bit(WB_has_dirty_io, &wb->state); + WARN_ON_ONCE(atomic_long_sub_return(wb->avg_write_bandwidth, + &wb->bdi->tot_write_bandwidth) < 0); + } +} + +/** + * inode_io_list_move_locked - move an inode onto a bdi_writeback IO list + * @inode: inode to be moved + * @wb: target bdi_writeback + * @head: one of @wb->b_{dirty|io|more_io|dirty_time} + * + * Move @inode->i_io_list to @list of @wb and set %WB_has_dirty_io. + * Returns %true if @inode is the first occupant of the !dirty_time IO + * lists; otherwise, %false. + */ +static bool inode_io_list_move_locked(struct inode *inode, + struct bdi_writeback *wb, + struct list_head *head) +{ + assert_spin_locked(&wb->list_lock); + assert_spin_locked(&inode->i_lock); + WARN_ON_ONCE(inode_state_read(inode) & I_FREEING); + + list_move(&inode->i_io_list, head); + + /* dirty_time doesn't count as dirty_io until expiration */ + if (head != &wb->b_dirty_time) + return wb_io_lists_populated(wb); + + wb_io_lists_depopulated(wb); + return false; +} + +static void wb_wakeup(struct bdi_writeback *wb) +{ + spin_lock_irq(&wb->work_lock); + if (test_bit(WB_registered, &wb->state)) + mod_delayed_work(bdi_wq, &wb->dwork, 0); + spin_unlock_irq(&wb->work_lock); +} + +/* + * This function is used when the first inode for this wb is marked dirty. It + * wakes-up the corresponding bdi thread which should then take care of the + * periodic background write-out of dirty inodes. Since the write-out would + * starts only 'dirty_writeback_interval' centisecs from now anyway, we just + * set up a timer which wakes the bdi thread up later. + * + * Note, we wouldn't bother setting up the timer, but this function is on the + * fast-path (used by '__mark_inode_dirty()'), so we save few context switches + * by delaying the wake-up. + * + * We have to be careful not to postpone flush work if it is scheduled for + * earlier. Thus we use queue_delayed_work(). + */ +static void wb_wakeup_delayed(struct bdi_writeback *wb) +{ + unsigned long timeout; + + timeout = msecs_to_jiffies(dirty_writeback_interval * 10); + spin_lock_irq(&wb->work_lock); + if (test_bit(WB_registered, &wb->state)) + queue_delayed_work(bdi_wq, &wb->dwork, timeout); + spin_unlock_irq(&wb->work_lock); +} + +static void finish_writeback_work(struct wb_writeback_work *work) +{ + struct wb_completion *done = work->done; + + if (work->auto_free) + kfree(work); + if (done) { + wait_queue_head_t *waitq = done->waitq; + + /* @done can't be accessed after the following dec */ + if (atomic_dec_and_test(&done->cnt)) + wake_up_all(waitq); + } +} + +static void wb_queue_work(struct bdi_writeback *wb, + struct wb_writeback_work *work) +{ + trace_writeback_queue(wb, work); + + if (work->done) + atomic_inc(&work->done->cnt); + + spin_lock_irq(&wb->work_lock); + + if (test_bit(WB_registered, &wb->state)) { + list_add_tail(&work->list, &wb->work_list); + mod_delayed_work(bdi_wq, &wb->dwork, 0); + } else + finish_writeback_work(work); + + spin_unlock_irq(&wb->work_lock); +} + +static bool wb_wait_for_completion_cb(struct wb_completion *done) +{ + unsigned long timeout = sysctl_hung_task_timeout_secs; + unsigned long waited_secs = (jiffies - done->wait_start) / HZ; + + done->progress_stamp = jiffies; + if (timeout && (waited_secs > timeout)) + pr_info("INFO: The task %s:%d has been waiting for writeback " + "completion for more than %lu seconds.", + current->comm, current->pid, waited_secs); + + return !atomic_read(&done->cnt); +} + +/** + * wb_wait_for_completion - wait for completion of bdi_writeback_works + * @done: target wb_completion + * + * Wait for one or more work items issued to @bdi with their ->done field + * set to @done, which should have been initialized with + * DEFINE_WB_COMPLETION(). This function returns after all such work items + * are completed. Work items which are waited upon aren't freed + * automatically on completion. + */ +void wb_wait_for_completion(struct wb_completion *done) +{ + done->wait_start = jiffies; + atomic_dec(&done->cnt); /* put down the initial count */ + wait_event(*done->waitq, wb_wait_for_completion_cb(done)); +} + +#ifdef CONFIG_CGROUP_WRITEBACK + +/* + * Parameters for foreign inode detection, see wbc_detach_inode() to see + * how they're used. + * + * These paramters are inherently heuristical as the detection target + * itself is fuzzy. All we want to do is detaching an inode from the + * current owner if it's being written to by some other cgroups too much. + * + * The current cgroup writeback is built on the assumption that multiple + * cgroups writing to the same inode concurrently is very rare and a mode + * of operation which isn't well supported. As such, the goal is not + * taking too long when a different cgroup takes over an inode while + * avoiding too aggressive flip-flops from occasional foreign writes. + * + * We record, very roughly, 2s worth of IO time history and if more than + * half of that is foreign, trigger the switch. The recording is quantized + * to 16 slots. To avoid tiny writes from swinging the decision too much, + * writes smaller than 1/8 of avg size are ignored. + */ +#define WB_FRN_TIME_SHIFT 13 /* 1s = 2^13, upto 8 secs w/ 16bit */ +#define WB_FRN_TIME_AVG_SHIFT 3 /* avg = avg * 7/8 + new * 1/8 */ +#define WB_FRN_TIME_CUT_DIV 8 /* ignore rounds < avg / 8 */ +#define WB_FRN_TIME_PERIOD (2 * (1 << WB_FRN_TIME_SHIFT)) /* 2s */ + +#define WB_FRN_HIST_SLOTS 16 /* inode->i_wb_frn_history is 16bit */ +#define WB_FRN_HIST_UNIT (WB_FRN_TIME_PERIOD / WB_FRN_HIST_SLOTS) + /* each slot's duration is 2s / 16 */ +#define WB_FRN_HIST_THR_SLOTS (WB_FRN_HIST_SLOTS / 2) + /* if foreign slots >= 8, switch */ +#define WB_FRN_HIST_MAX_SLOTS (WB_FRN_HIST_THR_SLOTS / 2 + 1) + /* one round can affect upto 5 slots */ +#define WB_FRN_MAX_IN_FLIGHT 1024 /* don't queue too many concurrently */ + +/* + * Maximum inodes per isw. A specific value has been chosen to make + * struct inode_switch_wbs_context fit into 1024 bytes kmalloc. + */ +#define WB_MAX_INODES_PER_ISW ((1024UL - sizeof(struct inode_switch_wbs_context)) \ + / sizeof(struct inode *)) + +static atomic_t isw_nr_in_flight = ATOMIC_INIT(0); +static struct workqueue_struct *isw_wq; + +void __inode_attach_wb(struct inode *inode, struct folio *folio) +{ + struct backing_dev_info *bdi = inode_to_bdi(inode); + struct bdi_writeback *wb = NULL; + + if (inode_cgwb_enabled(inode)) { + struct cgroup_subsys_state *memcg_css; + + /* must pin memcg_css, see wb_get_create() */ + if (folio) + memcg_css = get_mem_cgroup_css_from_folio(folio); + else + memcg_css = task_get_css(current, memory_cgrp_id); + wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC); + css_put(memcg_css); + } + + if (!wb) + wb = &bdi->wb; + + /* + * There may be multiple instances of this function racing to + * update the same inode. Use cmpxchg() to tell the winner. + */ + if (unlikely(cmpxchg(&inode->i_wb, NULL, wb))) + wb_put(wb); +} +EXPORT_SYMBOL_GPL(__inode_attach_wb); + +/** + * inode_cgwb_move_to_attached - put the inode onto wb->b_attached list + * @inode: inode of interest with i_lock held + * @wb: target bdi_writeback + * + * Remove the inode from wb's io lists and if necessarily put onto b_attached + * list. Only inodes attached to cgwb's are kept on this list. + */ +static void inode_cgwb_move_to_attached(struct inode *inode, + struct bdi_writeback *wb) +{ + assert_spin_locked(&wb->list_lock); + assert_spin_locked(&inode->i_lock); + WARN_ON_ONCE(inode_state_read(inode) & I_FREEING); + + inode_state_clear(inode, I_SYNC_QUEUED); + if (wb != &wb->bdi->wb) + list_move(&inode->i_io_list, &wb->b_attached); + else + list_del_init(&inode->i_io_list); + wb_io_lists_depopulated(wb); +} + +/** + * locked_inode_to_wb_and_lock_list - determine a locked inode's wb and lock it + * @inode: inode of interest with i_lock held + * + * Returns @inode's wb with its list_lock held. @inode->i_lock must be + * held on entry and is released on return. The returned wb is guaranteed + * to stay @inode's associated wb until its list_lock is released. + */ +static struct bdi_writeback * +locked_inode_to_wb_and_lock_list(struct inode *inode) + __releases(&inode->i_lock) + __acquires(&wb->list_lock) +{ + while (true) { + struct bdi_writeback *wb = inode_to_wb(inode); + + /* + * inode_to_wb() association is protected by both + * @inode->i_lock and @wb->list_lock but list_lock nests + * outside i_lock. Drop i_lock and verify that the + * association hasn't changed after acquiring list_lock. + */ + wb_get(wb); + spin_unlock(&inode->i_lock); + spin_lock(&wb->list_lock); + + /* i_wb may have changed inbetween, can't use inode_to_wb() */ + if (likely(wb == inode->i_wb)) { + wb_put(wb); /* @inode already has ref */ + return wb; + } + + spin_unlock(&wb->list_lock); + wb_put(wb); + cpu_relax(); + spin_lock(&inode->i_lock); + } +} + +/** + * inode_to_wb_and_lock_list - determine an inode's wb and lock it + * @inode: inode of interest + * + * Same as locked_inode_to_wb_and_lock_list() but @inode->i_lock isn't held + * on entry. + */ +static struct bdi_writeback *inode_to_wb_and_lock_list(struct inode *inode) + __acquires(&wb->list_lock) +{ + spin_lock(&inode->i_lock); + return locked_inode_to_wb_and_lock_list(inode); +} + +struct inode_switch_wbs_context { + /* List of queued switching contexts for the wb */ + struct llist_node list; + + /* + * Multiple inodes can be switched at once. The switching procedure + * consists of two parts, separated by a RCU grace period. To make + * sure that the second part is executed for each inode gone through + * the first part, all inode pointers are placed into a NULL-terminated + * array embedded into struct inode_switch_wbs_context. Otherwise + * an inode could be left in a non-consistent state. + */ + struct inode *inodes[]; +}; + +static void bdi_down_write_wb_switch_rwsem(struct backing_dev_info *bdi) +{ + down_write(&bdi->wb_switch_rwsem); +} + +static void bdi_up_write_wb_switch_rwsem(struct backing_dev_info *bdi) +{ + up_write(&bdi->wb_switch_rwsem); +} + +static bool inode_do_switch_wbs(struct inode *inode, + struct bdi_writeback *old_wb, + struct bdi_writeback *new_wb) +{ + struct address_space *mapping = inode->i_mapping; + XA_STATE(xas, &mapping->i_pages, 0); + struct folio *folio; + bool switched = false; + + spin_lock(&inode->i_lock); + xa_lock_irq(&mapping->i_pages); + + /* + * Once I_FREEING or I_WILL_FREE are visible under i_lock, the eviction + * path owns the inode and we shouldn't modify ->i_io_list. + */ + if (unlikely(inode_state_read(inode) & (I_FREEING | I_WILL_FREE))) + goto skip_switch; + + trace_inode_switch_wbs(inode, old_wb, new_wb); + + /* + * Count and transfer stats. Note that PAGECACHE_TAG_DIRTY points + * to possibly dirty folios while PAGECACHE_TAG_WRITEBACK points to + * folios actually under writeback. + */ + xas_for_each_marked(&xas, folio, ULONG_MAX, PAGECACHE_TAG_DIRTY) { + if (folio_test_dirty(folio)) { + long nr = folio_nr_pages(folio); + wb_stat_mod(old_wb, WB_RECLAIMABLE, -nr); + wb_stat_mod(new_wb, WB_RECLAIMABLE, nr); + if (folio_test_dropbehind(folio)) { + wb_stat_mod(old_wb, WB_DONTCACHE_DIRTY, -nr); + wb_stat_mod(new_wb, WB_DONTCACHE_DIRTY, nr); + } + } + } + + xas_set(&xas, 0); + xas_for_each_marked(&xas, folio, ULONG_MAX, PAGECACHE_TAG_WRITEBACK) { + long nr = folio_nr_pages(folio); + WARN_ON_ONCE(!folio_test_writeback(folio)); + wb_stat_mod(old_wb, WB_WRITEBACK, -nr); + wb_stat_mod(new_wb, WB_WRITEBACK, nr); + } + + if (mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK)) { + atomic_dec(&old_wb->writeback_inodes); + atomic_inc(&new_wb->writeback_inodes); + } + + wb_get(new_wb); + + /* + * Transfer to @new_wb's IO list if necessary. If the @inode is dirty, + * the specific list @inode was on is ignored and the @inode is put on + * ->b_dirty which is always correct including from ->b_dirty_time. + * If the @inode was clean, it means it was on the b_attached list, so + * move it onto the b_attached list of @new_wb. + */ + if (!list_empty(&inode->i_io_list)) { + inode->i_wb = new_wb; + + if (inode_state_read(inode) & I_DIRTY_ALL) { + /* + * We need to keep b_dirty list sorted by + * dirtied_time_when. However properly sorting the + * inode in the list gets too expensive when switching + * many inodes. So just attach inode at the end of the + * dirty list and clobber the dirtied_time_when. + */ + inode->dirtied_time_when = jiffies; + inode_io_list_move_locked(inode, new_wb, + &new_wb->b_dirty); + } else { + inode_cgwb_move_to_attached(inode, new_wb); + } + } else { + inode->i_wb = new_wb; + } + + /* ->i_wb_frn updates may race wbc_detach_inode() but doesn't matter */ + inode->i_wb_frn_winner = 0; + inode->i_wb_frn_avg_time = 0; + inode->i_wb_frn_history = 0; + switched = true; +skip_switch: + /* + * Paired with an acquire fence in unlocked_inode_to_wb_begin() and + * ensures that the new wb is visible if they see !I_WB_SWITCH. + */ + smp_wmb(); + inode_state_clear(inode, I_WB_SWITCH); + + xa_unlock_irq(&mapping->i_pages); + spin_unlock(&inode->i_lock); + + return switched; +} + +static inline void cgroup_writeback_pin(struct super_block *sb) +{ + atomic_inc(&sb->s_isw_nr_in_flight); +} + +static inline void cgroup_writeback_unpin(struct super_block *sb) +{ + if (atomic_dec_and_test(&sb->s_isw_nr_in_flight)) + wake_up_var(&sb->s_isw_nr_in_flight); +} + +static inline void cgroup_writeback_drain(struct super_block *sb) +{ + wait_var_event(&sb->s_isw_nr_in_flight, + !atomic_read(&sb->s_isw_nr_in_flight)); +} + +static void process_inode_switch_wbs(struct bdi_writeback *new_wb, + struct inode_switch_wbs_context *isw) +{ + struct backing_dev_info *bdi = inode_to_bdi(isw->inodes[0]); + struct bdi_writeback *old_wb = isw->inodes[0]->i_wb; + unsigned long nr_switched = 0; + struct inode **inodep; + + /* + * If @inode switches cgwb membership while sync_inodes_sb() is + * being issued, sync_inodes_sb() might miss it. Synchronize. + */ + down_read(&bdi->wb_switch_rwsem); + + inodep = isw->inodes; + /* + * By the time control reaches here, RCU grace period has passed + * since I_WB_SWITCH assertion and all wb stat update transactions + * between unlocked_inode_to_wb_begin/end() are guaranteed to be + * synchronizing against the i_pages lock. + * + * Grabbing old_wb->list_lock, inode->i_lock and the i_pages lock + * gives us exclusion against all wb related operations on @inode + * including IO list manipulations and stat updates. + */ +relock: + if (old_wb < new_wb) { + spin_lock(&old_wb->list_lock); + spin_lock_nested(&new_wb->list_lock, SINGLE_DEPTH_NESTING); + } else { + spin_lock(&new_wb->list_lock); + spin_lock_nested(&old_wb->list_lock, SINGLE_DEPTH_NESTING); + } + + while (*inodep) { + WARN_ON_ONCE((*inodep)->i_wb != old_wb); + if (inode_do_switch_wbs(*inodep, old_wb, new_wb)) + nr_switched++; + inodep++; + if (*inodep && need_resched()) { + spin_unlock(&new_wb->list_lock); + spin_unlock(&old_wb->list_lock); + cond_resched(); + goto relock; + } + } + + spin_unlock(&new_wb->list_lock); + spin_unlock(&old_wb->list_lock); + + up_read(&bdi->wb_switch_rwsem); + + if (nr_switched) { + wb_wakeup(new_wb); + wb_put_many(old_wb, nr_switched); + } + + for (inodep = isw->inodes; *inodep; inodep++) { + struct super_block *sb = (*inodep)->i_sb; + + iput(*inodep); + cgroup_writeback_unpin(sb); + } + wb_put(new_wb); + kfree(isw); + atomic_dec(&isw_nr_in_flight); +} + +void inode_switch_wbs_work_fn(struct work_struct *work) +{ + struct bdi_writeback *new_wb = container_of(work, struct bdi_writeback, + switch_work); + struct inode_switch_wbs_context *isw, *next_isw; + struct llist_node *list; + + list = llist_del_all(&new_wb->switch_wbs_ctxs); + /* + * Nothing to do? That would be a problem as references held by isw + * items protect wb from freeing... + */ + if (WARN_ON_ONCE(!list)) + return; + + /* + * Grab our reference to wb so that it cannot get freed under us + * after we process all the isw items. + */ + wb_get(new_wb); + /* + * In addition to synchronizing among switchers, I_WB_SWITCH + * tells the RCU protected stat update paths to grab the i_page + * lock so that stat transfer can synchronize against them. + * Let's continue after I_WB_SWITCH is guaranteed to be + * visible. + */ + synchronize_rcu(); + + llist_for_each_entry_safe(isw, next_isw, list, list) + process_inode_switch_wbs(new_wb, isw); + wb_put(new_wb); +} + +static bool inode_prepare_wbs_switch(struct inode *inode, + struct bdi_writeback *new_wb) +{ + /* Avoid the atomic_inc/smp_mb dance once SB_ACTIVE is gone. */ + if (!(inode->i_sb->s_flags & SB_ACTIVE)) + return false; + + /* + * Pairs with smp_mb() in cgroup_writeback_umount(): the umounter either + * sees a non-zero counter and waits, or we see SB_ACTIVE clear below. + */ + cgroup_writeback_pin(inode->i_sb); + smp_mb(); + + if (IS_DAX(inode)) + goto out_unpin; + + /* while holding I_WB_SWITCH, no one else can update the association */ + spin_lock(&inode->i_lock); + if (!(inode->i_sb->s_flags & SB_ACTIVE) || + inode_state_read(inode) & (I_WB_SWITCH | I_FREEING | I_WILL_FREE) || + inode_to_wb(inode) == new_wb) { + spin_unlock(&inode->i_lock); + goto out_unpin; + } + inode_state_set(inode, I_WB_SWITCH); + __iget(inode); + spin_unlock(&inode->i_lock); + + return true; + +out_unpin: + cgroup_writeback_unpin(inode->i_sb); + return false; +} + +static void wb_queue_isw(struct bdi_writeback *wb, + struct inode_switch_wbs_context *isw) +{ + if (llist_add(&isw->list, &wb->switch_wbs_ctxs)) + queue_work(isw_wq, &wb->switch_work); +} + +/** + * inode_switch_wbs - change the wb association of an inode + * @inode: target inode + * @new_wb_id: ID of the new wb + * + * Switch @inode's wb association to the wb identified by @new_wb_id. The + * switching is performed asynchronously and may fail silently. + */ +static void inode_switch_wbs(struct inode *inode, int new_wb_id) +{ + struct backing_dev_info *bdi = inode_to_bdi(inode); + struct cgroup_subsys_state *memcg_css; + struct inode_switch_wbs_context *isw; + struct bdi_writeback *new_wb = NULL; + + /* noop if seems to be already in progress */ + if (inode_state_read_once(inode) & I_WB_SWITCH) + return; + + /* avoid queueing a new switch if too many are already in flight */ + if (atomic_read(&isw_nr_in_flight) > WB_FRN_MAX_IN_FLIGHT) + return; + + isw = kzalloc_flex(*isw, inodes, 2, GFP_ATOMIC); + if (!isw) + return; + + atomic_inc(&isw_nr_in_flight); + + /* find and pin the new wb */ + rcu_read_lock(); + memcg_css = css_from_id(new_wb_id, &memory_cgrp_subsys); + if (memcg_css && !css_tryget(memcg_css)) + memcg_css = NULL; + rcu_read_unlock(); + if (!memcg_css) + goto out_free; + + new_wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC); + css_put(memcg_css); + if (!new_wb) + goto out_free; + + if (!inode_prepare_wbs_switch(inode, new_wb)) + goto out_free; + + isw->inodes[0] = inode; + + trace_inode_switch_wbs_queue(inode->i_wb, new_wb, 1); + wb_queue_isw(new_wb, isw); + return; + +out_free: + atomic_dec(&isw_nr_in_flight); + if (new_wb) + wb_put(new_wb); + kfree(isw); +} + +static bool isw_prepare_wbs_switch(struct bdi_writeback *new_wb, + struct inode_switch_wbs_context *isw, + struct list_head *list, int *nr) +{ + struct inode *inode; + + list_for_each_entry(inode, list, i_io_list) { + if (!inode_prepare_wbs_switch(inode, new_wb)) + continue; + + isw->inodes[*nr] = inode; + (*nr)++; + + if (*nr >= WB_MAX_INODES_PER_ISW - 1) + return true; + } + return false; +} + +/** + * cleanup_offline_cgwb - detach associated inodes + * @wb: target wb + * + * Switch all inodes attached to @wb to a nearest living ancestor's wb in order + * to eventually release the dying @wb. Returns %true if not all inodes were + * switched and the function has to be restarted. + */ +bool cleanup_offline_cgwb(struct bdi_writeback *wb) +{ + struct cgroup_subsys_state *memcg_css; + struct inode_switch_wbs_context *isw; + struct bdi_writeback *new_wb; + int nr; + bool restart = false; + + isw = kzalloc_flex(*isw, inodes, WB_MAX_INODES_PER_ISW); + if (!isw) + return restart; + + atomic_inc(&isw_nr_in_flight); + + for (memcg_css = wb->memcg_css->parent; memcg_css; + memcg_css = memcg_css->parent) { + new_wb = wb_get_create(wb->bdi, memcg_css, GFP_KERNEL); + if (new_wb) + break; + } + if (unlikely(!new_wb)) + new_wb = &wb->bdi->wb; /* wb_get() is noop for bdi's wb */ + + nr = 0; + spin_lock(&wb->list_lock); + /* + * In addition to the inodes that have completed writeback, also switch + * cgwbs for those inodes only with dirty timestamps. Otherwise, those + * inodes won't be written back for a long time when lazytime is + * enabled, and thus pinning the dying cgwbs. It won't break the + * bandwidth restrictions, as writeback of inode metadata is not + * accounted for. + */ + restart = isw_prepare_wbs_switch(new_wb, isw, &wb->b_attached, &nr); + if (!restart) + restart = isw_prepare_wbs_switch(new_wb, isw, &wb->b_dirty_time, + &nr); + spin_unlock(&wb->list_lock); + + /* no attached inodes? bail out */ + if (nr == 0) { + atomic_dec(&isw_nr_in_flight); + wb_put(new_wb); + kfree(isw); + return restart; + } + + trace_inode_switch_wbs_queue(wb, new_wb, nr); + wb_queue_isw(new_wb, isw); + + return restart; +} + +/** + * wbc_attach_and_unlock_inode - associate wbc with target inode and unlock it + * @wbc: writeback_control of interest + * @inode: target inode + * + * @inode is locked and about to be written back under the control of @wbc. + * Record @inode's writeback context into @wbc and unlock the i_lock. On + * writeback completion, wbc_detach_inode() should be called. This is used + * to track the cgroup writeback context. + */ +static void wbc_attach_and_unlock_inode(struct writeback_control *wbc, + struct inode *inode) + __releases(&inode->i_lock) +{ + if (!inode_cgwb_enabled(inode)) { + spin_unlock(&inode->i_lock); + return; + } + + wbc->wb = inode_to_wb(inode); + wbc->inode = inode; + + wbc->wb_id = wbc->wb->memcg_css->id; + wbc->wb_lcand_id = inode->i_wb_frn_winner; + wbc->wb_tcand_id = 0; + wbc->wb_bytes = 0; + wbc->wb_lcand_bytes = 0; + wbc->wb_tcand_bytes = 0; + + wb_get(wbc->wb); + spin_unlock(&inode->i_lock); + + /* + * A dying wb indicates that either the blkcg associated with the + * memcg changed or the associated memcg is dying. In the first + * case, a replacement wb should already be available and we should + * refresh the wb immediately. In the second case, trying to + * refresh will keep failing. + */ + if (unlikely(wb_dying(wbc->wb) && !css_is_dying(wbc->wb->memcg_css))) + inode_switch_wbs(inode, wbc->wb_id); +} + +/** + * wbc_attach_fdatawrite_inode - associate wbc and inode for fdatawrite + * @wbc: writeback_control of interest + * @inode: target inode + * + * This function is to be used by filemap_writeback(), which is an alternative + * entry point into writeback code, and first ensures @inode is associated with + * a bdi_writeback and attaches it to @wbc. + */ +void wbc_attach_fdatawrite_inode(struct writeback_control *wbc, + struct inode *inode) +{ + spin_lock(&inode->i_lock); + inode_attach_wb(inode, NULL); + wbc_attach_and_unlock_inode(wbc, inode); +} +EXPORT_SYMBOL_GPL(wbc_attach_fdatawrite_inode); + +/** + * wbc_detach_inode - disassociate wbc from inode and perform foreign detection + * @wbc: writeback_control of the just finished writeback + * + * To be called after a writeback attempt of an inode finishes and undoes + * wbc_attach_and_unlock_inode(). Can be called under any context. + * + * As concurrent write sharing of an inode is expected to be very rare and + * memcg only tracks page ownership on first-use basis severely confining + * the usefulness of such sharing, cgroup writeback tracks ownership + * per-inode. While the support for concurrent write sharing of an inode + * is deemed unnecessary, an inode being written to by different cgroups at + * different points in time is a lot more common, and, more importantly, + * charging only by first-use can too readily lead to grossly incorrect + * behaviors (single foreign page can lead to gigabytes of writeback to be + * incorrectly attributed). + * + * To resolve this issue, cgroup writeback detects the majority dirtier of + * an inode and transfers the ownership to it. To avoid unnecessary + * oscillation, the detection mechanism keeps track of history and gives + * out the switch verdict only if the foreign usage pattern is stable over + * a certain amount of time and/or writeback attempts. + * + * On each writeback attempt, @wbc tries to detect the majority writer + * using Boyer-Moore majority vote algorithm. In addition to the byte + * count from the majority voting, it also counts the bytes written for the + * current wb and the last round's winner wb (max of last round's current + * wb, the winner from two rounds ago, and the last round's majority + * candidate). Keeping track of the historical winner helps the algorithm + * to semi-reliably detect the most active writer even when it's not the + * absolute majority. + * + * Once the winner of the round is determined, whether the winner is + * foreign or not and how much IO time the round consumed is recorded in + * inode->i_wb_frn_history. If the amount of recorded foreign IO time is + * over a certain threshold, the switch verdict is given. + */ +void wbc_detach_inode(struct writeback_control *wbc) +{ + struct bdi_writeback *wb = wbc->wb; + struct inode *inode = wbc->inode; + unsigned long avg_time, max_bytes, max_time; + u16 history; + int max_id; + + if (!wb) + return; + + history = inode->i_wb_frn_history; + avg_time = inode->i_wb_frn_avg_time; + + /* pick the winner of this round */ + if (wbc->wb_bytes >= wbc->wb_lcand_bytes && + wbc->wb_bytes >= wbc->wb_tcand_bytes) { + max_id = wbc->wb_id; + max_bytes = wbc->wb_bytes; + } else if (wbc->wb_lcand_bytes >= wbc->wb_tcand_bytes) { + max_id = wbc->wb_lcand_id; + max_bytes = wbc->wb_lcand_bytes; + } else { + max_id = wbc->wb_tcand_id; + max_bytes = wbc->wb_tcand_bytes; + } + + /* + * Calculate the amount of IO time the winner consumed and fold it + * into the running average kept per inode. If the consumed IO + * time is lower than avag / WB_FRN_TIME_CUT_DIV, ignore it for + * deciding whether to switch or not. This is to prevent one-off + * small dirtiers from skewing the verdict. + */ + max_time = DIV_ROUND_UP((max_bytes >> PAGE_SHIFT) << WB_FRN_TIME_SHIFT, + wb->avg_write_bandwidth); + if (avg_time) + avg_time += (max_time >> WB_FRN_TIME_AVG_SHIFT) - + (avg_time >> WB_FRN_TIME_AVG_SHIFT); + else + avg_time = max_time; /* immediate catch up on first run */ + + if (max_time >= avg_time / WB_FRN_TIME_CUT_DIV) { + int slots; + + /* + * The switch verdict is reached if foreign wb's consume + * more than a certain proportion of IO time in a + * WB_FRN_TIME_PERIOD. This is loosely tracked by 16 slot + * history mask where each bit represents one sixteenth of + * the period. Determine the number of slots to shift into + * history from @max_time. + */ + slots = min(DIV_ROUND_UP(max_time, WB_FRN_HIST_UNIT), + (unsigned long)WB_FRN_HIST_MAX_SLOTS); + history <<= slots; + if (wbc->wb_id != max_id) + history |= (1U << slots) - 1; + + if (history) + trace_inode_foreign_history(inode, wbc, history); + + /* + * Switch if the current wb isn't the consistent winner. + * If there are multiple closely competing dirtiers, the + * inode may switch across them repeatedly over time, which + * is okay. The main goal is avoiding keeping an inode on + * the wrong wb for an extended period of time. + */ + if (hweight16(history) > WB_FRN_HIST_THR_SLOTS) + inode_switch_wbs(inode, max_id); + } + + /* + * Multiple instances of this function may race to update the + * following fields but we don't mind occassional inaccuracies. + */ + inode->i_wb_frn_winner = max_id; + inode->i_wb_frn_avg_time = min(avg_time, (unsigned long)U16_MAX); + inode->i_wb_frn_history = history; + + wb_put(wbc->wb); + wbc->wb = NULL; +} +EXPORT_SYMBOL_GPL(wbc_detach_inode); + +/** + * wbc_account_cgroup_owner - account writeback to update inode cgroup ownership + * @wbc: writeback_control of the writeback in progress + * @folio: folio being written out + * @bytes: number of bytes being written out + * + * @bytes from @folio are about to written out during the writeback + * controlled by @wbc. Keep the book for foreign inode detection. See + * wbc_detach_inode(). + */ +void wbc_account_cgroup_owner(struct writeback_control *wbc, struct folio *folio, + size_t bytes) +{ + struct cgroup_subsys_state *css; + int id; + + /* + * pageout() path doesn't attach @wbc to the inode being written + * out. This is intentional as we don't want the function to block + * behind a slow cgroup. Ultimately, we want pageout() to kick off + * regular writeback instead of writing things out itself. + */ + if (!wbc->wb || wbc->no_cgroup_owner) + return; + + css = get_mem_cgroup_css_from_folio(folio); + /* dead cgroups shouldn't contribute to inode ownership arbitration */ + if (!css_is_online(css)) + goto out; + + id = css->id; + + if (id == wbc->wb_id) { + wbc->wb_bytes += bytes; + goto out; + } + + if (id == wbc->wb_lcand_id) + wbc->wb_lcand_bytes += bytes; + + /* Boyer-Moore majority vote algorithm */ + if (!wbc->wb_tcand_bytes) + wbc->wb_tcand_id = id; + if (id == wbc->wb_tcand_id) + wbc->wb_tcand_bytes += bytes; + else + wbc->wb_tcand_bytes -= min(bytes, wbc->wb_tcand_bytes); +out: + css_put(css); +} +EXPORT_SYMBOL_GPL(wbc_account_cgroup_owner); + +/** + * wb_split_bdi_pages - split nr_pages to write according to bandwidth + * @wb: target bdi_writeback to split @nr_pages to + * @nr_pages: number of pages to write for the whole bdi + * + * Split @wb's portion of @nr_pages according to @wb's write bandwidth in + * relation to the total write bandwidth of all wb's w/ dirty inodes on + * @wb->bdi. + */ +static long wb_split_bdi_pages(struct bdi_writeback *wb, long nr_pages) +{ + unsigned long this_bw = wb->avg_write_bandwidth; + unsigned long tot_bw = atomic_long_read(&wb->bdi->tot_write_bandwidth); + + if (nr_pages == LONG_MAX) + return LONG_MAX; + + /* + * This may be called on clean wb's and proportional distribution + * may not make sense, just use the original @nr_pages in those + * cases. In general, we wanna err on the side of writing more. + */ + if (!tot_bw || this_bw >= tot_bw) + return nr_pages; + else + return DIV_ROUND_UP_ULL((u64)nr_pages * this_bw, tot_bw); +} + +/** + * bdi_split_work_to_wbs - split a wb_writeback_work to all wb's of a bdi + * @bdi: target backing_dev_info + * @base_work: wb_writeback_work to issue + * @skip_if_busy: skip wb's which already have writeback in progress + * + * Split and issue @base_work to all wb's (bdi_writeback's) of @bdi which + * have dirty inodes. If @base_work->nr_page isn't %LONG_MAX, it's + * distributed to the busy wbs according to each wb's proportion in the + * total active write bandwidth of @bdi. + */ +static void bdi_split_work_to_wbs(struct backing_dev_info *bdi, + struct wb_writeback_work *base_work, + bool skip_if_busy) +{ + struct bdi_writeback *last_wb = NULL; + struct bdi_writeback *wb = list_entry(&bdi->wb_list, + struct bdi_writeback, bdi_node); + + might_sleep(); +restart: + rcu_read_lock(); + list_for_each_entry_continue_rcu(wb, &bdi->wb_list, bdi_node) { + DEFINE_WB_COMPLETION(fallback_work_done, bdi); + struct wb_writeback_work fallback_work; + struct wb_writeback_work *work; + long nr_pages; + + if (last_wb) { + wb_put(last_wb); + last_wb = NULL; + } + + /* SYNC_ALL writes out I_DIRTY_TIME too */ + if (!wb_has_dirty_io(wb) && + (base_work->sync_mode == WB_SYNC_NONE || + list_empty(&wb->b_dirty_time))) + continue; + if (skip_if_busy && writeback_in_progress(wb)) + continue; + + nr_pages = wb_split_bdi_pages(wb, base_work->nr_pages); + + work = kmalloc_obj(*work, GFP_ATOMIC); + if (work) { + *work = *base_work; + work->nr_pages = nr_pages; + work->auto_free = 1; + wb_queue_work(wb, work); + continue; + } + + /* + * If wb_tryget fails, the wb has been shutdown, skip it. + * + * Pin @wb so that it stays on @bdi->wb_list. This allows + * continuing iteration from @wb after dropping and + * regrabbing rcu read lock. + */ + if (!wb_tryget(wb)) + continue; + + /* alloc failed, execute synchronously using on-stack fallback */ + work = &fallback_work; + *work = *base_work; + work->nr_pages = nr_pages; + work->auto_free = 0; + work->done = &fallback_work_done; + + wb_queue_work(wb, work); + last_wb = wb; + + rcu_read_unlock(); + wb_wait_for_completion(&fallback_work_done); + goto restart; + } + rcu_read_unlock(); + + if (last_wb) + wb_put(last_wb); +} + +/** + * cgroup_writeback_by_id - initiate cgroup writeback from bdi and memcg IDs + * @bdi_id: target bdi id + * @memcg_id: target memcg css id + * @reason: reason why some writeback work initiated + * @done: target wb_completion + * + * Initiate flush of the bdi_writeback identified by @bdi_id and @memcg_id + * with the specified parameters. + */ +int cgroup_writeback_by_id(u64 bdi_id, int memcg_id, + enum wb_reason reason, struct wb_completion *done) +{ + struct backing_dev_info *bdi; + struct cgroup_subsys_state *memcg_css; + struct bdi_writeback *wb; + struct wb_writeback_work *work; + unsigned long dirty; + int ret; + + /* lookup bdi and memcg */ + bdi = bdi_get_by_id(bdi_id); + if (!bdi) + return -ENOENT; + + rcu_read_lock(); + memcg_css = css_from_id(memcg_id, &memory_cgrp_subsys); + if (memcg_css && !css_tryget(memcg_css)) + memcg_css = NULL; + rcu_read_unlock(); + if (!memcg_css) { + ret = -ENOENT; + goto out_bdi_put; + } + + /* + * And find the associated wb. If the wb isn't there already + * there's nothing to flush, don't create one. + */ + wb = wb_get_lookup(bdi, memcg_css); + if (!wb) { + ret = -ENOENT; + goto out_css_put; + } + + /* + * The caller is attempting to write out most of + * the currently dirty pages. Let's take the current dirty page + * count and inflate it by 25% which should be large enough to + * flush out most dirty pages while avoiding getting livelocked by + * concurrent dirtiers. + * + * BTW the memcg stats are flushed periodically and this is best-effort + * estimation, so some potential error is ok. + */ + dirty = memcg_page_state(mem_cgroup_from_css(memcg_css), NR_FILE_DIRTY); + dirty = dirty * 10 / 8; + + /* issue the writeback work */ + work = kzalloc_obj(*work, GFP_NOWAIT); + if (work) { + work->nr_pages = dirty; + work->sync_mode = WB_SYNC_NONE; + work->range_cyclic = 1; + work->reason = reason; + work->done = done; + work->auto_free = 1; + wb_queue_work(wb, work); + ret = 0; + } else { + ret = -ENOMEM; + } + + wb_put(wb); +out_css_put: + css_put(memcg_css); +out_bdi_put: + bdi_put(bdi); + return ret; +} + +/** + * cgroup_writeback_umount - wait for in-flight inode wb switches on @sb + * @sb: target super_block + * + * Wait until every inode wb switch that already passed the SB_ACTIVE + * check on this superblock has been completed by the worker. Since + * SB_ACTIVE is cleared before this is called, no new switches can start + * for @sb, so s_isw_nr_in_flight will monotonically drop to zero. + */ +void cgroup_writeback_umount(struct super_block *sb) +{ + if (!(sb->s_bdi->capabilities & BDI_CAP_WRITEBACK)) + return; + + /* + * Pairs with smp_mb() in inode_prepare_wbs_switch(): we either observe + * a non-zero counter and wait, or the switcher sees SB_ACTIVE clear + * (cleared by generic_shutdown_super()) and bails before grabbing the + * inode. + */ + smp_mb(); + cgroup_writeback_drain(sb); +} + +static int __init cgroup_writeback_init(void) +{ + isw_wq = alloc_workqueue("inode_switch_wbs", WQ_PERCPU, 0); + if (!isw_wq) + return -ENOMEM; + return 0; +} +fs_initcall(cgroup_writeback_init); + +#else /* CONFIG_CGROUP_WRITEBACK */ + +static void bdi_down_write_wb_switch_rwsem(struct backing_dev_info *bdi) { } +static void bdi_up_write_wb_switch_rwsem(struct backing_dev_info *bdi) { } + +static void inode_cgwb_move_to_attached(struct inode *inode, + struct bdi_writeback *wb) +{ + assert_spin_locked(&wb->list_lock); + assert_spin_locked(&inode->i_lock); + WARN_ON_ONCE(inode_state_read(inode) & I_FREEING); + + inode_state_clear(inode, I_SYNC_QUEUED); + list_del_init(&inode->i_io_list); + wb_io_lists_depopulated(wb); +} + +static struct bdi_writeback * +locked_inode_to_wb_and_lock_list(struct inode *inode) + __releases(&inode->i_lock) + __acquires(&wb->list_lock) +{ + struct bdi_writeback *wb = inode_to_wb(inode); + + spin_unlock(&inode->i_lock); + spin_lock(&wb->list_lock); + return wb; +} + +static struct bdi_writeback *inode_to_wb_and_lock_list(struct inode *inode) + __acquires(&wb->list_lock) +{ + struct bdi_writeback *wb = inode_to_wb(inode); + + spin_lock(&wb->list_lock); + return wb; +} + +static long wb_split_bdi_pages(struct bdi_writeback *wb, long nr_pages) +{ + return nr_pages; +} + +static void bdi_split_work_to_wbs(struct backing_dev_info *bdi, + struct wb_writeback_work *base_work, + bool skip_if_busy) +{ + might_sleep(); + + if (!skip_if_busy || !writeback_in_progress(&bdi->wb)) { + base_work->auto_free = 0; + wb_queue_work(&bdi->wb, base_work); + } +} + +static inline void wbc_attach_and_unlock_inode(struct writeback_control *wbc, + struct inode *inode) + __releases(&inode->i_lock) +{ + spin_unlock(&inode->i_lock); +} + +#endif /* CONFIG_CGROUP_WRITEBACK */ + +/* + * Add in the number of potentially dirty inodes, because each inode + * write can dirty pagecache in the underlying blockdev. + */ +static unsigned long get_nr_dirty_pages(void) +{ + return global_node_page_state(NR_FILE_DIRTY) + + get_nr_dirty_inodes(); +} + +static void wb_start_writeback(struct bdi_writeback *wb, enum wb_reason reason) +{ + if (!wb_has_dirty_io(wb)) + return; + + /* + * All callers of this function want to start writeback of all + * dirty pages. Places like vmscan can call this at a very + * high frequency, causing pointless allocations of tons of + * work items and keeping the flusher threads busy retrieving + * that work. Ensure that we only allow one of them pending and + * inflight at the time. + */ + if (test_bit(WB_start_all, &wb->state) || + test_and_set_bit(WB_start_all, &wb->state)) + return; + + wb->start_all_reason = reason; + wb_wakeup(wb); +} + +/** + * wb_start_background_writeback - start background writeback + * @wb: bdi_writback to write from + * + * Description: + * This makes sure WB_SYNC_NONE background writeback happens. When + * this function returns, it is only guaranteed that for given wb + * some IO is happening if we are over background dirty threshold. + * Caller need not hold sb s_umount semaphore. + */ +void wb_start_background_writeback(struct bdi_writeback *wb) +{ + /* + * We just wake up the flusher thread. It will perform background + * writeback as soon as there is no other work to do. + */ + trace_writeback_wake_background(wb); + wb_wakeup(wb); +} + +/* + * Remove the inode from the writeback list it is on. + */ +void inode_io_list_del(struct inode *inode) +{ + struct bdi_writeback *wb; + + /* + * FIXME: ext4 can call here from ext4_evict_inode() after evict() already + * unlinked the inode. + */ + if (list_empty_careful(&inode->i_io_list)) + return; + + wb = inode_to_wb_and_lock_list(inode); + spin_lock(&inode->i_lock); + + inode_state_clear(inode, I_SYNC_QUEUED); + list_del_init(&inode->i_io_list); + wb_io_lists_depopulated(wb); + + spin_unlock(&inode->i_lock); + spin_unlock(&wb->list_lock); +} +EXPORT_SYMBOL(inode_io_list_del); + +/* + * mark an inode as under writeback on the sb + */ +void sb_mark_inode_writeback(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + unsigned long flags; + + if (list_empty(&inode->i_wb_list)) { + spin_lock_irqsave(&sb->s_inode_wblist_lock, flags); + if (list_empty(&inode->i_wb_list)) { + list_add_tail(&inode->i_wb_list, &sb->s_inodes_wb); + trace_sb_mark_inode_writeback(inode); + } + spin_unlock_irqrestore(&sb->s_inode_wblist_lock, flags); + } +} + +/* + * clear an inode as under writeback on the sb + */ +void sb_clear_inode_writeback(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + unsigned long flags; + + if (!list_empty(&inode->i_wb_list)) { + spin_lock_irqsave(&sb->s_inode_wblist_lock, flags); + if (!list_empty(&inode->i_wb_list)) { + list_del_init(&inode->i_wb_list); + trace_sb_clear_inode_writeback(inode); + } + spin_unlock_irqrestore(&sb->s_inode_wblist_lock, flags); + } +} + +/* + * Redirty an inode: set its when-it-was dirtied timestamp and move it to the + * furthest end of its superblock's dirty-inode list. + * + * Before stamping the inode's ->dirtied_when, we check to see whether it is + * already the most-recently-dirtied inode on the b_dirty list. If that is + * the case then the inode must have been redirtied while it was being written + * out and we don't reset its dirtied_when. + */ +static void redirty_tail_locked(struct inode *inode, struct bdi_writeback *wb) +{ + assert_spin_locked(&inode->i_lock); + + inode_state_clear(inode, I_SYNC_QUEUED); + /* + * When the inode is being freed just don't bother with dirty list + * tracking. Flush worker will ignore this inode anyway and it will + * trigger assertions in inode_io_list_move_locked(). + */ + if (inode_state_read(inode) & I_FREEING) { + list_del_init(&inode->i_io_list); + wb_io_lists_depopulated(wb); + return; + } + if (!list_empty(&wb->b_dirty)) { + struct inode *tail; + + tail = wb_inode(wb->b_dirty.next); + if (time_before(inode->dirtied_when, tail->dirtied_when)) + inode->dirtied_when = jiffies; + } + inode_io_list_move_locked(inode, wb, &wb->b_dirty); +} + +static void redirty_tail(struct inode *inode, struct bdi_writeback *wb) +{ + spin_lock(&inode->i_lock); + redirty_tail_locked(inode, wb); + spin_unlock(&inode->i_lock); +} + +/* + * requeue inode for re-scanning after bdi->b_io list is exhausted. + */ +static void requeue_io(struct inode *inode, struct bdi_writeback *wb) +{ + inode_io_list_move_locked(inode, wb, &wb->b_more_io); +} + +static void inode_sync_complete(struct inode *inode) +{ + assert_spin_locked(&inode->i_lock); + + inode_state_clear(inode, I_SYNC); + /* If inode is clean an unused, put it into LRU now... */ + inode_lru_list_add(inode); + /* Called with inode->i_lock which ensures memory ordering. */ + inode_wake_up_bit(inode, __I_SYNC); +} + +static bool inode_dirtied_after(struct inode *inode, unsigned long t) +{ + bool ret = time_after(inode->dirtied_when, t); +#ifndef CONFIG_64BIT + /* + * For inodes being constantly redirtied, dirtied_when can get stuck. + * It _appears_ to be in the future, but is actually in distant past. + * This test is necessary to prevent such wrapped-around relative times + * from permanently stopping the whole bdi writeback. + */ + ret = ret && time_before_eq(inode->dirtied_when, jiffies); +#endif + return ret; +} + +/* + * Move expired (dirtied before dirtied_before) dirty inodes from + * @delaying_queue to @dispatch_queue. + */ +static int move_expired_inodes(struct list_head *delaying_queue, + struct list_head *dispatch_queue, + unsigned long dirtied_before) +{ + LIST_HEAD(tmp); + struct list_head *pos, *node; + struct super_block *sb = NULL; + struct inode *inode; + int do_sb_sort = 0; + int moved = 0; + + while (!list_empty(delaying_queue)) { + inode = wb_inode(delaying_queue->prev); + if (inode_dirtied_after(inode, dirtied_before)) + break; + spin_lock(&inode->i_lock); + list_move(&inode->i_io_list, &tmp); + moved++; + inode_state_set(inode, I_SYNC_QUEUED); + spin_unlock(&inode->i_lock); + if (sb_is_blkdev_sb(inode->i_sb)) + continue; + if (sb && sb != inode->i_sb) + do_sb_sort = 1; + sb = inode->i_sb; + } + + /* just one sb in list, splice to dispatch_queue and we're done */ + if (!do_sb_sort) { + list_splice(&tmp, dispatch_queue); + goto out; + } + + /* + * Although inode's i_io_list is moved from 'tmp' to 'dispatch_queue', + * we don't take inode->i_lock here because it is just a pointless overhead. + * Inode is already marked as I_SYNC_QUEUED so writeback list handling is + * fully under our control. + */ + while (!list_empty(&tmp)) { + sb = wb_inode(tmp.prev)->i_sb; + list_for_each_prev_safe(pos, node, &tmp) { + inode = wb_inode(pos); + if (inode->i_sb == sb) + list_move(&inode->i_io_list, dispatch_queue); + } + } +out: + return moved; +} + +/* + * Queue all expired dirty inodes for io, eldest first. + * Before + * newly dirtied b_dirty b_io b_more_io + * =============> gf edc BA + * After + * newly dirtied b_dirty b_io b_more_io + * =============> g fBAedc + * | + * +--> dequeue for IO + */ +static void queue_io(struct bdi_writeback *wb, struct wb_writeback_work *work, + unsigned long dirtied_before) +{ + int moved; + unsigned long time_expire_jif = dirtied_before; + + assert_spin_locked(&wb->list_lock); + list_splice_init(&wb->b_more_io, &wb->b_io); + moved = move_expired_inodes(&wb->b_dirty, &wb->b_io, dirtied_before); + if (!work->for_sync) + time_expire_jif = jiffies - dirtytime_expire_interval * HZ; + moved += move_expired_inodes(&wb->b_dirty_time, &wb->b_io, + time_expire_jif); + if (moved) + wb_io_lists_populated(wb); + trace_writeback_queue_io(wb, work, dirtied_before, moved); +} + +static int write_inode(struct inode *inode, struct writeback_control *wbc) +{ + int ret; + + if (inode->i_sb->s_op->write_inode && !is_bad_inode(inode)) { + trace_writeback_write_inode_start(inode, wbc); + ret = inode->i_sb->s_op->write_inode(inode, wbc); + trace_writeback_write_inode(inode, wbc); + return ret; + } + return 0; +} + +/* + * Wait for writeback on an inode to complete. Called with i_lock held. + * Caller must make sure inode cannot go away when we drop i_lock. + */ +void inode_wait_for_writeback(struct inode *inode) +{ + struct wait_bit_queue_entry wqe; + struct wait_queue_head *wq_head; + + assert_spin_locked(&inode->i_lock); + + if (!(inode_state_read(inode) & I_SYNC)) + return; + + wq_head = inode_bit_waitqueue(&wqe, inode, __I_SYNC); + for (;;) { + prepare_to_wait_event(wq_head, &wqe.wq_entry, TASK_UNINTERRUPTIBLE); + /* Checking I_SYNC with inode->i_lock guarantees memory ordering. */ + if (!(inode_state_read(inode) & I_SYNC)) + break; + spin_unlock(&inode->i_lock); + schedule(); + spin_lock(&inode->i_lock); + } + finish_wait(wq_head, &wqe.wq_entry); +} + +/* + * Sleep until I_SYNC is cleared. This function must be called with i_lock + * held and drops it. It is aimed for callers not holding any inode reference + * so once i_lock is dropped, inode can go away. + */ +static void inode_sleep_on_writeback(struct inode *inode) + __releases(inode->i_lock) +{ + struct wait_bit_queue_entry wqe; + struct wait_queue_head *wq_head; + bool sleep; + + assert_spin_locked(&inode->i_lock); + + wq_head = inode_bit_waitqueue(&wqe, inode, __I_SYNC); + prepare_to_wait_event(wq_head, &wqe.wq_entry, TASK_UNINTERRUPTIBLE); + /* Checking I_SYNC with inode->i_lock guarantees memory ordering. */ + sleep = !!(inode_state_read(inode) & I_SYNC); + spin_unlock(&inode->i_lock); + if (sleep) + schedule(); + finish_wait(wq_head, &wqe.wq_entry); +} + +/* + * Find proper writeback list for the inode depending on its current state and + * possibly also change of its state while we were doing writeback. Here we + * handle things such as livelock prevention or fairness of writeback among + * inodes. This function can be called only by flusher thread - noone else + * processes all inodes in writeback lists and requeueing inodes behind flusher + * thread's back can have unexpected consequences. + */ +static void requeue_inode(struct inode *inode, struct bdi_writeback *wb, + struct writeback_control *wbc, + unsigned long dirtied_before) +{ + if (inode_state_read(inode) & I_FREEING) + return; + + /* + * Sync livelock prevention. Each inode is tagged and synced in one + * shot. If still dirty, it will be redirty_tail()'ed below. Update + * the dirty time to prevent enqueue and sync it again. + */ + if ((inode_state_read(inode) & I_DIRTY) && + (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)) + inode->dirtied_when = jiffies; + + if (wbc->pages_skipped) { + /* + * Writeback is not making progress due to locked buffers. + * Skip this inode for now. Although having skipped pages + * is odd for clean inodes, it can happen for some + * filesystems so handle that gracefully. + */ + if (inode_state_read(inode) & I_DIRTY_ALL) + redirty_tail_locked(inode, wb); + else + inode_cgwb_move_to_attached(inode, wb); + return; + } + + if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY)) { + /* + * We didn't write back all the pages. nfs_writepages() + * sometimes bales out without doing anything. + */ + if (wbc->nr_to_write <= 0 && + !inode_dirtied_after(inode, dirtied_before)) { + /* Slice used up. Queue for next turn. */ + requeue_io(inode, wb); + } else { + /* + * Writeback blocked by something other than + * congestion. Delay the inode for some time to + * avoid spinning on the CPU (100% iowait) + * retrying writeback of the dirty page/inode + * that cannot be performed immediately. + */ + redirty_tail_locked(inode, wb); + } + } else if (inode_state_read(inode) & I_DIRTY) { + /* + * Filesystems can dirty the inode during writeback operations, + * such as delayed allocation during submission or metadata + * updates after data IO completion. + */ + redirty_tail_locked(inode, wb); + } else if (inode_state_read(inode) & I_DIRTY_TIME) { + inode->dirtied_when = jiffies; + inode_io_list_move_locked(inode, wb, &wb->b_dirty_time); + inode_state_clear(inode, I_SYNC_QUEUED); + } else { + /* The inode is clean. Remove from writeback lists. */ + inode_cgwb_move_to_attached(inode, wb); + } +} + +static bool __sync_lazytime(struct inode *inode) +{ + spin_lock(&inode->i_lock); + if (!(inode_state_read(inode) & I_DIRTY_TIME)) { + spin_unlock(&inode->i_lock); + return false; + } + inode_state_clear(inode, I_DIRTY_TIME); + spin_unlock(&inode->i_lock); + inode->i_op->sync_lazytime(inode); + return true; +} + +bool sync_lazytime(struct inode *inode) +{ + if (!(inode_state_read_once(inode) & I_DIRTY_TIME)) + return false; + + trace_writeback_lazytime(inode); + if (inode->i_op->sync_lazytime) + return __sync_lazytime(inode); + mark_inode_dirty_sync(inode); + return true; +} + +/* + * Write out an inode and its dirty pages (or some of its dirty pages, depending + * on @wbc->nr_to_write), and clear the relevant dirty flags from i_state. + * + * This doesn't remove the inode from the writeback list it is on, except + * potentially to move it from b_dirty_time to b_dirty due to timestamp + * expiration. The caller is otherwise responsible for writeback list handling. + * + * The caller is also responsible for setting the I_SYNC flag beforehand and + * calling inode_sync_complete() to clear it afterwards. + */ +static int +__writeback_single_inode(struct inode *inode, struct writeback_control *wbc) +{ + struct address_space *mapping = inode->i_mapping; + long nr_to_write = wbc->nr_to_write; + unsigned dirty; + int ret; + + WARN_ON(!(inode_state_read_once(inode) & I_SYNC)); + + trace_writeback_single_inode_start(inode, wbc, nr_to_write); + + ret = do_writepages(mapping, wbc); + + /* + * Make sure to wait on the data before writing out the metadata. + * This is important for filesystems that modify metadata on data + * I/O completion. We don't do it for sync(2) writeback because it has a + * separate, external IO completion path and ->sync_fs for guaranteeing + * inode metadata is written back correctly. + */ + if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync) { + int err = filemap_fdatawait(mapping); + if (ret == 0) + ret = err; + } + + /* + * For data integrity writeback, or when the dirty interval expired, + * ask the file system to propagata lazy timestamp updates into real + * dirty state. + */ + if ((inode_state_read_once(inode) & I_DIRTY_TIME) && + (wbc->sync_mode == WB_SYNC_ALL || + time_after(jiffies, inode->dirtied_time_when + + dirtytime_expire_interval * HZ))) + sync_lazytime(inode); + + /* + * Get and clear the dirty flags from i_state. This needs to be done + * after calling writepages because some filesystems may redirty the + * inode during writepages due to delalloc. It also needs to be done + * after handling timestamp expiration, as that may dirty the inode too. + */ + spin_lock(&inode->i_lock); + dirty = inode_state_read(inode) & I_DIRTY; + inode_state_clear(inode, dirty); + + /* + * Paired with smp_mb() in __mark_inode_dirty(). This allows + * __mark_inode_dirty() to test i_state without grabbing i_lock - + * either they see the I_DIRTY bits cleared or we see the dirtied + * inode. + * + * I_DIRTY_PAGES is always cleared together above even if @mapping + * still has dirty pages. The flag is reinstated after smp_mb() if + * necessary. This guarantees that either __mark_inode_dirty() + * sees clear I_DIRTY_PAGES or we see PAGECACHE_TAG_DIRTY. + */ + smp_mb(); + + if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) + inode_state_set(inode, I_DIRTY_PAGES); + else if (unlikely(inode_state_read(inode) & I_PINNING_NETFS_WB)) { + if (!(inode_state_read(inode) & I_DIRTY_PAGES)) { + inode_state_clear(inode, I_PINNING_NETFS_WB); + wbc->unpinned_netfs_wb = true; + dirty |= I_PINNING_NETFS_WB; /* Cause write_inode */ + } + } + + spin_unlock(&inode->i_lock); + + /* Don't write the inode if only I_DIRTY_PAGES was set */ + if (dirty & ~I_DIRTY_PAGES) { + int err = write_inode(inode, wbc); + if (ret == 0) + ret = err; + } + + /* + * Do we need to wait for inode metadata IO possibly submitted + * by previous WB_SYNC_NONE writeback? + */ + if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync && + inode_state_read_once(inode) & I_METADATA_WRITEBACK) { + int err; + + spin_lock(&inode->i_lock); + inode_state_clear(inode, I_METADATA_WRITEBACK); + spin_unlock(&inode->i_lock); + err = inode->i_sb->s_op->sync_inode_metadata(inode, wbc); + if (ret == 0) + ret = err; + } + wbc->unpinned_netfs_wb = false; + trace_writeback_single_inode(inode, wbc, nr_to_write); + return ret; +} + +/* + * Write out an inode's dirty data and metadata on-demand, i.e. separately from + * the regular batched writeback done by the flusher threads in + * writeback_sb_inodes(). @wbc controls various aspects of the write, such as + * whether it is a data-integrity sync (%WB_SYNC_ALL) or not (%WB_SYNC_NONE). + * + * To prevent the inode from going away, either the caller must have a reference + * to the inode, or the inode must have I_WILL_FREE or I_FREEING set. + */ +static int writeback_single_inode(struct inode *inode, + struct writeback_control *wbc) +{ + struct bdi_writeback *wb; + int ret = 0; + + spin_lock(&inode->i_lock); + if (!icount_read(inode)) + WARN_ON(!(inode_state_read(inode) & (I_WILL_FREE | I_FREEING))); + else + WARN_ON(inode_state_read(inode) & I_WILL_FREE); + + if (inode_state_read(inode) & I_SYNC) { + /* + * Writeback is already running on the inode. For WB_SYNC_NONE, + * that's enough and we can just return. For WB_SYNC_ALL, we + * must wait for the existing writeback to complete, then do + * writeback again if there's anything left. + */ + if (wbc->sync_mode != WB_SYNC_ALL) + goto out; + inode_wait_for_writeback(inode); + } + WARN_ON(inode_state_read(inode) & I_SYNC); + /* + * If the inode is already fully clean, then there's nothing to do. + * + * For data-integrity syncs we also need to check whether any folios or + * metadata are still under writeback, e.g. due to prior WB_SYNC_NONE + * writeback. If there, we'll need to wait for them. + */ + if (!(inode_state_read(inode) & I_DIRTY_ALL)) { + if (wbc->sync_mode != WB_SYNC_ALL) + goto out; + if (!mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK) && + !(inode_state_read(inode) & I_METADATA_WRITEBACK)) + goto out; + } + inode_state_set(inode, I_SYNC); + wbc_attach_and_unlock_inode(wbc, inode); + + ret = __writeback_single_inode(inode, wbc); + + wbc_detach_inode(wbc); + + wb = inode_to_wb_and_lock_list(inode); + spin_lock(&inode->i_lock); + /* + * If the inode is freeing, its i_io_list shoudn't be updated + * as it can be finally deleted at this moment. + */ + if (!(inode_state_read(inode) & I_FREEING)) { + /* + * If the inode is now fully clean, then it can be safely + * removed from its writeback list (if any). Otherwise the + * flusher threads are responsible for the writeback lists. + */ + if (!(inode_state_read(inode) & I_DIRTY_ALL)) + inode_cgwb_move_to_attached(inode, wb); + else if (!(inode_state_read(inode) & I_SYNC_QUEUED)) { + if ((inode_state_read(inode) & I_DIRTY)) + redirty_tail_locked(inode, wb); + else if (inode_state_read(inode) & I_DIRTY_TIME) { + inode->dirtied_when = jiffies; + inode_io_list_move_locked(inode, + wb, + &wb->b_dirty_time); + } + } + } + + spin_unlock(&wb->list_lock); + inode_sync_complete(inode); +out: + spin_unlock(&inode->i_lock); + return ret; +} + +static long writeback_chunk_size(struct super_block *sb, + struct bdi_writeback *wb, struct wb_writeback_work *work) +{ + long pages; + + /* + * WB_SYNC_ALL mode does livelock avoidance by syncing dirty + * inodes/pages in one big loop. Setting wbc.nr_to_write=LONG_MAX + * here avoids calling into writeback_inodes_wb() more than once. + * + * The intended call sequence for WB_SYNC_ALL writeback is: + * + * wb_writeback() + * writeback_sb_inodes() <== called only once + * write_cache_pages() <== called once for each inode + * (quickly) tag currently dirty pages + * (maybe slowly) sync all tagged pages + */ + if (work->sync_mode == WB_SYNC_ALL || work->tagged_writepages) + return LONG_MAX; + + pages = min(wb->avg_write_bandwidth / 2, + global_wb_domain.dirty_limit / DIRTY_SCOPE); + pages = min(pages, work->nr_pages); + return round_down(pages + sb->s_min_writeback_pages, + sb->s_min_writeback_pages); +} + +/* + * Write a portion of b_io inodes which belong to @sb. + * + * Return the number of pages and/or inodes written. + * + * NOTE! This is called with wb->list_lock held, and will + * unlock and relock that for each inode it ends up doing + * IO for. + */ +static long writeback_sb_inodes(struct super_block *sb, + struct bdi_writeback *wb, + struct wb_writeback_work *work) +{ + struct writeback_control wbc = { + .sync_mode = work->sync_mode, + .tagged_writepages = work->tagged_writepages, + .for_kupdate = work->for_kupdate, + .for_background = work->for_background, + .for_sync = work->for_sync, + .range_cyclic = work->range_cyclic, + .range_start = 0, + .range_end = LLONG_MAX, + }; + unsigned long start_time = jiffies; + unsigned long timeout = sysctl_hung_task_timeout_secs; + long write_chunk; + long total_wrote = 0; /* count both pages and inodes */ + unsigned long dirtied_before = jiffies; + + if (work->for_kupdate) + dirtied_before = jiffies - + msecs_to_jiffies(dirty_expire_interval * 10); + + while (!list_empty(&wb->b_io)) { + struct inode *inode = wb_inode(wb->b_io.prev); + struct bdi_writeback *tmp_wb; + long wrote; + + if (inode->i_sb != sb) { + if (work->sb) { + /* + * We only want to write back data for this + * superblock, move all inodes not belonging + * to it back onto the dirty list. + */ + redirty_tail(inode, wb); + continue; + } + + /* + * The inode belongs to a different superblock. + * Bounce back to the caller to unpin this and + * pin the next superblock. + */ + break; + } + + /* + * Don't bother with new inodes or inodes being freed, first + * kind does not need periodic writeout yet, and for the latter + * kind writeout is handled by the freer. + */ + spin_lock(&inode->i_lock); + if (inode_state_read(inode) & (I_NEW | I_FREEING | I_WILL_FREE)) { + redirty_tail_locked(inode, wb); + spin_unlock(&inode->i_lock); + continue; + } + if ((inode_state_read(inode) & I_SYNC) && wbc.sync_mode != WB_SYNC_ALL) { + /* + * If this inode is locked for writeback and we are not + * doing writeback-for-data-integrity, move it to + * b_more_io so that writeback can proceed with the + * other inodes on s_io. + * + * We'll have another go at writing back this inode + * when we completed a full scan of b_io. + */ + requeue_io(inode, wb); + spin_unlock(&inode->i_lock); + trace_writeback_sb_inodes_requeue(inode); + continue; + } + spin_unlock(&wb->list_lock); + + /* + * We already requeued the inode if it had I_SYNC set and we + * are doing WB_SYNC_NONE writeback. So this catches only the + * WB_SYNC_ALL case. + */ + if (inode_state_read(inode) & I_SYNC) { + /* Wait for I_SYNC. This function drops i_lock... */ + inode_sleep_on_writeback(inode); + /* Inode may be gone, start again */ + spin_lock(&wb->list_lock); + continue; + } + inode_state_set(inode, I_SYNC); + wbc_attach_and_unlock_inode(&wbc, inode); + + write_chunk = writeback_chunk_size(inode->i_sb, wb, work); + wbc.nr_to_write = write_chunk; + wbc.pages_skipped = 0; + + /* + * We use I_SYNC to pin the inode in memory. While it is set + * evict_inode() will wait so the inode cannot be freed. + */ + __writeback_single_inode(inode, &wbc); + + /* Report progress to inform the hung task detector of the progress. */ + if (work->done && work->done->progress_stamp && timeout && + (jiffies - work->done->progress_stamp) > HZ * timeout / 2) + wake_up_all(work->done->waitq); + + wbc_detach_inode(&wbc); + work->nr_pages -= write_chunk - wbc.nr_to_write; + wrote = write_chunk - wbc.nr_to_write - wbc.pages_skipped; + wrote = wrote < 0 ? 0 : wrote; + total_wrote += wrote; + + if (need_resched()) { + /* + * We're trying to balance between building up a nice + * long list of IOs to improve our merge rate, and + * getting those IOs out quickly for anyone throttling + * in balance_dirty_pages(). cond_resched() doesn't + * unplug, so get our IOs out the door before we + * give up the CPU. + */ + blk_flush_plug(current->plug, false); + cond_resched(); + } + + /* + * Requeue @inode if still dirty. Be careful as @inode may + * have been switched to another wb in the meantime. + */ + tmp_wb = inode_to_wb_and_lock_list(inode); + spin_lock(&inode->i_lock); + if (!(inode_state_read(inode) & I_DIRTY_ALL)) + total_wrote++; + requeue_inode(inode, tmp_wb, &wbc, dirtied_before); + inode_sync_complete(inode); + spin_unlock(&inode->i_lock); + + if (unlikely(tmp_wb != wb)) { + spin_unlock(&tmp_wb->list_lock); + spin_lock(&wb->list_lock); + } + + /* + * bail out to wb_writeback() often enough to check + * background threshold and other termination conditions. + */ + if (total_wrote) { + if (time_is_before_jiffies(start_time + HZ / 10UL)) + break; + if (work->nr_pages <= 0) + break; + } + } + return total_wrote; +} + +static long __writeback_inodes_wb(struct bdi_writeback *wb, + struct wb_writeback_work *work) +{ + unsigned long start_time = jiffies; + long wrote = 0; + + while (!list_empty(&wb->b_io)) { + struct inode *inode = wb_inode(wb->b_io.prev); + struct super_block *sb = inode->i_sb; + + if (!super_trylock_shared(sb)) { + /* + * super_trylock_shared() may fail consistently due to + * s_umount being grabbed by someone else. Don't use + * requeue_io() to avoid busy retrying the inode/sb. + */ + redirty_tail(inode, wb); + continue; + } + wrote += writeback_sb_inodes(sb, wb, work); + up_read(&sb->s_umount); + + /* refer to the same tests at the end of writeback_sb_inodes */ + if (wrote) { + if (time_is_before_jiffies(start_time + HZ / 10UL)) + break; + if (work->nr_pages <= 0) + break; + } + } + /* Leave any unwritten inodes on b_io */ + return wrote; +} + +static long writeback_inodes_wb(struct bdi_writeback *wb, long nr_pages, + enum wb_reason reason) +{ + struct wb_writeback_work work = { + .nr_pages = nr_pages, + .sync_mode = WB_SYNC_NONE, + .range_cyclic = 1, + .reason = reason, + }; + struct blk_plug plug; + + blk_start_plug(&plug); + spin_lock(&wb->list_lock); + if (list_empty(&wb->b_io)) + queue_io(wb, &work, jiffies); + __writeback_inodes_wb(wb, &work); + spin_unlock(&wb->list_lock); + blk_finish_plug(&plug); + + return nr_pages - work.nr_pages; +} + +/* + * Explicit flushing or periodic writeback of "old" data. + * + * Define "old": the first time one of an inode's pages is dirtied, we mark the + * dirtying-time in the inode's address_space. So this periodic writeback code + * just walks the superblock inode list, writing back any inodes which are + * older than a specific point in time. + * + * Try to run once per dirty_writeback_interval. But if a writeback event + * takes longer than a dirty_writeback_interval interval, then leave a + * one-second gap. + * + * dirtied_before takes precedence over nr_to_write. So we'll only write back + * all dirty pages if they are all attached to "old" mappings. + */ +static long wb_writeback(struct bdi_writeback *wb, + struct wb_writeback_work *work) +{ + long nr_pages = work->nr_pages; + unsigned long dirtied_before = jiffies; + struct inode *inode; + long progress; + struct blk_plug plug; + bool queued = false; + + blk_start_plug(&plug); + for (;;) { + /* + * Stop writeback when nr_pages has been consumed + */ + if (work->nr_pages <= 0) + break; + + /* + * Background writeout and kupdate-style writeback may + * run forever. Stop them if there is other work to do + * so that e.g. sync can proceed. They'll be restarted + * after the other works are all done. + */ + if ((work->for_background || work->for_kupdate) && + !list_empty(&wb->work_list)) + break; + + /* + * For background writeout, stop when we are below the + * background dirty threshold + */ + if (work->for_background && !wb_over_bg_thresh(wb)) + break; + + + spin_lock(&wb->list_lock); + + trace_writeback_start(wb, work); + if (list_empty(&wb->b_io)) { + /* + * Kupdate and background works are special and we want + * to include all inodes that need writing. Livelock + * avoidance is handled by these works yielding to any + * other work so we are safe. + */ + if (work->for_kupdate) { + dirtied_before = jiffies - + msecs_to_jiffies(dirty_expire_interval * + 10); + } else if (work->for_background) + dirtied_before = jiffies; + + queue_io(wb, work, dirtied_before); + queued = true; + } + if (work->sb) + progress = writeback_sb_inodes(work->sb, wb, work); + else + progress = __writeback_inodes_wb(wb, work); + trace_writeback_written(wb, work); + + /* + * Did we write something? Try for more + * + * Dirty inodes are moved to b_io for writeback in batches. + * The completion of the current batch does not necessarily + * mean the overall work is done. So we keep looping as long + * as made some progress on cleaning pages or inodes. + */ + if (progress || !queued) { + spin_unlock(&wb->list_lock); + continue; + } + + /* + * No more inodes for IO, bail + */ + if (list_empty(&wb->b_more_io)) { + spin_unlock(&wb->list_lock); + break; + } + + /* + * Nothing written. Wait for some inode to + * become available for writeback. Otherwise + * we'll just busyloop. + */ + trace_writeback_wait(wb, work); + inode = wb_inode(wb->b_more_io.prev); + spin_lock(&inode->i_lock); + spin_unlock(&wb->list_lock); + /* This function drops i_lock... */ + inode_sleep_on_writeback(inode); + } + blk_finish_plug(&plug); + + return nr_pages - work->nr_pages; +} + +/* + * Return the next wb_writeback_work struct that hasn't been processed yet. + */ +static struct wb_writeback_work *get_next_work_item(struct bdi_writeback *wb) +{ + struct wb_writeback_work *work = NULL; + + spin_lock_irq(&wb->work_lock); + if (!list_empty(&wb->work_list)) { + work = list_entry(wb->work_list.next, + struct wb_writeback_work, list); + list_del_init(&work->list); + } + spin_unlock_irq(&wb->work_lock); + return work; +} + +static long wb_check_background_flush(struct bdi_writeback *wb) +{ + if (wb_over_bg_thresh(wb)) { + + struct wb_writeback_work work = { + .nr_pages = LONG_MAX, + .sync_mode = WB_SYNC_NONE, + .for_background = 1, + .range_cyclic = 1, + .reason = WB_REASON_BACKGROUND, + }; + + return wb_writeback(wb, &work); + } + + return 0; +} + +static long wb_check_old_data_flush(struct bdi_writeback *wb) +{ + unsigned long expired; + long nr_pages; + + /* + * When set to zero, disable periodic writeback + */ + if (!dirty_writeback_interval) + return 0; + + expired = wb->last_old_flush + + msecs_to_jiffies(dirty_writeback_interval * 10); + if (time_before(jiffies, expired)) + return 0; + + wb->last_old_flush = jiffies; + nr_pages = get_nr_dirty_pages(); + + if (nr_pages) { + struct wb_writeback_work work = { + .nr_pages = nr_pages, + .sync_mode = WB_SYNC_NONE, + .for_kupdate = 1, + .range_cyclic = 1, + .reason = WB_REASON_PERIODIC, + }; + + return wb_writeback(wb, &work); + } + + return 0; +} + +static long wb_check_start_all(struct bdi_writeback *wb) +{ + long nr_pages; + + if (!test_bit(WB_start_all, &wb->state)) + return 0; + + nr_pages = get_nr_dirty_pages(); + if (nr_pages) { + struct wb_writeback_work work = { + .nr_pages = wb_split_bdi_pages(wb, nr_pages), + .sync_mode = WB_SYNC_NONE, + .range_cyclic = 1, + .reason = wb->start_all_reason, + }; + + nr_pages = wb_writeback(wb, &work); + } + + clear_bit(WB_start_all, &wb->state); + return nr_pages; +} + +static long wb_check_start_dontcache(struct bdi_writeback *wb) +{ + long nr_pages; + + if (!test_and_clear_bit(WB_start_dontcache, &wb->state)) + return 0; + + nr_pages = wb_stat_sum(wb, WB_DONTCACHE_DIRTY); + if (nr_pages) { + struct wb_writeback_work work = { + .nr_pages = nr_pages, + .sync_mode = WB_SYNC_NONE, + .range_cyclic = 1, + .reason = WB_REASON_DONTCACHE, + }; + + nr_pages = wb_writeback(wb, &work); + } + + return nr_pages; +} + +/* + * Retrieve work items and do the writeback they describe + */ +static long wb_do_writeback(struct bdi_writeback *wb) +{ + struct wb_writeback_work *work; + long wrote = 0; + + set_bit(WB_writeback_running, &wb->state); + while ((work = get_next_work_item(wb)) != NULL) { + trace_writeback_exec(wb, work); + wrote += wb_writeback(wb, work); + finish_writeback_work(work); + } + + /* + * Check for a flush-everything request + */ + wrote += wb_check_start_all(wb); + + /* + * Check for dontcache writeback request + */ + wrote += wb_check_start_dontcache(wb); + + /* + * Check for periodic writeback, kupdated() style + */ + wrote += wb_check_old_data_flush(wb); + wrote += wb_check_background_flush(wb); + clear_bit(WB_writeback_running, &wb->state); + + return wrote; +} + +/* + * Handle writeback of dirty data for the device backed by this bdi. Also + * reschedules periodically and does kupdated style flushing. + */ +void wb_workfn(struct work_struct *work) +{ + struct bdi_writeback *wb = container_of(to_delayed_work(work), + struct bdi_writeback, dwork); + long pages_written; + + set_worker_desc("flush-%s", bdi_dev_name(wb->bdi)); + + if (likely(!current_is_workqueue_rescuer() || + !test_bit(WB_registered, &wb->state))) { + /* + * The normal path. Keep writing back @wb until its + * work_list is empty. Note that this path is also taken + * if @wb is shutting down even when we're running off the + * rescuer as work_list needs to be drained. + */ + do { + pages_written = wb_do_writeback(wb); + trace_writeback_pages_written(pages_written); + } while (!list_empty(&wb->work_list)); + } else { + /* + * bdi_wq can't get enough workers and we're running off + * the emergency worker. Don't hog it. Hopefully, 1024 is + * enough for efficient IO. + */ + pages_written = writeback_inodes_wb(wb, 1024, + WB_REASON_FORKER_THREAD); + trace_writeback_pages_written(pages_written); + } + + if (!list_empty(&wb->work_list)) + wb_wakeup(wb); + else if (wb_has_dirty_io(wb) && dirty_writeback_interval) + wb_wakeup_delayed(wb); +} + +/* + * Start writeback of all dirty pages on this bdi. + */ +static void __wakeup_flusher_threads_bdi(struct backing_dev_info *bdi, + enum wb_reason reason) +{ + struct bdi_writeback *wb; + + if (!bdi_has_dirty_io(bdi)) + return; + + list_for_each_entry_rcu(wb, &bdi->wb_list, bdi_node) + wb_start_writeback(wb, reason); +} + +void wakeup_flusher_threads_bdi(struct backing_dev_info *bdi, + enum wb_reason reason) +{ + rcu_read_lock(); + __wakeup_flusher_threads_bdi(bdi, reason); + rcu_read_unlock(); +} + +/** + * filemap_dontcache_kick_writeback - kick flusher for IOCB_DONTCACHE writes + * @mapping: address_space that was just written to + * + * Kick the writeback flusher thread to expedite writeback of dontcache dirty + * pages. Queue writeback for the inode's wb for as many pages as there are + * dontcache pages, but don't restrict writeback to dontcache pages only. + * + * This significantly improves performance over either writing all wb's pages + * or writing only dontcache pages. Although it doesn't guarantee quick + * writeback and reclaim of dontcache pages, it keeps the amount of dirty pages + * in check. Over longer term dontcache pages get written and reclaimed by + * background writeback even with this rough heuristic. + */ +void filemap_dontcache_kick_writeback(struct address_space *mapping) +{ + struct inode *inode = mapping->host; + struct bdi_writeback *wb; + struct wb_lock_cookie cookie = {}; + bool need_wakeup = false; + + wb = unlocked_inode_to_wb_begin(inode, &cookie); + if (wb_has_dirty_io(wb) && + !test_bit(WB_start_dontcache, &wb->state) && + !test_and_set_bit(WB_start_dontcache, &wb->state)) { + wb_get(wb); + need_wakeup = true; + } + unlocked_inode_to_wb_end(inode, &cookie); + + if (need_wakeup) { + wb_wakeup(wb); + wb_put(wb); + } +} +EXPORT_SYMBOL_GPL(filemap_dontcache_kick_writeback); + +/* + * Wakeup the flusher threads to start writeback of all currently dirty pages + */ +void wakeup_flusher_threads(enum wb_reason reason) +{ + struct backing_dev_info *bdi; + + /* + * If we are expecting writeback progress we must submit plugged IO. + */ + blk_flush_plug(current->plug, true); + + rcu_read_lock(); + list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) + __wakeup_flusher_threads_bdi(bdi, reason); + rcu_read_unlock(); +} + +/* + * Wake up bdi's periodically to make sure dirtytime inodes gets + * written back periodically. We deliberately do *not* check the + * b_dirtytime list in wb_has_dirty_io(), since this would cause the + * kernel to be constantly waking up once there are any dirtytime + * inodes on the system. So instead we define a separate delayed work + * function which gets called much more rarely. (By default, only + * once every 12 hours.) + * + * If there is any other write activity going on in the file system, + * this function won't be necessary. But if the only thing that has + * happened on the file system is a dirtytime inode caused by an atime + * update, we need this infrastructure below to make sure that inode + * eventually gets pushed out to disk. + */ +static void wakeup_dirtytime_writeback(struct work_struct *w); +static DECLARE_DELAYED_WORK(dirtytime_work, wakeup_dirtytime_writeback); + +static void wakeup_dirtytime_writeback(struct work_struct *w) +{ + struct backing_dev_info *bdi; + + rcu_read_lock(); + list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) { + struct bdi_writeback *wb; + + list_for_each_entry_rcu(wb, &bdi->wb_list, bdi_node) + if (!list_empty(&wb->b_dirty_time)) + wb_wakeup(wb); + } + rcu_read_unlock(); + if (dirtytime_expire_interval) + schedule_delayed_work(&dirtytime_work, + round_jiffies_relative(dirtytime_expire_interval * HZ)); +} + +static int dirtytime_interval_handler(const struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + int ret; + + ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); + if (ret == 0 && write) { + if (dirtytime_expire_interval) + mod_delayed_work(system_percpu_wq, &dirtytime_work, 0); + else + cancel_delayed_work_sync(&dirtytime_work); + } + return ret; +} + +static const struct ctl_table vm_fs_writeback_table[] = { + { + .procname = "dirtytime_expire_seconds", + .data = &dirtytime_expire_interval, + .maxlen = sizeof(dirtytime_expire_interval), + .mode = 0644, + .proc_handler = dirtytime_interval_handler, + .extra1 = SYSCTL_ZERO, + }, +}; + +static int __init start_dirtytime_writeback(void) +{ + if (dirtytime_expire_interval) + schedule_delayed_work(&dirtytime_work, + round_jiffies_relative(dirtytime_expire_interval * HZ)); + register_sysctl_init("vm", vm_fs_writeback_table); + return 0; +} +__initcall(start_dirtytime_writeback); + +/** + * __mark_inode_dirty - internal function to mark an inode dirty + * + * @inode: inode to mark + * @flags: what kind of dirty, e.g. I_DIRTY_SYNC. This can be a combination of + * multiple I_DIRTY_* flags, except that I_DIRTY_TIME can't be combined + * with I_DIRTY_PAGES. + * + * Mark an inode as dirty. We notify the filesystem, then update the inode's + * dirty flags. Then, if needed we add the inode to the appropriate dirty list. + * + * Most callers should use mark_inode_dirty() or mark_inode_dirty_sync() + * instead of calling this directly. + * + * CAREFUL! We only add the inode to the dirty list if it is hashed or if it + * refers to a blockdev. Unhashed inodes will never be added to the dirty list + * even if they are later hashed, as they will have been marked dirty already. + * + * In short, ensure you hash any inodes _before_ you start marking them dirty. + * + * Note that for blockdevs, inode->dirtied_when represents the dirtying time of + * the block-special inode (/dev/hda1) itself. And the ->dirtied_when field of + * the kernel-internal blockdev inode represents the dirtying time of the + * blockdev's pages. This is why for I_DIRTY_PAGES we always use + * page->mapping->host, so the page-dirtying time is recorded in the internal + * blockdev inode. + */ +void __mark_inode_dirty(struct inode *inode, int flags) +{ + struct super_block *sb = inode->i_sb; + int dirtytime = 0; + struct bdi_writeback *wb = NULL; + + trace_writeback_mark_inode_dirty(inode, flags); + + if (flags & I_DIRTY_INODE) { + bool was_dirty_time = false; + + /* + * Inode timestamp update will piggback on this dirtying. + * We tell ->dirty_inode callback that timestamps need to + * be updated by setting I_DIRTY_TIME in flags. + */ + if (inode_state_read_once(inode) & I_DIRTY_TIME) { + spin_lock(&inode->i_lock); + if (inode_state_read(inode) & I_DIRTY_TIME) { + inode_state_clear(inode, I_DIRTY_TIME); + flags |= I_DIRTY_TIME; + was_dirty_time = true; + } + spin_unlock(&inode->i_lock); + } + + /* + * Notify the filesystem about the inode being dirtied, so that + * (if needed) it can update on-disk fields and journal the + * inode. This is only needed when the inode itself is being + * dirtied now. I.e. it's only needed for I_DIRTY_INODE, not + * for just I_DIRTY_PAGES or I_DIRTY_TIME. + */ + trace_writeback_dirty_inode_start(inode, flags); + if (sb->s_op->dirty_inode) { + sb->s_op->dirty_inode(inode, + flags & (I_DIRTY_INODE | I_DIRTY_TIME)); + } else if (was_dirty_time && inode->i_op->sync_lazytime) { + inode->i_op->sync_lazytime(inode); + } + trace_writeback_dirty_inode(inode, flags); + + /* I_DIRTY_INODE supersedes I_DIRTY_TIME. */ + flags &= ~I_DIRTY_TIME; + } else { + /* + * Else it's either I_DIRTY_PAGES, I_DIRTY_TIME, or nothing. + * (We don't support setting both I_DIRTY_PAGES and I_DIRTY_TIME + * in one call to __mark_inode_dirty().) + */ + dirtytime = flags & I_DIRTY_TIME; + WARN_ON_ONCE(dirtytime && flags != I_DIRTY_TIME); + } + + /* + * Paired with smp_mb() in __writeback_single_inode() for the + * following lockless i_state test. See there for details. + */ + smp_mb(); + + if ((inode_state_read_once(inode) & flags) == flags) + return; + + spin_lock(&inode->i_lock); + if ((inode_state_read(inode) & flags) != flags) { + const int was_dirty = inode_state_read(inode) & I_DIRTY; + + inode_attach_wb(inode, NULL); + + inode_state_set(inode, flags); + + /* + * Grab inode's wb early because it requires dropping i_lock and we + * need to make sure following checks happen atomically with dirty + * list handling so that we don't move inodes under flush worker's + * hands. + */ + if (!was_dirty) { + wb = locked_inode_to_wb_and_lock_list(inode); + spin_lock(&inode->i_lock); + } + + /* + * If the inode is queued for writeback by flush worker, just + * update its dirty state. Once the flush worker is done with + * the inode it will place it on the appropriate superblock + * list, based upon its state. + */ + if (inode_state_read(inode) & I_SYNC_QUEUED) + goto out_unlock; + + /* + * Only add valid (hashed) inodes to the superblock's + * dirty list. Add blockdev inodes as well. + */ + if (!S_ISBLK(inode->i_mode)) { + if (inode_unhashed(inode)) + goto out_unlock; + } + if (inode_state_read(inode) & I_FREEING) + goto out_unlock; + + /* + * If the inode was already on b_dirty/b_io/b_more_io, don't + * reposition it (that would break b_dirty time-ordering). + */ + if (!was_dirty) { + struct list_head *dirty_list; + bool wakeup_bdi = false; + + inode->dirtied_when = jiffies; + if (dirtytime) + inode->dirtied_time_when = jiffies; + + if (inode_state_read(inode) & I_DIRTY) + dirty_list = &wb->b_dirty; + else + dirty_list = &wb->b_dirty_time; + + wakeup_bdi = inode_io_list_move_locked(inode, wb, + dirty_list); + + /* + * If this is the first dirty inode for this bdi, + * we have to wake-up the corresponding bdi thread + * to make sure background write-back happens + * later. + */ + if (wakeup_bdi && + (wb->bdi->capabilities & BDI_CAP_WRITEBACK)) + wb_wakeup_delayed(wb); + + spin_unlock(&wb->list_lock); + spin_unlock(&inode->i_lock); + trace_writeback_dirty_inode_enqueue(inode); + + return; + } + } +out_unlock: + if (wb) + spin_unlock(&wb->list_lock); + spin_unlock(&inode->i_lock); +} +EXPORT_SYMBOL(__mark_inode_dirty); + +/* + * The @s_sync_lock is used to serialise concurrent sync operations + * to avoid lock contention problems with concurrent wait_sb_inodes() calls. + * Concurrent callers will block on the s_sync_lock rather than doing contending + * walks. The queueing maintains sync(2) required behaviour as all the IO that + * has been issued up to the time this function is enter is guaranteed to be + * completed by the time we have gained the lock and waited for all IO that is + * in progress regardless of the order callers are granted the lock. + */ +static void wait_sb_inodes(struct super_block *sb) +{ + LIST_HEAD(sync_list); + + /* + * We need to be protected against the filesystem going from + * r/o to r/w or vice versa. + */ + WARN_ON(!rwsem_is_locked(&sb->s_umount)); + + mutex_lock(&sb->s_sync_lock); + + /* + * Splice the writeback list onto a temporary list to avoid waiting on + * inodes that have started writeback after this point. + * + * Use rcu_read_lock() to keep the inodes around until we have a + * reference. s_inode_wblist_lock protects sb->s_inodes_wb as well as + * the local list because inodes can be dropped from either by writeback + * completion. + */ + rcu_read_lock(); + spin_lock_irq(&sb->s_inode_wblist_lock); + list_splice_init(&sb->s_inodes_wb, &sync_list); + + /* + * Data integrity sync. Must wait for all pages under writeback, because + * there may have been pages dirtied before our sync call, but which had + * writeout started before we write it out. In which case, the inode + * may not be on the dirty list, but we still have to wait for that + * writeout. + */ + while (!list_empty(&sync_list)) { + struct inode *inode = list_first_entry(&sync_list, struct inode, + i_wb_list); + struct address_space *mapping = inode->i_mapping; + + /* + * Move each inode back to the wb list before we drop the lock + * to preserve consistency between i_wb_list and the mapping + * writeback tag. Writeback completion is responsible to remove + * the inode from either list once the writeback tag is cleared. + */ + list_move_tail(&inode->i_wb_list, &sb->s_inodes_wb); + + /* + * The mapping can appear untagged while still on-list since we + * do not have the mapping lock. Skip it here, wb completion + * will remove it. + */ + if (!mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK)) + continue; + + spin_unlock_irq(&sb->s_inode_wblist_lock); + + spin_lock(&inode->i_lock); + if (inode_state_read(inode) & (I_FREEING | I_WILL_FREE | I_NEW)) { + spin_unlock(&inode->i_lock); + + spin_lock_irq(&sb->s_inode_wblist_lock); + continue; + } + __iget(inode); + spin_unlock(&inode->i_lock); + rcu_read_unlock(); + + /* + * We keep the error status of individual mapping so that + * applications can catch the writeback error using fsync(2). + * See filemap_fdatawait_keep_errors() for details. + */ + filemap_fdatawait_keep_errors(mapping); + + cond_resched(); + + iput(inode); + + rcu_read_lock(); + spin_lock_irq(&sb->s_inode_wblist_lock); + } + spin_unlock_irq(&sb->s_inode_wblist_lock); + rcu_read_unlock(); + mutex_unlock(&sb->s_sync_lock); +} + +static void __writeback_inodes_sb_nr(struct super_block *sb, unsigned long nr, + enum wb_reason reason, bool skip_if_busy) +{ + struct backing_dev_info *bdi = sb->s_bdi; + DEFINE_WB_COMPLETION(done, bdi); + struct wb_writeback_work work = { + .sb = sb, + .sync_mode = WB_SYNC_NONE, + .tagged_writepages = 1, + .done = &done, + .nr_pages = nr, + .reason = reason, + }; + + if (!bdi_has_dirty_io(bdi) || bdi == &noop_backing_dev_info) + return; + WARN_ON(!rwsem_is_locked(&sb->s_umount)); + + bdi_split_work_to_wbs(sb->s_bdi, &work, skip_if_busy); + wb_wait_for_completion(&done); +} + +/** + * writeback_inodes_sb_nr - writeback dirty inodes from given super_block + * @sb: the superblock + * @nr: the number of pages to write + * @reason: reason why some writeback work initiated + * + * Start writeback on some inodes on this super_block. No guarantees are made + * on how many (if any) will be written, and this function does not wait + * for IO completion of submitted IO. + */ +void writeback_inodes_sb_nr(struct super_block *sb, + unsigned long nr, + enum wb_reason reason) +{ + __writeback_inodes_sb_nr(sb, nr, reason, false); +} +EXPORT_SYMBOL(writeback_inodes_sb_nr); + +/** + * writeback_inodes_sb - writeback dirty inodes from given super_block + * @sb: the superblock + * @reason: reason why some writeback work was initiated + * + * Start writeback on some inodes on this super_block. No guarantees are made + * on how many (if any) will be written, and this function does not wait + * for IO completion of submitted IO. + */ +void writeback_inodes_sb(struct super_block *sb, enum wb_reason reason) +{ + writeback_inodes_sb_nr(sb, get_nr_dirty_pages(), reason); +} +EXPORT_SYMBOL(writeback_inodes_sb); + +/** + * try_to_writeback_inodes_sb - try to start writeback if none underway + * @sb: the superblock + * @reason: reason why some writeback work was initiated + * + * Invoke __writeback_inodes_sb_nr if no writeback is currently underway. + */ +void try_to_writeback_inodes_sb(struct super_block *sb, enum wb_reason reason) +{ + if (!down_read_trylock(&sb->s_umount)) + return; + + __writeback_inodes_sb_nr(sb, get_nr_dirty_pages(), reason, true); + up_read(&sb->s_umount); +} +EXPORT_SYMBOL(try_to_writeback_inodes_sb); + +/** + * sync_inodes_sb - sync sb inode pages + * @sb: the superblock + * + * This function writes and waits on any dirty inode belonging to this + * super_block. + */ +void sync_inodes_sb(struct super_block *sb) +{ + struct backing_dev_info *bdi = sb->s_bdi; + DEFINE_WB_COMPLETION(done, bdi); + struct wb_writeback_work work = { + .sb = sb, + .sync_mode = WB_SYNC_ALL, + .nr_pages = LONG_MAX, + .range_cyclic = 0, + .done = &done, + .reason = WB_REASON_SYNC, + .for_sync = 1, + }; + + /* + * Can't skip on !bdi_has_dirty() because we should wait for !dirty + * inodes under writeback and I_DIRTY_TIME inodes ignored by + * bdi_has_dirty() need to be written out too. + */ + if (bdi == &noop_backing_dev_info) + return; + + /* + * If the superblock has SB_I_NO_DATA_INTEGRITY set, there's no need to + * wait for the writeout to complete, as the filesystem cannot guarantee + * data persistence on sync. Just kick off writeback and return. + */ + if (sb->s_iflags & SB_I_NO_DATA_INTEGRITY) { + wakeup_flusher_threads_bdi(bdi, WB_REASON_SYNC); + return; + } + + WARN_ON(!rwsem_is_locked(&sb->s_umount)); + + /* protect against inode wb switch, see inode_switch_wbs_work_fn() */ + bdi_down_write_wb_switch_rwsem(bdi); + bdi_split_work_to_wbs(bdi, &work, false); + wb_wait_for_completion(&done); + bdi_up_write_wb_switch_rwsem(bdi); + + wait_sb_inodes(sb); +} +EXPORT_SYMBOL(sync_inodes_sb); + +/** + * write_inode_now - write an inode to disk + * @inode: inode to write to disk + * @sync: whether the write should be synchronous or not + * + * This function commits an inode to disk immediately if it is dirty. This is + * primarily needed by knfsd. + * + * The caller must either have a ref on the inode or must have set I_WILL_FREE. + */ +int write_inode_now(struct inode *inode, int sync) +{ + struct writeback_control wbc = { + .nr_to_write = LONG_MAX, + .sync_mode = sync ? WB_SYNC_ALL : WB_SYNC_NONE, + .range_start = 0, + .range_end = LLONG_MAX, + }; + + if (!mapping_can_writeback(inode->i_mapping)) + wbc.nr_to_write = 0; + + might_sleep(); + return writeback_single_inode(inode, &wbc); +} +EXPORT_SYMBOL(write_inode_now); + +/** + * sync_inode_metadata - write an inode to disk + * @inode: the inode to sync + * @wait: wait for I/O to complete. + * + * Write an inode to disk and adjust its dirty state after completion. + * + * Note: only writes the actual inode, no associated data or other metadata. + */ +int sync_inode_metadata(struct inode *inode, int wait) +{ + struct writeback_control wbc = { + .sync_mode = wait ? WB_SYNC_ALL : WB_SYNC_NONE, + .nr_to_write = 0, /* metadata-only */ + }; + + return writeback_single_inode(inode, &wbc); +} +EXPORT_SYMBOL(sync_inode_metadata); diff --git a/fs/fs_context.c b/fs/fs_context.c new file mode 100644 index 000000000..23ad66cd9 --- /dev/null +++ b/fs/fs_context.c @@ -0,0 +1,568 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Provide a way to create a superblock configuration context within the kernel + * that allows a superblock to be set up prior to mounting. + * + * Copyright (C) 2017 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "mount.h" +#include "internal.h" + +static const struct constant_table common_set_sb_flag[] = { + { "dirsync", SB_DIRSYNC }, + { "lazytime", SB_LAZYTIME }, + { "mand", SB_MANDLOCK }, + { "ro", SB_RDONLY }, + { "sync", SB_SYNCHRONOUS }, + { }, +}; + +static const struct constant_table common_clear_sb_flag[] = { + { "async", SB_SYNCHRONOUS }, + { "nolazytime", SB_LAZYTIME }, + { "nomand", SB_MANDLOCK }, + { "rw", SB_RDONLY }, + { }, +}; + +/* + * Check for a common mount option that manipulates s_flags. + */ +static int vfs_parse_sb_flag(struct fs_context *fc, const char *key) +{ + unsigned int token; + + token = lookup_constant(common_set_sb_flag, key, 0); + if (token) { + fc->sb_flags |= token; + fc->sb_flags_mask |= token; + return 0; + } + + token = lookup_constant(common_clear_sb_flag, key, 0); + if (token) { + fc->sb_flags &= ~token; + fc->sb_flags_mask |= token; + return 0; + } + + return -ENOPARAM; +} + +/** + * vfs_parse_fs_param_source - Handle setting "source" via parameter + * @fc: The filesystem context to modify + * @param: The parameter + * + * This is a simple helper for filesystems to verify that the "source" they + * accept is sane. + * + * Returns 0 on success, -ENOPARAM if this is not "source" parameter, and + * -EINVAL otherwise. In the event of failure, supplementary error information + * is logged. + */ +int vfs_parse_fs_param_source(struct fs_context *fc, struct fs_parameter *param) +{ + if (strcmp(param->key, "source") != 0) + return -ENOPARAM; + + if (param->type != fs_value_is_string) + return invalf(fc, "Non-string source"); + + if (fc->source) + return invalf(fc, "Multiple sources"); + + fc->source = param->string; + param->string = NULL; + return 0; +} +EXPORT_SYMBOL(vfs_parse_fs_param_source); + +/** + * vfs_parse_fs_param - Add a single parameter to a superblock config + * @fc: The filesystem context to modify + * @param: The parameter + * + * A single mount option in string form is applied to the filesystem context + * being set up. Certain standard options (for example "ro") are translated + * into flag bits without going to the filesystem. The active security module + * is allowed to observe and poach options. Any other options are passed over + * to the filesystem to parse. + * + * This may be called multiple times for a context. + * + * Returns 0 on success and a negative error code on failure. In the event of + * failure, supplementary error information may have been set. + */ +int vfs_parse_fs_param(struct fs_context *fc, struct fs_parameter *param) +{ + int ret; + + if (!param->key) + return invalf(fc, "Unnamed parameter\n"); + + ret = vfs_parse_sb_flag(fc, param->key); + if (ret != -ENOPARAM) + return ret; + + ret = security_fs_context_parse_param(fc, param); + if (ret != -ENOPARAM) + /* Param belongs to the LSM or is disallowed by the LSM; so + * don't pass to the FS. + */ + return ret; + + if (fc->ops->parse_param) { + ret = fc->ops->parse_param(fc, param); + if (ret != -ENOPARAM) + return ret; + } + + /* If the filesystem doesn't take any arguments, give it the + * default handling of source. + */ + ret = vfs_parse_fs_param_source(fc, param); + if (ret != -ENOPARAM) + return ret; + + return invalf(fc, "%s: Unknown parameter '%s'", + fc->fs_type->name, param->key); +} +EXPORT_SYMBOL(vfs_parse_fs_param); + +/** + * vfs_parse_fs_qstr - Convenience function to just parse a string. + * @fc: Filesystem context. + * @key: Parameter name. + * @value: Default value. + */ +int vfs_parse_fs_qstr(struct fs_context *fc, const char *key, + const struct qstr *value) +{ + int ret; + + struct fs_parameter param = { + .key = key, + .type = fs_value_is_flag, + .size = value ? value->len : 0, + }; + + if (value) { + param.string = kmemdup_nul(value->name, value->len, GFP_KERNEL); + if (!param.string) + return -ENOMEM; + param.type = fs_value_is_string; + } + + ret = vfs_parse_fs_param(fc, ¶m); + kfree(param.string); + return ret; +} +EXPORT_SYMBOL(vfs_parse_fs_qstr); + +/** + * vfs_parse_monolithic_sep - Parse key[=val][,key[=val]]* mount data + * @fc: The superblock configuration to fill in. + * @data: The data to parse + * @sep: callback for separating next option + * + * Parse a blob of data that's in key[=val][,key[=val]]* form with a custom + * option separator callback. + * + * Returns 0 on success or the error returned by the ->parse_option() fs_context + * operation on failure. + */ +int vfs_parse_monolithic_sep(struct fs_context *fc, void *data, + char *(*sep)(char **)) +{ + char *options = data, *key; + int ret = 0; + + if (!options) + return 0; + + ret = security_sb_eat_lsm_opts(options, &fc->security); + if (ret) + return ret; + + while ((key = sep(&options)) != NULL) { + if (*key) { + char *value = strchr(key, '='); + + if (value) { + if (unlikely(value == key)) + continue; + *value++ = 0; + } + ret = vfs_parse_fs_string(fc, key, value); + if (ret < 0) + break; + } + } + + return ret; +} +EXPORT_SYMBOL(vfs_parse_monolithic_sep); + +static char *vfs_parse_comma_sep(char **s) +{ + return strsep(s, ","); +} + +/** + * generic_parse_monolithic - Parse key[=val][,key[=val]]* mount data + * @fc: The superblock configuration to fill in. + * @data: The data to parse + * + * Parse a blob of data that's in key[=val][,key[=val]]* form. This can be + * called from the ->monolithic_mount_data() fs_context operation. + * + * Returns 0 on success or the error returned by the ->parse_option() fs_context + * operation on failure. + */ +int generic_parse_monolithic(struct fs_context *fc, void *data) +{ + return vfs_parse_monolithic_sep(fc, data, vfs_parse_comma_sep); +} +EXPORT_SYMBOL(generic_parse_monolithic); + +/** + * alloc_fs_context - Create a filesystem context. + * @fs_type: The filesystem type. + * @reference: The dentry from which this one derives (or NULL) + * @sb_flags: Filesystem/superblock flags (SB_*) + * @sb_flags_mask: Applicable members of @sb_flags + * @purpose: The purpose that this configuration shall be used for. + * + * Open a filesystem and create a mount context. The mount context is + * initialised with the supplied flags and, if a submount/automount from + * another superblock (referred to by @reference) is supplied, may have + * parameters such as namespaces copied across from that superblock. + */ +static struct fs_context *alloc_fs_context(struct file_system_type *fs_type, + struct dentry *reference, + unsigned int sb_flags, + unsigned int sb_flags_mask, + enum fs_context_purpose purpose) +{ + struct fs_context *fc; + int ret = -ENOMEM; + + fc = kzalloc_obj(struct fs_context, GFP_KERNEL_ACCOUNT); + if (!fc) + return ERR_PTR(-ENOMEM); + + fc->purpose = purpose; + fc->sb_flags = sb_flags; + fc->sb_flags_mask = sb_flags_mask; + fc->fs_type = get_filesystem(fs_type); + fc->cred = get_current_cred(); + fc->net_ns = get_net(current->nsproxy->net_ns); + fc->log.prefix = fs_type->name; + + mutex_init(&fc->uapi_mutex); + + switch (purpose) { + case FS_CONTEXT_FOR_MOUNT: + fc->user_ns = get_user_ns(fc->cred->user_ns); + break; + case FS_CONTEXT_FOR_SUBMOUNT: + fc->user_ns = get_user_ns(reference->d_sb->s_user_ns); + break; + case FS_CONTEXT_FOR_RECONFIGURE: + atomic_inc(&reference->d_sb->s_active); + fc->user_ns = get_user_ns(reference->d_sb->s_user_ns); + fc->root = dget(reference); + break; + } + + ret = fc->fs_type->init_fs_context(fc); + if (ret < 0) + goto err_fc; + fc->need_free = true; + return fc; + +err_fc: + put_fs_context(fc); + return ERR_PTR(ret); +} + +struct fs_context *fs_context_for_mount(struct file_system_type *fs_type, + unsigned int sb_flags) +{ + return alloc_fs_context(fs_type, NULL, sb_flags, 0, + FS_CONTEXT_FOR_MOUNT); +} +EXPORT_SYMBOL(fs_context_for_mount); + +struct fs_context *fs_context_for_reconfigure(struct dentry *dentry, + unsigned int sb_flags, + unsigned int sb_flags_mask) +{ + return alloc_fs_context(dentry->d_sb->s_type, dentry, sb_flags, + sb_flags_mask, FS_CONTEXT_FOR_RECONFIGURE); +} + +/** + * fs_context_for_submount: allocate a new fs_context for a submount + * @type: file_system_type of the new context + * @reference: reference dentry from which to copy relevant info + * + * Allocate a new fs_context suitable for a submount. This also ensures that + * the fc->security object is inherited from @reference (if needed). + */ +struct fs_context *fs_context_for_submount(struct file_system_type *type, + struct dentry *reference) +{ + struct fs_context *fc; + int ret; + + fc = alloc_fs_context(type, reference, 0, 0, FS_CONTEXT_FOR_SUBMOUNT); + if (IS_ERR(fc)) + return fc; + + ret = security_fs_context_submount(fc, reference->d_sb); + if (ret) { + put_fs_context(fc); + return ERR_PTR(ret); + } + + return fc; +} +EXPORT_SYMBOL(fs_context_for_submount); + +void fc_drop_locked(struct fs_context *fc) +{ + struct super_block *sb = fc->root->d_sb; + dput(fc->root); + fc->root = NULL; + deactivate_locked_super(sb); +} + +/** + * vfs_dup_fs_context - Duplicate a filesystem context. + * @src_fc: The context to copy. + */ +struct fs_context *vfs_dup_fs_context(struct fs_context *src_fc) +{ + struct fs_context *fc; + int ret; + + if (!src_fc->ops->dup) + return ERR_PTR(-EOPNOTSUPP); + + fc = kmemdup(src_fc, sizeof(struct fs_context), GFP_KERNEL); + if (!fc) + return ERR_PTR(-ENOMEM); + + mutex_init(&fc->uapi_mutex); + + fc->fs_private = NULL; + fc->s_fs_info = NULL; + fc->source = NULL; + fc->security = NULL; + get_filesystem(fc->fs_type); + get_net(fc->net_ns); + get_user_ns(fc->user_ns); + get_cred(fc->cred); + if (fc->log.log) + refcount_inc(&fc->log.log->usage); + + /* Can't call put until we've called ->dup */ + ret = fc->ops->dup(fc, src_fc); + if (ret < 0) + goto err_fc; + + ret = security_fs_context_dup(fc, src_fc); + if (ret < 0) + goto err_fc; + return fc; + +err_fc: + put_fs_context(fc); + return ERR_PTR(ret); +} +EXPORT_SYMBOL(vfs_dup_fs_context); + +/** + * logfc - Log a message to a filesystem context + * @log: The filesystem context to log to, or NULL to use printk. + * @prefix: A string to prefix the output with, or NULL. + * @level: 'w' for a warning, 'e' for an error. Anything else is a notice. + * @fmt: The format of the buffer. + */ +void logfc(struct fc_log *log, const char *prefix, char level, const char *fmt, ...) +{ + va_list va; + struct va_format vaf = {.fmt = fmt, .va = &va}; + + va_start(va, fmt); + if (!log) { + switch (level) { + case 'w': + printk(KERN_WARNING "%s%s%pV\n", prefix ? prefix : "", + prefix ? ": " : "", &vaf); + break; + case 'e': + printk(KERN_ERR "%s%s%pV\n", prefix ? prefix : "", + prefix ? ": " : "", &vaf); + break; + case 'i': + printk(KERN_INFO "%s%s%pV\n", prefix ? prefix : "", + prefix ? ": " : "", &vaf); + break; + default: + printk(KERN_NOTICE "%s%s%pV\n", prefix ? prefix : "", + prefix ? ": " : "", &vaf); + break; + } + } else { + unsigned int logsize = ARRAY_SIZE(log->buffer); + u8 index; + char *q = kasprintf(GFP_KERNEL, "%c %s%s%pV\n", level, + prefix ? prefix : "", + prefix ? ": " : "", &vaf); + + index = log->head & (logsize - 1); + BUILD_BUG_ON(sizeof(log->head) != sizeof(u8) || + sizeof(log->tail) != sizeof(u8)); + if ((u8)(log->head - log->tail) == logsize) { + /* The buffer is full, discard the oldest message */ + if (log->need_free & (1 << index)) + kfree(log->buffer[index]); + log->tail++; + } + + log->buffer[index] = q ? q : "OOM: Can't store error string"; + if (q) + log->need_free |= 1 << index; + else + log->need_free &= ~(1 << index); + log->head++; + } + va_end(va); +} +EXPORT_SYMBOL(logfc); + +/* + * Free a logging structure. + */ +static void put_fc_log(struct fs_context *fc) +{ + struct fc_log *log = fc->log.log; + int i; + + if (log) { + if (refcount_dec_and_test(&log->usage)) { + fc->log.log = NULL; + for (i = 0; i < ARRAY_SIZE(log->buffer) ; i++) + if (log->need_free & (1 << i)) + kfree(log->buffer[i]); + kfree(log); + } + } +} + +/** + * put_fs_context - Dispose of a superblock configuration context. + * @fc: The context to dispose of. + */ +void put_fs_context(struct fs_context *fc) +{ + struct super_block *sb; + + if (fc->root) { + sb = fc->root->d_sb; + dput(fc->root); + fc->root = NULL; + deactivate_super(sb); + } + + if (fc->need_free && fc->ops && fc->ops->free) + fc->ops->free(fc); + + security_free_mnt_opts(&fc->security); + put_net(fc->net_ns); + put_user_ns(fc->user_ns); + put_cred(fc->cred); + put_fc_log(fc); + put_filesystem(fc->fs_type); + kfree(fc->source); + kfree(fc); +} +EXPORT_SYMBOL(put_fs_context); + +int parse_monolithic_mount_data(struct fs_context *fc, void *data) +{ + int (*monolithic_mount_data)(struct fs_context *, void *); + + monolithic_mount_data = fc->ops->parse_monolithic; + if (!monolithic_mount_data) + monolithic_mount_data = generic_parse_monolithic; + + return monolithic_mount_data(fc, data); +} + +/* + * Clean up a context after performing an action on it and put it into a state + * from where it can be used to reconfigure a superblock. + * + * Note that here we do only the parts that can't fail; the rest is in + * finish_clean_context() below and in between those fs_context is marked + * FS_CONTEXT_AWAITING_RECONF. The reason for splitup is that after + * successful mount or remount we need to report success to userland. + * Trying to do full reinit (for the sake of possible subsequent remount) + * and failing to allocate memory would've put us into a nasty situation. + * So here we only discard the old state and reinitialization is left + * until we actually try to reconfigure. + */ +void vfs_clean_context(struct fs_context *fc) +{ + if (fc->need_free && fc->ops && fc->ops->free) + fc->ops->free(fc); + fc->need_free = false; + fc->fs_private = NULL; + fc->s_fs_info = NULL; + fc->sb_flags = 0; + security_free_mnt_opts(&fc->security); + kfree(fc->source); + fc->source = NULL; + fc->exclusive = false; + + fc->purpose = FS_CONTEXT_FOR_RECONFIGURE; + fc->phase = FS_CONTEXT_AWAITING_RECONF; +} + +int finish_clean_context(struct fs_context *fc) +{ + int error; + + if (fc->phase != FS_CONTEXT_AWAITING_RECONF) + return 0; + + error = fc->fs_type->init_fs_context(fc); + + if (unlikely(error)) { + fc->phase = FS_CONTEXT_FAILED; + return error; + } + fc->need_free = true; + fc->phase = FS_CONTEXT_RECONF_PARAMS; + return 0; +} diff --git a/fs/fs_dirent.c b/fs/fs_dirent.c new file mode 100644 index 000000000..e5e08f213 --- /dev/null +++ b/fs/fs_dirent.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +/* + * fs on-disk file type to dirent file type conversion + */ +static const unsigned char fs_dtype_by_ftype[FT_MAX] = { + [FT_UNKNOWN] = DT_UNKNOWN, + [FT_REG_FILE] = DT_REG, + [FT_DIR] = DT_DIR, + [FT_CHRDEV] = DT_CHR, + [FT_BLKDEV] = DT_BLK, + [FT_FIFO] = DT_FIFO, + [FT_SOCK] = DT_SOCK, + [FT_SYMLINK] = DT_LNK +}; + +/** + * fs_ftype_to_dtype() - fs on-disk file type to dirent type. + * @filetype: The on-disk file type to convert. + * + * This function converts the on-disk file type value (FT_*) to the directory + * entry type (DT_*). + * + * Context: Any context. + * Return: + * * DT_UNKNOWN - Unknown type + * * DT_FIFO - FIFO + * * DT_CHR - Character device + * * DT_DIR - Directory + * * DT_BLK - Block device + * * DT_REG - Regular file + * * DT_LNK - Symbolic link + * * DT_SOCK - Local-domain socket + */ +unsigned char fs_ftype_to_dtype(unsigned int filetype) +{ + if (filetype >= FT_MAX) + return DT_UNKNOWN; + + return fs_dtype_by_ftype[filetype]; +} +EXPORT_SYMBOL_GPL(fs_ftype_to_dtype); + +/* + * dirent file type to fs on-disk file type conversion + * Values not initialized explicitly are FT_UNKNOWN (0). + */ +static const unsigned char fs_ftype_by_dtype[DT_MAX] = { + [DT_REG] = FT_REG_FILE, + [DT_DIR] = FT_DIR, + [DT_LNK] = FT_SYMLINK, + [DT_CHR] = FT_CHRDEV, + [DT_BLK] = FT_BLKDEV, + [DT_FIFO] = FT_FIFO, + [DT_SOCK] = FT_SOCK, +}; + +/** + * fs_umode_to_ftype() - file mode to on-disk file type. + * @mode: The file mode to convert. + * + * This function converts the file mode value to the on-disk file type (FT_*). + * + * Context: Any context. + * Return: + * * FT_UNKNOWN - Unknown type + * * FT_REG_FILE - Regular file + * * FT_DIR - Directory + * * FT_CHRDEV - Character device + * * FT_BLKDEV - Block device + * * FT_FIFO - FIFO + * * FT_SOCK - Local-domain socket + * * FT_SYMLINK - Symbolic link + */ +unsigned char fs_umode_to_ftype(umode_t mode) +{ + return fs_ftype_by_dtype[S_DT(mode)]; +} +EXPORT_SYMBOL_GPL(fs_umode_to_ftype); + +/** + * fs_umode_to_dtype() - file mode to dirent file type. + * @mode: The file mode to convert. + * + * This function converts the file mode value to the directory + * entry type (DT_*). + * + * Context: Any context. + * Return: + * * DT_UNKNOWN - Unknown type + * * DT_FIFO - FIFO + * * DT_CHR - Character device + * * DT_DIR - Directory + * * DT_BLK - Block device + * * DT_REG - Regular file + * * DT_LNK - Symbolic link + * * DT_SOCK - Local-domain socket + */ +unsigned char fs_umode_to_dtype(umode_t mode) +{ + return fs_ftype_to_dtype(fs_umode_to_ftype(mode)); +} +EXPORT_SYMBOL_GPL(fs_umode_to_dtype); diff --git a/fs/fs_parser.c b/fs/fs_parser.c new file mode 100644 index 000000000..b4cc4cce5 --- /dev/null +++ b/fs/fs_parser.c @@ -0,0 +1,390 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Filesystem parameter parser. + * + * Copyright (C) 2018 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include "internal.h" + +static const struct constant_table bool_names[] = { + { "0", false }, + { "1", true }, + { "false", false }, + { "no", false }, + { "true", true }, + { "yes", true }, + { }, +}; + +static const struct constant_table * +__lookup_constant(const struct constant_table *tbl, const char *name) +{ + for ( ; tbl->name; tbl++) + if (strcmp(name, tbl->name) == 0) + return tbl; + return NULL; +} + +/** + * lookup_constant - Look up a constant by name in an ordered table + * @tbl: The table of constants to search. + * @name: The name to look up. + * @not_found: The value to return if the name is not found. + */ +int lookup_constant(const struct constant_table *tbl, const char *name, int not_found) +{ + const struct constant_table *p = __lookup_constant(tbl, name); + + return p ? p->value : not_found; +} +EXPORT_SYMBOL(lookup_constant); + +static inline bool is_flag(const struct fs_parameter_spec *p) +{ + return p->type == NULL; +} + +static const struct fs_parameter_spec *fs_lookup_key( + const struct fs_parameter_spec *desc, + struct fs_parameter *param, bool *negated) +{ + const struct fs_parameter_spec *p, *other = NULL; + const char *name = param->key; + bool want_flag = param->type == fs_value_is_flag; + + *negated = false; + for (p = desc; p->name; p++) { + if (strcmp(p->name, name) != 0) + continue; + if (likely(is_flag(p) == want_flag)) + return p; + other = p; + } + if (want_flag) { + if (name[0] == 'n' && name[1] == 'o' && name[2]) { + for (p = desc; p->name; p++) { + if (strcmp(p->name, name + 2) != 0) + continue; + if (!(p->flags & fs_param_neg_with_no)) + continue; + *negated = true; + return p; + } + } + } + return other; +} + +/* + * __fs_parse - Parse a filesystem configuration parameter + * @log: The filesystem context to log errors through. + * @desc: The parameter description to use. + * @param: The parameter. + * @result: Where to place the result of the parse + * + * Parse a filesystem configuration parameter and attempt a conversion for a + * simple parameter for which this is requested. If successful, the determined + * parameter ID is placed into @result->key, the desired type is indicated in + * @result->t and any converted value is placed into an appropriate member of + * the union in @result. + * + * The function returns the parameter number if the parameter was matched, + * -ENOPARAM if it wasn't matched and @desc->ignore_unknown indicated that + * unknown parameters are okay and -EINVAL if there was a conversion issue or + * the parameter wasn't recognised and unknowns aren't okay. + */ +int __fs_parse(struct p_log *log, + const struct fs_parameter_spec *desc, + struct fs_parameter *param, + struct fs_parse_result *result) +{ + const struct fs_parameter_spec *p; + + result->uint_64 = 0; + + p = fs_lookup_key(desc, param, &result->negated); + if (!p) + return -ENOPARAM; + + if (p->flags & fs_param_deprecated) + warn_plog(log, "Deprecated parameter '%s'", param->key); + + /* Try to turn the type we were given into the type desired by the + * parameter and give an error if we can't. + */ + if (is_flag(p)) { + if (param->type != fs_value_is_flag) + return inval_plog(log, "Unexpected value for '%s'", + param->key); + result->boolean = !result->negated; + } else { + int ret = p->type(log, p, param, result); + if (ret) + return ret; + } + return p->opt; +} +EXPORT_SYMBOL(__fs_parse); + +/** + * fs_lookup_param - Look up a path referred to by a parameter + * @fc: The filesystem context to log errors through. + * @param: The parameter. + * @want_bdev: T if want a blockdev + * @flags: Pathwalk flags passed to filename_lookup() + * @_path: The result of the lookup + */ +int fs_lookup_param(struct fs_context *fc, + struct fs_parameter *param, + bool want_bdev, + unsigned int flags, + struct path *_path) +{ + struct filename *f; + bool put_f; + int ret; + + switch (param->type) { + case fs_value_is_string: + f = getname_kernel(param->string); + if (IS_ERR(f)) + return PTR_ERR(f); + param->dirfd = AT_FDCWD; + put_f = true; + break; + case fs_value_is_filename: + f = param->name; + put_f = false; + break; + default: + return invalf(fc, "%s: not usable as path", param->key); + } + + ret = filename_lookup(param->dirfd, f, flags, _path, NULL); + if (ret < 0) { + errorf(fc, "%s: Lookup failure for '%s'", param->key, f->name); + goto out; + } + + if (want_bdev && + !S_ISBLK(d_backing_inode(_path->dentry)->i_mode)) { + path_put(_path); + _path->dentry = NULL; + _path->mnt = NULL; + errorf(fc, "%s: Non-blockdev passed as '%s'", + param->key, f->name); + ret = -ENOTBLK; + } + +out: + if (put_f) + putname(f); + return ret; +} +EXPORT_SYMBOL(fs_lookup_param); + +static int fs_param_bad_value(struct p_log *log, struct fs_parameter *param) +{ + return inval_plog(log, "Bad value for '%s'", param->key); +} + +int fs_param_is_bool(struct p_log *log, const struct fs_parameter_spec *p, + struct fs_parameter *param, struct fs_parse_result *result) +{ + int b; + if (param->type != fs_value_is_string) + return fs_param_bad_value(log, param); + if (!*param->string && (p->flags & fs_param_can_be_empty)) + return 0; + b = lookup_constant(bool_names, param->string, -1); + if (b == -1) + return fs_param_bad_value(log, param); + result->boolean = b; + return 0; +} +EXPORT_SYMBOL(fs_param_is_bool); + +int fs_param_is_u32(struct p_log *log, const struct fs_parameter_spec *p, + struct fs_parameter *param, struct fs_parse_result *result) +{ + int base = (unsigned long)p->data; + if (param->type != fs_value_is_string) + return fs_param_bad_value(log, param); + if (!*param->string && (p->flags & fs_param_can_be_empty)) + return 0; + if (kstrtouint(param->string, base, &result->uint_32) < 0) + return fs_param_bad_value(log, param); + return 0; +} +EXPORT_SYMBOL(fs_param_is_u32); + +int fs_param_is_s32(struct p_log *log, const struct fs_parameter_spec *p, + struct fs_parameter *param, struct fs_parse_result *result) +{ + if (param->type != fs_value_is_string) + return fs_param_bad_value(log, param); + if (!*param->string && (p->flags & fs_param_can_be_empty)) + return 0; + if (kstrtoint(param->string, 0, &result->int_32) < 0) + return fs_param_bad_value(log, param); + return 0; +} +EXPORT_SYMBOL(fs_param_is_s32); + +int fs_param_is_u64(struct p_log *log, const struct fs_parameter_spec *p, + struct fs_parameter *param, struct fs_parse_result *result) +{ + if (param->type != fs_value_is_string) + return fs_param_bad_value(log, param); + if (!*param->string && (p->flags & fs_param_can_be_empty)) + return 0; + if (kstrtoull(param->string, 0, &result->uint_64) < 0) + return fs_param_bad_value(log, param); + return 0; +} +EXPORT_SYMBOL(fs_param_is_u64); + +int fs_param_is_enum(struct p_log *log, const struct fs_parameter_spec *p, + struct fs_parameter *param, struct fs_parse_result *result) +{ + const struct constant_table *c; + if (param->type != fs_value_is_string) + return fs_param_bad_value(log, param); + if (!*param->string && (p->flags & fs_param_can_be_empty)) + return 0; + c = __lookup_constant(p->data, param->string); + if (!c) + return fs_param_bad_value(log, param); + result->uint_32 = c->value; + return 0; +} +EXPORT_SYMBOL(fs_param_is_enum); + +int fs_param_is_string(struct p_log *log, const struct fs_parameter_spec *p, + struct fs_parameter *param, struct fs_parse_result *result) +{ + if (param->type != fs_value_is_string || + (!*param->string && !(p->flags & fs_param_can_be_empty))) + return fs_param_bad_value(log, param); + return 0; +} +EXPORT_SYMBOL(fs_param_is_string); + +int fs_param_is_fd(struct p_log *log, const struct fs_parameter_spec *p, + struct fs_parameter *param, struct fs_parse_result *result) +{ + switch (param->type) { + case fs_value_is_string: + if ((!*param->string && !(p->flags & fs_param_can_be_empty)) || + kstrtouint(param->string, 0, &result->uint_32) < 0) + break; + if (result->uint_32 <= INT_MAX) + return 0; + break; + case fs_value_is_file: + result->uint_32 = param->dirfd; + if (result->uint_32 <= INT_MAX) + return 0; + break; + default: + break; + } + return fs_param_bad_value(log, param); +} +EXPORT_SYMBOL(fs_param_is_fd); + +int fs_param_is_file_or_string(struct p_log *log, + const struct fs_parameter_spec *p, + struct fs_parameter *param, + struct fs_parse_result *result) +{ + switch (param->type) { + case fs_value_is_string: + return fs_param_is_string(log, p, param, result); + case fs_value_is_file: + result->uint_32 = param->dirfd; + if (result->uint_32 <= INT_MAX) + return 0; + break; + default: + break; + } + return fs_param_bad_value(log, param); +} +EXPORT_SYMBOL(fs_param_is_file_or_string); + +int fs_param_is_uid(struct p_log *log, const struct fs_parameter_spec *p, + struct fs_parameter *param, struct fs_parse_result *result) +{ + kuid_t uid; + + if (fs_param_is_u32(log, p, param, result) != 0) + return fs_param_bad_value(log, param); + + uid = make_kuid(current_user_ns(), result->uint_32); + if (!uid_valid(uid)) + return inval_plog(log, "Invalid uid '%s'", param->string); + + result->uid = uid; + return 0; +} +EXPORT_SYMBOL(fs_param_is_uid); + +int fs_param_is_gid(struct p_log *log, const struct fs_parameter_spec *p, + struct fs_parameter *param, struct fs_parse_result *result) +{ + kgid_t gid; + + if (fs_param_is_u32(log, p, param, result) != 0) + return fs_param_bad_value(log, param); + + gid = make_kgid(current_user_ns(), result->uint_32); + if (!gid_valid(gid)) + return inval_plog(log, "Invalid gid '%s'", param->string); + + result->gid = gid; + return 0; +} +EXPORT_SYMBOL(fs_param_is_gid); + +int fs_param_is_blockdev(struct p_log *log, const struct fs_parameter_spec *p, + struct fs_parameter *param, struct fs_parse_result *result) +{ + return 0; +} +EXPORT_SYMBOL(fs_param_is_blockdev); + +#ifdef CONFIG_VALIDATE_FS_PARSER +/** + * fs_validate_description - Validate a parameter specification array + * @name: Owner name of the parameter specification array + * @desc: The parameter specification array to validate. + */ +bool fs_validate_description(const char *name, + const struct fs_parameter_spec *desc) +{ + const struct fs_parameter_spec *param, *p2; + bool good = true; + + for (param = desc; param->name; param++) { + /* Check for duplicate parameter names */ + for (p2 = desc; p2 < param; p2++) { + if (strcmp(param->name, p2->name) == 0) { + if (is_flag(param) != is_flag(p2)) + continue; + pr_err("VALIDATE %s: PARAM[%s]: Duplicate\n", + name, param->name); + good = false; + } + } + } + return good; +} +#endif /* CONFIG_VALIDATE_FS_PARSER */ diff --git a/fs/fs_pin.c b/fs/fs_pin.c new file mode 100644 index 000000000..47ef3c71c --- /dev/null +++ b/fs/fs_pin.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include "internal.h" +#include "mount.h" + +static DEFINE_SPINLOCK(pin_lock); + +void pin_remove(struct fs_pin *pin) +{ + spin_lock(&pin_lock); + hlist_del_init(&pin->m_list); + hlist_del_init(&pin->s_list); + spin_unlock(&pin_lock); + spin_lock_irq(&pin->wait.lock); + pin->done = 1; + wake_up_locked(&pin->wait); + spin_unlock_irq(&pin->wait.lock); +} + +void pin_insert(struct fs_pin *pin, struct vfsmount *m) +{ + spin_lock(&pin_lock); + hlist_add_head(&pin->s_list, &m->mnt_sb->s_pins); + hlist_add_head(&pin->m_list, &real_mount(m)->mnt_pins); + spin_unlock(&pin_lock); +} + +void pin_kill(struct fs_pin *p) +{ + wait_queue_entry_t wait; + + if (!p) { + rcu_read_unlock(); + return; + } + init_wait(&wait); + spin_lock_irq(&p->wait.lock); + if (likely(!p->done)) { + p->done = -1; + spin_unlock_irq(&p->wait.lock); + rcu_read_unlock(); + p->kill(p); + return; + } + if (p->done > 0) { + spin_unlock_irq(&p->wait.lock); + rcu_read_unlock(); + return; + } + __add_wait_queue(&p->wait, &wait); + while (1) { + set_current_state(TASK_UNINTERRUPTIBLE); + spin_unlock_irq(&p->wait.lock); + rcu_read_unlock(); + schedule(); + rcu_read_lock(); + if (likely(list_empty(&wait.entry))) + break; + /* OK, we know p couldn't have been freed yet */ + spin_lock_irq(&p->wait.lock); + if (p->done > 0) { + spin_unlock_irq(&p->wait.lock); + break; + } + } + rcu_read_unlock(); +} + +void mnt_pin_kill(struct mount *m) +{ + while (1) { + struct hlist_node *p; + rcu_read_lock(); + p = READ_ONCE(m->mnt_pins.first); + if (!p) { + rcu_read_unlock(); + break; + } + pin_kill(hlist_entry(p, struct fs_pin, m_list)); + } +} + +void group_pin_kill(struct hlist_head *p) +{ + while (1) { + struct hlist_node *q; + rcu_read_lock(); + q = READ_ONCE(p->first); + if (!q) { + rcu_read_unlock(); + break; + } + pin_kill(hlist_entry(q, struct fs_pin, s_list)); + } +} diff --git a/fs/fs_struct.c b/fs/fs_struct.c new file mode 100644 index 000000000..34699f3b6 --- /dev/null +++ b/fs/fs_struct.c @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" +#include "mount.h" + +/* + * Replace the fs->{rootmnt,root} with {mnt,dentry}. Put the old values. + * It can block. + */ +void set_fs_root(struct fs_struct *fs, const struct path *path) +{ + struct path old_root; + + path_get(path); + write_seqlock(&fs->seq); + old_root = fs->root; + fs->root = *path; + write_sequnlock(&fs->seq); + if (old_root.dentry) + path_put(&old_root); +} + +/* + * Replace the fs->{pwdmnt,pwd} with {mnt,dentry}. Put the old values. + * It can block. + */ +void set_fs_pwd(struct fs_struct *fs, const struct path *path) +{ + struct path old_pwd; + + path_get(path); + write_seqlock(&fs->seq); + old_pwd = fs->pwd; + fs->pwd = *path; + write_sequnlock(&fs->seq); + + if (old_pwd.dentry) + path_put(&old_pwd); +} + +static inline int replace_path(struct path *p, const struct path *old, const struct path *new) +{ + if (likely(p->dentry != old->dentry || p->mnt != old->mnt)) + return 0; + *p = *new; + return 1; +} + +void chroot_fs_refs(const struct path *old_root, const struct path *new_root) +{ + struct task_struct *g, *p; + struct fs_struct *fs; + int count = 0; + + read_lock(&tasklist_lock); + for_each_process_thread(g, p) { + if (p->flags & (PF_KTHREAD | PF_EXITING | PF_DUMPCORE)) + continue; + + task_lock(p); + fs = p->real_fs; + if (fs) { + int hits = 0; + write_seqlock(&fs->seq); + hits += replace_path(&fs->root, old_root, new_root); + hits += replace_path(&fs->pwd, old_root, new_root); + while (hits--) { + count++; + path_get(new_root); + } + write_sequnlock(&fs->seq); + } + task_unlock(p); + } + read_unlock(&tasklist_lock); + while (count--) + path_put(old_root); +} + +void free_fs_struct(struct fs_struct *fs) +{ + path_put(&fs->root); + path_put(&fs->pwd); + kmem_cache_free(fs_cachep, fs); +} + +void exit_fs(struct task_struct *tsk) +{ + struct fs_struct *fs = tsk->real_fs; + + if (fs) { + int kill; + task_lock(tsk); + read_seqlock_excl(&fs->seq); + tsk->real_fs = NULL; + tsk->fs = NULL; + kill = !--fs->users; + read_sequnlock_excl(&fs->seq); + task_unlock(tsk); + if (kill) + free_fs_struct(fs); + } +} + +struct fs_struct *copy_fs_struct(struct fs_struct *old) +{ + struct fs_struct *fs = kmem_cache_alloc(fs_cachep, GFP_KERNEL); + /* We don't need to lock fs - think why ;-) */ + if (fs) { + fs->users = 1; + fs->in_exec = 0; + seqlock_init(&fs->seq); + fs->umask = old->umask; + + read_seqlock_excl(&old->seq); + fs->root = old->root; + path_get(&fs->root); + fs->pwd = old->pwd; + path_get(&fs->pwd); + read_sequnlock_excl(&old->seq); + } + return fs; +} + +int unshare_fs_struct(void) +{ + struct fs_struct *fs = current->real_fs; + struct fs_struct *new_fs = copy_fs_struct(fs); + int kill; + + if (!new_fs) + return -ENOMEM; + + task_lock(current); + read_seqlock_excl(&fs->seq); + VFS_WARN_ON_ONCE(fs != current->fs); + kill = !--fs->users; + current->fs = new_fs; + current->real_fs = new_fs; + read_sequnlock_excl(&fs->seq); + task_unlock(current); + + if (kill) + free_fs_struct(fs); + + return 0; +} +EXPORT_SYMBOL_GPL(unshare_fs_struct); + +/* + * PID 1 may choose to stop sharing fs_struct state with us. + * Either via unshare(CLONE_FS) or unshare(CLONE_NEWNS). Of + * course, PID 1 could have chosen to create arbitrary process + * trees that all share fs_struct state via CLONE_FS. This is a + * strong statement: We only care about PID 1 aka the thread-group + * leader so subthread's fs_struct state doesn't matter. + * + * PID 1 unsharing fs_struct state is a bug. PID 1 relies on + * various kthreads to be able to perform work based on its + * fs_struct state. Breaking that contract sucks for both sides. + * So just don't bother with extra work for this. No sane init + * system should ever do this. + * + * On older kernels if PID 1 unshared its filesystem state with us the + * kernel simply used the stale fs_struct state implicitly pinning + * anything that PID 1 had last used. Even if PID 1 might've moved on to + * some completely different fs_struct state and might've even unmounted + * the old root. + * + * This has hilarious consequences: Think continuing to dump coredump + * state into an implicitly pinned directory somewhere. Calling random + * binaries in the old rootfs via usermodehelpers. + * + * Be aggressive about this: We simply reject operating on stale + * fs_struct state by reverting to nullfs. Every kworker that does + * lookups after this point will fail. Every usermodehelper call will + * fail. Tough luck but let's be kind and emit a warning to userspace. + */ +static inline void validate_fs_switch(struct fs_struct *old_fs) +{ + might_sleep(); + + if (likely(current->pid != 1)) + return; + /* @old_fs may be dangling but for comparison it's fine */ + if (old_fs != userspace_init_fs) + return; + pr_warn("VFS: Pid 1 stopped sharing filesystem state\n"); + set_fs_root(userspace_init_fs, &init_fs.root); + set_fs_pwd(userspace_init_fs, &init_fs.root); +} + +struct fs_struct *switch_fs_struct(struct fs_struct *new_fs) +{ + struct fs_struct *fs; + + scoped_guard(task_lock, current) { + fs = current->fs; + VFS_WARN_ON_ONCE(fs != current->real_fs); + read_seqlock_excl(&fs->seq); + current->fs = new_fs; + current->real_fs = new_fs; + if (--fs->users) + new_fs = NULL; + else + new_fs = fs; + read_sequnlock_excl(&fs->seq); + } + + validate_fs_switch(fs); + return new_fs; +} + +/* to be mentioned only in INIT_TASK */ +struct fs_struct init_fs = { + .users = 1, + .seq = __SEQLOCK_UNLOCKED(init_fs.seq), + .umask = 0022, +}; + +struct fs_struct *userspace_init_fs __ro_after_init; +EXPORT_SYMBOL_GPL(userspace_init_fs); + +void __init init_userspace_fs(void) +{ + struct mount *m; + struct path root; + + /* Move PID 1 from nullfs into the initramfs. */ + m = topmost_overmount(current->nsproxy->mnt_ns->root); + root.mnt = &m->mnt; + root.dentry = root.mnt->mnt_root; + + VFS_WARN_ON_ONCE(current->pid != 1); + + set_fs_root(current->fs, &root); + set_fs_pwd(current->fs, &root); + + /* Hold a reference for the global pointer. */ + read_seqlock_excl(¤t->fs->seq); + current->fs->users++; + read_sequnlock_excl(¤t->fs->seq); + + userspace_init_fs = current->fs; +} diff --git a/fs/fserror.c b/fs/fserror.c new file mode 100644 index 000000000..1e4d11fd9 --- /dev/null +++ b/fs/fserror.c @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2025 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include +#include +#include +#include + +#define FSERROR_DEFAULT_EVENT_POOL_SIZE (32) + +static struct mempool fserror_events_pool; + +void fserror_mount(struct super_block *sb) +{ + /* + * The pending error counter is biased by 1 so that we don't wake_var + * until we're actually trying to unmount. + */ + refcount_set(&sb->s_pending_errors, 1); +} + +void fserror_unmount(struct super_block *sb) +{ + /* + * If we don't drop the pending error count to zero, then wait for it + * to drop below 1, which means that the pending errors cleared and + * hopefully we didn't saturate with 1 billion+ concurrent events. + */ + if (!refcount_dec_and_test(&sb->s_pending_errors)) + wait_var_event(&sb->s_pending_errors, + refcount_read(&sb->s_pending_errors) < 1); +} + +static inline void fserror_pending_dec(struct super_block *sb) +{ + if (refcount_dec_and_test(&sb->s_pending_errors)) + wake_up_var(&sb->s_pending_errors); +} + +static inline void fserror_free_event(struct fserror_event *event) +{ + fserror_pending_dec(event->sb); + mempool_free(event, &fserror_events_pool); +} + +static void fserror_worker(struct work_struct *work) +{ + struct fserror_event *event = + container_of(work, struct fserror_event, work); + struct super_block *sb = event->sb; + + if (sb->s_flags & SB_ACTIVE) { + struct fs_error_report report = { + /* send positive error number to userspace */ + .error = -event->error, + .inode = event->inode, + .sb = event->sb, + }; + + if (sb->s_op->report_error) + sb->s_op->report_error(event); + + fsnotify(FS_ERROR, &report, FSNOTIFY_EVENT_ERROR, NULL, NULL, + NULL, 0); + } + + iput(event->inode); + fserror_free_event(event); +} + +static inline struct fserror_event *fserror_alloc_event(struct super_block *sb, + gfp_t gfp_flags) +{ + struct fserror_event *event = NULL; + + /* + * If pending_errors already reached zero or is no longer active, + * the superblock is being deactivated so there's no point in + * continuing. + * + * The order of the check of s_pending_errors and SB_ACTIVE are + * mandated by order of accesses in generic_shutdown_super and + * fserror_unmount. Barriers are implicitly provided by the refcount + * manipulations in this function and fserror_unmount. + */ + if (!refcount_inc_not_zero(&sb->s_pending_errors)) + return NULL; + if (!(sb->s_flags & SB_ACTIVE)) + goto out_pending; + + event = mempool_alloc(&fserror_events_pool, gfp_flags); + if (!event) + goto out_pending; + + /* mempool_alloc doesn't support GFP_ZERO */ + memset(event, 0, sizeof(*event)); + event->sb = sb; + INIT_WORK(&event->work, fserror_worker); + + return event; + +out_pending: + fserror_pending_dec(sb); + return NULL; +} + +/** + * fserror_report - report a filesystem error of some kind + * + * @sb: superblock of the filesystem + * @inode: inode within that filesystem, if applicable + * @type: type of error encountered + * @pos: start of inode range affected, if applicable + * @len: length of inode range affected, if applicable + * @error: error number encountered, must be negative + * @gfp: memory allocation flags for conveying the event to a worker, + * since this function can be called from atomic contexts + * + * Report details of a filesystem error to the super_operations::report_error + * callback if present; and to fsnotify for distribution to userspace. @sb, + * @gfp, @type, and @error must all be specified. For file I/O errors, the + * @inode, @pos, and @len fields must also be specified. For file metadata + * errors, @inode must be specified. If @inode is not NULL, then @inode->i_sb + * must point to @sb. + * + * Reporting work is deferred to a workqueue to ensure that ->report_error is + * called from process context without any locks held. An active reference to + * the inode is maintained until event handling is complete, and unmount will + * wait for queued events to drain. + */ +void fserror_report(struct super_block *sb, struct inode *inode, + enum fserror_type type, loff_t pos, u64 len, int error, + gfp_t gfp) +{ + struct fserror_event *event; + + /* sb and inode must be from the same filesystem */ + WARN_ON_ONCE(inode && inode->i_sb != sb); + + /* error number must be negative */ + WARN_ON_ONCE(error >= 0); + + event = fserror_alloc_event(sb, gfp); + if (!event) + goto lost; + + event->type = type; + event->pos = pos; + event->len = len; + event->error = error; + + /* + * Can't iput from non-sleeping context, so grabbing another reference + * to the inode must be the last thing before submitting the event. + */ + if (inode) { + event->inode = igrab(inode); + if (!event->inode) + goto lost_event; + } + + /* + * Use schedule_work here even if we're already in process context so + * that fsnotify and super_operations::report_error implementations are + * guaranteed to run in process context without any locks held. Since + * errors are supposed to be rare, the overhead shouldn't kill us any + * more than the failing device will. + */ + schedule_work(&event->work); + return; + +lost_event: + fserror_free_event(event); +lost: + if (inode) + pr_err_ratelimited( + "%s: lost file I/O error report for ino %llu type %u pos 0x%llx len 0x%llx error %d", + sb->s_id, inode->i_ino, type, pos, len, error); + else + pr_err_ratelimited( + "%s: lost filesystem error report for type %u error %d", + sb->s_id, type, error); +} +EXPORT_SYMBOL_GPL(fserror_report); + +static int __init fserror_init(void) +{ + return mempool_init_kmalloc_pool(&fserror_events_pool, + FSERROR_DEFAULT_EVENT_POOL_SIZE, + sizeof(struct fserror_event)); +} +fs_initcall(fserror_init); diff --git a/fs/fsopen.c b/fs/fsopen.c new file mode 100644 index 000000000..ae19e5136 --- /dev/null +++ b/fs/fsopen.c @@ -0,0 +1,491 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Filesystem access-by-fd. + * + * Copyright (C) 2017 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" +#include "mount.h" + +static inline const char *fetch_message_locked(struct fc_log *log, size_t len, + bool *need_free) +{ + const char *p; + int index; + + if (unlikely(log->head == log->tail)) + return ERR_PTR(-ENODATA); + + index = log->tail & (ARRAY_SIZE(log->buffer) - 1); + p = log->buffer[index]; + if (unlikely(strlen(p) > len)) + return ERR_PTR(-EMSGSIZE); + + log->buffer[index] = NULL; + *need_free = log->need_free & (1 << index); + log->need_free &= ~(1 << index); + log->tail++; + + return p; +} + +/* + * Allow the user to read back any error, warning or informational messages. + * Only one message is returned for each read(2) call. + */ +static ssize_t fscontext_read(struct file *file, + char __user *_buf, size_t len, loff_t *pos) +{ + struct fs_context *fc = file->private_data; + ssize_t err; + const char *p __free(kfree) = NULL, *message; + bool need_free; + int n; + + err = mutex_lock_interruptible(&fc->uapi_mutex); + if (err < 0) + return err; + message = fetch_message_locked(fc->log.log, len, &need_free); + mutex_unlock(&fc->uapi_mutex); + if (IS_ERR(message)) + return PTR_ERR(message); + + if (need_free) + p = message; + + n = strlen(message); + if (copy_to_user(_buf, message, n)) + return -EFAULT; + return n; +} + +static int fscontext_release(struct inode *inode, struct file *file) +{ + struct fs_context *fc = file->private_data; + + if (fc) { + file->private_data = NULL; + put_fs_context(fc); + } + return 0; +} + +const struct file_operations fscontext_fops = { + .read = fscontext_read, + .release = fscontext_release, +}; + +/* + * Attach a filesystem context to a file and an fd. + */ +static int fscontext_create_fd(struct fs_context *fc, unsigned int o_flags) +{ + int fd; + + fd = anon_inode_getfd("[fscontext]", &fscontext_fops, fc, + O_RDWR | o_flags); + if (fd < 0) + put_fs_context(fc); + return fd; +} + +static int fscontext_alloc_log(struct fs_context *fc) +{ + fc->log.log = kzalloc_obj(*fc->log.log); + if (!fc->log.log) + return -ENOMEM; + refcount_set(&fc->log.log->usage, 1); + fc->log.log->owner = fc->fs_type->owner; + return 0; +} + +/* + * Open a filesystem by name so that it can be configured for mounting. + * + * We are allowed to specify a container in which the filesystem will be + * opened, thereby indicating which namespaces will be used (notably, which + * network namespace will be used for network filesystems). + */ +SYSCALL_DEFINE2(fsopen, const char __user *, _fs_name, unsigned int, flags) +{ + struct file_system_type *fs_type; + struct fs_context *fc; + const char *fs_name; + int ret; + + if (!may_mount()) + return -EPERM; + + if (flags & ~FSOPEN_CLOEXEC) + return -EINVAL; + + fs_name = strndup_user(_fs_name, PAGE_SIZE); + if (IS_ERR(fs_name)) + return PTR_ERR(fs_name); + + fs_type = get_fs_type(fs_name); + kfree(fs_name); + if (!fs_type) + return -ENODEV; + + fc = fs_context_for_mount(fs_type, 0); + put_filesystem(fs_type); + if (IS_ERR(fc)) + return PTR_ERR(fc); + + fc->phase = FS_CONTEXT_CREATE_PARAMS; + + ret = fscontext_alloc_log(fc); + if (ret < 0) + goto err_fc; + + return fscontext_create_fd(fc, flags & FSOPEN_CLOEXEC ? O_CLOEXEC : 0); + +err_fc: + put_fs_context(fc); + return ret; +} + +/* + * Pick a superblock into a context for reconfiguration. + */ +SYSCALL_DEFINE3(fspick, int, dfd, const char __user *, path, unsigned int, flags) +{ + struct fs_context *fc; + struct path target; + unsigned int lookup_flags; + int ret; + + if (!may_mount()) + return -EPERM; + + if ((flags & ~(FSPICK_CLOEXEC | + FSPICK_SYMLINK_NOFOLLOW | + FSPICK_NO_AUTOMOUNT | + FSPICK_EMPTY_PATH)) != 0) + return -EINVAL; + + lookup_flags = LOOKUP_FOLLOW | LOOKUP_AUTOMOUNT; + if (flags & FSPICK_SYMLINK_NOFOLLOW) + lookup_flags &= ~LOOKUP_FOLLOW; + if (flags & FSPICK_NO_AUTOMOUNT) + lookup_flags &= ~LOOKUP_AUTOMOUNT; + CLASS(filename_flags, filename)(path, + (flags & FSPICK_EMPTY_PATH) ? LOOKUP_EMPTY : 0); + ret = filename_lookup(dfd, filename, lookup_flags, &target, NULL); + if (ret < 0) + goto err; + + ret = -EINVAL; + if (target.mnt->mnt_root != target.dentry) + goto err_path; + + fc = fs_context_for_reconfigure(target.dentry, 0, 0); + if (IS_ERR(fc)) { + ret = PTR_ERR(fc); + goto err_path; + } + + fc->phase = FS_CONTEXT_RECONF_PARAMS; + + ret = fscontext_alloc_log(fc); + if (ret < 0) + goto err_fc; + + path_put(&target); + return fscontext_create_fd(fc, flags & FSPICK_CLOEXEC ? O_CLOEXEC : 0); + +err_fc: + put_fs_context(fc); +err_path: + path_put(&target); +err: + return ret; +} + +static int vfs_cmd_create(struct fs_context *fc, bool exclusive) +{ + struct super_block *sb; + int ret; + + if (fc->phase != FS_CONTEXT_CREATE_PARAMS) + return -EBUSY; + + if (!mount_capable(fc)) + return -EPERM; + + fc->phase = FS_CONTEXT_CREATING; + fc->exclusive = exclusive; + + ret = vfs_get_tree(fc); + if (ret) { + fc->phase = FS_CONTEXT_FAILED; + return ret; + } + + sb = fc->root->d_sb; + ret = security_sb_kern_mount(sb); + if (unlikely(ret)) { + fc_drop_locked(fc); + fc->phase = FS_CONTEXT_FAILED; + return ret; + } + + /* vfs_get_tree() callchains will have grabbed @s_umount */ + up_write(&sb->s_umount); + fc->phase = FS_CONTEXT_AWAITING_MOUNT; + return 0; +} + +static int vfs_cmd_reconfigure(struct fs_context *fc) +{ + struct super_block *sb; + int ret; + + if (fc->phase != FS_CONTEXT_RECONF_PARAMS) + return -EBUSY; + + fc->phase = FS_CONTEXT_RECONFIGURING; + + sb = fc->root->d_sb; + if (!ns_capable(sb->s_user_ns, CAP_SYS_ADMIN)) { + fc->phase = FS_CONTEXT_FAILED; + return -EPERM; + } + + down_write(&sb->s_umount); + ret = reconfigure_super(fc); + up_write(&sb->s_umount); + if (ret) { + fc->phase = FS_CONTEXT_FAILED; + return ret; + } + + vfs_clean_context(fc); + return 0; +} + +/* + * Check the state and apply the configuration. Note that this function is + * allowed to 'steal' the value by setting param->xxx to NULL before returning. + */ +static int vfs_fsconfig_locked(struct fs_context *fc, int cmd, + struct fs_parameter *param) +{ + int ret; + + ret = finish_clean_context(fc); + if (ret) + return ret; + switch (cmd) { + case FSCONFIG_CMD_CREATE: + return vfs_cmd_create(fc, false); + case FSCONFIG_CMD_CREATE_EXCL: + return vfs_cmd_create(fc, true); + case FSCONFIG_CMD_RECONFIGURE: + return vfs_cmd_reconfigure(fc); + default: + if (fc->phase != FS_CONTEXT_CREATE_PARAMS && + fc->phase != FS_CONTEXT_RECONF_PARAMS) + return -EBUSY; + + return vfs_parse_fs_param(fc, param); + } +} + +/** + * sys_fsconfig - Set parameters and trigger actions on a context + * @fd: The filesystem context to act upon + * @cmd: The action to take + * @_key: Where appropriate, the parameter key to set + * @_value: Where appropriate, the parameter value to set + * @aux: Additional information for the value + * + * This system call is used to set parameters on a context, including + * superblock settings, data source and security labelling. + * + * Actions include triggering the creation of a superblock and the + * reconfiguration of the superblock attached to the specified context. + * + * When setting a parameter, @cmd indicates the type of value being proposed + * and @_key indicates the parameter to be altered. + * + * @_value and @aux are used to specify the value, should a value be required: + * + * (*) fsconfig_set_flag: No value is specified. The parameter must be boolean + * in nature. The key may be prefixed with "no" to invert the + * setting. @_value must be NULL and @aux must be 0. + * + * (*) fsconfig_set_string: A string value is specified. The parameter can be + * expecting boolean, integer, string or take a path. A conversion to an + * appropriate type will be attempted (which may include looking up as a + * path). @_value points to a NUL-terminated string and @aux must be 0. + * + * (*) fsconfig_set_binary: A binary blob is specified. @_value points to the + * blob and @aux indicates its size. The parameter must be expecting a + * blob. + * + * (*) fsconfig_set_path: A non-empty path is specified. The parameter must be + * expecting a path object. @_value points to a NUL-terminated string that + * is the path and @aux is a file descriptor at which to start a relative + * lookup or AT_FDCWD. + * + * (*) fsconfig_set_path_empty: As fsconfig_set_path, but with AT_EMPTY_PATH + * implied. + * + * (*) fsconfig_set_fd: An open file descriptor is specified. @_value must be + * NULL and @aux indicates the file descriptor. + */ +SYSCALL_DEFINE5(fsconfig, + int, fd, + unsigned int, cmd, + const char __user *, _key, + const void __user *, _value, + int, aux) +{ + struct fs_context *fc; + int ret; + int lookup_flags = 0; + + struct fs_parameter param = { + .type = fs_value_is_undefined, + }; + + if (fd < 0) + return -EINVAL; + + switch (cmd) { + case FSCONFIG_SET_FLAG: + if (!_key || _value || aux) + return -EINVAL; + break; + case FSCONFIG_SET_STRING: + if (!_key || !_value || aux) + return -EINVAL; + break; + case FSCONFIG_SET_BINARY: + if (!_key || !_value || aux <= 0 || aux > 1024 * 1024) + return -EINVAL; + break; + case FSCONFIG_SET_PATH: + case FSCONFIG_SET_PATH_EMPTY: + if (!_key || !_value || (aux != AT_FDCWD && aux < 0)) + return -EINVAL; + break; + case FSCONFIG_SET_FD: + if (!_key || _value || aux < 0) + return -EINVAL; + break; + case FSCONFIG_CMD_CREATE: + case FSCONFIG_CMD_CREATE_EXCL: + case FSCONFIG_CMD_RECONFIGURE: + if (_key || _value || aux) + return -EINVAL; + break; + default: + return -EOPNOTSUPP; + } + + CLASS(fd, f)(fd); + if (fd_empty(f)) + return -EBADF; + if (fd_file(f)->f_op != &fscontext_fops) + return -EINVAL; + + fc = fd_file(f)->private_data; + + if (_key) { + param.key = strndup_user(_key, 256); + if (IS_ERR(param.key)) + return PTR_ERR(param.key); + } + + switch (cmd) { + case FSCONFIG_SET_FLAG: + param.type = fs_value_is_flag; + break; + case FSCONFIG_SET_STRING: + param.type = fs_value_is_string; + param.string = strndup_user(_value, 256); + if (IS_ERR(param.string)) { + ret = PTR_ERR(param.string); + goto out_key; + } + param.size = strlen(param.string); + break; + case FSCONFIG_SET_BINARY: + param.type = fs_value_is_blob; + param.size = aux; + param.blob = memdup_user_nul(_value, aux); + if (IS_ERR(param.blob)) { + ret = PTR_ERR(param.blob); + goto out_key; + } + break; + case FSCONFIG_SET_PATH_EMPTY: + lookup_flags = LOOKUP_EMPTY; + fallthrough; + case FSCONFIG_SET_PATH: + param.type = fs_value_is_filename; + param.name = getname_flags(_value, lookup_flags); + if (IS_ERR(param.name)) { + ret = PTR_ERR(param.name); + goto out_key; + } + param.dirfd = aux; + param.size = strlen(param.name->name); + break; + case FSCONFIG_SET_FD: + param.type = fs_value_is_file; + ret = -EBADF; + param.file = fget_raw(aux); + if (!param.file) + goto out_key; + param.dirfd = aux; + break; + default: + break; + } + + ret = mutex_lock_interruptible(&fc->uapi_mutex); + if (ret == 0) { + ret = vfs_fsconfig_locked(fc, cmd, ¶m); + mutex_unlock(&fc->uapi_mutex); + } + + /* Clean up the our record of any value that we obtained from + * userspace. Note that the value may have been stolen by the LSM or + * filesystem, in which case the value pointer will have been cleared. + */ + switch (cmd) { + case FSCONFIG_SET_STRING: + case FSCONFIG_SET_BINARY: + kfree(param.string); + break; + case FSCONFIG_SET_PATH: + case FSCONFIG_SET_PATH_EMPTY: + if (param.name) + putname(param.name); + break; + case FSCONFIG_SET_FD: + if (param.file) + fput(param.file); + break; + default: + break; + } +out_key: + kfree(param.key); + return ret; +} diff --git a/fs/fuse/Kconfig b/fs/fuse/Kconfig new file mode 100644 index 000000000..3a4ae632c --- /dev/null +++ b/fs/fuse/Kconfig @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: GPL-2.0-only +config FUSE_FS + tristate "FUSE (Filesystem in Userspace) support" + select FS_POSIX_ACL + select FS_IOMAP + help + With FUSE it is possible to implement a fully functional filesystem + in a userspace program. + + There's also a companion library: libfuse2. This library is available + from the FUSE homepage: + + although chances are your distribution already has that library + installed if you've installed the "fuse" package itself. + + See for more information. + See for needed library/utility version. + + If you want to develop a userspace FS, or if you want to use + a filesystem based on FUSE, answer Y or M. + +config CUSE + tristate "Character device in Userspace support" + depends on FUSE_FS + help + This FUSE extension allows character devices to be + implemented in userspace. + + If you want to develop or use a userspace character device + based on CUSE, answer Y or M. + +config VIRTIO_FS + tristate "Virtio Filesystem" + depends on FUSE_FS + select VIRTIO + help + The Virtio Filesystem allows guests to mount file systems from the + host. + + If you want to share files between guests or with the host, answer Y + or M. + +config FUSE_DAX + bool "Virtio Filesystem Direct Host Memory Access support" + default y + select INTERVAL_TREE + depends on VIRTIO_FS + depends on FS_DAX + depends on DAX + help + This allows bypassing guest page cache and allows mapping host page + cache directly in guest address space. + + If you want to allow mounting a Virtio Filesystem with the "dax" + option, answer Y. + +config FUSE_PASSTHROUGH + bool "FUSE passthrough operations support" + default y + depends on FUSE_FS + select FS_STACK + help + This allows bypassing FUSE server by mapping specific FUSE operations + to be performed directly on a backing file. + + If you want to allow passthrough operations, answer Y. + +config FUSE_IO_URING + bool "FUSE communication over io-uring" + default y + depends on FUSE_FS + depends on IO_URING + help + This allows sending FUSE requests over the io-uring interface and + also adds request core affinity. + + If you want to allow fuse server/client communication through io-uring, + answer Y diff --git a/fs/fuse/Makefile b/fs/fuse/Makefile new file mode 100644 index 000000000..245e67852 --- /dev/null +++ b/fs/fuse/Makefile @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the FUSE filesystem. +# + +# Needed for trace events +ccflags-y = -I$(src) + +obj-$(CONFIG_FUSE_FS) += fuse.o +obj-$(CONFIG_CUSE) += cuse.o +obj-$(CONFIG_VIRTIO_FS) += virtiofs.o + +fuse-y := trace.o # put trace.o first so we see ftrace errors sooner +fuse-y += dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o req_timeout.o req.o +fuse-y += poll.o notify.o +fuse-y += iomode.o +fuse-$(CONFIG_FUSE_DAX) += dax.o +fuse-$(CONFIG_FUSE_PASSTHROUGH) += passthrough.o backing.o +fuse-$(CONFIG_SYSCTL) += sysctl.o +fuse-$(CONFIG_FUSE_IO_URING) += dev_uring.o + +virtiofs-y := virtio_fs.o diff --git a/fs/fuse/acl.c b/fs/fuse/acl.c new file mode 100644 index 000000000..31fb50e16 --- /dev/null +++ b/fs/fuse/acl.c @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * FUSE: Filesystem in Userspace + * Copyright (C) 2016 Canonical Ltd. + */ + +#include "fuse_i.h" + +#include +#include + +static struct posix_acl *__fuse_get_acl(struct fuse_conn *fc, + struct inode *inode, int type, bool rcu) +{ + int size; + const char *name; + void *value = NULL; + struct posix_acl *acl; + + if (rcu) + return ERR_PTR(-ECHILD); + + if (fuse_is_bad(inode)) + return ERR_PTR(-EIO); + + if (fc->no_getxattr) + return NULL; + + if (type == ACL_TYPE_ACCESS) + name = XATTR_NAME_POSIX_ACL_ACCESS; + else if (type == ACL_TYPE_DEFAULT) + name = XATTR_NAME_POSIX_ACL_DEFAULT; + else + return ERR_PTR(-EOPNOTSUPP); + + value = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!value) + return ERR_PTR(-ENOMEM); + size = fuse_getxattr(inode, name, value, PAGE_SIZE); + if (size > 0) + acl = posix_acl_from_xattr(fc->user_ns, value, size); + else if ((size == 0) || (size == -ENODATA) || + (size == -EOPNOTSUPP && fc->no_getxattr)) + acl = NULL; + else if (size == -ERANGE) + acl = ERR_PTR(-E2BIG); + else + acl = ERR_PTR(size); + + kfree(value); + return acl; +} + +static inline bool fuse_no_acl(const struct fuse_conn *fc, + const struct inode *inode) +{ + /* + * Refuse interacting with POSIX ACLs for daemons that + * don't support FUSE_POSIX_ACL and are not mounted on + * the host to retain backwards compatibility. + */ + return !fc->posix_acl && (i_user_ns(inode) != &init_user_ns); +} + +struct posix_acl *fuse_get_acl(struct mnt_idmap *idmap, + struct dentry *dentry, int type) +{ + struct inode *inode = d_inode(dentry); + struct fuse_conn *fc = get_fuse_conn(inode); + + if (fuse_no_acl(fc, inode)) + return ERR_PTR(-EOPNOTSUPP); + + return __fuse_get_acl(fc, inode, type, false); +} + +struct posix_acl *fuse_get_inode_acl(struct inode *inode, int type, bool rcu) +{ + struct fuse_conn *fc = get_fuse_conn(inode); + + /* + * FUSE daemons before FUSE_POSIX_ACL was introduced could get and set + * POSIX ACLs without them being used for permission checking by the + * vfs. Retain that behavior for backwards compatibility as there are + * filesystems that do all permission checking for acls in the daemon + * and not in the kernel. + */ + if (!fc->posix_acl) + return NULL; + return __fuse_get_acl(fc, inode, type, rcu); +} + +int fuse_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type) +{ + struct inode *inode = d_inode(dentry); + struct fuse_conn *fc = get_fuse_conn(inode); + const char *name; + int ret; + + if (fuse_is_bad(inode)) + return -EIO; + + if (fc->no_setxattr || fuse_no_acl(fc, inode)) + return -EOPNOTSUPP; + + if (type == ACL_TYPE_ACCESS) + name = XATTR_NAME_POSIX_ACL_ACCESS; + else if (type == ACL_TYPE_DEFAULT) + name = XATTR_NAME_POSIX_ACL_DEFAULT; + else + return -EINVAL; + + if (acl) { + unsigned int extra_flags = 0; + /* + * Fuse userspace is responsible for updating access + * permissions in the inode, if needed. fuse_setxattr + * invalidates the inode attributes, which will force + * them to be refreshed the next time they are used, + * and it also updates i_ctime. + */ + size_t size; + void *value; + + value = posix_acl_to_xattr(fc->user_ns, acl, &size, GFP_KERNEL); + if (!value) + return -ENOMEM; + + if (size > PAGE_SIZE) { + kfree(value); + return -E2BIG; + } + + /* + * Fuse daemons without FUSE_POSIX_ACL never changed the passed + * through POSIX ACLs. Such daemons don't expect setgid bits to + * be stripped. + */ + if (fc->posix_acl && + !in_group_or_capable(idmap, inode, + i_gid_into_vfsgid(idmap, inode))) + extra_flags |= FUSE_SETXATTR_ACL_KILL_SGID; + + ret = fuse_setxattr(inode, name, value, size, 0, extra_flags); + kfree(value); + } else { + ret = fuse_removexattr(inode, name); + } + + if (fc->posix_acl) { + /* + * Fuse daemons without FUSE_POSIX_ACL never cached POSIX ACLs + * and didn't invalidate attributes. Retain that behavior. + */ + forget_all_cached_acls(inode); + fuse_invalidate_attr(inode); + } + + return ret; +} diff --git a/fs/fuse/args.h b/fs/fuse/args.h new file mode 100644 index 000000000..ecfe51a19 --- /dev/null +++ b/fs/fuse/args.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _FS_FUSE_ARGS_H +#define _FS_FUSE_ARGS_H + +#include + +struct fuse_mount; + +/** One input argument of a request */ +struct fuse_in_arg { + unsigned size; + const void *value; +}; + +/** One output argument of a request */ +struct fuse_arg { + unsigned size; + void *value; +}; + +struct fuse_args { + u64 nodeid; + u32 opcode; + u32 uid; + u32 gid; + u32 pid; + u8 in_numargs; + u8 out_numargs; + u8 ext_idx; + bool force:1; + bool noreply:1; + bool nocreds:1; + bool in_pages:1; + bool out_pages:1; + bool user_pages:1; + bool out_argvar:1; + bool page_zeroing:1; + bool page_replace:1; + bool may_block:1; + bool is_ext:1; + bool is_pinned:1; + bool invalidate_vmap:1; + bool abort_on_kill:1; + struct fuse_in_arg in_args[4]; + struct fuse_arg out_args[2]; + void (*end)(struct fuse_args *args, int error); + /* Used for kvec iter backed by vmalloc address */ + void *vmap_base; +}; + +/** FUSE folio descriptor */ +struct fuse_folio_desc { + unsigned int length; + unsigned int offset; +}; + +struct fuse_args_pages { + struct fuse_args args; + struct folio **folios; + struct fuse_folio_desc *descs; + unsigned int num_folios; +}; + +#endif /* _FS_FUSE_ARGS_H */ diff --git a/fs/fuse/backing.c b/fs/fuse/backing.c new file mode 100644 index 000000000..472b6afa7 --- /dev/null +++ b/fs/fuse/backing.c @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * FUSE passthrough to backing file. + * + * Copyright (c) 2023 CTERA Networks. + */ + +#include "dev.h" +#include "fuse_i.h" + +#include + +struct fuse_backing *fuse_backing_get(struct fuse_backing *fb) +{ + if (fb && refcount_inc_not_zero(&fb->count)) + return fb; + return NULL; +} + +static void fuse_backing_free(struct fuse_backing *fb) +{ + pr_debug("%s: fb=0x%p\n", __func__, fb); + + if (fb->file) + fput(fb->file); + put_cred(fb->cred); + kfree_rcu(fb, rcu); +} + +void fuse_backing_put(struct fuse_backing *fb) +{ + if (fb && refcount_dec_and_test(&fb->count)) + fuse_backing_free(fb); +} + +void fuse_backing_files_init(struct fuse_conn *fc) +{ + idr_init(&fc->backing_files_map); +} + +static int fuse_backing_id_alloc(struct fuse_conn *fc, struct fuse_backing *fb) +{ + int id; + + idr_preload(GFP_KERNEL); + spin_lock(&fc->lock); + /* FIXME: xarray might be space inefficient */ + id = idr_alloc_cyclic(&fc->backing_files_map, fb, 1, 0, GFP_ATOMIC); + spin_unlock(&fc->lock); + idr_preload_end(); + + WARN_ON_ONCE(id == 0); + return id; +} + +static struct fuse_backing *fuse_backing_id_remove(struct fuse_conn *fc, + int id) +{ + struct fuse_backing *fb; + + spin_lock(&fc->lock); + fb = idr_remove(&fc->backing_files_map, id); + spin_unlock(&fc->lock); + + return fb; +} + +static int fuse_backing_id_free(int id, void *p, void *data) +{ + struct fuse_backing *fb = p; + + WARN_ON_ONCE(refcount_read(&fb->count) != 1); + fuse_backing_free(fb); + return 0; +} + +void fuse_backing_files_free(struct fuse_conn *fc) +{ + idr_for_each(&fc->backing_files_map, fuse_backing_id_free, NULL); + idr_destroy(&fc->backing_files_map); +} + +int fuse_backing_open(struct fuse_conn *fc, struct fuse_backing_map *map) +{ + struct file *file; + struct super_block *backing_sb; + struct fuse_backing *fb = NULL; + int res; + + pr_debug("%s: fd=%d flags=0x%x\n", __func__, map->fd, map->flags); + + /* TODO: relax CAP_SYS_ADMIN once backing files are visible to lsof */ + res = -EPERM; + if (!fc->passthrough || !capable(CAP_SYS_ADMIN)) + goto out; + + res = -EINVAL; + if (map->flags || map->padding) + goto out; + + file = fget_raw(map->fd); + res = -EBADF; + if (!file) + goto out; + + /* read/write/splice/mmap passthrough only relevant for regular files */ + res = d_is_dir(file->f_path.dentry) ? -EISDIR : -EINVAL; + if (!d_is_reg(file->f_path.dentry)) + goto out_fput; + + backing_sb = file_inode(file)->i_sb; + res = -ELOOP; + if (backing_sb->s_stack_depth >= fc->max_stack_depth) + goto out_fput; + + fb = kmalloc_obj(struct fuse_backing); + res = -ENOMEM; + if (!fb) + goto out_fput; + + fb->file = file; + fb->cred = get_current_cred(); + refcount_set(&fb->count, 1); + + res = fuse_backing_id_alloc(fc, fb); + if (res < 0) { + fuse_backing_free(fb); + fb = NULL; + } + +out: + pr_debug("%s: fb=0x%p, ret=%i\n", __func__, fb, res); + + return res; + +out_fput: + fput(file); + goto out; +} + +int fuse_backing_close(struct fuse_conn *fc, int backing_id) +{ + struct fuse_backing *fb = NULL; + int err; + + pr_debug("%s: backing_id=%d\n", __func__, backing_id); + + /* TODO: relax CAP_SYS_ADMIN once backing files are visible to lsof */ + err = -EPERM; + if (!fc->passthrough || !capable(CAP_SYS_ADMIN)) + goto out; + + err = -EINVAL; + if (backing_id <= 0) + goto out; + + err = -ENOENT; + fb = fuse_backing_id_remove(fc, backing_id); + if (!fb) + goto out; + + fuse_backing_put(fb); + err = 0; +out: + pr_debug("%s: fb=0x%p, err=%i\n", __func__, fb, err); + + return err; +} + +struct fuse_backing *fuse_backing_lookup(struct fuse_conn *fc, int backing_id) +{ + struct fuse_backing *fb; + + rcu_read_lock(); + fb = idr_find(&fc->backing_files_map, backing_id); + fb = fuse_backing_get(fb); + rcu_read_unlock(); + + return fb; +} diff --git a/fs/fuse/control.c b/fs/fuse/control.c new file mode 100644 index 000000000..21ffde596 --- /dev/null +++ b/fs/fuse/control.c @@ -0,0 +1,366 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + FUSE: Filesystem in Userspace + Copyright (C) 2001-2008 Miklos Szeredi +*/ + +#include "fuse_i.h" +#include "dev.h" + +#include +#include +#include +#include + +#define FUSE_CTL_SUPER_MAGIC 0x65735543 + +/* + * This is non-NULL when the single instance of the control filesystem + * exists. Protected by fuse_mutex + */ +static struct super_block *fuse_control_sb; + +static struct fuse_conn *fuse_ctl_file_conn_get(struct file *file) +{ + struct fuse_conn *fc; + mutex_lock(&fuse_mutex); + fc = file_inode(file)->i_private; + if (fc) + fc = fuse_conn_get(fc); + mutex_unlock(&fuse_mutex); + return fc; +} + +static ssize_t fuse_conn_abort_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct fuse_conn *fc = fuse_ctl_file_conn_get(file); + if (fc) { + fuse_chan_abort(fc->chan, fc->abort_err); + fuse_conn_put(fc); + } + return count; +} + +static ssize_t fuse_conn_waiting_read(struct file *file, char __user *buf, + size_t len, loff_t *ppos) +{ + char tmp[32]; + size_t size; + + if (!*ppos) { + long value; + struct fuse_conn *fc = fuse_ctl_file_conn_get(file); + if (!fc) + return 0; + + value = fuse_chan_num_waiting(fc->chan); + file->private_data = (void *)value; + fuse_conn_put(fc); + } + size = sprintf(tmp, "%ld\n", (long)file->private_data); + return simple_read_from_buffer(buf, len, ppos, tmp, size); +} + +static ssize_t fuse_conn_limit_read(struct file *file, char __user *buf, + size_t len, loff_t *ppos, unsigned val) +{ + char tmp[32]; + size_t size = sprintf(tmp, "%u\n", val); + + return simple_read_from_buffer(buf, len, ppos, tmp, size); +} + +static ssize_t fuse_conn_limit_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos, unsigned *val, + unsigned global_limit) +{ + unsigned long t; + unsigned limit = (1 << 16) - 1; + int err; + + if (*ppos) + return -EINVAL; + + err = kstrtoul_from_user(buf, count, 0, &t); + if (err) + return err; + + if (!capable(CAP_SYS_ADMIN)) + limit = min(limit, global_limit); + + if (t > limit) + return -EINVAL; + + *val = t; + + return count; +} + +static ssize_t fuse_conn_max_background_read(struct file *file, + char __user *buf, size_t len, + loff_t *ppos) +{ + struct fuse_conn *fc; + unsigned val; + + fc = fuse_ctl_file_conn_get(file); + if (!fc) + return 0; + + val = fuse_chan_max_background(fc->chan); + fuse_conn_put(fc); + + return fuse_conn_limit_read(file, buf, len, ppos, val); +} + +static ssize_t fuse_conn_max_background_write(struct file *file, + const char __user *buf, + size_t count, loff_t *ppos) +{ + unsigned int val = 0; + ssize_t ret; + + ret = fuse_conn_limit_write(file, buf, count, ppos, &val, + max_user_bgreq); + if (ret > 0) { + struct fuse_conn *fc = fuse_ctl_file_conn_get(file); + if (fc) { + fuse_chan_max_background_set(fc->chan, val); + fuse_conn_put(fc); + } + } + + return ret; +} + +static ssize_t fuse_conn_congestion_threshold_read(struct file *file, + char __user *buf, size_t len, + loff_t *ppos) +{ + struct fuse_conn *fc; + unsigned val; + + fc = fuse_ctl_file_conn_get(file); + if (!fc) + return 0; + + val = READ_ONCE(fc->congestion_threshold); + fuse_conn_put(fc); + + return fuse_conn_limit_read(file, buf, len, ppos, val); +} + +static ssize_t fuse_conn_congestion_threshold_write(struct file *file, + const char __user *buf, + size_t count, loff_t *ppos) +{ + unsigned int val = 0; + struct fuse_conn *fc; + ssize_t ret; + + ret = fuse_conn_limit_write(file, buf, count, ppos, &val, + max_user_congthresh); + if (ret <= 0) + goto out; + fc = fuse_ctl_file_conn_get(file); + if (!fc) + goto out; + + WRITE_ONCE(fc->congestion_threshold, val); + fuse_conn_put(fc); +out: + return ret; +} + +static const struct file_operations fuse_ctl_abort_ops = { + .open = nonseekable_open, + .write = fuse_conn_abort_write, +}; + +static const struct file_operations fuse_ctl_waiting_ops = { + .open = nonseekable_open, + .read = fuse_conn_waiting_read, +}; + +static const struct file_operations fuse_conn_max_background_ops = { + .open = nonseekable_open, + .read = fuse_conn_max_background_read, + .write = fuse_conn_max_background_write, +}; + +static const struct file_operations fuse_conn_congestion_threshold_ops = { + .open = nonseekable_open, + .read = fuse_conn_congestion_threshold_read, + .write = fuse_conn_congestion_threshold_write, +}; + +static struct dentry *fuse_ctl_add_dentry(struct dentry *parent, + struct fuse_conn *fc, + const char *name, int mode, + const struct inode_operations *iop, + const struct file_operations *fop) +{ + struct dentry *dentry; + struct inode *inode; + + dentry = d_alloc_name(parent, name); + if (!dentry) + return NULL; + + inode = new_inode(fuse_control_sb); + if (!inode) { + dput(dentry); + return NULL; + } + + inode->i_ino = get_next_ino(); + inode->i_mode = mode; + inode->i_uid = fc->user_id; + inode->i_gid = fc->group_id; + simple_inode_init_ts(inode); + /* setting ->i_op to NULL is not allowed */ + if (iop) + inode->i_op = iop; + inode->i_fop = fop; + if (S_ISDIR(mode)) { + inc_nlink(d_inode(parent)); + inc_nlink(inode); + } + inode->i_private = fc; + d_make_persistent(dentry, inode); + dput(dentry); + + /* + * We are returning a borrowed reference here - it's only good while + * fuse_mutex is held. Actually it's d_make_persistent() return + * value... + */ + return dentry; +} + +/* + * Add a connection to the control filesystem (if it exists). Caller + * must hold fuse_mutex + */ +int fuse_ctl_add_conn(struct fuse_conn *fc) +{ + struct dentry *parent; + char name[32]; + + if (!fuse_control_sb || fc->no_control) + return 0; + + parent = fuse_control_sb->s_root; + sprintf(name, "%u", fc->dev); + parent = fuse_ctl_add_dentry(parent, fc, name, S_IFDIR | 0500, + &simple_dir_inode_operations, + &simple_dir_operations); + if (!parent) + goto err; + + if (!fuse_ctl_add_dentry(parent, fc, "waiting", S_IFREG | 0400, + NULL, &fuse_ctl_waiting_ops) || + !fuse_ctl_add_dentry(parent, fc, "abort", S_IFREG | 0200, + NULL, &fuse_ctl_abort_ops) || + !fuse_ctl_add_dentry(parent, fc, "max_background", S_IFREG | 0600, + NULL, &fuse_conn_max_background_ops) || + !fuse_ctl_add_dentry(parent, fc, "congestion_threshold", + S_IFREG | 0600, NULL, + &fuse_conn_congestion_threshold_ops)) + goto err; + + return 0; + + err: + fuse_ctl_remove_conn(fc); + return -ENOMEM; +} + +static void remove_one(struct dentry *dentry) +{ + d_inode(dentry)->i_private = NULL; +} + +/* + * Remove a connection from the control filesystem (if it exists). + * Caller must hold fuse_mutex + */ +void fuse_ctl_remove_conn(struct fuse_conn *fc) +{ + char name[32]; + + if (!fuse_control_sb || fc->no_control) + return; + + sprintf(name, "%u", fc->dev); + simple_remove_by_name(fuse_control_sb->s_root, name, remove_one); +} + +static int fuse_ctl_fill_super(struct super_block *sb, struct fs_context *fsc) +{ + static const struct tree_descr empty_descr = {""}; + struct fuse_conn *fc; + int err; + + err = simple_fill_super(sb, FUSE_CTL_SUPER_MAGIC, &empty_descr); + if (err) + return err; + + mutex_lock(&fuse_mutex); + BUG_ON(fuse_control_sb); + fuse_control_sb = sb; + list_for_each_entry(fc, &fuse_conn_list, entry) { + err = fuse_ctl_add_conn(fc); + if (err) { + fuse_control_sb = NULL; + mutex_unlock(&fuse_mutex); + return err; + } + } + mutex_unlock(&fuse_mutex); + + return 0; +} + +static int fuse_ctl_get_tree(struct fs_context *fsc) +{ + return get_tree_single(fsc, fuse_ctl_fill_super); +} + +static const struct fs_context_operations fuse_ctl_context_ops = { + .get_tree = fuse_ctl_get_tree, +}; + +static int fuse_ctl_init_fs_context(struct fs_context *fsc) +{ + fsc->ops = &fuse_ctl_context_ops; + return 0; +} + +static void fuse_ctl_kill_sb(struct super_block *sb) +{ + mutex_lock(&fuse_mutex); + fuse_control_sb = NULL; + mutex_unlock(&fuse_mutex); + + kill_anon_super(sb); +} + +static struct file_system_type fuse_ctl_fs_type = { + .owner = THIS_MODULE, + .name = "fusectl", + .init_fs_context = fuse_ctl_init_fs_context, + .kill_sb = fuse_ctl_kill_sb, +}; +MODULE_ALIAS_FS("fusectl"); + +int __init fuse_ctl_init(void) +{ + return register_filesystem(&fuse_ctl_fs_type); +} + +void __exit fuse_ctl_cleanup(void) +{ + unregister_filesystem(&fuse_ctl_fs_type); +} diff --git a/fs/fuse/cuse.c b/fs/fuse/cuse.c new file mode 100644 index 000000000..3c15b5ba1 --- /dev/null +++ b/fs/fuse/cuse.c @@ -0,0 +1,663 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * CUSE: Character device in Userspace + * + * Copyright (C) 2008-2009 SUSE Linux Products GmbH + * Copyright (C) 2008-2009 Tejun Heo + * + * CUSE enables character devices to be implemented from userland much + * like FUSE allows filesystems. On initialization /dev/cuse is + * created. By opening the file and replying to the CUSE_INIT request + * userland CUSE server can create a character device. After that the + * operation is very similar to FUSE. + * + * A CUSE instance involves the following objects. + * + * cuse_conn : contains fuse_conn and serves as bonding structure + * channel : file handle connected to the userland CUSE server + * cdev : the implemented character device + * dev : generic device for cdev + * + * Note that 'channel' is what 'dev' is in FUSE. As CUSE deals with + * devices, it's called 'channel' to reduce confusion. + * + * channel determines when the character device dies. When channel is + * closed, everything begins to destruct. The cuse_conn is taken off + * the lookup table preventing further access from cdev, cdev and + * generic device are removed and the base reference of cuse_conn is + * put. + * + * On each open, the matching cuse_conn is looked up and if found an + * additional reference is taken which is released when the file is + * closed. + */ + +#define pr_fmt(fmt) "CUSE: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dev.h" +#include "fuse_i.h" +#include "fuse_dev_i.h" + +#define CUSE_CONNTBL_LEN 64 + +struct cuse_conn { + struct list_head list; /* linked on cuse_conntbl */ + struct fuse_mount fm; /* Dummy mount referencing fc */ + struct fuse_conn fc; /* fuse connection */ + struct cdev *cdev; /* associated character device */ + struct device *dev; /* device representing @cdev */ + + /* init parameters, set once during initialization */ + bool unrestricted_ioctl; +}; + +static DEFINE_MUTEX(cuse_lock); /* protects registration */ +static struct list_head cuse_conntbl[CUSE_CONNTBL_LEN]; +static struct class *cuse_class; + +static struct cuse_conn *fc_to_cc(struct fuse_conn *fc) +{ + return container_of(fc, struct cuse_conn, fc); +} + +static struct list_head *cuse_conntbl_head(dev_t devt) +{ + return &cuse_conntbl[(MAJOR(devt) + MINOR(devt)) % CUSE_CONNTBL_LEN]; +} + + +/************************************************************************** + * CUSE frontend operations + * + * These are file operations for the character device. + * + * On open, CUSE opens a file from the FUSE mnt and stores it to + * private_data of the open file. All other ops call FUSE ops on the + * FUSE file. + */ + +static ssize_t cuse_read_iter(struct kiocb *kiocb, struct iov_iter *to) +{ + struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(kiocb); + loff_t pos = 0; + + return fuse_direct_io(&io, to, &pos, FUSE_DIO_CUSE); +} + +static ssize_t cuse_write_iter(struct kiocb *kiocb, struct iov_iter *from) +{ + struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(kiocb); + loff_t pos = 0; + /* + * No locking or generic_write_checks(), the server is + * responsible for locking and sanity checks. + */ + return fuse_direct_io(&io, from, &pos, + FUSE_DIO_WRITE | FUSE_DIO_CUSE); +} + +static int cuse_open(struct inode *inode, struct file *file) +{ + dev_t devt = inode->i_cdev->dev; + struct cuse_conn *cc = NULL, *pos; + int rc; + + /* look up and get the connection */ + mutex_lock(&cuse_lock); + list_for_each_entry(pos, cuse_conntbl_head(devt), list) + if (pos->dev->devt == devt) { + fuse_conn_get(&pos->fc); + cc = pos; + break; + } + mutex_unlock(&cuse_lock); + + /* dead? */ + if (!cc) + return -ENODEV; + + /* + * Generic permission check is already done against the chrdev + * file, proceed to open. + */ + rc = fuse_do_open(&cc->fm, 0, file, 0); + if (rc) + fuse_conn_put(&cc->fc); + return rc; +} + +static int cuse_release(struct inode *inode, struct file *file) +{ + struct fuse_file *ff = file->private_data; + struct fuse_mount *fm = ff->fm; + + fuse_sync_release(NULL, ff, file->f_flags); + fuse_conn_put(fm->fc); + + return 0; +} + +static long cuse_file_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct fuse_file *ff = file->private_data; + struct cuse_conn *cc = fc_to_cc(ff->fm->fc); + unsigned int flags = 0; + + if (cc->unrestricted_ioctl) + flags |= FUSE_IOCTL_UNRESTRICTED; + + return fuse_do_ioctl(file, cmd, arg, flags); +} + +static long cuse_file_compat_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct fuse_file *ff = file->private_data; + struct cuse_conn *cc = fc_to_cc(ff->fm->fc); + unsigned int flags = FUSE_IOCTL_COMPAT; + + if (cc->unrestricted_ioctl) + flags |= FUSE_IOCTL_UNRESTRICTED; + + return fuse_do_ioctl(file, cmd, arg, flags); +} + +static const struct file_operations cuse_frontend_fops = { + .owner = THIS_MODULE, + .read_iter = cuse_read_iter, + .write_iter = cuse_write_iter, + .open = cuse_open, + .release = cuse_release, + .unlocked_ioctl = cuse_file_ioctl, + .compat_ioctl = cuse_file_compat_ioctl, + .poll = fuse_file_poll, + .llseek = noop_llseek, +}; + + +/************************************************************************** + * CUSE channel initialization and destruction + */ + +struct cuse_devinfo { + const char *name; +}; + +/** + * cuse_parse_one - parse one key=value pair + * @pp: i/o parameter for the current position + * @end: points to one past the end of the packed string + * @keyp: out parameter for key + * @valp: out parameter for value + * + * *@pp points to packed strings - "key0=val0\0key1=val1\0" which ends + * at @end - 1. This function parses one pair and set *@keyp to the + * start of the key and *@valp to the start of the value. Note that + * the original string is modified such that the key string is + * terminated with '\0'. *@pp is updated to point to the next string. + * + * RETURNS: + * 1 on successful parse, 0 on EOF, -errno on failure. + */ +static int cuse_parse_one(char **pp, char *end, char **keyp, char **valp) +{ + char *p = *pp; + char *key, *val; + + while (p < end && *p == '\0') + p++; + if (p == end) + return 0; + + if (end[-1] != '\0') { + pr_err("info not properly terminated\n"); + return -EINVAL; + } + + key = val = p; + p += strlen(p); + + if (valp) { + strsep(&val, "="); + if (!val) + val = key + strlen(key); + key = strstrip(key); + val = strstrip(val); + } else + key = strstrip(key); + + if (!strlen(key)) { + pr_err("zero length info key specified\n"); + return -EINVAL; + } + + *pp = p; + *keyp = key; + if (valp) + *valp = val; + + return 1; +} + +/** + * cuse_parse_devinfo - parse device info + * @p: device info string + * @len: length of device info string + * @devinfo: out parameter for parsed device info + * + * Parse @p to extract device info and store it into @devinfo. String + * pointed to by @p is modified by parsing and @devinfo points into + * them, so @p shouldn't be freed while @devinfo is in use. + * + * RETURNS: + * 0 on success, -errno on failure. + */ +static int cuse_parse_devinfo(char *p, size_t len, struct cuse_devinfo *devinfo) +{ + char *end = p + len; + char *key, *val; + int rc; + + while (true) { + rc = cuse_parse_one(&p, end, &key, &val); + if (rc < 0) + return rc; + if (!rc) + break; + if (strcmp(key, "DEVNAME") == 0) + devinfo->name = val; + else + pr_warn("unknown device info \"%s\"\n", key); + } + + if (!devinfo->name || !strlen(devinfo->name)) { + pr_err("DEVNAME unspecified\n"); + return -EINVAL; + } + + return 0; +} + +static void cuse_gendev_release(struct device *dev) +{ + kfree(dev); +} + +struct cuse_init_args { + struct fuse_args_pages ap; + struct cuse_init_in in; + struct cuse_init_out out; + struct folio *folio; + struct fuse_folio_desc desc; + struct fuse_conn *fc; +}; + +/** + * cuse_process_init_reply - finish initializing CUSE channel + * + * @fm: The fuse mount information containing the CUSE connection. + * @args: The arguments passed to the init reply. + * @error: The error code signifying if any error occurred during the process. + * + * This function creates the character device and sets up all the + * required data structures for it. Please read the comment at the + * top of this file for high level overview. + */ +static void cuse_process_init_reply(struct fuse_args *args, int error) +{ + struct cuse_init_args *ia = container_of(args, typeof(*ia), ap.args); + struct fuse_conn *fc = ia->fc; + struct fuse_args_pages *ap = &ia->ap; + struct cuse_conn *cc = fc_to_cc(fc), *pos; + struct cuse_init_out *arg = &ia->out; + struct folio *folio = ap->folios[0]; + struct cuse_devinfo devinfo = { }; + struct device *dev; + struct cdev *cdev; + dev_t devt; + int rc, i; + + if (error || arg->major != FUSE_KERNEL_VERSION || arg->minor < 11) + goto err; + + fc->minor = arg->minor; + fc->max_read = max_t(unsigned, arg->max_read, 4096); + fc->max_write = max_t(unsigned, arg->max_write, 4096); + + /* parse init reply */ + cc->unrestricted_ioctl = arg->flags & CUSE_UNRESTRICTED_IOCTL; + + rc = cuse_parse_devinfo(folio_address(folio), ap->args.out_args[1].size, + &devinfo); + if (rc) + goto err; + + /* determine and reserve devt */ + devt = MKDEV(arg->dev_major, arg->dev_minor); + if (!MAJOR(devt)) + rc = alloc_chrdev_region(&devt, MINOR(devt), 1, devinfo.name); + else + rc = register_chrdev_region(devt, 1, devinfo.name); + if (rc) { + pr_err("failed to register chrdev region\n"); + goto err; + } + + /* devt determined, create device */ + rc = -ENOMEM; + dev = kzalloc_obj(*dev); + if (!dev) + goto err_region; + + device_initialize(dev); + dev_set_uevent_suppress(dev, 1); + dev->class = cuse_class; + dev->devt = devt; + dev->release = cuse_gendev_release; + dev_set_drvdata(dev, cc); + dev_set_name(dev, "%s", devinfo.name); + + mutex_lock(&cuse_lock); + + /* make sure the device-name is unique */ + for (i = 0; i < CUSE_CONNTBL_LEN; ++i) { + list_for_each_entry(pos, &cuse_conntbl[i], list) + if (!strcmp(dev_name(pos->dev), dev_name(dev))) + goto err_unlock; + } + + rc = device_add(dev); + if (rc) + goto err_unlock; + + /* register cdev */ + rc = -ENOMEM; + cdev = cdev_alloc(); + if (!cdev) + goto err_dev; + + cdev->owner = THIS_MODULE; + cdev->ops = &cuse_frontend_fops; + + rc = cdev_add(cdev, devt, 1); + if (rc) + goto err_cdev; + + cc->dev = dev; + cc->cdev = cdev; + + /* make the device available */ + list_add(&cc->list, cuse_conntbl_head(devt)); + mutex_unlock(&cuse_lock); + + /* announce device availability */ + dev_set_uevent_suppress(dev, 0); + kobject_uevent(&dev->kobj, KOBJ_ADD); +out: + kfree(ia); + folio_put(folio); + return; + +err_cdev: + cdev_del(cdev); +err_dev: + device_del(dev); +err_unlock: + mutex_unlock(&cuse_lock); + put_device(dev); +err_region: + unregister_chrdev_region(devt, 1); +err: + fuse_chan_abort(fc->chan, false); + goto out; +} + +static int cuse_send_init(struct cuse_conn *cc) +{ + int rc; + struct folio *folio; + struct fuse_mount *fm = &cc->fm; + struct cuse_init_args *ia; + struct fuse_args_pages *ap; + + BUILD_BUG_ON(CUSE_INIT_INFO_MAX > PAGE_SIZE); + + rc = -ENOMEM; + + folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0); + if (!folio) + goto err; + + ia = kzalloc_obj(*ia); + if (!ia) + goto err_free_folio; + + ap = &ia->ap; + ia->in.major = FUSE_KERNEL_VERSION; + ia->in.minor = FUSE_KERNEL_MINOR_VERSION; + ia->in.flags |= CUSE_UNRESTRICTED_IOCTL; + ap->args.opcode = CUSE_INIT; + ap->args.in_numargs = 1; + ap->args.in_args[0].size = sizeof(ia->in); + ap->args.in_args[0].value = &ia->in; + ap->args.out_numargs = 2; + ap->args.out_args[0].size = sizeof(ia->out); + ap->args.out_args[0].value = &ia->out; + ap->args.out_args[1].size = CUSE_INIT_INFO_MAX; + ap->args.out_argvar = true; + ap->args.out_pages = true; + ap->num_folios = 1; + ap->folios = &ia->folio; + ap->descs = &ia->desc; + ia->folio = folio; + ia->desc.length = ap->args.out_args[1].size; + ia->fc = &cc->fc; + ap->args.end = cuse_process_init_reply; + + rc = fuse_simple_background(fm, &ap->args, GFP_KERNEL); + if (rc) { + kfree(ia); +err_free_folio: + folio_put(folio); + } +err: + return rc; +} + +static void cuse_fc_release(struct fuse_conn *fc) +{ + kfree(fc_to_cc(fc)); +} + +/** + * cuse_channel_open - open method for /dev/cuse + * @inode: inode for /dev/cuse + * @file: file struct being opened + * + * Userland CUSE server can create a CUSE device by opening /dev/cuse + * and replying to the initialization request kernel sends. This + * function is responsible for handling CUSE device initialization. + * Because the fd opened by this function is used during + * initialization, this function only creates cuse_conn and sends + * init. The rest is delegated to a kthread. + * + * RETURNS: + * 0 on success, -errno on failure. + */ +static int cuse_channel_open(struct inode *inode, struct file *file) +{ + struct fuse_dev *fud; + struct cuse_conn *cc; + struct fuse_chan *fch __free(fuse_chan_free) = fuse_dev_chan_new(); + int rc; + + if (!fch) + return -ENOMEM; + + /* set up cuse_conn */ + cc = kzalloc_obj(*cc); + if (!cc) + return -ENOMEM; + + /* + * Limit the cuse channel to requests that can + * be represented in file->f_cred->user_ns. + */ + fuse_conn_init(&cc->fc, &cc->fm, file->f_cred->user_ns, no_free_ptr(fch)); + cc->fc.release = cuse_fc_release; + fud = fuse_dev_alloc_install(cc->fc.chan); + fuse_conn_put(&cc->fc); + if (!fud) + return -ENOMEM; + + INIT_LIST_HEAD(&cc->list); + + cc->fc.chan->initialized = 1; + rc = cuse_send_init(cc); + if (rc) { + fuse_dev_put(fud); + return rc; + } + file->private_data = fud; + + return 0; +} + +/** + * cuse_channel_release - release method for /dev/cuse + * @inode: inode for /dev/cuse + * @file: file struct being closed + * + * Disconnect the channel, deregister CUSE device and initiate + * destruction by putting the default reference. + * + * RETURNS: + * 0 on success, -errno on failure. + */ +static int cuse_channel_release(struct inode *inode, struct file *file) +{ + struct fuse_dev *fud = __fuse_get_dev(file); + struct cuse_conn *cc = fc_to_cc(fud->chan->conn); + + /* remove from the conntbl, no more access from this point on */ + mutex_lock(&cuse_lock); + list_del_init(&cc->list); + mutex_unlock(&cuse_lock); + + /* remove device */ + if (cc->dev) + device_unregister(cc->dev); + if (cc->cdev) { + unregister_chrdev_region(cc->cdev->dev, 1); + cdev_del(cc->cdev); + } + + return fuse_dev_release(inode, file); +} + +static struct file_operations cuse_channel_fops; /* initialized during init */ + + +/************************************************************************** + * Misc stuff and module initializatiion + * + * CUSE exports the same set of attributes to sysfs as fusectl. + */ + +static ssize_t cuse_class_waiting_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct cuse_conn *cc = dev_get_drvdata(dev); + + return sprintf(buf, "%d\n", atomic_read(&cc->fc.chan->num_waiting)); +} +static DEVICE_ATTR(waiting, 0400, cuse_class_waiting_show, NULL); + +static ssize_t cuse_class_abort_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct cuse_conn *cc = dev_get_drvdata(dev); + + fuse_chan_abort(cc->fc.chan, false); + return count; +} +static DEVICE_ATTR(abort, 0200, NULL, cuse_class_abort_store); + +static struct attribute *cuse_class_dev_attrs[] = { + &dev_attr_waiting.attr, + &dev_attr_abort.attr, + NULL, +}; +ATTRIBUTE_GROUPS(cuse_class_dev); + +static struct miscdevice cuse_miscdev = { + .minor = CUSE_MINOR, + .name = "cuse", + .fops = &cuse_channel_fops, +}; + +MODULE_ALIAS_MISCDEV(CUSE_MINOR); +MODULE_ALIAS("devname:cuse"); + +static int __init cuse_init(void) +{ + int i, rc; + + /* init conntbl */ + for (i = 0; i < CUSE_CONNTBL_LEN; i++) + INIT_LIST_HEAD(&cuse_conntbl[i]); + + /* inherit and extend fuse_dev_operations */ + cuse_channel_fops = fuse_dev_operations; + cuse_channel_fops.owner = THIS_MODULE; + cuse_channel_fops.open = cuse_channel_open; + cuse_channel_fops.release = cuse_channel_release; + /* CUSE is not prepared for FUSE_DEV_IOC_CLONE */ + cuse_channel_fops.unlocked_ioctl = NULL; + + cuse_class = class_create("cuse"); + if (IS_ERR(cuse_class)) + return PTR_ERR(cuse_class); + + cuse_class->dev_groups = cuse_class_dev_groups; + + rc = misc_register(&cuse_miscdev); + if (rc) { + class_destroy(cuse_class); + return rc; + } + + return 0; +} + +static void __exit cuse_exit(void) +{ + misc_deregister(&cuse_miscdev); + class_destroy(cuse_class); +} + +module_init(cuse_init); +module_exit(cuse_exit); + +MODULE_AUTHOR("Tejun Heo "); +MODULE_DESCRIPTION("Character device in Userspace"); +MODULE_LICENSE("GPL"); diff --git a/fs/fuse/dax.c b/fs/fuse/dax.c new file mode 100644 index 000000000..85cdf0199 --- /dev/null +++ b/fs/fuse/dax.c @@ -0,0 +1,1356 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * dax: direct host memory access + * Copyright (C) 2020 Red Hat, Inc. + */ + +#include "fuse_i.h" + +#include +#include +#include +#include +#include +#include + +/* + * Default memory range size. A power of 2 so it agrees with common FUSE_INIT + * map_alignment values 4KB and 64KB. + */ +#define FUSE_DAX_SHIFT 21 +#define FUSE_DAX_SZ (1 << FUSE_DAX_SHIFT) +#define FUSE_DAX_PAGES (FUSE_DAX_SZ / PAGE_SIZE) + +/* Number of ranges reclaimer will try to free in one invocation */ +#define FUSE_DAX_RECLAIM_CHUNK (10) + +/* + * Dax memory reclaim threshold in percetage of total ranges. When free + * number of free ranges drops below this threshold, reclaim can trigger + * Default is 20% + */ +#define FUSE_DAX_RECLAIM_THRESHOLD (20) + +/** Translation information for file offsets to DAX window offsets */ +struct fuse_dax_mapping { + /* Pointer to inode where this memory range is mapped */ + struct inode *inode; + + /* Will connect in fcd->free_ranges to keep track of free memory */ + struct list_head list; + + /* For interval tree in file/inode */ + struct interval_tree_node itn; + + /* Will connect in fc->busy_ranges to keep track busy memory */ + struct list_head busy_list; + + /** Position in DAX window */ + u64 window_offset; + + /** Length of mapping, in bytes */ + loff_t length; + + /* Is this mapping read-only or read-write */ + bool writable; + + /* reference count when the mapping is used by dax iomap. */ + refcount_t refcnt; +}; + +/* Per-inode dax map */ +struct fuse_inode_dax { + /* Semaphore to protect modifications to the dmap tree */ + struct rw_semaphore sem; + + /* Sorted rb tree of struct fuse_dax_mapping elements */ + struct rb_root_cached tree; + unsigned long nr; +}; + +struct fuse_conn_dax { + /* DAX device */ + struct dax_device *dev; + + /* Lock protecting accessess to members of this structure */ + spinlock_t lock; + + /* List of memory ranges which are busy */ + unsigned long nr_busy_ranges; + struct list_head busy_ranges; + + /* Worker to free up memory ranges */ + struct delayed_work free_work; + + /* Wait queue for a dax range to become free */ + wait_queue_head_t range_waitq; + + /* DAX Window Free Ranges */ + long nr_free_ranges; + struct list_head free_ranges; + + unsigned long nr_ranges; +}; + +static inline struct fuse_dax_mapping * +node_to_dmap(struct interval_tree_node *node) +{ + if (!node) + return NULL; + + return container_of(node, struct fuse_dax_mapping, itn); +} + +static struct fuse_dax_mapping * +alloc_dax_mapping_reclaim(struct fuse_conn_dax *fcd, struct inode *inode); + +static void +__kick_dmap_free_worker(struct fuse_conn_dax *fcd, unsigned long delay_ms) +{ + unsigned long free_threshold; + + /* If number of free ranges are below threshold, start reclaim */ + free_threshold = max_t(unsigned long, fcd->nr_ranges * FUSE_DAX_RECLAIM_THRESHOLD / 100, + 1); + if (fcd->nr_free_ranges < free_threshold) + queue_delayed_work(system_dfl_long_wq, &fcd->free_work, + msecs_to_jiffies(delay_ms)); +} + +static void kick_dmap_free_worker(struct fuse_conn_dax *fcd, + unsigned long delay_ms) +{ + spin_lock(&fcd->lock); + __kick_dmap_free_worker(fcd, delay_ms); + spin_unlock(&fcd->lock); +} + +static struct fuse_dax_mapping *alloc_dax_mapping(struct fuse_conn_dax *fcd) +{ + struct fuse_dax_mapping *dmap; + + spin_lock(&fcd->lock); + dmap = list_first_entry_or_null(&fcd->free_ranges, + struct fuse_dax_mapping, list); + if (dmap) { + list_del_init(&dmap->list); + WARN_ON(fcd->nr_free_ranges <= 0); + fcd->nr_free_ranges--; + } + __kick_dmap_free_worker(fcd, 0); + spin_unlock(&fcd->lock); + + return dmap; +} + +/* This assumes fcd->lock is held */ +static void __dmap_remove_busy_list(struct fuse_conn_dax *fcd, + struct fuse_dax_mapping *dmap) +{ + list_del_init(&dmap->busy_list); + WARN_ON(fcd->nr_busy_ranges == 0); + fcd->nr_busy_ranges--; +} + +static void dmap_remove_busy_list(struct fuse_conn_dax *fcd, + struct fuse_dax_mapping *dmap) +{ + spin_lock(&fcd->lock); + __dmap_remove_busy_list(fcd, dmap); + spin_unlock(&fcd->lock); +} + +/* This assumes fcd->lock is held */ +static void __dmap_add_to_free_pool(struct fuse_conn_dax *fcd, + struct fuse_dax_mapping *dmap) +{ + list_add_tail(&dmap->list, &fcd->free_ranges); + fcd->nr_free_ranges++; + wake_up(&fcd->range_waitq); +} + +static void dmap_add_to_free_pool(struct fuse_conn_dax *fcd, + struct fuse_dax_mapping *dmap) +{ + /* Return fuse_dax_mapping to free list */ + spin_lock(&fcd->lock); + __dmap_add_to_free_pool(fcd, dmap); + spin_unlock(&fcd->lock); +} + +static int fuse_setup_one_mapping(struct inode *inode, unsigned long start_idx, + struct fuse_dax_mapping *dmap, bool writable, + bool upgrade) +{ + struct fuse_mount *fm = get_fuse_mount(inode); + struct fuse_conn_dax *fcd = fm->fc->dax; + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_setupmapping_in inarg; + loff_t offset = start_idx << FUSE_DAX_SHIFT; + FUSE_ARGS(args); + ssize_t err; + + WARN_ON(fcd->nr_free_ranges < 0); + + /* Ask fuse daemon to setup mapping */ + memset(&inarg, 0, sizeof(inarg)); + inarg.foffset = offset; + inarg.fh = -1; + inarg.moffset = dmap->window_offset; + inarg.len = FUSE_DAX_SZ; + inarg.flags |= FUSE_SETUPMAPPING_FLAG_READ; + if (writable) + inarg.flags |= FUSE_SETUPMAPPING_FLAG_WRITE; + args.opcode = FUSE_SETUPMAPPING; + args.nodeid = fi->nodeid; + args.in_numargs = 1; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + err = fuse_simple_request(fm, &args); + if (err < 0) + return err; + dmap->writable = writable; + if (!upgrade) { + /* + * We don't take a reference on inode. inode is valid right now + * and when inode is going away, cleanup logic should first + * cleanup dmap entries. + */ + dmap->inode = inode; + dmap->itn.start = dmap->itn.last = start_idx; + /* Protected by fi->dax->sem */ + interval_tree_insert(&dmap->itn, &fi->dax->tree); + fi->dax->nr++; + spin_lock(&fcd->lock); + list_add_tail(&dmap->busy_list, &fcd->busy_ranges); + fcd->nr_busy_ranges++; + spin_unlock(&fcd->lock); + } + return 0; +} + +static int fuse_send_removemapping(struct inode *inode, + struct fuse_removemapping_in *inargp, + struct fuse_removemapping_one *remove_one) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_mount *fm = get_fuse_mount(inode); + FUSE_ARGS(args); + + args.opcode = FUSE_REMOVEMAPPING; + args.nodeid = fi->nodeid; + args.in_numargs = 3; + fuse_set_zero_arg0(&args); + args.in_args[1].size = sizeof(*inargp); + args.in_args[1].value = inargp; + args.in_args[2].size = inargp->count * sizeof(*remove_one); + args.in_args[2].value = remove_one; + return fuse_simple_request(fm, &args); +} + +static int dmap_removemapping_list(struct inode *inode, unsigned int num, + struct list_head *to_remove) +{ + struct fuse_removemapping_one *remove_one, *ptr; + struct fuse_removemapping_in inarg; + struct fuse_dax_mapping *dmap; + int ret, i = 0, nr_alloc; + + nr_alloc = min_t(unsigned int, num, FUSE_REMOVEMAPPING_MAX_ENTRY); + remove_one = kmalloc_objs(*remove_one, nr_alloc, GFP_NOFS); + if (!remove_one) + return -ENOMEM; + + ptr = remove_one; + list_for_each_entry(dmap, to_remove, list) { + ptr->moffset = dmap->window_offset; + ptr->len = dmap->length; + ptr++; + i++; + num--; + if (i >= nr_alloc || num == 0) { + memset(&inarg, 0, sizeof(inarg)); + inarg.count = i; + ret = fuse_send_removemapping(inode, &inarg, + remove_one); + if (ret) + goto out; + ptr = remove_one; + i = 0; + } + } +out: + kfree(remove_one); + return ret; +} + +/* + * Cleanup dmap entry and add back to free list. This should be called with + * fcd->lock held. + */ +static void dmap_reinit_add_to_free_pool(struct fuse_conn_dax *fcd, + struct fuse_dax_mapping *dmap) +{ + pr_debug("fuse: freeing memory range start_idx=0x%lx end_idx=0x%lx window_offset=0x%llx length=0x%llx\n", + dmap->itn.start, dmap->itn.last, dmap->window_offset, + dmap->length); + __dmap_remove_busy_list(fcd, dmap); + dmap->inode = NULL; + dmap->itn.start = dmap->itn.last = 0; + __dmap_add_to_free_pool(fcd, dmap); +} + +/* + * Free inode dmap entries whose range falls inside [start, end]. + * Does not take any locks. At this point of time it should only be + * called from evict_inode() path where we know all dmap entries can be + * reclaimed. + */ +static void inode_reclaim_dmap_range(struct fuse_conn_dax *fcd, + struct inode *inode, + loff_t start, loff_t end) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_dax_mapping *dmap, *n; + int err, num = 0; + LIST_HEAD(to_remove); + unsigned long start_idx = start >> FUSE_DAX_SHIFT; + unsigned long end_idx = end >> FUSE_DAX_SHIFT; + struct interval_tree_node *node; + + while (1) { + node = interval_tree_iter_first(&fi->dax->tree, start_idx, + end_idx); + if (!node) + break; + dmap = node_to_dmap(node); + /* inode is going away. There should not be any users of dmap */ + WARN_ON(refcount_read(&dmap->refcnt) > 1); + interval_tree_remove(&dmap->itn, &fi->dax->tree); + num++; + list_add(&dmap->list, &to_remove); + } + + /* Nothing to remove */ + if (list_empty(&to_remove)) + return; + + WARN_ON(fi->dax->nr < num); + fi->dax->nr -= num; + err = dmap_removemapping_list(inode, num, &to_remove); + if (err && err != -ENOTCONN) { + pr_warn("Failed to removemappings. start=0x%llx end=0x%llx\n", + start, end); + } + spin_lock(&fcd->lock); + list_for_each_entry_safe(dmap, n, &to_remove, list) { + list_del_init(&dmap->list); + dmap_reinit_add_to_free_pool(fcd, dmap); + } + spin_unlock(&fcd->lock); +} + +static int dmap_removemapping_one(struct inode *inode, + struct fuse_dax_mapping *dmap) +{ + struct fuse_removemapping_one forget_one; + struct fuse_removemapping_in inarg; + + memset(&inarg, 0, sizeof(inarg)); + inarg.count = 1; + memset(&forget_one, 0, sizeof(forget_one)); + forget_one.moffset = dmap->window_offset; + forget_one.len = dmap->length; + + return fuse_send_removemapping(inode, &inarg, &forget_one); +} + +/* + * It is called from evict_inode() and by that time inode is going away. So + * this function does not take any locks like fi->dax->sem for traversing + * that fuse inode interval tree. If that lock is taken then lock validator + * complains of deadlock situation w.r.t fs_reclaim lock. + */ +void fuse_dax_inode_cleanup(struct inode *inode) +{ + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + + /* + * fuse_evict_inode() has already called truncate_inode_pages_final() + * before we arrive here. So we should not have to worry about any + * pages/exception entries still associated with inode. + */ + inode_reclaim_dmap_range(fc->dax, inode, 0, -1); + WARN_ON(fi->dax->nr); +} + +static void fuse_fill_iomap_hole(struct iomap *iomap, loff_t length) +{ + iomap->addr = IOMAP_NULL_ADDR; + iomap->length = length; + iomap->type = IOMAP_HOLE; +} + +static void fuse_fill_iomap(struct inode *inode, loff_t pos, loff_t length, + struct iomap *iomap, struct fuse_dax_mapping *dmap, + unsigned int flags) +{ + loff_t offset, len; + loff_t i_size = i_size_read(inode); + + offset = pos - (dmap->itn.start << FUSE_DAX_SHIFT); + len = min(length, dmap->length - offset); + + /* If length is beyond end of file, truncate further */ + if (pos + len > i_size) + len = i_size - pos; + + if (len > 0) { + iomap->addr = dmap->window_offset + offset; + iomap->length = len; + if (flags & IOMAP_FAULT) + iomap->length = ALIGN(len, PAGE_SIZE); + iomap->type = IOMAP_MAPPED; + /* + * increace refcnt so that reclaim code knows this dmap is in + * use. This assumes fi->dax->sem mutex is held either + * shared/exclusive. + */ + refcount_inc(&dmap->refcnt); + + /* iomap->private should be NULL */ + WARN_ON_ONCE(iomap->private); + iomap->private = dmap; + } else { + /* Mapping beyond end of file is hole */ + fuse_fill_iomap_hole(iomap, length); + } +} + +static int fuse_setup_new_dax_mapping(struct inode *inode, loff_t pos, + loff_t length, unsigned int flags, + struct iomap *iomap) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_conn_dax *fcd = fc->dax; + struct fuse_dax_mapping *dmap, *alloc_dmap = NULL; + int ret; + bool writable = flags & IOMAP_WRITE; + unsigned long start_idx = pos >> FUSE_DAX_SHIFT; + struct interval_tree_node *node; + + /* + * Can't do inline reclaim in fault path. We call + * dax_layout_busy_page() before we free a range. And + * fuse_wait_dax_page() drops mapping->invalidate_lock and requires it. + * In fault path we enter with mapping->invalidate_lock held and can't + * drop it. Also in fault path we hold mapping->invalidate_lock shared + * and not exclusive, so that creates further issues with + * fuse_wait_dax_page(). Hence return -EAGAIN and fuse_dax_fault() + * will wait for a memory range to become free and retry. + */ + if (flags & IOMAP_FAULT) { + alloc_dmap = alloc_dax_mapping(fcd); + if (!alloc_dmap) + return -EAGAIN; + } else { + alloc_dmap = alloc_dax_mapping_reclaim(fcd, inode); + if (IS_ERR(alloc_dmap)) + return PTR_ERR(alloc_dmap); + } + + /* If we are here, we should have memory allocated */ + if (WARN_ON(!alloc_dmap)) + return -EIO; + + /* + * Take write lock so that only one caller can try to setup mapping + * and other waits. + */ + down_write(&fi->dax->sem); + /* + * We dropped lock. Check again if somebody else setup + * mapping already. + */ + node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx); + if (node) { + dmap = node_to_dmap(node); + fuse_fill_iomap(inode, pos, length, iomap, dmap, flags); + dmap_add_to_free_pool(fcd, alloc_dmap); + up_write(&fi->dax->sem); + return 0; + } + + /* Setup one mapping */ + ret = fuse_setup_one_mapping(inode, pos >> FUSE_DAX_SHIFT, alloc_dmap, + writable, false); + if (ret < 0) { + dmap_add_to_free_pool(fcd, alloc_dmap); + up_write(&fi->dax->sem); + return ret; + } + fuse_fill_iomap(inode, pos, length, iomap, alloc_dmap, flags); + up_write(&fi->dax->sem); + return 0; +} + +static int fuse_upgrade_dax_mapping(struct inode *inode, loff_t pos, + loff_t length, unsigned int flags, + struct iomap *iomap) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_dax_mapping *dmap; + int ret; + unsigned long idx = pos >> FUSE_DAX_SHIFT; + struct interval_tree_node *node; + + /* + * Take exclusive lock so that only one caller can try to setup + * mapping and others wait. + */ + down_write(&fi->dax->sem); + node = interval_tree_iter_first(&fi->dax->tree, idx, idx); + + /* We are holding either inode lock or invalidate_lock, and that should + * ensure that dmap can't be truncated. We are holding a reference + * on dmap and that should make sure it can't be reclaimed. So dmap + * should still be there in tree despite the fact we dropped and + * re-acquired the fi->dax->sem lock. + */ + ret = -EIO; + if (WARN_ON(!node)) + goto out_err; + + dmap = node_to_dmap(node); + + /* We took an extra reference on dmap to make sure its not reclaimd. + * Now we hold fi->dax->sem lock and that reference is not needed + * anymore. Drop it. + */ + if (refcount_dec_and_test(&dmap->refcnt)) { + /* refcount should not hit 0. This object only goes + * away when fuse connection goes away + */ + WARN_ON_ONCE(1); + } + + /* Maybe another thread already upgraded mapping while we were not + * holding lock. + */ + if (dmap->writable) { + ret = 0; + goto out_fill_iomap; + } + + ret = fuse_setup_one_mapping(inode, pos >> FUSE_DAX_SHIFT, dmap, true, + true); + if (ret < 0) + goto out_err; +out_fill_iomap: + fuse_fill_iomap(inode, pos, length, iomap, dmap, flags); +out_err: + up_write(&fi->dax->sem); + return ret; +} + +/* This is just for DAX and the mapping is ephemeral, do not use it for other + * purposes since there is no block device with a permanent mapping. + */ +static int fuse_iomap_begin(struct inode *inode, loff_t pos, loff_t length, + unsigned int flags, struct iomap *iomap, + struct iomap *srcmap) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_dax_mapping *dmap; + bool writable = flags & IOMAP_WRITE; + unsigned long start_idx = pos >> FUSE_DAX_SHIFT; + struct interval_tree_node *node; + + /* We don't support FIEMAP */ + if (WARN_ON(flags & IOMAP_REPORT)) + return -EIO; + + iomap->offset = pos; + iomap->flags = 0; + iomap->bdev = NULL; + iomap->dax_dev = fc->dax->dev; + + /* + * Both read/write and mmap path can race here. So we need something + * to make sure if we are setting up mapping, then other path waits + * + * For now, use a semaphore for this. It probably needs to be + * optimized later. + */ + down_read(&fi->dax->sem); + node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx); + if (node) { + dmap = node_to_dmap(node); + if (writable && !dmap->writable) { + /* Upgrade read-only mapping to read-write. This will + * require exclusive fi->dax->sem lock as we don't want + * two threads to be trying to this simultaneously + * for same dmap. So drop shared lock and acquire + * exclusive lock. + * + * Before dropping fi->dax->sem lock, take reference + * on dmap so that its not freed by range reclaim. + */ + refcount_inc(&dmap->refcnt); + up_read(&fi->dax->sem); + pr_debug("%s: Upgrading mapping at offset 0x%llx length 0x%llx\n", + __func__, pos, length); + return fuse_upgrade_dax_mapping(inode, pos, length, + flags, iomap); + } else { + fuse_fill_iomap(inode, pos, length, iomap, dmap, flags); + up_read(&fi->dax->sem); + return 0; + } + } else { + up_read(&fi->dax->sem); + pr_debug("%s: no mapping at offset 0x%llx length 0x%llx\n", + __func__, pos, length); + if (pos >= i_size_read(inode)) + goto iomap_hole; + + return fuse_setup_new_dax_mapping(inode, pos, length, flags, + iomap); + } + + /* + * If read beyond end of file happens, fs code seems to return + * it as hole + */ +iomap_hole: + fuse_fill_iomap_hole(iomap, length); + pr_debug("%s returning hole mapping. pos=0x%llx length_asked=0x%llx length_returned=0x%llx\n", + __func__, pos, length, iomap->length); + return 0; +} + +static int fuse_iomap_end(struct inode *inode, loff_t pos, loff_t length, + ssize_t written, unsigned int flags, + struct iomap *iomap) +{ + struct fuse_dax_mapping *dmap = iomap->private; + + if (dmap) { + if (refcount_dec_and_test(&dmap->refcnt)) { + /* refcount should not hit 0. This object only goes + * away when fuse connection goes away + */ + WARN_ON_ONCE(1); + } + } + + /* DAX writes beyond end-of-file aren't handled using iomap, so the + * file size is unchanged and there is nothing to do here. + */ + return 0; +} + +static DEFINE_IOMAP_ITER_NEXT_END(fuse_iomap_next, fuse_iomap_begin, + fuse_iomap_end); + +static const struct iomap_ops fuse_iomap_ops = { + .iomap_next = fuse_iomap_next, +}; + +static void fuse_wait_dax_page(struct inode *inode) +{ + filemap_invalidate_unlock(inode->i_mapping); + schedule(); + filemap_invalidate_lock(inode->i_mapping); +} + +/* Should be called with mapping->invalidate_lock held exclusively. */ +int fuse_dax_break_layouts(struct inode *inode, u64 dmap_start, + u64 dmap_end) +{ + return dax_break_layout(inode, dmap_start, dmap_end, + fuse_wait_dax_page); +} + +ssize_t fuse_dax_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct inode *inode = file_inode(iocb->ki_filp); + ssize_t ret; + + if (iocb->ki_flags & IOCB_NOWAIT) { + if (!inode_trylock_shared(inode)) + return -EAGAIN; + } else { + inode_lock_shared(inode); + } + + ret = dax_iomap_rw(iocb, to, &fuse_iomap_ops); + inode_unlock_shared(inode); + + /* TODO file_accessed(iocb->f_filp) */ + return ret; +} + +static bool file_extending_write(struct kiocb *iocb, struct iov_iter *from) +{ + struct inode *inode = file_inode(iocb->ki_filp); + + return (iov_iter_rw(from) == WRITE && + ((iocb->ki_pos) >= i_size_read(inode) || + (iocb->ki_pos + iov_iter_count(from) > i_size_read(inode)))); +} + +static ssize_t fuse_dax_direct_write(struct kiocb *iocb, struct iov_iter *from) +{ + struct inode *inode = file_inode(iocb->ki_filp); + struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); + ssize_t ret; + + ret = fuse_direct_io(&io, from, &iocb->ki_pos, FUSE_DIO_WRITE); + + fuse_write_update_attr(inode, iocb->ki_pos, ret); + return ret; +} + +ssize_t fuse_dax_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + struct inode *inode = file_inode(iocb->ki_filp); + ssize_t ret; + + if (iocb->ki_flags & IOCB_NOWAIT) { + if (!inode_trylock(inode)) + return -EAGAIN; + } else { + inode_lock(inode); + } + + ret = generic_write_checks(iocb, from); + if (ret <= 0) + goto out; + + ret = file_remove_privs(iocb->ki_filp); + if (ret) + goto out; + /* TODO file_update_time() but we don't want metadata I/O */ + + /* Do not use dax for file extending writes as write and on + * disk i_size increase are not atomic otherwise. + */ + if (file_extending_write(iocb, from)) + ret = fuse_dax_direct_write(iocb, from); + else + ret = dax_iomap_rw(iocb, from, &fuse_iomap_ops); + +out: + inode_unlock(inode); + + if (ret > 0) + ret = generic_write_sync(iocb, ret); + return ret; +} + +static vm_fault_t __fuse_dax_fault(struct vm_fault *vmf, unsigned int order, + bool write) +{ + vm_fault_t ret; + struct inode *inode = file_inode(vmf->vma->vm_file); + struct super_block *sb = inode->i_sb; + unsigned long pfn; + int error = 0; + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_conn_dax *fcd = fc->dax; + bool retry = false; + + if (write) + sb_start_pagefault(sb); +retry: + if (retry && !(fcd->nr_free_ranges > 0)) + wait_event(fcd->range_waitq, (fcd->nr_free_ranges > 0)); + + /* + * We need to serialize against not only truncate but also against + * fuse dax memory range reclaim. While a range is being reclaimed, + * we do not want any read/write/mmap to make progress and try + * to populate page cache or access memory we are trying to free. + */ + filemap_invalidate_lock_shared(inode->i_mapping); + ret = dax_iomap_fault(vmf, order, &pfn, &error, &fuse_iomap_ops); + if ((ret & VM_FAULT_ERROR) && error == -EAGAIN) { + error = 0; + retry = true; + filemap_invalidate_unlock_shared(inode->i_mapping); + goto retry; + } + + if (ret & VM_FAULT_NEEDDSYNC) + ret = dax_finish_sync_fault(vmf, order, pfn); + filemap_invalidate_unlock_shared(inode->i_mapping); + + if (write) + sb_end_pagefault(sb); + + return ret; +} + +static vm_fault_t fuse_dax_fault(struct vm_fault *vmf) +{ + return __fuse_dax_fault(vmf, 0, vmf->flags & FAULT_FLAG_WRITE); +} + +static vm_fault_t fuse_dax_huge_fault(struct vm_fault *vmf, unsigned int order) +{ + return __fuse_dax_fault(vmf, order, vmf->flags & FAULT_FLAG_WRITE); +} + +static vm_fault_t fuse_dax_page_mkwrite(struct vm_fault *vmf) +{ + return __fuse_dax_fault(vmf, 0, true); +} + +static vm_fault_t fuse_dax_pfn_mkwrite(struct vm_fault *vmf) +{ + return __fuse_dax_fault(vmf, 0, true); +} + +static const struct vm_operations_struct fuse_dax_vm_ops = { + .fault = fuse_dax_fault, + .huge_fault = fuse_dax_huge_fault, + .page_mkwrite = fuse_dax_page_mkwrite, + .pfn_mkwrite = fuse_dax_pfn_mkwrite, +}; + +int fuse_dax_mmap(struct file *file, struct vm_area_struct *vma) +{ + file_accessed(file); + vma->vm_ops = &fuse_dax_vm_ops; + vm_flags_set(vma, VM_MIXEDMAP | VM_HUGEPAGE); + return 0; +} + +static int dmap_writeback_invalidate(struct inode *inode, + struct fuse_dax_mapping *dmap) +{ + int ret; + loff_t start_pos = dmap->itn.start << FUSE_DAX_SHIFT; + loff_t end_pos = (start_pos + FUSE_DAX_SZ - 1); + + ret = filemap_fdatawrite_range(inode->i_mapping, start_pos, end_pos); + if (ret) { + pr_debug("fuse: filemap_fdatawrite_range() failed. err=%d start_pos=0x%llx, end_pos=0x%llx\n", + ret, start_pos, end_pos); + return ret; + } + + ret = invalidate_inode_pages2_range(inode->i_mapping, + start_pos >> PAGE_SHIFT, + end_pos >> PAGE_SHIFT); + if (ret) + pr_debug("fuse: invalidate_inode_pages2_range() failed err=%d\n", + ret); + + return ret; +} + +static int reclaim_one_dmap_locked(struct inode *inode, + struct fuse_dax_mapping *dmap) +{ + int ret; + struct fuse_inode *fi = get_fuse_inode(inode); + + /* + * igrab() was done to make sure inode won't go under us, and this + * further avoids the race with evict(). + */ + ret = dmap_writeback_invalidate(inode, dmap); + if (ret) + return ret; + + /* Remove dax mapping from inode interval tree now */ + interval_tree_remove(&dmap->itn, &fi->dax->tree); + fi->dax->nr--; + + /* It is possible that umount/shutdown has killed the fuse connection + * and worker thread is trying to reclaim memory in parallel. Don't + * warn in that case. + */ + ret = dmap_removemapping_one(inode, dmap); + if (ret && ret != -ENOTCONN) { + pr_warn("Failed to remove mapping. offset=0x%llx len=0x%llx ret=%d\n", + dmap->window_offset, dmap->length, ret); + } + return 0; +} + +/* Find first mapped dmap for an inode and return file offset. Caller needs + * to hold fi->dax->sem lock either shared or exclusive. + */ +static struct fuse_dax_mapping *inode_lookup_first_dmap(struct inode *inode) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_dax_mapping *dmap; + struct interval_tree_node *node; + + for (node = interval_tree_iter_first(&fi->dax->tree, 0, -1); node; + node = interval_tree_iter_next(node, 0, -1)) { + dmap = node_to_dmap(node); + /* still in use. */ + if (refcount_read(&dmap->refcnt) > 1) + continue; + + return dmap; + } + + return NULL; +} + +/* + * Find first mapping in the tree and free it and return it. Do not add + * it back to free pool. + */ +static struct fuse_dax_mapping * +inode_inline_reclaim_one_dmap(struct fuse_conn_dax *fcd, struct inode *inode, + bool *retry) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_dax_mapping *dmap; + u64 dmap_start, dmap_end; + unsigned long start_idx; + int ret; + struct interval_tree_node *node; + + filemap_invalidate_lock(inode->i_mapping); + + /* Lookup a dmap and corresponding file offset to reclaim. */ + down_read(&fi->dax->sem); + dmap = inode_lookup_first_dmap(inode); + if (dmap) { + start_idx = dmap->itn.start; + dmap_start = start_idx << FUSE_DAX_SHIFT; + dmap_end = dmap_start + FUSE_DAX_SZ - 1; + } + up_read(&fi->dax->sem); + + if (!dmap) + goto out_mmap_sem; + /* + * Make sure there are no references to inode pages using + * get_user_pages() + */ + ret = fuse_dax_break_layouts(inode, dmap_start, dmap_end); + if (ret) { + pr_debug("fuse: fuse_dax_break_layouts() failed. err=%d\n", + ret); + dmap = ERR_PTR(ret); + goto out_mmap_sem; + } + + down_write(&fi->dax->sem); + node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx); + /* Range already got reclaimed by somebody else */ + if (!node) { + if (retry) + *retry = true; + goto out_write_dmap_sem; + } + + dmap = node_to_dmap(node); + /* still in use. */ + if (refcount_read(&dmap->refcnt) > 1) { + dmap = NULL; + if (retry) + *retry = true; + goto out_write_dmap_sem; + } + + ret = reclaim_one_dmap_locked(inode, dmap); + if (ret < 0) { + dmap = ERR_PTR(ret); + goto out_write_dmap_sem; + } + + /* Clean up dmap. Do not add back to free list */ + dmap_remove_busy_list(fcd, dmap); + dmap->inode = NULL; + dmap->itn.start = dmap->itn.last = 0; + + pr_debug("fuse: %s: inline reclaimed memory range. inode=%p, window_offset=0x%llx, length=0x%llx\n", + __func__, inode, dmap->window_offset, dmap->length); + +out_write_dmap_sem: + up_write(&fi->dax->sem); +out_mmap_sem: + filemap_invalidate_unlock(inode->i_mapping); + return dmap; +} + +static struct fuse_dax_mapping * +alloc_dax_mapping_reclaim(struct fuse_conn_dax *fcd, struct inode *inode) +{ + struct fuse_dax_mapping *dmap; + struct fuse_inode *fi = get_fuse_inode(inode); + + while (1) { + bool retry = false; + + dmap = alloc_dax_mapping(fcd); + if (dmap) + return dmap; + + dmap = inode_inline_reclaim_one_dmap(fcd, inode, &retry); + /* + * Either we got a mapping or it is an error, return in both + * the cases. + */ + if (dmap) + return dmap; + + /* If we could not reclaim a mapping because it + * had a reference or some other temporary failure, + * Try again. We want to give up inline reclaim only + * if there is no range assigned to this node. Otherwise + * if a deadlock is possible if we sleep with + * mapping->invalidate_lock held and worker to free memory + * can't make progress due to unavailability of + * mapping->invalidate_lock. So sleep only if fi->dax->nr=0 + */ + if (retry) + continue; + /* + * There are no mappings which can be reclaimed. Wait for one. + * We are not holding fi->dax->sem. So it is possible + * that range gets added now. But as we are not holding + * mapping->invalidate_lock, worker should still be able to + * free up a range and wake us up. + */ + if (!fi->dax->nr && !(fcd->nr_free_ranges > 0)) { + if (wait_event_killable_exclusive(fcd->range_waitq, + (fcd->nr_free_ranges > 0))) { + return ERR_PTR(-EINTR); + } + } + } +} + +static int lookup_and_reclaim_dmap_locked(struct fuse_conn_dax *fcd, + struct inode *inode, + unsigned long start_idx) +{ + int ret; + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_dax_mapping *dmap; + struct interval_tree_node *node; + + /* Find fuse dax mapping at file offset inode. */ + node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx); + + /* Range already got cleaned up by somebody else */ + if (!node) + return 0; + dmap = node_to_dmap(node); + + /* still in use. */ + if (refcount_read(&dmap->refcnt) > 1) + return 0; + + ret = reclaim_one_dmap_locked(inode, dmap); + if (ret < 0) + return ret; + + /* Cleanup dmap entry and add back to free list */ + spin_lock(&fcd->lock); + dmap_reinit_add_to_free_pool(fcd, dmap); + spin_unlock(&fcd->lock); + return ret; +} + +/* + * Free a range of memory. + * Locking: + * 1. Take mapping->invalidate_lock to block dax faults. + * 2. Take fi->dax->sem to protect interval tree and also to make sure + * read/write can not reuse a dmap which we might be freeing. + */ +static int lookup_and_reclaim_dmap(struct fuse_conn_dax *fcd, + struct inode *inode, + unsigned long start_idx, + unsigned long end_idx) +{ + int ret; + struct fuse_inode *fi = get_fuse_inode(inode); + loff_t dmap_start = start_idx << FUSE_DAX_SHIFT; + loff_t dmap_end = (dmap_start + FUSE_DAX_SZ) - 1; + + filemap_invalidate_lock(inode->i_mapping); + ret = fuse_dax_break_layouts(inode, dmap_start, dmap_end); + if (ret) { + pr_debug("virtio_fs: fuse_dax_break_layouts() failed. err=%d\n", + ret); + goto out_mmap_sem; + } + + down_write(&fi->dax->sem); + ret = lookup_and_reclaim_dmap_locked(fcd, inode, start_idx); + up_write(&fi->dax->sem); +out_mmap_sem: + filemap_invalidate_unlock(inode->i_mapping); + return ret; +} + +static int try_to_free_dmap_chunks(struct fuse_conn_dax *fcd, + unsigned long nr_to_free) +{ + struct fuse_dax_mapping *dmap, *pos, *temp; + int ret, nr_freed = 0; + unsigned long start_idx = 0, end_idx = 0; + struct inode *inode = NULL; + + /* Pick first busy range and free it for now*/ + while (1) { + if (nr_freed >= nr_to_free) + break; + + dmap = NULL; + spin_lock(&fcd->lock); + + if (!fcd->nr_busy_ranges) { + spin_unlock(&fcd->lock); + return 0; + } + + list_for_each_entry_safe(pos, temp, &fcd->busy_ranges, + busy_list) { + /* skip this range if it's in use. */ + if (refcount_read(&pos->refcnt) > 1) + continue; + + inode = igrab(pos->inode); + /* + * This inode is going away. That will free + * up all the ranges anyway, continue to + * next range. + */ + if (!inode) + continue; + /* + * Take this element off list and add it tail. If + * this element can't be freed, it will help with + * selecting new element in next iteration of loop. + */ + dmap = pos; + list_move_tail(&dmap->busy_list, &fcd->busy_ranges); + start_idx = end_idx = dmap->itn.start; + break; + } + spin_unlock(&fcd->lock); + if (!dmap) + return 0; + + ret = lookup_and_reclaim_dmap(fcd, inode, start_idx, end_idx); + iput(inode); + if (ret) + return ret; + nr_freed++; + } + return 0; +} + +static void fuse_dax_free_mem_worker(struct work_struct *work) +{ + int ret; + struct fuse_conn_dax *fcd = container_of(work, struct fuse_conn_dax, + free_work.work); + ret = try_to_free_dmap_chunks(fcd, FUSE_DAX_RECLAIM_CHUNK); + if (ret) { + pr_debug("fuse: try_to_free_dmap_chunks() failed with err=%d\n", + ret); + } + + /* If number of free ranges are still below threshold, requeue */ + kick_dmap_free_worker(fcd, 1); +} + +static void fuse_free_dax_mem_ranges(struct list_head *mem_list) +{ + struct fuse_dax_mapping *range, *temp; + + /* Free All allocated elements */ + list_for_each_entry_safe(range, temp, mem_list, list) { + list_del(&range->list); + if (!list_empty(&range->busy_list)) + list_del(&range->busy_list); + kfree(range); + } +} + +void fuse_dax_conn_free(struct fuse_conn *fc) +{ + if (fc->dax) { + fuse_free_dax_mem_ranges(&fc->dax->free_ranges); + kfree(fc->dax); + fc->dax = NULL; + } +} + +static int fuse_dax_mem_range_init(struct fuse_conn_dax *fcd) +{ + long nr_pages, nr_ranges; + struct fuse_dax_mapping *range; + int ret, id; + size_t dax_size = -1; + unsigned long i; + + init_waitqueue_head(&fcd->range_waitq); + INIT_LIST_HEAD(&fcd->free_ranges); + INIT_LIST_HEAD(&fcd->busy_ranges); + INIT_DELAYED_WORK(&fcd->free_work, fuse_dax_free_mem_worker); + + id = dax_read_lock(); + nr_pages = dax_direct_access(fcd->dev, 0, PHYS_PFN(dax_size), + DAX_ACCESS, NULL, NULL); + dax_read_unlock(id); + if (nr_pages < 0) { + pr_debug("dax_direct_access() returned %ld\n", nr_pages); + return nr_pages; + } + + nr_ranges = nr_pages/FUSE_DAX_PAGES; + pr_debug("%s: dax mapped %ld pages. nr_ranges=%ld\n", + __func__, nr_pages, nr_ranges); + + for (i = 0; i < nr_ranges; i++) { + range = kzalloc_obj(struct fuse_dax_mapping); + ret = -ENOMEM; + if (!range) + goto out_err; + + /* TODO: This offset only works if virtio-fs driver is not + * having some memory hidden at the beginning. This needs + * better handling + */ + range->window_offset = i * FUSE_DAX_SZ; + range->length = FUSE_DAX_SZ; + INIT_LIST_HEAD(&range->busy_list); + refcount_set(&range->refcnt, 1); + list_add_tail(&range->list, &fcd->free_ranges); + } + + fcd->nr_free_ranges = nr_ranges; + fcd->nr_ranges = nr_ranges; + return 0; +out_err: + /* Free All allocated elements */ + fuse_free_dax_mem_ranges(&fcd->free_ranges); + return ret; +} + +int fuse_dax_conn_alloc(struct fuse_conn *fc, enum fuse_dax_mode dax_mode, + struct dax_device *dax_dev) +{ + struct fuse_conn_dax *fcd; + int err; + + fc->dax_mode = dax_mode; + + if (!dax_dev) + return 0; + + fcd = kzalloc_obj(*fcd); + if (!fcd) + return -ENOMEM; + + spin_lock_init(&fcd->lock); + fcd->dev = dax_dev; + err = fuse_dax_mem_range_init(fcd); + if (err) { + kfree(fcd); + return err; + } + + fc->dax = fcd; + return 0; +} + +bool fuse_dax_inode_alloc(struct super_block *sb, struct fuse_inode *fi) +{ + struct fuse_conn *fc = get_fuse_conn_super(sb); + + fi->dax = NULL; + if (fc->dax) { + fi->dax = kzalloc_obj(*fi->dax, GFP_KERNEL_ACCOUNT); + if (!fi->dax) + return false; + + init_rwsem(&fi->dax->sem); + fi->dax->tree = RB_ROOT_CACHED; + } + + return true; +} + +static const struct address_space_operations fuse_dax_file_aops = { + .direct_IO = noop_direct_IO, + .dirty_folio = noop_dirty_folio, +}; + +static bool fuse_should_enable_dax(struct inode *inode, unsigned int flags) +{ + struct fuse_conn *fc = get_fuse_conn(inode); + enum fuse_dax_mode dax_mode = fc->dax_mode; + + if (dax_mode == FUSE_DAX_NEVER) + return false; + + /* + * fc->dax may be NULL in 'inode' mode when filesystem device doesn't + * support DAX, in which case it will silently fallback to 'never' mode. + */ + if (!fc->dax) + return false; + + if (dax_mode == FUSE_DAX_ALWAYS) + return true; + + /* dax_mode is FUSE_DAX_INODE* */ + return fc->inode_dax && (flags & FUSE_ATTR_DAX); +} + +void fuse_dax_inode_init(struct inode *inode, unsigned int flags) +{ + if (!fuse_should_enable_dax(inode, flags)) + return; + + inode->i_flags |= S_DAX; + inode->i_data.a_ops = &fuse_dax_file_aops; +} + +void fuse_dax_dontcache(struct inode *inode, unsigned int flags) +{ + struct fuse_conn *fc = get_fuse_conn(inode); + + if (fuse_is_inode_dax_mode(fc->dax_mode) && + ((bool) IS_DAX(inode) != (bool) (flags & FUSE_ATTR_DAX))) + d_mark_dontcache(inode); +} + +bool fuse_dax_check_alignment(struct fuse_conn *fc, unsigned int map_alignment) +{ + if (fc->dax && (map_alignment > FUSE_DAX_SHIFT)) { + pr_warn("FUSE: map_alignment %u incompatible with dax mem range size %u\n", + map_alignment, FUSE_DAX_SZ); + return false; + } + return true; +} + +void fuse_dax_cancel_work(struct fuse_conn *fc) +{ + struct fuse_conn_dax *fcd = fc->dax; + + if (fcd) + cancel_delayed_work_sync(&fcd->free_work); + +} +EXPORT_SYMBOL_GPL(fuse_dax_cancel_work); diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c new file mode 100644 index 000000000..5763a7cd3 --- /dev/null +++ b/fs/fuse/dev.c @@ -0,0 +1,2430 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + FUSE: Filesystem in Userspace + Copyright (C) 2001-2008 Miklos Szeredi +*/ + +#include "dev.h" +#include "args.h" +#include "dev_uring_i.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fuse_trace.h" + +MODULE_ALIAS_MISCDEV(FUSE_MINOR); +MODULE_ALIAS("devname:fuse"); + +static DECLARE_WAIT_QUEUE_HEAD(fuse_dev_waitq); + +static struct kmem_cache *fuse_req_cachep; + +static void fuse_request_init(struct fuse_chan *fch, struct fuse_req *req) +{ + INIT_LIST_HEAD(&req->list); + INIT_LIST_HEAD(&req->intr_entry); + init_waitqueue_head(&req->waitq); + refcount_set(&req->count, 1); + __set_bit(FR_PENDING, &req->flags); + req->chan = fch; + req->create_time = jiffies; +} + +static struct fuse_req *fuse_request_alloc(struct fuse_chan *fch, gfp_t flags) +{ + struct fuse_req *req = kmem_cache_zalloc(fuse_req_cachep, flags); + if (req) + fuse_request_init(fch, req); + + return req; +} + +static void fuse_request_free(struct fuse_req *req) +{ + WARN_ON(!list_empty(&req->intr_entry)); + kmem_cache_free(fuse_req_cachep, req); +} + +static void __fuse_get_request(struct fuse_req *req) +{ + refcount_inc(&req->count); +} + +/* Must be called with > 1 refcount */ +static void __fuse_put_request(struct fuse_req *req) +{ + refcount_dec(&req->count); +} + +void fuse_chan_set_initialized(struct fuse_chan *fch, struct fuse_chan_param *param) +{ + if (param) { + fch->minor = param->minor; + fch->max_write = param->max_write; + fch->max_pages = param->max_pages; + } + + /* Make sure stores before this are seen on another CPU */ + smp_wmb(); + fch->initialized = 1; + wake_up_all(&fch->blocked_waitq); +} + +static bool fuse_block_alloc(struct fuse_chan *fch, bool for_background) +{ + return !fch->initialized || (for_background && fch->blocked) || + (fch->io_uring && fch->connected && !fuse_uring_ready(fch)); +} + +static void fuse_drop_waiting(struct fuse_chan *fch) +{ + /* + * lockess check of fch->connected is okay, because atomic_dec_and_test() + * provides a memory barrier matched with the one in fuse_chan_wait_aborted() + * to ensure no wake-up is missed. + */ + if (atomic_dec_and_test(&fch->num_waiting) && + !READ_ONCE(fch->connected)) { + /* wake up aborters */ + wake_up_all(&fch->blocked_waitq); + } +} + +static void fuse_put_request(struct fuse_req *req); + +static struct fuse_req *fuse_get_req(struct fuse_chan *fch, bool for_background) +{ + struct fuse_req *req; + int err; + + atomic_inc(&fch->num_waiting); + + if (fuse_block_alloc(fch, for_background)) { + err = -EINTR; + if (wait_event_state_exclusive(fch->blocked_waitq, + !fuse_block_alloc(fch, for_background), + (TASK_KILLABLE | TASK_FREEZABLE))) + goto out; + } + + /* Matches smp_wmb() in fuse_chan_set_initialized() */ + smp_rmb(); + + err = -ENOTCONN; + if (!fch->connected) + goto out; + + req = fuse_request_alloc(fch, GFP_KERNEL); + err = -ENOMEM; + if (!req) { + if (for_background) + wake_up(&fch->blocked_waitq); + goto out; + } + + __set_bit(FR_WAITING, &req->flags); + if (for_background) + __set_bit(FR_BACKGROUND, &req->flags); + + return req; + + out: + fuse_drop_waiting(fch); + return ERR_PTR(err); +} + +static void fuse_put_request(struct fuse_req *req) +{ + struct fuse_chan *fch = req->chan; + + if (refcount_dec_and_test(&req->count)) { + if (test_bit(FR_BACKGROUND, &req->flags)) { + /* + * We get here in the unlikely case that a background + * request was allocated but not sent + */ + spin_lock(&fch->bg_lock); + if (!fch->blocked) + wake_up(&fch->blocked_waitq); + spin_unlock(&fch->bg_lock); + } + + if (test_bit(FR_WAITING, &req->flags)) { + __clear_bit(FR_WAITING, &req->flags); + fuse_drop_waiting(fch); + } + + fuse_request_free(req); + } +} + +unsigned int fuse_len_args(unsigned int numargs, struct fuse_arg *args) +{ + unsigned nbytes = 0; + unsigned i; + + for (i = 0; i < numargs; i++) + nbytes += args[i].size; + + return nbytes; +} +EXPORT_SYMBOL_GPL(fuse_len_args); + +static u64 fuse_get_unique_locked(struct fuse_iqueue *fiq) +{ + fiq->reqctr += FUSE_REQ_ID_STEP; + return fiq->reqctr; +} + +u64 fuse_get_unique(struct fuse_iqueue *fiq) +{ + u64 ret; + + spin_lock(&fiq->lock); + ret = fuse_get_unique_locked(fiq); + spin_unlock(&fiq->lock); + + return ret; +} +EXPORT_SYMBOL_GPL(fuse_get_unique); + +unsigned int fuse_req_hash(u64 unique) +{ + return hash_long(unique & ~FUSE_INT_REQ_BIT, FUSE_PQ_HASH_BITS); +} +EXPORT_SYMBOL_GPL(fuse_req_hash); + +/* + * A new request is available, wake fiq->waitq + */ +static void fuse_dev_wake_and_unlock(struct fuse_iqueue *fiq) +__releases(fiq->lock) +{ + wake_up(&fiq->waitq); + kill_fasync(&fiq->fasync, SIGIO, POLL_IN); + spin_unlock(&fiq->lock); +} + +struct fuse_forget_link *fuse_alloc_forget(void) +{ + return kzalloc_obj(struct fuse_forget_link, GFP_KERNEL_ACCOUNT); +} + +void fuse_dev_queue_forget(struct fuse_iqueue *fiq, + struct fuse_forget_link *forget) +{ + spin_lock(&fiq->lock); + if (fiq->connected) { + fiq->forget_list_tail->next = forget; + fiq->forget_list_tail = forget; + fuse_dev_wake_and_unlock(fiq); + } else { + kfree(forget); + spin_unlock(&fiq->lock); + } +} + +void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req) +{ + spin_lock(&fiq->lock); + if (list_empty(&req->intr_entry)) { + list_add_tail(&req->intr_entry, &fiq->interrupts); + /* + * Pairs with smp_mb() implied by test_and_set_bit() + * from fuse_request_end(). + */ + smp_mb(); + if (test_bit(FR_FINISHED, &req->flags)) { + list_del_init(&req->intr_entry); + spin_unlock(&fiq->lock); + } else { + fuse_dev_wake_and_unlock(fiq); + } + } else { + spin_unlock(&fiq->lock); + } +} + +static inline void fuse_request_assign_unique_locked(struct fuse_iqueue *fiq, + struct fuse_req *req) +{ + if (req->in.h.opcode != FUSE_NOTIFY_REPLY) + req->in.h.unique = fuse_get_unique_locked(fiq); + + /* tracepoint captures in.h.unique and in.h.len */ + trace_fuse_request_send(req); +} + +inline void fuse_request_assign_unique(struct fuse_iqueue *fiq, + struct fuse_req *req) +{ + if (req->in.h.opcode != FUSE_NOTIFY_REPLY) + req->in.h.unique = fuse_get_unique(fiq); + + /* tracepoint captures in.h.unique and in.h.len */ + trace_fuse_request_send(req); +} +EXPORT_SYMBOL_GPL(fuse_request_assign_unique); + +static void fuse_dev_queue_req(struct fuse_iqueue *fiq, struct fuse_req *req) +{ + spin_lock(&fiq->lock); + if (fiq->connected) { + fuse_request_assign_unique_locked(fiq, req); + list_add_tail(&req->list, &fiq->pending); + fuse_dev_wake_and_unlock(fiq); + } else { + spin_unlock(&fiq->lock); + req->out.h.error = -ENOTCONN; + clear_bit(FR_PENDING, &req->flags); + fuse_request_end(req); + } +} + +static const struct fuse_iqueue_ops fuse_dev_fiq_ops = { + .send_forget = fuse_dev_queue_forget, + .send_interrupt = fuse_dev_queue_interrupt, + .send_req = fuse_dev_queue_req, +}; + +void fuse_iqueue_init(struct fuse_iqueue *fiq, const struct fuse_iqueue_ops *ops, void *priv) +{ + spin_lock_init(&fiq->lock); + init_waitqueue_head(&fiq->waitq); + INIT_LIST_HEAD(&fiq->pending); + INIT_LIST_HEAD(&fiq->interrupts); + fiq->forget_list_tail = &fiq->forget_list_head; + fiq->connected = 1; + fiq->ops = ops; + fiq->priv = priv; +} +EXPORT_SYMBOL_GPL(fuse_iqueue_init); + +void fuse_chan_release(struct fuse_chan *fch) +{ + struct fuse_iqueue *fiq = &fch->iq; + + if (fiq->ops->release) + fiq->ops->release(fiq); + + if (fch->timeout.req_timeout) + cancel_delayed_work_sync(&fch->timeout.work); +} + +void fuse_chan_free(struct fuse_chan *fch) +{ + WARN_ON(!list_empty(&fch->devices)); + kfree(fch->pq_prealloc); + kfree(fch); +} +EXPORT_SYMBOL_GPL(fuse_chan_free); + +struct fuse_chan *fuse_chan_new(void) +{ + struct fuse_chan *fch = kzalloc_obj(struct fuse_chan); + if (!fch) + return NULL; + + spin_lock_init(&fch->lock); + INIT_LIST_HEAD(&fch->devices); + spin_lock_init(&fch->bg_lock); + INIT_LIST_HEAD(&fch->bg_queue); + init_waitqueue_head(&fch->blocked_waitq); + atomic_set(&fch->num_waiting, 0); + fch->max_background = FUSE_DEFAULT_MAX_BACKGROUND; + fch->initialized = 0; + fch->blocked = 0; + fch->connected = 1; + fch->timeout.req_timeout = 0; + + return fch; +} +EXPORT_SYMBOL_GPL(fuse_chan_new); + +struct list_head *fuse_pqueue_alloc(void) +{ + struct list_head *pq = kzalloc_objs(struct list_head, FUSE_PQ_HASH_SIZE); + + if (pq) { + for (int i = 0; i < FUSE_PQ_HASH_SIZE; i++) + INIT_LIST_HEAD(&pq[i]); + } + return pq; +} + +struct fuse_chan *fuse_dev_chan_new(void) +{ + struct fuse_chan *fch __free(kfree) = fuse_chan_new(); + if (!fch) + return NULL; + + fch->pq_prealloc = fuse_pqueue_alloc(); + if (!fch->pq_prealloc) + return NULL; + + fuse_iqueue_init(&fch->iq, &fuse_dev_fiq_ops, NULL); + + return no_free_ptr(fch); +} +EXPORT_SYMBOL_GPL(fuse_dev_chan_new); + +unsigned int fuse_chan_num_background(struct fuse_chan *fch) +{ + return READ_ONCE(fch->num_background); +} + +unsigned int fuse_chan_max_background(struct fuse_chan *fch) +{ + return READ_ONCE(fch->max_background); +} + +void fuse_chan_max_background_set(struct fuse_chan *fch, unsigned int val) +{ + spin_lock(&fch->bg_lock); + fch->max_background = val; + fch->blocked = fch->num_background >= fch->max_background; + if (!fch->blocked) + wake_up(&fch->blocked_waitq); + spin_unlock(&fch->bg_lock); +} + +unsigned int fuse_chan_num_waiting(struct fuse_chan *fch) +{ + return atomic_read(&fch->num_waiting); +} + +void fuse_chan_set_fc(struct fuse_chan *fch, struct fuse_conn *fc) +{ + fch->conn = fc; +} + +void fuse_chan_io_uring_enable(struct fuse_chan *fch) +{ + fch->io_uring = 1; +} + +void fuse_pqueue_init(struct fuse_pqueue *fpq) +{ + spin_lock_init(&fpq->lock); + INIT_LIST_HEAD(&fpq->io); + fpq->connected = 1; + fpq->processing = NULL; +} + +static struct fuse_dev *fuse_dev_alloc_no_pq(void) +{ + struct fuse_dev *fud; + + fud = kzalloc_obj(struct fuse_dev); + if (!fud) + return NULL; + + refcount_set(&fud->ref, 1); + fuse_pqueue_init(&fud->pq); + + return fud; +} + +struct fuse_dev *fuse_dev_alloc(void) +{ + struct fuse_dev *fud __free(kfree) = fuse_dev_alloc_no_pq(); + if (!fud) + return NULL; + + fud->pq.processing = fuse_pqueue_alloc(); + if (!fud->pq.processing) + return NULL; + + return no_free_ptr(fud); +} +EXPORT_SYMBOL_GPL(fuse_dev_alloc); + +/* + * Installs @fch into @fud, return true on success. "Consumes" @pq in either case. + */ +static bool fuse_dev_install_with_pq(struct fuse_dev *fud, struct fuse_chan *fch, + struct list_head *pq) +{ + struct fuse_chan *old_fch; + + guard(spinlock)(&fch->lock); + /* + * Pairs with: + * - xchg() in fuse_dev_release() + * - smp_load_acquire() in fuse_dev_fc_get() + */ + old_fch = cmpxchg(&fud->chan, NULL, fch); + if (old_fch) { + /* + * failed to set fud->chan because + * - it was already set to a different fc + * - it was set to disconneted + */ + kfree(pq); + return false; + } + if (pq) { + WARN_ON(fud->pq.processing); + fud->pq.processing = pq; + } + list_add_tail(&fud->entry, &fch->devices); + fuse_conn_get(fch->conn); + wake_up_all(&fuse_dev_waitq); + return true; +} + +void fuse_dev_install(struct fuse_dev *fud, struct fuse_chan *fch) +{ + struct list_head *pq = fch->pq_prealloc; + + fch->pq_prealloc = NULL; + if (!fuse_dev_install_with_pq(fud, fch, pq)) { + /* Channel is not usable without a dev */ + fuse_chan_abort(fch, false); + } +} +EXPORT_SYMBOL_GPL(fuse_dev_install); + +struct fuse_dev *fuse_dev_alloc_install(struct fuse_chan *fch) +{ + struct fuse_dev *fud; + + fud = fuse_dev_alloc_no_pq(); + if (!fud) + return NULL; + + fuse_dev_install(fud, fch); + return fud; +} +EXPORT_SYMBOL_GPL(fuse_dev_alloc_install); + +void fuse_dev_put(struct fuse_dev *fud) +{ + struct fuse_chan *fch; + + if (!refcount_dec_and_test(&fud->ref)) + return; + + fch = fuse_dev_chan_get(fud); + if (fch && fch != FUSE_DEV_CHAN_DISCONNECTED) { + /* This is the virtiofs case (fuse_dev_release() not called) */ + spin_lock(&fch->lock); + list_del(&fud->entry); + spin_unlock(&fch->lock); + + fuse_conn_put(fch->conn); + } + kfree(fud->pq.processing); + kfree(fud); +} +EXPORT_SYMBOL_GPL(fuse_dev_put); + +bool fuse_dev_is_installed(struct fuse_dev *fud) +{ + struct fuse_chan *fch = fuse_dev_chan_get(fud); + + return fch != NULL && fch != FUSE_DEV_CHAN_DISCONNECTED; +} + +/* + * Checks if @fc matches the one installed in @fud + */ +bool fuse_dev_verify(struct fuse_dev *fud, struct fuse_chan *fch) +{ + return fuse_dev_chan_get(fud) == fch; +} + +bool fuse_dev_is_sync_init(struct fuse_dev *fud) +{ + return fud->sync_init; +} + +struct fuse_dev *fuse_dev_grab(struct file *file) +{ + struct fuse_dev *fud = fuse_file_to_fud(file); + + refcount_inc(&fud->ref); + return fud; +} + +static void fuse_send_one(struct fuse_iqueue *fiq, struct fuse_req *req) +{ + req->in.h.len = sizeof(struct fuse_in_header) + + fuse_len_args(req->args->in_numargs, + (struct fuse_arg *) req->args->in_args); + fiq->ops->send_req(fiq, req); +} + +void fuse_chan_queue_forget(struct fuse_chan *fch, struct fuse_forget_link *forget, + u64 nodeid, u64 nlookup) +{ + struct fuse_iqueue *fiq = &fch->iq; + + forget->forget_one.nodeid = nodeid; + forget->forget_one.nlookup = nlookup; + + fiq->ops->send_forget(fiq, forget); +} + +static void flush_bg_queue(struct fuse_chan *fch) +{ + struct fuse_iqueue *fiq = &fch->iq; + + while (fch->active_background < fch->max_background && + !list_empty(&fch->bg_queue)) { + struct fuse_req *req; + + req = list_first_entry(&fch->bg_queue, struct fuse_req, list); + list_del(&req->list); + fch->active_background++; + fuse_send_one(fiq, req); + } +} + +void fuse_request_bg_finish(struct fuse_chan *fch, struct fuse_req *req) +{ + lockdep_assert_held(&fch->bg_lock); + + clear_bit(FR_BACKGROUND, &req->flags); + if (fch->num_background == fch->max_background) { + fch->blocked = 0; + wake_up(&fch->blocked_waitq); + } else if (!fch->blocked) { + /* + * Wake up next waiter, if any. It's okay to use + * waitqueue_active(), as we've already synced up + * fch->blocked with waiters with the wake_up() call + * above. + */ + if (waitqueue_active(&fch->blocked_waitq)) + wake_up(&fch->blocked_waitq); + } + + fch->num_background--; + fch->active_background--; +} + +/* + * This function is called when a request is finished. Either a reply + * has arrived or it was aborted (and not yet sent) or some error + * occurred during communication with userspace, or the device file + * was closed. The requester thread is woken up (if still waiting), + * the 'end' callback is called if given, else the reference to the + * request is released + */ +void fuse_request_end(struct fuse_req *req) +{ + struct fuse_chan *fch = req->chan; + struct fuse_iqueue *fiq = &fch->iq; + + if (test_and_set_bit(FR_FINISHED, &req->flags)) + goto put_request; + + trace_fuse_request_end(req); + /* + * test_and_set_bit() implies smp_mb() between bit + * changing and below FR_INTERRUPTED check. Pairs with + * smp_mb() from queue_interrupt(). + */ + if (test_bit(FR_INTERRUPTED, &req->flags)) { + spin_lock(&fiq->lock); + list_del_init(&req->intr_entry); + spin_unlock(&fiq->lock); + } + WARN_ON(test_bit(FR_PENDING, &req->flags)); + WARN_ON(test_bit(FR_SENT, &req->flags)); + if (test_bit(FR_BACKGROUND, &req->flags)) { + spin_lock(&fch->bg_lock); + fuse_request_bg_finish(fch, req); + flush_bg_queue(fch); + spin_unlock(&fch->bg_lock); + } else { + /* Wake up waiter sleeping in request_wait_answer() */ + wake_up(&req->waitq); + } + + if (test_bit(FR_ASYNC, &req->flags)) + req->args->end(req->args, req->out.h.error); +put_request: + fuse_put_request(req); +} +EXPORT_SYMBOL_GPL(fuse_request_end); + +static int queue_interrupt(struct fuse_req *req) +{ + struct fuse_iqueue *fiq = &req->chan->iq; + + /* Check for we've sent request to interrupt this req */ + if (unlikely(!test_bit(FR_INTERRUPTED, &req->flags))) + return -EINVAL; + + fiq->ops->send_interrupt(fiq, req); + + return 0; +} + +bool fuse_remove_pending_req(struct fuse_req *req, spinlock_t *lock) +{ + spin_lock(lock); + if (test_bit(FR_PENDING, &req->flags)) { + /* + * FR_PENDING does not get cleared as the request will end + * up in destruction anyway. + */ + list_del(&req->list); + spin_unlock(lock); + __fuse_put_request(req); + req->out.h.error = -EINTR; + return true; + } + spin_unlock(lock); + return false; +} + +static void request_wait_answer(struct fuse_req *req) +{ + struct fuse_chan *fch = req->chan; + struct fuse_iqueue *fiq = &fch->iq; + int err; + + if (!fch->no_interrupt) { + /* Any signal may interrupt this */ + err = wait_event_interruptible(req->waitq, + test_bit(FR_FINISHED, &req->flags)); + if (!err) + return; + + set_bit(FR_INTERRUPTED, &req->flags); + /* matches barrier in fuse_dev_do_read() */ + smp_mb__after_atomic(); + if (test_bit(FR_SENT, &req->flags)) + queue_interrupt(req); + } + + if (!test_bit(FR_FORCE, &req->flags)) { + bool removed; + + /* Only fatal signals may interrupt this */ + err = wait_event_killable(req->waitq, + test_bit(FR_FINISHED, &req->flags)); + if (!err) + return; + + if (req->args->abort_on_kill) { + fuse_chan_abort(fch, false); + return; + } + + if (test_bit(FR_URING, &req->flags)) + removed = fuse_uring_remove_pending_req(req); + else + removed = fuse_remove_pending_req(req, &fiq->lock); + if (removed) + return; + } + + /* + * Either request is already in userspace, or it was forced. + * Wait it out. + */ + wait_event(req->waitq, test_bit(FR_FINISHED, &req->flags)); +} + +static void __fuse_request_send(struct fuse_req *req) +{ + struct fuse_iqueue *fiq = &req->chan->iq; + + BUG_ON(test_bit(FR_BACKGROUND, &req->flags)); + + /* acquire extra reference, since request is still needed after + fuse_request_end() */ + __fuse_get_request(req); + fuse_send_one(fiq, req); + + request_wait_answer(req); + /* Pairs with smp_wmb() in fuse_request_end() */ + smp_rmb(); +} + +static void fuse_adjust_compat(struct fuse_chan *fch, struct fuse_args *args) +{ + if (fch->minor < 4 && args->opcode == FUSE_STATFS) + args->out_args[0].size = FUSE_COMPAT_STATFS_SIZE; + + if (fch->minor < 9) { + switch (args->opcode) { + case FUSE_LOOKUP: + case FUSE_CREATE: + case FUSE_MKNOD: + case FUSE_MKDIR: + case FUSE_SYMLINK: + case FUSE_LINK: + args->out_args[0].size = FUSE_COMPAT_ENTRY_OUT_SIZE; + break; + case FUSE_GETATTR: + case FUSE_SETATTR: + args->out_args[0].size = FUSE_COMPAT_ATTR_OUT_SIZE; + break; + } + } + if (fch->minor < 12) { + switch (args->opcode) { + case FUSE_CREATE: + args->in_args[0].size = sizeof(struct fuse_open_in); + break; + case FUSE_MKNOD: + args->in_args[0].size = FUSE_COMPAT_MKNOD_IN_SIZE; + break; + } + } +} + +static void fuse_args_to_req(struct fuse_req *req, struct fuse_args *args) +{ + req->in.h.opcode = args->opcode; + req->in.h.nodeid = args->nodeid; + req->in.h.uid = args->uid; + req->in.h.gid = args->gid; + req->in.h.pid = args->pid; + req->args = args; + if (args->is_ext) + req->in.h.total_extlen = args->in_args[args->ext_idx].size / 8; + if (args->end) + __set_bit(FR_ASYNC, &req->flags); +} + +ssize_t fuse_chan_send(struct fuse_chan *fch, struct fuse_args *args) +{ + struct fuse_req *req; + ssize_t ret; + + if (args->force) { + atomic_inc(&fch->num_waiting); + req = fuse_request_alloc(fch, GFP_KERNEL | __GFP_NOFAIL); + + __set_bit(FR_WAITING, &req->flags); + if (!args->abort_on_kill) + __set_bit(FR_FORCE, &req->flags); + } else { + req = fuse_get_req(fch, false); + if (IS_ERR(req)) + return PTR_ERR(req); + } + + /* Needs to be done after fuse_get_req() so that fch->minor is valid */ + fuse_adjust_compat(fch, args); + fuse_args_to_req(req, args); + + if (!args->noreply) + __set_bit(FR_ISREPLY, &req->flags); + __fuse_request_send(req); + ret = req->out.h.error; + if (!ret && args->out_argvar) { + BUG_ON(args->out_numargs == 0); + ret = args->out_args[args->out_numargs - 1].size; + } + fuse_put_request(req); + + return ret; +} + +#ifdef CONFIG_FUSE_IO_URING +static bool fuse_request_queue_background_uring(struct fuse_req *req) +{ + struct fuse_iqueue *fiq = &req->chan->iq; + + req->in.h.len = sizeof(struct fuse_in_header) + + fuse_len_args(req->args->in_numargs, + (struct fuse_arg *) req->args->in_args); + fuse_request_assign_unique(fiq, req); + + return fuse_uring_queue_bq_req(req); +} +#endif + +/* + * @return true if queued + */ +static int fuse_request_queue_background(struct fuse_req *req) +{ + struct fuse_chan *fch = req->chan; + bool queued = false; + + WARN_ON(!test_bit(FR_BACKGROUND, &req->flags)); + if (!test_bit(FR_WAITING, &req->flags)) { + __set_bit(FR_WAITING, &req->flags); + atomic_inc(&fch->num_waiting); + } + __set_bit(FR_ISREPLY, &req->flags); + +#ifdef CONFIG_FUSE_IO_URING + if (fuse_uring_ready(fch)) + return fuse_request_queue_background_uring(req); +#endif + + spin_lock(&fch->bg_lock); + if (likely(fch->connected)) { + fch->num_background++; + if (fch->num_background == fch->max_background) + fch->blocked = 1; + list_add_tail(&req->list, &fch->bg_queue); + flush_bg_queue(fch); + queued = true; + } + spin_unlock(&fch->bg_lock); + + return queued; +} + +int fuse_chan_send_bg(struct fuse_chan *fch, struct fuse_args *args, gfp_t gfp_flags) +{ + struct fuse_req *req; + + if (args->force) { + req = fuse_request_alloc(fch, gfp_flags); + if (!req) + return -ENOMEM; + __set_bit(FR_BACKGROUND, &req->flags); + } else { + req = fuse_get_req(fch, true); + if (IS_ERR(req)) + return PTR_ERR(req); + } + + fuse_args_to_req(req, args); + + if (!fuse_request_queue_background(req)) { + fuse_put_request(req); + return -ENOTCONN; + } + + return 0; +} + +int fuse_chan_send_notify_reply(struct fuse_chan *fch, struct fuse_args *args, u64 unique) +{ + struct fuse_req *req; + struct fuse_iqueue *fiq = &fch->iq; + + req = fuse_get_req(fch, false); + if (IS_ERR(req)) + return PTR_ERR(req); + + __clear_bit(FR_ISREPLY, &req->flags); + req->in.h.unique = unique; + + fuse_args_to_req(req, args); + + fuse_send_one(fiq, req); + + return 0; +} + +/* + * Lock the request. Up to the next unlock_request() there mustn't be + * anything that could cause a page-fault. If the request was already + * aborted bail out. + */ +static int lock_request(struct fuse_req *req) +{ + int err = 0; + if (req) { + spin_lock(&req->waitq.lock); + if (test_bit(FR_ABORTED, &req->flags)) + err = -ENOENT; + else + set_bit(FR_LOCKED, &req->flags); + spin_unlock(&req->waitq.lock); + } + return err; +} + +/* + * Unlock request. If it was aborted while locked, caller is responsible + * for unlocking and ending the request. + */ +static int unlock_request(struct fuse_req *req) +{ + int err = 0; + if (req) { + spin_lock(&req->waitq.lock); + if (test_bit(FR_ABORTED, &req->flags)) + err = -ENOENT; + else + clear_bit(FR_LOCKED, &req->flags); + spin_unlock(&req->waitq.lock); + } + return err; +} + +void fuse_copy_init(struct fuse_copy_state *cs, bool write, + struct iov_iter *iter) +{ + memset(cs, 0, sizeof(*cs)); + cs->write = write; + cs->iter = iter; +} + +/* Unmap and put previous page of userspace buffer */ +void fuse_copy_finish(struct fuse_copy_state *cs) +{ + if (cs->currbuf) { + struct pipe_buffer *buf = cs->currbuf; + + if (cs->write) + buf->len = PAGE_SIZE - cs->len; + cs->currbuf = NULL; + } else if (cs->pg) { + if (cs->write) { + flush_dcache_page(cs->pg); + set_page_dirty_lock(cs->pg); + } + put_page(cs->pg); + } + cs->pg = NULL; +} + +/* + * Get another pagefull of userspace buffer, and map it to kernel + * address space, and lock request + */ +static int fuse_copy_fill(struct fuse_copy_state *cs) +{ + struct page *page; + int err; + + err = unlock_request(cs->req); + if (err) + return err; + + fuse_copy_finish(cs); + if (cs->pipebufs) { + struct pipe_buffer *buf = cs->pipebufs; + + if (!cs->write) { + err = pipe_buf_confirm(cs->pipe, buf); + if (err) + return err; + + BUG_ON(!cs->nr_segs); + cs->currbuf = buf; + cs->pg = buf->page; + cs->offset = buf->offset; + cs->len = buf->len; + cs->pipebufs++; + cs->nr_segs--; + } else { + if (cs->nr_segs >= cs->pipe->max_usage) + return -EIO; + + page = alloc_page(GFP_HIGHUSER); + if (!page) + return -ENOMEM; + + buf->page = page; + buf->offset = 0; + buf->len = 0; + + cs->currbuf = buf; + cs->pg = page; + cs->offset = 0; + cs->len = PAGE_SIZE; + cs->pipebufs++; + cs->nr_segs++; + } + } else { + size_t off; + err = iov_iter_get_pages2(cs->iter, &page, PAGE_SIZE, 1, &off); + if (err < 0) + return err; + BUG_ON(!err); + cs->len = err; + cs->offset = off; + cs->pg = page; + } + + return lock_request(cs->req); +} + +/* Do as much copy to/from userspace buffer as we can */ +static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size) +{ + unsigned ncpy = min(*size, cs->len); + if (val) { + void *pgaddr = kmap_local_page(cs->pg); + void *buf = pgaddr + cs->offset; + + if (cs->write) + memcpy(buf, *val, ncpy); + else + memcpy(*val, buf, ncpy); + + kunmap_local(pgaddr); + *val += ncpy; + } + *size -= ncpy; + cs->len -= ncpy; + cs->offset += ncpy; + if (cs->is_uring) + cs->ring.copied_sz += ncpy; + + return ncpy; +} + +static int fuse_check_folio(struct folio *folio) +{ + if (folio_mapped(folio) || + folio->mapping != NULL || + (folio->flags.f & PAGE_FLAGS_CHECK_AT_PREP & + ~(1 << PG_locked | + 1 << PG_referenced | + 1 << PG_lru | + 1 << PG_active | + 1 << PG_workingset | + 1 << PG_reclaim | + 1 << PG_waiters | + LRU_GEN_MASK | LRU_REFS_MASK))) { + dump_page(&folio->page, "fuse: trying to steal weird page"); + return 1; + } + return 0; +} + +/* + * Attempt to steal a page from the splice() pipe and move it into the + * pagecache. If successful, the pointer in @pagep will be updated. The + * folio that was originally in @pagep will lose a reference and the new + * folio returned in @pagep will carry a reference. + */ +static int fuse_try_move_folio(struct fuse_copy_state *cs, struct folio **foliop) +{ + int err; + struct folio *oldfolio = *foliop; + struct folio *newfolio; + struct pipe_buffer *buf = cs->pipebufs; + + folio_get(oldfolio); + err = unlock_request(cs->req); + if (err) + goto out_put_old; + + fuse_copy_finish(cs); + + err = pipe_buf_confirm(cs->pipe, buf); + if (err) + goto out_put_old; + + BUG_ON(!cs->nr_segs); + cs->currbuf = buf; + cs->len = buf->len; + cs->pipebufs++; + cs->nr_segs--; + + if (cs->len != folio_size(oldfolio)) + goto out_fallback; + + if (!pipe_buf_try_steal(cs->pipe, buf)) + goto out_fallback; + + newfolio = page_folio(buf->page); + + folio_clear_uptodate(newfolio); + folio_clear_mappedtodisk(newfolio); + + if (folio_test_large(newfolio)) + goto out_fallback_unlock; + + if (fuse_check_folio(newfolio) != 0) + goto out_fallback_unlock; + + /* + * This is a new and locked page, it shouldn't be mapped or + * have any special flags on it + */ + if (WARN_ON(folio_mapped(oldfolio))) + goto out_fallback_unlock; + if (WARN_ON(folio_has_private(oldfolio))) + goto out_fallback_unlock; + if (WARN_ON(folio_test_dirty(oldfolio) || + folio_test_writeback(oldfolio))) + goto out_fallback_unlock; + if (WARN_ON(folio_test_mlocked(oldfolio))) + goto out_fallback_unlock; + + err = lock_request(cs->req); + if (err) + goto out_fallback_unlock; + + replace_page_cache_folio(oldfolio, newfolio); + + folio_get(newfolio); + + if (!(buf->flags & PIPE_BUF_FLAG_LRU)) + folio_add_lru(newfolio); + + /* + * Release while we have extra ref on stolen page. Otherwise + * anon_pipe_buf_release() might think the page can be reused. + */ + pipe_buf_release(cs->pipe, buf); + + *foliop = newfolio; + folio_unlock(oldfolio); + /* Drop ref for ap->pages[] array */ + folio_put(oldfolio); + cs->len = 0; + + err = 0; +out_put_old: + /* Drop ref obtained in this function */ + folio_put(oldfolio); + return err; + +out_fallback_unlock: + folio_unlock(newfolio); +out_fallback: + cs->pg = buf->page; + cs->offset = buf->offset; + + err = lock_request(cs->req); + if (!err) + err = 1; + + goto out_put_old; +} + +static int fuse_ref_folio(struct fuse_copy_state *cs, struct folio *folio, + unsigned offset, unsigned count) +{ + struct pipe_buffer *buf; + int err; + + if (cs->nr_segs >= cs->pipe->max_usage) + return -EIO; + + folio_get(folio); + err = unlock_request(cs->req); + if (err) { + folio_put(folio); + return err; + } + + fuse_copy_finish(cs); + + buf = cs->pipebufs; + buf->page = &folio->page; + buf->offset = offset; + buf->len = count; + + cs->pipebufs++; + cs->nr_segs++; + cs->len = 0; + + return lock_request(cs->req); +} + +/* + * Copy a folio in the request to/from the userspace buffer. Must be + * done atomically + */ +int fuse_copy_folio(struct fuse_copy_state *cs, struct folio **foliop, + unsigned offset, unsigned count, int zeroing) +{ + int err; + struct folio *folio = *foliop; + size_t size; + + if (folio) { + size = folio_size(folio); + if (zeroing && count < size) + folio_zero_range(folio, 0, size); + } + + while (count) { + if (cs->write && cs->pipebufs && folio) { + /* + * Can't control lifetime of pipe buffers, so always + * copy user pages. + */ + if (cs->req->args->user_pages) { + err = fuse_copy_fill(cs); + if (err) + return err; + } else { + return fuse_ref_folio(cs, folio, offset, count); + } + } else if (!cs->len) { + if (cs->move_folios && folio && + offset == 0 && count == size) { + err = fuse_try_move_folio(cs, foliop); + if (err <= 0) + return err; + } else { + err = fuse_copy_fill(cs); + if (err) + return err; + } + } + if (folio) { + void *mapaddr = kmap_local_folio(folio, offset); + void *buf = mapaddr; + unsigned int copy = count; + unsigned int bytes_copied; + + if (folio_test_highmem(folio) && count > PAGE_SIZE - offset_in_page(offset)) + copy = PAGE_SIZE - offset_in_page(offset); + + bytes_copied = fuse_copy_do(cs, &buf, ©); + kunmap_local(mapaddr); + offset += bytes_copied; + count -= bytes_copied; + } else + offset += fuse_copy_do(cs, NULL, &count); + } + if (folio && !cs->write) + flush_dcache_folio(folio); + return 0; +} + +/* Copy folios in the request to/from userspace buffer */ +static int fuse_copy_folios(struct fuse_copy_state *cs, unsigned nbytes, + int zeroing) +{ + unsigned i; + struct fuse_req *req = cs->req; + struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args); + + for (i = 0; i < ap->num_folios && (nbytes || zeroing); i++) { + int err; + unsigned int offset = ap->descs[i].offset; + unsigned int count = min(nbytes, ap->descs[i].length); + + err = fuse_copy_folio(cs, &ap->folios[i], offset, count, zeroing); + if (err) + return err; + + nbytes -= count; + } + return 0; +} + +/* Copy a single argument in the request to/from userspace buffer */ +int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size) +{ + while (size) { + if (!cs->len) { + int err = fuse_copy_fill(cs); + if (err) + return err; + } + fuse_copy_do(cs, &val, &size); + } + return 0; +} + +/* Copy request arguments to/from userspace buffer */ +int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs, + unsigned argpages, struct fuse_arg *args, + int zeroing) +{ + int err = 0; + unsigned i; + + for (i = 0; !err && i < numargs; i++) { + struct fuse_arg *arg = &args[i]; + if (i == numargs - 1 && argpages) + err = fuse_copy_folios(cs, arg->size, zeroing); + else + err = fuse_copy_one(cs, arg->value, arg->size); + } + return err; +} + +static int forget_pending(struct fuse_iqueue *fiq) +{ + return fiq->forget_list_head.next != NULL; +} + +static int request_pending(struct fuse_iqueue *fiq) +{ + return !list_empty(&fiq->pending) || !list_empty(&fiq->interrupts) || + forget_pending(fiq); +} + +/* + * Transfer an interrupt request to userspace + * + * Unlike other requests this is assembled on demand, without a need + * to allocate a separate fuse_req structure. + * + * Called with fiq->lock held, releases it + */ +static int fuse_read_interrupt(struct fuse_iqueue *fiq, struct fuse_copy_state *cs) +__releases(fiq->lock) +{ + struct fuse_req *req = list_first_entry(&fiq->interrupts, struct fuse_req, intr_entry); + struct fuse_interrupt_in arg = { + .unique = req->in.h.unique, + }; + struct fuse_in_header ih = { + .opcode = FUSE_INTERRUPT, + .unique = (req->in.h.unique | FUSE_INT_REQ_BIT), + .len = sizeof(ih) + sizeof(arg), + }; + int err; + + list_del_init(&req->intr_entry); + spin_unlock(&fiq->lock); + + err = fuse_copy_one(cs, &ih, sizeof(ih)); + if (!err) + err = fuse_copy_one(cs, &arg, sizeof(arg)); + fuse_copy_finish(cs); + + return err ? err : ih.len; +} + +static struct fuse_forget_link *fuse_dequeue_forget(struct fuse_iqueue *fiq, + unsigned int max, + unsigned int *countp) +{ + struct fuse_forget_link *head = fiq->forget_list_head.next; + struct fuse_forget_link **newhead = &head; + unsigned count; + + for (count = 0; *newhead != NULL && count < max; count++) + newhead = &(*newhead)->next; + + fiq->forget_list_head.next = *newhead; + *newhead = NULL; + if (fiq->forget_list_head.next == NULL) + fiq->forget_list_tail = &fiq->forget_list_head; + + if (countp != NULL) + *countp = count; + + return head; +} + +static int fuse_read_single_forget(struct fuse_iqueue *fiq, + struct fuse_copy_state *cs) +__releases(fiq->lock) +{ + int err; + struct fuse_forget_link *forget = fuse_dequeue_forget(fiq, 1, NULL); + struct fuse_forget_in arg = { + .nlookup = forget->forget_one.nlookup, + }; + struct fuse_in_header ih = { + .opcode = FUSE_FORGET, + .nodeid = forget->forget_one.nodeid, + .unique = fuse_get_unique_locked(fiq), + .len = sizeof(ih) + sizeof(arg), + }; + + spin_unlock(&fiq->lock); + kfree(forget); + + err = fuse_copy_one(cs, &ih, sizeof(ih)); + if (!err) + err = fuse_copy_one(cs, &arg, sizeof(arg)); + fuse_copy_finish(cs); + + if (err) + return err; + + return ih.len; +} + +static int fuse_read_batch_forget(struct fuse_iqueue *fiq, + struct fuse_copy_state *cs, size_t nbytes) +__releases(fiq->lock) +{ + int err; + unsigned max_forgets; + unsigned count; + struct fuse_forget_link *head; + struct fuse_batch_forget_in arg = { .count = 0 }; + struct fuse_in_header ih = { + .opcode = FUSE_BATCH_FORGET, + .unique = fuse_get_unique_locked(fiq), + .len = sizeof(ih) + sizeof(arg), + }; + + max_forgets = (nbytes - ih.len) / sizeof(struct fuse_forget_one); + head = fuse_dequeue_forget(fiq, max_forgets, &count); + spin_unlock(&fiq->lock); + + arg.count = count; + ih.len += count * sizeof(struct fuse_forget_one); + err = fuse_copy_one(cs, &ih, sizeof(ih)); + if (!err) + err = fuse_copy_one(cs, &arg, sizeof(arg)); + + while (head) { + struct fuse_forget_link *forget = head; + + if (!err) { + err = fuse_copy_one(cs, &forget->forget_one, + sizeof(forget->forget_one)); + } + head = forget->next; + kfree(forget); + } + + fuse_copy_finish(cs); + + if (err) + return err; + + return ih.len; +} + +static int fuse_read_forget(struct fuse_chan *fch, struct fuse_iqueue *fiq, + struct fuse_copy_state *cs, + size_t nbytes) +__releases(fiq->lock) +{ + if (fch->minor < 16 || fiq->forget_list_head.next->next == NULL) + return fuse_read_single_forget(fiq, cs); + else + return fuse_read_batch_forget(fiq, cs, nbytes); +} + +/* + * Read a single request into the userspace filesystem's buffer. This + * function waits until a request is available, then removes it from + * the pending list and copies request data to userspace buffer. If + * no reply is needed (FORGET) or request has been aborted or there + * was an error during the copying then it's finished by calling + * fuse_request_end(). Otherwise add it to the processing list, and set + * the 'sent' flag. + */ +static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file, + struct fuse_copy_state *cs, size_t nbytes) +{ + ssize_t err; + struct fuse_chan *fch = fud->chan; + struct fuse_iqueue *fiq = &fch->iq; + struct fuse_pqueue *fpq = &fud->pq; + struct fuse_req *req; + struct fuse_args *args; + unsigned reqsize; + unsigned int hash; + + /* + * Require sane minimum read buffer - that has capacity for fixed part + * of any request header + negotiated max_write room for data. + * + * Historically libfuse reserves 4K for fixed header room, but e.g. + * GlusterFS reserves only 80 bytes + * + * = `sizeof(fuse_in_header) + sizeof(fuse_write_in)` + * + * which is the absolute minimum any sane filesystem should be using + * for header room. + */ + if (nbytes < max_t(size_t, FUSE_MIN_READ_BUFFER, + sizeof(struct fuse_in_header) + + sizeof(struct fuse_write_in) + + fch->max_write)) + return -EINVAL; + + restart: + for (;;) { + spin_lock(&fiq->lock); + if (!fiq->connected || request_pending(fiq)) + break; + spin_unlock(&fiq->lock); + + if (file->f_flags & O_NONBLOCK) + return -EAGAIN; + err = wait_event_interruptible_exclusive(fiq->waitq, + !fiq->connected || request_pending(fiq)); + if (err) + return err; + } + + if (!fiq->connected) { + err = fch->abort_with_err ? -ECONNABORTED : -ENODEV; + goto err_unlock; + } + + if (!list_empty(&fiq->interrupts)) + return fuse_read_interrupt(fiq, cs); + + if (forget_pending(fiq)) { + if (list_empty(&fiq->pending) || fiq->forget_batch-- > 0) + return fuse_read_forget(fch, fiq, cs, nbytes); + + if (fiq->forget_batch <= -8) + fiq->forget_batch = 16; + } + + req = list_entry(fiq->pending.next, struct fuse_req, list); + clear_bit(FR_PENDING, &req->flags); + list_del_init(&req->list); + spin_unlock(&fiq->lock); + + args = req->args; + reqsize = req->in.h.len; + + /* If request is too large, reply with an error and restart the read */ + if (nbytes < reqsize) { + req->out.h.error = -EIO; + /* SETXATTR is special, since it may contain too large data */ + if (args->opcode == FUSE_SETXATTR) + req->out.h.error = -E2BIG; + fuse_request_end(req); + goto restart; + } + spin_lock(&fpq->lock); + /* + * Must not put request on fpq->io queue after having been shut down by + * fuse_chan_abort() + */ + if (!fpq->connected) { + req->out.h.error = err = -ECONNABORTED; + goto out_end; + } + list_add(&req->list, &fpq->io); + spin_unlock(&fpq->lock); + cs->req = req; + err = fuse_copy_one(cs, &req->in.h, sizeof(req->in.h)); + if (!err) + err = fuse_copy_args(cs, args->in_numargs, args->in_pages, + (struct fuse_arg *) args->in_args, 0); + fuse_copy_finish(cs); + spin_lock(&fpq->lock); + clear_bit(FR_LOCKED, &req->flags); + if (!fpq->connected) { + err = fch->abort_with_err ? -ECONNABORTED : -ENODEV; + goto out_end; + } + if (err) { + req->out.h.error = -EIO; + goto out_end; + } + if (!test_bit(FR_ISREPLY, &req->flags)) { + err = reqsize; + goto out_end; + } + hash = fuse_req_hash(req->in.h.unique); + list_move_tail(&req->list, &fpq->processing[hash]); + __fuse_get_request(req); + set_bit(FR_SENT, &req->flags); + trace_fuse_request_sent(req); + spin_unlock(&fpq->lock); + /* matches barrier in request_wait_answer() */ + smp_mb__after_atomic(); + if (test_bit(FR_INTERRUPTED, &req->flags)) + queue_interrupt(req); + fuse_put_request(req); + + return reqsize; + +out_end: + if (!test_bit(FR_PRIVATE, &req->flags)) + list_del_init(&req->list); + spin_unlock(&fpq->lock); + fuse_request_end(req); + return err; + + err_unlock: + spin_unlock(&fiq->lock); + return err; +} + +static int fuse_dev_open(struct inode *inode, struct file *file) +{ + struct fuse_dev *fud = fuse_dev_alloc_no_pq(); + + if (!fud) + return -ENOMEM; + + file->private_data = fud; + return 0; +} + +struct fuse_dev *fuse_get_dev(struct file *file) +{ + struct fuse_dev *fud = fuse_file_to_fud(file); + int err; + + if (unlikely(!fuse_dev_chan_get(fud))) { + /* only block waiting for mount if sync init was requested */ + if (!fud->sync_init) + return ERR_PTR(-EPERM); + + err = wait_event_interruptible(fuse_dev_waitq, fuse_dev_chan_get(fud) != NULL); + if (err) + return ERR_PTR(err); + } + + return fud; +} + +static ssize_t fuse_dev_read(struct kiocb *iocb, struct iov_iter *to) +{ + struct fuse_copy_state cs; + struct file *file = iocb->ki_filp; + struct fuse_dev *fud = fuse_get_dev(file); + + if (IS_ERR(fud)) + return PTR_ERR(fud); + + if (!user_backed_iter(to)) + return -EINVAL; + + fuse_copy_init(&cs, true, to); + + return fuse_dev_do_read(fud, file, &cs, iov_iter_count(to)); +} + +static ssize_t fuse_dev_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, + size_t len, unsigned int flags) +{ + int total, ret; + int page_nr = 0; + struct pipe_buffer *bufs; + struct fuse_copy_state cs; + struct fuse_dev *fud = fuse_get_dev(in); + + if (IS_ERR(fud)) + return PTR_ERR(fud); + + bufs = kvmalloc_objs(struct pipe_buffer, pipe->max_usage); + if (!bufs) + return -ENOMEM; + + fuse_copy_init(&cs, true, NULL); + cs.pipebufs = bufs; + cs.pipe = pipe; + ret = fuse_dev_do_read(fud, in, &cs, len); + if (ret < 0) + goto out; + + if (pipe_buf_usage(pipe) + cs.nr_segs > pipe->max_usage) { + ret = -EIO; + goto out; + } + + for (ret = total = 0; page_nr < cs.nr_segs; total += ret) { + /* + * Need to be careful about this. Having buf->ops in module + * code can Oops if the buffer persists after module unload. + */ + bufs[page_nr].ops = &nosteal_pipe_buf_ops; + bufs[page_nr].flags = 0; + ret = add_to_pipe(pipe, &bufs[page_nr++]); + if (unlikely(ret < 0)) + break; + } + if (total) + ret = total; +out: + for (; page_nr < cs.nr_segs; page_nr++) + put_page(bufs[page_nr].page); + + kvfree(bufs); + return ret; +} + +/* + * Resending all processing queue requests. + * + * During a FUSE daemon panics and failover, it is possible for some inflight + * requests to be lost and never returned. As a result, applications awaiting + * replies would become stuck forever. To address this, we can use notification + * to trigger resending of these pending requests to the FUSE daemon, ensuring + * they are properly processed again. + * + * Please note that this strategy is applicable only to idempotent requests or + * if the FUSE daemon takes careful measures to avoid processing duplicated + * non-idempotent requests. + */ +void fuse_chan_resend(struct fuse_chan *fch) +{ + struct fuse_dev *fud; + struct fuse_req *req, *next; + struct fuse_iqueue *fiq = &fch->iq; + LIST_HEAD(to_queue); + unsigned int i; + + spin_lock(&fch->lock); + if (!fch->connected) { + spin_unlock(&fch->lock); + return; + } + + list_for_each_entry(fud, &fch->devices, entry) { + struct fuse_pqueue *fpq = &fud->pq; + + spin_lock(&fpq->lock); + for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) + list_splice_tail_init(&fpq->processing[i], &to_queue); + spin_unlock(&fpq->lock); + } + spin_unlock(&fch->lock); + + list_for_each_entry_safe(req, next, &to_queue, list) { + set_bit(FR_PENDING, &req->flags); + clear_bit(FR_SENT, &req->flags); + /* mark the request as resend request */ + req->in.h.unique |= FUSE_UNIQUE_RESEND; + } + + spin_lock(&fiq->lock); + if (!fiq->connected) { + spin_unlock(&fiq->lock); + list_for_each_entry(req, &to_queue, list) + clear_bit(FR_PENDING, &req->flags); + fuse_dev_end_requests(&to_queue); + return; + } + /* + * Remove interrupt entries for resent requests to prevent stale + * intr_entry on fiq->interrupts after the request is re-queued. + */ + list_for_each_entry(req, &to_queue, list) { + if (test_bit(FR_INTERRUPTED, &req->flags)) + list_del_init(&req->intr_entry); + } + /* iq and pq requests are both oldest to newest */ + list_splice(&to_queue, &fiq->pending); + fuse_dev_wake_and_unlock(fiq); +} + +/* Look up request on processing list by unique ID */ +struct fuse_req *fuse_request_find(struct fuse_pqueue *fpq, u64 unique) +{ + unsigned int hash = fuse_req_hash(unique); + struct fuse_req *req; + + list_for_each_entry(req, &fpq->processing[hash], list) { + if (req->in.h.unique == unique) + return req; + } + return NULL; +} + +int fuse_copy_out_args(struct fuse_copy_state *cs, struct fuse_args *args, + unsigned nbytes) +{ + + unsigned int reqsize = 0; + + /* + * Uring has all headers separated from args - args is payload only + */ + if (!cs->is_uring) + reqsize = sizeof(struct fuse_out_header); + + reqsize += fuse_len_args(args->out_numargs, args->out_args); + + if (reqsize < nbytes || (reqsize > nbytes && !args->out_argvar)) + return -EINVAL; + else if (reqsize > nbytes) { + struct fuse_arg *lastarg = &args->out_args[args->out_numargs-1]; + unsigned diffsize = reqsize - nbytes; + + if (diffsize > lastarg->size) + return -EINVAL; + lastarg->size -= diffsize; + } + return fuse_copy_args(cs, args->out_numargs, args->out_pages, + args->out_args, args->page_zeroing); +} + +/* + * Write a single reply to a request. First the header is copied from + * the write buffer. The request is then searched on the processing + * list by the unique ID found in the header. If found, then remove + * it from the list and copy the rest of the buffer to the request. + * The request is finished by calling fuse_request_end(). + */ +static ssize_t fuse_dev_do_write(struct fuse_dev *fud, + struct fuse_copy_state *cs, size_t nbytes) +{ + int err; + struct fuse_chan *fch = fud->chan; + struct fuse_pqueue *fpq = &fud->pq; + struct fuse_req *req; + struct fuse_out_header oh; + + err = -EINVAL; + if (nbytes < sizeof(struct fuse_out_header)) + goto out; + + err = fuse_copy_one(cs, &oh, sizeof(oh)); + if (err) + goto copy_finish; + + err = -EINVAL; + if (oh.len != nbytes) + goto copy_finish; + + /* + * Zero oh.unique indicates unsolicited notification message + * and error contains notification code. + */ + if (!oh.unique) { + /* + * Only allow notifications during while the connection is in an + * initialized and connected state + */ + err = -EINVAL; + if (!fch->initialized || !fch->connected) + goto copy_finish; + + /* Don't try to move folios (yet) */ + cs->move_folios = false; + + err = fuse_notify(fch->conn, oh.error, nbytes - sizeof(oh), cs); + goto copy_finish; + } + + err = -EINVAL; + if (oh.error <= -512 || oh.error > 0) + goto copy_finish; + + spin_lock(&fpq->lock); + req = NULL; + if (fpq->connected) + req = fuse_request_find(fpq, oh.unique & ~FUSE_INT_REQ_BIT); + + err = -ENOENT; + if (!req) { + spin_unlock(&fpq->lock); + goto copy_finish; + } + + /* Is it an interrupt reply ID? */ + if (oh.unique & FUSE_INT_REQ_BIT) { + __fuse_get_request(req); + spin_unlock(&fpq->lock); + + err = 0; + if (nbytes != sizeof(struct fuse_out_header)) + err = -EINVAL; + else if (oh.error == -ENOSYS) + fch->no_interrupt = 1; + else if (oh.error == -EAGAIN) + err = queue_interrupt(req); + + fuse_put_request(req); + + goto copy_finish; + } + + clear_bit(FR_SENT, &req->flags); + list_move(&req->list, &fpq->io); + req->out.h = oh; + set_bit(FR_LOCKED, &req->flags); + spin_unlock(&fpq->lock); + cs->req = req; + if (!req->args->page_replace) + cs->move_folios = false; + + if (oh.error) + err = nbytes != sizeof(oh) ? -EINVAL : 0; + else + err = fuse_copy_out_args(cs, req->args, nbytes); + fuse_copy_finish(cs); + + spin_lock(&fpq->lock); + clear_bit(FR_LOCKED, &req->flags); + if (!fpq->connected) + err = -ENOENT; + else if (err) + req->out.h.error = -EIO; + if (!test_bit(FR_PRIVATE, &req->flags)) + list_del_init(&req->list); + spin_unlock(&fpq->lock); + + fuse_request_end(req); +out: + return err ? err : nbytes; + +copy_finish: + fuse_copy_finish(cs); + goto out; +} + +static ssize_t fuse_dev_write(struct kiocb *iocb, struct iov_iter *from) +{ + struct fuse_copy_state cs; + struct fuse_dev *fud = __fuse_get_dev(iocb->ki_filp); + + if (!fud) + return -EPERM; + + if (!user_backed_iter(from)) + return -EINVAL; + + fuse_copy_init(&cs, false, from); + + return fuse_dev_do_write(fud, &cs, iov_iter_count(from)); +} + +static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe, + struct file *out, loff_t *ppos, + size_t len, unsigned int flags) +{ + unsigned int head, tail, count; + unsigned nbuf; + unsigned idx; + struct pipe_buffer *bufs; + struct fuse_copy_state cs; + struct fuse_dev *fud = __fuse_get_dev(out); + size_t rem; + ssize_t ret; + + if (!fud) + return -EPERM; + + pipe_lock(pipe); + + head = pipe->head; + tail = pipe->tail; + count = pipe_occupancy(head, tail); + + bufs = kvmalloc_objs(struct pipe_buffer, count); + if (!bufs) { + pipe_unlock(pipe); + return -ENOMEM; + } + + nbuf = 0; + rem = 0; + for (idx = tail; !pipe_empty(head, idx) && rem < len; idx++) + rem += pipe_buf(pipe, idx)->len; + + ret = -EINVAL; + if (rem < len) + goto out_free; + + rem = len; + while (rem) { + struct pipe_buffer *ibuf; + struct pipe_buffer *obuf; + + if (WARN_ON(nbuf >= count || pipe_empty(head, tail))) + goto out_free; + + ibuf = pipe_buf(pipe, tail); + obuf = &bufs[nbuf]; + + if (rem >= ibuf->len) { + *obuf = *ibuf; + ibuf->ops = NULL; + tail++; + pipe->tail = tail; + } else { + if (!pipe_buf_get(pipe, ibuf)) + goto out_free; + + *obuf = *ibuf; + obuf->flags &= ~PIPE_BUF_FLAG_GIFT; + obuf->len = rem; + ibuf->offset += obuf->len; + ibuf->len -= obuf->len; + } + nbuf++; + rem -= obuf->len; + } + pipe_unlock(pipe); + + fuse_copy_init(&cs, false, NULL); + cs.pipebufs = bufs; + cs.nr_segs = nbuf; + cs.pipe = pipe; + + if (flags & SPLICE_F_MOVE) + cs.move_folios = true; + + ret = fuse_dev_do_write(fud, &cs, len); + + pipe_lock(pipe); +out_free: + for (idx = 0; idx < nbuf; idx++) { + struct pipe_buffer *buf = &bufs[idx]; + + if (buf->ops) + pipe_buf_release(pipe, buf); + } + pipe_unlock(pipe); + + kvfree(bufs); + return ret; +} + +static __poll_t fuse_dev_poll(struct file *file, poll_table *wait) +{ + __poll_t mask = EPOLLOUT | EPOLLWRNORM; + struct fuse_iqueue *fiq; + struct fuse_dev *fud = fuse_get_dev(file); + + if (IS_ERR(fud)) + return EPOLLERR; + + fiq = &fud->chan->iq; + poll_wait(file, &fiq->waitq, wait); + + spin_lock(&fiq->lock); + if (!fiq->connected) + mask = EPOLLERR; + else if (request_pending(fiq)) + mask |= EPOLLIN | EPOLLRDNORM; + spin_unlock(&fiq->lock); + + return mask; +} + +/* Abort all requests on the given list (pending or processing) */ +void fuse_dev_end_requests(struct list_head *head) +{ + while (!list_empty(head)) { + struct fuse_req *req; + req = list_entry(head->next, struct fuse_req, list); + req->out.h.error = -ECONNABORTED; + clear_bit(FR_SENT, &req->flags); + list_del_init(&req->list); + fuse_request_end(req); + } +} + +/* + * Abort all requests. + * + * Emergency exit in case of a malicious or accidental deadlock, or just a hung + * filesystem. + * + * The same effect is usually achievable through killing the filesystem daemon + * and all users of the filesystem. The exception is the combination of an + * asynchronous request and the tricky deadlock (see + * Documentation/filesystems/fuse/fuse.rst). + * + * Aborting requests under I/O goes as follows: 1: Separate out unlocked + * requests, they should be finished off immediately. Locked requests will be + * finished after unlock; see unlock_request(). 2: Finish off the unlocked + * requests. It is possible that some request will finish before we can. This + * is OK, the request will in that case be removed from the list before we touch + * it. + */ +void fuse_chan_abort(struct fuse_chan *fch, bool abort_with_err) +{ + struct fuse_iqueue *fiq = &fch->iq; + + fch->abort_with_err = abort_with_err; + + spin_lock(&fch->lock); + if (fch->connected) { + struct fuse_dev *fud; + struct fuse_req *req, *next; + LIST_HEAD(to_end); + unsigned int i; + + if (fch->timeout.req_timeout) + cancel_delayed_work(&fch->timeout.work); + + /* Background queuing checks fch->connected under bg_lock */ + spin_lock(&fch->bg_lock); + fch->connected = 0; + spin_unlock(&fch->bg_lock); + + fuse_chan_set_initialized(fch, NULL); + list_for_each_entry(fud, &fch->devices, entry) { + struct fuse_pqueue *fpq = &fud->pq; + + spin_lock(&fpq->lock); + fpq->connected = 0; + list_for_each_entry_safe(req, next, &fpq->io, list) { + req->out.h.error = -ECONNABORTED; + spin_lock(&req->waitq.lock); + set_bit(FR_ABORTED, &req->flags); + if (!test_bit(FR_LOCKED, &req->flags)) { + set_bit(FR_PRIVATE, &req->flags); + __fuse_get_request(req); + list_move(&req->list, &to_end); + } + spin_unlock(&req->waitq.lock); + } + for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) + list_splice_tail_init(&fpq->processing[i], + &to_end); + spin_unlock(&fpq->lock); + } + spin_lock(&fch->bg_lock); + fch->blocked = 0; + fch->max_background = UINT_MAX; + flush_bg_queue(fch); + spin_unlock(&fch->bg_lock); + + spin_lock(&fiq->lock); + fiq->connected = 0; + list_for_each_entry(req, &fiq->pending, list) + clear_bit(FR_PENDING, &req->flags); + list_splice_tail_init(&fiq->pending, &to_end); + while (forget_pending(fiq)) + kfree(fuse_dequeue_forget(fiq, 1, NULL)); + wake_up_all(&fiq->waitq); + spin_unlock(&fiq->lock); + kill_fasync(&fiq->fasync, SIGIO, POLL_IN); + fuse_end_polls(fch->conn); + wake_up_all(&fch->blocked_waitq); + spin_unlock(&fch->lock); + + fuse_dev_end_requests(&to_end); + + /* + * fch->lock must not be taken to avoid conflicts with io-uring + * locks + */ + fuse_uring_abort(fch); + } else { + spin_unlock(&fch->lock); + } +} +EXPORT_SYMBOL_GPL(fuse_chan_abort); + +void fuse_chan_wait_aborted(struct fuse_chan *fch) +{ + /* matches implicit memory barrier in fuse_drop_waiting() */ + smp_mb(); + wait_event(fch->blocked_waitq, fuse_chan_num_waiting(fch) == 0); + + fuse_uring_wait_stopped_queues(fch); +} + +int fuse_dev_release(struct inode *inode, struct file *file) +{ + struct fuse_dev *fud = fuse_file_to_fud(file); + /* Pairs with cmpxchg() in fuse_dev_install() */ + struct fuse_chan *fch = xchg(&fud->chan, FUSE_DEV_CHAN_DISCONNECTED); + + if (fch) { + struct fuse_pqueue *fpq = &fud->pq; + LIST_HEAD(to_end); + unsigned int i; + bool last; + + /* Make sure fuse_dev_install_with_pq() has finished */ + spin_lock(&fch->lock); + spin_lock(&fpq->lock); + WARN_ON(!list_empty(&fpq->io)); + for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) + list_splice_init(&fpq->processing[i], &to_end); + spin_unlock(&fpq->lock); + + list_del(&fud->entry); + /* Are we the last open device? */ + last = list_empty(&fch->devices); + spin_unlock(&fch->lock); + + fuse_dev_end_requests(&to_end); + + if (last) { + WARN_ON(fch->iq.fasync != NULL); + fuse_chan_abort(fch, false); + } + fuse_conn_put(fch->conn); + } + fuse_dev_put(fud); + return 0; +} +EXPORT_SYMBOL_GPL(fuse_dev_release); + +static int fuse_dev_fasync(int fd, struct file *file, int on) +{ + struct fuse_dev *fud = fuse_get_dev(file); + + if (IS_ERR(fud)) + return PTR_ERR(fud); + + /* No locking - fasync_helper does its own locking */ + return fasync_helper(fd, file, on, &fud->chan->iq.fasync); +} + +static long fuse_dev_ioctl_clone(struct file *file, __u32 __user *argp) +{ + int oldfd; + struct fuse_dev *fud, *new_fud; + struct list_head *pq; + + if (get_user(oldfd, argp)) + return -EFAULT; + + CLASS(fd, f)(oldfd); + if (fd_empty(f)) + return -EINVAL; + + /* + * Check against file->f_op because CUSE + * uses the same ioctl handler. + */ + if (fd_file(f)->f_op != file->f_op) + return -EINVAL; + + fud = fuse_get_dev(fd_file(f)); + if (IS_ERR(fud)) + return PTR_ERR(fud); + + pq = fuse_pqueue_alloc(); + if (!pq) + return -ENOMEM; + + new_fud = fuse_file_to_fud(file); + if (!fuse_dev_install_with_pq(new_fud, fud->chan, pq)) + return -EINVAL; + + return 0; +} + +static long fuse_dev_ioctl_backing_open(struct file *file, + struct fuse_backing_map __user *argp) +{ + struct fuse_dev *fud = fuse_get_dev(file); + struct fuse_backing_map map; + + if (IS_ERR(fud)) + return PTR_ERR(fud); + + if (!IS_ENABLED(CONFIG_FUSE_PASSTHROUGH)) + return -EOPNOTSUPP; + + if (copy_from_user(&map, argp, sizeof(map))) + return -EFAULT; + + return fuse_backing_open(fud->chan->conn, &map); +} + +static long fuse_dev_ioctl_backing_close(struct file *file, __u32 __user *argp) +{ + struct fuse_dev *fud = fuse_get_dev(file); + int backing_id; + + if (IS_ERR(fud)) + return PTR_ERR(fud); + + if (!IS_ENABLED(CONFIG_FUSE_PASSTHROUGH)) + return -EOPNOTSUPP; + + if (get_user(backing_id, argp)) + return -EFAULT; + + return fuse_backing_close(fud->chan->conn, backing_id); +} + +static long fuse_dev_ioctl_sync_init(struct file *file) +{ + struct fuse_dev *fud = fuse_file_to_fud(file); + + if (fuse_dev_chan_get(fud)) + return -EINVAL; + + fud->sync_init = true; + return 0; +} + +static long fuse_dev_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + void __user *argp = (void __user *)arg; + + switch (cmd) { + case FUSE_DEV_IOC_CLONE: + return fuse_dev_ioctl_clone(file, argp); + + case FUSE_DEV_IOC_BACKING_OPEN: + return fuse_dev_ioctl_backing_open(file, argp); + + case FUSE_DEV_IOC_BACKING_CLOSE: + return fuse_dev_ioctl_backing_close(file, argp); + + case FUSE_DEV_IOC_SYNC_INIT: + return fuse_dev_ioctl_sync_init(file); + + default: + return -ENOTTY; + } +} + +#ifdef CONFIG_PROC_FS +static void fuse_dev_show_fdinfo(struct seq_file *seq, struct file *file) +{ + struct fuse_dev *fud = __fuse_get_dev(file); + if (!fud) + return; + + seq_printf(seq, "fuse_connection:\t%u\n", fuse_conn_get_id(fud->chan->conn)); +} +#endif + +const struct file_operations fuse_dev_operations = { + .owner = THIS_MODULE, + .open = fuse_dev_open, + .read_iter = fuse_dev_read, + .splice_read = fuse_dev_splice_read, + .write_iter = fuse_dev_write, + .splice_write = fuse_dev_splice_write, + .poll = fuse_dev_poll, + .release = fuse_dev_release, + .fasync = fuse_dev_fasync, + .unlocked_ioctl = fuse_dev_ioctl, + .compat_ioctl = compat_ptr_ioctl, +#ifdef CONFIG_FUSE_IO_URING + .uring_cmd = fuse_uring_cmd, +#endif +#ifdef CONFIG_PROC_FS + .show_fdinfo = fuse_dev_show_fdinfo, +#endif +}; +EXPORT_SYMBOL_GPL(fuse_dev_operations); + +static struct miscdevice fuse_miscdevice = { + .minor = FUSE_MINOR, + .name = "fuse", + .fops = &fuse_dev_operations, +}; + +int __init fuse_dev_init(void) +{ + int err = -ENOMEM; + fuse_req_cachep = kmem_cache_create("fuse_request", + sizeof(struct fuse_req), + 0, 0, NULL); + if (!fuse_req_cachep) + goto out; + + err = misc_register(&fuse_miscdevice); + if (err) + goto out_cache_clean; + + return 0; + + out_cache_clean: + kmem_cache_destroy(fuse_req_cachep); + out: + return err; +} + +void fuse_dev_cleanup(void) +{ + misc_deregister(&fuse_miscdevice); + kmem_cache_destroy(fuse_req_cachep); +} diff --git a/fs/fuse/dev.h b/fs/fuse/dev.h new file mode 100644 index 000000000..aed69fd14 --- /dev/null +++ b/fs/fuse/dev.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _FS_FUSE_DEV_H +#define _FS_FUSE_DEV_H + +#include + +/** Maximum number of outstanding background requests */ +#define FUSE_DEFAULT_MAX_BACKGROUND 12 + +struct fuse_conn; +struct fuse_chan; +struct fuse_dev; +struct fuse_args; +struct fuse_copy_state; +struct fuse_backing_map; +struct file; +struct folio; +enum fuse_notify_code; + +struct fuse_chan_param { + unsigned int minor; + unsigned int max_write; + unsigned int max_pages; +}; + +struct fuse_chan *fuse_chan_new(void); +struct fuse_chan *fuse_dev_chan_new(void); +void fuse_chan_release(struct fuse_chan *fch); +void fuse_chan_free(struct fuse_chan *fch); +unsigned int fuse_chan_num_background(struct fuse_chan *fch); +unsigned int fuse_chan_max_background(struct fuse_chan *fch); +void fuse_chan_max_background_set(struct fuse_chan *fch, unsigned int val); +unsigned int fuse_chan_num_waiting(struct fuse_chan *fch); +void fuse_chan_set_fc(struct fuse_chan *fch, struct fuse_conn *fc); +void fuse_chan_set_initialized(struct fuse_chan *fch, struct fuse_chan_param *param); +void fuse_chan_io_uring_enable(struct fuse_chan *fch); +ssize_t fuse_chan_send(struct fuse_chan *fch, struct fuse_args *args); +int fuse_chan_send_bg(struct fuse_chan *fch, struct fuse_args *args, gfp_t gfp_flags); +int fuse_chan_send_notify_reply(struct fuse_chan *fch, struct fuse_args *args, u64 unique); +void fuse_chan_resend(struct fuse_chan *fch); + +struct fuse_forget_link *fuse_alloc_forget(void); +void fuse_chan_queue_forget(struct fuse_chan *fch, struct fuse_forget_link *forget, + u64 nodeid, u64 nlookup); + +DEFINE_FREE(fuse_chan_free, struct fuse_chan *, if (_T) fuse_chan_free(_T)) + +/** + * Initialize the client device + */ +int fuse_dev_init(void); + +/** + * Cleanup the client device + */ +void fuse_dev_cleanup(void); + +void fuse_dev_install(struct fuse_dev *fud, struct fuse_chan *fch); +bool fuse_dev_verify(struct fuse_dev *fud, struct fuse_chan *fch); +void fuse_dev_put(struct fuse_dev *fud); +bool fuse_dev_is_installed(struct fuse_dev *fud); +bool fuse_dev_is_sync_init(struct fuse_dev *fud); +struct fuse_dev *fuse_dev_grab(struct file *file); + +void fuse_init_server_timeout(struct fuse_chan *fch, unsigned int timeout); + +/* Abort all requests */ +void fuse_chan_abort(struct fuse_chan *fch, bool abort_with_err); +void fuse_chan_wait_aborted(struct fuse_chan *fch); + +/** + * Acquire reference to fuse_conn + */ +struct fuse_conn *fuse_conn_get(struct fuse_conn *fc); + +/** + * Release reference to fuse_conn + */ +void fuse_conn_put(struct fuse_conn *fc); + +dev_t fuse_conn_get_id(struct fuse_conn *fc); + +void fuse_end_polls(struct fuse_conn *fc); +int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code, + unsigned int size, struct fuse_copy_state *cs); + +int fuse_backing_open(struct fuse_conn *fc, struct fuse_backing_map *map); +int fuse_backing_close(struct fuse_conn *fc, int backing_id); + +int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size); +int fuse_copy_folio(struct fuse_copy_state *cs, struct folio **foliop, + unsigned offset, unsigned count, int zeroing); +void fuse_copy_finish(struct fuse_copy_state *cs); + +#ifdef CONFIG_FUSE_IO_URING +bool fuse_uring_enabled(void); +void fuse_uring_destruct(struct fuse_chan *fch); +#else /* CONFIG_FUSE_IO_URING */ +static inline bool fuse_uring_enabled(void) +{ + return false; +} + +static inline void fuse_uring_destruct(struct fuse_chan *fch) +{ +} +#endif /* CONFIG_FUSE_IO_URING */ + +#endif /* _FS_FUSE_DEV_H */ diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c new file mode 100644 index 000000000..77c8cec43 --- /dev/null +++ b/fs/fuse/dev_uring.c @@ -0,0 +1,1456 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * FUSE: Filesystem in Userspace + * Copyright (c) 2023-2024 DataDirect Networks. + */ + +#include "dev.h" +#include "args.h" +#include "dev_uring_i.h" +#include "fuse_trace.h" + +#include +#include + +static bool __read_mostly enable_uring; +module_param(enable_uring, bool, 0644); +MODULE_PARM_DESC(enable_uring, + "Enable userspace communication through io-uring"); + +#define FUSE_URING_IOV_SEGS 2 /* header and payload */ +#define FUSE_URING_IOV_HEADERS 0 +#define FUSE_URING_IOV_PAYLOAD 1 + +bool fuse_uring_enabled(void) +{ + return enable_uring; +} + +struct fuse_uring_pdu { + struct fuse_ring_ent *ent; +}; + +static const struct fuse_iqueue_ops fuse_io_uring_ops; + +enum fuse_uring_header_type { + /* struct fuse_in_header / struct fuse_out_header */ + FUSE_URING_HEADER_IN_OUT, + /* per op code header */ + FUSE_URING_HEADER_OP, + /* struct fuse_uring_ent_in_out header */ + FUSE_URING_HEADER_RING_ENT, +}; + +static void uring_cmd_set_ring_ent(struct io_uring_cmd *cmd, + struct fuse_ring_ent *ring_ent) +{ + struct fuse_uring_pdu *pdu = + io_uring_cmd_to_pdu(cmd, struct fuse_uring_pdu); + + pdu->ent = ring_ent; +} + +static struct fuse_ring_ent *uring_cmd_to_ring_ent(struct io_uring_cmd *cmd) +{ + struct fuse_uring_pdu *pdu = + io_uring_cmd_to_pdu(cmd, struct fuse_uring_pdu); + + return pdu->ent; +} + +static void fuse_uring_flush_bg(struct fuse_ring_queue *queue) +{ + struct fuse_ring *ring = queue->ring; + struct fuse_chan *fch = ring->chan; + + lockdep_assert_held(&queue->lock); + lockdep_assert_held(&fch->bg_lock); + + /* + * Allow one bg request per queue, ignoring global fc limits. + * This prevents a single queue from consuming all resources and + * eliminates the need for remote queue wake-ups when global + * limits are met but this queue has no more waiting requests. + */ + while ((fch->active_background < fch->max_background || + !queue->active_background) && + (!list_empty(&queue->fuse_req_bg_queue))) { + struct fuse_req *req; + + req = list_first_entry(&queue->fuse_req_bg_queue, + struct fuse_req, list); + fch->active_background++; + queue->active_background++; + + list_move_tail(&req->list, &queue->fuse_req_queue); + } +} + +static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req, + int error) +{ + struct fuse_ring_queue *queue = ent->queue; + struct fuse_ring *ring = queue->ring; + struct fuse_chan *fch = ring->chan; + + lockdep_assert_not_held(&queue->lock); + spin_lock(&queue->lock); + ent->fuse_req = NULL; + list_del_init(&req->list); + if (test_bit(FR_BACKGROUND, &req->flags)) { + queue->active_background--; + spin_lock(&fch->bg_lock); + fuse_request_bg_finish(fch, req); + fuse_uring_flush_bg(queue); + spin_unlock(&fch->bg_lock); + } + + spin_unlock(&queue->lock); + + if (error) + req->out.h.error = error; + + clear_bit(FR_SENT, &req->flags); + fuse_request_end(req); +} + +/* Abort all list queued request on the given ring queue */ +static void fuse_uring_abort_end_queue_requests(struct fuse_ring_queue *queue) +{ + struct fuse_req *req; + LIST_HEAD(req_list); + + spin_lock(&queue->lock); + list_for_each_entry(req, &queue->fuse_req_queue, list) + clear_bit(FR_PENDING, &req->flags); + list_splice_init(&queue->fuse_req_queue, &req_list); + spin_unlock(&queue->lock); + + /* must not hold queue lock to avoid order issues with fi->lock */ + fuse_dev_end_requests(&req_list); +} + +void fuse_uring_abort_end_requests(struct fuse_ring *ring) +{ + int qid; + struct fuse_ring_queue *queue; + struct fuse_chan *fch = ring->chan; + + for (qid = 0; qid < ring->nr_queues; qid++) { + queue = READ_ONCE(ring->queues[qid]); + if (!queue) + continue; + + WARN_ON_ONCE(fch->max_background != UINT_MAX); + spin_lock(&queue->lock); + queue->stopped = true; + spin_lock(&fch->bg_lock); + fuse_uring_flush_bg(queue); + spin_unlock(&fch->bg_lock); + spin_unlock(&queue->lock); + fuse_uring_abort_end_queue_requests(queue); + } +} + +static bool ent_list_request_expired(struct fuse_chan *fch, struct list_head *list) +{ + struct fuse_ring_ent *ent; + struct fuse_req *req; + + ent = list_first_entry_or_null(list, struct fuse_ring_ent, list); + if (!ent) + return false; + + req = ent->fuse_req; + + return time_is_before_jiffies(req->create_time + + fch->timeout.req_timeout); +} + +bool fuse_uring_request_expired(struct fuse_chan *fch) +{ + struct fuse_ring *ring = fch->ring; + struct fuse_ring_queue *queue; + int qid; + + if (!ring) + return false; + + for (qid = 0; qid < ring->nr_queues; qid++) { + queue = READ_ONCE(ring->queues[qid]); + if (!queue) + continue; + + spin_lock(&queue->lock); + if (fuse_request_expired(fch, &queue->fuse_req_queue) || + fuse_request_expired(fch, &queue->fuse_req_bg_queue) || + ent_list_request_expired(fch, &queue->ent_w_req_queue) || + ent_list_request_expired(fch, &queue->ent_in_userspace)) { + spin_unlock(&queue->lock); + return true; + } + spin_unlock(&queue->lock); + } + + return false; +} + +void fuse_uring_destruct(struct fuse_chan *fch) +{ + struct fuse_ring *ring = fch->ring; + int qid; + + if (!ring) + return; + + for (qid = 0; qid < ring->nr_queues; qid++) { + struct fuse_ring_queue *queue = ring->queues[qid]; + struct fuse_ring_ent *ent, *next; + + if (!queue) + continue; + + WARN_ON(!list_empty(&queue->ent_avail_queue)); + WARN_ON(!list_empty(&queue->ent_w_req_queue)); + WARN_ON(!list_empty(&queue->ent_commit_queue)); + WARN_ON(!list_empty(&queue->ent_in_userspace)); + + list_for_each_entry_safe(ent, next, &queue->ent_released, + list) { + list_del_init(&ent->list); + kfree(ent); + } + + kfree(queue->fpq.processing); + kfree(queue); + ring->queues[qid] = NULL; + } + + kfree(ring->queues); + kfree(ring); + fch->ring = NULL; +} + +/* + * Basic ring setup for this connection based on the provided configuration + */ +static struct fuse_ring *fuse_uring_create(struct fuse_chan *fch) +{ + struct fuse_ring *ring; + size_t nr_queues = num_possible_cpus(); + struct fuse_ring *res = NULL; + size_t max_payload_size; + + ring = kzalloc_obj(*ring, GFP_KERNEL_ACCOUNT); + if (!ring) + return NULL; + + ring->queues = kzalloc_objs(struct fuse_ring_queue *, nr_queues, + GFP_KERNEL_ACCOUNT); + if (!ring->queues) + goto out_err; + + max_payload_size = max(FUSE_MIN_READ_BUFFER, fch->max_write); + max_payload_size = max(max_payload_size, fch->max_pages * PAGE_SIZE); + + spin_lock(&fch->lock); + if (!fch->connected) { + spin_unlock(&fch->lock); + goto out_err; + } + if (fch->ring) { + /* race, another thread created the ring in the meantime */ + spin_unlock(&fch->lock); + res = fch->ring; + goto out_err; + } + + init_waitqueue_head(&ring->stop_waitq); + + ring->nr_queues = nr_queues; + ring->chan = fch; + ring->max_payload_sz = max_payload_size; + smp_store_release(&fch->ring, ring); + + spin_unlock(&fch->lock); + return ring; + +out_err: + kfree(ring->queues); + kfree(ring); + return res; +} + +static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, + int qid) +{ + struct fuse_chan *fch = ring->chan; + struct fuse_ring_queue *queue; + struct list_head *pq; + + queue = kzalloc_obj(*queue, GFP_KERNEL_ACCOUNT); + if (!queue) + return NULL; + pq = fuse_pqueue_alloc(); + if (!pq) { + kfree(queue); + return NULL; + } + + queue->qid = qid; + queue->ring = ring; + spin_lock_init(&queue->lock); + + INIT_LIST_HEAD(&queue->ent_avail_queue); + INIT_LIST_HEAD(&queue->ent_commit_queue); + INIT_LIST_HEAD(&queue->ent_w_req_queue); + INIT_LIST_HEAD(&queue->ent_in_userspace); + INIT_LIST_HEAD(&queue->fuse_req_queue); + INIT_LIST_HEAD(&queue->fuse_req_bg_queue); + INIT_LIST_HEAD(&queue->ent_released); + + fuse_pqueue_init(&queue->fpq); + queue->fpq.processing = pq; + + spin_lock(&fch->lock); + if (ring->queues[qid]) { + spin_unlock(&fch->lock); + kfree(queue->fpq.processing); + kfree(queue); + return ring->queues[qid]; + } + + /* + * write_once and lock as the caller mostly doesn't take the lock at all + */ + WRITE_ONCE(ring->queues[qid], queue); + spin_unlock(&fch->lock); + + return queue; +} + +static void fuse_uring_stop_fuse_req_end(struct fuse_req *req) +{ + clear_bit(FR_SENT, &req->flags); + req->out.h.error = -ECONNABORTED; + fuse_request_end(req); +} + +/* + * Release a request/entry on connection tear down + */ +static void fuse_uring_entry_teardown(struct fuse_ring_ent *ent) +{ + struct fuse_req *req; + struct io_uring_cmd *cmd; + + struct fuse_ring_queue *queue = ent->queue; + + spin_lock(&queue->lock); + cmd = ent->cmd; + ent->cmd = NULL; + req = ent->fuse_req; + ent->fuse_req = NULL; + if (req) { + /* remove entry from queue->fpq->processing */ + list_del_init(&req->list); + } + + /* + * The entry must not be freed immediately, due to access of direct + * pointer access of entries through IO_URING_F_CANCEL - there is a risk + * of race between daemon termination (which triggers IO_URING_F_CANCEL + * and accesses entries without checking the list state first + */ + list_move(&ent->list, &queue->ent_released); + ent->state = FRRS_RELEASED; + spin_unlock(&queue->lock); + + if (cmd) + io_uring_cmd_done(cmd, -ENOTCONN, IO_URING_F_UNLOCKED); + + if (req) + fuse_uring_stop_fuse_req_end(req); +} + +static void fuse_uring_stop_list_entries(struct list_head *head, + struct fuse_ring_queue *queue, + enum fuse_ring_req_state exp_state) +{ + struct fuse_ring *ring = queue->ring; + struct fuse_ring_ent *ent, *next; + ssize_t queue_refs = SSIZE_MAX; + LIST_HEAD(to_teardown); + + spin_lock(&queue->lock); + list_for_each_entry_safe(ent, next, head, list) { + if (ent->state != exp_state) { + pr_warn("entry teardown qid=%d state=%d expected=%d", + queue->qid, ent->state, exp_state); + continue; + } + + ent->state = FRRS_TEARDOWN; + list_move(&ent->list, &to_teardown); + } + spin_unlock(&queue->lock); + + /* no queue lock to avoid lock order issues */ + list_for_each_entry_safe(ent, next, &to_teardown, list) { + fuse_uring_entry_teardown(ent); + queue_refs = atomic_dec_return(&ring->queue_refs); + WARN_ON_ONCE(queue_refs < 0); + } +} + +static void fuse_uring_teardown_entries(struct fuse_ring_queue *queue) +{ + fuse_uring_stop_list_entries(&queue->ent_in_userspace, queue, + FRRS_USERSPACE); + fuse_uring_stop_list_entries(&queue->ent_avail_queue, queue, + FRRS_AVAILABLE); +} + +static void fuse_uring_teardown_all_queues(struct fuse_ring *ring) +{ + int qid; + + for (qid = 0; qid < ring->nr_queues; qid++) { + struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]); + + if (!queue) + continue; + + fuse_uring_teardown_entries(queue); + } +} + +/* + * Log state debug info + */ +static void fuse_uring_log_ent_state(struct fuse_ring *ring) +{ + int qid; + struct fuse_ring_ent *ent; + + for (qid = 0; qid < ring->nr_queues; qid++) { + struct fuse_ring_queue *queue = ring->queues[qid]; + + if (!queue) + continue; + + spin_lock(&queue->lock); + /* + * Log entries from the intermediate queue, the other queues + * should be empty + */ + list_for_each_entry(ent, &queue->ent_w_req_queue, list) { + pr_info(" ent-req-queue ring=%p qid=%d ent=%p state=%d\n", + ring, qid, ent, ent->state); + } + list_for_each_entry(ent, &queue->ent_commit_queue, list) { + pr_info(" ent-commit-queue ring=%p qid=%d ent=%p state=%d\n", + ring, qid, ent, ent->state); + } + spin_unlock(&queue->lock); + } + ring->stop_debug_log = 1; +} + +static void fuse_uring_async_stop_queues(struct work_struct *work) +{ + struct fuse_ring *ring = + container_of(work, struct fuse_ring, async_teardown_work.work); + + fuse_uring_teardown_all_queues(ring); + + /* + * Some ring entries might be in the middle of IO operations, + * i.e. in process to get handled by file_operations::uring_cmd + * or on the way to userspace - we could handle that with conditions in + * run time code, but easier/cleaner to have an async tear down handler + * If there are still queue references left + */ + if (atomic_read(&ring->queue_refs) > 0) { + if (time_after(jiffies, + ring->teardown_time + FUSE_URING_TEARDOWN_TIMEOUT)) + fuse_uring_log_ent_state(ring); + + schedule_delayed_work(&ring->async_teardown_work, + FUSE_URING_TEARDOWN_INTERVAL); + } else { + wake_up_all(&ring->stop_waitq); + fuse_conn_put(ring->chan->conn); + } +} + +/* + * Stop the ring queues + */ +void fuse_uring_stop_queues(struct fuse_ring *ring) +{ + fuse_uring_teardown_all_queues(ring); + + if (atomic_read(&ring->queue_refs) > 0) { + fuse_conn_get(ring->chan->conn); + ring->teardown_time = jiffies; + INIT_DELAYED_WORK(&ring->async_teardown_work, + fuse_uring_async_stop_queues); + schedule_delayed_work(&ring->async_teardown_work, + FUSE_URING_TEARDOWN_INTERVAL); + } else { + wake_up_all(&ring->stop_waitq); + } +} + +/* + * Handle IO_URING_F_CANCEL, typically should come on daemon termination. + * + * Releasing the last entry should trigger fuse_dev_release() if + * the daemon was terminated + */ +static void fuse_uring_cancel(struct io_uring_cmd *cmd, + unsigned int issue_flags) +{ + struct fuse_ring_ent *ent = uring_cmd_to_ring_ent(cmd); + struct fuse_ring_queue *queue; + bool need_cmd_done = false; + + /* + * direct access on ent - it must not be destructed as long as + * IO_URING_F_CANCEL might come up + */ + queue = ent->queue; + spin_lock(&queue->lock); + if (ent->state == FRRS_AVAILABLE) { + list_del_init(&ent->list); + need_cmd_done = true; + ent->cmd = NULL; + } + spin_unlock(&queue->lock); + + if (need_cmd_done) { + /* no queue lock to avoid lock order issues */ + io_uring_cmd_done(cmd, -ENOTCONN, issue_flags); + kfree(ent); + if (atomic_dec_and_test(&queue->ring->queue_refs)) + wake_up_all(&queue->ring->stop_waitq); + } +} + +static void fuse_uring_prepare_cancel(struct io_uring_cmd *cmd, int issue_flags, + struct fuse_ring_ent *ring_ent) +{ + uring_cmd_set_ring_ent(cmd, ring_ent); + io_uring_cmd_mark_cancelable(cmd, issue_flags); +} + +/* + * Checks for errors and stores it into the request + */ +static int fuse_uring_out_header_has_err(struct fuse_out_header *oh, + struct fuse_req *req) +{ + int err; + + err = -EINVAL; + if (oh->unique == 0) { + /* Not supported through io-uring yet */ + pr_warn_once("notify through fuse-io-uring not supported\n"); + goto err; + } + + if (oh->error <= -ERESTARTSYS || oh->error > 0) + goto err; + + if (oh->error) { + err = oh->error; + goto err; + } + + err = -ENOENT; + if ((oh->unique & ~FUSE_INT_REQ_BIT) != req->in.h.unique) { + pr_warn_ratelimited("unique mismatch, expected: %llu got %llu\n", + req->in.h.unique, + oh->unique & ~FUSE_INT_REQ_BIT); + goto err; + } + + /* + * Is it an interrupt reply ID? + * XXX: Not supported through fuse-io-uring yet, it should not even + * find the request - should not happen. + */ + WARN_ON_ONCE(oh->unique & FUSE_INT_REQ_BIT); + + err = 0; +err: + return err; +} + +static int ring_header_type_offset(enum fuse_uring_header_type type) +{ + switch (type) { + case FUSE_URING_HEADER_IN_OUT: + return 0; + case FUSE_URING_HEADER_OP: + return offsetof(struct fuse_uring_req_header, op_in); + case FUSE_URING_HEADER_RING_ENT: + return offsetof(struct fuse_uring_req_header, ring_ent_in_out); + default: + WARN_ONCE(1, "Invalid header type: %d\n", type); + return -EINVAL; + } +} + +static int copy_header_to_ring(struct fuse_ring_ent *ent, + enum fuse_uring_header_type type, + const void *header, size_t header_size) +{ + int offset = ring_header_type_offset(type); + void __user *ring; + + if (offset < 0) + return offset; + + ring = (void __user *)ent->headers + offset; + + if (copy_to_user(ring, header, header_size)) { + pr_info_ratelimited("Copying header to ring failed.\n"); + return -EFAULT; + } + + return 0; +} + +static int copy_header_from_ring(struct fuse_ring_ent *ent, + enum fuse_uring_header_type type, void *header, + size_t header_size) +{ + int offset = ring_header_type_offset(type); + const void __user *ring; + + if (offset < 0) + return offset; + + ring = (void __user *)ent->headers + offset; + + if (copy_from_user(header, ring, header_size)) { + pr_info_ratelimited("Copying header from ring failed.\n"); + return -EFAULT; + } + + return 0; +} + +static int setup_fuse_copy_state(struct fuse_copy_state *cs, + struct fuse_ring *ring, struct fuse_req *req, + struct fuse_ring_ent *ent, int dir, + struct iov_iter *iter) +{ + int err; + + err = import_ubuf(dir, ent->payload, ring->max_payload_sz, iter); + if (err) { + pr_info_ratelimited("fuse: Import of user buffer failed\n"); + return err; + } + + fuse_copy_init(cs, dir == ITER_DEST, iter); + + cs->is_uring = true; + cs->req = req; + + return 0; +} + +static int fuse_uring_copy_from_ring(struct fuse_ring *ring, + struct fuse_req *req, + struct fuse_ring_ent *ent) +{ + struct fuse_copy_state cs; + struct fuse_args *args = req->args; + struct iov_iter iter; + int err; + struct fuse_uring_ent_in_out ring_in_out; + + err = copy_header_from_ring(ent, FUSE_URING_HEADER_RING_ENT, + &ring_in_out, sizeof(ring_in_out)); + if (err) + return err; + + err = setup_fuse_copy_state(&cs, ring, req, ent, ITER_SOURCE, &iter); + if (err) + return err; + + err = fuse_copy_out_args(&cs, args, ring_in_out.payload_sz); + fuse_copy_finish(&cs); + return err; +} + +/* + * Copy data from the req to the ring buffer + */ +static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, + struct fuse_ring_ent *ent) +{ + struct fuse_copy_state cs; + struct fuse_args *args = req->args; + struct fuse_in_arg *in_args = args->in_args; + int num_args = args->in_numargs; + int err; + struct iov_iter iter; + struct fuse_uring_ent_in_out ent_in_out = { + .flags = 0, + .commit_id = req->in.h.unique, + }; + + err = setup_fuse_copy_state(&cs, ring, req, ent, ITER_DEST, &iter); + if (err) + return err; + + if (num_args > 0) { + /* + * Expectation is that the first argument is the per op header. + * Some op code have that as zero size. + */ + if (args->in_args[0].size > 0) { + err = copy_header_to_ring(ent, FUSE_URING_HEADER_OP, + in_args->value, + in_args->size); + if (err) + return err; + } + in_args++; + num_args--; + } + + /* copy the payload */ + err = fuse_copy_args(&cs, num_args, args->in_pages, + (struct fuse_arg *)in_args, 0); + fuse_copy_finish(&cs); + if (err) { + pr_info_ratelimited("%s fuse_copy_args failed\n", __func__); + return err; + } + + ent_in_out.payload_sz = cs.ring.copied_sz; + return copy_header_to_ring(ent, FUSE_URING_HEADER_RING_ENT, + &ent_in_out, sizeof(ent_in_out)); +} + +static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, + struct fuse_req *req) +{ + struct fuse_ring_queue *queue = ent->queue; + struct fuse_ring *ring = queue->ring; + int err; + + err = -EIO; + if (WARN_ON(ent->state != FRRS_FUSE_REQ)) { + pr_err("qid=%d ring-req=%p invalid state %d on send\n", + queue->qid, ent, ent->state); + return err; + } + + err = -EINVAL; + if (WARN_ON(req->in.h.unique == 0)) + return err; + + /* copy the request */ + err = fuse_uring_args_to_ring(ring, req, ent); + if (unlikely(err)) { + pr_info_ratelimited("Copy to ring failed: %d\n", err); + return err; + } + + /* copy fuse_in_header */ + return copy_header_to_ring(ent, FUSE_URING_HEADER_IN_OUT, &req->in.h, + sizeof(req->in.h)); +} + +static int fuse_uring_prepare_send(struct fuse_ring_ent *ent, + struct fuse_req *req) +{ + int err; + + err = fuse_uring_copy_to_ring(ent, req); + if (!err) { + set_bit(FR_SENT, &req->flags); + trace_fuse_request_sent(req); + } else { + /* + * Copying the request failed. Remove the entry from the + * ent_w_req_queue list and terminate the request + */ + spin_lock(&ent->queue->lock); + list_del_init(&ent->list); + ent->state = FRRS_INVALID; + spin_unlock(&ent->queue->lock); + + fuse_uring_req_end(ent, req, err); + } + + return err; +} + +/* Used to find the request on SQE commit */ +static void fuse_uring_add_to_pq(struct fuse_ring_ent *ent) +{ + struct fuse_ring_queue *queue = ent->queue; + struct fuse_pqueue *fpq = &queue->fpq; + unsigned int hash; + struct fuse_req *req = ent->fuse_req; + + req->ring_entry = ent; + hash = fuse_req_hash(req->in.h.unique); + list_move_tail(&req->list, &fpq->processing[hash]); +} + +/* + * Make a ring entry available for fuse_req assignment + */ +static void fuse_uring_ent_avail(struct fuse_ring_ent *ent, + struct fuse_ring_queue *queue) +{ + WARN_ON_ONCE(!ent->cmd); + list_move(&ent->list, &queue->ent_avail_queue); + ent->state = FRRS_AVAILABLE; +} + +/* + * Assign a fuse queue entry to the given entry + */ +static void fuse_uring_add_req_to_ring_ent(struct fuse_ring_ent *ent, + struct fuse_req *req) +{ + struct fuse_ring_queue *queue = ent->queue; + + lockdep_assert_held(&queue->lock); + + if (WARN_ON_ONCE(ent->state != FRRS_AVAILABLE && + ent->state != FRRS_COMMIT)) { + pr_warn("%s qid=%d state=%d\n", __func__, ent->queue->qid, + ent->state); + } + + clear_bit(FR_PENDING, &req->flags); + + /* Until fuse_uring_add_to_pq() the req is not attached to any list */ + list_del_init(&req->list); + + ent->fuse_req = req; + ent->state = FRRS_FUSE_REQ; + list_move_tail(&ent->list, &queue->ent_w_req_queue); +} + +/* Fetch the next fuse request if available */ +static struct fuse_req *fuse_uring_ent_assign_req(struct fuse_ring_ent *ent) + __must_hold(&queue->lock) +{ + struct fuse_req *req; + struct fuse_ring_queue *queue = ent->queue; + struct list_head *req_queue = &queue->fuse_req_queue; + + lockdep_assert_held(&queue->lock); + + /* get and assign the next entry while it is still holding the lock */ + req = list_first_entry_or_null(req_queue, struct fuse_req, list); + if (req) + fuse_uring_add_req_to_ring_ent(ent, req); + + return req; +} + +/* + * Read data from the ring buffer, which user space has written to + * This is comparible with handling of classical write(/dev/fuse). + * Also make the ring request available again for new fuse requests. + */ +static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, + unsigned int issue_flags) +{ + struct fuse_ring *ring = ent->queue->ring; + ssize_t err = -EFAULT; + + if (copy_header_from_ring(ent, FUSE_URING_HEADER_IN_OUT, &req->out.h, + sizeof(req->out.h))) + goto out; + + err = fuse_uring_out_header_has_err(&req->out.h, req); + if (err) { + /* req->out.h.error already set */ + goto out; + } + + err = fuse_uring_copy_from_ring(ring, req, ent); +out: + fuse_uring_req_end(ent, req, err); +} + +/* + * Get the next fuse req. + * + * Returns true if the next fuse request has been assigned to the ent. + * Else, there is no next fuse request and this returns false. + */ +static bool fuse_uring_get_next_fuse_req(struct fuse_ring_ent *ent, + struct fuse_ring_queue *queue) +{ + int err; + struct fuse_req *req; + +retry: + spin_lock(&queue->lock); + fuse_uring_ent_avail(ent, queue); + req = fuse_uring_ent_assign_req(ent); + spin_unlock(&queue->lock); + + if (req) { + err = fuse_uring_prepare_send(ent, req); + if (err) + goto retry; + } + + return req != NULL; +} + +static int fuse_ring_ent_set_commit(struct fuse_ring_ent *ent) +{ + struct fuse_ring_queue *queue = ent->queue; + + lockdep_assert_held(&queue->lock); + + if (WARN_ON_ONCE(ent->state != FRRS_USERSPACE)) + return -EIO; + + ent->state = FRRS_COMMIT; + list_move(&ent->list, &queue->ent_commit_queue); + + return 0; +} + +static void fuse_uring_send(struct fuse_ring_ent *ent, struct io_uring_cmd *cmd, + ssize_t ret, unsigned int issue_flags) +{ + struct fuse_ring_queue *queue = ent->queue; + + spin_lock(&queue->lock); + ent->state = FRRS_USERSPACE; + list_move_tail(&ent->list, &queue->ent_in_userspace); + ent->cmd = NULL; + fuse_uring_add_to_pq(ent); + spin_unlock(&queue->lock); + + io_uring_cmd_done(cmd, ret, issue_flags); +} + +/* FUSE_URING_CMD_COMMIT_AND_FETCH handler */ +static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, + struct fuse_chan *fch) +{ + const struct fuse_uring_cmd_req *cmd_req = io_uring_sqe128_cmd(cmd->sqe, + struct fuse_uring_cmd_req); + struct fuse_ring_ent *ent; + int err; + struct fuse_ring *ring = fch->ring; + struct fuse_ring_queue *queue; + uint64_t commit_id = READ_ONCE(cmd_req->commit_id); + unsigned int qid = READ_ONCE(cmd_req->qid); + struct fuse_pqueue *fpq; + struct fuse_req *req; + + err = -ENOTCONN; + if (!ring) + return err; + + if (qid >= ring->nr_queues) + return -EINVAL; + + queue = ring->queues[qid]; + if (!queue) + return err; + fpq = &queue->fpq; + + if (!READ_ONCE(fch->connected)) + return err; + + spin_lock(&queue->lock); + if (unlikely(queue->stopped)) { + spin_unlock(&queue->lock); + return err; + } + + /* Find a request based on the unique ID of the fuse request + * This should get revised, as it needs a hash calculation and list + * search. And full struct fuse_pqueue is needed (memory overhead). + * As well as the link from req to ring_ent. + */ + req = fuse_request_find(fpq, commit_id); + err = -ENOENT; + if (!req) { + pr_info("qid=%d commit_id %llu not found\n", queue->qid, + commit_id); + spin_unlock(&queue->lock); + return err; + } + list_del_init(&req->list); + ent = req->ring_entry; + req->ring_entry = NULL; + + err = fuse_ring_ent_set_commit(ent); + if (err != 0) { + pr_info_ratelimited("qid=%d commit_id %llu state %d", + queue->qid, commit_id, ent->state); + spin_unlock(&queue->lock); + fuse_uring_req_end(ent, req, err); + return err; + } + + ent->cmd = cmd; + spin_unlock(&queue->lock); + + /* without the queue lock, as other locks are taken */ + fuse_uring_prepare_cancel(cmd, issue_flags, ent); + fuse_uring_commit(ent, req, issue_flags); + + /* + * Fetching the next request is absolutely required as queued + * fuse requests would otherwise not get processed - committing + * and fetching is done in one step vs legacy fuse, which has separated + * read (fetch request) and write (commit result). + */ + if (fuse_uring_get_next_fuse_req(ent, queue)) + fuse_uring_send(ent, cmd, 0, issue_flags); + return 0; +} + +static bool is_ring_ready(struct fuse_ring *ring, int current_qid) +{ + int qid; + struct fuse_ring_queue *queue; + bool ready = true; + + for (qid = 0; qid < ring->nr_queues && ready; qid++) { + if (current_qid == qid) + continue; + + queue = ring->queues[qid]; + if (!queue) { + ready = false; + break; + } + + spin_lock(&queue->lock); + if (list_empty(&queue->ent_avail_queue)) + ready = false; + spin_unlock(&queue->lock); + } + + return ready; +} + +/* + * fuse_uring_req_fetch command handling + */ +static int fuse_uring_do_register(struct fuse_ring_ent *ent, + struct io_uring_cmd *cmd, + unsigned int issue_flags) +{ + struct fuse_ring_queue *queue = ent->queue; + struct fuse_ring *ring = queue->ring; + struct fuse_chan *fch = ring->chan; + struct fuse_iqueue *fiq = &fch->iq; + + spin_lock(&fch->lock); + /* abort teardown path is running or has run */ + if (!fch->connected) { + spin_unlock(&fch->lock); + if (atomic_dec_and_test(&ring->queue_refs)) + wake_up_all(&ring->stop_waitq); + kfree(ent); + return -ECONNABORTED; + } + spin_unlock(&fch->lock); + + fuse_uring_prepare_cancel(cmd, issue_flags, ent); + + spin_lock(&queue->lock); + ent->cmd = cmd; + fuse_uring_ent_avail(ent, queue); + spin_unlock(&queue->lock); + + if (!READ_ONCE(ring->ready)) { + bool ready = is_ring_ready(ring, queue->qid); + + if (ready) { + WRITE_ONCE(fiq->ops, &fuse_io_uring_ops); + smp_store_release(&ring->ready, true); + wake_up_all(&fch->blocked_waitq); + } + } + return 0; +} + +/* + * sqe->addr is a ptr to an iovec array, iov[FUSE_URING_IOV_HEADERS] has the + * headers, iov[FUSE_URING_IOV_PAYLOAD] the payload + */ +static int fuse_uring_get_iovec_from_sqe(const struct io_uring_sqe *sqe, + struct iovec iov[FUSE_URING_IOV_SEGS]) +{ + struct iovec __user *uiov = u64_to_user_ptr(READ_ONCE(sqe->addr)); + struct iov_iter iter; + ssize_t ret; + + if (sqe->len != FUSE_URING_IOV_SEGS) + return -EINVAL; + + /* + * Direction for buffer access will actually be READ and WRITE, + * using write for the import should include READ access as well. + */ + ret = import_iovec(WRITE, uiov, FUSE_URING_IOV_SEGS, + FUSE_URING_IOV_SEGS, &iov, &iter); + if (ret < 0) + return ret; + + return 0; +} + +static struct fuse_ring_ent * +fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, + struct fuse_ring_queue *queue) +{ + struct fuse_ring *ring = queue->ring; + struct fuse_ring_ent *ent; + struct iovec iov[FUSE_URING_IOV_SEGS]; + struct iovec *headers, *payload; + int err; + + err = fuse_uring_get_iovec_from_sqe(cmd->sqe, iov); + if (err) { + pr_info_ratelimited("Failed to get iovec from sqe, err=%d\n", + err); + return ERR_PTR(err); + } + + err = -EINVAL; + headers = &iov[FUSE_URING_IOV_HEADERS]; + if (headers->iov_len < sizeof(struct fuse_uring_req_header)) { + pr_info_ratelimited("Invalid header len %zu\n", headers->iov_len); + return ERR_PTR(err); + } + + payload = &iov[FUSE_URING_IOV_PAYLOAD]; + if (payload->iov_len < ring->max_payload_sz) { + pr_info_ratelimited("Invalid req payload len %zu\n", + payload->iov_len); + return ERR_PTR(err); + } + + err = -ENOMEM; + ent = kzalloc_obj(*ent, GFP_KERNEL_ACCOUNT); + if (!ent) + return ERR_PTR(err); + + INIT_LIST_HEAD(&ent->list); + + ent->queue = queue; + ent->headers = headers->iov_base; + ent->payload = payload->iov_base; + + atomic_inc(&ring->queue_refs); + return ent; +} + +/* + * Register header and payload buffer with the kernel and puts the + * entry as "ready to get fuse requests" on the queue + */ +static int fuse_uring_register(struct io_uring_cmd *cmd, + unsigned int issue_flags, struct fuse_chan *fch) +{ + const struct fuse_uring_cmd_req *cmd_req = io_uring_sqe128_cmd(cmd->sqe, + struct fuse_uring_cmd_req); + struct fuse_ring *ring = smp_load_acquire(&fch->ring); + struct fuse_ring_queue *queue; + struct fuse_ring_ent *ent; + int err; + unsigned int qid = READ_ONCE(cmd_req->qid); + + err = -ENOMEM; + if (!ring) { + ring = fuse_uring_create(fch); + if (!ring) + return err; + } + + if (qid >= ring->nr_queues) { + pr_info_ratelimited("fuse: Invalid ring qid %u\n", qid); + return -EINVAL; + } + + queue = ring->queues[qid]; + if (!queue) { + queue = fuse_uring_create_queue(ring, qid); + if (!queue) + return err; + } + + /* + * The created queue above does not need to be destructed in + * case of entry errors below, will be done at ring destruction time. + */ + + ent = fuse_uring_create_ring_ent(cmd, queue); + if (IS_ERR(ent)) + return PTR_ERR(ent); + + return fuse_uring_do_register(ent, cmd, issue_flags); +} + +/* + * Entry function from io_uring to handle the given passthrough command + * (op code IORING_OP_URING_CMD) + */ +int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) +{ + struct fuse_dev *fud; + struct fuse_chan *fch; + u32 cmd_op = cmd->cmd_op; + int err; + + if ((unlikely(issue_flags & IO_URING_F_CANCEL))) { + fuse_uring_cancel(cmd, issue_flags); + return 0; + } + + /* This extra SQE size holds struct fuse_uring_cmd_req */ + if (!(issue_flags & IO_URING_F_SQE128)) + return -EINVAL; + + fud = fuse_get_dev(cmd->file); + if (IS_ERR(fud)) { + pr_info_ratelimited("No fuse device found\n"); + return PTR_ERR(fud); + } + fch = fud->chan; + + /* Once a connection has io-uring enabled on it, it can't be disabled */ + if (!enable_uring && !fch->io_uring) { + pr_info_ratelimited("fuse-io-uring is disabled\n"); + return -EOPNOTSUPP; + } + + if (fch->abort_with_err) + return -ECONNABORTED; + if (!fch->connected) + return -ENOTCONN; + + /* + * fuse_uring_register() needs the ring to be initialized, + * we need to know the max payload size + */ + if (!fch->initialized) + return -EAGAIN; + + switch (cmd_op) { + case FUSE_IO_URING_CMD_REGISTER: + err = fuse_uring_register(cmd, issue_flags, fch); + if (err) { + pr_info_once("FUSE_IO_URING_CMD_REGISTER failed err=%d\n", + err); + fch->io_uring = 0; + wake_up_all(&fch->blocked_waitq); + return err; + } + break; + case FUSE_IO_URING_CMD_COMMIT_AND_FETCH: + err = fuse_uring_commit_fetch(cmd, issue_flags, fch); + if (err) { + pr_info_once("FUSE_IO_URING_COMMIT_AND_FETCH failed err=%d\n", + err); + return err; + } + break; + default: + return -EINVAL; + } + + return -EIOCBQUEUED; +} + +/* + * This prepares and sends the ring request in fuse-uring task context. + * User buffers are not mapped yet - the application does not have permission + * to write to it - this has to be executed in ring task context. + */ +static void fuse_uring_send_in_task(struct io_tw_req tw_req, io_tw_token_t tw) +{ + unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS; + struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req); + struct fuse_ring_ent *ent = uring_cmd_to_ring_ent(cmd); + struct fuse_ring_queue *queue = ent->queue; + int err; + + if (!tw.cancel) { + err = fuse_uring_prepare_send(ent, ent->fuse_req); + if (err) { + if (!fuse_uring_get_next_fuse_req(ent, queue)) + return; + err = 0; + } + fuse_uring_send(ent, cmd, err, issue_flags); + } else { + err = -ECANCELED; + + spin_lock(&queue->lock); + list_del_init(&ent->list); + spin_unlock(&queue->lock); + + io_uring_cmd_done(cmd, err, issue_flags); + + fuse_uring_req_end(ent, ent->fuse_req, err); + kfree(ent); + if (atomic_dec_and_test(&queue->ring->queue_refs)) + wake_up_all(&queue->ring->stop_waitq); + } +} + +static struct fuse_ring_queue *fuse_uring_task_to_queue(struct fuse_ring *ring) +{ + unsigned int qid; + struct fuse_ring_queue *queue; + + qid = task_cpu(current); + + if (WARN_ONCE(qid >= ring->nr_queues, + "Core number (%u) exceeds nr queues (%zu)\n", qid, + ring->nr_queues)) + qid = 0; + + queue = ring->queues[qid]; + WARN_ONCE(!queue, "Missing queue for qid %d\n", qid); + + return queue; +} + +static void fuse_uring_dispatch_ent(struct fuse_ring_ent *ent) +{ + struct io_uring_cmd *cmd = ent->cmd; + + uring_cmd_set_ring_ent(cmd, ent); + io_uring_cmd_complete_in_task(cmd, fuse_uring_send_in_task); +} + +/* queue a fuse request and send it if a ring entry is available */ +void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req) +{ + struct fuse_ring *ring = req->chan->ring; + struct fuse_ring_queue *queue; + struct fuse_ring_ent *ent = NULL; + int err; + + err = -EINVAL; + queue = fuse_uring_task_to_queue(ring); + if (!queue) + goto err; + + fuse_request_assign_unique(fiq, req); + + spin_lock(&queue->lock); + err = -ENOTCONN; + if (unlikely(queue->stopped)) + goto err_unlock; + + set_bit(FR_URING, &req->flags); + req->ring_queue = queue; + ent = list_first_entry_or_null(&queue->ent_avail_queue, + struct fuse_ring_ent, list); + if (ent) + fuse_uring_add_req_to_ring_ent(ent, req); + else + list_add_tail(&req->list, &queue->fuse_req_queue); + spin_unlock(&queue->lock); + + if (ent) + fuse_uring_dispatch_ent(ent); + + return; + +err_unlock: + spin_unlock(&queue->lock); +err: + req->out.h.error = err; + clear_bit(FR_PENDING, &req->flags); + fuse_request_end(req); +} + +bool fuse_uring_queue_bq_req(struct fuse_req *req) +{ + struct fuse_chan *fch = req->chan; + struct fuse_ring *ring = fch->ring; + struct fuse_ring_queue *queue; + struct fuse_ring_ent *ent = NULL; + + queue = fuse_uring_task_to_queue(ring); + if (!queue) + return false; + + spin_lock(&queue->lock); + if (unlikely(queue->stopped)) { + spin_unlock(&queue->lock); + return false; + } + + set_bit(FR_URING, &req->flags); + req->ring_queue = queue; + list_add_tail(&req->list, &queue->fuse_req_bg_queue); + + ent = list_first_entry_or_null(&queue->ent_avail_queue, + struct fuse_ring_ent, list); + spin_lock(&fch->bg_lock); + fch->num_background++; + if (fch->num_background == fch->max_background) + fch->blocked = 1; + fuse_uring_flush_bg(queue); + spin_unlock(&fch->bg_lock); + + /* + * Due to bg_queue flush limits there might be other bg requests + * in the queue that need to be handled first. Or no further req + * might be available. + */ + req = list_first_entry_or_null(&queue->fuse_req_queue, struct fuse_req, + list); + if (ent && req) { + fuse_uring_add_req_to_ring_ent(ent, req); + spin_unlock(&queue->lock); + + fuse_uring_dispatch_ent(ent); + } else { + spin_unlock(&queue->lock); + } + + return true; +} + +bool fuse_uring_remove_pending_req(struct fuse_req *req) +{ + struct fuse_ring_queue *queue = req->ring_queue; + + return fuse_remove_pending_req(req, &queue->lock); +} + +static const struct fuse_iqueue_ops fuse_io_uring_ops = { + /* should be send over io-uring as enhancement */ + .send_forget = fuse_dev_queue_forget, + + /* + * could be send over io-uring, but interrupts should be rare, + * no need to make the code complex + */ + .send_interrupt = fuse_dev_queue_interrupt, + .send_req = fuse_uring_queue_fuse_req, +}; diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h new file mode 100644 index 000000000..55f8d04e4 --- /dev/null +++ b/fs/fuse/dev_uring_i.h @@ -0,0 +1,202 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * FUSE: Filesystem in Userspace + * Copyright (c) 2023-2024 DataDirect Networks. + */ + +#ifndef _FS_FUSE_DEV_URING_I_H +#define _FS_FUSE_DEV_URING_I_H + +#include "fuse_dev_i.h" + +#ifdef CONFIG_FUSE_IO_URING + +#define FUSE_URING_TEARDOWN_TIMEOUT (5 * HZ) +#define FUSE_URING_TEARDOWN_INTERVAL (HZ/20) + +enum fuse_ring_req_state { + FRRS_INVALID = 0, + + /* The ring entry received from userspace and it is being processed */ + FRRS_COMMIT, + + /* The ring entry is waiting for new fuse requests */ + FRRS_AVAILABLE, + + /* The ring entry got assigned a fuse req */ + FRRS_FUSE_REQ, + + /* The ring entry is in or on the way to user space */ + FRRS_USERSPACE, + + /* The ring entry is in teardown */ + FRRS_TEARDOWN, + + /* The ring entry is released, but not freed yet */ + FRRS_RELEASED, +}; + +/** A fuse ring entry, part of the ring queue */ +struct fuse_ring_ent { + /* userspace buffer */ + struct fuse_uring_req_header __user *headers; + void __user *payload; + + /* the ring queue that owns the request */ + struct fuse_ring_queue *queue; + + /* fields below are protected by queue->lock */ + + struct io_uring_cmd *cmd; + + struct list_head list; + + enum fuse_ring_req_state state; + + struct fuse_req *fuse_req; +}; + +struct fuse_ring_queue { + /* + * back pointer to the main fuse uring structure that holds this + * queue + */ + struct fuse_ring *ring; + + /* queue id, corresponds to the cpu core */ + unsigned int qid; + + /* + * queue lock, taken when any value in the queue changes _and_ also + * a ring entry state changes. + */ + spinlock_t lock; + + /* available ring entries (struct fuse_ring_ent) */ + struct list_head ent_avail_queue; + + /* + * entries in the process of being committed or in the process + * to be sent to userspace + */ + struct list_head ent_w_req_queue; + struct list_head ent_commit_queue; + + /* entries in userspace */ + struct list_head ent_in_userspace; + + /* entries that are released */ + struct list_head ent_released; + + /* fuse requests waiting for an entry slot */ + struct list_head fuse_req_queue; + + /* background fuse requests */ + struct list_head fuse_req_bg_queue; + + struct fuse_pqueue fpq; + + unsigned int active_background; + + bool stopped; +}; + +/* + * Describes if uring is for communication and holds alls the data needed + * for uring communication + */ +struct fuse_ring { + /* back pointer */ + struct fuse_chan *chan; + + /* number of ring queues */ + size_t nr_queues; + + /* maximum payload/arg size */ + size_t max_payload_sz; + + struct fuse_ring_queue **queues; + + /* + * Log ring entry states on stop when entries cannot be released + */ + unsigned int stop_debug_log : 1; + + wait_queue_head_t stop_waitq; + + /* async tear down */ + struct delayed_work async_teardown_work; + + /* log */ + unsigned long teardown_time; + + atomic_t queue_refs; + + bool ready; +}; + +void fuse_uring_stop_queues(struct fuse_ring *ring); +void fuse_uring_abort_end_requests(struct fuse_ring *ring); +int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags); +void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req); +bool fuse_uring_queue_bq_req(struct fuse_req *req); +bool fuse_uring_remove_pending_req(struct fuse_req *req); +bool fuse_uring_request_expired(struct fuse_chan *fch); + +static inline void fuse_uring_abort(struct fuse_chan *fch) +{ + struct fuse_ring *ring = fch->ring; + + if (ring == NULL) + return; + + fuse_uring_abort_end_requests(ring); + + if (atomic_read(&ring->queue_refs) > 0) + fuse_uring_stop_queues(ring); +} + +static inline void fuse_uring_wait_stopped_queues(struct fuse_chan *fch) +{ + struct fuse_ring *ring = fch->ring; + + if (ring) + wait_event(ring->stop_waitq, + atomic_read(&ring->queue_refs) == 0); +} + +static inline bool fuse_uring_ready(struct fuse_chan *fch) +{ + struct fuse_ring *ring = READ_ONCE(fch->ring); + + return ring && smp_load_acquire(&ring->ready); +} + +#else /* CONFIG_FUSE_IO_URING */ + +static inline void fuse_uring_abort(struct fuse_chan *fch) +{ +} + +static inline void fuse_uring_wait_stopped_queues(struct fuse_chan *fch) +{ +} + +static inline bool fuse_uring_ready(struct fuse_chan *fch) +{ + return false; +} + +static inline bool fuse_uring_remove_pending_req(struct fuse_req *req) +{ + return false; +} + +static inline bool fuse_uring_request_expired(struct fuse_chan *fch) +{ + return false; +} + +#endif /* CONFIG_FUSE_IO_URING */ + +#endif /* _FS_FUSE_DEV_URING_I_H */ diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c new file mode 100644 index 000000000..d4e002981 --- /dev/null +++ b/fs/fuse/dir.c @@ -0,0 +1,2497 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + FUSE: Filesystem in Userspace + Copyright (C) 2001-2008 Miklos Szeredi +*/ + +#include "dev.h" +#include "fuse_i.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static bool __read_mostly allow_sys_admin_access; +module_param(allow_sys_admin_access, bool, 0644); +MODULE_PARM_DESC(allow_sys_admin_access, + "Allow users with CAP_SYS_ADMIN in initial userns to bypass allow_other access check"); + +struct dentry_bucket { + struct rb_root tree; + spinlock_t lock; +}; + +#define FUSE_HASH_BITS 5 +#define FUSE_HASH_SIZE (1 << FUSE_HASH_BITS) +static struct dentry_bucket dentry_hash[FUSE_HASH_SIZE]; +struct delayed_work dentry_tree_work; + +/* Minimum invalidation work queue frequency */ +#define FUSE_DENTRY_INVAL_FREQ_MIN 5 + +unsigned __read_mostly inval_wq; +static int inval_wq_set(const char *val, const struct kernel_param *kp) +{ + unsigned int num; + unsigned int old = inval_wq; + int ret; + + if (!val) + return -EINVAL; + + ret = kstrtouint(val, 0, &num); + if (ret) + return ret; + + if ((num < FUSE_DENTRY_INVAL_FREQ_MIN) && (num != 0)) + return -EINVAL; + + /* This should prevent overflow in secs_to_jiffies() */ + if (num > USHRT_MAX) + return -EINVAL; + + *((unsigned int *)kp->arg) = num; + + if (num && !old) + schedule_delayed_work(&dentry_tree_work, + secs_to_jiffies(num)); + else if (!num && old) + cancel_delayed_work_sync(&dentry_tree_work); + + return 0; +} +static const struct kernel_param_ops inval_wq_ops = { + .set = inval_wq_set, + .get = param_get_uint, +}; +module_param_cb(inval_wq, &inval_wq_ops, &inval_wq, 0644); +__MODULE_PARM_TYPE(inval_wq, "uint"); +MODULE_PARM_DESC(inval_wq, + "Dentries invalidation work queue period in secs (>= " + __stringify(FUSE_DENTRY_INVAL_FREQ_MIN) ")."); + +static inline struct dentry_bucket *get_dentry_bucket(struct dentry *dentry) +{ + int i = hash_ptr(dentry, FUSE_HASH_BITS); + + return &dentry_hash[i]; +} + +static void fuse_advise_use_readdirplus(struct inode *dir) +{ + struct fuse_inode *fi = get_fuse_inode(dir); + + set_bit(FUSE_I_ADVISE_RDPLUS, &fi->state); +} + +struct fuse_dentry { + u64 time; + union { + struct rcu_head rcu; + struct rb_node node; + }; + struct dentry *dentry; +}; + +static void __fuse_dentry_tree_del_node(struct fuse_dentry *fd, + struct dentry_bucket *bucket) +{ + if (!RB_EMPTY_NODE(&fd->node)) { + rb_erase(&fd->node, &bucket->tree); + RB_CLEAR_NODE(&fd->node); + } +} + +static void fuse_dentry_tree_del_node(struct dentry *dentry) +{ + struct fuse_dentry *fd = dentry->d_fsdata; + struct dentry_bucket *bucket = get_dentry_bucket(dentry); + + spin_lock(&bucket->lock); + __fuse_dentry_tree_del_node(fd, bucket); + spin_unlock(&bucket->lock); +} + +static void fuse_dentry_tree_add_node(struct dentry *dentry) +{ + struct fuse_dentry *fd = dentry->d_fsdata; + struct dentry_bucket *bucket; + struct fuse_dentry *cur; + struct rb_node **p, *parent = NULL; + + if (!inval_wq) + return; + + bucket = get_dentry_bucket(dentry); + + spin_lock(&bucket->lock); + + __fuse_dentry_tree_del_node(fd, bucket); + + p = &bucket->tree.rb_node; + while (*p) { + parent = *p; + cur = rb_entry(*p, struct fuse_dentry, node); + if (fd->time < cur->time) + p = &(*p)->rb_left; + else + p = &(*p)->rb_right; + } + rb_link_node(&fd->node, parent, p); + rb_insert_color(&fd->node, &bucket->tree); + spin_unlock(&bucket->lock); +} + +/* + * work queue which, when enabled, will periodically check for expired dentries + * in the dentries tree. + */ +static void fuse_dentry_tree_work(struct work_struct *work) +{ + LIST_HEAD(dispose); + struct fuse_dentry *fd; + struct rb_node *node; + int i; + + for (i = 0; i < FUSE_HASH_SIZE; i++) { + spin_lock(&dentry_hash[i].lock); + node = rb_first(&dentry_hash[i].tree); + while (node) { + fd = rb_entry(node, struct fuse_dentry, node); + if (!time_before64(fd->time, get_jiffies_64())) + break; + + rb_erase(&fd->node, &dentry_hash[i].tree); + RB_CLEAR_NODE(&fd->node); + spin_lock(&fd->dentry->d_lock); + /* If dentry is still referenced, let next dput release it */ + fd->dentry->d_flags |= DCACHE_OP_DELETE; + __move_to_shrink_list(fd->dentry, &dispose); + spin_unlock(&fd->dentry->d_lock); + if (need_resched()) { + spin_unlock(&dentry_hash[i].lock); + cond_resched(); + spin_lock(&dentry_hash[i].lock); + } + node = rb_first(&dentry_hash[i].tree); + } + spin_unlock(&dentry_hash[i].lock); + } + shrink_dentry_list(&dispose); + + if (inval_wq) + schedule_delayed_work(&dentry_tree_work, + secs_to_jiffies(inval_wq)); +} + +void fuse_epoch_work(struct work_struct *work) +{ + struct fuse_conn *fc = container_of(work, struct fuse_conn, + epoch_work); + struct fuse_mount *fm; + struct inode *inode; + + down_read(&fc->killsb); + + inode = fuse_ilookup(fc, FUSE_ROOT_ID, &fm); + if (inode) { + iput(inode); + /* Remove all possible active references to cached inodes */ + shrink_dcache_sb(fm->sb); + } else + pr_warn("Failed to get root inode"); + + up_read(&fc->killsb); +} + +void fuse_dentry_tree_init(void) +{ + int i; + + for (i = 0; i < FUSE_HASH_SIZE; i++) { + spin_lock_init(&dentry_hash[i].lock); + dentry_hash[i].tree = RB_ROOT; + } + INIT_DELAYED_WORK(&dentry_tree_work, fuse_dentry_tree_work); +} + +void fuse_dentry_tree_cleanup(void) +{ + int i; + + inval_wq = 0; + cancel_delayed_work_sync(&dentry_tree_work); + + for (i = 0; i < FUSE_HASH_SIZE; i++) + WARN_ON_ONCE(!RB_EMPTY_ROOT(&dentry_hash[i].tree)); +} + +static inline void __fuse_dentry_settime(struct dentry *dentry, u64 time) +{ + ((struct fuse_dentry *) dentry->d_fsdata)->time = time; +} + +static inline u64 fuse_dentry_time(const struct dentry *entry) +{ + return ((struct fuse_dentry *) entry->d_fsdata)->time; +} + +static void fuse_dentry_settime(struct dentry *dentry, u64 time) +{ + struct fuse_conn *fc = get_fuse_conn_super(dentry->d_sb); + bool delete = !time && fc->delete_stale; + /* + * Mess with DCACHE_OP_DELETE because dput() will be faster without it. + * Don't care about races, either way it's just an optimization + */ + if ((!delete && (dentry->d_flags & DCACHE_OP_DELETE)) || + (delete && !(dentry->d_flags & DCACHE_OP_DELETE))) { + spin_lock(&dentry->d_lock); + if (!delete) + dentry->d_flags &= ~DCACHE_OP_DELETE; + else + dentry->d_flags |= DCACHE_OP_DELETE; + spin_unlock(&dentry->d_lock); + } + + __fuse_dentry_settime(dentry, time); + fuse_dentry_tree_add_node(dentry); +} + +/* + * FUSE caches dentries and attributes with separate timeout. The + * time in jiffies until the dentry/attributes are valid is stored in + * dentry->d_fsdata and fuse_inode->i_time respectively. + */ + +/* + * Calculate the time in jiffies until a dentry/attributes are valid + */ +u64 fuse_time_to_jiffies(u64 sec, u32 nsec) +{ + if (sec || nsec) { + struct timespec64 ts = { + sec, + min_t(u32, nsec, NSEC_PER_SEC - 1) + }; + + return get_jiffies_64() + timespec64_to_jiffies(&ts); + } else + return 0; +} + +/* + * Set dentry and possibly attribute timeouts from the lookup/mk* + * replies + */ +void fuse_change_entry_timeout(struct dentry *entry, struct fuse_entry_out *o) +{ + fuse_dentry_settime(entry, + fuse_time_to_jiffies(o->entry_valid, o->entry_valid_nsec)); +} + +void fuse_invalidate_attr_mask(struct inode *inode, u32 mask) +{ + set_mask_bits(&get_fuse_inode(inode)->inval_mask, 0, mask); +} + +/* + * Mark the attributes as stale, so that at the next call to + * ->getattr() they will be fetched from userspace + */ +void fuse_invalidate_attr(struct inode *inode) +{ + fuse_invalidate_attr_mask(inode, STATX_BASIC_STATS); +} + +static void fuse_dir_changed(struct inode *dir) +{ + fuse_invalidate_attr_mask(dir, FUSE_STATX_MODDIR); + inode_maybe_inc_iversion(dir, false); +} + +/* + * Mark the attributes as stale due to an atime change. Avoid the invalidate if + * atime is not used. + */ +void fuse_invalidate_atime(struct inode *inode) +{ + if (!IS_RDONLY(inode)) + fuse_invalidate_attr_mask(inode, STATX_ATIME); +} + +/* + * Just mark the entry as stale, so that a next attempt to look it up + * will result in a new lookup call to userspace + * + * This is called when a dentry is about to become negative and the + * timeout is unknown (unlink, rmdir, rename and in some cases + * lookup) + */ +void fuse_invalidate_entry_cache(struct dentry *entry) +{ + fuse_dentry_settime(entry, 0); +} + +/* + * Same as fuse_invalidate_entry_cache(), but also try to remove the + * dentry from the hash + */ +static void fuse_invalidate_entry(struct dentry *entry) +{ + d_invalidate(entry); + fuse_invalidate_entry_cache(entry); +} + +static void fuse_lookup_init(struct fuse_args *args, u64 nodeid, + const struct qstr *name, + struct fuse_entry_out *outarg) +{ + memset(outarg, 0, sizeof(struct fuse_entry_out)); + args->opcode = FUSE_LOOKUP; + args->nodeid = nodeid; + args->in_numargs = 3; + fuse_set_zero_arg0(args); + args->in_args[1].size = name->len; + args->in_args[1].value = name->name; + args->in_args[2].size = 1; + args->in_args[2].value = ""; + args->out_numargs = 1; + args->out_args[0].size = sizeof(struct fuse_entry_out); + args->out_args[0].value = outarg; +} + +/* + * Check whether the dentry is still valid + * + * If the entry validity timeout has expired and the dentry is + * positive, try to redo the lookup. If the lookup results in a + * different inode, then let the VFS invalidate the dentry and redo + * the lookup once more. If the lookup results in the same inode, + * then refresh the attributes, timeouts and mark the dentry valid. + */ +static int fuse_dentry_revalidate(struct inode *dir, const struct qstr *name, + struct dentry *entry, unsigned int flags) +{ + struct inode *inode; + struct fuse_mount *fm; + struct fuse_conn *fc; + struct fuse_inode *fi; + int ret; + + fc = get_fuse_conn_super(dir->i_sb); + if (entry->d_time < atomic_read(&fc->epoch)) + goto invalid; + + inode = d_inode_rcu(entry); + if (inode && fuse_is_bad(inode)) + goto invalid; + else if (time_before64(fuse_dentry_time(entry), get_jiffies_64()) || + (flags & (LOOKUP_EXCL | LOOKUP_REVAL | LOOKUP_RENAME_TARGET))) { + struct fuse_entry_out outarg; + FUSE_ARGS(args); + struct fuse_forget_link *forget; + u64 attr_version; + + /* For negative dentries, always do a fresh lookup */ + if (!inode) + goto invalid; + + ret = -ECHILD; + if (flags & LOOKUP_RCU) + goto out; + + fm = get_fuse_mount(inode); + + forget = fuse_alloc_forget(); + ret = -ENOMEM; + if (!forget) + goto out; + + attr_version = fuse_get_attr_version(fm->fc); + + fuse_lookup_init(&args, get_node_id(dir), name, &outarg); + ret = fuse_simple_request(fm, &args); + /* Zero nodeid is same as -ENOENT */ + if (!ret && !outarg.nodeid) + ret = -ENOENT; + if (!ret) { + fi = get_fuse_inode(inode); + if (outarg.nodeid != get_node_id(inode) || + (bool) IS_AUTOMOUNT(inode) != (bool) (outarg.attr.flags & FUSE_ATTR_SUBMOUNT)) { + fuse_chan_queue_forget(fm->fc->chan, forget, + outarg.nodeid, 1); + goto invalid; + } + spin_lock(&fi->lock); + fi->nlookup++; + spin_unlock(&fi->lock); + } + kfree(forget); + if (ret == -ENOMEM || ret == -EINTR) + goto out; + if (ret || fuse_invalid_attr(&outarg.attr) || + fuse_stale_inode(inode, outarg.generation, &outarg.attr)) + goto invalid; + + forget_all_cached_acls(inode); + fuse_change_attributes(inode, &outarg.attr, NULL, + ATTR_TIMEOUT(&outarg), + attr_version); + fuse_change_entry_timeout(entry, &outarg); + } else if (inode) { + fi = get_fuse_inode(inode); + if (flags & LOOKUP_RCU) { + if (test_bit(FUSE_I_INIT_RDPLUS, &fi->state)) + return -ECHILD; + } else if (test_and_clear_bit(FUSE_I_INIT_RDPLUS, &fi->state)) { + fuse_advise_use_readdirplus(dir); + } + } + ret = 1; +out: + return ret; + +invalid: + ret = 0; + goto out; +} + +static int fuse_dentry_init(struct dentry *dentry) +{ + struct fuse_dentry *fd; + + fd = kzalloc_obj(struct fuse_dentry, + GFP_KERNEL_ACCOUNT | __GFP_RECLAIMABLE); + if (!fd) + return -ENOMEM; + + fd->dentry = dentry; + RB_CLEAR_NODE(&fd->node); + dentry->d_fsdata = fd; + /* + * Initialising d_time (epoch) to '0' ensures the dentry is invalid + * if compared to fc->epoch, which is initialized to '1'. + */ + dentry->d_time = 0; + + return 0; +} + +static void fuse_dentry_release(struct dentry *dentry) +{ + struct fuse_dentry *fd = dentry->d_fsdata; + + if (!RB_EMPTY_NODE(&fd->node)) + fuse_dentry_tree_del_node(dentry); + kfree_rcu(fd, rcu); +} + +static int fuse_dentry_delete(const struct dentry *dentry) +{ + return time_before64(fuse_dentry_time(dentry), get_jiffies_64()); +} + +/* + * Create a fuse_mount object with a new superblock (with path->dentry + * as the root), and return that mount so it can be auto-mounted on + * @path. + */ +static struct vfsmount *fuse_dentry_automount(struct path *path) +{ + struct fs_context *fsc; + struct vfsmount *mnt; + struct fuse_inode *mp_fi = get_fuse_inode(d_inode(path->dentry)); + + fsc = fs_context_for_submount(path->mnt->mnt_sb->s_type, path->dentry); + if (IS_ERR(fsc)) + return ERR_CAST(fsc); + + /* Pass the FUSE inode of the mount for fuse_get_tree_submount() */ + fsc->fs_private = mp_fi; + + /* Create the submount */ + mnt = fc_mount(fsc); + put_fs_context(fsc); + return mnt; +} + +const struct dentry_operations fuse_dentry_operations = { + .d_revalidate = fuse_dentry_revalidate, + .d_delete = fuse_dentry_delete, + .d_init = fuse_dentry_init, + .d_release = fuse_dentry_release, + .d_automount = fuse_dentry_automount, +}; + +int fuse_valid_type(int m) +{ + return S_ISREG(m) || S_ISDIR(m) || S_ISLNK(m) || S_ISCHR(m) || + S_ISBLK(m) || S_ISFIFO(m) || S_ISSOCK(m); +} + +static bool fuse_valid_size(u64 size) +{ + return size <= LLONG_MAX; +} + +bool fuse_invalid_attr(struct fuse_attr *attr) +{ + return !fuse_valid_type(attr->mode) || !fuse_valid_size(attr->size); +} + +int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name, + struct fuse_entry_out *outarg, struct inode **inode) +{ + struct fuse_mount *fm = get_fuse_mount_super(sb); + FUSE_ARGS(args); + struct fuse_forget_link *forget; + u64 attr_version, evict_ctr; + int err; + + *inode = NULL; + err = -ENAMETOOLONG; + if (name->len > fm->fc->name_max) + goto out; + + + forget = fuse_alloc_forget(); + err = -ENOMEM; + if (!forget) + goto out; + + attr_version = fuse_get_attr_version(fm->fc); + evict_ctr = fuse_get_evict_ctr(fm->fc); + + fuse_lookup_init(&args, nodeid, name, outarg); + err = fuse_simple_request(fm, &args); + /* Zero nodeid is same as -ENOENT, but with valid timeout */ + if (err || !outarg->nodeid) + goto out_put_forget; + + err = -EIO; + if (fuse_invalid_attr(&outarg->attr)) + goto out_put_forget; + if (outarg->nodeid == FUSE_ROOT_ID && outarg->generation != 0) { + pr_warn_once("root generation should be zero\n"); + outarg->generation = 0; + } + + *inode = fuse_iget(sb, outarg->nodeid, outarg->generation, + &outarg->attr, ATTR_TIMEOUT(outarg), + attr_version, evict_ctr); + err = -ENOMEM; + if (!*inode) { + fuse_chan_queue_forget(fm->fc->chan, forget, outarg->nodeid, 1); + goto out; + } + err = 0; + + out_put_forget: + kfree(forget); + out: + return err; +} + +static struct dentry *fuse_lookup(struct inode *dir, struct dentry *entry, + unsigned int flags) +{ + struct fuse_entry_out outarg; + struct fuse_conn *fc; + struct inode *inode; + struct dentry *newent; + int err, epoch; + bool outarg_valid = true; + bool locked; + + if (fuse_is_bad(dir)) + return ERR_PTR(-EIO); + + fc = get_fuse_conn_super(dir->i_sb); + epoch = atomic_read(&fc->epoch); + + locked = fuse_lock_inode(dir); + err = fuse_lookup_name(dir->i_sb, get_node_id(dir), &entry->d_name, + &outarg, &inode); + fuse_unlock_inode(dir, locked); + if (err == -ENOENT) { + outarg_valid = false; + err = 0; + } + if (err) + goto out_err; + + err = -EIO; + if (inode && get_node_id(inode) == FUSE_ROOT_ID) + goto out_iput; + + newent = d_splice_alias(inode, entry); + err = PTR_ERR(newent); + if (IS_ERR(newent)) + goto out_err; + + entry = newent ? newent : entry; + entry->d_time = epoch; + if (outarg_valid) + fuse_change_entry_timeout(entry, &outarg); + else + fuse_invalidate_entry_cache(entry); + + if (inode) + fuse_advise_use_readdirplus(dir); + return newent; + + out_iput: + iput(inode); + out_err: + return ERR_PTR(err); +} + +static int get_security_context(struct dentry *entry, umode_t mode, + struct fuse_in_arg *ext) +{ + struct fuse_secctx *fctx; + struct fuse_secctx_header *header; + struct lsm_context lsmctx = { }; + void *ptr; + u32 total_len = sizeof(*header); + int err, nr_ctx = 0; + const char *name = NULL; + size_t namesize; + + err = security_dentry_init_security(entry, mode, &entry->d_name, + &name, &lsmctx); + + /* If no LSM is supporting this security hook ignore error */ + if (err && err != -EOPNOTSUPP) + goto out_err; + + if (lsmctx.len) { + nr_ctx = 1; + namesize = strlen(name) + 1; + err = -EIO; + if (WARN_ON(namesize > XATTR_NAME_MAX + 1 || + lsmctx.len > S32_MAX)) + goto out_err; + total_len += FUSE_REC_ALIGN(sizeof(*fctx) + namesize + + lsmctx.len); + } + + err = -ENOMEM; + header = ptr = kzalloc(total_len, GFP_KERNEL); + if (!ptr) + goto out_err; + + header->nr_secctx = nr_ctx; + header->size = total_len; + ptr += sizeof(*header); + if (nr_ctx) { + fctx = ptr; + fctx->size = lsmctx.len; + ptr += sizeof(*fctx); + + strscpy(ptr, name, namesize); + ptr += namesize; + + memcpy(ptr, lsmctx.context, lsmctx.len); + } + ext->size = total_len; + ext->value = header; + err = 0; +out_err: + if (nr_ctx) + security_release_secctx(&lsmctx); + return err; +} + +static void *extend_arg(struct fuse_in_arg *buf, u32 bytes) +{ + void *p; + u32 newlen = buf->size + bytes; + + p = krealloc(buf->value, newlen, GFP_KERNEL); + if (!p) { + kfree(buf->value); + buf->size = 0; + buf->value = NULL; + return NULL; + } + + memset(p + buf->size, 0, bytes); + buf->value = p; + buf->size = newlen; + + return p + newlen - bytes; +} + +static u32 fuse_ext_size(size_t size) +{ + return FUSE_REC_ALIGN(sizeof(struct fuse_ext_header) + size); +} + +/* + * This adds just a single supplementary group that matches the parent's group. + */ +static int get_create_supp_group(struct mnt_idmap *idmap, + struct inode *dir, + struct fuse_in_arg *ext) +{ + struct fuse_conn *fc = get_fuse_conn(dir); + struct fuse_ext_header *xh; + struct fuse_supp_groups *sg; + kgid_t kgid = dir->i_gid; + vfsgid_t vfsgid = make_vfsgid(idmap, fc->user_ns, kgid); + gid_t parent_gid = from_kgid(fc->user_ns, kgid); + + u32 sg_len = fuse_ext_size(sizeof(*sg) + sizeof(sg->groups[0])); + + if (parent_gid == (gid_t) -1 || vfsgid_eq_kgid(vfsgid, current_fsgid()) || + !vfsgid_in_group_p(vfsgid)) + return 0; + + xh = extend_arg(ext, sg_len); + if (!xh) + return -ENOMEM; + + xh->size = sg_len; + xh->type = FUSE_EXT_GROUPS; + + sg = (struct fuse_supp_groups *) &xh[1]; + sg->nr_groups = 1; + sg->groups[0] = parent_gid; + + return 0; +} + +static int get_create_ext(struct mnt_idmap *idmap, + struct fuse_args *args, + struct inode *dir, struct dentry *dentry, + umode_t mode) +{ + struct fuse_conn *fc = get_fuse_conn_super(dentry->d_sb); + struct fuse_in_arg ext = { .size = 0, .value = NULL }; + int err = 0; + + if (fc->init_security) + err = get_security_context(dentry, mode, &ext); + if (!err && fc->create_supp_group) + err = get_create_supp_group(idmap, dir, &ext); + + if (!err && ext.size) { + WARN_ON(args->in_numargs >= ARRAY_SIZE(args->in_args)); + args->is_ext = true; + args->ext_idx = args->in_numargs++; + args->in_args[args->ext_idx] = ext; + } else { + kfree(ext.value); + } + + return err; +} + +static void free_ext_value(struct fuse_args *args) +{ + if (args->is_ext) + kfree(args->in_args[args->ext_idx].value); +} + +/* + * Atomic create+open operation + * + * If the filesystem doesn't support this, then fall back to separate + * 'mknod' + 'open' requests. + */ +static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *entry, struct file *file, + unsigned int flags, umode_t mode, u32 opcode) +{ + struct inode *inode; + struct fuse_mount *fm = get_fuse_mount(dir); + FUSE_ARGS(args); + struct fuse_forget_link *forget; + struct fuse_create_in inarg; + struct fuse_open_out *outopenp; + struct fuse_entry_out outentry; + struct fuse_inode *fi; + struct fuse_file *ff; + int epoch, err; + bool trunc = flags & O_TRUNC; + + /* Userspace expects S_IFREG in create mode */ + BUG_ON((mode & S_IFMT) != S_IFREG); + + epoch = atomic_read(&fm->fc->epoch); + forget = fuse_alloc_forget(); + err = -ENOMEM; + if (!forget) + goto out_err; + + ff = fuse_file_alloc(fm, true); + if (!ff) + goto out_put_forget_req; + + if (!fm->fc->dont_mask) + mode &= ~current_umask(); + + flags &= ~O_NOCTTY; + memset(&inarg, 0, sizeof(inarg)); + memset(&outentry, 0, sizeof(outentry)); + inarg.flags = flags; + inarg.mode = mode; + inarg.umask = current_umask(); + + if (fm->fc->handle_killpriv_v2 && trunc && + !(flags & O_EXCL) && !capable(CAP_FSETID)) { + inarg.open_flags |= FUSE_OPEN_KILL_SUIDGID; + } + + args.opcode = opcode; + args.nodeid = get_node_id(dir); + args.in_numargs = 2; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + args.in_args[1].size = entry->d_name.len + 1; + args.in_args[1].value = entry->d_name.name; + args.out_numargs = 2; + args.out_args[0].size = sizeof(outentry); + args.out_args[0].value = &outentry; + /* Store outarg for fuse_finish_open() */ + outopenp = &ff->args->open_outarg; + args.out_args[1].size = sizeof(*outopenp); + args.out_args[1].value = outopenp; + + err = get_create_ext(idmap, &args, dir, entry, mode); + if (err) + goto out_free_ff; + + err = fuse_simple_idmap_request(idmap, fm, &args); + free_ext_value(&args); + if (err) + goto out_free_ff; + + err = -EIO; + if (!S_ISREG(outentry.attr.mode) || invalid_nodeid(outentry.nodeid) || + fuse_invalid_attr(&outentry.attr)) + goto out_free_ff; + + ff->fh = outopenp->fh; + ff->nodeid = outentry.nodeid; + ff->open_flags = outopenp->open_flags; + inode = fuse_iget(dir->i_sb, outentry.nodeid, outentry.generation, + &outentry.attr, ATTR_TIMEOUT(&outentry), 0, 0); + if (!inode) { + flags &= ~(O_CREAT | O_EXCL | O_TRUNC); + fuse_sync_release(NULL, ff, flags); + fuse_chan_queue_forget(fm->fc->chan, forget, outentry.nodeid, 1); + err = -ENOMEM; + goto out_err; + } + kfree(forget); + d_instantiate(entry, inode); + entry->d_time = epoch; + fuse_change_entry_timeout(entry, &outentry); + fuse_dir_changed(dir); + err = generic_file_open(inode, file); + if (!err) { + file->private_data = ff; + err = finish_open(file, entry, fuse_finish_open); + } + if (err) { + fi = get_fuse_inode(inode); + fuse_sync_release(fi, ff, flags); + } else { + if (fm->fc->atomic_o_trunc && trunc) + truncate_pagecache(inode, 0); + else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) + invalidate_inode_pages2(inode->i_mapping); + } + return err; + +out_free_ff: + fuse_file_free(ff); +out_put_forget_req: + kfree(forget); +out_err: + return err; +} + +static int fuse_mknod(struct mnt_idmap *, struct inode *, struct dentry *, + umode_t, dev_t); +static int fuse_atomic_open(struct inode *dir, struct dentry *entry, + struct file *file, unsigned flags, + umode_t mode) +{ + int err; + struct mnt_idmap *idmap = file_mnt_idmap(file); + struct fuse_conn *fc = get_fuse_conn(dir); + + if (fuse_is_bad(dir)) + return -EIO; + + if (d_in_lookup(entry)) { + struct dentry *res = fuse_lookup(dir, entry, 0); + if (res || d_really_is_positive(entry)) + return finish_no_open(file, res); + } + + if (!(flags & O_CREAT)) + return finish_no_open(file, NULL); + + /* Only creates */ + file->f_mode |= FMODE_CREATED; + + if (fc->no_create) + goto mknod; + + err = fuse_create_open(idmap, dir, entry, file, flags, mode, FUSE_CREATE); + if (err == -ENOSYS) { + fc->no_create = 1; + goto mknod; + } else if (err == -EEXIST) + fuse_invalidate_entry(entry); + return err; + +mknod: + err = fuse_mknod(idmap, dir, entry, mode, 0); + if (err) + return err; + return finish_no_open(file, NULL); +} + +/* + * Code shared between mknod, mkdir, symlink and link + */ +static struct dentry *create_new_entry(struct mnt_idmap *idmap, struct fuse_mount *fm, + struct fuse_args *args, struct inode *dir, + struct dentry *entry, umode_t mode) +{ + struct fuse_entry_out outarg; + struct inode *inode; + struct dentry *d; + struct fuse_forget_link *forget; + int epoch, err; + + if (fuse_is_bad(dir)) + return ERR_PTR(-EIO); + + epoch = atomic_read(&fm->fc->epoch); + + forget = fuse_alloc_forget(); + if (!forget) + return ERR_PTR(-ENOMEM); + + memset(&outarg, 0, sizeof(outarg)); + args->nodeid = get_node_id(dir); + args->out_numargs = 1; + args->out_args[0].size = sizeof(outarg); + args->out_args[0].value = &outarg; + + if (args->opcode != FUSE_LINK) { + err = get_create_ext(idmap, args, dir, entry, mode); + if (err) + goto out_put_forget_req; + } + + err = fuse_simple_idmap_request(idmap, fm, args); + free_ext_value(args); + if (err) + goto out_put_forget_req; + + err = -EIO; + if (invalid_nodeid(outarg.nodeid) || fuse_invalid_attr(&outarg.attr)) + goto out_put_forget_req; + + if ((outarg.attr.mode ^ mode) & S_IFMT) + goto out_put_forget_req; + + inode = fuse_iget(dir->i_sb, outarg.nodeid, outarg.generation, + &outarg.attr, ATTR_TIMEOUT(&outarg), 0, 0); + if (!inode) { + fuse_chan_queue_forget(fm->fc->chan, forget, outarg.nodeid, 1); + return ERR_PTR(-ENOMEM); + } + kfree(forget); + + d_drop(entry); + d = d_splice_alias(inode, entry); + if (IS_ERR(d)) + return d; + + if (d) { + d->d_time = epoch; + fuse_change_entry_timeout(d, &outarg); + } else { + entry->d_time = epoch; + fuse_change_entry_timeout(entry, &outarg); + } + fuse_dir_changed(dir); + return d; + + out_put_forget_req: + if (err == -EEXIST) + fuse_invalidate_entry(entry); + kfree(forget); + return ERR_PTR(err); +} + +static int create_new_nondir(struct mnt_idmap *idmap, struct fuse_mount *fm, + struct fuse_args *args, struct inode *dir, + struct dentry *entry, umode_t mode) +{ + /* + * Note that when creating anything other than a directory we + * can be sure create_new_entry() will NOT return an alternate + * dentry as d_splice_alias() only returns an alternate dentry + * for directories. So we don't need to check for that case + * when passing back the result. + */ + WARN_ON_ONCE(S_ISDIR(mode)); + + return PTR_ERR(create_new_entry(idmap, fm, args, dir, entry, mode)); +} + +static int fuse_mknod(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *entry, umode_t mode, dev_t rdev) +{ + struct fuse_mknod_in inarg; + struct fuse_mount *fm = get_fuse_mount(dir); + FUSE_ARGS(args); + + if (!fm->fc->dont_mask) + mode &= ~current_umask(); + + memset(&inarg, 0, sizeof(inarg)); + inarg.mode = mode; + inarg.rdev = new_encode_dev(rdev); + inarg.umask = current_umask(); + args.opcode = FUSE_MKNOD; + args.in_numargs = 2; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + args.in_args[1].size = entry->d_name.len + 1; + args.in_args[1].value = entry->d_name.name; + return create_new_nondir(idmap, fm, &args, dir, entry, mode); +} + +static int fuse_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *entry, umode_t mode) +{ + return fuse_mknod(idmap, dir, entry, mode, 0); +} + +static int fuse_tmpfile(struct mnt_idmap *idmap, struct inode *dir, + struct file *file, umode_t mode) +{ + struct fuse_conn *fc = get_fuse_conn(dir); + int err; + + if (fc->no_tmpfile) + return -EOPNOTSUPP; + + err = fuse_create_open(idmap, dir, file->f_path.dentry, file, + file->f_flags, mode, FUSE_TMPFILE); + if (err == -ENOSYS) { + fc->no_tmpfile = 1; + err = -EOPNOTSUPP; + } + return err; +} + +static struct dentry *fuse_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *entry, umode_t mode) +{ + struct fuse_mkdir_in inarg; + struct fuse_mount *fm = get_fuse_mount(dir); + FUSE_ARGS(args); + + if (!fm->fc->dont_mask) + mode &= ~current_umask(); + + /* + * vfs_mkdir() now passes S_IFDIR in @mode, but @mode is forwarded + * verbatim to the userspace server which has only ever been given the + * permission bits. Strip the type bit until the protocol is known to + * cope with it. + */ + mode &= ~S_IFDIR; + + memset(&inarg, 0, sizeof(inarg)); + inarg.mode = mode; + inarg.umask = current_umask(); + args.opcode = FUSE_MKDIR; + args.in_numargs = 2; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + args.in_args[1].size = entry->d_name.len + 1; + args.in_args[1].value = entry->d_name.name; + return create_new_entry(idmap, fm, &args, dir, entry, S_IFDIR); +} + +static int fuse_symlink(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *entry, const char *link) +{ + struct fuse_mount *fm = get_fuse_mount(dir); + unsigned len = strlen(link) + 1; + FUSE_ARGS(args); + + args.opcode = FUSE_SYMLINK; + args.in_numargs = 3; + fuse_set_zero_arg0(&args); + args.in_args[1].size = entry->d_name.len + 1; + args.in_args[1].value = entry->d_name.name; + args.in_args[2].size = len; + args.in_args[2].value = link; + return create_new_nondir(idmap, fm, &args, dir, entry, S_IFLNK); +} + +void fuse_flush_time_update(struct inode *inode) +{ + int err = sync_inode_metadata(inode, 1); + + mapping_set_error(inode->i_mapping, err); +} + +static void fuse_update_ctime_in_cache(struct inode *inode) +{ + if (!IS_NOCMTIME(inode)) { + inode_set_ctime_current(inode); + mark_inode_dirty_sync(inode); + fuse_flush_time_update(inode); + } +} + +void fuse_update_ctime(struct inode *inode) +{ + fuse_invalidate_attr_mask(inode, STATX_CTIME); + fuse_update_ctime_in_cache(inode); +} + +static void fuse_entry_unlinked(struct dentry *entry) +{ + struct inode *inode = d_inode(entry); + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + + spin_lock(&fi->lock); + fi->attr_version = atomic64_inc_return(&fc->attr_version); + /* + * If i_nlink == 0 then unlink doesn't make sense, yet this can + * happen if userspace filesystem is careless. It would be + * difficult to enforce correct nlink usage so just ignore this + * condition here + */ + if (S_ISDIR(inode->i_mode)) + clear_nlink(inode); + else if (inode->i_nlink > 0) + drop_nlink(inode); + spin_unlock(&fi->lock); + fuse_invalidate_entry_cache(entry); + fuse_update_ctime(inode); +} + +static int fuse_unlink(struct inode *dir, struct dentry *entry) +{ + int err; + struct fuse_mount *fm = get_fuse_mount(dir); + FUSE_ARGS(args); + + if (fuse_is_bad(dir)) + return -EIO; + + args.opcode = FUSE_UNLINK; + args.nodeid = get_node_id(dir); + args.in_numargs = 2; + fuse_set_zero_arg0(&args); + args.in_args[1].size = entry->d_name.len + 1; + args.in_args[1].value = entry->d_name.name; + err = fuse_simple_request(fm, &args); + if (!err) { + fuse_dir_changed(dir); + fuse_entry_unlinked(entry); + } else if (err == -EINTR || err == -ENOENT) + fuse_invalidate_entry(entry); + return err; +} + +static int fuse_rmdir(struct inode *dir, struct dentry *entry) +{ + int err; + struct fuse_mount *fm = get_fuse_mount(dir); + FUSE_ARGS(args); + + if (fuse_is_bad(dir)) + return -EIO; + + args.opcode = FUSE_RMDIR; + args.nodeid = get_node_id(dir); + args.in_numargs = 2; + fuse_set_zero_arg0(&args); + args.in_args[1].size = entry->d_name.len + 1; + args.in_args[1].value = entry->d_name.name; + err = fuse_simple_request(fm, &args); + if (!err) { + fuse_dir_changed(dir); + fuse_entry_unlinked(entry); + } else if (err == -EINTR || err == -ENOENT) + fuse_invalidate_entry(entry); + return err; +} + +static int fuse_rename_common(struct mnt_idmap *idmap, struct inode *olddir, struct dentry *oldent, + struct inode *newdir, struct dentry *newent, + unsigned int flags, int opcode, size_t argsize) +{ + int err; + struct fuse_rename2_in inarg; + struct fuse_mount *fm = get_fuse_mount(olddir); + FUSE_ARGS(args); + + memset(&inarg, 0, argsize); + inarg.newdir = get_node_id(newdir); + inarg.flags = flags; + args.opcode = opcode; + args.nodeid = get_node_id(olddir); + args.in_numargs = 3; + args.in_args[0].size = argsize; + args.in_args[0].value = &inarg; + args.in_args[1].size = oldent->d_name.len + 1; + args.in_args[1].value = oldent->d_name.name; + args.in_args[2].size = newent->d_name.len + 1; + args.in_args[2].value = newent->d_name.name; + err = fuse_simple_idmap_request(idmap, fm, &args); + if (!err) { + /* ctime changes */ + fuse_update_ctime(d_inode(oldent)); + + if (flags & RENAME_EXCHANGE) + fuse_update_ctime(d_inode(newent)); + + fuse_dir_changed(olddir); + if (olddir != newdir) + fuse_dir_changed(newdir); + + /* newent will end up negative */ + if (!(flags & RENAME_EXCHANGE) && d_really_is_positive(newent)) + fuse_entry_unlinked(newent); + } else if (err == -EINTR || err == -ENOENT) { + /* If request was interrupted, DEITY only knows if the + rename actually took place. If the invalidation + fails (e.g. some process has CWD under the renamed + directory), then there can be inconsistency between + the dcache and the real filesystem. Tough luck. */ + fuse_invalidate_entry(oldent); + if (d_really_is_positive(newent)) + fuse_invalidate_entry(newent); + } + + return err; +} + +static int fuse_rename2(struct mnt_idmap *idmap, struct inode *olddir, + struct dentry *oldent, struct inode *newdir, + struct dentry *newent, unsigned int flags) +{ + struct fuse_conn *fc = get_fuse_conn(olddir); + int err; + + if (fuse_is_bad(olddir)) + return -EIO; + + if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) + return -EINVAL; + + if (flags) { + if (fc->no_rename2 || fc->minor < 23) + return -EINVAL; + + err = fuse_rename_common((flags & RENAME_WHITEOUT) ? idmap : &invalid_mnt_idmap, + olddir, oldent, newdir, newent, flags, + FUSE_RENAME2, + sizeof(struct fuse_rename2_in)); + if (err == -ENOSYS) { + fc->no_rename2 = 1; + err = -EINVAL; + } + } else { + err = fuse_rename_common(&invalid_mnt_idmap, olddir, oldent, newdir, newent, 0, + FUSE_RENAME, + sizeof(struct fuse_rename_in)); + } + + return err; +} + +static int fuse_link(struct dentry *entry, struct inode *newdir, + struct dentry *newent) +{ + int err; + struct fuse_link_in inarg; + struct inode *inode = d_inode(entry); + struct fuse_mount *fm = get_fuse_mount(inode); + FUSE_ARGS(args); + + if (fm->fc->no_link) + goto out; + + memset(&inarg, 0, sizeof(inarg)); + inarg.oldnodeid = get_node_id(inode); + args.opcode = FUSE_LINK; + args.in_numargs = 2; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + args.in_args[1].size = newent->d_name.len + 1; + args.in_args[1].value = newent->d_name.name; + err = create_new_nondir(&invalid_mnt_idmap, fm, &args, newdir, newent, inode->i_mode); + if (!err) + fuse_update_ctime_in_cache(inode); + else if (err == -EINTR) + fuse_invalidate_attr(inode); + + if (err == -ENOSYS) + fm->fc->no_link = 1; +out: + if (fm->fc->no_link) + return -EPERM; + + return err; +} + +static void fuse_fillattr(struct mnt_idmap *idmap, struct inode *inode, + struct fuse_attr *attr, struct kstat *stat) +{ + unsigned int blkbits; + struct fuse_conn *fc = get_fuse_conn(inode); + vfsuid_t vfsuid = make_vfsuid(idmap, fc->user_ns, + make_kuid(fc->user_ns, attr->uid)); + vfsgid_t vfsgid = make_vfsgid(idmap, fc->user_ns, + make_kgid(fc->user_ns, attr->gid)); + + stat->dev = inode->i_sb->s_dev; + stat->ino = attr->ino; + stat->mode = (inode->i_mode & S_IFMT) | (attr->mode & 07777); + stat->nlink = attr->nlink; + stat->uid = vfsuid_into_kuid(vfsuid); + stat->gid = vfsgid_into_kgid(vfsgid); + stat->rdev = inode->i_rdev; + stat->atime.tv_sec = attr->atime; + stat->atime.tv_nsec = attr->atimensec; + stat->mtime.tv_sec = attr->mtime; + stat->mtime.tv_nsec = attr->mtimensec; + stat->ctime.tv_sec = attr->ctime; + stat->ctime.tv_nsec = attr->ctimensec; + stat->size = attr->size; + stat->blocks = attr->blocks; + + if (attr->blksize != 0) + blkbits = ilog2(attr->blksize); + else + blkbits = inode->i_sb->s_blocksize_bits; + + stat->blksize = 1 << blkbits; +} + +static void fuse_statx_to_attr(struct fuse_statx *sx, struct fuse_attr *attr) +{ + memset(attr, 0, sizeof(*attr)); + attr->ino = sx->ino; + attr->size = sx->size; + attr->blocks = sx->blocks; + attr->atime = sx->atime.tv_sec; + attr->mtime = sx->mtime.tv_sec; + attr->ctime = sx->ctime.tv_sec; + attr->atimensec = sx->atime.tv_nsec; + attr->mtimensec = sx->mtime.tv_nsec; + attr->ctimensec = sx->ctime.tv_nsec; + attr->mode = sx->mode; + attr->nlink = sx->nlink; + attr->uid = sx->uid; + attr->gid = sx->gid; + attr->rdev = new_encode_dev(MKDEV(sx->rdev_major, sx->rdev_minor)); + attr->blksize = sx->blksize; +} + +static int fuse_do_statx(struct mnt_idmap *idmap, struct inode *inode, + struct file *file, struct kstat *stat) +{ + int err; + struct fuse_attr attr; + struct fuse_statx *sx; + struct fuse_statx_in inarg; + struct fuse_statx_out outarg; + struct fuse_mount *fm = get_fuse_mount(inode); + u64 attr_version = fuse_get_attr_version(fm->fc); + FUSE_ARGS(args); + + memset(&inarg, 0, sizeof(inarg)); + memset(&outarg, 0, sizeof(outarg)); + /* Directories have separate file-handle space */ + if (file && S_ISREG(inode->i_mode)) { + struct fuse_file *ff = file->private_data; + + inarg.getattr_flags |= FUSE_GETATTR_FH; + inarg.fh = ff->fh; + } + /* For now leave sync hints as the default, request all stats. */ + inarg.sx_flags = 0; + inarg.sx_mask = STATX_BASIC_STATS | STATX_BTIME; + args.opcode = FUSE_STATX; + args.nodeid = get_node_id(inode); + args.in_numargs = 1; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + args.out_numargs = 1; + args.out_args[0].size = sizeof(outarg); + args.out_args[0].value = &outarg; + err = fuse_simple_request(fm, &args); + if (err) + return err; + + sx = &outarg.stat; + if (((sx->mask & STATX_SIZE) && !fuse_valid_size(sx->size)) || + ((sx->mask & STATX_TYPE) && (!fuse_valid_type(sx->mode) || + inode_wrong_type(inode, sx->mode)))) { + fuse_make_bad(inode); + return -EIO; + } + + fuse_statx_to_attr(&outarg.stat, &attr); + if ((sx->mask & STATX_BASIC_STATS) == STATX_BASIC_STATS) { + fuse_change_attributes(inode, &attr, &outarg.stat, + ATTR_TIMEOUT(&outarg), attr_version); + } + + if (stat) { + stat->result_mask = sx->mask & (STATX_BASIC_STATS | STATX_BTIME); + stat->btime.tv_sec = sx->btime.tv_sec; + stat->btime.tv_nsec = min_t(u32, sx->btime.tv_nsec, NSEC_PER_SEC - 1); + fuse_fillattr(idmap, inode, &attr, stat); + stat->result_mask |= STATX_TYPE; + } + + return 0; +} + +static int fuse_do_getattr(struct mnt_idmap *idmap, struct inode *inode, + struct kstat *stat, struct file *file) +{ + int err; + struct fuse_getattr_in inarg; + struct fuse_attr_out outarg; + struct fuse_mount *fm = get_fuse_mount(inode); + FUSE_ARGS(args); + u64 attr_version; + + attr_version = fuse_get_attr_version(fm->fc); + + memset(&inarg, 0, sizeof(inarg)); + memset(&outarg, 0, sizeof(outarg)); + /* Directories have separate file-handle space */ + if (file && S_ISREG(inode->i_mode)) { + struct fuse_file *ff = file->private_data; + + inarg.getattr_flags |= FUSE_GETATTR_FH; + inarg.fh = ff->fh; + } + args.opcode = FUSE_GETATTR; + args.nodeid = get_node_id(inode); + args.in_numargs = 1; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + args.out_numargs = 1; + args.out_args[0].size = sizeof(outarg); + args.out_args[0].value = &outarg; + err = fuse_simple_request(fm, &args); + if (!err) { + if (fuse_invalid_attr(&outarg.attr) || + inode_wrong_type(inode, outarg.attr.mode)) { + fuse_make_bad(inode); + err = -EIO; + } else { + fuse_change_attributes(inode, &outarg.attr, NULL, + ATTR_TIMEOUT(&outarg), + attr_version); + if (stat) + fuse_fillattr(idmap, inode, &outarg.attr, stat); + } + } + return err; +} + +static int fuse_update_get_attr(struct mnt_idmap *idmap, struct inode *inode, + struct file *file, struct kstat *stat, + u32 request_mask, unsigned int flags) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_conn *fc = get_fuse_conn(inode); + int err = 0; + bool sync; + u32 inval_mask = READ_ONCE(fi->inval_mask); + u32 cache_mask = fuse_get_cache_mask(inode); + + + /* FUSE only supports basic stats and possibly btime */ + request_mask &= STATX_BASIC_STATS | STATX_BTIME; +retry: + if (fc->no_statx) + request_mask &= STATX_BASIC_STATS; + + if (!request_mask) + sync = false; + else if (flags & AT_STATX_FORCE_SYNC) + sync = true; + else if (flags & AT_STATX_DONT_SYNC) + sync = false; + else if (request_mask & inval_mask & ~cache_mask) + sync = true; + else + sync = time_before64(fi->i_time, get_jiffies_64()); + + if (sync) { + forget_all_cached_acls(inode); + /* Try statx if BTIME is requested */ + if (!fc->no_statx && (request_mask & ~STATX_BASIC_STATS)) { + err = fuse_do_statx(idmap, inode, file, stat); + if (err == -ENOSYS) { + fc->no_statx = 1; + err = 0; + goto retry; + } + } else { + err = fuse_do_getattr(idmap, inode, stat, file); + } + } else if (stat) { + generic_fillattr(idmap, request_mask, inode, stat); + stat->mode = fi->orig_i_mode; + stat->ino = fi->orig_ino; + stat->blksize = 1 << fi->cached_i_blkbits; + if (test_bit(FUSE_I_BTIME, &fi->state)) { + stat->btime = fi->i_btime; + stat->result_mask |= STATX_BTIME; + } + } + + return err; +} + +int fuse_update_attributes(struct inode *inode, struct file *file, u32 mask) +{ + return fuse_update_get_attr(&nop_mnt_idmap, inode, file, NULL, mask, 0); +} + +int fuse_reverse_inval_entry(struct fuse_conn *fc, u64 parent_nodeid, + u64 child_nodeid, struct qstr *name, u32 flags) +{ + int err = -ENOTDIR; + struct inode *parent; + struct dentry *dir; + struct dentry *entry; + + parent = fuse_ilookup(fc, parent_nodeid, NULL); + if (!parent) + return -ENOENT; + + inode_lock_nested(parent, I_MUTEX_PARENT); + if (!S_ISDIR(parent->i_mode)) + goto unlock; + + err = -ENOENT; + dir = d_find_alias(parent); + if (!dir) + goto unlock; + + name->hash = full_name_hash(dir, name->name, name->len); + entry = d_lookup(dir, name); + dput(dir); + if (!entry) + goto unlock; + + fuse_dir_changed(parent); + if (!(flags & FUSE_EXPIRE_ONLY)) + d_invalidate(entry); + fuse_invalidate_entry_cache(entry); + + if (child_nodeid != 0 && d_really_is_positive(entry)) { + inode_lock(d_inode(entry)); + if (get_node_id(d_inode(entry)) != child_nodeid) { + err = -ENOENT; + goto badentry; + } + if (d_mountpoint(entry)) { + err = -EBUSY; + goto badentry; + } + if (d_is_dir(entry)) { + shrink_dcache_parent(entry); + if (!simple_empty(entry)) { + err = -ENOTEMPTY; + goto badentry; + } + d_inode(entry)->i_flags |= S_DEAD; + } + dont_mount(entry); + clear_nlink(d_inode(entry)); + err = 0; + badentry: + inode_unlock(d_inode(entry)); + if (!err) + d_delete(entry); + } else { + err = 0; + } + dput(entry); + + unlock: + inode_unlock(parent); + iput(parent); + return err; +} + +static inline bool fuse_permissible_uidgid(struct fuse_conn *fc) +{ + const struct cred *cred = current_cred(); + + return (uid_eq(cred->euid, fc->user_id) && + uid_eq(cred->suid, fc->user_id) && + uid_eq(cred->uid, fc->user_id) && + gid_eq(cred->egid, fc->group_id) && + gid_eq(cred->sgid, fc->group_id) && + gid_eq(cred->gid, fc->group_id)); +} + +/* + * Calling into a user-controlled filesystem gives the filesystem + * daemon ptrace-like capabilities over the current process. This + * means, that the filesystem daemon is able to record the exact + * filesystem operations performed, and can also control the behavior + * of the requester process in otherwise impossible ways. For example + * it can delay the operation for arbitrary length of time allowing + * DoS against the requester. + * + * For this reason only those processes can call into the filesystem, + * for which the owner of the mount has ptrace privilege. This + * excludes processes started by other users, suid or sgid processes. + */ +bool fuse_allow_current_process(struct fuse_conn *fc) +{ + bool allow; + + if (fc->allow_other) + allow = current_in_userns(fc->user_ns); + else + allow = fuse_permissible_uidgid(fc); + + if (!allow && allow_sys_admin_access && capable(CAP_SYS_ADMIN)) + allow = true; + + return allow; +} + +static int fuse_access(struct inode *inode, int mask) +{ + struct fuse_mount *fm = get_fuse_mount(inode); + FUSE_ARGS(args); + struct fuse_access_in inarg; + int err; + + BUG_ON(mask & MAY_NOT_BLOCK); + + /* + * We should not send FUSE_ACCESS to the userspace + * when idmapped mounts are enabled as for this case + * we have fc->default_permissions = 1 and access + * permission checks are done on the kernel side. + */ + WARN_ON_ONCE(!(fm->sb->s_iflags & SB_I_NOIDMAP)); + + if (fm->fc->no_access) + return 0; + + memset(&inarg, 0, sizeof(inarg)); + inarg.mask = mask & (MAY_READ | MAY_WRITE | MAY_EXEC); + args.opcode = FUSE_ACCESS; + args.nodeid = get_node_id(inode); + args.in_numargs = 1; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + err = fuse_simple_request(fm, &args); + if (err == -ENOSYS) { + fm->fc->no_access = 1; + err = 0; + } + return err; +} + +static int fuse_perm_getattr(struct inode *inode, int mask) +{ + if (mask & MAY_NOT_BLOCK) + return -ECHILD; + + forget_all_cached_acls(inode); + return fuse_do_getattr(&nop_mnt_idmap, inode, NULL, NULL); +} + +/* + * Check permission. The two basic access models of FUSE are: + * + * 1) Local access checking ('default_permissions' mount option) based + * on file mode. This is the plain old disk filesystem permission + * model. + * + * 2) "Remote" access checking, where server is responsible for + * checking permission in each inode operation. An exception to this + * is if ->permission() was invoked from sys_access() in which case an + * access request is sent. Execute permission is still checked + * locally based on file mode. + */ +static int fuse_permission(struct mnt_idmap *idmap, + struct inode *inode, int mask) +{ + struct fuse_conn *fc = get_fuse_conn(inode); + bool refreshed = false; + int err = 0; + + if (fuse_is_bad(inode)) + return -EIO; + + if (!fuse_allow_current_process(fc)) + return -EACCES; + + /* + * If attributes are needed, refresh them before proceeding + */ + if (fc->default_permissions || + ((mask & MAY_EXEC) && S_ISREG(inode->i_mode))) { + struct fuse_inode *fi = get_fuse_inode(inode); + u32 perm_mask = STATX_MODE | STATX_UID | STATX_GID; + + if (perm_mask & READ_ONCE(fi->inval_mask) || + time_before64(fi->i_time, get_jiffies_64())) { + refreshed = true; + + err = fuse_perm_getattr(inode, mask); + if (err) + return err; + } + } + + if (fc->default_permissions) { + err = generic_permission(idmap, inode, mask); + + /* If permission is denied, try to refresh file + attributes. This is also needed, because the root + node will at first have no permissions */ + if (err == -EACCES && !refreshed) { + err = fuse_perm_getattr(inode, mask); + if (!err) + err = generic_permission(idmap, + inode, mask); + } + + /* Note: the opposite of the above test does not + exist. So if permissions are revoked this won't be + noticed immediately, only after the attribute + timeout has expired */ + } else if (mask & (MAY_ACCESS | MAY_CHDIR)) { + err = fuse_access(inode, mask); + } else if ((mask & MAY_EXEC) && S_ISREG(inode->i_mode)) { + if (!(inode->i_mode & S_IXUGO)) { + if (refreshed) + return -EACCES; + + err = fuse_perm_getattr(inode, mask); + if (!err && !(inode->i_mode & S_IXUGO)) + return -EACCES; + } + } + return err; +} + +static int fuse_readlink_folio(struct inode *inode, struct folio *folio) +{ + struct fuse_mount *fm = get_fuse_mount(inode); + struct fuse_folio_desc desc = { .length = folio_size(folio) - 1 }; + struct fuse_args_pages ap = { + .num_folios = 1, + .folios = &folio, + .descs = &desc, + }; + char *link; + ssize_t res; + + ap.args.opcode = FUSE_READLINK; + ap.args.nodeid = get_node_id(inode); + ap.args.out_pages = true; + ap.args.out_argvar = true; + ap.args.page_zeroing = true; + ap.args.out_numargs = 1; + ap.args.out_args[0].size = desc.length; + res = fuse_simple_request(fm, &ap.args); + + fuse_invalidate_atime(inode); + + if (res < 0) + return res; + + if (WARN_ON(res >= PAGE_SIZE)) + return -EIO; + + link = folio_address(folio); + link[res] = '\0'; + + return 0; +} + +static const char *fuse_get_link(struct dentry *dentry, struct inode *inode, + struct delayed_call *callback) +{ + struct fuse_conn *fc = get_fuse_conn(inode); + struct folio *folio; + int err; + + err = -EIO; + if (fuse_is_bad(inode)) + goto out_err; + + if (fc->cache_symlinks) + return page_get_link_raw(dentry, inode, callback); + + err = -ECHILD; + if (!dentry) + goto out_err; + + folio = folio_alloc(GFP_KERNEL, 0); + err = -ENOMEM; + if (!folio) + goto out_err; + + err = fuse_readlink_folio(inode, folio); + if (err) { + folio_put(folio); + goto out_err; + } + + set_delayed_call(callback, page_put_link, folio); + + return folio_address(folio); + +out_err: + return ERR_PTR(err); +} + +static int fuse_dir_open(struct inode *inode, struct file *file) +{ + struct fuse_mount *fm = get_fuse_mount(inode); + int err; + + if (fuse_is_bad(inode)) + return -EIO; + + err = generic_file_open(inode, file); + if (err) + return err; + + err = fuse_do_open(fm, get_node_id(inode), file, true); + if (!err) { + struct fuse_file *ff = file->private_data; + + /* + * Keep handling FOPEN_STREAM and FOPEN_NONSEEKABLE for + * directories for backward compatibility, though it's unlikely + * to be useful. + */ + if (ff->open_flags & (FOPEN_STREAM | FOPEN_NONSEEKABLE)) + nonseekable_open(inode, file); + if (!(ff->open_flags & FOPEN_KEEP_CACHE)) + invalidate_inode_pages2(inode->i_mapping); + } + + return err; +} + +static int fuse_dir_release(struct inode *inode, struct file *file) +{ + fuse_release_common(file, true); + + return 0; +} + +static int fuse_dir_fsync(struct file *file, loff_t start, loff_t end, + int datasync) +{ + struct inode *inode = file->f_mapping->host; + struct fuse_conn *fc = get_fuse_conn(inode); + int err; + + if (fuse_is_bad(inode)) + return -EIO; + + if (fc->no_fsyncdir) + return 0; + + inode_lock(inode); + err = fuse_fsync_common(file, start, end, datasync, FUSE_FSYNCDIR); + if (err == -ENOSYS) { + fc->no_fsyncdir = 1; + err = 0; + } + inode_unlock(inode); + + return err; +} + +static long fuse_dir_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct fuse_conn *fc = get_fuse_conn(file->f_mapping->host); + + /* FUSE_IOCTL_DIR only supported for API version >= 7.18 */ + if (fc->minor < 18) + return -ENOTTY; + + return fuse_ioctl_common(file, cmd, arg, FUSE_IOCTL_DIR); +} + +static long fuse_dir_compat_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct fuse_conn *fc = get_fuse_conn(file->f_mapping->host); + + if (fc->minor < 18) + return -ENOTTY; + + return fuse_ioctl_common(file, cmd, arg, + FUSE_IOCTL_COMPAT | FUSE_IOCTL_DIR); +} + +static bool update_mtime(unsigned ivalid, bool trust_local_mtime) +{ + /* Always update if mtime is explicitly set */ + if (ivalid & ATTR_MTIME_SET) + return true; + + /* Or if kernel i_mtime is the official one */ + if (trust_local_mtime) + return true; + + /* If it's an open(O_TRUNC) or an ftruncate(), don't update */ + if ((ivalid & ATTR_SIZE) && (ivalid & (ATTR_OPEN | ATTR_FILE))) + return false; + + /* In all other cases update */ + return true; +} + +static void iattr_to_fattr(struct mnt_idmap *idmap, struct fuse_conn *fc, + struct iattr *iattr, struct fuse_setattr_in *arg, + bool trust_local_cmtime) +{ + unsigned ivalid = iattr->ia_valid; + + if (ivalid & ATTR_MODE) + arg->valid |= FATTR_MODE, arg->mode = iattr->ia_mode; + + if (ivalid & ATTR_UID) { + kuid_t fsuid = from_vfsuid(idmap, fc->user_ns, iattr->ia_vfsuid); + + arg->valid |= FATTR_UID; + arg->uid = from_kuid(fc->user_ns, fsuid); + } + + if (ivalid & ATTR_GID) { + kgid_t fsgid = from_vfsgid(idmap, fc->user_ns, iattr->ia_vfsgid); + + arg->valid |= FATTR_GID; + arg->gid = from_kgid(fc->user_ns, fsgid); + } + + if (ivalid & ATTR_SIZE) + arg->valid |= FATTR_SIZE, arg->size = iattr->ia_size; + if (ivalid & ATTR_ATIME) { + arg->valid |= FATTR_ATIME; + arg->atime = iattr->ia_atime.tv_sec; + arg->atimensec = iattr->ia_atime.tv_nsec; + if (!(ivalid & ATTR_ATIME_SET)) + arg->valid |= FATTR_ATIME_NOW; + } + if ((ivalid & ATTR_MTIME) && update_mtime(ivalid, trust_local_cmtime)) { + arg->valid |= FATTR_MTIME; + arg->mtime = iattr->ia_mtime.tv_sec; + arg->mtimensec = iattr->ia_mtime.tv_nsec; + if (!(ivalid & ATTR_MTIME_SET) && !trust_local_cmtime) + arg->valid |= FATTR_MTIME_NOW; + } + if ((ivalid & ATTR_CTIME) && trust_local_cmtime) { + arg->valid |= FATTR_CTIME; + arg->ctime = iattr->ia_ctime.tv_sec; + arg->ctimensec = iattr->ia_ctime.tv_nsec; + } +} + +/* + * Prevent concurrent writepages on inode + * + * This is done by adding a negative bias to the inode write counter + * and waiting for all pending writes to finish. + */ +void fuse_set_nowrite(struct inode *inode) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + + BUG_ON(!inode_is_locked(inode)); + + spin_lock(&fi->lock); + BUG_ON(fi->writectr < 0); + fi->writectr += FUSE_NOWRITE; + spin_unlock(&fi->lock); + wait_event(fi->page_waitq, fi->writectr == FUSE_NOWRITE); +} + +/* + * Allow writepages on inode + * + * Remove the bias from the writecounter and send any queued + * writepages. + */ +static void __fuse_release_nowrite(struct inode *inode) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + + BUG_ON(fi->writectr != FUSE_NOWRITE); + fi->writectr = 0; + fuse_flush_writepages(inode); +} + +void fuse_release_nowrite(struct inode *inode) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + + spin_lock(&fi->lock); + __fuse_release_nowrite(inode); + spin_unlock(&fi->lock); +} + +static void fuse_setattr_fill(struct fuse_conn *fc, struct fuse_args *args, + struct inode *inode, + struct fuse_setattr_in *inarg_p, + struct fuse_attr_out *outarg_p) +{ + args->opcode = FUSE_SETATTR; + args->nodeid = get_node_id(inode); + args->in_numargs = 1; + args->in_args[0].size = sizeof(*inarg_p); + args->in_args[0].value = inarg_p; + args->out_numargs = 1; + args->out_args[0].size = sizeof(*outarg_p); + args->out_args[0].value = outarg_p; +} + +/* + * Flush inode->i_mtime to the server + */ +int fuse_flush_times(struct inode *inode, struct fuse_file *ff) +{ + struct fuse_mount *fm = get_fuse_mount(inode); + FUSE_ARGS(args); + struct fuse_setattr_in inarg; + struct fuse_attr_out outarg; + + memset(&inarg, 0, sizeof(inarg)); + memset(&outarg, 0, sizeof(outarg)); + + inarg.valid = FATTR_MTIME; + inarg.mtime = inode_get_mtime_sec(inode); + inarg.mtimensec = inode_get_mtime_nsec(inode); + if (fm->fc->minor >= 23) { + inarg.valid |= FATTR_CTIME; + inarg.ctime = inode_get_ctime_sec(inode); + inarg.ctimensec = inode_get_ctime_nsec(inode); + } + if (ff) { + inarg.valid |= FATTR_FH; + inarg.fh = ff->fh; + } + fuse_setattr_fill(fm->fc, &args, inode, &inarg, &outarg); + + return fuse_simple_request(fm, &args); +} + +/* + * Set attributes, and at the same time refresh them. + * + * Truncation is slightly complicated, because the 'truncate' request + * may fail, in which case we don't want to touch the mapping. + * vmtruncate() doesn't allow for this case, so do the rlimit checking + * and the actual truncation by hand. + */ +int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr, struct file *file) +{ + struct inode *inode = d_inode(dentry); + struct fuse_mount *fm = get_fuse_mount(inode); + struct fuse_conn *fc = fm->fc; + struct fuse_inode *fi = get_fuse_inode(inode); + struct address_space *mapping = inode->i_mapping; + FUSE_ARGS(args); + struct fuse_setattr_in inarg; + struct fuse_attr_out outarg; + bool is_truncate = false; + bool is_wb = fc->writeback_cache && S_ISREG(inode->i_mode); + loff_t oldsize; + int err; + bool trust_local_cmtime = is_wb; + bool fault_blocked = false; + u64 attr_version; + + if (!fc->default_permissions) + attr->ia_valid |= ATTR_FORCE; + + err = setattr_prepare(idmap, dentry, attr); + if (err) + return err; + + if (attr->ia_valid & ATTR_SIZE) { + if (WARN_ON(!S_ISREG(inode->i_mode))) + return -EIO; + is_truncate = true; + } + + if (FUSE_IS_DAX(inode) && is_truncate) { + filemap_invalidate_lock(mapping); + fault_blocked = true; + err = fuse_dax_break_layouts(inode, 0, -1); + if (err) { + filemap_invalidate_unlock(mapping); + return err; + } + } + + if (attr->ia_valid & ATTR_OPEN) { + /* This is coming from open(..., ... | O_TRUNC); */ + WARN_ON(!(attr->ia_valid & ATTR_SIZE)); + WARN_ON(attr->ia_size != 0); + if (fc->atomic_o_trunc) { + /* + * No need to send request to userspace, since actual + * truncation has already been done by OPEN. But still + * need to truncate page cache. + */ + i_size_write(inode, 0); + truncate_pagecache(inode, 0); + goto out; + } + file = NULL; + } + + /* Flush dirty data/metadata before non-truncate SETATTR */ + if (is_wb && + attr->ia_valid & + (ATTR_MODE | ATTR_UID | ATTR_GID | ATTR_MTIME_SET | + ATTR_TIMES_SET)) { + err = write_inode_now(inode, true); + if (err) + return err; + + fuse_set_nowrite(inode); + fuse_release_nowrite(inode); + } + + if (is_truncate) { + fuse_set_nowrite(inode); + set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); + if (trust_local_cmtime && attr->ia_size != inode->i_size) + attr->ia_valid |= ATTR_MTIME | ATTR_CTIME; + } + + memset(&inarg, 0, sizeof(inarg)); + memset(&outarg, 0, sizeof(outarg)); + iattr_to_fattr(idmap, fc, attr, &inarg, trust_local_cmtime); + if (file) { + struct fuse_file *ff = file->private_data; + inarg.valid |= FATTR_FH; + inarg.fh = ff->fh; + } + + /* Kill suid/sgid for non-directory chown unconditionally */ + if (fc->handle_killpriv_v2 && !S_ISDIR(inode->i_mode) && + attr->ia_valid & (ATTR_UID | ATTR_GID)) + inarg.valid |= FATTR_KILL_SUIDGID; + + if (attr->ia_valid & ATTR_SIZE) { + /* For mandatory locking in truncate */ + inarg.valid |= FATTR_LOCKOWNER; + inarg.lock_owner = fuse_lock_owner_id(fc, current->files); + + /* Kill suid/sgid for truncate only if no CAP_FSETID */ + if (fc->handle_killpriv_v2 && !capable(CAP_FSETID)) + inarg.valid |= FATTR_KILL_SUIDGID; + } + + attr_version = fuse_get_attr_version(fm->fc); + fuse_setattr_fill(fc, &args, inode, &inarg, &outarg); + err = fuse_simple_request(fm, &args); + if (err) { + if (err == -EINTR) + fuse_invalidate_attr(inode); + goto error; + } + + if (fuse_invalid_attr(&outarg.attr) || + inode_wrong_type(inode, outarg.attr.mode)) { + fuse_make_bad(inode); + err = -EIO; + goto error; + } + + spin_lock(&fi->lock); + /* the kernel maintains i_mtime locally */ + if (trust_local_cmtime) { + if (attr->ia_valid & ATTR_MTIME) + inode_set_mtime_to_ts(inode, attr->ia_mtime); + if (attr->ia_valid & ATTR_CTIME) + inode_set_ctime_to_ts(inode, attr->ia_ctime); + /* FIXME: clear I_DIRTY_SYNC? */ + } + + if (fi->attr_version > attr_version) { + /* + * Apply attributes, for example for fsnotify_change(), but set + * attribute timeout to zero. + */ + outarg.attr_valid = outarg.attr_valid_nsec = 0; + } + + fuse_change_attributes_common(inode, &outarg.attr, NULL, + ATTR_TIMEOUT(&outarg), + fuse_get_cache_mask(inode), 0); + oldsize = inode->i_size; + /* see the comment in fuse_change_attributes() */ + if (!is_wb || is_truncate) + i_size_write(inode, outarg.attr.size); + + if (is_truncate) { + /* NOTE: this may release/reacquire fi->lock */ + __fuse_release_nowrite(inode); + } + spin_unlock(&fi->lock); + + /* + * Only call invalidate_inode_pages2() after removing + * FUSE_NOWRITE, otherwise fuse_launder_folio() would deadlock. + */ + if ((is_truncate || !is_wb) && + S_ISREG(inode->i_mode) && oldsize != outarg.attr.size) { + truncate_pagecache(inode, outarg.attr.size); + invalidate_inode_pages2(mapping); + } + + clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); +out: + if (fault_blocked) + filemap_invalidate_unlock(mapping); + + return 0; + +error: + if (is_truncate) + fuse_release_nowrite(inode); + + clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); + + if (fault_blocked) + filemap_invalidate_unlock(mapping); + return err; +} + +static int fuse_setattr(struct mnt_idmap *idmap, struct dentry *entry, + struct iattr *attr) +{ + struct inode *inode = d_inode(entry); + struct fuse_conn *fc = get_fuse_conn(inode); + struct file *file = (attr->ia_valid & ATTR_FILE) ? attr->ia_file : NULL; + int ret; + + if (fuse_is_bad(inode)) + return -EIO; + + if (!fuse_allow_current_process(get_fuse_conn(inode))) + return -EACCES; + + if (attr->ia_valid & (ATTR_KILL_SUID | ATTR_KILL_SGID)) { + attr->ia_valid &= ~(ATTR_KILL_SUID | ATTR_KILL_SGID | + ATTR_MODE); + + /* + * The only sane way to reliably kill suid/sgid is to do it in + * the userspace filesystem + * + * This should be done on write(), truncate() and chown(). + */ + if (!fc->handle_killpriv && !fc->handle_killpriv_v2) { + /* + * ia_mode calculation may have used stale i_mode. + * Refresh and recalculate. + */ + ret = fuse_do_getattr(idmap, inode, NULL, file); + if (ret) + return ret; + + attr->ia_mode = inode->i_mode; + if (inode->i_mode & S_ISUID) { + attr->ia_valid |= ATTR_MODE; + attr->ia_mode &= ~S_ISUID; + } + if ((inode->i_mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP)) { + attr->ia_valid |= ATTR_MODE; + attr->ia_mode &= ~S_ISGID; + } + } + } + if (!attr->ia_valid) + return 0; + + ret = fuse_do_setattr(idmap, entry, attr, file); + if (!ret) { + /* + * If filesystem supports acls it may have updated acl xattrs in + * the filesystem, so forget cached acls for the inode. + */ + if (fc->posix_acl) + forget_all_cached_acls(inode); + + /* Directory mode changed, may need to revalidate access */ + if (d_is_dir(entry) && (attr->ia_valid & ATTR_MODE)) + fuse_invalidate_entry_cache(entry); + } + return ret; +} + +static int fuse_getattr(struct mnt_idmap *idmap, + const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int flags) +{ + struct inode *inode = d_inode(path->dentry); + struct fuse_conn *fc = get_fuse_conn(inode); + + if (fuse_is_bad(inode)) + return -EIO; + + if (!fuse_allow_current_process(fc)) { + if (!request_mask) { + /* + * If user explicitly requested *nothing* then don't + * error out, but return st_dev only. + */ + stat->result_mask = 0; + stat->dev = inode->i_sb->s_dev; + return 0; + } + return -EACCES; + } + + return fuse_update_get_attr(idmap, inode, NULL, stat, request_mask, flags); +} + +static const struct inode_operations fuse_dir_inode_operations = { + .lookup = fuse_lookup, + .mkdir = fuse_mkdir, + .symlink = fuse_symlink, + .unlink = fuse_unlink, + .rmdir = fuse_rmdir, + .rename = fuse_rename2, + .link = fuse_link, + .setattr = fuse_setattr, + .create = fuse_create, + .atomic_open = fuse_atomic_open, + .tmpfile = fuse_tmpfile, + .mknod = fuse_mknod, + .permission = fuse_permission, + .getattr = fuse_getattr, + .listxattr = fuse_listxattr, + .get_inode_acl = fuse_get_inode_acl, + .get_acl = fuse_get_acl, + .set_acl = fuse_set_acl, + .fileattr_get = fuse_fileattr_get, + .fileattr_set = fuse_fileattr_set, +}; + +static const struct file_operations fuse_dir_operations = { + .llseek = generic_file_llseek, + .read = generic_read_dir, + .iterate_shared = fuse_readdir, + .open = fuse_dir_open, + .release = fuse_dir_release, + .fsync = fuse_dir_fsync, + .unlocked_ioctl = fuse_dir_ioctl, + .compat_ioctl = fuse_dir_compat_ioctl, +}; + +static const struct inode_operations fuse_common_inode_operations = { + .setattr = fuse_setattr, + .permission = fuse_permission, + .getattr = fuse_getattr, + .listxattr = fuse_listxattr, + .get_inode_acl = fuse_get_inode_acl, + .get_acl = fuse_get_acl, + .set_acl = fuse_set_acl, + .fileattr_get = fuse_fileattr_get, + .fileattr_set = fuse_fileattr_set, +}; + +static const struct inode_operations fuse_symlink_inode_operations = { + .setattr = fuse_setattr, + .get_link = fuse_get_link, + .getattr = fuse_getattr, + .listxattr = fuse_listxattr, +}; + +void fuse_init_common(struct inode *inode) +{ + inode->i_op = &fuse_common_inode_operations; +} + +void fuse_init_dir(struct inode *inode) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + + inode->i_op = &fuse_dir_inode_operations; + inode->i_fop = &fuse_dir_operations; + + spin_lock_init(&fi->rdc.lock); + fi->rdc.cached = false; + fi->rdc.size = 0; + fi->rdc.pos = 0; + fi->rdc.version = 0; +} + +static int fuse_symlink_read_folio(struct file *null, struct folio *folio) +{ + int err = fuse_readlink_folio(folio->mapping->host, folio); + + if (!err) + folio_mark_uptodate(folio); + + folio_unlock(folio); + + return err; +} + +static const struct address_space_operations fuse_symlink_aops = { + .read_folio = fuse_symlink_read_folio, +}; + +void fuse_init_symlink(struct inode *inode) +{ + inode->i_op = &fuse_symlink_inode_operations; + inode->i_data.a_ops = &fuse_symlink_aops; + inode_nohighmem(inode); +} diff --git a/fs/fuse/file.c b/fs/fuse/file.c new file mode 100644 index 000000000..f2c081f09 --- /dev/null +++ b/fs/fuse/file.c @@ -0,0 +1,3120 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + FUSE: Filesystem in Userspace + Copyright (C) 2001-2008 Miklos Szeredi +*/ + +#include "fuse_i.h" +#include "dev.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static int fuse_send_open(struct fuse_mount *fm, u64 nodeid, + unsigned int open_flags, int opcode, + struct fuse_open_out *outargp) +{ + struct fuse_open_in inarg; + FUSE_ARGS(args); + + memset(&inarg, 0, sizeof(inarg)); + inarg.flags = open_flags & ~(O_CREAT | O_EXCL | O_NOCTTY); + if (!fm->fc->atomic_o_trunc) + inarg.flags &= ~O_TRUNC; + + if (fm->fc->handle_killpriv_v2 && + (inarg.flags & O_TRUNC) && !capable(CAP_FSETID)) { + inarg.open_flags |= FUSE_OPEN_KILL_SUIDGID; + } + + args.opcode = opcode; + args.nodeid = nodeid; + args.in_numargs = 1; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + args.out_numargs = 1; + args.out_args[0].size = sizeof(*outargp); + args.out_args[0].value = outargp; + + return fuse_simple_request(fm, &args); +} + +struct fuse_file *fuse_file_alloc(struct fuse_mount *fm, bool release) +{ + struct fuse_file *ff; + + ff = kzalloc_obj(struct fuse_file, GFP_KERNEL_ACCOUNT); + if (unlikely(!ff)) + return NULL; + + ff->fm = fm; + if (release) { + ff->args = kzalloc_obj(*ff->args, GFP_KERNEL_ACCOUNT); + if (!ff->args) { + kfree(ff); + return NULL; + } + } + + INIT_LIST_HEAD(&ff->write_entry); + refcount_set(&ff->count, 1); + RB_CLEAR_NODE(&ff->polled_node); + init_waitqueue_head(&ff->poll_wait); + + ff->kh = atomic64_inc_return(&fm->fc->khctr); + + return ff; +} + +void fuse_file_free(struct fuse_file *ff) +{ + kfree(ff->args); + kfree(ff); +} + +static struct fuse_file *fuse_file_get(struct fuse_file *ff) +{ + refcount_inc(&ff->count); + return ff; +} + +static void fuse_release_end(struct fuse_args *args, int error) +{ + struct fuse_release_args *ra = container_of(args, typeof(*ra), args); + + iput(ra->inode); + kfree(ra); +} + +static void fuse_file_put(struct fuse_file *ff, bool sync) +{ + if (refcount_dec_and_test(&ff->count)) { + struct fuse_release_args *ra = &ff->args->release_args; + struct fuse_args *args = (ra ? &ra->args : NULL); + + if (ra && ra->inode) + fuse_file_io_release(ff, ra->inode); + + if (!args) { + /* Do nothing when server does not implement 'opendir' */ + } else if (args->opcode == FUSE_RELEASE && ff->fm->fc->no_open) { + fuse_release_end(args, 0); + } else if (sync) { + fuse_simple_request(ff->fm, args); + fuse_release_end(args, 0); + } else { + /* + * DAX inodes may need to issue a number of synchronous + * request for clearing the mappings. + */ + if (ra && ra->inode && FUSE_IS_DAX(ra->inode)) + args->may_block = true; + args->end = fuse_release_end; + if (fuse_simple_background(ff->fm, args, + GFP_KERNEL | __GFP_NOFAIL)) + fuse_release_end(args, -ENOTCONN); + } + kfree(ff); + } +} + +struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid, + unsigned int open_flags, bool isdir) +{ + struct fuse_conn *fc = fm->fc; + struct fuse_file *ff; + int opcode = isdir ? FUSE_OPENDIR : FUSE_OPEN; + bool open = isdir ? !fc->no_opendir : !fc->no_open; + bool release = !isdir || open; + + /* + * ff->args->release_args still needs to be allocated (so we can hold an + * inode reference while there are pending inflight file operations when + * ->release() is called, see fuse_prepare_release()) even if + * fc->no_open is set else it becomes possible for reclaim to deadlock + * if while servicing the readahead request the server triggers reclaim + * and reclaim evicts the inode of the file being read ahead. + */ + ff = fuse_file_alloc(fm, release); + if (!ff) + return ERR_PTR(-ENOMEM); + + ff->fh = 0; + /* Default for no-open */ + ff->open_flags = FOPEN_KEEP_CACHE | (isdir ? FOPEN_CACHE_DIR : 0); + if (open) { + /* Store outarg for fuse_finish_open() */ + struct fuse_open_out *outargp = &ff->args->open_outarg; + int err; + + err = fuse_send_open(fm, nodeid, open_flags, opcode, outargp); + if (!err) { + ff->fh = outargp->fh; + ff->open_flags = outargp->open_flags; + } else if (err != -ENOSYS) { + fuse_file_free(ff); + return ERR_PTR(err); + } else { + if (isdir) { + /* No release needed */ + kfree(ff->args); + ff->args = NULL; + fc->no_opendir = 1; + } else { + fc->no_open = 1; + } + } + } + + if (isdir) + ff->open_flags &= ~FOPEN_DIRECT_IO; + + ff->nodeid = nodeid; + + return ff; +} + +int fuse_do_open(struct fuse_mount *fm, u64 nodeid, struct file *file, + bool isdir) +{ + struct fuse_file *ff = fuse_file_open(fm, nodeid, file->f_flags, isdir); + + if (!IS_ERR(ff)) + file->private_data = ff; + + return PTR_ERR_OR_ZERO(ff); +} +EXPORT_SYMBOL_GPL(fuse_do_open); + +static void fuse_link_write_file(struct file *file) +{ + struct inode *inode = file_inode(file); + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_file *ff = file->private_data; + /* + * file may be written through mmap, so chain it onto the + * inodes's write_file list + */ + spin_lock(&fi->lock); + if (list_empty(&ff->write_entry)) + list_add(&ff->write_entry, &fi->write_files); + spin_unlock(&fi->lock); +} + +int fuse_finish_open(struct inode *inode, struct file *file) +{ + struct fuse_file *ff = file->private_data; + struct fuse_conn *fc = get_fuse_conn(inode); + int err; + + err = fuse_file_io_open(file, inode); + if (err) + return err; + + if (ff->open_flags & FOPEN_STREAM) + stream_open(inode, file); + else if (ff->open_flags & FOPEN_NONSEEKABLE) + nonseekable_open(inode, file); + + if ((file->f_mode & FMODE_WRITE) && fc->writeback_cache) + fuse_link_write_file(file); + + return 0; +} + +static void fuse_truncate_update_attr(struct inode *inode, struct file *file) +{ + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + + spin_lock(&fi->lock); + fi->attr_version = atomic64_inc_return(&fc->attr_version); + i_size_write(inode, 0); + spin_unlock(&fi->lock); + file_update_time(file); + fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE); +} + +static int fuse_open(struct inode *inode, struct file *file) +{ + struct fuse_mount *fm = get_fuse_mount(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_conn *fc = fm->fc; + struct fuse_file *ff; + int err; + bool is_truncate = (file->f_flags & O_TRUNC) && fc->atomic_o_trunc; + bool is_wb_truncate = is_truncate && fc->writeback_cache; + bool dax_truncate = is_truncate && FUSE_IS_DAX(inode); + + if (fuse_is_bad(inode)) + return -EIO; + + err = generic_file_open(inode, file); + if (err) + return err; + + if (is_wb_truncate || dax_truncate) + inode_lock(inode); + + if (dax_truncate) { + filemap_invalidate_lock(inode->i_mapping); + err = fuse_dax_break_layouts(inode, 0, -1); + if (err) + goto out_inode_unlock; + } + + if (is_wb_truncate || dax_truncate) + fuse_set_nowrite(inode); + + err = fuse_do_open(fm, get_node_id(inode), file, false); + if (!err) { + ff = file->private_data; + err = fuse_finish_open(inode, file); + if (err) + fuse_sync_release(fi, ff, file->f_flags); + else if (is_truncate) + fuse_truncate_update_attr(inode, file); + } + + if (is_wb_truncate || dax_truncate) + fuse_release_nowrite(inode); + if (!err) { + if (is_truncate) + truncate_pagecache(inode, 0); + else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) + invalidate_inode_pages2(inode->i_mapping); + } + if (dax_truncate) + filemap_invalidate_unlock(inode->i_mapping); +out_inode_unlock: + if (is_wb_truncate || dax_truncate) + inode_unlock(inode); + + return err; +} + +static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff, + unsigned int flags, int opcode, bool sync) +{ + struct fuse_conn *fc = ff->fm->fc; + struct fuse_release_args *ra = &ff->args->release_args; + + if (fuse_file_passthrough(ff)) + fuse_passthrough_release(ff, fuse_inode_backing(fi)); + + /* Inode is NULL on error path of fuse_create_open() */ + if (likely(fi)) { + spin_lock(&fi->lock); + list_del(&ff->write_entry); + spin_unlock(&fi->lock); + } + spin_lock(&fc->lock); + if (!RB_EMPTY_NODE(&ff->polled_node)) + rb_erase(&ff->polled_node, &fc->polled_files); + spin_unlock(&fc->lock); + + wake_up_interruptible_all(&ff->poll_wait); + + if (!ra) + return; + + /* ff->args was used for open outarg */ + memset(ff->args, 0, sizeof(*ff->args)); + ra->inarg.fh = ff->fh; + ra->inarg.flags = flags; + ra->args.in_numargs = 1; + ra->args.in_args[0].size = sizeof(struct fuse_release_in); + ra->args.in_args[0].value = &ra->inarg; + ra->args.opcode = opcode; + ra->args.nodeid = ff->nodeid; + ra->args.force = true; + ra->args.nocreds = true; + + /* + * Hold inode until release is finished. + * From fuse_sync_release() the refcount is 1 and everything's + * synchronous, so we are fine with not doing igrab() here. + */ + ra->inode = sync ? NULL : igrab(&fi->inode); +} + +void fuse_file_release(struct inode *inode, struct fuse_file *ff, + unsigned int open_flags, fl_owner_t id, bool isdir) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_release_args *ra = &ff->args->release_args; + int opcode = isdir ? FUSE_RELEASEDIR : FUSE_RELEASE; + + fuse_prepare_release(fi, ff, open_flags, opcode, false); + + if (ra && ff->flock) { + ra->inarg.release_flags |= FUSE_RELEASE_FLOCK_UNLOCK; + ra->inarg.lock_owner = fuse_lock_owner_id(ff->fm->fc, id); + } + + /* + * Normally this will send the RELEASE request, however if + * some asynchronous READ or WRITE requests are outstanding, + * the sending will be delayed. + * + * Make the release synchronous if this is a fuseblk mount, + * synchronous RELEASE is allowed (and desirable) in this case + * because the server can be trusted not to screw up. + * + * Always use the asynchronous file put because the current thread + * might be the fuse server. This can happen if a process starts some + * aio and closes the fd before the aio completes. Since aio takes its + * own ref to the file, the IO completion has to drop the ref, which is + * how the fuse server can end up closing its clients' files. + * + * Exception is virtio-fs, which is not affected by the above (server is + * on host, cannot close open files in guest). Virtio-fs needs sync + * release, because the num_waiting mechanism to wait for all requests + * before commencing with fs shutdown doesn't work if submounts are + * used. + */ + fuse_file_put(ff, ff->fm->fc->auto_submounts); +} + +void fuse_release_common(struct file *file, bool isdir) +{ + fuse_file_release(file_inode(file), file->private_data, file->f_flags, + (fl_owner_t) file, isdir); +} + +static int fuse_release(struct inode *inode, struct file *file) +{ + struct fuse_conn *fc = get_fuse_conn(inode); + + /* + * Dirty pages might remain despite write_inode_now() call from + * fuse_flush() due to writes racing with the close. + */ + if (fc->writeback_cache) + write_inode_now(inode, 1); + + fuse_release_common(file, false); + + /* return value is ignored by VFS */ + return 0; +} + +void fuse_sync_release(struct fuse_inode *fi, struct fuse_file *ff, + unsigned int flags) +{ + WARN_ON(refcount_read(&ff->count) > 1); + fuse_prepare_release(fi, ff, flags, FUSE_RELEASE, true); + fuse_file_put(ff, true); +} +EXPORT_SYMBOL_GPL(fuse_sync_release); + +/* + * Scramble the ID space with XTEA, so that the value of the files_struct + * pointer is not exposed to userspace. + */ +u64 fuse_lock_owner_id(struct fuse_conn *fc, fl_owner_t id) +{ + u32 *k = fc->scramble_key; + u64 v = (unsigned long) id; + u32 v0 = v; + u32 v1 = v >> 32; + u32 sum = 0; + int i; + + for (i = 0; i < 32; i++) { + v0 += ((v1 << 4 ^ v1 >> 5) + v1) ^ (sum + k[sum & 3]); + sum += 0x9E3779B9; + v1 += ((v0 << 4 ^ v0 >> 5) + v0) ^ (sum + k[sum>>11 & 3]); + } + + return (u64) v0 + ((u64) v1 << 32); +} + +struct fuse_writepage_args { + struct fuse_io_args ia; + struct list_head queue_entry; + struct inode *inode; + struct fuse_sync_bucket *bucket; +}; + +/* + * Wait for all pending writepages on the inode to finish. + * + * This is currently done by blocking further writes with FUSE_NOWRITE + * and waiting for all sent writes to complete. + * + * This must be called under i_mutex, otherwise the FUSE_NOWRITE usage + * could conflict with truncation. + */ +static void fuse_sync_writes(struct inode *inode) +{ + fuse_set_nowrite(inode); + fuse_release_nowrite(inode); +} + +static int fuse_flush(struct file *file, fl_owner_t id) +{ + struct inode *inode = file_inode(file); + struct fuse_mount *fm = get_fuse_mount(inode); + struct fuse_file *ff = file->private_data; + struct fuse_flush_in inarg; + FUSE_ARGS(args); + int err; + + if (fuse_is_bad(inode)) + return -EIO; + + if (ff->open_flags & FOPEN_NOFLUSH && !fm->fc->writeback_cache) + return 0; + + err = write_inode_now(inode, 1); + if (err) + return err; + + err = filemap_check_errors(file->f_mapping); + if (err) + return err; + + err = 0; + if (fm->fc->no_flush) + goto inval_attr_out; + + memset(&inarg, 0, sizeof(inarg)); + inarg.fh = ff->fh; + inarg.lock_owner = fuse_lock_owner_id(fm->fc, id); + args.opcode = FUSE_FLUSH; + args.nodeid = get_node_id(inode); + args.in_numargs = 1; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + args.force = true; + + err = fuse_simple_request(fm, &args); + if (err == -ENOSYS) { + fm->fc->no_flush = 1; + err = 0; + } + +inval_attr_out: + /* + * In memory i_blocks is not maintained by fuse, if writeback cache is + * enabled, i_blocks from cached attr may not be accurate. + */ + if (!err && fm->fc->writeback_cache) + fuse_invalidate_attr_mask(inode, STATX_BLOCKS); + return err; +} + +int fuse_fsync_common(struct file *file, loff_t start, loff_t end, + int datasync, int opcode) +{ + struct inode *inode = file->f_mapping->host; + struct fuse_mount *fm = get_fuse_mount(inode); + struct fuse_file *ff = file->private_data; + FUSE_ARGS(args); + struct fuse_fsync_in inarg; + + memset(&inarg, 0, sizeof(inarg)); + inarg.fh = ff->fh; + inarg.fsync_flags = datasync ? FUSE_FSYNC_FDATASYNC : 0; + args.opcode = opcode; + args.nodeid = get_node_id(inode); + args.in_numargs = 1; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + return fuse_simple_request(fm, &args); +} + +static int fuse_fsync(struct file *file, loff_t start, loff_t end, + int datasync) +{ + struct inode *inode = file->f_mapping->host; + struct fuse_conn *fc = get_fuse_conn(inode); + int err; + + if (fuse_is_bad(inode)) + return -EIO; + + inode_lock(inode); + + /* + * Start writeback against all dirty pages of the inode, then + * wait for all outstanding writes, before sending the FSYNC + * request. + */ + err = file_write_and_wait_range(file, start, end); + if (err) + goto out; + + fuse_sync_writes(inode); + + /* + * Due to implementation of fuse writeback + * file_write_and_wait_range() does not catch errors. + * We have to do this directly after fuse_sync_writes() + */ + err = file_check_and_advance_wb_err(file); + if (err) + goto out; + + err = sync_inode_metadata(inode, 1); + if (err) + goto out; + + if (fc->no_fsync) + goto out; + + err = fuse_fsync_common(file, start, end, datasync, FUSE_FSYNC); + if (err == -ENOSYS) { + fc->no_fsync = 1; + err = 0; + } +out: + inode_unlock(inode); + + return err; +} + +void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos, + size_t count, int opcode) +{ + struct fuse_file *ff = file->private_data; + struct fuse_args *args = &ia->ap.args; + + ia->read.in.fh = ff->fh; + ia->read.in.offset = pos; + ia->read.in.size = count; + ia->read.in.flags = file->f_flags; + args->opcode = opcode; + args->nodeid = ff->nodeid; + args->in_numargs = 1; + args->in_args[0].size = sizeof(ia->read.in); + args->in_args[0].value = &ia->read.in; + args->out_argvar = true; + args->out_numargs = 1; + args->out_args[0].size = count; +} + +static void fuse_release_user_pages(struct fuse_args_pages *ap, ssize_t nres, + bool should_dirty) +{ + unsigned int i; + + for (i = 0; i < ap->num_folios; i++) { + if (should_dirty) + folio_mark_dirty_lock(ap->folios[i]); + if (ap->args.is_pinned) + unpin_folio(ap->folios[i]); + } + + if (nres > 0 && ap->args.invalidate_vmap) + invalidate_kernel_vmap_range(ap->args.vmap_base, nres); +} + +static void fuse_io_release(struct kref *kref) +{ + kfree(container_of(kref, struct fuse_io_priv, refcnt)); +} + +static ssize_t fuse_get_res_by_io(struct fuse_io_priv *io) +{ + if (io->err) + return io->err; + + if (io->bytes >= 0 && io->write) + return -EIO; + + return io->bytes < 0 ? io->size : io->bytes; +} + +static void fuse_aio_invalidate_worker(struct work_struct *work) +{ + struct fuse_io_priv *io = container_of(work, struct fuse_io_priv, work); + struct address_space *mapping = io->iocb->ki_filp->f_mapping; + ssize_t res = fuse_get_res_by_io(io); + pgoff_t start = io->offset >> PAGE_SHIFT; + pgoff_t end = (io->offset + res - 1) >> PAGE_SHIFT; + + invalidate_inode_pages2_range(mapping, start, end); + io->iocb->ki_complete(io->iocb, res); + kref_put(&io->refcnt, fuse_io_release); +} + +/* + * In case of short read, the caller sets 'pos' to the position of + * actual end of fuse request in IO request. Otherwise, if bytes_requested + * == bytes_transferred or rw == WRITE, the caller sets 'pos' to -1. + * + * An example: + * User requested DIO read of 64K. It was split into two 32K fuse requests, + * both submitted asynchronously. The first of them was ACKed by userspace as + * fully completed (req->out.args[0].size == 32K) resulting in pos == -1. The + * second request was ACKed as short, e.g. only 1K was read, resulting in + * pos == 33K. + * + * Thus, when all fuse requests are completed, the minimal non-negative 'pos' + * will be equal to the length of the longest contiguous fragment of + * transferred data starting from the beginning of IO request. + */ +static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos) +{ + int left; + + spin_lock(&io->lock); + if (err) + io->err = io->err ? : err; + else if (pos >= 0 && (io->bytes < 0 || pos < io->bytes)) + io->bytes = pos; + + left = --io->reqs; + if (!left && io->blocking) + complete(io->done); + spin_unlock(&io->lock); + + if (!left && !io->blocking) { + struct inode *inode = file_inode(io->iocb->ki_filp); + struct address_space *mapping = io->iocb->ki_filp->f_mapping; + ssize_t res = fuse_get_res_by_io(io); + + if (res >= 0) { + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + + spin_lock(&fi->lock); + fi->attr_version = atomic64_inc_return(&fc->attr_version); + spin_unlock(&fi->lock); + } + + if (io->write && res > 0 && mapping->nrpages) { + /* + * As in generic_file_direct_write(), invalidate after the + * write, to invalidate read-ahead cache that may have competed + * with the write. + */ + INIT_WORK(&io->work, fuse_aio_invalidate_worker); + queue_work(inode->i_sb->s_dio_done_wq, &io->work); + return; + } + + io->iocb->ki_complete(io->iocb, res); + } + + kref_put(&io->refcnt, fuse_io_release); +} + +static struct fuse_io_args *fuse_io_alloc(struct fuse_io_priv *io, + unsigned int nfolios) +{ + struct fuse_io_args *ia; + + ia = kzalloc_obj(*ia); + if (ia) { + ia->io = io; + ia->ap.folios = fuse_folios_alloc(nfolios, GFP_KERNEL, + &ia->ap.descs); + if (!ia->ap.folios) { + kfree(ia); + ia = NULL; + } + } + return ia; +} + +static void fuse_io_free(struct fuse_io_args *ia) +{ + kfree(ia->ap.folios); + kfree(ia); +} + +static void fuse_aio_complete_req(struct fuse_args *args, int err) +{ + struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args); + struct fuse_io_priv *io = ia->io; + ssize_t pos = -1; + size_t nres; + + if (err) { + /* Nothing */ + } else if (io->write) { + if (ia->write.out.size > ia->write.in.size) { + err = -EIO; + } else { + nres = ia->write.out.size; + if (ia->write.in.size != ia->write.out.size) + pos = ia->write.in.offset - io->offset + + ia->write.out.size; + } + } else { + u32 outsize = args->out_args[0].size; + + nres = outsize; + if (ia->read.in.size != outsize) + pos = ia->read.in.offset - io->offset + outsize; + } + + fuse_release_user_pages(&ia->ap, err ?: nres, io->should_dirty); + + fuse_aio_complete(io, err, pos); + fuse_io_free(ia); +} + +static ssize_t fuse_async_req_send(struct fuse_mount *fm, + struct fuse_io_args *ia, size_t num_bytes) +{ + ssize_t err; + struct fuse_io_priv *io = ia->io; + + spin_lock(&io->lock); + kref_get(&io->refcnt); + io->size += num_bytes; + io->reqs++; + spin_unlock(&io->lock); + + ia->ap.args.end = fuse_aio_complete_req; + ia->ap.args.may_block = io->should_dirty; + err = fuse_simple_background(fm, &ia->ap.args, GFP_KERNEL); + if (err) + fuse_aio_complete_req(&ia->ap.args, err); + + return num_bytes; +} + +static ssize_t fuse_send_read(struct fuse_io_args *ia, loff_t pos, size_t count, + fl_owner_t owner) +{ + struct file *file = ia->io->iocb->ki_filp; + struct fuse_file *ff = file->private_data; + struct fuse_mount *fm = ff->fm; + + fuse_read_args_fill(ia, file, pos, count, FUSE_READ); + if (owner != NULL) { + ia->read.in.read_flags |= FUSE_READ_LOCKOWNER; + ia->read.in.lock_owner = fuse_lock_owner_id(fm->fc, owner); + } + + if (ia->io->async) + return fuse_async_req_send(fm, ia, count); + + return fuse_simple_request(fm, &ia->ap.args); +} + +static void fuse_read_update_size(struct inode *inode, loff_t size, + u64 attr_ver) +{ + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + + spin_lock(&fi->lock); + if (attr_ver >= fi->attr_version && size < inode->i_size && + !test_bit(FUSE_I_SIZE_UNSTABLE, &fi->state)) { + fi->attr_version = atomic64_inc_return(&fc->attr_version); + i_size_write(inode, size); + } + spin_unlock(&fi->lock); +} + +static void fuse_short_read(struct inode *inode, u64 attr_ver, size_t num_read, + struct fuse_args_pages *ap) +{ + struct fuse_conn *fc = get_fuse_conn(inode); + + /* + * If writeback_cache is enabled, a short read means there's a hole in + * the file. Some data after the hole is in page cache, but has not + * reached the client fs yet. So the hole is not present there. + */ + if (!fc->writeback_cache) { + loff_t pos = folio_pos(ap->folios[0]) + num_read; + fuse_read_update_size(inode, pos, attr_ver); + } +} + +static int fuse_do_readfolio(struct file *file, struct folio *folio, + size_t off, size_t len) +{ + struct inode *inode = folio->mapping->host; + struct fuse_mount *fm = get_fuse_mount(inode); + loff_t pos = folio_pos(folio) + off; + struct fuse_folio_desc desc = { + .offset = off, + .length = len, + }; + struct fuse_io_args ia = { + .ap.args.page_zeroing = true, + .ap.args.out_pages = true, + .ap.num_folios = 1, + .ap.folios = &folio, + .ap.descs = &desc, + }; + ssize_t res; + u64 attr_ver; + + attr_ver = fuse_get_attr_version(fm->fc); + + /* Don't overflow end offset */ + if (pos + (desc.length - 1) == LLONG_MAX) + desc.length--; + + fuse_read_args_fill(&ia, file, pos, desc.length, FUSE_READ); + res = fuse_simple_request(fm, &ia.ap.args); + if (res < 0) + return res; + /* + * Short read means EOF. If file size is larger, truncate it + */ + if (res < desc.length) + fuse_short_read(inode, attr_ver, res, &ia.ap); + + return 0; +} + +static int fuse_iomap_begin(struct inode *inode, loff_t offset, loff_t length, + unsigned int flags, struct iomap *iomap, + struct iomap *srcmap) +{ + iomap->type = IOMAP_MAPPED; + iomap->length = length; + iomap->offset = offset; + return 0; +} + +static DEFINE_IOMAP_ITER_NEXT(fuse_iomap_next, fuse_iomap_begin); + +static const struct iomap_ops fuse_iomap_ops = { + .iomap_next = fuse_iomap_next, +}; + +struct fuse_fill_read_data { + struct file *file; + + /* Fields below are used if sending the read request asynchronously */ + struct fuse_conn *fc; + struct fuse_io_args *ia; + unsigned int nr_bytes; +}; + +/* forward declarations */ +static bool fuse_folios_need_send(struct fuse_conn *fc, loff_t pos, + unsigned len, struct fuse_args_pages *ap, + unsigned cur_bytes, bool write); +static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file, + unsigned int count, bool async); + +static int fuse_handle_readahead(struct folio *folio, + struct readahead_control *rac, + struct fuse_fill_read_data *data, loff_t pos, + size_t len) +{ + struct fuse_io_args *ia = data->ia; + size_t off = offset_in_folio(folio, pos); + struct fuse_conn *fc = data->fc; + struct fuse_args_pages *ap; + unsigned int nr_pages; + + if (ia && fuse_folios_need_send(fc, pos, len, &ia->ap, data->nr_bytes, + false)) { + fuse_send_readpages(ia, data->file, data->nr_bytes, + fc->async_read); + data->nr_bytes = 0; + data->ia = NULL; + ia = NULL; + } + if (!ia) { + if (fuse_chan_num_background(fc->chan) >= fc->congestion_threshold && + rac->ra->async_size >= readahead_count(rac)) + /* + * Congested and only async pages left, so skip the + * rest. + */ + return -EAGAIN; + + nr_pages = min(fc->max_pages, readahead_count(rac)); + data->ia = fuse_io_alloc(NULL, nr_pages); + if (!data->ia) + return -ENOMEM; + ia = data->ia; + } + folio_get(folio); + ap = &ia->ap; + ap->folios[ap->num_folios] = folio; + ap->descs[ap->num_folios].offset = off; + ap->descs[ap->num_folios].length = len; + data->nr_bytes += len; + ap->num_folios++; + + return 0; +} + +static int fuse_iomap_read_folio_range_async(const struct iomap_iter *iter, + struct iomap_read_folio_ctx *ctx, + size_t len) +{ + struct fuse_fill_read_data *data = ctx->read_ctx; + struct folio *folio = ctx->cur_folio; + loff_t pos = iter->pos; + size_t off = offset_in_folio(folio, pos); + struct file *file = data->file; + int ret; + + if (ctx->rac) { + ret = fuse_handle_readahead(folio, ctx->rac, data, pos, len); + } else { + /* + * for non-readahead read requests, do reads synchronously + * since it's not guaranteed that the server can handle + * out-of-order reads + */ + ret = fuse_do_readfolio(file, folio, off, len); + if (!ret) + iomap_finish_folio_read(folio, off, len, ret); + } + return ret; +} + +static const struct iomap_read_ops fuse_iomap_read_ops = { + .read_folio_range = fuse_iomap_read_folio_range_async, +}; + +static int fuse_read_folio(struct file *file, struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + struct fuse_fill_read_data data = { + .file = file, + }; + struct iomap_read_folio_ctx ctx = { + .cur_folio = folio, + .ops = &fuse_iomap_read_ops, + .read_ctx = &data, + + }; + + if (fuse_is_bad(inode)) { + folio_unlock(folio); + return -EIO; + } + + iomap_read_folio(&fuse_iomap_ops, &ctx, NULL); + fuse_invalidate_atime(inode); + return 0; +} + +static int fuse_iomap_read_folio_range(const struct iomap_iter *iter, + struct folio *folio, loff_t pos, + size_t len) +{ + struct file *file = iter->private; + size_t off = offset_in_folio(folio, pos); + + return fuse_do_readfolio(file, folio, off, len); +} + +static void fuse_readpages_end(struct fuse_args *args, int err) +{ + int i; + struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args); + struct fuse_args_pages *ap = &ia->ap; + size_t count = ia->read.in.size; + size_t num_read = args->out_args[0].size; + struct address_space *mapping; + struct inode *inode; + + WARN_ON_ONCE(!ap->num_folios); + mapping = ap->folios[0]->mapping; + inode = mapping->host; + + /* + * Short read means EOF. If file size is larger, truncate it + */ + if (!err && num_read < count) + fuse_short_read(inode, ia->read.attr_ver, num_read, ap); + + fuse_invalidate_atime(inode); + + for (i = 0; i < ap->num_folios; i++) { + iomap_finish_folio_read(ap->folios[i], ap->descs[i].offset, + ap->descs[i].length, err); + folio_put(ap->folios[i]); + } + if (ia->ff) + fuse_file_put(ia->ff, false); + + fuse_io_free(ia); +} + +static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file, + unsigned int count, bool async) +{ + struct fuse_file *ff = file->private_data; + struct fuse_mount *fm = ff->fm; + struct fuse_args_pages *ap = &ia->ap; + loff_t pos = folio_pos(ap->folios[0]); + ssize_t res; + int err; + + ap->args.out_pages = true; + ap->args.page_zeroing = true; + ap->args.page_replace = true; + + /* Don't overflow end offset */ + if (pos + (count - 1) == LLONG_MAX) { + count--; + ap->descs[ap->num_folios - 1].length--; + } + WARN_ON((loff_t) (pos + count) < 0); + + fuse_read_args_fill(ia, file, pos, count, FUSE_READ); + ia->read.attr_ver = fuse_get_attr_version(fm->fc); + if (async) { + ia->ff = fuse_file_get(ff); + ap->args.end = fuse_readpages_end; + err = fuse_simple_background(fm, &ap->args, GFP_KERNEL); + if (!err) + return; + } else { + res = fuse_simple_request(fm, &ap->args); + err = res < 0 ? res : 0; + } + fuse_readpages_end(&ap->args, err); +} + +static void fuse_readahead(struct readahead_control *rac) +{ + struct inode *inode = rac->mapping->host; + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_fill_read_data data = { + .file = rac->file, + .fc = fc, + }; + struct iomap_read_folio_ctx ctx = { + .ops = &fuse_iomap_read_ops, + .rac = rac, + .read_ctx = &data + }; + + if (fuse_is_bad(inode)) + return; + + iomap_readahead(&fuse_iomap_ops, &ctx, NULL); + if (data.ia) + fuse_send_readpages(data.ia, data.file, data.nr_bytes, + fc->async_read); +} + +static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct inode *inode = iocb->ki_filp->f_mapping->host; + struct fuse_conn *fc = get_fuse_conn(inode); + + /* + * In auto invalidate mode, always update attributes on read. + * Otherwise, only update if we attempt to read past EOF (to ensure + * i_size is up to date). + */ + if (fc->auto_inval_data || + (iocb->ki_pos + iov_iter_count(to) > i_size_read(inode))) { + int err; + err = fuse_update_attributes(inode, iocb->ki_filp, STATX_SIZE); + if (err) + return err; + } + + return generic_file_read_iter(iocb, to); +} + +static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff, + loff_t pos, size_t count) +{ + struct fuse_args *args = &ia->ap.args; + + ia->write.in.fh = ff->fh; + ia->write.in.offset = pos; + ia->write.in.size = count; + args->opcode = FUSE_WRITE; + args->nodeid = ff->nodeid; + args->in_numargs = 2; + if (ff->fm->fc->minor < 9) + args->in_args[0].size = FUSE_COMPAT_WRITE_IN_SIZE; + else + args->in_args[0].size = sizeof(ia->write.in); + args->in_args[0].value = &ia->write.in; + args->in_args[1].size = count; + args->out_numargs = 1; + args->out_args[0].size = sizeof(ia->write.out); + args->out_args[0].value = &ia->write.out; +} + +static unsigned int fuse_write_flags(struct kiocb *iocb) +{ + unsigned int flags = iocb->ki_filp->f_flags; + + if (iocb_is_dsync(iocb)) + flags |= O_DSYNC; + if (iocb->ki_flags & IOCB_SYNC) + flags |= O_SYNC; + + return flags; +} + +static ssize_t fuse_send_write(struct fuse_io_args *ia, loff_t pos, + size_t count, fl_owner_t owner) +{ + struct kiocb *iocb = ia->io->iocb; + struct file *file = iocb->ki_filp; + struct fuse_file *ff = file->private_data; + struct fuse_mount *fm = ff->fm; + struct fuse_write_in *inarg = &ia->write.in; + ssize_t err; + + fuse_write_args_fill(ia, ff, pos, count); + inarg->flags = fuse_write_flags(iocb); + if (owner != NULL) { + inarg->write_flags |= FUSE_WRITE_LOCKOWNER; + inarg->lock_owner = fuse_lock_owner_id(fm->fc, owner); + } + + if (ia->io->async) + return fuse_async_req_send(fm, ia, count); + + err = fuse_simple_request(fm, &ia->ap.args); + if (!err && ia->write.out.size > count) + err = -EIO; + + return err ?: ia->write.out.size; +} + +bool fuse_write_update_attr(struct inode *inode, loff_t pos, ssize_t written) +{ + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + bool ret = false; + + spin_lock(&fi->lock); + fi->attr_version = atomic64_inc_return(&fc->attr_version); + if (written > 0 && pos > inode->i_size) { + i_size_write(inode, pos); + ret = true; + } + spin_unlock(&fi->lock); + + fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE); + + return ret; +} + +static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, + struct kiocb *iocb, struct inode *inode, + loff_t pos, size_t count) +{ + struct fuse_args_pages *ap = &ia->ap; + struct file *file = iocb->ki_filp; + struct fuse_file *ff = file->private_data; + struct fuse_mount *fm = ff->fm; + unsigned int i; + int err; + + for (i = 0; i < ap->num_folios; i++) + folio_wait_writeback(ap->folios[i]); + + fuse_write_args_fill(ia, ff, pos, count); + ia->write.in.flags = fuse_write_flags(iocb); + if (fm->fc->handle_killpriv_v2 && !capable(CAP_FSETID)) + ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID; + + err = fuse_simple_request(fm, &ap->args); + if (!err && ia->write.out.size > count) + err = -EIO; + + for (i = 0; i < ap->num_folios; i++) { + struct folio *folio = ap->folios[i]; + + if (ia->write.folio_locked && (i == ap->num_folios - 1)) + folio_unlock(folio); + folio_put(folio); + } + + return err; +} + +static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, + struct address_space *mapping, + struct iov_iter *ii, loff_t pos, + unsigned int max_folios) +{ + struct fuse_args_pages *ap = &ia->ap; + struct fuse_conn *fc = get_fuse_conn(mapping->host); + size_t count = 0; + unsigned int num; + int err = 0; + + num = min(iov_iter_count(ii), fc->max_write); + + ap->args.in_pages = true; + + while (num && ap->num_folios < max_folios) { + size_t tmp; + struct folio *folio; + pgoff_t index = pos >> PAGE_SHIFT; + unsigned int bytes; + unsigned int folio_offset; + + again: + folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN, + mapping_gfp_mask(mapping)); + if (IS_ERR(folio)) { + err = PTR_ERR(folio); + break; + } + + if (mapping_writably_mapped(mapping)) + flush_dcache_folio(folio); + + folio_offset = offset_in_folio(folio, pos); + bytes = min(folio_size(folio) - folio_offset, num); + + tmp = copy_folio_from_iter_atomic(folio, folio_offset, bytes, ii); + flush_dcache_folio(folio); + + if (!tmp) { + folio_unlock(folio); + folio_put(folio); + + /* + * Ensure forward progress by faulting in + * while not holding the folio lock: + */ + if (fault_in_iov_iter_readable(ii, bytes)) { + err = -EFAULT; + break; + } + + goto again; + } + + ap->folios[ap->num_folios] = folio; + ap->descs[ap->num_folios].offset = folio_offset; + ap->descs[ap->num_folios].length = tmp; + ap->num_folios++; + + count += tmp; + pos += tmp; + num -= tmp; + + /* If we copied full folio, mark it uptodate */ + if (tmp == folio_size(folio)) + iomap_folio_mark_uptodate(folio); + + if (folio_test_uptodate(folio)) { + folio_unlock(folio); + } else { + ia->write.folio_locked = true; + break; + } + if (!fc->big_writes) + break; + if (folio_offset + tmp != folio_size(folio)) + break; + } + + return count > 0 ? count : err; +} + +static inline unsigned int fuse_wr_pages(loff_t pos, size_t len, + unsigned int max_pages) +{ + unsigned int pages = ((pos + len - 1) >> PAGE_SHIFT) - + (pos >> PAGE_SHIFT) + 1; + + return min(pages, max_pages); +} + +static ssize_t fuse_perform_write(struct kiocb *iocb, struct iov_iter *ii) +{ + struct address_space *mapping = iocb->ki_filp->f_mapping; + struct inode *inode = mapping->host; + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + loff_t pos = iocb->ki_pos; + int err = 0; + ssize_t res = 0; + + if (inode->i_size < pos + iov_iter_count(ii)) + set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); + + do { + ssize_t count; + struct fuse_io_args ia = {}; + struct fuse_args_pages *ap = &ia.ap; + unsigned int nr_pages = fuse_wr_pages(pos, iov_iter_count(ii), + fc->max_pages); + + ap->folios = fuse_folios_alloc(nr_pages, GFP_KERNEL, &ap->descs); + if (!ap->folios) { + err = -ENOMEM; + break; + } + + count = fuse_fill_write_pages(&ia, mapping, ii, pos, nr_pages); + if (count <= 0) { + err = count; + } else { + err = fuse_send_write_pages(&ia, iocb, inode, + pos, count); + if (!err) { + size_t num_written = ia.write.out.size; + + res += num_written; + pos += num_written; + + /* break out of the loop on short write */ + if (num_written != count) + err = -EIO; + } + } + kfree(ap->folios); + } while (!err && iov_iter_count(ii)); + + fuse_write_update_attr(inode, pos, res); + clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); + + if (!res) + return err; + iocb->ki_pos += res; + return res; +} + +static bool fuse_io_past_eof(struct kiocb *iocb, struct iov_iter *iter) +{ + struct inode *inode = file_inode(iocb->ki_filp); + + return iocb->ki_pos + iov_iter_count(iter) > i_size_read(inode); +} + +/* + * @return true if an exclusive lock for direct IO writes is needed + */ +static bool fuse_dio_wr_exclusive_lock(struct kiocb *iocb, struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct fuse_file *ff = file->private_data; + struct inode *inode = file_inode(iocb->ki_filp); + struct fuse_inode *fi = get_fuse_inode(inode); + + /* Server side has to advise that it supports parallel dio writes. */ + if (!(ff->open_flags & FOPEN_PARALLEL_DIRECT_WRITES)) + return true; + + /* + * Append will need to know the eventual EOF - always needs an + * exclusive lock. + */ + if (iocb->ki_flags & IOCB_APPEND) + return true; + + /* shared locks are not allowed with parallel page cache IO */ + if (test_bit(FUSE_I_CACHE_IO_MODE, &fi->state)) + return true; + + /* Parallel dio beyond EOF is not supported, at least for now. */ + if (fuse_io_past_eof(iocb, from)) + return true; + + return false; +} + +static void fuse_dio_lock(struct kiocb *iocb, struct iov_iter *from, + bool *exclusive) +{ + struct inode *inode = file_inode(iocb->ki_filp); + struct fuse_inode *fi = get_fuse_inode(inode); + + *exclusive = fuse_dio_wr_exclusive_lock(iocb, from); + if (*exclusive) { + inode_lock(inode); + } else { + inode_lock_shared(inode); + /* + * New parallal dio allowed only if inode is not in caching + * mode and denies new opens in caching mode. This check + * should be performed only after taking shared inode lock. + * Previous past eof check was without inode lock and might + * have raced, so check it again. + */ + if (fuse_io_past_eof(iocb, from) || + fuse_inode_uncached_io_start(fi, NULL) != 0) { + inode_unlock_shared(inode); + inode_lock(inode); + *exclusive = true; + } + } +} + +static void fuse_dio_unlock(struct kiocb *iocb, bool exclusive) +{ + struct inode *inode = file_inode(iocb->ki_filp); + struct fuse_inode *fi = get_fuse_inode(inode); + + if (exclusive) { + inode_unlock(inode); + } else { + /* Allow opens in caching mode after last parallel dio end */ + fuse_inode_uncached_io_end(fi); + inode_unlock_shared(inode); + } +} + +static const struct iomap_write_ops fuse_iomap_write_ops = { + .read_folio_range = fuse_iomap_read_folio_range, +}; + +static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct mnt_idmap *idmap = file_mnt_idmap(file); + struct address_space *mapping = file->f_mapping; + ssize_t written = 0; + struct inode *inode = mapping->host; + ssize_t err, count; + struct fuse_conn *fc = get_fuse_conn(inode); + bool writeback = false; + + if (fc->writeback_cache) { + /* Update size (EOF optimization) and mode (SUID clearing) */ + err = fuse_update_attributes(mapping->host, file, + STATX_SIZE | STATX_MODE); + if (err) + return err; + + if (!fc->handle_killpriv_v2 || + !setattr_should_drop_suidgid(idmap, file_inode(file))) + writeback = true; + } + + inode_lock(inode); + + err = count = generic_write_checks(iocb, from); + if (err <= 0) + goto out; + + task_io_account_write(count); + + err = kiocb_modified(iocb); + if (err) + goto out; + + if (iocb->ki_flags & IOCB_DIRECT) { + written = generic_file_direct_write(iocb, from); + if (written < 0 || !iov_iter_count(from)) + goto out; + written = direct_write_fallback(iocb, from, written, + fuse_perform_write(iocb, from)); + } else if (writeback) { + /* + * Use iomap so that we can do granular uptodate reads + * and granular dirty tracking for large folios. + */ + written = iomap_file_buffered_write(iocb, from, + &fuse_iomap_ops, + &fuse_iomap_write_ops, + file); + } else { + written = fuse_perform_write(iocb, from); + } +out: + inode_unlock(inode); + if (written > 0) + written = generic_write_sync(iocb, written); + + return written ? written : err; +} + +static inline unsigned long fuse_get_user_addr(const struct iov_iter *ii) +{ + return (unsigned long)iter_iov(ii)->iov_base + ii->iov_offset; +} + +static inline size_t fuse_get_frag_size(const struct iov_iter *ii, + size_t max_size) +{ + return min(iov_iter_single_seg_count(ii), max_size); +} + +static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii, + size_t *nbytesp, int write, + unsigned int max_pages, + bool use_pages_for_kvec_io) +{ + bool flush_or_invalidate = false; + unsigned int nr_pages = 0; + size_t nbytes = 0; /* # bytes already packed in req */ + ssize_t ret = 0; + + /* Special case for kernel I/O: can copy directly into the buffer. + * However if the implementation of fuse_conn requires pages instead of + * pointer (e.g., virtio-fs), use iov_iter_extract_pages() instead. + */ + if (iov_iter_is_kvec(ii)) { + void *user_addr = (void *)fuse_get_user_addr(ii); + + if (!use_pages_for_kvec_io) { + size_t frag_size = fuse_get_frag_size(ii, *nbytesp); + + if (write) + ap->args.in_args[1].value = user_addr; + else + ap->args.out_args[0].value = user_addr; + + iov_iter_advance(ii, frag_size); + *nbytesp = frag_size; + return 0; + } + + if (is_vmalloc_addr(user_addr)) { + ap->args.vmap_base = user_addr; + flush_or_invalidate = true; + } + } + + /* + * Until there is support for iov_iter_extract_folios(), we have to + * manually extract pages using iov_iter_extract_pages() and then + * copy that to a folios array. + */ + struct page **pages = kcalloc(max_pages, sizeof(struct page *), + GFP_KERNEL); + if (!pages) { + ret = -ENOMEM; + goto out; + } + + while (nbytes < *nbytesp && nr_pages < max_pages) { + unsigned nfolios, i; + size_t start; + + ret = iov_iter_extract_pages(ii, &pages, + *nbytesp - nbytes, + max_pages - nr_pages, + 0, &start); + if (ret < 0) + break; + + nbytes += ret; + + nfolios = DIV_ROUND_UP(ret + start, PAGE_SIZE); + + for (i = 0; i < nfolios; i++) { + struct folio *folio = page_folio(pages[i]); + unsigned int offset = start + + (folio_page_idx(folio, pages[i]) << PAGE_SHIFT); + unsigned int len = umin(ret, PAGE_SIZE - start); + + ap->descs[ap->num_folios].offset = offset; + ap->descs[ap->num_folios].length = len; + ap->folios[ap->num_folios] = folio; + start = 0; + ret -= len; + ap->num_folios++; + } + + nr_pages += nfolios; + } + kfree(pages); + + if (write && flush_or_invalidate) + flush_kernel_vmap_range(ap->args.vmap_base, nbytes); + + ap->args.invalidate_vmap = !write && flush_or_invalidate; + ap->args.is_pinned = iov_iter_extract_will_pin(ii); + ap->args.user_pages = true; + if (write) + ap->args.in_pages = true; + else + ap->args.out_pages = true; + +out: + *nbytesp = nbytes; + + return ret < 0 ? ret : 0; +} + +ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, + loff_t *ppos, int flags) +{ + int write = flags & FUSE_DIO_WRITE; + int cuse = flags & FUSE_DIO_CUSE; + struct file *file = io->iocb->ki_filp; + struct address_space *mapping = file->f_mapping; + struct inode *inode = mapping->host; + struct fuse_file *ff = file->private_data; + struct fuse_conn *fc = ff->fm->fc; + size_t nmax = write ? fc->max_write : fc->max_read; + loff_t pos = *ppos; + size_t count = iov_iter_count(iter); + pgoff_t idx_from = pos >> PAGE_SHIFT; + pgoff_t idx_to = (pos + count - 1) >> PAGE_SHIFT; + ssize_t res = 0; + int err = 0; + struct fuse_io_args *ia; + unsigned int max_pages; + bool fopen_direct_io = ff->open_flags & FOPEN_DIRECT_IO; + + max_pages = iov_iter_npages(iter, fc->max_pages); + ia = fuse_io_alloc(io, max_pages); + if (!ia) + return -ENOMEM; + + if (fopen_direct_io) { + res = filemap_write_and_wait_range(mapping, pos, pos + count - 1); + if (res) { + fuse_io_free(ia); + return res; + } + } + if (!cuse && filemap_range_has_writeback(mapping, pos, (pos + count - 1))) { + if (!write) + inode_lock(inode); + fuse_sync_writes(inode); + if (!write) + inode_unlock(inode); + } + + if (fopen_direct_io && write) { + res = invalidate_inode_pages2_range(mapping, idx_from, idx_to); + if (res) { + fuse_io_free(ia); + return res; + } + } + + io->should_dirty = !write && user_backed_iter(iter); + while (count) { + ssize_t nres; + fl_owner_t owner = current->files; + size_t nbytes = min(count, nmax); + + err = fuse_get_user_pages(&ia->ap, iter, &nbytes, write, + max_pages, fc->use_pages_for_kvec_io); + if (err && !nbytes) + break; + + if (write) { + if (!capable(CAP_FSETID)) + ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID; + + nres = fuse_send_write(ia, pos, nbytes, owner); + } else { + nres = fuse_send_read(ia, pos, nbytes, owner); + } + + if (!io->async || nres < 0) { + fuse_release_user_pages(&ia->ap, nres, io->should_dirty); + fuse_io_free(ia); + } + ia = NULL; + if (nres < 0) { + iov_iter_revert(iter, nbytes); + err = nres; + break; + } + WARN_ON(nres > nbytes); + + count -= nres; + res += nres; + pos += nres; + if (nres != nbytes) { + iov_iter_revert(iter, nbytes - nres); + break; + } + if (count) { + max_pages = iov_iter_npages(iter, fc->max_pages); + ia = fuse_io_alloc(io, max_pages); + if (!ia) + break; + } + } + if (ia) + fuse_io_free(ia); + if (res > 0) + *ppos = pos; + + return res > 0 ? res : err; +} +EXPORT_SYMBOL_GPL(fuse_direct_io); + +static ssize_t __fuse_direct_read(struct fuse_io_priv *io, + struct iov_iter *iter, + loff_t *ppos) +{ + ssize_t res; + struct inode *inode = file_inode(io->iocb->ki_filp); + + res = fuse_direct_io(io, iter, ppos, 0); + + fuse_invalidate_atime(inode); + + return res; +} + +static ssize_t fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter); + +static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + ssize_t res; + + if (!is_sync_kiocb(iocb)) { + res = fuse_direct_IO(iocb, to); + } else { + struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); + + res = __fuse_direct_read(&io, to, &iocb->ki_pos); + } + + return res; +} + +static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + struct inode *inode = file_inode(iocb->ki_filp); + struct address_space *mapping = inode->i_mapping; + loff_t pos = iocb->ki_pos; + ssize_t res; + bool exclusive; + + fuse_dio_lock(iocb, from, &exclusive); + res = generic_write_checks(iocb, from); + if (res > 0) { + task_io_account_write(res); + if (!is_sync_kiocb(iocb)) { + res = fuse_direct_IO(iocb, from); + } else { + struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); + + res = fuse_direct_io(&io, from, &iocb->ki_pos, + FUSE_DIO_WRITE); + fuse_write_update_attr(inode, iocb->ki_pos, res); + } + if (res > 0 && mapping->nrpages) { + /* + * As in generic_file_direct_write(), invalidate after + * write, to invalidate read-ahead cache that may have + * with the write. + */ + invalidate_inode_pages2_range(mapping, + pos >> PAGE_SHIFT, + (pos + res - 1) >> PAGE_SHIFT); + } + } + fuse_dio_unlock(iocb, exclusive); + + return res; +} + +static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct file *file = iocb->ki_filp; + struct fuse_file *ff = file->private_data; + struct inode *inode = file_inode(file); + + if (fuse_is_bad(inode)) + return -EIO; + + if (FUSE_IS_DAX(inode)) + return fuse_dax_read_iter(iocb, to); + + /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ + if (ff->open_flags & FOPEN_DIRECT_IO) + return fuse_direct_read_iter(iocb, to); + else if (fuse_file_passthrough(ff)) + return fuse_passthrough_read_iter(iocb, to); + else + return fuse_cache_read_iter(iocb, to); +} + +static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct fuse_file *ff = file->private_data; + struct inode *inode = file_inode(file); + + if (fuse_is_bad(inode)) + return -EIO; + + if (FUSE_IS_DAX(inode)) + return fuse_dax_write_iter(iocb, from); + + /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ + if (ff->open_flags & FOPEN_DIRECT_IO) + return fuse_direct_write_iter(iocb, from); + else if (fuse_file_passthrough(ff)) + return fuse_passthrough_write_iter(iocb, from); + else + return fuse_cache_write_iter(iocb, from); +} + +static ssize_t fuse_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, + unsigned int flags) +{ + struct fuse_file *ff = in->private_data; + + /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ + if (fuse_file_passthrough(ff) && !(ff->open_flags & FOPEN_DIRECT_IO)) + return fuse_passthrough_splice_read(in, ppos, pipe, len, flags); + else + return filemap_splice_read(in, ppos, pipe, len, flags); +} + +static ssize_t fuse_splice_write(struct pipe_inode_info *pipe, struct file *out, + loff_t *ppos, size_t len, unsigned int flags) +{ + struct fuse_file *ff = out->private_data; + + /* FOPEN_DIRECT_IO overrides FOPEN_PASSTHROUGH */ + if (fuse_file_passthrough(ff) && !(ff->open_flags & FOPEN_DIRECT_IO)) + return fuse_passthrough_splice_write(pipe, out, ppos, len, flags); + else + return iter_file_splice_write(pipe, out, ppos, len, flags); +} + +static void fuse_writepage_free(struct fuse_writepage_args *wpa) +{ + struct fuse_args_pages *ap = &wpa->ia.ap; + + if (wpa->bucket) + fuse_sync_bucket_dec(wpa->bucket); + + fuse_file_put(wpa->ia.ff, false); + + kfree(ap->folios); + kfree(wpa); +} + +static void fuse_writepage_finish(struct fuse_writepage_args *wpa) +{ + struct fuse_args_pages *ap = &wpa->ia.ap; + struct inode *inode = wpa->inode; + struct fuse_inode *fi = get_fuse_inode(inode); + int i; + + for (i = 0; i < ap->num_folios; i++) + /* + * Benchmarks showed that ending writeback within the + * scope of the fi->lock alleviates xarray lock + * contention and noticeably improves performance. + */ + iomap_finish_folio_write(inode, ap->folios[i], + ap->descs[i].length); + + wake_up(&fi->page_waitq); +} + +/* Called under fi->lock, may release and reacquire it */ +static void fuse_send_writepage(struct fuse_mount *fm, + struct fuse_writepage_args *wpa, loff_t size) +__releases(fi->lock) +__acquires(fi->lock) +{ + struct fuse_inode *fi = get_fuse_inode(wpa->inode); + struct fuse_args_pages *ap = &wpa->ia.ap; + struct fuse_write_in *inarg = &wpa->ia.write.in; + struct fuse_args *args = &ap->args; + __u64 data_size = 0; + int err, i; + + for (i = 0; i < ap->num_folios; i++) + data_size += ap->descs[i].length; + + fi->writectr++; + if (inarg->offset + data_size <= size) { + inarg->size = data_size; + } else if (inarg->offset < size) { + inarg->size = size - inarg->offset; + } else { + /* Got truncated off completely */ + goto out_free; + } + + args->in_args[1].size = inarg->size; + args->force = true; + args->nocreds = true; + + err = fuse_simple_background(fm, args, GFP_ATOMIC); + if (err == -ENOMEM) { + spin_unlock(&fi->lock); + err = fuse_simple_background(fm, args, GFP_NOFS | __GFP_NOFAIL); + spin_lock(&fi->lock); + } + + /* Fails on broken connection only */ + if (unlikely(err)) + goto out_free; + + return; + + out_free: + fi->writectr--; + fuse_writepage_finish(wpa); + spin_unlock(&fi->lock); + fuse_writepage_free(wpa); + spin_lock(&fi->lock); +} + +/* + * If fi->writectr is positive (no truncate or fsync going on) send + * all queued writepage requests. + * + * Called with fi->lock + */ +void fuse_flush_writepages(struct inode *inode) +__releases(fi->lock) +__acquires(fi->lock) +{ + struct fuse_mount *fm = get_fuse_mount(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + loff_t crop = i_size_read(inode); + struct fuse_writepage_args *wpa; + + while (fi->writectr >= 0 && !list_empty(&fi->queued_writes)) { + wpa = list_entry(fi->queued_writes.next, + struct fuse_writepage_args, queue_entry); + list_del_init(&wpa->queue_entry); + fuse_send_writepage(fm, wpa, crop); + } +} + +static void fuse_writepage_end(struct fuse_args *args, int error) +{ + struct fuse_writepage_args *wpa = + container_of(args, typeof(*wpa), ia.ap.args); + struct inode *inode = wpa->inode; + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_conn *fc = get_fuse_conn(inode); + + mapping_set_error(inode->i_mapping, error); + /* + * A writeback finished and this might have updated mtime/ctime on + * server making local mtime/ctime stale. Hence invalidate attrs. + * Do this only if writeback_cache is not enabled. If writeback_cache + * is enabled, we trust local ctime/mtime. + */ + if (!fc->writeback_cache) + fuse_invalidate_attr_mask(inode, FUSE_STATX_MODIFY); + spin_lock(&fi->lock); + fi->writectr--; + fuse_writepage_finish(wpa); + spin_unlock(&fi->lock); + fuse_writepage_free(wpa); +} + +static struct fuse_file *__fuse_write_file_get(struct fuse_inode *fi) +{ + struct fuse_file *ff; + + spin_lock(&fi->lock); + ff = list_first_entry_or_null(&fi->write_files, struct fuse_file, + write_entry); + if (ff) + fuse_file_get(ff); + spin_unlock(&fi->lock); + + return ff; +} + +static struct fuse_file *fuse_write_file_get(struct fuse_inode *fi) +{ + struct fuse_file *ff = __fuse_write_file_get(fi); + WARN_ON(!ff); + return ff; +} + +int fuse_write_inode(struct inode *inode, struct writeback_control *wbc) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_file *ff; + int err; + + ff = __fuse_write_file_get(fi); + err = fuse_flush_times(inode, ff); + if (ff) + fuse_file_put(ff, false); + + return err; +} + +static struct fuse_writepage_args *fuse_writepage_args_alloc(void) +{ + struct fuse_writepage_args *wpa; + struct fuse_args_pages *ap; + + wpa = kzalloc_obj(*wpa, GFP_NOFS); + if (wpa) { + ap = &wpa->ia.ap; + ap->num_folios = 0; + ap->folios = fuse_folios_alloc(1, GFP_NOFS, &ap->descs); + if (!ap->folios) { + kfree(wpa); + wpa = NULL; + } + } + return wpa; + +} + +static void fuse_writepage_add_to_bucket(struct fuse_conn *fc, + struct fuse_writepage_args *wpa) +{ + if (!fc->sync_fs) + return; + + rcu_read_lock(); + /* Prevent resurrection of dead bucket in unlikely race with syncfs */ + do { + wpa->bucket = rcu_dereference(fc->curr_bucket); + } while (unlikely(!atomic_inc_not_zero(&wpa->bucket->count))); + rcu_read_unlock(); +} + +static void fuse_writepage_args_page_fill(struct fuse_writepage_args *wpa, struct folio *folio, + uint32_t folio_index, loff_t offset, unsigned len) +{ + struct fuse_args_pages *ap = &wpa->ia.ap; + + ap->folios[folio_index] = folio; + ap->descs[folio_index].offset = offset; + ap->descs[folio_index].length = len; +} + +static struct fuse_writepage_args *fuse_writepage_args_setup(struct folio *folio, + size_t offset, + struct fuse_file *ff) +{ + struct inode *inode = folio->mapping->host; + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_writepage_args *wpa; + struct fuse_args_pages *ap; + + wpa = fuse_writepage_args_alloc(); + if (!wpa) + return NULL; + + fuse_writepage_add_to_bucket(fc, wpa); + fuse_write_args_fill(&wpa->ia, ff, folio_pos(folio) + offset, 0); + wpa->ia.write.in.write_flags |= FUSE_WRITE_CACHE; + wpa->inode = inode; + wpa->ia.ff = ff; + + ap = &wpa->ia.ap; + ap->args.in_pages = true; + ap->args.end = fuse_writepage_end; + + return wpa; +} + +struct fuse_fill_wb_data { + struct fuse_writepage_args *wpa; + struct fuse_file *ff; + unsigned int max_folios; + /* + * nr_bytes won't overflow since fuse_folios_need_send() caps + * wb requests to never exceed fc->max_pages (which has an upper bound + * of U16_MAX). + */ + unsigned int nr_bytes; +}; + +static bool fuse_pages_realloc(struct fuse_fill_wb_data *data, + unsigned int max_pages) +{ + struct fuse_args_pages *ap = &data->wpa->ia.ap; + struct folio **folios; + struct fuse_folio_desc *descs; + unsigned int nfolios = min_t(unsigned int, + max_t(unsigned int, data->max_folios * 2, + FUSE_DEFAULT_MAX_PAGES_PER_REQ), + max_pages); + WARN_ON(nfolios <= data->max_folios); + + folios = fuse_folios_alloc(nfolios, GFP_NOFS, &descs); + if (!folios) + return false; + + memcpy(folios, ap->folios, sizeof(struct folio *) * ap->num_folios); + memcpy(descs, ap->descs, sizeof(struct fuse_folio_desc) * ap->num_folios); + kfree(ap->folios); + ap->folios = folios; + ap->descs = descs; + data->max_folios = nfolios; + + return true; +} + +static void fuse_writepages_send(struct inode *inode, + struct fuse_fill_wb_data *data) +{ + struct fuse_writepage_args *wpa = data->wpa; + struct fuse_inode *fi = get_fuse_inode(inode); + + spin_lock(&fi->lock); + list_add_tail(&wpa->queue_entry, &fi->queued_writes); + fuse_flush_writepages(inode); + spin_unlock(&fi->lock); +} + +static bool fuse_folios_need_send(struct fuse_conn *fc, loff_t pos, + unsigned len, struct fuse_args_pages *ap, + unsigned cur_bytes, bool write) +{ + struct folio *prev_folio; + struct fuse_folio_desc prev_desc; + unsigned bytes = cur_bytes + len; + loff_t prev_pos; + size_t max_bytes = write ? fc->max_write : fc->max_read; + + WARN_ON(!ap->num_folios); + + /* Reached max pages or max folio slots */ + if (ap->num_folios >= fc->max_pages) + return true; + + if (DIV_ROUND_UP(bytes, PAGE_SIZE) > fc->max_pages) + return true; + + if (bytes > max_bytes) + return true; + + /* Discontinuity */ + prev_folio = ap->folios[ap->num_folios - 1]; + prev_desc = ap->descs[ap->num_folios - 1]; + prev_pos = folio_pos(prev_folio) + prev_desc.offset + prev_desc.length; + if (prev_pos != pos) + return true; + + return false; +} + +static ssize_t fuse_iomap_writeback_range(struct iomap_writepage_ctx *wpc, + struct folio *folio, u64 pos, + unsigned len, u64 end_pos) +{ + struct fuse_fill_wb_data *data = wpc->wb_ctx; + struct fuse_writepage_args *wpa = data->wpa; + struct fuse_args_pages *ap = &wpa->ia.ap; + struct inode *inode = wpc->inode; + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_conn *fc = get_fuse_conn(inode); + loff_t offset = offset_in_folio(folio, pos); + + WARN_ON_ONCE(!data); + + if (!data->ff) { + data->ff = fuse_write_file_get(fi); + if (!data->ff) + return -EIO; + } + + if (wpa) { + bool send = fuse_folios_need_send(fc, pos, len, ap, + data->nr_bytes, true); + + if (!send) { + /* + * Need to grow the pages array? If so, did the + * expansion fail? + */ + send = (ap->num_folios == data->max_folios) && + !fuse_pages_realloc(data, fc->max_pages); + } + + if (send) { + fuse_writepages_send(inode, data); + data->wpa = NULL; + data->nr_bytes = 0; + } + } + + if (data->wpa == NULL) { + wpa = fuse_writepage_args_setup(folio, offset, data->ff); + if (!wpa) + return -ENOMEM; + fuse_file_get(wpa->ia.ff); + data->max_folios = 1; + ap = &wpa->ia.ap; + } + + fuse_writepage_args_page_fill(wpa, folio, ap->num_folios, + offset, len); + data->nr_bytes += len; + + ap->num_folios++; + if (!data->wpa) + data->wpa = wpa; + + return len; +} + +static int fuse_iomap_writeback_submit(struct iomap_writepage_ctx *wpc, + int error) +{ + struct fuse_fill_wb_data *data = wpc->wb_ctx; + + WARN_ON_ONCE(!data); + + if (data->wpa) { + WARN_ON(!data->wpa->ia.ap.num_folios); + fuse_writepages_send(wpc->inode, data); + } + + if (data->ff) + fuse_file_put(data->ff, false); + + return error; +} + +static const struct iomap_writeback_ops fuse_writeback_ops = { + .writeback_range = fuse_iomap_writeback_range, + .writeback_submit = fuse_iomap_writeback_submit, +}; + +static int fuse_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct inode *inode = mapping->host; + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_fill_wb_data data = {}; + struct iomap_writepage_ctx wpc = { + .inode = inode, + .iomap.type = IOMAP_MAPPED, + .wbc = wbc, + .ops = &fuse_writeback_ops, + .wb_ctx = &data, + }; + + if (fuse_is_bad(inode)) + return -EIO; + + if (wbc->sync_mode == WB_SYNC_NONE && + fuse_chan_num_background(fc->chan) >= fc->congestion_threshold) + return 0; + + return iomap_writepages(&wpc); +} + +static int fuse_launder_folio(struct folio *folio) +{ + int err = 0; + struct fuse_fill_wb_data data = {}; + struct iomap_writepage_ctx wpc = { + .inode = folio->mapping->host, + .iomap.type = IOMAP_MAPPED, + .ops = &fuse_writeback_ops, + .wb_ctx = &data, + }; + + if (folio_clear_dirty_for_io(folio)) { + err = iomap_writeback_folio(&wpc, folio); + err = fuse_iomap_writeback_submit(&wpc, err); + if (!err) + folio_wait_writeback(folio); + } + return err; +} + +/* + * Write back dirty data/metadata now (there may not be any suitable + * open files later for data) + */ +static void fuse_vma_close(struct vm_area_struct *vma) +{ + int err; + + err = write_inode_now(vma->vm_file->f_mapping->host, 1); + mapping_set_error(vma->vm_file->f_mapping, err); +} + +/* + * Wait for writeback against this page to complete before allowing it + * to be marked dirty again, and hence written back again, possibly + * before the previous writepage completed. + * + * Block here, instead of in ->writepage(), so that the userspace fs + * can only block processes actually operating on the filesystem. + * + * Otherwise unprivileged userspace fs would be able to block + * unrelated: + * + * - page migration + * - sync(2) + * - try_to_free_pages() with order > PAGE_ALLOC_COSTLY_ORDER + */ +static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf) +{ + struct folio *folio = page_folio(vmf->page); + struct inode *inode = file_inode(vmf->vma->vm_file); + + file_update_time(vmf->vma->vm_file); + folio_lock(folio); + if (folio->mapping != inode->i_mapping) { + folio_unlock(folio); + return VM_FAULT_NOPAGE; + } + + folio_wait_writeback(folio); + return VM_FAULT_LOCKED; +} + +static const struct vm_operations_struct fuse_file_vm_ops = { + .close = fuse_vma_close, + .fault = filemap_fault, + .map_pages = filemap_map_pages, + .page_mkwrite = fuse_page_mkwrite, +}; + +static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct fuse_file *ff = file->private_data; + struct fuse_conn *fc = ff->fm->fc; + struct inode *inode = file_inode(file); + int rc; + + /* DAX mmap is superior to direct_io mmap */ + if (FUSE_IS_DAX(inode)) + return fuse_dax_mmap(file, vma); + + /* + * If inode is in passthrough io mode, because it has some file open + * in passthrough mode, either mmap to backing file or fail mmap, + * because mixing cached mmap and passthrough io mode is not allowed. + */ + if (fuse_file_passthrough(ff)) + return fuse_passthrough_mmap(file, vma); + else if (fuse_inode_backing(get_fuse_inode(inode))) + return -ENODEV; + + /* + * FOPEN_DIRECT_IO handling is special compared to O_DIRECT, + * as does not allow MAP_SHARED mmap without FUSE_DIRECT_IO_ALLOW_MMAP. + */ + if (ff->open_flags & FOPEN_DIRECT_IO) { + /* + * Can't provide the coherency needed for MAP_SHARED + * if FUSE_DIRECT_IO_ALLOW_MMAP isn't set. + */ + if ((vma->vm_flags & VM_MAYSHARE) && !fc->direct_io_allow_mmap) + return -ENODEV; + + invalidate_inode_pages2(file->f_mapping); + + if (!(vma->vm_flags & VM_MAYSHARE)) { + /* MAP_PRIVATE */ + return generic_file_mmap(file, vma); + } + + /* + * First mmap of direct_io file enters caching inode io mode. + * Also waits for parallel dio writers to go into serial mode + * (exclusive instead of shared lock). + * After first mmap, the inode stays in caching io mode until + * the direct_io file release. + */ + rc = fuse_file_cached_io_open(inode, ff); + if (rc) + return rc; + } + + if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) + fuse_link_write_file(file); + + file_accessed(file); + vma->vm_ops = &fuse_file_vm_ops; + return 0; +} + +static int convert_fuse_file_lock(struct fuse_conn *fc, + const struct fuse_file_lock *ffl, + struct file_lock *fl) +{ + switch (ffl->type) { + case F_UNLCK: + break; + + case F_RDLCK: + case F_WRLCK: + if (ffl->start > OFFSET_MAX || ffl->end > OFFSET_MAX || + ffl->end < ffl->start) + return -EIO; + + fl->fl_start = ffl->start; + fl->fl_end = ffl->end; + + /* + * Convert pid into init's pid namespace. The locks API will + * translate it into the caller's pid namespace. + */ + rcu_read_lock(); + fl->c.flc_pid = pid_nr_ns(find_pid_ns(ffl->pid, fc->pid_ns), &init_pid_ns); + rcu_read_unlock(); + break; + + default: + return -EIO; + } + fl->c.flc_type = ffl->type; + return 0; +} + +static void fuse_lk_fill(struct fuse_args *args, struct file *file, + const struct file_lock *fl, int opcode, pid_t pid, + int flock, struct fuse_lk_in *inarg) +{ + struct inode *inode = file_inode(file); + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_file *ff = file->private_data; + + memset(inarg, 0, sizeof(*inarg)); + inarg->fh = ff->fh; + inarg->owner = fuse_lock_owner_id(fc, fl->c.flc_owner); + inarg->lk.start = fl->fl_start; + inarg->lk.end = fl->fl_end; + inarg->lk.type = fl->c.flc_type; + inarg->lk.pid = pid; + if (flock) + inarg->lk_flags |= FUSE_LK_FLOCK; + args->opcode = opcode; + args->nodeid = get_node_id(inode); + args->in_numargs = 1; + args->in_args[0].size = sizeof(*inarg); + args->in_args[0].value = inarg; +} + +static int fuse_getlk(struct file *file, struct file_lock *fl) +{ + struct inode *inode = file_inode(file); + struct fuse_mount *fm = get_fuse_mount(inode); + FUSE_ARGS(args); + struct fuse_lk_in inarg; + struct fuse_lk_out outarg; + int err; + + fuse_lk_fill(&args, file, fl, FUSE_GETLK, 0, 0, &inarg); + args.out_numargs = 1; + args.out_args[0].size = sizeof(outarg); + args.out_args[0].value = &outarg; + err = fuse_simple_request(fm, &args); + if (!err) + err = convert_fuse_file_lock(fm->fc, &outarg.lk, fl); + + return err; +} + +static int fuse_setlk(struct file *file, struct file_lock *fl, int flock) +{ + struct inode *inode = file_inode(file); + struct fuse_mount *fm = get_fuse_mount(inode); + FUSE_ARGS(args); + struct fuse_lk_in inarg; + int opcode = (fl->c.flc_flags & FL_SLEEP) ? FUSE_SETLKW : FUSE_SETLK; + struct pid *pid = fl->c.flc_type != F_UNLCK ? task_tgid(current) : NULL; + pid_t pid_nr = pid_nr_ns(pid, fm->fc->pid_ns); + int err; + + if (fl->fl_lmops && fl->fl_lmops->lm_grant) { + /* NLM needs asynchronous locks, which we don't support yet */ + return -ENOLCK; + } + + fuse_lk_fill(&args, file, fl, opcode, pid_nr, flock, &inarg); + err = fuse_simple_request(fm, &args); + + /* locking is restartable */ + if (err == -EINTR) + err = -ERESTARTSYS; + + return err; +} + +static int fuse_file_lock(struct file *file, int cmd, struct file_lock *fl) +{ + struct inode *inode = file_inode(file); + struct fuse_conn *fc = get_fuse_conn(inode); + int err; + + if (cmd == F_CANCELLK) { + err = 0; + } else if (cmd == F_GETLK) { + if (fc->no_lock) { + posix_test_lock(file, fl); + err = 0; + } else + err = fuse_getlk(file, fl); + } else { + if (fc->no_lock) + err = posix_lock_file(file, fl, NULL); + else + err = fuse_setlk(file, fl, 0); + } + return err; +} + +static int fuse_file_flock(struct file *file, int cmd, struct file_lock *fl) +{ + struct inode *inode = file_inode(file); + struct fuse_conn *fc = get_fuse_conn(inode); + int err; + + if (fc->no_flock) { + err = locks_lock_file_wait(file, fl); + } else { + struct fuse_file *ff = file->private_data; + + /* emulate flock with POSIX locks */ + err = fuse_setlk(file, fl, 1); + if (!err) + ff->flock = true; + } + + return err; +} + +static sector_t fuse_bmap(struct address_space *mapping, sector_t block) +{ + struct inode *inode = mapping->host; + struct fuse_mount *fm = get_fuse_mount(inode); + FUSE_ARGS(args); + struct fuse_bmap_in inarg; + struct fuse_bmap_out outarg; + int err; + + if (!inode->i_sb->s_bdev || fm->fc->no_bmap) + return 0; + + memset(&inarg, 0, sizeof(inarg)); + inarg.block = block; + inarg.blocksize = inode->i_sb->s_blocksize; + args.opcode = FUSE_BMAP; + args.nodeid = get_node_id(inode); + args.in_numargs = 1; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + args.out_numargs = 1; + args.out_args[0].size = sizeof(outarg); + args.out_args[0].value = &outarg; + err = fuse_simple_request(fm, &args); + if (err == -ENOSYS) + fm->fc->no_bmap = 1; + + return err ? 0 : outarg.block; +} + +static loff_t fuse_lseek(struct file *file, loff_t offset, int whence) +{ + struct inode *inode = file->f_mapping->host; + struct fuse_mount *fm = get_fuse_mount(inode); + struct fuse_file *ff = file->private_data; + FUSE_ARGS(args); + struct fuse_lseek_in inarg = { + .fh = ff->fh, + .offset = offset, + .whence = whence + }; + struct fuse_lseek_out outarg; + int err; + + if (fm->fc->no_lseek) + goto fallback; + + args.opcode = FUSE_LSEEK; + args.nodeid = ff->nodeid; + args.in_numargs = 1; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + args.out_numargs = 1; + args.out_args[0].size = sizeof(outarg); + args.out_args[0].value = &outarg; + err = fuse_simple_request(fm, &args); + if (err) { + if (err == -ENOSYS) { + fm->fc->no_lseek = 1; + goto fallback; + } + return err; + } + + return vfs_setpos(file, outarg.offset, inode->i_sb->s_maxbytes); + +fallback: + err = fuse_update_attributes(inode, file, STATX_SIZE); + if (!err) + return generic_file_llseek(file, offset, whence); + else + return err; +} + +static loff_t fuse_file_llseek(struct file *file, loff_t offset, int whence) +{ + loff_t retval; + struct inode *inode = file_inode(file); + + switch (whence) { + case SEEK_SET: + case SEEK_CUR: + /* No i_mutex protection necessary for SEEK_CUR and SEEK_SET */ + retval = generic_file_llseek(file, offset, whence); + break; + case SEEK_END: + inode_lock(inode); + retval = fuse_update_attributes(inode, file, STATX_SIZE); + if (!retval) + retval = generic_file_llseek(file, offset, whence); + inode_unlock(inode); + break; + case SEEK_HOLE: + case SEEK_DATA: + inode_lock(inode); + retval = fuse_lseek(file, offset, whence); + inode_unlock(inode); + break; + default: + retval = -EINVAL; + } + + return retval; +} + +static void fuse_do_truncate(struct file *file) +{ + struct inode *inode = file->f_mapping->host; + struct iattr attr; + + attr.ia_valid = ATTR_SIZE; + attr.ia_size = i_size_read(inode); + + attr.ia_file = file; + attr.ia_valid |= ATTR_FILE; + + fuse_do_setattr(file_mnt_idmap(file), file_dentry(file), &attr, file); +} + +static inline loff_t fuse_round_up(struct fuse_conn *fc, loff_t off) +{ + return round_up(off, fc->max_pages << PAGE_SHIFT); +} + +static ssize_t +fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter) +{ + DECLARE_COMPLETION_ONSTACK(wait); + ssize_t ret = 0; + struct file *file = iocb->ki_filp; + struct fuse_file *ff = file->private_data; + loff_t pos = 0; + struct inode *inode; + loff_t i_size; + size_t count = iov_iter_count(iter), shortened = 0; + loff_t offset = iocb->ki_pos; + struct fuse_io_priv *io; + bool async = ff->fm->fc->async_dio; + + pos = offset; + inode = file->f_mapping->host; + i_size = i_size_read(inode); + + if ((iov_iter_rw(iter) == READ) && (offset >= i_size)) + return 0; + + if ((iov_iter_rw(iter) == WRITE) && async && !inode->i_sb->s_dio_done_wq) { + ret = sb_init_dio_done_wq(inode->i_sb); + if (ret < 0) + return ret; + } + + io = kmalloc_obj(struct fuse_io_priv); + if (!io) + return -ENOMEM; + spin_lock_init(&io->lock); + kref_init(&io->refcnt); + io->reqs = 1; + io->bytes = -1; + io->size = 0; + io->offset = offset; + io->write = (iov_iter_rw(iter) == WRITE); + io->err = 0; + /* + * By default, we want to optimize all I/Os with async request + * submission to the client filesystem if supported. + */ + io->async = async; + io->iocb = iocb; + io->blocking = is_sync_kiocb(iocb); + + /* optimization for short read */ + if (io->async && !io->write && offset + count > i_size) { + iov_iter_truncate(iter, fuse_round_up(ff->fm->fc, i_size - offset)); + shortened = count - iov_iter_count(iter); + count -= shortened; + } + + /* + * We cannot asynchronously extend the size of a file. + * In such case the aio will behave exactly like sync io. + */ + if ((offset + count > i_size) && io->write) + io->blocking = true; + + if (io->async && io->blocking) { + /* + * Additional reference to keep io around after + * calling fuse_aio_complete() + */ + kref_get(&io->refcnt); + io->done = &wait; + } + + if (iov_iter_rw(iter) == WRITE) { + ret = fuse_direct_io(io, iter, &pos, FUSE_DIO_WRITE); + fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE); + } else { + ret = __fuse_direct_read(io, iter, &pos); + } + iov_iter_reexpand(iter, iov_iter_count(iter) + shortened); + + if (io->async) { + bool blocking = io->blocking; + + fuse_aio_complete(io, ret < 0 ? ret : 0, -1); + + /* we have a non-extending, async request, so return */ + if (!blocking) + return -EIOCBQUEUED; + + wait_for_completion(&wait); + ret = fuse_get_res_by_io(io); + } + + kref_put(&io->refcnt, fuse_io_release); + + if (iov_iter_rw(iter) == WRITE) { + fuse_write_update_attr(inode, pos, ret); + /* For extending writes we already hold exclusive lock */ + if (ret < 0 && offset + count > i_size) + fuse_do_truncate(file); + } + + return ret; +} + +static int fuse_writeback_range(struct inode *inode, loff_t start, loff_t end) +{ + int err = filemap_write_and_wait_range(inode->i_mapping, start, LLONG_MAX); + + if (!err) + fuse_sync_writes(inode); + + return err; +} + +static long fuse_file_fallocate(struct file *file, int mode, loff_t offset, + loff_t length) +{ + struct fuse_file *ff = file->private_data; + struct inode *inode = file_inode(file); + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_mount *fm = ff->fm; + FUSE_ARGS(args); + struct fuse_fallocate_in inarg = { + .fh = ff->fh, + .offset = offset, + .length = length, + .mode = mode + }; + int err; + bool block_faults = FUSE_IS_DAX(inode) && + (!(mode & FALLOC_FL_KEEP_SIZE) || + (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE))); + + if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE | + FALLOC_FL_ZERO_RANGE)) + return -EOPNOTSUPP; + + if (fm->fc->no_fallocate) + return -EOPNOTSUPP; + + inode_lock(inode); + if (block_faults) { + filemap_invalidate_lock(inode->i_mapping); + err = fuse_dax_break_layouts(inode, 0, -1); + if (err) + goto out; + } + + if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE)) { + loff_t endbyte = offset + length - 1; + + err = fuse_writeback_range(inode, offset, endbyte); + if (err) + goto out; + } + + if (!(mode & FALLOC_FL_KEEP_SIZE) && + offset + length > i_size_read(inode)) { + err = inode_newsize_ok(inode, offset + length); + if (err) + goto out; + } + + err = file_modified(file); + if (err) + goto out; + + if (!(mode & FALLOC_FL_KEEP_SIZE)) + set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); + + args.opcode = FUSE_FALLOCATE; + args.nodeid = ff->nodeid; + args.in_numargs = 1; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + err = fuse_simple_request(fm, &args); + if (err == -ENOSYS) { + fm->fc->no_fallocate = 1; + err = -EOPNOTSUPP; + } + if (err) + goto out; + + /* we could have extended the file */ + if (!(mode & FALLOC_FL_KEEP_SIZE)) { + if (fuse_write_update_attr(inode, offset + length, length)) + file_update_time(file); + } + + if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE)) + truncate_pagecache_range(inode, offset, offset + length - 1); + + fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE); + +out: + if (!(mode & FALLOC_FL_KEEP_SIZE)) + clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); + + if (block_faults) + filemap_invalidate_unlock(inode->i_mapping); + + inode_unlock(inode); + + fuse_flush_time_update(inode); + + return err; +} + +static ssize_t __fuse_copy_file_range(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + size_t len, unsigned int flags) +{ + struct fuse_file *ff_in = file_in->private_data; + struct fuse_file *ff_out = file_out->private_data; + struct inode *inode_in = file_inode(file_in); + struct inode *inode_out = file_inode(file_out); + struct fuse_inode *fi_out = get_fuse_inode(inode_out); + struct fuse_mount *fm = ff_in->fm; + struct fuse_conn *fc = fm->fc; + FUSE_ARGS(args); + struct fuse_copy_file_range_in inarg = { + .fh_in = ff_in->fh, + .off_in = pos_in, + .nodeid_out = ff_out->nodeid, + .fh_out = ff_out->fh, + .off_out = pos_out, + .len = len, + .flags = flags + }; + struct fuse_write_out outarg; + struct fuse_copy_file_range_out outarg_64; + u64 bytes_copied; + ssize_t err; + /* mark unstable when write-back is not used, and file_out gets + * extended */ + bool is_unstable = (!fc->writeback_cache) && + ((pos_out + len) > inode_out->i_size); + + if (fc->no_copy_file_range) + return -EOPNOTSUPP; + + if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb) + return -EXDEV; + + inode_lock(inode_in); + err = fuse_writeback_range(inode_in, pos_in, pos_in + len - 1); + inode_unlock(inode_in); + if (err) + return err; + + inode_lock(inode_out); + + err = file_modified(file_out); + if (err) + goto out; + + /* + * Write out dirty pages in the destination file before sending the COPY + * request to userspace. After the request is completed, truncate off + * pages (including partial ones) from the cache that have been copied, + * since these contain stale data at that point. + * + * This should be mostly correct, but if the COPY writes to partial + * pages (at the start or end) and the parts not covered by the COPY are + * written through a memory map after calling fuse_writeback_range(), + * then these partial page modifications will be lost on truncation. + * + * It is unlikely that someone would rely on such mixed style + * modifications. Yet this does give less guarantees than if the + * copying was performed with write(2). + * + * To fix this a mapping->invalidate_lock could be used to prevent new + * faults while the copy is ongoing. + */ + err = fuse_writeback_range(inode_out, pos_out, pos_out + len - 1); + if (err) + goto out; + + if (is_unstable) + set_bit(FUSE_I_SIZE_UNSTABLE, &fi_out->state); + + args.opcode = FUSE_COPY_FILE_RANGE_64; + args.nodeid = ff_in->nodeid; + args.in_numargs = 1; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + args.out_numargs = 1; + args.out_args[0].size = sizeof(outarg_64); + args.out_args[0].value = &outarg_64; + if (fc->no_copy_file_range_64) { +fallback: + /* Fall back to old op that can't handle large copy length */ + args.opcode = FUSE_COPY_FILE_RANGE; + args.out_args[0].size = sizeof(outarg); + args.out_args[0].value = &outarg; + inarg.len = len = min_t(size_t, len, UINT_MAX & PAGE_MASK); + } + err = fuse_simple_request(fm, &args); + if (err == -ENOSYS) { + if (fc->no_copy_file_range_64) { + fc->no_copy_file_range = 1; + err = -EOPNOTSUPP; + } else { + fc->no_copy_file_range_64 = 1; + goto fallback; + } + } + if (err) + goto out; + + bytes_copied = fc->no_copy_file_range_64 ? + outarg.size : outarg_64.bytes_copied; + + if (bytes_copied > len) { + err = -EIO; + goto out; + } + + truncate_inode_pages_range(inode_out->i_mapping, + ALIGN_DOWN(pos_out, PAGE_SIZE), + ALIGN(pos_out + bytes_copied, PAGE_SIZE) - 1); + + file_update_time(file_out); + fuse_write_update_attr(inode_out, pos_out + bytes_copied, bytes_copied); + + err = bytes_copied; +out: + if (is_unstable) + clear_bit(FUSE_I_SIZE_UNSTABLE, &fi_out->state); + + inode_unlock(inode_out); + file_accessed(file_in); + + fuse_flush_time_update(inode_out); + + return err; +} + +static ssize_t fuse_copy_file_range(struct file *src_file, loff_t src_off, + struct file *dst_file, loff_t dst_off, + size_t len, unsigned int flags) +{ + ssize_t ret; + + ret = __fuse_copy_file_range(src_file, src_off, dst_file, dst_off, + len, flags); + + if (ret == -EOPNOTSUPP || ret == -EXDEV) + ret = splice_copy_file_range(src_file, src_off, dst_file, + dst_off, len); + return ret; +} + +static const struct file_operations fuse_file_operations = { + .llseek = fuse_file_llseek, + .read_iter = fuse_file_read_iter, + .write_iter = fuse_file_write_iter, + .mmap = fuse_file_mmap, + .open = fuse_open, + .flush = fuse_flush, + .release = fuse_release, + .fsync = fuse_fsync, + .lock = fuse_file_lock, + .get_unmapped_area = thp_get_unmapped_area, + .flock = fuse_file_flock, + .splice_read = fuse_splice_read, + .splice_write = fuse_splice_write, + .unlocked_ioctl = fuse_file_ioctl, + .compat_ioctl = fuse_file_compat_ioctl, + .poll = fuse_file_poll, + .fallocate = fuse_file_fallocate, + .copy_file_range = fuse_copy_file_range, + .setlease = generic_setlease, +}; + +static const struct address_space_operations fuse_file_aops = { + .read_folio = fuse_read_folio, + .readahead = fuse_readahead, + .writepages = fuse_writepages, + .launder_folio = fuse_launder_folio, + .dirty_folio = iomap_dirty_folio, + .release_folio = iomap_release_folio, + .invalidate_folio = iomap_invalidate_folio, + .is_partially_uptodate = iomap_is_partially_uptodate, + .migrate_folio = filemap_migrate_folio, + .bmap = fuse_bmap, + .direct_IO = fuse_direct_IO, +}; + +void fuse_init_file_inode(struct inode *inode, unsigned int flags) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + struct fuse_conn *fc = get_fuse_conn(inode); + + inode->i_fop = &fuse_file_operations; + inode->i_data.a_ops = &fuse_file_aops; + if (fc->writeback_cache) + mapping_set_writeback_may_deadlock_on_reclaim(&inode->i_data); + + INIT_LIST_HEAD(&fi->write_files); + INIT_LIST_HEAD(&fi->queued_writes); + fi->writectr = 0; + fi->iocachectr = 0; + init_waitqueue_head(&fi->page_waitq); + init_waitqueue_head(&fi->direct_io_waitq); + + if (IS_ENABLED(CONFIG_FUSE_DAX)) + fuse_dax_inode_init(inode, flags); +} diff --git a/fs/fuse/fuse_dev_i.h b/fs/fuse/fuse_dev_i.h new file mode 100644 index 000000000..668c8391d --- /dev/null +++ b/fs/fuse/fuse_dev_i.h @@ -0,0 +1,422 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * FUSE: Filesystem in Userspace + * Copyright (C) 2001-2008 Miklos Szeredi + */ +#ifndef _FS_FUSE_DEV_I_H +#define _FS_FUSE_DEV_I_H + +#include +#include +#include +#include +#include +#include + +/* Ordinary requests have even IDs, while interrupts IDs are odd */ +#define FUSE_INT_REQ_BIT (1ULL << 0) +#define FUSE_REQ_ID_STEP (1ULL << 1) + +struct fuse_arg; +struct fuse_args; +struct fuse_pqueue; +struct fuse_iqueue; + +/** + * enum fuse_req_flag - Request flags + * + * @FR_ISREPLY: set if the request has reply + * @FR_FORCE: force sending of the request even if interrupted + * @FR_BACKGROUND: request is sent in the background + * @FR_WAITING: request is counted as "waiting" + * @FR_ABORTED: the request was aborted + * @FR_INTERRUPTED: the request has been interrupted + * @FR_LOCKED: data is being copied to/from the request + * @FR_PENDING: request is not yet in userspace + * @FR_SENT: request is in userspace, waiting for an answer + * @FR_FINISHED: request is finished + * @FR_PRIVATE: request is on private list + * @FR_ASYNC: request is asynchronous + * @FR_URING: request is handled through fuse-io-uring + */ +enum fuse_req_flag { + FR_ISREPLY, + FR_FORCE, + FR_BACKGROUND, + FR_WAITING, + FR_ABORTED, + FR_INTERRUPTED, + FR_LOCKED, + FR_PENDING, + FR_SENT, + FR_FINISHED, + FR_PRIVATE, + FR_ASYNC, + FR_URING, +}; + +/** + * struct fuse_req - A request to the client + * + * .waitq.lock protects the following fields: + * - FR_ABORTED + * - FR_LOCKED (may also be modified under fpq->lock, tested under both) + */ +struct fuse_req { + /** + * @list: This can be on either pending processing or io lists in + * fuse_conn + */ + struct list_head list; + + /** @intr_entry: Entry on the interrupts list */ + struct list_head intr_entry; + + /** @args: Input/output arguments */ + struct fuse_args *args; + + /** @count: refcount */ + refcount_t count; + + /** @flags: Request flags, updated with test/set/clear_bit() */ + unsigned long flags; + + /** @in: The request input header */ + struct { + /** @in.h: The request input header */ + struct fuse_in_header h; + } in; + + /** @out: The request output header */ + struct { + /** @out.h: The request output header */ + struct fuse_out_header h; + } out; + + /** @waitq: Used to wake up the task waiting for completion of request */ + wait_queue_head_t waitq; + +#if IS_ENABLED(CONFIG_VIRTIO_FS) + /** + * @argbuf: virtio-fs's physically contiguous buffer for in and out + * args + */ + void *argbuf; +#endif + + /** @chan: fuse_chan this request belongs to */ + struct fuse_chan *chan; + +#ifdef CONFIG_FUSE_IO_URING + void *ring_entry; + void *ring_queue; +#endif + /** @create_time: When (in jiffies) the request was created */ + unsigned long create_time; +}; + +/* One forget request */ +struct fuse_forget_link { + struct fuse_forget_one forget_one; + struct fuse_forget_link *next; +}; + +/** + * struct fuse_iqueue_ops - Input queue callbacks + * + * Input queue signalling is device-specific. For example, the /dev/fuse file + * uses fiq->waitq and fasync to wake processes that are waiting on queue + * readiness. These callbacks allow other device types to respond to input + * queue activity. + */ +struct fuse_iqueue_ops { + /** + * @send_forget: Send one forget + */ + void (*send_forget)(struct fuse_iqueue *fiq, struct fuse_forget_link *link); + + /** + * @send_interrupt: Send interrupt for request + */ + void (*send_interrupt)(struct fuse_iqueue *fiq, struct fuse_req *req); + + /** + * @send_req: Send one request + */ + void (*send_req)(struct fuse_iqueue *fiq, struct fuse_req *req); + + /** + * @release: Clean up when fuse_iqueue is destroyed + */ + void (*release)(struct fuse_iqueue *fiq); +}; + +struct fuse_iqueue { + /** Connection established */ + unsigned connected; + + /** Lock protecting accesses to members of this structure */ + spinlock_t lock; + + /** Readers of the connection are waiting on this */ + wait_queue_head_t waitq; + + /** The next unique request id */ + u64 reqctr; + + /** The list of pending requests */ + struct list_head pending; + + /** Pending interrupts */ + struct list_head interrupts; + + /** Queue of pending forgets */ + struct fuse_forget_link forget_list_head; + struct fuse_forget_link *forget_list_tail; + + /** Batching of FORGET requests (positive indicates FORGET batch) */ + int forget_batch; + + /** O_ASYNC requests */ + struct fasync_struct *fasync; + + /** Device-specific callbacks */ + const struct fuse_iqueue_ops *ops; + + /** Device-specific state */ + void *priv; +}; + +struct fuse_chan { + /** Lock protecting: + - devices + - connected + - ring + - ring->queues[qid] + */ + spinlock_t lock; + + /* back pointer: fc->chan->conn == fc */ + struct fuse_conn *conn; + + /** Input queue */ + struct fuse_iqueue iq; + + /** List of device instances belonging to this connection */ + struct list_head devices; + + /** Maximum number of outstanding background requests */ + unsigned max_background; + + /** Number of requests currently in the background */ + unsigned num_background; + + /** Number of background requests currently queued for userspace */ + unsigned active_background; + + /** The list of background requests set aside for later queuing */ + struct list_head bg_queue; + + /** Protects: max_background, num_background, active_background, bg_queue, blocked */ + spinlock_t bg_lock; + + /** Flag indicating that INIT reply has been received. Allocating + * any fuse request will be suspended until the flag is set */ + int initialized; + + /** Flag indicating if connection is blocked. This will be + the case before the INIT reply is received, and if there + are too many outstading backgrounds requests */ + int blocked; + + /** waitq for blocked connection */ + wait_queue_head_t blocked_waitq; + + /** Connection established, cleared on umount, connection + abort and device release */ + unsigned connected; + + /** The number of requests waiting for completion */ + atomic_t num_waiting; + + /** Is interrupt not implemented by fs? */ + bool no_interrupt; + + /* Use io_uring for communication */ + unsigned int io_uring; + + /* Negotiated minor version */ + unsigned int minor; + + /* Maximum write size */ + unsigned int max_write; + + /* Maximum number of pages that can be used in a single request */ + unsigned int max_pages; + + /* Before being installed into fud, contains the preallocated pq array*/ + struct list_head *pq_prealloc; + + /** Connection aborted via sysfs, respond with ECONNABORTED on device I/O */ + bool abort_with_err; + +#ifdef CONFIG_FUSE_IO_URING + /** uring connection information*/ + struct fuse_ring *ring; +#endif + + /** Only used if the connection opts into request timeouts */ + struct { + /* Worker for checking if any requests have timed out */ + struct delayed_work work; + + /* Request timeout (in jiffies). 0 = no timeout */ + unsigned int req_timeout; + } timeout; +}; + +#define FUSE_PQ_HASH_BITS 8 +#define FUSE_PQ_HASH_SIZE (1 << FUSE_PQ_HASH_BITS) + +struct fuse_pqueue { + /** Connection established */ + unsigned connected; + + /** Lock protecting accessess to members of this structure */ + spinlock_t lock; + + /** Hash table of requests being processed */ + struct list_head *processing; + + /** The list of requests under I/O */ + struct list_head io; +}; + +/** + * struct fuse_dev - Fuse device instance + */ +struct fuse_dev { + /** @ref: Reference count of this object */ + refcount_t ref; + + /** @sync_init: Issue FUSE_INIT synchronously */ + bool sync_init; + + /** @chan: Fuse channel for this device */ + struct fuse_chan *chan; + + /** @pq: Processing queue */ + struct fuse_pqueue pq; + + /** @entry: list entry on fch->devices */ + struct list_head entry; +}; + +struct fuse_copy_state { + struct fuse_req *req; + struct iov_iter *iter; + struct pipe_buffer *pipebufs; + struct pipe_buffer *currbuf; + struct pipe_inode_info *pipe; + unsigned long nr_segs; + struct page *pg; + unsigned int len; + unsigned int offset; + bool write:1; + bool move_folios:1; + bool is_uring:1; + struct { + unsigned int copied_sz; /* copied size into the user buffer */ + } ring; +}; + +/* fud->chan gets assigned to this value when /dev/fuse is closed */ +#define FUSE_DEV_CHAN_DISCONNECTED ((struct fuse_chan *) 1) + +/* + * Lockless access is OK, because fud->chan is set once during mount and is valid + * until the file is released. + * + * fud->chan is set to FUSE_DEV_CHAN_DISCONNECTED only after the containing file is + * released, so result is safe to dereference in most cases. Exceptions are: + * fuse_dev_put() and fuse_fill_super_common(). + */ +static inline struct fuse_chan *fuse_dev_chan_get(struct fuse_dev *fud) +{ + /* Pairs with xchg() in fuse_dev_install() */ + return smp_load_acquire(&fud->chan); +} + +static inline struct fuse_dev *fuse_file_to_fud(struct file *file) +{ + return file->private_data; +} + +static inline struct fuse_dev *__fuse_get_dev(struct file *file) +{ + struct fuse_dev *fud = fuse_file_to_fud(file); + + if (!fuse_dev_chan_get(fud)) + return NULL; + + return fud; +} + +void fuse_iqueue_init(struct fuse_iqueue *fiq, const struct fuse_iqueue_ops *ops, void *priv); + +struct fuse_dev *fuse_get_dev(struct file *file); + +unsigned int fuse_req_hash(u64 unique); +struct fuse_req *fuse_request_find(struct fuse_pqueue *fpq, u64 unique); + +void fuse_dev_end_requests(struct list_head *head); +void fuse_request_bg_finish(struct fuse_chan *fch, struct fuse_req *req); + +void fuse_copy_init(struct fuse_copy_state *cs, bool write, + struct iov_iter *iter); +/* + * Return the number of bytes in an arguments list + */ +unsigned int fuse_len_args(unsigned int numargs, struct fuse_arg *args); + +int fuse_copy_args(struct fuse_copy_state *cs, unsigned int numargs, + unsigned int argpages, struct fuse_arg *args, + int zeroing); +int fuse_copy_out_args(struct fuse_copy_state *cs, struct fuse_args *args, + unsigned int nbytes); +void fuse_dev_queue_forget(struct fuse_iqueue *fiq, + struct fuse_forget_link *forget); +void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req); +bool fuse_remove_pending_req(struct fuse_req *req, spinlock_t *lock); + +bool fuse_request_expired(struct fuse_chan *fch, struct list_head *list); + +/* + * Assign a unique id to a fuse request + */ +void fuse_request_assign_unique(struct fuse_iqueue *fiq, struct fuse_req *req); + +/* + * Get the next unique ID for a request + */ +u64 fuse_get_unique(struct fuse_iqueue *fiq); + +struct fuse_dev *fuse_dev_alloc_install(struct fuse_chan *fch); +struct fuse_dev *fuse_dev_alloc(void); + +int fuse_dev_release(struct inode *inode, struct file *file); + +struct list_head *fuse_pqueue_alloc(void); + +/* + * Initialize the fuse processing queue + */ +void fuse_pqueue_init(struct fuse_pqueue *fpq); + +/* + * End a finished request + */ +void fuse_request_end(struct fuse_req *req); + +#endif + diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h new file mode 100644 index 000000000..85f738c53 --- /dev/null +++ b/fs/fuse/fuse_i.h @@ -0,0 +1,1332 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + FUSE: Filesystem in Userspace + Copyright (C) 2001-2008 Miklos Szeredi +*/ + +#ifndef _FS_FUSE_I_H +#define _FS_FUSE_I_H + +#ifndef pr_fmt +# define pr_fmt(fmt) "fuse: " fmt +#endif + +#include "args.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/** Default max number of pages that can be used in a single read request */ +#define FUSE_DEFAULT_MAX_PAGES_PER_REQ 32 + +/** Bias for fi->writectr, meaning new writepages must not be sent */ +#define FUSE_NOWRITE INT_MIN + +/** Maximum length of a filename, not including terminating null */ + +/* maximum, small enough for FUSE_MIN_READ_BUFFER*/ +#define FUSE_NAME_LOW_MAX 1024 +/* maximum, but needs a request buffer > FUSE_MIN_READ_BUFFER */ +#define FUSE_NAME_MAX (PATH_MAX - 1) + +/** Number of dentries for each connection in the control filesystem */ +#define FUSE_CTL_NUM_DENTRIES 5 + +/* + * Dentries invalidation workqueue period, in seconds. The value of this + * parameter shall be >= FUSE_DENTRY_INVAL_FREQ_MIN seconds, or 0 (zero), in + * which case no workqueue will be created. + */ +extern unsigned inval_wq __read_mostly; + +/** Maximum of max_pages received in init_out */ +extern unsigned int fuse_max_pages_limit; + +/** List of active connections */ +extern struct list_head fuse_conn_list; + +/** Global mutex protecting fuse_conn_list and the control filesystem */ +extern struct mutex fuse_mutex; + +/** Module parameters */ +extern unsigned int max_user_bgreq; +extern unsigned int max_user_congthresh; + +struct fuse_forget_link; + +/** + * struct fuse_submount_lookup - Submount lookup tracking + */ +struct fuse_submount_lookup { + /** @count: Refcount */ + refcount_t count; + + /** + * @nodeid: Unique ID, which identifies the inode between userspace + * and kernel + */ + u64 nodeid; + + /** @forget: The request used for sending the FORGET message */ + struct fuse_forget_link *forget; +}; + +/* Container for data related to mapping to backing file */ +struct fuse_backing { + struct file *file; + const struct cred *cred; + + /* refcount */ + refcount_t count; + struct rcu_head rcu; +}; + +/** + * struct fuse_inode - FUSE inode + */ +struct fuse_inode { + /** @inode: Inode data */ + struct inode inode; + + /** + * @nodeid: Unique ID, which identifies the inode between userspace + * and kernel + */ + u64 nodeid; + + /** @nlookup: Number of lookups on this inode */ + u64 nlookup; + + /** @forget: The request used for sending the FORGET message */ + struct fuse_forget_link *forget; + + /** @i_time: Time in jiffies until the file attributes are valid */ + u64 i_time; + + /** @inval_mask: Which attributes are invalid */ + u32 inval_mask; + + /** + * @orig_i_mode: The sticky bit in inode->i_mode may have been removed, + * so preserve the original mode + */ + umode_t orig_i_mode; + + /** @i_btime: Cache birthtime */ + struct timespec64 i_btime; + + /** @orig_ino: 64-bit inode number */ + u64 orig_ino; + + /** @attr_version: Version of last attribute change */ + u64 attr_version; + + union { + /* read/write io cache (regular file only) */ + struct { + /** + * @write_files: Files usable in writepage. + * Protected by fi->lock + */ + struct list_head write_files; + + /** + * @queued_writes: Writepages pending on truncate or + * fsync + */ + struct list_head queued_writes; + + /** + * @writectr: Number of sent writes, a negative bias + * (FUSE_NOWRITE) means more writes are blocked + */ + int writectr; + + /** @iocachectr: Number of files/maps using page cache */ + int iocachectr; + + /** @page_waitq: Waitq for writepage completion */ + wait_queue_head_t page_waitq; + + /** @direct_io_waitq: waitq for direct-io completion */ + wait_queue_head_t direct_io_waitq; + }; + + /** @rdc: readdir cache (directory only) */ + struct { + /** @cached: true if fully cached */ + bool cached; + + /** @size: size of cache */ + loff_t size; + + /** + * @pos: position at end of cache (position of next + * entry) + */ + loff_t pos; + + /** @version: version of the cache */ + u64 version; + + /** + * @mtime: modification time of directory when cache was + * started + */ + struct timespec64 mtime; + + /** + * @epoch: epoch of fc when cache was started + */ + int epoch; + + /** + * @iversion: iversion of directory when cache was + * started + */ + u64 iversion; + + /** @lock: protects above fields */ + spinlock_t lock; + } rdc; + }; + + /** @state: Miscellaneous bits describing inode state */ + unsigned long state; + + /** + * @mutex: Lock for serializing lookup and readdir for back + * compatibility + */ + struct mutex mutex; + + /** @lock: Lock to protect write-related fields */ + spinlock_t lock; + +#ifdef CONFIG_FUSE_DAX + /** + * @dax: Dax specific inode data + */ + struct fuse_inode_dax *dax; +#endif + /** @submount_lookup: Submount specific lookup tracking */ + struct fuse_submount_lookup *submount_lookup; +#ifdef CONFIG_FUSE_PASSTHROUGH + /** @fb: Reference to backing file in passthrough mode */ + struct fuse_backing *fb; +#endif + + /** + * @cached_i_blkbits: The underlying inode->i_blkbits value will not + * be modified, so preserve the blocksize specified by the server. + */ + u8 cached_i_blkbits; +}; + +/** FUSE inode state bits */ +enum { + /** Advise readdirplus */ + FUSE_I_ADVISE_RDPLUS, + /** Initialized with readdirplus */ + FUSE_I_INIT_RDPLUS, + /** An operation changing file size is in progress */ + FUSE_I_SIZE_UNSTABLE, + /* Bad inode */ + FUSE_I_BAD, + /* Has btime */ + FUSE_I_BTIME, + /* Wants or already has page cache IO */ + FUSE_I_CACHE_IO_MODE, + /* + * Client has exclusive access to the inode, either because fs is local + * or the fuse server has an exclusive "lease" on distributed fs + */ + FUSE_I_EXCLUSIVE, +}; + +struct fuse_conn; +struct fuse_mount; +union fuse_file_args; + +/** + * struct fuse_file - FUSE-specific file data + */ +struct fuse_file { + /** @fm: Fuse connection for this file */ + struct fuse_mount *fm; + + /** @args: Argument space reserved for open/release */ + union fuse_file_args *args; + + /** @kh: Kernel file handle guaranteed to be unique */ + u64 kh; + + /** @fh: File handle used by userspace */ + u64 fh; + + /** @nodeid: Node id of this file */ + u64 nodeid; + + /** @count: Refcount */ + refcount_t count; + + /** @open_flags: FOPEN_* flags returned by open */ + u32 open_flags; + + /** @write_entry: Entry on inode's write_files list */ + struct list_head write_entry; + + /** @readdir: Readdir-related */ + struct { + /** @pos: Dir stream position */ + loff_t pos; + + /** @cache_off: Offset in cache */ + loff_t cache_off; + + /** @version: Version of cache we are reading */ + u64 version; + + } readdir; + + /** @polled_node: RB node to be linked on fuse_conn->polled_files */ + struct rb_node polled_node; + + /** @poll_wait: Wait queue head for poll */ + wait_queue_head_t poll_wait; + + /** @iomode: Does file hold a fi->iocachectr refcount? */ + enum { IOM_NONE, IOM_CACHED, IOM_UNCACHED } iomode; + +#ifdef CONFIG_FUSE_PASSTHROUGH + /** @passthrough: Reference to backing file in passthrough mode */ + struct file *passthrough; + /** @cred: passthrough file credentials */ + const struct cred *cred; +#endif + + /** @flock: Has flock been performed on this file? */ + bool flock:1; +}; + +struct fuse_release_args { + struct fuse_args args; + struct fuse_release_in inarg; + struct inode *inode; +}; + +union fuse_file_args { + /* Used during open() */ + struct fuse_open_out open_outarg; + /* Used during release() */ + struct fuse_release_args release_args; +}; + +#define FUSE_ARGS(args) struct fuse_args args = {} + +/** The request IO state (for asynchronous processing) */ +struct fuse_io_priv { + struct kref refcnt; + struct work_struct work; + int async; + spinlock_t lock; + unsigned reqs; + ssize_t bytes; + size_t size; + __u64 offset; + bool write; + bool should_dirty; + int err; + struct kiocb *iocb; + struct completion *done; + bool blocking; +}; + +#define FUSE_IO_PRIV_SYNC(i) \ +{ \ + .refcnt = KREF_INIT(1), \ + .async = 0, \ + .iocb = i, \ +} + +enum fuse_dax_mode { + FUSE_DAX_INODE_DEFAULT, /* default */ + FUSE_DAX_ALWAYS, /* "-o dax=always" */ + FUSE_DAX_NEVER, /* "-o dax=never" */ + FUSE_DAX_INODE_USER, /* "-o dax=inode" */ +}; + +static inline bool fuse_is_inode_dax_mode(enum fuse_dax_mode mode) +{ + return mode == FUSE_DAX_INODE_DEFAULT || mode == FUSE_DAX_INODE_USER; +} + +struct fuse_fs_context { + struct fuse_dev *fud; + unsigned int rootmode; + kuid_t user_id; + kgid_t group_id; + bool is_bdev:1; + bool rootmode_present:1; + bool user_id_present:1; + bool group_id_present:1; + bool default_permissions:1; + bool allow_other:1; + bool destroy:1; + bool no_control:1; + bool no_force_umount:1; + bool legacy_opts_show:1; + enum fuse_dax_mode dax_mode; + unsigned int max_read; + unsigned int blksize; + const char *subtype; + + /* DAX device, may be NULL */ + struct dax_device *dax_dev; +}; + +struct fuse_sync_bucket { + /* count is a possible scalability bottleneck */ + atomic_t count; + wait_queue_head_t waitq; + struct rcu_head rcu; +}; + +/** + * struct fuse_conn - A Fuse connection. + * + * This structure is created, when the root filesystem is mounted, and + * is destroyed, when the client device is closed and the last + * fuse_mount is destroyed. + */ +struct fuse_conn { + /** + * @lock: Lock protecting: + * - polled_files + * - backing_files_map + * - curr_bucket + */ + spinlock_t lock; + + /** @count: Refcount */ + refcount_t count; + + /** @epoch: Current epoch for up-to-date dentries */ + atomic_t epoch; + + /** @epoch_work: Used to invalidate dentries from old epochs */ + struct work_struct epoch_work; + + /** @rcu: Used to delay freeing fuse_conn, making it safe */ + struct rcu_head rcu; + + /** @user_id: The user id for this mount */ + kuid_t user_id; + + /** @group_id: The group id for this mount */ + kgid_t group_id; + + /** @pid_ns: The pid namespace for this mount */ + struct pid_namespace *pid_ns; + + /** @user_ns: The user namespace for this mount */ + struct user_namespace *user_ns; + + /** @max_read: Maximum read size */ + unsigned max_read; + + /** @max_write: Maximum write size */ + unsigned max_write; + + /** + * @max_pages: Maximum number of pages that can be used in a + * single request + */ + unsigned int max_pages; + + /** + * @max_pages_limit: Constrain ->max_pages to this value during + * feature negotiation + */ + unsigned int max_pages_limit; + + /** @chan: transport layer object */ + struct fuse_chan *chan; + + /** @khctr: The next unique kernel file handle */ + atomic64_t khctr; + + /** + * @polled_files: rbtree of fuse_files waiting for poll events + * indexed by ph + */ + struct rb_root polled_files; + + /** + * @congestion_threshold: Number of background requests at which + * congestion starts + */ + unsigned congestion_threshold; + + /** + * @conn_error: Connection failed (version mismatch). Cannot race with + * setting other bitfields since it is only set once in INIT + * reply, before any other request, and never cleared + */ + unsigned conn_error:1; + + /** @conn_init: Connection successful. Only set in INIT */ + unsigned conn_init:1; + + /** @async_read: Do readahead asynchronously? Only set in INIT */ + unsigned async_read:1; + + /** + * @abort_err: Return an unique read error after abort. + * Only set in INIT + */ + unsigned abort_err:1; + + /** + * @atomic_o_trunc: Do not send separate SETATTR request before + * open(O_TRUNC) + */ + unsigned atomic_o_trunc:1; + + /** + * @export_support: Filesystem supports NFS exporting. + * Only set in INIT + */ + unsigned export_support:1; + + /** @writeback_cache: write-back cache policy (default is write-through) */ + unsigned writeback_cache:1; + + /** + * @parallel_dirops: allow parallel lookups and readdir (default is + * serialized) + */ + unsigned parallel_dirops:1; + + /** + * @handle_killpriv: handle fs handles killing suid/sgid/cap on + * write/chown/trunc + */ + unsigned handle_killpriv:1; + + /** @cache_symlinks: cache READLINK responses in page cache */ + unsigned cache_symlinks:1; + + /** @legacy_opts_show: show legacy mount options */ + unsigned int legacy_opts_show:1; + + /** + * @handle_killpriv_v2: + * fs kills suid/sgid/cap on write/chown/trunc. suid is killed on + * write/trunc only if caller did not have CAP_FSETID. sgid is killed + * on write/truncate only if caller did not have CAP_FSETID as well as + * file has group execute permission. + */ + unsigned handle_killpriv_v2:1; + + /* + * The following bitfields are only for optimization purposes + * and hence races in setting them will not cause malfunction + */ + + /** @no_open: Is open/release not implemented by fs? */ + unsigned no_open:1; + + /** @no_opendir: Is opendir/releasedir not implemented by fs? */ + unsigned no_opendir:1; + + /** @no_fsync: Is fsync not implemented by fs? */ + unsigned no_fsync:1; + + /** @no_fsyncdir: Is fsyncdir not implemented by fs? */ + unsigned no_fsyncdir:1; + + /** @no_flush: Is flush not implemented by fs? */ + unsigned no_flush:1; + + /** @no_setxattr: Is setxattr not implemented by fs? */ + unsigned no_setxattr:1; + + /** @setxattr_ext: Does file server support extended setxattr */ + unsigned setxattr_ext:1; + + /** @no_getxattr: Is getxattr not implemented by fs? */ + unsigned no_getxattr:1; + + /** @no_listxattr: Is listxattr not implemented by fs? */ + unsigned no_listxattr:1; + + /** @no_removexattr: Is removexattr not implemented by fs? */ + unsigned no_removexattr:1; + + /** @no_lock: Are posix file locking primitives not implemented by fs? */ + unsigned no_lock:1; + + /** @no_access: Is access not implemented by fs? */ + unsigned no_access:1; + + /** @no_create: Is create not implemented by fs? */ + unsigned no_create:1; + + /** @no_bmap: Is bmap not implemented by fs? */ + unsigned no_bmap:1; + + /** @no_poll: Is poll not implemented by fs? */ + unsigned no_poll:1; + + /** @big_writes: Do multi-page cached writes */ + unsigned big_writes:1; + + /** @dont_mask: Don't apply umask to creation modes */ + unsigned dont_mask:1; + + /** @no_flock: Are BSD file locking primitives not implemented by fs? */ + unsigned no_flock:1; + + /** @no_fallocate: Is fallocate not implemented by fs? */ + unsigned no_fallocate:1; + + /** @no_rename2: Is rename with flags implemented by fs? */ + unsigned no_rename2:1; + + /** @auto_inval_data: Use enhanced/automatic page cache invalidation. */ + unsigned auto_inval_data:1; + + /** + * @explicit_inval_data: Filesystem is fully responsible for page cache + * invalidation. + */ + unsigned explicit_inval_data:1; + + /** @do_readdirplus: Does the filesystem support readdirplus? */ + unsigned do_readdirplus:1; + + /** @readdirplus_auto: Does the filesystem want adaptive readdirplus? */ + unsigned readdirplus_auto:1; + + /** + * @async_dio: Does the filesystem support asynchronous direct-IO + * submission? + */ + unsigned async_dio:1; + + /** @no_lseek: Is lseek not implemented by fs? */ + unsigned no_lseek:1; + + /** @posix_acl: Does the filesystem support posix acls? */ + unsigned posix_acl:1; + + /** + * @default_permissions: Check permissions based on the file mode + * or not? + */ + unsigned default_permissions:1; + + /** + * @allow_other: Allow other than the mounter user to access the + * filesystem ? + */ + unsigned allow_other:1; + + /** @no_copy_file_range: Does the filesystem support copy_file_range? */ + unsigned no_copy_file_range:1; + + /** + * @no_copy_file_range_64: Does the filesystem support + * copy_file_range_64? + */ + unsigned no_copy_file_range_64:1; + + /** @destroy: Send DESTROY request */ + unsigned int destroy:1; + + /** @delete_stale: Delete dentries that have gone stale */ + unsigned int delete_stale:1; + + /** @no_control: Do not create entry in fusectl fs */ + unsigned int no_control:1; + + /** @no_force_umount: Do not allow MNT_FORCE umount */ + unsigned int no_force_umount:1; + + /** @auto_submounts: Auto-mount submounts announced by the server */ + unsigned int auto_submounts:1; + + /** @sync_fs: Propagate syncfs() to server */ + unsigned int sync_fs:1; + + /** @init_security: Initialize security xattrs when creating a new inode */ + unsigned int init_security:1; + + /** + * @create_supp_group: Add supplementary group info when creating + * a new inode + */ + unsigned int create_supp_group:1; + + /** @inode_dax: Does the filesystem support per inode DAX? */ + unsigned int inode_dax:1; + + /** @no_tmpfile: Is tmpfile not implemented by fs? */ + unsigned int no_tmpfile:1; + + /** + * @direct_io_allow_mmap: Relax restrictions to allow shared mmap + * in FOPEN_DIRECT_IO mode + */ + unsigned int direct_io_allow_mmap:1; + + /** @no_statx: Is statx not implemented by fs? */ + unsigned int no_statx:1; + + /** @passthrough: Passthrough support for read/write IO */ + unsigned int passthrough:1; + + /** @use_pages_for_kvec_io: Use pages instead of pointer for kernel I/O */ + unsigned int use_pages_for_kvec_io:1; + + /** @no_link: Is link not implemented by fs? */ + unsigned int no_link:1; + + /** @sync_init: Is synchronous FUSE_INIT allowed? */ + unsigned int sync_init:1; + + /** @max_stack_depth: Maximum stack depth for passthrough backing files */ + int max_stack_depth; + + /** @minor: Negotiated minor version */ + unsigned minor; + + /** @entry: Entry on the fuse_conn_list */ + struct list_head entry; + + /** @dev: Device ID from the root super block */ + dev_t dev; + + /** @scramble_key: Key for lock owner ID scrambling */ + u32 scramble_key[4]; + + /** @attr_version: Version counter for attribute changes */ + atomic64_t attr_version; + + /** @evict_ctr: Version counter for evict inode */ + atomic64_t evict_ctr; + + /** @name_max: maximum file name length */ + u32 name_max; + + /** @release: Called on final put */ + void (*release)(struct fuse_conn *); + + /** + * @killsb: Read/write semaphore to hold when accessing the sb of any + * fuse_mount belonging to this connection + */ + struct rw_semaphore killsb; + +#ifdef CONFIG_FUSE_DAX + /** @dax_mode: Dax mode */ + enum fuse_dax_mode dax_mode; + + /** @dax: Dax specific conn data, non-NULL if DAX is enabled */ + struct fuse_conn_dax *dax; +#endif + + /** @mounts: List of filesystems using this connection */ + struct list_head mounts; + + /** @curr_bucket: New writepages go into this bucket */ + struct fuse_sync_bucket __rcu *curr_bucket; + +#ifdef CONFIG_FUSE_PASSTHROUGH + /** @backing_files_map: IDR for backing files ids */ + struct idr backing_files_map; +#endif +}; + +/* + * Represents a mounted filesystem, potentially a submount. + * + * This object allows sharing a fuse_conn between separate mounts to + * allow submounts with dedicated superblocks and thus separate device + * IDs. + */ +struct fuse_mount { + /* Underlying (potentially shared) connection to the FUSE server */ + struct fuse_conn *fc; + + /* + * Super block for this connection (fc->killsb must be held when + * accessing this). + */ + struct super_block *sb; + + /* Entry on fc->mounts */ + struct list_head fc_entry; + struct rcu_head rcu; +}; + +/* + * Empty header for FUSE opcodes without specific header needs. + * Used as a placeholder in args->in_args[0] for consistency + * across all FUSE operations, simplifying request handling. + */ +struct fuse_zero_header {}; + +static inline void fuse_set_zero_arg0(struct fuse_args *args) +{ + args->in_args[0].size = sizeof(struct fuse_zero_header); + args->in_args[0].value = NULL; +} + +static inline struct fuse_mount *get_fuse_mount_super(struct super_block *sb) +{ + return sb->s_fs_info; +} + +static inline struct fuse_conn *get_fuse_conn_super(struct super_block *sb) +{ + return get_fuse_mount_super(sb)->fc; +} + +static inline struct fuse_mount *get_fuse_mount(struct inode *inode) +{ + return get_fuse_mount_super(inode->i_sb); +} + +static inline struct fuse_conn *get_fuse_conn(struct inode *inode) +{ + return get_fuse_mount_super(inode->i_sb)->fc; +} + +static inline struct fuse_inode *get_fuse_inode(const struct inode *inode) +{ + return container_of(inode, struct fuse_inode, inode); +} + +static inline u64 get_node_id(struct inode *inode) +{ + return get_fuse_inode(inode)->nodeid; +} + +static inline int invalid_nodeid(u64 nodeid) +{ + return !nodeid || nodeid == FUSE_ROOT_ID; +} + +static inline u64 fuse_get_attr_version(struct fuse_conn *fc) +{ + return atomic64_read(&fc->attr_version); +} + +static inline u64 fuse_get_evict_ctr(struct fuse_conn *fc) +{ + return atomic64_read(&fc->evict_ctr); +} + +static inline bool fuse_stale_inode(const struct inode *inode, int generation, + struct fuse_attr *attr) +{ + return inode->i_generation != generation || + inode_wrong_type(inode, attr->mode); +} + +static inline void fuse_make_bad(struct inode *inode) +{ + set_bit(FUSE_I_BAD, &get_fuse_inode(inode)->state); +} + +static inline bool fuse_is_bad(struct inode *inode) +{ + return unlikely(test_bit(FUSE_I_BAD, &get_fuse_inode(inode)->state)); +} + +static inline bool fuse_inode_is_exclusive(const struct inode *inode) +{ + const struct fuse_inode *fi = get_fuse_inode(inode); + + return test_bit(FUSE_I_EXCLUSIVE, &fi->state); +} + +static inline struct folio **fuse_folios_alloc(unsigned int nfolios, gfp_t flags, + struct fuse_folio_desc **desc) +{ + struct folio **folios; + + folios = kzalloc(nfolios * (sizeof(struct folio *) + + sizeof(struct fuse_folio_desc)), flags); + *desc = (void *) (folios + nfolios); + + return folios; +} + +static inline void fuse_folio_descs_length_init(struct fuse_folio_desc *descs, + unsigned int index, + unsigned int nr_folios) +{ + int i; + + for (i = index; i < index + nr_folios; i++) + descs[i].length = PAGE_SIZE - descs[i].offset; +} + +static inline void fuse_sync_bucket_dec(struct fuse_sync_bucket *bucket) +{ + /* Need RCU protection to prevent use after free after the decrement */ + rcu_read_lock(); + if (atomic_dec_and_test(&bucket->count)) + wake_up(&bucket->waitq); + rcu_read_unlock(); +} + +/** Device operations */ +extern const struct file_operations fuse_dev_operations; + +extern const struct dentry_operations fuse_dentry_operations; + +/* + * Get a filled in inode + */ +struct inode *fuse_iget(struct super_block *sb, u64 nodeid, + int generation, struct fuse_attr *attr, + u64 attr_valid, u64 attr_version, + u64 evict_ctr); + +int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name, + struct fuse_entry_out *outarg, struct inode **inode); + +/* + * Initialize READ or READDIR request + */ +struct fuse_io_args { + union { + struct { + struct fuse_read_in in; + u64 attr_ver; + } read; + struct { + struct fuse_write_in in; + struct fuse_write_out out; + bool folio_locked; + } write; + }; + struct fuse_args_pages ap; + struct fuse_io_priv *io; + struct fuse_file *ff; +}; + +void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos, + size_t count, int opcode); + + +struct fuse_file *fuse_file_alloc(struct fuse_mount *fm, bool release); +void fuse_file_free(struct fuse_file *ff); +int fuse_finish_open(struct inode *inode, struct file *file); + +void fuse_sync_release(struct fuse_inode *fi, struct fuse_file *ff, + unsigned int flags); + +/* + * Send RELEASE or RELEASEDIR request + */ +void fuse_release_common(struct file *file, bool isdir); + +/* + * Send FSYNC or FSYNCDIR request + */ +int fuse_fsync_common(struct file *file, loff_t start, loff_t end, + int datasync, int opcode); + +/* + * Notify poll wakeup + */ +int fuse_notify_poll_wakeup(struct fuse_conn *fc, + struct fuse_notify_poll_wakeup_out *outarg); + +/* + * Initialize file operations on a regular file + */ +void fuse_init_file_inode(struct inode *inode, unsigned int flags); + +/* + * Initialize inode operations on regular files and special files + */ +void fuse_init_common(struct inode *inode); + +/* + * Initialize inode and file operations on a directory + */ +void fuse_init_dir(struct inode *inode); + +/* + * Initialize inode operations on a symlink + */ +void fuse_init_symlink(struct inode *inode); + +/* + * Change attributes of an inode + */ +void fuse_change_attributes(struct inode *inode, struct fuse_attr *attr, + struct fuse_statx *sx, + u64 attr_valid, u64 attr_version); + +void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr, + struct fuse_statx *sx, + u64 attr_valid, u32 cache_mask, + u64 evict_ctr); + +u32 fuse_get_cache_mask(struct inode *inode); + +int fuse_ctl_init(void); +void __exit fuse_ctl_cleanup(void); + +/* + * Simple request sending that does request allocation and freeing + */ +ssize_t __fuse_simple_request(struct mnt_idmap *idmap, + struct fuse_mount *fm, + struct fuse_args *args); + +static inline ssize_t fuse_simple_request(struct fuse_mount *fm, struct fuse_args *args) +{ + return __fuse_simple_request(&invalid_mnt_idmap, fm, args); +} + +static inline ssize_t fuse_simple_idmap_request(struct mnt_idmap *idmap, + struct fuse_mount *fm, + struct fuse_args *args) +{ + return __fuse_simple_request(idmap, fm, args); +} + +int fuse_simple_background(struct fuse_mount *fm, struct fuse_args *args, + gfp_t gfp_flags); +int fuse_simple_notify_reply(struct fuse_mount *fm, struct fuse_args *args, u64 unique); + +void fuse_dentry_tree_init(void); +void fuse_dentry_tree_cleanup(void); + +void fuse_epoch_work(struct work_struct *work); + +/* + * Invalidate inode attributes + */ + +/* Attributes possibly changed on data modification */ +#define FUSE_STATX_MODIFY (STATX_MTIME | STATX_CTIME | STATX_BLOCKS) + +/* Attributes possibly changed on data and/or size modification */ +#define FUSE_STATX_MODSIZE (FUSE_STATX_MODIFY | STATX_SIZE) + +/* Attributes possibly changed on directory modification */ +#define FUSE_STATX_MODDIR (FUSE_STATX_MODSIZE | STATX_NLINK) + +void fuse_invalidate_attr(struct inode *inode); +void fuse_invalidate_attr_mask(struct inode *inode, u32 mask); + +void fuse_invalidate_entry_cache(struct dentry *entry); + +void fuse_invalidate_atime(struct inode *inode); + +u64 fuse_time_to_jiffies(u64 sec, u32 nsec); +#define ATTR_TIMEOUT(o) \ + fuse_time_to_jiffies((o)->attr_valid, (o)->attr_valid_nsec) + +void fuse_change_entry_timeout(struct dentry *entry, struct fuse_entry_out *o); + +/* + * Initialize fuse_conn + */ +void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm, + struct user_namespace *user_ns, struct fuse_chan *fch); + +int fuse_send_init(struct fuse_mount *fm); + +/** + * fuse_fill_super_common - Fill in superblock and initialize fuse connection + * @sb: partially-initialized superblock to fill in + * @ctx: mount context + */ +int fuse_fill_super_common(struct super_block *sb, struct fuse_fs_context *ctx); + +/** + * fuse_mount_remove - Remove the mount from the connection + * @fm: fuse_mount to remove + * + * Returns: whether this was the last mount + */ +bool fuse_mount_remove(struct fuse_mount *fm); + +/* + * Setup context ops for submounts + */ +int fuse_init_fs_context_submount(struct fs_context *fsc); + +/* + * Shut down the connection (possibly sending DESTROY request). + */ +void fuse_conn_destroy(struct fuse_mount *fm); + +/* Drop the connection and free the fuse mount */ +void fuse_mount_destroy(struct fuse_mount *fm); + +/** + * fuse_ctl_add_conn - Add connection to control filesystem + * @fc: Fuse connection to add + */ +int fuse_ctl_add_conn(struct fuse_conn *fc); + +/** + * fuse_ctl_remove_conn - Remove connection from control filesystem + * @fc: Fuse connection to remove + */ +void fuse_ctl_remove_conn(struct fuse_conn *fc); + +/* + * Is file type valid? + */ +int fuse_valid_type(int m); + +bool fuse_invalid_attr(struct fuse_attr *attr); + +/* + * Is current process allowed to perform filesystem operation? + */ +bool fuse_allow_current_process(struct fuse_conn *fc); + +u64 fuse_lock_owner_id(struct fuse_conn *fc, fl_owner_t id); + +void fuse_flush_time_update(struct inode *inode); +void fuse_update_ctime(struct inode *inode); + +int fuse_update_attributes(struct inode *inode, struct file *file, u32 mask); + +void fuse_flush_writepages(struct inode *inode); + +void fuse_set_nowrite(struct inode *inode); +void fuse_release_nowrite(struct inode *inode); + +/* + * Scan all fuse_mounts belonging to fc to find the first where + * ilookup5() returns a result. Return that result and the + * respective fuse_mount in *fm (unless fm is NULL). + * + * The caller must hold fc->killsb. + */ +struct inode *fuse_ilookup(struct fuse_conn *fc, u64 nodeid, + struct fuse_mount **fm); + +/* + * File-system tells the kernel to invalidate cache for the given node id. + */ +int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, + loff_t offset, loff_t len); + +/* + * File-system tells the kernel to invalidate parent attributes and + * the dentry matching parent/name. + * + * If the child_nodeid is non-zero and: + * - matches the inode number for the dentry matching parent/name, + * - is not a mount point + * - is a file or oan empty directory + * then the dentry is unhashed (d_delete()). + */ +int fuse_reverse_inval_entry(struct fuse_conn *fc, u64 parent_nodeid, + u64 child_nodeid, struct qstr *name, u32 flags); + +/* + * Try to prune this inode. If neither the inode itself nor dentries associated + * with this inode have any external reference, then the inode can be freed. + */ +void fuse_try_prune_one_inode(struct fuse_conn *fc, u64 nodeid); + +int fuse_do_open(struct fuse_mount *fm, u64 nodeid, struct file *file, + bool isdir); + +/* + * fuse_direct_io() flags + */ + +/** If set, it is WRITE; otherwise - READ */ +#define FUSE_DIO_WRITE (1 << 0) + +/** CUSE pass fuse_direct_io() a file which f_mapping->host is not from FUSE */ +#define FUSE_DIO_CUSE (1 << 1) + +ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, + loff_t *ppos, int flags); +long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, + unsigned int flags); +long fuse_ioctl_common(struct file *file, unsigned int cmd, + unsigned long arg, unsigned int flags); +__poll_t fuse_file_poll(struct file *file, poll_table *wait); + +bool fuse_write_update_attr(struct inode *inode, loff_t pos, ssize_t written); + +int fuse_flush_times(struct inode *inode, struct fuse_file *ff); +int fuse_write_inode(struct inode *inode, struct writeback_control *wbc); + +int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr, struct file *file); + +void fuse_unlock_inode(struct inode *inode, bool locked); +bool fuse_lock_inode(struct inode *inode); + +int fuse_setxattr(struct inode *inode, const char *name, const void *value, + size_t size, int flags, unsigned int extra_flags); +ssize_t fuse_getxattr(struct inode *inode, const char *name, void *value, + size_t size); +ssize_t fuse_listxattr(struct dentry *entry, char *list, size_t size); +int fuse_removexattr(struct inode *inode, const char *name); +extern const struct xattr_handler * const fuse_xattr_handlers[]; + +struct posix_acl; +struct posix_acl *fuse_get_inode_acl(struct inode *inode, int type, bool rcu); +struct posix_acl *fuse_get_acl(struct mnt_idmap *idmap, + struct dentry *dentry, int type); +int fuse_set_acl(struct mnt_idmap *, struct dentry *dentry, + struct posix_acl *acl, int type); + +/* readdir.c */ +int fuse_readdir(struct file *file, struct dir_context *ctx); + +void fuse_free_conn(struct fuse_conn *fc); + +/* dax.c */ + +#define FUSE_IS_DAX(inode) (IS_ENABLED(CONFIG_FUSE_DAX) && IS_DAX(inode)) + +ssize_t fuse_dax_read_iter(struct kiocb *iocb, struct iov_iter *to); +ssize_t fuse_dax_write_iter(struct kiocb *iocb, struct iov_iter *from); +int fuse_dax_mmap(struct file *file, struct vm_area_struct *vma); +int fuse_dax_break_layouts(struct inode *inode, u64 dmap_start, u64 dmap_end); +int fuse_dax_conn_alloc(struct fuse_conn *fc, enum fuse_dax_mode mode, + struct dax_device *dax_dev); +void fuse_dax_conn_free(struct fuse_conn *fc); +bool fuse_dax_inode_alloc(struct super_block *sb, struct fuse_inode *fi); +void fuse_dax_inode_init(struct inode *inode, unsigned int flags); +void fuse_dax_inode_cleanup(struct inode *inode); +void fuse_dax_dontcache(struct inode *inode, unsigned int flags); +bool fuse_dax_check_alignment(struct fuse_conn *fc, unsigned int map_alignment); +void fuse_dax_cancel_work(struct fuse_conn *fc); + +/* ioctl.c */ +long fuse_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg); +long fuse_file_compat_ioctl(struct file *file, unsigned int cmd, + unsigned long arg); +int fuse_fileattr_get(struct dentry *dentry, struct file_kattr *fa); +int fuse_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa); + +/* iomode.c */ +int fuse_file_cached_io_open(struct inode *inode, struct fuse_file *ff); +int fuse_inode_uncached_io_start(struct fuse_inode *fi, + struct fuse_backing *fb); +void fuse_inode_uncached_io_end(struct fuse_inode *fi); + +int fuse_file_io_open(struct file *file, struct inode *inode); +void fuse_file_io_release(struct fuse_file *ff, struct inode *inode); + +/* file.c */ +struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid, + unsigned int open_flags, bool isdir); +void fuse_file_release(struct inode *inode, struct fuse_file *ff, + unsigned int open_flags, fl_owner_t id, bool isdir); + +/* backing.c */ +#ifdef CONFIG_FUSE_PASSTHROUGH +struct fuse_backing *fuse_backing_get(struct fuse_backing *fb); +void fuse_backing_put(struct fuse_backing *fb); +struct fuse_backing *fuse_backing_lookup(struct fuse_conn *fc, int backing_id); +#else + +static inline struct fuse_backing *fuse_backing_get(struct fuse_backing *fb) +{ + return NULL; +} + +static inline void fuse_backing_put(struct fuse_backing *fb) +{ +} +static inline struct fuse_backing *fuse_backing_lookup(struct fuse_conn *fc, + int backing_id) +{ + return NULL; +} +#endif + +void fuse_backing_files_init(struct fuse_conn *fc); +void fuse_backing_files_free(struct fuse_conn *fc); + +/* passthrough.c */ +static inline struct fuse_backing *fuse_inode_backing(struct fuse_inode *fi) +{ +#ifdef CONFIG_FUSE_PASSTHROUGH + return READ_ONCE(fi->fb); +#else + return NULL; +#endif +} + +static inline struct fuse_backing *fuse_inode_backing_set(struct fuse_inode *fi, + struct fuse_backing *fb) +{ +#ifdef CONFIG_FUSE_PASSTHROUGH + return xchg(&fi->fb, fb); +#else + return NULL; +#endif +} + +struct fuse_backing *fuse_passthrough_open(struct file *file, int backing_id); +void fuse_passthrough_release(struct fuse_file *ff, struct fuse_backing *fb); + +static inline struct file *fuse_file_passthrough(struct fuse_file *ff) +{ +#ifdef CONFIG_FUSE_PASSTHROUGH + return ff->passthrough; +#else + return NULL; +#endif +} + +ssize_t fuse_passthrough_read_iter(struct kiocb *iocb, struct iov_iter *iter); +ssize_t fuse_passthrough_write_iter(struct kiocb *iocb, struct iov_iter *iter); +ssize_t fuse_passthrough_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, + size_t len, unsigned int flags); +ssize_t fuse_passthrough_splice_write(struct pipe_inode_info *pipe, + struct file *out, loff_t *ppos, + size_t len, unsigned int flags); +ssize_t fuse_passthrough_mmap(struct file *file, struct vm_area_struct *vma); + +#ifdef CONFIG_SYSCTL +extern int fuse_sysctl_register(void); +extern void fuse_sysctl_unregister(void); +#else +#define fuse_sysctl_register() (0) +#define fuse_sysctl_unregister() do { } while (0) +#endif /* CONFIG_SYSCTL */ + +#endif /* _FS_FUSE_I_H */ diff --git a/fs/fuse/fuse_trace.h b/fs/fuse/fuse_trace.h new file mode 100644 index 000000000..60baa10bb --- /dev/null +++ b/fs/fuse/fuse_trace.h @@ -0,0 +1,154 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM fuse + +#if !defined(_TRACE_FUSE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_FUSE_H + +#include + +#define OPCODES \ + EM( FUSE_LOOKUP, "FUSE_LOOKUP") \ + EM( FUSE_FORGET, "FUSE_FORGET") \ + EM( FUSE_GETATTR, "FUSE_GETATTR") \ + EM( FUSE_SETATTR, "FUSE_SETATTR") \ + EM( FUSE_READLINK, "FUSE_READLINK") \ + EM( FUSE_SYMLINK, "FUSE_SYMLINK") \ + EM( FUSE_MKNOD, "FUSE_MKNOD") \ + EM( FUSE_MKDIR, "FUSE_MKDIR") \ + EM( FUSE_UNLINK, "FUSE_UNLINK") \ + EM( FUSE_RMDIR, "FUSE_RMDIR") \ + EM( FUSE_RENAME, "FUSE_RENAME") \ + EM( FUSE_LINK, "FUSE_LINK") \ + EM( FUSE_OPEN, "FUSE_OPEN") \ + EM( FUSE_READ, "FUSE_READ") \ + EM( FUSE_WRITE, "FUSE_WRITE") \ + EM( FUSE_STATFS, "FUSE_STATFS") \ + EM( FUSE_RELEASE, "FUSE_RELEASE") \ + EM( FUSE_FSYNC, "FUSE_FSYNC") \ + EM( FUSE_SETXATTR, "FUSE_SETXATTR") \ + EM( FUSE_GETXATTR, "FUSE_GETXATTR") \ + EM( FUSE_LISTXATTR, "FUSE_LISTXATTR") \ + EM( FUSE_REMOVEXATTR, "FUSE_REMOVEXATTR") \ + EM( FUSE_FLUSH, "FUSE_FLUSH") \ + EM( FUSE_INIT, "FUSE_INIT") \ + EM( FUSE_OPENDIR, "FUSE_OPENDIR") \ + EM( FUSE_READDIR, "FUSE_READDIR") \ + EM( FUSE_RELEASEDIR, "FUSE_RELEASEDIR") \ + EM( FUSE_FSYNCDIR, "FUSE_FSYNCDIR") \ + EM( FUSE_GETLK, "FUSE_GETLK") \ + EM( FUSE_SETLK, "FUSE_SETLK") \ + EM( FUSE_SETLKW, "FUSE_SETLKW") \ + EM( FUSE_ACCESS, "FUSE_ACCESS") \ + EM( FUSE_CREATE, "FUSE_CREATE") \ + EM( FUSE_INTERRUPT, "FUSE_INTERRUPT") \ + EM( FUSE_BMAP, "FUSE_BMAP") \ + EM( FUSE_DESTROY, "FUSE_DESTROY") \ + EM( FUSE_IOCTL, "FUSE_IOCTL") \ + EM( FUSE_POLL, "FUSE_POLL") \ + EM( FUSE_NOTIFY_REPLY, "FUSE_NOTIFY_REPLY") \ + EM( FUSE_BATCH_FORGET, "FUSE_BATCH_FORGET") \ + EM( FUSE_FALLOCATE, "FUSE_FALLOCATE") \ + EM( FUSE_READDIRPLUS, "FUSE_READDIRPLUS") \ + EM( FUSE_RENAME2, "FUSE_RENAME2") \ + EM( FUSE_LSEEK, "FUSE_LSEEK") \ + EM( FUSE_COPY_FILE_RANGE, "FUSE_COPY_FILE_RANGE") \ + EM( FUSE_SETUPMAPPING, "FUSE_SETUPMAPPING") \ + EM( FUSE_REMOVEMAPPING, "FUSE_REMOVEMAPPING") \ + EM( FUSE_SYNCFS, "FUSE_SYNCFS") \ + EM( FUSE_TMPFILE, "FUSE_TMPFILE") \ + EM( FUSE_STATX, "FUSE_STATX") \ + EMe(CUSE_INIT, "CUSE_INIT") + +/* + * This will turn the above table into TRACE_DEFINE_ENUM() for each of the + * entries. + */ +#undef EM +#undef EMe +#define EM(a, b) TRACE_DEFINE_ENUM(a); +#define EMe(a, b) TRACE_DEFINE_ENUM(a); + +OPCODES + +/* Now we redfine it with the table that __print_symbolic needs. */ +#undef EM +#undef EMe +#define EM(a, b) {a, b}, +#define EMe(a, b) {a, b} + +TRACE_EVENT(fuse_request_send, + TP_PROTO(const struct fuse_req *req), + + TP_ARGS(req), + + TP_STRUCT__entry( + __field(dev_t, connection) + __field(uint64_t, unique) + __field(enum fuse_opcode, opcode) + __field(uint32_t, len) + ), + + TP_fast_assign( + __entry->connection = req->chan->conn->dev; + __entry->unique = req->in.h.unique; + __entry->opcode = req->in.h.opcode; + __entry->len = req->in.h.len; + ), + + TP_printk("connection %u req %llu opcode %u (%s) len %u ", + __entry->connection, __entry->unique, __entry->opcode, + __print_symbolic(__entry->opcode, OPCODES), __entry->len) +); + +TRACE_EVENT(fuse_request_sent, + TP_PROTO(const struct fuse_req *req), + + TP_ARGS(req), + + TP_STRUCT__entry( + __field(dev_t, connection) + __field(uint64_t, unique) + __field(enum fuse_opcode, opcode) + ), + + TP_fast_assign( + __entry->connection = req->chan->conn->dev; + __entry->unique = req->in.h.unique; + __entry->opcode = req->in.h.opcode; + ), + + TP_printk("connection %u req %llu opcode %u (%s)", + __entry->connection, __entry->unique, __entry->opcode, + __print_symbolic(__entry->opcode, OPCODES)) +); + +TRACE_EVENT(fuse_request_end, + TP_PROTO(const struct fuse_req *req), + + TP_ARGS(req), + + TP_STRUCT__entry( + __field(dev_t, connection) + __field(uint64_t, unique) + __field(uint32_t, len) + __field(int32_t, error) + ), + + TP_fast_assign( + __entry->connection = req->chan->conn->dev; + __entry->unique = req->in.h.unique; + __entry->len = req->out.h.len; + __entry->error = req->out.h.error; + ), + + TP_printk("connection %u req %llu len %u error %d", __entry->connection, + __entry->unique, __entry->len, __entry->error) +); + +#endif /* _TRACE_FUSE_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE fuse_trace +#include diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c new file mode 100644 index 000000000..d975073c6 --- /dev/null +++ b/fs/fuse/inode.c @@ -0,0 +1,2189 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + FUSE: Filesystem in Userspace + Copyright (C) 2001-2008 Miklos Szeredi +*/ + +#include "dev.h" +#include "fuse_i.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +MODULE_AUTHOR("Miklos Szeredi "); +MODULE_DESCRIPTION("Filesystem in Userspace"); +MODULE_LICENSE("GPL"); + +static struct kmem_cache *fuse_inode_cachep; +struct list_head fuse_conn_list; +DEFINE_MUTEX(fuse_mutex); + +static int set_global_limit(const char *val, const struct kernel_param *kp); + +unsigned int fuse_max_pages_limit = 256; +/* default is no timeout */ + +unsigned int max_user_bgreq; +module_param_call(max_user_bgreq, set_global_limit, param_get_uint, + &max_user_bgreq, 0644); +__MODULE_PARM_TYPE(max_user_bgreq, "uint"); +MODULE_PARM_DESC(max_user_bgreq, + "Global limit for the maximum number of backgrounded requests an " + "unprivileged user can set"); + +unsigned int max_user_congthresh; +module_param_call(max_user_congthresh, set_global_limit, param_get_uint, + &max_user_congthresh, 0644); +__MODULE_PARM_TYPE(max_user_congthresh, "uint"); +MODULE_PARM_DESC(max_user_congthresh, + "Global limit for the maximum congestion threshold an " + "unprivileged user can set"); + +#define FUSE_DEFAULT_BLKSIZE 512 + +/** Congestion starts at 75% of maximum */ +#define FUSE_DEFAULT_CONGESTION_THRESHOLD (FUSE_DEFAULT_MAX_BACKGROUND * 3 / 4) + +#ifdef CONFIG_BLOCK +static struct file_system_type fuseblk_fs_type; +#endif + +static struct fuse_submount_lookup *fuse_alloc_submount_lookup(void) +{ + struct fuse_submount_lookup *sl; + + sl = kzalloc_obj(struct fuse_submount_lookup, GFP_KERNEL_ACCOUNT); + if (!sl) + return NULL; + sl->forget = fuse_alloc_forget(); + if (!sl->forget) + goto out_free; + + return sl; + +out_free: + kfree(sl); + return NULL; +} + +static struct inode *fuse_alloc_inode(struct super_block *sb) +{ + struct fuse_inode *fi; + + fi = alloc_inode_sb(sb, fuse_inode_cachep, GFP_KERNEL); + if (!fi) + return NULL; + + /* Initialize private data (i.e. everything except fi->inode) */ + BUILD_BUG_ON(offsetof(struct fuse_inode, inode) != 0); + memset((void *) fi + sizeof(fi->inode), 0, sizeof(*fi) - sizeof(fi->inode)); + + fi->inval_mask = ~0; + mutex_init(&fi->mutex); + spin_lock_init(&fi->lock); + fi->forget = fuse_alloc_forget(); + if (!fi->forget) + goto out_free; + + if (IS_ENABLED(CONFIG_FUSE_DAX) && !fuse_dax_inode_alloc(sb, fi)) + goto out_free_forget; + + if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH)) + fuse_inode_backing_set(fi, NULL); + + return &fi->inode; + +out_free_forget: + kfree(fi->forget); +out_free: + kmem_cache_free(fuse_inode_cachep, fi); + return NULL; +} + +static void fuse_free_inode(struct inode *inode) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + + mutex_destroy(&fi->mutex); + kfree(fi->forget); +#ifdef CONFIG_FUSE_DAX + kfree(fi->dax); +#endif + if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH)) + fuse_backing_put(fuse_inode_backing(fi)); + + kmem_cache_free(fuse_inode_cachep, fi); +} + +static void fuse_cleanup_submount_lookup(struct fuse_conn *fc, + struct fuse_submount_lookup *sl) +{ + if (!refcount_dec_and_test(&sl->count)) + return; + + fuse_chan_queue_forget(fc->chan, sl->forget, sl->nodeid, 1); + sl->forget = NULL; + kfree(sl); +} + +static void fuse_evict_inode(struct inode *inode) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + + /* Will write inode on close/munmap and in all other dirtiers */ + WARN_ON(inode_state_read_once(inode) & I_DIRTY_INODE); + + if (FUSE_IS_DAX(inode)) + dax_break_layout_final(inode); + + truncate_inode_pages_final(&inode->i_data); + clear_inode(inode); + if (inode->i_sb->s_flags & SB_ACTIVE) { + struct fuse_conn *fc = get_fuse_conn(inode); + + if (FUSE_IS_DAX(inode)) + fuse_dax_inode_cleanup(inode); + if (fi->nlookup) { + fuse_chan_queue_forget(fc->chan, fi->forget, fi->nodeid, + fi->nlookup); + fi->forget = NULL; + } + + if (fi->submount_lookup) { + fuse_cleanup_submount_lookup(fc, fi->submount_lookup); + fi->submount_lookup = NULL; + } + /* + * Evict of non-deleted inode may race with outstanding + * LOOKUP/READDIRPLUS requests and result in inconsistency when + * the request finishes. Deal with that here by bumping a + * counter that can be compared to the starting value. + */ + if (inode->i_nlink > 0) + atomic64_inc(&fc->evict_ctr); + } + if (S_ISREG(inode->i_mode) && !fuse_is_bad(inode)) { + WARN_ON(fi->iocachectr != 0); + WARN_ON(!list_empty(&fi->write_files)); + WARN_ON(!list_empty(&fi->queued_writes)); + } +} + +static int fuse_reconfigure(struct fs_context *fsc) +{ + struct super_block *sb = fsc->root->d_sb; + + sync_filesystem(sb); + if (fsc->sb_flags & SB_MANDLOCK) + return -EINVAL; + + return 0; +} + +/* + * ino_t is 32-bits on 32-bit arch. We have to squash the 64-bit value down + * so that it will fit. + */ +static ino_t fuse_squash_ino(u64 ino64) +{ + ino_t ino = (ino_t) ino64; + if (sizeof(ino_t) < sizeof(u64)) + ino ^= ino64 >> (sizeof(u64) - sizeof(ino_t)) * 8; + return ino; +} + +void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr, + struct fuse_statx *sx, + u64 attr_valid, u32 cache_mask, + u64 evict_ctr) +{ + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + + lockdep_assert_held(&fi->lock); + + /* + * Clear basic stats from invalid mask. + * + * Don't do this if this is coming from a fuse_iget() call and there + * might have been a racing evict which would've invalidated the result + * if the attr_version would've been preserved. + * + * !evict_ctr -> this is create + * fi->attr_version != 0 -> this is not a new inode + * evict_ctr == fuse_get_evict_ctr() -> no evicts while during request + */ + if (!evict_ctr || fi->attr_version || evict_ctr == fuse_get_evict_ctr(fc)) + set_mask_bits(&fi->inval_mask, STATX_BASIC_STATS, 0); + + fi->attr_version = atomic64_inc_return(&fc->attr_version); + fi->i_time = attr_valid; + + inode->i_ino = fuse_squash_ino(attr->ino); + inode->i_mode = (inode->i_mode & S_IFMT) | (attr->mode & 07777); + set_nlink(inode, attr->nlink); + inode->i_uid = make_kuid(fc->user_ns, attr->uid); + inode->i_gid = make_kgid(fc->user_ns, attr->gid); + inode->i_blocks = attr->blocks; + + /* Sanitize nsecs */ + attr->atimensec = min_t(u32, attr->atimensec, NSEC_PER_SEC - 1); + attr->mtimensec = min_t(u32, attr->mtimensec, NSEC_PER_SEC - 1); + attr->ctimensec = min_t(u32, attr->ctimensec, NSEC_PER_SEC - 1); + + inode_set_atime(inode, attr->atime, attr->atimensec); + /* mtime from server may be stale due to local buffered write */ + if (!(cache_mask & STATX_MTIME)) { + inode_set_mtime(inode, attr->mtime, attr->mtimensec); + } + if (!(cache_mask & STATX_CTIME)) { + inode_set_ctime(inode, attr->ctime, attr->ctimensec); + } + if (sx) { + /* Sanitize nsecs */ + sx->btime.tv_nsec = + min_t(u32, sx->btime.tv_nsec, NSEC_PER_SEC - 1); + + /* + * Btime has been queried, cache is valid (whether or not btime + * is available or not) so clear STATX_BTIME from inval_mask. + * + * Availability of the btime attribute is indicated in + * FUSE_I_BTIME + */ + set_mask_bits(&fi->inval_mask, STATX_BTIME, 0); + if (sx->mask & STATX_BTIME) { + set_bit(FUSE_I_BTIME, &fi->state); + fi->i_btime.tv_sec = sx->btime.tv_sec; + fi->i_btime.tv_nsec = sx->btime.tv_nsec; + } + } + + if (attr->blksize) + fi->cached_i_blkbits = ilog2(attr->blksize); + else + fi->cached_i_blkbits = inode->i_sb->s_blocksize_bits; + + /* + * Don't set the sticky bit in i_mode, unless we want the VFS + * to check permissions. This prevents failures due to the + * check in may_delete(). + */ + fi->orig_i_mode = inode->i_mode; + if (!fc->default_permissions) + inode->i_mode &= ~S_ISVTX; + + fi->orig_ino = attr->ino; + + /* + * We are refreshing inode data and it is possible that another + * client set suid/sgid or security.capability xattr. So clear + * S_NOSEC. Ideally, we could have cleared it only if suid/sgid + * was set or if security.capability xattr was set. But we don't + * know if security.capability has been set or not. So clear it + * anyway. Its less efficient but should be safe. + */ + inode->i_flags &= ~S_NOSEC; +} + +u32 fuse_get_cache_mask(struct inode *inode) +{ + struct fuse_conn *fc = get_fuse_conn(inode); + + if (!fc->writeback_cache || !S_ISREG(inode->i_mode)) + return 0; + + return STATX_MTIME | STATX_CTIME | STATX_SIZE; +} + +static void fuse_change_attributes_i(struct inode *inode, struct fuse_attr *attr, + struct fuse_statx *sx, u64 attr_valid, + u64 attr_version, u64 evict_ctr) +{ + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + u32 cache_mask; + loff_t oldsize; + struct timespec64 old_mtime; + + spin_lock(&fi->lock); + /* + * In case of writeback_cache enabled, writes update mtime, ctime and + * may update i_size. In these cases trust the cached value in the + * inode. + */ + cache_mask = fuse_get_cache_mask(inode); + if (cache_mask & STATX_SIZE) + attr->size = i_size_read(inode); + + if (cache_mask & STATX_MTIME) { + attr->mtime = inode_get_mtime_sec(inode); + attr->mtimensec = inode_get_mtime_nsec(inode); + } + if (cache_mask & STATX_CTIME) { + attr->ctime = inode_get_ctime_sec(inode); + attr->ctimensec = inode_get_ctime_nsec(inode); + } + + if ((attr_version != 0 && fi->attr_version > attr_version) || + test_bit(FUSE_I_SIZE_UNSTABLE, &fi->state)) { + spin_unlock(&fi->lock); + return; + } + + old_mtime = inode_get_mtime(inode); + fuse_change_attributes_common(inode, attr, sx, attr_valid, cache_mask, + evict_ctr); + + oldsize = inode->i_size; + /* + * In case of writeback_cache enabled, the cached writes beyond EOF + * extend local i_size without keeping userspace server in sync. So, + * attr->size coming from server can be stale. We cannot trust it. + */ + if (!(cache_mask & STATX_SIZE)) + i_size_write(inode, attr->size); + spin_unlock(&fi->lock); + + if (!cache_mask && S_ISREG(inode->i_mode)) { + bool inval = false; + + if (oldsize != attr->size) { + truncate_pagecache(inode, attr->size); + if (!fc->explicit_inval_data) + inval = true; + } else if (fc->auto_inval_data) { + struct timespec64 new_mtime = { + .tv_sec = attr->mtime, + .tv_nsec = attr->mtimensec, + }; + + /* + * Auto inval mode also checks and invalidates if mtime + * has changed. + */ + if (!timespec64_equal(&old_mtime, &new_mtime)) + inval = true; + } + + if (inval) + invalidate_inode_pages2(inode->i_mapping); + } + + if (IS_ENABLED(CONFIG_FUSE_DAX)) + fuse_dax_dontcache(inode, attr->flags); +} + +void fuse_change_attributes(struct inode *inode, struct fuse_attr *attr, + struct fuse_statx *sx, u64 attr_valid, + u64 attr_version) +{ + fuse_change_attributes_i(inode, attr, sx, attr_valid, attr_version, 0); +} + +static void fuse_init_submount_lookup(struct fuse_submount_lookup *sl, + u64 nodeid) +{ + sl->nodeid = nodeid; + refcount_set(&sl->count, 1); +} + +static void fuse_init_inode(struct inode *inode, struct fuse_attr *attr, + struct fuse_conn *fc) +{ + inode->i_mode = attr->mode & S_IFMT; + inode->i_size = attr->size; + inode_set_mtime(inode, attr->mtime, attr->mtimensec); + inode_set_ctime(inode, attr->ctime, attr->ctimensec); + if (S_ISREG(inode->i_mode)) { + fuse_init_common(inode); + fuse_init_file_inode(inode, attr->flags); + } else if (S_ISDIR(inode->i_mode)) + fuse_init_dir(inode); + else if (S_ISLNK(inode->i_mode)) + fuse_init_symlink(inode); + else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) || + S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) { + fuse_init_common(inode); + init_special_inode(inode, inode->i_mode, + new_decode_dev(attr->rdev)); + } else + BUG(); + /* + * Ensure that we don't cache acls for daemons without FUSE_POSIX_ACL + * so they see the exact same behavior as before. + */ + if (!fc->posix_acl) + inode->i_acl = inode->i_default_acl = ACL_DONT_CACHE; +} + +static int fuse_inode_eq(struct inode *inode, void *_nodeidp) +{ + u64 nodeid = *(u64 *) _nodeidp; + if (get_node_id(inode) == nodeid) + return 1; + else + return 0; +} + +static int fuse_inode_set(struct inode *inode, void *_nodeidp) +{ + u64 nodeid = *(u64 *) _nodeidp; + get_fuse_inode(inode)->nodeid = nodeid; + return 0; +} + +struct inode *fuse_iget(struct super_block *sb, u64 nodeid, + int generation, struct fuse_attr *attr, + u64 attr_valid, u64 attr_version, + u64 evict_ctr) +{ + struct inode *inode; + struct fuse_inode *fi; + struct fuse_conn *fc = get_fuse_conn_super(sb); + bool is_new_inode = false; + + /* + * Auto mount points get their node id from the submount root, which is + * not a unique identifier within this filesystem. + * + * To avoid conflicts, do not place submount points into the inode hash + * table. + */ + if (fc->auto_submounts && (attr->flags & FUSE_ATTR_SUBMOUNT) && + S_ISDIR(attr->mode)) { + struct fuse_inode *fi; + + inode = new_inode(sb); + if (!inode) + return NULL; + + fuse_init_inode(inode, attr, fc); + fi = get_fuse_inode(inode); + fi->nodeid = nodeid; + fi->submount_lookup = fuse_alloc_submount_lookup(); + if (!fi->submount_lookup) { + iput(inode); + return NULL; + } + /* Sets nlookup = 1 on fi->submount_lookup->nlookup */ + fuse_init_submount_lookup(fi->submount_lookup, nodeid); + inode->i_flags |= S_AUTOMOUNT; + goto done; + } + +retry: + inode = iget5_locked(sb, nodeid, fuse_inode_eq, fuse_inode_set, &nodeid); + if (!inode) + return NULL; + + is_new_inode = inode_state_read_once(inode) & I_NEW; + if (is_new_inode) { + inode->i_flags |= S_NOATIME; + if (!fc->writeback_cache || !S_ISREG(attr->mode)) + inode->i_flags |= S_NOCMTIME; + inode->i_generation = generation; + fuse_init_inode(inode, attr, fc); + } else if (fuse_stale_inode(inode, generation, attr)) { + /* nodeid was reused, any I/O on the old inode should fail */ + fuse_make_bad(inode); + if (inode != d_inode(sb->s_root)) { + remove_inode_hash(inode); + iput(inode); + goto retry; + } + } + fi = get_fuse_inode(inode); + spin_lock(&fi->lock); + fi->nlookup++; + spin_unlock(&fi->lock); +done: + fuse_change_attributes_i(inode, attr, NULL, attr_valid, attr_version, + evict_ctr); + if (is_new_inode) + unlock_new_inode(inode); + return inode; +} + +struct inode *fuse_ilookup(struct fuse_conn *fc, u64 nodeid, + struct fuse_mount **fm) +{ + struct fuse_mount *fm_iter; + struct inode *inode; + + WARN_ON(!rwsem_is_locked(&fc->killsb)); + list_for_each_entry(fm_iter, &fc->mounts, fc_entry) { + if (!fm_iter->sb) + continue; + + inode = ilookup5(fm_iter->sb, nodeid, fuse_inode_eq, &nodeid); + if (inode) { + if (fm) + *fm = fm_iter; + return inode; + } + } + + return NULL; +} + +int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid, + loff_t offset, loff_t len) +{ + struct fuse_inode *fi; + struct inode *inode; + pgoff_t pg_start; + pgoff_t pg_end; + + inode = fuse_ilookup(fc, nodeid, NULL); + if (!inode) + return -ENOENT; + + fi = get_fuse_inode(inode); + spin_lock(&fi->lock); + fi->attr_version = atomic64_inc_return(&fc->attr_version); + spin_unlock(&fi->lock); + + fuse_invalidate_attr(inode); + forget_all_cached_acls(inode); + if (offset >= 0) { + pg_start = offset >> PAGE_SHIFT; + if (len <= 0) + pg_end = -1; + else + pg_end = (offset + len - 1) >> PAGE_SHIFT; + invalidate_inode_pages2_range(inode->i_mapping, + pg_start, pg_end); + } + iput(inode); + return 0; +} + +void fuse_try_prune_one_inode(struct fuse_conn *fc, u64 nodeid) +{ + struct inode *inode; + + inode = fuse_ilookup(fc, nodeid, NULL); + if (!inode) + return; + d_prune_aliases(inode); + iput(inode); +} + +bool fuse_lock_inode(struct inode *inode) +{ + bool locked = false; + + if (!get_fuse_conn(inode)->parallel_dirops) { + mutex_lock(&get_fuse_inode(inode)->mutex); + locked = true; + } + + return locked; +} + +void fuse_unlock_inode(struct inode *inode, bool locked) +{ + if (locked) + mutex_unlock(&get_fuse_inode(inode)->mutex); +} + +static void fuse_umount_begin(struct super_block *sb) +{ + struct fuse_conn *fc = get_fuse_conn_super(sb); + + if (fc->no_force_umount) + return; + + fuse_chan_abort(fc->chan, false); + + // Only retire block-device-based superblocks. + if (sb->s_bdev != NULL) + retire_super(sb); +} + +static void fuse_send_destroy(struct fuse_mount *fm) +{ + if (fm->fc->conn_init) { + FUSE_ARGS(args); + + args.opcode = FUSE_DESTROY; + args.force = true; + args.nocreds = true; + fuse_simple_request(fm, &args); + } +} + +static void convert_fuse_statfs(struct kstatfs *stbuf, struct fuse_kstatfs *attr) +{ + stbuf->f_type = FUSE_SUPER_MAGIC; + stbuf->f_bsize = attr->bsize; + stbuf->f_frsize = attr->frsize; + stbuf->f_blocks = attr->blocks; + stbuf->f_bfree = attr->bfree; + stbuf->f_bavail = attr->bavail; + stbuf->f_files = attr->files; + stbuf->f_ffree = attr->ffree; + stbuf->f_namelen = attr->namelen; + /* fsid is left zero */ +} + +static int fuse_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + struct fuse_mount *fm = get_fuse_mount_super(sb); + FUSE_ARGS(args); + struct fuse_statfs_out outarg; + int err; + + if (!fuse_allow_current_process(fm->fc)) { + buf->f_type = FUSE_SUPER_MAGIC; + return 0; + } + + memset(&outarg, 0, sizeof(outarg)); + args.in_numargs = 0; + args.opcode = FUSE_STATFS; + args.nodeid = get_node_id(d_inode(dentry)); + args.out_numargs = 1; + args.out_args[0].size = sizeof(outarg); + args.out_args[0].value = &outarg; + err = fuse_simple_request(fm, &args); + if (!err) + convert_fuse_statfs(buf, &outarg.st); + return err; +} + +static struct fuse_sync_bucket *fuse_sync_bucket_alloc(void) +{ + struct fuse_sync_bucket *bucket; + + bucket = kzalloc_obj(*bucket, GFP_KERNEL | __GFP_NOFAIL); + init_waitqueue_head(&bucket->waitq); + /* Initial active count */ + atomic_set(&bucket->count, 1); + return bucket; +} + +static void fuse_sync_fs_writes(struct fuse_conn *fc) +{ + struct fuse_sync_bucket *bucket, *new_bucket; + int count; + + new_bucket = fuse_sync_bucket_alloc(); + spin_lock(&fc->lock); + bucket = rcu_dereference_protected(fc->curr_bucket, 1); + count = atomic_read(&bucket->count); + WARN_ON(count < 1); + /* No outstanding writes? */ + if (count == 1) { + spin_unlock(&fc->lock); + kfree(new_bucket); + return; + } + + /* + * Completion of new bucket depends on completion of this bucket, so add + * one more count. + */ + atomic_inc(&new_bucket->count); + rcu_assign_pointer(fc->curr_bucket, new_bucket); + spin_unlock(&fc->lock); + /* + * Drop initial active count. At this point if all writes in this and + * ancestor buckets complete, the count will go to zero and this task + * will be woken up. + */ + atomic_dec(&bucket->count); + + wait_event(bucket->waitq, atomic_read(&bucket->count) == 0); + + /* Drop temp count on descendant bucket */ + fuse_sync_bucket_dec(new_bucket); + kfree_rcu(bucket, rcu); +} + +static int fuse_sync_fs(struct super_block *sb, int wait) +{ + struct fuse_mount *fm = get_fuse_mount_super(sb); + struct fuse_conn *fc = fm->fc; + struct fuse_syncfs_in inarg; + FUSE_ARGS(args); + int err; + + /* + * Userspace cannot handle the wait == 0 case. Avoid a + * gratuitous roundtrip. + */ + if (!wait) + return 0; + + /* The filesystem is being unmounted. Nothing to do. */ + if (!sb->s_root) + return 0; + + if (!fc->sync_fs) + return 0; + + fuse_sync_fs_writes(fc); + + memset(&inarg, 0, sizeof(inarg)); + args.in_numargs = 1; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + args.opcode = FUSE_SYNCFS; + args.nodeid = get_node_id(sb->s_root->d_inode); + args.out_numargs = 0; + + err = fuse_simple_request(fm, &args); + if (err == -ENOSYS) { + fc->sync_fs = 0; + err = 0; + } + + return err; +} + +enum { + OPT_SOURCE, + OPT_SUBTYPE, + OPT_FD, + OPT_ROOTMODE, + OPT_USER_ID, + OPT_GROUP_ID, + OPT_DEFAULT_PERMISSIONS, + OPT_ALLOW_OTHER, + OPT_MAX_READ, + OPT_BLKSIZE, + OPT_ERR +}; + +static const struct fs_parameter_spec fuse_fs_parameters[] = { + fsparam_string ("source", OPT_SOURCE), + fsparam_fd ("fd", OPT_FD), + fsparam_u32oct ("rootmode", OPT_ROOTMODE), + fsparam_uid ("user_id", OPT_USER_ID), + fsparam_gid ("group_id", OPT_GROUP_ID), + fsparam_flag ("default_permissions", OPT_DEFAULT_PERMISSIONS), + fsparam_flag ("allow_other", OPT_ALLOW_OTHER), + fsparam_u32 ("max_read", OPT_MAX_READ), + fsparam_u32 ("blksize", OPT_BLKSIZE), + fsparam_string ("subtype", OPT_SUBTYPE), + {} +}; + +static int fuse_opt_fd(struct fs_context *fsc, struct file *file) +{ + struct fuse_fs_context *ctx = fsc->fs_private; + + if (file->f_op != &fuse_dev_operations) + return invalfc(fsc, "fd is not a fuse device"); + /* + * Require mount to happen from the same user namespace which + * opened /dev/fuse to prevent potential attacks. + */ + if (file->f_cred->user_ns != fsc->user_ns) + return invalfc(fsc, "wrong user namespace for fuse device"); + + ctx->fud = fuse_dev_grab(file); + + return 0; +} + +static int fuse_parse_param(struct fs_context *fsc, struct fs_parameter *param) +{ + struct fs_parse_result result; + struct fuse_fs_context *ctx = fsc->fs_private; + int opt; + kuid_t kuid; + kgid_t kgid; + + if (fsc->purpose == FS_CONTEXT_FOR_RECONFIGURE) { + /* + * Ignore options coming from mount(MS_REMOUNT) for backward + * compatibility. + */ + if (fsc->oldapi) + return 0; + + return invalfc(fsc, "No changes allowed in reconfigure"); + } + + opt = fs_parse(fsc, fuse_fs_parameters, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case OPT_SOURCE: + if (fsc->source) + return invalfc(fsc, "Multiple sources specified"); + fsc->source = param->string; + param->string = NULL; + break; + + case OPT_SUBTYPE: + if (ctx->subtype) + return invalfc(fsc, "Multiple subtypes specified"); + ctx->subtype = param->string; + param->string = NULL; + return 0; + + case OPT_FD: + if (param->type == fs_value_is_file) { + return fuse_opt_fd(fsc, param->file); + } else { + struct file *file __free(fput) = fget(result.uint_32); + if (!file) + return -EBADF; + + return fuse_opt_fd(fsc, file); + } + + case OPT_ROOTMODE: + if (!fuse_valid_type(result.uint_32)) + return invalfc(fsc, "Invalid rootmode"); + ctx->rootmode = result.uint_32; + ctx->rootmode_present = true; + break; + + case OPT_USER_ID: + kuid = result.uid; + /* + * The requested uid must be representable in the + * filesystem's idmapping. + */ + if (!kuid_has_mapping(fsc->user_ns, kuid)) + return invalfc(fsc, "Invalid user_id"); + ctx->user_id = kuid; + ctx->user_id_present = true; + break; + + case OPT_GROUP_ID: + kgid = result.gid; + /* + * The requested gid must be representable in the + * filesystem's idmapping. + */ + if (!kgid_has_mapping(fsc->user_ns, kgid)) + return invalfc(fsc, "Invalid group_id"); + ctx->group_id = kgid; + ctx->group_id_present = true; + break; + + case OPT_DEFAULT_PERMISSIONS: + ctx->default_permissions = true; + break; + + case OPT_ALLOW_OTHER: + ctx->allow_other = true; + break; + + case OPT_MAX_READ: + ctx->max_read = result.uint_32; + break; + + case OPT_BLKSIZE: + if (!ctx->is_bdev) + return invalfc(fsc, "blksize only supported for fuseblk"); + ctx->blksize = result.uint_32; + break; + + default: + return -EINVAL; + } + + return 0; +} + +static void fuse_free_fsc(struct fs_context *fsc) +{ + struct fuse_fs_context *ctx = fsc->fs_private; + + if (ctx) { + if (ctx->fud) + fuse_dev_put(ctx->fud); + kfree(ctx->subtype); + kfree(ctx); + } +} + +static int fuse_show_options(struct seq_file *m, struct dentry *root) +{ + struct super_block *sb = root->d_sb; + struct fuse_conn *fc = get_fuse_conn_super(sb); + + if (fc->legacy_opts_show) { + seq_printf(m, ",user_id=%u", + from_kuid_munged(fc->user_ns, fc->user_id)); + seq_printf(m, ",group_id=%u", + from_kgid_munged(fc->user_ns, fc->group_id)); + if (fc->default_permissions) + seq_puts(m, ",default_permissions"); + if (fc->allow_other) + seq_puts(m, ",allow_other"); + if (fc->max_read != ~0) + seq_printf(m, ",max_read=%u", fc->max_read); + if (sb->s_bdev && sb->s_blocksize != FUSE_DEFAULT_BLKSIZE) + seq_printf(m, ",blksize=%lu", sb->s_blocksize); + } +#ifdef CONFIG_FUSE_DAX + if (fc->dax_mode == FUSE_DAX_ALWAYS) + seq_puts(m, ",dax=always"); + else if (fc->dax_mode == FUSE_DAX_NEVER) + seq_puts(m, ",dax=never"); + else if (fc->dax_mode == FUSE_DAX_INODE_USER) + seq_puts(m, ",dax=inode"); +#endif + + return 0; +} + +void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm, + struct user_namespace *user_ns, struct fuse_chan *fch) +{ + memset(fc, 0, sizeof(*fc)); + spin_lock_init(&fc->lock); + init_rwsem(&fc->killsb); + refcount_set(&fc->count, 1); + atomic_set(&fc->epoch, 1); + INIT_WORK(&fc->epoch_work, fuse_epoch_work); + INIT_LIST_HEAD(&fc->entry); + fc->congestion_threshold = FUSE_DEFAULT_CONGESTION_THRESHOLD; + atomic64_set(&fc->khctr, 0); + fc->polled_files = RB_ROOT; + atomic64_set(&fc->attr_version, 1); + atomic64_set(&fc->evict_ctr, 1); + get_random_bytes(&fc->scramble_key, sizeof(fc->scramble_key)); + fc->pid_ns = get_pid_ns(task_active_pid_ns(current)); + fc->user_ns = get_user_ns(user_ns); + fc->max_pages = FUSE_DEFAULT_MAX_PAGES_PER_REQ; + fc->max_pages_limit = fuse_max_pages_limit; + fc->name_max = FUSE_NAME_LOW_MAX; + + if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH)) + fuse_backing_files_init(fc); + + INIT_LIST_HEAD(&fc->mounts); + list_add(&fm->fc_entry, &fc->mounts); + fm->fc = fc; + fuse_chan_set_fc(fch, fc); + fc->chan = fch; +} +EXPORT_SYMBOL_GPL(fuse_conn_init); + +static void delayed_release(struct rcu_head *p) +{ + struct fuse_conn *fc = container_of(p, struct fuse_conn, rcu); + + fuse_uring_destruct(fc->chan); + fuse_chan_free(fc->chan); + + put_user_ns(fc->user_ns); + fc->release(fc); +} + +void fuse_conn_put(struct fuse_conn *fc) +{ + struct fuse_sync_bucket *bucket; + + if (!refcount_dec_and_test(&fc->count)) + return; + + if (IS_ENABLED(CONFIG_FUSE_DAX)) + fuse_dax_conn_free(fc); + cancel_work_sync(&fc->epoch_work); + fuse_chan_release(fc->chan); + put_pid_ns(fc->pid_ns); + bucket = rcu_dereference_protected(fc->curr_bucket, 1); + if (bucket) { + WARN_ON(atomic_read(&bucket->count) != 1); + kfree(bucket); + } + if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH)) + fuse_backing_files_free(fc); + call_rcu(&fc->rcu, delayed_release); +} +EXPORT_SYMBOL_GPL(fuse_conn_put); + +struct fuse_conn *fuse_conn_get(struct fuse_conn *fc) +{ + refcount_inc(&fc->count); + return fc; +} +EXPORT_SYMBOL_GPL(fuse_conn_get); + +dev_t fuse_conn_get_id(struct fuse_conn *fc) +{ + return fc->dev; +} + +static struct inode *fuse_get_root_inode(struct super_block *sb, unsigned int mode) +{ + struct fuse_attr attr; + memset(&attr, 0, sizeof(attr)); + + attr.mode = mode; + attr.ino = FUSE_ROOT_ID; + attr.nlink = 1; + return fuse_iget(sb, FUSE_ROOT_ID, 0, &attr, 0, 0, 0); +} + +struct fuse_inode_handle { + u64 nodeid; + u32 generation; +}; + +static struct dentry *fuse_get_dentry(struct super_block *sb, + struct fuse_inode_handle *handle) +{ + struct fuse_conn *fc = get_fuse_conn_super(sb); + struct inode *inode; + struct dentry *entry; + int err = -ESTALE; + + if (handle->nodeid == 0) + goto out_err; + + inode = ilookup5(sb, handle->nodeid, fuse_inode_eq, &handle->nodeid); + if (!inode) { + struct fuse_entry_out outarg; + + if (!fc->export_support) + goto out_err; + + err = fuse_lookup_name(sb, handle->nodeid, &QSTR("."), &outarg, + &inode); + if (err && err != -ENOENT) + goto out_err; + if (err || !inode) { + err = -ESTALE; + goto out_err; + } + err = -EIO; + if (get_node_id(inode) != handle->nodeid) + goto out_iput; + } + err = -ESTALE; + if (inode->i_generation != handle->generation) + goto out_iput; + + entry = d_obtain_alias(inode); + if (!IS_ERR(entry) && get_node_id(inode) != FUSE_ROOT_ID) + fuse_invalidate_entry_cache(entry); + + return entry; + + out_iput: + iput(inode); + out_err: + return ERR_PTR(err); +} + +static int fuse_encode_fh(struct inode *inode, u32 *fh, int *max_len, + struct inode *parent) +{ + int len = parent ? 6 : 3; + u64 nodeid; + u32 generation; + + if (*max_len < len) { + *max_len = len; + return FILEID_INVALID; + } + + nodeid = get_fuse_inode(inode)->nodeid; + generation = inode->i_generation; + + fh[0] = (u32)(nodeid >> 32); + fh[1] = (u32)(nodeid & 0xffffffff); + fh[2] = generation; + + if (parent) { + nodeid = get_fuse_inode(parent)->nodeid; + generation = parent->i_generation; + + fh[3] = (u32)(nodeid >> 32); + fh[4] = (u32)(nodeid & 0xffffffff); + fh[5] = generation; + } + + *max_len = len; + return parent ? FILEID_INO64_GEN_PARENT : FILEID_INO64_GEN; +} + +static struct dentry *fuse_fh_to_dentry(struct super_block *sb, + struct fid *fid, int fh_len, int fh_type) +{ + struct fuse_inode_handle handle; + + if ((fh_type != FILEID_INO64_GEN && + fh_type != FILEID_INO64_GEN_PARENT) || fh_len < 3) + return NULL; + + handle.nodeid = (u64) fid->raw[0] << 32; + handle.nodeid |= (u64) fid->raw[1]; + handle.generation = fid->raw[2]; + return fuse_get_dentry(sb, &handle); +} + +static struct dentry *fuse_fh_to_parent(struct super_block *sb, + struct fid *fid, int fh_len, int fh_type) +{ + struct fuse_inode_handle parent; + + if (fh_type != FILEID_INO64_GEN_PARENT || fh_len < 6) + return NULL; + + parent.nodeid = (u64) fid->raw[3] << 32; + parent.nodeid |= (u64) fid->raw[4]; + parent.generation = fid->raw[5]; + return fuse_get_dentry(sb, &parent); +} + +static struct dentry *fuse_get_parent(struct dentry *child) +{ + struct inode *child_inode = d_inode(child); + struct fuse_conn *fc = get_fuse_conn(child_inode); + struct inode *inode; + struct dentry *parent; + struct fuse_entry_out outarg; + int err; + + if (!fc->export_support) + return ERR_PTR(-ESTALE); + + err = fuse_lookup_name(child_inode->i_sb, get_node_id(child_inode), + &dotdot_name, &outarg, &inode); + if (err) { + if (err == -ENOENT) + return ERR_PTR(-ESTALE); + return ERR_PTR(err); + } + + parent = d_obtain_alias(inode); + if (!IS_ERR(parent) && get_node_id(inode) != FUSE_ROOT_ID) + fuse_invalidate_entry_cache(parent); + + return parent; +} + +/* only for fid encoding; no support for file handle */ +static const struct export_operations fuse_export_fid_operations = { + .encode_fh = fuse_encode_fh, +}; + +static const struct export_operations fuse_export_operations = { + .fh_to_dentry = fuse_fh_to_dentry, + .fh_to_parent = fuse_fh_to_parent, + .encode_fh = fuse_encode_fh, + .get_parent = fuse_get_parent, +}; + +static const struct super_operations fuse_super_operations = { + .alloc_inode = fuse_alloc_inode, + .free_inode = fuse_free_inode, + .evict_inode = fuse_evict_inode, + .write_inode = fuse_write_inode, + .drop_inode = inode_just_drop, + .umount_begin = fuse_umount_begin, + .statfs = fuse_statfs, + .sync_fs = fuse_sync_fs, + .show_options = fuse_show_options, +}; + +static void sanitize_global_limit(unsigned int *limit) +{ + /* + * The default maximum number of async requests is calculated to consume + * 1/2^13 of the total memory, assuming 392 bytes per request. + */ + if (*limit == 0) + *limit = ((totalram_pages() << PAGE_SHIFT) >> 13) / 392; + + if (*limit >= 1 << 16) + *limit = (1 << 16) - 1; +} + +static int set_global_limit(const char *val, const struct kernel_param *kp) +{ + int rv; + + rv = param_set_uint(val, kp); + if (rv) + return rv; + + sanitize_global_limit((unsigned int *)kp->arg); + + return 0; +} + +static void process_init_limits(struct fuse_conn *fc, struct fuse_init_out *arg) +{ + int cap_sys_admin = capable(CAP_SYS_ADMIN); + + if (arg->minor < 13) + return; + + sanitize_global_limit(&max_user_bgreq); + sanitize_global_limit(&max_user_congthresh); + + if (arg->max_background) { + unsigned int max_background = arg->max_background; + + if (!cap_sys_admin && max_background > max_user_bgreq) + max_background = max_user_bgreq; + + fuse_chan_max_background_set(fc->chan, max_background); + } + if (arg->congestion_threshold) { + fc->congestion_threshold = arg->congestion_threshold; + + if (!cap_sys_admin && + fc->congestion_threshold > max_user_congthresh) + fc->congestion_threshold = max_user_congthresh; + } +} + +struct fuse_init_args { + struct fuse_args args; + struct fuse_init_in in; + struct fuse_init_out out; + struct fuse_mount *fm; +}; + +static void process_init_reply(struct fuse_args *args, int error) +{ + struct fuse_init_args *ia = container_of(args, typeof(*ia), args); + struct fuse_mount *fm = ia->fm; + struct fuse_conn *fc = fm->fc; + struct fuse_init_out *arg = &ia->out; + bool ok = true; + + if (error || arg->major != FUSE_KERNEL_VERSION) + ok = false; + else { + unsigned long ra_pages; + unsigned int timeout = 0; + + process_init_limits(fc, arg); + + if (arg->minor >= 6) { + u64 flags = arg->flags; + + if (flags & FUSE_INIT_EXT) + flags |= (u64) arg->flags2 << 32; + + ra_pages = arg->max_readahead / PAGE_SIZE; + if (flags & FUSE_ASYNC_READ) + fc->async_read = 1; + if (!(flags & FUSE_POSIX_LOCKS)) + fc->no_lock = 1; + if (arg->minor >= 17) { + if (!(flags & FUSE_FLOCK_LOCKS)) + fc->no_flock = 1; + } else { + if (!(flags & FUSE_POSIX_LOCKS)) + fc->no_flock = 1; + } + if (flags & FUSE_ATOMIC_O_TRUNC) + fc->atomic_o_trunc = 1; + if (arg->minor >= 9) { + /* LOOKUP has dependency on proto version */ + if (flags & FUSE_EXPORT_SUPPORT) + fc->export_support = 1; + } + if (flags & FUSE_BIG_WRITES) + fc->big_writes = 1; + if (flags & FUSE_DONT_MASK) + fc->dont_mask = 1; + if (flags & FUSE_AUTO_INVAL_DATA) + fc->auto_inval_data = 1; + else if (flags & FUSE_EXPLICIT_INVAL_DATA) + fc->explicit_inval_data = 1; + if (flags & FUSE_DO_READDIRPLUS) { + fc->do_readdirplus = 1; + if (flags & FUSE_READDIRPLUS_AUTO) + fc->readdirplus_auto = 1; + } + if (flags & FUSE_ASYNC_DIO) + fc->async_dio = 1; + if (flags & FUSE_WRITEBACK_CACHE) + fc->writeback_cache = 1; + if (flags & FUSE_PARALLEL_DIROPS) + fc->parallel_dirops = 1; + if (flags & FUSE_HANDLE_KILLPRIV) + fc->handle_killpriv = 1; + if (arg->time_gran && arg->time_gran <= 1000000000) + fm->sb->s_time_gran = arg->time_gran; + if ((flags & FUSE_POSIX_ACL)) { + fc->default_permissions = 1; + fc->posix_acl = 1; + } + if (flags & FUSE_CACHE_SYMLINKS) + fc->cache_symlinks = 1; + if (flags & FUSE_ABORT_ERROR) + fc->abort_err = 1; + if (flags & FUSE_MAX_PAGES) { + fc->max_pages = + min_t(unsigned int, fc->max_pages_limit, + max_t(unsigned int, arg->max_pages, 1)); + + /* + * PATH_MAX file names might need two pages for + * ops like rename + */ + if (fc->max_pages > 1) + fc->name_max = FUSE_NAME_MAX; + } + if (IS_ENABLED(CONFIG_FUSE_DAX)) { + if (flags & FUSE_MAP_ALIGNMENT && + !fuse_dax_check_alignment(fc, arg->map_alignment)) { + ok = false; + } + if (flags & FUSE_HAS_INODE_DAX) + fc->inode_dax = 1; + } + if (flags & FUSE_HANDLE_KILLPRIV_V2) { + fc->handle_killpriv_v2 = 1; + fm->sb->s_flags |= SB_NOSEC; + } + if (flags & FUSE_SETXATTR_EXT) + fc->setxattr_ext = 1; + if (flags & FUSE_SECURITY_CTX) + fc->init_security = 1; + if (flags & FUSE_CREATE_SUPP_GROUP) + fc->create_supp_group = 1; + if (flags & FUSE_DIRECT_IO_ALLOW_MMAP) + fc->direct_io_allow_mmap = 1; + /* + * max_stack_depth is the max stack depth of FUSE fs, + * so it has to be at least 1 to support passthrough + * to backing files. + * + * with max_stack_depth > 1, the backing files can be + * on a stacked fs (e.g. overlayfs) themselves and with + * max_stack_depth == 1, FUSE fs can be stacked as the + * underlying fs of a stacked fs (e.g. overlayfs). + * + * Also don't allow the combination of FUSE_PASSTHROUGH + * and FUSE_WRITEBACK_CACHE, current design doesn't handle + * them together. + */ + if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH) && + (flags & FUSE_PASSTHROUGH) && + arg->max_stack_depth > 0 && + arg->max_stack_depth <= FILESYSTEM_MAX_STACK_DEPTH && + !(flags & FUSE_WRITEBACK_CACHE)) { + fc->passthrough = 1; + fc->max_stack_depth = arg->max_stack_depth; + fm->sb->s_stack_depth = arg->max_stack_depth; + } + if (flags & FUSE_NO_EXPORT_SUPPORT) + fm->sb->s_export_op = &fuse_export_fid_operations; + if (flags & FUSE_ALLOW_IDMAP) { + if (fc->default_permissions) + fm->sb->s_iflags &= ~SB_I_NOIDMAP; + else + ok = false; + } + if (flags & FUSE_OVER_IO_URING && fuse_uring_enabled()) + fuse_chan_io_uring_enable(fc->chan); + + if (flags & FUSE_REQUEST_TIMEOUT) + timeout = arg->request_timeout; + } else { + ra_pages = fc->max_read / PAGE_SIZE; + fc->no_lock = 1; + fc->no_flock = 1; + } + + fuse_init_server_timeout(fc->chan, timeout); + + fm->sb->s_bdi->ra_pages = + min(fm->sb->s_bdi->ra_pages, ra_pages); + fc->minor = arg->minor; + fc->max_write = arg->minor < 5 ? 4096 : arg->max_write; + fc->max_write = max_t(unsigned, 4096, fc->max_write); + fc->conn_init = 1; + } + kfree(ia); + + if (!ok) { + fc->conn_init = 0; + fc->conn_error = 1; + fuse_chan_set_initialized(fc->chan, NULL); + } else { + struct fuse_chan_param cp = { + .minor = fc->minor, + .max_write = fc->max_write, + .max_pages = fc->max_pages, + }; + fuse_chan_set_initialized(fc->chan, &cp); + } +} + +static struct fuse_init_args *fuse_new_init(struct fuse_mount *fm) +{ + struct fuse_init_args *ia; + u64 flags; + + ia = kzalloc_obj(*ia, GFP_KERNEL | __GFP_NOFAIL); + + ia->fm = fm; + ia->in.major = FUSE_KERNEL_VERSION; + ia->in.minor = FUSE_KERNEL_MINOR_VERSION; + ia->in.max_readahead = fm->sb->s_bdi->ra_pages * PAGE_SIZE; + flags = + FUSE_ASYNC_READ | FUSE_POSIX_LOCKS | FUSE_ATOMIC_O_TRUNC | + FUSE_EXPORT_SUPPORT | FUSE_BIG_WRITES | FUSE_DONT_MASK | + FUSE_SPLICE_WRITE | FUSE_SPLICE_MOVE | FUSE_SPLICE_READ | + FUSE_FLOCK_LOCKS | FUSE_HAS_IOCTL_DIR | FUSE_AUTO_INVAL_DATA | + FUSE_DO_READDIRPLUS | FUSE_READDIRPLUS_AUTO | FUSE_ASYNC_DIO | + FUSE_WRITEBACK_CACHE | FUSE_NO_OPEN_SUPPORT | + FUSE_PARALLEL_DIROPS | FUSE_HANDLE_KILLPRIV | FUSE_POSIX_ACL | + FUSE_ABORT_ERROR | FUSE_MAX_PAGES | FUSE_CACHE_SYMLINKS | + FUSE_NO_OPENDIR_SUPPORT | FUSE_EXPLICIT_INVAL_DATA | + FUSE_HANDLE_KILLPRIV_V2 | FUSE_SETXATTR_EXT | FUSE_INIT_EXT | + FUSE_SECURITY_CTX | FUSE_CREATE_SUPP_GROUP | + FUSE_HAS_EXPIRE_ONLY | FUSE_DIRECT_IO_ALLOW_MMAP | + FUSE_NO_EXPORT_SUPPORT | FUSE_HAS_RESEND | FUSE_ALLOW_IDMAP | + FUSE_REQUEST_TIMEOUT; +#ifdef CONFIG_FUSE_DAX + if (fm->fc->dax) + flags |= FUSE_MAP_ALIGNMENT; + if (fuse_is_inode_dax_mode(fm->fc->dax_mode)) + flags |= FUSE_HAS_INODE_DAX; +#endif + if (fm->fc->auto_submounts) + flags |= FUSE_SUBMOUNTS; + if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH)) + flags |= FUSE_PASSTHROUGH; + + /* + * This is just an information flag for fuse server. No need to check + * the reply - server is either sending IORING_OP_URING_CMD or not. + */ + if (fuse_uring_enabled()) + flags |= FUSE_OVER_IO_URING; + + ia->in.flags = flags; + ia->in.flags2 = flags >> 32; + + ia->args.opcode = FUSE_INIT; + ia->args.in_numargs = 1; + ia->args.in_args[0].size = sizeof(ia->in); + ia->args.in_args[0].value = &ia->in; + ia->args.out_numargs = 1; + /* Variable length argument used for backward compatibility + with interface version < 7.5. Rest of init_out is zeroed + by do_get_request(), so a short reply is not a problem */ + ia->args.out_argvar = true; + ia->args.out_args[0].size = sizeof(ia->out); + ia->args.out_args[0].value = &ia->out; + ia->args.force = true; + ia->args.nocreds = true; + + return ia; +} + +int fuse_send_init(struct fuse_mount *fm) +{ + struct fuse_init_args *ia = fuse_new_init(fm); + int err; + + if (fm->fc->sync_init) { + ia->args.abort_on_kill = true; + err = fuse_simple_request(fm, &ia->args); + /* Ignore size of init reply */ + if (err > 0) + err = 0; + } else { + ia->args.end = process_init_reply; + err = fuse_simple_background(fm, &ia->args, GFP_KERNEL); + if (!err) + return 0; + } + process_init_reply(&ia->args, err); + if (fm->fc->conn_error) + return -ENOTCONN; + return 0; +} +EXPORT_SYMBOL_GPL(fuse_send_init); + +void fuse_free_conn(struct fuse_conn *fc) +{ + kfree(fc); +} +EXPORT_SYMBOL_GPL(fuse_free_conn); + +static int fuse_bdi_init(struct fuse_conn *fc, struct super_block *sb) +{ + int err; + char *suffix = ""; + + if (sb->s_bdev) { + suffix = "-fuseblk"; + /* + * sb->s_bdi points to blkdev's bdi however we want to redirect + * it to our private bdi... + */ + bdi_put(sb->s_bdi); + sb->s_bdi = &noop_backing_dev_info; + } + err = super_setup_bdi_name(sb, "%u:%u%s", MAJOR(fc->dev), + MINOR(fc->dev), suffix); + if (err) + return err; + + sb->s_bdi->capabilities |= BDI_CAP_STRICTLIMIT; + + /* + * For a single fuse filesystem use max 1% of dirty + + * writeback threshold. + * + * This gives about 1M of write buffer for memory maps on a + * machine with 1G and 10% dirty_ratio, which should be more + * than enough. + * + * Privileged users can raise it by writing to + * + * /sys/class/bdi//max_ratio + */ + bdi_set_max_ratio(sb->s_bdi, 1); + + return 0; +} + +static void fuse_fill_attr_from_inode(struct fuse_attr *attr, + const struct fuse_inode *fi) +{ + struct timespec64 atime = inode_get_atime(&fi->inode); + struct timespec64 mtime = inode_get_mtime(&fi->inode); + struct timespec64 ctime = inode_get_ctime(&fi->inode); + + *attr = (struct fuse_attr){ + .ino = fi->inode.i_ino, + .size = fi->inode.i_size, + .blocks = fi->inode.i_blocks, + .atime = atime.tv_sec, + .mtime = mtime.tv_sec, + .ctime = ctime.tv_sec, + .atimensec = atime.tv_nsec, + .mtimensec = mtime.tv_nsec, + .ctimensec = ctime.tv_nsec, + .mode = fi->inode.i_mode, + .nlink = fi->inode.i_nlink, + .uid = __kuid_val(fi->inode.i_uid), + .gid = __kgid_val(fi->inode.i_gid), + .rdev = fi->inode.i_rdev, + .blksize = 1u << fi->inode.i_blkbits, + }; +} + +static void fuse_sb_defaults(struct super_block *sb) +{ + sb->s_magic = FUSE_SUPER_MAGIC; + sb->s_op = &fuse_super_operations; + sb->s_xattr = fuse_xattr_handlers; + sb->s_maxbytes = MAX_LFS_FILESIZE; + sb->s_time_gran = 1; + sb->s_export_op = &fuse_export_operations; + sb->s_iflags |= SB_I_IMA_UNVERIFIABLE_SIGNATURE; + sb->s_iflags |= SB_I_NOIDMAP; + sb->s_iflags |= SB_I_NO_DATA_INTEGRITY; + if (sb->s_user_ns != &init_user_ns) + sb->s_iflags |= SB_I_UNTRUSTED_MOUNTER; + sb->s_flags &= ~(SB_NOSEC | SB_I_VERSION); +} + +static int fuse_fill_super_submount(struct super_block *sb, + struct fuse_inode *parent_fi) +{ + struct fuse_mount *fm = get_fuse_mount_super(sb); + struct super_block *parent_sb = parent_fi->inode.i_sb; + struct fuse_attr root_attr; + struct inode *root; + struct fuse_submount_lookup *sl; + struct fuse_inode *fi; + + fuse_sb_defaults(sb); + fm->sb = sb; + + WARN_ON(sb->s_bdi != &noop_backing_dev_info); + sb->s_bdi = bdi_get(parent_sb->s_bdi); + + sb->s_xattr = parent_sb->s_xattr; + sb->s_export_op = parent_sb->s_export_op; + sb->s_time_gran = parent_sb->s_time_gran; + sb->s_blocksize = parent_sb->s_blocksize; + sb->s_blocksize_bits = parent_sb->s_blocksize_bits; + sb->s_subtype = kstrdup(parent_sb->s_subtype, GFP_KERNEL); + if (parent_sb->s_subtype && !sb->s_subtype) + return -ENOMEM; + + fuse_fill_attr_from_inode(&root_attr, parent_fi); + root = fuse_iget(sb, parent_fi->nodeid, 0, &root_attr, 0, 0, + fuse_get_evict_ctr(fm->fc)); + /* + * This inode is just a duplicate, so it is not looked up and + * its nlookup should not be incremented. fuse_iget() does + * that, though, so undo it here. + */ + fi = get_fuse_inode(root); + fi->nlookup--; + + set_default_d_op(sb, &fuse_dentry_operations); + sb->s_root = d_make_root(root); + if (!sb->s_root) + return -ENOMEM; + + /* + * Grab the parent's submount_lookup pointer and take a + * reference on the shared nlookup from the parent. This is to + * prevent the last forget for this nodeid from getting + * triggered until all users have finished with it. + */ + sl = parent_fi->submount_lookup; + WARN_ON(!sl); + if (sl) { + refcount_inc(&sl->count); + fi->submount_lookup = sl; + } + + return 0; +} + +/* Filesystem context private data holds the FUSE inode of the mount point */ +static int fuse_get_tree_submount(struct fs_context *fsc) +{ + struct fuse_mount *fm; + struct fuse_inode *mp_fi = fsc->fs_private; + struct fuse_conn *fc = get_fuse_conn(&mp_fi->inode); + struct super_block *sb; + int err; + + fm = kzalloc_obj(struct fuse_mount); + if (!fm) + return -ENOMEM; + + fm->fc = fuse_conn_get(fc); + fsc->s_fs_info = fm; + sb = sget_fc(fsc, NULL, set_anon_super_fc); + if (fsc->s_fs_info) + fuse_mount_destroy(fm); + if (IS_ERR(sb)) + return PTR_ERR(sb); + + /* Initialize superblock, making @mp_fi its root */ + err = fuse_fill_super_submount(sb, mp_fi); + if (err) { + deactivate_locked_super(sb); + return err; + } + + down_write(&fc->killsb); + list_add_tail(&fm->fc_entry, &fc->mounts); + up_write(&fc->killsb); + + sb->s_flags |= SB_ACTIVE; + fsc->root = dget(sb->s_root); + + return 0; +} + +static const struct fs_context_operations fuse_context_submount_ops = { + .get_tree = fuse_get_tree_submount, +}; + +int fuse_init_fs_context_submount(struct fs_context *fsc) +{ + fsc->ops = &fuse_context_submount_ops; + return 0; +} +EXPORT_SYMBOL_GPL(fuse_init_fs_context_submount); + +int fuse_fill_super_common(struct super_block *sb, struct fuse_fs_context *ctx) +{ + struct fuse_dev *fud = ctx->fud; + struct fuse_mount *fm = get_fuse_mount_super(sb); + struct fuse_conn *fc = fm->fc; + struct inode *root; + struct dentry *root_dentry; + int err; + + err = -EINVAL; + if (sb->s_flags & SB_MANDLOCK) + goto err; + + rcu_assign_pointer(fc->curr_bucket, fuse_sync_bucket_alloc()); + fuse_sb_defaults(sb); + + if (ctx->is_bdev) { +#ifdef CONFIG_BLOCK + err = -EINVAL; + if (!sb_set_blocksize(sb, ctx->blksize)) + goto err; +#endif + fc->sync_fs = 1; + } else { + sb->s_blocksize = PAGE_SIZE; + sb->s_blocksize_bits = PAGE_SHIFT; + } + + sb->s_subtype = ctx->subtype; + ctx->subtype = NULL; + if (IS_ENABLED(CONFIG_FUSE_DAX)) { + err = fuse_dax_conn_alloc(fc, ctx->dax_mode, ctx->dax_dev); + if (err) + goto err; + } + + fc->dev = sb->s_dev; + fm->sb = sb; + err = fuse_bdi_init(fc, sb); + if (err) + goto err_free_dax; + + /* Handle umasking inside the fuse code */ + if (sb->s_flags & SB_POSIXACL) + fc->dont_mask = 1; + sb->s_flags |= SB_POSIXACL; + + fc->default_permissions = ctx->default_permissions; + fc->allow_other = ctx->allow_other; + fc->user_id = ctx->user_id; + fc->group_id = ctx->group_id; + fc->legacy_opts_show = ctx->legacy_opts_show; + fc->max_read = max_t(unsigned int, 4096, ctx->max_read); + fc->destroy = ctx->destroy; + fc->no_control = ctx->no_control; + fc->no_force_umount = ctx->no_force_umount; + + err = -ENOMEM; + root = fuse_get_root_inode(sb, ctx->rootmode); + set_default_d_op(sb, &fuse_dentry_operations); + root_dentry = d_make_root(root); + if (!root_dentry) + goto err_free_dax; + + mutex_lock(&fuse_mutex); + err = -EINVAL; + if (fud) { + if (fuse_dev_is_installed(fud)) + goto err_unlock; + if (fuse_dev_is_sync_init(fud)) + fc->sync_init = 1; + } + + err = fuse_ctl_add_conn(fc); + if (err) + goto err_unlock; + + list_add_tail(&fc->entry, &fuse_conn_list); + sb->s_root = root_dentry; + if (fud) + fuse_dev_install(fud, fc->chan); + + mutex_unlock(&fuse_mutex); + return 0; + + err_unlock: + mutex_unlock(&fuse_mutex); + dput(root_dentry); + err_free_dax: + if (IS_ENABLED(CONFIG_FUSE_DAX)) + fuse_dax_conn_free(fc); + err: + return err; +} +EXPORT_SYMBOL_GPL(fuse_fill_super_common); + +static int fuse_fill_super(struct super_block *sb, struct fs_context *fsc) +{ + struct fuse_fs_context *ctx = fsc->fs_private; + struct fuse_mount *fm; + int err; + + if (!ctx->fud || !ctx->rootmode_present || + !ctx->user_id_present || !ctx->group_id_present) + return -EINVAL; + + err = fuse_fill_super_common(sb, ctx); + if (err) + return err; + + fm = get_fuse_mount_super(sb); + + return fuse_send_init(fm); +} + +/* + * This is the path where user supplied an already initialized fuse dev. In + * this case never create a new super if the old one is gone. + */ +static int fuse_set_no_super(struct super_block *sb, struct fs_context *fsc) +{ + return -ENOTCONN; +} + +static int fuse_test_super(struct super_block *sb, struct fs_context *fsc) +{ + return fuse_dev_verify(fsc->sget_key, get_fuse_conn_super(sb)->chan); +} + +static int fuse_get_tree(struct fs_context *fsc) +{ + struct fuse_fs_context *ctx = fsc->fs_private; + struct fuse_conn *fc; + struct fuse_mount *fm; + struct super_block *sb; + struct fuse_chan *fch __free(fuse_chan_free) = fuse_dev_chan_new(); + int err; + + if (!fch) + return -ENOMEM; + + fc = kmalloc_obj(*fc); + if (!fc) + return -ENOMEM; + + fm = kzalloc_obj(*fm); + if (!fm) { + kfree(fc); + return -ENOMEM; + } + + fuse_conn_init(fc, fm, fsc->user_ns, no_free_ptr(fch)); + fc->release = fuse_free_conn; + + fsc->s_fs_info = fm; + + if (IS_ENABLED(CONFIG_BLOCK) && ctx->is_bdev) { + err = get_tree_bdev(fsc, fuse_fill_super); + goto out; + } + /* + * While block dev mount can be initialized with a dummy device fd + * (found by device name), normal fuse mounts can't + */ + err = -EINVAL; + if (!ctx->fud) + goto out; + + /* + * Allow creating a fuse mount with an already initialized fuse + * connection + */ + if (fuse_dev_is_installed(ctx->fud)) { + fsc->sget_key = ctx->fud; + sb = sget_fc(fsc, fuse_test_super, fuse_set_no_super); + err = PTR_ERR_OR_ZERO(sb); + if (!IS_ERR(sb)) + fsc->root = dget(sb->s_root); + } else { + err = get_tree_nodev(fsc, fuse_fill_super); + } +out: + if (fsc->s_fs_info) + fuse_mount_destroy(fm); + return err; +} + +static const struct fs_context_operations fuse_context_ops = { + .free = fuse_free_fsc, + .parse_param = fuse_parse_param, + .reconfigure = fuse_reconfigure, + .get_tree = fuse_get_tree, +}; + +/* + * Set up the filesystem mount context. + */ +static int fuse_init_fs_context(struct fs_context *fsc) +{ + struct fuse_fs_context *ctx; + + ctx = kzalloc_obj(struct fuse_fs_context); + if (!ctx) + return -ENOMEM; + + ctx->max_read = ~0; + ctx->blksize = FUSE_DEFAULT_BLKSIZE; + ctx->legacy_opts_show = true; + +#ifdef CONFIG_BLOCK + if (fsc->fs_type == &fuseblk_fs_type) { + ctx->is_bdev = true; + ctx->destroy = true; + } +#endif + + fsc->fs_private = ctx; + fsc->ops = &fuse_context_ops; + return 0; +} + +bool fuse_mount_remove(struct fuse_mount *fm) +{ + struct fuse_conn *fc = fm->fc; + bool last = false; + + down_write(&fc->killsb); + list_del_init(&fm->fc_entry); + if (list_empty(&fc->mounts)) + last = true; + up_write(&fc->killsb); + + return last; +} +EXPORT_SYMBOL_GPL(fuse_mount_remove); + +void fuse_conn_destroy(struct fuse_mount *fm) +{ + struct fuse_conn *fc = fm->fc; + + if (fc->destroy) + fuse_send_destroy(fm); + + fuse_chan_abort(fc->chan, false); + fuse_chan_wait_aborted(fc->chan); + + if (!list_empty(&fc->entry)) { + mutex_lock(&fuse_mutex); + list_del(&fc->entry); + fuse_ctl_remove_conn(fc); + mutex_unlock(&fuse_mutex); + } +} +EXPORT_SYMBOL_GPL(fuse_conn_destroy); + +static void fuse_sb_destroy(struct super_block *sb) +{ + struct fuse_mount *fm = get_fuse_mount_super(sb); + bool last; + + if (sb->s_root) { + last = fuse_mount_remove(fm); + if (last) + fuse_conn_destroy(fm); + } +} + +void fuse_mount_destroy(struct fuse_mount *fm) +{ + fuse_conn_put(fm->fc); + kfree_rcu(fm, rcu); +} +EXPORT_SYMBOL(fuse_mount_destroy); + +static void fuse_kill_sb_anon(struct super_block *sb) +{ + fuse_sb_destroy(sb); + kill_anon_super(sb); + fuse_mount_destroy(get_fuse_mount_super(sb)); +} + +static struct file_system_type fuse_fs_type = { + .owner = THIS_MODULE, + .name = "fuse", + .fs_flags = FS_HAS_SUBTYPE | FS_USERNS_MOUNT | FS_ALLOW_IDMAP, + .init_fs_context = fuse_init_fs_context, + .parameters = fuse_fs_parameters, + .kill_sb = fuse_kill_sb_anon, +}; +MODULE_ALIAS_FS("fuse"); + +#ifdef CONFIG_BLOCK +static void fuse_kill_sb_blk(struct super_block *sb) +{ + fuse_sb_destroy(sb); + kill_block_super(sb); + fuse_mount_destroy(get_fuse_mount_super(sb)); +} + +static struct file_system_type fuseblk_fs_type = { + .owner = THIS_MODULE, + .name = "fuseblk", + .init_fs_context = fuse_init_fs_context, + .parameters = fuse_fs_parameters, + .kill_sb = fuse_kill_sb_blk, + .fs_flags = FS_REQUIRES_DEV | FS_HAS_SUBTYPE | FS_ALLOW_IDMAP, +}; +MODULE_ALIAS_FS("fuseblk"); + +static inline int register_fuseblk(void) +{ + return register_filesystem(&fuseblk_fs_type); +} + +static inline void unregister_fuseblk(void) +{ + unregister_filesystem(&fuseblk_fs_type); +} +#else +static inline int register_fuseblk(void) +{ + return 0; +} + +static inline void unregister_fuseblk(void) +{ +} +#endif + +static void fuse_inode_init_once(void *foo) +{ + struct inode *inode = foo; + + inode_init_once(inode); +} + +static int __init fuse_fs_init(void) +{ + int err; + + fuse_inode_cachep = kmem_cache_create("fuse_inode", + sizeof(struct fuse_inode), 0, + SLAB_HWCACHE_ALIGN|SLAB_ACCOUNT|SLAB_RECLAIM_ACCOUNT, + fuse_inode_init_once); + err = -ENOMEM; + if (!fuse_inode_cachep) + goto out; + + err = register_fuseblk(); + if (err) + goto out2; + + err = register_filesystem(&fuse_fs_type); + if (err) + goto out3; + + err = fuse_sysctl_register(); + if (err) + goto out4; + + return 0; + + out4: + unregister_filesystem(&fuse_fs_type); + out3: + unregister_fuseblk(); + out2: + kmem_cache_destroy(fuse_inode_cachep); + out: + return err; +} + +static void fuse_fs_cleanup(void) +{ + fuse_sysctl_unregister(); + unregister_filesystem(&fuse_fs_type); + unregister_fuseblk(); + + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(fuse_inode_cachep); +} + +static struct kobject *fuse_kobj; + +static int fuse_sysfs_init(void) +{ + int err; + + fuse_kobj = kobject_create_and_add("fuse", fs_kobj); + if (!fuse_kobj) { + err = -ENOMEM; + goto out_err; + } + + err = sysfs_create_mount_point(fuse_kobj, "connections"); + if (err) + goto out_fuse_unregister; + + return 0; + + out_fuse_unregister: + kobject_put(fuse_kobj); + out_err: + return err; +} + +static void fuse_sysfs_cleanup(void) +{ + sysfs_remove_mount_point(fuse_kobj, "connections"); + kobject_put(fuse_kobj); +} + +static int __init fuse_init(void) +{ + int res; + + pr_info("init (API version %i.%i)\n", + FUSE_KERNEL_VERSION, FUSE_KERNEL_MINOR_VERSION); + + INIT_LIST_HEAD(&fuse_conn_list); + res = fuse_fs_init(); + if (res) + goto err; + + res = fuse_dev_init(); + if (res) + goto err_fs_cleanup; + + res = fuse_sysfs_init(); + if (res) + goto err_dev_cleanup; + + res = fuse_ctl_init(); + if (res) + goto err_sysfs_cleanup; + + fuse_dentry_tree_init(); + + sanitize_global_limit(&max_user_bgreq); + sanitize_global_limit(&max_user_congthresh); + + return 0; + + err_sysfs_cleanup: + fuse_sysfs_cleanup(); + err_dev_cleanup: + fuse_dev_cleanup(); + err_fs_cleanup: + fuse_fs_cleanup(); + err: + return res; +} + +static void __exit fuse_exit(void) +{ + pr_debug("exit\n"); + + fuse_dentry_tree_cleanup(); + fuse_ctl_cleanup(); + fuse_sysfs_cleanup(); + fuse_fs_cleanup(); + fuse_dev_cleanup(); +} + +module_init(fuse_init); +module_exit(fuse_exit); diff --git a/fs/fuse/ioctl.c b/fs/fuse/ioctl.c new file mode 100644 index 000000000..3614ea603 --- /dev/null +++ b/fs/fuse/ioctl.c @@ -0,0 +1,577 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017 Red Hat, Inc. + */ + +#include "fuse_i.h" + +#include +#include +#include +#include + +#include +#define FUSE_VERITY_ENABLE_ARG_MAX_PAGES 256 + +static ssize_t fuse_send_ioctl(struct fuse_mount *fm, struct fuse_args *args, + struct fuse_ioctl_out *outarg) +{ + ssize_t ret; + + args->out_args[0].size = sizeof(*outarg); + args->out_args[0].value = outarg; + + ret = fuse_simple_request(fm, args); + + /* Translate ENOSYS, which shouldn't be returned from fs */ + if (ret == -ENOSYS) + ret = -ENOTTY; + + if (ret >= 0 && outarg->result == -ENOSYS) + outarg->result = -ENOTTY; + + return ret; +} + +/* + * CUSE servers compiled on 32bit broke on 64bit kernels because the + * ABI was defined to be 'struct iovec' which is different on 32bit + * and 64bit. Fortunately we can determine which structure the server + * used from the size of the reply. + */ +static int fuse_copy_ioctl_iovec_old(struct iovec *dst, void *src, + size_t transferred, unsigned count, + bool is_compat) +{ +#ifdef CONFIG_COMPAT + if (count * sizeof(struct compat_iovec) == transferred) { + struct compat_iovec *ciov = src; + unsigned i; + + /* + * With this interface a 32bit server cannot support + * non-compat (i.e. ones coming from 64bit apps) ioctl + * requests + */ + if (!is_compat) + return -EINVAL; + + for (i = 0; i < count; i++) { + dst[i].iov_base = compat_ptr(ciov[i].iov_base); + dst[i].iov_len = ciov[i].iov_len; + } + return 0; + } +#endif + + if (count * sizeof(struct iovec) != transferred) + return -EIO; + + memcpy(dst, src, transferred); + return 0; +} + +/* Make sure iov_length() won't overflow */ +static int fuse_verify_ioctl_iov(struct fuse_conn *fc, struct iovec *iov, + size_t count) +{ + size_t n; + u32 max = fc->max_pages << PAGE_SHIFT; + + for (n = 0; n < count; n++, iov++) { + if (iov->iov_len > (size_t) max) + return -ENOMEM; + max -= iov->iov_len; + } + return 0; +} + +static int fuse_copy_ioctl_iovec(struct fuse_conn *fc, struct iovec *dst, + void *src, size_t transferred, unsigned count, + bool is_compat) +{ + unsigned i; + struct fuse_ioctl_iovec *fiov = src; + + if (fc->minor < 16) { + return fuse_copy_ioctl_iovec_old(dst, src, transferred, + count, is_compat); + } + + if (count * sizeof(struct fuse_ioctl_iovec) != transferred) + return -EIO; + + for (i = 0; i < count; i++) { + /* Did the server supply an inappropriate value? */ + if (fiov[i].base != (unsigned long) fiov[i].base || + fiov[i].len != (unsigned long) fiov[i].len) + return -EIO; + + dst[i].iov_base = (void __user *) (unsigned long) fiov[i].base; + dst[i].iov_len = (size_t) fiov[i].len; + +#ifdef CONFIG_COMPAT + if (is_compat && + (ptr_to_compat(dst[i].iov_base) != fiov[i].base || + (compat_size_t) dst[i].iov_len != fiov[i].len)) + return -EIO; +#endif + } + + return 0; +} + +/* For fs-verity, determine iov lengths from input */ +static int fuse_setup_measure_verity(unsigned long arg, struct iovec *iov) +{ + __u16 digest_size; + struct fsverity_digest __user *uarg = (void __user *)arg; + + if (copy_from_user(&digest_size, &uarg->digest_size, sizeof(digest_size))) + return -EFAULT; + + if (digest_size > SIZE_MAX - sizeof(struct fsverity_digest)) + return -EINVAL; + + iov->iov_len = sizeof(struct fsverity_digest) + digest_size; + + return 0; +} + +static int fuse_setup_enable_verity(unsigned long arg, struct iovec *iov, + unsigned int *in_iovs) +{ + struct fsverity_enable_arg enable; + struct fsverity_enable_arg __user *uarg = (void __user *)arg; + const __u32 max_buffer_len = FUSE_VERITY_ENABLE_ARG_MAX_PAGES * PAGE_SIZE; + + if (copy_from_user(&enable, uarg, sizeof(enable))) + return -EFAULT; + + if (enable.salt_size > max_buffer_len || enable.sig_size > max_buffer_len) + return -ENOMEM; + + if (enable.salt_size > 0) { + iov++; + (*in_iovs)++; + + iov->iov_base = u64_to_user_ptr(enable.salt_ptr); + iov->iov_len = enable.salt_size; + } + + if (enable.sig_size > 0) { + iov++; + (*in_iovs)++; + + iov->iov_base = u64_to_user_ptr(enable.sig_ptr); + iov->iov_len = enable.sig_size; + } + return 0; +} + +/* + * For ioctls, there is no generic way to determine how much memory + * needs to be read and/or written. Furthermore, ioctls are allowed + * to dereference the passed pointer, so the parameter requires deep + * copying but FUSE has no idea whatsoever about what to copy in or + * out. + * + * This is solved by allowing FUSE server to retry ioctl with + * necessary in/out iovecs. Let's assume the ioctl implementation + * needs to read in the following structure. + * + * struct a { + * char *buf; + * size_t buflen; + * } + * + * On the first callout to FUSE server, inarg->in_size and + * inarg->out_size will be NULL; then, the server completes the ioctl + * with FUSE_IOCTL_RETRY set in out->flags, out->in_iovs set to 1 and + * the actual iov array to + * + * { { .iov_base = inarg.arg, .iov_len = sizeof(struct a) } } + * + * which tells FUSE to copy in the requested area and retry the ioctl. + * On the second round, the server has access to the structure and + * from that it can tell what to look for next, so on the invocation, + * it sets FUSE_IOCTL_RETRY, out->in_iovs to 2 and iov array to + * + * { { .iov_base = inarg.arg, .iov_len = sizeof(struct a) }, + * { .iov_base = a.buf, .iov_len = a.buflen } } + * + * FUSE will copy both struct a and the pointed buffer from the + * process doing the ioctl and retry ioctl with both struct a and the + * buffer. + * + * This time, FUSE server has everything it needs and completes ioctl + * without FUSE_IOCTL_RETRY which finishes the ioctl call. + * + * Copying data out works the same way. + * + * Note that if FUSE_IOCTL_UNRESTRICTED is clear, the kernel + * automatically initializes in and out iovs by decoding @cmd with + * _IOC_* macros and the server is not allowed to request RETRY. This + * limits ioctl data transfers to well-formed ioctls and is the forced + * behavior for all FUSE servers. + */ +long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg, + unsigned int flags) +{ + struct fuse_file *ff = file->private_data; + struct fuse_mount *fm = ff->fm; + struct fuse_ioctl_in inarg = { + .fh = ff->fh, + .cmd = cmd, + .arg = arg, + .flags = flags + }; + struct fuse_ioctl_out outarg; + struct iovec *iov_page = NULL; + struct iovec *in_iov = NULL, *out_iov = NULL; + unsigned int in_iovs = 0, out_iovs = 0, max_pages; + size_t in_size, out_size, c; + ssize_t transferred; + int err, i; + struct iov_iter ii; + struct fuse_args_pages ap = {}; + +#if BITS_PER_LONG == 32 + inarg.flags |= FUSE_IOCTL_32BIT; +#else + if (flags & FUSE_IOCTL_COMPAT) { + inarg.flags |= FUSE_IOCTL_32BIT; +#ifdef CONFIG_X86_X32_ABI + if (in_x32_syscall()) + inarg.flags |= FUSE_IOCTL_COMPAT_X32; +#endif + } +#endif + + /* assume all the iovs returned by client always fits in a page */ + BUILD_BUG_ON(sizeof(struct fuse_ioctl_iovec) * FUSE_IOCTL_MAX_IOV > PAGE_SIZE); + + err = -ENOMEM; + ap.folios = fuse_folios_alloc(fm->fc->max_pages, GFP_KERNEL, &ap.descs); + iov_page = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!ap.folios || !iov_page) + goto out; + + fuse_folio_descs_length_init(ap.descs, 0, fm->fc->max_pages); + + /* + * If restricted, initialize IO parameters as encoded in @cmd. + * RETRY from server is not allowed. + */ + if (!(flags & FUSE_IOCTL_UNRESTRICTED)) { + struct iovec *iov = iov_page; + + iov->iov_base = (void __user *)arg; + iov->iov_len = _IOC_SIZE(cmd); + + if (_IOC_DIR(cmd) & _IOC_WRITE) { + in_iov = iov; + in_iovs = 1; + } + + if (_IOC_DIR(cmd) & _IOC_READ) { + out_iov = iov; + out_iovs = 1; + } + + err = 0; + switch (cmd) { + case FS_IOC_MEASURE_VERITY: + err = fuse_setup_measure_verity(arg, iov); + break; + case FS_IOC_ENABLE_VERITY: + err = fuse_setup_enable_verity(arg, iov, &in_iovs); + break; + } + if (err) + goto out; + } + + retry: + inarg.in_size = in_size = iov_length(in_iov, in_iovs); + inarg.out_size = out_size = iov_length(out_iov, out_iovs); + + /* + * Out data can be used either for actual out data or iovs, + * make sure there always is at least one page. + */ + out_size = max_t(size_t, out_size, PAGE_SIZE); + max_pages = DIV_ROUND_UP(max(in_size, out_size), PAGE_SIZE); + + /* make sure there are enough buffer pages and init request with them */ + err = -ENOMEM; + if (max_pages > fm->fc->max_pages) + goto out; + while (ap.num_folios < max_pages) { + ap.folios[ap.num_folios] = folio_alloc(GFP_KERNEL | __GFP_HIGHMEM, 0); + if (!ap.folios[ap.num_folios]) + goto out; + ap.num_folios++; + } + + /* okay, let's send it to the client */ + ap.args.opcode = FUSE_IOCTL; + ap.args.nodeid = ff->nodeid; + ap.args.in_numargs = 1; + ap.args.in_args[0].size = sizeof(inarg); + ap.args.in_args[0].value = &inarg; + if (in_size) { + ap.args.in_numargs++; + ap.args.in_args[1].size = in_size; + ap.args.in_pages = true; + + err = -EFAULT; + iov_iter_init(&ii, ITER_SOURCE, in_iov, in_iovs, in_size); + for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= ap.num_folios); i++) { + c = copy_folio_from_iter(ap.folios[i], 0, PAGE_SIZE, &ii); + if (c != PAGE_SIZE && iov_iter_count(&ii)) + goto out; + } + } + + ap.args.out_numargs = 2; + ap.args.out_args[1].size = out_size; + ap.args.out_pages = true; + ap.args.out_argvar = true; + + transferred = fuse_send_ioctl(fm, &ap.args, &outarg); + err = transferred; + if (transferred < 0) + goto out; + + /* did it ask for retry? */ + if (outarg.flags & FUSE_IOCTL_RETRY) { + void *vaddr; + + /* no retry if in restricted mode */ + err = -EIO; + if (!(flags & FUSE_IOCTL_UNRESTRICTED)) + goto out; + + in_iovs = outarg.in_iovs; + out_iovs = outarg.out_iovs; + + /* + * Make sure things are in boundary, separate checks + * are to protect against overflow. + */ + err = -ENOMEM; + if (in_iovs > FUSE_IOCTL_MAX_IOV || + out_iovs > FUSE_IOCTL_MAX_IOV || + in_iovs + out_iovs > FUSE_IOCTL_MAX_IOV) + goto out; + + vaddr = kmap_local_folio(ap.folios[0], 0); + err = fuse_copy_ioctl_iovec(fm->fc, iov_page, vaddr, + transferred, in_iovs + out_iovs, + (flags & FUSE_IOCTL_COMPAT) != 0); + kunmap_local(vaddr); + if (err) + goto out; + + in_iov = iov_page; + out_iov = in_iov + in_iovs; + + err = fuse_verify_ioctl_iov(fm->fc, in_iov, in_iovs); + if (err) + goto out; + + err = fuse_verify_ioctl_iov(fm->fc, out_iov, out_iovs); + if (err) + goto out; + + goto retry; + } + + err = -EIO; + if (transferred > inarg.out_size) + goto out; + + err = -EFAULT; + iov_iter_init(&ii, ITER_DEST, out_iov, out_iovs, transferred); + for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= ap.num_folios); i++) { + c = copy_folio_to_iter(ap.folios[i], 0, PAGE_SIZE, &ii); + if (c != PAGE_SIZE && iov_iter_count(&ii)) + goto out; + } + err = 0; + out: + kfree(iov_page); + while (ap.num_folios) + folio_put(ap.folios[--ap.num_folios]); + kfree(ap.folios); + + return err ? err : outarg.result; +} +EXPORT_SYMBOL_GPL(fuse_do_ioctl); + +long fuse_ioctl_common(struct file *file, unsigned int cmd, + unsigned long arg, unsigned int flags) +{ + struct inode *inode = file_inode(file); + struct fuse_conn *fc = get_fuse_conn(inode); + + if (!fuse_allow_current_process(fc)) + return -EACCES; + + if (fuse_is_bad(inode)) + return -EIO; + + return fuse_do_ioctl(file, cmd, arg, flags); +} + +long fuse_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + return fuse_ioctl_common(file, cmd, arg, 0); +} + +long fuse_file_compat_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + return fuse_ioctl_common(file, cmd, arg, FUSE_IOCTL_COMPAT); +} + +static int fuse_priv_ioctl(struct inode *inode, struct fuse_file *ff, + unsigned int cmd, void *ptr, size_t size) +{ + struct fuse_mount *fm = ff->fm; + struct fuse_ioctl_in inarg; + struct fuse_ioctl_out outarg; + FUSE_ARGS(args); + int err; + + memset(&inarg, 0, sizeof(inarg)); + inarg.fh = ff->fh; + inarg.cmd = cmd; + +#if BITS_PER_LONG == 32 + inarg.flags |= FUSE_IOCTL_32BIT; +#endif + if (S_ISDIR(inode->i_mode)) + inarg.flags |= FUSE_IOCTL_DIR; + + if (_IOC_DIR(cmd) & _IOC_READ) + inarg.out_size = size; + if (_IOC_DIR(cmd) & _IOC_WRITE) + inarg.in_size = size; + + args.opcode = FUSE_IOCTL; + args.nodeid = ff->nodeid; + args.in_numargs = 2; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + args.in_args[1].size = inarg.in_size; + args.in_args[1].value = ptr; + args.out_numargs = 2; + args.out_args[1].size = inarg.out_size; + args.out_args[1].value = ptr; + + err = fuse_send_ioctl(fm, &args, &outarg); + if (!err) { + if (outarg.result < 0) + err = outarg.result; + else if (outarg.flags & FUSE_IOCTL_RETRY) + err = -EIO; + } + return err; +} + +static struct fuse_file *fuse_priv_ioctl_prepare(struct inode *inode) +{ + struct fuse_mount *fm = get_fuse_mount(inode); + bool isdir = S_ISDIR(inode->i_mode); + + if (!fuse_allow_current_process(fm->fc)) + return ERR_PTR(-EACCES); + + if (fuse_is_bad(inode)) + return ERR_PTR(-EIO); + + if (!S_ISREG(inode->i_mode) && !isdir) + return ERR_PTR(-ENOTTY); + + return fuse_file_open(fm, get_node_id(inode), O_RDONLY, isdir); +} + +static void fuse_priv_ioctl_cleanup(struct inode *inode, struct fuse_file *ff) +{ + fuse_file_release(inode, ff, O_RDONLY, NULL, S_ISDIR(inode->i_mode)); +} + +int fuse_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + struct fuse_file *ff; + unsigned int flags; + struct fsxattr xfa; + int err; + + ff = fuse_priv_ioctl_prepare(inode); + if (IS_ERR(ff)) + return PTR_ERR(ff); + + if (fa->flags_valid) { + err = fuse_priv_ioctl(inode, ff, FS_IOC_GETFLAGS, + &flags, sizeof(flags)); + if (err) + goto cleanup; + + fileattr_fill_flags(fa, flags); + } else { + err = fuse_priv_ioctl(inode, ff, FS_IOC_FSGETXATTR, + &xfa, sizeof(xfa)); + if (err) + goto cleanup; + + fileattr_fill_xflags(fa, xfa.fsx_xflags); + fa->fsx_extsize = xfa.fsx_extsize; + fa->fsx_nextents = xfa.fsx_nextents; + fa->fsx_projid = xfa.fsx_projid; + fa->fsx_cowextsize = xfa.fsx_cowextsize; + } +cleanup: + fuse_priv_ioctl_cleanup(inode, ff); + + return err; +} + +int fuse_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + struct fuse_file *ff; + unsigned int flags = fa->flags; + struct fsxattr xfa; + int err; + + ff = fuse_priv_ioctl_prepare(inode); + if (IS_ERR(ff)) + return PTR_ERR(ff); + + if (fa->flags_valid) { + err = fuse_priv_ioctl(inode, ff, FS_IOC_SETFLAGS, + &flags, sizeof(flags)); + if (err) + goto cleanup; + } else { + memset(&xfa, 0, sizeof(xfa)); + xfa.fsx_xflags = fa->fsx_xflags; + xfa.fsx_extsize = fa->fsx_extsize; + xfa.fsx_nextents = fa->fsx_nextents; + xfa.fsx_projid = fa->fsx_projid; + xfa.fsx_cowextsize = fa->fsx_cowextsize; + + err = fuse_priv_ioctl(inode, ff, FS_IOC_FSSETXATTR, + &xfa, sizeof(xfa)); + } + +cleanup: + fuse_priv_ioctl_cleanup(inode, ff); + + return err; +} diff --git a/fs/fuse/iomode.c b/fs/fuse/iomode.c new file mode 100644 index 000000000..372893318 --- /dev/null +++ b/fs/fuse/iomode.c @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * FUSE inode io modes. + * + * Copyright (c) 2024 CTERA Networks. + */ + +#include "fuse_i.h" + +#include +#include +#include +#include + +/* + * Return true if need to wait for new opens in caching mode. + */ +static inline bool fuse_is_io_cache_wait(struct fuse_inode *fi) +{ + return READ_ONCE(fi->iocachectr) < 0 && !fuse_inode_backing(fi); +} + +/* + * Called on cached file open() and on first mmap() of direct_io file. + * Takes cached_io inode mode reference to be dropped on file release. + * + * Blocks new parallel dio writes and waits for the in-progress parallel dio + * writes to complete. + */ +int fuse_file_cached_io_open(struct inode *inode, struct fuse_file *ff) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + + /* There are no io modes if server does not implement open */ + if (!ff->args) + return 0; + + spin_lock(&fi->lock); + /* + * Setting the bit advises new direct-io writes to use an exclusive + * lock - without it the wait below might be forever. + */ + while (fuse_is_io_cache_wait(fi)) { + set_bit(FUSE_I_CACHE_IO_MODE, &fi->state); + spin_unlock(&fi->lock); + wait_event(fi->direct_io_waitq, !fuse_is_io_cache_wait(fi)); + spin_lock(&fi->lock); + } + + /* + * Check if inode entered passthrough io mode while waiting for parallel + * dio write completion. + */ + if (fuse_inode_backing(fi)) { + clear_bit(FUSE_I_CACHE_IO_MODE, &fi->state); + spin_unlock(&fi->lock); + return -ETXTBSY; + } + + WARN_ON(ff->iomode == IOM_UNCACHED); + if (ff->iomode == IOM_NONE) { + ff->iomode = IOM_CACHED; + if (fi->iocachectr == 0) + set_bit(FUSE_I_CACHE_IO_MODE, &fi->state); + fi->iocachectr++; + } + spin_unlock(&fi->lock); + return 0; +} + +static void fuse_file_cached_io_release(struct fuse_file *ff, + struct fuse_inode *fi) +{ + spin_lock(&fi->lock); + WARN_ON(fi->iocachectr <= 0); + WARN_ON(ff->iomode != IOM_CACHED); + ff->iomode = IOM_NONE; + fi->iocachectr--; + if (fi->iocachectr == 0) + clear_bit(FUSE_I_CACHE_IO_MODE, &fi->state); + spin_unlock(&fi->lock); +} + +/* Start strictly uncached io mode where cache access is not allowed */ +int fuse_inode_uncached_io_start(struct fuse_inode *fi, struct fuse_backing *fb) +{ + struct fuse_backing *oldfb; + int err = 0; + + spin_lock(&fi->lock); + /* deny conflicting backing files on same fuse inode */ + oldfb = fuse_inode_backing(fi); + if (fb && oldfb && oldfb != fb) { + err = -EBUSY; + goto unlock; + } + if (fi->iocachectr > 0) { + err = -ETXTBSY; + goto unlock; + } + fi->iocachectr--; + + /* fuse inode holds a single refcount of backing file */ + if (fb && !oldfb) { + oldfb = fuse_inode_backing_set(fi, fb); + WARN_ON_ONCE(oldfb != NULL); + } else { + fuse_backing_put(fb); + } +unlock: + spin_unlock(&fi->lock); + return err; +} + +/* Takes uncached_io inode mode reference to be dropped on file release */ +static int fuse_file_uncached_io_open(struct inode *inode, + struct fuse_file *ff, + struct fuse_backing *fb) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + int err; + + err = fuse_inode_uncached_io_start(fi, fb); + if (err) + return err; + + WARN_ON(ff->iomode != IOM_NONE); + ff->iomode = IOM_UNCACHED; + return 0; +} + +void fuse_inode_uncached_io_end(struct fuse_inode *fi) +{ + struct fuse_backing *oldfb = NULL; + + spin_lock(&fi->lock); + WARN_ON(fi->iocachectr >= 0); + fi->iocachectr++; + if (!fi->iocachectr) { + wake_up(&fi->direct_io_waitq); + oldfb = fuse_inode_backing_set(fi, NULL); + } + spin_unlock(&fi->lock); + if (oldfb) + fuse_backing_put(oldfb); +} + +/* Drop uncached_io reference from passthrough open */ +static void fuse_file_uncached_io_release(struct fuse_file *ff, + struct fuse_inode *fi) +{ + WARN_ON(ff->iomode != IOM_UNCACHED); + ff->iomode = IOM_NONE; + fuse_inode_uncached_io_end(fi); +} + +/* + * Open flags that are allowed in combination with FOPEN_PASSTHROUGH. + * A combination of FOPEN_PASSTHROUGH and FOPEN_DIRECT_IO means that read/write + * operations go directly to the server, but mmap is done on the backing file. + * FOPEN_PASSTHROUGH mode should not co-exist with any users of the fuse inode + * page cache, so FOPEN_KEEP_CACHE is a strange and undesired combination. + */ +#define FOPEN_PASSTHROUGH_MASK \ + (FOPEN_PASSTHROUGH | FOPEN_DIRECT_IO | FOPEN_PARALLEL_DIRECT_WRITES | \ + FOPEN_NOFLUSH) + +static int fuse_file_passthrough_open(struct inode *inode, struct file *file) +{ + struct fuse_file *ff = file->private_data; + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_backing *fb; + int err; + + /* Check allowed conditions for file open in passthrough mode */ + if (!IS_ENABLED(CONFIG_FUSE_PASSTHROUGH) || !fc->passthrough || + (ff->open_flags & ~FOPEN_PASSTHROUGH_MASK)) + return -EINVAL; + + fb = fuse_passthrough_open(file, ff->args->open_outarg.backing_id); + if (IS_ERR(fb)) + return PTR_ERR(fb); + + /* First passthrough file open denies caching inode io mode */ + err = fuse_file_uncached_io_open(inode, ff, fb); + if (!err) + return 0; + + fuse_passthrough_release(ff, fb); + fuse_backing_put(fb); + + return err; +} + +/* Request access to submit new io to inode via open file */ +int fuse_file_io_open(struct file *file, struct inode *inode) +{ + struct fuse_file *ff = file->private_data; + struct fuse_inode *fi = get_fuse_inode(inode); + int err; + + /* + * io modes are not relevant with DAX and with server that does not + * implement open. + */ + if (FUSE_IS_DAX(inode) || !ff->args) + return 0; + + /* + * Server is expected to use FOPEN_PASSTHROUGH for all opens of an inode + * which is already open for passthrough. + */ + err = -EINVAL; + if (fuse_inode_backing(fi) && !(ff->open_flags & FOPEN_PASSTHROUGH)) + goto fail; + + /* + * FOPEN_PARALLEL_DIRECT_WRITES requires FOPEN_DIRECT_IO. + */ + if (!(ff->open_flags & FOPEN_DIRECT_IO)) + ff->open_flags &= ~FOPEN_PARALLEL_DIRECT_WRITES; + + /* + * First passthrough file open denies caching inode io mode. + * First caching file open enters caching inode io mode. + * + * Note that if user opens a file open with O_DIRECT, but server did + * not specify FOPEN_DIRECT_IO, a later fcntl() could remove O_DIRECT, + * so we put the inode in caching mode to prevent parallel dio. + */ + if ((ff->open_flags & FOPEN_DIRECT_IO) && + !(ff->open_flags & FOPEN_PASSTHROUGH)) + return 0; + + if (ff->open_flags & FOPEN_PASSTHROUGH) + err = fuse_file_passthrough_open(inode, file); + else + err = fuse_file_cached_io_open(inode, ff); + if (err) + goto fail; + + return 0; + +fail: + pr_debug("failed to open file in requested io mode (open_flags=0x%x, err=%i).\n", + ff->open_flags, err); + /* + * The file open mode determines the inode io mode. + * Using incorrect open mode is a server mistake, which results in + * user visible failure of open() with EIO error. + */ + return -EIO; +} + +/* No more pending io and no new io possible to inode via open/mmapped file */ +void fuse_file_io_release(struct fuse_file *ff, struct inode *inode) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + + /* + * Last passthrough file close allows caching inode io mode. + * Last caching file close exits caching inode io mode. + */ + switch (ff->iomode) { + case IOM_NONE: + /* Nothing to do */ + break; + case IOM_UNCACHED: + fuse_file_uncached_io_release(ff, fi); + break; + case IOM_CACHED: + fuse_file_cached_io_release(ff, fi); + break; + } +} diff --git a/fs/fuse/notify.c b/fs/fuse/notify.c new file mode 100644 index 000000000..1ba763705 --- /dev/null +++ b/fs/fuse/notify.c @@ -0,0 +1,446 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include "dev.h" +#include "fuse_i.h" + +#include +#include + +static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size, + struct fuse_copy_state *cs) +{ + struct fuse_notify_poll_wakeup_out outarg; + int err; + + if (size != sizeof(outarg)) + return -EINVAL; + + err = fuse_copy_one(cs, &outarg, sizeof(outarg)); + if (err) + return err; + + fuse_copy_finish(cs); + return fuse_notify_poll_wakeup(fc, &outarg); +} + +static int fuse_notify_inval_inode(struct fuse_conn *fc, unsigned int size, + struct fuse_copy_state *cs) +{ + struct fuse_notify_inval_inode_out outarg; + int err; + + if (size != sizeof(outarg)) + return -EINVAL; + + err = fuse_copy_one(cs, &outarg, sizeof(outarg)); + if (err) + return err; + fuse_copy_finish(cs); + + down_read(&fc->killsb); + err = fuse_reverse_inval_inode(fc, outarg.ino, + outarg.off, outarg.len); + up_read(&fc->killsb); + return err; +} + +static int fuse_notify_inval_entry(struct fuse_conn *fc, unsigned int size, + struct fuse_copy_state *cs) +{ + struct fuse_notify_inval_entry_out outarg; + int err; + char *buf; + struct qstr name; + + if (size < sizeof(outarg)) + return -EINVAL; + + err = fuse_copy_one(cs, &outarg, sizeof(outarg)); + if (err) + return err; + + if (outarg.namelen > fc->name_max) + return -ENAMETOOLONG; + + err = -EINVAL; + if (size != sizeof(outarg) + outarg.namelen + 1) + return -EINVAL; + + buf = kzalloc(outarg.namelen + 1, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + name.name = buf; + name.len = outarg.namelen; + err = fuse_copy_one(cs, buf, outarg.namelen + 1); + if (err) + goto err; + fuse_copy_finish(cs); + buf[outarg.namelen] = 0; + + down_read(&fc->killsb); + err = fuse_reverse_inval_entry(fc, outarg.parent, 0, &name, outarg.flags); + up_read(&fc->killsb); +err: + kfree(buf); + return err; +} + +static int fuse_notify_delete(struct fuse_conn *fc, unsigned int size, + struct fuse_copy_state *cs) +{ + struct fuse_notify_delete_out outarg; + int err; + char *buf; + struct qstr name; + + if (size < sizeof(outarg)) + return -EINVAL; + + err = fuse_copy_one(cs, &outarg, sizeof(outarg)); + if (err) + return err; + + if (outarg.namelen > fc->name_max) + return -ENAMETOOLONG; + + if (size != sizeof(outarg) + outarg.namelen + 1) + return -EINVAL; + + buf = kzalloc(outarg.namelen + 1, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + name.name = buf; + name.len = outarg.namelen; + err = fuse_copy_one(cs, buf, outarg.namelen + 1); + if (err) + goto err; + fuse_copy_finish(cs); + buf[outarg.namelen] = 0; + + down_read(&fc->killsb); + err = fuse_reverse_inval_entry(fc, outarg.parent, outarg.child, &name, 0); + up_read(&fc->killsb); +err: + kfree(buf); + return err; +} + +static int fuse_notify_store(struct fuse_conn *fc, unsigned int size, + struct fuse_copy_state *cs) +{ + struct fuse_notify_store_out outarg; + struct inode *inode; + struct address_space *mapping; + u64 nodeid; + int err; + unsigned int num; + loff_t file_size; + loff_t pos; + loff_t end; + + if (size < sizeof(outarg)) + return -EINVAL; + + err = fuse_copy_one(cs, &outarg, sizeof(outarg)); + if (err) + return err; + + if (size - sizeof(outarg) != outarg.size) + return -EINVAL; + + if (outarg.offset >= MAX_LFS_FILESIZE) + return -EINVAL; + + nodeid = outarg.nodeid; + pos = outarg.offset; + num = min(outarg.size, MAX_LFS_FILESIZE - pos); + + down_read(&fc->killsb); + + err = -ENOENT; + inode = fuse_ilookup(fc, nodeid, NULL); + if (!inode) + goto out_up_killsb; + if (!S_ISREG(inode->i_mode)) { + err = -EINVAL; + goto out_iput; + } + + mapping = inode->i_mapping; + file_size = i_size_read(inode); + end = pos + num; + if (end > file_size) { + file_size = end; + fuse_write_update_attr(inode, file_size, num); + } + + while (num) { + struct folio *folio; + unsigned int folio_offset; + unsigned int nr_bytes; + pgoff_t index = pos >> PAGE_SHIFT; + + folio = filemap_grab_folio(mapping, index); + err = PTR_ERR(folio); + if (IS_ERR(folio)) + goto out_iput; + + folio_offset = offset_in_folio(folio, pos); + nr_bytes = min(num, folio_size(folio) - folio_offset); + + err = fuse_copy_folio(cs, &folio, folio_offset, nr_bytes, 0); + if (!folio_test_uptodate(folio) && !err && folio_offset == 0 && + (nr_bytes == folio_size(folio) || file_size == end)) { + folio_zero_segment(folio, nr_bytes, folio_size(folio)); + iomap_folio_mark_uptodate(folio); + } + folio_unlock(folio); + folio_put(folio); + + if (err) + goto out_iput; + + pos += nr_bytes; + num -= nr_bytes; + } + + err = 0; + +out_iput: + iput(inode); +out_up_killsb: + up_read(&fc->killsb); + return err; +} + +struct fuse_retrieve_args { + struct fuse_args_pages ap; + struct fuse_notify_retrieve_in inarg; +}; + +static void fuse_retrieve_end(struct fuse_args *args, int error) +{ + struct fuse_retrieve_args *ra = + container_of(args, typeof(*ra), ap.args); + + release_pages(ra->ap.folios, ra->ap.num_folios); + kfree(ra); +} + +static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode, + struct fuse_notify_retrieve_out *outarg) +{ + int err; + struct address_space *mapping = inode->i_mapping; + loff_t file_size; + unsigned int num; + unsigned int offset; + size_t total_len = 0; + unsigned int num_pages; + struct fuse_conn *fc = fm->fc; + struct fuse_retrieve_args *ra; + size_t args_size = sizeof(*ra); + struct fuse_args_pages *ap; + struct fuse_args *args; + loff_t pos = outarg->offset; + + offset = offset_in_page(pos); + file_size = i_size_read(inode); + + num = min(outarg->size, fc->max_write); + if (pos > file_size) + num = 0; + else if (num > file_size - pos) + num = file_size - pos; + + num_pages = DIV_ROUND_UP(num + offset, PAGE_SIZE); + num_pages = min(num_pages, fc->max_pages); + num = min(num, num_pages << PAGE_SHIFT); + + args_size += num_pages * (sizeof(ap->folios[0]) + sizeof(ap->descs[0])); + + ra = kzalloc(args_size, GFP_KERNEL); + if (!ra) + return -ENOMEM; + + ap = &ra->ap; + ap->folios = (void *) (ra + 1); + ap->descs = (void *) (ap->folios + num_pages); + + args = &ap->args; + args->nodeid = outarg->nodeid; + args->opcode = FUSE_NOTIFY_REPLY; + args->in_numargs = 3; + args->in_pages = true; + args->end = fuse_retrieve_end; + + while (num && ap->num_folios < num_pages) { + struct folio *folio; + unsigned int folio_offset; + unsigned int nr_bytes; + pgoff_t index = pos >> PAGE_SHIFT; + + folio = filemap_get_folio(mapping, index); + if (IS_ERR(folio)) + break; + if (!folio_test_uptodate(folio)) { + folio_put(folio); + break; + } + + folio_offset = offset_in_folio(folio, pos); + nr_bytes = min(folio_size(folio) - folio_offset, num); + + ap->folios[ap->num_folios] = folio; + ap->descs[ap->num_folios].offset = folio_offset; + ap->descs[ap->num_folios].length = nr_bytes; + ap->num_folios++; + + pos += nr_bytes; + num -= nr_bytes; + total_len += nr_bytes; + } + ra->inarg.offset = outarg->offset; + ra->inarg.size = total_len; + fuse_set_zero_arg0(args); + args->in_args[1].size = sizeof(ra->inarg); + args->in_args[1].value = &ra->inarg; + args->in_args[2].size = total_len; + + err = fuse_simple_notify_reply(fm, args, outarg->notify_unique); + if (err) + fuse_retrieve_end(args, err); + + return err; +} + +static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size, + struct fuse_copy_state *cs) +{ + struct fuse_notify_retrieve_out outarg; + struct fuse_mount *fm; + struct inode *inode; + u64 nodeid; + int err; + + if (size != sizeof(outarg)) + return -EINVAL; + + err = fuse_copy_one(cs, &outarg, sizeof(outarg)); + if (err) + return err; + + fuse_copy_finish(cs); + + if (outarg.offset >= MAX_LFS_FILESIZE) + return -EINVAL; + + down_read(&fc->killsb); + err = -ENOENT; + nodeid = outarg.nodeid; + + inode = fuse_ilookup(fc, nodeid, &fm); + if (inode) { + err = -EINVAL; + if (S_ISREG(inode->i_mode)) + err = fuse_retrieve(fm, inode, &outarg); + iput(inode); + } + up_read(&fc->killsb); + + return err; +} + +static int fuse_notify_resend(struct fuse_conn *fc) +{ + fuse_chan_resend(fc->chan); + return 0; +} + +/* + * Increments the fuse connection epoch. This will cause dentries and + * readdir caches from previous epochs to be invalidated. Additionally, + * if inval_wq is set, a work queue is scheduled to trigger the invalidation. + */ +static int fuse_notify_inc_epoch(struct fuse_conn *fc) +{ + atomic_inc(&fc->epoch); + if (inval_wq) + schedule_work(&fc->epoch_work); + + return 0; +} + +static int fuse_notify_prune(struct fuse_conn *fc, unsigned int size, + struct fuse_copy_state *cs) +{ + struct fuse_notify_prune_out outarg; + const unsigned int batch = 512; + u64 *nodeids __free(kfree) = kmalloc(sizeof(u64) * batch, GFP_KERNEL); + unsigned int num, i; + int err; + + if (!nodeids) + return -ENOMEM; + + if (size < sizeof(outarg)) + return -EINVAL; + + err = fuse_copy_one(cs, &outarg, sizeof(outarg)); + if (err) + return err; + + if (size - sizeof(outarg) != array_size(outarg.count, sizeof(u64))) + return -EINVAL; + + for (; outarg.count; outarg.count -= num) { + num = min(batch, outarg.count); + err = fuse_copy_one(cs, nodeids, num * sizeof(u64)); + if (err) + return err; + + scoped_guard(rwsem_read, &fc->killsb) { + for (i = 0; i < num; i++) + fuse_try_prune_one_inode(fc, nodeids[i]); + } + } + return 0; +} + +int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code, + unsigned int size, struct fuse_copy_state *cs) +{ + switch (code) { + case FUSE_NOTIFY_POLL: + return fuse_notify_poll(fc, size, cs); + + case FUSE_NOTIFY_INVAL_INODE: + return fuse_notify_inval_inode(fc, size, cs); + + case FUSE_NOTIFY_INVAL_ENTRY: + return fuse_notify_inval_entry(fc, size, cs); + + case FUSE_NOTIFY_STORE: + return fuse_notify_store(fc, size, cs); + + case FUSE_NOTIFY_RETRIEVE: + return fuse_notify_retrieve(fc, size, cs); + + case FUSE_NOTIFY_DELETE: + return fuse_notify_delete(fc, size, cs); + + case FUSE_NOTIFY_RESEND: + return fuse_notify_resend(fc); + + case FUSE_NOTIFY_INC_EPOCH: + return fuse_notify_inc_epoch(fc); + + case FUSE_NOTIFY_PRUNE: + return fuse_notify_prune(fc, size, cs); + + default: + return -EINVAL; + } +} diff --git a/fs/fuse/passthrough.c b/fs/fuse/passthrough.c new file mode 100644 index 000000000..f2d08ac24 --- /dev/null +++ b/fs/fuse/passthrough.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * FUSE passthrough to backing file. + * + * Copyright (c) 2023 CTERA Networks. + */ + +#include "fuse_i.h" + +#include +#include +#include + +static void fuse_file_accessed(struct file *file) +{ + struct inode *inode = file_inode(file); + + fuse_invalidate_atime(inode); +} + +static void fuse_passthrough_end_write(struct kiocb *iocb, ssize_t ret) +{ + struct inode *inode = file_inode(iocb->ki_filp); + + fuse_write_update_attr(inode, iocb->ki_pos, ret); +} + +ssize_t fuse_passthrough_read_iter(struct kiocb *iocb, struct iov_iter *iter) +{ + struct file *file = iocb->ki_filp; + struct fuse_file *ff = file->private_data; + struct file *backing_file = fuse_file_passthrough(ff); + size_t count = iov_iter_count(iter); + ssize_t ret; + struct backing_file_ctx ctx = { + .cred = ff->cred, + .accessed = fuse_file_accessed, + }; + + + pr_debug("%s: backing_file=0x%p, pos=%lld, len=%zu\n", __func__, + backing_file, iocb->ki_pos, count); + + if (!count) + return 0; + + ret = backing_file_read_iter(backing_file, iter, iocb, iocb->ki_flags, + &ctx); + + return ret; +} + +ssize_t fuse_passthrough_write_iter(struct kiocb *iocb, + struct iov_iter *iter) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct fuse_file *ff = file->private_data; + struct file *backing_file = fuse_file_passthrough(ff); + size_t count = iov_iter_count(iter); + ssize_t ret; + struct backing_file_ctx ctx = { + .cred = ff->cred, + .end_write = fuse_passthrough_end_write, + }; + + pr_debug("%s: backing_file=0x%p, pos=%lld, len=%zu\n", __func__, + backing_file, iocb->ki_pos, count); + + if (!count) + return 0; + + inode_lock(inode); + ret = backing_file_write_iter(backing_file, iter, iocb, iocb->ki_flags, + &ctx); + inode_unlock(inode); + + return ret; +} + +ssize_t fuse_passthrough_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, + size_t len, unsigned int flags) +{ + struct fuse_file *ff = in->private_data; + struct file *backing_file = fuse_file_passthrough(ff); + struct backing_file_ctx ctx = { + .cred = ff->cred, + .accessed = fuse_file_accessed, + }; + struct kiocb iocb; + ssize_t ret; + + pr_debug("%s: backing_file=0x%p, pos=%lld, len=%zu, flags=0x%x\n", __func__, + backing_file, *ppos, len, flags); + + init_sync_kiocb(&iocb, in); + iocb.ki_pos = *ppos; + ret = backing_file_splice_read(backing_file, &iocb, pipe, len, flags, &ctx); + *ppos = iocb.ki_pos; + + return ret; +} + +ssize_t fuse_passthrough_splice_write(struct pipe_inode_info *pipe, + struct file *out, loff_t *ppos, + size_t len, unsigned int flags) +{ + struct fuse_file *ff = out->private_data; + struct file *backing_file = fuse_file_passthrough(ff); + struct inode *inode = file_inode(out); + ssize_t ret; + struct backing_file_ctx ctx = { + .cred = ff->cred, + .end_write = fuse_passthrough_end_write, + }; + struct kiocb iocb; + + pr_debug("%s: backing_file=0x%p, pos=%lld, len=%zu, flags=0x%x\n", __func__, + backing_file, *ppos, len, flags); + + inode_lock(inode); + init_sync_kiocb(&iocb, out); + iocb.ki_pos = *ppos; + ret = backing_file_splice_write(pipe, backing_file, &iocb, len, flags, &ctx); + *ppos = iocb.ki_pos; + inode_unlock(inode); + + return ret; +} + +ssize_t fuse_passthrough_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct fuse_file *ff = file->private_data; + struct file *backing_file = fuse_file_passthrough(ff); + struct backing_file_ctx ctx = { + .cred = ff->cred, + .accessed = fuse_file_accessed, + }; + + pr_debug("%s: backing_file=0x%p, start=%lu, end=%lu\n", __func__, + backing_file, vma->vm_start, vma->vm_end); + + return backing_file_mmap(backing_file, vma, &ctx); +} + +/* + * Setup passthrough to a backing file. + * + * Returns an fb object with elevated refcount to be stored in fuse inode. + */ +struct fuse_backing *fuse_passthrough_open(struct file *file, int backing_id) +{ + struct fuse_file *ff = file->private_data; + struct fuse_conn *fc = ff->fm->fc; + struct fuse_backing *fb = NULL; + struct file *backing_file; + int err; + + err = -EINVAL; + if (backing_id <= 0) + goto out; + + err = -ENOENT; + fb = fuse_backing_lookup(fc, backing_id); + if (!fb) + goto out; + + /* Allocate backing file per fuse file to store fuse path */ + backing_file = backing_file_open(file, file->f_flags, + &fb->file->f_path, fb->cred); + err = PTR_ERR(backing_file); + if (IS_ERR(backing_file)) { + fuse_backing_put(fb); + goto out; + } + + err = 0; + ff->passthrough = backing_file; + ff->cred = get_cred(fb->cred); +out: + pr_debug("%s: backing_id=%d, fb=0x%p, backing_file=0x%p, err=%i\n", __func__, + backing_id, fb, ff->passthrough, err); + + return err ? ERR_PTR(err) : fb; +} + +void fuse_passthrough_release(struct fuse_file *ff, struct fuse_backing *fb) +{ + pr_debug("%s: fb=0x%p, backing_file=0x%p\n", __func__, + fb, ff->passthrough); + + fput(ff->passthrough); + ff->passthrough = NULL; + put_cred(ff->cred); + ff->cred = NULL; +} diff --git a/fs/fuse/poll.c b/fs/fuse/poll.c new file mode 100644 index 000000000..bce3ee2e8 --- /dev/null +++ b/fs/fuse/poll.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include "dev.h" +#include "fuse_i.h" + +void fuse_end_polls(struct fuse_conn *fc) +{ + struct rb_node *p; + + spin_lock(&fc->lock); + p = rb_first(&fc->polled_files); + + while (p) { + struct fuse_file *ff; + ff = rb_entry(p, struct fuse_file, polled_node); + wake_up_interruptible_all(&ff->poll_wait); + + p = rb_next(p); + } + spin_unlock(&fc->lock); +} + +/* + * All files which have been polled are linked to RB tree + * fuse_conn->polled_files which is indexed by kh. Walk the tree and + * find the matching one. + */ +static struct rb_node **fuse_find_polled_node(struct fuse_conn *fc, u64 kh, + struct rb_node **parent_out) +{ + struct rb_node **link = &fc->polled_files.rb_node; + struct rb_node *last = NULL; + + while (*link) { + struct fuse_file *ff; + + last = *link; + ff = rb_entry(last, struct fuse_file, polled_node); + + if (kh < ff->kh) + link = &last->rb_left; + else if (kh > ff->kh) + link = &last->rb_right; + else + return link; + } + + if (parent_out) + *parent_out = last; + return link; +} + +/* + * The file is about to be polled. Make sure it's on the polled_files + * RB tree. Note that files once added to the polled_files tree are + * not removed before the file is released. This is because a file + * polled once is likely to be polled again. + */ +static void fuse_register_polled_file(struct fuse_conn *fc, + struct fuse_file *ff) +{ + spin_lock(&fc->lock); + if (RB_EMPTY_NODE(&ff->polled_node)) { + struct rb_node **link, *parent; + + link = fuse_find_polled_node(fc, ff->kh, &parent); + BUG_ON(*link); + rb_link_node(&ff->polled_node, parent, link); + rb_insert_color(&ff->polled_node, &fc->polled_files); + } + spin_unlock(&fc->lock); +} + +__poll_t fuse_file_poll(struct file *file, poll_table *wait) +{ + struct fuse_file *ff = file->private_data; + struct fuse_mount *fm = ff->fm; + struct fuse_poll_in inarg = { .fh = ff->fh, .kh = ff->kh }; + struct fuse_poll_out outarg; + FUSE_ARGS(args); + int err; + + if (fm->fc->no_poll) + return DEFAULT_POLLMASK; + + poll_wait(file, &ff->poll_wait, wait); + inarg.events = mangle_poll(poll_requested_events(wait)); + + /* + * Ask for notification iff there's someone waiting for it. + * The client may ignore the flag and always notify. + */ + if (waitqueue_active(&ff->poll_wait)) { + inarg.flags |= FUSE_POLL_SCHEDULE_NOTIFY; + fuse_register_polled_file(fm->fc, ff); + } + + args.opcode = FUSE_POLL; + args.nodeid = ff->nodeid; + args.in_numargs = 1; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + args.out_numargs = 1; + args.out_args[0].size = sizeof(outarg); + args.out_args[0].value = &outarg; + err = fuse_simple_request(fm, &args); + + if (!err) + return demangle_poll(outarg.revents); + if (err == -ENOSYS) { + fm->fc->no_poll = 1; + return DEFAULT_POLLMASK; + } + return EPOLLERR; +} +EXPORT_SYMBOL_GPL(fuse_file_poll); + +/* + * This is called from fuse_handle_notify() on FUSE_NOTIFY_POLL and + * wakes up the poll waiters. + */ +int fuse_notify_poll_wakeup(struct fuse_conn *fc, + struct fuse_notify_poll_wakeup_out *outarg) +{ + u64 kh = outarg->kh; + struct rb_node **link; + + spin_lock(&fc->lock); + + link = fuse_find_polled_node(fc, kh, NULL); + if (*link) { + struct fuse_file *ff; + + ff = rb_entry(*link, struct fuse_file, polled_node); + wake_up_interruptible_sync(&ff->poll_wait); + } + + spin_unlock(&fc->lock); + return 0; +} + diff --git a/fs/fuse/readdir.c b/fs/fuse/readdir.c new file mode 100644 index 000000000..0e1321491 --- /dev/null +++ b/fs/fuse/readdir.c @@ -0,0 +1,656 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + FUSE: Filesystem in Userspace + Copyright (C) 2001-2018 Miklos Szeredi +*/ + + +#include "fuse_i.h" +#include +#include +#include +#include +#include + +static bool fuse_use_readdirplus(struct inode *dir, struct dir_context *ctx) +{ + struct fuse_conn *fc = get_fuse_conn(dir); + struct fuse_inode *fi = get_fuse_inode(dir); + + if (!fc->do_readdirplus) + return false; + if (!fc->readdirplus_auto) + return true; + if (test_and_clear_bit(FUSE_I_ADVISE_RDPLUS, &fi->state)) + return true; + if (ctx->pos == 0) + return true; + return false; +} + +static void fuse_add_dirent_to_cache(struct file *file, + struct fuse_dirent *dirent, loff_t pos) +{ + struct fuse_inode *fi = get_fuse_inode(file_inode(file)); + size_t reclen = FUSE_DIRENT_SIZE(dirent); + pgoff_t index; + struct page *page; + loff_t size; + u64 version; + unsigned int offset; + void *addr; + + /* Dirent doesn't fit in readdir cache page? Skip caching. */ + if (reclen > PAGE_SIZE) + return; + + spin_lock(&fi->rdc.lock); + /* + * Is cache already completed? Or this entry does not go at the end of + * cache? + */ + if (fi->rdc.cached || pos != fi->rdc.pos) { + spin_unlock(&fi->rdc.lock); + return; + } + version = fi->rdc.version; + size = fi->rdc.size; + offset = offset_in_page(size); + index = size >> PAGE_SHIFT; + /* Dirent doesn't fit in current page? Jump to next page. */ + if (offset + reclen > PAGE_SIZE) { + index++; + offset = 0; + } + spin_unlock(&fi->rdc.lock); + + if (offset) { + page = find_lock_page(file->f_mapping, index); + } else { + page = find_or_create_page(file->f_mapping, index, + mapping_gfp_mask(file->f_mapping)); + } + if (!page) + return; + + spin_lock(&fi->rdc.lock); + /* Raced with another readdir */ + if (fi->rdc.version != version || fi->rdc.size != size || + WARN_ON(fi->rdc.pos != pos)) + goto unlock; + + addr = kmap_local_page(page); + if (!offset) { + clear_page(addr); + SetPageUptodate(page); + } + memcpy(addr + offset, dirent, reclen); + kunmap_local(addr); + fi->rdc.size = (index << PAGE_SHIFT) + offset + reclen; + fi->rdc.pos = dirent->off; +unlock: + spin_unlock(&fi->rdc.lock); + unlock_page(page); + put_page(page); +} + +static void fuse_readdir_cache_end(struct file *file, loff_t pos) +{ + struct fuse_inode *fi = get_fuse_inode(file_inode(file)); + loff_t end; + + spin_lock(&fi->rdc.lock); + /* does cache end position match current position? */ + if (fi->rdc.pos != pos) { + spin_unlock(&fi->rdc.lock); + return; + } + + fi->rdc.cached = true; + end = ALIGN(fi->rdc.size, PAGE_SIZE); + spin_unlock(&fi->rdc.lock); + + /* truncate unused tail of cache */ + truncate_inode_pages(file->f_mapping, end); +} + +static bool fuse_emit(struct file *file, struct dir_context *ctx, + struct fuse_dirent *dirent) +{ + struct fuse_file *ff = file->private_data; + + if (ff->open_flags & FOPEN_CACHE_DIR) + fuse_add_dirent_to_cache(file, dirent, ctx->pos); + + return dir_emit(ctx, dirent->name, dirent->namelen, dirent->ino, + dirent->type | FILLDIR_FLAG_NOINTR); +} + +static int parse_dirfile(char *buf, size_t nbytes, struct file *file, + struct dir_context *ctx) +{ + while (nbytes >= FUSE_NAME_OFFSET) { + struct fuse_dirent *dirent = (struct fuse_dirent *) buf; + size_t reclen = FUSE_DIRENT_SIZE(dirent); + if (!dirent->namelen || dirent->namelen > FUSE_NAME_MAX) + return -EIO; + if (reclen > nbytes) + break; + if (memchr(dirent->name, '/', dirent->namelen) != NULL) + return -EIO; + + if (!fuse_emit(file, ctx, dirent)) + break; + + buf += reclen; + nbytes -= reclen; + ctx->pos = dirent->off; + } + + return 0; +} + +static int fuse_direntplus_link(struct file *file, + struct fuse_direntplus *direntplus, + u64 attr_version, u64 evict_ctr) +{ + struct fuse_entry_out *o = &direntplus->entry_out; + struct fuse_dirent *dirent = &direntplus->dirent; + struct dentry *parent = file->f_path.dentry; + struct qstr name = QSTR_INIT(dirent->name, dirent->namelen); + struct dentry *dentry; + struct dentry *alias; + struct inode *dir = d_inode(parent); + struct fuse_conn *fc; + struct inode *inode; + int epoch; + + if (!o->nodeid) { + /* + * Unlike in the case of fuse_lookup, zero nodeid does not mean + * ENOENT. Instead, it only means the userspace filesystem did + * not want to return attributes/handle for this entry. + * + * So do nothing. + */ + return 0; + } + + if (name.name[0] == '.') { + /* + * We could potentially refresh the attributes of the directory + * and its parent? + */ + if (name.len == 1) + return 0; + if (name.name[1] == '.' && name.len == 2) + return 0; + } + + if (invalid_nodeid(o->nodeid)) + return -EIO; + if (fuse_invalid_attr(&o->attr)) + return -EIO; + + fc = get_fuse_conn(dir); + epoch = atomic_read(&fc->epoch); + + name.hash = full_name_hash(parent, name.name, name.len); + dentry = d_lookup(parent, &name); + if (!dentry) { +retry: + dentry = d_alloc_parallel(parent, &name); + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + } + if (!d_in_lookup(dentry)) { + struct fuse_inode *fi; + inode = d_inode(dentry); + if (inode && get_node_id(inode) != o->nodeid) + inode = NULL; + if (!inode || + fuse_stale_inode(inode, o->generation, &o->attr)) { + if (inode) + fuse_make_bad(inode); + d_invalidate(dentry); + dput(dentry); + goto retry; + } + if (fuse_is_bad(inode)) { + dput(dentry); + return -EIO; + } + + fi = get_fuse_inode(inode); + spin_lock(&fi->lock); + fi->nlookup++; + spin_unlock(&fi->lock); + + forget_all_cached_acls(inode); + fuse_change_attributes(inode, &o->attr, NULL, + ATTR_TIMEOUT(o), + attr_version); + /* + * The other branch comes via fuse_iget() + * which bumps nlookup inside + */ + } else { + inode = fuse_iget(dir->i_sb, o->nodeid, o->generation, + &o->attr, ATTR_TIMEOUT(o), + attr_version, evict_ctr); + if (!inode) + inode = ERR_PTR(-ENOMEM); + + alias = d_splice_alias(inode, dentry); + d_lookup_done(dentry); + if (alias) { + dput(dentry); + dentry = alias; + } + if (IS_ERR(dentry)) { + if (!IS_ERR(inode)) { + struct fuse_inode *fi = get_fuse_inode(inode); + + spin_lock(&fi->lock); + fi->nlookup--; + spin_unlock(&fi->lock); + } + return PTR_ERR(dentry); + } + } + if (fc->readdirplus_auto) + set_bit(FUSE_I_INIT_RDPLUS, &get_fuse_inode(inode)->state); + dentry->d_time = epoch; + fuse_change_entry_timeout(dentry, o); + + dput(dentry); + return 0; +} + +static void fuse_force_forget(struct file *file, u64 nodeid) +{ + struct inode *inode = file_inode(file); + struct fuse_mount *fm = get_fuse_mount(inode); + struct fuse_forget_in inarg; + FUSE_ARGS(args); + + memset(&inarg, 0, sizeof(inarg)); + inarg.nlookup = 1; + args.opcode = FUSE_FORGET; + args.nodeid = nodeid; + args.in_numargs = 1; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + args.force = true; + args.noreply = true; + + fuse_simple_request(fm, &args); + /* ignore errors */ +} + +static int parse_dirplusfile(char *buf, size_t nbytes, struct file *file, + struct dir_context *ctx, u64 attr_version, + u64 evict_ctr) +{ + struct fuse_direntplus *direntplus; + struct fuse_dirent *dirent; + size_t reclen; + int over = 0; + int ret; + + while (nbytes >= FUSE_NAME_OFFSET_DIRENTPLUS) { + direntplus = (struct fuse_direntplus *) buf; + dirent = &direntplus->dirent; + reclen = FUSE_DIRENTPLUS_SIZE(direntplus); + + if (!dirent->namelen || dirent->namelen > FUSE_NAME_MAX) + return -EIO; + if (reclen > nbytes) + break; + if (memchr(dirent->name, '/', dirent->namelen) != NULL) + return -EIO; + + if (!over) { + /* We fill entries into dstbuf only as much as + it can hold. But we still continue iterating + over remaining entries to link them. If not, + we need to send a FORGET for each of those + which we did not link. + */ + over = !fuse_emit(file, ctx, dirent); + if (!over) + ctx->pos = dirent->off; + } + + buf += reclen; + nbytes -= reclen; + + ret = fuse_direntplus_link(file, direntplus, attr_version, evict_ctr); + if (ret) + fuse_force_forget(file, direntplus->entry_out.nodeid); + } + + return 0; +} + +static struct page **fuse_readdir_alloc_buf(struct fuse_args_pages *ap, size_t *bufsize) +{ + unsigned int i, nr_alloc, nr_pages = DIV_ROUND_UP(*bufsize, PAGE_SIZE); + struct page **pages = kcalloc(nr_pages, sizeof(*pages), GFP_KERNEL); + + if (!pages) + return NULL; + + nr_alloc = alloc_pages_bulk(GFP_KERNEL, nr_pages, pages); + if (!nr_alloc) + goto free_array; + + if (nr_alloc < nr_pages) { + nr_pages = nr_alloc; + *bufsize = (size_t) nr_pages << PAGE_SHIFT; + } + + ap->folios = fuse_folios_alloc(nr_pages, GFP_KERNEL, &ap->descs); + if (!ap->folios) + goto release_pages; + + for (i = 0; i < nr_pages; i++) { + ap->folios[i] = page_folio(pages[i]); + ap->descs[i].length = min_t(size_t, *bufsize - (size_t)i * PAGE_SIZE, PAGE_SIZE); + } + ap->num_folios = nr_pages; + ap->args.out_pages = true; + + return pages; + +release_pages: + release_pages(pages, nr_pages); +free_array: + kfree(pages); + return NULL; +} + +static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx) +{ + int plus; + ssize_t res; + struct inode *inode = file_inode(file); + struct fuse_mount *fm = get_fuse_mount(inode); + struct fuse_conn *fc = fm->fc; + struct fuse_io_args ia = {}; + struct fuse_args_pages *ap = &ia.ap; + void *buf; + size_t bufsize = clamp((unsigned int) ctx->count, PAGE_SIZE, fc->max_pages << PAGE_SHIFT); + u64 attr_version = 0, evict_ctr = 0; + bool locked; + struct page **pages = fuse_readdir_alloc_buf(ap, &bufsize); + + if (!pages) + return -ENOMEM; + + plus = fuse_use_readdirplus(inode, ctx); + if (plus) { + attr_version = fuse_get_attr_version(fm->fc); + evict_ctr = fuse_get_evict_ctr(fm->fc); + fuse_read_args_fill(&ia, file, ctx->pos, bufsize, FUSE_READDIRPLUS); + } else { + fuse_read_args_fill(&ia, file, ctx->pos, bufsize, FUSE_READDIR); + } + locked = fuse_lock_inode(inode); + res = fuse_simple_request(fm, &ap->args); + fuse_unlock_inode(inode, locked); + if (res < 0) + goto out; + + if (!res) { + struct fuse_file *ff = file->private_data; + + if (ff->open_flags & FOPEN_CACHE_DIR) + fuse_readdir_cache_end(file, ctx->pos); + goto out; + } + + buf = vm_map_ram(pages, ap->num_folios, -1); + if (!buf) { + res = -ENOMEM; + } else { + if (plus) + res = parse_dirplusfile(buf, res, file, ctx, attr_version, evict_ctr); + else + res = parse_dirfile(buf, res, file, ctx); + + vm_unmap_ram(buf, ap->num_folios); + } +out: + kfree(ap->folios); + release_pages(pages, ap->num_folios); + kfree(pages); + + fuse_invalidate_atime(inode); + + return res; +} + +enum fuse_parse_result { + FOUND_ERR = -1, + FOUND_NONE = 0, + FOUND_SOME, + FOUND_ALL, +}; + +static enum fuse_parse_result fuse_parse_cache(struct fuse_file *ff, + void *addr, unsigned int size, + struct dir_context *ctx) +{ + unsigned int offset = offset_in_page(ff->readdir.cache_off); + enum fuse_parse_result res = FOUND_NONE; + + WARN_ON(offset >= size); + + for (;;) { + struct fuse_dirent *dirent = addr + offset; + unsigned int nbytes = size - offset; + size_t reclen; + + if (nbytes < FUSE_NAME_OFFSET || !dirent->namelen) + break; + + reclen = FUSE_DIRENT_SIZE(dirent); /* derefs ->namelen */ + + if (WARN_ON(dirent->namelen > FUSE_NAME_MAX)) + return FOUND_ERR; + if (WARN_ON(reclen > nbytes)) + return FOUND_ERR; + if (WARN_ON(memchr(dirent->name, '/', dirent->namelen) != NULL)) + return FOUND_ERR; + + if (ff->readdir.pos == ctx->pos) { + res = FOUND_SOME; + if (!dir_emit(ctx, dirent->name, dirent->namelen, + dirent->ino, dirent->type | FILLDIR_FLAG_NOINTR)) + return FOUND_ALL; + ctx->pos = dirent->off; + } + ff->readdir.pos = dirent->off; + ff->readdir.cache_off += reclen; + + offset += reclen; + } + + return res; +} + +static void fuse_rdc_reset(struct inode *inode) +{ + struct fuse_inode *fi = get_fuse_inode(inode); + + fi->rdc.cached = false; + fi->rdc.version++; + fi->rdc.size = 0; + fi->rdc.pos = 0; + fi->rdc.epoch = 0; +} + +#define UNCACHED 1 + +static int fuse_readdir_cached(struct file *file, struct dir_context *ctx) +{ + struct fuse_file *ff = file->private_data; + struct inode *inode = file_inode(file); + struct fuse_conn *fc = get_fuse_conn(inode); + struct fuse_inode *fi = get_fuse_inode(inode); + enum fuse_parse_result res; + pgoff_t index; + unsigned int size; + struct page *page; + void *addr; + + /* Seeked? If so, reset the cache stream */ + if (ff->readdir.pos != ctx->pos) { + ff->readdir.pos = 0; + ff->readdir.cache_off = 0; + } + + /* + * We're just about to start reading into the cache or reading the + * cache; both cases require an up-to-date mtime value. + */ + if (!ctx->pos && fc->auto_inval_data) { + int err = fuse_update_attributes(inode, file, STATX_MTIME); + + if (err) + return err; + } + +retry: + spin_lock(&fi->rdc.lock); +retry_locked: + if (!fi->rdc.cached) { + /* Starting cache? Set cache mtime. */ + if (!ctx->pos && !fi->rdc.size) { + fi->rdc.mtime = inode_get_mtime(inode); + fi->rdc.iversion = inode_query_iversion(inode); + fi->rdc.epoch = atomic_read(&fc->epoch); + } + spin_unlock(&fi->rdc.lock); + return UNCACHED; + } + /* + * When at the beginning of the directory (i.e. just after opendir(3) or + * rewinddir(3)), then need to check whether directory contents have + * changed, and reset the cache if so. + */ + if (!ctx->pos) { + struct timespec64 mtime = inode_get_mtime(inode); + + if (inode_peek_iversion(inode) != fi->rdc.iversion || + !timespec64_equal(&fi->rdc.mtime, &mtime) || + fi->rdc.epoch != atomic_read(&fc->epoch)) { + fuse_rdc_reset(inode); + goto retry_locked; + } + } + + /* + * If cache version changed since the last getdents() call, then reset + * the cache stream. + */ + if (ff->readdir.version != fi->rdc.version) { + ff->readdir.pos = 0; + ff->readdir.cache_off = 0; + } + /* + * If at the beginning of the cache, than reset version to + * current. + */ + if (ff->readdir.pos == 0) + ff->readdir.version = fi->rdc.version; + + WARN_ON(fi->rdc.size < ff->readdir.cache_off); + + index = ff->readdir.cache_off >> PAGE_SHIFT; + + if (index == (fi->rdc.size >> PAGE_SHIFT)) + size = offset_in_page(fi->rdc.size); + else + size = PAGE_SIZE; + spin_unlock(&fi->rdc.lock); + + /* EOF? */ + if (offset_in_page(ff->readdir.cache_off) == size) + return 0; + + page = find_get_page_flags(file->f_mapping, index, + FGP_ACCESSED | FGP_LOCK); + /* Page gone missing, then re-added to cache, but not initialized? */ + if (page && !PageUptodate(page)) { + unlock_page(page); + put_page(page); + page = NULL; + } + spin_lock(&fi->rdc.lock); + if (!page) { + /* + * Uh-oh: page gone missing, cache is useless + */ + if (fi->rdc.version == ff->readdir.version) + fuse_rdc_reset(inode); + goto retry_locked; + } + + /* Make sure it's still the same version after getting the page. */ + if (ff->readdir.version != fi->rdc.version) { + spin_unlock(&fi->rdc.lock); + unlock_page(page); + put_page(page); + goto retry; + } + spin_unlock(&fi->rdc.lock); + + /* + * Contents of the page are now protected against changing by holding + * the page lock. + */ + addr = kmap_local_page(page); + res = fuse_parse_cache(ff, addr, size, ctx); + kunmap_local(addr); + unlock_page(page); + put_page(page); + + if (res == FOUND_ERR) + return -EIO; + + if (res == FOUND_ALL) + return 0; + + if (size == PAGE_SIZE) { + /* We hit end of page: skip to next page. */ + ff->readdir.cache_off = ALIGN(ff->readdir.cache_off, PAGE_SIZE); + goto retry; + } + + /* + * End of cache reached. If found position, then we are done, otherwise + * need to fall back to uncached, since the position we were looking for + * wasn't in the cache. + */ + return res == FOUND_SOME ? 0 : UNCACHED; +} + +int fuse_readdir(struct file *file, struct dir_context *ctx) +{ + struct fuse_file *ff = file->private_data; + struct inode *inode = file_inode(file); + int err; + + if (fuse_is_bad(inode)) + return -EIO; + + err = UNCACHED; + if (ff->open_flags & FOPEN_CACHE_DIR) + err = fuse_readdir_cached(file, ctx); + if (err == UNCACHED) + err = fuse_readdir_uncached(file, ctx); + + return err; +} diff --git a/fs/fuse/req.c b/fs/fuse/req.c new file mode 100644 index 000000000..a01ee743d --- /dev/null +++ b/fs/fuse/req.c @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include "dev.h" +#include "fuse_i.h" + +static int fuse_fill_creds(struct fuse_mount *fm, struct fuse_args *args, struct mnt_idmap *idmap) +{ + struct fuse_conn *fc = fm->fc; + bool no_idmap = !fm->sb || (fm->sb->s_iflags & SB_I_NOIDMAP); + kuid_t fsuid = mapped_fsuid(idmap, fc->user_ns); + kgid_t fsgid = mapped_fsgid(idmap, fc->user_ns); + + args->pid = pid_nr_ns(task_pid(current), fc->pid_ns); + + if (args->force) { + if (args->nocreds) + return 0; + + if (no_idmap) { + args->uid = from_kuid_munged(fc->user_ns, current_fsuid()); + args->gid = from_kgid_munged(fc->user_ns, current_fsgid()); + } else { + args->uid = FUSE_INVALID_UIDGID; + args->gid = FUSE_INVALID_UIDGID; + } + return 0; + } + + WARN_ON(args->nocreds); + /* + * Keep the old behavior when idmappings support was not + * declared by a FUSE server. + * + * For those FUSE servers who support idmapped mounts, we send UID/GID + * only along with "inode creation" fuse requests, otherwise idmap == + * &invalid_mnt_idmap and req->in.h.{u,g}id will be equal to + * FUSE_INVALID_UIDGID. + */ + if (no_idmap) { + fsuid = current_fsuid(); + fsgid = current_fsgid(); + } + args->uid = from_kuid(fc->user_ns, fsuid); + args->gid = from_kgid(fc->user_ns, fsgid); + + if (no_idmap && unlikely(args->uid == ((uid_t)-1) || args->gid == ((gid_t)-1))) + return -EOVERFLOW; + + return 0; +} + +static int fuse_req_prep(struct fuse_mount *fm, struct fuse_args *args, struct mnt_idmap *idmap) +{ + if (!args->force && fm->fc->conn_error) + return -ECONNREFUSED; + + return fuse_fill_creds(fm, args, idmap); +} + +ssize_t __fuse_simple_request(struct mnt_idmap *idmap, struct fuse_mount *fm, + struct fuse_args *args) +{ + struct fuse_conn *fc = fm->fc; + int err = fuse_req_prep(fm, args, idmap); + + if (err) + return err; + + return fuse_chan_send(fc->chan, args); +} + +int fuse_simple_background(struct fuse_mount *fm, struct fuse_args *args, gfp_t gfp_flags) +{ + struct fuse_conn *fc = fm->fc; + int err; + + WARN_ON(args->force && !args->nocreds); + + err = fuse_req_prep(fm, args, &invalid_mnt_idmap); + if (err) + return err; + + return fuse_chan_send_bg(fc->chan, args, gfp_flags); +} +EXPORT_SYMBOL_GPL(fuse_simple_background); + +int fuse_simple_notify_reply(struct fuse_mount *fm, struct fuse_args *args, u64 unique) +{ + struct fuse_conn *fc = fm->fc; + int err; + + WARN_ON(args->force && !args->nocreds); + + err = fuse_req_prep(fm, args, &invalid_mnt_idmap); + if (err) + return err; + + return fuse_chan_send_notify_reply(fc->chan, args, unique); +} diff --git a/fs/fuse/req_timeout.c b/fs/fuse/req_timeout.c new file mode 100644 index 000000000..6cc6fc491 --- /dev/null +++ b/fs/fuse/req_timeout.c @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include "dev.h" +#include "sysctl.h" +#include "fuse_dev_i.h" +#include "dev_uring_i.h" + +/* Frequency (in seconds) of request timeout checks, if opted into */ +#define FUSE_TIMEOUT_TIMER_FREQ 15 + +/* Frequency (in jiffies) of request timeout checks, if opted into */ +static const unsigned long fuse_timeout_timer_freq = + secs_to_jiffies(FUSE_TIMEOUT_TIMER_FREQ); + +/* + * Default timeout (in seconds) for the server to reply to a request + * before the connection is aborted, if no timeout was specified on mount. + * + * Exported via sysctl + */ +unsigned int fuse_default_req_timeout; + +/* + * Max timeout (in seconds) for the server to reply to a request before + * the connection is aborted. + * + * Exported via sysctl + */ +unsigned int fuse_max_req_timeout; + +bool fuse_request_expired(struct fuse_chan *fch, struct list_head *list) +{ + struct fuse_req *req; + + req = list_first_entry_or_null(list, struct fuse_req, list); + if (!req) + return false; + return time_is_before_jiffies(req->create_time + fch->timeout.req_timeout); +} + +static bool fuse_fpq_processing_expired(struct fuse_chan *fch, struct list_head *processing) +{ + int i; + + for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) + if (fuse_request_expired(fch, &processing[i])) + return true; + + return false; +} + +/* + * Check if any requests aren't being completed by the time the request timeout + * elapses. To do so, we: + * - check the fiq pending list + * - check the bg queue + * - check the fpq io and processing lists + * + * To make this fast, we only check against the head request on each list since + * these are generally queued in order of creation time (eg newer requests get + * queued to the tail). We might miss a few edge cases (eg requests transitioning + * between lists, re-sent requests at the head of the pending list having a + * later creation time than other requests on that list, etc.) but that is fine + * since if the request never gets fulfilled, it will eventually be caught. + */ +static void fuse_check_timeout(struct work_struct *work) +{ + struct delayed_work *dwork = to_delayed_work(work); + struct fuse_chan *fch = container_of(dwork, struct fuse_chan, timeout.work); + struct fuse_iqueue *fiq = &fch->iq; + struct fuse_dev *fud; + struct fuse_pqueue *fpq; + bool expired = false; + + if (!atomic_read(&fch->num_waiting)) + goto out; + + spin_lock(&fiq->lock); + expired = fuse_request_expired(fch, &fiq->pending); + spin_unlock(&fiq->lock); + if (expired) + goto chan_abort; + + spin_lock(&fch->bg_lock); + expired = fuse_request_expired(fch, &fch->bg_queue); + spin_unlock(&fch->bg_lock); + if (expired) + goto chan_abort; + + spin_lock(&fch->lock); + if (!fch->connected) { + spin_unlock(&fch->lock); + return; + } + list_for_each_entry(fud, &fch->devices, entry) { + fpq = &fud->pq; + spin_lock(&fpq->lock); + if (fuse_request_expired(fch, &fpq->io) || + fuse_fpq_processing_expired(fch, fpq->processing)) { + spin_unlock(&fpq->lock); + spin_unlock(&fch->lock); + goto chan_abort; + } + + spin_unlock(&fpq->lock); + } + spin_unlock(&fch->lock); + + if (fuse_uring_request_expired(fch)) + goto chan_abort; + +out: + queue_delayed_work(system_percpu_wq, &fch->timeout.work, + fuse_timeout_timer_freq); + return; + +chan_abort: + fuse_chan_abort(fch, false); +} + +static void set_request_timeout(struct fuse_chan *fch, unsigned int timeout) +{ + fch->timeout.req_timeout = secs_to_jiffies(timeout); + INIT_DELAYED_WORK(&fch->timeout.work, fuse_check_timeout); + queue_delayed_work(system_percpu_wq, &fch->timeout.work, + fuse_timeout_timer_freq); +} + +void fuse_init_server_timeout(struct fuse_chan *fch, unsigned int timeout) +{ + if (!timeout && !fuse_max_req_timeout && !fuse_default_req_timeout) + return; + + if (!timeout) + timeout = fuse_default_req_timeout; + + if (fuse_max_req_timeout) { + if (timeout) + timeout = min(fuse_max_req_timeout, timeout); + else + timeout = fuse_max_req_timeout; + } + + timeout = max(FUSE_TIMEOUT_TIMER_FREQ, timeout); + + set_request_timeout(fch, timeout); +} + diff --git a/fs/fuse/sysctl.c b/fs/fuse/sysctl.c new file mode 100644 index 000000000..74eca5ce9 --- /dev/null +++ b/fs/fuse/sysctl.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/fuse/fuse_sysctl.c + * + * Sysctl interface to fuse parameters + */ +#include + +#include "sysctl.h" +#include "fuse_i.h" + +static struct ctl_table_header *fuse_table_header; + +/* Bound by fuse_init_out max_pages, which is a u16 */ +static unsigned int sysctl_fuse_max_pages_limit = 65535; + +/* + * fuse_init_out request timeouts are u16. + * This goes up to ~18 hours, which is plenty for a timeout. + */ +static unsigned int sysctl_fuse_req_timeout_limit = 65535; + +static const struct ctl_table fuse_sysctl_table[] = { + { + .procname = "max_pages_limit", + .data = &fuse_max_pages_limit, + .maxlen = sizeof(fuse_max_pages_limit), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = SYSCTL_ONE, + .extra2 = &sysctl_fuse_max_pages_limit, + }, + { + .procname = "default_request_timeout", + .data = &fuse_default_req_timeout, + .maxlen = sizeof(fuse_default_req_timeout), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &sysctl_fuse_req_timeout_limit, + }, + { + .procname = "max_request_timeout", + .data = &fuse_max_req_timeout, + .maxlen = sizeof(fuse_max_req_timeout), + .mode = 0644, + .proc_handler = proc_douintvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = &sysctl_fuse_req_timeout_limit, + }, +}; + +int fuse_sysctl_register(void) +{ + fuse_table_header = register_sysctl("fs/fuse", fuse_sysctl_table); + if (!fuse_table_header) + return -ENOMEM; + return 0; +} + +void fuse_sysctl_unregister(void) +{ + unregister_sysctl_table(fuse_table_header); + fuse_table_header = NULL; +} diff --git a/fs/fuse/sysctl.h b/fs/fuse/sysctl.h new file mode 100644 index 000000000..948d88417 --- /dev/null +++ b/fs/fuse/sysctl.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _FS_FUSE_SYSCTL_H +#define _FS_FUSE_SYSCTL_H + +extern unsigned int fuse_default_req_timeout; +extern unsigned int fuse_max_req_timeout; + +#endif /* _FS_FUSE_SYSCTL_H */ diff --git a/fs/fuse/trace.c b/fs/fuse/trace.c new file mode 100644 index 000000000..93bd72efc --- /dev/null +++ b/fs/fuse/trace.c @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2025 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "dev_uring_i.h" +#include "fuse_i.h" +#include "fuse_dev_i.h" + +#include + +#define CREATE_TRACE_POINTS +#include "fuse_trace.h" diff --git a/fs/fuse/virtio_fs.c b/fs/fuse/virtio_fs.c new file mode 100644 index 000000000..f15e516eb --- /dev/null +++ b/fs/fuse/virtio_fs.c @@ -0,0 +1,1854 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * virtio-fs: Virtio Filesystem + * Copyright (C) 2018 Red Hat, Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "dev.h" +#include "fuse_i.h" +#include "fuse_dev_i.h" + +/* Used to help calculate the FUSE connection's max_pages limit for a request's + * size. Parts of the struct fuse_req are sliced into scattergather lists in + * addition to the pages used, so this can help account for that overhead. + */ +#define FUSE_HEADER_OVERHEAD 4 + +/* List of virtio-fs device instances and a lock for the list. Also provides + * mutual exclusion in device removal and mounting path + */ +static DEFINE_MUTEX(virtio_fs_mutex); +static LIST_HEAD(virtio_fs_instances); + +/* The /sys/fs/virtio_fs/ kset */ +static struct kset *virtio_fs_kset; + +enum { + VQ_HIPRIO, + VQ_REQUEST +}; + +#define VQ_NAME_LEN 24 + +/* Per-virtqueue state */ +struct virtio_fs_vq { + spinlock_t lock; + struct virtqueue *vq; /* protected by ->lock */ + struct work_struct done_work; + struct list_head queued_reqs; + struct list_head end_reqs; /* End these requests */ + struct work_struct dispatch_work; + struct fuse_dev *fud; + bool connected; + long in_flight; + struct completion in_flight_zero; /* No inflight requests */ + struct kobject *kobj; + char name[VQ_NAME_LEN]; +} ____cacheline_aligned_in_smp; + +/* A virtio-fs device instance */ +struct virtio_fs { + struct kobject kobj; + struct kobject *mqs_kobj; + struct list_head list; /* on virtio_fs_instances */ + char *tag; + struct virtio_fs_vq *vqs; + unsigned int nvqs; /* number of virtqueues */ + unsigned int num_request_queues; /* number of request queues */ + struct dax_device *dax_dev; + + unsigned int *mq_map; /* index = cpu id, value = request vq id */ + + /* DAX memory window where file contents are mapped */ + void *window_kaddr; + phys_addr_t window_phys_addr; + size_t window_len; +}; + +struct virtio_fs_forget_req { + struct fuse_in_header ih; + struct fuse_forget_in arg; +}; + +struct virtio_fs_forget { + /* This request can be temporarily queued on virt queue */ + struct list_head list; + struct virtio_fs_forget_req req; +}; + +struct virtio_fs_req_work { + struct fuse_req *req; + struct virtio_fs_vq *fsvq; + struct work_struct done_work; +}; + +static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, + struct fuse_req *req, bool in_flight, + gfp_t gfp); + +static const struct constant_table dax_param_enums[] = { + {"always", FUSE_DAX_ALWAYS }, + {"never", FUSE_DAX_NEVER }, + {"inode", FUSE_DAX_INODE_USER }, + {} +}; + +enum { + OPT_DAX, + OPT_DAX_ENUM, +}; + +static const struct fs_parameter_spec virtio_fs_parameters[] = { + fsparam_flag("dax", OPT_DAX), + fsparam_enum("dax", OPT_DAX_ENUM, dax_param_enums), + {} +}; + +static int virtio_fs_parse_param(struct fs_context *fsc, + struct fs_parameter *param) +{ + struct fs_parse_result result; + struct fuse_fs_context *ctx = fsc->fs_private; + int opt; + + opt = fs_parse(fsc, virtio_fs_parameters, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case OPT_DAX: + ctx->dax_mode = FUSE_DAX_ALWAYS; + break; + case OPT_DAX_ENUM: + ctx->dax_mode = result.uint_32; + break; + default: + return -EINVAL; + } + + return 0; +} + +static void virtio_fs_free_fsc(struct fs_context *fsc) +{ + struct fuse_fs_context *ctx = fsc->fs_private; + + kfree(ctx); +} + +static inline struct virtio_fs_vq *vq_to_fsvq(struct virtqueue *vq) +{ + struct virtio_fs *fs = vq->vdev->priv; + + return &fs->vqs[vq->index]; +} + +/* Should be called with fsvq->lock held. */ +static inline void inc_in_flight_req(struct virtio_fs_vq *fsvq) +{ + fsvq->in_flight++; +} + +/* Should be called with fsvq->lock held. */ +static inline void dec_in_flight_req(struct virtio_fs_vq *fsvq) +{ + WARN_ON(fsvq->in_flight <= 0); + fsvq->in_flight--; + if (!fsvq->in_flight) + complete(&fsvq->in_flight_zero); +} + +static ssize_t tag_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct virtio_fs *fs = container_of(kobj, struct virtio_fs, kobj); + + return sysfs_emit(buf, "%s\n", fs->tag); +} + +static struct kobj_attribute virtio_fs_tag_attr = __ATTR_RO(tag); + +static struct attribute *virtio_fs_attrs[] = { + &virtio_fs_tag_attr.attr, + NULL +}; +ATTRIBUTE_GROUPS(virtio_fs); + +static void virtio_fs_ktype_release(struct kobject *kobj) +{ + struct virtio_fs *vfs = container_of(kobj, struct virtio_fs, kobj); + + kfree(vfs->mq_map); + kfree(vfs->vqs); + kfree(vfs); +} + +static const struct kobj_type virtio_fs_ktype = { + .release = virtio_fs_ktype_release, + .sysfs_ops = &kobj_sysfs_ops, + .default_groups = virtio_fs_groups, +}; + +static struct virtio_fs_vq *virtio_fs_kobj_to_vq(struct virtio_fs *fs, + struct kobject *kobj) +{ + int i; + + for (i = 0; i < fs->nvqs; i++) { + if (kobj == fs->vqs[i].kobj) + return &fs->vqs[i]; + } + return NULL; +} + +static ssize_t name_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct virtio_fs *fs = container_of(kobj->parent->parent, struct virtio_fs, kobj); + struct virtio_fs_vq *fsvq = virtio_fs_kobj_to_vq(fs, kobj); + + if (!fsvq) + return -EINVAL; + return sysfs_emit(buf, "%s\n", fsvq->name); +} + +static struct kobj_attribute virtio_fs_vq_name_attr = __ATTR_RO(name); + +static ssize_t cpu_list_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct virtio_fs *fs = container_of(kobj->parent->parent, struct virtio_fs, kobj); + struct virtio_fs_vq *fsvq = virtio_fs_kobj_to_vq(fs, kobj); + unsigned int cpu, qid; + const size_t size = PAGE_SIZE - 1; + bool first = true; + int ret = 0, pos = 0; + + if (!fsvq) + return -EINVAL; + + qid = fsvq->vq->index; + for (cpu = 0; cpu < nr_cpu_ids; cpu++) { + if (qid < VQ_REQUEST || (fs->mq_map[cpu] == qid)) { + if (first) + ret = snprintf(buf + pos, size - pos, "%u", cpu); + else + ret = snprintf(buf + pos, size - pos, ", %u", cpu); + + if (ret >= size - pos) + break; + first = false; + pos += ret; + } + } + ret = snprintf(buf + pos, size + 1 - pos, "\n"); + return pos + ret; +} + +static struct kobj_attribute virtio_fs_vq_cpu_list_attr = __ATTR_RO(cpu_list); + +static struct attribute *virtio_fs_vq_attrs[] = { + &virtio_fs_vq_name_attr.attr, + &virtio_fs_vq_cpu_list_attr.attr, + NULL +}; + +static struct attribute_group virtio_fs_vq_attr_group = { + .attrs = virtio_fs_vq_attrs, +}; + +/* Make sure virtiofs_mutex is held */ +static void virtio_fs_put_locked(struct virtio_fs *fs) +{ + lockdep_assert_held(&virtio_fs_mutex); + + kobject_put(&fs->kobj); +} + +static void virtio_fs_put(struct virtio_fs *fs) +{ + mutex_lock(&virtio_fs_mutex); + virtio_fs_put_locked(fs); + mutex_unlock(&virtio_fs_mutex); +} + +static void virtio_fs_fiq_release(struct fuse_iqueue *fiq) +{ + struct virtio_fs *vfs = fiq->priv; + + virtio_fs_put(vfs); +} + +static void virtio_fs_drain_queue(struct virtio_fs_vq *fsvq) +{ + WARN_ON(fsvq->in_flight < 0); + + /* Wait for in flight requests to finish.*/ + spin_lock(&fsvq->lock); + if (fsvq->in_flight) { + /* We are holding virtio_fs_mutex. There should not be any + * waiters waiting for completion. + */ + reinit_completion(&fsvq->in_flight_zero); + spin_unlock(&fsvq->lock); + wait_for_completion(&fsvq->in_flight_zero); + } else { + spin_unlock(&fsvq->lock); + } + + flush_work(&fsvq->done_work); + flush_work(&fsvq->dispatch_work); +} + +static void virtio_fs_drain_all_queues_locked(struct virtio_fs *fs) +{ + struct virtio_fs_vq *fsvq; + int i; + + for (i = 0; i < fs->nvqs; i++) { + fsvq = &fs->vqs[i]; + virtio_fs_drain_queue(fsvq); + } +} + +static void virtio_fs_drain_all_queues(struct virtio_fs *fs) +{ + /* Provides mutual exclusion between ->remove and ->kill_sb + * paths. We don't want both of these draining queue at the + * same time. Current completion logic reinits completion + * and that means there should not be any other thread + * doing reinit or waiting for completion already. + */ + mutex_lock(&virtio_fs_mutex); + virtio_fs_drain_all_queues_locked(fs); + mutex_unlock(&virtio_fs_mutex); +} + +static void virtio_fs_start_all_queues(struct virtio_fs *fs) +{ + struct virtio_fs_vq *fsvq; + int i; + + for (i = 0; i < fs->nvqs; i++) { + fsvq = &fs->vqs[i]; + spin_lock(&fsvq->lock); + fsvq->connected = true; + spin_unlock(&fsvq->lock); + } +} + +static void virtio_fs_delete_queues_sysfs(struct virtio_fs *fs) +{ + struct virtio_fs_vq *fsvq; + int i; + + for (i = 0; i < fs->nvqs; i++) { + fsvq = &fs->vqs[i]; + kobject_put(fsvq->kobj); + } +} + +static int virtio_fs_add_queues_sysfs(struct virtio_fs *fs) +{ + struct virtio_fs_vq *fsvq; + char buff[12]; + int i, j, ret; + + for (i = 0; i < fs->nvqs; i++) { + fsvq = &fs->vqs[i]; + + sprintf(buff, "%d", i); + fsvq->kobj = kobject_create_and_add(buff, fs->mqs_kobj); + if (!fsvq->kobj) { + ret = -ENOMEM; + goto out_del; + } + + ret = sysfs_create_group(fsvq->kobj, &virtio_fs_vq_attr_group); + if (ret) { + kobject_put(fsvq->kobj); + goto out_del; + } + } + + return 0; + +out_del: + for (j = 0; j < i; j++) { + fsvq = &fs->vqs[j]; + kobject_put(fsvq->kobj); + } + return ret; +} + +/* Add a new instance to the list or return -EEXIST if tag name exists*/ +static int virtio_fs_add_instance(struct virtio_device *vdev, + struct virtio_fs *fs) +{ + struct virtio_fs *fs2; + int ret; + + mutex_lock(&virtio_fs_mutex); + + list_for_each_entry(fs2, &virtio_fs_instances, list) { + if (strcmp(fs->tag, fs2->tag) == 0) { + mutex_unlock(&virtio_fs_mutex); + return -EEXIST; + } + } + + /* Use the virtio_device's index as a unique identifier, there is no + * need to allocate our own identifiers because the virtio_fs instance + * is only visible to userspace as long as the underlying virtio_device + * exists. + */ + fs->kobj.kset = virtio_fs_kset; + ret = kobject_add(&fs->kobj, NULL, "%d", vdev->index); + if (ret < 0) + goto out_unlock; + + fs->mqs_kobj = kobject_create_and_add("mqs", &fs->kobj); + if (!fs->mqs_kobj) { + ret = -ENOMEM; + goto out_del; + } + + ret = sysfs_create_link(&fs->kobj, &vdev->dev.kobj, "device"); + if (ret < 0) + goto out_put; + + ret = virtio_fs_add_queues_sysfs(fs); + if (ret) + goto out_remove; + + list_add_tail(&fs->list, &virtio_fs_instances); + + mutex_unlock(&virtio_fs_mutex); + + kobject_uevent(&fs->kobj, KOBJ_ADD); + + return 0; + +out_remove: + sysfs_remove_link(&fs->kobj, "device"); +out_put: + kobject_put(fs->mqs_kobj); +out_del: + kobject_del(&fs->kobj); +out_unlock: + mutex_unlock(&virtio_fs_mutex); + return ret; +} + +/* Return the virtio_fs with a given tag, or NULL */ +static struct virtio_fs *virtio_fs_find_instance(const char *tag) +{ + struct virtio_fs *fs; + + mutex_lock(&virtio_fs_mutex); + + list_for_each_entry(fs, &virtio_fs_instances, list) { + if (strcmp(fs->tag, tag) == 0) { + kobject_get(&fs->kobj); + goto found; + } + } + + fs = NULL; /* not found */ + +found: + mutex_unlock(&virtio_fs_mutex); + + return fs; +} + +static void virtio_fs_free_devs(struct virtio_fs *fs) +{ + unsigned int i; + + for (i = 0; i < fs->nvqs; i++) { + struct virtio_fs_vq *fsvq = &fs->vqs[i]; + + if (!fsvq->fud) + continue; + + fuse_dev_put(fsvq->fud); + fsvq->fud = NULL; + } +} + +/* Read filesystem name from virtio config into fs->tag (must kfree()). */ +static int virtio_fs_read_tag(struct virtio_device *vdev, struct virtio_fs *fs) +{ + char tag_buf[sizeof_field(struct virtio_fs_config, tag)]; + char *end; + size_t len; + + virtio_cread_bytes(vdev, offsetof(struct virtio_fs_config, tag), + &tag_buf, sizeof(tag_buf)); + end = memchr(tag_buf, '\0', sizeof(tag_buf)); + if (end == tag_buf) + return -EINVAL; /* empty tag */ + if (!end) + end = &tag_buf[sizeof(tag_buf)]; + + len = end - tag_buf; + fs->tag = devm_kmalloc(&vdev->dev, len + 1, GFP_KERNEL); + if (!fs->tag) + return -ENOMEM; + memcpy(fs->tag, tag_buf, len); + fs->tag[len] = '\0'; + + /* While the VIRTIO specification allows any character, newlines are + * awkward on mount(8) command-lines and cause problems in the sysfs + * "tag" attr and uevent TAG= properties. Forbid them. + */ + if (strchr(fs->tag, '\n')) { + dev_dbg(&vdev->dev, "refusing virtiofs tag with newline character\n"); + return -EINVAL; + } + + dev_info(&vdev->dev, "discovered new tag: %s\n", fs->tag); + return 0; +} + +/* Work function for hiprio completion */ +static void virtio_fs_hiprio_done_work(struct work_struct *work) +{ + struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, + done_work); + struct virtqueue *vq = fsvq->vq; + + /* Free completed FUSE_FORGET requests */ + spin_lock(&fsvq->lock); + do { + unsigned int len; + void *req; + + virtqueue_disable_cb(vq); + + while ((req = virtqueue_get_buf(vq, &len)) != NULL) { + kfree(req); + dec_in_flight_req(fsvq); + } + } while (!virtqueue_enable_cb(vq)); + + if (!list_empty(&fsvq->queued_reqs)) + schedule_work(&fsvq->dispatch_work); + + spin_unlock(&fsvq->lock); +} + +static void virtio_fs_request_dispatch_work(struct work_struct *work) +{ + struct fuse_req *req; + struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, + dispatch_work); + int ret; + + pr_debug("virtio-fs: worker %s called.\n", __func__); + while (1) { + spin_lock(&fsvq->lock); + req = list_first_entry_or_null(&fsvq->end_reqs, struct fuse_req, + list); + if (!req) { + spin_unlock(&fsvq->lock); + break; + } + + list_del_init(&req->list); + spin_unlock(&fsvq->lock); + fuse_request_end(req); + } + + /* Dispatch pending requests */ + while (1) { + unsigned int flags; + + spin_lock(&fsvq->lock); + req = list_first_entry_or_null(&fsvq->queued_reqs, + struct fuse_req, list); + if (!req) { + spin_unlock(&fsvq->lock); + return; + } + list_del_init(&req->list); + spin_unlock(&fsvq->lock); + + flags = memalloc_nofs_save(); + ret = virtio_fs_enqueue_req(fsvq, req, true, GFP_KERNEL); + memalloc_nofs_restore(flags); + if (ret < 0) { + if (ret == -ENOSPC) { + spin_lock(&fsvq->lock); + list_add_tail(&req->list, &fsvq->queued_reqs); + spin_unlock(&fsvq->lock); + return; + } + req->out.h.error = ret; + spin_lock(&fsvq->lock); + dec_in_flight_req(fsvq); + spin_unlock(&fsvq->lock); + pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n", + ret); + fuse_request_end(req); + } + } +} + +/* + * Returns 1 if queue is full and sender should wait a bit before sending + * next request, 0 otherwise. + */ +static int send_forget_request(struct virtio_fs_vq *fsvq, + struct virtio_fs_forget *forget, + bool in_flight) +{ + struct scatterlist sg; + struct virtqueue *vq; + int ret = 0; + bool notify; + struct virtio_fs_forget_req *req = &forget->req; + + spin_lock(&fsvq->lock); + if (!fsvq->connected) { + if (in_flight) + dec_in_flight_req(fsvq); + kfree(forget); + goto out; + } + + sg_init_one(&sg, req, sizeof(*req)); + vq = fsvq->vq; + dev_dbg(&vq->vdev->dev, "%s\n", __func__); + + ret = virtqueue_add_outbuf(vq, &sg, 1, forget, GFP_ATOMIC); + if (ret < 0) { + if (ret == -ENOSPC) { + pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later\n", + ret); + list_add_tail(&forget->list, &fsvq->queued_reqs); + if (!in_flight) + inc_in_flight_req(fsvq); + /* Queue is full */ + ret = 1; + } else { + pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n", + ret); + kfree(forget); + if (in_flight) + dec_in_flight_req(fsvq); + } + goto out; + } + + if (!in_flight) + inc_in_flight_req(fsvq); + notify = virtqueue_kick_prepare(vq); + spin_unlock(&fsvq->lock); + + if (notify) + virtqueue_notify(vq); + return ret; +out: + spin_unlock(&fsvq->lock); + return ret; +} + +static void virtio_fs_hiprio_dispatch_work(struct work_struct *work) +{ + struct virtio_fs_forget *forget; + struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, + dispatch_work); + pr_debug("virtio-fs: worker %s called.\n", __func__); + while (1) { + spin_lock(&fsvq->lock); + forget = list_first_entry_or_null(&fsvq->queued_reqs, + struct virtio_fs_forget, list); + if (!forget) { + spin_unlock(&fsvq->lock); + return; + } + + list_del(&forget->list); + spin_unlock(&fsvq->lock); + if (send_forget_request(fsvq, forget, true)) + return; + } +} + +/* Allocate and copy args into req->argbuf */ +static int copy_args_to_argbuf(struct fuse_req *req, gfp_t gfp) +{ + struct fuse_args *args = req->args; + unsigned int offset = 0; + unsigned int num_in; + unsigned int num_out; + unsigned int len; + unsigned int i; + + num_in = args->in_numargs - args->in_pages; + num_out = args->out_numargs - args->out_pages; + len = fuse_len_args(num_in, (struct fuse_arg *) args->in_args) + + fuse_len_args(num_out, args->out_args); + + req->argbuf = kmalloc(len, gfp); + if (!req->argbuf) + return -ENOMEM; + + for (i = 0; i < num_in; i++) { + memcpy(req->argbuf + offset, + args->in_args[i].value, + args->in_args[i].size); + offset += args->in_args[i].size; + } + + return 0; +} + +/* Copy args out of and free req->argbuf */ +static void copy_args_from_argbuf(struct fuse_args *args, struct fuse_req *req) +{ + unsigned int remaining; + unsigned int offset; + unsigned int num_in; + unsigned int num_out; + unsigned int i; + + remaining = req->out.h.len - sizeof(req->out.h); + num_in = args->in_numargs - args->in_pages; + num_out = args->out_numargs - args->out_pages; + offset = fuse_len_args(num_in, (struct fuse_arg *)args->in_args); + + for (i = 0; i < num_out; i++) { + unsigned int argsize = args->out_args[i].size; + + if (args->out_argvar && + i == args->out_numargs - 1 && + argsize > remaining) { + argsize = remaining; + } + + memcpy(args->out_args[i].value, req->argbuf + offset, argsize); + offset += argsize; + + if (i != args->out_numargs - 1) + remaining -= argsize; + } + + /* Store the actual size of the variable-length arg */ + if (args->out_argvar) + args->out_args[args->out_numargs - 1].size = remaining; + + kfree(req->argbuf); + req->argbuf = NULL; +} + +/* Verify that the server properly follows the FUSE protocol */ +static bool virtio_fs_verify_response(struct fuse_req *req, unsigned int len) +{ + struct fuse_out_header *oh = &req->out.h; + + if (len < sizeof(*oh)) { + pr_warn("virtio-fs: response too short (%u)\n", len); + return false; + } + if (oh->len != len) { + pr_warn("virtio-fs: oh.len mismatch (%u != %u)\n", oh->len, len); + return false; + } + if (oh->unique != req->in.h.unique) { + pr_warn("virtio-fs: oh.unique mismatch (%llu != %llu)\n", + oh->unique, req->in.h.unique); + return false; + } + return true; +} + +/* Work function for request completion */ +static void virtio_fs_request_complete(struct fuse_req *req, + struct virtio_fs_vq *fsvq) +{ + struct fuse_args *args; + struct fuse_args_pages *ap; + unsigned int len, i, thislen; + struct folio *folio; + + args = req->args; + copy_args_from_argbuf(args, req); + + if (args->out_pages && args->page_zeroing) { + len = args->out_args[args->out_numargs - 1].size; + ap = container_of(args, typeof(*ap), args); + for (i = 0; i < ap->num_folios; i++) { + thislen = ap->descs[i].length; + if (len < thislen) { + WARN_ON(ap->descs[i].offset); + folio = ap->folios[i]; + folio_zero_segment(folio, len, thislen); + len = 0; + } else { + len -= thislen; + } + } + } + + clear_bit(FR_SENT, &req->flags); + + fuse_request_end(req); + spin_lock(&fsvq->lock); + dec_in_flight_req(fsvq); + spin_unlock(&fsvq->lock); +} + +static void virtio_fs_complete_req_work(struct work_struct *work) +{ + struct virtio_fs_req_work *w = + container_of(work, typeof(*w), done_work); + + virtio_fs_request_complete(w->req, w->fsvq); + kfree(w); +} + +static void virtio_fs_requests_done_work(struct work_struct *work) +{ + struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, + done_work); + struct fuse_pqueue *fpq = &fsvq->fud->pq; + struct virtqueue *vq = fsvq->vq; + struct fuse_req *req; + struct fuse_req *next; + unsigned int len; + LIST_HEAD(reqs); + + /* Collect completed requests off the virtqueue */ + spin_lock(&fsvq->lock); + do { + virtqueue_disable_cb(vq); + + while ((req = virtqueue_get_buf(vq, &len)) != NULL) { + if (!virtio_fs_verify_response(req, len)) { + req->out.h.error = -EIO; + req->out.h.len = sizeof(struct fuse_out_header); + } + spin_lock(&fpq->lock); + list_move_tail(&req->list, &reqs); + spin_unlock(&fpq->lock); + } + } while (!virtqueue_enable_cb(vq)); + spin_unlock(&fsvq->lock); + + /* End requests */ + list_for_each_entry_safe(req, next, &reqs, list) { + list_del_init(&req->list); + + /* blocking async request completes in a worker context */ + if (req->args->may_block) { + struct virtio_fs_req_work *w; + + w = kzalloc_obj(*w, GFP_NOFS | __GFP_NOFAIL); + INIT_WORK(&w->done_work, virtio_fs_complete_req_work); + w->fsvq = fsvq; + w->req = req; + schedule_work(&w->done_work); + } else { + virtio_fs_request_complete(req, fsvq); + } + } + + /* Try to push previously queued requests, as the queue might no longer be full */ + spin_lock(&fsvq->lock); + if (!list_empty(&fsvq->queued_reqs)) + schedule_work(&fsvq->dispatch_work); + spin_unlock(&fsvq->lock); +} + +static void virtio_fs_map_queues(struct virtio_device *vdev, struct virtio_fs *fs) +{ + const struct cpumask *mask, *masks; + unsigned int q, cpu, nr_masks; + + /* First attempt to map using existing transport layer affinities + * e.g. PCIe MSI-X + */ + if (!vdev->config->get_vq_affinity) + goto fallback; + + for (q = 0; q < fs->num_request_queues; q++) { + mask = vdev->config->get_vq_affinity(vdev, VQ_REQUEST + q); + if (!mask) + goto fallback; + + for_each_cpu(cpu, mask) + fs->mq_map[cpu] = q + VQ_REQUEST; + } + + return; +fallback: + /* Attempt to map evenly in groups over the CPUs */ + masks = group_cpus_evenly(fs->num_request_queues, &nr_masks); + /* If even this fails we default to all CPUs use first request queue */ + if (!masks) { + for_each_possible_cpu(cpu) + fs->mq_map[cpu] = VQ_REQUEST; + return; + } + + for (q = 0; q < fs->num_request_queues; q++) { + for_each_cpu(cpu, &masks[q % nr_masks]) + fs->mq_map[cpu] = q + VQ_REQUEST; + } + kfree(masks); +} + +/* Virtqueue interrupt handler */ +static void virtio_fs_vq_done(struct virtqueue *vq) +{ + struct virtio_fs_vq *fsvq = vq_to_fsvq(vq); + + dev_dbg(&vq->vdev->dev, "%s %s\n", __func__, fsvq->name); + + schedule_work(&fsvq->done_work); +} + +static void virtio_fs_init_vq(struct virtio_fs_vq *fsvq, char *name, + int vq_type) +{ + strscpy(fsvq->name, name, VQ_NAME_LEN); + spin_lock_init(&fsvq->lock); + INIT_LIST_HEAD(&fsvq->queued_reqs); + INIT_LIST_HEAD(&fsvq->end_reqs); + init_completion(&fsvq->in_flight_zero); + + if (vq_type == VQ_REQUEST) { + INIT_WORK(&fsvq->done_work, virtio_fs_requests_done_work); + INIT_WORK(&fsvq->dispatch_work, + virtio_fs_request_dispatch_work); + } else { + INIT_WORK(&fsvq->done_work, virtio_fs_hiprio_done_work); + INIT_WORK(&fsvq->dispatch_work, + virtio_fs_hiprio_dispatch_work); + } +} + +/* Initialize virtqueues */ +static int virtio_fs_setup_vqs(struct virtio_device *vdev, + struct virtio_fs *fs) +{ + struct virtqueue_info *vqs_info; + struct virtqueue **vqs; + /* Specify pre_vectors to ensure that the queues before the + * request queues (e.g. hiprio) don't claim any of the CPUs in + * the multi-queue mapping and interrupt affinities + */ + struct irq_affinity desc = { .pre_vectors = VQ_REQUEST }; + unsigned int i; + int ret = 0; + + virtio_cread_le(vdev, struct virtio_fs_config, num_request_queues, + &fs->num_request_queues); + if (fs->num_request_queues == 0) + return -EINVAL; + + /* Truncate nr of request queues to nr_cpu_id */ + fs->num_request_queues = min_t(unsigned int, fs->num_request_queues, + nr_cpu_ids); + fs->nvqs = VQ_REQUEST + fs->num_request_queues; + fs->vqs = kzalloc_objs(fs->vqs[VQ_HIPRIO], fs->nvqs); + if (!fs->vqs) + return -ENOMEM; + + vqs = kmalloc_objs(vqs[VQ_HIPRIO], fs->nvqs); + fs->mq_map = kcalloc_node(nr_cpu_ids, sizeof(*fs->mq_map), GFP_KERNEL, + dev_to_node(&vdev->dev)); + vqs_info = kzalloc_objs(*vqs_info, fs->nvqs); + if (!vqs || !vqs_info || !fs->mq_map) { + ret = -ENOMEM; + goto out; + } + + /* Initialize the hiprio/forget request virtqueue */ + vqs_info[VQ_HIPRIO].callback = virtio_fs_vq_done; + virtio_fs_init_vq(&fs->vqs[VQ_HIPRIO], "hiprio", VQ_HIPRIO); + vqs_info[VQ_HIPRIO].name = fs->vqs[VQ_HIPRIO].name; + + /* Initialize the requests virtqueues */ + for (i = VQ_REQUEST; i < fs->nvqs; i++) { + char vq_name[VQ_NAME_LEN]; + + snprintf(vq_name, VQ_NAME_LEN, "requests.%u", i - VQ_REQUEST); + virtio_fs_init_vq(&fs->vqs[i], vq_name, VQ_REQUEST); + vqs_info[i].callback = virtio_fs_vq_done; + vqs_info[i].name = fs->vqs[i].name; + } + + ret = virtio_find_vqs(vdev, fs->nvqs, vqs, vqs_info, &desc); + if (ret < 0) + goto out; + + for (i = 0; i < fs->nvqs; i++) + fs->vqs[i].vq = vqs[i]; + + virtio_fs_start_all_queues(fs); +out: + kfree(vqs_info); + kfree(vqs); + if (ret) { + kfree(fs->vqs); + fs->vqs = NULL; + kfree(fs->mq_map); + fs->mq_map = NULL; + } + return ret; +} + +/* Free virtqueues (device must already be reset) */ +static void virtio_fs_cleanup_vqs(struct virtio_device *vdev) +{ + vdev->config->del_vqs(vdev); +} + +/* Map a window offset to a page frame number. The window offset will have + * been produced by .iomap_next(), which maps a file offset to a window offset. + */ +static long virtio_fs_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, + long nr_pages, enum dax_access_mode mode, + void **kaddr, unsigned long *pfn) +{ + struct virtio_fs *fs = dax_get_private(dax_dev); + phys_addr_t offset = PFN_PHYS(pgoff); + size_t max_nr_pages = fs->window_len / PAGE_SIZE - pgoff; + + if (kaddr) + *kaddr = fs->window_kaddr + offset; + if (pfn) + *pfn = PHYS_PFN(fs->window_phys_addr + offset); + return nr_pages > max_nr_pages ? max_nr_pages : nr_pages; +} + +static int virtio_fs_zero_page_range(struct dax_device *dax_dev, + pgoff_t pgoff, size_t nr_pages) +{ + long rc; + void *kaddr; + + rc = dax_direct_access(dax_dev, pgoff, nr_pages, DAX_ACCESS, &kaddr, + NULL); + if (rc < 0) + return dax_mem2blk_err(rc); + + memset(kaddr, 0, nr_pages << PAGE_SHIFT); + dax_flush(dax_dev, kaddr, nr_pages << PAGE_SHIFT); + return 0; +} + +static const struct dax_operations virtio_fs_dax_ops = { + .direct_access = virtio_fs_direct_access, + .zero_page_range = virtio_fs_zero_page_range, +}; + +static void virtio_fs_cleanup_dax(void *data) +{ + struct dax_device *dax_dev = data; + + kill_dax(dax_dev); + put_dax(dax_dev); +} + +DEFINE_FREE(cleanup_dax, struct dax_dev *, if (!IS_ERR_OR_NULL(_T)) virtio_fs_cleanup_dax(_T)) + +static int virtio_fs_setup_dax(struct virtio_device *vdev, struct virtio_fs *fs) +{ + struct dax_device *dax_dev __free(cleanup_dax) = NULL; + struct virtio_shm_region cache_reg; + struct dev_pagemap *pgmap; + bool have_cache; + + if (!IS_ENABLED(CONFIG_FUSE_DAX)) + return 0; + + dax_dev = alloc_dax(fs, &virtio_fs_dax_ops); + if (IS_ERR(dax_dev)) { + int rc = PTR_ERR(dax_dev); + return rc == -EOPNOTSUPP ? 0 : rc; + } + + /* Get cache region */ + have_cache = virtio_get_shm_region(vdev, &cache_reg, + (u8)VIRTIO_FS_SHMCAP_ID_CACHE); + if (!have_cache) { + dev_notice(&vdev->dev, "%s: No cache capability\n", __func__); + return 0; + } + + if (!devm_request_mem_region(&vdev->dev, cache_reg.addr, cache_reg.len, + dev_name(&vdev->dev))) { + dev_warn(&vdev->dev, "could not reserve region addr=0x%llx len=0x%llx\n", + cache_reg.addr, cache_reg.len); + return -EBUSY; + } + + dev_notice(&vdev->dev, "Cache len: 0x%llx @ 0x%llx\n", cache_reg.len, + cache_reg.addr); + + pgmap = devm_kzalloc(&vdev->dev, sizeof(*pgmap), GFP_KERNEL); + if (!pgmap) + return -ENOMEM; + + pgmap->type = MEMORY_DEVICE_FS_DAX; + + /* Ideally we would directly use the PCI BAR resource but + * devm_memremap_pages() wants its own copy in pgmap. So + * initialize a struct resource from scratch (only the start + * and end fields will be used). + */ + pgmap->range = (struct range) { + .start = (phys_addr_t) cache_reg.addr, + .end = (phys_addr_t) cache_reg.addr + cache_reg.len - 1, + }; + pgmap->nr_range = 1; + + fs->window_kaddr = devm_memremap_pages(&vdev->dev, pgmap); + if (IS_ERR(fs->window_kaddr)) + return PTR_ERR(fs->window_kaddr); + + fs->window_phys_addr = (phys_addr_t) cache_reg.addr; + fs->window_len = (phys_addr_t) cache_reg.len; + + dev_dbg(&vdev->dev, "%s: window kaddr 0x%px phys_addr 0x%llx len 0x%llx\n", + __func__, fs->window_kaddr, cache_reg.addr, cache_reg.len); + + fs->dax_dev = no_free_ptr(dax_dev); + return devm_add_action_or_reset(&vdev->dev, virtio_fs_cleanup_dax, + fs->dax_dev); +} + +static int virtio_fs_probe(struct virtio_device *vdev) +{ + struct virtio_fs *fs; + int ret; + + fs = kzalloc_obj(*fs); + if (!fs) + return -ENOMEM; + kobject_init(&fs->kobj, &virtio_fs_ktype); + vdev->priv = fs; + + ret = virtio_fs_read_tag(vdev, fs); + if (ret < 0) + goto out; + + ret = virtio_fs_setup_vqs(vdev, fs); + if (ret < 0) + goto out; + + virtio_fs_map_queues(vdev, fs); + + ret = virtio_fs_setup_dax(vdev, fs); + if (ret < 0) + goto out_vqs; + + /* Bring the device online in case the filesystem is mounted and + * requests need to be sent before we return. + */ + virtio_device_ready(vdev); + + ret = virtio_fs_add_instance(vdev, fs); + if (ret < 0) + goto out_vqs; + + return 0; + +out_vqs: + virtio_reset_device(vdev); + virtio_fs_cleanup_vqs(vdev); + +out: + vdev->priv = NULL; + kobject_put(&fs->kobj); + return ret; +} + +static void virtio_fs_stop_all_queues(struct virtio_fs *fs) +{ + struct virtio_fs_vq *fsvq; + int i; + + for (i = 0; i < fs->nvqs; i++) { + fsvq = &fs->vqs[i]; + spin_lock(&fsvq->lock); + fsvq->connected = false; + spin_unlock(&fsvq->lock); + } +} + +static void virtio_fs_remove(struct virtio_device *vdev) +{ + struct virtio_fs *fs = vdev->priv; + + mutex_lock(&virtio_fs_mutex); + /* This device is going away. No one should get new reference */ + list_del_init(&fs->list); + virtio_fs_delete_queues_sysfs(fs); + sysfs_remove_link(&fs->kobj, "device"); + kobject_put(fs->mqs_kobj); + kobject_del(&fs->kobj); + virtio_fs_stop_all_queues(fs); + virtio_fs_drain_all_queues_locked(fs); + virtio_reset_device(vdev); + virtio_fs_cleanup_vqs(vdev); + + vdev->priv = NULL; + /* Put device reference on virtio_fs object */ + virtio_fs_put_locked(fs); + mutex_unlock(&virtio_fs_mutex); +} + +#ifdef CONFIG_PM_SLEEP +static int virtio_fs_freeze(struct virtio_device *vdev) +{ + /* TODO need to save state here */ + pr_warn("virtio-fs: suspend/resume not yet supported\n"); + return -EOPNOTSUPP; +} + +static int virtio_fs_restore(struct virtio_device *vdev) +{ + /* TODO need to restore state here */ + return 0; +} +#endif /* CONFIG_PM_SLEEP */ + +static const struct virtio_device_id id_table[] = { + { VIRTIO_ID_FS, VIRTIO_DEV_ANY_ID }, + {}, +}; + +static const unsigned int feature_table[] = {}; + +static struct virtio_driver virtio_fs_driver = { + .driver.name = KBUILD_MODNAME, + .id_table = id_table, + .feature_table = feature_table, + .feature_table_size = ARRAY_SIZE(feature_table), + .probe = virtio_fs_probe, + .remove = virtio_fs_remove, +#ifdef CONFIG_PM_SLEEP + .freeze = virtio_fs_freeze, + .restore = virtio_fs_restore, +#endif +}; + +static void virtio_fs_send_forget(struct fuse_iqueue *fiq, struct fuse_forget_link *link) +{ + struct virtio_fs_forget *forget; + struct virtio_fs_forget_req *req; + struct virtio_fs *fs = fiq->priv; + struct virtio_fs_vq *fsvq = &fs->vqs[VQ_HIPRIO]; + u64 unique = fuse_get_unique(fiq); + + /* Allocate a buffer for the request */ + forget = kmalloc_obj(*forget, GFP_NOFS | __GFP_NOFAIL); + req = &forget->req; + + req->ih = (struct fuse_in_header){ + .opcode = FUSE_FORGET, + .nodeid = link->forget_one.nodeid, + .unique = unique, + .len = sizeof(*req), + }; + req->arg = (struct fuse_forget_in){ + .nlookup = link->forget_one.nlookup, + }; + + send_forget_request(fsvq, forget, false); + kfree(link); +} + +static void virtio_fs_send_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req) +{ + /* + * TODO interrupts. + * + * Normal fs operations on a local filesystems aren't interruptible. + * Exceptions are blocking lock operations; for example fcntl(F_SETLKW) + * with shared lock between host and guest. + */ +} + +/* Count number of scatter-gather elements required */ +static unsigned int sg_count_fuse_folios(struct fuse_folio_desc *folio_descs, + unsigned int num_folios, + unsigned int total_len) +{ + unsigned int i; + unsigned int this_len; + + for (i = 0; i < num_folios && total_len; i++) { + this_len = min(folio_descs[i].length, total_len); + total_len -= this_len; + } + + return i; +} + +/* Return the number of scatter-gather list elements required */ +static unsigned int sg_count_fuse_req(struct fuse_req *req) +{ + struct fuse_args *args = req->args; + struct fuse_args_pages *ap = container_of(args, typeof(*ap), args); + unsigned int size, total_sgs = 1 /* fuse_in_header */; + + if (args->in_numargs - args->in_pages) + total_sgs += 1; + + if (args->in_pages) { + size = args->in_args[args->in_numargs - 1].size; + total_sgs += sg_count_fuse_folios(ap->descs, ap->num_folios, + size); + } + + if (!test_bit(FR_ISREPLY, &req->flags)) + return total_sgs; + + total_sgs += 1 /* fuse_out_header */; + + if (args->out_numargs - args->out_pages) + total_sgs += 1; + + if (args->out_pages) { + size = args->out_args[args->out_numargs - 1].size; + total_sgs += sg_count_fuse_folios(ap->descs, ap->num_folios, + size); + } + + return total_sgs; +} + +/* Add folios to scatter-gather list and return number of elements used */ +static unsigned int sg_init_fuse_folios(struct scatterlist *sg, + struct folio **folios, + struct fuse_folio_desc *folio_descs, + unsigned int num_folios, + unsigned int total_len) +{ + unsigned int i; + unsigned int this_len; + + for (i = 0; i < num_folios && total_len; i++) { + sg_init_table(&sg[i], 1); + this_len = min(folio_descs[i].length, total_len); + sg_set_folio(&sg[i], folios[i], this_len, folio_descs[i].offset); + total_len -= this_len; + } + + return i; +} + +/* Add args to scatter-gather list and return number of elements used */ +static unsigned int sg_init_fuse_args(struct scatterlist *sg, + struct fuse_req *req, + struct fuse_arg *args, + unsigned int numargs, + bool argpages, + void *argbuf, + unsigned int *len_used) +{ + struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args); + unsigned int total_sgs = 0; + unsigned int len; + + len = fuse_len_args(numargs - argpages, args); + if (len) + sg_init_one(&sg[total_sgs++], argbuf, len); + + if (argpages) + total_sgs += sg_init_fuse_folios(&sg[total_sgs], + ap->folios, ap->descs, + ap->num_folios, + args[numargs - 1].size); + + if (len_used) + *len_used = len; + + return total_sgs; +} + +/* Add a request to a virtqueue and kick the device */ +static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, + struct fuse_req *req, bool in_flight, + gfp_t gfp) +{ + /* requests need at least 4 elements */ + struct scatterlist *stack_sgs[6]; + struct scatterlist stack_sg[ARRAY_SIZE(stack_sgs)]; + struct scatterlist **sgs = stack_sgs; + struct scatterlist *sg = stack_sg; + struct virtqueue *vq; + struct fuse_args *args = req->args; + unsigned int argbuf_used = 0; + unsigned int out_sgs = 0; + unsigned int in_sgs = 0; + unsigned int total_sgs; + unsigned int i, hash; + int ret; + bool notify; + struct fuse_pqueue *fpq; + + /* Does the sglist fit on the stack? */ + total_sgs = sg_count_fuse_req(req); + if (total_sgs > ARRAY_SIZE(stack_sgs)) { + sgs = kmalloc_objs(sgs[0], total_sgs, gfp); + sg = kmalloc_objs(sg[0], total_sgs, gfp); + if (!sgs || !sg) { + ret = -ENOMEM; + goto out; + } + } + + /* Use a bounce buffer since stack args cannot be mapped */ + ret = copy_args_to_argbuf(req, gfp); + if (ret < 0) + goto out; + + /* Request elements */ + sg_init_one(&sg[out_sgs++], &req->in.h, sizeof(req->in.h)); + out_sgs += sg_init_fuse_args(&sg[out_sgs], req, + (struct fuse_arg *)args->in_args, + args->in_numargs, args->in_pages, + req->argbuf, &argbuf_used); + + /* Reply elements */ + if (test_bit(FR_ISREPLY, &req->flags)) { + sg_init_one(&sg[out_sgs + in_sgs++], + &req->out.h, sizeof(req->out.h)); + in_sgs += sg_init_fuse_args(&sg[out_sgs + in_sgs], req, + args->out_args, args->out_numargs, + args->out_pages, + req->argbuf + argbuf_used, NULL); + } + + WARN_ON(out_sgs + in_sgs != total_sgs); + + for (i = 0; i < total_sgs; i++) + sgs[i] = &sg[i]; + + spin_lock(&fsvq->lock); + + if (!fsvq->connected) { + spin_unlock(&fsvq->lock); + ret = -ENOTCONN; + goto out; + } + + vq = fsvq->vq; + ret = virtqueue_add_sgs(vq, sgs, out_sgs, in_sgs, req, GFP_ATOMIC); + if (ret < 0) { + spin_unlock(&fsvq->lock); + goto out; + } + + /* Request successfully sent. */ + fpq = &fsvq->fud->pq; + hash = fuse_req_hash(req->in.h.unique); + spin_lock(&fpq->lock); + list_add_tail(&req->list, &fpq->processing[hash]); + spin_unlock(&fpq->lock); + set_bit(FR_SENT, &req->flags); + /* matches barrier in request_wait_answer() */ + smp_mb__after_atomic(); + + if (!in_flight) + inc_in_flight_req(fsvq); + notify = virtqueue_kick_prepare(vq); + + spin_unlock(&fsvq->lock); + + if (notify) + virtqueue_notify(vq); + +out: + if (ret < 0 && req->argbuf) { + kfree(req->argbuf); + req->argbuf = NULL; + } + if (sgs != stack_sgs) { + kfree(sgs); + kfree(sg); + } + + return ret; +} + +static void virtio_fs_send_req(struct fuse_iqueue *fiq, struct fuse_req *req) +{ + unsigned int queue_id; + struct virtio_fs *fs; + struct virtio_fs_vq *fsvq; + int ret; + + fuse_request_assign_unique(fiq, req); + + clear_bit(FR_PENDING, &req->flags); + + fs = fiq->priv; + queue_id = fs->mq_map[raw_smp_processor_id()]; + + pr_debug("%s: opcode %u unique %#llx nodeid %#llx in.len %u out.len %u queue_id %u\n", + __func__, req->in.h.opcode, req->in.h.unique, + req->in.h.nodeid, req->in.h.len, + fuse_len_args(req->args->out_numargs, req->args->out_args), + queue_id); + + fsvq = &fs->vqs[queue_id]; + ret = virtio_fs_enqueue_req(fsvq, req, false, GFP_ATOMIC); + if (ret < 0) { + if (ret == -ENOSPC) { + /* + * Virtqueue full. Retry submission from worker + * context as we might be holding fc->chan->bg_lock. + */ + spin_lock(&fsvq->lock); + list_add_tail(&req->list, &fsvq->queued_reqs); + inc_in_flight_req(fsvq); + spin_unlock(&fsvq->lock); + return; + } + req->out.h.error = ret; + pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n", ret); + + /* Can't end request in submission context. Use a worker */ + spin_lock(&fsvq->lock); + list_add_tail(&req->list, &fsvq->end_reqs); + schedule_work(&fsvq->dispatch_work); + spin_unlock(&fsvq->lock); + return; + } +} + +static const struct fuse_iqueue_ops virtio_fs_fiq_ops = { + .send_forget = virtio_fs_send_forget, + .send_interrupt = virtio_fs_send_interrupt, + .send_req = virtio_fs_send_req, + .release = virtio_fs_fiq_release, +}; + +static inline void virtio_fs_ctx_set_defaults(struct fuse_fs_context *ctx) +{ + ctx->rootmode = S_IFDIR; + ctx->default_permissions = 1; + ctx->allow_other = 1; + ctx->max_read = UINT_MAX; + ctx->blksize = 512; + ctx->destroy = true; + ctx->no_control = true; + ctx->no_force_umount = true; +} + +static int virtio_fs_fill_super(struct super_block *sb, struct fs_context *fsc) +{ + struct fuse_mount *fm = get_fuse_mount_super(sb); + struct fuse_conn *fc = fm->fc; + struct virtio_fs *fs = fc->chan->iq.priv; + struct fuse_fs_context *ctx = fsc->fs_private; + unsigned int i; + int err; + + virtio_fs_ctx_set_defaults(ctx); + mutex_lock(&virtio_fs_mutex); + + /* After holding mutex, make sure virtiofs device is still there. + * Though we are holding a reference to it, drive ->remove might + * still have cleaned up virtual queues. In that case bail out. + */ + err = -EINVAL; + if (list_empty(&fs->list)) { + pr_info("virtio-fs: tag <%s> not found\n", fs->tag); + goto err; + } + + err = -ENOMEM; + /* Allocate fuse_dev for hiprio and notification queues */ + for (i = 0; i < fs->nvqs; i++) { + struct virtio_fs_vq *fsvq = &fs->vqs[i]; + + fsvq->fud = fuse_dev_alloc(); + if (!fsvq->fud) + goto err_free_fuse_devs; + } + + if (ctx->dax_mode != FUSE_DAX_NEVER) { + if (ctx->dax_mode == FUSE_DAX_ALWAYS && !fs->dax_dev) { + err = -EINVAL; + pr_err("virtio-fs: dax can't be enabled as filesystem" + " device does not support it.\n"); + goto err_free_fuse_devs; + } + ctx->dax_dev = fs->dax_dev; + } + err = fuse_fill_super_common(sb, ctx); + if (err < 0) + goto err_free_fuse_devs; + + for (i = 0; i < fs->nvqs; i++) { + struct virtio_fs_vq *fsvq = &fs->vqs[i]; + + fuse_dev_install(fsvq->fud, fc->chan); + } + + /* Previous unmount will stop all queues. Start these again */ + virtio_fs_start_all_queues(fs); + fuse_send_init(fm); + mutex_unlock(&virtio_fs_mutex); + return 0; + +err_free_fuse_devs: + virtio_fs_free_devs(fs); +err: + mutex_unlock(&virtio_fs_mutex); + return err; +} + +static void virtio_fs_conn_destroy(struct fuse_mount *fm) +{ + struct fuse_conn *fc = fm->fc; + struct virtio_fs *vfs = fc->chan->iq.priv; + struct virtio_fs_vq *fsvq = &vfs->vqs[VQ_HIPRIO]; + + /* Stop dax worker. Soon evict_inodes() will be called which + * will free all memory ranges belonging to all inodes. + */ + if (IS_ENABLED(CONFIG_FUSE_DAX)) + fuse_dax_cancel_work(fc); + + /* Stop forget queue. Soon destroy will be sent */ + spin_lock(&fsvq->lock); + fsvq->connected = false; + spin_unlock(&fsvq->lock); + virtio_fs_drain_all_queues(vfs); + + fuse_conn_destroy(fm); + + /* fuse_conn_destroy() must have sent destroy. Stop all queues + * and drain one more time and free fuse devices. Freeing fuse + * devices will drop their reference on fuse_conn and that in + * turn will drop its reference on virtio_fs object. + */ + virtio_fs_stop_all_queues(vfs); + virtio_fs_drain_all_queues(vfs); + virtio_fs_free_devs(vfs); +} + +static void virtio_kill_sb(struct super_block *sb) +{ + struct fuse_mount *fm = get_fuse_mount_super(sb); + bool last; + + /* If mount failed, we can still be called without any fc */ + if (sb->s_root) { + last = fuse_mount_remove(fm); + if (last) + virtio_fs_conn_destroy(fm); + } + kill_anon_super(sb); + fuse_mount_destroy(fm); +} + +static int virtio_fs_test_super(struct super_block *sb, + struct fs_context *fsc) +{ + struct fuse_mount *fsc_fm = fsc->s_fs_info; + struct fuse_mount *sb_fm = get_fuse_mount_super(sb); + + return fsc_fm->fc->chan->iq.priv == sb_fm->fc->chan->iq.priv; +} + +static int virtio_fs_get_tree(struct fs_context *fsc) +{ + struct virtio_fs *fs; + struct super_block *sb; + struct fuse_conn *fc = NULL; + struct fuse_mount *fm; + unsigned int virtqueue_size; + struct fuse_chan *fch __free(fuse_chan_free) = fuse_chan_new(); + int err = -EIO; + + if (!fch) + return -ENOMEM; + + if (!fsc->source) + return invalf(fsc, "No source specified"); + + /* This gets a reference on virtio_fs object. This ptr gets installed + * in chan->iq->priv. Once fuse_conn is going away, it calls ->put() + * to drop the reference to this object. + */ + fs = virtio_fs_find_instance(fsc->source); + if (!fs) { + pr_info("virtio-fs: tag <%s> not found\n", fsc->source); + return -EINVAL; + } + + virtqueue_size = virtqueue_get_vring_size(fs->vqs[VQ_REQUEST].vq); + if (WARN_ON(virtqueue_size <= FUSE_HEADER_OVERHEAD)) + goto out_err; + + err = -ENOMEM; + fc = kzalloc_obj(struct fuse_conn); + if (!fc) + goto out_err; + + fm = kzalloc_obj(struct fuse_mount); + if (!fm) + goto out_err; + + fuse_iqueue_init(&fch->iq, &virtio_fs_fiq_ops, fs); + fuse_conn_init(fc, fm, fsc->user_ns, no_free_ptr(fch)); + + fc->release = fuse_free_conn; + fc->delete_stale = true; + fc->auto_submounts = true; + fc->sync_fs = true; + fc->use_pages_for_kvec_io = true; + + /* Tell FUSE to split requests that exceed the virtqueue's size */ + fc->max_pages_limit = min_t(unsigned int, fc->max_pages_limit, + virtqueue_size - FUSE_HEADER_OVERHEAD); + + fsc->s_fs_info = fm; + sb = sget_fc(fsc, virtio_fs_test_super, set_anon_super_fc); + if (fsc->s_fs_info) + fuse_mount_destroy(fm); + if (IS_ERR(sb)) + return PTR_ERR(sb); + + if (!sb->s_root) { + err = virtio_fs_fill_super(sb, fsc); + if (err) { + deactivate_locked_super(sb); + return err; + } + + sb->s_flags |= SB_ACTIVE; + } + + WARN_ON(fsc->root); + fsc->root = dget(sb->s_root); + return 0; + +out_err: + kfree(fc); + virtio_fs_put(fs); + return err; +} + +static const struct fs_context_operations virtio_fs_context_ops = { + .free = virtio_fs_free_fsc, + .parse_param = virtio_fs_parse_param, + .get_tree = virtio_fs_get_tree, +}; + +static int virtio_fs_init_fs_context(struct fs_context *fsc) +{ + struct fuse_fs_context *ctx; + + if (fsc->purpose == FS_CONTEXT_FOR_SUBMOUNT) + return fuse_init_fs_context_submount(fsc); + + ctx = kzalloc_obj(struct fuse_fs_context); + if (!ctx) + return -ENOMEM; + fsc->fs_private = ctx; + fsc->ops = &virtio_fs_context_ops; + return 0; +} + +static struct file_system_type virtio_fs_type = { + .owner = THIS_MODULE, + .name = "virtiofs", + .init_fs_context = virtio_fs_init_fs_context, + .kill_sb = virtio_kill_sb, + .fs_flags = FS_ALLOW_IDMAP, +}; + +static int virtio_fs_uevent(const struct kobject *kobj, struct kobj_uevent_env *env) +{ + const struct virtio_fs *fs = container_of(kobj, struct virtio_fs, kobj); + + add_uevent_var(env, "TAG=%s", fs->tag); + return 0; +} + +static const struct kset_uevent_ops virtio_fs_uevent_ops = { + .uevent = virtio_fs_uevent, +}; + +static int __init virtio_fs_sysfs_init(void) +{ + virtio_fs_kset = kset_create_and_add("virtiofs", &virtio_fs_uevent_ops, + fs_kobj); + if (!virtio_fs_kset) + return -ENOMEM; + return 0; +} + +static void virtio_fs_sysfs_exit(void) +{ + kset_unregister(virtio_fs_kset); + virtio_fs_kset = NULL; +} + +static int __init virtio_fs_init(void) +{ + int ret; + + ret = virtio_fs_sysfs_init(); + if (ret < 0) + return ret; + + ret = register_virtio_driver(&virtio_fs_driver); + if (ret < 0) + goto sysfs_exit; + + ret = register_filesystem(&virtio_fs_type); + if (ret < 0) + goto unregister_virtio_driver; + + return 0; + +unregister_virtio_driver: + unregister_virtio_driver(&virtio_fs_driver); +sysfs_exit: + virtio_fs_sysfs_exit(); + return ret; +} +module_init(virtio_fs_init); + +static void __exit virtio_fs_exit(void) +{ + unregister_filesystem(&virtio_fs_type); + unregister_virtio_driver(&virtio_fs_driver); + virtio_fs_sysfs_exit(); +} +module_exit(virtio_fs_exit); + +MODULE_AUTHOR("Stefan Hajnoczi "); +MODULE_DESCRIPTION("Virtio Filesystem"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS_FS(KBUILD_MODNAME); +MODULE_DEVICE_TABLE(virtio, id_table); diff --git a/fs/fuse/xattr.c b/fs/fuse/xattr.c new file mode 100644 index 000000000..cab2685ac --- /dev/null +++ b/fs/fuse/xattr.c @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * FUSE: Filesystem in Userspace + * Copyright (C) 2001-2016 Miklos Szeredi + */ + +#include "fuse_i.h" + +#include +#include + +int fuse_setxattr(struct inode *inode, const char *name, const void *value, + size_t size, int flags, unsigned int extra_flags) +{ + struct fuse_mount *fm = get_fuse_mount(inode); + FUSE_ARGS(args); + struct fuse_setxattr_in inarg; + int err; + + if (fm->fc->no_setxattr) + return -EOPNOTSUPP; + + memset(&inarg, 0, sizeof(inarg)); + inarg.size = size; + inarg.flags = flags; + inarg.setxattr_flags = extra_flags; + + args.opcode = FUSE_SETXATTR; + args.nodeid = get_node_id(inode); + args.in_numargs = 3; + args.in_args[0].size = fm->fc->setxattr_ext ? + sizeof(inarg) : FUSE_COMPAT_SETXATTR_IN_SIZE; + args.in_args[0].value = &inarg; + args.in_args[1].size = strlen(name) + 1; + args.in_args[1].value = name; + args.in_args[2].size = size; + args.in_args[2].value = value; + err = fuse_simple_request(fm, &args); + if (err == -ENOSYS) { + fm->fc->no_setxattr = 1; + err = -EOPNOTSUPP; + } + if (!err) + fuse_update_ctime(inode); + + return err; +} + +ssize_t fuse_getxattr(struct inode *inode, const char *name, void *value, + size_t size) +{ + struct fuse_mount *fm = get_fuse_mount(inode); + FUSE_ARGS(args); + struct fuse_getxattr_in inarg; + struct fuse_getxattr_out outarg; + ssize_t ret; + + if (fm->fc->no_getxattr) + return -EOPNOTSUPP; + + memset(&inarg, 0, sizeof(inarg)); + inarg.size = size; + args.opcode = FUSE_GETXATTR; + args.nodeid = get_node_id(inode); + args.in_numargs = 2; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + args.in_args[1].size = strlen(name) + 1; + args.in_args[1].value = name; + /* This is really two different operations rolled into one */ + args.out_numargs = 1; + if (size) { + args.out_argvar = true; + args.out_args[0].size = size; + args.out_args[0].value = value; + } else { + args.out_args[0].size = sizeof(outarg); + args.out_args[0].value = &outarg; + } + ret = fuse_simple_request(fm, &args); + if (!ret && !size) + ret = min_t(size_t, outarg.size, XATTR_SIZE_MAX); + if (ret == -ENOSYS) { + fm->fc->no_getxattr = 1; + ret = -EOPNOTSUPP; + } + return ret; +} + +static int fuse_verify_xattr_list(char *list, size_t size) +{ + size_t origsize = size; + + while (size) { + size_t thislen = strnlen(list, size); + + if (!thislen || thislen == size) + return -EIO; + + size -= thislen + 1; + list += thislen + 1; + } + + return origsize; +} + +ssize_t fuse_listxattr(struct dentry *entry, char *list, size_t size) +{ + struct inode *inode = d_inode(entry); + struct fuse_mount *fm = get_fuse_mount(inode); + FUSE_ARGS(args); + struct fuse_getxattr_in inarg; + struct fuse_getxattr_out outarg; + ssize_t ret; + + if (fuse_is_bad(inode)) + return -EIO; + + if (!fuse_allow_current_process(fm->fc)) + return -EACCES; + + if (fm->fc->no_listxattr) + return -EOPNOTSUPP; + + memset(&inarg, 0, sizeof(inarg)); + inarg.size = size; + args.opcode = FUSE_LISTXATTR; + args.nodeid = get_node_id(inode); + args.in_numargs = 1; + args.in_args[0].size = sizeof(inarg); + args.in_args[0].value = &inarg; + /* This is really two different operations rolled into one */ + args.out_numargs = 1; + if (size) { + args.out_argvar = true; + args.out_args[0].size = size; + args.out_args[0].value = list; + } else { + args.out_args[0].size = sizeof(outarg); + args.out_args[0].value = &outarg; + } + ret = fuse_simple_request(fm, &args); + if (!ret && !size) + ret = min_t(size_t, outarg.size, XATTR_LIST_MAX); + if (ret > 0 && size) + ret = fuse_verify_xattr_list(list, ret); + if (ret == -ENOSYS) { + fm->fc->no_listxattr = 1; + ret = -EOPNOTSUPP; + } + return ret; +} + +int fuse_removexattr(struct inode *inode, const char *name) +{ + struct fuse_mount *fm = get_fuse_mount(inode); + FUSE_ARGS(args); + int err; + + if (fm->fc->no_removexattr) + return -EOPNOTSUPP; + + args.opcode = FUSE_REMOVEXATTR; + args.nodeid = get_node_id(inode); + args.in_numargs = 2; + fuse_set_zero_arg0(&args); + args.in_args[1].size = strlen(name) + 1; + args.in_args[1].value = name; + err = fuse_simple_request(fm, &args); + if (err == -ENOSYS) { + fm->fc->no_removexattr = 1; + err = -EOPNOTSUPP; + } + if (!err) + fuse_update_ctime(inode); + + return err; +} + +static int fuse_xattr_get(const struct xattr_handler *handler, + struct dentry *dentry, struct inode *inode, + const char *name, void *value, size_t size) +{ + if (fuse_is_bad(inode)) + return -EIO; + + return fuse_getxattr(inode, name, value, size); +} + +static int fuse_xattr_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *dentry, struct inode *inode, + const char *name, const void *value, size_t size, + int flags) +{ + if (fuse_is_bad(inode)) + return -EIO; + + if (!value) + return fuse_removexattr(inode, name); + + return fuse_setxattr(inode, name, value, size, flags, 0); +} + +static const struct xattr_handler fuse_xattr_handler = { + .prefix = "", + .get = fuse_xattr_get, + .set = fuse_xattr_set, +}; + +const struct xattr_handler * const fuse_xattr_handlers[] = { + &fuse_xattr_handler, + NULL +}; diff --git a/fs/gfs2/Kconfig b/fs/gfs2/Kconfig new file mode 100644 index 000000000..8beee571b --- /dev/null +++ b/fs/gfs2/Kconfig @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: GPL-2.0-only +config GFS2_FS + tristate "GFS2 file system support" + select BUFFER_HEAD + select FS_POSIX_ACL + select CRC32 + select QUOTACTL + select FS_IOMAP + help + A cluster filesystem. + + Allows a cluster of computers to simultaneously use a block device + that is shared between them (with FC, iSCSI, NBD, etc...). GFS reads + and writes to the block device like a local filesystem, but also uses + a lock module to allow the computers coordinate their I/O so + filesystem consistency is maintained. One of the nifty features of + GFS is perfect consistency -- changes made to the filesystem on one + machine show up immediately on all other machines in the cluster. + + To use the GFS2 filesystem in a cluster, you will need to enable + the locking module below. Documentation and utilities for GFS2 can + be found here: http://sources.redhat.com/cluster + + The "nolock" lock module is now built in to GFS2 by default. If + you want to use the DLM, be sure to enable IPv4/6 networking. + +config GFS2_FS_LOCKING_DLM + bool "GFS2 DLM locking" + depends on (GFS2_FS!=n) && NET && INET && \ + CONFIGFS_FS && SYSFS && (DLM=y || DLM=GFS2_FS) + help + Multiple node locking module for GFS2 + + Most users of GFS2 will require this. It provides the locking + interface between GFS2 and the DLM, which is required to use GFS2 + in a cluster environment. diff --git a/fs/gfs2/Makefile b/fs/gfs2/Makefile new file mode 100644 index 000000000..41b2aa4bc --- /dev/null +++ b/fs/gfs2/Makefile @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0 +ccflags-y := -I$(src) +obj-$(CONFIG_GFS2_FS) += gfs2.o +gfs2-y := acl.o bmap.o dir.o xattr.o glock.o \ + glops.o log.o lops.o main.o meta_io.o \ + aops.o dentry.o export.o file.o \ + ops_fstype.o inode.o quota.o \ + recovery.o rgrp.o super.o sys.o trans.o util.o + +gfs2-$(CONFIG_GFS2_FS_LOCKING_DLM) += lock_dlm.o + diff --git a/fs/gfs2/acl.c b/fs/gfs2/acl.c new file mode 100644 index 000000000..a5b60778b --- /dev/null +++ b/fs/gfs2/acl.c @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gfs2.h" +#include "incore.h" +#include "acl.h" +#include "xattr.h" +#include "glock.h" +#include "inode.h" +#include "meta_io.h" +#include "quota.h" +#include "rgrp.h" +#include "trans.h" +#include "util.h" + +static const char *gfs2_acl_name(int type) +{ + switch (type) { + case ACL_TYPE_ACCESS: + return XATTR_POSIX_ACL_ACCESS; + case ACL_TYPE_DEFAULT: + return XATTR_POSIX_ACL_DEFAULT; + } + return NULL; +} + +static struct posix_acl *__gfs2_get_acl(struct inode *inode, int type) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct posix_acl *acl; + const char *name; + char *data; + int len; + + if (!ip->i_eattr) + return NULL; + + name = gfs2_acl_name(type); + len = gfs2_xattr_acl_get(ip, name, &data); + if (len <= 0) + return ERR_PTR(len); + acl = posix_acl_from_xattr(&init_user_ns, data, len); + kfree(data); + return acl; +} + +struct posix_acl *gfs2_get_acl(struct inode *inode, int type, bool rcu) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_holder gh; + bool need_unlock = false; + struct posix_acl *acl; + + if (rcu) + return ERR_PTR(-ECHILD); + + if (!gfs2_glock_is_locked_by_me(ip->i_gl)) { + int ret = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, + LM_FLAG_ANY, &gh); + if (ret) + return ERR_PTR(ret); + need_unlock = true; + } + acl = __gfs2_get_acl(inode, type); + if (need_unlock) + gfs2_glock_dq_uninit(&gh); + return acl; +} + +int __gfs2_set_acl(struct inode *inode, struct posix_acl *acl, int type) +{ + int error; + size_t len = 0; + char *data = NULL; + const char *name = gfs2_acl_name(type); + + if (acl) { + data = posix_acl_to_xattr(&init_user_ns, acl, &len, GFP_NOFS); + if (data == NULL) + return -ENOMEM; + } + + error = __gfs2_xattr_set(inode, name, data, len, 0, GFS2_EATYPE_SYS); + if (error) + goto out; + set_cached_acl(inode, type, acl); +out: + kfree(data); + return error; +} + +int gfs2_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type) +{ + struct inode *inode = d_inode(dentry); + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_holder gh; + bool need_unlock = false; + int ret; + umode_t mode; + + if (acl && acl->a_count > GFS2_ACL_MAX_ENTRIES(GFS2_SB(inode))) + return -E2BIG; + + ret = gfs2_qa_get(ip); + if (ret) + return ret; + + if (!gfs2_glock_is_locked_by_me(ip->i_gl)) { + ret = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &gh); + if (ret) + goto out; + need_unlock = true; + } + + mode = inode->i_mode; + if (type == ACL_TYPE_ACCESS && acl) { + ret = posix_acl_update_mode(&nop_mnt_idmap, inode, &mode, &acl); + if (ret) + goto unlock; + } + + ret = __gfs2_set_acl(inode, acl, type); + if (!ret && mode != inode->i_mode) { + inode_set_ctime_current(inode); + inode->i_mode = mode; + mark_inode_dirty(inode); + } +unlock: + if (need_unlock) + gfs2_glock_dq_uninit(&gh); +out: + gfs2_qa_put(ip); + return ret; +} diff --git a/fs/gfs2/acl.h b/fs/gfs2/acl.h new file mode 100644 index 000000000..82f5b09c0 --- /dev/null +++ b/fs/gfs2/acl.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#ifndef __ACL_DOT_H__ +#define __ACL_DOT_H__ + +#include "incore.h" + +#define GFS2_ACL_MAX_ENTRIES(sdp) ((300 << (sdp)->sd_sb.sb_bsize_shift) >> 12) + +struct posix_acl *gfs2_get_acl(struct inode *inode, int type, bool rcu); +int __gfs2_set_acl(struct inode *inode, struct posix_acl *acl, int type); +int gfs2_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type); + +#endif /* __ACL_DOT_H__ */ diff --git a/fs/gfs2/aops.c b/fs/gfs2/aops.c new file mode 100644 index 000000000..0a7b8076a --- /dev/null +++ b/fs/gfs2/aops.c @@ -0,0 +1,743 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gfs2.h" +#include "incore.h" +#include "bmap.h" +#include "glock.h" +#include "inode.h" +#include "log.h" +#include "meta_io.h" +#include "quota.h" +#include "trans.h" +#include "rgrp.h" +#include "super.h" +#include "util.h" +#include "glops.h" +#include "aops.h" + + +/** + * gfs2_get_block_noalloc - Fills in a buffer head with details about a block + * @inode: The inode + * @lblock: The block number to look up + * @bh_result: The buffer head to return the result in + * @create: Non-zero if we may add block to the file + * + * Returns: errno + */ + +static int gfs2_get_block_noalloc(struct inode *inode, sector_t lblock, + struct buffer_head *bh_result, int create) +{ + int error; + + error = gfs2_block_map(inode, lblock, bh_result, 0); + if (error) + return error; + if (!buffer_mapped(bh_result)) + return -ENODATA; + return 0; +} + +/** + * gfs2_write_jdata_folio - gfs2 jdata-specific version of block_write_full_folio + * @folio: The folio to write + * @wbc: The writeback control + * + * This is the same as calling block_write_full_folio, but it also + * writes pages outside of i_size + */ +static int gfs2_write_jdata_folio(struct folio *folio, + struct writeback_control *wbc) +{ + struct inode * const inode = folio->mapping->host; + loff_t i_size = i_size_read(inode); + + /* + * The folio straddles i_size. It must be zeroed out on each and every + * writepage invocation because it may be mmapped. "A file is mapped + * in multiples of the page size. For a file that is not a multiple of + * the page size, the remaining memory is zeroed when mapped, and + * writes to that region are not written out to the file." + */ + if (folio_pos(folio) < i_size && i_size < folio_next_pos(folio)) + folio_zero_segment(folio, offset_in_folio(folio, i_size), + folio_size(folio)); + + return __block_write_full_folio(inode, folio, gfs2_get_block_noalloc, + wbc); +} + +/** + * __gfs2_jdata_write_folio - The core of jdata writepage + * @folio: The folio to write + * @wbc: The writeback control + * + * Implements the core of write back. If a transaction is required then + * the checked flag will have been set and the transaction will have + * already been started before this is called. + */ +static int __gfs2_jdata_write_folio(struct folio *folio, + struct writeback_control *wbc) +{ + struct inode *inode = folio->mapping->host; + struct gfs2_inode *ip = GFS2_I(inode); + + if (folio_test_checked(folio)) { + folio_clear_checked(folio); + if (!folio_buffers(folio)) { + create_empty_buffers(folio, + inode->i_sb->s_blocksize, + BIT(BH_Dirty)|BIT(BH_Uptodate)); + } + gfs2_trans_add_databufs(ip->i_gl, folio, 0, folio_size(folio)); + } + return gfs2_write_jdata_folio(folio, wbc); +} + +/** + * gfs2_jdata_writeback - Write jdata folios to the log + * @mapping: The mapping to write + * @wbc: The writeback control + * + * Returns: errno + */ +int gfs2_jdata_writeback(struct address_space *mapping, struct writeback_control *wbc) +{ + struct inode *inode = mapping->host; + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(mapping->host); + struct folio *folio = NULL; + int error; + + BUG_ON(current->journal_info); + if (gfs2_assert_withdraw(sdp, ip->i_gl->gl_state == LM_ST_EXCLUSIVE)) + return 0; + + while ((folio = writeback_iter(mapping, wbc, folio, &error))) { + if (folio_test_checked(folio)) { + folio_redirty_for_writepage(wbc, folio); + folio_unlock(folio); + continue; + } + error = __gfs2_jdata_write_folio(folio, wbc); + } + + return error; +} + +/** + * gfs2_writepages - Write a bunch of dirty pages back to disk + * @mapping: The mapping to write + * @wbc: Write-back control + * + * Used for both ordered and writeback modes. + */ +static int gfs2_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct gfs2_sbd *sdp = gfs2_mapping2sbd(mapping); + long initial_nr_to_write = wbc->nr_to_write; + struct iomap_writepage_ctx wpc = { + .inode = mapping->host, + .wbc = wbc, + .ops = &gfs2_writeback_ops, + }; + int ret; + + /* + * Even if we didn't write any pages here, we might still be holding + * dirty pages in the ail. We forcibly flush the ail because we don't + * want balance_dirty_pages() to loop indefinitely trying to write out + * pages held in the ail that it can't find. + */ + ret = iomap_writepages(&wpc); + if (ret == 0 && wbc->nr_to_write == initial_nr_to_write) + set_bit(SDF_FORCE_AIL_FLUSH, &sdp->sd_flags); + return ret; +} + +/** + * gfs2_write_jdata_batch - Write back a folio batch's worth of folios + * @mapping: The mapping + * @wbc: The writeback control + * @fbatch: The batch of folios + * @done_index: Page index + * + * Returns: non-zero if loop should terminate, zero otherwise + */ + +static int gfs2_write_jdata_batch(struct address_space *mapping, + struct writeback_control *wbc, + struct folio_batch *fbatch, + pgoff_t *done_index) +{ + struct inode *inode = mapping->host; + struct gfs2_sbd *sdp = GFS2_SB(inode); + unsigned nrblocks; + int i; + int ret; + size_t size = 0; + int nr_folios = folio_batch_count(fbatch); + + for (i = 0; i < nr_folios; i++) + size += folio_size(fbatch->folios[i]); + nrblocks = size >> inode->i_blkbits; + + ret = gfs2_trans_begin(sdp, nrblocks, nrblocks); + if (ret < 0) + return ret; + + for (i = 0; i < nr_folios; i++) { + struct folio *folio = fbatch->folios[i]; + + *done_index = folio->index; + + folio_lock(folio); + + if (unlikely(folio->mapping != mapping)) { +continue_unlock: + folio_unlock(folio); + continue; + } + + if (!folio_test_dirty(folio)) { + /* someone wrote it for us */ + goto continue_unlock; + } + + if (folio_test_writeback(folio)) { + if (wbc->sync_mode != WB_SYNC_NONE) + folio_wait_writeback(folio); + else + goto continue_unlock; + } + + BUG_ON(folio_test_writeback(folio)); + if (!folio_clear_dirty_for_io(folio)) + goto continue_unlock; + + trace_wbc_writepage(wbc, inode_to_bdi(inode)); + + ret = __gfs2_jdata_write_folio(folio, wbc); + if (unlikely(ret)) { + /* + * done_index is set past this page, so media errors + * will not choke background writeout for the entire + * file. This has consequences for range_cyclic + * semantics (ie. it may not be suitable for data + * integrity writeout). + */ + *done_index = folio_next_index(folio); + ret = 1; + break; + } + + /* + * We stop writing back only if we are not doing + * integrity sync. In case of integrity sync we have to + * keep going until we have written all the pages + * we tagged for writeback prior to entering this loop. + */ + if (--wbc->nr_to_write <= 0 && wbc->sync_mode == WB_SYNC_NONE) { + ret = 1; + break; + } + + } + gfs2_trans_end(sdp); + return ret; +} + +/** + * gfs2_write_cache_jdata - Like write_cache_pages but different + * @mapping: The mapping to write + * @wbc: The writeback control + * + * The reason that we use our own function here is that we need to + * start transactions before we grab page locks. This allows us + * to get the ordering right. + */ + +static int gfs2_write_cache_jdata(struct address_space *mapping, + struct writeback_control *wbc) +{ + int ret = 0; + int done = 0; + struct folio_batch fbatch; + int nr_folios; + pgoff_t writeback_index; + pgoff_t index; + pgoff_t end; + pgoff_t done_index; + int cycled; + int range_whole = 0; + xa_mark_t tag; + + folio_batch_init(&fbatch); + if (wbc->range_cyclic) { + writeback_index = mapping->writeback_index; /* prev offset */ + index = writeback_index; + if (index == 0) + cycled = 1; + else + cycled = 0; + end = -1; + } else { + index = wbc->range_start >> PAGE_SHIFT; + end = wbc->range_end >> PAGE_SHIFT; + if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) + range_whole = 1; + cycled = 1; /* ignore range_cyclic tests */ + } + tag = wbc_to_tag(wbc); + +retry: + if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) + tag_pages_for_writeback(mapping, index, end); + done_index = index; + while (!done && (index <= end)) { + nr_folios = filemap_get_folios_tag(mapping, &index, end, + tag, &fbatch); + if (nr_folios == 0) + break; + + ret = gfs2_write_jdata_batch(mapping, wbc, &fbatch, + &done_index); + if (ret) + done = 1; + if (ret > 0) + ret = 0; + folio_batch_release(&fbatch); + cond_resched(); + } + + if (!cycled && !done) { + /* + * range_cyclic: + * We hit the last page and there is more work to be done: wrap + * back to the start of the file + */ + cycled = 1; + index = 0; + end = writeback_index - 1; + goto retry; + } + + if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) + mapping->writeback_index = done_index; + + return ret; +} + + +/** + * gfs2_jdata_writepages - Write a bunch of dirty pages back to disk + * @mapping: The mapping to write + * @wbc: The writeback control + * + */ + +static int gfs2_jdata_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct gfs2_inode *ip = GFS2_I(mapping->host); + struct gfs2_sbd *sdp = GFS2_SB(mapping->host); + int ret; + + ret = gfs2_write_cache_jdata(mapping, wbc); + if (ret == 0 && wbc->sync_mode == WB_SYNC_ALL) { + gfs2_log_flush(sdp, ip->i_gl, GFS2_LOG_HEAD_FLUSH_NORMAL | + GFS2_LFC_JDATA_WPAGES); + ret = gfs2_write_cache_jdata(mapping, wbc); + } + return ret; +} + +/** + * stuffed_read_folio - Fill in a Linux folio with stuffed file data + * @ip: the inode + * @folio: the folio + * + * Returns: errno + */ +static int stuffed_read_folio(struct gfs2_inode *ip, struct folio *folio) +{ + struct buffer_head *dibh = NULL; + size_t dsize = i_size_read(&ip->i_inode); + void *from = NULL; + int error = 0; + + /* + * Due to the order of unstuffing files and ->fault(), we can be + * asked for a zero folio in the case of a stuffed file being extended, + * so we need to supply one here. It doesn't happen often. + */ + if (unlikely(folio->index)) { + dsize = 0; + } else { + error = gfs2_meta_inode_buffer(ip, &dibh); + if (error) + goto out; + from = dibh->b_data + sizeof(struct gfs2_dinode); + } + + folio_fill_tail(folio, 0, from, dsize); + brelse(dibh); +out: + folio_end_read(folio, error == 0); + + return error; +} + +/** + * gfs2_read_folio - read a folio from a file + * @file: The file to read + * @folio: The folio in the file + */ +static int gfs2_read_folio(struct file *file, struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + int error = 0; + + if (!gfs2_is_jdata(ip) || + (i_blocksize(inode) == PAGE_SIZE && !folio_buffers(folio))) { + iomap_bio_read_folio(folio, &gfs2_iomap_ops); + } else if (gfs2_is_stuffed(ip)) { + error = stuffed_read_folio(ip, folio); + } else { + error = mpage_read_folio(folio, gfs2_block_map); + } + + if (gfs2_withdrawn(sdp)) + return -EIO; + + return error; +} + +/** + * gfs2_internal_read - read an internal file + * @ip: The gfs2 inode + * @buf: The buffer to fill + * @pos: The file position + * @size: The amount to read + * + */ + +ssize_t gfs2_internal_read(struct gfs2_inode *ip, char *buf, loff_t *pos, + size_t size) +{ + struct address_space *mapping = ip->i_inode.i_mapping; + unsigned long index = *pos >> PAGE_SHIFT; + size_t copied = 0; + + do { + size_t offset, chunk; + struct folio *folio; + + folio = read_cache_folio(mapping, index, gfs2_read_folio, NULL); + if (IS_ERR(folio)) { + if (PTR_ERR(folio) == -EINTR) + continue; + return PTR_ERR(folio); + } + offset = *pos + copied - folio_pos(folio); + chunk = min(size - copied, folio_size(folio) - offset); + memcpy_from_folio(buf + copied, folio, offset, chunk); + index = folio_next_index(folio); + folio_put(folio); + copied += chunk; + } while(copied < size); + (*pos) += size; + return size; +} + +/** + * gfs2_readahead - Read a bunch of pages at once + * @rac: Read-ahead control structure + * + * Some notes: + * 1. This is only for readahead, so we can simply ignore any things + * which are slightly inconvenient (such as locking conflicts between + * the page lock and the glock) and return having done no I/O. Its + * obviously not something we'd want to do on too regular a basis. + * Any I/O we ignore at this time will be done via readpage later. + * 2. We don't handle stuffed files here we let readpage do the honours. + * 3. mpage_readahead() does most of the heavy lifting in the common case. + * 4. gfs2_block_map() is relied upon to set BH_Boundary in the right places. + */ + +static void gfs2_readahead(struct readahead_control *rac) +{ + struct inode *inode = rac->mapping->host; + struct gfs2_inode *ip = GFS2_I(inode); + + if (gfs2_is_stuffed(ip)) + ; + else if (gfs2_is_jdata(ip)) + mpage_readahead(rac, gfs2_block_map); + else + iomap_bio_readahead(rac, &gfs2_iomap_ops); +} + +/** + * adjust_fs_space - Adjusts the free space available due to gfs2_grow + * @inode: the rindex inode + */ +void adjust_fs_space(struct inode *inode) +{ + struct gfs2_sbd *sdp = GFS2_SB(inode); + struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode); + struct gfs2_statfs_change_host *m_sc = &sdp->sd_statfs_master; + struct gfs2_statfs_change_host *l_sc = &sdp->sd_statfs_local; + struct buffer_head *m_bh; + u64 fs_total, new_free; + + if (gfs2_trans_begin(sdp, 2 * RES_STATFS, 0) != 0) + return; + + /* Total up the file system space, according to the latest rindex. */ + fs_total = gfs2_ri_total(sdp); + if (gfs2_meta_inode_buffer(m_ip, &m_bh) != 0) + goto out; + + spin_lock(&sdp->sd_statfs_spin); + gfs2_statfs_change_in(m_sc, m_bh->b_data + + sizeof(struct gfs2_dinode)); + if (fs_total > (m_sc->sc_total + l_sc->sc_total)) + new_free = fs_total - (m_sc->sc_total + l_sc->sc_total); + else + new_free = 0; + spin_unlock(&sdp->sd_statfs_spin); + fs_warn(sdp, "File system extended by %llu blocks.\n", + (unsigned long long)new_free); + gfs2_statfs_change(sdp, new_free, new_free, 0); + + update_statfs(sdp, m_bh); + brelse(m_bh); +out: + sdp->sd_rindex_uptodate = 0; + gfs2_trans_end(sdp); +} + +static bool gfs2_jdata_dirty_folio(struct address_space *mapping, + struct folio *folio) +{ + if (current->journal_info) + folio_set_checked(folio); + return block_dirty_folio(mapping, folio); +} + +/** + * gfs2_bmap - Block map function + * @mapping: Address space info + * @lblock: The block to map + * + * Returns: The disk address for the block or 0 on hole or error + */ + +static sector_t gfs2_bmap(struct address_space *mapping, sector_t lblock) +{ + struct gfs2_inode *ip = GFS2_I(mapping->host); + struct gfs2_holder i_gh; + sector_t dblock = 0; + int error; + + error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, LM_FLAG_ANY, &i_gh); + if (error) + return 0; + + if (!gfs2_is_stuffed(ip)) + dblock = iomap_bmap(mapping, lblock, &gfs2_iomap_ops); + + gfs2_glock_dq_uninit(&i_gh); + + return dblock; +} + +static void gfs2_discard(struct gfs2_sbd *sdp, struct buffer_head *bh) +{ + struct gfs2_bufdata *bd; + + lock_buffer(bh); + spin_lock(&sdp->sd_log_lock); + clear_buffer_dirty(bh); + bd = bh->b_private; + if (bd) { + if (!list_empty(&bd->bd_list) && !buffer_pinned(bh)) + list_del_init(&bd->bd_list); + else { + spin_lock(&sdp->sd_ail_lock); + gfs2_remove_from_journal(bh, REMOVE_JDATA); + spin_unlock(&sdp->sd_ail_lock); + } + } + bh->b_bdev = NULL; + clear_buffer_mapped(bh); + clear_buffer_req(bh); + clear_buffer_new(bh); + spin_unlock(&sdp->sd_log_lock); + unlock_buffer(bh); +} + +static void gfs2_invalidate_folio(struct folio *folio, size_t offset, + size_t length) +{ + struct gfs2_sbd *sdp = GFS2_SB(folio->mapping->host); + size_t stop = offset + length; + int partial_page = (offset || length < folio_size(folio)); + struct buffer_head *bh, *head; + unsigned long pos = 0; + + BUG_ON(!folio_test_locked(folio)); + if (!partial_page) + folio_clear_checked(folio); + head = folio_buffers(folio); + if (!head) + goto out; + + bh = head; + do { + if (pos + bh->b_size > stop) + return; + + if (offset <= pos) + gfs2_discard(sdp, bh); + pos += bh->b_size; + bh = bh->b_this_page; + } while (bh != head); +out: + if (!partial_page) + filemap_release_folio(folio, 0); +} + +/** + * gfs2_release_folio - free the metadata associated with a folio + * @folio: the folio that's being released + * @gfp_mask: passed from Linux VFS, ignored by us + * + * Calls try_to_free_buffers() to free the buffers and put the folio if the + * buffers can be released. + * + * Returns: true if the folio was put or else false + */ + +bool gfs2_release_folio(struct folio *folio, gfp_t gfp_mask) +{ + struct address_space *mapping = folio->mapping; + struct gfs2_sbd *sdp = gfs2_mapping2sbd(mapping); + struct buffer_head *bh, *head; + struct gfs2_bufdata *bd; + + head = folio_buffers(folio); + if (!head) + return false; + + /* + * mm accommodates an old ext3 case where clean folios might + * not have had the dirty bit cleared. Thus, it can send actual + * dirty folios to ->release_folio() via shrink_active_list(). + * + * As a workaround, we skip folios that contain dirty buffers + * below. Once ->release_folio isn't called on dirty folios + * anymore, we can warn on dirty buffers like we used to here + * again. + */ + + spin_lock(&sdp->sd_log_lock); + bh = head; + do { + if (atomic_read(&bh->b_count)) + goto cannot_release; + bd = bh->b_private; + if (bd && bd->bd_tr) + goto cannot_release; + if (buffer_dirty(bh) || WARN_ON(buffer_pinned(bh))) + goto cannot_release; + bh = bh->b_this_page; + } while (bh != head); + + bh = head; + do { + bd = bh->b_private; + if (bd) { + gfs2_assert_warn(sdp, bd->bd_bh == bh); + bd->bd_bh = NULL; + bh->b_private = NULL; + /* + * The bd may still be queued as a revoke, in which + * case we must not dequeue nor free it. + */ + if (!bd->bd_blkno && !list_empty(&bd->bd_list)) + list_del_init(&bd->bd_list); + if (list_empty(&bd->bd_list)) + kmem_cache_free(gfs2_bufdata_cachep, bd); + } + + bh = bh->b_this_page; + } while (bh != head); + spin_unlock(&sdp->sd_log_lock); + + return try_to_free_buffers(folio); + +cannot_release: + spin_unlock(&sdp->sd_log_lock); + return false; +} + +static const struct address_space_operations gfs2_aops = { + .writepages = gfs2_writepages, + .read_folio = gfs2_read_folio, + .readahead = gfs2_readahead, + .dirty_folio = iomap_dirty_folio, + .release_folio = iomap_release_folio, + .invalidate_folio = iomap_invalidate_folio, + .bmap = gfs2_bmap, + .migrate_folio = filemap_migrate_folio, + .is_partially_uptodate = iomap_is_partially_uptodate, + .error_remove_folio = generic_error_remove_folio, +}; + +static const struct address_space_operations gfs2_jdata_aops = { + .writepages = gfs2_jdata_writepages, + .read_folio = gfs2_read_folio, + .readahead = gfs2_readahead, + .dirty_folio = gfs2_jdata_dirty_folio, + .bmap = gfs2_bmap, + .migrate_folio = buffer_migrate_folio, + .invalidate_folio = gfs2_invalidate_folio, + .release_folio = gfs2_release_folio, + .is_partially_uptodate = block_is_partially_uptodate, + .error_remove_folio = generic_error_remove_folio, +}; + +void gfs2_set_aops(struct inode *inode) +{ + if (gfs2_is_jdata(GFS2_I(inode))) + inode->i_mapping->a_ops = &gfs2_jdata_aops; + else + inode->i_mapping->a_ops = &gfs2_aops; +} diff --git a/fs/gfs2/aops.h b/fs/gfs2/aops.h new file mode 100644 index 000000000..bf002522a --- /dev/null +++ b/fs/gfs2/aops.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2018 Red Hat, Inc. All rights reserved. + */ + +#ifndef __AOPS_DOT_H__ +#define __AOPS_DOT_H__ + +#include "incore.h" + +void adjust_fs_space(struct inode *inode); +int gfs2_jdata_writeback(struct address_space *mapping, struct writeback_control *wbc); + +#endif /* __AOPS_DOT_H__ */ diff --git a/fs/gfs2/bmap.c b/fs/gfs2/bmap.c new file mode 100644 index 000000000..73c626971 --- /dev/null +++ b/fs/gfs2/bmap.c @@ -0,0 +1,2530 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gfs2.h" +#include "incore.h" +#include "bmap.h" +#include "glock.h" +#include "inode.h" +#include "meta_io.h" +#include "quota.h" +#include "rgrp.h" +#include "log.h" +#include "super.h" +#include "trans.h" +#include "dir.h" +#include "util.h" +#include "aops.h" +#include "trace_gfs2.h" + +/* This doesn't need to be that large as max 64 bit pointers in a 4k + * block is 512, so __u16 is fine for that. It saves stack space to + * keep it small. + */ +struct metapath { + struct buffer_head *mp_bh[GFS2_MAX_META_HEIGHT]; + __u16 mp_list[GFS2_MAX_META_HEIGHT]; + int mp_fheight; /* find_metapath height */ + int mp_aheight; /* actual height (lookup height) */ +}; + +static int punch_hole(struct gfs2_inode *ip, u64 offset, u64 length); + +/** + * gfs2_unstuffer_folio - unstuff a stuffed inode into a block cached by a folio + * @ip: the inode + * @dibh: the dinode buffer + * @block: the block number that was allocated + * @folio: The folio. + * + * Returns: errno + */ +static int gfs2_unstuffer_folio(struct gfs2_inode *ip, struct buffer_head *dibh, + u64 block, struct folio *folio) +{ + struct inode *inode = &ip->i_inode; + + if (!folio_test_uptodate(folio)) { + void *kaddr = kmap_local_folio(folio, 0); + u64 dsize = i_size_read(inode); + + memcpy(kaddr, dibh->b_data + sizeof(struct gfs2_dinode), dsize); + memset(kaddr + dsize, 0, folio_size(folio) - dsize); + kunmap_local(kaddr); + + folio_mark_uptodate(folio); + } + + if (gfs2_is_jdata(ip)) { + struct buffer_head *bh = folio_buffers(folio); + + if (!bh) + bh = create_empty_buffers(folio, + BIT(inode->i_blkbits), BIT(BH_Uptodate)); + + if (!buffer_mapped(bh)) + map_bh(bh, inode->i_sb, block); + + set_buffer_uptodate(bh); + gfs2_trans_add_data(ip->i_gl, bh); + } else { + folio_mark_dirty(folio); + gfs2_ordered_add_inode(ip); + } + + return 0; +} + +static int __gfs2_unstuff_inode(struct gfs2_inode *ip, struct folio *folio) +{ + struct buffer_head *bh, *dibh; + struct gfs2_dinode *di; + u64 block = 0; + int isdir = gfs2_is_dir(ip); + int error; + + error = gfs2_meta_inode_buffer(ip, &dibh); + if (error) + return error; + + if (i_size_read(&ip->i_inode)) { + /* Get a free block, fill it with the stuffed data, + and write it out to disk */ + + unsigned int n = 1; + error = gfs2_alloc_blocks(ip, &block, &n, 0); + if (error) + goto out_brelse; + if (isdir) { + gfs2_trans_remove_revoke(GFS2_SB(&ip->i_inode), block, 1); + error = gfs2_dir_get_new_buffer(ip, block, &bh); + if (error) + goto out_brelse; + gfs2_buffer_copy_tail(bh, sizeof(struct gfs2_meta_header), + dibh, sizeof(struct gfs2_dinode)); + brelse(bh); + } else { + error = gfs2_unstuffer_folio(ip, dibh, block, folio); + if (error) + goto out_brelse; + } + } + + /* Set up the pointer to the new block */ + + gfs2_trans_add_meta(ip->i_gl, dibh); + di = (struct gfs2_dinode *)dibh->b_data; + gfs2_buffer_clear_tail(dibh, sizeof(struct gfs2_dinode)); + + if (i_size_read(&ip->i_inode)) { + *(__be64 *)(di + 1) = cpu_to_be64(block); + gfs2_add_inode_blocks(&ip->i_inode, 1); + di->di_blocks = cpu_to_be64(gfs2_get_inode_blocks(&ip->i_inode)); + } + + ip->i_height = 1; + di->di_height = cpu_to_be16(1); + +out_brelse: + brelse(dibh); + return error; +} + +/** + * gfs2_unstuff_dinode - Unstuff a dinode when the data has grown too big + * @ip: The GFS2 inode to unstuff + * + * This routine unstuffs a dinode and returns it to a "normal" state such + * that the height can be grown in the traditional way. + * + * Returns: errno + */ + +int gfs2_unstuff_dinode(struct gfs2_inode *ip) +{ + struct inode *inode = &ip->i_inode; + struct folio *folio; + int error; + + down_write(&ip->i_rw_mutex); + folio = filemap_grab_folio(inode->i_mapping, 0); + error = PTR_ERR(folio); + if (IS_ERR(folio)) + goto out; + error = __gfs2_unstuff_inode(ip, folio); + folio_unlock(folio); + folio_put(folio); +out: + up_write(&ip->i_rw_mutex); + return error; +} + +/** + * find_metapath - Find path through the metadata tree + * @sdp: The superblock + * @block: The disk block to look up + * @mp: The metapath to return the result in + * @height: The pre-calculated height of the metadata tree + * + * This routine returns a struct metapath structure that defines a path + * through the metadata of inode "ip" to get to block "block". + * + * Example: + * Given: "ip" is a height 3 file, "offset" is 101342453, and this is a + * filesystem with a blocksize of 4096. + * + * find_metapath() would return a struct metapath structure set to: + * mp_fheight = 3, mp_list[0] = 0, mp_list[1] = 48, and mp_list[2] = 165. + * + * That means that in order to get to the block containing the byte at + * offset 101342453, we would load the indirect block pointed to by pointer + * 0 in the dinode. We would then load the indirect block pointed to by + * pointer 48 in that indirect block. We would then load the data block + * pointed to by pointer 165 in that indirect block. + * + * ---------------------------------------- + * | Dinode | | + * | | 4| + * | |0 1 2 3 4 5 9| + * | | 6| + * ---------------------------------------- + * | + * | + * V + * ---------------------------------------- + * | Indirect Block | + * | 5| + * | 4 4 4 4 4 5 5 1| + * |0 5 6 7 8 9 0 1 2| + * ---------------------------------------- + * | + * | + * V + * ---------------------------------------- + * | Indirect Block | + * | 1 1 1 1 1 5| + * | 6 6 6 6 6 1| + * |0 3 4 5 6 7 2| + * ---------------------------------------- + * | + * | + * V + * ---------------------------------------- + * | Data block containing offset | + * | 101342453 | + * | | + * | | + * ---------------------------------------- + * + */ + +static void find_metapath(const struct gfs2_sbd *sdp, u64 block, + struct metapath *mp, unsigned int height) +{ + unsigned int i; + + mp->mp_fheight = height; + for (i = height; i--;) + mp->mp_list[i] = do_div(block, sdp->sd_inptrs); +} + +static inline unsigned int metapath_branch_start(const struct metapath *mp) +{ + if (mp->mp_list[0] == 0) + return 2; + return 1; +} + +/** + * metaptr1 - Return the first possible metadata pointer in a metapath buffer + * @height: The metadata height (0 = dinode) + * @mp: The metapath + */ +static inline __be64 *metaptr1(unsigned int height, const struct metapath *mp) +{ + struct buffer_head *bh = mp->mp_bh[height]; + if (height == 0) + return ((__be64 *)(bh->b_data + sizeof(struct gfs2_dinode))); + return ((__be64 *)(bh->b_data + sizeof(struct gfs2_meta_header))); +} + +/** + * metapointer - Return pointer to start of metadata in a buffer + * @height: The metadata height (0 = dinode) + * @mp: The metapath + * + * Return a pointer to the block number of the next height of the metadata + * tree given a buffer containing the pointer to the current height of the + * metadata tree. + */ + +static inline __be64 *metapointer(unsigned int height, const struct metapath *mp) +{ + __be64 *p = metaptr1(height, mp); + return p + mp->mp_list[height]; +} + +static inline const __be64 *metaend(unsigned int height, const struct metapath *mp) +{ + const struct buffer_head *bh = mp->mp_bh[height]; + return (const __be64 *)(bh->b_data + bh->b_size); +} + +static void clone_metapath(struct metapath *clone, struct metapath *mp) +{ + unsigned int hgt; + + *clone = *mp; + for (hgt = 0; hgt < mp->mp_aheight; hgt++) + get_bh(clone->mp_bh[hgt]); +} + +static void gfs2_metapath_ra(struct gfs2_glock *gl, __be64 *start, __be64 *end) +{ + const __be64 *t; + + for (t = start; t < end; t++) { + struct buffer_head *rabh; + + if (!*t) + continue; + + rabh = gfs2_getbuf(gl, be64_to_cpu(*t), CREATE); + if (trylock_buffer(rabh)) { + if (!buffer_uptodate(rabh)) { + bh_submit(rabh, + REQ_OP_READ | REQ_RAHEAD | REQ_META | + REQ_PRIO, + bh_end_read); + } else { + unlock_buffer(rabh); + } + } + put_bh(rabh); + } +} + +static inline struct buffer_head * +metapath_dibh(struct metapath *mp) +{ + return mp->mp_bh[0]; +} + +static int __fillup_metapath(struct gfs2_inode *ip, struct metapath *mp, + unsigned int x, unsigned int h) +{ + for (; x < h; x++) { + __be64 *ptr = metapointer(x, mp); + u64 dblock = be64_to_cpu(*ptr); + int ret; + + if (!dblock) + break; + ret = gfs2_meta_buffer(ip, GFS2_METATYPE_IN, dblock, &mp->mp_bh[x + 1]); + if (ret) + return ret; + } + mp->mp_aheight = x + 1; + return 0; +} + +/** + * lookup_metapath - Walk the metadata tree to a specific point + * @ip: The inode + * @mp: The metapath + * + * Assumes that the inode's buffer has already been looked up and + * hooked onto mp->mp_bh[0] and that the metapath has been initialised + * by find_metapath(). + * + * If this function encounters part of the tree which has not been + * allocated, it returns the current height of the tree at the point + * at which it found the unallocated block. Blocks which are found are + * added to the mp->mp_bh[] list. + * + * Returns: error + */ + +static int lookup_metapath(struct gfs2_inode *ip, struct metapath *mp) +{ + return __fillup_metapath(ip, mp, 0, ip->i_height - 1); +} + +/** + * fillup_metapath - fill up buffers for the metadata path to a specific height + * @ip: The inode + * @mp: The metapath + * @h: The height to which it should be mapped + * + * Similar to lookup_metapath, but does lookups for a range of heights + * + * Returns: error or the number of buffers filled + */ + +static int fillup_metapath(struct gfs2_inode *ip, struct metapath *mp, int h) +{ + unsigned int x = 0; + int ret; + + if (h) { + /* find the first buffer we need to look up. */ + for (x = h - 1; x > 0; x--) { + if (mp->mp_bh[x]) + break; + } + } + ret = __fillup_metapath(ip, mp, x, h); + if (ret) + return ret; + return mp->mp_aheight - x - 1; +} + +static sector_t metapath_to_block(struct gfs2_sbd *sdp, struct metapath *mp) +{ + sector_t factor = 1, block = 0; + int hgt; + + for (hgt = mp->mp_fheight - 1; hgt >= 0; hgt--) { + if (hgt < mp->mp_aheight) + block += mp->mp_list[hgt] * factor; + factor *= sdp->sd_inptrs; + } + return block; +} + +static void release_metapath(struct metapath *mp) +{ + int i; + + for (i = 0; i < GFS2_MAX_META_HEIGHT; i++) { + if (mp->mp_bh[i] == NULL) + break; + brelse(mp->mp_bh[i]); + mp->mp_bh[i] = NULL; + } +} + +/** + * gfs2_extent_length - Returns length of an extent of blocks + * @bh: The metadata block + * @ptr: Current position in @bh + * @eob: Set to 1 if we hit "end of block" + * + * Returns: The length of the extent (minimum of one block) + */ + +static inline unsigned int gfs2_extent_length(struct buffer_head *bh, __be64 *ptr, int *eob) +{ + const __be64 *end = (__be64 *)(bh->b_data + bh->b_size); + const __be64 *first = ptr; + u64 d = be64_to_cpu(*ptr); + + *eob = 0; + do { + ptr++; + if (ptr >= end) + break; + d++; + } while(be64_to_cpu(*ptr) == d); + if (ptr >= end) + *eob = 1; + return ptr - first; +} + +enum walker_status { WALK_STOP, WALK_FOLLOW, WALK_CONTINUE }; + +/* + * gfs2_metadata_walker - walk an indirect block + * @mp: Metapath to indirect block + * @ptrs: Number of pointers to look at + * + * When returning WALK_FOLLOW, the walker must update @mp to point at the right + * indirect block to follow. + */ +typedef enum walker_status (*gfs2_metadata_walker)(struct metapath *mp, + unsigned int ptrs); + +/* + * gfs2_walk_metadata - walk a tree of indirect blocks + * @inode: The inode + * @mp: Starting point of walk + * @max_len: Maximum number of blocks to walk + * @walker: Called during the walk + * + * Returns 1 if the walk was stopped by @walker, 0 if we went past @max_len or + * past the end of metadata, and a negative error code otherwise. + */ + +static int gfs2_walk_metadata(struct inode *inode, struct metapath *mp, + u64 max_len, gfs2_metadata_walker walker) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + u64 factor = 1; + unsigned int hgt; + int ret; + + /* + * The walk starts in the lowest allocated indirect block, which may be + * before the position indicated by @mp. Adjust @max_len accordingly + * to avoid a short walk. + */ + for (hgt = mp->mp_fheight - 1; hgt >= mp->mp_aheight; hgt--) { + max_len += mp->mp_list[hgt] * factor; + mp->mp_list[hgt] = 0; + factor *= sdp->sd_inptrs; + } + + for (;;) { + u16 start = mp->mp_list[hgt]; + enum walker_status status; + unsigned int ptrs; + u64 len; + + /* Walk indirect block. */ + ptrs = (hgt >= 1 ? sdp->sd_inptrs : sdp->sd_diptrs) - start; + len = ptrs * factor; + if (len > max_len) + ptrs = DIV_ROUND_UP_ULL(max_len, factor); + status = walker(mp, ptrs); + switch (status) { + case WALK_STOP: + return 1; + case WALK_FOLLOW: + BUG_ON(mp->mp_aheight == mp->mp_fheight); + ptrs = mp->mp_list[hgt] - start; + len = ptrs * factor; + break; + case WALK_CONTINUE: + break; + } + if (len >= max_len) + break; + max_len -= len; + if (status == WALK_FOLLOW) + goto fill_up_metapath; + +lower_metapath: + /* Decrease height of metapath. */ + brelse(mp->mp_bh[hgt]); + mp->mp_bh[hgt] = NULL; + mp->mp_list[hgt] = 0; + if (!hgt) + break; + hgt--; + factor *= sdp->sd_inptrs; + + /* Advance in metadata tree. */ + (mp->mp_list[hgt])++; + if (hgt) { + if (mp->mp_list[hgt] >= sdp->sd_inptrs) + goto lower_metapath; + } else { + if (mp->mp_list[hgt] >= sdp->sd_diptrs) + break; + } + +fill_up_metapath: + /* Increase height of metapath. */ + ret = fillup_metapath(ip, mp, ip->i_height - 1); + if (ret < 0) + return ret; + hgt += ret; + for (; ret; ret--) + do_div(factor, sdp->sd_inptrs); + mp->mp_aheight = hgt + 1; + } + return 0; +} + +static enum walker_status gfs2_hole_walker(struct metapath *mp, + unsigned int ptrs) +{ + const __be64 *start, *ptr, *end; + unsigned int hgt; + + hgt = mp->mp_aheight - 1; + start = metapointer(hgt, mp); + end = start + ptrs; + + for (ptr = start; ptr < end; ptr++) { + if (*ptr) { + mp->mp_list[hgt] += ptr - start; + if (mp->mp_aheight == mp->mp_fheight) + return WALK_STOP; + return WALK_FOLLOW; + } + } + return WALK_CONTINUE; +} + +/** + * gfs2_hole_size - figure out the size of a hole + * @inode: The inode + * @lblock: The logical starting block number + * @len: How far to look (in blocks) + * @mp: The metapath at lblock + * @iomap: The iomap to store the hole size in + * + * This function modifies @mp. + * + * Returns: errno on error + */ +static int gfs2_hole_size(struct inode *inode, sector_t lblock, u64 len, + struct metapath *mp, struct iomap *iomap) +{ + struct metapath clone; + u64 hole_size; + int ret; + + clone_metapath(&clone, mp); + ret = gfs2_walk_metadata(inode, &clone, len, gfs2_hole_walker); + if (ret < 0) + goto out; + + if (ret == 1) + hole_size = metapath_to_block(GFS2_SB(inode), &clone) - lblock; + else + hole_size = len; + iomap->length = hole_size << inode->i_blkbits; + ret = 0; + +out: + release_metapath(&clone); + return ret; +} + +static inline void gfs2_indirect_init(struct metapath *mp, + struct gfs2_glock *gl, unsigned int i, + unsigned offset, u64 bn) +{ + __be64 *ptr = (__be64 *)(mp->mp_bh[i - 1]->b_data + + ((i > 1) ? sizeof(struct gfs2_meta_header) : + sizeof(struct gfs2_dinode))); + BUG_ON(i < 1); + BUG_ON(mp->mp_bh[i] != NULL); + mp->mp_bh[i] = gfs2_meta_new(gl, bn); + gfs2_trans_add_meta(gl, mp->mp_bh[i]); + gfs2_metatype_set(mp->mp_bh[i], GFS2_METATYPE_IN, GFS2_FORMAT_IN); + gfs2_buffer_clear_tail(mp->mp_bh[i], sizeof(struct gfs2_meta_header)); + ptr += offset; + *ptr = cpu_to_be64(bn); +} + +enum alloc_state { + ALLOC_DATA = 0, + ALLOC_GROW_DEPTH = 1, + ALLOC_GROW_HEIGHT = 2, + /* ALLOC_UNSTUFF = 3, TBD and rather complicated */ +}; + +/** + * __gfs2_iomap_alloc - Build a metadata tree of the requested height + * @inode: The GFS2 inode + * @iomap: The iomap structure + * @mp: The metapath, with proper height information calculated + * + * In this routine we may have to alloc: + * i) Indirect blocks to grow the metadata tree height + * ii) Indirect blocks to fill in lower part of the metadata tree + * iii) Data blocks + * + * This function is called after __gfs2_iomap_get, which works out the + * total number of blocks which we need via gfs2_alloc_size. + * + * We then do the actual allocation asking for an extent at a time (if + * enough contiguous free blocks are available, there will only be one + * allocation request per call) and uses the state machine to initialise + * the blocks in order. + * + * Right now, this function will allocate at most one indirect block + * worth of data -- with a default block size of 4K, that's slightly + * less than 2M. If this limitation is ever removed to allow huge + * allocations, we would probably still want to limit the iomap size we + * return to avoid stalling other tasks during huge writes; the next + * iomap iteration would then find the blocks already allocated. + * + * Returns: errno on error + */ + +static int __gfs2_iomap_alloc(struct inode *inode, struct iomap *iomap, + struct metapath *mp) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + struct buffer_head *dibh = metapath_dibh(mp); + u64 bn; + unsigned n, i, blks, alloced = 0, iblks = 0, branch_start = 0; + size_t dblks = iomap->length >> inode->i_blkbits; + const unsigned end_of_metadata = mp->mp_fheight - 1; + int ret; + enum alloc_state state; + __be64 *ptr; + __be64 zero_bn = 0; + + BUG_ON(mp->mp_aheight < 1); + BUG_ON(dibh == NULL); + BUG_ON(dblks < 1); + + gfs2_trans_add_meta(ip->i_gl, dibh); + + down_write(&ip->i_rw_mutex); + + if (mp->mp_fheight == mp->mp_aheight) { + /* Bottom indirect block exists */ + state = ALLOC_DATA; + } else { + /* Need to allocate indirect blocks */ + if (mp->mp_fheight == ip->i_height) { + /* Writing into existing tree, extend tree down */ + iblks = mp->mp_fheight - mp->mp_aheight; + state = ALLOC_GROW_DEPTH; + } else { + /* Building up tree height */ + state = ALLOC_GROW_HEIGHT; + iblks = mp->mp_fheight - ip->i_height; + branch_start = metapath_branch_start(mp); + iblks += (mp->mp_fheight - branch_start); + } + } + + /* start of the second part of the function (state machine) */ + + blks = dblks + iblks; + i = mp->mp_aheight; + do { + n = blks - alloced; + ret = gfs2_alloc_blocks(ip, &bn, &n, 0); + if (ret) + goto out; + alloced += n; + if (state != ALLOC_DATA || gfs2_is_jdata(ip)) + gfs2_trans_remove_revoke(sdp, bn, n); + switch (state) { + /* Growing height of tree */ + case ALLOC_GROW_HEIGHT: + if (i == 1) { + ptr = (__be64 *)(dibh->b_data + + sizeof(struct gfs2_dinode)); + zero_bn = *ptr; + } + for (; i - 1 < mp->mp_fheight - ip->i_height && n > 0; + i++, n--) + gfs2_indirect_init(mp, ip->i_gl, i, 0, bn++); + if (i - 1 == mp->mp_fheight - ip->i_height) { + i--; + gfs2_buffer_copy_tail(mp->mp_bh[i], + sizeof(struct gfs2_meta_header), + dibh, sizeof(struct gfs2_dinode)); + gfs2_buffer_clear_tail(dibh, + sizeof(struct gfs2_dinode) + + sizeof(__be64)); + ptr = (__be64 *)(mp->mp_bh[i]->b_data + + sizeof(struct gfs2_meta_header)); + *ptr = zero_bn; + state = ALLOC_GROW_DEPTH; + for(i = branch_start; i < mp->mp_fheight; i++) { + if (mp->mp_bh[i] == NULL) + break; + brelse(mp->mp_bh[i]); + mp->mp_bh[i] = NULL; + } + i = branch_start; + } + if (n == 0) + break; + fallthrough; /* To branching from existing tree */ + case ALLOC_GROW_DEPTH: + if (i > 1 && i < mp->mp_fheight) + gfs2_trans_add_meta(ip->i_gl, mp->mp_bh[i-1]); + for (; i < mp->mp_fheight && n > 0; i++, n--) + gfs2_indirect_init(mp, ip->i_gl, i, + mp->mp_list[i-1], bn++); + if (i == mp->mp_fheight) + state = ALLOC_DATA; + if (n == 0) + break; + fallthrough; /* To tree complete, adding data blocks */ + case ALLOC_DATA: + BUG_ON(n > dblks); + BUG_ON(mp->mp_bh[end_of_metadata] == NULL); + gfs2_trans_add_meta(ip->i_gl, mp->mp_bh[end_of_metadata]); + dblks = n; + ptr = metapointer(end_of_metadata, mp); + iomap->addr = bn << inode->i_blkbits; + iomap->flags |= IOMAP_F_MERGED | IOMAP_F_NEW; + while (n-- > 0) + *ptr++ = cpu_to_be64(bn++); + break; + } + } while (iomap->addr == IOMAP_NULL_ADDR); + + iomap->type = IOMAP_MAPPED; + iomap->length = (u64)dblks << inode->i_blkbits; + ip->i_height = mp->mp_fheight; + gfs2_add_inode_blocks(&ip->i_inode, alloced); + gfs2_dinode_out(ip, dibh->b_data); +out: + up_write(&ip->i_rw_mutex); + return ret; +} + +#define IOMAP_F_GFS2_BOUNDARY IOMAP_F_PRIVATE + +/** + * gfs2_alloc_size - Compute the maximum allocation size + * @inode: The inode + * @mp: The metapath + * @size: Requested size in blocks + * + * Compute the maximum size of the next allocation at @mp. + * + * Returns: size in blocks + */ +static u64 gfs2_alloc_size(struct inode *inode, struct metapath *mp, u64 size) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + const __be64 *first, *ptr, *end; + + /* + * For writes to stuffed files, this function is called twice via + * __gfs2_iomap_get, before and after unstuffing. The size we return the + * first time needs to be large enough to get the reservation and + * allocation sizes right. The size we return the second time must + * be exact or else __gfs2_iomap_alloc won't do the right thing. + */ + + if (gfs2_is_stuffed(ip) || mp->mp_fheight != mp->mp_aheight) { + unsigned int maxsize = mp->mp_fheight > 1 ? + sdp->sd_inptrs : sdp->sd_diptrs; + maxsize -= mp->mp_list[mp->mp_fheight - 1]; + if (size > maxsize) + size = maxsize; + return size; + } + + first = metapointer(ip->i_height - 1, mp); + end = metaend(ip->i_height - 1, mp); + if (end - first > size) + end = first + size; + for (ptr = first; ptr < end; ptr++) { + if (*ptr) + break; + } + return ptr - first; +} + +/** + * __gfs2_iomap_get - Map blocks from an inode to disk blocks + * @inode: The inode + * @pos: Starting position in bytes + * @length: Length to map, in bytes + * @flags: iomap flags + * @iomap: The iomap structure + * @mp: The metapath + * + * Returns: errno + */ +static int __gfs2_iomap_get(struct inode *inode, loff_t pos, loff_t length, + unsigned flags, struct iomap *iomap, + struct metapath *mp) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + loff_t size = i_size_read(inode); + __be64 *ptr; + sector_t lblock; + sector_t lblock_stop; + int ret; + int eob; + u64 len; + struct buffer_head *dibh = NULL, *bh; + u8 height; + + if (!length) + return -EINVAL; + + down_read(&ip->i_rw_mutex); + + ret = gfs2_meta_inode_buffer(ip, &dibh); + if (ret) + goto unlock; + mp->mp_bh[0] = dibh; + + if (gfs2_is_stuffed(ip)) { + if (flags & IOMAP_WRITE) { + loff_t max_size = gfs2_max_stuffed_size(ip); + + if (pos + length > max_size) + goto unstuff; + iomap->length = max_size; + } else { + if (pos >= size) { + if (flags & IOMAP_REPORT) { + ret = -ENOENT; + goto unlock; + } else { + iomap->offset = pos; + iomap->length = length; + goto hole_found; + } + } + iomap->length = size; + } + iomap->addr = (ip->i_no_addr << inode->i_blkbits) + + sizeof(struct gfs2_dinode); + iomap->type = IOMAP_INLINE; + iomap->inline_data = dibh->b_data + sizeof(struct gfs2_dinode); + goto out; + } + +unstuff: + lblock = pos >> inode->i_blkbits; + iomap->offset = lblock << inode->i_blkbits; + lblock_stop = (pos + length - 1) >> inode->i_blkbits; + len = lblock_stop - lblock + 1; + iomap->length = len << inode->i_blkbits; + + height = ip->i_height; + while ((lblock + 1) * sdp->sd_sb.sb_bsize > sdp->sd_heightsize[height]) + height++; + find_metapath(sdp, lblock, mp, height); + if (height > ip->i_height || gfs2_is_stuffed(ip)) + goto do_alloc; + + ret = lookup_metapath(ip, mp); + if (ret) + goto unlock; + + if (mp->mp_aheight != ip->i_height) + goto do_alloc; + + ptr = metapointer(ip->i_height - 1, mp); + if (*ptr == 0) + goto do_alloc; + + bh = mp->mp_bh[ip->i_height - 1]; + len = gfs2_extent_length(bh, ptr, &eob); + + iomap->addr = be64_to_cpu(*ptr) << inode->i_blkbits; + iomap->length = len << inode->i_blkbits; + iomap->type = IOMAP_MAPPED; + iomap->flags |= IOMAP_F_MERGED; + if (eob) + iomap->flags |= IOMAP_F_GFS2_BOUNDARY; + +out: + iomap->bdev = inode->i_sb->s_bdev; +unlock: + up_read(&ip->i_rw_mutex); + return ret; + +do_alloc: + if (flags & IOMAP_REPORT) { + if (pos >= size) + ret = -ENOENT; + else if (height == ip->i_height) + ret = gfs2_hole_size(inode, lblock, len, mp, iomap); + else + iomap->length = size - iomap->offset; + } else if (flags & IOMAP_WRITE) { + u64 alloc_size; + + if (flags & IOMAP_DIRECT) + goto out; /* (see gfs2_file_direct_write) */ + + len = gfs2_alloc_size(inode, mp, len); + alloc_size = len << inode->i_blkbits; + if (alloc_size < iomap->length) + iomap->length = alloc_size; + } else { + if (pos < size && height == ip->i_height) + ret = gfs2_hole_size(inode, lblock, len, mp, iomap); + } +hole_found: + iomap->addr = IOMAP_NULL_ADDR; + iomap->type = IOMAP_HOLE; + goto out; +} + +static struct folio * +gfs2_iomap_get_folio(struct iomap_iter *iter, loff_t pos, unsigned len) +{ + struct inode *inode = iter->inode; + struct gfs2_inode *ip = GFS2_I(inode); + unsigned int blockmask = i_blocksize(inode) - 1; + struct gfs2_sbd *sdp = GFS2_SB(inode); + unsigned int blocks; + struct folio *folio; + int status; + + if (!gfs2_is_jdata(ip) && !gfs2_is_stuffed(ip)) + return iomap_get_folio(iter, pos, len); + + blocks = ((pos & blockmask) + len + blockmask) >> inode->i_blkbits; + status = gfs2_trans_begin(sdp, RES_DINODE + blocks, 0); + if (status) + return ERR_PTR(status); + + folio = iomap_get_folio(iter, pos, len); + if (IS_ERR(folio)) + gfs2_trans_end(sdp); + return folio; +} + +static void gfs2_iomap_put_folio(struct inode *inode, loff_t pos, + unsigned copied, struct folio *folio) +{ + struct gfs2_trans *tr = current->journal_info; + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + + if (gfs2_is_jdata(ip) && !gfs2_is_stuffed(ip)) + gfs2_trans_add_databufs(ip->i_gl, folio, + offset_in_folio(folio, pos), + copied); + + folio_unlock(folio); + folio_put(folio); + + if (gfs2_is_jdata(ip) || gfs2_is_stuffed(ip)) { + if (tr->tr_num_buf_new) + __mark_inode_dirty(inode, I_DIRTY_DATASYNC); + gfs2_trans_end(sdp); + } +} + +const struct iomap_write_ops gfs2_iomap_write_ops = { + .get_folio = gfs2_iomap_get_folio, + .put_folio = gfs2_iomap_put_folio, +}; + +static int gfs2_iomap_begin_write(struct inode *inode, loff_t pos, + loff_t length, unsigned flags, + struct iomap *iomap, + struct metapath *mp) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + bool unstuff; + int ret; + + unstuff = gfs2_is_stuffed(ip) && + pos + length > gfs2_max_stuffed_size(ip); + + if (unstuff || iomap->type == IOMAP_HOLE) { + unsigned int data_blocks, ind_blocks; + struct gfs2_alloc_parms ap = {}; + unsigned int rblocks; + struct gfs2_trans *tr; + + gfs2_write_calc_reserv(ip, iomap->length, &data_blocks, + &ind_blocks); + ap.target = data_blocks + ind_blocks; + ret = gfs2_quota_lock_check(ip, &ap); + if (ret) + return ret; + + ret = gfs2_inplace_reserve(ip, &ap); + if (ret) + goto out_qunlock; + + rblocks = RES_DINODE + ind_blocks; + if (gfs2_is_jdata(ip)) + rblocks += data_blocks; + if (ind_blocks || data_blocks) + rblocks += RES_STATFS + RES_QUOTA; + if (inode == sdp->sd_rindex) + rblocks += 2 * RES_STATFS; + rblocks += gfs2_rg_blocks(ip, data_blocks + ind_blocks); + + ret = gfs2_trans_begin(sdp, rblocks, + iomap->length >> inode->i_blkbits); + if (ret) + goto out_trans_fail; + + if (unstuff) { + ret = gfs2_unstuff_dinode(ip); + if (ret) + goto out_trans_end; + release_metapath(mp); + ret = __gfs2_iomap_get(inode, iomap->offset, + iomap->length, flags, iomap, mp); + if (ret) + goto out_trans_end; + } + + if (iomap->type == IOMAP_HOLE) { + ret = __gfs2_iomap_alloc(inode, iomap, mp); + if (ret) { + gfs2_trans_end(sdp); + gfs2_inplace_release(ip); + punch_hole(ip, iomap->offset, iomap->length); + goto out_qunlock; + } + } + + tr = current->journal_info; + if (tr->tr_num_buf_new) + __mark_inode_dirty(inode, I_DIRTY_DATASYNC); + + gfs2_trans_end(sdp); + } + + return 0; + +out_trans_end: + gfs2_trans_end(sdp); +out_trans_fail: + gfs2_inplace_release(ip); +out_qunlock: + gfs2_quota_unlock(ip); + return ret; +} + +static int gfs2_iomap_begin(struct inode *inode, loff_t pos, loff_t length, + unsigned flags, struct iomap *iomap, + struct iomap *srcmap) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct metapath mp = { .mp_aheight = 1, }; + int ret; + + if (gfs2_is_jdata(ip)) + iomap->flags |= IOMAP_F_BUFFER_HEAD; + + trace_gfs2_iomap_start(ip, pos, length, flags); + ret = __gfs2_iomap_get(inode, pos, length, flags, iomap, &mp); + if (ret) + goto out_unlock; + + switch(flags & (IOMAP_WRITE | IOMAP_ZERO)) { + case IOMAP_WRITE: + if (flags & IOMAP_DIRECT) { + /* + * Silently fall back to buffered I/O for stuffed files + * or if we've got a hole (see gfs2_file_direct_write). + */ + if (iomap->type != IOMAP_MAPPED) + ret = -ENOTBLK; + goto out_unlock; + } + break; + case IOMAP_ZERO: + if (iomap->type == IOMAP_HOLE) + goto out_unlock; + break; + default: + goto out; + } + + ret = gfs2_iomap_begin_write(inode, pos, length, flags, iomap, &mp); + if (ret) + goto out_unlock; + +out: + if (iomap->type == IOMAP_INLINE) { + iomap->private = metapath_dibh(&mp); + get_bh(iomap->private); + } + +out_unlock: + release_metapath(&mp); + trace_gfs2_iomap_end(ip, iomap, ret); + return ret; +} + +static int gfs2_iomap_end(struct inode *inode, loff_t pos, loff_t length, + ssize_t written, unsigned flags, struct iomap *iomap) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + + if (iomap->private) + brelse(iomap->private); + + switch (flags & (IOMAP_WRITE | IOMAP_ZERO)) { + case IOMAP_WRITE: + if (flags & IOMAP_DIRECT) + return 0; + break; + case IOMAP_ZERO: + if (iomap->type == IOMAP_HOLE) + return 0; + break; + default: + return 0; + } + + if (!gfs2_is_stuffed(ip)) + gfs2_ordered_add_inode(ip); + + if (inode == sdp->sd_rindex) + adjust_fs_space(inode); + + gfs2_inplace_release(ip); + + if (ip->i_qadata && ip->i_qadata->qa_qd_num) + gfs2_quota_unlock(ip); + + if (length != written && (iomap->flags & IOMAP_F_NEW)) { + /* Deallocate blocks that were just allocated. */ + loff_t hstart = round_up(pos + written, i_blocksize(inode)); + loff_t hend = iomap->offset + iomap->length; + + if (hstart < hend) { + truncate_pagecache_range(inode, hstart, hend - 1); + punch_hole(ip, hstart, hend - hstart); + } + } + + if (unlikely(!written)) + return 0; + + if (iomap->flags & IOMAP_F_SIZE_CHANGED) + mark_inode_dirty(inode); + set_bit(GLF_DIRTY, &ip->i_gl->gl_flags); + return 0; +} + +static DEFINE_IOMAP_ITER_NEXT_END(gfs2_iomap_next, gfs2_iomap_begin, + gfs2_iomap_end); + +const struct iomap_ops gfs2_iomap_ops = { + .iomap_next = gfs2_iomap_next, +}; + +/** + * gfs2_block_map - Map one or more blocks of an inode to a disk block + * @inode: The inode + * @lblock: The logical block number + * @bh_map: The bh to be mapped + * @create: True if its ok to alloc blocks to satify the request + * + * The size of the requested mapping is defined in bh_map->b_size. + * + * Clears buffer_mapped(bh_map) and leaves bh_map->b_size unchanged + * when @lblock is not mapped. Sets buffer_mapped(bh_map) and + * bh_map->b_size to indicate the size of the mapping when @lblock and + * successive blocks are mapped, up to the requested size. + * + * Sets buffer_boundary() if a read of metadata will be required + * before the next block can be mapped. Sets buffer_new() if new + * blocks were allocated. + * + * Returns: errno + */ + +int gfs2_block_map(struct inode *inode, sector_t lblock, + struct buffer_head *bh_map, int create) +{ + struct gfs2_inode *ip = GFS2_I(inode); + loff_t pos = (loff_t)lblock << inode->i_blkbits; + loff_t length = bh_map->b_size; + struct iomap iomap = { }; + int ret; + + clear_buffer_mapped(bh_map); + clear_buffer_new(bh_map); + clear_buffer_boundary(bh_map); + trace_gfs2_bmap(ip, bh_map, lblock, create, 1); + + if (!create) + ret = gfs2_iomap_get(inode, pos, length, &iomap); + else + ret = gfs2_iomap_alloc(inode, pos, length, &iomap); + if (ret) + goto out; + + if (iomap.length > bh_map->b_size) { + iomap.length = bh_map->b_size; + iomap.flags &= ~IOMAP_F_GFS2_BOUNDARY; + } + if (iomap.addr != IOMAP_NULL_ADDR) + map_bh(bh_map, inode->i_sb, iomap.addr >> inode->i_blkbits); + bh_map->b_size = iomap.length; + if (iomap.flags & IOMAP_F_GFS2_BOUNDARY) + set_buffer_boundary(bh_map); + if (iomap.flags & IOMAP_F_NEW) + set_buffer_new(bh_map); + +out: + trace_gfs2_bmap(ip, bh_map, lblock, create, ret); + return ret; +} + +int gfs2_get_extent(struct inode *inode, u64 lblock, u64 *dblock, + unsigned int *extlen) +{ + unsigned int blkbits = inode->i_blkbits; + struct iomap iomap = { }; + unsigned int len; + int ret; + + ret = gfs2_iomap_get(inode, lblock << blkbits, *extlen << blkbits, + &iomap); + if (ret) + return ret; + if (iomap.type != IOMAP_MAPPED) + return -EIO; + *dblock = iomap.addr >> blkbits; + len = iomap.length >> blkbits; + if (len < *extlen) + *extlen = len; + return 0; +} + +int gfs2_alloc_extent(struct inode *inode, u64 lblock, u64 *dblock, + unsigned int *extlen, bool *new) +{ + unsigned int blkbits = inode->i_blkbits; + struct iomap iomap = { }; + unsigned int len; + int ret; + + ret = gfs2_iomap_alloc(inode, lblock << blkbits, *extlen << blkbits, + &iomap); + if (ret) + return ret; + if (iomap.type != IOMAP_MAPPED) + return -EIO; + *dblock = iomap.addr >> blkbits; + len = iomap.length >> blkbits; + if (len < *extlen) + *extlen = len; + *new = iomap.flags & IOMAP_F_NEW; + return 0; +} + +/* + * NOTE: Never call gfs2_block_zero_range with an open transaction because it + * uses iomap write to perform its actions, which begin their own transactions + * (iomap_begin, get_folio, etc.) + */ +static int gfs2_block_zero_range(struct inode *inode, loff_t from, loff_t length) +{ + BUG_ON(current->journal_info); + if (from >= inode->i_size) + return 0; + length = min(length, inode->i_size - from); + return iomap_zero_range(inode, from, length, NULL, &gfs2_iomap_ops, + &gfs2_iomap_write_ops, NULL); +} + +int gfs2_clear_beyond_eof(struct inode *inode, loff_t end) +{ + loff_t isize = i_size_read(inode); + unsigned int len = isize & ~PAGE_MASK; + + if (!len || isize >= end) + return 0; + len = PAGE_SIZE - len; + if (end - isize < len) + len = end - isize; + return gfs2_block_zero_range(inode, isize, len); +} + +#define GFS2_JTRUNC_REVOKES 8192 + +/** + * gfs2_journaled_truncate - Wrapper for truncate_pagecache for jdata files + * @inode: The inode being truncated + * @oldsize: The original (larger) size + * @newsize: The new smaller size + * + * With jdata files, we have to journal a revoke for each block which is + * truncated. As a result, we need to split this into separate transactions + * if the number of pages being truncated gets too large. + */ + +static int gfs2_journaled_truncate(struct inode *inode, u64 oldsize, u64 newsize) +{ + struct gfs2_sbd *sdp = GFS2_SB(inode); + u64 max_chunk = GFS2_JTRUNC_REVOKES * sdp->sd_vfs->s_blocksize; + u64 chunk; + int error; + + while (oldsize != newsize) { + struct gfs2_trans *tr; + unsigned int offs; + + chunk = oldsize - newsize; + if (chunk > max_chunk) + chunk = max_chunk; + + offs = oldsize & ~PAGE_MASK; + if (offs && chunk > PAGE_SIZE) + chunk = offs + ((chunk - offs) & PAGE_MASK); + + truncate_pagecache(inode, oldsize - chunk); + oldsize -= chunk; + + tr = current->journal_info; + if (!test_bit(TR_TOUCHED, &tr->tr_flags)) + continue; + + gfs2_trans_end(sdp); + error = gfs2_trans_begin(sdp, RES_DINODE, GFS2_JTRUNC_REVOKES); + if (error) + return error; + } + + return 0; +} + +static int trunc_start(struct inode *inode, u64 newsize) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + struct buffer_head *dibh = NULL; + int journaled = gfs2_is_jdata(ip); + u64 oldsize = inode->i_size; + int error; + + if (!gfs2_is_stuffed(ip)) { + unsigned int blocksize = i_blocksize(inode); + unsigned int offs = newsize & (blocksize - 1); + if (offs) { + error = gfs2_block_zero_range(inode, newsize, + blocksize - offs); + if (error) + return error; + } + } + if (journaled) + error = gfs2_trans_begin(sdp, RES_DINODE + RES_JDATA, GFS2_JTRUNC_REVOKES); + else + error = gfs2_trans_begin(sdp, RES_DINODE, 0); + if (error) + return error; + + error = gfs2_meta_inode_buffer(ip, &dibh); + if (error) + goto out; + + gfs2_trans_add_meta(ip->i_gl, dibh); + + if (gfs2_is_stuffed(ip)) + gfs2_buffer_clear_tail(dibh, sizeof(struct gfs2_dinode) + newsize); + else + ip->i_diskflags |= GFS2_DIF_TRUNC_IN_PROG; + + i_size_write(inode, newsize); + inode_set_mtime_to_ts(&ip->i_inode, inode_set_ctime_current(&ip->i_inode)); + gfs2_dinode_out(ip, dibh->b_data); + + if (journaled) + error = gfs2_journaled_truncate(inode, oldsize, newsize); + else + truncate_pagecache(inode, newsize); + +out: + brelse(dibh); + if (current->journal_info) + gfs2_trans_end(sdp); + return error; +} + +int gfs2_iomap_get(struct inode *inode, loff_t pos, loff_t length, + struct iomap *iomap) +{ + struct metapath mp = { .mp_aheight = 1, }; + int ret; + + ret = __gfs2_iomap_get(inode, pos, length, 0, iomap, &mp); + release_metapath(&mp); + return ret; +} + +int gfs2_iomap_alloc(struct inode *inode, loff_t pos, loff_t length, + struct iomap *iomap) +{ + struct metapath mp = { .mp_aheight = 1, }; + int ret; + + ret = __gfs2_iomap_get(inode, pos, length, IOMAP_WRITE, iomap, &mp); + if (!ret && iomap->type == IOMAP_HOLE) + ret = __gfs2_iomap_alloc(inode, iomap, &mp); + release_metapath(&mp); + return ret; +} + +/** + * sweep_bh_for_rgrps - find an rgrp in a meta buffer and free blocks therein + * @ip: inode + * @rd_gh: holder of resource group glock + * @bh: buffer head to sweep + * @start: starting point in bh + * @end: end point in bh + * @meta: true if bh points to metadata (rather than data) + * @btotal: place to keep count of total blocks freed + * + * We sweep a metadata buffer (provided by the metapath) for blocks we need to + * free, and free them all. However, we do it one rgrp at a time. If this + * block has references to multiple rgrps, we break it into individual + * transactions. This allows other processes to use the rgrps while we're + * focused on a single one, for better concurrency / performance. + * At every transaction boundary, we rewrite the inode into the journal. + * That way the bitmaps are kept consistent with the inode and we can recover + * if we're interrupted by power-outages. + * + * Returns: 0, or return code if an error occurred. + * *btotal has the total number of blocks freed + */ +static int sweep_bh_for_rgrps(struct gfs2_inode *ip, struct gfs2_holder *rd_gh, + struct buffer_head *bh, __be64 *start, __be64 *end, + bool meta, u32 *btotal) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + struct gfs2_rgrpd *rgd; + struct gfs2_trans *tr; + __be64 *p; + int blks_outside_rgrp; + u64 bn, bstart, isize_blks; + s64 blen; /* needs to be s64 or gfs2_add_inode_blocks breaks */ + int ret = 0; + bool buf_in_tr = false; /* buffer was added to transaction */ + +more_rgrps: + rgd = NULL; + if (gfs2_holder_initialized(rd_gh)) { + rgd = gfs2_glock2rgrp(rd_gh->gh_gl); + gfs2_assert_withdraw(sdp, + gfs2_glock_is_locked_by_me(rd_gh->gh_gl)); + } + blks_outside_rgrp = 0; + bstart = 0; + blen = 0; + + for (p = start; p < end; p++) { + if (!*p) + continue; + bn = be64_to_cpu(*p); + + if (rgd) { + if (!rgrp_contains_block(rgd, bn)) { + blks_outside_rgrp++; + continue; + } + } else { + rgd = gfs2_blk2rgrpd(sdp, bn, true); + if (unlikely(!rgd)) { + ret = -EIO; + goto out; + } + ret = gfs2_glock_nq_init(rgd->rd_gl, LM_ST_EXCLUSIVE, + LM_FLAG_NODE_SCOPE, rd_gh); + if (ret) + goto out; + + /* Must be done with the rgrp glock held: */ + if (gfs2_rs_active(&ip->i_res) && + rgd == ip->i_res.rs_rgd) + gfs2_rs_deltree(&ip->i_res); + } + + /* The size of our transactions will be unknown until we + actually process all the metadata blocks that relate to + the rgrp. So we estimate. We know it can't be more than + the dinode's i_blocks and we don't want to exceed the + journal flush threshold, sd_log_thresh2. */ + if (current->journal_info == NULL) { + unsigned int jblocks_rqsted, revokes; + + jblocks_rqsted = rgd->rd_length + RES_DINODE + + RES_INDIRECT; + isize_blks = gfs2_get_inode_blocks(&ip->i_inode); + if (isize_blks > atomic_read(&sdp->sd_log_thresh2)) + jblocks_rqsted += + atomic_read(&sdp->sd_log_thresh2); + else + jblocks_rqsted += isize_blks; + revokes = jblocks_rqsted; + if (meta) + revokes += end - start; + else if (ip->i_diskflags & GFS2_DIF_EXHASH) + revokes += sdp->sd_inptrs; + ret = gfs2_trans_begin(sdp, jblocks_rqsted, revokes); + if (ret) + goto out_unlock; + down_write(&ip->i_rw_mutex); + } + /* check if we will exceed the transaction blocks requested */ + tr = current->journal_info; + if (tr->tr_num_buf_new + RES_STATFS + + RES_QUOTA >= atomic_read(&sdp->sd_log_thresh2)) { + /* We set blks_outside_rgrp to ensure the loop will + be repeated for the same rgrp, but with a new + transaction. */ + blks_outside_rgrp++; + /* This next part is tricky. If the buffer was added + to the transaction, we've already set some block + pointers to 0, so we better follow through and free + them, or we will introduce corruption (so break). + This may be impossible, or at least rare, but I + decided to cover the case regardless. + + If the buffer was not added to the transaction + (this call), doing so would exceed our transaction + size, so we need to end the transaction and start a + new one (so goto). */ + + if (buf_in_tr) + break; + goto out_unlock; + } + + gfs2_trans_add_meta(ip->i_gl, bh); + buf_in_tr = true; + *p = 0; + if (bstart + blen == bn) { + blen++; + continue; + } + if (bstart) { + __gfs2_free_blocks(ip, rgd, bstart, (u32)blen, meta); + (*btotal) += blen; + gfs2_add_inode_blocks(&ip->i_inode, -blen); + } + bstart = bn; + blen = 1; + } + if (bstart) { + __gfs2_free_blocks(ip, rgd, bstart, (u32)blen, meta); + (*btotal) += blen; + gfs2_add_inode_blocks(&ip->i_inode, -blen); + } +out_unlock: + if (!ret && blks_outside_rgrp) { /* If buffer still has non-zero blocks + outside the rgrp we just processed, + do it all over again. */ + if (current->journal_info) { + struct buffer_head *dibh; + + ret = gfs2_meta_inode_buffer(ip, &dibh); + if (ret) + goto out; + + /* Every transaction boundary, we rewrite the dinode + to keep its di_blocks current in case of failure. */ + inode_set_mtime_to_ts(&ip->i_inode, inode_set_ctime_current(&ip->i_inode)); + gfs2_trans_add_meta(ip->i_gl, dibh); + gfs2_dinode_out(ip, dibh->b_data); + brelse(dibh); + up_write(&ip->i_rw_mutex); + gfs2_trans_end(sdp); + buf_in_tr = false; + } + gfs2_glock_dq_uninit(rd_gh); + cond_resched(); + goto more_rgrps; + } +out: + return ret; +} + +static bool mp_eq_to_hgt(struct metapath *mp, __u16 *list, unsigned int h) +{ + if (memcmp(mp->mp_list, list, h * sizeof(mp->mp_list[0]))) + return false; + return true; +} + +/** + * find_nonnull_ptr - find a non-null pointer given a metapath and height + * @sdp: The superblock + * @mp: starting metapath + * @h: desired height to search + * @end_list: See punch_hole(). + * @end_aligned: See punch_hole(). + * + * Assumes the metapath is valid (with buffers) out to height h. + * Returns: true if a non-null pointer was found in the metapath buffer + * false if all remaining pointers are NULL in the buffer + */ +static bool find_nonnull_ptr(struct gfs2_sbd *sdp, struct metapath *mp, + unsigned int h, + __u16 *end_list, unsigned int end_aligned) +{ + struct buffer_head *bh = mp->mp_bh[h]; + __be64 *first, *ptr, *end; + + first = metaptr1(h, mp); + ptr = first + mp->mp_list[h]; + end = (__be64 *)(bh->b_data + bh->b_size); + if (end_list && mp_eq_to_hgt(mp, end_list, h)) { + bool keep_end = h < end_aligned; + end = first + end_list[h] + keep_end; + } + + while (ptr < end) { + if (*ptr) { /* if we have a non-null pointer */ + mp->mp_list[h] = ptr - first; + h++; + if (h < GFS2_MAX_META_HEIGHT) + mp->mp_list[h] = 0; + return true; + } + ptr++; + } + return false; +} + +enum dealloc_states { + DEALLOC_MP_FULL = 0, /* Strip a metapath with all buffers read in */ + DEALLOC_MP_LOWER = 1, /* lower the metapath strip height */ + DEALLOC_FILL_MP = 2, /* Fill in the metapath to the given height. */ + DEALLOC_DONE = 3, /* process complete */ +}; + +static inline void +metapointer_range(struct metapath *mp, int height, + __u16 *start_list, unsigned int start_aligned, + __u16 *end_list, unsigned int end_aligned, + __be64 **start, __be64 **end) +{ + struct buffer_head *bh = mp->mp_bh[height]; + __be64 *first; + + first = metaptr1(height, mp); + *start = first; + if (mp_eq_to_hgt(mp, start_list, height)) { + bool keep_start = height < start_aligned; + *start = first + start_list[height] + keep_start; + } + *end = (__be64 *)(bh->b_data + bh->b_size); + if (end_list && mp_eq_to_hgt(mp, end_list, height)) { + bool keep_end = height < end_aligned; + *end = first + end_list[height] + keep_end; + } +} + +static inline bool walk_done(struct gfs2_sbd *sdp, + struct metapath *mp, int height, + __u16 *end_list, unsigned int end_aligned) +{ + __u16 end; + + if (end_list) { + bool keep_end = height < end_aligned; + if (!mp_eq_to_hgt(mp, end_list, height)) + return false; + end = end_list[height] + keep_end; + } else + end = (height > 0) ? sdp->sd_inptrs : sdp->sd_diptrs; + return mp->mp_list[height] >= end; +} + +/** + * punch_hole - deallocate blocks in a file + * @ip: inode to truncate + * @offset: the start of the hole + * @length: the size of the hole (or 0 for truncate) + * + * Punch a hole into a file or truncate a file at a given position. This + * function operates in whole blocks (@offset and @length are rounded + * accordingly); partially filled blocks must be cleared otherwise. + * + * This function works from the bottom up, and from the right to the left. In + * other words, it strips off the highest layer (data) before stripping any of + * the metadata. Doing it this way is best in case the operation is interrupted + * by power failure, etc. The dinode is rewritten in every transaction to + * guarantee integrity. + */ +static int punch_hole(struct gfs2_inode *ip, u64 offset, u64 length) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + u64 maxsize = sdp->sd_heightsize[ip->i_height]; + struct metapath mp = {}; + struct buffer_head *dibh, *bh; + struct gfs2_holder rd_gh; + unsigned int bsize_shift = sdp->sd_sb.sb_bsize_shift; + unsigned int bsize = 1 << bsize_shift; + u64 lblock = (offset + bsize - 1) >> bsize_shift; + __u16 start_list[GFS2_MAX_META_HEIGHT]; + __u16 __end_list[GFS2_MAX_META_HEIGHT], *end_list = NULL; + unsigned int start_aligned, end_aligned; + unsigned int strip_h = ip->i_height - 1; + u32 btotal = 0; + int ret, state; + int mp_h; /* metapath buffers are read in to this height */ + u64 prev_bnr = 0; + __be64 *start, *end; + + if (offset + bsize - 1 >= maxsize) { + /* + * The starting point lies beyond the allocated metadata; + * there are no blocks to deallocate. + */ + return 0; + } + + /* + * The start position of the hole is defined by lblock, start_list, and + * start_aligned. The end position of the hole is defined by lend, + * end_list, and end_aligned. + * + * start_aligned and end_aligned define down to which height the start + * and end positions are aligned to the metadata tree (i.e., the + * position is a multiple of the metadata granularity at the height + * above). This determines at which heights additional meta pointers + * needs to be preserved for the remaining data. + */ + + if (length) { + u64 end_offset = offset + length; + u64 lend; + + /* + * Clip the end at the maximum file size for the given height: + * that's how far the metadata goes; files bigger than that + * will have additional layers of indirection. + */ + if (end_offset > maxsize) + end_offset = maxsize; + lend = end_offset >> bsize_shift; + + if (lblock >= lend) + return 0; + + find_metapath(sdp, lend, &mp, ip->i_height); + end_list = __end_list; + memcpy(end_list, mp.mp_list, sizeof(mp.mp_list)); + + for (mp_h = ip->i_height - 1; mp_h > 0; mp_h--) { + if (end_list[mp_h]) + break; + } + end_aligned = mp_h; + } + + find_metapath(sdp, lblock, &mp, ip->i_height); + memcpy(start_list, mp.mp_list, sizeof(start_list)); + + for (mp_h = ip->i_height - 1; mp_h > 0; mp_h--) { + if (start_list[mp_h]) + break; + } + start_aligned = mp_h; + + ret = gfs2_meta_inode_buffer(ip, &dibh); + if (ret) + return ret; + + mp.mp_bh[0] = dibh; + ret = lookup_metapath(ip, &mp); + if (ret) + goto out_metapath; + + /* issue read-ahead on metadata */ + for (mp_h = 0; mp_h < mp.mp_aheight - 1; mp_h++) { + metapointer_range(&mp, mp_h, start_list, start_aligned, + end_list, end_aligned, &start, &end); + gfs2_metapath_ra(ip->i_gl, start, end); + } + + if (mp.mp_aheight == ip->i_height) + state = DEALLOC_MP_FULL; /* We have a complete metapath */ + else + state = DEALLOC_FILL_MP; /* deal with partial metapath */ + + ret = gfs2_rindex_update(sdp); + if (ret) + goto out_metapath; + + ret = gfs2_quota_hold(ip, NO_UID_QUOTA_CHANGE, NO_GID_QUOTA_CHANGE); + if (ret) + goto out_metapath; + gfs2_holder_mark_uninitialized(&rd_gh); + + mp_h = strip_h; + + while (state != DEALLOC_DONE) { + switch (state) { + /* Truncate a full metapath at the given strip height. + * Note that strip_h == mp_h in order to be in this state. */ + case DEALLOC_MP_FULL: + bh = mp.mp_bh[mp_h]; + gfs2_assert_withdraw(sdp, bh); + if (gfs2_assert_withdraw(sdp, + prev_bnr != bh->b_blocknr)) { + fs_emerg(sdp, "inode %llu, block:%llu, i_h:%u, " + "s_h:%u, mp_h:%u\n", + (unsigned long long)ip->i_no_addr, + prev_bnr, ip->i_height, strip_h, mp_h); + } + prev_bnr = bh->b_blocknr; + + if (gfs2_metatype_check(sdp, bh, + (mp_h ? GFS2_METATYPE_IN : + GFS2_METATYPE_DI))) { + ret = -EIO; + goto out; + } + + /* + * Below, passing end_aligned as 0 gives us the + * metapointer range excluding the end point: the end + * point is the first metapath we must not deallocate! + */ + + metapointer_range(&mp, mp_h, start_list, start_aligned, + end_list, 0 /* end_aligned */, + &start, &end); + ret = sweep_bh_for_rgrps(ip, &rd_gh, mp.mp_bh[mp_h], + start, end, + mp_h != ip->i_height - 1, + &btotal); + + /* If we hit an error or just swept dinode buffer, + just exit. */ + if (ret || !mp_h) { + state = DEALLOC_DONE; + break; + } + state = DEALLOC_MP_LOWER; + break; + + /* lower the metapath strip height */ + case DEALLOC_MP_LOWER: + /* We're done with the current buffer, so release it, + unless it's the dinode buffer. Then back up to the + previous pointer. */ + if (mp_h) { + brelse(mp.mp_bh[mp_h]); + mp.mp_bh[mp_h] = NULL; + } + /* If we can't get any lower in height, we've stripped + off all we can. Next step is to back up and start + stripping the previous level of metadata. */ + if (mp_h == 0) { + strip_h--; + memcpy(mp.mp_list, start_list, sizeof(start_list)); + mp_h = strip_h; + state = DEALLOC_FILL_MP; + break; + } + mp.mp_list[mp_h] = 0; + mp_h--; /* search one metadata height down */ + mp.mp_list[mp_h]++; + if (walk_done(sdp, &mp, mp_h, end_list, end_aligned)) + break; + /* Here we've found a part of the metapath that is not + * allocated. We need to search at that height for the + * next non-null pointer. */ + if (find_nonnull_ptr(sdp, &mp, mp_h, end_list, end_aligned)) { + state = DEALLOC_FILL_MP; + mp_h++; + } + /* No more non-null pointers at this height. Back up + to the previous height and try again. */ + break; /* loop around in the same state */ + + /* Fill the metapath with buffers to the given height. */ + case DEALLOC_FILL_MP: + /* Fill the buffers out to the current height. */ + ret = fillup_metapath(ip, &mp, mp_h); + if (ret < 0) + goto out; + + /* On the first pass, issue read-ahead on metadata. */ + if (mp.mp_aheight > 1 && strip_h == ip->i_height - 1) { + unsigned int height = mp.mp_aheight - 1; + + /* No read-ahead for data blocks. */ + if (mp.mp_aheight - 1 == strip_h) + height--; + + for (; height >= mp.mp_aheight - ret; height--) { + metapointer_range(&mp, height, + start_list, start_aligned, + end_list, end_aligned, + &start, &end); + gfs2_metapath_ra(ip->i_gl, start, end); + } + } + + /* If buffers found for the entire strip height */ + if (mp.mp_aheight - 1 == strip_h) { + state = DEALLOC_MP_FULL; + break; + } + if (mp.mp_aheight < ip->i_height) /* We have a partial height */ + mp_h = mp.mp_aheight - 1; + + /* If we find a non-null block pointer, crawl a bit + higher up in the metapath and try again, otherwise + we need to look lower for a new starting point. */ + if (find_nonnull_ptr(sdp, &mp, mp_h, end_list, end_aligned)) + mp_h++; + else + state = DEALLOC_MP_LOWER; + break; + } + } + + if (btotal) { + if (current->journal_info == NULL) { + ret = gfs2_trans_begin(sdp, RES_DINODE + RES_STATFS + + RES_QUOTA, 0); + if (ret) + goto out; + down_write(&ip->i_rw_mutex); + } + gfs2_statfs_change(sdp, 0, +btotal, 0); + gfs2_quota_change(ip, -(s64)btotal, ip->i_inode.i_uid, + ip->i_inode.i_gid); + inode_set_mtime_to_ts(&ip->i_inode, inode_set_ctime_current(&ip->i_inode)); + gfs2_trans_add_meta(ip->i_gl, dibh); + gfs2_dinode_out(ip, dibh->b_data); + up_write(&ip->i_rw_mutex); + gfs2_trans_end(sdp); + } + +out: + if (gfs2_holder_initialized(&rd_gh)) + gfs2_glock_dq_uninit(&rd_gh); + if (current->journal_info) { + up_write(&ip->i_rw_mutex); + gfs2_trans_end(sdp); + cond_resched(); + } + gfs2_quota_unhold(ip); +out_metapath: + release_metapath(&mp); + return ret; +} + +static int trunc_end(struct gfs2_inode *ip) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + struct buffer_head *dibh; + int error; + + error = gfs2_trans_begin(sdp, RES_DINODE, 0); + if (error) + return error; + + down_write(&ip->i_rw_mutex); + + error = gfs2_meta_inode_buffer(ip, &dibh); + if (error) + goto out; + + if (!i_size_read(&ip->i_inode)) { + ip->i_height = 0; + ip->i_goal = ip->i_no_addr; + gfs2_buffer_clear_tail(dibh, sizeof(struct gfs2_dinode)); + gfs2_ordered_del_inode(ip); + } + inode_set_mtime_to_ts(&ip->i_inode, inode_set_ctime_current(&ip->i_inode)); + ip->i_diskflags &= ~GFS2_DIF_TRUNC_IN_PROG; + + gfs2_trans_add_meta(ip->i_gl, dibh); + gfs2_dinode_out(ip, dibh->b_data); + brelse(dibh); + +out: + up_write(&ip->i_rw_mutex); + gfs2_trans_end(sdp); + return error; +} + +/** + * do_shrink - make a file smaller + * @inode: the inode + * @newsize: the size to make the file + * + * Called with an exclusive lock on @inode. The @size must + * be equal to or smaller than the current inode size. + * + * Returns: errno + */ + +static int do_shrink(struct inode *inode, u64 newsize) +{ + struct gfs2_inode *ip = GFS2_I(inode); + int error; + + error = trunc_start(inode, newsize); + if (error < 0) + return error; + if (gfs2_is_stuffed(ip)) + return 0; + + error = punch_hole(ip, newsize, 0); + if (error == 0) + error = trunc_end(ip); + + return error; +} + +/** + * do_grow - Touch and update inode size + * @inode: The inode + * @size: The new size + * + * This function updates the timestamps on the inode and + * may also increase the size of the inode. This function + * must not be called with @size any smaller than the current + * inode size. + * + * Although it is not strictly required to unstuff files here, + * earlier versions of GFS2 have a bug in the stuffed file reading + * code which will result in a buffer overrun if the size is larger + * than the max stuffed file size. In order to prevent this from + * occurring, such files are unstuffed, but in other cases we can + * just update the inode size directly. + * + * Returns: 0 on success, or -ve on error + */ + +static int do_grow(struct inode *inode, u64 size) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + struct gfs2_alloc_parms ap = { .target = 1, }; + struct buffer_head *dibh; + int error; + int unstuff = 0; + + if (gfs2_is_stuffed(ip) && size > gfs2_max_stuffed_size(ip)) { + error = gfs2_quota_lock_check(ip, &ap); + if (error) + return error; + + error = gfs2_inplace_reserve(ip, &ap); + if (error) + goto do_grow_qunlock; + unstuff = 1; + } + + if (!unstuff) { + error = gfs2_clear_beyond_eof(inode, size); + if (error) + goto do_grow_qunlock; + } + + error = gfs2_trans_begin(sdp, RES_DINODE + RES_STATFS + RES_RG_BIT + + (unstuff && + gfs2_is_jdata(ip) ? RES_JDATA : 0) + + (sdp->sd_args.ar_quota == GFS2_QUOTA_OFF ? + 0 : RES_QUOTA), 0); + if (error) + goto do_grow_release; + + if (unstuff) { + error = gfs2_unstuff_dinode(ip); + if (error) + goto do_end_trans; + } + + error = gfs2_meta_inode_buffer(ip, &dibh); + if (error) + goto do_end_trans; + + truncate_setsize(inode, size); + inode_set_mtime_to_ts(&ip->i_inode, inode_set_ctime_current(&ip->i_inode)); + gfs2_trans_add_meta(ip->i_gl, dibh); + gfs2_dinode_out(ip, dibh->b_data); + brelse(dibh); + +do_end_trans: + gfs2_trans_end(sdp); +do_grow_release: + if (unstuff) { + gfs2_inplace_release(ip); +do_grow_qunlock: + gfs2_quota_unlock(ip); + } + return error; +} + +/** + * gfs2_setattr_size - make a file a given size + * @inode: the inode + * @newsize: the size to make the file + * + * The file size can grow, shrink, or stay the same size. This + * is called holding i_rwsem and an exclusive glock on the inode + * in question. + * + * Returns: errno + */ + +int gfs2_setattr_size(struct inode *inode, u64 newsize) +{ + struct gfs2_inode *ip = GFS2_I(inode); + int ret; + + BUG_ON(!S_ISREG(inode->i_mode)); + + ret = inode_newsize_ok(inode, newsize); + if (ret) + return ret; + + inode_dio_wait(inode); + + ret = gfs2_qa_get(ip); + if (ret) + goto out; + + if (newsize >= inode->i_size) { + ret = do_grow(inode, newsize); + goto out; + } + + ret = do_shrink(inode, newsize); +out: + gfs2_rs_delete(ip); + gfs2_qa_put(ip); + return ret; +} + +int gfs2_truncatei_resume(struct gfs2_inode *ip) +{ + int error; + error = punch_hole(ip, i_size_read(&ip->i_inode), 0); + if (!error) + error = trunc_end(ip); + return error; +} + +int gfs2_file_dealloc(struct gfs2_inode *ip) +{ + return punch_hole(ip, 0, 0); +} + +/** + * gfs2_free_journal_extents - Free cached journal bmap info + * @jd: The journal + * + */ + +void gfs2_free_journal_extents(struct gfs2_jdesc *jd) +{ + struct gfs2_journal_extent *jext; + + while(!list_empty(&jd->extent_list)) { + jext = list_first_entry(&jd->extent_list, struct gfs2_journal_extent, list); + list_del(&jext->list); + kfree(jext); + } +} + +/** + * gfs2_add_jextent - Add or merge a new extent to extent cache + * @jd: The journal descriptor + * @lblock: The logical block at start of new extent + * @dblock: The physical block at start of new extent + * @blocks: Size of extent in fs blocks + * + * Returns: 0 on success or -ENOMEM + */ + +static int gfs2_add_jextent(struct gfs2_jdesc *jd, u64 lblock, u64 dblock, u64 blocks) +{ + struct gfs2_journal_extent *jext; + + if (!list_empty(&jd->extent_list)) { + jext = list_last_entry(&jd->extent_list, struct gfs2_journal_extent, list); + if ((jext->dblock + jext->blocks) == dblock) { + jext->blocks += blocks; + return 0; + } + } + + jext = kzalloc_obj(struct gfs2_journal_extent, GFP_NOFS); + if (jext == NULL) + return -ENOMEM; + jext->dblock = dblock; + jext->lblock = lblock; + jext->blocks = blocks; + list_add_tail(&jext->list, &jd->extent_list); + jd->nr_extents++; + return 0; +} + +/** + * gfs2_map_journal_extents - Cache journal bmap info + * @sdp: The super block + * @jd: The journal to map + * + * Create a reusable "extent" mapping from all logical + * blocks to all physical blocks for the given journal. This will save + * us time when writing journal blocks. Most journals will have only one + * extent that maps all their logical blocks. That's because gfs2.mkfs + * arranges the journal blocks sequentially to maximize performance. + * So the extent would map the first block for the entire file length. + * However, gfs2_jadd can happen while file activity is happening, so + * those journals may not be sequential. Less likely is the case where + * the users created their own journals by mounting the metafs and + * laying it out. But it's still possible. These journals might have + * several extents. + * + * Returns: 0 on success, or error on failure + */ + +int gfs2_map_journal_extents(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd) +{ + u64 lblock = 0; + u64 lblock_stop; + struct gfs2_inode *ip = GFS2_I(jd->jd_inode); + struct buffer_head bh; + unsigned int shift = sdp->sd_sb.sb_bsize_shift; + u64 size; + int rc; + ktime_t start, end; + + start = ktime_get(); + lblock_stop = i_size_read(jd->jd_inode) >> shift; + size = (lblock_stop - lblock) << shift; + jd->nr_extents = 0; + WARN_ON(!list_empty(&jd->extent_list)); + + do { + bh.b_state = 0; + bh.b_blocknr = 0; + bh.b_size = size; + rc = gfs2_block_map(jd->jd_inode, lblock, &bh, 0); + if (rc || !buffer_mapped(&bh)) + goto fail; + rc = gfs2_add_jextent(jd, lblock, bh.b_blocknr, bh.b_size >> shift); + if (rc) + goto fail; + size -= bh.b_size; + lblock += (bh.b_size >> ip->i_inode.i_blkbits); + } while(size > 0); + + end = ktime_get(); + fs_info(sdp, "journal %d mapped with %u extents in %lldms\n", jd->jd_jid, + jd->nr_extents, ktime_ms_delta(end, start)); + return 0; + +fail: + fs_warn(sdp, "error %d mapping journal %u at offset %llu (extent %u)\n", + rc, jd->jd_jid, + (unsigned long long)(i_size_read(jd->jd_inode) - size), + jd->nr_extents); + fs_warn(sdp, "bmap=%d lblock=%llu block=%llu, state=0x%08lx, size=%llu\n", + rc, (unsigned long long)lblock, (unsigned long long)bh.b_blocknr, + bh.b_state, (unsigned long long)bh.b_size); + gfs2_free_journal_extents(jd); + return rc; +} + +/** + * gfs2_write_alloc_required - figure out if a write will require an allocation + * @ip: the file being written to + * @offset: the offset to write to + * @len: the number of bytes being written + * + * Returns: 1 if an alloc is required, 0 otherwise + */ + +int gfs2_write_alloc_required(struct gfs2_inode *ip, u64 offset, + unsigned int len) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + struct buffer_head bh; + unsigned int shift; + u64 lblock, lblock_stop, size; + u64 end_of_file; + + if (!len) + return 0; + + if (gfs2_is_stuffed(ip)) { + if (offset + len > gfs2_max_stuffed_size(ip)) + return 1; + return 0; + } + + shift = sdp->sd_sb.sb_bsize_shift; + BUG_ON(gfs2_is_dir(ip)); + end_of_file = (i_size_read(&ip->i_inode) + sdp->sd_sb.sb_bsize - 1) >> shift; + lblock = offset >> shift; + lblock_stop = (offset + len + sdp->sd_sb.sb_bsize - 1) >> shift; + if (lblock_stop > end_of_file && ip != GFS2_I(sdp->sd_rindex)) + return 1; + + size = (lblock_stop - lblock) << shift; + do { + bh.b_state = 0; + bh.b_size = size; + gfs2_block_map(&ip->i_inode, lblock, &bh, 0); + if (!buffer_mapped(&bh)) + return 1; + size -= bh.b_size; + lblock += (bh.b_size >> ip->i_inode.i_blkbits); + } while(size > 0); + + return 0; +} + +static int stuffed_zero_range(struct inode *inode, loff_t offset, loff_t length) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct buffer_head *dibh; + int error; + + if (offset >= inode->i_size) + return 0; + if (offset + length > inode->i_size) + length = inode->i_size - offset; + + error = gfs2_meta_inode_buffer(ip, &dibh); + if (error) + return error; + gfs2_trans_add_meta(ip->i_gl, dibh); + memset(dibh->b_data + sizeof(struct gfs2_dinode) + offset, 0, + length); + brelse(dibh); + return 0; +} + +static int gfs2_journaled_truncate_range(struct inode *inode, loff_t offset, + loff_t length) +{ + struct gfs2_sbd *sdp = GFS2_SB(inode); + loff_t max_chunk = GFS2_JTRUNC_REVOKES * sdp->sd_vfs->s_blocksize; + int error; + + while (length) { + struct gfs2_trans *tr; + loff_t chunk; + unsigned int offs; + + chunk = length; + if (chunk > max_chunk) + chunk = max_chunk; + + offs = offset & ~PAGE_MASK; + if (offs && chunk > PAGE_SIZE) + chunk = offs + ((chunk - offs) & PAGE_MASK); + + truncate_pagecache_range(inode, offset, chunk); + offset += chunk; + length -= chunk; + + tr = current->journal_info; + if (!test_bit(TR_TOUCHED, &tr->tr_flags)) + continue; + + gfs2_trans_end(sdp); + error = gfs2_trans_begin(sdp, RES_DINODE, GFS2_JTRUNC_REVOKES); + if (error) + return error; + } + return 0; +} + +int __gfs2_punch_hole(struct file *file, loff_t offset, loff_t length) +{ + struct inode *inode = file_inode(file); + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + unsigned int blocksize = i_blocksize(inode); + loff_t start, end; + int error; + + if (!gfs2_is_stuffed(ip)) { + unsigned int start_off, end_len; + + start_off = offset & (blocksize - 1); + end_len = (offset + length) & (blocksize - 1); + if (start_off) { + unsigned int len = length; + if (length > blocksize - start_off) + len = blocksize - start_off; + error = gfs2_block_zero_range(inode, offset, len); + if (error) + goto out; + if (start_off + length < blocksize) + end_len = 0; + } + if (end_len) { + error = gfs2_block_zero_range(inode, + offset + length - end_len, end_len); + if (error) + goto out; + } + } + + start = round_down(offset, blocksize); + end = round_up(offset + length, blocksize) - 1; + error = filemap_write_and_wait_range(inode->i_mapping, start, end); + if (error) + return error; + + if (gfs2_is_jdata(ip)) + error = gfs2_trans_begin(sdp, RES_DINODE + 2 * RES_JDATA, + GFS2_JTRUNC_REVOKES); + else + error = gfs2_trans_begin(sdp, RES_DINODE, 0); + if (error) + return error; + + if (gfs2_is_stuffed(ip)) { + error = stuffed_zero_range(inode, offset, length); + if (error) + goto out; + } + + if (gfs2_is_jdata(ip)) { + BUG_ON(!current->journal_info); + gfs2_journaled_truncate_range(inode, offset, length); + } else + truncate_pagecache_range(inode, offset, offset + length - 1); + + file_update_time(file); + mark_inode_dirty(inode); + + if (current->journal_info) + gfs2_trans_end(sdp); + + if (!gfs2_is_stuffed(ip)) + error = punch_hole(ip, offset, length); + +out: + if (current->journal_info) + gfs2_trans_end(sdp); + return error; +} + +static ssize_t gfs2_writeback_range(struct iomap_writepage_ctx *wpc, + struct folio *folio, u64 offset, unsigned int len, u64 end_pos) +{ + if (WARN_ON_ONCE(gfs2_is_stuffed(GFS2_I(wpc->inode)))) + return -EIO; + + if (offset < wpc->iomap.offset || + offset >= wpc->iomap.offset + wpc->iomap.length) { + int ret; + + memset(&wpc->iomap, 0, sizeof(wpc->iomap)); + ret = gfs2_iomap_get(wpc->inode, offset, INT_MAX, &wpc->iomap); + if (ret) + return ret; + } + + return iomap_add_to_ioend(wpc, folio, offset, end_pos, len); +} + +const struct iomap_writeback_ops gfs2_writeback_ops = { + .writeback_range = gfs2_writeback_range, + .writeback_submit = iomap_ioend_writeback_submit, +}; diff --git a/fs/gfs2/bmap.h b/fs/gfs2/bmap.h new file mode 100644 index 000000000..e3d6efdfd --- /dev/null +++ b/fs/gfs2/bmap.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#ifndef __BMAP_DOT_H__ +#define __BMAP_DOT_H__ + +#include + +#include "inode.h" + +struct inode; +struct gfs2_inode; +struct page; + + +/** + * gfs2_write_calc_reserv - calculate number of blocks needed to write to a file + * @ip: the file + * @len: the number of bytes to be written to the file + * @data_blocks: returns the number of data blocks required + * @ind_blocks: returns the number of indirect blocks required + * + */ + +static inline void gfs2_write_calc_reserv(const struct gfs2_inode *ip, + unsigned int len, + unsigned int *data_blocks, + unsigned int *ind_blocks) +{ + const struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + unsigned int tmp; + + BUG_ON(gfs2_is_dir(ip)); + *data_blocks = (len >> sdp->sd_sb.sb_bsize_shift) + 3; + *ind_blocks = 3 * (sdp->sd_max_height - 1); + + for (tmp = *data_blocks; tmp > sdp->sd_diptrs;) { + tmp = DIV_ROUND_UP(tmp, sdp->sd_inptrs); + *ind_blocks += tmp; + } +} + +extern const struct iomap_ops gfs2_iomap_ops; +extern const struct iomap_write_ops gfs2_iomap_write_ops; +extern const struct iomap_writeback_ops gfs2_writeback_ops; + +int gfs2_unstuff_dinode(struct gfs2_inode *ip); +int gfs2_block_map(struct inode *inode, sector_t lblock, + struct buffer_head *bh, int create); +int gfs2_iomap_get(struct inode *inode, loff_t pos, loff_t length, + struct iomap *iomap); +int gfs2_iomap_alloc(struct inode *inode, loff_t pos, loff_t length, + struct iomap *iomap); +int gfs2_get_extent(struct inode *inode, u64 lblock, u64 *dblock, + unsigned int *extlen); +int gfs2_alloc_extent(struct inode *inode, u64 lblock, u64 *dblock, + unsigned *extlen, bool *new); +int gfs2_clear_beyond_eof(struct inode *inode, loff_t end); +int gfs2_setattr_size(struct inode *inode, u64 size); +int gfs2_truncatei_resume(struct gfs2_inode *ip); +int gfs2_file_dealloc(struct gfs2_inode *ip); +int gfs2_write_alloc_required(struct gfs2_inode *ip, u64 offset, + unsigned int len); +int gfs2_map_journal_extents(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd); +void gfs2_free_journal_extents(struct gfs2_jdesc *jd); +int __gfs2_punch_hole(struct file *file, loff_t offset, loff_t length); + +#endif /* __BMAP_DOT_H__ */ diff --git a/fs/gfs2/dentry.c b/fs/gfs2/dentry.c new file mode 100644 index 000000000..95050e719 --- /dev/null +++ b/fs/gfs2/dentry.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include + +#include "gfs2.h" +#include "incore.h" +#include "dir.h" +#include "glock.h" +#include "super.h" +#include "util.h" +#include "inode.h" + +/** + * gfs2_drevalidate - Check directory lookup consistency + * @dir: expected parent directory inode + * @name: expexted name + * @dentry: dentry to check + * @flags: lookup flags + * + * Check to make sure the lookup necessary to arrive at this inode from its + * parent is still good. + * + * Returns: 1 if the dentry is ok, 0 if it isn't + */ + +static int gfs2_drevalidate(struct inode *dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + struct gfs2_sbd *sdp = GFS2_SB(dir); + struct gfs2_inode *dip = GFS2_I(dir); + struct inode *inode; + struct gfs2_holder d_gh; + struct gfs2_inode *ip = NULL; + int error, valid; + int had_lock = 0; + + if (flags & LOOKUP_RCU) + return -ECHILD; + + inode = d_inode(dentry); + + if (inode) { + if (is_bad_inode(inode)) + return 0; + ip = GFS2_I(inode); + } + + if (sdp->sd_lockstruct.ls_ops->lm_mount == NULL) + return 1; + + had_lock = (gfs2_glock_is_locked_by_me(dip->i_gl) != NULL); + if (!had_lock) { + error = gfs2_glock_nq_init(dip->i_gl, LM_ST_SHARED, 0, &d_gh); + if (error) + return 0; + } + + error = gfs2_dir_check(dir, name, ip); + valid = inode ? !error : (error == -ENOENT); + + if (!had_lock) + gfs2_glock_dq_uninit(&d_gh); + return valid; +} + +static int gfs2_dhash(const struct dentry *dentry, struct qstr *str) +{ + str->hash = gfs2_disk_hash(str->name, str->len); + return 0; +} + +static int gfs2_dentry_delete(const struct dentry *dentry) +{ + struct gfs2_inode *ginode; + + if (d_really_is_negative(dentry)) + return 0; + + ginode = GFS2_I(d_inode(dentry)); + if (!gfs2_holder_initialized(&ginode->i_iopen_gh)) + return 0; + + if (test_bit(GLF_DEMOTE, &ginode->i_iopen_gh.gh_gl->gl_flags)) + return 1; + + return 0; +} + +const struct dentry_operations gfs2_dops = { + .d_revalidate = gfs2_drevalidate, + .d_hash = gfs2_dhash, + .d_delete = gfs2_dentry_delete, +}; + diff --git a/fs/gfs2/dir.c b/fs/gfs2/dir.c new file mode 100644 index 000000000..0237b36b9 --- /dev/null +++ b/fs/gfs2/dir.c @@ -0,0 +1,2187 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +/* + * Implements Extendible Hashing as described in: + * "Extendible Hashing" by Fagin, et al in + * __ACM Trans. on Database Systems__, Sept 1979. + * + * + * Here's the layout of dirents which is essentially the same as that of ext2 + * within a single block. The field de_name_len is the number of bytes + * actually required for the name (no null terminator). The field de_rec_len + * is the number of bytes allocated to the dirent. The offset of the next + * dirent in the block is (dirent + dirent->de_rec_len). When a dirent is + * deleted, the preceding dirent inherits its allocated space, ie + * prev->de_rec_len += deleted->de_rec_len. Since the next dirent is obtained + * by adding de_rec_len to the current dirent, this essentially causes the + * deleted dirent to get jumped over when iterating through all the dirents. + * + * When deleting the first dirent in a block, there is no previous dirent so + * the field de_ino is set to zero to designate it as deleted. When allocating + * a dirent, gfs2_dirent_alloc iterates through the dirents in a block. If the + * first dirent has (de_ino == 0) and de_rec_len is large enough, this first + * dirent is allocated. Otherwise it must go through all the 'used' dirents + * searching for one in which the amount of total space minus the amount of + * used space will provide enough space for the new dirent. + * + * There are two types of blocks in which dirents reside. In a stuffed dinode, + * the dirents begin at offset sizeof(struct gfs2_dinode) from the beginning of + * the block. In leaves, they begin at offset sizeof(struct gfs2_leaf) from the + * beginning of the leaf block. The dirents reside in leaves when + * + * dip->i_diskflags & GFS2_DIF_EXHASH is true + * + * Otherwise, the dirents are "linear", within a single stuffed dinode block. + * + * When the dirents are in leaves, the actual contents of the directory file are + * used as an array of 64-bit block pointers pointing to the leaf blocks. The + * dirents are NOT in the directory file itself. There can be more than one + * block pointer in the array that points to the same leaf. In fact, when a + * directory is first converted from linear to exhash, all of the pointers + * point to the same leaf. + * + * When a leaf is completely full, the size of the hash table can be + * doubled unless it is already at the maximum size which is hard coded into + * GFS2_DIR_MAX_DEPTH. After that, leaves are chained together in a linked list, + * but never before the maximum hash table size has been reached. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gfs2.h" +#include "incore.h" +#include "dir.h" +#include "glock.h" +#include "inode.h" +#include "meta_io.h" +#include "quota.h" +#include "rgrp.h" +#include "trans.h" +#include "bmap.h" +#include "util.h" + +#define MAX_RA_BLOCKS 32 /* max read-ahead blocks */ + +#define gfs2_disk_hash2offset(h) (((u64)(h)) >> 1) +#define gfs2_dir_offset2hash(p) ((u32)(((u64)(p)) << 1)) +#define GFS2_HASH_INDEX_MASK 0xffffc000 +#define GFS2_USE_HASH_FLAG 0x2000 + +struct qstr gfs2_qdot __read_mostly; +struct qstr gfs2_qdotdot __read_mostly; + +typedef int (*gfs2_dscan_t)(const struct gfs2_dirent *dent, + const struct qstr *name, void *opaque); + +int gfs2_dir_get_new_buffer(struct gfs2_inode *ip, u64 block, + struct buffer_head **bhp) +{ + struct buffer_head *bh; + + bh = gfs2_meta_new(ip->i_gl, block); + gfs2_trans_add_meta(ip->i_gl, bh); + gfs2_metatype_set(bh, GFS2_METATYPE_JD, GFS2_FORMAT_JD); + gfs2_buffer_clear_tail(bh, sizeof(struct gfs2_meta_header)); + *bhp = bh; + return 0; +} + +static int gfs2_dir_get_existing_buffer(struct gfs2_inode *ip, u64 block, + struct buffer_head **bhp) +{ + struct buffer_head *bh; + int error; + + error = gfs2_meta_read(ip->i_gl, block, DIO_WAIT, 0, &bh); + if (error) + return error; + if (gfs2_metatype_check(GFS2_SB(&ip->i_inode), bh, GFS2_METATYPE_JD)) { + brelse(bh); + return -EIO; + } + *bhp = bh; + return 0; +} + +static int gfs2_dir_write_stuffed(struct gfs2_inode *ip, const char *buf, + unsigned int offset, unsigned int size) +{ + struct buffer_head *dibh; + int error; + + error = gfs2_meta_inode_buffer(ip, &dibh); + if (error) + return error; + + gfs2_trans_add_meta(ip->i_gl, dibh); + memcpy(dibh->b_data + offset + sizeof(struct gfs2_dinode), buf, size); + if (ip->i_inode.i_size < offset + size) + i_size_write(&ip->i_inode, offset + size); + inode_set_mtime_to_ts(&ip->i_inode, inode_set_ctime_current(&ip->i_inode)); + gfs2_dinode_out(ip, dibh->b_data); + + brelse(dibh); + + return size; +} + + + +/** + * gfs2_dir_write_data - Write directory information to the inode + * @ip: The GFS2 inode + * @buf: The buffer containing information to be written + * @offset: The file offset to start writing at + * @size: The amount of data to write + * + * Returns: The number of bytes correctly written or error code + */ +static int gfs2_dir_write_data(struct gfs2_inode *ip, const char *buf, + u64 offset, unsigned int size) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + struct buffer_head *dibh; + u64 lblock, dblock; + u32 extlen = 0; + unsigned int o; + int copied = 0; + int error = 0; + bool new = false; + + if (!size) + return 0; + + if (gfs2_is_stuffed(ip) && offset + size <= gfs2_max_stuffed_size(ip)) + return gfs2_dir_write_stuffed(ip, buf, (unsigned int)offset, + size); + + if (gfs2_assert_warn(sdp, gfs2_is_jdata(ip))) + return -EINVAL; + + if (gfs2_is_stuffed(ip)) { + error = gfs2_unstuff_dinode(ip); + if (error) + return error; + } + + lblock = offset; + o = do_div(lblock, sdp->sd_jbsize) + sizeof(struct gfs2_meta_header); + + while (copied < size) { + unsigned int amount; + struct buffer_head *bh; + + amount = size - copied; + if (amount > sdp->sd_sb.sb_bsize - o) + amount = sdp->sd_sb.sb_bsize - o; + + if (!extlen) { + extlen = 1; + error = gfs2_alloc_extent(&ip->i_inode, lblock, &dblock, + &extlen, &new); + if (error) + goto fail; + error = -EIO; + if (gfs2_assert_withdraw(sdp, dblock)) + goto fail; + } + + if (amount == sdp->sd_jbsize || new) + error = gfs2_dir_get_new_buffer(ip, dblock, &bh); + else + error = gfs2_dir_get_existing_buffer(ip, dblock, &bh); + + if (error) + goto fail; + + gfs2_trans_add_meta(ip->i_gl, bh); + memcpy(bh->b_data + o, buf, amount); + brelse(bh); + + buf += amount; + copied += amount; + lblock++; + dblock++; + extlen--; + + o = sizeof(struct gfs2_meta_header); + } + +out: + error = gfs2_meta_inode_buffer(ip, &dibh); + if (error) + return error; + + if (ip->i_inode.i_size < offset + copied) + i_size_write(&ip->i_inode, offset + copied); + inode_set_mtime_to_ts(&ip->i_inode, inode_set_ctime_current(&ip->i_inode)); + + gfs2_trans_add_meta(ip->i_gl, dibh); + gfs2_dinode_out(ip, dibh->b_data); + brelse(dibh); + + return copied; +fail: + if (copied) + goto out; + return error; +} + +static int gfs2_dir_read_stuffed(struct gfs2_inode *ip, __be64 *buf, + unsigned int size) +{ + struct buffer_head *dibh; + int error; + + error = gfs2_meta_inode_buffer(ip, &dibh); + if (!error) { + memcpy(buf, dibh->b_data + sizeof(struct gfs2_dinode), size); + brelse(dibh); + } + + return (error) ? error : size; +} + + +/** + * gfs2_dir_read_data - Read a data from a directory inode + * @ip: The GFS2 Inode + * @buf: The buffer to place result into + * @size: Amount of data to transfer + * + * Returns: The amount of data actually copied or the error + */ +static int gfs2_dir_read_data(struct gfs2_inode *ip, __be64 *buf, + unsigned int size) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + u64 lblock, dblock; + u32 extlen = 0; + unsigned int o; + int copied = 0; + int error = 0; + + if (gfs2_is_stuffed(ip)) + return gfs2_dir_read_stuffed(ip, buf, size); + + if (gfs2_assert_warn(sdp, gfs2_is_jdata(ip))) + return -EINVAL; + + lblock = 0; + o = do_div(lblock, sdp->sd_jbsize) + sizeof(struct gfs2_meta_header); + + while (copied < size) { + unsigned int amount; + struct buffer_head *bh; + + amount = size - copied; + if (amount > sdp->sd_sb.sb_bsize - o) + amount = sdp->sd_sb.sb_bsize - o; + + if (!extlen) { + extlen = 32; + error = gfs2_get_extent(&ip->i_inode, lblock, + &dblock, &extlen); + if (error || !dblock) + goto fail; + BUG_ON(extlen < 1); + bh = gfs2_meta_ra(ip->i_gl, dblock, extlen); + } else { + error = gfs2_meta_read(ip->i_gl, dblock, DIO_WAIT, 0, &bh); + if (error) + goto fail; + } + error = gfs2_metatype_check(sdp, bh, GFS2_METATYPE_JD); + if (error) { + brelse(bh); + goto fail; + } + dblock++; + extlen--; + memcpy(buf, bh->b_data + o, amount); + brelse(bh); + buf += (amount/sizeof(__be64)); + copied += amount; + lblock++; + o = sizeof(struct gfs2_meta_header); + } + + return copied; +fail: + return (copied) ? copied : error; +} + +/** + * gfs2_dir_get_hash_table - Get pointer to the dir hash table + * @ip: The inode in question + * + * Returns: The hash table or an error + */ + +static __be64 *gfs2_dir_get_hash_table(struct gfs2_inode *ip) +{ + struct inode *inode = &ip->i_inode; + int ret; + u32 hsize; + __be64 *hc; + + BUG_ON(!(ip->i_diskflags & GFS2_DIF_EXHASH)); + + hc = ip->i_hash_cache; + if (hc) + return hc; + + hsize = BIT(ip->i_depth); + hsize *= sizeof(__be64); + if (hsize != i_size_read(&ip->i_inode)) { + gfs2_consist_inode(ip); + return ERR_PTR(-EIO); + } + + hc = kmalloc(hsize, GFP_NOFS | __GFP_NOWARN); + if (hc == NULL) + hc = __vmalloc(hsize, GFP_NOFS); + + if (hc == NULL) + return ERR_PTR(-ENOMEM); + + ret = gfs2_dir_read_data(ip, hc, hsize); + if (ret < 0) { + kvfree(hc); + return ERR_PTR(ret); + } + + spin_lock(&inode->i_lock); + if (likely(!ip->i_hash_cache)) { + ip->i_hash_cache = hc; + hc = NULL; + } + spin_unlock(&inode->i_lock); + kvfree(hc); + + return ip->i_hash_cache; +} + +/** + * gfs2_dir_hash_inval - Invalidate dir hash + * @ip: The directory inode + * + * Must be called with an exclusive glock, or during glock invalidation. + */ +void gfs2_dir_hash_inval(struct gfs2_inode *ip) +{ + __be64 *hc; + + spin_lock(&ip->i_inode.i_lock); + hc = ip->i_hash_cache; + ip->i_hash_cache = NULL; + spin_unlock(&ip->i_inode.i_lock); + + kvfree(hc); +} + +static inline int gfs2_dirent_sentinel(const struct gfs2_dirent *dent) +{ + return dent->de_inum.no_addr == 0 || dent->de_inum.no_formal_ino == 0; +} + +static inline int __gfs2_dirent_find(const struct gfs2_dirent *dent, + const struct qstr *name, int ret) +{ + if (!gfs2_dirent_sentinel(dent) && + be32_to_cpu(dent->de_hash) == name->hash && + be16_to_cpu(dent->de_name_len) == name->len && + memcmp(dent+1, name->name, name->len) == 0) + return ret; + return 0; +} + +static int gfs2_dirent_find(const struct gfs2_dirent *dent, + const struct qstr *name, + void *opaque) +{ + return __gfs2_dirent_find(dent, name, 1); +} + +static int gfs2_dirent_prev(const struct gfs2_dirent *dent, + const struct qstr *name, + void *opaque) +{ + return __gfs2_dirent_find(dent, name, 2); +} + +/* + * name->name holds ptr to start of block. + * name->len holds size of block. + */ +static int gfs2_dirent_last(const struct gfs2_dirent *dent, + const struct qstr *name, + void *opaque) +{ + const char *start = name->name; + const char *end = (const char *)dent + be16_to_cpu(dent->de_rec_len); + if (name->len == (end - start)) + return 1; + return 0; +} + +/* Look for the dirent that contains the offset specified in data. Once we + * find that dirent, there must be space available there for the new dirent */ +static int gfs2_dirent_find_offset(const struct gfs2_dirent *dent, + const struct qstr *name, + void *ptr) +{ + unsigned required = GFS2_DIRENT_SIZE(name->len); + unsigned actual = GFS2_DIRENT_SIZE(be16_to_cpu(dent->de_name_len)); + unsigned totlen = be16_to_cpu(dent->de_rec_len); + + if (ptr < (void *)dent || ptr >= (void *)dent + totlen) + return 0; + if (gfs2_dirent_sentinel(dent)) + actual = 0; + if (ptr < (void *)dent + actual) + return -1; + if ((void *)dent + totlen >= ptr + required) + return 1; + return -1; +} + +static int gfs2_dirent_find_space(const struct gfs2_dirent *dent, + const struct qstr *name, + void *opaque) +{ + unsigned required = GFS2_DIRENT_SIZE(name->len); + unsigned actual = GFS2_DIRENT_SIZE(be16_to_cpu(dent->de_name_len)); + unsigned totlen = be16_to_cpu(dent->de_rec_len); + + if (gfs2_dirent_sentinel(dent)) + actual = 0; + if (totlen - actual >= required) + return 1; + return 0; +} + +struct dirent_gather { + const struct gfs2_dirent **pdent; + unsigned offset; +}; + +static int gfs2_dirent_gather(const struct gfs2_dirent *dent, + const struct qstr *name, + void *opaque) +{ + struct dirent_gather *g = opaque; + if (!gfs2_dirent_sentinel(dent)) { + g->pdent[g->offset++] = dent; + } + return 0; +} + +/* + * Other possible things to check: + * - Inode located within filesystem size (and on valid block) + * - Valid directory entry type + * Not sure how heavy-weight we want to make this... could also check + * hash is correct for example, but that would take a lot of extra time. + * For now the most important thing is to check that the various sizes + * are correct. + */ +static int gfs2_check_dirent(struct gfs2_sbd *sdp, + struct gfs2_dirent *dent, unsigned int offset, + unsigned int size, unsigned int len, int first) +{ + const char *msg = "gfs2_dirent too small"; + if (unlikely(size < sizeof(struct gfs2_dirent))) + goto error; + msg = "gfs2_dirent misaligned"; + if (unlikely(offset & 0x7)) + goto error; + msg = "gfs2_dirent points beyond end of block"; + if (unlikely(offset + size > len)) + goto error; + msg = "zero inode number"; + if (unlikely(!first && gfs2_dirent_sentinel(dent))) + goto error; + msg = "name length is greater than space in dirent"; + if (!gfs2_dirent_sentinel(dent) && + unlikely(sizeof(struct gfs2_dirent)+be16_to_cpu(dent->de_name_len) > + size)) + goto error; + return 0; +error: + fs_warn(sdp, "%s: %s (%s)\n", + __func__, msg, first ? "first in block" : "not first in block"); + return -EIO; +} + +static int gfs2_dirent_offset(struct gfs2_sbd *sdp, const void *buf) +{ + const struct gfs2_meta_header *h = buf; + int offset; + + BUG_ON(buf == NULL); + + switch(be32_to_cpu(h->mh_type)) { + case GFS2_METATYPE_LF: + offset = sizeof(struct gfs2_leaf); + break; + case GFS2_METATYPE_DI: + offset = sizeof(struct gfs2_dinode); + break; + default: + goto wrong_type; + } + return offset; +wrong_type: + fs_warn(sdp, "%s: wrong block type %u\n", __func__, + be32_to_cpu(h->mh_type)); + return -1; +} + +static struct gfs2_dirent *gfs2_dirent_scan(struct inode *inode, void *buf, + unsigned int len, gfs2_dscan_t scan, + const struct qstr *name, + void *opaque) +{ + struct gfs2_dirent *dent, *prev; + unsigned offset; + unsigned size; + int ret = 0; + + ret = gfs2_dirent_offset(GFS2_SB(inode), buf); + if (ret < 0) { + gfs2_consist_inode(GFS2_I(inode)); + return ERR_PTR(-EIO); + } + offset = ret; + prev = NULL; + dent = buf + offset; + size = be16_to_cpu(dent->de_rec_len); + if (gfs2_check_dirent(GFS2_SB(inode), dent, offset, size, len, 1)) { + gfs2_consist_inode(GFS2_I(inode)); + return ERR_PTR(-EIO); + } + do { + ret = scan(dent, name, opaque); + if (ret) + break; + offset += size; + if (offset == len) + break; + prev = dent; + dent = buf + offset; + size = be16_to_cpu(dent->de_rec_len); + if (gfs2_check_dirent(GFS2_SB(inode), dent, offset, size, + len, 0)) { + gfs2_consist_inode(GFS2_I(inode)); + return ERR_PTR(-EIO); + } + } while(1); + + switch(ret) { + case 0: + return NULL; + case 1: + return dent; + case 2: + return prev ? prev : dent; + default: + BUG_ON(ret > 0); + return ERR_PTR(ret); + } +} + +static int dirent_check_reclen(struct gfs2_inode *dip, + const struct gfs2_dirent *d, const void *end_p) +{ + const void *ptr = d; + u16 rec_len = be16_to_cpu(d->de_rec_len); + + if (unlikely(rec_len < sizeof(struct gfs2_dirent))) { + gfs2_consist_inode(dip); + return -EIO; + } + ptr += rec_len; + if (ptr < end_p) + return rec_len; + if (ptr == end_p) + return -ENOENT; + + gfs2_consist_inode(dip); + return -EIO; +} + +/** + * dirent_next - Next dirent + * @dip: the directory + * @bh: The buffer + * @dent: Pointer to list of dirents + * + * Returns: 0 on success, error code otherwise + */ + +static int dirent_next(struct gfs2_inode *dip, struct buffer_head *bh, + struct gfs2_dirent **dent) +{ + struct gfs2_dirent *cur = *dent, *tmp; + char *bh_end = bh->b_data + bh->b_size; + int ret; + + ret = dirent_check_reclen(dip, cur, bh_end); + if (ret < 0) + return ret; + + tmp = (void *)cur + ret; + ret = dirent_check_reclen(dip, tmp, bh_end); + if (ret == -EIO) + return ret; + + /* Only the first dent could ever have de_inum.no_addr == 0 */ + if (gfs2_dirent_sentinel(tmp)) { + gfs2_consist_inode(dip); + return -EIO; + } + + *dent = tmp; + return 0; +} + +/** + * dirent_del - Delete a dirent + * @dip: The GFS2 inode + * @bh: The buffer + * @prev: The previous dirent + * @cur: The current dirent + * + */ + +static void dirent_del(struct gfs2_inode *dip, struct buffer_head *bh, + struct gfs2_dirent *prev, struct gfs2_dirent *cur) +{ + u16 cur_rec_len, prev_rec_len; + + if (gfs2_dirent_sentinel(cur)) { + gfs2_consist_inode(dip); + return; + } + + gfs2_trans_add_meta(dip->i_gl, bh); + + /* If there is no prev entry, this is the first entry in the block. + The de_rec_len is already as big as it needs to be. Just zero + out the inode number and return. */ + + if (!prev) { + cur->de_inum.no_addr = 0; + cur->de_inum.no_formal_ino = 0; + return; + } + + /* Combine this dentry with the previous one. */ + + prev_rec_len = be16_to_cpu(prev->de_rec_len); + cur_rec_len = be16_to_cpu(cur->de_rec_len); + + if ((char *)prev + prev_rec_len != (char *)cur) + gfs2_consist_inode(dip); + if ((char *)cur + cur_rec_len > bh->b_data + bh->b_size) + gfs2_consist_inode(dip); + + prev_rec_len += cur_rec_len; + prev->de_rec_len = cpu_to_be16(prev_rec_len); +} + + +static struct gfs2_dirent *do_init_dirent(struct inode *inode, + struct gfs2_dirent *dent, + const struct qstr *name, + struct buffer_head *bh, + unsigned offset) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_dirent *ndent; + unsigned totlen; + + totlen = be16_to_cpu(dent->de_rec_len); + BUG_ON(offset + name->len > totlen); + gfs2_trans_add_meta(ip->i_gl, bh); + ndent = (struct gfs2_dirent *)((char *)dent + offset); + dent->de_rec_len = cpu_to_be16(offset); + gfs2_qstr2dirent(name, totlen - offset, ndent); + return ndent; +} + + +/* + * Takes a dent from which to grab space as an argument. Returns the + * newly created dent. + */ +static struct gfs2_dirent *gfs2_init_dirent(struct inode *inode, + struct gfs2_dirent *dent, + const struct qstr *name, + struct buffer_head *bh) +{ + unsigned offset = 0; + + if (!gfs2_dirent_sentinel(dent)) + offset = GFS2_DIRENT_SIZE(be16_to_cpu(dent->de_name_len)); + return do_init_dirent(inode, dent, name, bh, offset); +} + +static struct gfs2_dirent *gfs2_dirent_split_alloc(struct inode *inode, + struct buffer_head *bh, + const struct qstr *name, + void *ptr) +{ + struct gfs2_dirent *dent; + dent = gfs2_dirent_scan(inode, bh->b_data, bh->b_size, + gfs2_dirent_find_offset, name, ptr); + if (IS_ERR_OR_NULL(dent)) + return dent; + return do_init_dirent(inode, dent, name, bh, + (unsigned)(ptr - (void *)dent)); +} + +static int get_leaf(struct gfs2_inode *dip, u64 leaf_no, + struct buffer_head **bhp) +{ + int error; + + error = gfs2_meta_read(dip->i_gl, leaf_no, DIO_WAIT, 0, bhp); + if (!error && gfs2_metatype_check(GFS2_SB(&dip->i_inode), *bhp, GFS2_METATYPE_LF)) { + /* pr_info("block num=%llu\n", leaf_no); */ + error = -EIO; + } + + return error; +} + +/** + * get_leaf_nr - Get a leaf number associated with the index + * @dip: The GFS2 inode + * @index: hash table index of the targeted leaf + * @leaf_out: Resulting leaf block number + * + * Returns: 0 on success, error code otherwise + */ + +static int get_leaf_nr(struct gfs2_inode *dip, u32 index, u64 *leaf_out) +{ + __be64 *hash; + int error; + + hash = gfs2_dir_get_hash_table(dip); + error = PTR_ERR_OR_ZERO(hash); + + if (!error) + *leaf_out = be64_to_cpu(*(hash + index)); + + return error; +} + +static int get_first_leaf(struct gfs2_inode *dip, u32 index, + struct buffer_head **bh_out) +{ + u64 leaf_no; + int error; + + error = get_leaf_nr(dip, index, &leaf_no); + if (!error) + error = get_leaf(dip, leaf_no, bh_out); + + return error; +} + +static struct gfs2_dirent *gfs2_dirent_search(struct inode *inode, + const struct qstr *name, + gfs2_dscan_t scan, + struct buffer_head **pbh) +{ + struct buffer_head *bh; + struct gfs2_dirent *dent; + struct gfs2_inode *ip = GFS2_I(inode); + int error; + + if (ip->i_diskflags & GFS2_DIF_EXHASH) { + struct gfs2_leaf *leaf; + unsigned int hsize = BIT(ip->i_depth); + unsigned int index; + u64 ln; + if (hsize * sizeof(u64) != i_size_read(inode)) { + gfs2_consist_inode(ip); + return ERR_PTR(-EIO); + } + + index = name->hash >> (32 - ip->i_depth); + error = get_first_leaf(ip, index, &bh); + if (error) + return ERR_PTR(error); + do { + dent = gfs2_dirent_scan(inode, bh->b_data, bh->b_size, + scan, name, NULL); + if (dent) + goto got_dent; + leaf = (struct gfs2_leaf *)bh->b_data; + ln = be64_to_cpu(leaf->lf_next); + brelse(bh); + if (!ln) + break; + + error = get_leaf(ip, ln, &bh); + } while(!error); + + return error ? ERR_PTR(error) : NULL; + } + + + error = gfs2_meta_inode_buffer(ip, &bh); + if (error) + return ERR_PTR(error); + dent = gfs2_dirent_scan(inode, bh->b_data, bh->b_size, scan, name, NULL); +got_dent: + if (IS_ERR_OR_NULL(dent)) { + brelse(bh); + bh = NULL; + } + *pbh = bh; + return dent; +} + +static struct gfs2_leaf *new_leaf(struct inode *inode, struct buffer_head **pbh, u16 depth) +{ + struct gfs2_inode *ip = GFS2_I(inode); + unsigned int n = 1; + u64 bn; + int error; + struct buffer_head *bh; + struct gfs2_leaf *leaf; + struct gfs2_dirent *dent; + struct timespec64 tv = current_time(inode); + + error = gfs2_alloc_blocks(ip, &bn, &n, 0); + if (error) + return NULL; + bh = gfs2_meta_new(ip->i_gl, bn); + if (!bh) + return NULL; + + gfs2_trans_remove_revoke(GFS2_SB(inode), bn, 1); + gfs2_trans_add_meta(ip->i_gl, bh); + gfs2_metatype_set(bh, GFS2_METATYPE_LF, GFS2_FORMAT_LF); + leaf = (struct gfs2_leaf *)bh->b_data; + leaf->lf_depth = cpu_to_be16(depth); + leaf->lf_entries = 0; + leaf->lf_dirent_format = cpu_to_be32(GFS2_FORMAT_DE); + leaf->lf_next = 0; + leaf->lf_inode = cpu_to_be64(ip->i_no_addr); + leaf->lf_dist = cpu_to_be32(1); + leaf->lf_nsec = cpu_to_be32(tv.tv_nsec); + leaf->lf_sec = cpu_to_be64(tv.tv_sec); + memset(leaf->lf_reserved2, 0, sizeof(leaf->lf_reserved2)); + dent = (struct gfs2_dirent *)(leaf+1); + gfs2_qstr2dirent(&empty_name, bh->b_size - sizeof(struct gfs2_leaf), dent); + *pbh = bh; + return leaf; +} + +/** + * dir_make_exhash - Convert a stuffed directory into an ExHash directory + * @inode: The directory inode to be converted to exhash + * + * Returns: 0 on success, error code otherwise + */ + +static int dir_make_exhash(struct inode *inode) +{ + struct gfs2_inode *dip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + struct gfs2_dirent *dent; + struct qstr args; + struct buffer_head *bh, *dibh; + struct gfs2_leaf *leaf; + u32 x; + __be64 *lp; + u64 bn; + int error; + + error = gfs2_meta_inode_buffer(dip, &dibh); + if (error) + return error; + + /* Turn over a new leaf */ + + leaf = new_leaf(inode, &bh, 0); + if (!leaf) + return -ENOSPC; + bn = bh->b_blocknr; + + gfs2_assert(sdp, dip->i_entries < BIT(16)); + leaf->lf_entries = cpu_to_be16(dip->i_entries); + + /* Copy dirents */ + + gfs2_buffer_copy_tail(bh, sizeof(struct gfs2_leaf), dibh, + sizeof(struct gfs2_dinode)); + + /* Find last entry */ + + x = 0; + args.len = bh->b_size - sizeof(struct gfs2_dinode) + + sizeof(struct gfs2_leaf); + args.name = bh->b_data; + dent = gfs2_dirent_scan(&dip->i_inode, bh->b_data, bh->b_size, + gfs2_dirent_last, &args, NULL); + if (!dent) { + brelse(bh); + brelse(dibh); + return -EIO; + } + if (IS_ERR(dent)) { + brelse(bh); + brelse(dibh); + return PTR_ERR(dent); + } + + /* Adjust the last dirent's record length + (Remember that dent still points to the last entry.) */ + + dent->de_rec_len = cpu_to_be16(be16_to_cpu(dent->de_rec_len) + + sizeof(struct gfs2_dinode) - + sizeof(struct gfs2_leaf)); + + brelse(bh); + + /* We're done with the new leaf block, now setup the new + hash table. */ + + gfs2_trans_add_meta(dip->i_gl, dibh); + gfs2_buffer_clear_tail(dibh, sizeof(struct gfs2_dinode)); + + lp = (__be64 *)(dibh->b_data + sizeof(struct gfs2_dinode)); + + for (x = sdp->sd_hash_ptrs; x--; lp++) + *lp = cpu_to_be64(bn); + + i_size_write(inode, sdp->sd_sb.sb_bsize / 2); + gfs2_add_inode_blocks(&dip->i_inode, 1); + dip->i_diskflags |= GFS2_DIF_EXHASH; + dip->i_depth = ilog2(sdp->sd_hash_ptrs); + + gfs2_dinode_out(dip, dibh->b_data); + + brelse(dibh); + + return 0; +} + +/** + * dir_split_leaf - Split a leaf block into two + * @inode: The directory inode to be split + * @name: name of the dirent we're trying to insert + * + * Returns: 0 on success, error code on failure + */ + +static int dir_split_leaf(struct inode *inode, const struct qstr *name) +{ + struct gfs2_inode *dip = GFS2_I(inode); + struct buffer_head *nbh, *obh, *dibh; + struct gfs2_leaf *nleaf, *oleaf; + struct gfs2_dirent *dent = NULL, *prev = NULL, *next = NULL, *new; + u32 start, len, half_len, divider; + u64 bn, leaf_no; + __be64 *lp; + u32 index; + int x; + int error; + + index = name->hash >> (32 - dip->i_depth); + error = get_leaf_nr(dip, index, &leaf_no); + if (error) + return error; + + /* Get the old leaf block */ + error = get_leaf(dip, leaf_no, &obh); + if (error) + return error; + + oleaf = (struct gfs2_leaf *)obh->b_data; + if (dip->i_depth == be16_to_cpu(oleaf->lf_depth)) { + brelse(obh); + return 1; /* can't split */ + } + + gfs2_trans_add_meta(dip->i_gl, obh); + + nleaf = new_leaf(inode, &nbh, be16_to_cpu(oleaf->lf_depth) + 1); + if (!nleaf) { + brelse(obh); + return -ENOSPC; + } + bn = nbh->b_blocknr; + + /* Compute the start and len of leaf pointers in the hash table. */ + len = BIT(dip->i_depth - be16_to_cpu(oleaf->lf_depth)); + half_len = len >> 1; + if (!half_len) { + fs_warn(GFS2_SB(inode), "i_depth %u lf_depth %u index %u\n", + dip->i_depth, be16_to_cpu(oleaf->lf_depth), index); + gfs2_consist_inode(dip); + error = -EIO; + goto fail_brelse; + } + + start = (index & ~(len - 1)); + + /* Change the pointers. + Don't bother distinguishing stuffed from non-stuffed. + This code is complicated enough already. */ + lp = kmalloc_array(half_len, sizeof(__be64), GFP_NOFS); + if (!lp) { + error = -ENOMEM; + goto fail_brelse; + } + + /* Change the pointers */ + for (x = 0; x < half_len; x++) + lp[x] = cpu_to_be64(bn); + + gfs2_dir_hash_inval(dip); + + error = gfs2_dir_write_data(dip, (char *)lp, start * sizeof(u64), + half_len * sizeof(u64)); + if (error != half_len * sizeof(u64)) { + if (error >= 0) + error = -EIO; + goto fail_lpfree; + } + + kfree(lp); + + /* Compute the divider */ + divider = (start + half_len) << (32 - dip->i_depth); + + /* Copy the entries */ + dent = (struct gfs2_dirent *)(obh->b_data + sizeof(struct gfs2_leaf)); + + do { + next = dent; + if (dirent_next(dip, obh, &next)) + next = NULL; + + if (!gfs2_dirent_sentinel(dent) && + be32_to_cpu(dent->de_hash) < divider) { + struct qstr str; + void *ptr = ((char *)dent - obh->b_data) + nbh->b_data; + str.name = (char*)(dent+1); + str.len = be16_to_cpu(dent->de_name_len); + str.hash = be32_to_cpu(dent->de_hash); + new = gfs2_dirent_split_alloc(inode, nbh, &str, ptr); + if (IS_ERR(new)) { + error = PTR_ERR(new); + break; + } + + new->de_inum = dent->de_inum; /* No endian worries */ + new->de_type = dent->de_type; /* No endian worries */ + be16_add_cpu(&nleaf->lf_entries, 1); + + dirent_del(dip, obh, prev, dent); + + if (!oleaf->lf_entries) + gfs2_consist_inode(dip); + be16_add_cpu(&oleaf->lf_entries, -1); + + if (!prev) + prev = dent; + } else { + prev = dent; + } + dent = next; + } while (dent); + + oleaf->lf_depth = nleaf->lf_depth; + + error = gfs2_meta_inode_buffer(dip, &dibh); + if (!gfs2_assert_withdraw(GFS2_SB(&dip->i_inode), !error)) { + gfs2_trans_add_meta(dip->i_gl, dibh); + gfs2_add_inode_blocks(&dip->i_inode, 1); + gfs2_dinode_out(dip, dibh->b_data); + brelse(dibh); + } + + brelse(obh); + brelse(nbh); + + return error; + +fail_lpfree: + kfree(lp); + +fail_brelse: + brelse(obh); + brelse(nbh); + return error; +} + +/** + * dir_double_exhash - Double size of ExHash table + * @dip: The GFS2 dinode + * + * Returns: 0 on success, error code on failure + */ + +static int dir_double_exhash(struct gfs2_inode *dip) +{ + struct buffer_head *dibh; + u32 hsize; + u32 hsize_bytes; + __be64 *hc; + __be64 *hc2, *h; + int x; + int error = 0; + + hsize = BIT(dip->i_depth); + hsize_bytes = hsize * sizeof(__be64); + + hc = gfs2_dir_get_hash_table(dip); + if (IS_ERR(hc)) + return PTR_ERR(hc); + + hc2 = kmalloc_array(hsize_bytes, 2, GFP_NOFS | __GFP_NOWARN); + if (hc2 == NULL) + hc2 = __vmalloc(hsize_bytes * 2, GFP_NOFS); + + if (!hc2) + return -ENOMEM; + + h = hc2; + error = gfs2_meta_inode_buffer(dip, &dibh); + if (error) + goto out_kfree; + + for (x = 0; x < hsize; x++) { + *h++ = *hc; + *h++ = *hc; + hc++; + } + + error = gfs2_dir_write_data(dip, (char *)hc2, 0, hsize_bytes * 2); + if (error != (hsize_bytes * 2)) + goto fail; + + gfs2_dir_hash_inval(dip); + dip->i_hash_cache = hc2; + dip->i_depth++; + gfs2_dinode_out(dip, dibh->b_data); + brelse(dibh); + return 0; + +fail: + /* Replace original hash table & size */ + gfs2_dir_write_data(dip, (char *)hc, 0, hsize_bytes); + i_size_write(&dip->i_inode, hsize_bytes); + gfs2_dinode_out(dip, dibh->b_data); + brelse(dibh); +out_kfree: + kvfree(hc2); + return error; +} + +/** + * compare_dents - compare directory entries by hash value + * @a: first dent + * @b: second dent + * + * When comparing the hash entries of @a to @b: + * gt: returns 1 + * lt: returns -1 + * eq: returns 0 + */ + +static int compare_dents(const void *a, const void *b) +{ + const struct gfs2_dirent *dent_a, *dent_b; + u32 hash_a, hash_b; + int ret = 0; + + dent_a = *(const struct gfs2_dirent **)a; + hash_a = dent_a->de_cookie; + + dent_b = *(const struct gfs2_dirent **)b; + hash_b = dent_b->de_cookie; + + if (hash_a > hash_b) + ret = 1; + else if (hash_a < hash_b) + ret = -1; + else { + unsigned int len_a = be16_to_cpu(dent_a->de_name_len); + unsigned int len_b = be16_to_cpu(dent_b->de_name_len); + + if (len_a > len_b) + ret = 1; + else if (len_a < len_b) + ret = -1; + else + ret = memcmp(dent_a + 1, dent_b + 1, len_a); + } + + return ret; +} + +/** + * do_filldir_main - read out directory entries + * @dip: The GFS2 inode + * @ctx: what to feed the entries to + * @darr: an array of struct gfs2_dirent pointers to read + * @entries: the number of entries in darr + * @sort_start: index of the directory array to start our sort + * @copied: pointer to int that's non-zero if a entry has been copied out + * + * Jump through some hoops to make sure that if there are hash collsions, + * they are read out at the beginning of a buffer. We want to minimize + * the possibility that they will fall into different readdir buffers or + * that someone will want to seek to that location. + * + * Returns: errno, >0 if the actor tells you to stop + */ + +static int do_filldir_main(struct gfs2_inode *dip, struct dir_context *ctx, + struct gfs2_dirent **darr, u32 entries, + u32 sort_start, int *copied) +{ + const struct gfs2_dirent *dent, *dent_next; + u64 off, off_next; + unsigned int x, y; + int run = 0; + + if (sort_start < entries) + sort(&darr[sort_start], entries - sort_start, + sizeof(struct gfs2_dirent *), compare_dents, NULL); + + dent_next = darr[0]; + off_next = dent_next->de_cookie; + + for (x = 0, y = 1; x < entries; x++, y++) { + dent = dent_next; + off = off_next; + + if (y < entries) { + dent_next = darr[y]; + off_next = dent_next->de_cookie; + + if (off < ctx->pos) + continue; + ctx->pos = off; + + if (off_next == off) { + if (*copied && !run) + return 1; + run = 1; + } else + run = 0; + } else { + if (off < ctx->pos) + continue; + ctx->pos = off; + } + + if (!dir_emit(ctx, (const char *)(dent + 1), + be16_to_cpu(dent->de_name_len), + be64_to_cpu(dent->de_inum.no_addr), + be16_to_cpu(dent->de_type))) + return 1; + + *copied = 1; + } + + /* Increment the ctx->pos by one, so the next time we come into the + do_filldir fxn, we get the next entry instead of the last one in the + current leaf */ + + ctx->pos++; + + return 0; +} + +static void *gfs2_alloc_sort_buffer(unsigned size) +{ + void *ptr = NULL; + + if (size < KMALLOC_MAX_SIZE) + ptr = kmalloc(size, GFP_NOFS | __GFP_NOWARN); + if (!ptr) + ptr = __vmalloc(size, GFP_NOFS); + return ptr; +} + + +static int gfs2_set_cookies(struct gfs2_sbd *sdp, struct buffer_head *bh, + unsigned leaf_nr, struct gfs2_dirent **darr, + unsigned entries) +{ + int sort_id = -1; + int i; + + for (i = 0; i < entries; i++) { + unsigned offset; + + darr[i]->de_cookie = be32_to_cpu(darr[i]->de_hash); + darr[i]->de_cookie = gfs2_disk_hash2offset(darr[i]->de_cookie); + + if (!sdp->sd_args.ar_loccookie) + continue; + offset = (char *)(darr[i]) - + (bh->b_data + gfs2_dirent_offset(sdp, bh->b_data)); + offset /= GFS2_MIN_DIRENT_SIZE; + offset += leaf_nr * sdp->sd_max_dents_per_leaf; + if (offset >= GFS2_USE_HASH_FLAG || + leaf_nr >= GFS2_USE_HASH_FLAG) { + darr[i]->de_cookie |= GFS2_USE_HASH_FLAG; + if (sort_id < 0) + sort_id = i; + continue; + } + darr[i]->de_cookie &= GFS2_HASH_INDEX_MASK; + darr[i]->de_cookie |= offset; + } + return sort_id; +} + + +static int gfs2_dir_read_leaf(struct inode *inode, struct dir_context *ctx, + int *copied, unsigned *depth, + u64 leaf_no) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + struct buffer_head *bh; + struct gfs2_leaf *lf; + unsigned entries = 0, entries2 = 0; + unsigned leaves = 0, leaf = 0, offset, sort_offset; + struct gfs2_dirent **darr, *dent; + struct dirent_gather g; + struct buffer_head **larr; + int error, i, need_sort = 0, sort_id; + u64 lfn = leaf_no; + + do { + error = get_leaf(ip, lfn, &bh); + if (error) + goto out; + lf = (struct gfs2_leaf *)bh->b_data; + if (leaves == 0) + *depth = be16_to_cpu(lf->lf_depth); + entries += be16_to_cpu(lf->lf_entries); + leaves++; + lfn = be64_to_cpu(lf->lf_next); + brelse(bh); + } while(lfn); + + if (*depth < GFS2_DIR_MAX_DEPTH || !sdp->sd_args.ar_loccookie) { + need_sort = 1; + sort_offset = 0; + } + + if (!entries) + return 0; + + error = -ENOMEM; + /* + * The extra 99 entries are not normally used, but are a buffer + * zone in case the number of entries in the leaf is corrupt. + * 99 is the maximum number of entries that can fit in a single + * leaf block. + */ + larr = gfs2_alloc_sort_buffer((leaves + entries + 99) * sizeof(void *)); + if (!larr) + goto out; + darr = (struct gfs2_dirent **)(larr + leaves); + g.pdent = (const struct gfs2_dirent **)darr; + g.offset = 0; + lfn = leaf_no; + + do { + error = get_leaf(ip, lfn, &bh); + if (error) + goto out_free; + lf = (struct gfs2_leaf *)bh->b_data; + lfn = be64_to_cpu(lf->lf_next); + if (lf->lf_entries) { + offset = g.offset; + entries2 += be16_to_cpu(lf->lf_entries); + dent = gfs2_dirent_scan(inode, bh->b_data, bh->b_size, + gfs2_dirent_gather, NULL, &g); + error = PTR_ERR(dent); + if (IS_ERR(dent)) + goto out_free; + if (entries2 != g.offset) { + fs_warn(sdp, "Number of entries corrupt in dir " + "leaf %llu, entries2 (%u) != " + "g.offset (%u)\n", + (unsigned long long)bh->b_blocknr, + entries2, g.offset); + gfs2_consist_inode(ip); + error = -EIO; + goto out_free; + } + error = 0; + sort_id = gfs2_set_cookies(sdp, bh, leaf, &darr[offset], + be16_to_cpu(lf->lf_entries)); + if (!need_sort && sort_id >= 0) { + need_sort = 1; + sort_offset = offset + sort_id; + } + larr[leaf++] = bh; + } else { + larr[leaf++] = NULL; + brelse(bh); + } + } while(lfn); + + BUG_ON(entries2 != entries); + error = do_filldir_main(ip, ctx, darr, entries, need_sort ? + sort_offset : entries, copied); +out_free: + for(i = 0; i < leaf; i++) + brelse(larr[i]); + kvfree(larr); +out: + return error; +} + +/** + * gfs2_dir_readahead - Issue read-ahead requests for leaf blocks. + * @inode: the directory inode + * @hsize: hash table size + * @index: index into the hash table + * @f_ra: read-ahead parameters + * + * Note: we can't calculate each index like dir_e_read can because we don't + * have the leaf, and therefore we don't have the depth, and therefore we + * don't have the length. So we have to just read enough ahead to make up + * for the loss of information. + */ +static void gfs2_dir_readahead(struct inode *inode, unsigned hsize, u32 index, + struct file_ra_state *f_ra) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_glock *gl = ip->i_gl; + struct buffer_head *bh; + u64 blocknr = 0, last; + unsigned count; + + /* First check if we've already read-ahead for the whole range. */ + if (index + MAX_RA_BLOCKS < f_ra->start) + return; + + f_ra->start = max((pgoff_t)index, f_ra->start); + for (count = 0; count < MAX_RA_BLOCKS; count++) { + if (f_ra->start >= hsize) /* if exceeded the hash table */ + break; + + last = blocknr; + blocknr = be64_to_cpu(ip->i_hash_cache[f_ra->start]); + f_ra->start++; + if (blocknr == last) + continue; + + bh = gfs2_getbuf(gl, blocknr, 1); + if (trylock_buffer(bh)) { + if (buffer_uptodate(bh)) { + unlock_buffer(bh); + } else { + bh_submit(bh, REQ_OP_READ | REQ_RAHEAD | + REQ_META | REQ_PRIO, + bh_end_read); + } + } + put_bh(bh); + } +} + +/** + * dir_e_read - Reads the entries from a directory into a filldir buffer + * @inode: the directory inode + * @ctx: actor to feed the entries to + * @f_ra: read-ahead parameters + * + * Returns: errno + */ + +static int dir_e_read(struct inode *inode, struct dir_context *ctx, + struct file_ra_state *f_ra) +{ + struct gfs2_inode *dip = GFS2_I(inode); + u32 hsize, len = 0; + u32 hash, index; + __be64 *lp; + int copied = 0; + int error = 0; + unsigned depth = 0; + + hsize = BIT(dip->i_depth); + hash = gfs2_dir_offset2hash(ctx->pos); + index = hash >> (32 - dip->i_depth); + + if (dip->i_hash_cache == NULL) + f_ra->start = 0; + lp = gfs2_dir_get_hash_table(dip); + if (IS_ERR(lp)) + return PTR_ERR(lp); + + gfs2_dir_readahead(inode, hsize, index, f_ra); + + while (index < hsize) { + error = gfs2_dir_read_leaf(inode, ctx, + &copied, &depth, + be64_to_cpu(lp[index])); + if (error) + break; + + len = BIT(dip->i_depth - depth); + index = (index & ~(len - 1)) + len; + } + + if (error > 0) + error = 0; + return error; +} + +int gfs2_dir_read(struct inode *inode, struct dir_context *ctx, + struct file_ra_state *f_ra) +{ + struct gfs2_inode *dip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + struct dirent_gather g; + struct gfs2_dirent **darr, *dent; + struct buffer_head *dibh; + int copied = 0; + int error; + + if (!dip->i_entries) + return 0; + + if (dip->i_diskflags & GFS2_DIF_EXHASH) + return dir_e_read(inode, ctx, f_ra); + + if (!gfs2_is_stuffed(dip)) { + gfs2_consist_inode(dip); + return -EIO; + } + + error = gfs2_meta_inode_buffer(dip, &dibh); + if (error) + return error; + + error = -ENOMEM; + /* 96 is max number of dirents which can be stuffed into an inode */ + darr = kmalloc_objs(struct gfs2_dirent *, 96, GFP_NOFS); + if (darr) { + g.pdent = (const struct gfs2_dirent **)darr; + g.offset = 0; + dent = gfs2_dirent_scan(inode, dibh->b_data, dibh->b_size, + gfs2_dirent_gather, NULL, &g); + if (IS_ERR(dent)) { + error = PTR_ERR(dent); + goto out; + } + if (dip->i_entries != g.offset) { + fs_warn(sdp, "Number of entries corrupt in dir %llu, " + "ip->i_entries (%u) != g.offset (%u)\n", + (unsigned long long)dip->i_no_addr, + dip->i_entries, + g.offset); + gfs2_consist_inode(dip); + error = -EIO; + goto out; + } + gfs2_set_cookies(sdp, dibh, 0, darr, dip->i_entries); + error = do_filldir_main(dip, ctx, darr, + dip->i_entries, 0, &copied); +out: + kfree(darr); + } + + if (error > 0) + error = 0; + + brelse(dibh); + + return error; +} + +/** + * gfs2_dir_search - Search a directory + * @dir: The GFS2 directory inode + * @name: The name we are looking up + * @fail_on_exist: Fail if the name exists rather than looking it up + * + * This routine searches a directory for a file or another directory. + * Assumes a glock is held on dip. + * + * Returns: errno + */ + +struct inode *gfs2_dir_search(struct inode *dir, const struct qstr *name, + bool fail_on_exist) +{ + struct buffer_head *bh; + struct gfs2_dirent *dent; + u64 addr, formal_ino; + u16 dtype; + + dent = gfs2_dirent_search(dir, name, gfs2_dirent_find, &bh); + if (dent) { + struct inode *inode; + u16 rahead; + + if (IS_ERR(dent)) + return ERR_CAST(dent); + dtype = be16_to_cpu(dent->de_type); + rahead = be16_to_cpu(dent->de_rahead); + addr = be64_to_cpu(dent->de_inum.no_addr); + formal_ino = be64_to_cpu(dent->de_inum.no_formal_ino); + brelse(bh); + if (fail_on_exist) + return ERR_PTR(-EEXIST); + inode = gfs2_inode_lookup(dir->i_sb, dtype, addr, formal_ino, + GFS2_BLKST_FREE /* ignore */); + if (!IS_ERR(inode)) + GFS2_I(inode)->i_rahead = rahead; + return inode; + } + return ERR_PTR(-ENOENT); +} + +int gfs2_dir_check(struct inode *dir, const struct qstr *name, + const struct gfs2_inode *ip) +{ + struct buffer_head *bh; + struct gfs2_dirent *dent; + int ret = -ENOENT; + + dent = gfs2_dirent_search(dir, name, gfs2_dirent_find, &bh); + if (dent) { + if (IS_ERR(dent)) + return PTR_ERR(dent); + if (ip) { + if (be64_to_cpu(dent->de_inum.no_addr) != ip->i_no_addr) + goto out; + if (be64_to_cpu(dent->de_inum.no_formal_ino) != + ip->i_no_formal_ino) + goto out; + if (unlikely(IF2DT(ip->i_inode.i_mode) != + be16_to_cpu(dent->de_type))) { + gfs2_consist_inode(GFS2_I(dir)); + ret = -EIO; + goto out; + } + } + ret = 0; +out: + brelse(bh); + } + return ret; +} + +/** + * dir_new_leaf - Add a new leaf onto hash chain + * @inode: The directory + * @name: The name we are adding + * + * This adds a new dir leaf onto an existing leaf when there is not + * enough space to add a new dir entry. This is a last resort after + * we've expanded the hash table to max size and also split existing + * leaf blocks, so it will only occur for very large directories. + * + * The dist parameter is set to 1 for leaf blocks directly attached + * to the hash table, 2 for one layer of indirection, 3 for two layers + * etc. We are thus able to tell the difference between an old leaf + * with dist set to zero (i.e. "don't know") and a new one where we + * set this information for debug/fsck purposes. + * + * Returns: 0 on success, or -ve on error + */ + +static int dir_new_leaf(struct inode *inode, const struct qstr *name) +{ + struct buffer_head *bh, *obh; + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_leaf *leaf, *oleaf; + u32 dist = 1; + int error; + u32 index; + u64 bn; + + index = name->hash >> (32 - ip->i_depth); + error = get_first_leaf(ip, index, &obh); + if (error) + return error; + do { + dist++; + oleaf = (struct gfs2_leaf *)obh->b_data; + bn = be64_to_cpu(oleaf->lf_next); + if (!bn) + break; + brelse(obh); + error = get_leaf(ip, bn, &obh); + if (error) + return error; + } while(1); + + gfs2_trans_add_meta(ip->i_gl, obh); + + leaf = new_leaf(inode, &bh, be16_to_cpu(oleaf->lf_depth)); + if (!leaf) { + brelse(obh); + return -ENOSPC; + } + leaf->lf_dist = cpu_to_be32(dist); + oleaf->lf_next = cpu_to_be64(bh->b_blocknr); + brelse(bh); + brelse(obh); + + error = gfs2_meta_inode_buffer(ip, &bh); + if (error) + return error; + gfs2_trans_add_meta(ip->i_gl, bh); + gfs2_add_inode_blocks(&ip->i_inode, 1); + gfs2_dinode_out(ip, bh->b_data); + brelse(bh); + return 0; +} + +static u16 gfs2_inode_ra_len(const struct gfs2_inode *ip) +{ + u64 where = ip->i_no_addr + 1; + if (ip->i_eattr == where) + return 1; + return 0; +} + +/** + * gfs2_dir_add - Add new filename into directory + * @inode: The directory inode + * @name: The new name + * @nip: The GFS2 inode to be linked in to the directory + * @da: The directory addition info + * + * If the call to gfs2_diradd_alloc_required resulted in there being + * no need to allocate any new directory blocks, then it will contain + * a pointer to the directory entry and the bh in which it resides. We + * can use that without having to repeat the search. If there was no + * free space, then we must now create more space. + * + * Returns: 0 on success, error code on failure + */ + +int gfs2_dir_add(struct inode *inode, const struct qstr *name, + const struct gfs2_inode *nip, struct gfs2_diradd *da) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct buffer_head *bh = da->bh; + struct gfs2_dirent *dent = da->dent; + struct timespec64 tv; + struct gfs2_leaf *leaf; + int error; + + while(1) { + if (da->bh == NULL) { + dent = gfs2_dirent_search(inode, name, + gfs2_dirent_find_space, &bh); + } + if (dent) { + if (IS_ERR(dent)) + return PTR_ERR(dent); + dent = gfs2_init_dirent(inode, dent, name, bh); + gfs2_inum_out(nip, dent); + dent->de_type = cpu_to_be16(IF2DT(nip->i_inode.i_mode)); + dent->de_rahead = cpu_to_be16(gfs2_inode_ra_len(nip)); + tv = inode_set_ctime_current(&ip->i_inode); + if (ip->i_diskflags & GFS2_DIF_EXHASH) { + leaf = (struct gfs2_leaf *)bh->b_data; + be16_add_cpu(&leaf->lf_entries, 1); + leaf->lf_nsec = cpu_to_be32(tv.tv_nsec); + leaf->lf_sec = cpu_to_be64(tv.tv_sec); + } + da->dent = NULL; + da->bh = NULL; + brelse(bh); + ip->i_entries++; + inode_set_mtime_to_ts(&ip->i_inode, tv); + if (S_ISDIR(nip->i_inode.i_mode)) + inc_nlink(&ip->i_inode); + mark_inode_dirty(inode); + error = 0; + break; + } + if (!(ip->i_diskflags & GFS2_DIF_EXHASH)) { + error = dir_make_exhash(inode); + if (error) + break; + continue; + } + error = dir_split_leaf(inode, name); + if (error == 0) + continue; + if (error < 0) + break; + if (ip->i_depth < GFS2_DIR_MAX_DEPTH) { + error = dir_double_exhash(ip); + if (error) + break; + error = dir_split_leaf(inode, name); + if (error < 0) + break; + if (error == 0) + continue; + } + error = dir_new_leaf(inode, name); + if (!error) + continue; + error = -ENOSPC; + break; + } + return error; +} + + +/** + * gfs2_dir_del - Delete a directory entry + * @dip: The GFS2 inode + * @dentry: The directory entry we want to delete + * + * Returns: 0 on success, error code on failure + */ + +int gfs2_dir_del(struct gfs2_inode *dip, const struct dentry *dentry) +{ + const struct qstr *name = &dentry->d_name; + struct gfs2_dirent *dent, *prev = NULL; + struct buffer_head *bh; + struct timespec64 tv; + + /* Returns _either_ the entry (if its first in block) or the + previous entry otherwise */ + dent = gfs2_dirent_search(&dip->i_inode, name, gfs2_dirent_prev, &bh); + if (!dent) { + gfs2_consist_inode(dip); + return -EIO; + } + if (IS_ERR(dent)) { + gfs2_consist_inode(dip); + return PTR_ERR(dent); + } + /* If not first in block, adjust pointers accordingly */ + if (gfs2_dirent_find(dent, name, NULL) == 0) { + prev = dent; + dent = (struct gfs2_dirent *)((char *)dent + be16_to_cpu(prev->de_rec_len)); + } + + dirent_del(dip, bh, prev, dent); + tv = inode_set_ctime_current(&dip->i_inode); + if (dip->i_diskflags & GFS2_DIF_EXHASH) { + struct gfs2_leaf *leaf = (struct gfs2_leaf *)bh->b_data; + u16 entries = be16_to_cpu(leaf->lf_entries); + if (!entries) + gfs2_consist_inode(dip); + leaf->lf_entries = cpu_to_be16(--entries); + leaf->lf_nsec = cpu_to_be32(tv.tv_nsec); + leaf->lf_sec = cpu_to_be64(tv.tv_sec); + } + brelse(bh); + + if (!dip->i_entries) + gfs2_consist_inode(dip); + dip->i_entries--; + inode_set_mtime_to_ts(&dip->i_inode, tv); + if (d_is_dir(dentry)) + drop_nlink(&dip->i_inode); + mark_inode_dirty(&dip->i_inode); + + return 0; +} + +/** + * gfs2_dir_mvino - Change inode number of directory entry + * @dip: The GFS2 directory inode + * @filename: the filename to be moved + * @nip: the new GFS2 inode + * @new_type: the de_type of the new dirent + * + * This routine changes the inode number of a directory entry. It's used + * by rename to change ".." when a directory is moved. + * Assumes a glock is held on dvp. + * + * Returns: errno + */ + +int gfs2_dir_mvino(struct gfs2_inode *dip, const struct qstr *filename, + const struct gfs2_inode *nip, unsigned int new_type) +{ + struct buffer_head *bh; + struct gfs2_dirent *dent; + + dent = gfs2_dirent_search(&dip->i_inode, filename, gfs2_dirent_find, &bh); + if (!dent) { + gfs2_consist_inode(dip); + return -EIO; + } + if (IS_ERR(dent)) + return PTR_ERR(dent); + + gfs2_trans_add_meta(dip->i_gl, bh); + gfs2_inum_out(nip, dent); + dent->de_type = cpu_to_be16(new_type); + brelse(bh); + + inode_set_mtime_to_ts(&dip->i_inode, inode_set_ctime_current(&dip->i_inode)); + mark_inode_dirty_sync(&dip->i_inode); + return 0; +} + +/** + * leaf_dealloc - Deallocate a directory leaf + * @dip: the directory + * @index: the hash table offset in the directory + * @len: the number of pointers to this leaf + * @leaf_no: the leaf number + * @leaf_bh: buffer_head for the starting leaf + * @last_dealloc: 1 if this is the final dealloc for the leaf, else 0 + * + * Returns: errno + */ + +static int leaf_dealloc(struct gfs2_inode *dip, u32 index, u32 len, + u64 leaf_no, struct buffer_head *leaf_bh, + int last_dealloc) +{ + struct gfs2_sbd *sdp = GFS2_SB(&dip->i_inode); + struct gfs2_leaf *tmp_leaf; + struct gfs2_rgrp_list rlist; + struct buffer_head *bh, *dibh; + u64 blk, nblk; + unsigned int rg_blocks = 0, l_blocks = 0; + char *ht; + unsigned int x, size = len * sizeof(u64); + int error; + + error = gfs2_rindex_update(sdp); + if (error) + return error; + + memset(&rlist, 0, sizeof(struct gfs2_rgrp_list)); + + ht = kzalloc(size, GFP_NOFS | __GFP_NOWARN); + if (ht == NULL) + ht = __vmalloc(size, GFP_NOFS | __GFP_NOWARN | __GFP_ZERO); + if (!ht) + return -ENOMEM; + + error = gfs2_quota_hold(dip, NO_UID_QUOTA_CHANGE, NO_GID_QUOTA_CHANGE); + if (error) + goto out; + + /* Count the number of leaves */ + bh = leaf_bh; + + for (blk = leaf_no; blk; blk = nblk) { + if (blk != leaf_no) { + error = get_leaf(dip, blk, &bh); + if (error) + goto out_rlist; + } + tmp_leaf = (struct gfs2_leaf *)bh->b_data; + nblk = be64_to_cpu(tmp_leaf->lf_next); + if (blk != leaf_no) + brelse(bh); + + gfs2_rlist_add(dip, &rlist, blk); + l_blocks++; + } + + gfs2_rlist_alloc(&rlist, LM_ST_EXCLUSIVE, LM_FLAG_NODE_SCOPE); + + for (x = 0; x < rlist.rl_rgrps; x++) { + struct gfs2_rgrpd *rgd = gfs2_glock2rgrp(rlist.rl_ghs[x].gh_gl); + + rg_blocks += rgd->rd_length; + } + + error = gfs2_glock_nq_m(rlist.rl_rgrps, rlist.rl_ghs); + if (error) + goto out_rlist; + + error = gfs2_trans_begin(sdp, + rg_blocks + (DIV_ROUND_UP(size, sdp->sd_jbsize) + 1) + + RES_DINODE + RES_STATFS + RES_QUOTA, RES_DINODE + + l_blocks); + if (error) + goto out_rg_gunlock; + + bh = leaf_bh; + + for (blk = leaf_no; blk; blk = nblk) { + struct gfs2_rgrpd *rgd; + + if (blk != leaf_no) { + error = get_leaf(dip, blk, &bh); + if (error) + goto out_end_trans; + } + tmp_leaf = (struct gfs2_leaf *)bh->b_data; + nblk = be64_to_cpu(tmp_leaf->lf_next); + if (blk != leaf_no) + brelse(bh); + + rgd = gfs2_blk2rgrpd(sdp, blk, true); + gfs2_free_meta(dip, rgd, blk, 1); + gfs2_add_inode_blocks(&dip->i_inode, -1); + } + + error = gfs2_dir_write_data(dip, ht, index * sizeof(u64), size); + if (error != size) { + if (error >= 0) + error = -EIO; + goto out_end_trans; + } + + error = gfs2_meta_inode_buffer(dip, &dibh); + if (error) + goto out_end_trans; + + gfs2_trans_add_meta(dip->i_gl, dibh); + /* On the last dealloc, make this a regular file in case we crash. + (We don't want to free these blocks a second time.) */ + if (last_dealloc) + dip->i_inode.i_mode = S_IFREG; + gfs2_dinode_out(dip, dibh->b_data); + brelse(dibh); + +out_end_trans: + gfs2_trans_end(sdp); +out_rg_gunlock: + gfs2_glock_dq_m(rlist.rl_rgrps, rlist.rl_ghs); +out_rlist: + gfs2_rlist_free(&rlist); + gfs2_quota_unhold(dip); +out: + kvfree(ht); + return error; +} + +/** + * gfs2_dir_exhash_dealloc - free all the leaf blocks in a directory + * @dip: the directory + * + * Dealloc all on-disk directory leaves to FREEMETA state + * Change on-disk inode type to "regular file" + * + * Returns: errno + */ + +int gfs2_dir_exhash_dealloc(struct gfs2_inode *dip) +{ + struct buffer_head *bh; + struct gfs2_leaf *leaf; + u32 hsize, len; + u32 index = 0, next_index; + __be64 *lp; + u64 leaf_no; + int error = 0, last; + + hsize = BIT(dip->i_depth); + + lp = gfs2_dir_get_hash_table(dip); + if (IS_ERR(lp)) + return PTR_ERR(lp); + + while (index < hsize) { + leaf_no = be64_to_cpu(lp[index]); + if (leaf_no) { + error = get_leaf(dip, leaf_no, &bh); + if (error) + goto out; + leaf = (struct gfs2_leaf *)bh->b_data; + len = BIT(dip->i_depth - be16_to_cpu(leaf->lf_depth)); + + next_index = (index & ~(len - 1)) + len; + last = ((next_index >= hsize) ? 1 : 0); + error = leaf_dealloc(dip, index, len, leaf_no, bh, + last); + brelse(bh); + if (error) + goto out; + index = next_index; + } else + index++; + } + + if (index != hsize) { + gfs2_consist_inode(dip); + error = -EIO; + } + +out: + + return error; +} + +/** + * gfs2_diradd_alloc_required - find if adding entry will require an allocation + * @inode: the directory inode being written to + * @name: the filename that's going to be added + * @da: The structure to return dir alloc info + * + * Returns: 0 if ok, -ve on error + */ + +int gfs2_diradd_alloc_required(struct inode *inode, const struct qstr *name, + struct gfs2_diradd *da) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + const unsigned int extra = sizeof(struct gfs2_dinode) - sizeof(struct gfs2_leaf); + struct gfs2_dirent *dent; + struct buffer_head *bh; + + da->nr_blocks = 0; + da->bh = NULL; + da->dent = NULL; + + dent = gfs2_dirent_search(inode, name, gfs2_dirent_find_space, &bh); + if (!dent) { + da->nr_blocks = sdp->sd_max_dirres; + if (!(ip->i_diskflags & GFS2_DIF_EXHASH) && + (GFS2_DIRENT_SIZE(name->len) < extra)) + da->nr_blocks = 1; + return 0; + } + if (IS_ERR(dent)) + return PTR_ERR(dent); + + if (da->save_loc) { + da->bh = bh; + da->dent = dent; + } else { + brelse(bh); + } + return 0; +} + diff --git a/fs/gfs2/dir.h b/fs/gfs2/dir.h new file mode 100644 index 000000000..25a857c78 --- /dev/null +++ b/fs/gfs2/dir.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#ifndef __DIR_DOT_H__ +#define __DIR_DOT_H__ + +#include +#include + +struct inode; +struct gfs2_inode; +struct gfs2_inum; +struct buffer_head; +struct gfs2_dirent; + +struct gfs2_diradd { + unsigned nr_blocks; + struct gfs2_dirent *dent; + struct buffer_head *bh; + int save_loc; +}; + +struct inode *gfs2_dir_search(struct inode *dir, + const struct qstr *filename, + bool fail_on_exist); +int gfs2_dir_check(struct inode *dir, const struct qstr *filename, + const struct gfs2_inode *ip); +int gfs2_dir_add(struct inode *inode, const struct qstr *filename, + const struct gfs2_inode *ip, struct gfs2_diradd *da); +static inline void gfs2_dir_no_add(struct gfs2_diradd *da) +{ + brelse(da->bh); + da->bh = NULL; +} +int gfs2_dir_del(struct gfs2_inode *dip, const struct dentry *dentry); +int gfs2_dir_read(struct inode *inode, struct dir_context *ctx, + struct file_ra_state *f_ra); +int gfs2_dir_mvino(struct gfs2_inode *dip, const struct qstr *filename, + const struct gfs2_inode *nip, unsigned int new_type); + +int gfs2_dir_exhash_dealloc(struct gfs2_inode *dip); + +int gfs2_diradd_alloc_required(struct inode *dir, + const struct qstr *filename, + struct gfs2_diradd *da); +int gfs2_dir_get_new_buffer(struct gfs2_inode *ip, u64 block, + struct buffer_head **bhp); +void gfs2_dir_hash_inval(struct gfs2_inode *ip); + +static inline u32 gfs2_disk_hash(const char *data, int len) +{ + return crc32_le((u32)~0, data, len) ^ (u32)~0; +} + + +static inline void gfs2_str2qstr(struct qstr *name, const char *fname) +{ + name->name = fname; + name->len = strlen(fname); + name->hash = gfs2_disk_hash(name->name, name->len); +} + +/* N.B. This probably ought to take inum & type as args as well */ +static inline void gfs2_qstr2dirent(const struct qstr *name, u16 reclen, struct gfs2_dirent *dent) +{ + dent->de_inum.no_addr = cpu_to_be64(0); + dent->de_inum.no_formal_ino = cpu_to_be64(0); + dent->de_hash = cpu_to_be32(name->hash); + dent->de_rec_len = cpu_to_be16(reclen); + dent->de_name_len = cpu_to_be16(name->len); + dent->de_type = cpu_to_be16(0); + memset(dent->__pad, 0, sizeof(dent->__pad)); + memcpy(dent + 1, name->name, name->len); +} + +extern struct qstr gfs2_qdot; +extern struct qstr gfs2_qdotdot; + +#endif /* __DIR_DOT_H__ */ diff --git a/fs/gfs2/export.c b/fs/gfs2/export.c new file mode 100644 index 000000000..3334c394c --- /dev/null +++ b/fs/gfs2/export.c @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include + +#include "gfs2.h" +#include "incore.h" +#include "dir.h" +#include "glock.h" +#include "glops.h" +#include "inode.h" +#include "super.h" +#include "rgrp.h" +#include "util.h" + +#define GFS2_SMALL_FH_SIZE 4 +#define GFS2_LARGE_FH_SIZE 8 +#define GFS2_OLD_FH_SIZE 10 + +static int gfs2_encode_fh(struct inode *inode, __u32 *p, int *len, + struct inode *parent) +{ + __be32 *fh = (__force __be32 *)p; + struct super_block *sb = inode->i_sb; + struct gfs2_inode *ip = GFS2_I(inode); + + if (parent && (*len < GFS2_LARGE_FH_SIZE)) { + *len = GFS2_LARGE_FH_SIZE; + return FILEID_INVALID; + } else if (*len < GFS2_SMALL_FH_SIZE) { + *len = GFS2_SMALL_FH_SIZE; + return FILEID_INVALID; + } + + fh[0] = cpu_to_be32(ip->i_no_formal_ino >> 32); + fh[1] = cpu_to_be32(ip->i_no_formal_ino & 0xFFFFFFFF); + fh[2] = cpu_to_be32(ip->i_no_addr >> 32); + fh[3] = cpu_to_be32(ip->i_no_addr & 0xFFFFFFFF); + *len = GFS2_SMALL_FH_SIZE; + + if (!parent || inode == d_inode(sb->s_root)) + return *len; + + ip = GFS2_I(parent); + + fh[4] = cpu_to_be32(ip->i_no_formal_ino >> 32); + fh[5] = cpu_to_be32(ip->i_no_formal_ino & 0xFFFFFFFF); + fh[6] = cpu_to_be32(ip->i_no_addr >> 32); + fh[7] = cpu_to_be32(ip->i_no_addr & 0xFFFFFFFF); + *len = GFS2_LARGE_FH_SIZE; + + return *len; +} + +struct get_name_filldir { + struct dir_context ctx; + struct gfs2_inum_host inum; + char *name; +}; + +static bool get_name_filldir(struct dir_context *ctx, const char *name, + int length, loff_t offset, u64 inum, + unsigned int type) +{ + struct get_name_filldir *gnfd = + container_of(ctx, struct get_name_filldir, ctx); + + if (inum != gnfd->inum.no_addr) + return true; + + memcpy(gnfd->name, name, length); + gnfd->name[length] = 0; + + return false; +} + +static int gfs2_get_name(struct dentry *parent, char *name, + struct dentry *child) +{ + struct inode *dir = d_inode(parent); + struct inode *inode = d_inode(child); + struct gfs2_inode *dip, *ip; + struct get_name_filldir gnfd = { + .ctx.actor = get_name_filldir, + .name = name + }; + struct gfs2_holder gh; + int error; + struct file_ra_state f_ra = { .start = 0 }; + + if (!dir) + return -EINVAL; + + if (!S_ISDIR(dir->i_mode) || !inode) + return -EINVAL; + + dip = GFS2_I(dir); + ip = GFS2_I(inode); + + *name = 0; + gnfd.inum.no_addr = ip->i_no_addr; + gnfd.inum.no_formal_ino = ip->i_no_formal_ino; + + error = gfs2_glock_nq_init(dip->i_gl, LM_ST_SHARED, 0, &gh); + if (error) + return error; + + error = gfs2_dir_read(dir, &gnfd.ctx, &f_ra); + + gfs2_glock_dq_uninit(&gh); + + if (!error && !*name) + error = -ENOENT; + + return error; +} + +static struct dentry *gfs2_get_parent(struct dentry *child) +{ + return d_obtain_alias(gfs2_lookupi(d_inode(child), &gfs2_qdotdot, 1)); +} + +static struct dentry *gfs2_get_dentry(struct super_block *sb, + struct gfs2_inum_host *inum) +{ + struct gfs2_sbd *sdp = sb->s_fs_info; + struct inode *inode; + + if (!inum->no_formal_ino) + return ERR_PTR(-ESTALE); + inode = gfs2_lookup_by_inum(sdp, inum->no_addr, inum->no_formal_ino, + GFS2_BLKST_DINODE); + return d_obtain_alias(inode); +} + +static struct dentry *gfs2_fh_to_dentry(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + struct gfs2_inum_host this; + __be32 *fh = (__force __be32 *)fid->raw; + + switch (fh_type) { + case GFS2_SMALL_FH_SIZE: + case GFS2_LARGE_FH_SIZE: + case GFS2_OLD_FH_SIZE: + if (fh_len < GFS2_SMALL_FH_SIZE) + return NULL; + this.no_formal_ino = ((u64)be32_to_cpu(fh[0])) << 32; + this.no_formal_ino |= be32_to_cpu(fh[1]); + this.no_addr = ((u64)be32_to_cpu(fh[2])) << 32; + this.no_addr |= be32_to_cpu(fh[3]); + return gfs2_get_dentry(sb, &this); + default: + return NULL; + } +} + +static struct dentry *gfs2_fh_to_parent(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + struct gfs2_inum_host parent; + __be32 *fh = (__force __be32 *)fid->raw; + + switch (fh_type) { + case GFS2_LARGE_FH_SIZE: + case GFS2_OLD_FH_SIZE: + if (fh_len < GFS2_LARGE_FH_SIZE) + return NULL; + parent.no_formal_ino = ((u64)be32_to_cpu(fh[4])) << 32; + parent.no_formal_ino |= be32_to_cpu(fh[5]); + parent.no_addr = ((u64)be32_to_cpu(fh[6])) << 32; + parent.no_addr |= be32_to_cpu(fh[7]); + return gfs2_get_dentry(sb, &parent); + default: + return NULL; + } +} + +const struct export_operations gfs2_export_ops = { + .encode_fh = gfs2_encode_fh, + .fh_to_dentry = gfs2_fh_to_dentry, + .fh_to_parent = gfs2_fh_to_parent, + .get_name = gfs2_get_name, + .get_parent = gfs2_get_parent, +}; + diff --git a/fs/gfs2/file.c b/fs/gfs2/file.c new file mode 100644 index 000000000..b8c10de11 --- /dev/null +++ b/fs/gfs2/file.c @@ -0,0 +1,1651 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gfs2.h" +#include "incore.h" +#include "bmap.h" +#include "aops.h" +#include "dir.h" +#include "glock.h" +#include "glops.h" +#include "inode.h" +#include "log.h" +#include "meta_io.h" +#include "quota.h" +#include "rgrp.h" +#include "trans.h" +#include "util.h" + +/** + * gfs2_llseek - seek to a location in a file + * @file: the file + * @offset: the offset + * @whence: Where to seek from (SEEK_SET, SEEK_CUR, or SEEK_END) + * + * SEEK_END requires the glock for the file because it references the + * file's size. + * + * Returns: The new offset, or errno + */ + +static loff_t gfs2_llseek(struct file *file, loff_t offset, int whence) +{ + struct gfs2_inode *ip = GFS2_I(file->f_mapping->host); + struct gfs2_holder i_gh; + loff_t error; + + switch (whence) { + case SEEK_END: + error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, LM_FLAG_ANY, + &i_gh); + if (!error) { + error = generic_file_llseek(file, offset, whence); + gfs2_glock_dq_uninit(&i_gh); + } + break; + + case SEEK_DATA: + error = gfs2_seek_data(file, offset); + break; + + case SEEK_HOLE: + error = gfs2_seek_hole(file, offset); + break; + + case SEEK_CUR: + case SEEK_SET: + /* + * These don't reference inode->i_size and don't depend on the + * block mapping, so we don't need the glock. + */ + error = generic_file_llseek(file, offset, whence); + break; + default: + error = -EINVAL; + } + + return error; +} + +/** + * gfs2_readdir - Iterator for a directory + * @file: The directory to read from + * @ctx: What to feed directory entries to + * + * Returns: errno + */ + +static int gfs2_readdir(struct file *file, struct dir_context *ctx) +{ + struct inode *dir = file->f_mapping->host; + struct gfs2_inode *dip = GFS2_I(dir); + struct gfs2_holder d_gh; + int error; + + error = gfs2_glock_nq_init(dip->i_gl, LM_ST_SHARED, 0, &d_gh); + if (error) + return error; + + error = gfs2_dir_read(dir, ctx, &file->f_ra); + + gfs2_glock_dq_uninit(&d_gh); + + return error; +} + +/* + * struct fsflag_gfs2flag + * + * The FS_JOURNAL_DATA_FL flag maps to GFS2_DIF_INHERIT_JDATA for directories, + * and to GFS2_DIF_JDATA for non-directories. + */ +static struct { + u32 fsflag; + u32 gfsflag; +} fsflag_gfs2flag[] = { + {FS_SYNC_FL, GFS2_DIF_SYNC}, + {FS_IMMUTABLE_FL, GFS2_DIF_IMMUTABLE}, + {FS_APPEND_FL, GFS2_DIF_APPENDONLY}, + {FS_NOATIME_FL, GFS2_DIF_NOATIME}, + {FS_INDEX_FL, GFS2_DIF_EXHASH}, + {FS_TOPDIR_FL, GFS2_DIF_TOPDIR}, + {FS_JOURNAL_DATA_FL, GFS2_DIF_JDATA | GFS2_DIF_INHERIT_JDATA}, +}; + +static inline u32 gfs2_gfsflags_to_fsflags(struct inode *inode, u32 gfsflags) +{ + int i; + u32 fsflags = 0; + + if (S_ISDIR(inode->i_mode)) + gfsflags &= ~GFS2_DIF_JDATA; + else + gfsflags &= ~GFS2_DIF_INHERIT_JDATA; + + for (i = 0; i < ARRAY_SIZE(fsflag_gfs2flag); i++) + if (gfsflags & fsflag_gfs2flag[i].gfsflag) + fsflags |= fsflag_gfs2flag[i].fsflag; + return fsflags; +} + +int gfs2_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_holder gh; + int error; + u32 fsflags; + + if (d_is_special(dentry)) + return -ENOTTY; + + gfs2_holder_init(ip->i_gl, LM_ST_SHARED, 0, &gh); + error = gfs2_glock_nq(&gh); + if (error) + goto out_uninit; + + fsflags = gfs2_gfsflags_to_fsflags(inode, ip->i_diskflags); + + fileattr_fill_flags(fa, fsflags); + + gfs2_glock_dq(&gh); +out_uninit: + gfs2_holder_uninit(&gh); + return error; +} + +void gfs2_set_inode_flags(struct inode *inode) +{ + struct gfs2_inode *ip = GFS2_I(inode); + unsigned int flags = inode->i_flags; + + flags &= ~(S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC|S_NOSEC); + if ((ip->i_eattr == 0) && !is_sxid(inode->i_mode)) + flags |= S_NOSEC; + if (ip->i_diskflags & GFS2_DIF_IMMUTABLE) + flags |= S_IMMUTABLE; + if (ip->i_diskflags & GFS2_DIF_APPENDONLY) + flags |= S_APPEND; + if (ip->i_diskflags & GFS2_DIF_NOATIME) + flags |= S_NOATIME; + if (ip->i_diskflags & GFS2_DIF_SYNC) + flags |= S_SYNC; + inode->i_flags = flags; +} + +/* Flags that can be set by user space */ +#define GFS2_FLAGS_USER_SET (GFS2_DIF_JDATA| \ + GFS2_DIF_IMMUTABLE| \ + GFS2_DIF_APPENDONLY| \ + GFS2_DIF_NOATIME| \ + GFS2_DIF_SYNC| \ + GFS2_DIF_TOPDIR| \ + GFS2_DIF_INHERIT_JDATA) + +/** + * do_gfs2_set_flags - set flags on an inode + * @inode: The inode + * @reqflags: The flags to set + * @mask: Indicates which flags are valid + * + */ +static int do_gfs2_set_flags(struct inode *inode, u32 reqflags, u32 mask) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + struct buffer_head *bh; + struct gfs2_holder gh; + int error; + u32 new_flags, flags; + + error = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &gh); + if (error) + return error; + + error = 0; + flags = ip->i_diskflags; + new_flags = (flags & ~mask) | (reqflags & mask); + if ((new_flags ^ flags) == 0) + goto out; + + if (!IS_IMMUTABLE(inode)) { + error = gfs2_permission(&nop_mnt_idmap, inode, MAY_WRITE); + if (error) + goto out; + } + if ((flags ^ new_flags) & GFS2_DIF_JDATA) { + if (new_flags & GFS2_DIF_JDATA) + gfs2_log_flush(sdp, ip->i_gl, + GFS2_LOG_HEAD_FLUSH_NORMAL | + GFS2_LFC_SET_FLAGS); + error = filemap_fdatawrite(inode->i_mapping); + if (error) + goto out; + error = filemap_fdatawait(inode->i_mapping); + if (error) + goto out; + truncate_inode_pages(inode->i_mapping, 0); + if (new_flags & GFS2_DIF_JDATA) + gfs2_ordered_del_inode(ip); + } + error = gfs2_trans_begin(sdp, RES_DINODE, 0); + if (error) + goto out; + error = gfs2_meta_inode_buffer(ip, &bh); + if (error) + goto out_trans_end; + inode_set_ctime_current(inode); + gfs2_trans_add_meta(ip->i_gl, bh); + ip->i_diskflags = new_flags; + gfs2_dinode_out(ip, bh->b_data); + brelse(bh); + gfs2_set_inode_flags(inode); + gfs2_set_aops(inode); +out_trans_end: + gfs2_trans_end(sdp); +out: + gfs2_glock_dq_uninit(&gh); + return error; +} + +int gfs2_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + u32 fsflags = fa->flags, gfsflags = 0; + u32 mask; + int i; + + if (d_is_special(dentry)) + return -ENOTTY; + + if (fileattr_has_fsx(fa)) + return -EOPNOTSUPP; + + for (i = 0; i < ARRAY_SIZE(fsflag_gfs2flag); i++) { + if (fsflags & fsflag_gfs2flag[i].fsflag) { + fsflags &= ~fsflag_gfs2flag[i].fsflag; + gfsflags |= fsflag_gfs2flag[i].gfsflag; + } + } + if (fsflags || gfsflags & ~GFS2_FLAGS_USER_SET) + return -EINVAL; + + mask = GFS2_FLAGS_USER_SET; + if (S_ISDIR(inode->i_mode)) { + mask &= ~GFS2_DIF_JDATA; + } else { + /* The GFS2_DIF_TOPDIR flag is only valid for directories. */ + if (gfsflags & GFS2_DIF_TOPDIR) + return -EINVAL; + mask &= ~(GFS2_DIF_TOPDIR | GFS2_DIF_INHERIT_JDATA); + } + + return do_gfs2_set_flags(inode, gfsflags, mask); +} + +static int gfs2_getlabel(struct file *filp, char __user *label) +{ + struct inode *inode = file_inode(filp); + struct gfs2_sbd *sdp = GFS2_SB(inode); + + if (copy_to_user(label, sdp->sd_sb.sb_locktable, GFS2_LOCKNAME_LEN)) + return -EFAULT; + + return 0; +} + +static long gfs2_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + switch(cmd) { + case FITRIM: + return gfs2_fitrim(filp, (void __user *)arg); + case FS_IOC_GETFSLABEL: + return gfs2_getlabel(filp, (char __user *)arg); + } + + return -ENOTTY; +} + +#ifdef CONFIG_COMPAT +static long gfs2_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + switch(cmd) { + /* Keep this list in sync with gfs2_ioctl */ + case FITRIM: + case FS_IOC_GETFSLABEL: + break; + default: + return -ENOIOCTLCMD; + } + + return gfs2_ioctl(filp, cmd, (unsigned long)compat_ptr(arg)); +} +#else +#define gfs2_compat_ioctl NULL +#endif + +/** + * gfs2_size_hint - Give a hint to the size of a write request + * @filep: The struct file + * @offset: The file offset of the write + * @size: The length of the write + * + * When we are about to do a write, this function records the total + * write size in order to provide a suitable hint to the lower layers + * about how many blocks will be required. + * + */ + +static void gfs2_size_hint(struct file *filep, loff_t offset, size_t size) +{ + struct inode *inode = file_inode(filep); + struct gfs2_sbd *sdp = GFS2_SB(inode); + struct gfs2_inode *ip = GFS2_I(inode); + size_t blks = (size + sdp->sd_sb.sb_bsize - 1) >> sdp->sd_sb.sb_bsize_shift; + int hint = min_t(size_t, INT_MAX, blks); + + if (hint > atomic_read(&ip->i_sizehint)) + atomic_set(&ip->i_sizehint, hint); +} + +/** + * gfs2_allocate_folio_backing - Allocate blocks for a write fault + * @folio: The (locked) folio to allocate backing for + * @length: Size of the allocation + * + * We try to allocate all the blocks required for the folio in one go. This + * might fail for various reasons, so we keep trying until all the blocks to + * back this folio are allocated. If some of the blocks are already allocated, + * that is ok too. + */ +static int gfs2_allocate_folio_backing(struct folio *folio, size_t length) +{ + u64 pos = folio_pos(folio); + + do { + struct iomap iomap = { }; + + if (gfs2_iomap_alloc(folio->mapping->host, pos, length, &iomap)) + return -EIO; + + if (length < iomap.length) + iomap.length = length; + length -= iomap.length; + pos += iomap.length; + } while (length > 0); + + return 0; +} + +/** + * gfs2_page_mkwrite - Make a shared, mmap()ed, page writable + * @vmf: The virtual memory fault containing the page to become writable + * + * When the page becomes writable, we need to ensure that we have + * blocks allocated on disk to back that page. + */ + +static vm_fault_t gfs2_page_mkwrite(struct vm_fault *vmf) +{ + struct folio *folio = page_folio(vmf->page); + struct inode *inode = file_inode(vmf->vma->vm_file); + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + struct gfs2_alloc_parms ap = {}; + u64 pos = folio_pos(folio); + unsigned int data_blocks, ind_blocks, rblocks; + vm_fault_t ret = VM_FAULT_LOCKED; + struct gfs2_holder gh; + size_t length; + loff_t size; + int err; + + sb_start_pagefault(inode->i_sb); + + gfs2_holder_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &gh); + err = gfs2_glock_nq(&gh); + if (err) { + ret = vmf_fs_error(err); + goto out_uninit; + } + + /* Check folio index against inode size */ + size = i_size_read(inode); + if (pos >= size) { + ret = VM_FAULT_SIGBUS; + goto out_unlock; + } + + /* Update file times before taking folio lock */ + file_update_time(vmf->vma->vm_file); + + /* folio is wholly or partially inside EOF */ + if (size - pos < folio_size(folio)) + length = size - pos; + else + length = folio_size(folio); + + gfs2_size_hint(vmf->vma->vm_file, pos, length); + + set_bit(GLF_DIRTY, &ip->i_gl->gl_flags); + set_bit(GIF_SW_PAGED, &ip->i_flags); + + /* + * iomap_writepage / iomap_writepages currently don't support inline + * files, so always unstuff here. + */ + + if (!gfs2_is_stuffed(ip) && + !gfs2_write_alloc_required(ip, pos, length)) { + folio_lock(folio); + if (!folio_test_uptodate(folio) || + folio->mapping != inode->i_mapping) { + ret = VM_FAULT_NOPAGE; + folio_unlock(folio); + } + goto out_unlock; + } + + err = gfs2_rindex_update(sdp); + if (err) { + ret = vmf_fs_error(err); + goto out_unlock; + } + + gfs2_write_calc_reserv(ip, length, &data_blocks, &ind_blocks); + ap.target = data_blocks + ind_blocks; + err = gfs2_quota_lock_check(ip, &ap); + if (err) { + ret = vmf_fs_error(err); + goto out_unlock; + } + err = gfs2_inplace_reserve(ip, &ap); + if (err) { + ret = vmf_fs_error(err); + goto out_quota_unlock; + } + + rblocks = RES_DINODE + ind_blocks; + if (gfs2_is_jdata(ip)) + rblocks += data_blocks ? data_blocks : 1; + if (ind_blocks || data_blocks) { + rblocks += RES_STATFS + RES_QUOTA; + rblocks += gfs2_rg_blocks(ip, data_blocks + ind_blocks); + } + err = gfs2_trans_begin(sdp, rblocks, 0); + if (err) { + ret = vmf_fs_error(err); + goto out_trans_fail; + } + + /* Unstuff, if required, and allocate backing blocks for folio */ + if (gfs2_is_stuffed(ip)) { + err = gfs2_unstuff_dinode(ip); + if (err) { + ret = vmf_fs_error(err); + goto out_trans_end; + } + } + + folio_lock(folio); + /* If truncated, we must retry the operation, we may have raced + * with the glock demotion code. + */ + if (!folio_test_uptodate(folio) || folio->mapping != inode->i_mapping) { + ret = VM_FAULT_NOPAGE; + goto out_page_locked; + } + + err = gfs2_allocate_folio_backing(folio, length); + if (err) + ret = vmf_fs_error(err); + +out_page_locked: + if (ret != VM_FAULT_LOCKED) + folio_unlock(folio); +out_trans_end: + gfs2_trans_end(sdp); +out_trans_fail: + gfs2_inplace_release(ip); +out_quota_unlock: + gfs2_quota_unlock(ip); +out_unlock: + gfs2_glock_dq(&gh); +out_uninit: + gfs2_holder_uninit(&gh); + if (ret == VM_FAULT_LOCKED) { + folio_mark_dirty(folio); + folio_wait_stable(folio); + } + sb_end_pagefault(inode->i_sb); + return ret; +} + +static vm_fault_t gfs2_fault(struct vm_fault *vmf) +{ + struct inode *inode = file_inode(vmf->vma->vm_file); + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_holder gh; + vm_fault_t ret; + int err; + + gfs2_holder_init(ip->i_gl, LM_ST_SHARED, 0, &gh); + err = gfs2_glock_nq(&gh); + if (err) { + ret = vmf_fs_error(err); + goto out_uninit; + } + ret = filemap_fault(vmf); + gfs2_glock_dq(&gh); +out_uninit: + gfs2_holder_uninit(&gh); + return ret; +} + +static const struct vm_operations_struct gfs2_vm_ops = { + .fault = gfs2_fault, + .map_pages = filemap_map_pages, + .page_mkwrite = gfs2_page_mkwrite, +}; + +/** + * gfs2_mmap + * @file: The file to map + * @vma: The VMA which described the mapping + * + * There is no need to get a lock here unless we should be updating + * atime. We ignore any locking errors since the only consequence is + * a missed atime update (which will just be deferred until later). + * + * Returns: 0 + */ + +static int gfs2_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct gfs2_inode *ip = GFS2_I(file->f_mapping->host); + + if (!(file->f_flags & O_NOATIME) && + !IS_NOATIME(&ip->i_inode)) { + struct gfs2_holder i_gh; + int error; + + error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, LM_FLAG_ANY, + &i_gh); + if (error) + return error; + /* grab lock to update inode */ + gfs2_glock_dq_uninit(&i_gh); + file_accessed(file); + } + vma->vm_ops = &gfs2_vm_ops; + + return 0; +} + +/** + * gfs2_open_common - This is common to open and atomic_open + * @inode: The inode being opened + * @file: The file being opened + * + * This maybe called under a glock or not depending upon how it has + * been called. We must always be called under a glock for regular + * files, however. For other file types, it does not matter whether + * we hold the glock or not. + * + * Returns: Error code or 0 for success + */ + +int gfs2_open_common(struct inode *inode, struct file *file) +{ + struct gfs2_file *fp; + int ret; + + if (S_ISREG(inode->i_mode)) { + ret = generic_file_open(inode, file); + if (ret) + return ret; + + if (!gfs2_is_jdata(GFS2_I(inode))) + file->f_mode |= FMODE_CAN_ODIRECT; + } + + fp = kzalloc_obj(struct gfs2_file, GFP_NOFS); + if (!fp) + return -ENOMEM; + + mutex_init(&fp->f_fl_mutex); + + gfs2_assert_warn(GFS2_SB(inode), !file->private_data); + file->private_data = fp; + if (file->f_mode & FMODE_WRITE) { + ret = gfs2_qa_get(GFS2_I(inode)); + if (ret) + goto fail; + } + return 0; + +fail: + kfree(file->private_data); + file->private_data = NULL; + return ret; +} + +/** + * gfs2_open - open a file + * @inode: the inode to open + * @file: the struct file for this opening + * + * After atomic_open, this function is only used for opening files + * which are already cached. We must still get the glock for regular + * files to ensure that we have the file size uptodate for the large + * file check which is in the common code. That is only an issue for + * regular files though. + * + * Returns: errno + */ + +static int gfs2_open(struct inode *inode, struct file *file) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_holder i_gh; + int error; + bool need_unlock = false; + + if (S_ISREG(ip->i_inode.i_mode)) { + error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, LM_FLAG_ANY, + &i_gh); + if (error) + return error; + need_unlock = true; + } + + error = gfs2_open_common(inode, file); + + if (need_unlock) + gfs2_glock_dq_uninit(&i_gh); + + return error; +} + +/** + * gfs2_release - called to close a struct file + * @inode: the inode the struct file belongs to + * @file: the struct file being closed + * + * Returns: errno + */ + +static int gfs2_release(struct inode *inode, struct file *file) +{ + struct gfs2_inode *ip = GFS2_I(inode); + + kfree(file->private_data); + file->private_data = NULL; + + if (file->f_mode & FMODE_WRITE) { + if (gfs2_rs_active(&ip->i_res)) + gfs2_rs_delete(ip); + gfs2_qa_put(ip); + } + return 0; +} + +/** + * gfs2_fsync - sync the dirty data for a file (across the cluster) + * @file: the file that points to the dentry + * @start: the start position in the file to sync + * @end: the end position in the file to sync + * @datasync: set if we can ignore timestamp changes + * + * We split the data flushing here so that we don't wait for the data + * until after we've also sent the metadata to disk. Note that for + * data=ordered, we will write & wait for the data at the log flush + * stage anyway, so this is unlikely to make much of a difference + * except in the data=writeback case. + * + * If the fdatawrite fails due to any reason except -EIO, we will + * continue the remainder of the fsync, although we'll still report + * the error at the end. This is to match filemap_write_and_wait_range() + * behaviour. + * + * Returns: errno + */ + +static int gfs2_fsync(struct file *file, loff_t start, loff_t end, + int datasync) +{ + struct address_space *mapping = file->f_mapping; + struct inode *inode = mapping->host; + int sync_state = inode_state_read_once(inode) & I_DIRTY; + struct gfs2_inode *ip = GFS2_I(inode); + int ret = 0, ret1 = 0; + + if (mapping->nrpages) { + ret1 = filemap_fdatawrite_range(mapping, start, end); + if (ret1 == -EIO) + return ret1; + } + + if (!gfs2_is_jdata(ip)) + sync_state &= ~I_DIRTY_PAGES; + if (datasync) + sync_state &= ~I_DIRTY_SYNC; + + if (sync_state) { + ret = sync_inode_metadata(inode, 1); + if (ret) + return ret; + if (gfs2_is_jdata(ip)) + ret = file_write_and_wait(file); + if (ret) + return ret; + gfs2_ail_flush(ip->i_gl, 1); + } + + if (mapping->nrpages) + ret = file_fdatawait_range(file, start, end); + + return ret ? ret : ret1; +} + +static inline bool should_fault_in_pages(struct iov_iter *i, + struct kiocb *iocb, + size_t *prev_count, + size_t *window_size) +{ + size_t count = iov_iter_count(i); + size_t size, offs; + + if (!count) + return false; + if (!user_backed_iter(i)) + return false; + + /* + * Try to fault in multiple pages initially. When that doesn't result + * in any progress, fall back to a single page. + */ + size = PAGE_SIZE; + offs = offset_in_page(iocb->ki_pos); + if (*prev_count != count) { + size_t nr_dirtied; + + nr_dirtied = max(current->nr_dirtied_pause - + current->nr_dirtied, 8); + size = min_t(size_t, SZ_1M, nr_dirtied << PAGE_SHIFT); + } + + *prev_count = count; + *window_size = size - offs; + return true; +} + +static ssize_t gfs2_file_direct_read(struct kiocb *iocb, struct iov_iter *to, + struct gfs2_holder *gh) +{ + struct file *file = iocb->ki_filp; + struct gfs2_inode *ip = GFS2_I(file->f_mapping->host); + size_t prev_count = 0, window_size = 0; + size_t read = 0; + ssize_t ret; + + /* + * In this function, we disable page faults when we're holding the + * inode glock while doing I/O. If a page fault occurs, we indicate + * that the inode glock should be dropped, fault in the pages manually, + * and retry. + * + * Unlike generic_file_read_iter, for reads, iomap_dio_rw can trigger + * physical as well as manual page faults, and we need to disable both + * kinds. + * + * For direct I/O, gfs2 takes the inode glock in deferred mode. This + * locking mode is compatible with other deferred holders, so multiple + * processes and nodes can do direct I/O to a file at the same time. + * There's no guarantee that reads or writes will be atomic. Any + * coordination among readers and writers needs to happen externally. + */ + + if (!iov_iter_count(to)) + return 0; /* skip atime */ + + gfs2_holder_init(ip->i_gl, LM_ST_DEFERRED, 0, gh); +retry: + ret = gfs2_glock_nq(gh); + if (ret) + goto out_uninit; + pagefault_disable(); + to->nofault = true; + ret = iomap_dio_rw(iocb, to, &gfs2_iomap_ops, NULL, + IOMAP_DIO_PARTIAL, NULL, read); + to->nofault = false; + pagefault_enable(); + if (ret <= 0 && ret != -EFAULT) + goto out_unlock; + /* No increment (+=) because iomap_dio_rw returns a cumulative value. */ + if (ret > 0) + read = ret; + + if (should_fault_in_pages(to, iocb, &prev_count, &window_size)) { + gfs2_glock_dq(gh); + window_size -= fault_in_iov_iter_writeable(to, window_size); + if (window_size) + goto retry; + } +out_unlock: + if (gfs2_holder_queued(gh)) + gfs2_glock_dq(gh); +out_uninit: + gfs2_holder_uninit(gh); + /* User space doesn't expect partial success. */ + if (ret < 0) + return ret; + return read; +} + +static ssize_t gfs2_file_direct_write(struct kiocb *iocb, struct iov_iter *from, + struct gfs2_holder *gh) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file->f_mapping->host; + struct gfs2_inode *ip = GFS2_I(inode); + size_t prev_count = 0, window_size = 0; + size_t written = 0; + bool enough_retries; + ssize_t ret; + + /* + * In this function, we disable page faults when we're holding the + * inode glock while doing I/O. If a page fault occurs, we indicate + * that the inode glock should be dropped, fault in the pages manually, + * and retry. + * + * For writes, iomap_dio_rw only triggers manual page faults, so we + * don't need to disable physical ones. + */ + + /* + * Deferred lock, even if its a write, since we do no allocation on + * this path. All we need to change is the atime, and this lock mode + * ensures that other nodes have flushed their buffered read caches + * (i.e. their page cache entries for this inode). We do not, + * unfortunately, have the option of only flushing a range like the + * VFS does. + */ + gfs2_holder_init(ip->i_gl, LM_ST_DEFERRED, 0, gh); +retry: + ret = gfs2_glock_nq(gh); + if (ret) + goto out_uninit; + /* Silently fall back to buffered I/O when writing beyond EOF */ + if (iocb->ki_pos + iov_iter_count(from) > i_size_read(&ip->i_inode)) + goto out_unlock; + + from->nofault = true; + ret = iomap_dio_rw(iocb, from, &gfs2_iomap_ops, NULL, + IOMAP_DIO_PARTIAL, NULL, written); + from->nofault = false; + if (ret <= 0) { + if (ret == -ENOTBLK) + ret = 0; + if (ret != -EFAULT) + goto out_unlock; + } + /* No increment (+=) because iomap_dio_rw returns a cumulative value. */ + if (ret > 0) + written = ret; + + enough_retries = prev_count == iov_iter_count(from) && + window_size <= PAGE_SIZE; + if (should_fault_in_pages(from, iocb, &prev_count, &window_size)) { + gfs2_glock_dq(gh); + window_size -= fault_in_iov_iter_readable(from, window_size); + if (window_size) { + if (!enough_retries) + goto retry; + /* fall back to buffered I/O */ + ret = 0; + } + } +out_unlock: + if (gfs2_holder_queued(gh)) + gfs2_glock_dq(gh); +out_uninit: + gfs2_holder_uninit(gh); + /* User space doesn't expect partial success. */ + if (ret < 0) + return ret; + return written; +} + +static ssize_t gfs2_file_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct gfs2_inode *ip; + struct gfs2_holder gh; + size_t prev_count = 0, window_size = 0; + size_t read = 0; + ssize_t ret; + + /* + * In this function, we disable page faults when we're holding the + * inode glock while doing I/O. If a page fault occurs, we indicate + * that the inode glock should be dropped, fault in the pages manually, + * and retry. + */ + + if (iocb->ki_flags & IOCB_DIRECT) + return gfs2_file_direct_read(iocb, to, &gh); + + pagefault_disable(); + iocb->ki_flags |= IOCB_NOIO; + ret = generic_file_read_iter(iocb, to); + iocb->ki_flags &= ~IOCB_NOIO; + pagefault_enable(); + if (ret >= 0) { + if (!iov_iter_count(to)) + return ret; + read = ret; + } else if (ret != -EFAULT) { + if (ret != -EAGAIN) + return ret; + if (iocb->ki_flags & IOCB_NOWAIT) + return ret; + } + ip = GFS2_I(iocb->ki_filp->f_mapping->host); + gfs2_holder_init(ip->i_gl, LM_ST_SHARED, 0, &gh); +retry: + ret = gfs2_glock_nq(&gh); + if (ret) + goto out_uninit; + pagefault_disable(); + ret = generic_file_read_iter(iocb, to); + pagefault_enable(); + if (ret <= 0 && ret != -EFAULT) + goto out_unlock; + if (ret > 0) + read += ret; + + if (should_fault_in_pages(to, iocb, &prev_count, &window_size)) { + gfs2_glock_dq(&gh); + window_size -= fault_in_iov_iter_writeable(to, window_size); + if (window_size) + goto retry; + } +out_unlock: + if (gfs2_holder_queued(&gh)) + gfs2_glock_dq(&gh); +out_uninit: + gfs2_holder_uninit(&gh); + return read ? read : ret; +} + +static ssize_t gfs2_file_buffered_write(struct kiocb *iocb, + struct iov_iter *from, + struct gfs2_holder *gh) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + struct gfs2_holder *statfs_gh = NULL; + size_t prev_count = 0, window_size = 0; + size_t orig_count = iov_iter_count(from); + size_t written = 0; + ssize_t ret; + + /* + * In this function, we disable page faults when we're holding the + * inode glock while doing I/O. If a page fault occurs, we indicate + * that the inode glock should be dropped, fault in the pages manually, + * and retry. + */ + + if (inode == sdp->sd_rindex) { + statfs_gh = kmalloc_obj(*statfs_gh, GFP_NOFS); + if (!statfs_gh) + return -ENOMEM; + } + + gfs2_holder_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, gh); + if (should_fault_in_pages(from, iocb, &prev_count, &window_size)) { +retry: + window_size -= fault_in_iov_iter_readable(from, window_size); + if (!window_size) { + ret = -EFAULT; + goto out_uninit; + } + from->count = min(from->count, window_size); + } + ret = gfs2_glock_nq(gh); + if (ret) + goto out_uninit; + + if (inode == sdp->sd_rindex) { + struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode); + + ret = gfs2_glock_nq_init(m_ip->i_gl, LM_ST_EXCLUSIVE, + GL_NOCACHE, statfs_gh); + if (ret) + goto out_unlock; + } + + ret = gfs2_clear_beyond_eof(inode, iocb->ki_pos); + if (ret) + goto out_unlock; + + pagefault_disable(); + ret = iomap_file_buffered_write(iocb, from, &gfs2_iomap_ops, + &gfs2_iomap_write_ops, NULL); + pagefault_enable(); + if (ret > 0) + written += ret; + + if (inode == sdp->sd_rindex) + gfs2_glock_dq_uninit(statfs_gh); + + if (ret <= 0 && ret != -EFAULT) + goto out_unlock; + + from->count = orig_count - written; + if (should_fault_in_pages(from, iocb, &prev_count, &window_size)) { + gfs2_glock_dq(gh); + goto retry; + } +out_unlock: + if (gfs2_holder_queued(gh)) + gfs2_glock_dq(gh); +out_uninit: + gfs2_holder_uninit(gh); + kfree(statfs_gh); + from->count = orig_count - written; + return written ? written : ret; +} + +/** + * gfs2_file_write_iter - Perform a write to a file + * @iocb: The io context + * @from: The data to write + * + * We have to do a lock/unlock here to refresh the inode size for + * O_APPEND writes, otherwise we can land up writing at the wrong + * offset. There is still a race, but provided the app is using its + * own file locking, this will make O_APPEND work as expected. + * + */ + +static ssize_t gfs2_file_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_holder gh; + ssize_t ret; + + gfs2_size_hint(file, iocb->ki_pos, iov_iter_count(from)); + + if (iocb->ki_flags & IOCB_APPEND) { + ret = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, 0, &gh); + if (ret) + return ret; + gfs2_glock_dq_uninit(&gh); + } + + inode_lock(inode); + ret = generic_write_checks(iocb, from); + if (ret <= 0) + goto out_unlock; + + ret = file_remove_privs(file); + if (ret) + goto out_unlock; + + if (iocb->ki_flags & IOCB_DIRECT) { + struct address_space *mapping = file->f_mapping; + ssize_t buffered, ret2; + + /* + * Note that under direct I/O, we don't allow and inode + * timestamp updates, so we're not calling file_update_time() + * here. + */ + + ret = gfs2_file_direct_write(iocb, from, &gh); + if (ret < 0 || !iov_iter_count(from)) + goto out_unlock; + + iocb->ki_flags |= IOCB_DSYNC; + buffered = gfs2_file_buffered_write(iocb, from, &gh); + if (unlikely(buffered <= 0)) { + if (!ret) + ret = buffered; + goto out_unlock; + } + + /* + * We need to ensure that the page cache pages are written to + * disk and invalidated to preserve the expected O_DIRECT + * semantics. If the writeback or invalidate fails, only report + * the direct I/O range as we don't know if the buffered pages + * made it to disk. + */ + ret2 = generic_write_sync(iocb, buffered); + invalidate_mapping_pages(mapping, + (iocb->ki_pos - buffered) >> PAGE_SHIFT, + (iocb->ki_pos - 1) >> PAGE_SHIFT); + if (!ret || ret2 > 0) + ret += ret2; + } else { + ret = file_update_time(file); + if (ret) + goto out_unlock; + + ret = gfs2_file_buffered_write(iocb, from, &gh); + if (likely(ret > 0)) + ret = generic_write_sync(iocb, ret); + } + +out_unlock: + inode_unlock(inode); + return ret; +} + +static int fallocate_chunk(struct inode *inode, loff_t offset, loff_t len) +{ + struct super_block *sb = inode->i_sb; + struct gfs2_inode *ip = GFS2_I(inode); + loff_t end = offset + len; + struct buffer_head *dibh; + int error; + + error = gfs2_meta_inode_buffer(ip, &dibh); + if (unlikely(error)) + return error; + + gfs2_trans_add_meta(ip->i_gl, dibh); + + if (gfs2_is_stuffed(ip)) { + error = gfs2_unstuff_dinode(ip); + if (unlikely(error)) + goto out; + } + + while (offset < end) { + struct iomap iomap = { }; + + error = gfs2_iomap_alloc(inode, offset, end - offset, &iomap); + if (error) + goto out; + offset = iomap.offset + iomap.length; + if (!(iomap.flags & IOMAP_F_NEW)) + continue; + error = sb_issue_zeroout(sb, iomap.addr >> inode->i_blkbits, + iomap.length >> inode->i_blkbits, + GFP_NOFS); + if (error) { + fs_err(GFS2_SB(inode), "Failed to zero data buffers\n"); + goto out; + } + } +out: + brelse(dibh); + return error; +} + +/** + * calc_max_reserv() - Reverse of write_calc_reserv. Given a number of + * blocks, determine how many bytes can be written. + * @ip: The inode in question. + * @len: Max cap of bytes. What we return in *len must be <= this. + * @data_blocks: Compute and return the number of data blocks needed + * @ind_blocks: Compute and return the number of indirect blocks needed + * @max_blocks: The total blocks available to work with. + * + * Returns: void, but @len, @data_blocks and @ind_blocks are filled in. + */ +static void calc_max_reserv(struct gfs2_inode *ip, loff_t *len, + unsigned int *data_blocks, unsigned int *ind_blocks, + unsigned int max_blocks) +{ + loff_t max = *len; + const struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + unsigned int tmp, max_data = max_blocks - 3 * (sdp->sd_max_height - 1); + + for (tmp = max_data; tmp > sdp->sd_diptrs;) { + tmp = DIV_ROUND_UP(tmp, sdp->sd_inptrs); + max_data -= tmp; + } + + *data_blocks = max_data; + *ind_blocks = max_blocks - max_data; + *len = ((loff_t)max_data - 3) << sdp->sd_sb.sb_bsize_shift; + if (*len > max) { + *len = max; + gfs2_write_calc_reserv(ip, max, data_blocks, ind_blocks); + } +} + +static long __gfs2_fallocate(struct file *file, int mode, loff_t offset, loff_t len) +{ + struct inode *inode = file_inode(file); + struct gfs2_sbd *sdp = GFS2_SB(inode); + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_alloc_parms ap = {}; + unsigned int data_blocks = 0, ind_blocks = 0, rblocks; + loff_t bytes, max_bytes, max_blks; + int error; + const loff_t pos = offset; + const loff_t count = len; + loff_t bsize_mask = ~((loff_t)sdp->sd_sb.sb_bsize - 1); + loff_t next = (offset + len - 1) >> sdp->sd_sb.sb_bsize_shift; + loff_t max_chunk_size = UINT_MAX & bsize_mask; + + next = (next + 1) << sdp->sd_sb.sb_bsize_shift; + + if (!(mode & FALLOC_FL_KEEP_SIZE)) { + error = gfs2_clear_beyond_eof(inode, offset + len); + if (error) + return error; + } + + offset &= bsize_mask; + + len = next - offset; + bytes = sdp->sd_max_rg_data * sdp->sd_sb.sb_bsize / 2; + if (!bytes) + bytes = UINT_MAX; + bytes &= bsize_mask; + if (bytes == 0) + bytes = sdp->sd_sb.sb_bsize; + + gfs2_size_hint(file, offset, len); + + gfs2_write_calc_reserv(ip, PAGE_SIZE, &data_blocks, &ind_blocks); + ap.min_target = data_blocks + ind_blocks; + + while (len > 0) { + if (len < bytes) + bytes = len; + if (!gfs2_write_alloc_required(ip, offset, bytes)) { + len -= bytes; + offset += bytes; + continue; + } + + /* We need to determine how many bytes we can actually + * fallocate without exceeding quota or going over the + * end of the fs. We start off optimistically by assuming + * we can write max_bytes */ + max_bytes = (len > max_chunk_size) ? max_chunk_size : len; + + /* Since max_bytes is most likely a theoretical max, we + * calculate a more realistic 'bytes' to serve as a good + * starting point for the number of bytes we may be able + * to write */ + gfs2_write_calc_reserv(ip, bytes, &data_blocks, &ind_blocks); + ap.target = data_blocks + ind_blocks; + + error = gfs2_quota_lock_check(ip, &ap); + if (error) + return error; + /* ap.allowed tells us how many blocks quota will allow + * us to write. Check if this reduces max_blks */ + max_blks = UINT_MAX; + if (ap.allowed) + max_blks = ap.allowed; + + error = gfs2_inplace_reserve(ip, &ap); + if (error) + goto out_qunlock; + + /* check if the selected rgrp limits our max_blks further */ + if (ip->i_res.rs_reserved < max_blks) + max_blks = ip->i_res.rs_reserved; + + /* Almost done. Calculate bytes that can be written using + * max_blks. We also recompute max_bytes, data_blocks and + * ind_blocks */ + calc_max_reserv(ip, &max_bytes, &data_blocks, + &ind_blocks, max_blks); + + rblocks = RES_DINODE + ind_blocks + RES_STATFS + RES_QUOTA + + RES_RG_HDR + gfs2_rg_blocks(ip, data_blocks + ind_blocks); + if (gfs2_is_jdata(ip)) + rblocks += data_blocks ? data_blocks : 1; + + error = gfs2_trans_begin(sdp, rblocks, + PAGE_SIZE >> inode->i_blkbits); + if (error) + goto out_trans_fail; + + error = fallocate_chunk(inode, offset, max_bytes); + gfs2_trans_end(sdp); + + if (error) + goto out_trans_fail; + + len -= max_bytes; + offset += max_bytes; + gfs2_inplace_release(ip); + gfs2_quota_unlock(ip); + } + + if (!(mode & FALLOC_FL_KEEP_SIZE) && (pos + count) > inode->i_size) + i_size_write(inode, pos + count); + file_update_time(file); + mark_inode_dirty(inode); + + if ((file->f_flags & O_DSYNC) || IS_SYNC(file->f_mapping->host)) + return vfs_fsync_range(file, pos, pos + count - 1, + (file->f_flags & __O_SYNC) ? 0 : 1); + return 0; + +out_trans_fail: + gfs2_inplace_release(ip); +out_qunlock: + gfs2_quota_unlock(ip); + return error; +} + +static long gfs2_fallocate(struct file *file, int mode, loff_t offset, loff_t len) +{ + struct inode *inode = file_inode(file); + struct gfs2_sbd *sdp = GFS2_SB(inode); + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_holder gh; + int ret; + + if (mode & ~(FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE)) + return -EOPNOTSUPP; + /* fallocate is needed by gfs2_grow to reserve space in the rindex */ + if (gfs2_is_jdata(ip) && inode != sdp->sd_rindex) + return -EOPNOTSUPP; + + inode_lock(inode); + + gfs2_holder_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &gh); + ret = gfs2_glock_nq(&gh); + if (ret) + goto out_uninit; + + if (!(mode & FALLOC_FL_KEEP_SIZE) && + (offset + len) > inode->i_size) { + ret = inode_newsize_ok(inode, offset + len); + if (ret) + goto out_unlock; + } + + ret = get_write_access(inode); + if (ret) + goto out_unlock; + + if (mode & FALLOC_FL_PUNCH_HOLE) { + ret = __gfs2_punch_hole(file, offset, len); + } else { + ret = __gfs2_fallocate(file, mode, offset, len); + if (ret) + gfs2_rs_deltree(&ip->i_res); + } + + put_write_access(inode); +out_unlock: + gfs2_glock_dq(&gh); +out_uninit: + gfs2_holder_uninit(&gh); + inode_unlock(inode); + return ret; +} + +static ssize_t gfs2_file_splice_write(struct pipe_inode_info *pipe, + struct file *out, loff_t *ppos, + size_t len, unsigned int flags) +{ + ssize_t ret; + + gfs2_size_hint(out, *ppos, len); + + ret = iter_file_splice_write(pipe, out, ppos, len, flags); + return ret; +} + +#ifdef CONFIG_GFS2_FS_LOCKING_DLM + +/** + * gfs2_lock - acquire/release a posix lock on a file + * @file: the file pointer + * @cmd: either modify or retrieve lock state, possibly wait + * @fl: type and range of lock + * + * Returns: errno + */ + +static int gfs2_lock(struct file *file, int cmd, struct file_lock *fl) +{ + struct gfs2_inode *ip = GFS2_I(file->f_mapping->host); + struct gfs2_sbd *sdp = GFS2_SB(file->f_mapping->host); + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + int ret; + + if (!(fl->c.flc_flags & FL_POSIX)) + return -ENOLCK; + if (gfs2_withdrawn(sdp)) { + if (lock_is_unlock(fl)) + locks_lock_file_wait(file, fl); + return -EIO; + } + down_read(&ls->ls_sem); + ret = -ENODEV; + if (likely(ls->ls_dlm != NULL)) { + if (cmd == F_CANCELLK) + ret = dlm_posix_cancel(ls->ls_dlm, ip->i_no_addr, file, fl); + else if (IS_GETLK(cmd)) + ret = dlm_posix_get(ls->ls_dlm, ip->i_no_addr, file, fl); + else if (lock_is_unlock(fl)) + ret = dlm_posix_unlock(ls->ls_dlm, ip->i_no_addr, file, fl); + else + ret = dlm_posix_lock(ls->ls_dlm, ip->i_no_addr, file, cmd, fl); + } + up_read(&ls->ls_sem); + return ret; +} + +static void __flock_holder_uninit(struct file *file, struct gfs2_holder *fl_gh) +{ + struct gfs2_glock *gl = gfs2_glock_hold(fl_gh->gh_gl); + + /* + * Make sure gfs2_glock_put() won't sleep under the file->f_lock + * spinlock. + */ + + spin_lock(&file->f_lock); + gfs2_holder_uninit(fl_gh); + spin_unlock(&file->f_lock); + gfs2_glock_put(gl); +} + +static int do_flock(struct file *file, int cmd, struct file_lock *fl) +{ + struct gfs2_file *fp = file->private_data; + struct gfs2_holder *fl_gh = &fp->f_fl_gh; + struct gfs2_inode *ip = GFS2_I(file_inode(file)); + struct gfs2_glock *gl; + unsigned int state; + u16 flags; + int error = 0; + int sleeptime; + + state = lock_is_write(fl) ? LM_ST_EXCLUSIVE : LM_ST_SHARED; + flags = GL_EXACT | GL_NOPID; + if (!IS_SETLKW(cmd)) + flags |= LM_FLAG_TRY_1CB; + + mutex_lock(&fp->f_fl_mutex); + + if (gfs2_holder_initialized(fl_gh)) { + struct file_lock request; + if (fl_gh->gh_state == state) + goto out; + locks_init_lock(&request); + request.c.flc_type = F_UNLCK; + request.c.flc_flags = FL_FLOCK; + locks_lock_file_wait(file, &request); + gfs2_glock_dq(fl_gh); + gfs2_holder_reinit(state, flags, fl_gh); + } else { + error = gfs2_glock_get(GFS2_SB(&ip->i_inode), ip->i_no_addr, + &gfs2_flock_glops, CREATE, &gl); + if (error) + goto out; + spin_lock(&file->f_lock); + gfs2_holder_init(gl, state, flags, fl_gh); + spin_unlock(&file->f_lock); + gfs2_glock_put(gl); + } + for (sleeptime = 1; sleeptime <= 4; sleeptime <<= 1) { + error = gfs2_glock_nq(fl_gh); + if (error != GLR_TRYFAILED) + break; + fl_gh->gh_flags &= ~LM_FLAG_TRY_1CB; + fl_gh->gh_flags |= LM_FLAG_TRY; + msleep(sleeptime); + } + if (error) { + __flock_holder_uninit(file, fl_gh); + if (error == GLR_TRYFAILED) + error = -EAGAIN; + } else { + error = locks_lock_file_wait(file, fl); + gfs2_assert_warn(GFS2_SB(&ip->i_inode), !error); + } + +out: + mutex_unlock(&fp->f_fl_mutex); + return error; +} + +static void do_unflock(struct file *file, struct file_lock *fl) +{ + struct gfs2_file *fp = file->private_data; + struct gfs2_holder *fl_gh = &fp->f_fl_gh; + + mutex_lock(&fp->f_fl_mutex); + locks_lock_file_wait(file, fl); + if (gfs2_holder_initialized(fl_gh)) { + gfs2_glock_dq(fl_gh); + __flock_holder_uninit(file, fl_gh); + } + mutex_unlock(&fp->f_fl_mutex); +} + +/** + * gfs2_flock - acquire/release a flock lock on a file + * @file: the file pointer + * @cmd: either modify or retrieve lock state, possibly wait + * @fl: type and range of lock + * + * Returns: errno + */ + +static int gfs2_flock(struct file *file, int cmd, struct file_lock *fl) +{ + if (!(fl->c.flc_flags & FL_FLOCK)) + return -ENOLCK; + + if (lock_is_unlock(fl)) { + do_unflock(file, fl); + return 0; + } else { + return do_flock(file, cmd, fl); + } +} + +const struct file_operations gfs2_file_fops = { + .llseek = gfs2_llseek, + .read_iter = gfs2_file_read_iter, + .write_iter = gfs2_file_write_iter, + .iopoll = iocb_bio_iopoll, + .unlocked_ioctl = gfs2_ioctl, + .compat_ioctl = gfs2_compat_ioctl, + .mmap = gfs2_mmap, + .open = gfs2_open, + .release = gfs2_release, + .fsync = gfs2_fsync, + .lock = gfs2_lock, + .flock = gfs2_flock, + .splice_read = copy_splice_read, + .splice_write = gfs2_file_splice_write, + .fallocate = gfs2_fallocate, + .fop_flags = FOP_ASYNC_LOCK, +}; + +const struct file_operations gfs2_dir_fops = { + .iterate_shared = gfs2_readdir, + .unlocked_ioctl = gfs2_ioctl, + .compat_ioctl = gfs2_compat_ioctl, + .open = gfs2_open, + .release = gfs2_release, + .fsync = gfs2_fsync, + .lock = gfs2_lock, + .flock = gfs2_flock, + .llseek = default_llseek, + .fop_flags = FOP_ASYNC_LOCK, +}; + +#endif /* CONFIG_GFS2_FS_LOCKING_DLM */ + +const struct file_operations gfs2_file_fops_nolock = { + .llseek = gfs2_llseek, + .read_iter = gfs2_file_read_iter, + .write_iter = gfs2_file_write_iter, + .iopoll = iocb_bio_iopoll, + .unlocked_ioctl = gfs2_ioctl, + .compat_ioctl = gfs2_compat_ioctl, + .mmap = gfs2_mmap, + .open = gfs2_open, + .release = gfs2_release, + .fsync = gfs2_fsync, + .splice_read = copy_splice_read, + .splice_write = gfs2_file_splice_write, + .setlease = generic_setlease, + .fallocate = gfs2_fallocate, +}; + +const struct file_operations gfs2_dir_fops_nolock = { + .iterate_shared = gfs2_readdir, + .unlocked_ioctl = gfs2_ioctl, + .compat_ioctl = gfs2_compat_ioctl, + .open = gfs2_open, + .release = gfs2_release, + .fsync = gfs2_fsync, + .llseek = default_llseek, + .setlease = generic_setlease, +}; + diff --git a/fs/gfs2/gfs2.h b/fs/gfs2/gfs2.h new file mode 100644 index 000000000..ed78e5f20 --- /dev/null +++ b/fs/gfs2/gfs2.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#ifndef __GFS2_DOT_H__ +#define __GFS2_DOT_H__ + +enum { + NO_CREATE = 0, + CREATE = 1, +}; + +enum { + NO_FORCE = 0, + FORCE = 1, +}; + +#define GFS2_FAST_NAME_SIZE 8 + +#endif /* __GFS2_DOT_H__ */ + diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c new file mode 100644 index 000000000..d59ea71a8 --- /dev/null +++ b/fs/gfs2/glock.c @@ -0,0 +1,2664 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gfs2.h" +#include "incore.h" +#include "glock.h" +#include "glops.h" +#include "inode.h" +#include "lops.h" +#include "meta_io.h" +#include "quota.h" +#include "super.h" +#include "util.h" +#include "bmap.h" +#define CREATE_TRACE_POINTS +#include "trace_gfs2.h" + +struct gfs2_glock_iter { + struct gfs2_sbd *sdp; /* incore superblock */ + struct rhashtable_iter hti; /* rhashtable iterator */ + struct gfs2_glock *gl; /* current glock struct */ + loff_t last_pos; /* last position */ +}; + +typedef void (*glock_examiner) (struct gfs2_glock * gl); + +static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, + unsigned int target, bool may_cancel); +static void request_demote(struct gfs2_glock *gl, unsigned int state, + unsigned long delay, bool remote); + +static struct dentry *gfs2_root; + +#define GFS2_GL_HASH_SHIFT 15 +#define GFS2_GL_HASH_SIZE BIT(GFS2_GL_HASH_SHIFT) + +static const struct rhashtable_params ht_parms = { + .nelem_hint = GFS2_GL_HASH_SIZE * 3 / 4, + .key_len = offsetofend(struct lm_lockname, ln_type), + .key_offset = offsetof(struct gfs2_glock, gl_name), + .head_offset = offsetof(struct gfs2_glock, gl_node), + .automatic_shrinking = true, +}; + +static struct rhashtable gl_hash_table; + +#define GLOCK_WAIT_TABLE_BITS 12 +#define GLOCK_WAIT_TABLE_SIZE (1 << GLOCK_WAIT_TABLE_BITS) +static wait_queue_head_t glock_wait_table[GLOCK_WAIT_TABLE_SIZE] __cacheline_aligned; + +struct wait_glock_queue { + struct lm_lockname *name; + wait_queue_entry_t wait; +}; + +static int glock_wake_function(wait_queue_entry_t *wait, unsigned int mode, + int sync, void *key) +{ + struct wait_glock_queue *wait_glock = + container_of(wait, struct wait_glock_queue, wait); + struct lm_lockname *wait_name = wait_glock->name; + struct lm_lockname *wake_name = key; + + if (wake_name->ln_sbd != wait_name->ln_sbd || + wake_name->ln_number != wait_name->ln_number || + wake_name->ln_type != wait_name->ln_type) + return 0; + return autoremove_wake_function(wait, mode, sync, key); +} + +static wait_queue_head_t *glock_waitqueue(struct lm_lockname *name) +{ + u32 hash = jhash2((u32 *)name, ht_parms.key_len / 4, 0); + + return glock_wait_table + hash_32(hash, GLOCK_WAIT_TABLE_BITS); +} + +/** + * wake_up_glock - Wake up waiters on a glock + * @gl: the glock + */ +static void wake_up_glock(struct gfs2_glock *gl) +{ + wait_queue_head_t *wq = glock_waitqueue(&gl->gl_name); + + if (waitqueue_active(wq)) + __wake_up(wq, TASK_NORMAL, 1, &gl->gl_name); +} + +static void gfs2_glock_dealloc(struct rcu_head *rcu) +{ + struct gfs2_glock *gl = container_of(rcu, struct gfs2_glock, gl_rcu); + + kfree(gl->gl_lksb.sb_lvbptr); + if (gl->gl_ops->go_flags & GLOF_ASPACE) { + struct gfs2_glock_aspace *gla = + container_of(gl, struct gfs2_glock_aspace, glock); + kmem_cache_free(gfs2_glock_aspace_cachep, gla); + } else + kmem_cache_free(gfs2_glock_cachep, gl); +} + +static void __gfs2_glock_free(struct gfs2_glock *gl) +{ + rhashtable_remove_fast(&gl_hash_table, &gl->gl_node, ht_parms); + smp_mb(); + wake_up_glock(gl); + call_rcu(&gl->gl_rcu, gfs2_glock_dealloc); +} + +void gfs2_glock_free(struct gfs2_glock *gl) { + struct gfs2_sbd *sdp = glock_sbd(gl); + + __gfs2_glock_free(gl); + if (atomic_dec_and_test(&sdp->sd_glock_disposal)) + wake_up(&sdp->sd_kill_wait); +} + +void gfs2_glock_free_later(struct gfs2_glock *gl) { + struct gfs2_sbd *sdp = glock_sbd(gl); + + spin_lock(&sdp->sd_dead_lock); + list_add(&gl->gl_dead, &sdp->sd_dead_glocks); + spin_unlock(&sdp->sd_dead_lock); + if (atomic_dec_and_test(&sdp->sd_glock_disposal)) + wake_up(&sdp->sd_kill_wait); +} + +static void gfs2_free_dead_glocks(struct gfs2_sbd *sdp) +{ + struct list_head *list = &sdp->sd_dead_glocks; + + while(!list_empty(list)) { + struct gfs2_glock *gl; + + gl = list_first_entry(list, struct gfs2_glock, gl_dead); + list_del(&gl->gl_dead); + __gfs2_glock_free(gl); + } +} + +/** + * gfs2_glock_hold() - increment reference count on glock + * @gl: The glock to hold + * + */ + +struct gfs2_glock *gfs2_glock_hold(struct gfs2_glock *gl) +{ + if (!lockref_get_not_zero(&gl->gl_lockref)) + GLOCK_BUG_ON(gl, 1); + return gl; +} + +/* + * Enqueue the glock on the work queue. Passes one glock reference on to the + * work queue. + */ +static void gfs2_glock_queue_work(struct gfs2_glock *gl, unsigned long delay) { + struct gfs2_sbd *sdp = glock_sbd(gl); + + if (!queue_delayed_work(sdp->sd_glock_wq, &gl->gl_work, delay)) { + /* + * We are holding the lockref spinlock, and the work was still + * queued above. The queued work (glock_work_func) takes that + * spinlock before dropping its glock reference(s), so it + * cannot have dropped them in the meantime. + */ + GLOCK_BUG_ON(gl, gl->gl_lockref.count < 2); + gl->gl_lockref.count--; + } +} + +static void __gfs2_glock_put(struct gfs2_glock *gl) +{ + struct gfs2_sbd *sdp = glock_sbd(gl); + struct address_space *mapping = gfs2_glock2aspace(gl); + + lockref_mark_dead(&gl->gl_lockref); + spin_unlock(&gl->gl_lockref.lock); + GLOCK_BUG_ON(gl, !list_empty(&gl->gl_holders)); + if (mapping) { + truncate_inode_pages_final(mapping); + if (!gfs2_withdrawn(sdp)) + GLOCK_BUG_ON(gl, !mapping_empty(mapping)); + } + trace_gfs2_glock_put(gl); + sdp->sd_lockstruct.ls_ops->lm_put_lock(gl); +} + +static bool __gfs2_glock_put_or_lock(struct gfs2_glock *gl) +{ + if (lockref_put_or_lock(&gl->gl_lockref)) + return true; + GLOCK_BUG_ON(gl, gl->gl_lockref.count != 1); + if (gl->gl_state != LM_ST_UNLOCKED) { + request_demote(gl, LM_ST_UNLOCKED, 0, false); + gfs2_glock_queue_work(gl, 0); + spin_unlock(&gl->gl_lockref.lock); + return true; + } + return false; +} + +/** + * gfs2_glock_put() - Decrement reference count on glock + * @gl: The glock to put + * + */ + +void gfs2_glock_put(struct gfs2_glock *gl) +{ + if (__gfs2_glock_put_or_lock(gl)) + return; + + __gfs2_glock_put(gl); +} + +/* + * gfs2_glock_put_async - Decrement reference count without sleeping + * @gl: The glock to put + * + * Decrement the reference count on glock immediately unless it is the last + * reference. Defer putting the last reference to work queue context. + */ +void gfs2_glock_put_async(struct gfs2_glock *gl) +{ + if (__gfs2_glock_put_or_lock(gl)) + return; + + gfs2_glock_queue_work(gl, 0); + spin_unlock(&gl->gl_lockref.lock); +} + +/** + * may_grant - check if it's ok to grant a new lock + * @gl: The glock + * @current_gh: One of the current holders of @gl + * @gh: The lock request which we wish to grant + * + * With our current compatibility rules, if a glock has one or more active + * holders (HIF_HOLDER flag set), any of those holders can be passed in as + * @current_gh; they are all the same as far as compatibility with the new @gh + * goes. + * + * Returns true if it's ok to grant the lock. + */ + +static inline bool may_grant(struct gfs2_glock *gl, + struct gfs2_holder *current_gh, + struct gfs2_holder *gh) +{ + if (current_gh) { + GLOCK_BUG_ON(gl, !test_bit(HIF_HOLDER, ¤t_gh->gh_iflags)); + + switch(current_gh->gh_state) { + case LM_ST_EXCLUSIVE: + /* + * Here we make a special exception to grant holders + * who agree to share the EX lock with other holders + * who also have the bit set. If the original holder + * has the LM_FLAG_NODE_SCOPE bit set, we grant more + * holders with the bit set. + */ + return gh->gh_state == LM_ST_EXCLUSIVE && + (current_gh->gh_flags & LM_FLAG_NODE_SCOPE) && + (gh->gh_flags & LM_FLAG_NODE_SCOPE); + + case LM_ST_SHARED: + case LM_ST_DEFERRED: + return gh->gh_state == current_gh->gh_state; + + default: + return false; + } + } + + if (gl->gl_state == gh->gh_state) + return true; + if (gh->gh_flags & GL_EXACT) + return false; + if (gl->gl_state == LM_ST_EXCLUSIVE) { + return gh->gh_state == LM_ST_SHARED || + gh->gh_state == LM_ST_DEFERRED; + } + if (gh->gh_flags & LM_FLAG_ANY) + return gl->gl_state != LM_ST_UNLOCKED; + return false; +} + +static void gfs2_holder_wake(struct gfs2_holder *gh) +{ + clear_bit(HIF_WAIT, &gh->gh_iflags); + smp_mb__after_atomic(); + wake_up_bit(&gh->gh_iflags, HIF_WAIT); + if (gh->gh_flags & GL_ASYNC) { + struct gfs2_sbd *sdp = glock_sbd(gh->gh_gl); + + wake_up(&sdp->sd_async_glock_wait); + } +} + +/** + * do_error - Something unexpected has happened during a lock request + * @gl: The glock + * @ret: The status from the DLM + */ + +static void do_error(struct gfs2_glock *gl, const int ret) +{ + struct gfs2_holder *gh, *tmp; + + list_for_each_entry_safe(gh, tmp, &gl->gl_holders, gh_list) { + if (test_bit(HIF_HOLDER, &gh->gh_iflags)) + continue; + if (ret & LM_OUT_ERROR) + gh->gh_error = -EIO; + else if (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)) + gh->gh_error = GLR_TRYFAILED; + else + continue; + list_del_init(&gh->gh_list); + trace_gfs2_glock_queue(gh, 0); + gfs2_holder_wake(gh); + } +} + +/** + * find_first_holder - find the first "holder" gh + * @gl: the glock + */ + +static inline struct gfs2_holder *find_first_holder(const struct gfs2_glock *gl) +{ + struct gfs2_holder *gh; + + if (!list_empty(&gl->gl_holders)) { + gh = list_first_entry(&gl->gl_holders, struct gfs2_holder, + gh_list); + if (test_bit(HIF_HOLDER, &gh->gh_iflags)) + return gh; + } + return NULL; +} + +/* + * gfs2_instantiate - Call the glops instantiate function + * @gh: The glock holder + * + * Returns: 0 if instantiate was successful, or error. + */ +int gfs2_instantiate(struct gfs2_holder *gh) +{ + struct gfs2_glock *gl = gh->gh_gl; + const struct gfs2_glock_operations *glops = gl->gl_ops; + int ret; + +again: + if (!test_bit(GLF_INSTANTIATE_NEEDED, &gl->gl_flags)) + goto done; + + /* + * Since we unlock the lockref lock, we set a flag to indicate + * instantiate is in progress. + */ + if (test_and_set_bit(GLF_INSTANTIATE_IN_PROG, &gl->gl_flags)) { + wait_on_bit(&gl->gl_flags, GLF_INSTANTIATE_IN_PROG, + TASK_UNINTERRUPTIBLE); + /* + * Here we just waited for a different instantiate to finish. + * But that may not have been successful, as when a process + * locks an inode glock _before_ it has an actual inode to + * instantiate into. So we check again. This process might + * have an inode to instantiate, so might be successful. + */ + goto again; + } + + ret = glops->go_instantiate(gl); + if (!ret) + clear_bit(GLF_INSTANTIATE_NEEDED, &gl->gl_flags); + clear_and_wake_up_bit(GLF_INSTANTIATE_IN_PROG, &gl->gl_flags); + if (ret) + return ret; + +done: + if (glops->go_held) + return glops->go_held(gh); + return 0; +} + +/** + * do_promote - promote as many requests as possible on the current queue + * @gl: The glock + */ + +static void do_promote(struct gfs2_glock *gl) +{ + struct gfs2_sbd *sdp = glock_sbd(gl); + struct gfs2_holder *gh, *current_gh; + + if (gfs2_withdrawn(sdp)) { + do_error(gl, LM_OUT_ERROR); + return; + } + + current_gh = find_first_holder(gl); + list_for_each_entry(gh, &gl->gl_holders, gh_list) { + if (test_bit(HIF_HOLDER, &gh->gh_iflags)) + continue; + if (!may_grant(gl, current_gh, gh)) { + /* + * If we get here, it means we may not grant this + * holder for some reason. + */ + if (current_gh) + do_error(gl, 0); /* Fail queued try locks */ + break; + } + set_bit(HIF_HOLDER, &gh->gh_iflags); + trace_gfs2_promote(gh); + gfs2_holder_wake(gh); + if (!current_gh) + current_gh = gh; + } +} + +/** + * find_first_waiter - find the first gh that's waiting for the glock + * @gl: the glock + */ + +static inline struct gfs2_holder *find_first_waiter(const struct gfs2_glock *gl) +{ + struct gfs2_holder *gh; + + list_for_each_entry(gh, &gl->gl_holders, gh_list) { + if (!test_bit(HIF_HOLDER, &gh->gh_iflags)) + return gh; + } + return NULL; +} + +/** + * find_last_waiter - find the last gh that's waiting for the glock + * @gl: the glock + * + * This also is a fast way of finding out if there are any waiters. + */ + +static inline struct gfs2_holder *find_last_waiter(const struct gfs2_glock *gl) +{ + struct gfs2_holder *gh; + + if (list_empty(&gl->gl_holders)) + return NULL; + gh = list_last_entry(&gl->gl_holders, struct gfs2_holder, gh_list); + return test_bit(HIF_HOLDER, &gh->gh_iflags) ? NULL : gh; +} + +/** + * state_change - record that the glock is now in a different state + * @gl: the glock + * @new_state: the new state + */ + +static void state_change(struct gfs2_glock *gl, unsigned int new_state) +{ + if (new_state != gl->gl_target) + /* shorten our minimum hold time */ + gl->gl_hold_time = max(gl->gl_hold_time - GL_GLOCK_HOLD_DECR, + GL_GLOCK_MIN_HOLD); + gl->gl_state = new_state; + gl->gl_tchange = jiffies; +} + +static void gfs2_set_demote(int nr, struct gfs2_glock *gl) +{ + struct gfs2_sbd *sdp = glock_sbd(gl); + + set_bit(nr, &gl->gl_flags); + smp_mb(); + wake_up(&sdp->sd_async_glock_wait); +} + +static void gfs2_demote_wake(struct gfs2_glock *gl) +{ + gl->gl_demote_state = LM_ST_EXCLUSIVE; + clear_bit(GLF_DEMOTE, &gl->gl_flags); + smp_mb__after_atomic(); + wake_up_bit(&gl->gl_flags, GLF_DEMOTE); +} + +/** + * finish_xmote - The DLM has replied to one of our lock requests + * @gl: The glock + * @ret: The status from the DLM + * + */ + +static void finish_xmote(struct gfs2_glock *gl, unsigned int ret) +{ + const struct gfs2_glock_operations *glops = gl->gl_ops; + + if (!(ret & ~LM_OUT_ST_MASK)) { + unsigned state = ret & LM_OUT_ST_MASK; + + trace_gfs2_glock_state_change(gl, state); + state_change(gl, state); + } + + /* Demote to UN request arrived during demote to SH or DF */ + if (test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags) && + gl->gl_state != LM_ST_UNLOCKED && + gl->gl_demote_state == LM_ST_UNLOCKED) + gl->gl_target = LM_ST_UNLOCKED; + + /* Check for state != intended state */ + if (unlikely(gl->gl_state != gl->gl_target)) { + struct gfs2_holder *gh = find_first_waiter(gl); + + if (gh && !test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags)) { + if (ret & LM_OUT_CANCELED) { + list_del_init(&gh->gh_list); + trace_gfs2_glock_queue(gh, 0); + gfs2_holder_wake(gh); + gl->gl_target = gl->gl_state; + goto out; + } + /* Some error or failed "try lock" - report it */ + if ((ret & LM_OUT_ERROR) || + (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) { + gl->gl_target = gl->gl_state; + do_error(gl, ret); + goto out; + } + } + switch(gl->gl_state) { + /* Unlocked due to conversion deadlock, try again */ + case LM_ST_UNLOCKED: + do_xmote(gl, gh, gl->gl_target, + !test_bit(GLF_DEMOTE_IN_PROGRESS, + &gl->gl_flags)); + break; + /* Conversion fails, unlock and try again */ + case LM_ST_SHARED: + case LM_ST_DEFERRED: + do_xmote(gl, gh, LM_ST_UNLOCKED, false); + break; + default: /* Everything else */ + fs_err(glock_sbd(gl), + "glock %u:%llu requested=%u ret=%u\n", + glock_type(gl), glock_number(gl), + gl->gl_req, ret); + GLOCK_BUG_ON(gl, 1); + } + return; + } + + /* Fast path - we got what we asked for */ + if (test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags)) { + clear_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags); + gfs2_demote_wake(gl); + } + if (gl->gl_state != LM_ST_UNLOCKED) { + if (glops->go_xmote_bh) { + int rv; + + spin_unlock(&gl->gl_lockref.lock); + rv = glops->go_xmote_bh(gl); + spin_lock(&gl->gl_lockref.lock); + if (rv) { + do_error(gl, rv); + goto out; + } + } + do_promote(gl); + } +out: + if (!test_bit(GLF_CANCELING, &gl->gl_flags)) + clear_and_wake_up_bit(GLF_LOCK, &gl->gl_flags); +} + +/** + * do_xmote - Calls the DLM to change the state of a lock + * @gl: The lock state + * @gh: The holder (only for promotes) + * @target: The target lock state + * @may_cancel: Operation may be canceled + * + */ + +static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, + unsigned int target, bool may_cancel) +__releases(&gl->gl_lockref.lock) +__acquires(&gl->gl_lockref.lock) +{ + const struct gfs2_glock_operations *glops = gl->gl_ops; + struct gfs2_sbd *sdp = glock_sbd(gl); + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + int ret; + + /* + * When a filesystem is withdrawing, the remaining cluster nodes will + * take care of recovering the withdrawing node's journal. We only + * need to make sure that once we trigger remote recovery, we won't + * write to the shared block device anymore. This means that here, + * + * - no new writes to the filesystem must be triggered (->go_sync()). + * + * - any cached data should be discarded by calling ->go_inval(), dirty + * or not and journaled or unjournaled. + * + * - no more dlm locking operations should be issued (->lm_lock()). + */ + + GLOCK_BUG_ON(gl, gl->gl_state == target); + GLOCK_BUG_ON(gl, gl->gl_state == gl->gl_target); + + if (!glops->go_inval || !glops->go_sync) + goto skip_inval; + + spin_unlock(&gl->gl_lockref.lock); + if (!gfs2_withdrawn(sdp)) { + ret = glops->go_sync(gl); + if (ret) { + if (cmpxchg(&sdp->sd_log_error, 0, ret)) { + fs_err(sdp, "Error %d syncing glock\n", ret); + gfs2_dump_glock(NULL, gl, true); + gfs2_withdraw(sdp); + } + } + } + + if (target == LM_ST_UNLOCKED || target == LM_ST_DEFERRED) + glops->go_inval(gl, target == LM_ST_DEFERRED ? 0 : DIO_METADATA); + spin_lock(&gl->gl_lockref.lock); + +skip_inval: + if (gfs2_withdrawn(sdp)) { + if (target != LM_ST_UNLOCKED) + target = LM_OUT_ERROR; + goto out; + } + + if (ls->ls_ops->lm_lock) { + spin_unlock(&gl->gl_lockref.lock); + ret = ls->ls_ops->lm_lock(gl, target, gh ? gh->gh_flags : 0); + spin_lock(&gl->gl_lockref.lock); + + if (!ret) { + if (may_cancel) { + set_bit(GLF_MAY_CANCEL, &gl->gl_flags); + smp_mb__after_atomic(); + wake_up_bit(&gl->gl_flags, GLF_LOCK); + } + /* The operation will be completed asynchronously. */ + gl->gl_lockref.count++; + return; + } + + if (ret == -ENODEV) { + /* + * The lockspace has been released and the lock has + * been unlocked implicitly. + */ + if (target != LM_ST_UNLOCKED) { + target = LM_OUT_ERROR; + goto out; + } + } else { + fs_err(sdp, "lm_lock ret %d\n", ret); + GLOCK_BUG_ON(gl, !gfs2_withdrawn(sdp)); + return; + } + } + +out: + /* Complete the operation now. */ + finish_xmote(gl, target); + gl->gl_lockref.count++; + gfs2_glock_queue_work(gl, 0); +} + +/** + * run_queue - do all outstanding tasks related to a glock + * @gl: The glock in question + * @nonblock: True if we must not block in run_queue + * + */ + +static void run_queue(struct gfs2_glock *gl, const int nonblock) +__releases(&gl->gl_lockref.lock) +__acquires(&gl->gl_lockref.lock) +{ + struct gfs2_holder *gh; + + if (test_bit(GLF_LOCK, &gl->gl_flags)) + return; + + /* + * The GLF_DEMOTE_IN_PROGRESS flag must only be set when the GLF_LOCK + * flag is set as well. + */ + GLOCK_BUG_ON(gl, test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags)); + + if (test_bit(GLF_DEMOTE, &gl->gl_flags)) { + if (gl->gl_demote_state == gl->gl_state) { + gfs2_demote_wake(gl); + goto promote; + } + + if (find_first_holder(gl)) + return; + if (nonblock) + goto out_sched; + set_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags); + GLOCK_BUG_ON(gl, gl->gl_demote_state == LM_ST_EXCLUSIVE); + gl->gl_target = gl->gl_demote_state; + set_bit(GLF_LOCK, &gl->gl_flags); + do_xmote(gl, NULL, gl->gl_target, false); + return; + } + +promote: + do_promote(gl); + if (find_first_holder(gl)) + return; + gh = find_first_waiter(gl); + if (!gh) + return; + if (nonblock) + goto out_sched; + gl->gl_target = gh->gh_state; + if (!(gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) + do_error(gl, 0); /* Fail queued try locks */ + set_bit(GLF_LOCK, &gl->gl_flags); + do_xmote(gl, gh, gl->gl_target, true); + return; + +out_sched: + gl->gl_lockref.count++; + gfs2_glock_queue_work(gl, 0); +} + +/** + * glock_set_object - set the gl_object field of a glock + * @gl: the glock + * @object: the object + */ +void glock_set_object(struct gfs2_glock *gl, void *object) +{ + void *prev_object; + + spin_lock(&gl->gl_lockref.lock); + prev_object = gl->gl_object; + gl->gl_object = object; + spin_unlock(&gl->gl_lockref.lock); + if (gfs2_assert_warn(glock_sbd(gl), prev_object == NULL)) + gfs2_dump_glock(NULL, gl, true); +} + +/** + * glock_clear_object - clear the gl_object field of a glock + * @gl: the glock + * @object: object the glock currently points at + */ +void glock_clear_object(struct gfs2_glock *gl, void *object) +{ + void *prev_object; + + spin_lock(&gl->gl_lockref.lock); + prev_object = gl->gl_object; + gl->gl_object = NULL; + spin_unlock(&gl->gl_lockref.lock); + if (gfs2_assert_warn(glock_sbd(gl), prev_object == object)) + gfs2_dump_glock(NULL, gl, true); +} + +void gfs2_inode_remember_delete(struct gfs2_glock *gl, u64 generation) +{ + struct gfs2_inode_lvb *ri = (void *)gl->gl_lksb.sb_lvbptr; + + if (ri->ri_magic == 0) + ri->ri_magic = cpu_to_be32(GFS2_MAGIC); + if (ri->ri_magic == cpu_to_be32(GFS2_MAGIC)) + ri->ri_generation_deleted = cpu_to_be64(generation); +} + +bool gfs2_inode_already_deleted(struct gfs2_glock *gl, u64 generation) +{ + struct gfs2_inode_lvb *ri = (void *)gl->gl_lksb.sb_lvbptr; + + if (ri->ri_magic != cpu_to_be32(GFS2_MAGIC)) + return false; + return generation <= be64_to_cpu(ri->ri_generation_deleted); +} + +static void gfs2_glock_poke(struct gfs2_glock *gl) +{ + int flags = LM_FLAG_TRY_1CB | LM_FLAG_ANY | GL_SKIP; + struct gfs2_holder gh; + int error; + + __gfs2_holder_init(gl, LM_ST_SHARED, flags, &gh, _RET_IP_); + error = gfs2_glock_nq(&gh); + if (!error) + gfs2_glock_dq(&gh); + gfs2_holder_uninit(&gh); +} + +static struct gfs2_inode *gfs2_grab_existing_inode(struct gfs2_glock *gl) +{ + struct gfs2_inode *ip; + + spin_lock(&gl->gl_lockref.lock); + ip = gl->gl_object; + if (ip && !igrab(&ip->i_inode)) + ip = NULL; + spin_unlock(&gl->gl_lockref.lock); + if (ip) { + wait_on_new_inode(&ip->i_inode); + if (is_bad_inode(&ip->i_inode)) { + iput(&ip->i_inode); + ip = NULL; + } + } + return ip; +} + +static void gfs2_try_to_evict(struct gfs2_glock *gl) +{ + struct gfs2_inode *ip; + + /* + * If there is contention on the iopen glock and we have an inode, try + * to grab and release the inode so that it can be evicted. The + * GLF_DEFER_DELETE flag indicates to gfs2_evict_inode() that the inode + * should not be deleted locally. This will allow the remote node to + * go ahead and delete the inode without us having to do it, which will + * avoid rgrp glock thrashing. + * + * The remote node is likely still holding the corresponding inode + * glock, so it will run before we get to verify that the delete has + * happened below. (Verification is triggered by the call to + * gfs2_queue_verify_delete() in gfs2_evict_inode().) + */ + ip = gfs2_grab_existing_inode(gl); + if (ip) { + set_bit(GLF_DEFER_DELETE, &gl->gl_flags); + d_prune_aliases(&ip->i_inode); + iput(&ip->i_inode); + clear_bit(GLF_DEFER_DELETE, &gl->gl_flags); + + /* If the inode was evicted, gl->gl_object will now be NULL. */ + ip = gfs2_grab_existing_inode(gl); + if (ip) { + gfs2_glock_poke(ip->i_gl); + iput(&ip->i_inode); + } + } +} + +bool gfs2_queue_try_to_evict(struct gfs2_glock *gl) +{ + struct gfs2_sbd *sdp = glock_sbd(gl); + + if (test_and_set_bit(GLF_TRY_TO_EVICT, &gl->gl_flags)) + return false; + return !mod_delayed_work(sdp->sd_delete_wq, &gl->gl_delete, 0); +} + +bool gfs2_queue_verify_delete(struct gfs2_glock *gl, bool later) +{ + struct gfs2_sbd *sdp = glock_sbd(gl); + unsigned long delay; + + if (test_and_set_bit(GLF_VERIFY_DELETE, &gl->gl_flags)) + return false; + delay = later ? HZ + get_random_long() % (HZ * 9) : 0; + return queue_delayed_work(sdp->sd_delete_wq, &gl->gl_delete, delay); +} + +static void delete_work_func(struct work_struct *work) +{ + struct delayed_work *dwork = to_delayed_work(work); + struct gfs2_glock *gl = container_of(dwork, struct gfs2_glock, gl_delete); + struct gfs2_sbd *sdp = glock_sbd(gl); + bool verify_delete = test_and_clear_bit(GLF_VERIFY_DELETE, &gl->gl_flags); + + /* + * Check for the GLF_VERIFY_DELETE above: this ensures that we won't + * immediately process GLF_VERIFY_DELETE work that the below call to + * gfs2_try_to_evict() queues. + */ + + if (test_and_clear_bit(GLF_TRY_TO_EVICT, &gl->gl_flags)) + gfs2_try_to_evict(gl); + + if (verify_delete) { + u64 no_addr = glock_number(gl); + struct inode *inode; + + inode = gfs2_lookup_by_inum(sdp, no_addr, gl->gl_no_formal_ino, + GFS2_BLKST_UNLINKED); + if (IS_ERR(inode)) { + if (PTR_ERR(inode) == -EAGAIN && + !test_bit(SDF_KILL, &sdp->sd_flags) && + gfs2_queue_verify_delete(gl, true)) + return; + } else { + d_prune_aliases(inode); + iput(inode); + } + } + + gfs2_glock_put(gl); +} + +static void glock_work_func(struct work_struct *work) +{ + struct gfs2_glock *gl = container_of(work, struct gfs2_glock, gl_work.work); + unsigned int drop_refs; + unsigned long delay; + + spin_lock(&gl->gl_lockref.lock); +again: + drop_refs = 1; + if (test_bit(GLF_HAVE_REPLY, &gl->gl_flags)) { + clear_bit(GLF_HAVE_REPLY, &gl->gl_flags); + finish_xmote(gl, gl->gl_reply); + drop_refs++; + } + delay = 0; + if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) && + gl->gl_state != LM_ST_UNLOCKED && + gl->gl_demote_state != LM_ST_EXCLUSIVE) { + if (glock_type(gl) == LM_TYPE_INODE) { + unsigned long holdtime, now = jiffies; + + holdtime = gl->gl_tchange + gl->gl_hold_time; + if (time_before(now, holdtime)) + delay = holdtime - now; + } + + if (!delay) { + clear_bit(GLF_PENDING_DEMOTE, &gl->gl_flags); + gfs2_set_demote(GLF_DEMOTE, gl); + } + } + run_queue(gl, 0); + if (delay) { + /* Keep one glock reference for the work we requeue. */ + drop_refs--; + gfs2_glock_queue_work(gl, delay); + } + + /* Drop the remaining glock references manually. */ + GLOCK_BUG_ON(gl, gl->gl_lockref.count < drop_refs); + gl->gl_lockref.count -= drop_refs; + if (!gl->gl_lockref.count) { + if (gl->gl_state != LM_ST_UNLOCKED) { + gl->gl_lockref.count++; + request_demote(gl, LM_ST_UNLOCKED, 0, false); + goto again; + } + __gfs2_glock_put(gl); + return; + } + spin_unlock(&gl->gl_lockref.lock); +} + +static struct gfs2_glock *find_insert_glock(struct lm_lockname *name, + struct gfs2_glock *new) +{ + struct wait_glock_queue wait; + wait_queue_head_t *wq = glock_waitqueue(name); + struct gfs2_glock *gl; + + wait.name = name; + init_wait(&wait.wait); + wait.wait.func = glock_wake_function; + +again: + prepare_to_wait(wq, &wait.wait, TASK_UNINTERRUPTIBLE); + rcu_read_lock(); + if (new) { + gl = rhashtable_lookup_get_insert_fast(&gl_hash_table, + &new->gl_node, ht_parms); + if (IS_ERR(gl)) + goto out; + } else { + gl = rhashtable_lookup_fast(&gl_hash_table, + name, ht_parms); + } + if (gl && !lockref_get_not_dead(&gl->gl_lockref)) { + rcu_read_unlock(); + schedule(); + goto again; + } +out: + rcu_read_unlock(); + finish_wait(wq, &wait.wait); + return gl; +} + +/** + * gfs2_glock_get() - Get a glock, or create one if one doesn't exist + * @sdp: The GFS2 superblock + * @number: the lock number + * @glops: The glock_operations to use + * @create: If 0, don't create the glock if it doesn't exist + * @glp: the glock is returned here + * + * This does not lock a glock, just finds/creates structures for one. + * + * Returns: errno + */ + +int gfs2_glock_get(struct gfs2_sbd *sdp, u64 number, + const struct gfs2_glock_operations *glops, int create, + struct gfs2_glock **glp) +{ + struct lm_lockname name = { .ln_number = number, + .ln_type = glops->go_type, + .ln_sbd = sdp }; + struct gfs2_glock *gl, *tmp; + struct address_space *mapping; + + gl = find_insert_glock(&name, NULL); + if (gl) + goto found; + if (!create) + return -ENOENT; + + if (glops->go_flags & GLOF_ASPACE) { + struct gfs2_glock_aspace *gla = + kmem_cache_alloc(gfs2_glock_aspace_cachep, GFP_NOFS); + if (!gla) + return -ENOMEM; + gl = &gla->glock; + } else { + gl = kmem_cache_alloc(gfs2_glock_cachep, GFP_NOFS); + if (!gl) + return -ENOMEM; + } + memset(&gl->gl_lksb, 0, sizeof(struct dlm_lksb)); + gl->gl_ops = glops; + + if (glops->go_flags & GLOF_LVB) { + gl->gl_lksb.sb_lvbptr = kzalloc(GDLM_LVB_SIZE, GFP_NOFS); + if (!gl->gl_lksb.sb_lvbptr) { + gfs2_glock_dealloc(&gl->gl_rcu); + return -ENOMEM; + } + } + + atomic_inc(&sdp->sd_glock_disposal); + gl->gl_node.next = NULL; + gl->gl_flags = BIT(GLF_INITIAL); + if (glops->go_instantiate) + gl->gl_flags |= BIT(GLF_INSTANTIATE_NEEDED); + gl->gl_name = name; + lockref_init(&gl->gl_lockref); + lockdep_set_subclass(&gl->gl_lockref.lock, glops->go_subclass); + gl->gl_state = LM_ST_UNLOCKED; + gl->gl_target = LM_ST_UNLOCKED; + gl->gl_demote_state = LM_ST_EXCLUSIVE; + gl->gl_dstamp = 0; + preempt_disable(); + /* We use the global stats to estimate the initial per-glock stats */ + gl->gl_stats = this_cpu_ptr(sdp->sd_lkstats)->lkstats[glops->go_type]; + preempt_enable(); + gl->gl_stats.stats[GFS2_LKS_DCOUNT] = 0; + gl->gl_stats.stats[GFS2_LKS_QCOUNT] = 0; + gl->gl_tchange = jiffies; + gl->gl_object = NULL; + gl->gl_hold_time = GL_GLOCK_DFT_HOLD; + INIT_DELAYED_WORK(&gl->gl_work, glock_work_func); + if (glock_type(gl) == LM_TYPE_IOPEN) + INIT_DELAYED_WORK(&gl->gl_delete, delete_work_func); + + mapping = gfs2_glock2aspace(gl); + if (mapping) { + gfp_t gfp_mask; + + mapping->a_ops = &gfs2_meta_aops; + mapping->host = sdp->sd_inode; + mapping->flags = 0; + gfp_mask = mapping_gfp_mask(sdp->sd_inode->i_mapping); + mapping_set_gfp_mask(mapping, gfp_mask); + mapping->writeback_index = 0; + } + + tmp = find_insert_glock(&name, gl); + if (tmp) { + gfs2_glock_dealloc(&gl->gl_rcu); + if (atomic_dec_and_test(&sdp->sd_glock_disposal)) + wake_up(&sdp->sd_kill_wait); + + if (IS_ERR(tmp)) + return PTR_ERR(tmp); + gl = tmp; + } + +found: + *glp = gl; + return 0; +} + +/** + * __gfs2_holder_init - initialize a struct gfs2_holder in the default way + * @gl: the glock + * @state: the state we're requesting + * @flags: the modifier flags + * @gh: the holder structure + * @ip: caller's return address for debugging + */ + +void __gfs2_holder_init(struct gfs2_glock *gl, unsigned int state, u16 flags, + struct gfs2_holder *gh, unsigned long ip) +{ + INIT_LIST_HEAD(&gh->gh_list); + gh->gh_gl = gfs2_glock_hold(gl); + gh->gh_ip = ip; + gh->gh_owner_pid = get_pid(task_pid(current)); + gh->gh_state = state; + gh->gh_flags = flags; + gh->gh_iflags = 0; +} + +/** + * gfs2_holder_reinit - reinitialize a struct gfs2_holder so we can requeue it + * @state: the state we're requesting + * @flags: the modifier flags + * @gh: the holder structure + * + * Don't mess with the glock. + * + */ + +void gfs2_holder_reinit(unsigned int state, u16 flags, struct gfs2_holder *gh) +{ + gh->gh_state = state; + gh->gh_flags = flags; + gh->gh_iflags = 0; + gh->gh_ip = _RET_IP_; + put_pid(gh->gh_owner_pid); + gh->gh_owner_pid = get_pid(task_pid(current)); +} + +/** + * gfs2_holder_uninit - uninitialize a holder structure (drop glock reference) + * @gh: the holder structure + * + */ + +void gfs2_holder_uninit(struct gfs2_holder *gh) +{ + put_pid(gh->gh_owner_pid); + gfs2_glock_put(gh->gh_gl); + gfs2_holder_mark_uninitialized(gh); + gh->gh_ip = 0; +} + +static void gfs2_glock_update_hold_time(struct gfs2_glock *gl, + unsigned long start_time) +{ + /* Have we waited longer that a second? */ + if (time_after(jiffies, start_time + HZ)) { + /* Lengthen the minimum hold time. */ + gl->gl_hold_time = min(gl->gl_hold_time + GL_GLOCK_HOLD_INCR, + GL_GLOCK_MAX_HOLD); + } +} + +/** + * gfs2_glock_holder_ready - holder is ready and its error code can be collected + * @gh: the glock holder + * + * Called when a glock holder no longer needs to be waited for because it is + * now either held (HIF_HOLDER set; gh_error == 0), or acquiring the lock has + * failed (gh_error != 0). + */ + +int gfs2_glock_holder_ready(struct gfs2_holder *gh) +{ + if (gh->gh_error || (gh->gh_flags & GL_SKIP)) + return gh->gh_error; + gh->gh_error = gfs2_instantiate(gh); + if (gh->gh_error) + gfs2_glock_dq(gh); + return gh->gh_error; +} + +/** + * gfs2_glock_wait - wait on a glock acquisition + * @gh: the glock holder + * + * Returns: 0 on success + */ + +int gfs2_glock_wait(struct gfs2_holder *gh) +{ + unsigned long start_time = jiffies; + + might_sleep(); + wait_on_bit(&gh->gh_iflags, HIF_WAIT, TASK_UNINTERRUPTIBLE); + gfs2_glock_update_hold_time(gh->gh_gl, start_time); + return gfs2_glock_holder_ready(gh); +} + +static int glocks_pending(unsigned int num_gh, struct gfs2_holder *ghs) +{ + int i; + + for (i = 0; i < num_gh; i++) + if (test_bit(HIF_WAIT, &ghs[i].gh_iflags)) + return 1; + return 0; +} + +/** + * gfs2_glock_async_wait - wait on multiple asynchronous glock acquisitions + * @num_gh: the number of holders in the array + * @ghs: the glock holder array + * @retries: number of retries attempted so far + * + * Returns: 0 on success, meaning all glocks have been granted and are held. + * -ESTALE if the request timed out, meaning all glocks were released, + * and the caller should retry the operation. + */ + +int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs, + unsigned int retries) +{ + struct gfs2_sbd *sdp = glock_sbd(ghs[0].gh_gl); + unsigned long start_time = jiffies; + int i, ret = 0; + long timeout; + + might_sleep(); + + timeout = GL_GLOCK_MIN_HOLD; + if (retries) { + unsigned int max_shift; + long incr; + + /* Add a random delay and increase the timeout exponentially. */ + max_shift = BITS_PER_LONG - 2 - __fls(GL_GLOCK_HOLD_INCR); + incr = min(GL_GLOCK_HOLD_INCR << min(retries - 1, max_shift), + 10 * HZ - GL_GLOCK_MIN_HOLD); + schedule_timeout_interruptible(get_random_long() % (incr / 3)); + if (signal_pending(current)) + goto interrupted; + timeout += (incr / 3) + get_random_long() % (incr / 3); + } + + if (!wait_event_interruptible_timeout(sdp->sd_async_glock_wait, + !glocks_pending(num_gh, ghs), timeout)) { + ret = -ESTALE; /* request timed out. */ + goto out; + } + if (signal_pending(current)) + goto interrupted; + + for (i = 0; i < num_gh; i++) { + struct gfs2_holder *gh = &ghs[i]; + int ret2; + + if (test_bit(HIF_HOLDER, &gh->gh_iflags)) { + gfs2_glock_update_hold_time(gh->gh_gl, + start_time); + } + ret2 = gfs2_glock_holder_ready(gh); + if (!ret) + ret = ret2; + } + +out: + if (ret) { + for (i = 0; i < num_gh; i++) { + struct gfs2_holder *gh = &ghs[i]; + + gfs2_glock_dq(gh); + } + } + return ret; + +interrupted: + ret = -EINTR; + goto out; +} + +/** + * request_demote - process a demote request + * @gl: the glock + * @state: the state the caller wants us to change to + * @delay: zero to demote immediately; otherwise pending demote + * @remote: true if this came from a different cluster node + * + * There are only two requests that we are going to see in actual + * practise: LM_ST_SHARED and LM_ST_UNLOCKED + */ + +static void request_demote(struct gfs2_glock *gl, unsigned int state, + unsigned long delay, bool remote) +{ + gfs2_set_demote(delay ? GLF_PENDING_DEMOTE : GLF_DEMOTE, gl); + if (gl->gl_demote_state == LM_ST_EXCLUSIVE) { + gl->gl_demote_state = state; + gl->gl_demote_time = jiffies; + } else if (gl->gl_demote_state != LM_ST_UNLOCKED && + gl->gl_demote_state != state) { + gl->gl_demote_state = LM_ST_UNLOCKED; + } + if (gl->gl_ops->go_callback) + gl->gl_ops->go_callback(gl, remote); + trace_gfs2_demote_rq(gl, remote); +} + +void gfs2_print_dbg(struct seq_file *seq, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + + if (seq) { + seq_vprintf(seq, fmt, args); + } else { + vaf.fmt = fmt; + vaf.va = &args; + + pr_err("%pV", &vaf); + } + + va_end(args); +} + +static bool gfs2_should_queue_trylock(struct gfs2_glock *gl, + struct gfs2_holder *gh) +{ + struct gfs2_holder *current_gh, *gh2; + + current_gh = find_first_holder(gl); + if (current_gh && !may_grant(gl, current_gh, gh)) + return false; + + list_for_each_entry(gh2, &gl->gl_holders, gh_list) { + if (test_bit(HIF_HOLDER, &gh2->gh_iflags)) + continue; + if (!(gh2->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) + return false; + } + return true; +} + +static inline bool pid_is_meaningful(const struct gfs2_holder *gh) +{ + if (!(gh->gh_flags & GL_NOPID)) + return true; + return !test_bit(HIF_HOLDER, &gh->gh_iflags); +} + +/** + * add_to_queue - Add a holder to the wait queue (but look for recursion) + * @gh: the holder structure to add + * + * Eventually we should move the recursive locking trap to a + * debugging option or something like that. This is the fast + * path and needs to have the minimum number of distractions. + * + */ + +static inline void add_to_queue(struct gfs2_holder *gh) +{ + struct gfs2_glock *gl = gh->gh_gl; + struct gfs2_sbd *sdp = glock_sbd(gl); + struct gfs2_holder *gh2; + + GLOCK_BUG_ON(gl, gh->gh_owner_pid == NULL); + if (test_and_set_bit(HIF_WAIT, &gh->gh_iflags)) + GLOCK_BUG_ON(gl, true); + + if ((gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)) && + !gfs2_should_queue_trylock(gl, gh)) { + gh->gh_error = GLR_TRYFAILED; + gfs2_holder_wake(gh); + return; + } + + list_for_each_entry(gh2, &gl->gl_holders, gh_list) { + if (likely(gh2->gh_owner_pid != gh->gh_owner_pid)) + continue; + if (gh->gh_gl->gl_ops->go_type == LM_TYPE_FLOCK) + continue; + if (!pid_is_meaningful(gh2)) + continue; + goto trap_recursive; + } + trace_gfs2_glock_queue(gh, 1); + gfs2_glstats_inc(gl, GFS2_LKS_QCOUNT); + gfs2_sbstats_inc(gl, GFS2_LKS_QCOUNT); + list_add_tail(&gh->gh_list, &gl->gl_holders); + return; + +trap_recursive: + fs_err(sdp, "original: %pSR\n", (void *)gh2->gh_ip); + fs_err(sdp, "pid: %d\n", pid_nr(gh2->gh_owner_pid)); + fs_err(sdp, "lock type: %d req lock state : %d\n", + glock_type(gh2->gh_gl), gh2->gh_state); + fs_err(sdp, "new: %pSR\n", (void *)gh->gh_ip); + fs_err(sdp, "pid: %d\n", pid_nr(gh->gh_owner_pid)); + fs_err(sdp, "lock type: %d req lock state : %d\n", + glock_type(gh->gh_gl), gh->gh_state); + gfs2_dump_glock(NULL, gl, true); + BUG(); +} + +/** + * gfs2_glock_nq - enqueue a struct gfs2_holder onto a glock (acquire a glock) + * @gh: the holder structure + * + * if (gh->gh_flags & GL_ASYNC), this never returns an error + * + * Returns: 0, GLR_TRYFAILED, or errno on failure + */ + +int gfs2_glock_nq(struct gfs2_holder *gh) +{ + struct gfs2_glock *gl = gh->gh_gl; + struct gfs2_sbd *sdp = glock_sbd(gl); + int error; + + if (gfs2_withdrawn(sdp)) + return -EIO; + + if (gh->gh_flags & GL_NOBLOCK) { + struct gfs2_holder *current_gh; + + error = -ECHILD; + spin_lock(&gl->gl_lockref.lock); + if (find_last_waiter(gl)) + goto unlock; + current_gh = find_first_holder(gl); + if (!may_grant(gl, current_gh, gh)) + goto unlock; + set_bit(HIF_HOLDER, &gh->gh_iflags); + list_add_tail(&gh->gh_list, &gl->gl_holders); + trace_gfs2_promote(gh); + error = 0; +unlock: + spin_unlock(&gl->gl_lockref.lock); + return error; + } + + gh->gh_error = 0; + spin_lock(&gl->gl_lockref.lock); + add_to_queue(gh); + if (unlikely((LM_FLAG_RECOVER & gh->gh_flags) && + test_and_clear_bit(GLF_HAVE_FROZEN_REPLY, &gl->gl_flags))) { + set_bit(GLF_HAVE_REPLY, &gl->gl_flags); + gl->gl_lockref.count++; + gfs2_glock_queue_work(gl, 0); + } + run_queue(gl, 1); + spin_unlock(&gl->gl_lockref.lock); + + error = 0; + if (!(gh->gh_flags & GL_ASYNC)) + error = gfs2_glock_wait(gh); + + return error; +} + +/** + * gfs2_glock_poll - poll to see if an async request has been completed + * @gh: the holder + * + * Returns: 1 if the request is ready to be gfs2_glock_wait()ed on + */ + +int gfs2_glock_poll(struct gfs2_holder *gh) +{ + return test_bit(HIF_WAIT, &gh->gh_iflags) ? 0 : 1; +} + +static void __gfs2_glock_dq(struct gfs2_holder *gh) +{ + struct gfs2_glock *gl = gh->gh_gl; + unsigned delay = 0; + int fast_path = 0; + + /* + * This holder should not be cached, so mark it for demote. + * Note: this should be done before the glock_needs_demote + * check below. + */ + if (gh->gh_flags & GL_NOCACHE) + request_demote(gl, LM_ST_UNLOCKED, 0, false); + + list_del_init(&gh->gh_list); + clear_bit(HIF_HOLDER, &gh->gh_iflags); + trace_gfs2_glock_queue(gh, 0); + if (test_bit(HIF_WAIT, &gh->gh_iflags)) + gfs2_holder_wake(gh); + + /* + * If there hasn't been a demote request we are done. + * (Let the remaining holders, if any, keep holding it.) + */ + if (!glock_needs_demote(gl)) { + if (list_empty(&gl->gl_holders)) + fast_path = 1; + } + + if (unlikely(!fast_path)) { + gl->gl_lockref.count++; + if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) && + !test_bit(GLF_DEMOTE, &gl->gl_flags) && + glock_type(gl) == LM_TYPE_INODE) + delay = gl->gl_hold_time; + gfs2_glock_queue_work(gl, delay); + } +} + +/** + * gfs2_glock_dq - dequeue a struct gfs2_holder from a glock (release a glock) + * @gh: the glock holder + * + */ +void gfs2_glock_dq(struct gfs2_holder *gh) +{ + struct gfs2_glock *gl = gh->gh_gl; + +again: + spin_lock(&gl->gl_lockref.lock); + if (!gfs2_holder_queued(gh)) { + /* + * May have already been dequeued because the locking request + * was GL_ASYNC and it has failed in the meantime. + */ + goto out; + } + + if (list_is_first(&gh->gh_list, &gl->gl_holders) && + !test_bit(HIF_HOLDER, &gh->gh_iflags) && + test_bit(GLF_LOCK, &gl->gl_flags) && + !test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags) && + !test_bit(GLF_CANCELING, &gl->gl_flags)) { + if (!test_bit(GLF_MAY_CANCEL, &gl->gl_flags)) { + struct wait_queue_head *wq; + DEFINE_WAIT(wait); + + wq = bit_waitqueue(&gl->gl_flags, GLF_LOCK); + prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE); + spin_unlock(&gl->gl_lockref.lock); + schedule(); + finish_wait(wq, &wait); + goto again; + } + + set_bit(GLF_CANCELING, &gl->gl_flags); + spin_unlock(&gl->gl_lockref.lock); + glock_sbd(gl)->sd_lockstruct.ls_ops->lm_cancel(gl); + wait_on_bit(&gh->gh_iflags, HIF_WAIT, TASK_UNINTERRUPTIBLE); + spin_lock(&gl->gl_lockref.lock); + clear_bit(GLF_CANCELING, &gl->gl_flags); + clear_and_wake_up_bit(GLF_LOCK, &gl->gl_flags); + if (!gfs2_holder_queued(gh)) + goto out; + } + + __gfs2_glock_dq(gh); +out: + spin_unlock(&gl->gl_lockref.lock); +} + +void gfs2_glock_dq_wait(struct gfs2_holder *gh) +{ + struct gfs2_glock *gl = gh->gh_gl; + gfs2_glock_dq(gh); + might_sleep(); + wait_on_bit(&gl->gl_flags, GLF_DEMOTE, TASK_UNINTERRUPTIBLE); +} + +/** + * gfs2_glock_dq_uninit - dequeue a holder from a glock and initialize it + * @gh: the holder structure + * + */ + +void gfs2_glock_dq_uninit(struct gfs2_holder *gh) +{ + gfs2_glock_dq(gh); + gfs2_holder_uninit(gh); +} + +/** + * gfs2_glock_nq_num - acquire a glock based on lock number + * @sdp: the filesystem + * @number: the lock number + * @glops: the glock operations for the type of glock + * @state: the state to acquire the glock in + * @flags: modifier flags for the acquisition + * @gh: the struct gfs2_holder + * + * Returns: errno + */ + +int gfs2_glock_nq_num(struct gfs2_sbd *sdp, u64 number, + const struct gfs2_glock_operations *glops, + unsigned int state, u16 flags, struct gfs2_holder *gh) +{ + struct gfs2_glock *gl; + int error; + + error = gfs2_glock_get(sdp, number, glops, CREATE, &gl); + if (!error) { + error = gfs2_glock_nq_init(gl, state, flags, gh); + gfs2_glock_put(gl); + } + + return error; +} + +/** + * glock_compare - Compare two struct gfs2_glock structures for sorting + * @arg_a: the first structure + * @arg_b: the second structure + * + */ + +static int glock_compare(const void *arg_a, const void *arg_b) +{ + const struct gfs2_holder *gh_a = *(const struct gfs2_holder **)arg_a; + const struct gfs2_holder *gh_b = *(const struct gfs2_holder **)arg_b; + const struct lm_lockname *a = &gh_a->gh_gl->gl_name; + const struct lm_lockname *b = &gh_b->gh_gl->gl_name; + + if (a->ln_number > b->ln_number) + return 1; + if (a->ln_number < b->ln_number) + return -1; + BUG_ON(gh_a->gh_gl->gl_ops->go_type == gh_b->gh_gl->gl_ops->go_type); + return 0; +} + +/** + * nq_m_sync - synchronously acquire more than one glock in deadlock free order + * @num_gh: the number of structures + * @ghs: an array of struct gfs2_holder structures + * @p: placeholder for the holder structure to pass back + * + * Returns: 0 on success (all glocks acquired), + * errno on failure (no glocks acquired) + */ + +static int nq_m_sync(unsigned int num_gh, struct gfs2_holder *ghs, + struct gfs2_holder **p) +{ + unsigned int x; + int error = 0; + + for (x = 0; x < num_gh; x++) + p[x] = &ghs[x]; + + sort(p, num_gh, sizeof(struct gfs2_holder *), glock_compare, NULL); + + for (x = 0; x < num_gh; x++) { + error = gfs2_glock_nq(p[x]); + if (error) { + while (x--) + gfs2_glock_dq(p[x]); + break; + } + } + + return error; +} + +/** + * gfs2_glock_nq_m - acquire multiple glocks + * @num_gh: the number of structures + * @ghs: an array of struct gfs2_holder structures + * + * Returns: 0 on success (all glocks acquired), + * errno on failure (no glocks acquired) + */ + +int gfs2_glock_nq_m(unsigned int num_gh, struct gfs2_holder *ghs) +{ + struct gfs2_holder *tmp[4]; + struct gfs2_holder **pph = tmp; + int error = 0; + + switch(num_gh) { + case 0: + return 0; + case 1: + return gfs2_glock_nq(ghs); + default: + if (num_gh <= 4) + break; + pph = kmalloc_objs(struct gfs2_holder *, num_gh, GFP_NOFS); + if (!pph) + return -ENOMEM; + } + + error = nq_m_sync(num_gh, ghs, pph); + + if (pph != tmp) + kfree(pph); + + return error; +} + +/** + * gfs2_glock_dq_m - release multiple glocks + * @num_gh: the number of structures + * @ghs: an array of struct gfs2_holder structures + * + */ + +void gfs2_glock_dq_m(unsigned int num_gh, struct gfs2_holder *ghs) +{ + while (num_gh--) + gfs2_glock_dq(&ghs[num_gh]); +} + +void gfs2_glock_cb(struct gfs2_glock *gl, unsigned int state) +{ + unsigned long delay = 0; + + if (!lockref_get_not_dead(&gl->gl_lockref)) + return; + + spin_lock(&gl->gl_lockref.lock); + if (!list_empty(&gl->gl_holders) && + glock_type(gl) == LM_TYPE_INODE) { + unsigned long now = jiffies; + unsigned long holdtime; + + holdtime = gl->gl_tchange + gl->gl_hold_time; + + if (time_before(now, holdtime)) + delay = holdtime - now; + if (test_bit(GLF_HAVE_REPLY, &gl->gl_flags)) + delay = gl->gl_hold_time; + } + request_demote(gl, state, delay, true); + gfs2_glock_queue_work(gl, delay); + spin_unlock(&gl->gl_lockref.lock); +} + +/** + * gfs2_should_freeze - Figure out if glock should be frozen + * @gl: The glock in question + * + * Glocks are not frozen if (a) the result of the dlm operation is + * an error, (b) the locking operation was an unlock operation or + * (c) if there is a "recover" flagged request anywhere in the queue + * + * Returns: 1 if freezing should occur, 0 otherwise + */ + +static int gfs2_should_freeze(const struct gfs2_glock *gl) +{ + const struct gfs2_holder *gh; + + if (gl->gl_reply & ~LM_OUT_ST_MASK) + return 0; + if (gl->gl_target == LM_ST_UNLOCKED) + return 0; + + list_for_each_entry(gh, &gl->gl_holders, gh_list) { + if (test_bit(HIF_HOLDER, &gh->gh_iflags)) + continue; + if (LM_FLAG_RECOVER & gh->gh_flags) + return 0; + } + + return 1; +} + +/** + * gfs2_glock_complete - Callback used by locking + * @gl: Pointer to the glock + * @ret: The return value from the dlm + * + * The gl_reply field is under the gl_lockref.lock lock so that it is ok + * to use a bitfield shared with other glock state fields. + */ + +void gfs2_glock_complete(struct gfs2_glock *gl, int ret) +{ + struct lm_lockstruct *ls = &glock_sbd(gl)->sd_lockstruct; + + spin_lock(&gl->gl_lockref.lock); + clear_bit(GLF_MAY_CANCEL, &gl->gl_flags); + gl->gl_reply = ret; + + if (unlikely(test_bit(DFL_BLOCK_LOCKS, &ls->ls_recover_flags))) { + if (gfs2_should_freeze(gl)) { + set_bit(GLF_HAVE_FROZEN_REPLY, &gl->gl_flags); + spin_unlock(&gl->gl_lockref.lock); + return; + } + } + + gl->gl_lockref.count++; + set_bit(GLF_HAVE_REPLY, &gl->gl_flags); + gfs2_glock_queue_work(gl, 0); + spin_unlock(&gl->gl_lockref.lock); +} + +/** + * glock_hash_walk - Call a function for glock in a hash bucket + * @examiner: the function + * @sdp: the filesystem + * + * Note that the function can be called multiple times on the same + * object. So the user must ensure that the function can cope with + * that. + */ + +static void glock_hash_walk(glock_examiner examiner, const struct gfs2_sbd *sdp) +{ + struct gfs2_glock *gl; + struct rhashtable_iter iter; + + rhashtable_walk_enter(&gl_hash_table, &iter); + + do { + rhashtable_walk_start(&iter); + + while ((gl = rhashtable_walk_next(&iter)) && !IS_ERR(gl)) { + if (glock_sbd(gl) == sdp) + examiner(gl); + } + + rhashtable_walk_stop(&iter); + } while (cond_resched(), gl == ERR_PTR(-EAGAIN)); + + rhashtable_walk_exit(&iter); +} + +void gfs2_cancel_delete_work(struct gfs2_glock *gl) +{ + clear_bit(GLF_TRY_TO_EVICT, &gl->gl_flags); + clear_bit(GLF_VERIFY_DELETE, &gl->gl_flags); + if (cancel_delayed_work(&gl->gl_delete)) + gfs2_glock_put(gl); +} + +static void flush_delete_work(struct gfs2_glock *gl) +{ + if (glock_type(gl) == LM_TYPE_IOPEN) { + struct gfs2_sbd *sdp = glock_sbd(gl); + + if (cancel_delayed_work(&gl->gl_delete)) { + queue_delayed_work(sdp->sd_delete_wq, + &gl->gl_delete, 0); + } + } +} + +void gfs2_flush_delete_work(struct gfs2_sbd *sdp) +{ + glock_hash_walk(flush_delete_work, sdp); + flush_workqueue(sdp->sd_delete_wq); +} + +/** + * thaw_glock - thaw out a glock which has an unprocessed reply waiting + * @gl: The glock to thaw + * + */ + +static void thaw_glock(struct gfs2_glock *gl) +{ + if (!test_and_clear_bit(GLF_HAVE_FROZEN_REPLY, &gl->gl_flags)) + return; + if (!lockref_get_not_dead(&gl->gl_lockref)) + return; + + spin_lock(&gl->gl_lockref.lock); + set_bit(GLF_HAVE_REPLY, &gl->gl_flags); + gfs2_glock_queue_work(gl, 0); + spin_unlock(&gl->gl_lockref.lock); +} + +/** + * gfs2_glock_thaw - Thaw any frozen glocks + * @sdp: The super block + * + */ + +void gfs2_glock_thaw(struct gfs2_sbd *sdp) +{ + glock_hash_walk(thaw_glock, sdp); +} + +static void dump_glock(struct seq_file *seq, struct gfs2_glock *gl, bool fsid) +{ + spin_lock(&gl->gl_lockref.lock); + gfs2_dump_glock(seq, gl, fsid); + spin_unlock(&gl->gl_lockref.lock); +} + +static void dump_glock_func(struct gfs2_glock *gl) +{ + dump_glock(NULL, gl, true); +} + +static void withdraw_glock(struct gfs2_glock *gl) +{ + spin_lock(&gl->gl_lockref.lock); + if (!lockref_is_dead(&gl->gl_lockref)) { + /* + * We don't want to write back any more dirty data. Unlock the + * remaining inode and resource group glocks; this will cause + * their ->go_inval() hooks to toss out all the remaining + * cached data, dirty or not. + */ + if (gl->gl_ops->go_inval && gl->gl_state != LM_ST_UNLOCKED) + request_demote(gl, LM_ST_UNLOCKED, 0, false); + do_error(gl, LM_OUT_ERROR); /* remove pending waiters */ + } + spin_unlock(&gl->gl_lockref.lock); +} + +void gfs2_withdraw_glocks(struct gfs2_sbd *sdp) +{ + glock_hash_walk(withdraw_glock, sdp); +} + +/** + * gfs2_wait_glocks - Wait for the remaining glocks to go away + * @sdp: the filesystem + * + * Called when unmounting the filesystem. + */ + +void gfs2_wait_glocks(struct gfs2_sbd *sdp) +{ + unsigned long start = jiffies; + bool timed_out = false; + + while (!timed_out) { + wait_event_timeout(sdp->sd_kill_wait, + !atomic_read(&sdp->sd_glock_disposal), + HZ * 60); + if (!atomic_read(&sdp->sd_glock_disposal)) + break; + timed_out = time_after(jiffies, start + (HZ * 600)); + fs_warn(sdp, "%u glocks left after %u seconds%s\n", + atomic_read(&sdp->sd_glock_disposal), + jiffies_to_msecs(jiffies - start) / 1000, + timed_out ? ":" : "; still waiting"); + } + gfs2_lm_unmount(sdp); + gfs2_free_dead_glocks(sdp); + glock_hash_walk(dump_glock_func, sdp); + destroy_workqueue(sdp->sd_glock_wq); + sdp->sd_glock_wq = NULL; +} + +static const char *state2str(unsigned state) +{ + switch(state) { + case LM_ST_UNLOCKED: + return "UN"; + case LM_ST_SHARED: + return "SH"; + case LM_ST_DEFERRED: + return "DF"; + case LM_ST_EXCLUSIVE: + return "EX"; + } + return "??"; +} + +static const char *hflags2str(char *buf, u16 flags, unsigned long iflags) +{ + char *p = buf; + if (flags & LM_FLAG_TRY) + *p++ = 't'; + if (flags & LM_FLAG_TRY_1CB) + *p++ = 'T'; + if (flags & LM_FLAG_RECOVER) + *p++ = 'e'; + if (flags & LM_FLAG_ANY) + *p++ = 'A'; + if (flags & LM_FLAG_NODE_SCOPE) + *p++ = 'n'; + if (flags & GL_ASYNC) + *p++ = 'a'; + if (flags & GL_EXACT) + *p++ = 'E'; + if (flags & GL_NOCACHE) + *p++ = 'c'; + if (test_bit(HIF_HOLDER, &iflags)) + *p++ = 'H'; + if (test_bit(HIF_WAIT, &iflags)) + *p++ = 'W'; + if (flags & GL_SKIP) + *p++ = 's'; + *p = 0; + return buf; +} + +/** + * dump_holder - print information about a glock holder + * @seq: the seq_file struct + * @gh: the glock holder + * @fs_id_buf: pointer to file system id (if requested) + * + */ + +static void dump_holder(struct seq_file *seq, const struct gfs2_holder *gh, + const char *fs_id_buf) +{ + const char *comm = "(none)"; + pid_t owner_pid = 0; + char flags_buf[32]; + + rcu_read_lock(); + if (pid_is_meaningful(gh)) { + struct task_struct *gh_owner; + + comm = "(ended)"; + owner_pid = pid_nr(gh->gh_owner_pid); + gh_owner = pid_task(gh->gh_owner_pid, PIDTYPE_PID); + if (gh_owner) + comm = gh_owner->comm; + } + gfs2_print_dbg(seq, "%s H: s:%s f:%s e:%d p:%ld [%s] %pS\n", + fs_id_buf, state2str(gh->gh_state), + hflags2str(flags_buf, gh->gh_flags, gh->gh_iflags), + gh->gh_error, (long)owner_pid, comm, (void *)gh->gh_ip); + rcu_read_unlock(); +} + +static const char *gflags2str(char *buf, const struct gfs2_glock *gl) +{ + const unsigned long *gflags = &gl->gl_flags; + char *p = buf; + + if (test_bit(GLF_LOCK, gflags)) + *p++ = 'l'; + if (test_bit(GLF_DEMOTE, gflags)) + *p++ = 'D'; + if (test_bit(GLF_PENDING_DEMOTE, gflags)) + *p++ = 'd'; + if (test_bit(GLF_DEMOTE_IN_PROGRESS, gflags)) + *p++ = 'p'; + if (test_bit(GLF_DIRTY, gflags)) + *p++ = 'y'; + if (test_bit(GLF_LFLUSH, gflags)) + *p++ = 'f'; + if (test_bit(GLF_MAY_CANCEL, gflags)) + *p++ = 'c'; + if (test_bit(GLF_HAVE_REPLY, gflags)) + *p++ = 'r'; + if (test_bit(GLF_INITIAL, gflags)) + *p++ = 'a'; + if (test_bit(GLF_HAVE_FROZEN_REPLY, gflags)) + *p++ = 'F'; + if (!list_empty(&gl->gl_holders)) + *p++ = 'q'; + if (gl->gl_object) + *p++ = 'o'; + if (test_bit(GLF_BLOCKING, gflags)) + *p++ = 'b'; + if (test_bit(GLF_INSTANTIATE_NEEDED, gflags)) + *p++ = 'n'; + if (test_bit(GLF_INSTANTIATE_IN_PROG, gflags)) + *p++ = 'N'; + if (test_bit(GLF_TRY_TO_EVICT, gflags)) + *p++ = 'e'; + if (test_bit(GLF_VERIFY_DELETE, gflags)) + *p++ = 'E'; + if (test_bit(GLF_DEFER_DELETE, gflags)) + *p++ = 's'; + if (test_bit(GLF_CANCELING, gflags)) + *p++ = 'C'; + *p = 0; + return buf; +} + +/** + * gfs2_dump_glock - print information about a glock + * @seq: The seq_file struct + * @gl: the glock + * @fsid: If true, also dump the file system id + * + * The file format is as follows: + * One line per object, capital letters are used to indicate objects + * G = glock, I = Inode, R = rgrp, H = holder. Glocks are not indented, + * other objects are indented by a single space and follow the glock to + * which they are related. Fields are indicated by lower case letters + * followed by a colon and the field value, except for strings which are in + * [] so that its possible to see if they are composed of spaces for + * example. The field's are n = number (id of the object), f = flags, + * t = type, s = state, r = refcount, e = error, p = pid. + * + */ + +void gfs2_dump_glock(struct seq_file *seq, struct gfs2_glock *gl, bool fsid) +{ + const struct gfs2_glock_operations *glops = gl->gl_ops; + unsigned long long dtime; + const struct gfs2_holder *gh; + char gflags_buf[32]; + struct gfs2_sbd *sdp = glock_sbd(gl); + char fs_id_buf[sizeof(sdp->sd_fsname) + 7]; + unsigned long nrpages = 0; + + if (gl->gl_ops->go_flags & GLOF_ASPACE) { + struct address_space *mapping = gfs2_glock2aspace(gl); + + nrpages = mapping->nrpages; + } + memset(fs_id_buf, 0, sizeof(fs_id_buf)); + if (fsid && sdp) /* safety precaution */ + sprintf(fs_id_buf, "fsid=%s: ", sdp->sd_fsname); + dtime = jiffies - gl->gl_demote_time; + dtime *= 1000000/HZ; /* demote time in uSec */ + if (!test_bit(GLF_DEMOTE, &gl->gl_flags)) + dtime = 0; + gfs2_print_dbg(seq, "%sG: s:%s n:%u/%llx f:%s t:%s d:%s/%llu a:%d " + "v:%d r:%d m:%ld p:%lu\n", + fs_id_buf, state2str(gl->gl_state), + glock_type(gl), + (unsigned long long) glock_number(gl), + gflags2str(gflags_buf, gl), + state2str(gl->gl_target), + state2str(gl->gl_demote_state), dtime, + atomic_read(&gl->gl_ail_count), + atomic_read(&gl->gl_revokes), + (int)gl->gl_lockref.count, gl->gl_hold_time, nrpages); + + list_for_each_entry(gh, &gl->gl_holders, gh_list) + dump_holder(seq, gh, fs_id_buf); + + if (gl->gl_state != LM_ST_UNLOCKED && glops->go_dump) + glops->go_dump(seq, gl, fs_id_buf); +} + +static int gfs2_glstats_seq_show(struct seq_file *seq, void *iter_ptr) +{ + struct gfs2_glock *gl = iter_ptr; + + seq_printf(seq, "G: n:%u/%llx rtt:%llu/%llu rttb:%llu/%llu irt:%llu/%llu dcnt: %llu qcnt: %llu\n", + glock_type(gl), + (unsigned long long) glock_number(gl), + (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTT], + (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTVAR], + (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTB], + (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTVARB], + (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SIRT], + (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SIRTVAR], + (unsigned long long)gl->gl_stats.stats[GFS2_LKS_DCOUNT], + (unsigned long long)gl->gl_stats.stats[GFS2_LKS_QCOUNT]); + return 0; +} + +static const char *gfs2_gltype[] = { + "type", + "reserved", + "nondisk", + "inode", + "rgrp", + "meta", + "iopen", + "flock", + "plock", + "quota", + "journal", +}; + +static const char *gfs2_stype[] = { + [GFS2_LKS_SRTT] = "srtt", + [GFS2_LKS_SRTTVAR] = "srttvar", + [GFS2_LKS_SRTTB] = "srttb", + [GFS2_LKS_SRTTVARB] = "srttvarb", + [GFS2_LKS_SIRT] = "sirt", + [GFS2_LKS_SIRTVAR] = "sirtvar", + [GFS2_LKS_DCOUNT] = "dlm", + [GFS2_LKS_QCOUNT] = "queue", +}; + +#define GFS2_NR_SBSTATS (ARRAY_SIZE(gfs2_gltype) * ARRAY_SIZE(gfs2_stype)) + +static int gfs2_sbstats_seq_show(struct seq_file *seq, void *iter_ptr) +{ + struct gfs2_sbd *sdp = seq->private; + loff_t pos = *(loff_t *)iter_ptr; + unsigned index = pos >> 3; + unsigned subindex = pos & 0x07; + int i; + + if (index == 0 && subindex != 0) + return 0; + + seq_printf(seq, "%-10s %8s:", gfs2_gltype[index], + (index == 0) ? "cpu": gfs2_stype[subindex]); + + for_each_possible_cpu(i) { + const struct gfs2_pcpu_lkstats *lkstats = per_cpu_ptr(sdp->sd_lkstats, i); + + if (index == 0) + seq_printf(seq, " %15u", i); + else + seq_printf(seq, " %15llu", (unsigned long long)lkstats-> + lkstats[index - 1].stats[subindex]); + } + seq_putc(seq, '\n'); + return 0; +} + +int __init gfs2_glock_init(void) +{ + int i, ret; + + ret = rhashtable_init(&gl_hash_table, &ht_parms); + if (ret < 0) + return ret; + + for (i = 0; i < GLOCK_WAIT_TABLE_SIZE; i++) + init_waitqueue_head(glock_wait_table + i); + + return 0; +} + +void gfs2_glock_exit(void) +{ + rhashtable_destroy(&gl_hash_table); +} + +static void gfs2_glock_iter_next(struct gfs2_glock_iter *gi, loff_t n) +{ + struct gfs2_glock *gl = gi->gl; + + if (gl) { + if (n == 0) + return; + gfs2_glock_put_async(gl); + } + for (;;) { + gl = rhashtable_walk_next(&gi->hti); + if (IS_ERR_OR_NULL(gl)) { + if (gl == ERR_PTR(-EAGAIN)) { + n = 1; + continue; + } + gl = NULL; + break; + } + if (glock_sbd(gl) != gi->sdp) + continue; + if (n <= 1) { + if (!lockref_get_not_dead(&gl->gl_lockref)) + continue; + break; + } else { + if (lockref_is_dead(&gl->gl_lockref)) + continue; + n--; + } + } + gi->gl = gl; +} + +static void *gfs2_glock_seq_start(struct seq_file *seq, loff_t *pos) + __acquires(RCU) +{ + struct gfs2_glock_iter *gi = seq->private; + loff_t n; + + /* + * We can either stay where we are, skip to the next hash table + * entry, or start from the beginning. + */ + if (*pos < gi->last_pos) { + rhashtable_walk_exit(&gi->hti); + rhashtable_walk_enter(&gl_hash_table, &gi->hti); + n = *pos + 1; + } else { + n = *pos - gi->last_pos; + } + + rhashtable_walk_start(&gi->hti); + + gfs2_glock_iter_next(gi, n); + gi->last_pos = *pos; + return gi->gl; +} + +static void *gfs2_glock_seq_next(struct seq_file *seq, void *iter_ptr, + loff_t *pos) +{ + struct gfs2_glock_iter *gi = seq->private; + + (*pos)++; + gi->last_pos = *pos; + gfs2_glock_iter_next(gi, 1); + return gi->gl; +} + +static void gfs2_glock_seq_stop(struct seq_file *seq, void *iter_ptr) + __releases(RCU) +{ + struct gfs2_glock_iter *gi = seq->private; + + rhashtable_walk_stop(&gi->hti); +} + +static int gfs2_glock_seq_show(struct seq_file *seq, void *iter_ptr) +{ + dump_glock(seq, iter_ptr, false); + return 0; +} + +static void *gfs2_sbstats_seq_start(struct seq_file *seq, loff_t *pos) +{ + preempt_disable(); + if (*pos >= GFS2_NR_SBSTATS) + return NULL; + return pos; +} + +static void *gfs2_sbstats_seq_next(struct seq_file *seq, void *iter_ptr, + loff_t *pos) +{ + (*pos)++; + if (*pos >= GFS2_NR_SBSTATS) + return NULL; + return pos; +} + +static void gfs2_sbstats_seq_stop(struct seq_file *seq, void *iter_ptr) +{ + preempt_enable(); +} + +static const struct seq_operations gfs2_glock_seq_ops = { + .start = gfs2_glock_seq_start, + .next = gfs2_glock_seq_next, + .stop = gfs2_glock_seq_stop, + .show = gfs2_glock_seq_show, +}; + +static const struct seq_operations gfs2_glstats_seq_ops = { + .start = gfs2_glock_seq_start, + .next = gfs2_glock_seq_next, + .stop = gfs2_glock_seq_stop, + .show = gfs2_glstats_seq_show, +}; + +static const struct seq_operations gfs2_sbstats_sops = { + .start = gfs2_sbstats_seq_start, + .next = gfs2_sbstats_seq_next, + .stop = gfs2_sbstats_seq_stop, + .show = gfs2_sbstats_seq_show, +}; + +#define GFS2_SEQ_GOODSIZE min(PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER, 65536UL) + +static int __gfs2_glocks_open(struct inode *inode, struct file *file, + const struct seq_operations *ops) +{ + int ret = seq_open_private(file, ops, sizeof(struct gfs2_glock_iter)); + if (ret == 0) { + struct seq_file *seq = file->private_data; + struct gfs2_glock_iter *gi = seq->private; + + gi->sdp = inode->i_private; + seq->buf = kmalloc(GFS2_SEQ_GOODSIZE, GFP_KERNEL | __GFP_NOWARN); + if (seq->buf) + seq->size = GFS2_SEQ_GOODSIZE; + /* + * Initially, we are "before" the first hash table entry; the + * first call to rhashtable_walk_next gets us the first entry. + */ + gi->last_pos = -1; + gi->gl = NULL; + rhashtable_walk_enter(&gl_hash_table, &gi->hti); + } + return ret; +} + +static int gfs2_glocks_open(struct inode *inode, struct file *file) +{ + return __gfs2_glocks_open(inode, file, &gfs2_glock_seq_ops); +} + +static int gfs2_glocks_release(struct inode *inode, struct file *file) +{ + struct seq_file *seq = file->private_data; + struct gfs2_glock_iter *gi = seq->private; + + if (gi->gl) + gfs2_glock_put(gi->gl); + rhashtable_walk_exit(&gi->hti); + return seq_release_private(inode, file); +} + +static int gfs2_glstats_open(struct inode *inode, struct file *file) +{ + return __gfs2_glocks_open(inode, file, &gfs2_glstats_seq_ops); +} + +static const struct file_operations gfs2_glocks_fops = { + .owner = THIS_MODULE, + .open = gfs2_glocks_open, + .read = seq_read, + .llseek = seq_lseek, + .release = gfs2_glocks_release, +}; + +static const struct file_operations gfs2_glstats_fops = { + .owner = THIS_MODULE, + .open = gfs2_glstats_open, + .read = seq_read, + .llseek = seq_lseek, + .release = gfs2_glocks_release, +}; + +struct gfs2_glockfd_iter { + struct super_block *sb; + unsigned int tgid; + struct task_struct *task; + unsigned int fd; + struct file *file; +}; + +static struct task_struct *gfs2_glockfd_next_task(struct gfs2_glockfd_iter *i) +{ + struct pid_namespace *ns = task_active_pid_ns(current); + struct pid *pid; + + if (i->task) + put_task_struct(i->task); + + rcu_read_lock(); +retry: + i->task = NULL; + pid = find_ge_pid(i->tgid, ns); + if (pid) { + i->tgid = pid_nr_ns(pid, ns); + i->task = pid_task(pid, PIDTYPE_TGID); + if (!i->task) { + i->tgid++; + goto retry; + } + get_task_struct(i->task); + } + rcu_read_unlock(); + return i->task; +} + +static struct file *gfs2_glockfd_next_file(struct gfs2_glockfd_iter *i) +{ + if (i->file) { + fput(i->file); + i->file = NULL; + } + + for(;; i->fd++) { + i->file = fget_task_next(i->task, &i->fd); + if (!i->file) { + i->fd = 0; + break; + } + + if (file_inode(i->file)->i_sb == i->sb) + break; + + fput(i->file); + } + return i->file; +} + +static void *gfs2_glockfd_seq_start(struct seq_file *seq, loff_t *pos) +{ + struct gfs2_glockfd_iter *i = seq->private; + + if (*pos) + return NULL; + while (gfs2_glockfd_next_task(i)) { + if (gfs2_glockfd_next_file(i)) + return i; + i->tgid++; + } + return NULL; +} + +static void *gfs2_glockfd_seq_next(struct seq_file *seq, void *iter_ptr, + loff_t *pos) +{ + struct gfs2_glockfd_iter *i = seq->private; + + (*pos)++; + i->fd++; + do { + if (gfs2_glockfd_next_file(i)) + return i; + i->tgid++; + } while (gfs2_glockfd_next_task(i)); + return NULL; +} + +static void gfs2_glockfd_seq_stop(struct seq_file *seq, void *iter_ptr) +{ + struct gfs2_glockfd_iter *i = seq->private; + + if (i->file) + fput(i->file); + if (i->task) + put_task_struct(i->task); +} + +static void gfs2_glockfd_seq_show_flock(struct seq_file *seq, + struct gfs2_glockfd_iter *i) +{ + struct gfs2_file *fp = i->file->private_data; + struct gfs2_holder *fl_gh = &fp->f_fl_gh; + struct lm_lockname gl_name = { .ln_type = LM_TYPE_RESERVED }; + + if (!READ_ONCE(fl_gh->gh_gl)) + return; + + spin_lock(&i->file->f_lock); + if (gfs2_holder_initialized(fl_gh)) + gl_name = fl_gh->gh_gl->gl_name; + spin_unlock(&i->file->f_lock); + + if (gl_name.ln_type != LM_TYPE_RESERVED) { + seq_printf(seq, "%d %u %u/%llx\n", + i->tgid, i->fd, gl_name.ln_type, + (unsigned long long)gl_name.ln_number); + } +} + +static int gfs2_glockfd_seq_show(struct seq_file *seq, void *iter_ptr) +{ + struct gfs2_glockfd_iter *i = seq->private; + struct inode *inode = file_inode(i->file); + struct gfs2_glock *gl; + + inode_lock_shared(inode); + gl = GFS2_I(inode)->i_iopen_gh.gh_gl; + if (gl) { + seq_printf(seq, "%d %u %u/%llx\n", + i->tgid, i->fd, glock_type(gl), + (unsigned long long) glock_number(gl)); + } + gfs2_glockfd_seq_show_flock(seq, i); + inode_unlock_shared(inode); + return 0; +} + +static const struct seq_operations gfs2_glockfd_seq_ops = { + .start = gfs2_glockfd_seq_start, + .next = gfs2_glockfd_seq_next, + .stop = gfs2_glockfd_seq_stop, + .show = gfs2_glockfd_seq_show, +}; + +static int gfs2_glockfd_open(struct inode *inode, struct file *file) +{ + struct gfs2_glockfd_iter *i; + struct gfs2_sbd *sdp = inode->i_private; + + i = __seq_open_private(file, &gfs2_glockfd_seq_ops, + sizeof(struct gfs2_glockfd_iter)); + if (!i) + return -ENOMEM; + i->sb = sdp->sd_vfs; + return 0; +} + +static const struct file_operations gfs2_glockfd_fops = { + .owner = THIS_MODULE, + .open = gfs2_glockfd_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release_private, +}; + +DEFINE_SEQ_ATTRIBUTE(gfs2_sbstats); + +void gfs2_create_debugfs_file(struct gfs2_sbd *sdp) +{ + sdp->debugfs_dir = debugfs_create_dir(sdp->sd_table_name, gfs2_root); + + debugfs_create_file("glocks", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp, + &gfs2_glocks_fops); + + debugfs_create_file("glockfd", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp, + &gfs2_glockfd_fops); + + debugfs_create_file("glstats", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp, + &gfs2_glstats_fops); + + debugfs_create_file("sbstats", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp, + &gfs2_sbstats_fops); +} + +void gfs2_delete_debugfs_file(struct gfs2_sbd *sdp) +{ + debugfs_remove_recursive(sdp->debugfs_dir); + sdp->debugfs_dir = NULL; +} + +void gfs2_register_debugfs(void) +{ + gfs2_root = debugfs_create_dir("gfs2", NULL); +} + +void gfs2_unregister_debugfs(void) +{ + debugfs_remove(gfs2_root); + gfs2_root = NULL; +} diff --git a/fs/gfs2/glock.h b/fs/gfs2/glock.h new file mode 100644 index 000000000..8b9e22bef --- /dev/null +++ b/fs/gfs2/glock.h @@ -0,0 +1,309 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#ifndef __GLOCK_DOT_H__ +#define __GLOCK_DOT_H__ + +#include +#include +#include "incore.h" +#include "util.h" + +/* Options for hostdata parser */ + +enum { + Opt_jid, + Opt_id, + Opt_first, + Opt_nodir, + Opt_err, +}; + +/* + * lm_lockname types + */ + +#define LM_TYPE_RESERVED 0x00 +#define LM_TYPE_NONDISK 0x01 +#define LM_TYPE_INODE 0x02 +#define LM_TYPE_RGRP 0x03 +#define LM_TYPE_META 0x04 +#define LM_TYPE_IOPEN 0x05 +#define LM_TYPE_FLOCK 0x06 +#define LM_TYPE_PLOCK 0x07 +#define LM_TYPE_QUOTA 0x08 +#define LM_TYPE_JOURNAL 0x09 + +/* + * lm_lock() states + * + * SHARED is compatible with SHARED, not with DEFERRED or EX. + * DEFERRED is compatible with DEFERRED, not with SHARED or EX. + */ + +#define LM_ST_UNLOCKED 0 +#define LM_ST_EXCLUSIVE 1 +#define LM_ST_DEFERRED 2 +#define LM_ST_SHARED 3 + +/* + * lm_lock() flags + * + * LM_FLAG_TRY + * Don't wait to acquire the lock if it can't be granted immediately. + * + * LM_FLAG_TRY_1CB + * Send one blocking callback if TRY is set and the lock is not granted. + * + * LM_FLAG_RECOVER + * GFS sets this flag on lock requests it makes while doing journal recovery. + * While ordinary requests are blocked until the end of recovery, requests + * with this flag set do proceed. + * + * LM_FLAG_ANY + * A SHARED request may also be granted in DEFERRED, or a DEFERRED request may + * also be granted in SHARED. The preferred state is whichever is compatible + * with other granted locks, or the specified state if no other locks exist. + * + * In addition, when a lock is already held in EX mode locally, a SHARED or + * DEFERRED mode request with the LM_FLAG_ANY flag set will be granted. + * (The LM_FLAG_ANY flag is only use for SHARED mode requests currently.) + * + * LM_FLAG_NODE_SCOPE + * This holder agrees to share the lock within this node. In other words, + * the glock is held in EX mode according to DLM, but local holders on the + * same node can share it. + */ + +#define LM_FLAG_TRY 0x0001 +#define LM_FLAG_TRY_1CB 0x0002 +#define LM_FLAG_RECOVER 0x0004 +#define LM_FLAG_ANY 0x0008 +#define LM_FLAG_NODE_SCOPE 0x0020 +#define GL_ASYNC 0x0040 +#define GL_EXACT 0x0080 +#define GL_SKIP 0x0100 +#define GL_NOPID 0x0200 +#define GL_NOCACHE 0x0400 +#define GL_NOBLOCK 0x0800 + +/* + * lm_async_cb return flags + * + * LM_OUT_ST_MASK + * Masks the lower two bits of lock state in the returned value. + * + * LM_OUT_TRY_AGAIN + * The trylock request failed. + * + * LM_OUT_DEADLOCK + * The lock request failed because it would deadlock. + * + * LM_OUT_CANCELED + * The lock request was canceled. + * + * LM_OUT_ERROR + * The lock request timed out or failed. + */ + +#define LM_OUT_ST_MASK 0x00000003 +#define LM_OUT_TRY_AGAIN 0x00000020 +#define LM_OUT_DEADLOCK 0x00000010 +#define LM_OUT_CANCELED 0x00000008 +#define LM_OUT_ERROR 0x00000004 + +/* + * lm_recovery_done() messages + */ + +#define LM_RD_GAVEUP 308 +#define LM_RD_SUCCESS 309 + +#define GLR_TRYFAILED 13 + +#define GL_GLOCK_MAX_HOLD (long)(HZ / 5) +#define GL_GLOCK_DFT_HOLD (long)(HZ / 5) +#define GL_GLOCK_MIN_HOLD (long)(HZ / 100) +#define GL_GLOCK_HOLD_INCR (long)(HZ / 20) +#define GL_GLOCK_HOLD_DECR (long)(HZ / 40) + +struct lm_lockops { + const char *lm_proto_name; + int (*lm_mount) (struct gfs2_sbd *sdp, const char *table); + void (*lm_first_done) (struct gfs2_sbd *sdp); + void (*lm_recovery_result) (struct gfs2_sbd *sdp, unsigned int jid, + unsigned int result); + void (*lm_unmount) (struct gfs2_sbd *sdp, bool clean); + void (*lm_withdraw) (struct gfs2_sbd *sdp); + void (*lm_put_lock) (struct gfs2_glock *gl); + int (*lm_lock) (struct gfs2_glock *gl, unsigned int req_state, + unsigned int flags); + void (*lm_cancel) (struct gfs2_glock *gl); + const match_table_t *lm_tokens; +}; + +struct gfs2_glock_aspace { + struct gfs2_glock glock; + struct address_space mapping; +}; + +static inline struct gfs2_holder *gfs2_glock_is_locked_by_me(struct gfs2_glock *gl) +{ + struct gfs2_holder *gh; + struct pid *pid; + + /* Look in glock's list of holders for one with current task as owner */ + spin_lock(&gl->gl_lockref.lock); + pid = task_pid(current); + list_for_each_entry(gh, &gl->gl_holders, gh_list) { + if (!test_bit(HIF_HOLDER, &gh->gh_iflags)) + break; + if (gh->gh_owner_pid == pid) + goto out; + } + gh = NULL; +out: + spin_unlock(&gl->gl_lockref.lock); + + return gh; +} + +static inline struct address_space *gfs2_glock2aspace(struct gfs2_glock *gl) +{ + if (gl->gl_ops->go_flags & GLOF_ASPACE) { + struct gfs2_glock_aspace *gla = + container_of(gl, struct gfs2_glock_aspace, glock); + return &gla->mapping; + } + return NULL; +} + +int gfs2_glock_get(struct gfs2_sbd *sdp, u64 number, + const struct gfs2_glock_operations *glops, + int create, struct gfs2_glock **glp); +struct gfs2_glock *gfs2_glock_hold(struct gfs2_glock *gl); +void gfs2_glock_put(struct gfs2_glock *gl); +void gfs2_glock_put_async(struct gfs2_glock *gl); + +void __gfs2_holder_init(struct gfs2_glock *gl, unsigned int state, + u16 flags, struct gfs2_holder *gh, + unsigned long ip); +static inline void gfs2_holder_init(struct gfs2_glock *gl, unsigned int state, + u16 flags, struct gfs2_holder *gh) { + __gfs2_holder_init(gl, state, flags, gh, _RET_IP_); +} + +void gfs2_holder_reinit(unsigned int state, u16 flags, + struct gfs2_holder *gh); +void gfs2_holder_uninit(struct gfs2_holder *gh); +int gfs2_glock_nq(struct gfs2_holder *gh); +int gfs2_glock_poll(struct gfs2_holder *gh); +int gfs2_instantiate(struct gfs2_holder *gh); +int gfs2_glock_holder_ready(struct gfs2_holder *gh); +int gfs2_glock_wait(struct gfs2_holder *gh); +int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs, + unsigned int retries); +void gfs2_glock_dq(struct gfs2_holder *gh); +void gfs2_glock_dq_wait(struct gfs2_holder *gh); +void gfs2_glock_dq_uninit(struct gfs2_holder *gh); +int gfs2_glock_nq_num(struct gfs2_sbd *sdp, u64 number, + const struct gfs2_glock_operations *glops, + unsigned int state, u16 flags, + struct gfs2_holder *gh); +int gfs2_glock_nq_m(unsigned int num_gh, struct gfs2_holder *ghs); +void gfs2_glock_dq_m(unsigned int num_gh, struct gfs2_holder *ghs); +void gfs2_dump_glock(struct seq_file *seq, struct gfs2_glock *gl, + bool fsid); +#define GLOCK_BUG_ON(gl,x) do { if (unlikely(x)) { \ + gfs2_dump_glock(NULL, gl, true); \ + BUG(); } } while(0) +#define gfs2_glock_assert_warn(gl, x) do { if (unlikely(!(x))) { \ + gfs2_dump_glock(NULL, gl, true); \ + gfs2_assert_warn(glock_sbd(gl), (x)); } } \ + while (0) +#define gfs2_glock_assert_withdraw(gl, x) do { if (unlikely(!(x))) { \ + gfs2_dump_glock(NULL, gl, true); \ + gfs2_assert_withdraw(glock_sbd(gl), (x)); } } \ + while (0) + +__printf(2, 3) +void gfs2_print_dbg(struct seq_file *seq, const char *fmt, ...); + +/** + * gfs2_glock_nq_init - initialize a holder and enqueue it on a glock + * @gl: the glock + * @state: the state we're requesting + * @flags: the modifier flags + * @gh: the holder structure + * + * Returns: 0, GLR_*, or errno + */ + +static inline int gfs2_glock_nq_init(struct gfs2_glock *gl, + unsigned int state, u16 flags, + struct gfs2_holder *gh) +{ + int error; + + __gfs2_holder_init(gl, state, flags, gh, _RET_IP_); + + error = gfs2_glock_nq(gh); + if (error) + gfs2_holder_uninit(gh); + + return error; +} + +void gfs2_glock_cb(struct gfs2_glock *gl, unsigned int state); +void gfs2_glock_complete(struct gfs2_glock *gl, int ret); +bool gfs2_queue_try_to_evict(struct gfs2_glock *gl); +bool gfs2_queue_verify_delete(struct gfs2_glock *gl, bool later); +void gfs2_cancel_delete_work(struct gfs2_glock *gl); +void gfs2_flush_delete_work(struct gfs2_sbd *sdp); +void gfs2_wait_glocks(struct gfs2_sbd *sdp); +void gfs2_withdraw_glocks(struct gfs2_sbd *sdp); +void gfs2_glock_thaw(struct gfs2_sbd *sdp); +void gfs2_glock_free(struct gfs2_glock *gl); +void gfs2_glock_free_later(struct gfs2_glock *gl); + +int __init gfs2_glock_init(void); +void gfs2_glock_exit(void); + +void gfs2_create_debugfs_file(struct gfs2_sbd *sdp); +void gfs2_delete_debugfs_file(struct gfs2_sbd *sdp); +void gfs2_register_debugfs(void); +void gfs2_unregister_debugfs(void); + +void glock_set_object(struct gfs2_glock *gl, void *object); +void glock_clear_object(struct gfs2_glock *gl, void *object); + +extern const struct lm_lockops gfs2_dlm_ops; + +static inline void gfs2_holder_mark_uninitialized(struct gfs2_holder *gh) +{ + gh->gh_gl = NULL; +} + +static inline bool gfs2_holder_initialized(struct gfs2_holder *gh) +{ + return gh->gh_gl; +} + +static inline bool gfs2_holder_queued(struct gfs2_holder *gh) +{ + return !list_empty(&gh->gh_list); +} + +void gfs2_inode_remember_delete(struct gfs2_glock *gl, u64 generation); +bool gfs2_inode_already_deleted(struct gfs2_glock *gl, u64 generation); + +static inline bool glock_needs_demote(struct gfs2_glock *gl) +{ + return (test_bit(GLF_DEMOTE, &gl->gl_flags) || + test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags)); +} + +#endif /* __GLOCK_DOT_H__ */ diff --git a/fs/gfs2/glops.c b/fs/gfs2/glops.c new file mode 100644 index 000000000..28f32424e --- /dev/null +++ b/fs/gfs2/glops.c @@ -0,0 +1,710 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gfs2.h" +#include "incore.h" +#include "bmap.h" +#include "glock.h" +#include "glops.h" +#include "inode.h" +#include "log.h" +#include "meta_io.h" +#include "recovery.h" +#include "rgrp.h" +#include "util.h" +#include "trans.h" +#include "dir.h" +#include "lops.h" + +struct workqueue_struct *gfs2_freeze_wq; + +static void gfs2_ail_error(struct gfs2_glock *gl, const struct buffer_head *bh) +{ + struct gfs2_sbd *sdp = glock_sbd(gl); + + fs_err(sdp, + "AIL buffer %p: blocknr %llu state 0x%08lx mapping %p page " + "state 0x%lx\n", + bh, (unsigned long long)bh->b_blocknr, bh->b_state, + bh->b_folio->mapping, bh->b_folio->flags.f); + fs_err(sdp, "AIL glock %u:%llu mapping %p\n", + glock_type(gl), glock_number(gl), + gfs2_glock2aspace(gl)); + gfs2_lm(sdp, "AIL error\n"); + gfs2_withdraw(sdp); +} + +/** + * __gfs2_ail_flush - remove all buffers for a given lock from the AIL + * @gl: the glock + * @fsync: set when called from fsync (not all buffers will be clean) + * @nr_revokes: Number of buffers to revoke + * + * None of the buffers should be dirty, locked, or pinned. + */ + +static void __gfs2_ail_flush(struct gfs2_glock *gl, bool fsync, + unsigned int nr_revokes) +{ + struct gfs2_sbd *sdp = glock_sbd(gl); + struct list_head *head = &gl->gl_ail_list; + struct gfs2_bufdata *bd, *tmp; + struct buffer_head *bh; + const unsigned long b_state = (1UL << BH_Dirty)|(1UL << BH_Pinned)|(1UL << BH_Lock); + + spin_lock(&sdp->sd_log_lock); + spin_lock(&sdp->sd_ail_lock); + list_for_each_entry_safe_reverse(bd, tmp, head, bd_ail_gl_list) { + if (nr_revokes == 0) + break; + bh = bd->bd_bh; + if (bh->b_state & b_state) { + if (fsync) + continue; + gfs2_ail_error(gl, bh); + } + gfs2_trans_add_revoke(sdp, bd); + nr_revokes--; + } + GLOCK_BUG_ON(gl, !fsync && atomic_read(&gl->gl_ail_count)); + spin_unlock(&sdp->sd_ail_lock); + spin_unlock(&sdp->sd_log_lock); +} + + +static int gfs2_ail_empty_gl(struct gfs2_glock *gl) +{ + struct gfs2_sbd *sdp = glock_sbd(gl); + struct gfs2_trans tr; + unsigned int revokes; + int ret = 0; + + revokes = atomic_read(&gl->gl_ail_count); + + if (!revokes) { + bool have_revokes; + bool log_in_flight; + + /* + * We have nothing on the ail, but there could be revokes on + * the sdp revoke queue, in which case, we still want to flush + * the log and wait for it to finish. + * + * If the sdp revoke list is empty too, we might still have an + * io outstanding for writing revokes, so we should wait for + * it before returning. + * + * If none of these conditions are true, our revokes are all + * flushed and we can return. + */ + spin_lock(&sdp->sd_log_lock); + have_revokes = !list_empty(&sdp->sd_log_revokes); + log_in_flight = atomic_read(&sdp->sd_log_in_flight); + spin_unlock(&sdp->sd_log_lock); + if (have_revokes) + goto flush; + if (log_in_flight) + log_flush_wait(sdp); + return 0; + } + + memset(&tr, 0, sizeof(tr)); + set_bit(TR_ONSTACK, &tr.tr_flags); + ret = __gfs2_trans_begin(&tr, sdp, 0, revokes, _RET_IP_); + if (ret) { + fs_err(sdp, "Transaction error %d: Unable to write revokes.", ret); + goto flush; + } + __gfs2_ail_flush(gl, 0, revokes); + gfs2_trans_end(sdp); + +flush: + if (!ret) + gfs2_log_flush(sdp, NULL, GFS2_LOG_HEAD_FLUSH_NORMAL | + GFS2_LFC_AIL_EMPTY_GL); + return ret; +} + +void gfs2_ail_flush(struct gfs2_glock *gl, bool fsync) +{ + struct gfs2_sbd *sdp = glock_sbd(gl); + unsigned int revokes = atomic_read(&gl->gl_ail_count); + int ret; + + if (!revokes) + return; + + ret = gfs2_trans_begin(sdp, 0, revokes); + if (ret) + return; + __gfs2_ail_flush(gl, fsync, revokes); + gfs2_trans_end(sdp); + gfs2_log_flush(sdp, NULL, GFS2_LOG_HEAD_FLUSH_NORMAL | + GFS2_LFC_AIL_FLUSH); +} + +/** + * gfs2_rgrp_metasync - sync out the metadata of a resource group + * @gl: the glock protecting the resource group + * + */ + +static int gfs2_rgrp_metasync(struct gfs2_glock *gl) +{ + struct gfs2_sbd *sdp = glock_sbd(gl); + struct address_space *metamapping = gfs2_aspace(sdp); + struct gfs2_rgrpd *rgd = gfs2_glock2rgrp(gl); + const unsigned bsize = sdp->sd_sb.sb_bsize; + loff_t start = (rgd->rd_addr * bsize) & PAGE_MASK; + loff_t end = PAGE_ALIGN((rgd->rd_addr + rgd->rd_length) * bsize) - 1; + int error; + + filemap_fdatawrite_range(metamapping, start, end); + error = filemap_fdatawait_range(metamapping, start, end); + WARN_ON_ONCE(error && !gfs2_withdrawn(sdp)); + mapping_set_error(metamapping, error); + if (error) + gfs2_io_error(sdp); + return error; +} + +/** + * rgrp_go_sync - sync out the metadata for this glock + * @gl: the glock + * + * Called when demoting or unlocking an EX glock. We must flush + * to disk all dirty buffers/pages relating to this glock, and must not + * return to caller to demote/unlock the glock until I/O is complete. + */ + +static int rgrp_go_sync(struct gfs2_glock *gl) +{ + struct gfs2_sbd *sdp = glock_sbd(gl); + struct gfs2_rgrpd *rgd = gfs2_glock2rgrp(gl); + int error; + + if (!rgd || !test_and_clear_bit(GLF_DIRTY, &gl->gl_flags)) + return 0; + GLOCK_BUG_ON(gl, gl->gl_state != LM_ST_EXCLUSIVE); + + gfs2_log_flush(sdp, gl, GFS2_LOG_HEAD_FLUSH_NORMAL | + GFS2_LFC_RGRP_GO_SYNC); + error = gfs2_rgrp_metasync(gl); + if (!error) + error = gfs2_ail_empty_gl(gl); + gfs2_free_clones(rgd); + return error; +} + +/** + * rgrp_go_inval - invalidate the metadata for this glock + * @gl: the glock + * @flags: + * + * We never used LM_ST_DEFERRED with resource groups, so that we + * should always see the metadata flag set here. + * + */ + +static void rgrp_go_inval(struct gfs2_glock *gl, int flags) +{ + struct gfs2_sbd *sdp = glock_sbd(gl); + struct address_space *mapping = gfs2_aspace(sdp); + struct gfs2_rgrpd *rgd = gfs2_glock2rgrp(gl); + const unsigned bsize = sdp->sd_sb.sb_bsize; + loff_t start, end; + + if (!rgd) + return; + start = (rgd->rd_addr * bsize) & PAGE_MASK; + end = PAGE_ALIGN((rgd->rd_addr + rgd->rd_length) * bsize) - 1; + gfs2_rgrp_brelse(rgd); + WARN_ON_ONCE(!(flags & DIO_METADATA)); + gfs2_assert_withdraw(sdp, !atomic_read(&gl->gl_ail_count)); + truncate_inode_pages_range(mapping, start, end); +} + +static void gfs2_rgrp_go_dump(struct seq_file *seq, const struct gfs2_glock *gl, + const char *fs_id_buf) +{ + struct gfs2_rgrpd *rgd = gl->gl_object; + + if (rgd) + gfs2_rgrp_dump(seq, rgd, fs_id_buf); +} + +static struct gfs2_inode *gfs2_glock2inode(struct gfs2_glock *gl) +{ + struct gfs2_inode *ip; + + spin_lock(&gl->gl_lockref.lock); + ip = gl->gl_object; + if (ip) + set_bit(GIF_GLOP_PENDING, &ip->i_flags); + spin_unlock(&gl->gl_lockref.lock); + return ip; +} + +struct gfs2_rgrpd *gfs2_glock2rgrp(struct gfs2_glock *gl) +{ + struct gfs2_rgrpd *rgd; + + spin_lock(&gl->gl_lockref.lock); + rgd = gl->gl_object; + spin_unlock(&gl->gl_lockref.lock); + + return rgd; +} + +static void gfs2_clear_glop_pending(struct gfs2_inode *ip) +{ + if (!ip) + return; + + clear_bit_unlock(GIF_GLOP_PENDING, &ip->i_flags); + wake_up_bit(&ip->i_flags, GIF_GLOP_PENDING); +} + +/** + * gfs2_inode_metasync - sync out the metadata of an inode + * @gl: the glock protecting the inode + * + */ +int gfs2_inode_metasync(struct gfs2_glock *gl) +{ + struct address_space *metamapping = gfs2_glock2aspace(gl); + int error; + + filemap_fdatawrite(metamapping); + error = filemap_fdatawait(metamapping); + if (error) + gfs2_io_error(glock_sbd(gl)); + return error; +} + +/** + * inode_go_sync - Sync the dirty metadata of an inode + * @gl: the glock protecting the inode + * + */ + +static int inode_go_sync(struct gfs2_glock *gl) +{ + struct gfs2_inode *ip = gfs2_glock2inode(gl); + int isreg = ip && S_ISREG(ip->i_inode.i_mode); + struct address_space *metamapping = gfs2_glock2aspace(gl); + int error = 0, ret; + + if (isreg) { + if (test_and_clear_bit(GIF_SW_PAGED, &ip->i_flags)) + unmap_shared_mapping_range(ip->i_inode.i_mapping, 0, 0); + inode_dio_wait(&ip->i_inode); + } + if (!test_and_clear_bit(GLF_DIRTY, &gl->gl_flags)) + goto out; + + GLOCK_BUG_ON(gl, gl->gl_state != LM_ST_EXCLUSIVE); + + gfs2_log_flush(glock_sbd(gl), gl, GFS2_LOG_HEAD_FLUSH_NORMAL | + GFS2_LFC_INODE_GO_SYNC); + filemap_fdatawrite(metamapping); + if (isreg) { + struct address_space *mapping = ip->i_inode.i_mapping; + filemap_fdatawrite(mapping); + error = filemap_fdatawait(mapping); + mapping_set_error(mapping, error); + } + ret = gfs2_inode_metasync(gl); + if (!error) + error = ret; + ret = gfs2_ail_empty_gl(gl); + if (!error) + error = ret; + /* + * Writeback of the data mapping may cause the dirty flag to be set + * so we have to clear it again here. + */ + smp_mb__before_atomic(); + clear_bit(GLF_DIRTY, &gl->gl_flags); + +out: + gfs2_clear_glop_pending(ip); + return error; +} + +/** + * inode_go_inval - prepare a inode glock to be released + * @gl: the glock + * @flags: + * + * Normally we invalidate everything, but if we are moving into + * LM_ST_DEFERRED from LM_ST_SHARED or LM_ST_EXCLUSIVE then we + * can keep hold of the metadata, since it won't have changed. + * + */ + +static void inode_go_inval(struct gfs2_glock *gl, int flags) +{ + struct gfs2_inode *ip = gfs2_glock2inode(gl); + + gfs2_assert_withdraw(glock_sbd(gl), !atomic_read(&gl->gl_ail_count)); + + if (flags & DIO_METADATA) { + struct address_space *mapping = gfs2_glock2aspace(gl); + truncate_inode_pages(mapping, 0); + if (ip) { + set_bit(GLF_INSTANTIATE_NEEDED, &gl->gl_flags); + forget_all_cached_acls(&ip->i_inode); + security_inode_invalidate_secctx(&ip->i_inode); + gfs2_dir_hash_inval(ip); + } + } + + if (ip == GFS2_I(glock_sbd(gl)->sd_rindex)) { + gfs2_log_flush(glock_sbd(gl), NULL, + GFS2_LOG_HEAD_FLUSH_NORMAL | + GFS2_LFC_INODE_GO_INVAL); + glock_sbd(gl)->sd_rindex_uptodate = 0; + } + if (ip && S_ISREG(ip->i_inode.i_mode)) + truncate_inode_pages(ip->i_inode.i_mapping, 0); + + gfs2_clear_glop_pending(ip); +} + +static int gfs2_dinode_in(struct gfs2_inode *ip, const void *buf) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + const struct gfs2_dinode *str = buf; + struct timespec64 atime, iatime; + u16 height, depth; + umode_t mode = be32_to_cpu(str->di_mode); + struct inode *inode = &ip->i_inode; + bool is_new = inode_state_read_once(inode) & I_NEW; + + if (unlikely(ip->i_no_addr != be64_to_cpu(str->di_num.no_addr))) { + gfs2_consist_inode(ip); + return -EIO; + } + if (unlikely(!is_new && inode_wrong_type(inode, mode))) { + gfs2_consist_inode(ip); + return -EIO; + } + ip->i_no_formal_ino = be64_to_cpu(str->di_num.no_formal_ino); + inode->i_mode = mode; + if (is_new) { + inode->i_rdev = 0; + switch (mode & S_IFMT) { + case S_IFBLK: + case S_IFCHR: + inode->i_rdev = MKDEV(be32_to_cpu(str->di_major), + be32_to_cpu(str->di_minor)); + break; + } + } + + i_uid_write(inode, be32_to_cpu(str->di_uid)); + i_gid_write(inode, be32_to_cpu(str->di_gid)); + set_nlink(inode, be32_to_cpu(str->di_nlink)); + i_size_write(inode, be64_to_cpu(str->di_size)); + gfs2_set_inode_blocks(inode, be64_to_cpu(str->di_blocks)); + atime.tv_sec = be64_to_cpu(str->di_atime); + atime.tv_nsec = be32_to_cpu(str->di_atime_nsec); + iatime = inode_get_atime(inode); + if (timespec64_compare(&iatime, &atime) < 0) + inode_set_atime_to_ts(inode, atime); + inode_set_mtime(inode, be64_to_cpu(str->di_mtime), + be32_to_cpu(str->di_mtime_nsec)); + inode_set_ctime(inode, be64_to_cpu(str->di_ctime), + be32_to_cpu(str->di_ctime_nsec)); + + ip->i_goal = be64_to_cpu(str->di_goal_meta); + ip->i_generation = be64_to_cpu(str->di_generation); + + ip->i_diskflags = be32_to_cpu(str->di_flags); + ip->i_eattr = be64_to_cpu(str->di_eattr); + /* i_diskflags and i_eattr must be set before gfs2_set_inode_flags() */ + gfs2_set_inode_flags(inode); + height = be16_to_cpu(str->di_height); + if (unlikely(height > sdp->sd_max_height)) { + gfs2_consist_inode(ip); + return -EIO; + } + ip->i_height = (u8)height; + + depth = be16_to_cpu(str->di_depth); + if (unlikely(depth > GFS2_DIR_MAX_DEPTH)) { + gfs2_consist_inode(ip); + return -EIO; + } + if ((ip->i_diskflags & GFS2_DIF_EXHASH) && + depth < ilog2(sdp->sd_hash_ptrs)) { + gfs2_consist_inode(ip); + return -EIO; + } + ip->i_depth = (u8)depth; + ip->i_entries = be32_to_cpu(str->di_entries); + + if (!S_ISDIR(inode->i_mode) && (ip->i_diskflags & GFS2_DIF_EXHASH)) { + gfs2_consist_inode(ip); + return -EIO; + } + + if (gfs2_is_stuffed(ip) && inode->i_size > gfs2_max_stuffed_size(ip)) { + gfs2_consist_inode(ip); + return -EIO; + } + if (S_ISREG(inode->i_mode)) + gfs2_set_aops(inode); + + return 0; +} + +/** + * gfs2_inode_refresh - Refresh the incore copy of the dinode + * @ip: The GFS2 inode + * + * Returns: errno + */ + +static int gfs2_inode_refresh(struct gfs2_inode *ip) +{ + struct buffer_head *dibh; + int error; + + error = gfs2_meta_inode_buffer(ip, &dibh); + if (error) + return error; + + error = gfs2_dinode_in(ip, dibh->b_data); + brelse(dibh); + return error; +} + +/** + * inode_go_instantiate - read in an inode if necessary + * @gl: The glock + * + * Returns: errno + */ + +static int inode_go_instantiate(struct gfs2_glock *gl) +{ + struct gfs2_inode *ip = gl->gl_object; + struct gfs2_glock *io_gl; + int error; + + if (!ip) /* no inode to populate - read it in later */ + return 0; + + error = gfs2_inode_refresh(ip); + if (error) + return error; + io_gl = ip->i_iopen_gh.gh_gl; + io_gl->gl_no_formal_ino = ip->i_no_formal_ino; + return 0; +} + +static int inode_go_held(struct gfs2_holder *gh) +{ + struct gfs2_glock *gl = gh->gh_gl; + struct gfs2_inode *ip = gl->gl_object; + int error = 0; + + if (!ip) /* no inode to populate - read it in later */ + return 0; + + if (gh->gh_state != LM_ST_DEFERRED) + inode_dio_wait(&ip->i_inode); + + if ((ip->i_diskflags & GFS2_DIF_TRUNC_IN_PROG) && + (gl->gl_state == LM_ST_EXCLUSIVE) && + (gh->gh_state == LM_ST_EXCLUSIVE)) + error = gfs2_truncatei_resume(ip); + + return error; +} + +/** + * inode_go_dump - print information about an inode + * @seq: The iterator + * @gl: The glock + * @fs_id_buf: file system id (may be empty) + * + */ + +static void inode_go_dump(struct seq_file *seq, const struct gfs2_glock *gl, + const char *fs_id_buf) +{ + struct gfs2_inode *ip = gl->gl_object; + const struct inode *inode = &ip->i_inode; + + if (ip == NULL) + return; + + gfs2_print_dbg(seq, "%s I: n:%llu/%llu t:%u f:0x%02lx d:0x%08x s:%llu " + "p:%lu\n", fs_id_buf, + (unsigned long long)ip->i_no_formal_ino, + (unsigned long long)ip->i_no_addr, + IF2DT(inode->i_mode), ip->i_flags, + (unsigned int)ip->i_diskflags, + (unsigned long long)i_size_read(inode), + inode->i_data.nrpages); +} + +/** + * freeze_go_callback - A cluster node is requesting a freeze + * @gl: the glock + * @remote: true if this came from a different cluster node + */ + +static void freeze_go_callback(struct gfs2_glock *gl, bool remote) +{ + struct gfs2_sbd *sdp = glock_sbd(gl); + struct super_block *sb = sdp->sd_vfs; + + if (!remote || + (gl->gl_state != LM_ST_SHARED && + gl->gl_state != LM_ST_UNLOCKED) || + gl->gl_demote_state != LM_ST_UNLOCKED) + return; + + /* + * Try to get an active super block reference to prevent racing with + * unmount (see super_trylock_shared()). But note that unmount isn't + * the only place where a write lock on s_umount is taken, and we can + * fail here because of things like remount as well. + */ + if (down_read_trylock(&sb->s_umount)) { + atomic_inc(&sb->s_active); + up_read(&sb->s_umount); + if (!queue_work(gfs2_freeze_wq, &sdp->sd_freeze_work)) + deactivate_super(sb); + } +} + +/** + * freeze_go_xmote_bh - After promoting/demoting the freeze glock + * @gl: the glock + */ +static int freeze_go_xmote_bh(struct gfs2_glock *gl) +{ + struct gfs2_sbd *sdp = glock_sbd(gl); + struct gfs2_inode *ip = GFS2_I(sdp->sd_jdesc->jd_inode); + struct gfs2_glock *j_gl = ip->i_gl; + struct gfs2_log_header_host head; + int error; + + if (test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags)) { + j_gl->gl_ops->go_inval(j_gl, DIO_METADATA); + + error = gfs2_find_jhead(sdp->sd_jdesc, &head); + if (gfs2_assert_withdraw(sdp, !error)) + return error; + if (gfs2_assert_withdraw(sdp, head.lh_flags & + GFS2_LOG_HEAD_UNMOUNT)) + return -EIO; + gfs2_log_pointers_init(sdp, &head); + } + return 0; +} + +/** + * iopen_go_callback - schedule the dcache entry for the inode to be deleted + * @gl: the glock + * @remote: true if this came from a different cluster node + * + * gl_lockref.lock lock is held while calling this + */ +static void iopen_go_callback(struct gfs2_glock *gl, bool remote) +{ + struct gfs2_inode *ip = gl->gl_object; + struct gfs2_sbd *sdp = glock_sbd(gl); + + if (!remote || test_bit(SDF_KILL, &sdp->sd_flags)) + return; + + if (gl->gl_demote_state == LM_ST_UNLOCKED && + gl->gl_state == LM_ST_SHARED && ip) { + gl->gl_lockref.count++; + if (!gfs2_queue_try_to_evict(gl)) + gl->gl_lockref.count--; + } +} + +const struct gfs2_glock_operations gfs2_meta_glops = { + .go_type = LM_TYPE_META, +}; + +const struct gfs2_glock_operations gfs2_inode_glops = { + .go_sync = inode_go_sync, + .go_inval = inode_go_inval, + .go_instantiate = inode_go_instantiate, + .go_held = inode_go_held, + .go_dump = inode_go_dump, + .go_type = LM_TYPE_INODE, + .go_flags = GLOF_ASPACE | GLOF_LVB, +}; + +const struct gfs2_glock_operations gfs2_rgrp_glops = { + .go_sync = rgrp_go_sync, + .go_inval = rgrp_go_inval, + .go_instantiate = gfs2_rgrp_go_instantiate, + .go_dump = gfs2_rgrp_go_dump, + .go_type = LM_TYPE_RGRP, + .go_flags = GLOF_LVB, +}; + +const struct gfs2_glock_operations gfs2_freeze_glops = { + .go_xmote_bh = freeze_go_xmote_bh, + .go_callback = freeze_go_callback, + .go_type = LM_TYPE_NONDISK, +}; + +const struct gfs2_glock_operations gfs2_iopen_glops = { + .go_type = LM_TYPE_IOPEN, + .go_callback = iopen_go_callback, + .go_dump = inode_go_dump, + .go_subclass = 1, +}; + +const struct gfs2_glock_operations gfs2_flock_glops = { + .go_type = LM_TYPE_FLOCK, +}; + +const struct gfs2_glock_operations gfs2_nondisk_glops = { + .go_type = LM_TYPE_NONDISK, +}; + +const struct gfs2_glock_operations gfs2_quota_glops = { + .go_type = LM_TYPE_QUOTA, + .go_flags = GLOF_LVB, +}; + +const struct gfs2_glock_operations gfs2_journal_glops = { + .go_type = LM_TYPE_JOURNAL, +}; + +const struct gfs2_glock_operations *gfs2_glops_list[] = { + [LM_TYPE_META] = &gfs2_meta_glops, + [LM_TYPE_INODE] = &gfs2_inode_glops, + [LM_TYPE_RGRP] = &gfs2_rgrp_glops, + [LM_TYPE_IOPEN] = &gfs2_iopen_glops, + [LM_TYPE_FLOCK] = &gfs2_flock_glops, + [LM_TYPE_NONDISK] = &gfs2_nondisk_glops, + [LM_TYPE_QUOTA] = &gfs2_quota_glops, + [LM_TYPE_JOURNAL] = &gfs2_journal_glops, +}; + diff --git a/fs/gfs2/glops.h b/fs/gfs2/glops.h new file mode 100644 index 000000000..934142379 --- /dev/null +++ b/fs/gfs2/glops.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#ifndef __GLOPS_DOT_H__ +#define __GLOPS_DOT_H__ + +#include "incore.h" + +extern struct workqueue_struct *gfs2_freeze_wq; + +extern const struct gfs2_glock_operations gfs2_meta_glops; +extern const struct gfs2_glock_operations gfs2_inode_glops; +extern const struct gfs2_glock_operations gfs2_rgrp_glops; +extern const struct gfs2_glock_operations gfs2_freeze_glops; +extern const struct gfs2_glock_operations gfs2_iopen_glops; +extern const struct gfs2_glock_operations gfs2_flock_glops; +extern const struct gfs2_glock_operations gfs2_nondisk_glops; +extern const struct gfs2_glock_operations gfs2_quota_glops; +extern const struct gfs2_glock_operations gfs2_journal_glops; +extern const struct gfs2_glock_operations *gfs2_glops_list[]; + +int gfs2_inode_metasync(struct gfs2_glock *gl); +void gfs2_ail_flush(struct gfs2_glock *gl, bool fsync); + +#endif /* __GLOPS_DOT_H__ */ diff --git a/fs/gfs2/incore.h b/fs/gfs2/incore.h new file mode 100644 index 000000000..dadb4d3c9 --- /dev/null +++ b/fs/gfs2/incore.h @@ -0,0 +1,883 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. + */ + +#ifndef __INCORE_DOT_H__ +#define __INCORE_DOT_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DIO_WAIT 0x00000010 +#define DIO_METADATA 0x00000020 + +struct gfs2_log_operations; +struct gfs2_bufdata; +struct gfs2_holder; +struct gfs2_glock; +struct gfs2_quota_data; +struct gfs2_trans; +struct gfs2_jdesc; +struct gfs2_sbd; +struct lm_lockops; + +typedef void (*gfs2_glop_bh_t) (struct gfs2_glock *gl, unsigned int ret); + +struct gfs2_log_header_host { + u64 lh_sequence; /* Sequence number of this transaction */ + u32 lh_flags; /* GFS2_LOG_HEAD_... */ + u32 lh_tail; /* Block number of log tail */ + u32 lh_blkno; + + s64 lh_local_total; + s64 lh_local_free; + s64 lh_local_dinodes; +}; + +/* + * Structure of operations that are associated with each + * type of element in the log. + */ + +struct gfs2_log_operations { + void (*lo_before_commit) (struct gfs2_sbd *sdp, struct gfs2_trans *tr); + void (*lo_after_commit) (struct gfs2_sbd *sdp, struct gfs2_trans *tr); + void (*lo_before_scan) (struct gfs2_jdesc *jd, + struct gfs2_log_header_host *head, int pass); + int (*lo_scan_elements) (struct gfs2_jdesc *jd, unsigned int start, + struct gfs2_log_descriptor *ld, __be64 *ptr, + int pass); + void (*lo_after_scan) (struct gfs2_jdesc *jd, int error, int pass); + const char *lo_name; +}; + +#define GBF_FULL 1 + +/** + * Clone bitmaps (bi_clone): + * + * - When a block is freed, we remember the previous state of the block in the + * clone bitmap, and only mark the block as free in the real bitmap. + * + * - When looking for a block to allocate, we check for a free block in the + * clone bitmap, and if no clone bitmap exists, in the real bitmap. + * + * - For allocating a block, we mark it as allocated in the real bitmap, and if + * a clone bitmap exists, also in the clone bitmap. + * + * - At the end of a log_flush, we copy the real bitmap into the clone bitmap + * to make the clone bitmap reflect the current allocation state. + * (Alternatively, we could remove the clone bitmap.) + * + * The clone bitmaps are in-core only, and is never written to disk. + * + * These steps ensure that blocks which have been freed in a transaction cannot + * be reallocated in that same transaction. + */ +struct gfs2_bitmap { + struct buffer_head *bi_bh; + char *bi_clone; + unsigned long bi_flags; + u32 bi_offset; + u32 bi_start; + u32 bi_bytes; + u32 bi_blocks; +}; + +struct gfs2_rgrpd { + struct rb_node rd_node; /* Link with superblock */ + struct gfs2_glock *rd_gl; /* Glock for this rgrp */ + u64 rd_addr; /* grp block disk address */ + u64 rd_data0; /* first data location */ + u32 rd_length; /* length of rgrp header in fs blocks */ + u32 rd_data; /* num of data blocks in rgrp */ + u32 rd_bitbytes; /* number of bytes in data bitmaps */ + u32 rd_free; + u32 rd_requested; /* number of blocks in rd_rstree */ + u32 rd_reserved; /* number of reserved blocks */ + u32 rd_free_clone; + u32 rd_dinodes; + u64 rd_igeneration; + struct gfs2_bitmap *rd_bits; + struct gfs2_sbd *rd_sbd; + struct gfs2_rgrp_lvb *rd_rgl; + u32 rd_last_alloc; + u32 rd_flags; + u32 rd_extfail_pt; /* extent failure point */ +#define GFS2_RDF_CHECK 0x10000000 /* check for unlinked inodes */ +#define GFS2_RDF_ERROR 0x40000000 /* error in rg */ +#define GFS2_RDF_PREFERRED 0x80000000 /* This rgrp is preferred */ +#define GFS2_RDF_MASK 0xf0000000 /* mask for internal flags */ + spinlock_t rd_rsspin; /* protects reservation related vars */ + struct mutex rd_mutex; + struct rb_root rd_rstree; /* multi-block reservation tree */ +}; + +enum gfs2_state_bits { + BH_Pinned = BH_PrivateStart, + BH_Escaped = BH_PrivateStart + 1, +}; + +BUFFER_FNS(Pinned, pinned) +TAS_BUFFER_FNS(Pinned, pinned) +BUFFER_FNS(Escaped, escaped) +TAS_BUFFER_FNS(Escaped, escaped) + +struct gfs2_bufdata { + struct buffer_head *bd_bh; + struct gfs2_glock *bd_gl; + u64 bd_blkno; + + struct list_head bd_list; + + struct gfs2_trans *bd_tr; + struct list_head bd_ail_st_list; + struct list_head bd_ail_gl_list; +}; + +/* + * Internally, we prefix things with gdlm_ and GDLM_ (for gfs-dlm) since a + * prefix of lock_dlm_ gets awkward. + */ + +#define GDLM_STRNAME_BYTES 25 +#define GDLM_LVB_SIZE 32 + +/* + * ls_recover_flags: + * + * DFL_BLOCK_LOCKS: dlm is in recovery and will grant locks that had been + * held by failed nodes whose journals need recovery. Those locks should + * only be used for journal recovery until the journal recovery is done. + * This is set by the dlm recover_prep callback and cleared by the + * gfs2_control thread when journal recovery is complete. To avoid + * races between recover_prep setting and gfs2_control clearing, recover_spin + * is held while changing this bit and reading/writing recover_block + * and recover_start. + * + * DFL_NO_DLM_OPS: dlm lockspace ops/callbacks are not being used. + * + * DFL_FIRST_MOUNT: this node is the first to mount this fs and is doing + * recovery of all journals before allowing other nodes to mount the fs. + * This is cleared when FIRST_MOUNT_DONE is set. + * + * DFL_FIRST_MOUNT_DONE: this node was the first mounter, and has finished + * recovery of all journals, and now allows other nodes to mount the fs. + * + * DFL_MOUNT_DONE: gdlm_mount has completed successfully and cleared + * BLOCK_LOCKS for the first time. The gfs2_control thread should now + * control clearing BLOCK_LOCKS for further recoveries. + * + * DFL_UNMOUNT: gdlm_unmount sets to keep sdp off gfs2_control_wq. + * + * DFL_DLM_RECOVERY: set while dlm is in recovery, between recover_prep() + * and recover_done(), i.e. set while recover_block == recover_start. + */ + +enum { + DFL_BLOCK_LOCKS = 0, + DFL_NO_DLM_OPS = 1, + DFL_FIRST_MOUNT = 2, + DFL_FIRST_MOUNT_DONE = 3, + DFL_MOUNT_DONE = 4, + DFL_UNMOUNT = 5, + DFL_DLM_RECOVERY = 6, +}; + +/* + * We are using struct lm_lockname as an rhashtable key. Avoid holes within + * the struct; padding at the end is fine. + */ +struct lm_lockname { + u64 ln_number; + struct gfs2_sbd *ln_sbd; + unsigned int ln_type; +}; + +#define lm_name_equal(name1, name2) \ + (((name1)->ln_number == (name2)->ln_number) && \ + ((name1)->ln_type == (name2)->ln_type) && \ + ((name1)->ln_sbd == (name2)->ln_sbd)) + + +struct gfs2_glock_operations { + int (*go_sync) (struct gfs2_glock *gl); + int (*go_xmote_bh)(struct gfs2_glock *gl); + void (*go_inval) (struct gfs2_glock *gl, int flags); + int (*go_instantiate) (struct gfs2_glock *gl); + int (*go_held)(struct gfs2_holder *gh); + void (*go_dump)(struct seq_file *seq, const struct gfs2_glock *gl, + const char *fs_id_buf); + void (*go_callback)(struct gfs2_glock *gl, bool remote); + const int go_subclass; + const int go_type; + const unsigned long go_flags; +#define GLOF_ASPACE 1 /* address space attached */ +#define GLOF_LVB 2 /* Lock Value Block attached */ +}; + +enum { + GFS2_LKS_SRTT = 0, /* Non blocking smoothed round trip time */ + GFS2_LKS_SRTTVAR = 1, /* Non blocking smoothed variance */ + GFS2_LKS_SRTTB = 2, /* Blocking smoothed round trip time */ + GFS2_LKS_SRTTVARB = 3, /* Blocking smoothed variance */ + GFS2_LKS_SIRT = 4, /* Smoothed Inter-request time */ + GFS2_LKS_SIRTVAR = 5, /* Smoothed Inter-request variance */ + GFS2_LKS_DCOUNT = 6, /* Count of dlm requests */ + GFS2_LKS_QCOUNT = 7, /* Count of gfs2_holder queues */ + GFS2_NR_LKSTATS +}; + +struct gfs2_lkstats { + u64 stats[GFS2_NR_LKSTATS]; +}; + +enum { + /* States */ + HIF_HOLDER = 6, /* Set for gh that "holds" the glock */ + HIF_WAIT = 10, +}; + +struct gfs2_holder { + struct list_head gh_list; + + struct gfs2_glock *gh_gl; + struct pid *gh_owner_pid; + u16 gh_flags; + u16 gh_state; + + int gh_error; + unsigned long gh_iflags; /* HIF_... */ + unsigned long gh_ip; +}; + +/* Number of quota types we support */ +#define GFS2_MAXQUOTAS 2 + +struct gfs2_qadata { /* quota allocation data */ + /* Quota stuff */ + struct gfs2_quota_data *qa_qd[2 * GFS2_MAXQUOTAS]; + struct gfs2_holder qa_qd_ghs[2 * GFS2_MAXQUOTAS]; + unsigned int qa_qd_num; + int qa_ref; +}; + +/* Resource group multi-block reservation, in order of appearance: + + Step 1. Function prepares to write, allocates a mb, sets the size hint. + Step 2. User calls inplace_reserve to target an rgrp, sets the rgrp info + Step 3. Function get_local_rgrp locks the rgrp, determines which bits to use + Step 4. Bits are assigned from the rgrp based on either the reservation + or wherever it can. +*/ + +struct gfs2_blkreserv { + struct rb_node rs_node; /* node within rd_rstree */ + struct gfs2_rgrpd *rs_rgd; + u64 rs_start; + u32 rs_requested; + u32 rs_reserved; /* number of reserved blocks */ +}; + +/* + * Allocation parameters + * @target: The number of blocks we'd ideally like to allocate + * @aflags: The flags (e.g. Orlov flag) + * + * The intent is to gradually expand this structure over time in + * order to give more information, e.g. alignment, min extent size + * to the allocation code. + */ +struct gfs2_alloc_parms { + u64 target; + u32 min_target; + u32 aflags; + u64 allowed; +}; + +enum { + GLF_LOCK = 1, + GLF_INSTANTIATE_NEEDED = 2, /* needs instantiate */ + GLF_DEMOTE = 3, + GLF_PENDING_DEMOTE = 4, + GLF_DEMOTE_IN_PROGRESS = 5, + GLF_DIRTY = 6, + GLF_LFLUSH = 7, + GLF_HAVE_REPLY = 9, + GLF_INITIAL = 10, + GLF_HAVE_FROZEN_REPLY = 11, + GLF_INSTANTIATE_IN_PROG = 12, /* instantiate happening now */ + GLF_OBJECT = 14, /* Used only for tracing */ + GLF_BLOCKING = 15, + GLF_TRY_TO_EVICT = 17, /* iopen glocks only */ + GLF_VERIFY_DELETE = 18, /* iopen glocks only */ + GLF_MAY_CANCEL = 19, + GLF_DEFER_DELETE = 20, /* iopen glocks only */ + GLF_CANCELING = 21, +}; + +struct gfs2_glock { + unsigned long gl_flags; /* GLF_... */ + struct lm_lockname gl_name; + + struct lockref gl_lockref; + + /* State fields protected by gl_lockref.lock */ + unsigned int gl_state:2, /* Current state */ + gl_target:2, /* Target state */ + gl_demote_state:2, /* State requested by remote node */ + gl_req:2, /* State in last dlm request */ + gl_reply:8; /* Last reply from the dlm */ + + unsigned long gl_demote_time; /* time of first demote request */ + long gl_hold_time; + struct list_head gl_holders; + + const struct gfs2_glock_operations *gl_ops; + ktime_t gl_dstamp; + struct gfs2_lkstats gl_stats; + struct dlm_lksb gl_lksb; + unsigned long gl_tchange; + void *gl_object; + + struct list_head gl_dead; + struct list_head gl_ail_list; + atomic_t gl_ail_count; + atomic_t gl_revokes; + struct delayed_work gl_work; + /* For iopen glocks only */ + struct { + struct delayed_work gl_delete; + u64 gl_no_formal_ino; + }; + struct rcu_head gl_rcu; + struct rhash_head gl_node; +}; + +static inline unsigned int glock_type(const struct gfs2_glock *gl) +{ + return gl->gl_name.ln_type; +} + +static inline u64 glock_number(const struct gfs2_glock *gl) +{ + return gl->gl_name.ln_number; +} + +enum { + GIF_QD_LOCKED = 1, + GIF_SW_PAGED = 3, + GIF_GLOP_PENDING = 6, +}; + +struct gfs2_inode { + struct inode i_inode; + u64 i_no_addr; + u64 i_no_formal_ino; + u64 i_generation; + u64 i_eattr; + unsigned long i_flags; /* GIF_... */ + struct gfs2_glock *i_gl; + struct gfs2_holder i_iopen_gh; + struct gfs2_qadata *i_qadata; /* quota allocation data */ + struct gfs2_holder i_rgd_gh; + struct gfs2_blkreserv i_res; /* rgrp multi-block reservation */ + u64 i_goal; /* goal block for allocations */ + atomic_t i_sizehint; /* hint of the write size */ + struct rw_semaphore i_rw_mutex; + struct list_head i_ordered; + __be64 *i_hash_cache; + u32 i_entries; + u32 i_diskflags; + u8 i_height; + u8 i_depth; + u16 i_rahead; +}; + +/* + * Since i_inode is the first element of struct gfs2_inode, + * this is effectively a cast. + */ +static inline struct gfs2_inode *GFS2_I(struct inode *inode) +{ + return container_of(inode, struct gfs2_inode, i_inode); +} + +static inline struct gfs2_sbd *GFS2_SB(const struct inode *inode) +{ + return inode->i_sb->s_fs_info; +} + +struct gfs2_file { + struct mutex f_fl_mutex; + struct gfs2_holder f_fl_gh; +}; + +struct gfs2_revoke_replay { + struct list_head rr_list; + u64 rr_blkno; + unsigned int rr_where; +}; + +enum { + QDF_CHANGE = 1, + QDF_LOCKED = 2, + QDF_REFRESH = 3, + QDF_QMSG_QUIET = 4, +}; + +struct gfs2_quota_data { + struct hlist_bl_node qd_hlist; + struct list_head qd_list; + struct kqid qd_id; + struct gfs2_sbd *qd_sbd; + struct lockref qd_lockref; + struct list_head qd_lru; + unsigned qd_hash; + + unsigned long qd_flags; /* QDF_... */ + + s64 qd_change; + s64 qd_change_sync; + + unsigned int qd_slot; + unsigned int qd_slot_ref; + + struct buffer_head *qd_bh; + struct gfs2_quota_change *qd_bh_qc; + unsigned int qd_bh_count; + + struct gfs2_glock *qd_gl; + struct gfs2_quota_lvb qd_qb; + + u64 qd_sync_gen; + unsigned long qd_last_warn; + struct rcu_head qd_rcu; +}; + +enum { + TR_TOUCHED = 1, + TR_ATTACHED = 2, + TR_ONSTACK = 3, +}; + +struct gfs2_trans { + unsigned long tr_ip; + + unsigned int tr_blocks; + unsigned int tr_revokes; + unsigned int tr_reserved; + unsigned long tr_flags; + + unsigned int tr_num_buf_new; + unsigned int tr_num_databuf_new; + unsigned int tr_num_buf_rm; + unsigned int tr_num_databuf_rm; + unsigned int tr_num_revoke; + + struct list_head tr_list; + struct list_head tr_databuf; + struct list_head tr_buf; + + unsigned int tr_first; + struct list_head tr_ail1_list; + struct list_head tr_ail2_list; +}; + +struct gfs2_journal_extent { + struct list_head list; + + unsigned int lblock; /* First logical block */ + u64 dblock; /* First disk block */ + u64 blocks; +}; + +struct gfs2_jdesc { + struct list_head jd_list; + struct list_head extent_list; + unsigned int nr_extents; + struct work_struct jd_work; + struct inode *jd_inode; + struct bio *jd_log_bio; + unsigned long jd_flags; +#define JDF_RECOVERY 1 + unsigned int jd_jid; + u32 jd_blocks; + int jd_recover_error; + /* Replay stuff */ + + unsigned int jd_found_blocks; + unsigned int jd_found_revokes; + unsigned int jd_replayed_blocks; + + struct list_head jd_revoke_list; + unsigned int jd_replay_tail; +}; + +struct gfs2_statfs_change_host { + s64 sc_total; + s64 sc_free; + s64 sc_dinodes; +}; + +#define GFS2_QUOTA_DEFAULT GFS2_QUOTA_OFF +#define GFS2_QUOTA_OFF 0 +#define GFS2_QUOTA_ACCOUNT 1 +#define GFS2_QUOTA_ON 2 +#define GFS2_QUOTA_QUIET 3 /* on but not complaining */ + +#define GFS2_DATA_DEFAULT GFS2_DATA_ORDERED +#define GFS2_DATA_WRITEBACK 1 +#define GFS2_DATA_ORDERED 2 + +#define GFS2_ERRORS_DEFAULT GFS2_ERRORS_WITHDRAW +#define GFS2_ERRORS_WITHDRAW 0 +#define GFS2_ERRORS_DEACTIVATE 1 +#define GFS2_ERRORS_PANIC 3 + +struct gfs2_args { + char ar_lockproto[GFS2_LOCKNAME_LEN]; /* Name of the Lock Protocol */ + char ar_locktable[GFS2_LOCKNAME_LEN]; /* Name of the Lock Table */ + char ar_hostdata[GFS2_LOCKNAME_LEN]; /* Host specific data */ + unsigned int ar_spectator:1; /* Don't get a journal */ + unsigned int ar_localflocks:1; /* Let the VFS do flock|fcntl */ + unsigned int ar_debug:1; /* Oops on errors */ + unsigned int ar_posix_acl:1; /* Enable posix acls */ + unsigned int ar_quota:2; /* off/account/on */ + unsigned int ar_suiddir:1; /* suiddir support */ + unsigned int ar_data:2; /* ordered/writeback */ + unsigned int ar_meta:1; /* mount metafs */ + unsigned int ar_discard:1; /* discard requests */ + unsigned int ar_errors:2; /* errors=withdraw | deactivate | panic */ + unsigned int ar_nobarrier:1; /* do not send barriers */ + unsigned int ar_rgrplvb:1; /* use lvbs for rgrp info */ + unsigned int ar_got_rgrplvb:1; /* Was the rgrplvb opt given? */ + unsigned int ar_loccookie:1; /* use location based readdir + cookies */ + s32 ar_commit; /* Commit interval */ + s32 ar_statfs_quantum; /* The fast statfs interval */ + s32 ar_quota_quantum; /* The quota interval */ + s32 ar_statfs_percent; /* The % change to force sync */ +}; + +struct gfs2_tune { + spinlock_t gt_spin; + + unsigned int gt_logd_secs; + + unsigned int gt_quota_warn_period; /* Secs between quota warn msgs */ + unsigned int gt_quota_scale_num; /* Numerator */ + unsigned int gt_quota_scale_den; /* Denominator */ + unsigned int gt_quota_quantum; /* Secs between syncs to quota file */ + unsigned int gt_new_files_jdata; + unsigned int gt_max_readahead; /* Max bytes to read-ahead from disk */ + unsigned int gt_complain_secs; + unsigned int gt_statfs_quantum; + unsigned int gt_statfs_slow; + unsigned int gt_withdraw_helper_timeout; +}; + +enum { + SDF_JOURNAL_CHECKED = 0, + SDF_JOURNAL_LIVE = 1, + SDF_WITHDRAWN = 2, + SDF_NOBARRIERS = 3, + SDF_NORECOVERY = 4, + SDF_DEMOTE = 5, + SDF_NOJOURNALID = 6, + SDF_RORECOVERY = 7, /* read only recovery */ + SDF_FORCE_AIL_FLUSH = 9, + SDF_FREEZE_INITIATOR = 10, + SDF_KILL = 15, + SDF_EVICTING = 16, + SDF_FROZEN = 17, +}; + +#define GFS2_FSNAME_LEN 256 + +struct gfs2_inum_host { + u64 no_formal_ino; + u64 no_addr; +}; + +struct gfs2_sb_host { + u32 sb_magic; + u32 sb_type; + + u32 sb_fs_format; + u32 sb_multihost_format; + u32 sb_bsize; + u32 sb_bsize_shift; + + struct gfs2_inum_host sb_master_dir; + struct gfs2_inum_host sb_root_dir; + + char sb_lockproto[GFS2_LOCKNAME_LEN]; + char sb_locktable[GFS2_LOCKNAME_LEN]; +}; + +/* + * lm_mount() return values + * + * ls_jid - the journal ID this node should use + * ls_first - this node is the first to mount the file system + * ls_lockspace - lock module's context for this file system + * ls_ops - lock module's functions + */ + +struct lm_lockstruct { + int ls_jid; + unsigned int ls_first; + const struct lm_lockops *ls_ops; + dlm_lockspace_t *ls_dlm; + + int ls_recover_jid_done; /* These two are deprecated, */ + int ls_recover_jid_status; /* used previously by gfs_controld */ + + struct dlm_lksb ls_mounted_lksb; /* mounted_lock */ + struct dlm_lksb ls_control_lksb; /* control_lock */ + char ls_control_lvb[GDLM_LVB_SIZE]; /* control_lock lvb */ + struct completion ls_sync_wait; /* {control,mounted}_{lock,unlock} */ + char *ls_lvb_bits; + + struct rw_semaphore ls_sem; + + spinlock_t ls_recover_spin; /* protects following fields */ + unsigned long ls_recover_flags; /* DFL_ */ + uint32_t ls_recover_mount; /* gen in first recover_done cb */ + uint32_t ls_recover_start; /* gen in last recover_done cb */ + uint32_t ls_recover_block; /* copy recover_start in last recover_prep */ + uint32_t ls_recover_size; /* size of recover_submit, recover_result */ + uint32_t *ls_recover_submit; /* gen in last recover_slot cb per jid */ + uint32_t *ls_recover_result; /* result of last jid recovery */ +}; + +struct gfs2_pcpu_lkstats { + /* One struct for each glock type */ + struct gfs2_lkstats lkstats[10]; +}; + +/* List of local (per node) statfs inodes */ +struct local_statfs_inode { + struct list_head si_list; + struct inode *si_sc_inode; + unsigned int si_jid; /* journal id this statfs inode corresponds to */ +}; + +struct gfs2_sbd { + struct super_block *sd_vfs; + struct gfs2_pcpu_lkstats __percpu *sd_lkstats; + struct kobject sd_kobj; + struct completion sd_kobj_unregister; + unsigned long sd_flags; /* SDF_... */ + struct gfs2_sb_host sd_sb; + + /* Constants computed on mount */ + + u32 sd_fsb2bb; + u32 sd_fsb2bb_shift; + u32 sd_diptrs; /* Number of pointers in a dinode */ + u32 sd_inptrs; /* Number of pointers in a indirect block */ + u32 sd_ldptrs; /* Number of pointers in a log descriptor block */ + u32 sd_jbsize; /* Size of a journaled data block */ + u32 sd_hash_bsize; /* sizeof(exhash block) */ + u32 sd_hash_bsize_shift; + u32 sd_hash_ptrs; /* Number of pointers in a hash block */ + u32 sd_qc_per_block; + u32 sd_blocks_per_bitmap; + u32 sd_max_dirres; /* Max blocks needed to add a directory entry */ + u32 sd_max_height; /* Max height of a file's metadata tree */ + u64 sd_heightsize[GFS2_MAX_META_HEIGHT + 1]; + u32 sd_max_dents_per_leaf; /* Max number of dirents in a leaf block */ + + struct gfs2_args sd_args; /* Mount arguments */ + struct gfs2_tune sd_tune; /* Filesystem tuning structure */ + + /* Lock Stuff */ + + struct lm_lockstruct sd_lockstruct; + struct gfs2_holder sd_live_gh; + struct gfs2_glock *sd_rename_gl; + struct gfs2_glock *sd_freeze_gl; + struct work_struct sd_freeze_work; + struct work_struct sd_withdraw_work; + wait_queue_head_t sd_kill_wait; + wait_queue_head_t sd_async_glock_wait; + atomic_t sd_glock_disposal; + struct completion sd_locking_init; + struct completion sd_withdraw_helper; + int sd_withdraw_helper_status; + struct delayed_work sd_control_work; + + /* Inode Stuff */ + + struct dentry *sd_master_dir; + struct dentry *sd_root_dir; + + struct inode *sd_jindex; + struct inode *sd_statfs_inode; + struct inode *sd_sc_inode; + struct list_head sd_sc_inodes_list; + struct inode *sd_qc_inode; + struct inode *sd_rindex; + struct inode *sd_quota_inode; + + /* StatFS stuff */ + + spinlock_t sd_statfs_spin; + struct gfs2_statfs_change_host sd_statfs_master; + struct gfs2_statfs_change_host sd_statfs_local; + int sd_statfs_force_sync; + + /* Resource group stuff */ + + int sd_rindex_uptodate; + spinlock_t sd_rindex_spin; + struct rb_root sd_rindex_tree; + unsigned int sd_rgrps; + unsigned int sd_max_rg_data; + + /* Journal index stuff */ + + struct list_head sd_jindex_list; + spinlock_t sd_jindex_spin; + struct mutex sd_jindex_mutex; + unsigned int sd_journals; + + struct gfs2_jdesc *sd_jdesc; + struct gfs2_holder sd_journal_gh; + struct gfs2_holder sd_jinode_gh; + + struct gfs2_holder sd_sc_gh; + struct buffer_head *sd_sc_bh; + struct gfs2_holder sd_qc_gh; + + struct completion sd_journal_ready; + + /* Workqueue stuff */ + + struct workqueue_struct *sd_glock_wq; + struct workqueue_struct *sd_delete_wq; + + /* Daemon stuff */ + + struct task_struct *sd_logd_process; + struct task_struct *sd_quotad_process; + + /* Quota stuff */ + + struct list_head sd_quota_list; + atomic_t sd_quota_count; + struct mutex sd_quota_sync_mutex; + wait_queue_head_t sd_quota_wait; + + unsigned int sd_quota_slots; + unsigned long *sd_quota_bitmap; + spinlock_t sd_bitmap_lock; + + u64 sd_quota_sync_gen; + + /* Log stuff */ + + struct inode *sd_inode; + + spinlock_t sd_log_lock; + + struct gfs2_trans *sd_log_tr; + unsigned int sd_log_blks_reserved; + + atomic_t sd_log_pinned; + unsigned int sd_log_num_revoke; + + struct list_head sd_log_revokes; + struct list_head sd_log_ordered; + spinlock_t sd_ordered_lock; + + atomic_t sd_log_thresh1; + atomic_t sd_log_thresh2; + atomic_t sd_log_blks_free; + atomic_t sd_log_blks_needed; + atomic_t sd_log_revokes_available; + wait_queue_head_t sd_log_waitq; + wait_queue_head_t sd_logd_waitq; + + u64 sd_log_sequence; + int sd_log_idle; + + struct rw_semaphore sd_log_flush_lock; + atomic_t sd_log_in_flight; + wait_queue_head_t sd_log_flush_wait; + int sd_log_error; /* First log error */ + + unsigned int sd_log_tail; + unsigned int sd_log_flush_tail; + unsigned int sd_log_head; + unsigned int sd_log_flush_head; + + spinlock_t sd_ail_lock; + struct list_head sd_ail1_list; + struct list_head sd_ail2_list; + + /* glocks */ + spinlock_t sd_dead_lock; + + /* For quiescing the filesystem */ + struct gfs2_holder sd_freeze_gh; + struct mutex sd_freeze_mutex; + struct list_head sd_dead_glocks; + + char sd_fsname[GFS2_FSNAME_LEN + 3 * sizeof(int) + 2]; + char sd_table_name[GFS2_FSNAME_LEN]; + char sd_proto_name[GFS2_FSNAME_LEN]; + + /* Debugging crud */ + + unsigned long sd_last_warning; + struct dentry *debugfs_dir; /* debugfs directory */ +}; + +#define glock_sbd(gl) ((gl)->gl_name.ln_sbd) + +#define GFS2_BAD_INO 1 + +static inline struct address_space *gfs2_aspace(struct gfs2_sbd *sdp) +{ + return sdp->sd_inode->i_mapping; +} + +static inline void gfs2_glstats_inc(struct gfs2_glock *gl, int which) +{ + gl->gl_stats.stats[which]++; +} + +static inline void gfs2_sbstats_inc(const struct gfs2_glock *gl, int which) +{ + const struct gfs2_sbd *sdp = glock_sbd(gl); + preempt_disable(); + this_cpu_ptr(sdp->sd_lkstats)->lkstats[glock_type(gl)].stats[which]++; + preempt_enable(); +} + +struct gfs2_rgrpd *gfs2_glock2rgrp(struct gfs2_glock *gl); + +static inline unsigned gfs2_max_stuffed_size(const struct gfs2_inode *ip) +{ + return GFS2_SB(&ip->i_inode)->sd_sb.sb_bsize - sizeof(struct gfs2_dinode); +} + +#endif /* __INCORE_DOT_H__ */ + diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c new file mode 100644 index 000000000..f361876c5 --- /dev/null +++ b/fs/gfs2/inode.c @@ -0,0 +1,2338 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2011 Red Hat, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gfs2.h" +#include "incore.h" +#include "acl.h" +#include "bmap.h" +#include "dir.h" +#include "xattr.h" +#include "glock.h" +#include "inode.h" +#include "meta_io.h" +#include "quota.h" +#include "rgrp.h" +#include "trans.h" +#include "util.h" +#include "super.h" +#include "glops.h" + +static const struct inode_operations gfs2_file_iops; +static const struct inode_operations gfs2_dir_iops; +static const struct inode_operations gfs2_symlink_iops; + +/** + * gfs2_set_iop - Sets inode operations + * @inode: The inode with correct i_mode filled in + * + * GFS2 lookup code fills in vfs inode contents based on info obtained + * from directory entry inside gfs2_inode_lookup(). + */ + +static void gfs2_set_iop(struct inode *inode) +{ + struct gfs2_sbd *sdp = GFS2_SB(inode); + umode_t mode = inode->i_mode; + + if (S_ISREG(mode)) { + inode->i_op = &gfs2_file_iops; + if (gfs2_localflocks(sdp)) + inode->i_fop = &gfs2_file_fops_nolock; + else + inode->i_fop = &gfs2_file_fops; + } else if (S_ISDIR(mode)) { + inode->i_op = &gfs2_dir_iops; + if (gfs2_localflocks(sdp)) + inode->i_fop = &gfs2_dir_fops_nolock; + else + inode->i_fop = &gfs2_dir_fops; + } else if (S_ISLNK(mode)) { + inode->i_op = &gfs2_symlink_iops; + } else { + inode->i_op = &gfs2_file_iops; + init_special_inode(inode, inode->i_mode, inode->i_rdev); + } +} + +static int iget_test(struct inode *inode, void *opaque) +{ + u64 no_addr = *(u64 *)opaque; + + return GFS2_I(inode)->i_no_addr == no_addr; +} + +static int iget_set(struct inode *inode, void *opaque) +{ + u64 no_addr = *(u64 *)opaque; + + GFS2_I(inode)->i_no_addr = no_addr; + inode->i_ino = no_addr; + return 0; +} + +void gfs2_setup_inode(struct inode *inode) +{ + gfp_t gfp_mask; + + /* + * Ensure all page cache allocations are done from GFP_NOFS context to + * prevent direct reclaim recursion back into the filesystem and blowing + * stacks or deadlocking. + */ + gfp_mask = mapping_gfp_mask(inode->i_mapping); + mapping_set_gfp_mask(inode->i_mapping, gfp_mask & ~__GFP_FS); +} + +/** + * gfs2_inode_lookup - Lookup an inode + * @sb: The super block + * @type: The type of the inode + * @no_addr: The inode number + * @no_formal_ino: The inode generation number + * @blktype: Requested block type (GFS2_BLKST_DINODE or GFS2_BLKST_UNLINKED; + * GFS2_BLKST_FREE to indicate not to verify) + * + * If @type is DT_UNKNOWN, the inode type is fetched from disk. + * + * If @blktype is anything other than GFS2_BLKST_FREE (which is used as a + * placeholder because it doesn't otherwise make sense), the on-disk block type + * is verified to be @blktype. + * + * When @no_formal_ino is non-zero, this function will return ERR_PTR(-ESTALE) + * if it detects that @no_formal_ino doesn't match the actual inode generation + * number. However, it doesn't always know unless @type is DT_UNKNOWN. + * + * Returns: A VFS inode, or an error + */ + +struct inode *gfs2_inode_lookup(struct super_block *sb, unsigned int type, + u64 no_addr, u64 no_formal_ino, + unsigned int blktype) +{ + struct inode *inode; + struct gfs2_inode *ip; + struct gfs2_holder i_gh; + int error; + + gfs2_holder_mark_uninitialized(&i_gh); + inode = iget5_locked(sb, no_addr, iget_test, iget_set, &no_addr); + if (!inode) + return ERR_PTR(-ENOMEM); + + ip = GFS2_I(inode); + + if (inode_state_read_once(inode) & I_NEW) { + struct gfs2_sbd *sdp = GFS2_SB(inode); + struct gfs2_glock *io_gl; + int extra_flags = 0; + + gfs2_setup_inode(inode); + error = gfs2_glock_get(sdp, no_addr, &gfs2_inode_glops, CREATE, + &ip->i_gl); + if (unlikely(error)) + goto fail; + + error = gfs2_glock_get(sdp, no_addr, &gfs2_iopen_glops, CREATE, + &io_gl); + if (unlikely(error)) + goto fail; + + /* + * The only caller that sets @blktype to GFS2_BLKST_UNLINKED is + * delete_work_func(). Make sure not to cancel the delete work + * from within itself here. + */ + if (blktype == GFS2_BLKST_UNLINKED) + extra_flags |= LM_FLAG_TRY; + else + gfs2_cancel_delete_work(io_gl); + error = gfs2_glock_nq_init(io_gl, LM_ST_SHARED, + GL_EXACT | GL_NOPID | extra_flags, + &ip->i_iopen_gh); + gfs2_glock_put(io_gl); + if (unlikely(error)) + goto fail; + + if (type == DT_UNKNOWN || blktype != GFS2_BLKST_FREE) { + /* + * The GL_SKIP flag indicates to skip reading the inode + * block. We read the inode when instantiating it + * after possibly checking the block type. + */ + error = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, + GL_SKIP, &i_gh); + if (error) + goto fail; + + error = -ESTALE; + if (no_formal_ino && + gfs2_inode_already_deleted(ip->i_gl, no_formal_ino)) + goto fail; + + if (blktype != GFS2_BLKST_FREE) { + error = gfs2_check_blk_type(sdp, no_addr, + blktype); + if (error) + goto fail; + } + } + + set_bit(GLF_INSTANTIATE_NEEDED, &ip->i_gl->gl_flags); + + /* Lowest possible timestamp; will be overwritten in gfs2_dinode_in. */ + inode_set_atime(inode, + 1LL << (8 * sizeof(inode_get_atime_sec(inode)) - 1), + 0); + + glock_set_object(ip->i_gl, ip); + + if (type == DT_UNKNOWN) { + /* Inode glock must be locked already */ + error = gfs2_instantiate(&i_gh); + if (error) { + glock_clear_object(ip->i_gl, ip); + goto fail; + } + } else { + ip->i_no_formal_ino = no_formal_ino; + inode->i_mode = DT2IF(type); + } + + if (gfs2_holder_initialized(&i_gh)) + gfs2_glock_dq_uninit(&i_gh); + glock_set_object(ip->i_iopen_gh.gh_gl, ip); + + gfs2_set_iop(inode); + unlock_new_inode(inode); + } + + if (no_formal_ino && ip->i_no_formal_ino && + no_formal_ino != ip->i_no_formal_ino) { + iput(inode); + return ERR_PTR(-ESTALE); + } + + return inode; + +fail: + if (error == GLR_TRYFAILED) + error = -EAGAIN; + if (gfs2_holder_initialized(&ip->i_iopen_gh)) + gfs2_glock_dq_uninit(&ip->i_iopen_gh); + if (gfs2_holder_initialized(&i_gh)) + gfs2_glock_dq_uninit(&i_gh); + if (ip->i_gl) { + gfs2_glock_put(ip->i_gl); + ip->i_gl = NULL; + } + iget_failed(inode); + return ERR_PTR(error); +} + +/** + * gfs2_lookup_by_inum - look up an inode by inode number + * @sdp: The super block + * @no_addr: The inode number + * @no_formal_ino: The inode generation number (0 for any) + * @blktype: Requested block type (see gfs2_inode_lookup) + */ +struct inode *gfs2_lookup_by_inum(struct gfs2_sbd *sdp, u64 no_addr, + u64 no_formal_ino, unsigned int blktype) +{ + struct super_block *sb = sdp->sd_vfs; + struct inode *inode; + int error; + + inode = gfs2_inode_lookup(sb, DT_UNKNOWN, no_addr, no_formal_ino, + blktype); + if (IS_ERR(inode)) + return inode; + + if (no_formal_ino) { + error = -EIO; + if (GFS2_I(inode)->i_diskflags & GFS2_DIF_SYSTEM) + goto fail_iput; + } + return inode; + +fail_iput: + iput(inode); + return ERR_PTR(error); +} + + +/** + * gfs2_lookup_meta - Look up an inode in a metadata directory + * @dip: The directory + * @name: The name of the inode + */ +struct inode *gfs2_lookup_meta(struct inode *dip, const char *name) +{ + struct qstr qstr; + struct inode *inode; + + gfs2_str2qstr(&qstr, name); + inode = gfs2_lookupi(dip, &qstr, 1); + if (IS_ERR_OR_NULL(inode)) + return inode ? inode : ERR_PTR(-ENOENT); + + /* + * Must not call back into the filesystem when allocating + * pages in the metadata inode's address space. + */ + mapping_set_gfp_mask(inode->i_mapping, GFP_NOFS); + + return inode; +} + + +/** + * gfs2_lookupi - Look up a filename in a directory and return its inode + * @dir: The inode of the directory containing the inode to look-up + * @name: The name of the inode to look for + * @is_root: If 1, ignore the caller's permissions + * + * This can be called via the VFS filldir function when NFS is doing + * a readdirplus and the inode which its intending to stat isn't + * already in cache. In this case we must not take the directory glock + * again, since the readdir call will have already taken that lock. + * + * Returns: errno + */ + +struct inode *gfs2_lookupi(struct inode *dir, const struct qstr *name, + int is_root) +{ + struct super_block *sb = dir->i_sb; + struct gfs2_inode *dip = GFS2_I(dir); + struct gfs2_holder d_gh; + int error = 0; + struct inode *inode = NULL; + + gfs2_holder_mark_uninitialized(&d_gh); + if (!name->len || name->len > GFS2_FNAMESIZE) + return ERR_PTR(-ENAMETOOLONG); + + if ((name->len == 1 && memcmp(name->name, ".", 1) == 0) || + (name->len == 2 && memcmp(name->name, "..", 2) == 0 && + dir == d_inode(sb->s_root))) { + igrab(dir); + return dir; + } + + if (gfs2_glock_is_locked_by_me(dip->i_gl) == NULL) { + error = gfs2_glock_nq_init(dip->i_gl, LM_ST_SHARED, 0, &d_gh); + if (error) + return ERR_PTR(error); + } + + if (!is_root) { + error = gfs2_permission(&nop_mnt_idmap, dir, MAY_EXEC); + if (error) + goto out; + } + + inode = gfs2_dir_search(dir, name, false); + if (IS_ERR(inode)) + error = PTR_ERR(inode); +out: + if (gfs2_holder_initialized(&d_gh)) + gfs2_glock_dq_uninit(&d_gh); + if (error == -ENOENT) + return NULL; + return inode ? inode : ERR_PTR(error); +} + +/** + * create_ok - OK to create a new on-disk inode here? + * @dip: Directory in which dinode is to be created + * @name: Name of new dinode + * @mode: + * + * Returns: errno + */ + +static int create_ok(struct gfs2_inode *dip, const struct qstr *name, + umode_t mode) +{ + int error; + + error = gfs2_permission(&nop_mnt_idmap, &dip->i_inode, + MAY_WRITE | MAY_EXEC); + if (error) + return error; + + /* Don't create entries in an unlinked directory */ + if (!dip->i_inode.i_nlink) + return -ENOENT; + + if (dip->i_entries == (u32)-1) + return -EFBIG; + if (S_ISDIR(mode) && dip->i_inode.i_nlink == (u32)-1) + return -EMLINK; + + return 0; +} + +static void munge_mode_uid_gid(const struct gfs2_inode *dip, + struct inode *inode) +{ + if (GFS2_SB(&dip->i_inode)->sd_args.ar_suiddir && + (dip->i_inode.i_mode & S_ISUID) && + !uid_eq(dip->i_inode.i_uid, GLOBAL_ROOT_UID)) { + if (S_ISDIR(inode->i_mode)) + inode->i_mode |= S_ISUID; + else if (!uid_eq(dip->i_inode.i_uid, current_fsuid())) + inode->i_mode &= ~07111; + inode->i_uid = dip->i_inode.i_uid; + } else + inode->i_uid = current_fsuid(); + + if (dip->i_inode.i_mode & S_ISGID) { + if (S_ISDIR(inode->i_mode)) + inode->i_mode |= S_ISGID; + inode->i_gid = dip->i_inode.i_gid; + } else + inode->i_gid = current_fsgid(); +} + +static int alloc_dinode(struct gfs2_inode *ip, u32 flags, unsigned *dblocks) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + struct gfs2_alloc_parms ap = { .target = *dblocks, .aflags = flags, }; + int error; + + error = gfs2_quota_lock_check(ip, &ap); + if (error) + goto out; + + error = gfs2_inplace_reserve(ip, &ap); + if (error) + goto out_quota; + + error = gfs2_trans_begin(sdp, (*dblocks * RES_RG_BIT) + RES_STATFS + RES_QUOTA, 0); + if (error) + goto out_ipreserv; + + error = gfs2_alloc_blocks(ip, &ip->i_no_addr, dblocks, 1); + if (error) + goto out_trans_end; + + ip->i_no_formal_ino = ip->i_generation; + ip->i_inode.i_ino = ip->i_no_addr; + ip->i_goal = ip->i_no_addr; + if (*dblocks > 1) + ip->i_eattr = ip->i_no_addr + 1; + +out_trans_end: + gfs2_trans_end(sdp); +out_ipreserv: + gfs2_inplace_release(ip); +out_quota: + gfs2_quota_unlock(ip); +out: + return error; +} + +static void gfs2_final_release_pages(struct gfs2_inode *ip) +{ + struct inode *inode = &ip->i_inode; + struct gfs2_glock *gl = ip->i_gl; + + /* This can only happen during incomplete inode creation. */ + if (unlikely(!gl)) + return; + + truncate_inode_pages(gfs2_glock2aspace(gl), 0); + truncate_inode_pages(&inode->i_data, 0); + + if (atomic_read(&gl->gl_revokes) == 0) { + clear_bit(GLF_LFLUSH, &gl->gl_flags); + clear_bit(GLF_DIRTY, &gl->gl_flags); + } +} + +int gfs2_dinode_dealloc(struct gfs2_inode *ip) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + struct gfs2_rgrpd *rgd; + struct gfs2_holder gh; + int error; + + if (gfs2_get_inode_blocks(&ip->i_inode) != 1) { + gfs2_consist_inode(ip); + return -EIO; + } + + gfs2_rindex_update(sdp); + + error = gfs2_quota_hold(ip, NO_UID_QUOTA_CHANGE, NO_GID_QUOTA_CHANGE); + if (error) + return error; + + rgd = gfs2_blk2rgrpd(sdp, ip->i_no_addr, 1); + if (!rgd) { + gfs2_consist_inode(ip); + error = -EIO; + goto out_qs; + } + + error = gfs2_glock_nq_init(rgd->rd_gl, LM_ST_EXCLUSIVE, + LM_FLAG_NODE_SCOPE, &gh); + if (error) + goto out_qs; + + error = gfs2_trans_begin(sdp, RES_RG_BIT + RES_STATFS + RES_QUOTA, + sdp->sd_jdesc->jd_blocks); + if (error) + goto out_rg_gunlock; + + gfs2_free_di(rgd, ip); + + gfs2_final_release_pages(ip); + + gfs2_trans_end(sdp); + +out_rg_gunlock: + gfs2_glock_dq_uninit(&gh); +out_qs: + gfs2_quota_unhold(ip); + return error; +} + +static void gfs2_init_dir(struct buffer_head *dibh, + const struct gfs2_inode *parent) +{ + struct gfs2_dinode *di = (struct gfs2_dinode *)dibh->b_data; + struct gfs2_dirent *dent = (struct gfs2_dirent *)(di+1); + + gfs2_qstr2dirent(&gfs2_qdot, GFS2_DIRENT_SIZE(gfs2_qdot.len), dent); + dent->de_inum = di->di_num; /* already GFS2 endian */ + dent->de_type = cpu_to_be16(DT_DIR); + + dent = (struct gfs2_dirent *)((char*)dent + GFS2_DIRENT_SIZE(1)); + gfs2_qstr2dirent(&gfs2_qdotdot, dibh->b_size - GFS2_DIRENT_SIZE(1) - sizeof(struct gfs2_dinode), dent); + gfs2_inum_out(parent, dent); + dent->de_type = cpu_to_be16(DT_DIR); + +} + +/** + * gfs2_init_xattr - Initialise an xattr block for a new inode + * @ip: The inode in question + * + * This sets up an empty xattr block for a new inode, ready to + * take any ACLs, LSM xattrs, etc. + */ + +static void gfs2_init_xattr(struct gfs2_inode *ip) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + struct buffer_head *bh; + struct gfs2_ea_header *ea; + + bh = gfs2_meta_new(ip->i_gl, ip->i_eattr); + gfs2_trans_add_meta(ip->i_gl, bh); + gfs2_metatype_set(bh, GFS2_METATYPE_EA, GFS2_FORMAT_EA); + gfs2_buffer_clear_tail(bh, sizeof(struct gfs2_meta_header)); + + ea = GFS2_EA_BH2FIRST(bh); + ea->ea_rec_len = cpu_to_be32(sdp->sd_jbsize); + ea->ea_type = GFS2_EATYPE_UNUSED; + ea->ea_flags = GFS2_EAFLAG_LAST; + + brelse(bh); +} + +/** + * init_dinode - Fill in a new dinode structure + * @dip: The directory this inode is being created in + * @ip: The inode + * @symname: The symlink destination (if a symlink) + * + */ + +static void init_dinode(struct gfs2_inode *dip, struct gfs2_inode *ip, + const char *symname) +{ + struct gfs2_dinode *di; + struct buffer_head *dibh; + + dibh = gfs2_meta_new(ip->i_gl, ip->i_no_addr); + gfs2_trans_add_meta(ip->i_gl, dibh); + di = (struct gfs2_dinode *)dibh->b_data; + gfs2_dinode_out(ip, di); + + di->di_major = cpu_to_be32(imajor(&ip->i_inode)); + di->di_minor = cpu_to_be32(iminor(&ip->i_inode)); + di->__pad1 = 0; + di->__pad2 = 0; + di->__pad3 = 0; + memset(&di->__pad4, 0, sizeof(di->__pad4)); + memset(&di->di_reserved, 0, sizeof(di->di_reserved)); + gfs2_buffer_clear_tail(dibh, sizeof(struct gfs2_dinode)); + + switch(ip->i_inode.i_mode & S_IFMT) { + case S_IFDIR: + gfs2_init_dir(dibh, dip); + break; + case S_IFLNK: + memcpy(dibh->b_data + sizeof(struct gfs2_dinode), symname, ip->i_inode.i_size); + break; + } + + set_buffer_uptodate(dibh); + brelse(dibh); +} + +/** + * gfs2_trans_da_blks - Calculate number of blocks to link inode + * @dip: The directory we are linking into + * @da: The dir add information + * @nr_inodes: The number of inodes involved + * + * This calculate the number of blocks we need to reserve in a + * transaction to link @nr_inodes into a directory. In most cases + * @nr_inodes will be 2 (the directory plus the inode being linked in) + * but in case of rename, 4 may be required. + * + * Returns: Number of blocks + */ + +static unsigned gfs2_trans_da_blks(const struct gfs2_inode *dip, + const struct gfs2_diradd *da, + unsigned nr_inodes) +{ + return da->nr_blocks + gfs2_rg_blocks(dip, da->nr_blocks) + + (nr_inodes * RES_DINODE) + RES_QUOTA + RES_STATFS; +} + +static int link_dinode(struct gfs2_inode *dip, const struct qstr *name, + struct gfs2_inode *ip, struct gfs2_diradd *da) +{ + struct gfs2_sbd *sdp = GFS2_SB(&dip->i_inode); + struct gfs2_alloc_parms ap = { .target = da->nr_blocks, }; + int error; + + if (da->nr_blocks) { + error = gfs2_quota_lock_check(dip, &ap); + if (error) + goto fail_quota_locks; + + error = gfs2_inplace_reserve(dip, &ap); + if (error) + goto fail_quota_locks; + + error = gfs2_trans_begin(sdp, gfs2_trans_da_blks(dip, da, 2), 0); + if (error) + goto fail_ipreserv; + } else { + error = gfs2_trans_begin(sdp, RES_LEAF + 2 * RES_DINODE, 0); + if (error) + goto fail_quota_locks; + } + + error = gfs2_dir_add(&dip->i_inode, name, ip, da); + + gfs2_trans_end(sdp); +fail_ipreserv: + gfs2_inplace_release(dip); +fail_quota_locks: + gfs2_quota_unlock(dip); + return error; +} + +static int gfs2_initxattrs(struct inode *inode, const struct xattr *xattr_array, + void *fs_info) +{ + const struct xattr *xattr; + int err = 0; + + for (xattr = xattr_array; xattr->name != NULL; xattr++) { + err = __gfs2_xattr_set(inode, xattr->name, xattr->value, + xattr->value_len, 0, + GFS2_EATYPE_SECURITY); + if (err < 0) + break; + } + return err; +} + +/** + * gfs2_create_inode - Create a new inode + * @dir: The parent directory + * @dentry: The new dentry + * @file: If non-NULL, the file which is being opened + * @mode: The permissions on the new inode + * @dev: For device nodes, this is the device number + * @symname: For symlinks, this is the link destination + * @size: The initial size of the inode (ignored for directories) + * @excl: Force fail if inode exists + * + * FIXME: Change to allocate the disk blocks and write them out in the same + * transaction. That way, we can no longer end up in a situation in which an + * inode is allocated, the node crashes, and the block looks like a valid + * inode. (With atomic creates in place, we will also no longer need to zero + * the link count and dirty the inode here on failure.) + * + * Returns: 0 on success, or error code + */ + +static int gfs2_create_inode(struct inode *dir, struct dentry *dentry, + struct file *file, + umode_t mode, dev_t dev, const char *symname, + unsigned int size, int excl) +{ + const struct qstr *name = &dentry->d_name; + struct posix_acl *default_acl, *acl; + struct gfs2_holder d_gh, gh; + struct inode *inode = NULL; + struct gfs2_inode *dip = GFS2_I(dir), *ip; + struct gfs2_sbd *sdp = GFS2_SB(&dip->i_inode); + struct gfs2_glock *io_gl; + int error, dealloc_error; + u32 aflags = 0; + unsigned blocks = 1; + struct gfs2_diradd da = { .bh = NULL, .save_loc = 1, }; + bool xattr_initialized = false; + + if (!name->len || name->len > GFS2_FNAMESIZE) + return -ENAMETOOLONG; + + error = gfs2_qa_get(dip); + if (error) + return error; + + error = gfs2_rindex_update(sdp); + if (error) + goto fail; + + error = gfs2_glock_nq_init(dip->i_gl, LM_ST_EXCLUSIVE, 0, &d_gh); + if (error) + goto fail; + gfs2_holder_mark_uninitialized(&gh); + + error = create_ok(dip, name, mode); + if (error) + goto fail_gunlock; + + inode = gfs2_dir_search(dir, &dentry->d_name, !S_ISREG(mode) || excl); + error = PTR_ERR(inode); + if (!IS_ERR(inode)) { + if (file && (file->f_flags & __O_REGULAR) && + !S_ISREG(inode->i_mode)) { + iput(inode); + inode = NULL; + error = -EFTYPE; + goto fail_gunlock; + } + if (S_ISDIR(inode->i_mode)) { + iput(inode); + inode = NULL; + error = -EISDIR; + goto fail_gunlock; + } + d_instantiate(dentry, inode); + error = 0; + if (file) { + if (S_ISREG(inode->i_mode)) + error = finish_open(file, dentry, gfs2_open_common); + else + error = finish_no_open(file, NULL); + } + gfs2_glock_dq_uninit(&d_gh); + goto fail; + } else if (error != -ENOENT) { + goto fail_gunlock; + } + + error = gfs2_diradd_alloc_required(dir, name, &da); + if (error < 0) + goto fail_gunlock; + + inode = new_inode(sdp->sd_vfs); + error = -ENOMEM; + if (!inode) + goto fail_gunlock; + gfs2_setup_inode(inode); + ip = GFS2_I(inode); + + error = posix_acl_create(dir, &mode, &default_acl, &acl); + if (error) + goto fail_gunlock; + + error = gfs2_qa_get(ip); + if (error) + goto fail_free_acls; + + inode->i_mode = mode; + set_nlink(inode, S_ISDIR(mode) ? 2 : 1); + inode->i_rdev = dev; + inode->i_size = size; + simple_inode_init_ts(inode); + munge_mode_uid_gid(dip, inode); + check_and_update_goal(dip); + ip->i_goal = dip->i_goal; + ip->i_diskflags = 0; + ip->i_eattr = 0; + ip->i_height = 0; + ip->i_depth = 0; + ip->i_entries = 0; + ip->i_no_addr = 0; /* Temporarily zero until real addr is assigned */ + + switch(mode & S_IFMT) { + case S_IFREG: + if ((dip->i_diskflags & GFS2_DIF_INHERIT_JDATA) || + gfs2_tune_get(sdp, gt_new_files_jdata)) + ip->i_diskflags |= GFS2_DIF_JDATA; + gfs2_set_aops(inode); + break; + case S_IFDIR: + ip->i_diskflags |= (dip->i_diskflags & GFS2_DIF_INHERIT_JDATA); + ip->i_diskflags |= GFS2_DIF_JDATA; + ip->i_entries = 2; + break; + } + + /* Force SYSTEM flag on all files and subdirs of a SYSTEM directory */ + if (dip->i_diskflags & GFS2_DIF_SYSTEM) + ip->i_diskflags |= GFS2_DIF_SYSTEM; + + gfs2_set_inode_flags(inode); + + if ((GFS2_I(d_inode(sdp->sd_root_dir)) == dip) || + (dip->i_diskflags & GFS2_DIF_TOPDIR)) + aflags |= GFS2_AF_ORLOV; + + if (default_acl || acl) + blocks++; + + error = alloc_dinode(ip, aflags, &blocks); + if (error) + goto fail_free_inode; + + gfs2_set_inode_blocks(inode, blocks); + + error = gfs2_glock_get(sdp, ip->i_no_addr, &gfs2_inode_glops, CREATE, &ip->i_gl); + if (error) + goto fail_dealloc_inode; + + error = gfs2_glock_get(sdp, ip->i_no_addr, &gfs2_iopen_glops, CREATE, &io_gl); + if (error) + goto fail_dealloc_inode; + gfs2_cancel_delete_work(io_gl); + io_gl->gl_no_formal_ino = ip->i_no_formal_ino; + +retry: + error = insert_inode_locked4(inode, ip->i_no_addr, iget_test, &ip->i_no_addr); + if (error == -EBUSY) + goto retry; + if (error) + goto fail_gunlock2; + + error = gfs2_glock_nq_init(io_gl, LM_ST_SHARED, GL_EXACT | GL_NOPID, + &ip->i_iopen_gh); + if (error) + goto fail_gunlock2; + + error = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, GL_SKIP, &gh); + if (error) + goto fail_gunlock3; + clear_bit(GLF_INSTANTIATE_NEEDED, &ip->i_gl->gl_flags); + + error = gfs2_trans_begin(sdp, blocks, 0); + if (error) + goto fail_gunlock3; + + if (blocks > 1) { + gfs2_init_xattr(ip); + xattr_initialized = true; + } + init_dinode(dip, ip, symname); + gfs2_trans_end(sdp); + + glock_set_object(ip->i_gl, ip); + glock_set_object(io_gl, ip); + gfs2_set_iop(inode); + + if (default_acl) { + error = __gfs2_set_acl(inode, default_acl, ACL_TYPE_DEFAULT); + if (error) + goto fail_gunlock4; + posix_acl_release(default_acl); + default_acl = NULL; + } + if (acl) { + error = __gfs2_set_acl(inode, acl, ACL_TYPE_ACCESS); + if (error) + goto fail_gunlock4; + posix_acl_release(acl); + acl = NULL; + } + + error = security_inode_init_security(&ip->i_inode, &dip->i_inode, name, + &gfs2_initxattrs, NULL); + if (error) + goto fail_gunlock4; + + error = link_dinode(dip, name, ip, &da); + if (error) + goto fail_gunlock4; + + mark_inode_dirty(inode); + d_instantiate_new(dentry, inode); + /* After instantiate, errors should result in evict which will destroy + * both inode and iopen glocks properly. */ + if (file) { + file->f_mode |= FMODE_CREATED; + error = finish_open(file, dentry, gfs2_open_common); + } + gfs2_glock_dq_uninit(&d_gh); + gfs2_qa_put(ip); + gfs2_glock_dq_uninit(&gh); + gfs2_glock_put(io_gl); + gfs2_qa_put(dip); + return error; + +fail_gunlock4: + glock_clear_object(ip->i_gl, ip); + glock_clear_object(io_gl, ip); +fail_gunlock3: + gfs2_glock_dq_uninit(&ip->i_iopen_gh); +fail_gunlock2: + gfs2_glock_put(io_gl); +fail_dealloc_inode: + dealloc_error = 0; + if (ip->i_eattr) + dealloc_error = gfs2_ea_dealloc(ip, xattr_initialized); + clear_nlink(inode); + mark_inode_dirty(inode); + if (!dealloc_error) + dealloc_error = gfs2_dinode_dealloc(ip); + if (dealloc_error) + fs_warn(sdp, "%s: %d\n", __func__, dealloc_error); + ip->i_no_addr = 0; +fail_free_inode: + if (ip->i_gl) { + gfs2_glock_put(ip->i_gl); + ip->i_gl = NULL; + } + gfs2_rs_deltree(&ip->i_res); + gfs2_qa_put(ip); +fail_free_acls: + posix_acl_release(default_acl); + posix_acl_release(acl); +fail_gunlock: + gfs2_dir_no_add(&da); + gfs2_glock_dq_uninit(&d_gh); + if (!IS_ERR_OR_NULL(inode)) { + if (inode_state_read_once(inode) & I_NEW) + iget_failed(inode); + else + iput(inode); + } + if (gfs2_holder_initialized(&gh)) + gfs2_glock_dq_uninit(&gh); +fail: + gfs2_qa_put(dip); + return error; +} + +/** + * gfs2_create - Create a file + * @idmap: idmap of the mount the inode was found from + * @dir: The directory in which to create the file + * @dentry: The dentry of the new file + * @mode: The mode of the new file + * + * Returns: errno + */ + +static int gfs2_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + return gfs2_create_inode(dir, dentry, NULL, S_IFREG | mode, 0, NULL, 0, 1); +} + +/** + * __gfs2_lookup - Look up a filename in a directory and return its inode + * @dir: The directory inode + * @dentry: The dentry of the new inode + * @file: File to be opened + * + * + * Returns: errno + */ + +static struct dentry *__gfs2_lookup(struct inode *dir, struct dentry *dentry, + struct file *file) +{ + struct inode *inode; + struct dentry *d; + struct gfs2_holder gh; + struct gfs2_glock *gl; + int error; + + inode = gfs2_lookupi(dir, &dentry->d_name, 0); + if (inode == NULL || IS_ERR(inode)) + return d_splice_alias(inode, dentry); + + gl = GFS2_I(inode)->i_gl; + error = gfs2_glock_nq_init(gl, LM_ST_SHARED, LM_FLAG_ANY, &gh); + if (error) { + iput(inode); + return ERR_PTR(error); + } + + d = d_splice_alias(inode, dentry); + if (IS_ERR(d)) { + gfs2_glock_dq_uninit(&gh); + return d; + } + if (file && S_ISREG(inode->i_mode)) + error = finish_open(file, dentry, gfs2_open_common); + + gfs2_glock_dq_uninit(&gh); + if (error) { + dput(d); + return ERR_PTR(error); + } + return d; +} + +static struct dentry *gfs2_lookup(struct inode *dir, struct dentry *dentry, + unsigned flags) +{ + return __gfs2_lookup(dir, dentry, NULL); +} + +/** + * gfs2_link - Link to a file + * @old_dentry: The inode to link + * @dir: Add link to this directory + * @dentry: The name of the link + * + * Link the inode in "old_dentry" into the directory "dir" with the + * name in "dentry". + * + * Returns: errno + */ + +static int gfs2_link(struct dentry *old_dentry, struct inode *dir, + struct dentry *dentry) +{ + struct gfs2_inode *dip = GFS2_I(dir); + struct gfs2_sbd *sdp = GFS2_SB(dir); + struct inode *inode = d_inode(old_dentry); + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_holder d_gh, gh; + struct buffer_head *dibh; + struct gfs2_diradd da = { .bh = NULL, .save_loc = 1, }; + int error; + + if (S_ISDIR(inode->i_mode)) + return -EPERM; + + error = gfs2_qa_get(dip); + if (error) + return error; + + gfs2_holder_init(dip->i_gl, LM_ST_EXCLUSIVE, 0, &d_gh); + gfs2_holder_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &gh); + + error = gfs2_glock_nq(&d_gh); + if (error) + goto out_parent; + + error = gfs2_glock_nq(&gh); + if (error) + goto out_child; + + error = -ENOENT; + if (inode->i_nlink == 0) + goto out_gunlock; + + error = gfs2_permission(&nop_mnt_idmap, dir, MAY_WRITE | MAY_EXEC); + if (error) + goto out_gunlock; + + error = gfs2_dir_check(dir, &dentry->d_name, NULL); + switch (error) { + case -ENOENT: + break; + case 0: + error = -EEXIST; + goto out_gunlock; + default: + goto out_gunlock; + } + + error = -EINVAL; + if (!dip->i_inode.i_nlink) + goto out_gunlock; + error = -EFBIG; + if (dip->i_entries == (u32)-1) + goto out_gunlock; + error = -EPERM; + if (IS_IMMUTABLE(inode) || IS_APPEND(inode)) + goto out_gunlock; + error = -EMLINK; + if (ip->i_inode.i_nlink == (u32)-1) + goto out_gunlock; + + error = gfs2_diradd_alloc_required(dir, &dentry->d_name, &da); + if (error < 0) + goto out_gunlock; + + if (da.nr_blocks) { + struct gfs2_alloc_parms ap = { .target = da.nr_blocks, }; + error = gfs2_quota_lock_check(dip, &ap); + if (error) + goto out_gunlock; + + error = gfs2_inplace_reserve(dip, &ap); + if (error) + goto out_gunlock_q; + + error = gfs2_trans_begin(sdp, gfs2_trans_da_blks(dip, &da, 2), 0); + if (error) + goto out_ipres; + } else { + error = gfs2_trans_begin(sdp, 2 * RES_DINODE + RES_LEAF, 0); + if (error) + goto out_ipres; + } + + error = gfs2_meta_inode_buffer(ip, &dibh); + if (error) + goto out_end_trans; + + error = gfs2_dir_add(dir, &dentry->d_name, ip, &da); + if (error) + goto out_brelse; + + gfs2_trans_add_meta(ip->i_gl, dibh); + inc_nlink(&ip->i_inode); + inode_set_ctime_current(&ip->i_inode); + ihold(inode); + d_instantiate(dentry, inode); + mark_inode_dirty(inode); + +out_brelse: + brelse(dibh); +out_end_trans: + gfs2_trans_end(sdp); +out_ipres: + if (da.nr_blocks) + gfs2_inplace_release(dip); +out_gunlock_q: + if (da.nr_blocks) + gfs2_quota_unlock(dip); +out_gunlock: + gfs2_dir_no_add(&da); + gfs2_glock_dq(&gh); +out_child: + gfs2_glock_dq(&d_gh); +out_parent: + gfs2_qa_put(dip); + gfs2_holder_uninit(&d_gh); + gfs2_holder_uninit(&gh); + return error; +} + +/* + * gfs2_unlink_ok - check to see that a inode is still in a directory + * @dip: the directory + * @name: the name of the file + * @ip: the inode + * + * Assumes that the lock on (at least) @dip is held. + * + * Returns: 0 if the parent/child relationship is correct, errno if it isn't + */ + +static int gfs2_unlink_ok(struct gfs2_inode *dip, const struct qstr *name, + const struct gfs2_inode *ip) +{ + int error; + + if (IS_IMMUTABLE(&ip->i_inode) || IS_APPEND(&ip->i_inode)) + return -EPERM; + + if ((dip->i_inode.i_mode & S_ISVTX) && + !uid_eq(dip->i_inode.i_uid, current_fsuid()) && + !uid_eq(ip->i_inode.i_uid, current_fsuid()) && !capable(CAP_FOWNER)) + return -EPERM; + + if (IS_APPEND(&dip->i_inode)) + return -EPERM; + + error = gfs2_permission(&nop_mnt_idmap, &dip->i_inode, + MAY_WRITE | MAY_EXEC); + if (error) + return error; + + return gfs2_dir_check(&dip->i_inode, name, ip); +} + +/** + * gfs2_unlink_inode - Removes an inode from its parent dir and unlinks it + * @dip: The parent directory + * @dentry: The dentry to unlink + * + * Called with all the locks and in a transaction. This will only be + * called for a directory after it has been checked to ensure it is empty. + * + * Returns: 0 on success, or an error + */ + +static int gfs2_unlink_inode(struct gfs2_inode *dip, + const struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + struct gfs2_inode *ip = GFS2_I(inode); + int error; + + error = gfs2_dir_del(dip, dentry); + if (error) + return error; + + ip->i_entries = 0; + inode_set_ctime_current(inode); + if (S_ISDIR(inode->i_mode)) + clear_nlink(inode); + else + drop_nlink(inode); + mark_inode_dirty(inode); + if (inode->i_nlink == 0) + gfs2_unlink_di(inode); + return 0; +} + + +/** + * gfs2_unlink - Unlink an inode (this does rmdir as well) + * @dir: The inode of the directory containing the inode to unlink + * @dentry: The file itself + * + * This routine uses the type of the inode as a flag to figure out + * whether this is an unlink or an rmdir. + * + * Returns: errno + */ + +static int gfs2_unlink(struct inode *dir, struct dentry *dentry) +{ + struct gfs2_inode *dip = GFS2_I(dir); + struct gfs2_sbd *sdp = GFS2_SB(dir); + struct inode *inode = d_inode(dentry); + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_holder d_gh, r_gh, gh; + struct gfs2_rgrpd *rgd; + int error; + + error = gfs2_rindex_update(sdp); + if (error) + return error; + + error = -EROFS; + + gfs2_holder_init(dip->i_gl, LM_ST_EXCLUSIVE, 0, &d_gh); + gfs2_holder_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &gh); + + rgd = gfs2_blk2rgrpd(sdp, ip->i_no_addr, 1); + if (!rgd) + goto out_inodes; + + gfs2_holder_init(rgd->rd_gl, LM_ST_EXCLUSIVE, LM_FLAG_NODE_SCOPE, &r_gh); + + + error = gfs2_glock_nq(&d_gh); + if (error) + goto out_parent; + + error = gfs2_glock_nq(&gh); + if (error) + goto out_child; + + error = -ENOENT; + if (inode->i_nlink == 0) + goto out_rgrp; + + if (S_ISDIR(inode->i_mode)) { + error = -ENOTEMPTY; + if (ip->i_entries > 2 || inode->i_nlink > 2) + goto out_rgrp; + } + + error = gfs2_glock_nq(&r_gh); /* rgrp */ + if (error) + goto out_rgrp; + + error = gfs2_unlink_ok(dip, &dentry->d_name, ip); + if (error) + goto out_gunlock; + + error = gfs2_trans_begin(sdp, 2*RES_DINODE + 3*RES_LEAF + RES_RG_BIT, 0); + if (error) + goto out_gunlock; + + error = gfs2_unlink_inode(dip, dentry); + gfs2_trans_end(sdp); + +out_gunlock: + gfs2_glock_dq(&r_gh); +out_rgrp: + gfs2_glock_dq(&gh); +out_child: + gfs2_glock_dq(&d_gh); +out_parent: + gfs2_holder_uninit(&r_gh); +out_inodes: + gfs2_holder_uninit(&gh); + gfs2_holder_uninit(&d_gh); + return error; +} + +/** + * gfs2_symlink - Create a symlink + * @idmap: idmap of the mount the inode was found from + * @dir: The directory to create the symlink in + * @dentry: The dentry to put the symlink in + * @symname: The thing which the link points to + * + * Returns: errno + */ + +static int gfs2_symlink(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const char *symname) +{ + unsigned int size; + + size = strlen(symname); + if (size >= gfs2_max_stuffed_size(GFS2_I(dir))) + return -ENAMETOOLONG; + + return gfs2_create_inode(dir, dentry, NULL, S_IFLNK | S_IRWXUGO, 0, symname, size, 0); +} + +/** + * gfs2_mkdir - Make a directory + * @idmap: idmap of the mount the inode was found from + * @dir: The parent directory of the new one + * @dentry: The dentry of the new directory + * @mode: The mode of the new directory + * + * Returns: the dentry, or ERR_PTR(errno) + */ + +static struct dentry *gfs2_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + unsigned dsize = gfs2_max_stuffed_size(GFS2_I(dir)); + + return ERR_PTR(gfs2_create_inode(dir, dentry, NULL, mode, 0, NULL, dsize, 0)); +} + +/** + * gfs2_mknod - Make a special file + * @idmap: idmap of the mount the inode was found from + * @dir: The directory in which the special file will reside + * @dentry: The dentry of the special file + * @mode: The mode of the special file + * @dev: The device specification of the special file + * + */ + +static int gfs2_mknod(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t dev) +{ + return gfs2_create_inode(dir, dentry, NULL, mode, dev, NULL, 0, 0); +} + +/** + * gfs2_atomic_open - Atomically open a file + * @dir: The directory + * @dentry: The proposed new entry + * @file: The proposed new struct file + * @flags: open flags + * @mode: File mode + * + * Returns: error code or 0 for success + */ + +static int gfs2_atomic_open(struct inode *dir, struct dentry *dentry, + struct file *file, unsigned flags, + umode_t mode) +{ + bool excl = !!(flags & O_EXCL); + + if (d_in_lookup(dentry)) { + struct dentry *d = __gfs2_lookup(dir, dentry, file); + if (file->f_mode & FMODE_OPENED) { + if (IS_ERR(d)) + return PTR_ERR(d); + dput(d); + return excl && (flags & O_CREAT) ? -EEXIST : 0; + } + if (d || d_really_is_positive(dentry)) + return finish_no_open(file, d); + } + if (!(flags & O_CREAT)) + return -ENOENT; + + return gfs2_create_inode(dir, dentry, file, S_IFREG | mode, 0, NULL, 0, excl); +} + +/* + * gfs2_ok_to_move - check if it's ok to move a directory to another directory + * @this: move this + * @to: to here + * + * Follow @to back to the root and make sure we don't encounter @this + * Assumes we already hold the rename lock. + * + * Returns: errno + */ + +static int gfs2_ok_to_move(struct gfs2_inode *this, struct gfs2_inode *to) +{ + struct inode *dir = &to->i_inode; + struct super_block *sb = dir->i_sb; + struct inode *tmp; + int error = 0; + + igrab(dir); + + for (;;) { + if (dir == &this->i_inode) { + error = -EINVAL; + break; + } + if (dir == d_inode(sb->s_root)) { + error = 0; + break; + } + + tmp = gfs2_lookupi(dir, &gfs2_qdotdot, 1); + if (!tmp) { + error = -ENOENT; + break; + } + if (IS_ERR(tmp)) { + error = PTR_ERR(tmp); + break; + } + + iput(dir); + dir = tmp; + } + + iput(dir); + + return error; +} + +/** + * update_moved_ino - Update an inode that's being moved + * @ip: The inode being moved + * @ndip: The parent directory of the new filename + * @dir_rename: True of ip is a directory + * + * Returns: errno + */ + +static int update_moved_ino(struct gfs2_inode *ip, struct gfs2_inode *ndip, + int dir_rename) +{ + if (dir_rename) + return gfs2_dir_mvino(ip, &gfs2_qdotdot, ndip, DT_DIR); + + inode_set_ctime_current(&ip->i_inode); + mark_inode_dirty_sync(&ip->i_inode); + return 0; +} + + +/** + * gfs2_rename - Rename a file + * @odir: Parent directory of old file name + * @odentry: The old dentry of the file + * @ndir: Parent directory of new file name + * @ndentry: The new dentry of the file + * + * Returns: errno + */ + +static int gfs2_rename(struct inode *odir, struct dentry *odentry, + struct inode *ndir, struct dentry *ndentry) +{ + struct gfs2_inode *odip = GFS2_I(odir); + struct gfs2_inode *ndip = GFS2_I(ndir); + struct gfs2_inode *ip = GFS2_I(d_inode(odentry)); + struct gfs2_inode *nip = NULL; + struct gfs2_sbd *sdp = GFS2_SB(odir); + struct gfs2_holder ghs[4], r_gh, rd_gh; + struct gfs2_rgrpd *nrgd; + unsigned int num_gh; + int dir_rename = 0; + struct gfs2_diradd da = { .nr_blocks = 0, .save_loc = 0, }; + unsigned int retries = 0, x; + int error; + + gfs2_holder_mark_uninitialized(&r_gh); + gfs2_holder_mark_uninitialized(&rd_gh); + if (d_really_is_positive(ndentry)) { + nip = GFS2_I(d_inode(ndentry)); + if (ip == nip) + return 0; + } + + error = gfs2_rindex_update(sdp); + if (error) + return error; + + error = gfs2_qa_get(ndip); + if (error) + return error; + + if (odip != ndip) { + error = gfs2_glock_nq_init(sdp->sd_rename_gl, LM_ST_EXCLUSIVE, + 0, &r_gh); + if (error) + goto out; + + if (S_ISDIR(ip->i_inode.i_mode)) { + dir_rename = 1; + /* don't move a directory into its subdir */ + error = gfs2_ok_to_move(ip, ndip); + if (error) + goto out_gunlock_r; + } + } + + num_gh = 1; + gfs2_holder_init(odip->i_gl, LM_ST_EXCLUSIVE, GL_ASYNC, ghs); + if (odip != ndip) { + gfs2_holder_init(ndip->i_gl, LM_ST_EXCLUSIVE,GL_ASYNC, + ghs + num_gh); + num_gh++; + } + gfs2_holder_init(ip->i_gl, LM_ST_EXCLUSIVE, GL_ASYNC, ghs + num_gh); + num_gh++; + + if (nip) { + gfs2_holder_init(nip->i_gl, LM_ST_EXCLUSIVE, GL_ASYNC, + ghs + num_gh); + num_gh++; + } + +again: + for (x = 0; x < num_gh; x++) { + error = gfs2_glock_nq(ghs + x); + if (error) + goto out_gunlock; + } + error = gfs2_glock_async_wait(num_gh, ghs, retries); + if (error == -ESTALE) { + retries++; + goto again; + } + if (error) + goto out_gunlock; + + if (nip) { + /* Grab the resource group glock for unlink flag twiddling. + * This is the case where the target dinode already exists + * so we unlink before doing the rename. + */ + nrgd = gfs2_blk2rgrpd(sdp, nip->i_no_addr, 1); + if (!nrgd) { + error = -ENOENT; + goto out_gunlock; + } + error = gfs2_glock_nq_init(nrgd->rd_gl, LM_ST_EXCLUSIVE, + LM_FLAG_NODE_SCOPE, &rd_gh); + if (error) + goto out_gunlock; + } + + error = -ENOENT; + if (ip->i_inode.i_nlink == 0) + goto out_gunlock; + + /* Check out the old directory */ + + error = gfs2_unlink_ok(odip, &odentry->d_name, ip); + if (error) + goto out_gunlock; + + /* Check out the new directory */ + + if (nip) { + error = gfs2_unlink_ok(ndip, &ndentry->d_name, nip); + if (error) + goto out_gunlock; + + if (nip->i_inode.i_nlink == 0) { + error = -EAGAIN; + goto out_gunlock; + } + + if (S_ISDIR(nip->i_inode.i_mode)) { + if (nip->i_entries < 2) { + gfs2_consist_inode(nip); + error = -EIO; + goto out_gunlock; + } + if (nip->i_entries > 2) { + error = -ENOTEMPTY; + goto out_gunlock; + } + } + } else { + error = gfs2_permission(&nop_mnt_idmap, ndir, + MAY_WRITE | MAY_EXEC); + if (error) + goto out_gunlock; + + error = gfs2_dir_check(ndir, &ndentry->d_name, NULL); + switch (error) { + case -ENOENT: + error = 0; + break; + case 0: + error = -EEXIST; + goto out_gunlock; + default: + goto out_gunlock; + } + + if (odip != ndip) { + if (!ndip->i_inode.i_nlink) { + error = -ENOENT; + goto out_gunlock; + } + if (ndip->i_entries == (u32)-1) { + error = -EFBIG; + goto out_gunlock; + } + if (S_ISDIR(ip->i_inode.i_mode) && + ndip->i_inode.i_nlink == (u32)-1) { + error = -EMLINK; + goto out_gunlock; + } + } + } + + /* Check out the dir to be renamed */ + + if (dir_rename) { + error = gfs2_permission(&nop_mnt_idmap, d_inode(odentry), + MAY_WRITE); + if (error) + goto out_gunlock; + } + + if (nip == NULL) { + error = gfs2_diradd_alloc_required(ndir, &ndentry->d_name, &da); + if (error) + goto out_gunlock; + } + + if (da.nr_blocks) { + struct gfs2_alloc_parms ap = { .target = da.nr_blocks, }; + error = gfs2_quota_lock_check(ndip, &ap); + if (error) + goto out_gunlock; + + error = gfs2_inplace_reserve(ndip, &ap); + if (error) + goto out_gunlock_q; + + error = gfs2_trans_begin(sdp, gfs2_trans_da_blks(ndip, &da, 4) + + 4 * RES_LEAF + 4, 0); + if (error) + goto out_ipreserv; + } else { + error = gfs2_trans_begin(sdp, 4 * RES_DINODE + + 5 * RES_LEAF + 4, 0); + if (error) + goto out_gunlock; + } + + /* Remove the target file, if it exists */ + + if (nip) + error = gfs2_unlink_inode(ndip, ndentry); + + error = update_moved_ino(ip, ndip, dir_rename); + if (error) + goto out_end_trans; + + error = gfs2_dir_del(odip, odentry); + if (error) + goto out_end_trans; + + error = gfs2_dir_add(ndir, &ndentry->d_name, ip, &da); + if (error) + goto out_end_trans; + +out_end_trans: + gfs2_trans_end(sdp); +out_ipreserv: + if (da.nr_blocks) + gfs2_inplace_release(ndip); +out_gunlock_q: + if (da.nr_blocks) + gfs2_quota_unlock(ndip); +out_gunlock: + gfs2_dir_no_add(&da); + if (gfs2_holder_initialized(&rd_gh)) + gfs2_glock_dq_uninit(&rd_gh); + + while (x--) { + if (gfs2_holder_queued(ghs + x)) + gfs2_glock_dq(ghs + x); + gfs2_holder_uninit(ghs + x); + } +out_gunlock_r: + if (gfs2_holder_initialized(&r_gh)) + gfs2_glock_dq_uninit(&r_gh); +out: + gfs2_qa_put(ndip); + return error; +} + +/** + * gfs2_exchange - exchange two files + * @odir: Parent directory of old file name + * @odentry: The old dentry of the file + * @ndir: Parent directory of new file name + * @ndentry: The new dentry of the file + * @flags: The rename flags + * + * Returns: errno + */ + +static int gfs2_exchange(struct inode *odir, struct dentry *odentry, + struct inode *ndir, struct dentry *ndentry, + unsigned int flags) +{ + struct gfs2_inode *odip = GFS2_I(odir); + struct gfs2_inode *ndip = GFS2_I(ndir); + struct gfs2_inode *oip = GFS2_I(odentry->d_inode); + struct gfs2_inode *nip = GFS2_I(ndentry->d_inode); + struct gfs2_sbd *sdp = GFS2_SB(odir); + struct gfs2_holder ghs[4], r_gh; + unsigned int num_gh; + unsigned int retries = 0, x; + umode_t old_mode = oip->i_inode.i_mode; + umode_t new_mode = nip->i_inode.i_mode; + int error; + + gfs2_holder_mark_uninitialized(&r_gh); + error = gfs2_rindex_update(sdp); + if (error) + return error; + + if (odip != ndip) { + error = gfs2_glock_nq_init(sdp->sd_rename_gl, LM_ST_EXCLUSIVE, + 0, &r_gh); + if (error) + goto out; + + if (S_ISDIR(old_mode)) { + /* don't move a directory into its subdir */ + error = gfs2_ok_to_move(oip, ndip); + if (error) + goto out_gunlock_r; + } + + if (S_ISDIR(new_mode)) { + /* don't move a directory into its subdir */ + error = gfs2_ok_to_move(nip, odip); + if (error) + goto out_gunlock_r; + } + } + + num_gh = 1; + gfs2_holder_init(odip->i_gl, LM_ST_EXCLUSIVE, GL_ASYNC, ghs); + if (odip != ndip) { + gfs2_holder_init(ndip->i_gl, LM_ST_EXCLUSIVE, GL_ASYNC, + ghs + num_gh); + num_gh++; + } + gfs2_holder_init(oip->i_gl, LM_ST_EXCLUSIVE, GL_ASYNC, ghs + num_gh); + num_gh++; + + gfs2_holder_init(nip->i_gl, LM_ST_EXCLUSIVE, GL_ASYNC, ghs + num_gh); + num_gh++; + +again: + for (x = 0; x < num_gh; x++) { + error = gfs2_glock_nq(ghs + x); + if (error) + goto out_gunlock; + } + + error = gfs2_glock_async_wait(num_gh, ghs, retries); + if (error == -ESTALE) { + retries++; + goto again; + } + if (error) + goto out_gunlock; + + error = -ENOENT; + if (oip->i_inode.i_nlink == 0 || nip->i_inode.i_nlink == 0) + goto out_gunlock; + + error = gfs2_unlink_ok(odip, &odentry->d_name, oip); + if (error) + goto out_gunlock; + error = gfs2_unlink_ok(ndip, &ndentry->d_name, nip); + if (error) + goto out_gunlock; + + if (S_ISDIR(old_mode)) { + error = gfs2_permission(&nop_mnt_idmap, odentry->d_inode, + MAY_WRITE); + if (error) + goto out_gunlock; + } + if (S_ISDIR(new_mode)) { + error = gfs2_permission(&nop_mnt_idmap, ndentry->d_inode, + MAY_WRITE); + if (error) + goto out_gunlock; + } + error = gfs2_trans_begin(sdp, 4 * RES_DINODE + 4 * RES_LEAF, 0); + if (error) + goto out_gunlock; + + error = update_moved_ino(oip, ndip, S_ISDIR(old_mode)); + if (error) + goto out_end_trans; + + error = update_moved_ino(nip, odip, S_ISDIR(new_mode)); + if (error) + goto out_end_trans; + + error = gfs2_dir_mvino(ndip, &ndentry->d_name, oip, + IF2DT(old_mode)); + if (error) + goto out_end_trans; + + error = gfs2_dir_mvino(odip, &odentry->d_name, nip, + IF2DT(new_mode)); + if (error) + goto out_end_trans; + + if (odip != ndip) { + if (S_ISDIR(new_mode) && !S_ISDIR(old_mode)) { + inc_nlink(&odip->i_inode); + drop_nlink(&ndip->i_inode); + } else if (S_ISDIR(old_mode) && !S_ISDIR(new_mode)) { + inc_nlink(&ndip->i_inode); + drop_nlink(&odip->i_inode); + } + } + mark_inode_dirty(&ndip->i_inode); + if (odip != ndip) + mark_inode_dirty(&odip->i_inode); + +out_end_trans: + gfs2_trans_end(sdp); +out_gunlock: + while (x--) { + if (gfs2_holder_queued(ghs + x)) + gfs2_glock_dq(ghs + x); + gfs2_holder_uninit(ghs + x); + } +out_gunlock_r: + if (gfs2_holder_initialized(&r_gh)) + gfs2_glock_dq_uninit(&r_gh); +out: + return error; +} + +static int gfs2_rename2(struct mnt_idmap *idmap, struct inode *odir, + struct dentry *odentry, struct inode *ndir, + struct dentry *ndentry, unsigned int flags) +{ + flags &= ~RENAME_NOREPLACE; + + if (flags & ~RENAME_EXCHANGE) + return -EINVAL; + + if (flags & RENAME_EXCHANGE) + return gfs2_exchange(odir, odentry, ndir, ndentry, flags); + + return gfs2_rename(odir, odentry, ndir, ndentry); +} + +/** + * gfs2_get_link - Follow a symbolic link + * @dentry: The dentry of the link + * @inode: The inode of the link + * @done: destructor for return value + * + * This can handle symlinks of any size. + * + * Returns: 0 on success or error code + */ + +static const char *gfs2_get_link(struct dentry *dentry, + struct inode *inode, + struct delayed_call *done) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_holder i_gh; + struct buffer_head *dibh; + unsigned int size; + char *buf; + int error; + + if (!dentry) + return ERR_PTR(-ECHILD); + + gfs2_holder_init(ip->i_gl, LM_ST_SHARED, 0, &i_gh); + error = gfs2_glock_nq(&i_gh); + if (error) { + gfs2_holder_uninit(&i_gh); + return ERR_PTR(error); + } + + size = (unsigned int)i_size_read(&ip->i_inode); + if (size == 0) { + gfs2_consist_inode(ip); + buf = ERR_PTR(-EIO); + goto out; + } + + error = gfs2_meta_inode_buffer(ip, &dibh); + if (error) { + buf = ERR_PTR(error); + goto out; + } + + buf = kzalloc(size + 1, GFP_NOFS); + if (!buf) + buf = ERR_PTR(-ENOMEM); + else + memcpy(buf, dibh->b_data + sizeof(struct gfs2_dinode), size); + brelse(dibh); +out: + gfs2_glock_dq_uninit(&i_gh); + if (!IS_ERR(buf)) + set_delayed_call(done, kfree_link, buf); + return buf; +} + +/** + * gfs2_permission + * @idmap: idmap of the mount the inode was found from + * @inode: The inode + * @mask: The mask to be tested + * + * This may be called from the VFS directly, or from within GFS2 with the + * inode locked, so we look to see if the glock is already locked and only + * lock the glock if its not already been done. + * + * Returns: errno + */ + +int gfs2_permission(struct mnt_idmap *idmap, struct inode *inode, + int mask) +{ + int may_not_block = mask & MAY_NOT_BLOCK; + struct gfs2_inode *ip; + struct gfs2_holder i_gh; + struct gfs2_glock *gl; + int error; + + gfs2_holder_mark_uninitialized(&i_gh); + ip = GFS2_I(inode); + gl = rcu_dereference_check(ip->i_gl, !may_not_block); + if (unlikely(!gl)) { + /* inode is getting torn down, must be RCU mode */ + WARN_ON_ONCE(!may_not_block); + return -ECHILD; + } + if (gfs2_glock_is_locked_by_me(gl) == NULL) { + if (may_not_block) + return -ECHILD; + error = gfs2_glock_nq_init(gl, LM_ST_SHARED, LM_FLAG_ANY, &i_gh); + if (error) + return error; + } + + if ((mask & MAY_WRITE) && IS_IMMUTABLE(inode)) + error = -EPERM; + else + error = generic_permission(&nop_mnt_idmap, inode, mask); + if (gfs2_holder_initialized(&i_gh)) + gfs2_glock_dq_uninit(&i_gh); + + return error; +} + +static int __gfs2_setattr_simple(struct inode *inode, struct iattr *attr) +{ + setattr_copy(&nop_mnt_idmap, inode, attr); + mark_inode_dirty(inode); + return 0; +} + +static int gfs2_setattr_simple(struct inode *inode, struct iattr *attr) +{ + int error; + + if (current->journal_info) + return __gfs2_setattr_simple(inode, attr); + + error = gfs2_trans_begin(GFS2_SB(inode), RES_DINODE, 0); + if (error) + return error; + + error = __gfs2_setattr_simple(inode, attr); + gfs2_trans_end(GFS2_SB(inode)); + return error; +} + +static int setattr_chown(struct inode *inode, struct iattr *attr) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + kuid_t ouid, nuid; + kgid_t ogid, ngid; + int error; + struct gfs2_alloc_parms ap = {}; + + ouid = inode->i_uid; + ogid = inode->i_gid; + nuid = attr->ia_uid; + ngid = attr->ia_gid; + + if (!(attr->ia_valid & ATTR_UID) || uid_eq(ouid, nuid)) + ouid = nuid = NO_UID_QUOTA_CHANGE; + if (!(attr->ia_valid & ATTR_GID) || gid_eq(ogid, ngid)) + ogid = ngid = NO_GID_QUOTA_CHANGE; + error = gfs2_qa_get(ip); + if (error) + return error; + + error = gfs2_rindex_update(sdp); + if (error) + goto out; + + error = gfs2_quota_lock(ip, nuid, ngid); + if (error) + goto out; + + ap.target = gfs2_get_inode_blocks(&ip->i_inode); + + if (!uid_eq(ouid, NO_UID_QUOTA_CHANGE) || + !gid_eq(ogid, NO_GID_QUOTA_CHANGE)) { + error = gfs2_quota_check(ip, nuid, ngid, &ap); + if (error) + goto out_gunlock_q; + } + + error = gfs2_trans_begin(sdp, RES_DINODE + 2 * RES_QUOTA, 0); + if (error) + goto out_gunlock_q; + + error = gfs2_setattr_simple(inode, attr); + if (error) + goto out_end_trans; + + if (!uid_eq(ouid, NO_UID_QUOTA_CHANGE) || + !gid_eq(ogid, NO_GID_QUOTA_CHANGE)) { + gfs2_quota_change(ip, -(s64)ap.target, ouid, ogid); + gfs2_quota_change(ip, ap.target, nuid, ngid); + } + +out_end_trans: + gfs2_trans_end(sdp); +out_gunlock_q: + gfs2_quota_unlock(ip); +out: + gfs2_qa_put(ip); + return error; +} + +/** + * gfs2_setattr - Change attributes on an inode + * @idmap: idmap of the mount the inode was found from + * @dentry: The dentry which is changing + * @attr: The structure describing the change + * + * The VFS layer wants to change one or more of an inodes attributes. Write + * that change out to disk. + * + * Returns: errno + */ + +static int gfs2_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, struct iattr *attr) +{ + struct inode *inode = d_inode(dentry); + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_holder i_gh; + int error; + + error = gfs2_qa_get(ip); + if (error) + return error; + + error = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &i_gh); + if (error) + goto out; + + error = may_setattr(&nop_mnt_idmap, inode, attr->ia_valid); + if (error) + goto error; + + error = setattr_prepare(&nop_mnt_idmap, dentry, attr); + if (error) + goto error; + + if (attr->ia_valid & ATTR_SIZE) + error = gfs2_setattr_size(inode, attr->ia_size); + else if (attr->ia_valid & (ATTR_UID | ATTR_GID)) + error = setattr_chown(inode, attr); + else { + error = gfs2_setattr_simple(inode, attr); + if (!error && attr->ia_valid & ATTR_MODE) + error = posix_acl_chmod(&nop_mnt_idmap, dentry, + inode->i_mode); + } + +error: + if (!error) + mark_inode_dirty(inode); + gfs2_glock_dq_uninit(&i_gh); +out: + gfs2_qa_put(ip); + return error; +} + +/** + * gfs2_getattr - Read out an inode's attributes + * @idmap: idmap of the mount the inode was found from + * @path: Object to query + * @stat: The inode's stats + * @request_mask: Mask of STATX_xxx flags indicating the caller's interests + * @flags: AT_STATX_xxx setting + * + * This may be called from the VFS directly, or from within GFS2 with the + * inode locked, so we look to see if the glock is already locked and only + * lock the glock if its not already been done. Note that its the NFS + * readdirplus operation which causes this to be called (from filldir) + * with the glock already held. + * + * Returns: errno + */ + +static int gfs2_getattr(struct mnt_idmap *idmap, + const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int flags) +{ + struct inode *inode = d_inode(path->dentry); + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_holder gh; + u32 gfsflags; + int error; + + gfs2_holder_mark_uninitialized(&gh); + if (gfs2_glock_is_locked_by_me(ip->i_gl) == NULL) { + error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, LM_FLAG_ANY, &gh); + if (error) + return error; + } + + gfsflags = ip->i_diskflags; + if (gfsflags & GFS2_DIF_APPENDONLY) + stat->attributes |= STATX_ATTR_APPEND; + if (gfsflags & GFS2_DIF_IMMUTABLE) + stat->attributes |= STATX_ATTR_IMMUTABLE; + + stat->attributes_mask |= (STATX_ATTR_APPEND | + STATX_ATTR_COMPRESSED | + STATX_ATTR_ENCRYPTED | + STATX_ATTR_IMMUTABLE | + STATX_ATTR_NODUMP); + + generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat); + + if (gfs2_holder_initialized(&gh)) + gfs2_glock_dq_uninit(&gh); + + return 0; +} + +static bool fault_in_fiemap(struct fiemap_extent_info *fi) +{ + struct fiemap_extent __user *dest = fi->fi_extents_start; + size_t size = sizeof(*dest) * fi->fi_extents_max; + + return fault_in_safe_writeable((char __user *)dest, size) == 0; +} + +static int gfs2_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + u64 start, u64 len) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_holder gh; + int ret; + + inode_lock_shared(inode); + +retry: + ret = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, 0, &gh); + if (ret) + goto out; + + pagefault_disable(); + ret = iomap_fiemap(inode, fieinfo, start, len, &gfs2_iomap_ops); + pagefault_enable(); + + gfs2_glock_dq_uninit(&gh); + + if (ret == -EFAULT && fault_in_fiemap(fieinfo)) { + fieinfo->fi_extents_mapped = 0; + goto retry; + } + +out: + inode_unlock_shared(inode); + return ret; +} + +loff_t gfs2_seek_data(struct file *file, loff_t offset) +{ + struct inode *inode = file->f_mapping->host; + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_holder gh; + loff_t ret; + + inode_lock_shared(inode); + ret = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, 0, &gh); + if (!ret) + ret = iomap_seek_data(inode, offset, &gfs2_iomap_ops); + gfs2_glock_dq_uninit(&gh); + inode_unlock_shared(inode); + + if (ret < 0) + return ret; + return vfs_setpos(file, ret, inode->i_sb->s_maxbytes); +} + +loff_t gfs2_seek_hole(struct file *file, loff_t offset) +{ + struct inode *inode = file->f_mapping->host; + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_holder gh; + loff_t ret; + + inode_lock_shared(inode); + ret = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, 0, &gh); + if (!ret) + ret = iomap_seek_hole(inode, offset, &gfs2_iomap_ops); + gfs2_glock_dq_uninit(&gh); + inode_unlock_shared(inode); + + if (ret < 0) + return ret; + return vfs_setpos(file, ret, inode->i_sb->s_maxbytes); +} + +static int gfs2_update_time(struct inode *inode, enum fs_update_time type, + unsigned int flags) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_glock *gl = ip->i_gl; + struct gfs2_holder *gh; + int error; + + if (flags & IOCB_NOWAIT) + return -EAGAIN; + + gh = gfs2_glock_is_locked_by_me(gl); + if (gh && gl->gl_state != LM_ST_EXCLUSIVE) { + gfs2_glock_dq(gh); + gfs2_holder_reinit(LM_ST_EXCLUSIVE, 0, gh); + error = gfs2_glock_nq(gh); + if (error) + return error; + } + return generic_update_time(inode, type, flags); +} + +static const struct inode_operations gfs2_file_iops = { + .permission = gfs2_permission, + .setattr = gfs2_setattr, + .getattr = gfs2_getattr, + .listxattr = gfs2_listxattr, + .fiemap = gfs2_fiemap, + .get_inode_acl = gfs2_get_acl, + .set_acl = gfs2_set_acl, + .update_time = gfs2_update_time, + .fileattr_get = gfs2_fileattr_get, + .fileattr_set = gfs2_fileattr_set, +}; + +static const struct inode_operations gfs2_dir_iops = { + .create = gfs2_create, + .lookup = gfs2_lookup, + .link = gfs2_link, + .unlink = gfs2_unlink, + .symlink = gfs2_symlink, + .mkdir = gfs2_mkdir, + .rmdir = gfs2_unlink, + .mknod = gfs2_mknod, + .rename = gfs2_rename2, + .permission = gfs2_permission, + .setattr = gfs2_setattr, + .getattr = gfs2_getattr, + .listxattr = gfs2_listxattr, + .fiemap = gfs2_fiemap, + .get_inode_acl = gfs2_get_acl, + .set_acl = gfs2_set_acl, + .update_time = gfs2_update_time, + .atomic_open = gfs2_atomic_open, + .fileattr_get = gfs2_fileattr_get, + .fileattr_set = gfs2_fileattr_set, +}; + +static const struct inode_operations gfs2_symlink_iops = { + .get_link = gfs2_get_link, + .permission = gfs2_permission, + .setattr = gfs2_setattr, + .getattr = gfs2_getattr, + .listxattr = gfs2_listxattr, + .fiemap = gfs2_fiemap, +}; + diff --git a/fs/gfs2/inode.h b/fs/gfs2/inode.h new file mode 100644 index 000000000..2fcd96dd1 --- /dev/null +++ b/fs/gfs2/inode.h @@ -0,0 +1,135 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#ifndef __INODE_DOT_H__ +#define __INODE_DOT_H__ + +#include +#include +#include +#include "util.h" + +bool gfs2_release_folio(struct folio *folio, gfp_t gfp_mask); +ssize_t gfs2_internal_read(struct gfs2_inode *ip, + char *buf, loff_t *pos, size_t size); +void gfs2_set_aops(struct inode *inode); + +static inline int gfs2_is_stuffed(const struct gfs2_inode *ip) +{ + return !ip->i_height; +} + +static inline int gfs2_is_jdata(const struct gfs2_inode *ip) +{ + return ip->i_diskflags & GFS2_DIF_JDATA; +} + +static inline bool gfs2_is_ordered(const struct gfs2_sbd *sdp) +{ + return sdp->sd_args.ar_data == GFS2_DATA_ORDERED; +} + +static inline bool gfs2_is_writeback(const struct gfs2_sbd *sdp) +{ + return sdp->sd_args.ar_data == GFS2_DATA_WRITEBACK; +} + +static inline int gfs2_is_dir(const struct gfs2_inode *ip) +{ + return S_ISDIR(ip->i_inode.i_mode); +} + +static inline void gfs2_set_inode_blocks(struct inode *inode, u64 blocks) +{ + inode->i_blocks = blocks << (inode->i_blkbits - SECTOR_SHIFT); +} + +static inline u64 gfs2_get_inode_blocks(const struct inode *inode) +{ + return inode->i_blocks >> (inode->i_blkbits - SECTOR_SHIFT); +} + +static inline void gfs2_add_inode_blocks(struct inode *inode, s64 change) +{ + change <<= inode->i_blkbits - SECTOR_SHIFT; + gfs2_assert(GFS2_SB(inode), (change >= 0 || inode->i_blocks >= -change)); + inode->i_blocks += change; +} + +static inline int gfs2_check_inum(const struct gfs2_inode *ip, u64 no_addr, + u64 no_formal_ino) +{ + return ip->i_no_addr == no_addr && ip->i_no_formal_ino == no_formal_ino; +} + +static inline void gfs2_inum_out(const struct gfs2_inode *ip, + struct gfs2_dirent *dent) +{ + dent->de_inum.no_formal_ino = cpu_to_be64(ip->i_no_formal_ino); + dent->de_inum.no_addr = cpu_to_be64(ip->i_no_addr); +} + +static inline int gfs2_check_internal_file_size(struct inode *inode, + u64 minsize, u64 maxsize) +{ + u64 size = i_size_read(inode); + if (size < minsize || size > maxsize) + goto err; + if (size & (BIT(inode->i_blkbits) - 1)) + goto err; + return 0; +err: + gfs2_consist_inode(GFS2_I(inode)); + return -EIO; +} + +void gfs2_setup_inode(struct inode *inode); +struct inode *gfs2_inode_lookup(struct super_block *sb, unsigned type, + u64 no_addr, u64 no_formal_ino, + unsigned int blktype); +struct inode *gfs2_lookup_by_inum(struct gfs2_sbd *sdp, u64 no_addr, + u64 no_formal_ino, + unsigned int blktype); +int gfs2_dinode_dealloc(struct gfs2_inode *ip); + +struct inode *gfs2_lookupi(struct inode *dir, const struct qstr *name, + int is_root); +int gfs2_permission(struct mnt_idmap *idmap, + struct inode *inode, int mask); +struct inode *gfs2_lookup_meta(struct inode *dip, const char *name); +void gfs2_dinode_out(const struct gfs2_inode *ip, void *buf); +int gfs2_open_common(struct inode *inode, struct file *file); +loff_t gfs2_seek_data(struct file *file, loff_t offset); +loff_t gfs2_seek_hole(struct file *file, loff_t offset); + +extern const struct file_operations gfs2_file_fops_nolock; +extern const struct file_operations gfs2_dir_fops_nolock; + +int gfs2_fileattr_get(struct dentry *dentry, struct file_kattr *fa); +int gfs2_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa); +void gfs2_set_inode_flags(struct inode *inode); + +#ifdef CONFIG_GFS2_FS_LOCKING_DLM +extern const struct file_operations gfs2_file_fops; +extern const struct file_operations gfs2_dir_fops; + +static inline int gfs2_localflocks(const struct gfs2_sbd *sdp) +{ + return sdp->sd_args.ar_localflocks; +} +#else /* Single node only */ +#define gfs2_file_fops gfs2_file_fops_nolock +#define gfs2_dir_fops gfs2_dir_fops_nolock + +static inline int gfs2_localflocks(const struct gfs2_sbd *sdp) +{ + return 1; +} +#endif /* CONFIG_GFS2_FS_LOCKING_DLM */ + +#endif /* __INODE_DOT_H__ */ + diff --git a/fs/gfs2/lock_dlm.c b/fs/gfs2/lock_dlm.c new file mode 100644 index 000000000..cef4bdee9 --- /dev/null +++ b/fs/gfs2/lock_dlm.c @@ -0,0 +1,1459 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright 2004-2011 Red Hat, Inc. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "incore.h" +#include "util.h" +#include "sys.h" +#include "trace_gfs2.h" + +/** + * gfs2_update_stats - Update time based stats + * @s: The stats to update (local or global) + * @index: The index inside @s + * @sample: New data to include + */ +static inline void gfs2_update_stats(struct gfs2_lkstats *s, unsigned index, + s64 sample) +{ + /* + * @delta is the difference between the current rtt sample and the + * running average srtt. We add 1/8 of that to the srtt in order to + * update the current srtt estimate. The variance estimate is a bit + * more complicated. We subtract the current variance estimate from + * the abs value of the @delta and add 1/4 of that to the running + * total. That's equivalent to 3/4 of the current variance + * estimate plus 1/4 of the abs of @delta. + * + * Note that the index points at the array entry containing the + * smoothed mean value, and the variance is always in the following + * entry + * + * Reference: TCP/IP Illustrated, vol 2, p. 831,832 + * All times are in units of integer nanoseconds. Unlike the TCP/IP + * case, they are not scaled fixed point. + */ + + s64 delta = sample - s->stats[index]; + s->stats[index] += (delta >> 3); + index++; + s->stats[index] += (s64)(abs(delta) - s->stats[index]) >> 2; +} + +/** + * gfs2_update_reply_times - Update locking statistics + * @gl: The glock to update + * @blocking: The operation may have been blocking + * + * This assumes that gl->gl_dstamp has been set earlier. + * + * The rtt (lock round trip time) is an estimate of the time + * taken to perform a dlm lock request. We update it on each + * reply from the dlm. + * + * The blocking flag is set on the glock for all dlm requests + * which may potentially block due to lock requests from other nodes. + * DLM requests where the current lock state is exclusive, the + * requested state is null (or unlocked) or where the TRY or + * TRY_1CB flags are set are classified as non-blocking. All + * other DLM requests are counted as (potentially) blocking. + */ +static inline void gfs2_update_reply_times(struct gfs2_glock *gl, + bool blocking) +{ + struct gfs2_pcpu_lkstats *lks; + const unsigned gltype = glock_type(gl); + unsigned index = blocking ? GFS2_LKS_SRTTB : GFS2_LKS_SRTT; + s64 rtt; + + preempt_disable(); + rtt = ktime_to_ns(ktime_sub(ktime_get_real(), gl->gl_dstamp)); + lks = this_cpu_ptr(glock_sbd(gl)->sd_lkstats); + gfs2_update_stats(&gl->gl_stats, index, rtt); /* Local */ + gfs2_update_stats(&lks->lkstats[gltype], index, rtt); /* Global */ + preempt_enable(); + + trace_gfs2_glock_lock_time(gl, rtt); +} + +/** + * gfs2_update_request_times - Update locking statistics + * @gl: The glock to update + * + * The irt (lock inter-request times) measures the average time + * between requests to the dlm. It is updated immediately before + * each dlm call. + */ + +static inline void gfs2_update_request_times(struct gfs2_glock *gl) +{ + struct gfs2_pcpu_lkstats *lks; + const unsigned gltype = glock_type(gl); + ktime_t dstamp; + s64 irt; + + preempt_disable(); + dstamp = gl->gl_dstamp; + gl->gl_dstamp = ktime_get_real(); + irt = ktime_to_ns(ktime_sub(gl->gl_dstamp, dstamp)); + lks = this_cpu_ptr(glock_sbd(gl)->sd_lkstats); + gfs2_update_stats(&gl->gl_stats, GFS2_LKS_SIRT, irt); /* Local */ + gfs2_update_stats(&lks->lkstats[gltype], GFS2_LKS_SIRT, irt); /* Global */ + preempt_enable(); +} + +static void gdlm_ast(void *arg) +{ + struct gfs2_glock *gl = arg; + bool blocking; + unsigned ret; + + blocking = test_bit(GLF_BLOCKING, &gl->gl_flags); + gfs2_update_reply_times(gl, blocking); + clear_bit(GLF_BLOCKING, &gl->gl_flags); + + /* If the glock is dead, we only react to a dlm_unlock() reply. */ + if (lockref_is_dead(&gl->gl_lockref) && + gl->gl_lksb.sb_status != -DLM_EUNLOCK) + return; + + BUG_ON(gl->gl_lksb.sb_flags & DLM_SBF_DEMOTED); + + if ((gl->gl_lksb.sb_flags & DLM_SBF_VALNOTVALID) && gl->gl_lksb.sb_lvbptr) + memset(gl->gl_lksb.sb_lvbptr, 0, GDLM_LVB_SIZE); + + switch (gl->gl_lksb.sb_status) { + case -DLM_EUNLOCK: /* Unlocked, so glock can be freed */ + gfs2_glock_free(gl); + return; + case -DLM_ECANCEL: /* Cancel while getting lock */ + ret = LM_OUT_CANCELED; + goto out; + case -EAGAIN: /* Try lock fails */ + ret = LM_OUT_TRY_AGAIN; + goto out; + case -EDEADLK: /* Deadlock detected */ + ret = LM_OUT_DEADLOCK; + goto out; + case -ETIMEDOUT: /* Canceled due to timeout */ + ret = LM_OUT_ERROR; + goto out; + case 0: /* Success */ + break; + default: /* Something unexpected */ + BUG(); + } + + ret = gl->gl_req; + + /* + * The GLF_INITIAL flag is initially set for new glocks. Upon the + * first successful new (non-conversion) request, we clear this flag to + * indicate that a DLM lock exists and that gl->gl_lksb.sb_lkid is the + * identifier to use for identifying it. + * + * Any failed initial requests do not create a DLM lock, so we ignore + * the gl->gl_lksb.sb_lkid values that come with such requests. + */ + + clear_bit(GLF_INITIAL, &gl->gl_flags); + gfs2_glock_complete(gl, ret); + return; +out: + if (test_bit(GLF_INITIAL, &gl->gl_flags)) + gl->gl_lksb.sb_lkid = 0; + gfs2_glock_complete(gl, ret); +} + +static void gdlm_bast(void *arg, int mode) +{ + struct gfs2_glock *gl = arg; + + switch (mode) { + case DLM_LOCK_EX: + gfs2_glock_cb(gl, LM_ST_UNLOCKED); + break; + case DLM_LOCK_CW: + gfs2_glock_cb(gl, LM_ST_DEFERRED); + break; + case DLM_LOCK_PR: + gfs2_glock_cb(gl, LM_ST_SHARED); + break; + default: + fs_err(glock_sbd(gl), "unknown bast mode %d\n", mode); + BUG(); + } +} + +/* convert gfs lock-state to dlm lock-mode */ + +static int make_mode(struct gfs2_sbd *sdp, const unsigned int lmstate) +{ + switch (lmstate) { + case LM_ST_UNLOCKED: + return DLM_LOCK_NL; + case LM_ST_EXCLUSIVE: + return DLM_LOCK_EX; + case LM_ST_DEFERRED: + return DLM_LOCK_CW; + case LM_ST_SHARED: + return DLM_LOCK_PR; + } + fs_err(sdp, "unknown LM state %d\n", lmstate); + BUG(); + return -1; +} + +/* Taken from fs/dlm/lock.c. */ + +static bool middle_conversion(int cur, int req) +{ + return (cur == DLM_LOCK_PR && req == DLM_LOCK_CW) || + (cur == DLM_LOCK_CW && req == DLM_LOCK_PR); +} + +static bool down_conversion(int cur, int req) +{ + return !middle_conversion(cur, req) && req < cur; +} + +static u32 make_flags(struct gfs2_glock *gl, const unsigned int gfs_flags, + const int req, bool blocking) +{ + u32 lkf = 0; + + if (gl->gl_lksb.sb_lvbptr) + lkf |= DLM_LKF_VALBLK; + + if (gfs_flags & LM_FLAG_TRY) + lkf |= DLM_LKF_NOQUEUE; + + if (gfs_flags & LM_FLAG_TRY_1CB) { + lkf |= DLM_LKF_NOQUEUE; + lkf |= DLM_LKF_NOQUEUEBAST; + } + + if (!test_bit(GLF_INITIAL, &gl->gl_flags)) { + lkf |= DLM_LKF_CONVERT; + + /* + * The DLM_LKF_QUECVT flag needs to be set for "first come, + * first served" semantics, but it must only be set for + * "upward" lock conversions or else DLM will reject the + * request as invalid. + */ + if (blocking) + lkf |= DLM_LKF_QUECVT; + } + + return lkf; +} + +static void gfs2_reverse_hex(char *c, u64 value) +{ + *c = '0'; + while (value) { + *c-- = hex_asc[value & 0x0f]; + value >>= 4; + } +} + +static int gdlm_lock(struct gfs2_glock *gl, unsigned int req_state, + unsigned int flags) +{ + struct lm_lockstruct *ls = &glock_sbd(gl)->sd_lockstruct; + bool blocking; + int cur, req; + u32 lkf; + char strname[GDLM_STRNAME_BYTES] = ""; + int error; + + gl->gl_req = req_state; + cur = make_mode(glock_sbd(gl), gl->gl_state); + req = make_mode(glock_sbd(gl), req_state); + blocking = !down_conversion(cur, req) && + !(flags & (LM_FLAG_TRY|LM_FLAG_TRY_1CB)); + lkf = make_flags(gl, flags, req, blocking); + if (blocking) + set_bit(GLF_BLOCKING, &gl->gl_flags); + gfs2_glstats_inc(gl, GFS2_LKS_DCOUNT); + gfs2_sbstats_inc(gl, GFS2_LKS_DCOUNT); + if (test_bit(GLF_INITIAL, &gl->gl_flags)) { + memset(strname, ' ', GDLM_STRNAME_BYTES - 1); + strname[GDLM_STRNAME_BYTES - 1] = '\0'; + gfs2_reverse_hex(strname + 7, glock_type(gl)); + gfs2_reverse_hex(strname + 23, glock_number(gl)); + gl->gl_dstamp = ktime_get_real(); + } else { + gfs2_update_request_times(gl); + } + /* + * Submit the actual lock request. + */ + +again: + down_read(&ls->ls_sem); + error = -ENODEV; + if (likely(ls->ls_dlm != NULL)) { + error = dlm_lock(ls->ls_dlm, req, &gl->gl_lksb, lkf, strname, + GDLM_STRNAME_BYTES - 1, 0, gdlm_ast, gl, gdlm_bast); + } + up_read(&ls->ls_sem); + if (error == -EBUSY) { + msleep(20); + goto again; + } + return error; +} + +static void gdlm_put_lock(struct gfs2_glock *gl) +{ + struct gfs2_sbd *sdp = glock_sbd(gl); + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + uint32_t flags = 0; + int error; + + BUG_ON(!lockref_is_dead(&gl->gl_lockref)); + + if (test_bit(GLF_INITIAL, &gl->gl_flags)) { + gfs2_glock_free(gl); + return; + } + + gfs2_glstats_inc(gl, GFS2_LKS_DCOUNT); + gfs2_sbstats_inc(gl, GFS2_LKS_DCOUNT); + gfs2_update_request_times(gl); + + /* + * When the lockspace is released, all remaining glocks will be + * unlocked automatically. This is more efficient than unlocking them + * individually, but when the lock is held in DLM_LOCK_EX or + * DLM_LOCK_PW mode, the lock value block (LVB) would be lost. + */ + + if (test_bit(SDF_KILL, &sdp->sd_flags) && + (!gl->gl_lksb.sb_lvbptr || gl->gl_state != LM_ST_EXCLUSIVE)) { + gfs2_glock_free_later(gl); + return; + } + + if (gl->gl_lksb.sb_lvbptr) + flags |= DLM_LKF_VALBLK; + +again: + down_read(&ls->ls_sem); + error = -ENODEV; + if (likely(ls->ls_dlm != NULL)) { + error = dlm_unlock(ls->ls_dlm, gl->gl_lksb.sb_lkid, flags, + NULL, gl); + } + up_read(&ls->ls_sem); + if (error == -EBUSY) { + msleep(20); + goto again; + } + + if (error == -ENODEV) { + gfs2_glock_free(gl); + return; + } + + if (error) { + fs_err(sdp, "gdlm_unlock %x,%llx err=%d\n", + glock_type(gl), + (unsigned long long) glock_number(gl), error); + } +} + +static void gdlm_cancel(struct gfs2_glock *gl) +{ + struct lm_lockstruct *ls = &glock_sbd(gl)->sd_lockstruct; + + down_read(&ls->ls_sem); + if (likely(ls->ls_dlm != NULL)) { + dlm_unlock(ls->ls_dlm, gl->gl_lksb.sb_lkid, DLM_LKF_CANCEL, NULL, gl); + } + up_read(&ls->ls_sem); +} + +/* + * dlm/gfs2 recovery coordination using dlm_recover callbacks + * + * 1. dlm_controld sees lockspace members change + * 2. dlm_controld blocks dlm-kernel locking activity + * 3. dlm_controld within dlm-kernel notifies gfs2 (recover_prep) + * 4. dlm_controld starts and finishes its own user level recovery + * 5. dlm_controld starts dlm-kernel dlm_recoverd to do kernel recovery + * 6. dlm_recoverd notifies gfs2 of failed nodes (recover_slot) + * 7. dlm_recoverd does its own lock recovery + * 8. dlm_recoverd unblocks dlm-kernel locking activity + * 9. dlm_recoverd notifies gfs2 when done (recover_done with new generation) + * 10. gfs2_control updates control_lock lvb with new generation and jid bits + * 11. gfs2_control enqueues journals for gfs2_recover to recover (maybe none) + * 12. gfs2_recover dequeues and recovers journals of failed nodes + * 13. gfs2_recover provides recovery results to gfs2_control (recovery_result) + * 14. gfs2_control updates control_lock lvb jid bits for recovered journals + * 15. gfs2_control unblocks normal locking when all journals are recovered + * + * - failures during recovery + * + * recover_prep() may set BLOCK_LOCKS (step 3) again before gfs2_control + * clears BLOCK_LOCKS (step 15), e.g. another node fails while still + * recovering for a prior failure. gfs2_control needs a way to detect + * this so it can leave BLOCK_LOCKS set in step 15. This is managed using + * the recover_block and recover_start values. + * + * recover_done() provides a new lockspace generation number each time it + * is called (step 9). This generation number is saved as recover_start. + * When recover_prep() is called, it sets BLOCK_LOCKS and sets + * recover_block = recover_start. So, while recover_block is equal to + * recover_start, BLOCK_LOCKS should remain set. (recover_spin must + * be held around the BLOCK_LOCKS/recover_block/recover_start logic.) + * + * - more specific gfs2 steps in sequence above + * + * 3. recover_prep sets BLOCK_LOCKS and sets recover_block = recover_start + * 6. recover_slot records any failed jids (maybe none) + * 9. recover_done sets recover_start = new generation number + * 10. gfs2_control sets control_lock lvb = new gen + bits for failed jids + * 12. gfs2_recover does journal recoveries for failed jids identified above + * 14. gfs2_control clears control_lock lvb bits for recovered jids + * 15. gfs2_control checks if recover_block == recover_start (step 3 occured + * again) then do nothing, otherwise if recover_start > recover_block + * then clear BLOCK_LOCKS. + * + * - parallel recovery steps across all nodes + * + * All nodes attempt to update the control_lock lvb with the new generation + * number and jid bits, but only the first to get the control_lock EX will + * do so; others will see that it's already done (lvb already contains new + * generation number.) + * + * . All nodes get the same recover_prep/recover_slot/recover_done callbacks + * . All nodes attempt to set control_lock lvb gen + bits for the new gen + * . One node gets control_lock first and writes the lvb, others see it's done + * . All nodes attempt to recover jids for which they see control_lock bits set + * . One node succeeds for a jid, and that one clears the jid bit in the lvb + * . All nodes will eventually see all lvb bits clear and unblock locks + * + * - is there a problem with clearing an lvb bit that should be set + * and missing a journal recovery? + * + * 1. jid fails + * 2. lvb bit set for step 1 + * 3. jid recovered for step 1 + * 4. jid taken again (new mount) + * 5. jid fails (for step 4) + * 6. lvb bit set for step 5 (will already be set) + * 7. lvb bit cleared for step 3 + * + * This is not a problem because the failure in step 5 does not + * require recovery, because the mount in step 4 could not have + * progressed far enough to unblock locks and access the fs. The + * control_mount() function waits for all recoveries to be complete + * for the latest lockspace generation before ever unblocking locks + * and returning. The mount in step 4 waits until the recovery in + * step 1 is done. + * + * - special case of first mounter: first node to mount the fs + * + * The first node to mount a gfs2 fs needs to check all the journals + * and recover any that need recovery before other nodes are allowed + * to mount the fs. (Others may begin mounting, but they must wait + * for the first mounter to be done before taking locks on the fs + * or accessing the fs.) This has two parts: + * + * 1. The mounted_lock tells a node it's the first to mount the fs. + * Each node holds the mounted_lock in PR while it's mounted. + * Each node tries to acquire the mounted_lock in EX when it mounts. + * If a node is granted the mounted_lock EX it means there are no + * other mounted nodes (no PR locks exist), and it is the first mounter. + * The mounted_lock is demoted to PR when first recovery is done, so + * others will fail to get an EX lock, but will get a PR lock. + * + * 2. The control_lock blocks others in control_mount() while the first + * mounter is doing first mount recovery of all journals. + * A mounting node needs to acquire control_lock in EX mode before + * it can proceed. The first mounter holds control_lock in EX while doing + * the first mount recovery, blocking mounts from other nodes, then demotes + * control_lock to NL when it's done (others_may_mount/first_done), + * allowing other nodes to continue mounting. + * + * first mounter: + * control_lock EX/NOQUEUE success + * mounted_lock EX/NOQUEUE success (no other PR, so no other mounters) + * set first=1 + * do first mounter recovery + * mounted_lock EX->PR + * control_lock EX->NL, write lvb generation + * + * other mounter: + * control_lock EX/NOQUEUE success (if fail -EAGAIN, retry) + * mounted_lock EX/NOQUEUE fail -EAGAIN (expected due to other mounters PR) + * mounted_lock PR/NOQUEUE success + * read lvb generation + * control_lock EX->NL + * set first=0 + * + * - mount during recovery + * + * If a node mounts while others are doing recovery (not first mounter), + * the mounting node will get its initial recover_done() callback without + * having seen any previous failures/callbacks. + * + * It must wait for all recoveries preceding its mount to be finished + * before it unblocks locks. It does this by repeating the "other mounter" + * steps above until the lvb generation number is >= its mount generation + * number (from initial recover_done) and all lvb bits are clear. + * + * - control_lock lvb format + * + * 4 bytes generation number: the latest dlm lockspace generation number + * from recover_done callback. Indicates the jid bitmap has been updated + * to reflect all slot failures through that generation. + * 4 bytes unused. + * GDLM_LVB_SIZE-8 bytes of jid bit map. If bit N is set, it indicates + * that jid N needs recovery. + */ + +#define JID_BITMAP_OFFSET 8 /* 4 byte generation number + 4 byte unused */ + +static void control_lvb_read(struct lm_lockstruct *ls, uint32_t *lvb_gen, + char *lvb_bits) +{ + __le32 gen; + memcpy(lvb_bits, ls->ls_control_lvb, GDLM_LVB_SIZE); + memcpy(&gen, lvb_bits, sizeof(__le32)); + *lvb_gen = le32_to_cpu(gen); +} + +static void control_lvb_write(struct lm_lockstruct *ls, uint32_t lvb_gen, + char *lvb_bits) +{ + __le32 gen; + memcpy(ls->ls_control_lvb, lvb_bits, GDLM_LVB_SIZE); + gen = cpu_to_le32(lvb_gen); + memcpy(ls->ls_control_lvb, &gen, sizeof(__le32)); +} + +static int all_jid_bits_clear(char *lvb) +{ + return !memchr_inv(lvb + JID_BITMAP_OFFSET, 0, + GDLM_LVB_SIZE - JID_BITMAP_OFFSET); +} + +static void sync_wait_cb(void *arg) +{ + struct lm_lockstruct *ls = arg; + complete(&ls->ls_sync_wait); +} + +static int sync_unlock(struct gfs2_sbd *sdp, struct dlm_lksb *lksb, char *name) +{ + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + int error; + + down_read(&ls->ls_sem); + error = -ENODEV; + if (likely(ls->ls_dlm != NULL)) + error = dlm_unlock(ls->ls_dlm, lksb->sb_lkid, 0, lksb, ls); + up_read(&ls->ls_sem); + if (error) { + fs_err(sdp, "%s lkid %x error %d\n", + name, lksb->sb_lkid, error); + return error; + } + + wait_for_completion(&ls->ls_sync_wait); + + if (lksb->sb_status != -DLM_EUNLOCK) { + fs_err(sdp, "%s lkid %x status %d\n", + name, lksb->sb_lkid, lksb->sb_status); + return -1; + } + return 0; +} + +static int sync_lock(struct gfs2_sbd *sdp, int mode, uint32_t flags, + unsigned int num, struct dlm_lksb *lksb, char *name) +{ + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + char strname[GDLM_STRNAME_BYTES]; + int error, status; + + memset(strname, 0, GDLM_STRNAME_BYTES); + snprintf(strname, GDLM_STRNAME_BYTES, "%8x%16x", LM_TYPE_NONDISK, num); + + down_read(&ls->ls_sem); + error = -ENODEV; + if (likely(ls->ls_dlm != NULL)) { + error = dlm_lock(ls->ls_dlm, mode, lksb, flags, + strname, GDLM_STRNAME_BYTES - 1, + 0, sync_wait_cb, ls, NULL); + } + up_read(&ls->ls_sem); + if (error) { + fs_err(sdp, "%s lkid %x flags %x mode %d error %d\n", + name, lksb->sb_lkid, flags, mode, error); + return error; + } + + wait_for_completion(&ls->ls_sync_wait); + + status = lksb->sb_status; + + if (status && status != -EAGAIN) { + fs_err(sdp, "%s lkid %x flags %x mode %d status %d\n", + name, lksb->sb_lkid, flags, mode, status); + } + + return status; +} + +static int mounted_unlock(struct gfs2_sbd *sdp) +{ + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + return sync_unlock(sdp, &ls->ls_mounted_lksb, "mounted_lock"); +} + +static int mounted_lock(struct gfs2_sbd *sdp, int mode, uint32_t flags) +{ + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + return sync_lock(sdp, mode, flags, GFS2_MOUNTED_LOCK, + &ls->ls_mounted_lksb, "mounted_lock"); +} + +static int control_unlock(struct gfs2_sbd *sdp) +{ + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + return sync_unlock(sdp, &ls->ls_control_lksb, "control_lock"); +} + +static int control_lock(struct gfs2_sbd *sdp, int mode, uint32_t flags) +{ + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + return sync_lock(sdp, mode, flags, GFS2_CONTROL_LOCK, + &ls->ls_control_lksb, "control_lock"); +} + +static void gfs2_control_func(struct work_struct *work) +{ + struct gfs2_sbd *sdp = container_of(work, struct gfs2_sbd, sd_control_work.work); + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + uint32_t block_gen, start_gen, lvb_gen, flags; + int recover_set = 0; + int write_lvb = 0; + int recover_size; + int i, error; + + spin_lock(&ls->ls_recover_spin); + /* + * No MOUNT_DONE means we're still mounting; control_mount() + * will set this flag, after which this thread will take over + * all further clearing of BLOCK_LOCKS. + * + * FIRST_MOUNT means this node is doing first mounter recovery, + * for which recovery control is handled by + * control_mount()/control_first_done(), not this thread. + */ + if (!test_bit(DFL_MOUNT_DONE, &ls->ls_recover_flags) || + test_bit(DFL_FIRST_MOUNT, &ls->ls_recover_flags)) { + spin_unlock(&ls->ls_recover_spin); + return; + } + block_gen = ls->ls_recover_block; + start_gen = ls->ls_recover_start; + spin_unlock(&ls->ls_recover_spin); + + /* + * Equal block_gen and start_gen implies we are between + * recover_prep and recover_done callbacks, which means + * dlm recovery is in progress and dlm locking is blocked. + * There's no point trying to do any work until recover_done. + */ + + if (block_gen == start_gen) + return; + + /* + * Propagate recover_submit[] and recover_result[] to lvb: + * dlm_recoverd adds to recover_submit[] jids needing recovery + * gfs2_recover adds to recover_result[] journal recovery results + * + * set lvb bit for jids in recover_submit[] if the lvb has not + * yet been updated for the generation of the failure + * + * clear lvb bit for jids in recover_result[] if the result of + * the journal recovery is SUCCESS + */ + + error = control_lock(sdp, DLM_LOCK_EX, DLM_LKF_CONVERT|DLM_LKF_VALBLK); + if (error) { + fs_err(sdp, "control lock EX error %d\n", error); + return; + } + + control_lvb_read(ls, &lvb_gen, ls->ls_lvb_bits); + + spin_lock(&ls->ls_recover_spin); + if (block_gen != ls->ls_recover_block || + start_gen != ls->ls_recover_start) { + fs_info(sdp, "recover generation %u block1 %u %u\n", + start_gen, block_gen, ls->ls_recover_block); + spin_unlock(&ls->ls_recover_spin); + control_lock(sdp, DLM_LOCK_NL, DLM_LKF_CONVERT); + return; + } + + recover_size = ls->ls_recover_size; + + if (lvb_gen <= start_gen) { + /* + * Clear lvb bits for jids we've successfully recovered. + * Because all nodes attempt to recover failed journals, + * a journal can be recovered multiple times successfully + * in succession. Only the first will really do recovery, + * the others find it clean, but still report a successful + * recovery. So, another node may have already recovered + * the jid and cleared the lvb bit for it. + */ + for (i = 0; i < recover_size; i++) { + if (ls->ls_recover_result[i] != LM_RD_SUCCESS) + continue; + + ls->ls_recover_result[i] = 0; + + if (!test_bit_le(i, ls->ls_lvb_bits + JID_BITMAP_OFFSET)) + continue; + + __clear_bit_le(i, ls->ls_lvb_bits + JID_BITMAP_OFFSET); + write_lvb = 1; + } + } + + if (lvb_gen == start_gen) { + /* + * Failed slots before start_gen are already set in lvb. + */ + for (i = 0; i < recover_size; i++) { + if (!ls->ls_recover_submit[i]) + continue; + if (ls->ls_recover_submit[i] < lvb_gen) + ls->ls_recover_submit[i] = 0; + } + } else if (lvb_gen < start_gen) { + /* + * Failed slots before start_gen are not yet set in lvb. + */ + for (i = 0; i < recover_size; i++) { + if (!ls->ls_recover_submit[i]) + continue; + if (ls->ls_recover_submit[i] < start_gen) { + ls->ls_recover_submit[i] = 0; + __set_bit_le(i, ls->ls_lvb_bits + JID_BITMAP_OFFSET); + } + } + /* even if there are no bits to set, we need to write the + latest generation to the lvb */ + write_lvb = 1; + } else { + /* + * we should be getting a recover_done() for lvb_gen soon + */ + } + spin_unlock(&ls->ls_recover_spin); + + if (write_lvb) { + control_lvb_write(ls, start_gen, ls->ls_lvb_bits); + flags = DLM_LKF_CONVERT | DLM_LKF_VALBLK; + } else { + flags = DLM_LKF_CONVERT; + } + + error = control_lock(sdp, DLM_LOCK_NL, flags); + if (error) { + fs_err(sdp, "control lock NL error %d\n", error); + return; + } + + /* + * Everyone will see jid bits set in the lvb, run gfs2_recover_set(), + * and clear a jid bit in the lvb if the recovery is a success. + * Eventually all journals will be recovered, all jid bits will + * be cleared in the lvb, and everyone will clear BLOCK_LOCKS. + */ + + for (i = 0; i < recover_size; i++) { + if (test_bit_le(i, ls->ls_lvb_bits + JID_BITMAP_OFFSET)) { + fs_info(sdp, "recover generation %u jid %d\n", + start_gen, i); + gfs2_recover_set(sdp, i); + recover_set++; + } + } + if (recover_set) + return; + + /* + * No more jid bits set in lvb, all recovery is done, unblock locks + * (unless a new recover_prep callback has occured blocking locks + * again while working above) + */ + + spin_lock(&ls->ls_recover_spin); + if (ls->ls_recover_block == block_gen && + ls->ls_recover_start == start_gen) { + clear_bit(DFL_BLOCK_LOCKS, &ls->ls_recover_flags); + spin_unlock(&ls->ls_recover_spin); + fs_info(sdp, "recover generation %u done\n", start_gen); + gfs2_glock_thaw(sdp); + } else { + fs_info(sdp, "recover generation %u block2 %u %u\n", + start_gen, block_gen, ls->ls_recover_block); + spin_unlock(&ls->ls_recover_spin); + } +} + +static int control_mount(struct gfs2_sbd *sdp) +{ + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + uint32_t start_gen, block_gen, mount_gen, lvb_gen; + int mounted_mode; + int retries = 0; + int error; + + memset(&ls->ls_mounted_lksb, 0, sizeof(struct dlm_lksb)); + memset(&ls->ls_control_lksb, 0, sizeof(struct dlm_lksb)); + memset(&ls->ls_control_lvb, 0, GDLM_LVB_SIZE); + ls->ls_control_lksb.sb_lvbptr = ls->ls_control_lvb; + init_completion(&ls->ls_sync_wait); + + set_bit(DFL_BLOCK_LOCKS, &ls->ls_recover_flags); + + error = control_lock(sdp, DLM_LOCK_NL, DLM_LKF_VALBLK); + if (error) { + fs_err(sdp, "control_mount control_lock NL error %d\n", error); + return error; + } + + error = mounted_lock(sdp, DLM_LOCK_NL, 0); + if (error) { + fs_err(sdp, "control_mount mounted_lock NL error %d\n", error); + control_unlock(sdp); + return error; + } + mounted_mode = DLM_LOCK_NL; + +restart: + if (retries++ && signal_pending(current)) { + error = -EINTR; + goto fail; + } + + /* + * We always start with both locks in NL. control_lock is + * demoted to NL below so we don't need to do it here. + */ + + if (mounted_mode != DLM_LOCK_NL) { + error = mounted_lock(sdp, DLM_LOCK_NL, DLM_LKF_CONVERT); + if (error) + goto fail; + mounted_mode = DLM_LOCK_NL; + } + + /* + * Other nodes need to do some work in dlm recovery and gfs2_control + * before the recover_done and control_lock will be ready for us below. + * A delay here is not required but often avoids having to retry. + */ + + msleep_interruptible(500); + + /* + * Acquire control_lock in EX and mounted_lock in either EX or PR. + * control_lock lvb keeps track of any pending journal recoveries. + * mounted_lock indicates if any other nodes have the fs mounted. + */ + + error = control_lock(sdp, DLM_LOCK_EX, DLM_LKF_CONVERT|DLM_LKF_NOQUEUE|DLM_LKF_VALBLK); + if (error == -EAGAIN) { + goto restart; + } else if (error) { + fs_err(sdp, "control_mount control_lock EX error %d\n", error); + goto fail; + } + + /** + * If we're a spectator, we don't want to take the lock in EX because + * we cannot do the first-mount responsibility it implies: recovery. + */ + if (sdp->sd_args.ar_spectator) + goto locks_done; + + error = mounted_lock(sdp, DLM_LOCK_EX, DLM_LKF_CONVERT|DLM_LKF_NOQUEUE); + if (!error) { + mounted_mode = DLM_LOCK_EX; + goto locks_done; + } else if (error != -EAGAIN) { + fs_err(sdp, "control_mount mounted_lock EX error %d\n", error); + goto fail; + } + + error = mounted_lock(sdp, DLM_LOCK_PR, DLM_LKF_CONVERT|DLM_LKF_NOQUEUE); + if (!error) { + mounted_mode = DLM_LOCK_PR; + goto locks_done; + } else { + /* not even -EAGAIN should happen here */ + fs_err(sdp, "control_mount mounted_lock PR error %d\n", error); + goto fail; + } + +locks_done: + /* + * If we got both locks above in EX, then we're the first mounter. + * If not, then we need to wait for the control_lock lvb to be + * updated by other mounted nodes to reflect our mount generation. + * + * In simple first mounter cases, first mounter will see zero lvb_gen, + * but in cases where all existing nodes leave/fail before mounting + * nodes finish control_mount, then all nodes will be mounting and + * lvb_gen will be non-zero. + */ + + control_lvb_read(ls, &lvb_gen, ls->ls_lvb_bits); + + if (lvb_gen == 0xFFFFFFFF) { + /* special value to force mount attempts to fail */ + fs_err(sdp, "control_mount control_lock disabled\n"); + error = -EINVAL; + goto fail; + } + + if (mounted_mode == DLM_LOCK_EX) { + /* first mounter, keep both EX while doing first recovery */ + spin_lock(&ls->ls_recover_spin); + clear_bit(DFL_BLOCK_LOCKS, &ls->ls_recover_flags); + set_bit(DFL_MOUNT_DONE, &ls->ls_recover_flags); + set_bit(DFL_FIRST_MOUNT, &ls->ls_recover_flags); + spin_unlock(&ls->ls_recover_spin); + fs_info(sdp, "first mounter control generation %u\n", lvb_gen); + return 0; + } + + error = control_lock(sdp, DLM_LOCK_NL, DLM_LKF_CONVERT); + if (error) + goto fail; + + /* + * We are not first mounter, now we need to wait for the control_lock + * lvb generation to be >= the generation from our first recover_done + * and all lvb bits to be clear (no pending journal recoveries.) + */ + + if (!all_jid_bits_clear(ls->ls_lvb_bits)) { + /* journals need recovery, wait until all are clear */ + fs_info(sdp, "control_mount wait for journal recovery\n"); + goto restart; + } + + spin_lock(&ls->ls_recover_spin); + block_gen = ls->ls_recover_block; + start_gen = ls->ls_recover_start; + mount_gen = ls->ls_recover_mount; + + if (lvb_gen < mount_gen) { + /* wait for mounted nodes to update control_lock lvb to our + generation, which might include new recovery bits set */ + if (sdp->sd_args.ar_spectator) { + fs_info(sdp, "Recovery is required. Waiting for a " + "non-spectator to mount.\n"); + spin_unlock(&ls->ls_recover_spin); + msleep_interruptible(1000); + } else { + fs_info(sdp, "control_mount wait1 block %u start %u " + "mount %u lvb %u flags %lx\n", block_gen, + start_gen, mount_gen, lvb_gen, + ls->ls_recover_flags); + spin_unlock(&ls->ls_recover_spin); + } + goto restart; + } + + if (lvb_gen != start_gen) { + /* wait for mounted nodes to update control_lock lvb to the + latest recovery generation */ + fs_info(sdp, "control_mount wait2 block %u start %u mount %u " + "lvb %u flags %lx\n", block_gen, start_gen, mount_gen, + lvb_gen, ls->ls_recover_flags); + spin_unlock(&ls->ls_recover_spin); + goto restart; + } + + if (block_gen == start_gen) { + /* dlm recovery in progress, wait for it to finish */ + fs_info(sdp, "control_mount wait3 block %u start %u mount %u " + "lvb %u flags %lx\n", block_gen, start_gen, mount_gen, + lvb_gen, ls->ls_recover_flags); + spin_unlock(&ls->ls_recover_spin); + goto restart; + } + + clear_bit(DFL_BLOCK_LOCKS, &ls->ls_recover_flags); + set_bit(DFL_MOUNT_DONE, &ls->ls_recover_flags); + memset(ls->ls_recover_submit, 0, ls->ls_recover_size*sizeof(uint32_t)); + memset(ls->ls_recover_result, 0, ls->ls_recover_size*sizeof(uint32_t)); + spin_unlock(&ls->ls_recover_spin); + return 0; + +fail: + mounted_unlock(sdp); + control_unlock(sdp); + return error; +} + +static int control_first_done(struct gfs2_sbd *sdp) +{ + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + uint32_t start_gen, block_gen; + int error; + +restart: + spin_lock(&ls->ls_recover_spin); + start_gen = ls->ls_recover_start; + block_gen = ls->ls_recover_block; + + if (test_bit(DFL_BLOCK_LOCKS, &ls->ls_recover_flags) || + !test_bit(DFL_MOUNT_DONE, &ls->ls_recover_flags) || + !test_bit(DFL_FIRST_MOUNT, &ls->ls_recover_flags)) { + /* sanity check, should not happen */ + fs_err(sdp, "control_first_done start %u block %u flags %lx\n", + start_gen, block_gen, ls->ls_recover_flags); + spin_unlock(&ls->ls_recover_spin); + control_unlock(sdp); + return -1; + } + + if (start_gen == block_gen) { + /* + * Wait for the end of a dlm recovery cycle to switch from + * first mounter recovery. We can ignore any recover_slot + * callbacks between the recover_prep and next recover_done + * because we are still the first mounter and any failed nodes + * have not fully mounted, so they don't need recovery. + */ + spin_unlock(&ls->ls_recover_spin); + fs_info(sdp, "control_first_done wait gen %u\n", start_gen); + + wait_on_bit(&ls->ls_recover_flags, DFL_DLM_RECOVERY, + TASK_UNINTERRUPTIBLE); + goto restart; + } + + clear_bit(DFL_FIRST_MOUNT, &ls->ls_recover_flags); + set_bit(DFL_FIRST_MOUNT_DONE, &ls->ls_recover_flags); + memset(ls->ls_recover_submit, 0, ls->ls_recover_size*sizeof(uint32_t)); + memset(ls->ls_recover_result, 0, ls->ls_recover_size*sizeof(uint32_t)); + spin_unlock(&ls->ls_recover_spin); + + memset(ls->ls_lvb_bits, 0, GDLM_LVB_SIZE); + control_lvb_write(ls, start_gen, ls->ls_lvb_bits); + + error = mounted_lock(sdp, DLM_LOCK_PR, DLM_LKF_CONVERT); + if (error) + fs_err(sdp, "control_first_done mounted PR error %d\n", error); + + error = control_lock(sdp, DLM_LOCK_NL, DLM_LKF_CONVERT|DLM_LKF_VALBLK); + if (error) + fs_err(sdp, "control_first_done control NL error %d\n", error); + + return error; +} + +/* + * Expand static jid arrays if necessary (by increments of RECOVER_SIZE_INC) + * to accommodate the largest slot number. (NB dlm slot numbers start at 1, + * gfs2 jids start at 0, so jid = slot - 1) + */ + +#define RECOVER_SIZE_INC 16 + +static int set_recover_size(struct gfs2_sbd *sdp, struct dlm_slot *slots, + int num_slots) +{ + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + uint32_t *submit = NULL; + uint32_t *result = NULL; + uint32_t old_size, new_size; + int i, max_jid; + + if (!ls->ls_lvb_bits) { + ls->ls_lvb_bits = kzalloc(GDLM_LVB_SIZE, GFP_NOFS); + if (!ls->ls_lvb_bits) + return -ENOMEM; + } + + max_jid = 0; + for (i = 0; i < num_slots; i++) { + if (max_jid < slots[i].slot - 1) + max_jid = slots[i].slot - 1; + } + + old_size = ls->ls_recover_size; + new_size = old_size; + while (new_size < max_jid + 1) + new_size += RECOVER_SIZE_INC; + if (new_size == old_size) + return 0; + + submit = kcalloc(new_size, sizeof(uint32_t), GFP_NOFS); + result = kcalloc(new_size, sizeof(uint32_t), GFP_NOFS); + if (!submit || !result) { + kfree(submit); + kfree(result); + return -ENOMEM; + } + + spin_lock(&ls->ls_recover_spin); + memcpy(submit, ls->ls_recover_submit, old_size * sizeof(uint32_t)); + memcpy(result, ls->ls_recover_result, old_size * sizeof(uint32_t)); + kfree(ls->ls_recover_submit); + kfree(ls->ls_recover_result); + ls->ls_recover_submit = submit; + ls->ls_recover_result = result; + ls->ls_recover_size = new_size; + spin_unlock(&ls->ls_recover_spin); + return 0; +} + +static void free_recover_size(struct lm_lockstruct *ls) +{ + kfree(ls->ls_lvb_bits); + kfree(ls->ls_recover_submit); + kfree(ls->ls_recover_result); + ls->ls_recover_submit = NULL; + ls->ls_recover_result = NULL; + ls->ls_recover_size = 0; + ls->ls_lvb_bits = NULL; +} + +/* dlm calls before it does lock recovery */ + +static void gdlm_recover_prep(void *arg) +{ + struct gfs2_sbd *sdp = arg; + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + + if (gfs2_withdrawn(sdp)) { + fs_err(sdp, "recover_prep ignored due to withdraw.\n"); + return; + } + spin_lock(&ls->ls_recover_spin); + ls->ls_recover_block = ls->ls_recover_start; + set_bit(DFL_DLM_RECOVERY, &ls->ls_recover_flags); + + if (!test_bit(DFL_MOUNT_DONE, &ls->ls_recover_flags) || + test_bit(DFL_FIRST_MOUNT, &ls->ls_recover_flags)) { + spin_unlock(&ls->ls_recover_spin); + return; + } + set_bit(DFL_BLOCK_LOCKS, &ls->ls_recover_flags); + spin_unlock(&ls->ls_recover_spin); +} + +/* dlm calls after recover_prep has been completed on all lockspace members; + identifies slot/jid of failed member */ + +static void gdlm_recover_slot(void *arg, struct dlm_slot *slot) +{ + struct gfs2_sbd *sdp = arg; + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + int jid = slot->slot - 1; + + if (gfs2_withdrawn(sdp)) { + fs_err(sdp, "recover_slot jid %d ignored due to withdraw.\n", + jid); + return; + } + spin_lock(&ls->ls_recover_spin); + if (ls->ls_recover_size < jid + 1) { + fs_err(sdp, "recover_slot jid %d gen %u short size %d\n", + jid, ls->ls_recover_block, ls->ls_recover_size); + spin_unlock(&ls->ls_recover_spin); + return; + } + + if (ls->ls_recover_submit[jid]) { + fs_info(sdp, "recover_slot jid %d gen %u prev %u\n", + jid, ls->ls_recover_block, ls->ls_recover_submit[jid]); + } + ls->ls_recover_submit[jid] = ls->ls_recover_block; + spin_unlock(&ls->ls_recover_spin); +} + +/* dlm calls after recover_slot and after it completes lock recovery */ + +static void gdlm_recover_done(void *arg, struct dlm_slot *slots, int num_slots, + int our_slot, uint32_t generation) +{ + struct gfs2_sbd *sdp = arg; + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + + if (gfs2_withdrawn(sdp)) { + fs_err(sdp, "recover_done ignored due to withdraw.\n"); + return; + } + /* ensure the ls jid arrays are large enough */ + set_recover_size(sdp, slots, num_slots); + + spin_lock(&ls->ls_recover_spin); + ls->ls_recover_start = generation; + + if (!ls->ls_recover_mount) { + ls->ls_recover_mount = generation; + ls->ls_jid = our_slot - 1; + } + + if (!test_bit(DFL_UNMOUNT, &ls->ls_recover_flags)) + queue_delayed_work(gfs2_control_wq, &sdp->sd_control_work, 0); + + clear_bit(DFL_DLM_RECOVERY, &ls->ls_recover_flags); + smp_mb__after_atomic(); + wake_up_bit(&ls->ls_recover_flags, DFL_DLM_RECOVERY); + spin_unlock(&ls->ls_recover_spin); +} + +/* gfs2_recover thread has a journal recovery result */ + +static void gdlm_recovery_result(struct gfs2_sbd *sdp, unsigned int jid, + unsigned int result) +{ + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + + if (gfs2_withdrawn(sdp)) { + fs_err(sdp, "recovery_result jid %d ignored due to withdraw.\n", + jid); + return; + } + if (test_bit(DFL_NO_DLM_OPS, &ls->ls_recover_flags)) + return; + + /* don't care about the recovery of own journal during mount */ + if (jid == ls->ls_jid) + return; + + spin_lock(&ls->ls_recover_spin); + if (test_bit(DFL_FIRST_MOUNT, &ls->ls_recover_flags)) { + spin_unlock(&ls->ls_recover_spin); + return; + } + if (ls->ls_recover_size < jid + 1) { + fs_err(sdp, "recovery_result jid %d short size %d\n", + jid, ls->ls_recover_size); + spin_unlock(&ls->ls_recover_spin); + return; + } + + fs_info(sdp, "recover jid %d result %s\n", jid, + result == LM_RD_GAVEUP ? "busy" : "success"); + + ls->ls_recover_result[jid] = result; + + /* GAVEUP means another node is recovering the journal; delay our + next attempt to recover it, to give the other node a chance to + finish before trying again */ + + if (!test_bit(DFL_UNMOUNT, &ls->ls_recover_flags)) + queue_delayed_work(gfs2_control_wq, &sdp->sd_control_work, + result == LM_RD_GAVEUP ? HZ : 0); + spin_unlock(&ls->ls_recover_spin); +} + +static const struct dlm_lockspace_ops gdlm_lockspace_ops = { + .recover_prep = gdlm_recover_prep, + .recover_slot = gdlm_recover_slot, + .recover_done = gdlm_recover_done, +}; + +static int gdlm_mount(struct gfs2_sbd *sdp, const char *table) +{ + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + char cluster[GFS2_LOCKNAME_LEN]; + const char *fsname; + uint32_t flags; + int error, ops_result; + + /* + * initialize everything + */ + + INIT_DELAYED_WORK(&sdp->sd_control_work, gfs2_control_func); + ls->ls_dlm = NULL; + spin_lock_init(&ls->ls_recover_spin); + ls->ls_recover_flags = 0; + ls->ls_recover_mount = 0; + ls->ls_recover_start = 0; + ls->ls_recover_block = 0; + ls->ls_recover_size = 0; + ls->ls_recover_submit = NULL; + ls->ls_recover_result = NULL; + ls->ls_lvb_bits = NULL; + + error = set_recover_size(sdp, NULL, 0); + if (error) + goto fail; + + /* + * prepare dlm_new_lockspace args + */ + + fsname = strchr(table, ':'); + if (!fsname) { + fs_info(sdp, "no fsname found\n"); + error = -EINVAL; + goto fail_free; + } + memset(cluster, 0, sizeof(cluster)); + memcpy(cluster, table, strlen(table) - strlen(fsname)); + fsname++; + + flags = DLM_LSFL_NEWEXCL; + + /* + * create/join lockspace + */ + + init_rwsem(&ls->ls_sem); + error = dlm_new_lockspace(fsname, cluster, flags, GDLM_LVB_SIZE, + &gdlm_lockspace_ops, sdp, &ops_result, + &ls->ls_dlm); + if (error) { + fs_err(sdp, "dlm_new_lockspace error %d\n", error); + goto fail_free; + } + + if (ops_result < 0) { + /* + * dlm does not support ops callbacks, + * old dlm_controld/gfs_controld are used, try without ops. + */ + fs_info(sdp, "dlm lockspace ops not used\n"); + free_recover_size(ls); + set_bit(DFL_NO_DLM_OPS, &ls->ls_recover_flags); + return 0; + } + + if (!test_bit(SDF_NOJOURNALID, &sdp->sd_flags)) { + fs_err(sdp, "dlm lockspace ops disallow jid preset\n"); + error = -EINVAL; + goto fail_release; + } + + /* + * control_mount() uses control_lock to determine first mounter, + * and for later mounts, waits for any recoveries to be cleared. + */ + + error = control_mount(sdp); + if (error) { + fs_err(sdp, "mount control error %d\n", error); + goto fail_release; + } + + ls->ls_first = !!test_bit(DFL_FIRST_MOUNT, &ls->ls_recover_flags); + clear_bit(SDF_NOJOURNALID, &sdp->sd_flags); + smp_mb__after_atomic(); + wake_up_bit(&sdp->sd_flags, SDF_NOJOURNALID); + return 0; + +fail_release: + dlm_release_lockspace(ls->ls_dlm, DLM_RELEASE_NORMAL); +fail_free: + free_recover_size(ls); +fail: + return error; +} + +static void gdlm_first_done(struct gfs2_sbd *sdp) +{ + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + int error; + + if (test_bit(DFL_NO_DLM_OPS, &ls->ls_recover_flags)) + return; + + error = control_first_done(sdp); + if (error) + fs_err(sdp, "mount first_done error %d\n", error); +} + +/* + * gdlm_unmount - release our lockspace + * @sdp: the superblock + * @clean: Indicates whether or not the remaining nodes in the cluster should + * perform recovery. Recovery is necessary when a node withdraws and + * its journal remains dirty. Recovery isn't necessary when a node + * cleanly unmounts a filesystem. + */ +static void gdlm_unmount(struct gfs2_sbd *sdp, bool clean) +{ + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + + if (test_bit(DFL_NO_DLM_OPS, &ls->ls_recover_flags)) + goto release; + + /* wait for gfs2_control_wq to be done with this mount */ + + spin_lock(&ls->ls_recover_spin); + set_bit(DFL_UNMOUNT, &ls->ls_recover_flags); + spin_unlock(&ls->ls_recover_spin); + flush_delayed_work(&sdp->sd_control_work); + + /* mounted_lock and control_lock will be purged in dlm recovery */ +release: + down_write(&ls->ls_sem); + if (ls->ls_dlm) { + dlm_release_lockspace(ls->ls_dlm, + clean ? DLM_RELEASE_NORMAL : + DLM_RELEASE_RECOVER); + ls->ls_dlm = NULL; + } + up_write(&ls->ls_sem); + + free_recover_size(ls); +} + +static const match_table_t dlm_tokens = { + { Opt_jid, "jid=%d"}, + { Opt_id, "id=%d"}, + { Opt_first, "first=%d"}, + { Opt_nodir, "nodir=%d"}, + { Opt_err, NULL }, +}; + +const struct lm_lockops gfs2_dlm_ops = { + .lm_proto_name = "lock_dlm", + .lm_mount = gdlm_mount, + .lm_first_done = gdlm_first_done, + .lm_recovery_result = gdlm_recovery_result, + .lm_unmount = gdlm_unmount, + .lm_put_lock = gdlm_put_lock, + .lm_lock = gdlm_lock, + .lm_cancel = gdlm_cancel, + .lm_tokens = &dlm_tokens, +}; + diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c new file mode 100644 index 000000000..78bba8cc1 --- /dev/null +++ b/fs/gfs2/log.c @@ -0,0 +1,1385 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2007 Red Hat, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gfs2.h" +#include "incore.h" +#include "bmap.h" +#include "glock.h" +#include "log.h" +#include "lops.h" +#include "meta_io.h" +#include "util.h" +#include "dir.h" +#include "trace_gfs2.h" +#include "trans.h" +#include "aops.h" + +static void gfs2_log_shutdown(struct gfs2_sbd *sdp); + +/** + * gfs2_struct2blk - compute stuff + * @sdp: the filesystem + * @nstruct: the number of structures + * + * Compute the number of log descriptor blocks needed to hold a certain number + * of structures of a certain size. + * + * Returns: the number of blocks needed (minimum is always 1) + */ + +unsigned int gfs2_struct2blk(struct gfs2_sbd *sdp, unsigned int nstruct) +{ + unsigned int blks; + unsigned int first, second; + + /* The initial struct gfs2_log_descriptor block */ + blks = 1; + first = sdp->sd_ldptrs; + + if (nstruct > first) { + /* Subsequent struct gfs2_meta_header blocks */ + second = sdp->sd_inptrs; + blks += DIV_ROUND_UP(nstruct - first, second); + } + + return blks; +} + +/** + * gfs2_remove_from_ail - Remove an entry from the ail lists, updating counters + * @bd: The gfs2_bufdata to remove + * + * The ail lock _must_ be held when calling this function + * + */ + +static void gfs2_remove_from_ail(struct gfs2_bufdata *bd) +{ + bd->bd_tr = NULL; + list_del_init(&bd->bd_ail_st_list); + list_del_init(&bd->bd_ail_gl_list); + atomic_dec(&bd->bd_gl->gl_ail_count); + brelse(bd->bd_bh); +} + +/** + * gfs2_ail1_start_one - Start I/O on a transaction + * @sdp: The superblock + * @wbc: The writeback control structure + * @tr: The transaction to start I/O on + * @plug: The block plug currently active + */ + +static int gfs2_ail1_start_one(struct gfs2_sbd *sdp, + struct writeback_control *wbc, + struct gfs2_trans *tr, struct blk_plug *plug) +__releases(&sdp->sd_ail_lock) +__acquires(&sdp->sd_ail_lock) +{ + struct gfs2_glock *gl = NULL; + struct address_space *mapping; + struct gfs2_bufdata *bd, *s; + struct buffer_head *bh; + int ret = 0; + + list_for_each_entry_safe_reverse(bd, s, &tr->tr_ail1_list, bd_ail_st_list) { + bh = bd->bd_bh; + + gfs2_assert(sdp, bd->bd_tr == tr); + + if (!buffer_busy(bh)) { + if (buffer_uptodate(bh)) { + list_move(&bd->bd_ail_st_list, + &tr->tr_ail2_list); + continue; + } + if (!cmpxchg(&sdp->sd_log_error, 0, -EIO)) + gfs2_io_error_bh(sdp, bh); + } + + if (gfs2_withdrawn(sdp)) { + gfs2_remove_from_ail(bd); + continue; + } + if (!buffer_dirty(bh)) + continue; + if (gl == bd->bd_gl) + continue; + gl = bd->bd_gl; + list_move(&bd->bd_ail_st_list, &tr->tr_ail1_list); + mapping = bh->b_folio->mapping; + if (!mapping) + continue; + spin_unlock(&sdp->sd_ail_lock); + BUG_ON(GFS2_SB(mapping->host) != sdp); + if (gfs2_is_jdata(GFS2_I(mapping->host))) + ret = gfs2_jdata_writeback(mapping, wbc); + else + ret = mapping->a_ops->writepages(mapping, wbc); + if (need_resched()) { + blk_finish_plug(plug); + cond_resched(); + blk_start_plug(plug); + } + spin_lock(&sdp->sd_ail_lock); + if (ret == -ENODATA) /* if a jdata write into a new hole */ + ret = 0; /* ignore it */ + mapping_set_error(mapping, ret); + if (ret || wbc->nr_to_write <= 0) + break; + return -EBUSY; + } + + return ret; +} + +static void dump_ail_list(struct gfs2_sbd *sdp) +{ + struct gfs2_trans *tr; + struct gfs2_bufdata *bd; + struct buffer_head *bh; + + list_for_each_entry_reverse(tr, &sdp->sd_ail1_list, tr_list) { + list_for_each_entry_reverse(bd, &tr->tr_ail1_list, + bd_ail_st_list) { + bh = bd->bd_bh; + fs_err(sdp, "bd %p: blk:0x%llx bh=%p ", bd, + (unsigned long long)bd->bd_blkno, bh); + if (!bh) { + fs_err(sdp, "\n"); + continue; + } + fs_err(sdp, "0x%llx up2:%d dirt:%d lkd:%d req:%d " + "map:%d new:%d ar:%d aw:%d delay:%d " + "io err:%d unwritten:%d dfr:%d pin:%d esc:%d\n", + (unsigned long long)bh->b_blocknr, + buffer_uptodate(bh), buffer_dirty(bh), + buffer_locked(bh), buffer_req(bh), + buffer_mapped(bh), buffer_new(bh), + buffer_async_read(bh), buffer_async_write(bh), + buffer_delay(bh), buffer_write_io_error(bh), + buffer_unwritten(bh), + buffer_defer_completion(bh), + buffer_pinned(bh), buffer_escaped(bh)); + } + } +} + +/** + * gfs2_ail1_flush - start writeback of some ail1 entries + * @sdp: The super block + * @wbc: The writeback control structure + * + * Writes back some ail1 entries, according to the limits in the + * writeback control structure + */ + +void gfs2_ail1_flush(struct gfs2_sbd *sdp, struct writeback_control *wbc) +{ + struct list_head *head = &sdp->sd_ail1_list; + struct gfs2_trans *tr; + struct blk_plug plug; + int ret; + unsigned long flush_start = jiffies; + + trace_gfs2_ail_flush(sdp, wbc, 1); + blk_start_plug(&plug); + spin_lock(&sdp->sd_ail_lock); +restart: + ret = 0; + if (time_after(jiffies, flush_start + (HZ * 600))) { + fs_err(sdp, "Error: In %s for ten minutes! t=%d\n", + __func__, current->journal_info ? 1 : 0); + dump_ail_list(sdp); + goto out; + } + list_for_each_entry_reverse(tr, head, tr_list) { + if (wbc->nr_to_write <= 0) + break; + ret = gfs2_ail1_start_one(sdp, wbc, tr, &plug); + if (ret) { + if (ret == -EBUSY) + goto restart; + break; + } + } +out: + spin_unlock(&sdp->sd_ail_lock); + blk_finish_plug(&plug); + if (ret) { + gfs2_lm(sdp, "gfs2_ail1_start_one returned: %d\n", ret); + gfs2_withdraw(sdp); + } + trace_gfs2_ail_flush(sdp, wbc, 0); +} + +/** + * gfs2_ail1_start - start writeback of all ail1 entries + * @sdp: The superblock + */ + +static void gfs2_ail1_start(struct gfs2_sbd *sdp) +{ + struct writeback_control wbc = { + .sync_mode = WB_SYNC_NONE, + .nr_to_write = LONG_MAX, + .range_start = 0, + .range_end = LLONG_MAX, + }; + + return gfs2_ail1_flush(sdp, &wbc); +} + +static void gfs2_log_update_flush_tail(struct gfs2_sbd *sdp) +{ + unsigned int new_flush_tail = sdp->sd_log_head; + struct gfs2_trans *tr; + + if (!list_empty(&sdp->sd_ail1_list)) { + tr = list_last_entry(&sdp->sd_ail1_list, + struct gfs2_trans, tr_list); + new_flush_tail = tr->tr_first; + } + sdp->sd_log_flush_tail = new_flush_tail; +} + +static void gfs2_log_update_head(struct gfs2_sbd *sdp) +{ + unsigned int new_head = sdp->sd_log_flush_head; + + if (sdp->sd_log_flush_tail == sdp->sd_log_head) + sdp->sd_log_flush_tail = new_head; + sdp->sd_log_head = new_head; +} + +/* + * gfs2_ail_empty_tr - empty one of the ail lists of a transaction + */ + +static void gfs2_ail_empty_tr(struct gfs2_sbd *sdp, struct gfs2_trans *tr, + struct list_head *head) +{ + struct gfs2_bufdata *bd; + + while (!list_empty(head)) { + bd = list_first_entry(head, struct gfs2_bufdata, + bd_ail_st_list); + gfs2_assert(sdp, bd->bd_tr == tr); + gfs2_remove_from_ail(bd); + } +} + +/** + * gfs2_ail1_empty_one - Check whether or not a trans in the AIL has been synced + * @sdp: the filesystem + * @tr: the transaction + * @max_revokes: If nonzero, issue revokes for the bd items for written buffers + * + * returns: the transaction's count of remaining active items + */ + +static int gfs2_ail1_empty_one(struct gfs2_sbd *sdp, struct gfs2_trans *tr, + int *max_revokes) +{ + struct gfs2_bufdata *bd, *s; + struct buffer_head *bh; + int active_count = 0; + + list_for_each_entry_safe_reverse(bd, s, &tr->tr_ail1_list, + bd_ail_st_list) { + bh = bd->bd_bh; + gfs2_assert(sdp, bd->bd_tr == tr); + /* + * If another process flagged an io error, e.g. writing to the + * journal, error all other bhs and move them off the ail1 to + * prevent a tight loop when unmount tries to flush ail1, + * regardless of whether they're still busy. If no outside + * errors were found and the buffer is busy, move to the next. + * If the ail buffer is not busy and caught an error, flag it + * for others. + */ + if (!sdp->sd_log_error && buffer_busy(bh)) { + active_count++; + continue; + } + if (!buffer_uptodate(bh) && + !cmpxchg(&sdp->sd_log_error, 0, -EIO)) + gfs2_io_error_bh(sdp, bh); + /* + * If we have space for revokes and the bd is no longer on any + * buf list, we can just add a revoke for it immediately and + * avoid having to put it on the ail2 list, where it would need + * to be revoked later. + */ + if (*max_revokes && list_empty(&bd->bd_list)) { + gfs2_add_revoke(sdp, bd); + (*max_revokes)--; + continue; + } + list_move(&bd->bd_ail_st_list, &tr->tr_ail2_list); + } + return active_count; +} + +/** + * gfs2_ail1_empty - Try to empty the ail1 lists + * @sdp: The superblock + * @max_revokes: If non-zero, add revokes where appropriate + * + * Tries to empty the ail1 lists, starting with the oldest first. + * Returns %true if the ail1 list is now empty. + */ + +static bool gfs2_ail1_empty(struct gfs2_sbd *sdp, int max_revokes) +{ + struct gfs2_trans *tr, *s; + int oldest_tr = 1; + bool empty; + + spin_lock(&sdp->sd_ail_lock); + list_for_each_entry_safe_reverse(tr, s, &sdp->sd_ail1_list, tr_list) { + if (!gfs2_ail1_empty_one(sdp, tr, &max_revokes) && oldest_tr) + list_move(&tr->tr_list, &sdp->sd_ail2_list); + else + oldest_tr = 0; + } + gfs2_log_update_flush_tail(sdp); + empty = list_empty(&sdp->sd_ail1_list); + spin_unlock(&sdp->sd_ail_lock); + + return empty; +} + +static void gfs2_ail1_wait(struct gfs2_sbd *sdp) +{ + struct gfs2_trans *tr; + struct gfs2_bufdata *bd; + struct buffer_head *bh; + + spin_lock(&sdp->sd_ail_lock); + list_for_each_entry_reverse(tr, &sdp->sd_ail1_list, tr_list) { + list_for_each_entry(bd, &tr->tr_ail1_list, bd_ail_st_list) { + bh = bd->bd_bh; + if (!buffer_locked(bh)) + continue; + get_bh(bh); + spin_unlock(&sdp->sd_ail_lock); + wait_on_buffer(bh); + brelse(bh); + return; + } + } + spin_unlock(&sdp->sd_ail_lock); +} + +static void __ail2_empty(struct gfs2_sbd *sdp, struct gfs2_trans *tr) +{ + gfs2_ail_empty_tr(sdp, tr, &tr->tr_ail2_list); + list_del(&tr->tr_list); + gfs2_assert_warn(sdp, list_empty(&tr->tr_ail1_list)); + gfs2_assert_warn(sdp, list_empty(&tr->tr_ail2_list)); + gfs2_trans_free(sdp, tr); +} + +static void ail2_empty(struct gfs2_sbd *sdp, unsigned int new_tail) +{ + struct list_head *ail2_list = &sdp->sd_ail2_list; + unsigned int old_tail = sdp->sd_log_tail; + struct gfs2_trans *tr, *safe; + + spin_lock(&sdp->sd_ail_lock); + if (old_tail <= new_tail) { + list_for_each_entry_safe(tr, safe, ail2_list, tr_list) { + if (old_tail <= tr->tr_first && tr->tr_first < new_tail) + __ail2_empty(sdp, tr); + } + } else { + list_for_each_entry_safe(tr, safe, ail2_list, tr_list) { + if (old_tail <= tr->tr_first || tr->tr_first < new_tail) + __ail2_empty(sdp, tr); + } + } + spin_unlock(&sdp->sd_ail_lock); +} + +/** + * gfs2_log_is_empty - Check if the log is empty + * @sdp: The GFS2 superblock + */ + +bool gfs2_log_is_empty(struct gfs2_sbd *sdp) { + return atomic_read(&sdp->sd_log_blks_free) == sdp->sd_jdesc->jd_blocks; +} + +static bool __gfs2_log_try_reserve_revokes(struct gfs2_sbd *sdp, unsigned int revokes) +{ + unsigned int available; + + available = atomic_read(&sdp->sd_log_revokes_available); + while (available >= revokes) { + if (atomic_try_cmpxchg(&sdp->sd_log_revokes_available, + &available, available - revokes)) + return true; + } + return false; +} + +/** + * gfs2_log_release_revokes - Release a given number of revokes + * @sdp: The GFS2 superblock + * @revokes: The number of revokes to release + * + * sdp->sd_log_flush_lock must be held. + */ +void gfs2_log_release_revokes(struct gfs2_sbd *sdp, unsigned int revokes) +{ + if (revokes) + atomic_add(revokes, &sdp->sd_log_revokes_available); +} + +/** + * gfs2_log_release - Release a given number of log blocks + * @sdp: The GFS2 superblock + * @blks: The number of blocks + * + */ + +void gfs2_log_release(struct gfs2_sbd *sdp, unsigned int blks) +{ + atomic_add(blks, &sdp->sd_log_blks_free); + trace_gfs2_log_blocks(sdp, blks); + gfs2_assert_withdraw(sdp, !sdp->sd_jdesc || + atomic_read(&sdp->sd_log_blks_free) <= + sdp->sd_jdesc->jd_blocks); + if (atomic_read(&sdp->sd_log_blks_needed)) + wake_up(&sdp->sd_log_waitq); +} + +/** + * __gfs2_log_try_reserve - Try to make a log reservation + * @sdp: The GFS2 superblock + * @blks: The number of blocks to reserve + * @taboo_blks: The number of blocks to leave free + * + * Try to do the same as __gfs2_log_reserve(), but fail if no more log + * space is immediately available. + */ +static bool __gfs2_log_try_reserve(struct gfs2_sbd *sdp, unsigned int blks, + unsigned int taboo_blks) +{ + unsigned wanted = blks + taboo_blks; + unsigned int free_blocks; + + free_blocks = atomic_read(&sdp->sd_log_blks_free); + while (free_blocks >= wanted) { + if (atomic_try_cmpxchg(&sdp->sd_log_blks_free, &free_blocks, + free_blocks - blks)) { + trace_gfs2_log_blocks(sdp, -blks); + return true; + } + } + return false; +} + +/** + * __gfs2_log_reserve - Make a log reservation + * @sdp: The GFS2 superblock + * @blks: The number of blocks to reserve + * @taboo_blks: The number of blocks to leave free + * + * @taboo_blks is set to 0 for logd, and to GFS2_LOG_FLUSH_MIN_BLOCKS + * for all other processes. This ensures that when the log is almost full, + * logd will still be able to call gfs2_log_flush one more time without + * blocking, which will advance the tail and make some more log space + * available. + * + * We no longer flush the log here, instead we wake up logd to do that + * for us. To avoid the thundering herd and to ensure that we deal fairly + * with queued waiters, we use an exclusive wait. This means that when we + * get woken with enough journal space to get our reservation, we need to + * wake the next waiter on the list. + */ + +static void __gfs2_log_reserve(struct gfs2_sbd *sdp, unsigned int blks, + unsigned int taboo_blks) +{ + unsigned wanted = blks + taboo_blks; + unsigned int free_blocks; + + atomic_add(blks, &sdp->sd_log_blks_needed); + for (;;) { + if (current != sdp->sd_logd_process) + wake_up(&sdp->sd_logd_waitq); + io_wait_event(sdp->sd_log_waitq, + (free_blocks = atomic_read(&sdp->sd_log_blks_free), + free_blocks >= wanted)); + do { + if (atomic_try_cmpxchg(&sdp->sd_log_blks_free, + &free_blocks, + free_blocks - blks)) + goto reserved; + } while (free_blocks >= wanted); + } + +reserved: + trace_gfs2_log_blocks(sdp, -blks); + if (atomic_sub_return(blks, &sdp->sd_log_blks_needed)) + wake_up(&sdp->sd_log_waitq); +} + +/** + * gfs2_log_try_reserve - Try to make a log reservation + * @sdp: The GFS2 superblock + * @tr: The transaction + * @extra_revokes: The number of additional revokes reserved (output) + * + * This is similar to gfs2_log_reserve, but sdp->sd_log_flush_lock must be + * held for correct revoke accounting. + */ + +bool gfs2_log_try_reserve(struct gfs2_sbd *sdp, struct gfs2_trans *tr, + unsigned int *extra_revokes) +{ + unsigned int blks = tr->tr_reserved; + unsigned int revokes = tr->tr_revokes; + unsigned int revoke_blks = 0; + + *extra_revokes = 0; + if (revokes && !__gfs2_log_try_reserve_revokes(sdp, revokes)) { + revoke_blks = DIV_ROUND_UP(revokes, sdp->sd_inptrs); + *extra_revokes = revoke_blks * sdp->sd_inptrs - revokes; + blks += revoke_blks; + } + if (!blks) + return true; + if (__gfs2_log_try_reserve(sdp, blks, GFS2_LOG_FLUSH_MIN_BLOCKS)) + return true; + if (!revoke_blks) + gfs2_log_release_revokes(sdp, revokes); + return false; +} + +/** + * gfs2_log_reserve - Make a log reservation + * @sdp: The GFS2 superblock + * @tr: The transaction + * @extra_revokes: The number of additional revokes reserved (output) + * + * sdp->sd_log_flush_lock must not be held. + */ + +void gfs2_log_reserve(struct gfs2_sbd *sdp, struct gfs2_trans *tr, + unsigned int *extra_revokes) +{ + unsigned int blks = tr->tr_reserved; + unsigned int revokes = tr->tr_revokes; + unsigned int revoke_blks; + + *extra_revokes = 0; + if (revokes) { + revoke_blks = DIV_ROUND_UP(revokes, sdp->sd_inptrs); + *extra_revokes = revoke_blks * sdp->sd_inptrs - revokes; + blks += revoke_blks; + } + __gfs2_log_reserve(sdp, blks, GFS2_LOG_FLUSH_MIN_BLOCKS); +} + +/** + * log_distance - Compute distance between two journal blocks + * @sdp: The GFS2 superblock + * @newer: The most recent journal block of the pair + * @older: The older journal block of the pair + * + * Compute the distance (in the journal direction) between two + * blocks in the journal + * + * Returns: the distance in blocks + */ + +static inline unsigned int log_distance(struct gfs2_sbd *sdp, unsigned int newer, + unsigned int older) +{ + int dist; + + dist = newer - older; + if (dist < 0) + dist += sdp->sd_jdesc->jd_blocks; + + return dist; +} + +/** + * calc_reserved - Calculate the number of blocks to keep reserved + * @sdp: The GFS2 superblock + * + * This is complex. We need to reserve room for all our currently used + * metadata blocks (e.g. normal file I/O rewriting file time stamps) and + * all our journaled data blocks for journaled files (e.g. files in the + * meta_fs like rindex, or files for which chattr +j was done.) + * If we don't reserve enough space, corruption will follow. + * + * We can have metadata blocks and jdata blocks in the same journal. Each + * type gets its own log descriptor, for which we need to reserve a block. + * In fact, each type has the potential for needing more than one log descriptor + * in cases where we have more blocks than will fit in a log descriptor. + * Metadata journal entries take up half the space of journaled buffer entries. + * + * Also, we need to reserve blocks for revoke journal entries and one for an + * overall header for the lot. + * + * Returns: the number of blocks reserved + */ +static unsigned int calc_reserved(struct gfs2_sbd *sdp) +{ + unsigned int reserved = GFS2_LOG_FLUSH_MIN_BLOCKS; + unsigned int blocks; + struct gfs2_trans *tr = sdp->sd_log_tr; + + if (tr) { + blocks = tr->tr_num_buf_new - tr->tr_num_buf_rm; + reserved += blocks + DIV_ROUND_UP(blocks, buf_limit(sdp)); + blocks = tr->tr_num_databuf_new - tr->tr_num_databuf_rm; + reserved += blocks + DIV_ROUND_UP(blocks, databuf_limit(sdp)); + } + return reserved; +} + +static void log_pull_tail(struct gfs2_sbd *sdp) +{ + unsigned int new_tail = sdp->sd_log_flush_tail; + unsigned int dist; + + if (new_tail == sdp->sd_log_tail) + return; + dist = log_distance(sdp, new_tail, sdp->sd_log_tail); + ail2_empty(sdp, new_tail); + gfs2_log_release(sdp, dist); + sdp->sd_log_tail = new_tail; +} + + +void log_flush_wait(struct gfs2_sbd *sdp) +{ + DEFINE_WAIT(wait); + + if (atomic_read(&sdp->sd_log_in_flight)) { + do { + prepare_to_wait(&sdp->sd_log_flush_wait, &wait, + TASK_UNINTERRUPTIBLE); + if (atomic_read(&sdp->sd_log_in_flight)) + io_schedule(); + } while(atomic_read(&sdp->sd_log_in_flight)); + finish_wait(&sdp->sd_log_flush_wait, &wait); + } +} + +static int ip_cmp(void *priv, const struct list_head *a, const struct list_head *b) +{ + struct gfs2_inode *ipa, *ipb; + + ipa = list_entry(a, struct gfs2_inode, i_ordered); + ipb = list_entry(b, struct gfs2_inode, i_ordered); + + if (ipa->i_no_addr < ipb->i_no_addr) + return -1; + if (ipa->i_no_addr > ipb->i_no_addr) + return 1; + return 0; +} + +static void __ordered_del_inode(struct gfs2_inode *ip) +{ + if (!list_empty(&ip->i_ordered)) + list_del_init(&ip->i_ordered); +} + +static void gfs2_ordered_write(struct gfs2_sbd *sdp) +{ + struct gfs2_inode *ip; + LIST_HEAD(written); + + spin_lock(&sdp->sd_ordered_lock); + list_sort(NULL, &sdp->sd_log_ordered, &ip_cmp); + while (!list_empty(&sdp->sd_log_ordered)) { + ip = list_first_entry(&sdp->sd_log_ordered, struct gfs2_inode, i_ordered); + if (ip->i_inode.i_mapping->nrpages == 0) { + __ordered_del_inode(ip); + continue; + } + list_move(&ip->i_ordered, &written); + spin_unlock(&sdp->sd_ordered_lock); + filemap_fdatawrite(ip->i_inode.i_mapping); + spin_lock(&sdp->sd_ordered_lock); + } + list_splice(&written, &sdp->sd_log_ordered); + spin_unlock(&sdp->sd_ordered_lock); +} + +static void gfs2_ordered_wait(struct gfs2_sbd *sdp) +{ + struct gfs2_inode *ip; + + spin_lock(&sdp->sd_ordered_lock); + while (!list_empty(&sdp->sd_log_ordered)) { + ip = list_first_entry(&sdp->sd_log_ordered, struct gfs2_inode, i_ordered); + __ordered_del_inode(ip); + if (ip->i_inode.i_mapping->nrpages == 0) + continue; + spin_unlock(&sdp->sd_ordered_lock); + filemap_fdatawait(ip->i_inode.i_mapping); + spin_lock(&sdp->sd_ordered_lock); + } + spin_unlock(&sdp->sd_ordered_lock); +} + +void gfs2_ordered_del_inode(struct gfs2_inode *ip) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + + spin_lock(&sdp->sd_ordered_lock); + __ordered_del_inode(ip); + spin_unlock(&sdp->sd_ordered_lock); +} + +void gfs2_add_revoke(struct gfs2_sbd *sdp, struct gfs2_bufdata *bd) +{ + struct buffer_head *bh = bd->bd_bh; + struct gfs2_glock *gl = bd->bd_gl; + + sdp->sd_log_num_revoke++; + if (atomic_inc_return(&gl->gl_revokes) == 1) + gfs2_glock_hold(gl); + bh->b_private = NULL; + bd->bd_blkno = bh->b_blocknr; + gfs2_remove_from_ail(bd); /* drops ref on bh */ + bd->bd_bh = NULL; + set_bit(GLF_LFLUSH, &gl->gl_flags); + list_add(&bd->bd_list, &sdp->sd_log_revokes); +} + +void gfs2_glock_remove_revoke(struct gfs2_glock *gl) +{ + if (atomic_dec_return(&gl->gl_revokes) == 0) { + clear_bit(GLF_LFLUSH, &gl->gl_flags); + gfs2_glock_put_async(gl); + } +} + +/** + * gfs2_flush_revokes - Add as many revokes to the system transaction as we can + * @sdp: The GFS2 superblock + * + * Our usual strategy is to defer writing revokes as much as we can in the hope + * that we'll eventually overwrite the journal, which will make those revokes + * go away. This changes when we flush the log: at that point, there will + * likely be some left-over space in the last revoke block of that transaction. + * We can fill that space with additional revokes for blocks that have already + * been written back. This will basically come at no cost now, and will save + * us from having to keep track of those blocks on the AIL2 list later. + */ +void gfs2_flush_revokes(struct gfs2_sbd *sdp) +{ + /* number of revokes we still have room for */ + unsigned int max_revokes = atomic_read(&sdp->sd_log_revokes_available); + + spin_lock(&sdp->sd_log_lock); + gfs2_ail1_empty(sdp, max_revokes); + spin_unlock(&sdp->sd_log_lock); +} + +/** + * gfs2_write_log_header - Write a journal log header buffer at lblock + * @sdp: The GFS2 superblock + * @jd: journal descriptor of the journal to which we are writing + * @seq: sequence number + * @tail: tail of the log + * @lblock: value for lh_blkno (block number relative to start of journal) + * @flags: log header flags GFS2_LOG_HEAD_* + * @op_flags: flags to pass to the bio + * + * Returns: the initialized log buffer descriptor + */ + +void gfs2_write_log_header(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd, + u64 seq, u32 tail, u32 lblock, u32 flags, + blk_opf_t op_flags) +{ + struct gfs2_log_header *lh; + u32 hash, crc; + struct page *page; + struct gfs2_statfs_change_host *l_sc = &sdp->sd_statfs_local; + struct timespec64 tv; + struct super_block *sb = sdp->sd_vfs; + u64 dblock; + + if (gfs2_withdrawn(sdp)) + return; + + page = mempool_alloc(gfs2_page_pool, GFP_NOIO); + lh = page_address(page); + clear_page(lh); + + lh->lh_header.mh_magic = cpu_to_be32(GFS2_MAGIC); + lh->lh_header.mh_type = cpu_to_be32(GFS2_METATYPE_LH); + lh->lh_header.__pad0 = cpu_to_be64(0); + lh->lh_header.mh_format = cpu_to_be32(GFS2_FORMAT_LH); + lh->lh_header.mh_jid = cpu_to_be32(sdp->sd_jdesc->jd_jid); + lh->lh_sequence = cpu_to_be64(seq); + lh->lh_flags = cpu_to_be32(flags); + lh->lh_tail = cpu_to_be32(tail); + lh->lh_blkno = cpu_to_be32(lblock); + hash = ~crc32(~0, lh, LH_V1_SIZE); + lh->lh_hash = cpu_to_be32(hash); + + ktime_get_coarse_real_ts64(&tv); + lh->lh_nsec = cpu_to_be32(tv.tv_nsec); + lh->lh_sec = cpu_to_be64(tv.tv_sec); + if (!list_empty(&jd->extent_list)) + dblock = gfs2_log_bmap(jd, lblock); + else { + unsigned int extlen; + int ret; + + extlen = 1; + ret = gfs2_get_extent(jd->jd_inode, lblock, &dblock, &extlen); + if (gfs2_assert_withdraw(sdp, ret == 0)) + return; + } + lh->lh_addr = cpu_to_be64(dblock); + lh->lh_jinode = cpu_to_be64(GFS2_I(jd->jd_inode)->i_no_addr); + + /* We may only write local statfs, quota, etc., when writing to our + own journal. The values are left 0 when recovering a journal + different from our own. */ + if (!(flags & GFS2_LOG_HEAD_RECOVERY)) { + lh->lh_statfs_addr = + cpu_to_be64(GFS2_I(sdp->sd_sc_inode)->i_no_addr); + lh->lh_quota_addr = + cpu_to_be64(GFS2_I(sdp->sd_qc_inode)->i_no_addr); + + spin_lock(&sdp->sd_statfs_spin); + lh->lh_local_total = cpu_to_be64(l_sc->sc_total); + lh->lh_local_free = cpu_to_be64(l_sc->sc_free); + lh->lh_local_dinodes = cpu_to_be64(l_sc->sc_dinodes); + spin_unlock(&sdp->sd_statfs_spin); + } + + BUILD_BUG_ON(offsetof(struct gfs2_log_header, lh_crc) != LH_V1_SIZE); + + crc = crc32c(~0, (void *)lh + LH_V1_SIZE + 4, + sb->s_blocksize - LH_V1_SIZE - 4); + lh->lh_crc = cpu_to_be32(crc); + + gfs2_log_write(sdp, jd, page, sb->s_blocksize, 0, dblock, + REQ_OP_WRITE | op_flags); + gfs2_log_submit_write(&jd->jd_log_bio); +} + +/** + * log_write_header - Get and initialize a journal header buffer + * @sdp: The GFS2 superblock + * @flags: The log header flags, including log header origin + * + * Returns: the initialized log buffer descriptor + */ + +static void log_write_header(struct gfs2_sbd *sdp, u32 flags) +{ + blk_opf_t op_flags = REQ_PREFLUSH | REQ_FUA | REQ_META | REQ_SYNC; + struct super_block *sb = sdp->sd_vfs; + + gfs2_assert_withdraw(sdp, sb->s_writers.frozen != SB_FREEZE_COMPLETE); + + if (test_bit(SDF_NOBARRIERS, &sdp->sd_flags)) { + gfs2_ordered_wait(sdp); + log_flush_wait(sdp); + op_flags = REQ_SYNC | REQ_META | REQ_PRIO; + } + sdp->sd_log_idle = (sdp->sd_log_flush_tail == sdp->sd_log_flush_head); + gfs2_write_log_header(sdp, sdp->sd_jdesc, sdp->sd_log_sequence++, + sdp->sd_log_flush_tail, sdp->sd_log_flush_head, + flags, op_flags); + gfs2_log_incr_head(sdp); + log_flush_wait(sdp); + log_pull_tail(sdp); + gfs2_log_update_head(sdp); +} + +/** + * gfs2_ail_drain - drain the ail lists after a withdraw + * @sdp: Pointer to GFS2 superblock + */ +void gfs2_ail_drain(struct gfs2_sbd *sdp) +{ + struct gfs2_trans *tr; + + spin_lock(&sdp->sd_ail_lock); + /* + * For transactions on the sd_ail1_list we need to drain both the + * ail1 and ail2 lists. That's because function gfs2_ail1_start_one + * (temporarily) moves items from its tr_ail1 list to tr_ail2 list + * before revokes are sent for that block. Items on the sd_ail2_list + * should have already gotten beyond that point, so no need. + */ + while (!list_empty(&sdp->sd_ail1_list)) { + tr = list_first_entry(&sdp->sd_ail1_list, struct gfs2_trans, + tr_list); + gfs2_ail_empty_tr(sdp, tr, &tr->tr_ail1_list); + gfs2_ail_empty_tr(sdp, tr, &tr->tr_ail2_list); + list_del(&tr->tr_list); + gfs2_trans_free(sdp, tr); + } + while (!list_empty(&sdp->sd_ail2_list)) { + tr = list_first_entry(&sdp->sd_ail2_list, struct gfs2_trans, + tr_list); + gfs2_ail_empty_tr(sdp, tr, &tr->tr_ail2_list); + list_del(&tr->tr_list); + gfs2_trans_free(sdp, tr); + } + gfs2_drain_revokes(sdp); + spin_unlock(&sdp->sd_ail_lock); +} + +/** + * empty_ail1_list - try to start IO and empty the ail1 list + * @sdp: Pointer to GFS2 superblock + */ +static void empty_ail1_list(struct gfs2_sbd *sdp) +{ + unsigned long start = jiffies; + bool empty = false; + + while (!empty) { + if (time_after(jiffies, start + (HZ * 600))) { + fs_err(sdp, "Error: In %s for 10 minutes! t=%d\n", + __func__, current->journal_info ? 1 : 0); + dump_ail_list(sdp); + return; + } + gfs2_ail1_start(sdp); + gfs2_ail1_wait(sdp); + empty = gfs2_ail1_empty(sdp, 0); + + if (gfs2_withdrawn(sdp)) + break; + } +} + +static void gfs2_trans_drain_list(struct gfs2_sbd *sdp, struct list_head *list) +{ + struct gfs2_bufdata *bd; + + while (!list_empty(list)) { + bd = list_first_entry(list, struct gfs2_bufdata, bd_list); + struct buffer_head *bh = bd->bd_bh; + + WARN_ON_ONCE(!buffer_pinned(bh)); + clear_buffer_pinned(bh); + trace_gfs2_pin(bd, 0); + atomic_dec(&sdp->sd_log_pinned); + list_del_init(&bd->bd_list); + brelse(bh); + } +} + +/** + * gfs2_trans_drain - drain the buf and databuf queue for a failed transaction + * @sdp: the filesystem + * @tr: the transaction to drain + * + * When this is called, we're taking an error exit for a log write that failed + * but since we bypassed the after_commit functions, we need to remove the + * items from the buf and databuf queue. + */ +static void gfs2_trans_drain(struct gfs2_sbd *sdp, struct gfs2_trans *tr) +{ + if (!tr) + return; + gfs2_trans_drain_list(sdp, &tr->tr_buf); + gfs2_trans_drain_list(sdp, &tr->tr_databuf); +} + +void gfs2_remove_from_journal(struct buffer_head *bh, int meta) +{ + struct address_space *mapping = bh->b_folio->mapping; + struct gfs2_sbd *sdp = gfs2_mapping2sbd(mapping); + struct gfs2_bufdata *bd = bh->b_private; + struct gfs2_trans *tr = current->journal_info; + int was_pinned = 0; + + if (test_clear_buffer_pinned(bh)) { + trace_gfs2_pin(bd, 0); + atomic_dec(&sdp->sd_log_pinned); + list_del_init(&bd->bd_list); + if (tr) { + if (meta == REMOVE_META) + tr->tr_num_buf_rm++; + else + tr->tr_num_databuf_rm++; + set_bit(TR_TOUCHED, &tr->tr_flags); + } + was_pinned = 1; + brelse(bh); + } + if (bd) { + if (bd->bd_tr) { + if (tr) + gfs2_trans_add_revoke(sdp, bd); + else + gfs2_remove_from_ail(bd); + } else if (was_pinned) { + bh->b_private = NULL; + kmem_cache_free(gfs2_bufdata_cachep, bd); + } else if (!list_empty(&bd->bd_ail_st_list) && + !list_empty(&bd->bd_ail_gl_list)) { + gfs2_remove_from_ail(bd); + } + } + clear_buffer_dirty(bh); + clear_buffer_uptodate(bh); +} + +/** + * __gfs2_log_flush - flush incore transaction(s) + * @sdp: The filesystem + * @gl: The glock structure to flush. If NULL, flush the whole incore log + * @flags: The log header flags: GFS2_LOG_HEAD_FLUSH_* and debug flags + * + */ + +static void __gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, + u32 flags) +{ + struct gfs2_trans *tr = NULL; + unsigned int reserved_blocks = 0, used_blocks = 0; + bool frozen = test_bit(SDF_FROZEN, &sdp->sd_flags); + unsigned int first_log_head; + unsigned int reserved_revokes = 0; + + trace_gfs2_log_flush(sdp, 1, flags); + +repeat: + /* + * Do this check while holding the log_flush_lock to prevent new + * buffers from being added to the ail via gfs2_pin() + */ + if (gfs2_withdrawn(sdp) || + !test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags)) + goto out; + + /* Log might have been flushed while we waited for the flush lock */ + if (gl && !test_bit(GLF_LFLUSH, &gl->gl_flags)) + goto out; + + first_log_head = sdp->sd_log_head; + sdp->sd_log_flush_head = first_log_head; + + tr = sdp->sd_log_tr; + if (tr || sdp->sd_log_num_revoke) { + if (reserved_blocks) + gfs2_log_release(sdp, reserved_blocks); + reserved_blocks = sdp->sd_log_blks_reserved; + reserved_revokes = sdp->sd_log_num_revoke; + if (tr) { + sdp->sd_log_tr = NULL; + tr->tr_first = first_log_head; + if (unlikely(frozen)) { + if (gfs2_assert_withdraw(sdp, + !tr->tr_num_buf_new && !tr->tr_num_databuf_new)) + goto out_withdraw; + } + } + } else if (!reserved_blocks) { + unsigned int taboo_blocks = GFS2_LOG_FLUSH_MIN_BLOCKS; + + reserved_blocks = GFS2_LOG_FLUSH_MIN_BLOCKS; + if (current == sdp->sd_logd_process) + taboo_blocks = 0; + + if (!__gfs2_log_try_reserve(sdp, reserved_blocks, taboo_blocks)) { + up_write(&sdp->sd_log_flush_lock); + __gfs2_log_reserve(sdp, reserved_blocks, taboo_blocks); + down_write(&sdp->sd_log_flush_lock); + goto repeat; + } + BUG_ON(sdp->sd_log_num_revoke); + } + + if (flags & GFS2_LOG_HEAD_FLUSH_SHUTDOWN) + clear_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags); + + if (unlikely(frozen)) + if (gfs2_assert_withdraw(sdp, !reserved_revokes)) + goto out_withdraw; + + gfs2_ordered_write(sdp); + if (gfs2_withdrawn(sdp)) + goto out_withdraw; + lops_before_commit(sdp, tr); + if (gfs2_withdrawn(sdp)) + goto out_withdraw; + if (sdp->sd_jdesc) + gfs2_log_submit_write(&sdp->sd_jdesc->jd_log_bio); + if (gfs2_withdrawn(sdp)) + goto out_withdraw; + + if (sdp->sd_log_head != sdp->sd_log_flush_head) { + log_write_header(sdp, flags); + } else if (sdp->sd_log_tail != sdp->sd_log_flush_tail && !sdp->sd_log_idle) { + log_write_header(sdp, flags); + } + if (gfs2_withdrawn(sdp)) + goto out_withdraw; + lops_after_commit(sdp, tr); + + spin_lock(&sdp->sd_log_lock); + sdp->sd_log_blks_reserved = 0; + + spin_lock(&sdp->sd_ail_lock); + if (tr && !list_empty(&tr->tr_ail1_list)) { + list_add(&tr->tr_list, &sdp->sd_ail1_list); + tr = NULL; + } + spin_unlock(&sdp->sd_ail_lock); + spin_unlock(&sdp->sd_log_lock); + + if (!(flags & GFS2_LOG_HEAD_FLUSH_NORMAL)) { + if (!sdp->sd_log_idle) { + empty_ail1_list(sdp); + if (gfs2_withdrawn(sdp)) + goto out_withdraw; + log_write_header(sdp, flags); + } + if (flags & (GFS2_LOG_HEAD_FLUSH_SHUTDOWN | + GFS2_LOG_HEAD_FLUSH_FREEZE)) + gfs2_log_shutdown(sdp); + } + +out_end: + used_blocks = log_distance(sdp, sdp->sd_log_flush_head, first_log_head); + reserved_revokes += atomic_read(&sdp->sd_log_revokes_available); + atomic_set(&sdp->sd_log_revokes_available, sdp->sd_ldptrs); + gfs2_assert_withdraw(sdp, reserved_revokes % sdp->sd_inptrs == sdp->sd_ldptrs); + if (reserved_revokes > sdp->sd_ldptrs) + reserved_blocks += (reserved_revokes - sdp->sd_ldptrs) / sdp->sd_inptrs; +out: + if (used_blocks != reserved_blocks) { + gfs2_assert_withdraw(sdp, used_blocks < reserved_blocks); + gfs2_log_release(sdp, reserved_blocks - used_blocks); + } + gfs2_trans_free(sdp, tr); + trace_gfs2_log_flush(sdp, 0, flags); + return; + +out_withdraw: + if (sdp->sd_jdesc->jd_log_bio) { + bio_io_error(sdp->sd_jdesc->jd_log_bio); + sdp->sd_jdesc->jd_log_bio = NULL; + } + gfs2_trans_drain(sdp, tr); + /** + * If the tr_list is empty, we're withdrawing during a log + * flush that targets a transaction, but the transaction was + * never queued onto any of the ail lists. Here we add it to + * ail1 just so that ail_drain() will find and free it. + */ + spin_lock(&sdp->sd_ail_lock); + if (tr && list_empty(&tr->tr_list)) + list_add(&tr->tr_list, &sdp->sd_ail1_list); + spin_unlock(&sdp->sd_ail_lock); + tr = NULL; + goto out_end; +} + +void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, u32 flags) +{ + down_write(&sdp->sd_log_flush_lock); + __gfs2_log_flush(sdp, gl, flags); + up_write(&sdp->sd_log_flush_lock); +} + +/** + * gfs2_merge_trans - Merge a new transaction into a cached transaction + * @sdp: the filesystem + * @new: New transaction to be merged + */ + +static void gfs2_merge_trans(struct gfs2_sbd *sdp, struct gfs2_trans *new) +{ + struct gfs2_trans *old = sdp->sd_log_tr; + + WARN_ON_ONCE(!test_bit(TR_ATTACHED, &old->tr_flags)); + + old->tr_num_buf_new += new->tr_num_buf_new; + old->tr_num_databuf_new += new->tr_num_databuf_new; + old->tr_num_buf_rm += new->tr_num_buf_rm; + old->tr_num_databuf_rm += new->tr_num_databuf_rm; + old->tr_revokes += new->tr_revokes; + old->tr_num_revoke += new->tr_num_revoke; + + list_splice_tail_init(&new->tr_databuf, &old->tr_databuf); + list_splice_tail_init(&new->tr_buf, &old->tr_buf); + + spin_lock(&sdp->sd_ail_lock); + list_splice_tail_init(&new->tr_ail1_list, &old->tr_ail1_list); + list_splice_tail_init(&new->tr_ail2_list, &old->tr_ail2_list); + spin_unlock(&sdp->sd_ail_lock); +} + +static void log_refund(struct gfs2_sbd *sdp, struct gfs2_trans *tr) +{ + unsigned int reserved; + unsigned int unused; + unsigned int maxres; + + spin_lock(&sdp->sd_log_lock); + + if (sdp->sd_log_tr) { + gfs2_merge_trans(sdp, tr); + } else if (tr->tr_num_buf_new || tr->tr_num_databuf_new) { + gfs2_assert_withdraw(sdp, !test_bit(TR_ONSTACK, &tr->tr_flags)); + sdp->sd_log_tr = tr; + set_bit(TR_ATTACHED, &tr->tr_flags); + } + + reserved = calc_reserved(sdp); + maxres = sdp->sd_log_blks_reserved + tr->tr_reserved; + gfs2_assert_withdraw(sdp, maxres >= reserved); + unused = maxres - reserved; + if (unused) + gfs2_log_release(sdp, unused); + sdp->sd_log_blks_reserved = reserved; + + spin_unlock(&sdp->sd_log_lock); +} + +static inline int gfs2_jrnl_flush_reqd(struct gfs2_sbd *sdp) +{ + return atomic_read(&sdp->sd_log_pinned) + + atomic_read(&sdp->sd_log_blks_needed) >= + atomic_read(&sdp->sd_log_thresh1); +} + +static inline int gfs2_ail_flush_reqd(struct gfs2_sbd *sdp) +{ + return sdp->sd_jdesc->jd_blocks - + atomic_read(&sdp->sd_log_blks_free) + + atomic_read(&sdp->sd_log_blks_needed) >= + atomic_read(&sdp->sd_log_thresh2); +} + +/** + * gfs2_log_commit - Commit a transaction to the log + * @sdp: the filesystem + * @tr: the transaction + * + * We wake up gfs2_logd if the number of pinned blocks exceed thresh1 + * or the total number of used blocks (pinned blocks plus AIL blocks) + * is greater than thresh2. + * + * At mount time thresh1 is 2/5ths of journal size, thresh2 is 4/5ths of + * journal size. + * + * Returns: errno + */ + +void gfs2_log_commit(struct gfs2_sbd *sdp, struct gfs2_trans *tr) +{ + log_refund(sdp, tr); + + if (gfs2_ail_flush_reqd(sdp) || gfs2_jrnl_flush_reqd(sdp)) + wake_up(&sdp->sd_logd_waitq); +} + +/** + * gfs2_log_shutdown - write a shutdown header into a journal + * @sdp: the filesystem + * + */ + +static void gfs2_log_shutdown(struct gfs2_sbd *sdp) +{ + gfs2_assert_withdraw(sdp, !sdp->sd_log_blks_reserved); + gfs2_assert_withdraw(sdp, !sdp->sd_log_num_revoke); + gfs2_assert_withdraw(sdp, list_empty(&sdp->sd_ail1_list)); + + log_write_header(sdp, GFS2_LOG_HEAD_UNMOUNT | GFS2_LFC_SHUTDOWN); + log_pull_tail(sdp); + + gfs2_assert_warn(sdp, sdp->sd_log_head == sdp->sd_log_tail); + gfs2_assert_warn(sdp, list_empty(&sdp->sd_ail2_list)); +} + +/** + * gfs2_logd - Update log tail as Active Items get flushed to in-place blocks + * @data: Pointer to GFS2 superblock + * + * Also, periodically check to make sure that we're using the most recent + * journal index. + */ + +int gfs2_logd(void *data) +{ + struct gfs2_sbd *sdp = data; + unsigned long t = 1; + + set_freezable(); + while (!kthread_should_stop()) { + if (gfs2_withdrawn(sdp)) + break; + + if (gfs2_jrnl_flush_reqd(sdp) || t == 0) { + down_write(&sdp->sd_log_flush_lock); + gfs2_ail1_empty(sdp, 0); + __gfs2_log_flush(sdp, NULL, + GFS2_LOG_HEAD_FLUSH_NORMAL | + GFS2_LFC_LOGD_JFLUSH_REQD); + up_write(&sdp->sd_log_flush_lock); + } + + if (test_bit(SDF_FORCE_AIL_FLUSH, &sdp->sd_flags) || + gfs2_ail_flush_reqd(sdp)) { + clear_bit(SDF_FORCE_AIL_FLUSH, &sdp->sd_flags); + down_write(&sdp->sd_log_flush_lock); + gfs2_ail1_start(sdp); + gfs2_ail1_wait(sdp); + gfs2_ail1_empty(sdp, 0); + __gfs2_log_flush(sdp, NULL, + GFS2_LOG_HEAD_FLUSH_NORMAL | + GFS2_LFC_LOGD_AIL_FLUSH_REQD); + up_write(&sdp->sd_log_flush_lock); + } + + t = gfs2_tune_get(sdp, gt_logd_secs) * HZ; + + t = wait_event_freezable_timeout(sdp->sd_logd_waitq, + test_bit(SDF_FORCE_AIL_FLUSH, &sdp->sd_flags) || + gfs2_ail_flush_reqd(sdp) || + gfs2_jrnl_flush_reqd(sdp) || + gfs2_withdrawn(sdp) || + kthread_should_stop(), + t); + } + + return 0; +} + diff --git a/fs/gfs2/log.h b/fs/gfs2/log.h new file mode 100644 index 000000000..10beda624 --- /dev/null +++ b/fs/gfs2/log.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#ifndef __LOG_DOT_H__ +#define __LOG_DOT_H__ + +#include +#include +#include +#include "incore.h" +#include "inode.h" + +/* + * The minimum amount of log space required for a log flush is one block for + * revokes and one block for the log header. Log flushes other than + * GFS2_LOG_HEAD_FLUSH_NORMAL may write one or two more log headers. + */ +#define GFS2_LOG_FLUSH_MIN_BLOCKS 4 + +static inline void gfs2_ordered_add_inode(struct gfs2_inode *ip) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + + if (gfs2_is_jdata(ip) || !gfs2_is_ordered(sdp)) + return; + + if (list_empty(&ip->i_ordered)) { + spin_lock(&sdp->sd_ordered_lock); + if (list_empty(&ip->i_ordered)) + list_add(&ip->i_ordered, &sdp->sd_log_ordered); + spin_unlock(&sdp->sd_ordered_lock); + } +} + +void gfs2_ordered_del_inode(struct gfs2_inode *ip); +unsigned int gfs2_struct2blk(struct gfs2_sbd *sdp, unsigned int nstruct); +bool gfs2_log_is_empty(struct gfs2_sbd *sdp); +void gfs2_log_release_revokes(struct gfs2_sbd *sdp, unsigned int revokes); +void gfs2_log_release(struct gfs2_sbd *sdp, unsigned int blks); +bool gfs2_log_try_reserve(struct gfs2_sbd *sdp, struct gfs2_trans *tr, + unsigned int *extra_revokes); +void gfs2_log_reserve(struct gfs2_sbd *sdp, struct gfs2_trans *tr, + unsigned int *extra_revokes); +void gfs2_write_log_header(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd, + u64 seq, u32 tail, u32 lblock, u32 flags, + blk_opf_t op_flags); +void gfs2_remove_from_journal(struct buffer_head *bh, int meta); +void gfs2_log_flush(struct gfs2_sbd *sdp, struct gfs2_glock *gl, + u32 type); +void gfs2_log_commit(struct gfs2_sbd *sdp, struct gfs2_trans *trans); +void gfs2_ail1_flush(struct gfs2_sbd *sdp, struct writeback_control *wbc); +void log_flush_wait(struct gfs2_sbd *sdp); + +int gfs2_logd(void *data); +void gfs2_add_revoke(struct gfs2_sbd *sdp, struct gfs2_bufdata *bd); +void gfs2_glock_remove_revoke(struct gfs2_glock *gl); +void gfs2_flush_revokes(struct gfs2_sbd *sdp); +void gfs2_ail_drain(struct gfs2_sbd *sdp); + +#endif /* __LOG_DOT_H__ */ diff --git a/fs/gfs2/lops.c b/fs/gfs2/lops.c new file mode 100644 index 000000000..6dabe73ad --- /dev/null +++ b/fs/gfs2/lops.c @@ -0,0 +1,1117 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "bmap.h" +#include "dir.h" +#include "gfs2.h" +#include "incore.h" +#include "inode.h" +#include "glock.h" +#include "glops.h" +#include "log.h" +#include "lops.h" +#include "meta_io.h" +#include "recovery.h" +#include "rgrp.h" +#include "trans.h" +#include "util.h" +#include "trace_gfs2.h" + +/** + * gfs2_pin - Pin a buffer in memory + * @sdp: The superblock + * @bh: The buffer to be pinned + * + * The log lock must be held when calling this function + */ +void gfs2_pin(struct gfs2_sbd *sdp, struct buffer_head *bh) +{ + struct gfs2_bufdata *bd; + + BUG_ON(!current->journal_info); + + clear_buffer_dirty(bh); + if (test_set_buffer_pinned(bh)) + gfs2_assert_withdraw(sdp, 0); + if (!buffer_uptodate(bh)) + gfs2_io_error_bh(sdp, bh); + bd = bh->b_private; + /* If this buffer is in the AIL and it has already been written + * to in-place disk block, remove it from the AIL. + */ + spin_lock(&sdp->sd_ail_lock); + if (bd->bd_tr) + list_move(&bd->bd_ail_st_list, &bd->bd_tr->tr_ail2_list); + spin_unlock(&sdp->sd_ail_lock); + get_bh(bh); + atomic_inc(&sdp->sd_log_pinned); + trace_gfs2_pin(bd, 1); +} + +static bool buffer_is_rgrp(const struct gfs2_bufdata *bd) +{ + return glock_type(bd->bd_gl) == LM_TYPE_RGRP; +} + +static void maybe_release_space(struct gfs2_bufdata *bd) +{ + struct gfs2_glock *gl = bd->bd_gl; + struct gfs2_sbd *sdp = glock_sbd(gl); + struct gfs2_rgrpd *rgd = gfs2_glock2rgrp(gl); + unsigned int index = bd->bd_bh->b_blocknr - glock_number(gl); + struct gfs2_bitmap *bi = rgd->rd_bits + index; + + rgrp_lock_local(rgd); + if (bi->bi_clone == NULL) + goto out; + if (sdp->sd_args.ar_discard) + gfs2_rgrp_send_discards(sdp, rgd->rd_data0, bd->bd_bh, bi, 1, NULL); + memcpy(bi->bi_clone + bi->bi_offset, + bd->bd_bh->b_data + bi->bi_offset, bi->bi_bytes); + clear_bit(GBF_FULL, &bi->bi_flags); + rgd->rd_free_clone = rgd->rd_free; + BUG_ON(rgd->rd_free_clone < rgd->rd_reserved); + rgd->rd_extfail_pt = rgd->rd_free; + +out: + rgrp_unlock_local(rgd); +} + +/** + * gfs2_unpin - Unpin a buffer + * @sdp: the filesystem the buffer belongs to + * @bh: The buffer to unpin + * @tr: The system transaction being flushed + */ + +static void gfs2_unpin(struct gfs2_sbd *sdp, struct buffer_head *bh, + struct gfs2_trans *tr) +{ + struct gfs2_bufdata *bd = bh->b_private; + + BUG_ON(!buffer_uptodate(bh)); + BUG_ON(!buffer_pinned(bh)); + + lock_buffer(bh); + mark_buffer_dirty(bh); + clear_buffer_pinned(bh); + + if (buffer_is_rgrp(bd)) + maybe_release_space(bd); + + spin_lock(&sdp->sd_ail_lock); + if (bd->bd_tr) { + list_del(&bd->bd_ail_st_list); + brelse(bh); + } else { + struct gfs2_glock *gl = bd->bd_gl; + list_add(&bd->bd_ail_gl_list, &gl->gl_ail_list); + atomic_inc(&gl->gl_ail_count); + } + bd->bd_tr = tr; + list_add(&bd->bd_ail_st_list, &tr->tr_ail1_list); + spin_unlock(&sdp->sd_ail_lock); + + clear_bit(GLF_LFLUSH, &bd->bd_gl->gl_flags); + trace_gfs2_pin(bd, 0); + unlock_buffer(bh); + atomic_dec(&sdp->sd_log_pinned); +} + +void gfs2_log_incr_head(struct gfs2_sbd *sdp) +{ + BUG_ON((sdp->sd_log_flush_head == sdp->sd_log_tail) && + (sdp->sd_log_flush_head != sdp->sd_log_head)); + + if (++sdp->sd_log_flush_head == sdp->sd_jdesc->jd_blocks) + sdp->sd_log_flush_head = 0; +} + +u64 gfs2_log_bmap(struct gfs2_jdesc *jd, unsigned int lblock) +{ + struct gfs2_journal_extent *je; + + list_for_each_entry(je, &jd->extent_list, list) { + if (lblock >= je->lblock && lblock < je->lblock + je->blocks) + return je->dblock + lblock - je->lblock; + } + + return -1; +} + +/** + * gfs2_end_log_write_bh - end log write of pagecache data with buffers + * @sdp: The superblock + * @folio: The folio + * @offset: The first byte within the folio that completed + * @size: The number of bytes that completed + * @error: The i/o status + * + * This finds the relevant buffers and unlocks them and sets the + * error flag according to the status of the i/o request. This is + * used when the log is writing data which has an in-place version + * that is pinned in the pagecache. + */ + +static void gfs2_end_log_write_bh(struct gfs2_sbd *sdp, struct folio *folio, + size_t offset, size_t size, blk_status_t error) +{ + struct buffer_head *bh, *next; + + bh = folio_buffers(folio); + while (bh_offset(bh) < offset) + bh = bh->b_this_page; + do { + if (error) + mark_buffer_write_io_error(bh); + unlock_buffer(bh); + next = bh->b_this_page; + size -= bh->b_size; + brelse(bh); + bh = next; + } while (bh && size); +} + +/** + * gfs2_end_log_write - end of i/o to the log + * @bio: The bio + * + * Each bio_vec contains either data from the pagecache or data + * relating to the log itself. Here we iterate over the bio_vec + * array, processing both kinds of data. + * + */ + +static void gfs2_end_log_write(struct bio *bio) +{ + struct gfs2_sbd *sdp = bio->bi_private; + struct bio_vec *bvec; + struct bvec_iter_all iter_all; + + if (bio->bi_status) { + int err = blk_status_to_errno(bio->bi_status); + + if (!cmpxchg(&sdp->sd_log_error, 0, err)) + fs_err(sdp, "Error %d writing to journal, jid=%u\n", + err, sdp->sd_jdesc->jd_jid); + gfs2_withdraw(sdp); + } + + bio_for_each_segment_all(bvec, bio, iter_all) { + struct page *page = bvec->bv_page; + struct folio *folio = page_folio(page); + + if (folio && folio_buffers(folio)) + gfs2_end_log_write_bh(sdp, folio, bvec->bv_offset, + bvec->bv_len, bio->bi_status); + else + mempool_free(page, gfs2_page_pool); + } + + bio_put(bio); + if (atomic_dec_and_test(&sdp->sd_log_in_flight)) + wake_up(&sdp->sd_log_flush_wait); +} + +/** + * gfs2_log_submit_write - Submit a pending log write bio + * @biop: Address of the bio pointer + * + * Submit any pending part-built or full bio to the block device. If + * there is no pending bio, then this is a no-op. + */ + +void gfs2_log_submit_write(struct bio **biop) +{ + struct bio *bio = *biop; + if (bio) { + struct gfs2_sbd *sdp = bio->bi_private; + atomic_inc(&sdp->sd_log_in_flight); + submit_bio(bio); + *biop = NULL; + } +} + +/** + * gfs2_log_alloc_bio - Allocate a bio + * @sdp: The super block + * @blkno: The device block number we want to write to + * @end_io: The bi_end_io callback + * @opf: REQ_OP | op_flags + * + * Allocate a new bio, initialize it with the given parameters and return it. + * + * Returns: The newly allocated bio + */ + +static struct bio *gfs2_log_alloc_bio(struct gfs2_sbd *sdp, u64 blkno, + bio_end_io_t *end_io, blk_opf_t opf) +{ + struct super_block *sb = sdp->sd_vfs; + struct bio *bio = bio_alloc(sb->s_bdev, BIO_MAX_VECS, opf, GFP_NOIO); + + bio->bi_iter.bi_sector = blkno << sdp->sd_fsb2bb_shift; + bio->bi_end_io = end_io; + bio->bi_private = sdp; + + return bio; +} + +/** + * gfs2_log_get_bio - Get cached log bio, or allocate a new one + * @sdp: The super block + * @blkno: The device block number we want to write to + * @biop: The bio to get or allocate + * @opf: REQ_OP | op_flags + * @end_io: The bi_end_io callback + * @flush: Always flush the current bio and allocate a new one? + * + * If there is a cached bio, then if the next block number is sequential + * with the previous one, return it, otherwise flush the bio to the + * device. If there is no cached bio, or we just flushed it, then + * allocate a new one. + * + * Returns: The bio to use for log writes + */ + +static struct bio *gfs2_log_get_bio(struct gfs2_sbd *sdp, u64 blkno, + struct bio **biop, blk_opf_t opf, + bio_end_io_t *end_io, bool flush) +{ + struct bio *bio = *biop; + + if (bio) { + u64 nblk; + + nblk = bio_end_sector(bio); + nblk >>= sdp->sd_fsb2bb_shift; + if (blkno == nblk && !flush) + return bio; + gfs2_log_submit_write(biop); + } + + *biop = gfs2_log_alloc_bio(sdp, blkno, end_io, opf); + return *biop; +} + +/** + * gfs2_log_write - write to log + * @sdp: the filesystem + * @jd: The journal descriptor + * @page: the page to write + * @size: the size of the data to write + * @offset: the offset within the page + * @blkno: block number of the log entry + * @opf: REQ_OP | op_flags + * + * Try and add the page segment to the current bio. If that fails, + * submit the current bio to the device and create a new one, and + * then add the page segment to that. + */ + +void gfs2_log_write(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd, + struct page *page, unsigned size, unsigned offset, + u64 blkno, blk_opf_t opf) +{ + struct bio *bio; + int ret; + + bio = gfs2_log_get_bio(sdp, blkno, &jd->jd_log_bio, opf, + gfs2_end_log_write, false); + ret = bio_add_page(bio, page, size, offset); + if (ret == 0) { + bio = gfs2_log_get_bio(sdp, blkno, &jd->jd_log_bio, + opf, gfs2_end_log_write, true); + ret = bio_add_page(bio, page, size, offset); + WARN_ON(ret == 0); + } +} + +/** + * gfs2_log_write_bh - write a buffer's content to the log + * @sdp: The super block + * @bh: The buffer pointing to the in-place location + * + * This writes the content of the buffer to the next available location + * in the log. The buffer will be unlocked once the i/o to the log has + * completed. + */ + +static void gfs2_log_write_bh(struct gfs2_sbd *sdp, struct buffer_head *bh) +{ + u64 dblock; + + dblock = gfs2_log_bmap(sdp->sd_jdesc, sdp->sd_log_flush_head); + gfs2_log_incr_head(sdp); + gfs2_log_write(sdp, sdp->sd_jdesc, folio_page(bh->b_folio, 0), + bh->b_size, bh_offset(bh), dblock, REQ_OP_WRITE); +} + +/** + * gfs2_log_write_page - write one block stored in a page, into the log + * @sdp: The superblock + * @page: The struct page + * + * This writes the first block-sized part of the page into the log. Note + * that the page must have been allocated from the gfs2_page_pool mempool + * and that after this has been called, ownership has been transferred and + * the page may be freed at any time. + */ + +static void gfs2_log_write_page(struct gfs2_sbd *sdp, struct page *page) +{ + struct super_block *sb = sdp->sd_vfs; + u64 dblock; + + dblock = gfs2_log_bmap(sdp->sd_jdesc, sdp->sd_log_flush_head); + gfs2_log_incr_head(sdp); + gfs2_log_write(sdp, sdp->sd_jdesc, page, sb->s_blocksize, 0, dblock, + REQ_OP_WRITE); +} + +/** + * gfs2_end_log_read - end I/O callback for reads from the log + * @bio: The bio + * + * Simply unlock the pages in the bio. The main thread will wait on them and + * process them in order as necessary. + */ +static void gfs2_end_log_read(struct bio *bio) +{ + int error = blk_status_to_errno(bio->bi_status); + struct folio_iter fi; + + bio_for_each_folio_all(fi, bio) { + /* We're abusing wb_err to get the error to gfs2_find_jhead */ + filemap_set_wb_err(fi.folio->mapping, error); + folio_end_read(fi.folio, !error); + } + + bio_put(bio); +} + +/** + * gfs2_jhead_folio_search - Look for the journal head in a given page. + * @jd: The journal descriptor + * @head: The journal head to start from + * @folio: The folio to look in + * + * Returns: 1 if found, 0 otherwise. + */ +static bool gfs2_jhead_folio_search(struct gfs2_jdesc *jd, + struct gfs2_log_header_host *head, + struct folio *folio) +{ + struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode); + struct gfs2_log_header_host lh; + void *kaddr; + unsigned int offset; + bool ret = false; + + VM_BUG_ON_FOLIO(folio_test_large(folio), folio); + kaddr = kmap_local_folio(folio, 0); + for (offset = 0; offset < PAGE_SIZE; offset += sdp->sd_sb.sb_bsize) { + if (!__get_log_header(sdp, kaddr + offset, 0, &lh)) { + if (lh.lh_sequence >= head->lh_sequence) + *head = lh; + else { + ret = true; + break; + } + } + } + kunmap_local(kaddr); + return ret; +} + +/** + * gfs2_jhead_process_page - Search/cleanup a page + * @jd: The journal descriptor + * @index: Index of the page to look into + * @head: The journal head to start from + * @done: If set, perform only cleanup, else search and set if found. + * + * Find the folio with 'index' in the journal's mapping. Search the folio for + * the journal head if requested (cleanup == false). Release refs on the + * folio so the page cache can reclaim it. We grabbed a + * reference on this folio twice, first when we did a filemap_grab_folio() + * to obtain the folio to add it to the bio and second when we do a + * filemap_get_folio() here to get the folio to wait on while I/O on it is being + * completed. + * This function is also used to free up a folio we might've grabbed but not + * used. Maybe we added it to a bio, but not submitted it for I/O. Or we + * submitted the I/O, but we already found the jhead so we only need to drop + * our references to the folio. + */ + +static void gfs2_jhead_process_page(struct gfs2_jdesc *jd, unsigned long index, + struct gfs2_log_header_host *head, + bool *done) +{ + struct folio *folio; + + folio = filemap_get_folio(jd->jd_inode->i_mapping, index); + + folio_wait_locked(folio); + if (!folio_test_uptodate(folio)) + *done = true; + + if (!*done) + *done = gfs2_jhead_folio_search(jd, head, folio); + + /* filemap_get_folio() and the earlier filemap_grab_folio() */ + folio_put_refs(folio, 2); +} + +static struct bio *gfs2_chain_bio(struct bio *prev, unsigned int nr_iovecs, + sector_t sector, blk_opf_t opf) +{ + struct bio *new; + + new = bio_alloc(prev->bi_bdev, nr_iovecs, opf, GFP_NOIO); + bio_clone_blkg_association(new, prev); + new->bi_iter.bi_sector = sector; + bio_chain(new, prev); + submit_bio(prev); + return new; +} + +/** + * gfs2_find_jhead - find the head of a log + * @jd: The journal descriptor + * @head: The log descriptor for the head of the log is returned here + * + * Do a search of a journal by reading it in large chunks using bios and find + * the valid log entry with the highest sequence number. (i.e. the log head) + * + * Returns: 0 on success, errno otherwise + */ +int gfs2_find_jhead(struct gfs2_jdesc *jd, struct gfs2_log_header_host *head) +{ + struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode); + struct address_space *mapping = jd->jd_inode->i_mapping; + unsigned int block = 0, blocks_submitted = 0, blocks_read = 0; + unsigned int bsize = sdp->sd_sb.sb_bsize, off; + unsigned int bsize_shift = sdp->sd_sb.sb_bsize_shift; + unsigned int shift = PAGE_SHIFT - bsize_shift; + unsigned int max_blocks = 2 * 1024 * 1024 >> bsize_shift; + struct gfs2_journal_extent *je; + int ret = 0; + struct bio *bio = NULL; + struct folio *folio = NULL; + bool done = false; + errseq_t since; + + memset(head, 0, sizeof(*head)); + if (list_empty(&jd->extent_list)) + gfs2_map_journal_extents(sdp, jd); + + since = filemap_sample_wb_err(mapping); + list_for_each_entry(je, &jd->extent_list, list) { + u64 dblock = je->dblock; + + for (; block < je->lblock + je->blocks; block++, dblock++) { + if (!folio) { + folio = filemap_grab_folio(mapping, + block >> shift); + if (IS_ERR(folio)) { + ret = PTR_ERR(folio); + done = true; + goto out; + } + off = 0; + } + + if (bio && (off || block < blocks_submitted + max_blocks)) { + sector_t sector = dblock << sdp->sd_fsb2bb_shift; + + if (bio_end_sector(bio) == sector) { + if (bio_add_folio(bio, folio, bsize, off)) + goto block_added; + } + if (off) { + unsigned int blocks = + (PAGE_SIZE - off) >> bsize_shift; + + bio = gfs2_chain_bio(bio, blocks, sector, + REQ_OP_READ); + goto add_block_to_new_bio; + } + } + + if (bio) { + blocks_submitted = block; + submit_bio(bio); + } + + bio = gfs2_log_alloc_bio(sdp, dblock, gfs2_end_log_read, + REQ_OP_READ); +add_block_to_new_bio: + bio_add_folio_nofail(bio, folio, bsize, off); +block_added: + off += bsize; + if (off == folio_size(folio)) + folio = NULL; + if (blocks_submitted <= blocks_read + max_blocks) { + /* Keep at least one bio in flight */ + continue; + } + + gfs2_jhead_process_page(jd, blocks_read >> shift, head, &done); + blocks_read += PAGE_SIZE >> bsize_shift; + if (done) + goto out; /* found */ + } + } + +out: + if (bio) + submit_bio(bio); + while (blocks_read < block) { + gfs2_jhead_process_page(jd, blocks_read >> shift, head, &done); + blocks_read += PAGE_SIZE >> bsize_shift; + } + + if (!ret) + ret = filemap_check_wb_err(mapping, since); + + truncate_inode_pages(mapping, 0); + + return ret; +} + +static struct page *gfs2_get_log_desc(struct gfs2_sbd *sdp, u32 ld_type, + u32 ld_length, u32 ld_data1) +{ + struct page *page = mempool_alloc(gfs2_page_pool, GFP_NOIO); + struct gfs2_log_descriptor *ld = page_address(page); + clear_page(ld); + ld->ld_header.mh_magic = cpu_to_be32(GFS2_MAGIC); + ld->ld_header.mh_type = cpu_to_be32(GFS2_METATYPE_LD); + ld->ld_header.mh_format = cpu_to_be32(GFS2_FORMAT_LD); + ld->ld_type = cpu_to_be32(ld_type); + ld->ld_length = cpu_to_be32(ld_length); + ld->ld_data1 = cpu_to_be32(ld_data1); + ld->ld_data2 = 0; + return page; +} + +static void gfs2_check_magic(struct buffer_head *bh) +{ + __be32 *ptr; + + clear_buffer_escaped(bh); + ptr = kmap_local_folio(bh->b_folio, bh_offset(bh)); + if (*ptr == cpu_to_be32(GFS2_MAGIC)) + set_buffer_escaped(bh); + kunmap_local(ptr); +} + +static int blocknr_cmp(void *priv, const struct list_head *a, + const struct list_head *b) +{ + struct gfs2_bufdata *bda, *bdb; + + bda = list_entry(a, struct gfs2_bufdata, bd_list); + bdb = list_entry(b, struct gfs2_bufdata, bd_list); + + if (bda->bd_bh->b_blocknr < bdb->bd_bh->b_blocknr) + return -1; + if (bda->bd_bh->b_blocknr > bdb->bd_bh->b_blocknr) + return 1; + return 0; +} + +static void gfs2_before_commit(struct gfs2_sbd *sdp, unsigned int limit, + unsigned int total, struct list_head *blist, + bool is_databuf) +{ + struct gfs2_log_descriptor *ld; + struct gfs2_bufdata *bd1 = NULL, *bd2; + struct page *page; + unsigned int num; + unsigned n; + __be64 *ptr; + + spin_lock(&sdp->sd_log_lock); + list_sort(NULL, blist, blocknr_cmp); + bd1 = bd2 = list_prepare_entry(bd1, blist, bd_list); + while(total) { + num = total; + if (total > limit) + num = limit; + spin_unlock(&sdp->sd_log_lock); + page = gfs2_get_log_desc(sdp, + is_databuf ? GFS2_LOG_DESC_JDATA : + GFS2_LOG_DESC_METADATA, num + 1, num); + ld = page_address(page); + spin_lock(&sdp->sd_log_lock); + ptr = (__be64 *)(ld + 1); + + n = 0; + list_for_each_entry_continue(bd1, blist, bd_list) { + *ptr++ = cpu_to_be64(bd1->bd_bh->b_blocknr); + if (is_databuf) { + gfs2_check_magic(bd1->bd_bh); + *ptr++ = cpu_to_be64(buffer_escaped(bd1->bd_bh) ? 1 : 0); + } + if (++n >= num) + break; + } + + spin_unlock(&sdp->sd_log_lock); + gfs2_log_write_page(sdp, page); + spin_lock(&sdp->sd_log_lock); + + n = 0; + list_for_each_entry_continue(bd2, blist, bd_list) { + get_bh(bd2->bd_bh); + spin_unlock(&sdp->sd_log_lock); + lock_buffer(bd2->bd_bh); + + if (buffer_escaped(bd2->bd_bh)) { + void *p; + + page = mempool_alloc(gfs2_page_pool, GFP_NOIO); + p = page_address(page); + memcpy_from_page(p, page, bh_offset(bd2->bd_bh), bd2->bd_bh->b_size); + *(__be32 *)p = 0; + clear_buffer_escaped(bd2->bd_bh); + unlock_buffer(bd2->bd_bh); + brelse(bd2->bd_bh); + gfs2_log_write_page(sdp, page); + } else { + gfs2_log_write_bh(sdp, bd2->bd_bh); + } + spin_lock(&sdp->sd_log_lock); + if (++n >= num) + break; + } + + BUG_ON(total < num); + total -= num; + } + spin_unlock(&sdp->sd_log_lock); +} + +static void buf_lo_before_commit(struct gfs2_sbd *sdp, struct gfs2_trans *tr) +{ + unsigned int limit = buf_limit(sdp); /* 503 for 4k blocks */ + unsigned int nbuf; + if (tr == NULL) + return; + nbuf = tr->tr_num_buf_new - tr->tr_num_buf_rm; + gfs2_before_commit(sdp, limit, nbuf, &tr->tr_buf, 0); +} + +static void buf_lo_after_commit(struct gfs2_sbd *sdp, struct gfs2_trans *tr) +{ + struct list_head *head; + struct gfs2_bufdata *bd; + + if (tr == NULL) + return; + + head = &tr->tr_buf; + while (!list_empty(head)) { + bd = list_first_entry(head, struct gfs2_bufdata, bd_list); + list_del_init(&bd->bd_list); + gfs2_unpin(sdp, bd->bd_bh, tr); + } +} + +static void buf_lo_before_scan(struct gfs2_jdesc *jd, + struct gfs2_log_header_host *head, int pass) +{ + if (pass != 0) + return; + + jd->jd_found_blocks = 0; + jd->jd_replayed_blocks = 0; +} + +#define obsolete_rgrp_replay \ +"Replaying 0x%llx from jid=%d/0x%llx but we already have a bh!\n" +#define obsolete_rgrp_replay2 \ +"busy:%d, pinned:%d rg_gen:0x%llx, j_gen:0x%llx\n" + +static void obsolete_rgrp(struct gfs2_jdesc *jd, struct buffer_head *bh_log, + u64 blkno) +{ + struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode); + struct gfs2_rgrpd *rgd; + struct gfs2_rgrp *jrgd = (struct gfs2_rgrp *)bh_log->b_data; + + rgd = gfs2_blk2rgrpd(sdp, blkno, false); + if (rgd && rgd->rd_addr == blkno && + rgd->rd_bits && rgd->rd_bits->bi_bh) { + fs_info(sdp, obsolete_rgrp_replay, (unsigned long long)blkno, + jd->jd_jid, bh_log->b_blocknr); + fs_info(sdp, obsolete_rgrp_replay2, + buffer_busy(rgd->rd_bits->bi_bh) ? 1 : 0, + buffer_pinned(rgd->rd_bits->bi_bh), + rgd->rd_igeneration, + be64_to_cpu(jrgd->rg_igeneration)); + gfs2_dump_glock(NULL, rgd->rd_gl, true); + } +} + +static int buf_lo_scan_elements(struct gfs2_jdesc *jd, u32 start, + struct gfs2_log_descriptor *ld, __be64 *ptr, + int pass) +{ + struct gfs2_inode *ip = GFS2_I(jd->jd_inode); + struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode); + struct gfs2_glock *gl = ip->i_gl; + unsigned int blks = be32_to_cpu(ld->ld_data1); + struct buffer_head *bh_log, *bh_ip; + u64 blkno; + int error = 0; + + if (pass != 1 || be32_to_cpu(ld->ld_type) != GFS2_LOG_DESC_METADATA) + return 0; + + gfs2_replay_incr_blk(jd, &start); + + for (; blks; gfs2_replay_incr_blk(jd, &start), blks--) { + blkno = be64_to_cpu(*ptr++); + + jd->jd_found_blocks++; + + if (gfs2_revoke_check(jd, blkno, start)) + continue; + + error = gfs2_replay_read_block(jd, start, &bh_log); + if (error) + return error; + + bh_ip = gfs2_meta_new(gl, blkno); + memcpy(bh_ip->b_data, bh_log->b_data, bh_log->b_size); + + if (gfs2_meta_check(sdp, bh_ip)) + error = -EIO; + else { + struct gfs2_meta_header *mh = + (struct gfs2_meta_header *)bh_ip->b_data; + + if (mh->mh_type == cpu_to_be32(GFS2_METATYPE_RG)) + obsolete_rgrp(jd, bh_log, blkno); + + mark_buffer_dirty(bh_ip); + } + brelse(bh_log); + brelse(bh_ip); + + if (error) + break; + + jd->jd_replayed_blocks++; + } + + return error; +} + +static void buf_lo_after_scan(struct gfs2_jdesc *jd, int error, int pass) +{ + struct gfs2_inode *ip = GFS2_I(jd->jd_inode); + struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode); + + if (error) { + gfs2_inode_metasync(ip->i_gl); + return; + } + if (pass != 1) + return; + + gfs2_inode_metasync(ip->i_gl); + + fs_info(sdp, "jid=%u: Replayed %u of %u blocks\n", + jd->jd_jid, jd->jd_replayed_blocks, jd->jd_found_blocks); +} + +static void revoke_lo_before_commit(struct gfs2_sbd *sdp, struct gfs2_trans *tr) +{ + struct gfs2_meta_header *mh; + unsigned int offset; + struct list_head *head = &sdp->sd_log_revokes; + struct gfs2_bufdata *bd; + struct page *page; + unsigned int length; + + gfs2_flush_revokes(sdp); + if (!sdp->sd_log_num_revoke) + return; + + length = gfs2_struct2blk(sdp, sdp->sd_log_num_revoke); + page = gfs2_get_log_desc(sdp, GFS2_LOG_DESC_REVOKE, length, sdp->sd_log_num_revoke); + offset = sizeof(struct gfs2_log_descriptor); + + list_for_each_entry(bd, head, bd_list) { + sdp->sd_log_num_revoke--; + + if (offset + sizeof(u64) > sdp->sd_sb.sb_bsize) { + gfs2_log_write_page(sdp, page); + page = mempool_alloc(gfs2_page_pool, GFP_NOIO); + mh = page_address(page); + clear_page(mh); + mh->mh_magic = cpu_to_be32(GFS2_MAGIC); + mh->mh_type = cpu_to_be32(GFS2_METATYPE_LB); + mh->mh_format = cpu_to_be32(GFS2_FORMAT_LB); + offset = sizeof(struct gfs2_meta_header); + } + + *(__be64 *)(page_address(page) + offset) = cpu_to_be64(bd->bd_blkno); + offset += sizeof(u64); + } + gfs2_assert_withdraw(sdp, !sdp->sd_log_num_revoke); + + gfs2_log_write_page(sdp, page); +} + +void gfs2_drain_revokes(struct gfs2_sbd *sdp) +{ + struct list_head *head = &sdp->sd_log_revokes; + struct gfs2_bufdata *bd; + struct gfs2_glock *gl; + + while (!list_empty(head)) { + bd = list_first_entry(head, struct gfs2_bufdata, bd_list); + list_del_init(&bd->bd_list); + gl = bd->bd_gl; + gfs2_glock_remove_revoke(gl); + kmem_cache_free(gfs2_bufdata_cachep, bd); + } +} + +static void revoke_lo_after_commit(struct gfs2_sbd *sdp, struct gfs2_trans *tr) +{ + gfs2_drain_revokes(sdp); +} + +static void revoke_lo_before_scan(struct gfs2_jdesc *jd, + struct gfs2_log_header_host *head, int pass) +{ + if (pass != 0) + return; + + jd->jd_found_revokes = 0; + jd->jd_replay_tail = head->lh_tail; +} + +static int revoke_lo_scan_elements(struct gfs2_jdesc *jd, u32 start, + struct gfs2_log_descriptor *ld, __be64 *ptr, + int pass) +{ + struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode); + unsigned int blks = be32_to_cpu(ld->ld_length); + unsigned int revokes = be32_to_cpu(ld->ld_data1); + struct buffer_head *bh; + unsigned int offset; + u64 blkno; + int first = 1; + int error; + + if (pass != 0 || be32_to_cpu(ld->ld_type) != GFS2_LOG_DESC_REVOKE) + return 0; + + offset = sizeof(struct gfs2_log_descriptor); + + for (; blks; gfs2_replay_incr_blk(jd, &start), blks--) { + error = gfs2_replay_read_block(jd, start, &bh); + if (error) + return error; + + if (!first) + gfs2_metatype_check(sdp, bh, GFS2_METATYPE_LB); + + while (offset + sizeof(u64) <= sdp->sd_sb.sb_bsize) { + blkno = be64_to_cpu(*(__be64 *)(bh->b_data + offset)); + + error = gfs2_revoke_add(jd, blkno, start); + if (error < 0) { + brelse(bh); + return error; + } + else if (error) + jd->jd_found_revokes++; + + if (!--revokes) + break; + offset += sizeof(u64); + } + + brelse(bh); + offset = sizeof(struct gfs2_meta_header); + first = 0; + } + + return 0; +} + +static void revoke_lo_after_scan(struct gfs2_jdesc *jd, int error, int pass) +{ + struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode); + + if (error) { + gfs2_revoke_clean(jd); + return; + } + if (pass != 1) + return; + + fs_info(sdp, "jid=%u: Found %u revoke tags\n", + jd->jd_jid, jd->jd_found_revokes); + + gfs2_revoke_clean(jd); +} + +/** + * databuf_lo_before_commit - Scan the data buffers, writing as we go + * @sdp: The filesystem + * @tr: The system transaction being flushed + */ + +static void databuf_lo_before_commit(struct gfs2_sbd *sdp, struct gfs2_trans *tr) +{ + unsigned int limit = databuf_limit(sdp); + unsigned int nbuf; + if (tr == NULL) + return; + nbuf = tr->tr_num_databuf_new - tr->tr_num_databuf_rm; + gfs2_before_commit(sdp, limit, nbuf, &tr->tr_databuf, 1); +} + +static int databuf_lo_scan_elements(struct gfs2_jdesc *jd, u32 start, + struct gfs2_log_descriptor *ld, + __be64 *ptr, int pass) +{ + struct gfs2_inode *ip = GFS2_I(jd->jd_inode); + struct gfs2_glock *gl = ip->i_gl; + unsigned int blks = be32_to_cpu(ld->ld_data1); + struct buffer_head *bh_log, *bh_ip; + u64 blkno; + u64 esc; + int error = 0; + + if (pass != 1 || be32_to_cpu(ld->ld_type) != GFS2_LOG_DESC_JDATA) + return 0; + + gfs2_replay_incr_blk(jd, &start); + for (; blks; gfs2_replay_incr_blk(jd, &start), blks--) { + blkno = be64_to_cpu(*ptr++); + esc = be64_to_cpu(*ptr++); + + jd->jd_found_blocks++; + + if (gfs2_revoke_check(jd, blkno, start)) + continue; + + error = gfs2_replay_read_block(jd, start, &bh_log); + if (error) + return error; + + bh_ip = gfs2_meta_new(gl, blkno); + memcpy(bh_ip->b_data, bh_log->b_data, bh_log->b_size); + + /* Unescape */ + if (esc) { + __be32 *eptr = (__be32 *)bh_ip->b_data; + *eptr = cpu_to_be32(GFS2_MAGIC); + } + mark_buffer_dirty(bh_ip); + + brelse(bh_log); + brelse(bh_ip); + + jd->jd_replayed_blocks++; + } + + return error; +} + +/* FIXME: sort out accounting for log blocks etc. */ + +static void databuf_lo_after_scan(struct gfs2_jdesc *jd, int error, int pass) +{ + struct gfs2_inode *ip = GFS2_I(jd->jd_inode); + struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode); + + if (error) { + gfs2_inode_metasync(ip->i_gl); + return; + } + if (pass != 1) + return; + + /* data sync? */ + gfs2_inode_metasync(ip->i_gl); + + fs_info(sdp, "jid=%u: Replayed %u of %u data blocks\n", + jd->jd_jid, jd->jd_replayed_blocks, jd->jd_found_blocks); +} + +static void databuf_lo_after_commit(struct gfs2_sbd *sdp, struct gfs2_trans *tr) +{ + struct list_head *head; + struct gfs2_bufdata *bd; + + if (tr == NULL) + return; + + head = &tr->tr_databuf; + while (!list_empty(head)) { + bd = list_first_entry(head, struct gfs2_bufdata, bd_list); + list_del_init(&bd->bd_list); + gfs2_unpin(sdp, bd->bd_bh, tr); + } +} + + +static const struct gfs2_log_operations gfs2_buf_lops = { + .lo_before_commit = buf_lo_before_commit, + .lo_after_commit = buf_lo_after_commit, + .lo_before_scan = buf_lo_before_scan, + .lo_scan_elements = buf_lo_scan_elements, + .lo_after_scan = buf_lo_after_scan, + .lo_name = "buf", +}; + +static const struct gfs2_log_operations gfs2_revoke_lops = { + .lo_before_commit = revoke_lo_before_commit, + .lo_after_commit = revoke_lo_after_commit, + .lo_before_scan = revoke_lo_before_scan, + .lo_scan_elements = revoke_lo_scan_elements, + .lo_after_scan = revoke_lo_after_scan, + .lo_name = "revoke", +}; + +static const struct gfs2_log_operations gfs2_databuf_lops = { + .lo_before_commit = databuf_lo_before_commit, + .lo_after_commit = databuf_lo_after_commit, + .lo_scan_elements = databuf_lo_scan_elements, + .lo_after_scan = databuf_lo_after_scan, + .lo_name = "databuf", +}; + +const struct gfs2_log_operations *gfs2_log_ops[] = { + &gfs2_databuf_lops, + &gfs2_buf_lops, + &gfs2_revoke_lops, + NULL, +}; + diff --git a/fs/gfs2/lops.h b/fs/gfs2/lops.h new file mode 100644 index 000000000..772557b63 --- /dev/null +++ b/fs/gfs2/lops.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. + */ + +#ifndef __LOPS_DOT_H__ +#define __LOPS_DOT_H__ + +#include +#include "incore.h" + +extern const struct gfs2_log_operations *gfs2_log_ops[]; + +void gfs2_log_incr_head(struct gfs2_sbd *sdp); +u64 gfs2_log_bmap(struct gfs2_jdesc *jd, unsigned int lbn); +void gfs2_log_write(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd, + struct page *page, unsigned size, unsigned offset, + u64 blkno, blk_opf_t opf); +void gfs2_log_submit_write(struct bio **biop); +void gfs2_pin(struct gfs2_sbd *sdp, struct buffer_head *bh); +int gfs2_find_jhead(struct gfs2_jdesc *jd, + struct gfs2_log_header_host *head); +void gfs2_drain_revokes(struct gfs2_sbd *sdp); + +static inline unsigned int buf_limit(struct gfs2_sbd *sdp) +{ + return sdp->sd_ldptrs; +} + +static inline unsigned int databuf_limit(struct gfs2_sbd *sdp) +{ + return sdp->sd_ldptrs / 2; +} + +static inline void lops_before_commit(struct gfs2_sbd *sdp, + struct gfs2_trans *tr) +{ + int x; + for (x = 0; gfs2_log_ops[x]; x++) + if (gfs2_log_ops[x]->lo_before_commit) + gfs2_log_ops[x]->lo_before_commit(sdp, tr); +} + +static inline void lops_after_commit(struct gfs2_sbd *sdp, + struct gfs2_trans *tr) +{ + int x; + for (x = 0; gfs2_log_ops[x]; x++) + if (gfs2_log_ops[x]->lo_after_commit) + gfs2_log_ops[x]->lo_after_commit(sdp, tr); +} + +static inline void lops_before_scan(struct gfs2_jdesc *jd, + struct gfs2_log_header_host *head, + unsigned int pass) +{ + int x; + for (x = 0; gfs2_log_ops[x]; x++) + if (gfs2_log_ops[x]->lo_before_scan) + gfs2_log_ops[x]->lo_before_scan(jd, head, pass); +} + +static inline int lops_scan_elements(struct gfs2_jdesc *jd, u32 start, + struct gfs2_log_descriptor *ld, + __be64 *ptr, + unsigned int pass) +{ + int x, error; + for (x = 0; gfs2_log_ops[x]; x++) + if (gfs2_log_ops[x]->lo_scan_elements) { + error = gfs2_log_ops[x]->lo_scan_elements(jd, start, + ld, ptr, pass); + if (error) + return error; + } + + return 0; +} + +static inline void lops_after_scan(struct gfs2_jdesc *jd, int error, + unsigned int pass) +{ + int x; + for (x = 0; gfs2_log_ops[x]; x++) + if (gfs2_log_ops[x]->lo_before_scan) + gfs2_log_ops[x]->lo_after_scan(jd, error, pass); +} + +#endif /* __LOPS_DOT_H__ */ + diff --git a/fs/gfs2/main.c b/fs/gfs2/main.c new file mode 100644 index 000000000..36c9c06e9 --- /dev/null +++ b/fs/gfs2/main.c @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gfs2.h" +#include "incore.h" +#include "super.h" +#include "sys.h" +#include "util.h" +#include "glock.h" +#include "quota.h" +#include "recovery.h" +#include "dir.h" +#include "glops.h" + +struct workqueue_struct *gfs2_control_wq; + +static void gfs2_init_inode_once(void *foo) +{ + struct gfs2_inode *ip = foo; + + inode_init_once(&ip->i_inode); + atomic_set(&ip->i_sizehint, 0); + init_rwsem(&ip->i_rw_mutex); + INIT_LIST_HEAD(&ip->i_ordered); + ip->i_qadata = NULL; + gfs2_holder_mark_uninitialized(&ip->i_rgd_gh); + memset(&ip->i_res, 0, sizeof(ip->i_res)); + RB_CLEAR_NODE(&ip->i_res.rs_node); + ip->i_hash_cache = NULL; + gfs2_holder_mark_uninitialized(&ip->i_iopen_gh); +} + +static void gfs2_init_glock_once(void *foo) +{ + struct gfs2_glock *gl = foo; + + INIT_LIST_HEAD(&gl->gl_holders); + INIT_LIST_HEAD(&gl->gl_ail_list); + atomic_set(&gl->gl_ail_count, 0); + atomic_set(&gl->gl_revokes, 0); +} + +static void gfs2_init_gl_aspace_once(void *foo) +{ + struct gfs2_glock_aspace *gla = foo; + + gfs2_init_glock_once(&gla->glock); + address_space_init_once(&gla->mapping); +} + +/** + * init_gfs2_fs - Register GFS2 as a filesystem + * + * Returns: 0 on success, error code on failure + */ + +static int __init init_gfs2_fs(void) +{ + int error; + + gfs2_str2qstr(&gfs2_qdot, "."); + gfs2_str2qstr(&gfs2_qdotdot, ".."); + gfs2_quota_hash_init(); + + error = gfs2_sys_init(); + if (error) + return error; + + error = list_lru_init(&gfs2_qd_lru); + if (error) + goto fail_lru; + + error = gfs2_glock_init(); + if (error) + goto fail_glock; + + error = -ENOMEM; + gfs2_glock_cachep = kmem_cache_create("gfs2_glock", + sizeof(struct gfs2_glock), + 0, SLAB_RECLAIM_ACCOUNT, + gfs2_init_glock_once); + if (!gfs2_glock_cachep) + goto fail_cachep1; + + gfs2_glock_aspace_cachep = kmem_cache_create("gfs2_glock(aspace)", + sizeof(struct gfs2_glock_aspace), + 0, 0, gfs2_init_gl_aspace_once); + + if (!gfs2_glock_aspace_cachep) + goto fail_cachep2; + + gfs2_inode_cachep = kmem_cache_create("gfs2_inode", + sizeof(struct gfs2_inode), + 0, SLAB_RECLAIM_ACCOUNT| + SLAB_ACCOUNT, + gfs2_init_inode_once); + if (!gfs2_inode_cachep) + goto fail_cachep3; + + gfs2_bufdata_cachep = kmem_cache_create("gfs2_bufdata", + sizeof(struct gfs2_bufdata), + 0, 0, NULL); + if (!gfs2_bufdata_cachep) + goto fail_cachep4; + + gfs2_rgrpd_cachep = kmem_cache_create("gfs2_rgrpd", + sizeof(struct gfs2_rgrpd), + 0, 0, NULL); + if (!gfs2_rgrpd_cachep) + goto fail_cachep5; + + gfs2_quotad_cachep = kmem_cache_create("gfs2_quotad", + sizeof(struct gfs2_quota_data), + 0, SLAB_RECLAIM_ACCOUNT, NULL); + if (!gfs2_quotad_cachep) + goto fail_cachep6; + + gfs2_qadata_cachep = kmem_cache_create("gfs2_qadata", + sizeof(struct gfs2_qadata), + 0, 0, NULL); + if (!gfs2_qadata_cachep) + goto fail_cachep7; + + gfs2_trans_cachep = kmem_cache_create("gfs2_trans", + sizeof(struct gfs2_trans), + 0, 0, NULL); + if (!gfs2_trans_cachep) + goto fail_cachep8; + + error = gfs2_qd_shrinker_init(); + if (error) + goto fail_shrinker; + + error = -ENOMEM; + gfs2_recovery_wq = alloc_workqueue("gfs2_recovery", + WQ_MEM_RECLAIM | WQ_FREEZABLE | WQ_PERCPU, + 0); + if (!gfs2_recovery_wq) + goto fail_wq1; + + gfs2_control_wq = alloc_workqueue("gfs2_control", + WQ_UNBOUND | WQ_FREEZABLE, 0); + if (!gfs2_control_wq) + goto fail_wq2; + + gfs2_freeze_wq = alloc_workqueue("gfs2_freeze", WQ_PERCPU, 0); + + if (!gfs2_freeze_wq) + goto fail_wq3; + + gfs2_page_pool = mempool_create_page_pool(64, 0); + if (!gfs2_page_pool) + goto fail_mempool; + + gfs2_register_debugfs(); + error = register_filesystem(&gfs2_fs_type); + if (error) + goto fail_fs1; + + error = register_filesystem(&gfs2meta_fs_type); + if (error) + goto fail_fs2; + + + pr_info("GFS2 installed\n"); + + return 0; + +fail_fs2: + unregister_filesystem(&gfs2_fs_type); +fail_fs1: + mempool_destroy(gfs2_page_pool); +fail_mempool: + destroy_workqueue(gfs2_freeze_wq); +fail_wq3: + destroy_workqueue(gfs2_control_wq); +fail_wq2: + destroy_workqueue(gfs2_recovery_wq); +fail_wq1: + gfs2_qd_shrinker_exit(); +fail_shrinker: + kmem_cache_destroy(gfs2_trans_cachep); +fail_cachep8: + kmem_cache_destroy(gfs2_qadata_cachep); +fail_cachep7: + kmem_cache_destroy(gfs2_quotad_cachep); +fail_cachep6: + kmem_cache_destroy(gfs2_rgrpd_cachep); +fail_cachep5: + kmem_cache_destroy(gfs2_bufdata_cachep); +fail_cachep4: + kmem_cache_destroy(gfs2_inode_cachep); +fail_cachep3: + kmem_cache_destroy(gfs2_glock_aspace_cachep); +fail_cachep2: + kmem_cache_destroy(gfs2_glock_cachep); +fail_cachep1: + gfs2_glock_exit(); +fail_glock: + list_lru_destroy(&gfs2_qd_lru); +fail_lru: + gfs2_sys_uninit(); + return error; +} + +/** + * exit_gfs2_fs - Unregister the file system + * + */ + +static void __exit exit_gfs2_fs(void) +{ + gfs2_qd_shrinker_exit(); + gfs2_glock_exit(); + gfs2_unregister_debugfs(); + unregister_filesystem(&gfs2_fs_type); + unregister_filesystem(&gfs2meta_fs_type); + destroy_workqueue(gfs2_recovery_wq); + destroy_workqueue(gfs2_control_wq); + destroy_workqueue(gfs2_freeze_wq); + list_lru_destroy(&gfs2_qd_lru); + + rcu_barrier(); + + mempool_destroy(gfs2_page_pool); + kmem_cache_destroy(gfs2_trans_cachep); + kmem_cache_destroy(gfs2_qadata_cachep); + kmem_cache_destroy(gfs2_quotad_cachep); + kmem_cache_destroy(gfs2_rgrpd_cachep); + kmem_cache_destroy(gfs2_bufdata_cachep); + kmem_cache_destroy(gfs2_inode_cachep); + kmem_cache_destroy(gfs2_glock_aspace_cachep); + kmem_cache_destroy(gfs2_glock_cachep); + + gfs2_sys_uninit(); +} + +MODULE_DESCRIPTION("Global File System"); +MODULE_AUTHOR("Red Hat, Inc."); +MODULE_LICENSE("GPL"); + +module_init(init_gfs2_fs); +module_exit(exit_gfs2_fs); + diff --git a/fs/gfs2/meta_io.c b/fs/gfs2/meta_io.c new file mode 100644 index 000000000..a87cfbf0d --- /dev/null +++ b/fs/gfs2/meta_io.c @@ -0,0 +1,516 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gfs2.h" +#include "incore.h" +#include "glock.h" +#include "glops.h" +#include "inode.h" +#include "log.h" +#include "lops.h" +#include "meta_io.h" +#include "rgrp.h" +#include "trans.h" +#include "util.h" +#include "trace_gfs2.h" + +static void gfs2_aspace_write_folio(struct folio *folio, + struct writeback_control *wbc) +{ + struct buffer_head *bh, *head; + int nr_underway = 0; + blk_opf_t write_flags = REQ_META | REQ_PRIO | wbc_to_write_flags(wbc); + + BUG_ON(!folio_test_locked(folio)); + + head = folio_buffers(folio); + bh = head; + + do { + if (!buffer_mapped(bh)) + continue; + /* + * If it's a fully non-blocking write attempt and we cannot + * lock the buffer then redirty the page. Note that this can + * potentially cause a busy-wait loop from flusher thread and kswapd + * activity, but those code paths have their own higher-level + * throttling. + */ + if (wbc->sync_mode != WB_SYNC_NONE) { + lock_buffer(bh); + } else if (!trylock_buffer(bh)) { + folio_redirty_for_writepage(wbc, folio); + continue; + } + if (test_clear_buffer_dirty(bh)) { + set_buffer_async_write(bh); + } else { + unlock_buffer(bh); + } + } while ((bh = bh->b_this_page) != head); + + /* + * The folio and its buffers are protected from truncation by + * the writeback flag, so we can drop the bh refcounts early. + */ + BUG_ON(folio_test_writeback(folio)); + folio_start_writeback(folio); + + do { + struct buffer_head *next = bh->b_this_page; + if (buffer_async_write(bh)) { + bh_submit(bh, REQ_OP_WRITE | write_flags, + bh_end_async_write); + nr_underway++; + } + bh = next; + } while (bh != head); + folio_unlock(folio); + + if (nr_underway == 0) + folio_end_writeback(folio); +} + +static int gfs2_aspace_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct folio *folio = NULL; + int error; + + while ((folio = writeback_iter(mapping, wbc, folio, &error))) + gfs2_aspace_write_folio(folio, wbc); + + return error; +} + +const struct address_space_operations gfs2_meta_aops = { + .dirty_folio = block_dirty_folio, + .invalidate_folio = block_invalidate_folio, + .writepages = gfs2_aspace_writepages, + .release_folio = gfs2_release_folio, + .migrate_folio = buffer_migrate_folio_norefs, +}; + +const struct address_space_operations gfs2_rgrp_aops = { + .dirty_folio = block_dirty_folio, + .invalidate_folio = block_invalidate_folio, + .writepages = gfs2_aspace_writepages, + .release_folio = gfs2_release_folio, + .migrate_folio = buffer_migrate_folio_norefs, +}; + +/** + * gfs2_getbuf - Get a buffer with a given address space + * @gl: the glock + * @blkno: the block number (filesystem scope) + * @create: 1 if the buffer should be created + * + * Returns: the buffer + */ + +struct buffer_head *gfs2_getbuf(struct gfs2_glock *gl, u64 blkno, int create) +{ + struct address_space *mapping = gfs2_glock2aspace(gl); + struct gfs2_sbd *sdp = glock_sbd(gl); + struct folio *folio; + struct buffer_head *bh; + unsigned int shift; + unsigned long index; + unsigned int bufnum; + + if (mapping == NULL) + mapping = gfs2_aspace(sdp); + + shift = PAGE_SHIFT - sdp->sd_sb.sb_bsize_shift; + index = blkno >> shift; /* convert block to page */ + bufnum = blkno - (index << shift); /* block buf index within page */ + + if (create) { + folio = __filemap_get_folio(mapping, index, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, + mapping_gfp_mask(mapping) | __GFP_NOFAIL); + bh = folio_buffers(folio); + if (!bh) + bh = create_empty_buffers(folio, + sdp->sd_sb.sb_bsize, 0); + } else { + folio = __filemap_get_folio(mapping, index, + FGP_LOCK | FGP_ACCESSED, 0); + if (IS_ERR(folio)) + return NULL; + bh = folio_buffers(folio); + } + + if (!bh) + goto out_unlock; + + bh = get_nth_bh(bh, bufnum); + if (!buffer_mapped(bh)) + map_bh(bh, sdp->sd_vfs, blkno); + +out_unlock: + folio_unlock(folio); + folio_put(folio); + + return bh; +} + +static void meta_prep_new(struct buffer_head *bh) +{ + struct gfs2_meta_header *mh = (struct gfs2_meta_header *)bh->b_data; + + lock_buffer(bh); + clear_buffer_dirty(bh); + set_buffer_uptodate(bh); + unlock_buffer(bh); + + mh->mh_magic = cpu_to_be32(GFS2_MAGIC); +} + +/** + * gfs2_meta_new - Get a block + * @gl: The glock associated with this block + * @blkno: The block number + * + * Returns: The buffer + */ + +struct buffer_head *gfs2_meta_new(struct gfs2_glock *gl, u64 blkno) +{ + struct buffer_head *bh; + bh = gfs2_getbuf(gl, blkno, CREATE); + meta_prep_new(bh); + return bh; +} + +static void gfs2_meta_read_endio(struct bio *bio) +{ + struct folio_iter fi; + + bio_for_each_folio_all(fi, bio) { + struct folio *folio = fi.folio; + struct buffer_head *bh = folio_buffers(folio); + size_t len = fi.length; + + while (bh_offset(bh) < fi.offset) + bh = bh->b_this_page; + do { + struct buffer_head *next = bh->b_this_page; + len -= bh->b_size; + end_buffer_read_sync(bh, bio->bi_status == BLK_STS_OK); + bh = next; + } while (bh && len); + } + bio_put(bio); +} + +/* + * Submit several consecutive buffer head I/O requests as a single bio I/O + * request. (See bh_submit.) + */ +static void gfs2_submit_bhs(blk_opf_t opf, struct buffer_head *bhs[], int num) +{ + while (num > 0) { + struct buffer_head *bh = *bhs; + struct bio *bio; + + bio = bio_alloc(bh->b_bdev, num, opf, GFP_NOIO); + bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> SECTOR_SHIFT); + while (num > 0) { + bh = *bhs; + if (!bio_add_folio(bio, bh->b_folio, bh->b_size, bh_offset(bh))) { + BUG_ON(bio->bi_iter.bi_size == 0); + break; + } + bhs++; + num--; + } + bio->bi_end_io = gfs2_meta_read_endio; + submit_bio(bio); + } +} + +/** + * gfs2_meta_read - Read a block from disk + * @gl: The glock covering the block + * @blkno: The block number + * @flags: flags + * @rahead: Do read-ahead + * @bhp: the place where the buffer is returned (NULL on failure) + * + * Returns: errno + */ + +int gfs2_meta_read(struct gfs2_glock *gl, u64 blkno, int flags, + int rahead, struct buffer_head **bhp) +{ + struct gfs2_sbd *sdp = glock_sbd(gl); + struct buffer_head *bh, *bhs[2]; + int num = 0; + + if (gfs2_withdrawn(sdp)) { + *bhp = NULL; + return -EIO; + } + + *bhp = bh = gfs2_getbuf(gl, blkno, CREATE); + + lock_buffer(bh); + if (buffer_uptodate(bh)) { + unlock_buffer(bh); + flags &= ~DIO_WAIT; + } else { + get_bh(bh); + bhs[num++] = bh; + } + + if (rahead) { + bh = gfs2_getbuf(gl, blkno + 1, CREATE); + + lock_buffer(bh); + if (buffer_uptodate(bh)) { + unlock_buffer(bh); + } else { + bhs[num++] = bh; + } + brelse(bh); + } + + gfs2_submit_bhs(REQ_OP_READ | REQ_META | REQ_PRIO, bhs, num); + if (!(flags & DIO_WAIT)) + return 0; + + bh = *bhp; + wait_on_buffer(bh); + if (unlikely(!buffer_uptodate(bh))) { + struct gfs2_trans *tr = current->journal_info; + if (tr && test_bit(TR_TOUCHED, &tr->tr_flags)) + gfs2_io_error_bh(sdp, bh); + brelse(bh); + *bhp = NULL; + return -EIO; + } + + return 0; +} + +/** + * gfs2_meta_wait - Reread a block from disk + * @sdp: the filesystem + * @bh: The block to wait for + * + * Returns: errno + */ + +int gfs2_meta_wait(struct gfs2_sbd *sdp, struct buffer_head *bh) +{ + if (gfs2_withdrawn(sdp)) + return -EIO; + + wait_on_buffer(bh); + + if (!buffer_uptodate(bh)) { + struct gfs2_trans *tr = current->journal_info; + if (tr && test_bit(TR_TOUCHED, &tr->tr_flags)) + gfs2_io_error_bh(sdp, bh); + return -EIO; + } + if (gfs2_withdrawn(sdp)) + return -EIO; + + return 0; +} + +/** + * gfs2_ail1_wipe - remove deleted/freed buffers from the ail1 list + * @sdp: superblock + * @bstart: starting block address of buffers to remove + * @blen: length of buffers to be removed + * + * This function is called from gfs2_journal wipe, whose job is to remove + * buffers, corresponding to deleted blocks, from the journal. If we find any + * bufdata elements on the system ail1 list, they haven't been written to + * the journal yet. So we remove them. + */ +static void gfs2_ail1_wipe(struct gfs2_sbd *sdp, u64 bstart, u32 blen) +{ + struct gfs2_trans *tr, *s; + struct gfs2_bufdata *bd, *bs; + struct buffer_head *bh; + u64 end = bstart + blen; + + spin_lock(&sdp->sd_log_lock); + spin_lock(&sdp->sd_ail_lock); + list_for_each_entry_safe(tr, s, &sdp->sd_ail1_list, tr_list) { + list_for_each_entry_safe(bd, bs, &tr->tr_ail1_list, + bd_ail_st_list) { + bh = bd->bd_bh; + if (bh->b_blocknr < bstart || bh->b_blocknr >= end) + continue; + + gfs2_remove_from_journal(bh, REMOVE_JDATA); + } + } + spin_unlock(&sdp->sd_ail_lock); + spin_unlock(&sdp->sd_log_lock); +} + +static struct buffer_head *gfs2_getjdatabuf(struct gfs2_inode *ip, u64 blkno) +{ + struct address_space *mapping = ip->i_inode.i_mapping; + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + struct folio *folio; + struct buffer_head *bh; + unsigned int shift = PAGE_SHIFT - sdp->sd_sb.sb_bsize_shift; + unsigned long index = blkno >> shift; /* convert block to page */ + unsigned int bufnum = blkno - (index << shift); + + folio = __filemap_get_folio(mapping, index, FGP_LOCK | FGP_ACCESSED, 0); + if (IS_ERR(folio)) + return NULL; + bh = folio_buffers(folio); + if (bh) + bh = get_nth_bh(bh, bufnum); + folio_unlock(folio); + folio_put(folio); + return bh; +} + +/** + * gfs2_journal_wipe - make inode's buffers so they aren't dirty/pinned anymore + * @ip: the inode who owns the buffers + * @bstart: the first buffer in the run + * @blen: the number of buffers in the run + * + */ + +void gfs2_journal_wipe(struct gfs2_inode *ip, u64 bstart, u32 blen) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + struct buffer_head *bh; + int ty; + + /* This can only happen during incomplete inode creation. */ + if (!ip->i_gl) + return; + + gfs2_ail1_wipe(sdp, bstart, blen); + while (blen) { + ty = REMOVE_META; + bh = gfs2_getbuf(ip->i_gl, bstart, NO_CREATE); + if (!bh && gfs2_is_jdata(ip)) { + bh = gfs2_getjdatabuf(ip, bstart); + ty = REMOVE_JDATA; + } + if (bh) { + lock_buffer(bh); + spin_lock(&sdp->sd_log_lock); + spin_lock(&sdp->sd_ail_lock); + gfs2_remove_from_journal(bh, ty); + spin_unlock(&sdp->sd_ail_lock); + spin_unlock(&sdp->sd_log_lock); + unlock_buffer(bh); + brelse(bh); + } + + bstart++; + blen--; + } +} + +/** + * gfs2_meta_buffer - Get a metadata buffer + * @ip: The GFS2 inode + * @mtype: The block type (GFS2_METATYPE_*) + * @num: The block number (device relative) of the buffer + * @bhp: the buffer is returned here + * + * Returns: errno + */ + +int gfs2_meta_buffer(struct gfs2_inode *ip, u32 mtype, u64 num, + struct buffer_head **bhp) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + struct gfs2_glock *gl = ip->i_gl; + struct buffer_head *bh; + int ret = 0; + int rahead = 0; + + if (num == ip->i_no_addr) + rahead = ip->i_rahead; + + ret = gfs2_meta_read(gl, num, DIO_WAIT, rahead, &bh); + if (ret == 0 && gfs2_metatype_check(sdp, bh, mtype)) { + brelse(bh); + ret = -EIO; + } else { + *bhp = bh; + } + return ret; +} + +/** + * gfs2_meta_ra - start readahead on an extent of a file + * @gl: the glock the blocks belong to + * @dblock: the starting disk block + * @extlen: the number of blocks in the extent + * + * returns: the first buffer in the extent + */ + +struct buffer_head *gfs2_meta_ra(struct gfs2_glock *gl, u64 dblock, u32 extlen) +{ + struct gfs2_sbd *sdp = glock_sbd(gl); + struct buffer_head *first_bh, *bh; + u32 max_ra = gfs2_tune_get(sdp, gt_max_readahead) >> + sdp->sd_sb.sb_bsize_shift; + + BUG_ON(!extlen); + + if (max_ra < 1) + max_ra = 1; + if (extlen > max_ra) + extlen = max_ra; + + first_bh = gfs2_getbuf(gl, dblock, CREATE); + + if (buffer_uptodate(first_bh)) + goto out; + bh_read_nowait(first_bh, REQ_META | REQ_PRIO); + + dblock++; + extlen--; + + while (extlen) { + bh = gfs2_getbuf(gl, dblock, CREATE); + + bh_readahead(bh, REQ_RAHEAD | REQ_META | REQ_PRIO); + brelse(bh); + dblock++; + extlen--; + if (!buffer_locked(first_bh) && buffer_uptodate(first_bh)) + goto out; + } + + wait_on_buffer(first_bh); +out: + return first_bh; +} + diff --git a/fs/gfs2/meta_io.h b/fs/gfs2/meta_io.h new file mode 100644 index 000000000..ec76215bf --- /dev/null +++ b/fs/gfs2/meta_io.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#ifndef __DIO_DOT_H__ +#define __DIO_DOT_H__ + +#include +#include +#include "incore.h" + +static inline void gfs2_buffer_clear(struct buffer_head *bh) +{ + memset(bh->b_data, 0, bh->b_size); +} + +static inline void gfs2_buffer_clear_tail(struct buffer_head *bh, int head) +{ + BUG_ON(head > bh->b_size); + memset(bh->b_data + head, 0, bh->b_size - head); +} + +static inline void gfs2_buffer_copy_tail(struct buffer_head *to_bh, + int to_head, + struct buffer_head *from_bh, + int from_head) +{ + BUG_ON(from_head < to_head); + memcpy(to_bh->b_data + to_head, from_bh->b_data + from_head, + from_bh->b_size - from_head); + memset(to_bh->b_data + to_bh->b_size + to_head - from_head, + 0, from_head - to_head); +} + +extern const struct address_space_operations gfs2_meta_aops; +extern const struct address_space_operations gfs2_rgrp_aops; + +static inline struct gfs2_sbd *gfs2_mapping2sbd(struct address_space *mapping) +{ + struct inode *inode = mapping->host; + if (mapping->a_ops == &gfs2_meta_aops) { + struct gfs2_glock_aspace *gla = + container_of(mapping, struct gfs2_glock_aspace, mapping); + return glock_sbd(&gla->glock); + } else + return inode->i_sb->s_fs_info; +} + +struct buffer_head *gfs2_meta_new(struct gfs2_glock *gl, u64 blkno); +int gfs2_meta_read(struct gfs2_glock *gl, u64 blkno, int flags, + int rahead, struct buffer_head **bhp); +int gfs2_meta_wait(struct gfs2_sbd *sdp, struct buffer_head *bh); +struct buffer_head *gfs2_getbuf(struct gfs2_glock *gl, u64 blkno, + int create); +enum { + REMOVE_JDATA = 0, + REMOVE_META = 1, +}; + +void gfs2_journal_wipe(struct gfs2_inode *ip, u64 bstart, u32 blen); +int gfs2_meta_buffer(struct gfs2_inode *ip, u32 mtype, u64 num, + struct buffer_head **bhp); + +static inline int gfs2_meta_inode_buffer(struct gfs2_inode *ip, + struct buffer_head **bhp) +{ + return gfs2_meta_buffer(ip, GFS2_METATYPE_DI, ip->i_no_addr, bhp); +} + +struct buffer_head *gfs2_meta_ra(struct gfs2_glock *gl, u64 dblock, u32 extlen); + +#define buffer_busy(bh) \ +((bh)->b_state & ((1ul << BH_Dirty) | (1ul << BH_Lock) | (1ul << BH_Pinned))) + +#endif /* __DIO_DOT_H__ */ + diff --git a/fs/gfs2/ops_fstype.c b/fs/gfs2/ops_fstype.c new file mode 100644 index 000000000..718e0da7d --- /dev/null +++ b/fs/gfs2/ops_fstype.c @@ -0,0 +1,1817 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gfs2.h" +#include "incore.h" +#include "bmap.h" +#include "glock.h" +#include "glops.h" +#include "inode.h" +#include "recovery.h" +#include "rgrp.h" +#include "super.h" +#include "sys.h" +#include "util.h" +#include "log.h" +#include "quota.h" +#include "dir.h" +#include "meta_io.h" +#include "trace_gfs2.h" +#include "lops.h" + +#define DO 0 +#define UNDO 1 + +/** + * gfs2_tune_init - Fill a gfs2_tune structure with default values + * @gt: tune + * + */ + +static void gfs2_tune_init(struct gfs2_tune *gt) +{ + spin_lock_init(>->gt_spin); + + gt->gt_quota_warn_period = 10; + gt->gt_quota_scale_num = 1; + gt->gt_quota_scale_den = 1; + gt->gt_new_files_jdata = 0; + gt->gt_max_readahead = BIT(18); + gt->gt_complain_secs = 10; + gt->gt_withdraw_helper_timeout = 5; +} + +void free_sbd(struct gfs2_sbd *sdp) +{ + struct super_block *sb = sdp->sd_vfs; + + free_percpu(sdp->sd_lkstats); + sb->s_fs_info = NULL; + kfree(sdp); +} + +static struct gfs2_sbd *init_sbd(struct super_block *sb) +{ + struct gfs2_sbd *sdp; + + sdp = kzalloc_obj(struct gfs2_sbd); + if (!sdp) + return NULL; + + sdp->sd_vfs = sb; + sdp->sd_lkstats = alloc_percpu(struct gfs2_pcpu_lkstats); + if (!sdp->sd_lkstats) + goto fail; + sb->s_fs_info = sdp; + + set_bit(SDF_NOJOURNALID, &sdp->sd_flags); + gfs2_tune_init(&sdp->sd_tune); + + init_waitqueue_head(&sdp->sd_kill_wait); + init_waitqueue_head(&sdp->sd_async_glock_wait); + atomic_set(&sdp->sd_glock_disposal, 0); + init_completion(&sdp->sd_locking_init); + init_completion(&sdp->sd_withdraw_helper); + spin_lock_init(&sdp->sd_statfs_spin); + + spin_lock_init(&sdp->sd_rindex_spin); + sdp->sd_rindex_tree.rb_node = NULL; + + INIT_LIST_HEAD(&sdp->sd_jindex_list); + spin_lock_init(&sdp->sd_jindex_spin); + mutex_init(&sdp->sd_jindex_mutex); + init_completion(&sdp->sd_journal_ready); + + INIT_LIST_HEAD(&sdp->sd_quota_list); + mutex_init(&sdp->sd_quota_sync_mutex); + init_waitqueue_head(&sdp->sd_quota_wait); + spin_lock_init(&sdp->sd_bitmap_lock); + + INIT_LIST_HEAD(&sdp->sd_sc_inodes_list); + + spin_lock_init(&sdp->sd_log_lock); + atomic_set(&sdp->sd_log_pinned, 0); + INIT_LIST_HEAD(&sdp->sd_log_revokes); + INIT_LIST_HEAD(&sdp->sd_log_ordered); + spin_lock_init(&sdp->sd_ordered_lock); + + init_waitqueue_head(&sdp->sd_log_waitq); + init_waitqueue_head(&sdp->sd_logd_waitq); + spin_lock_init(&sdp->sd_ail_lock); + INIT_LIST_HEAD(&sdp->sd_ail1_list); + INIT_LIST_HEAD(&sdp->sd_ail2_list); + + spin_lock_init(&sdp->sd_dead_lock); + + init_rwsem(&sdp->sd_log_flush_lock); + atomic_set(&sdp->sd_log_in_flight, 0); + init_waitqueue_head(&sdp->sd_log_flush_wait); + mutex_init(&sdp->sd_freeze_mutex); + INIT_LIST_HEAD(&sdp->sd_dead_glocks); + + return sdp; + +fail: + free_sbd(sdp); + return NULL; +} + +/** + * gfs2_check_sb - Check superblock + * @sdp: the filesystem + * @silent: Don't print a message if the check fails + * + * Checks the version code of the FS is one that we understand how to + * read and that the sizes of the various on-disk structures have not + * changed. + */ + +static int gfs2_check_sb(struct gfs2_sbd *sdp, int silent) +{ + struct gfs2_sb_host *sb = &sdp->sd_sb; + + if (sb->sb_magic != GFS2_MAGIC || + sb->sb_type != GFS2_METATYPE_SB) { + if (!silent) + pr_warn("not a GFS2 filesystem\n"); + return -EINVAL; + } + + if (sb->sb_fs_format < GFS2_FS_FORMAT_MIN || + sb->sb_fs_format > GFS2_FS_FORMAT_MAX || + sb->sb_multihost_format != GFS2_FORMAT_MULTI) { + fs_warn(sdp, "Unknown on-disk format, unable to mount\n"); + return -EINVAL; + } + + if (sb->sb_bsize < SECTOR_SIZE || sb->sb_bsize > PAGE_SIZE || + (sb->sb_bsize & (sb->sb_bsize - 1))) { + pr_warn("Invalid block size\n"); + return -EINVAL; + } + if (sb->sb_bsize_shift != ffs(sb->sb_bsize) - 1) { + pr_warn("Invalid block size shift\n"); + return -EINVAL; + } + return 0; +} + +static void gfs2_sb_in(struct gfs2_sbd *sdp, const struct gfs2_sb *str) +{ + struct gfs2_sb_host *sb = &sdp->sd_sb; + struct super_block *s = sdp->sd_vfs; + + sb->sb_magic = be32_to_cpu(str->sb_header.mh_magic); + sb->sb_type = be32_to_cpu(str->sb_header.mh_type); + sb->sb_fs_format = be32_to_cpu(str->sb_fs_format); + sb->sb_multihost_format = be32_to_cpu(str->sb_multihost_format); + sb->sb_bsize = be32_to_cpu(str->sb_bsize); + sb->sb_bsize_shift = be32_to_cpu(str->sb_bsize_shift); + sb->sb_master_dir.no_addr = be64_to_cpu(str->sb_master_dir.no_addr); + sb->sb_master_dir.no_formal_ino = be64_to_cpu(str->sb_master_dir.no_formal_ino); + sb->sb_root_dir.no_addr = be64_to_cpu(str->sb_root_dir.no_addr); + sb->sb_root_dir.no_formal_ino = be64_to_cpu(str->sb_root_dir.no_formal_ino); + + memcpy(sb->sb_lockproto, str->sb_lockproto, GFS2_LOCKNAME_LEN); + memcpy(sb->sb_locktable, str->sb_locktable, GFS2_LOCKNAME_LEN); + super_set_uuid(s, str->sb_uuid, 16); +} + +/** + * gfs2_read_super - Read the gfs2 super block from disk + * @sdp: The GFS2 super block + * @sector: The location of the super block + * @silent: Don't print a message if the check fails + * + * This uses the bio functions to read the super block from disk + * because we want to be 100% sure that we never read cached data. + * A super block is read twice only during each GFS2 mount and is + * never written to by the filesystem. The first time its read no + * locks are held, and the only details which are looked at are those + * relating to the locking protocol. Once locking is up and working, + * the sb is read again under the lock to establish the location of + * the master directory (contains pointers to journals etc) and the + * root directory. + * + * Returns: 0 on success or error + */ + +static int gfs2_read_super(struct gfs2_sbd *sdp, sector_t sector, int silent) +{ + struct gfs2_sb *sb; + int err; + + sb = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (unlikely(!sb)) + return -ENOMEM; + err = bdev_rw_virt(sdp->sd_vfs->s_bdev, + sector << (sdp->sd_vfs->s_blocksize_bits - SECTOR_SHIFT), + sb, PAGE_SIZE, REQ_OP_READ | REQ_META); + if (err) { + pr_warn("error %d reading superblock\n", err); + kfree(sb); + return err; + } + gfs2_sb_in(sdp, sb); + kfree(sb); + return gfs2_check_sb(sdp, silent); +} + +/** + * gfs2_read_sb - Read super block + * @sdp: The GFS2 superblock + * @silent: Don't print message if mount fails + * + */ + +static int gfs2_read_sb(struct gfs2_sbd *sdp, int silent) +{ + u32 hash_blocks, ind_blocks, leaf_blocks; + u32 tmp_blocks; + unsigned int x; + int error; + + error = gfs2_read_super(sdp, GFS2_SB_ADDR >> sdp->sd_fsb2bb_shift, silent); + if (error) { + if (!silent) + fs_err(sdp, "can't read superblock\n"); + return error; + } + + sdp->sd_fsb2bb_shift = sdp->sd_sb.sb_bsize_shift - SECTOR_SHIFT; + sdp->sd_fsb2bb = BIT(sdp->sd_fsb2bb_shift); + sdp->sd_diptrs = (sdp->sd_sb.sb_bsize - + sizeof(struct gfs2_dinode)) / sizeof(u64); + sdp->sd_inptrs = (sdp->sd_sb.sb_bsize - + sizeof(struct gfs2_meta_header)) / sizeof(u64); + sdp->sd_ldptrs = (sdp->sd_sb.sb_bsize - + sizeof(struct gfs2_log_descriptor)) / sizeof(u64); + sdp->sd_jbsize = sdp->sd_sb.sb_bsize - sizeof(struct gfs2_meta_header); + sdp->sd_hash_bsize = sdp->sd_sb.sb_bsize / 2; + sdp->sd_hash_bsize_shift = sdp->sd_sb.sb_bsize_shift - 1; + sdp->sd_hash_ptrs = sdp->sd_hash_bsize / sizeof(u64); + sdp->sd_qc_per_block = (sdp->sd_sb.sb_bsize - + sizeof(struct gfs2_meta_header)) / + sizeof(struct gfs2_quota_change); + sdp->sd_blocks_per_bitmap = (sdp->sd_sb.sb_bsize - + sizeof(struct gfs2_meta_header)) + * GFS2_NBBY; /* not the rgrp bitmap, subsequent bitmaps only */ + + /* + * We always keep at least one block reserved for revokes in + * transactions. This greatly simplifies allocating additional + * revoke blocks. + */ + atomic_set(&sdp->sd_log_revokes_available, sdp->sd_ldptrs); + + /* Compute maximum reservation required to add a entry to a directory */ + + hash_blocks = DIV_ROUND_UP(sizeof(u64) * BIT(GFS2_DIR_MAX_DEPTH), + sdp->sd_jbsize); + + ind_blocks = 0; + for (tmp_blocks = hash_blocks; tmp_blocks > sdp->sd_diptrs;) { + tmp_blocks = DIV_ROUND_UP(tmp_blocks, sdp->sd_inptrs); + ind_blocks += tmp_blocks; + } + + leaf_blocks = 2 + GFS2_DIR_MAX_DEPTH; + + sdp->sd_max_dirres = hash_blocks + ind_blocks + leaf_blocks; + + sdp->sd_heightsize[0] = sdp->sd_sb.sb_bsize - + sizeof(struct gfs2_dinode); + sdp->sd_heightsize[1] = sdp->sd_sb.sb_bsize * sdp->sd_diptrs; + for (x = 2;; x++) { + u64 space, d; + u32 m; + + space = sdp->sd_heightsize[x - 1] * sdp->sd_inptrs; + d = space; + m = do_div(d, sdp->sd_inptrs); + + if (d != sdp->sd_heightsize[x - 1] || m) + break; + sdp->sd_heightsize[x] = space; + } + sdp->sd_max_height = x; + sdp->sd_heightsize[x] = ~0; + gfs2_assert(sdp, sdp->sd_max_height <= GFS2_MAX_META_HEIGHT); + + sdp->sd_max_dents_per_leaf = (sdp->sd_sb.sb_bsize - + sizeof(struct gfs2_leaf)) / + GFS2_MIN_DIRENT_SIZE; + return 0; +} + +static int init_names(struct gfs2_sbd *sdp, int silent) +{ + char *proto, *table; + int error = 0; + + proto = sdp->sd_args.ar_lockproto; + table = sdp->sd_args.ar_locktable; + + /* Try to autodetect */ + + if (!proto[0] || !table[0]) { + error = gfs2_read_super(sdp, GFS2_SB_ADDR >> sdp->sd_fsb2bb_shift, silent); + if (error) + return error; + + if (!proto[0]) + proto = sdp->sd_sb.sb_lockproto; + if (!table[0]) + table = sdp->sd_sb.sb_locktable; + } + + if (!table[0]) + table = sdp->sd_vfs->s_id; + + BUILD_BUG_ON(GFS2_LOCKNAME_LEN > GFS2_FSNAME_LEN); + + strscpy(sdp->sd_proto_name, proto, GFS2_LOCKNAME_LEN); + strscpy(sdp->sd_table_name, table, GFS2_LOCKNAME_LEN); + + table = sdp->sd_table_name; + while ((table = strchr(table, '/'))) + *table = '_'; + + return error; +} + +static int init_locking(struct gfs2_sbd *sdp, struct gfs2_holder *mount_gh, + int undo) +{ + int error = 0; + + if (undo) + goto fail_trans; + + error = gfs2_glock_nq_num(sdp, + GFS2_MOUNT_LOCK, &gfs2_nondisk_glops, + LM_ST_EXCLUSIVE, + LM_FLAG_RECOVER | GL_NOCACHE | GL_NOPID, + mount_gh); + if (error) { + fs_err(sdp, "can't acquire mount glock: %d\n", error); + goto fail; + } + + error = gfs2_glock_nq_num(sdp, + GFS2_LIVE_LOCK, &gfs2_nondisk_glops, + LM_ST_SHARED, + LM_FLAG_RECOVER | GL_EXACT | GL_NOPID, + &sdp->sd_live_gh); + if (error) { + fs_err(sdp, "can't acquire live glock: %d\n", error); + goto fail_mount; + } + + error = gfs2_glock_get(sdp, GFS2_RENAME_LOCK, &gfs2_nondisk_glops, + CREATE, &sdp->sd_rename_gl); + if (error) { + fs_err(sdp, "can't create rename glock: %d\n", error); + goto fail_live; + } + + error = gfs2_glock_get(sdp, GFS2_FREEZE_LOCK, &gfs2_freeze_glops, + CREATE, &sdp->sd_freeze_gl); + if (error) { + fs_err(sdp, "can't create freeze glock: %d\n", error); + goto fail_rename; + } + + return 0; + +fail_trans: + gfs2_glock_put(sdp->sd_freeze_gl); +fail_rename: + gfs2_glock_put(sdp->sd_rename_gl); +fail_live: + gfs2_glock_dq_uninit(&sdp->sd_live_gh); +fail_mount: + gfs2_glock_dq_uninit(mount_gh); +fail: + return error; +} + +static int gfs2_lookup_root(struct super_block *sb, struct dentry **dptr, + u64 no_addr, const char *name) +{ + struct gfs2_sbd *sdp = sb->s_fs_info; + struct dentry *dentry; + struct inode *inode; + + inode = gfs2_inode_lookup(sb, DT_DIR, no_addr, 0, + GFS2_BLKST_FREE /* ignore */); + if (IS_ERR(inode)) { + fs_err(sdp, "can't read in %s inode: %ld\n", name, PTR_ERR(inode)); + return PTR_ERR(inode); + } + dentry = d_make_root(inode); + if (!dentry) { + fs_err(sdp, "can't alloc %s dentry\n", name); + return -ENOMEM; + } + *dptr = dentry; + return 0; +} + +static int init_sb(struct gfs2_sbd *sdp, int silent) +{ + struct super_block *sb = sdp->sd_vfs; + struct gfs2_holder sb_gh; + u64 no_addr; + int ret; + + ret = gfs2_glock_nq_num(sdp, GFS2_SB_LOCK, &gfs2_meta_glops, + LM_ST_SHARED, 0, &sb_gh); + if (ret) { + fs_err(sdp, "can't acquire superblock glock: %d\n", ret); + return ret; + } + + ret = gfs2_read_sb(sdp, silent); + if (ret) { + fs_err(sdp, "can't read superblock: %d\n", ret); + goto out; + } + + switch(sdp->sd_sb.sb_fs_format) { + case GFS2_FS_FORMAT_MAX: + sb->s_xattr = gfs2_xattr_handlers_max; + break; + + case GFS2_FS_FORMAT_MIN: + sb->s_xattr = gfs2_xattr_handlers_min; + break; + + default: + BUG(); + } + + /* Set up the buffer cache and SB for real */ + if (sdp->sd_sb.sb_bsize < bdev_logical_block_size(sb->s_bdev)) { + ret = -EINVAL; + fs_err(sdp, "FS block size (%u) is too small for device " + "block size (%u)\n", + sdp->sd_sb.sb_bsize, bdev_logical_block_size(sb->s_bdev)); + goto out; + } + if (sdp->sd_sb.sb_bsize > PAGE_SIZE) { + ret = -EINVAL; + fs_err(sdp, "FS block size (%u) is too big for machine " + "page size (%u)\n", + sdp->sd_sb.sb_bsize, (unsigned int)PAGE_SIZE); + goto out; + } + ret = -EINVAL; + if (!sb_set_blocksize(sb, sdp->sd_sb.sb_bsize)) + goto out; + + /* Get the root inode */ + no_addr = sdp->sd_sb.sb_root_dir.no_addr; + ret = gfs2_lookup_root(sb, &sdp->sd_root_dir, no_addr, "root"); + if (ret) + goto out; + + /* Get the master inode */ + no_addr = sdp->sd_sb.sb_master_dir.no_addr; + ret = gfs2_lookup_root(sb, &sdp->sd_master_dir, no_addr, "master"); + if (ret) { + dput(sdp->sd_root_dir); + goto out; + } + sb->s_root = dget(sdp->sd_args.ar_meta ? sdp->sd_master_dir : sdp->sd_root_dir); +out: + gfs2_glock_dq_uninit(&sb_gh); + return ret; +} + +static void gfs2_others_may_mount(struct gfs2_sbd *sdp) +{ + char *message = "FIRSTMOUNT=Done"; + char *envp[] = { message, NULL }; + + fs_info(sdp, "first mount done, others may mount\n"); + + if (sdp->sd_lockstruct.ls_ops->lm_first_done) + sdp->sd_lockstruct.ls_ops->lm_first_done(sdp); + + kobject_uevent_env(&sdp->sd_kobj, KOBJ_CHANGE, envp); +} + +/** + * gfs2_jindex_hold - Grab a lock on the jindex + * @sdp: The GFS2 superblock + * @ji_gh: the holder for the jindex glock + * + * Returns: errno + */ + +static int gfs2_jindex_hold(struct gfs2_sbd *sdp, struct gfs2_holder *ji_gh) +{ + struct gfs2_inode *dip = GFS2_I(sdp->sd_jindex); + struct qstr name; + char buf[20]; + struct gfs2_jdesc *jd; + int error; + + name.name = buf; + + mutex_lock(&sdp->sd_jindex_mutex); + + for (;;) { + error = gfs2_glock_nq_init(dip->i_gl, LM_ST_SHARED, 0, ji_gh); + if (error) + break; + + name.len = sprintf(buf, "journal%u", sdp->sd_journals); + name.hash = gfs2_disk_hash(name.name, name.len); + + error = gfs2_dir_check(sdp->sd_jindex, &name, NULL); + if (error == -ENOENT) { + error = 0; + break; + } + + gfs2_glock_dq_uninit(ji_gh); + + if (error) + break; + + error = -ENOMEM; + jd = kzalloc_obj(struct gfs2_jdesc); + if (!jd) + break; + + INIT_LIST_HEAD(&jd->extent_list); + INIT_LIST_HEAD(&jd->jd_revoke_list); + + INIT_WORK(&jd->jd_work, gfs2_recover_func); + jd->jd_inode = gfs2_lookupi(sdp->sd_jindex, &name, 1); + if (IS_ERR_OR_NULL(jd->jd_inode)) { + if (!jd->jd_inode) + error = -ENOENT; + else + error = PTR_ERR(jd->jd_inode); + kfree(jd); + break; + } + + d_mark_dontcache(jd->jd_inode); + spin_lock(&sdp->sd_jindex_spin); + jd->jd_jid = sdp->sd_journals++; + list_add_tail(&jd->jd_list, &sdp->sd_jindex_list); + spin_unlock(&sdp->sd_jindex_spin); + } + + mutex_unlock(&sdp->sd_jindex_mutex); + + return error; +} + +/** + * init_statfs - look up and initialize master and local (per node) statfs inodes + * @sdp: The GFS2 superblock + * + * This should be called after the jindex is initialized in init_journal() and + * before gfs2_journal_recovery() is called because we need to be able to write + * to these inodes during recovery. + * + * Returns: errno + */ +static int init_statfs(struct gfs2_sbd *sdp) +{ + int error = 0; + struct inode *master = d_inode(sdp->sd_master_dir); + struct inode *pn = NULL; + char buf[30]; + struct gfs2_jdesc *jd; + struct gfs2_inode *ip; + + sdp->sd_statfs_inode = gfs2_lookup_meta(master, "statfs"); + if (IS_ERR(sdp->sd_statfs_inode)) { + error = PTR_ERR(sdp->sd_statfs_inode); + fs_err(sdp, "can't read in statfs inode: %d\n", error); + goto out; + } + if (sdp->sd_args.ar_spectator) + goto out; + + pn = gfs2_lookup_meta(master, "per_node"); + if (IS_ERR(pn)) { + error = PTR_ERR(pn); + fs_err(sdp, "can't find per_node directory: %d\n", error); + goto put_statfs; + } + + /* For each jid, lookup the corresponding local statfs inode in the + * per_node metafs directory and save it in the sdp->sd_sc_inodes_list. */ + list_for_each_entry(jd, &sdp->sd_jindex_list, jd_list) { + struct local_statfs_inode *lsi = + kmalloc_obj(struct local_statfs_inode, GFP_NOFS); + if (!lsi) { + error = -ENOMEM; + goto free_local; + } + sprintf(buf, "statfs_change%u", jd->jd_jid); + lsi->si_sc_inode = gfs2_lookup_meta(pn, buf); + if (IS_ERR(lsi->si_sc_inode)) { + error = PTR_ERR(lsi->si_sc_inode); + fs_err(sdp, "can't find local \"sc\" file#%u: %d\n", + jd->jd_jid, error); + kfree(lsi); + goto free_local; + } + lsi->si_jid = jd->jd_jid; + if (jd->jd_jid == sdp->sd_jdesc->jd_jid) + sdp->sd_sc_inode = lsi->si_sc_inode; + + list_add_tail(&lsi->si_list, &sdp->sd_sc_inodes_list); + } + + iput(pn); + pn = NULL; + ip = GFS2_I(sdp->sd_sc_inode); + error = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, GL_NOPID, + &sdp->sd_sc_gh); + if (error) { + fs_err(sdp, "can't lock local \"sc\" file: %d\n", error); + goto free_local; + } + /* read in the local statfs buffer - other nodes don't change it. */ + error = gfs2_meta_inode_buffer(ip, &sdp->sd_sc_bh); + if (error) { + fs_err(sdp, "Cannot read in local statfs: %d\n", error); + goto unlock_sd_gh; + } + return 0; + +unlock_sd_gh: + gfs2_glock_dq_uninit(&sdp->sd_sc_gh); +free_local: + free_local_statfs_inodes(sdp); + iput(pn); +put_statfs: + iput(sdp->sd_statfs_inode); +out: + return error; +} + +/* Uninitialize and free up memory used by the list of statfs inodes */ +static void uninit_statfs(struct gfs2_sbd *sdp) +{ + if (!sdp->sd_args.ar_spectator) { + brelse(sdp->sd_sc_bh); + gfs2_glock_dq_uninit(&sdp->sd_sc_gh); + free_local_statfs_inodes(sdp); + } + iput(sdp->sd_statfs_inode); +} + +static int init_journal(struct gfs2_sbd *sdp, int undo) +{ + struct inode *master = d_inode(sdp->sd_master_dir); + struct gfs2_holder ji_gh; + struct gfs2_inode *ip; + int error = 0; + + gfs2_holder_mark_uninitialized(&ji_gh); + if (undo) + goto fail_statfs; + + sdp->sd_jindex = gfs2_lookup_meta(master, "jindex"); + if (IS_ERR(sdp->sd_jindex)) { + fs_err(sdp, "can't lookup journal index: %d\n", error); + return PTR_ERR(sdp->sd_jindex); + } + + /* Load in the journal index special file */ + + error = gfs2_jindex_hold(sdp, &ji_gh); + if (error) { + fs_err(sdp, "can't read journal index: %d\n", error); + goto fail; + } + + error = -EUSERS; + if (!gfs2_jindex_size(sdp)) { + fs_err(sdp, "no journals!\n"); + goto fail_jindex; + } + + atomic_set(&sdp->sd_log_blks_needed, 0); + if (sdp->sd_args.ar_spectator) { + sdp->sd_jdesc = gfs2_jdesc_find(sdp, 0); + atomic_set(&sdp->sd_log_blks_free, sdp->sd_jdesc->jd_blocks); + atomic_set(&sdp->sd_log_thresh1, 2*sdp->sd_jdesc->jd_blocks/5); + atomic_set(&sdp->sd_log_thresh2, 4*sdp->sd_jdesc->jd_blocks/5); + } else { + if (sdp->sd_lockstruct.ls_jid >= gfs2_jindex_size(sdp)) { + fs_err(sdp, "can't mount journal #%u\n", + sdp->sd_lockstruct.ls_jid); + fs_err(sdp, "there are only %u journals (0 - %u)\n", + gfs2_jindex_size(sdp), + gfs2_jindex_size(sdp) - 1); + goto fail_jindex; + } + sdp->sd_jdesc = gfs2_jdesc_find(sdp, sdp->sd_lockstruct.ls_jid); + + error = gfs2_glock_nq_num(sdp, sdp->sd_lockstruct.ls_jid, + &gfs2_journal_glops, + LM_ST_EXCLUSIVE, + LM_FLAG_RECOVER | GL_NOPID, + &sdp->sd_journal_gh); + if (error) { + fs_err(sdp, "can't acquire journal glock: %d\n", error); + goto fail_jindex; + } + + ip = GFS2_I(sdp->sd_jdesc->jd_inode); + error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, + LM_FLAG_RECOVER | GL_EXACT | + GL_NOCACHE | GL_NOPID, + &sdp->sd_jinode_gh); + if (error) { + fs_err(sdp, "can't acquire journal inode glock: %d\n", + error); + goto fail_journal_gh; + } + + error = gfs2_jdesc_check(sdp->sd_jdesc); + if (error) { + fs_err(sdp, "my journal (%u) is bad: %d\n", + sdp->sd_jdesc->jd_jid, error); + goto fail_jinode_gh; + } + atomic_set(&sdp->sd_log_blks_free, sdp->sd_jdesc->jd_blocks); + atomic_set(&sdp->sd_log_thresh1, 2*sdp->sd_jdesc->jd_blocks/5); + atomic_set(&sdp->sd_log_thresh2, 4*sdp->sd_jdesc->jd_blocks/5); + + /* Map the extents for this journal's blocks */ + gfs2_map_journal_extents(sdp, sdp->sd_jdesc); + } + trace_gfs2_log_blocks(sdp, atomic_read(&sdp->sd_log_blks_free)); + + /* Lookup statfs inodes here so journal recovery can use them. */ + error = init_statfs(sdp); + if (error) + goto fail_jinode_gh; + + if (sdp->sd_lockstruct.ls_first) { + unsigned int x; + for (x = 0; x < sdp->sd_journals; x++) { + struct gfs2_jdesc *jd = gfs2_jdesc_find(sdp, x); + + if (sdp->sd_args.ar_spectator) { + error = check_journal_clean(sdp, jd, true); + if (error) + goto fail_statfs; + continue; + } + error = gfs2_recover_journal(jd, true); + if (error) { + fs_err(sdp, "error recovering journal %u: %d\n", + x, error); + goto fail_statfs; + } + } + + gfs2_others_may_mount(sdp); + } else if (!sdp->sd_args.ar_spectator) { + error = gfs2_recover_journal(sdp->sd_jdesc, true); + if (error) { + fs_err(sdp, "error recovering my journal: %d\n", error); + goto fail_statfs; + } + } + + sdp->sd_log_idle = 1; + set_bit(SDF_JOURNAL_CHECKED, &sdp->sd_flags); + gfs2_glock_dq_uninit(&ji_gh); + INIT_WORK(&sdp->sd_freeze_work, gfs2_freeze_func); + return 0; + +fail_statfs: + uninit_statfs(sdp); +fail_jinode_gh: + if (!sdp->sd_args.ar_spectator) + gfs2_glock_dq_uninit(&sdp->sd_jinode_gh); +fail_journal_gh: + if (!sdp->sd_args.ar_spectator) + gfs2_glock_dq_uninit(&sdp->sd_journal_gh); +fail_jindex: + gfs2_jindex_free(sdp); + if (gfs2_holder_initialized(&ji_gh)) + gfs2_glock_dq_uninit(&ji_gh); +fail: + iput(sdp->sd_jindex); + return error; +} + +static struct lock_class_key gfs2_quota_imutex_key; + +static int init_inodes(struct gfs2_sbd *sdp, int undo) +{ + int error = 0; + struct inode *master = d_inode(sdp->sd_master_dir); + + if (undo) + goto fail_qinode; + + error = init_journal(sdp, undo); + complete_all(&sdp->sd_journal_ready); + if (error) + goto fail; + + /* Read in the resource index inode */ + sdp->sd_rindex = gfs2_lookup_meta(master, "rindex"); + if (IS_ERR(sdp->sd_rindex)) { + error = PTR_ERR(sdp->sd_rindex); + fs_err(sdp, "can't get resource index inode: %d\n", error); + goto fail_journal; + } + sdp->sd_rindex_uptodate = 0; + + /* Read in the quota inode */ + sdp->sd_quota_inode = gfs2_lookup_meta(master, "quota"); + if (IS_ERR(sdp->sd_quota_inode)) { + error = PTR_ERR(sdp->sd_quota_inode); + fs_err(sdp, "can't get quota file inode: %d\n", error); + goto fail_rindex; + } + /* + * i_rwsem on quota files is special. Since this inode is hidden system + * file, we are safe to define locking ourselves. + */ + lockdep_set_class(&sdp->sd_quota_inode->i_rwsem, + &gfs2_quota_imutex_key); + + error = gfs2_rindex_update(sdp); + if (error) + goto fail_qinode; + + return 0; + +fail_qinode: + iput(sdp->sd_quota_inode); +fail_rindex: + gfs2_clear_rgrpd(sdp); + iput(sdp->sd_rindex); +fail_journal: + init_journal(sdp, UNDO); +fail: + return error; +} + +static int init_per_node(struct gfs2_sbd *sdp, int undo) +{ + struct inode *pn = NULL; + char buf[30]; + int error = 0; + struct gfs2_inode *ip; + struct inode *master = d_inode(sdp->sd_master_dir); + + if (sdp->sd_args.ar_spectator) + return 0; + + if (undo) + goto fail_qc_gh; + + pn = gfs2_lookup_meta(master, "per_node"); + if (IS_ERR(pn)) { + error = PTR_ERR(pn); + fs_err(sdp, "can't find per_node directory: %d\n", error); + return error; + } + + sprintf(buf, "quota_change%u", sdp->sd_jdesc->jd_jid); + sdp->sd_qc_inode = gfs2_lookup_meta(pn, buf); + if (IS_ERR(sdp->sd_qc_inode)) { + error = PTR_ERR(sdp->sd_qc_inode); + fs_err(sdp, "can't find local \"qc\" file: %d\n", error); + goto fail_ut_i; + } + + iput(pn); + pn = NULL; + + ip = GFS2_I(sdp->sd_qc_inode); + error = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, GL_NOPID, + &sdp->sd_qc_gh); + if (error) { + fs_err(sdp, "can't lock local \"qc\" file: %d\n", error); + goto fail_qc_i; + } + + return 0; + +fail_qc_gh: + gfs2_glock_dq_uninit(&sdp->sd_qc_gh); +fail_qc_i: + iput(sdp->sd_qc_inode); +fail_ut_i: + iput(pn); + return error; +} + +static const match_table_t nolock_tokens = { + { Opt_jid, "jid=%d", }, + { Opt_err, NULL }, +}; + +static const struct lm_lockops nolock_ops = { + .lm_proto_name = "lock_nolock", + .lm_put_lock = gfs2_glock_free, + .lm_tokens = &nolock_tokens, +}; + +/** + * gfs2_lm_mount - mount a locking protocol + * @sdp: the filesystem + * @silent: if 1, don't complain if the FS isn't a GFS2 fs + * + * Returns: errno + */ + +static int gfs2_lm_mount(struct gfs2_sbd *sdp, int silent) +{ + const struct lm_lockops *lm; + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + struct gfs2_args *args = &sdp->sd_args; + const char *proto = sdp->sd_proto_name; + const char *table = sdp->sd_table_name; + char *o, *options; + int ret; + + if (!strcmp("lock_nolock", proto)) { + lm = &nolock_ops; + sdp->sd_args.ar_localflocks = 1; +#ifdef CONFIG_GFS2_FS_LOCKING_DLM + } else if (!strcmp("lock_dlm", proto)) { + lm = &gfs2_dlm_ops; +#endif + } else { + pr_info("can't find protocol %s\n", proto); + return -ENOENT; + } + + fs_info(sdp, "Trying to join cluster \"%s\", \"%s\"\n", proto, table); + + ls->ls_ops = lm; + ls->ls_first = 1; + + for (options = args->ar_hostdata; (o = strsep(&options, ":")); ) { + substring_t tmp[MAX_OPT_ARGS]; + int token, option; + + if (!o || !*o) + continue; + + token = match_token(o, *lm->lm_tokens, tmp); + switch (token) { + case Opt_jid: + ret = match_int(&tmp[0], &option); + if (ret || option < 0) + goto hostdata_error; + if (test_and_clear_bit(SDF_NOJOURNALID, &sdp->sd_flags)) + ls->ls_jid = option; + break; + case Opt_id: + case Opt_nodir: + /* Obsolete, but left for backward compat purposes */ + break; + case Opt_first: + ret = match_int(&tmp[0], &option); + if (ret || (option != 0 && option != 1)) + goto hostdata_error; + ls->ls_first = option; + break; + case Opt_err: + default: +hostdata_error: + fs_info(sdp, "unknown hostdata (%s)\n", o); + return -EINVAL; + } + } + + if (lm->lm_mount == NULL) { + fs_info(sdp, "Now mounting FS (format %u)...\n", sdp->sd_sb.sb_fs_format); + complete_all(&sdp->sd_locking_init); + return 0; + } + ret = lm->lm_mount(sdp, table); + if (ret == 0) + fs_info(sdp, "Joined cluster. Now mounting FS (format %u)...\n", + sdp->sd_sb.sb_fs_format); + complete_all(&sdp->sd_locking_init); + return ret; +} + +void gfs2_lm_unmount(struct gfs2_sbd *sdp) +{ + const struct lm_lockops *lm = sdp->sd_lockstruct.ls_ops; + if (!gfs2_withdrawn(sdp) && lm->lm_unmount) + lm->lm_unmount(sdp, true); +} + +static int wait_on_journal(struct gfs2_sbd *sdp) +{ + if (sdp->sd_lockstruct.ls_ops->lm_mount == NULL) + return 0; + + return wait_on_bit(&sdp->sd_flags, SDF_NOJOURNALID, TASK_INTERRUPTIBLE) + ? -EINTR : 0; +} + +void gfs2_online_uevent(struct gfs2_sbd *sdp) +{ + struct super_block *sb = sdp->sd_vfs; + char ro[20]; + char spectator[20]; + char *envp[] = { ro, spectator, NULL }; + sprintf(ro, "RDONLY=%d", sb_rdonly(sb)); + sprintf(spectator, "SPECTATOR=%d", sdp->sd_args.ar_spectator ? 1 : 0); + kobject_uevent_env(&sdp->sd_kobj, KOBJ_ONLINE, envp); +} + +static int init_threads(struct gfs2_sbd *sdp) +{ + struct task_struct *p; + int error = 0; + + p = kthread_create(gfs2_logd, sdp, "gfs2_logd/%s", sdp->sd_fsname); + if (IS_ERR(p)) { + error = PTR_ERR(p); + fs_err(sdp, "can't create logd thread: %d\n", error); + return error; + } + get_task_struct(p); + sdp->sd_logd_process = p; + + p = kthread_create(gfs2_quotad, sdp, "gfs2_quotad/%s", sdp->sd_fsname); + if (IS_ERR(p)) { + error = PTR_ERR(p); + fs_err(sdp, "can't create quotad thread: %d\n", error); + goto fail; + } + get_task_struct(p); + sdp->sd_quotad_process = p; + + wake_up_process(sdp->sd_logd_process); + wake_up_process(sdp->sd_quotad_process); + return 0; + +fail: + kthread_stop_put(sdp->sd_logd_process); + sdp->sd_logd_process = NULL; + return error; +} + +void gfs2_destroy_threads(struct gfs2_sbd *sdp) +{ + if (sdp->sd_logd_process) { + kthread_stop_put(sdp->sd_logd_process); + sdp->sd_logd_process = NULL; + } + if (sdp->sd_quotad_process) { + kthread_stop_put(sdp->sd_quotad_process); + sdp->sd_quotad_process = NULL; + } +} + +/** + * gfs2_fill_super - Read in superblock + * @sb: The VFS superblock + * @fc: Mount options and flags + * + * Returns: -errno + */ +static int gfs2_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct gfs2_args *args = fc->fs_private; + int silent = fc->sb_flags & SB_SILENT; + struct gfs2_sbd *sdp; + struct gfs2_holder mount_gh; + struct address_space *mapping; + int error; + + sdp = init_sbd(sb); + if (!sdp) { + pr_warn("can't alloc struct gfs2_sbd\n"); + return -ENOMEM; + } + sdp->sd_args = *args; + + if (sdp->sd_args.ar_spectator) { + sb->s_flags |= SB_RDONLY; + set_bit(SDF_RORECOVERY, &sdp->sd_flags); + } + if (sdp->sd_args.ar_posix_acl) + sb->s_flags |= SB_POSIXACL; + if (sdp->sd_args.ar_nobarrier) + set_bit(SDF_NOBARRIERS, &sdp->sd_flags); + + sb->s_flags |= SB_NOSEC; + sb->s_magic = GFS2_MAGIC; + sb->s_op = &gfs2_super_ops; + + set_default_d_op(sb, &gfs2_dops); + sb->s_export_op = &gfs2_export_ops; + sb->s_qcop = &gfs2_quotactl_ops; + sb->s_quota_types = QTYPE_MASK_USR | QTYPE_MASK_GRP; + sb_dqopt(sb)->flags |= DQUOT_QUOTA_SYS_FILE; + sb->s_time_gran = 1; + sb->s_maxbytes = MAX_LFS_FILESIZE; + + /* Set up the buffer cache and fill in some fake block size values + to allow us to read-in the on-disk superblock. */ + sdp->sd_sb.sb_bsize = sb_min_blocksize(sb, SECTOR_SIZE); + error = -EINVAL; + if (!sdp->sd_sb.sb_bsize) + goto fail_free; + sdp->sd_sb.sb_bsize_shift = sb->s_blocksize_bits; + sdp->sd_fsb2bb_shift = sdp->sd_sb.sb_bsize_shift - SECTOR_SHIFT; + sdp->sd_fsb2bb = BIT(sdp->sd_fsb2bb_shift); + + sdp->sd_tune.gt_logd_secs = sdp->sd_args.ar_commit; + sdp->sd_tune.gt_quota_quantum = sdp->sd_args.ar_quota_quantum; + if (sdp->sd_args.ar_statfs_quantum) { + sdp->sd_tune.gt_statfs_slow = 0; + sdp->sd_tune.gt_statfs_quantum = sdp->sd_args.ar_statfs_quantum; + } else { + sdp->sd_tune.gt_statfs_slow = 1; + sdp->sd_tune.gt_statfs_quantum = 30; + } + + /* Set up an address space for metadata writes */ + sdp->sd_inode = new_inode(sb); + error = -ENOMEM; + if (!sdp->sd_inode) + goto fail_free; + sdp->sd_inode->i_ino = GFS2_BAD_INO; + sdp->sd_inode->i_size = OFFSET_MAX; + + mapping = gfs2_aspace(sdp); + mapping->a_ops = &gfs2_rgrp_aops; + gfs2_setup_inode(sdp->sd_inode); + + error = init_names(sdp, silent); + if (error) + goto fail_iput; + + snprintf(sdp->sd_fsname, sizeof(sdp->sd_fsname), "%s", sdp->sd_table_name); + + error = -ENOMEM; + sdp->sd_glock_wq = alloc_workqueue("gfs2-glock/%s", + WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_FREEZABLE | WQ_PERCPU, + 0, + sdp->sd_fsname); + if (!sdp->sd_glock_wq) + goto fail_iput; + + sdp->sd_delete_wq = alloc_workqueue("gfs2-delete/%s", + WQ_MEM_RECLAIM | WQ_FREEZABLE | WQ_PERCPU, 0, + sdp->sd_fsname); + if (!sdp->sd_delete_wq) + goto fail_glock_wq; + + error = gfs2_sys_fs_add(sdp); + if (error) + goto fail_delete_wq; + + gfs2_create_debugfs_file(sdp); + + error = gfs2_lm_mount(sdp, silent); + if (error) + goto fail_debug; + + INIT_WORK(&sdp->sd_withdraw_work, gfs2_withdraw_func); + + error = init_locking(sdp, &mount_gh, DO); + if (error) + goto fail_lm; + + error = init_sb(sdp, silent); + if (error) + goto fail_locking; + + /* Turn rgrplvb on by default if fs format is recent enough */ + if (!sdp->sd_args.ar_got_rgrplvb && sdp->sd_sb.sb_fs_format > 1801) + sdp->sd_args.ar_rgrplvb = 1; + + error = wait_on_journal(sdp); + if (error) + goto fail_sb; + + /* + * If user space has failed to join the cluster or some similar + * failure has occurred, then the journal id will contain a + * negative (error) number. This will then be returned to the + * caller (of the mount syscall). We do this even for spectator + * mounts (which just write a jid of 0 to indicate "ok" even though + * the jid is unused in the spectator case) + */ + if (sdp->sd_lockstruct.ls_jid < 0) { + error = sdp->sd_lockstruct.ls_jid; + sdp->sd_lockstruct.ls_jid = 0; + goto fail_sb; + } + + if (sdp->sd_args.ar_spectator) + snprintf(sdp->sd_fsname, sizeof(sdp->sd_fsname), "%s.s", + sdp->sd_table_name); + else + snprintf(sdp->sd_fsname, sizeof(sdp->sd_fsname), "%s.%u", + sdp->sd_table_name, sdp->sd_lockstruct.ls_jid); + + error = init_inodes(sdp, DO); + if (error) + goto fail_sb; + + error = init_per_node(sdp, DO); + if (error) + goto fail_inodes; + + error = gfs2_statfs_init(sdp); + if (error) { + fs_err(sdp, "can't initialize statfs subsystem: %d\n", error); + goto fail_per_node; + } + + if (!sb_rdonly(sb)) { + error = init_threads(sdp); + if (error) + goto fail_per_node; + } + + error = gfs2_freeze_lock_shared(sdp); + if (error) + goto fail_per_node; + + if (!sb_rdonly(sb)) + error = gfs2_make_fs_rw(sdp); + + if (error) { + gfs2_freeze_unlock(sdp); + fs_err(sdp, "can't make FS RW: %d\n", error); + goto fail_per_node; + } + gfs2_glock_dq_uninit(&mount_gh); + gfs2_online_uevent(sdp); + return 0; + +fail_per_node: + init_per_node(sdp, UNDO); + gfs2_destroy_threads(sdp); +fail_inodes: + init_inodes(sdp, UNDO); +fail_sb: + if (sdp->sd_root_dir) + dput(sdp->sd_root_dir); + if (sdp->sd_master_dir) + dput(sdp->sd_master_dir); + if (sb->s_root) + dput(sb->s_root); + sb->s_root = NULL; +fail_locking: + init_locking(sdp, &mount_gh, UNDO); +fail_lm: + complete_all(&sdp->sd_journal_ready); + gfs2_lm_unmount(sdp); +fail_debug: + gfs2_delete_debugfs_file(sdp); + gfs2_sys_fs_del(sdp); +fail_delete_wq: + destroy_workqueue(sdp->sd_delete_wq); +fail_glock_wq: + if (sdp->sd_glock_wq) + destroy_workqueue(sdp->sd_glock_wq); +fail_iput: + iput(sdp->sd_inode); +fail_free: + free_sbd(sdp); + return error; +} + +/** + * gfs2_get_tree - Get the GFS2 superblock and root directory + * @fc: The filesystem context + * + * Returns: 0 or -errno on error + */ +static int gfs2_get_tree(struct fs_context *fc) +{ + struct gfs2_args *args = fc->fs_private; + struct gfs2_sbd *sdp; + int error; + + error = get_tree_bdev(fc, gfs2_fill_super); + if (error) + return error; + + sdp = fc->root->d_sb->s_fs_info; + dput(fc->root); + if (args->ar_meta) + fc->root = dget(sdp->sd_master_dir); + else + fc->root = dget(sdp->sd_root_dir); + return 0; +} + +static void gfs2_fc_free(struct fs_context *fc) +{ + struct gfs2_args *args = fc->fs_private; + + kfree(args); +} + +enum gfs2_param { + Opt_lockproto, + Opt_locktable, + Opt_hostdata, + Opt_spectator, + Opt_ignore_local_fs, + Opt_localflocks, + Opt_localcaching, + Opt_debug, + Opt_upgrade, + Opt_acl, + Opt_quota, + Opt_quota_flag, + Opt_suiddir, + Opt_data, + Opt_meta, + Opt_discard, + Opt_commit, + Opt_errors, + Opt_statfs_quantum, + Opt_statfs_percent, + Opt_quota_quantum, + Opt_barrier, + Opt_rgrplvb, + Opt_loccookie, +}; + +static const struct constant_table gfs2_param_quota[] = { + {"off", GFS2_QUOTA_OFF}, + {"account", GFS2_QUOTA_ACCOUNT}, + {"on", GFS2_QUOTA_ON}, + {"quiet", GFS2_QUOTA_QUIET}, + {} +}; + +enum opt_data { + Opt_data_writeback = GFS2_DATA_WRITEBACK, + Opt_data_ordered = GFS2_DATA_ORDERED, +}; + +static const struct constant_table gfs2_param_data[] = { + {"writeback", Opt_data_writeback }, + {"ordered", Opt_data_ordered }, + {} +}; + +enum opt_errors { + Opt_errors_withdraw = GFS2_ERRORS_WITHDRAW, + Opt_errors_deactivate = GFS2_ERRORS_DEACTIVATE, + Opt_errors_panic = GFS2_ERRORS_PANIC, +}; + +static const struct constant_table gfs2_param_errors[] = { + {"withdraw", Opt_errors_withdraw }, + {"deactivate", Opt_errors_deactivate }, + {"panic", Opt_errors_panic }, + {} +}; + +static const struct fs_parameter_spec gfs2_fs_parameters[] = { + fsparam_string ("lockproto", Opt_lockproto), + fsparam_string ("locktable", Opt_locktable), + fsparam_string ("hostdata", Opt_hostdata), + fsparam_flag ("spectator", Opt_spectator), + fsparam_flag ("norecovery", Opt_spectator), + fsparam_flag ("ignore_local_fs", Opt_ignore_local_fs), + fsparam_flag ("localflocks", Opt_localflocks), + fsparam_flag ("localcaching", Opt_localcaching), + fsparam_flag_no("debug", Opt_debug), + fsparam_flag ("upgrade", Opt_upgrade), + fsparam_flag_no("acl", Opt_acl), + fsparam_flag_no("suiddir", Opt_suiddir), + fsparam_enum ("data", Opt_data, gfs2_param_data), + fsparam_flag ("meta", Opt_meta), + fsparam_flag_no("discard", Opt_discard), + fsparam_s32 ("commit", Opt_commit), + fsparam_enum ("errors", Opt_errors, gfs2_param_errors), + fsparam_s32 ("statfs_quantum", Opt_statfs_quantum), + fsparam_s32 ("statfs_percent", Opt_statfs_percent), + fsparam_s32 ("quota_quantum", Opt_quota_quantum), + fsparam_flag_no("barrier", Opt_barrier), + fsparam_flag_no("rgrplvb", Opt_rgrplvb), + fsparam_flag_no("loccookie", Opt_loccookie), + /* quota can be a flag or an enum so it gets special treatment */ + fsparam_flag_no("quota", Opt_quota_flag), + fsparam_enum("quota", Opt_quota, gfs2_param_quota), + {} +}; + +/* Parse a single mount parameter */ +static int gfs2_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct gfs2_args *args = fc->fs_private; + struct fs_parse_result result; + int o; + + o = fs_parse(fc, gfs2_fs_parameters, param, &result); + if (o < 0) + return o; + + switch (o) { + case Opt_lockproto: + strscpy(args->ar_lockproto, param->string, GFS2_LOCKNAME_LEN); + break; + case Opt_locktable: + strscpy(args->ar_locktable, param->string, GFS2_LOCKNAME_LEN); + break; + case Opt_hostdata: + strscpy(args->ar_hostdata, param->string, GFS2_LOCKNAME_LEN); + break; + case Opt_spectator: + args->ar_spectator = 1; + break; + case Opt_ignore_local_fs: + /* Retained for backwards compat only */ + break; + case Opt_localflocks: + args->ar_localflocks = 1; + break; + case Opt_localcaching: + /* Retained for backwards compat only */ + break; + case Opt_debug: + if (result.boolean && args->ar_errors == GFS2_ERRORS_PANIC) + return invalfc(fc, "-o debug and -o errors=panic are mutually exclusive"); + args->ar_debug = result.boolean; + break; + case Opt_upgrade: + /* Retained for backwards compat only */ + break; + case Opt_acl: + args->ar_posix_acl = result.boolean; + break; + case Opt_quota_flag: + args->ar_quota = result.negated ? GFS2_QUOTA_OFF : GFS2_QUOTA_ON; + break; + case Opt_quota: + args->ar_quota = result.int_32; + break; + case Opt_suiddir: + args->ar_suiddir = result.boolean; + break; + case Opt_data: + /* The uint_32 result maps directly to GFS2_DATA_* */ + args->ar_data = result.uint_32; + break; + case Opt_meta: + args->ar_meta = 1; + break; + case Opt_discard: + args->ar_discard = result.boolean; + break; + case Opt_commit: + if (result.int_32 <= 0) + return invalfc(fc, "commit mount option requires a positive numeric argument"); + args->ar_commit = result.int_32; + break; + case Opt_statfs_quantum: + if (result.int_32 < 0) + return invalfc(fc, "statfs_quantum mount option requires a non-negative numeric argument"); + args->ar_statfs_quantum = result.int_32; + break; + case Opt_quota_quantum: + if (result.int_32 <= 0) + return invalfc(fc, "quota_quantum mount option requires a positive numeric argument"); + args->ar_quota_quantum = result.int_32; + break; + case Opt_statfs_percent: + if (result.int_32 < 0 || result.int_32 > 100) + return invalfc(fc, "statfs_percent mount option requires a numeric argument between 0 and 100"); + args->ar_statfs_percent = result.int_32; + break; + case Opt_errors: + if (args->ar_debug && result.uint_32 == GFS2_ERRORS_PANIC) + return invalfc(fc, "-o debug and -o errors=panic are mutually exclusive"); + args->ar_errors = result.uint_32; + break; + case Opt_barrier: + args->ar_nobarrier = result.boolean; + break; + case Opt_rgrplvb: + args->ar_rgrplvb = result.boolean; + args->ar_got_rgrplvb = 1; + break; + case Opt_loccookie: + args->ar_loccookie = result.boolean; + break; + default: + return invalfc(fc, "invalid mount option: %s", param->key); + } + return 0; +} + +static int gfs2_reconfigure(struct fs_context *fc) +{ + struct super_block *sb = fc->root->d_sb; + struct gfs2_sbd *sdp = sb->s_fs_info; + struct gfs2_args *oldargs = &sdp->sd_args; + struct gfs2_args *newargs = fc->fs_private; + struct gfs2_tune *gt = &sdp->sd_tune; + int error = 0; + + sync_filesystem(sb); + + spin_lock(>->gt_spin); + oldargs->ar_commit = gt->gt_logd_secs; + oldargs->ar_quota_quantum = gt->gt_quota_quantum; + if (gt->gt_statfs_slow) + oldargs->ar_statfs_quantum = 0; + else + oldargs->ar_statfs_quantum = gt->gt_statfs_quantum; + spin_unlock(>->gt_spin); + + if (strcmp(newargs->ar_lockproto, oldargs->ar_lockproto)) { + errorfc(fc, "reconfiguration of locking protocol not allowed"); + return -EINVAL; + } + if (strcmp(newargs->ar_locktable, oldargs->ar_locktable)) { + errorfc(fc, "reconfiguration of lock table not allowed"); + return -EINVAL; + } + if (strcmp(newargs->ar_hostdata, oldargs->ar_hostdata)) { + errorfc(fc, "reconfiguration of host data not allowed"); + return -EINVAL; + } + if (newargs->ar_spectator != oldargs->ar_spectator) { + errorfc(fc, "reconfiguration of spectator mode not allowed"); + return -EINVAL; + } + if (newargs->ar_localflocks != oldargs->ar_localflocks) { + errorfc(fc, "reconfiguration of localflocks not allowed"); + return -EINVAL; + } + if (newargs->ar_meta != oldargs->ar_meta) { + errorfc(fc, "switching between gfs2 and gfs2meta not allowed"); + return -EINVAL; + } + if (oldargs->ar_spectator) + fc->sb_flags |= SB_RDONLY; + + if ((sb->s_flags ^ fc->sb_flags) & SB_RDONLY) { + if (fc->sb_flags & SB_RDONLY) { + gfs2_make_fs_ro(sdp); + } else { + error = gfs2_make_fs_rw(sdp); + if (error) + errorfc(fc, "unable to remount read-write"); + } + } + sdp->sd_args = *newargs; + + if (sdp->sd_args.ar_posix_acl) + sb->s_flags |= SB_POSIXACL; + else + sb->s_flags &= ~SB_POSIXACL; + if (sdp->sd_args.ar_nobarrier) + set_bit(SDF_NOBARRIERS, &sdp->sd_flags); + else + clear_bit(SDF_NOBARRIERS, &sdp->sd_flags); + spin_lock(>->gt_spin); + gt->gt_logd_secs = newargs->ar_commit; + gt->gt_quota_quantum = newargs->ar_quota_quantum; + if (newargs->ar_statfs_quantum) { + gt->gt_statfs_slow = 0; + gt->gt_statfs_quantum = newargs->ar_statfs_quantum; + } + else { + gt->gt_statfs_slow = 1; + gt->gt_statfs_quantum = 30; + } + spin_unlock(>->gt_spin); + + gfs2_online_uevent(sdp); + return error; +} + +static const struct fs_context_operations gfs2_context_ops = { + .free = gfs2_fc_free, + .parse_param = gfs2_parse_param, + .get_tree = gfs2_get_tree, + .reconfigure = gfs2_reconfigure, +}; + +/* Set up the filesystem mount context */ +static int gfs2_init_fs_context(struct fs_context *fc) +{ + struct gfs2_args *args; + + args = kmalloc_obj(*args); + if (args == NULL) + return -ENOMEM; + + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE) { + struct gfs2_sbd *sdp = fc->root->d_sb->s_fs_info; + + *args = sdp->sd_args; + } else { + memset(args, 0, sizeof(*args)); + args->ar_quota = GFS2_QUOTA_DEFAULT; + args->ar_data = GFS2_DATA_DEFAULT; + args->ar_commit = 30; + args->ar_statfs_quantum = 30; + args->ar_quota_quantum = 60; + args->ar_errors = GFS2_ERRORS_DEFAULT; + } + fc->fs_private = args; + fc->ops = &gfs2_context_ops; + return 0; +} + +static int set_meta_super(struct super_block *s, struct fs_context *fc) +{ + return -EINVAL; +} + +static int test_meta_super(struct super_block *s, struct fs_context *fc) +{ + return (fc->sget_key == s->s_bdev); +} + +static int gfs2_meta_get_tree(struct fs_context *fc) +{ + struct super_block *s; + struct gfs2_sbd *sdp; + struct path path; + int error; + + if (!fc->source || !*fc->source) + return -EINVAL; + + error = kern_path(fc->source, LOOKUP_FOLLOW, &path); + if (error) { + pr_warn("path_lookup on %s returned error %d\n", + fc->source, error); + return error; + } + fc->fs_type = &gfs2_fs_type; + fc->sget_key = path.dentry->d_sb->s_bdev; + s = sget_fc(fc, test_meta_super, set_meta_super); + path_put(&path); + if (IS_ERR(s)) { + pr_warn("gfs2 mount does not exist\n"); + return PTR_ERR(s); + } + if ((fc->sb_flags ^ s->s_flags) & SB_RDONLY) { + deactivate_locked_super(s); + return -EBUSY; + } + sdp = s->s_fs_info; + fc->root = dget(sdp->sd_master_dir); + return 0; +} + +static const struct fs_context_operations gfs2_meta_context_ops = { + .free = gfs2_fc_free, + .get_tree = gfs2_meta_get_tree, +}; + +static int gfs2_meta_init_fs_context(struct fs_context *fc) +{ + int ret = gfs2_init_fs_context(fc); + + if (ret) + return ret; + + fc->ops = &gfs2_meta_context_ops; + return 0; +} + +/** + * gfs2_evict_inodes - evict inodes cooperatively + * @sb: the superblock + * + * When evicting an inode with a zero link count, we are trying to upgrade the + * inode's iopen glock from SH to EX mode in order to determine if we can + * delete the inode. The other nodes are supposed to evict the inode from + * their caches if they can, and to poke the inode's inode glock if they cannot + * do so. Either behavior allows gfs2_upgrade_iopen_glock() to proceed + * quickly, but if the other nodes are not cooperating, the lock upgrading + * attempt will time out. Since inodes are evicted sequentially, this can add + * up quickly. + * + * Function evict_inodes() tries to keep the s_inode_list_lock list locked over + * a long time, which prevents other inodes from being evicted concurrently. + * This precludes the cooperative behavior we are looking for. This special + * version of evict_inodes() avoids that. + * + * Modeled after drop_pagecache_sb(). + */ +static void gfs2_evict_inodes(struct super_block *sb) +{ + struct inode *inode, *toput_inode = NULL; + struct gfs2_sbd *sdp = sb->s_fs_info; + + set_bit(SDF_EVICTING, &sdp->sd_flags); + + spin_lock(&sb->s_inode_list_lock); + list_for_each_entry(inode, &sb->s_inodes, i_sb_list) { + spin_lock(&inode->i_lock); + if ((inode_state_read(inode) & (I_FREEING | I_WILL_FREE | I_NEW)) && + !need_resched()) { + spin_unlock(&inode->i_lock); + continue; + } + __iget(inode); + spin_unlock(&inode->i_lock); + spin_unlock(&sb->s_inode_list_lock); + + iput(toput_inode); + toput_inode = inode; + + cond_resched(); + spin_lock(&sb->s_inode_list_lock); + } + spin_unlock(&sb->s_inode_list_lock); + iput(toput_inode); +} + +static void gfs2_kill_sb(struct super_block *sb) +{ + struct gfs2_sbd *sdp = sb->s_fs_info; + + if (sdp == NULL) { + kill_block_super(sb); + return; + } + + gfs2_log_flush(sdp, NULL, GFS2_LOG_HEAD_FLUSH_SYNC | GFS2_LFC_KILL_SB); + dput(sdp->sd_root_dir); + dput(sdp->sd_master_dir); + sdp->sd_root_dir = NULL; + sdp->sd_master_dir = NULL; + shrink_dcache_sb(sb); + + set_bit(SDF_KILL, &sdp->sd_flags); + gfs2_evict_inodes(sb); + + /* + * Flush and then drain the delete workqueue here (via + * destroy_workqueue()) to ensure that any delete work that + * may be running will also see the SDF_KILL flag. + */ + gfs2_flush_delete_work(sdp); + destroy_workqueue(sdp->sd_delete_wq); + + kill_block_super(sb); +} + +struct file_system_type gfs2_fs_type = { + .name = "gfs2", + .fs_flags = FS_REQUIRES_DEV, + .init_fs_context = gfs2_init_fs_context, + .parameters = gfs2_fs_parameters, + .kill_sb = gfs2_kill_sb, + .owner = THIS_MODULE, +}; +MODULE_ALIAS_FS("gfs2"); + +struct file_system_type gfs2meta_fs_type = { + .name = "gfs2meta", + .fs_flags = FS_REQUIRES_DEV, + .init_fs_context = gfs2_meta_init_fs_context, + .owner = THIS_MODULE, +}; +MODULE_ALIAS_FS("gfs2meta"); diff --git a/fs/gfs2/quota.c b/fs/gfs2/quota.c new file mode 100644 index 000000000..001c8b39c --- /dev/null +++ b/fs/gfs2/quota.c @@ -0,0 +1,1829 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2007 Red Hat, Inc. All rights reserved. + */ + +/* + * Quota change tags are associated with each transaction that allocates or + * deallocates space. Those changes are accumulated locally to each node (in a + * per-node file) and then are periodically synced to the quota file. This + * avoids the bottleneck of constantly touching the quota file, but introduces + * fuzziness in the current usage value of IDs that are being used on different + * nodes in the cluster simultaneously. So, it is possible for a user on + * multiple nodes to overrun their quota, but that overrun is controlable. + * Since quota tags are part of transactions, there is no need for a quota check + * program to be run on node crashes or anything like that. + * + * There are couple of knobs that let the administrator manage the quota + * fuzziness. "quota_quantum" sets the maximum time a quota change can be + * sitting on one node before being synced to the quota file. (The default is + * 60 seconds.) Another knob, "quota_scale" controls how quickly the frequency + * of quota file syncs increases as the user moves closer to their limit. The + * more frequent the syncs, the more accurate the quota enforcement, but that + * means that there is more contention between the nodes for the quota file. + * The default value is one. This sets the maximum theoretical quota overrun + * (with infinite node with infinite bandwidth) to twice the user's limit. (In + * practice, the maximum overrun you see should be much less.) A "quota_scale" + * number greater than one makes quota syncs more frequent and reduces the + * maximum overrun. Numbers less than one (but greater than zero) make quota + * syncs less frequent. + * + * GFS quotas also use per-ID Lock Value Blocks (LVBs) to cache the contents of + * the quota file, so it is not being constantly read. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gfs2.h" +#include "incore.h" +#include "bmap.h" +#include "glock.h" +#include "glops.h" +#include "log.h" +#include "meta_io.h" +#include "quota.h" +#include "rgrp.h" +#include "super.h" +#include "trans.h" +#include "inode.h" +#include "util.h" + +#define GFS2_QD_HASH_SHIFT 12 +#define GFS2_QD_HASH_SIZE BIT(GFS2_QD_HASH_SHIFT) +#define GFS2_QD_HASH_MASK (GFS2_QD_HASH_SIZE - 1) + +/* Lock order: qd_lock -> bucket lock -> qd->lockref.lock -> lru lock */ +/* -> sd_bitmap_lock */ +static DEFINE_SPINLOCK(qd_lock); +struct list_lru gfs2_qd_lru; + +static struct hlist_bl_head qd_hash_table[GFS2_QD_HASH_SIZE]; + +static unsigned int gfs2_qd_hash(const struct gfs2_sbd *sdp, + const struct kqid qid) +{ + unsigned int h; + + h = jhash(&sdp, sizeof(struct gfs2_sbd *), 0); + h = jhash(&qid, sizeof(struct kqid), h); + + return h & GFS2_QD_HASH_MASK; +} + +static inline void spin_lock_bucket(unsigned int hash) +{ + hlist_bl_lock(&qd_hash_table[hash]); +} + +static inline void spin_unlock_bucket(unsigned int hash) +{ + hlist_bl_unlock(&qd_hash_table[hash]); +} + +static void gfs2_qd_dealloc(struct rcu_head *rcu) +{ + struct gfs2_quota_data *qd = container_of(rcu, struct gfs2_quota_data, qd_rcu); + struct gfs2_sbd *sdp = qd->qd_sbd; + + kmem_cache_free(gfs2_quotad_cachep, qd); + if (atomic_dec_and_test(&sdp->sd_quota_count)) + wake_up(&sdp->sd_kill_wait); +} + +static void gfs2_qd_dispose(struct gfs2_quota_data *qd) +{ + struct gfs2_sbd *sdp = qd->qd_sbd; + + spin_lock(&qd_lock); + list_del(&qd->qd_list); + spin_unlock(&qd_lock); + + spin_lock_bucket(qd->qd_hash); + hlist_bl_del_rcu(&qd->qd_hlist); + spin_unlock_bucket(qd->qd_hash); + + if (!gfs2_withdrawn(sdp)) { + gfs2_assert_warn(sdp, !qd->qd_change); + gfs2_assert_warn(sdp, !qd->qd_slot_ref); + gfs2_assert_warn(sdp, !qd->qd_bh_count); + } + + gfs2_glock_put(qd->qd_gl); + call_rcu(&qd->qd_rcu, gfs2_qd_dealloc); +} + +static void gfs2_qd_list_dispose(struct list_head *list) +{ + struct gfs2_quota_data *qd; + + while (!list_empty(list)) { + qd = list_first_entry(list, struct gfs2_quota_data, qd_lru); + list_del(&qd->qd_lru); + + gfs2_qd_dispose(qd); + } +} + + +static enum lru_status gfs2_qd_isolate(struct list_head *item, + struct list_lru_one *lru, void *arg) +{ + struct list_head *dispose = arg; + struct gfs2_quota_data *qd = + list_entry(item, struct gfs2_quota_data, qd_lru); + enum lru_status status; + + if (!spin_trylock(&qd->qd_lockref.lock)) + return LRU_SKIP; + + status = LRU_SKIP; + if (qd->qd_lockref.count == 0) { + lockref_mark_dead(&qd->qd_lockref); + list_lru_isolate_move(lru, &qd->qd_lru, dispose); + status = LRU_REMOVED; + } + + spin_unlock(&qd->qd_lockref.lock); + return status; +} + +static unsigned long gfs2_qd_shrink_scan(struct shrinker *shrink, + struct shrink_control *sc) +{ + LIST_HEAD(dispose); + unsigned long freed; + + if (!(sc->gfp_mask & __GFP_FS)) + return SHRINK_STOP; + + freed = list_lru_shrink_walk(&gfs2_qd_lru, sc, + gfs2_qd_isolate, &dispose); + + gfs2_qd_list_dispose(&dispose); + + return freed; +} + +static unsigned long gfs2_qd_shrink_count(struct shrinker *shrink, + struct shrink_control *sc) +{ + return vfs_pressure_ratio(list_lru_shrink_count(&gfs2_qd_lru, sc)); +} + +static struct shrinker *gfs2_qd_shrinker; + +int __init gfs2_qd_shrinker_init(void) +{ + gfs2_qd_shrinker = shrinker_alloc(SHRINKER_NUMA_AWARE, "gfs2-qd"); + if (!gfs2_qd_shrinker) + return -ENOMEM; + + gfs2_qd_shrinker->count_objects = gfs2_qd_shrink_count; + gfs2_qd_shrinker->scan_objects = gfs2_qd_shrink_scan; + + shrinker_register(gfs2_qd_shrinker); + + return 0; +} + +void gfs2_qd_shrinker_exit(void) +{ + shrinker_free(gfs2_qd_shrinker); +} + +static u64 qd2index(struct gfs2_quota_data *qd) +{ + struct kqid qid = qd->qd_id; + return (2 * (u64)from_kqid(&init_user_ns, qid)) + + ((qid.type == USRQUOTA) ? 0 : 1); +} + +static u64 qd2offset(struct gfs2_quota_data *qd) +{ + return qd2index(qd) * sizeof(struct gfs2_quota); +} + +static struct gfs2_quota_data *qd_alloc(unsigned hash, struct gfs2_sbd *sdp, struct kqid qid) +{ + struct gfs2_quota_data *qd; + int error; + + qd = kmem_cache_zalloc(gfs2_quotad_cachep, GFP_NOFS); + if (!qd) + return NULL; + + qd->qd_sbd = sdp; + lockref_init(&qd->qd_lockref); + qd->qd_id = qid; + qd->qd_slot = -1; + INIT_LIST_HEAD(&qd->qd_lru); + qd->qd_hash = hash; + + error = gfs2_glock_get(sdp, qd2index(qd), + &gfs2_quota_glops, CREATE, &qd->qd_gl); + if (error) + goto fail; + + return qd; + +fail: + kmem_cache_free(gfs2_quotad_cachep, qd); + return NULL; +} + +/* + * Lookup variant for callers which already hold qd_lock + bucket lock. + */ +static struct gfs2_quota_data * +gfs2_qd_search_bucket_noref(unsigned int hash, + const struct gfs2_sbd *sdp, + struct kqid qid) +{ + struct gfs2_quota_data *qd; + struct hlist_bl_node *h; + + hlist_bl_for_each_entry_rcu(qd, h, &qd_hash_table[hash], qd_hlist) { + if (!qid_eq(qd->qd_id, qid)) + continue; + if (qd->qd_sbd == sdp) + return qd; + } + + return NULL; +} + +static struct gfs2_quota_data * +gfs2_qd_search_bucket(unsigned int hash, const struct gfs2_sbd *sdp, struct kqid qid) +{ + struct gfs2_quota_data *qd; + + qd = gfs2_qd_search_bucket_noref(hash, sdp, qid); + if (qd && lockref_get_not_dead(&qd->qd_lockref)) { + list_lru_del_obj(&gfs2_qd_lru, &qd->qd_lru); + return qd; + } + + return NULL; +} + + +static int qd_get(struct gfs2_sbd *sdp, struct kqid qid, + struct gfs2_quota_data **qdp) +{ + struct gfs2_quota_data *qd, *new_qd; + unsigned int hash = gfs2_qd_hash(sdp, qid); + + rcu_read_lock(); + *qdp = qd = gfs2_qd_search_bucket(hash, sdp, qid); + rcu_read_unlock(); + + if (qd) + return 0; + + new_qd = qd_alloc(hash, sdp, qid); + if (!new_qd) + return -ENOMEM; + + spin_lock(&qd_lock); + spin_lock_bucket(hash); + *qdp = qd = gfs2_qd_search_bucket(hash, sdp, qid); + if (qd == NULL) { + *qdp = new_qd; + list_add(&new_qd->qd_list, &sdp->sd_quota_list); + hlist_bl_add_head_rcu(&new_qd->qd_hlist, &qd_hash_table[hash]); + atomic_inc(&sdp->sd_quota_count); + } + spin_unlock_bucket(hash); + spin_unlock(&qd_lock); + + if (qd) { + gfs2_glock_put(new_qd->qd_gl); + kmem_cache_free(gfs2_quotad_cachep, new_qd); + } + + return 0; +} + + +static void __qd_hold(struct gfs2_quota_data *qd) +{ + struct gfs2_sbd *sdp = qd->qd_sbd; + gfs2_assert(sdp, qd->qd_lockref.count > 0); + qd->qd_lockref.count++; +} + +static void qd_put(struct gfs2_quota_data *qd) +{ + struct gfs2_sbd *sdp; + + if (lockref_put_or_lock(&qd->qd_lockref)) + return; + + BUG_ON(lockref_is_dead(&qd->qd_lockref)); + sdp = qd->qd_sbd; + if (unlikely(!test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags))) { + lockref_mark_dead(&qd->qd_lockref); + spin_unlock(&qd->qd_lockref.lock); + + list_lru_del_obj(&gfs2_qd_lru, &qd->qd_lru); + gfs2_qd_dispose(qd); + return; + } + + qd->qd_lockref.count = 0; + list_lru_add_obj(&gfs2_qd_lru, &qd->qd_lru); + spin_unlock(&qd->qd_lockref.lock); +} + +static int slot_get(struct gfs2_quota_data *qd) +{ + struct gfs2_sbd *sdp = qd->qd_sbd; + unsigned int bit; + int error = 0; + + spin_lock(&sdp->sd_bitmap_lock); + if (qd->qd_slot_ref == 0) { + bit = find_first_zero_bit(sdp->sd_quota_bitmap, + sdp->sd_quota_slots); + if (bit >= sdp->sd_quota_slots) { + error = -ENOSPC; + goto out; + } + set_bit(bit, sdp->sd_quota_bitmap); + qd->qd_slot = bit; + } + qd->qd_slot_ref++; +out: + spin_unlock(&sdp->sd_bitmap_lock); + return error; +} + +static void slot_hold(struct gfs2_quota_data *qd) +{ + struct gfs2_sbd *sdp = qd->qd_sbd; + + spin_lock(&sdp->sd_bitmap_lock); + gfs2_assert(sdp, qd->qd_slot_ref); + qd->qd_slot_ref++; + spin_unlock(&sdp->sd_bitmap_lock); +} + +static void slot_put(struct gfs2_quota_data *qd) +{ + struct gfs2_sbd *sdp = qd->qd_sbd; + + spin_lock(&sdp->sd_bitmap_lock); + gfs2_assert(sdp, qd->qd_slot_ref); + if (!--qd->qd_slot_ref) { + BUG_ON(!test_and_clear_bit(qd->qd_slot, sdp->sd_quota_bitmap)); + qd->qd_slot = -1; + } + spin_unlock(&sdp->sd_bitmap_lock); +} + +static int bh_get(struct gfs2_quota_data *qd) +{ + struct gfs2_sbd *sdp = qd->qd_sbd; + struct inode *inode = sdp->sd_qc_inode; + struct gfs2_inode *ip = GFS2_I(inode); + unsigned int block, offset; + struct buffer_head *bh = NULL; + struct iomap iomap = { }; + int error; + + spin_lock(&qd->qd_lockref.lock); + if (qd->qd_bh_count) { + qd->qd_bh_count++; + spin_unlock(&qd->qd_lockref.lock); + return 0; + } + spin_unlock(&qd->qd_lockref.lock); + + block = qd->qd_slot / sdp->sd_qc_per_block; + offset = qd->qd_slot % sdp->sd_qc_per_block; + + error = gfs2_iomap_get(inode, + (loff_t)block << inode->i_blkbits, + i_blocksize(inode), &iomap); + if (error) + return error; + error = -ENOENT; + if (iomap.type != IOMAP_MAPPED) + return error; + + error = gfs2_meta_read(ip->i_gl, iomap.addr >> inode->i_blkbits, + DIO_WAIT, 0, &bh); + if (error) + return error; + error = -EIO; + if (gfs2_metatype_check(sdp, bh, GFS2_METATYPE_QC)) + goto out; + + spin_lock(&qd->qd_lockref.lock); + if (qd->qd_bh == NULL) { + qd->qd_bh = bh; + qd->qd_bh_qc = (struct gfs2_quota_change *) + (bh->b_data + sizeof(struct gfs2_meta_header) + + offset * sizeof(struct gfs2_quota_change)); + bh = NULL; + } + qd->qd_bh_count++; + spin_unlock(&qd->qd_lockref.lock); + error = 0; + +out: + brelse(bh); + return error; +} + +static void bh_put(struct gfs2_quota_data *qd) +{ + struct gfs2_sbd *sdp = qd->qd_sbd; + struct buffer_head *bh = NULL; + + spin_lock(&qd->qd_lockref.lock); + gfs2_assert(sdp, qd->qd_bh_count); + if (!--qd->qd_bh_count) { + bh = qd->qd_bh; + qd->qd_bh = NULL; + qd->qd_bh_qc = NULL; + } + spin_unlock(&qd->qd_lockref.lock); + brelse(bh); +} + +static bool qd_grab_sync(struct gfs2_sbd *sdp, struct gfs2_quota_data *qd, + u64 sync_gen) +{ + bool ret = false; + + spin_lock(&qd->qd_lockref.lock); + if (test_bit(QDF_LOCKED, &qd->qd_flags) || + !test_bit(QDF_CHANGE, &qd->qd_flags) || + qd->qd_sync_gen >= sync_gen) + goto out; + + if (lockref_is_dead(&qd->qd_lockref)) + goto out; + qd->qd_lockref.count++; + + list_move_tail(&qd->qd_list, &sdp->sd_quota_list); + set_bit(QDF_LOCKED, &qd->qd_flags); + qd->qd_change_sync = qd->qd_change; + slot_hold(qd); + ret = true; + +out: + spin_unlock(&qd->qd_lockref.lock); + return ret; +} + +static void qd_ungrab_sync(struct gfs2_quota_data *qd) +{ + clear_bit(QDF_LOCKED, &qd->qd_flags); + slot_put(qd); + qd_put(qd); +} + +static void qdsb_put(struct gfs2_quota_data *qd) +{ + bh_put(qd); + slot_put(qd); + qd_put(qd); +} + +static void qd_unlock(struct gfs2_quota_data *qd) +{ + spin_lock(&qd->qd_lockref.lock); + gfs2_assert_warn(qd->qd_sbd, test_bit(QDF_LOCKED, &qd->qd_flags)); + clear_bit(QDF_LOCKED, &qd->qd_flags); + spin_unlock(&qd->qd_lockref.lock); + qdsb_put(qd); +} + +static int qdsb_get(struct gfs2_sbd *sdp, struct kqid qid, + struct gfs2_quota_data **qdp) +{ + int error; + + error = qd_get(sdp, qid, qdp); + if (error) + return error; + + error = slot_get(*qdp); + if (error) + goto fail; + + error = bh_get(*qdp); + if (error) + goto fail_slot; + + return 0; + +fail_slot: + slot_put(*qdp); +fail: + qd_put(*qdp); + return error; +} + +/** + * gfs2_qa_get - make sure we have a quota allocations data structure, + * if necessary + * @ip: the inode for this reservation + */ +int gfs2_qa_get(struct gfs2_inode *ip) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + struct inode *inode = &ip->i_inode; + + if (sdp->sd_args.ar_quota == GFS2_QUOTA_OFF) + return 0; + + spin_lock(&inode->i_lock); + if (ip->i_qadata == NULL) { + struct gfs2_qadata *tmp; + + spin_unlock(&inode->i_lock); + tmp = kmem_cache_zalloc(gfs2_qadata_cachep, GFP_NOFS); + if (!tmp) + return -ENOMEM; + + spin_lock(&inode->i_lock); + if (ip->i_qadata == NULL) + ip->i_qadata = tmp; + else + kmem_cache_free(gfs2_qadata_cachep, tmp); + } + ip->i_qadata->qa_ref++; + spin_unlock(&inode->i_lock); + return 0; +} + +void gfs2_qa_put(struct gfs2_inode *ip) +{ + struct inode *inode = &ip->i_inode; + + spin_lock(&inode->i_lock); + if (ip->i_qadata && --ip->i_qadata->qa_ref == 0) { + kmem_cache_free(gfs2_qadata_cachep, ip->i_qadata); + ip->i_qadata = NULL; + } + spin_unlock(&inode->i_lock); +} + +int gfs2_quota_hold(struct gfs2_inode *ip, kuid_t uid, kgid_t gid) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + struct gfs2_quota_data **qd; + int error; + + if (sdp->sd_args.ar_quota == GFS2_QUOTA_OFF) + return 0; + + error = gfs2_qa_get(ip); + if (error) + return error; + + qd = ip->i_qadata->qa_qd; + + if (gfs2_assert_warn(sdp, !ip->i_qadata->qa_qd_num) || + gfs2_assert_warn(sdp, !test_bit(GIF_QD_LOCKED, &ip->i_flags))) { + error = -EIO; + gfs2_qa_put(ip); + goto out; + } + + error = qdsb_get(sdp, make_kqid_uid(ip->i_inode.i_uid), qd); + if (error) + goto out_unhold; + ip->i_qadata->qa_qd_num++; + qd++; + + error = qdsb_get(sdp, make_kqid_gid(ip->i_inode.i_gid), qd); + if (error) + goto out_unhold; + ip->i_qadata->qa_qd_num++; + qd++; + + if (!uid_eq(uid, NO_UID_QUOTA_CHANGE) && + !uid_eq(uid, ip->i_inode.i_uid)) { + error = qdsb_get(sdp, make_kqid_uid(uid), qd); + if (error) + goto out_unhold; + ip->i_qadata->qa_qd_num++; + qd++; + } + + if (!gid_eq(gid, NO_GID_QUOTA_CHANGE) && + !gid_eq(gid, ip->i_inode.i_gid)) { + error = qdsb_get(sdp, make_kqid_gid(gid), qd); + if (error) + goto out_unhold; + ip->i_qadata->qa_qd_num++; + qd++; + } + +out_unhold: + if (error) + gfs2_quota_unhold(ip); +out: + return error; +} + +void gfs2_quota_unhold(struct gfs2_inode *ip) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + u32 x; + + if (ip->i_qadata == NULL) + return; + + gfs2_assert_warn(sdp, !test_bit(GIF_QD_LOCKED, &ip->i_flags)); + + for (x = 0; x < ip->i_qadata->qa_qd_num; x++) { + qdsb_put(ip->i_qadata->qa_qd[x]); + ip->i_qadata->qa_qd[x] = NULL; + } + ip->i_qadata->qa_qd_num = 0; + gfs2_qa_put(ip); +} + +static int sort_qd(const void *a, const void *b) +{ + const struct gfs2_quota_data *qd_a = *(const struct gfs2_quota_data **)a; + const struct gfs2_quota_data *qd_b = *(const struct gfs2_quota_data **)b; + + if (qid_lt(qd_a->qd_id, qd_b->qd_id)) + return -1; + if (qid_lt(qd_b->qd_id, qd_a->qd_id)) + return 1; + return 0; +} + +static void do_qc(struct gfs2_quota_data *qd, s64 change) +{ + struct gfs2_sbd *sdp = qd->qd_sbd; + struct gfs2_inode *ip = GFS2_I(sdp->sd_qc_inode); + struct gfs2_quota_change *qc = qd->qd_bh_qc; + bool needs_put = false; + s64 x; + + gfs2_trans_add_meta(ip->i_gl, qd->qd_bh); + + /* + * The QDF_CHANGE flag indicates that the slot in the quota change file + * is used. Here, we use the value of qc->qc_change when the slot is + * used, and we assume a value of 0 otherwise. + */ + + spin_lock(&qd->qd_lockref.lock); + + x = 0; + if (test_bit(QDF_CHANGE, &qd->qd_flags)) + x = be64_to_cpu(qc->qc_change); + x += change; + qd->qd_change += change; + + if (!x && test_bit(QDF_CHANGE, &qd->qd_flags)) { + /* The slot in the quota change file becomes unused. */ + clear_bit(QDF_CHANGE, &qd->qd_flags); + qc->qc_flags = 0; + qc->qc_id = 0; + needs_put = true; + } else if (x && !test_bit(QDF_CHANGE, &qd->qd_flags)) { + /* The slot in the quota change file becomes used. */ + set_bit(QDF_CHANGE, &qd->qd_flags); + __qd_hold(qd); + slot_hold(qd); + + qc->qc_flags = 0; + if (qd->qd_id.type == USRQUOTA) + qc->qc_flags = cpu_to_be32(GFS2_QCF_USER); + qc->qc_id = cpu_to_be32(from_kqid(&init_user_ns, qd->qd_id)); + } + qc->qc_change = cpu_to_be64(x); + + spin_unlock(&qd->qd_lockref.lock); + + if (needs_put) { + slot_put(qd); + qd_put(qd); + } + if (change < 0) /* Reset quiet flag if we freed some blocks */ + clear_bit(QDF_QMSG_QUIET, &qd->qd_flags); +} + +static int gfs2_write_buf_to_page(struct gfs2_sbd *sdp, unsigned long index, + unsigned off, void *buf, unsigned bytes) +{ + struct gfs2_inode *ip = GFS2_I(sdp->sd_quota_inode); + struct inode *inode = &ip->i_inode; + struct address_space *mapping = inode->i_mapping; + struct folio *folio; + struct buffer_head *bh; + u64 blk; + unsigned bsize = sdp->sd_sb.sb_bsize, bnum = 0, boff = 0; + unsigned to_write = bytes, pg_off = off; + + blk = index << (PAGE_SHIFT - sdp->sd_sb.sb_bsize_shift); + boff = off % bsize; + + folio = filemap_grab_folio(mapping, index); + if (IS_ERR(folio)) + return PTR_ERR(folio); + bh = folio_buffers(folio); + if (!bh) + bh = create_empty_buffers(folio, bsize, 0); + + for (;;) { + /* Find the beginning block within the folio */ + if (pg_off >= ((bnum * bsize) + bsize)) { + bh = bh->b_this_page; + bnum++; + blk++; + continue; + } + if (!buffer_mapped(bh)) { + gfs2_block_map(inode, blk, bh, 1); + if (!buffer_mapped(bh)) + goto unlock_out; + /* If it's a newly allocated disk block, zero it */ + if (buffer_new(bh)) + folio_zero_range(folio, bnum * bsize, + bh->b_size); + } + if (folio_test_uptodate(folio)) + set_buffer_uptodate(bh); + if (bh_read(bh, REQ_META | REQ_PRIO) < 0) + goto unlock_out; + gfs2_trans_add_data(ip->i_gl, bh); + + /* If we need to write to the next block as well */ + if (to_write > (bsize - boff)) { + pg_off += (bsize - boff); + to_write -= (bsize - boff); + boff = pg_off % bsize; + continue; + } + break; + } + + /* Write to the folio, now that we have setup the buffer(s) */ + memcpy_to_folio(folio, off, buf, bytes); + flush_dcache_folio(folio); + folio_unlock(folio); + folio_put(folio); + + return 0; + +unlock_out: + folio_unlock(folio); + folio_put(folio); + return -EIO; +} + +static int gfs2_write_disk_quota(struct gfs2_sbd *sdp, struct gfs2_quota *qp, + loff_t loc) +{ + unsigned long pg_beg; + unsigned pg_off, nbytes, overflow = 0; + int error; + void *ptr; + + nbytes = sizeof(struct gfs2_quota); + + pg_beg = loc >> PAGE_SHIFT; + pg_off = offset_in_page(loc); + + /* If the quota straddles a page boundary, split the write in two */ + if ((pg_off + nbytes) > PAGE_SIZE) + overflow = (pg_off + nbytes) - PAGE_SIZE; + + ptr = qp; + error = gfs2_write_buf_to_page(sdp, pg_beg, pg_off, ptr, + nbytes - overflow); + /* If there's an overflow, write the remaining bytes to the next page */ + if (!error && overflow) + error = gfs2_write_buf_to_page(sdp, pg_beg + 1, 0, + ptr + nbytes - overflow, + overflow); + return error; +} + +/** + * gfs2_adjust_quota - adjust record of current block usage + * @sdp: The superblock + * @loc: Offset of the entry in the quota file + * @change: The amount of usage change to record + * @qd: The quota data + * @fdq: The updated limits to record + * + * This function was mostly borrowed from gfs2_block_truncate_page which was + * in turn mostly borrowed from ext3 + * + * Returns: 0 or -ve on error + */ + +static int gfs2_adjust_quota(struct gfs2_sbd *sdp, loff_t loc, + s64 change, struct gfs2_quota_data *qd, + struct qc_dqblk *fdq) +{ + struct gfs2_inode *ip = GFS2_I(sdp->sd_quota_inode); + struct inode *inode = &ip->i_inode; + struct gfs2_quota q; + int err; + u64 size; + + if (gfs2_is_stuffed(ip)) { + err = gfs2_unstuff_dinode(ip); + if (err) + return err; + } + + memset(&q, 0, sizeof(struct gfs2_quota)); + err = gfs2_internal_read(ip, (char *)&q, &loc, sizeof(q)); + if (err < 0) + return err; + + loc -= sizeof(q); /* gfs2_internal_read would've advanced the loc ptr */ + be64_add_cpu(&q.qu_value, change); + if (((s64)be64_to_cpu(q.qu_value)) < 0) + q.qu_value = 0; /* Never go negative on quota usage */ + spin_lock(&qd->qd_lockref.lock); + qd->qd_qb.qb_value = q.qu_value; + if (fdq) { + if (fdq->d_fieldmask & QC_SPC_SOFT) { + q.qu_warn = cpu_to_be64(fdq->d_spc_softlimit >> sdp->sd_sb.sb_bsize_shift); + qd->qd_qb.qb_warn = q.qu_warn; + } + if (fdq->d_fieldmask & QC_SPC_HARD) { + q.qu_limit = cpu_to_be64(fdq->d_spc_hardlimit >> sdp->sd_sb.sb_bsize_shift); + qd->qd_qb.qb_limit = q.qu_limit; + } + if (fdq->d_fieldmask & QC_SPACE) { + q.qu_value = cpu_to_be64(fdq->d_space >> sdp->sd_sb.sb_bsize_shift); + qd->qd_qb.qb_value = q.qu_value; + } + } + spin_unlock(&qd->qd_lockref.lock); + + err = gfs2_write_disk_quota(sdp, &q, loc); + if (!err) { + size = loc + sizeof(struct gfs2_quota); + if (size > inode->i_size) + i_size_write(inode, size); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + mark_inode_dirty(inode); + set_bit(QDF_REFRESH, &qd->qd_flags); + } + + return err; +} + +static int do_sync(unsigned int num_qd, struct gfs2_quota_data **qda, + u64 sync_gen) +{ + struct gfs2_sbd *sdp = (*qda)->qd_sbd; + struct gfs2_inode *ip = GFS2_I(sdp->sd_quota_inode); + struct gfs2_alloc_parms ap = {}; + unsigned int data_blocks, ind_blocks; + struct gfs2_holder *ghs, i_gh; + unsigned int qx, x; + struct gfs2_quota_data *qd; + unsigned reserved; + loff_t offset; + unsigned int nalloc = 0, blocks; + int error; + + gfs2_write_calc_reserv(ip, sizeof(struct gfs2_quota), + &data_blocks, &ind_blocks); + + ghs = kmalloc_objs(struct gfs2_holder, num_qd, GFP_NOFS); + if (!ghs) + return -ENOMEM; + + sort(qda, num_qd, sizeof(struct gfs2_quota_data *), sort_qd, NULL); + inode_lock(&ip->i_inode); + for (qx = 0; qx < num_qd; qx++) { + error = gfs2_glock_nq_init(qda[qx]->qd_gl, LM_ST_EXCLUSIVE, + GL_NOCACHE, &ghs[qx]); + if (error) + goto out_dq; + } + + error = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &i_gh); + if (error) + goto out_dq; + + for (x = 0; x < num_qd; x++) { + offset = qd2offset(qda[x]); + if (gfs2_write_alloc_required(ip, offset, + sizeof(struct gfs2_quota))) + nalloc++; + } + + /* + * 1 blk for unstuffing inode if stuffed. We add this extra + * block to the reservation unconditionally. If the inode + * doesn't need unstuffing, the block will be released to the + * rgrp since it won't be allocated during the transaction + */ + /* +3 in the end for unstuffing block, inode size update block + * and another block in case quota straddles page boundary and + * two blocks need to be updated instead of 1 */ + blocks = num_qd * data_blocks + RES_DINODE + num_qd + 3; + + reserved = 1 + (nalloc * (data_blocks + ind_blocks)); + ap.target = reserved; + error = gfs2_inplace_reserve(ip, &ap); + if (error) + goto out_alloc; + + if (nalloc) + blocks += gfs2_rg_blocks(ip, reserved) + nalloc * ind_blocks + RES_STATFS; + + error = gfs2_trans_begin(sdp, blocks, 0); + if (error) + goto out_ipres; + + for (x = 0; x < num_qd; x++) { + qd = qda[x]; + offset = qd2offset(qd); + error = gfs2_adjust_quota(sdp, offset, qd->qd_change_sync, qd, + NULL); + if (error) + goto out_end_trans; + + do_qc(qd, -qd->qd_change_sync); + set_bit(QDF_REFRESH, &qd->qd_flags); + } + +out_end_trans: + gfs2_trans_end(sdp); +out_ipres: + gfs2_inplace_release(ip); +out_alloc: + gfs2_glock_dq_uninit(&i_gh); +out_dq: + while (qx--) + gfs2_glock_dq_uninit(&ghs[qx]); + inode_unlock(&ip->i_inode); + kfree(ghs); + gfs2_log_flush(glock_sbd(ip->i_gl), ip->i_gl, + GFS2_LOG_HEAD_FLUSH_NORMAL | GFS2_LFC_DO_SYNC); + if (!error) { + for (x = 0; x < num_qd; x++) { + qd = qda[x]; + spin_lock(&qd->qd_lockref.lock); + if (qd->qd_sync_gen < sync_gen) + qd->qd_sync_gen = sync_gen; + spin_unlock(&qd->qd_lockref.lock); + } + } + return error; +} + +static int update_qd(struct gfs2_sbd *sdp, struct gfs2_quota_data *qd) +{ + struct gfs2_inode *ip = GFS2_I(sdp->sd_quota_inode); + struct gfs2_quota q; + struct gfs2_quota_lvb *qlvb; + loff_t pos; + int error; + + memset(&q, 0, sizeof(struct gfs2_quota)); + pos = qd2offset(qd); + error = gfs2_internal_read(ip, (char *)&q, &pos, sizeof(q)); + if (error < 0) + return error; + + qlvb = (struct gfs2_quota_lvb *)qd->qd_gl->gl_lksb.sb_lvbptr; + qlvb->qb_magic = cpu_to_be32(GFS2_MAGIC); + qlvb->__pad = 0; + qlvb->qb_limit = q.qu_limit; + qlvb->qb_warn = q.qu_warn; + qlvb->qb_value = q.qu_value; + spin_lock(&qd->qd_lockref.lock); + qd->qd_qb = *qlvb; + spin_unlock(&qd->qd_lockref.lock); + + return 0; +} + +static int do_glock(struct gfs2_quota_data *qd, int force_refresh, + struct gfs2_holder *q_gh) +{ + struct gfs2_sbd *sdp = qd->qd_sbd; + struct gfs2_inode *ip = GFS2_I(sdp->sd_quota_inode); + struct gfs2_holder i_gh; + int error; + + gfs2_assert_warn(sdp, sdp == glock_sbd(qd->qd_gl)); +restart: + error = gfs2_glock_nq_init(qd->qd_gl, LM_ST_SHARED, 0, q_gh); + if (error) + return error; + + if (test_and_clear_bit(QDF_REFRESH, &qd->qd_flags)) + force_refresh = FORCE; + + spin_lock(&qd->qd_lockref.lock); + qd->qd_qb = *(struct gfs2_quota_lvb *)qd->qd_gl->gl_lksb.sb_lvbptr; + spin_unlock(&qd->qd_lockref.lock); + + if (force_refresh || qd->qd_qb.qb_magic != cpu_to_be32(GFS2_MAGIC)) { + gfs2_glock_dq_uninit(q_gh); + error = gfs2_glock_nq_init(qd->qd_gl, LM_ST_EXCLUSIVE, + GL_NOCACHE, q_gh); + if (error) + return error; + + error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, 0, &i_gh); + if (error) + goto fail; + + error = update_qd(sdp, qd); + if (error) + goto fail_gunlock; + + gfs2_glock_dq_uninit(&i_gh); + gfs2_glock_dq_uninit(q_gh); + force_refresh = 0; + goto restart; + } + + return 0; + +fail_gunlock: + gfs2_glock_dq_uninit(&i_gh); +fail: + gfs2_glock_dq_uninit(q_gh); + return error; +} + +int gfs2_quota_lock(struct gfs2_inode *ip, kuid_t uid, kgid_t gid) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + struct gfs2_quota_data *qd; + u32 x; + int error; + + if (sdp->sd_args.ar_quota == GFS2_QUOTA_OFF) + return 0; + + error = gfs2_quota_hold(ip, uid, gid); + if (error) + return error; + + sort(ip->i_qadata->qa_qd, ip->i_qadata->qa_qd_num, + sizeof(struct gfs2_quota_data *), sort_qd, NULL); + + for (x = 0; x < ip->i_qadata->qa_qd_num; x++) { + qd = ip->i_qadata->qa_qd[x]; + error = do_glock(qd, NO_FORCE, &ip->i_qadata->qa_qd_ghs[x]); + if (error) + break; + } + + if (!error) + set_bit(GIF_QD_LOCKED, &ip->i_flags); + else { + while (x--) + gfs2_glock_dq_uninit(&ip->i_qadata->qa_qd_ghs[x]); + gfs2_quota_unhold(ip); + } + + return error; +} + +static bool need_sync(struct gfs2_quota_data *qd) +{ + struct gfs2_sbd *sdp = qd->qd_sbd; + struct gfs2_tune *gt = &sdp->sd_tune; + s64 value, change, limit; + unsigned int num, den; + int ret = false; + + spin_lock(&qd->qd_lockref.lock); + if (!qd->qd_qb.qb_limit) + goto out; + + change = qd->qd_change; + if (change <= 0) + goto out; + value = (s64)be64_to_cpu(qd->qd_qb.qb_value); + limit = (s64)be64_to_cpu(qd->qd_qb.qb_limit); + if (value >= limit) + goto out; + + spin_lock(>->gt_spin); + num = gt->gt_quota_scale_num; + den = gt->gt_quota_scale_den; + spin_unlock(>->gt_spin); + + change *= gfs2_jindex_size(sdp) * num; + change = div_s64(change, den); + if (value + change < limit) + goto out; + + ret = true; +out: + spin_unlock(&qd->qd_lockref.lock); + return ret; +} + +void gfs2_quota_unlock(struct gfs2_inode *ip) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + struct gfs2_quota_data *qda[2 * GFS2_MAXQUOTAS]; + unsigned int count = 0; + u32 x; + + if (!test_and_clear_bit(GIF_QD_LOCKED, &ip->i_flags)) + return; + + for (x = 0; x < ip->i_qadata->qa_qd_num; x++) { + struct gfs2_quota_data *qd; + bool sync; + int error; + + qd = ip->i_qadata->qa_qd[x]; + sync = need_sync(qd); + + gfs2_glock_dq_uninit(&ip->i_qadata->qa_qd_ghs[x]); + if (!sync) + continue; + + spin_lock(&qd_lock); + sync = qd_grab_sync(sdp, qd, U64_MAX); + spin_unlock(&qd_lock); + + if (!sync) + continue; + + gfs2_assert_warn(sdp, qd->qd_change_sync); + error = bh_get(qd); + if (error) { + qd_ungrab_sync(qd); + continue; + } + + qda[count++] = qd; + } + + if (count) { + u64 sync_gen = READ_ONCE(sdp->sd_quota_sync_gen); + + do_sync(count, qda, sync_gen); + for (x = 0; x < count; x++) + qd_unlock(qda[x]); + } + + gfs2_quota_unhold(ip); +} + +#define MAX_LINE 256 + +static void print_message(struct gfs2_quota_data *qd, char *type) +{ + struct gfs2_sbd *sdp = qd->qd_sbd; + + if (sdp->sd_args.ar_quota != GFS2_QUOTA_QUIET) { + fs_info(sdp, "quota %s for %s %u\n", + type, + (qd->qd_id.type == USRQUOTA) ? "user" : "group", + from_kqid(&init_user_ns, qd->qd_id)); + } +} + +/** + * gfs2_quota_check - check if allocating new blocks will exceed quota + * @ip: The inode for which this check is being performed + * @uid: The uid to check against + * @gid: The gid to check against + * @ap: The allocation parameters. ap->target contains the requested + * blocks. ap->min_target, if set, contains the minimum blks + * requested. + * + * Returns: 0 on success. + * min_req = ap->min_target ? ap->min_target : ap->target; + * quota must allow at least min_req blks for success and + * ap->allowed is set to the number of blocks allowed + * + * -EDQUOT otherwise, quota violation. ap->allowed is set to number + * of blocks available. + */ +int gfs2_quota_check(struct gfs2_inode *ip, kuid_t uid, kgid_t gid, + struct gfs2_alloc_parms *ap) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + struct gfs2_quota_data *qd; + s64 value, warn, limit; + u32 x; + int error = 0; + + ap->allowed = UINT_MAX; /* Assume we are permitted a whole lot */ + if (!test_bit(GIF_QD_LOCKED, &ip->i_flags)) + return 0; + + for (x = 0; x < ip->i_qadata->qa_qd_num; x++) { + qd = ip->i_qadata->qa_qd[x]; + + if (!(qid_eq(qd->qd_id, make_kqid_uid(uid)) || + qid_eq(qd->qd_id, make_kqid_gid(gid)))) + continue; + + spin_lock(&qd->qd_lockref.lock); + warn = (s64)be64_to_cpu(qd->qd_qb.qb_warn); + limit = (s64)be64_to_cpu(qd->qd_qb.qb_limit); + value = (s64)be64_to_cpu(qd->qd_qb.qb_value); + value += qd->qd_change; + spin_unlock(&qd->qd_lockref.lock); + + if (limit > 0 && (limit - value) < ap->allowed) + ap->allowed = limit - value; + /* If we can't meet the target */ + if (limit && limit < (value + (s64)ap->target)) { + /* If no min_target specified or we don't meet + * min_target, return -EDQUOT */ + if (!ap->min_target || ap->min_target > ap->allowed) { + if (!test_and_set_bit(QDF_QMSG_QUIET, + &qd->qd_flags)) { + print_message(qd, "exceeded"); + quota_send_warning(qd->qd_id, + sdp->sd_vfs->s_dev, + QUOTA_NL_BHARDWARN); + } + error = -EDQUOT; + break; + } + } else if (warn && warn < value && + time_after_eq(jiffies, qd->qd_last_warn + + gfs2_tune_get(sdp, gt_quota_warn_period) + * HZ)) { + quota_send_warning(qd->qd_id, + sdp->sd_vfs->s_dev, QUOTA_NL_BSOFTWARN); + print_message(qd, "warning"); + error = 0; + qd->qd_last_warn = jiffies; + } + } + return error; +} + +void gfs2_quota_change(struct gfs2_inode *ip, s64 change, + kuid_t uid, kgid_t gid) +{ + struct gfs2_quota_data *qd; + u32 x; + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + + if (sdp->sd_args.ar_quota == GFS2_QUOTA_OFF || + gfs2_assert_warn(sdp, change)) + return; + if (ip->i_diskflags & GFS2_DIF_SYSTEM) + return; + + if (gfs2_assert_withdraw(sdp, ip->i_qadata && + ip->i_qadata->qa_ref > 0)) + return; + for (x = 0; x < ip->i_qadata->qa_qd_num; x++) { + qd = ip->i_qadata->qa_qd[x]; + + if (qid_eq(qd->qd_id, make_kqid_uid(uid)) || + qid_eq(qd->qd_id, make_kqid_gid(gid))) { + do_qc(qd, change); + } + } +} + +int gfs2_quota_sync(struct super_block *sb, int type) +{ + struct gfs2_sbd *sdp = sb->s_fs_info; + struct gfs2_quota_data **qda; + unsigned int max_qd = PAGE_SIZE / sizeof(struct gfs2_holder); + u64 sync_gen; + int error = 0; + + if (sb_rdonly(sdp->sd_vfs)) + return 0; + + qda = kzalloc_objs(struct gfs2_quota_data *, max_qd); + if (!qda) + return -ENOMEM; + + mutex_lock(&sdp->sd_quota_sync_mutex); + sync_gen = sdp->sd_quota_sync_gen + 1; + + do { + struct gfs2_quota_data *iter; + unsigned int num_qd = 0; + unsigned int x; + + spin_lock(&qd_lock); + list_for_each_entry(iter, &sdp->sd_quota_list, qd_list) { + if (qd_grab_sync(sdp, iter, sync_gen)) { + qda[num_qd++] = iter; + if (num_qd == max_qd) + break; + } + } + spin_unlock(&qd_lock); + + if (!num_qd) + break; + + for (x = 0; x < num_qd; x++) { + error = bh_get(qda[x]); + if (!error) + continue; + + while (x < num_qd) + qd_ungrab_sync(qda[--num_qd]); + break; + } + + if (!error) { + WRITE_ONCE(sdp->sd_quota_sync_gen, sync_gen); + error = do_sync(num_qd, qda, sync_gen); + } + + for (x = 0; x < num_qd; x++) + qd_unlock(qda[x]); + } while (!error); + + mutex_unlock(&sdp->sd_quota_sync_mutex); + kfree(qda); + + return error; +} + +int gfs2_quota_refresh(struct gfs2_sbd *sdp, struct kqid qid) +{ + struct gfs2_quota_data *qd; + struct gfs2_holder q_gh; + int error; + + error = qd_get(sdp, qid, &qd); + if (error) + return error; + + error = do_glock(qd, FORCE, &q_gh); + if (!error) + gfs2_glock_dq_uninit(&q_gh); + + qd_put(qd); + return error; +} + +int gfs2_quota_init(struct gfs2_sbd *sdp) +{ + struct gfs2_inode *ip = GFS2_I(sdp->sd_qc_inode); + u64 size = i_size_read(sdp->sd_qc_inode); + unsigned int blocks = size >> sdp->sd_sb.sb_bsize_shift; + unsigned int x, slot = 0; + unsigned int found = 0; + unsigned int hash; + unsigned int bm_size; + struct buffer_head *bh; + u64 dblock; + u32 extlen = 0; + int error; + + if (gfs2_check_internal_file_size(sdp->sd_qc_inode, 1, 64 << 20)) + return -EIO; + + sdp->sd_quota_slots = blocks * sdp->sd_qc_per_block; + bm_size = DIV_ROUND_UP(sdp->sd_quota_slots, 8 * sizeof(unsigned long)); + bm_size *= sizeof(unsigned long); + error = -ENOMEM; + sdp->sd_quota_bitmap = kzalloc(bm_size, GFP_NOFS | __GFP_NOWARN); + if (sdp->sd_quota_bitmap == NULL) + sdp->sd_quota_bitmap = __vmalloc(bm_size, GFP_NOFS | + __GFP_ZERO); + if (!sdp->sd_quota_bitmap) + return error; + + for (x = 0; x < blocks; x++) { + struct gfs2_quota_change *qc; + unsigned int y; + + if (!extlen) { + extlen = 32; + error = gfs2_get_extent(&ip->i_inode, x, &dblock, &extlen); + if (error) + goto fail; + } + error = -EIO; + bh = gfs2_meta_ra(ip->i_gl, dblock, extlen); + if (!bh) + goto fail; + if (gfs2_metatype_check(sdp, bh, GFS2_METATYPE_QC)) + goto fail_brelse; + + qc = (struct gfs2_quota_change *)(bh->b_data + sizeof(struct gfs2_meta_header)); + for (y = 0; y < sdp->sd_qc_per_block && slot < sdp->sd_quota_slots; + y++, slot++, qc++) { + struct gfs2_quota_data *old_qd, *qd; + s64 qc_change = be64_to_cpu(qc->qc_change); + u32 qc_flags = be32_to_cpu(qc->qc_flags); + enum quota_type qtype = (qc_flags & GFS2_QCF_USER) ? + USRQUOTA : GRPQUOTA; + struct kqid qc_id = make_kqid(&init_user_ns, qtype, + be32_to_cpu(qc->qc_id)); + if (!qc_change) + continue; + + hash = gfs2_qd_hash(sdp, qc_id); + qd = qd_alloc(hash, sdp, qc_id); + if (qd == NULL) + goto fail_brelse; + + qd->qd_lockref.count = 0; + set_bit(QDF_CHANGE, &qd->qd_flags); + qd->qd_change = qc_change; + qd->qd_slot = slot; + qd->qd_slot_ref = 1; + + spin_lock(&qd_lock); + spin_lock_bucket(hash); + old_qd = gfs2_qd_search_bucket_noref(hash, sdp, qc_id); + if (old_qd) { + fs_err(sdp, "Corruption found in quota_change%u" + "file: duplicate identifier in " + "slot %u\n", + sdp->sd_jdesc->jd_jid, slot); + + spin_unlock_bucket(hash); + spin_unlock(&qd_lock); + + gfs2_glock_put(qd->qd_gl); + kmem_cache_free(gfs2_quotad_cachep, qd); + + /* zero out the duplicate slot */ + lock_buffer(bh); + memset(qc, 0, sizeof(*qc)); + mark_buffer_dirty(bh); + unlock_buffer(bh); + + continue; + } + BUG_ON(test_and_set_bit(slot, sdp->sd_quota_bitmap)); + list_add(&qd->qd_list, &sdp->sd_quota_list); + atomic_inc(&sdp->sd_quota_count); + hlist_bl_add_head_rcu(&qd->qd_hlist, &qd_hash_table[hash]); + spin_unlock_bucket(hash); + spin_unlock(&qd_lock); + + found++; + } + + if (buffer_dirty(bh)) + sync_dirty_buffer(bh); + brelse(bh); + dblock++; + extlen--; + } + + if (found) + fs_info(sdp, "found %u quota changes\n", found); + + return 0; + +fail_brelse: + if (buffer_dirty(bh)) + sync_dirty_buffer(bh); + brelse(bh); +fail: + gfs2_quota_cleanup(sdp); + return error; +} + +void gfs2_quota_cleanup(struct gfs2_sbd *sdp) +{ + struct gfs2_quota_data *qd; + LIST_HEAD(dispose); + int count; + + BUG_ON(!test_bit(SDF_NORECOVERY, &sdp->sd_flags) && + test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags)); + + spin_lock(&qd_lock); + list_for_each_entry(qd, &sdp->sd_quota_list, qd_list) { + spin_lock(&qd->qd_lockref.lock); + if (qd->qd_lockref.count != 0) { + spin_unlock(&qd->qd_lockref.lock); + continue; + } + lockref_mark_dead(&qd->qd_lockref); + spin_unlock(&qd->qd_lockref.lock); + + list_lru_del_obj(&gfs2_qd_lru, &qd->qd_lru); + list_add(&qd->qd_lru, &dispose); + } + spin_unlock(&qd_lock); + + gfs2_qd_list_dispose(&dispose); + + wait_event_timeout(sdp->sd_kill_wait, + (count = atomic_read(&sdp->sd_quota_count)) == 0, + HZ * 60); + + if (count != 0) + fs_err(sdp, "%d left-over quota data objects\n", count); + + kvfree(sdp->sd_quota_bitmap); + sdp->sd_quota_bitmap = NULL; +} + +static void quotad_error(struct gfs2_sbd *sdp, const char *msg, int error) +{ + if (error == 0 || error == -EROFS) + return; + if (!gfs2_withdrawn(sdp)) { + if (!cmpxchg(&sdp->sd_log_error, 0, error)) + fs_err(sdp, "gfs2_quotad: %s error %d\n", msg, error); + wake_up(&sdp->sd_logd_waitq); + } +} + +void gfs2_wake_up_statfs(struct gfs2_sbd *sdp) { + if (!sdp->sd_statfs_force_sync) { + sdp->sd_statfs_force_sync = 1; + wake_up(&sdp->sd_quota_wait); + } +} + + +/** + * gfs2_quotad - Write cached quota changes into the quota file + * @data: Pointer to GFS2 superblock + * + */ + +int gfs2_quotad(void *data) +{ + struct gfs2_sbd *sdp = data; + unsigned long now = jiffies; + unsigned long statfs_deadline = now; + unsigned long quotad_deadline = now; + + set_freezable(); + while (!kthread_should_stop()) { + unsigned long t; + + if (gfs2_withdrawn(sdp)) + break; + + now = jiffies; + if (sdp->sd_statfs_force_sync || + time_after(now, statfs_deadline)) { + unsigned int quantum; + int error; + + /* Update the master statfs file */ + error = gfs2_statfs_sync(sdp->sd_vfs, 0); + quotad_error(sdp, "statfs", error); + + quantum = gfs2_tune_get(sdp, gt_statfs_quantum); + statfs_deadline = now + quantum * HZ; + } + if (time_after(now, quotad_deadline)) { + unsigned int quantum; + int error; + + /* Update the quota file */ + error = gfs2_quota_sync(sdp->sd_vfs, 0); + quotad_error(sdp, "sync", error); + + quantum = gfs2_tune_get(sdp, gt_quota_quantum); + quotad_deadline = now + quantum * HZ; + } + + t = min(statfs_deadline - now, quotad_deadline - now); + wait_event_freezable_timeout(sdp->sd_quota_wait, + sdp->sd_statfs_force_sync || + gfs2_withdrawn(sdp) || + kthread_should_stop(), + t); + + if (sdp->sd_statfs_force_sync) + t = 0; + } + + return 0; +} + +static int gfs2_quota_get_state(struct super_block *sb, struct qc_state *state) +{ + struct gfs2_sbd *sdp = sb->s_fs_info; + + memset(state, 0, sizeof(*state)); + + switch (sdp->sd_args.ar_quota) { + case GFS2_QUOTA_QUIET: + fallthrough; + case GFS2_QUOTA_ON: + state->s_state[USRQUOTA].flags |= QCI_LIMITS_ENFORCED; + state->s_state[GRPQUOTA].flags |= QCI_LIMITS_ENFORCED; + fallthrough; + case GFS2_QUOTA_ACCOUNT: + state->s_state[USRQUOTA].flags |= QCI_ACCT_ENABLED | + QCI_SYSFILE; + state->s_state[GRPQUOTA].flags |= QCI_ACCT_ENABLED | + QCI_SYSFILE; + break; + case GFS2_QUOTA_OFF: + break; + } + if (sdp->sd_quota_inode) { + state->s_state[USRQUOTA].ino = + GFS2_I(sdp->sd_quota_inode)->i_no_addr; + state->s_state[USRQUOTA].blocks = sdp->sd_quota_inode->i_blocks; + } + state->s_state[USRQUOTA].nextents = 1; /* unsupported */ + state->s_state[GRPQUOTA] = state->s_state[USRQUOTA]; + state->s_incoredqs = list_lru_count(&gfs2_qd_lru); + return 0; +} + +static int gfs2_get_dqblk(struct super_block *sb, struct kqid qid, + struct qc_dqblk *fdq) +{ + struct gfs2_sbd *sdp = sb->s_fs_info; + struct gfs2_quota_lvb *qlvb; + struct gfs2_quota_data *qd; + struct gfs2_holder q_gh; + int error; + + memset(fdq, 0, sizeof(*fdq)); + + if (sdp->sd_args.ar_quota == GFS2_QUOTA_OFF) + return -ESRCH; /* Crazy XFS error code */ + + if ((qid.type != USRQUOTA) && + (qid.type != GRPQUOTA)) + return -EINVAL; + + error = qd_get(sdp, qid, &qd); + if (error) + return error; + error = do_glock(qd, FORCE, &q_gh); + if (error) + goto out; + + qlvb = (struct gfs2_quota_lvb *)qd->qd_gl->gl_lksb.sb_lvbptr; + fdq->d_spc_hardlimit = be64_to_cpu(qlvb->qb_limit) << sdp->sd_sb.sb_bsize_shift; + fdq->d_spc_softlimit = be64_to_cpu(qlvb->qb_warn) << sdp->sd_sb.sb_bsize_shift; + fdq->d_space = be64_to_cpu(qlvb->qb_value) << sdp->sd_sb.sb_bsize_shift; + + gfs2_glock_dq_uninit(&q_gh); +out: + qd_put(qd); + return error; +} + +/* GFS2 only supports a subset of the XFS fields */ +#define GFS2_FIELDMASK (QC_SPC_SOFT|QC_SPC_HARD|QC_SPACE) + +static int gfs2_set_dqblk(struct super_block *sb, struct kqid qid, + struct qc_dqblk *fdq) +{ + struct gfs2_sbd *sdp = sb->s_fs_info; + struct gfs2_inode *ip = GFS2_I(sdp->sd_quota_inode); + struct gfs2_quota_data *qd; + struct gfs2_holder q_gh, i_gh; + unsigned int data_blocks, ind_blocks; + unsigned int blocks = 0; + int alloc_required; + loff_t offset; + int error; + + if (sdp->sd_args.ar_quota == GFS2_QUOTA_OFF) + return -ESRCH; /* Crazy XFS error code */ + + if ((qid.type != USRQUOTA) && + (qid.type != GRPQUOTA)) + return -EINVAL; + + if (fdq->d_fieldmask & ~GFS2_FIELDMASK) + return -EINVAL; + + error = qd_get(sdp, qid, &qd); + if (error) + return error; + + error = gfs2_qa_get(ip); + if (error) + goto out_put; + + inode_lock(&ip->i_inode); + error = gfs2_glock_nq_init(qd->qd_gl, LM_ST_EXCLUSIVE, 0, &q_gh); + if (error) + goto out_unlockput; + error = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &i_gh); + if (error) + goto out_q; + + /* Check for existing entry, if none then alloc new blocks */ + error = update_qd(sdp, qd); + if (error) + goto out_i; + + /* If nothing has changed, this is a no-op */ + if ((fdq->d_fieldmask & QC_SPC_SOFT) && + ((fdq->d_spc_softlimit >> sdp->sd_sb.sb_bsize_shift) == be64_to_cpu(qd->qd_qb.qb_warn))) + fdq->d_fieldmask ^= QC_SPC_SOFT; + + if ((fdq->d_fieldmask & QC_SPC_HARD) && + ((fdq->d_spc_hardlimit >> sdp->sd_sb.sb_bsize_shift) == be64_to_cpu(qd->qd_qb.qb_limit))) + fdq->d_fieldmask ^= QC_SPC_HARD; + + if ((fdq->d_fieldmask & QC_SPACE) && + ((fdq->d_space >> sdp->sd_sb.sb_bsize_shift) == be64_to_cpu(qd->qd_qb.qb_value))) + fdq->d_fieldmask ^= QC_SPACE; + + if (fdq->d_fieldmask == 0) + goto out_i; + + offset = qd2offset(qd); + alloc_required = gfs2_write_alloc_required(ip, offset, sizeof(struct gfs2_quota)); + if (gfs2_is_stuffed(ip)) + alloc_required = 1; + if (alloc_required) { + struct gfs2_alloc_parms ap = {}; + gfs2_write_calc_reserv(ip, sizeof(struct gfs2_quota), + &data_blocks, &ind_blocks); + blocks = 1 + data_blocks + ind_blocks; + ap.target = blocks; + error = gfs2_inplace_reserve(ip, &ap); + if (error) + goto out_i; + blocks += gfs2_rg_blocks(ip, blocks); + } + + /* Some quotas span block boundaries and can update two blocks, + adding an extra block to the transaction to handle such quotas */ + error = gfs2_trans_begin(sdp, blocks + RES_DINODE + 2, 0); + if (error) + goto out_release; + + /* Apply changes */ + error = gfs2_adjust_quota(sdp, offset, 0, qd, fdq); + if (!error) + clear_bit(QDF_QMSG_QUIET, &qd->qd_flags); + + gfs2_trans_end(sdp); +out_release: + if (alloc_required) + gfs2_inplace_release(ip); +out_i: + gfs2_glock_dq_uninit(&i_gh); +out_q: + gfs2_glock_dq_uninit(&q_gh); +out_unlockput: + gfs2_qa_put(ip); + inode_unlock(&ip->i_inode); +out_put: + qd_put(qd); + return error; +} + +const struct quotactl_ops gfs2_quotactl_ops = { + .quota_sync = gfs2_quota_sync, + .get_state = gfs2_quota_get_state, + .get_dqblk = gfs2_get_dqblk, + .set_dqblk = gfs2_set_dqblk, +}; + +void __init gfs2_quota_hash_init(void) +{ + unsigned i; + + for(i = 0; i < GFS2_QD_HASH_SIZE; i++) + INIT_HLIST_BL_HEAD(&qd_hash_table[i]); +} diff --git a/fs/gfs2/quota.h b/fs/gfs2/quota.h new file mode 100644 index 000000000..988f38dc5 --- /dev/null +++ b/fs/gfs2/quota.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#ifndef __QUOTA_DOT_H__ +#define __QUOTA_DOT_H__ + +#include + +struct gfs2_inode; +struct gfs2_sbd; + +#define NO_UID_QUOTA_CHANGE INVALID_UID +#define NO_GID_QUOTA_CHANGE INVALID_GID + +int gfs2_qa_get(struct gfs2_inode *ip); +void gfs2_qa_put(struct gfs2_inode *ip); +int gfs2_quota_hold(struct gfs2_inode *ip, kuid_t uid, kgid_t gid); +void gfs2_quota_unhold(struct gfs2_inode *ip); + +int gfs2_quota_lock(struct gfs2_inode *ip, kuid_t uid, kgid_t gid); +void gfs2_quota_unlock(struct gfs2_inode *ip); + +int gfs2_quota_check(struct gfs2_inode *ip, kuid_t uid, kgid_t gid, + struct gfs2_alloc_parms *ap); +void gfs2_quota_change(struct gfs2_inode *ip, s64 change, + kuid_t uid, kgid_t gid); + +int gfs2_quota_sync(struct super_block *sb, int type); +int gfs2_quota_refresh(struct gfs2_sbd *sdp, struct kqid qid); + +int gfs2_quota_init(struct gfs2_sbd *sdp); +void gfs2_quota_cleanup(struct gfs2_sbd *sdp); +int gfs2_quotad(void *data); + +void gfs2_wake_up_statfs(struct gfs2_sbd *sdp); + +static inline int gfs2_quota_lock_check(struct gfs2_inode *ip, + struct gfs2_alloc_parms *ap) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + int ret; + + ap->allowed = UINT_MAX; /* Assume we are permitted a whole lot */ + if (sdp->sd_args.ar_quota == GFS2_QUOTA_OFF || + capable(CAP_SYS_RESOURCE)) + return 0; + ret = gfs2_quota_lock(ip, NO_UID_QUOTA_CHANGE, NO_GID_QUOTA_CHANGE); + if (ret) + return ret; + if (sdp->sd_args.ar_quota == GFS2_QUOTA_ACCOUNT) + return 0; + ret = gfs2_quota_check(ip, ip->i_inode.i_uid, ip->i_inode.i_gid, ap); + if (ret) + gfs2_quota_unlock(ip); + return ret; +} + +extern const struct quotactl_ops gfs2_quotactl_ops; +int __init gfs2_qd_shrinker_init(void); +void gfs2_qd_shrinker_exit(void); +extern struct list_lru gfs2_qd_lru; + +void __init gfs2_quota_hash_init(void); + +#endif /* __QUOTA_DOT_H__ */ diff --git a/fs/gfs2/recovery.c b/fs/gfs2/recovery.c new file mode 100644 index 000000000..616c46aa3 --- /dev/null +++ b/fs/gfs2/recovery.c @@ -0,0 +1,594 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gfs2.h" +#include "incore.h" +#include "bmap.h" +#include "glock.h" +#include "glops.h" +#include "log.h" +#include "lops.h" +#include "meta_io.h" +#include "recovery.h" +#include "super.h" +#include "util.h" +#include "dir.h" + +struct workqueue_struct *gfs2_recovery_wq; + +int gfs2_replay_read_block(struct gfs2_jdesc *jd, unsigned int blk, + struct buffer_head **bh) +{ + struct gfs2_inode *ip = GFS2_I(jd->jd_inode); + struct gfs2_glock *gl = ip->i_gl; + u64 dblock; + u32 extlen; + int error; + + extlen = 32; + error = gfs2_get_extent(&ip->i_inode, blk, &dblock, &extlen); + if (error) + return error; + if (!dblock) { + gfs2_consist_inode(ip); + return -EIO; + } + + *bh = gfs2_meta_ra(gl, dblock, extlen); + + return error; +} + +int gfs2_revoke_add(struct gfs2_jdesc *jd, u64 blkno, unsigned int where) +{ + struct list_head *head = &jd->jd_revoke_list; + struct gfs2_revoke_replay *rr = NULL, *iter; + + list_for_each_entry(iter, head, rr_list) { + if (iter->rr_blkno == blkno) { + rr = iter; + break; + } + } + + if (rr) { + rr->rr_where = where; + return 0; + } + + rr = kmalloc_obj(struct gfs2_revoke_replay, GFP_NOFS); + if (!rr) + return -ENOMEM; + + rr->rr_blkno = blkno; + rr->rr_where = where; + list_add(&rr->rr_list, head); + + return 1; +} + +int gfs2_revoke_check(struct gfs2_jdesc *jd, u64 blkno, unsigned int where) +{ + struct gfs2_revoke_replay *rr = NULL, *iter; + int wrap, a, b, revoke; + + list_for_each_entry(iter, &jd->jd_revoke_list, rr_list) { + if (iter->rr_blkno == blkno) { + rr = iter; + break; + } + } + + if (!rr) + return 0; + + wrap = (rr->rr_where < jd->jd_replay_tail); + a = (jd->jd_replay_tail < where); + b = (where < rr->rr_where); + revoke = (wrap) ? (a || b) : (a && b); + + return revoke; +} + +void gfs2_revoke_clean(struct gfs2_jdesc *jd) +{ + struct list_head *head = &jd->jd_revoke_list; + struct gfs2_revoke_replay *rr; + + while (!list_empty(head)) { + rr = list_first_entry(head, struct gfs2_revoke_replay, rr_list); + list_del(&rr->rr_list); + kfree(rr); + } +} + +int __get_log_header(struct gfs2_sbd *sdp, const struct gfs2_log_header *lh, + unsigned int blkno, struct gfs2_log_header_host *head) +{ + const u32 zero = 0; + u32 hash, crc; + + if (lh->lh_header.mh_magic != cpu_to_be32(GFS2_MAGIC) || + lh->lh_header.mh_type != cpu_to_be32(GFS2_METATYPE_LH) || + (blkno && be32_to_cpu(lh->lh_blkno) != blkno)) + return 1; + + hash = crc32(~0, lh, LH_V1_SIZE - 4); + hash = ~crc32(hash, &zero, 4); /* assume lh_hash is zero */ + + if (be32_to_cpu(lh->lh_hash) != hash) + return 1; + + crc = crc32c(~0, (void *)lh + LH_V1_SIZE + 4, + sdp->sd_sb.sb_bsize - LH_V1_SIZE - 4); + + if ((lh->lh_crc != 0 && be32_to_cpu(lh->lh_crc) != crc)) + return 1; + + head->lh_sequence = be64_to_cpu(lh->lh_sequence); + head->lh_flags = be32_to_cpu(lh->lh_flags); + head->lh_tail = be32_to_cpu(lh->lh_tail); + head->lh_blkno = be32_to_cpu(lh->lh_blkno); + + head->lh_local_total = be64_to_cpu(lh->lh_local_total); + head->lh_local_free = be64_to_cpu(lh->lh_local_free); + head->lh_local_dinodes = be64_to_cpu(lh->lh_local_dinodes); + + return 0; +} +/** + * get_log_header - read the log header for a given segment + * @jd: the journal + * @blk: the block to look at + * @head: the log header to return + * + * Read the log header for a given segement in a given journal. Do a few + * sanity checks on it. + * + * Returns: 0 on success, + * 1 if the header was invalid or incomplete, + * errno on error + */ + +static int get_log_header(struct gfs2_jdesc *jd, unsigned int blk, + struct gfs2_log_header_host *head) +{ + struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode); + struct buffer_head *bh; + int error; + + error = gfs2_replay_read_block(jd, blk, &bh); + if (error) + return error; + + error = __get_log_header(sdp, (const struct gfs2_log_header *)bh->b_data, + blk, head); + brelse(bh); + + return error; +} + +/** + * foreach_descriptor - go through the active part of the log + * @jd: the journal + * @start: the first log header in the active region + * @end: the last log header (don't process the contents of this entry)) + * @pass: iteration number (foreach_descriptor() is called in a for() loop) + * + * Call a given function once for every log descriptor in the active + * portion of the log. + * + * Returns: errno + */ + +static int foreach_descriptor(struct gfs2_jdesc *jd, u32 start, + unsigned int end, int pass) +{ + struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode); + struct buffer_head *bh; + struct gfs2_log_descriptor *ld; + int error = 0; + u32 length; + __be64 *ptr; + unsigned int offset = sizeof(struct gfs2_log_descriptor); + offset += sizeof(__be64) - 1; + offset &= ~(sizeof(__be64) - 1); + + while (start != end) { + error = gfs2_replay_read_block(jd, start, &bh); + if (error) + return error; + if (gfs2_meta_check(sdp, bh)) { + brelse(bh); + return -EIO; + } + ld = (struct gfs2_log_descriptor *)bh->b_data; + length = be32_to_cpu(ld->ld_length); + + if (be32_to_cpu(ld->ld_header.mh_type) == GFS2_METATYPE_LH) { + struct gfs2_log_header_host lh; + error = get_log_header(jd, start, &lh); + if (!error) { + gfs2_replay_incr_blk(jd, &start); + brelse(bh); + continue; + } + if (error == 1) { + gfs2_consist_inode(GFS2_I(jd->jd_inode)); + error = -EIO; + } + brelse(bh); + return error; + } else if (gfs2_metatype_check(sdp, bh, GFS2_METATYPE_LD)) { + brelse(bh); + return -EIO; + } + ptr = (__be64 *)(bh->b_data + offset); + error = lops_scan_elements(jd, start, ld, ptr, pass); + if (error) { + brelse(bh); + return error; + } + + while (length--) + gfs2_replay_incr_blk(jd, &start); + + brelse(bh); + } + + return 0; +} + +/** + * clean_journal - mark a dirty journal as being clean + * @jd: the journal + * @head: the head journal to start from + * + * Returns: errno + */ + +static void clean_journal(struct gfs2_jdesc *jd, + struct gfs2_log_header_host *head) +{ + struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode); + + gfs2_replay_incr_blk(jd, &head->lh_blkno); + head->lh_sequence++; + gfs2_write_log_header(sdp, jd, head->lh_sequence, 0, head->lh_blkno, + GFS2_LOG_HEAD_UNMOUNT | GFS2_LOG_HEAD_RECOVERY, + REQ_PREFLUSH | REQ_FUA | REQ_META | REQ_SYNC); +} + + +static void gfs2_recovery_done(struct gfs2_sbd *sdp, unsigned int jid, + unsigned int message) +{ + char env_jid[20]; + char env_status[20]; + char *envp[] = { env_jid, env_status, NULL }; + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + + ls->ls_recover_jid_done = jid; + ls->ls_recover_jid_status = message; + sprintf(env_jid, "JID=%u", jid); + sprintf(env_status, "RECOVERY=%s", + message == LM_RD_SUCCESS ? "Done" : "Failed"); + kobject_uevent_env(&sdp->sd_kobj, KOBJ_CHANGE, envp); + + if (sdp->sd_lockstruct.ls_ops->lm_recovery_result) + sdp->sd_lockstruct.ls_ops->lm_recovery_result(sdp, jid, message); +} + +/** + * update_statfs_inode - Update the master statfs inode or zero out the local + * statfs inode for a given journal. + * @jd: The journal + * @head: If NULL, @inode is the local statfs inode and we need to zero it out. + * Otherwise, it @head contains the statfs change info that needs to be + * synced to the master statfs inode (pointed to by @inode). + * @inode: statfs inode to update. + */ +static int update_statfs_inode(struct gfs2_jdesc *jd, + struct gfs2_log_header_host *head, + struct inode *inode) +{ + struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode); + struct gfs2_inode *ip; + struct buffer_head *bh; + struct gfs2_statfs_change_host sc; + int error = 0; + + BUG_ON(!inode); + ip = GFS2_I(inode); + + error = gfs2_meta_inode_buffer(ip, &bh); + if (error) + goto out; + + spin_lock(&sdp->sd_statfs_spin); + + if (head) { /* Update the master statfs inode */ + gfs2_statfs_change_in(&sc, bh->b_data + sizeof(struct gfs2_dinode)); + sc.sc_total += head->lh_local_total; + sc.sc_free += head->lh_local_free; + sc.sc_dinodes += head->lh_local_dinodes; + gfs2_statfs_change_out(&sc, bh->b_data + sizeof(struct gfs2_dinode)); + + fs_info(sdp, "jid=%u: Updated master statfs Total:%lld, " + "Free:%lld, Dinodes:%lld after change " + "[%+lld,%+lld,%+lld]\n", jd->jd_jid, sc.sc_total, + sc.sc_free, sc.sc_dinodes, head->lh_local_total, + head->lh_local_free, head->lh_local_dinodes); + } else { /* Zero out the local statfs inode */ + memset(bh->b_data + sizeof(struct gfs2_dinode), 0, + sizeof(struct gfs2_statfs_change)); + /* If it's our own journal, reset any in-memory changes too */ + if (jd->jd_jid == sdp->sd_lockstruct.ls_jid) { + memset(&sdp->sd_statfs_local, 0, + sizeof(struct gfs2_statfs_change_host)); + } + } + spin_unlock(&sdp->sd_statfs_spin); + + mark_buffer_dirty(bh); + brelse(bh); + gfs2_inode_metasync(ip->i_gl); + +out: + return error; +} + +/** + * recover_local_statfs - Update the master and local statfs changes for this + * journal. + * + * Previously, statfs updates would be read in from the local statfs inode and + * synced to the master statfs inode during recovery. + * + * We now use the statfs updates in the journal head to update the master statfs + * inode instead of reading in from the local statfs inode. To preserve backward + * compatibility with kernels that can't do this, we still need to keep the + * local statfs inode up to date by writing changes to it. At some point in the + * future, we can do away with the local statfs inodes altogether and keep the + * statfs changes solely in the journal. + * + * @jd: the journal + * @head: the journal head + * + * Returns: errno + */ +static void recover_local_statfs(struct gfs2_jdesc *jd, + struct gfs2_log_header_host *head) +{ + int error; + struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode); + + if (!head->lh_local_total && !head->lh_local_free + && !head->lh_local_dinodes) /* No change */ + goto zero_local; + + /* First update the master statfs inode with the changes we + * found in the journal. */ + error = update_statfs_inode(jd, head, sdp->sd_statfs_inode); + if (error) + goto out; + +zero_local: + /* Zero out the local statfs inode so any changes in there + * are not re-recovered. */ + error = update_statfs_inode(jd, NULL, + find_local_statfs_inode(sdp, jd->jd_jid)); +out: + return; +} + +void gfs2_recover_func(struct work_struct *work) +{ + struct gfs2_jdesc *jd = container_of(work, struct gfs2_jdesc, jd_work); + struct gfs2_inode *ip = GFS2_I(jd->jd_inode); + struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode); + struct gfs2_log_header_host head; + struct gfs2_holder j_gh, ji_gh; + ktime_t t_start, t_jlck, t_jhd, t_tlck, t_rep; + int ro = 0; + unsigned int pass; + int error = 0; + int jlocked = 0; + + if (gfs2_withdrawn(sdp)) { + fs_err(sdp, "jid=%u: Recovery not attempted due to withdraw.\n", + jd->jd_jid); + goto fail; + } + t_start = ktime_get(); + if (sdp->sd_args.ar_spectator) + goto fail; + if (jd->jd_jid != sdp->sd_lockstruct.ls_jid) { + fs_info(sdp, "jid=%u: Trying to acquire journal glock...\n", + jd->jd_jid); + jlocked = 1; + /* Acquire the journal glock so we can do recovery */ + + error = gfs2_glock_nq_num(sdp, jd->jd_jid, &gfs2_journal_glops, + LM_ST_EXCLUSIVE, + LM_FLAG_RECOVER | LM_FLAG_TRY | + GL_NOCACHE, + &j_gh); + switch (error) { + case 0: + break; + + case GLR_TRYFAILED: + fs_info(sdp, "jid=%u: Busy\n", jd->jd_jid); + error = 0; + goto fail; + + default: + goto fail; + } + + error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, + LM_FLAG_RECOVER | GL_NOCACHE, + &ji_gh); + if (error) + goto fail_gunlock_j; + } else { + fs_info(sdp, "jid=%u, already locked for use\n", jd->jd_jid); + } + + t_jlck = ktime_get(); + fs_info(sdp, "jid=%u: Looking at journal...\n", jd->jd_jid); + + error = gfs2_jdesc_check(jd); + if (error) + goto fail_gunlock_ji; + + error = gfs2_find_jhead(jd, &head); + if (error) + goto fail_gunlock_ji; + t_jhd = ktime_get(); + fs_info(sdp, "jid=%u: Journal head lookup took %lldms\n", jd->jd_jid, + ktime_ms_delta(t_jhd, t_jlck)); + + if (!(head.lh_flags & GFS2_LOG_HEAD_UNMOUNT)) { + mutex_lock(&sdp->sd_freeze_mutex); + + if (test_bit(SDF_FROZEN, &sdp->sd_flags)) { + mutex_unlock(&sdp->sd_freeze_mutex); + fs_warn(sdp, "jid=%u: Can't replay: filesystem " + "is frozen\n", jd->jd_jid); + goto fail_gunlock_ji; + } + + if (test_bit(SDF_RORECOVERY, &sdp->sd_flags)) { + ro = 1; + } else if (test_bit(SDF_JOURNAL_CHECKED, &sdp->sd_flags)) { + if (!test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags)) + ro = 1; + } else { + if (sb_rdonly(sdp->sd_vfs)) { + /* check if device itself is read-only */ + ro = bdev_read_only(sdp->sd_vfs->s_bdev); + if (!ro) { + fs_info(sdp, "recovery required on " + "read-only filesystem.\n"); + fs_info(sdp, "write access will be " + "enabled during recovery.\n"); + } + } + } + + if (ro) { + fs_warn(sdp, "jid=%u: Can't replay: read-only block " + "device\n", jd->jd_jid); + error = -EROFS; + goto fail_gunlock_nofreeze; + } + + t_tlck = ktime_get(); + fs_info(sdp, "jid=%u: Replaying journal...0x%x to 0x%x\n", + jd->jd_jid, head.lh_tail, head.lh_blkno); + + /* We take the sd_log_flush_lock here primarily to prevent log + * flushes and simultaneous journal replays from stomping on + * each other wrt jd_log_bio. */ + down_read(&sdp->sd_log_flush_lock); + for (pass = 0; pass < 2; pass++) { + lops_before_scan(jd, &head, pass); + error = foreach_descriptor(jd, head.lh_tail, + head.lh_blkno, pass); + lops_after_scan(jd, error, pass); + if (error) { + up_read(&sdp->sd_log_flush_lock); + goto fail_gunlock_nofreeze; + } + } + + recover_local_statfs(jd, &head); + clean_journal(jd, &head); + up_read(&sdp->sd_log_flush_lock); + + mutex_unlock(&sdp->sd_freeze_mutex); + t_rep = ktime_get(); + fs_info(sdp, "jid=%u: Journal replayed in %lldms [jlck:%lldms, " + "jhead:%lldms, tlck:%lldms, replay:%lldms]\n", + jd->jd_jid, ktime_ms_delta(t_rep, t_start), + ktime_ms_delta(t_jlck, t_start), + ktime_ms_delta(t_jhd, t_jlck), + ktime_ms_delta(t_tlck, t_jhd), + ktime_ms_delta(t_rep, t_tlck)); + } + + if (jd->jd_jid == sdp->sd_lockstruct.ls_jid) + gfs2_log_pointers_init(sdp, &head); + + gfs2_recovery_done(sdp, jd->jd_jid, LM_RD_SUCCESS); + + if (jlocked) { + gfs2_glock_dq_uninit(&ji_gh); + gfs2_glock_dq_uninit(&j_gh); + } + + fs_info(sdp, "jid=%u: Done\n", jd->jd_jid); + goto done; + +fail_gunlock_nofreeze: + mutex_unlock(&sdp->sd_freeze_mutex); +fail_gunlock_ji: + if (jlocked) { + gfs2_glock_dq_uninit(&ji_gh); +fail_gunlock_j: + gfs2_glock_dq_uninit(&j_gh); + } + + fs_info(sdp, "jid=%u: %s\n", jd->jd_jid, (error) ? "Failed" : "Done"); +fail: + jd->jd_recover_error = error; + gfs2_recovery_done(sdp, jd->jd_jid, LM_RD_GAVEUP); +done: + clear_bit(JDF_RECOVERY, &jd->jd_flags); + smp_mb__after_atomic(); + wake_up_bit(&jd->jd_flags, JDF_RECOVERY); +} + +int gfs2_recover_journal(struct gfs2_jdesc *jd, bool wait) +{ + int rv; + + if (test_and_set_bit(JDF_RECOVERY, &jd->jd_flags)) + return -EBUSY; + + /* we have JDF_RECOVERY, queue should always succeed */ + rv = queue_work(gfs2_recovery_wq, &jd->jd_work); + BUG_ON(!rv); + + if (wait) + wait_on_bit(&jd->jd_flags, JDF_RECOVERY, + TASK_UNINTERRUPTIBLE); + + return wait ? jd->jd_recover_error : 0; +} + +void gfs2_log_pointers_init(struct gfs2_sbd *sdp, + struct gfs2_log_header_host *head) +{ + sdp->sd_log_sequence = head->lh_sequence + 1; + gfs2_replay_incr_blk(sdp->sd_jdesc, &head->lh_blkno); + sdp->sd_log_tail = head->lh_blkno; + sdp->sd_log_flush_head = head->lh_blkno; + sdp->sd_log_flush_tail = head->lh_blkno; + sdp->sd_log_head = head->lh_blkno; +} diff --git a/fs/gfs2/recovery.h b/fs/gfs2/recovery.h new file mode 100644 index 000000000..5a5ba72ec --- /dev/null +++ b/fs/gfs2/recovery.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#ifndef __RECOVERY_DOT_H__ +#define __RECOVERY_DOT_H__ + +#include "incore.h" + +extern struct workqueue_struct *gfs2_recovery_wq; + +static inline void gfs2_replay_incr_blk(struct gfs2_jdesc *jd, u32 *blk) +{ + if (++*blk == jd->jd_blocks) + *blk = 0; +} + +int gfs2_replay_read_block(struct gfs2_jdesc *jd, unsigned int blk, + struct buffer_head **bh); + +int gfs2_revoke_add(struct gfs2_jdesc *jd, u64 blkno, unsigned int where); +int gfs2_revoke_check(struct gfs2_jdesc *jd, u64 blkno, unsigned int where); +void gfs2_revoke_clean(struct gfs2_jdesc *jd); + +int gfs2_recover_journal(struct gfs2_jdesc *gfs2_jd, bool wait); +void gfs2_recover_func(struct work_struct *work); +int __get_log_header(struct gfs2_sbd *sdp, + const struct gfs2_log_header *lh, unsigned int blkno, + struct gfs2_log_header_host *head); +void gfs2_log_pointers_init(struct gfs2_sbd *sdp, + struct gfs2_log_header_host *head); + +#endif /* __RECOVERY_DOT_H__ */ + diff --git a/fs/gfs2/rgrp.c b/fs/gfs2/rgrp.c new file mode 100644 index 000000000..5988a165a --- /dev/null +++ b/fs/gfs2/rgrp.c @@ -0,0 +1,2769 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gfs2.h" +#include "incore.h" +#include "glock.h" +#include "glops.h" +#include "lops.h" +#include "meta_io.h" +#include "quota.h" +#include "rgrp.h" +#include "super.h" +#include "trans.h" +#include "util.h" +#include "log.h" +#include "inode.h" +#include "trace_gfs2.h" +#include "dir.h" + +#define BFITNOENT ((u32)~0) +#define NO_BLOCK ((u64)~0) + +struct gfs2_rbm { + struct gfs2_rgrpd *rgd; + u32 offset; /* The offset is bitmap relative */ + int bii; /* Bitmap index */ +}; + +static inline struct gfs2_bitmap *rbm_bi(const struct gfs2_rbm *rbm) +{ + return rbm->rgd->rd_bits + rbm->bii; +} + +static inline u64 gfs2_rbm_to_block(const struct gfs2_rbm *rbm) +{ + BUG_ON(rbm->offset >= rbm->rgd->rd_data); + return rbm->rgd->rd_data0 + (rbm_bi(rbm)->bi_start * GFS2_NBBY) + + rbm->offset; +} + +/* + * These routines are used by the resource group routines (rgrp.c) + * to keep track of block allocation. Each block is represented by two + * bits. So, each byte represents GFS2_NBBY (i.e. 4) blocks. + * + * 0 = Free + * 1 = Used (not metadata) + * 2 = Unlinked (still in use) inode + * 3 = Used (metadata) + */ + +struct gfs2_extent { + struct gfs2_rbm rbm; + u32 len; +}; + +static const char valid_change[16] = { + /* current */ + /* n */ 0, 1, 1, 1, + /* e */ 1, 0, 0, 0, + /* w */ 0, 0, 0, 1, + 1, 0, 0, 0 +}; + +static int gfs2_rbm_find(struct gfs2_rbm *rbm, u8 state, u32 *minext, + struct gfs2_blkreserv *rs, bool nowrap); + + +/** + * gfs2_setbit - Set a bit in the bitmaps + * @rbm: The position of the bit to set + * @do_clone: Also set the clone bitmap, if it exists + * @new_state: the new state of the block + * + */ + +static inline void gfs2_setbit(const struct gfs2_rbm *rbm, bool do_clone, + unsigned char new_state) +{ + unsigned char *byte1, *byte2, *end, cur_state; + struct gfs2_bitmap *bi = rbm_bi(rbm); + unsigned int buflen = bi->bi_bytes; + const unsigned int bit = (rbm->offset % GFS2_NBBY) * GFS2_BIT_SIZE; + + byte1 = bi->bi_bh->b_data + bi->bi_offset + (rbm->offset / GFS2_NBBY); + end = bi->bi_bh->b_data + bi->bi_offset + buflen; + + BUG_ON(byte1 >= end); + + cur_state = (*byte1 >> bit) & GFS2_BIT_MASK; + + if (unlikely(!valid_change[new_state * 4 + cur_state])) { + struct gfs2_sbd *sdp = rbm->rgd->rd_sbd; + + fs_warn(sdp, "buf_blk = 0x%x old_state=%d, new_state=%d\n", + rbm->offset, cur_state, new_state); + fs_warn(sdp, "rgrp=0x%llx bi_start=0x%x biblk: 0x%llx\n", + (unsigned long long)rbm->rgd->rd_addr, bi->bi_start, + (unsigned long long)bi->bi_bh->b_blocknr); + fs_warn(sdp, "bi_offset=0x%x bi_bytes=0x%x block=0x%llx\n", + bi->bi_offset, bi->bi_bytes, + (unsigned long long)gfs2_rbm_to_block(rbm)); + dump_stack(); + gfs2_consist_rgrpd(rbm->rgd); + return; + } + *byte1 ^= (cur_state ^ new_state) << bit; + + if (do_clone && bi->bi_clone) { + byte2 = bi->bi_clone + bi->bi_offset + (rbm->offset / GFS2_NBBY); + cur_state = (*byte2 >> bit) & GFS2_BIT_MASK; + *byte2 ^= (cur_state ^ new_state) << bit; + } +} + +/** + * gfs2_testbit - test a bit in the bitmaps + * @rbm: The bit to test + * @use_clone: If true, test the clone bitmap, not the official bitmap. + * + * Some callers like gfs2_unaligned_extlen need to test the clone bitmaps, + * not the "real" bitmaps, to avoid allocating recently freed blocks. + * + * Returns: The two bit block state of the requested bit + */ + +static inline u8 gfs2_testbit(const struct gfs2_rbm *rbm, bool use_clone) +{ + struct gfs2_bitmap *bi = rbm_bi(rbm); + const u8 *buffer; + const u8 *byte; + unsigned int bit; + + if (use_clone && bi->bi_clone) + buffer = bi->bi_clone; + else + buffer = bi->bi_bh->b_data; + buffer += bi->bi_offset; + byte = buffer + (rbm->offset / GFS2_NBBY); + bit = (rbm->offset % GFS2_NBBY) * GFS2_BIT_SIZE; + + return (*byte >> bit) & GFS2_BIT_MASK; +} + +/** + * gfs2_bit_search - search bitmap for a state + * @ptr: Pointer to bitmap data + * @mask: Mask to use (normally 0x55555.... but adjusted for search start) + * @state: The state we are searching for + * + * We xor the bitmap data with a pattern which is the bitwise opposite + * of what we are looking for. This gives rise to a pattern of ones + * wherever there is a match. Since we have two bits per entry, we + * take this pattern, shift it down by one place and then and it with + * the original. All the even bit positions (0,2,4, etc) then represent + * successful matches, so we mask with 0x55555..... to remove the unwanted + * odd bit positions. + * + * This allows searching of a whole u64 at once (32 blocks) with a + * single test (on 64 bit arches). + */ + +static inline u64 gfs2_bit_search(const __le64 *ptr, u64 mask, u8 state) +{ + u64 tmp; + static const u64 search[] = { + [0] = 0xffffffffffffffffULL, + [1] = 0xaaaaaaaaaaaaaaaaULL, + [2] = 0x5555555555555555ULL, + [3] = 0x0000000000000000ULL, + }; + tmp = le64_to_cpu(*ptr) ^ search[state]; + tmp &= (tmp >> 1); + tmp &= mask; + return tmp; +} + +/** + * rs_cmp - multi-block reservation range compare + * @start: start of the new reservation + * @len: number of blocks in the new reservation + * @rs: existing reservation to compare against + * + * returns: 1 if the block range is beyond the reach of the reservation + * -1 if the block range is before the start of the reservation + * 0 if the block range overlaps with the reservation + */ +static inline int rs_cmp(u64 start, u32 len, struct gfs2_blkreserv *rs) +{ + if (start >= rs->rs_start + rs->rs_requested) + return 1; + if (rs->rs_start >= start + len) + return -1; + return 0; +} + +/** + * gfs2_bitfit - Search an rgrp's bitmap buffer to find a bit-pair representing + * a block in a given allocation state. + * @buf: the buffer that holds the bitmaps + * @len: the length (in bytes) of the buffer + * @goal: start search at this block's bit-pair (within @buffer) + * @state: GFS2_BLKST_XXX the state of the block we're looking for. + * + * Scope of @goal and returned block number is only within this bitmap buffer, + * not entire rgrp or filesystem. @buffer will be offset from the actual + * beginning of a bitmap block buffer, skipping any header structures, but + * headers are always a multiple of 64 bits long so that the buffer is + * always aligned to a 64 bit boundary. + * + * The size of the buffer is in bytes, but is it assumed that it is + * always ok to read a complete multiple of 64 bits at the end + * of the block in case the end is no aligned to a natural boundary. + * + * Return: the block number (bitmap buffer scope) that was found + */ + +static u32 gfs2_bitfit(const u8 *buf, const unsigned int len, + u32 goal, u8 state) +{ + u32 spoint = (goal << 1) & ((8*sizeof(u64)) - 1); + const __le64 *ptr = ((__le64 *)buf) + (goal >> 5); + const __le64 *end = (__le64 *)(buf + ALIGN(len, sizeof(u64))); + u64 tmp; + u64 mask = 0x5555555555555555ULL; + u32 bit; + + /* Mask off bits we don't care about at the start of the search */ + mask <<= spoint; + tmp = gfs2_bit_search(ptr, mask, state); + ptr++; + while(tmp == 0 && ptr < end) { + tmp = gfs2_bit_search(ptr, 0x5555555555555555ULL, state); + ptr++; + } + /* Mask off any bits which are more than len bytes from the start */ + if (ptr == end && (len & (sizeof(u64) - 1))) + tmp &= (((u64)~0) >> (64 - 8*(len & (sizeof(u64) - 1)))); + /* Didn't find anything, so return */ + if (tmp == 0) + return BFITNOENT; + ptr--; + bit = __ffs64(tmp); + bit /= 2; /* two bits per entry in the bitmap */ + return (((const unsigned char *)ptr - buf) * GFS2_NBBY) + bit; +} + +/** + * gfs2_rbm_from_block - Set the rbm based upon rgd and block number + * @rbm: The rbm with rgd already set correctly + * @block: The block number (filesystem relative) + * + * This sets the bi and offset members of an rbm based on a + * resource group and a filesystem relative block number. The + * resource group must be set in the rbm on entry, the bi and + * offset members will be set by this function. + * + * Returns: 0 on success, or an error code + */ + +static int gfs2_rbm_from_block(struct gfs2_rbm *rbm, u64 block) +{ + if (!rgrp_contains_block(rbm->rgd, block)) + return -E2BIG; + rbm->bii = 0; + rbm->offset = block - rbm->rgd->rd_data0; + /* Check if the block is within the first block */ + if (rbm->offset < rbm_bi(rbm)->bi_blocks) + return 0; + + /* Adjust for the size diff between gfs2_meta_header and gfs2_rgrp */ + rbm->offset += (sizeof(struct gfs2_rgrp) - + sizeof(struct gfs2_meta_header)) * GFS2_NBBY; + rbm->bii = rbm->offset / rbm->rgd->rd_sbd->sd_blocks_per_bitmap; + rbm->offset -= rbm->bii * rbm->rgd->rd_sbd->sd_blocks_per_bitmap; + return 0; +} + +/** + * gfs2_rbm_add - add a number of blocks to an rbm + * @rbm: The rbm with rgd already set correctly + * @blocks: The number of blocks to add to rpm + * + * This function takes an existing rbm structure and adds a number of blocks to + * it. + * + * Returns: True if the new rbm would point past the end of the rgrp. + */ + +static bool gfs2_rbm_add(struct gfs2_rbm *rbm, u32 blocks) +{ + struct gfs2_rgrpd *rgd = rbm->rgd; + struct gfs2_bitmap *bi = rgd->rd_bits + rbm->bii; + + if (rbm->offset + blocks < bi->bi_blocks) { + rbm->offset += blocks; + return false; + } + blocks -= bi->bi_blocks - rbm->offset; + + for(;;) { + bi++; + if (bi == rgd->rd_bits + rgd->rd_length) + return true; + if (blocks < bi->bi_blocks) { + rbm->offset = blocks; + rbm->bii = bi - rgd->rd_bits; + return false; + } + blocks -= bi->bi_blocks; + } +} + +/** + * gfs2_unaligned_extlen - Look for free blocks which are not byte aligned + * @rbm: Position to search (value/result) + * @n_unaligned: Number of unaligned blocks to check + * @len: Decremented for each block found (terminate on zero) + * + * Returns: true if a non-free block is encountered or the end of the resource + * group is reached. + */ + +static bool gfs2_unaligned_extlen(struct gfs2_rbm *rbm, u32 n_unaligned, u32 *len) +{ + u32 n; + u8 res; + + for (n = 0; n < n_unaligned; n++) { + res = gfs2_testbit(rbm, true); + if (res != GFS2_BLKST_FREE) + return true; + (*len)--; + if (*len == 0) + return true; + if (gfs2_rbm_add(rbm, 1)) + return true; + } + + return false; +} + +/** + * gfs2_free_extlen - Return extent length of free blocks + * @rrbm: Starting position + * @len: Max length to check + * + * Starting at the block specified by the rbm, see how many free blocks + * there are, not reading more than len blocks ahead. This can be done + * using memchr_inv when the blocks are byte aligned, but has to be done + * on a block by block basis in case of unaligned blocks. Also this + * function can cope with bitmap boundaries (although it must stop on + * a resource group boundary) + * + * Returns: Number of free blocks in the extent + */ + +static u32 gfs2_free_extlen(const struct gfs2_rbm *rrbm, u32 len) +{ + struct gfs2_rbm rbm = *rrbm; + u32 n_unaligned = rbm.offset & 3; + u32 size = len; + u32 bytes; + u32 chunk_size; + u8 *ptr, *start, *end; + u64 block; + struct gfs2_bitmap *bi; + + if (n_unaligned && + gfs2_unaligned_extlen(&rbm, 4 - n_unaligned, &len)) + goto out; + + n_unaligned = len & 3; + /* Start is now byte aligned */ + while (len > 3) { + bi = rbm_bi(&rbm); + start = bi->bi_bh->b_data; + if (bi->bi_clone) + start = bi->bi_clone; + start += bi->bi_offset; + end = start + bi->bi_bytes; + BUG_ON(rbm.offset & 3); + start += (rbm.offset / GFS2_NBBY); + bytes = min_t(u32, len / GFS2_NBBY, (end - start)); + ptr = memchr_inv(start, 0, bytes); + chunk_size = ((ptr == NULL) ? bytes : (ptr - start)); + chunk_size *= GFS2_NBBY; + BUG_ON(len < chunk_size); + len -= chunk_size; + block = gfs2_rbm_to_block(&rbm); + if (gfs2_rbm_from_block(&rbm, block + chunk_size)) { + n_unaligned = 0; + break; + } + if (ptr) { + n_unaligned = 3; + break; + } + n_unaligned = len & 3; + } + + /* Deal with any bits left over at the end */ + if (n_unaligned) + gfs2_unaligned_extlen(&rbm, n_unaligned, &len); +out: + return size - len; +} + +/** + * gfs2_bitcount - count the number of bits in a certain state + * @rgd: the resource group descriptor + * @buffer: the buffer that holds the bitmaps + * @buflen: the length (in bytes) of the buffer + * @state: the state of the block we're looking for + * + * Returns: The number of bits + */ + +static u32 gfs2_bitcount(struct gfs2_rgrpd *rgd, const u8 *buffer, + unsigned int buflen, u8 state) +{ + const u8 *byte = buffer; + const u8 *end = buffer + buflen; + const u8 state1 = state << 2; + const u8 state2 = state << 4; + const u8 state3 = state << 6; + u32 count = 0; + + for (; byte < end; byte++) { + if (((*byte) & 0x03) == state) + count++; + if (((*byte) & 0x0C) == state1) + count++; + if (((*byte) & 0x30) == state2) + count++; + if (((*byte) & 0xC0) == state3) + count++; + } + + return count; +} + +/** + * gfs2_rgrp_verify - Verify that a resource group is consistent + * @rgd: the rgrp + * + */ + +void gfs2_rgrp_verify(struct gfs2_rgrpd *rgd) +{ + struct gfs2_sbd *sdp = rgd->rd_sbd; + struct gfs2_bitmap *bi = NULL; + u32 length = rgd->rd_length; + u32 count[4], tmp; + int buf, x; + + memset(count, 0, 4 * sizeof(u32)); + + /* Count # blocks in each of 4 possible allocation states */ + for (buf = 0; buf < length; buf++) { + bi = rgd->rd_bits + buf; + for (x = 0; x < 4; x++) + count[x] += gfs2_bitcount(rgd, + bi->bi_bh->b_data + + bi->bi_offset, + bi->bi_bytes, x); + } + + if (count[0] != rgd->rd_free) { + gfs2_lm(sdp, "free data mismatch: %u != %u\n", + count[0], rgd->rd_free); + gfs2_consist_rgrpd(rgd); + return; + } + + tmp = rgd->rd_data - rgd->rd_free - rgd->rd_dinodes; + if (count[1] != tmp) { + gfs2_lm(sdp, "used data mismatch: %u != %u\n", + count[1], tmp); + gfs2_consist_rgrpd(rgd); + return; + } + + if (count[2] + count[3] != rgd->rd_dinodes) { + gfs2_lm(sdp, "used metadata mismatch: %u != %u\n", + count[2] + count[3], rgd->rd_dinodes); + gfs2_consist_rgrpd(rgd); + return; + } +} + +/** + * gfs2_blk2rgrpd - Find resource group for a given data/meta block number + * @sdp: The GFS2 superblock + * @blk: The data block number + * @exact: True if this needs to be an exact match + * + * The @exact argument should be set to true by most callers. The exception + * is when we need to match blocks which are not represented by the rgrp + * bitmap, but which are part of the rgrp (i.e. padding blocks) which are + * there for alignment purposes. Another way of looking at it is that @exact + * matches only valid data/metadata blocks, but with @exact false, it will + * match any block within the extent of the rgrp. + * + * Returns: The resource group, or NULL if not found + */ + +struct gfs2_rgrpd *gfs2_blk2rgrpd(struct gfs2_sbd *sdp, u64 blk, bool exact) +{ + struct rb_node *n, *next; + struct gfs2_rgrpd *cur; + + spin_lock(&sdp->sd_rindex_spin); + n = sdp->sd_rindex_tree.rb_node; + while (n) { + cur = rb_entry(n, struct gfs2_rgrpd, rd_node); + next = NULL; + if (blk < cur->rd_addr) + next = n->rb_left; + else if (blk >= cur->rd_data0 + cur->rd_data) + next = n->rb_right; + if (next == NULL) { + spin_unlock(&sdp->sd_rindex_spin); + if (exact) { + if (blk < cur->rd_addr) + return NULL; + if (blk >= cur->rd_data0 + cur->rd_data) + return NULL; + } + return cur; + } + n = next; + } + spin_unlock(&sdp->sd_rindex_spin); + + return NULL; +} + +/** + * gfs2_rgrpd_get_first - get the first Resource Group in the filesystem + * @sdp: The GFS2 superblock + * + * Returns: The first rgrp in the filesystem + */ + +struct gfs2_rgrpd *gfs2_rgrpd_get_first(struct gfs2_sbd *sdp) +{ + const struct rb_node *n; + struct gfs2_rgrpd *rgd; + + spin_lock(&sdp->sd_rindex_spin); + n = rb_first(&sdp->sd_rindex_tree); + rgd = rb_entry(n, struct gfs2_rgrpd, rd_node); + spin_unlock(&sdp->sd_rindex_spin); + + return rgd; +} + +/** + * gfs2_rgrpd_get_next - get the next RG + * @rgd: the resource group descriptor + * + * Returns: The next rgrp + */ + +struct gfs2_rgrpd *gfs2_rgrpd_get_next(struct gfs2_rgrpd *rgd) +{ + struct gfs2_sbd *sdp = rgd->rd_sbd; + const struct rb_node *n; + + spin_lock(&sdp->sd_rindex_spin); + n = rb_next(&rgd->rd_node); + if (n == NULL) + n = rb_first(&sdp->sd_rindex_tree); + + if (unlikely(&rgd->rd_node == n)) { + spin_unlock(&sdp->sd_rindex_spin); + return NULL; + } + rgd = rb_entry(n, struct gfs2_rgrpd, rd_node); + spin_unlock(&sdp->sd_rindex_spin); + return rgd; +} + +void check_and_update_goal(struct gfs2_inode *ip) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + if (!ip->i_goal || gfs2_blk2rgrpd(sdp, ip->i_goal, 1) == NULL) + ip->i_goal = ip->i_no_addr; +} + +void gfs2_free_clones(struct gfs2_rgrpd *rgd) +{ + int x; + + for (x = 0; x < rgd->rd_length; x++) { + struct gfs2_bitmap *bi = rgd->rd_bits + x; + kfree(bi->bi_clone); + bi->bi_clone = NULL; + } +} + +static void dump_rs(struct seq_file *seq, const struct gfs2_blkreserv *rs, + const char *fs_id_buf) +{ + struct gfs2_inode *ip = container_of(rs, struct gfs2_inode, i_res); + + gfs2_print_dbg(seq, "%s B: n:%llu s:%llu f:%u\n", + fs_id_buf, + (unsigned long long)ip->i_no_addr, + (unsigned long long)rs->rs_start, + rs->rs_requested); +} + +/** + * __rs_deltree - remove a multi-block reservation from the rgd tree + * @rs: The reservation to remove + * + */ +static void __rs_deltree(struct gfs2_blkreserv *rs) +{ + struct gfs2_rgrpd *rgd; + + if (!gfs2_rs_active(rs)) + return; + + rgd = rs->rs_rgd; + trace_gfs2_rs(rs, TRACE_RS_TREEDEL); + rb_erase(&rs->rs_node, &rgd->rd_rstree); + RB_CLEAR_NODE(&rs->rs_node); + + if (rs->rs_requested) { + /* return requested blocks to the rgrp */ + BUG_ON(rs->rs_rgd->rd_requested < rs->rs_requested); + rs->rs_rgd->rd_requested -= rs->rs_requested; + + /* The rgrp extent failure point is likely not to increase; + it will only do so if the freed blocks are somehow + contiguous with a span of free blocks that follows. Still, + it will force the number to be recalculated later. */ + rgd->rd_extfail_pt += rs->rs_requested; + rs->rs_requested = 0; + } +} + +/** + * gfs2_rs_deltree - remove a multi-block reservation from the rgd tree + * @rs: The reservation to remove + * + */ +void gfs2_rs_deltree(struct gfs2_blkreserv *rs) +{ + struct gfs2_rgrpd *rgd; + + rgd = rs->rs_rgd; + if (rgd) { + spin_lock(&rgd->rd_rsspin); + __rs_deltree(rs); + BUG_ON(rs->rs_requested); + spin_unlock(&rgd->rd_rsspin); + } +} + +/** + * gfs2_rs_delete - delete a multi-block reservation + * @ip: The inode for this reservation + * + */ +void gfs2_rs_delete(struct gfs2_inode *ip) +{ + struct inode *inode = &ip->i_inode; + + down_write(&ip->i_rw_mutex); + if (atomic_read(&inode->i_writecount) <= 1) + gfs2_rs_deltree(&ip->i_res); + up_write(&ip->i_rw_mutex); +} + +/** + * return_all_reservations - return all reserved blocks back to the rgrp. + * @rgd: the rgrp that needs its space back + * + * We previously reserved a bunch of blocks for allocation. Now we need to + * give them back. This leave the reservation structures in tact, but removes + * all of their corresponding "no-fly zones". + */ +static void return_all_reservations(struct gfs2_rgrpd *rgd) +{ + struct rb_node *n; + struct gfs2_blkreserv *rs; + + spin_lock(&rgd->rd_rsspin); + while ((n = rb_first(&rgd->rd_rstree))) { + rs = rb_entry(n, struct gfs2_blkreserv, rs_node); + __rs_deltree(rs); + } + spin_unlock(&rgd->rd_rsspin); +} + +void gfs2_clear_rgrpd(struct gfs2_sbd *sdp) +{ + struct rb_node *n; + struct gfs2_rgrpd *rgd; + struct gfs2_glock *gl; + + while ((n = rb_first(&sdp->sd_rindex_tree))) { + rgd = rb_entry(n, struct gfs2_rgrpd, rd_node); + gl = rgd->rd_gl; + + rb_erase(n, &sdp->sd_rindex_tree); + + if (gl) { + if (gl->gl_state != LM_ST_UNLOCKED) { + gfs2_glock_cb(gl, LM_ST_UNLOCKED); + flush_delayed_work(&gl->gl_work); + } + gfs2_rgrp_brelse(rgd); + glock_clear_object(gl, rgd); + gfs2_glock_put(gl); + } + + gfs2_free_clones(rgd); + return_all_reservations(rgd); + kfree(rgd->rd_bits); + rgd->rd_bits = NULL; + kmem_cache_free(gfs2_rgrpd_cachep, rgd); + } +} + +/** + * compute_bitstructs - Compute the bitmap sizes + * @rgd: The resource group descriptor + * + * Calculates bitmap descriptors, one for each block that contains bitmap data + * + * Returns: errno + */ + +static int compute_bitstructs(struct gfs2_rgrpd *rgd) +{ + struct gfs2_sbd *sdp = rgd->rd_sbd; + struct gfs2_bitmap *bi; + u32 length = rgd->rd_length; /* # blocks in hdr & bitmap */ + u32 bytes_left, bytes; + int x; + + if (!length) + return -EINVAL; + + rgd->rd_bits = kzalloc_objs(struct gfs2_bitmap, length, GFP_NOFS); + if (!rgd->rd_bits) + return -ENOMEM; + + bytes_left = rgd->rd_bitbytes; + + for (x = 0; x < length; x++) { + bi = rgd->rd_bits + x; + + bi->bi_flags = 0; + /* small rgrp; bitmap stored completely in header block */ + if (length == 1) { + bytes = bytes_left; + bi->bi_offset = sizeof(struct gfs2_rgrp); + bi->bi_start = 0; + bi->bi_bytes = bytes; + bi->bi_blocks = bytes * GFS2_NBBY; + /* header block */ + } else if (x == 0) { + bytes = sdp->sd_sb.sb_bsize - sizeof(struct gfs2_rgrp); + bi->bi_offset = sizeof(struct gfs2_rgrp); + bi->bi_start = 0; + bi->bi_bytes = bytes; + bi->bi_blocks = bytes * GFS2_NBBY; + /* last block */ + } else if (x + 1 == length) { + bytes = bytes_left; + bi->bi_offset = sizeof(struct gfs2_meta_header); + bi->bi_start = rgd->rd_bitbytes - bytes_left; + bi->bi_bytes = bytes; + bi->bi_blocks = bytes * GFS2_NBBY; + /* other blocks */ + } else { + bytes = sdp->sd_sb.sb_bsize - + sizeof(struct gfs2_meta_header); + bi->bi_offset = sizeof(struct gfs2_meta_header); + bi->bi_start = rgd->rd_bitbytes - bytes_left; + bi->bi_bytes = bytes; + bi->bi_blocks = bytes * GFS2_NBBY; + } + + bytes_left -= bytes; + } + + if (bytes_left) { + gfs2_consist_rgrpd(rgd); + return -EIO; + } + bi = rgd->rd_bits + (length - 1); + if ((bi->bi_start + bi->bi_bytes) * GFS2_NBBY != rgd->rd_data) { + gfs2_lm(sdp, + "ri_addr=%llu " + "ri_length=%u " + "ri_data0=%llu " + "ri_data=%u " + "ri_bitbytes=%u " + "start=%u len=%u offset=%u\n", + (unsigned long long)rgd->rd_addr, + rgd->rd_length, + (unsigned long long)rgd->rd_data0, + rgd->rd_data, + rgd->rd_bitbytes, + bi->bi_start, bi->bi_bytes, bi->bi_offset); + gfs2_consist_rgrpd(rgd); + return -EIO; + } + + return 0; +} + +/** + * gfs2_ri_total - Total up the file system space, according to the rindex. + * @sdp: the filesystem + * + */ +u64 gfs2_ri_total(struct gfs2_sbd *sdp) +{ + u64 total_data = 0; + struct inode *inode = sdp->sd_rindex; + struct gfs2_inode *ip = GFS2_I(inode); + char buf[sizeof(struct gfs2_rindex)]; + int error, rgrps; + + for (rgrps = 0;; rgrps++) { + loff_t pos = rgrps * sizeof(struct gfs2_rindex); + + if (pos + sizeof(struct gfs2_rindex) > i_size_read(inode)) + break; + error = gfs2_internal_read(ip, buf, &pos, + sizeof(struct gfs2_rindex)); + if (error != sizeof(struct gfs2_rindex)) + break; + total_data += be32_to_cpu(((struct gfs2_rindex *)buf)->ri_data); + } + return total_data; +} + +static int rgd_insert(struct gfs2_rgrpd *rgd) +{ + struct gfs2_sbd *sdp = rgd->rd_sbd; + struct rb_node **newn = &sdp->sd_rindex_tree.rb_node, *parent = NULL; + + /* Figure out where to put new node */ + while (*newn) { + struct gfs2_rgrpd *cur = rb_entry(*newn, struct gfs2_rgrpd, + rd_node); + + parent = *newn; + if (rgd->rd_addr < cur->rd_addr) + newn = &((*newn)->rb_left); + else if (rgd->rd_addr > cur->rd_addr) + newn = &((*newn)->rb_right); + else + return -EEXIST; + } + + rb_link_node(&rgd->rd_node, parent, newn); + rb_insert_color(&rgd->rd_node, &sdp->sd_rindex_tree); + sdp->sd_rgrps++; + return 0; +} + +/** + * read_rindex_entry - Pull in a new resource index entry from the disk + * @ip: Pointer to the rindex inode + * + * Returns: 0 on success, > 0 on EOF, error code otherwise + */ + +static int read_rindex_entry(struct gfs2_inode *ip) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + loff_t pos = sdp->sd_rgrps * sizeof(struct gfs2_rindex); + struct gfs2_rindex buf; + int error; + struct gfs2_rgrpd *rgd; + + if (pos >= i_size_read(&ip->i_inode)) + return 1; + + error = gfs2_internal_read(ip, (char *)&buf, &pos, + sizeof(struct gfs2_rindex)); + + if (error != sizeof(struct gfs2_rindex)) + return (error == 0) ? 1 : error; + + rgd = kmem_cache_zalloc(gfs2_rgrpd_cachep, GFP_NOFS); + error = -ENOMEM; + if (!rgd) + return error; + + rgd->rd_sbd = sdp; + rgd->rd_addr = be64_to_cpu(buf.ri_addr); + rgd->rd_length = be32_to_cpu(buf.ri_length); + rgd->rd_data0 = be64_to_cpu(buf.ri_data0); + rgd->rd_data = be32_to_cpu(buf.ri_data); + rgd->rd_bitbytes = be32_to_cpu(buf.ri_bitbytes); + spin_lock_init(&rgd->rd_rsspin); + mutex_init(&rgd->rd_mutex); + + error = gfs2_glock_get(sdp, rgd->rd_addr, + &gfs2_rgrp_glops, CREATE, &rgd->rd_gl); + if (error) + goto fail; + + error = compute_bitstructs(rgd); + if (error) + goto fail_glock; + + rgd->rd_rgl = (struct gfs2_rgrp_lvb *)rgd->rd_gl->gl_lksb.sb_lvbptr; + rgd->rd_flags &= ~GFS2_RDF_PREFERRED; + if (rgd->rd_data > sdp->sd_max_rg_data) + sdp->sd_max_rg_data = rgd->rd_data; + spin_lock(&sdp->sd_rindex_spin); + error = rgd_insert(rgd); + spin_unlock(&sdp->sd_rindex_spin); + if (!error) { + glock_set_object(rgd->rd_gl, rgd); + return 0; + } + + error = 0; /* someone else read in the rgrp; free it and ignore it */ +fail_glock: + gfs2_glock_put(rgd->rd_gl); + +fail: + kfree(rgd->rd_bits); + rgd->rd_bits = NULL; + kmem_cache_free(gfs2_rgrpd_cachep, rgd); + return error; +} + +/** + * set_rgrp_preferences - Run all the rgrps, selecting some we prefer to use + * @sdp: the GFS2 superblock + * + * The purpose of this function is to select a subset of the resource groups + * and mark them as PREFERRED. We do it in such a way that each node prefers + * to use a unique set of rgrps to minimize glock contention. + */ +static void set_rgrp_preferences(struct gfs2_sbd *sdp) +{ + struct gfs2_rgrpd *rgd, *first; + int i; + + /* Skip an initial number of rgrps, based on this node's journal ID. + That should start each node out on its own set. */ + rgd = gfs2_rgrpd_get_first(sdp); + for (i = 0; i < sdp->sd_lockstruct.ls_jid; i++) + rgd = gfs2_rgrpd_get_next(rgd); + first = rgd; + + do { + rgd->rd_flags |= GFS2_RDF_PREFERRED; + for (i = 0; i < sdp->sd_journals; i++) { + rgd = gfs2_rgrpd_get_next(rgd); + if (!rgd || rgd == first) + break; + } + } while (rgd && rgd != first); +} + +/** + * gfs2_ri_update - Pull in a new resource index from the disk + * @ip: pointer to the rindex inode + * + * Returns: 0 on successful update, error code otherwise + */ + +static int gfs2_ri_update(struct gfs2_inode *ip) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + int error; + + do { + error = read_rindex_entry(ip); + } while (error == 0); + + if (error < 0) + return error; + + if (RB_EMPTY_ROOT(&sdp->sd_rindex_tree)) { + fs_err(sdp, "no resource groups found in the file system.\n"); + return -ENOENT; + } + set_rgrp_preferences(sdp); + + sdp->sd_rindex_uptodate = 1; + return 0; +} + +/** + * gfs2_rindex_update - Update the rindex if required + * @sdp: The GFS2 superblock + * + * We grab a lock on the rindex inode to make sure that it doesn't + * change whilst we are performing an operation. We keep this lock + * for quite long periods of time compared to other locks. This + * doesn't matter, since it is shared and it is very, very rarely + * accessed in the exclusive mode (i.e. only when expanding the filesystem). + * + * This makes sure that we're using the latest copy of the resource index + * special file, which might have been updated if someone expanded the + * filesystem (via gfs2_grow utility), which adds new resource groups. + * + * Returns: 0 on succeess, error code otherwise + */ + +int gfs2_rindex_update(struct gfs2_sbd *sdp) +{ + struct gfs2_inode *ip = GFS2_I(sdp->sd_rindex); + struct gfs2_glock *gl = ip->i_gl; + struct gfs2_holder ri_gh; + int error = 0; + int unlock_required = 0; + + /* Read new copy from disk if we don't have the latest */ + if (!sdp->sd_rindex_uptodate) { + if (!gfs2_glock_is_locked_by_me(gl)) { + error = gfs2_glock_nq_init(gl, LM_ST_SHARED, 0, &ri_gh); + if (error) + return error; + unlock_required = 1; + } + if (!sdp->sd_rindex_uptodate) + error = gfs2_ri_update(ip); + if (unlock_required) + gfs2_glock_dq_uninit(&ri_gh); + } + + return error; +} + +static void gfs2_rgrp_in(struct gfs2_rgrpd *rgd, const void *buf) +{ + const struct gfs2_rgrp *str = buf; + u32 rg_flags; + + rg_flags = be32_to_cpu(str->rg_flags); + rg_flags &= ~GFS2_RDF_MASK; + rgd->rd_flags &= GFS2_RDF_MASK; + rgd->rd_flags |= rg_flags; + rgd->rd_free = be32_to_cpu(str->rg_free); + rgd->rd_dinodes = be32_to_cpu(str->rg_dinodes); + rgd->rd_igeneration = be64_to_cpu(str->rg_igeneration); + /* rd_data0, rd_data and rd_bitbytes already set from rindex */ +} + +static void gfs2_rgrp_ondisk2lvb(struct gfs2_rgrp_lvb *rgl, const void *buf) +{ + const struct gfs2_rgrp *str = buf; + + rgl->rl_magic = cpu_to_be32(GFS2_MAGIC); + rgl->rl_flags = str->rg_flags; + rgl->rl_free = str->rg_free; + rgl->rl_dinodes = str->rg_dinodes; + rgl->rl_igeneration = str->rg_igeneration; + rgl->__pad = 0UL; +} + +static void gfs2_rgrp_out(struct gfs2_rgrpd *rgd, void *buf) +{ + struct gfs2_rgrpd *next = gfs2_rgrpd_get_next(rgd); + struct gfs2_rgrp *str = buf; + u32 crc; + + str->rg_flags = cpu_to_be32(rgd->rd_flags & ~GFS2_RDF_MASK); + str->rg_free = cpu_to_be32(rgd->rd_free); + str->rg_dinodes = cpu_to_be32(rgd->rd_dinodes); + if (next == NULL) + str->rg_skip = 0; + else if (next->rd_addr > rgd->rd_addr) + str->rg_skip = cpu_to_be32(next->rd_addr - rgd->rd_addr); + str->rg_igeneration = cpu_to_be64(rgd->rd_igeneration); + str->rg_data0 = cpu_to_be64(rgd->rd_data0); + str->rg_data = cpu_to_be32(rgd->rd_data); + str->rg_bitbytes = cpu_to_be32(rgd->rd_bitbytes); + str->rg_crc = 0; + crc = gfs2_disk_hash(buf, sizeof(struct gfs2_rgrp)); + str->rg_crc = cpu_to_be32(crc); + + memset(&str->rg_reserved, 0, sizeof(str->rg_reserved)); + gfs2_rgrp_ondisk2lvb(rgd->rd_rgl, buf); +} + +static int gfs2_rgrp_lvb_valid(struct gfs2_rgrpd *rgd) +{ + struct gfs2_rgrp_lvb *rgl = rgd->rd_rgl; + struct gfs2_rgrp *str = (struct gfs2_rgrp *)rgd->rd_bits[0].bi_bh->b_data; + struct gfs2_sbd *sdp = rgd->rd_sbd; + int valid = 1; + + if (rgl->rl_flags != str->rg_flags) { + fs_warn(sdp, "GFS2: rgd: %llu lvb flag mismatch %u/%u", + (unsigned long long)rgd->rd_addr, + be32_to_cpu(rgl->rl_flags), be32_to_cpu(str->rg_flags)); + valid = 0; + } + if (rgl->rl_free != str->rg_free) { + fs_warn(sdp, "GFS2: rgd: %llu lvb free mismatch %u/%u", + (unsigned long long)rgd->rd_addr, + be32_to_cpu(rgl->rl_free), be32_to_cpu(str->rg_free)); + valid = 0; + } + if (rgl->rl_dinodes != str->rg_dinodes) { + fs_warn(sdp, "GFS2: rgd: %llu lvb dinode mismatch %u/%u", + (unsigned long long)rgd->rd_addr, + be32_to_cpu(rgl->rl_dinodes), + be32_to_cpu(str->rg_dinodes)); + valid = 0; + } + if (rgl->rl_igeneration != str->rg_igeneration) { + fs_warn(sdp, "GFS2: rgd: %llu lvb igen mismatch %llu/%llu", + (unsigned long long)rgd->rd_addr, + (unsigned long long)be64_to_cpu(rgl->rl_igeneration), + (unsigned long long)be64_to_cpu(str->rg_igeneration)); + valid = 0; + } + return valid; +} + +static u32 count_unlinked(struct gfs2_rgrpd *rgd) +{ + struct gfs2_bitmap *bi; + const u32 length = rgd->rd_length; + const u8 *buffer = NULL; + u32 i, goal, count = 0; + + for (i = 0, bi = rgd->rd_bits; i < length; i++, bi++) { + goal = 0; + buffer = bi->bi_bh->b_data + bi->bi_offset; + WARN_ON(!buffer_uptodate(bi->bi_bh)); + while (goal < bi->bi_blocks) { + goal = gfs2_bitfit(buffer, bi->bi_bytes, goal, + GFS2_BLKST_UNLINKED); + if (goal == BFITNOENT) + break; + count++; + goal++; + } + } + + return count; +} + +static void rgrp_set_bitmap_flags(struct gfs2_rgrpd *rgd) +{ + struct gfs2_bitmap *bi; + int x; + + if (rgd->rd_free) { + for (x = 0; x < rgd->rd_length; x++) { + bi = rgd->rd_bits + x; + clear_bit(GBF_FULL, &bi->bi_flags); + } + } else { + for (x = 0; x < rgd->rd_length; x++) { + bi = rgd->rd_bits + x; + set_bit(GBF_FULL, &bi->bi_flags); + } + } +} + +/** + * gfs2_rgrp_go_instantiate - Read in a RG's header and bitmaps + * @gl: the glock representing the rgrpd to read in + * + * Read in all of a Resource Group's header and bitmap blocks. + * Caller must eventually call gfs2_rgrp_brelse() to free the bitmaps. + * + * Returns: errno + */ + +int gfs2_rgrp_go_instantiate(struct gfs2_glock *gl) +{ + struct gfs2_rgrpd *rgd = gl->gl_object; + struct gfs2_sbd *sdp = rgd->rd_sbd; + unsigned int length = rgd->rd_length; + struct gfs2_bitmap *bi; + unsigned int x, y; + int error; + + if (rgd->rd_bits[0].bi_bh != NULL) + return 0; + + for (x = 0; x < length; x++) { + bi = rgd->rd_bits + x; + error = gfs2_meta_read(gl, rgd->rd_addr + x, 0, 0, &bi->bi_bh); + if (error) + goto fail; + } + + for (y = length; y--;) { + bi = rgd->rd_bits + y; + error = gfs2_meta_wait(sdp, bi->bi_bh); + if (error) + goto fail; + if (gfs2_metatype_check(sdp, bi->bi_bh, y ? GFS2_METATYPE_RB : + GFS2_METATYPE_RG)) { + error = -EIO; + goto fail; + } + } + + gfs2_rgrp_in(rgd, (rgd->rd_bits[0].bi_bh)->b_data); + rgrp_set_bitmap_flags(rgd); + rgd->rd_flags |= GFS2_RDF_CHECK; + rgd->rd_free_clone = rgd->rd_free; + GLOCK_BUG_ON(rgd->rd_gl, rgd->rd_reserved); + /* max out the rgrp allocation failure point */ + rgd->rd_extfail_pt = rgd->rd_free; + if (cpu_to_be32(GFS2_MAGIC) != rgd->rd_rgl->rl_magic) { + rgd->rd_rgl->rl_unlinked = cpu_to_be32(count_unlinked(rgd)); + gfs2_rgrp_ondisk2lvb(rgd->rd_rgl, + rgd->rd_bits[0].bi_bh->b_data); + } else if (sdp->sd_args.ar_rgrplvb) { + if (!gfs2_rgrp_lvb_valid(rgd)){ + gfs2_consist_rgrpd(rgd); + error = -EIO; + goto fail; + } + if (rgd->rd_rgl->rl_unlinked == 0) + rgd->rd_flags &= ~GFS2_RDF_CHECK; + } + return 0; + +fail: + while (x--) { + bi = rgd->rd_bits + x; + brelse(bi->bi_bh); + bi->bi_bh = NULL; + gfs2_assert_warn(sdp, !bi->bi_clone); + } + return error; +} + +static int update_rgrp_lvb(struct gfs2_rgrpd *rgd, struct gfs2_holder *gh) +{ + u32 rl_flags; + + if (!test_bit(GLF_INSTANTIATE_NEEDED, &gh->gh_gl->gl_flags)) + return 0; + + if (cpu_to_be32(GFS2_MAGIC) != rgd->rd_rgl->rl_magic) + return gfs2_instantiate(gh); + + rl_flags = be32_to_cpu(rgd->rd_rgl->rl_flags); + rl_flags &= ~GFS2_RDF_MASK; + rgd->rd_flags &= GFS2_RDF_MASK; + rgd->rd_flags |= (rl_flags | GFS2_RDF_CHECK); + if (rgd->rd_rgl->rl_unlinked == 0) + rgd->rd_flags &= ~GFS2_RDF_CHECK; + rgd->rd_free = be32_to_cpu(rgd->rd_rgl->rl_free); + rgrp_set_bitmap_flags(rgd); + rgd->rd_free_clone = rgd->rd_free; + GLOCK_BUG_ON(rgd->rd_gl, rgd->rd_reserved); + /* max out the rgrp allocation failure point */ + rgd->rd_extfail_pt = rgd->rd_free; + rgd->rd_dinodes = be32_to_cpu(rgd->rd_rgl->rl_dinodes); + rgd->rd_igeneration = be64_to_cpu(rgd->rd_rgl->rl_igeneration); + return 0; +} + +/** + * gfs2_rgrp_brelse - Release RG bitmaps read in with gfs2_rgrp_bh_get() + * @rgd: The resource group + * + */ + +void gfs2_rgrp_brelse(struct gfs2_rgrpd *rgd) +{ + int x, length = rgd->rd_length; + + for (x = 0; x < length; x++) { + struct gfs2_bitmap *bi = rgd->rd_bits + x; + if (bi->bi_bh) { + brelse(bi->bi_bh); + bi->bi_bh = NULL; + } + } + set_bit(GLF_INSTANTIATE_NEEDED, &rgd->rd_gl->gl_flags); +} + +int gfs2_rgrp_send_discards(struct gfs2_sbd *sdp, u64 offset, + struct buffer_head *bh, + const struct gfs2_bitmap *bi, unsigned minlen, u64 *ptrimmed) +{ + struct super_block *sb = sdp->sd_vfs; + u64 blk; + sector_t start = 0; + sector_t nr_blks = 0; + int rv = -EIO; + unsigned int x; + u32 trimmed = 0; + u8 diff; + + for (x = 0; x < bi->bi_bytes; x++) { + const u8 *clone = bi->bi_clone ? bi->bi_clone : bi->bi_bh->b_data; + clone += bi->bi_offset; + clone += x; + if (bh) { + const u8 *orig = bh->b_data + bi->bi_offset + x; + diff = ~(*orig | (*orig >> 1)) & (*clone | (*clone >> 1)); + } else { + diff = ~(*clone | (*clone >> 1)); + } + diff &= 0x55; + if (diff == 0) + continue; + blk = offset + ((bi->bi_start + x) * GFS2_NBBY); + while(diff) { + if (diff & 1) { + if (nr_blks == 0) + goto start_new_extent; + if ((start + nr_blks) != blk) { + if (nr_blks >= minlen) { + rv = sb_issue_discard(sb, + start, nr_blks, + GFP_NOFS, 0); + if (rv) + goto fail; + trimmed += nr_blks; + } + nr_blks = 0; +start_new_extent: + start = blk; + } + nr_blks++; + } + diff >>= 2; + blk++; + } + } + if (nr_blks >= minlen) { + rv = sb_issue_discard(sb, start, nr_blks, GFP_NOFS, 0); + if (rv) + goto fail; + trimmed += nr_blks; + } + if (ptrimmed) + *ptrimmed = trimmed; + return 0; + +fail: + if (sdp->sd_args.ar_discard) + fs_warn(sdp, "error %d on discard request, turning discards off for this filesystem\n", rv); + sdp->sd_args.ar_discard = 0; + return rv; +} + +/** + * gfs2_fitrim - Generate discard requests for unused bits of the filesystem + * @filp: Any file on the filesystem + * @argp: Pointer to the arguments (also used to pass result) + * + * Returns: 0 on success, otherwise error code + */ + +int gfs2_fitrim(struct file *filp, void __user *argp) +{ + struct inode *inode = file_inode(filp); + struct gfs2_sbd *sdp = GFS2_SB(inode); + struct block_device *bdev = sdp->sd_vfs->s_bdev; + struct buffer_head *bh; + struct gfs2_rgrpd *rgd; + struct gfs2_rgrpd *rgd_end; + struct gfs2_holder gh; + struct fstrim_range r; + int ret = 0; + u64 amt; + u64 trimmed = 0; + u64 start, end, minlen; + unsigned int x; + unsigned bs_shift = sdp->sd_sb.sb_bsize_shift; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (!test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags)) + return -EROFS; + + if (!bdev_max_discard_sectors(bdev)) + return -EOPNOTSUPP; + + if (copy_from_user(&r, argp, sizeof(r))) + return -EFAULT; + + ret = gfs2_rindex_update(sdp); + if (ret) + return ret; + + start = r.start >> bs_shift; + end = start + (r.len >> bs_shift); + minlen = max_t(u64, r.minlen, sdp->sd_sb.sb_bsize); + minlen = max_t(u64, minlen, bdev_discard_granularity(bdev)) >> bs_shift; + + if (end <= start || minlen > sdp->sd_max_rg_data) + return -EINVAL; + + rgd = gfs2_blk2rgrpd(sdp, start, 0); + rgd_end = gfs2_blk2rgrpd(sdp, end, 0); + + if ((gfs2_rgrpd_get_first(sdp) == gfs2_rgrpd_get_next(rgd_end)) + && (start > rgd_end->rd_data0 + rgd_end->rd_data)) + return -EINVAL; /* start is beyond the end of the fs */ + + while (1) { + + ret = gfs2_glock_nq_init(rgd->rd_gl, LM_ST_EXCLUSIVE, + LM_FLAG_NODE_SCOPE, &gh); + if (ret) + goto out; + + if (!(rgd->rd_flags & GFS2_RGF_TRIMMED)) { + /* Trim each bitmap in the rgrp */ + for (x = 0; x < rgd->rd_length; x++) { + struct gfs2_bitmap *bi = rgd->rd_bits + x; + rgrp_lock_local(rgd); + ret = gfs2_rgrp_send_discards(sdp, + rgd->rd_data0, NULL, bi, minlen, + &amt); + rgrp_unlock_local(rgd); + if (ret) { + gfs2_glock_dq_uninit(&gh); + goto out; + } + trimmed += amt; + } + + /* Mark rgrp as having been trimmed */ + ret = gfs2_trans_begin(sdp, RES_RG_HDR, 0); + if (ret == 0) { + bh = rgd->rd_bits[0].bi_bh; + rgrp_lock_local(rgd); + rgd->rd_flags |= GFS2_RGF_TRIMMED; + gfs2_trans_add_meta(rgd->rd_gl, bh); + gfs2_rgrp_out(rgd, bh->b_data); + rgrp_unlock_local(rgd); + gfs2_trans_end(sdp); + } + } + gfs2_glock_dq_uninit(&gh); + + if (rgd == rgd_end) + break; + + rgd = gfs2_rgrpd_get_next(rgd); + } + +out: + r.len = trimmed << bs_shift; + if (copy_to_user(argp, &r, sizeof(r))) + return -EFAULT; + + return ret; +} + +/** + * rs_insert - insert a new multi-block reservation into the rgrp's rb_tree + * @ip: the inode structure + * + */ +static void rs_insert(struct gfs2_inode *ip) +{ + struct rb_node **newn, *parent = NULL; + int rc; + struct gfs2_blkreserv *rs = &ip->i_res; + struct gfs2_rgrpd *rgd = rs->rs_rgd; + + BUG_ON(gfs2_rs_active(rs)); + + spin_lock(&rgd->rd_rsspin); + newn = &rgd->rd_rstree.rb_node; + while (*newn) { + struct gfs2_blkreserv *cur = + rb_entry(*newn, struct gfs2_blkreserv, rs_node); + + parent = *newn; + rc = rs_cmp(rs->rs_start, rs->rs_requested, cur); + if (rc > 0) + newn = &((*newn)->rb_right); + else if (rc < 0) + newn = &((*newn)->rb_left); + else { + spin_unlock(&rgd->rd_rsspin); + WARN_ON(1); + return; + } + } + + rb_link_node(&rs->rs_node, parent, newn); + rb_insert_color(&rs->rs_node, &rgd->rd_rstree); + + /* Do our rgrp accounting for the reservation */ + rgd->rd_requested += rs->rs_requested; /* blocks requested */ + spin_unlock(&rgd->rd_rsspin); + trace_gfs2_rs(rs, TRACE_RS_INSERT); +} + +/** + * rgd_free - return the number of free blocks we can allocate + * @rgd: the resource group + * @rs: The reservation to free + * + * This function returns the number of free blocks for an rgrp. + * That's the clone-free blocks (blocks that are free, not including those + * still being used for unlinked files that haven't been deleted.) + * + * It also subtracts any blocks reserved by someone else, but does not + * include free blocks that are still part of our current reservation, + * because obviously we can (and will) allocate them. + */ +static inline u32 rgd_free(struct gfs2_rgrpd *rgd, struct gfs2_blkreserv *rs) +{ + u32 tot_reserved, tot_free; + + if (WARN_ON_ONCE(rgd->rd_requested < rs->rs_requested)) + return 0; + tot_reserved = rgd->rd_requested - rs->rs_requested; + + if (rgd->rd_free_clone < tot_reserved) + tot_reserved = 0; + + tot_free = rgd->rd_free_clone - tot_reserved; + + return tot_free; +} + +/** + * rg_mblk_search - find a group of multiple free blocks to form a reservation + * @rgd: the resource group descriptor + * @ip: pointer to the inode for which we're reserving blocks + * @ap: the allocation parameters + * + */ + +static void rg_mblk_search(struct gfs2_rgrpd *rgd, struct gfs2_inode *ip, + const struct gfs2_alloc_parms *ap) +{ + struct gfs2_rbm rbm = { .rgd = rgd, }; + u64 goal; + struct gfs2_blkreserv *rs = &ip->i_res; + u32 extlen; + u32 free_blocks, blocks_available; + int ret; + struct inode *inode = &ip->i_inode; + + spin_lock(&rgd->rd_rsspin); + free_blocks = rgd_free(rgd, rs); + if (rgd->rd_free_clone < rgd->rd_requested) + free_blocks = 0; + blocks_available = rgd->rd_free_clone - rgd->rd_reserved; + if (rgd == rs->rs_rgd) + blocks_available += rs->rs_reserved; + spin_unlock(&rgd->rd_rsspin); + + if (S_ISDIR(inode->i_mode)) + extlen = 1; + else { + extlen = max_t(u32, atomic_read(&ip->i_sizehint), ap->target); + extlen = clamp(extlen, (u32)RGRP_RSRV_MINBLKS, free_blocks); + } + if (free_blocks < extlen || blocks_available < extlen) + return; + + /* Find bitmap block that contains bits for goal block */ + if (rgrp_contains_block(rgd, ip->i_goal)) + goal = ip->i_goal; + else + goal = rgd->rd_last_alloc + rgd->rd_data0; + + if (WARN_ON(gfs2_rbm_from_block(&rbm, goal))) + return; + + ret = gfs2_rbm_find(&rbm, GFS2_BLKST_FREE, &extlen, &ip->i_res, true); + if (ret == 0) { + rs->rs_start = gfs2_rbm_to_block(&rbm); + rs->rs_requested = extlen; + rs_insert(ip); + } else { + if (goal == rgd->rd_last_alloc + rgd->rd_data0) + rgd->rd_last_alloc = 0; + } +} + +/** + * gfs2_next_unreserved_block - Return next block that is not reserved + * @rgd: The resource group + * @block: The starting block + * @length: The required length + * @ignore_rs: Reservation to ignore + * + * If the block does not appear in any reservation, then return the + * block number unchanged. If it does appear in the reservation, then + * keep looking through the tree of reservations in order to find the + * first block number which is not reserved. + */ + +static u64 gfs2_next_unreserved_block(struct gfs2_rgrpd *rgd, u64 block, + u32 length, + struct gfs2_blkreserv *ignore_rs) +{ + struct gfs2_blkreserv *rs; + struct rb_node *n; + int rc; + + spin_lock(&rgd->rd_rsspin); + n = rgd->rd_rstree.rb_node; + while (n) { + rs = rb_entry(n, struct gfs2_blkreserv, rs_node); + rc = rs_cmp(block, length, rs); + if (rc < 0) + n = n->rb_left; + else if (rc > 0) + n = n->rb_right; + else + break; + } + + if (n) { + while (rs_cmp(block, length, rs) == 0 && rs != ignore_rs) { + block = rs->rs_start + rs->rs_requested; + n = n->rb_right; + if (n == NULL) + break; + rs = rb_entry(n, struct gfs2_blkreserv, rs_node); + } + } + + spin_unlock(&rgd->rd_rsspin); + return block; +} + +/** + * gfs2_reservation_check_and_update - Check for reservations during block alloc + * @rbm: The current position in the resource group + * @rs: Our own reservation + * @minext: The minimum extent length + * @maxext: A pointer to the maximum extent structure + * + * This checks the current position in the rgrp to see whether there is + * a reservation covering this block. If not then this function is a + * no-op. If there is, then the position is moved to the end of the + * contiguous reservation(s) so that we are pointing at the first + * non-reserved block. + * + * Returns: 0 if no reservation, 1 if @rbm has changed, otherwise an error + */ + +static int gfs2_reservation_check_and_update(struct gfs2_rbm *rbm, + struct gfs2_blkreserv *rs, + u32 minext, + struct gfs2_extent *maxext) +{ + u64 block = gfs2_rbm_to_block(rbm); + u32 extlen = 1; + u64 nblock; + + /* + * If we have a minimum extent length, then skip over any extent + * which is less than the min extent length in size. + */ + if (minext > 1) { + extlen = gfs2_free_extlen(rbm, minext); + if (extlen <= maxext->len) + goto fail; + } + + /* + * Check the extent which has been found against the reservations + * and skip if parts of it are already reserved + */ + nblock = gfs2_next_unreserved_block(rbm->rgd, block, extlen, rs); + if (nblock == block) { + if (!minext || extlen >= minext) + return 0; + + if (extlen > maxext->len) { + maxext->len = extlen; + maxext->rbm = *rbm; + } + } else { + u64 len = nblock - block; + if (len >= (u64)1 << 32) + return -E2BIG; + extlen = len; + } +fail: + if (gfs2_rbm_add(rbm, extlen)) + return -E2BIG; + return 1; +} + +/** + * gfs2_rbm_find - Look for blocks of a particular state + * @rbm: Value/result starting position and final position + * @state: The state which we want to find + * @minext: Pointer to the requested extent length + * This is updated to be the actual reservation size. + * @rs: Our own reservation (NULL to skip checking for reservations) + * @nowrap: Stop looking at the end of the rgrp, rather than wrapping + * around until we've reached the starting point. + * + * Side effects: + * - If looking for free blocks, we set GBF_FULL on each bitmap which + * has no free blocks in it. + * - If looking for free blocks, we set rd_extfail_pt on each rgrp which + * has come up short on a free block search. + * + * Returns: 0 on success, -ENOSPC if there is no block of the requested state + */ + +static int gfs2_rbm_find(struct gfs2_rbm *rbm, u8 state, u32 *minext, + struct gfs2_blkreserv *rs, bool nowrap) +{ + bool scan_from_start = rbm->bii == 0 && rbm->offset == 0; + struct buffer_head *bh; + int last_bii; + u32 offset; + u8 *buffer; + bool wrapped = false; + int ret; + struct gfs2_bitmap *bi; + struct gfs2_extent maxext = { .rbm.rgd = rbm->rgd, }; + + /* + * Determine the last bitmap to search. If we're not starting at the + * beginning of a bitmap, we need to search that bitmap twice to scan + * the entire resource group. + */ + last_bii = rbm->bii - (rbm->offset == 0); + + while(1) { + bi = rbm_bi(rbm); + if (test_bit(GBF_FULL, &bi->bi_flags) && + (state == GFS2_BLKST_FREE)) + goto next_bitmap; + + bh = bi->bi_bh; + buffer = bh->b_data + bi->bi_offset; + WARN_ON(!buffer_uptodate(bh)); + if (state != GFS2_BLKST_UNLINKED && bi->bi_clone) + buffer = bi->bi_clone + bi->bi_offset; + offset = gfs2_bitfit(buffer, bi->bi_bytes, rbm->offset, state); + if (offset == BFITNOENT) { + if (state == GFS2_BLKST_FREE && rbm->offset == 0) + set_bit(GBF_FULL, &bi->bi_flags); + goto next_bitmap; + } + rbm->offset = offset; + if (!rs || !minext) + return 0; + + ret = gfs2_reservation_check_and_update(rbm, rs, *minext, + &maxext); + if (ret == 0) + return 0; + if (ret > 0) + goto next_iter; + if (ret == -E2BIG) { + rbm->bii = 0; + rbm->offset = 0; + goto res_covered_end_of_rgrp; + } + return ret; + +next_bitmap: /* Find next bitmap in the rgrp */ + rbm->offset = 0; + rbm->bii++; + if (rbm->bii == rbm->rgd->rd_length) + rbm->bii = 0; +res_covered_end_of_rgrp: + if (rbm->bii == 0) { + if (wrapped) + break; + wrapped = true; + if (nowrap) + break; + } +next_iter: + /* Have we scanned the entire resource group? */ + if (wrapped && rbm->bii > last_bii) + break; + } + + if (state != GFS2_BLKST_FREE) + return -ENOSPC; + + /* If the extent was too small, and it's smaller than the smallest + to have failed before, remember for future reference that it's + useless to search this rgrp again for this amount or more. */ + if (wrapped && (scan_from_start || rbm->bii > last_bii) && + *minext < rbm->rgd->rd_extfail_pt) + rbm->rgd->rd_extfail_pt = *minext - 1; + + /* If the maximum extent we found is big enough to fulfill the + minimum requirements, use it anyway. */ + if (maxext.len) { + *rbm = maxext.rbm; + *minext = maxext.len; + return 0; + } + + return -ENOSPC; +} + +/** + * try_rgrp_unlink - Look for any unlinked, allocated, but unused inodes + * @rgd: The rgrp + * @last_unlinked: block address of the last dinode we unlinked + * @skip: block address we should explicitly not unlink + * + * Returns: 0 if no error + * The inode, if one has been found, in inode. + */ + +static void try_rgrp_unlink(struct gfs2_rgrpd *rgd, u64 *last_unlinked, u64 skip) +{ + u64 block; + struct gfs2_sbd *sdp = rgd->rd_sbd; + struct gfs2_glock *gl; + struct gfs2_inode *ip; + int error; + int found = 0; + struct gfs2_rbm rbm = { .rgd = rgd, .bii = 0, .offset = 0 }; + + while (1) { + error = gfs2_rbm_find(&rbm, GFS2_BLKST_UNLINKED, NULL, NULL, + true); + if (error == -ENOSPC) + break; + if (WARN_ON_ONCE(error)) + break; + + block = gfs2_rbm_to_block(&rbm); + if (gfs2_rbm_from_block(&rbm, block + 1)) + break; + if (*last_unlinked != NO_BLOCK && block <= *last_unlinked) + continue; + if (block == skip) + continue; + *last_unlinked = block; + + error = gfs2_glock_get(sdp, block, &gfs2_iopen_glops, CREATE, &gl); + if (error) + continue; + + /* If the inode is already in cache, we can ignore it here + * because the existing inode disposal code will deal with + * it when all refs have gone away. Accessing gl_object like + * this is not safe in general. Here it is ok because we do + * not dereference the pointer, and we only need an approx + * answer to whether it is NULL or not. + */ + ip = gl->gl_object; + + if (ip || !gfs2_queue_verify_delete(gl, false)) + gfs2_glock_put(gl); + else + found++; + + /* Limit reclaim to sensible number of tasks */ + if (found > NR_CPUS) + return; + } + + rgd->rd_flags &= ~GFS2_RDF_CHECK; + return; +} + +/** + * gfs2_rgrp_congested - Use stats to figure out whether an rgrp is congested + * @rgd: The rgrp in question + * @loops: An indication of how picky we can be (0=very, 1=less so) + * + * This function uses the recently added glock statistics in order to + * figure out whether a parciular resource group is suffering from + * contention from multiple nodes. This is done purely on the basis + * of timings, since this is the only data we have to work with and + * our aim here is to reject a resource group which is highly contended + * but (very important) not to do this too often in order to ensure that + * we do not land up introducing fragmentation by changing resource + * groups when not actually required. + * + * The calculation is fairly simple, we want to know whether the SRTTB + * (i.e. smoothed round trip time for blocking operations) to acquire + * the lock for this rgrp's glock is significantly greater than the + * time taken for resource groups on average. We introduce a margin in + * the form of the variable @var which is computed as the sum of the two + * respective variences, and multiplied by a factor depending on @loops + * and whether we have a lot of data to base the decision on. This is + * then tested against the square difference of the means in order to + * decide whether the result is statistically significant or not. + * + * Returns: A boolean verdict on the congestion status + */ + +static bool gfs2_rgrp_congested(const struct gfs2_rgrpd *rgd, int loops) +{ + const struct gfs2_glock *gl = rgd->rd_gl; + const struct gfs2_sbd *sdp = glock_sbd(gl); + struct gfs2_lkstats *st; + u64 r_dcount, l_dcount; + u64 l_srttb, a_srttb = 0; + s64 srttb_diff; + u64 sqr_diff; + u64 var; + int cpu, nonzero = 0; + + preempt_disable(); + for_each_present_cpu(cpu) { + st = &per_cpu_ptr(sdp->sd_lkstats, cpu)->lkstats[LM_TYPE_RGRP]; + if (st->stats[GFS2_LKS_SRTTB]) { + a_srttb += st->stats[GFS2_LKS_SRTTB]; + nonzero++; + } + } + st = &this_cpu_ptr(sdp->sd_lkstats)->lkstats[LM_TYPE_RGRP]; + if (nonzero) + do_div(a_srttb, nonzero); + r_dcount = st->stats[GFS2_LKS_DCOUNT]; + var = st->stats[GFS2_LKS_SRTTVARB] + + gl->gl_stats.stats[GFS2_LKS_SRTTVARB]; + preempt_enable(); + + l_srttb = gl->gl_stats.stats[GFS2_LKS_SRTTB]; + l_dcount = gl->gl_stats.stats[GFS2_LKS_DCOUNT]; + + if ((l_dcount < 1) || (r_dcount < 1) || (a_srttb == 0)) + return false; + + srttb_diff = a_srttb - l_srttb; + sqr_diff = srttb_diff * srttb_diff; + + var *= 2; + if (l_dcount < 8 || r_dcount < 8) + var *= 2; + if (loops == 1) + var *= 2; + + return ((srttb_diff < 0) && (sqr_diff > var)); +} + +/** + * gfs2_rgrp_used_recently - test if an rgrp has been used recently + * @rs: The block reservation with the rgrp to test + * @msecs: The time limit in milliseconds + * + * Returns: True if the rgrp glock has been used within the time limit + */ +static bool gfs2_rgrp_used_recently(const struct gfs2_blkreserv *rs, + u64 msecs) +{ + u64 tdiff; + + tdiff = ktime_to_ns(ktime_sub(ktime_get_real(), + rs->rs_rgd->rd_gl->gl_dstamp)); + + return tdiff > (msecs * 1000 * 1000); +} + +static u32 gfs2_orlov_skip(const struct gfs2_inode *ip) +{ + const struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + + return get_random_u32() % sdp->sd_rgrps; +} + +static bool gfs2_select_rgrp(struct gfs2_rgrpd **pos, const struct gfs2_rgrpd *begin) +{ + struct gfs2_rgrpd *rgd = *pos; + struct gfs2_sbd *sdp = rgd->rd_sbd; + + rgd = gfs2_rgrpd_get_next(rgd); + if (rgd == NULL) + rgd = gfs2_rgrpd_get_first(sdp); + *pos = rgd; + if (rgd != begin) /* If we didn't wrap */ + return true; + return false; +} + +/** + * fast_to_acquire - determine if a resource group will be fast to acquire + * @rgd: The rgrp + * + * If this is one of our preferred rgrps, it should be quicker to acquire, + * because we tried to set ourselves up as dlm lock master. + */ +static inline int fast_to_acquire(struct gfs2_rgrpd *rgd) +{ + struct gfs2_glock *gl = rgd->rd_gl; + + if (gl->gl_state != LM_ST_UNLOCKED && list_empty(&gl->gl_holders) && + !test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags) && + !test_bit(GLF_DEMOTE, &gl->gl_flags)) + return 1; + if (rgd->rd_flags & GFS2_RDF_PREFERRED) + return 1; + return 0; +} + +/** + * gfs2_inplace_reserve - Reserve space in the filesystem + * @ip: the inode to reserve space for + * @ap: the allocation parameters + * + * We try our best to find an rgrp that has at least ap->target blocks + * available. After a couple of passes (loops == 2), the prospects of finding + * such an rgrp diminish. At this stage, we return the first rgrp that has + * at least ap->min_target blocks available. + * + * Returns: 0 on success, + * -ENOMEM if a suitable rgrp can't be found + * errno otherwise + */ + +int gfs2_inplace_reserve(struct gfs2_inode *ip, struct gfs2_alloc_parms *ap) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + struct gfs2_rgrpd *begin = NULL; + struct gfs2_blkreserv *rs = &ip->i_res; + int error = 0, flags = LM_FLAG_NODE_SCOPE; + bool rg_locked; + u64 last_unlinked = NO_BLOCK; + u32 target = ap->target; + int loops = 0; + u32 free_blocks, blocks_available, skip = 0; + + BUG_ON(rs->rs_reserved); + + if (sdp->sd_args.ar_rgrplvb) + flags |= GL_SKIP; + if (gfs2_assert_warn(sdp, target)) + return -EINVAL; + if (gfs2_rs_active(rs)) { + begin = rs->rs_rgd; + } else if (rs->rs_rgd && + rgrp_contains_block(rs->rs_rgd, ip->i_goal)) { + begin = rs->rs_rgd; + } else { + check_and_update_goal(ip); + rs->rs_rgd = begin = gfs2_blk2rgrpd(sdp, ip->i_goal, 1); + } + if (S_ISDIR(ip->i_inode.i_mode) && (ap->aflags & GFS2_AF_ORLOV)) + skip = gfs2_orlov_skip(ip); + if (rs->rs_rgd == NULL) + return -EBADSLT; + + while (loops < 3) { + struct gfs2_rgrpd *rgd; + + rg_locked = gfs2_glock_is_locked_by_me(rs->rs_rgd->rd_gl); + if (rg_locked) { + rgrp_lock_local(rs->rs_rgd); + } else { + if (skip && skip--) + goto next_rgrp; + if (!gfs2_rs_active(rs)) { + if (loops == 0 && + !fast_to_acquire(rs->rs_rgd)) + goto next_rgrp; + if ((loops < 2) && + gfs2_rgrp_used_recently(rs, 1000) && + gfs2_rgrp_congested(rs->rs_rgd, loops)) + goto next_rgrp; + } + error = gfs2_glock_nq_init(rs->rs_rgd->rd_gl, + LM_ST_EXCLUSIVE, flags, + &ip->i_rgd_gh); + if (unlikely(error)) + return error; + rgrp_lock_local(rs->rs_rgd); + if (!gfs2_rs_active(rs) && (loops < 2) && + gfs2_rgrp_congested(rs->rs_rgd, loops)) + goto skip_rgrp; + if (sdp->sd_args.ar_rgrplvb) { + error = update_rgrp_lvb(rs->rs_rgd, + &ip->i_rgd_gh); + if (unlikely(error)) { + rgrp_unlock_local(rs->rs_rgd); + gfs2_glock_dq_uninit(&ip->i_rgd_gh); + return error; + } + } + } + + /* Skip unusable resource groups */ + if ((rs->rs_rgd->rd_flags & (GFS2_RGF_NOALLOC | + GFS2_RDF_ERROR)) || + (loops == 0 && target > rs->rs_rgd->rd_extfail_pt)) + goto skip_rgrp; + + if (sdp->sd_args.ar_rgrplvb) { + error = gfs2_instantiate(&ip->i_rgd_gh); + if (error) + goto skip_rgrp; + } + + /* Get a reservation if we don't already have one */ + if (!gfs2_rs_active(rs)) + rg_mblk_search(rs->rs_rgd, ip, ap); + + /* Skip rgrps when we can't get a reservation on first pass */ + if (!gfs2_rs_active(rs) && (loops < 1)) + goto check_rgrp; + + /* If rgrp has enough free space, use it */ + rgd = rs->rs_rgd; + spin_lock(&rgd->rd_rsspin); + free_blocks = rgd_free(rgd, rs); + blocks_available = rgd->rd_free_clone - rgd->rd_reserved; + if (free_blocks < target || blocks_available < target) { + spin_unlock(&rgd->rd_rsspin); + goto check_rgrp; + } + rs->rs_reserved = ap->target; + if (rs->rs_reserved > blocks_available) + rs->rs_reserved = blocks_available; + rgd->rd_reserved += rs->rs_reserved; + spin_unlock(&rgd->rd_rsspin); + rgrp_unlock_local(rs->rs_rgd); + return 0; +check_rgrp: + /* Check for unlinked inodes which can be reclaimed */ + if (rs->rs_rgd->rd_flags & GFS2_RDF_CHECK) + try_rgrp_unlink(rs->rs_rgd, &last_unlinked, + ip->i_no_addr); +skip_rgrp: + rgrp_unlock_local(rs->rs_rgd); + + /* Drop reservation, if we couldn't use reserved rgrp */ + if (gfs2_rs_active(rs)) + gfs2_rs_deltree(rs); + + /* Unlock rgrp if required */ + if (!rg_locked) + gfs2_glock_dq_uninit(&ip->i_rgd_gh); +next_rgrp: + /* Find the next rgrp, and continue looking */ + if (gfs2_select_rgrp(&rs->rs_rgd, begin)) + continue; + if (skip) + continue; + + /* If we've scanned all the rgrps, but found no free blocks + * then this checks for some less likely conditions before + * trying again. + */ + loops++; + /* Check that fs hasn't grown if writing to rindex */ + if (ip == GFS2_I(sdp->sd_rindex) && !sdp->sd_rindex_uptodate) { + error = gfs2_ri_update(ip); + if (error) + return error; + } + /* Flushing the log may release space */ + if (loops == 2) { + if (ap->min_target) + target = ap->min_target; + gfs2_log_flush(sdp, NULL, GFS2_LOG_HEAD_FLUSH_NORMAL | + GFS2_LFC_INPLACE_RESERVE); + } + } + + return -ENOSPC; +} + +/** + * gfs2_inplace_release - release an inplace reservation + * @ip: the inode the reservation was taken out on + * + * Release a reservation made by gfs2_inplace_reserve(). + */ + +void gfs2_inplace_release(struct gfs2_inode *ip) +{ + struct gfs2_blkreserv *rs = &ip->i_res; + + if (rs->rs_reserved) { + struct gfs2_rgrpd *rgd = rs->rs_rgd; + + spin_lock(&rgd->rd_rsspin); + GLOCK_BUG_ON(rgd->rd_gl, rgd->rd_reserved < rs->rs_reserved); + rgd->rd_reserved -= rs->rs_reserved; + spin_unlock(&rgd->rd_rsspin); + rs->rs_reserved = 0; + } + if (gfs2_holder_initialized(&ip->i_rgd_gh)) + gfs2_glock_dq_uninit(&ip->i_rgd_gh); +} + +/** + * gfs2_alloc_extent - allocate an extent from a given bitmap + * @rbm: the resource group information + * @dinode: TRUE if the first block we allocate is for a dinode + * @n: The extent length (value/result) + * + * Add the bitmap buffer to the transaction. + * Set the found bits to @new_state to change block's allocation state. + */ +static void gfs2_alloc_extent(const struct gfs2_rbm *rbm, bool dinode, + unsigned int *n) +{ + struct gfs2_rbm pos = { .rgd = rbm->rgd, }; + const unsigned int elen = *n; + u64 block; + int ret; + + *n = 1; + block = gfs2_rbm_to_block(rbm); + gfs2_trans_add_meta(rbm->rgd->rd_gl, rbm_bi(rbm)->bi_bh); + gfs2_setbit(rbm, true, dinode ? GFS2_BLKST_DINODE : GFS2_BLKST_USED); + block++; + while (*n < elen) { + ret = gfs2_rbm_from_block(&pos, block); + if (ret || gfs2_testbit(&pos, true) != GFS2_BLKST_FREE) + break; + gfs2_trans_add_meta(pos.rgd->rd_gl, rbm_bi(&pos)->bi_bh); + gfs2_setbit(&pos, true, GFS2_BLKST_USED); + (*n)++; + block++; + } +} + +/** + * rgblk_free - Change alloc state of given block(s) + * @sdp: the filesystem + * @rgd: the resource group the blocks are in + * @bstart: the start of a run of blocks to free + * @blen: the length of the block run (all must lie within ONE RG!) + * @new_state: GFS2_BLKST_XXX the after-allocation block state + */ + +static void rgblk_free(struct gfs2_sbd *sdp, struct gfs2_rgrpd *rgd, + u64 bstart, u32 blen, unsigned char new_state) +{ + struct gfs2_rbm rbm; + struct gfs2_bitmap *bi, *bi_prev = NULL; + + rbm.rgd = rgd; + if (WARN_ON_ONCE(gfs2_rbm_from_block(&rbm, bstart))) + return; + while (blen--) { + bi = rbm_bi(&rbm); + if (bi != bi_prev) { + if (!bi->bi_clone) { + bi->bi_clone = kmalloc(bi->bi_bh->b_size, + GFP_NOFS | __GFP_NOFAIL); + memcpy(bi->bi_clone + bi->bi_offset, + bi->bi_bh->b_data + bi->bi_offset, + bi->bi_bytes); + } + gfs2_trans_add_meta(rbm.rgd->rd_gl, bi->bi_bh); + bi_prev = bi; + } + gfs2_setbit(&rbm, false, new_state); + gfs2_rbm_add(&rbm, 1); + } +} + +/** + * gfs2_rgrp_dump - print out an rgrp + * @seq: The iterator + * @rgd: The rgrp in question + * @fs_id_buf: pointer to file system id (if requested) + * + */ + +void gfs2_rgrp_dump(struct seq_file *seq, struct gfs2_rgrpd *rgd, + const char *fs_id_buf) +{ + struct gfs2_blkreserv *trs; + const struct rb_node *n; + + spin_lock(&rgd->rd_rsspin); + gfs2_print_dbg(seq, "%s R: n:%llu f:%02x b:%u/%u i:%u q:%u r:%u e:%u\n", + fs_id_buf, + (unsigned long long)rgd->rd_addr, rgd->rd_flags, + rgd->rd_free, rgd->rd_free_clone, rgd->rd_dinodes, + rgd->rd_requested, rgd->rd_reserved, rgd->rd_extfail_pt); + if (rgd->rd_sbd->sd_args.ar_rgrplvb && rgd->rd_rgl) { + struct gfs2_rgrp_lvb *rgl = rgd->rd_rgl; + + gfs2_print_dbg(seq, "%s L: f:%02x b:%u i:%u\n", fs_id_buf, + be32_to_cpu(rgl->rl_flags), + be32_to_cpu(rgl->rl_free), + be32_to_cpu(rgl->rl_dinodes)); + } + for (n = rb_first(&rgd->rd_rstree); n; n = rb_next(&trs->rs_node)) { + trs = rb_entry(n, struct gfs2_blkreserv, rs_node); + dump_rs(seq, trs, fs_id_buf); + } + spin_unlock(&rgd->rd_rsspin); +} + +static void gfs2_rgrp_error(struct gfs2_rgrpd *rgd) +{ + struct gfs2_sbd *sdp = rgd->rd_sbd; + char fs_id_buf[sizeof(sdp->sd_fsname) + 7]; + + fs_warn(sdp, "rgrp %llu has an error, marking it readonly until umount\n", + (unsigned long long)rgd->rd_addr); + fs_warn(sdp, "umount on all nodes and run fsck.gfs2 to fix the error\n"); + sprintf(fs_id_buf, "fsid=%s: ", sdp->sd_fsname); + gfs2_rgrp_dump(NULL, rgd, fs_id_buf); + rgd->rd_flags |= GFS2_RDF_ERROR; +} + +/** + * gfs2_adjust_reservation - Adjust (or remove) a reservation after allocation + * @ip: The inode we have just allocated blocks for + * @rbm: The start of the allocated blocks + * @len: The extent length + * + * Adjusts a reservation after an allocation has taken place. If the + * reservation does not match the allocation, or if it is now empty + * then it is removed. + */ + +static void gfs2_adjust_reservation(struct gfs2_inode *ip, + const struct gfs2_rbm *rbm, unsigned len) +{ + struct gfs2_blkreserv *rs = &ip->i_res; + struct gfs2_rgrpd *rgd = rbm->rgd; + + BUG_ON(rs->rs_reserved < len); + rs->rs_reserved -= len; + if (gfs2_rs_active(rs)) { + u64 start = gfs2_rbm_to_block(rbm); + + if (rs->rs_start == start) { + unsigned int rlen; + + rs->rs_start += len; + rlen = min(rs->rs_requested, len); + rs->rs_requested -= rlen; + rgd->rd_requested -= rlen; + trace_gfs2_rs(rs, TRACE_RS_CLAIM); + if (rs->rs_start < rgd->rd_data0 + rgd->rd_data && + rs->rs_requested) + return; + /* We used up our block reservation, so we should + reserve more blocks next time. */ + atomic_add(RGRP_RSRV_ADDBLKS, &ip->i_sizehint); + } + __rs_deltree(rs); + } +} + +/** + * gfs2_set_alloc_start - Set starting point for block allocation + * @rbm: The rbm which will be set to the required location + * @ip: The gfs2 inode + * @dinode: Flag to say if allocation includes a new inode + * + * This sets the starting point from the reservation if one is active + * otherwise it falls back to guessing a start point based on the + * inode's goal block or the last allocation point in the rgrp. + */ + +static void gfs2_set_alloc_start(struct gfs2_rbm *rbm, + const struct gfs2_inode *ip, bool dinode) +{ + u64 goal; + + if (gfs2_rs_active(&ip->i_res)) { + goal = ip->i_res.rs_start; + } else { + if (!dinode && rgrp_contains_block(rbm->rgd, ip->i_goal)) + goal = ip->i_goal; + else + goal = rbm->rgd->rd_last_alloc + rbm->rgd->rd_data0; + } + if (WARN_ON_ONCE(gfs2_rbm_from_block(rbm, goal))) { + rbm->bii = 0; + rbm->offset = 0; + } +} + +/** + * gfs2_alloc_blocks - Allocate one or more blocks of data and/or a dinode + * @ip: the inode to allocate the block for + * @bn: Used to return the starting block number + * @nblocks: requested number of blocks/extent length (value/result) + * @dinode: 1 if we're allocating a dinode block, else 0 + * + * Returns: 0 or error + */ + +int gfs2_alloc_blocks(struct gfs2_inode *ip, u64 *bn, unsigned int *nblocks, + bool dinode) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + struct buffer_head *dibh; + struct gfs2_rbm rbm = { .rgd = ip->i_res.rs_rgd, }; + u64 block; /* block, within the file system scope */ + u32 minext = 1; + int error = -ENOSPC; + + BUG_ON(ip->i_res.rs_reserved < *nblocks); + + rgrp_lock_local(rbm.rgd); + if (gfs2_rs_active(&ip->i_res)) { + gfs2_set_alloc_start(&rbm, ip, dinode); + error = gfs2_rbm_find(&rbm, GFS2_BLKST_FREE, &minext, &ip->i_res, false); + } + if (error == -ENOSPC) { + gfs2_set_alloc_start(&rbm, ip, dinode); + error = gfs2_rbm_find(&rbm, GFS2_BLKST_FREE, &minext, NULL, false); + } + + /* Since all blocks are reserved in advance, this shouldn't happen */ + if (error) { + fs_warn(sdp, "inum=%llu error=%d, nblocks=%u, full=%d fail_pt=%d\n", + (unsigned long long)ip->i_no_addr, error, *nblocks, + test_bit(GBF_FULL, &rbm.rgd->rd_bits->bi_flags), + rbm.rgd->rd_extfail_pt); + goto rgrp_error; + } + + gfs2_alloc_extent(&rbm, dinode, nblocks); + block = gfs2_rbm_to_block(&rbm); + rbm.rgd->rd_last_alloc = block - rbm.rgd->rd_data0; + if (!dinode) { + ip->i_goal = block + *nblocks - 1; + error = gfs2_meta_inode_buffer(ip, &dibh); + if (error == 0) { + struct gfs2_dinode *di = + (struct gfs2_dinode *)dibh->b_data; + gfs2_trans_add_meta(ip->i_gl, dibh); + di->di_goal_meta = di->di_goal_data = + cpu_to_be64(ip->i_goal); + brelse(dibh); + } + } + spin_lock(&rbm.rgd->rd_rsspin); + gfs2_adjust_reservation(ip, &rbm, *nblocks); + if (rbm.rgd->rd_free < *nblocks || rbm.rgd->rd_reserved < *nblocks) { + fs_warn(sdp, "nblocks=%u\n", *nblocks); + spin_unlock(&rbm.rgd->rd_rsspin); + goto rgrp_error; + } + GLOCK_BUG_ON(rbm.rgd->rd_gl, rbm.rgd->rd_reserved < *nblocks); + GLOCK_BUG_ON(rbm.rgd->rd_gl, rbm.rgd->rd_free_clone < *nblocks); + GLOCK_BUG_ON(rbm.rgd->rd_gl, rbm.rgd->rd_free < *nblocks); + rbm.rgd->rd_reserved -= *nblocks; + rbm.rgd->rd_free_clone -= *nblocks; + rbm.rgd->rd_free -= *nblocks; + spin_unlock(&rbm.rgd->rd_rsspin); + if (dinode) { + u64 generation; + + rbm.rgd->rd_dinodes++; + generation = rbm.rgd->rd_igeneration++; + if (generation == 0) + generation = rbm.rgd->rd_igeneration++; + ip->i_generation = generation; + } + + gfs2_trans_add_meta(rbm.rgd->rd_gl, rbm.rgd->rd_bits[0].bi_bh); + gfs2_rgrp_out(rbm.rgd, rbm.rgd->rd_bits[0].bi_bh->b_data); + rgrp_unlock_local(rbm.rgd); + + gfs2_statfs_change(sdp, 0, -(s64)*nblocks, dinode ? 1 : 0); + if (dinode) + gfs2_trans_remove_revoke(sdp, block, *nblocks); + + gfs2_quota_change(ip, *nblocks, ip->i_inode.i_uid, ip->i_inode.i_gid); + + trace_gfs2_block_alloc(ip, rbm.rgd, block, *nblocks, + dinode ? GFS2_BLKST_DINODE : GFS2_BLKST_USED); + *bn = block; + return 0; + +rgrp_error: + rgrp_unlock_local(rbm.rgd); + gfs2_rgrp_error(rbm.rgd); + return -EIO; +} + +/** + * __gfs2_free_blocks - free a contiguous run of block(s) + * @ip: the inode these blocks are being freed from + * @rgd: the resource group the blocks are in + * @bstart: first block of a run of contiguous blocks + * @blen: the length of the block run + * @meta: 1 if the blocks represent metadata + * + */ + +void __gfs2_free_blocks(struct gfs2_inode *ip, struct gfs2_rgrpd *rgd, + u64 bstart, u32 blen, int meta) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + + rgrp_lock_local(rgd); + rgblk_free(sdp, rgd, bstart, blen, GFS2_BLKST_FREE); + trace_gfs2_block_alloc(ip, rgd, bstart, blen, GFS2_BLKST_FREE); + rgd->rd_free += blen; + rgd->rd_flags &= ~GFS2_RGF_TRIMMED; + gfs2_trans_add_meta(rgd->rd_gl, rgd->rd_bits[0].bi_bh); + gfs2_rgrp_out(rgd, rgd->rd_bits[0].bi_bh->b_data); + rgrp_unlock_local(rgd); + + /* Directories keep their data in the metadata address space */ + if (meta || (ip->i_diskflags & GFS2_DIF_EXHASH) || gfs2_is_jdata(ip)) + gfs2_journal_wipe(ip, bstart, blen); +} + +/** + * gfs2_free_meta - free a contiguous run of data block(s) + * @ip: the inode these blocks are being freed from + * @rgd: the resource group the blocks are in + * @bstart: first block of a run of contiguous blocks + * @blen: the length of the block run + * + */ + +void gfs2_free_meta(struct gfs2_inode *ip, struct gfs2_rgrpd *rgd, + u64 bstart, u32 blen) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + + __gfs2_free_blocks(ip, rgd, bstart, blen, 1); + gfs2_statfs_change(sdp, 0, +blen, 0); + gfs2_quota_change(ip, -(s64)blen, ip->i_inode.i_uid, ip->i_inode.i_gid); +} + +void gfs2_unlink_di(struct inode *inode) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + struct gfs2_rgrpd *rgd; + u64 blkno = ip->i_no_addr; + + rgd = gfs2_blk2rgrpd(sdp, blkno, true); + if (!rgd) + return; + rgrp_lock_local(rgd); + rgblk_free(sdp, rgd, blkno, 1, GFS2_BLKST_UNLINKED); + trace_gfs2_block_alloc(ip, rgd, blkno, 1, GFS2_BLKST_UNLINKED); + gfs2_trans_add_meta(rgd->rd_gl, rgd->rd_bits[0].bi_bh); + gfs2_rgrp_out(rgd, rgd->rd_bits[0].bi_bh->b_data); + be32_add_cpu(&rgd->rd_rgl->rl_unlinked, 1); + rgrp_unlock_local(rgd); +} + +void gfs2_free_di(struct gfs2_rgrpd *rgd, struct gfs2_inode *ip) +{ + struct gfs2_sbd *sdp = rgd->rd_sbd; + + rgrp_lock_local(rgd); + rgblk_free(sdp, rgd, ip->i_no_addr, 1, GFS2_BLKST_FREE); + if (!rgd->rd_dinodes) + gfs2_consist_rgrpd(rgd); + rgd->rd_dinodes--; + rgd->rd_free++; + + gfs2_trans_add_meta(rgd->rd_gl, rgd->rd_bits[0].bi_bh); + gfs2_rgrp_out(rgd, rgd->rd_bits[0].bi_bh->b_data); + be32_add_cpu(&rgd->rd_rgl->rl_unlinked, -1); + rgrp_unlock_local(rgd); + + gfs2_statfs_change(sdp, 0, +1, -1); + trace_gfs2_block_alloc(ip, rgd, ip->i_no_addr, 1, GFS2_BLKST_FREE); + gfs2_quota_change(ip, -1, ip->i_inode.i_uid, ip->i_inode.i_gid); + gfs2_journal_wipe(ip, ip->i_no_addr, 1); +} + +/** + * gfs2_check_blk_type - Check the type of a block + * @sdp: The superblock + * @no_addr: The block number to check + * @type: The block type we are looking for + * + * The inode glock of @no_addr must be held. The @type to check for is either + * GFS2_BLKST_DINODE or GFS2_BLKST_UNLINKED; checking for type GFS2_BLKST_FREE + * or GFS2_BLKST_USED would make no sense. + * + * Returns: 0 if the block type matches the expected type + * -ESTALE if it doesn't match + * or -ve errno if something went wrong while checking + */ + +int gfs2_check_blk_type(struct gfs2_sbd *sdp, u64 no_addr, unsigned int type) +{ + struct gfs2_rgrpd *rgd; + struct gfs2_holder rgd_gh; + struct gfs2_rbm rbm; + int error = -EINVAL; + + rgd = gfs2_blk2rgrpd(sdp, no_addr, 1); + if (!rgd) + goto fail; + + error = gfs2_glock_nq_init(rgd->rd_gl, LM_ST_SHARED, 0, &rgd_gh); + if (error) + goto fail; + + rbm.rgd = rgd; + error = gfs2_rbm_from_block(&rbm, no_addr); + if (!WARN_ON_ONCE(error)) { + /* + * No need to take the local resource group lock here; the + * inode glock of @no_addr provides the necessary + * synchronization in case the block is an inode. (In case + * the block is not an inode, the block type will not match + * the @type we are looking for.) + */ + if (gfs2_testbit(&rbm, false) != type) + error = -ESTALE; + } + + gfs2_glock_dq_uninit(&rgd_gh); + +fail: + return error; +} + +/** + * gfs2_rlist_add - add a RG to a list of RGs + * @ip: the inode + * @rlist: the list of resource groups + * @block: the block + * + * Figure out what RG a block belongs to and add that RG to the list + * + * FIXME: Don't use NOFAIL + * + */ + +void gfs2_rlist_add(struct gfs2_inode *ip, struct gfs2_rgrp_list *rlist, + u64 block) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + struct gfs2_rgrpd *rgd; + struct gfs2_rgrpd **tmp; + unsigned int new_space; + unsigned int x; + + if (gfs2_assert_warn(sdp, !rlist->rl_ghs)) + return; + + /* + * The resource group last accessed is kept in the last position. + */ + + if (rlist->rl_rgrps) { + rgd = rlist->rl_rgd[rlist->rl_rgrps - 1]; + if (rgrp_contains_block(rgd, block)) + return; + rgd = gfs2_blk2rgrpd(sdp, block, 1); + } else { + rgd = ip->i_res.rs_rgd; + if (!rgd || !rgrp_contains_block(rgd, block)) + rgd = gfs2_blk2rgrpd(sdp, block, 1); + } + + if (!rgd) { + fs_err(sdp, "rlist_add: no rgrp for block %llu\n", + (unsigned long long)block); + return; + } + + for (x = 0; x < rlist->rl_rgrps; x++) { + if (rlist->rl_rgd[x] == rgd) { + swap(rlist->rl_rgd[x], + rlist->rl_rgd[rlist->rl_rgrps - 1]); + return; + } + } + + if (rlist->rl_rgrps == rlist->rl_space) { + new_space = rlist->rl_space + 10; + + tmp = kzalloc_objs(struct gfs2_rgrpd *, new_space, + GFP_NOFS | __GFP_NOFAIL); + + if (rlist->rl_rgd) { + memcpy(tmp, rlist->rl_rgd, + rlist->rl_space * sizeof(struct gfs2_rgrpd *)); + kfree(rlist->rl_rgd); + } + + rlist->rl_space = new_space; + rlist->rl_rgd = tmp; + } + + rlist->rl_rgd[rlist->rl_rgrps++] = rgd; +} + +/** + * gfs2_rlist_alloc - all RGs have been added to the rlist, now allocate + * and initialize an array of glock holders for them + * @rlist: the list of resource groups + * @state: the state we're requesting + * @flags: the modifier flags + * + * FIXME: Don't use NOFAIL + * + */ + +void gfs2_rlist_alloc(struct gfs2_rgrp_list *rlist, + unsigned int state, u16 flags) +{ + unsigned int x; + + rlist->rl_ghs = kmalloc_objs(struct gfs2_holder, rlist->rl_rgrps, + GFP_NOFS | __GFP_NOFAIL); + for (x = 0; x < rlist->rl_rgrps; x++) + gfs2_holder_init(rlist->rl_rgd[x]->rd_gl, state, flags, + &rlist->rl_ghs[x]); +} + +/** + * gfs2_rlist_free - free a resource group list + * @rlist: the list of resource groups + * + */ + +void gfs2_rlist_free(struct gfs2_rgrp_list *rlist) +{ + unsigned int x; + + kfree(rlist->rl_rgd); + + if (rlist->rl_ghs) { + for (x = 0; x < rlist->rl_rgrps; x++) + gfs2_holder_uninit(&rlist->rl_ghs[x]); + kfree(rlist->rl_ghs); + rlist->rl_ghs = NULL; + } +} + +void rgrp_lock_local(struct gfs2_rgrpd *rgd) +{ + mutex_lock(&rgd->rd_mutex); +} + +void rgrp_unlock_local(struct gfs2_rgrpd *rgd) +{ + mutex_unlock(&rgd->rd_mutex); +} diff --git a/fs/gfs2/rgrp.h b/fs/gfs2/rgrp.h new file mode 100644 index 000000000..8d20e9938 --- /dev/null +++ b/fs/gfs2/rgrp.h @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. + */ + +#ifndef __RGRP_DOT_H__ +#define __RGRP_DOT_H__ + +#include +#include + +/* Since each block in the file system is represented by two bits in the + * bitmap, one 64-bit word in the bitmap will represent 32 blocks. + * By reserving 32 blocks at a time, we can optimize / shortcut how we search + * through the bitmaps by looking a word at a time. + */ +#define RGRP_RSRV_MINBLKS 32 +#define RGRP_RSRV_ADDBLKS 64 + +struct gfs2_rgrpd; +struct gfs2_sbd; +struct gfs2_holder; + +void gfs2_rgrp_verify(struct gfs2_rgrpd *rgd); + +struct gfs2_rgrpd *gfs2_blk2rgrpd(struct gfs2_sbd *sdp, u64 blk, bool exact); +struct gfs2_rgrpd *gfs2_rgrpd_get_first(struct gfs2_sbd *sdp); +struct gfs2_rgrpd *gfs2_rgrpd_get_next(struct gfs2_rgrpd *rgd); + +void gfs2_clear_rgrpd(struct gfs2_sbd *sdp); +int gfs2_rindex_update(struct gfs2_sbd *sdp); +void gfs2_free_clones(struct gfs2_rgrpd *rgd); +int gfs2_rgrp_go_instantiate(struct gfs2_glock *gl); +void gfs2_rgrp_brelse(struct gfs2_rgrpd *rgd); + +struct gfs2_alloc *gfs2_alloc_get(struct gfs2_inode *ip); + +#define GFS2_AF_ORLOV 1 +int gfs2_inplace_reserve(struct gfs2_inode *ip, + struct gfs2_alloc_parms *ap); +void gfs2_inplace_release(struct gfs2_inode *ip); + +int gfs2_alloc_blocks(struct gfs2_inode *ip, u64 *bn, unsigned int *n, + bool dinode); + +void gfs2_rs_deltree(struct gfs2_blkreserv *rs); +void gfs2_rs_delete(struct gfs2_inode *ip); +void __gfs2_free_blocks(struct gfs2_inode *ip, struct gfs2_rgrpd *rgd, + u64 bstart, u32 blen, int meta); +void gfs2_free_meta(struct gfs2_inode *ip, struct gfs2_rgrpd *rgd, + u64 bstart, u32 blen); +void gfs2_free_di(struct gfs2_rgrpd *rgd, struct gfs2_inode *ip); +void gfs2_unlink_di(struct inode *inode); +int gfs2_check_blk_type(struct gfs2_sbd *sdp, u64 no_addr, + unsigned int type); + +struct gfs2_rgrp_list { + unsigned int rl_rgrps; + unsigned int rl_space; + struct gfs2_rgrpd **rl_rgd; + struct gfs2_holder *rl_ghs; +}; + +void gfs2_rlist_add(struct gfs2_inode *ip, struct gfs2_rgrp_list *rlist, + u64 block); +void gfs2_rlist_alloc(struct gfs2_rgrp_list *rlist, + unsigned int state, u16 flags); +void gfs2_rlist_free(struct gfs2_rgrp_list *rlist); +u64 gfs2_ri_total(struct gfs2_sbd *sdp); +void gfs2_rgrp_dump(struct seq_file *seq, struct gfs2_rgrpd *rgd, + const char *fs_id_buf); +int gfs2_rgrp_send_discards(struct gfs2_sbd *sdp, u64 offset, + struct buffer_head *bh, + const struct gfs2_bitmap *bi, unsigned minlen, + u64 *ptrimmed); +int gfs2_fitrim(struct file *filp, void __user *argp); + +/* This is how to tell if a reservation is in the rgrp tree: */ +static inline bool gfs2_rs_active(const struct gfs2_blkreserv *rs) +{ + return !RB_EMPTY_NODE(&rs->rs_node); +} + +static inline int rgrp_contains_block(struct gfs2_rgrpd *rgd, u64 block) +{ + u64 first = rgd->rd_data0; + u64 last = first + rgd->rd_data; + return first <= block && block < last; +} + +void check_and_update_goal(struct gfs2_inode *ip); + +void rgrp_lock_local(struct gfs2_rgrpd *rgd); +void rgrp_unlock_local(struct gfs2_rgrpd *rgd); + +#endif /* __RGRP_DOT_H__ */ diff --git a/fs/gfs2/super.c b/fs/gfs2/super.c new file mode 100644 index 000000000..06302c293 --- /dev/null +++ b/fs/gfs2/super.c @@ -0,0 +1,1584 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2007 Red Hat, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gfs2.h" +#include "incore.h" +#include "bmap.h" +#include "dir.h" +#include "glock.h" +#include "glops.h" +#include "inode.h" +#include "log.h" +#include "meta_io.h" +#include "quota.h" +#include "recovery.h" +#include "rgrp.h" +#include "super.h" +#include "trans.h" +#include "util.h" +#include "sys.h" +#include "xattr.h" +#include "lops.h" + +enum evict_behavior { + EVICT_SHOULD_DELETE, + EVICT_SHOULD_SKIP_DELETE, + EVICT_SHOULD_DEFER_DELETE, +}; + +/** + * gfs2_jindex_free - Clear all the journal index information + * @sdp: The GFS2 superblock + * + */ + +void gfs2_jindex_free(struct gfs2_sbd *sdp) +{ + struct list_head list; + struct gfs2_jdesc *jd; + + spin_lock(&sdp->sd_jindex_spin); + list_add(&list, &sdp->sd_jindex_list); + list_del_init(&sdp->sd_jindex_list); + sdp->sd_journals = 0; + spin_unlock(&sdp->sd_jindex_spin); + + down_write(&sdp->sd_log_flush_lock); + sdp->sd_jdesc = NULL; + up_write(&sdp->sd_log_flush_lock); + + while (!list_empty(&list)) { + jd = list_first_entry(&list, struct gfs2_jdesc, jd_list); + BUG_ON(jd->jd_log_bio); + gfs2_free_journal_extents(jd); + list_del(&jd->jd_list); + iput(jd->jd_inode); + jd->jd_inode = NULL; + kfree(jd); + } +} + +static struct gfs2_jdesc *jdesc_find_i(struct list_head *head, unsigned int jid) +{ + struct gfs2_jdesc *jd; + + list_for_each_entry(jd, head, jd_list) { + if (jd->jd_jid == jid) + return jd; + } + return NULL; +} + +struct gfs2_jdesc *gfs2_jdesc_find(struct gfs2_sbd *sdp, unsigned int jid) +{ + struct gfs2_jdesc *jd; + + spin_lock(&sdp->sd_jindex_spin); + jd = jdesc_find_i(&sdp->sd_jindex_list, jid); + spin_unlock(&sdp->sd_jindex_spin); + + return jd; +} + +int gfs2_jdesc_check(struct gfs2_jdesc *jd) +{ + struct gfs2_inode *ip = GFS2_I(jd->jd_inode); + struct gfs2_sbd *sdp = GFS2_SB(jd->jd_inode); + u64 size = i_size_read(jd->jd_inode); + + if (gfs2_check_internal_file_size(jd->jd_inode, 8 << 20, BIT(30))) + return -EIO; + + jd->jd_blocks = size >> sdp->sd_sb.sb_bsize_shift; + + if (gfs2_write_alloc_required(ip, 0, size)) { + gfs2_consist_inode(ip); + return -EIO; + } + + return 0; +} + +/** + * gfs2_make_fs_rw - Turn a Read-Only FS into a Read-Write one + * @sdp: the filesystem + * + * Returns: errno + */ + +int gfs2_make_fs_rw(struct gfs2_sbd *sdp) +{ + struct gfs2_inode *ip = GFS2_I(sdp->sd_jdesc->jd_inode); + struct gfs2_glock *j_gl = ip->i_gl; + int error; + + j_gl->gl_ops->go_inval(j_gl, DIO_METADATA); + if (gfs2_withdrawn(sdp)) + return -EIO; + + if (sdp->sd_log_sequence == 0) { + fs_err(sdp, "unknown status of our own journal jid %d", + sdp->sd_lockstruct.ls_jid); + return -EIO; + } + + error = gfs2_quota_init(sdp); + if (!error && gfs2_withdrawn(sdp)) { + gfs2_quota_cleanup(sdp); + error = -EIO; + } + if (!error) + set_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags); + return error; +} + +void gfs2_statfs_change_in(struct gfs2_statfs_change_host *sc, const void *buf) +{ + const struct gfs2_statfs_change *str = buf; + + sc->sc_total = be64_to_cpu(str->sc_total); + sc->sc_free = be64_to_cpu(str->sc_free); + sc->sc_dinodes = be64_to_cpu(str->sc_dinodes); +} + +void gfs2_statfs_change_out(const struct gfs2_statfs_change_host *sc, void *buf) +{ + struct gfs2_statfs_change *str = buf; + + str->sc_total = cpu_to_be64(sc->sc_total); + str->sc_free = cpu_to_be64(sc->sc_free); + str->sc_dinodes = cpu_to_be64(sc->sc_dinodes); +} + +int gfs2_statfs_init(struct gfs2_sbd *sdp) +{ + struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode); + struct gfs2_statfs_change_host *m_sc = &sdp->sd_statfs_master; + struct gfs2_statfs_change_host *l_sc = &sdp->sd_statfs_local; + struct buffer_head *m_bh; + struct gfs2_holder gh; + int error; + + error = gfs2_glock_nq_init(m_ip->i_gl, LM_ST_EXCLUSIVE, GL_NOCACHE, + &gh); + if (error) + return error; + + error = gfs2_meta_inode_buffer(m_ip, &m_bh); + if (error) + goto out; + + if (sdp->sd_args.ar_spectator) { + spin_lock(&sdp->sd_statfs_spin); + gfs2_statfs_change_in(m_sc, m_bh->b_data + + sizeof(struct gfs2_dinode)); + spin_unlock(&sdp->sd_statfs_spin); + } else { + spin_lock(&sdp->sd_statfs_spin); + gfs2_statfs_change_in(m_sc, m_bh->b_data + + sizeof(struct gfs2_dinode)); + gfs2_statfs_change_in(l_sc, sdp->sd_sc_bh->b_data + + sizeof(struct gfs2_dinode)); + spin_unlock(&sdp->sd_statfs_spin); + + } + + brelse(m_bh); +out: + gfs2_glock_dq_uninit(&gh); + return 0; +} + +void gfs2_statfs_change(struct gfs2_sbd *sdp, s64 total, s64 free, + s64 dinodes) +{ + struct gfs2_inode *l_ip = GFS2_I(sdp->sd_sc_inode); + struct gfs2_statfs_change_host *l_sc = &sdp->sd_statfs_local; + struct gfs2_statfs_change_host *m_sc = &sdp->sd_statfs_master; + s64 x, y; + int need_sync = 0; + + gfs2_trans_add_meta(l_ip->i_gl, sdp->sd_sc_bh); + + spin_lock(&sdp->sd_statfs_spin); + l_sc->sc_total += total; + l_sc->sc_free += free; + l_sc->sc_dinodes += dinodes; + gfs2_statfs_change_out(l_sc, sdp->sd_sc_bh->b_data + + sizeof(struct gfs2_dinode)); + if (sdp->sd_args.ar_statfs_percent) { + x = 100 * l_sc->sc_free; + y = m_sc->sc_free * sdp->sd_args.ar_statfs_percent; + if (x >= y || x <= -y) + need_sync = 1; + } + spin_unlock(&sdp->sd_statfs_spin); + + if (need_sync) + gfs2_wake_up_statfs(sdp); +} + +void update_statfs(struct gfs2_sbd *sdp, struct buffer_head *m_bh) +{ + struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode); + struct gfs2_inode *l_ip = GFS2_I(sdp->sd_sc_inode); + struct gfs2_statfs_change_host *m_sc = &sdp->sd_statfs_master; + struct gfs2_statfs_change_host *l_sc = &sdp->sd_statfs_local; + + gfs2_trans_add_meta(l_ip->i_gl, sdp->sd_sc_bh); + gfs2_trans_add_meta(m_ip->i_gl, m_bh); + + spin_lock(&sdp->sd_statfs_spin); + m_sc->sc_total += l_sc->sc_total; + m_sc->sc_free += l_sc->sc_free; + m_sc->sc_dinodes += l_sc->sc_dinodes; + memset(l_sc, 0, sizeof(struct gfs2_statfs_change)); + memset(sdp->sd_sc_bh->b_data + sizeof(struct gfs2_dinode), + 0, sizeof(struct gfs2_statfs_change)); + gfs2_statfs_change_out(m_sc, m_bh->b_data + sizeof(struct gfs2_dinode)); + spin_unlock(&sdp->sd_statfs_spin); +} + +int gfs2_statfs_sync(struct super_block *sb, int type) +{ + struct gfs2_sbd *sdp = sb->s_fs_info; + struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode); + struct gfs2_statfs_change_host *m_sc = &sdp->sd_statfs_master; + struct gfs2_statfs_change_host *l_sc = &sdp->sd_statfs_local; + struct gfs2_holder gh; + struct buffer_head *m_bh; + int error; + + error = gfs2_glock_nq_init(m_ip->i_gl, LM_ST_EXCLUSIVE, GL_NOCACHE, + &gh); + if (error) + goto out; + + error = gfs2_meta_inode_buffer(m_ip, &m_bh); + if (error) + goto out_unlock; + + spin_lock(&sdp->sd_statfs_spin); + gfs2_statfs_change_in(m_sc, m_bh->b_data + + sizeof(struct gfs2_dinode)); + if (!l_sc->sc_total && !l_sc->sc_free && !l_sc->sc_dinodes) { + spin_unlock(&sdp->sd_statfs_spin); + goto out_bh; + } + spin_unlock(&sdp->sd_statfs_spin); + + error = gfs2_trans_begin(sdp, 2 * RES_DINODE, 0); + if (error) + goto out_bh; + + update_statfs(sdp, m_bh); + sdp->sd_statfs_force_sync = 0; + + gfs2_trans_end(sdp); + +out_bh: + brelse(m_bh); +out_unlock: + gfs2_glock_dq_uninit(&gh); +out: + return error; +} + +struct lfcc { + struct list_head list; + struct gfs2_holder gh; +}; + +/** + * gfs2_lock_fs_check_clean - Stop all writes to the FS and check that all + * journals are clean + * @sdp: the file system + * + * Returns: errno + */ + +static int gfs2_lock_fs_check_clean(struct gfs2_sbd *sdp) +{ + struct gfs2_inode *ip; + struct gfs2_jdesc *jd; + struct lfcc *lfcc; + LIST_HEAD(list); + struct gfs2_log_header_host lh; + int error, error2; + + /* + * Grab all the journal glocks in SH mode. We are *probably* doing + * that to prevent recovery. + */ + + list_for_each_entry(jd, &sdp->sd_jindex_list, jd_list) { + lfcc = kmalloc_obj(struct lfcc); + if (!lfcc) { + error = -ENOMEM; + goto out; + } + ip = GFS2_I(jd->jd_inode); + error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, 0, &lfcc->gh); + if (error) { + kfree(lfcc); + goto out; + } + list_add(&lfcc->list, &list); + } + + gfs2_freeze_unlock(sdp); + + error = gfs2_glock_nq_init(sdp->sd_freeze_gl, LM_ST_EXCLUSIVE, + LM_FLAG_RECOVER | GL_NOPID, + &sdp->sd_freeze_gh); + if (error) + goto relock_shared; + + list_for_each_entry(jd, &sdp->sd_jindex_list, jd_list) { + error = gfs2_jdesc_check(jd); + if (error) + break; + error = gfs2_find_jhead(jd, &lh); + if (error) + break; + if (!(lh.lh_flags & GFS2_LOG_HEAD_UNMOUNT)) { + error = -EBUSY; + break; + } + } + + if (!error) + goto out; /* success */ + + gfs2_freeze_unlock(sdp); + +relock_shared: + error2 = gfs2_freeze_lock_shared(sdp); + gfs2_assert_withdraw(sdp, !error2); + +out: + while (!list_empty(&list)) { + lfcc = list_first_entry(&list, struct lfcc, list); + list_del(&lfcc->list); + gfs2_glock_dq_uninit(&lfcc->gh); + kfree(lfcc); + } + return error; +} + +void gfs2_dinode_out(const struct gfs2_inode *ip, void *buf) +{ + const struct inode *inode = &ip->i_inode; + struct gfs2_dinode *str = buf; + + str->di_header.mh_magic = cpu_to_be32(GFS2_MAGIC); + str->di_header.mh_type = cpu_to_be32(GFS2_METATYPE_DI); + str->di_header.mh_format = cpu_to_be32(GFS2_FORMAT_DI); + str->di_num.no_addr = cpu_to_be64(ip->i_no_addr); + str->di_num.no_formal_ino = cpu_to_be64(ip->i_no_formal_ino); + str->di_mode = cpu_to_be32(inode->i_mode); + str->di_uid = cpu_to_be32(i_uid_read(inode)); + str->di_gid = cpu_to_be32(i_gid_read(inode)); + str->di_nlink = cpu_to_be32(inode->i_nlink); + str->di_size = cpu_to_be64(i_size_read(inode)); + str->di_blocks = cpu_to_be64(gfs2_get_inode_blocks(inode)); + str->di_atime = cpu_to_be64(inode_get_atime_sec(inode)); + str->di_mtime = cpu_to_be64(inode_get_mtime_sec(inode)); + str->di_ctime = cpu_to_be64(inode_get_ctime_sec(inode)); + + str->di_goal_meta = cpu_to_be64(ip->i_goal); + str->di_goal_data = cpu_to_be64(ip->i_goal); + str->di_generation = cpu_to_be64(ip->i_generation); + + str->di_flags = cpu_to_be32(ip->i_diskflags); + str->di_height = cpu_to_be16(ip->i_height); + str->di_payload_format = cpu_to_be32(S_ISDIR(inode->i_mode) && + !(ip->i_diskflags & GFS2_DIF_EXHASH) ? + GFS2_FORMAT_DE : 0); + str->di_depth = cpu_to_be16(ip->i_depth); + str->di_entries = cpu_to_be32(ip->i_entries); + + str->di_eattr = cpu_to_be64(ip->i_eattr); + str->di_atime_nsec = cpu_to_be32(inode_get_atime_nsec(inode)); + str->di_mtime_nsec = cpu_to_be32(inode_get_mtime_nsec(inode)); + str->di_ctime_nsec = cpu_to_be32(inode_get_ctime_nsec(inode)); +} + +/** + * gfs2_write_inode - Make sure the inode is stable on the disk + * @inode: The inode + * @wbc: The writeback control structure + * + * Returns: errno + */ + +static int gfs2_write_inode(struct inode *inode, struct writeback_control *wbc) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + struct address_space *metamapping = gfs2_glock2aspace(ip->i_gl); + struct backing_dev_info *bdi = inode_to_bdi(metamapping->host); + int ret = 0; + bool flush_all = (wbc->sync_mode == WB_SYNC_ALL || gfs2_is_jdata(ip)); + + if (flush_all) + gfs2_log_flush(GFS2_SB(inode), ip->i_gl, + GFS2_LOG_HEAD_FLUSH_NORMAL | + GFS2_LFC_WRITE_INODE); + if (bdi_wb_dirty_exceeded(bdi)) + gfs2_ail1_flush(sdp, wbc); + else + filemap_fdatawrite(metamapping); + if (flush_all) + ret = filemap_fdatawait(metamapping); + if (ret) + mark_inode_dirty_sync(inode); + else { + spin_lock(&inode->i_lock); + if (!(inode->i_flags & I_DIRTY)) + gfs2_ordered_del_inode(ip); + spin_unlock(&inode->i_lock); + } + return ret; +} + +/** + * gfs2_dirty_inode - check for atime updates + * @inode: The inode in question + * @flags: The type of dirty + * + * Unfortunately it can be called under any combination of inode + * glock and freeze glock, so we have to check carefully. + * + * At the moment this deals only with atime - it should be possible + * to expand that role in future, once a review of the locking has + * been carried out. + */ + +static void gfs2_dirty_inode(struct inode *inode, int flags) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + struct buffer_head *bh; + struct gfs2_holder gh; + int need_unlock = 0; + int need_endtrans = 0; + int ret; + + /* This can only happen during incomplete inode creation. */ + if (unlikely(!ip->i_gl)) + return; + + if (gfs2_withdrawn(sdp)) + return; + if (!gfs2_glock_is_locked_by_me(ip->i_gl)) { + ret = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &gh); + if (ret) { + fs_err(sdp, "dirty_inode: glock %d\n", ret); + gfs2_dump_glock(NULL, ip->i_gl, true); + return; + } + need_unlock = 1; + } else if (WARN_ON_ONCE(ip->i_gl->gl_state != LM_ST_EXCLUSIVE)) + return; + + if (current->journal_info == NULL) { + ret = gfs2_trans_begin(sdp, RES_DINODE, 0); + if (ret) { + fs_err(sdp, "dirty_inode: gfs2_trans_begin %d\n", ret); + goto out; + } + need_endtrans = 1; + } + + ret = gfs2_meta_inode_buffer(ip, &bh); + if (ret == 0) { + gfs2_trans_add_meta(ip->i_gl, bh); + gfs2_dinode_out(ip, bh->b_data); + brelse(bh); + } + + if (need_endtrans) + gfs2_trans_end(sdp); +out: + if (need_unlock) + gfs2_glock_dq_uninit(&gh); +} + +/** + * gfs2_make_fs_ro - Turn a Read-Write FS into a Read-Only one + * @sdp: the filesystem + * + * Returns: errno + */ + +void gfs2_make_fs_ro(struct gfs2_sbd *sdp) +{ + int log_write_allowed = test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags); + + if (!test_bit(SDF_KILL, &sdp->sd_flags)) + gfs2_flush_delete_work(sdp); + + gfs2_destroy_threads(sdp); + + if (log_write_allowed) { + gfs2_quota_sync(sdp->sd_vfs, 0); + gfs2_statfs_sync(sdp->sd_vfs, 0); + + /* We do two log flushes here. The first one commits dirty inodes + * and rgrps to the journal, but queues up revokes to the ail list. + * The second flush writes out and removes the revokes. + * + * The first must be done before the FLUSH_SHUTDOWN code + * clears the LIVE flag, otherwise it will not be able to start + * a transaction to write its revokes, and the error will cause + * a withdraw of the file system. */ + gfs2_log_flush(sdp, NULL, GFS2_LFC_MAKE_FS_RO); + gfs2_log_flush(sdp, NULL, GFS2_LOG_HEAD_FLUSH_SHUTDOWN | + GFS2_LFC_MAKE_FS_RO); + wait_event_timeout(sdp->sd_log_waitq, + gfs2_log_is_empty(sdp), + HZ * 5); + gfs2_assert_warn(sdp, gfs2_log_is_empty(sdp)); + } + gfs2_quota_cleanup(sdp); +} + +/** + * gfs2_put_super - Unmount the filesystem + * @sb: The VFS superblock + * + */ + +static void gfs2_put_super(struct super_block *sb) +{ + struct gfs2_sbd *sdp = sb->s_fs_info; + struct gfs2_jdesc *jd; + + /* No more recovery requests */ + set_bit(SDF_NORECOVERY, &sdp->sd_flags); + smp_mb(); + + /* Wait on outstanding recovery */ +restart: + spin_lock(&sdp->sd_jindex_spin); + list_for_each_entry(jd, &sdp->sd_jindex_list, jd_list) { + if (!test_bit(JDF_RECOVERY, &jd->jd_flags)) + continue; + spin_unlock(&sdp->sd_jindex_spin); + wait_on_bit(&jd->jd_flags, JDF_RECOVERY, + TASK_UNINTERRUPTIBLE); + goto restart; + } + spin_unlock(&sdp->sd_jindex_spin); + + /* Wait for withdraw to complete */ + flush_work(&sdp->sd_withdraw_work); + + if (!sb_rdonly(sb)) + gfs2_make_fs_ro(sdp); + else { + if (gfs2_withdrawn(sdp)) + gfs2_destroy_threads(sdp); + + gfs2_quota_cleanup(sdp); + } + + /* At this point, we're through modifying the disk */ + + /* Release stuff */ + + gfs2_freeze_unlock(sdp); + + iput(sdp->sd_jindex); + iput(sdp->sd_statfs_inode); + iput(sdp->sd_rindex); + iput(sdp->sd_quota_inode); + + gfs2_glock_put(sdp->sd_rename_gl); + gfs2_glock_put(sdp->sd_freeze_gl); + + if (!sdp->sd_args.ar_spectator) { + if (gfs2_holder_initialized(&sdp->sd_journal_gh)) + gfs2_glock_dq_uninit(&sdp->sd_journal_gh); + if (gfs2_holder_initialized(&sdp->sd_jinode_gh)) + gfs2_glock_dq_uninit(&sdp->sd_jinode_gh); + brelse(sdp->sd_sc_bh); + gfs2_glock_dq_uninit(&sdp->sd_sc_gh); + gfs2_glock_dq_uninit(&sdp->sd_qc_gh); + free_local_statfs_inodes(sdp); + iput(sdp->sd_qc_inode); + } + + gfs2_glock_dq_uninit(&sdp->sd_live_gh); + gfs2_clear_rgrpd(sdp); + gfs2_jindex_free(sdp); + /* Take apart glock structures and buffer lists */ + gfs2_wait_glocks(sdp); + iput(sdp->sd_inode); + gfs2_delete_debugfs_file(sdp); + + gfs2_sys_fs_del(sdp); + rcu_barrier(); + free_sbd(sdp); +} + +/** + * gfs2_sync_fs - sync the filesystem + * @sb: the superblock + * @wait: true to wait for completion + * + * Flushes the log to disk. + */ + +static int gfs2_sync_fs(struct super_block *sb, int wait) +{ + struct gfs2_sbd *sdp = sb->s_fs_info; + + gfs2_quota_sync(sb, -1); + if (wait) + gfs2_log_flush(sdp, NULL, GFS2_LOG_HEAD_FLUSH_NORMAL | + GFS2_LFC_SYNC_FS); + return sdp->sd_log_error; +} + +static int gfs2_do_thaw(struct gfs2_sbd *sdp, enum freeze_holder who, const void *freeze_owner) +{ + struct super_block *sb = sdp->sd_vfs; + int error; + + error = gfs2_freeze_lock_shared(sdp); + if (error) + goto fail; + error = thaw_super(sb, who, freeze_owner); + if (!error) + return 0; + +fail: + fs_info(sdp, "GFS2: couldn't thaw filesystem: %d\n", error); + gfs2_assert_withdraw(sdp, 0); + return error; +} + +void gfs2_freeze_func(struct work_struct *work) +{ + struct gfs2_sbd *sdp = container_of(work, struct gfs2_sbd, sd_freeze_work); + struct super_block *sb = sdp->sd_vfs; + int error; + + mutex_lock(&sdp->sd_freeze_mutex); + error = -EBUSY; + if (test_bit(SDF_FROZEN, &sdp->sd_flags)) + goto freeze_failed; + + error = freeze_super(sb, FREEZE_HOLDER_USERSPACE, NULL); + if (error) + goto freeze_failed; + + gfs2_freeze_unlock(sdp); + set_bit(SDF_FROZEN, &sdp->sd_flags); + + error = gfs2_do_thaw(sdp, FREEZE_HOLDER_USERSPACE, NULL); + if (error) + goto out; + + clear_bit(SDF_FROZEN, &sdp->sd_flags); + goto out; + +freeze_failed: + fs_info(sdp, "GFS2: couldn't freeze filesystem: %d\n", error); + +out: + mutex_unlock(&sdp->sd_freeze_mutex); + deactivate_super(sb); +} + +/** + * gfs2_freeze_super - prevent further writes to the filesystem + * @sb: the VFS structure for the filesystem + * @who: freeze flags + * @freeze_owner: owner of the freeze + * + */ + +static int gfs2_freeze_super(struct super_block *sb, enum freeze_holder who, + const void *freeze_owner) +{ + struct gfs2_sbd *sdp = sb->s_fs_info; + int error; + + if (!mutex_trylock(&sdp->sd_freeze_mutex)) + return -EBUSY; + if (test_bit(SDF_FROZEN, &sdp->sd_flags)) { + mutex_unlock(&sdp->sd_freeze_mutex); + return -EBUSY; + } + + for (;;) { + error = freeze_super(sb, who, freeze_owner); + if (error) { + fs_info(sdp, "GFS2: couldn't freeze filesystem: %d\n", + error); + goto out; + } + + error = gfs2_lock_fs_check_clean(sdp); + if (!error) { + set_bit(SDF_FREEZE_INITIATOR, &sdp->sd_flags); + set_bit(SDF_FROZEN, &sdp->sd_flags); + break; + } + + (void)gfs2_do_thaw(sdp, who, freeze_owner); + + if (error == -EBUSY) + fs_err(sdp, "waiting for recovery before freeze\n"); + else if (error == -EIO) { + fs_err(sdp, "Fatal IO error: cannot freeze gfs2 due " + "to recovery error.\n"); + goto out; + } else { + fs_err(sdp, "error freezing FS: %d\n", error); + } + fs_err(sdp, "retrying...\n"); + msleep(1000); + } + +out: + mutex_unlock(&sdp->sd_freeze_mutex); + return error; +} + +static int gfs2_freeze_fs(struct super_block *sb) +{ + struct gfs2_sbd *sdp = sb->s_fs_info; + + if (test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags)) { + gfs2_log_flush(sdp, NULL, GFS2_LOG_HEAD_FLUSH_FREEZE | + GFS2_LFC_FREEZE_GO_SYNC); + if (gfs2_withdrawn(sdp)) + return -EIO; + } + return 0; +} + +/** + * gfs2_thaw_super - reallow writes to the filesystem + * @sb: the VFS structure for the filesystem + * @who: freeze flags + * @freeze_owner: owner of the freeze + * + */ + +static int gfs2_thaw_super(struct super_block *sb, enum freeze_holder who, + const void *freeze_owner) +{ + struct gfs2_sbd *sdp = sb->s_fs_info; + int error; + + if (!mutex_trylock(&sdp->sd_freeze_mutex)) + return -EBUSY; + if (!test_bit(SDF_FREEZE_INITIATOR, &sdp->sd_flags)) { + mutex_unlock(&sdp->sd_freeze_mutex); + return -EINVAL; + } + + atomic_inc(&sb->s_active); + gfs2_freeze_unlock(sdp); + + error = gfs2_do_thaw(sdp, who, freeze_owner); + + if (!error) { + clear_bit(SDF_FREEZE_INITIATOR, &sdp->sd_flags); + clear_bit(SDF_FROZEN, &sdp->sd_flags); + } + mutex_unlock(&sdp->sd_freeze_mutex); + deactivate_super(sb); + return error; +} + +/** + * statfs_slow_fill - fill in the sg for a given RG + * @rgd: the RG + * @sc: the sc structure + * + * Returns: 0 on success, -ESTALE if the LVB is invalid + */ + +static int statfs_slow_fill(struct gfs2_rgrpd *rgd, + struct gfs2_statfs_change_host *sc) +{ + gfs2_rgrp_verify(rgd); + sc->sc_total += rgd->rd_data; + sc->sc_free += rgd->rd_free; + sc->sc_dinodes += rgd->rd_dinodes; + return 0; +} + +/** + * gfs2_statfs_slow - Stat a filesystem using asynchronous locking + * @sdp: the filesystem + * @sc: the sc info that will be returned + * + * Any error (other than a signal) will cause this routine to fall back + * to the synchronous version. + * + * FIXME: This really shouldn't busy wait like this. + * + * Returns: errno + */ + +static int gfs2_statfs_slow(struct gfs2_sbd *sdp, struct gfs2_statfs_change_host *sc) +{ + struct gfs2_rgrpd *rgd_next; + struct gfs2_holder *gha, *gh; + unsigned int slots = 64; + unsigned int x; + int done; + int error = 0, err; + + memset(sc, 0, sizeof(struct gfs2_statfs_change_host)); + gha = kmalloc_objs(struct gfs2_holder, slots); + if (!gha) + return -ENOMEM; + for (x = 0; x < slots; x++) + gfs2_holder_mark_uninitialized(gha + x); + + rgd_next = gfs2_rgrpd_get_first(sdp); + + for (;;) { + done = 1; + + for (x = 0; x < slots; x++) { + gh = gha + x; + + if (gfs2_holder_initialized(gh) && gfs2_glock_poll(gh)) { + err = gfs2_glock_wait(gh); + if (err) { + gfs2_holder_uninit(gh); + error = err; + } else { + if (!error) { + struct gfs2_rgrpd *rgd = + gfs2_glock2rgrp(gh->gh_gl); + + error = statfs_slow_fill(rgd, sc); + } + gfs2_glock_dq_uninit(gh); + } + } + + if (gfs2_holder_initialized(gh)) + done = 0; + else if (rgd_next && !error) { + error = gfs2_glock_nq_init(rgd_next->rd_gl, + LM_ST_SHARED, + GL_ASYNC, + gh); + rgd_next = gfs2_rgrpd_get_next(rgd_next); + done = 0; + } + + if (signal_pending(current)) + error = -ERESTARTSYS; + } + + if (done) + break; + + yield(); + } + + kfree(gha); + return error; +} + +/** + * gfs2_statfs_i - Do a statfs + * @sdp: the filesystem + * @sc: the sc structure + * + * Returns: errno + */ + +static int gfs2_statfs_i(struct gfs2_sbd *sdp, struct gfs2_statfs_change_host *sc) +{ + struct gfs2_statfs_change_host *m_sc = &sdp->sd_statfs_master; + struct gfs2_statfs_change_host *l_sc = &sdp->sd_statfs_local; + + spin_lock(&sdp->sd_statfs_spin); + + *sc = *m_sc; + sc->sc_total += l_sc->sc_total; + sc->sc_free += l_sc->sc_free; + sc->sc_dinodes += l_sc->sc_dinodes; + + spin_unlock(&sdp->sd_statfs_spin); + + if (sc->sc_free < 0) + sc->sc_free = 0; + if (sc->sc_free > sc->sc_total) + sc->sc_free = sc->sc_total; + if (sc->sc_dinodes < 0) + sc->sc_dinodes = 0; + + return 0; +} + +/** + * gfs2_statfs - Gather and return stats about the filesystem + * @dentry: The name of the link + * @buf: The buffer + * + * Returns: 0 on success or error code + */ + +static int gfs2_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + struct gfs2_sbd *sdp = sb->s_fs_info; + struct gfs2_statfs_change_host sc; + int error; + + error = gfs2_rindex_update(sdp); + if (error) + return error; + + if (gfs2_tune_get(sdp, gt_statfs_slow)) + error = gfs2_statfs_slow(sdp, &sc); + else + error = gfs2_statfs_i(sdp, &sc); + + if (error) + return error; + + buf->f_type = GFS2_MAGIC; + buf->f_bsize = sdp->sd_sb.sb_bsize; + buf->f_blocks = sc.sc_total; + buf->f_bfree = sc.sc_free; + buf->f_bavail = sc.sc_free; + buf->f_files = sc.sc_dinodes + sc.sc_free; + buf->f_ffree = sc.sc_free; + buf->f_namelen = GFS2_FNAMESIZE; + buf->f_fsid = uuid_to_fsid(sb->s_uuid.b); + + return 0; +} + +/** + * gfs2_drop_inode - Drop an inode (test for remote unlink) + * @inode: The inode to drop + * + * If we've received a callback on an iopen lock then it's because a + * remote node tried to deallocate the inode but failed due to this node + * still having the inode open. Here we mark the link count zero + * since we know that it must have reached zero if the GLF_DEMOTE flag + * is set on the iopen glock. If we didn't do a disk read since the + * remote node removed the final link then we might otherwise miss + * this event. This check ensures that this node will deallocate the + * inode's blocks, or alternatively pass the baton on to another + * node for later deallocation. + */ + +static int gfs2_drop_inode(struct inode *inode) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + + if (inode->i_nlink && + gfs2_holder_initialized(&ip->i_iopen_gh)) { + struct gfs2_glock *gl = ip->i_iopen_gh.gh_gl; + if (glock_needs_demote(gl)) + clear_nlink(inode); + } + + /* + * When under memory pressure when an inode's link count has dropped to + * zero, defer deleting the inode to the delete workqueue. This avoids + * calling into DLM under memory pressure, which can deadlock. + */ + if (!inode->i_nlink && + unlikely(current->flags & PF_MEMALLOC) && + gfs2_holder_initialized(&ip->i_iopen_gh)) { + struct gfs2_glock *gl = ip->i_iopen_gh.gh_gl; + + gfs2_glock_hold(gl); + if (!gfs2_queue_verify_delete(gl, true)) + gfs2_glock_put_async(gl); + return 0; + } + + /* + * No longer cache inodes when trying to evict them all. + */ + if (test_bit(SDF_EVICTING, &sdp->sd_flags)) + return 1; + + return inode_generic_drop(inode); +} + +/** + * gfs2_show_options - Show mount options for /proc/mounts + * @s: seq_file structure + * @root: root of this (sub)tree + * + * Returns: 0 on success or error code + */ + +static int gfs2_show_options(struct seq_file *s, struct dentry *root) +{ + struct gfs2_sbd *sdp = root->d_sb->s_fs_info; + struct gfs2_args *args = &sdp->sd_args; + unsigned int logd_secs, statfs_slow, statfs_quantum, quota_quantum; + + spin_lock(&sdp->sd_tune.gt_spin); + logd_secs = sdp->sd_tune.gt_logd_secs; + quota_quantum = sdp->sd_tune.gt_quota_quantum; + statfs_quantum = sdp->sd_tune.gt_statfs_quantum; + statfs_slow = sdp->sd_tune.gt_statfs_slow; + spin_unlock(&sdp->sd_tune.gt_spin); + + if (is_subdir(root, sdp->sd_master_dir)) + seq_puts(s, ",meta"); + if (args->ar_lockproto[0]) + seq_show_option(s, "lockproto", args->ar_lockproto); + if (args->ar_locktable[0]) + seq_show_option(s, "locktable", args->ar_locktable); + if (args->ar_hostdata[0]) + seq_show_option(s, "hostdata", args->ar_hostdata); + if (args->ar_spectator) + seq_puts(s, ",spectator"); + if (args->ar_localflocks) + seq_puts(s, ",localflocks"); + if (args->ar_debug) + seq_puts(s, ",debug"); + if (args->ar_posix_acl) + seq_puts(s, ",acl"); + if (args->ar_quota != GFS2_QUOTA_DEFAULT) { + char *state; + switch (args->ar_quota) { + case GFS2_QUOTA_OFF: + state = "off"; + break; + case GFS2_QUOTA_ACCOUNT: + state = "account"; + break; + case GFS2_QUOTA_ON: + state = "on"; + break; + case GFS2_QUOTA_QUIET: + state = "quiet"; + break; + default: + state = "unknown"; + break; + } + seq_printf(s, ",quota=%s", state); + } + if (args->ar_suiddir) + seq_puts(s, ",suiddir"); + if (args->ar_data != GFS2_DATA_DEFAULT) { + char *state; + switch (args->ar_data) { + case GFS2_DATA_WRITEBACK: + state = "writeback"; + break; + case GFS2_DATA_ORDERED: + state = "ordered"; + break; + default: + state = "unknown"; + break; + } + seq_printf(s, ",data=%s", state); + } + if (args->ar_discard) + seq_puts(s, ",discard"); + if (logd_secs != 30) + seq_printf(s, ",commit=%d", logd_secs); + if (statfs_quantum != 30) + seq_printf(s, ",statfs_quantum=%d", statfs_quantum); + else if (statfs_slow) + seq_puts(s, ",statfs_quantum=0"); + if (quota_quantum != 60) + seq_printf(s, ",quota_quantum=%d", quota_quantum); + if (args->ar_statfs_percent) + seq_printf(s, ",statfs_percent=%d", args->ar_statfs_percent); + if (args->ar_errors != GFS2_ERRORS_DEFAULT) { + const char *state; + + switch (args->ar_errors) { + case GFS2_ERRORS_WITHDRAW: + state = "withdraw"; + break; + case GFS2_ERRORS_DEACTIVATE: + state = "deactivate"; + break; + case GFS2_ERRORS_PANIC: + state = "panic"; + break; + default: + state = "unknown"; + break; + } + seq_printf(s, ",errors=%s", state); + } + if (test_bit(SDF_NOBARRIERS, &sdp->sd_flags)) + seq_puts(s, ",nobarrier"); + if (test_bit(SDF_DEMOTE, &sdp->sd_flags)) + seq_puts(s, ",demote_interface_used"); + if (args->ar_rgrplvb) + seq_puts(s, ",rgrplvb"); + if (args->ar_loccookie) + seq_puts(s, ",loccookie"); + return 0; +} + +/** + * gfs2_glock_put_eventually + * @gl: The glock to put + * + * When under memory pressure, trigger a deferred glock put to make sure we + * won't call into DLM and deadlock. Otherwise, put the glock directly. + */ + +static void gfs2_glock_put_eventually(struct gfs2_glock *gl) +{ + if (current->flags & PF_MEMALLOC) + gfs2_glock_put_async(gl); + else + gfs2_glock_put(gl); +} + +static enum evict_behavior gfs2_upgrade_iopen_glock(struct inode *inode) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + struct gfs2_holder *gh = &ip->i_iopen_gh; + int error; + + gh->gh_flags |= GL_NOCACHE; + gfs2_glock_dq_wait(gh); + + /* + * If there are no other lock holders, we will immediately get + * exclusive access to the iopen glock here. + * + * Otherwise, the other nodes holding the lock will be notified about + * our locking request (see iopen_go_callback()). If they do not have + * the inode open, they are expected to evict the cached inode and + * release the lock, allowing us to proceed. + * + * Otherwise, if they cannot evict the inode, they are expected to poke + * the inode glock (note: not the iopen glock). We will notice that + * and stop waiting for the iopen glock immediately. The other node(s) + * are then expected to take care of deleting the inode when they no + * longer use it. + * + * As a last resort, if another node keeps holding the iopen glock + * without showing any activity on the inode glock, we will eventually + * time out and fail the iopen glock upgrade. + */ + + gfs2_holder_reinit(LM_ST_EXCLUSIVE, GL_ASYNC | GL_NOCACHE, gh); + error = gfs2_glock_nq(gh); + if (error) + return EVICT_SHOULD_SKIP_DELETE; + + wait_event_interruptible_timeout(sdp->sd_async_glock_wait, + !test_bit(HIF_WAIT, &gh->gh_iflags) || + glock_needs_demote(ip->i_gl), + 5 * HZ); + if (!test_bit(HIF_HOLDER, &gh->gh_iflags)) { + gfs2_glock_dq(gh); + if (glock_needs_demote(ip->i_gl)) + return EVICT_SHOULD_SKIP_DELETE; + return EVICT_SHOULD_DEFER_DELETE; + } + error = gfs2_glock_holder_ready(gh); + if (error) + return EVICT_SHOULD_SKIP_DELETE; + return EVICT_SHOULD_DELETE; +} + +/** + * evict_should_delete - determine whether the inode is eligible for deletion + * @inode: The inode to evict + * @gh: The glock holder structure + * + * This function determines whether the evicted inode is eligible to be deleted + * and locks the inode glock. + * + * Returns: the fate of the dinode + */ +static enum evict_behavior evict_should_delete(struct inode *inode, + struct gfs2_holder *gh) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct super_block *sb = inode->i_sb; + struct gfs2_sbd *sdp = sb->s_fs_info; + int ret; + + if (inode->i_nlink) + return EVICT_SHOULD_SKIP_DELETE; + + if (gfs2_holder_initialized(&ip->i_iopen_gh) && + test_bit(GLF_DEFER_DELETE, &ip->i_iopen_gh.gh_gl->gl_flags)) + return EVICT_SHOULD_DEFER_DELETE; + + /* Deletes should never happen under memory pressure anymore. */ + if (WARN_ON_ONCE(current->flags & PF_MEMALLOC)) + return EVICT_SHOULD_DEFER_DELETE; + + /* Must not read inode block until block type has been verified */ + ret = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, GL_SKIP, gh); + if (unlikely(ret)) + return EVICT_SHOULD_SKIP_DELETE; + + if (gfs2_inode_already_deleted(ip->i_gl, ip->i_no_formal_ino)) + return EVICT_SHOULD_SKIP_DELETE; + ret = gfs2_check_blk_type(sdp, ip->i_no_addr, GFS2_BLKST_UNLINKED); + if (ret) + return EVICT_SHOULD_SKIP_DELETE; + + ret = gfs2_instantiate(gh); + if (ret) + return EVICT_SHOULD_SKIP_DELETE; + + /* + * The inode may have been recreated in the meantime. + */ + if (inode->i_nlink) + return EVICT_SHOULD_SKIP_DELETE; + + if (gfs2_holder_initialized(&ip->i_iopen_gh) && + test_bit(HIF_HOLDER, &ip->i_iopen_gh.gh_iflags)) + return gfs2_upgrade_iopen_glock(inode); + return EVICT_SHOULD_DELETE; +} + +/** + * evict_unlinked_inode - delete the pieces of an unlinked evicted inode + * @inode: The inode to evict + * @gh: The glock holder structure + */ +static int evict_unlinked_inode(struct inode *inode, struct gfs2_holder *gh) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_glock *gl = ip->i_gl; + int ret; + + /* The inode glock must be held exclusively and be instantiated. */ + BUG_ON(!gfs2_holder_initialized(gh) || + test_bit(GLF_INSTANTIATE_NEEDED, &gl->gl_flags)); + + if (S_ISDIR(inode->i_mode) && + (ip->i_diskflags & GFS2_DIF_EXHASH)) { + ret = gfs2_dir_exhash_dealloc(ip); + if (ret) + goto out; + } + + if (ip->i_eattr) { + ret = gfs2_ea_dealloc(ip, true); + if (ret) + goto out; + } + + if (!gfs2_is_stuffed(ip)) { + ret = gfs2_file_dealloc(ip); + if (ret) + goto out; + } + + /* + * As soon as we clear the bitmap for the dinode, gfs2_create_inode() + * can get called to recreate it, or even gfs2_inode_lookup() if the + * inode was recreated on another node in the meantime. + * + * However, inserting the new inode into the inode hash table will not + * succeed until the old inode is removed, and that only happens after + * ->evict_inode() returns. The new inode is attached to its inode and + * iopen glocks after inserting it into the inode hash table, so at + * that point we can be sure that both glocks are unused. + */ + + ret = gfs2_dinode_dealloc(ip); + if (!ret) + gfs2_inode_remember_delete(gl, ip->i_no_formal_ino); + +out: + return ret; +} + +static int gfs2_truncate_inode_pages(struct inode *inode) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + struct address_space *mapping = &inode->i_data; + bool need_trans = gfs2_is_jdata(ip) && mapping->nrpages; + int ret = 0; + + /* + * Truncating a jdata inode address space may create revokes in + * truncate_inode_pages() -> gfs2_invalidate_folio() -> ... -> + * gfs2_remove_from_journal(), so we need a transaction here. + * + * During a withdraw, no new transactions can be created. We still + * take the log flush lock to prevent truncate from racing with + * gfs2_log_flush(). + */ + if (need_trans) { + ret = gfs2_trans_begin(sdp, 0, sdp->sd_jdesc->jd_blocks); + if (ret) + down_read(&sdp->sd_log_flush_lock); + } + truncate_inode_pages(mapping, 0); + if (need_trans) { + if (ret) + up_read(&sdp->sd_log_flush_lock); + else + gfs2_trans_end(sdp); + } + return ret; +} + +static void gfs2_truncate_inode_pages_final(struct inode *inode) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + struct address_space *mapping = &inode->i_data; + bool need_lock = gfs2_is_jdata(ip) && mapping->nrpages; + + if (need_lock) + down_read(&sdp->sd_log_flush_lock); + truncate_inode_pages_final(mapping); + if (need_lock) + up_read(&sdp->sd_log_flush_lock); +} + +/* + * evict_linked_inode - evict an inode whose dinode has not been unlinked + * @inode: The inode to evict + * @gh: The glock holder structure + */ +static int evict_linked_inode(struct inode *inode, struct gfs2_holder *gh) +{ + struct super_block *sb = inode->i_sb; + struct gfs2_sbd *sdp = sb->s_fs_info; + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_glock *gl = ip->i_gl; + struct address_space *metamapping = gfs2_glock2aspace(gl); + int ret; + + if (!(test_bit(GLF_DIRTY, &gl->gl_flags) || inode->i_flags & I_DIRTY)) + goto clean; + + /* The inode glock must be held exclusively and be instantiated. */ + if (!gfs2_holder_initialized(gh)) + ret = gfs2_glock_nq_init(gl, LM_ST_EXCLUSIVE, 0, gh); + else + ret = gfs2_instantiate(gh); + if (ret) + return ret; + + gfs2_log_flush(sdp, gl, GFS2_LOG_HEAD_FLUSH_NORMAL | + GFS2_LFC_EVICT_INODE); + if (test_bit(GLF_DIRTY, &gl->gl_flags)) { + filemap_fdatawrite(metamapping); + filemap_fdatawait(metamapping); + } + write_inode_now(inode, 1); + gfs2_ail_flush(gl, 0); + +clean: + ret = gfs2_truncate_inode_pages(inode); + truncate_inode_pages(metamapping, 0); + return ret; +} + +/** + * gfs2_evict_inode - Remove an inode from cache + * @inode: The inode to evict + * + * There are three cases to consider: + * 1. i_nlink == 0, we are final opener (and must deallocate) + * 2. i_nlink == 0, we are not the final opener (and cannot deallocate) + * 3. i_nlink > 0 + * + * If the fs is read only, then we have to treat all cases as per #3 + * since we are unable to do any deallocation. The inode will be + * deallocated by the next read/write node to attempt an allocation + * in the same resource group + * + * We have to (at the moment) hold the inodes main lock to cover + * the gap between unlocking the shared lock on the iopen lock and + * taking the exclusive lock. I'd rather do a shared -> exclusive + * conversion on the iopen lock, but we can change that later. This + * is safe, just less efficient. + */ + +static void gfs2_evict_inode(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + struct gfs2_sbd *sdp = sb->s_fs_info; + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_holder gh; + enum evict_behavior behavior; + int ret; + + gfs2_holder_mark_uninitialized(&gh); + if (sb_rdonly(sb) || !ip->i_no_addr || !ip->i_gl) + goto out; + + /* + * In case of an incomplete mount, gfs2_evict_inode() may be called for + * system files without having an active journal to write to. In that + * case, skip the filesystem evict. + */ + if (!sdp->sd_jdesc) + goto out; + + behavior = evict_should_delete(inode, &gh); + if (behavior == EVICT_SHOULD_DEFER_DELETE && + !test_bit(SDF_KILL, &sdp->sd_flags)) { + struct gfs2_glock *io_gl = ip->i_iopen_gh.gh_gl; + + if (io_gl) { + gfs2_glock_hold(io_gl); + if (!gfs2_queue_verify_delete(io_gl, true)) + gfs2_glock_put(io_gl); + goto out; + } + behavior = EVICT_SHOULD_SKIP_DELETE; + } + if (behavior == EVICT_SHOULD_DELETE) + ret = evict_unlinked_inode(inode, &gh); + else + ret = evict_linked_inode(inode, &gh); + + if (gfs2_rs_active(&ip->i_res)) + gfs2_rs_deltree(&ip->i_res); + + if (ret && !gfs2_withdrawn(sdp) && ret != -EROFS) + fs_warn(sdp, "gfs2_evict_inode: %d\n", ret); +out: + if (gfs2_holder_initialized(&gh)) + gfs2_glock_dq_uninit(&gh); + gfs2_truncate_inode_pages_final(inode); + if (ip->i_qadata) + gfs2_assert_warn(sdp, ip->i_qadata->qa_ref == 0); + gfs2_rs_deltree(&ip->i_res); + gfs2_ordered_del_inode(ip); + clear_inode(inode); + gfs2_dir_hash_inval(ip); + if (gfs2_holder_initialized(&ip->i_iopen_gh)) { + struct gfs2_glock *gl = ip->i_iopen_gh.gh_gl; + + glock_clear_object(gl, ip); + gfs2_glock_hold(gl); + ip->i_iopen_gh.gh_flags |= GL_NOCACHE; + gfs2_glock_dq_uninit(&ip->i_iopen_gh); + gfs2_glock_put_eventually(gl); + } + if (ip->i_gl) { + glock_clear_object(ip->i_gl, ip); + wait_on_bit_io(&ip->i_flags, GIF_GLOP_PENDING, TASK_UNINTERRUPTIBLE); + gfs2_glock_put_eventually(ip->i_gl); + rcu_assign_pointer(ip->i_gl, NULL); + } +} + +static struct inode *gfs2_alloc_inode(struct super_block *sb) +{ + struct gfs2_inode *ip; + + ip = alloc_inode_sb(sb, gfs2_inode_cachep, GFP_KERNEL); + if (!ip) + return NULL; + ip->i_no_addr = 0; + ip->i_no_formal_ino = 0; + ip->i_flags = 0; + ip->i_gl = NULL; + gfs2_holder_mark_uninitialized(&ip->i_iopen_gh); + memset(&ip->i_res, 0, sizeof(ip->i_res)); + RB_CLEAR_NODE(&ip->i_res.rs_node); + ip->i_diskflags = 0; + ip->i_rahead = 0; + return &ip->i_inode; +} + +static void gfs2_free_inode(struct inode *inode) +{ + kmem_cache_free(gfs2_inode_cachep, GFS2_I(inode)); +} + +void free_local_statfs_inodes(struct gfs2_sbd *sdp) +{ + struct local_statfs_inode *lsi, *safe; + + /* Run through the statfs inodes list to iput and free memory */ + list_for_each_entry_safe(lsi, safe, &sdp->sd_sc_inodes_list, si_list) { + if (lsi->si_jid == sdp->sd_jdesc->jd_jid) + sdp->sd_sc_inode = NULL; /* belongs to this node */ + if (lsi->si_sc_inode) + iput(lsi->si_sc_inode); + list_del(&lsi->si_list); + kfree(lsi); + } +} + +struct inode *find_local_statfs_inode(struct gfs2_sbd *sdp, + unsigned int index) +{ + struct local_statfs_inode *lsi; + + /* Return the local (per node) statfs inode in the + * sdp->sd_sc_inodes_list corresponding to the 'index'. */ + list_for_each_entry(lsi, &sdp->sd_sc_inodes_list, si_list) { + if (lsi->si_jid == index) + return lsi->si_sc_inode; + } + return NULL; +} + +const struct super_operations gfs2_super_ops = { + .alloc_inode = gfs2_alloc_inode, + .free_inode = gfs2_free_inode, + .write_inode = gfs2_write_inode, + .dirty_inode = gfs2_dirty_inode, + .evict_inode = gfs2_evict_inode, + .put_super = gfs2_put_super, + .sync_fs = gfs2_sync_fs, + .freeze_super = gfs2_freeze_super, + .freeze_fs = gfs2_freeze_fs, + .thaw_super = gfs2_thaw_super, + .statfs = gfs2_statfs, + .drop_inode = gfs2_drop_inode, + .show_options = gfs2_show_options, +}; + diff --git a/fs/gfs2/super.h b/fs/gfs2/super.h new file mode 100644 index 000000000..173f1e74c --- /dev/null +++ b/fs/gfs2/super.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#ifndef __SUPER_DOT_H__ +#define __SUPER_DOT_H__ + +#include +#include +#include "incore.h" + +/* Supported fs format version range */ +#define GFS2_FS_FORMAT_MIN (1801) +#define GFS2_FS_FORMAT_MAX (1802) + +void gfs2_lm_unmount(struct gfs2_sbd *sdp); + +static inline unsigned int gfs2_jindex_size(struct gfs2_sbd *sdp) +{ + unsigned int x; + spin_lock(&sdp->sd_jindex_spin); + x = sdp->sd_journals; + spin_unlock(&sdp->sd_jindex_spin); + return x; +} + +void gfs2_jindex_free(struct gfs2_sbd *sdp); + +struct gfs2_jdesc *gfs2_jdesc_find(struct gfs2_sbd *sdp, unsigned int jid); +int gfs2_jdesc_check(struct gfs2_jdesc *jd); +int gfs2_lookup_in_master_dir(struct gfs2_sbd *sdp, char *filename, + struct gfs2_inode **ipp); + +int gfs2_make_fs_rw(struct gfs2_sbd *sdp); +void gfs2_make_fs_ro(struct gfs2_sbd *sdp); +void gfs2_online_uevent(struct gfs2_sbd *sdp); +void gfs2_destroy_threads(struct gfs2_sbd *sdp); +int gfs2_statfs_init(struct gfs2_sbd *sdp); +void gfs2_statfs_change(struct gfs2_sbd *sdp, s64 total, s64 free, + s64 dinodes); +void gfs2_statfs_change_in(struct gfs2_statfs_change_host *sc, + const void *buf); +void gfs2_statfs_change_out(const struct gfs2_statfs_change_host *sc, + void *buf); +void update_statfs(struct gfs2_sbd *sdp, struct buffer_head *m_bh); +int gfs2_statfs_sync(struct super_block *sb, int type); +void gfs2_freeze_func(struct work_struct *work); + +void free_local_statfs_inodes(struct gfs2_sbd *sdp); +struct inode *find_local_statfs_inode(struct gfs2_sbd *sdp, + unsigned int index); +void free_sbd(struct gfs2_sbd *sdp); + +extern struct file_system_type gfs2_fs_type; +extern struct file_system_type gfs2meta_fs_type; +extern const struct export_operations gfs2_export_ops; +extern const struct super_operations gfs2_super_ops; +extern const struct dentry_operations gfs2_dops; + +extern const struct xattr_handler * const gfs2_xattr_handlers_max[]; +extern const struct xattr_handler * const *gfs2_xattr_handlers_min; + +#endif /* __SUPER_DOT_H__ */ + diff --git a/fs/gfs2/sys.c b/fs/gfs2/sys.c new file mode 100644 index 000000000..ea2c7b9e4 --- /dev/null +++ b/fs/gfs2/sys.c @@ -0,0 +1,795 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gfs2.h" +#include "incore.h" +#include "sys.h" +#include "super.h" +#include "glock.h" +#include "quota.h" +#include "util.h" +#include "glops.h" +#include "recovery.h" + +struct gfs2_attr { + struct attribute attr; + ssize_t (*show)(struct gfs2_sbd *, char *); + ssize_t (*store)(struct gfs2_sbd *, const char *, size_t); +}; + +static ssize_t gfs2_attr_show(struct kobject *kobj, struct attribute *attr, + char *buf) +{ + struct gfs2_sbd *sdp = container_of(kobj, struct gfs2_sbd, sd_kobj); + struct gfs2_attr *a = container_of(attr, struct gfs2_attr, attr); + return a->show ? a->show(sdp, buf) : 0; +} + +static ssize_t gfs2_attr_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t len) +{ + struct gfs2_sbd *sdp = container_of(kobj, struct gfs2_sbd, sd_kobj); + struct gfs2_attr *a = container_of(attr, struct gfs2_attr, attr); + return a->store ? a->store(sdp, buf, len) : len; +} + +static const struct sysfs_ops gfs2_attr_ops = { + .show = gfs2_attr_show, + .store = gfs2_attr_store, +}; + + +static struct kset *gfs2_kset; + +static ssize_t id_show(struct gfs2_sbd *sdp, char *buf) +{ + return sysfs_emit(buf, "%u:%u\n", + MAJOR(sdp->sd_vfs->s_dev), MINOR(sdp->sd_vfs->s_dev)); +} + +static ssize_t status_show(struct gfs2_sbd *sdp, char *buf) +{ + unsigned long f = sdp->sd_flags; + ssize_t s; + + s = sysfs_emit(buf, + "Journal Checked: %d\n" + "Journal Live: %d\n" + "Journal ID: %d\n" + "Spectator: %d\n" + "Withdrawn: %d\n" + "No barriers: %d\n" + "No recovery: %d\n" + "Demote: %d\n" + "No Journal ID: %d\n" + "Mounted RO: %d\n" + "RO Recovery: %d\n" + "Force AIL Flush: %d\n" + "FS Freeze Initiator: %d\n" + "FS Frozen: %d\n" + "Killing: %d\n" + "sd_log_error: %d\n" + "sd_log_flush_lock: %d\n" + "sd_log_num_revoke: %u\n" + "sd_log_in_flight: %d\n" + "sd_log_blks_needed: %d\n" + "sd_log_blks_free: %d\n" + "sd_log_flush_head: %d\n" + "sd_log_flush_tail: %d\n" + "sd_log_blks_reserved: %d\n" + "sd_log_revokes_available: %d\n" + "sd_log_pinned: %d\n" + "sd_log_thresh1: %d\n" + "sd_log_thresh2: %d\n", + test_bit(SDF_JOURNAL_CHECKED, &f), + test_bit(SDF_JOURNAL_LIVE, &f), + (sdp->sd_jdesc ? sdp->sd_jdesc->jd_jid : 0), + (sdp->sd_args.ar_spectator ? 1 : 0), + test_bit(SDF_WITHDRAWN, &f), + test_bit(SDF_NOBARRIERS, &f), + test_bit(SDF_NORECOVERY, &f), + test_bit(SDF_DEMOTE, &f), + test_bit(SDF_NOJOURNALID, &f), + (sb_rdonly(sdp->sd_vfs) ? 1 : 0), + test_bit(SDF_RORECOVERY, &f), + test_bit(SDF_FORCE_AIL_FLUSH, &f), + test_bit(SDF_FREEZE_INITIATOR, &f), + test_bit(SDF_FROZEN, &f), + test_bit(SDF_KILL, &f), + sdp->sd_log_error, + rwsem_is_locked(&sdp->sd_log_flush_lock), + sdp->sd_log_num_revoke, + atomic_read(&sdp->sd_log_in_flight), + atomic_read(&sdp->sd_log_blks_needed), + atomic_read(&sdp->sd_log_blks_free), + sdp->sd_log_flush_head, + sdp->sd_log_flush_tail, + sdp->sd_log_blks_reserved, + atomic_read(&sdp->sd_log_revokes_available), + atomic_read(&sdp->sd_log_pinned), + atomic_read(&sdp->sd_log_thresh1), + atomic_read(&sdp->sd_log_thresh2)); + return s; +} + +static ssize_t fsname_show(struct gfs2_sbd *sdp, char *buf) +{ + return sysfs_emit(buf, "%s\n", sdp->sd_fsname); +} + +static ssize_t uuid_show(struct gfs2_sbd *sdp, char *buf) +{ + struct super_block *s = sdp->sd_vfs; + + buf[0] = '\0'; + if (uuid_is_null(&s->s_uuid)) + return 0; + return sysfs_emit(buf, "%pUB\n", &s->s_uuid); +} + +static ssize_t freeze_show(struct gfs2_sbd *sdp, char *buf) +{ + struct super_block *sb = sdp->sd_vfs; + int frozen = (sb->s_writers.frozen == SB_UNFROZEN) ? 0 : 1; + + return sysfs_emit(buf, "%d\n", frozen); +} + +static ssize_t freeze_store(struct gfs2_sbd *sdp, const char *buf, size_t len) +{ + int error, n; + + error = kstrtoint(buf, 0, &n); + if (error) + return error; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + switch (n) { + case 0: + error = thaw_super(sdp->sd_vfs, FREEZE_HOLDER_USERSPACE, NULL); + break; + case 1: + error = freeze_super(sdp->sd_vfs, FREEZE_HOLDER_USERSPACE, NULL); + break; + default: + return -EINVAL; + } + + if (error) { + fs_warn(sdp, "freeze %d error %d\n", n, error); + return error; + } + + return len; +} + +static ssize_t withdraw_show(struct gfs2_sbd *sdp, char *buf) +{ + unsigned int b = gfs2_withdrawn(sdp); + return sysfs_emit(buf, "%u\n", b); +} + +static ssize_t withdraw_store(struct gfs2_sbd *sdp, const char *buf, size_t len) +{ + int error, val; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + error = kstrtoint(buf, 0, &val); + if (error) + return error; + + if (val != 1) + return -EINVAL; + + gfs2_lm(sdp, "withdrawing from cluster at user's request\n"); + gfs2_withdraw(sdp); + + return len; +} + +static ssize_t statfs_sync_store(struct gfs2_sbd *sdp, const char *buf, + size_t len) +{ + int error, val; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + error = kstrtoint(buf, 0, &val); + if (error) + return error; + + if (val != 1) + return -EINVAL; + + gfs2_statfs_sync(sdp->sd_vfs, 0); + return len; +} + +static ssize_t quota_sync_store(struct gfs2_sbd *sdp, const char *buf, + size_t len) +{ + int error, val; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + error = kstrtoint(buf, 0, &val); + if (error) + return error; + + if (val != 1) + return -EINVAL; + + gfs2_quota_sync(sdp->sd_vfs, 0); + return len; +} + +static ssize_t quota_refresh_user_store(struct gfs2_sbd *sdp, const char *buf, + size_t len) +{ + struct kqid qid; + int error; + u32 id; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + error = kstrtou32(buf, 0, &id); + if (error) + return error; + + qid = make_kqid(current_user_ns(), USRQUOTA, id); + if (!qid_valid(qid)) + return -EINVAL; + + error = gfs2_quota_refresh(sdp, qid); + return error ? error : len; +} + +static ssize_t quota_refresh_group_store(struct gfs2_sbd *sdp, const char *buf, + size_t len) +{ + struct kqid qid; + int error; + u32 id; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + error = kstrtou32(buf, 0, &id); + if (error) + return error; + + qid = make_kqid(current_user_ns(), GRPQUOTA, id); + if (!qid_valid(qid)) + return -EINVAL; + + error = gfs2_quota_refresh(sdp, qid); + return error ? error : len; +} + +static ssize_t demote_rq_store(struct gfs2_sbd *sdp, const char *buf, size_t len) +{ + struct gfs2_glock *gl; + const struct gfs2_glock_operations *glops; + unsigned int glmode; + unsigned int gltype; + unsigned long long glnum; + char mode[16]; + int rv; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + rv = sscanf(buf, "%u:%llu %15s", &gltype, &glnum, + mode); + if (rv != 3) + return -EINVAL; + + if (strcmp(mode, "EX") == 0) + glmode = LM_ST_UNLOCKED; + else if ((strcmp(mode, "CW") == 0) || (strcmp(mode, "DF") == 0)) + glmode = LM_ST_DEFERRED; + else if ((strcmp(mode, "PR") == 0) || (strcmp(mode, "SH") == 0)) + glmode = LM_ST_SHARED; + else + return -EINVAL; + + if (gltype > LM_TYPE_JOURNAL) + return -EINVAL; + if (gltype == LM_TYPE_NONDISK && glnum == GFS2_FREEZE_LOCK) + glops = &gfs2_freeze_glops; + else + glops = gfs2_glops_list[gltype]; + if (glops == NULL) + return -EINVAL; + if (!test_and_set_bit(SDF_DEMOTE, &sdp->sd_flags)) + fs_info(sdp, "demote interface used\n"); + rv = gfs2_glock_get(sdp, glnum, glops, NO_CREATE, &gl); + if (rv) + return rv; + gfs2_glock_cb(gl, glmode); + gfs2_glock_put(gl); + return len; +} + + +#define GFS2_ATTR(name, mode, show, store) \ +static struct gfs2_attr gfs2_attr_##name = __ATTR(name, mode, show, store) + +GFS2_ATTR(id, 0444, id_show, NULL); +GFS2_ATTR(fsname, 0444, fsname_show, NULL); +GFS2_ATTR(uuid, 0444, uuid_show, NULL); +GFS2_ATTR(freeze, 0644, freeze_show, freeze_store); +GFS2_ATTR(withdraw, 0644, withdraw_show, withdraw_store); +GFS2_ATTR(statfs_sync, 0200, NULL, statfs_sync_store); +GFS2_ATTR(quota_sync, 0200, NULL, quota_sync_store); +GFS2_ATTR(quota_refresh_user, 0200, NULL, quota_refresh_user_store); +GFS2_ATTR(quota_refresh_group, 0200, NULL, quota_refresh_group_store); +GFS2_ATTR(demote_rq, 0200, NULL, demote_rq_store); +GFS2_ATTR(status, 0400, status_show, NULL); + +static struct attribute *gfs2_attrs[] = { + &gfs2_attr_id.attr, + &gfs2_attr_fsname.attr, + &gfs2_attr_uuid.attr, + &gfs2_attr_freeze.attr, + &gfs2_attr_withdraw.attr, + &gfs2_attr_statfs_sync.attr, + &gfs2_attr_quota_sync.attr, + &gfs2_attr_quota_refresh_user.attr, + &gfs2_attr_quota_refresh_group.attr, + &gfs2_attr_demote_rq.attr, + &gfs2_attr_status.attr, + NULL, +}; +ATTRIBUTE_GROUPS(gfs2); + +static void gfs2_sbd_release(struct kobject *kobj) +{ + struct gfs2_sbd *sdp = container_of(kobj, struct gfs2_sbd, sd_kobj); + + complete(&sdp->sd_kobj_unregister); +} + +static struct kobj_type gfs2_ktype = { + .release = gfs2_sbd_release, + .default_groups = gfs2_groups, + .sysfs_ops = &gfs2_attr_ops, +}; + + +/* + * lock_module. Originally from lock_dlm + */ + +static ssize_t proto_name_show(struct gfs2_sbd *sdp, char *buf) +{ + const struct lm_lockops *ops = sdp->sd_lockstruct.ls_ops; + return sysfs_emit(buf, "%s\n", ops->lm_proto_name); +} + +static ssize_t block_show(struct gfs2_sbd *sdp, char *buf) +{ + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + ssize_t ret; + int val = 0; + + if (test_bit(DFL_BLOCK_LOCKS, &ls->ls_recover_flags)) + val = 1; + ret = sysfs_emit(buf, "%d\n", val); + return ret; +} + +static ssize_t block_store(struct gfs2_sbd *sdp, const char *buf, size_t len) +{ + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + int ret, val; + + ret = kstrtoint(buf, 0, &val); + if (ret) + return ret; + + if (val == 1) + set_bit(DFL_BLOCK_LOCKS, &ls->ls_recover_flags); + else if (val == 0) { + clear_bit(DFL_BLOCK_LOCKS, &ls->ls_recover_flags); + smp_mb__after_atomic(); + gfs2_glock_thaw(sdp); + } else { + return -EINVAL; + } + return len; +} + +static ssize_t withdraw_helper_status_store(struct gfs2_sbd *sdp, + const char *buf, + size_t len) +{ + int ret, val; + + ret = kstrtoint(buf, 0, &val); + if (ret) + return ret; + if (val < 0 || val > 1) + return -EINVAL; + + sdp->sd_withdraw_helper_status = val; + complete(&sdp->sd_withdraw_helper); + return len; +} + +static ssize_t lkfirst_show(struct gfs2_sbd *sdp, char *buf) +{ + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + return sysfs_emit(buf, "%d\n", ls->ls_first); +} + +static ssize_t lkfirst_store(struct gfs2_sbd *sdp, const char *buf, size_t len) +{ + unsigned first; + int rv; + + rv = sscanf(buf, "%u", &first); + if (rv != 1 || first > 1) + return -EINVAL; + rv = wait_for_completion_killable(&sdp->sd_locking_init); + if (rv) + return rv; + spin_lock(&sdp->sd_jindex_spin); + rv = -EBUSY; + if (test_bit(SDF_NOJOURNALID, &sdp->sd_flags) == 0) + goto out; + rv = -EINVAL; + if (sdp->sd_args.ar_spectator) + goto out; + if (sdp->sd_lockstruct.ls_ops->lm_mount == NULL) + goto out; + sdp->sd_lockstruct.ls_first = first; + rv = 0; +out: + spin_unlock(&sdp->sd_jindex_spin); + return rv ? rv : len; +} + +static ssize_t first_done_show(struct gfs2_sbd *sdp, char *buf) +{ + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + return sysfs_emit(buf, "%d\n", !!test_bit(DFL_FIRST_MOUNT_DONE, &ls->ls_recover_flags)); +} + +int gfs2_recover_set(struct gfs2_sbd *sdp, unsigned jid) +{ + struct gfs2_jdesc *jd; + int rv; + + /* Wait for our primary journal to be initialized */ + wait_for_completion(&sdp->sd_journal_ready); + + spin_lock(&sdp->sd_jindex_spin); + rv = -EBUSY; + /** + * If we're a spectator, we use journal0, but it's not really ours. + * So we need to wait for its recovery too. If we skip it we'd never + * queue work to the recovery workqueue, and so its completion would + * never clear the DFL_BLOCK_LOCKS flag, so all our locks would + * permanently stop working. + */ + if (!sdp->sd_jdesc) + goto out; + if (sdp->sd_jdesc->jd_jid == jid && !sdp->sd_args.ar_spectator) + goto out; + rv = -ENOENT; + list_for_each_entry(jd, &sdp->sd_jindex_list, jd_list) { + if (jd->jd_jid != jid && !sdp->sd_args.ar_spectator) + continue; + rv = gfs2_recover_journal(jd, false); + break; + } +out: + spin_unlock(&sdp->sd_jindex_spin); + return rv; +} + +static ssize_t recover_store(struct gfs2_sbd *sdp, const char *buf, size_t len) +{ + unsigned jid; + int rv; + + rv = sscanf(buf, "%u", &jid); + if (rv != 1) + return -EINVAL; + + if (test_bit(SDF_NORECOVERY, &sdp->sd_flags)) { + rv = -ESHUTDOWN; + goto out; + } + + rv = gfs2_recover_set(sdp, jid); +out: + return rv ? rv : len; +} + +static ssize_t recover_done_show(struct gfs2_sbd *sdp, char *buf) +{ + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + return sysfs_emit(buf, "%d\n", ls->ls_recover_jid_done); +} + +static ssize_t recover_status_show(struct gfs2_sbd *sdp, char *buf) +{ + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + return sysfs_emit(buf, "%d\n", ls->ls_recover_jid_status); +} + +static ssize_t jid_show(struct gfs2_sbd *sdp, char *buf) +{ + return sysfs_emit(buf, "%d\n", sdp->sd_lockstruct.ls_jid); +} + +static ssize_t jid_store(struct gfs2_sbd *sdp, const char *buf, size_t len) +{ + int jid; + int rv; + + rv = sscanf(buf, "%d", &jid); + if (rv != 1) + return -EINVAL; + rv = wait_for_completion_killable(&sdp->sd_locking_init); + if (rv) + return rv; + spin_lock(&sdp->sd_jindex_spin); + rv = -EINVAL; + if (sdp->sd_lockstruct.ls_ops->lm_mount == NULL) + goto out; + rv = -EBUSY; + if (test_bit(SDF_NOJOURNALID, &sdp->sd_flags) == 0) + goto out; + rv = 0; + if (sdp->sd_args.ar_spectator && jid > 0) + rv = jid = -EINVAL; + sdp->sd_lockstruct.ls_jid = jid; + clear_bit(SDF_NOJOURNALID, &sdp->sd_flags); + smp_mb__after_atomic(); + wake_up_bit(&sdp->sd_flags, SDF_NOJOURNALID); +out: + spin_unlock(&sdp->sd_jindex_spin); + return rv ? rv : len; +} + +#define GDLM_ATTR(_name,_mode,_show,_store) \ +static struct gfs2_attr gdlm_attr_##_name = __ATTR(_name,_mode,_show,_store) + +GDLM_ATTR(proto_name, 0444, proto_name_show, NULL); +GDLM_ATTR(block, 0644, block_show, block_store); +GDLM_ATTR(withdraw, 0200, NULL, withdraw_helper_status_store); +GDLM_ATTR(jid, 0644, jid_show, jid_store); +GDLM_ATTR(first, 0644, lkfirst_show, lkfirst_store); +GDLM_ATTR(first_done, 0444, first_done_show, NULL); +GDLM_ATTR(recover, 0600, NULL, recover_store); +GDLM_ATTR(recover_done, 0444, recover_done_show, NULL); +GDLM_ATTR(recover_status, 0444, recover_status_show, NULL); + +static struct attribute *lock_module_attrs[] = { + &gdlm_attr_proto_name.attr, + &gdlm_attr_block.attr, + &gdlm_attr_withdraw.attr, + &gdlm_attr_jid.attr, + &gdlm_attr_first.attr, + &gdlm_attr_first_done.attr, + &gdlm_attr_recover.attr, + &gdlm_attr_recover_done.attr, + &gdlm_attr_recover_status.attr, + NULL, +}; + +/* + * get and set struct gfs2_tune fields + */ + +static ssize_t quota_scale_show(struct gfs2_sbd *sdp, char *buf) +{ + return sysfs_emit(buf, "%u %u\n", + sdp->sd_tune.gt_quota_scale_num, + sdp->sd_tune.gt_quota_scale_den); +} + +static ssize_t quota_scale_store(struct gfs2_sbd *sdp, const char *buf, + size_t len) +{ + struct gfs2_tune *gt = &sdp->sd_tune; + unsigned int x, y; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (sscanf(buf, "%u %u", &x, &y) != 2 || !y) + return -EINVAL; + + spin_lock(>->gt_spin); + gt->gt_quota_scale_num = x; + gt->gt_quota_scale_den = y; + spin_unlock(>->gt_spin); + return len; +} + +static ssize_t tune_set(struct gfs2_sbd *sdp, unsigned int *field, + int check_zero, const char *buf, size_t len) +{ + struct gfs2_tune *gt = &sdp->sd_tune; + unsigned int x; + int error; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + error = kstrtouint(buf, 0, &x); + if (error) + return error; + + if (check_zero && !x) + return -EINVAL; + + spin_lock(>->gt_spin); + *field = x; + spin_unlock(>->gt_spin); + return len; +} + +#define TUNE_ATTR_3(name, show, store) \ +static struct gfs2_attr tune_attr_##name = __ATTR(name, 0644, show, store) + +#define TUNE_ATTR_2(name, store) \ +static ssize_t name##_show(struct gfs2_sbd *sdp, char *buf) \ +{ \ + return sysfs_emit(buf, "%u\n", sdp->sd_tune.gt_##name); \ +} \ +TUNE_ATTR_3(name, name##_show, store) + +#define TUNE_ATTR(name, check_zero) \ +static ssize_t name##_store(struct gfs2_sbd *sdp, const char *buf, size_t len)\ +{ \ + return tune_set(sdp, &sdp->sd_tune.gt_##name, check_zero, buf, len); \ +} \ +TUNE_ATTR_2(name, name##_store) + +TUNE_ATTR(quota_warn_period, 0); +TUNE_ATTR(quota_quantum, 0); +TUNE_ATTR(max_readahead, 0); +TUNE_ATTR(complain_secs, 0); +TUNE_ATTR(statfs_slow, 0); +TUNE_ATTR(new_files_jdata, 0); +TUNE_ATTR(statfs_quantum, 1); +TUNE_ATTR_3(quota_scale, quota_scale_show, quota_scale_store); +TUNE_ATTR(withdraw_helper_timeout, 1); + +static struct attribute *tune_attrs[] = { + &tune_attr_quota_warn_period.attr, + &tune_attr_quota_quantum.attr, + &tune_attr_max_readahead.attr, + &tune_attr_complain_secs.attr, + &tune_attr_statfs_slow.attr, + &tune_attr_statfs_quantum.attr, + &tune_attr_quota_scale.attr, + &tune_attr_new_files_jdata.attr, + &tune_attr_withdraw_helper_timeout.attr, + NULL, +}; + +static const struct attribute_group tune_group = { + .name = "tune", + .attrs = tune_attrs, +}; + +static const struct attribute_group lock_module_group = { + .name = "lock_module", + .attrs = lock_module_attrs, +}; + +int gfs2_sys_fs_add(struct gfs2_sbd *sdp) +{ + struct super_block *sb = sdp->sd_vfs; + int error; + char ro[20]; + char spectator[20]; + char *envp[] = { ro, spectator, NULL }; + + sprintf(ro, "RDONLY=%d", sb_rdonly(sb)); + sprintf(spectator, "SPECTATOR=%d", sdp->sd_args.ar_spectator ? 1 : 0); + + init_completion(&sdp->sd_kobj_unregister); + sdp->sd_kobj.kset = gfs2_kset; + error = kobject_init_and_add(&sdp->sd_kobj, &gfs2_ktype, NULL, + "%s", sdp->sd_table_name); + if (error) + goto fail_reg; + + error = sysfs_create_group(&sdp->sd_kobj, &tune_group); + if (error) + goto fail_reg; + + error = sysfs_create_group(&sdp->sd_kobj, &lock_module_group); + if (error) + goto fail_tune; + + error = sysfs_create_link(&sdp->sd_kobj, + &disk_to_dev(sb->s_bdev->bd_disk)->kobj, + "device"); + if (error) + goto fail_lock_module; + + kobject_uevent_env(&sdp->sd_kobj, KOBJ_ADD, envp); + return 0; + +fail_lock_module: + sysfs_remove_group(&sdp->sd_kobj, &lock_module_group); +fail_tune: + sysfs_remove_group(&sdp->sd_kobj, &tune_group); +fail_reg: + fs_err(sdp, "error %d adding sysfs files\n", error); + kobject_put(&sdp->sd_kobj); + wait_for_completion(&sdp->sd_kobj_unregister); + return error; +} + +void gfs2_sys_fs_del(struct gfs2_sbd *sdp) +{ + sysfs_remove_link(&sdp->sd_kobj, "device"); + sysfs_remove_group(&sdp->sd_kobj, &tune_group); + sysfs_remove_group(&sdp->sd_kobj, &lock_module_group); + kobject_put(&sdp->sd_kobj); + wait_for_completion(&sdp->sd_kobj_unregister); +} + +static int gfs2_uevent(const struct kobject *kobj, struct kobj_uevent_env *env) +{ + const struct gfs2_sbd *sdp = container_of(kobj, struct gfs2_sbd, sd_kobj); + const struct super_block *s = sdp->sd_vfs; + + add_uevent_var(env, "LOCKTABLE=%s", sdp->sd_table_name); + add_uevent_var(env, "LOCKPROTO=%s", sdp->sd_proto_name); + if (!test_bit(SDF_NOJOURNALID, &sdp->sd_flags)) + add_uevent_var(env, "JOURNALID=%d", sdp->sd_lockstruct.ls_jid); + if (!uuid_is_null(&s->s_uuid)) + add_uevent_var(env, "UUID=%pUB", &s->s_uuid); + return 0; +} + +static const struct kset_uevent_ops gfs2_uevent_ops = { + .uevent = gfs2_uevent, +}; + +int gfs2_sys_init(void) +{ + gfs2_kset = kset_create_and_add("gfs2", &gfs2_uevent_ops, fs_kobj); + if (!gfs2_kset) + return -ENOMEM; + return 0; +} + +void gfs2_sys_uninit(void) +{ + kset_unregister(gfs2_kset); +} + diff --git a/fs/gfs2/sys.h b/fs/gfs2/sys.h new file mode 100644 index 000000000..f8dacf20e --- /dev/null +++ b/fs/gfs2/sys.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#ifndef __SYS_DOT_H__ +#define __SYS_DOT_H__ + +#include +struct gfs2_sbd; + +int gfs2_sys_fs_add(struct gfs2_sbd *sdp); +void gfs2_sys_fs_del(struct gfs2_sbd *sdp); + +int gfs2_sys_init(void); +void gfs2_sys_uninit(void); + +int gfs2_recover_set(struct gfs2_sbd *sdp, unsigned jid); + +#endif /* __SYS_DOT_H__ */ + diff --git a/fs/gfs2/trace_gfs2.h b/fs/gfs2/trace_gfs2.h new file mode 100644 index 000000000..bc40320ef --- /dev/null +++ b/fs/gfs2/trace_gfs2.h @@ -0,0 +1,647 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM gfs2 + +#if !defined(_TRACE_GFS2_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_GFS2_H + +#include + +#include +#include +#include +#include +#include +#include +#include +#include "incore.h" +#include "glock.h" +#include "rgrp.h" + +#define dlm_state_name(nn) { DLM_LOCK_##nn, #nn } +#define glock_trace_name(x) __print_symbolic(x, \ + dlm_state_name(IV), \ + dlm_state_name(NL), \ + dlm_state_name(CR), \ + dlm_state_name(CW), \ + dlm_state_name(PR), \ + dlm_state_name(PW), \ + dlm_state_name(EX)) + +#define block_state_name(x) __print_symbolic(x, \ + { GFS2_BLKST_FREE, "free" }, \ + { GFS2_BLKST_USED, "used" }, \ + { GFS2_BLKST_DINODE, "dinode" }, \ + { GFS2_BLKST_UNLINKED, "unlinked" }) + +#define TRACE_RS_DELETE 0 +#define TRACE_RS_TREEDEL 1 +#define TRACE_RS_INSERT 2 +#define TRACE_RS_CLAIM 3 + +#define rs_func_name(x) __print_symbolic(x, \ + { 0, "del " }, \ + { 1, "tdel" }, \ + { 2, "ins " }, \ + { 3, "clm " }) + +#define show_glock_flags(flags) __print_flags(flags, "", \ + {(1UL << GLF_LOCK), "l" }, \ + {(1UL << GLF_DEMOTE), "D" }, \ + {(1UL << GLF_PENDING_DEMOTE), "d" }, \ + {(1UL << GLF_DEMOTE_IN_PROGRESS), "p" }, \ + {(1UL << GLF_DIRTY), "y" }, \ + {(1UL << GLF_LFLUSH), "f" }, \ + {(1UL << GLF_MAY_CANCEL), "c" }, \ + {(1UL << GLF_HAVE_REPLY), "r" }, \ + {(1UL << GLF_INITIAL), "a" }, \ + {(1UL << GLF_HAVE_FROZEN_REPLY), "F" }, \ + {(1UL << GLF_OBJECT), "o" }, \ + {(1UL << GLF_BLOCKING), "b" }, \ + {(1UL << GLF_INSTANTIATE_NEEDED), "n" }, \ + {(1UL << GLF_INSTANTIATE_IN_PROG), "N" }, \ + {(1UL << GLF_TRY_TO_EVICT), "e" }, \ + {(1UL << GLF_VERIFY_DELETE), "E" }, \ + {(1UL << GLF_DEFER_DELETE), "s" }, \ + {(1UL << GLF_CANCELING), "C" }) + +#ifndef NUMPTY +#define NUMPTY +static inline u8 glock_trace_state(unsigned int state) +{ + switch(state) { + case LM_ST_SHARED: + return DLM_LOCK_PR; + case LM_ST_DEFERRED: + return DLM_LOCK_CW; + case LM_ST_EXCLUSIVE: + return DLM_LOCK_EX; + } + return DLM_LOCK_NL; +} +#endif + +/* Section 1 - Locking + * + * Objectives: + * Latency: Remote demote request to state change + * Latency: Local lock request to state change + * Latency: State change to lock grant + * Correctness: Ordering of local lock state vs. I/O requests + * Correctness: Responses to remote demote requests + */ + +/* General glock state change (DLM lock request completes) */ +TRACE_EVENT(gfs2_glock_state_change, + + TP_PROTO(const struct gfs2_glock *gl, unsigned int new_state), + + TP_ARGS(gl, new_state), + + TP_STRUCT__entry( + __field( dev_t, dev ) + __field( u64, glnum ) + __field( u32, gltype ) + __field( u8, cur_state ) + __field( u8, new_state ) + __field( u8, dmt_state ) + __field( u8, tgt_state ) + __field( unsigned long, flags ) + ), + + TP_fast_assign( + __entry->dev = glock_sbd(gl)->sd_vfs->s_dev; + __entry->glnum = glock_number(gl); + __entry->gltype = glock_type(gl); + __entry->cur_state = glock_trace_state(gl->gl_state); + __entry->new_state = glock_trace_state(new_state); + __entry->tgt_state = glock_trace_state(gl->gl_target); + __entry->dmt_state = glock_trace_state(gl->gl_demote_state); + __entry->flags = gl->gl_flags | (gl->gl_object ? (1UL<dev), MINOR(__entry->dev), __entry->gltype, + (unsigned long long)__entry->glnum, + glock_trace_name(__entry->cur_state), + glock_trace_name(__entry->new_state), + glock_trace_name(__entry->tgt_state), + glock_trace_name(__entry->dmt_state), + show_glock_flags(__entry->flags)) +); + +/* State change -> unlocked, glock is being deallocated */ +TRACE_EVENT(gfs2_glock_put, + + TP_PROTO(const struct gfs2_glock *gl), + + TP_ARGS(gl), + + TP_STRUCT__entry( + __field( dev_t, dev ) + __field( u64, glnum ) + __field( u32, gltype ) + __field( u8, cur_state ) + __field( unsigned long, flags ) + ), + + TP_fast_assign( + __entry->dev = glock_sbd(gl)->sd_vfs->s_dev; + __entry->gltype = glock_type(gl); + __entry->glnum = glock_number(gl); + __entry->cur_state = glock_trace_state(gl->gl_state); + __entry->flags = gl->gl_flags | (gl->gl_object ? (1UL< %s flags:%s", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->gltype, (unsigned long long)__entry->glnum, + glock_trace_name(__entry->cur_state), + glock_trace_name(DLM_LOCK_IV), + show_glock_flags(__entry->flags)) + +); + +/* Callback (local or remote) requesting lock demotion */ +TRACE_EVENT(gfs2_demote_rq, + + TP_PROTO(const struct gfs2_glock *gl, bool remote), + + TP_ARGS(gl, remote), + + TP_STRUCT__entry( + __field( dev_t, dev ) + __field( u64, glnum ) + __field( u32, gltype ) + __field( u8, cur_state ) + __field( u8, dmt_state ) + __field( unsigned long, flags ) + __field( bool, remote ) + ), + + TP_fast_assign( + __entry->dev = glock_sbd(gl)->sd_vfs->s_dev; + __entry->gltype = glock_type(gl); + __entry->glnum = glock_number(gl); + __entry->cur_state = glock_trace_state(gl->gl_state); + __entry->dmt_state = glock_trace_state(gl->gl_demote_state); + __entry->flags = gl->gl_flags | (gl->gl_object ? (1UL<remote = remote; + ), + + TP_printk("%u,%u glock %d:%lld demote %s to %s flags:%s %s", + MAJOR(__entry->dev), MINOR(__entry->dev), __entry->gltype, + (unsigned long long)__entry->glnum, + glock_trace_name(__entry->cur_state), + glock_trace_name(__entry->dmt_state), + show_glock_flags(__entry->flags), + __entry->remote ? "remote" : "local") + +); + +/* Promotion/grant of a glock */ +TRACE_EVENT(gfs2_promote, + + TP_PROTO(const struct gfs2_holder *gh), + + TP_ARGS(gh), + + TP_STRUCT__entry( + __field( dev_t, dev ) + __field( u64, glnum ) + __field( u32, gltype ) + __field( u8, state ) + ), + + TP_fast_assign( + __entry->dev = glock_sbd(gh->gh_gl)->sd_vfs->s_dev; + __entry->glnum = glock_number(gh->gh_gl); + __entry->gltype = glock_type(gh->gh_gl); + __entry->state = glock_trace_state(gh->gh_state); + ), + + TP_printk("%u,%u glock %u:%llu promote %s", + MAJOR(__entry->dev), MINOR(__entry->dev), __entry->gltype, + (unsigned long long)__entry->glnum, + glock_trace_name(__entry->state)) +); + +/* Queue/dequeue a lock request */ +TRACE_EVENT(gfs2_glock_queue, + + TP_PROTO(const struct gfs2_holder *gh, int queue), + + TP_ARGS(gh, queue), + + TP_STRUCT__entry( + __field( dev_t, dev ) + __field( u64, glnum ) + __field( u32, gltype ) + __field( int, queue ) + __field( u8, state ) + ), + + TP_fast_assign( + __entry->dev = glock_sbd(gh->gh_gl)->sd_vfs->s_dev; + __entry->glnum = glock_number(gh->gh_gl); + __entry->gltype = glock_type(gh->gh_gl); + __entry->queue = queue; + __entry->state = glock_trace_state(gh->gh_state); + ), + + TP_printk("%u,%u glock %u:%llu %squeue %s", + MAJOR(__entry->dev), MINOR(__entry->dev), __entry->gltype, + (unsigned long long)__entry->glnum, + __entry->queue ? "" : "de", + glock_trace_name(__entry->state)) +); + +/* DLM sends a reply to GFS2 */ +TRACE_EVENT(gfs2_glock_lock_time, + + TP_PROTO(const struct gfs2_glock *gl, s64 tdiff), + + TP_ARGS(gl, tdiff), + + TP_STRUCT__entry( + __field( dev_t, dev ) + __field( u64, glnum ) + __field( u32, gltype ) + __field( int, status ) + __field( char, flags ) + __field( s64, tdiff ) + __field( u64, srtt ) + __field( u64, srttvar ) + __field( u64, srttb ) + __field( u64, srttvarb ) + __field( u64, sirt ) + __field( u64, sirtvar ) + __field( u64, dcount ) + __field( u64, qcount ) + ), + + TP_fast_assign( + __entry->dev = glock_sbd(gl)->sd_vfs->s_dev; + __entry->glnum = glock_number(gl); + __entry->gltype = glock_type(gl); + __entry->status = gl->gl_lksb.sb_status; + __entry->flags = gl->gl_lksb.sb_flags; + __entry->tdiff = tdiff; + __entry->srtt = gl->gl_stats.stats[GFS2_LKS_SRTT]; + __entry->srttvar = gl->gl_stats.stats[GFS2_LKS_SRTTVAR]; + __entry->srttb = gl->gl_stats.stats[GFS2_LKS_SRTTB]; + __entry->srttvarb = gl->gl_stats.stats[GFS2_LKS_SRTTVARB]; + __entry->sirt = gl->gl_stats.stats[GFS2_LKS_SIRT]; + __entry->sirtvar = gl->gl_stats.stats[GFS2_LKS_SIRTVAR]; + __entry->dcount = gl->gl_stats.stats[GFS2_LKS_DCOUNT]; + __entry->qcount = gl->gl_stats.stats[GFS2_LKS_QCOUNT]; + ), + + TP_printk("%u,%u glock %d:%lld status:%d flags:%02x tdiff:%lld srtt:%lld/%lld srttb:%lld/%lld sirt:%lld/%lld dcnt:%lld qcnt:%lld", + MAJOR(__entry->dev), MINOR(__entry->dev), __entry->gltype, + (unsigned long long)__entry->glnum, + __entry->status, __entry->flags, + (long long)__entry->tdiff, + (long long)__entry->srtt, + (long long)__entry->srttvar, + (long long)__entry->srttb, + (long long)__entry->srttvarb, + (long long)__entry->sirt, + (long long)__entry->sirtvar, + (long long)__entry->dcount, + (long long)__entry->qcount) +); + +/* Section 2 - Log/journal + * + * Objectives: + * Latency: Log flush time + * Correctness: pin/unpin vs. disk I/O ordering + * Performance: Log usage stats + */ + +/* Pin/unpin a block in the log */ +TRACE_EVENT(gfs2_pin, + + TP_PROTO(const struct gfs2_bufdata *bd, int pin), + + TP_ARGS(bd, pin), + + TP_STRUCT__entry( + __field( dev_t, dev ) + __field( int, pin ) + __field( u32, len ) + __field( sector_t, block ) + __field( u64, ino ) + ), + + TP_fast_assign( + __entry->dev = glock_sbd(bd->bd_gl)->sd_vfs->s_dev; + __entry->pin = pin; + __entry->len = bd->bd_bh->b_size; + __entry->block = bd->bd_bh->b_blocknr; + __entry->ino = glock_number(bd->bd_gl); + ), + + TP_printk("%u,%u log %s %llu/%lu inode %llu", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->pin ? "pin" : "unpin", + (unsigned long long)__entry->block, + (unsigned long)__entry->len, + (unsigned long long)__entry->ino) +); + +/* Flushing the log */ +TRACE_EVENT(gfs2_log_flush, + + TP_PROTO(const struct gfs2_sbd *sdp, int start, u32 flags), + + TP_ARGS(sdp, start, flags), + + TP_STRUCT__entry( + __field( dev_t, dev ) + __field( int, start ) + __field( u64, log_seq ) + __field( u32, flags ) + ), + + TP_fast_assign( + __entry->dev = sdp->sd_vfs->s_dev; + __entry->start = start; + __entry->log_seq = sdp->sd_log_sequence; + __entry->flags = flags; + ), + + TP_printk("%u,%u log flush %s %llu %llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->start ? "start" : "end", + (unsigned long long)__entry->log_seq, + (unsigned long long)__entry->flags) +); + +/* Reserving/releasing blocks in the log */ +TRACE_EVENT(gfs2_log_blocks, + + TP_PROTO(const struct gfs2_sbd *sdp, int blocks), + + TP_ARGS(sdp, blocks), + + TP_STRUCT__entry( + __field( dev_t, dev ) + __field( int, blocks ) + __field( int, blks_free ) + ), + + TP_fast_assign( + __entry->dev = sdp->sd_vfs->s_dev; + __entry->blocks = blocks; + __entry->blks_free = atomic_read(&sdp->sd_log_blks_free); + ), + + TP_printk("%u,%u log reserve %d %d", MAJOR(__entry->dev), + MINOR(__entry->dev), __entry->blocks, __entry->blks_free) +); + +/* Writing back the AIL */ +TRACE_EVENT(gfs2_ail_flush, + + TP_PROTO(const struct gfs2_sbd *sdp, const struct writeback_control *wbc, int start), + + TP_ARGS(sdp, wbc, start), + + TP_STRUCT__entry( + __field( dev_t, dev ) + __field( int, start ) + __field( int, sync_mode ) + __field( long, nr_to_write ) + ), + + TP_fast_assign( + __entry->dev = sdp->sd_vfs->s_dev; + __entry->start = start; + __entry->sync_mode = wbc->sync_mode; + __entry->nr_to_write = wbc->nr_to_write; + ), + + TP_printk("%u,%u ail flush %s %s %ld", MAJOR(__entry->dev), + MINOR(__entry->dev), __entry->start ? "start" : "end", + __entry->sync_mode == WB_SYNC_ALL ? "all" : "none", + __entry->nr_to_write) +); + +/* Section 3 - bmap + * + * Objectives: + * Latency: Bmap request time + * Performance: Block allocator tracing + * Correctness: Test of disard generation vs. blocks allocated + */ + +/* Map an extent of blocks, possibly a new allocation */ +TRACE_EVENT(gfs2_bmap, + + TP_PROTO(const struct gfs2_inode *ip, const struct buffer_head *bh, + sector_t lblock, int create, int errno), + + TP_ARGS(ip, bh, lblock, create, errno), + + TP_STRUCT__entry( + __field( dev_t, dev ) + __field( sector_t, lblock ) + __field( sector_t, pblock ) + __field( u64, inum ) + __field( unsigned long, state ) + __field( u32, len ) + __field( int, create ) + __field( int, errno ) + ), + + TP_fast_assign( + __entry->dev = glock_sbd(ip->i_gl)->sd_vfs->s_dev; + __entry->lblock = lblock; + __entry->pblock = buffer_mapped(bh) ? bh->b_blocknr : 0; + __entry->inum = ip->i_no_addr; + __entry->state = bh->b_state; + __entry->len = bh->b_size; + __entry->create = create; + __entry->errno = errno; + ), + + TP_printk("%u,%u bmap %llu map %llu/%lu to %llu flags:%08lx %s %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->inum, + (unsigned long long)__entry->lblock, + (unsigned long)__entry->len, + (unsigned long long)__entry->pblock, + __entry->state, __entry->create ? "create " : "nocreate", + __entry->errno) +); + +TRACE_EVENT(gfs2_iomap_start, + + TP_PROTO(const struct gfs2_inode *ip, loff_t pos, ssize_t length, + u16 flags), + + TP_ARGS(ip, pos, length, flags), + + TP_STRUCT__entry( + __field( dev_t, dev ) + __field( u64, inum ) + __field( loff_t, pos ) + __field( ssize_t, length ) + __field( u16, flags ) + ), + + TP_fast_assign( + __entry->dev = glock_sbd(ip->i_gl)->sd_vfs->s_dev; + __entry->inum = ip->i_no_addr; + __entry->pos = pos; + __entry->length = length; + __entry->flags = flags; + ), + + TP_printk("%u,%u bmap %llu iomap start %llu/%lu flags:%08x", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->inum, + (unsigned long long)__entry->pos, + (unsigned long)__entry->length, (u16)__entry->flags) +); + +TRACE_EVENT(gfs2_iomap_end, + + TP_PROTO(const struct gfs2_inode *ip, struct iomap *iomap, int ret), + + TP_ARGS(ip, iomap, ret), + + TP_STRUCT__entry( + __field( dev_t, dev ) + __field( u64, inum ) + __field( loff_t, offset ) + __field( ssize_t, length ) + __field( sector_t, pblock ) + __field( u16, flags ) + __field( u16, type ) + __field( int, ret ) + ), + + TP_fast_assign( + __entry->dev = glock_sbd(ip->i_gl)->sd_vfs->s_dev; + __entry->inum = ip->i_no_addr; + __entry->offset = iomap->offset; + __entry->length = iomap->length; + __entry->pblock = iomap->addr == IOMAP_NULL_ADDR ? 0 : + (iomap->addr >> ip->i_inode.i_blkbits); + __entry->flags = iomap->flags; + __entry->type = iomap->type; + __entry->ret = ret; + ), + + TP_printk("%u,%u bmap %llu iomap end %llu/%lu to %llu ty:%d flags:%08x rc:%d", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->inum, + (unsigned long long)__entry->offset, + (unsigned long)__entry->length, + (long long)__entry->pblock, + (u16)__entry->type, + (u16)__entry->flags, __entry->ret) +); + +/* Keep track of blocks as they are allocated/freed */ +TRACE_EVENT(gfs2_block_alloc, + + TP_PROTO(const struct gfs2_inode *ip, struct gfs2_rgrpd *rgd, + u64 block, unsigned len, u8 block_state), + + TP_ARGS(ip, rgd, block, len, block_state), + + TP_STRUCT__entry( + __field( dev_t, dev ) + __field( u64, start ) + __field( u64, inum ) + __field( u32, len ) + __field( u8, block_state ) + __field( u64, rd_addr ) + __field( u32, rd_free_clone ) + __field( u32, rd_requested ) + __field( u32, rd_reserved ) + ), + + TP_fast_assign( + __entry->dev = glock_sbd(rgd->rd_gl)->sd_vfs->s_dev; + __entry->start = block; + __entry->inum = ip->i_no_addr; + __entry->len = len; + __entry->block_state = block_state; + __entry->rd_addr = rgd->rd_addr; + __entry->rd_free_clone = rgd->rd_free_clone; + __entry->rd_requested = rgd->rd_requested; + __entry->rd_reserved = rgd->rd_reserved; + ), + + TP_printk("%u,%u bmap %llu alloc %llu/%lu %s rg:%llu rf:%u rq:%u rr:%u", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->inum, + (unsigned long long)__entry->start, + (unsigned long)__entry->len, + block_state_name(__entry->block_state), + (unsigned long long)__entry->rd_addr, + __entry->rd_free_clone, + __entry->rd_requested, + __entry->rd_reserved) +); + +/* Keep track of multi-block reservations as they are allocated/freed */ +TRACE_EVENT(gfs2_rs, + + TP_PROTO(const struct gfs2_blkreserv *rs, u8 func), + + TP_ARGS(rs, func), + + TP_STRUCT__entry( + __field( dev_t, dev ) + __field( u64, rd_addr ) + __field( u32, rd_free_clone ) + __field( u32, rd_requested ) + __field( u32, rd_reserved ) + __field( u64, inum ) + __field( u64, start ) + __field( u32, requested ) + __field( u32, reserved ) + __field( u8, func ) + ), + + TP_fast_assign( + __entry->dev = rs->rs_rgd->rd_sbd->sd_vfs->s_dev; + __entry->rd_addr = rs->rs_rgd->rd_addr; + __entry->rd_free_clone = rs->rs_rgd->rd_free_clone; + __entry->rd_requested = rs->rs_rgd->rd_requested; + __entry->rd_reserved = rs->rs_rgd->rd_reserved; + __entry->inum = container_of(rs, struct gfs2_inode, + i_res)->i_no_addr; + __entry->start = rs->rs_start; + __entry->requested = rs->rs_requested; + __entry->reserved = rs->rs_reserved; + __entry->func = func; + ), + + TP_printk("%u,%u bmap %llu resrv %llu rg:%llu rf:%u rq:%u rr:%u %s q:%u r:%u", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->inum, + (unsigned long long)__entry->start, + (unsigned long long)__entry->rd_addr, + __entry->rd_free_clone, + __entry->rd_requested, + __entry->rd_reserved, + rs_func_name(__entry->func), + __entry->requested, + __entry->reserved) +); + +#endif /* _TRACE_GFS2_H */ + +/* This part must be outside protection */ +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE trace_gfs2 +#include + diff --git a/fs/gfs2/trans.c b/fs/gfs2/trans.c new file mode 100644 index 000000000..65cbe06e3 --- /dev/null +++ b/fs/gfs2/trans.c @@ -0,0 +1,362 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include + +#include "gfs2.h" +#include "incore.h" +#include "glock.h" +#include "inode.h" +#include "log.h" +#include "lops.h" +#include "meta_io.h" +#include "trans.h" +#include "util.h" +#include "trace_gfs2.h" + +static void gfs2_print_trans(struct gfs2_sbd *sdp, const struct gfs2_trans *tr) +{ + fs_warn(sdp, "Transaction created at: %pSR\n", (void *)tr->tr_ip); + fs_warn(sdp, "blocks=%u revokes=%u reserved=%u touched=%u\n", + tr->tr_blocks, tr->tr_revokes, tr->tr_reserved, + test_bit(TR_TOUCHED, &tr->tr_flags)); + fs_warn(sdp, "Buf %u/%u Databuf %u/%u Revoke %u\n", + tr->tr_num_buf_new, tr->tr_num_buf_rm, + tr->tr_num_databuf_new, tr->tr_num_databuf_rm, + tr->tr_num_revoke); +} + +int __gfs2_trans_begin(struct gfs2_trans *tr, struct gfs2_sbd *sdp, + unsigned int blocks, unsigned int revokes, + unsigned long ip) +{ + unsigned int extra_revokes; + + if (current->journal_info) { + gfs2_print_trans(sdp, current->journal_info); + BUG(); + } + BUG_ON(blocks == 0 && revokes == 0); + + if (gfs2_withdrawn(sdp)) + return -EROFS; + + tr->tr_ip = ip; + tr->tr_blocks = blocks; + tr->tr_revokes = revokes; + tr->tr_reserved = GFS2_LOG_FLUSH_MIN_BLOCKS; + if (blocks) { + /* + * The reserved blocks are either used for data or metadata. + * We can have mixed data and metadata, each with its own log + * descriptor block; see calc_reserved(). + */ + tr->tr_reserved += blocks + 1 + DIV_ROUND_UP(blocks - 1, databuf_limit(sdp)); + } + INIT_LIST_HEAD(&tr->tr_databuf); + INIT_LIST_HEAD(&tr->tr_buf); + INIT_LIST_HEAD(&tr->tr_list); + INIT_LIST_HEAD(&tr->tr_ail1_list); + INIT_LIST_HEAD(&tr->tr_ail2_list); + + if (gfs2_assert_warn(sdp, tr->tr_reserved <= sdp->sd_jdesc->jd_blocks)) + return -EINVAL; + + sb_start_intwrite(sdp->sd_vfs); + + /* + * Try the reservations under sd_log_flush_lock to prevent log flushes + * from creating inconsistencies between the number of allocated and + * reserved revokes. If that fails, do a full-block allocation outside + * of the lock to avoid stalling log flushes. Then, allot the + * appropriate number of blocks to revokes, use as many revokes locally + * as needed, and "release" the surplus into the revokes pool. + */ + + down_read(&sdp->sd_log_flush_lock); + if (unlikely(!test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags))) + goto out_not_live; + if (gfs2_log_try_reserve(sdp, tr, &extra_revokes)) + goto reserved; + + up_read(&sdp->sd_log_flush_lock); + gfs2_log_reserve(sdp, tr, &extra_revokes); + down_read(&sdp->sd_log_flush_lock); + if (unlikely(!test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags))) { + revokes = tr->tr_revokes + extra_revokes; + gfs2_log_release_revokes(sdp, revokes); + gfs2_log_release(sdp, tr->tr_reserved); + goto out_not_live; + } + +reserved: + gfs2_log_release_revokes(sdp, extra_revokes); + current->journal_info = tr; + return 0; + +out_not_live: + up_read(&sdp->sd_log_flush_lock); + sb_end_intwrite(sdp->sd_vfs); + return -EROFS; +} + +int gfs2_trans_begin(struct gfs2_sbd *sdp, unsigned int blocks, + unsigned int revokes) +{ + struct gfs2_trans *tr; + int error; + + tr = kmem_cache_zalloc(gfs2_trans_cachep, GFP_NOFS); + if (!tr) + return -ENOMEM; + error = __gfs2_trans_begin(tr, sdp, blocks, revokes, _RET_IP_); + if (error) + kmem_cache_free(gfs2_trans_cachep, tr); + return error; +} + +void gfs2_trans_end(struct gfs2_sbd *sdp) +{ + struct gfs2_trans *tr = current->journal_info; + s64 nbuf; + + current->journal_info = NULL; + + if (!test_bit(TR_TOUCHED, &tr->tr_flags)) { + gfs2_log_release_revokes(sdp, tr->tr_revokes); + up_read(&sdp->sd_log_flush_lock); + gfs2_log_release(sdp, tr->tr_reserved); + if (!test_bit(TR_ONSTACK, &tr->tr_flags)) + gfs2_trans_free(sdp, tr); + sb_end_intwrite(sdp->sd_vfs); + return; + } + + gfs2_log_release_revokes(sdp, tr->tr_revokes - tr->tr_num_revoke); + + nbuf = tr->tr_num_buf_new + tr->tr_num_databuf_new; + nbuf -= tr->tr_num_buf_rm; + nbuf -= tr->tr_num_databuf_rm; + + if (gfs2_assert_withdraw(sdp, nbuf <= tr->tr_blocks) || + gfs2_assert_withdraw(sdp, tr->tr_num_revoke <= tr->tr_revokes)) + gfs2_print_trans(sdp, tr); + + gfs2_log_commit(sdp, tr); + if (!test_bit(TR_ONSTACK, &tr->tr_flags) && + !test_bit(TR_ATTACHED, &tr->tr_flags)) + gfs2_trans_free(sdp, tr); + up_read(&sdp->sd_log_flush_lock); + + if (sdp->sd_vfs->s_flags & SB_SYNCHRONOUS) + gfs2_log_flush(sdp, NULL, GFS2_LOG_HEAD_FLUSH_NORMAL | + GFS2_LFC_TRANS_END); + sb_end_intwrite(sdp->sd_vfs); +} + +static struct gfs2_bufdata *gfs2_alloc_bufdata(struct gfs2_glock *gl, + struct buffer_head *bh) +{ + struct gfs2_bufdata *bd; + + bd = kmem_cache_zalloc(gfs2_bufdata_cachep, GFP_NOFS | __GFP_NOFAIL); + bd->bd_bh = bh; + bd->bd_gl = gl; + INIT_LIST_HEAD(&bd->bd_list); + INIT_LIST_HEAD(&bd->bd_ail_st_list); + INIT_LIST_HEAD(&bd->bd_ail_gl_list); + return bd; +} + +/** + * gfs2_trans_add_data - Add a databuf to the transaction. + * @gl: The inode glock associated with the buffer + * @bh: The buffer to add + * + * This is used in journaled data mode. + * We need to journal the data block in the same way as metadata in + * the functions above. The difference is that here we have a tag + * which is two __be64's being the block number (as per meta data) + * and a flag which says whether the data block needs escaping or + * not. This means we need a new log entry for each 251 or so data + * blocks, which isn't an enormous overhead but twice as much as + * for normal metadata blocks. + */ +void gfs2_trans_add_data(struct gfs2_glock *gl, struct buffer_head *bh) +{ + struct gfs2_trans *tr = current->journal_info; + struct gfs2_sbd *sdp = glock_sbd(gl); + struct gfs2_bufdata *bd; + + lock_buffer(bh); + if (buffer_pinned(bh)) { + set_bit(TR_TOUCHED, &tr->tr_flags); + goto out; + } + spin_lock(&sdp->sd_log_lock); + bd = bh->b_private; + if (bd == NULL) { + spin_unlock(&sdp->sd_log_lock); + unlock_buffer(bh); + bd = gfs2_alloc_bufdata(gl, bh); + lock_buffer(bh); + spin_lock(&sdp->sd_log_lock); + if (bh->b_private) { + kmem_cache_free(gfs2_bufdata_cachep, bd); + bd = bh->b_private; + } else { + bh->b_private = bd; + } + } + gfs2_assert(sdp, bd->bd_gl == gl); + set_bit(TR_TOUCHED, &tr->tr_flags); + if (list_empty(&bd->bd_list)) { + set_bit(GLF_LFLUSH, &bd->bd_gl->gl_flags); + set_bit(GLF_DIRTY, &bd->bd_gl->gl_flags); + gfs2_pin(sdp, bd->bd_bh); + tr->tr_num_databuf_new++; + list_add_tail(&bd->bd_list, &tr->tr_databuf); + } + spin_unlock(&sdp->sd_log_lock); +out: + unlock_buffer(bh); +} + +void gfs2_trans_add_databufs(struct gfs2_glock *gl, struct folio *folio, + size_t from, size_t len) +{ + struct buffer_head *head = folio_buffers(folio); + unsigned int bsize = head->b_size; + struct buffer_head *bh; + size_t to = from + len; + size_t start, end; + + for (bh = head, start = 0; bh != head || !start; + bh = bh->b_this_page, start = end) { + end = start + bsize; + if (end <= from) + continue; + if (start >= to) + break; + set_buffer_uptodate(bh); + gfs2_trans_add_data(gl, bh); + } +} + +void gfs2_trans_add_meta(struct gfs2_glock *gl, struct buffer_head *bh) +{ + + struct gfs2_sbd *sdp = glock_sbd(gl); + struct super_block *sb = sdp->sd_vfs; + struct gfs2_bufdata *bd; + struct gfs2_meta_header *mh; + struct gfs2_trans *tr = current->journal_info; + + lock_buffer(bh); + if (buffer_pinned(bh)) { + set_bit(TR_TOUCHED, &tr->tr_flags); + goto out; + } + spin_lock(&sdp->sd_log_lock); + bd = bh->b_private; + if (bd == NULL) { + spin_unlock(&sdp->sd_log_lock); + unlock_buffer(bh); + bd = gfs2_alloc_bufdata(gl, bh); + lock_buffer(bh); + spin_lock(&sdp->sd_log_lock); + if (bh->b_private) { + kmem_cache_free(gfs2_bufdata_cachep, bd); + bd = bh->b_private; + } else { + bh->b_private = bd; + } + } + gfs2_assert(sdp, bd->bd_gl == gl); + set_bit(TR_TOUCHED, &tr->tr_flags); + if (!list_empty(&bd->bd_list)) + goto out_unlock; + set_bit(GLF_LFLUSH, &bd->bd_gl->gl_flags); + set_bit(GLF_DIRTY, &bd->bd_gl->gl_flags); + mh = (struct gfs2_meta_header *)bd->bd_bh->b_data; + if (unlikely(mh->mh_magic != cpu_to_be32(GFS2_MAGIC))) { + fs_err(sdp, "Attempting to add uninitialised block to " + "journal (inplace block=%lld)\n", + (unsigned long long)bd->bd_bh->b_blocknr); + BUG(); + } + if (gfs2_withdrawn(sdp)) { + fs_info(sdp, "GFS2:adding buf while withdrawn! 0x%llx\n", + (unsigned long long)bd->bd_bh->b_blocknr); + goto out_unlock; + } + if (unlikely(sb->s_writers.frozen == SB_FREEZE_COMPLETE)) { + fs_info(sdp, "GFS2:adding buf while frozen\n"); + gfs2_withdraw(sdp); + goto out_unlock; + } + gfs2_pin(sdp, bd->bd_bh); + mh->__pad0 = cpu_to_be64(0); + mh->mh_jid = cpu_to_be32(sdp->sd_jdesc->jd_jid); + list_add(&bd->bd_list, &tr->tr_buf); + tr->tr_num_buf_new++; +out_unlock: + spin_unlock(&sdp->sd_log_lock); +out: + unlock_buffer(bh); +} + +void gfs2_trans_add_revoke(struct gfs2_sbd *sdp, struct gfs2_bufdata *bd) +{ + struct gfs2_trans *tr = current->journal_info; + + BUG_ON(!list_empty(&bd->bd_list)); + gfs2_add_revoke(sdp, bd); + set_bit(TR_TOUCHED, &tr->tr_flags); + tr->tr_num_revoke++; +} + +void gfs2_trans_remove_revoke(struct gfs2_sbd *sdp, u64 blkno, unsigned int len) +{ + struct gfs2_bufdata *bd, *tmp; + unsigned int n = len; + + spin_lock(&sdp->sd_log_lock); + list_for_each_entry_safe(bd, tmp, &sdp->sd_log_revokes, bd_list) { + if ((bd->bd_blkno >= blkno) && (bd->bd_blkno < (blkno + len))) { + list_del_init(&bd->bd_list); + gfs2_assert_withdraw(sdp, sdp->sd_log_num_revoke); + sdp->sd_log_num_revoke--; + if (bd->bd_gl) + gfs2_glock_remove_revoke(bd->bd_gl); + kmem_cache_free(gfs2_bufdata_cachep, bd); + gfs2_log_release_revokes(sdp, 1); + if (--n == 0) + break; + } + } + spin_unlock(&sdp->sd_log_lock); +} + +void gfs2_trans_free(struct gfs2_sbd *sdp, struct gfs2_trans *tr) +{ + if (tr == NULL) + return; + + gfs2_assert_warn(sdp, list_empty(&tr->tr_ail1_list)); + gfs2_assert_warn(sdp, list_empty(&tr->tr_ail2_list)); + gfs2_assert_warn(sdp, list_empty(&tr->tr_databuf)); + gfs2_assert_warn(sdp, list_empty(&tr->tr_buf)); + kmem_cache_free(gfs2_trans_cachep, tr); +} diff --git a/fs/gfs2/trans.h b/fs/gfs2/trans.h new file mode 100644 index 000000000..790c55f59 --- /dev/null +++ b/fs/gfs2/trans.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#ifndef __TRANS_DOT_H__ +#define __TRANS_DOT_H__ + +#include +struct gfs2_sbd; +struct gfs2_rgrpd; +struct gfs2_glock; + +#define RES_DINODE 1 +#define RES_INDIRECT 1 +#define RES_JDATA 1 +#define RES_DATA 1 +#define RES_LEAF 1 +#define RES_RG_HDR 1 +#define RES_RG_BIT 2 +#define RES_EATTR 1 +#define RES_STATFS 1 +#define RES_QUOTA 2 + +/* reserve either the number of blocks to be allocated plus the rg header + * block, or all of the blocks in the rg, whichever is smaller */ +static inline unsigned int gfs2_rg_blocks(const struct gfs2_inode *ip, unsigned requested) +{ + struct gfs2_rgrpd *rgd = ip->i_res.rs_rgd; + + if (requested < rgd->rd_length) + return requested + 1; + return rgd->rd_length; +} + +int __gfs2_trans_begin(struct gfs2_trans *tr, struct gfs2_sbd *sdp, + unsigned int blocks, unsigned int revokes, + unsigned long ip); +int gfs2_trans_begin(struct gfs2_sbd *sdp, unsigned int blocks, + unsigned int revokes); + +void gfs2_trans_end(struct gfs2_sbd *sdp); +void gfs2_trans_add_data(struct gfs2_glock *gl, struct buffer_head *bh); +void gfs2_trans_add_databufs(struct gfs2_glock *gl, struct folio *folio, + size_t from, size_t len); +void gfs2_trans_add_meta(struct gfs2_glock *gl, struct buffer_head *bh); +void gfs2_trans_add_revoke(struct gfs2_sbd *sdp, struct gfs2_bufdata *bd); +void gfs2_trans_remove_revoke(struct gfs2_sbd *sdp, u64 blkno, unsigned int len); +void gfs2_trans_free(struct gfs2_sbd *sdp, struct gfs2_trans *tr); + +#endif /* __TRANS_DOT_H__ */ diff --git a/fs/gfs2/util.c b/fs/gfs2/util.c new file mode 100644 index 000000000..83b8bb644 --- /dev/null +++ b/fs/gfs2/util.c @@ -0,0 +1,436 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gfs2.h" +#include "incore.h" +#include "glock.h" +#include "glops.h" +#include "log.h" +#include "lops.h" +#include "recovery.h" +#include "rgrp.h" +#include "super.h" +#include "util.h" + +struct kmem_cache *gfs2_glock_cachep __read_mostly; +struct kmem_cache *gfs2_glock_aspace_cachep __read_mostly; +struct kmem_cache *gfs2_inode_cachep __read_mostly; +struct kmem_cache *gfs2_bufdata_cachep __read_mostly; +struct kmem_cache *gfs2_rgrpd_cachep __read_mostly; +struct kmem_cache *gfs2_quotad_cachep __read_mostly; +struct kmem_cache *gfs2_qadata_cachep __read_mostly; +struct kmem_cache *gfs2_trans_cachep __read_mostly; +mempool_t *gfs2_page_pool __read_mostly; + +void gfs2_assert_i(struct gfs2_sbd *sdp) +{ + fs_emerg(sdp, "fatal assertion failed\n"); +} + +/** + * check_journal_clean - Make sure a journal is clean for a spectator mount + * @sdp: The GFS2 superblock + * @jd: The journal descriptor + * @verbose: Show more prints in the log + * + * Returns: 0 if the journal is clean or locked, else an error + */ +int check_journal_clean(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd, + bool verbose) +{ + int error; + struct gfs2_holder j_gh; + struct gfs2_log_header_host head; + struct gfs2_inode *ip; + + ip = GFS2_I(jd->jd_inode); + error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, LM_FLAG_RECOVER | + GL_EXACT | GL_NOCACHE, &j_gh); + if (error) { + if (verbose) + fs_err(sdp, "Error %d locking journal for spectator " + "mount.\n", error); + return -EPERM; + } + error = gfs2_jdesc_check(jd); + if (error) { + if (verbose) + fs_err(sdp, "Error checking journal for spectator " + "mount.\n"); + goto out_unlock; + } + error = gfs2_find_jhead(jd, &head); + if (error) { + if (verbose) + fs_err(sdp, "Error parsing journal for spectator " + "mount.\n"); + goto out_unlock; + } + if (!(head.lh_flags & GFS2_LOG_HEAD_UNMOUNT)) { + error = -EPERM; + if (verbose) + fs_err(sdp, "jid=%u: Journal is dirty, so the first " + "mounter must not be a spectator.\n", + jd->jd_jid); + } + +out_unlock: + gfs2_glock_dq_uninit(&j_gh); + return error; +} + +/** + * gfs2_freeze_lock_shared - hold the freeze glock + * @sdp: the superblock + */ +int gfs2_freeze_lock_shared(struct gfs2_sbd *sdp) +{ + int flags = LM_FLAG_RECOVER | GL_EXACT; + int error; + + error = gfs2_glock_nq_init(sdp->sd_freeze_gl, LM_ST_SHARED, flags, + &sdp->sd_freeze_gh); + if (error && error != GLR_TRYFAILED) + fs_err(sdp, "can't lock the freeze glock: %d\n", error); + return error; +} + +void gfs2_freeze_unlock(struct gfs2_sbd *sdp) +{ + if (gfs2_holder_initialized(&sdp->sd_freeze_gh)) + gfs2_glock_dq_uninit(&sdp->sd_freeze_gh); +} + +static void do_withdraw(struct gfs2_sbd *sdp) +{ + down_write(&sdp->sd_log_flush_lock); + if (!test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags)) { + up_write(&sdp->sd_log_flush_lock); + return; + } + clear_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags); + gfs2_ail_drain(sdp); /* frees all transactions */ + up_write(&sdp->sd_log_flush_lock); + + wake_up(&sdp->sd_logd_waitq); + wake_up(&sdp->sd_quota_wait); + + wait_event_timeout(sdp->sd_log_waitq, + gfs2_log_is_empty(sdp), + HZ * 5); + + sdp->sd_vfs->s_flags |= SB_RDONLY; + + /* + * Dequeue any pending non-system glock holders that can no + * longer be granted because the file system is withdrawn. + */ + gfs2_withdraw_glocks(sdp); +} + +void gfs2_lm(struct gfs2_sbd *sdp, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + if (sdp->sd_args.ar_errors == GFS2_ERRORS_WITHDRAW && + test_bit(SDF_WITHDRAWN, &sdp->sd_flags)) + return; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + fs_err(sdp, "%pV", &vaf); + va_end(args); +} + +/** + * gfs2_offline_uevent - run gfs2_withdraw_helper + * @sdp: The GFS2 superblock + */ +static bool gfs2_offline_uevent(struct gfs2_sbd *sdp) +{ + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + long timeout; + + /* Skip protocol "lock_nolock" which doesn't require shared storage. */ + if (!ls->ls_ops->lm_lock) + return false; + + /* + * The gfs2_withdraw_helper replies by writing one of the following + * status codes to "/sys$DEVPATH/lock_module/withdraw": + * + * 0 - The shared block device has been marked inactive. Future write + * operations will fail. + * + * 1 - The shared block device may still be active and carry out + * write operations. + * + * If the "offline" uevent isn't reacted upon in time, the event + * handler is assumed to have failed. + */ + + sdp->sd_withdraw_helper_status = -1; + kobject_uevent(&sdp->sd_kobj, KOBJ_OFFLINE); + timeout = gfs2_tune_get(sdp, gt_withdraw_helper_timeout) * HZ; + wait_for_completion_timeout(&sdp->sd_withdraw_helper, timeout); + if (sdp->sd_withdraw_helper_status == -1) { + fs_err(sdp, "%s timed out\n", "gfs2_withdraw_helper"); + } else { + fs_err(sdp, "%s %s with status %d\n", + "gfs2_withdraw_helper", + sdp->sd_withdraw_helper_status == 0 ? + "succeeded" : "failed", + sdp->sd_withdraw_helper_status); + } + return sdp->sd_withdraw_helper_status == 0; +} + +void gfs2_withdraw_func(struct work_struct *work) +{ + struct gfs2_sbd *sdp = container_of(work, struct gfs2_sbd, sd_withdraw_work); + struct lm_lockstruct *ls = &sdp->sd_lockstruct; + const struct lm_lockops *lm = ls->ls_ops; + bool device_inactive; + + if (test_bit(SDF_KILL, &sdp->sd_flags)) + return; + + BUG_ON(sdp->sd_args.ar_debug); + + /* + * Try to deactivate the shared block device so that no more I/O will + * go through. If successful, we can immediately trigger remote + * recovery. Otherwise, we must first empty out all our local caches. + */ + + device_inactive = gfs2_offline_uevent(sdp); + + if (sdp->sd_args.ar_errors == GFS2_ERRORS_DEACTIVATE && !device_inactive) + panic("GFS2: fsid=%s: panic requested\n", sdp->sd_fsname); + + if (lm->lm_unmount) { + if (device_inactive) { + lm->lm_unmount(sdp, false); + do_withdraw(sdp); + } else { + do_withdraw(sdp); + lm->lm_unmount(sdp, false); + } + } else { + do_withdraw(sdp); + } + + fs_err(sdp, "file system withdrawn\n"); +} + +void gfs2_withdraw(struct gfs2_sbd *sdp) +{ + if (sdp->sd_args.ar_errors == GFS2_ERRORS_WITHDRAW || + sdp->sd_args.ar_errors == GFS2_ERRORS_DEACTIVATE) { + if (test_and_set_bit(SDF_WITHDRAWN, &sdp->sd_flags)) + return; + + dump_stack(); + /* + * There is no need to withdraw when the superblock hasn't been + * fully initialized, yet. + */ + if (!(sdp->sd_vfs->s_flags & SB_BORN)) + return; + fs_err(sdp, "about to withdraw this file system\n"); + schedule_work(&sdp->sd_withdraw_work); + return; + } + + if (sdp->sd_args.ar_errors == GFS2_ERRORS_PANIC) + panic("GFS2: fsid=%s: panic requested\n", sdp->sd_fsname); +} + +/* + * gfs2_assert_withdraw_i - Cause the machine to withdraw if @assertion is false + */ + +void gfs2_assert_withdraw_i(struct gfs2_sbd *sdp, char *assertion, + const char *function, char *file, unsigned int line) +{ + if (gfs2_withdrawn(sdp)) + return; + + fs_err(sdp, + "fatal: assertion \"%s\" failed - " + "function = %s, file = %s, line = %u\n", + assertion, function, file, line); + + gfs2_withdraw(sdp); + dump_stack(); +} + +/* + * gfs2_assert_warn_i - Print a message to the console if @assertion is false + */ + +void gfs2_assert_warn_i(struct gfs2_sbd *sdp, char *assertion, + const char *function, char *file, unsigned int line) +{ + if (time_before(jiffies, + sdp->sd_last_warning + + gfs2_tune_get(sdp, gt_complain_secs) * HZ)) + return; + + if (sdp->sd_args.ar_errors == GFS2_ERRORS_WITHDRAW) + fs_warn(sdp, "warning: assertion \"%s\" failed - " + "function = %s, file = %s, line = %u\n", + assertion, function, file, line); + + if (sdp->sd_args.ar_debug) + BUG(); + else + dump_stack(); + + if (sdp->sd_args.ar_errors == GFS2_ERRORS_PANIC) + panic("GFS2: fsid=%s: warning: assertion \"%s\" failed - " + "function = %s, file = %s, line = %u\n", + sdp->sd_fsname, assertion, + function, file, line); + + sdp->sd_last_warning = jiffies; +} + +/* + * gfs2_consist_i - Flag a filesystem consistency error and withdraw + */ + +void gfs2_consist_i(struct gfs2_sbd *sdp, const char *function, + char *file, unsigned int line) +{ + gfs2_lm(sdp, + "fatal: filesystem consistency error - " + "function = %s, file = %s, line = %u\n", + function, file, line); + gfs2_withdraw(sdp); +} + +/* + * gfs2_consist_inode_i - Flag an inode consistency error and withdraw + */ + +void gfs2_consist_inode_i(struct gfs2_inode *ip, + const char *function, char *file, unsigned int line) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + + gfs2_lm(sdp, + "fatal: filesystem consistency error - " + "inode = %llu %llu, " + "function = %s, file = %s, line = %u\n", + (unsigned long long)ip->i_no_formal_ino, + (unsigned long long)ip->i_no_addr, + function, file, line); + gfs2_dump_glock(NULL, ip->i_gl, 1); + gfs2_withdraw(sdp); +} + +/* + * gfs2_consist_rgrpd_i - Flag a RG consistency error and withdraw + */ + +void gfs2_consist_rgrpd_i(struct gfs2_rgrpd *rgd, + const char *function, char *file, unsigned int line) +{ + struct gfs2_sbd *sdp = rgd->rd_sbd; + char fs_id_buf[sizeof(sdp->sd_fsname) + 7]; + + sprintf(fs_id_buf, "fsid=%s: ", sdp->sd_fsname); + gfs2_rgrp_dump(NULL, rgd, fs_id_buf); + gfs2_lm(sdp, + "fatal: filesystem consistency error - " + "RG = %llu, " + "function = %s, file = %s, line = %u\n", + (unsigned long long)rgd->rd_addr, + function, file, line); + gfs2_dump_glock(NULL, rgd->rd_gl, 1); + gfs2_withdraw(sdp); +} + +/* + * gfs2_meta_check_ii - Flag a magic number consistency error and withdraw + */ + +void gfs2_meta_check_ii(struct gfs2_sbd *sdp, struct buffer_head *bh, + const char *function, char *file, + unsigned int line) +{ + gfs2_lm(sdp, + "fatal: invalid metadata block - " + "bh = %llu (bad magic number), " + "function = %s, file = %s, line = %u\n", + (unsigned long long)bh->b_blocknr, + function, file, line); + gfs2_withdraw(sdp); +} + +/* + * gfs2_metatype_check_ii - Flag a metadata type consistency error and withdraw + */ + +void gfs2_metatype_check_ii(struct gfs2_sbd *sdp, struct buffer_head *bh, + u16 type, u16 t, const char *function, + char *file, unsigned int line) +{ + gfs2_lm(sdp, + "fatal: invalid metadata block - " + "bh = %llu (type: exp=%u, found=%u), " + "function = %s, file = %s, line = %u\n", + (unsigned long long)bh->b_blocknr, type, t, + function, file, line); + gfs2_withdraw(sdp); +} + +/* + * gfs2_io_error_i - Flag an I/O error and withdraw + * Returns: -1 if this call withdrew the machine, + * 0 if it was already withdrawn + */ + +void gfs2_io_error_i(struct gfs2_sbd *sdp, const char *function, char *file, + unsigned int line) +{ + gfs2_lm(sdp, + "fatal: I/O error - " + "function = %s, file = %s, line = %u\n", + function, file, line); + gfs2_withdraw(sdp); +} + +/* + * gfs2_io_error_bh_i - Flag a buffer I/O error and withdraw + */ + +void gfs2_io_error_bh_i(struct gfs2_sbd *sdp, struct buffer_head *bh, + const char *function, char *file, unsigned int line) +{ + if (gfs2_withdrawn(sdp)) + return; + + fs_err(sdp, "fatal: I/O error - " + "block = %llu, " + "function = %s, file = %s, line = %u\n", + (unsigned long long)bh->b_blocknr, function, file, line); + gfs2_withdraw(sdp); +} diff --git a/fs/gfs2/util.h b/fs/gfs2/util.h new file mode 100644 index 000000000..ffcc47d6b --- /dev/null +++ b/fs/gfs2/util.h @@ -0,0 +1,199 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#ifndef __UTIL_DOT_H__ +#define __UTIL_DOT_H__ + +#ifdef pr_fmt +#undef pr_fmt +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#endif + +#include + +#include "incore.h" + +#define fs_emerg(fs, fmt, ...) \ + pr_emerg("fsid=%s: " fmt, (fs)->sd_fsname, ##__VA_ARGS__) +#define fs_warn(fs, fmt, ...) \ + pr_warn("fsid=%s: " fmt, (fs)->sd_fsname, ##__VA_ARGS__) +#define fs_err(fs, fmt, ...) \ + pr_err("fsid=%s: " fmt, (fs)->sd_fsname, ##__VA_ARGS__) +#define fs_info(fs, fmt, ...) \ + pr_info("fsid=%s: " fmt, (fs)->sd_fsname, ##__VA_ARGS__) + +void gfs2_assert_i(struct gfs2_sbd *sdp); + +#define gfs2_assert(sdp, assertion) \ +do { \ + if (unlikely(!(assertion))) { \ + gfs2_assert_i(sdp); \ + BUG(); \ + } \ +} while (0) + + +void gfs2_assert_withdraw_i(struct gfs2_sbd *sdp, char *assertion, + const char *function, char *file, unsigned int line); + +#define gfs2_assert_withdraw(sdp, assertion) \ + ({ \ + bool _bool = (assertion); \ + if (unlikely(!_bool)) \ + gfs2_assert_withdraw_i((sdp), #assertion, \ + __func__, __FILE__, __LINE__); \ + !_bool; \ + }) + +void gfs2_assert_warn_i(struct gfs2_sbd *sdp, char *assertion, + const char *function, char *file, unsigned int line); + +#define gfs2_assert_warn(sdp, assertion) \ + ({ \ + bool _bool = (assertion); \ + if (unlikely(!_bool)) \ + gfs2_assert_warn_i((sdp), #assertion, \ + __func__, __FILE__, __LINE__); \ + !_bool; \ + }) + +void gfs2_consist_i(struct gfs2_sbd *sdp, + const char *function, char *file, unsigned int line); + +#define gfs2_consist(sdp) \ +gfs2_consist_i((sdp), __func__, __FILE__, __LINE__) + + +void gfs2_consist_inode_i(struct gfs2_inode *ip, + const char *function, char *file, unsigned int line); + +#define gfs2_consist_inode(ip) \ +gfs2_consist_inode_i((ip), __func__, __FILE__, __LINE__) + + +void gfs2_consist_rgrpd_i(struct gfs2_rgrpd *rgd, + const char *function, char *file, unsigned int line); + +#define gfs2_consist_rgrpd(rgd) \ +gfs2_consist_rgrpd_i((rgd), __func__, __FILE__, __LINE__) + + +void gfs2_meta_check_ii(struct gfs2_sbd *sdp, struct buffer_head *bh, + const char *function, + char *file, unsigned int line); + +static inline int gfs2_meta_check(struct gfs2_sbd *sdp, + struct buffer_head *bh) +{ + struct gfs2_meta_header *mh = (struct gfs2_meta_header *)bh->b_data; + u32 magic = be32_to_cpu(mh->mh_magic); + if (unlikely(magic != GFS2_MAGIC)) { + fs_err(sdp, "Magic number missing at %llu\n", + (unsigned long long)bh->b_blocknr); + return -EIO; + } + return 0; +} + +void gfs2_metatype_check_ii(struct gfs2_sbd *sdp, struct buffer_head *bh, + u16 type, u16 t, + const char *function, + char *file, unsigned int line); + +static inline int gfs2_metatype_check_i(struct gfs2_sbd *sdp, + struct buffer_head *bh, + u16 type, + const char *function, + char *file, unsigned int line) +{ + struct gfs2_meta_header *mh = (struct gfs2_meta_header *)bh->b_data; + u32 magic = be32_to_cpu(mh->mh_magic); + u16 t = be32_to_cpu(mh->mh_type); + if (unlikely(magic != GFS2_MAGIC)) { + gfs2_meta_check_ii(sdp, bh, function, + file, line); + return -EIO; + } + if (unlikely(t != type)) { + gfs2_metatype_check_ii(sdp, bh, type, t, function, + file, line); + return -EIO; + } + return 0; +} + +#define gfs2_metatype_check(sdp, bh, type) \ +gfs2_metatype_check_i((sdp), (bh), (type), __func__, __FILE__, __LINE__) + +static inline void gfs2_metatype_set(struct buffer_head *bh, u16 type, + u16 format) +{ + struct gfs2_meta_header *mh; + mh = (struct gfs2_meta_header *)bh->b_data; + mh->mh_type = cpu_to_be32(type); + mh->mh_format = cpu_to_be32(format); +} + + +void gfs2_io_error_i(struct gfs2_sbd *sdp, const char *function, + char *file, unsigned int line); + +int check_journal_clean(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd, + bool verbose); +int gfs2_freeze_lock_shared(struct gfs2_sbd *sdp); +void gfs2_freeze_unlock(struct gfs2_sbd *sdp); + +#define gfs2_io_error(sdp) \ +gfs2_io_error_i((sdp), __func__, __FILE__, __LINE__) + + +void gfs2_io_error_bh_i(struct gfs2_sbd *sdp, struct buffer_head *bh, + const char *function, char *file, unsigned int line); + +#define gfs2_io_error_bh(sdp, bh) \ +gfs2_io_error_bh_i((sdp), (bh), __func__, __FILE__, __LINE__) + + +extern struct kmem_cache *gfs2_glock_cachep; +extern struct kmem_cache *gfs2_glock_aspace_cachep; +extern struct kmem_cache *gfs2_inode_cachep; +extern struct kmem_cache *gfs2_bufdata_cachep; +extern struct kmem_cache *gfs2_rgrpd_cachep; +extern struct kmem_cache *gfs2_quotad_cachep; +extern struct kmem_cache *gfs2_qadata_cachep; +extern struct kmem_cache *gfs2_trans_cachep; +extern mempool_t *gfs2_page_pool; +extern struct workqueue_struct *gfs2_control_wq; + +static inline unsigned int gfs2_tune_get_i(struct gfs2_tune *gt, + unsigned int *p) +{ + unsigned int x; + spin_lock(>->gt_spin); + x = *p; + spin_unlock(>->gt_spin); + return x; +} + +/** + * gfs2_withdrawn - test whether the file system is withdrawn + * @sdp: the superblock + */ +static inline bool gfs2_withdrawn(struct gfs2_sbd *sdp) +{ + return unlikely(test_bit(SDF_WITHDRAWN, &sdp->sd_flags)); +} + +#define gfs2_tune_get(sdp, field) \ +gfs2_tune_get_i(&(sdp)->sd_tune, &(sdp)->sd_tune.field) + +__printf(2, 3) +void gfs2_lm(struct gfs2_sbd *sdp, const char *fmt, ...); + +void gfs2_withdraw_func(struct work_struct *work); +void gfs2_withdraw(struct gfs2_sbd *sdp); + +#endif /* __UTIL_DOT_H__ */ diff --git a/fs/gfs2/xattr.c b/fs/gfs2/xattr.c new file mode 100644 index 000000000..b9f48d6f1 --- /dev/null +++ b/fs/gfs2/xattr.c @@ -0,0 +1,1512 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gfs2.h" +#include "incore.h" +#include "acl.h" +#include "xattr.h" +#include "glock.h" +#include "inode.h" +#include "meta_io.h" +#include "quota.h" +#include "rgrp.h" +#include "super.h" +#include "trans.h" +#include "util.h" + +/* + * ea_calc_size - returns the actual number of bytes the request will take up + * (not counting any unstuffed data blocks) + * + * Returns: 1 if the EA should be stuffed + */ + +static int ea_calc_size(struct gfs2_sbd *sdp, unsigned int nsize, size_t dsize, + unsigned int *size) +{ + unsigned int jbsize = sdp->sd_jbsize; + + /* Stuffed */ + *size = ALIGN(sizeof(struct gfs2_ea_header) + nsize + dsize, 8); + + if (*size <= jbsize) + return 1; + + /* Unstuffed */ + *size = ALIGN(sizeof(struct gfs2_ea_header) + nsize + + (sizeof(__be64) * DIV_ROUND_UP(dsize, jbsize)), 8); + + return 0; +} + +static int ea_check_size(struct gfs2_sbd *sdp, unsigned int nsize, size_t dsize) +{ + unsigned int size; + + if (dsize > GFS2_EA_MAX_DATA_LEN) + return -ERANGE; + + ea_calc_size(sdp, nsize, dsize, &size); + + /* This can only happen with 512 byte blocks */ + if (size > sdp->sd_jbsize) + return -ERANGE; + + return 0; +} + +static bool gfs2_eatype_valid(struct gfs2_sbd *sdp, u8 type) +{ + switch(sdp->sd_sb.sb_fs_format) { + case GFS2_FS_FORMAT_MAX: + return true; + + case GFS2_FS_FORMAT_MIN: + return type <= GFS2_EATYPE_SECURITY; + + default: + return false; + } +} + +typedef int (*ea_call_t) (struct gfs2_inode *ip, struct buffer_head *bh, + struct gfs2_ea_header *ea, + struct gfs2_ea_header *prev, void *private); + +static int ea_foreach_i(struct gfs2_inode *ip, struct buffer_head *bh, + ea_call_t ea_call, void *data) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + struct gfs2_ea_header *ea, *prev = NULL; + int error = 0; + + if (gfs2_metatype_check(GFS2_SB(&ip->i_inode), bh, GFS2_METATYPE_EA)) + return -EIO; + + for (ea = GFS2_EA_BH2FIRST(bh);; prev = ea, ea = GFS2_EA2NEXT(ea)) { + if (!GFS2_EA_REC_LEN(ea)) { + gfs2_consist_inode(ip); + return -EIO; + } + if (!(bh->b_data <= (char *)ea && (char *)GFS2_EA2NEXT(ea) <= + bh->b_data + bh->b_size)) { + gfs2_consist_inode(ip); + return -EIO; + } + if (!gfs2_eatype_valid(sdp, ea->ea_type)) { + gfs2_consist_inode(ip); + return -EIO; + } + error = ea_call(ip, bh, ea, prev, data); + if (error) + return error; + + if (GFS2_EA_IS_LAST(ea)) { + if ((char *)GFS2_EA2NEXT(ea) != + bh->b_data + bh->b_size) { + gfs2_consist_inode(ip); + return -EIO; + } + break; + } + } + + return error; +} + +static int ea_foreach(struct gfs2_inode *ip, ea_call_t ea_call, void *data) +{ + struct buffer_head *bh, *eabh; + __be64 *eablk, *end; + int error; + + error = gfs2_meta_read(ip->i_gl, ip->i_eattr, DIO_WAIT, 0, &bh); + if (error) + return error; + + if (!(ip->i_diskflags & GFS2_DIF_EA_INDIRECT)) { + error = ea_foreach_i(ip, bh, ea_call, data); + goto out; + } + + if (gfs2_metatype_check(GFS2_SB(&ip->i_inode), bh, GFS2_METATYPE_IN)) { + error = -EIO; + goto out; + } + + eablk = (__be64 *)(bh->b_data + sizeof(struct gfs2_meta_header)); + end = eablk + GFS2_SB(&ip->i_inode)->sd_inptrs; + + for (; eablk < end; eablk++) { + u64 bn; + + if (!*eablk) + break; + bn = be64_to_cpu(*eablk); + + error = gfs2_meta_read(ip->i_gl, bn, DIO_WAIT, 0, &eabh); + if (error) + break; + error = ea_foreach_i(ip, eabh, ea_call, data); + brelse(eabh); + if (error) + break; + } +out: + brelse(bh); + return error; +} + +struct ea_find { + int type; + const char *name; + size_t namel; + struct gfs2_ea_location *ef_el; +}; + +static int ea_find_i(struct gfs2_inode *ip, struct buffer_head *bh, + struct gfs2_ea_header *ea, struct gfs2_ea_header *prev, + void *private) +{ + struct ea_find *ef = private; + + if (ea->ea_type == GFS2_EATYPE_UNUSED) + return 0; + + if (ea->ea_type == ef->type) { + if (ea->ea_name_len == ef->namel && + !memcmp(GFS2_EA2NAME(ea), ef->name, ea->ea_name_len)) { + struct gfs2_ea_location *el = ef->ef_el; + get_bh(bh); + el->el_bh = bh; + el->el_ea = ea; + el->el_prev = prev; + return 1; + } + } + + return 0; +} + +static int gfs2_ea_find(struct gfs2_inode *ip, int type, const char *name, + struct gfs2_ea_location *el) +{ + struct ea_find ef; + int error; + + ef.type = type; + ef.name = name; + ef.namel = strlen(name); + ef.ef_el = el; + + memset(el, 0, sizeof(struct gfs2_ea_location)); + + error = ea_foreach(ip, ea_find_i, &ef); + if (error > 0) + return 0; + + return error; +} + +/* + * ea_dealloc_unstuffed + * + * Take advantage of the fact that all unstuffed blocks are + * allocated from the same RG. But watch, this may not always + * be true. + * + * Returns: errno + */ + +static int ea_dealloc_unstuffed(struct gfs2_inode *ip, struct buffer_head *bh, + struct gfs2_ea_header *ea, + struct gfs2_ea_header *prev, void *private) +{ + int *leave = private; + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + struct gfs2_rgrpd *rgd; + struct gfs2_holder rg_gh; + __be64 *dataptrs; + u64 bn = 0; + u64 bstart = 0; + unsigned int blen = 0; + unsigned int blks = 0; + unsigned int x; + int error; + + error = gfs2_rindex_update(sdp); + if (error) + return error; + + if (GFS2_EA_IS_STUFFED(ea)) + return 0; + + dataptrs = GFS2_EA2DATAPTRS(ea); + for (x = 0; x < ea->ea_num_ptrs; x++, dataptrs++) { + if (*dataptrs) { + blks++; + bn = be64_to_cpu(*dataptrs); + } + } + if (!blks) + return 0; + + rgd = gfs2_blk2rgrpd(sdp, bn, 1); + if (!rgd) { + gfs2_consist_inode(ip); + return -EIO; + } + + error = gfs2_glock_nq_init(rgd->rd_gl, LM_ST_EXCLUSIVE, + LM_FLAG_NODE_SCOPE, &rg_gh); + if (error) + return error; + + error = gfs2_trans_begin(sdp, rgd->rd_length + RES_DINODE + + RES_EATTR + RES_STATFS + RES_QUOTA, blks); + if (error) + goto out_gunlock; + + gfs2_trans_add_meta(ip->i_gl, bh); + + dataptrs = GFS2_EA2DATAPTRS(ea); + for (x = 0; x < ea->ea_num_ptrs; x++, dataptrs++) { + if (!*dataptrs) + break; + bn = be64_to_cpu(*dataptrs); + + if (bstart + blen == bn) + blen++; + else { + if (bstart) + gfs2_free_meta(ip, rgd, bstart, blen); + bstart = bn; + blen = 1; + } + + *dataptrs = 0; + gfs2_add_inode_blocks(&ip->i_inode, -1); + } + if (bstart) + gfs2_free_meta(ip, rgd, bstart, blen); + + if (prev && !leave) { + u32 len; + + len = GFS2_EA_REC_LEN(prev) + GFS2_EA_REC_LEN(ea); + prev->ea_rec_len = cpu_to_be32(len); + + if (GFS2_EA_IS_LAST(ea)) + prev->ea_flags |= GFS2_EAFLAG_LAST; + } else { + ea->ea_type = GFS2_EATYPE_UNUSED; + ea->ea_num_ptrs = 0; + } + + inode_set_ctime_current(&ip->i_inode); + __mark_inode_dirty(&ip->i_inode, I_DIRTY_DATASYNC); + + gfs2_trans_end(sdp); + +out_gunlock: + gfs2_glock_dq_uninit(&rg_gh); + return error; +} + +static int ea_remove_unstuffed(struct gfs2_inode *ip, struct buffer_head *bh, + struct gfs2_ea_header *ea, + struct gfs2_ea_header *prev, int leave) +{ + int error; + + error = gfs2_rindex_update(GFS2_SB(&ip->i_inode)); + if (error) + return error; + + error = gfs2_quota_hold(ip, NO_UID_QUOTA_CHANGE, NO_GID_QUOTA_CHANGE); + if (error) + goto out_alloc; + + error = ea_dealloc_unstuffed(ip, bh, ea, prev, (leave) ? &error : NULL); + + gfs2_quota_unhold(ip); +out_alloc: + return error; +} + +struct ea_list { + struct gfs2_ea_request *ei_er; + unsigned int ei_size; +}; + +static int ea_list_i(struct gfs2_inode *ip, struct buffer_head *bh, + struct gfs2_ea_header *ea, struct gfs2_ea_header *prev, + void *private) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + struct ea_list *ei = private; + struct gfs2_ea_request *er = ei->ei_er; + unsigned int ea_size; + char *prefix; + unsigned int l; + + if (ea->ea_type == GFS2_EATYPE_UNUSED) + return 0; + + BUG_ON(ea->ea_type > GFS2_EATYPE_SECURITY && + sdp->sd_sb.sb_fs_format == GFS2_FS_FORMAT_MIN); + switch (ea->ea_type) { + case GFS2_EATYPE_USR: + prefix = "user."; + l = 5; + break; + case GFS2_EATYPE_SYS: + prefix = "system."; + l = 7; + break; + case GFS2_EATYPE_SECURITY: + prefix = "security."; + l = 9; + break; + case GFS2_EATYPE_TRUSTED: + prefix = "trusted."; + l = 8; + break; + default: + return 0; + } + + ea_size = l + ea->ea_name_len + 1; + if (er->er_data_len) { + if (ei->ei_size + ea_size > er->er_data_len) + return -ERANGE; + + memcpy(er->er_data + ei->ei_size, prefix, l); + memcpy(er->er_data + ei->ei_size + l, GFS2_EA2NAME(ea), + ea->ea_name_len); + er->er_data[ei->ei_size + ea_size - 1] = 0; + } + + ei->ei_size += ea_size; + + return 0; +} + +/** + * gfs2_listxattr - List gfs2 extended attributes + * @dentry: The dentry whose inode we are interested in + * @buffer: The buffer to write the results + * @size: The size of the buffer + * + * Returns: actual size of data on success, -errno on error + */ + +ssize_t gfs2_listxattr(struct dentry *dentry, char *buffer, size_t size) +{ + struct gfs2_inode *ip = GFS2_I(d_inode(dentry)); + struct gfs2_ea_request er; + struct gfs2_holder i_gh; + int error; + + memset(&er, 0, sizeof(struct gfs2_ea_request)); + if (size) { + er.er_data = buffer; + er.er_data_len = size; + } + + error = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, LM_FLAG_ANY, &i_gh); + if (error) + return error; + + if (ip->i_eattr) { + struct ea_list ei = { .ei_er = &er, .ei_size = 0 }; + + error = ea_foreach(ip, ea_list_i, &ei); + if (!error) + error = ei.ei_size; + } + + gfs2_glock_dq_uninit(&i_gh); + + return error; +} + +/** + * gfs2_iter_unstuffed - copies the unstuffed xattr data to/from the + * request buffer + * @ip: The GFS2 inode + * @ea: The extended attribute header structure + * @din: The data to be copied in + * @dout: The data to be copied out (one of din,dout will be NULL) + * + * Returns: errno + */ + +static int gfs2_iter_unstuffed(struct gfs2_inode *ip, struct gfs2_ea_header *ea, + const char *din, char *dout) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + struct buffer_head **bh; + unsigned int amount = GFS2_EA_DATA_LEN(ea); + unsigned int nptrs = DIV_ROUND_UP(amount, sdp->sd_jbsize); + __be64 *dataptrs = GFS2_EA2DATAPTRS(ea); + unsigned int x; + int error = 0; + unsigned char *pos; + unsigned cp_size; + + bh = kzalloc_objs(struct buffer_head *, nptrs, GFP_NOFS); + if (!bh) + return -ENOMEM; + + for (x = 0; x < nptrs; x++) { + error = gfs2_meta_read(ip->i_gl, be64_to_cpu(*dataptrs), 0, 0, + bh + x); + if (error) { + while (x--) + brelse(bh[x]); + goto out; + } + dataptrs++; + } + + for (x = 0; x < nptrs; x++) { + error = gfs2_meta_wait(sdp, bh[x]); + if (error) { + for (; x < nptrs; x++) + brelse(bh[x]); + goto out; + } + if (gfs2_metatype_check(sdp, bh[x], GFS2_METATYPE_ED)) { + for (; x < nptrs; x++) + brelse(bh[x]); + error = -EIO; + goto out; + } + + pos = bh[x]->b_data + sizeof(struct gfs2_meta_header); + cp_size = (sdp->sd_jbsize > amount) ? amount : sdp->sd_jbsize; + + if (dout) { + memcpy(dout, pos, cp_size); + dout += sdp->sd_jbsize; + } + + if (din) { + gfs2_trans_add_meta(ip->i_gl, bh[x]); + memcpy(pos, din, cp_size); + din += sdp->sd_jbsize; + } + + amount -= sdp->sd_jbsize; + brelse(bh[x]); + } + +out: + kfree(bh); + return error; +} + +static int gfs2_ea_get_copy(struct gfs2_inode *ip, struct gfs2_ea_location *el, + char *data, size_t size) +{ + int ret; + size_t len = GFS2_EA_DATA_LEN(el->el_ea); + if (len > size) + return -ERANGE; + + if (GFS2_EA_IS_STUFFED(el->el_ea)) { + memcpy(data, GFS2_EA2DATA(el->el_ea), len); + return len; + } + ret = gfs2_iter_unstuffed(ip, el->el_ea, NULL, data); + if (ret < 0) + return ret; + return len; +} + +int gfs2_xattr_acl_get(struct gfs2_inode *ip, const char *name, char **ppdata) +{ + struct gfs2_ea_location el; + int error; + int len; + char *data; + + error = gfs2_ea_find(ip, GFS2_EATYPE_SYS, name, &el); + if (error) + return error; + if (!el.el_ea) + goto out; + if (!GFS2_EA_DATA_LEN(el.el_ea)) + goto out; + + len = GFS2_EA_DATA_LEN(el.el_ea); + data = kmalloc(len, GFP_NOFS); + error = -ENOMEM; + if (data == NULL) + goto out; + + error = gfs2_ea_get_copy(ip, &el, data, len); + if (error < 0) + kfree(data); + else + *ppdata = data; +out: + brelse(el.el_bh); + return error; +} + +/** + * __gfs2_xattr_get - Get a GFS2 extended attribute + * @inode: The inode + * @name: The name of the extended attribute + * @buffer: The buffer to write the result into + * @size: The size of the buffer + * @type: The type of extended attribute + * + * Returns: actual size of data on success, -errno on error + */ +static int __gfs2_xattr_get(struct inode *inode, const char *name, + void *buffer, size_t size, int type) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_ea_location el; + int error; + + if (!ip->i_eattr) + return -ENODATA; + if (strlen(name) > GFS2_EA_MAX_NAME_LEN) + return -EINVAL; + + error = gfs2_ea_find(ip, type, name, &el); + if (error) + return error; + if (!el.el_ea) + return -ENODATA; + if (size) + error = gfs2_ea_get_copy(ip, &el, buffer, size); + else + error = GFS2_EA_DATA_LEN(el.el_ea); + brelse(el.el_bh); + + return error; +} + +static int gfs2_xattr_get(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_holder gh; + int ret; + + /* During lookup, SELinux calls this function with the glock locked. */ + + if (!gfs2_glock_is_locked_by_me(ip->i_gl)) { + ret = gfs2_glock_nq_init(ip->i_gl, LM_ST_SHARED, LM_FLAG_ANY, &gh); + if (ret) + return ret; + } else { + gfs2_holder_mark_uninitialized(&gh); + } + ret = __gfs2_xattr_get(inode, name, buffer, size, handler->flags); + if (gfs2_holder_initialized(&gh)) + gfs2_glock_dq_uninit(&gh); + return ret; +} + +/** + * ea_alloc_blk - allocates a new block for extended attributes. + * @ip: A pointer to the inode that's getting extended attributes + * @bhp: Pointer to pointer to a struct buffer_head + * + * Returns: errno + */ + +static int ea_alloc_blk(struct gfs2_inode *ip, struct buffer_head **bhp) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + struct gfs2_ea_header *ea; + unsigned int n = 1; + u64 block; + int error; + + error = gfs2_alloc_blocks(ip, &block, &n, 0); + if (error) + return error; + gfs2_trans_remove_revoke(sdp, block, 1); + *bhp = gfs2_meta_new(ip->i_gl, block); + gfs2_trans_add_meta(ip->i_gl, *bhp); + gfs2_metatype_set(*bhp, GFS2_METATYPE_EA, GFS2_FORMAT_EA); + gfs2_buffer_clear_tail(*bhp, sizeof(struct gfs2_meta_header)); + + ea = GFS2_EA_BH2FIRST(*bhp); + ea->ea_rec_len = cpu_to_be32(sdp->sd_jbsize); + ea->ea_type = GFS2_EATYPE_UNUSED; + ea->ea_flags = GFS2_EAFLAG_LAST; + ea->ea_num_ptrs = 0; + + gfs2_add_inode_blocks(&ip->i_inode, 1); + + return 0; +} + +/** + * ea_write - writes the request info to an ea, creating new blocks if + * necessary + * @ip: inode that is being modified + * @ea: the location of the new ea in a block + * @er: the write request + * + * Note: does not update ea_rec_len or the GFS2_EAFLAG_LAST bin of ea_flags + * + * returns : errno + */ + +static int ea_write(struct gfs2_inode *ip, struct gfs2_ea_header *ea, + struct gfs2_ea_request *er) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + int error; + + ea->ea_data_len = cpu_to_be32(er->er_data_len); + ea->ea_name_len = er->er_name_len; + ea->ea_type = er->er_type; + ea->__pad = 0; + + memcpy(GFS2_EA2NAME(ea), er->er_name, er->er_name_len); + + if (GFS2_EAREQ_SIZE_STUFFED(er) <= sdp->sd_jbsize) { + ea->ea_num_ptrs = 0; + memcpy(GFS2_EA2DATA(ea), er->er_data, er->er_data_len); + } else { + __be64 *dataptr = GFS2_EA2DATAPTRS(ea); + const char *data = er->er_data; + unsigned int data_len = er->er_data_len; + unsigned int copy; + unsigned int x; + + ea->ea_num_ptrs = DIV_ROUND_UP(er->er_data_len, sdp->sd_jbsize); + for (x = 0; x < ea->ea_num_ptrs; x++) { + struct buffer_head *bh; + u64 block; + int mh_size = sizeof(struct gfs2_meta_header); + unsigned int n = 1; + + error = gfs2_alloc_blocks(ip, &block, &n, 0); + if (error) + return error; + gfs2_trans_remove_revoke(sdp, block, 1); + bh = gfs2_meta_new(ip->i_gl, block); + gfs2_trans_add_meta(ip->i_gl, bh); + gfs2_metatype_set(bh, GFS2_METATYPE_ED, GFS2_FORMAT_ED); + + gfs2_add_inode_blocks(&ip->i_inode, 1); + + copy = data_len > sdp->sd_jbsize ? sdp->sd_jbsize : + data_len; + memcpy(bh->b_data + mh_size, data, copy); + if (copy < sdp->sd_jbsize) + memset(bh->b_data + mh_size + copy, 0, + sdp->sd_jbsize - copy); + + *dataptr++ = cpu_to_be64(bh->b_blocknr); + data += copy; + data_len -= copy; + + brelse(bh); + } + + gfs2_assert_withdraw(sdp, !data_len); + } + + return 0; +} + +typedef int (*ea_skeleton_call_t) (struct gfs2_inode *ip, + struct gfs2_ea_request *er, void *private); + +static int ea_alloc_skeleton(struct gfs2_inode *ip, struct gfs2_ea_request *er, + unsigned int blks, + ea_skeleton_call_t skeleton_call, void *private) +{ + struct gfs2_alloc_parms ap = { .target = blks }; + int error; + + error = gfs2_rindex_update(GFS2_SB(&ip->i_inode)); + if (error) + return error; + + error = gfs2_quota_lock_check(ip, &ap); + if (error) + return error; + + error = gfs2_inplace_reserve(ip, &ap); + if (error) + goto out_gunlock_q; + + error = gfs2_trans_begin(GFS2_SB(&ip->i_inode), + blks + gfs2_rg_blocks(ip, blks) + + RES_DINODE + RES_STATFS + RES_QUOTA, 0); + if (error) + goto out_ipres; + + error = skeleton_call(ip, er, private); + if (error) + goto out_end_trans; + + inode_set_ctime_current(&ip->i_inode); + __mark_inode_dirty(&ip->i_inode, I_DIRTY_DATASYNC); + +out_end_trans: + gfs2_trans_end(GFS2_SB(&ip->i_inode)); +out_ipres: + gfs2_inplace_release(ip); +out_gunlock_q: + gfs2_quota_unlock(ip); + return error; +} + +static int ea_init_i(struct gfs2_inode *ip, struct gfs2_ea_request *er, + void *private) +{ + struct buffer_head *bh; + int error; + + error = ea_alloc_blk(ip, &bh); + if (error) + return error; + + ip->i_eattr = bh->b_blocknr; + error = ea_write(ip, GFS2_EA_BH2FIRST(bh), er); + + brelse(bh); + + return error; +} + +/* + * ea_init - initializes a new eattr block + * + * Returns: errno + */ +static int ea_init(struct gfs2_inode *ip, int type, const char *name, + const void *data, size_t size) +{ + struct gfs2_ea_request er; + unsigned int jbsize = GFS2_SB(&ip->i_inode)->sd_jbsize; + unsigned int blks = 1; + + er.er_type = type; + er.er_name = name; + er.er_name_len = strlen(name); + er.er_data = (void *)data; + er.er_data_len = size; + + if (GFS2_EAREQ_SIZE_STUFFED(&er) > jbsize) + blks += DIV_ROUND_UP(er.er_data_len, jbsize); + + return ea_alloc_skeleton(ip, &er, blks, ea_init_i, NULL); +} + +static struct gfs2_ea_header *ea_split_ea(struct gfs2_ea_header *ea) +{ + u32 ea_size = GFS2_EA_SIZE(ea); + struct gfs2_ea_header *new = (struct gfs2_ea_header *)((char *)ea + + ea_size); + u32 new_size = GFS2_EA_REC_LEN(ea) - ea_size; + int last = ea->ea_flags & GFS2_EAFLAG_LAST; + + ea->ea_rec_len = cpu_to_be32(ea_size); + ea->ea_flags ^= last; + + new->ea_rec_len = cpu_to_be32(new_size); + new->ea_flags = last; + + return new; +} + +static void ea_set_remove_stuffed(struct gfs2_inode *ip, + struct gfs2_ea_location *el) +{ + struct gfs2_ea_header *ea = el->el_ea; + struct gfs2_ea_header *prev = el->el_prev; + u32 len; + + gfs2_trans_add_meta(ip->i_gl, el->el_bh); + + if (!prev || !GFS2_EA_IS_STUFFED(ea)) { + ea->ea_type = GFS2_EATYPE_UNUSED; + return; + } else if (GFS2_EA2NEXT(prev) != ea) { + prev = GFS2_EA2NEXT(prev); + gfs2_assert_withdraw(GFS2_SB(&ip->i_inode), GFS2_EA2NEXT(prev) == ea); + } + + len = GFS2_EA_REC_LEN(prev) + GFS2_EA_REC_LEN(ea); + prev->ea_rec_len = cpu_to_be32(len); + + if (GFS2_EA_IS_LAST(ea)) + prev->ea_flags |= GFS2_EAFLAG_LAST; +} + +struct ea_set { + int ea_split; + + struct gfs2_ea_request *es_er; + struct gfs2_ea_location *es_el; + + struct buffer_head *es_bh; + struct gfs2_ea_header *es_ea; +}; + +static int ea_set_simple_noalloc(struct gfs2_inode *ip, struct buffer_head *bh, + struct gfs2_ea_header *ea, struct ea_set *es) +{ + struct gfs2_ea_request *er = es->es_er; + int error; + + error = gfs2_trans_begin(GFS2_SB(&ip->i_inode), RES_DINODE + 2 * RES_EATTR, 0); + if (error) + return error; + + gfs2_trans_add_meta(ip->i_gl, bh); + + if (es->ea_split) + ea = ea_split_ea(ea); + + ea_write(ip, ea, er); + + if (es->es_el) + ea_set_remove_stuffed(ip, es->es_el); + + inode_set_ctime_current(&ip->i_inode); + __mark_inode_dirty(&ip->i_inode, I_DIRTY_DATASYNC); + + gfs2_trans_end(GFS2_SB(&ip->i_inode)); + return error; +} + +static int ea_set_simple_alloc(struct gfs2_inode *ip, + struct gfs2_ea_request *er, void *private) +{ + struct ea_set *es = private; + struct gfs2_ea_header *ea = es->es_ea; + int error; + + gfs2_trans_add_meta(ip->i_gl, es->es_bh); + + if (es->ea_split) + ea = ea_split_ea(ea); + + error = ea_write(ip, ea, er); + if (error) + return error; + + if (es->es_el) + ea_set_remove_stuffed(ip, es->es_el); + + return 0; +} + +static int ea_set_simple(struct gfs2_inode *ip, struct buffer_head *bh, + struct gfs2_ea_header *ea, struct gfs2_ea_header *prev, + void *private) +{ + struct ea_set *es = private; + unsigned int size; + int stuffed; + int error; + + stuffed = ea_calc_size(GFS2_SB(&ip->i_inode), es->es_er->er_name_len, + es->es_er->er_data_len, &size); + + if (ea->ea_type == GFS2_EATYPE_UNUSED) { + if (GFS2_EA_REC_LEN(ea) < size) + return 0; + if (!GFS2_EA_IS_STUFFED(ea)) { + error = ea_remove_unstuffed(ip, bh, ea, prev, 1); + if (error) + return error; + } + es->ea_split = 0; + } else if (GFS2_EA_REC_LEN(ea) - GFS2_EA_SIZE(ea) >= size) + es->ea_split = 1; + else + return 0; + + if (stuffed) { + error = ea_set_simple_noalloc(ip, bh, ea, es); + if (error) + return error; + } else { + unsigned int blks; + + es->es_bh = bh; + es->es_ea = ea; + blks = 2 + DIV_ROUND_UP(es->es_er->er_data_len, + GFS2_SB(&ip->i_inode)->sd_jbsize); + + error = ea_alloc_skeleton(ip, es->es_er, blks, + ea_set_simple_alloc, es); + if (error) + return error; + } + + return 1; +} + +static int ea_set_block(struct gfs2_inode *ip, struct gfs2_ea_request *er, + void *private) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + struct buffer_head *indbh, *newbh; + __be64 *eablk; + int error; + int mh_size = sizeof(struct gfs2_meta_header); + + if (ip->i_diskflags & GFS2_DIF_EA_INDIRECT) { + __be64 *end; + + error = gfs2_meta_read(ip->i_gl, ip->i_eattr, DIO_WAIT, 0, + &indbh); + if (error) + return error; + + if (gfs2_metatype_check(sdp, indbh, GFS2_METATYPE_IN)) { + error = -EIO; + goto out; + } + + eablk = (__be64 *)(indbh->b_data + mh_size); + end = eablk + sdp->sd_inptrs; + + for (; eablk < end; eablk++) + if (!*eablk) + break; + + if (eablk == end) { + error = -ENOSPC; + goto out; + } + + gfs2_trans_add_meta(ip->i_gl, indbh); + } else { + u64 blk; + unsigned int n = 1; + error = gfs2_alloc_blocks(ip, &blk, &n, 0); + if (error) + return error; + gfs2_trans_remove_revoke(sdp, blk, 1); + indbh = gfs2_meta_new(ip->i_gl, blk); + gfs2_trans_add_meta(ip->i_gl, indbh); + gfs2_metatype_set(indbh, GFS2_METATYPE_IN, GFS2_FORMAT_IN); + gfs2_buffer_clear_tail(indbh, mh_size); + + eablk = (__be64 *)(indbh->b_data + mh_size); + *eablk = cpu_to_be64(ip->i_eattr); + ip->i_eattr = blk; + ip->i_diskflags |= GFS2_DIF_EA_INDIRECT; + gfs2_add_inode_blocks(&ip->i_inode, 1); + + eablk++; + } + + error = ea_alloc_blk(ip, &newbh); + if (error) + goto out; + + *eablk = cpu_to_be64((u64)newbh->b_blocknr); + error = ea_write(ip, GFS2_EA_BH2FIRST(newbh), er); + brelse(newbh); + if (error) + goto out; + + if (private) + ea_set_remove_stuffed(ip, private); + +out: + brelse(indbh); + return error; +} + +static int ea_set_i(struct gfs2_inode *ip, int type, const char *name, + const void *value, size_t size, struct gfs2_ea_location *el) +{ + struct gfs2_ea_request er; + struct ea_set es; + unsigned int blks = 2; + int error; + + er.er_type = type; + er.er_name = name; + er.er_data = (void *)value; + er.er_name_len = strlen(name); + er.er_data_len = size; + + memset(&es, 0, sizeof(struct ea_set)); + es.es_er = &er; + es.es_el = el; + + error = ea_foreach(ip, ea_set_simple, &es); + if (error > 0) + return 0; + if (error) + return error; + + if (!(ip->i_diskflags & GFS2_DIF_EA_INDIRECT)) + blks++; + if (GFS2_EAREQ_SIZE_STUFFED(&er) > GFS2_SB(&ip->i_inode)->sd_jbsize) + blks += DIV_ROUND_UP(er.er_data_len, GFS2_SB(&ip->i_inode)->sd_jbsize); + + return ea_alloc_skeleton(ip, &er, blks, ea_set_block, el); +} + +static int ea_set_remove_unstuffed(struct gfs2_inode *ip, + struct gfs2_ea_location *el) +{ + if (el->el_prev && GFS2_EA2NEXT(el->el_prev) != el->el_ea) { + el->el_prev = GFS2_EA2NEXT(el->el_prev); + gfs2_assert_withdraw(GFS2_SB(&ip->i_inode), + GFS2_EA2NEXT(el->el_prev) == el->el_ea); + } + + return ea_remove_unstuffed(ip, el->el_bh, el->el_ea, el->el_prev, 0); +} + +static int ea_remove_stuffed(struct gfs2_inode *ip, struct gfs2_ea_location *el) +{ + struct gfs2_ea_header *ea = el->el_ea; + struct gfs2_ea_header *prev = el->el_prev; + int error; + + error = gfs2_trans_begin(GFS2_SB(&ip->i_inode), RES_DINODE + RES_EATTR, 0); + if (error) + return error; + + gfs2_trans_add_meta(ip->i_gl, el->el_bh); + + if (prev) { + u32 len; + + len = GFS2_EA_REC_LEN(prev) + GFS2_EA_REC_LEN(ea); + prev->ea_rec_len = cpu_to_be32(len); + + if (GFS2_EA_IS_LAST(ea)) + prev->ea_flags |= GFS2_EAFLAG_LAST; + } else { + ea->ea_type = GFS2_EATYPE_UNUSED; + } + + inode_set_ctime_current(&ip->i_inode); + __mark_inode_dirty(&ip->i_inode, I_DIRTY_DATASYNC); + + gfs2_trans_end(GFS2_SB(&ip->i_inode)); + + return error; +} + +/** + * gfs2_xattr_remove - Remove a GFS2 extended attribute + * @ip: The inode + * @type: The type of the extended attribute + * @name: The name of the extended attribute + * + * This is not called directly by the VFS since we use the (common) + * scheme of making a "set with NULL data" mean a remove request. Note + * that this is different from a set with zero length data. + * + * Returns: 0, or errno on failure + */ + +static int gfs2_xattr_remove(struct gfs2_inode *ip, int type, const char *name) +{ + struct gfs2_ea_location el; + int error; + + if (!ip->i_eattr) + return -ENODATA; + + error = gfs2_ea_find(ip, type, name, &el); + if (error) + return error; + if (!el.el_ea) + return -ENODATA; + + if (GFS2_EA_IS_STUFFED(el.el_ea)) + error = ea_remove_stuffed(ip, &el); + else + error = ea_remove_unstuffed(ip, el.el_bh, el.el_ea, el.el_prev, 0); + + brelse(el.el_bh); + + return error; +} + +/** + * __gfs2_xattr_set - Set (or remove) a GFS2 extended attribute + * @inode: The inode + * @name: The name of the extended attribute + * @value: The value of the extended attribute (NULL for remove) + * @size: The size of the @value argument + * @flags: Create or Replace + * @type: The type of the extended attribute + * + * See gfs2_xattr_remove() for details of the removal of xattrs. + * + * Returns: 0 or errno on failure + */ + +int __gfs2_xattr_set(struct inode *inode, const char *name, + const void *value, size_t size, int flags, int type) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_sbd *sdp = GFS2_SB(inode); + struct gfs2_ea_location el; + unsigned int namel = strlen(name); + int error; + + if (IS_IMMUTABLE(inode) || IS_APPEND(inode)) + return -EPERM; + if (namel > GFS2_EA_MAX_NAME_LEN) + return -ERANGE; + + if (value == NULL) { + error = gfs2_xattr_remove(ip, type, name); + if (error == -ENODATA && !(flags & XATTR_REPLACE)) + error = 0; + return error; + } + + if (ea_check_size(sdp, namel, size)) + return -ERANGE; + + if (!ip->i_eattr) { + if (flags & XATTR_REPLACE) + return -ENODATA; + return ea_init(ip, type, name, value, size); + } + + error = gfs2_ea_find(ip, type, name, &el); + if (error) + return error; + + if (el.el_ea) { + if (ip->i_diskflags & GFS2_DIF_APPENDONLY) { + brelse(el.el_bh); + return -EPERM; + } + + error = -EEXIST; + if (!(flags & XATTR_CREATE)) { + int unstuffed = !GFS2_EA_IS_STUFFED(el.el_ea); + error = ea_set_i(ip, type, name, value, size, &el); + if (!error && unstuffed) + ea_set_remove_unstuffed(ip, &el); + } + + brelse(el.el_bh); + return error; + } + + error = -ENODATA; + if (!(flags & XATTR_REPLACE)) + error = ea_set_i(ip, type, name, value, size, NULL); + + return error; +} + +static int gfs2_xattr_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +{ + struct gfs2_inode *ip = GFS2_I(inode); + struct gfs2_holder gh; + int ret; + + ret = gfs2_qa_get(ip); + if (ret) + return ret; + + /* May be called from gfs_setattr with the glock locked. */ + + if (!gfs2_glock_is_locked_by_me(ip->i_gl)) { + ret = gfs2_glock_nq_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &gh); + if (ret) + goto out; + } else { + if (WARN_ON_ONCE(ip->i_gl->gl_state != LM_ST_EXCLUSIVE)) { + ret = -EIO; + goto out; + } + gfs2_holder_mark_uninitialized(&gh); + } + ret = __gfs2_xattr_set(inode, name, value, size, flags, handler->flags); + if (gfs2_holder_initialized(&gh)) + gfs2_glock_dq_uninit(&gh); +out: + gfs2_qa_put(ip); + return ret; +} + +static int ea_dealloc_indirect(struct gfs2_inode *ip) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + struct gfs2_rgrp_list rlist; + struct gfs2_rgrpd *rgd; + struct buffer_head *indbh, *dibh; + __be64 *eablk, *end; + unsigned int rg_blocks = 0; + u64 bstart = 0; + unsigned int blen = 0; + unsigned int blks = 0; + unsigned int x; + int error; + + error = gfs2_rindex_update(sdp); + if (error) + return error; + + memset(&rlist, 0, sizeof(struct gfs2_rgrp_list)); + + error = gfs2_meta_read(ip->i_gl, ip->i_eattr, DIO_WAIT, 0, &indbh); + if (error) + return error; + + if (gfs2_metatype_check(sdp, indbh, GFS2_METATYPE_IN)) { + error = -EIO; + goto out; + } + + eablk = (__be64 *)(indbh->b_data + sizeof(struct gfs2_meta_header)); + end = eablk + sdp->sd_inptrs; + + for (; eablk < end; eablk++) { + u64 bn; + + if (!*eablk) + break; + bn = be64_to_cpu(*eablk); + + if (bstart + blen == bn) + blen++; + else { + if (bstart) + gfs2_rlist_add(ip, &rlist, bstart); + bstart = bn; + blen = 1; + } + blks++; + } + if (bstart) + gfs2_rlist_add(ip, &rlist, bstart); + else + goto out; + + gfs2_rlist_alloc(&rlist, LM_ST_EXCLUSIVE, LM_FLAG_NODE_SCOPE); + + for (x = 0; x < rlist.rl_rgrps; x++) { + rgd = gfs2_glock2rgrp(rlist.rl_ghs[x].gh_gl); + rg_blocks += rgd->rd_length; + } + + error = gfs2_glock_nq_m(rlist.rl_rgrps, rlist.rl_ghs); + if (error) + goto out_rlist_free; + + error = gfs2_trans_begin(sdp, rg_blocks + RES_DINODE + RES_INDIRECT + + RES_STATFS + RES_QUOTA, blks); + if (error) + goto out_gunlock; + + gfs2_trans_add_meta(ip->i_gl, indbh); + + eablk = (__be64 *)(indbh->b_data + sizeof(struct gfs2_meta_header)); + bstart = 0; + rgd = NULL; + blen = 0; + + for (; eablk < end; eablk++) { + u64 bn; + + if (!*eablk) + break; + bn = be64_to_cpu(*eablk); + + if (bstart + blen == bn) + blen++; + else { + if (bstart) + gfs2_free_meta(ip, rgd, bstart, blen); + bstart = bn; + rgd = gfs2_blk2rgrpd(sdp, bstart, true); + blen = 1; + } + + *eablk = 0; + gfs2_add_inode_blocks(&ip->i_inode, -1); + } + if (bstart) + gfs2_free_meta(ip, rgd, bstart, blen); + + ip->i_diskflags &= ~GFS2_DIF_EA_INDIRECT; + + error = gfs2_meta_inode_buffer(ip, &dibh); + if (!error) { + gfs2_trans_add_meta(ip->i_gl, dibh); + gfs2_dinode_out(ip, dibh->b_data); + brelse(dibh); + } + + gfs2_trans_end(sdp); + +out_gunlock: + gfs2_glock_dq_m(rlist.rl_rgrps, rlist.rl_ghs); +out_rlist_free: + gfs2_rlist_free(&rlist); +out: + brelse(indbh); + return error; +} + +static int ea_dealloc_block(struct gfs2_inode *ip, bool initialized) +{ + struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode); + struct gfs2_rgrpd *rgd; + struct buffer_head *dibh; + struct gfs2_holder gh; + int error; + + error = gfs2_rindex_update(sdp); + if (error) + return error; + + rgd = gfs2_blk2rgrpd(sdp, ip->i_eattr, 1); + if (!rgd) { + gfs2_consist_inode(ip); + return -EIO; + } + + error = gfs2_glock_nq_init(rgd->rd_gl, LM_ST_EXCLUSIVE, + LM_FLAG_NODE_SCOPE, &gh); + if (error) + return error; + + error = gfs2_trans_begin(sdp, RES_RG_BIT + RES_DINODE + RES_STATFS + + RES_QUOTA, 1); + if (error) + goto out_gunlock; + + gfs2_free_meta(ip, rgd, ip->i_eattr, 1); + + ip->i_eattr = 0; + gfs2_add_inode_blocks(&ip->i_inode, -1); + + if (initialized) { + error = gfs2_meta_inode_buffer(ip, &dibh); + if (!error) { + gfs2_trans_add_meta(ip->i_gl, dibh); + gfs2_dinode_out(ip, dibh->b_data); + brelse(dibh); + } + } + + gfs2_trans_end(sdp); + +out_gunlock: + gfs2_glock_dq_uninit(&gh); + return error; +} + +/** + * gfs2_ea_dealloc - deallocate the extended attribute fork + * @ip: the inode + * @initialized: xattrs have been initialized + * + * Returns: errno + */ + +int gfs2_ea_dealloc(struct gfs2_inode *ip, bool initialized) +{ + int error; + + error = gfs2_rindex_update(GFS2_SB(&ip->i_inode)); + if (error) + return error; + + error = gfs2_quota_hold(ip, NO_UID_QUOTA_CHANGE, NO_GID_QUOTA_CHANGE); + if (error) + return error; + + if (initialized) { + error = ea_foreach(ip, ea_dealloc_unstuffed, NULL); + if (error) + goto out_quota; + + if (ip->i_diskflags & GFS2_DIF_EA_INDIRECT) { + error = ea_dealloc_indirect(ip); + if (error) + goto out_quota; + } + } + + error = ea_dealloc_block(ip, initialized); + +out_quota: + gfs2_quota_unhold(ip); + return error; +} + +static const struct xattr_handler gfs2_xattr_user_handler = { + .prefix = XATTR_USER_PREFIX, + .flags = GFS2_EATYPE_USR, + .get = gfs2_xattr_get, + .set = gfs2_xattr_set, +}; + +static const struct xattr_handler gfs2_xattr_security_handler = { + .prefix = XATTR_SECURITY_PREFIX, + .flags = GFS2_EATYPE_SECURITY, + .get = gfs2_xattr_get, + .set = gfs2_xattr_set, +}; + +static bool +gfs2_xattr_trusted_list(struct dentry *dentry) +{ + return capable(CAP_SYS_ADMIN); +} + +static const struct xattr_handler gfs2_xattr_trusted_handler = { + .prefix = XATTR_TRUSTED_PREFIX, + .flags = GFS2_EATYPE_TRUSTED, + .list = gfs2_xattr_trusted_list, + .get = gfs2_xattr_get, + .set = gfs2_xattr_set, +}; + +const struct xattr_handler * const gfs2_xattr_handlers_max[] = { + /* GFS2_FS_FORMAT_MAX */ + &gfs2_xattr_trusted_handler, + + /* GFS2_FS_FORMAT_MIN */ + &gfs2_xattr_user_handler, + &gfs2_xattr_security_handler, + NULL, +}; + +const struct xattr_handler * const *gfs2_xattr_handlers_min = gfs2_xattr_handlers_max + 1; diff --git a/fs/gfs2/xattr.h b/fs/gfs2/xattr.h new file mode 100644 index 000000000..3c9788e0e --- /dev/null +++ b/fs/gfs2/xattr.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. + * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. + */ + +#ifndef __EATTR_DOT_H__ +#define __EATTR_DOT_H__ + +struct gfs2_inode; +struct iattr; + +#define GFS2_EA_REC_LEN(ea) be32_to_cpu((ea)->ea_rec_len) +#define GFS2_EA_DATA_LEN(ea) be32_to_cpu((ea)->ea_data_len) + +#define GFS2_EA_SIZE(ea) \ +ALIGN(sizeof(struct gfs2_ea_header) + (ea)->ea_name_len + \ + ((GFS2_EA_IS_STUFFED(ea)) ? GFS2_EA_DATA_LEN(ea) : \ + (sizeof(__be64) * (ea)->ea_num_ptrs)), 8) + +#define GFS2_EA_IS_STUFFED(ea) (!(ea)->ea_num_ptrs) +#define GFS2_EA_IS_LAST(ea) ((ea)->ea_flags & GFS2_EAFLAG_LAST) + +#define GFS2_EAREQ_SIZE_STUFFED(er) \ +ALIGN(sizeof(struct gfs2_ea_header) + (er)->er_name_len + (er)->er_data_len, 8) + +#define GFS2_EA2NAME(ea) ((char *)((struct gfs2_ea_header *)(ea) + 1)) +#define GFS2_EA2DATA(ea) (GFS2_EA2NAME(ea) + (ea)->ea_name_len) + +#define GFS2_EA2DATAPTRS(ea) \ +((__be64 *)(GFS2_EA2NAME(ea) + ALIGN((ea)->ea_name_len, 8))) + +#define GFS2_EA2NEXT(ea) \ +((struct gfs2_ea_header *)((char *)(ea) + GFS2_EA_REC_LEN(ea))) + +#define GFS2_EA_BH2FIRST(bh) \ +((struct gfs2_ea_header *)((bh)->b_data + sizeof(struct gfs2_meta_header))) + +struct gfs2_ea_request { + const char *er_name; + char *er_data; + unsigned int er_name_len; + unsigned int er_data_len; + unsigned int er_type; /* GFS2_EATYPE_... */ +}; + +struct gfs2_ea_location { + struct buffer_head *el_bh; + struct gfs2_ea_header *el_ea; + struct gfs2_ea_header *el_prev; +}; + +int __gfs2_xattr_set(struct inode *inode, const char *name, + const void *value, size_t size, + int flags, int type); +ssize_t gfs2_listxattr(struct dentry *dentry, char *buffer, size_t size); +int gfs2_ea_dealloc(struct gfs2_inode *ip, bool initialized); + +/* Exported to acl.c */ + +int gfs2_xattr_acl_get(struct gfs2_inode *ip, const char *name, char **data); + +#endif /* __EATTR_DOT_H__ */ diff --git a/fs/hfs/.kunitconfig b/fs/hfs/.kunitconfig new file mode 100644 index 000000000..5caa9af1e --- /dev/null +++ b/fs/hfs/.kunitconfig @@ -0,0 +1,7 @@ +CONFIG_KUNIT=y +CONFIG_HFS_FS=y +CONFIG_HFS_KUNIT_TEST=y +CONFIG_BLOCK=y +CONFIG_BUFFER_HEAD=y +CONFIG_NLS=y +CONFIG_LEGACY_DIRECT_IO=y diff --git a/fs/hfs/Kconfig b/fs/hfs/Kconfig new file mode 100644 index 000000000..7f3cbe43b --- /dev/null +++ b/fs/hfs/Kconfig @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: GPL-2.0-only +config HFS_FS + tristate "Apple Macintosh file system support" + depends on BLOCK + select BUFFER_HEAD + select NLS + select LEGACY_DIRECT_IO + help + If you say Y here, you will be able to mount Macintosh-formatted + floppy disks and hard drive partitions with full read-write access. + Please read to learn about + the available mount options. + + To compile this file system support as a module, choose M here: the + module will be called hfs. + +config HFS_KUNIT_TEST + tristate "KUnit tests for HFS filesystem" if !KUNIT_ALL_TESTS + depends on HFS_FS && KUNIT + default KUNIT_ALL_TESTS + help + This builds KUnit tests for the HFS filesystem. + + KUnit tests run during boot and output the results to the debug + log in TAP format (https://testanything.org/). Only useful for + kernel devs running KUnit test harness and are not for inclusion + into a production build. + + For more information on KUnit and unit tests in general please + refer to the KUnit documentation in Documentation/dev-tools/kunit/. diff --git a/fs/hfs/Makefile b/fs/hfs/Makefile new file mode 100644 index 000000000..a7c9ce6b4 --- /dev/null +++ b/fs/hfs/Makefile @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the Linux hfs filesystem routines. +# + +obj-$(CONFIG_HFS_FS) += hfs.o + +hfs-objs := bitmap.o bfind.o bnode.o brec.o btree.o \ + catalog.o dir.o extent.o inode.o attr.o mdb.o \ + part_tbl.o string.o super.o sysdep.o trans.o + +# KUnit tests +obj-$(CONFIG_HFS_KUNIT_TEST) += string_test.o diff --git a/fs/hfs/attr.c b/fs/hfs/attr.c new file mode 100644 index 000000000..f8395cdd1 --- /dev/null +++ b/fs/hfs/attr.c @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hfs/attr.c + * + * (C) 2003 Ardis Technologies + * + * Export hfs data via xattr + */ + + +#include +#include + +#include "hfs_fs.h" +#include "btree.h" + +enum hfs_xattr_type { + HFS_TYPE, + HFS_CREATOR, +}; + +static int __hfs_setxattr(struct inode *inode, enum hfs_xattr_type type, + const void *value, size_t size, int flags) +{ + struct hfs_find_data fd; + hfs_cat_rec rec; + struct hfs_cat_file *file; + int res; + + if (!S_ISREG(inode->i_mode) || HFS_IS_RSRC(inode)) + return -EOPNOTSUPP; + + res = hfs_find_init(HFS_SB(inode->i_sb)->cat_tree, &fd); + if (res) + return res; + fd.search_key->cat = HFS_I(inode)->cat_key; + res = hfs_brec_find(&fd); + if (res) + goto out; + hfs_bnode_read(fd.bnode, &rec, fd.entryoffset, + sizeof(struct hfs_cat_file)); + file = &rec.file; + + switch (type) { + case HFS_TYPE: + if (size == 4) + memcpy(&file->UsrWds.fdType, value, 4); + else + res = -ERANGE; + break; + + case HFS_CREATOR: + if (size == 4) + memcpy(&file->UsrWds.fdCreator, value, 4); + else + res = -ERANGE; + break; + } + + if (!res) + hfs_bnode_write(fd.bnode, &rec, fd.entryoffset, + sizeof(struct hfs_cat_file)); +out: + hfs_find_exit(&fd); + return res; +} + +static ssize_t __hfs_getxattr(struct inode *inode, enum hfs_xattr_type type, + void *value, size_t size) +{ + struct hfs_find_data fd; + hfs_cat_rec rec; + struct hfs_cat_file *file; + ssize_t res = 0; + + if (!S_ISREG(inode->i_mode) || HFS_IS_RSRC(inode)) + return -EOPNOTSUPP; + + if (size) { + res = hfs_find_init(HFS_SB(inode->i_sb)->cat_tree, &fd); + if (res) + return res; + fd.search_key->cat = HFS_I(inode)->cat_key; + res = hfs_brec_find(&fd); + if (res) + goto out; + hfs_bnode_read(fd.bnode, &rec, fd.entryoffset, + sizeof(struct hfs_cat_file)); + } + file = &rec.file; + + switch (type) { + case HFS_TYPE: + if (size >= 4) { + memcpy(value, &file->UsrWds.fdType, 4); + res = 4; + } else + res = size ? -ERANGE : 4; + break; + + case HFS_CREATOR: + if (size >= 4) { + memcpy(value, &file->UsrWds.fdCreator, 4); + res = 4; + } else + res = size ? -ERANGE : 4; + break; + } + +out: + if (size) + hfs_find_exit(&fd); + return res; +} + +static int hfs_xattr_get(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *value, size_t size) +{ + return __hfs_getxattr(inode, handler->flags, value, size); +} + +static int hfs_xattr_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, size_t size, + int flags) +{ + if (!value) + return -EOPNOTSUPP; + + return __hfs_setxattr(inode, handler->flags, value, size, flags); +} + +static const struct xattr_handler hfs_creator_handler = { + .name = "hfs.creator", + .flags = HFS_CREATOR, + .get = hfs_xattr_get, + .set = hfs_xattr_set, +}; + +static const struct xattr_handler hfs_type_handler = { + .name = "hfs.type", + .flags = HFS_TYPE, + .get = hfs_xattr_get, + .set = hfs_xattr_set, +}; + +const struct xattr_handler * const hfs_xattr_handlers[] = { + &hfs_creator_handler, + &hfs_type_handler, + NULL +}; diff --git a/fs/hfs/bfind.c b/fs/hfs/bfind.c new file mode 100644 index 000000000..d56e47bdc --- /dev/null +++ b/fs/hfs/bfind.c @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hfs/bfind.c + * + * Copyright (C) 2001 + * Brad Boyer (flar@allandria.com) + * (C) 2003 Ardis Technologies + * + * Search routines for btrees + */ + +#include +#include "btree.h" + +int hfs_find_init(struct hfs_btree *tree, struct hfs_find_data *fd) +{ + void *ptr; + + if (!tree || !fd) + return -EINVAL; + + fd->tree = tree; + fd->bnode = NULL; + ptr = kzalloc(tree->max_key_len * 2 + 4, GFP_KERNEL); + if (!ptr) + return -ENOMEM; + fd->search_key = ptr; + fd->key = ptr + tree->max_key_len + 2; + hfs_dbg("cnid %d, caller %ps\n", + tree->cnid, __builtin_return_address(0)); + switch (tree->cnid) { + case HFS_CAT_CNID: + mutex_lock_nested(&tree->tree_lock, CATALOG_BTREE_MUTEX); + break; + case HFS_EXT_CNID: + mutex_lock_nested(&tree->tree_lock, EXTENTS_BTREE_MUTEX); + break; + case HFS_ATTR_CNID: + mutex_lock_nested(&tree->tree_lock, ATTR_BTREE_MUTEX); + break; + default: + return -EINVAL; + } + return 0; +} + +void hfs_find_exit(struct hfs_find_data *fd) +{ + hfs_bnode_put(fd->bnode); + kfree(fd->search_key); + hfs_dbg("cnid %d, caller %ps\n", + fd->tree->cnid, __builtin_return_address(0)); + mutex_unlock(&fd->tree->tree_lock); + fd->tree = NULL; +} + +/* Find the record in bnode that best matches key (not greater than...)*/ +int __hfs_brec_find(struct hfs_bnode *bnode, struct hfs_find_data *fd) +{ + int cmpval; + u16 off, len, keylen; + int rec; + int b, e; + int res; + + b = 0; + e = bnode->num_recs - 1; + res = -ENOENT; + do { + rec = (e + b) / 2; + len = hfs_brec_lenoff(bnode, rec, &off); + keylen = hfs_brec_keylen(bnode, rec); + if (keylen == 0) { + res = -EINVAL; + goto fail; + } + hfs_bnode_read(bnode, fd->key, off, keylen); + cmpval = bnode->tree->keycmp(fd->key, fd->search_key); + if (!cmpval) { + e = rec; + res = 0; + goto done; + } + if (cmpval < 0) + b = rec + 1; + else + e = rec - 1; + } while (b <= e); + if (rec != e && e >= 0) { + len = hfs_brec_lenoff(bnode, e, &off); + keylen = hfs_brec_keylen(bnode, e); + if (keylen == 0) { + res = -EINVAL; + goto fail; + } + hfs_bnode_read(bnode, fd->key, off, keylen); + } +done: + fd->record = e; + fd->keyoffset = off; + fd->keylength = keylen; + fd->entryoffset = off + keylen; + fd->entrylength = len - keylen; +fail: + return res; +} + +/* Traverse a B*Tree from the root to a leaf finding best fit to key */ +/* Return allocated copy of node found, set recnum to best record */ +int hfs_brec_find(struct hfs_find_data *fd) +{ + struct hfs_btree *tree; + struct hfs_bnode *bnode; + u32 nidx, parent; + __be32 data; + int height, res; + + fd->record = -1; + fd->keyoffset = -1; + fd->keylength = -1; + fd->entryoffset = -1; + fd->entrylength = -1; + + tree = fd->tree; + if (fd->bnode) + hfs_bnode_put(fd->bnode); + fd->bnode = NULL; + nidx = tree->root; + if (!nidx) + return -ENOENT; + height = tree->depth; + res = 0; + parent = 0; + for (;;) { + bnode = hfs_bnode_find(tree, nidx); + if (IS_ERR(bnode)) { + res = PTR_ERR(bnode); + bnode = NULL; + break; + } + if (bnode->height != height) + goto invalid; + if (bnode->type != (--height ? HFS_NODE_INDEX : HFS_NODE_LEAF)) + goto invalid; + bnode->parent = parent; + + res = __hfs_brec_find(bnode, fd); + if (!height) + break; + if (fd->record < 0) + goto release; + + parent = nidx; + hfs_bnode_read(bnode, &data, fd->entryoffset, 4); + nidx = be32_to_cpu(data); + hfs_bnode_put(bnode); + } + fd->bnode = bnode; + return res; + +invalid: + pr_err("inconsistency in B*Tree (%d,%d,%d,%u,%u)\n", + height, bnode->height, bnode->type, nidx, parent); + res = -EIO; +release: + hfs_bnode_put(bnode); + return res; +} + +int hfs_brec_read(struct hfs_find_data *fd, void *rec, u32 rec_len) +{ + int res; + + res = hfs_brec_find(fd); + if (res) + return res; + if (fd->entrylength > rec_len) + return -EINVAL; + hfs_bnode_read(fd->bnode, rec, fd->entryoffset, fd->entrylength); + return 0; +} + +int hfs_brec_goto(struct hfs_find_data *fd, int cnt) +{ + struct hfs_btree *tree; + struct hfs_bnode *bnode; + int idx, res = 0; + u16 off, len, keylen; + + bnode = fd->bnode; + tree = bnode->tree; + + if (cnt < 0) { + cnt = -cnt; + while (cnt > fd->record) { + cnt -= fd->record + 1; + fd->record = bnode->num_recs - 1; + idx = bnode->prev; + if (!idx) { + res = -ENOENT; + goto out; + } + hfs_bnode_put(bnode); + bnode = hfs_bnode_find(tree, idx); + if (IS_ERR(bnode)) { + res = PTR_ERR(bnode); + bnode = NULL; + goto out; + } + } + fd->record -= cnt; + } else { + while (cnt >= bnode->num_recs - fd->record) { + cnt -= bnode->num_recs - fd->record; + fd->record = 0; + idx = bnode->next; + if (!idx) { + res = -ENOENT; + goto out; + } + hfs_bnode_put(bnode); + bnode = hfs_bnode_find(tree, idx); + if (IS_ERR(bnode)) { + res = PTR_ERR(bnode); + bnode = NULL; + goto out; + } + } + fd->record += cnt; + } + + len = hfs_brec_lenoff(bnode, fd->record, &off); + keylen = hfs_brec_keylen(bnode, fd->record); + if (keylen == 0) { + res = -EINVAL; + goto out; + } + fd->keyoffset = off; + fd->keylength = keylen; + fd->entryoffset = off + keylen; + fd->entrylength = len - keylen; + hfs_bnode_read(bnode, fd->key, off, keylen); +out: + fd->bnode = bnode; + return res; +} diff --git a/fs/hfs/bitmap.c b/fs/hfs/bitmap.c new file mode 100644 index 000000000..5e84833a4 --- /dev/null +++ b/fs/hfs/bitmap.c @@ -0,0 +1,243 @@ +/* + * linux/fs/hfs/bitmap.c + * + * Copyright (C) 1996-1997 Paul H. Hargrove + * (C) 2003 Ardis Technologies + * This file may be distributed under the terms of the GNU General Public License. + * + * Based on GPLed code Copyright (C) 1995 Michael Dreher + * + * This file contains the code to modify the volume bitmap: + * search/set/clear bits. + */ + +#include "hfs_fs.h" + +/* + * hfs_find_zero_bit() + * + * Description: + * Given a block of memory, its length in bits, and a starting bit number, + * determine the number of the first zero bits (in left-to-right ordering) + * in that range. + * + * Returns >= 'size' if no zero bits are found in the range. + * + * Accesses memory in 32-bit aligned chunks of 32-bits and thus + * may read beyond the 'size'th bit. + */ +static u32 hfs_find_set_zero_bits(__be32 *bitmap, u32 size, u32 offset, u32 *max) +{ + __be32 *curr, *end; + u32 mask, start, len, n; + __be32 val; + int i; + + len = *max; + if (!len) + return size; + + curr = bitmap + (offset / 32); + end = bitmap + ((size + 31) / 32); + + /* scan the first partial u32 for zero bits */ + val = *curr; + if (~val) { + n = be32_to_cpu(val); + i = offset % 32; + mask = (1U << 31) >> i; + for (; i < 32; mask >>= 1, i++) { + if (!(n & mask)) + goto found; + } + } + + /* scan complete u32s for the first zero bit */ + while (++curr < end) { + val = *curr; + if (~val) { + n = be32_to_cpu(val); + mask = 1 << 31; + for (i = 0; i < 32; mask >>= 1, i++) { + if (!(n & mask)) + goto found; + } + } + } + return size; + +found: + start = (curr - bitmap) * 32 + i; + if (start >= size) + return start; + /* do any partial u32 at the start */ + len = min(size - start, len); + while (1) { + n |= mask; + if (++i >= 32) + break; + mask >>= 1; + if (!--len || n & mask) + goto done; + } + if (!--len) + goto done; + *curr++ = cpu_to_be32(n); + /* do full u32s */ + while (1) { + n = be32_to_cpu(*curr); + if (len < 32) + break; + if (n) { + len = 32; + break; + } + *curr++ = cpu_to_be32(0xffffffff); + len -= 32; + } + /* do any partial u32 at end */ + mask = 1U << 31; + for (i = 0; i < len; i++) { + if (n & mask) + break; + n |= mask; + mask >>= 1; + } +done: + *curr = cpu_to_be32(n); + *max = (curr - bitmap) * 32 + i - start; + return start; +} + +/* + * hfs_vbm_search_free() + * + * Description: + * Search for 'num_bits' consecutive cleared bits in the bitmap blocks of + * the hfs MDB. 'mdb' had better be locked or the returned range + * may be no longer free, when this functions returns! + * XXX Currently the search starts from bit 0, but it should start with + * the bit number stored in 's_alloc_ptr' of the MDB. + * Input Variable(s): + * struct hfs_mdb *mdb: Pointer to the hfs MDB + * u16 *num_bits: Pointer to the number of cleared bits + * to search for + * Output Variable(s): + * u16 *num_bits: The number of consecutive clear bits of the + * returned range. If the bitmap is fragmented, this will be less than + * requested and it will be zero, when the disk is full. + * Returns: + * The number of the first bit of the range of cleared bits which has been + * found. When 'num_bits' is zero, this is invalid! + * Preconditions: + * 'mdb' points to a "valid" (struct hfs_mdb). + * 'num_bits' points to a variable of type (u16), which contains + * the number of cleared bits to find. + * Postconditions: + * 'num_bits' is set to the length of the found sequence. + */ +u32 hfs_vbm_search_free(struct super_block *sb, u32 goal, u32 *num_bits) +{ + void *bitmap; + u32 pos; + + /* make sure we have actual work to perform */ + if (!*num_bits) + return 0; + + mutex_lock(&HFS_SB(sb)->bitmap_lock); + bitmap = HFS_SB(sb)->bitmap; + + pos = hfs_find_set_zero_bits(bitmap, HFS_SB(sb)->fs_ablocks, goal, num_bits); + if (pos >= HFS_SB(sb)->fs_ablocks) { + if (goal) + pos = hfs_find_set_zero_bits(bitmap, goal, 0, num_bits); + if (pos >= HFS_SB(sb)->fs_ablocks) { + *num_bits = pos = 0; + goto out; + } + } + + hfs_dbg("pos %u, num_bits %u\n", pos, *num_bits); + HFS_SB(sb)->free_ablocks -= *num_bits; + hfs_bitmap_dirty(sb); +out: + mutex_unlock(&HFS_SB(sb)->bitmap_lock); + return pos; +} + + +/* + * hfs_clear_vbm_bits() + * + * Description: + * Clear the requested bits in the volume bitmap of the hfs filesystem + * Input Variable(s): + * struct hfs_mdb *mdb: Pointer to the hfs MDB + * u16 start: The offset of the first bit + * u16 count: The number of bits + * Output Variable(s): + * None + * Returns: + * 0: no error + * -1: One of the bits was already clear. This is a strange + * error and when it happens, the filesystem must be repaired! + * -2: One or more of the bits are out of range of the bitmap. + * Preconditions: + * 'mdb' points to a "valid" (struct hfs_mdb). + * Postconditions: + * Starting with bit number 'start', 'count' bits in the volume bitmap + * are cleared. The affected bitmap blocks are marked "dirty", the free + * block count of the MDB is updated and the MDB is marked dirty. + */ +int hfs_clear_vbm_bits(struct super_block *sb, u16 start, u16 count) +{ + __be32 *curr; + u32 mask; + int i, len; + + /* is there any actual work to be done? */ + if (!count) + return 0; + + hfs_dbg("start %u, count %u\n", start, count); + /* are all of the bits in range? */ + if ((start + count) > HFS_SB(sb)->fs_ablocks) + return -2; + + mutex_lock(&HFS_SB(sb)->bitmap_lock); + /* bitmap is always on a 32-bit boundary */ + curr = HFS_SB(sb)->bitmap + (start / 32); + len = count; + + /* do any partial u32 at the start */ + i = start % 32; + if (i) { + int j = 32 - i; + mask = 0xffffffffU << j; + if (j > count) { + mask |= 0xffffffffU >> (i + count); + *curr &= cpu_to_be32(mask); + goto out; + } + *curr++ &= cpu_to_be32(mask); + count -= j; + } + + /* do full u32s */ + while (count >= 32) { + *curr++ = 0; + count -= 32; + } + /* do any partial u32 at end */ + if (count) { + mask = 0xffffffffU >> count; + *curr &= cpu_to_be32(mask); + } +out: + HFS_SB(sb)->free_ablocks += len; + mutex_unlock(&HFS_SB(sb)->bitmap_lock); + hfs_bitmap_dirty(sb); + + return 0; +} diff --git a/fs/hfs/bnode.c b/fs/hfs/bnode.c new file mode 100644 index 000000000..1b331108d --- /dev/null +++ b/fs/hfs/bnode.c @@ -0,0 +1,542 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hfs/bnode.c + * + * Copyright (C) 2001 + * Brad Boyer (flar@allandria.com) + * (C) 2003 Ardis Technologies + * + * Handle basic btree node operations + */ + +#include +#include +#include + +#include "btree.h" + +void hfs_bnode_read(struct hfs_bnode *node, void *buf, u32 off, u32 len) +{ + struct page *page; + u32 pagenum; + u32 bytes_read; + u32 bytes_to_read; + + memset(buf, 0, len); + + if (!is_bnode_offset_valid(node, off)) + return; + + if (len == 0) { + pr_err("requested zero length: " + "NODE: id %u, type %#x, height %u, " + "node_size %u, offset %u, len %u\n", + node->this, node->type, node->height, + node->tree->node_size, off, len); + return; + } + + len = check_and_correct_requested_length(node, off, len); + + off += node->page_offset; + pagenum = off >> PAGE_SHIFT; + off &= ~PAGE_MASK; /* compute page offset for the first page */ + + for (bytes_read = 0; bytes_read < len; bytes_read += bytes_to_read) { + if (pagenum >= node->tree->pages_per_bnode) + break; + page = node->page[pagenum]; + bytes_to_read = min_t(u32, len - bytes_read, PAGE_SIZE - off); + + memcpy_from_page(buf + bytes_read, page, off, bytes_to_read); + + pagenum++; + off = 0; /* page offset only applies to the first page */ + } +} + +u16 hfs_bnode_read_u16(struct hfs_bnode *node, u32 off) +{ + __be16 data; + // optimize later... + hfs_bnode_read(node, &data, off, 2); + return be16_to_cpu(data); +} + +u8 hfs_bnode_read_u8(struct hfs_bnode *node, u32 off) +{ + u8 data; + // optimize later... + hfs_bnode_read(node, &data, off, 1); + return data; +} + +void hfs_bnode_read_key(struct hfs_bnode *node, void *key, u32 off) +{ + struct hfs_btree *tree; + u32 key_len; + + tree = node->tree; + if (node->type == HFS_NODE_LEAF || + tree->attributes & HFS_TREE_VARIDXKEYS) + key_len = hfs_bnode_read_u8(node, off) + 1; + else + key_len = tree->max_key_len + 1; + + if (key_len > sizeof(hfs_btree_key) || key_len < 1) { + memset(key, 0, sizeof(hfs_btree_key)); + pr_err("hfs: Invalid key length: %u\n", key_len); + return; + } + + hfs_bnode_read(node, key, off, key_len); +} + +void hfs_bnode_write(struct hfs_bnode *node, void *buf, u32 off, u32 len) +{ + struct page *page; + + if (!is_bnode_offset_valid(node, off)) + return; + + if (len == 0) { + pr_err("requested zero length: " + "NODE: id %u, type %#x, height %u, " + "node_size %u, offset %u, len %u\n", + node->this, node->type, node->height, + node->tree->node_size, off, len); + return; + } + + len = check_and_correct_requested_length(node, off, len); + + off += node->page_offset; + page = node->page[0]; + + memcpy_to_page(page, off, buf, len); + set_page_dirty(page); +} + +void hfs_bnode_write_u16(struct hfs_bnode *node, u32 off, u16 data) +{ + __be16 v = cpu_to_be16(data); + // optimize later... + hfs_bnode_write(node, &v, off, 2); +} + +void hfs_bnode_write_u8(struct hfs_bnode *node, u32 off, u8 data) +{ + // optimize later... + hfs_bnode_write(node, &data, off, 1); +} + +void hfs_bnode_clear(struct hfs_bnode *node, u32 off, u32 len) +{ + struct page *page; + + if (!is_bnode_offset_valid(node, off)) + return; + + if (len == 0) { + pr_err("requested zero length: " + "NODE: id %u, type %#x, height %u, " + "node_size %u, offset %u, len %u\n", + node->this, node->type, node->height, + node->tree->node_size, off, len); + return; + } + + len = check_and_correct_requested_length(node, off, len); + + off += node->page_offset; + page = node->page[0]; + + memzero_page(page, off, len); + set_page_dirty(page); +} + +void hfs_bnode_copy(struct hfs_bnode *dst_node, u32 dst, + struct hfs_bnode *src_node, u32 src, u32 len) +{ + struct page *src_page, *dst_page; + + hfs_dbg("dst %u, src %u, len %u\n", dst, src, len); + if (!len) + return; + + len = check_and_correct_requested_length(src_node, src, len); + len = check_and_correct_requested_length(dst_node, dst, len); + + src += src_node->page_offset; + dst += dst_node->page_offset; + src_page = src_node->page[0]; + dst_page = dst_node->page[0]; + + memcpy_page(dst_page, dst, src_page, src, len); + set_page_dirty(dst_page); +} + +void hfs_bnode_move(struct hfs_bnode *node, u32 dst, u32 src, u32 len) +{ + struct page *page; + void *ptr; + + hfs_dbg("dst %u, src %u, len %u\n", dst, src, len); + if (!len) + return; + + len = check_and_correct_requested_length(node, src, len); + len = check_and_correct_requested_length(node, dst, len); + + src += node->page_offset; + dst += node->page_offset; + page = node->page[0]; + ptr = kmap_local_page(page); + memmove(ptr + dst, ptr + src, len); + kunmap_local(ptr); + set_page_dirty(page); +} + +void hfs_bnode_dump(struct hfs_bnode *node) +{ + struct hfs_bnode_desc desc; + __be32 cnid; + int i, off, key_off; + + hfs_dbg("node %d\n", node->this); + hfs_bnode_read(node, &desc, 0, sizeof(desc)); + hfs_dbg("next %d, prev %d, type %d, height %d, num_recs %d\n", + be32_to_cpu(desc.next), be32_to_cpu(desc.prev), + desc.type, desc.height, be16_to_cpu(desc.num_recs)); + + off = node->tree->node_size - 2; + for (i = be16_to_cpu(desc.num_recs); i >= 0; off -= 2, i--) { + key_off = hfs_bnode_read_u16(node, off); + hfs_dbg(" key_off %d", key_off); + if (i && node->type == HFS_NODE_INDEX) { + int tmp; + + if (node->tree->attributes & HFS_TREE_VARIDXKEYS) + tmp = (hfs_bnode_read_u8(node, key_off) | 1) + 1; + else + tmp = node->tree->max_key_len + 1; + hfs_dbg(" (%d,%d", + tmp, hfs_bnode_read_u8(node, key_off)); + hfs_bnode_read(node, &cnid, key_off + tmp, 4); + hfs_dbg(", cnid %d)", be32_to_cpu(cnid)); + } else if (i && node->type == HFS_NODE_LEAF) { + int tmp; + + tmp = hfs_bnode_read_u8(node, key_off); + hfs_dbg(" (%d)", tmp); + } + } + hfs_dbg("\n"); +} + +void hfs_bnode_unlink(struct hfs_bnode *node) +{ + struct hfs_btree *tree; + struct hfs_bnode *tmp; + __be32 cnid; + + tree = node->tree; + if (node->prev) { + tmp = hfs_bnode_find(tree, node->prev); + if (IS_ERR(tmp)) + return; + tmp->next = node->next; + cnid = cpu_to_be32(tmp->next); + hfs_bnode_write(tmp, &cnid, offsetof(struct hfs_bnode_desc, next), 4); + hfs_bnode_put(tmp); + } else if (node->type == HFS_NODE_LEAF) + tree->leaf_head = node->next; + + if (node->next) { + tmp = hfs_bnode_find(tree, node->next); + if (IS_ERR(tmp)) + return; + tmp->prev = node->prev; + cnid = cpu_to_be32(tmp->prev); + hfs_bnode_write(tmp, &cnid, offsetof(struct hfs_bnode_desc, prev), 4); + hfs_bnode_put(tmp); + } else if (node->type == HFS_NODE_LEAF) + tree->leaf_tail = node->prev; + + // move down? + if (!node->prev && !node->next) { + printk(KERN_DEBUG "hfs_btree_del_level\n"); + } + if (!node->parent) { + tree->root = 0; + tree->depth = 0; + } + set_bit(HFS_BNODE_DELETED, &node->flags); +} + +static inline int hfs_bnode_hash(u32 num) +{ + num = (num >> 16) + num; + num += num >> 8; + return num & (NODE_HASH_SIZE - 1); +} + +struct hfs_bnode *hfs_bnode_findhash(struct hfs_btree *tree, u32 cnid) +{ + struct hfs_bnode *node; + + if (cnid >= tree->node_count) { + pr_err("request for non-existent node %d in B*Tree\n", cnid); + return NULL; + } + + for (node = tree->node_hash[hfs_bnode_hash(cnid)]; + node; node = node->next_hash) { + if (node->this == cnid) { + return node; + } + } + return NULL; +} + +static struct hfs_bnode *__hfs_bnode_create(struct hfs_btree *tree, u32 cnid) +{ + struct hfs_bnode *node, *node2; + struct address_space *mapping; + struct page *page; + int block, i, hash; + loff_t off; + + if (cnid >= tree->node_count) { + pr_err("request for non-existent node %d in B*Tree\n", cnid); + return NULL; + } + + node = kzalloc_flex(*node, page, tree->pages_per_bnode, GFP_KERNEL); + if (!node) + return NULL; + node->tree = tree; + node->this = cnid; + set_bit(HFS_BNODE_NEW, &node->flags); + atomic_set(&node->refcnt, 1); + hfs_dbg("cnid %d, node %d, refcnt 1\n", + node->tree->cnid, node->this); + init_waitqueue_head(&node->lock_wq); + spin_lock(&tree->hash_lock); + node2 = hfs_bnode_findhash(tree, cnid); + if (!node2) { + hash = hfs_bnode_hash(cnid); + node->next_hash = tree->node_hash[hash]; + tree->node_hash[hash] = node; + tree->node_hash_cnt++; + } else { + hfs_bnode_get(node2); + spin_unlock(&tree->hash_lock); + kfree(node); + wait_event(node2->lock_wq, !test_bit(HFS_BNODE_NEW, &node2->flags)); + return node2; + } + spin_unlock(&tree->hash_lock); + + mapping = tree->inode->i_mapping; + off = (loff_t)cnid * tree->node_size; + block = off >> PAGE_SHIFT; + node->page_offset = off & ~PAGE_MASK; + for (i = 0; i < tree->pages_per_bnode; i++) { + page = read_mapping_page(mapping, block++, NULL); + if (IS_ERR(page)) + goto fail; + node->page[i] = page; + } + + return node; +fail: + set_bit(HFS_BNODE_ERROR, &node->flags); + return node; +} + +void hfs_bnode_unhash(struct hfs_bnode *node) +{ + struct hfs_bnode **p; + + hfs_dbg("cnid %d, node %d, refcnt %d\n", + node->tree->cnid, node->this, atomic_read(&node->refcnt)); + for (p = &node->tree->node_hash[hfs_bnode_hash(node->this)]; + *p && *p != node; p = &(*p)->next_hash) + ; + BUG_ON(!*p); + *p = node->next_hash; + node->tree->node_hash_cnt--; +} + +/* Load a particular node out of a tree */ +struct hfs_bnode *hfs_bnode_find(struct hfs_btree *tree, u32 num) +{ + struct hfs_bnode *node; + struct hfs_bnode_desc *desc; + int i, rec_off, off, next_off; + int entry_size, key_size; + + spin_lock(&tree->hash_lock); + node = hfs_bnode_findhash(tree, num); + if (node) { + hfs_bnode_get(node); + spin_unlock(&tree->hash_lock); + wait_event(node->lock_wq, !test_bit(HFS_BNODE_NEW, &node->flags)); + if (test_bit(HFS_BNODE_ERROR, &node->flags)) + goto node_error; + return node; + } + spin_unlock(&tree->hash_lock); + node = __hfs_bnode_create(tree, num); + if (!node) + return ERR_PTR(-ENOMEM); + if (test_bit(HFS_BNODE_ERROR, &node->flags)) + goto node_error; + if (!test_bit(HFS_BNODE_NEW, &node->flags)) + return node; + + desc = (struct hfs_bnode_desc *)(kmap_local_page(node->page[0]) + + node->page_offset); + node->prev = be32_to_cpu(desc->prev); + node->next = be32_to_cpu(desc->next); + node->num_recs = be16_to_cpu(desc->num_recs); + node->type = desc->type; + node->height = desc->height; + kunmap_local(desc); + + switch (node->type) { + case HFS_NODE_HEADER: + case HFS_NODE_MAP: + if (node->height != 0) + goto node_error; + break; + case HFS_NODE_LEAF: + if (node->height != 1) + goto node_error; + break; + case HFS_NODE_INDEX: + if (node->height <= 1 || node->height > tree->depth) + goto node_error; + break; + default: + goto node_error; + } + + rec_off = tree->node_size - 2; + off = hfs_bnode_read_u16(node, rec_off); + if (off != sizeof(struct hfs_bnode_desc)) + goto node_error; + for (i = 1; i <= node->num_recs; off = next_off, i++) { + rec_off -= 2; + next_off = hfs_bnode_read_u16(node, rec_off); + if (next_off <= off || + next_off > tree->node_size || + next_off & 1) + goto node_error; + entry_size = next_off - off; + if (node->type != HFS_NODE_INDEX && + node->type != HFS_NODE_LEAF) + continue; + key_size = hfs_bnode_read_u8(node, off) + 1; + if (key_size >= entry_size /*|| key_size & 1*/) + goto node_error; + } + clear_bit(HFS_BNODE_NEW, &node->flags); + wake_up(&node->lock_wq); + return node; + +node_error: + set_bit(HFS_BNODE_ERROR, &node->flags); + clear_bit(HFS_BNODE_NEW, &node->flags); + wake_up(&node->lock_wq); + hfs_bnode_put(node); + return ERR_PTR(-EIO); +} + +void hfs_bnode_free(struct hfs_bnode *node) +{ + int i; + + for (i = 0; i < node->tree->pages_per_bnode; i++) + if (node->page[i]) + put_page(node->page[i]); + kfree(node); +} + +struct hfs_bnode *hfs_bnode_create(struct hfs_btree *tree, u32 num) +{ + struct hfs_bnode *node; + struct page **pagep; + int i; + + spin_lock(&tree->hash_lock); + node = hfs_bnode_findhash(tree, num); + spin_unlock(&tree->hash_lock); + if (node) { + pr_crit("new node %u already hashed?\n", num); + WARN_ON(1); + return node; + } + node = __hfs_bnode_create(tree, num); + if (!node) + return ERR_PTR(-ENOMEM); + if (test_bit(HFS_BNODE_ERROR, &node->flags)) { + hfs_bnode_put(node); + return ERR_PTR(-EIO); + } + + pagep = node->page; + memzero_page(*pagep, node->page_offset, + min((int)PAGE_SIZE, (int)tree->node_size)); + set_page_dirty(*pagep); + for (i = 1; i < tree->pages_per_bnode; i++) { + memzero_page(*++pagep, 0, PAGE_SIZE); + set_page_dirty(*pagep); + } + clear_bit(HFS_BNODE_NEW, &node->flags); + wake_up(&node->lock_wq); + + return node; +} + +void hfs_bnode_get(struct hfs_bnode *node) +{ + if (node) { + atomic_inc(&node->refcnt); + hfs_dbg("cnid %d, node %d, refcnt %d\n", + node->tree->cnid, node->this, + atomic_read(&node->refcnt)); + } +} + +/* Dispose of resources used by a node */ +void hfs_bnode_put(struct hfs_bnode *node) +{ + if (node) { + struct hfs_btree *tree = node->tree; + int i; + + hfs_dbg("cnid %d, node %d, refcnt %d\n", + node->tree->cnid, node->this, + atomic_read(&node->refcnt)); + BUG_ON(!atomic_read(&node->refcnt)); + if (!atomic_dec_and_lock(&node->refcnt, &tree->hash_lock)) + return; + for (i = 0; i < tree->pages_per_bnode; i++) { + if (!node->page[i]) + continue; + mark_page_accessed(node->page[i]); + } + + if (test_bit(HFS_BNODE_DELETED, &node->flags)) { + hfs_bnode_unhash(node); + spin_unlock(&tree->hash_lock); + hfs_bnode_clear(node, 0, tree->node_size); + hfs_bmap_free(node); + hfs_bnode_free(node); + return; + } + spin_unlock(&tree->hash_lock); + } +} diff --git a/fs/hfs/brec.c b/fs/hfs/brec.c new file mode 100644 index 000000000..5a2f740dd --- /dev/null +++ b/fs/hfs/brec.c @@ -0,0 +1,548 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hfs/brec.c + * + * Copyright (C) 2001 + * Brad Boyer (flar@allandria.com) + * (C) 2003 Ardis Technologies + * + * Handle individual btree records + */ + +#include "btree.h" + +static struct hfs_bnode *hfs_bnode_split(struct hfs_find_data *fd); +static int hfs_brec_update_parent(struct hfs_find_data *fd); +static int hfs_btree_inc_height(struct hfs_btree *tree); + +/* Get the length and offset of the given record in the given node */ +u16 hfs_brec_lenoff(struct hfs_bnode *node, u16 rec, u16 *off) +{ + __be16 retval[2]; + u16 dataoff; + + dataoff = node->tree->node_size - (rec + 2) * 2; + hfs_bnode_read(node, retval, dataoff, 4); + *off = be16_to_cpu(retval[1]); + return be16_to_cpu(retval[0]) - *off; +} + +/* Get the length of the key from a keyed record */ +u16 hfs_brec_keylen(struct hfs_bnode *node, u16 rec) +{ + u16 retval, recoff; + + if (node->type != HFS_NODE_INDEX && node->type != HFS_NODE_LEAF) + return 0; + + if ((node->type == HFS_NODE_INDEX) && + !(node->tree->attributes & HFS_TREE_VARIDXKEYS)) { + if (node->tree->attributes & HFS_TREE_BIGKEYS) + retval = node->tree->max_key_len + 2; + else + retval = node->tree->max_key_len + 1; + } else { + recoff = hfs_bnode_read_u16(node, node->tree->node_size - (rec + 1) * 2); + if (!recoff) + return 0; + if (node->tree->attributes & HFS_TREE_BIGKEYS) { + retval = hfs_bnode_read_u16(node, recoff) + 2; + if (retval > node->tree->max_key_len + 2) { + pr_err("keylen %d too large\n", retval); + retval = 0; + } + } else { + retval = (hfs_bnode_read_u8(node, recoff) | 1) + 1; + if (retval > node->tree->max_key_len + 1) { + pr_err("keylen %d too large\n", retval); + retval = 0; + } + } + } + return retval; +} + +int hfs_brec_insert(struct hfs_find_data *fd, void *entry, u32 entry_len) +{ + struct hfs_btree *tree; + struct hfs_bnode *node, *new_node; + int size, key_len, rec; + int data_off, end_off; + int idx_rec_off, data_rec_off, end_rec_off; + __be32 cnid; + + tree = fd->tree; + if (!fd->bnode) { + if (!tree->root) + hfs_btree_inc_height(tree); + node = hfs_bnode_find(tree, tree->leaf_head); + if (IS_ERR(node)) + return PTR_ERR(node); + fd->bnode = node; + fd->record = -1; + } + new_node = NULL; + key_len = (fd->search_key->key_len | 1) + 1; +again: + /* new record idx and complete record size */ + rec = fd->record + 1; + size = key_len + entry_len; + + node = fd->bnode; + hfs_bnode_dump(node); + /* get last offset */ + end_rec_off = tree->node_size - (node->num_recs + 1) * 2; + end_off = hfs_bnode_read_u16(node, end_rec_off); + end_rec_off -= 2; + hfs_dbg("rec %d, size %d, end_off %d, end_rec_off %d\n", + rec, size, end_off, end_rec_off); + if (size > end_rec_off - end_off) { + if (new_node) + panic("not enough room!\n"); + new_node = hfs_bnode_split(fd); + if (IS_ERR(new_node)) + return PTR_ERR(new_node); + goto again; + } + if (node->type == HFS_NODE_LEAF) { + tree->leaf_count++; + mark_inode_dirty(tree->inode); + } + node->num_recs++; + /* write new last offset */ + hfs_bnode_write_u16(node, offsetof(struct hfs_bnode_desc, num_recs), node->num_recs); + hfs_bnode_write_u16(node, end_rec_off, end_off + size); + data_off = end_off; + data_rec_off = end_rec_off + 2; + idx_rec_off = tree->node_size - (rec + 1) * 2; + if (idx_rec_off == data_rec_off) + goto skip; + /* move all following entries */ + do { + data_off = hfs_bnode_read_u16(node, data_rec_off + 2); + hfs_bnode_write_u16(node, data_rec_off, data_off + size); + data_rec_off += 2; + } while (data_rec_off < idx_rec_off); + + /* move data away */ + hfs_bnode_move(node, data_off + size, data_off, + end_off - data_off); + +skip: + hfs_bnode_write(node, fd->search_key, data_off, key_len); + hfs_bnode_write(node, entry, data_off + key_len, entry_len); + hfs_bnode_dump(node); + + /* + * update parent key if we inserted a key + * at the start of the node and it is not the new node + */ + if (!rec && new_node != node) { + hfs_bnode_read_key(node, fd->search_key, data_off + size); + hfs_brec_update_parent(fd); + } + + if (new_node) { + hfs_bnode_put(fd->bnode); + if (!new_node->parent) { + hfs_btree_inc_height(tree); + new_node->parent = tree->root; + } + fd->bnode = hfs_bnode_find(tree, new_node->parent); + + /* create index data entry */ + cnid = cpu_to_be32(new_node->this); + entry = &cnid; + entry_len = sizeof(cnid); + + /* get index key */ + hfs_bnode_read_key(new_node, fd->search_key, 14); + __hfs_brec_find(fd->bnode, fd); + + hfs_bnode_put(new_node); + new_node = NULL; + + if (tree->attributes & HFS_TREE_VARIDXKEYS) + key_len = fd->search_key->key_len + 1; + else { + fd->search_key->key_len = tree->max_key_len; + key_len = tree->max_key_len + 1; + } + goto again; + } + + return 0; +} + +int hfs_brec_remove(struct hfs_find_data *fd) +{ + struct hfs_btree *tree; + struct hfs_bnode *node, *parent; + int end_off, rec_off, data_off, size; + int src, dst, len; + + tree = fd->tree; + node = fd->bnode; +again: + rec_off = tree->node_size - (fd->record + 2) * 2; + end_off = tree->node_size - (node->num_recs + 1) * 2; + + if (node->type == HFS_NODE_LEAF) { + tree->leaf_count--; + mark_inode_dirty(tree->inode); + } + hfs_bnode_dump(node); + hfs_dbg("rec %d, len %d\n", + fd->record, fd->keylength + fd->entrylength); + if (!--node->num_recs) { + hfs_bnode_unlink(node); + if (!node->parent) + return 0; + parent = hfs_bnode_find(tree, node->parent); + if (IS_ERR(parent)) + return PTR_ERR(parent); + hfs_bnode_put(node); + node = fd->bnode = parent; + + __hfs_brec_find(node, fd); + goto again; + } + hfs_bnode_write_u16(node, offsetof(struct hfs_bnode_desc, num_recs), node->num_recs); + + size = fd->keylength + fd->entrylength; + + if (rec_off == end_off) { + src = fd->keyoffset; + hfs_bnode_clear(node, src, size); + goto skip; + } + + do { + data_off = hfs_bnode_read_u16(node, rec_off); + hfs_bnode_write_u16(node, rec_off + 2, data_off - size); + rec_off -= 2; + } while (rec_off >= end_off); + + /* fill hole */ + dst = fd->keyoffset; + src = fd->keyoffset + size; + len = data_off - src; + + hfs_bnode_move(node, dst, src, len); + + src = dst + len; + len = data_off - src; + + hfs_bnode_clear(node, src, len); + +skip: + /* + * Remove the obsolete offset to free space. + */ + hfs_bnode_write_u16(node, end_off, 0); + + hfs_bnode_dump(node); + if (!fd->record) + hfs_brec_update_parent(fd); + return 0; +} + +static struct hfs_bnode *hfs_bnode_split(struct hfs_find_data *fd) +{ + struct hfs_btree *tree; + struct hfs_bnode *node, *new_node, *next_node; + struct hfs_bnode_desc node_desc; + int num_recs, new_rec_off, new_off, old_rec_off; + int data_start, data_end, size; + + tree = fd->tree; + node = fd->bnode; + new_node = hfs_bmap_alloc(tree); + if (IS_ERR(new_node)) + return new_node; + hfs_bnode_get(node); + hfs_dbg("this %d, new %d, next %d\n", + node->this, new_node->this, node->next); + new_node->next = node->next; + new_node->prev = node->this; + new_node->parent = node->parent; + new_node->type = node->type; + new_node->height = node->height; + + if (node->next) + next_node = hfs_bnode_find(tree, node->next); + else + next_node = NULL; + + if (IS_ERR(next_node)) { + hfs_bnode_put(node); + hfs_bnode_put(new_node); + return next_node; + } + + size = tree->node_size / 2 - node->num_recs * 2 - 14; + old_rec_off = tree->node_size - 4; + num_recs = 1; + for (;;) { + data_start = hfs_bnode_read_u16(node, old_rec_off); + if (data_start > size) + break; + old_rec_off -= 2; + if (++num_recs < node->num_recs) + continue; + /* panic? */ + hfs_bnode_put(node); + hfs_bnode_put(new_node); + if (next_node) + hfs_bnode_put(next_node); + return ERR_PTR(-ENOSPC); + } + + if (fd->record + 1 < num_recs) { + /* new record is in the lower half, + * so leave some more space there + */ + old_rec_off += 2; + num_recs--; + data_start = hfs_bnode_read_u16(node, old_rec_off); + } else { + hfs_bnode_put(node); + hfs_bnode_get(new_node); + fd->bnode = new_node; + fd->record -= num_recs; + fd->keyoffset -= data_start - 14; + fd->entryoffset -= data_start - 14; + } + new_node->num_recs = node->num_recs - num_recs; + node->num_recs = num_recs; + + new_rec_off = tree->node_size - 2; + new_off = 14; + size = data_start - new_off; + num_recs = new_node->num_recs; + data_end = data_start; + while (num_recs) { + hfs_bnode_write_u16(new_node, new_rec_off, new_off); + old_rec_off -= 2; + new_rec_off -= 2; + data_end = hfs_bnode_read_u16(node, old_rec_off); + new_off = data_end - size; + num_recs--; + } + hfs_bnode_write_u16(new_node, new_rec_off, new_off); + hfs_bnode_copy(new_node, 14, node, data_start, data_end - data_start); + + /* update new bnode header */ + node_desc.next = cpu_to_be32(new_node->next); + node_desc.prev = cpu_to_be32(new_node->prev); + node_desc.type = new_node->type; + node_desc.height = new_node->height; + node_desc.num_recs = cpu_to_be16(new_node->num_recs); + node_desc.reserved = 0; + hfs_bnode_write(new_node, &node_desc, 0, sizeof(node_desc)); + + /* update previous bnode header */ + node->next = new_node->this; + hfs_bnode_read(node, &node_desc, 0, sizeof(node_desc)); + node_desc.next = cpu_to_be32(node->next); + node_desc.num_recs = cpu_to_be16(node->num_recs); + hfs_bnode_write(node, &node_desc, 0, sizeof(node_desc)); + + /* update next bnode header */ + if (next_node) { + next_node->prev = new_node->this; + hfs_bnode_read(next_node, &node_desc, 0, sizeof(node_desc)); + node_desc.prev = cpu_to_be32(next_node->prev); + hfs_bnode_write(next_node, &node_desc, 0, sizeof(node_desc)); + hfs_bnode_put(next_node); + } else if (node->this == tree->leaf_tail) { + /* if there is no next node, this might be the new tail */ + tree->leaf_tail = new_node->this; + mark_inode_dirty(tree->inode); + } + + hfs_bnode_dump(node); + hfs_bnode_dump(new_node); + hfs_bnode_put(node); + + return new_node; +} + +static int hfs_brec_update_parent(struct hfs_find_data *fd) +{ + struct hfs_btree *tree; + struct hfs_bnode *node, *new_node, *parent; + int newkeylen, diff; + int rec, rec_off, end_rec_off; + int start_off, end_off; + + tree = fd->tree; + node = fd->bnode; + new_node = NULL; + if (!node->parent) + return 0; + +again: + parent = hfs_bnode_find(tree, node->parent); + if (IS_ERR(parent)) + return PTR_ERR(parent); + __hfs_brec_find(parent, fd); + if (fd->record < 0) + return -ENOENT; + hfs_bnode_dump(parent); + rec = fd->record; + + /* size difference between old and new key */ + if (tree->attributes & HFS_TREE_VARIDXKEYS) + newkeylen = (hfs_bnode_read_u8(node, 14) | 1) + 1; + else + fd->keylength = newkeylen = tree->max_key_len + 1; + hfs_dbg("rec %d, keylength %d, newkeylen %d\n", + rec, fd->keylength, newkeylen); + + rec_off = tree->node_size - (rec + 2) * 2; + end_rec_off = tree->node_size - (parent->num_recs + 1) * 2; + diff = newkeylen - fd->keylength; + if (!diff) + goto skip; + if (diff > 0) { + end_off = hfs_bnode_read_u16(parent, end_rec_off); + if (end_rec_off - end_off < diff) { + + printk(KERN_DEBUG "splitting index node...\n"); + fd->bnode = parent; + new_node = hfs_bnode_split(fd); + if (IS_ERR(new_node)) + return PTR_ERR(new_node); + parent = fd->bnode; + rec = fd->record; + rec_off = tree->node_size - (rec + 2) * 2; + end_rec_off = tree->node_size - (parent->num_recs + 1) * 2; + } + } + + end_off = start_off = hfs_bnode_read_u16(parent, rec_off); + hfs_bnode_write_u16(parent, rec_off, start_off + diff); + start_off -= 4; /* move previous cnid too */ + + while (rec_off > end_rec_off) { + rec_off -= 2; + end_off = hfs_bnode_read_u16(parent, rec_off); + hfs_bnode_write_u16(parent, rec_off, end_off + diff); + } + hfs_bnode_move(parent, start_off + diff, start_off, + end_off - start_off); +skip: + hfs_bnode_copy(parent, fd->keyoffset, node, 14, newkeylen); + if (!(tree->attributes & HFS_TREE_VARIDXKEYS)) + hfs_bnode_write_u8(parent, fd->keyoffset, newkeylen - 1); + hfs_bnode_dump(parent); + + hfs_bnode_put(node); + node = parent; + + if (new_node) { + __be32 cnid; + + if (!new_node->parent) { + hfs_btree_inc_height(tree); + new_node->parent = tree->root; + } + fd->bnode = hfs_bnode_find(tree, new_node->parent); + /* create index key and entry */ + hfs_bnode_read_key(new_node, fd->search_key, 14); + cnid = cpu_to_be32(new_node->this); + + __hfs_brec_find(fd->bnode, fd); + hfs_brec_insert(fd, &cnid, sizeof(cnid)); + hfs_bnode_put(fd->bnode); + hfs_bnode_put(new_node); + + if (!rec) { + if (new_node == node) + goto out; + /* restore search_key */ + hfs_bnode_read_key(node, fd->search_key, 14); + } + new_node = NULL; + } + + if (!rec && node->parent) + goto again; +out: + fd->bnode = node; + return 0; +} + +static int hfs_btree_inc_height(struct hfs_btree *tree) +{ + struct hfs_bnode *node, *new_node; + struct hfs_bnode_desc node_desc; + int key_size, rec; + __be32 cnid; + + node = NULL; + if (tree->root) { + node = hfs_bnode_find(tree, tree->root); + if (IS_ERR(node)) + return PTR_ERR(node); + } + new_node = hfs_bmap_alloc(tree); + if (IS_ERR(new_node)) { + hfs_bnode_put(node); + return PTR_ERR(new_node); + } + + tree->root = new_node->this; + if (!tree->depth) { + tree->leaf_head = tree->leaf_tail = new_node->this; + new_node->type = HFS_NODE_LEAF; + new_node->num_recs = 0; + } else { + new_node->type = HFS_NODE_INDEX; + new_node->num_recs = 1; + } + new_node->parent = 0; + new_node->next = 0; + new_node->prev = 0; + new_node->height = ++tree->depth; + + node_desc.next = cpu_to_be32(new_node->next); + node_desc.prev = cpu_to_be32(new_node->prev); + node_desc.type = new_node->type; + node_desc.height = new_node->height; + node_desc.num_recs = cpu_to_be16(new_node->num_recs); + node_desc.reserved = 0; + hfs_bnode_write(new_node, &node_desc, 0, sizeof(node_desc)); + + rec = tree->node_size - 2; + hfs_bnode_write_u16(new_node, rec, 14); + + if (node) { + /* insert old root idx into new root */ + node->parent = tree->root; + if (node->type == HFS_NODE_LEAF || + tree->attributes & HFS_TREE_VARIDXKEYS) + key_size = hfs_bnode_read_u8(node, 14) + 1; + else + key_size = tree->max_key_len + 1; + hfs_bnode_copy(new_node, 14, node, 14, key_size); + + if (!(tree->attributes & HFS_TREE_VARIDXKEYS)) { + key_size = tree->max_key_len + 1; + hfs_bnode_write_u8(new_node, 14, tree->max_key_len); + } + key_size = (key_size + 1) & -2; + cnid = cpu_to_be32(node->this); + hfs_bnode_write(new_node, &cnid, 14 + key_size, 4); + + rec -= 2; + hfs_bnode_write_u16(new_node, rec, 14 + key_size + 4); + + hfs_bnode_put(node); + } + hfs_bnode_put(new_node); + mark_inode_dirty(tree->inode); + + return 0; +} diff --git a/fs/hfs/btree.c b/fs/hfs/btree.c new file mode 100644 index 000000000..41b4e8fc9 --- /dev/null +++ b/fs/hfs/btree.c @@ -0,0 +1,544 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hfs/btree.c + * + * Copyright (C) 2001 + * Brad Boyer (flar@allandria.com) + * (C) 2003 Ardis Technologies + * + * Handle opening/closing btree + */ + +#include +#include +#include + +#include "btree.h" + +/* Context for iterating b-tree map pages + * @page_idx: The index of the page within the b-node's page array + * @off: The byte offset within the mapped page + * @len: The remaining length of the map record + */ +struct hfs_bmap_ctx { + unsigned int page_idx; + unsigned int off; + u16 len; +}; + +/* + * Finds the specific page containing the requested byte offset within the map + * record. Automatically handles the difference between header and map nodes. + * Returns the struct page pointer, or an ERR_PTR on failure. + * Note: The caller is responsible for mapping/unmapping the returned page. + */ +static struct page *hfs_bmap_get_map_page(struct hfs_bnode *node, + struct hfs_bmap_ctx *ctx, + u32 byte_offset) +{ + u16 rec_idx, off16; + unsigned int page_off; + + if (node->this == HFS_TREE_HEAD) { + if (node->type != HFS_NODE_HEADER) { + pr_err("hfs: invalid btree header node\n"); + return ERR_PTR(-EIO); + } + rec_idx = HFS_BTREE_HDR_MAP_REC_INDEX; + } else { + if (node->type != HFS_NODE_MAP) { + pr_err("hfs: invalid btree map node\n"); + return ERR_PTR(-EIO); + } + rec_idx = HFS_BTREE_MAP_NODE_REC_INDEX; + } + + ctx->len = hfs_brec_lenoff(node, rec_idx, &off16); + if (!ctx->len) + return ERR_PTR(-ENOENT); + + if (!is_bnode_offset_valid(node, off16)) + return ERR_PTR(-EIO); + + ctx->len = check_and_correct_requested_length(node, off16, ctx->len); + + if (byte_offset >= ctx->len) + return ERR_PTR(-EINVAL); + + page_off = (u32)off16 + node->page_offset + byte_offset; + ctx->page_idx = page_off >> PAGE_SHIFT; + ctx->off = page_off & ~PAGE_MASK; + + return node->page[ctx->page_idx]; +} + +/** + * hfs_bmap_test_bit - test a bit in the b-tree map + * @node: the b-tree node containing the map record + * @node_bit_idx: the relative bit index within the node's map record + * + * Returns true if set, false if clear or on failure. + */ +static bool hfs_bmap_test_bit(struct hfs_bnode *node, u32 node_bit_idx) +{ + struct hfs_bmap_ctx ctx; + struct page *page; + u8 *bmap, byte, mask; + + page = hfs_bmap_get_map_page(node, &ctx, node_bit_idx / BITS_PER_BYTE); + if (IS_ERR(page)) + return false; + + bmap = kmap_local_page(page); + byte = bmap[ctx.off]; + kunmap_local(bmap); + + mask = 1 << (7 - (node_bit_idx % BITS_PER_BYTE)); + return (byte & mask) != 0; +} + +/** + * hfs_bmap_clear_bit - clear a bit in the b-tree map + * @node: the b-tree node containing the map record + * @node_bit_idx: the relative bit index within the node's map record + * + * Returns 0 on success, -EINVAL if already clear, or negative error code. + */ +static int hfs_bmap_clear_bit(struct hfs_bnode *node, u32 node_bit_idx) +{ + struct hfs_bmap_ctx ctx; + struct page *page; + u8 *bmap, mask; + + page = hfs_bmap_get_map_page(node, &ctx, node_bit_idx / BITS_PER_BYTE); + if (IS_ERR(page)) + return PTR_ERR(page); + + bmap = kmap_local_page(page); + + mask = 1 << (7 - (node_bit_idx % BITS_PER_BYTE)); + + if (!(bmap[ctx.off] & mask)) { + kunmap_local(bmap); + return -EINVAL; + } + + bmap[ctx.off] &= ~mask; + set_page_dirty(page); + kunmap_local(bmap); + + return 0; +} + +/* Get a reference to a B*Tree and do some initial checks */ +struct hfs_btree *hfs_btree_open(struct super_block *sb, u32 id, btree_keycmp keycmp) +{ + struct hfs_btree *tree; + struct hfs_btree_header_rec *head; + struct address_space *mapping; + struct folio *folio; + struct buffer_head *bh; + struct hfs_bnode *node; + unsigned int size; + u16 dblock; + sector_t start_block; + loff_t offset; + + tree = kzalloc_obj(*tree); + if (!tree) + return NULL; + + mutex_init(&tree->tree_lock); + spin_lock_init(&tree->hash_lock); + /* Set the correct compare function */ + tree->sb = sb; + tree->cnid = id; + tree->keycmp = keycmp; + + tree->inode = iget_locked(sb, id); + if (!tree->inode) + goto free_tree; + BUG_ON(!(inode_state_read_once(tree->inode) & I_NEW)); + { + struct hfs_mdb *mdb = HFS_SB(sb)->mdb; + HFS_I(tree->inode)->flags = 0; + mutex_init(&HFS_I(tree->inode)->extents_lock); + switch (id) { + case HFS_EXT_CNID: + hfs_inode_read_fork(tree->inode, mdb->drXTExtRec, mdb->drXTFlSize, + mdb->drXTFlSize, be32_to_cpu(mdb->drXTClpSiz)); + if (HFS_I(tree->inode)->alloc_blocks > + HFS_I(tree->inode)->first_blocks) { + pr_err("invalid btree extent records\n"); + unlock_new_inode(tree->inode); + goto free_inode; + } + + tree->inode->i_mapping->a_ops = &hfs_btree_aops; + break; + case HFS_CAT_CNID: + hfs_inode_read_fork(tree->inode, mdb->drCTExtRec, mdb->drCTFlSize, + mdb->drCTFlSize, be32_to_cpu(mdb->drCTClpSiz)); + + if (!HFS_I(tree->inode)->first_blocks) { + pr_err("invalid btree extent records (0 size)\n"); + unlock_new_inode(tree->inode); + goto free_inode; + } + + tree->inode->i_mapping->a_ops = &hfs_btree_aops; + break; + default: + BUG(); + } + } + unlock_new_inode(tree->inode); + + mapping = tree->inode->i_mapping; + folio = filemap_grab_folio(mapping, 0); + if (IS_ERR(folio)) + goto free_inode; + + folio_zero_range(folio, 0, folio_size(folio)); + + dblock = hfs_ext_find_block(HFS_I(tree->inode)->first_extents, 0); + start_block = HFS_SB(sb)->fs_start + (dblock * HFS_SB(sb)->fs_div); + + size = folio_size(folio); + offset = 0; + while (size > 0) { + size_t len; + + bh = sb_bread(sb, start_block); + if (!bh) { + pr_err("unable to read tree header\n"); + goto put_folio; + } + + len = min_t(size_t, folio_size(folio), sb->s_blocksize); + memcpy_to_folio(folio, offset, bh->b_data, sb->s_blocksize); + + brelse(bh); + + start_block++; + offset += len; + size -= len; + } + + folio_mark_uptodate(folio); + + /* Load the header */ + head = (struct hfs_btree_header_rec *)(kmap_local_folio(folio, 0) + + sizeof(struct hfs_bnode_desc)); + tree->root = be32_to_cpu(head->root); + tree->leaf_count = be32_to_cpu(head->leaf_count); + tree->leaf_head = be32_to_cpu(head->leaf_head); + tree->leaf_tail = be32_to_cpu(head->leaf_tail); + tree->node_count = be32_to_cpu(head->node_count); + tree->free_nodes = be32_to_cpu(head->free_nodes); + tree->attributes = be32_to_cpu(head->attributes); + tree->node_size = be16_to_cpu(head->node_size); + tree->max_key_len = be16_to_cpu(head->max_key_len); + tree->depth = be16_to_cpu(head->depth); + + size = tree->node_size; + if (!is_power_of_2(size)) + goto fail_folio; + if (!tree->node_count) + goto fail_folio; + switch (id) { + case HFS_EXT_CNID: + if (tree->max_key_len != HFS_MAX_EXT_KEYLEN) { + pr_err("invalid extent max_key_len %d\n", + tree->max_key_len); + goto fail_folio; + } + break; + case HFS_CAT_CNID: + if (tree->max_key_len != HFS_MAX_CAT_KEYLEN) { + pr_err("invalid catalog max_key_len %d\n", + tree->max_key_len); + goto fail_folio; + } + break; + default: + BUG(); + } + + tree->node_size_shift = ffs(size) - 1; + tree->pages_per_bnode = (tree->node_size + PAGE_SIZE - 1) >> PAGE_SHIFT; + + kunmap_local(head); + folio_unlock(folio); + folio_put(folio); + + node = hfs_bnode_find(tree, HFS_TREE_HEAD); + if (IS_ERR(node)) + goto free_inode; + + if (!hfs_bmap_test_bit(node, HFS_TREE_HEAD)) { + pr_warn("(%s): %s (cnid 0x%x) bitmap corrupted, forcing rdonly\n", + sb->s_id, id == HFS_EXT_CNID ? "extents" : "catalog", id); + pr_warn("Run fsck.hfs to repair.\n"); + sb->s_flags |= SB_RDONLY; + } + + hfs_bnode_put(node); + + return tree; + +fail_folio: + kunmap_local(head); +put_folio: + folio_unlock(folio); + folio_put(folio); +free_inode: + tree->inode->i_mapping->a_ops = &hfs_aops; + iput(tree->inode); +free_tree: + kfree(tree); + return NULL; +} + +/* Release resources used by a btree */ +void hfs_btree_close(struct hfs_btree *tree) +{ + struct hfs_bnode *node; + int i; + + if (!tree) + return; + + for (i = 0; i < NODE_HASH_SIZE; i++) { + while ((node = tree->node_hash[i])) { + tree->node_hash[i] = node->next_hash; + if (atomic_read(&node->refcnt)) + pr_err("node %d:%d still has %d user(s)!\n", + node->tree->cnid, node->this, + atomic_read(&node->refcnt)); + hfs_bnode_free(node); + tree->node_hash_cnt--; + } + } + iput(tree->inode); + kfree(tree); +} + +void hfs_btree_write(struct hfs_btree *tree) +{ + struct hfs_btree_header_rec *head; + struct hfs_bnode *node; + struct page *page; + + node = hfs_bnode_find(tree, 0); + if (IS_ERR(node)) + /* panic? */ + return; + /* Load the header */ + page = node->page[0]; + head = (struct hfs_btree_header_rec *)(kmap_local_page(page) + + sizeof(struct hfs_bnode_desc)); + + head->root = cpu_to_be32(tree->root); + head->leaf_count = cpu_to_be32(tree->leaf_count); + head->leaf_head = cpu_to_be32(tree->leaf_head); + head->leaf_tail = cpu_to_be32(tree->leaf_tail); + head->node_count = cpu_to_be32(tree->node_count); + head->free_nodes = cpu_to_be32(tree->free_nodes); + head->attributes = cpu_to_be32(tree->attributes); + head->depth = cpu_to_be16(tree->depth); + + kunmap_local(head); + set_page_dirty(page); + hfs_bnode_put(node); +} + +static struct hfs_bnode *hfs_bmap_new_bmap(struct hfs_bnode *prev, u32 idx) +{ + struct hfs_btree *tree = prev->tree; + struct hfs_bnode *node; + struct hfs_bnode_desc desc; + __be32 cnid; + + node = hfs_bnode_create(tree, idx); + if (IS_ERR(node)) + return node; + + if (!tree->free_nodes) + panic("FIXME!!!"); + tree->free_nodes--; + prev->next = idx; + cnid = cpu_to_be32(idx); + hfs_bnode_write(prev, &cnid, offsetof(struct hfs_bnode_desc, next), 4); + + node->type = HFS_NODE_MAP; + node->num_recs = 1; + hfs_bnode_clear(node, 0, tree->node_size); + desc.next = 0; + desc.prev = 0; + desc.type = HFS_NODE_MAP; + desc.height = 0; + desc.num_recs = cpu_to_be16(1); + desc.reserved = 0; + hfs_bnode_write(node, &desc, 0, sizeof(desc)); + hfs_bnode_write_u16(node, 14, 0x8000); + hfs_bnode_write_u16(node, tree->node_size - 2, 14); + hfs_bnode_write_u16(node, tree->node_size - 4, tree->node_size - 6); + + return node; +} + +/* Make sure @tree has enough space for the @rsvd_nodes */ +int hfs_bmap_reserve(struct hfs_btree *tree, u32 rsvd_nodes) +{ + struct inode *inode = tree->inode; + u32 count; + int res; + + while (tree->free_nodes < rsvd_nodes) { + res = hfs_extend_file(inode); + if (res) + return res; + HFS_I(inode)->phys_size = inode->i_size = + (loff_t)HFS_I(inode)->alloc_blocks * + HFS_SB(tree->sb)->alloc_blksz; + HFS_I(inode)->fs_blocks = inode->i_size >> + tree->sb->s_blocksize_bits; + inode_set_bytes(inode, inode->i_size); + count = inode->i_size >> tree->node_size_shift; + tree->free_nodes += count - tree->node_count; + tree->node_count = count; + } + return 0; +} + +struct hfs_bnode *hfs_bmap_alloc(struct hfs_btree *tree) +{ + struct hfs_bnode *node, *next_node; + struct hfs_bmap_ctx ctx; + struct page *page; + u32 nidx, idx; + u8 *data, byte, m; + int i, res; + + res = hfs_bmap_reserve(tree, 1); + if (res) + return ERR_PTR(res); + + nidx = 0; + node = hfs_bnode_find(tree, nidx); + if (IS_ERR(node)) + return node; + + page = hfs_bmap_get_map_page(node, &ctx, 0); + if (IS_ERR(page)) { + res = PTR_ERR(page); + hfs_bnode_put(node); + return ERR_PTR(res); + } + + data = kmap_local_page(page); + idx = 0; + + for (;;) { + while (ctx.len) { + byte = data[ctx.off]; + if (byte != 0xff) { + for (m = 0x80, i = 0; i < 8; m >>= 1, i++) { + if (!(byte & m)) { + idx += i; + data[ctx.off] |= m; + set_page_dirty(page); + kunmap_local(data); + tree->free_nodes--; + mark_inode_dirty(tree->inode); + hfs_bnode_put(node); + return hfs_bnode_create(tree, idx); + } + } + } + if (++ctx.off >= PAGE_SIZE) { + kunmap_local(data); + page = node->page[++ctx.page_idx]; + data = kmap_local_page(page); + ctx.off = 0; + } + idx += 8; + ctx.len--; + } + kunmap_local(data); + nidx = node->next; + if (!nidx) { + printk(KERN_DEBUG "create new bmap node...\n"); + next_node = hfs_bmap_new_bmap(node, idx); + } else + next_node = hfs_bnode_find(tree, nidx); + hfs_bnode_put(node); + if (IS_ERR(next_node)) + return next_node; + node = next_node; + + page = hfs_bmap_get_map_page(node, &ctx, 0); + if (IS_ERR(page)) { + res = PTR_ERR(page); + hfs_bnode_put(node); + return ERR_PTR(res); + } + data = kmap_local_page(page); + } +} + +void hfs_bmap_free(struct hfs_bnode *node) +{ + struct hfs_btree *tree; + u16 off, len; + u32 nidx; + int res; + + hfs_dbg("node %u\n", node->this); + tree = node->tree; + nidx = node->this; + node = hfs_bnode_find(tree, 0); + if (IS_ERR(node)) + return; + len = hfs_brec_lenoff(node, 2, &off); + while (nidx >= len * 8) { + u32 i; + + nidx -= len * 8; + i = node->next; + if (!i) { + /* panic */; + pr_crit("unable to free bnode %u. bmap not found!\n", + node->this); + hfs_bnode_put(node); + return; + } + hfs_bnode_put(node); + node = hfs_bnode_find(tree, i); + if (IS_ERR(node)) + return; + if (node->type != HFS_NODE_MAP) { + /* panic */; + pr_crit("invalid bmap found! (%u,%d)\n", + node->this, node->type); + hfs_bnode_put(node); + return; + } + len = hfs_brec_lenoff(node, 0, &off); + } + + res = hfs_bmap_clear_bit(node, nidx); + if (res == -EINVAL) { + pr_crit("trying to free free bnode %u(%d)\n", + nidx, node->type); + } else if (res) { + pr_crit("fail to free bnode %u(%d)\n", + nidx, node->type); + } else { + tree->free_nodes++; + mark_inode_dirty(tree->inode); + } + + hfs_bnode_put(node); +} diff --git a/fs/hfs/btree.h b/fs/hfs/btree.h new file mode 100644 index 000000000..b4c3f2a31 --- /dev/null +++ b/fs/hfs/btree.h @@ -0,0 +1,171 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * linux/fs/hfs/btree.h + * + * Copyright (C) 2001 + * Brad Boyer (flar@allandria.com) + * (C) 2003 Ardis Technologies + */ + +#include "hfs_fs.h" + +typedef int (*btree_keycmp)(const btree_key *, const btree_key *); + +#define NODE_HASH_SIZE 256 + +/* B-tree mutex nested subclasses */ +enum hfs_btree_mutex_classes { + CATALOG_BTREE_MUTEX, + EXTENTS_BTREE_MUTEX, + ATTR_BTREE_MUTEX, +}; + +/* A HFS BTree held in memory */ +struct hfs_btree { + struct super_block *sb; + struct inode *inode; + btree_keycmp keycmp; + + u32 cnid; + u32 root; + u32 leaf_count; + u32 leaf_head; + u32 leaf_tail; + u32 node_count; + u32 free_nodes; + u32 attributes; + + unsigned int node_size; + unsigned int node_size_shift; + unsigned int max_key_len; + unsigned int depth; + + //unsigned int map1_size, map_size; + struct mutex tree_lock; + + unsigned int pages_per_bnode; + spinlock_t hash_lock; + struct hfs_bnode *node_hash[NODE_HASH_SIZE]; + int node_hash_cnt; +}; + +/* A HFS BTree node in memory */ +struct hfs_bnode { + struct hfs_btree *tree; + + u32 prev; + u32 this; + u32 next; + u32 parent; + + u16 num_recs; + u8 type; + u8 height; + + struct hfs_bnode *next_hash; + unsigned long flags; + wait_queue_head_t lock_wq; + atomic_t refcnt; + unsigned int page_offset; + struct page *page[]; +}; + +#define HFS_BNODE_ERROR 0 +#define HFS_BNODE_NEW 1 +#define HFS_BNODE_DELETED 2 + +struct hfs_find_data { + btree_key *key; + btree_key *search_key; + struct hfs_btree *tree; + struct hfs_bnode *bnode; + int record; + int keyoffset, keylength; + int entryoffset, entrylength; +}; + + +/* btree.c */ +extern struct hfs_btree *hfs_btree_open(struct super_block *sb, u32 id, + btree_keycmp keycmp); +extern void hfs_btree_close(struct hfs_btree *tree); +extern void hfs_btree_write(struct hfs_btree *tree); +extern int hfs_bmap_reserve(struct hfs_btree *tree, u32 rsvd_nodes); +extern struct hfs_bnode *hfs_bmap_alloc(struct hfs_btree *tree); +extern void hfs_bmap_free(struct hfs_bnode *node); + +/* bnode.c */ +extern void hfs_bnode_read(struct hfs_bnode *node, void *buf, u32 off, u32 len); +extern u16 hfs_bnode_read_u16(struct hfs_bnode *node, u32 off); +extern u8 hfs_bnode_read_u8(struct hfs_bnode *node, u32 off); +extern void hfs_bnode_read_key(struct hfs_bnode *node, void *key, u32 off); +extern void hfs_bnode_write(struct hfs_bnode *node, void *buf, u32 off, u32 len); +extern void hfs_bnode_write_u16(struct hfs_bnode *node, u32 off, u16 data); +extern void hfs_bnode_write_u8(struct hfs_bnode *node, u32 off, u8 data); +extern void hfs_bnode_clear(struct hfs_bnode *node, u32 off, u32 len); +extern void hfs_bnode_copy(struct hfs_bnode *dst_node, u32 dst, + struct hfs_bnode *src_node, u32 src, u32 len); +extern void hfs_bnode_move(struct hfs_bnode *node, u32 dst, u32 src, u32 len); +extern void hfs_bnode_dump(struct hfs_bnode *node); +extern void hfs_bnode_unlink(struct hfs_bnode *node); +extern struct hfs_bnode *hfs_bnode_findhash(struct hfs_btree *tree, u32 cnid); +extern struct hfs_bnode *hfs_bnode_find(struct hfs_btree *tree, u32 num); +extern void hfs_bnode_unhash(struct hfs_bnode *node); +extern void hfs_bnode_free(struct hfs_bnode *node); +extern struct hfs_bnode *hfs_bnode_create(struct hfs_btree *tree, u32 num); +extern void hfs_bnode_get(struct hfs_bnode *node); +extern void hfs_bnode_put(struct hfs_bnode *node); + +/* brec.c */ +extern u16 hfs_brec_lenoff(struct hfs_bnode *node, u16 rec, u16 *off); +extern u16 hfs_brec_keylen(struct hfs_bnode *node, u16 rec); +extern int hfs_brec_insert(struct hfs_find_data *fd, void *entry, u32 entry_len); +extern int hfs_brec_remove(struct hfs_find_data *fd); + +/* bfind.c */ +extern int hfs_find_init(struct hfs_btree *tree, struct hfs_find_data *fd); +extern void hfs_find_exit(struct hfs_find_data *fd); +extern int __hfs_brec_find(struct hfs_bnode *bnode, struct hfs_find_data *fd); +extern int hfs_brec_find(struct hfs_find_data *fd); +extern int hfs_brec_read(struct hfs_find_data *fd, void *rec, u32 rec_len); +extern int hfs_brec_goto(struct hfs_find_data *fd, int cnt); + +static inline bool is_bnode_offset_valid(struct hfs_bnode *node, u32 off) +{ + bool is_valid; + + if (!node || !node->tree) + return false; + + is_valid = off < node->tree->node_size; + + if (!is_valid) { + pr_err("invalid offset: id %u, type %#x, h %u, sz %u, off %u\n", + node->this, node->type, node->height, + node->tree->node_size, off); + } + + return is_valid; +} + +static inline u32 check_and_correct_requested_length(struct hfs_bnode *node, u32 off, u32 len) +{ + unsigned int node_size; + + if (!is_bnode_offset_valid(node, off)) + return 0; + + node_size = node->tree->node_size; + + if ((u64)off + len > node_size) { + u32 new_len = node_size - off; + + pr_err("corrected len: id %u, type %#x, h %u, sz %u, off %u, len %u->%u\n", + node->this, node->type, node->height, + node_size, off, len, new_len); + + return new_len; + } + + return len; +} diff --git a/fs/hfs/catalog.c b/fs/hfs/catalog.c new file mode 100644 index 000000000..490c5e130 --- /dev/null +++ b/fs/hfs/catalog.c @@ -0,0 +1,536 @@ +/* + * linux/fs/hfs/catalog.c + * + * Copyright (C) 1995-1997 Paul H. Hargrove + * (C) 2003 Ardis Technologies + * This file may be distributed under the terms of the GNU General Public License. + * + * This file contains the functions related to the catalog B-tree. + * + * Cache code shamelessly stolen from + * linux/fs/inode.c Copyright (C) 1991, 1992 Linus Torvalds + * re-shamelessly stolen Copyright (C) 1997 Linus Torvalds + */ + +#include "hfs_fs.h" +#include "btree.h" + +/* + * hfs_cat_build_key() + * + * Given the ID of the parent and the name build a search key. + */ +void hfs_cat_build_key(struct super_block *sb, btree_key *key, u32 parent, const struct qstr *name) +{ + key->cat.reserved = 0; + key->cat.ParID = cpu_to_be32(parent); + if (name) { + hfs_asc2mac(sb, &key->cat.CName, name); + key->key_len = 6 + key->cat.CName.len; + } else { + memset(&key->cat.CName, 0, sizeof(struct hfs_name)); + key->key_len = 6; + } +} + +static int hfs_cat_build_record(hfs_cat_rec *rec, u32 cnid, struct inode *inode) +{ + __be32 mtime = hfs_mtime(); + + memset(rec, 0, sizeof(*rec)); + if (S_ISDIR(inode->i_mode)) { + rec->type = HFS_CDR_DIR; + rec->dir.DirID = cpu_to_be32(cnid); + rec->dir.CrDat = mtime; + rec->dir.MdDat = mtime; + rec->dir.BkDat = 0; + rec->dir.UsrInfo.frView = cpu_to_be16(0xff); + return sizeof(struct hfs_cat_dir); + } else { + /* init some fields for the file record */ + rec->type = HFS_CDR_FIL; + rec->file.Flags = HFS_FIL_USED | HFS_FIL_THD; + if (!(inode->i_mode & S_IWUSR)) + rec->file.Flags |= HFS_FIL_LOCK; + rec->file.FlNum = cpu_to_be32(cnid); + rec->file.CrDat = mtime; + rec->file.MdDat = mtime; + rec->file.BkDat = 0; + rec->file.UsrWds.fdType = HFS_SB(inode->i_sb)->s_type; + rec->file.UsrWds.fdCreator = HFS_SB(inode->i_sb)->s_creator; + return sizeof(struct hfs_cat_file); + } +} + +static int hfs_cat_build_thread(struct super_block *sb, + hfs_cat_rec *rec, int type, + u32 parentid, const struct qstr *name) +{ + rec->type = type; + memset(rec->thread.reserved, 0, sizeof(rec->thread.reserved)); + rec->thread.ParID = cpu_to_be32(parentid); + hfs_asc2mac(sb, &rec->thread.CName, name); + return sizeof(struct hfs_cat_thread); +} + +/* + * create_entry() + * + * Add a new file or directory to the catalog B-tree and + * return a (struct hfs_cat_entry) for it in '*result'. + */ +int hfs_cat_create(u32 cnid, struct inode *dir, const struct qstr *str, struct inode *inode) +{ + struct hfs_find_data fd; + struct super_block *sb; + union hfs_cat_rec entry; + int entry_size; + int err; + + hfs_dbg("name %s, cnid %u, i_nlink %d\n", + str->name, cnid, inode->i_nlink); + if (dir->i_size >= HFS_MAX_VALENCE) + return -ENOSPC; + + sb = dir->i_sb; + err = hfs_find_init(HFS_SB(sb)->cat_tree, &fd); + if (err) + return err; + + /* + * Fail early and avoid ENOSPC during the btree operations. We may + * have to split the root node at most once. + */ + err = hfs_bmap_reserve(fd.tree, 2 * fd.tree->depth); + if (err) + goto err2; + + hfs_cat_build_key(sb, fd.search_key, cnid, NULL); + entry_size = hfs_cat_build_thread(sb, &entry, S_ISDIR(inode->i_mode) ? + HFS_CDR_THD : HFS_CDR_FTH, + dir->i_ino, str); + err = hfs_brec_find(&fd); + if (err != -ENOENT) { + if (!err) + err = -EEXIST; + goto err2; + } + err = hfs_brec_insert(&fd, &entry, entry_size); + if (err) + goto err2; + + hfs_cat_build_key(sb, fd.search_key, dir->i_ino, str); + entry_size = hfs_cat_build_record(&entry, cnid, inode); + err = hfs_brec_find(&fd); + if (err != -ENOENT) { + /* panic? */ + if (!err) + err = -EEXIST; + goto err1; + } + err = hfs_brec_insert(&fd, &entry, entry_size); + if (err) + goto err1; + + dir->i_size++; + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + mark_inode_dirty(dir); + hfs_find_exit(&fd); + return 0; + +err1: + hfs_cat_build_key(sb, fd.search_key, cnid, NULL); + if (!hfs_brec_find(&fd)) + hfs_brec_remove(&fd); +err2: + hfs_find_exit(&fd); + return err; +} + +/* + * hfs_cat_compare() + * + * Description: + * This is the comparison function used for the catalog B-tree. In + * comparing catalog B-tree entries, the parent id is the most + * significant field (compared as unsigned ints). The name field is + * the least significant (compared in "Macintosh lexical order", + * see hfs_strcmp() in string.c) + * Input Variable(s): + * struct hfs_cat_key *key1: pointer to the first key to compare + * struct hfs_cat_key *key2: pointer to the second key to compare + * Output Variable(s): + * NONE + * Returns: + * int: negative if key1key2, and 0 if key1==key2 + * Preconditions: + * key1 and key2 point to "valid" (struct hfs_cat_key)s. + * Postconditions: + * This function has no side-effects + */ +int hfs_cat_keycmp(const btree_key *key1, const btree_key *key2) +{ + __be32 k1p, k2p; + + k1p = key1->cat.ParID; + k2p = key2->cat.ParID; + + if (k1p != k2p) + return be32_to_cpu(k1p) < be32_to_cpu(k2p) ? -1 : 1; + + return hfs_strcmp(key1->cat.CName.name, key1->cat.CName.len, + key2->cat.CName.name, key2->cat.CName.len); +} + +static int hfs_cat_validate_found_cnid(struct hfs_find_data *fd, u32 cnid) +{ + hfs_cat_rec rec; + u32 found_cnid; + int rec_len; + + rec_len = fd->entrylength; + if (rec_len <= 0 || (size_t)rec_len > sizeof(rec)) + return -EIO; + + memset(&rec, 0, sizeof(rec)); + hfs_bnode_read(fd->bnode, &rec, fd->entryoffset, rec_len); + + switch (rec.type) { + case HFS_CDR_FIL: + if ((size_t)rec_len != sizeof(struct hfs_cat_file)) + return -EIO; + found_cnid = be32_to_cpu(rec.file.FlNum); + break; + case HFS_CDR_DIR: + if ((size_t)rec_len != sizeof(struct hfs_cat_dir)) + return -EIO; + found_cnid = be32_to_cpu(rec.dir.DirID); + break; + default: + return -EIO; + } + + if (found_cnid != cnid) + return -EIO; + + return 0; +} + +/* Try to get a catalog entry for given catalog id */ +// move to read_super??? +int hfs_cat_find_brec(struct super_block *sb, u32 cnid, + struct hfs_find_data *fd) +{ + hfs_cat_rec rec; + int res, len, type; + + hfs_cat_build_key(sb, fd->search_key, cnid, NULL); + res = hfs_brec_read(fd, &rec, sizeof(rec)); + if (res) + return res; + + type = rec.type; + if (type != HFS_CDR_THD && type != HFS_CDR_FTH) { + pr_err("found bad thread record in catalog\n"); + return -EIO; + } + + fd->search_key->cat.ParID = rec.thread.ParID; + len = fd->search_key->cat.CName.len = rec.thread.CName.len; + if (len > HFS_NAMELEN) { + pr_err("bad catalog namelength\n"); + return -EIO; + } + memcpy(fd->search_key->cat.CName.name, rec.thread.CName.name, len); + + res = hfs_brec_find(fd); + if (res) + return res; + + return hfs_cat_validate_found_cnid(fd, cnid); +} + +static inline +void hfs_set_next_unused_CNID(struct super_block *sb, + u32 deleted_cnid, u32 found_cnid) +{ + if (found_cnid < HFS_FIRSTUSER_CNID) { + atomic64_cmpxchg(&HFS_SB(sb)->next_id, + deleted_cnid + 1, HFS_FIRSTUSER_CNID); + } else { + atomic64_cmpxchg(&HFS_SB(sb)->next_id, + deleted_cnid + 1, found_cnid + 1); + } +} + +/* + * hfs_correct_next_unused_CNID() + * + * Correct the next unused CNID of Catalog Tree. + */ +static +int hfs_correct_next_unused_CNID(struct super_block *sb, u32 cnid) +{ + struct hfs_btree *cat_tree; + struct hfs_bnode *node; + s64 leaf_head; + s64 leaf_tail; + s64 node_id; + + hfs_dbg("cnid %u, next_id %lld\n", + cnid, atomic64_read(&HFS_SB(sb)->next_id)); + + if ((cnid + 1) < atomic64_read(&HFS_SB(sb)->next_id)) { + /* next ID should be unchanged */ + return 0; + } + + cat_tree = HFS_SB(sb)->cat_tree; + leaf_head = cat_tree->leaf_head; + leaf_tail = cat_tree->leaf_tail; + + if (leaf_head > leaf_tail) { + pr_err("node is corrupted: leaf_head %lld, leaf_tail %lld\n", + leaf_head, leaf_tail); + return -ERANGE; + } + + node = hfs_bnode_find(cat_tree, leaf_tail); + if (IS_ERR(node)) { + pr_err("fail to find leaf node: node ID %lld\n", + leaf_tail); + return -ENOENT; + } + + node_id = leaf_tail; + + do { + int i; + + if (node_id != leaf_tail) { + node = hfs_bnode_find(cat_tree, node_id); + if (IS_ERR(node)) + return -ENOENT; + } + + hfs_dbg("node %lld, leaf_tail %lld, leaf_head %lld\n", + node_id, leaf_tail, leaf_head); + + hfs_bnode_dump(node); + + for (i = node->num_recs - 1; i >= 0; i--) { + hfs_cat_rec rec; + u16 off, len, keylen; + int entryoffset; + int entrylength; + u32 found_cnid; + + len = hfs_brec_lenoff(node, i, &off); + keylen = hfs_brec_keylen(node, i); + if (keylen == 0) { + pr_err("fail to get the keylen: " + "node_id %lld, record index %d\n", + node_id, i); + return -EINVAL; + } + + entryoffset = off + keylen; + entrylength = len - keylen; + + if (entrylength > sizeof(rec)) { + pr_err("unexpected record length: " + "entrylength %d\n", + entrylength); + return -EINVAL; + } + + hfs_bnode_read(node, &rec, entryoffset, entrylength); + + if (rec.type == HFS_CDR_DIR) { + found_cnid = be32_to_cpu(rec.dir.DirID); + hfs_dbg("found_cnid %u\n", found_cnid); + hfs_set_next_unused_CNID(sb, cnid, found_cnid); + hfs_bnode_put(node); + return 0; + } else if (rec.type == HFS_CDR_FIL) { + found_cnid = be32_to_cpu(rec.file.FlNum); + hfs_dbg("found_cnid %u\n", found_cnid); + hfs_set_next_unused_CNID(sb, cnid, found_cnid); + hfs_bnode_put(node); + return 0; + } + } + + node_id = node->prev; + hfs_bnode_put(node); + + } while (node_id >= leaf_head); + + return -ENOENT; +} + +/* + * hfs_cat_delete() + * + * Delete the indicated file or directory. + * The associated thread is also removed unless ('with_thread'==0). + */ +int hfs_cat_delete(u32 cnid, struct inode *dir, const struct qstr *str) +{ + struct super_block *sb; + struct hfs_find_data fd; + int res, type; + + hfs_dbg("name %s, cnid %u\n", str ? str->name : NULL, cnid); + sb = dir->i_sb; + res = hfs_find_init(HFS_SB(sb)->cat_tree, &fd); + if (res) + return res; + + hfs_cat_build_key(sb, fd.search_key, dir->i_ino, str); + res = hfs_brec_find(&fd); + if (res) + goto out; + + type = hfs_bnode_read_u8(fd.bnode, fd.entryoffset); + if (type == HFS_CDR_FIL) { + struct hfs_cat_file file; + hfs_bnode_read(fd.bnode, &file, fd.entryoffset, sizeof(file)); + if (be32_to_cpu(file.FlNum) == cnid) { +#if 0 + hfs_free_fork(sb, &file, HFS_FK_DATA); +#endif + hfs_free_fork(sb, &file, HFS_FK_RSRC); + } + } + + res = hfs_brec_remove(&fd); + if (res) + goto out; + + hfs_cat_build_key(sb, fd.search_key, cnid, NULL); + res = hfs_brec_find(&fd); + if (!res) { + res = hfs_brec_remove(&fd); + if (res) + goto out; + } + + dir->i_size--; + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + mark_inode_dirty(dir); + + res = hfs_correct_next_unused_CNID(sb, cnid); + if (res) + goto out; + + res = 0; +out: + hfs_find_exit(&fd); + + return res; +} + +/* + * hfs_cat_move() + * + * Rename a file or directory, possibly to a new directory. + * If the destination exists it is removed and a + * (struct hfs_cat_entry) for it is returned in '*result'. + */ +int hfs_cat_move(u32 cnid, struct inode *src_dir, const struct qstr *src_name, + struct inode *dst_dir, const struct qstr *dst_name) +{ + struct super_block *sb; + struct hfs_find_data src_fd, dst_fd; + union hfs_cat_rec entry; + int entry_size, type; + int err; + + hfs_dbg("cnid %u - (ino %llu, name %s) - (ino %llu, name %s)\n", + cnid, src_dir->i_ino, src_name->name, + dst_dir->i_ino, dst_name->name); + sb = src_dir->i_sb; + err = hfs_find_init(HFS_SB(sb)->cat_tree, &src_fd); + if (err) + return err; + dst_fd = src_fd; + + /* + * Fail early and avoid ENOSPC during the btree operations. We may + * have to split the root node at most once. + */ + err = hfs_bmap_reserve(src_fd.tree, 2 * src_fd.tree->depth); + if (err) + goto out; + + /* find the old dir entry and read the data */ + hfs_cat_build_key(sb, src_fd.search_key, src_dir->i_ino, src_name); + err = hfs_brec_find(&src_fd); + if (err) + goto out; + if (src_fd.entrylength > sizeof(entry) || src_fd.entrylength < 0) { + err = -EIO; + goto out; + } + + hfs_bnode_read(src_fd.bnode, &entry, src_fd.entryoffset, + src_fd.entrylength); + + /* create new dir entry with the data from the old entry */ + hfs_cat_build_key(sb, dst_fd.search_key, dst_dir->i_ino, dst_name); + err = hfs_brec_find(&dst_fd); + if (err != -ENOENT) { + if (!err) + err = -EEXIST; + goto out; + } + + err = hfs_brec_insert(&dst_fd, &entry, src_fd.entrylength); + if (err) + goto out; + dst_dir->i_size++; + inode_set_mtime_to_ts(dst_dir, inode_set_ctime_current(dst_dir)); + mark_inode_dirty(dst_dir); + + /* finally remove the old entry */ + hfs_cat_build_key(sb, src_fd.search_key, src_dir->i_ino, src_name); + err = hfs_brec_find(&src_fd); + if (err) + goto out; + err = hfs_brec_remove(&src_fd); + if (err) + goto out; + src_dir->i_size--; + inode_set_mtime_to_ts(src_dir, inode_set_ctime_current(src_dir)); + mark_inode_dirty(src_dir); + + type = entry.type; + if (type == HFS_CDR_FIL && !(entry.file.Flags & HFS_FIL_THD)) + goto out; + + /* remove old thread entry */ + hfs_cat_build_key(sb, src_fd.search_key, cnid, NULL); + err = hfs_brec_find(&src_fd); + if (err) + goto out; + err = hfs_brec_remove(&src_fd); + if (err) + goto out; + + /* create new thread entry */ + hfs_cat_build_key(sb, dst_fd.search_key, cnid, NULL); + entry_size = hfs_cat_build_thread(sb, &entry, type == HFS_CDR_FIL ? HFS_CDR_FTH : HFS_CDR_THD, + dst_dir->i_ino, dst_name); + err = hfs_brec_find(&dst_fd); + if (err != -ENOENT) { + if (!err) + err = -EEXIST; + goto out; + } + err = hfs_brec_insert(&dst_fd, &entry, entry_size); +out: + hfs_bnode_put(dst_fd.bnode); + hfs_find_exit(&src_fd); + return err; +} diff --git a/fs/hfs/dir.c b/fs/hfs/dir.c new file mode 100644 index 000000000..e1f1fb351 --- /dev/null +++ b/fs/hfs/dir.c @@ -0,0 +1,331 @@ +/* + * linux/fs/hfs/dir.c + * + * Copyright (C) 1995-1997 Paul H. Hargrove + * (C) 2003 Ardis Technologies + * This file may be distributed under the terms of the GNU General Public License. + * + * This file contains directory-related functions independent of which + * scheme is being used to represent forks. + * + * Based on the minix file system code, (C) 1991, 1992 by Linus Torvalds + */ + +#include "hfs_fs.h" +#include "btree.h" + +/* + * hfs_lookup() + */ +static struct dentry *hfs_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + hfs_cat_rec rec; + struct hfs_find_data fd; + struct inode *inode = NULL; + int res; + + res = hfs_find_init(HFS_SB(dir->i_sb)->cat_tree, &fd); + if (res) + return ERR_PTR(res); + hfs_cat_build_key(dir->i_sb, fd.search_key, dir->i_ino, &dentry->d_name); + res = hfs_brec_read(&fd, &rec, sizeof(rec)); + if (res) { + if (res != -ENOENT) + inode = ERR_PTR(res); + } else { + inode = hfs_iget(dir->i_sb, &fd.search_key->cat, &rec); + if (!inode) + inode = ERR_PTR(-EACCES); + } + hfs_find_exit(&fd); + return d_splice_alias(inode, dentry); +} + +/* + * hfs_readdir + */ +static int hfs_readdir(struct file *file, struct dir_context *ctx) +{ + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + int len, err; + char strbuf[HFS_MAX_NAMELEN]; + union hfs_cat_rec entry; + struct hfs_find_data fd; + struct hfs_readdir_data *rd; + u16 type; + + if (ctx->pos >= inode->i_size) + return 0; + + err = hfs_find_init(HFS_SB(sb)->cat_tree, &fd); + if (err) + return err; + hfs_cat_build_key(sb, fd.search_key, inode->i_ino, NULL); + err = hfs_brec_find(&fd); + if (err) + goto out; + + if (ctx->pos == 0) { + /* This is completely artificial... */ + if (!dir_emit_dot(file, ctx)) + goto out; + ctx->pos = 1; + } + if (ctx->pos == 1) { + if (fd.entrylength > sizeof(entry) || fd.entrylength < 0) { + err = -EIO; + goto out; + } + + hfs_bnode_read(fd.bnode, &entry, fd.entryoffset, fd.entrylength); + if (entry.type != HFS_CDR_THD) { + pr_err("bad catalog folder thread\n"); + err = -EIO; + goto out; + } + //if (fd.entrylength < HFS_MIN_THREAD_SZ) { + // pr_err("truncated catalog thread\n"); + // err = -EIO; + // goto out; + //} + if (!dir_emit(ctx, "..", 2, + be32_to_cpu(entry.thread.ParID), DT_DIR)) + goto out; + ctx->pos = 2; + } + if (ctx->pos >= inode->i_size) + goto out; + rd = file->private_data; + if (rd && rd->pos == ctx->pos) { + memcpy(fd.search_key, &rd->key, sizeof(struct hfs_cat_key)); + err = hfs_brec_find(&fd); + if (err == -ENOENT) + err = hfs_brec_goto(&fd, 1); + } else { + err = hfs_brec_goto(&fd, ctx->pos - 1); + } + if (err) + goto out; + + for (;;) { + if (be32_to_cpu(fd.key->cat.ParID) != inode->i_ino) { + pr_err("walked past end of dir\n"); + err = -EIO; + goto out; + } + + if (fd.entrylength > sizeof(entry) || fd.entrylength < 0) { + err = -EIO; + goto out; + } + + hfs_bnode_read(fd.bnode, &entry, fd.entryoffset, fd.entrylength); + type = entry.type; + len = hfs_mac2asc(sb, strbuf, &fd.key->cat.CName); + if (type == HFS_CDR_DIR) { + if (fd.entrylength < sizeof(struct hfs_cat_dir)) { + pr_err("small dir entry\n"); + err = -EIO; + goto out; + } + if (!dir_emit(ctx, strbuf, len, + be32_to_cpu(entry.dir.DirID), DT_DIR)) + break; + } else if (type == HFS_CDR_FIL) { + if (fd.entrylength < sizeof(struct hfs_cat_file)) { + pr_err("small file entry\n"); + err = -EIO; + goto out; + } + if (!dir_emit(ctx, strbuf, len, + be32_to_cpu(entry.file.FlNum), DT_REG)) + break; + } else { + pr_err("bad catalog entry type %d\n", type); + err = -EIO; + goto out; + } + ctx->pos++; + if (ctx->pos >= inode->i_size) + goto out; + err = hfs_brec_goto(&fd, 1); + if (err) + goto out; + } + if (!rd) { + rd = kmalloc_obj(struct hfs_readdir_data); + if (!rd) { + err = -ENOMEM; + goto out; + } + file->private_data = rd; + } + rd->pos = ctx->pos; + memcpy(&rd->key, &fd.key->cat, sizeof(struct hfs_cat_key)); +out: + hfs_find_exit(&fd); + return err; +} + +static int hfs_dir_release(struct inode *inode, struct file *file) +{ + kfree(file->private_data); + return 0; +} + +/* + * hfs_create() + * + * This is the create() entry in the inode_operations structure for + * regular HFS directories. The purpose is to create a new file in + * a directory and return a corresponding inode, given the inode for + * the directory and the name (and its length) of the new file. + */ +static int hfs_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct inode *inode; + int res; + + inode = hfs_new_inode(dir, &dentry->d_name, mode); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + res = hfs_cat_create(inode->i_ino, dir, &dentry->d_name, inode); + if (res) { + clear_nlink(inode); + hfs_delete_inode(inode); + iput(inode); + return res; + } + d_instantiate(dentry, inode); + mark_inode_dirty(inode); + return 0; +} + +/* + * hfs_mkdir() + * + * This is the mkdir() entry in the inode_operations structure for + * regular HFS directories. The purpose is to create a new directory + * in a directory, given the inode for the parent directory and the + * name (and its length) of the new directory. + */ +static struct dentry *hfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct inode *inode; + int res; + + inode = hfs_new_inode(dir, &dentry->d_name, mode); + if (IS_ERR(inode)) + return ERR_CAST(inode); + + res = hfs_cat_create(inode->i_ino, dir, &dentry->d_name, inode); + if (res) { + clear_nlink(inode); + hfs_delete_inode(inode); + iput(inode); + return ERR_PTR(res); + } + d_instantiate(dentry, inode); + mark_inode_dirty(inode); + return NULL; +} + +/* + * hfs_remove() + * + * This serves as both unlink() and rmdir() in the inode_operations + * structure for regular HFS directories. The purpose is to delete + * an existing child, given the inode for the parent directory and + * the name (and its length) of the existing directory. + * + * HFS does not have hardlinks, so both rmdir and unlink set the + * link count to 0. The only difference is the emptiness check. + */ +static int hfs_remove(struct inode *dir, struct dentry *dentry) +{ + struct super_block *sb = dir->i_sb; + struct inode *inode = d_inode(dentry); + int res; + + if (S_ISDIR(inode->i_mode) && inode->i_size != 2) + return -ENOTEMPTY; + + if (unlikely(!is_hfs_cnid_counts_valid(sb))) { + pr_err("cannot remove file/folder\n"); + return -ERANGE; + } + + res = hfs_cat_delete(inode->i_ino, dir, &dentry->d_name); + if (res) + return res; + clear_nlink(inode); + inode_set_ctime_current(inode); + hfs_delete_inode(inode); + mark_inode_dirty(inode); + return 0; +} + +/* + * hfs_rename() + * + * This is the rename() entry in the inode_operations structure for + * regular HFS directories. The purpose is to rename an existing + * file or directory, given the inode for the current directory and + * the name (and its length) of the existing file/directory and the + * inode for the new directory and the name (and its length) of the + * new file/directory. + * XXX: how do you handle must_be dir? + */ +static int hfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) +{ + int res; + + if (flags & ~RENAME_NOREPLACE) + return -EINVAL; + + /* Unlink destination if it already exists */ + if (d_really_is_positive(new_dentry)) { + res = hfs_remove(new_dir, new_dentry); + if (res) + return res; + } + + res = hfs_cat_move(d_inode(old_dentry)->i_ino, + old_dir, &old_dentry->d_name, + new_dir, &new_dentry->d_name); + if (!res) { + struct inode *inode = d_inode(old_dentry); + + hfs_cat_build_key(old_dir->i_sb, + (btree_key *)&HFS_I(inode)->cat_key, + new_dir->i_ino, &new_dentry->d_name); + inode_set_ctime_current(inode); + mark_inode_dirty(inode); + } + return res; +} + +const struct file_operations hfs_dir_operations = { + .read = generic_read_dir, + .iterate_shared = hfs_readdir, + .llseek = generic_file_llseek, + .release = hfs_dir_release, +}; + +const struct inode_operations hfs_dir_inode_operations = { + .create = hfs_create, + .lookup = hfs_lookup, + .unlink = hfs_remove, + .mkdir = hfs_mkdir, + .rmdir = hfs_remove, + .rename = hfs_rename, + .setattr = hfs_inode_setattr, + .fileattr_get = hfs_fileattr_get, +}; diff --git a/fs/hfs/extent.c b/fs/hfs/extent.c new file mode 100644 index 000000000..f066a99a8 --- /dev/null +++ b/fs/hfs/extent.c @@ -0,0 +1,552 @@ +/* + * linux/fs/hfs/extent.c + * + * Copyright (C) 1995-1997 Paul H. Hargrove + * (C) 2003 Ardis Technologies + * This file may be distributed under the terms of the GNU General Public License. + * + * This file contains the functions related to the extents B-tree. + */ + +#include + +#include "hfs_fs.h" +#include "btree.h" + +/*================ File-local functions ================*/ + +/* + * build_key + */ +static void hfs_ext_build_key(hfs_btree_key *key, u32 cnid, u16 block, u8 type) +{ + key->key_len = 7; + key->ext.FkType = type; + key->ext.FNum = cpu_to_be32(cnid); + key->ext.FABN = cpu_to_be16(block); +} + +/* + * hfs_ext_compare() + * + * Description: + * This is the comparison function used for the extents B-tree. In + * comparing extent B-tree entries, the file id is the most + * significant field (compared as unsigned ints); the fork type is + * the second most significant field (compared as unsigned chars); + * and the allocation block number field is the least significant + * (compared as unsigned ints). + * Input Variable(s): + * struct hfs_ext_key *key1: pointer to the first key to compare + * struct hfs_ext_key *key2: pointer to the second key to compare + * Output Variable(s): + * NONE + * Returns: + * int: negative if key1key2, and 0 if key1==key2 + * Preconditions: + * key1 and key2 point to "valid" (struct hfs_ext_key)s. + * Postconditions: + * This function has no side-effects */ +int hfs_ext_keycmp(const btree_key *key1, const btree_key *key2) +{ + __be32 fnum1, fnum2; + __be16 block1, block2; + + fnum1 = key1->ext.FNum; + fnum2 = key2->ext.FNum; + if (fnum1 != fnum2) + return be32_to_cpu(fnum1) < be32_to_cpu(fnum2) ? -1 : 1; + if (key1->ext.FkType != key2->ext.FkType) + return key1->ext.FkType < key2->ext.FkType ? -1 : 1; + + block1 = key1->ext.FABN; + block2 = key2->ext.FABN; + if (block1 == block2) + return 0; + return be16_to_cpu(block1) < be16_to_cpu(block2) ? -1 : 1; +} + +/* + * hfs_ext_find_block + * + * Find a block within an extent record + */ +u16 hfs_ext_find_block(struct hfs_extent *ext, u16 off) +{ + int i; + u16 count; + + for (i = 0; i < 3; ext++, i++) { + count = be16_to_cpu(ext->count); + if (off < count) + return be16_to_cpu(ext->block) + off; + off -= count; + } + /* panic? */ + return 0; +} + +static int hfs_ext_block_count(struct hfs_extent *ext) +{ + int i; + u16 count = 0; + + for (i = 0; i < 3; ext++, i++) + count += be16_to_cpu(ext->count); + return count; +} + +static u16 hfs_ext_lastblock(struct hfs_extent *ext) +{ + int i; + + ext += 2; + for (i = 0; i < 2; ext--, i++) + if (ext->count) + break; + return be16_to_cpu(ext->block) + be16_to_cpu(ext->count); +} + +static int __hfs_ext_write_extent(struct inode *inode, struct hfs_find_data *fd) +{ + int res; + + hfs_ext_build_key(fd->search_key, inode->i_ino, HFS_I(inode)->cached_start, + HFS_IS_RSRC(inode) ? HFS_FK_RSRC : HFS_FK_DATA); + res = hfs_brec_find(fd); + if (HFS_I(inode)->flags & HFS_FLG_EXT_NEW) { + if (res != -ENOENT) + return res; + /* Fail early and avoid ENOSPC during the btree operation */ + res = hfs_bmap_reserve(fd->tree, fd->tree->depth + 1); + if (res) + return res; + hfs_brec_insert(fd, HFS_I(inode)->cached_extents, sizeof(hfs_extent_rec)); + HFS_I(inode)->flags &= ~(HFS_FLG_EXT_DIRTY|HFS_FLG_EXT_NEW); + } else { + if (res) + return res; + hfs_bnode_write(fd->bnode, HFS_I(inode)->cached_extents, fd->entryoffset, fd->entrylength); + HFS_I(inode)->flags &= ~HFS_FLG_EXT_DIRTY; + } + return 0; +} + +int hfs_ext_write_extent(struct inode *inode) +{ + struct hfs_find_data fd; + int res = 0; + + if (HFS_I(inode)->flags & HFS_FLG_EXT_DIRTY) { + res = hfs_find_init(HFS_SB(inode->i_sb)->ext_tree, &fd); + if (res) + return res; + res = __hfs_ext_write_extent(inode, &fd); + hfs_find_exit(&fd); + } + return res; +} + +static inline int __hfs_ext_read_extent(struct hfs_find_data *fd, struct hfs_extent *extent, + u32 cnid, u32 block, u8 type) +{ + int res; + + hfs_ext_build_key(fd->search_key, cnid, block, type); + fd->key->ext.FNum = 0; + res = hfs_brec_find(fd); + if (res && res != -ENOENT) + return res; + if (fd->key->ext.FNum != fd->search_key->ext.FNum || + fd->key->ext.FkType != fd->search_key->ext.FkType) + return -ENOENT; + if (fd->entrylength != sizeof(hfs_extent_rec)) + return -EIO; + hfs_bnode_read(fd->bnode, extent, fd->entryoffset, sizeof(hfs_extent_rec)); + return 0; +} + +static inline int __hfs_ext_cache_extent(struct hfs_find_data *fd, struct inode *inode, u32 block) +{ + int res; + + if (HFS_I(inode)->flags & HFS_FLG_EXT_DIRTY) { + res = __hfs_ext_write_extent(inode, fd); + if (res) + return res; + } + + res = __hfs_ext_read_extent(fd, HFS_I(inode)->cached_extents, inode->i_ino, + block, HFS_IS_RSRC(inode) ? HFS_FK_RSRC : HFS_FK_DATA); + if (!res) { + HFS_I(inode)->cached_start = be16_to_cpu(fd->key->ext.FABN); + HFS_I(inode)->cached_blocks = hfs_ext_block_count(HFS_I(inode)->cached_extents); + } else { + HFS_I(inode)->cached_start = HFS_I(inode)->cached_blocks = 0; + HFS_I(inode)->flags &= ~(HFS_FLG_EXT_DIRTY|HFS_FLG_EXT_NEW); + } + return res; +} + +static int hfs_ext_read_extent(struct inode *inode, u16 block) +{ + struct hfs_find_data fd; + int res; + + if (block >= HFS_I(inode)->cached_start && + block < HFS_I(inode)->cached_start + HFS_I(inode)->cached_blocks) + return 0; + + res = hfs_find_init(HFS_SB(inode->i_sb)->ext_tree, &fd); + if (!res) { + res = __hfs_ext_cache_extent(&fd, inode, block); + hfs_find_exit(&fd); + } + return res; +} + +static void hfs_dump_extent(struct hfs_extent *extent) +{ + int i; + + hfs_dbg("extent: "); + for (i = 0; i < 3; i++) + hfs_dbg(" block %u, count %u", + be16_to_cpu(extent[i].block), + be16_to_cpu(extent[i].count)); + hfs_dbg("\n"); +} + +static int hfs_add_extent(struct hfs_extent *extent, u16 offset, + u16 alloc_block, u16 block_count) +{ + u16 count, start; + int i; + + hfs_dump_extent(extent); + for (i = 0; i < 3; extent++, i++) { + count = be16_to_cpu(extent->count); + if (offset == count) { + start = be16_to_cpu(extent->block); + if (alloc_block != start + count) { + if (++i >= 3) + return -ENOSPC; + extent++; + extent->block = cpu_to_be16(alloc_block); + } else + block_count += count; + extent->count = cpu_to_be16(block_count); + return 0; + } else if (offset < count) + break; + offset -= count; + } + /* panic? */ + return -EIO; +} + +static int hfs_free_extents(struct super_block *sb, struct hfs_extent *extent, + u16 offset, u16 block_nr) +{ + u16 count, start; + int i; + + hfs_dump_extent(extent); + for (i = 0; i < 3; extent++, i++) { + count = be16_to_cpu(extent->count); + if (offset == count) + goto found; + else if (offset < count) + break; + offset -= count; + } + /* panic? */ + return -EIO; +found: + for (;;) { + start = be16_to_cpu(extent->block); + if (count <= block_nr) { + hfs_clear_vbm_bits(sb, start, count); + extent->block = 0; + extent->count = 0; + block_nr -= count; + } else { + count -= block_nr; + hfs_clear_vbm_bits(sb, start + count, block_nr); + extent->count = cpu_to_be16(count); + block_nr = 0; + } + if (!block_nr || !i) + return 0; + i--; + extent--; + count = be16_to_cpu(extent->count); + } +} + +int hfs_free_fork(struct super_block *sb, struct hfs_cat_file *file, int type) +{ + struct hfs_find_data fd; + u32 total_blocks, blocks, start; + u32 cnid = be32_to_cpu(file->FlNum); + struct hfs_extent *extent; + int res, i; + + if (type == HFS_FK_DATA) { + total_blocks = be32_to_cpu(file->PyLen); + extent = file->ExtRec; + } else { + total_blocks = be32_to_cpu(file->RPyLen); + extent = file->RExtRec; + } + total_blocks /= HFS_SB(sb)->alloc_blksz; + if (!total_blocks) + return 0; + + blocks = 0; + for (i = 0; i < 3; i++) + blocks += be16_to_cpu(extent[i].count); + + res = hfs_free_extents(sb, extent, blocks, blocks); + if (res) + return res; + if (total_blocks == blocks) + return 0; + + res = hfs_find_init(HFS_SB(sb)->ext_tree, &fd); + if (res) + return res; + do { + res = __hfs_ext_read_extent(&fd, extent, cnid, total_blocks, type); + if (res) + break; + start = be16_to_cpu(fd.key->ext.FABN); + hfs_free_extents(sb, extent, total_blocks - start, total_blocks); + hfs_brec_remove(&fd); + total_blocks = start; + } while (total_blocks > blocks); + hfs_find_exit(&fd); + + return res; +} + +/* + * hfs_get_block + */ +int hfs_get_block(struct inode *inode, sector_t block, + struct buffer_head *bh_result, int create) +{ + struct super_block *sb; + u16 dblock, ablock; + int res; + + sb = inode->i_sb; + /* Convert inode block to disk allocation block */ + ablock = (u32)block / HFS_SB(sb)->fs_div; + + if (block >= HFS_I(inode)->fs_blocks) { + if (!create) + return 0; + if (block > HFS_I(inode)->fs_blocks) + return -EIO; + if (ablock >= HFS_I(inode)->alloc_blocks) { + res = hfs_extend_file(inode); + if (res) + return res; + } + } else + create = 0; + + if (ablock < HFS_I(inode)->first_blocks) { + dblock = hfs_ext_find_block(HFS_I(inode)->first_extents, ablock); + goto done; + } + + mutex_lock(&HFS_I(inode)->extents_lock); + res = hfs_ext_read_extent(inode, ablock); + if (!res) + dblock = hfs_ext_find_block(HFS_I(inode)->cached_extents, + ablock - HFS_I(inode)->cached_start); + else { + mutex_unlock(&HFS_I(inode)->extents_lock); + return -EIO; + } + mutex_unlock(&HFS_I(inode)->extents_lock); + +done: + map_bh(bh_result, sb, HFS_SB(sb)->fs_start + + dblock * HFS_SB(sb)->fs_div + + (u32)block % HFS_SB(sb)->fs_div); + + if (create) { + set_buffer_new(bh_result); + HFS_I(inode)->phys_size += sb->s_blocksize; + HFS_I(inode)->fs_blocks++; + inode_add_bytes(inode, sb->s_blocksize); + mark_inode_dirty(inode); + } + return 0; +} + +int hfs_extend_file(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + u32 start, len, goal; + int res; + + mutex_lock(&HFS_I(inode)->extents_lock); + if (HFS_I(inode)->alloc_blocks == HFS_I(inode)->first_blocks) + goal = hfs_ext_lastblock(HFS_I(inode)->first_extents); + else { + res = hfs_ext_read_extent(inode, HFS_I(inode)->alloc_blocks); + if (res) + goto out; + goal = hfs_ext_lastblock(HFS_I(inode)->cached_extents); + } + + len = HFS_I(inode)->clump_blocks; + start = hfs_vbm_search_free(sb, goal, &len); + if (!len) { + res = -ENOSPC; + goto out; + } + + hfs_dbg("ino %llu, start %u, len %u\n", inode->i_ino, start, len); + if (HFS_I(inode)->alloc_blocks == HFS_I(inode)->first_blocks) { + if (!HFS_I(inode)->first_blocks) { + hfs_dbg("first_extent: start %u, len %u\n", + start, len); + /* no extents yet */ + HFS_I(inode)->first_extents[0].block = cpu_to_be16(start); + HFS_I(inode)->first_extents[0].count = cpu_to_be16(len); + res = 0; + } else { + /* try to append to extents in inode */ + res = hfs_add_extent(HFS_I(inode)->first_extents, + HFS_I(inode)->alloc_blocks, + start, len); + if (res == -ENOSPC) + goto insert_extent; + } + if (!res) { + hfs_dump_extent(HFS_I(inode)->first_extents); + HFS_I(inode)->first_blocks += len; + } + } else { + res = hfs_add_extent(HFS_I(inode)->cached_extents, + HFS_I(inode)->alloc_blocks - + HFS_I(inode)->cached_start, + start, len); + if (!res) { + hfs_dump_extent(HFS_I(inode)->cached_extents); + HFS_I(inode)->flags |= HFS_FLG_EXT_DIRTY; + HFS_I(inode)->cached_blocks += len; + } else if (res == -ENOSPC) + goto insert_extent; + } +out: + mutex_unlock(&HFS_I(inode)->extents_lock); + if (!res) { + HFS_I(inode)->alloc_blocks += len; + mark_inode_dirty(inode); + if (inode->i_ino < HFS_FIRSTUSER_CNID) + set_bit(HFS_FLG_ALT_MDB_DIRTY, &HFS_SB(sb)->flags); + set_bit(HFS_FLG_MDB_DIRTY, &HFS_SB(sb)->flags); + hfs_mark_mdb_dirty(sb); + } + return res; + +insert_extent: + hfs_dbg("insert new extent\n"); + res = hfs_ext_write_extent(inode); + if (res) + goto out; + + memset(HFS_I(inode)->cached_extents, 0, sizeof(hfs_extent_rec)); + HFS_I(inode)->cached_extents[0].block = cpu_to_be16(start); + HFS_I(inode)->cached_extents[0].count = cpu_to_be16(len); + hfs_dump_extent(HFS_I(inode)->cached_extents); + HFS_I(inode)->flags |= HFS_FLG_EXT_DIRTY|HFS_FLG_EXT_NEW; + HFS_I(inode)->cached_start = HFS_I(inode)->alloc_blocks; + HFS_I(inode)->cached_blocks = len; + + res = 0; + goto out; +} + +void hfs_file_truncate(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + struct hfs_find_data fd; + u16 blk_cnt, alloc_cnt, start; + u32 size; + int res; + + hfs_dbg("ino %llu, phys_size %llu -> i_size %llu\n", + inode->i_ino, (long long)HFS_I(inode)->phys_size, + inode->i_size); + if (inode->i_size > HFS_I(inode)->phys_size) { + struct address_space *mapping = inode->i_mapping; + void *fsdata = NULL; + struct folio *folio; + + /* XXX: Can use generic_cont_expand? */ + size = inode->i_size - 1; + res = hfs_write_begin(NULL, mapping, size + 1, 0, &folio, + &fsdata); + if (!res) { + res = generic_write_end(NULL, mapping, size + 1, 0, 0, + folio, fsdata); + } + if (res) + inode->i_size = HFS_I(inode)->phys_size; + return; + } else if (inode->i_size == HFS_I(inode)->phys_size) + return; + size = inode->i_size + HFS_SB(sb)->alloc_blksz - 1; + blk_cnt = size / HFS_SB(sb)->alloc_blksz; + alloc_cnt = HFS_I(inode)->alloc_blocks; + if (blk_cnt == alloc_cnt) + goto out; + + mutex_lock(&HFS_I(inode)->extents_lock); + res = hfs_find_init(HFS_SB(sb)->ext_tree, &fd); + if (res) { + mutex_unlock(&HFS_I(inode)->extents_lock); + /* XXX: We lack error handling of hfs_file_truncate() */ + return; + } + while (1) { + if (alloc_cnt == HFS_I(inode)->first_blocks) { + hfs_free_extents(sb, HFS_I(inode)->first_extents, + alloc_cnt, alloc_cnt - blk_cnt); + hfs_dump_extent(HFS_I(inode)->first_extents); + HFS_I(inode)->first_blocks = blk_cnt; + break; + } + res = __hfs_ext_cache_extent(&fd, inode, alloc_cnt); + if (res) + break; + start = HFS_I(inode)->cached_start; + hfs_free_extents(sb, HFS_I(inode)->cached_extents, + alloc_cnt - start, alloc_cnt - blk_cnt); + hfs_dump_extent(HFS_I(inode)->cached_extents); + if (blk_cnt > start) { + HFS_I(inode)->flags |= HFS_FLG_EXT_DIRTY; + break; + } + alloc_cnt = start; + HFS_I(inode)->cached_start = HFS_I(inode)->cached_blocks = 0; + HFS_I(inode)->flags &= ~(HFS_FLG_EXT_DIRTY|HFS_FLG_EXT_NEW); + hfs_brec_remove(&fd); + } + hfs_find_exit(&fd); + mutex_unlock(&HFS_I(inode)->extents_lock); + + HFS_I(inode)->alloc_blocks = blk_cnt; +out: + HFS_I(inode)->phys_size = inode->i_size; + HFS_I(inode)->fs_blocks = (inode->i_size + sb->s_blocksize - 1) >> sb->s_blocksize_bits; + inode_set_bytes(inode, HFS_I(inode)->fs_blocks << sb->s_blocksize_bits); + mark_inode_dirty(inode); +} diff --git a/fs/hfs/hfs.h b/fs/hfs/hfs.h new file mode 100644 index 000000000..2e958c46b --- /dev/null +++ b/fs/hfs/hfs.h @@ -0,0 +1,21 @@ +/* + * linux/fs/hfs/hfs.h + * + * Copyright (C) 1995-1997 Paul H. Hargrove + * (C) 2003 Ardis Technologies + * This file may be distributed under the terms of the GNU General Public License. + */ + +#ifndef _HFS_H +#define _HFS_H + +#include + +/*======== Data structures kept in memory ========*/ + +struct hfs_readdir_data { + loff_t pos; + struct hfs_cat_key key; +}; + +#endif diff --git a/fs/hfs/hfs_fs.h b/fs/hfs/hfs_fs.h new file mode 100644 index 000000000..e250f87a5 --- /dev/null +++ b/fs/hfs/hfs_fs.h @@ -0,0 +1,319 @@ +/* + * linux/fs/hfs/hfs_fs.h + * + * Copyright (C) 1995-1997 Paul H. Hargrove + * (C) 2003 Ardis Technologies + * This file may be distributed under the terms of the GNU General Public License. + */ + +#ifndef _LINUX_HFS_FS_H +#define _LINUX_HFS_FS_H + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "hfs.h" + +/* + * struct hfs_inode_info + * + * The HFS-specific part of a Linux (struct inode) + */ +struct hfs_inode_info { + atomic_t opencnt; + + unsigned int flags; + + /* to deal with localtime ugliness */ + int tz_secondswest; + + struct hfs_cat_key cat_key; + + struct inode *rsrc_inode; + + struct mutex extents_lock; + + u16 alloc_blocks, clump_blocks; + sector_t fs_blocks; + /* Allocation extents from catlog record or volume header */ + hfs_extent_rec first_extents; + u16 first_blocks; + hfs_extent_rec cached_extents; + u16 cached_start, cached_blocks; + + loff_t phys_size; + struct inode vfs_inode; +}; + +#define HFS_FLG_RSRC 0x0001 +#define HFS_FLG_EXT_DIRTY 0x0002 +#define HFS_FLG_EXT_NEW 0x0004 + +#define HFS_IS_RSRC(inode) (HFS_I(inode)->flags & HFS_FLG_RSRC) + +/* + * struct hfs_sb_info + * + * The HFS-specific part of a Linux (struct super_block) + */ +struct hfs_sb_info { + struct mutex mdb_lock; /* MDB operations lock */ + struct buffer_head *mdb_bh; /* The hfs_buffer + holding the real + superblock (aka VIB + or MDB) */ + unsigned int mdb_offset; /* byte offset of the MDB + sector within mdb_bh's + data */ + struct hfs_mdb *mdb; /* in-memory copy of the MDB */ + struct buffer_head *alt_mdb_bh; /* The hfs_buffer holding + the alternate superblock */ + unsigned int alt_mdb_offset; /* byte offset of the alternate + MDB sector within + alt_mdb_bh's data */ + struct hfs_mdb *alt_mdb; /* in-memory copy of the + alternate MDB */ + __be32 *bitmap; /* The page holding the + allocation bitmap */ + struct hfs_btree *ext_tree; /* Information about + the extents b-tree */ + struct hfs_btree *cat_tree; /* Information about + the catalog b-tree */ + atomic64_t file_count; /* The number of + regular files in + the filesystem */ + atomic64_t folder_count; /* The number of + directories in the + filesystem */ + atomic64_t next_id; /* The next available + file id number */ + u32 clumpablks; /* The number of allocation + blocks to try to add when + extending a file */ + u32 fs_start; /* The first 512-byte + block represented + in the bitmap */ + u32 part_start; + u16 root_files; /* The number of + regular + (non-directory) + files in the root + directory */ + u16 root_dirs; /* The number of + directories in the + root directory */ + u16 fs_ablocks; /* The number of + allocation blocks + in the filesystem */ + u16 free_ablocks; /* the number of unused + allocation blocks + in the filesystem */ + u32 alloc_blksz; /* The size of an + "allocation block" */ + int s_quiet; /* Silent failure when + changing owner or mode? */ + __be32 s_type; /* Type for new files */ + __be32 s_creator; /* Creator for new files */ + umode_t s_file_umask; /* The umask applied to the + permissions on all files */ + umode_t s_dir_umask; /* The umask applied to the + permissions on all dirs */ + kuid_t s_uid; /* The uid of all files */ + kgid_t s_gid; /* The gid of all files */ + + int session, part; + struct nls_table *nls_io, *nls_disk; + struct mutex bitmap_lock; + unsigned long flags; + u16 blockoffset; + int fs_div; + struct super_block *sb; + int work_queued; /* non-zero delayed work is queued */ + struct delayed_work mdb_work; /* MDB flush delayed work */ + spinlock_t work_lock; /* protects mdb_work and work_queued */ +}; + +#define HFS_FLG_BITMAP_DIRTY 0 +#define HFS_FLG_MDB_DIRTY 1 +#define HFS_FLG_ALT_MDB_DIRTY 2 + +/* bitmap.c */ +extern u32 hfs_vbm_search_free(struct super_block *sb, u32 goal, u32 *num_bits); +extern int hfs_clear_vbm_bits(struct super_block *sb, u16 start, u16 count); + +/* catalog.c */ +extern int hfs_cat_keycmp(const btree_key *key1, const btree_key *key2); +struct hfs_find_data; +extern int hfs_cat_find_brec(struct super_block *sb, u32 cnid, + struct hfs_find_data *fd); +extern int hfs_cat_create(u32 cnid, struct inode *dir, + const struct qstr *str, struct inode *inode); +extern int hfs_cat_delete(u32 cnid, struct inode *dir, const struct qstr *str); +extern int hfs_cat_move(u32 cnid, struct inode *src_dir, + const struct qstr *src_name, + struct inode *dst_dir, + const struct qstr *dst_name); +extern void hfs_cat_build_key(struct super_block *sb, btree_key *key, + u32 parent, const struct qstr *name); + +/* + * Validate the CNID of a catalog record. + */ +static inline bool hfs_is_valid_cnid(u32 cnid, u8 type) +{ + if (likely(cnid >= HFS_FIRSTUSER_CNID)) + return true; + + switch (cnid) { + case HFS_ROOT_CNID: + return type == HFS_CDR_DIR; + case HFS_EXT_CNID: + case HFS_CAT_CNID: + return type == HFS_CDR_FIL; + default: + return false; + } +} + +/* dir.c */ +extern const struct file_operations hfs_dir_operations; +extern const struct inode_operations hfs_dir_inode_operations; + +/* extent.c */ +extern int hfs_ext_keycmp(const btree_key *key1, const btree_key *key2); +extern u16 hfs_ext_find_block(struct hfs_extent *ext, u16 off); +extern int hfs_free_fork(struct super_block *sb, + struct hfs_cat_file *file, int type); +extern int hfs_ext_write_extent(struct inode *inode); +extern int hfs_extend_file(struct inode *inode); +extern void hfs_file_truncate(struct inode *inode); + +extern int hfs_get_block(struct inode *inode, sector_t block, + struct buffer_head *bh_result, int create); + +/* inode.c */ +extern const struct address_space_operations hfs_aops; +extern const struct address_space_operations hfs_btree_aops; + +struct file_kattr; +int hfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa); +int hfs_write_begin(const struct kiocb *iocb, struct address_space *mapping, + loff_t pos, unsigned int len, struct folio **foliop, + void **fsdata); +extern struct inode *hfs_new_inode(struct inode *dir, const struct qstr *name, + umode_t mode); +extern void hfs_inode_write_fork(struct inode *inode, struct hfs_extent *ext, + __be32 *log_size, __be32 *phys_size); +extern int hfs_write_inode(struct inode *inode, struct writeback_control *wbc); +extern int hfs_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr); +extern void hfs_inode_read_fork(struct inode *inode, struct hfs_extent *ext, + __be32 __log_size, __be32 phys_size, + u32 clump_size); +extern struct inode *hfs_iget(struct super_block *sb, struct hfs_cat_key *key, + hfs_cat_rec *rec); +extern void hfs_evict_inode(struct inode *inode); +extern void hfs_delete_inode(struct inode *inode); + +/* attr.c */ +extern const struct xattr_handler * const hfs_xattr_handlers[]; + +/* mdb.c */ +extern bool is_hfs_cnid_counts_valid(struct super_block *sb); +extern int hfs_mdb_get(struct super_block *sb); +extern int hfs_mdb_commit(struct super_block *sb); +extern void hfs_mdb_close(struct super_block *sb); +extern void hfs_mdb_put(struct super_block *sb); + +/* part_tbl.c */ +extern int hfs_part_find(struct super_block *sb, + sector_t *part_start, sector_t *part_size); + +/* string.c */ +extern const struct dentry_operations hfs_dentry_operations; + +extern int hfs_hash_dentry(const struct dentry *dentry, struct qstr *this); +extern int hfs_strcmp(const unsigned char *s1, unsigned int len1, + const unsigned char *s2, unsigned int len2); +extern int hfs_compare_dentry(const struct dentry *dentry, + unsigned int len, const char *str, + const struct qstr *name); + +/* trans.c */ +extern void hfs_asc2mac(struct super_block *sb, + struct hfs_name *out, const struct qstr *in); +extern int hfs_mac2asc(struct super_block *sb, + char *out, const struct hfs_name *in); + +/* super.c */ +extern void hfs_mark_mdb_dirty(struct super_block *sb); + +/* + * There are two time systems. Both are based on seconds since + * a particular time/date. + * Unix: signed little-endian since 00:00 GMT, Jan. 1, 1970 + * mac: unsigned big-endian since 00:00 GMT, Jan. 1, 1904 + * + * HFS implementations are highly inconsistent, this one matches the + * traditional behavior of 64-bit Linux, giving the most useful + * time range between 1970 and 2106, by treating any on-disk timestamp + * under HFS_UTC_OFFSET (Jan 1 1970) as a time between 2040 and 2106. + */ +#define HFS_UTC_OFFSET 2082844800U + +static inline time64_t __hfs_m_to_utime(__be32 mt) +{ + time64_t ut = (u32)(be32_to_cpu(mt) - HFS_UTC_OFFSET); + + return ut + sys_tz.tz_minuteswest * 60; +} + +static inline __be32 __hfs_u_to_mtime(time64_t ut) +{ + ut -= sys_tz.tz_minuteswest * 60; + + return cpu_to_be32(lower_32_bits(ut) + HFS_UTC_OFFSET); +} +#define HFS_I(inode) (container_of(inode, struct hfs_inode_info, vfs_inode)) +#define HFS_SB(sb) ((struct hfs_sb_info *)(sb)->s_fs_info) + +#define hfs_m_to_utime(time) (struct timespec64){ .tv_sec = __hfs_m_to_utime(time) } +#define hfs_u_to_mtime(time) __hfs_u_to_mtime((time).tv_sec) +#define hfs_mtime() __hfs_u_to_mtime(ktime_get_real_seconds()) + +static inline const char *hfs_mdb_name(struct super_block *sb) +{ + return sb->s_id; +} + +static inline void hfs_bitmap_dirty(struct super_block *sb) +{ + set_bit(HFS_FLG_BITMAP_DIRTY, &HFS_SB(sb)->flags); + hfs_mark_mdb_dirty(sb); +} + +#define sb_bread512(sb, sec, data) ({ \ + struct buffer_head *__bh; \ + sector_t __block; \ + loff_t __start; \ + int __offset; \ + \ + __start = (loff_t)(sec) << HFS_SECTOR_SIZE_BITS;\ + __block = __start >> (sb)->s_blocksize_bits; \ + __offset = __start & ((sb)->s_blocksize - 1); \ + __bh = sb_bread((sb), __block); \ + if (likely(__bh != NULL)) \ + data = (void *)(__bh->b_data + __offset);\ + else \ + data = NULL; \ + __bh; \ +}) + +#endif diff --git a/fs/hfs/inode.c b/fs/hfs/inode.c new file mode 100644 index 000000000..2aef3c36a --- /dev/null +++ b/fs/hfs/inode.c @@ -0,0 +1,752 @@ +/* + * linux/fs/hfs/inode.c + * + * Copyright (C) 1995-1997 Paul H. Hargrove + * (C) 2003 Ardis Technologies + * This file may be distributed under the terms of the GNU General Public License. + * + * This file contains inode-related functions which do not depend on + * which scheme is being used to represent forks. + * + * Based on the minix file system code, (C) 1991, 1992 by Linus Torvalds + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hfs_fs.h" +#include "btree.h" + +static const struct file_operations hfs_file_operations; +static const struct inode_operations hfs_file_inode_operations; + +/*================ Variable-like macros ================*/ + +#define HFS_VALID_MODE_BITS (S_IFREG | S_IFDIR | S_IRWXUGO) + +static int hfs_read_folio(struct file *file, struct folio *folio) +{ + return block_read_full_folio(folio, hfs_get_block); +} + +static void hfs_write_failed(struct address_space *mapping, loff_t to) +{ + struct inode *inode = mapping->host; + + if (to > inode->i_size) { + truncate_pagecache(inode, inode->i_size); + hfs_file_truncate(inode); + } +} + +int hfs_write_begin(const struct kiocb *iocb, struct address_space *mapping, + loff_t pos, unsigned int len, struct folio **foliop, + void **fsdata) +{ + struct inode *inode = mapping->host; + struct hfs_sb_info *sbi = HFS_SB(inode->i_sb); + loff_t total_capacity; + int ret; + + total_capacity = (loff_t)sbi->fs_ablocks * sbi->alloc_blksz; + + if (pos >= total_capacity) + return -EFBIG; + + ret = cont_write_begin(iocb, mapping, pos, len, foliop, fsdata, + hfs_get_block, + &HFS_I(inode)->phys_size); + if (unlikely(ret)) + hfs_write_failed(mapping, pos + len); + + return ret; +} + +static sector_t hfs_bmap(struct address_space *mapping, sector_t block) +{ + return generic_block_bmap(mapping, block, hfs_get_block); +} + +static bool hfs_release_folio(struct folio *folio, gfp_t mask) +{ + struct inode *inode = folio->mapping->host; + struct super_block *sb = inode->i_sb; + struct hfs_btree *tree; + struct hfs_bnode *node; + u32 nidx; + int i; + bool res = true; + + switch (inode->i_ino) { + case HFS_EXT_CNID: + tree = HFS_SB(sb)->ext_tree; + break; + case HFS_CAT_CNID: + tree = HFS_SB(sb)->cat_tree; + break; + default: + BUG(); + return false; + } + + if (!tree) + return false; + + if (tree->node_size >= PAGE_SIZE) { + nidx = folio->index >> (tree->node_size_shift - PAGE_SHIFT); + spin_lock(&tree->hash_lock); + node = hfs_bnode_findhash(tree, nidx); + if (!node) + ; + else if (atomic_read(&node->refcnt)) + res = false; + if (res && node) { + hfs_bnode_unhash(node); + hfs_bnode_free(node); + } + spin_unlock(&tree->hash_lock); + } else { + nidx = folio->index << (PAGE_SHIFT - tree->node_size_shift); + i = 1 << (PAGE_SHIFT - tree->node_size_shift); + spin_lock(&tree->hash_lock); + do { + node = hfs_bnode_findhash(tree, nidx++); + if (!node) + continue; + if (atomic_read(&node->refcnt)) { + res = false; + break; + } + hfs_bnode_unhash(node); + hfs_bnode_free(node); + } while (--i && nidx < tree->node_count); + spin_unlock(&tree->hash_lock); + } + return res ? try_to_free_buffers(folio) : false; +} + +static ssize_t hfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) +{ + struct file *file = iocb->ki_filp; + struct address_space *mapping = file->f_mapping; + struct inode *inode = mapping->host; + size_t count = iov_iter_count(iter); + ssize_t ret; + + ret = blockdev_direct_IO(iocb, inode, iter, hfs_get_block); + + /* + * In case of error extending write may have instantiated a few + * blocks outside i_size. Trim these off again. + */ + if (unlikely(iov_iter_rw(iter) == WRITE && ret < 0)) { + loff_t isize = i_size_read(inode); + loff_t end = iocb->ki_pos + count; + + if (end > isize) + hfs_write_failed(mapping, end); + } + + return ret; +} + +static int hfs_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + return mpage_writepages(mapping, wbc, hfs_get_block); +} + +const struct address_space_operations hfs_btree_aops = { + .dirty_folio = block_dirty_folio, + .invalidate_folio = block_invalidate_folio, + .read_folio = hfs_read_folio, + .writepages = hfs_writepages, + .write_begin = hfs_write_begin, + .write_end = generic_write_end, + .migrate_folio = buffer_migrate_folio, + .bmap = hfs_bmap, + .release_folio = hfs_release_folio, +}; + +const struct address_space_operations hfs_aops = { + .dirty_folio = block_dirty_folio, + .invalidate_folio = block_invalidate_folio, + .read_folio = hfs_read_folio, + .write_begin = hfs_write_begin, + .write_end = generic_write_end, + .bmap = hfs_bmap, + .direct_IO = hfs_direct_IO, + .writepages = hfs_writepages, + .migrate_folio = buffer_migrate_folio, +}; + +/* + * hfs_new_inode + */ +struct inode *hfs_new_inode(struct inode *dir, const struct qstr *name, umode_t mode) +{ + struct super_block *sb = dir->i_sb; + struct inode *inode = new_inode(sb); + s64 next_id; + s64 file_count; + s64 folder_count; + int err = -ENOMEM; + + if (!inode) + goto out_err; + + err = -ERANGE; + + mutex_init(&HFS_I(inode)->extents_lock); + hfs_cat_build_key(sb, (btree_key *)&HFS_I(inode)->cat_key, dir->i_ino, name); + next_id = atomic64_inc_return(&HFS_SB(sb)->next_id); + if (next_id > U32_MAX) { + atomic64_dec(&HFS_SB(sb)->next_id); + pr_err("cannot create new inode: next CNID exceeds limit\n"); + goto out_discard; + } + inode->i_ino = (u32)next_id - 1; + inode->i_mode = mode; + inode->i_uid = current_fsuid(); + inode->i_gid = current_fsgid(); + set_nlink(inode, 1); + simple_inode_init_ts(inode); + HFS_I(inode)->flags = 0; + HFS_I(inode)->rsrc_inode = NULL; + HFS_I(inode)->fs_blocks = 0; + HFS_I(inode)->tz_secondswest = sys_tz.tz_minuteswest * 60; + if (S_ISDIR(mode)) { + inode->i_size = 2; + folder_count = atomic64_inc_return(&HFS_SB(sb)->folder_count); + if (folder_count> U32_MAX) { + atomic64_dec(&HFS_SB(sb)->folder_count); + pr_err("cannot create new inode: folder count exceeds limit\n"); + goto out_discard; + } + if (dir->i_ino == HFS_ROOT_CNID) + HFS_SB(sb)->root_dirs++; + inode->i_op = &hfs_dir_inode_operations; + inode->i_fop = &hfs_dir_operations; + inode->i_mode |= S_IRWXUGO; + inode->i_mode &= ~HFS_SB(inode->i_sb)->s_dir_umask; + } else if (S_ISREG(mode)) { + HFS_I(inode)->clump_blocks = HFS_SB(sb)->clumpablks; + file_count = atomic64_inc_return(&HFS_SB(sb)->file_count); + if (file_count > U32_MAX) { + atomic64_dec(&HFS_SB(sb)->file_count); + pr_err("cannot create new inode: file count exceeds limit\n"); + goto out_discard; + } + if (dir->i_ino == HFS_ROOT_CNID) + HFS_SB(sb)->root_files++; + inode->i_op = &hfs_file_inode_operations; + inode->i_fop = &hfs_file_operations; + inode->i_mapping->a_ops = &hfs_aops; + inode->i_mode |= S_IRUGO|S_IXUGO; + if (mode & S_IWUSR) + inode->i_mode |= S_IWUGO; + inode->i_mode &= ~HFS_SB(inode->i_sb)->s_file_umask; + HFS_I(inode)->phys_size = 0; + HFS_I(inode)->alloc_blocks = 0; + HFS_I(inode)->first_blocks = 0; + HFS_I(inode)->cached_start = 0; + HFS_I(inode)->cached_blocks = 0; + memset(HFS_I(inode)->first_extents, 0, sizeof(hfs_extent_rec)); + memset(HFS_I(inode)->cached_extents, 0, sizeof(hfs_extent_rec)); + } + insert_inode_hash(inode); + mark_inode_dirty(inode); + set_bit(HFS_FLG_MDB_DIRTY, &HFS_SB(sb)->flags); + hfs_mark_mdb_dirty(sb); + + return inode; + + out_discard: + iput(inode); + out_err: + return ERR_PTR(err); +} + +void hfs_delete_inode(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + + hfs_dbg("ino %llu\n", inode->i_ino); + if (S_ISDIR(inode->i_mode)) { + atomic64_dec(&HFS_SB(sb)->folder_count); + if (HFS_I(inode)->cat_key.ParID == cpu_to_be32(HFS_ROOT_CNID)) + HFS_SB(sb)->root_dirs--; + set_bit(HFS_FLG_MDB_DIRTY, &HFS_SB(sb)->flags); + hfs_mark_mdb_dirty(sb); + return; + } + + atomic64_dec(&HFS_SB(sb)->file_count); + if (HFS_I(inode)->cat_key.ParID == cpu_to_be32(HFS_ROOT_CNID)) + HFS_SB(sb)->root_files--; + if (S_ISREG(inode->i_mode)) { + if (!inode->i_nlink) { + inode->i_size = 0; + hfs_file_truncate(inode); + } + } + set_bit(HFS_FLG_MDB_DIRTY, &HFS_SB(sb)->flags); + hfs_mark_mdb_dirty(sb); +} + +void hfs_inode_read_fork(struct inode *inode, struct hfs_extent *ext, + __be32 __log_size, __be32 phys_size, u32 clump_size) +{ + struct super_block *sb = inode->i_sb; + u32 log_size = be32_to_cpu(__log_size); + u16 count; + int i; + + memcpy(HFS_I(inode)->first_extents, ext, sizeof(hfs_extent_rec)); + for (count = 0, i = 0; i < 3; i++) + count += be16_to_cpu(ext[i].count); + HFS_I(inode)->first_blocks = count; + HFS_I(inode)->cached_start = 0; + HFS_I(inode)->cached_blocks = 0; + + inode->i_size = HFS_I(inode)->phys_size = log_size; + HFS_I(inode)->fs_blocks = (log_size + sb->s_blocksize - 1) >> sb->s_blocksize_bits; + inode_set_bytes(inode, HFS_I(inode)->fs_blocks << sb->s_blocksize_bits); + HFS_I(inode)->alloc_blocks = be32_to_cpu(phys_size) / + HFS_SB(sb)->alloc_blksz; + HFS_I(inode)->clump_blocks = clump_size / HFS_SB(sb)->alloc_blksz; + if (!HFS_I(inode)->clump_blocks) + HFS_I(inode)->clump_blocks = HFS_SB(sb)->clumpablks; +} + +struct hfs_iget_data { + struct hfs_cat_key *key; + hfs_cat_rec *rec; +}; + +static int hfs_test_inode(struct inode *inode, void *data) +{ + struct hfs_iget_data *idata = data; + hfs_cat_rec *rec; + + rec = idata->rec; + switch (rec->type) { + case HFS_CDR_DIR: + return inode->i_ino == be32_to_cpu(rec->dir.DirID); + case HFS_CDR_FIL: + return inode->i_ino == be32_to_cpu(rec->file.FlNum); + default: + BUG(); + return 1; + } +} + +/* + * hfs_read_inode + */ +static int hfs_read_inode(struct inode *inode, void *data) +{ + struct hfs_iget_data *idata = data; + struct hfs_sb_info *hsb = HFS_SB(inode->i_sb); + hfs_cat_rec *rec; + struct timespec64 mtime; + + HFS_I(inode)->flags = 0; + HFS_I(inode)->rsrc_inode = NULL; + mutex_init(&HFS_I(inode)->extents_lock); + + /* Initialize the inode */ + inode->i_uid = hsb->s_uid; + inode->i_gid = hsb->s_gid; + set_nlink(inode, 1); + + if (idata->key) + HFS_I(inode)->cat_key = *idata->key; + else + HFS_I(inode)->flags |= HFS_FLG_RSRC; + HFS_I(inode)->tz_secondswest = sys_tz.tz_minuteswest * 60; + + rec = idata->rec; + switch (rec->type) { + case HFS_CDR_FIL: + if (!hfs_is_valid_cnid(be32_to_cpu(rec->file.FlNum), rec->type)) + return -EIO; + + if (!HFS_IS_RSRC(inode)) { + hfs_inode_read_fork(inode, rec->file.ExtRec, rec->file.LgLen, + rec->file.PyLen, be16_to_cpu(rec->file.ClpSize)); + } else { + hfs_inode_read_fork(inode, rec->file.RExtRec, rec->file.RLgLen, + rec->file.RPyLen, be16_to_cpu(rec->file.ClpSize)); + } + + inode->i_ino = be32_to_cpu(rec->file.FlNum); + inode->i_mode = S_IRUGO | S_IXUGO; + if (!(rec->file.Flags & HFS_FIL_LOCK)) + inode->i_mode |= S_IWUGO; + inode->i_mode &= ~hsb->s_file_umask; + inode->i_mode |= S_IFREG; + mtime = hfs_m_to_utime(rec->file.MdDat); + inode_set_ctime_to_ts(inode, mtime); + inode_set_atime_to_ts(inode, mtime); + inode_set_mtime_to_ts(inode, mtime); + inode->i_op = &hfs_file_inode_operations; + inode->i_fop = &hfs_file_operations; + inode->i_mapping->a_ops = &hfs_aops; + break; + case HFS_CDR_DIR: + if (!hfs_is_valid_cnid(be32_to_cpu(rec->dir.DirID), rec->type)) + return -EIO; + + inode->i_ino = be32_to_cpu(rec->dir.DirID); + inode->i_size = be16_to_cpu(rec->dir.Val) + 2; + HFS_I(inode)->fs_blocks = 0; + inode->i_mode = S_IFDIR | (S_IRWXUGO & ~hsb->s_dir_umask); + mtime = hfs_m_to_utime(rec->dir.MdDat); + inode_set_ctime_to_ts(inode, mtime); + inode_set_atime_to_ts(inode, mtime); + inode_set_mtime_to_ts(inode, mtime); + inode->i_op = &hfs_dir_inode_operations; + inode->i_fop = &hfs_dir_operations; + break; + default: + make_bad_inode(inode); + } + return 0; +} + +/* + * __hfs_iget() + * + * Given the MDB for a HFS filesystem, a 'key' and an 'entry' in + * the catalog B-tree and the 'type' of the desired file return the + * inode for that file/directory or NULL. Note that 'type' indicates + * whether we want the actual file or directory, or the corresponding + * metadata (AppleDouble header file or CAP metadata file). + */ +struct inode *hfs_iget(struct super_block *sb, struct hfs_cat_key *key, hfs_cat_rec *rec) +{ + struct hfs_iget_data data = { key, rec }; + struct inode *inode; + u32 cnid; + + switch (rec->type) { + case HFS_CDR_DIR: + cnid = be32_to_cpu(rec->dir.DirID); + break; + case HFS_CDR_FIL: + cnid = be32_to_cpu(rec->file.FlNum); + break; + default: + return NULL; + } + inode = iget5_locked(sb, cnid, hfs_test_inode, hfs_read_inode, &data); + if (inode && (inode_state_read_once(inode) & I_NEW)) + unlock_new_inode(inode); + return inode; +} + +void hfs_inode_write_fork(struct inode *inode, struct hfs_extent *ext, + __be32 *log_size, __be32 *phys_size) +{ + memcpy(ext, HFS_I(inode)->first_extents, sizeof(hfs_extent_rec)); + + if (log_size) + *log_size = cpu_to_be32(inode->i_size); + if (phys_size) + *phys_size = cpu_to_be32(HFS_I(inode)->alloc_blocks * + HFS_SB(inode->i_sb)->alloc_blksz); +} + +int hfs_write_inode(struct inode *inode, struct writeback_control *wbc) +{ + struct inode *main_inode = inode; + struct hfs_find_data fd; + hfs_cat_rec rec; + int res; + + hfs_dbg("ino %llu\n", inode->i_ino); + res = hfs_ext_write_extent(inode); + if (res) + return res; + + if (inode->i_ino < HFS_FIRSTUSER_CNID) { + switch (inode->i_ino) { + case HFS_ROOT_CNID: + break; + case HFS_EXT_CNID: + hfs_btree_write(HFS_SB(inode->i_sb)->ext_tree); + return 0; + case HFS_CAT_CNID: + hfs_btree_write(HFS_SB(inode->i_sb)->cat_tree); + return 0; + default: + BUG(); + return -EIO; + } + } + + if (HFS_IS_RSRC(inode)) + main_inode = HFS_I(inode)->rsrc_inode; + + if (!main_inode->i_nlink) + return 0; + + if (hfs_find_init(HFS_SB(main_inode->i_sb)->cat_tree, &fd)) + /* panic? */ + return -EIO; + + res = -EIO; + if (HFS_I(main_inode)->cat_key.CName.len > HFS_NAMELEN) + goto out; + fd.search_key->cat = HFS_I(main_inode)->cat_key; + if (hfs_brec_find(&fd)) + goto out; + + if (S_ISDIR(main_inode->i_mode)) { + if (fd.entrylength < sizeof(struct hfs_cat_dir)) + goto out; + hfs_bnode_read(fd.bnode, &rec, fd.entryoffset, + sizeof(struct hfs_cat_dir)); + if (rec.type != HFS_CDR_DIR || + be32_to_cpu(rec.dir.DirID) != inode->i_ino) { + } + + rec.dir.MdDat = hfs_u_to_mtime(inode_get_mtime(inode)); + rec.dir.Val = cpu_to_be16(inode->i_size - 2); + + hfs_bnode_write(fd.bnode, &rec, fd.entryoffset, + sizeof(struct hfs_cat_dir)); + } else if (HFS_IS_RSRC(inode)) { + if (fd.entrylength < sizeof(struct hfs_cat_file)) + goto out; + hfs_bnode_read(fd.bnode, &rec, fd.entryoffset, + sizeof(struct hfs_cat_file)); + hfs_inode_write_fork(inode, rec.file.RExtRec, + &rec.file.RLgLen, &rec.file.RPyLen); + hfs_bnode_write(fd.bnode, &rec, fd.entryoffset, + sizeof(struct hfs_cat_file)); + } else { + if (fd.entrylength < sizeof(struct hfs_cat_file)) + goto out; + hfs_bnode_read(fd.bnode, &rec, fd.entryoffset, + sizeof(struct hfs_cat_file)); + if (rec.type != HFS_CDR_FIL || + be32_to_cpu(rec.file.FlNum) != inode->i_ino) { + } + + if (inode->i_mode & S_IWUSR) + rec.file.Flags &= ~HFS_FIL_LOCK; + else + rec.file.Flags |= HFS_FIL_LOCK; + hfs_inode_write_fork(inode, rec.file.ExtRec, &rec.file.LgLen, &rec.file.PyLen); + rec.file.MdDat = hfs_u_to_mtime(inode_get_mtime(inode)); + + hfs_bnode_write(fd.bnode, &rec, fd.entryoffset, + sizeof(struct hfs_cat_file)); + } + res = 0; +out: + hfs_find_exit(&fd); + return res; +} + +static struct dentry *hfs_file_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + struct inode *inode = NULL; + hfs_cat_rec rec; + struct hfs_find_data fd; + int res; + + if (HFS_IS_RSRC(dir) || strcmp(dentry->d_name.name, "rsrc")) + goto out; + + inode = HFS_I(dir)->rsrc_inode; + if (inode) + goto out; + + inode = new_inode(dir->i_sb); + if (!inode) + return ERR_PTR(-ENOMEM); + + res = hfs_find_init(HFS_SB(dir->i_sb)->cat_tree, &fd); + if (res) { + iput(inode); + return ERR_PTR(res); + } + fd.search_key->cat = HFS_I(dir)->cat_key; + res = hfs_brec_read(&fd, &rec, sizeof(rec)); + if (!res) { + struct hfs_iget_data idata = { NULL, &rec }; + res = hfs_read_inode(inode, &idata); + } + hfs_find_exit(&fd); + if (res) { + iput(inode); + return ERR_PTR(res); + } + + if (is_bad_inode(inode)) { + iput(inode); + return ERR_PTR(-EIO); + } + HFS_I(inode)->rsrc_inode = dir; + HFS_I(dir)->rsrc_inode = inode; + igrab(dir); + inode_fake_hash(inode); + mark_inode_dirty(inode); + dont_mount(dentry); +out: + return d_splice_alias(inode, dentry); +} + +void hfs_evict_inode(struct inode *inode) +{ + truncate_inode_pages_final(&inode->i_data); + clear_inode(inode); + if (HFS_IS_RSRC(inode) && HFS_I(inode)->rsrc_inode) { + HFS_I(HFS_I(inode)->rsrc_inode)->rsrc_inode = NULL; + iput(HFS_I(inode)->rsrc_inode); + } +} + +static int hfs_file_open(struct inode *inode, struct file *file) +{ + if (HFS_IS_RSRC(inode)) + inode = HFS_I(inode)->rsrc_inode; + atomic_inc(&HFS_I(inode)->opencnt); + return 0; +} + +static int hfs_file_release(struct inode *inode, struct file *file) +{ + //struct super_block *sb = inode->i_sb; + + if (HFS_IS_RSRC(inode)) + inode = HFS_I(inode)->rsrc_inode; + if (atomic_dec_and_test(&HFS_I(inode)->opencnt)) { + inode_lock(inode); + hfs_file_truncate(inode); + //if (inode->i_flags & S_DEAD) { + // hfs_delete_cat(inode->i_ino, HFSPLUS_SB(sb).hidden_dir, NULL); + // hfs_delete_inode(inode); + //} + inode_unlock(inode); + } + return 0; +} + +int hfs_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +{ + struct inode *inode = d_inode(dentry); + struct hfs_sb_info *hsb = HFS_SB(inode->i_sb); + int error; + + error = setattr_prepare(&nop_mnt_idmap, dentry, attr); + if (error) + return error; + + /* no uig/gid changes and limit which mode bits can be set */ + if (((attr->ia_valid & ATTR_UID) && + (!uid_eq(attr->ia_uid, hsb->s_uid))) || + ((attr->ia_valid & ATTR_GID) && + (!gid_eq(attr->ia_gid, hsb->s_gid))) || + ((attr->ia_valid & ATTR_MODE) && + ((S_ISDIR(inode->i_mode) && + (attr->ia_mode != inode->i_mode)) || + (attr->ia_mode & ~HFS_VALID_MODE_BITS)))) { + return hsb->s_quiet ? 0 : error; + } + + /* map file permissions to the closest allowable permissions in HFS */ + if (attr->ia_valid & ATTR_MODE) { + /* Only the 'w' bits can ever change and only all together. */ + if (attr->ia_mode & S_IWUSR) + attr->ia_mode = inode->i_mode | S_IWUGO; + else + attr->ia_mode = inode->i_mode & ~S_IWUGO; + attr->ia_mode &= S_ISDIR(inode->i_mode) ? ~hsb->s_dir_umask: ~hsb->s_file_umask; + } + + if ((attr->ia_valid & ATTR_SIZE) && + attr->ia_size != i_size_read(inode)) { + inode_dio_wait(inode); + + error = inode_newsize_ok(inode, attr->ia_size); + if (error) + return error; + + truncate_setsize(inode, attr->ia_size); + hfs_file_truncate(inode); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + } + + setattr_copy(&nop_mnt_idmap, inode, attr); + mark_inode_dirty(inode); + return 0; +} + +static int hfs_file_fsync(struct file *filp, loff_t start, loff_t end, + int datasync) +{ + struct inode *inode = filp->f_mapping->host; + struct super_block * sb; + int ret, err; + + ret = file_write_and_wait_range(filp, start, end); + if (ret) + return ret; + inode_lock(inode); + + /* sync the inode to buffers */ + ret = write_inode_now(inode, 0); + + /* sync the superblock to buffers */ + sb = inode->i_sb; + flush_delayed_work(&HFS_SB(sb)->mdb_work); + /* .. finally sync the buffers to disk */ + err = sync_blockdev(sb->s_bdev); + if (!ret) + ret = err; + inode_unlock(inode); + return ret; +} + +int hfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + /* + * HFS compares filenames using Mac OS Roman case folding, so + * lookup is always case-insensitive. Names are stored on disk + * with case intact; CASENONPRESERVING stays clear. + */ + fa->fsx_xflags |= FS_XFLAG_CASEFOLD; + fa->flags |= FS_CASEFOLD_FL; + return 0; +} + +static const struct file_operations hfs_file_operations = { + .llseek = generic_file_llseek, + .read_iter = generic_file_read_iter, + .write_iter = generic_file_write_iter, + .mmap_prepare = generic_file_mmap_prepare, + .splice_read = filemap_splice_read, + .splice_write = iter_file_splice_write, + .fsync = hfs_file_fsync, + .open = hfs_file_open, + .release = hfs_file_release, +}; + +static const struct inode_operations hfs_file_inode_operations = { + .lookup = hfs_file_lookup, + .setattr = hfs_inode_setattr, + .listxattr = generic_listxattr, + .fileattr_get = hfs_fileattr_get, +}; diff --git a/fs/hfs/mdb.c b/fs/hfs/mdb.c new file mode 100644 index 000000000..277de712f --- /dev/null +++ b/fs/hfs/mdb.c @@ -0,0 +1,462 @@ +/* + * linux/fs/hfs/mdb.c + * + * Copyright (C) 1995-1997 Paul H. Hargrove + * (C) 2003 Ardis Technologies + * This file may be distributed under the terms of the GNU General Public License. + * + * This file contains functions for reading/writing the MDB. + */ + +#include +#include +#include +#include + +#include "hfs_fs.h" +#include "btree.h" + +/*================ File-local data types ================*/ + +/* + * The HFS Master Directory Block (MDB). + * + * Also known as the Volume Information Block (VIB), this structure is + * the HFS equivalent of a superblock. + * + * Reference: _Inside Macintosh: Files_ pages 2-59 through 2-62 + * + * modified for HFS Extended + */ + +static int hfs_get_last_session(struct super_block *sb, + sector_t *start, sector_t *size) +{ + struct cdrom_device_info *cdi = disk_to_cdi(sb->s_bdev->bd_disk); + + /* default values */ + *start = 0; + *size = bdev_nr_sectors(sb->s_bdev); + + if (HFS_SB(sb)->session >= 0) { + struct cdrom_tocentry te; + + if (!cdi) + return -EINVAL; + + te.cdte_track = HFS_SB(sb)->session; + te.cdte_format = CDROM_LBA; + if (cdrom_read_tocentry(cdi, &te) || + (te.cdte_ctrl & CDROM_DATA_TRACK) != 4) { + pr_err("invalid session number or type of track\n"); + return -EINVAL; + } + + *start = (sector_t)te.cdte_addr.lba << 2; + } else if (cdi) { + struct cdrom_multisession ms_info; + + ms_info.addr_format = CDROM_LBA; + if (cdrom_multisession(cdi, &ms_info) == 0 && ms_info.xa_flag) + *start = (sector_t)ms_info.addr.lba << 2; + } + + return 0; +} + +bool is_hfs_cnid_counts_valid(struct super_block *sb) +{ + struct hfs_sb_info *sbi = HFS_SB(sb); + bool corrupted = false; + + if (unlikely(atomic64_read(&sbi->next_id) > U32_MAX)) { + pr_warn("next CNID exceeds limit\n"); + corrupted = true; + } + if (unlikely(atomic64_read(&sbi->file_count) > U32_MAX)) { + pr_warn("file count exceeds limit\n"); + corrupted = true; + } + if (unlikely(atomic64_read(&sbi->folder_count) > U32_MAX)) { + pr_warn("folder count exceeds limit\n"); + corrupted = true; + } + + return !corrupted; +} + +/* + * hfs_sect_offset() - get byte offset within the buffer_head. + */ +static unsigned int hfs_sect_offset(struct super_block *sb, sector_t sec) +{ + loff_t start = (loff_t)sec << HFS_SECTOR_SIZE_BITS; + + return start & (sb->s_blocksize - 1); +} + +/* + * hfs_mdb_publish() - copy the in-memory primary MDB to the on-disk buffer. + */ +static void hfs_mdb_publish(struct hfs_sb_info *sbi) +{ + lock_buffer(sbi->mdb_bh); + memcpy(sbi->mdb_bh->b_data + sbi->mdb_offset, sbi->mdb, HFS_SECTOR_SIZE); + mark_buffer_dirty(sbi->mdb_bh); + unlock_buffer(sbi->mdb_bh); +} + +/* + * hfs_alt_mdb_publish() - copy the in-memory alternate MDB to its buffer. + */ +static void hfs_alt_mdb_publish(struct hfs_sb_info *sbi) +{ + lock_buffer(sbi->alt_mdb_bh); + memcpy(sbi->alt_mdb_bh->b_data + sbi->alt_mdb_offset, sbi->alt_mdb, + HFS_SECTOR_SIZE); + mark_buffer_dirty(sbi->alt_mdb_bh); + unlock_buffer(sbi->alt_mdb_bh); +} + +/* + * hfs_mdb_get() + * + * Build the in-core MDB for a filesystem, including + * the B-trees and the volume bitmap. + */ +int hfs_mdb_get(struct super_block *sb) +{ + struct buffer_head *bh; + struct hfs_mdb *mdb, *alt_mdb; + unsigned int block; + char *ptr; + int off2, len, size, sect; + sector_t part_start, part_size; + loff_t off; + __be16 attrib; + + /* set the device driver to 512-byte blocks */ + size = sb_min_blocksize(sb, HFS_SECTOR_SIZE); + if (!size) + return -EINVAL; + + if (hfs_get_last_session(sb, &part_start, &part_size)) + return -EINVAL; + while (1) { + /* See if this is an HFS filesystem */ + bh = sb_bread512(sb, part_start + HFS_MDB_BLK, mdb); + if (!bh) + return -EIO; + + if (mdb->drSigWord == cpu_to_be16(HFS_SUPER_MAGIC)) + break; + brelse(bh); + + /* check for a partition block + * (should do this only for cdrom/loop though) + */ + if (hfs_part_find(sb, &part_start, &part_size)) + return -EIO; + } + + HFS_SB(sb)->alloc_blksz = size = be32_to_cpu(mdb->drAlBlkSiz); + if (!size || (size & (HFS_SECTOR_SIZE - 1))) { + pr_err("bad allocation block size %d\n", size); + brelse(bh); + return -EIO; + } + + size = min(HFS_SB(sb)->alloc_blksz, (u32)PAGE_SIZE); + /* size must be a multiple of 512 */ + while (size & (size - 1)) + size -= HFS_SECTOR_SIZE; + sect = be16_to_cpu(mdb->drAlBlSt) + part_start; + /* align block size to first sector */ + while (sect & ((size - 1) >> HFS_SECTOR_SIZE_BITS)) + size >>= 1; + /* align block size to weird alloc size */ + while (HFS_SB(sb)->alloc_blksz & (size - 1)) + size >>= 1; + brelse(bh); + if (!sb_set_blocksize(sb, size)) { + pr_err("unable to set blocksize to %u\n", size); + return -EIO; + } + + bh = sb_bread512(sb, part_start + HFS_MDB_BLK, mdb); + if (!bh) + return -EIO; + if (mdb->drSigWord != cpu_to_be16(HFS_SUPER_MAGIC)) { + brelse(bh); + return -EIO; + } + + mdb = kmemdup(mdb, HFS_SECTOR_SIZE, GFP_KERNEL); + if (!mdb) { + brelse(bh); + return -ENOMEM; + } + + HFS_SB(sb)->mdb_bh = bh; + HFS_SB(sb)->mdb_offset = hfs_sect_offset(sb, part_start + HFS_MDB_BLK); + HFS_SB(sb)->mdb = mdb; + + /* These parameters are read from the MDB, and never written */ + HFS_SB(sb)->part_start = part_start; + HFS_SB(sb)->fs_ablocks = be16_to_cpu(mdb->drNmAlBlks); + HFS_SB(sb)->fs_div = HFS_SB(sb)->alloc_blksz >> sb->s_blocksize_bits; + HFS_SB(sb)->clumpablks = be32_to_cpu(mdb->drClpSiz) / + HFS_SB(sb)->alloc_blksz; + if (!HFS_SB(sb)->clumpablks) + HFS_SB(sb)->clumpablks = 1; + HFS_SB(sb)->fs_start = (be16_to_cpu(mdb->drAlBlSt) + part_start) >> + (sb->s_blocksize_bits - HFS_SECTOR_SIZE_BITS); + + /* These parameters are read from and written to the MDB */ + HFS_SB(sb)->free_ablocks = be16_to_cpu(mdb->drFreeBks); + atomic64_set(&HFS_SB(sb)->next_id, be32_to_cpu(mdb->drNxtCNID)); + HFS_SB(sb)->root_files = be16_to_cpu(mdb->drNmFls); + HFS_SB(sb)->root_dirs = be16_to_cpu(mdb->drNmRtDirs); + atomic64_set(&HFS_SB(sb)->file_count, be32_to_cpu(mdb->drFilCnt)); + atomic64_set(&HFS_SB(sb)->folder_count, be32_to_cpu(mdb->drDirCnt)); + + if (!is_hfs_cnid_counts_valid(sb)) { + pr_warn("filesystem possibly corrupted, running fsck.hfs is recommended. Mounting read-only.\n"); + sb->s_flags |= SB_RDONLY; + } + + /* TRY to get the alternate (backup) MDB. */ + sect = part_start + part_size - 2; + bh = sb_bread512(sb, sect, alt_mdb); + if (bh) { + if (alt_mdb->drSigWord == cpu_to_be16(HFS_SUPER_MAGIC)) { + alt_mdb = kmemdup(alt_mdb, HFS_SECTOR_SIZE, GFP_KERNEL); + if (alt_mdb) { + HFS_SB(sb)->alt_mdb_bh = bh; + HFS_SB(sb)->alt_mdb_offset = + hfs_sect_offset(sb, sect); + HFS_SB(sb)->alt_mdb = alt_mdb; + } else { + brelse(bh); + } + } else + brelse(bh); + } + + if (!HFS_SB(sb)->alt_mdb) { + pr_warn("unable to locate alternate MDB\n"); + pr_warn("continuing without an alternate MDB\n"); + } + + HFS_SB(sb)->bitmap = kzalloc(8192, GFP_KERNEL); + if (!HFS_SB(sb)->bitmap) + return -EIO; + + /* read in the bitmap */ + block = be16_to_cpu(mdb->drVBMSt) + part_start; + off = (loff_t)block << HFS_SECTOR_SIZE_BITS; + size = (HFS_SB(sb)->fs_ablocks + 8) / 8; + ptr = (u8 *)HFS_SB(sb)->bitmap; + while (size) { + bh = sb_bread(sb, off >> sb->s_blocksize_bits); + if (!bh) { + pr_err("unable to read volume bitmap\n"); + return -EIO; + } + off2 = off & (sb->s_blocksize - 1); + len = min((int)sb->s_blocksize - off2, size); + memcpy(ptr, bh->b_data + off2, len); + brelse(bh); + ptr += len; + off += len; + size -= len; + } + + HFS_SB(sb)->ext_tree = hfs_btree_open(sb, HFS_EXT_CNID, hfs_ext_keycmp); + if (!HFS_SB(sb)->ext_tree) { + pr_err("unable to open extent tree\n"); + return -EIO; + } + HFS_SB(sb)->cat_tree = hfs_btree_open(sb, HFS_CAT_CNID, hfs_cat_keycmp); + if (!HFS_SB(sb)->cat_tree) { + pr_err("unable to open catalog tree\n"); + return -EIO; + } + + attrib = mdb->drAtrb; + if (!(attrib & cpu_to_be16(HFS_SB_ATTRIB_UNMNT))) { + pr_warn("filesystem was not cleanly unmounted, running fsck.hfs is recommended. Mounting read-only.\n"); + sb->s_flags |= SB_RDONLY; + } + if ((attrib & cpu_to_be16(HFS_SB_ATTRIB_SLOCK))) { + pr_warn("filesystem is marked locked, mounting read-only.\n"); + sb->s_flags |= SB_RDONLY; + } + if (!sb_rdonly(sb)) { + /* Mark the volume uncleanly unmounted in case we crash */ + attrib &= cpu_to_be16(~HFS_SB_ATTRIB_UNMNT); + attrib |= cpu_to_be16(HFS_SB_ATTRIB_INCNSTNT); + mdb->drAtrb = attrib; + be32_add_cpu(&mdb->drWrCnt, 1); + mdb->drLsMod = hfs_mtime(); + + hfs_mdb_publish(HFS_SB(sb)); + sync_dirty_buffer(HFS_SB(sb)->mdb_bh); + } + + return 0; +} + +/* + * hfs_mdb_commit() + * + * Description: + * This updates the MDB on disk. + * It does not check, if the superblock has been modified, or + * if the filesystem has been mounted read-only. It is mainly + * called by hfs_sync_fs() and flush_mdb(). + * Input Variable(s): + * struct hfs_mdb *mdb: Pointer to the hfs MDB + * int backup; + * Output Variable(s): + * NONE + * Returns: + * 0 on success, -EIO if the MDB or alternate MDB buffer is no longer + * valid (e.g. after a prior write error), in which case the volume is + * remounted read-only. + * Preconditions: + * 'mdb' points to a "valid" (struct hfs_mdb). + * Postconditions: + * The HFS MDB and on disk will be updated, by copying the possibly + * modified fields from the in memory MDB (in native byte order) to + * the disk block buffer. + * If 'backup' is non-zero then the alternate MDB is also written + * and the function doesn't return until it is actually on disk. + */ +int hfs_mdb_commit(struct super_block *sb) +{ + struct hfs_mdb *mdb = HFS_SB(sb)->mdb; + int ret = 0; + + if (sb_rdonly(sb)) + return 0; + + if (!buffer_uptodate(HFS_SB(sb)->mdb_bh)) { + pr_err("primary MDB is corrupt, mounting read-only\n"); + sb->s_flags |= SB_RDONLY; + return -EIO; + } + + if (test_and_clear_bit(HFS_FLG_MDB_DIRTY, &HFS_SB(sb)->flags)) { + /* These parameters may have been modified, so write them back */ + mdb->drLsMod = hfs_mtime(); + mdb->drFreeBks = cpu_to_be16(HFS_SB(sb)->free_ablocks); + mdb->drNxtCNID = + cpu_to_be32((u32)atomic64_read(&HFS_SB(sb)->next_id)); + mdb->drNmFls = cpu_to_be16(HFS_SB(sb)->root_files); + mdb->drNmRtDirs = cpu_to_be16(HFS_SB(sb)->root_dirs); + mdb->drFilCnt = + cpu_to_be32((u32)atomic64_read(&HFS_SB(sb)->file_count)); + mdb->drDirCnt = + cpu_to_be32((u32)atomic64_read(&HFS_SB(sb)->folder_count)); + + hfs_inode_write_fork(HFS_SB(sb)->ext_tree->inode, mdb->drXTExtRec, + &mdb->drXTFlSize, NULL); + hfs_inode_write_fork(HFS_SB(sb)->cat_tree->inode, mdb->drCTExtRec, + &mdb->drCTFlSize, NULL); + + /* write MDB to disk */ + hfs_mdb_publish(HFS_SB(sb)); + sync_dirty_buffer(HFS_SB(sb)->mdb_bh); + } + + /* write the backup MDB, not returning until it is written. + * we only do this when either the catalog or extents overflow + * files grow. */ + if (test_and_clear_bit(HFS_FLG_ALT_MDB_DIRTY, &HFS_SB(sb)->flags) && + HFS_SB(sb)->alt_mdb) { + if (!buffer_uptodate(HFS_SB(sb)->alt_mdb_bh)) { + pr_err("alternate MDB is corrupt, mounting read-only\n"); + sb->s_flags |= SB_RDONLY; + ret = -EIO; + goto out; + } + + memcpy(HFS_SB(sb)->alt_mdb, mdb, HFS_SECTOR_SIZE); + HFS_SB(sb)->alt_mdb->drAtrb |= cpu_to_be16(HFS_SB_ATTRIB_UNMNT); + HFS_SB(sb)->alt_mdb->drAtrb &= cpu_to_be16(~HFS_SB_ATTRIB_INCNSTNT); + + hfs_alt_mdb_publish(HFS_SB(sb)); + sync_dirty_buffer(HFS_SB(sb)->alt_mdb_bh); + } + + if (test_and_clear_bit(HFS_FLG_BITMAP_DIRTY, &HFS_SB(sb)->flags)) { + struct buffer_head *bh; + sector_t block; + char *ptr; + int off, size, len; + + block = be16_to_cpu(HFS_SB(sb)->mdb->drVBMSt) + HFS_SB(sb)->part_start; + off = (block << HFS_SECTOR_SIZE_BITS) & (sb->s_blocksize - 1); + block >>= sb->s_blocksize_bits - HFS_SECTOR_SIZE_BITS; + size = (HFS_SB(sb)->fs_ablocks + 7) / 8; + ptr = (u8 *)HFS_SB(sb)->bitmap; + while (size) { + bh = sb_bread(sb, block); + if (!bh) { + pr_err("unable to read volume bitmap\n"); + break; + } + len = min((int)sb->s_blocksize - off, size); + + lock_buffer(bh); + memcpy(bh->b_data + off, ptr, len); + unlock_buffer(bh); + + mark_buffer_dirty(bh); + brelse(bh); + block++; + off = 0; + ptr += len; + size -= len; + } + } +out: + return ret; +} + +void hfs_mdb_close(struct super_block *sb) +{ + /* update volume attributes */ + if (sb_rdonly(sb)) + return; + + if (!buffer_uptodate(HFS_SB(sb)->mdb_bh)) + return; + + HFS_SB(sb)->mdb->drAtrb |= cpu_to_be16(HFS_SB_ATTRIB_UNMNT); + HFS_SB(sb)->mdb->drAtrb &= cpu_to_be16(~HFS_SB_ATTRIB_INCNSTNT); + hfs_mdb_publish(HFS_SB(sb)); +} + +/* + * hfs_mdb_put() + * + * Release the resources associated with the in-core MDB. */ +void hfs_mdb_put(struct super_block *sb) +{ + /* free the B-trees */ + hfs_btree_close(HFS_SB(sb)->ext_tree); + hfs_btree_close(HFS_SB(sb)->cat_tree); + + /* free the buffers holding the primary and alternate MDBs */ + brelse(HFS_SB(sb)->mdb_bh); + brelse(HFS_SB(sb)->alt_mdb_bh); + kfree(HFS_SB(sb)->mdb); + kfree(HFS_SB(sb)->alt_mdb); + + unload_nls(HFS_SB(sb)->nls_io); + unload_nls(HFS_SB(sb)->nls_disk); + + kfree(HFS_SB(sb)->bitmap); +} diff --git a/fs/hfs/part_tbl.c b/fs/hfs/part_tbl.c new file mode 100644 index 000000000..36add537d --- /dev/null +++ b/fs/hfs/part_tbl.c @@ -0,0 +1,117 @@ +/* + * linux/fs/hfs/part_tbl.c + * + * Copyright (C) 1996-1997 Paul H. Hargrove + * (C) 2003 Ardis Technologies + * This file may be distributed under the terms of the GNU General Public License. + * + * Original code to handle the new style Mac partition table based on + * a patch contributed by Holger Schemel (aeglos@valinor.owl.de). + */ + +#include "hfs_fs.h" + +/* + * The new style Mac partition map + * + * For each partition on the media there is a physical block (512-byte + * block) containing one of these structures. These blocks are + * contiguous starting at block 1. + */ +struct new_pmap { + __be16 pmSig; /* signature */ + __be16 reSigPad; /* padding */ + __be32 pmMapBlkCnt; /* partition blocks count */ + __be32 pmPyPartStart; /* physical block start of partition */ + __be32 pmPartBlkCnt; /* physical block count of partition */ + u8 pmPartName[32]; /* (null terminated?) string + giving the name of this + partition */ + u8 pmPartType[32]; /* (null terminated?) string + giving the type of this + partition */ + /* a bunch more stuff we don't need */ +} __packed; + +/* + * The old style Mac partition map + * + * The partition map consists for a 2-byte signature followed by an + * array of these structures. The map is terminated with an all-zero + * one of these. + */ +struct old_pmap { + __be16 pdSig; /* Signature bytes */ + struct old_pmap_entry { + __be32 pdStart; + __be32 pdSize; + __be32 pdFSID; + } pdEntry[42]; +} __packed; + +/* + * hfs_part_find() + * + * Parse the partition map looking for the + * start and length of the 'part'th HFS partition. + */ +int hfs_part_find(struct super_block *sb, + sector_t *part_start, sector_t *part_size) +{ + struct buffer_head *bh; + __be16 *data; + int i, size, res; + + res = -ENOENT; + bh = sb_bread512(sb, *part_start + HFS_PMAP_BLK, data); + if (!bh) + return -EIO; + + switch (be16_to_cpu(*data)) { + case HFS_OLD_PMAP_MAGIC: + { + struct old_pmap *pm; + struct old_pmap_entry *p; + + pm = (struct old_pmap *)bh->b_data; + p = pm->pdEntry; + size = 42; + for (i = 0; i < size; p++, i++) { + if (p->pdStart && p->pdSize && + p->pdFSID == cpu_to_be32(0x54465331)/*"TFS1"*/ && + (HFS_SB(sb)->part < 0 || HFS_SB(sb)->part == i)) { + *part_start += be32_to_cpu(p->pdStart); + *part_size = be32_to_cpu(p->pdSize); + res = 0; + } + } + break; + } + case HFS_NEW_PMAP_MAGIC: + { + struct new_pmap *pm; + + pm = (struct new_pmap *)bh->b_data; + size = be32_to_cpu(pm->pmMapBlkCnt); + for (i = 0; i < size;) { + if (!memcmp(pm->pmPartType,"Apple_HFS", 9) && + (HFS_SB(sb)->part < 0 || HFS_SB(sb)->part == i)) { + *part_start += be32_to_cpu(pm->pmPyPartStart); + *part_size = be32_to_cpu(pm->pmPartBlkCnt); + res = 0; + break; + } + brelse(bh); + bh = sb_bread512(sb, *part_start + HFS_PMAP_BLK + ++i, pm); + if (!bh) + return -EIO; + if (pm->pmSig != cpu_to_be16(HFS_NEW_PMAP_MAGIC)) + break; + } + break; + } + } + brelse(bh); + + return res; +} diff --git a/fs/hfs/string.c b/fs/hfs/string.c new file mode 100644 index 000000000..0cfa35e82 --- /dev/null +++ b/fs/hfs/string.c @@ -0,0 +1,119 @@ +/* + * linux/fs/hfs/string.c + * + * Copyright (C) 1995-1997 Paul H. Hargrove + * (C) 2003 Ardis Technologies + * This file may be distributed under the terms of the GNU General Public License. + * + * This file contains the string comparison function for the + * Macintosh character set. + * + * The code in this file is derived from code which is copyright + * 1986, 1989, 1990 by Abacus Research and Development, Inc. (ARDI) + * It is used here by the permission of ARDI's president Cliff Matthews. + */ + +#include "hfs_fs.h" +#include + +#include + +/*================ File-local variables ================*/ + +/* + * unsigned char caseorder[] + * + * Defines the lexical ordering of characters on the Macintosh + * + * Composition of the 'casefold' and 'order' tables from ARDI's code + * with the entry for 0x20 changed to match that for 0xCA to remove + * special case for those two characters. + */ +static unsigned char caseorder[256] = { + 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0A,0x0B,0x0C,0x0D,0x0E,0x0F, + 0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,0x19,0x1A,0x1B,0x1C,0x1D,0x1E,0x1F, + 0x20,0x22,0x23,0x28,0x29,0x2A,0x2B,0x2C,0x2F,0x30,0x31,0x32,0x33,0x34,0x35,0x36, + 0x37,0x38,0x39,0x3A,0x3B,0x3C,0x3D,0x3E,0x3F,0x40,0x41,0x42,0x43,0x44,0x45,0x46, + 0x47,0x48,0x57,0x59,0x5D,0x5F,0x66,0x68,0x6A,0x6C,0x72,0x74,0x76,0x78,0x7A,0x7E, + 0x8C,0x8E,0x90,0x92,0x95,0x97,0x9E,0xA0,0xA2,0xA4,0xA7,0xA9,0xAA,0xAB,0xAC,0xAD, + 0x4E,0x48,0x57,0x59,0x5D,0x5F,0x66,0x68,0x6A,0x6C,0x72,0x74,0x76,0x78,0x7A,0x7E, + 0x8C,0x8E,0x90,0x92,0x95,0x97,0x9E,0xA0,0xA2,0xA4,0xA7,0xAF,0xB0,0xB1,0xB2,0xB3, + 0x4A,0x4C,0x5A,0x60,0x7B,0x7F,0x98,0x4F,0x49,0x51,0x4A,0x4B,0x4C,0x5A,0x60,0x63, + 0x64,0x65,0x6E,0x6F,0x70,0x71,0x7B,0x84,0x85,0x86,0x7F,0x80,0x9A,0x9B,0x9C,0x98, + 0xB4,0xB5,0xB6,0xB7,0xB8,0xB9,0xBA,0x94,0xBB,0xBC,0xBD,0xBE,0xBF,0xC0,0x4D,0x81, + 0xC1,0xC2,0xC3,0xC4,0xC5,0xC6,0xC7,0xC8,0xC9,0xCA,0xCB,0x55,0x8A,0xCC,0x4D,0x81, + 0xCD,0xCE,0xCF,0xD0,0xD1,0xD2,0xD3,0x26,0x27,0xD4,0x20,0x49,0x4B,0x80,0x82,0x82, + 0xD5,0xD6,0x24,0x25,0x2D,0x2E,0xD7,0xD8,0xA6,0xD9,0xDA,0xDB,0xDC,0xDD,0xDE,0xDF, + 0xE0,0xE1,0xE2,0xE3,0xE4,0xE5,0xE6,0xE7,0xE8,0xE9,0xEA,0xEB,0xEC,0xED,0xEE,0xEF, + 0xF0,0xF1,0xF2,0xF3,0xF4,0xF5,0xF6,0xF7,0xF8,0xF9,0xFA,0xFB,0xFC,0xFD,0xFE,0xFF +}; + +/*================ Global functions ================*/ + +/* + * Hash a string to an integer in a case-independent way + */ +int hfs_hash_dentry(const struct dentry *dentry, struct qstr *this) +{ + const unsigned char *name = this->name; + unsigned int hash, len = this->len; + + if (len > HFS_NAMELEN) + len = HFS_NAMELEN; + + hash = init_name_hash(dentry); + for (; len; len--) + hash = partial_name_hash(caseorder[*name++], hash); + this->hash = end_name_hash(hash); + return 0; +} +EXPORT_SYMBOL_IF_KUNIT(hfs_hash_dentry); + +/* + * Compare two strings in the HFS filename character ordering + * Returns positive, negative, or zero, not just 0 or (+/-)1 + * + * Equivalent to ARDI's call: + * ROMlib_RelString(s1+1, s2+1, true, false, (s1[0]<<16) | s2[0]) + */ +int hfs_strcmp(const unsigned char *s1, unsigned int len1, + const unsigned char *s2, unsigned int len2) +{ + int len, tmp; + + len = (len1 > len2) ? len2 : len1; + + while (len--) { + tmp = (int)caseorder[*(s1++)] - (int)caseorder[*(s2++)]; + if (tmp) + return tmp; + } + return len1 - len2; +} +EXPORT_SYMBOL_IF_KUNIT(hfs_strcmp); + +/* + * Test for equality of two strings in the HFS filename character ordering. + * return 1 on failure and 0 on success + */ +int hfs_compare_dentry(const struct dentry *dentry, + unsigned int len, const char *str, const struct qstr *name) +{ + const unsigned char *n1, *n2; + + if (len >= HFS_NAMELEN) { + if (name->len < HFS_NAMELEN) + return 1; + len = HFS_NAMELEN; + } else if (len != name->len) + return 1; + + n1 = str; + n2 = name->name; + while (len--) { + if (caseorder[*n1++] != caseorder[*n2++]) + return 1; + } + return 0; +} +EXPORT_SYMBOL_IF_KUNIT(hfs_compare_dentry); diff --git a/fs/hfs/string_test.c b/fs/hfs/string_test.c new file mode 100644 index 000000000..e1bf6f954 --- /dev/null +++ b/fs/hfs/string_test.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit tests for HFS string operations + * + * Copyright (C) 2025 Viacheslav Dubeyko + */ + +#include +#include +#include "hfs_fs.h" + +/* Test hfs_strcmp function */ +static void hfs_strcmp_test(struct kunit *test) +{ + /* Test equal strings */ + KUNIT_EXPECT_EQ(test, 0, hfs_strcmp("hello", 5, "hello", 5)); + KUNIT_EXPECT_EQ(test, 0, hfs_strcmp("test", 4, "test", 4)); + KUNIT_EXPECT_EQ(test, 0, hfs_strcmp("", 0, "", 0)); + + /* Test unequal strings */ + KUNIT_EXPECT_NE(test, 0, hfs_strcmp("hello", 5, "world", 5)); + KUNIT_EXPECT_NE(test, 0, hfs_strcmp("test", 4, "testing", 7)); + + /* Test different lengths */ + KUNIT_EXPECT_LT(test, hfs_strcmp("test", 4, "testing", 7), 0); + KUNIT_EXPECT_GT(test, hfs_strcmp("testing", 7, "test", 4), 0); + + /* Test case insensitive comparison (HFS should handle case) */ + KUNIT_EXPECT_EQ(test, 0, hfs_strcmp("Test", 4, "TEST", 4)); + KUNIT_EXPECT_EQ(test, 0, hfs_strcmp("hello", 5, "HELLO", 5)); + + /* Test with special characters */ + KUNIT_EXPECT_EQ(test, 0, hfs_strcmp("file.txt", 8, "file.txt", 8)); + KUNIT_EXPECT_NE(test, 0, hfs_strcmp("file.txt", 8, "file.dat", 8)); + + /* Test boundary cases */ + KUNIT_EXPECT_EQ(test, 0, hfs_strcmp("a", 1, "a", 1)); + KUNIT_EXPECT_NE(test, 0, hfs_strcmp("a", 1, "b", 1)); +} + +/* Test hfs_hash_dentry function */ +static void hfs_hash_dentry_test(struct kunit *test) +{ + struct qstr test_name1, test_name2, test_name3; + struct dentry dentry = {}; + char name1[] = "testfile"; + char name2[] = "TestFile"; + char name3[] = "different"; + + /* Initialize test strings */ + test_name1.name = name1; + test_name1.len = strlen(name1); + test_name1.hash = 0; + + test_name2.name = name2; + test_name2.len = strlen(name2); + test_name2.hash = 0; + + test_name3.name = name3; + test_name3.len = strlen(name3); + test_name3.hash = 0; + + /* Test hashing */ + KUNIT_EXPECT_EQ(test, 0, hfs_hash_dentry(&dentry, &test_name1)); + KUNIT_EXPECT_EQ(test, 0, hfs_hash_dentry(&dentry, &test_name2)); + KUNIT_EXPECT_EQ(test, 0, hfs_hash_dentry(&dentry, &test_name3)); + + /* Case insensitive names should hash the same */ + KUNIT_EXPECT_EQ(test, test_name1.hash, test_name2.hash); + + /* Different names should have different hashes */ + KUNIT_EXPECT_NE(test, test_name1.hash, test_name3.hash); +} + +/* Test hfs_compare_dentry function */ +static void hfs_compare_dentry_test(struct kunit *test) +{ + struct qstr test_name; + struct dentry dentry = {}; + char name[] = "TestFile"; + + test_name.name = name; + test_name.len = strlen(name); + + /* Test exact match */ + KUNIT_EXPECT_EQ(test, 0, hfs_compare_dentry(&dentry, 8, + "TestFile", &test_name)); + + /* Test case insensitive match */ + KUNIT_EXPECT_EQ(test, 0, hfs_compare_dentry(&dentry, 8, + "testfile", &test_name)); + KUNIT_EXPECT_EQ(test, 0, hfs_compare_dentry(&dentry, 8, + "TESTFILE", &test_name)); + + /* Test different names */ + KUNIT_EXPECT_EQ(test, 1, hfs_compare_dentry(&dentry, 8, + "DiffFile", &test_name)); + + /* Test different lengths */ + KUNIT_EXPECT_EQ(test, 1, hfs_compare_dentry(&dentry, 7, + "TestFil", &test_name)); + KUNIT_EXPECT_EQ(test, 1, hfs_compare_dentry(&dentry, 9, + "TestFiles", &test_name)); + + /* Test empty string */ + test_name.name = ""; + test_name.len = 0; + KUNIT_EXPECT_EQ(test, 0, hfs_compare_dentry(&dentry, 0, "", &test_name)); + + /* Test HFS_NAMELEN boundary */ + test_name.name = "This_is_a_very_long_filename_that_exceeds_normal_limits"; + test_name.len = strlen(test_name.name); + KUNIT_EXPECT_EQ(test, 0, hfs_compare_dentry(&dentry, HFS_NAMELEN, + "This_is_a_very_long_filename_th", &test_name)); +} + +static struct kunit_case hfs_string_test_cases[] = { + KUNIT_CASE(hfs_strcmp_test), + KUNIT_CASE(hfs_hash_dentry_test), + KUNIT_CASE(hfs_compare_dentry_test), + {} +}; + +static struct kunit_suite hfs_string_test_suite = { + .name = "hfs_string", + .test_cases = hfs_string_test_cases, +}; + +kunit_test_suite(hfs_string_test_suite); + +MODULE_DESCRIPTION("KUnit tests for HFS string operations"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/fs/hfs/super.c b/fs/hfs/super.c new file mode 100644 index 000000000..5e6d66ff4 --- /dev/null +++ b/fs/hfs/super.c @@ -0,0 +1,503 @@ +/* + * linux/fs/hfs/super.c + * + * Copyright (C) 1995-1997 Paul H. Hargrove + * (C) 2003 Ardis Technologies + * This file may be distributed under the terms of the GNU General Public License. + * + * This file contains hfs_read_super(), some of the super_ops and + * init_hfs_fs() and exit_hfs_fs(). The remaining super_ops are in + * inode.c since they deal with inodes. + * + * Based on the minix file system code, (C) 1991, 1992 by Linus Torvalds + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hfs_fs.h" +#include "btree.h" + +static struct kmem_cache *hfs_inode_cachep; + +MODULE_DESCRIPTION("Apple Macintosh file system support"); +MODULE_LICENSE("GPL"); + +static int hfs_sync_fs(struct super_block *sb, int wait) +{ + int ret; + + mutex_lock(&HFS_SB(sb)->mdb_lock); + is_hfs_cnid_counts_valid(sb); + ret = hfs_mdb_commit(sb); + mutex_unlock(&HFS_SB(sb)->mdb_lock); + + return ret; +} + +/* + * hfs_put_super() + * + * This is the put_super() entry in the super_operations structure for + * HFS filesystems. The purpose is to release the resources + * associated with the superblock sb. + */ +static void hfs_put_super(struct super_block *sb) +{ + cancel_delayed_work_sync(&HFS_SB(sb)->mdb_work); + hfs_mdb_close(sb); + /* release the MDB's resources */ + hfs_mdb_put(sb); +} + +static void flush_mdb(struct work_struct *work) +{ + struct hfs_sb_info *sbi; + struct super_block *sb; + + sbi = container_of(work, struct hfs_sb_info, mdb_work.work); + sb = sbi->sb; + + spin_lock(&sbi->work_lock); + sbi->work_queued = 0; + spin_unlock(&sbi->work_lock); + + mutex_lock(&sbi->mdb_lock); + is_hfs_cnid_counts_valid(sb); + hfs_mdb_commit(sb); + mutex_unlock(&sbi->mdb_lock); +} + +void hfs_mark_mdb_dirty(struct super_block *sb) +{ + struct hfs_sb_info *sbi = HFS_SB(sb); + unsigned long delay; + + if (sb_rdonly(sb)) + return; + + spin_lock(&sbi->work_lock); + if (!sbi->work_queued) { + delay = msecs_to_jiffies(dirty_writeback_interval * 10); + queue_delayed_work(system_dfl_long_wq, &sbi->mdb_work, delay); + sbi->work_queued = 1; + } + spin_unlock(&sbi->work_lock); +} + +/* + * hfs_statfs() + * + * This is the statfs() entry in the super_operations structure for + * HFS filesystems. The purpose is to return various data about the + * filesystem. + * + * changed f_files/f_ffree to reflect the fs_ablock/free_ablocks. + */ +static int hfs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + u64 id = huge_encode_dev(sb->s_bdev->bd_dev); + + buf->f_type = HFS_SUPER_MAGIC; + buf->f_bsize = sb->s_blocksize; + buf->f_blocks = (u32)HFS_SB(sb)->fs_ablocks * HFS_SB(sb)->fs_div; + buf->f_bfree = (u32)HFS_SB(sb)->free_ablocks * HFS_SB(sb)->fs_div; + buf->f_bavail = buf->f_bfree; + buf->f_files = HFS_SB(sb)->fs_ablocks; + buf->f_ffree = HFS_SB(sb)->free_ablocks; + buf->f_fsid = u64_to_fsid(id); + buf->f_namelen = HFS_NAMELEN; + + return 0; +} + +static int hfs_reconfigure(struct fs_context *fc) +{ + struct super_block *sb = fc->root->d_sb; + + sync_filesystem(sb); + fc->sb_flags |= SB_NODIRATIME; + if ((bool)(fc->sb_flags & SB_RDONLY) == sb_rdonly(sb)) + return 0; + + if (!(fc->sb_flags & SB_RDONLY)) { + if (!(HFS_SB(sb)->mdb->drAtrb & cpu_to_be16(HFS_SB_ATTRIB_UNMNT))) { + pr_warn("filesystem was not cleanly unmounted, running fsck.hfs is recommended. leaving read-only.\n"); + sb->s_flags |= SB_RDONLY; + fc->sb_flags |= SB_RDONLY; + } else if (HFS_SB(sb)->mdb->drAtrb & cpu_to_be16(HFS_SB_ATTRIB_SLOCK)) { + pr_warn("filesystem is marked locked, leaving read-only.\n"); + sb->s_flags |= SB_RDONLY; + fc->sb_flags |= SB_RDONLY; + } + } + return 0; +} + +static int hfs_show_options(struct seq_file *seq, struct dentry *root) +{ + struct hfs_sb_info *sbi = HFS_SB(root->d_sb); + + if (sbi->s_creator != cpu_to_be32(0x3f3f3f3f)) + seq_show_option_n(seq, "creator", (char *)&sbi->s_creator, 4); + if (sbi->s_type != cpu_to_be32(0x3f3f3f3f)) + seq_show_option_n(seq, "type", (char *)&sbi->s_type, 4); + seq_printf(seq, ",uid=%u,gid=%u", + from_kuid_munged(&init_user_ns, sbi->s_uid), + from_kgid_munged(&init_user_ns, sbi->s_gid)); + if (sbi->s_file_umask != 0133) + seq_printf(seq, ",file_umask=%o", sbi->s_file_umask); + if (sbi->s_dir_umask != 0022) + seq_printf(seq, ",dir_umask=%o", sbi->s_dir_umask); + if (sbi->part >= 0) + seq_printf(seq, ",part=%u", sbi->part); + if (sbi->session >= 0) + seq_printf(seq, ",session=%u", sbi->session); + if (sbi->nls_disk) + seq_printf(seq, ",codepage=%s", sbi->nls_disk->charset); + if (sbi->nls_io) + seq_printf(seq, ",iocharset=%s", sbi->nls_io->charset); + if (sbi->s_quiet) + seq_printf(seq, ",quiet"); + return 0; +} + +static struct inode *hfs_alloc_inode(struct super_block *sb) +{ + struct hfs_inode_info *i; + + i = alloc_inode_sb(sb, hfs_inode_cachep, GFP_KERNEL); + return i ? &i->vfs_inode : NULL; +} + +static void hfs_free_inode(struct inode *inode) +{ + kmem_cache_free(hfs_inode_cachep, HFS_I(inode)); +} + +static const struct super_operations hfs_super_operations = { + .alloc_inode = hfs_alloc_inode, + .free_inode = hfs_free_inode, + .write_inode = hfs_write_inode, + .evict_inode = hfs_evict_inode, + .put_super = hfs_put_super, + .sync_fs = hfs_sync_fs, + .statfs = hfs_statfs, + .show_options = hfs_show_options, +}; + +enum { + opt_uid, opt_gid, opt_umask, opt_file_umask, opt_dir_umask, + opt_part, opt_session, opt_type, opt_creator, opt_quiet, + opt_codepage, opt_iocharset, +}; + +static const struct fs_parameter_spec hfs_param_spec[] = { + fsparam_u32 ("uid", opt_uid), + fsparam_u32 ("gid", opt_gid), + fsparam_u32oct ("umask", opt_umask), + fsparam_u32oct ("file_umask", opt_file_umask), + fsparam_u32oct ("dir_umask", opt_dir_umask), + fsparam_u32 ("part", opt_part), + fsparam_u32 ("session", opt_session), + fsparam_string ("type", opt_type), + fsparam_string ("creator", opt_creator), + fsparam_flag ("quiet", opt_quiet), + fsparam_string ("codepage", opt_codepage), + fsparam_string ("iocharset", opt_iocharset), + {} +}; + +/* + * hfs_parse_param() + * + * This function is called by the vfs to parse the mount options. + */ +static int hfs_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct hfs_sb_info *hsb = fc->s_fs_info; + struct fs_parse_result result; + int opt; + + /* hfs does not honor any fs-specific options on remount */ + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE) + return 0; + + opt = fs_parse(fc, hfs_param_spec, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case opt_uid: + hsb->s_uid = result.uid; + break; + case opt_gid: + hsb->s_gid = result.gid; + break; + case opt_umask: + hsb->s_file_umask = (umode_t)result.uint_32; + hsb->s_dir_umask = (umode_t)result.uint_32; + break; + case opt_file_umask: + hsb->s_file_umask = (umode_t)result.uint_32; + break; + case opt_dir_umask: + hsb->s_dir_umask = (umode_t)result.uint_32; + break; + case opt_part: + hsb->part = result.uint_32; + break; + case opt_session: + hsb->session = result.uint_32; + break; + case opt_type: + if (strlen(param->string) != 4) { + pr_err("type requires a 4 character value\n"); + return -EINVAL; + } + memcpy(&hsb->s_type, param->string, 4); + break; + case opt_creator: + if (strlen(param->string) != 4) { + pr_err("creator requires a 4 character value\n"); + return -EINVAL; + } + memcpy(&hsb->s_creator, param->string, 4); + break; + case opt_quiet: + hsb->s_quiet = 1; + break; + case opt_codepage: + if (hsb->nls_disk) { + pr_err("unable to change codepage\n"); + return -EINVAL; + } + hsb->nls_disk = load_nls(param->string); + if (!hsb->nls_disk) { + pr_err("unable to load codepage \"%s\"\n", + param->string); + return -EINVAL; + } + break; + case opt_iocharset: + if (hsb->nls_io) { + pr_err("unable to change iocharset\n"); + return -EINVAL; + } + hsb->nls_io = load_nls(param->string); + if (!hsb->nls_io) { + pr_err("unable to load iocharset \"%s\"\n", + param->string); + return -EINVAL; + } + break; + default: + return -EINVAL; + } + + return 0; +} + +/* + * hfs_read_super() + * + * This is the function that is responsible for mounting an HFS + * filesystem. It performs all the tasks necessary to get enough data + * from the disk to read the root inode. This includes parsing the + * mount options, dealing with Macintosh partitions, reading the + * superblock and the allocation bitmap blocks, calling + * hfs_btree_init() to get the necessary data about the extents and + * catalog B-trees and, finally, reading the root inode into memory. + */ +static int hfs_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct hfs_sb_info *sbi = HFS_SB(sb); + struct hfs_find_data fd; + hfs_cat_rec rec; + struct inode *root_inode; + int silent = fc->sb_flags & SB_SILENT; + int res; + + atomic64_set(&sbi->file_count, 0); + atomic64_set(&sbi->folder_count, 0); + atomic64_set(&sbi->next_id, 0); + + /* load_nls_default does not fail */ + if (sbi->nls_disk && !sbi->nls_io) + sbi->nls_io = load_nls_default(); + sbi->s_dir_umask &= 0777; + sbi->s_file_umask &= 0577; + + spin_lock_init(&sbi->work_lock); + INIT_DELAYED_WORK(&sbi->mdb_work, flush_mdb); + + sbi->sb = sb; + sb->s_op = &hfs_super_operations; + sb->s_xattr = hfs_xattr_handlers; + sb->s_flags |= SB_NOATIME | SB_NODIRATIME; + mutex_init(&sbi->mdb_lock); + mutex_init(&sbi->bitmap_lock); + + mutex_lock(&sbi->mdb_lock); + res = hfs_mdb_get(sb); + mutex_unlock(&sbi->mdb_lock); + if (res) { + if (!silent) + pr_warn("can't find a HFS filesystem on dev %s\n", + hfs_mdb_name(sb)); + res = -EINVAL; + goto bail; + } + + /* try to get the root inode */ + res = hfs_find_init(HFS_SB(sb)->cat_tree, &fd); + if (res) + goto bail_no_root; + res = hfs_cat_find_brec(sb, HFS_ROOT_CNID, &fd); + if (!res) { + if (fd.entrylength != sizeof(rec.dir)) { + res = -EIO; + goto bail_hfs_find; + } + hfs_bnode_read(fd.bnode, &rec, fd.entryoffset, fd.entrylength); + if (rec.type != HFS_CDR_DIR) + res = -EIO; + } + if (res) + goto bail_hfs_find; + res = -EINVAL; + root_inode = hfs_iget(sb, &fd.search_key->cat, &rec); + hfs_find_exit(&fd); + if (!root_inode) + goto bail_no_root; + + if (is_bad_inode(root_inode)) { + iput(root_inode); + goto bail_no_root; + } + + set_default_d_op(sb, &hfs_dentry_operations); + res = -ENOMEM; + sb->s_root = d_make_root(root_inode); + if (!sb->s_root) + goto bail_no_root; + + /* everything's okay */ + return 0; + +bail_hfs_find: + hfs_find_exit(&fd); +bail_no_root: + pr_err("get root inode failed\n"); +bail: + hfs_mdb_put(sb); + return res; +} + +static int hfs_get_tree(struct fs_context *fc) +{ + return get_tree_bdev(fc, hfs_fill_super); +} + +static void hfs_free_fc(struct fs_context *fc) +{ + kfree(fc->s_fs_info); +} + +static const struct fs_context_operations hfs_context_ops = { + .parse_param = hfs_parse_param, + .get_tree = hfs_get_tree, + .reconfigure = hfs_reconfigure, + .free = hfs_free_fc, +}; + +static int hfs_init_fs_context(struct fs_context *fc) +{ + struct hfs_sb_info *hsb; + + hsb = kzalloc_obj(struct hfs_sb_info); + if (!hsb) + return -ENOMEM; + + fc->s_fs_info = hsb; + fc->ops = &hfs_context_ops; + + if (fc->purpose != FS_CONTEXT_FOR_RECONFIGURE) { + /* initialize options with defaults */ + hsb->s_uid = current_uid(); + hsb->s_gid = current_gid(); + hsb->s_file_umask = 0133; + hsb->s_dir_umask = 0022; + hsb->s_type = cpu_to_be32(0x3f3f3f3f); /* == '????' */ + hsb->s_creator = cpu_to_be32(0x3f3f3f3f); /* == '????' */ + hsb->s_quiet = 0; + hsb->part = -1; + hsb->session = -1; + } + + return 0; +} + +static void hfs_kill_super(struct super_block *sb) +{ + struct hfs_sb_info *hsb = HFS_SB(sb); + + kill_block_super(sb); + kfree(hsb); +} + +static struct file_system_type hfs_fs_type = { + .owner = THIS_MODULE, + .name = "hfs", + .kill_sb = hfs_kill_super, + .fs_flags = FS_REQUIRES_DEV, + .init_fs_context = hfs_init_fs_context, +}; +MODULE_ALIAS_FS("hfs"); + +static void hfs_init_once(void *p) +{ + struct hfs_inode_info *i = p; + + inode_init_once(&i->vfs_inode); +} + +static int __init init_hfs_fs(void) +{ + int err; + + hfs_inode_cachep = kmem_cache_create("hfs_inode_cache", + sizeof(struct hfs_inode_info), 0, + SLAB_HWCACHE_ALIGN|SLAB_ACCOUNT, hfs_init_once); + if (!hfs_inode_cachep) + return -ENOMEM; + err = register_filesystem(&hfs_fs_type); + if (err) + kmem_cache_destroy(hfs_inode_cachep); + return err; +} + +static void __exit exit_hfs_fs(void) +{ + unregister_filesystem(&hfs_fs_type); + + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(hfs_inode_cachep); +} + +module_init(init_hfs_fs) +module_exit(exit_hfs_fs) diff --git a/fs/hfs/sysdep.c b/fs/hfs/sysdep.c new file mode 100644 index 000000000..ef54fc809 --- /dev/null +++ b/fs/hfs/sysdep.c @@ -0,0 +1,50 @@ +/* + * linux/fs/hfs/sysdep.c + * + * Copyright (C) 1996 Paul H. Hargrove + * (C) 2003 Ardis Technologies + * This file may be distributed under the terms of the GNU General Public License. + * + * This file contains the code to do various system dependent things. + */ + +#include +#include "hfs_fs.h" + +/* dentry case-handling: just lowercase everything */ + +static int hfs_revalidate_dentry(struct inode *dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + struct inode *inode; + int diff; + + if (flags & LOOKUP_RCU) + return -ECHILD; + + inode = d_inode(dentry); + if(!inode) + return 1; + + /* fix up inode on a timezone change */ + diff = sys_tz.tz_minuteswest * 60 - HFS_I(inode)->tz_secondswest; + if (diff) { + struct timespec64 ts = inode_get_ctime(inode); + + inode_set_ctime(inode, ts.tv_sec + diff, ts.tv_nsec); + ts = inode_get_atime(inode); + inode_set_atime(inode, ts.tv_sec + diff, ts.tv_nsec); + ts = inode_get_mtime(inode); + inode_set_mtime(inode, ts.tv_sec + diff, ts.tv_nsec); + HFS_I(inode)->tz_secondswest += diff; + } + return 1; +} + +const struct dentry_operations hfs_dentry_operations = +{ + .d_revalidate = hfs_revalidate_dentry, + .d_hash = hfs_hash_dentry, + .d_compare = hfs_compare_dentry, +}; + diff --git a/fs/hfs/trans.c b/fs/hfs/trans.c new file mode 100644 index 000000000..fdb0edb8a --- /dev/null +++ b/fs/hfs/trans.c @@ -0,0 +1,150 @@ +/* + * linux/fs/hfs/trans.c + * + * Copyright (C) 1995-1997 Paul H. Hargrove + * This file may be distributed under the terms of the GNU General Public License. + * + * This file contains routines for converting between the Macintosh + * character set and various other encodings. This includes dealing + * with ':' vs. '/' as the path-element separator. + */ + +#include +#include + +#include "hfs_fs.h" + +/*================ Global functions ================*/ + +/* + * hfs_mac2asc() + * + * Given a 'Pascal String' (a string preceded by a length byte) in + * the Macintosh character set produce the corresponding filename using + * the 'trivial' name-mangling scheme, returning the length of the + * mangled filename. Note that the output string is not NULL + * terminated. + * + * The name-mangling works as follows: + * The character '/', which is illegal in Linux filenames is replaced + * by ':' which never appears in HFS filenames. All other characters + * are passed unchanged from input to output. + */ +int hfs_mac2asc(struct super_block *sb, char *out, const struct hfs_name *in) +{ + struct nls_table *nls_disk = HFS_SB(sb)->nls_disk; + struct nls_table *nls_io = HFS_SB(sb)->nls_io; + const char *src; + char *dst; + int srclen, dstlen, size; + + src = in->name; + srclen = in->len; + if (srclen > HFS_NAMELEN) + srclen = HFS_NAMELEN; + dst = out; + dstlen = HFS_MAX_NAMELEN; + if (nls_io) { + wchar_t ch; + + while (srclen > 0) { + if (nls_disk) { + size = nls_disk->char2uni(src, srclen, &ch); + if (size <= 0) { + ch = '?'; + size = 1; + } + src += size; + srclen -= size; + } else { + ch = *src++; + srclen--; + } + if (ch == '/') + ch = ':'; + size = nls_io->uni2char(ch, dst, dstlen); + if (size < 0) { + if (size == -ENAMETOOLONG) + goto out; + *dst = '?'; + size = 1; + } + dst += size; + dstlen -= size; + } + } else { + char ch; + + while (--srclen >= 0) + *dst++ = (ch = *src++) == '/' ? ':' : ch; + } +out: + return dst - out; +} + +/* + * hfs_asc2mac() + * + * Given an ASCII string (not null-terminated) and its length, + * generate the corresponding filename in the Macintosh character set + * using the 'trivial' name-mangling scheme, returning the length of + * the mangled filename. Note that the output string is not NULL + * terminated. + * + * This routine is a inverse to hfs_mac2triv(). + * A ':' is replaced by a '/'. + */ +void hfs_asc2mac(struct super_block *sb, struct hfs_name *out, const struct qstr *in) +{ + struct nls_table *nls_disk = HFS_SB(sb)->nls_disk; + struct nls_table *nls_io = HFS_SB(sb)->nls_io; + const char *src; + char *dst; + int srclen, dstlen, size; + + src = in->name; + srclen = in->len; + dst = out->name; + dstlen = HFS_NAMELEN; + if (nls_io) { + wchar_t ch; + + while (srclen > 0 && dstlen > 0) { + size = nls_io->char2uni(src, srclen, &ch); + if (size < 0) { + ch = '?'; + size = 1; + } + src += size; + srclen -= size; + if (ch == ':') + ch = '/'; + if (nls_disk) { + size = nls_disk->uni2char(ch, dst, dstlen); + if (size < 0) { + if (size == -ENAMETOOLONG) + goto out; + *dst = '?'; + size = 1; + } + dst += size; + dstlen -= size; + } else { + *dst++ = ch > 0xff ? '?' : ch; + dstlen--; + } + } + } else { + char ch; + + if (dstlen > srclen) + dstlen = srclen; + while (--dstlen >= 0) + *dst++ = (ch = *src++) == ':' ? '/' : ch; + } +out: + out->len = dst - (char *)out->name; + dstlen = HFS_NAMELEN - out->len; + while (--dstlen >= 0) + *dst++ = 0; +} diff --git a/fs/hfsplus/.kunitconfig b/fs/hfsplus/.kunitconfig new file mode 100644 index 000000000..6c96dc7e8 --- /dev/null +++ b/fs/hfsplus/.kunitconfig @@ -0,0 +1,8 @@ +CONFIG_KUNIT=y +CONFIG_HFSPLUS_FS=y +CONFIG_HFSPLUS_KUNIT_TEST=y +CONFIG_BLOCK=y +CONFIG_BUFFER_HEAD=y +CONFIG_NLS=y +CONFIG_NLS_UTF8=y +CONFIG_LEGACY_DIRECT_IO=y diff --git a/fs/hfsplus/Kconfig b/fs/hfsplus/Kconfig new file mode 100644 index 000000000..ca8401cb6 --- /dev/null +++ b/fs/hfsplus/Kconfig @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: GPL-2.0-only +config HFSPLUS_FS + tristate "Apple Extended HFS file system support" + depends on BLOCK + select BUFFER_HEAD + select NLS + select NLS_UTF8 + select LEGACY_DIRECT_IO + help + If you say Y here, you will be able to mount extended format + Macintosh-formatted hard drive partitions with full read-write access. + + This file system is often called HFS+ and was introduced with + MacOS 8. It includes all Mac specific filesystem data such as + data forks and creator codes, but it also has several UNIX + style features such as file ownership and permissions. + +config HFSPLUS_KUNIT_TEST + tristate "KUnit tests for HFS+ filesystem" if !KUNIT_ALL_TESTS + depends on HFSPLUS_FS && KUNIT + default KUNIT_ALL_TESTS + help + This builds KUnit tests for the HFS+ filesystem. + + KUnit tests run during boot and output the results to the debug + log in TAP format (https://testanything.org/). Only useful for + kernel devs running KUnit test harness and are not for inclusion + into a production build. + + For more information on KUnit and unit tests in general please + refer to the KUnit documentation in Documentation/dev-tools/kunit/. diff --git a/fs/hfsplus/Makefile b/fs/hfsplus/Makefile new file mode 100644 index 000000000..f2a9ae697 --- /dev/null +++ b/fs/hfsplus/Makefile @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0 +# +## Makefile for the linux hfsplus filesystem routines. +# + +obj-$(CONFIG_HFSPLUS_FS) += hfsplus.o + +hfsplus-objs := super.o options.o inode.o ioctl.o extents.o catalog.o dir.o btree.o \ + bnode.o brec.o bfind.o tables.o unicode.o wrapper.o bitmap.o part_tbl.o \ + attributes.o xattr.o xattr_user.o xattr_security.o xattr_trusted.o + +# KUnit tests +obj-$(CONFIG_HFSPLUS_KUNIT_TEST) += unicode_test.o diff --git a/fs/hfsplus/attributes.c b/fs/hfsplus/attributes.c new file mode 100644 index 000000000..7c2e589d4 --- /dev/null +++ b/fs/hfsplus/attributes.c @@ -0,0 +1,499 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hfsplus/attributes.c + * + * Vyacheslav Dubeyko + * + * Handling of records in attributes tree + */ + +#include "hfsplus_fs.h" +#include "hfsplus_raw.h" + +static struct kmem_cache *hfsplus_attr_tree_cachep; + +int __init hfsplus_create_attr_tree_cache(void) +{ + if (hfsplus_attr_tree_cachep) + return -EEXIST; + + hfsplus_attr_tree_cachep = + kmem_cache_create("hfsplus_attr_cache", + sizeof(hfsplus_attr_entry), 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!hfsplus_attr_tree_cachep) + return -ENOMEM; + + return 0; +} + +void hfsplus_destroy_attr_tree_cache(void) +{ + kmem_cache_destroy(hfsplus_attr_tree_cachep); +} + +int hfsplus_attr_bin_cmp_key(const hfsplus_btree_key *k1, + const hfsplus_btree_key *k2) +{ + __be32 k1_cnid, k2_cnid; + + k1_cnid = k1->attr.cnid; + k2_cnid = k2->attr.cnid; + if (k1_cnid != k2_cnid) + return be32_to_cpu(k1_cnid) < be32_to_cpu(k2_cnid) ? -1 : 1; + + return hfsplus_strcmp( + (const struct hfsplus_unistr *)&k1->attr.key_name, + (const struct hfsplus_unistr *)&k2->attr.key_name); +} + +int hfsplus_attr_build_key(struct super_block *sb, hfsplus_btree_key *key, + u32 cnid, const char *name) +{ + int len; + + memset(key, 0, sizeof(struct hfsplus_attr_key)); + key->attr.cnid = cpu_to_be32(cnid); + if (name) { + int res = hfsplus_asc2uni(sb, + (struct hfsplus_unistr *)&key->attr.key_name, + HFSPLUS_ATTR_MAX_STRLEN, name, strlen(name), + HFS_XATTR_NAME); + if (res) + return res; + len = be16_to_cpu(key->attr.key_name.length); + } else { + key->attr.key_name.length = 0; + len = 0; + } + + /* The length of the key, as stored in key_len field, does not include + * the size of the key_len field itself. + * So, offsetof(hfsplus_attr_key, key_name) is a trick because + * it takes into consideration key_len field (__be16) of + * hfsplus_attr_key structure instead of length field (__be16) of + * hfsplus_attr_unistr structure. + */ + key->key_len = + cpu_to_be16(offsetof(struct hfsplus_attr_key, key_name) + + 2 * len); + + return 0; +} + +hfsplus_attr_entry *hfsplus_alloc_attr_entry(void) +{ + return kmem_cache_alloc(hfsplus_attr_tree_cachep, GFP_KERNEL); +} + +void hfsplus_destroy_attr_entry(hfsplus_attr_entry *entry) +{ + if (entry) + kmem_cache_free(hfsplus_attr_tree_cachep, entry); +} + +#define HFSPLUS_INVALID_ATTR_RECORD -1 + +static int hfsplus_attr_build_record(hfsplus_attr_entry *entry, int record_type, + u32 cnid, const void *value, size_t size) +{ + if (record_type == HFSPLUS_ATTR_FORK_DATA) { + /* + * Mac OS X supports only inline data attributes. + * Do nothing + */ + memset(entry, 0, sizeof(*entry)); + return sizeof(struct hfsplus_attr_fork_data); + } else if (record_type == HFSPLUS_ATTR_EXTENTS) { + /* + * Mac OS X supports only inline data attributes. + * Do nothing. + */ + memset(entry, 0, sizeof(*entry)); + return sizeof(struct hfsplus_attr_extents); + } else if (record_type == HFSPLUS_ATTR_INLINE_DATA) { + u16 len; + + memset(entry, 0, sizeof(struct hfsplus_attr_inline_data)); + entry->inline_data.record_type = cpu_to_be32(record_type); + if (size <= HFSPLUS_MAX_INLINE_DATA_SIZE) + len = size; + else { + hfs_dbg("value size %zu is too big\n", size); + return HFSPLUS_INVALID_ATTR_RECORD; + } + entry->inline_data.length = cpu_to_be16(len); + memcpy(entry->inline_data.raw_bytes, value, len); + /* + * Align len on two-byte boundary. + * It needs to add pad byte if we have odd len. + */ + len = round_up(len, 2); + return offsetof(struct hfsplus_attr_inline_data, raw_bytes) + + len; + } else /* invalid input */ + memset(entry, 0, sizeof(*entry)); + + return HFSPLUS_INVALID_ATTR_RECORD; +} + +int hfsplus_find_attr(struct super_block *sb, u32 cnid, + const char *name, struct hfs_find_data *fd) +{ + int err = 0; + + hfs_dbg("name %s, cnid %d\n", name ? name : NULL, cnid); + + if (!HFSPLUS_SB(sb)->attr_tree) { + pr_err("attributes file doesn't exist\n"); + return -EINVAL; + } + + if (name) { + err = hfsplus_attr_build_key(sb, fd->search_key, cnid, name); + if (err) + goto failed_find_attr; + err = hfs_brec_find(fd, hfs_find_rec_by_key); + if (err == -ENOENT) { + /* file exists but xattr is absent */ + err = -ENODATA; + goto failed_find_attr; + } else if (err) + goto failed_find_attr; + } else { + err = hfsplus_attr_build_key(sb, fd->search_key, cnid, NULL); + if (err) + goto failed_find_attr; + err = hfs_brec_find(fd, hfs_find_1st_rec_by_cnid); + if (err == -ENOENT) { + /* file exists but xattr is absent */ + err = -ENODATA; + goto failed_find_attr; + } else if (err) + goto failed_find_attr; + } + +failed_find_attr: + return err; +} + +int hfsplus_attr_exists(struct inode *inode, const char *name) +{ + int err = 0; + struct super_block *sb = inode->i_sb; + struct hfs_find_data fd; + + hfs_dbg("name %s, ino %llu\n", + name ? name : NULL, inode->i_ino); + + if (!HFSPLUS_SB(sb)->attr_tree) + return 0; + + err = hfs_find_init(HFSPLUS_SB(sb)->attr_tree, &fd); + if (err) + return 0; + + err = hfsplus_find_attr(sb, inode->i_ino, name, &fd); + if (err) + goto attr_not_found; + + hfs_find_exit(&fd); + return 1; + +attr_not_found: + hfs_find_exit(&fd); + return 0; +} + +static +int hfsplus_create_attr_nolock(struct inode *inode, const char *name, + const void *value, size_t size, + struct hfs_find_data *fd, + hfsplus_attr_entry *entry_ptr) +{ + struct super_block *sb = inode->i_sb; + int entry_size; + int err; + + hfs_dbg("name %s, ino %llu\n", + name ? name : NULL, inode->i_ino); + + if (name) { + err = hfsplus_attr_build_key(sb, fd->search_key, + inode->i_ino, name); + if (err) + return err; + } else + return -EINVAL; + + /* Mac OS X supports only inline data attributes. */ + entry_size = hfsplus_attr_build_record(entry_ptr, + HFSPLUS_ATTR_INLINE_DATA, + inode->i_ino, + value, size); + if (entry_size == HFSPLUS_INVALID_ATTR_RECORD) { + if (size > HFSPLUS_MAX_INLINE_DATA_SIZE) + err = -E2BIG; + else + err = -EINVAL; + hfs_dbg("unable to store value: err %d\n", err); + return err; + } + + err = hfs_brec_find(fd, hfs_find_rec_by_key); + if (err != -ENOENT) { + if (!err) + err = -EEXIST; + return err; + } + + err = hfs_brec_insert(fd, entry_ptr, entry_size); + if (err) { + hfs_dbg("unable to store value: err %d\n", err); + return err; + } + + hfsplus_mark_inode_dirty(HFSPLUS_ATTR_TREE_I(sb), HFSPLUS_I_ATTR_DIRTY); + hfsplus_mark_inode_dirty(inode, HFSPLUS_I_ATTR_DIRTY); + + return 0; +} + +int hfsplus_create_attr(struct inode *inode, + const char *name, + const void *value, size_t size) +{ + struct super_block *sb = inode->i_sb; + struct hfs_find_data fd; + hfsplus_attr_entry *entry_ptr; + int err; + + hfs_dbg("name %s, ino %llu\n", + name ? name : NULL, inode->i_ino); + + if (!HFSPLUS_SB(sb)->attr_tree) { + pr_err("attributes file doesn't exist\n"); + return -EINVAL; + } + + entry_ptr = hfsplus_alloc_attr_entry(); + if (!entry_ptr) + return -ENOMEM; + + err = hfs_find_init(HFSPLUS_SB(sb)->attr_tree, &fd); + if (err) + goto failed_init_create_attr; + + /* Fail early and avoid ENOSPC during the btree operation */ + err = hfs_bmap_reserve(fd.tree, fd.tree->depth + 1); + if (err) + goto failed_create_attr; + + err = hfsplus_create_attr_nolock(inode, name, value, size, + &fd, entry_ptr); + if (err) + goto failed_create_attr; + +failed_create_attr: + hfs_find_exit(&fd); + +failed_init_create_attr: + hfsplus_destroy_attr_entry(entry_ptr); + return err; +} + +static int __hfsplus_delete_attr(struct inode *inode, u32 cnid, + struct hfs_find_data *fd) +{ + int err; + __be32 found_cnid, record_type; + + found_cnid = U32_MAX; + hfs_bnode_read(fd->bnode, &found_cnid, + fd->keyoffset + + offsetof(struct hfsplus_attr_key, cnid), + sizeof(__be32)); + if (cnid != be32_to_cpu(found_cnid)) + return -ENODATA; + + hfs_bnode_read(fd->bnode, &record_type, + fd->entryoffset, sizeof(record_type)); + + switch (be32_to_cpu(record_type)) { + case HFSPLUS_ATTR_INLINE_DATA: + /* All is OK. Do nothing. */ + break; + case HFSPLUS_ATTR_FORK_DATA: + case HFSPLUS_ATTR_EXTENTS: + pr_err("only inline data xattr are supported\n"); + return -EOPNOTSUPP; + default: + pr_err("invalid extended attribute record\n"); + return -ENOENT; + } + + /* Avoid btree corruption */ + hfs_bnode_read(fd->bnode, fd->search_key, + fd->keyoffset, fd->keylength); + + err = hfs_brec_remove(fd); + if (err) + return err; + + hfsplus_mark_inode_dirty(HFSPLUS_ATTR_TREE_I(inode->i_sb), + HFSPLUS_I_ATTR_DIRTY); + hfsplus_mark_inode_dirty(inode, HFSPLUS_I_ATTR_DIRTY); + return 0; +} + +static +int hfsplus_delete_attr_nolock(struct inode *inode, const char *name, + struct hfs_find_data *fd) +{ + struct super_block *sb = inode->i_sb; + int err; + + hfs_dbg("name %s, ino %llu\n", + name ? name : NULL, inode->i_ino); + + if (name) { + err = hfsplus_attr_build_key(sb, fd->search_key, + inode->i_ino, name); + if (err) + return err; + } else { + pr_err("invalid extended attribute name\n"); + return -EINVAL; + } + + err = hfs_brec_find(fd, hfs_find_rec_by_key); + if (err == -ENOENT) { + /* file exists but xattr is absent */ + return -ENODATA; + } else if (err) + return err; + + err = __hfsplus_delete_attr(inode, inode->i_ino, fd); + if (err) + return err; + + return 0; +} + +int hfsplus_delete_attr(struct inode *inode, const char *name) +{ + int err = 0; + struct super_block *sb = inode->i_sb; + struct hfs_find_data fd; + + hfs_dbg("name %s, ino %llu\n", + name ? name : NULL, inode->i_ino); + + if (!HFSPLUS_SB(sb)->attr_tree) { + pr_err("attributes file doesn't exist\n"); + return -EINVAL; + } + + err = hfs_find_init(HFSPLUS_SB(sb)->attr_tree, &fd); + if (err) + return err; + + /* Fail early and avoid ENOSPC during the btree operation */ + err = hfs_bmap_reserve(fd.tree, fd.tree->depth); + if (err) + goto out; + + err = hfsplus_delete_attr_nolock(inode, name, &fd); + if (err) + goto out; + +out: + hfs_find_exit(&fd); + return err; +} + +int hfsplus_delete_all_attrs(struct inode *dir, u32 cnid) +{ + int err = 0; + struct hfs_find_data fd; + + hfs_dbg("cnid %d\n", cnid); + + if (!HFSPLUS_SB(dir->i_sb)->attr_tree) { + pr_err("attributes file doesn't exist\n"); + return -EINVAL; + } + + err = hfs_find_init(HFSPLUS_SB(dir->i_sb)->attr_tree, &fd); + if (err) + return err; + + for (;;) { + err = hfsplus_find_attr(dir->i_sb, cnid, NULL, &fd); + if (err == -ENOENT || err == -ENODATA) { + /* + * xattr has not been found + */ + err = -ENODATA; + goto end_delete_all; + } else if (err) { + pr_err("xattr search failed\n"); + goto end_delete_all; + } + + err = __hfsplus_delete_attr(dir, cnid, &fd); + if (err) + goto end_delete_all; + } + +end_delete_all: + hfs_find_exit(&fd); + return err; +} + +int hfsplus_replace_attr(struct inode *inode, + const char *name, + const void *value, size_t size) +{ + struct super_block *sb = inode->i_sb; + struct hfs_find_data fd; + hfsplus_attr_entry *entry_ptr; + int err = 0; + + hfs_dbg("name %s, ino %llu\n", + name ? name : NULL, inode->i_ino); + + if (!HFSPLUS_SB(sb)->attr_tree) { + pr_err("attributes file doesn't exist\n"); + return -EINVAL; + } + + entry_ptr = hfsplus_alloc_attr_entry(); + if (!entry_ptr) + return -ENOMEM; + + err = hfs_find_init(HFSPLUS_SB(sb)->attr_tree, &fd); + if (err) + goto failed_init_replace_attr; + + /* Fail early and avoid ENOSPC during the btree operation */ + err = hfs_bmap_reserve(fd.tree, fd.tree->depth + 1); + if (err) + goto failed_replace_attr; + + err = hfsplus_delete_attr_nolock(inode, name, &fd); + if (err) + goto failed_replace_attr; + + err = hfsplus_create_attr_nolock(inode, name, value, size, + &fd, entry_ptr); + if (err) + goto failed_replace_attr; + +failed_replace_attr: + hfs_find_exit(&fd); + +failed_init_replace_attr: + hfsplus_destroy_attr_entry(entry_ptr); + return err; +} diff --git a/fs/hfsplus/bfind.c b/fs/hfsplus/bfind.c new file mode 100644 index 000000000..ca9813f58 --- /dev/null +++ b/fs/hfsplus/bfind.c @@ -0,0 +1,343 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hfsplus/bfind.c + * + * Copyright (C) 2001 + * Brad Boyer (flar@allandria.com) + * (C) 2003 Ardis Technologies + * + * Search routines for btrees + */ + +#include +#include "hfsplus_fs.h" + +int hfs_find_init(struct hfs_btree *tree, struct hfs_find_data *fd) +{ + void *ptr; + + fd->tree = tree; + fd->bnode = NULL; + hfs_find_result_init(fd); + ptr = kzalloc(tree->max_key_len * 2 + 4, GFP_KERNEL); + if (!ptr) + return -ENOMEM; + fd->search_key = ptr; + fd->key = ptr + tree->max_key_len + 2; + hfs_dbg("cnid %d, caller %ps\n", + tree->cnid, __builtin_return_address(0)); + mutex_lock_nested(&tree->tree_lock, + hfsplus_btree_lock_class(tree)); + return 0; +} + +void hfs_find_exit(struct hfs_find_data *fd) +{ + hfs_bnode_put(fd->bnode); + kfree(fd->search_key); + hfs_dbg("cnid %d, caller %ps\n", + fd->tree->cnid, __builtin_return_address(0)); + mutex_unlock(&fd->tree->tree_lock); + fd->tree = NULL; +} + +int hfs_find_1st_rec_by_cnid(struct hfs_bnode *bnode, + struct hfs_find_data *fd, + int *begin, + int *end, + int *cur_rec) +{ + __be32 cur_cnid; + __be32 search_cnid; + + if (bnode->tree->cnid == HFSPLUS_EXT_CNID) { + cur_cnid = fd->key->ext.cnid; + search_cnid = fd->search_key->ext.cnid; + } else if (bnode->tree->cnid == HFSPLUS_CAT_CNID) { + cur_cnid = fd->key->cat.parent; + search_cnid = fd->search_key->cat.parent; + } else if (bnode->tree->cnid == HFSPLUS_ATTR_CNID) { + cur_cnid = fd->key->attr.cnid; + search_cnid = fd->search_key->attr.cnid; + } else { + cur_cnid = 0; /* used-uninitialized warning */ + search_cnid = 0; + BUG(); + } + + if (cur_cnid == search_cnid) { + (*end) = (*cur_rec); + if ((*begin) == (*end)) + return 1; + } else { + if (be32_to_cpu(cur_cnid) < be32_to_cpu(search_cnid)) + (*begin) = (*cur_rec) + 1; + else + (*end) = (*cur_rec) - 1; + } + + return 0; +} + +int hfs_find_rec_by_key(struct hfs_bnode *bnode, + struct hfs_find_data *fd, + int *begin, + int *end, + int *cur_rec) +{ + int cmpval; + + cmpval = bnode->tree->keycmp(fd->key, fd->search_key); + if (!cmpval) { + (*end) = (*cur_rec); + return 1; + } + if (cmpval < 0) + (*begin) = (*cur_rec) + 1; + else + *(end) = (*cur_rec) - 1; + + return 0; +} + +/* Find the record in bnode that best matches key (not greater than...)*/ +int __hfs_brec_find(struct hfs_bnode *bnode, struct hfs_find_data *fd, + search_strategy_t rec_found) +{ + u16 off, len, keylen; + int rec; + int b, e; + int res = -ENOENT; + + BUG_ON(!rec_found); + hfs_find_result_init(fd); + if (hfs_bnode_num_recs_invalid(bnode)) + goto fail; + + b = 0; + e = bnode->num_recs - 1; + do { + rec = (e + b) / 2; + len = hfs_brec_lenoff(bnode, rec, &off); + keylen = hfs_brec_keylen(bnode, rec); + if (hfs_brec_len_invalid(bnode, len) || + hfs_brec_len_invalid(bnode, keylen)) { + res = -EINVAL; + goto fail; + } + hfs_bnode_read(bnode, fd->key, off, keylen); + if (rec_found(bnode, fd, &b, &e, &rec)) { + res = 0; + goto done; + } + } while (b <= e); + + if (rec != e && e >= 0) { + len = hfs_brec_lenoff(bnode, e, &off); + keylen = hfs_brec_keylen(bnode, e); + if (hfs_brec_len_invalid(bnode, keylen) || + hfs_brec_len_invalid(bnode, len)) { + res = -EINVAL; + goto fail; + } + hfs_bnode_read(bnode, fd->key, off, keylen); + } + +done: + fd->record = e; + fd->keyoffset = off; + fd->keylength = keylen; + fd->entryoffset = off + keylen; + fd->entrylength = len - keylen; + +fail: + return res; +} + +/* Traverse a B*Tree from the root to a leaf finding best fit to key */ +/* Return allocated copy of node found, set recnum to best record */ +int hfs_brec_find(struct hfs_find_data *fd, search_strategy_t do_key_compare) +{ + struct hfs_btree *tree; + struct hfs_bnode *bnode; + u32 nidx, parent; + __be32 data; + int height, res; + + hfs_find_result_init(fd); + + tree = fd->tree; + if (fd->bnode) + hfs_bnode_put(fd->bnode); + fd->bnode = NULL; + nidx = tree->root; + if (!nidx) + return -ENOENT; + height = tree->depth; + res = 0; + parent = 0; + for (;;) { + bnode = hfs_bnode_find(tree, nidx); + if (IS_ERR(bnode)) { + res = PTR_ERR(bnode); + bnode = NULL; + break; + } + if (bnode->height != height) + goto invalid; + if (bnode->type != (--height ? HFS_NODE_INDEX : HFS_NODE_LEAF)) + goto invalid; + bnode->parent = parent; + + res = __hfs_brec_find(bnode, fd, do_key_compare); + if (!height) + break; + if (fd->record < 0) + goto release; + + parent = nidx; + hfs_bnode_read(bnode, &data, fd->entryoffset, 4); + nidx = be32_to_cpu(data); + hfs_bnode_put(bnode); + } + fd->bnode = bnode; + return res; + +invalid: + pr_err("inconsistency in B*Tree (%d,%d,%d,%u,%u)\n", + height, bnode->height, bnode->type, nidx, parent); + res = -EIO; +release: + hfs_bnode_put(bnode); + return res; +} + +int hfs_brec_read(struct hfs_find_data *fd, void *rec, u32 rec_len) +{ + int res; + + res = hfs_brec_find(fd, hfs_find_rec_by_key); + if (res) + return res; + if (fd->entrylength > rec_len) + return -EINVAL; + hfs_bnode_read(fd->bnode, rec, fd->entryoffset, fd->entrylength); + return 0; +} + +int hfs_brec_goto(struct hfs_find_data *fd, int cnt) +{ + struct hfs_btree *tree; + struct hfs_bnode *bnode; + int idx, res = 0; + u16 off, len, keylen; + + bnode = fd->bnode; + tree = bnode->tree; + + if (cnt < 0) { + cnt = -cnt; + while (cnt > fd->record) { + cnt -= fd->record + 1; + fd->record = bnode->num_recs - 1; + idx = bnode->prev; + if (!idx) { + res = -ENOENT; + goto out; + } + hfs_bnode_put(bnode); + bnode = hfs_bnode_find(tree, idx); + if (IS_ERR(bnode)) { + res = PTR_ERR(bnode); + bnode = NULL; + goto out; + } + } + fd->record -= cnt; + } else { + while (cnt >= bnode->num_recs - fd->record) { + cnt -= bnode->num_recs - fd->record; + fd->record = 0; + idx = bnode->next; + if (!idx) { + res = -ENOENT; + goto out; + } + hfs_bnode_put(bnode); + bnode = hfs_bnode_find(tree, idx); + if (IS_ERR(bnode)) { + res = PTR_ERR(bnode); + bnode = NULL; + goto out; + } + } + fd->record += cnt; + } + + len = hfs_brec_lenoff(bnode, fd->record, &off); + keylen = hfs_brec_keylen(bnode, fd->record); + if (hfs_brec_len_invalid(bnode, len) || + hfs_brec_len_invalid(bnode, keylen)) { + res = -EINVAL; + goto out; + } + fd->keyoffset = off; + fd->keylength = keylen; + fd->entryoffset = off + keylen; + fd->entrylength = len - keylen; + hfs_bnode_read(bnode, fd->key, off, keylen); +out: + fd->bnode = bnode; + return res; +} + +/** + * hfsplus_brec_read_cat - read and validate a catalog record + * @fd: find data structure + * @entry: pointer to catalog entry to read into + * + * Reads a catalog record and validates its size matches the expected + * size based on the record type. + * + * Returns 0 on success, or negative error code on failure. + */ +int hfsplus_brec_read_cat(struct hfs_find_data *fd, hfsplus_cat_entry *entry) +{ + int res; + u32 expected_size; + + res = hfs_brec_read(fd, entry, sizeof(hfsplus_cat_entry)); + if (res) + return res; + + /* Validate catalog record size based on type */ + switch (be16_to_cpu(entry->type)) { + case HFSPLUS_FOLDER: + expected_size = sizeof(struct hfsplus_cat_folder); + break; + case HFSPLUS_FILE: + expected_size = sizeof(struct hfsplus_cat_file); + break; + case HFSPLUS_FOLDER_THREAD: + case HFSPLUS_FILE_THREAD: + /* Ensure we have at least the fixed fields before reading nodeName.length */ + if (fd->entrylength < HFSPLUS_MIN_THREAD_SZ) { + pr_err("thread record too short (got %u)\n", fd->entrylength); + return -EIO; + } + expected_size = hfsplus_cat_thread_size(&entry->thread); + break; + default: + pr_err("unknown catalog record type %d\n", + be16_to_cpu(entry->type)); + return -EIO; + } + + if (fd->entrylength != expected_size) { + pr_err("catalog record size mismatch (type %d, got %u, expected %u)\n", + be16_to_cpu(entry->type), fd->entrylength, expected_size); + return -EIO; + } + + return 0; +} diff --git a/fs/hfsplus/bitmap.c b/fs/hfsplus/bitmap.c new file mode 100644 index 000000000..1b3af8c87 --- /dev/null +++ b/fs/hfsplus/bitmap.c @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hfsplus/bitmap.c + * + * Copyright (C) 2001 + * Brad Boyer (flar@allandria.com) + * (C) 2003 Ardis Technologies + * + * Handling of allocation file + */ + +#include + +#include "hfsplus_fs.h" +#include "hfsplus_raw.h" + +#define PAGE_CACHE_BITS (PAGE_SIZE * 8) + +int hfsplus_block_allocate(struct super_block *sb, u32 size, + u32 offset, u32 *max) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(sb); + struct page *page; + struct address_space *mapping; + __be32 *pptr, *curr, *end; + u32 mask, start, len, n; + __be32 val; + int i; + + len = *max; + if (!len) + return size; + + hfs_dbg("size %u, offset %u, len %u\n", size, offset, len); + mutex_lock(&sbi->alloc_mutex); + mapping = sbi->alloc_file->i_mapping; + page = read_mapping_page(mapping, offset / PAGE_CACHE_BITS, NULL); + if (IS_ERR(page)) { + start = size; + goto out; + } + pptr = kmap_local_page(page); + curr = pptr + (offset & (PAGE_CACHE_BITS - 1)) / 32; + i = offset % 32; + offset &= ~(PAGE_CACHE_BITS - 1); + if ((size ^ offset) / PAGE_CACHE_BITS) + end = pptr + PAGE_CACHE_BITS / 32; + else + end = pptr + ((size + 31) & (PAGE_CACHE_BITS - 1)) / 32; + + /* scan the first partial u32 for zero bits */ + val = *curr; + if (~val) { + n = be32_to_cpu(val); + mask = (1U << 31) >> i; + for (; i < 32; mask >>= 1, i++) { + if (!(n & mask)) + goto found; + } + } + curr++; + + /* scan complete u32s for the first zero bit */ + while (1) { + while (curr < end) { + val = *curr; + if (~val) { + n = be32_to_cpu(val); + mask = 1 << 31; + for (i = 0; i < 32; mask >>= 1, i++) { + if (!(n & mask)) + goto found; + } + } + curr++; + } + kunmap_local(pptr); + offset += PAGE_CACHE_BITS; + if (offset >= size) + break; + page = read_mapping_page(mapping, offset / PAGE_CACHE_BITS, + NULL); + if (IS_ERR(page)) { + start = size; + goto out; + } + curr = pptr = kmap_local_page(page); + if ((size ^ offset) / PAGE_CACHE_BITS) + end = pptr + PAGE_CACHE_BITS / 32; + else + end = pptr + ((size + 31) & (PAGE_CACHE_BITS - 1)) / 32; + } + hfs_dbg("bitmap full\n"); + start = size; + goto out; + +found: + start = offset + (curr - pptr) * 32 + i; + if (start >= size) { + hfs_dbg("bitmap full\n"); + goto out; + } + /* do any partial u32 at the start */ + len = min(size - start, len); + while (1) { + n |= mask; + if (++i >= 32) + break; + mask >>= 1; + if (!--len || n & mask) + goto done; + } + if (!--len) + goto done; + *curr++ = cpu_to_be32(n); + /* do full u32s */ + while (1) { + while (curr < end) { + n = be32_to_cpu(*curr); + if (len < 32) + goto last; + if (n) { + len = 32; + goto last; + } + *curr++ = cpu_to_be32(0xffffffff); + len -= 32; + } + set_page_dirty(page); + kunmap_local(pptr); + offset += PAGE_CACHE_BITS; + page = read_mapping_page(mapping, offset / PAGE_CACHE_BITS, + NULL); + if (IS_ERR(page)) { + start = size; + goto out; + } + pptr = kmap_local_page(page); + curr = pptr; + end = pptr + PAGE_CACHE_BITS / 32; + } +last: + /* do any partial u32 at end */ + mask = 1U << 31; + for (i = 0; i < len; i++) { + if (n & mask) + break; + n |= mask; + mask >>= 1; + } +done: + *curr = cpu_to_be32(n); + set_page_dirty(page); + kunmap_local(pptr); + *max = offset + (curr - pptr) * 32 + i - start; + sbi->free_blocks -= *max; + hfsplus_mark_mdb_dirty(sb); + hfs_dbg("start %u, max %u\n", start, *max); +out: + mutex_unlock(&sbi->alloc_mutex); + return start; +} + +int hfsplus_block_free(struct super_block *sb, u32 offset, u32 count) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(sb); + struct page *page; + struct address_space *mapping; + __be32 *pptr, *curr, *end; + u32 mask, len, pnr; + int i; + + /* is there any actual work to be done? */ + if (!count) + return 0; + + hfs_dbg("offset %u, count %u\n", offset, count); + /* are all of the bits in range? */ + if ((offset + count) > sbi->total_blocks) + return -ENOENT; + + mutex_lock(&sbi->alloc_mutex); + mapping = sbi->alloc_file->i_mapping; + pnr = offset / PAGE_CACHE_BITS; + page = read_mapping_page(mapping, pnr, NULL); + if (IS_ERR(page)) + goto kaboom; + pptr = kmap_local_page(page); + curr = pptr + (offset & (PAGE_CACHE_BITS - 1)) / 32; + end = pptr + PAGE_CACHE_BITS / 32; + len = count; + + /* do any partial u32 at the start */ + i = offset % 32; + if (i) { + int j = 32 - i; + mask = 0xffffffffU << j; + if (j > count) { + mask |= 0xffffffffU >> (i + count); + *curr++ &= cpu_to_be32(mask); + goto out; + } + *curr++ &= cpu_to_be32(mask); + count -= j; + } + + /* do full u32s */ + while (1) { + while (curr < end) { + if (count < 32) + goto done; + *curr++ = 0; + count -= 32; + } + if (!count) + break; + set_page_dirty(page); + kunmap_local(pptr); + page = read_mapping_page(mapping, ++pnr, NULL); + if (IS_ERR(page)) + goto kaboom; + pptr = kmap_local_page(page); + curr = pptr; + end = pptr + PAGE_CACHE_BITS / 32; + } +done: + /* do any partial u32 at end */ + if (count) { + mask = 0xffffffffU >> count; + *curr &= cpu_to_be32(mask); + } +out: + set_page_dirty(page); + kunmap_local(pptr); + sbi->free_blocks += len; + hfsplus_mark_mdb_dirty(sb); + mutex_unlock(&sbi->alloc_mutex); + + return 0; + +kaboom: + pr_crit("unable to mark blocks free: error %ld\n", PTR_ERR(page)); + mutex_unlock(&sbi->alloc_mutex); + + return -EIO; +} diff --git a/fs/hfsplus/bnode.c b/fs/hfsplus/bnode.c new file mode 100644 index 000000000..3a1a1fa6f --- /dev/null +++ b/fs/hfsplus/bnode.c @@ -0,0 +1,720 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hfsplus/bnode.c + * + * Copyright (C) 2001 + * Brad Boyer (flar@allandria.com) + * (C) 2003 Ardis Technologies + * + * Handle basic btree node operations + */ + +#include +#include +#include +#include +#include + +#include "hfsplus_fs.h" +#include "hfsplus_raw.h" + + +/* Copy a specified range of bytes from the raw data of a node */ +void hfs_bnode_read(struct hfs_bnode *node, void *buf, u32 off, u32 len) +{ + struct page **pagep; + u32 l; + + memset(buf, 0, len); + + if (!is_bnode_offset_valid(node, off)) + return; + + if (len == 0) { + pr_err("requested zero length: " + "NODE: id %u, type %#x, height %u, " + "node_size %u, offset %u, len %u\n", + node->this, node->type, node->height, + node->tree->node_size, off, len); + return; + } + + len = check_and_correct_requested_length(node, off, len); + + off += node->page_offset; + pagep = node->page + (off >> PAGE_SHIFT); + off &= ~PAGE_MASK; + + l = min_t(u32, len, PAGE_SIZE - off); + memcpy_from_page(buf, *pagep, off, l); + + while ((len -= l) != 0) { + buf += l; + l = min_t(u32, len, PAGE_SIZE); + memcpy_from_page(buf, *++pagep, 0, l); + } +} + +u16 hfs_bnode_read_u16(struct hfs_bnode *node, u32 off) +{ + __be16 data; + /* TODO: optimize later... */ + hfs_bnode_read(node, &data, off, 2); + return be16_to_cpu(data); +} + +u8 hfs_bnode_read_u8(struct hfs_bnode *node, u32 off) +{ + u8 data; + /* TODO: optimize later... */ + hfs_bnode_read(node, &data, off, 1); + return data; +} + +void hfs_bnode_read_key(struct hfs_bnode *node, void *key, u32 off) +{ + struct hfs_btree *tree; + u32 key_len; + + tree = node->tree; + if (node->type == HFS_NODE_LEAF || + tree->attributes & HFS_TREE_VARIDXKEYS || + node->tree->cnid == HFSPLUS_ATTR_CNID) + key_len = hfs_bnode_read_u16(node, off) + 2; + else + key_len = tree->max_key_len + 2; + + if (key_len > sizeof(hfsplus_btree_key) || key_len < 1) { + memset(key, 0, sizeof(hfsplus_btree_key)); + pr_err("hfsplus: Invalid key length: %u\n", key_len); + return; + } + + hfs_bnode_read(node, key, off, key_len); +} + +void hfs_bnode_write(struct hfs_bnode *node, void *buf, u32 off, u32 len) +{ + struct page **pagep; + u32 l; + + if (!is_bnode_offset_valid(node, off)) + return; + + if (len == 0) { + pr_err("requested zero length: " + "NODE: id %u, type %#x, height %u, " + "node_size %u, offset %u, len %u\n", + node->this, node->type, node->height, + node->tree->node_size, off, len); + return; + } + + len = check_and_correct_requested_length(node, off, len); + + off += node->page_offset; + pagep = node->page + (off >> PAGE_SHIFT); + off &= ~PAGE_MASK; + + l = min_t(u32, len, PAGE_SIZE - off); + memcpy_to_page(*pagep, off, buf, l); + set_page_dirty(*pagep); + + while ((len -= l) != 0) { + buf += l; + l = min_t(u32, len, PAGE_SIZE); + memcpy_to_page(*++pagep, 0, buf, l); + set_page_dirty(*pagep); + } +} + +void hfs_bnode_write_u16(struct hfs_bnode *node, u32 off, u16 data) +{ + __be16 v = cpu_to_be16(data); + /* TODO: optimize later... */ + hfs_bnode_write(node, &v, off, 2); +} + +void hfs_bnode_clear(struct hfs_bnode *node, u32 off, u32 len) +{ + struct page **pagep; + u32 l; + + if (!is_bnode_offset_valid(node, off)) + return; + + if (len == 0) { + pr_err("requested zero length: " + "NODE: id %u, type %#x, height %u, " + "node_size %u, offset %u, len %u\n", + node->this, node->type, node->height, + node->tree->node_size, off, len); + return; + } + + len = check_and_correct_requested_length(node, off, len); + + off += node->page_offset; + pagep = node->page + (off >> PAGE_SHIFT); + off &= ~PAGE_MASK; + + l = min_t(u32, len, PAGE_SIZE - off); + memzero_page(*pagep, off, l); + set_page_dirty(*pagep); + + while ((len -= l) != 0) { + l = min_t(u32, len, PAGE_SIZE); + memzero_page(*++pagep, 0, l); + set_page_dirty(*pagep); + } +} + +void hfs_bnode_copy(struct hfs_bnode *dst_node, u32 dst, + struct hfs_bnode *src_node, u32 src, u32 len) +{ + struct page **src_page, **dst_page; + u32 l; + + hfs_dbg("dst %u, src %u, len %u\n", dst, src, len); + if (!len) + return; + + len = check_and_correct_requested_length(src_node, src, len); + len = check_and_correct_requested_length(dst_node, dst, len); + + src += src_node->page_offset; + dst += dst_node->page_offset; + src_page = src_node->page + (src >> PAGE_SHIFT); + src &= ~PAGE_MASK; + dst_page = dst_node->page + (dst >> PAGE_SHIFT); + dst &= ~PAGE_MASK; + + if (src == dst) { + l = min_t(u32, len, PAGE_SIZE - src); + memcpy_page(*dst_page, src, *src_page, src, l); + set_page_dirty(*dst_page); + + while ((len -= l) != 0) { + l = min_t(u32, len, PAGE_SIZE); + memcpy_page(*++dst_page, 0, *++src_page, 0, l); + set_page_dirty(*dst_page); + } + } else { + void *src_ptr, *dst_ptr; + + do { + dst_ptr = kmap_local_page(*dst_page) + dst; + src_ptr = kmap_local_page(*src_page) + src; + if (PAGE_SIZE - src < PAGE_SIZE - dst) { + l = PAGE_SIZE - src; + src = 0; + dst += l; + } else { + l = PAGE_SIZE - dst; + src += l; + dst = 0; + } + l = min(len, l); + memcpy(dst_ptr, src_ptr, l); + kunmap_local(src_ptr); + set_page_dirty(*dst_page); + kunmap_local(dst_ptr); + if (!dst) + dst_page++; + else + src_page++; + } while ((len -= l)); + } +} + +void hfs_bnode_move(struct hfs_bnode *node, u32 dst, u32 src, u32 len) +{ + struct page **src_page, **dst_page; + void *src_ptr, *dst_ptr; + u32 l; + + hfs_dbg("dst %u, src %u, len %u\n", dst, src, len); + if (!len) + return; + + len = check_and_correct_requested_length(node, src, len); + len = check_and_correct_requested_length(node, dst, len); + + src += node->page_offset; + dst += node->page_offset; + if (dst > src) { + src += len - 1; + src_page = node->page + (src >> PAGE_SHIFT); + src = (src & ~PAGE_MASK) + 1; + dst += len - 1; + dst_page = node->page + (dst >> PAGE_SHIFT); + dst = (dst & ~PAGE_MASK) + 1; + + if (src == dst) { + while (src < len) { + dst_ptr = kmap_local_page(*dst_page); + src_ptr = kmap_local_page(*src_page); + memmove(dst_ptr, src_ptr, src); + kunmap_local(src_ptr); + set_page_dirty(*dst_page); + kunmap_local(dst_ptr); + len -= src; + src = PAGE_SIZE; + src_page--; + dst_page--; + } + src -= len; + dst_ptr = kmap_local_page(*dst_page); + src_ptr = kmap_local_page(*src_page); + memmove(dst_ptr + src, src_ptr + src, len); + kunmap_local(src_ptr); + set_page_dirty(*dst_page); + kunmap_local(dst_ptr); + } else { + do { + dst_ptr = kmap_local_page(*dst_page) + dst; + src_ptr = kmap_local_page(*src_page) + src; + if (src < dst) { + l = src; + src = PAGE_SIZE; + dst -= l; + } else { + l = dst; + src -= l; + dst = PAGE_SIZE; + } + l = min(len, l); + memmove(dst_ptr - l, src_ptr - l, l); + kunmap_local(src_ptr); + set_page_dirty(*dst_page); + kunmap_local(dst_ptr); + if (dst == PAGE_SIZE) + dst_page--; + else + src_page--; + } while ((len -= l)); + } + } else { + src_page = node->page + (src >> PAGE_SHIFT); + src &= ~PAGE_MASK; + dst_page = node->page + (dst >> PAGE_SHIFT); + dst &= ~PAGE_MASK; + + if (src == dst) { + l = min_t(u32, len, PAGE_SIZE - src); + + dst_ptr = kmap_local_page(*dst_page) + src; + src_ptr = kmap_local_page(*src_page) + src; + memmove(dst_ptr, src_ptr, l); + kunmap_local(src_ptr); + set_page_dirty(*dst_page); + kunmap_local(dst_ptr); + + while ((len -= l) != 0) { + l = min_t(u32, len, PAGE_SIZE); + dst_ptr = kmap_local_page(*++dst_page); + src_ptr = kmap_local_page(*++src_page); + memmove(dst_ptr, src_ptr, l); + kunmap_local(src_ptr); + set_page_dirty(*dst_page); + kunmap_local(dst_ptr); + } + } else { + do { + dst_ptr = kmap_local_page(*dst_page) + dst; + src_ptr = kmap_local_page(*src_page) + src; + if (PAGE_SIZE - src < + PAGE_SIZE - dst) { + l = PAGE_SIZE - src; + src = 0; + dst += l; + } else { + l = PAGE_SIZE - dst; + src += l; + dst = 0; + } + l = min(len, l); + memmove(dst_ptr, src_ptr, l); + kunmap_local(src_ptr); + set_page_dirty(*dst_page); + kunmap_local(dst_ptr); + if (!dst) + dst_page++; + else + src_page++; + } while ((len -= l)); + } + } +} + +void hfs_bnode_dump(struct hfs_bnode *node) +{ + struct hfs_bnode_desc desc; + __be32 cnid; + int i, off, key_off; + u16 num_recs; + + hfs_dbg("node %d\n", node->this); + hfs_bnode_read(node, &desc, 0, sizeof(desc)); + num_recs = node->num_recs; + hfs_dbg("next %d, prev %d, type %d, height %d, num_recs %d\n", + be32_to_cpu(desc.next), be32_to_cpu(desc.prev), + desc.type, desc.height, be16_to_cpu(desc.num_recs)); + + if (hfs_bnode_num_recs_invalid(node)) { + hfs_dbg("invalid num_recs %u\n", num_recs); + return; + } + + off = node->tree->node_size - 2; + for (i = num_recs; i >= 0; off -= 2, i--) { + key_off = hfs_bnode_read_u16(node, off); + hfs_dbg(" key_off %d", key_off); + if (i && node->type == HFS_NODE_INDEX) { + int tmp; + + if (node->tree->attributes & HFS_TREE_VARIDXKEYS || + node->tree->cnid == HFSPLUS_ATTR_CNID) + tmp = hfs_bnode_read_u16(node, key_off) + 2; + else + tmp = node->tree->max_key_len + 2; + hfs_dbg(" (%d", tmp); + hfs_bnode_read(node, &cnid, key_off + tmp, 4); + hfs_dbg(", cnid %d)", be32_to_cpu(cnid)); + } else if (i && node->type == HFS_NODE_LEAF) { + int tmp; + + tmp = hfs_bnode_read_u16(node, key_off); + hfs_dbg(" (%d)", tmp); + } + } + hfs_dbg("\n"); +} + +void hfs_bnode_unlink(struct hfs_bnode *node) +{ + struct hfs_btree *tree; + struct hfs_bnode *tmp; + __be32 cnid; + + tree = node->tree; + if (node->prev) { + tmp = hfs_bnode_find(tree, node->prev); + if (IS_ERR(tmp)) + return; + tmp->next = node->next; + cnid = cpu_to_be32(tmp->next); + hfs_bnode_write(tmp, &cnid, + offsetof(struct hfs_bnode_desc, next), 4); + hfs_bnode_put(tmp); + } else if (node->type == HFS_NODE_LEAF) + tree->leaf_head = node->next; + + if (node->next) { + tmp = hfs_bnode_find(tree, node->next); + if (IS_ERR(tmp)) + return; + tmp->prev = node->prev; + cnid = cpu_to_be32(tmp->prev); + hfs_bnode_write(tmp, &cnid, + offsetof(struct hfs_bnode_desc, prev), 4); + hfs_bnode_put(tmp); + } else if (node->type == HFS_NODE_LEAF) + tree->leaf_tail = node->prev; + + /* move down? */ + if (!node->prev && !node->next) + hfs_dbg("btree delete level\n"); + if (!node->parent) { + tree->root = 0; + tree->depth = 0; + } + + spin_lock(&tree->hash_lock); + set_bit(HFS_BNODE_DELETED, &node->flags); + spin_unlock(&tree->hash_lock); +} + +static inline int hfs_bnode_hash(u32 num) +{ + num = (num >> 16) + num; + num += num >> 8; + return num & (NODE_HASH_SIZE - 1); +} + +struct hfs_bnode *hfs_bnode_findhash(struct hfs_btree *tree, u32 cnid) +{ + struct hfs_bnode *node; + + if (cnid >= tree->node_count) { + pr_err("request for non-existent node %d in B*Tree\n", + cnid); + return NULL; + } + + for (node = tree->node_hash[hfs_bnode_hash(cnid)]; + node; node = node->next_hash) + if (node->this == cnid) + return node; + return NULL; +} + +static struct hfs_bnode *__hfs_bnode_create(struct hfs_btree *tree, u32 cnid) +{ + struct hfs_bnode *node, *node2; + struct address_space *mapping; + struct page *page; + int block, i, hash; + loff_t off; + + if (cnid >= tree->node_count) { + pr_err("request for non-existent node %d in B*Tree\n", + cnid); + return NULL; + } + + node = kzalloc_flex(*node, page, tree->pages_per_bnode); + if (!node) + return NULL; + node->tree = tree; + node->this = cnid; + set_bit(HFS_BNODE_NEW, &node->flags); + atomic_set(&node->refcnt, 1); + hfs_dbg("cnid %d, node %d, refcnt 1\n", + node->tree->cnid, node->this); + init_waitqueue_head(&node->lock_wq); + spin_lock(&tree->hash_lock); + node2 = hfs_bnode_findhash(tree, cnid); + if (!node2) { + hash = hfs_bnode_hash(cnid); + node->next_hash = tree->node_hash[hash]; + tree->node_hash[hash] = node; + tree->node_hash_cnt++; + } else { + hfs_bnode_get(node2); + spin_unlock(&tree->hash_lock); + kfree(node); + wait_event(node2->lock_wq, + !test_bit(HFS_BNODE_NEW, &node2->flags)); + return node2; + } + spin_unlock(&tree->hash_lock); + + mapping = tree->inode->i_mapping; + off = (loff_t)cnid << tree->node_size_shift; + block = off >> PAGE_SHIFT; + node->page_offset = off & ~PAGE_MASK; + for (i = 0; i < tree->pages_per_bnode; block++, i++) { + page = read_mapping_page(mapping, block, NULL); + if (IS_ERR(page)) + goto fail; + node->page[i] = page; + } + + return node; +fail: + set_bit(HFS_BNODE_ERROR, &node->flags); + return node; +} + +void hfs_bnode_unhash(struct hfs_bnode *node) +{ + struct hfs_bnode **p; + + hfs_dbg("cnid %d, node %d, refcnt %d\n", + node->tree->cnid, node->this, atomic_read(&node->refcnt)); + for (p = &node->tree->node_hash[hfs_bnode_hash(node->this)]; + *p && *p != node; p = &(*p)->next_hash) + ; + BUG_ON(!*p); + *p = node->next_hash; + node->tree->node_hash_cnt--; +} + +/* Load a particular node out of a tree */ +struct hfs_bnode *hfs_bnode_find(struct hfs_btree *tree, u32 num) +{ + struct hfs_bnode *node; + struct hfs_bnode_desc *desc; + int i, rec_off, off, next_off; + int entry_size, key_size; + + spin_lock(&tree->hash_lock); + node = hfs_bnode_findhash(tree, num); + if (node) { + hfs_bnode_get(node); + spin_unlock(&tree->hash_lock); + wait_event(node->lock_wq, + !test_bit(HFS_BNODE_NEW, &node->flags)); + if (test_bit(HFS_BNODE_ERROR, &node->flags)) + goto node_error; + return node; + } + spin_unlock(&tree->hash_lock); + node = __hfs_bnode_create(tree, num); + if (!node) + return ERR_PTR(-ENOMEM); + if (test_bit(HFS_BNODE_ERROR, &node->flags)) + goto node_error; + if (!test_bit(HFS_BNODE_NEW, &node->flags)) + return node; + + desc = (struct hfs_bnode_desc *)(kmap_local_page(node->page[0]) + + node->page_offset); + node->prev = be32_to_cpu(desc->prev); + node->next = be32_to_cpu(desc->next); + node->num_recs = be16_to_cpu(desc->num_recs); + node->type = desc->type; + node->height = desc->height; + kunmap_local(desc); + + if (hfs_bnode_num_recs_invalid(node)) + goto node_error; + + switch (node->type) { + case HFS_NODE_HEADER: + case HFS_NODE_MAP: + if (node->height != 0) + goto node_error; + break; + case HFS_NODE_LEAF: + if (node->height != 1) + goto node_error; + break; + case HFS_NODE_INDEX: + if (node->height <= 1 || node->height > tree->depth) + goto node_error; + break; + default: + goto node_error; + } + + rec_off = tree->node_size - 2; + off = hfs_bnode_read_u16(node, rec_off); + if (off != sizeof(struct hfs_bnode_desc)) + goto node_error; + for (i = 1; i <= node->num_recs; off = next_off, i++) { + rec_off -= 2; + next_off = hfs_bnode_read_u16(node, rec_off); + if (hfs_brec_offsets_invalid(node, off, next_off)) + goto node_error; + entry_size = next_off - off; + if (node->type != HFS_NODE_INDEX && + node->type != HFS_NODE_LEAF) + continue; + key_size = hfs_bnode_read_u16(node, off) + 2; + if (key_size >= entry_size || key_size & 1) + goto node_error; + } + clear_bit(HFS_BNODE_NEW, &node->flags); + wake_up(&node->lock_wq); + return node; + +node_error: + set_bit(HFS_BNODE_ERROR, &node->flags); + clear_bit(HFS_BNODE_NEW, &node->flags); + wake_up(&node->lock_wq); + hfs_bnode_put(node); + return ERR_PTR(-EIO); +} + +void hfs_bnode_free(struct hfs_bnode *node) +{ + int i; + + for (i = 0; i < node->tree->pages_per_bnode; i++) + if (node->page[i]) + put_page(node->page[i]); + kfree(node); +} + +struct hfs_bnode *hfs_bnode_create(struct hfs_btree *tree, u32 num) +{ + struct hfs_bnode *node; + struct page **pagep; + int i; + + spin_lock(&tree->hash_lock); + node = hfs_bnode_findhash(tree, num); + spin_unlock(&tree->hash_lock); + if (node) { + pr_crit("new node %u already hashed?\n", num); + WARN_ON(1); + return ERR_PTR(-EEXIST); + } + node = __hfs_bnode_create(tree, num); + if (!node) + return ERR_PTR(-ENOMEM); + if (test_bit(HFS_BNODE_ERROR, &node->flags)) { + hfs_bnode_put(node); + return ERR_PTR(-EIO); + } + + pagep = node->page; + memzero_page(*pagep, node->page_offset, + min_t(int, PAGE_SIZE, tree->node_size)); + set_page_dirty(*pagep); + for (i = 1; i < tree->pages_per_bnode; i++) { + memzero_page(*++pagep, 0, PAGE_SIZE); + set_page_dirty(*pagep); + } + clear_bit(HFS_BNODE_NEW, &node->flags); + wake_up(&node->lock_wq); + + return node; +} + +void hfs_bnode_get(struct hfs_bnode *node) +{ + if (node) { + atomic_inc(&node->refcnt); + hfs_dbg("cnid %d, node %d, refcnt %d\n", + node->tree->cnid, node->this, + atomic_read(&node->refcnt)); + } +} + +/* Dispose of resources used by a node */ +void hfs_bnode_put(struct hfs_bnode *node) +{ + if (node) { + struct hfs_btree *tree = node->tree; + int i; + + hfs_dbg("cnid %d, node %d, refcnt %d\n", + node->tree->cnid, node->this, + atomic_read(&node->refcnt)); + BUG_ON(!atomic_read(&node->refcnt)); + if (!atomic_dec_and_lock(&node->refcnt, &tree->hash_lock)) + return; + for (i = 0; i < tree->pages_per_bnode; i++) { + if (!node->page[i]) + continue; + mark_page_accessed(node->page[i]); + } + + if (test_bit(HFS_BNODE_DELETED, &node->flags)) { + hfs_bnode_unhash(node); + spin_unlock(&tree->hash_lock); + if (hfs_bnode_need_zeroout(tree)) + hfs_bnode_clear(node, 0, tree->node_size); + hfs_bmap_free(node); + hfs_bnode_free(node); + return; + } + spin_unlock(&tree->hash_lock); + } +} + +/* + * Unused nodes have to be zeroed if this is the catalog tree and + * a corresponding flag in the volume header is set. + */ +bool hfs_bnode_need_zeroout(struct hfs_btree *tree) +{ + struct super_block *sb = tree->inode->i_sb; + struct hfsplus_sb_info *sbi = HFSPLUS_SB(sb); + const u32 volume_attr = be32_to_cpu(sbi->s_vhdr->attributes); + + return volume_attr & HFSPLUS_VOL_UNUSED_NODE_FIX; +} diff --git a/fs/hfsplus/brec.c b/fs/hfsplus/brec.c new file mode 100644 index 000000000..f416e562d --- /dev/null +++ b/fs/hfsplus/brec.c @@ -0,0 +1,563 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hfsplus/brec.c + * + * Copyright (C) 2001 + * Brad Boyer (flar@allandria.com) + * (C) 2003 Ardis Technologies + * + * Handle individual btree records + */ + +#include + +#include "hfsplus_fs.h" +#include "hfsplus_raw.h" + +static struct hfs_bnode *hfs_bnode_split(struct hfs_find_data *fd); +static int hfs_brec_update_parent(struct hfs_find_data *fd); +static int hfs_btree_inc_height(struct hfs_btree *); + +/* Get the length and offset of the given record in the given node */ +u16 hfs_brec_lenoff(struct hfs_bnode *node, u16 rec, u16 *off) +{ + __be16 retval[2]; + u16 data_off; + u16 next_off; + + if (hfs_brec_record_invalid(node, rec)) { + *off = U16_MAX; + return U16_MAX; + } + + data_off = node->tree->node_size - (rec + 2) * 2; + hfs_bnode_read(node, retval, data_off, 4); + *off = be16_to_cpu(retval[1]); + next_off = be16_to_cpu(retval[0]); + if (hfs_brec_offsets_invalid(node, *off, next_off)) { + *off = U16_MAX; + return U16_MAX; + } + return next_off - *off; +} + +/* Get the length of the key from a keyed record */ +u16 hfs_brec_keylen(struct hfs_bnode *node, u16 rec) +{ + u16 retval, recoff, len; + + if (node->type != HFS_NODE_INDEX && node->type != HFS_NODE_LEAF) + return 0; + if (hfs_brec_record_invalid(node, rec)) + return U16_MAX; + + if ((node->type == HFS_NODE_INDEX) && + !(node->tree->attributes & HFS_TREE_VARIDXKEYS) && + (node->tree->cnid != HFSPLUS_ATTR_CNID)) { + retval = node->tree->max_key_len + 2; + } else { + len = hfs_brec_lenoff(node, rec, &recoff); + if (hfs_brec_len_invalid(node, len)) + return len; + + retval = hfs_bnode_read_u16(node, recoff) + 2; + if (retval > node->tree->max_key_len + 2) { + pr_err("keylen %d too large\n", + retval); + retval = U16_MAX; + } + } + return retval; +} + +int hfs_brec_insert(struct hfs_find_data *fd, void *entry, u32 entry_len) +{ + struct hfs_btree *tree; + struct hfs_bnode *node, *new_node; + int size, key_len, rec; + int data_off, end_off; + int idx_rec_off, data_rec_off, end_rec_off; + __be32 cnid; + + tree = fd->tree; + if (!fd->bnode) { + if (!tree->root) + hfs_btree_inc_height(tree); + node = hfs_bnode_find(tree, tree->leaf_head); + if (IS_ERR(node)) + return PTR_ERR(node); + fd->bnode = node; + fd->record = -1; + } + new_node = NULL; + key_len = be16_to_cpu(fd->search_key->key_len) + 2; +again: + /* new record idx and complete record size */ + rec = fd->record + 1; + size = key_len + entry_len; + + node = fd->bnode; + hfs_bnode_dump(node); + /* get last offset */ + end_rec_off = tree->node_size - (node->num_recs + 1) * 2; + end_off = hfs_bnode_read_u16(node, end_rec_off); + end_rec_off -= 2; + hfs_dbg("rec %d, size %d, end_off %d, end_rec_off %d\n", + rec, size, end_off, end_rec_off); + if (size > end_rec_off - end_off) { + if (new_node) + panic("not enough room!\n"); + new_node = hfs_bnode_split(fd); + if (IS_ERR(new_node)) + return PTR_ERR(new_node); + goto again; + } + if (node->type == HFS_NODE_LEAF) { + tree->leaf_count++; + mark_inode_dirty(tree->inode); + } + node->num_recs++; + /* write new last offset */ + hfs_bnode_write_u16(node, + offsetof(struct hfs_bnode_desc, num_recs), + node->num_recs); + hfs_bnode_write_u16(node, end_rec_off, end_off + size); + data_off = end_off; + data_rec_off = end_rec_off + 2; + idx_rec_off = tree->node_size - (rec + 1) * 2; + if (idx_rec_off == data_rec_off) + goto skip; + /* move all following entries */ + do { + data_off = hfs_bnode_read_u16(node, data_rec_off + 2); + hfs_bnode_write_u16(node, data_rec_off, data_off + size); + data_rec_off += 2; + } while (data_rec_off < idx_rec_off); + + /* move data away */ + hfs_bnode_move(node, data_off + size, data_off, + end_off - data_off); + +skip: + hfs_bnode_write(node, fd->search_key, data_off, key_len); + hfs_bnode_write(node, entry, data_off + key_len, entry_len); + hfs_bnode_dump(node); + + /* + * update parent key if we inserted a key + * at the start of the node and it is not the new node + */ + if (!rec && new_node != node) { + hfs_bnode_read_key(node, fd->search_key, data_off + size); + hfs_brec_update_parent(fd); + } + + if (new_node) { + hfs_bnode_put(fd->bnode); + if (!new_node->parent) { + hfs_btree_inc_height(tree); + new_node->parent = tree->root; + } + fd->bnode = hfs_bnode_find(tree, new_node->parent); + + /* create index data entry */ + cnid = cpu_to_be32(new_node->this); + entry = &cnid; + entry_len = sizeof(cnid); + + /* get index key */ + hfs_bnode_read_key(new_node, fd->search_key, 14); + __hfs_brec_find(fd->bnode, fd, hfs_find_rec_by_key); + + hfs_bnode_put(new_node); + new_node = NULL; + + if ((tree->attributes & HFS_TREE_VARIDXKEYS) || + (tree->cnid == HFSPLUS_ATTR_CNID)) + key_len = be16_to_cpu(fd->search_key->key_len) + 2; + else { + fd->search_key->key_len = + cpu_to_be16(tree->max_key_len); + key_len = tree->max_key_len + 2; + } + goto again; + } + + return 0; +} + +int hfs_brec_remove(struct hfs_find_data *fd) +{ + struct hfs_btree *tree; + struct hfs_bnode *node, *parent; + int end_off, rec_off, data_off, size; + int res; + + tree = fd->tree; + node = fd->bnode; +again: + if (hfs_brec_record_invalid(node, fd->record)) + return -EINVAL; + + rec_off = tree->node_size - (fd->record + 2) * 2; + end_off = tree->node_size - (node->num_recs + 1) * 2; + + if (node->type == HFS_NODE_LEAF) { + if (tree->leaf_count == 0) + return -EINVAL; + tree->leaf_count--; + mark_inode_dirty(tree->inode); + } + hfs_bnode_dump(node); + hfs_dbg("rec %d, len %d\n", + fd->record, fd->keylength + fd->entrylength); + if (!--node->num_recs) { + hfs_bnode_unlink(node); + if (!node->parent) + return 0; + parent = hfs_bnode_find(tree, node->parent); + if (IS_ERR(parent)) + return PTR_ERR(parent); + hfs_bnode_put(node); + node = fd->bnode = parent; + + res = __hfs_brec_find(node, fd, hfs_find_rec_by_key); + if (res && res != -ENOENT) + return res; + goto again; + } + hfs_bnode_write_u16(node, + offsetof(struct hfs_bnode_desc, num_recs), + node->num_recs); + + if (rec_off == end_off) + goto skip; + size = fd->keylength + fd->entrylength; + + do { + data_off = hfs_bnode_read_u16(node, rec_off); + hfs_bnode_write_u16(node, rec_off + 2, data_off - size); + rec_off -= 2; + } while (rec_off >= end_off); + + /* fill hole */ + hfs_bnode_move(node, fd->keyoffset, fd->keyoffset + size, + data_off - fd->keyoffset - size); +skip: + hfs_bnode_dump(node); + if (!fd->record) + hfs_brec_update_parent(fd); + return 0; +} + +static struct hfs_bnode *hfs_bnode_split(struct hfs_find_data *fd) +{ + struct hfs_btree *tree; + struct hfs_bnode *node, *new_node, *next_node; + struct hfs_bnode_desc node_desc; + int num_recs, new_rec_off, new_off, old_rec_off; + int data_start, data_end, size; + size_t rec_off_tbl_size; + size_t node_desc_size = sizeof(struct hfs_bnode_desc); + size_t rec_size = sizeof(__be16); + + tree = fd->tree; + node = fd->bnode; + new_node = hfs_bmap_alloc(tree); + if (IS_ERR(new_node)) + return new_node; + hfs_bnode_get(node); + hfs_dbg("this %d - new %d - next %d\n", + node->this, new_node->this, node->next); + new_node->next = node->next; + new_node->prev = node->this; + new_node->parent = node->parent; + new_node->type = node->type; + new_node->height = node->height; + + if (node->next) + next_node = hfs_bnode_find(tree, node->next); + else + next_node = NULL; + + if (IS_ERR(next_node)) { + hfs_bnode_put(node); + hfs_bnode_put(new_node); + return next_node; + } + + rec_off_tbl_size = node->num_recs * rec_size; + size = tree->node_size / 2; + size -= node_desc_size; + size -= rec_off_tbl_size; + old_rec_off = tree->node_size - (2 * rec_size); + + num_recs = 1; + for (;;) { + data_start = hfs_bnode_read_u16(node, old_rec_off); + if (data_start > size) + break; + old_rec_off -= rec_size; + if (++num_recs < node->num_recs) + continue; + hfs_bnode_put(node); + hfs_bnode_unlink(new_node); + hfs_bnode_put(new_node); + if (next_node) + hfs_bnode_put(next_node); + return ERR_PTR(-ENOSPC); + } + + if (fd->record + 1 < num_recs) { + /* new record is in the lower half, + * so leave some more space there + */ + old_rec_off += rec_size; + num_recs--; + data_start = hfs_bnode_read_u16(node, old_rec_off); + } else { + hfs_bnode_put(node); + hfs_bnode_get(new_node); + fd->bnode = new_node; + fd->record -= num_recs; + fd->keyoffset -= data_start - node_desc_size; + fd->entryoffset -= data_start - node_desc_size; + } + new_node->num_recs = node->num_recs - num_recs; + node->num_recs = num_recs; + + new_rec_off = tree->node_size - rec_size; + new_off = node_desc_size; + size = data_start - new_off; + num_recs = new_node->num_recs; + data_end = data_start; + while (num_recs) { + hfs_bnode_write_u16(new_node, new_rec_off, new_off); + old_rec_off -= rec_size; + new_rec_off -= rec_size; + data_end = hfs_bnode_read_u16(node, old_rec_off); + new_off = data_end - size; + num_recs--; + } + hfs_bnode_write_u16(new_node, new_rec_off, new_off); + hfs_bnode_copy(new_node, node_desc_size, + node, data_start, data_end - data_start); + + /* update new bnode header */ + node_desc.next = cpu_to_be32(new_node->next); + node_desc.prev = cpu_to_be32(new_node->prev); + node_desc.type = new_node->type; + node_desc.height = new_node->height; + node_desc.num_recs = cpu_to_be16(new_node->num_recs); + node_desc.reserved = 0; + hfs_bnode_write(new_node, &node_desc, 0, sizeof(node_desc)); + + /* update previous bnode header */ + node->next = new_node->this; + hfs_bnode_read(node, &node_desc, 0, sizeof(node_desc)); + node_desc.next = cpu_to_be32(node->next); + node_desc.num_recs = cpu_to_be16(node->num_recs); + hfs_bnode_write(node, &node_desc, 0, sizeof(node_desc)); + + /* update next bnode header */ + if (next_node) { + next_node->prev = new_node->this; + hfs_bnode_read(next_node, &node_desc, 0, sizeof(node_desc)); + node_desc.prev = cpu_to_be32(next_node->prev); + hfs_bnode_write(next_node, &node_desc, 0, sizeof(node_desc)); + hfs_bnode_put(next_node); + } else if (node->this == tree->leaf_tail) { + /* if there is no next node, this might be the new tail */ + tree->leaf_tail = new_node->this; + mark_inode_dirty(tree->inode); + } + + hfs_bnode_dump(node); + hfs_bnode_dump(new_node); + hfs_bnode_put(node); + + return new_node; +} + +static int hfs_brec_update_parent(struct hfs_find_data *fd) +{ + struct hfs_btree *tree; + struct hfs_bnode *node, *new_node, *parent; + int newkeylen, diff; + int rec, rec_off, end_rec_off; + int start_off, end_off; + int res; + + tree = fd->tree; + node = fd->bnode; + new_node = NULL; + if (!node->parent) + return 0; + +again: + parent = hfs_bnode_find(tree, node->parent); + if (IS_ERR(parent)) + return PTR_ERR(parent); + res = __hfs_brec_find(parent, fd, hfs_find_rec_by_key); + if (res && res != -ENOENT) + return res; + if (fd->record < 0) + return -ENOENT; + hfs_bnode_dump(parent); + rec = fd->record; + + /* size difference between old and new key */ + if ((tree->attributes & HFS_TREE_VARIDXKEYS) || + (tree->cnid == HFSPLUS_ATTR_CNID)) + newkeylen = hfs_bnode_read_u16(node, 14) + 2; + else + fd->keylength = newkeylen = tree->max_key_len + 2; + hfs_dbg("rec %d, keylength %d, newkeylen %d\n", + rec, fd->keylength, newkeylen); + + rec_off = tree->node_size - (rec + 2) * 2; + end_rec_off = tree->node_size - (parent->num_recs + 1) * 2; + diff = newkeylen - fd->keylength; + if (!diff) + goto skip; + if (diff > 0) { + end_off = hfs_bnode_read_u16(parent, end_rec_off); + if (end_rec_off - end_off < diff) { + + hfs_dbg("splitting index node\n"); + fd->bnode = parent; + new_node = hfs_bnode_split(fd); + if (IS_ERR(new_node)) + return PTR_ERR(new_node); + parent = fd->bnode; + rec = fd->record; + rec_off = tree->node_size - (rec + 2) * 2; + end_rec_off = tree->node_size - + (parent->num_recs + 1) * 2; + } + } + + end_off = start_off = hfs_bnode_read_u16(parent, rec_off); + hfs_bnode_write_u16(parent, rec_off, start_off + diff); + start_off -= 4; /* move previous cnid too */ + + while (rec_off > end_rec_off) { + rec_off -= 2; + end_off = hfs_bnode_read_u16(parent, rec_off); + hfs_bnode_write_u16(parent, rec_off, end_off + diff); + } + hfs_bnode_move(parent, start_off + diff, start_off, + end_off - start_off); +skip: + hfs_bnode_copy(parent, fd->keyoffset, node, 14, newkeylen); + hfs_bnode_dump(parent); + + hfs_bnode_put(node); + node = parent; + + if (new_node) { + __be32 cnid; + + if (!new_node->parent) { + hfs_btree_inc_height(tree); + new_node->parent = tree->root; + } + fd->bnode = hfs_bnode_find(tree, new_node->parent); + /* create index key and entry */ + hfs_bnode_read_key(new_node, fd->search_key, 14); + cnid = cpu_to_be32(new_node->this); + + __hfs_brec_find(fd->bnode, fd, hfs_find_rec_by_key); + hfs_brec_insert(fd, &cnid, sizeof(cnid)); + hfs_bnode_put(fd->bnode); + hfs_bnode_put(new_node); + + if (!rec) { + if (new_node == node) + goto out; + /* restore search_key */ + hfs_bnode_read_key(node, fd->search_key, 14); + } + new_node = NULL; + } + + if (!rec && node->parent) + goto again; +out: + fd->bnode = node; + return 0; +} + +static int hfs_btree_inc_height(struct hfs_btree *tree) +{ + struct hfs_bnode *node, *new_node; + struct hfs_bnode_desc node_desc; + int key_size, rec; + __be32 cnid; + + node = NULL; + if (tree->root) { + node = hfs_bnode_find(tree, tree->root); + if (IS_ERR(node)) + return PTR_ERR(node); + } + new_node = hfs_bmap_alloc(tree); + if (IS_ERR(new_node)) { + hfs_bnode_put(node); + return PTR_ERR(new_node); + } + + tree->root = new_node->this; + if (!tree->depth) { + tree->leaf_head = tree->leaf_tail = new_node->this; + new_node->type = HFS_NODE_LEAF; + new_node->num_recs = 0; + } else { + new_node->type = HFS_NODE_INDEX; + new_node->num_recs = 1; + } + new_node->parent = 0; + new_node->next = 0; + new_node->prev = 0; + new_node->height = ++tree->depth; + + node_desc.next = cpu_to_be32(new_node->next); + node_desc.prev = cpu_to_be32(new_node->prev); + node_desc.type = new_node->type; + node_desc.height = new_node->height; + node_desc.num_recs = cpu_to_be16(new_node->num_recs); + node_desc.reserved = 0; + hfs_bnode_write(new_node, &node_desc, 0, sizeof(node_desc)); + + rec = tree->node_size - 2; + hfs_bnode_write_u16(new_node, rec, 14); + + if (node) { + /* insert old root idx into new root */ + node->parent = tree->root; + if (node->type == HFS_NODE_LEAF || + tree->attributes & HFS_TREE_VARIDXKEYS || + tree->cnid == HFSPLUS_ATTR_CNID) + key_size = hfs_bnode_read_u16(node, 14) + 2; + else + key_size = tree->max_key_len + 2; + hfs_bnode_copy(new_node, 14, node, 14, key_size); + + if (!(tree->attributes & HFS_TREE_VARIDXKEYS) && + (tree->cnid != HFSPLUS_ATTR_CNID)) { + key_size = tree->max_key_len + 2; + hfs_bnode_write_u16(new_node, 14, tree->max_key_len); + } + cnid = cpu_to_be32(node->this); + hfs_bnode_write(new_node, &cnid, 14 + key_size, 4); + + rec -= 2; + hfs_bnode_write_u16(new_node, rec, 14 + key_size + 4); + + hfs_bnode_put(node); + } + hfs_bnode_put(new_node); + mark_inode_dirty(tree->inode); + + return 0; +} diff --git a/fs/hfsplus/btree.c b/fs/hfsplus/btree.c new file mode 100644 index 000000000..2ea8cd565 --- /dev/null +++ b/fs/hfsplus/btree.c @@ -0,0 +1,679 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hfsplus/btree.c + * + * Copyright (C) 2001 + * Brad Boyer (flar@allandria.com) + * (C) 2003 Ardis Technologies + * + * Handle opening/closing btree + */ + +#include +#include +#include + +#include "hfsplus_fs.h" +#include "hfsplus_raw.h" + +/* + * Initial source code of clump size calculation is gotten + * from http://opensource.apple.com/tarballs/diskdev_cmds/ + */ +#define CLUMP_ENTRIES 15 + +static short clumptbl[CLUMP_ENTRIES * 3] = { +/* + * Volume Attributes Catalog Extents + * Size Clump (MB) Clump (MB) Clump (MB) + */ + /* 1GB */ 4, 4, 4, + /* 2GB */ 6, 6, 4, + /* 4GB */ 8, 8, 4, + /* 8GB */ 11, 11, 5, + /* + * For volumes 16GB and larger, we want to make sure that a full OS + * install won't require fragmentation of the Catalog or Attributes + * B-trees. We do this by making the clump sizes sufficiently large, + * and by leaving a gap after the B-trees for them to grow into. + * + * For SnowLeopard 10A298, a FullNetInstall with all packages selected + * results in: + * Catalog B-tree Header + * nodeSize: 8192 + * totalNodes: 31616 + * freeNodes: 1978 + * (used = 231.55 MB) + * Attributes B-tree Header + * nodeSize: 8192 + * totalNodes: 63232 + * freeNodes: 958 + * (used = 486.52 MB) + * + * We also want Time Machine backup volumes to have a sufficiently + * large clump size to reduce fragmentation. + * + * The series of numbers for Catalog and Attribute form a geometric + * series. For Catalog (16GB to 512GB), each term is 8**(1/5) times + * the previous term. For Attributes (16GB to 512GB), each term is + * 4**(1/5) times the previous term. For 1TB to 16TB, each term is + * 2**(1/5) times the previous term. + */ + /* 16GB */ 64, 32, 5, + /* 32GB */ 84, 49, 6, + /* 64GB */ 111, 74, 7, + /* 128GB */ 147, 111, 8, + /* 256GB */ 194, 169, 9, + /* 512GB */ 256, 256, 11, + /* 1TB */ 294, 294, 14, + /* 2TB */ 338, 338, 16, + /* 4TB */ 388, 388, 20, + /* 8TB */ 446, 446, 25, + /* 16TB */ 512, 512, 32 +}; + +u32 hfsplus_calc_btree_clump_size(u32 block_size, u32 node_size, + u64 sectors, int file_id) +{ + u32 mod = max(node_size, block_size); + u32 clump_size; + int column; + int i; + + /* Figure out which column of the above table to use for this file. */ + switch (file_id) { + case HFSPLUS_ATTR_CNID: + column = 0; + break; + case HFSPLUS_CAT_CNID: + column = 1; + break; + default: + column = 2; + break; + } + + /* + * The default clump size is 0.8% of the volume size. And + * it must also be a multiple of the node and block size. + */ + if (sectors < 0x200000) { + clump_size = sectors << 2; /* 0.8 % */ + if (clump_size < (8 * node_size)) + clump_size = 8 * node_size; + } else { + /* turn exponent into table index... */ + for (i = 0, sectors = sectors >> 22; + sectors && (i < CLUMP_ENTRIES - 1); + ++i, sectors = sectors >> 1) { + /* empty body */ + } + + clump_size = clumptbl[column + (i) * 3] * 1024 * 1024; + } + + /* + * Round the clump size to a multiple of node and block size. + * NOTE: This rounds down. + */ + clump_size /= mod; + clump_size *= mod; + + /* + * Rounding down could have rounded down to 0 if the block size was + * greater than the clump size. If so, just use one block or node. + */ + if (clump_size == 0) + clump_size = mod; + + return clump_size; +} + +/* Context for iterating b-tree map pages + * @page_idx: The index of the page within the b-node's page array + * @off: The byte offset within the mapped page + * @len: The remaining length of the map record + */ +struct hfs_bmap_ctx { + unsigned int page_idx; + unsigned int off; + u16 len; +}; + +/* + * Finds the specific page containing the requested byte offset within the map + * record. Automatically handles the difference between header and map nodes. + * Returns the struct page pointer, or an ERR_PTR on failure. + * Note: The caller is responsible for mapping/unmapping the returned page. + */ +static struct page *hfs_bmap_get_map_page(struct hfs_bnode *node, + struct hfs_bmap_ctx *ctx, + u32 byte_offset) +{ + u16 rec_idx, off16; + unsigned int page_off; + + if (node->this == HFSPLUS_TREE_HEAD) { + if (node->type != HFS_NODE_HEADER) { + pr_err("hfsplus: invalid btree header node\n"); + return ERR_PTR(-EIO); + } + rec_idx = HFSPLUS_BTREE_HDR_MAP_REC_INDEX; + } else { + if (node->type != HFS_NODE_MAP) { + pr_err("hfsplus: invalid btree map node\n"); + return ERR_PTR(-EIO); + } + rec_idx = HFSPLUS_BTREE_MAP_NODE_REC_INDEX; + } + + ctx->len = hfs_brec_lenoff(node, rec_idx, &off16); + if (hfs_brec_len_invalid(node, ctx->len)) + return ERR_PTR(-EINVAL); + + if (!is_bnode_offset_valid(node, off16)) + return ERR_PTR(-EIO); + + ctx->len = check_and_correct_requested_length(node, off16, ctx->len); + + if (byte_offset >= ctx->len) + return ERR_PTR(-EINVAL); + + page_off = (u32)off16 + node->page_offset + byte_offset; + ctx->page_idx = page_off >> PAGE_SHIFT; + ctx->off = page_off & ~PAGE_MASK; + + return node->page[ctx->page_idx]; +} + +/** + * hfs_bmap_test_bit - test a bit in the b-tree map + * @node: the b-tree node containing the map record + * @node_bit_idx: the relative bit index within the node's map record + * + * Returns true if set, false if clear or on failure. + */ +static bool hfs_bmap_test_bit(struct hfs_bnode *node, u32 node_bit_idx) +{ + struct hfs_bmap_ctx ctx; + struct page *page; + u8 *bmap, byte, mask; + + page = hfs_bmap_get_map_page(node, &ctx, node_bit_idx / BITS_PER_BYTE); + if (IS_ERR(page)) + return false; + + bmap = kmap_local_page(page); + byte = bmap[ctx.off]; + kunmap_local(bmap); + + mask = 1 << (7 - (node_bit_idx % BITS_PER_BYTE)); + return (byte & mask) != 0; +} + + +/** + * hfs_bmap_clear_bit - clear a bit in the b-tree map + * @node: the b-tree node containing the map record + * @node_bit_idx: the relative bit index within the node's map record + * + * Returns 0 on success, -EINVAL if already clear, or negative error code. + */ +static int hfs_bmap_clear_bit(struct hfs_bnode *node, u32 node_bit_idx) +{ + struct hfs_bmap_ctx ctx; + struct page *page; + u8 *bmap, mask; + + page = hfs_bmap_get_map_page(node, &ctx, node_bit_idx / BITS_PER_BYTE); + if (IS_ERR(page)) + return PTR_ERR(page); + + bmap = kmap_local_page(page); + + mask = 1 << (7 - (node_bit_idx % BITS_PER_BYTE)); + + if (!(bmap[ctx.off] & mask)) { + kunmap_local(bmap); + return -EINVAL; + } + + bmap[ctx.off] &= ~mask; + set_page_dirty(page); + kunmap_local(bmap); + + return 0; +} + +#define HFS_EXTENT_TREE_NAME "Extents Overflow File" +#define HFS_CATALOG_TREE_NAME "Catalog File" +#define HFS_ATTR_TREE_NAME "Attributes File" +#define HFS_UNKNOWN_TREE_NAME "Unknown B-tree" + +static const char *hfs_btree_name(u32 cnid) +{ + switch (cnid) { + case HFSPLUS_EXT_CNID: + return HFS_EXTENT_TREE_NAME; + case HFSPLUS_CAT_CNID: + return HFS_CATALOG_TREE_NAME; + case HFSPLUS_ATTR_CNID: + return HFS_ATTR_TREE_NAME; + default: + return HFS_UNKNOWN_TREE_NAME; + } +} + +/* Get a reference to a B*Tree and do some initial checks */ +struct hfs_btree *hfs_btree_open(struct super_block *sb, u32 id) +{ + struct hfs_btree *tree; + struct hfs_btree_header_rec *head; + struct address_space *mapping; + struct hfs_bnode *node; + struct inode *inode; + struct page *page; + unsigned int size; + + tree = kzalloc_obj(*tree); + if (!tree) + return NULL; + + mutex_init(&tree->tree_lock); + spin_lock_init(&tree->hash_lock); + tree->sb = sb; + tree->cnid = id; + inode = hfsplus_iget(sb, id); + if (IS_ERR(inode)) + goto free_tree; + tree->inode = inode; + + if (!HFSPLUS_I(tree->inode)->first_blocks) { + pr_err("invalid btree extent records (0 size)\n"); + goto free_inode; + } + + mapping = tree->inode->i_mapping; + page = read_mapping_page(mapping, 0, NULL); + if (IS_ERR(page)) + goto free_inode; + + /* Load the header */ + head = (struct hfs_btree_header_rec *)(kmap_local_page(page) + + sizeof(struct hfs_bnode_desc)); + tree->root = be32_to_cpu(head->root); + tree->leaf_count = be32_to_cpu(head->leaf_count); + tree->leaf_head = be32_to_cpu(head->leaf_head); + tree->leaf_tail = be32_to_cpu(head->leaf_tail); + tree->node_count = be32_to_cpu(head->node_count); + tree->free_nodes = be32_to_cpu(head->free_nodes); + tree->attributes = be32_to_cpu(head->attributes); + tree->node_size = be16_to_cpu(head->node_size); + tree->max_key_len = be16_to_cpu(head->max_key_len); + tree->depth = be16_to_cpu(head->depth); + + /* Verify the tree and set the correct compare function */ + switch (id) { + case HFSPLUS_EXT_CNID: + if (tree->max_key_len != HFSPLUS_EXT_KEYLEN - sizeof(u16)) { + pr_err("invalid extent max_key_len %d\n", + tree->max_key_len); + goto fail_page; + } + if (tree->attributes & HFS_TREE_VARIDXKEYS) { + pr_err("invalid extent btree flag\n"); + goto fail_page; + } + + tree->keycmp = hfsplus_ext_cmp_key; + break; + case HFSPLUS_CAT_CNID: + if (tree->max_key_len != HFSPLUS_CAT_KEYLEN - sizeof(u16)) { + pr_err("invalid catalog max_key_len %d\n", + tree->max_key_len); + goto fail_page; + } + if (!(tree->attributes & HFS_TREE_VARIDXKEYS)) { + pr_err("invalid catalog btree flag\n"); + goto fail_page; + } + + if (test_bit(HFSPLUS_SB_HFSX, &HFSPLUS_SB(sb)->flags) && + (head->key_type == HFSPLUS_KEY_BINARY)) + tree->keycmp = hfsplus_cat_bin_cmp_key; + else { + tree->keycmp = hfsplus_cat_case_cmp_key; + set_bit(HFSPLUS_SB_CASEFOLD, &HFSPLUS_SB(sb)->flags); + } + break; + case HFSPLUS_ATTR_CNID: + if (tree->max_key_len != HFSPLUS_ATTR_KEYLEN - sizeof(u16)) { + pr_err("invalid attributes max_key_len %d\n", + tree->max_key_len); + goto fail_page; + } + tree->keycmp = hfsplus_attr_bin_cmp_key; + break; + default: + pr_err("unknown B*Tree requested\n"); + goto fail_page; + } + + if (!(tree->attributes & HFS_TREE_BIGKEYS)) { + pr_err("invalid btree flag\n"); + goto fail_page; + } + + size = tree->node_size; + if (size < HFSPLUS_NODE_MINSZ || size > HFSPLUS_NODE_MXSZ) + goto fail_page; + if (!is_power_of_2(size)) + goto fail_page; + if (!tree->node_count) + goto fail_page; + + tree->node_size_shift = ffs(size) - 1; + + tree->pages_per_bnode = + (tree->node_size + PAGE_SIZE - 1) >> + PAGE_SHIFT; + + kunmap_local(head); + put_page(page); + + node = hfs_bnode_find(tree, HFSPLUS_TREE_HEAD); + if (IS_ERR(node)) + goto free_inode; + + if (!hfs_bmap_test_bit(node, 0)) { + pr_warn("(%s): %s (cnid 0x%x) map record invalid or bitmap corruption detected, forcing read-only.\n", + sb->s_id, hfs_btree_name(id), id); + pr_warn("Run fsck.hfsplus to repair.\n"); + sb->s_flags |= SB_RDONLY; + } + + hfs_bnode_put(node); + + return tree; + + fail_page: + kunmap_local(head); + put_page(page); + free_inode: + tree->inode->i_mapping->a_ops = &hfsplus_aops; + iput(tree->inode); + free_tree: + kfree(tree); + return NULL; +} + +/* Release resources used by a btree */ +void hfs_btree_close(struct hfs_btree *tree) +{ + struct hfs_bnode *node; + int i; + + if (!tree) + return; + + for (i = 0; i < NODE_HASH_SIZE; i++) { + while ((node = tree->node_hash[i])) { + tree->node_hash[i] = node->next_hash; + if (atomic_read(&node->refcnt)) + pr_crit("node %d:%d " + "still has %d user(s)!\n", + node->tree->cnid, node->this, + atomic_read(&node->refcnt)); + hfs_bnode_free(node); + tree->node_hash_cnt--; + } + } + iput(tree->inode); + kfree(tree); +} + +int hfs_btree_write(struct hfs_btree *tree) +{ + struct hfs_btree_header_rec *head; + struct hfs_bnode *node; + struct page *page; + + node = hfs_bnode_find(tree, 0); + if (IS_ERR(node)) + /* panic? */ + return -EIO; + /* Load the header */ + page = node->page[0]; + head = (struct hfs_btree_header_rec *)(kmap_local_page(page) + + sizeof(struct hfs_bnode_desc)); + + head->root = cpu_to_be32(tree->root); + head->leaf_count = cpu_to_be32(tree->leaf_count); + head->leaf_head = cpu_to_be32(tree->leaf_head); + head->leaf_tail = cpu_to_be32(tree->leaf_tail); + head->node_count = cpu_to_be32(tree->node_count); + head->free_nodes = cpu_to_be32(tree->free_nodes); + head->attributes = cpu_to_be32(tree->attributes); + head->depth = cpu_to_be16(tree->depth); + + kunmap_local(head); + set_page_dirty(page); + hfs_bnode_put(node); + return 0; +} + +static struct hfs_bnode *hfs_bmap_new_bmap(struct hfs_bnode *prev, u32 idx) +{ + struct hfs_btree *tree = prev->tree; + struct hfs_bnode *node; + struct hfs_bnode_desc desc; + __be32 cnid; + + node = hfs_bnode_create(tree, idx); + if (IS_ERR(node)) + return node; + + tree->free_nodes--; + prev->next = idx; + cnid = cpu_to_be32(idx); + hfs_bnode_write(prev, &cnid, offsetof(struct hfs_bnode_desc, next), 4); + + node->type = HFS_NODE_MAP; + node->num_recs = 1; + hfs_bnode_clear(node, 0, tree->node_size); + desc.next = 0; + desc.prev = 0; + desc.type = HFS_NODE_MAP; + desc.height = 0; + desc.num_recs = cpu_to_be16(1); + desc.reserved = 0; + hfs_bnode_write(node, &desc, 0, sizeof(desc)); + hfs_bnode_write_u16(node, 14, 0x8000); + hfs_bnode_write_u16(node, tree->node_size - 2, 14); + hfs_bnode_write_u16(node, tree->node_size - 4, tree->node_size - 6); + + return node; +} + +/* Make sure @tree has enough space for the @rsvd_nodes */ +int hfs_bmap_reserve(struct hfs_btree *tree, u32 rsvd_nodes) +{ + struct inode *inode = tree->inode; + struct hfsplus_inode_info *hip = HFSPLUS_I(inode); + u32 count; + int res; + + lockdep_assert_held(&tree->tree_lock); + + if (rsvd_nodes <= 0) + return 0; + + while (tree->free_nodes < rsvd_nodes) { + res = hfsplus_file_extend(inode, hfs_bnode_need_zeroout(tree)); + if (res) + return res; + hip->phys_size = inode->i_size = + (loff_t)hip->alloc_blocks << + HFSPLUS_SB(tree->sb)->alloc_blksz_shift; + hip->fs_blocks = + hip->alloc_blocks << HFSPLUS_SB(tree->sb)->fs_shift; + inode_set_bytes(inode, inode->i_size); + count = inode->i_size >> tree->node_size_shift; + tree->free_nodes += count - tree->node_count; + tree->node_count = count; + } + return 0; +} + +struct hfs_bnode *hfs_bmap_alloc(struct hfs_btree *tree) +{ + struct hfs_bnode *node, *next_node; + struct hfs_bmap_ctx ctx; + struct page *page; + u32 nidx, idx; + u8 *data, byte, m; + int i, res; + + lockdep_assert_held(&tree->tree_lock); + + res = hfs_bmap_reserve(tree, 1); + if (res) + return ERR_PTR(res); + + nidx = 0; + node = hfs_bnode_find(tree, nidx); + if (IS_ERR(node)) + return node; + + page = hfs_bmap_get_map_page(node, &ctx, 0); + if (IS_ERR(page)) { + res = PTR_ERR(page); + hfs_bnode_put(node); + return ERR_PTR(res); + } + + data = kmap_local_page(page); + idx = 0; + + for (;;) { + while (ctx.len) { + byte = data[ctx.off]; + if (byte != 0xff) { + for (m = 0x80, i = 0; i < 8; m >>= 1, i++) { + if (!(byte & m)) { + idx += i; + data[ctx.off] |= m; + set_page_dirty(page); + kunmap_local(data); + tree->free_nodes--; + hfs_btree_write(tree); + mark_inode_dirty(tree->inode); + hfs_bnode_put(node); + return hfs_bnode_create(tree, + idx); + } + } + } + if (++ctx.off >= PAGE_SIZE) { + kunmap_local(data); + page = node->page[++ctx.page_idx]; + data = kmap_local_page(page); + ctx.off = 0; + } + idx += 8; + ctx.len--; + } + kunmap_local(data); + nidx = node->next; + if (!nidx) { + hfs_dbg("create new bmap node\n"); + next_node = hfs_bmap_new_bmap(node, idx); + hfs_btree_write(tree); + } else + next_node = hfs_bnode_find(tree, nidx); + hfs_bnode_put(node); + if (IS_ERR(next_node)) + return next_node; + node = next_node; + + page = hfs_bmap_get_map_page(node, &ctx, 0); + if (IS_ERR(page)) { + res = PTR_ERR(page); + hfs_bnode_put(node); + return ERR_PTR(res); + } + data = kmap_local_page(page); + } +} + +void hfs_bmap_free(struct hfs_bnode *node) +{ + struct hfs_btree *tree; + u16 off, len; + u32 nidx; + int res; + + hfs_dbg("node %u\n", node->this); + BUG_ON(!node->this); + tree = node->tree; + lockdep_assert_held(&tree->tree_lock); + nidx = node->this; + node = hfs_bnode_find(tree, 0); + if (IS_ERR(node)) + return; + len = hfs_brec_lenoff(node, 2, &off); + if (hfs_brec_len_invalid(node, len)) { + pr_err("invalid bmap record length: node %u, len %u\n", + node->this, len); + hfs_bnode_put(node); + return; + } + while (nidx >= len * 8) { + u32 i; + + nidx -= len * 8; + i = node->next; + if (!i) { + /* panic */; + pr_crit("unable to free bnode %u. " + "bmap not found!\n", + node->this); + hfs_bnode_put(node); + return; + } + hfs_bnode_put(node); + node = hfs_bnode_find(tree, i); + if (IS_ERR(node)) + return; + if (node->type != HFS_NODE_MAP) { + /* panic */; + pr_crit("invalid bmap found! " + "(%u,%d)\n", + node->this, node->type); + hfs_bnode_put(node); + return; + } + len = hfs_brec_lenoff(node, 0, &off); + if (hfs_brec_len_invalid(node, len)) { + pr_err("invalid bmap record length: node %u, len %u\n", + node->this, len); + hfs_bnode_put(node); + return; + } + } + + res = hfs_bmap_clear_bit(node, nidx); + if (res == -EINVAL) { + pr_crit("trying to free the freed bnode %u(%d)\n", + nidx, node->type); + } else if (res) { + pr_crit("fail to free bnode %u(%d)\n", + nidx, node->type); + } else { + tree->free_nodes++; + hfs_btree_write(tree); + mark_inode_dirty(tree->inode); + } + + hfs_bnode_put(node); +} diff --git a/fs/hfsplus/catalog.c b/fs/hfsplus/catalog.c new file mode 100644 index 000000000..fe11c4b6d --- /dev/null +++ b/fs/hfsplus/catalog.c @@ -0,0 +1,540 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hfsplus/catalog.c + * + * Copyright (C) 2001 + * Brad Boyer (flar@allandria.com) + * (C) 2003 Ardis Technologies + * + * Handling of catalog records + */ + + +#include "hfsplus_fs.h" +#include "hfsplus_raw.h" + +int hfsplus_cat_case_cmp_key(const hfsplus_btree_key *k1, + const hfsplus_btree_key *k2) +{ + __be32 k1p, k2p; + + k1p = k1->cat.parent; + k2p = k2->cat.parent; + if (k1p != k2p) + return be32_to_cpu(k1p) < be32_to_cpu(k2p) ? -1 : 1; + + return hfsplus_strcasecmp(&k1->cat.name, &k2->cat.name); +} + +int hfsplus_cat_bin_cmp_key(const hfsplus_btree_key *k1, + const hfsplus_btree_key *k2) +{ + __be32 k1p, k2p; + + k1p = k1->cat.parent; + k2p = k2->cat.parent; + if (k1p != k2p) + return be32_to_cpu(k1p) < be32_to_cpu(k2p) ? -1 : 1; + + return hfsplus_strcmp(&k1->cat.name, &k2->cat.name); +} + +/* Generates key for catalog file/folders record. */ +int hfsplus_cat_build_key(struct super_block *sb, + hfsplus_btree_key *key, u32 parent, const struct qstr *str) +{ + int len, err; + + key->cat.parent = cpu_to_be32(parent); + err = hfsplus_asc2uni(sb, &key->cat.name, HFSPLUS_MAX_STRLEN, + str->name, str->len, HFS_REGULAR_NAME); + if (unlikely(err < 0)) + return err; + + len = be16_to_cpu(key->cat.name.length); + key->key_len = cpu_to_be16(6 + 2 * len); + return 0; +} + +/* Generates key for catalog thread record. */ +void hfsplus_cat_build_key_with_cnid(struct super_block *sb, + hfsplus_btree_key *key, u32 parent) +{ + key->cat.parent = cpu_to_be32(parent); + key->cat.name.length = 0; + key->key_len = cpu_to_be16(6); +} + +static void hfsplus_cat_build_key_uni(hfsplus_btree_key *key, u32 parent, + struct hfsplus_unistr *name) +{ + int ustrlen; + + ustrlen = be16_to_cpu(name->length); + key->cat.parent = cpu_to_be32(parent); + key->cat.name.length = cpu_to_be16(ustrlen); + ustrlen *= 2; + memcpy(key->cat.name.unicode, name->unicode, ustrlen); + key->key_len = cpu_to_be16(6 + ustrlen); +} + +void hfsplus_cat_set_perms(struct inode *inode, struct hfsplus_perm *perms) +{ + if (inode->i_flags & S_IMMUTABLE) + perms->rootflags |= HFSPLUS_FLG_IMMUTABLE; + else + perms->rootflags &= ~HFSPLUS_FLG_IMMUTABLE; + if (inode->i_flags & S_APPEND) + perms->rootflags |= HFSPLUS_FLG_APPEND; + else + perms->rootflags &= ~HFSPLUS_FLG_APPEND; + + perms->userflags = HFSPLUS_I(inode)->userflags; + perms->mode = cpu_to_be16(inode->i_mode); + perms->owner = cpu_to_be32(i_uid_read(inode)); + perms->group = cpu_to_be32(i_gid_read(inode)); + + if (S_ISREG(inode->i_mode)) + perms->dev = cpu_to_be32(inode->i_nlink); + else if (S_ISBLK(inode->i_mode) || S_ISCHR(inode->i_mode)) + perms->dev = cpu_to_be32(inode->i_rdev); + else + perms->dev = 0; +} + +static int hfsplus_cat_build_record(hfsplus_cat_entry *entry, + u32 cnid, struct inode *inode) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(inode->i_sb); + + if (S_ISDIR(inode->i_mode)) { + struct hfsplus_cat_folder *folder; + + folder = &entry->folder; + memset(folder, 0, sizeof(*folder)); + folder->type = cpu_to_be16(HFSPLUS_FOLDER); + if (test_bit(HFSPLUS_SB_HFSX, &sbi->flags)) + folder->flags |= cpu_to_be16(HFSPLUS_HAS_FOLDER_COUNT); + folder->id = cpu_to_be32(inode->i_ino); + HFSPLUS_I(inode)->create_date = + folder->create_date = + folder->content_mod_date = + folder->attribute_mod_date = + folder->access_date = hfsp_now2mt(); + hfsplus_cat_set_perms(inode, &folder->permissions); + if (inode == sbi->hidden_dir) + /* invisible and namelocked */ + folder->user_info.frFlags = cpu_to_be16(0x5000); + return sizeof(*folder); + } else { + struct hfsplus_cat_file *file; + + file = &entry->file; + memset(file, 0, sizeof(*file)); + file->type = cpu_to_be16(HFSPLUS_FILE); + file->flags = cpu_to_be16(HFSPLUS_FILE_THREAD_EXISTS); + file->id = cpu_to_be32(cnid); + HFSPLUS_I(inode)->create_date = + file->create_date = + file->content_mod_date = + file->attribute_mod_date = + file->access_date = hfsp_now2mt(); + if (cnid == inode->i_ino) { + hfsplus_cat_set_perms(inode, &file->permissions); + if (S_ISLNK(inode->i_mode)) { + file->user_info.fdType = + cpu_to_be32(HFSP_SYMLINK_TYPE); + file->user_info.fdCreator = + cpu_to_be32(HFSP_SYMLINK_CREATOR); + } else { + file->user_info.fdType = + cpu_to_be32(sbi->type); + file->user_info.fdCreator = + cpu_to_be32(sbi->creator); + } + if (HFSPLUS_FLG_IMMUTABLE & + (file->permissions.rootflags | + file->permissions.userflags)) + file->flags |= + cpu_to_be16(HFSPLUS_FILE_LOCKED); + } else { + file->user_info.fdType = + cpu_to_be32(HFSP_HARDLINK_TYPE); + file->user_info.fdCreator = + cpu_to_be32(HFSP_HFSPLUS_CREATOR); + file->user_info.fdFlags = + cpu_to_be16(0x100); + file->create_date = + HFSPLUS_I(sbi->hidden_dir)->create_date; + file->permissions.dev = + cpu_to_be32(HFSPLUS_I(inode)->linkid); + } + return sizeof(*file); + } +} + +static int hfsplus_fill_cat_thread(struct super_block *sb, + hfsplus_cat_entry *entry, int type, + u32 parentid, const struct qstr *str) +{ + int err; + + entry->type = cpu_to_be16(type); + entry->thread.reserved = 0; + entry->thread.parentID = cpu_to_be32(parentid); + err = hfsplus_asc2uni(sb, &entry->thread.nodeName, HFSPLUS_MAX_STRLEN, + str->name, str->len, HFS_REGULAR_NAME); + if (unlikely(err < 0)) + return err; + + return 10 + be16_to_cpu(entry->thread.nodeName.length) * 2; +} + +/* Try to get a catalog entry for given catalog id */ +int hfsplus_find_cat(struct super_block *sb, u32 cnid, + struct hfs_find_data *fd) +{ + hfsplus_cat_entry tmp = {0}; + int err; + u16 type; + + hfsplus_cat_build_key_with_cnid(sb, fd->search_key, cnid); + err = hfsplus_brec_read_cat(fd, &tmp); + if (err) + return err; + + type = be16_to_cpu(tmp.type); + if (!is_hfs_thread_record_type(type)) { + pr_err("found bad thread record in catalog\n"); + return -EIO; + } + + if (be16_to_cpu(tmp.thread.nodeName.length) > 255) { + pr_err("catalog name length corrupted\n"); + return -EIO; + } + + hfsplus_cat_build_key_uni(fd->search_key, + be32_to_cpu(tmp.thread.parentID), + &tmp.thread.nodeName); + return hfs_brec_find(fd, hfs_find_rec_by_key); +} + +static void hfsplus_subfolders_inc(struct inode *dir) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(dir->i_sb); + + if (test_bit(HFSPLUS_SB_HFSX, &sbi->flags)) { + /* + * Increment subfolder count. Note, the value is only meaningful + * for folders with HFSPLUS_HAS_FOLDER_COUNT flag set. + */ + HFSPLUS_I(dir)->subfolders++; + } +} + +static void hfsplus_subfolders_dec(struct inode *dir) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(dir->i_sb); + + if (test_bit(HFSPLUS_SB_HFSX, &sbi->flags)) { + /* + * Decrement subfolder count. Note, the value is only meaningful + * for folders with HFSPLUS_HAS_FOLDER_COUNT flag set. + * + * Check for zero. Some subfolders may have been created + * by an implementation ignorant of this counter. + */ + if (HFSPLUS_I(dir)->subfolders) + HFSPLUS_I(dir)->subfolders--; + } +} + +int hfsplus_create_cat(u32 cnid, struct inode *dir, + const struct qstr *str, struct inode *inode) +{ + struct super_block *sb = dir->i_sb; + struct hfs_find_data fd; + hfsplus_cat_entry entry; + int entry_size; + int err; + + hfs_dbg("name %s, cnid %u, i_nlink %d\n", + str->name, cnid, inode->i_nlink); + err = hfs_find_init(HFSPLUS_SB(sb)->cat_tree, &fd); + if (err) + return err; + + /* + * Fail early and avoid ENOSPC during the btree operations. We may + * have to split the root node at most once. + */ + err = hfs_bmap_reserve(fd.tree, 2 * fd.tree->depth); + if (err) + goto err2; + + hfsplus_cat_build_key_with_cnid(sb, fd.search_key, cnid); + entry_size = hfsplus_fill_cat_thread(sb, &entry, + S_ISDIR(inode->i_mode) ? + HFSPLUS_FOLDER_THREAD : HFSPLUS_FILE_THREAD, + dir->i_ino, str); + if (unlikely(entry_size < 0)) { + err = entry_size; + goto err2; + } + + err = hfs_brec_find(&fd, hfs_find_rec_by_key); + if (err != -ENOENT) { + if (!err) + err = -EEXIST; + goto err2; + } + err = hfs_brec_insert(&fd, &entry, entry_size); + if (err) + goto err2; + + err = hfsplus_cat_build_key(sb, fd.search_key, dir->i_ino, str); + if (unlikely(err)) + goto err1; + + entry_size = hfsplus_cat_build_record(&entry, cnid, inode); + err = hfs_brec_find(&fd, hfs_find_rec_by_key); + if (err != -ENOENT) { + /* panic? */ + if (!err) + err = -EEXIST; + goto err1; + } + err = hfs_brec_insert(&fd, &entry, entry_size); + if (err) + goto err1; + + dir->i_size++; + if (S_ISDIR(inode->i_mode)) + hfsplus_subfolders_inc(dir); + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + hfsplus_mark_inode_dirty(HFSPLUS_CAT_TREE_I(sb), HFSPLUS_I_CAT_DIRTY); + hfsplus_mark_inode_dirty(dir, HFSPLUS_I_CAT_DIRTY); + + hfs_find_exit(&fd); + return 0; + +err1: + hfsplus_cat_build_key_with_cnid(sb, fd.search_key, cnid); + if (!hfs_brec_find(&fd, hfs_find_rec_by_key)) + hfs_brec_remove(&fd); +err2: + hfs_find_exit(&fd); + return err; +} + +int hfsplus_delete_cat(u32 cnid, struct inode *dir, const struct qstr *str) +{ + struct super_block *sb = dir->i_sb; + struct hfs_find_data fd; + struct hfsplus_fork_raw fork; + int err, off; + u16 type; + + hfs_dbg("name %s, cnid %u\n", str ? str->name : NULL, cnid); + err = hfs_find_init(HFSPLUS_SB(sb)->cat_tree, &fd); + if (err) + return err; + + /* + * Fail early and avoid ENOSPC during the btree operations. We may + * have to split the root node at most once. + */ + err = hfs_bmap_reserve(fd.tree, 2 * (int)fd.tree->depth - 2); + if (err) + goto out; + + if (!str) { + hfsplus_cat_entry entry = {0}; + + hfsplus_cat_build_key_with_cnid(sb, fd.search_key, cnid); + err = hfsplus_brec_read_cat(&fd, &entry); + if (err) + goto out; + + type = be16_to_cpu(entry.type); + if (!is_hfs_thread_record_type(type)) { + pr_err("found bad thread record in catalog\n"); + err = -EIO; + goto out; + } + + hfsplus_cat_build_key_uni(fd.search_key, dir->i_ino, + &entry.thread.nodeName); + } else { + err = hfsplus_cat_build_key(sb, fd.search_key, dir->i_ino, str); + if (unlikely(err)) + goto out; + } + + err = hfs_brec_find(&fd, hfs_find_rec_by_key); + if (err) + goto out; + + type = hfs_bnode_read_u16(fd.bnode, fd.entryoffset); + if (type == HFSPLUS_FILE) { +#if 0 + off = fd.entryoffset + offsetof(hfsplus_cat_file, data_fork); + hfs_bnode_read(fd.bnode, &fork, off, sizeof(fork)); + hfsplus_free_fork(sb, cnid, &fork, HFSPLUS_TYPE_DATA); +#endif + + off = fd.entryoffset + + offsetof(struct hfsplus_cat_file, rsrc_fork); + hfs_bnode_read(fd.bnode, &fork, off, sizeof(fork)); + hfsplus_free_fork(sb, cnid, &fork, HFSPLUS_TYPE_RSRC); + } + + err = hfs_brec_remove(&fd); + if (err) + goto out; + + hfsplus_cat_build_key_with_cnid(sb, fd.search_key, cnid); + err = hfs_brec_find(&fd, hfs_find_rec_by_key); + if (err) + goto out; + + err = hfs_brec_remove(&fd); + if (err) + goto out; + + dir->i_size--; + if (type == HFSPLUS_FOLDER) + hfsplus_subfolders_dec(dir); + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + hfsplus_mark_inode_dirty(HFSPLUS_CAT_TREE_I(sb), HFSPLUS_I_CAT_DIRTY); + hfsplus_mark_inode_dirty(dir, HFSPLUS_I_CAT_DIRTY); + + if (type == HFSPLUS_FILE || type == HFSPLUS_FOLDER) { + if (HFSPLUS_SB(sb)->attr_tree) + hfsplus_delete_all_attrs(dir, cnid); + } + +out: + hfs_find_exit(&fd); + + return err; +} + +int hfsplus_rename_cat(u32 cnid, + struct inode *src_dir, const struct qstr *src_name, + struct inode *dst_dir, const struct qstr *dst_name) +{ + struct super_block *sb = src_dir->i_sb; + struct hfs_find_data src_fd, dst_fd; + hfsplus_cat_entry entry; + int entry_size, type; + int err; + + hfs_dbg("cnid %u - ino %llu, name %s - ino %llu, name %s\n", + cnid, src_dir->i_ino, src_name->name, + dst_dir->i_ino, dst_name->name); + err = hfs_find_init(HFSPLUS_SB(sb)->cat_tree, &src_fd); + if (err) + return err; + dst_fd = src_fd; + + /* + * Fail early and avoid ENOSPC during the btree operations. We may + * have to split the root node at most twice. + */ + err = hfs_bmap_reserve(src_fd.tree, 4 * (int)src_fd.tree->depth - 1); + if (err) + goto out; + + /* find the old dir entry and read the data */ + err = hfsplus_cat_build_key(sb, src_fd.search_key, + src_dir->i_ino, src_name); + if (unlikely(err)) + goto out; + + err = hfs_brec_find(&src_fd, hfs_find_rec_by_key); + if (err) + goto out; + if (src_fd.entrylength > sizeof(entry) || src_fd.entrylength < 0) { + err = -EIO; + goto out; + } + + hfs_bnode_read(src_fd.bnode, &entry, src_fd.entryoffset, + src_fd.entrylength); + type = be16_to_cpu(entry.type); + + /* create new dir entry with the data from the old entry */ + err = hfsplus_cat_build_key(sb, dst_fd.search_key, + dst_dir->i_ino, dst_name); + if (unlikely(err)) + goto out; + + err = hfs_brec_find(&dst_fd, hfs_find_rec_by_key); + if (err != -ENOENT) { + if (!err) + err = -EEXIST; + goto out; + } + + err = hfs_brec_insert(&dst_fd, &entry, src_fd.entrylength); + if (err) + goto out; + dst_dir->i_size++; + if (type == HFSPLUS_FOLDER) + hfsplus_subfolders_inc(dst_dir); + inode_set_mtime_to_ts(dst_dir, inode_set_ctime_current(dst_dir)); + + /* finally remove the old entry */ + err = hfsplus_cat_build_key(sb, src_fd.search_key, + src_dir->i_ino, src_name); + if (unlikely(err)) + goto out; + + err = hfs_brec_find(&src_fd, hfs_find_rec_by_key); + if (err) + goto out; + err = hfs_brec_remove(&src_fd); + if (err) + goto out; + src_dir->i_size--; + if (type == HFSPLUS_FOLDER) + hfsplus_subfolders_dec(src_dir); + inode_set_mtime_to_ts(src_dir, inode_set_ctime_current(src_dir)); + + /* remove old thread entry */ + hfsplus_cat_build_key_with_cnid(sb, src_fd.search_key, cnid); + err = hfs_brec_find(&src_fd, hfs_find_rec_by_key); + if (err) + goto out; + type = hfs_bnode_read_u16(src_fd.bnode, src_fd.entryoffset); + err = hfs_brec_remove(&src_fd); + if (err) + goto out; + + /* create new thread entry */ + hfsplus_cat_build_key_with_cnid(sb, dst_fd.search_key, cnid); + entry_size = hfsplus_fill_cat_thread(sb, &entry, type, + dst_dir->i_ino, dst_name); + if (unlikely(entry_size < 0)) { + err = entry_size; + goto out; + } + + err = hfs_brec_find(&dst_fd, hfs_find_rec_by_key); + if (err != -ENOENT) { + if (!err) + err = -EEXIST; + goto out; + } + err = hfs_brec_insert(&dst_fd, &entry, entry_size); + + hfsplus_mark_inode_dirty(HFSPLUS_CAT_TREE_I(sb), HFSPLUS_I_CAT_DIRTY); + hfsplus_mark_inode_dirty(dst_dir, HFSPLUS_I_CAT_DIRTY); + hfsplus_mark_inode_dirty(src_dir, HFSPLUS_I_CAT_DIRTY); +out: + hfs_bnode_put(dst_fd.bnode); + hfs_find_exit(&src_fd); + return err; +} diff --git a/fs/hfsplus/dir.c b/fs/hfsplus/dir.c new file mode 100644 index 000000000..51fcba2e6 --- /dev/null +++ b/fs/hfsplus/dir.c @@ -0,0 +1,643 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hfsplus/dir.c + * + * Copyright (C) 2001 + * Brad Boyer (flar@allandria.com) + * (C) 2003 Ardis Technologies + * + * Handling of directories + */ + +#include +#include +#include +#include +#include + +#include "hfsplus_fs.h" +#include "hfsplus_raw.h" +#include "xattr.h" + +static inline void hfsplus_instantiate(struct dentry *dentry, + struct inode *inode, u32 cnid) +{ + dentry->d_fsdata = (void *)(unsigned long)cnid; + d_instantiate(dentry, inode); +} + +/* Find the entry inside dir named dentry->d_name */ +static struct dentry *hfsplus_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + struct inode *inode = NULL; + struct hfs_find_data fd; + struct super_block *sb; + hfsplus_cat_entry entry; + int err; + u32 cnid, linkid = 0; + u16 type; + + sb = dir->i_sb; + + dentry->d_fsdata = NULL; + err = hfs_find_init(HFSPLUS_SB(sb)->cat_tree, &fd); + if (err) + return ERR_PTR(err); + err = hfsplus_cat_build_key(sb, fd.search_key, dir->i_ino, + &dentry->d_name); + if (unlikely(err < 0)) + goto fail; +again: + err = hfsplus_brec_read_cat(&fd, &entry); + if (err) { + if (err == -ENOENT) { + hfs_find_exit(&fd); + /* No such entry */ + inode = NULL; + goto out; + } + goto fail; + } + type = be16_to_cpu(entry.type); + if (type == HFSPLUS_FOLDER) { + if (fd.entrylength < sizeof(struct hfsplus_cat_folder)) { + err = -EIO; + goto fail; + } + cnid = be32_to_cpu(entry.folder.id); + dentry->d_fsdata = (void *)(unsigned long)cnid; + } else if (type == HFSPLUS_FILE) { + if (fd.entrylength < sizeof(struct hfsplus_cat_file)) { + err = -EIO; + goto fail; + } + cnid = be32_to_cpu(entry.file.id); + if (entry.file.user_info.fdType == + cpu_to_be32(HFSP_HARDLINK_TYPE) && + entry.file.user_info.fdCreator == + cpu_to_be32(HFSP_HFSPLUS_CREATOR) && + HFSPLUS_SB(sb)->hidden_dir && + (entry.file.create_date == + HFSPLUS_I(HFSPLUS_SB(sb)->hidden_dir)-> + create_date || + entry.file.create_date == + HFSPLUS_I(d_inode(sb->s_root))-> + create_date)) { + struct qstr str; + char name[32]; + + if (dentry->d_fsdata) { + /* + * We found a link pointing to another link, + * so ignore it and treat it as regular file. + */ + cnid = (unsigned long)dentry->d_fsdata; + linkid = 0; + } else { + dentry->d_fsdata = (void *)(unsigned long)cnid; + linkid = + be32_to_cpu(entry.file.permissions.dev); + str.len = sprintf(name, "iNode%d", linkid); + str.name = name; + err = hfsplus_cat_build_key(sb, fd.search_key, + HFSPLUS_SB(sb)->hidden_dir->i_ino, + &str); + if (unlikely(err < 0)) + goto fail; + goto again; + } + } else if (!dentry->d_fsdata) + dentry->d_fsdata = (void *)(unsigned long)cnid; + } else { + pr_err("invalid catalog entry type in lookup\n"); + err = -EIO; + goto fail; + } + hfs_find_exit(&fd); + inode = hfsplus_iget(dir->i_sb, cnid); + if (IS_ERR(inode)) + return ERR_CAST(inode); + if (S_ISREG(inode->i_mode)) + HFSPLUS_I(inode)->linkid = linkid; +out: + return d_splice_alias(inode, dentry); +fail: + hfs_find_exit(&fd); + return ERR_PTR(err); +} + +static int hfsplus_readdir(struct file *file, struct dir_context *ctx) +{ + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + int len, err; + char *strbuf; + hfsplus_cat_entry entry; + struct hfs_find_data fd; + struct hfsplus_readdir_data *rd; + u16 type; + + if (file->f_pos >= inode->i_size) + return 0; + + err = hfs_find_init(HFSPLUS_SB(sb)->cat_tree, &fd); + if (err) + return err; + strbuf = kmalloc(NLS_MAX_CHARSET_SIZE * HFSPLUS_MAX_STRLEN + 1, GFP_KERNEL); + if (!strbuf) { + err = -ENOMEM; + goto out; + } + hfsplus_cat_build_key_with_cnid(sb, fd.search_key, inode->i_ino); + err = hfs_brec_find(&fd, hfs_find_rec_by_key); + if (err) + goto out; + + if (ctx->pos == 0) { + /* This is completely artificial... */ + if (!dir_emit_dot(file, ctx)) + goto out; + ctx->pos = 1; + } + if (ctx->pos == 1) { + if (fd.entrylength > sizeof(entry) || fd.entrylength < 0) { + err = -EIO; + goto out; + } + + hfs_bnode_read(fd.bnode, &entry, fd.entryoffset, + fd.entrylength); + if (be16_to_cpu(entry.type) != HFSPLUS_FOLDER_THREAD) { + pr_err("bad catalog folder thread\n"); + err = -EIO; + goto out; + } + if (fd.entrylength < HFSPLUS_MIN_THREAD_SZ) { + pr_err("truncated catalog thread\n"); + err = -EIO; + goto out; + } + if (!dir_emit(ctx, "..", 2, + be32_to_cpu(entry.thread.parentID), DT_DIR)) + goto out; + ctx->pos = 2; + } + if (ctx->pos >= inode->i_size) + goto out; + rd = file->private_data; + if (rd && rd->pos == ctx->pos) { + memcpy(fd.search_key, &rd->key, sizeof(struct hfsplus_cat_key)); + err = hfs_brec_find(&fd, hfs_find_rec_by_key); + if (err == -ENOENT) + err = hfs_brec_goto(&fd, 1); + } else { + err = hfs_brec_goto(&fd, ctx->pos - 1); + } + if (err) + goto out; + for (;;) { + if (be32_to_cpu(fd.key->cat.parent) != inode->i_ino) { + pr_err("walked past end of dir\n"); + err = -EIO; + goto out; + } + + if (fd.entrylength > sizeof(entry) || fd.entrylength < 0) { + err = -EIO; + goto out; + } + + hfs_bnode_read(fd.bnode, &entry, fd.entryoffset, + fd.entrylength); + type = be16_to_cpu(entry.type); + len = NLS_MAX_CHARSET_SIZE * HFSPLUS_MAX_STRLEN; + err = hfsplus_uni2asc_str(sb, &fd.key->cat.name, strbuf, &len); + if (err) + goto out; + if (type == HFSPLUS_FOLDER) { + if (fd.entrylength < + sizeof(struct hfsplus_cat_folder)) { + pr_err("small dir entry\n"); + err = -EIO; + goto out; + } + if (HFSPLUS_SB(sb)->hidden_dir && + HFSPLUS_SB(sb)->hidden_dir->i_ino == + be32_to_cpu(entry.folder.id)) + goto next; + if (!dir_emit(ctx, strbuf, len, + be32_to_cpu(entry.folder.id), DT_DIR)) + break; + } else if (type == HFSPLUS_FILE) { + u16 mode; + unsigned type = DT_UNKNOWN; + + if (fd.entrylength < sizeof(struct hfsplus_cat_file)) { + pr_err("small file entry\n"); + err = -EIO; + goto out; + } + + mode = be16_to_cpu(entry.file.permissions.mode); + if (S_ISREG(mode)) + type = DT_REG; + else if (S_ISLNK(mode)) + type = DT_LNK; + else if (S_ISFIFO(mode)) + type = DT_FIFO; + else if (S_ISCHR(mode)) + type = DT_CHR; + else if (S_ISBLK(mode)) + type = DT_BLK; + else if (S_ISSOCK(mode)) + type = DT_SOCK; + + if (!dir_emit(ctx, strbuf, len, + be32_to_cpu(entry.file.id), type)) + break; + } else { + pr_err("bad catalog entry type\n"); + err = -EIO; + goto out; + } +next: + ctx->pos++; + if (ctx->pos >= inode->i_size) + goto out; + err = hfs_brec_goto(&fd, 1); + if (err) + goto out; + } + if (!rd) { + rd = kmalloc_obj(struct hfsplus_readdir_data); + if (!rd) { + err = -ENOMEM; + goto out; + } + file->private_data = rd; + } + rd->pos = ctx->pos; + memcpy(&rd->key, fd.key, sizeof(struct hfsplus_cat_key)); +out: + kfree(strbuf); + hfs_find_exit(&fd); + return err; +} + +static int hfsplus_dir_release(struct inode *inode, struct file *file) +{ + kfree(file->private_data); + return 0; +} + +static int hfsplus_link(struct dentry *src_dentry, struct inode *dst_dir, + struct dentry *dst_dentry) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(dst_dir->i_sb); + struct inode *inode = d_inode(src_dentry); + struct inode *src_dir = d_inode(src_dentry->d_parent); + struct qstr str; + char name[32]; + u32 cnid, id; + int res; + + if (HFSPLUS_IS_RSRC(inode)) + return -EPERM; + if (!S_ISREG(inode->i_mode)) + return -EPERM; + + hfs_dbg("src_dir->i_ino %llu, dst_dir->i_ino %llu, inode->i_ino %llu\n", + src_dir->i_ino, dst_dir->i_ino, inode->i_ino); + + mutex_lock(&sbi->vh_mutex); + if (inode->i_ino == (u32)(unsigned long)src_dentry->d_fsdata) { + for (;;) { + get_random_bytes(&id, sizeof(cnid)); + id &= 0x3fffffff; + str.name = name; + str.len = sprintf(name, "iNode%d", id); + res = hfsplus_rename_cat(inode->i_ino, + src_dir, &src_dentry->d_name, + sbi->hidden_dir, &str); + if (!res) + break; + if (res != -EEXIST) + goto out; + } + HFSPLUS_I(inode)->linkid = id; + cnid = sbi->next_cnid++; + src_dentry->d_fsdata = (void *)(unsigned long)cnid; + res = hfsplus_create_cat(cnid, src_dir, + &src_dentry->d_name, inode); + if (res) + /* panic? */ + goto out; + sbi->file_count++; + } + cnid = sbi->next_cnid++; + res = hfsplus_create_cat(cnid, dst_dir, &dst_dentry->d_name, inode); + if (res) + goto out; + + inc_nlink(inode); + hfsplus_instantiate(dst_dentry, inode, cnid); + ihold(inode); + inode_set_ctime_current(inode); + mark_inode_dirty(inode); + sbi->file_count++; + hfsplus_mark_mdb_dirty(dst_dir->i_sb); + + res = hfsplus_cat_write_inode(src_dir); + if (res) + goto out; + + res = hfsplus_cat_write_inode(dst_dir); + if (res) + goto out; + + res = hfsplus_cat_write_inode(sbi->hidden_dir); + if (res) + goto out; + + res = hfsplus_cat_write_inode(inode); + +out: + mutex_unlock(&sbi->vh_mutex); + return res; +} + +static int hfsplus_unlink(struct inode *dir, struct dentry *dentry) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(dir->i_sb); + struct inode *inode = d_inode(dentry); + struct qstr str; + char name[32]; + u32 cnid; + int res; + + if (HFSPLUS_IS_RSRC(inode)) + return -EPERM; + + hfs_dbg("dir->i_ino %llu, inode->i_ino %llu\n", + dir->i_ino, inode->i_ino); + + mutex_lock(&sbi->vh_mutex); + cnid = (u32)(unsigned long)dentry->d_fsdata; + if (inode->i_ino == cnid && + atomic_read(&HFSPLUS_I(inode)->opencnt)) { + str.name = name; + str.len = sprintf(name, "temp%llu", inode->i_ino); + res = hfsplus_rename_cat(inode->i_ino, + dir, &dentry->d_name, + sbi->hidden_dir, &str); + if (!res) { + inode->i_flags |= S_DEAD; + drop_nlink(inode); + } + goto out; + } + res = hfsplus_delete_cat(cnid, dir, &dentry->d_name); + if (res) + goto out; + + if (inode->i_nlink > 0) + drop_nlink(inode); + if (inode->i_ino == cnid) + clear_nlink(inode); + if (!inode->i_nlink) { + if (inode->i_ino != cnid) { + sbi->file_count--; + if (!atomic_read(&HFSPLUS_I(inode)->opencnt)) { + res = hfsplus_delete_cat(inode->i_ino, + sbi->hidden_dir, + NULL); + if (!res) + hfsplus_delete_inode(inode); + } else + inode->i_flags |= S_DEAD; + } else + hfsplus_delete_inode(inode); + } else + sbi->file_count--; + inode_set_ctime_current(inode); + mark_inode_dirty(inode); +out: + if (!res) { + res = hfsplus_cat_write_inode(dir); + if (!res) { + res = hfsplus_cat_write_inode(sbi->hidden_dir); + if (!res) + res = hfsplus_cat_write_inode(inode); + } + } + + mutex_unlock(&sbi->vh_mutex); + return res; +} + +static int hfsplus_rmdir(struct inode *dir, struct dentry *dentry) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(dir->i_sb); + struct inode *inode = d_inode(dentry); + int res; + + if (inode->i_size != 2) + return -ENOTEMPTY; + + mutex_lock(&sbi->vh_mutex); + res = hfsplus_delete_cat(inode->i_ino, dir, &dentry->d_name); + if (res) + goto out; + clear_nlink(inode); + inode_set_ctime_current(inode); + hfsplus_delete_inode(inode); + mark_inode_dirty(inode); + + res = hfsplus_cat_write_inode(dir); +out: + mutex_unlock(&sbi->vh_mutex); + return res; +} + +static int hfsplus_symlink(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const char *symname) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(dir->i_sb); + struct inode *inode; + int res = -ENOMEM; + + mutex_lock(&sbi->vh_mutex); + inode = hfsplus_new_inode(dir->i_sb, dir, S_IFLNK | S_IRWXUGO); + if (!inode) + goto out; + + hfs_dbg("dir->i_ino %llu, inode->i_ino %llu\n", + dir->i_ino, inode->i_ino); + + res = page_symlink(inode, symname, strlen(symname) + 1); + if (res) + goto out_err; + + res = hfsplus_create_cat(inode->i_ino, dir, &dentry->d_name, inode); + if (res) + goto out_err; + + res = hfsplus_init_security(inode, dir, &dentry->d_name); + if (res == -EOPNOTSUPP) + res = 0; /* Operation is not supported. */ + else if (res) { + /* Try to delete anyway without error analysis. */ + hfsplus_delete_cat(inode->i_ino, dir, &dentry->d_name); + goto out_err; + } + + hfsplus_instantiate(dentry, inode, inode->i_ino); + mark_inode_dirty(inode); + + res = hfsplus_cat_write_inode(dir); + if (res) + goto out; + + res = hfsplus_cat_write_inode(inode); + goto out; + +out_err: + clear_nlink(inode); + hfsplus_delete_inode(inode); + iput(inode); +out: + mutex_unlock(&sbi->vh_mutex); + return res; +} + +static int hfsplus_mknod(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(dir->i_sb); + struct inode *inode; + int res = -ENOMEM; + + mutex_lock(&sbi->vh_mutex); + inode = hfsplus_new_inode(dir->i_sb, dir, mode); + if (!inode) + goto out; + + hfs_dbg("dir->i_ino %llu, inode->i_ino %llu\n", + dir->i_ino, inode->i_ino); + + if (S_ISBLK(mode) || S_ISCHR(mode) || S_ISFIFO(mode) || S_ISSOCK(mode)) + init_special_inode(inode, mode, rdev); + + res = hfsplus_create_cat(inode->i_ino, dir, &dentry->d_name, inode); + if (res) + goto failed_mknod; + + res = hfsplus_init_security(inode, dir, &dentry->d_name); + if (res == -EOPNOTSUPP) + res = 0; /* Operation is not supported. */ + else if (res) { + /* Try to delete anyway without error analysis. */ + hfsplus_delete_cat(inode->i_ino, dir, &dentry->d_name); + goto failed_mknod; + } + + hfsplus_instantiate(dentry, inode, inode->i_ino); + mark_inode_dirty(inode); + + res = hfsplus_cat_write_inode(dir); + if (res) + goto out; + + res = hfsplus_cat_write_inode(inode); + goto out; + +failed_mknod: + clear_nlink(inode); + hfsplus_delete_inode(inode); + iput(inode); +out: + mutex_unlock(&sbi->vh_mutex); + return res; +} + +static int hfsplus_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + return hfsplus_mknod(&nop_mnt_idmap, dir, dentry, mode, 0); +} + +static struct dentry *hfsplus_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + return ERR_PTR(hfsplus_mknod(&nop_mnt_idmap, dir, dentry, mode, 0)); +} + +static int hfsplus_rename(struct mnt_idmap *idmap, + struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry, + unsigned int flags) +{ + int res; + + if (flags & ~RENAME_NOREPLACE) + return -EINVAL; + + /* Unlink destination if it already exists */ + if (d_really_is_positive(new_dentry)) { + if (d_is_dir(new_dentry)) + res = hfsplus_rmdir(new_dir, new_dentry); + else + res = hfsplus_unlink(new_dir, new_dentry); + if (res) + return res; + } + + res = hfsplus_rename_cat((u32)(unsigned long)old_dentry->d_fsdata, + old_dir, &old_dentry->d_name, + new_dir, &new_dentry->d_name); + if (!res) { + struct inode *inode = d_inode(old_dentry); + + new_dentry->d_fsdata = old_dentry->d_fsdata; + + inode_set_ctime_current(inode); + mark_inode_dirty(inode); + + res = hfsplus_cat_write_inode(old_dir); + if (res) + return res; + + res = hfsplus_cat_write_inode(new_dir); + if (res) + return res; + + res = hfsplus_cat_write_inode(inode); + } + return res; +} + +const struct inode_operations hfsplus_dir_inode_operations = { + .lookup = hfsplus_lookup, + .create = hfsplus_create, + .link = hfsplus_link, + .unlink = hfsplus_unlink, + .mkdir = hfsplus_mkdir, + .rmdir = hfsplus_rmdir, + .symlink = hfsplus_symlink, + .mknod = hfsplus_mknod, + .rename = hfsplus_rename, + .getattr = hfsplus_getattr, + .listxattr = hfsplus_listxattr, + .fileattr_get = hfsplus_fileattr_get, + .fileattr_set = hfsplus_fileattr_set, +}; + +const struct file_operations hfsplus_dir_operations = { + .fsync = hfsplus_file_fsync, + .read = generic_read_dir, + .iterate_shared = hfsplus_readdir, + .unlocked_ioctl = hfsplus_ioctl, + .llseek = generic_file_llseek, + .release = hfsplus_dir_release, +}; diff --git a/fs/hfsplus/extents.c b/fs/hfsplus/extents.c new file mode 100644 index 000000000..eb7c11524 --- /dev/null +++ b/fs/hfsplus/extents.c @@ -0,0 +1,636 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hfsplus/extents.c + * + * Copyright (C) 2001 + * Brad Boyer (flar@allandria.com) + * (C) 2003 Ardis Technologies + * + * Handling of Extents both in catalog and extents overflow trees + */ + +#include +#include +#include + +#include "hfsplus_fs.h" +#include "hfsplus_raw.h" + +/* Compare two extents keys, returns 0 on same, pos/neg for difference */ +int hfsplus_ext_cmp_key(const hfsplus_btree_key *k1, + const hfsplus_btree_key *k2) +{ + __be32 k1id, k2id; + __be32 k1s, k2s; + + k1id = k1->ext.cnid; + k2id = k2->ext.cnid; + if (k1id != k2id) + return be32_to_cpu(k1id) < be32_to_cpu(k2id) ? -1 : 1; + + if (k1->ext.fork_type != k2->ext.fork_type) + return k1->ext.fork_type < k2->ext.fork_type ? -1 : 1; + + k1s = k1->ext.start_block; + k2s = k2->ext.start_block; + if (k1s == k2s) + return 0; + return be32_to_cpu(k1s) < be32_to_cpu(k2s) ? -1 : 1; +} + +static void hfsplus_ext_build_key(hfsplus_btree_key *key, u32 cnid, + u32 block, u8 type) +{ + key->key_len = cpu_to_be16(HFSPLUS_EXT_KEYLEN - 2); + key->ext.cnid = cpu_to_be32(cnid); + key->ext.start_block = cpu_to_be32(block); + key->ext.fork_type = type; + key->ext.pad = 0; +} + +static u32 hfsplus_ext_find_block(struct hfsplus_extent *ext, u32 off) +{ + int i; + u32 count; + + for (i = 0; i < 8; ext++, i++) { + count = be32_to_cpu(ext->block_count); + if (off < count) + return be32_to_cpu(ext->start_block) + off; + off -= count; + } + /* panic? */ + return 0; +} + +static int hfsplus_ext_block_count(struct hfsplus_extent *ext) +{ + int i; + u32 count = 0; + + for (i = 0; i < 8; ext++, i++) + count += be32_to_cpu(ext->block_count); + return count; +} + +static u32 hfsplus_ext_lastblock(struct hfsplus_extent *ext) +{ + int i; + + ext += 7; + for (i = 0; i < 7; ext--, i++) + if (ext->block_count) + break; + return be32_to_cpu(ext->start_block) + be32_to_cpu(ext->block_count); +} + +static int __hfsplus_ext_write_extent(struct inode *inode, + struct hfs_find_data *fd) +{ + struct hfsplus_inode_info *hip = HFSPLUS_I(inode); + int res; + + WARN_ON(!mutex_is_locked(&hip->extents_lock)); + + hfsplus_ext_build_key(fd->search_key, inode->i_ino, hip->cached_start, + HFSPLUS_IS_RSRC(inode) ? + HFSPLUS_TYPE_RSRC : HFSPLUS_TYPE_DATA); + + res = hfs_brec_find(fd, hfs_find_rec_by_key); + if (hip->extent_state & HFSPLUS_EXT_NEW) { + if (res != -ENOENT) + return res; + /* Fail early and avoid ENOSPC during the btree operation */ + res = hfs_bmap_reserve(fd->tree, fd->tree->depth + 1); + if (res) + return res; + hfs_brec_insert(fd, hip->cached_extents, + sizeof(hfsplus_extent_rec)); + hip->extent_state &= ~(HFSPLUS_EXT_DIRTY | HFSPLUS_EXT_NEW); + } else { + if (res) + return res; + if (fd->entrylength != sizeof(hfsplus_extent_rec)) + return -EIO; + hfs_bnode_write(fd->bnode, hip->cached_extents, + fd->entryoffset, fd->entrylength); + hip->extent_state &= ~HFSPLUS_EXT_DIRTY; + } + + /* + * We can't just use hfsplus_mark_inode_dirty here, because we + * also get called from hfsplus_write_inode, which should not + * redirty the inode. Instead the callers have to be careful + * to explicily mark the inode dirty, too. + */ + set_bit(HFSPLUS_I_EXT_DIRTY, + &HFSPLUS_I(HFSPLUS_EXT_TREE_I(inode->i_sb))->flags); + set_bit(HFSPLUS_I_EXT_DIRTY, &hip->flags); + + return 0; +} + +static int hfsplus_ext_write_extent_locked(struct inode *inode) +{ + int res = 0; + + if (HFSPLUS_I(inode)->extent_state & HFSPLUS_EXT_DIRTY) { + struct hfs_find_data fd; + + res = hfs_find_init(HFSPLUS_SB(inode->i_sb)->ext_tree, &fd); + if (res) + return res; + res = __hfsplus_ext_write_extent(inode, &fd); + hfs_find_exit(&fd); + } + return res; +} + +int hfsplus_ext_write_extent(struct inode *inode) +{ + int res; + + mutex_lock(&HFSPLUS_I(inode)->extents_lock); + res = hfsplus_ext_write_extent_locked(inode); + mutex_unlock(&HFSPLUS_I(inode)->extents_lock); + + return res; +} + +static inline int __hfsplus_ext_read_extent(struct hfs_find_data *fd, + struct hfsplus_extent *extent, + u32 cnid, u32 block, u8 type) +{ + int res; + + hfsplus_ext_build_key(fd->search_key, cnid, block, type); + fd->key->ext.cnid = 0; + res = hfs_brec_find(fd, hfs_find_rec_by_key); + if (res && res != -ENOENT) + return res; + if (fd->key->ext.cnid != fd->search_key->ext.cnid || + fd->key->ext.fork_type != fd->search_key->ext.fork_type) + return -ENOENT; + if (fd->entrylength != sizeof(hfsplus_extent_rec)) + return -EIO; + hfs_bnode_read(fd->bnode, extent, fd->entryoffset, + sizeof(hfsplus_extent_rec)); + return 0; +} + +static inline int __hfsplus_ext_cache_extent(struct hfs_find_data *fd, + struct inode *inode, u32 block) +{ + struct hfsplus_inode_info *hip = HFSPLUS_I(inode); + int res; + + WARN_ON(!mutex_is_locked(&hip->extents_lock)); + + if (hip->extent_state & HFSPLUS_EXT_DIRTY) { + res = __hfsplus_ext_write_extent(inode, fd); + if (res) + return res; + } + + res = __hfsplus_ext_read_extent(fd, hip->cached_extents, inode->i_ino, + block, HFSPLUS_IS_RSRC(inode) ? + HFSPLUS_TYPE_RSRC : + HFSPLUS_TYPE_DATA); + if (!res) { + hip->cached_start = be32_to_cpu(fd->key->ext.start_block); + hip->cached_blocks = + hfsplus_ext_block_count(hip->cached_extents); + } else { + hip->cached_start = hip->cached_blocks = 0; + hip->extent_state &= ~(HFSPLUS_EXT_DIRTY | HFSPLUS_EXT_NEW); + } + return res; +} + +static int hfsplus_ext_read_extent(struct inode *inode, u32 block) +{ + struct hfsplus_inode_info *hip = HFSPLUS_I(inode); + struct hfs_find_data fd; + int res; + + if (block >= hip->cached_start && + block < hip->cached_start + hip->cached_blocks) + return 0; + + res = hfs_find_init(HFSPLUS_SB(inode->i_sb)->ext_tree, &fd); + if (!res) { + res = __hfsplus_ext_cache_extent(&fd, inode, block); + hfs_find_exit(&fd); + } + return res; +} + +/* Get a block at iblock for inode, possibly allocating if create */ +int hfsplus_get_block(struct inode *inode, sector_t iblock, + struct buffer_head *bh_result, int create) +{ + struct super_block *sb = inode->i_sb; + struct hfsplus_sb_info *sbi = HFSPLUS_SB(sb); + struct hfsplus_inode_info *hip = HFSPLUS_I(inode); + int res = -EIO; + u32 ablock, dblock, mask; + sector_t sector; + int was_dirty = 0; + + /* Convert inode block to disk allocation block */ + ablock = iblock >> sbi->fs_shift; + + if (iblock >= hip->fs_blocks) { + if (!create) + return 0; + if (iblock > hip->fs_blocks) + return -EIO; + if (ablock >= hip->alloc_blocks) { + res = hfsplus_file_extend(inode, false); + if (res) + return res; + } + } else + create = 0; + + if (ablock < hip->first_blocks) { + dblock = hfsplus_ext_find_block(hip->first_extents, ablock); + goto done; + } + + if (inode->i_ino == HFSPLUS_EXT_CNID) + return -EIO; + + mutex_lock(&hip->extents_lock); + + /* + * hfsplus_ext_read_extent will write out a cached extent into + * the extents btree. In that case we may have to mark the inode + * dirty even for a pure read of an extent here. + */ + was_dirty = (hip->extent_state & HFSPLUS_EXT_DIRTY); + res = hfsplus_ext_read_extent(inode, ablock); + if (res) { + mutex_unlock(&hip->extents_lock); + return -EIO; + } + dblock = hfsplus_ext_find_block(hip->cached_extents, + ablock - hip->cached_start); + mutex_unlock(&hip->extents_lock); + +done: + hfs_dbg("ino %llu, iblock %llu - dblock %u\n", + inode->i_ino, (long long)iblock, dblock); + + mask = (1 << sbi->fs_shift) - 1; + sector = ((sector_t)dblock << sbi->fs_shift) + + sbi->blockoffset + (iblock & mask); + map_bh(bh_result, sb, sector); + + if (create) { + set_buffer_new(bh_result); + hip->phys_size += sb->s_blocksize; + hip->fs_blocks++; + inode_add_bytes(inode, sb->s_blocksize); + } + if (create || was_dirty) + mark_inode_dirty(inode); + return 0; +} + +static void hfsplus_dump_extent(struct hfsplus_extent *extent) +{ + int i; + + hfs_dbg("extent "); + for (i = 0; i < 8; i++) + hfs_dbg(" start_block %u, block_count %u", + be32_to_cpu(extent[i].start_block), + be32_to_cpu(extent[i].block_count)); + hfs_dbg("\n"); +} + +static int hfsplus_add_extent(struct hfsplus_extent *extent, u32 offset, + u32 alloc_block, u32 block_count) +{ + u32 count, start; + int i; + + hfsplus_dump_extent(extent); + for (i = 0; i < 8; extent++, i++) { + count = be32_to_cpu(extent->block_count); + if (offset == count) { + start = be32_to_cpu(extent->start_block); + if (alloc_block != start + count) { + if (++i >= 8) + return -ENOSPC; + extent++; + extent->start_block = cpu_to_be32(alloc_block); + } else + block_count += count; + extent->block_count = cpu_to_be32(block_count); + return 0; + } else if (offset < count) + break; + offset -= count; + } + /* panic? */ + return -EIO; +} + +static int hfsplus_free_extents(struct super_block *sb, + struct hfsplus_extent *extent, + u32 offset, u32 block_nr) +{ + u32 count, start; + int i; + int err = 0; + + hfsplus_dump_extent(extent); + for (i = 0; i < 8; extent++, i++) { + count = be32_to_cpu(extent->block_count); + if (offset == count) + goto found; + else if (offset < count) + break; + offset -= count; + } + /* panic? */ + return -EIO; +found: + for (;;) { + start = be32_to_cpu(extent->start_block); + if (count <= block_nr) { + err = hfsplus_block_free(sb, start, count); + if (err) { + pr_err("can't free extent: start %u, count %u\n", + start, count); + } + extent->block_count = 0; + extent->start_block = 0; + block_nr -= count; + } else { + count -= block_nr; + err = hfsplus_block_free(sb, start + count, block_nr); + if (err) { + pr_err("can't free extent: start %u, count %u\n", + start, count); + } + extent->block_count = cpu_to_be32(count); + block_nr = 0; + } + if (!block_nr || !i) { + /* + * Try to free all extents and + * return only last error + */ + return err; + } + i--; + extent--; + count = be32_to_cpu(extent->block_count); + } +} + +int hfsplus_free_fork(struct super_block *sb, u32 cnid, + struct hfsplus_fork_raw *fork, int type) +{ + struct hfs_find_data fd; + hfsplus_extent_rec ext_entry; + u32 total_blocks, blocks, start; + int res, i; + + total_blocks = be32_to_cpu(fork->total_blocks); + if (!total_blocks) + return 0; + + blocks = 0; + for (i = 0; i < 8; i++) + blocks += be32_to_cpu(fork->extents[i].block_count); + + res = hfsplus_free_extents(sb, fork->extents, blocks, blocks); + if (res) + return res; + if (total_blocks == blocks) + return 0; + + res = hfs_find_init(HFSPLUS_SB(sb)->ext_tree, &fd); + if (res) + return res; + do { + res = __hfsplus_ext_read_extent(&fd, ext_entry, cnid, + total_blocks, type); + if (res) + break; + start = be32_to_cpu(fd.key->ext.start_block); + hfs_brec_remove(&fd); + + mutex_unlock(&fd.tree->tree_lock); + hfsplus_free_extents(sb, ext_entry, total_blocks - start, + total_blocks); + total_blocks = start; + mutex_lock_nested(&fd.tree->tree_lock, + hfsplus_btree_lock_class(fd.tree)); + } while (total_blocks > blocks); + hfs_find_exit(&fd); + + return res; +} + +int hfsplus_file_extend(struct inode *inode, bool zeroout) +{ + struct super_block *sb = inode->i_sb; + struct hfsplus_sb_info *sbi = HFSPLUS_SB(sb); + struct hfsplus_inode_info *hip = HFSPLUS_I(inode); + u32 start, len, goal; + int res; + + if (sbi->alloc_file->i_size * 8 < + sbi->total_blocks - sbi->free_blocks + 8) { + /* extend alloc file */ + pr_err_ratelimited("extend alloc file! (%llu,%u,%u)\n", + sbi->alloc_file->i_size * 8, + sbi->total_blocks, sbi->free_blocks); + return -ENOSPC; + } + + mutex_lock(&hip->extents_lock); + if (hip->alloc_blocks == hip->first_blocks) + goal = hfsplus_ext_lastblock(hip->first_extents); + else { + res = hfsplus_ext_read_extent(inode, hip->alloc_blocks); + if (res) + goto out; + goal = hfsplus_ext_lastblock(hip->cached_extents); + } + + len = hip->clump_blocks; + start = hfsplus_block_allocate(sb, sbi->total_blocks, goal, &len); + if (start >= sbi->total_blocks) { + start = hfsplus_block_allocate(sb, goal, 0, &len); + if (start >= goal) { + res = -ENOSPC; + goto out; + } + } + + if (zeroout) { + res = sb_issue_zeroout(sb, start, len, GFP_NOFS); + if (res) + goto out; + } + + hfs_dbg("ino %llu, start %u, len %u\n", inode->i_ino, start, len); + + if (hip->alloc_blocks <= hip->first_blocks) { + if (!hip->first_blocks) { + hfs_dbg("first_extent: start %u, len %u\n", + start, len); + /* no extents yet */ + hip->first_extents[0].start_block = cpu_to_be32(start); + hip->first_extents[0].block_count = cpu_to_be32(len); + res = 0; + } else { + /* try to append to extents in inode */ + res = hfsplus_add_extent(hip->first_extents, + hip->alloc_blocks, + start, len); + if (res == -ENOSPC) + goto insert_extent; + } + if (!res) { + hfsplus_dump_extent(hip->first_extents); + hip->first_blocks += len; + } + } else { + res = hfsplus_add_extent(hip->cached_extents, + hip->alloc_blocks - hip->cached_start, + start, len); + if (!res) { + hfsplus_dump_extent(hip->cached_extents); + hip->extent_state |= HFSPLUS_EXT_DIRTY; + hip->cached_blocks += len; + } else if (res == -ENOSPC) + goto insert_extent; + } +out: + if (!res) { + hip->alloc_blocks += len; + mutex_unlock(&hip->extents_lock); + hfsplus_mark_inode_dirty(HFSPLUS_SB(sb)->alloc_file, + HFSPLUS_I_ALLOC_DIRTY); + hfsplus_mark_inode_dirty(inode, HFSPLUS_I_ALLOC_DIRTY); + return 0; + } + mutex_unlock(&hip->extents_lock); + return res; + +insert_extent: + hfs_dbg("insert new extent\n"); + res = hfsplus_ext_write_extent_locked(inode); + if (res) + goto out; + + memset(hip->cached_extents, 0, sizeof(hfsplus_extent_rec)); + hip->cached_extents[0].start_block = cpu_to_be32(start); + hip->cached_extents[0].block_count = cpu_to_be32(len); + hfsplus_dump_extent(hip->cached_extents); + hip->extent_state |= HFSPLUS_EXT_DIRTY | HFSPLUS_EXT_NEW; + hip->cached_start = hip->alloc_blocks; + hip->cached_blocks = len; + + res = 0; + goto out; +} + +void hfsplus_file_truncate(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + struct hfsplus_inode_info *hip = HFSPLUS_I(inode); + struct hfs_find_data fd; + u32 alloc_cnt, blk_cnt, start; + int res; + + hfs_dbg("ino %llu, phys_size %llu -> i_size %llu\n", + inode->i_ino, (long long)hip->phys_size, inode->i_size); + + if (inode->i_size > hip->phys_size) { + struct address_space *mapping = inode->i_mapping; + struct folio *folio; + void *fsdata = NULL; + loff_t size = inode->i_size; + + res = hfsplus_write_begin(NULL, mapping, size, 0, + &folio, &fsdata); + if (res) + return; + res = generic_write_end(NULL, mapping, size, 0, 0, + folio, fsdata); + if (res < 0) + return; + mark_inode_dirty(inode); + return; + } else if (inode->i_size == hip->phys_size) + return; + + blk_cnt = (inode->i_size + HFSPLUS_SB(sb)->alloc_blksz - 1) >> + HFSPLUS_SB(sb)->alloc_blksz_shift; + + mutex_lock(&hip->extents_lock); + + alloc_cnt = hip->alloc_blocks; + if (blk_cnt == alloc_cnt) + goto out_unlock; + + res = hfs_find_init(HFSPLUS_SB(sb)->ext_tree, &fd); + if (res) { + mutex_unlock(&hip->extents_lock); + /* XXX: We lack error handling of hfsplus_file_truncate() */ + return; + } + + while (1) { + if (alloc_cnt == hip->first_blocks) { + mutex_unlock(&fd.tree->tree_lock); + hfsplus_free_extents(sb, hip->first_extents, + alloc_cnt, alloc_cnt - blk_cnt); + hfsplus_dump_extent(hip->first_extents); + hip->first_blocks = blk_cnt; + mutex_lock_nested(&fd.tree->tree_lock, + hfsplus_btree_lock_class(fd.tree)); + break; + } + res = __hfsplus_ext_cache_extent(&fd, inode, alloc_cnt); + if (res) + break; + + start = hip->cached_start; + if (blk_cnt <= start) + hfs_brec_remove(&fd); + mutex_unlock(&fd.tree->tree_lock); + hfsplus_free_extents(sb, hip->cached_extents, + alloc_cnt - start, alloc_cnt - blk_cnt); + hfsplus_dump_extent(hip->cached_extents); + mutex_lock_nested(&fd.tree->tree_lock, + hfsplus_btree_lock_class(fd.tree)); + if (blk_cnt > start) { + hip->extent_state |= HFSPLUS_EXT_DIRTY; + break; + } + alloc_cnt = start; + hip->cached_start = hip->cached_blocks = 0; + hip->extent_state &= ~(HFSPLUS_EXT_DIRTY | HFSPLUS_EXT_NEW); + } + hfs_find_exit(&fd); + + hip->alloc_blocks = blk_cnt; +out_unlock: + mutex_unlock(&hip->extents_lock); + hip->phys_size = inode->i_size; + hip->fs_blocks = (inode->i_size + sb->s_blocksize - 1) >> + sb->s_blocksize_bits; + inode_set_bytes(inode, hip->fs_blocks << sb->s_blocksize_bits); + hfsplus_mark_inode_dirty(HFSPLUS_SB(sb)->alloc_file, + HFSPLUS_I_ALLOC_DIRTY); + hfsplus_mark_inode_dirty(inode, HFSPLUS_I_ALLOC_DIRTY); +} diff --git a/fs/hfsplus/hfsplus_fs.h b/fs/hfsplus/hfsplus_fs.h new file mode 100644 index 000000000..1e5b58e6a --- /dev/null +++ b/fs/hfsplus/hfsplus_fs.h @@ -0,0 +1,706 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * linux/include/linux/hfsplus_fs.h + * + * Copyright (C) 1999 + * Brad Boyer (flar@pants.nu) + * (C) 2003 Ardis Technologies + * + */ + +#ifndef _LINUX_HFSPLUS_FS_H +#define _LINUX_HFSPLUS_FS_H + +#include +#include +#include +#include +#include +#include "hfsplus_raw.h" + +/* Runtime config options */ +#define HFSPLUS_DEF_CR_TYPE 0x3F3F3F3F /* '????' */ + +#define HFSPLUS_TYPE_DATA 0x00 +#define HFSPLUS_TYPE_RSRC 0xFF + +typedef int (*btree_keycmp)(const hfsplus_btree_key *, + const hfsplus_btree_key *); + +#define NODE_HASH_SIZE 256 + +/* B-tree mutex nested subclasses */ +enum hfsplus_btree_mutex_classes { + CATALOG_BTREE_MUTEX, + EXTENTS_BTREE_MUTEX, + ATTR_BTREE_MUTEX, +}; + +/* An HFS+ BTree held in memory */ +struct hfs_btree { + struct super_block *sb; + struct inode *inode; + btree_keycmp keycmp; + + u32 cnid; + u32 root; + u32 leaf_count; + u32 leaf_head; + u32 leaf_tail; + u32 node_count; + u32 free_nodes; + u32 attributes; + + unsigned int node_size; + unsigned int node_size_shift; + unsigned int max_key_len; + unsigned int depth; + + struct mutex tree_lock; + + unsigned int pages_per_bnode; + spinlock_t hash_lock; + struct hfs_bnode *node_hash[NODE_HASH_SIZE]; + int node_hash_cnt; +}; + +struct page; + +/* An HFS+ BTree node in memory */ +struct hfs_bnode { + struct hfs_btree *tree; + + u32 prev; + u32 this; + u32 next; + u32 parent; + + u16 num_recs; + u8 type; + u8 height; + + struct hfs_bnode *next_hash; + unsigned long flags; + wait_queue_head_t lock_wq; + atomic_t refcnt; + unsigned int page_offset; + struct page *page[]; +}; + +#define HFS_BNODE_LOCK 0 +#define HFS_BNODE_ERROR 1 +#define HFS_BNODE_NEW 2 +#define HFS_BNODE_DIRTY 3 +#define HFS_BNODE_DELETED 4 + +/* + * Attributes file states + */ +#define HFSPLUS_EMPTY_ATTR_TREE 0 +#define HFSPLUS_CREATING_ATTR_TREE 1 +#define HFSPLUS_VALID_ATTR_TREE 2 +#define HFSPLUS_FAILED_ATTR_TREE 3 + +/* + * HFS+ superblock info (built from Volume Header on disk) + */ + +struct hfsplus_vh; +struct hfs_btree; + +struct hfsplus_sb_info { + void *s_vhdr_buf; + struct hfsplus_vh *s_vhdr; + void *s_backup_vhdr_buf; + struct hfsplus_vh *s_backup_vhdr; + struct hfs_btree *ext_tree; + struct hfs_btree *cat_tree; + struct hfs_btree *attr_tree; + atomic_t attr_tree_state; + struct inode *alloc_file; + struct inode *hidden_dir; + struct nls_table *nls; + + /* Runtime variables */ + u32 blockoffset; + u32 min_io_size; + sector_t part_start; + sector_t sect_count; + int fs_shift; + + /* immutable data from the volume header */ + u32 alloc_blksz; + int alloc_blksz_shift; + u32 total_blocks; + u32 data_clump_blocks, rsrc_clump_blocks; + + /* mutable data from the volume header, protected by alloc_mutex */ + u32 free_blocks; + struct mutex alloc_mutex; + + /* mutable data from the volume header, protected by vh_mutex */ + u32 next_cnid; + u32 file_count; + u32 folder_count; + struct mutex vh_mutex; + + /* Config options */ + u32 creator; + u32 type; + + umode_t umask; + kuid_t uid; + kgid_t gid; + + int part, session; + unsigned long flags; + + int work_queued; /* non-zero delayed work is queued */ + struct delayed_work sync_work; /* FS sync delayed work */ + spinlock_t work_lock; /* protects sync_work and work_queued */ + struct rcu_head rcu; +}; + +#define HFSPLUS_SB_WRITEBACKUP 0 +#define HFSPLUS_SB_NODECOMPOSE 1 +#define HFSPLUS_SB_FORCE 2 +#define HFSPLUS_SB_HFSX 3 +#define HFSPLUS_SB_CASEFOLD 4 +#define HFSPLUS_SB_NOBARRIER 5 +#define HFSPLUS_SB_UID 6 +#define HFSPLUS_SB_GID 7 + +static inline struct hfsplus_sb_info *HFSPLUS_SB(struct super_block *sb) +{ + return sb->s_fs_info; +} + + +struct hfsplus_inode_info { + atomic_t opencnt; + + /* + * Extent allocation information, protected by extents_lock. + */ + u32 first_blocks; + u32 clump_blocks; + u32 alloc_blocks; + u32 cached_start; + u32 cached_blocks; + hfsplus_extent_rec first_extents; + hfsplus_extent_rec cached_extents; + unsigned int extent_state; + struct mutex extents_lock; + + /* + * Immutable data. + */ + struct inode *rsrc_inode; + __be32 create_date; + + /* + * Protected by sbi->vh_mutex. + */ + u32 linkid; + + /* + * Accessed using atomic bitops. + */ + unsigned long flags; + + /* + * Protected by i_mutex. + */ + sector_t fs_blocks; + u8 userflags; /* BSD user file flags */ + u32 subfolders; /* Subfolder count (HFSX only) */ + loff_t phys_size; + + struct inode vfs_inode; +}; + +#define HFSPLUS_EXT_DIRTY 0x0001 +#define HFSPLUS_EXT_NEW 0x0002 + +#define HFSPLUS_I_RSRC 0 /* represents a resource fork */ +#define HFSPLUS_I_CAT_DIRTY 1 /* has changes in the catalog tree */ +#define HFSPLUS_I_EXT_DIRTY 2 /* has changes in the extent tree */ +#define HFSPLUS_I_ALLOC_DIRTY 3 /* has changes in the allocation file */ +#define HFSPLUS_I_ATTR_DIRTY 4 /* has changes in the attributes tree */ + +#define HFSPLUS_IS_RSRC(inode) \ + test_bit(HFSPLUS_I_RSRC, &HFSPLUS_I(inode)->flags) + +static inline struct hfsplus_inode_info *HFSPLUS_I(struct inode *inode) +{ + return container_of(inode, struct hfsplus_inode_info, vfs_inode); +} + +#define HFSPLUS_CAT_TREE_I(sb) \ + HFSPLUS_SB(sb)->cat_tree->inode +#define HFSPLUS_EXT_TREE_I(sb) \ + HFSPLUS_SB(sb)->ext_tree->inode +#define HFSPLUS_ATTR_TREE_I(sb) \ + HFSPLUS_SB(sb)->attr_tree->inode + +/* + * Mark an inode dirty, and also mark the btree in which the + * specific type of metadata is stored. + * For data or metadata that gets written back by into the catalog btree + * by hfsplus_write_inode a plain mark_inode_dirty call is enough. + */ +static inline void hfsplus_mark_inode_dirty(struct inode *inode, + unsigned int flag) +{ + set_bit(flag, &HFSPLUS_I(inode)->flags); + mark_inode_dirty(inode); +} + +struct hfs_find_data { + /* filled by caller */ + hfsplus_btree_key *search_key; + hfsplus_btree_key *key; + /* filled by find */ + struct hfs_btree *tree; + struct hfs_bnode *bnode; + /* filled by findrec */ + int record; + int keyoffset, keylength; + int entryoffset, entrylength; +}; + +struct hfsplus_readdir_data { + loff_t pos; + struct hfsplus_cat_key key; +}; + +/* + * Find minimum acceptible I/O size for an hfsplus sb. + */ +static inline unsigned short hfsplus_min_io_size(struct super_block *sb) +{ + return max_t(unsigned short, HFSPLUS_SB(sb)->min_io_size, + HFSPLUS_SECTOR_SIZE); +} + +#define hfs_btree_open hfsplus_btree_open +#define hfs_btree_close hfsplus_btree_close +#define hfs_btree_write hfsplus_btree_write +#define hfs_bmap_reserve hfsplus_bmap_reserve +#define hfs_bmap_alloc hfsplus_bmap_alloc +#define hfs_bmap_free hfsplus_bmap_free +#define hfs_bnode_read hfsplus_bnode_read +#define hfs_bnode_read_u16 hfsplus_bnode_read_u16 +#define hfs_bnode_read_u8 hfsplus_bnode_read_u8 +#define hfs_bnode_read_key hfsplus_bnode_read_key +#define hfs_bnode_write hfsplus_bnode_write +#define hfs_bnode_write_u16 hfsplus_bnode_write_u16 +#define hfs_bnode_clear hfsplus_bnode_clear +#define hfs_bnode_copy hfsplus_bnode_copy +#define hfs_bnode_move hfsplus_bnode_move +#define hfs_bnode_dump hfsplus_bnode_dump +#define hfs_bnode_unlink hfsplus_bnode_unlink +#define hfs_bnode_findhash hfsplus_bnode_findhash +#define hfs_bnode_find hfsplus_bnode_find +#define hfs_bnode_unhash hfsplus_bnode_unhash +#define hfs_bnode_free hfsplus_bnode_free +#define hfs_bnode_create hfsplus_bnode_create +#define hfs_bnode_get hfsplus_bnode_get +#define hfs_bnode_put hfsplus_bnode_put +#define hfs_brec_lenoff hfsplus_brec_lenoff +#define hfs_brec_keylen hfsplus_brec_keylen +#define hfs_brec_insert hfsplus_brec_insert +#define hfs_brec_remove hfsplus_brec_remove +#define hfs_find_init hfsplus_find_init +#define hfs_find_exit hfsplus_find_exit +#define __hfs_brec_find __hfsplus_brec_find +#define hfs_brec_find hfsplus_brec_find +#define hfs_brec_read hfsplus_brec_read +#define hfs_brec_goto hfsplus_brec_goto +#define hfs_part_find hfsplus_part_find + +/* + * hfs+-specific ioctl for making the filesystem bootable + */ +#define HFSPLUS_IOC_BLESS _IO('h', 0x80) + +typedef int (*search_strategy_t)(struct hfs_bnode *, + struct hfs_find_data *, + int *, int *, int *); + +/* + * Functions in any *.c used in other files + */ + +/* attributes.c */ +int __init hfsplus_create_attr_tree_cache(void); +void hfsplus_destroy_attr_tree_cache(void); +int hfsplus_attr_bin_cmp_key(const hfsplus_btree_key *k1, + const hfsplus_btree_key *k2); +int hfsplus_attr_build_key(struct super_block *sb, hfsplus_btree_key *key, + u32 cnid, const char *name); +hfsplus_attr_entry *hfsplus_alloc_attr_entry(void); +void hfsplus_destroy_attr_entry(hfsplus_attr_entry *entry); +int hfsplus_find_attr(struct super_block *sb, u32 cnid, const char *name, + struct hfs_find_data *fd); +int hfsplus_attr_exists(struct inode *inode, const char *name); +int hfsplus_create_attr(struct inode *inode, const char *name, + const void *value, size_t size); +int hfsplus_delete_attr(struct inode *inode, const char *name); +int hfsplus_delete_all_attrs(struct inode *dir, u32 cnid); +int hfsplus_replace_attr(struct inode *inode, + const char *name, + const void *value, size_t size); + +/* bitmap.c */ +int hfsplus_block_allocate(struct super_block *sb, u32 size, u32 offset, + u32 *max); +int hfsplus_block_free(struct super_block *sb, u32 offset, u32 count); + +/* btree.c */ +u32 hfsplus_calc_btree_clump_size(u32 block_size, u32 node_size, u64 sectors, + int file_id); +struct hfs_btree *hfs_btree_open(struct super_block *sb, u32 id); +void hfs_btree_close(struct hfs_btree *tree); +int hfs_btree_write(struct hfs_btree *tree); +int hfs_bmap_reserve(struct hfs_btree *tree, u32 rsvd_nodes); +struct hfs_bnode *hfs_bmap_alloc(struct hfs_btree *tree); +void hfs_bmap_free(struct hfs_bnode *node); + +/* bnode.c */ +void hfs_bnode_read(struct hfs_bnode *node, void *buf, u32 off, u32 len); +u16 hfs_bnode_read_u16(struct hfs_bnode *node, u32 off); +u8 hfs_bnode_read_u8(struct hfs_bnode *node, u32 off); +void hfs_bnode_read_key(struct hfs_bnode *node, void *key, u32 off); +void hfs_bnode_write(struct hfs_bnode *node, void *buf, u32 off, u32 len); +void hfs_bnode_write_u16(struct hfs_bnode *node, u32 off, u16 data); +void hfs_bnode_clear(struct hfs_bnode *node, u32 off, u32 len); +void hfs_bnode_copy(struct hfs_bnode *dst_node, u32 dst, + struct hfs_bnode *src_node, u32 src, u32 len); +void hfs_bnode_move(struct hfs_bnode *node, u32 dst, u32 src, u32 len); +void hfs_bnode_dump(struct hfs_bnode *node); +void hfs_bnode_unlink(struct hfs_bnode *node); +struct hfs_bnode *hfs_bnode_findhash(struct hfs_btree *tree, u32 cnid); +void hfs_bnode_unhash(struct hfs_bnode *node); +struct hfs_bnode *hfs_bnode_find(struct hfs_btree *tree, u32 num); +void hfs_bnode_free(struct hfs_bnode *node); +struct hfs_bnode *hfs_bnode_create(struct hfs_btree *tree, u32 num); +void hfs_bnode_get(struct hfs_bnode *node); +void hfs_bnode_put(struct hfs_bnode *node); +bool hfs_bnode_need_zeroout(struct hfs_btree *tree); + +/* brec.c */ +u16 hfs_brec_lenoff(struct hfs_bnode *node, u16 rec, u16 *off); +u16 hfs_brec_keylen(struct hfs_bnode *node, u16 rec); +int hfs_brec_insert(struct hfs_find_data *fd, void *entry, u32 entry_len); +int hfs_brec_remove(struct hfs_find_data *fd); + +/* bfind.c */ +int hfs_find_init(struct hfs_btree *tree, struct hfs_find_data *fd); +void hfs_find_exit(struct hfs_find_data *fd); +int hfs_find_1st_rec_by_cnid(struct hfs_bnode *bnode, struct hfs_find_data *fd, + int *begin, int *end, int *cur_rec); +int hfs_find_rec_by_key(struct hfs_bnode *bnode, struct hfs_find_data *fd, + int *begin, int *end, int *cur_rec); +int __hfs_brec_find(struct hfs_bnode *bnode, struct hfs_find_data *fd, + search_strategy_t rec_found); +int hfs_brec_find(struct hfs_find_data *fd, search_strategy_t do_key_compare); +int hfs_brec_read(struct hfs_find_data *fd, void *rec, u32 rec_len); +int hfs_brec_goto(struct hfs_find_data *fd, int cnt); + +/* catalog.c */ +int hfsplus_cat_case_cmp_key(const hfsplus_btree_key *k1, + const hfsplus_btree_key *k2); +int hfsplus_cat_bin_cmp_key(const hfsplus_btree_key *k1, + const hfsplus_btree_key *k2); +int hfsplus_cat_build_key(struct super_block *sb, hfsplus_btree_key *key, + u32 parent, const struct qstr *str); +void hfsplus_cat_build_key_with_cnid(struct super_block *sb, + hfsplus_btree_key *key, u32 parent); +void hfsplus_cat_set_perms(struct inode *inode, struct hfsplus_perm *perms); +int hfsplus_find_cat(struct super_block *sb, u32 cnid, + struct hfs_find_data *fd); +int hfsplus_create_cat(u32 cnid, struct inode *dir, const struct qstr *str, + struct inode *inode); +int hfsplus_delete_cat(u32 cnid, struct inode *dir, const struct qstr *str); +int hfsplus_rename_cat(u32 cnid, struct inode *src_dir, const struct qstr *src_name, + struct inode *dst_dir, const struct qstr *dst_name); + +/* dir.c */ +extern const struct inode_operations hfsplus_dir_inode_operations; +extern const struct file_operations hfsplus_dir_operations; + +/* extents.c */ +int hfsplus_ext_cmp_key(const hfsplus_btree_key *k1, + const hfsplus_btree_key *k2); +int hfsplus_ext_write_extent(struct inode *inode); +int hfsplus_get_block(struct inode *inode, sector_t iblock, + struct buffer_head *bh_result, int create); +int hfsplus_free_fork(struct super_block *sb, u32 cnid, + struct hfsplus_fork_raw *fork, int type); +int hfsplus_file_extend(struct inode *inode, bool zeroout); +void hfsplus_file_truncate(struct inode *inode); + +/* inode.c */ +extern const struct address_space_operations hfsplus_aops; +extern const struct address_space_operations hfsplus_btree_aops; +extern const struct dentry_operations hfsplus_dentry_operations; + +int hfsplus_write_begin(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, struct folio **foliop, + void **fsdata); +struct inode *hfsplus_new_inode(struct super_block *sb, struct inode *dir, + umode_t mode); +void hfsplus_delete_inode(struct inode *inode); +void hfsplus_inode_read_fork(struct inode *inode, + struct hfsplus_fork_raw *fork); +void hfsplus_inode_write_fork(struct inode *inode, + struct hfsplus_fork_raw *fork); +int hfsplus_cat_read_inode(struct inode *inode, struct hfs_find_data *fd); +int hfsplus_cat_write_inode(struct inode *inode); +int hfsplus_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, + unsigned int query_flags); +int hfsplus_file_fsync(struct file *file, loff_t start, loff_t end, + int datasync); +int hfsplus_fileattr_get(struct dentry *dentry, struct file_kattr *fa); +int hfsplus_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa); + +/* ioctl.c */ +long hfsplus_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); + +/* options.c */ +void hfsplus_fill_defaults(struct hfsplus_sb_info *opts); +int hfsplus_parse_param(struct fs_context *fc, struct fs_parameter *param); +int hfsplus_show_options(struct seq_file *seq, struct dentry *root); + +/* part_tbl.c */ +int hfs_part_find(struct super_block *sb, sector_t *part_start, + sector_t *part_size); + +/* super.c */ +struct inode *hfsplus_iget(struct super_block *sb, unsigned long ino); +void hfsplus_mark_mdb_dirty(struct super_block *sb); +void hfsplus_prepare_volume_header_for_commit(struct hfsplus_vh *vhdr); +int hfsplus_commit_superblock(struct super_block *sb); + +/* tables.c */ +extern u16 hfsplus_case_fold_table[]; +extern u16 hfsplus_decompose_table[]; +extern u16 hfsplus_compose_table[]; + +/* unicode.c */ +int hfsplus_strcasecmp(const struct hfsplus_unistr *s1, + const struct hfsplus_unistr *s2); +int hfsplus_strcmp(const struct hfsplus_unistr *s1, + const struct hfsplus_unistr *s2); +int hfsplus_uni2asc_str(struct super_block *sb, + const struct hfsplus_unistr *ustr, char *astr, + int *len_p); +int hfsplus_uni2asc_xattr_str(struct super_block *sb, + const struct hfsplus_attr_unistr *ustr, + char *astr, int *len_p); +int hfsplus_asc2uni(struct super_block *sb, struct hfsplus_unistr *ustr, + int max_unistr_len, const char *astr, int len, + int name_type); +int hfsplus_hash_dentry(const struct dentry *dentry, struct qstr *str); +int hfsplus_compare_dentry(const struct dentry *dentry, unsigned int len, + const char *str, const struct qstr *name); + +/* wrapper.c */ +int hfsplus_submit_bio(struct super_block *sb, sector_t sector, void *buf, + void **data, blk_opf_t opf); +int hfsplus_read_wrapper(struct super_block *sb); + +static inline u32 hfsplus_cat_thread_size(const struct hfsplus_cat_thread *thread) +{ + return offsetof(struct hfsplus_cat_thread, nodeName) + + offsetof(struct hfsplus_unistr, unicode) + + be16_to_cpu(thread->nodeName.length) * sizeof(hfsplus_unichr); +} + +static inline +bool is_hfs_thread_record_type(u16 type) +{ + return type == HFSPLUS_FOLDER_THREAD || type == HFSPLUS_FILE_THREAD; +} + +int hfsplus_brec_read_cat(struct hfs_find_data *fd, hfsplus_cat_entry *entry); + +/* + * time helpers: convert between 1904-base and 1970-base timestamps + * + * HFS+ implementations are highly inconsistent, this one matches the + * traditional behavior of 64-bit Linux, giving the most useful + * time range between 1970 and 2106, by treating any on-disk timestamp + * under HFSPLUS_UTC_OFFSET (Jan 1 1970) as a time between 2040 and 2106. + */ +#define HFSPLUS_UTC_OFFSET 2082844800U + +static inline time64_t __hfsp_mt2ut(__be32 mt) +{ + time64_t ut = (u32)(be32_to_cpu(mt) - HFSPLUS_UTC_OFFSET); + + return ut; +} + +static inline __be32 __hfsp_ut2mt(time64_t ut) +{ + return cpu_to_be32(lower_32_bits(ut) + HFSPLUS_UTC_OFFSET); +} + +static inline enum hfsplus_btree_mutex_classes +hfsplus_btree_lock_class(struct hfs_btree *tree) +{ + enum hfsplus_btree_mutex_classes class; + + switch (tree->cnid) { + case HFSPLUS_CAT_CNID: + class = CATALOG_BTREE_MUTEX; + break; + case HFSPLUS_EXT_CNID: + class = EXTENTS_BTREE_MUTEX; + break; + case HFSPLUS_ATTR_CNID: + class = ATTR_BTREE_MUTEX; + break; + default: + BUG(); + } + return class; +} + +static inline +bool is_bnode_offset_valid(struct hfs_bnode *node, u32 off) +{ + bool is_valid; + + if (!node || !node->tree) + return false; + + is_valid = off < node->tree->node_size; + + if (!is_valid) { + pr_err("requested invalid offset: " + "NODE: id %u, type %#x, height %u, " + "node_size %u, offset %u\n", + node->this, node->type, node->height, + node->tree->node_size, off); + } + + return is_valid; +} + +static inline +bool hfs_bnode_num_recs_invalid(struct hfs_bnode *node) +{ + u32 node_size; + u32 table_size; + u32 area_size; + u32 rec_size = sizeof(__be16); + u32 desc_size = sizeof(struct hfs_bnode_desc); + + if (!node || !node->tree) + return true; + + node_size = node->tree->node_size; + if (node_size < desc_size) + return true; + + area_size = node_size - desc_size; + table_size = ((u32)node->num_recs + 1) * rec_size; + + return table_size > area_size; +} + +static inline +bool hfs_brec_record_invalid(struct hfs_bnode *node, int record) +{ + if (hfs_bnode_num_recs_invalid(node)) + return true; + if (record < 0) + return true; + + return record >= node->num_recs; +} + +static inline +bool hfs_brec_offsets_invalid(struct hfs_bnode *node, u16 off, u16 next_off) +{ + u32 table_size; + u32 table_start; + u32 rec_size = sizeof(__be16); + u32 desc_size = sizeof(struct hfs_bnode_desc); + + if (!node || !node->tree) + return true; + + if (off < desc_size || (off & 1)) + return true; + + if (next_off <= off || + next_off > node->tree->node_size || + (next_off & 1)) + return true; + + table_size = ((u32)node->num_recs + 1) * rec_size; + table_start = node->tree->node_size - table_size; + if (next_off > table_start) + return true; + + return false; +} + +static inline +bool hfs_brec_len_invalid(struct hfs_bnode *node, u16 len) +{ + if (!node || !node->tree) + return true; + + return len == 0 || len > node->tree->node_size; +} + +static inline +void hfs_find_result_init(struct hfs_find_data *fd) +{ + fd->record = -1; + fd->keyoffset = -1; + fd->keylength = -1; + fd->entryoffset = -1; + fd->entrylength = -1; +} + +static inline +u32 check_and_correct_requested_length(struct hfs_bnode *node, u32 off, u32 len) +{ + unsigned int node_size; + + if (!is_bnode_offset_valid(node, off)) + return 0; + + node_size = node->tree->node_size; + + if ((u64)off + len > node_size) { + u32 new_len = node_size - off; + + pr_err("requested length has been corrected: " + "NODE: id %u, type %#x, height %u, " + "node_size %u, offset %u, " + "requested_len %u, corrected_len %u\n", + node->this, node->type, node->height, + node->tree->node_size, off, len, new_len); + + return new_len; + } + + return len; +} + +/* compatibility */ +#define hfsp_mt2ut(t) (struct timespec64){ .tv_sec = __hfsp_mt2ut(t) } +#define hfsp_ut2mt(t) __hfsp_ut2mt((t).tv_sec) +#define hfsp_now2mt() __hfsp_ut2mt(ktime_get_real_seconds()) + +#endif diff --git a/fs/hfsplus/hfsplus_raw.h b/fs/hfsplus/hfsplus_raw.h new file mode 100644 index 000000000..83b5dbde9 --- /dev/null +++ b/fs/hfsplus/hfsplus_raw.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * linux/include/linux/hfsplus_raw.h + * + * Copyright (C) 1999 + * Brad Boyer (flar@pants.nu) + * (C) 2003 Ardis Technologies + * + * Format of structures on disk + * Information taken from Apple Technote #1150 (HFS Plus Volume Format) + * + */ + +#ifndef _LINUX_HFSPLUS_RAW_H +#define _LINUX_HFSPLUS_RAW_H + +#include +#include + +#endif diff --git a/fs/hfsplus/inode.c b/fs/hfsplus/inode.c new file mode 100644 index 000000000..2ce6de574 --- /dev/null +++ b/fs/hfsplus/inode.c @@ -0,0 +1,842 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hfsplus/inode.c + * + * Copyright (C) 2001 + * Brad Boyer (flar@allandria.com) + * (C) 2003 Ardis Technologies + * + * Inode handling routines + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hfsplus_fs.h" +#include "hfsplus_raw.h" +#include "xattr.h" + +static int hfsplus_read_folio(struct file *file, struct folio *folio) +{ + return block_read_full_folio(folio, hfsplus_get_block); +} + +static void hfsplus_write_failed(struct address_space *mapping, loff_t to) +{ + struct inode *inode = mapping->host; + + if (to > inode->i_size) { + truncate_pagecache(inode, inode->i_size); + hfsplus_file_truncate(inode); + } +} + +int hfsplus_write_begin(const struct kiocb *iocb, + struct address_space *mapping, loff_t pos, + unsigned len, struct folio **foliop, + void **fsdata) +{ + struct inode *inode = mapping->host; + struct hfsplus_sb_info *sbi = HFSPLUS_SB(inode->i_sb); + loff_t total_capacity; + int ret; + + total_capacity = (loff_t)sbi->total_blocks << sbi->alloc_blksz_shift; + + if (pos >= total_capacity) + return -EFBIG; + + ret = cont_write_begin(iocb, mapping, pos, len, foliop, fsdata, + hfsplus_get_block, + &HFSPLUS_I(inode)->phys_size); + if (unlikely(ret)) + hfsplus_write_failed(mapping, pos + len); + + return ret; +} + +static sector_t hfsplus_bmap(struct address_space *mapping, sector_t block) +{ + return generic_block_bmap(mapping, block, hfsplus_get_block); +} + +static bool hfsplus_release_folio(struct folio *folio, gfp_t mask) +{ + struct inode *inode = folio->mapping->host; + struct super_block *sb = inode->i_sb; + struct hfs_btree *tree; + struct hfs_bnode *node; + u32 nidx; + int i; + bool res = true; + + switch (inode->i_ino) { + case HFSPLUS_EXT_CNID: + tree = HFSPLUS_SB(sb)->ext_tree; + break; + case HFSPLUS_CAT_CNID: + tree = HFSPLUS_SB(sb)->cat_tree; + break; + case HFSPLUS_ATTR_CNID: + tree = HFSPLUS_SB(sb)->attr_tree; + break; + default: + BUG(); + return false; + } + if (!tree) + return false; + if (tree->node_size >= PAGE_SIZE) { + nidx = folio->index >> + (tree->node_size_shift - PAGE_SHIFT); + spin_lock(&tree->hash_lock); + node = hfs_bnode_findhash(tree, nidx); + if (!node) + ; + else if (atomic_read(&node->refcnt)) + res = false; + if (res && node) { + hfs_bnode_unhash(node); + hfs_bnode_free(node); + } + spin_unlock(&tree->hash_lock); + } else { + nidx = folio->index << + (PAGE_SHIFT - tree->node_size_shift); + i = 1 << (PAGE_SHIFT - tree->node_size_shift); + spin_lock(&tree->hash_lock); + do { + node = hfs_bnode_findhash(tree, nidx++); + if (!node) + continue; + if (atomic_read(&node->refcnt)) { + res = false; + break; + } + hfs_bnode_unhash(node); + hfs_bnode_free(node); + } while (--i && nidx < tree->node_count); + spin_unlock(&tree->hash_lock); + } + return res ? try_to_free_buffers(folio) : false; +} + +static ssize_t hfsplus_direct_IO(struct kiocb *iocb, struct iov_iter *iter) +{ + struct file *file = iocb->ki_filp; + struct address_space *mapping = file->f_mapping; + struct inode *inode = mapping->host; + loff_t isize; + size_t count = iov_iter_count(iter); + loff_t end = iocb->ki_pos + count; + ssize_t ret; + + /* + * The hfsplus_get_block() only allows creating the next sequential block. + * For direct writes beyond EOF, expand the file first. + */ + if (iov_iter_rw(iter) == WRITE && iocb->ki_pos > i_size_read(inode)) { + loff_t start_off, end_off; + loff_t start_page, end_page; + + isize = i_size_read(inode); + + /* + * Wait for any in-flight DIO on this inode to finish before + * calling generic_cont_expand_simple(). + */ + inode_dio_wait(inode); + + ret = generic_cont_expand_simple(inode, iocb->ki_pos); + if (ret) + return ret; + + start_off = isize; + end_off = (end > 0) ? end - 1 : end; + + ret = filemap_write_and_wait_range(mapping, start_off, end_off); + if (ret) + return ret; + + start_page = start_off >> PAGE_SHIFT; + end_page = end_off >> PAGE_SHIFT; + + invalidate_inode_pages2_range(mapping, start_page, end_page); + } + + ret = blockdev_direct_IO(iocb, inode, iter, hfsplus_get_block); + + /* + * In case of error extending write may have instantiated a few + * blocks outside i_size. Trim these off again. + */ + if (unlikely(iov_iter_rw(iter) == WRITE && ret < 0)) { + isize = i_size_read(inode); + + if (end > isize) + hfsplus_write_failed(mapping, end); + } + + return ret; +} + +static int hfsplus_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + return mpage_writepages(mapping, wbc, hfsplus_get_block); +} + +const struct address_space_operations hfsplus_btree_aops = { + .dirty_folio = block_dirty_folio, + .invalidate_folio = block_invalidate_folio, + .read_folio = hfsplus_read_folio, + .writepages = hfsplus_writepages, + .write_begin = hfsplus_write_begin, + .write_end = generic_write_end, + .migrate_folio = buffer_migrate_folio, + .bmap = hfsplus_bmap, + .release_folio = hfsplus_release_folio, +}; + +const struct address_space_operations hfsplus_aops = { + .dirty_folio = block_dirty_folio, + .invalidate_folio = block_invalidate_folio, + .read_folio = hfsplus_read_folio, + .write_begin = hfsplus_write_begin, + .write_end = generic_write_end, + .bmap = hfsplus_bmap, + .direct_IO = hfsplus_direct_IO, + .writepages = hfsplus_writepages, + .migrate_folio = buffer_migrate_folio, +}; + +const struct dentry_operations hfsplus_dentry_operations = { + .d_hash = hfsplus_hash_dentry, + .d_compare = hfsplus_compare_dentry, +}; + +static int hfsplus_get_perms(struct inode *inode, + struct hfsplus_perm *perms, int dir) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(inode->i_sb); + u16 mode; + + mode = be16_to_cpu(perms->mode); + if (dir) { + if (mode && !S_ISDIR(mode)) + goto bad_type; + } else if (mode) { + switch (mode & S_IFMT) { + case S_IFREG: + case S_IFLNK: + case S_IFCHR: + case S_IFBLK: + case S_IFIFO: + case S_IFSOCK: + break; + default: + goto bad_type; + } + } + + i_uid_write(inode, be32_to_cpu(perms->owner)); + if ((test_bit(HFSPLUS_SB_UID, &sbi->flags)) || (!i_uid_read(inode) && !mode)) + inode->i_uid = sbi->uid; + + i_gid_write(inode, be32_to_cpu(perms->group)); + if ((test_bit(HFSPLUS_SB_GID, &sbi->flags)) || (!i_gid_read(inode) && !mode)) + inode->i_gid = sbi->gid; + + if (dir) { + mode = mode ? (mode & S_IALLUGO) : (S_IRWXUGO & ~(sbi->umask)); + mode |= S_IFDIR; + } else if (!mode) + mode = S_IFREG | ((S_IRUGO|S_IWUGO) & ~(sbi->umask)); + inode->i_mode = mode; + + HFSPLUS_I(inode)->userflags = perms->userflags; + if (perms->rootflags & HFSPLUS_FLG_IMMUTABLE) + inode->i_flags |= S_IMMUTABLE; + else + inode->i_flags &= ~S_IMMUTABLE; + if (perms->rootflags & HFSPLUS_FLG_APPEND) + inode->i_flags |= S_APPEND; + else + inode->i_flags &= ~S_APPEND; + return 0; +bad_type: + pr_err("invalid file type 0%04o for inode %llu\n", mode, inode->i_ino); + return -EIO; +} + +static int hfsplus_file_open(struct inode *inode, struct file *file) +{ + if (HFSPLUS_IS_RSRC(inode)) + inode = HFSPLUS_I(inode)->rsrc_inode; + if (!(file->f_flags & O_LARGEFILE) && i_size_read(inode) > MAX_NON_LFS) + return -EOVERFLOW; + atomic_inc(&HFSPLUS_I(inode)->opencnt); + return 0; +} + +static int hfsplus_file_release(struct inode *inode, struct file *file) +{ + struct super_block *sb = inode->i_sb; + + if (HFSPLUS_IS_RSRC(inode)) + inode = HFSPLUS_I(inode)->rsrc_inode; + if (atomic_dec_and_test(&HFSPLUS_I(inode)->opencnt)) { + inode_lock(inode); + hfsplus_file_truncate(inode); + if (inode->i_flags & S_DEAD) { + hfsplus_delete_cat(inode->i_ino, + HFSPLUS_SB(sb)->hidden_dir, NULL); + hfsplus_delete_inode(inode); + } + inode_unlock(inode); + } + return 0; +} + +static int hfsplus_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, struct iattr *attr) +{ + struct inode *inode = d_inode(dentry); + int error; + + error = setattr_prepare(&nop_mnt_idmap, dentry, attr); + if (error) + return error; + + if ((attr->ia_valid & ATTR_SIZE) && + attr->ia_size != i_size_read(inode)) { + inode_dio_wait(inode); + if (attr->ia_size > inode->i_size) { + error = generic_cont_expand_simple(inode, + attr->ia_size); + if (error) + return error; + } + truncate_setsize(inode, attr->ia_size); + hfsplus_file_truncate(inode); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + } + + setattr_copy(&nop_mnt_idmap, inode, attr); + mark_inode_dirty(inode); + + return 0; +} + +int hfsplus_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, + unsigned int query_flags) +{ + struct inode *inode = d_inode(path->dentry); + struct hfsplus_inode_info *hip = HFSPLUS_I(inode); + + if (request_mask & STATX_BTIME) { + stat->result_mask |= STATX_BTIME; + stat->btime = hfsp_mt2ut(hip->create_date); + } + + if (inode->i_flags & S_APPEND) + stat->attributes |= STATX_ATTR_APPEND; + if (inode->i_flags & S_IMMUTABLE) + stat->attributes |= STATX_ATTR_IMMUTABLE; + if (hip->userflags & HFSPLUS_FLG_NODUMP) + stat->attributes |= STATX_ATTR_NODUMP; + + stat->attributes_mask |= STATX_ATTR_APPEND | STATX_ATTR_IMMUTABLE | + STATX_ATTR_NODUMP; + + generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat); + return 0; +} + +int hfsplus_file_fsync(struct file *file, loff_t start, loff_t end, + int datasync) +{ + struct inode *inode = file->f_mapping->host; + struct hfsplus_inode_info *hip = HFSPLUS_I(inode); + struct super_block *sb = inode->i_sb; + struct hfsplus_sb_info *sbi = HFSPLUS_SB(inode->i_sb); + struct hfsplus_vh *vhdr = sbi->s_vhdr; + int error = 0, error2; + + hfs_dbg("inode->i_ino %llu, start %llu, end %llu\n", + inode->i_ino, start, end); + + error = file_write_and_wait_range(file, start, end); + if (error) + return error; + inode_lock(inode); + + /* + * Sync inode metadata into the catalog and extent trees. + */ + sync_inode_metadata(inode, 1); + + /* + * And explicitly write out the btrees. + */ + if (test_and_clear_bit(HFSPLUS_I_CAT_DIRTY, + &HFSPLUS_I(HFSPLUS_CAT_TREE_I(sb))->flags)) { + clear_bit(HFSPLUS_I_CAT_DIRTY, &hip->flags); + error = filemap_write_and_wait(sbi->cat_tree->inode->i_mapping); + } + + if (test_and_clear_bit(HFSPLUS_I_EXT_DIRTY, + &HFSPLUS_I(HFSPLUS_EXT_TREE_I(sb))->flags)) { + clear_bit(HFSPLUS_I_EXT_DIRTY, &hip->flags); + error2 = + filemap_write_and_wait(sbi->ext_tree->inode->i_mapping); + if (!error) + error = error2; + } + + if (sbi->attr_tree) { + if (test_and_clear_bit(HFSPLUS_I_ATTR_DIRTY, + &HFSPLUS_I(HFSPLUS_ATTR_TREE_I(sb))->flags)) { + clear_bit(HFSPLUS_I_ATTR_DIRTY, &hip->flags); + error2 = + filemap_write_and_wait( + sbi->attr_tree->inode->i_mapping); + if (!error) + error = error2; + } + } else { + if (test_and_clear_bit(HFSPLUS_I_ATTR_DIRTY, &hip->flags)) + pr_err("sync non-existent attributes tree\n"); + } + + if (test_and_clear_bit(HFSPLUS_I_ALLOC_DIRTY, + &HFSPLUS_I(sbi->alloc_file)->flags)) { + clear_bit(HFSPLUS_I_ALLOC_DIRTY, &hip->flags); + error2 = filemap_write_and_wait(sbi->alloc_file->i_mapping); + if (!error) + error = error2; + } + + mutex_lock(&sbi->vh_mutex); + hfsplus_prepare_volume_header_for_commit(vhdr); + mutex_unlock(&sbi->vh_mutex); + + error2 = hfsplus_commit_superblock(inode->i_sb); + if (!error) + error = error2; + + if (!test_bit(HFSPLUS_SB_NOBARRIER, &sbi->flags)) + blkdev_issue_flush(inode->i_sb->s_bdev); + + inode_unlock(inode); + + return error; +} + +static const struct inode_operations hfsplus_file_inode_operations = { + .setattr = hfsplus_setattr, + .getattr = hfsplus_getattr, + .listxattr = hfsplus_listxattr, + .fileattr_get = hfsplus_fileattr_get, + .fileattr_set = hfsplus_fileattr_set, +}; + +static const struct inode_operations hfsplus_symlink_inode_operations = { + .get_link = page_get_link, + .setattr = hfsplus_setattr, + .getattr = hfsplus_getattr, + .listxattr = hfsplus_listxattr, +}; + +static const struct inode_operations hfsplus_special_inode_operations = { + .setattr = hfsplus_setattr, + .getattr = hfsplus_getattr, + .listxattr = hfsplus_listxattr, +}; + +static const struct file_operations hfsplus_file_operations = { + .llseek = generic_file_llseek, + .read_iter = generic_file_read_iter, + .write_iter = generic_file_write_iter, + .mmap_prepare = generic_file_mmap_prepare, + .splice_read = filemap_splice_read, + .splice_write = iter_file_splice_write, + .fsync = hfsplus_file_fsync, + .open = hfsplus_file_open, + .release = hfsplus_file_release, + .unlocked_ioctl = hfsplus_ioctl, +}; + +struct inode *hfsplus_new_inode(struct super_block *sb, struct inode *dir, + umode_t mode) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(sb); + struct inode *inode = new_inode(sb); + struct hfsplus_inode_info *hip; + + if (!inode) + return NULL; + + inode->i_ino = sbi->next_cnid++; + inode_init_owner(&nop_mnt_idmap, inode, dir, mode); + set_nlink(inode, 1); + simple_inode_init_ts(inode); + + hip = HFSPLUS_I(inode); + mutex_init(&hip->extents_lock); + atomic_set(&hip->opencnt, 0); + hip->extent_state = 0; + hip->flags = 0; + hip->userflags = 0; + hip->subfolders = 0; + memset(hip->first_extents, 0, sizeof(hfsplus_extent_rec)); + memset(hip->cached_extents, 0, sizeof(hfsplus_extent_rec)); + hip->alloc_blocks = 0; + hip->first_blocks = 0; + hip->cached_start = 0; + hip->cached_blocks = 0; + hip->phys_size = 0; + hip->fs_blocks = 0; + hip->rsrc_inode = NULL; + if (S_ISDIR(inode->i_mode)) { + inode->i_size = 2; + sbi->folder_count++; + inode->i_op = &hfsplus_dir_inode_operations; + inode->i_fop = &hfsplus_dir_operations; + } else if (S_ISREG(inode->i_mode)) { + sbi->file_count++; + inode->i_op = &hfsplus_file_inode_operations; + inode->i_fop = &hfsplus_file_operations; + inode->i_mapping->a_ops = &hfsplus_aops; + hip->clump_blocks = sbi->data_clump_blocks; + } else if (S_ISLNK(inode->i_mode)) { + sbi->file_count++; + inode->i_op = &hfsplus_symlink_inode_operations; + inode_nohighmem(inode); + inode->i_mapping->a_ops = &hfsplus_aops; + hip->clump_blocks = 1; + } else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) || + S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) { + sbi->file_count++; + inode->i_op = &hfsplus_special_inode_operations; + } else + sbi->file_count++; + + insert_inode_hash(inode); + mark_inode_dirty(inode); + hfsplus_mark_mdb_dirty(sb); + + return inode; +} + +void hfsplus_delete_inode(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + + if (S_ISDIR(inode->i_mode)) { + HFSPLUS_SB(sb)->folder_count--; + hfsplus_mark_mdb_dirty(sb); + return; + } + HFSPLUS_SB(sb)->file_count--; + if (S_ISREG(inode->i_mode)) { + if (!inode->i_nlink) { + inode->i_size = 0; + hfsplus_file_truncate(inode); + } + } else if (S_ISLNK(inode->i_mode)) { + inode->i_size = 0; + hfsplus_file_truncate(inode); + } + hfsplus_mark_mdb_dirty(sb); +} + +void hfsplus_inode_read_fork(struct inode *inode, struct hfsplus_fork_raw *fork) +{ + struct super_block *sb = inode->i_sb; + struct hfsplus_sb_info *sbi = HFSPLUS_SB(sb); + struct hfsplus_inode_info *hip = HFSPLUS_I(inode); + u32 count; + int i; + + memcpy(&hip->first_extents, &fork->extents, sizeof(hfsplus_extent_rec)); + for (count = 0, i = 0; i < 8; i++) + count += be32_to_cpu(fork->extents[i].block_count); + hip->first_blocks = count; + memset(hip->cached_extents, 0, sizeof(hfsplus_extent_rec)); + hip->cached_start = 0; + hip->cached_blocks = 0; + + hip->alloc_blocks = be32_to_cpu(fork->total_blocks); + hip->phys_size = inode->i_size = be64_to_cpu(fork->total_size); + hip->fs_blocks = + (inode->i_size + sb->s_blocksize - 1) >> sb->s_blocksize_bits; + inode_set_bytes(inode, hip->fs_blocks << sb->s_blocksize_bits); + hip->clump_blocks = + be32_to_cpu(fork->clump_size) >> sbi->alloc_blksz_shift; + if (!hip->clump_blocks) { + hip->clump_blocks = HFSPLUS_IS_RSRC(inode) ? + sbi->rsrc_clump_blocks : + sbi->data_clump_blocks; + } +} + +void hfsplus_inode_write_fork(struct inode *inode, + struct hfsplus_fork_raw *fork) +{ + memcpy(&fork->extents, &HFSPLUS_I(inode)->first_extents, + sizeof(hfsplus_extent_rec)); + fork->total_size = cpu_to_be64(inode->i_size); + fork->total_blocks = cpu_to_be32(HFSPLUS_I(inode)->alloc_blocks); +} + +int hfsplus_cat_read_inode(struct inode *inode, struct hfs_find_data *fd) +{ + hfsplus_cat_entry entry; + int res = 0; + u16 type; + + type = hfs_bnode_read_u16(fd->bnode, fd->entryoffset); + + HFSPLUS_I(inode)->linkid = 0; + if (type == HFSPLUS_FOLDER) { + struct hfsplus_cat_folder *folder = &entry.folder; + + if (fd->entrylength < sizeof(struct hfsplus_cat_folder)) { + pr_err("bad catalog folder entry\n"); + res = -EIO; + goto out; + } + hfs_bnode_read(fd->bnode, &entry, fd->entryoffset, + sizeof(struct hfsplus_cat_folder)); + res = hfsplus_get_perms(inode, &folder->permissions, 1); + if (res) + goto out; + set_nlink(inode, 1); + inode->i_size = 2 + be32_to_cpu(folder->valence); + inode_set_atime_to_ts(inode, hfsp_mt2ut(folder->access_date)); + inode_set_mtime_to_ts(inode, + hfsp_mt2ut(folder->content_mod_date)); + inode_set_ctime_to_ts(inode, + hfsp_mt2ut(folder->attribute_mod_date)); + HFSPLUS_I(inode)->create_date = folder->create_date; + HFSPLUS_I(inode)->fs_blocks = 0; + if (folder->flags & cpu_to_be16(HFSPLUS_HAS_FOLDER_COUNT)) { + HFSPLUS_I(inode)->subfolders = + be32_to_cpu(folder->subfolders); + } + inode->i_op = &hfsplus_dir_inode_operations; + inode->i_fop = &hfsplus_dir_operations; + } else if (type == HFSPLUS_FILE) { + struct hfsplus_cat_file *file = &entry.file; + + if (fd->entrylength < sizeof(struct hfsplus_cat_file)) { + pr_err("bad catalog file entry\n"); + res = -EIO; + goto out; + } + hfs_bnode_read(fd->bnode, &entry, fd->entryoffset, + sizeof(struct hfsplus_cat_file)); + + hfsplus_inode_read_fork(inode, HFSPLUS_IS_RSRC(inode) ? + &file->rsrc_fork : &file->data_fork); + res = hfsplus_get_perms(inode, &file->permissions, 0); + if (res) + goto out; + set_nlink(inode, 1); + if (S_ISREG(inode->i_mode)) { + if (file->permissions.dev) + set_nlink(inode, + be32_to_cpu(file->permissions.dev)); + inode->i_op = &hfsplus_file_inode_operations; + inode->i_fop = &hfsplus_file_operations; + inode->i_mapping->a_ops = &hfsplus_aops; + } else if (S_ISLNK(inode->i_mode)) { + inode->i_op = &hfsplus_symlink_inode_operations; + inode_nohighmem(inode); + inode->i_mapping->a_ops = &hfsplus_aops; + } else { + inode->i_op = &hfsplus_special_inode_operations; + init_special_inode(inode, inode->i_mode, + be32_to_cpu(file->permissions.dev)); + } + inode_set_atime_to_ts(inode, hfsp_mt2ut(file->access_date)); + inode_set_mtime_to_ts(inode, + hfsp_mt2ut(file->content_mod_date)); + inode_set_ctime_to_ts(inode, + hfsp_mt2ut(file->attribute_mod_date)); + HFSPLUS_I(inode)->create_date = file->create_date; + } else { + pr_err("bad catalog entry used to create inode\n"); + res = -EIO; + } +out: + return res; +} + +int hfsplus_cat_write_inode(struct inode *inode) +{ + struct inode *main_inode = inode; + struct hfs_btree *tree = HFSPLUS_SB(inode->i_sb)->cat_tree; + struct hfs_find_data fd; + hfsplus_cat_entry entry; + int res = 0; + + hfs_dbg("inode->i_ino %llu\n", inode->i_ino); + + if (HFSPLUS_IS_RSRC(inode)) + main_inode = HFSPLUS_I(inode)->rsrc_inode; + + if (!main_inode->i_nlink) + return 0; + + if (hfs_find_init(tree, &fd)) + /* panic? */ + return -EIO; + + if (hfsplus_find_cat(main_inode->i_sb, main_inode->i_ino, &fd)) + /* panic? */ + goto out; + + if (S_ISDIR(main_inode->i_mode)) { + struct hfsplus_cat_folder *folder = &entry.folder; + + if (fd.entrylength < sizeof(struct hfsplus_cat_folder)) { + pr_err("bad catalog folder entry\n"); + res = -EIO; + goto out; + } + hfs_bnode_read(fd.bnode, &entry, fd.entryoffset, + sizeof(struct hfsplus_cat_folder)); + /* simple node checks? */ + hfsplus_cat_set_perms(inode, &folder->permissions); + folder->access_date = hfsp_ut2mt(inode_get_atime(inode)); + folder->content_mod_date = hfsp_ut2mt(inode_get_mtime(inode)); + folder->attribute_mod_date = hfsp_ut2mt(inode_get_ctime(inode)); + folder->valence = cpu_to_be32(inode->i_size - 2); + if (folder->flags & cpu_to_be16(HFSPLUS_HAS_FOLDER_COUNT)) { + folder->subfolders = + cpu_to_be32(HFSPLUS_I(inode)->subfolders); + } + hfs_bnode_write(fd.bnode, &entry, fd.entryoffset, + sizeof(struct hfsplus_cat_folder)); + } else if (HFSPLUS_IS_RSRC(inode)) { + struct hfsplus_cat_file *file = &entry.file; + hfs_bnode_read(fd.bnode, &entry, fd.entryoffset, + sizeof(struct hfsplus_cat_file)); + hfsplus_inode_write_fork(inode, &file->rsrc_fork); + hfs_bnode_write(fd.bnode, &entry, fd.entryoffset, + sizeof(struct hfsplus_cat_file)); + } else { + struct hfsplus_cat_file *file = &entry.file; + + if (fd.entrylength < sizeof(struct hfsplus_cat_file)) { + pr_err("bad catalog file entry\n"); + res = -EIO; + goto out; + } + hfs_bnode_read(fd.bnode, &entry, fd.entryoffset, + sizeof(struct hfsplus_cat_file)); + hfsplus_inode_write_fork(inode, &file->data_fork); + hfsplus_cat_set_perms(inode, &file->permissions); + if (HFSPLUS_FLG_IMMUTABLE & + (file->permissions.rootflags | + file->permissions.userflags)) + file->flags |= cpu_to_be16(HFSPLUS_FILE_LOCKED); + else + file->flags &= cpu_to_be16(~HFSPLUS_FILE_LOCKED); + file->access_date = hfsp_ut2mt(inode_get_atime(inode)); + file->content_mod_date = hfsp_ut2mt(inode_get_mtime(inode)); + file->attribute_mod_date = hfsp_ut2mt(inode_get_ctime(inode)); + hfs_bnode_write(fd.bnode, &entry, fd.entryoffset, + sizeof(struct hfsplus_cat_file)); + } + + res = hfs_btree_write(tree); + if (res) { + pr_err("b-tree write err: %d, ino %llu\n", + res, inode->i_ino); + goto out; + } + + set_bit(HFSPLUS_I_CAT_DIRTY, + &HFSPLUS_I(HFSPLUS_CAT_TREE_I(inode->i_sb))->flags); + set_bit(HFSPLUS_I_CAT_DIRTY, &HFSPLUS_I(inode)->flags); +out: + hfs_find_exit(&fd); + + return res; +} + +int hfsplus_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + struct hfsplus_inode_info *hip = HFSPLUS_I(inode); + struct hfsplus_sb_info *sbi = HFSPLUS_SB(inode->i_sb); + unsigned int flags = 0; + + if (inode->i_flags & S_IMMUTABLE) + flags |= FS_IMMUTABLE_FL; + if (inode->i_flags & S_APPEND) + flags |= FS_APPEND_FL; + if (hip->userflags & HFSPLUS_FLG_NODUMP) + flags |= FS_NODUMP_FL; + if (test_bit(HFSPLUS_SB_CASEFOLD, &sbi->flags)) + flags |= FS_CASEFOLD_FL; + + fileattr_fill_flags(fa, flags); + + return 0; +} + +int hfsplus_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + struct hfsplus_inode_info *hip = HFSPLUS_I(inode); + struct hfsplus_sb_info *sbi = HFSPLUS_SB(inode->i_sb); + unsigned int allowed = FS_IMMUTABLE_FL | FS_APPEND_FL | FS_NODUMP_FL; + unsigned int new_fl = 0; + + if (fileattr_has_fsx(fa)) + return -EOPNOTSUPP; + + /* + * FS_CASEFOLD_FL reflects HFSPLUS_SB_CASEFOLD, a mount-time + * property. Accept it as a no-op so chattr's RMW round-trip + * succeeds; reject any attempt to enable it on a volume that + * was not formatted case-insensitive. + */ + if (test_bit(HFSPLUS_SB_CASEFOLD, &sbi->flags)) + allowed |= FS_CASEFOLD_FL; + + /* don't silently ignore unsupported ext2 flags */ + if (fa->flags & ~allowed) + return -EOPNOTSUPP; + + if (fa->flags & FS_IMMUTABLE_FL) + new_fl |= S_IMMUTABLE; + + if (fa->flags & FS_APPEND_FL) + new_fl |= S_APPEND; + + inode_set_flags(inode, new_fl, S_IMMUTABLE | S_APPEND); + + if (fa->flags & FS_NODUMP_FL) + hip->userflags |= HFSPLUS_FLG_NODUMP; + else + hip->userflags &= ~HFSPLUS_FLG_NODUMP; + + inode_set_ctime_current(inode); + mark_inode_dirty(inode); + + return 0; +} diff --git a/fs/hfsplus/ioctl.c b/fs/hfsplus/ioctl.c new file mode 100644 index 000000000..40d04dba1 --- /dev/null +++ b/fs/hfsplus/ioctl.c @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hfsplus/ioctl.c + * + * Copyright (C) 2003 + * Ethan Benson + * partially derived from linux/fs/ext2/ioctl.c + * Copyright (C) 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * hfsplus ioctls + */ + +#include +#include +#include +#include +#include +#include "hfsplus_fs.h" + +/* + * "Blessing" an HFS+ filesystem writes metadata to the superblock informing + * the platform firmware which file to boot from + */ +static int hfsplus_ioctl_bless(struct file *file, int __user *user_flags) +{ + struct dentry *dentry = file->f_path.dentry; + struct inode *inode = d_inode(dentry); + struct hfsplus_sb_info *sbi = HFSPLUS_SB(inode->i_sb); + struct hfsplus_vh *vh = sbi->s_vhdr; + struct hfsplus_vh *bvh = sbi->s_backup_vhdr; + u32 cnid = (unsigned long)dentry->d_fsdata; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + mutex_lock(&sbi->vh_mutex); + + /* Directory containing the bootable system */ + vh->finder_info[0] = bvh->finder_info[0] = + cpu_to_be32(d_parent_ino(dentry)); + + /* + * Bootloader. Just using the inode here breaks in the case of + * hard links - the firmware wants the ID of the hard link file, + * but the inode points at the indirect inode + */ + vh->finder_info[1] = bvh->finder_info[1] = cpu_to_be32(cnid); + + /* Per spec, the OS X system folder - same as finder_info[0] here */ + vh->finder_info[5] = bvh->finder_info[5] = + cpu_to_be32(d_parent_ino(dentry)); + + mutex_unlock(&sbi->vh_mutex); + return 0; +} + +long hfsplus_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + void __user *argp = (void __user *)arg; + + switch (cmd) { + case HFSPLUS_IOC_BLESS: + return hfsplus_ioctl_bless(file, argp); + default: + return -ENOTTY; + } +} diff --git a/fs/hfsplus/options.c b/fs/hfsplus/options.c new file mode 100644 index 000000000..9b377481f --- /dev/null +++ b/fs/hfsplus/options.c @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hfsplus/options.c + * + * Copyright (C) 2001 + * Brad Boyer (flar@allandria.com) + * (C) 2003 Ardis Technologies + * + * Option parsing + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "hfsplus_fs.h" + +enum { + opt_creator, opt_type, + opt_umask, opt_uid, opt_gid, + opt_part, opt_session, opt_nls, + opt_decompose, opt_barrier, + opt_force, +}; + +static const struct fs_parameter_spec hfs_param_spec[] = { + fsparam_string ("creator", opt_creator), + fsparam_string ("type", opt_type), + fsparam_u32oct ("umask", opt_umask), + fsparam_u32 ("uid", opt_uid), + fsparam_u32 ("gid", opt_gid), + fsparam_u32 ("part", opt_part), + fsparam_u32 ("session", opt_session), + fsparam_string ("nls", opt_nls), + fsparam_flag_no ("decompose", opt_decompose), + fsparam_flag_no ("barrier", opt_barrier), + fsparam_flag ("force", opt_force), + {} +}; + +/* Initialize an options object to reasonable defaults */ +void hfsplus_fill_defaults(struct hfsplus_sb_info *opts) +{ + if (!opts) + return; + + opts->creator = HFSPLUS_DEF_CR_TYPE; + opts->type = HFSPLUS_DEF_CR_TYPE; + opts->umask = current_umask(); + opts->uid = current_uid(); + opts->gid = current_gid(); + opts->part = -1; + opts->session = -1; +} + +/* Parse options from mount. Returns nonzero errno on failure */ +int hfsplus_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct hfsplus_sb_info *sbi = fc->s_fs_info; + struct fs_parse_result result; + int opt; + + /* + * Only the force option is examined during remount, all others + * are ignored. + */ + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE && + strncmp(param->key, "force", 5)) + return 0; + + opt = fs_parse(fc, hfs_param_spec, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case opt_creator: + if (strlen(param->string) != 4) { + pr_err("creator requires a 4 character value\n"); + return -EINVAL; + } + memcpy(&sbi->creator, param->string, 4); + break; + case opt_type: + if (strlen(param->string) != 4) { + pr_err("type requires a 4 character value\n"); + return -EINVAL; + } + memcpy(&sbi->type, param->string, 4); + break; + case opt_umask: + sbi->umask = (umode_t)result.uint_32; + break; + case opt_uid: + sbi->uid = result.uid; + set_bit(HFSPLUS_SB_UID, &sbi->flags); + break; + case opt_gid: + sbi->gid = result.gid; + set_bit(HFSPLUS_SB_GID, &sbi->flags); + break; + case opt_part: + sbi->part = result.uint_32; + break; + case opt_session: + sbi->session = result.uint_32; + break; + case opt_nls: + if (sbi->nls) { + pr_err("unable to change nls mapping\n"); + return -EINVAL; + } + sbi->nls = load_nls(param->string); + if (!sbi->nls) { + pr_err("unable to load nls mapping \"%s\"\n", + param->string); + return -EINVAL; + } + break; + case opt_decompose: + if (result.negated) + set_bit(HFSPLUS_SB_NODECOMPOSE, &sbi->flags); + else + clear_bit(HFSPLUS_SB_NODECOMPOSE, &sbi->flags); + break; + case opt_barrier: + if (result.negated) + set_bit(HFSPLUS_SB_NOBARRIER, &sbi->flags); + else + clear_bit(HFSPLUS_SB_NOBARRIER, &sbi->flags); + break; + case opt_force: + set_bit(HFSPLUS_SB_FORCE, &sbi->flags); + break; + default: + return -EINVAL; + } + + return 0; +} + +int hfsplus_show_options(struct seq_file *seq, struct dentry *root) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(root->d_sb); + + if (sbi->creator != HFSPLUS_DEF_CR_TYPE) + seq_show_option_n(seq, "creator", (char *)&sbi->creator, 4); + if (sbi->type != HFSPLUS_DEF_CR_TYPE) + seq_show_option_n(seq, "type", (char *)&sbi->type, 4); + seq_printf(seq, ",umask=%o,uid=%u,gid=%u", sbi->umask, + from_kuid_munged(&init_user_ns, sbi->uid), + from_kgid_munged(&init_user_ns, sbi->gid)); + if (sbi->part >= 0) + seq_printf(seq, ",part=%u", sbi->part); + if (sbi->session >= 0) + seq_printf(seq, ",session=%u", sbi->session); + if (sbi->nls) + seq_printf(seq, ",nls=%s", sbi->nls->charset); + if (test_bit(HFSPLUS_SB_NODECOMPOSE, &sbi->flags)) + seq_puts(seq, ",nodecompose"); + if (test_bit(HFSPLUS_SB_NOBARRIER, &sbi->flags)) + seq_puts(seq, ",nobarrier"); + return 0; +} diff --git a/fs/hfsplus/part_tbl.c b/fs/hfsplus/part_tbl.c new file mode 100644 index 000000000..9ec21664e --- /dev/null +++ b/fs/hfsplus/part_tbl.c @@ -0,0 +1,157 @@ +/* + * linux/fs/hfsplus/part_tbl.c + * + * Copyright (C) 1996-1997 Paul H. Hargrove + * This file may be distributed under the terms of + * the GNU General Public License. + * + * Original code to handle the new style Mac partition table based on + * a patch contributed by Holger Schemel (aeglos@valinor.owl.de). + * + * In function preconditions the term "valid" applied to a pointer to + * a structure means that the pointer is non-NULL and the structure it + * points to has all fields initialized to consistent values. + * + */ + +#include +#include "hfsplus_fs.h" + +/* offsets to various blocks */ +#define HFS_DD_BLK 0 /* Driver Descriptor block */ +#define HFS_PMAP_BLK 1 /* First block of partition map */ +#define HFS_MDB_BLK 2 /* Block (w/i partition) of MDB */ + +/* magic numbers for various disk blocks */ +#define HFS_DRVR_DESC_MAGIC 0x4552 /* "ER": driver descriptor map */ +#define HFS_OLD_PMAP_MAGIC 0x5453 /* "TS": old-type partition map */ +#define HFS_NEW_PMAP_MAGIC 0x504D /* "PM": new-type partition map */ +#define HFS_SUPER_MAGIC 0x4244 /* "BD": HFS MDB (super block) */ +#define HFS_MFS_SUPER_MAGIC 0xD2D7 /* MFS MDB (super block) */ + +/* + * The new style Mac partition map + * + * For each partition on the media there is a physical block (512-byte + * block) containing one of these structures. These blocks are + * contiguous starting at block 1. + */ +struct new_pmap { + __be16 pmSig; /* signature */ + __be16 reSigPad; /* padding */ + __be32 pmMapBlkCnt; /* partition blocks count */ + __be32 pmPyPartStart; /* physical block start of partition */ + __be32 pmPartBlkCnt; /* physical block count of partition */ + u8 pmPartName[32]; /* (null terminated?) string + giving the name of this + partition */ + u8 pmPartType[32]; /* (null terminated?) string + giving the type of this + partition */ + /* a bunch more stuff we don't need */ +} __packed; + +/* + * The old style Mac partition map + * + * The partition map consists for a 2-byte signature followed by an + * array of these structures. The map is terminated with an all-zero + * one of these. + */ +struct old_pmap { + __be16 pdSig; /* Signature bytes */ + struct old_pmap_entry { + __be32 pdStart; + __be32 pdSize; + __be32 pdFSID; + } pdEntry[42]; +} __packed; + +static int hfs_parse_old_pmap(struct super_block *sb, struct old_pmap *pm, + sector_t *part_start, sector_t *part_size) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(sb); + int i; + + for (i = 0; i < 42; i++) { + struct old_pmap_entry *p = &pm->pdEntry[i]; + + if (p->pdStart && p->pdSize && + p->pdFSID == cpu_to_be32(0x54465331)/*"TFS1"*/ && + (sbi->part < 0 || sbi->part == i)) { + *part_start += be32_to_cpu(p->pdStart); + *part_size = be32_to_cpu(p->pdSize); + return 0; + } + } + + return -ENOENT; +} + +static int hfs_parse_new_pmap(struct super_block *sb, void *buf, + struct new_pmap *pm, sector_t *part_start, sector_t *part_size) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(sb); + int size = be32_to_cpu(pm->pmMapBlkCnt); + int buf_size = hfsplus_min_io_size(sb); + int res; + int i = 0; + + do { + if (!memcmp(pm->pmPartType, "Apple_HFS", 9) && + (sbi->part < 0 || sbi->part == i)) { + *part_start += be32_to_cpu(pm->pmPyPartStart); + *part_size = be32_to_cpu(pm->pmPartBlkCnt); + return 0; + } + + if (++i >= size) + return -ENOENT; + + pm = (struct new_pmap *)((u8 *)pm + HFSPLUS_SECTOR_SIZE); + if ((u8 *)pm - (u8 *)buf >= buf_size) { + res = hfsplus_submit_bio(sb, + *part_start + HFS_PMAP_BLK + i, + buf, (void **)&pm, REQ_OP_READ); + if (res) + return res; + } + } while (pm->pmSig == cpu_to_be16(HFS_NEW_PMAP_MAGIC)); + + return -ENOENT; +} + +/* + * Parse the partition map looking for the start and length of a + * HFS/HFS+ partition. + */ +int hfs_part_find(struct super_block *sb, + sector_t *part_start, sector_t *part_size) +{ + void *buf, *data; + int res; + + buf = kmalloc(hfsplus_min_io_size(sb), GFP_KERNEL); + if (!buf) + return -ENOMEM; + + res = hfsplus_submit_bio(sb, *part_start + HFS_PMAP_BLK, + buf, &data, REQ_OP_READ); + if (res) + goto out; + + switch (be16_to_cpu(*((__be16 *)data))) { + case HFS_OLD_PMAP_MAGIC: + res = hfs_parse_old_pmap(sb, data, part_start, part_size); + break; + case HFS_NEW_PMAP_MAGIC: + res = hfs_parse_new_pmap(sb, buf, data, part_start, part_size); + break; + default: + res = -ENOENT; + break; + } +out: + kfree(buf); + return res; +} diff --git a/fs/hfsplus/super.c b/fs/hfsplus/super.c new file mode 100644 index 000000000..ff7d6b333 --- /dev/null +++ b/fs/hfsplus/super.c @@ -0,0 +1,803 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/hfsplus/super.c + * + * Copyright (C) 2001 + * Brad Boyer (flar@allandria.com) + * (C) 2003 Ardis Technologies + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static struct inode *hfsplus_alloc_inode(struct super_block *sb); +static void hfsplus_free_inode(struct inode *inode); + +#include "hfsplus_fs.h" +#include "xattr.h" + +static int hfsplus_system_read_inode(struct inode *inode) +{ + struct hfsplus_vh *vhdr = HFSPLUS_SB(inode->i_sb)->s_vhdr; + + switch (inode->i_ino) { + case HFSPLUS_EXT_CNID: + hfsplus_inode_read_fork(inode, &vhdr->ext_file); + inode->i_mapping->a_ops = &hfsplus_btree_aops; + break; + case HFSPLUS_CAT_CNID: + hfsplus_inode_read_fork(inode, &vhdr->cat_file); + inode->i_mapping->a_ops = &hfsplus_btree_aops; + break; + case HFSPLUS_ALLOC_CNID: + hfsplus_inode_read_fork(inode, &vhdr->alloc_file); + inode->i_mapping->a_ops = &hfsplus_aops; + break; + case HFSPLUS_START_CNID: + hfsplus_inode_read_fork(inode, &vhdr->start_file); + break; + case HFSPLUS_ATTR_CNID: + hfsplus_inode_read_fork(inode, &vhdr->attr_file); + inode->i_mapping->a_ops = &hfsplus_btree_aops; + break; + default: + return -EIO; + } + + /* + * Assign a dummy file type, for may_open() requires that + * an inode has a valid file type. + */ + inode->i_mode = S_IFREG; + + return 0; +} + +struct inode *hfsplus_iget(struct super_block *sb, unsigned long ino) +{ + struct hfs_find_data fd; + struct inode *inode; + int err; + + inode = iget_locked(sb, ino); + if (!inode) + return ERR_PTR(-ENOMEM); + if (!(inode_state_read_once(inode) & I_NEW)) + return inode; + + atomic_set(&HFSPLUS_I(inode)->opencnt, 0); + HFSPLUS_I(inode)->first_blocks = 0; + HFSPLUS_I(inode)->clump_blocks = 0; + HFSPLUS_I(inode)->alloc_blocks = 0; + HFSPLUS_I(inode)->cached_start = U32_MAX; + HFSPLUS_I(inode)->cached_blocks = 0; + memset(HFSPLUS_I(inode)->first_extents, 0, sizeof(hfsplus_extent_rec)); + memset(HFSPLUS_I(inode)->cached_extents, 0, sizeof(hfsplus_extent_rec)); + HFSPLUS_I(inode)->extent_state = 0; + mutex_init(&HFSPLUS_I(inode)->extents_lock); + HFSPLUS_I(inode)->rsrc_inode = NULL; + HFSPLUS_I(inode)->create_date = 0; + HFSPLUS_I(inode)->linkid = 0; + HFSPLUS_I(inode)->flags = 0; + HFSPLUS_I(inode)->fs_blocks = 0; + HFSPLUS_I(inode)->userflags = 0; + HFSPLUS_I(inode)->subfolders = 0; + HFSPLUS_I(inode)->phys_size = 0; + + if (inode->i_ino >= HFSPLUS_FIRSTUSER_CNID || + inode->i_ino == HFSPLUS_ROOT_CNID) { + err = hfs_find_init(HFSPLUS_SB(inode->i_sb)->cat_tree, &fd); + if (!err) { + err = hfsplus_find_cat(inode->i_sb, inode->i_ino, &fd); + if (!err) + err = hfsplus_cat_read_inode(inode, &fd); + hfs_find_exit(&fd); + } + } else { + err = hfsplus_system_read_inode(inode); + } + + if (err) { + iget_failed(inode); + return ERR_PTR(err); + } + + unlock_new_inode(inode); + return inode; +} + +static int hfsplus_system_write_inode(struct inode *inode) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(inode->i_sb); + struct hfsplus_vh *vhdr = sbi->s_vhdr; + struct hfsplus_fork_raw *fork; + struct hfs_btree *tree = NULL; + + switch (inode->i_ino) { + case HFSPLUS_EXT_CNID: + fork = &vhdr->ext_file; + tree = sbi->ext_tree; + break; + case HFSPLUS_CAT_CNID: + fork = &vhdr->cat_file; + tree = sbi->cat_tree; + break; + case HFSPLUS_ALLOC_CNID: + fork = &vhdr->alloc_file; + break; + case HFSPLUS_START_CNID: + fork = &vhdr->start_file; + break; + case HFSPLUS_ATTR_CNID: + fork = &vhdr->attr_file; + tree = sbi->attr_tree; + break; + default: + return -EIO; + } + + if (fork->total_size != cpu_to_be64(inode->i_size)) { + set_bit(HFSPLUS_SB_WRITEBACKUP, &sbi->flags); + hfsplus_mark_mdb_dirty(inode->i_sb); + } + hfsplus_inode_write_fork(inode, fork); + if (tree) { + mutex_lock_nested(&tree->tree_lock, + hfsplus_btree_lock_class(tree)); + int err = hfs_btree_write(tree); + mutex_unlock(&tree->tree_lock); + + if (err) { + pr_err("b-tree write err: %d, ino %llu\n", + err, inode->i_ino); + return err; + } + } + return 0; +} + +static int hfsplus_write_inode(struct inode *inode, + struct writeback_control *wbc) +{ + int err; + + hfs_dbg("ino %llu\n", inode->i_ino); + + err = hfsplus_ext_write_extent(inode); + if (err) + return err; + + if (inode->i_ino >= HFSPLUS_FIRSTUSER_CNID || + inode->i_ino == HFSPLUS_ROOT_CNID) + return hfsplus_cat_write_inode(inode); + else + return hfsplus_system_write_inode(inode); +} + +static void hfsplus_evict_inode(struct inode *inode) +{ + hfs_dbg("ino %llu\n", inode->i_ino); + truncate_inode_pages_final(&inode->i_data); + clear_inode(inode); + if (HFSPLUS_IS_RSRC(inode)) { + HFSPLUS_I(HFSPLUS_I(inode)->rsrc_inode)->rsrc_inode = NULL; + iput(HFSPLUS_I(inode)->rsrc_inode); + } +} + +int hfsplus_commit_superblock(struct super_block *sb) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(sb); + struct hfsplus_vh *vhdr = sbi->s_vhdr; + int write_backup = 0; + int error = 0, error2; + + hfs_dbg("starting...\n"); + + mutex_lock(&sbi->vh_mutex); + mutex_lock(&sbi->alloc_mutex); + vhdr->free_blocks = cpu_to_be32(sbi->free_blocks); + vhdr->next_cnid = cpu_to_be32(sbi->next_cnid); + vhdr->folder_count = cpu_to_be32(sbi->folder_count); + vhdr->file_count = cpu_to_be32(sbi->file_count); + + hfs_dbg("free_blocks %u, next_cnid %u, folder_count %u, file_count %u\n", + sbi->free_blocks, sbi->next_cnid, + sbi->folder_count, sbi->file_count); + + if (test_and_clear_bit(HFSPLUS_SB_WRITEBACKUP, &sbi->flags)) { + memcpy(sbi->s_backup_vhdr, sbi->s_vhdr, sizeof(*sbi->s_vhdr)); + write_backup = 1; + } + + error2 = hfsplus_submit_bio(sb, + sbi->part_start + HFSPLUS_VOLHEAD_SECTOR, + sbi->s_vhdr_buf, NULL, REQ_OP_WRITE); + if (!error) + error = error2; + if (!write_backup) + goto out; + + error2 = hfsplus_submit_bio(sb, + sbi->part_start + sbi->sect_count - 2, + sbi->s_backup_vhdr_buf, NULL, REQ_OP_WRITE); + if (!error) + error = error2; +out: + mutex_unlock(&sbi->alloc_mutex); + mutex_unlock(&sbi->vh_mutex); + + hfs_dbg("finished: err %d\n", error); + + return error; +} + +static int hfsplus_sync_fs(struct super_block *sb, int wait) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(sb); + int error, error2; + + if (!wait) + return 0; + + hfs_dbg("starting...\n"); + + /* + * Explicitly write out the special metadata inodes. + * + * While these special inodes are marked as hashed and written + * out peridocically by the flusher threads we redirty them + * during writeout of normal inodes, and thus the life lock + * prevents us from getting the latest state to disk. + */ + error = filemap_write_and_wait(sbi->cat_tree->inode->i_mapping); + error2 = filemap_write_and_wait(sbi->ext_tree->inode->i_mapping); + if (!error) + error = error2; + if (sbi->attr_tree) { + error2 = + filemap_write_and_wait(sbi->attr_tree->inode->i_mapping); + if (!error) + error = error2; + } + error2 = filemap_write_and_wait(sbi->alloc_file->i_mapping); + if (!error) + error = error2; + + error2 = hfsplus_commit_superblock(sb); + if (!error) + error = error2; + + if (!test_bit(HFSPLUS_SB_NOBARRIER, &sbi->flags)) + blkdev_issue_flush(sb->s_bdev); + + hfs_dbg("finished: err %d\n", error); + + return error; +} + +static void delayed_sync_fs(struct work_struct *work) +{ + int err; + struct hfsplus_sb_info *sbi; + + sbi = container_of(work, struct hfsplus_sb_info, sync_work.work); + + spin_lock(&sbi->work_lock); + sbi->work_queued = 0; + spin_unlock(&sbi->work_lock); + + err = hfsplus_sync_fs(sbi->alloc_file->i_sb, 1); + if (err) + pr_err("delayed sync fs err %d\n", err); +} + +void hfsplus_mark_mdb_dirty(struct super_block *sb) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(sb); + unsigned long delay; + + if (sb_rdonly(sb)) + return; + + spin_lock(&sbi->work_lock); + if (!sbi->work_queued) { + delay = msecs_to_jiffies(dirty_writeback_interval * 10); + queue_delayed_work(system_dfl_long_wq, &sbi->sync_work, delay); + sbi->work_queued = 1; + } + spin_unlock(&sbi->work_lock); +} + +static void delayed_free(struct rcu_head *p) +{ + struct hfsplus_sb_info *sbi = container_of(p, struct hfsplus_sb_info, rcu); + + unload_nls(sbi->nls); + kfree(sbi); +} + +static void hfsplus_put_super(struct super_block *sb) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(sb); + + hfs_dbg("starting...\n"); + + cancel_delayed_work_sync(&sbi->sync_work); + + if (!sb_rdonly(sb) && sbi->s_vhdr) { + struct hfsplus_vh *vhdr = sbi->s_vhdr; + + vhdr->modify_date = hfsp_now2mt(); + vhdr->attributes |= cpu_to_be32(HFSPLUS_VOL_UNMNT); + vhdr->attributes &= cpu_to_be32(~HFSPLUS_VOL_INCNSTNT); + + hfsplus_sync_fs(sb, 1); + } + + iput(sbi->alloc_file); + iput(sbi->hidden_dir); + hfs_btree_close(sbi->attr_tree); + hfs_btree_close(sbi->cat_tree); + hfs_btree_close(sbi->ext_tree); + kfree(sbi->s_vhdr_buf); + kfree(sbi->s_backup_vhdr_buf); + hfs_dbg("finished\n"); +} + +static int hfsplus_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + struct hfsplus_sb_info *sbi = HFSPLUS_SB(sb); + u64 id = huge_encode_dev(sb->s_bdev->bd_dev); + + buf->f_type = HFSPLUS_SUPER_MAGIC; + buf->f_bsize = sb->s_blocksize; + buf->f_blocks = sbi->total_blocks << sbi->fs_shift; + buf->f_bfree = sbi->free_blocks << sbi->fs_shift; + buf->f_bavail = buf->f_bfree; + buf->f_files = 0xFFFFFFFF; + buf->f_ffree = 0xFFFFFFFF - sbi->next_cnid; + buf->f_fsid = u64_to_fsid(id); + buf->f_namelen = HFSPLUS_MAX_STRLEN; + + return 0; +} + +static int hfsplus_reconfigure(struct fs_context *fc) +{ + struct super_block *sb = fc->root->d_sb; + + sync_filesystem(sb); + if ((bool)(fc->sb_flags & SB_RDONLY) == sb_rdonly(sb)) + return 0; + if (!(fc->sb_flags & SB_RDONLY)) { + struct hfsplus_sb_info *sbi = HFSPLUS_SB(sb); + struct hfsplus_vh *vhdr = sbi->s_vhdr; + + if (!(vhdr->attributes & cpu_to_be32(HFSPLUS_VOL_UNMNT))) { + pr_warn("filesystem was not cleanly unmounted, running fsck.hfsplus is recommended. leaving read-only.\n"); + sb->s_flags |= SB_RDONLY; + fc->sb_flags |= SB_RDONLY; + } else if (test_bit(HFSPLUS_SB_FORCE, &sbi->flags)) { + /* nothing */ + } else if (vhdr->attributes & + cpu_to_be32(HFSPLUS_VOL_SOFTLOCK)) { + pr_warn("filesystem is marked locked, leaving read-only.\n"); + sb->s_flags |= SB_RDONLY; + fc->sb_flags |= SB_RDONLY; + } else if (vhdr->attributes & + cpu_to_be32(HFSPLUS_VOL_JOURNALED)) { + pr_warn("filesystem is marked journaled, leaving read-only.\n"); + sb->s_flags |= SB_RDONLY; + fc->sb_flags |= SB_RDONLY; + } + } + return 0; +} + +static const struct super_operations hfsplus_sops = { + .alloc_inode = hfsplus_alloc_inode, + .free_inode = hfsplus_free_inode, + .write_inode = hfsplus_write_inode, + .evict_inode = hfsplus_evict_inode, + .put_super = hfsplus_put_super, + .sync_fs = hfsplus_sync_fs, + .statfs = hfsplus_statfs, + .show_options = hfsplus_show_options, +}; + +void hfsplus_prepare_volume_header_for_commit(struct hfsplus_vh *vhdr) +{ + vhdr->last_mount_vers = cpu_to_be32(HFSP_MOUNT_VERSION); + vhdr->modify_date = hfsp_now2mt(); + be32_add_cpu(&vhdr->write_count, 1); + vhdr->attributes &= cpu_to_be32(~HFSPLUS_VOL_UNMNT); + vhdr->attributes |= cpu_to_be32(HFSPLUS_VOL_INCNSTNT); +} + +static inline int hfsplus_get_hidden_dir_entry(struct super_block *sb, + const struct qstr *str, + hfsplus_cat_entry *entry) +{ + struct hfs_find_data fd; + int err; + + err = hfs_find_init(HFSPLUS_SB(sb)->cat_tree, &fd); + if (unlikely(err)) + return err; + + err = hfsplus_cat_build_key(sb, fd.search_key, HFSPLUS_ROOT_CNID, str); + if (unlikely(err)) + goto free_fd; + + err = hfsplus_brec_read_cat(&fd, entry); + if (err) + err = -ENOENT; + +free_fd: + hfs_find_exit(&fd); + return err; +} + +static int hfsplus_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct hfsplus_vh *vhdr; + struct hfsplus_sb_info *sbi = HFSPLUS_SB(sb); + hfsplus_cat_entry entry; + struct inode *root, *inode; + struct qstr str; + struct nls_table *nls; + u64 last_fs_block, last_fs_page; + int silent = fc->sb_flags & SB_SILENT; + int err; + + mutex_init(&sbi->alloc_mutex); + mutex_init(&sbi->vh_mutex); + spin_lock_init(&sbi->work_lock); + INIT_DELAYED_WORK(&sbi->sync_work, delayed_sync_fs); + + err = -EINVAL; + if (!sbi->nls) { + /* try utf8 first, as this is the old default behaviour */ + sbi->nls = load_nls("utf8"); + if (!sbi->nls) + sbi->nls = load_nls_default(); + } + + /* temporarily use utf8 to correctly find the hidden dir below */ + nls = sbi->nls; + sbi->nls = load_nls("utf8"); + if (!sbi->nls) { + pr_err("unable to load nls for utf8\n"); + goto out_unload_nls; + } + + /* Grab the volume header */ + if (hfsplus_read_wrapper(sb)) { + if (!silent) + pr_warn("unable to find HFS+ superblock\n"); + goto out_unload_nls; + } + vhdr = sbi->s_vhdr; + + /* Copy parts of the volume header into the superblock */ + sb->s_magic = HFSPLUS_VOLHEAD_SIG; + if (be16_to_cpu(vhdr->version) < HFSPLUS_MIN_VERSION || + be16_to_cpu(vhdr->version) > HFSPLUS_CURRENT_VERSION) { + pr_err("wrong filesystem version\n"); + goto out_free_vhdr; + } + sbi->total_blocks = be32_to_cpu(vhdr->total_blocks); + sbi->free_blocks = be32_to_cpu(vhdr->free_blocks); + sbi->next_cnid = be32_to_cpu(vhdr->next_cnid); + sbi->file_count = be32_to_cpu(vhdr->file_count); + sbi->folder_count = be32_to_cpu(vhdr->folder_count); + sbi->data_clump_blocks = + be32_to_cpu(vhdr->data_clump_sz) >> sbi->alloc_blksz_shift; + if (!sbi->data_clump_blocks) + sbi->data_clump_blocks = 1; + sbi->rsrc_clump_blocks = + be32_to_cpu(vhdr->rsrc_clump_sz) >> sbi->alloc_blksz_shift; + if (!sbi->rsrc_clump_blocks) + sbi->rsrc_clump_blocks = 1; + + err = -EFBIG; + last_fs_block = sbi->total_blocks - 1; + last_fs_page = (last_fs_block << sbi->alloc_blksz_shift) >> + PAGE_SHIFT; + + if ((last_fs_block > (sector_t)(~0ULL) >> (sbi->alloc_blksz_shift - 9)) || + (last_fs_page > (pgoff_t)(~0ULL))) { + pr_err("filesystem size too large\n"); + goto out_free_vhdr; + } + + /* Set up operations so we can load metadata */ + sb->s_op = &hfsplus_sops; + sb->s_maxbytes = MAX_LFS_FILESIZE; + + if (!(vhdr->attributes & cpu_to_be32(HFSPLUS_VOL_UNMNT))) { + pr_warn("Filesystem was not cleanly unmounted, running fsck.hfsplus is recommended. mounting read-only.\n"); + sb->s_flags |= SB_RDONLY; + } else if (test_and_clear_bit(HFSPLUS_SB_FORCE, &sbi->flags)) { + /* nothing */ + } else if (vhdr->attributes & cpu_to_be32(HFSPLUS_VOL_SOFTLOCK)) { + pr_warn("Filesystem is marked locked, mounting read-only.\n"); + sb->s_flags |= SB_RDONLY; + } else if ((vhdr->attributes & cpu_to_be32(HFSPLUS_VOL_JOURNALED)) && + !sb_rdonly(sb)) { + pr_warn("write access to a journaled filesystem is not supported, use the force option at your own risk, mounting read-only.\n"); + sb->s_flags |= SB_RDONLY; + } + + err = -EINVAL; + + /* Load metadata objects (B*Trees) */ + sbi->ext_tree = hfs_btree_open(sb, HFSPLUS_EXT_CNID); + if (!sbi->ext_tree) { + pr_err("failed to load extents file\n"); + goto out_free_vhdr; + } + sbi->cat_tree = hfs_btree_open(sb, HFSPLUS_CAT_CNID); + if (!sbi->cat_tree) { + pr_err("failed to load catalog file\n"); + goto out_close_ext_tree; + } + atomic_set(&sbi->attr_tree_state, HFSPLUS_EMPTY_ATTR_TREE); + if (vhdr->attr_file.total_blocks != 0) { + sbi->attr_tree = hfs_btree_open(sb, HFSPLUS_ATTR_CNID); + if (!sbi->attr_tree) { + pr_err("failed to load attributes file\n"); + goto out_close_cat_tree; + } + atomic_set(&sbi->attr_tree_state, HFSPLUS_VALID_ATTR_TREE); + } + sb->s_xattr = hfsplus_xattr_handlers; + + inode = hfsplus_iget(sb, HFSPLUS_ALLOC_CNID); + if (IS_ERR(inode)) { + pr_err("failed to load allocation file\n"); + err = PTR_ERR(inode); + goto out_close_attr_tree; + } + sbi->alloc_file = inode; + + /* Load the root directory */ + root = hfsplus_iget(sb, HFSPLUS_ROOT_CNID); + if (IS_ERR(root)) { + pr_err("failed to load root directory\n"); + err = PTR_ERR(root); + goto out_put_alloc_file; + } + + set_default_d_op(sb, &hfsplus_dentry_operations); + sb->s_root = d_make_root(root); + if (!sb->s_root) { + err = -ENOMEM; + goto out_put_alloc_file; + } + + str.len = sizeof(HFSP_HIDDENDIR_NAME) - 1; + str.name = HFSP_HIDDENDIR_NAME; + err = hfsplus_get_hidden_dir_entry(sb, &str, &entry); + if (err == -ENOENT) { + /* + * Hidden directory is absent or it cannot be read. + */ + } else if (unlikely(err)) { + goto out_put_root; + } else { + if (entry.type != cpu_to_be16(HFSPLUS_FOLDER)) { + err = -EIO; + goto out_put_root; + } + inode = hfsplus_iget(sb, be32_to_cpu(entry.folder.id)); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + goto out_put_root; + } + sbi->hidden_dir = inode; + } + + if (!sb_rdonly(sb)) { + /* + * H+LX == hfsplusutils, H+Lx == this driver, H+lx is unused + * all three are registered with Apple for our use + */ + hfsplus_prepare_volume_header_for_commit(vhdr); + hfsplus_sync_fs(sb, 1); + + if (!sbi->hidden_dir) { + mutex_lock(&sbi->vh_mutex); + sbi->hidden_dir = hfsplus_new_inode(sb, root, S_IFDIR); + if (!sbi->hidden_dir) { + mutex_unlock(&sbi->vh_mutex); + err = -ENOMEM; + goto out_put_root; + } + err = hfsplus_create_cat(sbi->hidden_dir->i_ino, root, + &str, sbi->hidden_dir); + if (err) { + mutex_unlock(&sbi->vh_mutex); + goto out_put_hidden_dir; + } + + err = hfsplus_init_security(sbi->hidden_dir, + root, &str); + if (err == -EOPNOTSUPP) + err = 0; /* Operation is not supported. */ + else if (err) { + /* + * Try to delete anyway without + * error analysis. + */ + hfsplus_delete_cat(sbi->hidden_dir->i_ino, + root, &str); + mutex_unlock(&sbi->vh_mutex); + goto out_put_hidden_dir; + } + + mutex_unlock(&sbi->vh_mutex); + hfsplus_mark_inode_dirty(HFSPLUS_CAT_TREE_I(sb), + HFSPLUS_I_CAT_DIRTY); + hfsplus_mark_inode_dirty(sbi->hidden_dir, + HFSPLUS_I_CAT_DIRTY); + } + } + + unload_nls(sbi->nls); + sbi->nls = nls; + return 0; + +out_put_hidden_dir: + cancel_delayed_work_sync(&sbi->sync_work); + iput(sbi->hidden_dir); +out_put_root: + dput(sb->s_root); + sb->s_root = NULL; +out_put_alloc_file: + iput(sbi->alloc_file); +out_close_attr_tree: + hfs_btree_close(sbi->attr_tree); +out_close_cat_tree: + hfs_btree_close(sbi->cat_tree); +out_close_ext_tree: + hfs_btree_close(sbi->ext_tree); +out_free_vhdr: + kfree(sbi->s_vhdr_buf); + kfree(sbi->s_backup_vhdr_buf); +out_unload_nls: + unload_nls(nls); + return err; +} + +MODULE_AUTHOR("Brad Boyer"); +MODULE_DESCRIPTION("Extended Macintosh Filesystem"); +MODULE_LICENSE("GPL"); + +static struct kmem_cache *hfsplus_inode_cachep; + +static struct inode *hfsplus_alloc_inode(struct super_block *sb) +{ + struct hfsplus_inode_info *i; + + i = alloc_inode_sb(sb, hfsplus_inode_cachep, GFP_KERNEL); + return i ? &i->vfs_inode : NULL; +} + +static void hfsplus_free_inode(struct inode *inode) +{ + kmem_cache_free(hfsplus_inode_cachep, HFSPLUS_I(inode)); +} + +#define HFSPLUS_INODE_SIZE sizeof(struct hfsplus_inode_info) + +static int hfsplus_get_tree(struct fs_context *fc) +{ + return get_tree_bdev(fc, hfsplus_fill_super); +} + +static void hfsplus_free_fc(struct fs_context *fc) +{ + kfree(fc->s_fs_info); +} + +static const struct fs_context_operations hfsplus_context_ops = { + .parse_param = hfsplus_parse_param, + .get_tree = hfsplus_get_tree, + .reconfigure = hfsplus_reconfigure, + .free = hfsplus_free_fc, +}; + +static int hfsplus_init_fs_context(struct fs_context *fc) +{ + struct hfsplus_sb_info *sbi; + + sbi = kzalloc_obj(struct hfsplus_sb_info); + if (!sbi) + return -ENOMEM; + + if (fc->purpose != FS_CONTEXT_FOR_RECONFIGURE) + hfsplus_fill_defaults(sbi); + + fc->s_fs_info = sbi; + fc->ops = &hfsplus_context_ops; + + return 0; +} + +static void hfsplus_kill_super(struct super_block *sb) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(sb); + + kill_block_super(sb); + call_rcu(&sbi->rcu, delayed_free); +} + +static struct file_system_type hfsplus_fs_type = { + .owner = THIS_MODULE, + .name = "hfsplus", + .kill_sb = hfsplus_kill_super, + .fs_flags = FS_REQUIRES_DEV, + .init_fs_context = hfsplus_init_fs_context, +}; +MODULE_ALIAS_FS("hfsplus"); + +static void hfsplus_init_once(void *p) +{ + struct hfsplus_inode_info *i = p; + + inode_init_once(&i->vfs_inode); +} + +static int __init init_hfsplus_fs(void) +{ + int err; + + hfsplus_inode_cachep = kmem_cache_create("hfsplus_icache", + HFSPLUS_INODE_SIZE, 0, SLAB_HWCACHE_ALIGN|SLAB_ACCOUNT, + hfsplus_init_once); + if (!hfsplus_inode_cachep) + return -ENOMEM; + err = hfsplus_create_attr_tree_cache(); + if (err) + goto destroy_inode_cache; + err = register_filesystem(&hfsplus_fs_type); + if (err) + goto destroy_attr_tree_cache; + return 0; + +destroy_attr_tree_cache: + hfsplus_destroy_attr_tree_cache(); + +destroy_inode_cache: + kmem_cache_destroy(hfsplus_inode_cachep); + + return err; +} + +static void __exit exit_hfsplus_fs(void) +{ + unregister_filesystem(&hfsplus_fs_type); + + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + hfsplus_destroy_attr_tree_cache(); + kmem_cache_destroy(hfsplus_inode_cachep); +} + +module_init(init_hfsplus_fs) +module_exit(exit_hfsplus_fs) diff --git a/fs/hfsplus/tables.c b/fs/hfsplus/tables.c new file mode 100644 index 000000000..a5fb8ee7d --- /dev/null +++ b/fs/hfsplus/tables.c @@ -0,0 +1,3246 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hfsplus/tables.c + * + * Various data tables + */ + +#include "hfsplus_fs.h" + +/* + * Unicode case folding table taken from Apple Technote #1150 + * (HFS Plus Volume Format) + */ + +u16 hfsplus_case_fold_table[] = { +/* + * The lower case table consists of a 256-entry high-byte table followed by + * some number of 256-entry subtables. The high-byte table contains either an + * offset to the subtable for characters with that high byte or zero, which + * means that there are no case mappings or ignored characters in that block. + * Ignored characters are mapped to zero. + */ + + // High-byte indices ( == 0 iff no case mapping and no ignorables ) + + + /* 0 */ 0x0100, 0x0200, 0x0000, 0x0300, 0x0400, 0x0500, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* 1 */ 0x0600, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* 2 */ 0x0700, 0x0800, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* 3 */ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* 4 */ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* 5 */ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* 6 */ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* 7 */ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* 8 */ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* 9 */ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* A */ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* B */ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* C */ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* D */ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* E */ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* F */ 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0900, 0x0A00, + + // Table 1 (for high byte 0x00) + + /* 0 */ 0xFFFF, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F, + /* 1 */ 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001A, 0x001B, 0x001C, 0x001D, 0x001E, 0x001F, + /* 2 */ 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002A, 0x002B, 0x002C, 0x002D, 0x002E, 0x002F, + /* 3 */ 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003A, 0x003B, 0x003C, 0x003D, 0x003E, 0x003F, + /* 4 */ 0x0040, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006A, 0x006B, 0x006C, 0x006D, 0x006E, 0x006F, + /* 5 */ 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007A, 0x005B, 0x005C, 0x005D, 0x005E, 0x005F, + /* 6 */ 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006A, 0x006B, 0x006C, 0x006D, 0x006E, 0x006F, + /* 7 */ 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007A, 0x007B, 0x007C, 0x007D, 0x007E, 0x007F, + /* 8 */ 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F, + /* 9 */ 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F, + /* A */ 0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, + 0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, + /* B */ 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, + 0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, + /* C */ 0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00E6, 0x00C7, + 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF, + /* D */ 0x00F0, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x00D7, + 0x00F8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x00DD, 0x00FE, 0x00DF, + /* E */ 0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7, + 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF, + /* F */ 0x00F0, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7, + 0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x00FE, 0x00FF, + + // Table 2 (for high byte 0x01) + + /* 0 */ 0x0100, 0x0101, 0x0102, 0x0103, 0x0104, 0x0105, 0x0106, 0x0107, + 0x0108, 0x0109, 0x010A, 0x010B, 0x010C, 0x010D, 0x010E, 0x010F, + /* 1 */ 0x0111, 0x0111, 0x0112, 0x0113, 0x0114, 0x0115, 0x0116, 0x0117, + 0x0118, 0x0119, 0x011A, 0x011B, 0x011C, 0x011D, 0x011E, 0x011F, + /* 2 */ 0x0120, 0x0121, 0x0122, 0x0123, 0x0124, 0x0125, 0x0127, 0x0127, + 0x0128, 0x0129, 0x012A, 0x012B, 0x012C, 0x012D, 0x012E, 0x012F, + /* 3 */ 0x0130, 0x0131, 0x0133, 0x0133, 0x0134, 0x0135, 0x0136, 0x0137, + 0x0138, 0x0139, 0x013A, 0x013B, 0x013C, 0x013D, 0x013E, 0x0140, + /* 4 */ 0x0140, 0x0142, 0x0142, 0x0143, 0x0144, 0x0145, 0x0146, 0x0147, + 0x0148, 0x0149, 0x014B, 0x014B, 0x014C, 0x014D, 0x014E, 0x014F, + /* 5 */ 0x0150, 0x0151, 0x0153, 0x0153, 0x0154, 0x0155, 0x0156, 0x0157, + 0x0158, 0x0159, 0x015A, 0x015B, 0x015C, 0x015D, 0x015E, 0x015F, + /* 6 */ 0x0160, 0x0161, 0x0162, 0x0163, 0x0164, 0x0165, 0x0167, 0x0167, + 0x0168, 0x0169, 0x016A, 0x016B, 0x016C, 0x016D, 0x016E, 0x016F, + /* 7 */ 0x0170, 0x0171, 0x0172, 0x0173, 0x0174, 0x0175, 0x0176, 0x0177, + 0x0178, 0x0179, 0x017A, 0x017B, 0x017C, 0x017D, 0x017E, 0x017F, + /* 8 */ 0x0180, 0x0253, 0x0183, 0x0183, 0x0185, 0x0185, 0x0254, 0x0188, + 0x0188, 0x0256, 0x0257, 0x018C, 0x018C, 0x018D, 0x01DD, 0x0259, + /* 9 */ 0x025B, 0x0192, 0x0192, 0x0260, 0x0263, 0x0195, 0x0269, 0x0268, + 0x0199, 0x0199, 0x019A, 0x019B, 0x026F, 0x0272, 0x019E, 0x0275, + /* A */ 0x01A0, 0x01A1, 0x01A3, 0x01A3, 0x01A5, 0x01A5, 0x01A6, 0x01A8, + 0x01A8, 0x0283, 0x01AA, 0x01AB, 0x01AD, 0x01AD, 0x0288, 0x01AF, + /* B */ 0x01B0, 0x028A, 0x028B, 0x01B4, 0x01B4, 0x01B6, 0x01B6, 0x0292, + 0x01B9, 0x01B9, 0x01BA, 0x01BB, 0x01BD, 0x01BD, 0x01BE, 0x01BF, + /* C */ 0x01C0, 0x01C1, 0x01C2, 0x01C3, 0x01C6, 0x01C6, 0x01C6, 0x01C9, + 0x01C9, 0x01C9, 0x01CC, 0x01CC, 0x01CC, 0x01CD, 0x01CE, 0x01CF, + /* D */ 0x01D0, 0x01D1, 0x01D2, 0x01D3, 0x01D4, 0x01D5, 0x01D6, 0x01D7, + 0x01D8, 0x01D9, 0x01DA, 0x01DB, 0x01DC, 0x01DD, 0x01DE, 0x01DF, + /* E */ 0x01E0, 0x01E1, 0x01E2, 0x01E3, 0x01E5, 0x01E5, 0x01E6, 0x01E7, + 0x01E8, 0x01E9, 0x01EA, 0x01EB, 0x01EC, 0x01ED, 0x01EE, 0x01EF, + /* F */ 0x01F0, 0x01F3, 0x01F3, 0x01F3, 0x01F4, 0x01F5, 0x01F6, 0x01F7, + 0x01F8, 0x01F9, 0x01FA, 0x01FB, 0x01FC, 0x01FD, 0x01FE, 0x01FF, + + // Table 3 (for high byte 0x03) + + /* 0 */ 0x0300, 0x0301, 0x0302, 0x0303, 0x0304, 0x0305, 0x0306, 0x0307, + 0x0308, 0x0309, 0x030A, 0x030B, 0x030C, 0x030D, 0x030E, 0x030F, + /* 1 */ 0x0310, 0x0311, 0x0312, 0x0313, 0x0314, 0x0315, 0x0316, 0x0317, + 0x0318, 0x0319, 0x031A, 0x031B, 0x031C, 0x031D, 0x031E, 0x031F, + /* 2 */ 0x0320, 0x0321, 0x0322, 0x0323, 0x0324, 0x0325, 0x0326, 0x0327, + 0x0328, 0x0329, 0x032A, 0x032B, 0x032C, 0x032D, 0x032E, 0x032F, + /* 3 */ 0x0330, 0x0331, 0x0332, 0x0333, 0x0334, 0x0335, 0x0336, 0x0337, + 0x0338, 0x0339, 0x033A, 0x033B, 0x033C, 0x033D, 0x033E, 0x033F, + /* 4 */ 0x0340, 0x0341, 0x0342, 0x0343, 0x0344, 0x0345, 0x0346, 0x0347, + 0x0348, 0x0349, 0x034A, 0x034B, 0x034C, 0x034D, 0x034E, 0x034F, + /* 5 */ 0x0350, 0x0351, 0x0352, 0x0353, 0x0354, 0x0355, 0x0356, 0x0357, + 0x0358, 0x0359, 0x035A, 0x035B, 0x035C, 0x035D, 0x035E, 0x035F, + /* 6 */ 0x0360, 0x0361, 0x0362, 0x0363, 0x0364, 0x0365, 0x0366, 0x0367, + 0x0368, 0x0369, 0x036A, 0x036B, 0x036C, 0x036D, 0x036E, 0x036F, + /* 7 */ 0x0370, 0x0371, 0x0372, 0x0373, 0x0374, 0x0375, 0x0376, 0x0377, + 0x0378, 0x0379, 0x037A, 0x037B, 0x037C, 0x037D, 0x037E, 0x037F, + /* 8 */ 0x0380, 0x0381, 0x0382, 0x0383, 0x0384, 0x0385, 0x0386, 0x0387, + 0x0388, 0x0389, 0x038A, 0x038B, 0x038C, 0x038D, 0x038E, 0x038F, + /* 9 */ 0x0390, 0x03B1, 0x03B2, 0x03B3, 0x03B4, 0x03B5, 0x03B6, 0x03B7, + 0x03B8, 0x03B9, 0x03BA, 0x03BB, 0x03BC, 0x03BD, 0x03BE, 0x03BF, + /* A */ 0x03C0, 0x03C1, 0x03A2, 0x03C3, 0x03C4, 0x03C5, 0x03C6, 0x03C7, + 0x03C8, 0x03C9, 0x03AA, 0x03AB, 0x03AC, 0x03AD, 0x03AE, 0x03AF, + /* B */ 0x03B0, 0x03B1, 0x03B2, 0x03B3, 0x03B4, 0x03B5, 0x03B6, 0x03B7, + 0x03B8, 0x03B9, 0x03BA, 0x03BB, 0x03BC, 0x03BD, 0x03BE, 0x03BF, + /* C */ 0x03C0, 0x03C1, 0x03C2, 0x03C3, 0x03C4, 0x03C5, 0x03C6, 0x03C7, + 0x03C8, 0x03C9, 0x03CA, 0x03CB, 0x03CC, 0x03CD, 0x03CE, 0x03CF, + /* D */ 0x03D0, 0x03D1, 0x03D2, 0x03D3, 0x03D4, 0x03D5, 0x03D6, 0x03D7, + 0x03D8, 0x03D9, 0x03DA, 0x03DB, 0x03DC, 0x03DD, 0x03DE, 0x03DF, + /* E */ 0x03E0, 0x03E1, 0x03E3, 0x03E3, 0x03E5, 0x03E5, 0x03E7, 0x03E7, + 0x03E9, 0x03E9, 0x03EB, 0x03EB, 0x03ED, 0x03ED, 0x03EF, 0x03EF, + /* F */ 0x03F0, 0x03F1, 0x03F2, 0x03F3, 0x03F4, 0x03F5, 0x03F6, 0x03F7, + 0x03F8, 0x03F9, 0x03FA, 0x03FB, 0x03FC, 0x03FD, 0x03FE, 0x03FF, + + // Table 4 (for high byte 0x04) + + /* 0 */ 0x0400, 0x0401, 0x0452, 0x0403, 0x0454, 0x0455, 0x0456, 0x0407, + 0x0458, 0x0459, 0x045A, 0x045B, 0x040C, 0x040D, 0x040E, 0x045F, + /* 1 */ 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, + 0x0438, 0x0419, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F, + /* 2 */ 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, + 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F, + /* 3 */ 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, + 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F, + /* 4 */ 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, + 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F, + /* 5 */ 0x0450, 0x0451, 0x0452, 0x0453, 0x0454, 0x0455, 0x0456, 0x0457, + 0x0458, 0x0459, 0x045A, 0x045B, 0x045C, 0x045D, 0x045E, 0x045F, + /* 6 */ 0x0461, 0x0461, 0x0463, 0x0463, 0x0465, 0x0465, 0x0467, 0x0467, + 0x0469, 0x0469, 0x046B, 0x046B, 0x046D, 0x046D, 0x046F, 0x046F, + /* 7 */ 0x0471, 0x0471, 0x0473, 0x0473, 0x0475, 0x0475, 0x0476, 0x0477, + 0x0479, 0x0479, 0x047B, 0x047B, 0x047D, 0x047D, 0x047F, 0x047F, + /* 8 */ 0x0481, 0x0481, 0x0482, 0x0483, 0x0484, 0x0485, 0x0486, 0x0487, + 0x0488, 0x0489, 0x048A, 0x048B, 0x048C, 0x048D, 0x048E, 0x048F, + /* 9 */ 0x0491, 0x0491, 0x0493, 0x0493, 0x0495, 0x0495, 0x0497, 0x0497, + 0x0499, 0x0499, 0x049B, 0x049B, 0x049D, 0x049D, 0x049F, 0x049F, + /* A */ 0x04A1, 0x04A1, 0x04A3, 0x04A3, 0x04A5, 0x04A5, 0x04A7, 0x04A7, + 0x04A9, 0x04A9, 0x04AB, 0x04AB, 0x04AD, 0x04AD, 0x04AF, 0x04AF, + /* B */ 0x04B1, 0x04B1, 0x04B3, 0x04B3, 0x04B5, 0x04B5, 0x04B7, 0x04B7, + 0x04B9, 0x04B9, 0x04BB, 0x04BB, 0x04BD, 0x04BD, 0x04BF, 0x04BF, + /* C */ 0x04C0, 0x04C1, 0x04C2, 0x04C4, 0x04C4, 0x04C5, 0x04C6, 0x04C8, + 0x04C8, 0x04C9, 0x04CA, 0x04CC, 0x04CC, 0x04CD, 0x04CE, 0x04CF, + /* D */ 0x04D0, 0x04D1, 0x04D2, 0x04D3, 0x04D4, 0x04D5, 0x04D6, 0x04D7, + 0x04D8, 0x04D9, 0x04DA, 0x04DB, 0x04DC, 0x04DD, 0x04DE, 0x04DF, + /* E */ 0x04E0, 0x04E1, 0x04E2, 0x04E3, 0x04E4, 0x04E5, 0x04E6, 0x04E7, + 0x04E8, 0x04E9, 0x04EA, 0x04EB, 0x04EC, 0x04ED, 0x04EE, 0x04EF, + /* F */ 0x04F0, 0x04F1, 0x04F2, 0x04F3, 0x04F4, 0x04F5, 0x04F6, 0x04F7, + 0x04F8, 0x04F9, 0x04FA, 0x04FB, 0x04FC, 0x04FD, 0x04FE, 0x04FF, + + // Table 5 (for high byte 0x05) + + /* 0 */ 0x0500, 0x0501, 0x0502, 0x0503, 0x0504, 0x0505, 0x0506, 0x0507, + 0x0508, 0x0509, 0x050A, 0x050B, 0x050C, 0x050D, 0x050E, 0x050F, + /* 1 */ 0x0510, 0x0511, 0x0512, 0x0513, 0x0514, 0x0515, 0x0516, 0x0517, + 0x0518, 0x0519, 0x051A, 0x051B, 0x051C, 0x051D, 0x051E, 0x051F, + /* 2 */ 0x0520, 0x0521, 0x0522, 0x0523, 0x0524, 0x0525, 0x0526, 0x0527, + 0x0528, 0x0529, 0x052A, 0x052B, 0x052C, 0x052D, 0x052E, 0x052F, + /* 3 */ 0x0530, 0x0561, 0x0562, 0x0563, 0x0564, 0x0565, 0x0566, 0x0567, + 0x0568, 0x0569, 0x056A, 0x056B, 0x056C, 0x056D, 0x056E, 0x056F, + /* 4 */ 0x0570, 0x0571, 0x0572, 0x0573, 0x0574, 0x0575, 0x0576, 0x0577, + 0x0578, 0x0579, 0x057A, 0x057B, 0x057C, 0x057D, 0x057E, 0x057F, + /* 5 */ 0x0580, 0x0581, 0x0582, 0x0583, 0x0584, 0x0585, 0x0586, 0x0557, + 0x0558, 0x0559, 0x055A, 0x055B, 0x055C, 0x055D, 0x055E, 0x055F, + /* 6 */ 0x0560, 0x0561, 0x0562, 0x0563, 0x0564, 0x0565, 0x0566, 0x0567, + 0x0568, 0x0569, 0x056A, 0x056B, 0x056C, 0x056D, 0x056E, 0x056F, + /* 7 */ 0x0570, 0x0571, 0x0572, 0x0573, 0x0574, 0x0575, 0x0576, 0x0577, + 0x0578, 0x0579, 0x057A, 0x057B, 0x057C, 0x057D, 0x057E, 0x057F, + /* 8 */ 0x0580, 0x0581, 0x0582, 0x0583, 0x0584, 0x0585, 0x0586, 0x0587, + 0x0588, 0x0589, 0x058A, 0x058B, 0x058C, 0x058D, 0x058E, 0x058F, + /* 9 */ 0x0590, 0x0591, 0x0592, 0x0593, 0x0594, 0x0595, 0x0596, 0x0597, + 0x0598, 0x0599, 0x059A, 0x059B, 0x059C, 0x059D, 0x059E, 0x059F, + /* A */ 0x05A0, 0x05A1, 0x05A2, 0x05A3, 0x05A4, 0x05A5, 0x05A6, 0x05A7, + 0x05A8, 0x05A9, 0x05AA, 0x05AB, 0x05AC, 0x05AD, 0x05AE, 0x05AF, + /* B */ 0x05B0, 0x05B1, 0x05B2, 0x05B3, 0x05B4, 0x05B5, 0x05B6, 0x05B7, + 0x05B8, 0x05B9, 0x05BA, 0x05BB, 0x05BC, 0x05BD, 0x05BE, 0x05BF, + /* C */ 0x05C0, 0x05C1, 0x05C2, 0x05C3, 0x05C4, 0x05C5, 0x05C6, 0x05C7, + 0x05C8, 0x05C9, 0x05CA, 0x05CB, 0x05CC, 0x05CD, 0x05CE, 0x05CF, + /* D */ 0x05D0, 0x05D1, 0x05D2, 0x05D3, 0x05D4, 0x05D5, 0x05D6, 0x05D7, + 0x05D8, 0x05D9, 0x05DA, 0x05DB, 0x05DC, 0x05DD, 0x05DE, 0x05DF, + /* E */ 0x05E0, 0x05E1, 0x05E2, 0x05E3, 0x05E4, 0x05E5, 0x05E6, 0x05E7, + 0x05E8, 0x05E9, 0x05EA, 0x05EB, 0x05EC, 0x05ED, 0x05EE, 0x05EF, + /* F */ 0x05F0, 0x05F1, 0x05F2, 0x05F3, 0x05F4, 0x05F5, 0x05F6, 0x05F7, + 0x05F8, 0x05F9, 0x05FA, 0x05FB, 0x05FC, 0x05FD, 0x05FE, 0x05FF, + + // Table 6 (for high byte 0x10) + + /* 0 */ 0x1000, 0x1001, 0x1002, 0x1003, 0x1004, 0x1005, 0x1006, 0x1007, + 0x1008, 0x1009, 0x100A, 0x100B, 0x100C, 0x100D, 0x100E, 0x100F, + /* 1 */ 0x1010, 0x1011, 0x1012, 0x1013, 0x1014, 0x1015, 0x1016, 0x1017, + 0x1018, 0x1019, 0x101A, 0x101B, 0x101C, 0x101D, 0x101E, 0x101F, + /* 2 */ 0x1020, 0x1021, 0x1022, 0x1023, 0x1024, 0x1025, 0x1026, 0x1027, + 0x1028, 0x1029, 0x102A, 0x102B, 0x102C, 0x102D, 0x102E, 0x102F, + /* 3 */ 0x1030, 0x1031, 0x1032, 0x1033, 0x1034, 0x1035, 0x1036, 0x1037, + 0x1038, 0x1039, 0x103A, 0x103B, 0x103C, 0x103D, 0x103E, 0x103F, + /* 4 */ 0x1040, 0x1041, 0x1042, 0x1043, 0x1044, 0x1045, 0x1046, 0x1047, + 0x1048, 0x1049, 0x104A, 0x104B, 0x104C, 0x104D, 0x104E, 0x104F, + /* 5 */ 0x1050, 0x1051, 0x1052, 0x1053, 0x1054, 0x1055, 0x1056, 0x1057, + 0x1058, 0x1059, 0x105A, 0x105B, 0x105C, 0x105D, 0x105E, 0x105F, + /* 6 */ 0x1060, 0x1061, 0x1062, 0x1063, 0x1064, 0x1065, 0x1066, 0x1067, + 0x1068, 0x1069, 0x106A, 0x106B, 0x106C, 0x106D, 0x106E, 0x106F, + /* 7 */ 0x1070, 0x1071, 0x1072, 0x1073, 0x1074, 0x1075, 0x1076, 0x1077, + 0x1078, 0x1079, 0x107A, 0x107B, 0x107C, 0x107D, 0x107E, 0x107F, + /* 8 */ 0x1080, 0x1081, 0x1082, 0x1083, 0x1084, 0x1085, 0x1086, 0x1087, + 0x1088, 0x1089, 0x108A, 0x108B, 0x108C, 0x108D, 0x108E, 0x108F, + /* 9 */ 0x1090, 0x1091, 0x1092, 0x1093, 0x1094, 0x1095, 0x1096, 0x1097, + 0x1098, 0x1099, 0x109A, 0x109B, 0x109C, 0x109D, 0x109E, 0x109F, + /* A */ 0x10D0, 0x10D1, 0x10D2, 0x10D3, 0x10D4, 0x10D5, 0x10D6, 0x10D7, + 0x10D8, 0x10D9, 0x10DA, 0x10DB, 0x10DC, 0x10DD, 0x10DE, 0x10DF, + /* B */ 0x10E0, 0x10E1, 0x10E2, 0x10E3, 0x10E4, 0x10E5, 0x10E6, 0x10E7, + 0x10E8, 0x10E9, 0x10EA, 0x10EB, 0x10EC, 0x10ED, 0x10EE, 0x10EF, + /* C */ 0x10F0, 0x10F1, 0x10F2, 0x10F3, 0x10F4, 0x10F5, 0x10C6, 0x10C7, + 0x10C8, 0x10C9, 0x10CA, 0x10CB, 0x10CC, 0x10CD, 0x10CE, 0x10CF, + /* D */ 0x10D0, 0x10D1, 0x10D2, 0x10D3, 0x10D4, 0x10D5, 0x10D6, 0x10D7, + 0x10D8, 0x10D9, 0x10DA, 0x10DB, 0x10DC, 0x10DD, 0x10DE, 0x10DF, + /* E */ 0x10E0, 0x10E1, 0x10E2, 0x10E3, 0x10E4, 0x10E5, 0x10E6, 0x10E7, + 0x10E8, 0x10E9, 0x10EA, 0x10EB, 0x10EC, 0x10ED, 0x10EE, 0x10EF, + /* F */ 0x10F0, 0x10F1, 0x10F2, 0x10F3, 0x10F4, 0x10F5, 0x10F6, 0x10F7, + 0x10F8, 0x10F9, 0x10FA, 0x10FB, 0x10FC, 0x10FD, 0x10FE, 0x10FF, + + // Table 7 (for high byte 0x20) + + /* 0 */ 0x2000, 0x2001, 0x2002, 0x2003, 0x2004, 0x2005, 0x2006, 0x2007, + 0x2008, 0x2009, 0x200A, 0x200B, 0x0000, 0x0000, 0x0000, 0x0000, + /* 1 */ 0x2010, 0x2011, 0x2012, 0x2013, 0x2014, 0x2015, 0x2016, 0x2017, + 0x2018, 0x2019, 0x201A, 0x201B, 0x201C, 0x201D, 0x201E, 0x201F, + /* 2 */ 0x2020, 0x2021, 0x2022, 0x2023, 0x2024, 0x2025, 0x2026, 0x2027, + 0x2028, 0x2029, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x202F, + /* 3 */ 0x2030, 0x2031, 0x2032, 0x2033, 0x2034, 0x2035, 0x2036, 0x2037, + 0x2038, 0x2039, 0x203A, 0x203B, 0x203C, 0x203D, 0x203E, 0x203F, + /* 4 */ 0x2040, 0x2041, 0x2042, 0x2043, 0x2044, 0x2045, 0x2046, 0x2047, + 0x2048, 0x2049, 0x204A, 0x204B, 0x204C, 0x204D, 0x204E, 0x204F, + /* 5 */ 0x2050, 0x2051, 0x2052, 0x2053, 0x2054, 0x2055, 0x2056, 0x2057, + 0x2058, 0x2059, 0x205A, 0x205B, 0x205C, 0x205D, 0x205E, 0x205F, + /* 6 */ 0x2060, 0x2061, 0x2062, 0x2063, 0x2064, 0x2065, 0x2066, 0x2067, + 0x2068, 0x2069, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* 7 */ 0x2070, 0x2071, 0x2072, 0x2073, 0x2074, 0x2075, 0x2076, 0x2077, + 0x2078, 0x2079, 0x207A, 0x207B, 0x207C, 0x207D, 0x207E, 0x207F, + /* 8 */ 0x2080, 0x2081, 0x2082, 0x2083, 0x2084, 0x2085, 0x2086, 0x2087, + 0x2088, 0x2089, 0x208A, 0x208B, 0x208C, 0x208D, 0x208E, 0x208F, + /* 9 */ 0x2090, 0x2091, 0x2092, 0x2093, 0x2094, 0x2095, 0x2096, 0x2097, + 0x2098, 0x2099, 0x209A, 0x209B, 0x209C, 0x209D, 0x209E, 0x209F, + /* A */ 0x20A0, 0x20A1, 0x20A2, 0x20A3, 0x20A4, 0x20A5, 0x20A6, 0x20A7, + 0x20A8, 0x20A9, 0x20AA, 0x20AB, 0x20AC, 0x20AD, 0x20AE, 0x20AF, + /* B */ 0x20B0, 0x20B1, 0x20B2, 0x20B3, 0x20B4, 0x20B5, 0x20B6, 0x20B7, + 0x20B8, 0x20B9, 0x20BA, 0x20BB, 0x20BC, 0x20BD, 0x20BE, 0x20BF, + /* C */ 0x20C0, 0x20C1, 0x20C2, 0x20C3, 0x20C4, 0x20C5, 0x20C6, 0x20C7, + 0x20C8, 0x20C9, 0x20CA, 0x20CB, 0x20CC, 0x20CD, 0x20CE, 0x20CF, + /* D */ 0x20D0, 0x20D1, 0x20D2, 0x20D3, 0x20D4, 0x20D5, 0x20D6, 0x20D7, + 0x20D8, 0x20D9, 0x20DA, 0x20DB, 0x20DC, 0x20DD, 0x20DE, 0x20DF, + /* E */ 0x20E0, 0x20E1, 0x20E2, 0x20E3, 0x20E4, 0x20E5, 0x20E6, 0x20E7, + 0x20E8, 0x20E9, 0x20EA, 0x20EB, 0x20EC, 0x20ED, 0x20EE, 0x20EF, + /* F */ 0x20F0, 0x20F1, 0x20F2, 0x20F3, 0x20F4, 0x20F5, 0x20F6, 0x20F7, + 0x20F8, 0x20F9, 0x20FA, 0x20FB, 0x20FC, 0x20FD, 0x20FE, 0x20FF, + + // Table 8 (for high byte 0x21) + + /* 0 */ 0x2100, 0x2101, 0x2102, 0x2103, 0x2104, 0x2105, 0x2106, 0x2107, + 0x2108, 0x2109, 0x210A, 0x210B, 0x210C, 0x210D, 0x210E, 0x210F, + /* 1 */ 0x2110, 0x2111, 0x2112, 0x2113, 0x2114, 0x2115, 0x2116, 0x2117, + 0x2118, 0x2119, 0x211A, 0x211B, 0x211C, 0x211D, 0x211E, 0x211F, + /* 2 */ 0x2120, 0x2121, 0x2122, 0x2123, 0x2124, 0x2125, 0x2126, 0x2127, + 0x2128, 0x2129, 0x212A, 0x212B, 0x212C, 0x212D, 0x212E, 0x212F, + /* 3 */ 0x2130, 0x2131, 0x2132, 0x2133, 0x2134, 0x2135, 0x2136, 0x2137, + 0x2138, 0x2139, 0x213A, 0x213B, 0x213C, 0x213D, 0x213E, 0x213F, + /* 4 */ 0x2140, 0x2141, 0x2142, 0x2143, 0x2144, 0x2145, 0x2146, 0x2147, + 0x2148, 0x2149, 0x214A, 0x214B, 0x214C, 0x214D, 0x214E, 0x214F, + /* 5 */ 0x2150, 0x2151, 0x2152, 0x2153, 0x2154, 0x2155, 0x2156, 0x2157, + 0x2158, 0x2159, 0x215A, 0x215B, 0x215C, 0x215D, 0x215E, 0x215F, + /* 6 */ 0x2170, 0x2171, 0x2172, 0x2173, 0x2174, 0x2175, 0x2176, 0x2177, + 0x2178, 0x2179, 0x217A, 0x217B, 0x217C, 0x217D, 0x217E, 0x217F, + /* 7 */ 0x2170, 0x2171, 0x2172, 0x2173, 0x2174, 0x2175, 0x2176, 0x2177, + 0x2178, 0x2179, 0x217A, 0x217B, 0x217C, 0x217D, 0x217E, 0x217F, + /* 8 */ 0x2180, 0x2181, 0x2182, 0x2183, 0x2184, 0x2185, 0x2186, 0x2187, + 0x2188, 0x2189, 0x218A, 0x218B, 0x218C, 0x218D, 0x218E, 0x218F, + /* 9 */ 0x2190, 0x2191, 0x2192, 0x2193, 0x2194, 0x2195, 0x2196, 0x2197, + 0x2198, 0x2199, 0x219A, 0x219B, 0x219C, 0x219D, 0x219E, 0x219F, + /* A */ 0x21A0, 0x21A1, 0x21A2, 0x21A3, 0x21A4, 0x21A5, 0x21A6, 0x21A7, + 0x21A8, 0x21A9, 0x21AA, 0x21AB, 0x21AC, 0x21AD, 0x21AE, 0x21AF, + /* B */ 0x21B0, 0x21B1, 0x21B2, 0x21B3, 0x21B4, 0x21B5, 0x21B6, 0x21B7, + 0x21B8, 0x21B9, 0x21BA, 0x21BB, 0x21BC, 0x21BD, 0x21BE, 0x21BF, + /* C */ 0x21C0, 0x21C1, 0x21C2, 0x21C3, 0x21C4, 0x21C5, 0x21C6, 0x21C7, + 0x21C8, 0x21C9, 0x21CA, 0x21CB, 0x21CC, 0x21CD, 0x21CE, 0x21CF, + /* D */ 0x21D0, 0x21D1, 0x21D2, 0x21D3, 0x21D4, 0x21D5, 0x21D6, 0x21D7, + 0x21D8, 0x21D9, 0x21DA, 0x21DB, 0x21DC, 0x21DD, 0x21DE, 0x21DF, + /* E */ 0x21E0, 0x21E1, 0x21E2, 0x21E3, 0x21E4, 0x21E5, 0x21E6, 0x21E7, + 0x21E8, 0x21E9, 0x21EA, 0x21EB, 0x21EC, 0x21ED, 0x21EE, 0x21EF, + /* F */ 0x21F0, 0x21F1, 0x21F2, 0x21F3, 0x21F4, 0x21F5, 0x21F6, 0x21F7, + 0x21F8, 0x21F9, 0x21FA, 0x21FB, 0x21FC, 0x21FD, 0x21FE, 0x21FF, + + // Table 9 (for high byte 0xFE) + + /* 0 */ 0xFE00, 0xFE01, 0xFE02, 0xFE03, 0xFE04, 0xFE05, 0xFE06, 0xFE07, + 0xFE08, 0xFE09, 0xFE0A, 0xFE0B, 0xFE0C, 0xFE0D, 0xFE0E, 0xFE0F, + /* 1 */ 0xFE10, 0xFE11, 0xFE12, 0xFE13, 0xFE14, 0xFE15, 0xFE16, 0xFE17, + 0xFE18, 0xFE19, 0xFE1A, 0xFE1B, 0xFE1C, 0xFE1D, 0xFE1E, 0xFE1F, + /* 2 */ 0xFE20, 0xFE21, 0xFE22, 0xFE23, 0xFE24, 0xFE25, 0xFE26, 0xFE27, + 0xFE28, 0xFE29, 0xFE2A, 0xFE2B, 0xFE2C, 0xFE2D, 0xFE2E, 0xFE2F, + /* 3 */ 0xFE30, 0xFE31, 0xFE32, 0xFE33, 0xFE34, 0xFE35, 0xFE36, 0xFE37, + 0xFE38, 0xFE39, 0xFE3A, 0xFE3B, 0xFE3C, 0xFE3D, 0xFE3E, 0xFE3F, + /* 4 */ 0xFE40, 0xFE41, 0xFE42, 0xFE43, 0xFE44, 0xFE45, 0xFE46, 0xFE47, + 0xFE48, 0xFE49, 0xFE4A, 0xFE4B, 0xFE4C, 0xFE4D, 0xFE4E, 0xFE4F, + /* 5 */ 0xFE50, 0xFE51, 0xFE52, 0xFE53, 0xFE54, 0xFE55, 0xFE56, 0xFE57, + 0xFE58, 0xFE59, 0xFE5A, 0xFE5B, 0xFE5C, 0xFE5D, 0xFE5E, 0xFE5F, + /* 6 */ 0xFE60, 0xFE61, 0xFE62, 0xFE63, 0xFE64, 0xFE65, 0xFE66, 0xFE67, + 0xFE68, 0xFE69, 0xFE6A, 0xFE6B, 0xFE6C, 0xFE6D, 0xFE6E, 0xFE6F, + /* 7 */ 0xFE70, 0xFE71, 0xFE72, 0xFE73, 0xFE74, 0xFE75, 0xFE76, 0xFE77, + 0xFE78, 0xFE79, 0xFE7A, 0xFE7B, 0xFE7C, 0xFE7D, 0xFE7E, 0xFE7F, + /* 8 */ 0xFE80, 0xFE81, 0xFE82, 0xFE83, 0xFE84, 0xFE85, 0xFE86, 0xFE87, + 0xFE88, 0xFE89, 0xFE8A, 0xFE8B, 0xFE8C, 0xFE8D, 0xFE8E, 0xFE8F, + /* 9 */ 0xFE90, 0xFE91, 0xFE92, 0xFE93, 0xFE94, 0xFE95, 0xFE96, 0xFE97, + 0xFE98, 0xFE99, 0xFE9A, 0xFE9B, 0xFE9C, 0xFE9D, 0xFE9E, 0xFE9F, + /* A */ 0xFEA0, 0xFEA1, 0xFEA2, 0xFEA3, 0xFEA4, 0xFEA5, 0xFEA6, 0xFEA7, + 0xFEA8, 0xFEA9, 0xFEAA, 0xFEAB, 0xFEAC, 0xFEAD, 0xFEAE, 0xFEAF, + /* B */ 0xFEB0, 0xFEB1, 0xFEB2, 0xFEB3, 0xFEB4, 0xFEB5, 0xFEB6, 0xFEB7, + 0xFEB8, 0xFEB9, 0xFEBA, 0xFEBB, 0xFEBC, 0xFEBD, 0xFEBE, 0xFEBF, + /* C */ 0xFEC0, 0xFEC1, 0xFEC2, 0xFEC3, 0xFEC4, 0xFEC5, 0xFEC6, 0xFEC7, + 0xFEC8, 0xFEC9, 0xFECA, 0xFECB, 0xFECC, 0xFECD, 0xFECE, 0xFECF, + /* D */ 0xFED0, 0xFED1, 0xFED2, 0xFED3, 0xFED4, 0xFED5, 0xFED6, 0xFED7, + 0xFED8, 0xFED9, 0xFEDA, 0xFEDB, 0xFEDC, 0xFEDD, 0xFEDE, 0xFEDF, + /* E */ 0xFEE0, 0xFEE1, 0xFEE2, 0xFEE3, 0xFEE4, 0xFEE5, 0xFEE6, 0xFEE7, + 0xFEE8, 0xFEE9, 0xFEEA, 0xFEEB, 0xFEEC, 0xFEED, 0xFEEE, 0xFEEF, + /* F */ 0xFEF0, 0xFEF1, 0xFEF2, 0xFEF3, 0xFEF4, 0xFEF5, 0xFEF6, 0xFEF7, + 0xFEF8, 0xFEF9, 0xFEFA, 0xFEFB, 0xFEFC, 0xFEFD, 0xFEFE, 0x0000, + + // Table 10 (for high byte 0xFF) + + /* 0 */ 0xFF00, 0xFF01, 0xFF02, 0xFF03, 0xFF04, 0xFF05, 0xFF06, 0xFF07, + 0xFF08, 0xFF09, 0xFF0A, 0xFF0B, 0xFF0C, 0xFF0D, 0xFF0E, 0xFF0F, + /* 1 */ 0xFF10, 0xFF11, 0xFF12, 0xFF13, 0xFF14, 0xFF15, 0xFF16, 0xFF17, + 0xFF18, 0xFF19, 0xFF1A, 0xFF1B, 0xFF1C, 0xFF1D, 0xFF1E, 0xFF1F, + /* 2 */ 0xFF20, 0xFF41, 0xFF42, 0xFF43, 0xFF44, 0xFF45, 0xFF46, 0xFF47, + 0xFF48, 0xFF49, 0xFF4A, 0xFF4B, 0xFF4C, 0xFF4D, 0xFF4E, 0xFF4F, + /* 3 */ 0xFF50, 0xFF51, 0xFF52, 0xFF53, 0xFF54, 0xFF55, 0xFF56, 0xFF57, + 0xFF58, 0xFF59, 0xFF5A, 0xFF3B, 0xFF3C, 0xFF3D, 0xFF3E, 0xFF3F, + /* 4 */ 0xFF40, 0xFF41, 0xFF42, 0xFF43, 0xFF44, 0xFF45, 0xFF46, 0xFF47, + 0xFF48, 0xFF49, 0xFF4A, 0xFF4B, 0xFF4C, 0xFF4D, 0xFF4E, 0xFF4F, + /* 5 */ 0xFF50, 0xFF51, 0xFF52, 0xFF53, 0xFF54, 0xFF55, 0xFF56, 0xFF57, + 0xFF58, 0xFF59, 0xFF5A, 0xFF5B, 0xFF5C, 0xFF5D, 0xFF5E, 0xFF5F, + /* 6 */ 0xFF60, 0xFF61, 0xFF62, 0xFF63, 0xFF64, 0xFF65, 0xFF66, 0xFF67, + 0xFF68, 0xFF69, 0xFF6A, 0xFF6B, 0xFF6C, 0xFF6D, 0xFF6E, 0xFF6F, + /* 7 */ 0xFF70, 0xFF71, 0xFF72, 0xFF73, 0xFF74, 0xFF75, 0xFF76, 0xFF77, + 0xFF78, 0xFF79, 0xFF7A, 0xFF7B, 0xFF7C, 0xFF7D, 0xFF7E, 0xFF7F, + /* 8 */ 0xFF80, 0xFF81, 0xFF82, 0xFF83, 0xFF84, 0xFF85, 0xFF86, 0xFF87, + 0xFF88, 0xFF89, 0xFF8A, 0xFF8B, 0xFF8C, 0xFF8D, 0xFF8E, 0xFF8F, + /* 9 */ 0xFF90, 0xFF91, 0xFF92, 0xFF93, 0xFF94, 0xFF95, 0xFF96, 0xFF97, + 0xFF98, 0xFF99, 0xFF9A, 0xFF9B, 0xFF9C, 0xFF9D, 0xFF9E, 0xFF9F, + /* A */ 0xFFA0, 0xFFA1, 0xFFA2, 0xFFA3, 0xFFA4, 0xFFA5, 0xFFA6, 0xFFA7, + 0xFFA8, 0xFFA9, 0xFFAA, 0xFFAB, 0xFFAC, 0xFFAD, 0xFFAE, 0xFFAF, + /* B */ 0xFFB0, 0xFFB1, 0xFFB2, 0xFFB3, 0xFFB4, 0xFFB5, 0xFFB6, 0xFFB7, + 0xFFB8, 0xFFB9, 0xFFBA, 0xFFBB, 0xFFBC, 0xFFBD, 0xFFBE, 0xFFBF, + /* C */ 0xFFC0, 0xFFC1, 0xFFC2, 0xFFC3, 0xFFC4, 0xFFC5, 0xFFC6, 0xFFC7, + 0xFFC8, 0xFFC9, 0xFFCA, 0xFFCB, 0xFFCC, 0xFFCD, 0xFFCE, 0xFFCF, + /* D */ 0xFFD0, 0xFFD1, 0xFFD2, 0xFFD3, 0xFFD4, 0xFFD5, 0xFFD6, 0xFFD7, + 0xFFD8, 0xFFD9, 0xFFDA, 0xFFDB, 0xFFDC, 0xFFDD, 0xFFDE, 0xFFDF, + /* E */ 0xFFE0, 0xFFE1, 0xFFE2, 0xFFE3, 0xFFE4, 0xFFE5, 0xFFE6, 0xFFE7, + 0xFFE8, 0xFFE9, 0xFFEA, 0xFFEB, 0xFFEC, 0xFFED, 0xFFEE, 0xFFEF, + /* F */ 0xFFF0, 0xFFF1, 0xFFF2, 0xFFF3, 0xFFF4, 0xFFF5, 0xFFF6, 0xFFF7, + 0xFFF8, 0xFFF9, 0xFFFA, 0xFFFB, 0xFFFC, 0xFFFD, 0xFFFE, 0xFFFF, +}; + +u16 hfsplus_decompose_table[] = { + /* base table */ + 0x0010, 0x04c0, 0x0000, 0x06f0, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0xffff, 0xffff, 0xffff, 0xffff, 0x0000, 0x07b0, + /* char table 0x0___ */ + 0x0020, 0x0070, 0x0160, 0x0190, 0x0230, 0x0000, 0x0000, 0x0000, + 0x0000, 0x02d0, 0x0340, 0x0360, 0x03b0, 0x03e0, 0x0400, 0x0430, + /* char table 0x00__ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0030, 0x0040, 0x0050, 0x0060, + /* char values 0x00c_ */ + 0x2042, 0x204a, 0x2052, 0x205a, 0x2062, 0x206a, 0x0000, 0x2072, + 0x207a, 0x2082, 0x208a, 0x2092, 0x209a, 0x20a2, 0x20aa, 0x20b2, + /* char values 0x00d_ */ + 0x0000, 0x20ba, 0x20c2, 0x20ca, 0x20d2, 0x20da, 0x20e2, 0x0000, + 0x0000, 0x20ea, 0x20f2, 0x20fa, 0x2102, 0x210a, 0x0000, 0x0000, + /* char values 0x00e_ */ + 0x2112, 0x211a, 0x2122, 0x212a, 0x2132, 0x213a, 0x0000, 0x2142, + 0x214a, 0x2152, 0x215a, 0x2162, 0x216a, 0x2172, 0x217a, 0x2182, + /* char values 0x00f_ */ + 0x0000, 0x218a, 0x2192, 0x219a, 0x21a2, 0x21aa, 0x21b2, 0x0000, + 0x0000, 0x21ba, 0x21c2, 0x21ca, 0x21d2, 0x21da, 0x0000, 0x21e2, + /* char table 0x01__ */ + 0x0080, 0x0090, 0x00a0, 0x00b0, 0x00c0, 0x00d0, 0x00e0, 0x00f0, + 0x0000, 0x0000, 0x0100, 0x0110, 0x0120, 0x0130, 0x0140, 0x0150, + /* char values 0x010_ */ + 0x21ea, 0x21f2, 0x21fa, 0x2202, 0x220a, 0x2212, 0x221a, 0x2222, + 0x222a, 0x2232, 0x223a, 0x2242, 0x224a, 0x2252, 0x225a, 0x2262, + /* char values 0x011_ */ + 0x0000, 0x0000, 0x226a, 0x2272, 0x227a, 0x2282, 0x228a, 0x2292, + 0x229a, 0x22a2, 0x22aa, 0x22b2, 0x22ba, 0x22c2, 0x22ca, 0x22d2, + /* char values 0x012_ */ + 0x22da, 0x22e2, 0x22ea, 0x22f2, 0x22fa, 0x2302, 0x0000, 0x0000, + 0x230a, 0x2312, 0x231a, 0x2322, 0x232a, 0x2332, 0x233a, 0x2342, + /* char values 0x013_ */ + 0x234a, 0x0000, 0x0000, 0x0000, 0x2352, 0x235a, 0x2362, 0x236a, + 0x0000, 0x2372, 0x237a, 0x2382, 0x238a, 0x2392, 0x239a, 0x0000, + /* char values 0x014_ */ + 0x0000, 0x0000, 0x0000, 0x23a2, 0x23aa, 0x23b2, 0x23ba, 0x23c2, + 0x23ca, 0x0000, 0x0000, 0x0000, 0x23d2, 0x23da, 0x23e2, 0x23ea, + /* char values 0x015_ */ + 0x23f2, 0x23fa, 0x0000, 0x0000, 0x2402, 0x240a, 0x2412, 0x241a, + 0x2422, 0x242a, 0x2432, 0x243a, 0x2442, 0x244a, 0x2452, 0x245a, + /* char values 0x016_ */ + 0x2462, 0x246a, 0x2472, 0x247a, 0x2482, 0x248a, 0x0000, 0x0000, + 0x2492, 0x249a, 0x24a2, 0x24aa, 0x24b2, 0x24ba, 0x24c2, 0x24ca, + /* char values 0x017_ */ + 0x24d2, 0x24da, 0x24e2, 0x24ea, 0x24f2, 0x24fa, 0x2502, 0x250a, + 0x2512, 0x251a, 0x2522, 0x252a, 0x2532, 0x253a, 0x2542, 0x0000, + /* char values 0x01a_ */ + 0x254a, 0x2552, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x255a, + /* char values 0x01b_ */ + 0x2562, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0x01c_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x256a, 0x2572, 0x257a, + /* char values 0x01d_ */ + 0x2582, 0x258a, 0x2592, 0x259a, 0x25a2, 0x25ab, 0x25b7, 0x25c3, + 0x25cf, 0x25db, 0x25e7, 0x25f3, 0x25ff, 0x0000, 0x260b, 0x2617, + /* char values 0x01e_ */ + 0x2623, 0x262f, 0x263a, 0x2642, 0x0000, 0x0000, 0x264a, 0x2652, + 0x265a, 0x2662, 0x266a, 0x2672, 0x267b, 0x2687, 0x2692, 0x269a, + /* char values 0x01f_ */ + 0x26a2, 0x0000, 0x0000, 0x0000, 0x26aa, 0x26b2, 0x0000, 0x0000, + 0x0000, 0x0000, 0x26bb, 0x26c7, 0x26d2, 0x26da, 0x26e2, 0x26ea, + /* char table 0x02__ */ + 0x0170, 0x0180, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0x020_ */ + 0x26f2, 0x26fa, 0x2702, 0x270a, 0x2712, 0x271a, 0x2722, 0x272a, + 0x2732, 0x273a, 0x2742, 0x274a, 0x2752, 0x275a, 0x2762, 0x276a, + /* char values 0x021_ */ + 0x2772, 0x277a, 0x2782, 0x278a, 0x2792, 0x279a, 0x27a2, 0x27aa, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char table 0x03__ */ + 0x0000, 0x01a0, 0x0000, 0x0000, 0x01b0, 0x0000, 0x0000, 0x01c0, + 0x01d0, 0x01e0, 0x01f0, 0x0200, 0x0210, 0x0220, 0x0000, 0x0000, + /* char values 0x031_ */ + 0x27b2, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0x034_ */ + 0x27b9, 0x27bd, 0x0000, 0x27c1, 0x27c6, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0x037_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x27cd, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x27d1, 0x0000, + /* char values 0x038_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x27d6, 0x27de, 0x27e5, + 0x27ea, 0x27f2, 0x27fa, 0x0000, 0x2802, 0x0000, 0x280a, 0x2812, + /* char values 0x039_ */ + 0x281b, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0x03a_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x2826, 0x282e, 0x2836, 0x283e, 0x2846, 0x284e, + /* char values 0x03b_ */ + 0x2857, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0x03c_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x2862, 0x286a, 0x2872, 0x287a, 0x2882, 0x0000, + /* char values 0x03d_ */ + 0x0000, 0x0000, 0x0000, 0x288a, 0x2892, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char table 0x04__ */ + 0x0240, 0x0250, 0x0000, 0x0260, 0x0000, 0x0270, 0x0000, 0x0280, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0290, 0x02a0, 0x02b0, 0x02c0, + /* char values 0x040_ */ + 0x0000, 0x289a, 0x0000, 0x28a2, 0x0000, 0x0000, 0x0000, 0x28aa, + 0x0000, 0x0000, 0x0000, 0x0000, 0x28b2, 0x0000, 0x28ba, 0x0000, + /* char values 0x041_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x28c2, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0x043_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x28ca, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0x045_ */ + 0x0000, 0x28d2, 0x0000, 0x28da, 0x0000, 0x0000, 0x0000, 0x28e2, + 0x0000, 0x0000, 0x0000, 0x0000, 0x28ea, 0x0000, 0x28f2, 0x0000, + /* char values 0x047_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x28fa, 0x2902, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0x04c_ */ + 0x0000, 0x290a, 0x2912, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0x04d_ */ + 0x291a, 0x2922, 0x292a, 0x2932, 0x2939, 0x293d, 0x2942, 0x294a, + 0x2951, 0x2955, 0x295a, 0x2962, 0x296a, 0x2972, 0x297a, 0x2982, + /* char values 0x04e_ */ + 0x2989, 0x298d, 0x2992, 0x299a, 0x29a2, 0x29aa, 0x29b2, 0x29ba, + 0x29c1, 0x29c5, 0x29ca, 0x29d2, 0x0000, 0x0000, 0x29da, 0x29e2, + /* char values 0x04f_ */ + 0x29ea, 0x29f2, 0x29fa, 0x2a02, 0x2a0a, 0x2a12, 0x0000, 0x0000, + 0x2a1a, 0x2a22, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char table 0x09__ */ + 0x0000, 0x0000, 0x02e0, 0x02f0, 0x0000, 0x0300, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0310, 0x0320, 0x0330, 0x0000, 0x0000, + /* char values 0x092_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x2a2a, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0x093_ */ + 0x0000, 0x2a32, 0x0000, 0x0000, 0x2a3a, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0x095_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x2a42, 0x2a4a, 0x2a52, 0x2a5a, 0x2a62, 0x2a6a, 0x2a72, 0x2a7a, + /* char values 0x09b_ */ + 0x2a82, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0x09c_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x2a8a, 0x2a92, 0x0000, 0x0000, 0x0000, + /* char values 0x09d_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x2a9a, 0x2aa2, 0x0000, 0x2aaa, + /* char table 0x0a__ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0350, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0x0a5_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x2ab2, 0x2aba, 0x2ac2, 0x2aca, 0x0000, 0x2ad2, 0x0000, + /* char table 0x0b__ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0370, 0x0380, 0x0000, 0x0000, + 0x0000, 0x0390, 0x0000, 0x0000, 0x03a0, 0x0000, 0x0000, 0x0000, + /* char values 0x0b4_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x2ada, 0x0000, 0x0000, 0x2ae2, 0x2aea, 0x0000, 0x0000, 0x0000, + /* char values 0x0b5_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x2af2, 0x2afa, 0x0000, 0x2b02, + /* char values 0x0b9_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x2b0a, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0x0bc_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x2b12, 0x2b1a, 0x2b22, 0x0000, 0x0000, 0x0000, + /* char table 0x0c__ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x03c0, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x03d0, 0x0000, 0x0000, 0x0000, + /* char values 0x0c4_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x2b2a, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0x0cc_ */ + 0x2b32, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x2b3a, + 0x2b42, 0x0000, 0x2b4a, 0x2b53, 0x0000, 0x0000, 0x0000, 0x0000, + /* char table 0x0d__ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x03f0, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0x0d4_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x2b5e, 0x2b66, 0x2b6e, 0x0000, 0x0000, 0x0000, + /* char table 0x0e__ */ + 0x0000, 0x0000, 0x0000, 0x0410, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0420, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0x0e3_ */ + 0x0000, 0x0000, 0x0000, 0x2b76, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0x0eb_ */ + 0x0000, 0x0000, 0x0000, 0x2b7e, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char table 0x0f__ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0440, 0x0450, 0x0460, 0x0470, + 0x0480, 0x0490, 0x04a0, 0x04b0, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0x0f4_ */ + 0x0000, 0x0000, 0x0000, 0x2b86, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x2b8e, 0x0000, 0x0000, + /* char values 0x0f5_ */ + 0x0000, 0x0000, 0x2b96, 0x0000, 0x0000, 0x0000, 0x0000, 0x2b9e, + 0x0000, 0x0000, 0x0000, 0x0000, 0x2ba6, 0x0000, 0x0000, 0x0000, + /* char values 0x0f6_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x2bae, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0x0f7_ */ + 0x0000, 0x0000, 0x0000, 0x2bb6, 0x0000, 0x2bbe, 0x2bc6, 0x2bcf, + 0x2bda, 0x2be3, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0x0f8_ */ + 0x0000, 0x2bee, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0x0f9_ */ + 0x0000, 0x0000, 0x0000, 0x2bf6, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x2bfe, 0x0000, 0x0000, + /* char values 0x0fa_ */ + 0x0000, 0x0000, 0x2c06, 0x0000, 0x0000, 0x0000, 0x0000, 0x2c0e, + 0x0000, 0x0000, 0x0000, 0x0000, 0x2c16, 0x0000, 0x0000, 0x0000, + /* char values 0x0fb_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x2c1e, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char table 0x1___ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x04d0, 0x05e0, + /* char table 0x1e__ */ + 0x04e0, 0x04f0, 0x0500, 0x0510, 0x0520, 0x0530, 0x0540, 0x0550, + 0x0560, 0x0570, 0x0580, 0x0590, 0x05a0, 0x05b0, 0x05c0, 0x05d0, + /* char values 0x1e0_ */ + 0x2c26, 0x2c2e, 0x2c36, 0x2c3e, 0x2c46, 0x2c4e, 0x2c56, 0x2c5e, + 0x2c67, 0x2c73, 0x2c7e, 0x2c86, 0x2c8e, 0x2c96, 0x2c9e, 0x2ca6, + /* char values 0x1e1_ */ + 0x2cae, 0x2cb6, 0x2cbe, 0x2cc6, 0x2ccf, 0x2cdb, 0x2ce7, 0x2cf3, + 0x2cfe, 0x2d06, 0x2d0e, 0x2d16, 0x2d1f, 0x2d2b, 0x2d36, 0x2d3e, + /* char values 0x1e2_ */ + 0x2d46, 0x2d4e, 0x2d56, 0x2d5e, 0x2d66, 0x2d6e, 0x2d76, 0x2d7e, + 0x2d86, 0x2d8e, 0x2d96, 0x2d9e, 0x2da6, 0x2dae, 0x2db7, 0x2dc3, + /* char values 0x1e3_ */ + 0x2dce, 0x2dd6, 0x2dde, 0x2de6, 0x2dee, 0x2df6, 0x2dfe, 0x2e06, + 0x2e0f, 0x2e1b, 0x2e26, 0x2e2e, 0x2e36, 0x2e3e, 0x2e46, 0x2e4e, + /* char values 0x1e4_ */ + 0x2e56, 0x2e5e, 0x2e66, 0x2e6e, 0x2e76, 0x2e7e, 0x2e86, 0x2e8e, + 0x2e96, 0x2e9e, 0x2ea6, 0x2eae, 0x2eb7, 0x2ec3, 0x2ecf, 0x2edb, + /* char values 0x1e5_ */ + 0x2ee7, 0x2ef3, 0x2eff, 0x2f0b, 0x2f16, 0x2f1e, 0x2f26, 0x2f2e, + 0x2f36, 0x2f3e, 0x2f46, 0x2f4e, 0x2f57, 0x2f63, 0x2f6e, 0x2f76, + /* char values 0x1e6_ */ + 0x2f7e, 0x2f86, 0x2f8e, 0x2f96, 0x2f9f, 0x2fab, 0x2fb7, 0x2fc3, + 0x2fcf, 0x2fdb, 0x2fe6, 0x2fee, 0x2ff6, 0x2ffe, 0x3006, 0x300e, + /* char values 0x1e7_ */ + 0x3016, 0x301e, 0x3026, 0x302e, 0x3036, 0x303e, 0x3046, 0x304e, + 0x3057, 0x3063, 0x306f, 0x307b, 0x3086, 0x308e, 0x3096, 0x309e, + /* char values 0x1e8_ */ + 0x30a6, 0x30ae, 0x30b6, 0x30be, 0x30c6, 0x30ce, 0x30d6, 0x30de, + 0x30e6, 0x30ee, 0x30f6, 0x30fe, 0x3106, 0x310e, 0x3116, 0x311e, + /* char values 0x1e9_ */ + 0x3126, 0x312e, 0x3136, 0x313e, 0x3146, 0x314e, 0x3156, 0x315e, + 0x3166, 0x316e, 0x0000, 0x3176, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0x1ea_ */ + 0x317e, 0x3186, 0x318e, 0x3196, 0x319f, 0x31ab, 0x31b7, 0x31c3, + 0x31cf, 0x31db, 0x31e7, 0x31f3, 0x31ff, 0x320b, 0x3217, 0x3223, + /* char values 0x1eb_ */ + 0x322f, 0x323b, 0x3247, 0x3253, 0x325f, 0x326b, 0x3277, 0x3283, + 0x328e, 0x3296, 0x329e, 0x32a6, 0x32ae, 0x32b6, 0x32bf, 0x32cb, + /* char values 0x1ec_ */ + 0x32d7, 0x32e3, 0x32ef, 0x32fb, 0x3307, 0x3313, 0x331f, 0x332b, + 0x3336, 0x333e, 0x3346, 0x334e, 0x3356, 0x335e, 0x3366, 0x336e, + /* char values 0x1ed_ */ + 0x3377, 0x3383, 0x338f, 0x339b, 0x33a7, 0x33b3, 0x33bf, 0x33cb, + 0x33d7, 0x33e3, 0x33ef, 0x33fb, 0x3407, 0x3413, 0x341f, 0x342b, + /* char values 0x1ee_ */ + 0x3437, 0x3443, 0x344f, 0x345b, 0x3466, 0x346e, 0x3476, 0x347e, + 0x3487, 0x3493, 0x349f, 0x34ab, 0x34b7, 0x34c3, 0x34cf, 0x34db, + /* char values 0x1ef_ */ + 0x34e7, 0x34f3, 0x34fe, 0x3506, 0x350e, 0x3516, 0x351e, 0x3526, + 0x352e, 0x3536, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char table 0x1f__ */ + 0x05f0, 0x0600, 0x0610, 0x0620, 0x0630, 0x0640, 0x0650, 0x0660, + 0x0670, 0x0680, 0x0690, 0x06a0, 0x06b0, 0x06c0, 0x06d0, 0x06e0, + /* char values 0x1f0_ */ + 0x353e, 0x3546, 0x354f, 0x355b, 0x3567, 0x3573, 0x357f, 0x358b, + 0x3596, 0x359e, 0x35a7, 0x35b3, 0x35bf, 0x35cb, 0x35d7, 0x35e3, + /* char values 0x1f1_ */ + 0x35ee, 0x35f6, 0x35ff, 0x360b, 0x3617, 0x3623, 0x0000, 0x0000, + 0x362e, 0x3636, 0x363f, 0x364b, 0x3657, 0x3663, 0x0000, 0x0000, + /* char values 0x1f2_ */ + 0x366e, 0x3676, 0x367f, 0x368b, 0x3697, 0x36a3, 0x36af, 0x36bb, + 0x36c6, 0x36ce, 0x36d7, 0x36e3, 0x36ef, 0x36fb, 0x3707, 0x3713, + /* char values 0x1f3_ */ + 0x371e, 0x3726, 0x372f, 0x373b, 0x3747, 0x3753, 0x375f, 0x376b, + 0x3776, 0x377e, 0x3787, 0x3793, 0x379f, 0x37ab, 0x37b7, 0x37c3, + /* char values 0x1f4_ */ + 0x37ce, 0x37d6, 0x37df, 0x37eb, 0x37f7, 0x3803, 0x0000, 0x0000, + 0x380e, 0x3816, 0x381f, 0x382b, 0x3837, 0x3843, 0x0000, 0x0000, + /* char values 0x1f5_ */ + 0x384e, 0x3856, 0x385f, 0x386b, 0x3877, 0x3883, 0x388f, 0x389b, + 0x0000, 0x38a6, 0x0000, 0x38af, 0x0000, 0x38bb, 0x0000, 0x38c7, + /* char values 0x1f6_ */ + 0x38d2, 0x38da, 0x38e3, 0x38ef, 0x38fb, 0x3907, 0x3913, 0x391f, + 0x392a, 0x3932, 0x393b, 0x3947, 0x3953, 0x395f, 0x396b, 0x3977, + /* char values 0x1f7_ */ + 0x3982, 0x398a, 0x3992, 0x399a, 0x39a2, 0x39aa, 0x39b2, 0x39ba, + 0x39c2, 0x39ca, 0x39d2, 0x39da, 0x39e2, 0x39ea, 0x0000, 0x0000, + /* char values 0x1f8_ */ + 0x39f3, 0x39ff, 0x3a0c, 0x3a1c, 0x3a2c, 0x3a3c, 0x3a4c, 0x3a5c, + 0x3a6b, 0x3a77, 0x3a84, 0x3a94, 0x3aa4, 0x3ab4, 0x3ac4, 0x3ad4, + /* char values 0x1f9_ */ + 0x3ae3, 0x3aef, 0x3afc, 0x3b0c, 0x3b1c, 0x3b2c, 0x3b3c, 0x3b4c, + 0x3b5b, 0x3b67, 0x3b74, 0x3b84, 0x3b94, 0x3ba4, 0x3bb4, 0x3bc4, + /* char values 0x1fa_ */ + 0x3bd3, 0x3bdf, 0x3bec, 0x3bfc, 0x3c0c, 0x3c1c, 0x3c2c, 0x3c3c, + 0x3c4b, 0x3c57, 0x3c64, 0x3c74, 0x3c84, 0x3c94, 0x3ca4, 0x3cb4, + /* char values 0x1fb_ */ + 0x3cc2, 0x3cca, 0x3cd3, 0x3cde, 0x3ce7, 0x0000, 0x3cf2, 0x3cfb, + 0x3d06, 0x3d0e, 0x3d16, 0x3d1e, 0x3d26, 0x0000, 0x3d2d, 0x0000, + /* char values 0x1fc_ */ + 0x0000, 0x3d32, 0x3d3b, 0x3d46, 0x3d4f, 0x0000, 0x3d5a, 0x3d63, + 0x3d6e, 0x3d76, 0x3d7e, 0x3d86, 0x3d8e, 0x3d96, 0x3d9e, 0x3da6, + /* char values 0x1fd_ */ + 0x3dae, 0x3db6, 0x3dbf, 0x3dcb, 0x0000, 0x0000, 0x3dd6, 0x3ddf, + 0x3dea, 0x3df2, 0x3dfa, 0x3e02, 0x0000, 0x3e0a, 0x3e12, 0x3e1a, + /* char values 0x1fe_ */ + 0x3e22, 0x3e2a, 0x3e33, 0x3e3f, 0x3e4a, 0x3e52, 0x3e5a, 0x3e63, + 0x3e6e, 0x3e76, 0x3e7e, 0x3e86, 0x3e8e, 0x3e96, 0x3e9e, 0x3ea5, + /* char values 0x1ff_ */ + 0x0000, 0x0000, 0x3eab, 0x3eb6, 0x3ebf, 0x0000, 0x3eca, 0x3ed3, + 0x3ede, 0x3ee6, 0x3eee, 0x3ef6, 0x3efe, 0x3f05, 0x0000, 0x0000, + /* char table 0x3___ */ + 0x0700, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char table 0x30__ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0710, 0x0720, 0x0730, 0x0740, + 0x0000, 0x0750, 0x0760, 0x0770, 0x0780, 0x0790, 0x0000, 0x07a0, + /* char values 0x304_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x3f0a, 0x0000, 0x3f12, 0x0000, + /* char values 0x305_ */ + 0x3f1a, 0x0000, 0x3f22, 0x0000, 0x3f2a, 0x0000, 0x3f32, 0x0000, + 0x3f3a, 0x0000, 0x3f42, 0x0000, 0x3f4a, 0x0000, 0x3f52, 0x0000, + /* char values 0x306_ */ + 0x3f5a, 0x0000, 0x3f62, 0x0000, 0x0000, 0x3f6a, 0x0000, 0x3f72, + 0x0000, 0x3f7a, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0x307_ */ + 0x3f82, 0x3f8a, 0x0000, 0x3f92, 0x3f9a, 0x0000, 0x3fa2, 0x3faa, + 0x0000, 0x3fb2, 0x3fba, 0x0000, 0x3fc2, 0x3fca, 0x0000, 0x0000, + /* char values 0x309_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x3fd2, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x3fda, 0x0000, + /* char values 0x30a_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x3fe2, 0x0000, 0x3fea, 0x0000, + /* char values 0x30b_ */ + 0x3ff2, 0x0000, 0x3ffa, 0x0000, 0x4002, 0x0000, 0x400a, 0x0000, + 0x4012, 0x0000, 0x401a, 0x0000, 0x4022, 0x0000, 0x402a, 0x0000, + /* char values 0x30c_ */ + 0x4032, 0x0000, 0x403a, 0x0000, 0x0000, 0x4042, 0x0000, 0x404a, + 0x0000, 0x4052, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0x30d_ */ + 0x405a, 0x4062, 0x0000, 0x406a, 0x4072, 0x0000, 0x407a, 0x4082, + 0x0000, 0x408a, 0x4092, 0x0000, 0x409a, 0x40a2, 0x0000, 0x0000, + /* char values 0x30f_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x40aa, 0x0000, 0x0000, 0x40b2, + 0x40ba, 0x40c2, 0x40ca, 0x0000, 0x0000, 0x0000, 0x40d2, 0x0000, + /* char table 0xf___ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x07c0, 0x0000, 0x0000, 0x0000, 0x0000, + /* char table 0xfb__ */ + 0x0000, 0x07d0, 0x07e0, 0x07f0, 0x0800, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + /* char values 0xfb1_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x40da, + /* char values 0xfb2_ */ + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x40e2, 0x40ea, 0x40f3, 0x40ff, 0x410a, 0x4112, + /* char values 0xfb3_ */ + 0x411a, 0x4122, 0x412a, 0x4132, 0x413a, 0x4142, 0x414a, 0x0000, + 0x4152, 0x415a, 0x4162, 0x416a, 0x4172, 0x0000, 0x417a, 0x0000, + /* char values 0xfb4_ */ + 0x4182, 0x418a, 0x0000, 0x4192, 0x419a, 0x0000, 0x41a2, 0x41aa, + 0x41b2, 0x41ba, 0x41c2, 0x41ca, 0x41d2, 0x41da, 0x41e2, 0x0000, + /* decomposed characters */ + 0x0041, 0x0300, 0x0041, 0x0301, 0x0041, 0x0302, 0x0041, 0x0303, + 0x0041, 0x0308, 0x0041, 0x030a, 0x0043, 0x0327, 0x0045, 0x0300, + 0x0045, 0x0301, 0x0045, 0x0302, 0x0045, 0x0308, 0x0049, 0x0300, + 0x0049, 0x0301, 0x0049, 0x0302, 0x0049, 0x0308, 0x004e, 0x0303, + 0x004f, 0x0300, 0x004f, 0x0301, 0x004f, 0x0302, 0x004f, 0x0303, + 0x004f, 0x0308, 0x0055, 0x0300, 0x0055, 0x0301, 0x0055, 0x0302, + 0x0055, 0x0308, 0x0059, 0x0301, 0x0061, 0x0300, 0x0061, 0x0301, + 0x0061, 0x0302, 0x0061, 0x0303, 0x0061, 0x0308, 0x0061, 0x030a, + 0x0063, 0x0327, 0x0065, 0x0300, 0x0065, 0x0301, 0x0065, 0x0302, + 0x0065, 0x0308, 0x0069, 0x0300, 0x0069, 0x0301, 0x0069, 0x0302, + 0x0069, 0x0308, 0x006e, 0x0303, 0x006f, 0x0300, 0x006f, 0x0301, + 0x006f, 0x0302, 0x006f, 0x0303, 0x006f, 0x0308, 0x0075, 0x0300, + 0x0075, 0x0301, 0x0075, 0x0302, 0x0075, 0x0308, 0x0079, 0x0301, + 0x0079, 0x0308, 0x0041, 0x0304, 0x0061, 0x0304, 0x0041, 0x0306, + 0x0061, 0x0306, 0x0041, 0x0328, 0x0061, 0x0328, 0x0043, 0x0301, + 0x0063, 0x0301, 0x0043, 0x0302, 0x0063, 0x0302, 0x0043, 0x0307, + 0x0063, 0x0307, 0x0043, 0x030c, 0x0063, 0x030c, 0x0044, 0x030c, + 0x0064, 0x030c, 0x0045, 0x0304, 0x0065, 0x0304, 0x0045, 0x0306, + 0x0065, 0x0306, 0x0045, 0x0307, 0x0065, 0x0307, 0x0045, 0x0328, + 0x0065, 0x0328, 0x0045, 0x030c, 0x0065, 0x030c, 0x0047, 0x0302, + 0x0067, 0x0302, 0x0047, 0x0306, 0x0067, 0x0306, 0x0047, 0x0307, + 0x0067, 0x0307, 0x0047, 0x0327, 0x0067, 0x0327, 0x0048, 0x0302, + 0x0068, 0x0302, 0x0049, 0x0303, 0x0069, 0x0303, 0x0049, 0x0304, + 0x0069, 0x0304, 0x0049, 0x0306, 0x0069, 0x0306, 0x0049, 0x0328, + 0x0069, 0x0328, 0x0049, 0x0307, 0x004a, 0x0302, 0x006a, 0x0302, + 0x004b, 0x0327, 0x006b, 0x0327, 0x004c, 0x0301, 0x006c, 0x0301, + 0x004c, 0x0327, 0x006c, 0x0327, 0x004c, 0x030c, 0x006c, 0x030c, + 0x004e, 0x0301, 0x006e, 0x0301, 0x004e, 0x0327, 0x006e, 0x0327, + 0x004e, 0x030c, 0x006e, 0x030c, 0x004f, 0x0304, 0x006f, 0x0304, + 0x004f, 0x0306, 0x006f, 0x0306, 0x004f, 0x030b, 0x006f, 0x030b, + 0x0052, 0x0301, 0x0072, 0x0301, 0x0052, 0x0327, 0x0072, 0x0327, + 0x0052, 0x030c, 0x0072, 0x030c, 0x0053, 0x0301, 0x0073, 0x0301, + 0x0053, 0x0302, 0x0073, 0x0302, 0x0053, 0x0327, 0x0073, 0x0327, + 0x0053, 0x030c, 0x0073, 0x030c, 0x0054, 0x0327, 0x0074, 0x0327, + 0x0054, 0x030c, 0x0074, 0x030c, 0x0055, 0x0303, 0x0075, 0x0303, + 0x0055, 0x0304, 0x0075, 0x0304, 0x0055, 0x0306, 0x0075, 0x0306, + 0x0055, 0x030a, 0x0075, 0x030a, 0x0055, 0x030b, 0x0075, 0x030b, + 0x0055, 0x0328, 0x0075, 0x0328, 0x0057, 0x0302, 0x0077, 0x0302, + 0x0059, 0x0302, 0x0079, 0x0302, 0x0059, 0x0308, 0x005a, 0x0301, + 0x007a, 0x0301, 0x005a, 0x0307, 0x007a, 0x0307, 0x005a, 0x030c, + 0x007a, 0x030c, 0x004f, 0x031b, 0x006f, 0x031b, 0x0055, 0x031b, + 0x0075, 0x031b, 0x0041, 0x030c, 0x0061, 0x030c, 0x0049, 0x030c, + 0x0069, 0x030c, 0x004f, 0x030c, 0x006f, 0x030c, 0x0055, 0x030c, + 0x0075, 0x030c, 0x0055, 0x0308, 0x0304, 0x0075, 0x0308, 0x0304, + 0x0055, 0x0308, 0x0301, 0x0075, 0x0308, 0x0301, 0x0055, 0x0308, + 0x030c, 0x0075, 0x0308, 0x030c, 0x0055, 0x0308, 0x0300, 0x0075, + 0x0308, 0x0300, 0x0041, 0x0308, 0x0304, 0x0061, 0x0308, 0x0304, + 0x0041, 0x0307, 0x0304, 0x0061, 0x0307, 0x0304, 0x00c6, 0x0304, + 0x00e6, 0x0304, 0x0047, 0x030c, 0x0067, 0x030c, 0x004b, 0x030c, + 0x006b, 0x030c, 0x004f, 0x0328, 0x006f, 0x0328, 0x004f, 0x0328, + 0x0304, 0x006f, 0x0328, 0x0304, 0x01b7, 0x030c, 0x0292, 0x030c, + 0x006a, 0x030c, 0x0047, 0x0301, 0x0067, 0x0301, 0x0041, 0x030a, + 0x0301, 0x0061, 0x030a, 0x0301, 0x00c6, 0x0301, 0x00e6, 0x0301, + 0x00d8, 0x0301, 0x00f8, 0x0301, 0x0041, 0x030f, 0x0061, 0x030f, + 0x0041, 0x0311, 0x0061, 0x0311, 0x0045, 0x030f, 0x0065, 0x030f, + 0x0045, 0x0311, 0x0065, 0x0311, 0x0049, 0x030f, 0x0069, 0x030f, + 0x0049, 0x0311, 0x0069, 0x0311, 0x004f, 0x030f, 0x006f, 0x030f, + 0x004f, 0x0311, 0x006f, 0x0311, 0x0052, 0x030f, 0x0072, 0x030f, + 0x0052, 0x0311, 0x0072, 0x0311, 0x0055, 0x030f, 0x0075, 0x030f, + 0x0055, 0x0311, 0x0075, 0x0311, 0x0306, 0x0307, 0x0300, 0x0301, + 0x0313, 0x0308, 0x030d, 0x02b9, 0x003b, 0x00a8, 0x030d, 0x0391, + 0x030d, 0x00b7, 0x0395, 0x030d, 0x0397, 0x030d, 0x0399, 0x030d, + 0x039f, 0x030d, 0x03a5, 0x030d, 0x03a9, 0x030d, 0x03b9, 0x0308, + 0x030d, 0x0399, 0x0308, 0x03a5, 0x0308, 0x03b1, 0x030d, 0x03b5, + 0x030d, 0x03b7, 0x030d, 0x03b9, 0x030d, 0x03c5, 0x0308, 0x030d, + 0x03b9, 0x0308, 0x03c5, 0x0308, 0x03bf, 0x030d, 0x03c5, 0x030d, + 0x03c9, 0x030d, 0x03d2, 0x030d, 0x03d2, 0x0308, 0x0415, 0x0308, + 0x0413, 0x0301, 0x0406, 0x0308, 0x041a, 0x0301, 0x0423, 0x0306, + 0x0418, 0x0306, 0x0438, 0x0306, 0x0435, 0x0308, 0x0433, 0x0301, + 0x0456, 0x0308, 0x043a, 0x0301, 0x0443, 0x0306, 0x0474, 0x030f, + 0x0475, 0x030f, 0x0416, 0x0306, 0x0436, 0x0306, 0x0410, 0x0306, + 0x0430, 0x0306, 0x0410, 0x0308, 0x0430, 0x0308, 0x00c6, 0x00e6, + 0x0415, 0x0306, 0x0435, 0x0306, 0x018f, 0x0259, 0x018f, 0x0308, + 0x0259, 0x0308, 0x0416, 0x0308, 0x0436, 0x0308, 0x0417, 0x0308, + 0x0437, 0x0308, 0x01b7, 0x0292, 0x0418, 0x0304, 0x0438, 0x0304, + 0x0418, 0x0308, 0x0438, 0x0308, 0x041e, 0x0308, 0x043e, 0x0308, + 0x019f, 0x0275, 0x019f, 0x0308, 0x0275, 0x0308, 0x0423, 0x0304, + 0x0443, 0x0304, 0x0423, 0x0308, 0x0443, 0x0308, 0x0423, 0x030b, + 0x0443, 0x030b, 0x0427, 0x0308, 0x0447, 0x0308, 0x042b, 0x0308, + 0x044b, 0x0308, 0x0928, 0x093c, 0x0930, 0x093c, 0x0933, 0x093c, + 0x0915, 0x093c, 0x0916, 0x093c, 0x0917, 0x093c, 0x091c, 0x093c, + 0x0921, 0x093c, 0x0922, 0x093c, 0x092b, 0x093c, 0x092f, 0x093c, + 0x09ac, 0x09bc, 0x09c7, 0x09be, 0x09c7, 0x09d7, 0x09a1, 0x09bc, + 0x09a2, 0x09bc, 0x09af, 0x09bc, 0x0a16, 0x0a3c, 0x0a17, 0x0a3c, + 0x0a1c, 0x0a3c, 0x0a21, 0x0a3c, 0x0a2b, 0x0a3c, 0x0b47, 0x0b56, + 0x0b47, 0x0b3e, 0x0b47, 0x0b57, 0x0b21, 0x0b3c, 0x0b22, 0x0b3c, + 0x0b2f, 0x0b3c, 0x0b92, 0x0bd7, 0x0bc6, 0x0bbe, 0x0bc7, 0x0bbe, + 0x0bc6, 0x0bd7, 0x0c46, 0x0c56, 0x0cbf, 0x0cd5, 0x0cc6, 0x0cd5, + 0x0cc6, 0x0cd6, 0x0cc6, 0x0cc2, 0x0cc6, 0x0cc2, 0x0cd5, 0x0d46, + 0x0d3e, 0x0d47, 0x0d3e, 0x0d46, 0x0d57, 0x0e4d, 0x0e32, 0x0ecd, + 0x0eb2, 0x0f42, 0x0fb7, 0x0f4c, 0x0fb7, 0x0f51, 0x0fb7, 0x0f56, + 0x0fb7, 0x0f5b, 0x0fb7, 0x0f40, 0x0fb5, 0x0f72, 0x0f71, 0x0f74, + 0x0f71, 0x0fb2, 0x0f80, 0x0fb2, 0x0f80, 0x0f71, 0x0fb3, 0x0f80, + 0x0fb3, 0x0f80, 0x0f71, 0x0f80, 0x0f71, 0x0f92, 0x0fb7, 0x0f9c, + 0x0fb7, 0x0fa1, 0x0fb7, 0x0fa6, 0x0fb7, 0x0fab, 0x0fb7, 0x0f90, + 0x0fb5, 0x0041, 0x0325, 0x0061, 0x0325, 0x0042, 0x0307, 0x0062, + 0x0307, 0x0042, 0x0323, 0x0062, 0x0323, 0x0042, 0x0331, 0x0062, + 0x0331, 0x0043, 0x0327, 0x0301, 0x0063, 0x0327, 0x0301, 0x0044, + 0x0307, 0x0064, 0x0307, 0x0044, 0x0323, 0x0064, 0x0323, 0x0044, + 0x0331, 0x0064, 0x0331, 0x0044, 0x0327, 0x0064, 0x0327, 0x0044, + 0x032d, 0x0064, 0x032d, 0x0045, 0x0304, 0x0300, 0x0065, 0x0304, + 0x0300, 0x0045, 0x0304, 0x0301, 0x0065, 0x0304, 0x0301, 0x0045, + 0x032d, 0x0065, 0x032d, 0x0045, 0x0330, 0x0065, 0x0330, 0x0045, + 0x0327, 0x0306, 0x0065, 0x0327, 0x0306, 0x0046, 0x0307, 0x0066, + 0x0307, 0x0047, 0x0304, 0x0067, 0x0304, 0x0048, 0x0307, 0x0068, + 0x0307, 0x0048, 0x0323, 0x0068, 0x0323, 0x0048, 0x0308, 0x0068, + 0x0308, 0x0048, 0x0327, 0x0068, 0x0327, 0x0048, 0x032e, 0x0068, + 0x032e, 0x0049, 0x0330, 0x0069, 0x0330, 0x0049, 0x0308, 0x0301, + 0x0069, 0x0308, 0x0301, 0x004b, 0x0301, 0x006b, 0x0301, 0x004b, + 0x0323, 0x006b, 0x0323, 0x004b, 0x0331, 0x006b, 0x0331, 0x004c, + 0x0323, 0x006c, 0x0323, 0x004c, 0x0323, 0x0304, 0x006c, 0x0323, + 0x0304, 0x004c, 0x0331, 0x006c, 0x0331, 0x004c, 0x032d, 0x006c, + 0x032d, 0x004d, 0x0301, 0x006d, 0x0301, 0x004d, 0x0307, 0x006d, + 0x0307, 0x004d, 0x0323, 0x006d, 0x0323, 0x004e, 0x0307, 0x006e, + 0x0307, 0x004e, 0x0323, 0x006e, 0x0323, 0x004e, 0x0331, 0x006e, + 0x0331, 0x004e, 0x032d, 0x006e, 0x032d, 0x004f, 0x0303, 0x0301, + 0x006f, 0x0303, 0x0301, 0x004f, 0x0303, 0x0308, 0x006f, 0x0303, + 0x0308, 0x004f, 0x0304, 0x0300, 0x006f, 0x0304, 0x0300, 0x004f, + 0x0304, 0x0301, 0x006f, 0x0304, 0x0301, 0x0050, 0x0301, 0x0070, + 0x0301, 0x0050, 0x0307, 0x0070, 0x0307, 0x0052, 0x0307, 0x0072, + 0x0307, 0x0052, 0x0323, 0x0072, 0x0323, 0x0052, 0x0323, 0x0304, + 0x0072, 0x0323, 0x0304, 0x0052, 0x0331, 0x0072, 0x0331, 0x0053, + 0x0307, 0x0073, 0x0307, 0x0053, 0x0323, 0x0073, 0x0323, 0x0053, + 0x0301, 0x0307, 0x0073, 0x0301, 0x0307, 0x0053, 0x030c, 0x0307, + 0x0073, 0x030c, 0x0307, 0x0053, 0x0323, 0x0307, 0x0073, 0x0323, + 0x0307, 0x0054, 0x0307, 0x0074, 0x0307, 0x0054, 0x0323, 0x0074, + 0x0323, 0x0054, 0x0331, 0x0074, 0x0331, 0x0054, 0x032d, 0x0074, + 0x032d, 0x0055, 0x0324, 0x0075, 0x0324, 0x0055, 0x0330, 0x0075, + 0x0330, 0x0055, 0x032d, 0x0075, 0x032d, 0x0055, 0x0303, 0x0301, + 0x0075, 0x0303, 0x0301, 0x0055, 0x0304, 0x0308, 0x0075, 0x0304, + 0x0308, 0x0056, 0x0303, 0x0076, 0x0303, 0x0056, 0x0323, 0x0076, + 0x0323, 0x0057, 0x0300, 0x0077, 0x0300, 0x0057, 0x0301, 0x0077, + 0x0301, 0x0057, 0x0308, 0x0077, 0x0308, 0x0057, 0x0307, 0x0077, + 0x0307, 0x0057, 0x0323, 0x0077, 0x0323, 0x0058, 0x0307, 0x0078, + 0x0307, 0x0058, 0x0308, 0x0078, 0x0308, 0x0059, 0x0307, 0x0079, + 0x0307, 0x005a, 0x0302, 0x007a, 0x0302, 0x005a, 0x0323, 0x007a, + 0x0323, 0x005a, 0x0331, 0x007a, 0x0331, 0x0068, 0x0331, 0x0074, + 0x0308, 0x0077, 0x030a, 0x0079, 0x030a, 0x017f, 0x0307, 0x0041, + 0x0323, 0x0061, 0x0323, 0x0041, 0x0309, 0x0061, 0x0309, 0x0041, + 0x0302, 0x0301, 0x0061, 0x0302, 0x0301, 0x0041, 0x0302, 0x0300, + 0x0061, 0x0302, 0x0300, 0x0041, 0x0302, 0x0309, 0x0061, 0x0302, + 0x0309, 0x0041, 0x0302, 0x0303, 0x0061, 0x0302, 0x0303, 0x0041, + 0x0323, 0x0302, 0x0061, 0x0323, 0x0302, 0x0041, 0x0306, 0x0301, + 0x0061, 0x0306, 0x0301, 0x0041, 0x0306, 0x0300, 0x0061, 0x0306, + 0x0300, 0x0041, 0x0306, 0x0309, 0x0061, 0x0306, 0x0309, 0x0041, + 0x0306, 0x0303, 0x0061, 0x0306, 0x0303, 0x0041, 0x0323, 0x0306, + 0x0061, 0x0323, 0x0306, 0x0045, 0x0323, 0x0065, 0x0323, 0x0045, + 0x0309, 0x0065, 0x0309, 0x0045, 0x0303, 0x0065, 0x0303, 0x0045, + 0x0302, 0x0301, 0x0065, 0x0302, 0x0301, 0x0045, 0x0302, 0x0300, + 0x0065, 0x0302, 0x0300, 0x0045, 0x0302, 0x0309, 0x0065, 0x0302, + 0x0309, 0x0045, 0x0302, 0x0303, 0x0065, 0x0302, 0x0303, 0x0045, + 0x0323, 0x0302, 0x0065, 0x0323, 0x0302, 0x0049, 0x0309, 0x0069, + 0x0309, 0x0049, 0x0323, 0x0069, 0x0323, 0x004f, 0x0323, 0x006f, + 0x0323, 0x004f, 0x0309, 0x006f, 0x0309, 0x004f, 0x0302, 0x0301, + 0x006f, 0x0302, 0x0301, 0x004f, 0x0302, 0x0300, 0x006f, 0x0302, + 0x0300, 0x004f, 0x0302, 0x0309, 0x006f, 0x0302, 0x0309, 0x004f, + 0x0302, 0x0303, 0x006f, 0x0302, 0x0303, 0x004f, 0x0323, 0x0302, + 0x006f, 0x0323, 0x0302, 0x004f, 0x031b, 0x0301, 0x006f, 0x031b, + 0x0301, 0x004f, 0x031b, 0x0300, 0x006f, 0x031b, 0x0300, 0x004f, + 0x031b, 0x0309, 0x006f, 0x031b, 0x0309, 0x004f, 0x031b, 0x0303, + 0x006f, 0x031b, 0x0303, 0x004f, 0x031b, 0x0323, 0x006f, 0x031b, + 0x0323, 0x0055, 0x0323, 0x0075, 0x0323, 0x0055, 0x0309, 0x0075, + 0x0309, 0x0055, 0x031b, 0x0301, 0x0075, 0x031b, 0x0301, 0x0055, + 0x031b, 0x0300, 0x0075, 0x031b, 0x0300, 0x0055, 0x031b, 0x0309, + 0x0075, 0x031b, 0x0309, 0x0055, 0x031b, 0x0303, 0x0075, 0x031b, + 0x0303, 0x0055, 0x031b, 0x0323, 0x0075, 0x031b, 0x0323, 0x0059, + 0x0300, 0x0079, 0x0300, 0x0059, 0x0323, 0x0079, 0x0323, 0x0059, + 0x0309, 0x0079, 0x0309, 0x0059, 0x0303, 0x0079, 0x0303, 0x03b1, + 0x0313, 0x03b1, 0x0314, 0x03b1, 0x0313, 0x0300, 0x03b1, 0x0314, + 0x0300, 0x03b1, 0x0313, 0x0301, 0x03b1, 0x0314, 0x0301, 0x03b1, + 0x0313, 0x0342, 0x03b1, 0x0314, 0x0342, 0x0391, 0x0313, 0x0391, + 0x0314, 0x0391, 0x0313, 0x0300, 0x0391, 0x0314, 0x0300, 0x0391, + 0x0313, 0x0301, 0x0391, 0x0314, 0x0301, 0x0391, 0x0313, 0x0342, + 0x0391, 0x0314, 0x0342, 0x03b5, 0x0313, 0x03b5, 0x0314, 0x03b5, + 0x0313, 0x0300, 0x03b5, 0x0314, 0x0300, 0x03b5, 0x0313, 0x0301, + 0x03b5, 0x0314, 0x0301, 0x0395, 0x0313, 0x0395, 0x0314, 0x0395, + 0x0313, 0x0300, 0x0395, 0x0314, 0x0300, 0x0395, 0x0313, 0x0301, + 0x0395, 0x0314, 0x0301, 0x03b7, 0x0313, 0x03b7, 0x0314, 0x03b7, + 0x0313, 0x0300, 0x03b7, 0x0314, 0x0300, 0x03b7, 0x0313, 0x0301, + 0x03b7, 0x0314, 0x0301, 0x03b7, 0x0313, 0x0342, 0x03b7, 0x0314, + 0x0342, 0x0397, 0x0313, 0x0397, 0x0314, 0x0397, 0x0313, 0x0300, + 0x0397, 0x0314, 0x0300, 0x0397, 0x0313, 0x0301, 0x0397, 0x0314, + 0x0301, 0x0397, 0x0313, 0x0342, 0x0397, 0x0314, 0x0342, 0x03b9, + 0x0313, 0x03b9, 0x0314, 0x03b9, 0x0313, 0x0300, 0x03b9, 0x0314, + 0x0300, 0x03b9, 0x0313, 0x0301, 0x03b9, 0x0314, 0x0301, 0x03b9, + 0x0313, 0x0342, 0x03b9, 0x0314, 0x0342, 0x0399, 0x0313, 0x0399, + 0x0314, 0x0399, 0x0313, 0x0300, 0x0399, 0x0314, 0x0300, 0x0399, + 0x0313, 0x0301, 0x0399, 0x0314, 0x0301, 0x0399, 0x0313, 0x0342, + 0x0399, 0x0314, 0x0342, 0x03bf, 0x0313, 0x03bf, 0x0314, 0x03bf, + 0x0313, 0x0300, 0x03bf, 0x0314, 0x0300, 0x03bf, 0x0313, 0x0301, + 0x03bf, 0x0314, 0x0301, 0x039f, 0x0313, 0x039f, 0x0314, 0x039f, + 0x0313, 0x0300, 0x039f, 0x0314, 0x0300, 0x039f, 0x0313, 0x0301, + 0x039f, 0x0314, 0x0301, 0x03c5, 0x0313, 0x03c5, 0x0314, 0x03c5, + 0x0313, 0x0300, 0x03c5, 0x0314, 0x0300, 0x03c5, 0x0313, 0x0301, + 0x03c5, 0x0314, 0x0301, 0x03c5, 0x0313, 0x0342, 0x03c5, 0x0314, + 0x0342, 0x03a5, 0x0314, 0x03a5, 0x0314, 0x0300, 0x03a5, 0x0314, + 0x0301, 0x03a5, 0x0314, 0x0342, 0x03c9, 0x0313, 0x03c9, 0x0314, + 0x03c9, 0x0313, 0x0300, 0x03c9, 0x0314, 0x0300, 0x03c9, 0x0313, + 0x0301, 0x03c9, 0x0314, 0x0301, 0x03c9, 0x0313, 0x0342, 0x03c9, + 0x0314, 0x0342, 0x03a9, 0x0313, 0x03a9, 0x0314, 0x03a9, 0x0313, + 0x0300, 0x03a9, 0x0314, 0x0300, 0x03a9, 0x0313, 0x0301, 0x03a9, + 0x0314, 0x0301, 0x03a9, 0x0313, 0x0342, 0x03a9, 0x0314, 0x0342, + 0x03b1, 0x0300, 0x03b1, 0x0301, 0x03b5, 0x0300, 0x03b5, 0x0301, + 0x03b7, 0x0300, 0x03b7, 0x0301, 0x03b9, 0x0300, 0x03b9, 0x0301, + 0x03bf, 0x0300, 0x03bf, 0x0301, 0x03c5, 0x0300, 0x03c5, 0x0301, + 0x03c9, 0x0300, 0x03c9, 0x0301, 0x03b1, 0x0345, 0x0313, 0x03b1, + 0x0345, 0x0314, 0x03b1, 0x0345, 0x0313, 0x0300, 0x03b1, 0x0345, + 0x0314, 0x0300, 0x03b1, 0x0345, 0x0313, 0x0301, 0x03b1, 0x0345, + 0x0314, 0x0301, 0x03b1, 0x0345, 0x0313, 0x0342, 0x03b1, 0x0345, + 0x0314, 0x0342, 0x0391, 0x0345, 0x0313, 0x0391, 0x0345, 0x0314, + 0x0391, 0x0345, 0x0313, 0x0300, 0x0391, 0x0345, 0x0314, 0x0300, + 0x0391, 0x0345, 0x0313, 0x0301, 0x0391, 0x0345, 0x0314, 0x0301, + 0x0391, 0x0345, 0x0313, 0x0342, 0x0391, 0x0345, 0x0314, 0x0342, + 0x03b7, 0x0345, 0x0313, 0x03b7, 0x0345, 0x0314, 0x03b7, 0x0345, + 0x0313, 0x0300, 0x03b7, 0x0345, 0x0314, 0x0300, 0x03b7, 0x0345, + 0x0313, 0x0301, 0x03b7, 0x0345, 0x0314, 0x0301, 0x03b7, 0x0345, + 0x0313, 0x0342, 0x03b7, 0x0345, 0x0314, 0x0342, 0x0397, 0x0345, + 0x0313, 0x0397, 0x0345, 0x0314, 0x0397, 0x0345, 0x0313, 0x0300, + 0x0397, 0x0345, 0x0314, 0x0300, 0x0397, 0x0345, 0x0313, 0x0301, + 0x0397, 0x0345, 0x0314, 0x0301, 0x0397, 0x0345, 0x0313, 0x0342, + 0x0397, 0x0345, 0x0314, 0x0342, 0x03c9, 0x0345, 0x0313, 0x03c9, + 0x0345, 0x0314, 0x03c9, 0x0345, 0x0313, 0x0300, 0x03c9, 0x0345, + 0x0314, 0x0300, 0x03c9, 0x0345, 0x0313, 0x0301, 0x03c9, 0x0345, + 0x0314, 0x0301, 0x03c9, 0x0345, 0x0313, 0x0342, 0x03c9, 0x0345, + 0x0314, 0x0342, 0x03a9, 0x0345, 0x0313, 0x03a9, 0x0345, 0x0314, + 0x03a9, 0x0345, 0x0313, 0x0300, 0x03a9, 0x0345, 0x0314, 0x0300, + 0x03a9, 0x0345, 0x0313, 0x0301, 0x03a9, 0x0345, 0x0314, 0x0301, + 0x03a9, 0x0345, 0x0313, 0x0342, 0x03a9, 0x0345, 0x0314, 0x0342, + 0x03b1, 0x0306, 0x03b1, 0x0304, 0x03b1, 0x0345, 0x0300, 0x03b1, + 0x0345, 0x03b1, 0x0345, 0x0301, 0x03b1, 0x0342, 0x03b1, 0x0345, + 0x0342, 0x0391, 0x0306, 0x0391, 0x0304, 0x0391, 0x0300, 0x0391, + 0x0301, 0x0391, 0x0345, 0x03b9, 0x00a8, 0x0342, 0x03b7, 0x0345, + 0x0300, 0x03b7, 0x0345, 0x03b7, 0x0345, 0x0301, 0x03b7, 0x0342, + 0x03b7, 0x0345, 0x0342, 0x0395, 0x0300, 0x0395, 0x0301, 0x0397, + 0x0300, 0x0397, 0x0301, 0x0397, 0x0345, 0x1fbf, 0x0300, 0x1fbf, + 0x0301, 0x1fbf, 0x0342, 0x03b9, 0x0306, 0x03b9, 0x0304, 0x03b9, + 0x0308, 0x0300, 0x03b9, 0x0308, 0x0301, 0x03b9, 0x0342, 0x03b9, + 0x0308, 0x0342, 0x0399, 0x0306, 0x0399, 0x0304, 0x0399, 0x0300, + 0x0399, 0x0301, 0x1ffe, 0x0300, 0x1ffe, 0x0301, 0x1ffe, 0x0342, + 0x03c5, 0x0306, 0x03c5, 0x0304, 0x03c5, 0x0308, 0x0300, 0x03c5, + 0x0308, 0x0301, 0x03c1, 0x0313, 0x03c1, 0x0314, 0x03c5, 0x0342, + 0x03c5, 0x0308, 0x0342, 0x03a5, 0x0306, 0x03a5, 0x0304, 0x03a5, + 0x0300, 0x03a5, 0x0301, 0x03a1, 0x0314, 0x00a8, 0x0300, 0x00a8, + 0x0301, 0x0060, 0x03c9, 0x0345, 0x0300, 0x03c9, 0x0345, 0x03bf, + 0x0345, 0x0301, 0x03c9, 0x0342, 0x03c9, 0x0345, 0x0342, 0x039f, + 0x0300, 0x039f, 0x0301, 0x03a9, 0x0300, 0x03a9, 0x0301, 0x03a9, + 0x0345, 0x00b4, 0x304b, 0x3099, 0x304d, 0x3099, 0x304f, 0x3099, + 0x3051, 0x3099, 0x3053, 0x3099, 0x3055, 0x3099, 0x3057, 0x3099, + 0x3059, 0x3099, 0x305b, 0x3099, 0x305d, 0x3099, 0x305f, 0x3099, + 0x3061, 0x3099, 0x3064, 0x3099, 0x3066, 0x3099, 0x3068, 0x3099, + 0x306f, 0x3099, 0x306f, 0x309a, 0x3072, 0x3099, 0x3072, 0x309a, + 0x3075, 0x3099, 0x3075, 0x309a, 0x3078, 0x3099, 0x3078, 0x309a, + 0x307b, 0x3099, 0x307b, 0x309a, 0x3046, 0x3099, 0x309d, 0x3099, + 0x30ab, 0x3099, 0x30ad, 0x3099, 0x30af, 0x3099, 0x30b1, 0x3099, + 0x30b3, 0x3099, 0x30b5, 0x3099, 0x30b7, 0x3099, 0x30b9, 0x3099, + 0x30bb, 0x3099, 0x30bd, 0x3099, 0x30bf, 0x3099, 0x30c1, 0x3099, + 0x30c4, 0x3099, 0x30c6, 0x3099, 0x30c8, 0x3099, 0x30cf, 0x3099, + 0x30cf, 0x309a, 0x30d2, 0x3099, 0x30d2, 0x309a, 0x30d5, 0x3099, + 0x30d5, 0x309a, 0x30d8, 0x3099, 0x30d8, 0x309a, 0x30db, 0x3099, + 0x30db, 0x309a, 0x30a6, 0x3099, 0x30ef, 0x3099, 0x30f0, 0x3099, + 0x30f1, 0x3099, 0x30f2, 0x3099, 0x30fd, 0x3099, 0x05f2, 0x05b7, + 0x05e9, 0x05c1, 0x05e9, 0x05c2, 0x05e9, 0x05bc, 0x05c1, 0x05e9, + 0x05bc, 0x05c2, 0x05d0, 0x05b7, 0x05d0, 0x05b8, 0x05d0, 0x05bc, + 0x05d1, 0x05bc, 0x05d2, 0x05bc, 0x05d3, 0x05bc, 0x05d4, 0x05bc, + 0x05d5, 0x05bc, 0x05d6, 0x05bc, 0x05d8, 0x05bc, 0x05d9, 0x05bc, + 0x05da, 0x05bc, 0x05db, 0x05bc, 0x05dc, 0x05bc, 0x05de, 0x05bc, + 0x05e0, 0x05bc, 0x05e1, 0x05bc, 0x05e3, 0x05bc, 0x05e4, 0x05bc, + 0x05e6, 0x05bc, 0x05e7, 0x05bc, 0x05e8, 0x05bc, 0x05e9, 0x05bc, + 0x05ea, 0x05bc, 0x05d5, 0x05b9, 0x05d1, 0x05bf, 0x05db, 0x05bf, + 0x05e4, 0x05bf +}; + +u16 hfsplus_compose_table[] = { + /* base */ + 0x0000, 0x0050, 0x0300, 0x00a4, 0x0301, 0x00e4, 0x0302, 0x015c, + 0x0303, 0x0192, 0x0304, 0x01b4, 0x0306, 0x01e6, 0x0307, 0x0220, + 0x0308, 0x0270, 0x0309, 0x02d2, 0x030a, 0x02ec, 0x030b, 0x02fa, + 0x030c, 0x0308, 0x030d, 0x034c, 0x030f, 0x0370, 0x0311, 0x038e, + 0x0313, 0x03a8, 0x0314, 0x03c6, 0x031b, 0x03e8, 0x0323, 0x03f2, + 0x0324, 0x0440, 0x0325, 0x0446, 0x0327, 0x044c, 0x0328, 0x047a, + 0x032d, 0x0490, 0x032e, 0x04aa, 0x0330, 0x04b0, 0x0331, 0x04be, + 0x0342, 0x04e2, 0x0345, 0x04f4, 0x05b7, 0x0504, 0x05b8, 0x050a, + 0x05b9, 0x050e, 0x05bc, 0x0512, 0x05bf, 0x0540, 0x05c1, 0x0548, + 0x05c2, 0x054c, 0x093c, 0x0550, 0x09bc, 0x0568, 0x09be, 0x0572, + 0x09d7, 0x0576, 0x0a3c, 0x057a, 0x0b3c, 0x0586, 0x0b3e, 0x058e, + 0x0b56, 0x0592, 0x0b57, 0x0596, 0x0bbe, 0x059a, 0x0bd7, 0x05a0, + 0x0c56, 0x05a6, 0x0cc2, 0x05aa, 0x0cd5, 0x05ae, 0x0cd6, 0x05b4, + 0x0d3e, 0x05b8, 0x0d57, 0x05be, 0x0e32, 0x05c2, 0x0eb2, 0x05c6, + 0x0f71, 0x05ca, 0x0f80, 0x05d2, 0x0fb5, 0x05d8, 0x0fb7, 0x05de, + 0x1100, 0x00a2, 0x1101, 0x00a2, 0x1102, 0x00a2, 0x1103, 0x00a2, + 0x1104, 0x00a2, 0x1105, 0x00a2, 0x1106, 0x00a2, 0x1107, 0x00a2, + 0x1108, 0x00a2, 0x1109, 0x00a2, 0x110a, 0x00a2, 0x110b, 0x00a2, + 0x110c, 0x00a2, 0x110d, 0x00a2, 0x110e, 0x00a2, 0x110f, 0x00a2, + 0x1110, 0x00a2, 0x1111, 0x00a2, 0x1112, 0x00a2, 0x3099, 0x05f4, + 0x309a, 0x0656, + /* hangul marker */ + 0xffff, 0x0000, + /* 0x0300 */ + 0x0340, 0x001f, 0x0041, 0x066c, 0x0045, 0x066e, 0x0049, 0x0670, + 0x004f, 0x0672, 0x0055, 0x0674, 0x0057, 0x0676, 0x0059, 0x0678, + 0x0061, 0x067a, 0x0065, 0x067c, 0x0069, 0x067e, 0x006f, 0x0680, + 0x0075, 0x0682, 0x0077, 0x0684, 0x0079, 0x0686, 0x00a8, 0x0688, + 0x0391, 0x068a, 0x0395, 0x068c, 0x0397, 0x068e, 0x0399, 0x0690, + 0x039f, 0x0692, 0x03a5, 0x0694, 0x03a9, 0x0696, 0x03b1, 0x0698, + 0x03b5, 0x069a, 0x03b7, 0x069c, 0x03b9, 0x069e, 0x03bf, 0x06a0, + 0x03c5, 0x06a2, 0x03c9, 0x06a4, 0x1fbf, 0x06a6, 0x1ffe, 0x06a8, + /* 0x0301 */ + 0x0341, 0x003b, 0x0041, 0x06aa, 0x0043, 0x06ac, 0x0045, 0x06ae, + 0x0047, 0x06b0, 0x0049, 0x06b2, 0x004b, 0x06b4, 0x004c, 0x06b6, + 0x004d, 0x06b8, 0x004e, 0x06ba, 0x004f, 0x06bc, 0x0050, 0x06be, + 0x0052, 0x06c0, 0x0053, 0x06c2, 0x0055, 0x06c6, 0x0057, 0x06c8, + 0x0059, 0x06ca, 0x005a, 0x06cc, 0x0061, 0x06ce, 0x0063, 0x06d0, + 0x0065, 0x06d2, 0x0067, 0x06d4, 0x0069, 0x06d6, 0x006b, 0x06d8, + 0x006c, 0x06da, 0x006d, 0x06dc, 0x006e, 0x06de, 0x006f, 0x06e0, + 0x0070, 0x06e2, 0x0072, 0x06e4, 0x0073, 0x06e6, 0x0075, 0x06ea, + 0x0077, 0x06ec, 0x0079, 0x06ee, 0x007a, 0x06f0, 0x00a8, 0x06f2, + 0x00c6, 0x06f4, 0x00d8, 0x06f6, 0x00e6, 0x06f8, 0x00f8, 0x06fa, + 0x0391, 0x06fc, 0x0395, 0x06fe, 0x0397, 0x0700, 0x0399, 0x0702, + 0x039f, 0x0704, 0x03a5, 0x0706, 0x03a9, 0x0708, 0x03b1, 0x070a, + 0x03b5, 0x070c, 0x03b7, 0x070e, 0x03b9, 0x0710, 0x03bf, 0x0712, + 0x03c5, 0x0714, 0x03c9, 0x0716, 0x0413, 0x0718, 0x041a, 0x071a, + 0x0433, 0x071c, 0x043a, 0x071e, 0x1fbf, 0x0720, 0x1ffe, 0x0722, + /* 0x0302 */ + 0x0000, 0x001a, 0x0041, 0x0724, 0x0043, 0x072e, 0x0045, 0x0730, + 0x0047, 0x073a, 0x0048, 0x073c, 0x0049, 0x073e, 0x004a, 0x0740, + 0x004f, 0x0742, 0x0053, 0x074c, 0x0055, 0x074e, 0x0057, 0x0750, + 0x0059, 0x0752, 0x005a, 0x0754, 0x0061, 0x0756, 0x0063, 0x0760, + 0x0065, 0x0762, 0x0067, 0x076c, 0x0068, 0x076e, 0x0069, 0x0770, + 0x006a, 0x0772, 0x006f, 0x0774, 0x0073, 0x077e, 0x0075, 0x0780, + 0x0077, 0x0782, 0x0079, 0x0784, 0x007a, 0x0786, + /* 0x0303 */ + 0x0000, 0x0010, 0x0041, 0x0788, 0x0045, 0x078a, 0x0049, 0x078c, + 0x004e, 0x078e, 0x004f, 0x0790, 0x0055, 0x0796, 0x0056, 0x079a, + 0x0059, 0x079c, 0x0061, 0x079e, 0x0065, 0x07a0, 0x0069, 0x07a2, + 0x006e, 0x07a4, 0x006f, 0x07a6, 0x0075, 0x07ac, 0x0076, 0x07b0, + 0x0079, 0x07b2, + /* 0x0304 */ + 0x0000, 0x0018, 0x0041, 0x07b4, 0x0045, 0x07b6, 0x0047, 0x07bc, + 0x0049, 0x07be, 0x004f, 0x07c0, 0x0055, 0x07c6, 0x0061, 0x07ca, + 0x0065, 0x07cc, 0x0067, 0x07d2, 0x0069, 0x07d4, 0x006f, 0x07d6, + 0x0075, 0x07dc, 0x00c6, 0x07e0, 0x00e6, 0x07e2, 0x0391, 0x07e4, + 0x0399, 0x07e6, 0x03a5, 0x07e8, 0x03b1, 0x07ea, 0x03b9, 0x07ec, + 0x03c5, 0x07ee, 0x0418, 0x07f0, 0x0423, 0x07f2, 0x0438, 0x07f4, + 0x0443, 0x07f6, + /* 0x0306 */ + 0x0000, 0x001c, 0x0041, 0x07f8, 0x0045, 0x0802, 0x0047, 0x0804, + 0x0049, 0x0806, 0x004f, 0x0808, 0x0055, 0x080a, 0x0061, 0x080c, + 0x0065, 0x0816, 0x0067, 0x0818, 0x0069, 0x081a, 0x006f, 0x081c, + 0x0075, 0x081e, 0x0391, 0x0820, 0x0399, 0x0822, 0x03a5, 0x0824, + 0x03b1, 0x0826, 0x03b9, 0x0828, 0x03c5, 0x082a, 0x0410, 0x082c, + 0x0415, 0x082e, 0x0416, 0x0830, 0x0418, 0x0832, 0x0423, 0x0834, + 0x0430, 0x0836, 0x0435, 0x0838, 0x0436, 0x083a, 0x0438, 0x083c, + 0x0443, 0x083e, + /* 0x0307 */ + 0x0000, 0x0027, 0x0041, 0x0840, 0x0042, 0x0844, 0x0043, 0x0846, + 0x0044, 0x0848, 0x0045, 0x084a, 0x0046, 0x084c, 0x0047, 0x084e, + 0x0048, 0x0850, 0x0049, 0x0852, 0x004d, 0x0854, 0x004e, 0x0856, + 0x0050, 0x0858, 0x0052, 0x085a, 0x0053, 0x085c, 0x0054, 0x085e, + 0x0057, 0x0860, 0x0058, 0x0862, 0x0059, 0x0864, 0x005a, 0x0866, + 0x0061, 0x0868, 0x0062, 0x086c, 0x0063, 0x086e, 0x0064, 0x0870, + 0x0065, 0x0872, 0x0066, 0x0874, 0x0067, 0x0876, 0x0068, 0x0878, + 0x006d, 0x087a, 0x006e, 0x087c, 0x0070, 0x087e, 0x0072, 0x0880, + 0x0073, 0x0882, 0x0074, 0x0884, 0x0077, 0x0886, 0x0078, 0x0888, + 0x0079, 0x088a, 0x007a, 0x088c, 0x017f, 0x088e, 0x0306, 0x0890, + /* 0x0308 */ + 0x0000, 0x0030, 0x0041, 0x0892, 0x0045, 0x0896, 0x0048, 0x0898, + 0x0049, 0x089a, 0x004f, 0x089e, 0x0055, 0x08a0, 0x0057, 0x08aa, + 0x0058, 0x08ac, 0x0059, 0x08ae, 0x0061, 0x08b0, 0x0065, 0x08b4, + 0x0068, 0x08b6, 0x0069, 0x08b8, 0x006f, 0x08bc, 0x0074, 0x08be, + 0x0075, 0x08c0, 0x0077, 0x08ca, 0x0078, 0x08cc, 0x0079, 0x08ce, + 0x018f, 0x08d0, 0x019f, 0x08d2, 0x0259, 0x08d4, 0x0275, 0x08d6, + 0x0399, 0x08d8, 0x03a5, 0x08da, 0x03b9, 0x08dc, 0x03c5, 0x08e6, + 0x03d2, 0x08f0, 0x0406, 0x08f2, 0x0410, 0x08f4, 0x0415, 0x08f6, + 0x0416, 0x08f8, 0x0417, 0x08fa, 0x0418, 0x08fc, 0x041e, 0x08fe, + 0x0423, 0x0900, 0x0427, 0x0902, 0x042b, 0x0904, 0x0430, 0x0906, + 0x0435, 0x0908, 0x0436, 0x090a, 0x0437, 0x090c, 0x0438, 0x090e, + 0x043e, 0x0910, 0x0443, 0x0912, 0x0447, 0x0914, 0x044b, 0x0916, + 0x0456, 0x0918, + /* 0x0309 */ + 0x0000, 0x000c, 0x0041, 0x091a, 0x0045, 0x091c, 0x0049, 0x091e, + 0x004f, 0x0920, 0x0055, 0x0922, 0x0059, 0x0924, 0x0061, 0x0926, + 0x0065, 0x0928, 0x0069, 0x092a, 0x006f, 0x092c, 0x0075, 0x092e, + 0x0079, 0x0930, + /* 0x030a */ + 0x0000, 0x0006, 0x0041, 0x0932, 0x0055, 0x0936, 0x0061, 0x0938, + 0x0075, 0x093c, 0x0077, 0x093e, 0x0079, 0x0940, + /* 0x030b */ + 0x0000, 0x0006, 0x004f, 0x0942, 0x0055, 0x0944, 0x006f, 0x0946, + 0x0075, 0x0948, 0x0423, 0x094a, 0x0443, 0x094c, + /* 0x030c */ + 0x0000, 0x0021, 0x0041, 0x094e, 0x0043, 0x0950, 0x0044, 0x0952, + 0x0045, 0x0954, 0x0047, 0x0956, 0x0049, 0x0958, 0x004b, 0x095a, + 0x004c, 0x095c, 0x004e, 0x095e, 0x004f, 0x0960, 0x0052, 0x0962, + 0x0053, 0x0964, 0x0054, 0x0968, 0x0055, 0x096a, 0x005a, 0x096c, + 0x0061, 0x096e, 0x0063, 0x0970, 0x0064, 0x0972, 0x0065, 0x0974, + 0x0067, 0x0976, 0x0069, 0x0978, 0x006a, 0x097a, 0x006b, 0x097c, + 0x006c, 0x097e, 0x006e, 0x0980, 0x006f, 0x0982, 0x0072, 0x0984, + 0x0073, 0x0986, 0x0074, 0x098a, 0x0075, 0x098c, 0x007a, 0x098e, + 0x01b7, 0x0990, 0x0292, 0x0992, + /* 0x030d */ + 0x0000, 0x0011, 0x00a8, 0x0994, 0x0308, 0x0996, 0x0391, 0x0998, + 0x0395, 0x099a, 0x0397, 0x099c, 0x0399, 0x099e, 0x039f, 0x09a0, + 0x03a5, 0x09a2, 0x03a9, 0x09a4, 0x03b1, 0x09a6, 0x03b5, 0x09a8, + 0x03b7, 0x09aa, 0x03b9, 0x09ac, 0x03bf, 0x09ae, 0x03c5, 0x09b0, + 0x03c9, 0x09b2, 0x03d2, 0x09b4, + /* 0x030f */ + 0x0000, 0x000e, 0x0041, 0x09b6, 0x0045, 0x09b8, 0x0049, 0x09ba, + 0x004f, 0x09bc, 0x0052, 0x09be, 0x0055, 0x09c0, 0x0061, 0x09c2, + 0x0065, 0x09c4, 0x0069, 0x09c6, 0x006f, 0x09c8, 0x0072, 0x09ca, + 0x0075, 0x09cc, 0x0474, 0x09ce, 0x0475, 0x09d0, + /* 0x0311 */ + 0x0000, 0x000c, 0x0041, 0x09d2, 0x0045, 0x09d4, 0x0049, 0x09d6, + 0x004f, 0x09d8, 0x0052, 0x09da, 0x0055, 0x09dc, 0x0061, 0x09de, + 0x0065, 0x09e0, 0x0069, 0x09e2, 0x006f, 0x09e4, 0x0072, 0x09e6, + 0x0075, 0x09e8, + /* 0x0313 */ + 0x0343, 0x000e, 0x0391, 0x09ea, 0x0395, 0x09f2, 0x0397, 0x09f8, + 0x0399, 0x0a00, 0x039f, 0x0a08, 0x03a9, 0x0a0e, 0x03b1, 0x0a16, + 0x03b5, 0x0a1e, 0x03b7, 0x0a24, 0x03b9, 0x0a2c, 0x03bf, 0x0a34, + 0x03c1, 0x0a3a, 0x03c5, 0x0a3c, 0x03c9, 0x0a44, + /* 0x0314 */ + 0x0000, 0x0010, 0x0391, 0x0a4c, 0x0395, 0x0a54, 0x0397, 0x0a5a, + 0x0399, 0x0a62, 0x039f, 0x0a6a, 0x03a1, 0x0a70, 0x03a5, 0x0a72, + 0x03a9, 0x0a7a, 0x03b1, 0x0a82, 0x03b5, 0x0a8a, 0x03b7, 0x0a90, + 0x03b9, 0x0a98, 0x03bf, 0x0aa0, 0x03c1, 0x0aa6, 0x03c5, 0x0aa8, + 0x03c9, 0x0ab0, + /* 0x031b */ + 0x0000, 0x0004, 0x004f, 0x0ab8, 0x0055, 0x0ac4, 0x006f, 0x0ad0, + 0x0075, 0x0adc, + /* 0x0323 */ + 0x0000, 0x0026, 0x0041, 0x0ae8, 0x0042, 0x0aee, 0x0044, 0x0af0, + 0x0045, 0x0af2, 0x0048, 0x0af6, 0x0049, 0x0af8, 0x004b, 0x0afa, + 0x004c, 0x0afc, 0x004d, 0x0b00, 0x004e, 0x0b02, 0x004f, 0x0b04, + 0x0052, 0x0b08, 0x0053, 0x0b0c, 0x0054, 0x0b10, 0x0055, 0x0b12, + 0x0056, 0x0b14, 0x0057, 0x0b16, 0x0059, 0x0b18, 0x005a, 0x0b1a, + 0x0061, 0x0b1c, 0x0062, 0x0b22, 0x0064, 0x0b24, 0x0065, 0x0b26, + 0x0068, 0x0b2a, 0x0069, 0x0b2c, 0x006b, 0x0b2e, 0x006c, 0x0b30, + 0x006d, 0x0b34, 0x006e, 0x0b36, 0x006f, 0x0b38, 0x0072, 0x0b3c, + 0x0073, 0x0b40, 0x0074, 0x0b44, 0x0075, 0x0b46, 0x0076, 0x0b48, + 0x0077, 0x0b4a, 0x0079, 0x0b4c, 0x007a, 0x0b4e, + /* 0x0324 */ + 0x0000, 0x0002, 0x0055, 0x0b50, 0x0075, 0x0b52, + /* 0x0325 */ + 0x0000, 0x0002, 0x0041, 0x0b54, 0x0061, 0x0b56, + /* 0x0327 */ + 0x0000, 0x0016, 0x0043, 0x0b58, 0x0044, 0x0b5c, 0x0045, 0x0b5e, + 0x0047, 0x0b62, 0x0048, 0x0b64, 0x004b, 0x0b66, 0x004c, 0x0b68, + 0x004e, 0x0b6a, 0x0052, 0x0b6c, 0x0053, 0x0b6e, 0x0054, 0x0b70, + 0x0063, 0x0b72, 0x0064, 0x0b76, 0x0065, 0x0b78, 0x0067, 0x0b7c, + 0x0068, 0x0b7e, 0x006b, 0x0b80, 0x006c, 0x0b82, 0x006e, 0x0b84, + 0x0072, 0x0b86, 0x0073, 0x0b88, 0x0074, 0x0b8a, + /* 0x0328 */ + 0x0000, 0x000a, 0x0041, 0x0b8c, 0x0045, 0x0b8e, 0x0049, 0x0b90, + 0x004f, 0x0b92, 0x0055, 0x0b96, 0x0061, 0x0b98, 0x0065, 0x0b9a, + 0x0069, 0x0b9c, 0x006f, 0x0b9e, 0x0075, 0x0ba2, + /* 0x032d */ + 0x0000, 0x000c, 0x0044, 0x0ba4, 0x0045, 0x0ba6, 0x004c, 0x0ba8, + 0x004e, 0x0baa, 0x0054, 0x0bac, 0x0055, 0x0bae, 0x0064, 0x0bb0, + 0x0065, 0x0bb2, 0x006c, 0x0bb4, 0x006e, 0x0bb6, 0x0074, 0x0bb8, + 0x0075, 0x0bba, + /* 0x032e */ + 0x0000, 0x0002, 0x0048, 0x0bbc, 0x0068, 0x0bbe, + /* 0x0330 */ + 0x0000, 0x0006, 0x0045, 0x0bc0, 0x0049, 0x0bc2, 0x0055, 0x0bc4, + 0x0065, 0x0bc6, 0x0069, 0x0bc8, 0x0075, 0x0bca, + /* 0x0331 */ + 0x0000, 0x0011, 0x0042, 0x0bcc, 0x0044, 0x0bce, 0x004b, 0x0bd0, + 0x004c, 0x0bd2, 0x004e, 0x0bd4, 0x0052, 0x0bd6, 0x0054, 0x0bd8, + 0x005a, 0x0bda, 0x0062, 0x0bdc, 0x0064, 0x0bde, 0x0068, 0x0be0, + 0x006b, 0x0be2, 0x006c, 0x0be4, 0x006e, 0x0be6, 0x0072, 0x0be8, + 0x0074, 0x0bea, 0x007a, 0x0bec, + /* 0x0342 */ + 0x0000, 0x0008, 0x00a8, 0x0bee, 0x03b1, 0x0bf0, 0x03b7, 0x0bf2, + 0x03b9, 0x0bf4, 0x03c5, 0x0bf6, 0x03c9, 0x0bf8, 0x1fbf, 0x0bfa, + 0x1ffe, 0x0bfc, + /* 0x0345 */ + 0x0000, 0x0007, 0x0391, 0x0bfe, 0x0397, 0x0c04, 0x03a9, 0x0c0a, + 0x03b1, 0x0c10, 0x03b7, 0x0c1c, 0x03bf, 0x0c28, 0x03c9, 0x0c2c, + /* 0x05b7 */ + 0x0000, 0x0002, 0x05d0, 0x0c36, 0x05f2, 0x0c38, + /* 0x05b8 */ + 0x0000, 0x0001, 0x05d0, 0x0c3a, + /* 0x05b9 */ + 0x0000, 0x0001, 0x05d5, 0x0c3c, + /* 0x05bc */ + 0x0000, 0x0016, 0x05d0, 0x0c3e, 0x05d1, 0x0c40, 0x05d2, 0x0c42, + 0x05d3, 0x0c44, 0x05d4, 0x0c46, 0x05d5, 0x0c48, 0x05d6, 0x0c4a, + 0x05d8, 0x0c4c, 0x05d9, 0x0c4e, 0x05da, 0x0c50, 0x05db, 0x0c52, + 0x05dc, 0x0c54, 0x05de, 0x0c56, 0x05e0, 0x0c58, 0x05e1, 0x0c5a, + 0x05e3, 0x0c5c, 0x05e4, 0x0c5e, 0x05e6, 0x0c60, 0x05e7, 0x0c62, + 0x05e8, 0x0c64, 0x05e9, 0x0c66, 0x05ea, 0x0c6c, + /* 0x05bf */ + 0x0000, 0x0003, 0x05d1, 0x0c6e, 0x05db, 0x0c70, 0x05e4, 0x0c72, + /* 0x05c1 */ + 0x0000, 0x0001, 0x05e9, 0x0c74, + /* 0x05c2 */ + 0x0000, 0x0001, 0x05e9, 0x0c76, + /* 0x093c */ + 0x0000, 0x000b, 0x0915, 0x0c78, 0x0916, 0x0c7a, 0x0917, 0x0c7c, + 0x091c, 0x0c7e, 0x0921, 0x0c80, 0x0922, 0x0c82, 0x0928, 0x0c84, + 0x092b, 0x0c86, 0x092f, 0x0c88, 0x0930, 0x0c8a, 0x0933, 0x0c8c, + /* 0x09bc */ + 0x0000, 0x0004, 0x09a1, 0x0c8e, 0x09a2, 0x0c90, 0x09ac, 0x0c92, + 0x09af, 0x0c94, + /* 0x09be */ + 0x0000, 0x0001, 0x09c7, 0x0c96, + /* 0x09d7 */ + 0x0000, 0x0001, 0x09c7, 0x0c98, + /* 0x0a3c */ + 0x0000, 0x0005, 0x0a16, 0x0c9a, 0x0a17, 0x0c9c, 0x0a1c, 0x0c9e, + 0x0a21, 0x0ca0, 0x0a2b, 0x0ca2, + /* 0x0b3c */ + 0x0000, 0x0003, 0x0b21, 0x0ca4, 0x0b22, 0x0ca6, 0x0b2f, 0x0ca8, + /* 0x0b3e */ + 0x0000, 0x0001, 0x0b47, 0x0caa, + /* 0x0b56 */ + 0x0000, 0x0001, 0x0b47, 0x0cac, + /* 0x0b57 */ + 0x0000, 0x0001, 0x0b47, 0x0cae, + /* 0x0bbe */ + 0x0000, 0x0002, 0x0bc6, 0x0cb0, 0x0bc7, 0x0cb2, + /* 0x0bd7 */ + 0x0000, 0x0002, 0x0b92, 0x0cb4, 0x0bc6, 0x0cb6, + /* 0x0c56 */ + 0x0000, 0x0001, 0x0c46, 0x0cb8, + /* 0x0cc2 */ + 0x0000, 0x0001, 0x0cc6, 0x0cba, + /* 0x0cd5 */ + 0x0000, 0x0002, 0x0cbf, 0x0cbe, 0x0cc6, 0x0cc0, + /* 0x0cd6 */ + 0x0000, 0x0001, 0x0cc6, 0x0cc2, + /* 0x0d3e */ + 0x0000, 0x0002, 0x0d46, 0x0cc4, 0x0d47, 0x0cc6, + /* 0x0d57 */ + 0x0000, 0x0001, 0x0d46, 0x0cc8, + /* 0x0e32 */ + 0x0000, 0x0001, 0x0e4d, 0x0cca, + /* 0x0eb2 */ + 0x0000, 0x0001, 0x0ecd, 0x0ccc, + /* 0x0f71 */ + 0x0000, 0x0003, 0x0f72, 0x0cce, 0x0f74, 0x0cd0, 0x0f80, 0x0cd2, + /* 0x0f80 */ + 0x0000, 0x0002, 0x0fb2, 0x0cd4, 0x0fb3, 0x0cd8, + /* 0x0fb5 */ + 0x0000, 0x0002, 0x0f40, 0x0cdc, 0x0f90, 0x0cde, + /* 0x0fb7 */ + 0x0000, 0x000a, 0x0f42, 0x0ce0, 0x0f4c, 0x0ce2, 0x0f51, 0x0ce4, + 0x0f56, 0x0ce6, 0x0f5b, 0x0ce8, 0x0f92, 0x0cea, 0x0f9c, 0x0cec, + 0x0fa1, 0x0cee, 0x0fa6, 0x0cf0, 0x0fab, 0x0cf2, + /* 0x3099 */ + 0x0000, 0x0030, 0x3046, 0x0cf4, 0x304b, 0x0cf6, 0x304d, 0x0cf8, + 0x304f, 0x0cfa, 0x3051, 0x0cfc, 0x3053, 0x0cfe, 0x3055, 0x0d00, + 0x3057, 0x0d02, 0x3059, 0x0d04, 0x305b, 0x0d06, 0x305d, 0x0d08, + 0x305f, 0x0d0a, 0x3061, 0x0d0c, 0x3064, 0x0d0e, 0x3066, 0x0d10, + 0x3068, 0x0d12, 0x306f, 0x0d14, 0x3072, 0x0d16, 0x3075, 0x0d18, + 0x3078, 0x0d1a, 0x307b, 0x0d1c, 0x309d, 0x0d1e, 0x30a6, 0x0d20, + 0x30ab, 0x0d22, 0x30ad, 0x0d24, 0x30af, 0x0d26, 0x30b1, 0x0d28, + 0x30b3, 0x0d2a, 0x30b5, 0x0d2c, 0x30b7, 0x0d2e, 0x30b9, 0x0d30, + 0x30bb, 0x0d32, 0x30bd, 0x0d34, 0x30bf, 0x0d36, 0x30c1, 0x0d38, + 0x30c4, 0x0d3a, 0x30c6, 0x0d3c, 0x30c8, 0x0d3e, 0x30cf, 0x0d40, + 0x30d2, 0x0d42, 0x30d5, 0x0d44, 0x30d8, 0x0d46, 0x30db, 0x0d48, + 0x30ef, 0x0d4a, 0x30f0, 0x0d4c, 0x30f1, 0x0d4e, 0x30f2, 0x0d50, + 0x30fd, 0x0d52, + /* 0x309a */ + 0x0000, 0x000a, 0x306f, 0x0d54, 0x3072, 0x0d56, 0x3075, 0x0d58, + 0x3078, 0x0d5a, 0x307b, 0x0d5c, 0x30cf, 0x0d5e, 0x30d2, 0x0d60, + 0x30d5, 0x0d62, 0x30d8, 0x0d64, 0x30db, 0x0d66, + /* 0x0041 0x0300 */ + 0x00c0, 0x0000, + /* 0x0045 0x0300 */ + 0x00c8, 0x0000, + /* 0x0049 0x0300 */ + 0x00cc, 0x0000, + /* 0x004f 0x0300 */ + 0x00d2, 0x0000, + /* 0x0055 0x0300 */ + 0x00d9, 0x0000, + /* 0x0057 0x0300 */ + 0x1e80, 0x0000, + /* 0x0059 0x0300 */ + 0x1ef2, 0x0000, + /* 0x0061 0x0300 */ + 0x00e0, 0x0000, + /* 0x0065 0x0300 */ + 0x00e8, 0x0000, + /* 0x0069 0x0300 */ + 0x00ec, 0x0000, + /* 0x006f 0x0300 */ + 0x00f2, 0x0000, + /* 0x0075 0x0300 */ + 0x00f9, 0x0000, + /* 0x0077 0x0300 */ + 0x1e81, 0x0000, + /* 0x0079 0x0300 */ + 0x1ef3, 0x0000, + /* 0x00a8 0x0300 */ + 0x1fed, 0x0000, + /* 0x0391 0x0300 */ + 0x1fba, 0x0000, + /* 0x0395 0x0300 */ + 0x1fc8, 0x0000, + /* 0x0397 0x0300 */ + 0x1fca, 0x0000, + /* 0x0399 0x0300 */ + 0x1fda, 0x0000, + /* 0x039f 0x0300 */ + 0x1ff8, 0x0000, + /* 0x03a5 0x0300 */ + 0x1fea, 0x0000, + /* 0x03a9 0x0300 */ + 0x1ffa, 0x0000, + /* 0x03b1 0x0300 */ + 0x1f70, 0x0000, + /* 0x03b5 0x0300 */ + 0x1f72, 0x0000, + /* 0x03b7 0x0300 */ + 0x1f74, 0x0000, + /* 0x03b9 0x0300 */ + 0x1f76, 0x0000, + /* 0x03bf 0x0300 */ + 0x1f78, 0x0000, + /* 0x03c5 0x0300 */ + 0x1f7a, 0x0000, + /* 0x03c9 0x0300 */ + 0x1f7c, 0x0000, + /* 0x1fbf 0x0300 */ + 0x1fcd, 0x0000, + /* 0x1ffe 0x0300 */ + 0x1fdd, 0x0000, + /* 0x0041 0x0301 */ + 0x00c1, 0x0000, + /* 0x0043 0x0301 */ + 0x0106, 0x0000, + /* 0x0045 0x0301 */ + 0x00c9, 0x0000, + /* 0x0047 0x0301 */ + 0x01f4, 0x0000, + /* 0x0049 0x0301 */ + 0x00cd, 0x0000, + /* 0x004b 0x0301 */ + 0x1e30, 0x0000, + /* 0x004c 0x0301 */ + 0x0139, 0x0000, + /* 0x004d 0x0301 */ + 0x1e3e, 0x0000, + /* 0x004e 0x0301 */ + 0x0143, 0x0000, + /* 0x004f 0x0301 */ + 0x00d3, 0x0000, + /* 0x0050 0x0301 */ + 0x1e54, 0x0000, + /* 0x0052 0x0301 */ + 0x0154, 0x0000, + /* 0x0053 0x0301 */ + 0x015a, 0x0001, 0x0307, 0x0d68, + /* 0x0055 0x0301 */ + 0x00da, 0x0000, + /* 0x0057 0x0301 */ + 0x1e82, 0x0000, + /* 0x0059 0x0301 */ + 0x00dd, 0x0000, + /* 0x005a 0x0301 */ + 0x0179, 0x0000, + /* 0x0061 0x0301 */ + 0x00e1, 0x0000, + /* 0x0063 0x0301 */ + 0x0107, 0x0000, + /* 0x0065 0x0301 */ + 0x00e9, 0x0000, + /* 0x0067 0x0301 */ + 0x01f5, 0x0000, + /* 0x0069 0x0301 */ + 0x00ed, 0x0000, + /* 0x006b 0x0301 */ + 0x1e31, 0x0000, + /* 0x006c 0x0301 */ + 0x013a, 0x0000, + /* 0x006d 0x0301 */ + 0x1e3f, 0x0000, + /* 0x006e 0x0301 */ + 0x0144, 0x0000, + /* 0x006f 0x0301 */ + 0x00f3, 0x0000, + /* 0x0070 0x0301 */ + 0x1e55, 0x0000, + /* 0x0072 0x0301 */ + 0x0155, 0x0000, + /* 0x0073 0x0301 */ + 0x015b, 0x0001, 0x0307, 0x0d6a, + /* 0x0075 0x0301 */ + 0x00fa, 0x0000, + /* 0x0077 0x0301 */ + 0x1e83, 0x0000, + /* 0x0079 0x0301 */ + 0x00fd, 0x0000, + /* 0x007a 0x0301 */ + 0x017a, 0x0000, + /* 0x00a8 0x0301 */ + 0x1fee, 0x0000, + /* 0x00c6 0x0301 */ + 0x01fc, 0x0000, + /* 0x00d8 0x0301 */ + 0x01fe, 0x0000, + /* 0x00e6 0x0301 */ + 0x01fd, 0x0000, + /* 0x00f8 0x0301 */ + 0x01ff, 0x0000, + /* 0x0391 0x0301 */ + 0x1fbb, 0x0000, + /* 0x0395 0x0301 */ + 0x1fc9, 0x0000, + /* 0x0397 0x0301 */ + 0x1fcb, 0x0000, + /* 0x0399 0x0301 */ + 0x1fdb, 0x0000, + /* 0x039f 0x0301 */ + 0x1ff9, 0x0000, + /* 0x03a5 0x0301 */ + 0x1feb, 0x0000, + /* 0x03a9 0x0301 */ + 0x1ffb, 0x0000, + /* 0x03b1 0x0301 */ + 0x1f71, 0x0000, + /* 0x03b5 0x0301 */ + 0x1f73, 0x0000, + /* 0x03b7 0x0301 */ + 0x1f75, 0x0000, + /* 0x03b9 0x0301 */ + 0x1f77, 0x0000, + /* 0x03bf 0x0301 */ + 0x1f79, 0x0000, + /* 0x03c5 0x0301 */ + 0x1f7b, 0x0000, + /* 0x03c9 0x0301 */ + 0x1f7d, 0x0000, + /* 0x0413 0x0301 */ + 0x0403, 0x0000, + /* 0x041a 0x0301 */ + 0x040c, 0x0000, + /* 0x0433 0x0301 */ + 0x0453, 0x0000, + /* 0x043a 0x0301 */ + 0x045c, 0x0000, + /* 0x1fbf 0x0301 */ + 0x1fce, 0x0000, + /* 0x1ffe 0x0301 */ + 0x1fde, 0x0000, + /* 0x0041 0x0302 */ + 0x00c2, 0x0004, 0x0300, 0x0d6c, 0x0301, 0x0d6e, 0x0303, 0x0d70, + 0x0309, 0x0d72, + /* 0x0043 0x0302 */ + 0x0108, 0x0000, + /* 0x0045 0x0302 */ + 0x00ca, 0x0004, 0x0300, 0x0d74, 0x0301, 0x0d76, 0x0303, 0x0d78, + 0x0309, 0x0d7a, + /* 0x0047 0x0302 */ + 0x011c, 0x0000, + /* 0x0048 0x0302 */ + 0x0124, 0x0000, + /* 0x0049 0x0302 */ + 0x00ce, 0x0000, + /* 0x004a 0x0302 */ + 0x0134, 0x0000, + /* 0x004f 0x0302 */ + 0x00d4, 0x0004, 0x0300, 0x0d7c, 0x0301, 0x0d7e, 0x0303, 0x0d80, + 0x0309, 0x0d82, + /* 0x0053 0x0302 */ + 0x015c, 0x0000, + /* 0x0055 0x0302 */ + 0x00db, 0x0000, + /* 0x0057 0x0302 */ + 0x0174, 0x0000, + /* 0x0059 0x0302 */ + 0x0176, 0x0000, + /* 0x005a 0x0302 */ + 0x1e90, 0x0000, + /* 0x0061 0x0302 */ + 0x00e2, 0x0004, 0x0300, 0x0d84, 0x0301, 0x0d86, 0x0303, 0x0d88, + 0x0309, 0x0d8a, + /* 0x0063 0x0302 */ + 0x0109, 0x0000, + /* 0x0065 0x0302 */ + 0x00ea, 0x0004, 0x0300, 0x0d8c, 0x0301, 0x0d8e, 0x0303, 0x0d90, + 0x0309, 0x0d92, + /* 0x0067 0x0302 */ + 0x011d, 0x0000, + /* 0x0068 0x0302 */ + 0x0125, 0x0000, + /* 0x0069 0x0302 */ + 0x00ee, 0x0000, + /* 0x006a 0x0302 */ + 0x0135, 0x0000, + /* 0x006f 0x0302 */ + 0x00f4, 0x0004, 0x0300, 0x0d94, 0x0301, 0x0d96, 0x0303, 0x0d98, + 0x0309, 0x0d9a, + /* 0x0073 0x0302 */ + 0x015d, 0x0000, + /* 0x0075 0x0302 */ + 0x00fb, 0x0000, + /* 0x0077 0x0302 */ + 0x0175, 0x0000, + /* 0x0079 0x0302 */ + 0x0177, 0x0000, + /* 0x007a 0x0302 */ + 0x1e91, 0x0000, + /* 0x0041 0x0303 */ + 0x00c3, 0x0000, + /* 0x0045 0x0303 */ + 0x1ebc, 0x0000, + /* 0x0049 0x0303 */ + 0x0128, 0x0000, + /* 0x004e 0x0303 */ + 0x00d1, 0x0000, + /* 0x004f 0x0303 */ + 0x00d5, 0x0002, 0x0301, 0x0d9c, 0x0308, 0x0d9e, + /* 0x0055 0x0303 */ + 0x0168, 0x0001, 0x0301, 0x0da0, + /* 0x0056 0x0303 */ + 0x1e7c, 0x0000, + /* 0x0059 0x0303 */ + 0x1ef8, 0x0000, + /* 0x0061 0x0303 */ + 0x00e3, 0x0000, + /* 0x0065 0x0303 */ + 0x1ebd, 0x0000, + /* 0x0069 0x0303 */ + 0x0129, 0x0000, + /* 0x006e 0x0303 */ + 0x00f1, 0x0000, + /* 0x006f 0x0303 */ + 0x00f5, 0x0002, 0x0301, 0x0da2, 0x0308, 0x0da4, + /* 0x0075 0x0303 */ + 0x0169, 0x0001, 0x0301, 0x0da6, + /* 0x0076 0x0303 */ + 0x1e7d, 0x0000, + /* 0x0079 0x0303 */ + 0x1ef9, 0x0000, + /* 0x0041 0x0304 */ + 0x0100, 0x0000, + /* 0x0045 0x0304 */ + 0x0112, 0x0002, 0x0300, 0x0da8, 0x0301, 0x0daa, + /* 0x0047 0x0304 */ + 0x1e20, 0x0000, + /* 0x0049 0x0304 */ + 0x012a, 0x0000, + /* 0x004f 0x0304 */ + 0x014c, 0x0002, 0x0300, 0x0dac, 0x0301, 0x0dae, + /* 0x0055 0x0304 */ + 0x016a, 0x0001, 0x0308, 0x0db0, + /* 0x0061 0x0304 */ + 0x0101, 0x0000, + /* 0x0065 0x0304 */ + 0x0113, 0x0002, 0x0300, 0x0db2, 0x0301, 0x0db4, + /* 0x0067 0x0304 */ + 0x1e21, 0x0000, + /* 0x0069 0x0304 */ + 0x012b, 0x0000, + /* 0x006f 0x0304 */ + 0x014d, 0x0002, 0x0300, 0x0db6, 0x0301, 0x0db8, + /* 0x0075 0x0304 */ + 0x016b, 0x0001, 0x0308, 0x0dba, + /* 0x00c6 0x0304 */ + 0x01e2, 0x0000, + /* 0x00e6 0x0304 */ + 0x01e3, 0x0000, + /* 0x0391 0x0304 */ + 0x1fb9, 0x0000, + /* 0x0399 0x0304 */ + 0x1fd9, 0x0000, + /* 0x03a5 0x0304 */ + 0x1fe9, 0x0000, + /* 0x03b1 0x0304 */ + 0x1fb1, 0x0000, + /* 0x03b9 0x0304 */ + 0x1fd1, 0x0000, + /* 0x03c5 0x0304 */ + 0x1fe1, 0x0000, + /* 0x0418 0x0304 */ + 0x04e2, 0x0000, + /* 0x0423 0x0304 */ + 0x04ee, 0x0000, + /* 0x0438 0x0304 */ + 0x04e3, 0x0000, + /* 0x0443 0x0304 */ + 0x04ef, 0x0000, + /* 0x0041 0x0306 */ + 0x0102, 0x0004, 0x0300, 0x0dbc, 0x0301, 0x0dbe, 0x0303, 0x0dc0, + 0x0309, 0x0dc2, + /* 0x0045 0x0306 */ + 0x0114, 0x0000, + /* 0x0047 0x0306 */ + 0x011e, 0x0000, + /* 0x0049 0x0306 */ + 0x012c, 0x0000, + /* 0x004f 0x0306 */ + 0x014e, 0x0000, + /* 0x0055 0x0306 */ + 0x016c, 0x0000, + /* 0x0061 0x0306 */ + 0x0103, 0x0004, 0x0300, 0x0dc4, 0x0301, 0x0dc6, 0x0303, 0x0dc8, + 0x0309, 0x0dca, + /* 0x0065 0x0306 */ + 0x0115, 0x0000, + /* 0x0067 0x0306 */ + 0x011f, 0x0000, + /* 0x0069 0x0306 */ + 0x012d, 0x0000, + /* 0x006f 0x0306 */ + 0x014f, 0x0000, + /* 0x0075 0x0306 */ + 0x016d, 0x0000, + /* 0x0391 0x0306 */ + 0x1fb8, 0x0000, + /* 0x0399 0x0306 */ + 0x1fd8, 0x0000, + /* 0x03a5 0x0306 */ + 0x1fe8, 0x0000, + /* 0x03b1 0x0306 */ + 0x1fb0, 0x0000, + /* 0x03b9 0x0306 */ + 0x1fd0, 0x0000, + /* 0x03c5 0x0306 */ + 0x1fe0, 0x0000, + /* 0x0410 0x0306 */ + 0x04d0, 0x0000, + /* 0x0415 0x0306 */ + 0x04d6, 0x0000, + /* 0x0416 0x0306 */ + 0x04c1, 0x0000, + /* 0x0418 0x0306 */ + 0x0419, 0x0000, + /* 0x0423 0x0306 */ + 0x040e, 0x0000, + /* 0x0430 0x0306 */ + 0x04d1, 0x0000, + /* 0x0435 0x0306 */ + 0x04d7, 0x0000, + /* 0x0436 0x0306 */ + 0x04c2, 0x0000, + /* 0x0438 0x0306 */ + 0x0439, 0x0000, + /* 0x0443 0x0306 */ + 0x045e, 0x0000, + /* 0x0041 0x0307 */ + 0x0000, 0x0001, 0x0304, 0x0dcc, + /* 0x0042 0x0307 */ + 0x1e02, 0x0000, + /* 0x0043 0x0307 */ + 0x010a, 0x0000, + /* 0x0044 0x0307 */ + 0x1e0a, 0x0000, + /* 0x0045 0x0307 */ + 0x0116, 0x0000, + /* 0x0046 0x0307 */ + 0x1e1e, 0x0000, + /* 0x0047 0x0307 */ + 0x0120, 0x0000, + /* 0x0048 0x0307 */ + 0x1e22, 0x0000, + /* 0x0049 0x0307 */ + 0x0130, 0x0000, + /* 0x004d 0x0307 */ + 0x1e40, 0x0000, + /* 0x004e 0x0307 */ + 0x1e44, 0x0000, + /* 0x0050 0x0307 */ + 0x1e56, 0x0000, + /* 0x0052 0x0307 */ + 0x1e58, 0x0000, + /* 0x0053 0x0307 */ + 0x1e60, 0x0000, + /* 0x0054 0x0307 */ + 0x1e6a, 0x0000, + /* 0x0057 0x0307 */ + 0x1e86, 0x0000, + /* 0x0058 0x0307 */ + 0x1e8a, 0x0000, + /* 0x0059 0x0307 */ + 0x1e8e, 0x0000, + /* 0x005a 0x0307 */ + 0x017b, 0x0000, + /* 0x0061 0x0307 */ + 0x0000, 0x0001, 0x0304, 0x0dce, + /* 0x0062 0x0307 */ + 0x1e03, 0x0000, + /* 0x0063 0x0307 */ + 0x010b, 0x0000, + /* 0x0064 0x0307 */ + 0x1e0b, 0x0000, + /* 0x0065 0x0307 */ + 0x0117, 0x0000, + /* 0x0066 0x0307 */ + 0x1e1f, 0x0000, + /* 0x0067 0x0307 */ + 0x0121, 0x0000, + /* 0x0068 0x0307 */ + 0x1e23, 0x0000, + /* 0x006d 0x0307 */ + 0x1e41, 0x0000, + /* 0x006e 0x0307 */ + 0x1e45, 0x0000, + /* 0x0070 0x0307 */ + 0x1e57, 0x0000, + /* 0x0072 0x0307 */ + 0x1e59, 0x0000, + /* 0x0073 0x0307 */ + 0x1e61, 0x0000, + /* 0x0074 0x0307 */ + 0x1e6b, 0x0000, + /* 0x0077 0x0307 */ + 0x1e87, 0x0000, + /* 0x0078 0x0307 */ + 0x1e8b, 0x0000, + /* 0x0079 0x0307 */ + 0x1e8f, 0x0000, + /* 0x007a 0x0307 */ + 0x017c, 0x0000, + /* 0x017f 0x0307 */ + 0x1e9b, 0x0000, + /* 0x0306 0x0307 */ + 0x0310, 0x0000, + /* 0x0041 0x0308 */ + 0x00c4, 0x0001, 0x0304, 0x0dd0, + /* 0x0045 0x0308 */ + 0x00cb, 0x0000, + /* 0x0048 0x0308 */ + 0x1e26, 0x0000, + /* 0x0049 0x0308 */ + 0x00cf, 0x0001, 0x0301, 0x0dd2, + /* 0x004f 0x0308 */ + 0x00d6, 0x0000, + /* 0x0055 0x0308 */ + 0x00dc, 0x0004, 0x0300, 0x0dd4, 0x0301, 0x0dd6, 0x0304, 0x0dd8, + 0x030c, 0x0dda, + /* 0x0057 0x0308 */ + 0x1e84, 0x0000, + /* 0x0058 0x0308 */ + 0x1e8c, 0x0000, + /* 0x0059 0x0308 */ + 0x0178, 0x0000, + /* 0x0061 0x0308 */ + 0x00e4, 0x0001, 0x0304, 0x0ddc, + /* 0x0065 0x0308 */ + 0x00eb, 0x0000, + /* 0x0068 0x0308 */ + 0x1e27, 0x0000, + /* 0x0069 0x0308 */ + 0x00ef, 0x0001, 0x0301, 0x0dde, + /* 0x006f 0x0308 */ + 0x00f6, 0x0000, + /* 0x0074 0x0308 */ + 0x1e97, 0x0000, + /* 0x0075 0x0308 */ + 0x00fc, 0x0004, 0x0300, 0x0de0, 0x0301, 0x0de2, 0x0304, 0x0de4, + 0x030c, 0x0de6, + /* 0x0077 0x0308 */ + 0x1e85, 0x0000, + /* 0x0078 0x0308 */ + 0x1e8d, 0x0000, + /* 0x0079 0x0308 */ + 0x00ff, 0x0000, + /* 0x018f 0x0308 */ + 0x04da, 0x0000, + /* 0x019f 0x0308 */ + 0x04ea, 0x0000, + /* 0x0259 0x0308 */ + 0x04db, 0x0000, + /* 0x0275 0x0308 */ + 0x04eb, 0x0000, + /* 0x0399 0x0308 */ + 0x03aa, 0x0000, + /* 0x03a5 0x0308 */ + 0x03ab, 0x0000, + /* 0x03b9 0x0308 */ + 0x03ca, 0x0004, 0x0300, 0x0de8, 0x0301, 0x0dea, 0x030d, 0x0dec, + 0x0342, 0x0dee, + /* 0x03c5 0x0308 */ + 0x03cb, 0x0004, 0x0300, 0x0df0, 0x0301, 0x0df2, 0x030d, 0x0df4, + 0x0342, 0x0df6, + /* 0x03d2 0x0308 */ + 0x03d4, 0x0000, + /* 0x0406 0x0308 */ + 0x0407, 0x0000, + /* 0x0410 0x0308 */ + 0x04d2, 0x0000, + /* 0x0415 0x0308 */ + 0x0401, 0x0000, + /* 0x0416 0x0308 */ + 0x04dc, 0x0000, + /* 0x0417 0x0308 */ + 0x04de, 0x0000, + /* 0x0418 0x0308 */ + 0x04e4, 0x0000, + /* 0x041e 0x0308 */ + 0x04e6, 0x0000, + /* 0x0423 0x0308 */ + 0x04f0, 0x0000, + /* 0x0427 0x0308 */ + 0x04f4, 0x0000, + /* 0x042b 0x0308 */ + 0x04f8, 0x0000, + /* 0x0430 0x0308 */ + 0x04d3, 0x0000, + /* 0x0435 0x0308 */ + 0x0451, 0x0000, + /* 0x0436 0x0308 */ + 0x04dd, 0x0000, + /* 0x0437 0x0308 */ + 0x04df, 0x0000, + /* 0x0438 0x0308 */ + 0x04e5, 0x0000, + /* 0x043e 0x0308 */ + 0x04e7, 0x0000, + /* 0x0443 0x0308 */ + 0x04f1, 0x0000, + /* 0x0447 0x0308 */ + 0x04f5, 0x0000, + /* 0x044b 0x0308 */ + 0x04f9, 0x0000, + /* 0x0456 0x0308 */ + 0x0457, 0x0000, + /* 0x0041 0x0309 */ + 0x1ea2, 0x0000, + /* 0x0045 0x0309 */ + 0x1eba, 0x0000, + /* 0x0049 0x0309 */ + 0x1ec8, 0x0000, + /* 0x004f 0x0309 */ + 0x1ece, 0x0000, + /* 0x0055 0x0309 */ + 0x1ee6, 0x0000, + /* 0x0059 0x0309 */ + 0x1ef6, 0x0000, + /* 0x0061 0x0309 */ + 0x1ea3, 0x0000, + /* 0x0065 0x0309 */ + 0x1ebb, 0x0000, + /* 0x0069 0x0309 */ + 0x1ec9, 0x0000, + /* 0x006f 0x0309 */ + 0x1ecf, 0x0000, + /* 0x0075 0x0309 */ + 0x1ee7, 0x0000, + /* 0x0079 0x0309 */ + 0x1ef7, 0x0000, + /* 0x0041 0x030a */ + 0x00c5, 0x0001, 0x0301, 0x0df8, + /* 0x0055 0x030a */ + 0x016e, 0x0000, + /* 0x0061 0x030a */ + 0x00e5, 0x0001, 0x0301, 0x0dfa, + /* 0x0075 0x030a */ + 0x016f, 0x0000, + /* 0x0077 0x030a */ + 0x1e98, 0x0000, + /* 0x0079 0x030a */ + 0x1e99, 0x0000, + /* 0x004f 0x030b */ + 0x0150, 0x0000, + /* 0x0055 0x030b */ + 0x0170, 0x0000, + /* 0x006f 0x030b */ + 0x0151, 0x0000, + /* 0x0075 0x030b */ + 0x0171, 0x0000, + /* 0x0423 0x030b */ + 0x04f2, 0x0000, + /* 0x0443 0x030b */ + 0x04f3, 0x0000, + /* 0x0041 0x030c */ + 0x01cd, 0x0000, + /* 0x0043 0x030c */ + 0x010c, 0x0000, + /* 0x0044 0x030c */ + 0x010e, 0x0000, + /* 0x0045 0x030c */ + 0x011a, 0x0000, + /* 0x0047 0x030c */ + 0x01e6, 0x0000, + /* 0x0049 0x030c */ + 0x01cf, 0x0000, + /* 0x004b 0x030c */ + 0x01e8, 0x0000, + /* 0x004c 0x030c */ + 0x013d, 0x0000, + /* 0x004e 0x030c */ + 0x0147, 0x0000, + /* 0x004f 0x030c */ + 0x01d1, 0x0000, + /* 0x0052 0x030c */ + 0x0158, 0x0000, + /* 0x0053 0x030c */ + 0x0160, 0x0001, 0x0307, 0x0dfc, + /* 0x0054 0x030c */ + 0x0164, 0x0000, + /* 0x0055 0x030c */ + 0x01d3, 0x0000, + /* 0x005a 0x030c */ + 0x017d, 0x0000, + /* 0x0061 0x030c */ + 0x01ce, 0x0000, + /* 0x0063 0x030c */ + 0x010d, 0x0000, + /* 0x0064 0x030c */ + 0x010f, 0x0000, + /* 0x0065 0x030c */ + 0x011b, 0x0000, + /* 0x0067 0x030c */ + 0x01e7, 0x0000, + /* 0x0069 0x030c */ + 0x01d0, 0x0000, + /* 0x006a 0x030c */ + 0x01f0, 0x0000, + /* 0x006b 0x030c */ + 0x01e9, 0x0000, + /* 0x006c 0x030c */ + 0x013e, 0x0000, + /* 0x006e 0x030c */ + 0x0148, 0x0000, + /* 0x006f 0x030c */ + 0x01d2, 0x0000, + /* 0x0072 0x030c */ + 0x0159, 0x0000, + /* 0x0073 0x030c */ + 0x0161, 0x0001, 0x0307, 0x0dfe, + /* 0x0074 0x030c */ + 0x0165, 0x0000, + /* 0x0075 0x030c */ + 0x01d4, 0x0000, + /* 0x007a 0x030c */ + 0x017e, 0x0000, + /* 0x01b7 0x030c */ + 0x01ee, 0x0000, + /* 0x0292 0x030c */ + 0x01ef, 0x0000, + /* 0x00a8 0x030d */ + 0x0385, 0x0000, + /* 0x0308 0x030d */ + 0x0344, 0x0000, + /* 0x0391 0x030d */ + 0x0386, 0x0000, + /* 0x0395 0x030d */ + 0x0388, 0x0000, + /* 0x0397 0x030d */ + 0x0389, 0x0000, + /* 0x0399 0x030d */ + 0x038a, 0x0000, + /* 0x039f 0x030d */ + 0x038c, 0x0000, + /* 0x03a5 0x030d */ + 0x038e, 0x0000, + /* 0x03a9 0x030d */ + 0x038f, 0x0000, + /* 0x03b1 0x030d */ + 0x03ac, 0x0000, + /* 0x03b5 0x030d */ + 0x03ad, 0x0000, + /* 0x03b7 0x030d */ + 0x03ae, 0x0000, + /* 0x03b9 0x030d */ + 0x03af, 0x0000, + /* 0x03bf 0x030d */ + 0x03cc, 0x0000, + /* 0x03c5 0x030d */ + 0x03cd, 0x0000, + /* 0x03c9 0x030d */ + 0x03ce, 0x0000, + /* 0x03d2 0x030d */ + 0x03d3, 0x0000, + /* 0x0041 0x030f */ + 0x0200, 0x0000, + /* 0x0045 0x030f */ + 0x0204, 0x0000, + /* 0x0049 0x030f */ + 0x0208, 0x0000, + /* 0x004f 0x030f */ + 0x020c, 0x0000, + /* 0x0052 0x030f */ + 0x0210, 0x0000, + /* 0x0055 0x030f */ + 0x0214, 0x0000, + /* 0x0061 0x030f */ + 0x0201, 0x0000, + /* 0x0065 0x030f */ + 0x0205, 0x0000, + /* 0x0069 0x030f */ + 0x0209, 0x0000, + /* 0x006f 0x030f */ + 0x020d, 0x0000, + /* 0x0072 0x030f */ + 0x0211, 0x0000, + /* 0x0075 0x030f */ + 0x0215, 0x0000, + /* 0x0474 0x030f */ + 0x0476, 0x0000, + /* 0x0475 0x030f */ + 0x0477, 0x0000, + /* 0x0041 0x0311 */ + 0x0202, 0x0000, + /* 0x0045 0x0311 */ + 0x0206, 0x0000, + /* 0x0049 0x0311 */ + 0x020a, 0x0000, + /* 0x004f 0x0311 */ + 0x020e, 0x0000, + /* 0x0052 0x0311 */ + 0x0212, 0x0000, + /* 0x0055 0x0311 */ + 0x0216, 0x0000, + /* 0x0061 0x0311 */ + 0x0203, 0x0000, + /* 0x0065 0x0311 */ + 0x0207, 0x0000, + /* 0x0069 0x0311 */ + 0x020b, 0x0000, + /* 0x006f 0x0311 */ + 0x020f, 0x0000, + /* 0x0072 0x0311 */ + 0x0213, 0x0000, + /* 0x0075 0x0311 */ + 0x0217, 0x0000, + /* 0x0391 0x0313 */ + 0x1f08, 0x0003, 0x0300, 0x0e00, 0x0301, 0x0e02, 0x0342, 0x0e04, + /* 0x0395 0x0313 */ + 0x1f18, 0x0002, 0x0300, 0x0e06, 0x0301, 0x0e08, + /* 0x0397 0x0313 */ + 0x1f28, 0x0003, 0x0300, 0x0e0a, 0x0301, 0x0e0c, 0x0342, 0x0e0e, + /* 0x0399 0x0313 */ + 0x1f38, 0x0003, 0x0300, 0x0e10, 0x0301, 0x0e12, 0x0342, 0x0e14, + /* 0x039f 0x0313 */ + 0x1f48, 0x0002, 0x0300, 0x0e16, 0x0301, 0x0e18, + /* 0x03a9 0x0313 */ + 0x1f68, 0x0003, 0x0300, 0x0e1a, 0x0301, 0x0e1c, 0x0342, 0x0e1e, + /* 0x03b1 0x0313 */ + 0x1f00, 0x0003, 0x0300, 0x0e20, 0x0301, 0x0e22, 0x0342, 0x0e24, + /* 0x03b5 0x0313 */ + 0x1f10, 0x0002, 0x0300, 0x0e26, 0x0301, 0x0e28, + /* 0x03b7 0x0313 */ + 0x1f20, 0x0003, 0x0300, 0x0e2a, 0x0301, 0x0e2c, 0x0342, 0x0e2e, + /* 0x03b9 0x0313 */ + 0x1f30, 0x0003, 0x0300, 0x0e30, 0x0301, 0x0e32, 0x0342, 0x0e34, + /* 0x03bf 0x0313 */ + 0x1f40, 0x0002, 0x0300, 0x0e36, 0x0301, 0x0e38, + /* 0x03c1 0x0313 */ + 0x1fe4, 0x0000, + /* 0x03c5 0x0313 */ + 0x1f50, 0x0003, 0x0300, 0x0e3a, 0x0301, 0x0e3c, 0x0342, 0x0e3e, + /* 0x03c9 0x0313 */ + 0x1f60, 0x0003, 0x0300, 0x0e40, 0x0301, 0x0e42, 0x0342, 0x0e44, + /* 0x0391 0x0314 */ + 0x1f09, 0x0003, 0x0300, 0x0e46, 0x0301, 0x0e48, 0x0342, 0x0e4a, + /* 0x0395 0x0314 */ + 0x1f19, 0x0002, 0x0300, 0x0e4c, 0x0301, 0x0e4e, + /* 0x0397 0x0314 */ + 0x1f29, 0x0003, 0x0300, 0x0e50, 0x0301, 0x0e52, 0x0342, 0x0e54, + /* 0x0399 0x0314 */ + 0x1f39, 0x0003, 0x0300, 0x0e56, 0x0301, 0x0e58, 0x0342, 0x0e5a, + /* 0x039f 0x0314 */ + 0x1f49, 0x0002, 0x0300, 0x0e5c, 0x0301, 0x0e5e, + /* 0x03a1 0x0314 */ + 0x1fec, 0x0000, + /* 0x03a5 0x0314 */ + 0x1f59, 0x0003, 0x0300, 0x0e60, 0x0301, 0x0e62, 0x0342, 0x0e64, + /* 0x03a9 0x0314 */ + 0x1f69, 0x0003, 0x0300, 0x0e66, 0x0301, 0x0e68, 0x0342, 0x0e6a, + /* 0x03b1 0x0314 */ + 0x1f01, 0x0003, 0x0300, 0x0e6c, 0x0301, 0x0e6e, 0x0342, 0x0e70, + /* 0x03b5 0x0314 */ + 0x1f11, 0x0002, 0x0300, 0x0e72, 0x0301, 0x0e74, + /* 0x03b7 0x0314 */ + 0x1f21, 0x0003, 0x0300, 0x0e76, 0x0301, 0x0e78, 0x0342, 0x0e7a, + /* 0x03b9 0x0314 */ + 0x1f31, 0x0003, 0x0300, 0x0e7c, 0x0301, 0x0e7e, 0x0342, 0x0e80, + /* 0x03bf 0x0314 */ + 0x1f41, 0x0002, 0x0300, 0x0e82, 0x0301, 0x0e84, + /* 0x03c1 0x0314 */ + 0x1fe5, 0x0000, + /* 0x03c5 0x0314 */ + 0x1f51, 0x0003, 0x0300, 0x0e86, 0x0301, 0x0e88, 0x0342, 0x0e8a, + /* 0x03c9 0x0314 */ + 0x1f61, 0x0003, 0x0300, 0x0e8c, 0x0301, 0x0e8e, 0x0342, 0x0e90, + /* 0x004f 0x031b */ + 0x01a0, 0x0005, 0x0300, 0x0e92, 0x0301, 0x0e94, 0x0303, 0x0e96, + 0x0309, 0x0e98, 0x0323, 0x0e9a, + /* 0x0055 0x031b */ + 0x01af, 0x0005, 0x0300, 0x0e9c, 0x0301, 0x0e9e, 0x0303, 0x0ea0, + 0x0309, 0x0ea2, 0x0323, 0x0ea4, + /* 0x006f 0x031b */ + 0x01a1, 0x0005, 0x0300, 0x0ea6, 0x0301, 0x0ea8, 0x0303, 0x0eaa, + 0x0309, 0x0eac, 0x0323, 0x0eae, + /* 0x0075 0x031b */ + 0x01b0, 0x0005, 0x0300, 0x0eb0, 0x0301, 0x0eb2, 0x0303, 0x0eb4, + 0x0309, 0x0eb6, 0x0323, 0x0eb8, + /* 0x0041 0x0323 */ + 0x1ea0, 0x0002, 0x0302, 0x0eba, 0x0306, 0x0ebc, + /* 0x0042 0x0323 */ + 0x1e04, 0x0000, + /* 0x0044 0x0323 */ + 0x1e0c, 0x0000, + /* 0x0045 0x0323 */ + 0x1eb8, 0x0001, 0x0302, 0x0ebe, + /* 0x0048 0x0323 */ + 0x1e24, 0x0000, + /* 0x0049 0x0323 */ + 0x1eca, 0x0000, + /* 0x004b 0x0323 */ + 0x1e32, 0x0000, + /* 0x004c 0x0323 */ + 0x1e36, 0x0001, 0x0304, 0x0ec0, + /* 0x004d 0x0323 */ + 0x1e42, 0x0000, + /* 0x004e 0x0323 */ + 0x1e46, 0x0000, + /* 0x004f 0x0323 */ + 0x1ecc, 0x0001, 0x0302, 0x0ec2, + /* 0x0052 0x0323 */ + 0x1e5a, 0x0001, 0x0304, 0x0ec4, + /* 0x0053 0x0323 */ + 0x1e62, 0x0001, 0x0307, 0x0ec6, + /* 0x0054 0x0323 */ + 0x1e6c, 0x0000, + /* 0x0055 0x0323 */ + 0x1ee4, 0x0000, + /* 0x0056 0x0323 */ + 0x1e7e, 0x0000, + /* 0x0057 0x0323 */ + 0x1e88, 0x0000, + /* 0x0059 0x0323 */ + 0x1ef4, 0x0000, + /* 0x005a 0x0323 */ + 0x1e92, 0x0000, + /* 0x0061 0x0323 */ + 0x1ea1, 0x0002, 0x0302, 0x0ec8, 0x0306, 0x0eca, + /* 0x0062 0x0323 */ + 0x1e05, 0x0000, + /* 0x0064 0x0323 */ + 0x1e0d, 0x0000, + /* 0x0065 0x0323 */ + 0x1eb9, 0x0001, 0x0302, 0x0ecc, + /* 0x0068 0x0323 */ + 0x1e25, 0x0000, + /* 0x0069 0x0323 */ + 0x1ecb, 0x0000, + /* 0x006b 0x0323 */ + 0x1e33, 0x0000, + /* 0x006c 0x0323 */ + 0x1e37, 0x0001, 0x0304, 0x0ece, + /* 0x006d 0x0323 */ + 0x1e43, 0x0000, + /* 0x006e 0x0323 */ + 0x1e47, 0x0000, + /* 0x006f 0x0323 */ + 0x1ecd, 0x0001, 0x0302, 0x0ed0, + /* 0x0072 0x0323 */ + 0x1e5b, 0x0001, 0x0304, 0x0ed2, + /* 0x0073 0x0323 */ + 0x1e63, 0x0001, 0x0307, 0x0ed4, + /* 0x0074 0x0323 */ + 0x1e6d, 0x0000, + /* 0x0075 0x0323 */ + 0x1ee5, 0x0000, + /* 0x0076 0x0323 */ + 0x1e7f, 0x0000, + /* 0x0077 0x0323 */ + 0x1e89, 0x0000, + /* 0x0079 0x0323 */ + 0x1ef5, 0x0000, + /* 0x007a 0x0323 */ + 0x1e93, 0x0000, + /* 0x0055 0x0324 */ + 0x1e72, 0x0000, + /* 0x0075 0x0324 */ + 0x1e73, 0x0000, + /* 0x0041 0x0325 */ + 0x1e00, 0x0000, + /* 0x0061 0x0325 */ + 0x1e01, 0x0000, + /* 0x0043 0x0327 */ + 0x00c7, 0x0001, 0x0301, 0x0ed6, + /* 0x0044 0x0327 */ + 0x1e10, 0x0000, + /* 0x0045 0x0327 */ + 0x0000, 0x0001, 0x0306, 0x0ed8, + /* 0x0047 0x0327 */ + 0x0122, 0x0000, + /* 0x0048 0x0327 */ + 0x1e28, 0x0000, + /* 0x004b 0x0327 */ + 0x0136, 0x0000, + /* 0x004c 0x0327 */ + 0x013b, 0x0000, + /* 0x004e 0x0327 */ + 0x0145, 0x0000, + /* 0x0052 0x0327 */ + 0x0156, 0x0000, + /* 0x0053 0x0327 */ + 0x015e, 0x0000, + /* 0x0054 0x0327 */ + 0x0162, 0x0000, + /* 0x0063 0x0327 */ + 0x00e7, 0x0001, 0x0301, 0x0eda, + /* 0x0064 0x0327 */ + 0x1e11, 0x0000, + /* 0x0065 0x0327 */ + 0x0000, 0x0001, 0x0306, 0x0edc, + /* 0x0067 0x0327 */ + 0x0123, 0x0000, + /* 0x0068 0x0327 */ + 0x1e29, 0x0000, + /* 0x006b 0x0327 */ + 0x0137, 0x0000, + /* 0x006c 0x0327 */ + 0x013c, 0x0000, + /* 0x006e 0x0327 */ + 0x0146, 0x0000, + /* 0x0072 0x0327 */ + 0x0157, 0x0000, + /* 0x0073 0x0327 */ + 0x015f, 0x0000, + /* 0x0074 0x0327 */ + 0x0163, 0x0000, + /* 0x0041 0x0328 */ + 0x0104, 0x0000, + /* 0x0045 0x0328 */ + 0x0118, 0x0000, + /* 0x0049 0x0328 */ + 0x012e, 0x0000, + /* 0x004f 0x0328 */ + 0x01ea, 0x0001, 0x0304, 0x0ede, + /* 0x0055 0x0328 */ + 0x0172, 0x0000, + /* 0x0061 0x0328 */ + 0x0105, 0x0000, + /* 0x0065 0x0328 */ + 0x0119, 0x0000, + /* 0x0069 0x0328 */ + 0x012f, 0x0000, + /* 0x006f 0x0328 */ + 0x01eb, 0x0001, 0x0304, 0x0ee0, + /* 0x0075 0x0328 */ + 0x0173, 0x0000, + /* 0x0044 0x032d */ + 0x1e12, 0x0000, + /* 0x0045 0x032d */ + 0x1e18, 0x0000, + /* 0x004c 0x032d */ + 0x1e3c, 0x0000, + /* 0x004e 0x032d */ + 0x1e4a, 0x0000, + /* 0x0054 0x032d */ + 0x1e70, 0x0000, + /* 0x0055 0x032d */ + 0x1e76, 0x0000, + /* 0x0064 0x032d */ + 0x1e13, 0x0000, + /* 0x0065 0x032d */ + 0x1e19, 0x0000, + /* 0x006c 0x032d */ + 0x1e3d, 0x0000, + /* 0x006e 0x032d */ + 0x1e4b, 0x0000, + /* 0x0074 0x032d */ + 0x1e71, 0x0000, + /* 0x0075 0x032d */ + 0x1e77, 0x0000, + /* 0x0048 0x032e */ + 0x1e2a, 0x0000, + /* 0x0068 0x032e */ + 0x1e2b, 0x0000, + /* 0x0045 0x0330 */ + 0x1e1a, 0x0000, + /* 0x0049 0x0330 */ + 0x1e2c, 0x0000, + /* 0x0055 0x0330 */ + 0x1e74, 0x0000, + /* 0x0065 0x0330 */ + 0x1e1b, 0x0000, + /* 0x0069 0x0330 */ + 0x1e2d, 0x0000, + /* 0x0075 0x0330 */ + 0x1e75, 0x0000, + /* 0x0042 0x0331 */ + 0x1e06, 0x0000, + /* 0x0044 0x0331 */ + 0x1e0e, 0x0000, + /* 0x004b 0x0331 */ + 0x1e34, 0x0000, + /* 0x004c 0x0331 */ + 0x1e3a, 0x0000, + /* 0x004e 0x0331 */ + 0x1e48, 0x0000, + /* 0x0052 0x0331 */ + 0x1e5e, 0x0000, + /* 0x0054 0x0331 */ + 0x1e6e, 0x0000, + /* 0x005a 0x0331 */ + 0x1e94, 0x0000, + /* 0x0062 0x0331 */ + 0x1e07, 0x0000, + /* 0x0064 0x0331 */ + 0x1e0f, 0x0000, + /* 0x0068 0x0331 */ + 0x1e96, 0x0000, + /* 0x006b 0x0331 */ + 0x1e35, 0x0000, + /* 0x006c 0x0331 */ + 0x1e3b, 0x0000, + /* 0x006e 0x0331 */ + 0x1e49, 0x0000, + /* 0x0072 0x0331 */ + 0x1e5f, 0x0000, + /* 0x0074 0x0331 */ + 0x1e6f, 0x0000, + /* 0x007a 0x0331 */ + 0x1e95, 0x0000, + /* 0x00a8 0x0342 */ + 0x1fc1, 0x0000, + /* 0x03b1 0x0342 */ + 0x1fb6, 0x0000, + /* 0x03b7 0x0342 */ + 0x1fc6, 0x0000, + /* 0x03b9 0x0342 */ + 0x1fd6, 0x0000, + /* 0x03c5 0x0342 */ + 0x1fe6, 0x0000, + /* 0x03c9 0x0342 */ + 0x1ff6, 0x0000, + /* 0x1fbf 0x0342 */ + 0x1fcf, 0x0000, + /* 0x1ffe 0x0342 */ + 0x1fdf, 0x0000, + /* 0x0391 0x0345 */ + 0x1fbc, 0x0002, 0x0313, 0x0ee2, 0x0314, 0x0eea, + /* 0x0397 0x0345 */ + 0x1fcc, 0x0002, 0x0313, 0x0ef2, 0x0314, 0x0efa, + /* 0x03a9 0x0345 */ + 0x1ffc, 0x0002, 0x0313, 0x0f02, 0x0314, 0x0f0a, + /* 0x03b1 0x0345 */ + 0x1fb3, 0x0005, 0x0300, 0x0f12, 0x0301, 0x0f14, 0x0313, 0x0f16, + 0x0314, 0x0f1e, 0x0342, 0x0f26, + /* 0x03b7 0x0345 */ + 0x1fc3, 0x0005, 0x0300, 0x0f28, 0x0301, 0x0f2a, 0x0313, 0x0f2c, + 0x0314, 0x0f34, 0x0342, 0x0f3c, + /* 0x03bf 0x0345 */ + 0x0000, 0x0001, 0x0301, 0x0f3e, + /* 0x03c9 0x0345 */ + 0x1ff3, 0x0004, 0x0300, 0x0f40, 0x0313, 0x0f42, 0x0314, 0x0f4a, + 0x0342, 0x0f52, + /* 0x05d0 0x05b7 */ + 0xfb2e, 0x0000, + /* 0x05f2 0x05b7 */ + 0xfb1f, 0x0000, + /* 0x05d0 0x05b8 */ + 0xfb2f, 0x0000, + /* 0x05d5 0x05b9 */ + 0xfb4b, 0x0000, + /* 0x05d0 0x05bc */ + 0xfb30, 0x0000, + /* 0x05d1 0x05bc */ + 0xfb31, 0x0000, + /* 0x05d2 0x05bc */ + 0xfb32, 0x0000, + /* 0x05d3 0x05bc */ + 0xfb33, 0x0000, + /* 0x05d4 0x05bc */ + 0xfb34, 0x0000, + /* 0x05d5 0x05bc */ + 0xfb35, 0x0000, + /* 0x05d6 0x05bc */ + 0xfb36, 0x0000, + /* 0x05d8 0x05bc */ + 0xfb38, 0x0000, + /* 0x05d9 0x05bc */ + 0xfb39, 0x0000, + /* 0x05da 0x05bc */ + 0xfb3a, 0x0000, + /* 0x05db 0x05bc */ + 0xfb3b, 0x0000, + /* 0x05dc 0x05bc */ + 0xfb3c, 0x0000, + /* 0x05de 0x05bc */ + 0xfb3e, 0x0000, + /* 0x05e0 0x05bc */ + 0xfb40, 0x0000, + /* 0x05e1 0x05bc */ + 0xfb41, 0x0000, + /* 0x05e3 0x05bc */ + 0xfb43, 0x0000, + /* 0x05e4 0x05bc */ + 0xfb44, 0x0000, + /* 0x05e6 0x05bc */ + 0xfb46, 0x0000, + /* 0x05e7 0x05bc */ + 0xfb47, 0x0000, + /* 0x05e8 0x05bc */ + 0xfb48, 0x0000, + /* 0x05e9 0x05bc */ + 0xfb49, 0x0002, 0x05c1, 0x0f54, 0x05c2, 0x0f56, + /* 0x05ea 0x05bc */ + 0xfb4a, 0x0000, + /* 0x05d1 0x05bf */ + 0xfb4c, 0x0000, + /* 0x05db 0x05bf */ + 0xfb4d, 0x0000, + /* 0x05e4 0x05bf */ + 0xfb4e, 0x0000, + /* 0x05e9 0x05c1 */ + 0xfb2a, 0x0000, + /* 0x05e9 0x05c2 */ + 0xfb2b, 0x0000, + /* 0x0915 0x093c */ + 0x0958, 0x0000, + /* 0x0916 0x093c */ + 0x0959, 0x0000, + /* 0x0917 0x093c */ + 0x095a, 0x0000, + /* 0x091c 0x093c */ + 0x095b, 0x0000, + /* 0x0921 0x093c */ + 0x095c, 0x0000, + /* 0x0922 0x093c */ + 0x095d, 0x0000, + /* 0x0928 0x093c */ + 0x0929, 0x0000, + /* 0x092b 0x093c */ + 0x095e, 0x0000, + /* 0x092f 0x093c */ + 0x095f, 0x0000, + /* 0x0930 0x093c */ + 0x0931, 0x0000, + /* 0x0933 0x093c */ + 0x0934, 0x0000, + /* 0x09a1 0x09bc */ + 0x09dc, 0x0000, + /* 0x09a2 0x09bc */ + 0x09dd, 0x0000, + /* 0x09ac 0x09bc */ + 0x09b0, 0x0000, + /* 0x09af 0x09bc */ + 0x09df, 0x0000, + /* 0x09c7 0x09be */ + 0x09cb, 0x0000, + /* 0x09c7 0x09d7 */ + 0x09cc, 0x0000, + /* 0x0a16 0x0a3c */ + 0x0a59, 0x0000, + /* 0x0a17 0x0a3c */ + 0x0a5a, 0x0000, + /* 0x0a1c 0x0a3c */ + 0x0a5b, 0x0000, + /* 0x0a21 0x0a3c */ + 0x0a5c, 0x0000, + /* 0x0a2b 0x0a3c */ + 0x0a5e, 0x0000, + /* 0x0b21 0x0b3c */ + 0x0b5c, 0x0000, + /* 0x0b22 0x0b3c */ + 0x0b5d, 0x0000, + /* 0x0b2f 0x0b3c */ + 0x0b5f, 0x0000, + /* 0x0b47 0x0b3e */ + 0x0b4b, 0x0000, + /* 0x0b47 0x0b56 */ + 0x0b48, 0x0000, + /* 0x0b47 0x0b57 */ + 0x0b4c, 0x0000, + /* 0x0bc6 0x0bbe */ + 0x0bca, 0x0000, + /* 0x0bc7 0x0bbe */ + 0x0bcb, 0x0000, + /* 0x0b92 0x0bd7 */ + 0x0b94, 0x0000, + /* 0x0bc6 0x0bd7 */ + 0x0bcc, 0x0000, + /* 0x0c46 0x0c56 */ + 0x0c48, 0x0000, + /* 0x0cc6 0x0cc2 */ + 0x0cca, 0x0001, 0x0cd5, 0x0f58, + /* 0x0cbf 0x0cd5 */ + 0x0cc0, 0x0000, + /* 0x0cc6 0x0cd5 */ + 0x0cc7, 0x0000, + /* 0x0cc6 0x0cd6 */ + 0x0cc8, 0x0000, + /* 0x0d46 0x0d3e */ + 0x0d4a, 0x0000, + /* 0x0d47 0x0d3e */ + 0x0d4b, 0x0000, + /* 0x0d46 0x0d57 */ + 0x0d4c, 0x0000, + /* 0x0e4d 0x0e32 */ + 0x0e33, 0x0000, + /* 0x0ecd 0x0eb2 */ + 0x0eb3, 0x0000, + /* 0x0f72 0x0f71 */ + 0x0f73, 0x0000, + /* 0x0f74 0x0f71 */ + 0x0f75, 0x0000, + /* 0x0f80 0x0f71 */ + 0x0f81, 0x0000, + /* 0x0fb2 0x0f80 */ + 0x0f76, 0x0001, 0x0f71, 0x0f5a, + /* 0x0fb3 0x0f80 */ + 0x0f78, 0x0001, 0x0f71, 0x0f5c, + /* 0x0f40 0x0fb5 */ + 0x0f69, 0x0000, + /* 0x0f90 0x0fb5 */ + 0x0fb9, 0x0000, + /* 0x0f42 0x0fb7 */ + 0x0f43, 0x0000, + /* 0x0f4c 0x0fb7 */ + 0x0f4d, 0x0000, + /* 0x0f51 0x0fb7 */ + 0x0f52, 0x0000, + /* 0x0f56 0x0fb7 */ + 0x0f57, 0x0000, + /* 0x0f5b 0x0fb7 */ + 0x0f5c, 0x0000, + /* 0x0f92 0x0fb7 */ + 0x0f93, 0x0000, + /* 0x0f9c 0x0fb7 */ + 0x0f9d, 0x0000, + /* 0x0fa1 0x0fb7 */ + 0x0fa2, 0x0000, + /* 0x0fa6 0x0fb7 */ + 0x0fa7, 0x0000, + /* 0x0fab 0x0fb7 */ + 0x0fac, 0x0000, + /* 0x3046 0x3099 */ + 0x3094, 0x0000, + /* 0x304b 0x3099 */ + 0x304c, 0x0000, + /* 0x304d 0x3099 */ + 0x304e, 0x0000, + /* 0x304f 0x3099 */ + 0x3050, 0x0000, + /* 0x3051 0x3099 */ + 0x3052, 0x0000, + /* 0x3053 0x3099 */ + 0x3054, 0x0000, + /* 0x3055 0x3099 */ + 0x3056, 0x0000, + /* 0x3057 0x3099 */ + 0x3058, 0x0000, + /* 0x3059 0x3099 */ + 0x305a, 0x0000, + /* 0x305b 0x3099 */ + 0x305c, 0x0000, + /* 0x305d 0x3099 */ + 0x305e, 0x0000, + /* 0x305f 0x3099 */ + 0x3060, 0x0000, + /* 0x3061 0x3099 */ + 0x3062, 0x0000, + /* 0x3064 0x3099 */ + 0x3065, 0x0000, + /* 0x3066 0x3099 */ + 0x3067, 0x0000, + /* 0x3068 0x3099 */ + 0x3069, 0x0000, + /* 0x306f 0x3099 */ + 0x3070, 0x0000, + /* 0x3072 0x3099 */ + 0x3073, 0x0000, + /* 0x3075 0x3099 */ + 0x3076, 0x0000, + /* 0x3078 0x3099 */ + 0x3079, 0x0000, + /* 0x307b 0x3099 */ + 0x307c, 0x0000, + /* 0x309d 0x3099 */ + 0x309e, 0x0000, + /* 0x30a6 0x3099 */ + 0x30f4, 0x0000, + /* 0x30ab 0x3099 */ + 0x30ac, 0x0000, + /* 0x30ad 0x3099 */ + 0x30ae, 0x0000, + /* 0x30af 0x3099 */ + 0x30b0, 0x0000, + /* 0x30b1 0x3099 */ + 0x30b2, 0x0000, + /* 0x30b3 0x3099 */ + 0x30b4, 0x0000, + /* 0x30b5 0x3099 */ + 0x30b6, 0x0000, + /* 0x30b7 0x3099 */ + 0x30b8, 0x0000, + /* 0x30b9 0x3099 */ + 0x30ba, 0x0000, + /* 0x30bb 0x3099 */ + 0x30bc, 0x0000, + /* 0x30bd 0x3099 */ + 0x30be, 0x0000, + /* 0x30bf 0x3099 */ + 0x30c0, 0x0000, + /* 0x30c1 0x3099 */ + 0x30c2, 0x0000, + /* 0x30c4 0x3099 */ + 0x30c5, 0x0000, + /* 0x30c6 0x3099 */ + 0x30c7, 0x0000, + /* 0x30c8 0x3099 */ + 0x30c9, 0x0000, + /* 0x30cf 0x3099 */ + 0x30d0, 0x0000, + /* 0x30d2 0x3099 */ + 0x30d3, 0x0000, + /* 0x30d5 0x3099 */ + 0x30d6, 0x0000, + /* 0x30d8 0x3099 */ + 0x30d9, 0x0000, + /* 0x30db 0x3099 */ + 0x30dc, 0x0000, + /* 0x30ef 0x3099 */ + 0x30f7, 0x0000, + /* 0x30f0 0x3099 */ + 0x30f8, 0x0000, + /* 0x30f1 0x3099 */ + 0x30f9, 0x0000, + /* 0x30f2 0x3099 */ + 0x30fa, 0x0000, + /* 0x30fd 0x3099 */ + 0x30fe, 0x0000, + /* 0x306f 0x309a */ + 0x3071, 0x0000, + /* 0x3072 0x309a */ + 0x3074, 0x0000, + /* 0x3075 0x309a */ + 0x3077, 0x0000, + /* 0x3078 0x309a */ + 0x307a, 0x0000, + /* 0x307b 0x309a */ + 0x307d, 0x0000, + /* 0x30cf 0x309a */ + 0x30d1, 0x0000, + /* 0x30d2 0x309a */ + 0x30d4, 0x0000, + /* 0x30d5 0x309a */ + 0x30d7, 0x0000, + /* 0x30d8 0x309a */ + 0x30da, 0x0000, + /* 0x30db 0x309a */ + 0x30dd, 0x0000, + /* 0x0307 0x0053 0x0301 */ + 0x1e64, 0x0000, + /* 0x0307 0x0073 0x0301 */ + 0x1e65, 0x0000, + /* 0x0300 0x0041 0x0302 */ + 0x1ea6, 0x0000, + /* 0x0301 0x0041 0x0302 */ + 0x1ea4, 0x0000, + /* 0x0303 0x0041 0x0302 */ + 0x1eaa, 0x0000, + /* 0x0309 0x0041 0x0302 */ + 0x1ea8, 0x0000, + /* 0x0300 0x0045 0x0302 */ + 0x1ec0, 0x0000, + /* 0x0301 0x0045 0x0302 */ + 0x1ebe, 0x0000, + /* 0x0303 0x0045 0x0302 */ + 0x1ec4, 0x0000, + /* 0x0309 0x0045 0x0302 */ + 0x1ec2, 0x0000, + /* 0x0300 0x004f 0x0302 */ + 0x1ed2, 0x0000, + /* 0x0301 0x004f 0x0302 */ + 0x1ed0, 0x0000, + /* 0x0303 0x004f 0x0302 */ + 0x1ed6, 0x0000, + /* 0x0309 0x004f 0x0302 */ + 0x1ed4, 0x0000, + /* 0x0300 0x0061 0x0302 */ + 0x1ea7, 0x0000, + /* 0x0301 0x0061 0x0302 */ + 0x1ea5, 0x0000, + /* 0x0303 0x0061 0x0302 */ + 0x1eab, 0x0000, + /* 0x0309 0x0061 0x0302 */ + 0x1ea9, 0x0000, + /* 0x0300 0x0065 0x0302 */ + 0x1ec1, 0x0000, + /* 0x0301 0x0065 0x0302 */ + 0x1ebf, 0x0000, + /* 0x0303 0x0065 0x0302 */ + 0x1ec5, 0x0000, + /* 0x0309 0x0065 0x0302 */ + 0x1ec3, 0x0000, + /* 0x0300 0x006f 0x0302 */ + 0x1ed3, 0x0000, + /* 0x0301 0x006f 0x0302 */ + 0x1ed1, 0x0000, + /* 0x0303 0x006f 0x0302 */ + 0x1ed7, 0x0000, + /* 0x0309 0x006f 0x0302 */ + 0x1ed5, 0x0000, + /* 0x0301 0x004f 0x0303 */ + 0x1e4c, 0x0000, + /* 0x0308 0x004f 0x0303 */ + 0x1e4e, 0x0000, + /* 0x0301 0x0055 0x0303 */ + 0x1e78, 0x0000, + /* 0x0301 0x006f 0x0303 */ + 0x1e4d, 0x0000, + /* 0x0308 0x006f 0x0303 */ + 0x1e4f, 0x0000, + /* 0x0301 0x0075 0x0303 */ + 0x1e79, 0x0000, + /* 0x0300 0x0045 0x0304 */ + 0x1e14, 0x0000, + /* 0x0301 0x0045 0x0304 */ + 0x1e16, 0x0000, + /* 0x0300 0x004f 0x0304 */ + 0x1e50, 0x0000, + /* 0x0301 0x004f 0x0304 */ + 0x1e52, 0x0000, + /* 0x0308 0x0055 0x0304 */ + 0x1e7a, 0x0000, + /* 0x0300 0x0065 0x0304 */ + 0x1e15, 0x0000, + /* 0x0301 0x0065 0x0304 */ + 0x1e17, 0x0000, + /* 0x0300 0x006f 0x0304 */ + 0x1e51, 0x0000, + /* 0x0301 0x006f 0x0304 */ + 0x1e53, 0x0000, + /* 0x0308 0x0075 0x0304 */ + 0x1e7b, 0x0000, + /* 0x0300 0x0041 0x0306 */ + 0x1eb0, 0x0000, + /* 0x0301 0x0041 0x0306 */ + 0x1eae, 0x0000, + /* 0x0303 0x0041 0x0306 */ + 0x1eb4, 0x0000, + /* 0x0309 0x0041 0x0306 */ + 0x1eb2, 0x0000, + /* 0x0300 0x0061 0x0306 */ + 0x1eb1, 0x0000, + /* 0x0301 0x0061 0x0306 */ + 0x1eaf, 0x0000, + /* 0x0303 0x0061 0x0306 */ + 0x1eb5, 0x0000, + /* 0x0309 0x0061 0x0306 */ + 0x1eb3, 0x0000, + /* 0x0304 0x0041 0x0307 */ + 0x01e0, 0x0000, + /* 0x0304 0x0061 0x0307 */ + 0x01e1, 0x0000, + /* 0x0304 0x0041 0x0308 */ + 0x01de, 0x0000, + /* 0x0301 0x0049 0x0308 */ + 0x1e2e, 0x0000, + /* 0x0300 0x0055 0x0308 */ + 0x01db, 0x0000, + /* 0x0301 0x0055 0x0308 */ + 0x01d7, 0x0000, + /* 0x0304 0x0055 0x0308 */ + 0x01d5, 0x0000, + /* 0x030c 0x0055 0x0308 */ + 0x01d9, 0x0000, + /* 0x0304 0x0061 0x0308 */ + 0x01df, 0x0000, + /* 0x0301 0x0069 0x0308 */ + 0x1e2f, 0x0000, + /* 0x0300 0x0075 0x0308 */ + 0x01dc, 0x0000, + /* 0x0301 0x0075 0x0308 */ + 0x01d8, 0x0000, + /* 0x0304 0x0075 0x0308 */ + 0x01d6, 0x0000, + /* 0x030c 0x0075 0x0308 */ + 0x01da, 0x0000, + /* 0x0300 0x03b9 0x0308 */ + 0x1fd2, 0x0000, + /* 0x0301 0x03b9 0x0308 */ + 0x1fd3, 0x0000, + /* 0x030d 0x03b9 0x0308 */ + 0x0390, 0x0000, + /* 0x0342 0x03b9 0x0308 */ + 0x1fd7, 0x0000, + /* 0x0300 0x03c5 0x0308 */ + 0x1fe2, 0x0000, + /* 0x0301 0x03c5 0x0308 */ + 0x1fe3, 0x0000, + /* 0x030d 0x03c5 0x0308 */ + 0x03b0, 0x0000, + /* 0x0342 0x03c5 0x0308 */ + 0x1fe7, 0x0000, + /* 0x0301 0x0041 0x030a */ + 0x01fa, 0x0000, + /* 0x0301 0x0061 0x030a */ + 0x01fb, 0x0000, + /* 0x0307 0x0053 0x030c */ + 0x1e66, 0x0000, + /* 0x0307 0x0073 0x030c */ + 0x1e67, 0x0000, + /* 0x0300 0x0391 0x0313 */ + 0x1f0a, 0x0000, + /* 0x0301 0x0391 0x0313 */ + 0x1f0c, 0x0000, + /* 0x0342 0x0391 0x0313 */ + 0x1f0e, 0x0000, + /* 0x0300 0x0395 0x0313 */ + 0x1f1a, 0x0000, + /* 0x0301 0x0395 0x0313 */ + 0x1f1c, 0x0000, + /* 0x0300 0x0397 0x0313 */ + 0x1f2a, 0x0000, + /* 0x0301 0x0397 0x0313 */ + 0x1f2c, 0x0000, + /* 0x0342 0x0397 0x0313 */ + 0x1f2e, 0x0000, + /* 0x0300 0x0399 0x0313 */ + 0x1f3a, 0x0000, + /* 0x0301 0x0399 0x0313 */ + 0x1f3c, 0x0000, + /* 0x0342 0x0399 0x0313 */ + 0x1f3e, 0x0000, + /* 0x0300 0x039f 0x0313 */ + 0x1f4a, 0x0000, + /* 0x0301 0x039f 0x0313 */ + 0x1f4c, 0x0000, + /* 0x0300 0x03a9 0x0313 */ + 0x1f6a, 0x0000, + /* 0x0301 0x03a9 0x0313 */ + 0x1f6c, 0x0000, + /* 0x0342 0x03a9 0x0313 */ + 0x1f6e, 0x0000, + /* 0x0300 0x03b1 0x0313 */ + 0x1f02, 0x0000, + /* 0x0301 0x03b1 0x0313 */ + 0x1f04, 0x0000, + /* 0x0342 0x03b1 0x0313 */ + 0x1f06, 0x0000, + /* 0x0300 0x03b5 0x0313 */ + 0x1f12, 0x0000, + /* 0x0301 0x03b5 0x0313 */ + 0x1f14, 0x0000, + /* 0x0300 0x03b7 0x0313 */ + 0x1f22, 0x0000, + /* 0x0301 0x03b7 0x0313 */ + 0x1f24, 0x0000, + /* 0x0342 0x03b7 0x0313 */ + 0x1f26, 0x0000, + /* 0x0300 0x03b9 0x0313 */ + 0x1f32, 0x0000, + /* 0x0301 0x03b9 0x0313 */ + 0x1f34, 0x0000, + /* 0x0342 0x03b9 0x0313 */ + 0x1f36, 0x0000, + /* 0x0300 0x03bf 0x0313 */ + 0x1f42, 0x0000, + /* 0x0301 0x03bf 0x0313 */ + 0x1f44, 0x0000, + /* 0x0300 0x03c5 0x0313 */ + 0x1f52, 0x0000, + /* 0x0301 0x03c5 0x0313 */ + 0x1f54, 0x0000, + /* 0x0342 0x03c5 0x0313 */ + 0x1f56, 0x0000, + /* 0x0300 0x03c9 0x0313 */ + 0x1f62, 0x0000, + /* 0x0301 0x03c9 0x0313 */ + 0x1f64, 0x0000, + /* 0x0342 0x03c9 0x0313 */ + 0x1f66, 0x0000, + /* 0x0300 0x0391 0x0314 */ + 0x1f0b, 0x0000, + /* 0x0301 0x0391 0x0314 */ + 0x1f0d, 0x0000, + /* 0x0342 0x0391 0x0314 */ + 0x1f0f, 0x0000, + /* 0x0300 0x0395 0x0314 */ + 0x1f1b, 0x0000, + /* 0x0301 0x0395 0x0314 */ + 0x1f1d, 0x0000, + /* 0x0300 0x0397 0x0314 */ + 0x1f2b, 0x0000, + /* 0x0301 0x0397 0x0314 */ + 0x1f2d, 0x0000, + /* 0x0342 0x0397 0x0314 */ + 0x1f2f, 0x0000, + /* 0x0300 0x0399 0x0314 */ + 0x1f3b, 0x0000, + /* 0x0301 0x0399 0x0314 */ + 0x1f3d, 0x0000, + /* 0x0342 0x0399 0x0314 */ + 0x1f3f, 0x0000, + /* 0x0300 0x039f 0x0314 */ + 0x1f4b, 0x0000, + /* 0x0301 0x039f 0x0314 */ + 0x1f4d, 0x0000, + /* 0x0300 0x03a5 0x0314 */ + 0x1f5b, 0x0000, + /* 0x0301 0x03a5 0x0314 */ + 0x1f5d, 0x0000, + /* 0x0342 0x03a5 0x0314 */ + 0x1f5f, 0x0000, + /* 0x0300 0x03a9 0x0314 */ + 0x1f6b, 0x0000, + /* 0x0301 0x03a9 0x0314 */ + 0x1f6d, 0x0000, + /* 0x0342 0x03a9 0x0314 */ + 0x1f6f, 0x0000, + /* 0x0300 0x03b1 0x0314 */ + 0x1f03, 0x0000, + /* 0x0301 0x03b1 0x0314 */ + 0x1f05, 0x0000, + /* 0x0342 0x03b1 0x0314 */ + 0x1f07, 0x0000, + /* 0x0300 0x03b5 0x0314 */ + 0x1f13, 0x0000, + /* 0x0301 0x03b5 0x0314 */ + 0x1f15, 0x0000, + /* 0x0300 0x03b7 0x0314 */ + 0x1f23, 0x0000, + /* 0x0301 0x03b7 0x0314 */ + 0x1f25, 0x0000, + /* 0x0342 0x03b7 0x0314 */ + 0x1f27, 0x0000, + /* 0x0300 0x03b9 0x0314 */ + 0x1f33, 0x0000, + /* 0x0301 0x03b9 0x0314 */ + 0x1f35, 0x0000, + /* 0x0342 0x03b9 0x0314 */ + 0x1f37, 0x0000, + /* 0x0300 0x03bf 0x0314 */ + 0x1f43, 0x0000, + /* 0x0301 0x03bf 0x0314 */ + 0x1f45, 0x0000, + /* 0x0300 0x03c5 0x0314 */ + 0x1f53, 0x0000, + /* 0x0301 0x03c5 0x0314 */ + 0x1f55, 0x0000, + /* 0x0342 0x03c5 0x0314 */ + 0x1f57, 0x0000, + /* 0x0300 0x03c9 0x0314 */ + 0x1f63, 0x0000, + /* 0x0301 0x03c9 0x0314 */ + 0x1f65, 0x0000, + /* 0x0342 0x03c9 0x0314 */ + 0x1f67, 0x0000, + /* 0x0300 0x004f 0x031b */ + 0x1edc, 0x0000, + /* 0x0301 0x004f 0x031b */ + 0x1eda, 0x0000, + /* 0x0303 0x004f 0x031b */ + 0x1ee0, 0x0000, + /* 0x0309 0x004f 0x031b */ + 0x1ede, 0x0000, + /* 0x0323 0x004f 0x031b */ + 0x1ee2, 0x0000, + /* 0x0300 0x0055 0x031b */ + 0x1eea, 0x0000, + /* 0x0301 0x0055 0x031b */ + 0x1ee8, 0x0000, + /* 0x0303 0x0055 0x031b */ + 0x1eee, 0x0000, + /* 0x0309 0x0055 0x031b */ + 0x1eec, 0x0000, + /* 0x0323 0x0055 0x031b */ + 0x1ef0, 0x0000, + /* 0x0300 0x006f 0x031b */ + 0x1edd, 0x0000, + /* 0x0301 0x006f 0x031b */ + 0x1edb, 0x0000, + /* 0x0303 0x006f 0x031b */ + 0x1ee1, 0x0000, + /* 0x0309 0x006f 0x031b */ + 0x1edf, 0x0000, + /* 0x0323 0x006f 0x031b */ + 0x1ee3, 0x0000, + /* 0x0300 0x0075 0x031b */ + 0x1eeb, 0x0000, + /* 0x0301 0x0075 0x031b */ + 0x1ee9, 0x0000, + /* 0x0303 0x0075 0x031b */ + 0x1eef, 0x0000, + /* 0x0309 0x0075 0x031b */ + 0x1eed, 0x0000, + /* 0x0323 0x0075 0x031b */ + 0x1ef1, 0x0000, + /* 0x0302 0x0041 0x0323 */ + 0x1eac, 0x0000, + /* 0x0306 0x0041 0x0323 */ + 0x1eb6, 0x0000, + /* 0x0302 0x0045 0x0323 */ + 0x1ec6, 0x0000, + /* 0x0304 0x004c 0x0323 */ + 0x1e38, 0x0000, + /* 0x0302 0x004f 0x0323 */ + 0x1ed8, 0x0000, + /* 0x0304 0x0052 0x0323 */ + 0x1e5c, 0x0000, + /* 0x0307 0x0053 0x0323 */ + 0x1e68, 0x0000, + /* 0x0302 0x0061 0x0323 */ + 0x1ead, 0x0000, + /* 0x0306 0x0061 0x0323 */ + 0x1eb7, 0x0000, + /* 0x0302 0x0065 0x0323 */ + 0x1ec7, 0x0000, + /* 0x0304 0x006c 0x0323 */ + 0x1e39, 0x0000, + /* 0x0302 0x006f 0x0323 */ + 0x1ed9, 0x0000, + /* 0x0304 0x0072 0x0323 */ + 0x1e5d, 0x0000, + /* 0x0307 0x0073 0x0323 */ + 0x1e69, 0x0000, + /* 0x0301 0x0043 0x0327 */ + 0x1e08, 0x0000, + /* 0x0306 0x0045 0x0327 */ + 0x1e1c, 0x0000, + /* 0x0301 0x0063 0x0327 */ + 0x1e09, 0x0000, + /* 0x0306 0x0065 0x0327 */ + 0x1e1d, 0x0000, + /* 0x0304 0x004f 0x0328 */ + 0x01ec, 0x0000, + /* 0x0304 0x006f 0x0328 */ + 0x01ed, 0x0000, + /* 0x0313 0x0391 0x0345 */ + 0x1f88, 0x0003, 0x0300, 0x0f5e, 0x0301, 0x0f60, 0x0342, 0x0f62, + /* 0x0314 0x0391 0x0345 */ + 0x1f89, 0x0003, 0x0300, 0x0f64, 0x0301, 0x0f66, 0x0342, 0x0f68, + /* 0x0313 0x0397 0x0345 */ + 0x1f98, 0x0003, 0x0300, 0x0f6a, 0x0301, 0x0f6c, 0x0342, 0x0f6e, + /* 0x0314 0x0397 0x0345 */ + 0x1f99, 0x0003, 0x0300, 0x0f70, 0x0301, 0x0f72, 0x0342, 0x0f74, + /* 0x0313 0x03a9 0x0345 */ + 0x1fa8, 0x0003, 0x0300, 0x0f76, 0x0301, 0x0f78, 0x0342, 0x0f7a, + /* 0x0314 0x03a9 0x0345 */ + 0x1fa9, 0x0003, 0x0300, 0x0f7c, 0x0301, 0x0f7e, 0x0342, 0x0f80, + /* 0x0300 0x03b1 0x0345 */ + 0x1fb2, 0x0000, + /* 0x0301 0x03b1 0x0345 */ + 0x1fb4, 0x0000, + /* 0x0313 0x03b1 0x0345 */ + 0x1f80, 0x0003, 0x0300, 0x0f82, 0x0301, 0x0f84, 0x0342, 0x0f86, + /* 0x0314 0x03b1 0x0345 */ + 0x1f81, 0x0003, 0x0300, 0x0f88, 0x0301, 0x0f8a, 0x0342, 0x0f8c, + /* 0x0342 0x03b1 0x0345 */ + 0x1fb7, 0x0000, + /* 0x0300 0x03b7 0x0345 */ + 0x1fc2, 0x0000, + /* 0x0301 0x03b7 0x0345 */ + 0x1fc4, 0x0000, + /* 0x0313 0x03b7 0x0345 */ + 0x1f90, 0x0003, 0x0300, 0x0f8e, 0x0301, 0x0f90, 0x0342, 0x0f92, + /* 0x0314 0x03b7 0x0345 */ + 0x1f91, 0x0003, 0x0300, 0x0f94, 0x0301, 0x0f96, 0x0342, 0x0f98, + /* 0x0342 0x03b7 0x0345 */ + 0x1fc7, 0x0000, + /* 0x0301 0x03bf 0x0345 */ + 0x1ff4, 0x0000, + /* 0x0300 0x03c9 0x0345 */ + 0x1ff2, 0x0000, + /* 0x0313 0x03c9 0x0345 */ + 0x1fa0, 0x0003, 0x0300, 0x0f9a, 0x0301, 0x0f9c, 0x0342, 0x0f9e, + /* 0x0314 0x03c9 0x0345 */ + 0x1fa1, 0x0003, 0x0300, 0x0fa0, 0x0301, 0x0fa2, 0x0342, 0x0fa4, + /* 0x0342 0x03c9 0x0345 */ + 0x1ff7, 0x0000, + /* 0x05c1 0x05e9 0x05bc */ + 0xfb2c, 0x0000, + /* 0x05c2 0x05e9 0x05bc */ + 0xfb2d, 0x0000, + /* 0x0cd5 0x0cc6 0x0cc2 */ + 0x0ccb, 0x0000, + /* 0x0f71 0x0fb2 0x0f80 */ + 0x0f77, 0x0000, + /* 0x0f71 0x0fb3 0x0f80 */ + 0x0f79, 0x0000, + /* 0x0300 0x0313 0x0391 0x0345 */ + 0x1f8a, 0x0000, + /* 0x0301 0x0313 0x0391 0x0345 */ + 0x1f8c, 0x0000, + /* 0x0342 0x0313 0x0391 0x0345 */ + 0x1f8e, 0x0000, + /* 0x0300 0x0314 0x0391 0x0345 */ + 0x1f8b, 0x0000, + /* 0x0301 0x0314 0x0391 0x0345 */ + 0x1f8d, 0x0000, + /* 0x0342 0x0314 0x0391 0x0345 */ + 0x1f8f, 0x0000, + /* 0x0300 0x0313 0x0397 0x0345 */ + 0x1f9a, 0x0000, + /* 0x0301 0x0313 0x0397 0x0345 */ + 0x1f9c, 0x0000, + /* 0x0342 0x0313 0x0397 0x0345 */ + 0x1f9e, 0x0000, + /* 0x0300 0x0314 0x0397 0x0345 */ + 0x1f9b, 0x0000, + /* 0x0301 0x0314 0x0397 0x0345 */ + 0x1f9d, 0x0000, + /* 0x0342 0x0314 0x0397 0x0345 */ + 0x1f9f, 0x0000, + /* 0x0300 0x0313 0x03a9 0x0345 */ + 0x1faa, 0x0000, + /* 0x0301 0x0313 0x03a9 0x0345 */ + 0x1fac, 0x0000, + /* 0x0342 0x0313 0x03a9 0x0345 */ + 0x1fae, 0x0000, + /* 0x0300 0x0314 0x03a9 0x0345 */ + 0x1fab, 0x0000, + /* 0x0301 0x0314 0x03a9 0x0345 */ + 0x1fad, 0x0000, + /* 0x0342 0x0314 0x03a9 0x0345 */ + 0x1faf, 0x0000, + /* 0x0300 0x0313 0x03b1 0x0345 */ + 0x1f82, 0x0000, + /* 0x0301 0x0313 0x03b1 0x0345 */ + 0x1f84, 0x0000, + /* 0x0342 0x0313 0x03b1 0x0345 */ + 0x1f86, 0x0000, + /* 0x0300 0x0314 0x03b1 0x0345 */ + 0x1f83, 0x0000, + /* 0x0301 0x0314 0x03b1 0x0345 */ + 0x1f85, 0x0000, + /* 0x0342 0x0314 0x03b1 0x0345 */ + 0x1f87, 0x0000, + /* 0x0300 0x0313 0x03b7 0x0345 */ + 0x1f92, 0x0000, + /* 0x0301 0x0313 0x03b7 0x0345 */ + 0x1f94, 0x0000, + /* 0x0342 0x0313 0x03b7 0x0345 */ + 0x1f96, 0x0000, + /* 0x0300 0x0314 0x03b7 0x0345 */ + 0x1f93, 0x0000, + /* 0x0301 0x0314 0x03b7 0x0345 */ + 0x1f95, 0x0000, + /* 0x0342 0x0314 0x03b7 0x0345 */ + 0x1f97, 0x0000, + /* 0x0300 0x0313 0x03c9 0x0345 */ + 0x1fa2, 0x0000, + /* 0x0301 0x0313 0x03c9 0x0345 */ + 0x1fa4, 0x0000, + /* 0x0342 0x0313 0x03c9 0x0345 */ + 0x1fa6, 0x0000, + /* 0x0300 0x0314 0x03c9 0x0345 */ + 0x1fa3, 0x0000, + /* 0x0301 0x0314 0x03c9 0x0345 */ + 0x1fa5, 0x0000, + /* 0x0342 0x0314 0x03c9 0x0345 */ + 0x1fa7, 0x0000, +}; diff --git a/fs/hfsplus/unicode.c b/fs/hfsplus/unicode.c new file mode 100644 index 000000000..008fec186 --- /dev/null +++ b/fs/hfsplus/unicode.c @@ -0,0 +1,628 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hfsplus/unicode.c + * + * Copyright (C) 2001 + * Brad Boyer (flar@allandria.com) + * (C) 2003 Ardis Technologies + * + * Handler routines for unicode strings + */ + +#include +#include + +#include + +#include "hfsplus_fs.h" +#include "hfsplus_raw.h" + +/* Fold the case of a unicode char, given the 16 bit value */ +/* Returns folded char, or 0 if ignorable */ +static inline u16 case_fold(u16 c) +{ + u16 tmp; + + tmp = hfsplus_case_fold_table[c >> 8]; + if (tmp) + tmp = hfsplus_case_fold_table[tmp + (c & 0xff)]; + else + tmp = c; + return tmp; +} + +/* Compare unicode strings, return values like normal strcmp */ +int hfsplus_strcasecmp(const struct hfsplus_unistr *s1, + const struct hfsplus_unistr *s2) +{ + u16 len1, len2, c1, c2; + const hfsplus_unichr *p1, *p2; + + len1 = be16_to_cpu(s1->length); + len2 = be16_to_cpu(s2->length); + p1 = s1->unicode; + p2 = s2->unicode; + + if (len1 > HFSPLUS_MAX_STRLEN) { + len1 = HFSPLUS_MAX_STRLEN; + pr_err("invalid length %u has been corrected to %d\n", + be16_to_cpu(s1->length), len1); + } + + if (len2 > HFSPLUS_MAX_STRLEN) { + len2 = HFSPLUS_MAX_STRLEN; + pr_err("invalid length %u has been corrected to %d\n", + be16_to_cpu(s2->length), len2); + } + + while (1) { + c1 = c2 = 0; + + while (len1 && !c1) { + c1 = case_fold(be16_to_cpu(*p1)); + p1++; + len1--; + } + while (len2 && !c2) { + c2 = case_fold(be16_to_cpu(*p2)); + p2++; + len2--; + } + + if (c1 != c2) + return (c1 < c2) ? -1 : 1; + if (!c1 && !c2) + return 0; + } +} +EXPORT_SYMBOL_IF_KUNIT(hfsplus_strcasecmp); + +/* Compare names as a sequence of 16-bit unsigned integers */ +int hfsplus_strcmp(const struct hfsplus_unistr *s1, + const struct hfsplus_unistr *s2) +{ + u16 len1, len2, c1, c2; + const hfsplus_unichr *p1, *p2; + int len; + + len1 = be16_to_cpu(s1->length); + len2 = be16_to_cpu(s2->length); + p1 = s1->unicode; + p2 = s2->unicode; + + if (len1 > HFSPLUS_MAX_STRLEN) { + len1 = HFSPLUS_MAX_STRLEN; + pr_err("invalid length %u has been corrected to %d\n", + be16_to_cpu(s1->length), len1); + } + + if (len2 > HFSPLUS_MAX_STRLEN) { + len2 = HFSPLUS_MAX_STRLEN; + pr_err("invalid length %u has been corrected to %d\n", + be16_to_cpu(s2->length), len2); + } + + for (len = min(len1, len2); len > 0; len--) { + c1 = be16_to_cpu(*p1); + c2 = be16_to_cpu(*p2); + if (c1 != c2) + return c1 < c2 ? -1 : 1; + p1++; + p2++; + } + + return len1 < len2 ? -1 : + len1 > len2 ? 1 : 0; +} +EXPORT_SYMBOL_IF_KUNIT(hfsplus_strcmp); + +#define Hangul_SBase 0xac00 +#define Hangul_LBase 0x1100 +#define Hangul_VBase 0x1161 +#define Hangul_TBase 0x11a7 +#define Hangul_SCount 11172 +#define Hangul_LCount 19 +#define Hangul_VCount 21 +#define Hangul_TCount 28 +#define Hangul_NCount (Hangul_VCount * Hangul_TCount) + + +static u16 *hfsplus_compose_lookup(u16 *p, u16 cc) +{ + int i, s, e; + + s = 1; + e = p[1]; + if (!e || cc < p[s * 2] || cc > p[e * 2]) + return NULL; + do { + i = (s + e) / 2; + if (cc > p[i * 2]) + s = i + 1; + else if (cc < p[i * 2]) + e = i - 1; + else + return hfsplus_compose_table + p[i * 2 + 1]; + } while (s <= e); + return NULL; +} + +/* + * In HFS+, a filename can contain / because : is the separator. + * The slash is a valid filename character on macOS. + * But on Linux, / is the path separator and + * it cannot appear in a filename component. + * There's a parallel mapping for the NUL character (0 -> U+2400). + * NUL terminates strings in C/POSIX but is valid in HFS+ filenames. + */ +static inline +void hfsplus_mac2linux_compatibility_check(u16 symbol, u16 *conversion, + int name_type) +{ + *conversion = symbol; + + switch (name_type) { + case HFS_XATTR_NAME: + /* ignore conversion */ + return; + + default: + /* continue logic */ + break; + } + + switch (symbol) { + case 0: + *conversion = 0x2400; + break; + case '/': + *conversion = ':'; + break; + } +} + +static int hfsplus_uni2asc(struct super_block *sb, + const struct hfsplus_unistr *ustr, + int max_len, char *astr, int *len_p, + int name_type) +{ + const hfsplus_unichr *ip; + struct nls_table *nls = HFSPLUS_SB(sb)->nls; + u8 *op; + u16 cc, c0, c1; + u16 *ce1, *ce2; + int i, len, ustrlen, res, compose; + + op = astr; + ip = ustr->unicode; + + ustrlen = be16_to_cpu(ustr->length); + if (ustrlen > max_len) { + ustrlen = max_len; + pr_err("invalid length %u has been corrected to %d\n", + be16_to_cpu(ustr->length), ustrlen); + } + + len = *len_p; + ce1 = NULL; + compose = !test_bit(HFSPLUS_SB_NODECOMPOSE, &HFSPLUS_SB(sb)->flags); + + while (ustrlen > 0) { + c0 = be16_to_cpu(*ip++); + ustrlen--; + /* search for single decomposed char */ + if (likely(compose)) + ce1 = hfsplus_compose_lookup(hfsplus_compose_table, c0); + if (ce1) + cc = ce1[0]; + else + cc = 0; + if (cc) { + /* start of a possibly decomposed Hangul char */ + if (cc != 0xffff) + goto done; + if (!ustrlen) + goto same; + c1 = be16_to_cpu(*ip) - Hangul_VBase; + if (c1 < Hangul_VCount) { + /* compose the Hangul char */ + cc = (c0 - Hangul_LBase) * Hangul_VCount; + cc = (cc + c1) * Hangul_TCount; + cc += Hangul_SBase; + ip++; + ustrlen--; + if (!ustrlen) + goto done; + c1 = be16_to_cpu(*ip) - Hangul_TBase; + if (c1 > 0 && c1 < Hangul_TCount) { + cc += c1; + ip++; + ustrlen--; + } + goto done; + } + } + while (1) { + /* main loop for common case of not composed chars */ + if (!ustrlen) + goto same; + c1 = be16_to_cpu(*ip); + if (likely(compose)) + ce1 = hfsplus_compose_lookup( + hfsplus_compose_table, c1); + if (ce1) + break; + hfsplus_mac2linux_compatibility_check(c0, &c0, + name_type); + res = nls->uni2char(c0, op, len); + if (res < 0) { + if (res == -ENAMETOOLONG) + goto out; + *op = '?'; + res = 1; + } + op += res; + len -= res; + c0 = c1; + ip++; + ustrlen--; + } + ce2 = hfsplus_compose_lookup(ce1, c0); + if (ce2) { + i = 1; + while (i < ustrlen) { + ce1 = hfsplus_compose_lookup(ce2, + be16_to_cpu(ip[i])); + if (!ce1) + break; + i++; + ce2 = ce1; + } + cc = ce2[0]; + if (cc) { + ip += i; + ustrlen -= i; + goto done; + } + } +same: + hfsplus_mac2linux_compatibility_check(c0, &cc, + name_type); +done: + res = nls->uni2char(cc, op, len); + if (res < 0) { + if (res == -ENAMETOOLONG) + goto out; + *op = '?'; + res = 1; + } + op += res; + len -= res; + } + res = 0; +out: + *len_p = (char *)op - astr; + return res; +} + +inline int hfsplus_uni2asc_str(struct super_block *sb, + const struct hfsplus_unistr *ustr, char *astr, + int *len_p) +{ + return hfsplus_uni2asc(sb, + ustr, HFSPLUS_MAX_STRLEN, + astr, len_p, + HFS_REGULAR_NAME); +} +EXPORT_SYMBOL_IF_KUNIT(hfsplus_uni2asc_str); + +inline int hfsplus_uni2asc_xattr_str(struct super_block *sb, + const struct hfsplus_attr_unistr *ustr, + char *astr, int *len_p) +{ + return hfsplus_uni2asc(sb, (const struct hfsplus_unistr *)ustr, + HFSPLUS_ATTR_MAX_STRLEN, astr, len_p, + HFS_XATTR_NAME); +} +EXPORT_SYMBOL_IF_KUNIT(hfsplus_uni2asc_xattr_str); + +/* + * In HFS+, a filename can contain / because : is the separator. + * The slash is a valid filename character on macOS. + * But on Linux, / is the path separator and + * it cannot appear in a filename component. + * There's a parallel mapping for the NUL character (0 -> U+2400). + * NUL terminates strings in C/POSIX but is valid in HFS+ filenames. + */ +static inline +void hfsplus_linux2mac_compatibility_check(wchar_t *uc, int name_type) +{ + switch (name_type) { + case HFS_XATTR_NAME: + /* ignore conversion */ + return; + + default: + /* continue logic */ + break; + } + + switch (*uc) { + case 0x2400: + *uc = 0; + break; + case ':': + *uc = '/'; + break; + } +} + +/* + * Convert one or more ASCII characters into a single unicode character. + * Returns the number of ASCII characters corresponding to the unicode char. + */ +static inline int asc2unichar(struct super_block *sb, const char *astr, int len, + wchar_t *uc, int name_type) +{ + int size = HFSPLUS_SB(sb)->nls->char2uni(astr, len, uc); + + if (size <= 0) { + *uc = '?'; + size = 1; + } + + hfsplus_linux2mac_compatibility_check(uc, name_type); + return size; +} + +/* Decomposes a non-Hangul unicode character. */ +static u16 *hfsplus_decompose_nonhangul(wchar_t uc, int *size) +{ + int off; + + off = hfsplus_decompose_table[(uc >> 12) & 0xf]; + if (off == 0 || off == 0xffff) + return NULL; + + off = hfsplus_decompose_table[off + ((uc >> 8) & 0xf)]; + if (!off) + return NULL; + + off = hfsplus_decompose_table[off + ((uc >> 4) & 0xf)]; + if (!off) + return NULL; + + off = hfsplus_decompose_table[off + (uc & 0xf)]; + *size = off & 3; + if (*size == 0) + return NULL; + return hfsplus_decompose_table + (off / 4); +} + +/* + * Try to decompose a unicode character as Hangul. Return 0 if @uc is not + * precomposed Hangul, otherwise return the length of the decomposition. + * + * This function was adapted from sample code from the Unicode Standard + * Annex #15: Unicode Normalization Forms, version 3.2.0. + * + * Copyright (C) 1991-2018 Unicode, Inc. All rights reserved. Distributed + * under the Terms of Use in http://www.unicode.org/copyright.html. + */ +static int hfsplus_try_decompose_hangul(wchar_t uc, u16 *result) +{ + int index; + int l, v, t; + + index = uc - Hangul_SBase; + if (index < 0 || index >= Hangul_SCount) + return 0; + + l = Hangul_LBase + index / Hangul_NCount; + v = Hangul_VBase + (index % Hangul_NCount) / Hangul_TCount; + t = Hangul_TBase + index % Hangul_TCount; + + result[0] = l; + result[1] = v; + if (t != Hangul_TBase) { + result[2] = t; + return 3; + } + return 2; +} + +/* Decomposes a single unicode character. */ +static u16 *decompose_unichar(wchar_t uc, int *size, u16 *hangul_buffer) +{ + u16 *result; + + /* Hangul is handled separately */ + result = hangul_buffer; + *size = hfsplus_try_decompose_hangul(uc, result); + if (*size == 0) + result = hfsplus_decompose_nonhangul(uc, size); + return result; +} + +int hfsplus_asc2uni(struct super_block *sb, + struct hfsplus_unistr *ustr, int max_unistr_len, + const char *astr, int len, int name_type) +{ + int size, dsize, decompose; + u16 *dstr, outlen = 0; + wchar_t c; + u16 dhangul[3]; + + decompose = !test_bit(HFSPLUS_SB_NODECOMPOSE, &HFSPLUS_SB(sb)->flags); + while (outlen < max_unistr_len && len > 0) { + size = asc2unichar(sb, astr, len, &c, name_type); + + if (decompose) + dstr = decompose_unichar(c, &dsize, dhangul); + else + dstr = NULL; + if (dstr) { + if (outlen + dsize > max_unistr_len) + break; + do { + ustr->unicode[outlen++] = cpu_to_be16(*dstr++); + } while (--dsize > 0); + } else + ustr->unicode[outlen++] = cpu_to_be16(c); + + astr += size; + len -= size; + } + ustr->length = cpu_to_be16(outlen); + if (len > 0) + return -ENAMETOOLONG; + return 0; +} +EXPORT_SYMBOL_IF_KUNIT(hfsplus_asc2uni); + +/* + * Hash a string to an integer as appropriate for the HFS+ filesystem. + * Composed unicode characters are decomposed and case-folding is performed + * if the appropriate bits are (un)set on the superblock. + */ +int hfsplus_hash_dentry(const struct dentry *dentry, struct qstr *str) +{ + struct super_block *sb = dentry->d_sb; + const char *astr; + const u16 *dstr; + int casefold, decompose, size, len; + unsigned long hash; + wchar_t c; + u16 c2; + u16 dhangul[3]; + + casefold = test_bit(HFSPLUS_SB_CASEFOLD, &HFSPLUS_SB(sb)->flags); + decompose = !test_bit(HFSPLUS_SB_NODECOMPOSE, &HFSPLUS_SB(sb)->flags); + hash = init_name_hash(dentry); + astr = str->name; + len = str->len; + while (len > 0) { + int dsize; + size = asc2unichar(sb, astr, len, &c, HFS_REGULAR_NAME); + astr += size; + len -= size; + + if (decompose) + dstr = decompose_unichar(c, &dsize, dhangul); + else + dstr = NULL; + if (dstr) { + do { + c2 = *dstr++; + if (casefold) + c2 = case_fold(c2); + if (!casefold || c2) + hash = partial_name_hash(c2, hash); + } while (--dsize > 0); + } else { + c2 = c; + if (casefold) + c2 = case_fold(c2); + if (!casefold || c2) + hash = partial_name_hash(c2, hash); + } + } + str->hash = end_name_hash(hash); + + return 0; +} +EXPORT_SYMBOL_IF_KUNIT(hfsplus_hash_dentry); + +/* + * Compare strings with HFS+ filename ordering. + * Composed unicode characters are decomposed and case-folding is performed + * if the appropriate bits are (un)set on the superblock. + */ +int hfsplus_compare_dentry(const struct dentry *dentry, + unsigned int len, const char *str, const struct qstr *name) +{ + struct super_block *sb = dentry->d_sb; + int casefold, decompose, size; + int dsize1, dsize2, len1, len2; + const u16 *dstr1, *dstr2; + const char *astr1, *astr2; + u16 c1, c2; + wchar_t c; + u16 dhangul_1[3], dhangul_2[3]; + + casefold = test_bit(HFSPLUS_SB_CASEFOLD, &HFSPLUS_SB(sb)->flags); + decompose = !test_bit(HFSPLUS_SB_NODECOMPOSE, &HFSPLUS_SB(sb)->flags); + astr1 = str; + len1 = len; + astr2 = name->name; + len2 = name->len; + dsize1 = dsize2 = 0; + dstr1 = dstr2 = NULL; + + while (len1 > 0 && len2 > 0) { + if (!dsize1) { + size = asc2unichar(sb, astr1, len1, &c, + HFS_REGULAR_NAME); + astr1 += size; + len1 -= size; + + if (decompose) + dstr1 = decompose_unichar(c, &dsize1, + dhangul_1); + if (!decompose || !dstr1) { + c1 = c; + dstr1 = &c1; + dsize1 = 1; + } + } + + if (!dsize2) { + size = asc2unichar(sb, astr2, len2, &c, + HFS_REGULAR_NAME); + astr2 += size; + len2 -= size; + + if (decompose) + dstr2 = decompose_unichar(c, &dsize2, + dhangul_2); + if (!decompose || !dstr2) { + c2 = c; + dstr2 = &c2; + dsize2 = 1; + } + } + + c1 = *dstr1; + c2 = *dstr2; + if (casefold) { + c1 = case_fold(c1); + if (!c1) { + dstr1++; + dsize1--; + continue; + } + c2 = case_fold(c2); + if (!c2) { + dstr2++; + dsize2--; + continue; + } + } + if (c1 < c2) + return -1; + else if (c1 > c2) + return 1; + + dstr1++; + dsize1--; + dstr2++; + dsize2--; + } + + if (len1 < len2) + return -1; + if (len1 > len2) + return 1; + return 0; +} +EXPORT_SYMBOL_IF_KUNIT(hfsplus_compare_dentry); diff --git a/fs/hfsplus/unicode_test.c b/fs/hfsplus/unicode_test.c new file mode 100644 index 000000000..7f6b2a3c6 --- /dev/null +++ b/fs/hfsplus/unicode_test.c @@ -0,0 +1,1595 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * KUnit tests for HFS+ Unicode string operations + * + * Copyright (C) 2025 Viacheslav Dubeyko + */ + +#include +#include +#include +#include +#include "hfsplus_fs.h" + +struct test_mock_string_env { + struct hfsplus_unistr str1; + struct hfsplus_unistr str2; + char *buf; + u32 buf_size; +}; + +static struct test_mock_string_env *setup_mock_str_env(u32 buf_size) +{ + struct test_mock_string_env *env; + + env = kzalloc_obj(struct test_mock_string_env); + if (!env) + return NULL; + + env->buf = kzalloc(buf_size, GFP_KERNEL); + if (!env->buf) { + kfree(env); + return NULL; + } + + env->buf_size = buf_size; + + return env; +} + +static void free_mock_str_env(struct test_mock_string_env *env) +{ + kfree(env->buf); + kfree(env); +} + +/* Helper function to create hfsplus_unistr */ +static void create_unistr(struct hfsplus_unistr *ustr, const char *ascii_str) +{ + int len = strlen(ascii_str); + int i; + + memset(ustr->unicode, 0, sizeof(ustr->unicode)); + + ustr->length = cpu_to_be16(len); + for (i = 0; i < len && i < HFSPLUS_MAX_STRLEN; i++) + ustr->unicode[i] = cpu_to_be16((u16)ascii_str[i]); +} + +static void corrupt_unistr(struct hfsplus_unistr *ustr) +{ + ustr->length = cpu_to_be16(U16_MAX); +} + +/* Test hfsplus_strcasecmp function */ +static void hfsplus_strcasecmp_test(struct kunit *test) +{ + struct test_mock_string_env *mock_env; + + mock_env = setup_mock_str_env(HFSPLUS_MAX_STRLEN + 1); + KUNIT_ASSERT_NOT_NULL(test, mock_env); + + /* Test identical strings */ + create_unistr(&mock_env->str1, "hello"); + create_unistr(&mock_env->str2, "hello"); + KUNIT_EXPECT_EQ(test, 0, hfsplus_strcasecmp(&mock_env->str1, + &mock_env->str2)); + + /* Test case insensitive comparison */ + create_unistr(&mock_env->str1, "Hello"); + create_unistr(&mock_env->str2, "hello"); + KUNIT_EXPECT_EQ(test, 0, hfsplus_strcasecmp(&mock_env->str1, + &mock_env->str2)); + + create_unistr(&mock_env->str1, "HELLO"); + create_unistr(&mock_env->str2, "hello"); + KUNIT_EXPECT_EQ(test, 0, hfsplus_strcasecmp(&mock_env->str1, + &mock_env->str2)); + + /* Test different strings */ + create_unistr(&mock_env->str1, "apple"); + create_unistr(&mock_env->str2, "banana"); + KUNIT_EXPECT_LT(test, hfsplus_strcasecmp(&mock_env->str1, + &mock_env->str2), 0); + + create_unistr(&mock_env->str1, "zebra"); + create_unistr(&mock_env->str2, "apple"); + KUNIT_EXPECT_GT(test, hfsplus_strcasecmp(&mock_env->str1, + &mock_env->str2), 0); + + /* Test different lengths */ + create_unistr(&mock_env->str1, "test"); + create_unistr(&mock_env->str2, "testing"); + KUNIT_EXPECT_LT(test, hfsplus_strcasecmp(&mock_env->str1, + &mock_env->str2), 0); + + create_unistr(&mock_env->str1, "testing"); + create_unistr(&mock_env->str2, "test"); + KUNIT_EXPECT_GT(test, hfsplus_strcasecmp(&mock_env->str1, + &mock_env->str2), 0); + + /* Test empty strings */ + create_unistr(&mock_env->str1, ""); + create_unistr(&mock_env->str2, ""); + KUNIT_EXPECT_EQ(test, 0, hfsplus_strcasecmp(&mock_env->str1, + &mock_env->str2)); + + create_unistr(&mock_env->str1, ""); + create_unistr(&mock_env->str2, "test"); + KUNIT_EXPECT_LT(test, hfsplus_strcasecmp(&mock_env->str1, + &mock_env->str2), 0); + + /* Test single characters */ + create_unistr(&mock_env->str1, "A"); + create_unistr(&mock_env->str2, "a"); + KUNIT_EXPECT_EQ(test, 0, hfsplus_strcasecmp(&mock_env->str1, + &mock_env->str2)); + + create_unistr(&mock_env->str1, "A"); + create_unistr(&mock_env->str2, "B"); + KUNIT_EXPECT_LT(test, hfsplus_strcasecmp(&mock_env->str1, + &mock_env->str2), 0); + + /* Test maximum length strings */ + memset(mock_env->buf, 'a', HFSPLUS_MAX_STRLEN); + mock_env->buf[HFSPLUS_MAX_STRLEN] = '\0'; + create_unistr(&mock_env->str1, mock_env->buf); + create_unistr(&mock_env->str2, mock_env->buf); + KUNIT_EXPECT_EQ(test, 0, hfsplus_strcasecmp(&mock_env->str1, + &mock_env->str2)); + + /* Change one character in the middle */ + mock_env->buf[HFSPLUS_MAX_STRLEN / 2] = 'b'; + create_unistr(&mock_env->str2, mock_env->buf); + KUNIT_EXPECT_LT(test, hfsplus_strcasecmp(&mock_env->str1, + &mock_env->str2), 0); + + /* Test corrupted strings */ + create_unistr(&mock_env->str1, ""); + corrupt_unistr(&mock_env->str1); + create_unistr(&mock_env->str2, ""); + KUNIT_EXPECT_NE(test, 0, hfsplus_strcasecmp(&mock_env->str1, + &mock_env->str2)); + + create_unistr(&mock_env->str1, ""); + create_unistr(&mock_env->str2, ""); + corrupt_unistr(&mock_env->str2); + KUNIT_EXPECT_NE(test, 0, hfsplus_strcasecmp(&mock_env->str1, + &mock_env->str2)); + + create_unistr(&mock_env->str1, "test"); + corrupt_unistr(&mock_env->str1); + create_unistr(&mock_env->str2, "testing"); + KUNIT_EXPECT_GT(test, hfsplus_strcasecmp(&mock_env->str1, + &mock_env->str2), 0); + + create_unistr(&mock_env->str1, "test"); + create_unistr(&mock_env->str2, "testing"); + corrupt_unistr(&mock_env->str2); + KUNIT_EXPECT_LT(test, hfsplus_strcasecmp(&mock_env->str1, + &mock_env->str2), 0); + + create_unistr(&mock_env->str1, "testing"); + corrupt_unistr(&mock_env->str1); + create_unistr(&mock_env->str2, "test"); + KUNIT_EXPECT_GT(test, hfsplus_strcasecmp(&mock_env->str1, + &mock_env->str2), 0); + + create_unistr(&mock_env->str1, "testing"); + create_unistr(&mock_env->str2, "test"); + corrupt_unistr(&mock_env->str2); + KUNIT_EXPECT_LT(test, hfsplus_strcasecmp(&mock_env->str1, + &mock_env->str2), 0); + + free_mock_str_env(mock_env); +} + +/* Test hfsplus_strcmp function (case-sensitive) */ +static void hfsplus_strcmp_test(struct kunit *test) +{ + struct test_mock_string_env *mock_env; + + mock_env = setup_mock_str_env(HFSPLUS_MAX_STRLEN + 1); + KUNIT_ASSERT_NOT_NULL(test, mock_env); + + /* Test identical strings */ + create_unistr(&mock_env->str1, "hello"); + create_unistr(&mock_env->str2, "hello"); + KUNIT_EXPECT_EQ(test, 0, hfsplus_strcmp(&mock_env->str1, + &mock_env->str2)); + + /* Test case sensitive comparison - should NOT be equal */ + create_unistr(&mock_env->str1, "Hello"); + create_unistr(&mock_env->str2, "hello"); + KUNIT_EXPECT_NE(test, 0, hfsplus_strcmp(&mock_env->str1, + &mock_env->str2)); + /* 'H' < 'h' in Unicode */ + KUNIT_EXPECT_LT(test, hfsplus_strcmp(&mock_env->str1, + &mock_env->str2), 0); + + /* Test lexicographic ordering */ + create_unistr(&mock_env->str1, "apple"); + create_unistr(&mock_env->str2, "banana"); + KUNIT_EXPECT_LT(test, hfsplus_strcmp(&mock_env->str1, + &mock_env->str2), 0); + + create_unistr(&mock_env->str1, "zebra"); + create_unistr(&mock_env->str2, "apple"); + KUNIT_EXPECT_GT(test, hfsplus_strcmp(&mock_env->str1, + &mock_env->str2), 0); + + /* Test different lengths with common prefix */ + create_unistr(&mock_env->str1, "test"); + create_unistr(&mock_env->str2, "testing"); + KUNIT_EXPECT_LT(test, hfsplus_strcmp(&mock_env->str1, + &mock_env->str2), 0); + + create_unistr(&mock_env->str1, "testing"); + create_unistr(&mock_env->str2, "test"); + KUNIT_EXPECT_GT(test, hfsplus_strcmp(&mock_env->str1, + &mock_env->str2), 0); + + /* Test empty strings */ + create_unistr(&mock_env->str1, ""); + create_unistr(&mock_env->str2, ""); + KUNIT_EXPECT_EQ(test, 0, hfsplus_strcmp(&mock_env->str1, + &mock_env->str2)); + + /* Test maximum length strings */ + memset(mock_env->buf, 'a', HFSPLUS_MAX_STRLEN); + mock_env->buf[HFSPLUS_MAX_STRLEN] = '\0'; + create_unistr(&mock_env->str1, mock_env->buf); + create_unistr(&mock_env->str2, mock_env->buf); + KUNIT_EXPECT_EQ(test, 0, hfsplus_strcmp(&mock_env->str1, + &mock_env->str2)); + + /* Change one character in the middle */ + mock_env->buf[HFSPLUS_MAX_STRLEN / 2] = 'b'; + create_unistr(&mock_env->str2, mock_env->buf); + KUNIT_EXPECT_LT(test, hfsplus_strcmp(&mock_env->str1, + &mock_env->str2), 0); + + /* Test corrupted strings */ + create_unistr(&mock_env->str1, ""); + corrupt_unistr(&mock_env->str1); + create_unistr(&mock_env->str2, ""); + KUNIT_EXPECT_NE(test, 0, hfsplus_strcmp(&mock_env->str1, + &mock_env->str2)); + + create_unistr(&mock_env->str1, ""); + create_unistr(&mock_env->str2, ""); + corrupt_unistr(&mock_env->str2); + KUNIT_EXPECT_NE(test, 0, hfsplus_strcmp(&mock_env->str1, + &mock_env->str2)); + + create_unistr(&mock_env->str1, "test"); + corrupt_unistr(&mock_env->str1); + create_unistr(&mock_env->str2, "testing"); + KUNIT_EXPECT_LT(test, hfsplus_strcmp(&mock_env->str1, + &mock_env->str2), 0); + + create_unistr(&mock_env->str1, "test"); + create_unistr(&mock_env->str2, "testing"); + corrupt_unistr(&mock_env->str2); + KUNIT_EXPECT_LT(test, hfsplus_strcmp(&mock_env->str1, + &mock_env->str2), 0); + + create_unistr(&mock_env->str1, "testing"); + corrupt_unistr(&mock_env->str1); + create_unistr(&mock_env->str2, "test"); + KUNIT_EXPECT_GT(test, hfsplus_strcmp(&mock_env->str1, + &mock_env->str2), 0); + + create_unistr(&mock_env->str1, "testing"); + create_unistr(&mock_env->str2, "test"); + corrupt_unistr(&mock_env->str2); + KUNIT_EXPECT_GT(test, hfsplus_strcmp(&mock_env->str1, + &mock_env->str2), 0); + + free_mock_str_env(mock_env); +} + +/* Test Unicode edge cases */ +static void hfsplus_unicode_edge_cases_test(struct kunit *test) +{ + struct test_mock_string_env *mock_env; + + mock_env = setup_mock_str_env(HFSPLUS_MAX_STRLEN + 1); + KUNIT_ASSERT_NOT_NULL(test, mock_env); + + /* Test with special characters */ + mock_env->str1.length = cpu_to_be16(3); + mock_env->str1.unicode[0] = cpu_to_be16(0x00E9); /* é */ + mock_env->str1.unicode[1] = cpu_to_be16(0x00F1); /* ñ */ + mock_env->str1.unicode[2] = cpu_to_be16(0x00FC); /* ü */ + + mock_env->str2.length = cpu_to_be16(3); + mock_env->str2.unicode[0] = cpu_to_be16(0x00E9); /* é */ + mock_env->str2.unicode[1] = cpu_to_be16(0x00F1); /* ñ */ + mock_env->str2.unicode[2] = cpu_to_be16(0x00FC); /* ü */ + + KUNIT_EXPECT_EQ(test, 0, hfsplus_strcmp(&mock_env->str1, + &mock_env->str2)); + KUNIT_EXPECT_EQ(test, 0, hfsplus_strcasecmp(&mock_env->str1, + &mock_env->str2)); + + /* Test with different special characters */ + mock_env->str2.unicode[1] = cpu_to_be16(0x00F2); /* ò */ + KUNIT_EXPECT_NE(test, 0, hfsplus_strcmp(&mock_env->str1, + &mock_env->str2)); + + /* Test null characters within string (should be handled correctly) */ + mock_env->str1.length = cpu_to_be16(3); + mock_env->str1.unicode[0] = cpu_to_be16('a'); + mock_env->str1.unicode[1] = cpu_to_be16(0x0000); /* null */ + mock_env->str1.unicode[2] = cpu_to_be16('b'); + + mock_env->str2.length = cpu_to_be16(3); + mock_env->str2.unicode[0] = cpu_to_be16('a'); + mock_env->str2.unicode[1] = cpu_to_be16(0x0000); /* null */ + mock_env->str2.unicode[2] = cpu_to_be16('b'); + + KUNIT_EXPECT_EQ(test, 0, hfsplus_strcmp(&mock_env->str1, + &mock_env->str2)); + + free_mock_str_env(mock_env); +} + +/* Test boundary conditions */ +static void hfsplus_unicode_boundary_test(struct kunit *test) +{ + struct test_mock_string_env *mock_env; + int i; + + mock_env = setup_mock_str_env(HFSPLUS_MAX_STRLEN + 1); + KUNIT_ASSERT_NOT_NULL(test, mock_env); + + /* Test maximum length boundary */ + mock_env->str1.length = cpu_to_be16(HFSPLUS_MAX_STRLEN); + mock_env->str2.length = cpu_to_be16(HFSPLUS_MAX_STRLEN); + + for (i = 0; i < HFSPLUS_MAX_STRLEN; i++) { + mock_env->str1.unicode[i] = cpu_to_be16('A'); + mock_env->str2.unicode[i] = cpu_to_be16('A'); + } + + KUNIT_EXPECT_EQ(test, 0, hfsplus_strcmp(&mock_env->str1, + &mock_env->str2)); + + /* Change last character */ + mock_env->str2.unicode[HFSPLUS_MAX_STRLEN - 1] = cpu_to_be16('B'); + KUNIT_EXPECT_LT(test, hfsplus_strcmp(&mock_env->str1, + &mock_env->str2), 0); + + /* Test zero length strings */ + mock_env->str1.length = cpu_to_be16(0); + mock_env->str2.length = cpu_to_be16(0); + KUNIT_EXPECT_EQ(test, 0, hfsplus_strcmp(&mock_env->str1, + &mock_env->str2)); + KUNIT_EXPECT_EQ(test, 0, hfsplus_strcasecmp(&mock_env->str1, + &mock_env->str2)); + + /* Test one character vs empty */ + mock_env->str1.length = cpu_to_be16(1); + mock_env->str1.unicode[0] = cpu_to_be16('A'); + mock_env->str2.length = cpu_to_be16(0); + KUNIT_EXPECT_GT(test, hfsplus_strcmp(&mock_env->str1, + &mock_env->str2), 0); + KUNIT_EXPECT_GT(test, hfsplus_strcasecmp(&mock_env->str1, + &mock_env->str2), 0); + + free_mock_str_env(mock_env); +} + +/* Mock superblock and NLS table for testing hfsplus_uni2asc */ +struct test_mock_sb { + struct nls_table nls; + struct hfsplus_sb_info sb_info; + struct super_block sb; +}; + +static struct test_mock_sb *setup_mock_sb(void) +{ + struct test_mock_sb *ptr; + + ptr = kzalloc_obj(struct test_mock_sb); + if (!ptr) + return NULL; + + ptr->nls.charset = "utf8"; + ptr->nls.uni2char = NULL; /* Will use default behavior */ + ptr->sb_info.nls = &ptr->nls; + ptr->sb.s_fs_info = &ptr->sb_info; + + /* Set default flags - no decomposition, no case folding */ + clear_bit(HFSPLUS_SB_NODECOMPOSE, &ptr->sb_info.flags); + clear_bit(HFSPLUS_SB_CASEFOLD, &ptr->sb_info.flags); + + return ptr; +} + +static void free_mock_sb(struct test_mock_sb *ptr) +{ + kfree(ptr); +} + +/* Simple uni2char implementation for testing */ +static int test_uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + if (boundlen <= 0) + return -ENAMETOOLONG; + + if (uni < 0x80) { + *out = (unsigned char)uni; + return 1; + } + + /* For non-ASCII, just use '?' as fallback */ + *out = '?'; + return 1; +} + +/* Test hfsplus_uni2asc basic functionality */ +static void hfsplus_uni2asc_basic_test(struct kunit *test) +{ + struct test_mock_sb *mock_sb; + struct test_mock_string_env *mock_env; + int len, result; + + mock_env = setup_mock_str_env(HFSPLUS_MAX_STRLEN + 1); + KUNIT_ASSERT_NOT_NULL(test, mock_env); + + mock_sb = setup_mock_sb(); + KUNIT_ASSERT_NOT_NULL(test, mock_sb); + + mock_sb->nls.uni2char = test_uni2char; + + /* Test simple ASCII string conversion */ + create_unistr(&mock_env->str1, "hello"); + len = mock_env->buf_size; + result = hfsplus_uni2asc_str(&mock_sb->sb, &mock_env->str1, + mock_env->buf, &len); + + KUNIT_EXPECT_EQ(test, 0, result); + KUNIT_EXPECT_EQ(test, 5, len); + KUNIT_EXPECT_STREQ(test, "hello", mock_env->buf); + + /* Test empty string */ + create_unistr(&mock_env->str1, ""); + len = mock_env->buf_size; + result = hfsplus_uni2asc_str(&mock_sb->sb, &mock_env->str1, + mock_env->buf, &len); + + KUNIT_EXPECT_EQ(test, 0, result); + KUNIT_EXPECT_EQ(test, 0, len); + + /* Test single character */ + create_unistr(&mock_env->str1, "A"); + len = mock_env->buf_size; + result = hfsplus_uni2asc_str(&mock_sb->sb, &mock_env->str1, + mock_env->buf, &len); + + KUNIT_EXPECT_EQ(test, 0, result); + KUNIT_EXPECT_EQ(test, 1, len); + KUNIT_EXPECT_EQ(test, 'A', mock_env->buf[0]); + + free_mock_str_env(mock_env); + free_mock_sb(mock_sb); +} + +/* Test special character handling */ +static void hfsplus_uni2asc_special_chars_test(struct kunit *test) +{ + struct test_mock_sb *mock_sb; + struct test_mock_string_env *mock_env; + int len, result; + + mock_env = setup_mock_str_env(HFSPLUS_MAX_STRLEN + 1); + KUNIT_ASSERT_NOT_NULL(test, mock_env); + + mock_sb = setup_mock_sb(); + KUNIT_ASSERT_NOT_NULL(test, mock_sb); + + mock_sb->nls.uni2char = test_uni2char; + + /* Test null character conversion (should become 0x2400) */ + mock_env->str1.length = cpu_to_be16(1); + mock_env->str1.unicode[0] = cpu_to_be16(0x0000); + len = mock_env->buf_size; + result = hfsplus_uni2asc_str(&mock_sb->sb, &mock_env->str1, + mock_env->buf, &len); + + KUNIT_EXPECT_EQ(test, 0, result); + KUNIT_EXPECT_EQ(test, 1, len); + /* Our test implementation returns '?' for non-ASCII */ + KUNIT_EXPECT_EQ(test, '?', mock_env->buf[0]); + + /* Test forward slash conversion (should become colon) */ + mock_env->str1.length = cpu_to_be16(1); + mock_env->str1.unicode[0] = cpu_to_be16('/'); + len = mock_env->buf_size; + result = hfsplus_uni2asc_str(&mock_sb->sb, &mock_env->str1, + mock_env->buf, &len); + + KUNIT_EXPECT_EQ(test, 0, result); + KUNIT_EXPECT_EQ(test, 1, len); + KUNIT_EXPECT_EQ(test, ':', mock_env->buf[0]); + + /* Test string with mixed special characters */ + mock_env->str1.length = cpu_to_be16(3); + mock_env->str1.unicode[0] = cpu_to_be16('a'); + mock_env->str1.unicode[1] = cpu_to_be16('/'); + mock_env->str1.unicode[2] = cpu_to_be16('b'); + len = mock_env->buf_size; + result = hfsplus_uni2asc_str(&mock_sb->sb, &mock_env->str1, + mock_env->buf, &len); + + KUNIT_EXPECT_EQ(test, 0, result); + KUNIT_EXPECT_EQ(test, 3, len); + KUNIT_EXPECT_EQ(test, 'a', mock_env->buf[0]); + KUNIT_EXPECT_EQ(test, ':', mock_env->buf[1]); + KUNIT_EXPECT_EQ(test, 'b', mock_env->buf[2]); + + free_mock_str_env(mock_env); + free_mock_sb(mock_sb); +} + +/* Test buffer length handling */ +static void hfsplus_uni2asc_buffer_test(struct kunit *test) +{ + struct test_mock_sb *mock_sb; + struct test_mock_string_env *mock_env; + int len, result; + + mock_env = setup_mock_str_env(10); + KUNIT_ASSERT_NOT_NULL(test, mock_env); + + mock_sb = setup_mock_sb(); + KUNIT_ASSERT_NOT_NULL(test, mock_sb); + + mock_sb->nls.uni2char = test_uni2char; + + /* Test insufficient buffer space */ + create_unistr(&mock_env->str1, "toolongstring"); + len = 5; /* Buffer too small */ + result = hfsplus_uni2asc_str(&mock_sb->sb, &mock_env->str1, + mock_env->buf, &len); + + KUNIT_EXPECT_EQ(test, -ENAMETOOLONG, result); + KUNIT_EXPECT_EQ(test, 5, len); /* Should be set to consumed length */ + + /* Test exact buffer size */ + create_unistr(&mock_env->str1, "exact"); + len = 5; + result = hfsplus_uni2asc_str(&mock_sb->sb, &mock_env->str1, + mock_env->buf, &len); + + KUNIT_EXPECT_EQ(test, 0, result); + KUNIT_EXPECT_EQ(test, 5, len); + + /* Test zero length buffer */ + create_unistr(&mock_env->str1, "test"); + len = 0; + result = hfsplus_uni2asc_str(&mock_sb->sb, &mock_env->str1, + mock_env->buf, &len); + + KUNIT_EXPECT_EQ(test, -ENAMETOOLONG, result); + KUNIT_EXPECT_EQ(test, 0, len); + + free_mock_str_env(mock_env); + free_mock_sb(mock_sb); +} + +/* Test corrupted unicode string handling */ +static void hfsplus_uni2asc_corrupted_test(struct kunit *test) +{ + struct test_mock_sb *mock_sb; + struct test_mock_string_env *mock_env; + int len, result; + + mock_env = setup_mock_str_env(HFSPLUS_MAX_STRLEN + 1); + KUNIT_ASSERT_NOT_NULL(test, mock_env); + + mock_sb = setup_mock_sb(); + KUNIT_ASSERT_NOT_NULL(test, mock_sb); + + mock_sb->nls.uni2char = test_uni2char; + + /* Test corrupted length (too large) */ + create_unistr(&mock_env->str1, "test"); + corrupt_unistr(&mock_env->str1); /* Sets length to U16_MAX */ + len = mock_env->buf_size; + + result = hfsplus_uni2asc_str(&mock_sb->sb, &mock_env->str1, + mock_env->buf, &len); + + /* Should still work but with corrected length */ + KUNIT_EXPECT_EQ(test, 0, result); + /* + * Length should be corrected to HFSPLUS_MAX_STRLEN + * and processed accordingly + */ + KUNIT_EXPECT_GT(test, len, 0); + + free_mock_str_env(mock_env); + free_mock_sb(mock_sb); +} + +/* Test edge cases and boundary conditions */ +static void hfsplus_uni2asc_edge_cases_test(struct kunit *test) +{ + struct test_mock_sb *mock_sb; + struct test_mock_string_env *mock_env; + int len, result; + int i; + + mock_env = setup_mock_str_env(HFSPLUS_MAX_STRLEN * 2); + KUNIT_ASSERT_NOT_NULL(test, mock_env); + + mock_sb = setup_mock_sb(); + KUNIT_ASSERT_NOT_NULL(test, mock_sb); + + mock_sb->nls.uni2char = test_uni2char; + + /* Test maximum length string */ + mock_env->str1.length = cpu_to_be16(HFSPLUS_MAX_STRLEN); + for (i = 0; i < HFSPLUS_MAX_STRLEN; i++) + mock_env->str1.unicode[i] = cpu_to_be16('a'); + + len = mock_env->buf_size; + result = hfsplus_uni2asc_str(&mock_sb->sb, &mock_env->str1, + mock_env->buf, &len); + + KUNIT_EXPECT_EQ(test, 0, result); + KUNIT_EXPECT_EQ(test, HFSPLUS_MAX_STRLEN, len); + + /* Verify all characters are 'a' */ + for (i = 0; i < HFSPLUS_MAX_STRLEN; i++) + KUNIT_EXPECT_EQ(test, 'a', mock_env->buf[i]); + + /* Test string with high Unicode values (non-ASCII) */ + mock_env->str1.length = cpu_to_be16(3); + mock_env->str1.unicode[0] = cpu_to_be16(0x00E9); /* é */ + mock_env->str1.unicode[1] = cpu_to_be16(0x00F1); /* ñ */ + mock_env->str1.unicode[2] = cpu_to_be16(0x00FC); /* ü */ + len = mock_env->buf_size; + result = hfsplus_uni2asc_str(&mock_sb->sb, &mock_env->str1, + mock_env->buf, &len); + + KUNIT_EXPECT_EQ(test, 0, result); + KUNIT_EXPECT_EQ(test, 3, len); + /* Our test implementation converts non-ASCII to '?' */ + KUNIT_EXPECT_EQ(test, '?', mock_env->buf[0]); + KUNIT_EXPECT_EQ(test, '?', mock_env->buf[1]); + KUNIT_EXPECT_EQ(test, '?', mock_env->buf[2]); + + free_mock_str_env(mock_env); + free_mock_sb(mock_sb); +} + +/* Simple char2uni implementation for testing */ +static int test_char2uni(const unsigned char *rawstring, + int boundlen, wchar_t *uni) +{ + if (boundlen <= 0) + return -EINVAL; + + *uni = (wchar_t)*rawstring; + return 1; +} + +/* Helper function to check unicode string contents */ +static void check_unistr_content(struct kunit *test, + struct hfsplus_unistr *ustr, + const char *expected_ascii) +{ + int expected_len = strlen(expected_ascii); + int actual_len = be16_to_cpu(ustr->length); + int i; + + KUNIT_EXPECT_EQ(test, expected_len, actual_len); + + for (i = 0; i < expected_len && i < actual_len; i++) { + u16 expected_char = (u16)expected_ascii[i]; + u16 actual_char = be16_to_cpu(ustr->unicode[i]); + + KUNIT_EXPECT_EQ(test, expected_char, actual_char); + } +} + +/* Test hfsplus_asc2uni basic functionality */ +static void hfsplus_asc2uni_basic_test(struct kunit *test) +{ + struct test_mock_sb *mock_sb; + struct test_mock_string_env *mock_env; + int result; + + mock_env = setup_mock_str_env(HFSPLUS_MAX_STRLEN + 1); + KUNIT_ASSERT_NOT_NULL(test, mock_env); + + mock_sb = setup_mock_sb(); + KUNIT_ASSERT_NOT_NULL(test, mock_sb); + + mock_sb->nls.char2uni = test_char2uni; + + /* Test simple ASCII string conversion */ + result = hfsplus_asc2uni(&mock_sb->sb, &mock_env->str1, + HFSPLUS_MAX_STRLEN, "hello", 5, + HFS_REGULAR_NAME); + + KUNIT_EXPECT_EQ(test, 0, result); + check_unistr_content(test, &mock_env->str1, "hello"); + + /* Test empty string */ + result = hfsplus_asc2uni(&mock_sb->sb, &mock_env->str1, + HFSPLUS_MAX_STRLEN, "", 0, + HFS_REGULAR_NAME); + + KUNIT_EXPECT_EQ(test, 0, result); + KUNIT_EXPECT_EQ(test, 0, be16_to_cpu(mock_env->str1.length)); + + /* Test single character */ + result = hfsplus_asc2uni(&mock_sb->sb, &mock_env->str1, + HFSPLUS_MAX_STRLEN, "A", 1, + HFS_REGULAR_NAME); + + KUNIT_EXPECT_EQ(test, 0, result); + check_unistr_content(test, &mock_env->str1, "A"); + + /* Test null-terminated string with explicit length */ + result = hfsplus_asc2uni(&mock_sb->sb, &mock_env->str1, + HFSPLUS_MAX_STRLEN, "test\0extra", 4, + HFS_REGULAR_NAME); + + KUNIT_EXPECT_EQ(test, 0, result); + check_unistr_content(test, &mock_env->str1, "test"); + + free_mock_str_env(mock_env); + free_mock_sb(mock_sb); +} + +/* Test special character handling in asc2uni */ +static void hfsplus_asc2uni_special_chars_test(struct kunit *test) +{ + struct test_mock_sb *mock_sb; + struct test_mock_string_env *mock_env; + int result; + + mock_env = setup_mock_str_env(HFSPLUS_MAX_STRLEN + 1); + KUNIT_ASSERT_NOT_NULL(test, mock_env); + + mock_sb = setup_mock_sb(); + KUNIT_ASSERT_NOT_NULL(test, mock_sb); + + mock_sb->nls.char2uni = test_char2uni; + + /* Test colon conversion (should become forward slash) */ + result = hfsplus_asc2uni(&mock_sb->sb, &mock_env->str1, + HFSPLUS_MAX_STRLEN, ":", 1, + HFS_REGULAR_NAME); + + KUNIT_EXPECT_EQ(test, 0, result); + KUNIT_EXPECT_EQ(test, 1, be16_to_cpu(mock_env->str1.length)); + KUNIT_EXPECT_EQ(test, '/', be16_to_cpu(mock_env->str1.unicode[0])); + + /* Test string with mixed special characters */ + result = hfsplus_asc2uni(&mock_sb->sb, &mock_env->str1, + HFSPLUS_MAX_STRLEN, "a:b", 3, + HFS_REGULAR_NAME); + + KUNIT_EXPECT_EQ(test, 0, result); + KUNIT_EXPECT_EQ(test, 3, be16_to_cpu(mock_env->str1.length)); + KUNIT_EXPECT_EQ(test, 'a', be16_to_cpu(mock_env->str1.unicode[0])); + KUNIT_EXPECT_EQ(test, '/', be16_to_cpu(mock_env->str1.unicode[1])); + KUNIT_EXPECT_EQ(test, 'b', be16_to_cpu(mock_env->str1.unicode[2])); + + /* Test multiple special characters */ + result = hfsplus_asc2uni(&mock_sb->sb, &mock_env->str1, + HFSPLUS_MAX_STRLEN, ":::", 3, + HFS_REGULAR_NAME); + + KUNIT_EXPECT_EQ(test, 0, result); + KUNIT_EXPECT_EQ(test, 3, be16_to_cpu(mock_env->str1.length)); + KUNIT_EXPECT_EQ(test, '/', be16_to_cpu(mock_env->str1.unicode[0])); + KUNIT_EXPECT_EQ(test, '/', be16_to_cpu(mock_env->str1.unicode[1])); + KUNIT_EXPECT_EQ(test, '/', be16_to_cpu(mock_env->str1.unicode[2])); + + free_mock_str_env(mock_env); + free_mock_sb(mock_sb); +} + +/* Test buffer length limits */ +static void hfsplus_asc2uni_buffer_limits_test(struct kunit *test) +{ + struct test_mock_sb *mock_sb; + struct test_mock_string_env *mock_env; + int result; + + mock_env = setup_mock_str_env(HFSPLUS_MAX_STRLEN + 10); + KUNIT_ASSERT_NOT_NULL(test, mock_env); + + mock_sb = setup_mock_sb(); + KUNIT_ASSERT_NOT_NULL(test, mock_sb); + + mock_sb->nls.char2uni = test_char2uni; + + /* Test exact maximum length */ + memset(mock_env->buf, 'a', HFSPLUS_MAX_STRLEN); + result = hfsplus_asc2uni(&mock_sb->sb, + &mock_env->str1, HFSPLUS_MAX_STRLEN, + mock_env->buf, HFSPLUS_MAX_STRLEN, + HFS_REGULAR_NAME); + + KUNIT_EXPECT_EQ(test, 0, result); + KUNIT_EXPECT_EQ(test, HFSPLUS_MAX_STRLEN, + be16_to_cpu(mock_env->str1.length)); + + /* Test exceeding maximum length */ + memset(mock_env->buf, 'a', HFSPLUS_MAX_STRLEN + 5); + result = hfsplus_asc2uni(&mock_sb->sb, + &mock_env->str1, HFSPLUS_MAX_STRLEN, + mock_env->buf, HFSPLUS_MAX_STRLEN + 5, + HFS_REGULAR_NAME); + + KUNIT_EXPECT_EQ(test, -ENAMETOOLONG, result); + KUNIT_EXPECT_EQ(test, HFSPLUS_MAX_STRLEN, + be16_to_cpu(mock_env->str1.length)); + + /* Test with smaller max_unistr_len */ + result = hfsplus_asc2uni(&mock_sb->sb, + &mock_env->str1, 5, "toolongstring", 13, + HFS_REGULAR_NAME); + + KUNIT_EXPECT_EQ(test, -ENAMETOOLONG, result); + KUNIT_EXPECT_EQ(test, 5, be16_to_cpu(mock_env->str1.length)); + + /* Test zero max length */ + result = hfsplus_asc2uni(&mock_sb->sb, &mock_env->str1, 0, "test", 4, + HFS_REGULAR_NAME); + + KUNIT_EXPECT_EQ(test, -ENAMETOOLONG, result); + KUNIT_EXPECT_EQ(test, 0, be16_to_cpu(mock_env->str1.length)); + + free_mock_str_env(mock_env); + free_mock_sb(mock_sb); +} + +/* Test error handling and edge cases */ +static void hfsplus_asc2uni_edge_cases_test(struct kunit *test) +{ + struct test_mock_sb *mock_sb; + struct hfsplus_unistr ustr; + char test_str[] = {'a', '\0', 'b'}; + int result; + + mock_sb = setup_mock_sb(); + KUNIT_ASSERT_NOT_NULL(test, mock_sb); + + mock_sb->nls.char2uni = test_char2uni; + + /* Test zero length input */ + result = hfsplus_asc2uni(&mock_sb->sb, + &ustr, HFSPLUS_MAX_STRLEN, "test", 0, + HFS_REGULAR_NAME); + + KUNIT_EXPECT_EQ(test, 0, result); + KUNIT_EXPECT_EQ(test, 0, be16_to_cpu(ustr.length)); + + /* Test input with length mismatch */ + result = hfsplus_asc2uni(&mock_sb->sb, + &ustr, HFSPLUS_MAX_STRLEN, "hello", 3, + HFS_REGULAR_NAME); + + KUNIT_EXPECT_EQ(test, 0, result); + check_unistr_content(test, &ustr, "hel"); + + /* Test with various printable ASCII characters */ + result = hfsplus_asc2uni(&mock_sb->sb, + &ustr, HFSPLUS_MAX_STRLEN, "ABC123!@#", 9, + HFS_REGULAR_NAME); + + KUNIT_EXPECT_EQ(test, 0, result); + check_unistr_content(test, &ustr, "ABC123!@#"); + + /* Test null character in the middle */ + result = hfsplus_asc2uni(&mock_sb->sb, + &ustr, HFSPLUS_MAX_STRLEN, test_str, 3, + HFS_REGULAR_NAME); + + KUNIT_EXPECT_EQ(test, 0, result); + KUNIT_EXPECT_EQ(test, 3, be16_to_cpu(ustr.length)); + KUNIT_EXPECT_EQ(test, 'a', be16_to_cpu(ustr.unicode[0])); + KUNIT_EXPECT_EQ(test, 0, be16_to_cpu(ustr.unicode[1])); + KUNIT_EXPECT_EQ(test, 'b', be16_to_cpu(ustr.unicode[2])); + + free_mock_sb(mock_sb); +} + +/* Test decomposition flag behavior */ +static void hfsplus_asc2uni_decompose_test(struct kunit *test) +{ + struct test_mock_sb *mock_sb; + struct test_mock_string_env *mock_env; + int result; + + mock_env = setup_mock_str_env(HFSPLUS_MAX_STRLEN + 1); + KUNIT_ASSERT_NOT_NULL(test, mock_env); + + mock_sb = setup_mock_sb(); + KUNIT_ASSERT_NOT_NULL(test, mock_sb); + + mock_sb->nls.char2uni = test_char2uni; + + /* Test with decomposition disabled (default) */ + clear_bit(HFSPLUS_SB_NODECOMPOSE, &mock_sb->sb_info.flags); + result = hfsplus_asc2uni(&mock_sb->sb, &mock_env->str1, + HFSPLUS_MAX_STRLEN, "test", 4, + HFS_REGULAR_NAME); + + KUNIT_EXPECT_EQ(test, 0, result); + check_unistr_content(test, &mock_env->str1, "test"); + + /* Test with decomposition enabled */ + set_bit(HFSPLUS_SB_NODECOMPOSE, &mock_sb->sb_info.flags); + result = hfsplus_asc2uni(&mock_sb->sb, &mock_env->str2, + HFSPLUS_MAX_STRLEN, "test", 4, + HFS_REGULAR_NAME); + + KUNIT_EXPECT_EQ(test, 0, result); + check_unistr_content(test, &mock_env->str2, "test"); + + /* For simple ASCII, both should produce the same result */ + KUNIT_EXPECT_EQ(test, + be16_to_cpu(mock_env->str1.length), + be16_to_cpu(mock_env->str2.length)); + + free_mock_str_env(mock_env); + free_mock_sb(mock_sb); +} + +/* Mock dentry for testing hfsplus_hash_dentry */ +static struct dentry test_dentry; + +static void setup_mock_dentry(struct super_block *sb) +{ + memset(&test_dentry, 0, sizeof(test_dentry)); + test_dentry.d_sb = sb; +} + +/* Helper function to create qstr */ +static void create_qstr(struct qstr *str, const char *name) +{ + str->name = name; + str->len = strlen(name); + str->hash = 0; /* Will be set by hash function */ +} + +/* Test hfsplus_hash_dentry basic functionality */ +static void hfsplus_hash_dentry_basic_test(struct kunit *test) +{ + struct test_mock_sb *mock_sb; + struct qstr str1, str2; + int result; + + mock_sb = setup_mock_sb(); + KUNIT_ASSERT_NOT_NULL(test, mock_sb); + + setup_mock_dentry(&mock_sb->sb); + mock_sb->nls.char2uni = test_char2uni; + + /* Test basic string hashing */ + create_qstr(&str1, "hello"); + result = hfsplus_hash_dentry(&test_dentry, &str1); + + KUNIT_EXPECT_EQ(test, 0, result); + KUNIT_EXPECT_NE(test, 0, str1.hash); + + /* Test that identical strings produce identical hashes */ + create_qstr(&str2, "hello"); + result = hfsplus_hash_dentry(&test_dentry, &str2); + + KUNIT_EXPECT_EQ(test, 0, result); + KUNIT_EXPECT_EQ(test, str1.hash, str2.hash); + + /* Test empty string */ + create_qstr(&str1, ""); + result = hfsplus_hash_dentry(&test_dentry, &str1); + + /* Empty string should still produce a hash */ + KUNIT_EXPECT_EQ(test, 0, result); + + /* Test single character */ + create_qstr(&str1, "A"); + result = hfsplus_hash_dentry(&test_dentry, &str1); + + KUNIT_EXPECT_EQ(test, 0, result); + KUNIT_EXPECT_NE(test, 0, str1.hash); + + free_mock_sb(mock_sb); +} + +/* Test case folding behavior in hash */ +static void hfsplus_hash_dentry_casefold_test(struct kunit *test) +{ + struct test_mock_sb *mock_sb; + struct qstr str1, str2; + int result; + + mock_sb = setup_mock_sb(); + KUNIT_ASSERT_NOT_NULL(test, mock_sb); + + setup_mock_dentry(&mock_sb->sb); + mock_sb->nls.char2uni = test_char2uni; + + /* Test with case folding disabled (default) */ + clear_bit(HFSPLUS_SB_CASEFOLD, &mock_sb->sb_info.flags); + + create_qstr(&str1, "Hello"); + result = hfsplus_hash_dentry(&test_dentry, &str1); + KUNIT_EXPECT_EQ(test, 0, result); + + create_qstr(&str2, "hello"); + result = hfsplus_hash_dentry(&test_dentry, &str2); + KUNIT_EXPECT_EQ(test, 0, result); + + /* + * Without case folding, different cases + * should produce different hashes + */ + KUNIT_EXPECT_NE(test, str1.hash, str2.hash); + + /* Test with case folding enabled */ + set_bit(HFSPLUS_SB_CASEFOLD, &mock_sb->sb_info.flags); + + create_qstr(&str1, "Hello"); + result = hfsplus_hash_dentry(&test_dentry, &str1); + KUNIT_EXPECT_EQ(test, 0, result); + + create_qstr(&str2, "hello"); + result = hfsplus_hash_dentry(&test_dentry, &str2); + KUNIT_EXPECT_EQ(test, 0, result); + + /* With case folding, different cases should produce same hash */ + KUNIT_EXPECT_EQ(test, str1.hash, str2.hash); + + /* Test mixed case */ + create_qstr(&str1, "HeLLo"); + result = hfsplus_hash_dentry(&test_dentry, &str1); + KUNIT_EXPECT_EQ(test, 0, result); + KUNIT_EXPECT_EQ(test, str1.hash, str2.hash); + + free_mock_sb(mock_sb); +} + +/* Test special character handling in hash */ +static void hfsplus_hash_dentry_special_chars_test(struct kunit *test) +{ + struct test_mock_sb *mock_sb; + struct qstr str1, str2; + int result; + + mock_sb = setup_mock_sb(); + KUNIT_ASSERT_NOT_NULL(test, mock_sb); + + setup_mock_dentry(&mock_sb->sb); + mock_sb->nls.char2uni = test_char2uni; + + /* Test colon conversion (: becomes /) */ + create_qstr(&str1, "file:name"); + result = hfsplus_hash_dentry(&test_dentry, &str1); + KUNIT_EXPECT_EQ(test, 0, result); + + create_qstr(&str2, "file/name"); + result = hfsplus_hash_dentry(&test_dentry, &str2); + KUNIT_EXPECT_EQ(test, 0, result); + + /* After conversion, these should produce the same hash */ + KUNIT_EXPECT_EQ(test, str1.hash, str2.hash); + + /* Test multiple special characters */ + create_qstr(&str1, ":::"); + result = hfsplus_hash_dentry(&test_dentry, &str1); + KUNIT_EXPECT_EQ(test, 0, result); + + create_qstr(&str2, "///"); + result = hfsplus_hash_dentry(&test_dentry, &str2); + KUNIT_EXPECT_EQ(test, 0, result); + + KUNIT_EXPECT_EQ(test, str1.hash, str2.hash); + + free_mock_sb(mock_sb); +} + +/* Test decomposition flag behavior in hash */ +static void hfsplus_hash_dentry_decompose_test(struct kunit *test) +{ + struct test_mock_sb *mock_sb; + struct qstr str1, str2; + int result; + + mock_sb = setup_mock_sb(); + KUNIT_ASSERT_NOT_NULL(test, mock_sb); + + setup_mock_dentry(&mock_sb->sb); + mock_sb->nls.char2uni = test_char2uni; + + /* Test with decomposition disabled (default) */ + clear_bit(HFSPLUS_SB_NODECOMPOSE, &mock_sb->sb_info.flags); + + create_qstr(&str1, "test"); + result = hfsplus_hash_dentry(&test_dentry, &str1); + KUNIT_EXPECT_EQ(test, 0, result); + + /* Test with decomposition enabled */ + set_bit(HFSPLUS_SB_NODECOMPOSE, &mock_sb->sb_info.flags); + + create_qstr(&str2, "test"); + result = hfsplus_hash_dentry(&test_dentry, &str2); + KUNIT_EXPECT_EQ(test, 0, result); + + /* + * For simple ASCII, decomposition shouldn't change + * the hash much but the function should still work correctly + */ + KUNIT_EXPECT_NE(test, 0, str2.hash); + + free_mock_sb(mock_sb); +} + +/* Test hash consistency and distribution */ +static void hfsplus_hash_dentry_consistency_test(struct kunit *test) +{ + struct test_mock_sb *mock_sb; + struct qstr str1, str2, str3; + unsigned long hash1; + int result; + + mock_sb = setup_mock_sb(); + KUNIT_ASSERT_NOT_NULL(test, mock_sb); + + setup_mock_dentry(&mock_sb->sb); + mock_sb->nls.char2uni = test_char2uni; + + /* Test that same string always produces same hash */ + create_qstr(&str1, "consistent"); + result = hfsplus_hash_dentry(&test_dentry, &str1); + KUNIT_EXPECT_EQ(test, 0, result); + hash1 = str1.hash; + + create_qstr(&str2, "consistent"); + result = hfsplus_hash_dentry(&test_dentry, &str2); + KUNIT_EXPECT_EQ(test, 0, result); + + KUNIT_EXPECT_EQ(test, hash1, str2.hash); + + /* Test that different strings produce different hashes */ + create_qstr(&str3, "different"); + result = hfsplus_hash_dentry(&test_dentry, &str3); + KUNIT_EXPECT_EQ(test, 0, result); + + KUNIT_EXPECT_NE(test, str1.hash, str3.hash); + + /* Test similar strings should have different hashes */ + create_qstr(&str1, "file1"); + result = hfsplus_hash_dentry(&test_dentry, &str1); + KUNIT_EXPECT_EQ(test, 0, result); + + create_qstr(&str2, "file2"); + result = hfsplus_hash_dentry(&test_dentry, &str2); + KUNIT_EXPECT_EQ(test, 0, result); + + KUNIT_EXPECT_NE(test, str1.hash, str2.hash); + + free_mock_sb(mock_sb); +} + +/* Test edge cases and boundary conditions */ +static void hfsplus_hash_dentry_edge_cases_test(struct kunit *test) +{ + struct test_mock_sb *mock_sb; + struct test_mock_string_env *mock_env; + struct qstr str; + int result; + + mock_env = setup_mock_str_env(HFSPLUS_MAX_STRLEN + 1); + KUNIT_ASSERT_NOT_NULL(test, mock_env); + + mock_sb = setup_mock_sb(); + KUNIT_ASSERT_NOT_NULL(test, mock_sb); + + setup_mock_dentry(&mock_sb->sb); + mock_sb->nls.char2uni = test_char2uni; + + /* Test very long filename */ + memset(mock_env->buf, 'a', mock_env->buf_size - 1); + mock_env->buf[mock_env->buf_size - 1] = '\0'; + + create_qstr(&str, mock_env->buf); + result = hfsplus_hash_dentry(&test_dentry, &str); + + KUNIT_EXPECT_EQ(test, 0, result); + KUNIT_EXPECT_NE(test, 0, str.hash); + + /* Test filename with all printable ASCII characters */ + create_qstr(&str, "!@#$%^&*()_+-=[]{}|;':\",./<>?"); + result = hfsplus_hash_dentry(&test_dentry, &str); + + KUNIT_EXPECT_EQ(test, 0, result); + KUNIT_EXPECT_NE(test, 0, str.hash); + + /* Test with embedded null (though not typical for filenames) */ + str.name = "file\0hidden"; + str.len = 11; /* Include the null and text after it */ + str.hash = 0; + result = hfsplus_hash_dentry(&test_dentry, &str); + + KUNIT_EXPECT_EQ(test, 0, result); + KUNIT_EXPECT_NE(test, 0, str.hash); + + free_mock_str_env(mock_env); + free_mock_sb(mock_sb); +} + +/* Test hfsplus_compare_dentry basic functionality */ +static void hfsplus_compare_dentry_basic_test(struct kunit *test) +{ + struct test_mock_sb *mock_sb; + struct qstr name; + int result; + + mock_sb = setup_mock_sb(); + KUNIT_ASSERT_NOT_NULL(test, mock_sb); + + setup_mock_dentry(&mock_sb->sb); + mock_sb->nls.char2uni = test_char2uni; + + /* Test identical strings */ + create_qstr(&name, "hello"); + result = hfsplus_compare_dentry(&test_dentry, 5, "hello", &name); + KUNIT_EXPECT_EQ(test, 0, result); + + /* Test different strings - lexicographic order */ + create_qstr(&name, "world"); + result = hfsplus_compare_dentry(&test_dentry, 5, "hello", &name); + KUNIT_EXPECT_LT(test, result, 0); /* "hello" < "world" */ + + result = hfsplus_compare_dentry(&test_dentry, 5, "world", &name); + KUNIT_EXPECT_EQ(test, 0, result); + + create_qstr(&name, "hello"); + result = hfsplus_compare_dentry(&test_dentry, 5, "world", &name); + KUNIT_EXPECT_GT(test, result, 0); /* "world" > "hello" */ + + /* Test empty strings */ + create_qstr(&name, ""); + result = hfsplus_compare_dentry(&test_dentry, 0, "", &name); + KUNIT_EXPECT_EQ(test, 0, result); + + /* Test one empty, one non-empty */ + create_qstr(&name, "test"); + result = hfsplus_compare_dentry(&test_dentry, 0, "", &name); + KUNIT_EXPECT_LT(test, result, 0); /* "" < "test" */ + + create_qstr(&name, ""); + result = hfsplus_compare_dentry(&test_dentry, 4, "test", &name); + KUNIT_EXPECT_GT(test, result, 0); /* "test" > "" */ + + free_mock_sb(mock_sb); +} + +/* Test case folding behavior in comparison */ +static void hfsplus_compare_dentry_casefold_test(struct kunit *test) +{ + struct test_mock_sb *mock_sb; + struct qstr name; + int result; + + mock_sb = setup_mock_sb(); + KUNIT_ASSERT_NOT_NULL(test, mock_sb); + + setup_mock_dentry(&mock_sb->sb); + mock_sb->nls.char2uni = test_char2uni; + + /* Test with case folding disabled (default) */ + clear_bit(HFSPLUS_SB_CASEFOLD, &mock_sb->sb_info.flags); + + create_qstr(&name, "hello"); + result = hfsplus_compare_dentry(&test_dentry, 5, "Hello", &name); + /* Case sensitive: "Hello" != "hello" */ + KUNIT_EXPECT_NE(test, 0, result); + + create_qstr(&name, "Hello"); + result = hfsplus_compare_dentry(&test_dentry, 5, "hello", &name); + /* Case sensitive: "hello" != "Hello" */ + KUNIT_EXPECT_NE(test, 0, result); + + /* Test with case folding enabled */ + set_bit(HFSPLUS_SB_CASEFOLD, &mock_sb->sb_info.flags); + + create_qstr(&name, "hello"); + result = hfsplus_compare_dentry(&test_dentry, 5, "Hello", &name); + /* Case insensitive: "Hello" == "hello" */ + KUNIT_EXPECT_EQ(test, 0, result); + + create_qstr(&name, "Hello"); + result = hfsplus_compare_dentry(&test_dentry, 5, "hello", &name); + /* Case insensitive: "hello" == "Hello" */ + KUNIT_EXPECT_EQ(test, 0, result); + + /* Test mixed case */ + create_qstr(&name, "TeSt"); + result = hfsplus_compare_dentry(&test_dentry, 4, "test", &name); + KUNIT_EXPECT_EQ(test, 0, result); + + create_qstr(&name, "test"); + result = hfsplus_compare_dentry(&test_dentry, 4, "TEST", &name); + KUNIT_EXPECT_EQ(test, 0, result); + + free_mock_sb(mock_sb); +} + +/* Test special character handling in comparison */ +static void hfsplus_compare_dentry_special_chars_test(struct kunit *test) +{ + struct test_mock_sb *mock_sb; + struct qstr name; + int result; + + mock_sb = setup_mock_sb(); + KUNIT_ASSERT_NOT_NULL(test, mock_sb); + + setup_mock_dentry(&mock_sb->sb); + mock_sb->nls.char2uni = test_char2uni; + + /* Test colon conversion (: becomes /) */ + create_qstr(&name, "file/name"); + result = hfsplus_compare_dentry(&test_dentry, 9, "file:name", &name); + /* "file:name" == "file/name" after conversion */ + KUNIT_EXPECT_EQ(test, 0, result); + + create_qstr(&name, "file:name"); + result = hfsplus_compare_dentry(&test_dentry, 9, "file/name", &name); + /* "file/name" == "file:name" after conversion */ + KUNIT_EXPECT_EQ(test, 0, result); + + /* Test multiple special characters */ + create_qstr(&name, "///"); + result = hfsplus_compare_dentry(&test_dentry, 3, ":::", &name); + KUNIT_EXPECT_EQ(test, 0, result); + + /* Test mixed special and regular characters */ + create_qstr(&name, "a/b:c"); + result = hfsplus_compare_dentry(&test_dentry, 5, "a:b/c", &name); + /* Both become "a/b/c" after conversion */ + KUNIT_EXPECT_EQ(test, 0, result); + + free_mock_sb(mock_sb); +} + +/* Test length differences */ +static void hfsplus_compare_dentry_length_test(struct kunit *test) +{ + struct test_mock_sb *mock_sb; + struct qstr name; + int result; + + mock_sb = setup_mock_sb(); + KUNIT_ASSERT_NOT_NULL(test, mock_sb); + + setup_mock_dentry(&mock_sb->sb); + mock_sb->nls.char2uni = test_char2uni; + + /* Test different lengths with common prefix */ + create_qstr(&name, "testing"); + result = hfsplus_compare_dentry(&test_dentry, 4, "test", &name); + KUNIT_EXPECT_LT(test, result, 0); /* "test" < "testing" */ + + create_qstr(&name, "test"); + result = hfsplus_compare_dentry(&test_dentry, 7, "testing", &name); + KUNIT_EXPECT_GT(test, result, 0); /* "testing" > "test" */ + + /* Test exact length match */ + create_qstr(&name, "exact"); + result = hfsplus_compare_dentry(&test_dentry, 5, "exact", &name); + KUNIT_EXPECT_EQ(test, 0, result); + + /* Test length parameter vs actual string content */ + create_qstr(&name, "hello"); + result = hfsplus_compare_dentry(&test_dentry, 3, "hel", &name); + KUNIT_EXPECT_LT(test, result, 0); /* "hel" < "hello" */ + + /* Test longer first string but shorter length parameter */ + create_qstr(&name, "hi"); + result = hfsplus_compare_dentry(&test_dentry, 2, "hello", &name); + /* "he" < "hi" (only first 2 chars compared) */ + KUNIT_EXPECT_LT(test, result, 0); + + free_mock_sb(mock_sb); +} + +/* Test decomposition flag behavior */ +static void hfsplus_compare_dentry_decompose_test(struct kunit *test) +{ + struct test_mock_sb *mock_sb; + struct qstr name; + int result; + + mock_sb = setup_mock_sb(); + KUNIT_ASSERT_NOT_NULL(test, mock_sb); + + setup_mock_dentry(&mock_sb->sb); + mock_sb->nls.char2uni = test_char2uni; + + /* Test with decomposition disabled (default) */ + clear_bit(HFSPLUS_SB_NODECOMPOSE, &mock_sb->sb_info.flags); + + create_qstr(&name, "test"); + result = hfsplus_compare_dentry(&test_dentry, 4, "test", &name); + KUNIT_EXPECT_EQ(test, 0, result); + + /* Test with decomposition enabled */ + set_bit(HFSPLUS_SB_NODECOMPOSE, &mock_sb->sb_info.flags); + + create_qstr(&name, "test"); + result = hfsplus_compare_dentry(&test_dentry, 4, "test", &name); + KUNIT_EXPECT_EQ(test, 0, result); + + /* For simple ASCII, decomposition shouldn't affect the result */ + create_qstr(&name, "different"); + result = hfsplus_compare_dentry(&test_dentry, 4, "test", &name); + KUNIT_EXPECT_NE(test, 0, result); + + free_mock_sb(mock_sb); +} + +/* Test edge cases and boundary conditions */ +static void hfsplus_compare_dentry_edge_cases_test(struct kunit *test) +{ + struct test_mock_sb *mock_sb; + struct qstr name; + char *long_str; + char *long_str2; + u32 str_size = HFSPLUS_MAX_STRLEN + 1; + struct qstr null_name = { + .name = "a\0b", + .len = 3, + .hash = 0 + }; + int result; + + mock_sb = setup_mock_sb(); + KUNIT_ASSERT_NOT_NULL(test, mock_sb); + + setup_mock_dentry(&mock_sb->sb); + mock_sb->nls.char2uni = test_char2uni; + + long_str = kzalloc(str_size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, long_str); + + long_str2 = kzalloc(str_size, GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, long_str2); + + /* Test very long strings */ + memset(long_str, 'a', str_size - 1); + long_str[str_size - 1] = '\0'; + + create_qstr(&name, long_str); + result = hfsplus_compare_dentry(&test_dentry, str_size - 1, + long_str, &name); + KUNIT_EXPECT_EQ(test, 0, result); + + /* Test with difference at the end of long strings */ + memset(long_str2, 'a', str_size - 1); + long_str2[str_size - 1] = '\0'; + long_str2[str_size - 2] = 'b'; + create_qstr(&name, long_str2); + result = hfsplus_compare_dentry(&test_dentry, str_size - 1, + long_str, &name); + KUNIT_EXPECT_LT(test, result, 0); /* 'a' < 'b' */ + + /* Test single character differences */ + create_qstr(&name, "b"); + result = hfsplus_compare_dentry(&test_dentry, 1, "a", &name); + KUNIT_EXPECT_LT(test, result, 0); /* 'a' < 'b' */ + + create_qstr(&name, "a"); + result = hfsplus_compare_dentry(&test_dentry, 1, "b", &name); + KUNIT_EXPECT_GT(test, result, 0); /* 'b' > 'a' */ + + /* Test with null characters in the middle */ + result = hfsplus_compare_dentry(&test_dentry, 3, "a\0b", &null_name); + KUNIT_EXPECT_EQ(test, 0, result); + + /* Test all printable ASCII characters */ + create_qstr(&name, "!@#$%^&*()"); + result = hfsplus_compare_dentry(&test_dentry, 10, "!@#$%^&*()", &name); + KUNIT_EXPECT_EQ(test, 0, result); + + kfree(long_str); + kfree(long_str2); + free_mock_sb(mock_sb); +} + +/* Test combined flag behaviors */ +static void hfsplus_compare_dentry_combined_flags_test(struct kunit *test) +{ + struct test_mock_sb *mock_sb; + struct qstr name; + int result; + + mock_sb = setup_mock_sb(); + KUNIT_ASSERT_NOT_NULL(test, mock_sb); + + setup_mock_dentry(&mock_sb->sb); + mock_sb->nls.char2uni = test_char2uni; + + /* Test with both casefold and decompose enabled */ + set_bit(HFSPLUS_SB_CASEFOLD, &mock_sb->sb_info.flags); + set_bit(HFSPLUS_SB_NODECOMPOSE, &mock_sb->sb_info.flags); + + create_qstr(&name, "hello"); + result = hfsplus_compare_dentry(&test_dentry, 5, "HELLO", &name); + KUNIT_EXPECT_EQ(test, 0, result); + + /* Test special chars with case folding */ + create_qstr(&name, "File/Name"); + result = hfsplus_compare_dentry(&test_dentry, 9, "file:name", &name); + KUNIT_EXPECT_EQ(test, 0, result); + + /* Test with both flags disabled */ + clear_bit(HFSPLUS_SB_CASEFOLD, &mock_sb->sb_info.flags); + clear_bit(HFSPLUS_SB_NODECOMPOSE, &mock_sb->sb_info.flags); + + create_qstr(&name, "hello"); + result = hfsplus_compare_dentry(&test_dentry, 5, "HELLO", &name); + KUNIT_EXPECT_NE(test, 0, result); /* Case sensitive */ + + /* But special chars should still be converted */ + create_qstr(&name, "file/name"); + result = hfsplus_compare_dentry(&test_dentry, 9, "file:name", &name); + KUNIT_EXPECT_EQ(test, 0, result); + + free_mock_sb(mock_sb); +} + +static struct kunit_case hfsplus_unicode_test_cases[] = { + KUNIT_CASE(hfsplus_strcasecmp_test), + KUNIT_CASE(hfsplus_strcmp_test), + KUNIT_CASE(hfsplus_unicode_edge_cases_test), + KUNIT_CASE(hfsplus_unicode_boundary_test), + KUNIT_CASE(hfsplus_uni2asc_basic_test), + KUNIT_CASE(hfsplus_uni2asc_special_chars_test), + KUNIT_CASE(hfsplus_uni2asc_buffer_test), + KUNIT_CASE(hfsplus_uni2asc_corrupted_test), + KUNIT_CASE(hfsplus_uni2asc_edge_cases_test), + KUNIT_CASE(hfsplus_asc2uni_basic_test), + KUNIT_CASE(hfsplus_asc2uni_special_chars_test), + KUNIT_CASE(hfsplus_asc2uni_buffer_limits_test), + KUNIT_CASE(hfsplus_asc2uni_edge_cases_test), + KUNIT_CASE(hfsplus_asc2uni_decompose_test), + KUNIT_CASE(hfsplus_hash_dentry_basic_test), + KUNIT_CASE(hfsplus_hash_dentry_casefold_test), + KUNIT_CASE(hfsplus_hash_dentry_special_chars_test), + KUNIT_CASE(hfsplus_hash_dentry_decompose_test), + KUNIT_CASE(hfsplus_hash_dentry_consistency_test), + KUNIT_CASE(hfsplus_hash_dentry_edge_cases_test), + KUNIT_CASE(hfsplus_compare_dentry_basic_test), + KUNIT_CASE(hfsplus_compare_dentry_casefold_test), + KUNIT_CASE(hfsplus_compare_dentry_special_chars_test), + KUNIT_CASE(hfsplus_compare_dentry_length_test), + KUNIT_CASE(hfsplus_compare_dentry_decompose_test), + KUNIT_CASE(hfsplus_compare_dentry_edge_cases_test), + KUNIT_CASE(hfsplus_compare_dentry_combined_flags_test), + {} +}; + +static struct kunit_suite hfsplus_unicode_test_suite = { + .name = "hfsplus_unicode", + .test_cases = hfsplus_unicode_test_cases, +}; + +kunit_test_suite(hfsplus_unicode_test_suite); + +MODULE_DESCRIPTION("KUnit tests for HFS+ Unicode string operations"); +MODULE_LICENSE("GPL"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/fs/hfsplus/wrapper.c b/fs/hfsplus/wrapper.c new file mode 100644 index 000000000..30cf4fe78 --- /dev/null +++ b/fs/hfsplus/wrapper.c @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hfsplus/wrapper.c + * + * Copyright (C) 2001 + * Brad Boyer (flar@allandria.com) + * (C) 2003 Ardis Technologies + * + * Handling of HFS wrappers around HFS+ volumes + */ + +#include +#include +#include +#include + +#include "hfsplus_fs.h" +#include "hfsplus_raw.h" + +struct hfsplus_wd { + u32 ablk_size; + u16 ablk_start; + u16 embed_start; + u16 embed_count; +}; + +/** + * hfsplus_submit_bio - Perform block I/O + * @sb: super block of volume for I/O + * @sector: block to read or write, for blocks of HFSPLUS_SECTOR_SIZE bytes + * @buf: buffer for I/O + * @data: output pointer for location of requested data + * @opf: I/O operation type and flags + * + * The unit of I/O is hfsplus_min_io_size(sb), which may be bigger than + * HFSPLUS_SECTOR_SIZE, and @buf must be sized accordingly. On reads + * @data will return a pointer to the start of the requested sector, + * which may not be the same location as @buf. + * + * If @sector is not aligned to the bdev logical block size it will + * be rounded down. For writes this means that @buf should contain data + * that starts at the rounded-down address. As long as the data was + * read using hfsplus_submit_bio() and the same buffer is used things + * will work correctly. + * + * Returns: %0 on success else -errno code + */ +int hfsplus_submit_bio(struct super_block *sb, sector_t sector, + void *buf, void **data, blk_opf_t opf) +{ + u64 io_size = hfsplus_min_io_size(sb); + loff_t start = (loff_t)sector << HFSPLUS_SECTOR_SHIFT; + int offset = start & (io_size - 1); + + if ((opf & REQ_OP_MASK) != REQ_OP_WRITE && data) + *data = (u8 *)buf + offset; + + /* + * Align sector to hardware sector size and find offset. We assume that + * io_size is a power of two, which _should_ be true. + */ + sector &= ~((io_size >> HFSPLUS_SECTOR_SHIFT) - 1); + return bdev_rw_virt(sb->s_bdev, sector, buf, io_size, opf); +} + +static int hfsplus_read_mdb(void *bufptr, struct hfsplus_wd *wd) +{ + u32 extent; + u16 attrib; + __be16 sig; + + sig = *(__be16 *)(bufptr + HFSP_WRAPOFF_EMBEDSIG); + if (sig != cpu_to_be16(HFSPLUS_VOLHEAD_SIG) && + sig != cpu_to_be16(HFSPLUS_VOLHEAD_SIGX)) + return 0; + + attrib = be16_to_cpu(*(__be16 *)(bufptr + HFSP_WRAPOFF_ATTRIB)); + if (!(attrib & HFSP_WRAP_ATTRIB_SLOCK) || + !(attrib & HFSP_WRAP_ATTRIB_SPARED)) + return 0; + + wd->ablk_size = + be32_to_cpu(*(__be32 *)(bufptr + HFSP_WRAPOFF_ABLKSIZE)); + if (wd->ablk_size < HFSPLUS_SECTOR_SIZE) + return 0; + if (wd->ablk_size % HFSPLUS_SECTOR_SIZE) + return 0; + wd->ablk_start = + be16_to_cpu(*(__be16 *)(bufptr + HFSP_WRAPOFF_ABLKSTART)); + + extent = get_unaligned_be32(bufptr + HFSP_WRAPOFF_EMBEDEXT); + wd->embed_start = (extent >> 16) & 0xFFFF; + wd->embed_count = extent & 0xFFFF; + + return 1; +} + +static int hfsplus_get_last_session(struct super_block *sb, + sector_t *start, sector_t *size) +{ + struct cdrom_device_info *cdi = disk_to_cdi(sb->s_bdev->bd_disk); + + /* default values */ + *start = 0; + *size = bdev_nr_sectors(sb->s_bdev); + + if (HFSPLUS_SB(sb)->session >= 0) { + struct cdrom_tocentry te; + + if (!cdi) + return -EINVAL; + + te.cdte_track = HFSPLUS_SB(sb)->session; + te.cdte_format = CDROM_LBA; + if (cdrom_read_tocentry(cdi, &te) || + (te.cdte_ctrl & CDROM_DATA_TRACK) != 4) { + pr_err("invalid session number or type of track\n"); + return -EINVAL; + } + *start = (sector_t)te.cdte_addr.lba << 2; + } else if (cdi) { + struct cdrom_multisession ms_info; + + ms_info.addr_format = CDROM_LBA; + if (cdrom_multisession(cdi, &ms_info) == 0 && ms_info.xa_flag) + *start = (sector_t)ms_info.addr.lba << 2; + } + + return 0; +} + +/* Find the volume header and fill in some minimum bits in superblock */ +/* Takes in super block, returns true if good data read */ +int hfsplus_read_wrapper(struct super_block *sb) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(sb); + struct hfsplus_wd wd; + sector_t part_start, part_size; + u32 blocksize; + int error = 0; + + error = -EINVAL; + blocksize = sb_min_blocksize(sb, HFSPLUS_SECTOR_SIZE); + if (!blocksize) + goto out; + + sbi->min_io_size = blocksize; + + if (hfsplus_get_last_session(sb, &part_start, &part_size)) + goto out; + + error = -ENOMEM; + sbi->s_vhdr_buf = kmalloc(hfsplus_min_io_size(sb), GFP_KERNEL); + if (!sbi->s_vhdr_buf) + goto out; + sbi->s_backup_vhdr_buf = kmalloc(hfsplus_min_io_size(sb), GFP_KERNEL); + if (!sbi->s_backup_vhdr_buf) + goto out_free_vhdr; + +reread: + error = hfsplus_submit_bio(sb, part_start + HFSPLUS_VOLHEAD_SECTOR, + sbi->s_vhdr_buf, (void **)&sbi->s_vhdr, + REQ_OP_READ); + if (error) + goto out_free_backup_vhdr; + + error = -EINVAL; + switch (sbi->s_vhdr->signature) { + case cpu_to_be16(HFSPLUS_VOLHEAD_SIGX): + set_bit(HFSPLUS_SB_HFSX, &sbi->flags); + fallthrough; + case cpu_to_be16(HFSPLUS_VOLHEAD_SIG): + break; + case cpu_to_be16(HFSP_WRAP_MAGIC): + if (!hfsplus_read_mdb(sbi->s_vhdr, &wd)) + goto out_free_backup_vhdr; + wd.ablk_size >>= HFSPLUS_SECTOR_SHIFT; + part_start += (sector_t)wd.ablk_start + + (sector_t)wd.embed_start * wd.ablk_size; + part_size = (sector_t)wd.embed_count * wd.ablk_size; + goto reread; + default: + /* + * Check for a partition block. + * + * (should do this only for cdrom/loop though) + */ + if (hfs_part_find(sb, &part_start, &part_size)) + goto out_free_backup_vhdr; + goto reread; + } + + error = hfsplus_submit_bio(sb, part_start + part_size - 2, + sbi->s_backup_vhdr_buf, + (void **)&sbi->s_backup_vhdr, REQ_OP_READ); + if (error) + goto out_free_backup_vhdr; + + error = -EINVAL; + if (sbi->s_backup_vhdr->signature != sbi->s_vhdr->signature) { + pr_warn("invalid secondary volume header\n"); + goto out_free_backup_vhdr; + } + + blocksize = be32_to_cpu(sbi->s_vhdr->blocksize); + + /* + * Block size must be at least as large as a sector and a multiple of 2. + */ + if (blocksize < HFSPLUS_SECTOR_SIZE || ((blocksize - 1) & blocksize)) + goto out_free_backup_vhdr; + sbi->alloc_blksz = blocksize; + sbi->alloc_blksz_shift = ilog2(blocksize); + blocksize = min_t(u32, sbi->alloc_blksz, PAGE_SIZE); + + /* + * Align block size to block offset. + */ + while (part_start & ((blocksize >> HFSPLUS_SECTOR_SHIFT) - 1)) + blocksize >>= 1; + + if (sb_set_blocksize(sb, blocksize) != blocksize) { + pr_err("unable to set blocksize to %u!\n", blocksize); + goto out_free_backup_vhdr; + } + + sbi->blockoffset = + part_start >> (sb->s_blocksize_bits - HFSPLUS_SECTOR_SHIFT); + sbi->part_start = part_start; + sbi->sect_count = part_size; + sbi->fs_shift = sbi->alloc_blksz_shift - sb->s_blocksize_bits; + return 0; + +out_free_backup_vhdr: + kfree(sbi->s_backup_vhdr_buf); +out_free_vhdr: + kfree(sbi->s_vhdr_buf); +out: + return error; +} diff --git a/fs/hfsplus/xattr.c b/fs/hfsplus/xattr.c new file mode 100644 index 000000000..21a1c196c --- /dev/null +++ b/fs/hfsplus/xattr.c @@ -0,0 +1,1036 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hfsplus/xattr.c + * + * Vyacheslav Dubeyko + * + * Logic of processing extended attributes + */ + +#include "hfsplus_fs.h" +#include +#include "xattr.h" + +static int hfsplus_removexattr(struct inode *inode, const char *name); + +const struct xattr_handler * const hfsplus_xattr_handlers[] = { + &hfsplus_xattr_osx_handler, + &hfsplus_xattr_user_handler, + &hfsplus_xattr_trusted_handler, + &hfsplus_xattr_security_handler, + NULL +}; + +static int strcmp_xattr_finder_info(const char *name) +{ + if (name) { + return strncmp(name, HFSPLUS_XATTR_FINDER_INFO_NAME, + sizeof(HFSPLUS_XATTR_FINDER_INFO_NAME)); + } + return -1; +} + +static int strcmp_xattr_acl(const char *name) +{ + if (name) { + return strncmp(name, HFSPLUS_XATTR_ACL_NAME, + sizeof(HFSPLUS_XATTR_ACL_NAME)); + } + return -1; +} + +static bool is_known_namespace(const char *name) +{ + if (strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN) && + strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN) && + strncmp(name, XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN) && + strncmp(name, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN)) + return false; + + return true; +} + +static u32 hfsplus_init_header_node(struct inode *attr_file, + u32 clump_size, + char *buf, u16 node_size) +{ + struct hfs_bnode_desc *desc; + struct hfs_btree_header_rec *head; + u16 offset; + __be16 *rec_offsets; + u32 hdr_node_map_rec_bits; + u32 map_nodes = 0; + char *bmp; + u32 used_nodes; + u32 used_bmp_bytes; + u64 tmp; + + hfs_dbg("clump %u, node_size %u\n", + clump_size, node_size); + + /* The end of the node contains list of record offsets */ + rec_offsets = (__be16 *)(buf + node_size); + + desc = (struct hfs_bnode_desc *)buf; + desc->type = HFS_NODE_HEADER; + desc->num_recs = cpu_to_be16(HFSPLUS_BTREE_HDR_NODE_RECS_COUNT); + offset = sizeof(struct hfs_bnode_desc); + *--rec_offsets = cpu_to_be16(offset); + + head = (struct hfs_btree_header_rec *)(buf + offset); + head->node_size = cpu_to_be16(node_size); + tmp = i_size_read(attr_file); + do_div(tmp, node_size); + head->node_count = cpu_to_be32(tmp); + head->free_nodes = cpu_to_be32(be32_to_cpu(head->node_count) - 1); + head->clump_size = cpu_to_be32(clump_size); + head->attributes |= cpu_to_be32(HFS_TREE_BIGKEYS | HFS_TREE_VARIDXKEYS); + head->max_key_len = cpu_to_be16(HFSPLUS_ATTR_KEYLEN - sizeof(u16)); + offset += sizeof(struct hfs_btree_header_rec); + *--rec_offsets = cpu_to_be16(offset); + offset += HFSPLUS_BTREE_HDR_USER_BYTES; + *--rec_offsets = cpu_to_be16(offset); + + hdr_node_map_rec_bits = 8 * (node_size - offset - (4 * sizeof(u16))); + if (be32_to_cpu(head->node_count) > hdr_node_map_rec_bits) { + u32 map_node_bits; + + desc->next = cpu_to_be32(be32_to_cpu(head->leaf_tail) + 1); + map_node_bits = 8 * (node_size - sizeof(struct hfs_bnode_desc) - + (2 * sizeof(u16)) - 2); + map_nodes = (be32_to_cpu(head->node_count) - + hdr_node_map_rec_bits + + (map_node_bits - 1)) / map_node_bits; + be32_add_cpu(&head->free_nodes, 0 - map_nodes); + } + + bmp = buf + offset; + used_nodes = + be32_to_cpu(head->node_count) - be32_to_cpu(head->free_nodes); + used_bmp_bytes = used_nodes / 8; + if (used_bmp_bytes) { + memset(bmp, 0xFF, used_bmp_bytes); + bmp += used_bmp_bytes; + used_nodes %= 8; + } + *bmp = ~(0xFF >> used_nodes); + offset += hdr_node_map_rec_bits / 8; + *--rec_offsets = cpu_to_be16(offset); + + return map_nodes; +} + +/* + * Initialize a map node buffer. Map nodes have a single bitmap record. + */ +static void hfsplus_init_map_node(u8 *buf, u16 node_size, u32 next_node) +{ + struct hfs_bnode_desc *desc; + __be16 *rec_offsets; + size_t rec_size = sizeof(__be16); + u16 offset; + + memset(buf, 0, node_size); + + desc = (struct hfs_bnode_desc *)buf; + desc->type = HFS_NODE_MAP; + desc->num_recs = cpu_to_be16(1); + desc->next = cpu_to_be32(next_node); + + /* + * A map node consists of the node descriptor and a single + * map record. The map record is a continuation of the map + * record contained in the header node. The size of the map + * record is the size of the node, minus the size of + * the node descriptor (14 bytes), minus the size of two offsets + * (4 bytes), minus two bytes of free space. That is, the size of + * the map record is the size of the node minus 20 bytes; + * this keeps the length of the map record an even multiple of 4 bytes. + * The start of the map record is not aligned to a 4-byte boundary: + * it starts immediately after the node descriptor + * (at an offset of 14 bytes). + * + * Two record offsets stored at the end of the node: + * record[1] = start of record 0 -> sizeof(hfs_bnode_desc) + * record[2] = start of free space + */ + rec_offsets = (__be16 *)(buf + node_size); + + /* record #1 */ + offset = sizeof(struct hfs_bnode_desc); + *--rec_offsets = cpu_to_be16(offset); + + /* record #2 */ + offset = node_size; + offset -= (u16)HFSPLUS_BTREE_MAP_NODE_RECS_COUNT * rec_size; + offset -= HFSPLUS_BTREE_MAP_NODE_RESERVED_BYTES; + *--rec_offsets = cpu_to_be16(offset); +} + +static inline +int hfsplus_write_attributes_file_node(struct inode *attr_file, char *buf, + u16 node_size, int *index) +{ + struct address_space *mapping; + struct page *page; + void *kaddr; + u32 written = 0; + + mapping = attr_file->i_mapping; + + for (; written < node_size; (*index)++, written += PAGE_SIZE) { + page = read_mapping_page(mapping, *index, NULL); + if (IS_ERR(page)) + return PTR_ERR(page); + + kaddr = kmap_local_page(page); + memcpy(kaddr, buf + written, + min_t(size_t, PAGE_SIZE, node_size - written)); + kunmap_local(kaddr); + + set_page_dirty(page); + put_page(page); + } + + return 0; +} + +static int hfsplus_create_attributes_file(struct super_block *sb) +{ + struct hfsplus_sb_info *sbi = HFSPLUS_SB(sb); + struct inode *attr_file; + struct hfsplus_inode_info *hip; + struct hfs_bnode_desc *desc; + u32 clump_size; + u16 node_size = HFSPLUS_ATTR_TREE_NODE_SIZE; + u32 next_node; + u32 map_node_idx; + u32 map_nodes; + char *buf; + int index; + int old_state = HFSPLUS_EMPTY_ATTR_TREE; + int err = 0; + + hfs_dbg("ino %d\n", HFSPLUS_ATTR_CNID); + +check_attr_tree_state_again: + switch (atomic_read(&sbi->attr_tree_state)) { + case HFSPLUS_EMPTY_ATTR_TREE: + if (old_state != atomic_cmpxchg(&sbi->attr_tree_state, + old_state, + HFSPLUS_CREATING_ATTR_TREE)) + goto check_attr_tree_state_again; + break; + case HFSPLUS_CREATING_ATTR_TREE: + /* + * This state means that another thread is in process + * of AttributesFile creation. Theoretically, it is + * possible to be here. But really __setxattr() method + * first of all calls hfs_find_init() for lookup in + * B-tree of CatalogFile. This method locks mutex of + * CatalogFile's B-tree. As a result, if some thread + * is inside AttributedFile creation operation then + * another threads will be waiting unlocking of + * CatalogFile's B-tree's mutex. However, if code will + * change then we will return error code (-EAGAIN) from + * here. Really, it means that first try to set of xattr + * fails with error but second attempt will have success. + */ + return -EAGAIN; + case HFSPLUS_VALID_ATTR_TREE: + return 0; + case HFSPLUS_FAILED_ATTR_TREE: + return -EOPNOTSUPP; + default: + BUG(); + } + + attr_file = hfsplus_iget(sb, HFSPLUS_ATTR_CNID); + if (IS_ERR(attr_file)) { + pr_err("failed to load attributes file\n"); + return PTR_ERR(attr_file); + } + + if (i_size_read(attr_file) != 0) { + err = -EIO; + pr_err("detected inconsistent attributes file, running fsck.hfsplus is recommended.\n"); + goto end_attr_file_creation; + } + + hip = HFSPLUS_I(attr_file); + + clump_size = hfsplus_calc_btree_clump_size(sb->s_blocksize, + node_size, + sbi->sect_count, + HFSPLUS_ATTR_CNID); + + mutex_lock(&hip->extents_lock); + hip->clump_blocks = clump_size >> sbi->alloc_blksz_shift; + mutex_unlock(&hip->extents_lock); + + if (sbi->free_blocks <= (hip->clump_blocks << 1)) { + err = -ENOSPC; + goto end_attr_file_creation; + } + + while (hip->alloc_blocks < hip->clump_blocks) { + err = hfsplus_file_extend(attr_file, true); + if (unlikely(err)) { + pr_err("failed to extend attributes file\n"); + goto end_attr_file_creation; + } + hip->phys_size = attr_file->i_size = + (loff_t)hip->alloc_blocks << sbi->alloc_blksz_shift; + hip->fs_blocks = hip->alloc_blocks << sbi->fs_shift; + inode_set_bytes(attr_file, attr_file->i_size); + } + + buf = kzalloc(node_size, GFP_NOFS); + if (!buf) { + err = -ENOMEM; + goto end_attr_file_creation; + } + + map_nodes = hfsplus_init_header_node(attr_file, clump_size, buf, node_size); + + desc = (struct hfs_bnode_desc *)buf; + next_node = be32_to_cpu(desc->next); + + index = 0; + + err = hfsplus_write_attributes_file_node(attr_file, buf, + node_size, &index); + if (unlikely(err)) + goto failed_header_node_init; + + for (map_node_idx = 0; map_node_idx < map_nodes; map_node_idx++) { + if (next_node >= map_nodes) + next_node = 0; + + hfsplus_init_map_node(buf, node_size, next_node); + + err = hfsplus_write_attributes_file_node(attr_file, buf, + node_size, &index); + if (unlikely(err)) + goto failed_header_node_init; + + next_node++; + } + + hfsplus_mark_inode_dirty(attr_file, HFSPLUS_I_ATTR_DIRTY); + + sbi->attr_tree = hfs_btree_open(sb, HFSPLUS_ATTR_CNID); + if (!sbi->attr_tree) + pr_err("failed to load attributes file\n"); + +failed_header_node_init: + kfree(buf); + +end_attr_file_creation: + iput(attr_file); + + if (!err) + atomic_set(&sbi->attr_tree_state, HFSPLUS_VALID_ATTR_TREE); + else if (err == -ENOSPC) + atomic_set(&sbi->attr_tree_state, HFSPLUS_EMPTY_ATTR_TREE); + else + atomic_set(&sbi->attr_tree_state, HFSPLUS_FAILED_ATTR_TREE); + + return err; +} + +static inline +bool is_xattr_operation_supported(struct inode *inode) +{ + if (HFSPLUS_IS_RSRC(inode)) + return false; + + return true; +} + +int __hfsplus_setxattr(struct inode *inode, const char *name, + const void *value, size_t size, int flags) +{ + int err; + struct hfs_find_data cat_fd; + hfsplus_cat_entry entry; + u16 cat_entry_flags, cat_entry_type; + u16 folder_finderinfo_len = sizeof(DInfo) + sizeof(DXInfo); + u16 file_finderinfo_len = sizeof(FInfo) + sizeof(FXInfo); + + hfs_dbg("ino %llu, name %s, value %p, size %zu\n", + inode->i_ino, name ? name : NULL, + value, size); + + if (!is_xattr_operation_supported(inode)) + return -EOPNOTSUPP; + + if (value == NULL) + return hfsplus_removexattr(inode, name); + + err = hfs_find_init(HFSPLUS_SB(inode->i_sb)->cat_tree, &cat_fd); + if (err) { + pr_err("can't init xattr find struct\n"); + return err; + } + + err = hfsplus_find_cat(inode->i_sb, inode->i_ino, &cat_fd); + if (err) { + pr_err("catalog searching failed\n"); + goto end_setxattr; + } + + if (!strcmp_xattr_finder_info(name)) { + if (flags & XATTR_CREATE) { + pr_err("xattr exists yet\n"); + err = -EOPNOTSUPP; + goto end_setxattr; + } + hfs_bnode_read(cat_fd.bnode, &entry, cat_fd.entryoffset, + sizeof(hfsplus_cat_entry)); + if (be16_to_cpu(entry.type) == HFSPLUS_FOLDER) { + if (size == folder_finderinfo_len) { + memcpy(&entry.folder.info, value, + folder_finderinfo_len); + hfs_bnode_write(cat_fd.bnode, &entry, + cat_fd.entryoffset, + sizeof(struct hfsplus_cat_folder)); + hfsplus_mark_inode_dirty( + HFSPLUS_CAT_TREE_I(inode->i_sb), + HFSPLUS_I_CAT_DIRTY); + hfsplus_mark_inode_dirty(inode, + HFSPLUS_I_CAT_DIRTY); + } else { + err = -ERANGE; + goto end_setxattr; + } + } else if (be16_to_cpu(entry.type) == HFSPLUS_FILE) { + if (size == file_finderinfo_len) { + memcpy(&entry.file.info, value, + file_finderinfo_len); + hfs_bnode_write(cat_fd.bnode, &entry, + cat_fd.entryoffset, + sizeof(struct hfsplus_cat_file)); + hfsplus_mark_inode_dirty( + HFSPLUS_CAT_TREE_I(inode->i_sb), + HFSPLUS_I_CAT_DIRTY); + hfsplus_mark_inode_dirty(inode, + HFSPLUS_I_CAT_DIRTY); + } else { + err = -ERANGE; + goto end_setxattr; + } + } else { + err = -EOPNOTSUPP; + goto end_setxattr; + } + goto end_setxattr; + } + + if (!HFSPLUS_SB(inode->i_sb)->attr_tree) { + err = hfsplus_create_attributes_file(inode->i_sb); + if (unlikely(err)) + goto end_setxattr; + } + + if (hfsplus_attr_exists(inode, name)) { + if (flags & XATTR_CREATE) { + pr_err("xattr exists yet\n"); + err = -EOPNOTSUPP; + goto end_setxattr; + } + err = hfsplus_replace_attr(inode, name, value, size); + if (err) { + hfs_dbg("unable to replace xattr: err %d\n", err); + goto end_setxattr; + } + } else { + if (flags & XATTR_REPLACE) { + pr_err("cannot replace xattr\n"); + err = -EOPNOTSUPP; + goto end_setxattr; + } + err = hfsplus_create_attr(inode, name, value, size); + if (err) { + hfs_dbg("unable to store value: err %d\n", err); + goto end_setxattr; + } + } + + cat_entry_type = hfs_bnode_read_u16(cat_fd.bnode, cat_fd.entryoffset); + if (cat_entry_type == HFSPLUS_FOLDER) { + cat_entry_flags = hfs_bnode_read_u16(cat_fd.bnode, + cat_fd.entryoffset + + offsetof(struct hfsplus_cat_folder, flags)); + cat_entry_flags |= HFSPLUS_XATTR_EXISTS; + if (!strcmp_xattr_acl(name)) + cat_entry_flags |= HFSPLUS_ACL_EXISTS; + hfs_bnode_write_u16(cat_fd.bnode, cat_fd.entryoffset + + offsetof(struct hfsplus_cat_folder, flags), + cat_entry_flags); + hfsplus_mark_inode_dirty(HFSPLUS_CAT_TREE_I(inode->i_sb), + HFSPLUS_I_CAT_DIRTY); + hfsplus_mark_inode_dirty(inode, HFSPLUS_I_CAT_DIRTY); + } else if (cat_entry_type == HFSPLUS_FILE) { + cat_entry_flags = hfs_bnode_read_u16(cat_fd.bnode, + cat_fd.entryoffset + + offsetof(struct hfsplus_cat_file, flags)); + cat_entry_flags |= HFSPLUS_XATTR_EXISTS; + if (!strcmp_xattr_acl(name)) + cat_entry_flags |= HFSPLUS_ACL_EXISTS; + hfs_bnode_write_u16(cat_fd.bnode, cat_fd.entryoffset + + offsetof(struct hfsplus_cat_file, flags), + cat_entry_flags); + hfsplus_mark_inode_dirty(HFSPLUS_CAT_TREE_I(inode->i_sb), + HFSPLUS_I_CAT_DIRTY); + hfsplus_mark_inode_dirty(inode, HFSPLUS_I_CAT_DIRTY); + } else { + pr_err("invalid catalog entry type\n"); + err = -EIO; + goto end_setxattr; + } + + inode_set_ctime_current(inode); + +end_setxattr: + hfs_find_exit(&cat_fd); + hfs_dbg("finished: res %d\n", err); + return err; +} + +static size_t name_len(const char *xattr_name, size_t xattr_name_len) +{ + size_t len = xattr_name_len + 1; + + if (!is_known_namespace(xattr_name)) + len += XATTR_MAC_OSX_PREFIX_LEN; + + return len; +} + +static ssize_t copy_name(char *buffer, const char *xattr_name, size_t name_len) +{ + ssize_t len; + + memset(buffer, 0, name_len); + + if (!is_known_namespace(xattr_name)) { + len = scnprintf(buffer, name_len + XATTR_MAC_OSX_PREFIX_LEN, + "%s%s", XATTR_MAC_OSX_PREFIX, xattr_name); + } else { + len = strscpy(buffer, xattr_name, name_len + 1); + if (len < 0) { + pr_err("fail to copy name: err %zd\n", len); + len = 0; + } + } + + /* include NUL-byte in length for non-empty name */ + if (len >= 0) + len++; + return len; +} + +int hfsplus_setxattr(struct inode *inode, const char *name, + const void *value, size_t size, int flags, + const char *prefix, size_t prefixlen) +{ + char *xattr_name; + size_t xattr_name_len = + NLS_MAX_CHARSET_SIZE * HFSPLUS_ATTR_MAX_STRLEN + 1; + int res; + + hfs_dbg("ino %llu, name %s, prefix %s, prefixlen %zu, " + "value %p, size %zu\n", + inode->i_ino, name ? name : NULL, + prefix ? prefix : NULL, prefixlen, + value, size); + + xattr_name = kmalloc(xattr_name_len, GFP_KERNEL); + if (!xattr_name) + return -ENOMEM; + strcpy(xattr_name, prefix); + strcpy(xattr_name + prefixlen, name); + res = __hfsplus_setxattr(inode, xattr_name, value, size, flags); + kfree(xattr_name); + + hfs_dbg("finished: res %d\n", res); + + return res; +} + +static ssize_t hfsplus_getxattr_finder_info(struct inode *inode, + void *value, size_t size) +{ + ssize_t res = 0; + struct hfs_find_data fd; + u16 entry_type; + u16 folder_rec_len = sizeof(DInfo) + sizeof(DXInfo); + u16 file_rec_len = sizeof(FInfo) + sizeof(FXInfo); + u16 record_len = max(folder_rec_len, file_rec_len); + u8 folder_finder_info[sizeof(DInfo) + sizeof(DXInfo)]; + u8 file_finder_info[sizeof(FInfo) + sizeof(FXInfo)]; + + if (size >= record_len) { + res = hfs_find_init(HFSPLUS_SB(inode->i_sb)->cat_tree, &fd); + if (res) { + pr_err("can't init xattr find struct\n"); + return res; + } + res = hfsplus_find_cat(inode->i_sb, inode->i_ino, &fd); + if (res) + goto end_getxattr_finder_info; + entry_type = hfs_bnode_read_u16(fd.bnode, fd.entryoffset); + + if (entry_type == HFSPLUS_FOLDER) { + hfs_bnode_read(fd.bnode, folder_finder_info, + fd.entryoffset + + offsetof(struct hfsplus_cat_folder, user_info), + folder_rec_len); + memcpy(value, folder_finder_info, folder_rec_len); + res = folder_rec_len; + } else if (entry_type == HFSPLUS_FILE) { + hfs_bnode_read(fd.bnode, file_finder_info, + fd.entryoffset + + offsetof(struct hfsplus_cat_file, user_info), + file_rec_len); + memcpy(value, file_finder_info, file_rec_len); + res = file_rec_len; + } else { + res = -EOPNOTSUPP; + goto end_getxattr_finder_info; + } + } else + res = size ? -ERANGE : record_len; + +end_getxattr_finder_info: + if (size >= record_len) + hfs_find_exit(&fd); + return res; +} + +ssize_t __hfsplus_getxattr(struct inode *inode, const char *name, + void *value, size_t size) +{ + struct hfs_find_data fd; + hfsplus_attr_entry *entry; + __be32 xattr_record_type; + u32 record_type; + u16 record_length = 0; + ssize_t res; + + if (!is_xattr_operation_supported(inode)) + return -EOPNOTSUPP; + + if (!strcmp_xattr_finder_info(name)) + return hfsplus_getxattr_finder_info(inode, value, size); + + if (!HFSPLUS_SB(inode->i_sb)->attr_tree) + return -EOPNOTSUPP; + + entry = hfsplus_alloc_attr_entry(); + if (!entry) { + pr_err("can't allocate xattr entry\n"); + return -ENOMEM; + } + + res = hfs_find_init(HFSPLUS_SB(inode->i_sb)->attr_tree, &fd); + if (res) { + pr_err("can't init xattr find struct\n"); + goto failed_getxattr_init; + } + + res = hfsplus_find_attr(inode->i_sb, inode->i_ino, name, &fd); + if (res) { + if (res == -ENOENT || res == -ENODATA) + res = -ENODATA; + else + pr_err("xattr search failed\n"); + goto out; + } + + hfs_bnode_read(fd.bnode, &xattr_record_type, + fd.entryoffset, sizeof(xattr_record_type)); + record_type = be32_to_cpu(xattr_record_type); + if (record_type == HFSPLUS_ATTR_INLINE_DATA) { + record_length = hfs_bnode_read_u16(fd.bnode, + fd.entryoffset + + offsetof(struct hfsplus_attr_inline_data, + length)); + if (record_length > HFSPLUS_MAX_INLINE_DATA_SIZE) { + pr_err("invalid xattr record size\n"); + res = -EIO; + goto out; + } + } else if (record_type == HFSPLUS_ATTR_FORK_DATA || + record_type == HFSPLUS_ATTR_EXTENTS) { + pr_err("only inline data xattr are supported\n"); + res = -EOPNOTSUPP; + goto out; + } else { + pr_err("invalid xattr record\n"); + res = -EIO; + goto out; + } + + if (size) { + hfs_bnode_read(fd.bnode, entry, fd.entryoffset, + offsetof(struct hfsplus_attr_inline_data, + raw_bytes) + record_length); + } + + if (size >= record_length) { + memcpy(value, entry->inline_data.raw_bytes, record_length); + res = record_length; + } else + res = size ? -ERANGE : record_length; + +out: + hfs_find_exit(&fd); + +failed_getxattr_init: + hfsplus_destroy_attr_entry(entry); + return res; +} + +ssize_t hfsplus_getxattr(struct inode *inode, const char *name, + void *value, size_t size, + const char *prefix, size_t prefixlen) +{ + int res; + char *xattr_name; + + hfs_dbg("ino %llu, name %s, prefix %s\n", + inode->i_ino, name ? name : NULL, + prefix ? prefix : NULL); + + xattr_name = kmalloc(NLS_MAX_CHARSET_SIZE * HFSPLUS_ATTR_MAX_STRLEN + 1, + GFP_KERNEL); + if (!xattr_name) + return -ENOMEM; + + strcpy(xattr_name, prefix); + strcpy(xattr_name + prefixlen, name); + + res = __hfsplus_getxattr(inode, xattr_name, value, size); + kfree(xattr_name); + + hfs_dbg("finished: res %d\n", res); + + return res; + +} + +static inline int can_list(const char *xattr_name) +{ + if (!xattr_name) + return 0; + + return strncmp(xattr_name, XATTR_TRUSTED_PREFIX, + XATTR_TRUSTED_PREFIX_LEN) || + capable(CAP_SYS_ADMIN); +} + +static ssize_t hfsplus_listxattr_finder_info(struct dentry *dentry, + char *buffer, size_t size) +{ + ssize_t res; + struct inode *inode = d_inode(dentry); + struct hfs_find_data fd; + u16 entry_type; + u8 folder_finder_info[sizeof(DInfo) + sizeof(DXInfo)]; + u8 file_finder_info[sizeof(FInfo) + sizeof(FXInfo)]; + unsigned long len, found_bit; + int xattr_name_len, symbols_count; + + res = hfs_find_init(HFSPLUS_SB(inode->i_sb)->cat_tree, &fd); + if (res) { + pr_err("can't init xattr find struct\n"); + return res; + } + + res = hfsplus_find_cat(inode->i_sb, inode->i_ino, &fd); + if (res) + goto end_listxattr_finder_info; + + entry_type = hfs_bnode_read_u16(fd.bnode, fd.entryoffset); + if (entry_type == HFSPLUS_FOLDER) { + len = sizeof(DInfo) + sizeof(DXInfo); + hfs_bnode_read(fd.bnode, folder_finder_info, + fd.entryoffset + + offsetof(struct hfsplus_cat_folder, user_info), + len); + found_bit = find_first_bit((void *)folder_finder_info, len*8); + } else if (entry_type == HFSPLUS_FILE) { + len = sizeof(FInfo) + sizeof(FXInfo); + hfs_bnode_read(fd.bnode, file_finder_info, + fd.entryoffset + + offsetof(struct hfsplus_cat_file, user_info), + len); + found_bit = find_first_bit((void *)file_finder_info, len*8); + } else { + res = -EOPNOTSUPP; + goto end_listxattr_finder_info; + } + + if (found_bit >= (len*8)) + res = 0; + else { + symbols_count = sizeof(HFSPLUS_XATTR_FINDER_INFO_NAME) - 1; + xattr_name_len = + name_len(HFSPLUS_XATTR_FINDER_INFO_NAME, symbols_count); + if (!buffer || !size) { + if (can_list(HFSPLUS_XATTR_FINDER_INFO_NAME)) + res = xattr_name_len; + } else if (can_list(HFSPLUS_XATTR_FINDER_INFO_NAME)) { + if (size < xattr_name_len) + res = -ERANGE; + else { + res = copy_name(buffer, + HFSPLUS_XATTR_FINDER_INFO_NAME, + symbols_count); + } + } + } + +end_listxattr_finder_info: + hfs_find_exit(&fd); + + return res; +} + +ssize_t hfsplus_listxattr(struct dentry *dentry, char *buffer, size_t size) +{ + ssize_t err; + ssize_t res; + struct inode *inode = d_inode(dentry); + struct hfs_find_data fd; + struct hfsplus_attr_key attr_key; + char *strbuf; + size_t strbuf_size; + int xattr_name_len; + + hfs_dbg("ino %llu\n", inode->i_ino); + + if (!is_xattr_operation_supported(inode)) + return -EOPNOTSUPP; + + res = hfsplus_listxattr_finder_info(dentry, buffer, size); + if (res < 0) + return res; + else if (!HFSPLUS_SB(inode->i_sb)->attr_tree) + return (res == 0) ? -EOPNOTSUPP : res; + + err = hfs_find_init(HFSPLUS_SB(inode->i_sb)->attr_tree, &fd); + if (err) { + pr_err("can't init xattr find struct\n"); + return err; + } + + strbuf_size = NLS_MAX_CHARSET_SIZE * HFSPLUS_ATTR_MAX_STRLEN + + XATTR_MAC_OSX_PREFIX_LEN + 1; + strbuf = kzalloc(strbuf_size, GFP_KERNEL); + if (!strbuf) { + res = -ENOMEM; + goto out; + } + + err = hfsplus_find_attr(inode->i_sb, inode->i_ino, NULL, &fd); + if (err) { + if (err == -ENOENT || err == -ENODATA) { + res = 0; + goto end_listxattr; + } else { + res = err; + goto end_listxattr; + } + } + + for (;;) { + u16 key_len = hfs_bnode_read_u16(fd.bnode, fd.keyoffset); + + if (key_len == 0 || key_len > fd.tree->max_key_len) { + pr_err("invalid xattr key length: %d\n", key_len); + res = -EIO; + goto end_listxattr; + } + + hfs_bnode_read(fd.bnode, &attr_key, + fd.keyoffset, key_len + sizeof(key_len)); + + if (be32_to_cpu(attr_key.cnid) != inode->i_ino) + goto end_listxattr; + + xattr_name_len = NLS_MAX_CHARSET_SIZE * HFSPLUS_ATTR_MAX_STRLEN; + if (hfsplus_uni2asc_xattr_str(inode->i_sb, + &fd.key->attr.key_name, strbuf, + &xattr_name_len)) { + pr_err("unicode conversion failed\n"); + res = -EIO; + goto end_listxattr; + } + + if (!buffer || !size) { + if (can_list(strbuf)) + res += name_len(strbuf, xattr_name_len); + } else if (can_list(strbuf)) { + if (size < (res + name_len(strbuf, xattr_name_len))) { + pr_err("size %zu, res %zd, name_len %zu\n", + size, res, + name_len(strbuf, xattr_name_len)); + res = -ERANGE; + goto end_listxattr; + } else + res += copy_name(buffer + res, + strbuf, xattr_name_len); + } + + memset(fd.key->attr.key_name.unicode, 0, + sizeof(fd.key->attr.key_name.unicode)); + memset(strbuf, 0, strbuf_size); + + if (hfs_brec_goto(&fd, 1)) + goto end_listxattr; + } + +end_listxattr: + kfree(strbuf); +out: + hfs_find_exit(&fd); + + hfs_dbg("finished: res %zd\n", res); + + return res; +} + +static int hfsplus_removexattr(struct inode *inode, const char *name) +{ + int err; + struct hfs_find_data cat_fd; + u16 flags; + u16 cat_entry_type; + int is_xattr_acl_deleted; + int is_all_xattrs_deleted; + + hfs_dbg("ino %llu, name %s\n", + inode->i_ino, name ? name : NULL); + + if (!HFSPLUS_SB(inode->i_sb)->attr_tree) + return -EOPNOTSUPP; + + if (!strcmp_xattr_finder_info(name)) + return -EOPNOTSUPP; + + err = hfs_find_init(HFSPLUS_SB(inode->i_sb)->cat_tree, &cat_fd); + if (err) { + pr_err("can't init xattr find struct\n"); + return err; + } + + err = hfsplus_find_cat(inode->i_sb, inode->i_ino, &cat_fd); + if (err) { + pr_err("catalog searching failed\n"); + goto end_removexattr; + } + + err = hfsplus_delete_attr(inode, name); + if (err) + goto end_removexattr; + + is_xattr_acl_deleted = !strcmp_xattr_acl(name); + is_all_xattrs_deleted = !hfsplus_attr_exists(inode, NULL); + + if (!is_xattr_acl_deleted && !is_all_xattrs_deleted) + goto end_removexattr; + + cat_entry_type = hfs_bnode_read_u16(cat_fd.bnode, cat_fd.entryoffset); + + if (cat_entry_type == HFSPLUS_FOLDER) { + flags = hfs_bnode_read_u16(cat_fd.bnode, cat_fd.entryoffset + + offsetof(struct hfsplus_cat_folder, flags)); + if (is_xattr_acl_deleted) + flags &= ~HFSPLUS_ACL_EXISTS; + if (is_all_xattrs_deleted) + flags &= ~HFSPLUS_XATTR_EXISTS; + hfs_bnode_write_u16(cat_fd.bnode, cat_fd.entryoffset + + offsetof(struct hfsplus_cat_folder, flags), + flags); + hfsplus_mark_inode_dirty(HFSPLUS_CAT_TREE_I(inode->i_sb), + HFSPLUS_I_CAT_DIRTY); + hfsplus_mark_inode_dirty(inode, HFSPLUS_I_CAT_DIRTY); + } else if (cat_entry_type == HFSPLUS_FILE) { + flags = hfs_bnode_read_u16(cat_fd.bnode, cat_fd.entryoffset + + offsetof(struct hfsplus_cat_file, flags)); + if (is_xattr_acl_deleted) + flags &= ~HFSPLUS_ACL_EXISTS; + if (is_all_xattrs_deleted) + flags &= ~HFSPLUS_XATTR_EXISTS; + hfs_bnode_write_u16(cat_fd.bnode, cat_fd.entryoffset + + offsetof(struct hfsplus_cat_file, flags), + flags); + hfsplus_mark_inode_dirty(HFSPLUS_CAT_TREE_I(inode->i_sb), + HFSPLUS_I_CAT_DIRTY); + hfsplus_mark_inode_dirty(inode, HFSPLUS_I_CAT_DIRTY); + } else { + pr_err("invalid catalog entry type\n"); + err = -EIO; + goto end_removexattr; + } + + inode_set_ctime_current(inode); + +end_removexattr: + hfs_find_exit(&cat_fd); + + hfs_dbg("finished: err %d\n", err); + + return err; +} + +static int hfsplus_osx_getxattr(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + /* + * Don't allow retrieving properly prefixed attributes + * by prepending them with "osx." + */ + if (is_known_namespace(name)) + return -EOPNOTSUPP; + + /* + * osx is the namespace we use to indicate an unprefixed + * attribute on the filesystem (like the ones that OS X + * creates), so we pass the name through unmodified (after + * ensuring it doesn't conflict with another namespace). + */ + return __hfsplus_getxattr(inode, name, buffer, size); +} + +static int hfsplus_osx_setxattr(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *buffer, + size_t size, int flags) +{ + /* + * Don't allow setting properly prefixed attributes + * by prepending them with "osx." + */ + if (is_known_namespace(name)) + return -EOPNOTSUPP; + + /* + * osx is the namespace we use to indicate an unprefixed + * attribute on the filesystem (like the ones that OS X + * creates), so we pass the name through unmodified (after + * ensuring it doesn't conflict with another namespace). + */ + return __hfsplus_setxattr(inode, name, buffer, size, flags); +} + +const struct xattr_handler hfsplus_xattr_osx_handler = { + .prefix = XATTR_MAC_OSX_PREFIX, + .get = hfsplus_osx_getxattr, + .set = hfsplus_osx_setxattr, +}; diff --git a/fs/hfsplus/xattr.h b/fs/hfsplus/xattr.h new file mode 100644 index 000000000..15cc55e41 --- /dev/null +++ b/fs/hfsplus/xattr.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * linux/fs/hfsplus/xattr.h + * + * Vyacheslav Dubeyko + * + * Logic of processing extended attributes + */ + +#ifndef _LINUX_HFSPLUS_XATTR_H +#define _LINUX_HFSPLUS_XATTR_H + +#include + +extern const struct xattr_handler hfsplus_xattr_osx_handler; +extern const struct xattr_handler hfsplus_xattr_user_handler; +extern const struct xattr_handler hfsplus_xattr_trusted_handler; +extern const struct xattr_handler hfsplus_xattr_security_handler; + +extern const struct xattr_handler * const hfsplus_xattr_handlers[]; + +int __hfsplus_setxattr(struct inode *inode, const char *name, + const void *value, size_t size, int flags); + +int hfsplus_setxattr(struct inode *inode, const char *name, + const void *value, size_t size, int flags, + const char *prefix, size_t prefixlen); + +ssize_t __hfsplus_getxattr(struct inode *inode, const char *name, + void *value, size_t size); + +ssize_t hfsplus_getxattr(struct inode *inode, const char *name, + void *value, size_t size, + const char *prefix, size_t prefixlen); + +ssize_t hfsplus_listxattr(struct dentry *dentry, char *buffer, size_t size); + +int hfsplus_init_security(struct inode *inode, struct inode *dir, + const struct qstr *qstr); + +#endif diff --git a/fs/hfsplus/xattr_security.c b/fs/hfsplus/xattr_security.c new file mode 100644 index 000000000..90f68ec11 --- /dev/null +++ b/fs/hfsplus/xattr_security.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hfsplus/xattr_trusted.c + * + * Vyacheslav Dubeyko + * + * Handler for storing security labels as extended attributes. + */ + +#include +#include + +#include "hfsplus_fs.h" +#include "xattr.h" + +static int hfsplus_security_getxattr(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + return hfsplus_getxattr(inode, name, buffer, size, + XATTR_SECURITY_PREFIX, + XATTR_SECURITY_PREFIX_LEN); +} + +static int hfsplus_security_setxattr(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *buffer, + size_t size, int flags) +{ + return hfsplus_setxattr(inode, name, buffer, size, flags, + XATTR_SECURITY_PREFIX, + XATTR_SECURITY_PREFIX_LEN); +} + +static int hfsplus_initxattrs(struct inode *inode, + const struct xattr *xattr_array, + void *fs_info) +{ + const struct xattr *xattr; + char *xattr_name; + int err = 0; + + xattr_name = kmalloc(NLS_MAX_CHARSET_SIZE * HFSPLUS_ATTR_MAX_STRLEN + 1, + GFP_KERNEL); + if (!xattr_name) + return -ENOMEM; + for (xattr = xattr_array; xattr->name != NULL; xattr++) { + + if (!strcmp(xattr->name, "")) + continue; + + strcpy(xattr_name, XATTR_SECURITY_PREFIX); + strcpy(xattr_name + + XATTR_SECURITY_PREFIX_LEN, xattr->name); + memset(xattr_name + + XATTR_SECURITY_PREFIX_LEN + strlen(xattr->name), 0, 1); + + err = __hfsplus_setxattr(inode, xattr_name, + xattr->value, xattr->value_len, 0); + if (err) + break; + } + kfree(xattr_name); + return err; +} + +int hfsplus_init_security(struct inode *inode, struct inode *dir, + const struct qstr *qstr) +{ + return security_inode_init_security(inode, dir, qstr, + &hfsplus_initxattrs, NULL); +} + +const struct xattr_handler hfsplus_xattr_security_handler = { + .prefix = XATTR_SECURITY_PREFIX, + .get = hfsplus_security_getxattr, + .set = hfsplus_security_setxattr, +}; diff --git a/fs/hfsplus/xattr_trusted.c b/fs/hfsplus/xattr_trusted.c new file mode 100644 index 000000000..fdbaebc1c --- /dev/null +++ b/fs/hfsplus/xattr_trusted.c @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hfsplus/xattr_trusted.c + * + * Vyacheslav Dubeyko + * + * Handler for trusted extended attributes. + */ + +#include + +#include "hfsplus_fs.h" +#include "xattr.h" + +static int hfsplus_trusted_getxattr(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + return hfsplus_getxattr(inode, name, buffer, size, + XATTR_TRUSTED_PREFIX, + XATTR_TRUSTED_PREFIX_LEN); +} + +static int hfsplus_trusted_setxattr(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *buffer, + size_t size, int flags) +{ + return hfsplus_setxattr(inode, name, buffer, size, flags, + XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN); +} + +const struct xattr_handler hfsplus_xattr_trusted_handler = { + .prefix = XATTR_TRUSTED_PREFIX, + .get = hfsplus_trusted_getxattr, + .set = hfsplus_trusted_setxattr, +}; diff --git a/fs/hfsplus/xattr_user.c b/fs/hfsplus/xattr_user.c new file mode 100644 index 000000000..6464b6c3d --- /dev/null +++ b/fs/hfsplus/xattr_user.c @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hfsplus/xattr_user.c + * + * Vyacheslav Dubeyko + * + * Handler for user extended attributes. + */ + +#include + +#include "hfsplus_fs.h" +#include "xattr.h" + +static int hfsplus_user_getxattr(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + + return hfsplus_getxattr(inode, name, buffer, size, + XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN); +} + +static int hfsplus_user_setxattr(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *buffer, + size_t size, int flags) +{ + return hfsplus_setxattr(inode, name, buffer, size, flags, + XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN); +} + +const struct xattr_handler hfsplus_xattr_user_handler = { + .prefix = XATTR_USER_PREFIX, + .get = hfsplus_user_getxattr, + .set = hfsplus_user_setxattr, +}; diff --git a/fs/hostfs/Makefile b/fs/hostfs/Makefile new file mode 100644 index 000000000..16be592e8 --- /dev/null +++ b/fs/hostfs/Makefile @@ -0,0 +1,13 @@ +# +# Copyright (C) 2000 Jeff Dike (jdike@karaya.com) +# Licensed under the GPL +# + +hostfs-objs := hostfs_kern.o + +hostfs-builtin-$(CONFIG_HOSTFS) += hostfs_user.o hostfs_user_exp.o + +obj-y := $(hostfs-builtin-y) $(hostfs-builtin-m) +obj-$(CONFIG_HOSTFS) += hostfs.o + +include $(srctree)/arch/um/scripts/Makefile.rules diff --git a/fs/hostfs/hostfs.h b/fs/hostfs/hostfs.h new file mode 100644 index 000000000..aa02599b7 --- /dev/null +++ b/fs/hostfs/hostfs.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __UM_FS_HOSTFS +#define __UM_FS_HOSTFS + +#include +#include + +struct hostfs_timespec { + long long tv_sec; + long long tv_nsec; +}; + +struct hostfs_iattr { + unsigned int ia_valid; + unsigned short ia_mode; + uid_t ia_uid; + gid_t ia_gid; + loff_t ia_size; + struct hostfs_timespec ia_atime; + struct hostfs_timespec ia_mtime; + struct hostfs_timespec ia_ctime; +}; + +struct hostfs_stat { + unsigned long long ino; + unsigned int mode; + unsigned int nlink; + unsigned int uid; + unsigned int gid; + unsigned long long size; + struct hostfs_timespec atime, mtime, ctime, btime; + unsigned int blksize; + unsigned long long blocks; + struct { + unsigned int maj; + unsigned int min; + } rdev, dev; +}; + +extern int stat_file(const char *path, struct hostfs_stat *p, int fd); +extern int access_file(char *path, int r, int w, int x); +extern int open_file(char *path, int r, int w, int append); +extern void *open_dir(char *path, int *err_out); +extern void seek_dir(void *stream, unsigned long long pos); +extern char *read_dir(void *stream, unsigned long long *pos_out, + unsigned long long *ino_out, int *len_out, + unsigned int *type_out); +extern void close_file(void *stream); +extern int replace_file(int oldfd, int fd); +extern void close_dir(void *stream); +extern int read_file(int fd, unsigned long long *offset, char *buf, int len); +extern int write_file(int fd, unsigned long long *offset, const char *buf, + int len); +extern int lseek_file(int fd, long long offset, int whence); +extern int fsync_file(int fd, int datasync); +extern int file_create(char *name, int mode); +extern int set_attr(const char *file, struct hostfs_iattr *attrs, int fd); +extern int make_symlink(const char *from, const char *to); +extern int unlink_file(const char *file); +extern int do_mkdir(const char *file, int mode); +extern int hostfs_do_rmdir(const char *file); +extern int do_mknod(const char *file, int mode, unsigned int major, + unsigned int minor); +extern int link_file(const char *to, const char *from); +extern int hostfs_do_readlink(char *file, char *buf, int size); +extern int rename_file(char *from, char *to); +extern int rename2_file(char *from, char *to, unsigned int flags); +extern int do_statfs(char *root, long *bsize_out, long long *blocks_out, + long long *bfree_out, long long *bavail_out, + long long *files_out, long long *ffree_out, + void *fsid_out, int fsid_size, long *namelen_out); + +#endif diff --git a/fs/hostfs/hostfs_kern.c b/fs/hostfs/hostfs_kern.c new file mode 100644 index 000000000..7add056d4 --- /dev/null +++ b/fs/hostfs/hostfs_kern.c @@ -0,0 +1,1096 @@ +/* + * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com) + * Licensed under the GPL + * + * Ported the filesystem routines to 2.5. + * 2003-02-10 Petr Baudis + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "hostfs.h" +#include +#include + +struct hostfs_fs_info { + char *host_root_path; +}; + +struct hostfs_inode_info { + int fd; + fmode_t mode; + struct inode vfs_inode; + struct mutex open_mutex; + dev_t dev; + struct hostfs_timespec btime; +}; + +static inline struct hostfs_inode_info *HOSTFS_I(struct inode *inode) +{ + return list_entry(inode, struct hostfs_inode_info, vfs_inode); +} + +#define FILE_HOSTFS_I(file) HOSTFS_I(file_inode(file)) + +static struct kmem_cache *hostfs_inode_cache; + +/* Changed in hostfs_args before the kernel starts running */ +static char *root_ino = ""; +static int append = 0; + +static const struct inode_operations hostfs_iops; +static const struct inode_operations hostfs_dir_iops; +static const struct inode_operations hostfs_link_iops; + +#ifndef MODULE +static int __init hostfs_args(char *options, int *add) +{ + char *ptr; + + *add = 0; + ptr = strchr(options, ','); + if (ptr != NULL) + *ptr++ = '\0'; + if (*options != '\0') + root_ino = options; + + options = ptr; + while (options) { + ptr = strchr(options, ','); + if (ptr != NULL) + *ptr++ = '\0'; + if (*options != '\0') { + if (!strcmp(options, "append")) + append = 1; + else printf("hostfs_args - unsupported option - %s\n", + options); + } + options = ptr; + } + return 0; +} + +__uml_setup("hostfs=", hostfs_args, +"hostfs=,,...\n" +" This is used to set hostfs parameters. The root directory argument\n" +" is used to confine all hostfs mounts to within the specified directory\n" +" tree on the host. If this isn't specified, then a user inside UML can\n" +" mount anything on the host that's accessible to the user that's running\n" +" it.\n" +" The only flag currently supported is 'append', which specifies that all\n" +" files opened by hostfs will be opened in append mode.\n\n" +); +#endif + +static char *__dentry_name(struct dentry *dentry, char *name) +{ + char *p = dentry_path_raw(dentry, name, PATH_MAX); + struct hostfs_fs_info *fsi = dentry->d_sb->s_fs_info; + char *root = fsi->host_root_path; + size_t len = strlen(root); + + if (IS_ERR(p) || len > p - name) { + __putname(name); + return NULL; + } + + memcpy(name, root, len); + memmove(name + len, p, name + PATH_MAX - p); + + return name; +} + +static char *dentry_name(struct dentry *dentry) +{ + char *name = __getname(); + if (!name) + return NULL; + + return __dentry_name(dentry, name); +} + +static char *inode_name(struct inode *ino) +{ + struct dentry *dentry; + char *name; + + dentry = d_find_alias(ino); + if (!dentry) + return NULL; + + name = dentry_name(dentry); + + dput(dentry); + + return name; +} + +static char *follow_link(char *link) +{ + char *name, *resolved, *end; + int n; + + name = kmalloc(PATH_MAX, GFP_KERNEL); + if (!name) { + n = -ENOMEM; + goto out_free; + } + + n = hostfs_do_readlink(link, name, PATH_MAX); + if (n < 0) + goto out_free; + else if (n == PATH_MAX) { + n = -E2BIG; + goto out_free; + } + + if (*name == '/') + return name; + + end = strrchr(link, '/'); + if (end == NULL) + return name; + + *(end + 1) = '\0'; + + resolved = kasprintf(GFP_KERNEL, "%s%s", link, name); + if (resolved == NULL) { + n = -ENOMEM; + goto out_free; + } + + kfree(name); + return resolved; + + out_free: + kfree(name); + return ERR_PTR(n); +} + +static int hostfs_statfs(struct dentry *dentry, struct kstatfs *sf) +{ + /* + * do_statfs uses struct statfs64 internally, but the linux kernel + * struct statfs still has 32-bit versions for most of these fields, + * so we convert them here + */ + int err; + long long f_blocks; + long long f_bfree; + long long f_bavail; + long long f_files; + long long f_ffree; + struct hostfs_fs_info *fsi; + + fsi = dentry->d_sb->s_fs_info; + err = do_statfs(fsi->host_root_path, + &sf->f_bsize, &f_blocks, &f_bfree, &f_bavail, &f_files, + &f_ffree, &sf->f_fsid, sizeof(sf->f_fsid), + &sf->f_namelen); + if (err) + return err; + sf->f_blocks = f_blocks; + sf->f_bfree = f_bfree; + sf->f_bavail = f_bavail; + sf->f_files = f_files; + sf->f_ffree = f_ffree; + sf->f_type = HOSTFS_SUPER_MAGIC; + return 0; +} + +static struct inode *hostfs_alloc_inode(struct super_block *sb) +{ + struct hostfs_inode_info *hi; + + hi = alloc_inode_sb(sb, hostfs_inode_cache, GFP_KERNEL_ACCOUNT); + if (hi == NULL) + return NULL; + hi->fd = -1; + hi->mode = 0; + hi->dev = 0; + inode_init_once(&hi->vfs_inode); + mutex_init(&hi->open_mutex); + return &hi->vfs_inode; +} + +static void hostfs_evict_inode(struct inode *inode) +{ + truncate_inode_pages_final(&inode->i_data); + clear_inode(inode); + if (HOSTFS_I(inode)->fd != -1) { + close_file(&HOSTFS_I(inode)->fd); + HOSTFS_I(inode)->fd = -1; + HOSTFS_I(inode)->dev = 0; + } +} + +static void hostfs_free_inode(struct inode *inode) +{ + kmem_cache_free(hostfs_inode_cache, HOSTFS_I(inode)); +} + +static int hostfs_show_options(struct seq_file *seq, struct dentry *root) +{ + struct hostfs_fs_info *fsi; + const char *root_path; + + fsi = root->d_sb->s_fs_info; + root_path = fsi->host_root_path; + size_t offset = strlen(root_ino) + 1; + + if (strlen(root_path) > offset) + seq_show_option(seq, root_path + offset, NULL); + + if (append) + seq_puts(seq, ",append"); + + return 0; +} + +static const struct super_operations hostfs_sbops = { + .alloc_inode = hostfs_alloc_inode, + .free_inode = hostfs_free_inode, + .drop_inode = inode_just_drop, + .evict_inode = hostfs_evict_inode, + .statfs = hostfs_statfs, + .show_options = hostfs_show_options, +}; + +static int hostfs_readdir(struct file *file, struct dir_context *ctx) +{ + void *dir; + char *name; + unsigned long long next, ino; + int error, len; + unsigned int type; + + name = dentry_name(file->f_path.dentry); + if (name == NULL) + return -ENOMEM; + dir = open_dir(name, &error); + __putname(name); + if (dir == NULL) + return -error; + next = ctx->pos; + seek_dir(dir, next); + while ((name = read_dir(dir, &next, &ino, &len, &type)) != NULL) { + if (!dir_emit(ctx, name, len, ino, type)) + break; + ctx->pos = next; + } + close_dir(dir); + return 0; +} + +static int hostfs_open(struct inode *ino, struct file *file) +{ + char *name; + fmode_t mode; + int err; + int r, w, fd; + + mode = file->f_mode & (FMODE_READ | FMODE_WRITE); + if ((mode & HOSTFS_I(ino)->mode) == mode) + return 0; + + mode |= HOSTFS_I(ino)->mode; + +retry: + r = w = 0; + + if (mode & FMODE_READ) + r = 1; + if (mode & FMODE_WRITE) + r = w = 1; + + name = dentry_name(file_dentry(file)); + if (name == NULL) + return -ENOMEM; + + fd = open_file(name, r, w, append); + __putname(name); + if (fd < 0) + return fd; + + mutex_lock(&HOSTFS_I(ino)->open_mutex); + /* somebody else had handled it first? */ + if ((mode & HOSTFS_I(ino)->mode) == mode) { + mutex_unlock(&HOSTFS_I(ino)->open_mutex); + close_file(&fd); + return 0; + } + if ((mode | HOSTFS_I(ino)->mode) != mode) { + mode |= HOSTFS_I(ino)->mode; + mutex_unlock(&HOSTFS_I(ino)->open_mutex); + close_file(&fd); + goto retry; + } + if (HOSTFS_I(ino)->fd == -1) { + HOSTFS_I(ino)->fd = fd; + } else { + err = replace_file(fd, HOSTFS_I(ino)->fd); + close_file(&fd); + if (err < 0) { + mutex_unlock(&HOSTFS_I(ino)->open_mutex); + return err; + } + } + HOSTFS_I(ino)->mode = mode; + mutex_unlock(&HOSTFS_I(ino)->open_mutex); + + return 0; +} + +static int hostfs_file_release(struct inode *inode, struct file *file) +{ + filemap_write_and_wait(inode->i_mapping); + + return 0; +} + +static int hostfs_fsync(struct file *file, loff_t start, loff_t end, + int datasync) +{ + struct inode *inode = file->f_mapping->host; + int ret; + + ret = file_write_and_wait_range(file, start, end); + if (ret) + return ret; + + inode_lock(inode); + ret = fsync_file(HOSTFS_I(inode)->fd, datasync); + inode_unlock(inode); + + return ret; +} + +static const struct file_operations hostfs_file_fops = { + .llseek = generic_file_llseek, + .splice_read = filemap_splice_read, + .splice_write = iter_file_splice_write, + .read_iter = generic_file_read_iter, + .write_iter = generic_file_write_iter, + .mmap_prepare = generic_file_mmap_prepare, + .open = hostfs_open, + .release = hostfs_file_release, + .fsync = hostfs_fsync, +}; + +static const struct file_operations hostfs_dir_fops = { + .llseek = generic_file_llseek, + .iterate_shared = hostfs_readdir, + .read = generic_read_dir, + .open = hostfs_open, + .fsync = hostfs_fsync, +}; + +static int hostfs_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct inode *inode = mapping->host; + struct folio *folio = NULL; + loff_t i_size = i_size_read(inode); + int err = 0; + + while ((folio = writeback_iter(mapping, wbc, folio, &err))) { + loff_t pos = folio_pos(folio); + size_t count = folio_size(folio); + char *buffer; + int ret; + + if (count > i_size - pos) + count = i_size - pos; + + buffer = kmap_local_folio(folio, 0); + ret = write_file(HOSTFS_I(inode)->fd, &pos, buffer, count); + kunmap_local(buffer); + folio_unlock(folio); + if (ret != count) { + err = ret < 0 ? ret : -EIO; + mapping_set_error(mapping, err); + } + } + + return err; +} + +static int hostfs_read_folio(struct file *file, struct folio *folio) +{ + char *buffer; + loff_t start = folio_pos(folio); + int bytes_read, ret = 0; + + buffer = kmap_local_folio(folio, 0); + bytes_read = read_file(FILE_HOSTFS_I(file)->fd, &start, buffer, + PAGE_SIZE); + if (bytes_read < 0) + ret = bytes_read; + else + buffer = folio_zero_tail(folio, bytes_read, buffer + bytes_read); + kunmap_local(buffer); + + folio_end_read(folio, ret == 0); + return ret; +} + +static int hostfs_write_begin(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, + struct folio **foliop, void **fsdata) +{ + pgoff_t index = pos >> PAGE_SHIFT; + + *foliop = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN, + mapping_gfp_mask(mapping)); + if (IS_ERR(*foliop)) + return PTR_ERR(*foliop); + return 0; +} + +static int hostfs_write_end(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct folio *folio, void *fsdata) +{ + struct inode *inode = mapping->host; + void *buffer; + size_t from = offset_in_folio(folio, pos); + int err; + + buffer = kmap_local_folio(folio, from); + err = write_file(FILE_HOSTFS_I(iocb->ki_filp)->fd, &pos, buffer, copied); + kunmap_local(buffer); + + if (!folio_test_uptodate(folio) && err == folio_size(folio)) + folio_mark_uptodate(folio); + + /* + * If err > 0, write_file has added err to pos, so we are comparing + * i_size against the last byte written. + */ + if (err > 0 && (pos > inode->i_size)) + inode->i_size = pos; + folio_unlock(folio); + folio_put(folio); + + return err; +} + +static const struct address_space_operations hostfs_aops = { + .writepages = hostfs_writepages, + .read_folio = hostfs_read_folio, + .dirty_folio = filemap_dirty_folio, + .write_begin = hostfs_write_begin, + .write_end = hostfs_write_end, + .migrate_folio = filemap_migrate_folio, +}; + +static int hostfs_inode_update(struct inode *ino, const struct hostfs_stat *st) +{ + set_nlink(ino, st->nlink); + i_uid_write(ino, st->uid); + i_gid_write(ino, st->gid); + inode_set_atime_to_ts(ino, (struct timespec64){ + st->atime.tv_sec, + st->atime.tv_nsec, + }); + inode_set_mtime_to_ts(ino, (struct timespec64){ + st->mtime.tv_sec, + st->mtime.tv_nsec, + }); + inode_set_ctime(ino, st->ctime.tv_sec, st->ctime.tv_nsec); + ino->i_size = st->size; + ino->i_blocks = st->blocks; + return 0; +} + +static int hostfs_inode_set(struct inode *ino, void *data) +{ + struct hostfs_stat *st = data; + dev_t dev, rdev; + + /* Reencode maj and min with the kernel encoding.*/ + rdev = MKDEV(st->rdev.maj, st->rdev.min); + dev = MKDEV(st->dev.maj, st->dev.min); + + switch (st->mode & S_IFMT) { + case S_IFLNK: + ino->i_op = &hostfs_link_iops; + break; + case S_IFDIR: + ino->i_op = &hostfs_dir_iops; + ino->i_fop = &hostfs_dir_fops; + break; + case S_IFCHR: + case S_IFBLK: + case S_IFIFO: + case S_IFSOCK: + init_special_inode(ino, st->mode & S_IFMT, rdev); + ino->i_op = &hostfs_iops; + break; + case S_IFREG: + ino->i_op = &hostfs_iops; + ino->i_fop = &hostfs_file_fops; + ino->i_mapping->a_ops = &hostfs_aops; + break; + default: + return -EIO; + } + + HOSTFS_I(ino)->dev = dev; + HOSTFS_I(ino)->btime = st->btime; + ino->i_ino = st->ino; + ino->i_mode = st->mode; + return hostfs_inode_update(ino, st); +} + +static int hostfs_inode_test(struct inode *inode, void *data) +{ + const struct hostfs_stat *st = data; + dev_t dev = MKDEV(st->dev.maj, st->dev.min); + + return inode->i_ino == st->ino && HOSTFS_I(inode)->dev == dev && + (inode->i_mode & S_IFMT) == (st->mode & S_IFMT) && + HOSTFS_I(inode)->btime.tv_sec == st->btime.tv_sec && + HOSTFS_I(inode)->btime.tv_nsec == st->btime.tv_nsec; +} + +static struct inode *hostfs_iget(struct super_block *sb, char *name) +{ + struct inode *inode; + struct hostfs_stat st; + int err = stat_file(name, &st, -1); + + if (err) + return ERR_PTR(err); + + inode = iget5_locked(sb, st.ino, hostfs_inode_test, hostfs_inode_set, + &st); + if (!inode) + return ERR_PTR(-ENOMEM); + + if (inode_state_read_once(inode) & I_NEW) { + unlock_new_inode(inode); + } else { + spin_lock(&inode->i_lock); + hostfs_inode_update(inode, &st); + spin_unlock(&inode->i_lock); + } + + return inode; +} + +static int hostfs_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct inode *inode; + char *name; + int fd; + + name = dentry_name(dentry); + if (name == NULL) + return -ENOMEM; + + fd = file_create(name, mode & 0777); + if (fd < 0) { + __putname(name); + return fd; + } + + inode = hostfs_iget(dir->i_sb, name); + __putname(name); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + HOSTFS_I(inode)->fd = fd; + HOSTFS_I(inode)->mode = FMODE_READ | FMODE_WRITE; + d_instantiate(dentry, inode); + return 0; +} + +static struct dentry *hostfs_lookup(struct inode *ino, struct dentry *dentry, + unsigned int flags) +{ + struct inode *inode = NULL; + char *name; + + name = dentry_name(dentry); + if (name == NULL) + return ERR_PTR(-ENOMEM); + + inode = hostfs_iget(ino->i_sb, name); + __putname(name); + if (inode == ERR_PTR(-ENOENT)) + inode = NULL; + + return d_splice_alias(inode, dentry); +} + +static int hostfs_link(struct dentry *to, struct inode *ino, + struct dentry *from) +{ + char *from_name, *to_name; + int err; + + if ((from_name = dentry_name(from)) == NULL) + return -ENOMEM; + to_name = dentry_name(to); + if (to_name == NULL) { + __putname(from_name); + return -ENOMEM; + } + err = link_file(to_name, from_name); + __putname(from_name); + __putname(to_name); + return err; +} + +static int hostfs_unlink(struct inode *ino, struct dentry *dentry) +{ + char *file; + int err; + + if (append) + return -EPERM; + + if ((file = dentry_name(dentry)) == NULL) + return -ENOMEM; + + err = unlink_file(file); + __putname(file); + return err; +} + +static int hostfs_symlink(struct mnt_idmap *idmap, struct inode *ino, + struct dentry *dentry, const char *to) +{ + char *file; + int err; + + if ((file = dentry_name(dentry)) == NULL) + return -ENOMEM; + err = make_symlink(file, to); + __putname(file); + return err; +} + +static struct dentry *hostfs_mkdir(struct mnt_idmap *idmap, struct inode *ino, + struct dentry *dentry, umode_t mode) +{ + struct inode *inode; + char *file; + int err; + + if ((file = dentry_name(dentry)) == NULL) + return ERR_PTR(-ENOMEM); + err = do_mkdir(file, mode); + if (err) { + dentry = ERR_PTR(err); + } else { + inode = hostfs_iget(dentry->d_sb, file); + d_drop(dentry); + dentry = d_splice_alias(inode, dentry); + } + __putname(file); + return dentry; +} + +static int hostfs_rmdir(struct inode *ino, struct dentry *dentry) +{ + char *file; + int err; + + if ((file = dentry_name(dentry)) == NULL) + return -ENOMEM; + err = hostfs_do_rmdir(file); + __putname(file); + return err; +} + +static int hostfs_mknod(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t dev) +{ + struct inode *inode; + char *name; + int err; + + name = dentry_name(dentry); + if (name == NULL) + return -ENOMEM; + + err = do_mknod(name, mode, MAJOR(dev), MINOR(dev)); + if (err) { + __putname(name); + return err; + } + + inode = hostfs_iget(dir->i_sb, name); + __putname(name); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + d_instantiate(dentry, inode); + return 0; +} + +static int hostfs_rename2(struct mnt_idmap *idmap, + struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry, + unsigned int flags) +{ + char *old_name, *new_name; + int err; + + if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE)) + return -EINVAL; + + old_name = dentry_name(old_dentry); + if (old_name == NULL) + return -ENOMEM; + new_name = dentry_name(new_dentry); + if (new_name == NULL) { + __putname(old_name); + return -ENOMEM; + } + if (!flags) + err = rename_file(old_name, new_name); + else + err = rename2_file(old_name, new_name, flags); + + __putname(old_name); + __putname(new_name); + return err; +} + +static int hostfs_permission(struct mnt_idmap *idmap, + struct inode *ino, int desired) +{ + char *name; + int r = 0, w = 0, x = 0, err; + + if (desired & MAY_NOT_BLOCK) + return -ECHILD; + + if (desired & MAY_READ) r = 1; + if (desired & MAY_WRITE) w = 1; + if (desired & MAY_EXEC) x = 1; + name = inode_name(ino); + if (name == NULL) + return -ENOMEM; + + if (S_ISCHR(ino->i_mode) || S_ISBLK(ino->i_mode) || + S_ISFIFO(ino->i_mode) || S_ISSOCK(ino->i_mode)) + err = 0; + else + err = access_file(name, r, w, x); + __putname(name); + if (!err) + err = generic_permission(&nop_mnt_idmap, ino, desired); + return err; +} + +static int hostfs_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, struct iattr *attr) +{ + struct inode *inode = d_inode(dentry); + struct hostfs_iattr attrs; + char *name; + int err; + + int fd = HOSTFS_I(inode)->fd; + + err = setattr_prepare(&nop_mnt_idmap, dentry, attr); + if (err) + return err; + + if (append) + attr->ia_valid &= ~ATTR_SIZE; + + attrs.ia_valid = 0; + if (attr->ia_valid & ATTR_MODE) { + attrs.ia_valid |= HOSTFS_ATTR_MODE; + attrs.ia_mode = attr->ia_mode; + } + if (attr->ia_valid & ATTR_UID) { + attrs.ia_valid |= HOSTFS_ATTR_UID; + attrs.ia_uid = from_kuid(&init_user_ns, attr->ia_uid); + } + if (attr->ia_valid & ATTR_GID) { + attrs.ia_valid |= HOSTFS_ATTR_GID; + attrs.ia_gid = from_kgid(&init_user_ns, attr->ia_gid); + } + if (attr->ia_valid & ATTR_SIZE) { + attrs.ia_valid |= HOSTFS_ATTR_SIZE; + attrs.ia_size = attr->ia_size; + } + if (attr->ia_valid & ATTR_ATIME) { + attrs.ia_valid |= HOSTFS_ATTR_ATIME; + attrs.ia_atime = (struct hostfs_timespec) + { attr->ia_atime.tv_sec, attr->ia_atime.tv_nsec }; + } + if (attr->ia_valid & ATTR_MTIME) { + attrs.ia_valid |= HOSTFS_ATTR_MTIME; + attrs.ia_mtime = (struct hostfs_timespec) + { attr->ia_mtime.tv_sec, attr->ia_mtime.tv_nsec }; + } + if (attr->ia_valid & ATTR_CTIME) { + attrs.ia_valid |= HOSTFS_ATTR_CTIME; + attrs.ia_ctime = (struct hostfs_timespec) + { attr->ia_ctime.tv_sec, attr->ia_ctime.tv_nsec }; + } + if (attr->ia_valid & ATTR_ATIME_SET) { + attrs.ia_valid |= HOSTFS_ATTR_ATIME_SET; + } + if (attr->ia_valid & ATTR_MTIME_SET) { + attrs.ia_valid |= HOSTFS_ATTR_MTIME_SET; + } + name = dentry_name(dentry); + if (name == NULL) + return -ENOMEM; + err = set_attr(name, &attrs, fd); + __putname(name); + if (err) + return err; + + if ((attr->ia_valid & ATTR_SIZE) && + attr->ia_size != i_size_read(inode)) + truncate_setsize(inode, attr->ia_size); + + setattr_copy(&nop_mnt_idmap, inode, attr); + mark_inode_dirty(inode); + return 0; +} + +static const struct inode_operations hostfs_iops = { + .permission = hostfs_permission, + .setattr = hostfs_setattr, +}; + +static const struct inode_operations hostfs_dir_iops = { + .create = hostfs_create, + .lookup = hostfs_lookup, + .link = hostfs_link, + .unlink = hostfs_unlink, + .symlink = hostfs_symlink, + .mkdir = hostfs_mkdir, + .rmdir = hostfs_rmdir, + .mknod = hostfs_mknod, + .rename = hostfs_rename2, + .permission = hostfs_permission, + .setattr = hostfs_setattr, +}; + +static const char *hostfs_get_link(struct dentry *dentry, + struct inode *inode, + struct delayed_call *done) +{ + char *link; + if (!dentry) + return ERR_PTR(-ECHILD); + link = kmalloc(PATH_MAX, GFP_KERNEL); + if (link) { + char *path = dentry_name(dentry); + int err = -ENOMEM; + if (path) { + err = hostfs_do_readlink(path, link, PATH_MAX); + if (err == PATH_MAX) + err = -E2BIG; + __putname(path); + } + if (err < 0) { + kfree(link); + return ERR_PTR(err); + } + } else { + return ERR_PTR(-ENOMEM); + } + + set_delayed_call(done, kfree_link, link); + return link; +} + +static const struct inode_operations hostfs_link_iops = { + .get_link = hostfs_get_link, +}; + +static int hostfs_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct hostfs_fs_info *fsi = sb->s_fs_info; + struct inode *root_inode; + int err; + + sb->s_blocksize = 1024; + sb->s_blocksize_bits = 10; + sb->s_magic = HOSTFS_SUPER_MAGIC; + sb->s_op = &hostfs_sbops; + sb->s_d_flags = DCACHE_DONTCACHE; + sb->s_maxbytes = MAX_LFS_FILESIZE; + err = super_setup_bdi(sb); + if (err) + return err; + + root_inode = hostfs_iget(sb, fsi->host_root_path); + if (IS_ERR(root_inode)) + return PTR_ERR(root_inode); + + if (S_ISLNK(root_inode->i_mode)) { + char *name; + + iput(root_inode); + name = follow_link(fsi->host_root_path); + if (IS_ERR(name)) + return PTR_ERR(name); + + root_inode = hostfs_iget(sb, name); + kfree(name); + if (IS_ERR(root_inode)) + return PTR_ERR(root_inode); + } + + sb->s_root = d_make_root(root_inode); + if (sb->s_root == NULL) + return -ENOMEM; + + return 0; +} + +enum hostfs_parma { + Opt_hostfs, +}; + +static const struct fs_parameter_spec hostfs_param_specs[] = { + fsparam_string_empty("hostfs", Opt_hostfs), + {} +}; + +static int hostfs_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct hostfs_fs_info *fsi = fc->s_fs_info; + struct fs_parse_result result; + char *host_root, *tmp_root; + int opt; + + opt = fs_parse(fc, hostfs_param_specs, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_hostfs: + host_root = param->string; + if (!*host_root) + break; + tmp_root = kasprintf(GFP_KERNEL, "%s%s", + fsi->host_root_path, host_root); + if (!tmp_root) + return -ENOMEM; + kfree(fsi->host_root_path); + fsi->host_root_path = tmp_root; + break; + } + + return 0; +} + +static int hostfs_parse_monolithic(struct fs_context *fc, void *data) +{ + struct hostfs_fs_info *fsi = fc->s_fs_info; + char *tmp_root, *host_root = (char *)data; + + /* NULL is printed as '(null)' by printf(): avoid that. */ + if (host_root == NULL) + return 0; + + tmp_root = kasprintf(GFP_KERNEL, "%s%s", fsi->host_root_path, host_root); + if (!tmp_root) + return -ENOMEM; + kfree(fsi->host_root_path); + fsi->host_root_path = tmp_root; + return 0; +} + +static int hostfs_fc_get_tree(struct fs_context *fc) +{ + return get_tree_nodev(fc, hostfs_fill_super); +} + +static void hostfs_fc_free(struct fs_context *fc) +{ + struct hostfs_fs_info *fsi = fc->s_fs_info; + + if (!fsi) + return; + + kfree(fsi->host_root_path); + kfree(fsi); +} + +static const struct fs_context_operations hostfs_context_ops = { + .parse_monolithic = hostfs_parse_monolithic, + .parse_param = hostfs_parse_param, + .get_tree = hostfs_fc_get_tree, + .free = hostfs_fc_free, +}; + +static int hostfs_init_fs_context(struct fs_context *fc) +{ + struct hostfs_fs_info *fsi; + + fsi = kzalloc_obj(*fsi); + if (!fsi) + return -ENOMEM; + + fsi->host_root_path = kasprintf(GFP_KERNEL, "%s/", root_ino); + if (!fsi->host_root_path) { + kfree(fsi); + return -ENOMEM; + } + fc->s_fs_info = fsi; + fc->ops = &hostfs_context_ops; + return 0; +} + +static void hostfs_kill_sb(struct super_block *s) +{ + kill_anon_super(s); + kfree(s->s_fs_info); +} + +static struct file_system_type hostfs_type = { + .owner = THIS_MODULE, + .name = "hostfs", + .init_fs_context = hostfs_init_fs_context, + .kill_sb = hostfs_kill_sb, + .fs_flags = 0, +}; +MODULE_ALIAS_FS("hostfs"); + +static int __init init_hostfs(void) +{ + hostfs_inode_cache = KMEM_CACHE(hostfs_inode_info, 0); + if (!hostfs_inode_cache) + return -ENOMEM; + return register_filesystem(&hostfs_type); +} + +static void __exit exit_hostfs(void) +{ + unregister_filesystem(&hostfs_type); + kmem_cache_destroy(hostfs_inode_cache); +} + +module_init(init_hostfs) +module_exit(exit_hostfs) +MODULE_DESCRIPTION("User-Mode Linux Host filesystem"); +MODULE_LICENSE("GPL"); diff --git a/fs/hostfs/hostfs_user.c b/fs/hostfs/hostfs_user.c new file mode 100644 index 000000000..3bcd9f35e --- /dev/null +++ b/fs/hostfs/hostfs_user.c @@ -0,0 +1,421 @@ +/* + * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com) + * Licensed under the GPL + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "hostfs.h" +#include + +static void statx_to_hostfs(const struct statx *buf, struct hostfs_stat *p) +{ + p->ino = buf->stx_ino; + p->mode = buf->stx_mode; + p->nlink = buf->stx_nlink; + p->uid = buf->stx_uid; + p->gid = buf->stx_gid; + p->size = buf->stx_size; + p->atime.tv_sec = buf->stx_atime.tv_sec; + p->atime.tv_nsec = buf->stx_atime.tv_nsec; + p->ctime.tv_sec = buf->stx_ctime.tv_sec; + p->ctime.tv_nsec = buf->stx_ctime.tv_nsec; + p->mtime.tv_sec = buf->stx_mtime.tv_sec; + p->mtime.tv_nsec = buf->stx_mtime.tv_nsec; + if (buf->stx_mask & STATX_BTIME) { + p->btime.tv_sec = buf->stx_btime.tv_sec; + p->btime.tv_nsec = buf->stx_btime.tv_nsec; + } else { + memset(&p->btime, 0, sizeof(p->btime)); + } + p->blksize = buf->stx_blksize; + p->blocks = buf->stx_blocks; + p->rdev.maj = buf->stx_rdev_major; + p->rdev.min = buf->stx_rdev_minor; + p->dev.maj = buf->stx_dev_major; + p->dev.min = buf->stx_dev_minor; +} + +int stat_file(const char *path, struct hostfs_stat *p, int fd) +{ + struct statx buf; + int flags = AT_SYMLINK_NOFOLLOW; + + if (fd >= 0) { + flags |= AT_EMPTY_PATH; + path = ""; + } + + if ((statx(fd, path, flags, STATX_BASIC_STATS | STATX_BTIME, &buf)) < 0) + return -errno; + + statx_to_hostfs(&buf, p); + return 0; +} + +int access_file(char *path, int r, int w, int x) +{ + int mode = 0; + + if (r) + mode = R_OK; + if (w) + mode |= W_OK; + if (x) + mode |= X_OK; + if (access(path, mode) != 0) + return -errno; + else return 0; +} + +int open_file(char *path, int r, int w, int append) +{ + int mode = 0, fd; + + if (r && !w) + mode = O_RDONLY; + else if (!r && w) + mode = O_WRONLY; + else if (r && w) + mode = O_RDWR; + else panic("Impossible mode in open_file"); + + if (append) + mode |= O_APPEND; + fd = open64(path, mode); + if (fd < 0) + return -errno; + else return fd; +} + +void *open_dir(char *path, int *err_out) +{ + DIR *dir; + + dir = opendir(path); + *err_out = errno; + + return dir; +} + +void seek_dir(void *stream, unsigned long long pos) +{ + DIR *dir = stream; + + seekdir(dir, pos); +} + +char *read_dir(void *stream, unsigned long long *pos_out, + unsigned long long *ino_out, int *len_out, + unsigned int *type_out) +{ + DIR *dir = stream; + struct dirent *ent; + + ent = readdir(dir); + if (ent == NULL) + return NULL; + *len_out = strlen(ent->d_name); + *ino_out = ent->d_ino; + *type_out = ent->d_type; + *pos_out = ent->d_off; + return ent->d_name; +} + +int read_file(int fd, unsigned long long *offset, char *buf, int len) +{ + int n; + + n = pread64(fd, buf, len, *offset); + if (n < 0) + return -errno; + *offset += n; + return n; +} + +int write_file(int fd, unsigned long long *offset, const char *buf, int len) +{ + int n; + + n = pwrite64(fd, buf, len, *offset); + if (n < 0) + return -errno; + *offset += n; + return n; +} + +int lseek_file(int fd, long long offset, int whence) +{ + int ret; + + ret = lseek64(fd, offset, whence); + if (ret < 0) + return -errno; + return 0; +} + +int fsync_file(int fd, int datasync) +{ + int ret; + if (datasync) + ret = fdatasync(fd); + else + ret = fsync(fd); + + if (ret < 0) + return -errno; + return 0; +} + +int replace_file(int oldfd, int fd) +{ + return dup2(oldfd, fd); +} + +void close_file(void *stream) +{ + close(*((int *) stream)); +} + +void close_dir(void *stream) +{ + closedir(stream); +} + +int file_create(char *name, int mode) +{ + int fd; + + fd = open64(name, O_CREAT | O_RDWR, mode); + if (fd < 0) + return -errno; + return fd; +} + +int set_attr(const char *file, struct hostfs_iattr *attrs, int fd) +{ + struct hostfs_stat st; + struct timeval times[2]; + int err, ma; + + if (attrs->ia_valid & HOSTFS_ATTR_MODE) { + if (fd >= 0) { + if (fchmod(fd, attrs->ia_mode) != 0) + return -errno; + } else if (chmod(file, attrs->ia_mode) != 0) { + return -errno; + } + } + if (attrs->ia_valid & HOSTFS_ATTR_UID) { + if (fd >= 0) { + if (fchown(fd, attrs->ia_uid, -1)) + return -errno; + } else if (chown(file, attrs->ia_uid, -1)) { + return -errno; + } + } + if (attrs->ia_valid & HOSTFS_ATTR_GID) { + if (fd >= 0) { + if (fchown(fd, -1, attrs->ia_gid)) + return -errno; + } else if (chown(file, -1, attrs->ia_gid)) { + return -errno; + } + } + if (attrs->ia_valid & HOSTFS_ATTR_SIZE) { + if (fd >= 0) { + if (ftruncate(fd, attrs->ia_size)) + return -errno; + } else if (truncate(file, attrs->ia_size)) { + return -errno; + } + } + + /* + * Update accessed and/or modified time, in two parts: first set + * times according to the changes to perform, and then call futimes() + * or utimes() to apply them. + */ + ma = (HOSTFS_ATTR_ATIME_SET | HOSTFS_ATTR_MTIME_SET); + if (attrs->ia_valid & ma) { + err = stat_file(file, &st, fd); + if (err != 0) + return err; + + times[0].tv_sec = st.atime.tv_sec; + times[0].tv_usec = st.atime.tv_nsec / 1000; + times[1].tv_sec = st.mtime.tv_sec; + times[1].tv_usec = st.mtime.tv_nsec / 1000; + + if (attrs->ia_valid & HOSTFS_ATTR_ATIME_SET) { + times[0].tv_sec = attrs->ia_atime.tv_sec; + times[0].tv_usec = attrs->ia_atime.tv_nsec / 1000; + } + if (attrs->ia_valid & HOSTFS_ATTR_MTIME_SET) { + times[1].tv_sec = attrs->ia_mtime.tv_sec; + times[1].tv_usec = attrs->ia_mtime.tv_nsec / 1000; + } + + if (fd >= 0) { + if (futimes(fd, times) != 0) + return -errno; + } else if (utimes(file, times) != 0) { + return -errno; + } + } + + /* Note: ctime is not handled */ + if (attrs->ia_valid & (HOSTFS_ATTR_ATIME | HOSTFS_ATTR_MTIME)) { + err = stat_file(file, &st, fd); + attrs->ia_atime = st.atime; + attrs->ia_mtime = st.mtime; + if (err != 0) + return err; + } + return 0; +} + +int make_symlink(const char *from, const char *to) +{ + int err; + + err = symlink(to, from); + if (err) + return -errno; + return 0; +} + +int unlink_file(const char *file) +{ + int err; + + err = unlink(file); + if (err) + return -errno; + return 0; +} + +int do_mkdir(const char *file, int mode) +{ + int err; + + err = mkdir(file, mode); + if (err) + return -errno; + return 0; +} + +int hostfs_do_rmdir(const char *file) +{ + int err; + + err = rmdir(file); + if (err) + return -errno; + return 0; +} + +int do_mknod(const char *file, int mode, unsigned int major, unsigned int minor) +{ + int err; + + err = mknod(file, mode, os_makedev(major, minor)); + if (err) + return -errno; + return 0; +} + +int link_file(const char *to, const char *from) +{ + int err; + + err = link(to, from); + if (err) + return -errno; + return 0; +} + +int hostfs_do_readlink(char *file, char *buf, int size) +{ + int n; + + n = readlink(file, buf, size); + if (n < 0) + return -errno; + if (n < size) + buf[n] = '\0'; + return n; +} + +int rename_file(char *from, char *to) +{ + int err; + + err = rename(from, to); + if (err < 0) + return -errno; + return 0; +} + +int rename2_file(char *from, char *to, unsigned int flags) +{ + int err; + +#ifndef SYS_renameat2 +# ifdef __x86_64__ +# define SYS_renameat2 316 +# endif +# ifdef __i386__ +# define SYS_renameat2 353 +# endif +#endif + +#ifdef SYS_renameat2 + err = syscall(SYS_renameat2, AT_FDCWD, from, AT_FDCWD, to, flags); + if (err < 0) { + if (errno != ENOSYS) + return -errno; + else + return -EINVAL; + } + return 0; +#else + return -EINVAL; +#endif +} + +int do_statfs(char *root, long *bsize_out, long long *blocks_out, + long long *bfree_out, long long *bavail_out, + long long *files_out, long long *ffree_out, + void *fsid_out, int fsid_size, long *namelen_out) +{ + struct statfs64 buf; + int err; + + err = statfs64(root, &buf); + if (err < 0) + return -errno; + + *bsize_out = buf.f_bsize; + *blocks_out = buf.f_blocks; + *bfree_out = buf.f_bfree; + *bavail_out = buf.f_bavail; + *files_out = buf.f_files; + *ffree_out = buf.f_ffree; + memcpy(fsid_out, &buf.f_fsid, + sizeof(buf.f_fsid) > fsid_size ? fsid_size : + sizeof(buf.f_fsid)); + *namelen_out = buf.f_namelen; + + return 0; +} diff --git a/fs/hostfs/hostfs_user_exp.c b/fs/hostfs/hostfs_user_exp.c new file mode 100644 index 000000000..250c91c55 --- /dev/null +++ b/fs/hostfs/hostfs_user_exp.c @@ -0,0 +1,28 @@ +#include +#include "hostfs.h" + +EXPORT_SYMBOL_GPL(stat_file); +EXPORT_SYMBOL_GPL(access_file); +EXPORT_SYMBOL_GPL(open_file); +EXPORT_SYMBOL_GPL(open_dir); +EXPORT_SYMBOL_GPL(seek_dir); +EXPORT_SYMBOL_GPL(read_dir); +EXPORT_SYMBOL_GPL(read_file); +EXPORT_SYMBOL_GPL(write_file); +EXPORT_SYMBOL_GPL(lseek_file); +EXPORT_SYMBOL_GPL(fsync_file); +EXPORT_SYMBOL_GPL(replace_file); +EXPORT_SYMBOL_GPL(close_file); +EXPORT_SYMBOL_GPL(close_dir); +EXPORT_SYMBOL_GPL(file_create); +EXPORT_SYMBOL_GPL(set_attr); +EXPORT_SYMBOL_GPL(make_symlink); +EXPORT_SYMBOL_GPL(unlink_file); +EXPORT_SYMBOL_GPL(do_mkdir); +EXPORT_SYMBOL_GPL(hostfs_do_rmdir); +EXPORT_SYMBOL_GPL(do_mknod); +EXPORT_SYMBOL_GPL(link_file); +EXPORT_SYMBOL_GPL(hostfs_do_readlink); +EXPORT_SYMBOL_GPL(rename_file); +EXPORT_SYMBOL_GPL(rename2_file); +EXPORT_SYMBOL_GPL(do_statfs); diff --git a/fs/hpfs/Kconfig b/fs/hpfs/Kconfig new file mode 100644 index 000000000..ac1e9318e --- /dev/null +++ b/fs/hpfs/Kconfig @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: GPL-2.0-only +config HPFS_FS + tristate "OS/2 HPFS file system support" + depends on BLOCK + select BUFFER_HEAD + select FS_IOMAP + help + OS/2 is IBM's operating system for PC's, the same as Warp, and HPFS + is the file system used for organizing files on OS/2 hard disk + partitions. Say Y if you want to be able to read files from and + write files to an OS/2 HPFS partition on your hard drive. OS/2 + floppies however are in regular MSDOS format, so you don't need this + option in order to be able to read them. Read + . + + To compile this file system support as a module, choose M here: the + module will be called hpfs. If unsure, say N. diff --git a/fs/hpfs/Makefile b/fs/hpfs/Makefile new file mode 100644 index 000000000..153c17382 --- /dev/null +++ b/fs/hpfs/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the Linux hpfs filesystem routines. +# + +obj-$(CONFIG_HPFS_FS) += hpfs.o + +hpfs-objs := alloc.o anode.o buffer.o dentry.o dir.o dnode.o ea.o file.o \ + inode.o map.o name.o namei.o super.o diff --git a/fs/hpfs/alloc.c b/fs/hpfs/alloc.c new file mode 100644 index 000000000..f51503726 --- /dev/null +++ b/fs/hpfs/alloc.c @@ -0,0 +1,582 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hpfs/alloc.c + * + * Mikulas Patocka (mikulas@artax.karlin.mff.cuni.cz), 1998-1999 + * + * HPFS bitmap operations + */ + +#include "hpfs_fn.h" + +static void hpfs_claim_alloc(struct super_block *s, secno sec) +{ + struct hpfs_sb_info *sbi = hpfs_sb(s); + if (sbi->sb_n_free != (unsigned)-1) { + if (unlikely(!sbi->sb_n_free)) { + hpfs_error(s, "free count underflow, allocating sector %08x", sec); + sbi->sb_n_free = -1; + return; + } + sbi->sb_n_free--; + } +} + +static void hpfs_claim_free(struct super_block *s, secno sec) +{ + struct hpfs_sb_info *sbi = hpfs_sb(s); + if (sbi->sb_n_free != (unsigned)-1) { + if (unlikely(sbi->sb_n_free >= sbi->sb_fs_size)) { + hpfs_error(s, "free count overflow, freeing sector %08x", sec); + sbi->sb_n_free = -1; + return; + } + sbi->sb_n_free++; + } +} + +static void hpfs_claim_dirband_alloc(struct super_block *s, secno sec) +{ + struct hpfs_sb_info *sbi = hpfs_sb(s); + if (sbi->sb_n_free_dnodes != (unsigned)-1) { + if (unlikely(!sbi->sb_n_free_dnodes)) { + hpfs_error(s, "dirband free count underflow, allocating sector %08x", sec); + sbi->sb_n_free_dnodes = -1; + return; + } + sbi->sb_n_free_dnodes--; + } +} + +static void hpfs_claim_dirband_free(struct super_block *s, secno sec) +{ + struct hpfs_sb_info *sbi = hpfs_sb(s); + if (sbi->sb_n_free_dnodes != (unsigned)-1) { + if (unlikely(sbi->sb_n_free_dnodes >= sbi->sb_dirband_size / 4)) { + hpfs_error(s, "dirband free count overflow, freeing sector %08x", sec); + sbi->sb_n_free_dnodes = -1; + return; + } + sbi->sb_n_free_dnodes++; + } +} + +/* + * Check if a sector is allocated in bitmap + * This is really slow. Turned on only if chk==2 + */ + +static int chk_if_allocated(struct super_block *s, secno sec, char *msg) +{ + struct quad_buffer_head qbh; + __le32 *bmp; + if (!(bmp = hpfs_map_bitmap(s, sec >> 14, &qbh, "chk"))) goto fail; + if ((le32_to_cpu(bmp[(sec & 0x3fff) >> 5]) >> (sec & 0x1f)) & 1) { + hpfs_error(s, "sector '%s' - %08x not allocated in bitmap", msg, sec); + goto fail1; + } + hpfs_brelse4(&qbh); + if (sec >= hpfs_sb(s)->sb_dirband_start && sec < hpfs_sb(s)->sb_dirband_start + hpfs_sb(s)->sb_dirband_size) { + unsigned ssec = (sec - hpfs_sb(s)->sb_dirband_start) / 4; + if (!(bmp = hpfs_map_dnode_bitmap(s, &qbh))) goto fail; + if ((le32_to_cpu(bmp[ssec >> 5]) >> (ssec & 0x1f)) & 1) { + hpfs_error(s, "sector '%s' - %08x not allocated in directory bitmap", msg, sec); + goto fail1; + } + hpfs_brelse4(&qbh); + } + return 0; + fail1: + hpfs_brelse4(&qbh); + fail: + return 1; +} + +/* + * Check if sector(s) have proper number and additionally check if they're + * allocated in bitmap. + */ + +int hpfs_chk_sectors(struct super_block *s, secno start, int len, char *msg) +{ + if (start + len < start || start < 0x12 || + start + len > hpfs_sb(s)->sb_fs_size) { + hpfs_error(s, "sector(s) '%s' badly placed at %08x", msg, start); + return 1; + } + if (hpfs_sb(s)->sb_chk>=2) { + int i; + for (i = 0; i < len; i++) + if (chk_if_allocated(s, start + i, msg)) return 1; + } + return 0; +} + +static secno alloc_in_bmp(struct super_block *s, secno near, unsigned n, unsigned forward) +{ + struct quad_buffer_head qbh; + __le32 *bmp; + unsigned bs = near & ~0x3fff; + unsigned nr = (near & 0x3fff) & ~(n - 1); + /*unsigned mnr;*/ + unsigned i, q; + int a, b; + secno ret = 0; + if (n != 1 && n != 4) { + hpfs_error(s, "Bad allocation size: %d", n); + return 0; + } + if (bs != ~0x3fff) { + if (!(bmp = hpfs_map_bitmap(s, near >> 14, &qbh, "aib"))) goto uls; + } else { + if (!(bmp = hpfs_map_dnode_bitmap(s, &qbh))) goto uls; + } + if (!tstbits(bmp, nr, n + forward)) { + ret = bs + nr; + goto rt; + } + q = nr + n; b = 0; + while ((a = tstbits(bmp, q, n + forward)) != 0) { + q += a; + if (n != 1) q = ((q-1)&~(n-1))+n; + if (!b) { + if (q>>5 != nr>>5) { + b = 1; + q = nr & 0x1f; + } + } else if (q > nr) break; + } + if (!a) { + ret = bs + q; + goto rt; + } + nr >>= 5; + /*for (i = nr + 1; i != nr; i++, i &= 0x1ff) */ + i = nr; + do { + if (!le32_to_cpu(bmp[i])) goto cont; + if (n + forward >= 0x3f && le32_to_cpu(bmp[i]) != 0xffffffff) goto cont; + q = i<<5; + if (i > 0) { + unsigned k = le32_to_cpu(bmp[i-1]); + while (k & 0x80000000) { + q--; k <<= 1; + } + } + if (n != 1) q = ((q-1)&~(n-1))+n; + while ((a = tstbits(bmp, q, n + forward)) != 0) { + q += a; + if (n != 1) q = ((q-1)&~(n-1))+n; + if (q>>5 > i) break; + } + if (!a) { + ret = bs + q; + goto rt; + } + cont: + i++, i &= 0x1ff; + } while (i != nr); + rt: + if (ret) { + if (hpfs_sb(s)->sb_chk && ((ret >> 14) != (bs >> 14) || (le32_to_cpu(bmp[(ret & 0x3fff) >> 5]) | ~(((1 << n) - 1) << (ret & 0x1f))) != 0xffffffff)) { + hpfs_error(s, "Allocation doesn't work! Wanted %d, allocated at %08x", n, ret); + ret = 0; + goto b; + } + bmp[(ret & 0x3fff) >> 5] &= cpu_to_le32(~(((1 << n) - 1) << (ret & 0x1f))); + hpfs_mark_4buffers_dirty(&qbh); + } + b: + hpfs_brelse4(&qbh); + uls: + return ret; +} + +/* + * Allocation strategy: 1) search place near the sector specified + * 2) search bitmap where free sectors last found + * 3) search all bitmaps + * 4) search all bitmaps ignoring number of pre-allocated + * sectors + */ + +secno hpfs_alloc_sector(struct super_block *s, secno near, unsigned n, int forward) +{ + secno sec; + int i; + unsigned n_bmps; + struct hpfs_sb_info *sbi = hpfs_sb(s); + int f_p = 0; + int near_bmp; + if (forward < 0) { + forward = -forward; + f_p = 1; + } + n_bmps = (sbi->sb_fs_size + 0x4000 - 1) >> 14; + if (near && near < sbi->sb_fs_size) { + if ((sec = alloc_in_bmp(s, near, n, f_p ? forward : forward/4))) goto ret; + near_bmp = near >> 14; + } else near_bmp = n_bmps / 2; + /* + if (b != -1) { + if ((sec = alloc_in_bmp(s, b<<14, n, f_p ? forward : forward/2))) { + b &= 0x0fffffff; + goto ret; + } + if (b > 0x10000000) if ((sec = alloc_in_bmp(s, (b&0xfffffff)<<14, n, f_p ? forward : 0))) goto ret; + */ + if (!f_p) if (forward > sbi->sb_max_fwd_alloc) forward = sbi->sb_max_fwd_alloc; + less_fwd: + for (i = 0; i < n_bmps; i++) { + if (near_bmp+i < n_bmps && ((sec = alloc_in_bmp(s, (near_bmp+i) << 14, n, forward)))) { + sbi->sb_c_bitmap = near_bmp+i; + goto ret; + } + if (!forward) { + if (near_bmp-i-1 >= 0 && ((sec = alloc_in_bmp(s, (near_bmp-i-1) << 14, n, forward)))) { + sbi->sb_c_bitmap = near_bmp-i-1; + goto ret; + } + } else { + if (near_bmp+i >= n_bmps && ((sec = alloc_in_bmp(s, (near_bmp+i-n_bmps) << 14, n, forward)))) { + sbi->sb_c_bitmap = near_bmp+i-n_bmps; + goto ret; + } + } + if (i == 1 && sbi->sb_c_bitmap != -1 && ((sec = alloc_in_bmp(s, (sbi->sb_c_bitmap) << 14, n, forward)))) { + goto ret; + } + } + if (!f_p) { + if (forward) { + sbi->sb_max_fwd_alloc = forward * 3 / 4; + forward /= 2; + goto less_fwd; + } + } + sec = 0; + ret: + if (sec) { + i = 0; + do + hpfs_claim_alloc(s, sec + i); + while (unlikely(++i < n)); + } + if (sec && f_p) { + for (i = 0; i < forward; i++) { + if (!hpfs_alloc_if_possible(s, sec + n + i)) { + hpfs_error(s, "Prealloc doesn't work! Wanted %d, allocated at %08x, can't allocate %d", forward, sec, i); + sec = 0; + break; + } + } + } + return sec; +} + +static secno alloc_in_dirband(struct super_block *s, secno near) +{ + unsigned nr = near; + secno sec; + struct hpfs_sb_info *sbi = hpfs_sb(s); + if (nr < sbi->sb_dirband_start) + nr = sbi->sb_dirband_start; + if (nr >= sbi->sb_dirband_start + sbi->sb_dirband_size) + nr = sbi->sb_dirband_start + sbi->sb_dirband_size - 4; + nr -= sbi->sb_dirband_start; + nr >>= 2; + sec = alloc_in_bmp(s, (~0x3fff) | nr, 1, 0); + if (!sec) return 0; + hpfs_claim_dirband_alloc(s, sec); + return ((sec & 0x3fff) << 2) + sbi->sb_dirband_start; +} + +/* Alloc sector if it's free */ + +int hpfs_alloc_if_possible(struct super_block *s, secno sec) +{ + struct quad_buffer_head qbh; + __le32 *bmp; + if (!(bmp = hpfs_map_bitmap(s, sec >> 14, &qbh, "aip"))) goto end; + if (le32_to_cpu(bmp[(sec & 0x3fff) >> 5]) & (1 << (sec & 0x1f))) { + bmp[(sec & 0x3fff) >> 5] &= cpu_to_le32(~(1 << (sec & 0x1f))); + hpfs_mark_4buffers_dirty(&qbh); + hpfs_brelse4(&qbh); + hpfs_claim_alloc(s, sec); + return 1; + } + hpfs_brelse4(&qbh); + end: + return 0; +} + +/* Free sectors in bitmaps */ + +void hpfs_free_sectors(struct super_block *s, secno sec, unsigned n) +{ + struct quad_buffer_head qbh; + __le32 *bmp; + struct hpfs_sb_info *sbi = hpfs_sb(s); + /*pr_info("2 - ");*/ + if (!n) return; + if (sec < 0x12) { + hpfs_error(s, "Trying to free reserved sector %08x", sec); + return; + } + sbi->sb_max_fwd_alloc += n > 0xffff ? 0xffff : n; + if (sbi->sb_max_fwd_alloc > 0xffffff) sbi->sb_max_fwd_alloc = 0xffffff; + new_map: + if (!(bmp = hpfs_map_bitmap(s, sec >> 14, &qbh, "free"))) { + return; + } + new_tst: + if ((le32_to_cpu(bmp[(sec & 0x3fff) >> 5]) >> (sec & 0x1f) & 1)) { + hpfs_error(s, "sector %08x not allocated", sec); + hpfs_brelse4(&qbh); + return; + } + bmp[(sec & 0x3fff) >> 5] |= cpu_to_le32(1 << (sec & 0x1f)); + hpfs_claim_free(s, sec); + if (!--n) { + hpfs_mark_4buffers_dirty(&qbh); + hpfs_brelse4(&qbh); + return; + } + if (!(++sec & 0x3fff)) { + hpfs_mark_4buffers_dirty(&qbh); + hpfs_brelse4(&qbh); + goto new_map; + } + goto new_tst; +} + +/* + * Check if there are at least n free dnodes on the filesystem. + * Called before adding to dnode. If we run out of space while + * splitting dnodes, it would corrupt dnode tree. + */ + +int hpfs_check_free_dnodes(struct super_block *s, int n) +{ + int n_bmps = (hpfs_sb(s)->sb_fs_size + 0x4000 - 1) >> 14; + int b = hpfs_sb(s)->sb_c_bitmap & 0x0fffffff; + int i, j; + __le32 *bmp; + struct quad_buffer_head qbh; + if ((bmp = hpfs_map_dnode_bitmap(s, &qbh))) { + for (j = 0; j < 512; j++) { + unsigned k; + if (!le32_to_cpu(bmp[j])) continue; + for (k = le32_to_cpu(bmp[j]); k; k >>= 1) if (k & 1) if (!--n) { + hpfs_brelse4(&qbh); + return 0; + } + } + hpfs_brelse4(&qbh); + } + i = 0; + if (hpfs_sb(s)->sb_c_bitmap != -1) { + bmp = hpfs_map_bitmap(s, b, &qbh, "chkdn1"); + goto chk_bmp; + } + chk_next: + if (i == b) i++; + if (i >= n_bmps) return 1; + bmp = hpfs_map_bitmap(s, i, &qbh, "chkdn2"); + chk_bmp: + if (bmp) { + for (j = 0; j < 512; j++) { + u32 k; + if (!le32_to_cpu(bmp[j])) continue; + for (k = 0xf; k; k <<= 4) + if ((le32_to_cpu(bmp[j]) & k) == k) { + if (!--n) { + hpfs_brelse4(&qbh); + return 0; + } + } + } + hpfs_brelse4(&qbh); + } + i++; + goto chk_next; +} + +void hpfs_free_dnode(struct super_block *s, dnode_secno dno) +{ + if (hpfs_sb(s)->sb_chk) if (dno & 3) { + hpfs_error(s, "hpfs_free_dnode: dnode %08x not aligned", dno); + return; + } + if (dno < hpfs_sb(s)->sb_dirband_start || + dno >= hpfs_sb(s)->sb_dirband_start + hpfs_sb(s)->sb_dirband_size) { + hpfs_free_sectors(s, dno, 4); + } else { + struct quad_buffer_head qbh; + __le32 *bmp; + unsigned ssec = (dno - hpfs_sb(s)->sb_dirband_start) / 4; + if (!(bmp = hpfs_map_dnode_bitmap(s, &qbh))) { + return; + } + bmp[ssec >> 5] |= cpu_to_le32(1 << (ssec & 0x1f)); + hpfs_mark_4buffers_dirty(&qbh); + hpfs_brelse4(&qbh); + hpfs_claim_dirband_free(s, dno); + } +} + +struct dnode *hpfs_alloc_dnode(struct super_block *s, secno near, + dnode_secno *dno, struct quad_buffer_head *qbh) +{ + struct dnode *d; + if (hpfs_get_free_dnodes(s) > FREE_DNODES_ADD) { + if (!(*dno = alloc_in_dirband(s, near))) + if (!(*dno = hpfs_alloc_sector(s, near, 4, 0))) return NULL; + } else { + if (!(*dno = hpfs_alloc_sector(s, near, 4, 0))) + if (!(*dno = alloc_in_dirband(s, near))) return NULL; + } + if (!(d = hpfs_get_4sectors(s, *dno, qbh))) { + hpfs_free_dnode(s, *dno); + return NULL; + } + memset(d, 0, 2048); + d->magic = cpu_to_le32(DNODE_MAGIC); + d->first_free = cpu_to_le32(52); + d->dirent[0] = 32; + d->dirent[2] = 8; + d->dirent[30] = 1; + d->dirent[31] = 255; + d->self = cpu_to_le32(*dno); + return d; +} + +struct fnode *hpfs_alloc_fnode(struct super_block *s, secno near, fnode_secno *fno, + struct buffer_head **bh) +{ + struct fnode *f; + if (!(*fno = hpfs_alloc_sector(s, near, 1, FNODE_ALLOC_FWD))) return NULL; + if (!(f = hpfs_get_sector(s, *fno, bh))) { + hpfs_free_sectors(s, *fno, 1); + return NULL; + } + memset(f, 0, 512); + f->magic = cpu_to_le32(FNODE_MAGIC); + f->ea_offs = cpu_to_le16(0xc4); + f->btree.n_free_nodes = 8; + f->btree.first_free = cpu_to_le16(8); + return f; +} + +struct anode *hpfs_alloc_anode(struct super_block *s, secno near, anode_secno *ano, + struct buffer_head **bh) +{ + struct anode *a; + if (!(*ano = hpfs_alloc_sector(s, near, 1, ANODE_ALLOC_FWD))) return NULL; + if (!(a = hpfs_get_sector(s, *ano, bh))) { + hpfs_free_sectors(s, *ano, 1); + return NULL; + } + memset(a, 0, 512); + a->magic = cpu_to_le32(ANODE_MAGIC); + a->self = cpu_to_le32(*ano); + a->btree.n_free_nodes = 40; + a->btree.n_used_nodes = 0; + a->btree.first_free = cpu_to_le16(8); + return a; +} + +static unsigned find_run(__le32 *bmp, unsigned *idx) +{ + unsigned len; + while (tstbits(bmp, *idx, 1)) { + (*idx)++; + if (unlikely(*idx >= 0x4000)) + return 0; + } + len = 1; + while (!tstbits(bmp, *idx + len, 1)) + len++; + return len; +} + +static int do_trim(struct super_block *s, secno start, unsigned len, secno limit_start, secno limit_end, unsigned minlen, unsigned *result) +{ + int err; + secno end; + if (fatal_signal_pending(current)) + return -EINTR; + end = start + len; + if (start < limit_start) + start = limit_start; + if (end > limit_end) + end = limit_end; + if (start >= end) + return 0; + if (end - start < minlen) + return 0; + err = sb_issue_discard(s, start, end - start, GFP_NOFS, 0); + if (err) + return err; + *result += end - start; + return 0; +} + +int hpfs_trim_fs(struct super_block *s, u64 start, u64 end, u64 minlen, unsigned *result) +{ + int err = 0; + struct hpfs_sb_info *sbi = hpfs_sb(s); + unsigned idx, len, start_bmp, end_bmp; + __le32 *bmp; + struct quad_buffer_head qbh; + + *result = 0; + if (!end || end > sbi->sb_fs_size) + end = sbi->sb_fs_size; + if (start >= sbi->sb_fs_size) + return 0; + if (minlen > 0x4000) + return 0; + if (start < sbi->sb_dirband_start + sbi->sb_dirband_size && end > sbi->sb_dirband_start) { + hpfs_lock(s); + if (sb_rdonly(s)) { + err = -EROFS; + goto unlock_1; + } + if (!(bmp = hpfs_map_dnode_bitmap(s, &qbh))) { + err = -EIO; + goto unlock_1; + } + idx = 0; + while ((len = find_run(bmp, &idx)) && !err) { + err = do_trim(s, sbi->sb_dirband_start + idx * 4, len * 4, start, end, minlen, result); + idx += len; + } + hpfs_brelse4(&qbh); +unlock_1: + hpfs_unlock(s); + } + start_bmp = start >> 14; + end_bmp = (end + 0x3fff) >> 14; + while (start_bmp < end_bmp && !err) { + hpfs_lock(s); + if (sb_rdonly(s)) { + err = -EROFS; + goto unlock_2; + } + if (!(bmp = hpfs_map_bitmap(s, start_bmp, &qbh, "trim"))) { + err = -EIO; + goto unlock_2; + } + idx = 0; + while ((len = find_run(bmp, &idx)) && !err) { + err = do_trim(s, (start_bmp << 14) + idx, len, start, end, minlen, result); + idx += len; + } + hpfs_brelse4(&qbh); +unlock_2: + hpfs_unlock(s); + start_bmp++; + } + return err; +} diff --git a/fs/hpfs/anode.c b/fs/hpfs/anode.c new file mode 100644 index 000000000..a4f5321ea --- /dev/null +++ b/fs/hpfs/anode.c @@ -0,0 +1,498 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hpfs/anode.c + * + * Mikulas Patocka (mikulas@artax.karlin.mff.cuni.cz), 1998-1999 + * + * handling HPFS anode tree that contains file allocation info + */ + +#include "hpfs_fn.h" + +/* Find a sector in allocation tree */ + +secno hpfs_bplus_lookup(struct super_block *s, struct inode *inode, + struct bplus_header *btree, unsigned sec, + struct buffer_head *bh) +{ + anode_secno a = -1; + struct anode *anode; + int i; + int c1, c2 = 0; + go_down: + if (hpfs_sb(s)->sb_chk) if (hpfs_stop_cycles(s, a, &c1, &c2, "hpfs_bplus_lookup")) return -1; + if (bp_internal(btree)) { + for (i = 0; i < btree->n_used_nodes; i++) + if (le32_to_cpu(btree->u.internal[i].file_secno) > sec) { + a = le32_to_cpu(btree->u.internal[i].down); + brelse(bh); + if (!(anode = hpfs_map_anode(s, a, &bh))) return -1; + btree = GET_BTREE_PTR(&anode->btree); + goto go_down; + } + hpfs_error(s, "sector %08x not found in internal anode %08x", sec, a); + brelse(bh); + return -1; + } + for (i = 0; i < btree->n_used_nodes; i++) + if (le32_to_cpu(btree->u.external[i].file_secno) <= sec && + le32_to_cpu(btree->u.external[i].file_secno) + le32_to_cpu(btree->u.external[i].length) > sec) { + a = le32_to_cpu(btree->u.external[i].disk_secno) + sec - le32_to_cpu(btree->u.external[i].file_secno); + if (hpfs_sb(s)->sb_chk) if (hpfs_chk_sectors(s, a, 1, "data")) { + brelse(bh); + return -1; + } + if (inode) { + struct hpfs_inode_info *hpfs_inode = hpfs_i(inode); + hpfs_inode->i_file_sec = le32_to_cpu(btree->u.external[i].file_secno); + hpfs_inode->i_disk_sec = le32_to_cpu(btree->u.external[i].disk_secno); + hpfs_inode->i_n_secs = le32_to_cpu(btree->u.external[i].length); + } + brelse(bh); + return a; + } + hpfs_error(s, "sector %08x not found in external anode %08x", sec, a); + brelse(bh); + return -1; +} + +/* Add a sector to tree */ + +secno hpfs_add_sector_to_btree(struct super_block *s, secno node, int fnod, unsigned fsecno) +{ + struct bplus_header *btree; + struct anode *anode = NULL, *ranode = NULL; + struct fnode *fnode; + anode_secno a, na = -1, ra, up = -1; + secno se; + struct buffer_head *bh, *bh1, *bh2; + int n; + unsigned fs; + int c1, c2 = 0; + + if (fnod) { + if (!(fnode = hpfs_map_fnode(s, node, &bh))) return -1; + btree = GET_BTREE_PTR(&fnode->btree); + } else { + if (!(anode = hpfs_map_anode(s, node, &bh))) return -1; + btree = GET_BTREE_PTR(&anode->btree); + } + a = node; + go_down: + if ((n = btree->n_used_nodes - 1) < -!!fnod) { + hpfs_error(s, "anode %08x has no entries", a); + brelse(bh); + return -1; + } + if (bp_internal(btree)) { + a = le32_to_cpu(btree->u.internal[n].down); + btree->u.internal[n].file_secno = cpu_to_le32(-1); + mark_buffer_dirty(bh); + brelse(bh); + if (hpfs_sb(s)->sb_chk) + if (hpfs_stop_cycles(s, a, &c1, &c2, "hpfs_add_sector_to_btree #1")) return -1; + if (!(anode = hpfs_map_anode(s, a, &bh))) return -1; + btree = GET_BTREE_PTR(&anode->btree); + goto go_down; + } + if (n >= 0) { + if (le32_to_cpu(btree->u.external[n].file_secno) + le32_to_cpu(btree->u.external[n].length) != fsecno) { + hpfs_error(s, "allocated size %08x, trying to add sector %08x, %cnode %08x", + le32_to_cpu(btree->u.external[n].file_secno) + le32_to_cpu(btree->u.external[n].length), fsecno, + fnod?'f':'a', node); + brelse(bh); + return -1; + } + if (hpfs_alloc_if_possible(s, se = le32_to_cpu(btree->u.external[n].disk_secno) + le32_to_cpu(btree->u.external[n].length))) { + le32_add_cpu(&btree->u.external[n].length, 1); + mark_buffer_dirty(bh); + brelse(bh); + return se; + } + } else { + if (fsecno) { + hpfs_error(s, "empty file %08x, trying to add sector %08x", node, fsecno); + brelse(bh); + return -1; + } + se = !fnod ? node : (node + 16384) & ~16383; + } + if (!(se = hpfs_alloc_sector(s, se, 1, fsecno*ALLOC_M>ALLOC_FWD_MAX ? ALLOC_FWD_MAX : fsecno*ALLOC_Mu.external[n].file_secno) + le32_to_cpu(btree->u.external[n].length); + if (!btree->n_free_nodes) { + up = a != node ? le32_to_cpu(anode->up) : -1; + if (!(anode = hpfs_alloc_anode(s, a, &na, &bh1))) { + brelse(bh); + hpfs_free_sectors(s, se, 1); + return -1; + } + if (a == node && fnod) { + anode->up = cpu_to_le32(node); + anode->btree.flags |= BP_fnode_parent; + anode->btree.n_used_nodes = btree->n_used_nodes; + anode->btree.first_free = btree->first_free; + anode->btree.n_free_nodes = 40 - anode->btree.n_used_nodes; + memcpy(&anode->u, &btree->u, btree->n_used_nodes * 12); + btree->flags |= BP_internal; + btree->n_free_nodes = 11; + btree->n_used_nodes = 1; + btree->first_free = cpu_to_le16((char *)&(btree->u.internal[1]) - (char *)btree); + btree->u.internal[0].file_secno = cpu_to_le32(-1); + btree->u.internal[0].down = cpu_to_le32(na); + mark_buffer_dirty(bh); + } else if (!(ranode = hpfs_alloc_anode(s, /*a*/0, &ra, &bh2))) { + brelse(bh); + brelse(bh1); + hpfs_free_sectors(s, se, 1); + hpfs_free_sectors(s, na, 1); + return -1; + } + brelse(bh); + bh = bh1; + btree = GET_BTREE_PTR(&anode->btree); + } + btree->n_free_nodes--; n = btree->n_used_nodes++; + le16_add_cpu(&btree->first_free, 12); + btree->u.external[n].disk_secno = cpu_to_le32(se); + btree->u.external[n].file_secno = cpu_to_le32(fs); + btree->u.external[n].length = cpu_to_le32(1); + mark_buffer_dirty(bh); + brelse(bh); + if ((a == node && fnod) || na == -1) return se; + c2 = 0; + while (up != (anode_secno)-1) { + struct anode *new_anode; + if (hpfs_sb(s)->sb_chk) + if (hpfs_stop_cycles(s, up, &c1, &c2, "hpfs_add_sector_to_btree #2")) return -1; + if (up != node || !fnod) { + if (!(anode = hpfs_map_anode(s, up, &bh))) return -1; + btree = GET_BTREE_PTR(&anode->btree); + } else { + if (!(fnode = hpfs_map_fnode(s, up, &bh))) return -1; + btree = GET_BTREE_PTR(&fnode->btree); + } + if (btree->n_free_nodes) { + btree->n_free_nodes--; n = btree->n_used_nodes++; + le16_add_cpu(&btree->first_free, 8); + btree->u.internal[n].file_secno = cpu_to_le32(-1); + btree->u.internal[n].down = cpu_to_le32(na); + btree->u.internal[n-1].file_secno = cpu_to_le32(fs); + mark_buffer_dirty(bh); + brelse(bh); + brelse(bh2); + hpfs_free_sectors(s, ra, 1); + if ((anode = hpfs_map_anode(s, na, &bh))) { + anode->up = cpu_to_le32(up); + if (up == node && fnod) + anode->btree.flags |= BP_fnode_parent; + else + anode->btree.flags &= ~BP_fnode_parent; + mark_buffer_dirty(bh); + brelse(bh); + } + return se; + } + up = up != node ? le32_to_cpu(anode->up) : -1; + btree->u.internal[btree->n_used_nodes - 1].file_secno = cpu_to_le32(/*fs*/-1); + mark_buffer_dirty(bh); + brelse(bh); + a = na; + if ((new_anode = hpfs_alloc_anode(s, a, &na, &bh))) { + anode = new_anode; + /*anode->up = cpu_to_le32(up != -1 ? up : ra);*/ + anode->btree.flags |= BP_internal; + anode->btree.n_used_nodes = 1; + anode->btree.n_free_nodes = 59; + anode->btree.first_free = cpu_to_le16(16); + GET_BTREE_PTR(&anode->btree)->u.internal[0].down = cpu_to_le32(a); + GET_BTREE_PTR(&anode->btree)->u.internal[0].file_secno = cpu_to_le32(-1); + mark_buffer_dirty(bh); + brelse(bh); + if ((anode = hpfs_map_anode(s, a, &bh))) { + anode->up = cpu_to_le32(na); + mark_buffer_dirty(bh); + brelse(bh); + } + } else na = a; + } + if ((anode = hpfs_map_anode(s, na, &bh))) { + anode->up = cpu_to_le32(node); + if (fnod) + anode->btree.flags |= BP_fnode_parent; + mark_buffer_dirty(bh); + brelse(bh); + } + if (!fnod) { + if (!(anode = hpfs_map_anode(s, node, &bh))) { + brelse(bh2); + return -1; + } + btree = GET_BTREE_PTR(&anode->btree); + } else { + if (!(fnode = hpfs_map_fnode(s, node, &bh))) { + brelse(bh2); + return -1; + } + btree = GET_BTREE_PTR(&fnode->btree); + } + ranode->up = cpu_to_le32(node); + memcpy(&ranode->btree, btree, le16_to_cpu(btree->first_free)); + if (fnod) + ranode->btree.flags |= BP_fnode_parent; + GET_BTREE_PTR(&ranode->btree)->n_free_nodes = (bp_internal(GET_BTREE_PTR(&ranode->btree)) ? 60 : 40) - GET_BTREE_PTR(&ranode->btree)->n_used_nodes; + if (bp_internal(GET_BTREE_PTR(&ranode->btree))) for (n = 0; n < GET_BTREE_PTR(&ranode->btree)->n_used_nodes; n++) { + struct anode *unode; + if ((unode = hpfs_map_anode(s, le32_to_cpu(ranode->u.internal[n].down), &bh1))) { + unode->up = cpu_to_le32(ra); + unode->btree.flags &= ~BP_fnode_parent; + mark_buffer_dirty(bh1); + brelse(bh1); + } + } + btree->flags |= BP_internal; + btree->n_free_nodes = fnod ? 10 : 58; + btree->n_used_nodes = 2; + btree->first_free = cpu_to_le16((char *)&btree->u.internal[2] - (char *)btree); + btree->u.internal[0].file_secno = cpu_to_le32(fs); + btree->u.internal[0].down = cpu_to_le32(ra); + btree->u.internal[1].file_secno = cpu_to_le32(-1); + btree->u.internal[1].down = cpu_to_le32(na); + mark_buffer_dirty(bh); + brelse(bh); + mark_buffer_dirty(bh2); + brelse(bh2); + return se; +} + +/* + * Remove allocation tree. Recursion would look much nicer but + * I want to avoid it because it can cause stack overflow. + */ + +void hpfs_remove_btree(struct super_block *s, struct bplus_header *btree) +{ + struct bplus_header *btree1 = btree; + struct anode *anode = NULL; + anode_secno ano = 0, oano; + struct buffer_head *bh; + int level = 0; + int pos = 0; + int i; + int c1, c2 = 0; + int d1, d2; + go_down: + d2 = 0; + while (bp_internal(btree1)) { + ano = le32_to_cpu(btree1->u.internal[pos].down); + if (level) brelse(bh); + if (hpfs_sb(s)->sb_chk) + if (hpfs_stop_cycles(s, ano, &d1, &d2, "hpfs_remove_btree #1")) + return; + if (!(anode = hpfs_map_anode(s, ano, &bh))) return; + btree1 = GET_BTREE_PTR(&anode->btree); + level++; + pos = 0; + } + for (i = 0; i < btree1->n_used_nodes; i++) + hpfs_free_sectors(s, le32_to_cpu(btree1->u.external[i].disk_secno), le32_to_cpu(btree1->u.external[i].length)); + go_up: + if (!level) return; + brelse(bh); + if (hpfs_sb(s)->sb_chk) + if (hpfs_stop_cycles(s, ano, &c1, &c2, "hpfs_remove_btree #2")) return; + hpfs_free_sectors(s, ano, 1); + oano = ano; + ano = le32_to_cpu(anode->up); + if (--level) { + if (!(anode = hpfs_map_anode(s, ano, &bh))) return; + btree1 = GET_BTREE_PTR(&anode->btree); + } else btree1 = btree; + for (i = 0; i < btree1->n_used_nodes; i++) { + if (le32_to_cpu(btree1->u.internal[i].down) == oano) { + if ((pos = i + 1) < btree1->n_used_nodes) + goto go_down; + else + goto go_up; + } + } + hpfs_error(s, + "reference to anode %08x not found in anode %08x " + "(probably bad up pointer)", + oano, level ? ano : -1); + if (level) + brelse(bh); +} + +/* Just a wrapper around hpfs_bplus_lookup .. used for reading eas */ + +static secno anode_lookup(struct super_block *s, anode_secno a, unsigned sec) +{ + struct anode *anode; + struct buffer_head *bh; + if (!(anode = hpfs_map_anode(s, a, &bh))) return -1; + return hpfs_bplus_lookup(s, NULL, GET_BTREE_PTR(&anode->btree), sec, bh); +} + +int hpfs_ea_read(struct super_block *s, secno a, int ano, unsigned pos, + unsigned len, char *buf) +{ + struct buffer_head *bh; + char *data; + secno sec; + unsigned l; + while (len) { + if (ano) { + if ((sec = anode_lookup(s, a, pos >> 9)) == -1) + return -1; + } else sec = a + (pos >> 9); + if (hpfs_sb(s)->sb_chk) if (hpfs_chk_sectors(s, sec, 1, "ea #1")) return -1; + if (!(data = hpfs_map_sector(s, sec, &bh, (len - 1) >> 9))) + return -1; + l = 0x200 - (pos & 0x1ff); if (l > len) l = len; + memcpy(buf, data + (pos & 0x1ff), l); + brelse(bh); + buf += l; pos += l; len -= l; + } + return 0; +} + +int hpfs_ea_write(struct super_block *s, secno a, int ano, unsigned pos, + unsigned len, const char *buf) +{ + struct buffer_head *bh; + char *data; + secno sec; + unsigned l; + while (len) { + if (ano) { + if ((sec = anode_lookup(s, a, pos >> 9)) == -1) + return -1; + } else sec = a + (pos >> 9); + if (hpfs_sb(s)->sb_chk) if (hpfs_chk_sectors(s, sec, 1, "ea #2")) return -1; + if (!(data = hpfs_map_sector(s, sec, &bh, (len - 1) >> 9))) + return -1; + l = 0x200 - (pos & 0x1ff); if (l > len) l = len; + memcpy(data + (pos & 0x1ff), buf, l); + mark_buffer_dirty(bh); + brelse(bh); + buf += l; pos += l; len -= l; + } + return 0; +} + +void hpfs_ea_remove(struct super_block *s, secno a, int ano, unsigned len) +{ + struct anode *anode; + struct buffer_head *bh; + if (ano) { + if (!(anode = hpfs_map_anode(s, a, &bh))) return; + hpfs_remove_btree(s, GET_BTREE_PTR(&anode->btree)); + brelse(bh); + hpfs_free_sectors(s, a, 1); + } else hpfs_free_sectors(s, a, (len + 511) >> 9); +} + +/* Truncate allocation tree. Doesn't join anodes - I hope it doesn't matter */ + +void hpfs_truncate_btree(struct super_block *s, secno f, int fno, unsigned secs) +{ + struct fnode *fnode; + struct anode *anode; + struct buffer_head *bh; + struct bplus_header *btree; + anode_secno node = f; + int i, j, nodes; + int c1, c2 = 0; + if (fno) { + if (!(fnode = hpfs_map_fnode(s, f, &bh))) return; + btree = GET_BTREE_PTR(&fnode->btree); + } else { + if (!(anode = hpfs_map_anode(s, f, &bh))) return; + btree = GET_BTREE_PTR(&anode->btree); + } + if (!secs) { + hpfs_remove_btree(s, btree); + if (fno) { + btree->n_free_nodes = 8; + btree->n_used_nodes = 0; + btree->first_free = cpu_to_le16(8); + btree->flags &= ~BP_internal; + mark_buffer_dirty(bh); + } else hpfs_free_sectors(s, f, 1); + brelse(bh); + return; + } + while (bp_internal(btree)) { + nodes = btree->n_used_nodes + btree->n_free_nodes; + for (i = 0; i < btree->n_used_nodes; i++) + if (le32_to_cpu(btree->u.internal[i].file_secno) >= secs) goto f; + brelse(bh); + hpfs_error(s, "internal btree %08x doesn't end with -1", node); + return; + f: + for (j = i + 1; j < btree->n_used_nodes; j++) + hpfs_ea_remove(s, le32_to_cpu(btree->u.internal[j].down), 1, 0); + btree->n_used_nodes = i + 1; + btree->n_free_nodes = nodes - btree->n_used_nodes; + btree->first_free = cpu_to_le16(8 + 8 * btree->n_used_nodes); + mark_buffer_dirty(bh); + if (btree->u.internal[i].file_secno == cpu_to_le32(secs)) { + brelse(bh); + return; + } + node = le32_to_cpu(btree->u.internal[i].down); + brelse(bh); + if (hpfs_sb(s)->sb_chk) + if (hpfs_stop_cycles(s, node, &c1, &c2, "hpfs_truncate_btree")) + return; + if (!(anode = hpfs_map_anode(s, node, &bh))) return; + btree = GET_BTREE_PTR(&anode->btree); + } + nodes = btree->n_used_nodes + btree->n_free_nodes; + for (i = 0; i < btree->n_used_nodes; i++) + if (le32_to_cpu(btree->u.external[i].file_secno) + le32_to_cpu(btree->u.external[i].length) >= secs) goto ff; + brelse(bh); + return; + ff: + if (secs <= le32_to_cpu(btree->u.external[i].file_secno)) { + hpfs_error(s, "there is an allocation error in file %08x, sector %08x", f, secs); + if (i) i--; + } + else if (le32_to_cpu(btree->u.external[i].file_secno) + le32_to_cpu(btree->u.external[i].length) > secs) { + hpfs_free_sectors(s, le32_to_cpu(btree->u.external[i].disk_secno) + secs - + le32_to_cpu(btree->u.external[i].file_secno), le32_to_cpu(btree->u.external[i].length) + - secs + le32_to_cpu(btree->u.external[i].file_secno)); /* I hope gcc optimizes this :-) */ + btree->u.external[i].length = cpu_to_le32(secs - le32_to_cpu(btree->u.external[i].file_secno)); + } + for (j = i + 1; j < btree->n_used_nodes; j++) + hpfs_free_sectors(s, le32_to_cpu(btree->u.external[j].disk_secno), le32_to_cpu(btree->u.external[j].length)); + btree->n_used_nodes = i + 1; + btree->n_free_nodes = nodes - btree->n_used_nodes; + btree->first_free = cpu_to_le16(8 + 12 * btree->n_used_nodes); + mark_buffer_dirty(bh); + brelse(bh); +} + +/* Remove file or directory and it's eas - note that directory must + be empty when this is called. */ + +void hpfs_remove_fnode(struct super_block *s, fnode_secno fno) +{ + struct buffer_head *bh; + struct fnode *fnode; + struct extended_attribute *ea; + struct extended_attribute *ea_end; + if (!(fnode = hpfs_map_fnode(s, fno, &bh))) return; + if (!fnode_is_dir(fnode)) hpfs_remove_btree(s, GET_BTREE_PTR(&fnode->btree)); + else hpfs_remove_dtree(s, le32_to_cpu(fnode->u.external[0].disk_secno)); + ea_end = fnode_end_ea(fnode); + for (ea = fnode_ea(fnode); ea < ea_end; ea = next_ea(ea)) + if (ea_indirect(ea)) + hpfs_ea_remove(s, ea_sec(ea), ea_in_anode(ea), ea_len(ea)); + hpfs_ea_ext_remove(s, le32_to_cpu(fnode->ea_secno), fnode_in_anode(fnode), le32_to_cpu(fnode->ea_size_l)); + brelse(bh); + hpfs_free_sectors(s, fno, 1); +} diff --git a/fs/hpfs/buffer.c b/fs/hpfs/buffer.c new file mode 100644 index 000000000..d39246865 --- /dev/null +++ b/fs/hpfs/buffer.c @@ -0,0 +1,232 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hpfs/buffer.c + * + * Mikulas Patocka (mikulas@artax.karlin.mff.cuni.cz), 1998-1999 + * + * general buffer i/o + */ +#include +#include +#include +#include "hpfs_fn.h" + +secno hpfs_search_hotfix_map(struct super_block *s, secno sec) +{ + unsigned i; + struct hpfs_sb_info *sbi = hpfs_sb(s); + for (i = 0; unlikely(i < sbi->n_hotfixes); i++) { + if (sbi->hotfix_from[i] == sec) { + return sbi->hotfix_to[i]; + } + } + return sec; +} + +unsigned hpfs_search_hotfix_map_for_range(struct super_block *s, secno sec, unsigned n) +{ + unsigned i; + struct hpfs_sb_info *sbi = hpfs_sb(s); + for (i = 0; unlikely(i < sbi->n_hotfixes); i++) { + if (sbi->hotfix_from[i] >= sec && sbi->hotfix_from[i] < sec + n) { + n = sbi->hotfix_from[i] - sec; + } + } + return n; +} + +void hpfs_prefetch_sectors(struct super_block *s, unsigned secno, int n) +{ + struct buffer_head *bh; + struct blk_plug plug; + + if (n <= 0 || unlikely(secno >= hpfs_sb(s)->sb_fs_size)) + return; + + if (unlikely(hpfs_search_hotfix_map_for_range(s, secno, n) != n)) + return; + + bh = sb_find_get_block(s, secno); + if (bh) { + if (buffer_uptodate(bh)) { + brelse(bh); + return; + } + brelse(bh); + } + + blk_start_plug(&plug); + while (n > 0) { + if (unlikely(secno >= hpfs_sb(s)->sb_fs_size)) + break; + sb_breadahead(s, secno); + secno++; + n--; + } + blk_finish_plug(&plug); +} + +/* Map a sector into a buffer and return pointers to it and to the buffer. */ + +void *hpfs_map_sector(struct super_block *s, unsigned secno, struct buffer_head **bhp, + int ahead) +{ + struct buffer_head *bh; + + hpfs_lock_assert(s); + + hpfs_prefetch_sectors(s, secno, ahead); + + cond_resched(); + + *bhp = bh = sb_bread(s, hpfs_search_hotfix_map(s, secno)); + if (bh != NULL) + return bh->b_data; + else { + pr_err("%s(): read error\n", __func__); + return NULL; + } +} + +/* Like hpfs_map_sector but don't read anything */ + +void *hpfs_get_sector(struct super_block *s, unsigned secno, struct buffer_head **bhp) +{ + struct buffer_head *bh; + /*return hpfs_map_sector(s, secno, bhp, 0);*/ + + hpfs_lock_assert(s); + + cond_resched(); + + if ((*bhp = bh = sb_getblk(s, hpfs_search_hotfix_map(s, secno))) != NULL) { + if (!buffer_uptodate(bh)) wait_on_buffer(bh); + set_buffer_uptodate(bh); + return bh->b_data; + } else { + pr_err("%s(): getblk failed\n", __func__); + return NULL; + } +} + +/* Map 4 sectors into a 4buffer and return pointers to it and to the buffer. */ + +void *hpfs_map_4sectors(struct super_block *s, unsigned secno, struct quad_buffer_head *qbh, + int ahead) +{ + char *data; + + hpfs_lock_assert(s); + + cond_resched(); + + if (secno & 3) { + pr_err("%s(): unaligned read\n", __func__); + return NULL; + } + + hpfs_prefetch_sectors(s, secno, 4 + ahead); + + if (!hpfs_map_sector(s, secno + 0, &qbh->bh[0], 0)) goto bail0; + if (!hpfs_map_sector(s, secno + 1, &qbh->bh[1], 0)) goto bail1; + if (!hpfs_map_sector(s, secno + 2, &qbh->bh[2], 0)) goto bail2; + if (!hpfs_map_sector(s, secno + 3, &qbh->bh[3], 0)) goto bail3; + + if (likely(qbh->bh[1]->b_data == qbh->bh[0]->b_data + 1 * 512) && + likely(qbh->bh[2]->b_data == qbh->bh[0]->b_data + 2 * 512) && + likely(qbh->bh[3]->b_data == qbh->bh[0]->b_data + 3 * 512)) { + return qbh->data = qbh->bh[0]->b_data; + } + + qbh->data = data = kmalloc(2048, GFP_NOFS); + if (!data) { + pr_err("%s(): out of memory\n", __func__); + goto bail4; + } + + memcpy(data + 0 * 512, qbh->bh[0]->b_data, 512); + memcpy(data + 1 * 512, qbh->bh[1]->b_data, 512); + memcpy(data + 2 * 512, qbh->bh[2]->b_data, 512); + memcpy(data + 3 * 512, qbh->bh[3]->b_data, 512); + + return data; + + bail4: + brelse(qbh->bh[3]); + bail3: + brelse(qbh->bh[2]); + bail2: + brelse(qbh->bh[1]); + bail1: + brelse(qbh->bh[0]); + bail0: + return NULL; +} + +/* Don't read sectors */ + +void *hpfs_get_4sectors(struct super_block *s, unsigned secno, + struct quad_buffer_head *qbh) +{ + cond_resched(); + + hpfs_lock_assert(s); + + if (secno & 3) { + pr_err("%s(): unaligned read\n", __func__); + return NULL; + } + + if (!hpfs_get_sector(s, secno + 0, &qbh->bh[0])) goto bail0; + if (!hpfs_get_sector(s, secno + 1, &qbh->bh[1])) goto bail1; + if (!hpfs_get_sector(s, secno + 2, &qbh->bh[2])) goto bail2; + if (!hpfs_get_sector(s, secno + 3, &qbh->bh[3])) goto bail3; + + if (likely(qbh->bh[1]->b_data == qbh->bh[0]->b_data + 1 * 512) && + likely(qbh->bh[2]->b_data == qbh->bh[0]->b_data + 2 * 512) && + likely(qbh->bh[3]->b_data == qbh->bh[0]->b_data + 3 * 512)) { + return qbh->data = qbh->bh[0]->b_data; + } + + if (!(qbh->data = kmalloc(2048, GFP_NOFS))) { + pr_err("%s(): out of memory\n", __func__); + goto bail4; + } + return qbh->data; + +bail4: + brelse(qbh->bh[3]); +bail3: + brelse(qbh->bh[2]); +bail2: + brelse(qbh->bh[1]); +bail1: + brelse(qbh->bh[0]); +bail0: + return NULL; +} + + +void hpfs_brelse4(struct quad_buffer_head *qbh) +{ + if (unlikely(qbh->data != qbh->bh[0]->b_data)) + kfree(qbh->data); + brelse(qbh->bh[0]); + brelse(qbh->bh[1]); + brelse(qbh->bh[2]); + brelse(qbh->bh[3]); +} + +void hpfs_mark_4buffers_dirty(struct quad_buffer_head *qbh) +{ + if (unlikely(qbh->data != qbh->bh[0]->b_data)) { + memcpy(qbh->bh[0]->b_data, qbh->data + 0 * 512, 512); + memcpy(qbh->bh[1]->b_data, qbh->data + 1 * 512, 512); + memcpy(qbh->bh[2]->b_data, qbh->data + 2 * 512, 512); + memcpy(qbh->bh[3]->b_data, qbh->data + 3 * 512, 512); + } + mark_buffer_dirty(qbh->bh[0]); + mark_buffer_dirty(qbh->bh[1]); + mark_buffer_dirty(qbh->bh[2]); + mark_buffer_dirty(qbh->bh[3]); +} diff --git a/fs/hpfs/dentry.c b/fs/hpfs/dentry.c new file mode 100644 index 000000000..89a36fdc6 --- /dev/null +++ b/fs/hpfs/dentry.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hpfs/dentry.c + * + * Mikulas Patocka (mikulas@artax.karlin.mff.cuni.cz), 1998-1999 + * + * dcache operations + */ + +#include "hpfs_fn.h" + +/* + * Note: the dentry argument is the parent dentry. + */ + +static int hpfs_hash_dentry(const struct dentry *dentry, struct qstr *qstr) +{ + unsigned long hash; + int i; + unsigned l = qstr->len; + + if (l == 1) if (qstr->name[0]=='.') goto x; + if (l == 2) if (qstr->name[0]=='.' || qstr->name[1]=='.') goto x; + hpfs_adjust_length(qstr->name, &l); + /*if (hpfs_chk_name(qstr->name,&l))*/ + /*return -ENAMETOOLONG;*/ + /*return -ENOENT;*/ + x: + + hash = init_name_hash(dentry); + for (i = 0; i < l; i++) + hash = partial_name_hash(hpfs_upcase(hpfs_sb(dentry->d_sb)->sb_cp_table,qstr->name[i]), hash); + qstr->hash = end_name_hash(hash); + + return 0; +} + +static int hpfs_compare_dentry(const struct dentry *dentry, + unsigned int len, const char *str, const struct qstr *name) +{ + unsigned al = len; + unsigned bl = name->len; + + hpfs_adjust_length(str, &al); + /*hpfs_adjust_length(b->name, &bl);*/ + + /* + * 'str' is the nane of an already existing dentry, so the name + * must be valid. 'name' must be validated first. + */ + + if (hpfs_chk_name(name->name, &bl)) + return 1; + if (hpfs_compare_names(dentry->d_sb, str, al, name->name, bl, 0)) + return 1; + return 0; +} + +const struct dentry_operations hpfs_dentry_operations = { + .d_hash = hpfs_hash_dentry, + .d_compare = hpfs_compare_dentry, +}; diff --git a/fs/hpfs/dir.c b/fs/hpfs/dir.c new file mode 100644 index 000000000..3bf11202e --- /dev/null +++ b/fs/hpfs/dir.c @@ -0,0 +1,331 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hpfs/dir.c + * + * Mikulas Patocka (mikulas@artax.karlin.mff.cuni.cz), 1998-1999 + * + * directory VFS functions + */ + +#include +#include "hpfs_fn.h" + +static int hpfs_dir_release(struct inode *inode, struct file *filp) +{ + hpfs_lock(inode->i_sb); + hpfs_del_pos(inode, &filp->f_pos); + /*hpfs_write_if_changed(inode);*/ + hpfs_unlock(inode->i_sb); + return 0; +} + +/* This is slow, but it's not used often */ + +static loff_t hpfs_dir_lseek(struct file *filp, loff_t off, int whence) +{ + loff_t new_off = off + (whence == 1 ? filp->f_pos : 0); + loff_t pos; + struct quad_buffer_head qbh; + struct inode *i = file_inode(filp); + struct hpfs_inode_info *hpfs_inode = hpfs_i(i); + struct super_block *s = i->i_sb; + + /* Somebody else will have to figure out what to do here */ + if (whence == SEEK_DATA || whence == SEEK_HOLE) + return -EINVAL; + + inode_lock(i); + hpfs_lock(s); + + /*pr_info("dir lseek\n");*/ + if (new_off == 0 || new_off == 1 || new_off == 11 || new_off == 12 || new_off == 13) goto ok; + pos = ((loff_t) hpfs_de_as_down_as_possible(s, hpfs_inode->i_dno) << 4) + 1; + while (pos != new_off) { + if (map_pos_dirent(i, &pos, &qbh)) hpfs_brelse4(&qbh); + else goto fail; + if (pos == 12) goto fail; + } + if (unlikely(hpfs_add_pos(i, &filp->f_pos) < 0)) { + hpfs_unlock(s); + inode_unlock(i); + return -ENOMEM; + } +ok: + filp->f_pos = new_off; + hpfs_unlock(s); + inode_unlock(i); + return new_off; +fail: + /*pr_warn("illegal lseek: %016llx\n", new_off);*/ + hpfs_unlock(s); + inode_unlock(i); + return -ESPIPE; +} + +static int hpfs_readdir(struct file *file, struct dir_context *ctx) +{ + struct inode *inode = file_inode(file); + struct hpfs_inode_info *hpfs_inode = hpfs_i(inode); + struct quad_buffer_head qbh; + struct hpfs_dirent *de; + int lc; + loff_t next_pos; + unsigned char *tempname; + int c1, c2 = 0; + int ret = 0; + + hpfs_lock(inode->i_sb); + + if (hpfs_sb(inode->i_sb)->sb_chk) { + if (hpfs_chk_sectors(inode->i_sb, inode->i_ino, 1, "dir_fnode")) { + ret = -EFSERROR; + goto out; + } + if (hpfs_chk_sectors(inode->i_sb, hpfs_inode->i_dno, 4, "dir_dnode")) { + ret = -EFSERROR; + goto out; + } + } + if (hpfs_sb(inode->i_sb)->sb_chk >= 2) { + struct buffer_head *bh; + struct fnode *fno; + int e = 0; + if (!(fno = hpfs_map_fnode(inode->i_sb, inode->i_ino, &bh))) { + ret = -EIOERROR; + goto out; + } + if (!fnode_is_dir(fno)) { + e = 1; + hpfs_error(inode->i_sb, "not a directory, fnode %08llx", + inode->i_ino); + } + if (hpfs_inode->i_dno != le32_to_cpu(fno->u.external[0].disk_secno)) { + e = 1; + hpfs_error(inode->i_sb, "corrupted inode: i_dno == %08x, fnode -> dnode == %08x", hpfs_inode->i_dno, le32_to_cpu(fno->u.external[0].disk_secno)); + } + brelse(bh); + if (e) { + ret = -EFSERROR; + goto out; + } + } + lc = hpfs_sb(inode->i_sb)->sb_lowercase; + if (ctx->pos == 12) { /* diff -r requires this (note, that diff -r */ + ctx->pos = 13; /* also fails on msdos filesystem in 2.0) */ + goto out; + } + if (ctx->pos == 13) { + ret = -ENOENT; + goto out; + } + + while (1) { + again: + /* This won't work when cycle is longer than number of dirents + accepted by filldir, but what can I do? + maybe killall -9 ls helps */ + if (hpfs_sb(inode->i_sb)->sb_chk) + if (hpfs_stop_cycles(inode->i_sb, ctx->pos, &c1, &c2, "hpfs_readdir")) { + ret = -EFSERROR; + goto out; + } + if (ctx->pos == 12) + goto out; + if (ctx->pos == 3 || ctx->pos == 4 || ctx->pos == 5) { + pr_err("pos==%d\n", (int)ctx->pos); + goto out; + } + if (ctx->pos == 0) { + if (!dir_emit_dot(file, ctx)) + goto out; + ctx->pos = 11; + } + if (ctx->pos == 11) { + if (!dir_emit(ctx, "..", 2, hpfs_inode->i_parent_dir, DT_DIR)) + goto out; + ctx->pos = 1; + } + if (ctx->pos == 1) { + ret = hpfs_add_pos(inode, &file->f_pos); + if (unlikely(ret < 0)) + goto out; + ctx->pos = ((loff_t) hpfs_de_as_down_as_possible(inode->i_sb, hpfs_inode->i_dno) << 4) + 1; + } + next_pos = ctx->pos; + if (!(de = map_pos_dirent(inode, &next_pos, &qbh))) { + ctx->pos = next_pos; + ret = -EIOERROR; + goto out; + } + if (de->first || de->last) { + if (hpfs_sb(inode->i_sb)->sb_chk) { + if (de->first && !de->last && (de->namelen != 2 + || de ->name[0] != 1 || de->name[1] != 1)) + hpfs_error(inode->i_sb, "hpfs_readdir: bad ^A^A entry; pos = %08lx", (unsigned long)ctx->pos); + if (de->last && (de->namelen != 1 || de ->name[0] != 255)) + hpfs_error(inode->i_sb, "hpfs_readdir: bad \\377 entry; pos = %08lx", (unsigned long)ctx->pos); + } + hpfs_brelse4(&qbh); + ctx->pos = next_pos; + goto again; + } + tempname = hpfs_translate_name(inode->i_sb, de->name, de->namelen, lc, de->not_8x3); + if (!dir_emit(ctx, tempname, de->namelen, le32_to_cpu(de->fnode), DT_UNKNOWN)) { + if (tempname != de->name) kfree(tempname); + hpfs_brelse4(&qbh); + goto out; + } + ctx->pos = next_pos; + if (tempname != de->name) kfree(tempname); + hpfs_brelse4(&qbh); + } +out: + hpfs_unlock(inode->i_sb); + return ret; +} + +/* + * lookup. Search the specified directory for the specified name, set + * *result to the corresponding inode. + * + * lookup uses the inode number to tell read_inode whether it is reading + * the inode of a directory or a file -- file ino's are odd, directory + * ino's are even. read_inode avoids i/o for file inodes; everything + * needed is up here in the directory. (And file fnodes are out in + * the boondocks.) + * + * - M.P.: this is over, sometimes we've got to read file's fnode for eas + * inode numbers are just fnode sector numbers; iget lock is used + * to tell read_inode to read fnode or not. + */ + +struct dentry *hpfs_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) +{ + const unsigned char *name = dentry->d_name.name; + unsigned len = dentry->d_name.len; + struct quad_buffer_head qbh; + struct hpfs_dirent *de; + ino_t ino; + int err; + struct inode *result = NULL; + struct hpfs_inode_info *hpfs_result; + + hpfs_lock(dir->i_sb); + if ((err = hpfs_chk_name(name, &len))) { + if (err == -ENAMETOOLONG) { + hpfs_unlock(dir->i_sb); + return ERR_PTR(-ENAMETOOLONG); + } + goto end_add; + } + + /* + * '.' and '..' will never be passed here. + */ + + de = map_dirent(dir, hpfs_i(dir)->i_dno, name, len, NULL, &qbh); + + /* + * This is not really a bailout, just means file not found. + */ + + if (!de) goto end; + + /* + * Get inode number, what we're after. + */ + + ino = le32_to_cpu(de->fnode); + + /* + * Go find or make an inode. + */ + + result = iget_locked(dir->i_sb, ino); + if (!result) { + hpfs_error(dir->i_sb, "hpfs_lookup: can't get inode"); + result = ERR_PTR(-ENOMEM); + goto bail1; + } + if (inode_state_read_once(result) & I_NEW) { + hpfs_init_inode(result); + if (de->directory) + hpfs_read_inode(result); + else if (le32_to_cpu(de->ea_size) && hpfs_sb(dir->i_sb)->sb_eas) + hpfs_read_inode(result); + else { + result->i_mode |= S_IFREG; + result->i_mode &= ~0111; + result->i_op = &hpfs_file_iops; + result->i_fop = &hpfs_file_ops; + set_nlink(result, 1); + } + unlock_new_inode(result); + } + hpfs_result = hpfs_i(result); + if (!de->directory) hpfs_result->i_parent_dir = dir->i_ino; + + if (de->has_acl || de->has_xtd_perm) if (!sb_rdonly(dir->i_sb)) { + hpfs_error(result->i_sb, "ACLs or XPERM found. This is probably HPFS386. This driver doesn't support it now. Send me some info on these structures"); + iput(result); + result = ERR_PTR(-EINVAL); + goto bail1; + } + + /* + * Fill in the info from the directory if this is a newly created + * inode. + */ + + if (!inode_get_ctime_sec(result)) { + time64_t csec = local_to_gmt(dir->i_sb, le32_to_cpu(de->creation_date)); + + inode_set_ctime(result, csec ? csec : 1, 0); + inode_set_mtime(result, + local_to_gmt(dir->i_sb, le32_to_cpu(de->write_date)), + 0); + inode_set_atime(result, + local_to_gmt(dir->i_sb, le32_to_cpu(de->read_date)), + 0); + hpfs_result->i_ea_size = le32_to_cpu(de->ea_size); + if (!hpfs_result->i_ea_mode && de->read_only) + result->i_mode &= ~0222; + if (!de->directory) { + if (result->i_size == -1) { + result->i_size = le32_to_cpu(de->file_size); + result->i_data.a_ops = &hpfs_aops; + hpfs_i(result)->mmu_private = result->i_size; + /* + * i_blocks should count the fnode and any anodes. + * We count 1 for the fnode and don't bother about + * anodes -- the disk heads are on the directory band + * and we want them to stay there. + */ + result->i_blocks = 1 + ((result->i_size + 511) >> 9); + } + } + } + +bail1: + hpfs_brelse4(&qbh); + + /* + * Made it. + */ + +end: +end_add: + hpfs_unlock(dir->i_sb); + return d_splice_alias(result, dentry); +} + +const struct file_operations hpfs_dir_ops = +{ + .llseek = hpfs_dir_lseek, + .read = generic_read_dir, + .iterate_shared = hpfs_readdir, + .release = hpfs_dir_release, + .fsync = hpfs_file_fsync, + .unlocked_ioctl = hpfs_ioctl, + .compat_ioctl = compat_ptr_ioctl, +}; diff --git a/fs/hpfs/dnode.c b/fs/hpfs/dnode.c new file mode 100644 index 000000000..8c6aa060f --- /dev/null +++ b/fs/hpfs/dnode.c @@ -0,0 +1,1095 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hpfs/dnode.c + * + * Mikulas Patocka (mikulas@artax.karlin.mff.cuni.cz), 1998-1999 + * + * handling directory dnode tree - adding, deleteing & searching for dirents + */ + +#include "hpfs_fn.h" + +static loff_t get_pos(struct dnode *d, struct hpfs_dirent *fde) +{ + struct hpfs_dirent *de; + struct hpfs_dirent *de_end = dnode_end_de(d); + int i = 1; + for (de = dnode_first_de(d); de < de_end; de = de_next_de(de)) { + if (de == fde) return ((loff_t) le32_to_cpu(d->self) << 4) | (loff_t)i; + i++; + } + pr_info("%s(): not_found\n", __func__); + return ((loff_t)le32_to_cpu(d->self) << 4) | (loff_t)1; +} + +int hpfs_add_pos(struct inode *inode, loff_t *pos) +{ + struct hpfs_inode_info *hpfs_inode = hpfs_i(inode); + int i = 0; + loff_t **ppos; + + if (hpfs_inode->i_rddir_off) + for (; hpfs_inode->i_rddir_off[i]; i++) + if (hpfs_inode->i_rddir_off[i] == pos) + return 0; + if (!(i&0x0f)) { + ppos = kmalloc_objs(loff_t *, i + 0x11, GFP_NOFS); + if (!ppos) { + pr_err("out of memory for position list\n"); + return -ENOMEM; + } + if (hpfs_inode->i_rddir_off) { + memcpy(ppos, hpfs_inode->i_rddir_off, i * sizeof(loff_t)); + kfree(hpfs_inode->i_rddir_off); + } + hpfs_inode->i_rddir_off = ppos; + } + hpfs_inode->i_rddir_off[i] = pos; + hpfs_inode->i_rddir_off[i + 1] = NULL; + return 0; +} + +void hpfs_del_pos(struct inode *inode, loff_t *pos) +{ + struct hpfs_inode_info *hpfs_inode = hpfs_i(inode); + loff_t **i, **j; + + if (!hpfs_inode->i_rddir_off) goto not_f; + for (i = hpfs_inode->i_rddir_off; *i; i++) if (*i == pos) goto fnd; + goto not_f; + fnd: + for (j = i + 1; *j; j++) ; + *i = *(j - 1); + *(j - 1) = NULL; + if (j - 1 == hpfs_inode->i_rddir_off) { + kfree(hpfs_inode->i_rddir_off); + hpfs_inode->i_rddir_off = NULL; + } + return; + not_f: + /*pr_warn("position pointer %p->%08x not found\n", + pos, (int)*pos);*/ + return; +} + +static void for_all_poss(struct inode *inode, void (*f)(loff_t *, loff_t, loff_t), + loff_t p1, loff_t p2) +{ + struct hpfs_inode_info *hpfs_inode = hpfs_i(inode); + loff_t **i; + + if (!hpfs_inode->i_rddir_off) return; + for (i = hpfs_inode->i_rddir_off; *i; i++) (*f)(*i, p1, p2); + return; +} + +static void hpfs_pos_subst(loff_t *p, loff_t f, loff_t t) +{ + if (*p == f) *p = t; +} + +/*void hpfs_hpfs_pos_substd(loff_t *p, loff_t f, loff_t t) +{ + if ((*p & ~0x3f) == (f & ~0x3f)) *p = (t & ~0x3f) | (*p & 0x3f); +}*/ + +static void hpfs_pos_ins(loff_t *p, loff_t d, loff_t c) +{ + if ((*p & ~0x3f) == (d & ~0x3f) && (*p & 0x3f) >= (d & 0x3f)) { + int n = (*p & 0x3f) + c; + if (n > 0x3f) + pr_err("%s(): %08x + %d\n", + __func__, (int)*p, (int)c >> 8); + else + *p = (*p & ~0x3f) | n; + } +} + +static void hpfs_pos_del(loff_t *p, loff_t d, loff_t c) +{ + if ((*p & ~0x3f) == (d & ~0x3f) && (*p & 0x3f) >= (d & 0x3f)) { + int n = (*p & 0x3f) - c; + if (n < 1) + pr_err("%s(): %08x - %d\n", + __func__, (int)*p, (int)c >> 8); + else + *p = (*p & ~0x3f) | n; + } +} + +static struct hpfs_dirent *dnode_pre_last_de(struct dnode *d) +{ + struct hpfs_dirent *de, *de_end, *dee = NULL, *deee = NULL; + de_end = dnode_end_de(d); + for (de = dnode_first_de(d); de < de_end; de = de_next_de(de)) { + deee = dee; dee = de; + } + return deee; +} + +static struct hpfs_dirent *dnode_last_de(struct dnode *d) +{ + struct hpfs_dirent *de, *de_end, *dee = NULL; + de_end = dnode_end_de(d); + for (de = dnode_first_de(d); de < de_end; de = de_next_de(de)) { + dee = de; + } + return dee; +} + +static void set_last_pointer(struct super_block *s, struct dnode *d, dnode_secno ptr) +{ + struct hpfs_dirent *de; + if (!(de = dnode_last_de(d))) { + hpfs_error(s, "set_last_pointer: empty dnode %08x", le32_to_cpu(d->self)); + return; + } + if (hpfs_sb(s)->sb_chk) { + if (de->down) { + hpfs_error(s, "set_last_pointer: dnode %08x has already last pointer %08x", + le32_to_cpu(d->self), de_down_pointer(de)); + return; + } + if (le16_to_cpu(de->length) != 32) { + hpfs_error(s, "set_last_pointer: bad last dirent in dnode %08x", le32_to_cpu(d->self)); + return; + } + } + if (ptr) { + le32_add_cpu(&d->first_free, 4); + if (le32_to_cpu(d->first_free) > 2048) { + hpfs_error(s, "set_last_pointer: too long dnode %08x", le32_to_cpu(d->self)); + le32_add_cpu(&d->first_free, -4); + return; + } + de->length = cpu_to_le16(36); + de->down = 1; + *(__le32 *)((char *)de + 32) = cpu_to_le32(ptr); + } +} + +/* Add an entry to dnode and don't care if it grows over 2048 bytes */ + +struct hpfs_dirent *hpfs_add_de(struct super_block *s, struct dnode *d, + const unsigned char *name, + unsigned namelen, secno down_ptr) +{ + struct hpfs_dirent *de; + struct hpfs_dirent *de_end = dnode_end_de(d); + unsigned d_size = de_size(namelen, down_ptr); + for (de = dnode_first_de(d); de < de_end; de = de_next_de(de)) { + int c = hpfs_compare_names(s, name, namelen, de->name, de->namelen, de->last); + if (!c) { + hpfs_error(s, "name (%c,%d) already exists in dnode %08x", *name, namelen, le32_to_cpu(d->self)); + return NULL; + } + if (c < 0) break; + } + memmove((char *)de + d_size, de, (char *)de_end - (char *)de); + memset(de, 0, d_size); + if (down_ptr) { + *(__le32 *)((char *)de + d_size - 4) = cpu_to_le32(down_ptr); + de->down = 1; + } + de->length = cpu_to_le16(d_size); + de->not_8x3 = hpfs_is_name_long(name, namelen); + de->namelen = namelen; + memcpy(de->name, name, namelen); + le32_add_cpu(&d->first_free, d_size); + return de; +} + +/* Delete dirent and don't care about its subtree */ + +static void hpfs_delete_de(struct super_block *s, struct dnode *d, + struct hpfs_dirent *de) +{ + if (de->last) { + hpfs_error(s, "attempt to delete last dirent in dnode %08x", le32_to_cpu(d->self)); + return; + } + d->first_free = cpu_to_le32(le32_to_cpu(d->first_free) - le16_to_cpu(de->length)); + memmove(de, de_next_de(de), le32_to_cpu(d->first_free) + (char *)d - (char *)de); +} + +static void fix_up_ptrs(struct super_block *s, struct dnode *d) +{ + struct hpfs_dirent *de; + struct hpfs_dirent *de_end = dnode_end_de(d); + dnode_secno dno = le32_to_cpu(d->self); + for (de = dnode_first_de(d); de < de_end; de = de_next_de(de)) + if (de->down) { + struct quad_buffer_head qbh; + struct dnode *dd; + if ((dd = hpfs_map_dnode(s, de_down_pointer(de), &qbh))) { + if (le32_to_cpu(dd->up) != dno || dd->root_dnode) { + dd->up = cpu_to_le32(dno); + dd->root_dnode = 0; + hpfs_mark_4buffers_dirty(&qbh); + } + hpfs_brelse4(&qbh); + } + } +} + +/* Add an entry to dnode and do dnode splitting if required */ + +static int hpfs_add_to_dnode(struct inode *i, dnode_secno dno, + const unsigned char *name, unsigned namelen, + struct hpfs_dirent *new_de, dnode_secno down_ptr) +{ + struct quad_buffer_head qbh, qbh1, qbh2; + struct dnode *d, *ad, *rd, *nd = NULL; + dnode_secno adno, rdno; + struct hpfs_dirent *de; + struct hpfs_dirent nde; + unsigned char *nname; + int h; + int pos; + struct buffer_head *bh; + struct fnode *fnode; + int c1, c2 = 0; + if (!(nname = kmalloc(256, GFP_NOFS))) { + pr_err("out of memory, can't add to dnode\n"); + return 1; + } + go_up: + if (namelen >= 256) { + hpfs_error(i->i_sb, "%s(): namelen == %d", __func__, namelen); + kfree(nd); + kfree(nname); + return 1; + } + if (!(d = hpfs_map_dnode(i->i_sb, dno, &qbh))) { + kfree(nd); + kfree(nname); + return 1; + } + go_up_a: + if (hpfs_sb(i->i_sb)->sb_chk) + if (hpfs_stop_cycles(i->i_sb, dno, &c1, &c2, "hpfs_add_to_dnode")) { + hpfs_brelse4(&qbh); + kfree(nd); + kfree(nname); + return 1; + } + if (le32_to_cpu(d->first_free) + de_size(namelen, down_ptr) <= 2048) { + loff_t t; + copy_de(de=hpfs_add_de(i->i_sb, d, name, namelen, down_ptr), new_de); + t = get_pos(d, de); + for_all_poss(i, hpfs_pos_ins, t, 1); + for_all_poss(i, hpfs_pos_subst, 4, t); + for_all_poss(i, hpfs_pos_subst, 5, t + 1); + hpfs_mark_4buffers_dirty(&qbh); + hpfs_brelse4(&qbh); + kfree(nd); + kfree(nname); + return 0; + } + if (!nd) if (!(nd = kmalloc(0x924, GFP_NOFS))) { + /* 0x924 is a max size of dnode after adding a dirent with + max name length. We alloc this only once. There must + not be any error while splitting dnodes, otherwise the + whole directory, not only file we're adding, would + be lost. */ + pr_err("out of memory for dnode splitting\n"); + hpfs_brelse4(&qbh); + kfree(nname); + return 1; + } + memcpy(nd, d, le32_to_cpu(d->first_free)); + copy_de(de = hpfs_add_de(i->i_sb, nd, name, namelen, down_ptr), new_de); + for_all_poss(i, hpfs_pos_ins, get_pos(nd, de), 1); + h = ((char *)dnode_last_de(nd) - (char *)nd) / 2 + 10; + if (!(ad = hpfs_alloc_dnode(i->i_sb, le32_to_cpu(d->up), &adno, &qbh1))) { + hpfs_error(i->i_sb, "unable to alloc dnode - dnode tree will be corrupted"); + hpfs_brelse4(&qbh); + kfree(nd); + kfree(nname); + return 1; + } + i->i_size += 2048; + i->i_blocks += 4; + pos = 1; + for (de = dnode_first_de(nd); (char *)de_next_de(de) - (char *)nd < h; de = de_next_de(de)) { + copy_de(hpfs_add_de(i->i_sb, ad, de->name, de->namelen, de->down ? de_down_pointer(de) : 0), de); + for_all_poss(i, hpfs_pos_subst, ((loff_t)dno << 4) | pos, ((loff_t)adno << 4) | pos); + pos++; + } + copy_de(new_de = &nde, de); + memcpy(nname, de->name, de->namelen); + name = nname; + namelen = de->namelen; + for_all_poss(i, hpfs_pos_subst, ((loff_t)dno << 4) | pos, 4); + down_ptr = adno; + set_last_pointer(i->i_sb, ad, de->down ? de_down_pointer(de) : 0); + de = de_next_de(de); + memmove((char *)nd + 20, de, le32_to_cpu(nd->first_free) + (char *)nd - (char *)de); + le32_add_cpu(&nd->first_free, -((char *)de - (char *)nd - 20)); + memcpy(d, nd, le32_to_cpu(nd->first_free)); + for_all_poss(i, hpfs_pos_del, (loff_t)dno << 4, pos); + fix_up_ptrs(i->i_sb, ad); + if (!d->root_dnode) { + ad->up = d->up; + dno = le32_to_cpu(ad->up); + hpfs_mark_4buffers_dirty(&qbh); + hpfs_brelse4(&qbh); + hpfs_mark_4buffers_dirty(&qbh1); + hpfs_brelse4(&qbh1); + goto go_up; + } + if (!(rd = hpfs_alloc_dnode(i->i_sb, le32_to_cpu(d->up), &rdno, &qbh2))) { + hpfs_error(i->i_sb, "unable to alloc dnode - dnode tree will be corrupted"); + hpfs_brelse4(&qbh); + hpfs_brelse4(&qbh1); + kfree(nd); + kfree(nname); + return 1; + } + i->i_size += 2048; + i->i_blocks += 4; + rd->root_dnode = 1; + rd->up = d->up; + if (!(fnode = hpfs_map_fnode(i->i_sb, le32_to_cpu(d->up), &bh))) { + hpfs_free_dnode(i->i_sb, rdno); + hpfs_brelse4(&qbh); + hpfs_brelse4(&qbh1); + hpfs_brelse4(&qbh2); + kfree(nd); + kfree(nname); + return 1; + } + fnode->u.external[0].disk_secno = cpu_to_le32(rdno); + mark_buffer_dirty(bh); + brelse(bh); + hpfs_i(i)->i_dno = rdno; + d->up = ad->up = cpu_to_le32(rdno); + d->root_dnode = ad->root_dnode = 0; + hpfs_mark_4buffers_dirty(&qbh); + hpfs_brelse4(&qbh); + hpfs_mark_4buffers_dirty(&qbh1); + hpfs_brelse4(&qbh1); + qbh = qbh2; + set_last_pointer(i->i_sb, rd, dno); + dno = rdno; + d = rd; + goto go_up_a; +} + +/* + * Add an entry to directory btree. + * I hate such crazy directory structure. + * It's easy to read but terrible to write. + * I wrote this directory code 4 times. + * I hope, now it's finally bug-free. + */ + +int hpfs_add_dirent(struct inode *i, + const unsigned char *name, unsigned namelen, + struct hpfs_dirent *new_de) +{ + struct hpfs_inode_info *hpfs_inode = hpfs_i(i); + struct dnode *d; + struct hpfs_dirent *de, *de_end; + struct quad_buffer_head qbh; + dnode_secno dno; + int c; + int c1, c2 = 0; + dno = hpfs_inode->i_dno; + down: + if (hpfs_sb(i->i_sb)->sb_chk) + if (hpfs_stop_cycles(i->i_sb, dno, &c1, &c2, "hpfs_add_dirent")) return 1; + if (!(d = hpfs_map_dnode(i->i_sb, dno, &qbh))) return 1; + de_end = dnode_end_de(d); + for (de = dnode_first_de(d); de < de_end; de = de_next_de(de)) { + if (!(c = hpfs_compare_names(i->i_sb, name, namelen, de->name, de->namelen, de->last))) { + hpfs_brelse4(&qbh); + return -1; + } + if (c < 0) { + if (de->down) { + dno = de_down_pointer(de); + hpfs_brelse4(&qbh); + goto down; + } + break; + } + } + hpfs_brelse4(&qbh); + if (hpfs_check_free_dnodes(i->i_sb, FREE_DNODES_ADD)) { + c = 1; + goto ret; + } + c = hpfs_add_to_dnode(i, dno, name, namelen, new_de, 0); + ret: + return c; +} + +/* + * Find dirent with higher name in 'from' subtree and move it to 'to' dnode. + * Return the dnode we moved from (to be checked later if it's empty) + */ + +static secno move_to_top(struct inode *i, dnode_secno from, dnode_secno to) +{ + dnode_secno dno, ddno; + dnode_secno chk_up = to; + struct dnode *dnode; + struct quad_buffer_head qbh; + struct hpfs_dirent *de, *nde; + int a; + loff_t t; + int c1, c2 = 0; + dno = from; + while (1) { + if (hpfs_sb(i->i_sb)->sb_chk) + if (hpfs_stop_cycles(i->i_sb, dno, &c1, &c2, "move_to_top")) + return 0; + if (!(dnode = hpfs_map_dnode(i->i_sb, dno, &qbh))) return 0; + if (hpfs_sb(i->i_sb)->sb_chk) { + if (le32_to_cpu(dnode->up) != chk_up) { + hpfs_error(i->i_sb, "move_to_top: up pointer from %08x should be %08x, is %08x", + dno, chk_up, le32_to_cpu(dnode->up)); + hpfs_brelse4(&qbh); + return 0; + } + chk_up = dno; + } + if (!(de = dnode_last_de(dnode))) { + hpfs_error(i->i_sb, "move_to_top: dnode %08x has no last de", dno); + hpfs_brelse4(&qbh); + return 0; + } + if (!de->down) break; + dno = de_down_pointer(de); + hpfs_brelse4(&qbh); + } + while (!(de = dnode_pre_last_de(dnode))) { + dnode_secno up = le32_to_cpu(dnode->up); + hpfs_brelse4(&qbh); + hpfs_free_dnode(i->i_sb, dno); + i->i_size -= 2048; + i->i_blocks -= 4; + for_all_poss(i, hpfs_pos_subst, ((loff_t)dno << 4) | 1, 5); + if (up == to) return to; + if (!(dnode = hpfs_map_dnode(i->i_sb, up, &qbh))) return 0; + if (dnode->root_dnode) { + hpfs_error(i->i_sb, "move_to_top: got to root_dnode while moving from %08x to %08x", from, to); + hpfs_brelse4(&qbh); + return 0; + } + de = dnode_last_de(dnode); + if (!de || !de->down) { + hpfs_error(i->i_sb, "move_to_top: dnode %08x doesn't point down to %08x", up, dno); + hpfs_brelse4(&qbh); + return 0; + } + le32_add_cpu(&dnode->first_free, -4); + le16_add_cpu(&de->length, -4); + de->down = 0; + hpfs_mark_4buffers_dirty(&qbh); + dno = up; + } + t = get_pos(dnode, de); + for_all_poss(i, hpfs_pos_subst, t, 4); + for_all_poss(i, hpfs_pos_subst, t + 1, 5); + if (!(nde = kmalloc(le16_to_cpu(de->length), GFP_NOFS))) { + hpfs_error(i->i_sb, "out of memory for dirent - directory will be corrupted"); + hpfs_brelse4(&qbh); + return 0; + } + memcpy(nde, de, le16_to_cpu(de->length)); + ddno = de->down ? de_down_pointer(de) : 0; + hpfs_delete_de(i->i_sb, dnode, de); + set_last_pointer(i->i_sb, dnode, ddno); + hpfs_mark_4buffers_dirty(&qbh); + hpfs_brelse4(&qbh); + a = hpfs_add_to_dnode(i, to, nde->name, nde->namelen, nde, from); + kfree(nde); + if (a) return 0; + return dno; +} + +/* + * Check if a dnode is empty and delete it from the tree + * (chkdsk doesn't like empty dnodes) + */ + +static void delete_empty_dnode(struct inode *i, dnode_secno dno) +{ + struct hpfs_inode_info *hpfs_inode = hpfs_i(i); + struct quad_buffer_head qbh; + struct dnode *dnode; + dnode_secno down, up, ndown; + int p; + struct hpfs_dirent *de; + int c1, c2 = 0; + try_it_again: + if (hpfs_stop_cycles(i->i_sb, dno, &c1, &c2, "delete_empty_dnode")) return; + if (!(dnode = hpfs_map_dnode(i->i_sb, dno, &qbh))) return; + if (le32_to_cpu(dnode->first_free) > 56) goto end; + if (le32_to_cpu(dnode->first_free) == 52 || le32_to_cpu(dnode->first_free) == 56) { + struct hpfs_dirent *de_end; + int root = dnode->root_dnode; + up = le32_to_cpu(dnode->up); + de = dnode_first_de(dnode); + down = de->down ? de_down_pointer(de) : 0; + if (hpfs_sb(i->i_sb)->sb_chk) if (root && !down) { + hpfs_error(i->i_sb, "delete_empty_dnode: root dnode %08x is empty", dno); + goto end; + } + hpfs_brelse4(&qbh); + hpfs_free_dnode(i->i_sb, dno); + i->i_size -= 2048; + i->i_blocks -= 4; + if (root) { + struct fnode *fnode; + struct buffer_head *bh; + struct dnode *d1; + struct quad_buffer_head qbh1; + if (hpfs_sb(i->i_sb)->sb_chk) + if (up != i->i_ino) { + hpfs_error(i->i_sb, + "bad pointer to fnode, dnode %08x, pointing to %08x, should be %08llx", + dno, up, + i->i_ino); + return; + } + if ((d1 = hpfs_map_dnode(i->i_sb, down, &qbh1))) { + d1->up = cpu_to_le32(up); + d1->root_dnode = 1; + hpfs_mark_4buffers_dirty(&qbh1); + hpfs_brelse4(&qbh1); + } + if ((fnode = hpfs_map_fnode(i->i_sb, up, &bh))) { + fnode->u.external[0].disk_secno = cpu_to_le32(down); + mark_buffer_dirty(bh); + brelse(bh); + } + hpfs_inode->i_dno = down; + for_all_poss(i, hpfs_pos_subst, ((loff_t)dno << 4) | 1, (loff_t) 12); + return; + } + if (!(dnode = hpfs_map_dnode(i->i_sb, up, &qbh))) return; + p = 1; + de_end = dnode_end_de(dnode); + for (de = dnode_first_de(dnode); de < de_end; de = de_next_de(de), p++) + if (de->down) if (de_down_pointer(de) == dno) goto fnd; + hpfs_error(i->i_sb, "delete_empty_dnode: pointer to dnode %08x not found in dnode %08x", dno, up); + goto end; + fnd: + for_all_poss(i, hpfs_pos_subst, ((loff_t)dno << 4) | 1, ((loff_t)up << 4) | p); + if (!down) { + de->down = 0; + le16_add_cpu(&de->length, -4); + le32_add_cpu(&dnode->first_free, -4); + memmove(de_next_de(de), (char *)de_next_de(de) + 4, + (char *)dnode + le32_to_cpu(dnode->first_free) - (char *)de_next_de(de)); + } else { + struct dnode *d1; + struct quad_buffer_head qbh1; + *(dnode_secno *) ((void *) de + le16_to_cpu(de->length) - 4) = down; + if ((d1 = hpfs_map_dnode(i->i_sb, down, &qbh1))) { + d1->up = cpu_to_le32(up); + hpfs_mark_4buffers_dirty(&qbh1); + hpfs_brelse4(&qbh1); + } + } + } else { + hpfs_error(i->i_sb, "delete_empty_dnode: dnode %08x, first_free == %03x", dno, le32_to_cpu(dnode->first_free)); + goto end; + } + + if (!de->last) { + struct hpfs_dirent *de_next = de_next_de(de); + struct hpfs_dirent *de_cp; + struct dnode *d1; + struct quad_buffer_head qbh1; + if (!de_next->down) goto endm; + ndown = de_down_pointer(de_next); + if (!(de_cp = kmalloc(le16_to_cpu(de->length), GFP_NOFS))) { + pr_err("out of memory for dtree balancing\n"); + goto endm; + } + memcpy(de_cp, de, le16_to_cpu(de->length)); + hpfs_delete_de(i->i_sb, dnode, de); + hpfs_mark_4buffers_dirty(&qbh); + hpfs_brelse4(&qbh); + for_all_poss(i, hpfs_pos_subst, ((loff_t)up << 4) | p, 4); + for_all_poss(i, hpfs_pos_del, ((loff_t)up << 4) | p, 1); + if (de_cp->down) if ((d1 = hpfs_map_dnode(i->i_sb, de_down_pointer(de_cp), &qbh1))) { + d1->up = cpu_to_le32(ndown); + hpfs_mark_4buffers_dirty(&qbh1); + hpfs_brelse4(&qbh1); + } + hpfs_add_to_dnode(i, ndown, de_cp->name, de_cp->namelen, de_cp, de_cp->down ? de_down_pointer(de_cp) : 0); + /*pr_info("UP-TO-DNODE: %08x (ndown = %08x, down = %08x, dno = %08x)\n", + up, ndown, down, dno);*/ + dno = up; + kfree(de_cp); + goto try_it_again; + } else { + struct hpfs_dirent *de_prev = dnode_pre_last_de(dnode); + struct hpfs_dirent *de_cp; + struct dnode *d1; + struct quad_buffer_head qbh1; + dnode_secno dlp; + if (!de_prev) { + hpfs_error(i->i_sb, "delete_empty_dnode: empty dnode %08x", up); + hpfs_mark_4buffers_dirty(&qbh); + hpfs_brelse4(&qbh); + dno = up; + goto try_it_again; + } + if (!de_prev->down) goto endm; + ndown = de_down_pointer(de_prev); + if ((d1 = hpfs_map_dnode(i->i_sb, ndown, &qbh1))) { + struct hpfs_dirent *del = dnode_last_de(d1); + dlp = del->down ? de_down_pointer(del) : 0; + if (!dlp && down) { + if (le32_to_cpu(d1->first_free) > 2044) { + if (hpfs_sb(i->i_sb)->sb_chk >= 2) { + pr_err("unbalanced dnode tree, see hpfs.txt 4 more info\n"); + pr_err("terminating balancing operation\n"); + } + hpfs_brelse4(&qbh1); + goto endm; + } + if (hpfs_sb(i->i_sb)->sb_chk >= 2) { + pr_err("unbalanced dnode tree, see hpfs.txt 4 more info\n"); + pr_err("goin'on\n"); + } + le16_add_cpu(&del->length, 4); + del->down = 1; + le32_add_cpu(&d1->first_free, 4); + } + if (dlp && !down) { + le16_add_cpu(&del->length, -4); + del->down = 0; + le32_add_cpu(&d1->first_free, -4); + } else if (down) + *(__le32 *) ((void *) del + le16_to_cpu(del->length) - 4) = cpu_to_le32(down); + } else goto endm; + if (!(de_cp = kmalloc(le16_to_cpu(de_prev->length), GFP_NOFS))) { + pr_err("out of memory for dtree balancing\n"); + hpfs_brelse4(&qbh1); + goto endm; + } + hpfs_mark_4buffers_dirty(&qbh1); + hpfs_brelse4(&qbh1); + memcpy(de_cp, de_prev, le16_to_cpu(de_prev->length)); + hpfs_delete_de(i->i_sb, dnode, de_prev); + if (!de_prev->down) { + le16_add_cpu(&de_prev->length, 4); + de_prev->down = 1; + le32_add_cpu(&dnode->first_free, 4); + } + *(__le32 *) ((void *) de_prev + le16_to_cpu(de_prev->length) - 4) = cpu_to_le32(ndown); + hpfs_mark_4buffers_dirty(&qbh); + hpfs_brelse4(&qbh); + for_all_poss(i, hpfs_pos_subst, ((loff_t)up << 4) | (p - 1), 4); + for_all_poss(i, hpfs_pos_subst, ((loff_t)up << 4) | p, ((loff_t)up << 4) | (p - 1)); + if (down) if ((d1 = hpfs_map_dnode(i->i_sb, de_down_pointer(de), &qbh1))) { + d1->up = cpu_to_le32(ndown); + hpfs_mark_4buffers_dirty(&qbh1); + hpfs_brelse4(&qbh1); + } + hpfs_add_to_dnode(i, ndown, de_cp->name, de_cp->namelen, de_cp, dlp); + dno = up; + kfree(de_cp); + goto try_it_again; + } + endm: + hpfs_mark_4buffers_dirty(&qbh); + end: + hpfs_brelse4(&qbh); +} + + +/* Delete dirent from directory */ + +int hpfs_remove_dirent(struct inode *i, dnode_secno dno, struct hpfs_dirent *de, + struct quad_buffer_head *qbh, int depth) +{ + struct dnode *dnode = qbh->data; + dnode_secno down = 0; + loff_t t; + if (de->first || de->last) { + hpfs_error(i->i_sb, "hpfs_remove_dirent: attempt to delete first or last dirent in dnode %08x", dno); + hpfs_brelse4(qbh); + return 1; + } + if (de->down) down = de_down_pointer(de); + if (depth && (de->down || (de == dnode_first_de(dnode) && de_next_de(de)->last))) { + if (hpfs_check_free_dnodes(i->i_sb, FREE_DNODES_DEL)) { + hpfs_brelse4(qbh); + return 2; + } + } + for_all_poss(i, hpfs_pos_del, (t = get_pos(dnode, de)) + 1, 1); + hpfs_delete_de(i->i_sb, dnode, de); + hpfs_mark_4buffers_dirty(qbh); + hpfs_brelse4(qbh); + if (down) { + dnode_secno a = move_to_top(i, down, dno); + for_all_poss(i, hpfs_pos_subst, 5, t); + if (a) delete_empty_dnode(i, a); + return !a; + } + delete_empty_dnode(i, dno); + return 0; +} + +void hpfs_count_dnodes(struct super_block *s, dnode_secno dno, int *n_dnodes, + int *n_subdirs, int *n_items) +{ + struct dnode *dnode; + struct quad_buffer_head qbh; + struct hpfs_dirent *de; + dnode_secno ptr, odno = 0; + int c1, c2 = 0; + int d1, d2 = 0; + go_down: + if (n_dnodes) (*n_dnodes)++; + if (hpfs_sb(s)->sb_chk) + if (hpfs_stop_cycles(s, dno, &c1, &c2, "hpfs_count_dnodes #1")) return; + ptr = 0; + go_up: + if (!(dnode = hpfs_map_dnode(s, dno, &qbh))) return; + if (hpfs_sb(s)->sb_chk) if (odno && odno != -1 && le32_to_cpu(dnode->up) != odno) + hpfs_error(s, "hpfs_count_dnodes: bad up pointer; dnode %08x, down %08x points to %08x", odno, dno, le32_to_cpu(dnode->up)); + de = dnode_first_de(dnode); + if (ptr) while(1) { + if (de->down) if (de_down_pointer(de) == ptr) goto process_de; + if (de->last) { + hpfs_brelse4(&qbh); + hpfs_error(s, "hpfs_count_dnodes: pointer to dnode %08x not found in dnode %08x, got here from %08x", + ptr, dno, odno); + return; + } + de = de_next_de(de); + } + next_de: + if (de->down) { + odno = dno; + dno = de_down_pointer(de); + hpfs_brelse4(&qbh); + goto go_down; + } + process_de: + if (!de->first && !de->last && de->directory && n_subdirs) (*n_subdirs)++; + if (!de->first && !de->last && n_items) (*n_items)++; + if ((de = de_next_de(de)) < dnode_end_de(dnode)) goto next_de; + ptr = dno; + dno = le32_to_cpu(dnode->up); + if (dnode->root_dnode) { + hpfs_brelse4(&qbh); + return; + } + hpfs_brelse4(&qbh); + if (hpfs_sb(s)->sb_chk) + if (hpfs_stop_cycles(s, ptr, &d1, &d2, "hpfs_count_dnodes #2")) return; + odno = -1; + goto go_up; +} + +static struct hpfs_dirent *map_nth_dirent(struct super_block *s, dnode_secno dno, int n, + struct quad_buffer_head *qbh, struct dnode **dn) +{ + int i; + struct hpfs_dirent *de, *de_end; + struct dnode *dnode; + dnode = hpfs_map_dnode(s, dno, qbh); + if (!dnode) return NULL; + if (dn) *dn=dnode; + de = dnode_first_de(dnode); + de_end = dnode_end_de(dnode); + for (i = 1; de < de_end; i++, de = de_next_de(de)) { + if (i == n) { + return de; + } + if (de->last) break; + } + hpfs_brelse4(qbh); + hpfs_error(s, "map_nth_dirent: n too high; dnode = %08x, requested %08x", dno, n); + return NULL; +} + +dnode_secno hpfs_de_as_down_as_possible(struct super_block *s, dnode_secno dno) +{ + struct quad_buffer_head qbh; + dnode_secno d = dno; + dnode_secno up = 0; + struct hpfs_dirent *de; + int c1, c2 = 0; + + again: + if (hpfs_sb(s)->sb_chk) + if (hpfs_stop_cycles(s, d, &c1, &c2, "hpfs_de_as_down_as_possible")) + return d; + if (!(de = map_nth_dirent(s, d, 1, &qbh, NULL))) return dno; + if (hpfs_sb(s)->sb_chk) + if (up && le32_to_cpu(((struct dnode *)qbh.data)->up) != up) + hpfs_error(s, "hpfs_de_as_down_as_possible: bad up pointer; dnode %08x, down %08x points to %08x", up, d, le32_to_cpu(((struct dnode *)qbh.data)->up)); + if (!de->down) { + hpfs_brelse4(&qbh); + return d; + } + up = d; + d = de_down_pointer(de); + hpfs_brelse4(&qbh); + goto again; +} + +struct hpfs_dirent *map_pos_dirent(struct inode *inode, loff_t *posp, + struct quad_buffer_head *qbh) +{ + loff_t pos; + unsigned c; + dnode_secno dno; + struct hpfs_dirent *de, *d; + struct hpfs_dirent *up_de; + struct hpfs_dirent *end_up_de; + struct dnode *dnode; + struct dnode *up_dnode; + struct quad_buffer_head qbh0; + + pos = *posp; + dno = pos >> 6 << 2; + pos &= 077; + if (!(de = map_nth_dirent(inode->i_sb, dno, pos, qbh, &dnode))) + goto bail; + + /* Going to the next dirent */ + if ((d = de_next_de(de)) < dnode_end_de(dnode)) { + if (!(++*posp & 077)) { + hpfs_error(inode->i_sb, + "map_pos_dirent: pos crossed dnode boundary; pos = %08llx", + (unsigned long long)*posp); + goto bail; + } + /* We're going down the tree */ + if (d->down) { + *posp = ((loff_t) hpfs_de_as_down_as_possible(inode->i_sb, de_down_pointer(d)) << 4) + 1; + } + + return de; + } + + /* Going up */ + if (dnode->root_dnode) goto bail; + + if (!(up_dnode = hpfs_map_dnode(inode->i_sb, le32_to_cpu(dnode->up), &qbh0))) + goto bail; + + end_up_de = dnode_end_de(up_dnode); + c = 0; + for (up_de = dnode_first_de(up_dnode); up_de < end_up_de; + up_de = de_next_de(up_de)) { + if (!(++c & 077)) hpfs_error(inode->i_sb, + "map_pos_dirent: pos crossed dnode boundary; dnode = %08x", le32_to_cpu(dnode->up)); + if (up_de->down && de_down_pointer(up_de) == dno) { + *posp = ((loff_t) le32_to_cpu(dnode->up) << 4) + c; + hpfs_brelse4(&qbh0); + return de; + } + } + + hpfs_error(inode->i_sb, "map_pos_dirent: pointer to dnode %08x not found in parent dnode %08x", + dno, le32_to_cpu(dnode->up)); + hpfs_brelse4(&qbh0); + + bail: + *posp = 12; + return de; +} + +/* Find a dirent in tree */ + +struct hpfs_dirent *map_dirent(struct inode *inode, dnode_secno dno, + const unsigned char *name, unsigned len, + dnode_secno *dd, struct quad_buffer_head *qbh) +{ + struct dnode *dnode; + struct hpfs_dirent *de; + struct hpfs_dirent *de_end; + int c1, c2 = 0; + + if (!S_ISDIR(inode->i_mode)) hpfs_error(inode->i_sb, "map_dirent: not a directory\n"); + again: + if (hpfs_sb(inode->i_sb)->sb_chk) + if (hpfs_stop_cycles(inode->i_sb, dno, &c1, &c2, "map_dirent")) return NULL; + if (!(dnode = hpfs_map_dnode(inode->i_sb, dno, qbh))) return NULL; + + de_end = dnode_end_de(dnode); + for (de = dnode_first_de(dnode); de < de_end; de = de_next_de(de)) { + int t = hpfs_compare_names(inode->i_sb, name, len, de->name, de->namelen, de->last); + if (!t) { + if (dd) *dd = dno; + return de; + } + if (t < 0) { + if (de->down) { + dno = de_down_pointer(de); + hpfs_brelse4(qbh); + goto again; + } + break; + } + } + hpfs_brelse4(qbh); + return NULL; +} + +/* + * Remove empty directory. In normal cases it is only one dnode with two + * entries, but we must handle also such obscure cases when it's a tree + * of empty dnodes. + */ + +void hpfs_remove_dtree(struct super_block *s, dnode_secno dno) +{ + struct quad_buffer_head qbh; + struct dnode *dnode; + struct hpfs_dirent *de; + dnode_secno d1, d2, rdno = dno; + while (1) { + if (!(dnode = hpfs_map_dnode(s, dno, &qbh))) return; + de = dnode_first_de(dnode); + if (de->last) { + if (de->down) d1 = de_down_pointer(de); + else goto error; + hpfs_brelse4(&qbh); + hpfs_free_dnode(s, dno); + dno = d1; + } else break; + } + if (!de->first) goto error; + d1 = de->down ? de_down_pointer(de) : 0; + de = de_next_de(de); + if (!de->last) goto error; + d2 = de->down ? de_down_pointer(de) : 0; + hpfs_brelse4(&qbh); + hpfs_free_dnode(s, dno); + do { + while (d1) { + if (!(dnode = hpfs_map_dnode(s, dno = d1, &qbh))) return; + de = dnode_first_de(dnode); + if (!de->last) goto error; + d1 = de->down ? de_down_pointer(de) : 0; + hpfs_brelse4(&qbh); + hpfs_free_dnode(s, dno); + } + d1 = d2; + d2 = 0; + } while (d1); + return; + error: + hpfs_brelse4(&qbh); + hpfs_free_dnode(s, dno); + hpfs_error(s, "directory %08x is corrupted or not empty", rdno); +} + +/* + * Find dirent for specified fnode. Use truncated 15-char name in fnode as + * a help for searching. + */ + +struct hpfs_dirent *map_fnode_dirent(struct super_block *s, fnode_secno fno, + struct fnode *f, struct quad_buffer_head *qbh) +{ + unsigned char *name1; + unsigned char *name2; + int name1len, name2len; + struct dnode *d; + dnode_secno dno, downd; + struct fnode *upf; + struct buffer_head *bh; + struct hpfs_dirent *de, *de_end; + int c; + int c1, c2 = 0; + int d1, d2 = 0; + name1 = f->name; + if (!(name2 = kmalloc(256, GFP_NOFS))) { + pr_err("out of memory, can't map dirent\n"); + return NULL; + } + if (f->len <= 15) + memcpy(name2, name1, name1len = name2len = f->len); + else { + memcpy(name2, name1, 15); + memset(name2 + 15, 0xff, 256 - 15); + /*name2[15] = 0xff;*/ + name1len = 15; name2len = 256; + } + if (!(upf = hpfs_map_fnode(s, le32_to_cpu(f->up), &bh))) { + kfree(name2); + return NULL; + } + if (!fnode_is_dir(upf)) { + brelse(bh); + hpfs_error(s, "fnode %08x has non-directory parent %08x", fno, le32_to_cpu(f->up)); + kfree(name2); + return NULL; + } + dno = le32_to_cpu(upf->u.external[0].disk_secno); + brelse(bh); + go_down: + downd = 0; + go_up: + if (!(d = hpfs_map_dnode(s, dno, qbh))) { + kfree(name2); + return NULL; + } + de_end = dnode_end_de(d); + de = dnode_first_de(d); + if (downd) { + while (de < de_end) { + if (de->down) if (de_down_pointer(de) == downd) goto f; + de = de_next_de(de); + } + hpfs_error(s, "pointer to dnode %08x not found in dnode %08x", downd, dno); + hpfs_brelse4(qbh); + kfree(name2); + return NULL; + } + next_de: + if (le32_to_cpu(de->fnode) == fno) { + kfree(name2); + return de; + } + c = hpfs_compare_names(s, name1, name1len, de->name, de->namelen, de->last); + if (c < 0 && de->down) { + dno = de_down_pointer(de); + hpfs_brelse4(qbh); + if (hpfs_sb(s)->sb_chk) + if (hpfs_stop_cycles(s, dno, &c1, &c2, "map_fnode_dirent #1")) { + kfree(name2); + return NULL; + } + goto go_down; + } + f: + if (le32_to_cpu(de->fnode) == fno) { + kfree(name2); + return de; + } + c = hpfs_compare_names(s, name2, name2len, de->name, de->namelen, de->last); + if (c < 0 && !de->last) goto not_found; + if ((de = de_next_de(de)) < de_end) goto next_de; + if (d->root_dnode) goto not_found; + downd = dno; + dno = le32_to_cpu(d->up); + hpfs_brelse4(qbh); + if (hpfs_sb(s)->sb_chk) + if (hpfs_stop_cycles(s, downd, &d1, &d2, "map_fnode_dirent #2")) { + kfree(name2); + return NULL; + } + goto go_up; + not_found: + hpfs_brelse4(qbh); + hpfs_error(s, "dirent for fnode %08x not found", fno); + kfree(name2); + return NULL; +} diff --git a/fs/hpfs/ea.c b/fs/hpfs/ea.c new file mode 100644 index 000000000..4664f9ab0 --- /dev/null +++ b/fs/hpfs/ea.c @@ -0,0 +1,368 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hpfs/ea.c + * + * Mikulas Patocka (mikulas@artax.karlin.mff.cuni.cz), 1998-1999 + * + * handling extended attributes + */ + +#include "hpfs_fn.h" + +/* Remove external extended attributes. ano specifies whether a is a + direct sector where eas starts or an anode */ + +void hpfs_ea_ext_remove(struct super_block *s, secno a, int ano, unsigned len) +{ + unsigned pos = 0; + while (pos < len) { + char ex[4 + 255 + 1 + 8]; + struct extended_attribute *ea = (struct extended_attribute *)ex; + if (pos + 4 > len) { + hpfs_error(s, "EAs don't end correctly, %s %08x, len %08x", + ano ? "anode" : "sectors", a, len); + return; + } + if (hpfs_ea_read(s, a, ano, pos, 4, ex)) return; + if (ea_indirect(ea)) { + if (ea_valuelen(ea) != 8) { + hpfs_error(s, "ea_indirect(ea) set while ea->valuelen!=8, %s %08x, pos %08x", + ano ? "anode" : "sectors", a, pos); + return; + } + if (hpfs_ea_read(s, a, ano, pos + 4, ea->namelen + 9, ex+4)) + return; + hpfs_ea_remove(s, ea_sec(ea), ea_in_anode(ea), ea_len(ea)); + } + pos += ea->namelen + ea_valuelen(ea) + 5; + } + if (!ano) hpfs_free_sectors(s, a, (len+511) >> 9); + else { + struct buffer_head *bh; + struct anode *anode; + if ((anode = hpfs_map_anode(s, a, &bh))) { + hpfs_remove_btree(s, GET_BTREE_PTR(&anode->btree)); + brelse(bh); + hpfs_free_sectors(s, a, 1); + } + } +} + +static char *get_indirect_ea(struct super_block *s, int ano, secno a, int size) +{ + char *ret; + if (!(ret = kmalloc(size + 1, GFP_NOFS))) { + pr_err("out of memory for EA\n"); + return NULL; + } + if (hpfs_ea_read(s, a, ano, 0, size, ret)) { + kfree(ret); + return NULL; + } + ret[size] = 0; + return ret; +} + +static void set_indirect_ea(struct super_block *s, int ano, secno a, + const char *data, int size) +{ + hpfs_ea_write(s, a, ano, 0, size, data); +} + +/* Read an extended attribute named 'key' into the provided buffer */ + +int hpfs_read_ea(struct super_block *s, struct fnode *fnode, char *key, + char *buf, int size) +{ + unsigned pos; + int ano, len; + secno a; + char ex[4 + 255 + 1 + 8]; + struct extended_attribute *ea; + struct extended_attribute *ea_end = fnode_end_ea(fnode); + for (ea = fnode_ea(fnode); ea < ea_end; ea = next_ea(ea)) + if (!strcmp(ea->name, key)) { + if (ea_indirect(ea)) + goto indirect; + if (ea_valuelen(ea) >= size) + return -EINVAL; + memcpy(buf, ea_data(ea), ea_valuelen(ea)); + buf[ea_valuelen(ea)] = 0; + return 0; + } + a = le32_to_cpu(fnode->ea_secno); + len = le32_to_cpu(fnode->ea_size_l); + ano = fnode_in_anode(fnode); + pos = 0; + while (pos < len) { + ea = (struct extended_attribute *)ex; + if (pos + 4 > len) { + hpfs_error(s, "EAs don't end correctly, %s %08x, len %08x", + ano ? "anode" : "sectors", a, len); + return -EIO; + } + if (hpfs_ea_read(s, a, ano, pos, 4, ex)) return -EIO; + if (hpfs_ea_read(s, a, ano, pos + 4, ea->namelen + 1 + (ea_indirect(ea) ? 8 : 0), ex + 4)) + return -EIO; + if (!strcmp(ea->name, key)) { + if (ea_indirect(ea)) + goto indirect; + if (ea_valuelen(ea) >= size) + return -EINVAL; + if (hpfs_ea_read(s, a, ano, pos + 4 + ea->namelen + 1, ea_valuelen(ea), buf)) + return -EIO; + buf[ea_valuelen(ea)] = 0; + return 0; + } + pos += ea->namelen + ea_valuelen(ea) + 5; + } + return -ENOENT; +indirect: + if (ea_len(ea) >= size) + return -EINVAL; + if (hpfs_ea_read(s, ea_sec(ea), ea_in_anode(ea), 0, ea_len(ea), buf)) + return -EIO; + buf[ea_len(ea)] = 0; + return 0; +} + +/* Read an extended attribute named 'key' */ +char *hpfs_get_ea(struct super_block *s, struct fnode *fnode, char *key, int *size) +{ + char *ret; + unsigned pos; + int ano, len; + secno a; + struct extended_attribute *ea; + struct extended_attribute *ea_end = fnode_end_ea(fnode); + for (ea = fnode_ea(fnode); ea < ea_end; ea = next_ea(ea)) + if (!strcmp(ea->name, key)) { + if (ea_indirect(ea)) + return get_indirect_ea(s, ea_in_anode(ea), ea_sec(ea), *size = ea_len(ea)); + if (!(ret = kmalloc((*size = ea_valuelen(ea)) + 1, GFP_NOFS))) { + pr_err("out of memory for EA\n"); + return NULL; + } + memcpy(ret, ea_data(ea), ea_valuelen(ea)); + ret[ea_valuelen(ea)] = 0; + return ret; + } + a = le32_to_cpu(fnode->ea_secno); + len = le32_to_cpu(fnode->ea_size_l); + ano = fnode_in_anode(fnode); + pos = 0; + while (pos < len) { + char ex[4 + 255 + 1 + 8]; + ea = (struct extended_attribute *)ex; + if (pos + 4 > len) { + hpfs_error(s, "EAs don't end correctly, %s %08x, len %08x", + ano ? "anode" : "sectors", a, len); + return NULL; + } + if (hpfs_ea_read(s, a, ano, pos, 4, ex)) return NULL; + if (hpfs_ea_read(s, a, ano, pos + 4, ea->namelen + 1 + (ea_indirect(ea) ? 8 : 0), ex + 4)) + return NULL; + if (!strcmp(ea->name, key)) { + if (ea_indirect(ea)) + return get_indirect_ea(s, ea_in_anode(ea), ea_sec(ea), *size = ea_len(ea)); + if (!(ret = kmalloc((*size = ea_valuelen(ea)) + 1, GFP_NOFS))) { + pr_err("out of memory for EA\n"); + return NULL; + } + if (hpfs_ea_read(s, a, ano, pos + 4 + ea->namelen + 1, ea_valuelen(ea), ret)) { + kfree(ret); + return NULL; + } + ret[ea_valuelen(ea)] = 0; + return ret; + } + pos += ea->namelen + ea_valuelen(ea) + 5; + } + return NULL; +} + +/* + * Update or create extended attribute 'key' with value 'data'. Note that + * when this ea exists, it MUST have the same size as size of data. + * This driver can't change sizes of eas ('cause I just don't need it). + */ + +void hpfs_set_ea(struct inode *inode, struct fnode *fnode, const char *key, + const char *data, int size) +{ + fnode_secno fno = inode->i_ino; + struct super_block *s = inode->i_sb; + unsigned pos; + int ano, len; + secno a; + unsigned char h[4]; + struct extended_attribute *ea; + struct extended_attribute *ea_end = fnode_end_ea(fnode); + for (ea = fnode_ea(fnode); ea < ea_end; ea = next_ea(ea)) + if (!strcmp(ea->name, key)) { + if (ea_indirect(ea)) { + if (ea_len(ea) == size) + set_indirect_ea(s, ea_in_anode(ea), ea_sec(ea), data, size); + } else if (ea_valuelen(ea) == size) { + memcpy(ea_data(ea), data, size); + } + return; + } + a = le32_to_cpu(fnode->ea_secno); + len = le32_to_cpu(fnode->ea_size_l); + ano = fnode_in_anode(fnode); + pos = 0; + while (pos < len) { + char ex[4 + 255 + 1 + 8]; + ea = (struct extended_attribute *)ex; + if (pos + 4 > len) { + hpfs_error(s, "EAs don't end correctly, %s %08x, len %08x", + ano ? "anode" : "sectors", a, len); + return; + } + if (hpfs_ea_read(s, a, ano, pos, 4, ex)) return; + if (hpfs_ea_read(s, a, ano, pos + 4, ea->namelen + 1 + (ea_indirect(ea) ? 8 : 0), ex + 4)) + return; + if (!strcmp(ea->name, key)) { + if (ea_indirect(ea)) { + if (ea_len(ea) == size) + set_indirect_ea(s, ea_in_anode(ea), ea_sec(ea), data, size); + } + else { + if (ea_valuelen(ea) == size) + hpfs_ea_write(s, a, ano, pos + 4 + ea->namelen + 1, size, data); + } + return; + } + pos += ea->namelen + ea_valuelen(ea) + 5; + } + if (!le16_to_cpu(fnode->ea_offs)) { + /*if (le16_to_cpu(fnode->ea_size_s)) { + hpfs_error(s, "fnode %08x: ea_size_s == %03x, ea_offs == 0", + inode->i_ino, le16_to_cpu(fnode->ea_size_s)); + return; + }*/ + fnode->ea_offs = cpu_to_le16(0xc4); + } + if (le16_to_cpu(fnode->ea_offs) < 0xc4 || le16_to_cpu(fnode->ea_offs) + le16_to_cpu(fnode->acl_size_s) + le16_to_cpu(fnode->ea_size_s) > 0x200) { + hpfs_error(s, "fnode %08llx: ea_offs == %03x, ea_size_s == %03x", + inode->i_ino, + le16_to_cpu(fnode->ea_offs), le16_to_cpu(fnode->ea_size_s)); + return; + } + if ((le16_to_cpu(fnode->ea_size_s) || !le32_to_cpu(fnode->ea_size_l)) && + le16_to_cpu(fnode->ea_offs) + le16_to_cpu(fnode->acl_size_s) + le16_to_cpu(fnode->ea_size_s) + strlen(key) + size + 5 <= 0x200) { + ea = fnode_end_ea(fnode); + *(char *)ea = 0; + ea->namelen = strlen(key); + ea->valuelen_lo = size; + ea->valuelen_hi = size >> 8; + strcpy(ea->name, key); + memcpy(ea_data(ea), data, size); + fnode->ea_size_s = cpu_to_le16(le16_to_cpu(fnode->ea_size_s) + strlen(key) + size + 5); + goto ret; + } + /* Most the code here is 99.9993422% unused. I hope there are no bugs. + But what .. HPFS.IFS has also bugs in ea management. */ + if (le16_to_cpu(fnode->ea_size_s) && !le32_to_cpu(fnode->ea_size_l)) { + secno n; + struct buffer_head *bh; + char *data; + if (!(n = hpfs_alloc_sector(s, fno, 1, 0))) return; + if (!(data = hpfs_get_sector(s, n, &bh))) { + hpfs_free_sectors(s, n, 1); + return; + } + memcpy(data, fnode_ea(fnode), le16_to_cpu(fnode->ea_size_s)); + fnode->ea_size_l = cpu_to_le32(le16_to_cpu(fnode->ea_size_s)); + fnode->ea_size_s = cpu_to_le16(0); + fnode->ea_secno = cpu_to_le32(n); + fnode->flags &= ~FNODE_anode; + mark_buffer_dirty(bh); + brelse(bh); + } + pos = le32_to_cpu(fnode->ea_size_l) + 5 + strlen(key) + size; + len = (le32_to_cpu(fnode->ea_size_l) + 511) >> 9; + if (pos >= 30000) goto bail; + while (((pos + 511) >> 9) > len) { + if (!len) { + secno q = hpfs_alloc_sector(s, fno, 1, 0); + if (!q) goto bail; + fnode->ea_secno = cpu_to_le32(q); + fnode->flags &= ~FNODE_anode; + len++; + } else if (!fnode_in_anode(fnode)) { + if (hpfs_alloc_if_possible(s, le32_to_cpu(fnode->ea_secno) + len)) { + len++; + } else { + /* Aargh... don't know how to create ea anodes :-( */ + /*struct buffer_head *bh; + struct anode *anode; + anode_secno a_s; + if (!(anode = hpfs_alloc_anode(s, fno, &a_s, &bh))) + goto bail; + anode->up = cpu_to_le32(fno); + anode->btree.fnode_parent = 1; + anode->btree.n_free_nodes--; + anode->btree.n_used_nodes++; + anode->btree.first_free = cpu_to_le16(le16_to_cpu(anode->btree.first_free) + 12); + anode->u.external[0].disk_secno = cpu_to_le32(le32_to_cpu(fnode->ea_secno)); + anode->u.external[0].file_secno = cpu_to_le32(0); + anode->u.external[0].length = cpu_to_le32(len); + mark_buffer_dirty(bh); + brelse(bh); + fnode->flags |= FNODE_anode; + fnode->ea_secno = cpu_to_le32(a_s);*/ + secno new_sec; + int i; + if (!(new_sec = hpfs_alloc_sector(s, fno, 1, 1 - ((pos + 511) >> 9)))) + goto bail; + for (i = 0; i < len; i++) { + struct buffer_head *bh1, *bh2; + void *b1, *b2; + if (!(b1 = hpfs_map_sector(s, le32_to_cpu(fnode->ea_secno) + i, &bh1, len - i - 1))) { + hpfs_free_sectors(s, new_sec, (pos + 511) >> 9); + goto bail; + } + if (!(b2 = hpfs_get_sector(s, new_sec + i, &bh2))) { + brelse(bh1); + hpfs_free_sectors(s, new_sec, (pos + 511) >> 9); + goto bail; + } + memcpy(b2, b1, 512); + brelse(bh1); + mark_buffer_dirty(bh2); + brelse(bh2); + } + hpfs_free_sectors(s, le32_to_cpu(fnode->ea_secno), len); + fnode->ea_secno = cpu_to_le32(new_sec); + len = (pos + 511) >> 9; + } + } + if (fnode_in_anode(fnode)) { + if (hpfs_add_sector_to_btree(s, le32_to_cpu(fnode->ea_secno), + 0, len) != -1) { + len++; + } else { + goto bail; + } + } + } + h[0] = 0; + h[1] = strlen(key); + h[2] = size & 0xff; + h[3] = size >> 8; + if (hpfs_ea_write(s, le32_to_cpu(fnode->ea_secno), fnode_in_anode(fnode), le32_to_cpu(fnode->ea_size_l), 4, h)) goto bail; + if (hpfs_ea_write(s, le32_to_cpu(fnode->ea_secno), fnode_in_anode(fnode), le32_to_cpu(fnode->ea_size_l) + 4, h[1] + 1, key)) goto bail; + if (hpfs_ea_write(s, le32_to_cpu(fnode->ea_secno), fnode_in_anode(fnode), le32_to_cpu(fnode->ea_size_l) + 5 + h[1], size, data)) goto bail; + fnode->ea_size_l = cpu_to_le32(pos); + ret: + hpfs_i(inode)->i_ea_size += 5 + strlen(key) + size; + return; + bail: + if (le32_to_cpu(fnode->ea_secno)) + if (fnode_in_anode(fnode)) hpfs_truncate_btree(s, le32_to_cpu(fnode->ea_secno), 1, (le32_to_cpu(fnode->ea_size_l) + 511) >> 9); + else hpfs_free_sectors(s, le32_to_cpu(fnode->ea_secno) + ((le32_to_cpu(fnode->ea_size_l) + 511) >> 9), len - ((le32_to_cpu(fnode->ea_size_l) + 511) >> 9)); + else fnode->ea_secno = fnode->ea_size_l = cpu_to_le32(0); +} + diff --git a/fs/hpfs/file.c b/fs/hpfs/file.c new file mode 100644 index 000000000..6a629ab95 --- /dev/null +++ b/fs/hpfs/file.c @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hpfs/file.c + * + * Mikulas Patocka (mikulas@artax.karlin.mff.cuni.cz), 1998-1999 + * + * file VFS functions + */ + +#include "hpfs_fn.h" +#include +#include +#include + +#define BLOCKS(size) (((size) + 511) >> 9) + +static int hpfs_file_release(struct inode *inode, struct file *file) +{ + hpfs_lock(inode->i_sb); + hpfs_write_if_changed(inode); + hpfs_unlock(inode->i_sb); + return 0; +} + +int hpfs_file_fsync(struct file *file, loff_t start, loff_t end, int datasync) +{ + struct inode *inode = file->f_mapping->host; + int ret; + + ret = file_write_and_wait_range(file, start, end); + if (ret) + return ret; + return sync_blockdev(inode->i_sb->s_bdev); +} + +/* + * generic_file_read often calls bmap with non-existing sector, + * so we must ignore such errors. + */ + +static secno hpfs_bmap(struct inode *inode, unsigned file_secno, unsigned *n_secs) +{ + struct hpfs_inode_info *hpfs_inode = hpfs_i(inode); + unsigned n, disk_secno; + struct fnode *fnode; + struct buffer_head *bh; + if (BLOCKS(hpfs_i(inode)->mmu_private) <= file_secno) return 0; + n = file_secno - hpfs_inode->i_file_sec; + if (n < hpfs_inode->i_n_secs) { + *n_secs = hpfs_inode->i_n_secs - n; + return hpfs_inode->i_disk_sec + n; + } + if (!(fnode = hpfs_map_fnode(inode->i_sb, inode->i_ino, &bh))) return 0; + disk_secno = hpfs_bplus_lookup(inode->i_sb, inode, + GET_BTREE_PTR(&fnode->btree), + file_secno, bh); + if (disk_secno == -1) return 0; + if (hpfs_chk_sectors(inode->i_sb, disk_secno, 1, "bmap")) return 0; + n = file_secno - hpfs_inode->i_file_sec; + if (n < hpfs_inode->i_n_secs) { + *n_secs = hpfs_inode->i_n_secs - n; + return hpfs_inode->i_disk_sec + n; + } + *n_secs = 1; + return disk_secno; +} + +void hpfs_truncate(struct inode *i) +{ + if (IS_IMMUTABLE(i)) return /*-EPERM*/; + hpfs_lock_assert(i->i_sb); + + hpfs_i(i)->i_n_secs = 0; + i->i_blocks = 1 + ((i->i_size + 511) >> 9); + hpfs_i(i)->mmu_private = i->i_size; + hpfs_truncate_btree(i->i_sb, i->i_ino, 1, ((i->i_size + 511) >> 9)); + hpfs_write_inode(i); + hpfs_i(i)->i_n_secs = 0; +} + +static int hpfs_get_block(struct inode *inode, sector_t iblock, struct buffer_head *bh_result, int create) +{ + int r; + secno s; + unsigned n_secs; + hpfs_lock(inode->i_sb); + s = hpfs_bmap(inode, iblock, &n_secs); + if (s) { + if (bh_result->b_size >> 9 < n_secs) + n_secs = bh_result->b_size >> 9; + n_secs = hpfs_search_hotfix_map_for_range(inode->i_sb, s, n_secs); + if (unlikely(!n_secs)) { + s = hpfs_search_hotfix_map(inode->i_sb, s); + n_secs = 1; + } + map_bh(bh_result, inode->i_sb, s); + bh_result->b_size = n_secs << 9; + goto ret_0; + } + if (!create) goto ret_0; + if (iblock<<9 != hpfs_i(inode)->mmu_private) { + BUG(); + r = -EIO; + goto ret_r; + } + if ((s = hpfs_add_sector_to_btree(inode->i_sb, inode->i_ino, 1, inode->i_blocks - 1)) == -1) { + hpfs_truncate_btree(inode->i_sb, inode->i_ino, 1, inode->i_blocks - 1); + r = -ENOSPC; + goto ret_r; + } + inode->i_blocks++; + hpfs_i(inode)->mmu_private += 512; + set_buffer_new(bh_result); + map_bh(bh_result, inode->i_sb, hpfs_search_hotfix_map(inode->i_sb, s)); + ret_0: + r = 0; + ret_r: + hpfs_unlock(inode->i_sb); + return r; +} + +static int hpfs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, + unsigned flags, struct iomap *iomap, struct iomap *srcmap) +{ + struct super_block *sb = inode->i_sb; + unsigned int blkbits = inode->i_blkbits; + unsigned int n_secs; + secno s; + + if (WARN_ON_ONCE(flags & (IOMAP_WRITE | IOMAP_ZERO))) + return -EINVAL; + + iomap->bdev = inode->i_sb->s_bdev; + iomap->offset = offset; + + hpfs_lock(sb); + s = hpfs_bmap(inode, offset >> blkbits, &n_secs); + if (s) { + n_secs = hpfs_search_hotfix_map_for_range(sb, s, + min_t(loff_t, n_secs, length)); + if (unlikely(!n_secs)) { + s = hpfs_search_hotfix_map(sb, s); + n_secs = 1; + } + iomap->type = IOMAP_MAPPED; + iomap->flags = IOMAP_F_MERGED; + iomap->addr = (u64)s << blkbits; + iomap->length = (u64)n_secs << blkbits; + } else { + iomap->type = IOMAP_HOLE; + iomap->addr = IOMAP_NULL_ADDR; + iomap->length = 1 << blkbits; + } + + hpfs_unlock(sb); + return 0; +} + +static DEFINE_IOMAP_ITER_NEXT(hpfs_iomap_next, hpfs_iomap_begin); + +static const struct iomap_ops hpfs_iomap_ops = { + .iomap_next = hpfs_iomap_next, +}; + +static int hpfs_read_folio(struct file *file, struct folio *folio) +{ + return mpage_read_folio(folio, hpfs_get_block); +} + +static void hpfs_readahead(struct readahead_control *rac) +{ + mpage_readahead(rac, hpfs_get_block); +} + +static int hpfs_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + return mpage_writepages(mapping, wbc, hpfs_get_block); +} + +static void hpfs_write_failed(struct address_space *mapping, loff_t to) +{ + struct inode *inode = mapping->host; + + hpfs_lock(inode->i_sb); + + if (to > inode->i_size) { + truncate_pagecache(inode, inode->i_size); + hpfs_truncate(inode); + } + + hpfs_unlock(inode->i_sb); +} + +static int hpfs_write_begin(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, + struct folio **foliop, void **fsdata) +{ + int ret; + + ret = cont_write_begin(iocb, mapping, pos, len, foliop, fsdata, + hpfs_get_block, + &hpfs_i(mapping->host)->mmu_private); + if (unlikely(ret)) + hpfs_write_failed(mapping, pos + len); + + return ret; +} + +static int hpfs_write_end(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct folio *folio, void *fsdata) +{ + struct inode *inode = mapping->host; + int err; + err = generic_write_end(iocb, mapping, pos, len, copied, folio, fsdata); + if (err < len) + hpfs_write_failed(mapping, pos + len); + if (!(err < 0)) { + /* make sure we write it on close, if not earlier */ + hpfs_lock(inode->i_sb); + hpfs_i(inode)->i_dirty = 1; + hpfs_unlock(inode->i_sb); + } + return err; +} + +static sector_t _hpfs_bmap(struct address_space *mapping, sector_t block) +{ + return generic_block_bmap(mapping, block, hpfs_get_block); +} + +static int hpfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len) +{ + int ret; + + inode_lock(inode); + len = min_t(u64, len, i_size_read(inode)); + ret = iomap_fiemap(inode, fieinfo, start, len, &hpfs_iomap_ops); + inode_unlock(inode); + + return ret; +} + +const struct address_space_operations hpfs_aops = { + .dirty_folio = block_dirty_folio, + .invalidate_folio = block_invalidate_folio, + .read_folio = hpfs_read_folio, + .readahead = hpfs_readahead, + .writepages = hpfs_writepages, + .write_begin = hpfs_write_begin, + .write_end = hpfs_write_end, + .bmap = _hpfs_bmap, + .migrate_folio = buffer_migrate_folio, +}; + +const struct file_operations hpfs_file_ops = +{ + .llseek = generic_file_llseek, + .read_iter = generic_file_read_iter, + .write_iter = generic_file_write_iter, + .mmap_prepare = generic_file_mmap_prepare, + .release = hpfs_file_release, + .fsync = hpfs_file_fsync, + .splice_read = filemap_splice_read, + .unlocked_ioctl = hpfs_ioctl, + .compat_ioctl = compat_ptr_ioctl, +}; + +const struct inode_operations hpfs_file_iops = +{ + .setattr = hpfs_setattr, + .fiemap = hpfs_fiemap, +}; diff --git a/fs/hpfs/hpfs.h b/fs/hpfs/hpfs.h new file mode 100644 index 000000000..353f73c91 --- /dev/null +++ b/fs/hpfs/hpfs.h @@ -0,0 +1,579 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * linux/fs/hpfs/hpfs.h + * + * HPFS structures by Chris Smith, 1993 + * + * a little bit modified by Mikulas Patocka, 1998-1999 + */ + +/* The paper + + Duncan, Roy + Design goals and implementation of the new High Performance File System + Microsoft Systems Journal Sept 1989 v4 n5 p1(13) + + describes what HPFS looked like when it was new, and it is the source + of most of the information given here. The rest is conjecture. + + For definitive information on the Duncan paper, see it, not this file. + For definitive information on HPFS, ask somebody else -- this is guesswork. + There are certain to be many mistakes. */ + +#if !defined(__LITTLE_ENDIAN) && !defined(__BIG_ENDIAN) +#error unknown endian +#endif + +/* Notation */ + +typedef u32 secno; /* sector number, partition relative */ + +typedef secno dnode_secno; /* sector number of a dnode */ +typedef secno fnode_secno; /* sector number of an fnode */ +typedef secno anode_secno; /* sector number of an anode */ + +typedef u32 time32_t; /* 32-bit time_t type */ + +/* sector 0 */ + +/* The boot block is very like a FAT boot block, except that the + 29h signature byte is 28h instead, and the ID string is "HPFS". */ + +#define BB_MAGIC 0xaa55 + +struct hpfs_boot_block +{ + u8 jmp[3]; + u8 oem_id[8]; + u8 bytes_per_sector[2]; /* 512 */ + u8 sectors_per_cluster; + u8 n_reserved_sectors[2]; + u8 n_fats; + u8 n_rootdir_entries[2]; + u8 n_sectors_s[2]; + u8 media_byte; + __le16 sectors_per_fat; + __le16 sectors_per_track; + __le16 heads_per_cyl; + __le32 n_hidden_sectors; + __le32 n_sectors_l; /* size of partition */ + u8 drive_number; + u8 mbz; + u8 sig_28h; /* 28h */ + u8 vol_serno[4]; + u8 vol_label[11]; + u8 sig_hpfs[8]; /* "HPFS " */ + u8 pad[448]; + __le16 magic; /* aa55 */ +}; + + +/* sector 16 */ + +/* The super block has the pointer to the root directory. */ + +#define SB_MAGIC 0xf995e849 + +struct hpfs_super_block +{ + __le32 magic; /* f995 e849 */ + __le32 magic1; /* fa53 e9c5, more magic? */ + u8 version; /* version of a filesystem usually 2 */ + u8 funcversion; /* functional version - oldest version + of filesystem that can understand + this disk */ + __le16 zero; /* 0 */ + __le32 root; /* fnode of root directory */ + __le32 n_sectors; /* size of filesystem */ + __le32 n_badblocks; /* number of bad blocks */ + __le32 bitmaps; /* pointers to free space bit maps */ + __le32 zero1; /* 0 */ + __le32 badblocks; /* bad block list */ + __le32 zero3; /* 0 */ + __le32 last_chkdsk; /* date last checked, 0 if never */ + __le32 last_optimize; /* date last optimized, 0 if never */ + __le32 n_dir_band; /* number of sectors in dir band */ + __le32 dir_band_start; /* first sector in dir band */ + __le32 dir_band_end; /* last sector in dir band */ + __le32 dir_band_bitmap; /* free space map, 1 dnode per bit */ + u8 volume_name[32]; /* not used */ + __le32 user_id_table; /* 8 preallocated sectors - user id */ + u32 zero6[103]; /* 0 */ +}; + + +/* sector 17 */ + +/* The spare block has pointers to spare sectors. */ + +#define SP_MAGIC 0xf9911849 + +struct hpfs_spare_block +{ + __le32 magic; /* f991 1849 */ + __le32 magic1; /* fa52 29c5, more magic? */ + +#ifdef __LITTLE_ENDIAN + u8 dirty: 1; /* 0 clean, 1 "improperly stopped" */ + u8 sparedir_used: 1; /* spare dirblks used */ + u8 hotfixes_used: 1; /* hotfixes used */ + u8 bad_sector: 1; /* bad sector, corrupted disk (???) */ + u8 bad_bitmap: 1; /* bad bitmap */ + u8 fast: 1; /* partition was fast formatted */ + u8 old_wrote: 1; /* old version wrote to partition */ + u8 old_wrote_1: 1; /* old version wrote to partition (?) */ +#else + u8 old_wrote_1: 1; /* old version wrote to partition (?) */ + u8 old_wrote: 1; /* old version wrote to partition */ + u8 fast: 1; /* partition was fast formatted */ + u8 bad_bitmap: 1; /* bad bitmap */ + u8 bad_sector: 1; /* bad sector, corrupted disk (???) */ + u8 hotfixes_used: 1; /* hotfixes used */ + u8 sparedir_used: 1; /* spare dirblks used */ + u8 dirty: 1; /* 0 clean, 1 "improperly stopped" */ +#endif + +#ifdef __LITTLE_ENDIAN + u8 install_dasd_limits: 1; /* HPFS386 flags */ + u8 resynch_dasd_limits: 1; + u8 dasd_limits_operational: 1; + u8 multimedia_active: 1; + u8 dce_acls_active: 1; + u8 dasd_limits_dirty: 1; + u8 flag67: 2; +#else + u8 flag67: 2; + u8 dasd_limits_dirty: 1; + u8 dce_acls_active: 1; + u8 multimedia_active: 1; + u8 dasd_limits_operational: 1; + u8 resynch_dasd_limits: 1; + u8 install_dasd_limits: 1; /* HPFS386 flags */ +#endif + + u8 mm_contlgulty; + u8 unused; + + __le32 hotfix_map; /* info about remapped bad sectors */ + __le32 n_spares_used; /* number of hotfixes */ + __le32 n_spares; /* number of spares in hotfix map */ + __le32 n_dnode_spares_free; /* spare dnodes unused */ + __le32 n_dnode_spares; /* length of spare_dnodes[] list, + follows in this block*/ + __le32 code_page_dir; /* code page directory block */ + __le32 n_code_pages; /* number of code pages */ + __le32 super_crc; /* on HPFS386 and LAN Server this is + checksum of superblock, on normal + OS/2 unused */ + __le32 spare_crc; /* on HPFS386 checksum of spareblock */ + __le32 zero1[15]; /* unused */ + __le32 spare_dnodes[100]; /* emergency free dnode list */ + __le32 zero2[1]; /* room for more? */ +}; + +/* The bad block list is 4 sectors long. The first word must be zero, + the remaining words give n_badblocks bad block numbers. + I bet you can see it coming... */ + +#define BAD_MAGIC 0 + +/* The hotfix map is 4 sectors long. It looks like + + secno from[n_spares]; + secno to[n_spares]; + + The to[] list is initialized to point to n_spares preallocated empty + sectors. The from[] list contains the sector numbers of bad blocks + which have been remapped to corresponding sectors in the to[] list. + n_spares_used gives the length of the from[] list. */ + + +/* Sectors 18 and 19 are preallocated and unused. + Maybe they're spares for 16 and 17, but simple substitution fails. */ + + +/* The code page info pointed to by the spare block consists of an index + block and blocks containing uppercasing tables. I don't know what + these are for (CHKDSK, maybe?) -- OS/2 does not seem to use them + itself. Linux doesn't use them either. */ + +/* block pointed to by spareblock->code_page_dir */ + +#define CP_DIR_MAGIC 0x494521f7 + +struct code_page_directory +{ + __le32 magic; /* 4945 21f7 */ + __le32 n_code_pages; /* number of pointers following */ + __le32 zero1[2]; + struct { + __le16 ix; /* index */ + __le16 code_page_number; /* code page number */ + __le32 bounds; /* matches corresponding word + in data block */ + __le32 code_page_data; /* sector number of a code_page_data + containing c.p. array */ + __le16 index; /* index in c.p. array in that sector*/ + __le16 unknown; /* some unknown value; usually 0; + 2 in Japanese version */ + } array[31]; /* unknown length */ +}; + +/* blocks pointed to by code_page_directory */ + +#define CP_DATA_MAGIC 0x894521f7 + +struct code_page_data +{ + __le32 magic; /* 8945 21f7 */ + __le32 n_used; /* # elements used in c_p_data[] */ + __le32 bounds[3]; /* looks a bit like + (beg1,end1), (beg2,end2) + one byte each */ + __le16 offs[3]; /* offsets from start of sector + to start of c_p_data[ix] */ + struct { + __le16 ix; /* index */ + __le16 code_page_number; /* code page number */ + __le16 unknown; /* the same as in cp directory */ + u8 map[128]; /* upcase table for chars 80..ff */ + __le16 zero2; + } code_page[3]; + u8 incognita[78]; +}; + + +/* Free space bitmaps are 4 sectors long, which is 16384 bits. + 16384 sectors is 8 meg, and each 8 meg band has a 4-sector bitmap. + Bit order in the maps is little-endian. 0 means taken, 1 means free. + + Bit map sectors are marked allocated in the bit maps, and so are sectors + off the end of the partition. + + Band 0 is sectors 0-3fff, its map is in sectors 18-1b. + Band 1 is 4000-7fff, its map is in 7ffc-7fff. + Band 2 is 8000-ffff, its map is in 8000-8003. + The remaining bands have maps in their first (even) or last (odd) 4 sectors + -- if the last, partial, band is odd its map is in its last 4 sectors. + + The bitmap locations are given in a table pointed to by the super block. + No doubt they aren't constrained to be at 18, 7ffc, 8000, ...; that is + just where they usually are. + + The "directory band" is a bunch of sectors preallocated for dnodes. + It has a 4-sector free space bitmap of its own. Each bit in the map + corresponds to one 4-sector dnode, bit 0 of the map corresponding to + the first 4 sectors of the directory band. The entire band is marked + allocated in the main bitmap. The super block gives the locations + of the directory band and its bitmap. ("band" doesn't mean it is + 8 meg long; it isn't.) */ + + +/* dnode: directory. 4 sectors long */ + +/* A directory is a tree of dnodes. The fnode for a directory + contains one pointer, to the root dnode of the tree. The fnode + never moves, the dnodes do the B-tree thing, splitting and merging + as files are added and removed. */ + +#define DNODE_MAGIC 0x77e40aae + +struct dnode { + __le32 magic; /* 77e4 0aae */ + __le32 first_free; /* offset from start of dnode to + first free dir entry */ +#ifdef __LITTLE_ENDIAN + u8 root_dnode: 1; /* Is it root dnode? */ + u8 increment_me: 7; /* some kind of activity counter? */ + /* Neither HPFS.IFS nor CHKDSK cares + if you change this word */ +#else + u8 increment_me: 7; /* some kind of activity counter? */ + /* Neither HPFS.IFS nor CHKDSK cares + if you change this word */ + u8 root_dnode: 1; /* Is it root dnode? */ +#endif + u8 increment_me2[3]; + __le32 up; /* (root dnode) directory's fnode + (nonroot) parent dnode */ + __le32 self; /* pointer to this dnode */ + u8 dirent[2028]; /* one or more dirents */ +}; + +struct hpfs_dirent { + __le16 length; /* offset to next dirent */ + +#ifdef __LITTLE_ENDIAN + u8 first: 1; /* set on phony ^A^A (".") entry */ + u8 has_acl: 1; + u8 down: 1; /* down pointer present (after name) */ + u8 last: 1; /* set on phony \377 entry */ + u8 has_ea: 1; /* entry has EA */ + u8 has_xtd_perm: 1; /* has extended perm list (???) */ + u8 has_explicit_acl: 1; + u8 has_needea: 1; /* ?? some EA has NEEDEA set + I have no idea why this is + interesting in a dir entry */ +#else + u8 has_needea: 1; /* ?? some EA has NEEDEA set + I have no idea why this is + interesting in a dir entry */ + u8 has_explicit_acl: 1; + u8 has_xtd_perm: 1; /* has extended perm list (???) */ + u8 has_ea: 1; /* entry has EA */ + u8 last: 1; /* set on phony \377 entry */ + u8 down: 1; /* down pointer present (after name) */ + u8 has_acl: 1; + u8 first: 1; /* set on phony ^A^A (".") entry */ +#endif + +#ifdef __LITTLE_ENDIAN + u8 read_only: 1; /* dos attrib */ + u8 hidden: 1; /* dos attrib */ + u8 system: 1; /* dos attrib */ + u8 flag11: 1; /* would be volume label dos attrib */ + u8 directory: 1; /* dos attrib */ + u8 archive: 1; /* dos attrib */ + u8 not_8x3: 1; /* name is not 8.3 */ + u8 flag15: 1; +#else + u8 flag15: 1; + u8 not_8x3: 1; /* name is not 8.3 */ + u8 archive: 1; /* dos attrib */ + u8 directory: 1; /* dos attrib */ + u8 flag11: 1; /* would be volume label dos attrib */ + u8 system: 1; /* dos attrib */ + u8 hidden: 1; /* dos attrib */ + u8 read_only: 1; /* dos attrib */ +#endif + + __le32 fnode; /* fnode giving allocation info */ + __le32 write_date; /* mtime */ + __le32 file_size; /* file length, bytes */ + __le32 read_date; /* atime */ + __le32 creation_date; /* ctime */ + __le32 ea_size; /* total EA length, bytes */ + u8 no_of_acls; /* number of ACL's (low 3 bits) */ + u8 ix; /* code page index (of filename), see + struct code_page_data */ + u8 namelen; /* file name length */ + u8 name[]; /* file name */ + /* dnode_secno down; btree down pointer, if present, + follows name on next word boundary, or maybe it + precedes next dirent, which is on a word boundary. */ +}; + + +/* B+ tree: allocation info in fnodes and anodes */ + +/* dnodes point to fnodes which are responsible for listing the sectors + assigned to the file. This is done with trees of (length,address) + pairs. (Actually triples, of (length, file-address, disk-address) + which can represent holes. Find out if HPFS does that.) + At any rate, fnodes contain a small tree; if subtrees are needed + they occupy essentially a full block in anodes. A leaf-level tree node + has 3-word entries giving sector runs, a non-leaf node has 2-word + entries giving subtree pointers. A flag in the header says which. */ + +struct bplus_leaf_node +{ + __le32 file_secno; /* first file sector in extent */ + __le32 length; /* length, sectors */ + __le32 disk_secno; /* first corresponding disk sector */ +}; + +struct bplus_internal_node +{ + __le32 file_secno; /* subtree maps sectors < this */ + __le32 down; /* pointer to subtree */ +}; + +enum { + BP_hbff = 1, + BP_fnode_parent = 0x20, + BP_binary_search = 0x40, + BP_internal = 0x80 +}; + +/** + * GET_BTREE_PTR() - Get a pointer to struct bplus_header + * + * Wrapper around container_of() to retrieve a pointer to struct + * bplus_header from a pointer to struct bplus_header_fixed. + * + * @ptr: Pointer to struct bplus_header_fixed. + * + */ +#define GET_BTREE_PTR(ptr) \ + container_of(ptr, struct bplus_header, __hdr) + +struct bplus_header +{ + /* New members MUST be added within the struct_group() macro below. */ + struct_group_tagged(bplus_header_fixed, __hdr, + u8 flags; /* bit 0 - high bit of first free entry offset + bit 5 - we're pointed to by an fnode, + the data btree or some ea or the + main ea bootage pointer ea_secno + bit 6 - suggest binary search (unused) + bit 7 - 1 -> (internal) tree of anodes + 0 -> (leaf) list of extents */ + u8 fill[3]; + u8 n_free_nodes; /* free nodes in following array */ + u8 n_used_nodes; /* used nodes in following array */ + __le16 first_free; /* offset from start of header to + first free node in array */ + ); + union { + /* (internal) 2-word entries giving subtree pointers */ + DECLARE_FLEX_ARRAY(struct bplus_internal_node, internal); + /* (external) 3-word entries giving sector runs */ + DECLARE_FLEX_ARRAY(struct bplus_leaf_node, external); + } u; +}; +static_assert(offsetof(struct bplus_header, u.internal) == sizeof(struct bplus_header_fixed), + "struct member likely outside of struct_group_tagged()"); + +static inline bool bp_internal(struct bplus_header *bp) +{ + return bp->flags & BP_internal; +} + +static inline bool bp_fnode_parent(struct bplus_header *bp) +{ + return bp->flags & BP_fnode_parent; +} + +/* fnode: root of allocation b+ tree, and EA's */ + +/* Every file and every directory has one fnode, pointed to by the directory + entry and pointing to the file's sectors or directory's root dnode. EA's + are also stored here, and there are said to be ACL's somewhere here too. */ + +#define FNODE_MAGIC 0xf7e40aae + +enum {FNODE_anode = cpu_to_le16(2), FNODE_dir = cpu_to_le16(256)}; +struct fnode +{ + __le32 magic; /* f7e4 0aae */ + __le32 zero1[2]; /* read history */ + u8 len, name[15]; /* true length, truncated name */ + __le32 up; /* pointer to file's directory fnode */ + __le32 acl_size_l; + __le32 acl_secno; + __le16 acl_size_s; + u8 acl_anode; + u8 zero2; /* history bit count */ + __le32 ea_size_l; /* length of disk-resident ea's */ + __le32 ea_secno; /* first sector of disk-resident ea's*/ + __le16 ea_size_s; /* length of fnode-resident ea's */ + + __le16 flags; /* bit 1 set -> ea_secno is an anode */ + /* bit 8 set -> directory. first & only extent + points to dnode. */ + struct bplus_header_fixed btree; /* b+ tree, 8 extents or 12 subtrees */ + union { + struct bplus_leaf_node external[8]; + struct bplus_internal_node internal[12]; + } u; + + __le32 file_size; /* file length, bytes */ + __le32 n_needea; /* number of EA's with NEEDEA set */ + u8 user_id[16]; /* unused */ + __le16 ea_offs; /* offset from start of fnode + to first fnode-resident ea */ + u8 dasd_limit_treshhold; + u8 dasd_limit_delta; + __le32 dasd_limit; + __le32 dasd_usage; + u8 ea[316]; /* zero or more EA's, packed together + with no alignment padding. + (Do not use this name, get here + via fnode + ea_offs. I think.) */ +}; + +static inline bool fnode_in_anode(struct fnode *p) +{ + return (p->flags & FNODE_anode) != 0; +} + +static inline bool fnode_is_dir(struct fnode *p) +{ + return (p->flags & FNODE_dir) != 0; +} + + +/* anode: 99.44% pure allocation tree */ + +#define ANODE_MAGIC 0x37e40aae + +struct anode +{ + __le32 magic; /* 37e4 0aae */ + __le32 self; /* pointer to this anode */ + __le32 up; /* parent anode or fnode */ + + struct bplus_header_fixed btree; /* b+tree, 40 extents or 60 subtrees */ + union { + struct bplus_leaf_node external[40]; + struct bplus_internal_node internal[60]; + } u; + + __le32 fill[3]; /* unused */ +}; + + +/* extended attributes. + + A file's EA info is stored as a list of (name,value) pairs. It is + usually in the fnode, but (if it's large) it is moved to a single + sector run outside the fnode, or to multiple runs with an anode tree + that points to them. + + The value of a single EA is stored along with the name, or (if large) + it is moved to a single sector run, or multiple runs pointed to by an + anode tree, pointed to by the value field of the (name,value) pair. + + Flags in the EA tell whether the value is immediate, in a single sector + run, or in multiple runs. Flags in the fnode tell whether the EA list + is immediate, in a single run, or in multiple runs. */ + +enum {EA_indirect = 1, EA_anode = 2, EA_needea = 128 }; +struct extended_attribute +{ + u8 flags; /* bit 0 set -> value gives sector number + where real value starts */ + /* bit 1 set -> sector is an anode + that points to fragmented value */ + /* bit 7 set -> required ea */ + u8 namelen; /* length of name, bytes */ + u8 valuelen_lo; /* length of value, bytes */ + u8 valuelen_hi; /* length of value, bytes */ + u8 name[]; + /* + u8 name[namelen]; ascii attrib name + u8 nul; terminating '\0', not counted + u8 value[valuelen]; value, arbitrary + if this.flags & 1, valuelen is 8 and the value is + u32 length; real length of value, bytes + secno secno; sector address where it starts + if this.anode, the above sector number is the root of an anode tree + which points to the value. + */ +}; + +static inline bool ea_indirect(struct extended_attribute *ea) +{ + return ea->flags & EA_indirect; +} + +static inline bool ea_in_anode(struct extended_attribute *ea) +{ + return ea->flags & EA_anode; +} + +/* + Local Variables: + comment-column: 40 + End: +*/ diff --git a/fs/hpfs/hpfs_fn.h b/fs/hpfs/hpfs_fn.h new file mode 100644 index 000000000..237c1c23e --- /dev/null +++ b/fs/hpfs/hpfs_fn.h @@ -0,0 +1,379 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * linux/fs/hpfs/hpfs_fn.h + * + * Mikulas Patocka (mikulas@artax.karlin.mff.cuni.cz), 1998-1999 + * + * function headers + */ + +//#define DBG +//#define DEBUG_LOCKS +#ifdef pr_fmt +#undef pr_fmt +#endif + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include + +#include "hpfs.h" + +#define EIOERROR EIO +#define EFSERROR EUCLEAN + +#define ANODE_ALLOC_FWD 512 +#define FNODE_ALLOC_FWD 0 +#define ALLOC_FWD_MIN 16 +#define ALLOC_FWD_MAX 128 +#define ALLOC_M 1 +#define FNODE_RD_AHEAD 16 +#define ANODE_RD_AHEAD 0 +#define DNODE_RD_AHEAD 72 +#define COUNT_RD_AHEAD 62 + +#define FREE_DNODES_ADD 58 +#define FREE_DNODES_DEL 29 + +#define CHKCOND(x,y) if (!(x)) printk y + +struct hpfs_inode_info { + loff_t mmu_private; + ino_t i_parent_dir; /* (directories) gives fnode of parent dir */ + unsigned i_dno; /* (directories) root dnode */ + unsigned i_dpos; /* (directories) temp for readdir */ + unsigned i_dsubdno; /* (directories) temp for readdir */ + unsigned i_file_sec; /* (files) minimalist cache of alloc info */ + unsigned i_disk_sec; /* (files) minimalist cache of alloc info */ + unsigned i_n_secs; /* (files) minimalist cache of alloc info */ + unsigned i_ea_size; /* size of extended attributes */ + unsigned i_ea_mode : 1; /* file's permission is stored in ea */ + unsigned i_ea_uid : 1; /* file's uid is stored in ea */ + unsigned i_ea_gid : 1; /* file's gid is stored in ea */ + unsigned i_dirty : 1; + loff_t **i_rddir_off; + struct inode vfs_inode; +}; + +struct hpfs_sb_info { + struct mutex hpfs_mutex; /* global hpfs lock */ + ino_t sb_root; /* inode number of root dir */ + unsigned sb_fs_size; /* file system size, sectors */ + unsigned sb_bitmaps; /* sector number of bitmap list */ + unsigned sb_dirband_start; /* directory band start sector */ + unsigned sb_dirband_size; /* directory band size, dnodes */ + unsigned sb_dmap; /* sector number of dnode bit map */ + unsigned sb_n_free; /* free blocks for statfs, or -1 */ + unsigned sb_n_free_dnodes; /* free dnodes for statfs, or -1 */ + kuid_t sb_uid; /* uid from mount options */ + kgid_t sb_gid; /* gid from mount options */ + umode_t sb_mode; /* mode from mount options */ + unsigned sb_eas : 2; /* eas: 0-ignore, 1-ro, 2-rw */ + unsigned sb_err : 2; /* on errs: 0-cont, 1-ro, 2-panic */ + unsigned sb_chk : 2; /* checks: 0-no, 1-normal, 2-strict */ + unsigned sb_lowercase : 1; /* downcase filenames hackery */ + unsigned sb_was_error : 1; /* there was an error, set dirty flag */ + unsigned sb_chkdsk : 2; /* chkdsk: 0-no, 1-on errs, 2-allways */ + unsigned char *sb_cp_table; /* code page tables: */ + /* 128 bytes uppercasing table & */ + /* 128 bytes lowercasing table */ + __le32 *sb_bmp_dir; /* main bitmap directory */ + unsigned sb_c_bitmap; /* current bitmap */ + unsigned sb_max_fwd_alloc; /* max forwad allocation */ + int sb_timeshift; + struct rcu_head rcu; + + unsigned n_hotfixes; + secno hotfix_from[256]; + secno hotfix_to[256]; +}; + +/* Four 512-byte buffers and the 2k block obtained by concatenating them */ + +struct quad_buffer_head { + struct buffer_head *bh[4]; + void *data; +}; + +/* The b-tree down pointer from a dir entry */ + +static inline dnode_secno de_down_pointer (struct hpfs_dirent *de) +{ + CHKCOND(de->down,("HPFS: de_down_pointer: !de->down\n")); + return le32_to_cpu(*(__le32 *) ((void *) de + le16_to_cpu(de->length) - 4)); +} + +/* The first dir entry in a dnode */ + +static inline struct hpfs_dirent *dnode_first_de (struct dnode *dnode) +{ + return (void *) dnode->dirent; +} + +/* The end+1 of the dir entries */ + +static inline struct hpfs_dirent *dnode_end_de (struct dnode *dnode) +{ + CHKCOND(le32_to_cpu(dnode->first_free)>=0x14 && le32_to_cpu(dnode->first_free)<=0xa00,("HPFS: dnode_end_de: dnode->first_free = %x\n",(unsigned)le32_to_cpu(dnode->first_free))); + return (void *) dnode + le32_to_cpu(dnode->first_free); +} + +/* The dir entry after dir entry de */ + +static inline struct hpfs_dirent *de_next_de (struct hpfs_dirent *de) +{ + CHKCOND(le16_to_cpu(de->length)>=0x20 && le16_to_cpu(de->length)<0x800,("HPFS: de_next_de: de->length = %x\n",(unsigned)le16_to_cpu(de->length))); + return (void *) de + le16_to_cpu(de->length); +} + +static inline struct extended_attribute *fnode_ea(struct fnode *fnode) +{ + return (struct extended_attribute *)((char *)fnode + le16_to_cpu(fnode->ea_offs) + le16_to_cpu(fnode->acl_size_s)); +} + +static inline struct extended_attribute *fnode_end_ea(struct fnode *fnode) +{ + return (struct extended_attribute *)((char *)fnode + le16_to_cpu(fnode->ea_offs) + le16_to_cpu(fnode->acl_size_s) + le16_to_cpu(fnode->ea_size_s)); +} + +static unsigned ea_valuelen(struct extended_attribute *ea) +{ + return ea->valuelen_lo + 256 * ea->valuelen_hi; +} + +static inline struct extended_attribute *next_ea(struct extended_attribute *ea) +{ + return (struct extended_attribute *)((char *)ea + 5 + ea->namelen + ea_valuelen(ea)); +} + +static inline secno ea_sec(struct extended_attribute *ea) +{ + return le32_to_cpu(get_unaligned((__le32 *)((char *)ea + 9 + ea->namelen))); +} + +static inline secno ea_len(struct extended_attribute *ea) +{ + return le32_to_cpu(get_unaligned((__le32 *)((char *)ea + 5 + ea->namelen))); +} + +static inline char *ea_data(struct extended_attribute *ea) +{ + return (char *)((char *)ea + 5 + ea->namelen); +} + +static inline unsigned de_size(int namelen, secno down_ptr) +{ + return ((0x1f + namelen + 3) & ~3) + (down_ptr ? 4 : 0); +} + +static inline void copy_de(struct hpfs_dirent *dst, struct hpfs_dirent *src) +{ + int a; + int n; + if (!dst || !src) return; + a = dst->down; + n = dst->not_8x3; + memcpy((char *)dst + 2, (char *)src + 2, 28); + dst->down = a; + dst->not_8x3 = n; +} + +static inline unsigned tstbits(__le32 *bmp, unsigned b, unsigned n) +{ + int i; + if ((b >= 0x4000) || (b + n - 1 >= 0x4000)) return n; + if (!((le32_to_cpu(bmp[(b & 0x3fff) >> 5]) >> (b & 0x1f)) & 1)) return 1; + for (i = 1; i < n; i++) + if (!((le32_to_cpu(bmp[((b+i) & 0x3fff) >> 5]) >> ((b+i) & 0x1f)) & 1)) + return i + 1; + return 0; +} + +/* alloc.c */ + +int hpfs_chk_sectors(struct super_block *, secno, int, char *); +secno hpfs_alloc_sector(struct super_block *, secno, unsigned, int); +int hpfs_alloc_if_possible(struct super_block *, secno); +void hpfs_free_sectors(struct super_block *, secno, unsigned); +int hpfs_check_free_dnodes(struct super_block *, int); +void hpfs_free_dnode(struct super_block *, secno); +struct dnode *hpfs_alloc_dnode(struct super_block *, secno, dnode_secno *, struct quad_buffer_head *); +struct fnode *hpfs_alloc_fnode(struct super_block *, secno, fnode_secno *, struct buffer_head **); +struct anode *hpfs_alloc_anode(struct super_block *, secno, anode_secno *, struct buffer_head **); +int hpfs_trim_fs(struct super_block *, u64, u64, u64, unsigned *); + +/* anode.c */ + +secno hpfs_bplus_lookup(struct super_block *, struct inode *, struct bplus_header *, unsigned, struct buffer_head *); +secno hpfs_add_sector_to_btree(struct super_block *, secno, int, unsigned); +void hpfs_remove_btree(struct super_block *, struct bplus_header *); +int hpfs_ea_read(struct super_block *, secno, int, unsigned, unsigned, char *); +int hpfs_ea_write(struct super_block *, secno, int, unsigned, unsigned, const char *); +void hpfs_ea_remove(struct super_block *, secno, int, unsigned); +void hpfs_truncate_btree(struct super_block *, secno, int, unsigned); +void hpfs_remove_fnode(struct super_block *, fnode_secno fno); + +/* buffer.c */ + +secno hpfs_search_hotfix_map(struct super_block *s, secno sec); +unsigned hpfs_search_hotfix_map_for_range(struct super_block *s, secno sec, unsigned n); +void hpfs_prefetch_sectors(struct super_block *, unsigned, int); +void *hpfs_map_sector(struct super_block *, unsigned, struct buffer_head **, int); +void *hpfs_get_sector(struct super_block *, unsigned, struct buffer_head **); +void *hpfs_map_4sectors(struct super_block *, unsigned, struct quad_buffer_head *, int); +void *hpfs_get_4sectors(struct super_block *, unsigned, struct quad_buffer_head *); +void hpfs_brelse4(struct quad_buffer_head *); +void hpfs_mark_4buffers_dirty(struct quad_buffer_head *); + +/* dentry.c */ + +extern const struct dentry_operations hpfs_dentry_operations; + +/* dir.c */ + +struct dentry *hpfs_lookup(struct inode *, struct dentry *, unsigned int); +extern const struct file_operations hpfs_dir_ops; + +/* dnode.c */ + +int hpfs_add_pos(struct inode *, loff_t *); +void hpfs_del_pos(struct inode *, loff_t *); +struct hpfs_dirent *hpfs_add_de(struct super_block *, struct dnode *, + const unsigned char *, unsigned, secno); +int hpfs_add_dirent(struct inode *, const unsigned char *, unsigned, + struct hpfs_dirent *); +int hpfs_remove_dirent(struct inode *, dnode_secno, struct hpfs_dirent *, struct quad_buffer_head *, int); +void hpfs_count_dnodes(struct super_block *, dnode_secno, int *, int *, int *); +dnode_secno hpfs_de_as_down_as_possible(struct super_block *, dnode_secno dno); +struct hpfs_dirent *map_pos_dirent(struct inode *, loff_t *, struct quad_buffer_head *); +struct hpfs_dirent *map_dirent(struct inode *, dnode_secno, + const unsigned char *, unsigned, dnode_secno *, + struct quad_buffer_head *); +void hpfs_remove_dtree(struct super_block *, dnode_secno); +struct hpfs_dirent *map_fnode_dirent(struct super_block *, fnode_secno, struct fnode *, struct quad_buffer_head *); + +/* ea.c */ + +void hpfs_ea_ext_remove(struct super_block *, secno, int, unsigned); +int hpfs_read_ea(struct super_block *, struct fnode *, char *, char *, int); +char *hpfs_get_ea(struct super_block *, struct fnode *, char *, int *); +void hpfs_set_ea(struct inode *, struct fnode *, const char *, + const char *, int); + +/* file.c */ + +int hpfs_file_fsync(struct file *, loff_t, loff_t, int); +void hpfs_truncate(struct inode *); +extern const struct file_operations hpfs_file_ops; +extern const struct inode_operations hpfs_file_iops; +extern const struct address_space_operations hpfs_aops; + +/* inode.c */ + +void hpfs_init_inode(struct inode *); +void hpfs_read_inode(struct inode *); +void hpfs_write_inode(struct inode *); +void hpfs_write_inode_nolock(struct inode *); +int hpfs_setattr(struct mnt_idmap *, struct dentry *, struct iattr *); +void hpfs_write_if_changed(struct inode *); +void hpfs_evict_inode(struct inode *); + +/* map.c */ + +__le32 *hpfs_map_dnode_bitmap(struct super_block *, struct quad_buffer_head *); +__le32 *hpfs_map_bitmap(struct super_block *, unsigned, struct quad_buffer_head *, char *); +void hpfs_prefetch_bitmap(struct super_block *, unsigned); +unsigned char *hpfs_load_code_page(struct super_block *, secno); +__le32 *hpfs_load_bitmap_directory(struct super_block *, secno bmp); +void hpfs_load_hotfix_map(struct super_block *s, struct hpfs_spare_block *spareblock); +struct fnode *hpfs_map_fnode(struct super_block *s, ino_t, struct buffer_head **); +struct anode *hpfs_map_anode(struct super_block *s, anode_secno, struct buffer_head **); +struct dnode *hpfs_map_dnode(struct super_block *s, dnode_secno, struct quad_buffer_head *); +dnode_secno hpfs_fnode_dno(struct super_block *s, ino_t ino); + +/* name.c */ + +unsigned char hpfs_upcase(unsigned char *, unsigned char); +int hpfs_chk_name(const unsigned char *, unsigned *); +unsigned char *hpfs_translate_name(struct super_block *, unsigned char *, unsigned, int, int); +int hpfs_compare_names(struct super_block *, const unsigned char *, unsigned, + const unsigned char *, unsigned, int); +int hpfs_is_name_long(const unsigned char *, unsigned); +void hpfs_adjust_length(const unsigned char *, unsigned *); + +/* namei.c */ + +extern const struct inode_operations hpfs_dir_iops; +extern const struct address_space_operations hpfs_symlink_aops; + +static inline struct hpfs_inode_info *hpfs_i(struct inode *inode) +{ + return container_of(inode, struct hpfs_inode_info, vfs_inode); +} + +static inline struct hpfs_sb_info *hpfs_sb(struct super_block *sb) +{ + return sb->s_fs_info; +} + +/* super.c */ + +__printf(2, 3) +void hpfs_error(struct super_block *, const char *, ...); +int hpfs_stop_cycles(struct super_block *, int, int *, int *, char *); +unsigned hpfs_get_free_dnodes(struct super_block *); +long hpfs_ioctl(struct file *file, unsigned cmd, unsigned long arg); + +/* + * local time (HPFS) to GMT (Unix) + */ + +static inline time64_t local_to_gmt(struct super_block *s, time64_t t) +{ + extern struct timezone sys_tz; + return t + sys_tz.tz_minuteswest * 60 + hpfs_sb(s)->sb_timeshift; +} + +static inline time32_t gmt_to_local(struct super_block *s, time64_t t) +{ + extern struct timezone sys_tz; + return t - sys_tz.tz_minuteswest * 60 - hpfs_sb(s)->sb_timeshift; +} + +static inline time32_t local_get_seconds(struct super_block *s) +{ + return gmt_to_local(s, ktime_get_real_seconds()); +} + +/* + * Locking: + * + * hpfs_lock() locks the whole filesystem. It must be taken + * on any method called by the VFS. + * + * We don't do any per-file locking anymore, it is hard to + * review and HPFS is not performance-sensitive anyway. + */ +static inline void hpfs_lock(struct super_block *s) +{ + struct hpfs_sb_info *sbi = hpfs_sb(s); + mutex_lock(&sbi->hpfs_mutex); +} + +static inline void hpfs_unlock(struct super_block *s) +{ + struct hpfs_sb_info *sbi = hpfs_sb(s); + mutex_unlock(&sbi->hpfs_mutex); +} + +static inline void hpfs_lock_assert(struct super_block *s) +{ + struct hpfs_sb_info *sbi = hpfs_sb(s); + WARN_ON(!mutex_is_locked(&sbi->hpfs_mutex)); +} diff --git a/fs/hpfs/inode.c b/fs/hpfs/inode.c new file mode 100644 index 000000000..1b4fcf760 --- /dev/null +++ b/fs/hpfs/inode.c @@ -0,0 +1,318 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hpfs/inode.c + * + * Mikulas Patocka (mikulas@artax.karlin.mff.cuni.cz), 1998-1999 + * + * inode VFS functions + */ + +#include +#include +#include "hpfs_fn.h" + +void hpfs_init_inode(struct inode *i) +{ + struct super_block *sb = i->i_sb; + struct hpfs_inode_info *hpfs_inode = hpfs_i(i); + + i->i_uid = hpfs_sb(sb)->sb_uid; + i->i_gid = hpfs_sb(sb)->sb_gid; + i->i_mode = hpfs_sb(sb)->sb_mode; + i->i_size = -1; + i->i_blocks = -1; + + hpfs_inode->i_dno = 0; + hpfs_inode->i_n_secs = 0; + hpfs_inode->i_file_sec = 0; + hpfs_inode->i_disk_sec = 0; + hpfs_inode->i_dpos = 0; + hpfs_inode->i_dsubdno = 0; + hpfs_inode->i_ea_mode = 0; + hpfs_inode->i_ea_uid = 0; + hpfs_inode->i_ea_gid = 0; + hpfs_inode->i_ea_size = 0; + + hpfs_inode->i_rddir_off = NULL; + hpfs_inode->i_dirty = 0; + + inode_set_ctime(i, 0, 0); + inode_set_mtime(i, 0, 0); + inode_set_atime(i, 0, 0); +} + +void hpfs_read_inode(struct inode *i) +{ + struct buffer_head *bh; + struct fnode *fnode; + struct super_block *sb = i->i_sb; + struct hpfs_inode_info *hpfs_inode = hpfs_i(i); + void *ea; + int ea_size; + + if (!(fnode = hpfs_map_fnode(sb, i->i_ino, &bh))) { + /*i->i_mode |= S_IFREG; + i->i_mode &= ~0111; + i->i_op = &hpfs_file_iops; + i->i_fop = &hpfs_file_ops; + clear_nlink(i);*/ + make_bad_inode(i); + return; + } + if (hpfs_sb(i->i_sb)->sb_eas) { + if ((ea = hpfs_get_ea(i->i_sb, fnode, "UID", &ea_size))) { + if (ea_size == 2) { + i_uid_write(i, le16_to_cpu(*(__le16*)ea)); + hpfs_inode->i_ea_uid = 1; + } + kfree(ea); + } + if ((ea = hpfs_get_ea(i->i_sb, fnode, "GID", &ea_size))) { + if (ea_size == 2) { + i_gid_write(i, le16_to_cpu(*(__le16*)ea)); + hpfs_inode->i_ea_gid = 1; + } + kfree(ea); + } + if ((ea = hpfs_get_ea(i->i_sb, fnode, "SYMLINK", &ea_size))) { + kfree(ea); + i->i_mode = S_IFLNK | 0777; + i->i_op = &page_symlink_inode_operations; + inode_nohighmem(i); + i->i_data.a_ops = &hpfs_symlink_aops; + set_nlink(i, 1); + i->i_size = ea_size; + i->i_blocks = 1; + brelse(bh); + return; + } + if ((ea = hpfs_get_ea(i->i_sb, fnode, "MODE", &ea_size))) { + int rdev = 0; + umode_t mode = hpfs_sb(sb)->sb_mode; + if (ea_size == 2) { + mode = le16_to_cpu(*(__le16*)ea); + hpfs_inode->i_ea_mode = 1; + } + kfree(ea); + i->i_mode = mode; + if (S_ISBLK(mode) || S_ISCHR(mode)) { + if ((ea = hpfs_get_ea(i->i_sb, fnode, "DEV", &ea_size))) { + if (ea_size == 4) + rdev = le32_to_cpu(*(__le32*)ea); + kfree(ea); + } + } + if (S_ISBLK(mode) || S_ISCHR(mode) || S_ISFIFO(mode) || S_ISSOCK(mode)) { + brelse(bh); + set_nlink(i, 1); + i->i_size = 0; + i->i_blocks = 1; + init_special_inode(i, mode, + new_decode_dev(rdev)); + return; + } + } + } + if (fnode_is_dir(fnode)) { + int n_dnodes, n_subdirs; + i->i_mode |= S_IFDIR; + i->i_op = &hpfs_dir_iops; + i->i_fop = &hpfs_dir_ops; + hpfs_inode->i_parent_dir = le32_to_cpu(fnode->up); + hpfs_inode->i_dno = le32_to_cpu(fnode->u.external[0].disk_secno); + if (hpfs_sb(sb)->sb_chk >= 2) { + struct buffer_head *bh0; + if (hpfs_map_fnode(sb, hpfs_inode->i_parent_dir, &bh0)) brelse(bh0); + } + n_dnodes = 0; n_subdirs = 0; + hpfs_count_dnodes(i->i_sb, hpfs_inode->i_dno, &n_dnodes, &n_subdirs, NULL); + i->i_blocks = 4 * n_dnodes; + i->i_size = 2048 * n_dnodes; + set_nlink(i, 2 + n_subdirs); + } else { + i->i_mode |= S_IFREG; + if (!hpfs_inode->i_ea_mode) i->i_mode &= ~0111; + i->i_op = &hpfs_file_iops; + i->i_fop = &hpfs_file_ops; + set_nlink(i, 1); + i->i_size = le32_to_cpu(fnode->file_size); + i->i_blocks = ((i->i_size + 511) >> 9) + 1; + i->i_data.a_ops = &hpfs_aops; + hpfs_i(i)->mmu_private = i->i_size; + } + brelse(bh); +} + +static void hpfs_write_inode_ea(struct inode *i, struct fnode *fnode) +{ + struct hpfs_inode_info *hpfs_inode = hpfs_i(i); + /*if (le32_to_cpu(fnode->acl_size_l) || le16_to_cpu(fnode->acl_size_s)) { + Some unknown structures like ACL may be in fnode, + we'd better not overwrite them + hpfs_error(i->i_sb, "fnode %08x has some unknown HPFS386 structures", i->i_ino); + } else*/ if (hpfs_sb(i->i_sb)->sb_eas >= 2) { + __le32 ea; + if (!uid_eq(i->i_uid, hpfs_sb(i->i_sb)->sb_uid) || hpfs_inode->i_ea_uid) { + ea = cpu_to_le32(i_uid_read(i)); + hpfs_set_ea(i, fnode, "UID", (char*)&ea, 2); + hpfs_inode->i_ea_uid = 1; + } + if (!gid_eq(i->i_gid, hpfs_sb(i->i_sb)->sb_gid) || hpfs_inode->i_ea_gid) { + ea = cpu_to_le32(i_gid_read(i)); + hpfs_set_ea(i, fnode, "GID", (char *)&ea, 2); + hpfs_inode->i_ea_gid = 1; + } + if (!S_ISLNK(i->i_mode)) + if ((i->i_mode != ((hpfs_sb(i->i_sb)->sb_mode & ~(S_ISDIR(i->i_mode) ? 0 : 0111)) + | (S_ISDIR(i->i_mode) ? S_IFDIR : S_IFREG)) + && i->i_mode != ((hpfs_sb(i->i_sb)->sb_mode & ~(S_ISDIR(i->i_mode) ? 0222 : 0333)) + | (S_ISDIR(i->i_mode) ? S_IFDIR : S_IFREG))) || hpfs_inode->i_ea_mode) { + ea = cpu_to_le32(i->i_mode); + /* sick, but legal */ + hpfs_set_ea(i, fnode, "MODE", (char *)&ea, 2); + hpfs_inode->i_ea_mode = 1; + } + if (S_ISBLK(i->i_mode) || S_ISCHR(i->i_mode)) { + ea = cpu_to_le32(new_encode_dev(i->i_rdev)); + hpfs_set_ea(i, fnode, "DEV", (char *)&ea, 4); + } + } +} + +void hpfs_write_inode(struct inode *i) +{ + struct hpfs_inode_info *hpfs_inode = hpfs_i(i); + struct inode *parent; + if (i->i_ino == hpfs_sb(i->i_sb)->sb_root) return; + if (hpfs_inode->i_rddir_off && !icount_read_once(i)) { + if (*hpfs_inode->i_rddir_off) + pr_err("write_inode: some position still there\n"); + kfree(hpfs_inode->i_rddir_off); + hpfs_inode->i_rddir_off = NULL; + } + if (!i->i_nlink) { + return; + } + parent = iget_locked(i->i_sb, hpfs_inode->i_parent_dir); + if (parent) { + hpfs_inode->i_dirty = 0; + if (inode_state_read_once(parent) & I_NEW) { + hpfs_init_inode(parent); + hpfs_read_inode(parent); + unlock_new_inode(parent); + } + hpfs_write_inode_nolock(i); + iput(parent); + } +} + +void hpfs_write_inode_nolock(struct inode *i) +{ + struct hpfs_inode_info *hpfs_inode = hpfs_i(i); + struct buffer_head *bh; + struct fnode *fnode; + struct quad_buffer_head qbh; + struct hpfs_dirent *de; + if (i->i_ino == hpfs_sb(i->i_sb)->sb_root) return; + if (!(fnode = hpfs_map_fnode(i->i_sb, i->i_ino, &bh))) return; + if (i->i_ino != hpfs_sb(i->i_sb)->sb_root && i->i_nlink) { + if (!(de = map_fnode_dirent(i->i_sb, i->i_ino, fnode, &qbh))) { + brelse(bh); + return; + } + } else de = NULL; + if (S_ISREG(i->i_mode)) { + fnode->file_size = cpu_to_le32(i->i_size); + if (de) de->file_size = cpu_to_le32(i->i_size); + } else if (S_ISDIR(i->i_mode)) { + fnode->file_size = cpu_to_le32(0); + if (de) de->file_size = cpu_to_le32(0); + } + hpfs_write_inode_ea(i, fnode); + if (de) { + de->write_date = cpu_to_le32(gmt_to_local(i->i_sb, inode_get_mtime_sec(i))); + de->read_date = cpu_to_le32(gmt_to_local(i->i_sb, inode_get_atime_sec(i))); + de->creation_date = cpu_to_le32(gmt_to_local(i->i_sb, inode_get_ctime_sec(i))); + de->read_only = !(i->i_mode & 0222); + de->ea_size = cpu_to_le32(hpfs_inode->i_ea_size); + hpfs_mark_4buffers_dirty(&qbh); + hpfs_brelse4(&qbh); + } + if (S_ISDIR(i->i_mode)) { + if ((de = map_dirent(i, hpfs_inode->i_dno, "\001\001", 2, NULL, &qbh))) { + de->write_date = cpu_to_le32(gmt_to_local(i->i_sb, inode_get_mtime_sec(i))); + de->read_date = cpu_to_le32(gmt_to_local(i->i_sb, inode_get_atime_sec(i))); + de->creation_date = cpu_to_le32(gmt_to_local(i->i_sb, inode_get_ctime_sec(i))); + de->read_only = !(i->i_mode & 0222); + de->ea_size = cpu_to_le32(/*hpfs_inode->i_ea_size*/0); + de->file_size = cpu_to_le32(0); + hpfs_mark_4buffers_dirty(&qbh); + hpfs_brelse4(&qbh); + } else + hpfs_error(i->i_sb, + "directory %08llx doesn't have '.' entry", + i->i_ino); + } + mark_buffer_dirty(bh); + brelse(bh); +} + +int hpfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +{ + struct inode *inode = d_inode(dentry); + int error = -EINVAL; + + hpfs_lock(inode->i_sb); + if (inode->i_ino == hpfs_sb(inode->i_sb)->sb_root) + goto out_unlock; + if ((attr->ia_valid & ATTR_UID) && + from_kuid(&init_user_ns, attr->ia_uid) >= 0x10000) + goto out_unlock; + if ((attr->ia_valid & ATTR_GID) && + from_kgid(&init_user_ns, attr->ia_gid) >= 0x10000) + goto out_unlock; + if ((attr->ia_valid & ATTR_SIZE) && attr->ia_size > inode->i_size) + goto out_unlock; + + error = setattr_prepare(&nop_mnt_idmap, dentry, attr); + if (error) + goto out_unlock; + + if ((attr->ia_valid & ATTR_SIZE) && + attr->ia_size != i_size_read(inode)) { + error = inode_newsize_ok(inode, attr->ia_size); + if (error) + goto out_unlock; + + truncate_setsize(inode, attr->ia_size); + hpfs_truncate(inode); + } + + setattr_copy(&nop_mnt_idmap, inode, attr); + + hpfs_write_inode(inode); + + out_unlock: + hpfs_unlock(inode->i_sb); + return error; +} + +void hpfs_write_if_changed(struct inode *inode) +{ + struct hpfs_inode_info *hpfs_inode = hpfs_i(inode); + + if (hpfs_inode->i_dirty) + hpfs_write_inode(inode); +} + +void hpfs_evict_inode(struct inode *inode) +{ + truncate_inode_pages_final(&inode->i_data); + clear_inode(inode); + if (!inode->i_nlink) { + hpfs_lock(inode->i_sb); + hpfs_remove_fnode(inode->i_sb, inode->i_ino); + hpfs_unlock(inode->i_sb); + } +} diff --git a/fs/hpfs/map.c b/fs/hpfs/map.c new file mode 100644 index 000000000..be7323350 --- /dev/null +++ b/fs/hpfs/map.c @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hpfs/map.c + * + * Mikulas Patocka (mikulas@artax.karlin.mff.cuni.cz), 1998-1999 + * + * mapping structures to memory with some minimal checks + */ + +#include "hpfs_fn.h" + +__le32 *hpfs_map_dnode_bitmap(struct super_block *s, struct quad_buffer_head *qbh) +{ + return hpfs_map_4sectors(s, hpfs_sb(s)->sb_dmap, qbh, 0); +} + +__le32 *hpfs_map_bitmap(struct super_block *s, unsigned bmp_block, + struct quad_buffer_head *qbh, char *id) +{ + secno sec; + __le32 *ret; + unsigned n_bands = (hpfs_sb(s)->sb_fs_size + 0x3fff) >> 14; + if (hpfs_sb(s)->sb_chk) if (bmp_block >= n_bands) { + hpfs_error(s, "hpfs_map_bitmap called with bad parameter: %08x at %s", bmp_block, id); + return NULL; + } + sec = le32_to_cpu(hpfs_sb(s)->sb_bmp_dir[bmp_block]); + if (!sec || sec > hpfs_sb(s)->sb_fs_size-4) { + hpfs_error(s, "invalid bitmap block pointer %08x -> %08x at %s", bmp_block, sec, id); + return NULL; + } + ret = hpfs_map_4sectors(s, sec, qbh, 4); + if (ret) hpfs_prefetch_bitmap(s, bmp_block + 1); + return ret; +} + +void hpfs_prefetch_bitmap(struct super_block *s, unsigned bmp_block) +{ + unsigned to_prefetch, next_prefetch; + unsigned n_bands = (hpfs_sb(s)->sb_fs_size + 0x3fff) >> 14; + if (unlikely(bmp_block >= n_bands)) + return; + to_prefetch = le32_to_cpu(hpfs_sb(s)->sb_bmp_dir[bmp_block]); + if (unlikely(bmp_block + 1 >= n_bands)) + next_prefetch = 0; + else + next_prefetch = le32_to_cpu(hpfs_sb(s)->sb_bmp_dir[bmp_block + 1]); + hpfs_prefetch_sectors(s, to_prefetch, 4 + 4 * (to_prefetch + 4 == next_prefetch)); +} + +/* + * Load first code page into kernel memory, return pointer to 256-byte array, + * first 128 bytes are uppercasing table for chars 128-255, next 128 bytes are + * lowercasing table + */ + +unsigned char *hpfs_load_code_page(struct super_block *s, secno cps) +{ + struct buffer_head *bh; + secno cpds; + unsigned cpi; + unsigned char *ptr; + unsigned char *cp_table; + int i; + struct code_page_data *cpd; + struct code_page_directory *cp = hpfs_map_sector(s, cps, &bh, 0); + if (!cp) return NULL; + if (le32_to_cpu(cp->magic) != CP_DIR_MAGIC) { + pr_err("Code page directory magic doesn't match (magic = %08x)\n", + le32_to_cpu(cp->magic)); + brelse(bh); + return NULL; + } + if (!le32_to_cpu(cp->n_code_pages)) { + pr_err("n_code_pages == 0\n"); + brelse(bh); + return NULL; + } + cpds = le32_to_cpu(cp->array[0].code_page_data); + cpi = le16_to_cpu(cp->array[0].index); + brelse(bh); + + if (cpi >= 3) { + pr_err("Code page index out of array\n"); + return NULL; + } + + if (!(cpd = hpfs_map_sector(s, cpds, &bh, 0))) return NULL; + if (le16_to_cpu(cpd->offs[cpi]) > 0x178) { + pr_err("Code page index out of sector\n"); + brelse(bh); + return NULL; + } + ptr = (unsigned char *)cpd + le16_to_cpu(cpd->offs[cpi]) + 6; + if (!(cp_table = kmalloc(256, GFP_KERNEL))) { + pr_err("out of memory for code page table\n"); + brelse(bh); + return NULL; + } + memcpy(cp_table, ptr, 128); + brelse(bh); + + /* Try to build lowercasing table from uppercasing one */ + + for (i=128; i<256; i++) cp_table[i]=i; + for (i=128; i<256; i++) if (cp_table[i-128]!=i && cp_table[i-128]>=128) + cp_table[cp_table[i-128]] = i; + + return cp_table; +} + +__le32 *hpfs_load_bitmap_directory(struct super_block *s, secno bmp) +{ + struct buffer_head *bh; + int n = (hpfs_sb(s)->sb_fs_size + 0x200000 - 1) >> 21; + int i; + __le32 *b; + if (!(b = kmalloc_array(n, 512, GFP_KERNEL))) { + pr_err("can't allocate memory for bitmap directory\n"); + return NULL; + } + for (i=0;in_spares); + n_used_hotfixes = le32_to_cpu(spareblock->n_spares_used); + + if (n_hotfixes > 256 || n_used_hotfixes > n_hotfixes) { + hpfs_error(s, "invalid number of hotfixes: %u, used: %u", n_hotfixes, n_used_hotfixes); + return; + } + if (!(directory = hpfs_map_4sectors(s, le32_to_cpu(spareblock->hotfix_map), &qbh, 0))) { + hpfs_error(s, "can't load hotfix map"); + return; + } + for (i = 0; i < n_used_hotfixes; i++) { + hpfs_sb(s)->hotfix_from[i] = le32_to_cpu(directory[i]); + hpfs_sb(s)->hotfix_to[i] = le32_to_cpu(directory[n_hotfixes + i]); + } + hpfs_sb(s)->n_hotfixes = n_used_hotfixes; + hpfs_brelse4(&qbh); +} + +/* + * Load fnode to memory + */ + +struct fnode *hpfs_map_fnode(struct super_block *s, ino_t ino, struct buffer_head **bhp) +{ + struct fnode *fnode; + if (hpfs_sb(s)->sb_chk) if (hpfs_chk_sectors(s, ino, 1, "fnode")) { + return NULL; + } + if ((fnode = hpfs_map_sector(s, ino, bhp, FNODE_RD_AHEAD))) { + if (hpfs_sb(s)->sb_chk) { + struct extended_attribute *ea; + struct extended_attribute *ea_end; + if (le32_to_cpu(fnode->magic) != FNODE_MAGIC) { + hpfs_error(s, "bad magic on fnode %08lx", + (unsigned long)ino); + goto bail; + } + if (!fnode_is_dir(fnode)) { + if ((unsigned)fnode->btree.n_used_nodes + (unsigned)fnode->btree.n_free_nodes != + (bp_internal(GET_BTREE_PTR(&fnode->btree)) ? 12 : 8)) { + hpfs_error(s, + "bad number of nodes in fnode %08lx", + (unsigned long)ino); + goto bail; + } + if (le16_to_cpu(fnode->btree.first_free) != + 8 + fnode->btree.n_used_nodes * (bp_internal(GET_BTREE_PTR(&fnode->btree)) ? 8 : 12)) { + hpfs_error(s, + "bad first_free pointer in fnode %08lx", + (unsigned long)ino); + goto bail; + } + } + if (le16_to_cpu(fnode->ea_size_s) && (le16_to_cpu(fnode->ea_offs) < 0xc4 || + le16_to_cpu(fnode->ea_offs) + le16_to_cpu(fnode->acl_size_s) + le16_to_cpu(fnode->ea_size_s) > 0x200)) { + hpfs_error(s, + "bad EA info in fnode %08lx: ea_offs == %04x ea_size_s == %04x", + (unsigned long)ino, + le16_to_cpu(fnode->ea_offs), le16_to_cpu(fnode->ea_size_s)); + goto bail; + } + ea = fnode_ea(fnode); + ea_end = fnode_end_ea(fnode); + while (ea != ea_end) { + if (ea > ea_end) { + hpfs_error(s, "bad EA in fnode %08lx", + (unsigned long)ino); + goto bail; + } + ea = next_ea(ea); + } + } + } + return fnode; + bail: + brelse(*bhp); + return NULL; +} + +struct anode *hpfs_map_anode(struct super_block *s, anode_secno ano, struct buffer_head **bhp) +{ + struct anode *anode; + if (hpfs_sb(s)->sb_chk) if (hpfs_chk_sectors(s, ano, 1, "anode")) return NULL; + if ((anode = hpfs_map_sector(s, ano, bhp, ANODE_RD_AHEAD))) + if (hpfs_sb(s)->sb_chk) { + if (le32_to_cpu(anode->magic) != ANODE_MAGIC) { + hpfs_error(s, "bad magic on anode %08x", ano); + goto bail; + } + if (le32_to_cpu(anode->self) != ano) { + hpfs_error(s, "self pointer invalid on anode %08x", ano); + goto bail; + } + if ((unsigned)anode->btree.n_used_nodes + (unsigned)anode->btree.n_free_nodes != + (bp_internal(GET_BTREE_PTR(&anode->btree)) ? 60 : 40)) { + hpfs_error(s, "bad number of nodes in anode %08x", ano); + goto bail; + } + if (le16_to_cpu(anode->btree.first_free) != + 8 + anode->btree.n_used_nodes * (bp_internal(GET_BTREE_PTR(&anode->btree)) ? 8 : 12)) { + hpfs_error(s, "bad first_free pointer in anode %08x", ano); + goto bail; + } + } + return anode; + bail: + brelse(*bhp); + return NULL; +} + +/* + * Load dnode to memory and do some checks + */ + +struct dnode *hpfs_map_dnode(struct super_block *s, unsigned secno, + struct quad_buffer_head *qbh) +{ + struct dnode *dnode; + if (hpfs_sb(s)->sb_chk) { + if (hpfs_chk_sectors(s, secno, 4, "dnode")) return NULL; + if (secno & 3) { + hpfs_error(s, "dnode %08x not byte-aligned", secno); + return NULL; + } + } + if ((dnode = hpfs_map_4sectors(s, secno, qbh, DNODE_RD_AHEAD))) + if (hpfs_sb(s)->sb_chk) { + unsigned p, pp = 0; + unsigned char *d = (unsigned char *)dnode; + int b = 0; + if (le32_to_cpu(dnode->magic) != DNODE_MAGIC) { + hpfs_error(s, "bad magic on dnode %08x", secno); + goto bail; + } + if (le32_to_cpu(dnode->self) != secno) + hpfs_error(s, "bad self pointer on dnode %08x self = %08x", secno, le32_to_cpu(dnode->self)); + /* Check dirents - bad dirents would cause infinite + loops or shooting to memory */ + if (le32_to_cpu(dnode->first_free) > 2048) { + hpfs_error(s, "dnode %08x has first_free == %08x", secno, le32_to_cpu(dnode->first_free)); + goto bail; + } + for (p = 20; p < le32_to_cpu(dnode->first_free); p += d[p] + (d[p+1] << 8)) { + struct hpfs_dirent *de = (struct hpfs_dirent *)((char *)dnode + p); + if (le16_to_cpu(de->length) > 292 || (le16_to_cpu(de->length) < 32) || (le16_to_cpu(de->length) & 3) || p + le16_to_cpu(de->length) > 2048) { + hpfs_error(s, "bad dirent size in dnode %08x, dirent %03x, last %03x", secno, p, pp); + goto bail; + } + if (((31 + de->namelen + de->down*4 + 3) & ~3) != le16_to_cpu(de->length)) { + if (((31 + de->namelen + de->down*4 + 3) & ~3) < le16_to_cpu(de->length) && s->s_flags & SB_RDONLY) goto ok; + hpfs_error(s, "namelen does not match dirent size in dnode %08x, dirent %03x, last %03x", secno, p, pp); + goto bail; + } + ok: + if (hpfs_sb(s)->sb_chk >= 2) b |= 1 << de->down; + if (de->down) if (de_down_pointer(de) < 0x10) { + hpfs_error(s, "bad down pointer in dnode %08x, dirent %03x, last %03x", secno, p, pp); + goto bail; + } + pp = p; + + } + if (p != le32_to_cpu(dnode->first_free)) { + hpfs_error(s, "size on last dirent does not match first_free; dnode %08x", secno); + goto bail; + } + if (d[pp + 30] != 1 || d[pp + 31] != 255) { + hpfs_error(s, "dnode %08x does not end with \\377 entry", secno); + goto bail; + } + if (b == 3) + pr_err("unbalanced dnode tree, dnode %08x; see hpfs.txt 4 more info\n", + secno); + } + return dnode; + bail: + hpfs_brelse4(qbh); + return NULL; +} + +dnode_secno hpfs_fnode_dno(struct super_block *s, ino_t ino) +{ + struct buffer_head *bh; + struct fnode *fnode; + dnode_secno dno; + + fnode = hpfs_map_fnode(s, ino, &bh); + if (!fnode) + return 0; + + dno = le32_to_cpu(fnode->u.external[0].disk_secno); + brelse(bh); + return dno; +} diff --git a/fs/hpfs/name.c b/fs/hpfs/name.c new file mode 100644 index 000000000..ef7ba77f3 --- /dev/null +++ b/fs/hpfs/name.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hpfs/name.c + * + * Mikulas Patocka (mikulas@artax.karlin.mff.cuni.cz), 1998-1999 + * + * operations with filenames + */ + +#include "hpfs_fn.h" + +static inline int not_allowed_char(unsigned char c) +{ + return c<' ' || c=='"' || c=='*' || c=='/' || c==':' || c=='<' || + c=='>' || c=='?' || c=='\\' || c=='|'; +} + +static inline int no_dos_char(unsigned char c) +{ /* Characters that are allowed in HPFS but not in DOS */ + return c=='+' || c==',' || c==';' || c=='=' || c=='[' || c==']'; +} + +static inline unsigned char upcase(unsigned char *dir, unsigned char a) +{ + if (a<128 || a==255) return a>='a' && a<='z' ? a - 0x20 : a; + if (!dir) return a; + return dir[a-128]; +} + +unsigned char hpfs_upcase(unsigned char *dir, unsigned char a) +{ + return upcase(dir, a); +} + +static inline unsigned char locase(unsigned char *dir, unsigned char a) +{ + if (a<128 || a==255) return a>='A' && a<='Z' ? a + 0x20 : a; + if (!dir) return a; + return dir[a]; +} + +int hpfs_chk_name(const unsigned char *name, unsigned *len) +{ + int i; + if (*len > 254) return -ENAMETOOLONG; + hpfs_adjust_length(name, len); + if (!*len) return -EINVAL; + for (i = 0; i < *len; i++) if (not_allowed_char(name[i])) return -EINVAL; + if (*len == 1) if (name[0] == '.') return -EINVAL; + if (*len == 2) if (name[0] == '.' && name[1] == '.') return -EINVAL; + return 0; +} + +unsigned char *hpfs_translate_name(struct super_block *s, unsigned char *from, + unsigned len, int lc, int lng) +{ + unsigned char *to; + int i; + if (hpfs_sb(s)->sb_chk >= 2) if (hpfs_is_name_long(from, len) != lng) { + pr_err("Long name flag mismatch - name "); + for (i = 0; i < len; i++) + pr_cont("%c", from[i]); + pr_cont(" misidentified as %s.\n", lng ? "short" : "long"); + pr_err("It's nothing serious. It could happen because of bug in OS/2.\nSet checks=normal to disable this message.\n"); + } + if (!lc) return from; + if (!(to = kmalloc(len, GFP_KERNEL))) { + pr_err("can't allocate memory for name conversion buffer\n"); + return from; + } + for (i = 0; i < len; i++) to[i] = locase(hpfs_sb(s)->sb_cp_table,from[i]); + return to; +} + +int hpfs_compare_names(struct super_block *s, + const unsigned char *n1, unsigned l1, + const unsigned char *n2, unsigned l2, int last) +{ + unsigned l = l1 < l2 ? l1 : l2; + unsigned i; + if (last) return -1; + for (i = 0; i < l; i++) { + unsigned char c1 = upcase(hpfs_sb(s)->sb_cp_table,n1[i]); + unsigned char c2 = upcase(hpfs_sb(s)->sb_cp_table,n2[i]); + if (c1 < c2) return -1; + if (c1 > c2) return 1; + } + if (l1 < l2) return -1; + if (l1 > l2) return 1; + return 0; +} + +int hpfs_is_name_long(const unsigned char *name, unsigned len) +{ + int i,j; + for (i = 0; i < len && name[i] != '.'; i++) + if (no_dos_char(name[i])) return 1; + if (!i || i > 8) return 1; + if (i == len) return 0; + for (j = i + 1; j < len; j++) + if (name[j] == '.' || no_dos_char(name[i])) return 1; + return j - i > 4; +} + +/* OS/2 clears dots and spaces at the end of file name, so we have to */ + +void hpfs_adjust_length(const unsigned char *name, unsigned *len) +{ + if (!*len) return; + if (*len == 1 && name[0] == '.') return; + if (*len == 2 && name[0] == '.' && name[1] == '.') return; + while (*len && (name[*len - 1] == '.' || name[*len - 1] == ' ')) + (*len)--; +} diff --git a/fs/hpfs/namei.c b/fs/hpfs/namei.c new file mode 100644 index 000000000..9446f4038 --- /dev/null +++ b/fs/hpfs/namei.c @@ -0,0 +1,621 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/hpfs/namei.c + * + * Mikulas Patocka (mikulas@artax.karlin.mff.cuni.cz), 1998-1999 + * + * adding & removing files & directories + */ +#include +#include "hpfs_fn.h" + +static void hpfs_update_directory_times(struct inode *dir) +{ + time64_t t = local_to_gmt(dir->i_sb, local_get_seconds(dir->i_sb)); + if (t == inode_get_mtime_sec(dir) && + t == inode_get_ctime_sec(dir)) + return; + inode_set_mtime_to_ts(dir, inode_set_ctime(dir, t, 0)); + hpfs_write_inode_nolock(dir); +} + +static struct dentry *hpfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + const unsigned char *name = dentry->d_name.name; + unsigned len = dentry->d_name.len; + struct quad_buffer_head qbh0; + struct buffer_head *bh; + struct hpfs_dirent *de; + struct fnode *fnode; + struct dnode *dnode; + struct inode *result; + fnode_secno fno; + dnode_secno dno; + int r; + struct hpfs_dirent dee; + int err; + if ((err = hpfs_chk_name(name, &len))) return ERR_PTR(err==-ENOENT ? -EINVAL : err); + hpfs_lock(dir->i_sb); + err = -ENOSPC; + fnode = hpfs_alloc_fnode(dir->i_sb, hpfs_i(dir)->i_dno, &fno, &bh); + if (!fnode) + goto bail; + dnode = hpfs_alloc_dnode(dir->i_sb, fno, &dno, &qbh0); + if (!dnode) + goto bail1; + memset(&dee, 0, sizeof dee); + dee.directory = 1; + if (!(mode & 0222)) dee.read_only = 1; + /*dee.archive = 0;*/ + dee.hidden = name[0] == '.'; + dee.fnode = cpu_to_le32(fno); + dee.creation_date = dee.write_date = dee.read_date = cpu_to_le32(local_get_seconds(dir->i_sb)); + result = new_inode(dir->i_sb); + if (!result) { + err = -ENOMEM; + goto bail2; + } + hpfs_init_inode(result); + result->i_ino = fno; + hpfs_i(result)->i_parent_dir = dir->i_ino; + hpfs_i(result)->i_dno = dno; + inode_set_mtime_to_ts(result, + inode_set_atime_to_ts(result, inode_set_ctime(result, local_to_gmt(dir->i_sb, le32_to_cpu(dee.creation_date)), 0))); + hpfs_i(result)->i_ea_size = 0; + result->i_mode |= S_IFDIR; + result->i_op = &hpfs_dir_iops; + result->i_fop = &hpfs_dir_ops; + result->i_blocks = 4; + result->i_size = 2048; + set_nlink(result, 2); + if (dee.read_only) + result->i_mode &= ~0222; + + r = hpfs_add_dirent(dir, name, len, &dee); + if (r == 1) + goto bail3; + if (r == -1) { + err = -EEXIST; + goto bail3; + } + fnode->len = len; + memcpy(fnode->name, name, len > 15 ? 15 : len); + fnode->up = cpu_to_le32(dir->i_ino); + fnode->flags |= FNODE_dir; + fnode->btree.n_free_nodes = 7; + fnode->btree.n_used_nodes = 1; + fnode->btree.first_free = cpu_to_le16(0x14); + fnode->u.external[0].disk_secno = cpu_to_le32(dno); + fnode->u.external[0].file_secno = cpu_to_le32(-1); + dnode->root_dnode = 1; + dnode->up = cpu_to_le32(fno); + de = hpfs_add_de(dir->i_sb, dnode, "\001\001", 2, 0); + de->creation_date = de->write_date = de->read_date = cpu_to_le32(local_get_seconds(dir->i_sb)); + if (!(mode & 0222)) de->read_only = 1; + de->first = de->directory = 1; + /*de->hidden = de->system = 0;*/ + de->fnode = cpu_to_le32(fno); + mark_buffer_dirty(bh); + brelse(bh); + hpfs_mark_4buffers_dirty(&qbh0); + hpfs_brelse4(&qbh0); + inc_nlink(dir); + insert_inode_hash(result); + + if (!uid_eq(result->i_uid, current_fsuid()) || + !gid_eq(result->i_gid, current_fsgid()) || + result->i_mode != mode) { + result->i_uid = current_fsuid(); + result->i_gid = current_fsgid(); + result->i_mode = mode; + hpfs_write_inode_nolock(result); + } + hpfs_update_directory_times(dir); + d_instantiate(dentry, result); + hpfs_unlock(dir->i_sb); + return NULL; +bail3: + iput(result); +bail2: + hpfs_brelse4(&qbh0); + hpfs_free_dnode(dir->i_sb, dno); +bail1: + brelse(bh); + hpfs_free_sectors(dir->i_sb, fno, 1); +bail: + hpfs_unlock(dir->i_sb); + return ERR_PTR(err); +} + +static int hpfs_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + const unsigned char *name = dentry->d_name.name; + unsigned len = dentry->d_name.len; + struct inode *result = NULL; + struct buffer_head *bh; + struct fnode *fnode; + fnode_secno fno; + int r; + struct hpfs_dirent dee; + int err; + if ((err = hpfs_chk_name(name, &len))) + return err==-ENOENT ? -EINVAL : err; + hpfs_lock(dir->i_sb); + err = -ENOSPC; + fnode = hpfs_alloc_fnode(dir->i_sb, hpfs_i(dir)->i_dno, &fno, &bh); + if (!fnode) + goto bail; + memset(&dee, 0, sizeof dee); + if (!(mode & 0222)) dee.read_only = 1; + dee.archive = 1; + dee.hidden = name[0] == '.'; + dee.fnode = cpu_to_le32(fno); + dee.creation_date = dee.write_date = dee.read_date = cpu_to_le32(local_get_seconds(dir->i_sb)); + + result = new_inode(dir->i_sb); + if (!result) { + err = -ENOMEM; + goto bail1; + } + hpfs_init_inode(result); + result->i_ino = fno; + result->i_mode |= S_IFREG; + result->i_mode &= ~0111; + result->i_op = &hpfs_file_iops; + result->i_fop = &hpfs_file_ops; + set_nlink(result, 1); + hpfs_i(result)->i_parent_dir = dir->i_ino; + inode_set_mtime_to_ts(result, + inode_set_atime_to_ts(result, inode_set_ctime(result, local_to_gmt(dir->i_sb, le32_to_cpu(dee.creation_date)), 0))); + hpfs_i(result)->i_ea_size = 0; + if (dee.read_only) + result->i_mode &= ~0222; + result->i_blocks = 1; + result->i_size = 0; + result->i_data.a_ops = &hpfs_aops; + hpfs_i(result)->mmu_private = 0; + + r = hpfs_add_dirent(dir, name, len, &dee); + if (r == 1) + goto bail2; + if (r == -1) { + err = -EEXIST; + goto bail2; + } + fnode->len = len; + memcpy(fnode->name, name, len > 15 ? 15 : len); + fnode->up = cpu_to_le32(dir->i_ino); + mark_buffer_dirty(bh); + brelse(bh); + + insert_inode_hash(result); + + if (!uid_eq(result->i_uid, current_fsuid()) || + !gid_eq(result->i_gid, current_fsgid()) || + result->i_mode != (mode | S_IFREG)) { + result->i_uid = current_fsuid(); + result->i_gid = current_fsgid(); + result->i_mode = mode | S_IFREG; + hpfs_write_inode_nolock(result); + } + hpfs_update_directory_times(dir); + d_instantiate(dentry, result); + hpfs_unlock(dir->i_sb); + return 0; + +bail2: + iput(result); +bail1: + brelse(bh); + hpfs_free_sectors(dir->i_sb, fno, 1); +bail: + hpfs_unlock(dir->i_sb); + return err; +} + +static int hpfs_mknod(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) +{ + const unsigned char *name = dentry->d_name.name; + unsigned len = dentry->d_name.len; + struct buffer_head *bh; + struct fnode *fnode; + fnode_secno fno; + int r; + struct hpfs_dirent dee; + struct inode *result = NULL; + int err; + if ((err = hpfs_chk_name(name, &len))) return err==-ENOENT ? -EINVAL : err; + if (hpfs_sb(dir->i_sb)->sb_eas < 2) return -EPERM; + hpfs_lock(dir->i_sb); + err = -ENOSPC; + fnode = hpfs_alloc_fnode(dir->i_sb, hpfs_i(dir)->i_dno, &fno, &bh); + if (!fnode) + goto bail; + memset(&dee, 0, sizeof dee); + if (!(mode & 0222)) dee.read_only = 1; + dee.archive = 1; + dee.hidden = name[0] == '.'; + dee.fnode = cpu_to_le32(fno); + dee.creation_date = dee.write_date = dee.read_date = cpu_to_le32(local_get_seconds(dir->i_sb)); + + result = new_inode(dir->i_sb); + if (!result) { + err = -ENOMEM; + goto bail1; + } + hpfs_init_inode(result); + result->i_ino = fno; + hpfs_i(result)->i_parent_dir = dir->i_ino; + inode_set_mtime_to_ts(result, + inode_set_atime_to_ts(result, inode_set_ctime(result, local_to_gmt(dir->i_sb, le32_to_cpu(dee.creation_date)), 0))); + hpfs_i(result)->i_ea_size = 0; + result->i_uid = current_fsuid(); + result->i_gid = current_fsgid(); + set_nlink(result, 1); + result->i_size = 0; + result->i_blocks = 1; + init_special_inode(result, mode, rdev); + + r = hpfs_add_dirent(dir, name, len, &dee); + if (r == 1) + goto bail2; + if (r == -1) { + err = -EEXIST; + goto bail2; + } + fnode->len = len; + memcpy(fnode->name, name, len > 15 ? 15 : len); + fnode->up = cpu_to_le32(dir->i_ino); + mark_buffer_dirty(bh); + + insert_inode_hash(result); + + hpfs_write_inode_nolock(result); + hpfs_update_directory_times(dir); + d_instantiate(dentry, result); + brelse(bh); + hpfs_unlock(dir->i_sb); + return 0; +bail2: + iput(result); +bail1: + brelse(bh); + hpfs_free_sectors(dir->i_sb, fno, 1); +bail: + hpfs_unlock(dir->i_sb); + return err; +} + +static int hpfs_symlink(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const char *symlink) +{ + const unsigned char *name = dentry->d_name.name; + unsigned len = dentry->d_name.len; + struct buffer_head *bh; + struct fnode *fnode; + fnode_secno fno; + int r; + struct hpfs_dirent dee; + struct inode *result; + int err; + if ((err = hpfs_chk_name(name, &len))) return err==-ENOENT ? -EINVAL : err; + hpfs_lock(dir->i_sb); + if (hpfs_sb(dir->i_sb)->sb_eas < 2) { + hpfs_unlock(dir->i_sb); + return -EPERM; + } + err = -ENOSPC; + fnode = hpfs_alloc_fnode(dir->i_sb, hpfs_i(dir)->i_dno, &fno, &bh); + if (!fnode) + goto bail; + memset(&dee, 0, sizeof dee); + dee.archive = 1; + dee.hidden = name[0] == '.'; + dee.fnode = cpu_to_le32(fno); + dee.creation_date = dee.write_date = dee.read_date = cpu_to_le32(local_get_seconds(dir->i_sb)); + + result = new_inode(dir->i_sb); + if (!result) { + err = -ENOMEM; + goto bail1; + } + result->i_ino = fno; + hpfs_init_inode(result); + hpfs_i(result)->i_parent_dir = dir->i_ino; + inode_set_mtime_to_ts(result, + inode_set_atime_to_ts(result, inode_set_ctime(result, local_to_gmt(dir->i_sb, le32_to_cpu(dee.creation_date)), 0))); + hpfs_i(result)->i_ea_size = 0; + result->i_mode = S_IFLNK | 0777; + result->i_uid = current_fsuid(); + result->i_gid = current_fsgid(); + result->i_blocks = 1; + set_nlink(result, 1); + result->i_size = strlen(symlink); + inode_nohighmem(result); + result->i_op = &page_symlink_inode_operations; + result->i_data.a_ops = &hpfs_symlink_aops; + + r = hpfs_add_dirent(dir, name, len, &dee); + if (r == 1) + goto bail2; + if (r == -1) { + err = -EEXIST; + goto bail2; + } + fnode->len = len; + memcpy(fnode->name, name, len > 15 ? 15 : len); + fnode->up = cpu_to_le32(dir->i_ino); + hpfs_set_ea(result, fnode, "SYMLINK", symlink, strlen(symlink)); + mark_buffer_dirty(bh); + brelse(bh); + + insert_inode_hash(result); + + hpfs_write_inode_nolock(result); + hpfs_update_directory_times(dir); + d_instantiate(dentry, result); + hpfs_unlock(dir->i_sb); + return 0; +bail2: + iput(result); +bail1: + brelse(bh); + hpfs_free_sectors(dir->i_sb, fno, 1); +bail: + hpfs_unlock(dir->i_sb); + return err; +} + +static int hpfs_unlink(struct inode *dir, struct dentry *dentry) +{ + const unsigned char *name = dentry->d_name.name; + unsigned len = dentry->d_name.len; + struct quad_buffer_head qbh; + struct hpfs_dirent *de; + struct inode *inode = d_inode(dentry); + dnode_secno dno; + int r; + int err; + + hpfs_lock(dir->i_sb); + hpfs_adjust_length(name, &len); + + err = -ENOENT; + de = map_dirent(dir, hpfs_i(dir)->i_dno, name, len, &dno, &qbh); + if (!de) + goto out; + + err = -EPERM; + if (de->first) + goto out1; + + err = -EISDIR; + if (de->directory) + goto out1; + + r = hpfs_remove_dirent(dir, dno, de, &qbh, 1); + switch (r) { + case 1: + hpfs_error(dir->i_sb, "there was error when removing dirent"); + err = -EFSERROR; + break; + case 2: /* no space for deleting */ + err = -ENOSPC; + break; + default: + drop_nlink(inode); + err = 0; + } + goto out; + +out1: + hpfs_brelse4(&qbh); +out: + if (!err) + hpfs_update_directory_times(dir); + hpfs_unlock(dir->i_sb); + return err; +} + +static int hpfs_rmdir(struct inode *dir, struct dentry *dentry) +{ + const unsigned char *name = dentry->d_name.name; + unsigned len = dentry->d_name.len; + struct quad_buffer_head qbh; + struct hpfs_dirent *de; + struct inode *inode = d_inode(dentry); + dnode_secno dno; + int n_items = 0; + int err; + int r; + + hpfs_adjust_length(name, &len); + hpfs_lock(dir->i_sb); + err = -ENOENT; + de = map_dirent(dir, hpfs_i(dir)->i_dno, name, len, &dno, &qbh); + if (!de) + goto out; + + err = -EPERM; + if (de->first) + goto out1; + + err = -ENOTDIR; + if (!de->directory) + goto out1; + + hpfs_count_dnodes(dir->i_sb, hpfs_i(inode)->i_dno, NULL, NULL, &n_items); + err = -ENOTEMPTY; + if (n_items) + goto out1; + + r = hpfs_remove_dirent(dir, dno, de, &qbh, 1); + switch (r) { + case 1: + hpfs_error(dir->i_sb, "there was error when removing dirent"); + err = -EFSERROR; + break; + case 2: + err = -ENOSPC; + break; + default: + drop_nlink(dir); + clear_nlink(inode); + err = 0; + } + goto out; +out1: + hpfs_brelse4(&qbh); +out: + if (!err) + hpfs_update_directory_times(dir); + hpfs_unlock(dir->i_sb); + return err; +} + +static int hpfs_symlink_read_folio(struct file *file, struct folio *folio) +{ + char *link = folio_address(folio); + struct inode *i = folio->mapping->host; + struct fnode *fnode; + struct buffer_head *bh; + int err; + + err = -EIO; + hpfs_lock(i->i_sb); + if (!(fnode = hpfs_map_fnode(i->i_sb, i->i_ino, &bh))) + goto fail; + err = hpfs_read_ea(i->i_sb, fnode, "SYMLINK", link, PAGE_SIZE); + brelse(bh); +fail: + hpfs_unlock(i->i_sb); + folio_end_read(folio, err == 0); + return err; +} + +const struct address_space_operations hpfs_symlink_aops = { + .read_folio = hpfs_symlink_read_folio +}; + +static int hpfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) +{ + const unsigned char *old_name = old_dentry->d_name.name; + unsigned old_len = old_dentry->d_name.len; + const unsigned char *new_name = new_dentry->d_name.name; + unsigned new_len = new_dentry->d_name.len; + struct inode *i = d_inode(old_dentry); + struct inode *new_inode = d_inode(new_dentry); + struct quad_buffer_head qbh, qbh1; + struct hpfs_dirent *dep, *nde; + struct hpfs_dirent de; + dnode_secno dno; + int r; + struct buffer_head *bh; + struct fnode *fnode; + int err; + + if (flags & ~RENAME_NOREPLACE) + return -EINVAL; + + if ((err = hpfs_chk_name(new_name, &new_len))) return err; + err = 0; + hpfs_adjust_length(old_name, &old_len); + + hpfs_lock(i->i_sb); + /* order doesn't matter, due to VFS exclusion */ + + /* Erm? Moving over the empty non-busy directory is perfectly legal */ + if (new_inode && S_ISDIR(new_inode->i_mode)) { + err = -EINVAL; + goto end1; + } + + if (!(dep = map_dirent(old_dir, hpfs_i(old_dir)->i_dno, old_name, old_len, &dno, &qbh))) { + hpfs_error(i->i_sb, "lookup succeeded but map dirent failed"); + err = -ENOENT; + goto end1; + } + copy_de(&de, dep); + de.hidden = new_name[0] == '.'; + + if (new_inode) { + int r; + if ((r = hpfs_remove_dirent(old_dir, dno, dep, &qbh, 1)) != 2) { + if ((nde = map_dirent(new_dir, hpfs_i(new_dir)->i_dno, new_name, new_len, NULL, &qbh1))) { + clear_nlink(new_inode); + copy_de(nde, &de); + memcpy(nde->name, new_name, new_len); + hpfs_mark_4buffers_dirty(&qbh1); + hpfs_brelse4(&qbh1); + goto end; + } + hpfs_error(new_dir->i_sb, "hpfs_rename: could not find dirent"); + err = -EFSERROR; + goto end1; + } + err = -ENOSPC; + goto end1; + } + + if (new_dir == old_dir) hpfs_brelse4(&qbh); + + if ((r = hpfs_add_dirent(new_dir, new_name, new_len, &de))) { + if (r == -1) hpfs_error(new_dir->i_sb, "hpfs_rename: dirent already exists!"); + err = r == 1 ? -ENOSPC : -EFSERROR; + if (new_dir != old_dir) hpfs_brelse4(&qbh); + goto end1; + } + + if (new_dir == old_dir) + if (!(dep = map_dirent(old_dir, hpfs_i(old_dir)->i_dno, old_name, old_len, &dno, &qbh))) { + hpfs_error(i->i_sb, "lookup succeeded but map dirent failed at #2"); + err = -ENOENT; + goto end1; + } + + if ((r = hpfs_remove_dirent(old_dir, dno, dep, &qbh, 0))) { + hpfs_error(i->i_sb, "hpfs_rename: could not remove dirent"); + err = r == 2 ? -ENOSPC : -EFSERROR; + goto end1; + } + +end: + hpfs_i(i)->i_parent_dir = new_dir->i_ino; + if (S_ISDIR(i->i_mode)) { + inc_nlink(new_dir); + drop_nlink(old_dir); + } + if ((fnode = hpfs_map_fnode(i->i_sb, i->i_ino, &bh))) { + fnode->up = cpu_to_le32(new_dir->i_ino); + fnode->len = new_len; + memcpy(fnode->name, new_name, new_len>15?15:new_len); + if (new_len < 15) memset(&fnode->name[new_len], 0, 15 - new_len); + mark_buffer_dirty(bh); + brelse(bh); + } +end1: + if (!err) { + hpfs_update_directory_times(old_dir); + hpfs_update_directory_times(new_dir); + } + hpfs_unlock(i->i_sb); + return err; +} + +const struct inode_operations hpfs_dir_iops = +{ + .create = hpfs_create, + .lookup = hpfs_lookup, + .unlink = hpfs_unlink, + .symlink = hpfs_symlink, + .mkdir = hpfs_mkdir, + .rmdir = hpfs_rmdir, + .mknod = hpfs_mknod, + .rename = hpfs_rename, + .setattr = hpfs_setattr, +}; diff --git a/fs/hpfs/super.c b/fs/hpfs/super.c new file mode 100644 index 000000000..8fbdbf080 --- /dev/null +++ b/fs/hpfs/super.c @@ -0,0 +1,789 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/hpfs/super.c + * + * Mikulas Patocka (mikulas@artax.karlin.mff.cuni.cz), 1998-1999 + * + * mounting, unmounting, error handling + */ + +#include "hpfs_fn.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Mark the filesystem dirty, so that chkdsk checks it when os/2 booted */ + +static void mark_dirty(struct super_block *s, int remount) +{ + if (hpfs_sb(s)->sb_chkdsk && (remount || !sb_rdonly(s))) { + struct buffer_head *bh; + struct hpfs_spare_block *sb; + if ((sb = hpfs_map_sector(s, 17, &bh, 0))) { + sb->dirty = 1; + sb->old_wrote = 0; + mark_buffer_dirty(bh); + sync_dirty_buffer(bh); + brelse(bh); + } + } +} + +/* Mark the filesystem clean (mark it dirty for chkdsk if chkdsk==2 or if there + were errors) */ + +static void unmark_dirty(struct super_block *s) +{ + struct buffer_head *bh; + struct hpfs_spare_block *sb; + if (sb_rdonly(s)) return; + sync_blockdev(s->s_bdev); + if ((sb = hpfs_map_sector(s, 17, &bh, 0))) { + sb->dirty = hpfs_sb(s)->sb_chkdsk > 1 - hpfs_sb(s)->sb_was_error; + sb->old_wrote = hpfs_sb(s)->sb_chkdsk >= 2 && !hpfs_sb(s)->sb_was_error; + mark_buffer_dirty(bh); + sync_dirty_buffer(bh); + brelse(bh); + } +} + +/* Filesystem error... */ +void hpfs_error(struct super_block *s, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + pr_err("filesystem error: %pV", &vaf); + + va_end(args); + + if (!hpfs_sb(s)->sb_was_error) { + if (hpfs_sb(s)->sb_err == 2) { + pr_cont("; crashing the system because you wanted it\n"); + mark_dirty(s, 0); + panic("HPFS panic"); + } else if (hpfs_sb(s)->sb_err == 1) { + if (sb_rdonly(s)) + pr_cont("; already mounted read-only\n"); + else { + pr_cont("; remounting read-only\n"); + mark_dirty(s, 0); + s->s_flags |= SB_RDONLY; + } + } else if (sb_rdonly(s)) + pr_cont("; going on - but anything won't be destroyed because it's read-only\n"); + else + pr_cont("; corrupted filesystem mounted read/write - your computer will explode within 20 seconds ... but you wanted it so!\n"); + } else + pr_cont("\n"); + hpfs_sb(s)->sb_was_error = 1; +} + +/* + * A little trick to detect cycles in many hpfs structures and don't let the + * kernel crash on corrupted filesystem. When first called, set c2 to 0. + * + * BTW. chkdsk doesn't detect cycles correctly. When I had 2 lost directories + * nested each in other, chkdsk locked up happilly. + */ + +int hpfs_stop_cycles(struct super_block *s, int key, int *c1, int *c2, + char *msg) +{ + if (*c2 && *c1 == key) { + hpfs_error(s, "cycle detected on key %08x in %s", key, msg); + return 1; + } + (*c2)++; + if (!((*c2 - 1) & *c2)) *c1 = key; + return 0; +} + +static void free_sbi(struct hpfs_sb_info *sbi) +{ + kfree(sbi->sb_cp_table); + kfree(sbi->sb_bmp_dir); + kfree(sbi); +} + +static void lazy_free_sbi(struct rcu_head *rcu) +{ + free_sbi(container_of(rcu, struct hpfs_sb_info, rcu)); +} + +static void hpfs_put_super(struct super_block *s) +{ + hpfs_lock(s); + unmark_dirty(s); + hpfs_unlock(s); + call_rcu(&hpfs_sb(s)->rcu, lazy_free_sbi); +} + +static unsigned hpfs_count_one_bitmap(struct super_block *s, secno secno) +{ + struct quad_buffer_head qbh; + unsigned long *bits; + unsigned count; + + bits = hpfs_map_4sectors(s, secno, &qbh, 0); + if (!bits) + return (unsigned)-1; + count = bitmap_weight(bits, 2048 * BITS_PER_BYTE); + hpfs_brelse4(&qbh); + return count; +} + +static unsigned count_bitmaps(struct super_block *s) +{ + unsigned n, count, n_bands; + n_bands = (hpfs_sb(s)->sb_fs_size + 0x3fff) >> 14; + count = 0; + for (n = 0; n < COUNT_RD_AHEAD; n++) { + hpfs_prefetch_bitmap(s, n); + } + for (n = 0; n < n_bands; n++) { + unsigned c; + hpfs_prefetch_bitmap(s, n + COUNT_RD_AHEAD); + c = hpfs_count_one_bitmap(s, le32_to_cpu(hpfs_sb(s)->sb_bmp_dir[n])); + if (c != (unsigned)-1) + count += c; + } + return count; +} + +unsigned hpfs_get_free_dnodes(struct super_block *s) +{ + struct hpfs_sb_info *sbi = hpfs_sb(s); + if (sbi->sb_n_free_dnodes == (unsigned)-1) { + unsigned c = hpfs_count_one_bitmap(s, sbi->sb_dmap); + if (c == (unsigned)-1) + return 0; + sbi->sb_n_free_dnodes = c; + } + return sbi->sb_n_free_dnodes; +} + +static int hpfs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *s = dentry->d_sb; + struct hpfs_sb_info *sbi = hpfs_sb(s); + u64 id = huge_encode_dev(s->s_bdev->bd_dev); + + hpfs_lock(s); + + if (sbi->sb_n_free == (unsigned)-1) + sbi->sb_n_free = count_bitmaps(s); + + buf->f_type = s->s_magic; + buf->f_bsize = 512; + buf->f_blocks = sbi->sb_fs_size; + buf->f_bfree = sbi->sb_n_free; + buf->f_bavail = sbi->sb_n_free; + buf->f_files = sbi->sb_dirband_size / 4; + buf->f_ffree = hpfs_get_free_dnodes(s); + buf->f_fsid = u64_to_fsid(id); + buf->f_namelen = 254; + + hpfs_unlock(s); + + return 0; +} + + +long hpfs_ioctl(struct file *file, unsigned cmd, unsigned long arg) +{ + switch (cmd) { + case FITRIM: { + struct fstrim_range range; + secno n_trimmed; + int r; + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (copy_from_user(&range, (struct fstrim_range __user *)arg, sizeof(range))) + return -EFAULT; + r = hpfs_trim_fs(file_inode(file)->i_sb, range.start >> 9, (range.start + range.len) >> 9, (range.minlen + 511) >> 9, &n_trimmed); + if (r) + return r; + range.len = (u64)n_trimmed << 9; + if (copy_to_user((struct fstrim_range __user *)arg, &range, sizeof(range))) + return -EFAULT; + return 0; + } + default: { + return -ENOIOCTLCMD; + } + } +} + + +static struct kmem_cache * hpfs_inode_cachep; + +static struct inode *hpfs_alloc_inode(struct super_block *sb) +{ + struct hpfs_inode_info *ei; + ei = alloc_inode_sb(sb, hpfs_inode_cachep, GFP_NOFS); + if (!ei) + return NULL; + return &ei->vfs_inode; +} + +static void hpfs_free_inode(struct inode *inode) +{ + kmem_cache_free(hpfs_inode_cachep, hpfs_i(inode)); +} + +static void init_once(void *foo) +{ + struct hpfs_inode_info *ei = (struct hpfs_inode_info *) foo; + + inode_init_once(&ei->vfs_inode); +} + +static int init_inodecache(void) +{ + hpfs_inode_cachep = kmem_cache_create("hpfs_inode_cache", + sizeof(struct hpfs_inode_info), + 0, (SLAB_RECLAIM_ACCOUNT| + SLAB_ACCOUNT), + init_once); + if (hpfs_inode_cachep == NULL) + return -ENOMEM; + return 0; +} + +static void destroy_inodecache(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(hpfs_inode_cachep); +} + +enum { + Opt_help, Opt_uid, Opt_gid, Opt_umask, Opt_case, + Opt_check, Opt_err, Opt_eas, Opt_chkdsk, Opt_timeshift, +}; + +static const struct constant_table hpfs_param_case[] = { + {"asis", 0}, + {"lower", 1}, + {} +}; + +static const struct constant_table hpfs_param_check[] = { + {"none", 0}, + {"normal", 1}, + {"strict", 2}, + {} +}; + +static const struct constant_table hpfs_param_err[] = { + {"continue", 0}, + {"remount-ro", 1}, + {"panic", 2}, + {} +}; + +static const struct constant_table hpfs_param_eas[] = { + {"no", 0}, + {"ro", 1}, + {"rw", 2}, + {} +}; + +static const struct constant_table hpfs_param_chkdsk[] = { + {"no", 0}, + {"errors", 1}, + {"always", 2}, + {} +}; + +static const struct fs_parameter_spec hpfs_param_spec[] = { + fsparam_flag ("help", Opt_help), + fsparam_uid ("uid", Opt_uid), + fsparam_gid ("gid", Opt_gid), + fsparam_u32oct ("umask", Opt_umask), + fsparam_enum ("case", Opt_case, hpfs_param_case), + fsparam_enum ("check", Opt_check, hpfs_param_check), + fsparam_enum ("errors", Opt_err, hpfs_param_err), + fsparam_enum ("eas", Opt_eas, hpfs_param_eas), + fsparam_enum ("chkdsk", Opt_chkdsk, hpfs_param_chkdsk), + fsparam_s32 ("timeshift", Opt_timeshift), + {} +}; + +struct hpfs_fc_context { + kuid_t uid; + kgid_t gid; + umode_t umask; + int lowercase; + int eas; + int chk; + int errs; + int chkdsk; + int timeshift; +}; + +static inline void hpfs_help(void) +{ + pr_info("\n\ +HPFS filesystem options:\n\ + help do not mount and display this text\n\ + uid=xxx set uid of files that don't have uid specified in eas\n\ + gid=xxx set gid of files that don't have gid specified in eas\n\ + umask=xxx set mode of files that don't have mode specified in eas\n\ + case=lower lowercase all files\n\ + case=asis do not lowercase files (default)\n\ + check=none no fs checks - kernel may crash on corrupted filesystem\n\ + check=normal do some checks - it should not crash (default)\n\ + check=strict do extra time-consuming checks, used for debugging\n\ + errors=continue continue on errors\n\ + errors=remount-ro remount read-only if errors found (default)\n\ + errors=panic panic on errors\n\ + chkdsk=no do not mark fs for chkdsking even if there were errors\n\ + chkdsk=errors mark fs dirty if errors found (default)\n\ + chkdsk=always always mark fs dirty - used for debugging\n\ + eas=no ignore extended attributes\n\ + eas=ro read but do not write extended attributes\n\ + eas=rw r/w eas => enables chmod, chown, mknod, ln -s (default)\n\ + timeshift=nnn add nnn seconds to file times\n\ +\n"); +} + +static int hpfs_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct hpfs_fc_context *ctx = fc->fs_private; + struct fs_parse_result result; + int opt; + + opt = fs_parse(fc, hpfs_param_spec, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_help: + hpfs_help(); + return -EINVAL; + case Opt_uid: + ctx->uid = result.uid; + break; + case Opt_gid: + ctx->gid = result.gid; + break; + case Opt_umask: + ctx->umask = result.uint_32; + break; + case Opt_case: + ctx->lowercase = result.uint_32; + break; + case Opt_check: + ctx->chk = result.uint_32; + break; + case Opt_err: + ctx->errs = result.uint_32; + break; + case Opt_eas: + ctx->eas = result.uint_32; + break; + case Opt_chkdsk: + ctx->chkdsk = result.uint_32; + break; + case Opt_timeshift: + { + char *rhs = param->string; + int timeshift; + + if (kstrtoint(rhs, 0, ×hift)) + return -EINVAL; + ctx->timeshift = timeshift; + break; + } + default: + return -EINVAL; + } + + return 0; +} + +static int hpfs_reconfigure(struct fs_context *fc) +{ + struct hpfs_fc_context *ctx = fc->fs_private; + struct super_block *s = fc->root->d_sb; + struct hpfs_sb_info *sbi = hpfs_sb(s); + + sync_filesystem(s); + + fc->sb_flags |= SB_NOATIME; + + hpfs_lock(s); + + if (ctx->timeshift != sbi->sb_timeshift) { + pr_err("timeshift can't be changed using remount.\n"); + goto out_err; + } + + unmark_dirty(s); + + sbi->sb_uid = ctx->uid; sbi->sb_gid = ctx->gid; + sbi->sb_mode = 0777 & ~ctx->umask; + sbi->sb_lowercase = ctx->lowercase; + sbi->sb_eas = ctx->eas; sbi->sb_chk = ctx->chk; + sbi->sb_chkdsk = ctx->chkdsk; + sbi->sb_err = ctx->errs; sbi->sb_timeshift = ctx->timeshift; + + if (!(fc->sb_flags & SB_RDONLY)) mark_dirty(s, 1); + + hpfs_unlock(s); + return 0; + +out_err: + hpfs_unlock(s); + return -EINVAL; +} + +static int hpfs_show_options(struct seq_file *seq, struct dentry *root) +{ + struct hpfs_sb_info *sbi = hpfs_sb(root->d_sb); + + seq_printf(seq, ",uid=%u", from_kuid_munged(&init_user_ns, sbi->sb_uid)); + seq_printf(seq, ",gid=%u", from_kgid_munged(&init_user_ns, sbi->sb_gid)); + seq_printf(seq, ",umask=%03o", (~sbi->sb_mode & 0777)); + if (sbi->sb_lowercase) + seq_printf(seq, ",case=lower"); + if (!sbi->sb_chk) + seq_printf(seq, ",check=none"); + if (sbi->sb_chk == 2) + seq_printf(seq, ",check=strict"); + if (!sbi->sb_err) + seq_printf(seq, ",errors=continue"); + if (sbi->sb_err == 2) + seq_printf(seq, ",errors=panic"); + if (!sbi->sb_chkdsk) + seq_printf(seq, ",chkdsk=no"); + if (sbi->sb_chkdsk == 2) + seq_printf(seq, ",chkdsk=always"); + if (!sbi->sb_eas) + seq_printf(seq, ",eas=no"); + if (sbi->sb_eas == 1) + seq_printf(seq, ",eas=ro"); + if (sbi->sb_timeshift) + seq_printf(seq, ",timeshift=%d", sbi->sb_timeshift); + return 0; +} + +/* Super operations */ + +static const struct super_operations hpfs_sops = +{ + .alloc_inode = hpfs_alloc_inode, + .free_inode = hpfs_free_inode, + .evict_inode = hpfs_evict_inode, + .put_super = hpfs_put_super, + .statfs = hpfs_statfs, + .show_options = hpfs_show_options, +}; + +static int hpfs_fill_super(struct super_block *s, struct fs_context *fc) +{ + struct hpfs_fc_context *ctx = fc->fs_private; + struct buffer_head *bh0, *bh1, *bh2; + struct hpfs_boot_block *bootblock; + struct hpfs_super_block *superblock; + struct hpfs_spare_block *spareblock; + struct hpfs_sb_info *sbi; + struct inode *root; + int silent = fc->sb_flags & SB_SILENT; + + dnode_secno root_dno; + struct hpfs_dirent *de = NULL; + struct quad_buffer_head qbh; + + sbi = kzalloc_obj(*sbi); + if (!sbi) { + return -ENOMEM; + } + s->s_fs_info = sbi; + + mutex_init(&sbi->hpfs_mutex); + hpfs_lock(s); + + /*sbi->sb_mounting = 1;*/ + if (!sb_set_blocksize(s, 512)) + goto bail0; + sbi->sb_fs_size = -1; + if (!(bootblock = hpfs_map_sector(s, 0, &bh0, 0))) goto bail1; + if (!(superblock = hpfs_map_sector(s, 16, &bh1, 1))) goto bail2; + if (!(spareblock = hpfs_map_sector(s, 17, &bh2, 0))) goto bail3; + + /* Check magics */ + if (/*le16_to_cpu(bootblock->magic) != BB_MAGIC + ||*/ le32_to_cpu(superblock->magic) != SB_MAGIC + || le32_to_cpu(spareblock->magic) != SP_MAGIC) { + if (!silent) + pr_err("Bad magic ... probably not HPFS\n"); + goto bail4; + } + + /* Check version */ + if (!sb_rdonly(s) && superblock->funcversion != 2 && superblock->funcversion != 3) { + pr_err("Bad version %d,%d. Mount readonly to go around\n", + (int)superblock->version, (int)superblock->funcversion); + pr_err("please try recent version of HPFS driver at http://artax.karlin.mff.cuni.cz/~mikulas/vyplody/hpfs/index-e.cgi and if it still can't understand this format, contact author - mikulas@artax.karlin.mff.cuni.cz\n"); + goto bail4; + } + + s->s_flags |= SB_NOATIME; + + /* Fill superblock stuff */ + s->s_magic = HPFS_SUPER_MAGIC; + s->s_op = &hpfs_sops; + set_default_d_op(s, &hpfs_dentry_operations); + s->s_time_min = local_to_gmt(s, 0); + s->s_time_max = local_to_gmt(s, U32_MAX); + + sbi->sb_root = le32_to_cpu(superblock->root); + sbi->sb_fs_size = le32_to_cpu(superblock->n_sectors); + sbi->sb_bitmaps = le32_to_cpu(superblock->bitmaps); + sbi->sb_dirband_start = le32_to_cpu(superblock->dir_band_start); + sbi->sb_dirband_size = le32_to_cpu(superblock->n_dir_band); + sbi->sb_dmap = le32_to_cpu(superblock->dir_band_bitmap); + sbi->sb_uid = ctx->uid; + sbi->sb_gid = ctx->gid; + sbi->sb_mode = 0777 & ~ctx->umask; + sbi->sb_n_free = -1; + sbi->sb_n_free_dnodes = -1; + sbi->sb_lowercase = ctx->lowercase; + sbi->sb_eas = ctx->eas; + sbi->sb_chk = ctx->chk; + sbi->sb_chkdsk = ctx->chkdsk; + sbi->sb_err = ctx->errs; + sbi->sb_timeshift = ctx->timeshift; + sbi->sb_was_error = 0; + sbi->sb_cp_table = NULL; + sbi->sb_c_bitmap = -1; + sbi->sb_max_fwd_alloc = 0xffffff; + + if (sbi->sb_fs_size >= 0x80000000) { + hpfs_error(s, "invalid size in superblock: %08x", + (unsigned)sbi->sb_fs_size); + goto bail4; + } + + if (spareblock->n_spares_used) + hpfs_load_hotfix_map(s, spareblock); + + /* Load bitmap directory */ + if (!(sbi->sb_bmp_dir = hpfs_load_bitmap_directory(s, le32_to_cpu(superblock->bitmaps)))) + goto bail4; + + /* Check for general fs errors*/ + if (spareblock->dirty && !spareblock->old_wrote) { + if (sbi->sb_err == 2) { + pr_err("Improperly stopped, not mounted\n"); + goto bail4; + } + hpfs_error(s, "improperly stopped"); + } + + if (!sb_rdonly(s)) { + spareblock->dirty = 1; + spareblock->old_wrote = 0; + mark_buffer_dirty(bh2); + } + + if (le32_to_cpu(spareblock->n_dnode_spares) != le32_to_cpu(spareblock->n_dnode_spares_free)) { + if (sbi->sb_err >= 2) { + pr_err("Spare dnodes used, try chkdsk\n"); + mark_dirty(s, 0); + goto bail4; + } + hpfs_error(s, "warning: spare dnodes used, try chkdsk"); + if (sbi->sb_err == 0) + pr_err("Proceeding, but your filesystem could be corrupted if you delete files or directories\n"); + } + if (sbi->sb_chk) { + unsigned a; + if (le32_to_cpu(superblock->dir_band_end) - le32_to_cpu(superblock->dir_band_start) + 1 != le32_to_cpu(superblock->n_dir_band) || + le32_to_cpu(superblock->dir_band_end) < le32_to_cpu(superblock->dir_band_start) || le32_to_cpu(superblock->n_dir_band) > 0x4000) { + hpfs_error(s, "dir band size mismatch: dir_band_start==%08x, dir_band_end==%08x, n_dir_band==%08x", + le32_to_cpu(superblock->dir_band_start), le32_to_cpu(superblock->dir_band_end), le32_to_cpu(superblock->n_dir_band)); + goto bail4; + } + a = sbi->sb_dirband_size; + sbi->sb_dirband_size = 0; + if (hpfs_chk_sectors(s, le32_to_cpu(superblock->dir_band_start), le32_to_cpu(superblock->n_dir_band), "dir_band") || + hpfs_chk_sectors(s, le32_to_cpu(superblock->dir_band_bitmap), 4, "dir_band_bitmap") || + hpfs_chk_sectors(s, le32_to_cpu(superblock->bitmaps), 4, "bitmaps")) { + mark_dirty(s, 0); + goto bail4; + } + sbi->sb_dirband_size = a; + } else + pr_err("You really don't want any checks? You are crazy...\n"); + + /* Load code page table */ + if (le32_to_cpu(spareblock->n_code_pages)) + if (!(sbi->sb_cp_table = hpfs_load_code_page(s, le32_to_cpu(spareblock->code_page_dir)))) + pr_err("code page support is disabled\n"); + + brelse(bh2); + brelse(bh1); + brelse(bh0); + + root = iget_locked(s, sbi->sb_root); + if (!root) + goto bail0; + hpfs_init_inode(root); + hpfs_read_inode(root); + unlock_new_inode(root); + s->s_root = d_make_root(root); + if (!s->s_root) + goto bail0; + + /* + * find the root directory's . pointer & finish filling in the inode + */ + + root_dno = hpfs_fnode_dno(s, sbi->sb_root); + if (root_dno) + de = map_dirent(root, root_dno, "\001\001", 2, NULL, &qbh); + if (!de) + hpfs_error(s, "unable to find root dir"); + else { + inode_set_atime(root, + local_to_gmt(s, le32_to_cpu(de->read_date)), + 0); + inode_set_mtime(root, + local_to_gmt(s, le32_to_cpu(de->write_date)), + 0); + inode_set_ctime(root, + local_to_gmt(s, le32_to_cpu(de->creation_date)), + 0); + hpfs_i(root)->i_ea_size = le32_to_cpu(de->ea_size); + hpfs_i(root)->i_parent_dir = root->i_ino; + if (root->i_size == -1) + root->i_size = 2048; + if (root->i_blocks == -1) + root->i_blocks = 5; + hpfs_brelse4(&qbh); + } + hpfs_unlock(s); + return 0; + +bail4: brelse(bh2); +bail3: brelse(bh1); +bail2: brelse(bh0); +bail1: +bail0: + hpfs_unlock(s); + free_sbi(sbi); + return -EINVAL; +} + +static int hpfs_get_tree(struct fs_context *fc) +{ + return get_tree_bdev(fc, hpfs_fill_super); +} + +static void hpfs_free_fc(struct fs_context *fc) +{ + kfree(fc->fs_private); +} + +static const struct fs_context_operations hpfs_fc_context_ops = { + .parse_param = hpfs_parse_param, + .get_tree = hpfs_get_tree, + .reconfigure = hpfs_reconfigure, + .free = hpfs_free_fc, +}; + +static int hpfs_init_fs_context(struct fs_context *fc) +{ + struct hpfs_fc_context *ctx; + + ctx = kzalloc_obj(struct hpfs_fc_context); + if (!ctx) + return -ENOMEM; + + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE) { + struct super_block *sb = fc->root->d_sb; + struct hpfs_sb_info *sbi = hpfs_sb(sb); + + ctx->uid = sbi->sb_uid; + ctx->gid = sbi->sb_gid; + ctx->umask = 0777 & ~sbi->sb_mode; + ctx->lowercase = sbi->sb_lowercase; + ctx->eas = sbi->sb_eas; + ctx->chk = sbi->sb_chk; + ctx->chkdsk = sbi->sb_chkdsk; + ctx->errs = sbi->sb_err; + ctx->timeshift = sbi->sb_timeshift; + + } else { + ctx->uid = current_uid(); + ctx->gid = current_gid(); + ctx->umask = current_umask(); + ctx->lowercase = 0; + ctx->eas = 2; + ctx->chk = 1; + ctx->errs = 1; + ctx->chkdsk = 1; + ctx->timeshift = 0; + } + + fc->fs_private = ctx; + fc->ops = &hpfs_fc_context_ops; + + return 0; +}; + +static struct file_system_type hpfs_fs_type = { + .owner = THIS_MODULE, + .name = "hpfs", + .kill_sb = kill_block_super, + .fs_flags = FS_REQUIRES_DEV, + .init_fs_context = hpfs_init_fs_context, + .parameters = hpfs_param_spec, +}; +MODULE_ALIAS_FS("hpfs"); + +static int __init init_hpfs_fs(void) +{ + int err = init_inodecache(); + if (err) + goto out1; + err = register_filesystem(&hpfs_fs_type); + if (err) + goto out; + return 0; +out: + destroy_inodecache(); +out1: + return err; +} + +static void __exit exit_hpfs_fs(void) +{ + unregister_filesystem(&hpfs_fs_type); + destroy_inodecache(); +} + +module_init(init_hpfs_fs) +module_exit(exit_hpfs_fs) +MODULE_DESCRIPTION("OS/2 HPFS file system support"); +MODULE_LICENSE("GPL"); diff --git a/fs/hugetlbfs/Makefile b/fs/hugetlbfs/Makefile new file mode 100644 index 000000000..d876ecfbe --- /dev/null +++ b/fs/hugetlbfs/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the linux ramfs routines. +# + +obj-$(CONFIG_HUGETLBFS) += hugetlbfs.o + +hugetlbfs-objs := inode.o diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c new file mode 100644 index 000000000..7611a8470 --- /dev/null +++ b/fs/hugetlbfs/inode.c @@ -0,0 +1,1628 @@ +/* + * hugetlbpage-backed filesystem. Based on ramfs. + * + * Nadia Yvette Chambers, 2002 + * + * Copyright (C) 2002 Linus Torvalds. + * License: GPL + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#define CREATE_TRACE_POINTS +#include + +static const struct address_space_operations hugetlbfs_aops; +static const struct file_operations hugetlbfs_file_operations; +static const struct inode_operations hugetlbfs_dir_inode_operations; +static const struct inode_operations hugetlbfs_inode_operations; + +enum hugetlbfs_size_type { NO_SIZE, SIZE_STD, SIZE_PERCENT }; + +struct hugetlbfs_fs_context { + struct hstate *hstate; + unsigned long long max_size_opt; + unsigned long long min_size_opt; + long max_hpages; + long nr_inodes; + long min_hpages; + enum hugetlbfs_size_type max_val_type; + enum hugetlbfs_size_type min_val_type; + kuid_t uid; + kgid_t gid; + umode_t mode; +}; + +int sysctl_hugetlb_shm_group; + +enum hugetlb_param { + Opt_gid, + Opt_min_size, + Opt_mode, + Opt_nr_inodes, + Opt_pagesize, + Opt_size, + Opt_uid, +}; + +static const struct fs_parameter_spec hugetlb_fs_parameters[] = { + fsparam_gid ("gid", Opt_gid), + fsparam_string("min_size", Opt_min_size), + fsparam_u32oct("mode", Opt_mode), + fsparam_string("nr_inodes", Opt_nr_inodes), + fsparam_string("pagesize", Opt_pagesize), + fsparam_string("size", Opt_size), + fsparam_uid ("uid", Opt_uid), + {} +}; + +/* + * Mask used when checking the page offset value passed in via system + * calls. This value will be converted to a loff_t which is signed. + * Therefore, we want to check the upper PAGE_SHIFT + 1 bits of the + * value. The extra bit (- 1 in the shift value) is to take the sign + * bit into account. + */ +#define PGOFF_LOFFT_MAX \ + (((1UL << (PAGE_SHIFT + 1)) - 1) << (BITS_PER_LONG - (PAGE_SHIFT + 1))) + +static int hugetlbfs_file_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct inode *inode = file_inode(file); + loff_t len, vma_len; + int ret; + struct hstate *h = hstate_file(file); + vma_flags_t vma_flags; + + /* + * vma address alignment (but not the pgoff alignment) has + * already been checked by prepare_hugepage_range. If you add + * any error returns here, do so after setting VM_HUGETLB, so + * is_vm_hugetlb_page tests below unmap_region go the right + * way when do_mmap unwinds (may be important on powerpc + * and ia64). + */ + vma_set_flags(vma, VMA_HUGETLB_BIT, VMA_DONTEXPAND_BIT); + vma->vm_ops = &hugetlb_vm_ops; + + /* + * page based offset in vm_pgoff could be sufficiently large to + * overflow a loff_t when converted to byte offset. This can + * only happen on architectures where sizeof(loff_t) == + * sizeof(unsigned long). So, only check in those instances. + */ + if (sizeof(unsigned long) == sizeof(loff_t)) { + if (vma->vm_pgoff & PGOFF_LOFFT_MAX) + return -EINVAL; + } + + /* must be huge page aligned */ + if (vma->vm_pgoff & (~huge_page_mask(h) >> PAGE_SHIFT)) + return -EINVAL; + + vma_len = (loff_t)(vma->vm_end - vma->vm_start); + len = vma_len + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); + /* check for overflow */ + if (len < vma_len) + return -EINVAL; + + inode_lock(inode); + file_accessed(file); + + ret = -ENOMEM; + + vma_flags = vma->flags; + /* + * for SHM_HUGETLB, the pages are reserved in the shmget() call so skip + * reserving here. Note: only for SHM hugetlbfs file, the inode + * flag S_PRIVATE is set. + */ + if (inode->i_flags & S_PRIVATE) + vma_flags_set(&vma_flags, VMA_NORESERVE_BIT); + + if (hugetlb_reserve_pages(inode, + vma->vm_pgoff >> huge_page_order(h), + len >> huge_page_shift(h), vma, + vma_flags) < 0) + goto out; + + ret = 0; + if (vma_test(vma, VMA_WRITE_BIT) && inode->i_size < len) + i_size_write(inode, len); +out: + inode_unlock(inode); + + return ret; +} + +/* + * Called under mmap_write_lock(mm). + */ + +unsigned long +hugetlb_get_unmapped_area(struct file *file, unsigned long addr, + unsigned long len, unsigned long pgoff, + unsigned long flags) +{ + unsigned long addr0 = 0; + struct hstate *h = hstate_file(file); + + if (len & ~huge_page_mask(h)) + return -EINVAL; + if ((flags & MAP_FIXED) && (addr & ~huge_page_mask(h))) + return -EINVAL; + if (addr) + addr0 = ALIGN(addr, huge_page_size(h)); + + return mm_get_unmapped_area_vmaflags(file, addr0, len, pgoff, flags, + EMPTY_VMA_FLAGS); +} + +/* + * Someone wants to read @bytes from a HWPOISON hugetlb @folio from @offset. + * Returns the maximum number of bytes one can read without touching the 1st raw + * HWPOISON page. + */ +static size_t adjust_range_hwpoison(struct folio *folio, size_t offset, + size_t bytes) +{ + struct page *page = folio_page(folio, offset / PAGE_SIZE); + size_t safe_bytes; + + if (is_raw_hwpoison_page_in_hugepage(page)) + return 0; + /* Safe to read the remaining bytes in this page. */ + safe_bytes = PAGE_SIZE - (offset % PAGE_SIZE); + page++; + + /* Check each remaining page as long as we are not done yet. */ + for (; safe_bytes < bytes; safe_bytes += PAGE_SIZE, page++) + if (is_raw_hwpoison_page_in_hugepage(page)) + break; + + return min(safe_bytes, bytes); +} + +/* + * Support for read() - Find the page attached to f_mapping and copy out the + * data. This provides functionality similar to filemap_read(). + */ +static ssize_t hugetlbfs_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct file *file = iocb->ki_filp; + struct hstate *h = hstate_file(file); + struct address_space *mapping = file->f_mapping; + struct inode *inode = mapping->host; + unsigned long index = iocb->ki_pos >> huge_page_shift(h); + unsigned long offset = iocb->ki_pos & ~huge_page_mask(h); + unsigned long end_index; + loff_t isize; + ssize_t retval = 0; + + while (iov_iter_count(to)) { + struct folio *folio; + size_t nr, copied, want; + + /* nr is the maximum number of bytes to copy from this page */ + nr = huge_page_size(h); + isize = i_size_read(inode); + if (!isize) + break; + end_index = (isize - 1) >> huge_page_shift(h); + if (index > end_index) + break; + if (index == end_index) { + nr = ((isize - 1) & ~huge_page_mask(h)) + 1; + if (nr <= offset) + break; + } + nr = nr - offset; + + /* Find the folio */ + folio = filemap_lock_hugetlb_folio(h, mapping, index); + if (IS_ERR(folio)) { + /* + * We have a HOLE, zero out the user-buffer for the + * length of the hole or request. + */ + copied = iov_iter_zero(nr, to); + } else { + folio_unlock(folio); + + if (!folio_test_hwpoison(folio)) + want = nr; + else { + /* + * Adjust how many bytes safe to read without + * touching the 1st raw HWPOISON page after + * offset. + */ + want = adjust_range_hwpoison(folio, offset, nr); + if (want == 0) { + folio_put(folio); + retval = -EIO; + break; + } + } + + /* + * We have the folio, copy it to user space buffer. + */ + copied = copy_folio_to_iter(folio, offset, want, to); + folio_put(folio); + } + offset += copied; + retval += copied; + if (copied != nr && iov_iter_count(to)) { + if (!retval) + retval = -EFAULT; + break; + } + index += offset >> huge_page_shift(h); + offset &= ~huge_page_mask(h); + } + iocb->ki_pos = ((loff_t)index << huge_page_shift(h)) + offset; + return retval; +} + +static int hugetlbfs_write_begin(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, + struct folio **foliop, void **fsdata) +{ + return -EINVAL; +} + +static int hugetlbfs_write_end(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct folio *folio, void *fsdata) +{ + BUG(); + return -EINVAL; +} + +static void hugetlb_delete_from_page_cache(struct folio *folio) +{ + folio_clear_dirty(folio); + folio_clear_uptodate(folio); + filemap_remove_folio(folio); +} + +/* + * Called with i_mmap_rwsem held for inode based vma maps. This makes + * sure vma (and vm_mm) will not go away. We also hold the hugetlb fault + * mutex for the page in the mapping. So, we can not race with page being + * faulted into the vma. + */ +static bool hugetlb_vma_maps_pfn(struct vm_area_struct *vma, + unsigned long addr, unsigned long pfn) +{ + pte_t *ptep, pte; + + ptep = hugetlb_walk(vma, addr, huge_page_size(hstate_vma(vma))); + if (!ptep) + return false; + + pte = huge_ptep_get(vma->vm_mm, addr, ptep); + if (huge_pte_none(pte) || !pte_present(pte)) + return false; + + if (pte_pfn(pte) == pfn) + return true; + + return false; +} + +/* + * Can vma_offset_start/vma_offset_end overflow on 32-bit arches? + * No, because the interval tree returns us only those vmas + * which overlap the truncated area starting at pgoff, + * and no vma on a 32-bit arch can span beyond the 4GB. + */ +static unsigned long vma_offset_start(struct vm_area_struct *vma, pgoff_t start) +{ + unsigned long offset = 0; + + if (vma->vm_pgoff < start) + offset = (start - vma->vm_pgoff) << PAGE_SHIFT; + + return vma->vm_start + offset; +} + +static unsigned long vma_offset_end(struct vm_area_struct *vma, pgoff_t end) +{ + unsigned long t_end; + + if (!end) + return vma->vm_end; + + t_end = ((end - vma->vm_pgoff) << PAGE_SHIFT) + vma->vm_start; + if (t_end > vma->vm_end) + t_end = vma->vm_end; + return t_end; +} + +/* + * Called with hugetlb fault mutex held. Therefore, no more mappings to + * this folio can be created while executing the routine. + */ +static void hugetlb_unmap_file_folio(struct hstate *h, + struct address_space *mapping, + struct folio *folio, pgoff_t index) +{ + struct hugetlb_vma_lock *vma_lock; + unsigned long pfn = folio_pfn(folio); + struct vm_area_struct *vma; + unsigned long v_start; + unsigned long v_end; + pgoff_t start, end; + + start = index * pages_per_huge_page(h); + end = (index + 1) * pages_per_huge_page(h); + + i_mmap_lock_write(mapping); +retry: + vma_lock = NULL; + mapping_rmap_tree_foreach(vma, mapping, start, end - 1) { + v_start = vma_offset_start(vma, start); + v_end = vma_offset_end(vma, end); + + if (!hugetlb_vma_maps_pfn(vma, v_start, pfn)) + continue; + + if (!hugetlb_vma_trylock_write(vma)) { + vma_lock = vma->vm_private_data; + /* + * If we can not get vma lock, we need to drop + * immap_sema and take locks in order. First, + * take a ref on the vma_lock structure so that + * we can be guaranteed it will not go away when + * dropping immap_sema. + */ + kref_get(&vma_lock->refs); + break; + } + + unmap_hugepage_range(vma, v_start, v_end, NULL, + ZAP_FLAG_DROP_MARKER); + hugetlb_vma_unlock_write(vma); + } + + i_mmap_unlock_write(mapping); + + if (vma_lock) { + /* + * Wait on vma_lock. We know it is still valid as we have + * a reference. We must 'open code' vma locking as we do + * not know if vma_lock is still attached to vma. + */ + down_write(&vma_lock->rw_sema); + i_mmap_lock_write(mapping); + + vma = vma_lock->vma; + if (!vma) { + /* + * If lock is no longer attached to vma, then just + * unlock, drop our reference and retry looking for + * other vmas. + */ + up_write(&vma_lock->rw_sema); + kref_put(&vma_lock->refs, hugetlb_vma_lock_release); + goto retry; + } + + /* + * vma_lock is still attached to vma. Check to see if vma + * still maps page and if so, unmap. + */ + v_start = vma_offset_start(vma, start); + v_end = vma_offset_end(vma, end); + if (hugetlb_vma_maps_pfn(vma, v_start, pfn)) + unmap_hugepage_range(vma, v_start, v_end, NULL, + ZAP_FLAG_DROP_MARKER); + + kref_put(&vma_lock->refs, hugetlb_vma_lock_release); + hugetlb_vma_unlock_write(vma); + + goto retry; + } +} + +static void +hugetlb_vmdelete_list(struct address_space *mapping, pgoff_t start, + pgoff_t end, zap_flags_t zap_flags) +{ + struct vm_area_struct *vma; + + /* + * end == 0 indicates that the entire range after start should be + * unmapped. Note, end is exclusive, whereas the interval tree takes + * an inclusive "last". + */ + mapping_rmap_tree_foreach(vma, mapping, start, + end ? end - 1 : ULONG_MAX) { + unsigned long v_start; + unsigned long v_end; + + if (!hugetlb_vma_trylock_write(vma)) + continue; + + v_start = vma_offset_start(vma, start); + v_end = vma_offset_end(vma, end); + + unmap_hugepage_range(vma, v_start, v_end, NULL, zap_flags); + + /* + * Note that vma lock only exists for shared/non-private + * vmas. Therefore, lock is not held when calling + * unmap_hugepage_range for private vmas. + */ + hugetlb_vma_unlock_write(vma); + } +} + +/* + * Called with hugetlb fault mutex held. + */ +static void remove_inode_single_folio(struct hstate *h, struct inode *inode, + struct address_space *mapping, struct folio *folio, + pgoff_t index, bool truncate_op) +{ + /* + * If folio is mapped, it was faulted in after being + * unmapped in caller or hugetlb_vmdelete_list() skips + * unmapping it due to fail to grab lock. Unmap (again) + * while holding the fault mutex. The mutex will prevent + * faults until we finish removing the folio. Hold folio + * lock to guarantee no concurrent migration. + */ + folio_lock(folio); + if (unlikely(folio_mapped(folio))) + hugetlb_unmap_file_folio(h, mapping, folio, index); + + /* + * We must remove the folio from page cache before removing + * the region/ reserve map (hugetlb_unreserve_pages). In + * rare out of memory conditions, removal of the region/reserve + * map could fail. Correspondingly, the subpool and global + * reserve usage count can need to be adjusted. + */ + VM_BUG_ON_FOLIO(folio_test_hugetlb_restore_reserve(folio), folio); + hugetlb_delete_from_page_cache(folio); + if (!truncate_op) { + if (unlikely(hugetlb_unreserve_pages(inode, index, + index + 1, 1))) + hugetlb_fix_reserve_counts(inode); + } + + folio_unlock(folio); +} + +/* + * remove_inode_hugepages handles two distinct cases: truncation and hole + * punch. There are subtle differences in operation for each case. + * + * truncation is indicated by end of range being LLONG_MAX + * In this case, we first scan the range and release found pages. + * After releasing pages, hugetlb_unreserve_pages cleans up region/reserve + * maps and global counts. Page faults can race with truncation. + * During faults, hugetlb_no_page() checks i_size before page allocation, + * and again after obtaining page table lock. It will 'back out' + * allocations in the truncated range. + * hole punch is indicated if end is not LLONG_MAX + * In the hole punch case we scan the range and release found pages. + * Only when releasing a page is the associated region/reserve map + * deleted. The region/reserve map for ranges without associated + * pages are not modified. Page faults can race with hole punch. + * This is indicated if we find a mapped page. + * Note: If the passed end of range value is beyond the end of file, but + * not LLONG_MAX this routine still performs a hole punch operation. + */ +static void remove_inode_hugepages(struct inode *inode, loff_t lstart, + loff_t lend) +{ + struct hstate *h = hstate_inode(inode); + struct address_space *mapping = &inode->i_data; + const pgoff_t end = lend >> PAGE_SHIFT; + struct folio_batch fbatch; + pgoff_t next, index; + int i, freed = 0; + bool truncate_op = (lend == LLONG_MAX); + + folio_batch_init(&fbatch); + next = lstart >> PAGE_SHIFT; + while (filemap_get_folios(mapping, &next, end - 1, &fbatch)) { + for (i = 0; i < folio_batch_count(&fbatch); ++i) { + struct folio *folio = fbatch.folios[i]; + u32 hash = 0; + + index = folio->index >> huge_page_order(h); + hash = hugetlb_fault_mutex_hash(mapping, index); + mutex_lock(&hugetlb_fault_mutex_table[hash]); + + /* + * Remove folio that was part of folio_batch. + */ + remove_inode_single_folio(h, inode, mapping, folio, + index, truncate_op); + freed++; + + mutex_unlock(&hugetlb_fault_mutex_table[hash]); + } + folio_batch_release(&fbatch); + cond_resched(); + } + + if (truncate_op) + (void)hugetlb_unreserve_pages(inode, + lstart >> huge_page_shift(h), + LONG_MAX, freed); +} + +static void hugetlbfs_evict_inode(struct inode *inode) +{ + struct resv_map *resv_map; + + trace_hugetlbfs_evict_inode(inode); + remove_inode_hugepages(inode, 0, LLONG_MAX); + + resv_map = HUGETLBFS_I(inode)->resv_map; + /* Only regular and link inodes have associated reserve maps */ + if (resv_map) + resv_map_release(&resv_map->refs); + clear_inode(inode); +} + +static void hugetlb_vmtruncate(struct inode *inode, loff_t offset) +{ + pgoff_t pgoff; + struct address_space *mapping = inode->i_mapping; + struct hstate *h = hstate_inode(inode); + + BUG_ON(offset & ~huge_page_mask(h)); + pgoff = offset >> PAGE_SHIFT; + + i_size_write(inode, offset); + i_mmap_lock_write(mapping); + if (mapping_mapped(mapping)) + hugetlb_vmdelete_list(mapping, pgoff, 0, ZAP_FLAG_DROP_MARKER); + i_mmap_unlock_write(mapping); + remove_inode_hugepages(inode, offset, LLONG_MAX); +} + +static void hugetlbfs_zero_partial_page(struct hstate *h, + struct address_space *mapping, + loff_t start, + loff_t end) +{ + pgoff_t idx = start >> huge_page_shift(h); + struct folio *folio; + + folio = filemap_lock_hugetlb_folio(h, mapping, idx); + if (IS_ERR(folio)) + return; + + start = start & ~huge_page_mask(h); + end = end & ~huge_page_mask(h); + if (!end) + end = huge_page_size(h); + + folio_zero_segment(folio, (size_t)start, (size_t)end); + + folio_unlock(folio); + folio_put(folio); +} + +static long hugetlbfs_punch_hole(struct inode *inode, loff_t offset, loff_t len) +{ + struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode); + struct address_space *mapping = inode->i_mapping; + struct hstate *h = hstate_inode(inode); + loff_t hpage_size = huge_page_size(h); + loff_t hole_start, hole_end; + + /* + * hole_start and hole_end indicate the full pages within the hole. + */ + hole_start = round_up(offset, hpage_size); + hole_end = round_down(offset + len, hpage_size); + + inode_lock(inode); + + /* protected by i_rwsem */ + if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) { + inode_unlock(inode); + return -EPERM; + } + + i_mmap_lock_write(mapping); + + /* If range starts before first full page, zero partial page. */ + if (offset < hole_start) + hugetlbfs_zero_partial_page(h, mapping, + offset, min(offset + len, hole_start)); + + /* Unmap users of full pages in the hole. */ + if (hole_end > hole_start) { + if (mapping_mapped(mapping)) + hugetlb_vmdelete_list(mapping, + hole_start >> PAGE_SHIFT, + hole_end >> PAGE_SHIFT, 0); + } + + /* If range extends beyond last full page, zero partial page. */ + if ((offset + len) > hole_end && (offset + len) > hole_start) + hugetlbfs_zero_partial_page(h, mapping, + hole_end, offset + len); + + i_mmap_unlock_write(mapping); + + /* Remove full pages from the file. */ + if (hole_end > hole_start) + remove_inode_hugepages(inode, hole_start, hole_end); + + inode_unlock(inode); + + return 0; +} + +static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset, + loff_t len) +{ + struct inode *inode = file_inode(file); + struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode); + struct address_space *mapping = inode->i_mapping; + struct hstate *h = hstate_inode(inode); + struct vm_area_struct pseudo_vma; + struct mm_struct *mm = current->mm; + loff_t hpage_size = huge_page_size(h); + unsigned long hpage_shift = huge_page_shift(h); + pgoff_t start, index, end; + int error; + u32 hash; + + if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)) + return -EOPNOTSUPP; + + if (mode & FALLOC_FL_PUNCH_HOLE) { + error = hugetlbfs_punch_hole(inode, offset, len); + goto out_nolock; + } + + /* + * Default preallocate case. + * For this range, start is rounded down and end is rounded up + * as well as being converted to page offsets. + */ + start = offset >> hpage_shift; + end = (offset + len + hpage_size - 1) >> hpage_shift; + + inode_lock(inode); + + /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */ + error = inode_newsize_ok(inode, offset + len); + if (error) + goto out; + + if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) { + error = -EPERM; + goto out; + } + + /* + * Initialize a pseudo vma as this is required by the huge page + * allocation routines. + */ + vma_init(&pseudo_vma, mm); + vm_flags_init(&pseudo_vma, VM_HUGETLB | VM_MAYSHARE | VM_SHARED); + pseudo_vma.vm_file = file; + + for (index = start; index < end; index++) { + /* + * This is supposed to be the vaddr where the page is being + * faulted in, but we have no vaddr here. + */ + struct folio *folio; + unsigned long addr; + + cond_resched(); + + /* + * fallocate(2) manpage permits EINTR; we may have been + * interrupted because we are using up too much memory. + */ + if (signal_pending(current)) { + error = -EINTR; + break; + } + + /* addr is the offset within the file (zero based) */ + addr = index * hpage_size; + + /* mutex taken here, fault path and hole punch */ + hash = hugetlb_fault_mutex_hash(mapping, index); + mutex_lock(&hugetlb_fault_mutex_table[hash]); + + /* See if already present in mapping to avoid alloc/free */ + folio = filemap_get_folio(mapping, index << huge_page_order(h)); + if (!IS_ERR(folio)) { + folio_put(folio); + mutex_unlock(&hugetlb_fault_mutex_table[hash]); + continue; + } + + /* + * Allocate folio without setting the avoid_reserve argument. + * There certainly are no reserves associated with the + * pseudo_vma. However, there could be shared mappings with + * reserves for the file at the inode level. If we fallocate + * folios in these areas, we need to consume the reserves + * to keep reservation accounting consistent. + */ + folio = alloc_hugetlb_folio(&pseudo_vma, addr, false); + if (IS_ERR(folio)) { + mutex_unlock(&hugetlb_fault_mutex_table[hash]); + error = PTR_ERR(folio); + goto out; + } + folio_zero_user(folio, addr); + __folio_mark_uptodate(folio); + error = hugetlb_add_to_page_cache(folio, mapping, index); + if (unlikely(error)) { + restore_reserve_on_error(h, &pseudo_vma, addr, folio); + folio_put(folio); + mutex_unlock(&hugetlb_fault_mutex_table[hash]); + goto out; + } + + mutex_unlock(&hugetlb_fault_mutex_table[hash]); + + folio_set_hugetlb_migratable(folio); + /* + * folio_unlock because locked by hugetlb_add_to_page_cache() + * folio_put() due to reference from alloc_hugetlb_folio() + */ + folio_unlock(folio); + folio_put(folio); + } + + if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size) + i_size_write(inode, offset + len); + inode_set_ctime_current(inode); +out: + inode_unlock(inode); + +out_nolock: + trace_hugetlbfs_fallocate(inode, mode, offset, len, error); + return error; +} + +static int hugetlbfs_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, struct iattr *attr) +{ + struct inode *inode = d_inode(dentry); + struct hstate *h = hstate_inode(inode); + int error; + unsigned int ia_valid = attr->ia_valid; + struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode); + + error = setattr_prepare(idmap, dentry, attr); + if (error) + return error; + + trace_hugetlbfs_setattr(inode, dentry, attr); + + if (ia_valid & ATTR_SIZE) { + loff_t oldsize = inode->i_size; + loff_t newsize = attr->ia_size; + + if (newsize & ~huge_page_mask(h)) + return -EINVAL; + /* protected by i_rwsem */ + if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) || + (newsize > oldsize && (info->seals & F_SEAL_GROW))) + return -EPERM; + hugetlb_vmtruncate(inode, newsize); + } + + setattr_copy(idmap, inode, attr); + mark_inode_dirty(inode); + return 0; +} + +static struct inode *hugetlbfs_get_root(struct super_block *sb, + struct hugetlbfs_fs_context *ctx) +{ + struct inode *inode; + + inode = new_inode(sb); + if (inode) { + inode->i_ino = get_next_ino(); + inode->i_mode = S_IFDIR | ctx->mode; + inode->i_uid = ctx->uid; + inode->i_gid = ctx->gid; + simple_inode_init_ts(inode); + inode->i_op = &hugetlbfs_dir_inode_operations; + inode->i_fop = &simple_dir_operations; + HUGETLBFS_I(inode)->resv_map = NULL; + /* directory inodes start off with i_nlink == 2 (for "." entry) */ + inc_nlink(inode); + lockdep_annotate_inode_mutex_key(inode); + } + return inode; +} + +/* + * Hugetlbfs is not reclaimable; therefore its i_mmap_rwsem will never + * be taken from reclaim -- unlike regular filesystems. This needs an + * annotation because huge_pmd_share() does an allocation under hugetlb's + * i_mmap_rwsem. + */ +static struct lock_class_key hugetlbfs_i_mmap_rwsem_key; + +static struct inode *hugetlbfs_get_inode(struct super_block *sb, + struct mnt_idmap *idmap, + struct inode *dir, + umode_t mode, dev_t dev) +{ + struct inode *inode; + struct resv_map *resv_map = NULL; + + /* + * Reserve maps are only needed for inodes that can have associated + * page allocations. + */ + if (S_ISREG(mode) || S_ISLNK(mode)) { + resv_map = resv_map_alloc(); + if (!resv_map) + return NULL; + } + + inode = new_inode(sb); + if (inode) { + struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode); + + inode->i_ino = get_next_ino(); + inode_init_owner(idmap, inode, dir, mode); + lockdep_set_class(&inode->i_mapping->i_mmap_rwsem, + &hugetlbfs_i_mmap_rwsem_key); + inode->i_mapping->a_ops = &hugetlbfs_aops; + simple_inode_init_ts(inode); + info->resv_map = resv_map; + info->seals = F_SEAL_SEAL; + switch (mode & S_IFMT) { + default: + init_special_inode(inode, mode, dev); + break; + case S_IFREG: + inode->i_op = &hugetlbfs_inode_operations; + inode->i_fop = &hugetlbfs_file_operations; + break; + case S_IFDIR: + inode->i_op = &hugetlbfs_dir_inode_operations; + inode->i_fop = &simple_dir_operations; + + /* directory inodes start off with i_nlink == 2 (for "." entry) */ + inc_nlink(inode); + break; + case S_IFLNK: + inode->i_op = &page_symlink_inode_operations; + inode_nohighmem(inode); + break; + } + lockdep_annotate_inode_mutex_key(inode); + trace_hugetlbfs_alloc_inode(inode, dir, mode); + } else { + if (resv_map) + kref_put(&resv_map->refs, resv_map_release); + } + + return inode; +} + +/* + * File creation. Allocate an inode, and we're done.. + */ +static int hugetlbfs_mknod(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t dev) +{ + struct inode *inode; + + inode = hugetlbfs_get_inode(dir->i_sb, idmap, dir, mode, dev); + if (!inode) + return -ENOSPC; + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + d_make_persistent(dentry, inode); + return 0; +} + +static struct dentry *hugetlbfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + int retval = hugetlbfs_mknod(idmap, dir, dentry, + mode, 0); + if (!retval) + inc_nlink(dir); + return ERR_PTR(retval); +} + +static int hugetlbfs_create(struct mnt_idmap *idmap, + struct inode *dir, struct dentry *dentry, + umode_t mode) +{ + return hugetlbfs_mknod(idmap, dir, dentry, mode | S_IFREG, 0); +} + +static int hugetlbfs_tmpfile(struct mnt_idmap *idmap, + struct inode *dir, struct file *file, + umode_t mode) +{ + struct inode *inode; + + inode = hugetlbfs_get_inode(dir->i_sb, idmap, dir, mode | S_IFREG, 0); + if (!inode) + return -ENOSPC; + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + d_tmpfile(file, inode); + return finish_open_simple(file, 0); +} + +static int hugetlbfs_symlink(struct mnt_idmap *idmap, + struct inode *dir, struct dentry *dentry, + const char *symname) +{ + const umode_t mode = S_IFLNK|S_IRWXUGO; + struct inode *inode; + int error = -ENOSPC; + + inode = hugetlbfs_get_inode(dir->i_sb, idmap, dir, mode, 0); + if (inode) { + int l = strlen(symname)+1; + error = page_symlink(inode, symname, l); + if (!error) + d_make_persistent(dentry, inode); + else + iput(inode); + } + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + + return error; +} + +#ifdef CONFIG_MIGRATION +static int hugetlbfs_migrate_folio(struct address_space *mapping, + struct folio *dst, struct folio *src, + enum migrate_mode mode) +{ + int rc; + + rc = migrate_huge_page_move_mapping(mapping, dst, src); + if (rc) + return rc; + + if (hugetlb_folio_subpool(src)) { + hugetlb_set_folio_subpool(dst, + hugetlb_folio_subpool(src)); + hugetlb_set_folio_subpool(src, NULL); + } + + folio_migrate_flags(dst, src); + + return 0; +} +#else +#define hugetlbfs_migrate_folio NULL +#endif + +static int hugetlbfs_error_remove_folio(struct address_space *mapping, + struct folio *folio) +{ + return 0; +} + +/* + * Display the mount options in /proc/mounts. + */ +static int hugetlbfs_show_options(struct seq_file *m, struct dentry *root) +{ + struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(root->d_sb); + struct hugepage_subpool *spool = sbinfo->spool; + unsigned long hpage_size = huge_page_size(sbinfo->hstate); + unsigned hpage_shift = huge_page_shift(sbinfo->hstate); + char mod; + + if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID)) + seq_printf(m, ",uid=%u", + from_kuid_munged(&init_user_ns, sbinfo->uid)); + if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID)) + seq_printf(m, ",gid=%u", + from_kgid_munged(&init_user_ns, sbinfo->gid)); + if (sbinfo->mode != 0755) + seq_printf(m, ",mode=%o", sbinfo->mode); + if (sbinfo->max_inodes != -1) + seq_printf(m, ",nr_inodes=%lu", sbinfo->max_inodes); + + hpage_size /= 1024; + mod = 'K'; + if (hpage_size >= 1024) { + hpage_size /= 1024; + mod = 'M'; + } + seq_printf(m, ",pagesize=%lu%c", hpage_size, mod); + if (spool) { + if (spool->max_hpages != -1) + seq_printf(m, ",size=%llu", + (unsigned long long)spool->max_hpages << hpage_shift); + if (spool->min_hpages != -1) + seq_printf(m, ",min_size=%llu", + (unsigned long long)spool->min_hpages << hpage_shift); + } + return 0; +} + +static int hugetlbfs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(dentry->d_sb); + struct hstate *h = hstate_inode(d_inode(dentry)); + u64 id = huge_encode_dev(dentry->d_sb->s_dev); + + buf->f_fsid = u64_to_fsid(id); + buf->f_type = HUGETLBFS_MAGIC; + buf->f_bsize = huge_page_size(h); + if (sbinfo) { + spin_lock(&sbinfo->stat_lock); + /* If no limits set, just report 0 or -1 for max/free/used + * blocks, like simple_statfs() */ + if (sbinfo->spool) { + long free_pages; + + spin_lock_irq(&sbinfo->spool->lock); + buf->f_blocks = sbinfo->spool->max_hpages; + free_pages = sbinfo->spool->max_hpages + - sbinfo->spool->used_hpages; + buf->f_bavail = buf->f_bfree = free_pages; + spin_unlock_irq(&sbinfo->spool->lock); + buf->f_files = sbinfo->max_inodes; + buf->f_ffree = sbinfo->free_inodes; + } + spin_unlock(&sbinfo->stat_lock); + } + buf->f_namelen = NAME_MAX; + return 0; +} + +static void hugetlbfs_put_super(struct super_block *sb) +{ + struct hugetlbfs_sb_info *sbi = HUGETLBFS_SB(sb); + + if (sbi) { + sb->s_fs_info = NULL; + + hugepage_put_subpool(sbi->spool); + + kfree(sbi); + } +} + +static inline int hugetlbfs_dec_free_inodes(struct hugetlbfs_sb_info *sbinfo) +{ + if (sbinfo->free_inodes >= 0) { + spin_lock(&sbinfo->stat_lock); + if (unlikely(!sbinfo->free_inodes)) { + spin_unlock(&sbinfo->stat_lock); + return 0; + } + sbinfo->free_inodes--; + spin_unlock(&sbinfo->stat_lock); + } + + return 1; +} + +static void hugetlbfs_inc_free_inodes(struct hugetlbfs_sb_info *sbinfo) +{ + if (sbinfo->free_inodes >= 0) { + spin_lock(&sbinfo->stat_lock); + sbinfo->free_inodes++; + spin_unlock(&sbinfo->stat_lock); + } +} + + +static struct kmem_cache *hugetlbfs_inode_cachep; + +static struct inode *hugetlbfs_alloc_inode(struct super_block *sb) +{ + struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(sb); + struct hugetlbfs_inode_info *p; + + if (unlikely(!hugetlbfs_dec_free_inodes(sbinfo))) + return NULL; + p = alloc_inode_sb(sb, hugetlbfs_inode_cachep, GFP_KERNEL); + if (unlikely(!p)) { + hugetlbfs_inc_free_inodes(sbinfo); + return NULL; + } + return &p->vfs_inode; +} + +static void hugetlbfs_free_inode(struct inode *inode) +{ + trace_hugetlbfs_free_inode(inode); + kmem_cache_free(hugetlbfs_inode_cachep, HUGETLBFS_I(inode)); +} + +static void hugetlbfs_destroy_inode(struct inode *inode) +{ + hugetlbfs_inc_free_inodes(HUGETLBFS_SB(inode->i_sb)); +} + +static const struct address_space_operations hugetlbfs_aops = { + .write_begin = hugetlbfs_write_begin, + .write_end = hugetlbfs_write_end, + .dirty_folio = noop_dirty_folio, + .migrate_folio = hugetlbfs_migrate_folio, + .error_remove_folio = hugetlbfs_error_remove_folio, +}; + + +static void init_once(void *foo) +{ + struct hugetlbfs_inode_info *ei = foo; + + inode_init_once(&ei->vfs_inode); +} + +static const struct file_operations hugetlbfs_file_operations = { + .read_iter = hugetlbfs_read_iter, + .mmap = hugetlbfs_file_mmap, + .fsync = noop_fsync, + .get_unmapped_area = hugetlb_get_unmapped_area, + .llseek = default_llseek, + .fallocate = hugetlbfs_fallocate, + .fop_flags = FOP_HUGE_PAGES, +}; + +static const struct inode_operations hugetlbfs_dir_inode_operations = { + .create = hugetlbfs_create, + .lookup = simple_lookup, + .link = simple_link, + .unlink = simple_unlink, + .symlink = hugetlbfs_symlink, + .mkdir = hugetlbfs_mkdir, + .rmdir = simple_rmdir, + .mknod = hugetlbfs_mknod, + .rename = simple_rename, + .setattr = hugetlbfs_setattr, + .tmpfile = hugetlbfs_tmpfile, +}; + +static const struct inode_operations hugetlbfs_inode_operations = { + .setattr = hugetlbfs_setattr, +}; + +static const struct super_operations hugetlbfs_ops = { + .alloc_inode = hugetlbfs_alloc_inode, + .free_inode = hugetlbfs_free_inode, + .destroy_inode = hugetlbfs_destroy_inode, + .evict_inode = hugetlbfs_evict_inode, + .statfs = hugetlbfs_statfs, + .put_super = hugetlbfs_put_super, + .show_options = hugetlbfs_show_options, +}; + +/* + * Convert size option passed from command line to number of huge pages + * in the pool specified by hstate. Size option could be in bytes + * (val_type == SIZE_STD) or percentage of the pool (val_type == SIZE_PERCENT). + */ +static long +hugetlbfs_size_to_hpages(struct hstate *h, unsigned long long size_opt, + enum hugetlbfs_size_type val_type) +{ + if (val_type == NO_SIZE) + return -1; + + if (val_type == SIZE_PERCENT) { + size_opt <<= huge_page_shift(h); + size_opt *= h->max_huge_pages; + do_div(size_opt, 100); + } + + size_opt >>= huge_page_shift(h); + return size_opt; +} + +/* + * Parse one mount parameter. + */ +static int hugetlbfs_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct hugetlbfs_fs_context *ctx = fc->fs_private; + struct fs_parse_result result; + struct hstate *h; + char *rest; + unsigned long ps; + int opt; + + opt = fs_parse(fc, hugetlb_fs_parameters, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_uid: + ctx->uid = result.uid; + return 0; + + case Opt_gid: + ctx->gid = result.gid; + return 0; + + case Opt_mode: + ctx->mode = result.uint_32 & 01777U; + return 0; + + case Opt_size: + /* memparse() will accept a K/M/G without a digit */ + if (!param->string || !isdigit(param->string[0])) + goto bad_val; + ctx->max_size_opt = memparse(param->string, &rest); + ctx->max_val_type = SIZE_STD; + if (*rest == '%') + ctx->max_val_type = SIZE_PERCENT; + return 0; + + case Opt_nr_inodes: + /* memparse() will accept a K/M/G without a digit */ + if (!param->string || !isdigit(param->string[0])) + goto bad_val; + ctx->nr_inodes = memparse(param->string, &rest); + return 0; + + case Opt_pagesize: + ps = memparse(param->string, &rest); + h = size_to_hstate(ps); + if (!h) { + pr_err("Unsupported page size %lu MB\n", ps / SZ_1M); + return -EINVAL; + } + ctx->hstate = h; + return 0; + + case Opt_min_size: + /* memparse() will accept a K/M/G without a digit */ + if (!param->string || !isdigit(param->string[0])) + goto bad_val; + ctx->min_size_opt = memparse(param->string, &rest); + ctx->min_val_type = SIZE_STD; + if (*rest == '%') + ctx->min_val_type = SIZE_PERCENT; + return 0; + + default: + return -EINVAL; + } + +bad_val: + return invalfc(fc, "Bad value '%s' for mount option '%s'\n", + param->string, param->key); +} + +/* + * Validate the parsed options. + */ +static int hugetlbfs_validate(struct fs_context *fc) +{ + struct hugetlbfs_fs_context *ctx = fc->fs_private; + + /* + * Use huge page pool size (in hstate) to convert the size + * options to number of huge pages. If NO_SIZE, -1 is returned. + */ + ctx->max_hpages = hugetlbfs_size_to_hpages(ctx->hstate, + ctx->max_size_opt, + ctx->max_val_type); + ctx->min_hpages = hugetlbfs_size_to_hpages(ctx->hstate, + ctx->min_size_opt, + ctx->min_val_type); + + /* + * If max_size was specified, then min_size must be smaller + */ + if (ctx->max_val_type > NO_SIZE && + ctx->min_hpages > ctx->max_hpages) { + pr_err("Minimum size can not be greater than maximum size\n"); + return -EINVAL; + } + + return 0; +} + +static int +hugetlbfs_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct hugetlbfs_fs_context *ctx = fc->fs_private; + struct hugetlbfs_sb_info *sbinfo; + + sbinfo = kmalloc_obj(struct hugetlbfs_sb_info); + if (!sbinfo) + return -ENOMEM; + sb->s_fs_info = sbinfo; + spin_lock_init(&sbinfo->stat_lock); + sbinfo->hstate = ctx->hstate; + sbinfo->max_inodes = ctx->nr_inodes; + sbinfo->free_inodes = ctx->nr_inodes; + sbinfo->spool = NULL; + sbinfo->uid = ctx->uid; + sbinfo->gid = ctx->gid; + sbinfo->mode = ctx->mode; + + /* + * Allocate and initialize subpool if maximum or minimum size is + * specified. Any needed reservations (for minimum size) are taken + * when the subpool is created. + */ + if (ctx->max_hpages != -1 || ctx->min_hpages != -1) { + sbinfo->spool = hugepage_new_subpool(ctx->hstate, + ctx->max_hpages, + ctx->min_hpages); + if (!sbinfo->spool) + goto out_free; + } + sb->s_maxbytes = MAX_LFS_FILESIZE; + sb->s_blocksize = huge_page_size(ctx->hstate); + sb->s_blocksize_bits = huge_page_shift(ctx->hstate); + sb->s_magic = HUGETLBFS_MAGIC; + sb->s_op = &hugetlbfs_ops; + sb->s_d_flags = DCACHE_DONTCACHE; + sb->s_time_gran = 1; + + /* + * Due to the special and limited functionality of hugetlbfs, it does + * not work well as a stacking filesystem. + */ + sb->s_stack_depth = FILESYSTEM_MAX_STACK_DEPTH; + sb->s_root = d_make_root(hugetlbfs_get_root(sb, ctx)); + if (!sb->s_root) + goto out_free; + return 0; +out_free: + hugepage_put_subpool(sbinfo->spool); + kfree(sbinfo); + return -ENOMEM; +} + +static int hugetlbfs_get_tree(struct fs_context *fc) +{ + int err = hugetlbfs_validate(fc); + if (err) + return err; + return get_tree_nodev(fc, hugetlbfs_fill_super); +} + +static void hugetlbfs_fs_context_free(struct fs_context *fc) +{ + kfree(fc->fs_private); +} + +static const struct fs_context_operations hugetlbfs_fs_context_ops = { + .free = hugetlbfs_fs_context_free, + .parse_param = hugetlbfs_parse_param, + .get_tree = hugetlbfs_get_tree, +}; + +static int hugetlbfs_init_fs_context(struct fs_context *fc) +{ + struct hugetlbfs_fs_context *ctx; + + ctx = kzalloc_obj(struct hugetlbfs_fs_context); + if (!ctx) + return -ENOMEM; + + ctx->max_hpages = -1; /* No limit on size by default */ + ctx->nr_inodes = -1; /* No limit on number of inodes by default */ + ctx->uid = current_fsuid(); + ctx->gid = current_fsgid(); + ctx->mode = 0755; + ctx->hstate = &default_hstate; + ctx->min_hpages = -1; /* No default minimum size */ + ctx->max_val_type = NO_SIZE; + ctx->min_val_type = NO_SIZE; + fc->fs_private = ctx; + fc->ops = &hugetlbfs_fs_context_ops; + return 0; +} + +static struct file_system_type hugetlbfs_fs_type = { + .name = "hugetlbfs", + .init_fs_context = hugetlbfs_init_fs_context, + .parameters = hugetlb_fs_parameters, + .kill_sb = kill_anon_super, + .fs_flags = FS_ALLOW_IDMAP, +}; + +static struct vfsmount *hugetlbfs_vfsmount[HUGE_MAX_HSTATE]; + +static int can_do_hugetlb_shm(void) +{ + kgid_t shm_group; + shm_group = make_kgid(&init_user_ns, sysctl_hugetlb_shm_group); + return capable(CAP_IPC_LOCK) || in_group_p(shm_group); +} + +static int get_hstate_idx(int page_size_log) +{ + struct hstate *h = hstate_sizelog(page_size_log); + + if (!h) + return -1; + return hstate_index(h); +} + +/* + * Note that size should be aligned to proper hugepage size in caller side, + * otherwise hugetlb_reserve_pages reserves one less hugepages than intended. + */ +struct file *hugetlb_file_setup(const char *name, size_t size, + vma_flags_t acctflag, int creat_flags, + int page_size_log) +{ + struct inode *inode; + struct vfsmount *mnt; + int hstate_idx; + struct file *file; + + hstate_idx = get_hstate_idx(page_size_log); + if (hstate_idx < 0) + return ERR_PTR(-ENODEV); + + mnt = hugetlbfs_vfsmount[hstate_idx]; + if (!mnt) + return ERR_PTR(-ENOENT); + + if (creat_flags == HUGETLB_SHMFS_INODE && !can_do_hugetlb_shm()) { + struct ucounts *ucounts = current_ucounts(); + + if (user_shm_lock(size, ucounts)) { + pr_warn_once("%s (%d): Using mlock ulimits for SHM_HUGETLB is obsolete\n", + current->comm, current->pid); + user_shm_unlock(size, ucounts); + } + return ERR_PTR(-EPERM); + } + + file = ERR_PTR(-ENOSPC); + /* hugetlbfs_vfsmount[] mounts do not use idmapped mounts. */ + inode = hugetlbfs_get_inode(mnt->mnt_sb, &nop_mnt_idmap, NULL, + S_IFREG | S_IRWXUGO, 0); + if (!inode) + goto out; + if (creat_flags == HUGETLB_SHMFS_INODE) + inode->i_flags |= S_PRIVATE; + + inode->i_size = size; + clear_nlink(inode); + + if (hugetlb_reserve_pages(inode, 0, + size >> huge_page_shift(hstate_inode(inode)), NULL, + acctflag) < 0) + file = ERR_PTR(-ENOMEM); + else + file = alloc_file_pseudo(inode, mnt, name, O_RDWR, + &hugetlbfs_file_operations); + if (!IS_ERR(file)) + return file; + + iput(inode); +out: + return file; +} + +static struct vfsmount *__init mount_one_hugetlbfs(struct hstate *h) +{ + struct fs_context *fc; + struct vfsmount *mnt; + + fc = fs_context_for_mount(&hugetlbfs_fs_type, SB_KERNMOUNT); + if (IS_ERR(fc)) { + mnt = ERR_CAST(fc); + } else { + struct hugetlbfs_fs_context *ctx = fc->fs_private; + ctx->hstate = h; + mnt = fc_mount_longterm(fc); + put_fs_context(fc); + } + if (IS_ERR(mnt)) + pr_err("Cannot mount internal hugetlbfs for page size %luK", + huge_page_size(h) / SZ_1K); + return mnt; +} + +static int __init init_hugetlbfs_fs(void) +{ + struct vfsmount *mnt; + struct hstate *h; + int error; + int i; + + if (!hugepages_supported()) { + pr_info("disabling because there are no supported hugepage sizes\n"); + return -ENOTSUPP; + } + + error = -ENOMEM; + hugetlbfs_inode_cachep = kmem_cache_create("hugetlbfs_inode_cache", + sizeof(struct hugetlbfs_inode_info), + 0, SLAB_ACCOUNT, init_once); + if (hugetlbfs_inode_cachep == NULL) + goto out; + + error = register_filesystem(&hugetlbfs_fs_type); + if (error) + goto out_free; + + /* default hstate mount is required */ + mnt = mount_one_hugetlbfs(&default_hstate); + if (IS_ERR(mnt)) { + error = PTR_ERR(mnt); + goto out_unreg; + } + hugetlbfs_vfsmount[default_hstate_idx] = mnt; + + /* other hstates are optional */ + i = 0; + for_each_hstate(h) { + if (i == default_hstate_idx) { + i++; + continue; + } + + mnt = mount_one_hugetlbfs(h); + if (IS_ERR(mnt)) + hugetlbfs_vfsmount[i] = NULL; + else + hugetlbfs_vfsmount[i] = mnt; + i++; + } + + return 0; + + out_unreg: + (void)unregister_filesystem(&hugetlbfs_fs_type); + out_free: + kmem_cache_destroy(hugetlbfs_inode_cachep); + out: + return error; +} +fs_initcall(init_hugetlbfs_fs) diff --git a/fs/init.c b/fs/init.c new file mode 100644 index 000000000..33e312d74 --- /dev/null +++ b/fs/init.c @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Routines that mimic syscalls, but don't use the user address space or file + * descriptors. Only for init/ and related early init code. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +int __init init_pivot_root(const char *new_root, const char *put_old) +{ + struct path new_path __free(path_put) = {}; + struct path old_path __free(path_put) = {}; + int ret; + + ret = kern_path(new_root, LOOKUP_FOLLOW | LOOKUP_DIRECTORY, &new_path); + if (ret) + return ret; + + ret = kern_path(put_old, LOOKUP_FOLLOW | LOOKUP_DIRECTORY, &old_path); + if (ret) + return ret; + + return path_pivot_root(&new_path, &old_path); +} + +int __init init_mount(const char *dev_name, const char *dir_name, + const char *type_page, unsigned long flags, void *data_page) +{ + struct path path; + int ret; + + ret = kern_path(dir_name, LOOKUP_FOLLOW, &path); + if (ret) + return ret; + ret = path_mount(dev_name, &path, type_page, flags, data_page); + path_put(&path); + return ret; +} + +int __init init_umount(const char *name, int flags) +{ + int lookup_flags = LOOKUP_MOUNTPOINT; + struct path path; + int ret; + + if (!(flags & UMOUNT_NOFOLLOW)) + lookup_flags |= LOOKUP_FOLLOW; + ret = kern_path(name, lookup_flags, &path); + if (ret) + return ret; + return path_umount(&path, flags); +} + +int __init init_chdir(const char *filename) +{ + struct path path; + int error; + + error = kern_path(filename, LOOKUP_FOLLOW | LOOKUP_DIRECTORY, &path); + if (error) + return error; + error = path_permission(&path, MAY_EXEC | MAY_CHDIR); + if (!error) + set_fs_pwd(current->fs, &path); + path_put(&path); + return error; +} + +int __init init_chroot(const char *filename) +{ + struct path path; + int error; + + error = kern_path(filename, LOOKUP_FOLLOW | LOOKUP_DIRECTORY, &path); + if (error) + return error; + error = path_permission(&path, MAY_EXEC | MAY_CHDIR); + if (error) + goto dput_and_out; + error = -EPERM; + if (!ns_capable(current_user_ns(), CAP_SYS_CHROOT)) + goto dput_and_out; + error = security_path_chroot(&path); + if (error) + goto dput_and_out; + set_fs_root(current->fs, &path); +dput_and_out: + path_put(&path); + return error; +} + +int __init init_chown(const char *filename, uid_t user, gid_t group, int flags) +{ + int lookup_flags = (flags & AT_SYMLINK_NOFOLLOW) ? 0 : LOOKUP_FOLLOW; + struct path path; + int error; + + error = kern_path(filename, lookup_flags, &path); + if (error) + return error; + error = mnt_want_write(path.mnt); + if (!error) { + error = chown_common(&path, user, group); + mnt_drop_write(path.mnt); + } + path_put(&path); + return error; +} + +int __init init_chmod(const char *filename, umode_t mode) +{ + struct path path; + int error; + + error = kern_path(filename, LOOKUP_FOLLOW, &path); + if (error) + return error; + error = chmod_common(&path, mode); + path_put(&path); + return error; +} + +int __init init_eaccess(const char *filename) +{ + struct path path; + int error; + + error = kern_path(filename, LOOKUP_FOLLOW, &path); + if (error) + return error; + error = path_permission(&path, MAY_ACCESS); + path_put(&path); + return error; +} + +int __init init_stat(const char *filename, struct kstat *stat, int flags) +{ + int lookup_flags = (flags & AT_SYMLINK_NOFOLLOW) ? 0 : LOOKUP_FOLLOW; + struct path path; + int error; + + error = kern_path(filename, lookup_flags, &path); + if (error) + return error; + error = vfs_getattr(&path, stat, STATX_BASIC_STATS, + flags | AT_NO_AUTOMOUNT); + path_put(&path); + return error; +} + +int __init init_mknod(const char *filename, umode_t mode, unsigned int dev) +{ + CLASS(filename_kernel, name)(filename); + return filename_mknodat(AT_FDCWD, name, mode, dev); +} + +int __init init_link(const char *oldname, const char *newname) +{ + CLASS(filename_kernel, old)(oldname); + CLASS(filename_kernel, new)(newname); + return filename_linkat(AT_FDCWD, old, AT_FDCWD, new, 0); +} + +int __init init_symlink(const char *oldname, const char *newname) +{ + CLASS(filename_kernel, old)(oldname); + CLASS(filename_kernel, new)(newname); + return filename_symlinkat(old, AT_FDCWD, new); +} + +int __init init_unlink(const char *pathname) +{ + CLASS(filename_kernel, name)(pathname); + return filename_unlinkat(AT_FDCWD, name); +} + +int __init init_mkdir(const char *pathname, umode_t mode) +{ + CLASS(filename_kernel, name)(pathname); + return filename_mkdirat(AT_FDCWD, name, mode); +} + +int __init init_rmdir(const char *pathname) +{ + CLASS(filename_kernel, name)(pathname); + return filename_rmdir(AT_FDCWD, name); +} + +int __init init_utimes(char *filename, struct timespec64 *ts) +{ + struct path path; + int error; + + error = kern_path(filename, 0, &path); + if (error) + return error; + error = vfs_utimes(&path, ts); + path_put(&path); + return error; +} + +int __init init_dup(struct file *file) +{ + int fd; + + fd = get_unused_fd_flags(0); + if (fd < 0) + return fd; + fd_install(fd, get_file(file)); + return 0; +} diff --git a/fs/inode.c b/fs/inode.c new file mode 100644 index 000000000..ba7da39be --- /dev/null +++ b/fs/inode.c @@ -0,0 +1,3116 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * (C) 1997 Linus Torvalds + * (C) 1999 Andrea Arcangeli (dynamic inode allocation) + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define CREATE_TRACE_POINTS +#include + +#include "internal.h" + +/* + * Inode locking rules: + * + * inode->i_lock protects: + * inode->i_state, inode->i_hash, __iget(), inode->i_io_list + * Inode LRU list locks protect: + * inode->i_sb->s_inode_lru, inode->i_lru + * inode->i_sb->s_inode_list_lock protects: + * inode->i_sb->s_inodes, inode->i_sb_list + * bdi->wb.list_lock protects: + * bdi->wb.b_{dirty,io,more_io,dirty_time}, inode->i_io_list + * inode_hash_lock protects: + * inode_hashtable, inode->i_hash + * + * Lock ordering: + * + * inode->i_sb->s_inode_list_lock + * inode->i_lock + * Inode LRU list locks + * + * bdi->wb.list_lock + * inode->i_lock + * + * inode_hash_lock + * inode->i_lock + */ + +static unsigned int i_hash_mask __ro_after_init; +static unsigned int i_hash_shift __ro_after_init; +static struct hlist_head *inode_hashtable __ro_after_init; +static __cacheline_aligned_in_smp DEFINE_SPINLOCK(inode_hash_lock); + +/* + * Empty aops. Can be used for the cases where the user does not + * define any of the address_space operations. + */ +const struct address_space_operations empty_aops = { +}; +EXPORT_SYMBOL(empty_aops); + +static DEFINE_PER_CPU(unsigned long, nr_inodes); +static DEFINE_PER_CPU(unsigned long, nr_unused); + +static struct kmem_cache *inode_cachep __ro_after_init; + +static long get_nr_inodes(void) +{ + int i; + long sum = 0; + for_each_possible_cpu(i) + sum += per_cpu(nr_inodes, i); + return sum < 0 ? 0 : sum; +} + +static inline long get_nr_inodes_unused(void) +{ + int i; + long sum = 0; + for_each_possible_cpu(i) + sum += per_cpu(nr_unused, i); + return sum < 0 ? 0 : sum; +} + +long get_nr_dirty_inodes(void) +{ + /* not actually dirty inodes, but a wild approximation */ + long nr_dirty = get_nr_inodes() - get_nr_inodes_unused(); + return nr_dirty > 0 ? nr_dirty : 0; +} + +#ifdef CONFIG_DEBUG_FS +static DEFINE_PER_CPU(long, mg_ctime_updates); +static DEFINE_PER_CPU(long, mg_fine_stamps); +static DEFINE_PER_CPU(long, mg_ctime_swaps); + +static unsigned long get_mg_ctime_updates(void) +{ + unsigned long sum = 0; + int i; + + for_each_possible_cpu(i) + sum += data_race(per_cpu(mg_ctime_updates, i)); + return sum; +} + +static unsigned long get_mg_fine_stamps(void) +{ + unsigned long sum = 0; + int i; + + for_each_possible_cpu(i) + sum += data_race(per_cpu(mg_fine_stamps, i)); + return sum; +} + +static unsigned long get_mg_ctime_swaps(void) +{ + unsigned long sum = 0; + int i; + + for_each_possible_cpu(i) + sum += data_race(per_cpu(mg_ctime_swaps, i)); + return sum; +} + +#define mgtime_counter_inc(__var) this_cpu_inc(__var) + +static int mgts_show(struct seq_file *s, void *p) +{ + unsigned long ctime_updates = get_mg_ctime_updates(); + unsigned long ctime_swaps = get_mg_ctime_swaps(); + unsigned long fine_stamps = get_mg_fine_stamps(); + unsigned long floor_swaps = timekeeping_get_mg_floor_swaps(); + + seq_printf(s, "%lu %lu %lu %lu\n", + ctime_updates, ctime_swaps, fine_stamps, floor_swaps); + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(mgts); + +static int __init mg_debugfs_init(void) +{ + debugfs_create_file("multigrain_timestamps", S_IFREG | S_IRUGO, NULL, NULL, &mgts_fops); + return 0; +} +late_initcall(mg_debugfs_init); + +#else /* ! CONFIG_DEBUG_FS */ + +#define mgtime_counter_inc(__var) do { } while (0) + +#endif /* CONFIG_DEBUG_FS */ + +/* + * Handle nr_inode sysctl + */ +#ifdef CONFIG_SYSCTL +/* + * Statistics gathering.. + */ +static struct inodes_stat_t inodes_stat; + +static int proc_nr_inodes(const struct ctl_table *table, int write, void *buffer, + size_t *lenp, loff_t *ppos) +{ + inodes_stat.nr_inodes = get_nr_inodes(); + inodes_stat.nr_unused = get_nr_inodes_unused(); + return proc_doulongvec_minmax(table, write, buffer, lenp, ppos); +} + +static const struct ctl_table inodes_sysctls[] = { + { + .procname = "inode-nr", + .data = &inodes_stat, + .maxlen = 2*sizeof(long), + .mode = 0444, + .proc_handler = proc_nr_inodes, + }, + { + .procname = "inode-state", + .data = &inodes_stat, + .maxlen = 7*sizeof(long), + .mode = 0444, + .proc_handler = proc_nr_inodes, + }, +}; + +static int __init init_fs_inode_sysctls(void) +{ + register_sysctl_init("fs", inodes_sysctls); + return 0; +} +early_initcall(init_fs_inode_sysctls); +#endif + +static int no_open(struct inode *inode, struct file *file) +{ + return -ENXIO; +} + +/** + * inode_init_always_gfp - perform inode structure initialisation + * @sb: superblock inode belongs to + * @inode: inode to initialise + * @gfp: allocation flags + * + * These are initializations that need to be done on every inode + * allocation as the fields are not initialised by slab allocation. + * If there are additional allocations required @gfp is used. + */ +int inode_init_always_gfp(struct super_block *sb, struct inode *inode, gfp_t gfp) +{ + static const struct inode_operations empty_iops; + static const struct file_operations no_open_fops = {.open = no_open}; + struct address_space *const mapping = &inode->i_data; + + inode->i_sb = sb; + inode->i_blkbits = sb->s_blocksize_bits; + inode->i_flags = 0; + inode_state_assign_raw(inode, 0); + atomic64_set(&inode->i_sequence, 0); + atomic_set(&inode->i_count, 1); + inode->i_op = &empty_iops; + inode->i_fop = &no_open_fops; + inode->i_ino = 0; + inode->__i_nlink = 1; + inode->i_opflags = 0; + if (sb->s_xattr) + inode->i_opflags |= IOP_XATTR; + if (sb->s_type->fs_flags & FS_MGTIME) + inode->i_opflags |= IOP_MGTIME; + i_uid_write(inode, 0); + i_gid_write(inode, 0); + atomic_set(&inode->i_writecount, 0); + inode->i_size = 0; + inode->i_write_hint = WRITE_LIFE_NOT_SET; + inode->i_blocks = 0; + inode->i_bytes = 0; + inode->i_generation = 0; + inode->i_pipe = NULL; + inode->i_cdev = NULL; + inode->i_link = NULL; + inode->i_dir_seq = 0; + inode->i_rdev = 0; + inode->dirtied_when = 0; + +#ifdef CONFIG_CGROUP_WRITEBACK + inode->i_wb_frn_winner = 0; + inode->i_wb_frn_avg_time = 0; + inode->i_wb_frn_history = 0; +#endif + + spin_lock_init(&inode->i_lock); + lockdep_set_class(&inode->i_lock, &sb->s_type->i_lock_key); + + init_rwsem(&inode->i_rwsem); + lockdep_set_class(&inode->i_rwsem, &sb->s_type->i_mutex_key); + + atomic_set(&inode->i_dio_count, 0); + + mapping->a_ops = &empty_aops; + mapping->host = inode; + mapping->flags = 0; + mapping->wb_err = 0; + atomic_set(&mapping->i_mmap_writable, 0); + mapping_set_gfp_mask(mapping, GFP_HIGHUSER_MOVABLE); + mapping->writeback_index = 0; + init_rwsem(&mapping->invalidate_lock); + lockdep_set_class_and_name(&mapping->invalidate_lock, + &sb->s_type->invalidate_lock_key, + "mapping.invalidate_lock"); + if (sb->s_iflags & SB_I_STABLE_WRITES) + mapping_set_stable_writes(mapping); + inode->i_private = NULL; + inode->i_mapping = mapping; + INIT_HLIST_HEAD(&inode->i_dentry); /* buggered by rcu freeing */ +#ifdef CONFIG_FS_POSIX_ACL + inode->i_acl = inode->i_default_acl = ACL_NOT_CACHED; +#endif + +#ifdef CONFIG_FSNOTIFY + inode->i_fsnotify_mask = 0; +#endif + inode->i_flctx = NULL; + + if (unlikely(security_inode_alloc(inode, gfp))) + return -ENOMEM; + + this_cpu_inc(nr_inodes); + + return 0; +} +EXPORT_SYMBOL(inode_init_always_gfp); + +void free_inode_nonrcu(struct inode *inode) +{ + kmem_cache_free(inode_cachep, inode); +} +EXPORT_SYMBOL(free_inode_nonrcu); + +static void i_callback(struct rcu_head *head) +{ + struct inode *inode = container_of(head, struct inode, i_rcu); + if (inode->free_inode) + inode->free_inode(inode); + else + free_inode_nonrcu(inode); +} + +/** + * alloc_inode - obtain an inode + * @sb: superblock + * + * Allocates a new inode for given superblock. + * Inode wont be chained in superblock s_inodes list + * This means : + * - fs can't be unmount + * - quotas, fsnotify, writeback can't work + */ +struct inode *alloc_inode(struct super_block *sb) +{ + const struct super_operations *ops = sb->s_op; + struct inode *inode; + + if (ops->alloc_inode) + inode = ops->alloc_inode(sb); + else + inode = alloc_inode_sb(sb, inode_cachep, GFP_KERNEL); + + if (!inode) + return NULL; + + if (unlikely(inode_init_always(sb, inode))) { + if (ops->destroy_inode) { + ops->destroy_inode(inode); + if (!ops->free_inode) + return NULL; + } + inode->free_inode = ops->free_inode; + i_callback(&inode->i_rcu); + return NULL; + } + + return inode; +} + +void __destroy_inode(struct inode *inode) +{ + inode_detach_wb(inode); + security_inode_free(inode); + fsnotify_inode_delete(inode); + locks_free_lock_context(inode); + if (!inode->i_nlink) { + WARN_ON(atomic_long_read(&inode->i_sb->s_remove_count) == 0); + atomic_long_dec(&inode->i_sb->s_remove_count); + } + +#ifdef CONFIG_FS_POSIX_ACL + if (inode->i_acl && !is_uncached_acl(inode->i_acl)) + posix_acl_release(inode->i_acl); + if (inode->i_default_acl && !is_uncached_acl(inode->i_default_acl)) + posix_acl_release(inode->i_default_acl); +#endif + this_cpu_dec(nr_inodes); +} +EXPORT_SYMBOL(__destroy_inode); + +static void destroy_inode(struct inode *inode) +{ + const struct super_operations *ops = inode->i_sb->s_op; + + BUG_ON(!list_empty(&inode->i_lru)); + __destroy_inode(inode); + if (ops->destroy_inode) { + ops->destroy_inode(inode); + if (!ops->free_inode) + return; + } + inode->free_inode = ops->free_inode; + call_rcu(&inode->i_rcu, i_callback); +} + +/** + * drop_nlink - directly drop an inode's link count + * @inode: inode + * + * This is a low-level filesystem helper to replace any + * direct filesystem manipulation of i_nlink. In cases + * where we are attempting to track writes to the + * filesystem, a decrement to zero means an imminent + * write when the file is truncated and actually unlinked + * on the filesystem. + */ +void drop_nlink(struct inode *inode) +{ + WARN_ON(inode->i_nlink == 0); + inode->__i_nlink--; + if (!inode->i_nlink) + atomic_long_inc(&inode->i_sb->s_remove_count); +} +EXPORT_SYMBOL(drop_nlink); + +/** + * clear_nlink - directly zero an inode's link count + * @inode: inode + * + * This is a low-level filesystem helper to replace any + * direct filesystem manipulation of i_nlink. See + * drop_nlink() for why we care about i_nlink hitting zero. + */ +void clear_nlink(struct inode *inode) +{ + if (inode->i_nlink) { + inode->__i_nlink = 0; + atomic_long_inc(&inode->i_sb->s_remove_count); + } +} +EXPORT_SYMBOL(clear_nlink); + +/** + * set_nlink - directly set an inode's link count + * @inode: inode + * @nlink: new nlink (should be non-zero) + * + * This is a low-level filesystem helper to replace any + * direct filesystem manipulation of i_nlink. + */ +void set_nlink(struct inode *inode, unsigned int nlink) +{ + if (!nlink) { + clear_nlink(inode); + } else { + /* Yes, some filesystems do change nlink from zero to one */ + if (inode->i_nlink == 0) + atomic_long_dec(&inode->i_sb->s_remove_count); + + inode->__i_nlink = nlink; + } +} +EXPORT_SYMBOL(set_nlink); + +/** + * inc_nlink - directly increment an inode's link count + * @inode: inode + * + * This is a low-level filesystem helper to replace any + * direct filesystem manipulation of i_nlink. Currently, + * it is only here for parity with dec_nlink(). + */ +void inc_nlink(struct inode *inode) +{ + if (unlikely(inode->i_nlink == 0)) { + WARN_ON(!(inode_state_read_once(inode) & I_LINKABLE)); + atomic_long_dec(&inode->i_sb->s_remove_count); + } + + inode->__i_nlink++; +} +EXPORT_SYMBOL(inc_nlink); + +static void __address_space_init_once(struct address_space *mapping) +{ + xa_init_flags(&mapping->i_pages, XA_FLAGS_LOCK_IRQ | XA_FLAGS_ACCOUNT); + init_rwsem(&mapping->i_mmap_rwsem); + spin_lock_init(&mapping->i_private_lock); + mapping->i_mmap = RB_ROOT_CACHED; +} + +void address_space_init_once(struct address_space *mapping) +{ + memset(mapping, 0, sizeof(*mapping)); + __address_space_init_once(mapping); +} +EXPORT_SYMBOL(address_space_init_once); + +/* + * These are initializations that only need to be done + * once, because the fields are idempotent across use + * of the inode, so let the slab aware of that. + */ +void inode_init_once(struct inode *inode) +{ + memset(inode, 0, sizeof(*inode)); + INIT_HLIST_NODE(&inode->i_hash); + INIT_LIST_HEAD(&inode->i_devices); + INIT_LIST_HEAD(&inode->i_io_list); + INIT_LIST_HEAD(&inode->i_wb_list); + INIT_LIST_HEAD(&inode->i_lru); + INIT_LIST_HEAD(&inode->i_sb_list); + __address_space_init_once(&inode->i_data); + i_size_ordered_init(inode); +} +EXPORT_SYMBOL(inode_init_once); + +static void init_once(void *foo) +{ + struct inode *inode = (struct inode *) foo; + + inode_init_once(inode); +} + +struct wait_queue_head *inode_bit_waitqueue(struct wait_bit_queue_entry *wqe, + struct inode *inode, u32 bit) +{ + void *bit_address; + + bit_address = inode_state_wait_address(inode, bit); + init_wait_var_entry(wqe, bit_address, 0); + return __var_waitqueue(bit_address); +} +EXPORT_SYMBOL(inode_bit_waitqueue); + +void wait_on_new_inode(struct inode *inode) +{ + struct wait_bit_queue_entry wqe; + struct wait_queue_head *wq_head; + + spin_lock(&inode->i_lock); + if (!(inode_state_read(inode) & I_NEW)) { + spin_unlock(&inode->i_lock); + return; + } + + wq_head = inode_bit_waitqueue(&wqe, inode, __I_NEW); + for (;;) { + prepare_to_wait_event(wq_head, &wqe.wq_entry, TASK_UNINTERRUPTIBLE); + if (!(inode_state_read(inode) & I_NEW)) + break; + spin_unlock(&inode->i_lock); + schedule(); + spin_lock(&inode->i_lock); + } + finish_wait(wq_head, &wqe.wq_entry); + WARN_ON(inode_state_read(inode) & I_NEW); + spin_unlock(&inode->i_lock); +} +EXPORT_SYMBOL(wait_on_new_inode); + +static void __inode_lru_list_add(struct inode *inode, bool rotate) +{ + lockdep_assert_held(&inode->i_lock); + + if (inode_state_read(inode) & (I_DIRTY_ALL | I_SYNC | I_FREEING | I_WILL_FREE)) + return; + if (icount_read(inode)) + return; + if (!(inode->i_sb->s_flags & SB_ACTIVE)) + return; + if (!mapping_shrinkable(&inode->i_data)) + return; + + if (list_lru_add_obj(&inode->i_sb->s_inode_lru, &inode->i_lru)) + this_cpu_inc(nr_unused); + else if (rotate) + inode_state_set(inode, I_REFERENCED); +} + +/* + * Add inode to LRU if needed (inode is unused and clean). + */ +void inode_lru_list_add(struct inode *inode) +{ + __inode_lru_list_add(inode, false); +} + +static void inode_lru_list_del(struct inode *inode) +{ + if (list_empty(&inode->i_lru)) + return; + + if (list_lru_del_obj(&inode->i_sb->s_inode_lru, &inode->i_lru)) + this_cpu_dec(nr_unused); +} + +static void inode_pin_lru_isolating(struct inode *inode) +{ + lockdep_assert_held(&inode->i_lock); + WARN_ON(inode_state_read(inode) & (I_LRU_ISOLATING | I_FREEING | I_WILL_FREE)); + inode_state_set(inode, I_LRU_ISOLATING); +} + +static void inode_unpin_lru_isolating(struct inode *inode) +{ + spin_lock(&inode->i_lock); + WARN_ON(!(inode_state_read(inode) & I_LRU_ISOLATING)); + inode_state_clear(inode, I_LRU_ISOLATING); + /* Called with inode->i_lock which ensures memory ordering. */ + inode_wake_up_bit(inode, __I_LRU_ISOLATING); + spin_unlock(&inode->i_lock); +} + +static void inode_wait_for_lru_isolating(struct inode *inode) +{ + struct wait_bit_queue_entry wqe; + struct wait_queue_head *wq_head; + + lockdep_assert_held(&inode->i_lock); + if (!(inode_state_read(inode) & I_LRU_ISOLATING)) + return; + + wq_head = inode_bit_waitqueue(&wqe, inode, __I_LRU_ISOLATING); + for (;;) { + prepare_to_wait_event(wq_head, &wqe.wq_entry, TASK_UNINTERRUPTIBLE); + /* + * Checking I_LRU_ISOLATING with inode->i_lock guarantees + * memory ordering. + */ + if (!(inode_state_read(inode) & I_LRU_ISOLATING)) + break; + spin_unlock(&inode->i_lock); + schedule(); + spin_lock(&inode->i_lock); + } + finish_wait(wq_head, &wqe.wq_entry); + WARN_ON(inode_state_read(inode) & I_LRU_ISOLATING); +} + +/** + * inode_sb_list_add - add inode to the superblock list of inodes + * @inode: inode to add + */ +void inode_sb_list_add(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + + spin_lock(&sb->s_inode_list_lock); + list_add(&inode->i_sb_list, &sb->s_inodes); + spin_unlock(&sb->s_inode_list_lock); +} +EXPORT_SYMBOL_GPL(inode_sb_list_add); + +static inline void inode_sb_list_del(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + + if (!list_empty(&inode->i_sb_list)) { + spin_lock(&sb->s_inode_list_lock); + list_del_init(&inode->i_sb_list); + spin_unlock(&sb->s_inode_list_lock); + } +} + +static unsigned long hash(struct super_block *sb, u64 hashval) +{ + unsigned long tmp; + + tmp = (hashval * (unsigned long)sb) ^ (GOLDEN_RATIO_PRIME + hashval) / + L1_CACHE_BYTES; + tmp = tmp ^ ((tmp ^ GOLDEN_RATIO_PRIME) >> i_hash_shift); + return tmp & i_hash_mask; +} + +/** + * __insert_inode_hash - hash an inode + * @inode: unhashed inode + * @hashval: u64 value used to locate this object in the + * inode_hashtable. + * + * Add an inode to the inode hash for this superblock. + */ +void __insert_inode_hash(struct inode *inode, u64 hashval) +{ + struct hlist_head *b = inode_hashtable + hash(inode->i_sb, hashval); + + spin_lock(&inode_hash_lock); + spin_lock(&inode->i_lock); + hlist_add_head_rcu(&inode->i_hash, b); + spin_unlock(&inode->i_lock); + spin_unlock(&inode_hash_lock); +} +EXPORT_SYMBOL(__insert_inode_hash); + +/** + * __remove_inode_hash - remove an inode from the hash + * @inode: inode to unhash + * + * Remove an inode from the superblock. + */ +void __remove_inode_hash(struct inode *inode) +{ + spin_lock(&inode_hash_lock); + spin_lock(&inode->i_lock); + hlist_del_init_rcu(&inode->i_hash); + spin_unlock(&inode->i_lock); + spin_unlock(&inode_hash_lock); +} +EXPORT_SYMBOL(__remove_inode_hash); + +void dump_mapping(const struct address_space *mapping) +{ + struct inode *host; + const struct address_space_operations *a_ops; + struct hlist_node *dentry_first; + struct dentry *dentry_ptr; + struct dentry dentry; + char fname[64] = {}; + u64 ino; + + /* + * If mapping is an invalid pointer, we don't want to crash + * accessing it, so probe everything depending on it carefully. + */ + if (get_kernel_nofault(host, &mapping->host) || + get_kernel_nofault(a_ops, &mapping->a_ops)) { + pr_warn("invalid mapping:%px\n", mapping); + return; + } + + if (!host) { + pr_warn("aops:%ps\n", a_ops); + return; + } + + if (get_kernel_nofault(dentry_first, &host->i_dentry.first) || + get_kernel_nofault(ino, &host->i_ino)) { + pr_warn("aops:%ps invalid inode:%px\n", a_ops, host); + return; + } + + if (!dentry_first) { + pr_warn("aops:%ps ino:%llx\n", a_ops, ino); + return; + } + + dentry_ptr = container_of(dentry_first, struct dentry, d_alias); + if (get_kernel_nofault(dentry, dentry_ptr) || + !dentry.d_parent || !dentry.d_name.name) { + pr_warn("aops:%ps ino:%llx invalid dentry:%px\n", + a_ops, ino, dentry_ptr); + return; + } + + if (strncpy_from_kernel_nofault(fname, dentry.d_name.name, 63) < 0) + strscpy(fname, ""); + /* + * Even if strncpy_from_kernel_nofault() succeeded, + * the fname could be unreliable + */ + pr_warn("aops:%ps ino:%llx dentry name(?):\"%s\"\n", + a_ops, ino, fname); +} + +void clear_inode(struct inode *inode) +{ + /* + * Only IS_VERITY() inodes can have verity info, so start by checking + * for IS_VERITY() (which is faster than retrieving the pointer to the + * verity info). This minimizes overhead for non-verity inodes. + */ + if (IS_ENABLED(CONFIG_FS_VERITY) && IS_VERITY(inode)) + fsverity_cleanup_inode(inode); + + /* + * We have to cycle the i_pages lock here because reclaim + * can be in the process of removing the last page (in + * __filemap_remove_folio()) and we must not free the mapping + * under it. We also remove nodes which are empty; these + * can occur in two different ways. The first is that radix + * tree expansion can fail partway and the second is that THP + * collapse_file() can allocate some temporary nodes and not + * clean them up. + */ + xa_destroy(&inode->i_data.i_pages); + + BUG_ON(inode->i_data.nrpages); + BUG_ON(!(inode_state_read_once(inode) & I_FREEING)); + BUG_ON(inode_state_read_once(inode) & I_CLEAR); + BUG_ON(!list_empty(&inode->i_wb_list)); + /* don't need i_lock here, no concurrent mods to i_state */ + inode_state_assign_raw(inode, I_FREEING | I_CLEAR); +} +EXPORT_SYMBOL(clear_inode); + +/* + * Free the inode passed in, removing it from the lists it is still connected + * to. We remove any pages still attached to the inode and wait for any IO that + * is still in progress before finally destroying the inode. + * + * An inode must already be marked I_FREEING so that we avoid the inode being + * moved back onto lists if we race with other code that manipulates the lists + * (e.g. writeback_single_inode). The caller is responsible for setting this. + * + * An inode must already be removed from the LRU list before being evicted from + * the cache. This should occur atomically with setting the I_FREEING state + * flag, so no inodes here should ever be on the LRU when being evicted. + */ +static void evict(struct inode *inode) +{ + const struct super_operations *op = inode->i_sb->s_op; + + BUG_ON(!(inode_state_read_once(inode) & I_FREEING)); + BUG_ON(!list_empty(&inode->i_lru)); + + inode_io_list_del(inode); + inode_sb_list_del(inode); + + spin_lock(&inode->i_lock); + inode_wait_for_lru_isolating(inode); + + /* + * Wait for flusher thread to be done with the inode so that filesystem + * does not start destroying it while writeback is still running. Since + * the inode has I_FREEING set, flusher thread won't start new work on + * the inode. We just have to wait for running writeback to finish. + */ + inode_wait_for_writeback(inode); + spin_unlock(&inode->i_lock); + + if (op->evict_inode) { + op->evict_inode(inode); + } else { + truncate_inode_pages_final(&inode->i_data); + clear_inode(inode); + } + if (S_ISCHR(inode->i_mode) && inode->i_cdev) + cd_forget(inode); + + remove_inode_hash(inode); + + /* + * Wake up waiters in __wait_on_freeing_inode(). + * + * It is an invariant that any thread we need to wake up is already + * accounted for before remove_inode_hash() acquires ->i_lock -- both + * sides take the lock and sleep is aborted if the inode is found + * unhashed. Thus either the sleeper wins and goes off CPU, or removal + * wins and the sleeper aborts after testing with the lock. + * + * This also means we don't need any fences for the call below. + */ + inode_wake_up_bit(inode, __I_NEW); + BUG_ON(inode_state_read_once(inode) != (I_FREEING | I_CLEAR)); + + destroy_inode(inode); +} + +/* + * dispose_list - dispose of the contents of a local list + * @head: the head of the list to free + * + * Dispose-list gets a local list with local inodes in it, so it doesn't + * need to worry about list corruption and SMP locks. + */ +static void dispose_list(struct list_head *head) +{ + while (!list_empty(head)) { + struct inode *inode; + + inode = list_first_entry(head, struct inode, i_lru); + list_del_init(&inode->i_lru); + + evict(inode); + cond_resched(); + } +} + +/** + * evict_inodes - evict all evictable inodes for a superblock + * @sb: superblock to operate on + * + * Make sure that no inodes with zero refcount are retained. This is + * called by superblock shutdown after having SB_ACTIVE flag removed, + * so any inode reaching zero refcount during or after that call will + * be immediately evicted. + */ +void evict_inodes(struct super_block *sb) +{ + struct inode *inode; + LIST_HEAD(dispose); + +again: + spin_lock(&sb->s_inode_list_lock); + list_for_each_entry(inode, &sb->s_inodes, i_sb_list) { + if (icount_read_once(inode)) + continue; + + spin_lock(&inode->i_lock); + if (icount_read(inode)) { + spin_unlock(&inode->i_lock); + continue; + } + if (inode_state_read(inode) & (I_NEW | I_FREEING | I_WILL_FREE)) { + spin_unlock(&inode->i_lock); + continue; + } + + inode_state_set(inode, I_FREEING); + inode_lru_list_del(inode); + spin_unlock(&inode->i_lock); + list_add(&inode->i_lru, &dispose); + + /* + * We can have a ton of inodes to evict at unmount time given + * enough memory, check to see if we need to go to sleep for a + * bit so we don't livelock. + */ + if (need_resched()) { + spin_unlock(&sb->s_inode_list_lock); + cond_resched(); + dispose_list(&dispose); + goto again; + } + } + spin_unlock(&sb->s_inode_list_lock); + + dispose_list(&dispose); +} +EXPORT_SYMBOL_GPL(evict_inodes); + +/* + * Isolate the inode from the LRU in preparation for freeing it. + * + * If the inode has the I_REFERENCED flag set, then it means that it has been + * used recently - the flag is set in iput_final(). When we encounter such an + * inode, clear the flag and move it to the back of the LRU so it gets another + * pass through the LRU before it gets reclaimed. This is necessary because of + * the fact we are doing lazy LRU updates to minimise lock contention so the + * LRU does not have strict ordering. Hence we don't want to reclaim inodes + * with this flag set because they are the inodes that are out of order. + */ +static enum lru_status inode_lru_isolate(struct list_head *item, + struct list_lru_one *lru, void *arg) +{ + struct list_head *freeable = arg; + struct inode *inode = container_of(item, struct inode, i_lru); + + /* + * We are inverting the lru lock/inode->i_lock here, so use a + * trylock. If we fail to get the lock, just skip it. + */ + if (!spin_trylock(&inode->i_lock)) + return LRU_SKIP; + + /* + * Inodes can get referenced, redirtied, or repopulated while + * they're already on the LRU, and this can make them + * unreclaimable for a while. Remove them lazily here; iput, + * sync, or the last page cache deletion will requeue them. + */ + if (icount_read(inode) || + (inode_state_read(inode) & ~I_REFERENCED) || + !mapping_shrinkable(&inode->i_data)) { + list_lru_isolate(lru, &inode->i_lru); + spin_unlock(&inode->i_lock); + this_cpu_dec(nr_unused); + return LRU_REMOVED; + } + + /* Recently referenced inodes get one more pass */ + if (inode_state_read(inode) & I_REFERENCED) { + inode_state_clear(inode, I_REFERENCED); + spin_unlock(&inode->i_lock); + return LRU_ROTATE; + } + + /* + * On highmem systems, mapping_shrinkable() permits dropping + * page cache in order to free up struct inodes: lowmem might + * be under pressure before the cache inside the highmem zone. + */ + if (!mapping_empty(&inode->i_data)) { + unsigned long reap; + + inode_pin_lru_isolating(inode); + spin_unlock(&inode->i_lock); + spin_unlock(&lru->lock); + reap = invalidate_mapping_pages(&inode->i_data, 0, -1); + if (current_is_kswapd()) + __count_vm_events(KSWAPD_INODESTEAL, reap); + else + __count_vm_events(PGINODESTEAL, reap); + mm_account_reclaimed_pages(reap); + inode_unpin_lru_isolating(inode); + return LRU_RETRY; + } + + WARN_ON(inode_state_read(inode) & I_NEW); + inode_state_set(inode, I_FREEING); + list_lru_isolate_move(lru, &inode->i_lru, freeable); + spin_unlock(&inode->i_lock); + + this_cpu_dec(nr_unused); + return LRU_REMOVED; +} + +/* + * Walk the superblock inode LRU for freeable inodes and attempt to free them. + * This is called from the superblock shrinker function with a number of inodes + * to trim from the LRU. Inodes to be freed are moved to a temporary list and + * then are freed outside inode_lock by dispose_list(). + */ +long prune_icache_sb(struct super_block *sb, struct shrink_control *sc) +{ + LIST_HEAD(freeable); + long freed; + + freed = list_lru_shrink_walk(&sb->s_inode_lru, sc, + inode_lru_isolate, &freeable); + dispose_list(&freeable); + return freed; +} + +static void __wait_on_freeing_inode(struct inode *inode, bool hash_locked, bool rcu_locked); +static bool igrab_from_hash(struct inode *inode); + +/* + * Called with the inode lock held. + */ +static struct inode *find_inode(struct super_block *sb, + struct hlist_head *head, + int (*test)(struct inode *, void *), + void *data, bool hash_locked, + bool *isnew) +{ + struct inode *inode = NULL; + + if (hash_locked) + lockdep_assert_held(&inode_hash_lock); + else + lockdep_assert_not_held(&inode_hash_lock); + + rcu_read_lock(); +repeat: + hlist_for_each_entry_rcu(inode, head, i_hash) { + if (inode->i_sb != sb) + continue; + if (!test(inode, data)) + continue; + if (igrab_from_hash(inode)) { + rcu_read_unlock(); + *isnew = false; + return inode; + } + spin_lock(&inode->i_lock); + if (inode_state_read(inode) & (I_FREEING | I_WILL_FREE)) { + __wait_on_freeing_inode(inode, hash_locked, true); + goto repeat; + } + if (unlikely(inode_state_read(inode) & I_CREATING)) { + spin_unlock(&inode->i_lock); + rcu_read_unlock(); + return ERR_PTR(-ESTALE); + } + __iget(inode); + *isnew = !!(inode_state_read(inode) & I_NEW); + spin_unlock(&inode->i_lock); + rcu_read_unlock(); + return inode; + } + rcu_read_unlock(); + return NULL; +} + +/* + * find_inode_fast is the fast path version of find_inode, see the comment at + * iget_locked for details. + */ +static struct inode *find_inode_fast(struct super_block *sb, + struct hlist_head *head, u64 ino, + bool hash_locked, bool *isnew) +{ + struct inode *inode = NULL; + + if (hash_locked) + lockdep_assert_held(&inode_hash_lock); + else + lockdep_assert_not_held(&inode_hash_lock); + + rcu_read_lock(); +repeat: + hlist_for_each_entry_rcu(inode, head, i_hash) { + if (inode->i_ino != ino) + continue; + if (inode->i_sb != sb) + continue; + if (igrab_from_hash(inode)) { + rcu_read_unlock(); + *isnew = false; + return inode; + } + spin_lock(&inode->i_lock); + if (inode_state_read(inode) & (I_FREEING | I_WILL_FREE)) { + __wait_on_freeing_inode(inode, hash_locked, true); + goto repeat; + } + if (unlikely(inode_state_read(inode) & I_CREATING)) { + spin_unlock(&inode->i_lock); + rcu_read_unlock(); + return ERR_PTR(-ESTALE); + } + __iget(inode); + *isnew = !!(inode_state_read(inode) & I_NEW); + spin_unlock(&inode->i_lock); + rcu_read_unlock(); + return inode; + } + rcu_read_unlock(); + return NULL; +} + +/* + * Each cpu owns a range of LAST_INO_BATCH numbers. + * 'shared_last_ino' is dirtied only once out of LAST_INO_BATCH allocations, + * to renew the exhausted range. + * + * This does not significantly increase overflow rate because every CPU can + * consume at most LAST_INO_BATCH-1 unused inode numbers. So there is + * NR_CPUS*(LAST_INO_BATCH-1) wastage. At 4096 and 1024, this is ~0.1% of the + * 2^32 range, and is a worst-case. Even a 50% wastage would only increase + * overflow rate by 2x, which does not seem too significant. + * + * On a 32bit, non LFS stat() call, glibc will generate an EOVERFLOW + * error if st_ino won't fit in target struct field. Use 32bit counter + * here to attempt to avoid that. + */ +#define LAST_INO_BATCH 1024 +static DEFINE_PER_CPU(unsigned int, last_ino); + +unsigned int get_next_ino(void) +{ + unsigned int *p = &get_cpu_var(last_ino); + unsigned int res = *p; + +#ifdef CONFIG_SMP + if (unlikely((res & (LAST_INO_BATCH-1)) == 0)) { + static atomic_t shared_last_ino; + int next = atomic_add_return(LAST_INO_BATCH, &shared_last_ino); + + res = next - LAST_INO_BATCH; + } +#endif + + res++; + /* get_next_ino should not provide a 0 inode number */ + if (unlikely(!res)) + res++; + *p = res; + put_cpu_var(last_ino); + return res; +} +EXPORT_SYMBOL(get_next_ino); + +/** + * new_inode - obtain an inode + * @sb: superblock + * + * Allocates a new inode for given superblock. The default gfp_mask + * for allocations related to inode->i_mapping is GFP_HIGHUSER_MOVABLE. + * If HIGHMEM pages are unsuitable or it is known that pages allocated + * for the page cache are not reclaimable or migratable, + * mapping_set_gfp_mask() must be called with suitable flags on the + * newly created inode's mapping + * + */ +struct inode *new_inode(struct super_block *sb) +{ + struct inode *inode; + + inode = alloc_inode(sb); + if (inode) + inode_sb_list_add(inode); + return inode; +} +EXPORT_SYMBOL(new_inode); + +#ifdef CONFIG_DEBUG_LOCK_ALLOC +void lockdep_annotate_inode_mutex_key(struct inode *inode) +{ + if (S_ISDIR(inode->i_mode)) { + struct file_system_type *type = inode->i_sb->s_type; + + /* Set new key only if filesystem hasn't already changed it */ + if (lockdep_match_class(&inode->i_rwsem, &type->i_mutex_key)) { + /* + * ensure nobody is actually holding i_rwsem + */ + init_rwsem(&inode->i_rwsem); + lockdep_set_class(&inode->i_rwsem, + &type->i_mutex_dir_key); + } + } +} +EXPORT_SYMBOL(lockdep_annotate_inode_mutex_key); +#endif + +/** + * unlock_new_inode - clear the I_NEW state and wake up any waiters + * @inode: new inode to unlock + * + * Called when the inode is fully initialised to clear the new state of the + * inode and wake up anyone waiting for the inode to finish initialisation. + */ +void unlock_new_inode(struct inode *inode) +{ + lockdep_annotate_inode_mutex_key(inode); + spin_lock(&inode->i_lock); + WARN_ON(!(inode_state_read(inode) & I_NEW)); + /* + * Paired with igrab_from_hash() + */ + smp_wmb(); + inode_state_clear(inode, I_NEW | I_CREATING); + inode_wake_up_bit(inode, __I_NEW); + spin_unlock(&inode->i_lock); +} +EXPORT_SYMBOL(unlock_new_inode); + +void discard_new_inode(struct inode *inode) +{ + lockdep_annotate_inode_mutex_key(inode); + spin_lock(&inode->i_lock); + WARN_ON(!(inode_state_read(inode) & I_NEW)); + /* + * Paired with igrab_from_hash() + */ + smp_wmb(); + inode_state_clear(inode, I_NEW); + inode_wake_up_bit(inode, __I_NEW); + spin_unlock(&inode->i_lock); + iput(inode); +} +EXPORT_SYMBOL(discard_new_inode); + +/** + * lock_two_nondirectories - take two i_mutexes on non-directory objects + * + * Lock any non-NULL argument. Passed objects must not be directories. + * Zero, one or two objects may be locked by this function. + * + * @inode1: first inode to lock + * @inode2: second inode to lock + */ +void lock_two_nondirectories(struct inode *inode1, struct inode *inode2) +{ + if (inode1) + WARN_ON_ONCE(S_ISDIR(inode1->i_mode)); + if (inode2) + WARN_ON_ONCE(S_ISDIR(inode2->i_mode)); + if (inode1 > inode2) + swap(inode1, inode2); + if (inode1) + inode_lock(inode1); + if (inode2 && inode2 != inode1) + inode_lock_nested(inode2, I_MUTEX_NONDIR2); +} +EXPORT_SYMBOL(lock_two_nondirectories); + +/** + * unlock_two_nondirectories - release locks from lock_two_nondirectories() + * @inode1: first inode to unlock + * @inode2: second inode to unlock + */ +void unlock_two_nondirectories(struct inode *inode1, struct inode *inode2) +{ + if (inode1) { + WARN_ON_ONCE(S_ISDIR(inode1->i_mode)); + inode_unlock(inode1); + } + if (inode2 && inode2 != inode1) { + WARN_ON_ONCE(S_ISDIR(inode2->i_mode)); + inode_unlock(inode2); + } +} +EXPORT_SYMBOL(unlock_two_nondirectories); + +/** + * inode_insert5 - obtain an inode from a mounted file system + * @inode: pre-allocated inode to use for insert to cache + * @hashval: hash value (usually inode number) to get + * @test: callback used for comparisons between inodes + * @set: callback used to initialize a new struct inode + * @data: opaque data pointer to pass to @test and @set + * + * Search for the inode specified by @hashval and @data in the inode cache, + * and if present return it with an increased reference count. This is a + * variant of iget5_locked() that doesn't allocate an inode. + * + * If the inode is not present in the cache, insert the pre-allocated inode and + * return it locked, hashed, and with the I_NEW flag set. The file system gets + * to fill it in before unlocking it via unlock_new_inode(). + * + * Note that both @test and @set are called with the inode_hash_lock held, so + * they can't sleep. + */ +struct inode *inode_insert5(struct inode *inode, u64 hashval, + int (*test)(struct inode *, void *), + int (*set)(struct inode *, void *), void *data) +{ + struct hlist_head *head = inode_hashtable + hash(inode->i_sb, hashval); + struct inode *old; + bool isnew; + + might_sleep(); + +again: + spin_lock(&inode_hash_lock); + old = find_inode(inode->i_sb, head, test, data, true, &isnew); + if (unlikely(old)) { + /* + * Uhhuh, somebody else created the same inode under us. + * Use the old inode instead of the preallocated one. + */ + spin_unlock(&inode_hash_lock); + if (IS_ERR(old)) + return NULL; + if (unlikely(isnew)) + wait_on_new_inode(old); + if (unlikely(inode_unhashed(old))) { + iput(old); + goto again; + } + return old; + } + + if (set && unlikely(set(inode, data))) { + spin_unlock(&inode_hash_lock); + return NULL; + } + + /* + * Return the locked inode with I_NEW set, the + * caller is responsible for filling in the contents + */ + spin_lock(&inode->i_lock); + inode_state_set(inode, I_NEW); + hlist_add_head_rcu(&inode->i_hash, head); + spin_unlock(&inode->i_lock); + + spin_unlock(&inode_hash_lock); + + /* + * Add inode to the sb list if it's not already. It has I_NEW at this + * point, so it should be safe to test i_sb_list locklessly. + */ + if (list_empty(&inode->i_sb_list)) + inode_sb_list_add(inode); + + return inode; +} +EXPORT_SYMBOL(inode_insert5); + +/** + * iget5_locked - obtain an inode from a mounted file system + * @sb: super block of file system + * @hashval: hash value (usually inode number) to get + * @test: callback used for comparisons between inodes + * @set: callback used to initialize a new struct inode + * @data: opaque data pointer to pass to @test and @set + * + * Search for the inode specified by @hashval and @data in the inode cache, + * and if present return it with an increased reference count. This is a + * generalized version of iget_locked() for file systems where the inode + * number is not sufficient for unique identification of an inode. + * + * If the inode is not present in the cache, allocate and insert a new inode + * and return it locked, hashed, and with the I_NEW flag set. The file system + * gets to fill it in before unlocking it via unlock_new_inode(). + * + * Note that both @test and @set are called with the inode_hash_lock held, so + * they can't sleep. + */ +struct inode *iget5_locked(struct super_block *sb, u64 hashval, + int (*test)(struct inode *, void *), + int (*set)(struct inode *, void *), void *data) +{ + struct inode *inode = ilookup5(sb, hashval, test, data); + + if (!inode) { + struct inode *new = alloc_inode(sb); + + if (new) { + inode = inode_insert5(new, hashval, test, set, data); + if (unlikely(inode != new)) + destroy_inode(new); + } + } + return inode; +} +EXPORT_SYMBOL(iget5_locked); + +/** + * iget5_locked_rcu - obtain an inode from a mounted file system + * @sb: super block of file system + * @hashval: hash value (usually inode number) to get + * @test: callback used for comparisons between inodes + * @set: callback used to initialize a new struct inode + * @data: opaque data pointer to pass to @test and @set + * + * This is equivalent to iget5_locked, except the @test callback must + * tolerate the inode not being stable, including being mid-teardown. + */ +struct inode *iget5_locked_rcu(struct super_block *sb, u64 hashval, + int (*test)(struct inode *, void *), + int (*set)(struct inode *, void *), void *data) +{ + struct hlist_head *head = inode_hashtable + hash(sb, hashval); + struct inode *inode, *new; + bool isnew; + + might_sleep(); + +again: + inode = find_inode(sb, head, test, data, false, &isnew); + if (inode) { + if (IS_ERR(inode)) + return NULL; + if (unlikely(isnew)) + wait_on_new_inode(inode); + if (unlikely(inode_unhashed(inode))) { + iput(inode); + goto again; + } + return inode; + } + + new = alloc_inode(sb); + if (new) { + inode = inode_insert5(new, hashval, test, set, data); + if (unlikely(inode != new)) + destroy_inode(new); + } + return inode; +} +EXPORT_SYMBOL_GPL(iget5_locked_rcu); + +/** + * iget_locked - obtain an inode from a mounted file system + * @sb: super block of file system + * @ino: inode number to get + * + * Search for the inode specified by @ino in the inode cache and if present + * return it with an increased reference count. This is for file systems + * where the inode number is sufficient for unique identification of an inode. + * + * If the inode is not in cache, allocate a new inode and return it locked, + * hashed, and with the I_NEW flag set. The file system gets to fill it in + * before unlocking it via unlock_new_inode(). + */ +struct inode *iget_locked(struct super_block *sb, u64 ino) +{ + struct hlist_head *head = inode_hashtable + hash(sb, ino); + struct inode *inode; + bool isnew; + + might_sleep(); + +again: + inode = find_inode_fast(sb, head, ino, false, &isnew); + if (inode) { + if (IS_ERR(inode)) + return NULL; + if (unlikely(isnew)) + wait_on_new_inode(inode); + if (unlikely(inode_unhashed(inode))) { + iput(inode); + goto again; + } + return inode; + } + + inode = alloc_inode(sb); + if (inode) { + struct inode *old; + + spin_lock(&inode_hash_lock); + /* We released the lock, so.. */ + old = find_inode_fast(sb, head, ino, true, &isnew); + if (!old) { + inode->i_ino = ino; + spin_lock(&inode->i_lock); + inode_state_assign(inode, I_NEW); + hlist_add_head_rcu(&inode->i_hash, head); + spin_unlock(&inode->i_lock); + spin_unlock(&inode_hash_lock); + inode_sb_list_add(inode); + + /* Return the locked inode with I_NEW set, the + * caller is responsible for filling in the contents + */ + return inode; + } + + /* + * Uhhuh, somebody else created the same inode under + * us. Use the old inode instead of the one we just + * allocated. + */ + spin_unlock(&inode_hash_lock); + destroy_inode(inode); + if (IS_ERR(old)) + return NULL; + inode = old; + if (unlikely(isnew)) + wait_on_new_inode(inode); + if (unlikely(inode_unhashed(inode))) { + iput(inode); + goto again; + } + } + return inode; +} +EXPORT_SYMBOL(iget_locked); + +/* + * search the inode cache for a matching inode number. + * If we find one, then the inode number we are trying to + * allocate is not unique and so we should not use it. + * + * Returns 1 if the inode number is unique, 0 if it is not. + */ +static int test_inode_iunique(struct super_block *sb, u64 ino) +{ + struct hlist_head *b = inode_hashtable + hash(sb, ino); + struct inode *inode; + + hlist_for_each_entry_rcu(inode, b, i_hash) { + if (inode->i_ino == ino && inode->i_sb == sb) + return 0; + } + return 1; +} + +/** + * iunique - get a unique inode number + * @sb: superblock + * @max_reserved: highest reserved inode number + * + * Obtain an inode number that is unique on the system for a given + * superblock. This is used by file systems that have no natural + * permanent inode numbering system. An inode number is returned that + * is higher than the reserved limit but unique. + * + * BUGS: + * With a large number of inodes live on the file system this function + * currently becomes quite slow. + */ +ino_t iunique(struct super_block *sb, ino_t max_reserved) +{ + /* + * On a 32bit, non LFS stat() call, glibc will generate an EOVERFLOW + * error if st_ino won't fit in target struct field. Use 32bit counter + * here to attempt to avoid that. + */ + static DEFINE_SPINLOCK(iunique_lock); + static unsigned int counter; + ino_t res; + + rcu_read_lock(); + spin_lock(&iunique_lock); + do { + if (counter <= max_reserved) + counter = max_reserved + 1; + res = counter++; + } while (!test_inode_iunique(sb, res)); + spin_unlock(&iunique_lock); + rcu_read_unlock(); + + return res; +} +EXPORT_SYMBOL(iunique); + +/** + * ihold - get a reference on the inode, provided you already have one + * @inode: inode to operate on + */ +void ihold(struct inode *inode) +{ + VFS_BUG_ON_INODE(icount_read_once(inode) < 1, inode); + WARN_ON(atomic_inc_return(&inode->i_count) < 2); +} +EXPORT_SYMBOL(ihold); + +struct inode *igrab(struct inode *inode) +{ + /* + * Read commentary above igrab_from_hash() for an explanation why this works. + */ + if (atomic_add_unless(&inode->i_count, 1, 0)) { + VFS_BUG_ON_INODE(inode_state_read_once(inode) & (I_FREEING | I_WILL_FREE), inode); + return inode; + } + + spin_lock(&inode->i_lock); + if (!(inode_state_read(inode) & (I_FREEING | I_WILL_FREE))) { + __iget(inode); + spin_unlock(&inode->i_lock); + } else { + spin_unlock(&inode->i_lock); + /* + * Handle the case where s_op->clear_inode is not been + * called yet, and somebody is calling igrab + * while the inode is getting freed. + */ + inode = NULL; + } + return inode; +} +EXPORT_SYMBOL(igrab); + +/* + * igrab_from_hash - special inode refcount acquire primitive for the inode hash + * + * It provides lockless refcount acquire in the common case of no problematic + * flags being set and the count being > 0. + * + * There are 4 state flags to worry about and the routine makes sure to not bump the + * ref if any of them is present. + * + * I_NEW and I_CREATING can only legally get set *before* the inode becomes visible + * during lookup. Thus if the flags are not spotted, they are guaranteed to not be + * a factor. However, we need an acquire fence before returning the inode just + * in case we raced against clearing the state to make sure our consumer picks up + * any other changes made prior. atomic_add_unless provides a full fence, which + * takes care of it. + * + * I_FREEING and I_WILL_FREE can only legally get set if ->i_count == 0 and it is + * illegal to bump the ref if either is present. Consequently if atomic_add_unless + * managed to replace a non-0 value with a bigger one, we have a guarantee neither + * of these flags is set. Note this means explicitly checking of these flags below + * is not necessary, it is only done because it does not cost anything on top of the + * load which already needs to be done to handle the other flags. + */ +static bool igrab_from_hash(struct inode *inode) +{ + if (inode_state_read_once(inode) & (I_NEW | I_CREATING | I_FREEING | I_WILL_FREE)) + return false; + /* + * Paired with routines clearing I_NEW + */ + if (atomic_add_unless(&inode->i_count, 1, 0)) { + VFS_BUG_ON_INODE(inode_state_read_once(inode) & (I_FREEING | I_WILL_FREE), inode); + return true; + } + return false; +} + +/** + * ilookup5_nowait - search for an inode in the inode cache + * @sb: super block of file system to search + * @hashval: hash value (usually inode number) to search for + * @test: callback used for comparisons between inodes + * @data: opaque data pointer to pass to @test + * @isnew: return argument telling whether I_NEW was set when + * the inode was found in hash (the caller needs to + * wait for I_NEW to clear) + * + * Search for the inode specified by @hashval and @data in the inode cache. + * If the inode is in the cache, the inode is returned with an incremented + * reference count. + * + * Note: I_NEW is not waited upon so you have to be very careful what you do + * with the returned inode. You probably should be using ilookup5() instead. + * + * Note2: @test is called with the inode_hash_lock held, so can't sleep. + */ +struct inode *ilookup5_nowait(struct super_block *sb, u64 hashval, + int (*test)(struct inode *, void *), void *data, bool *isnew) +{ + struct hlist_head *head = inode_hashtable + hash(sb, hashval); + struct inode *inode; + + spin_lock(&inode_hash_lock); + inode = find_inode(sb, head, test, data, true, isnew); + spin_unlock(&inode_hash_lock); + + return IS_ERR(inode) ? NULL : inode; +} +EXPORT_SYMBOL(ilookup5_nowait); + +/** + * ilookup5 - search for an inode in the inode cache + * @sb: super block of file system to search + * @hashval: hash value (usually inode number) to search for + * @test: callback used for comparisons between inodes + * @data: opaque data pointer to pass to @test + * + * Search for the inode specified by @hashval and @data in the inode cache, + * and if the inode is in the cache, return the inode with an incremented + * reference count. Waits on I_NEW before returning the inode. + * returned with an incremented reference count. + * + * This is a generalized version of ilookup() for file systems where the + * inode number is not sufficient for unique identification of an inode. + * + * Note: @test is called with the inode_hash_lock held, so can't sleep. + */ +struct inode *ilookup5(struct super_block *sb, u64 hashval, + int (*test)(struct inode *, void *), void *data) +{ + struct inode *inode; + bool isnew; + + might_sleep(); + +again: + inode = ilookup5_nowait(sb, hashval, test, data, &isnew); + if (inode) { + if (unlikely(isnew)) + wait_on_new_inode(inode); + if (unlikely(inode_unhashed(inode))) { + iput(inode); + goto again; + } + } + return inode; +} +EXPORT_SYMBOL(ilookup5); + +/** + * ilookup - search for an inode in the inode cache + * @sb: super block of file system to search + * @ino: inode number to search for + * + * Search for the inode @ino in the inode cache, and if the inode is in the + * cache, the inode is returned with an incremented reference count. + */ +struct inode *ilookup(struct super_block *sb, u64 ino) +{ + struct hlist_head *head = inode_hashtable + hash(sb, ino); + struct inode *inode; + bool isnew; + + might_sleep(); + +again: + inode = find_inode_fast(sb, head, ino, false, &isnew); + + if (inode) { + if (IS_ERR(inode)) + return NULL; + if (unlikely(isnew)) + wait_on_new_inode(inode); + if (unlikely(inode_unhashed(inode))) { + iput(inode); + goto again; + } + } + return inode; +} +EXPORT_SYMBOL(ilookup); + +/** + * find_inode_nowait - find an inode in the inode cache + * @sb: super block of file system to search + * @hashval: hash value (usually inode number) to search for + * @match: callback used for comparisons between inodes + * @data: opaque data pointer to pass to @match + * + * Search for the inode specified by @hashval and @data in the inode + * cache, where the helper function @match will return 0 if the inode + * does not match, 1 if the inode does match, and -1 if the search + * should be stopped. The @match function must be responsible for + * taking the i_lock spin_lock and checking i_state for an inode being + * freed or being initialized, and incrementing the reference count + * before returning 1. It also must not sleep, since it is called with + * the inode_hash_lock spinlock held. + * + * This is a even more generalized version of ilookup5() when the + * function must never block --- find_inode() can block in + * __wait_on_freeing_inode() --- or when the caller can not increment + * the reference count because the resulting iput() might cause an + * inode eviction. The tradeoff is that the @match funtion must be + * very carefully implemented. + */ +struct inode *find_inode_nowait(struct super_block *sb, + u64 hashval, + int (*match)(struct inode *, u64, + void *), + void *data) +{ + struct hlist_head *head = inode_hashtable + hash(sb, hashval); + struct inode *inode, *ret_inode = NULL; + int mval; + + spin_lock(&inode_hash_lock); + hlist_for_each_entry(inode, head, i_hash) { + if (inode->i_sb != sb) + continue; + mval = match(inode, hashval, data); + if (mval == 0) + continue; + if (mval == 1) + ret_inode = inode; + goto out; + } +out: + spin_unlock(&inode_hash_lock); + return ret_inode; +} +EXPORT_SYMBOL(find_inode_nowait); + +/** + * find_inode_rcu - find an inode in the inode cache + * @sb: Super block of file system to search + * @hashval: Key to hash + * @test: Function to test match on an inode + * @data: Data for test function + * + * Search for the inode specified by @hashval and @data in the inode cache, + * where the helper function @test will return 0 if the inode does not match + * and 1 if it does. The @test function must be responsible for taking the + * i_lock spin_lock and checking i_state for an inode being freed or being + * initialized. + * + * If successful, this will return the inode for which the @test function + * returned 1 and NULL otherwise. + * + * The @test function is not permitted to take a ref on any inode presented. + * It is also not permitted to sleep. + * + * The caller must hold the RCU read lock. + */ +struct inode *find_inode_rcu(struct super_block *sb, u64 hashval, + int (*test)(struct inode *, void *), void *data) +{ + struct hlist_head *head = inode_hashtable + hash(sb, hashval); + struct inode *inode; + + RCU_LOCKDEP_WARN(!rcu_read_lock_held(), + "suspicious find_inode_rcu() usage"); + + hlist_for_each_entry_rcu(inode, head, i_hash) { + if (inode->i_sb == sb && + !(inode_state_read_once(inode) & (I_FREEING | I_WILL_FREE)) && + test(inode, data)) + return inode; + } + return NULL; +} +EXPORT_SYMBOL(find_inode_rcu); + +/** + * find_inode_by_ino_rcu - Find an inode in the inode cache + * @sb: Super block of file system to search + * @ino: The inode number to match + * + * Search for the inode specified by @hashval and @data in the inode cache, + * where the helper function @test will return 0 if the inode does not match + * and 1 if it does. The @test function must be responsible for taking the + * i_lock spin_lock and checking i_state for an inode being freed or being + * initialized. + * + * If successful, this will return the inode for which the @test function + * returned 1 and NULL otherwise. + * + * The @test function is not permitted to take a ref on any inode presented. + * It is also not permitted to sleep. + * + * The caller must hold the RCU read lock. + */ +struct inode *find_inode_by_ino_rcu(struct super_block *sb, + u64 ino) +{ + struct hlist_head *head = inode_hashtable + hash(sb, ino); + struct inode *inode; + + RCU_LOCKDEP_WARN(!rcu_read_lock_held(), + "suspicious find_inode_by_ino_rcu() usage"); + + hlist_for_each_entry_rcu(inode, head, i_hash) { + if (inode->i_ino == ino && + inode->i_sb == sb && + !(inode_state_read_once(inode) & (I_FREEING | I_WILL_FREE))) + return inode; + } + return NULL; +} +EXPORT_SYMBOL(find_inode_by_ino_rcu); + +int insert_inode_locked(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + u64 ino = inode->i_ino; + struct hlist_head *head = inode_hashtable + hash(sb, ino); + bool isnew; + + might_sleep(); + + while (1) { + struct inode *old = NULL; + spin_lock(&inode_hash_lock); +repeat: + hlist_for_each_entry(old, head, i_hash) { + if (old->i_ino != ino) + continue; + if (old->i_sb != sb) + continue; + spin_lock(&old->i_lock); + break; + } + if (likely(!old)) { + spin_lock(&inode->i_lock); + inode_state_set(inode, I_NEW | I_CREATING); + hlist_add_head_rcu(&inode->i_hash, head); + spin_unlock(&inode->i_lock); + spin_unlock(&inode_hash_lock); + return 0; + } + if (inode_state_read(old) & (I_FREEING | I_WILL_FREE)) { + __wait_on_freeing_inode(old, true, false); + old = NULL; + goto repeat; + } + if (unlikely(inode_state_read(old) & I_CREATING)) { + spin_unlock(&old->i_lock); + spin_unlock(&inode_hash_lock); + return -EBUSY; + } + __iget(old); + isnew = !!(inode_state_read(old) & I_NEW); + spin_unlock(&old->i_lock); + spin_unlock(&inode_hash_lock); + if (isnew) + wait_on_new_inode(old); + if (unlikely(!inode_unhashed(old))) { + iput(old); + return -EBUSY; + } + iput(old); + } +} +EXPORT_SYMBOL(insert_inode_locked); + +int insert_inode_locked4(struct inode *inode, u64 hashval, + int (*test)(struct inode *, void *), void *data) +{ + struct inode *old; + + might_sleep(); + + inode_state_set_raw(inode, I_CREATING); + old = inode_insert5(inode, hashval, test, NULL, data); + + if (old != inode) { + iput(old); + return -EBUSY; + } + return 0; +} +EXPORT_SYMBOL(insert_inode_locked4); + + +int inode_just_drop(struct inode *inode) +{ + return 1; +} +EXPORT_SYMBOL(inode_just_drop); + +/* + * Called when we're dropping the last reference + * to an inode. + * + * Call the FS "drop_inode()" function, defaulting to + * the legacy UNIX filesystem behaviour. If it tells + * us to evict inode, do so. Otherwise, retain inode + * in cache if fs is alive, sync and evict if fs is + * shutting down. + */ +static void iput_final(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + const struct super_operations *op = inode->i_sb->s_op; + int drop; + + WARN_ON(inode_state_read(inode) & I_NEW); + VFS_BUG_ON_INODE(icount_read(inode) != 0, inode); + + if (op->drop_inode) + drop = op->drop_inode(inode); + else + drop = inode_generic_drop(inode); + + if (!drop && + !(inode_state_read(inode) & I_DONTCACHE) && + (sb->s_flags & SB_ACTIVE)) { + __inode_lru_list_add(inode, true); + spin_unlock(&inode->i_lock); + return; + } + + /* + * Re-check ->i_count in case the ->drop_inode() hooks played games. + * Note we only execute this if the verdict was to drop the inode. + */ + VFS_BUG_ON_INODE(icount_read(inode) != 0, inode); + + if (drop) { + inode_state_set(inode, I_FREEING); + } else { + inode_state_set(inode, I_WILL_FREE); + spin_unlock(&inode->i_lock); + + write_inode_now(inode, 1); + + spin_lock(&inode->i_lock); + WARN_ON(inode_state_read(inode) & I_NEW); + inode_state_replace(inode, I_WILL_FREE, I_FREEING); + } + + inode_lru_list_del(inode); + spin_unlock(&inode->i_lock); + + evict(inode); +} + +/** + * iput - put an inode + * @inode: inode to put + * + * Puts an inode, dropping its usage count. If the inode use count hits + * zero, the inode is then freed and may also be destroyed. + * + * Consequently, iput() can sleep. + */ +void iput(struct inode *inode) +{ + might_sleep(); + if (unlikely(!inode)) + return; + +retry: + lockdep_assert_not_held(&inode->i_lock); + VFS_BUG_ON_INODE(inode_state_read_once(inode) & (I_FREEING | I_CLEAR), inode); + /* + * Note this assert is technically racy as if the count is bogusly + * equal to one, then two CPUs racing to further drop it can both + * conclude it's fine. + */ + VFS_BUG_ON_INODE(icount_read_once(inode) < 1, inode); + + if (atomic_add_unless(&inode->i_count, -1, 1)) + return; + + if (inode->i_nlink && sync_lazytime(inode)) + goto retry; + + spin_lock(&inode->i_lock); + if (unlikely((inode_state_read(inode) & I_DIRTY_TIME) && inode->i_nlink)) { + spin_unlock(&inode->i_lock); + goto retry; + } + + if (!atomic_dec_and_test(&inode->i_count)) { + spin_unlock(&inode->i_lock); + return; + } + + /* + * iput_final() drops ->i_lock, we can't assert on it as the inode may + * be deallocated by the time the call returns. + */ + iput_final(inode); +} +EXPORT_SYMBOL(iput); + +/** + * iput_not_last - put an inode assuming this is not the last reference + * @inode: inode to put + */ +void iput_not_last(struct inode *inode) +{ + VFS_BUG_ON_INODE(inode_state_read_once(inode) & (I_FREEING | I_CLEAR), inode); + VFS_BUG_ON_INODE(icount_read_once(inode) < 2, inode); + + WARN_ON(atomic_sub_return(1, &inode->i_count) == 0); +} +EXPORT_SYMBOL(iput_not_last); + +#ifdef CONFIG_BLOCK +/** + * bmap - find a block number in a file + * @inode: inode owning the block number being requested + * @block: pointer containing the block to find + * + * Replaces the value in ``*block`` with the block number on the device holding + * corresponding to the requested block number in the file. + * That is, asked for block 4 of inode 1 the function will replace the + * 4 in ``*block``, with disk block relative to the disk start that holds that + * block of the file. + * + * Returns -EINVAL in case of error, 0 otherwise. If mapping falls into a + * hole, returns 0 and ``*block`` is also set to 0. + */ +int bmap(struct inode *inode, sector_t *block) +{ + if (!inode->i_mapping->a_ops->bmap) + return -EINVAL; + + *block = inode->i_mapping->a_ops->bmap(inode->i_mapping, *block); + return 0; +} +EXPORT_SYMBOL(bmap); +#endif + +/* + * With relative atime, only update atime if the previous atime is + * earlier than or equal to either the ctime or mtime, + * or if at least a day has passed since the last atime update. + */ +static bool relatime_need_update(struct vfsmount *mnt, struct inode *inode, + struct timespec64 now) +{ + struct timespec64 atime, mtime, ctime; + + if (!(mnt->mnt_flags & MNT_RELATIME)) + return true; + /* + * Is mtime younger than or equal to atime? If yes, update atime: + */ + atime = inode_get_atime(inode); + mtime = inode_get_mtime(inode); + if (timespec64_compare(&mtime, &atime) >= 0) + return true; + /* + * Is ctime younger than or equal to atime? If yes, update atime: + */ + ctime = inode_get_ctime(inode); + if (timespec64_compare(&ctime, &atime) >= 0) + return true; + + /* + * Is the previous atime value older than a day? If yes, + * update atime: + */ + if ((long)(now.tv_sec - atime.tv_sec) >= 24*60*60) + return true; + /* + * Good, we can skip the atime update: + */ + return false; +} + +static int inode_update_atime(struct inode *inode) +{ + struct timespec64 atime = inode_get_atime(inode); + struct timespec64 now = current_time(inode); + + if (timespec64_equal(&now, &atime)) + return 0; + + inode_set_atime_to_ts(inode, now); + return inode_time_dirty_flag(inode); +} + +static int inode_update_cmtime(struct inode *inode, unsigned int flags) +{ + struct timespec64 ctime = inode_get_ctime(inode); + struct timespec64 mtime = inode_get_mtime(inode); + struct timespec64 now = inode_set_ctime_current(inode); + unsigned int dirty = 0; + bool mtime_changed; + + mtime_changed = !timespec64_equal(&now, &mtime); + if (mtime_changed || !timespec64_equal(&now, &ctime)) + dirty = inode_time_dirty_flag(inode); + + /* + * Pure timestamp updates can be recorded in the inode without blocking + * by not dirtying the inode. But when the file system requires + * i_version updates, the update of i_version can still block. + * Error out if we'd actually have to update i_version or don't support + * lazytime. + */ + if (IS_I_VERSION(inode)) { + if (flags & IOCB_NOWAIT) { + if (!(inode->i_sb->s_flags & SB_LAZYTIME) || + inode_iversion_need_inc(inode)) + return -EAGAIN; + } else { + /* + * Don't force iversion increment for pure lazytime + * updates (I_DIRTY_TIME only), let I_VERSION_QUERIED + * dictate whether the increment is needed. + */ + if (inode_maybe_inc_iversion(inode, + dirty != I_DIRTY_TIME)) + dirty |= I_DIRTY_SYNC; + } + } + + if (mtime_changed) + inode_set_mtime_to_ts(inode, now); + return dirty; +} + +/** + * inode_update_time - update either atime or c/mtime and i_version on the inode + * @inode: inode to be updated + * @type: timestamp to be updated + * @flags: flags for the update + * + * Update either atime or c/mtime and version in a inode if needed for a file + * access or modification. It is up to the caller to mark the inode dirty + * appropriately. + * + * Returns the positive I_DIRTY_* flags for __mark_inode_dirty() if the inode + * needs to be marked dirty, 0 if it did not, or a negative errno if an error + * happened. + */ +int inode_update_time(struct inode *inode, enum fs_update_time type, + unsigned int flags) +{ + switch (type) { + case FS_UPD_ATIME: + return inode_update_atime(inode); + case FS_UPD_CMTIME: + return inode_update_cmtime(inode, flags); + default: + WARN_ON_ONCE(1); + return -EIO; + } +} +EXPORT_SYMBOL(inode_update_time); + +/** + * generic_update_time - update the timestamps on the inode + * @inode: inode to be updated + * @type: timestamp to be updated + * @flags: flags for the update + * + * Returns a negative error value on error, else 0. + */ +int generic_update_time(struct inode *inode, enum fs_update_time type, + unsigned int flags) +{ + int dirty; + + /* + * ->dirty_inode is what could make generic timestamp updates block. + * Don't support non-blocking timestamp updates here if it is set. + * File systems that implement ->dirty_inode but want to support + * non-blocking timestamp updates should call inode_update_time + * directly. + */ + if ((flags & IOCB_NOWAIT) && inode->i_sb->s_op->dirty_inode) + return -EAGAIN; + + dirty = inode_update_time(inode, type, flags); + if (dirty <= 0) + return dirty; + __mark_inode_dirty(inode, dirty); + return 0; +} +EXPORT_SYMBOL(generic_update_time); + +/** + * atime_needs_update - update the access time + * @path: the &struct path to update + * @inode: inode to update + * + * Update the accessed time on an inode and mark it for writeback. + * This function automatically handles read only file systems and media, + * as well as the "noatime" flag and inode specific "noatime" markers. + */ +bool atime_needs_update(const struct path *path, struct inode *inode) +{ + struct vfsmount *mnt = path->mnt; + struct timespec64 now, atime; + + if (inode->i_flags & S_NOATIME) + return false; + + /* Atime updates will likely cause i_uid and i_gid to be written + * back improprely if their true value is unknown to the vfs. + */ + if (HAS_UNMAPPED_ID(mnt_idmap(mnt), inode)) + return false; + + if (IS_NOATIME(inode)) + return false; + if ((inode->i_sb->s_flags & SB_NODIRATIME) && S_ISDIR(inode->i_mode)) + return false; + + if (mnt->mnt_flags & MNT_NOATIME) + return false; + if ((mnt->mnt_flags & MNT_NODIRATIME) && S_ISDIR(inode->i_mode)) + return false; + + now = current_time(inode); + + if (!relatime_need_update(mnt, inode, now)) + return false; + + atime = inode_get_atime(inode); + if (timespec64_equal(&atime, &now)) + return false; + + return true; +} + +void touch_atime(const struct path *path) +{ + struct vfsmount *mnt = path->mnt; + struct inode *inode = d_inode(path->dentry); + + if (!atime_needs_update(path, inode)) + return; + + if (!sb_start_write_trylock(inode->i_sb)) + return; + + if (mnt_get_write_access(mnt) != 0) + goto skip_update; + /* + * File systems can error out when updating inodes if they need to + * allocate new space to modify an inode (such is the case for + * Btrfs), but since we touch atime while walking down the path we + * really don't care if we failed to update the atime of the file, + * so just ignore the return value. + * We may also fail on filesystems that have the ability to make parts + * of the fs read only, e.g. subvolumes in Btrfs. + */ + if (inode->i_op->update_time) + inode->i_op->update_time(inode, FS_UPD_ATIME, 0); + else + generic_update_time(inode, FS_UPD_ATIME, 0); + mnt_put_write_access(mnt); +skip_update: + sb_end_write(inode->i_sb); +} +EXPORT_SYMBOL(touch_atime); + +/* + * Return mask of changes for notify_change() that need to be done as a + * response to write or truncate. Return 0 if nothing has to be changed. + * Negative value on error (change should be denied). + */ +int dentry_needs_remove_privs(struct mnt_idmap *idmap, + struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + int mask = 0; + int ret; + + if (IS_NOSEC(inode)) + return 0; + + mask = setattr_should_drop_suidgid(idmap, inode); + ret = security_inode_need_killpriv(dentry); + if (ret < 0) + return ret; + if (ret) + mask |= ATTR_KILL_PRIV; + return mask; +} + +static int __remove_privs(struct mnt_idmap *idmap, + struct dentry *dentry, int kill) +{ + struct iattr newattrs; + + newattrs.ia_valid = ATTR_FORCE | kill; + /* + * Note we call this on write, so notify_change will not + * encounter any conflicting delegations: + */ + return notify_change(idmap, dentry, &newattrs, NULL); +} + +static int file_remove_privs_flags(struct file *file, unsigned int flags) +{ + struct dentry *dentry = file_dentry(file); + struct inode *inode = file_inode(file); + int error = 0; + int kill; + + if (IS_NOSEC(inode) || !S_ISREG(inode->i_mode)) + return 0; + + kill = dentry_needs_remove_privs(file_mnt_idmap(file), dentry); + if (kill < 0) + return kill; + + if (kill) { + if (flags & IOCB_NOWAIT) + return -EAGAIN; + + error = __remove_privs(file_mnt_idmap(file), dentry, kill); + } + + if (!error) + inode_has_no_xattr(inode); + return error; +} + +/** + * file_remove_privs - remove special file privileges (suid, capabilities) + * @file: file to remove privileges from + * + * When file is modified by a write or truncation ensure that special + * file privileges are removed. + * + * Return: 0 on success, negative errno on failure. + */ +int file_remove_privs(struct file *file) +{ + return file_remove_privs_flags(file, 0); +} +EXPORT_SYMBOL(file_remove_privs); + +/** + * current_time - Return FS time (possibly fine-grained) + * @inode: inode. + * + * Return the current time truncated to the time granularity supported by + * the fs, as suitable for a ctime/mtime change. If the ctime is flagged + * as having been QUERIED, get a fine-grained timestamp, but don't update + * the floor. + * + * For a multigrain inode, this is effectively an estimate of the timestamp + * that a file would receive. An actual update must go through + * inode_set_ctime_current(). + */ +struct timespec64 current_time(struct inode *inode) +{ + struct timespec64 now; + u32 cns; + + ktime_get_coarse_real_ts64_mg(&now); + + if (!is_mgtime(inode)) + goto out; + + /* If nothing has queried it, then coarse time is fine */ + cns = smp_load_acquire(&inode->i_ctime_nsec); + if (cns & I_CTIME_QUERIED) { + /* + * If there is no apparent change, then get a fine-grained + * timestamp. + */ + if (now.tv_nsec == (cns & ~I_CTIME_QUERIED)) + ktime_get_real_ts64(&now); + } +out: + return timestamp_truncate(now, inode); +} +EXPORT_SYMBOL(current_time); + +static inline bool need_cmtime_update(struct inode *inode) +{ + struct timespec64 now = current_time(inode), ts; + + ts = inode_get_mtime(inode); + if (!timespec64_equal(&ts, &now)) + return true; + ts = inode_get_ctime(inode); + if (!timespec64_equal(&ts, &now)) + return true; + return IS_I_VERSION(inode) && inode_iversion_need_inc(inode); +} + +static int file_update_time_flags(struct file *file, unsigned int flags) +{ + struct inode *inode = file_inode(file); + int ret; + + /* First try to exhaust all avenues to not sync */ + if (IS_NOCMTIME(inode)) + return 0; + if (unlikely(file->f_mode & FMODE_NOCMTIME)) + return 0; + if (!need_cmtime_update(inode)) + return 0; + + flags &= IOCB_NOWAIT; + if (mnt_get_write_access_file(file)) + return 0; + if (inode->i_op->update_time) + ret = inode->i_op->update_time(inode, FS_UPD_CMTIME, flags); + else + ret = generic_update_time(inode, FS_UPD_CMTIME, flags); + mnt_put_write_access_file(file); + return ret; +} + +/** + * file_update_time - update mtime and ctime time + * @file: file accessed + * + * Update the mtime and ctime members of an inode and mark the inode for + * writeback. Note that this function is meant exclusively for usage in + * the file write path of filesystems, and filesystems may choose to + * explicitly ignore updates via this function with the _NOCMTIME inode + * flag, e.g. for network filesystem where these imestamps are handled + * by the server. This can return an error for file systems who need to + * allocate space in order to update an inode. + * + * Return: 0 on success, negative errno on failure. + */ +int file_update_time(struct file *file) +{ + return file_update_time_flags(file, 0); +} +EXPORT_SYMBOL(file_update_time); + +/** + * file_modified_flags - handle mandated vfs changes when modifying a file + * @file: file that was modified + * @flags: kiocb flags + * + * When file has been modified ensure that special + * file privileges are removed and time settings are updated. + * + * If IOCB_NOWAIT is set, special file privileges will not be removed and + * time settings will not be updated. It will return -EAGAIN. + * + * Context: Caller must hold the file's inode lock. + * + * Return: 0 on success, negative errno on failure. + */ +static int file_modified_flags(struct file *file, int flags) +{ + int ret; + + /* + * Clear the security bits if the process is not being run by root. + * This keeps people from modifying setuid and setgid binaries. + */ + ret = file_remove_privs_flags(file, flags); + if (ret) + return ret; + return file_update_time_flags(file, flags); +} + +/** + * file_modified - handle mandated vfs changes when modifying a file + * @file: file that was modified + * + * When file has been modified ensure that special + * file privileges are removed and time settings are updated. + * + * Context: Caller must hold the file's inode lock. + * + * Return: 0 on success, negative errno on failure. + */ +int file_modified(struct file *file) +{ + return file_modified_flags(file, 0); +} +EXPORT_SYMBOL(file_modified); + +/** + * kiocb_modified - handle mandated vfs changes when modifying a file + * @iocb: iocb that was modified + * + * When file has been modified ensure that special + * file privileges are removed and time settings are updated. + * + * Context: Caller must hold the file's inode lock. + * + * Return: 0 on success, negative errno on failure. + */ +int kiocb_modified(struct kiocb *iocb) +{ + return file_modified_flags(iocb->ki_filp, iocb->ki_flags); +} +EXPORT_SYMBOL_GPL(kiocb_modified); + +int inode_needs_sync(struct inode *inode) +{ + if (IS_SYNC(inode)) + return 1; + if (S_ISDIR(inode->i_mode) && IS_DIRSYNC(inode)) + return 1; + return 0; +} +EXPORT_SYMBOL(inode_needs_sync); + +/* + * If we try to find an inode in the inode hash while it is being + * deleted, we have to wait until the filesystem completes its + * deletion before reporting that it isn't found. This function waits + * until the deletion _might_ have completed. Callers are responsible + * to recheck inode state. + * + * It doesn't matter if I_NEW is not set initially, a call to + * wake_up_bit(&inode->i_state, __I_NEW) after removing from the hash list + * will DTRT. + */ +static void __wait_on_freeing_inode(struct inode *inode, bool hash_locked, bool rcu_locked) +{ + struct wait_bit_queue_entry wqe; + struct wait_queue_head *wq_head; + + VFS_BUG_ON(!hash_locked && !rcu_locked); + + /* + * Handle racing against evict(), see that routine for more details. + */ + if (unlikely(inode_unhashed(inode))) { + WARN_ON(hash_locked); + spin_unlock(&inode->i_lock); + return; + } + + wq_head = inode_bit_waitqueue(&wqe, inode, __I_NEW); + prepare_to_wait_event(wq_head, &wqe.wq_entry, TASK_UNINTERRUPTIBLE); + spin_unlock(&inode->i_lock); + if (rcu_locked) + rcu_read_unlock(); + if (hash_locked) + spin_unlock(&inode_hash_lock); + schedule(); + finish_wait(wq_head, &wqe.wq_entry); + if (hash_locked) + spin_lock(&inode_hash_lock); + if (rcu_locked) + rcu_read_lock(); +} + +static __initdata unsigned long ihash_entries; +static int __init set_ihash_entries(char *str) +{ + return kstrtoul(str, 0, &ihash_entries) == 0; +} +__setup("ihash_entries=", set_ihash_entries); + +/* + * Initialize the waitqueues and inode hash table. + */ +void __init inode_init_early(void) +{ + /* If hashes are distributed across NUMA nodes, defer + * hash allocation until vmalloc space is available. + */ + if (hashdist) + return; + + inode_hashtable = + alloc_large_system_hash("Inode-cache", + sizeof(struct hlist_head), + ihash_entries, + 14, + HASH_EARLY | HASH_ZERO, + &i_hash_shift, + &i_hash_mask, + 0, + 0); +} + +void __init inode_init(void) +{ + /* inode slab cache */ + inode_cachep = kmem_cache_create("inode_cache", + sizeof(struct inode), + 0, + (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC| + SLAB_ACCOUNT), + init_once); + + /* Hash may have been set up in inode_init_early */ + if (!hashdist) + return; + + inode_hashtable = + alloc_large_system_hash("Inode-cache", + sizeof(struct hlist_head), + ihash_entries, + 14, + HASH_ZERO, + &i_hash_shift, + &i_hash_mask, + 0, + 0); +} + +void init_special_inode(struct inode *inode, umode_t mode, dev_t rdev) +{ + inode->i_mode = mode; + switch (inode->i_mode & S_IFMT) { + case S_IFCHR: + inode->i_fop = &def_chr_fops; + inode->i_rdev = rdev; + break; + case S_IFBLK: + if (IS_ENABLED(CONFIG_BLOCK)) + inode->i_fop = &def_blk_fops; + inode->i_rdev = rdev; + break; + case S_IFIFO: + inode->i_fop = &pipefifo_fops; + break; + case S_IFSOCK: + /* leave it no_open_fops */ + break; + default: + pr_debug("init_special_inode: bogus i_mode (%o) for inode %s:%llu\n", + mode, inode->i_sb->s_id, inode->i_ino); + break; + } +} +EXPORT_SYMBOL(init_special_inode); + +/** + * inode_init_owner - Init uid,gid,mode for new inode according to posix standards + * @idmap: idmap of the mount the inode was created from + * @inode: New inode + * @dir: Directory inode + * @mode: mode of the new inode + * + * If the inode has been created through an idmapped mount the idmap of + * the vfsmount must be passed through @idmap. This function will then take + * care to map the inode according to @idmap before checking permissions + * and initializing i_uid and i_gid. On non-idmapped mounts or if permission + * checking is to be performed on the raw inode simply pass @nop_mnt_idmap. + */ +void inode_init_owner(struct mnt_idmap *idmap, struct inode *inode, + const struct inode *dir, umode_t mode) +{ + inode_fsuid_set(inode, idmap); + if (dir && dir->i_mode & S_ISGID) { + inode->i_gid = dir->i_gid; + + /* Directories are special, and always inherit S_ISGID */ + if (S_ISDIR(mode)) + mode |= S_ISGID; + } else + inode_fsgid_set(inode, idmap); + inode->i_mode = mode; +} +EXPORT_SYMBOL(inode_init_owner); + +/** + * inode_owner_or_capable - check current task permissions to inode + * @idmap: idmap of the mount the inode was found from + * @inode: inode being checked + * + * Return true if current either has CAP_FOWNER in a namespace with the + * inode owner uid mapped, or owns the file. + * + * If the inode has been found through an idmapped mount the idmap of + * the vfsmount must be passed through @idmap. This function will then take + * care to map the inode according to @idmap before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply pass @nop_mnt_idmap. + */ +bool inode_owner_or_capable(struct mnt_idmap *idmap, + const struct inode *inode) +{ + vfsuid_t vfsuid; + struct user_namespace *ns; + + vfsuid = i_uid_into_vfsuid(idmap, inode); + if (vfsuid_eq_kuid(vfsuid, current_fsuid())) + return true; + + ns = current_user_ns(); + if (vfsuid_has_mapping(ns, vfsuid) && ns_capable(ns, CAP_FOWNER)) + return true; + return false; +} +EXPORT_SYMBOL(inode_owner_or_capable); + +/* + * Direct i/o helper functions + */ +bool inode_dio_finished(const struct inode *inode) +{ + return atomic_read(&inode->i_dio_count) == 0; +} +EXPORT_SYMBOL(inode_dio_finished); + +/** + * inode_dio_wait - wait for outstanding DIO requests to finish + * @inode: inode to wait for + * + * Waits for all pending direct I/O requests to finish so that we can + * proceed with a truncate or equivalent operation. + * + * Must be called under a lock that serializes taking new references + * to i_dio_count, usually by inode->i_rwsem. + */ +void inode_dio_wait(struct inode *inode) +{ + wait_var_event(&inode->i_dio_count, inode_dio_finished(inode)); +} +EXPORT_SYMBOL(inode_dio_wait); + +void inode_dio_wait_interruptible(struct inode *inode) +{ + wait_var_event_interruptible(&inode->i_dio_count, + inode_dio_finished(inode)); +} +EXPORT_SYMBOL(inode_dio_wait_interruptible); + +/* + * inode_set_flags - atomically set some inode flags + * + * Note: the caller should be holding i_rwsem exclusively, or else be sure that + * they have exclusive access to the inode structure (i.e., while the + * inode is being instantiated). The reason for the cmpxchg() loop + * --- which wouldn't be necessary if all code paths which modify + * i_flags actually followed this rule, is that there is at least one + * code path which doesn't today so we use cmpxchg() out of an abundance + * of caution. + * + * In the long run, i_rwsem is overkill, and we should probably look + * at using the i_lock spinlock to protect i_flags, and then make sure + * it is so documented in include/linux/fs.h and that all code follows + * the locking convention!! + */ +void inode_set_flags(struct inode *inode, unsigned int flags, + unsigned int mask) +{ + WARN_ON_ONCE(flags & ~mask); + set_mask_bits(&inode->i_flags, mask, flags); +} +EXPORT_SYMBOL(inode_set_flags); + +void inode_nohighmem(struct inode *inode) +{ + mapping_set_gfp_mask(inode->i_mapping, GFP_USER); +} +EXPORT_SYMBOL(inode_nohighmem); + +struct timespec64 inode_set_ctime_to_ts(struct inode *inode, struct timespec64 ts) +{ + trace_inode_set_ctime_to_ts(inode, &ts); + set_normalized_timespec64(&ts, ts.tv_sec, ts.tv_nsec); + WRITE_ONCE(inode->i_ctime_sec, ts.tv_sec); + WRITE_ONCE(inode->i_ctime_nsec, ts.tv_nsec); + return ts; +} +EXPORT_SYMBOL(inode_set_ctime_to_ts); + +/** + * timestamp_truncate - Truncate timespec to a granularity + * @t: Timespec + * @inode: inode being updated + * + * Truncate a timespec to the granularity supported by the fs + * containing the inode. Always rounds down. gran must + * not be 0 nor greater than a second (NSEC_PER_SEC, or 10^9 ns). + */ +struct timespec64 timestamp_truncate(struct timespec64 t, struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + unsigned int gran = sb->s_time_gran; + + t.tv_sec = clamp(t.tv_sec, sb->s_time_min, sb->s_time_max); + if (unlikely(t.tv_sec == sb->s_time_max || t.tv_sec == sb->s_time_min)) + t.tv_nsec = 0; + + /* Avoid division in the common cases 1 ns and 1 s. */ + if (gran == 1) + ; /* nothing */ + else if (gran == NSEC_PER_SEC) + t.tv_nsec = 0; + else if (gran > 1 && gran < NSEC_PER_SEC) + t.tv_nsec -= t.tv_nsec % gran; + else + WARN(1, "invalid file time granularity: %u", gran); + return t; +} +EXPORT_SYMBOL(timestamp_truncate); + +/** + * inode_set_ctime_current - set the ctime to current_time + * @inode: inode + * + * Set the inode's ctime to the current value for the inode. Returns the + * current value that was assigned. If this is not a multigrain inode, then we + * set it to the later of the coarse time and floor value. + * + * If it is multigrain, then we first see if the coarse-grained timestamp is + * distinct from what is already there. If so, then use that. Otherwise, get a + * fine-grained timestamp. + * + * After that, try to swap the new value into i_ctime_nsec. Accept the + * resulting ctime, regardless of the outcome of the swap. If it has + * already been replaced, then that timestamp is later than the earlier + * unacceptable one, and is thus acceptable. + */ +struct timespec64 inode_set_ctime_current(struct inode *inode) +{ + struct timespec64 now; + u32 cns, cur; + + ktime_get_coarse_real_ts64_mg(&now); + now = timestamp_truncate(now, inode); + + /* Just return that if this is not a multigrain fs */ + if (!is_mgtime(inode)) { + inode_set_ctime_to_ts(inode, now); + goto out; + } + + /* + * A fine-grained time is only needed if someone has queried + * for timestamps, and the current coarse grained time isn't + * later than what's already there. + */ + cns = smp_load_acquire(&inode->i_ctime_nsec); + if (cns & I_CTIME_QUERIED) { + struct timespec64 ctime = { .tv_sec = inode_get_ctime_sec(inode), + .tv_nsec = cns & ~I_CTIME_QUERIED }; + + if (timespec64_compare(&now, &ctime) <= 0) { + ktime_get_real_ts64_mg(&now); + now = timestamp_truncate(now, inode); + mgtime_counter_inc(mg_fine_stamps); + } + } + mgtime_counter_inc(mg_ctime_updates); + + /* No need to cmpxchg if it's exactly the same */ + if (cns == now.tv_nsec && inode_get_ctime_sec(inode) == now.tv_sec) { + trace_ctime_xchg_skip(inode, &now); + goto out; + } + cur = cns; +retry: + /* Try to swap the nsec value into place. */ + if (try_cmpxchg(&inode->i_ctime_nsec, &cur, now.tv_nsec)) { + /* If swap occurred, then we're (mostly) done */ + WRITE_ONCE(inode->i_ctime_sec, now.tv_sec); + trace_ctime_ns_xchg(inode, cns, now.tv_nsec, cur); + mgtime_counter_inc(mg_ctime_swaps); + } else { + /* + * Was the change due to someone marking the old ctime QUERIED? + * If so then retry the swap. This can only happen once since + * the only way to clear I_CTIME_QUERIED is to stamp the inode + * with a new ctime. + */ + if (!(cns & I_CTIME_QUERIED) && (cns | I_CTIME_QUERIED) == cur) { + cns = cur; + goto retry; + } + /* Otherwise, keep the existing ctime */ + now.tv_sec = inode_get_ctime_sec(inode); + now.tv_nsec = cur & ~I_CTIME_QUERIED; + } +out: + return now; +} +EXPORT_SYMBOL(inode_set_ctime_current); + +/** + * inode_set_ctime_deleg - try to update the ctime on a delegated inode + * @inode: inode to update + * @update: timespec64 to set the ctime + * + * Attempt to atomically update the ctime on behalf of a delegation holder. + * + * The nfs server can call back the holder of a delegation to get updated + * inode attributes, including the mtime. When updating the mtime, update + * the ctime to a value at least equal to that. + * + * This can race with concurrent updates to the inode, in which + * case the update is skipped. + * + * Note that this works even when multigrain timestamps are not enabled, + * so it is used in either case. + */ +struct timespec64 inode_set_ctime_deleg(struct inode *inode, struct timespec64 update) +{ + struct timespec64 now, cur_ts; + u32 cur, old; + + /* pairs with try_cmpxchg below */ + cur = smp_load_acquire(&inode->i_ctime_nsec); + cur_ts.tv_nsec = cur & ~I_CTIME_QUERIED; + cur_ts.tv_sec = inode_get_ctime_sec(inode); + + /* If the update is older than the existing value, skip it. */ + if (timespec64_compare(&update, &cur_ts) <= 0) + return cur_ts; + + ktime_get_coarse_real_ts64_mg(&now); + + /* Clamp the update to "now" if it's in the future */ + if (timespec64_compare(&update, &now) > 0) + update = now; + + update = timestamp_truncate(update, inode); + + /* No need to update if the values are already the same */ + if (timespec64_equal(&update, &cur_ts)) + return cur_ts; + + /* + * Try to swap the nsec value into place. If it fails, that means + * it raced with an update due to a write or similar activity. That + * stamp takes precedence, so just skip the update. + */ +retry: + old = cur; + if (try_cmpxchg(&inode->i_ctime_nsec, &cur, update.tv_nsec)) { + WRITE_ONCE(inode->i_ctime_sec, update.tv_sec); + mgtime_counter_inc(mg_ctime_swaps); + return update; + } + + /* + * Was the change due to another task marking the old ctime QUERIED? + * + * If so, then retry the swap. This can only happen once since + * the only way to clear I_CTIME_QUERIED is to stamp the inode + * with a new ctime. + */ + if (!(old & I_CTIME_QUERIED) && (cur == (old | I_CTIME_QUERIED))) + goto retry; + + /* Otherwise, it was a new timestamp. */ + cur_ts.tv_sec = inode_get_ctime_sec(inode); + cur_ts.tv_nsec = cur & ~I_CTIME_QUERIED; + return cur_ts; +} +EXPORT_SYMBOL(inode_set_ctime_deleg); + +/** + * in_group_or_capable - check whether caller is CAP_FSETID privileged + * @idmap: idmap of the mount @inode was found from + * @inode: inode to check + * @vfsgid: the new/current vfsgid of @inode + * + * Check whether @vfsgid is in the caller's group list or if the caller is + * privileged with CAP_FSETID over @inode. This can be used to determine + * whether the setgid bit can be kept or must be dropped. + * + * Return: true if the caller is sufficiently privileged, false if not. + */ +bool in_group_or_capable(struct mnt_idmap *idmap, + const struct inode *inode, vfsgid_t vfsgid) +{ + if (vfsgid_in_group_p(vfsgid)) + return true; + if (capable_wrt_inode_uidgid(idmap, inode, CAP_FSETID)) + return true; + return false; +} +EXPORT_SYMBOL(in_group_or_capable); + +/** + * mode_strip_sgid - handle the sgid bit for non-directories + * @idmap: idmap of the mount the inode was created from + * @dir: parent directory inode + * @mode: mode of the file to be created in @dir + * + * If the @mode of the new file has both the S_ISGID and S_IXGRP bit + * raised and @dir has the S_ISGID bit raised ensure that the caller is + * either in the group of the parent directory or they have CAP_FSETID + * in their user namespace and are privileged over the parent directory. + * In all other cases, strip the S_ISGID bit from @mode. + * + * Return: the new mode to use for the file + */ +umode_t mode_strip_sgid(struct mnt_idmap *idmap, + const struct inode *dir, umode_t mode) +{ + if ((mode & (S_ISGID | S_IXGRP)) != (S_ISGID | S_IXGRP)) + return mode; + if (S_ISDIR(mode) || !dir || !(dir->i_mode & S_ISGID)) + return mode; + if (in_group_or_capable(idmap, dir, i_gid_into_vfsgid(idmap, dir))) + return mode; + return mode & ~S_ISGID; +} +EXPORT_SYMBOL(mode_strip_sgid); + +#ifdef CONFIG_DEBUG_VFS +/** + * dump_inode - dump an inode. + * @inode: inode to dump + * @reason: reason for dumping + * + * If inode is an invalid pointer, we don't want to crash accessing it, + * so probe everything depending on it carefully with get_kernel_nofault(). + */ +void dump_inode(struct inode *inode, const char *reason) +{ + struct super_block *sb; + struct file_system_type *s_type; + const char *fs_name_ptr; + char fs_name[32] = {}; + umode_t mode; + unsigned short opflags; + unsigned int flags; + unsigned int state; + int count; + + if (get_kernel_nofault(sb, &inode->i_sb) || + get_kernel_nofault(mode, &inode->i_mode) || + get_kernel_nofault(opflags, &inode->i_opflags) || + get_kernel_nofault(flags, &inode->i_flags)) { + pr_warn("%s: unreadable inode:%px\n", reason, inode); + return; + } + + state = inode_state_read_once(inode); + count = icount_read_once(inode); + + if (!sb || + get_kernel_nofault(s_type, &sb->s_type) || !s_type || + get_kernel_nofault(fs_name_ptr, &s_type->name) || !fs_name_ptr || + strncpy_from_kernel_nofault(fs_name, fs_name_ptr, sizeof(fs_name) - 1) < 0) + strscpy(fs_name, ""); + + pr_warn("%s: inode:%px fs:%s mode:%ho opflags:%#x flags:%#x state:%#x count:%d\n", + reason, inode, fs_name, mode, opflags, flags, state, count); +} +EXPORT_SYMBOL(dump_inode); +#endif diff --git a/fs/internal.h b/fs/internal.h new file mode 100644 index 000000000..c658c8a5e --- /dev/null +++ b/fs/internal.h @@ -0,0 +1,369 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* fs/ internal definitions + * + * Copyright (C) 2006 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +struct super_block; +struct file_system_type; +struct iomap; +struct iomap_ops; +struct linux_binprm; +struct path; +struct mount; +struct shrink_control; +struct fs_context; +struct pipe_inode_info; +struct iov_iter; +struct mnt_idmap; +struct ns_common; + +/* + * block/bdev.c + */ +#ifdef CONFIG_BLOCK +extern void __init bdev_cache_init(void); +#else +static inline void bdev_cache_init(void) +{ +} +#endif /* CONFIG_BLOCK */ + +/* + * buffer.c + */ +int __block_write_begin_int(struct folio *folio, loff_t pos, unsigned len, + get_block_t *get_block, const struct iomap *iomap); + +/* + * char_dev.c + */ +extern void __init chrdev_init(void); + +/* + * fs_context.c + */ +extern int parse_monolithic_mount_data(struct fs_context *, void *); +extern void vfs_clean_context(struct fs_context *fc); +extern int finish_clean_context(struct fs_context *fc); + +/* + * namei.c + */ +extern int filename_lookup(int dfd, struct filename *name, unsigned flags, + struct path *path, const struct path *root); +int filename_rmdir(int dfd, struct filename *name); +int filename_unlinkat(int dfd, struct filename *name); +int may_linkat(struct mnt_idmap *idmap, const struct path *link); +int filename_renameat2(int olddfd, struct filename *oldname, int newdfd, + struct filename *newname, unsigned int flags); +int filename_mkdirat(int dfd, struct filename *name, umode_t mode); +int filename_mknodat(int dfd, struct filename *name, umode_t mode, unsigned int dev); +int filename_symlinkat(struct filename *from, int newdfd, struct filename *to); +int filename_linkat(int olddfd, struct filename *old, int newdfd, + struct filename *new, int flags); +int vfs_tmpfile(struct mnt_idmap *idmap, + const struct path *parentpath, + struct file *file, umode_t mode); +struct dentry *d_hash_and_lookup(struct dentry *, struct qstr *); +struct dentry *start_dirop(struct dentry *parent, struct qstr *name, + unsigned int lookup_flags); +int lookup_noperm_common(struct qstr *qname, struct dentry *base); + +void __init filename_init(void); + +/* + * namespace.c + */ +extern struct vfsmount *lookup_mnt(const struct path *); +extern int finish_automount(struct vfsmount *, const struct path *); + +extern int sb_prepare_remount_readonly(struct super_block *); + +extern void __init mnt_init(void); + +int mnt_get_write_access_file(struct file *file); +void mnt_put_write_access_file(struct file *file); + +extern void dissolve_on_fput(struct vfsmount *); +extern bool may_mount(void); + +int path_mount(const char *dev_name, const struct path *path, + const char *type_page, unsigned long flags, void *data_page); +int path_umount(const struct path *path, int flags); +int path_pivot_root(struct path *new, struct path *old); + +int show_path(struct seq_file *m, struct dentry *root); + +/* + * fs_struct.c + */ +extern void chroot_fs_refs(const struct path *, const struct path *); + +/* + * file_table.c + */ +struct file *alloc_empty_file(int flags, const struct cred *cred); +struct file *alloc_empty_file_noaccount(int flags, const struct cred *cred); +struct file *alloc_empty_backing_file(int flags, const struct cred *cred, + const struct file *user_file); +void backing_file_set_user_path(struct file *f, const struct path *path); + +static inline void file_put_write_access(struct file *file) +{ + put_write_access(file->f_inode); + mnt_put_write_access(file->f_path.mnt); + if (unlikely(file->f_mode & FMODE_BACKING)) + mnt_put_write_access(backing_file_user_path(file)->mnt); +} + +static inline void put_file_access(struct file *file) +{ + if ((file->f_mode & (FMODE_READ | FMODE_WRITE)) == FMODE_READ) { + i_readcount_dec(file->f_inode); + } else if (file->f_mode & FMODE_WRITER) { + file_put_write_access(file); + } +} + +void fput_close_sync(struct file *); +void fput_close(struct file *); + +/* + * super.c + */ +extern int reconfigure_super(struct fs_context *); +extern bool super_trylock_shared(struct super_block *sb); +struct super_block *user_get_super(dev_t, bool excl); +void put_super(struct super_block *sb); +void __init super_dev_init(void); +extern bool mount_capable(struct fs_context *); + +/* + * Prepare superblock for changing its read-only state (i.e., either remount + * read-write superblock read-only or vice versa). After this function returns + * mnt_is_readonly() will return true for any mount of the superblock if its + * caller is able to observe any changes done by the remount. This holds until + * sb_end_ro_state_change() is called. + */ +static inline void sb_start_ro_state_change(struct super_block *sb) +{ + WRITE_ONCE(sb->s_readonly_remount, 1); + /* + * For RO->RW transition, the barrier pairs with the barrier in + * mnt_is_readonly() making sure if mnt_is_readonly() sees SB_RDONLY + * cleared, it will see s_readonly_remount set. + * For RW->RO transition, the barrier pairs with the barrier in + * mnt_get_write_access() before the mnt_is_readonly() check. + * The barrier makes sure if mnt_get_write_access() sees MNT_WRITE_HOLD + * already cleared, it will see s_readonly_remount set. + */ + smp_wmb(); +} + +/* + * Ends section changing read-only state of the superblock. After this function + * returns if mnt_is_readonly() returns false, the caller will be able to + * observe all the changes remount did to the superblock. + */ +static inline void sb_end_ro_state_change(struct super_block *sb) +{ + /* + * This barrier provides release semantics that pairs with + * the smp_rmb() acquire semantics in mnt_is_readonly(). + * This barrier pair ensure that when mnt_is_readonly() sees + * 0 for sb->s_readonly_remount, it will also see all the + * preceding flag changes that were made during the RO state + * change. + */ + smp_wmb(); + WRITE_ONCE(sb->s_readonly_remount, 0); +} + +/* + * open.c + */ +struct open_flags { + int open_flag; + umode_t mode; + int acc_mode; + int intent; + int lookup_flags; +}; +extern struct file *do_file_open(int dfd, struct filename *pathname, + const struct open_flags *op); +extern struct file *do_file_open_root(const struct path *, + const char *, const struct open_flags *); +extern struct open_how build_open_how(int flags, umode_t mode); +extern int build_open_flags(const struct open_how *how, struct open_flags *op); +struct file *file_close_fd_locked(struct files_struct *files, unsigned fd); + +int do_ftruncate(struct file *file, loff_t length, unsigned int flags); +int chmod_common(const struct path *path, umode_t mode); +int do_fchownat(int dfd, const char __user *filename, uid_t user, gid_t group, + int flag); +int chown_common(const struct path *path, uid_t user, gid_t group); +extern int vfs_open(const struct path *, struct file *); + +/* + * inode.c + */ +extern long prune_icache_sb(struct super_block *sb, struct shrink_control *sc); +int dentry_needs_remove_privs(struct mnt_idmap *, struct dentry *dentry); +bool in_group_or_capable(struct mnt_idmap *idmap, + const struct inode *inode, vfsgid_t vfsgid); + +/* + * fs-writeback.c + */ +long get_nr_dirty_inodes(void); +bool sync_lazytime(struct inode *inode); + +/* + * dcache.c + */ +extern int d_set_mounted(struct dentry *dentry); +extern long prune_dcache_sb(struct super_block *sb, struct shrink_control *sc); +extern struct dentry *d_alloc_cursor(struct dentry *); +extern struct dentry * d_alloc_pseudo(struct super_block *, const struct qstr *); +extern char *simple_dname(struct dentry *, char *, int); +extern void dput_to_list(struct dentry *, struct list_head *); +extern void shrink_dentry_list(struct list_head *); +extern void shrink_dcache_for_umount(struct super_block *); +extern struct dentry *__d_lookup(const struct dentry *, const struct qstr *); +extern struct dentry *__d_lookup_rcu(const struct dentry *parent, + const struct qstr *name, unsigned *seq); + +/* + * pipe.c + */ +extern const struct file_operations pipefifo_fops; + +/* + * fs_pin.c + */ +extern void group_pin_kill(struct hlist_head *p); +extern void mnt_pin_kill(struct mount *m); + +/* + * fs/nsfs.c + */ +extern const struct dentry_operations ns_dentry_operations; +int open_namespace(struct ns_common *ns); +struct file *open_namespace_file(struct ns_common *ns); + +/* + * fs/stat.c: + */ + +int do_statx(int dfd, struct filename *filename, unsigned int flags, + unsigned int mask, struct statx __user *buffer); +int do_statx_fd(int fd, unsigned int flags, unsigned int mask, + struct statx __user *buffer); + +/* + * fs/splice.c: + */ +ssize_t splice_file_to_pipe(struct file *in, + struct pipe_inode_info *opipe, + loff_t *offset, + size_t len, unsigned int flags); + +/* + * fs/xattr.c: + */ +struct xattr_name { + char name[XATTR_NAME_MAX + 1]; +}; + +struct kernel_xattr_ctx { + /* Value of attribute */ + union { + const void __user *cvalue; + void __user *value; + }; + void *kvalue; + size_t size; + /* Attribute name */ + struct xattr_name *kname; + unsigned int flags; +}; + +ssize_t file_getxattr(struct file *file, struct kernel_xattr_ctx *ctx); +ssize_t filename_getxattr(int dfd, struct filename *filename, + unsigned int lookup_flags, struct kernel_xattr_ctx *ctx); +int file_setxattr(struct file *file, struct kernel_xattr_ctx *ctx); +int filename_setxattr(int dfd, struct filename *filename, + unsigned int lookup_flags, struct kernel_xattr_ctx *ctx); +int setxattr_copy(const char __user *name, struct kernel_xattr_ctx *ctx); +int import_xattr_name(struct xattr_name *kname, const char __user *name); + +int may_write_xattr(struct mnt_idmap *idmap, struct inode *inode); + +#ifdef CONFIG_FS_POSIX_ACL +int do_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + const char *acl_name, const void *kvalue, size_t size); +ssize_t do_get_acl(struct mnt_idmap *idmap, struct dentry *dentry, + const char *acl_name, void *kvalue, size_t size); +#else +static inline int do_set_acl(struct mnt_idmap *idmap, + struct dentry *dentry, const char *acl_name, + const void *kvalue, size_t size) +{ + return -EOPNOTSUPP; +} +static inline ssize_t do_get_acl(struct mnt_idmap *idmap, + struct dentry *dentry, const char *acl_name, + void *kvalue, size_t size) +{ + return -EOPNOTSUPP; +} +#endif + +ssize_t __kernel_write_iter(struct file *file, struct iov_iter *from, loff_t *pos); + +/* + * fs/attr.c + */ +struct mnt_idmap *alloc_mnt_idmap(struct user_namespace *mnt_userns); +struct mnt_idmap *mnt_idmap_get(struct mnt_idmap *idmap); +void mnt_idmap_put(struct mnt_idmap *idmap); +struct stashed_operations { + struct dentry *(*stash_dentry)(struct dentry **stashed, + struct dentry *dentry); + void (*put_data)(void *data); + int (*init_inode)(struct inode *inode, void *data); +}; +int path_from_stashed(struct dentry **stashed, struct vfsmount *mnt, void *data, + struct path *path); +void stashed_dentry_prune(struct dentry *dentry); +struct dentry *stash_dentry(struct dentry **stashed, struct dentry *dentry); +struct dentry *stashed_dentry_get(struct dentry **stashed); +/** + * path_mounted - check whether path is mounted + * @path: path to check + * + * Determine whether @path refers to the root of a mount. + * + * Return: true if @path is the root of a mount, false if not. + */ +static inline bool path_mounted(const struct path *path) +{ + return path->mnt->mnt_root == path->dentry; +} +void file_f_owner_release(struct file *file); +bool file_seek_cur_needs_f_lock(struct file *file); +int statmount_mnt_idmap(struct mnt_idmap *idmap, struct seq_file *seq, bool uid_map); +struct dentry *find_next_child(struct dentry *parent, struct dentry *prev); +int anon_inode_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, + unsigned int query_flags); +int anon_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr); +void pidfs_get_root(struct path *path); +void nsfs_get_root(struct path *path); +void failfs_get_root(struct path *path); +void __init failfs_init(void); +bool failfs_mnt(const struct vfsmount *mnt); +int failfs_current_chdir(void); diff --git a/fs/ioctl.c b/fs/ioctl.c new file mode 100644 index 000000000..1c152c2b1 --- /dev/null +++ b/fs/ioctl.c @@ -0,0 +1,702 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ioctl.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +#include + +/* So that the fiemap access checks can't overflow on 32 bit machines. */ +#define FIEMAP_MAX_EXTENTS (UINT_MAX / sizeof(struct fiemap_extent)) + +/** + * vfs_ioctl - call filesystem specific ioctl methods + * @filp: open file to invoke ioctl method on + * @cmd: ioctl command to execute + * @arg: command-specific argument for ioctl + * + * Invokes filesystem specific ->unlocked_ioctl, if one exists; otherwise + * returns -ENOTTY. + * + * Returns 0 on success, -errno on error. + */ +static int vfs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + int error = -ENOTTY; + + if (!filp->f_op->unlocked_ioctl) + goto out; + + error = filp->f_op->unlocked_ioctl(filp, cmd, arg); + if (error == -ENOIOCTLCMD) + error = -ENOTTY; + out: + return error; +} + +static int ioctl_fibmap(struct file *filp, int __user *p) +{ + struct inode *inode = file_inode(filp); + struct super_block *sb = inode->i_sb; + int error, ur_block; + sector_t block; + + if (!capable(CAP_SYS_RAWIO)) + return -EPERM; + + error = get_user(ur_block, p); + if (error) + return error; + + if (ur_block < 0) + return -EINVAL; + + block = ur_block; + error = bmap(inode, &block); + + if (block > INT_MAX) { + error = -ERANGE; + pr_warn_ratelimited("[%s/%d] FS: %s File: %pD4 would truncate fibmap result\n", + current->comm, task_pid_nr(current), + sb->s_id, filp); + } + + if (error) + ur_block = 0; + else + ur_block = block; + + if (put_user(ur_block, p)) + error = -EFAULT; + + return error; +} + +/** + * fiemap_fill_next_extent - Fiemap helper function + * @fieinfo: Fiemap context passed into ->fiemap + * @logical: Extent logical start offset, in bytes + * @phys: Extent physical start offset, in bytes + * @len: Extent length, in bytes + * @flags: FIEMAP_EXTENT flags that describe this extent + * + * Called from file system ->fiemap callback. Will populate extent + * info as passed in via arguments and copy to user memory. On + * success, extent count on fieinfo is incremented. + * + * Returns 0 on success, -errno on error, 1 if this was the last + * extent that will fit in user array. + */ +int fiemap_fill_next_extent(struct fiemap_extent_info *fieinfo, u64 logical, + u64 phys, u64 len, u32 flags) +{ + struct fiemap_extent extent; + struct fiemap_extent __user *dest = fieinfo->fi_extents_start; + + /* only count the extents */ + if (fieinfo->fi_extents_max == 0) { + fieinfo->fi_extents_mapped++; + return (flags & FIEMAP_EXTENT_LAST) ? 1 : 0; + } + + if (fieinfo->fi_extents_mapped >= fieinfo->fi_extents_max) + return 1; + +#define SET_UNKNOWN_FLAGS (FIEMAP_EXTENT_DELALLOC) +#define SET_NO_UNMOUNTED_IO_FLAGS (FIEMAP_EXTENT_DATA_ENCRYPTED) +#define SET_NOT_ALIGNED_FLAGS (FIEMAP_EXTENT_DATA_TAIL|FIEMAP_EXTENT_DATA_INLINE) + + if (flags & SET_UNKNOWN_FLAGS) + flags |= FIEMAP_EXTENT_UNKNOWN; + if (flags & SET_NO_UNMOUNTED_IO_FLAGS) + flags |= FIEMAP_EXTENT_ENCODED; + if (flags & SET_NOT_ALIGNED_FLAGS) + flags |= FIEMAP_EXTENT_NOT_ALIGNED; + + memset(&extent, 0, sizeof(extent)); + extent.fe_logical = logical; + extent.fe_physical = phys; + extent.fe_length = len; + extent.fe_flags = flags; + + dest += fieinfo->fi_extents_mapped; + if (copy_to_user(dest, &extent, sizeof(extent))) + return -EFAULT; + + fieinfo->fi_extents_mapped++; + if (fieinfo->fi_extents_mapped == fieinfo->fi_extents_max) + return 1; + return (flags & FIEMAP_EXTENT_LAST) ? 1 : 0; +} +EXPORT_SYMBOL(fiemap_fill_next_extent); + +/** + * fiemap_prep - check validity of requested flags for fiemap + * @inode: Inode to operate on + * @fieinfo: Fiemap context passed into ->fiemap + * @start: Start of the mapped range + * @len: Length of the mapped range, can be truncated by this function. + * @supported_flags: Set of fiemap flags that the file system understands + * + * This function must be called from each ->fiemap instance to validate the + * fiemap request against the file system parameters. + * + * Returns 0 on success, or a negative error on failure. + */ +int fiemap_prep(struct inode *inode, struct fiemap_extent_info *fieinfo, + u64 start, u64 *len, u32 supported_flags) +{ + u64 maxbytes = inode->i_sb->s_maxbytes; + u32 incompat_flags; + int ret = 0; + + if (*len == 0) + return -EINVAL; + if (start >= maxbytes) + return -EFBIG; + + /* + * Shrink request scope to what the fs can actually handle. + */ + if (*len > maxbytes || (maxbytes - *len) < start) + *len = maxbytes - start; + + supported_flags |= FIEMAP_FLAG_SYNC; + supported_flags &= FIEMAP_FLAGS_COMPAT; + incompat_flags = fieinfo->fi_flags & ~supported_flags; + if (incompat_flags) { + fieinfo->fi_flags = incompat_flags; + return -EBADR; + } + + if (fieinfo->fi_flags & FIEMAP_FLAG_SYNC) + ret = filemap_write_and_wait(inode->i_mapping); + return ret; +} +EXPORT_SYMBOL(fiemap_prep); + +static int ioctl_fiemap(struct file *filp, struct fiemap __user *ufiemap) +{ + struct fiemap fiemap; + struct fiemap_extent_info fieinfo = { 0, }; + struct inode *inode = file_inode(filp); + int error; + + if (!inode->i_op->fiemap) + return -EOPNOTSUPP; + + if (copy_from_user(&fiemap, ufiemap, sizeof(fiemap))) + return -EFAULT; + + if (fiemap.fm_extent_count > FIEMAP_MAX_EXTENTS) + return -EINVAL; + + fieinfo.fi_flags = fiemap.fm_flags; + fieinfo.fi_extents_max = fiemap.fm_extent_count; + fieinfo.fi_extents_start = ufiemap->fm_extents; + + error = inode->i_op->fiemap(inode, &fieinfo, fiemap.fm_start, + fiemap.fm_length); + + fiemap.fm_flags = fieinfo.fi_flags; + fiemap.fm_mapped_extents = fieinfo.fi_extents_mapped; + if (copy_to_user(ufiemap, &fiemap, sizeof(fiemap))) + error = -EFAULT; + + return error; +} + +static int ioctl_file_clone(struct file *dst_file, unsigned long srcfd, + u64 off, u64 olen, u64 destoff) +{ + CLASS(fd, src_file)(srcfd); + loff_t cloned; + int ret; + + if (fd_empty(src_file)) + return -EBADF; + cloned = vfs_clone_file_range(fd_file(src_file), off, dst_file, destoff, + olen, 0); + if (cloned < 0) + ret = cloned; + else if (olen && cloned != olen) + ret = -EINVAL; + else + ret = 0; + return ret; +} + +static int ioctl_file_clone_range(struct file *file, + struct file_clone_range __user *argp) +{ + struct file_clone_range args; + + if (copy_from_user(&args, argp, sizeof(args))) + return -EFAULT; + return ioctl_file_clone(file, args.src_fd, args.src_offset, + args.src_length, args.dest_offset); +} + +/* + * This provides compatibility with legacy XFS pre-allocation ioctls + * which predate the fallocate syscall. + * + * Only the l_start, l_len and l_whence fields of the 'struct space_resv' + * are used here, rest are ignored. + */ +static int ioctl_preallocate(struct file *filp, int mode, void __user *argp) +{ + struct inode *inode = file_inode(filp); + struct space_resv sr; + + if (copy_from_user(&sr, argp, sizeof(sr))) + return -EFAULT; + + switch (sr.l_whence) { + case SEEK_SET: + break; + case SEEK_CUR: + sr.l_start += filp->f_pos; + break; + case SEEK_END: + sr.l_start += i_size_read(inode); + break; + default: + return -EINVAL; + } + + return vfs_fallocate(filp, mode | FALLOC_FL_KEEP_SIZE, sr.l_start, + sr.l_len); +} + +/* on ia32 l_start is on a 32-bit boundary */ +#if defined CONFIG_COMPAT && defined(CONFIG_X86_64) +/* just account for different alignment */ +static int compat_ioctl_preallocate(struct file *file, int mode, + struct space_resv_32 __user *argp) +{ + struct inode *inode = file_inode(file); + struct space_resv_32 sr; + + if (copy_from_user(&sr, argp, sizeof(sr))) + return -EFAULT; + + switch (sr.l_whence) { + case SEEK_SET: + break; + case SEEK_CUR: + sr.l_start += file->f_pos; + break; + case SEEK_END: + sr.l_start += i_size_read(inode); + break; + default: + return -EINVAL; + } + + return vfs_fallocate(file, mode | FALLOC_FL_KEEP_SIZE, sr.l_start, sr.l_len); +} +#endif + +static int file_ioctl(struct file *filp, unsigned int cmd, int __user *p) +{ + switch (cmd) { + case FIBMAP: + return ioctl_fibmap(filp, p); + case FS_IOC_RESVSP: + case FS_IOC_RESVSP64: + return ioctl_preallocate(filp, 0, p); + case FS_IOC_UNRESVSP: + case FS_IOC_UNRESVSP64: + return ioctl_preallocate(filp, FALLOC_FL_PUNCH_HOLE, p); + case FS_IOC_ZERO_RANGE: + return ioctl_preallocate(filp, FALLOC_FL_ZERO_RANGE, p); + } + + return -ENOIOCTLCMD; +} + +static int ioctl_fionbio(struct file *filp, int __user *argp) +{ + unsigned int flag; + int on, error; + + error = get_user(on, argp); + if (error) + return error; + flag = O_NONBLOCK; +#ifdef __sparc__ + /* SunOS compatibility item. */ + if (O_NONBLOCK != O_NDELAY) + flag |= O_NDELAY; +#endif + spin_lock(&filp->f_lock); + if (on) + filp->f_flags |= flag; + else + filp->f_flags &= ~flag; + spin_unlock(&filp->f_lock); + return error; +} + +static int ioctl_fioasync(unsigned int fd, struct file *filp, + int __user *argp) +{ + unsigned int flag; + int on, error; + + error = get_user(on, argp); + if (error) + return error; + flag = on ? FASYNC : 0; + + /* Did FASYNC state change ? */ + if ((flag ^ filp->f_flags) & FASYNC) { + if (filp->f_op->fasync) + /* fasync() adjusts filp->f_flags */ + error = filp->f_op->fasync(fd, filp, on); + else + error = -ENOTTY; + } + return error < 0 ? error : 0; +} + +static int ioctl_fsfreeze(struct file *filp) +{ + struct super_block *sb = file_inode(filp)->i_sb; + + if (!ns_capable(sb->s_user_ns, CAP_SYS_ADMIN)) + return -EPERM; + + /* If filesystem doesn't support freeze feature, return. */ + if (sb->s_op->freeze_fs == NULL && sb->s_op->freeze_super == NULL) + return -EOPNOTSUPP; + + /* Freeze */ + if (sb->s_op->freeze_super) + return sb->s_op->freeze_super(sb, FREEZE_HOLDER_USERSPACE, NULL); + return freeze_super(sb, FREEZE_HOLDER_USERSPACE, NULL); +} + +static int ioctl_fsthaw(struct file *filp) +{ + struct super_block *sb = file_inode(filp)->i_sb; + + if (!ns_capable(sb->s_user_ns, CAP_SYS_ADMIN)) + return -EPERM; + + /* Thaw */ + if (sb->s_op->thaw_super) + return sb->s_op->thaw_super(sb, FREEZE_HOLDER_USERSPACE, NULL); + return thaw_super(sb, FREEZE_HOLDER_USERSPACE, NULL); +} + +static int ioctl_file_dedupe_range(struct file *file, + struct file_dedupe_range __user *argp) +{ + struct file_dedupe_range *same = NULL; + int ret; + unsigned long size; + u16 count; + + if (get_user(count, &argp->dest_count)) { + ret = -EFAULT; + goto out; + } + + size = struct_size(same, info, count); + if (size > PAGE_SIZE) { + ret = -ENOMEM; + goto out; + } + + same = memdup_user(argp, size); + if (IS_ERR(same)) { + ret = PTR_ERR(same); + same = NULL; + goto out; + } + + same->dest_count = count; + ret = vfs_dedupe_file_range(file, same); + if (ret) + goto out; + + ret = copy_to_user(argp, same, size); + if (ret) + ret = -EFAULT; + +out: + kfree(same); + return ret; +} + +static int ioctl_getfsuuid(struct file *file, void __user *argp) +{ + struct super_block *sb = file_inode(file)->i_sb; + struct fsuuid2 u = { .len = sb->s_uuid_len, }; + + if (!sb->s_uuid_len) + return -ENOTTY; + + memcpy(&u.uuid[0], &sb->s_uuid, sb->s_uuid_len); + + return copy_to_user(argp, &u, sizeof(u)) ? -EFAULT : 0; +} + +static int ioctl_get_fs_sysfs_path(struct file *file, void __user *argp) +{ + struct super_block *sb = file_inode(file)->i_sb; + + if (!strlen(sb->s_sysfs_name)) + return -ENOTTY; + + struct fs_sysfs_path u = {}; + + u.len = scnprintf(u.name, sizeof(u.name), "%s/%s", sb->s_type->name, sb->s_sysfs_name); + + return copy_to_user(argp, &u, sizeof(u)) ? -EFAULT : 0; +} + +/* + * do_vfs_ioctl() is not for drivers and not intended to be EXPORT_SYMBOL()'d. + * It's just a simple helper for sys_ioctl and compat_sys_ioctl. + * + * When you add any new common ioctls to the switches above and below, + * please ensure they have compatible arguments in compat mode. + * + * The LSM mailing list should also be notified of any command additions or + * changes, as specific LSMs may be affected. + */ +static int do_vfs_ioctl(struct file *filp, unsigned int fd, + unsigned int cmd, unsigned long arg) +{ + void __user *argp = (void __user *)arg; + struct inode *inode = file_inode(filp); + + switch (cmd) { + case FIOCLEX: + set_close_on_exec(fd, 1); + return 0; + + case FIONCLEX: + set_close_on_exec(fd, 0); + return 0; + + case FIONBIO: + return ioctl_fionbio(filp, argp); + + case FIOASYNC: + return ioctl_fioasync(fd, filp, argp); + + case FIOQSIZE: + if (S_ISDIR(inode->i_mode) || + (S_ISREG(inode->i_mode) && !IS_ANON_FILE(inode)) || + S_ISLNK(inode->i_mode)) { + loff_t res = inode_get_bytes(inode); + return copy_to_user(argp, &res, sizeof(res)) ? + -EFAULT : 0; + } + + return -ENOTTY; + + case FIFREEZE: + return ioctl_fsfreeze(filp); + + case FITHAW: + return ioctl_fsthaw(filp); + + case FS_IOC_FIEMAP: + return ioctl_fiemap(filp, argp); + + case FIGETBSZ: + /* anon_bdev filesystems may not have a block size */ + if (!inode->i_sb->s_blocksize) + return -EINVAL; + + return put_user(inode->i_sb->s_blocksize, (int __user *)argp); + + case FICLONE: + return ioctl_file_clone(filp, arg, 0, 0, 0); + + case FICLONERANGE: + return ioctl_file_clone_range(filp, argp); + + case FIDEDUPERANGE: + return ioctl_file_dedupe_range(filp, argp); + + case FIONREAD: + if (!S_ISREG(inode->i_mode) || IS_ANON_FILE(inode)) + return vfs_ioctl(filp, cmd, arg); + + return put_user(i_size_read(inode) - filp->f_pos, + (int __user *)argp); + + case FS_IOC_GETFLAGS: + return ioctl_getflags(filp, argp); + + case FS_IOC_SETFLAGS: + return ioctl_setflags(filp, argp); + + case FS_IOC_FSGETXATTR: + return ioctl_fsgetxattr(filp, argp); + + case FS_IOC_FSSETXATTR: + return ioctl_fssetxattr(filp, argp); + + case FS_IOC_GETFSUUID: + return ioctl_getfsuuid(filp, argp); + + case FS_IOC_GETFSSYSFSPATH: + return ioctl_get_fs_sysfs_path(filp, argp); + + default: + if (S_ISREG(inode->i_mode) && !IS_ANON_FILE(inode)) + return file_ioctl(filp, cmd, argp); + break; + } + + return -ENOIOCTLCMD; +} + +SYSCALL_DEFINE3(ioctl, unsigned int, fd, unsigned int, cmd, unsigned long, arg) +{ + CLASS(fd, f)(fd); + int error; + + if (fd_empty(f)) + return -EBADF; + + error = security_file_ioctl(fd_file(f), cmd, arg); + if (error) + return error; + + error = do_vfs_ioctl(fd_file(f), fd, cmd, arg); + if (error == -ENOIOCTLCMD) + error = vfs_ioctl(fd_file(f), cmd, arg); + + return error; +} + +#ifdef CONFIG_COMPAT +/** + * compat_ptr_ioctl - generic implementation of .compat_ioctl file operation + * @file: The file to operate on. + * @cmd: The ioctl command number. + * @arg: The argument to the ioctl. + * + * This is not normally called as a function, but instead set in struct + * file_operations as + * + * .compat_ioctl = compat_ptr_ioctl, + * + * On most architectures, the compat_ptr_ioctl() just passes all arguments + * to the corresponding ->ioctl handler. The exception is arch/s390, where + * compat_ptr() clears the top bit of a 32-bit pointer value, so user space + * pointers to the second 2GB alias the first 2GB, as is the case for + * native 32-bit s390 user space. + * + * The compat_ptr_ioctl() function must therefore be used only with ioctl + * functions that either ignore the argument or pass a pointer to a + * compatible data type. + * + * If any ioctl command handled by fops->unlocked_ioctl passes a plain + * integer instead of a pointer, or any of the passed data types + * is incompatible between 32-bit and 64-bit architectures, a proper + * handler is required instead of compat_ptr_ioctl. + */ +long compat_ptr_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + if (!file->f_op->unlocked_ioctl) + return -ENOIOCTLCMD; + + return file->f_op->unlocked_ioctl(file, cmd, (unsigned long)compat_ptr(arg)); +} +EXPORT_SYMBOL(compat_ptr_ioctl); + +COMPAT_SYSCALL_DEFINE3(ioctl, unsigned int, fd, unsigned int, cmd, + compat_ulong_t, arg) +{ + CLASS(fd, f)(fd); + int error; + + if (fd_empty(f)) + return -EBADF; + + error = security_file_ioctl_compat(fd_file(f), cmd, arg); + if (error) + return error; + + switch (cmd) { + /* FICLONE takes an int argument, so don't use compat_ptr() */ + case FICLONE: + error = ioctl_file_clone(fd_file(f), arg, 0, 0, 0); + break; + +#if defined(CONFIG_X86_64) + /* these get messy on amd64 due to alignment differences */ + case FS_IOC_RESVSP_32: + case FS_IOC_RESVSP64_32: + error = compat_ioctl_preallocate(fd_file(f), 0, compat_ptr(arg)); + break; + case FS_IOC_UNRESVSP_32: + case FS_IOC_UNRESVSP64_32: + error = compat_ioctl_preallocate(fd_file(f), FALLOC_FL_PUNCH_HOLE, + compat_ptr(arg)); + break; + case FS_IOC_ZERO_RANGE_32: + error = compat_ioctl_preallocate(fd_file(f), FALLOC_FL_ZERO_RANGE, + compat_ptr(arg)); + break; +#endif + + /* + * These access 32-bit values anyway so no further handling is + * necessary. + */ + case FS_IOC32_GETFLAGS: + case FS_IOC32_SETFLAGS: + cmd = (cmd == FS_IOC32_GETFLAGS) ? + FS_IOC_GETFLAGS : FS_IOC_SETFLAGS; + fallthrough; + /* + * everything else in do_vfs_ioctl() takes either a compatible + * pointer argument or no argument -- call it with a modified + * argument. + */ + default: + error = do_vfs_ioctl(fd_file(f), fd, cmd, + (unsigned long)compat_ptr(arg)); + if (error != -ENOIOCTLCMD) + break; + + if (fd_file(f)->f_op->compat_ioctl) + error = fd_file(f)->f_op->compat_ioctl(fd_file(f), cmd, arg); + if (error == -ENOIOCTLCMD) + error = -ENOTTY; + break; + } + return error; +} +#endif diff --git a/fs/iomap/Makefile b/fs/iomap/Makefile new file mode 100644 index 000000000..a572b8808 --- /dev/null +++ b/fs/iomap/Makefile @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Copyright (c) 2019 Oracle. +# All Rights Reserved. +# + +ccflags-y += -I $(src) # needed for trace events + +obj-$(CONFIG_FS_IOMAP) += iomap.o + +iomap-y += trace.o \ + iter.o \ + buffered-io.o +iomap-$(CONFIG_BLOCK) += direct-io.o \ + ioend.o \ + fiemap.o \ + seek.o \ + bio.o +iomap-$(CONFIG_SWAP) += swapfile.o diff --git a/fs/iomap/bio.c b/fs/iomap/bio.c new file mode 100644 index 000000000..48100c614 --- /dev/null +++ b/fs/iomap/bio.c @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2010 Red Hat, Inc. + * Copyright (C) 2016-2023 Christoph Hellwig. + */ +#include +#include +#include +#include "internal.h" +#include "trace.h" + +static DEFINE_SPINLOCK(failed_read_lock); +static struct bio_list failed_read_list = BIO_EMPTY_LIST; + +static u32 __iomap_read_end_io(struct bio *bio, int error) +{ + struct folio_iter fi; + u32 folio_count = 0; + + bio_for_each_folio_all(fi, bio) { + iomap_finish_folio_read(fi.folio, fi.offset, fi.length, error); + folio_count++; + } + if (bio_integrity(bio)) + fs_bio_integrity_free(bio); + bio_put(bio); + return folio_count; +} + +static void +iomap_fail_reads( + struct work_struct *work) +{ + struct bio *bio; + struct bio_list tmp = BIO_EMPTY_LIST; + unsigned long flags; + + spin_lock_irqsave(&failed_read_lock, flags); + bio_list_merge_init(&tmp, &failed_read_list); + spin_unlock_irqrestore(&failed_read_lock, flags); + + while ((bio = bio_list_pop(&tmp)) != NULL) { + __iomap_read_end_io(bio, blk_status_to_errno(bio->bi_status)); + cond_resched(); + } +} + +static DECLARE_WORK(failed_read_work, iomap_fail_reads); + +static void iomap_fail_buffered_read(struct bio *bio) +{ + unsigned long flags; + + /* + * Bounce I/O errors to a workqueue to avoid nested i_lock acquisitions + * in the fserror code. The caller no longer owns the bio reference + * after the spinlock drops. + */ + spin_lock_irqsave(&failed_read_lock, flags); + if (bio_list_empty(&failed_read_list)) + WARN_ON_ONCE(!schedule_work(&failed_read_work)); + bio_list_add(&failed_read_list, bio); + spin_unlock_irqrestore(&failed_read_lock, flags); +} + +static void iomap_read_end_io(struct bio *bio) +{ + if (bio->bi_status) { + iomap_fail_buffered_read(bio); + return; + } + + __iomap_read_end_io(bio, 0); +} + +u32 iomap_finish_ioend_buffered_read(struct iomap_ioend *ioend) +{ + return __iomap_read_end_io(&ioend->io_bio, ioend->io_error); +} + +void iomap_bio_submit_read_endio(const struct iomap_iter *iter, + struct iomap_read_folio_ctx *ctx, bio_end_io_t end_io) +{ + struct bio *bio = ctx->read_ctx; + + bio->bi_end_io = end_io; + if (iter->iomap.flags & IOMAP_F_INTEGRITY) + fs_bio_integrity_alloc(bio); + submit_bio(bio); + + ctx->read_ctx = NULL; +} +EXPORT_SYMBOL_GPL(iomap_bio_submit_read_endio); + +static void iomap_bio_submit_read(const struct iomap_iter *iter, + struct iomap_read_folio_ctx *ctx) +{ + return iomap_bio_submit_read_endio(iter, ctx, iomap_read_end_io); +} + +static struct bio_set *iomap_read_bio_set(struct iomap_read_folio_ctx *ctx) +{ + if (ctx->ops && ctx->ops->bio_set) + return ctx->ops->bio_set; + return &fs_bio_set; +} + +static void iomap_read_alloc_bio(const struct iomap_iter *iter, + struct iomap_read_folio_ctx *ctx, size_t plen) +{ + const struct iomap *iomap = &iter->iomap; + unsigned int nr_vecs = DIV_ROUND_UP(iomap_length(iter), PAGE_SIZE); + struct bio_set *bio_set = iomap_read_bio_set(ctx); + struct folio *folio = ctx->cur_folio; + gfp_t gfp = mapping_gfp_constraint(folio->mapping, GFP_KERNEL); + gfp_t orig_gfp = gfp; + struct bio *bio; + + /* Submit the existing range if there was one. */ + if (ctx->read_ctx) + ctx->ops->submit_read(iter, ctx); + + /* Same as readahead_gfp_mask: */ + if (ctx->rac) + gfp |= __GFP_NORETRY | __GFP_NOWARN; + + /* + * If the bio_alloc fails, try it again for a single page to avoid + * having to deal with partial page reads. This emulates what + * do_mpage_read_folio does. + */ + bio = bio_alloc_bioset(iomap->bdev, bio_max_segs(nr_vecs), REQ_OP_READ, + gfp, bio_set); + if (!bio) + bio = bio_alloc_bioset(iomap->bdev, 1, REQ_OP_READ, orig_gfp, + bio_set); + if (ctx->rac) + bio->bi_opf |= REQ_RAHEAD; + bio->bi_iter.bi_sector = iomap_sector(iomap, iter->pos); + bio_add_folio_nofail(bio, folio, plen, + offset_in_folio(folio, iter->pos)); + ctx->read_ctx = bio; + ctx->read_ctx_file_offset = iter->pos; +} + +int iomap_bio_read_folio_range(const struct iomap_iter *iter, + struct iomap_read_folio_ctx *ctx, size_t plen) +{ + struct folio *folio = ctx->cur_folio; + struct bio *bio = ctx->read_ctx; + + if (!bio || + bio_end_sector(bio) != iomap_sector(&iter->iomap, iter->pos) || + bio->bi_iter.bi_size > iomap_max_bio_size(&iter->iomap) - plen || + !bio_add_folio(bio, folio, plen, offset_in_folio(folio, iter->pos))) + iomap_read_alloc_bio(iter, ctx, plen); + return 0; +} +EXPORT_SYMBOL_GPL(iomap_bio_read_folio_range); + +const struct iomap_read_ops iomap_bio_read_ops = { + .read_folio_range = iomap_bio_read_folio_range, + .submit_read = iomap_bio_submit_read, +}; +EXPORT_SYMBOL_GPL(iomap_bio_read_ops); + +int iomap_bio_read_folio_range_sync(const struct iomap_iter *iter, + struct folio *folio, loff_t pos, size_t len) +{ + const struct iomap *srcmap = iomap_iter_srcmap(iter); + sector_t sector = iomap_sector(srcmap, pos); + struct bio_vec bvec; + struct bio bio; + int error; + + bio_init(&bio, srcmap->bdev, &bvec, 1, REQ_OP_READ); + bio.bi_iter.bi_sector = sector; + bio_add_folio_nofail(&bio, folio, len, offset_in_folio(folio, pos)); + if (srcmap->flags & IOMAP_F_INTEGRITY) + fs_bio_integrity_alloc(&bio); + error = submit_bio_wait(&bio); + if (bio_integrity(&bio)) { + if (!error) + error = fs_bio_integrity_verify(&bio, sector, len); + fs_bio_integrity_free(&bio); + } + bio_uninit(&bio); + return error; +} diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c new file mode 100644 index 000000000..0a5ebfda9 --- /dev/null +++ b/fs/iomap/buffered-io.c @@ -0,0 +1,2102 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2010 Red Hat, Inc. + * Copyright (C) 2016-2023 Christoph Hellwig. + */ +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" +#include "trace.h" + +#include "../internal.h" + +/* + * Structure allocated for each folio to track per-block uptodate, dirty state + * and I/O completions. + */ +struct iomap_folio_state { + spinlock_t state_lock; + unsigned int read_bytes_pending; + atomic_t write_bytes_pending; + + /* + * Each block has two bits in this bitmap: + * Bits [0..blocks_per_folio) has the uptodate status. + * Bits [b_p_f...(2*b_p_f)) has the dirty status. + */ + unsigned long state[]; +}; + +static inline bool ifs_is_fully_uptodate(struct folio *folio, + struct iomap_folio_state *ifs) +{ + struct inode *inode = folio->mapping->host; + + return bitmap_full(ifs->state, i_blocks_per_folio(inode, folio)); +} + +/* + * Find the next uptodate block in the folio. end_blk is inclusive. + * If no uptodate block is found, this will return end_blk + 1. + */ +static unsigned ifs_next_uptodate_block(struct folio *folio, + unsigned start_blk, unsigned end_blk) +{ + struct iomap_folio_state *ifs = folio->private; + + return find_next_bit(ifs->state, end_blk + 1, start_blk); +} + +/* + * Find the next non-uptodate block in the folio. end_blk is inclusive. + * If no non-uptodate block is found, this will return end_blk + 1. + */ +static unsigned ifs_next_nonuptodate_block(struct folio *folio, + unsigned start_blk, unsigned end_blk) +{ + struct iomap_folio_state *ifs = folio->private; + + return find_next_zero_bit(ifs->state, end_blk + 1, start_blk); +} + +static bool ifs_set_range_uptodate(struct folio *folio, + struct iomap_folio_state *ifs, size_t off, size_t len) +{ + struct inode *inode = folio->mapping->host; + unsigned int first_blk, last_blk; + + if (len) { + first_blk = off >> inode->i_blkbits; + last_blk = (off + len - 1) >> inode->i_blkbits; + bitmap_set(ifs->state, first_blk, last_blk - first_blk + 1); + } + return ifs_is_fully_uptodate(folio, ifs); +} + +static void iomap_set_range_uptodate(struct folio *folio, size_t off, + size_t len) +{ + struct iomap_folio_state *ifs = folio->private; + unsigned long flags; + bool mark_uptodate = true; + + if (folio_test_uptodate(folio)) + return; + + if (ifs) { + spin_lock_irqsave(&ifs->state_lock, flags); + /* + * If a read with bytes pending is in progress, we must not call + * folio_mark_uptodate(). The read completion path + * (iomap_read_end()) will call folio_end_read(), which uses XOR + * semantics to set the uptodate bit. If we set it here, the XOR + * in folio_end_read() will clear it, leaving the folio not + * uptodate. + */ + mark_uptodate = ifs_set_range_uptodate(folio, ifs, off, len) && + !ifs->read_bytes_pending; + spin_unlock_irqrestore(&ifs->state_lock, flags); + } + + if (mark_uptodate) + folio_mark_uptodate(folio); +} + +void iomap_folio_mark_uptodate(struct folio *folio) +{ + iomap_set_range_uptodate(folio, 0, folio_size(folio)); +} +EXPORT_SYMBOL_GPL(iomap_folio_mark_uptodate); + +/* + * Find the next dirty block in the folio. end_blk is inclusive. + * If no dirty block is found, this will return end_blk + 1. + */ +static unsigned ifs_next_dirty_block(struct folio *folio, + unsigned start_blk, unsigned end_blk) +{ + struct iomap_folio_state *ifs = folio->private; + struct inode *inode = folio->mapping->host; + unsigned int blks = i_blocks_per_folio(inode, folio); + + return find_next_bit(ifs->state, blks + end_blk + 1, + blks + start_blk) - blks; +} + +/* + * Find the next clean block in the folio. end_blk is inclusive. + * If no clean block is found, this will return end_blk + 1. + */ +static unsigned ifs_next_clean_block(struct folio *folio, + unsigned start_blk, unsigned end_blk) +{ + struct iomap_folio_state *ifs = folio->private; + struct inode *inode = folio->mapping->host; + unsigned int blks = i_blocks_per_folio(inode, folio); + + return find_next_zero_bit(ifs->state, blks + end_blk + 1, + blks + start_blk) - blks; +} + +static unsigned ifs_find_dirty_range(struct folio *folio, + struct iomap_folio_state *ifs, u64 *range_start, u64 range_end) +{ + struct inode *inode = folio->mapping->host; + unsigned start_blk = + offset_in_folio(folio, *range_start) >> inode->i_blkbits; + unsigned end_blk = min_not_zero( + offset_in_folio(folio, range_end) >> inode->i_blkbits, + i_blocks_per_folio(inode, folio)) - 1; + unsigned nblks; + + start_blk = ifs_next_dirty_block(folio, start_blk, end_blk); + if (start_blk > end_blk) + return 0; + if (start_blk == end_blk) + nblks = 1; + else + nblks = ifs_next_clean_block(folio, start_blk + 1, end_blk) - + start_blk; + + *range_start = folio_pos(folio) + (start_blk << inode->i_blkbits); + return nblks << inode->i_blkbits; +} + +static unsigned iomap_find_dirty_range(struct folio *folio, u64 *range_start, + u64 range_end) +{ + struct iomap_folio_state *ifs = folio->private; + + if (*range_start >= range_end) + return 0; + + if (ifs) + return ifs_find_dirty_range(folio, ifs, range_start, range_end); + return range_end - *range_start; +} + +/* + * Clear the per-block dirty bits for the range [@off, @off + @len) within a + * folio. The range is rounded inwards so that only blocks fully covered by + * the range are cleared. This is required for operations like folio + * invalidation, where we must ensure a block is fully clean before discarding + * it. + */ +static void ifs_clear_range_dirty(struct folio *folio, + struct iomap_folio_state *ifs, size_t off, size_t len) +{ + struct inode *inode = folio->mapping->host; + unsigned int blks_per_folio = i_blocks_per_folio(inode, folio); + unsigned int first_blk = round_up(off, i_blocksize(inode)) >> + inode->i_blkbits; + unsigned int last_blk = (off + len) >> inode->i_blkbits; + unsigned long flags; + + if (first_blk >= last_blk) + return; + + spin_lock_irqsave(&ifs->state_lock, flags); + bitmap_clear(ifs->state, first_blk + blks_per_folio, + last_blk - first_blk); + spin_unlock_irqrestore(&ifs->state_lock, flags); +} + +static void iomap_clear_range_dirty(struct folio *folio, size_t off, size_t len) +{ + struct iomap_folio_state *ifs = folio->private; + + if (ifs) + ifs_clear_range_dirty(folio, ifs, off, len); +} + +/* + * Set the per-block dirty bits for the range [@off, @off + @len) within a + * folio. The range is rounded outwards so that any block partially touched + * by the range is marked dirty. This ensures blocks containing even a + * single dirty byte will be included in subsequent writeback, preventing + * data loss when partial blocks are written. + */ +static void ifs_set_range_dirty(struct folio *folio, + struct iomap_folio_state *ifs, size_t off, size_t len) +{ + struct inode *inode = folio->mapping->host; + unsigned int blks_per_folio = i_blocks_per_folio(inode, folio); + unsigned int first_blk, last_blk; + unsigned long flags; + + if (!len) + return; + + first_blk = off >> inode->i_blkbits; + last_blk = (off + len - 1) >> inode->i_blkbits; + spin_lock_irqsave(&ifs->state_lock, flags); + bitmap_set(ifs->state, first_blk + blks_per_folio, + last_blk - first_blk + 1); + spin_unlock_irqrestore(&ifs->state_lock, flags); +} + +static void iomap_set_range_dirty(struct folio *folio, size_t off, size_t len) +{ + struct iomap_folio_state *ifs = folio->private; + + if (ifs) + ifs_set_range_dirty(folio, ifs, off, len); +} + +static struct iomap_folio_state *ifs_alloc(struct inode *inode, + struct folio *folio, unsigned int flags) +{ + struct iomap_folio_state *ifs = folio->private; + unsigned int nr_blocks = i_blocks_per_folio(inode, folio); + gfp_t gfp; + + if (ifs || nr_blocks <= 1) + return ifs; + + if (flags & IOMAP_NOWAIT) + gfp = GFP_NOWAIT; + else + gfp = GFP_NOFS | __GFP_NOFAIL; + + /* + * ifs->state tracks two sets of state flags when the + * filesystem block size is smaller than the folio size. + * The first state tracks per-block uptodate and the + * second tracks per-block dirty state. + */ + ifs = kzalloc_flex(*ifs, state, BITS_TO_LONGS(2 * nr_blocks), gfp); + if (!ifs) + return ifs; + + spin_lock_init(&ifs->state_lock); + if (folio_test_uptodate(folio)) + bitmap_set(ifs->state, 0, nr_blocks); + if (folio_test_dirty(folio)) + bitmap_set(ifs->state, nr_blocks, nr_blocks); + folio_attach_private(folio, ifs); + + return ifs; +} + +static void ifs_free(struct folio *folio) +{ + struct iomap_folio_state *ifs = folio_detach_private(folio); + + if (!ifs) + return; + WARN_ON_ONCE(ifs->read_bytes_pending != 0); + WARN_ON_ONCE(atomic_read(&ifs->write_bytes_pending)); + WARN_ON_ONCE(ifs_is_fully_uptodate(folio, ifs) != + folio_test_uptodate(folio)); + kfree(ifs); +} + +/* + * Calculate how many bytes to truncate based off the number of blocks to + * truncate and the end position to start truncating from. + */ +static size_t iomap_bytes_to_truncate(loff_t end_pos, unsigned block_bits, + unsigned blocks_truncated) +{ + unsigned block_size = 1 << block_bits; + unsigned block_offset = end_pos & (block_size - 1); + + if (!block_offset) + return blocks_truncated << block_bits; + + return ((blocks_truncated - 1) << block_bits) + block_offset; +} + +/* + * Calculate the range inside the folio that we actually need to read. + */ +static void iomap_adjust_read_range(struct inode *inode, struct folio *folio, + loff_t *pos, loff_t length, size_t *offp, size_t *lenp) +{ + struct iomap_folio_state *ifs = folio->private; + loff_t orig_pos = *pos; + loff_t isize = i_size_read(inode); + unsigned block_bits = inode->i_blkbits; + unsigned block_size = (1 << block_bits); + size_t poff = offset_in_folio(folio, *pos); + size_t plen = min_t(loff_t, folio_size(folio) - poff, length); + size_t orig_plen = plen; + unsigned first = poff >> block_bits; + unsigned last = (poff + plen - 1) >> block_bits; + + /* + * If the block size is smaller than the page size, we need to check the + * per-block uptodate status and adjust the offset and length if needed + * to avoid reading in already uptodate ranges. + */ + if (ifs) { + unsigned int next, blocks_skipped; + + next = ifs_next_nonuptodate_block(folio, first, last); + blocks_skipped = next - first; + + if (blocks_skipped) { + unsigned long block_offset = *pos & (block_size - 1); + unsigned bytes_skipped = + (blocks_skipped << block_bits) - block_offset; + + *pos += bytes_skipped; + poff += bytes_skipped; + plen -= bytes_skipped; + } + first = next; + + /* truncate len if we find any trailing uptodate block(s) */ + if (++next <= last) { + next = ifs_next_uptodate_block(folio, next, last); + if (next <= last) { + plen -= iomap_bytes_to_truncate(*pos + plen, + block_bits, last - next + 1); + last = next - 1; + } + } + } + + /* + * If the extent spans the block that contains the i_size, we need to + * handle both halves separately so that we properly zero data in the + * page cache for blocks that are entirely outside of i_size. + */ + if (orig_pos <= isize && orig_pos + orig_plen > isize) { + unsigned end = offset_in_folio(folio, isize - 1) >> block_bits; + + if (first <= end && last > end) + plen -= iomap_bytes_to_truncate(*pos + plen, block_bits, + last - end); + } + + *offp = poff; + *lenp = plen; +} + +static inline bool iomap_block_needs_zeroing(const struct iomap_iter *iter, + loff_t pos) +{ + const struct iomap *srcmap = iomap_iter_srcmap(iter); + + /* + * If this block has not been written, there's nothing to read + */ + if (srcmap->type != IOMAP_MAPPED) + return true; + + /* + * Newly allocated blocks have not been written + */ + if (srcmap->flags & IOMAP_F_NEW) + return true; + + /* + * fsverity metadata is stored past i_size, we need to read it instead + * of zeroing + */ + if (srcmap->flags & IOMAP_F_FSVERITY) + return false; + + return pos >= i_size_read(iter->inode); +} + +/** + * iomap_read_inline_data - copy inline data into the page cache + * @iter: iteration structure + * @folio: folio to copy to + * + * Copy the inline data in @iter into @folio and zero out the rest of the folio. + * Only a single IOMAP_INLINE extent is allowed at the end of each file. + * Returns zero for success to complete the read, or the usual negative errno. + */ +static int iomap_read_inline_data(const struct iomap_iter *iter, + struct folio *folio) +{ + const struct iomap *iomap = iomap_iter_srcmap(iter); + size_t size = i_size_read(iter->inode) - iomap->offset; + size_t offset = offset_in_folio(folio, iomap->offset); + + if (WARN_ON_ONCE(!iomap->inline_data)) + return -EIO; + + if (folio_test_uptodate(folio)) + return 0; + + if (WARN_ON_ONCE(size > iomap->length)) { + fserror_report_io(iter->inode, FSERR_BUFFERED_READ, + iomap->offset, size, -EIO, GFP_NOFS); + return -EIO; + } + if (offset > 0) + ifs_alloc(iter->inode, folio, iter->flags); + + folio_fill_tail(folio, offset, iomap->inline_data, size); + iomap_set_range_uptodate(folio, offset, folio_size(folio) - offset); + return 0; +} + +void iomap_finish_folio_read(struct folio *folio, size_t off, size_t len, + int error) +{ + struct iomap_folio_state *ifs = folio->private; + bool uptodate = !error; + bool finished = true; + + if (error) + fserror_report_io(folio->mapping->host, FSERR_BUFFERED_READ, + folio_pos(folio) + off, len, error, + GFP_ATOMIC); + + if (ifs) { + unsigned long flags; + + spin_lock_irqsave(&ifs->state_lock, flags); + if (!error) + uptodate = ifs_set_range_uptodate(folio, ifs, off, len); + ifs->read_bytes_pending -= len; + finished = !ifs->read_bytes_pending; + spin_unlock_irqrestore(&ifs->state_lock, flags); + } + + if (finished) + folio_end_read(folio, uptodate); +} +EXPORT_SYMBOL_GPL(iomap_finish_folio_read); + +static void iomap_read_init(struct folio *folio) +{ + struct iomap_folio_state *ifs = folio->private; + + if (ifs) { + /* + * ifs->read_bytes_pending is used to track how many bytes are + * read in asynchronously by the IO helper. We need to track + * this so that we can know when the IO helper has finished + * reading in all the necessary ranges of the folio and can end + * the read. + * + * Increase ->read_bytes_pending by the folio size to start. + * We'll subtract any uptodate / zeroed ranges that did not + * require IO in iomap_read_end() after we're done processing + * the folio. + * + * We do this because otherwise, we would have to increment + * ifs->read_bytes_pending every time a range in the folio needs + * to be read in, which can get expensive since the spinlock + * needs to be held whenever modifying ifs->read_bytes_pending. + */ + spin_lock_irq(&ifs->state_lock); + WARN_ON_ONCE(ifs->read_bytes_pending != 0); + ifs->read_bytes_pending = folio_size(folio); + spin_unlock_irq(&ifs->state_lock); + } +} + +/* + * This ends IO if no bytes were submitted to an IO helper. + * + * Otherwise, this calibrates ifs->read_bytes_pending to represent only the + * submitted bytes (see comment in iomap_read_init()). If all bytes submitted + * have already been completed by the IO helper, then this will end the read. + * Else the IO helper will end the read after all submitted ranges have been + * read. + */ +static void iomap_read_end(struct folio *folio, size_t bytes_submitted) +{ + struct iomap_folio_state *ifs = folio->private; + + if (ifs) { + bool end_read, uptodate; + + spin_lock_irq(&ifs->state_lock); + if (!ifs->read_bytes_pending) { + WARN_ON_ONCE(bytes_submitted); + spin_unlock_irq(&ifs->state_lock); + folio_unlock(folio); + return; + } + + /* + * Subtract any bytes that were initially accounted to + * read_bytes_pending but skipped for IO. + */ + ifs->read_bytes_pending -= folio_size(folio) - bytes_submitted; + + /* + * If !ifs->read_bytes_pending, this means all pending reads by + * the IO helper have already completed, which means we need to + * end the folio read here. If ifs->read_bytes_pending != 0, + * the IO helper will end the folio read. + */ + end_read = !ifs->read_bytes_pending; + if (end_read) + uptodate = ifs_is_fully_uptodate(folio, ifs); + spin_unlock_irq(&ifs->state_lock); + if (end_read) + folio_end_read(folio, uptodate); + } else { + /* + * If a folio without an ifs is submitted to the IO helper, the + * read must be on the entire folio and the IO helper takes + * ownership of the folio. This means we should only enter + * iomap_read_end() for the !ifs case if no bytes were submitted + * to the IO helper, in which case we are responsible for + * unlocking the folio here. + */ + WARN_ON_ONCE(bytes_submitted); + folio_unlock(folio); + } +} + +static int iomap_read_folio_iter(struct iomap_iter *iter, + struct iomap_read_folio_ctx *ctx, size_t *bytes_submitted) +{ + const struct iomap *iomap = &iter->iomap; + loff_t pos = iter->pos; + loff_t length = iomap_length(iter); + struct folio *folio = ctx->cur_folio; + size_t folio_len = folio_size(folio); + struct iomap_folio_state *ifs; + size_t poff, plen; + loff_t pos_diff; + int ret; + + if (iomap->type == IOMAP_INLINE) { + ret = iomap_read_inline_data(iter, folio); + if (ret) + return ret; + return iomap_iter_advance(iter, length); + } + + ifs = ifs_alloc(iter->inode, folio, iter->flags); + + length = min_t(loff_t, length, folio_len - offset_in_folio(folio, pos)); + while (length) { + iomap_adjust_read_range(iter->inode, folio, &pos, length, &poff, + &plen); + + pos_diff = pos - iter->pos; + if (WARN_ON_ONCE(pos_diff + plen > length)) + return -EIO; + + ret = iomap_iter_advance(iter, pos_diff); + if (ret) + return ret; + + if (plen == 0) + return 0; + + /* + * Handling of fsverity "holes". We hit this for two case: + * 1. No need to go further, the hole after fsverity + * descriptor is the end of the fsverity metadata. + * + * 2. This folio contains merkle tree blocks which need to be + * synthesized. If we already have fsverity info (ctx->vi) + * synthesize these blocks. + */ + if ((iomap->flags & IOMAP_F_FSVERITY) && + iomap->type == IOMAP_HOLE) { + if (ctx->vi) + fsverity_fill_zerohash(folio, poff, plen, + ctx->vi); + iomap_set_range_uptodate(folio, poff, plen); + } else if (iomap_block_needs_zeroing(iter, pos)) { + /* zero post-eof blocks as the page may be mapped */ + folio_zero_range(folio, poff, plen); + if (ctx->vi && + !fsverity_verify_blocks(ctx->vi, folio, plen, poff)) + return -EIO; + iomap_set_range_uptodate(folio, poff, plen); + } else { + if (!*bytes_submitted) + iomap_read_init(folio); + ret = ctx->ops->read_folio_range(iter, ctx, plen); + if (ret < 0) + fserror_report_io(iter->inode, + FSERR_BUFFERED_READ, pos, + plen, ret, GFP_NOFS); + if (ret) + return ret; + + *bytes_submitted += plen; + /* + * Hand off folio ownership to the IO helper when: + * 1) The entire folio has been submitted for IO, or + * 2) There is no ifs attached to the folio + * + * Case (2) occurs when 1 << i_blkbits matches the folio + * size but the underlying filesystem or block device + * uses a smaller granularity for IO. + */ + if (*bytes_submitted == folio_len || !ifs) + ctx->cur_folio = NULL; + } + + ret = iomap_iter_advance(iter, plen); + if (ret) + return ret; + length -= pos_diff + plen; + pos = iter->pos; + } + return 0; +} + +void iomap_read_folio(const struct iomap_ops *ops, + struct iomap_read_folio_ctx *ctx, void *private) +{ + struct folio *folio = ctx->cur_folio; + struct iomap_iter iter = { + .inode = folio->mapping->host, + .pos = folio_pos(folio), + .len = folio_size(folio), + .private = private, + }; + size_t bytes_submitted = 0; + int ret; + + trace_iomap_readpage(iter.inode, 1); + + /* + * Fetch fsverity_info for both data and fsverity metadata, as iomap + * needs zeroed hash for merkle tree block synthesis + */ + ctx->vi = fsverity_get_info(iter.inode); + if (ctx->vi && iter.pos < i_size_read(iter.inode)) + fsverity_readahead(ctx->vi, folio->index, + folio_nr_pages(folio)); + + while ((ret = iomap_iter(&iter, ops)) > 0) { + iter.status = iomap_read_folio_iter(&iter, ctx, + &bytes_submitted); + if (ctx->read_ctx && ctx->ops->submit_read) + ctx->ops->submit_read(&iter, ctx); + } + + if (ctx->cur_folio) + iomap_read_end(ctx->cur_folio, bytes_submitted); +} +EXPORT_SYMBOL_GPL(iomap_read_folio); + +static int iomap_readahead_iter(struct iomap_iter *iter, + struct iomap_read_folio_ctx *ctx, size_t *cur_bytes_submitted) +{ + int ret; + + while (iomap_length(iter)) { + if (ctx->cur_folio && + offset_in_folio(ctx->cur_folio, iter->pos) == 0) { + iomap_read_end(ctx->cur_folio, *cur_bytes_submitted); + ctx->cur_folio = NULL; + } + if (!ctx->cur_folio) { + ctx->cur_folio = readahead_folio(ctx->rac); + if (WARN_ON_ONCE(!ctx->cur_folio)) + return -EINVAL; + *cur_bytes_submitted = 0; + } + ret = iomap_read_folio_iter(iter, ctx, cur_bytes_submitted); + if (ret) + return ret; + } + + return 0; +} + +/** + * iomap_readahead - Attempt to read pages from a file. + * @ops: The operations vector for the filesystem. + * @ctx: The ctx used for issuing readahead. + * @private: The filesystem-specific information for issuing iomap_iter. + * + * This function is for filesystems to call to implement their readahead + * address_space operation. + * + * Context: The @ops callbacks may submit I/O (eg to read the addresses of + * blocks from disc), and may wait for it. The caller may be trying to + * access a different page, and so sleeping excessively should be avoided. + * It may allocate memory, but should avoid costly allocations. This + * function is called with memalloc_nofs set, so allocations will not cause + * the filesystem to be reentered. + */ +void iomap_readahead(const struct iomap_ops *ops, + struct iomap_read_folio_ctx *ctx, void *private) +{ + struct readahead_control *rac = ctx->rac; + struct iomap_iter iter = { + .inode = rac->mapping->host, + .pos = readahead_pos(rac), + .len = readahead_length(rac), + .private = private, + }; + size_t cur_bytes_submitted; + + trace_iomap_readahead(rac->mapping->host, readahead_count(rac)); + + /* + * Fetch fsverity_info for both data and fsverity metadata, as iomap + * needs zeroed hash for merkle tree block synthesis + */ + ctx->vi = fsverity_get_info(iter.inode); + if (ctx->vi && iter.pos < i_size_read(iter.inode)) + fsverity_readahead(ctx->vi, readahead_index(rac), + readahead_count(rac)); + + while (iomap_iter(&iter, ops) > 0) { + iter.status = iomap_readahead_iter(&iter, ctx, + &cur_bytes_submitted); + if (ctx->read_ctx && ctx->ops->submit_read) + ctx->ops->submit_read(&iter, ctx); + } + + if (ctx->cur_folio) + iomap_read_end(ctx->cur_folio, cur_bytes_submitted); +} +EXPORT_SYMBOL_GPL(iomap_readahead); + +/* + * iomap_is_partially_uptodate checks whether blocks within a folio are + * uptodate or not. + * + * Returns true if all blocks which correspond to the specified part + * of the folio are uptodate. + */ +bool iomap_is_partially_uptodate(struct folio *folio, size_t from, size_t count) +{ + struct iomap_folio_state *ifs = folio->private; + struct inode *inode = folio->mapping->host; + unsigned first, last; + + if (!ifs) + return false; + + /* Caller's range may extend past the end of this folio */ + count = min(folio_size(folio) - from, count); + + /* First and last blocks in range within folio */ + first = from >> inode->i_blkbits; + last = (from + count - 1) >> inode->i_blkbits; + + return ifs_next_nonuptodate_block(folio, first, last) > last; +} +EXPORT_SYMBOL_GPL(iomap_is_partially_uptodate); + +/** + * iomap_get_folio - get a folio reference for writing + * @iter: iteration structure + * @pos: start offset of write + * @len: Suggested size of folio to create. + * + * Returns a locked reference to the folio at @pos, or an error pointer if the + * folio could not be obtained. + */ +struct folio *iomap_get_folio(struct iomap_iter *iter, loff_t pos, size_t len) +{ + fgf_t fgp = FGP_WRITEBEGIN; + + if (iter->flags & IOMAP_NOWAIT) + fgp |= FGP_NOWAIT; + if (iter->flags & IOMAP_DONTCACHE) + fgp |= FGP_DONTCACHE; + fgp |= fgf_set_order(len); + + return __filemap_get_folio(iter->inode->i_mapping, pos >> PAGE_SHIFT, + fgp, mapping_gfp_mask(iter->inode->i_mapping)); +} +EXPORT_SYMBOL_GPL(iomap_get_folio); + +bool iomap_release_folio(struct folio *folio, gfp_t gfp_flags) +{ + trace_iomap_release_folio(folio->mapping->host, folio_pos(folio), + folio_size(folio)); + + /* + * If the folio is dirty, we refuse to release our metadata because + * it may be partially dirty. Once we track per-block dirty state, + * we can release the metadata if every block is dirty. + */ + if (folio_test_dirty(folio)) + return false; + ifs_free(folio); + return true; +} +EXPORT_SYMBOL_GPL(iomap_release_folio); + +void iomap_invalidate_folio(struct folio *folio, size_t offset, size_t len) +{ + trace_iomap_invalidate_folio(folio->mapping->host, + folio_pos(folio) + offset, len); + + /* + * If we're invalidating the entire folio, clear the dirty state + * from it and release it to avoid unnecessary buildup of the LRU. + */ + if (offset == 0 && len == folio_size(folio)) { + WARN_ON_ONCE(folio_test_writeback(folio)); + folio_cancel_dirty(folio); + ifs_free(folio); + } else { + iomap_clear_range_dirty(folio, offset, len); + } +} +EXPORT_SYMBOL_GPL(iomap_invalidate_folio); + +bool iomap_dirty_folio(struct address_space *mapping, struct folio *folio) +{ + struct inode *inode = mapping->host; + size_t len = folio_size(folio); + + ifs_alloc(inode, folio, 0); + iomap_set_range_dirty(folio, 0, len); + return filemap_dirty_folio(mapping, folio); +} +EXPORT_SYMBOL_GPL(iomap_dirty_folio); + +static void +iomap_write_failed(struct inode *inode, loff_t pos, unsigned len) +{ + loff_t i_size = i_size_read(inode); + + /* + * Only truncate newly allocated pages beyoned EOF, even if the + * write started inside the existing inode size. + */ + if (pos + len > i_size) + truncate_pagecache_range(inode, max(pos, i_size), + pos + len - 1); +} + +static int __iomap_write_begin(const struct iomap_iter *iter, + const struct iomap_write_ops *write_ops, size_t len, + struct folio *folio) +{ + struct iomap_folio_state *ifs; + loff_t pos = iter->pos; + loff_t block_size = i_blocksize(iter->inode); + loff_t block_start = round_down(pos, block_size); + loff_t block_end = round_up(pos + len, block_size); + unsigned int nr_blocks = i_blocks_per_folio(iter->inode, folio); + size_t from = offset_in_folio(folio, pos), to = from + len; + size_t poff, plen; + + /* + * If the write or zeroing completely overlaps the current folio, then + * entire folio will be dirtied so there is no need for + * per-block state tracking structures to be attached to this folio. + * For the unshare case, we must read in the ondisk contents because we + * are not changing pagecache contents. + */ + if (!(iter->flags & IOMAP_UNSHARE) && pos <= folio_pos(folio) && + pos + len >= folio_next_pos(folio)) + return 0; + + ifs = ifs_alloc(iter->inode, folio, iter->flags); + if ((iter->flags & IOMAP_NOWAIT) && !ifs && nr_blocks > 1) + return -EAGAIN; + + if (folio_test_uptodate(folio)) + return 0; + + do { + iomap_adjust_read_range(iter->inode, folio, &block_start, + block_end - block_start, &poff, &plen); + if (plen == 0) + break; + + /* + * If the read range will be entirely overwritten by the write, + * we can skip having to zero/read it in. + */ + if (!(iter->flags & IOMAP_UNSHARE) && from <= poff && + to >= poff + plen) + continue; + + if (iomap_block_needs_zeroing(iter, block_start)) { + if (WARN_ON_ONCE(iter->flags & IOMAP_UNSHARE)) + return -EIO; + folio_zero_segments(folio, poff, from, to, poff + plen); + } else { + const struct iomap *iomap = iomap_iter_srcmap(iter); + int status; + + if (iter->flags & IOMAP_NOWAIT) + return -EAGAIN; + + if (write_ops && write_ops->read_folio_range) + status = write_ops->read_folio_range(iter, + folio, block_start, plen); + else + status = iomap_bio_read_folio_range_sync(iter, + folio, block_start, plen); + if (status < 0) + fserror_report_io(iter->inode, + FSERR_BUFFERED_READ, pos, + plen, status, GFP_NOFS); + if (status) + return status; + + if (iomap->flags & IOMAP_F_ZERO_TAIL) + folio_zero_segment(folio, to, poff + plen); + } + iomap_set_range_uptodate(folio, poff, plen); + } while ((block_start += plen) < block_end); + + return 0; +} + +static struct folio *__iomap_get_folio(struct iomap_iter *iter, + const struct iomap_write_ops *write_ops, size_t len) +{ + loff_t pos = iter->pos; + + if (!mapping_large_folio_support(iter->inode->i_mapping)) + len = min_t(size_t, len, PAGE_SIZE - offset_in_page(pos)); + + if (iter->iomap.flags & IOMAP_F_FOLIO_BATCH) { + struct folio *folio = folio_batch_next(iter->fbatch); + + if (!folio) + return NULL; + + /* + * The folio mapping generally shouldn't have changed based on + * fs locks, but be consistent with filemap lookup and retry + * the iter if it does. + */ + folio_lock(folio); + if (unlikely(folio->mapping != iter->inode->i_mapping)) { + iter->iomap.flags |= IOMAP_F_STALE; + folio_unlock(folio); + return NULL; + } + + folio_get(folio); + folio_wait_stable(folio); + return folio; + } + + if (write_ops && write_ops->get_folio) + return write_ops->get_folio(iter, pos, len); + return iomap_get_folio(iter, pos, len); +} + +static void __iomap_put_folio(struct iomap_iter *iter, + const struct iomap_write_ops *write_ops, size_t ret, + struct folio *folio) +{ + loff_t pos = iter->pos; + + if (write_ops && write_ops->put_folio) { + write_ops->put_folio(iter->inode, pos, ret, folio); + } else { + folio_unlock(folio); + folio_put(folio); + } +} + +/* trim pos and bytes to within a given folio */ +static loff_t iomap_trim_folio_range(struct iomap_iter *iter, + struct folio *folio, size_t *offset, u64 *bytes) +{ + loff_t pos = iter->pos; + size_t fsize = folio_size(folio); + + WARN_ON_ONCE(pos < folio_pos(folio)); + WARN_ON_ONCE(pos >= folio_pos(folio) + fsize); + + *offset = offset_in_folio(folio, pos); + *bytes = min(*bytes, fsize - *offset); + + return pos; +} + +static int iomap_write_begin_inline(const struct iomap_iter *iter, + struct folio *folio) +{ + /* needs more work for the tailpacking case; disable for now */ + if (WARN_ON_ONCE(iomap_iter_srcmap(iter)->offset != 0)) + return -EIO; + return iomap_read_inline_data(iter, folio); +} + +/* + * Grab and prepare a folio for write based on iter state. Returns the folio, + * offset, and length. Callers can optionally pass a max length *plen, + * otherwise init to zero. + */ +static int iomap_write_begin(struct iomap_iter *iter, + const struct iomap_write_ops *write_ops, struct folio **foliop, + size_t *poffset, u64 *plen) +{ + const struct iomap *srcmap = iomap_iter_srcmap(iter); + loff_t pos; + u64 len = min_t(u64, SIZE_MAX, iomap_length(iter)); + struct folio *folio; + int status = 0; + + len = min_not_zero(len, *plen); + *foliop = NULL; + *plen = 0; + + if (fatal_signal_pending(current)) + return -EINTR; + + folio = __iomap_get_folio(iter, write_ops, len); + if (IS_ERR(folio)) + return PTR_ERR(folio); + + /* + * No folio means we're done with a batch. We still have range to + * process so return and let the caller iterate and refill the batch. + */ + if (!folio) { + WARN_ON_ONCE(!(iter->iomap.flags & IOMAP_F_FOLIO_BATCH)); + return 0; + } + + /* + * Now we have a locked folio, before we do anything with it we need to + * check that the iomap we have cached is not stale. The inode extent + * mapping can change due to concurrent IO in flight (e.g. + * IOMAP_UNWRITTEN state can change and memory reclaim could have + * reclaimed a previously partially written page at this index after IO + * completion before this write reaches this file offset) and hence we + * could do the wrong thing here (zero a page range incorrectly or fail + * to zero) and corrupt data. + */ + if (write_ops && write_ops->iomap_valid) { + bool iomap_valid = write_ops->iomap_valid(iter->inode, + &iter->iomap); + if (!iomap_valid) { + iter->iomap.flags |= IOMAP_F_STALE; + status = 0; + goto out_unlock; + } + } + + /* + * The folios in a batch may not be contiguous. If we've skipped + * forward, advance the iter to the pos of the current folio. If the + * folio starts beyond the end of the mapping, it may have been trimmed + * since the lookup for whatever reason. Return a NULL folio to + * terminate the op. + */ + if (folio_pos(folio) > iter->pos) { + len = min_t(u64, folio_pos(folio) - iter->pos, + iomap_length(iter)); + status = iomap_iter_advance(iter, len); + len = iomap_length(iter); + if (status || !len) + goto out_unlock; + } + + pos = iomap_trim_folio_range(iter, folio, poffset, &len); + + if (srcmap->type == IOMAP_INLINE) + status = iomap_write_begin_inline(iter, folio); + else if (srcmap->flags & IOMAP_F_BUFFER_HEAD) + status = __block_write_begin_int(folio, pos, len, NULL, srcmap); + else + status = __iomap_write_begin(iter, write_ops, len, folio); + + if (unlikely(status)) + goto out_unlock; + + *foliop = folio; + *plen = len; + return 0; + +out_unlock: + __iomap_put_folio(iter, write_ops, 0, folio); + return status; +} + +static bool __iomap_write_end(struct inode *inode, loff_t pos, size_t len, + size_t copied, struct folio *folio) +{ + flush_dcache_folio(folio); + + /* + * The blocks that were entirely written will now be uptodate, so we + * don't have to worry about a read_folio reading them and overwriting a + * partial write. However, if we've encountered a short write and only + * partially written into a block, it will not be marked uptodate, so a + * read_folio might come in and destroy our partial write. + * + * Do the simplest thing and just treat any short write to a + * non-uptodate page as a zero-length write, and force the caller to + * redo the whole thing. + */ + if (unlikely(copied < len && !folio_test_uptodate(folio))) + return false; + iomap_set_range_uptodate(folio, offset_in_folio(folio, pos), len); + iomap_set_range_dirty(folio, offset_in_folio(folio, pos), copied); + filemap_dirty_folio(inode->i_mapping, folio); + return true; +} + +static bool iomap_write_end_inline(const struct iomap_iter *iter, + struct folio *folio, loff_t pos, size_t copied) +{ + const struct iomap *iomap = &iter->iomap; + void *addr; + + WARN_ON_ONCE(!folio_test_uptodate(folio)); + + if (WARN_ON_ONCE(!iomap->inline_data)) + return false; + + flush_dcache_folio(folio); + addr = kmap_local_folio(folio, pos); + memcpy(iomap_inline_data(iomap, pos), addr, copied); + kunmap_local(addr); + + mark_inode_dirty(iter->inode); + return true; +} + +/* + * Returns true if all copied bytes have been written to the pagecache, + * otherwise return false. + */ +static bool iomap_write_end(struct iomap_iter *iter, size_t len, size_t copied, + struct folio *folio) +{ + const struct iomap *srcmap = iomap_iter_srcmap(iter); + loff_t pos = iter->pos; + + if (srcmap->type == IOMAP_INLINE) + return iomap_write_end_inline(iter, folio, pos, copied); + + if (srcmap->flags & IOMAP_F_BUFFER_HEAD) { + size_t bh_written; + + bh_written = block_write_end(pos, len, copied, folio); + WARN_ON_ONCE(bh_written != copied && bh_written != 0); + return bh_written == copied; + } + + return __iomap_write_end(iter->inode, pos, len, copied, folio); +} + +static int iomap_write_iter(struct iomap_iter *iter, struct iov_iter *i, + const struct iomap_write_ops *write_ops) +{ + int status = 0; + struct address_space *mapping = iter->inode->i_mapping; + size_t chunk = mapping_max_folio_size(mapping); + unsigned int bdp_flags = (iter->flags & IOMAP_NOWAIT) ? BDP_ASYNC : 0; + + do { + struct folio *folio; + loff_t old_size; + size_t offset; /* Offset into folio */ + u64 bytes; /* Bytes to write to folio */ + size_t copied; /* Bytes copied from user */ + u64 written; /* Bytes have been written */ + loff_t pos; + + bytes = iov_iter_count(i); +retry: + offset = iter->pos & (chunk - 1); + bytes = min(chunk - offset, bytes); + status = balance_dirty_pages_ratelimited_flags(mapping, + bdp_flags); + if (unlikely(status)) + break; + + if (bytes > iomap_length(iter)) + bytes = iomap_length(iter); + + /* + * Bring in the user page that we'll copy from _first_. + * Otherwise there's a nasty deadlock on copying from the + * same page as we're writing to, without it being marked + * up-to-date. + * + * For async buffered writes the assumption is that the user + * page has already been faulted in. This can be optimized by + * faulting the user page. + */ + if (unlikely(fault_in_iov_iter_readable(i, bytes) == bytes)) { + status = -EFAULT; + break; + } + + status = iomap_write_begin(iter, write_ops, &folio, &offset, + &bytes); + if (unlikely(status)) { + iomap_write_failed(iter->inode, iter->pos, bytes); + break; + } + if (iter->iomap.flags & IOMAP_F_STALE) + break; + + pos = iter->pos; + + if (mapping_writably_mapped(mapping)) + flush_dcache_folio(folio); + + copied = copy_folio_from_iter_atomic(folio, offset, bytes, i); + written = iomap_write_end(iter, bytes, copied, folio) ? + copied : 0; + + /* + * Update the in-memory inode size after copying the data into + * the page cache. It's up to the file system to write the + * updated size to disk, preferably after I/O completion so that + * no stale data is exposed. Only once that's done can we + * unlock and release the folio. + */ + old_size = iter->inode->i_size; + if (pos + written > old_size && + !(iter->iomap.flags & IOMAP_F_FSVERITY)) { + i_size_write(iter->inode, pos + written); + iter->iomap.flags |= IOMAP_F_SIZE_CHANGED; + } + __iomap_put_folio(iter, write_ops, written, folio); + + if (old_size < pos && !(iter->iomap.flags & IOMAP_F_FSVERITY)) + pagecache_isize_extended(iter->inode, old_size, pos); + + cond_resched(); + if (unlikely(written == 0)) { + /* + * A short copy made iomap_write_end() reject the + * thing entirely. Might be memory poisoning + * halfway through, might be a race with munmap, + * might be severe memory pressure. + */ + iomap_write_failed(iter->inode, pos, bytes); + iov_iter_revert(i, copied); + + if (chunk > PAGE_SIZE) + chunk /= 2; + if (copied) { + bytes = copied; + goto retry; + } + } else { + iomap_iter_advance(iter, written); + } + } while (iov_iter_count(i) && iomap_length(iter)); + + return status; +} + +ssize_t +iomap_file_buffered_write(struct kiocb *iocb, struct iov_iter *i, + const struct iomap_ops *ops, + const struct iomap_write_ops *write_ops, void *private) +{ + struct iomap_iter iter = { + .inode = iocb->ki_filp->f_mapping->host, + .pos = iocb->ki_pos, + .len = iov_iter_count(i), + .flags = IOMAP_WRITE, + .private = private, + }; + ssize_t ret; + + if (iocb->ki_flags & IOCB_NOWAIT) + iter.flags |= IOMAP_NOWAIT; + if (iocb->ki_flags & IOCB_DONTCACHE) + iter.flags |= IOMAP_DONTCACHE; + + while ((ret = iomap_iter(&iter, ops)) > 0) + iter.status = iomap_write_iter(&iter, i, write_ops); + + if (unlikely(iter.pos == iocb->ki_pos)) + return ret; + ret = iter.pos - iocb->ki_pos; + iocb->ki_pos = iter.pos; + return ret; +} +EXPORT_SYMBOL_GPL(iomap_file_buffered_write); + +int iomap_fsverity_write(struct file *file, loff_t pos, size_t length, + const void *buf, const struct iomap_ops *ops, + const struct iomap_write_ops *write_ops) +{ + int ret; + struct iov_iter iiter; + struct kvec kvec = { + .iov_base = (void *)buf, + .iov_len = length, + }; + struct kiocb iocb = { + .ki_filp = file, + .ki_ioprio = get_current_ioprio(), + .ki_pos = pos, + }; + + iov_iter_kvec(&iiter, WRITE, &kvec, 1, length); + + ret = iomap_file_buffered_write(&iocb, &iiter, ops, write_ops, NULL); + if (ret < 0) + return ret; + return ret == length ? 0 : -EIO; +} +EXPORT_SYMBOL_GPL(iomap_fsverity_write); + +static void iomap_write_delalloc_ifs_punch(struct inode *inode, + struct folio *folio, loff_t start_byte, loff_t end_byte, + struct iomap *iomap, iomap_punch_t punch) +{ + unsigned int first_blk, last_blk; + loff_t last_byte; + u8 blkbits = inode->i_blkbits; + struct iomap_folio_state *ifs; + + /* + * When we have per-block dirty tracking, there can be + * blocks within a folio which are marked uptodate + * but not dirty. In that case it is necessary to punch + * out such blocks to avoid leaking any delalloc blocks. + */ + ifs = folio->private; + if (!ifs) + return; + + last_byte = min_t(loff_t, end_byte - 1, folio_next_pos(folio) - 1); + first_blk = offset_in_folio(folio, start_byte) >> blkbits; + last_blk = offset_in_folio(folio, last_byte) >> blkbits; + while ((first_blk = ifs_next_clean_block(folio, first_blk, last_blk)) + <= last_blk) { + punch(inode, folio_pos(folio) + (first_blk << blkbits), + 1 << blkbits, iomap); + first_blk++; + } +} + +static void iomap_write_delalloc_punch(struct inode *inode, struct folio *folio, + loff_t *punch_start_byte, loff_t start_byte, loff_t end_byte, + struct iomap *iomap, iomap_punch_t punch) +{ + if (!folio_test_dirty(folio)) + return; + + /* if dirty, punch up to offset */ + if (start_byte > *punch_start_byte) { + punch(inode, *punch_start_byte, start_byte - *punch_start_byte, + iomap); + } + + /* Punch non-dirty blocks within folio */ + iomap_write_delalloc_ifs_punch(inode, folio, start_byte, end_byte, + iomap, punch); + + /* + * Make sure the next punch start is correctly bound to + * the end of this data range, not the end of the folio. + */ + *punch_start_byte = min_t(loff_t, end_byte, folio_next_pos(folio)); +} + +/* + * Scan the data range passed to us for dirty page cache folios. If we find a + * dirty folio, punch out the preceding range and update the offset from which + * the next punch will start from. + * + * We can punch out storage reservations under clean pages because they either + * contain data that has been written back - in which case the delalloc punch + * over that range is a no-op - or they have been read faults in which case they + * contain zeroes and we can remove the delalloc backing range and any new + * writes to those pages will do the normal hole filling operation... + * + * This makes the logic simple: we only need to keep the delalloc extents only + * over the dirty ranges of the page cache. + * + * This function uses [start_byte, end_byte) intervals (i.e. open ended) to + * simplify range iterations. + */ +static void iomap_write_delalloc_scan(struct inode *inode, + loff_t *punch_start_byte, loff_t start_byte, loff_t end_byte, + struct iomap *iomap, iomap_punch_t punch) +{ + while (start_byte < end_byte) { + struct folio *folio; + + /* grab locked page */ + folio = filemap_lock_folio(inode->i_mapping, + start_byte >> PAGE_SHIFT); + if (IS_ERR(folio)) { + start_byte = ALIGN_DOWN(start_byte, PAGE_SIZE) + + PAGE_SIZE; + continue; + } + + iomap_write_delalloc_punch(inode, folio, punch_start_byte, + start_byte, end_byte, iomap, punch); + + /* move offset to start of next folio in range */ + start_byte = folio_next_pos(folio); + folio_unlock(folio); + folio_put(folio); + } +} + +/* + * When a short write occurs, the filesystem might need to use ->iomap_end + * to remove space reservations created in ->iomap_begin. + * + * For filesystems that use delayed allocation, there can be dirty pages over + * the delalloc extent outside the range of a short write but still within the + * delalloc extent allocated for this iomap if the write raced with page + * faults. + * + * Punch out all the delalloc blocks in the range given except for those that + * have dirty data still pending in the page cache - those are going to be + * written and so must still retain the delalloc backing for writeback. + * + * The punch() callback *must* only punch delalloc extents in the range passed + * to it. It must skip over all other types of extents in the range and leave + * them completely unchanged. It must do this punch atomically with respect to + * other extent modifications. + * + * The punch() callback may be called with a folio locked to prevent writeback + * extent allocation racing at the edge of the range we are currently punching. + * The locked folio may or may not cover the range being punched, so it is not + * safe for the punch() callback to lock folios itself. + * + * Lock order is: + * + * inode->i_rwsem (shared or exclusive) + * inode->i_mapping->invalidate_lock (exclusive) + * folio_lock() + * ->punch + * internal filesystem allocation lock + * + * As we are scanning the page cache for data, we don't need to reimplement the + * wheel - mapping_seek_hole_data() does exactly what we need to identify the + * start and end of data ranges correctly even for sub-folio block sizes. This + * byte range based iteration is especially convenient because it means we + * don't have to care about variable size folios, nor where the start or end of + * the data range lies within a folio, if they lie within the same folio or even + * if there are multiple discontiguous data ranges within the folio. + * + * It should be noted that mapping_seek_hole_data() is not aware of EOF, and so + * can return data ranges that exist in the cache beyond EOF. e.g. a page fault + * spanning EOF will initialise the post-EOF data to zeroes and mark it up to + * date. A write page fault can then mark it dirty. If we then fail a write() + * beyond EOF into that up to date cached range, we allocate a delalloc block + * beyond EOF and then have to punch it out. Because the range is up to date, + * mapping_seek_hole_data() will return it, and we will skip the punch because + * the folio is dirty. THis is incorrect - we always need to punch out delalloc + * beyond EOF in this case as writeback will never write back and covert that + * delalloc block beyond EOF. Hence we limit the cached data scan range to EOF, + * resulting in always punching out the range from the EOF to the end of the + * range the iomap spans. + * + * Intervals are of the form [start_byte, end_byte) (i.e. open ended) because it + * matches the intervals returned by mapping_seek_hole_data(). i.e. SEEK_DATA + * returns the start of a data range (start_byte), and SEEK_HOLE(start_byte) + * returns the end of the data range (data_end). Using closed intervals would + * require sprinkling this code with magic "+ 1" and "- 1" arithmetic and expose + * the code to subtle off-by-one bugs.... + */ +void iomap_write_delalloc_release(struct inode *inode, loff_t start_byte, + loff_t end_byte, unsigned flags, struct iomap *iomap, + iomap_punch_t punch) +{ + loff_t punch_start_byte = start_byte; + loff_t scan_end_byte = min(i_size_read(inode), end_byte); + + /* + * The caller must hold invalidate_lock to avoid races with page faults + * re-instantiating folios and dirtying them via ->page_mkwrite whilst + * we walk the cache and perform delalloc extent removal. Failing to do + * this can leave dirty pages with no space reservation in the cache. + */ + lockdep_assert_held_write(&inode->i_mapping->invalidate_lock); + + while (start_byte < scan_end_byte) { + loff_t data_end; + + start_byte = mapping_seek_hole_data(inode->i_mapping, + start_byte, scan_end_byte, SEEK_DATA); + /* + * If there is no more data to scan, all that is left is to + * punch out the remaining range. + * + * Note that mapping_seek_hole_data is only supposed to return + * either an offset or -ENXIO, so WARN on any other error as + * that would be an API change without updating the callers. + */ + if (start_byte == -ENXIO || start_byte == scan_end_byte) + break; + if (WARN_ON_ONCE(start_byte < 0)) + return; + WARN_ON_ONCE(start_byte < punch_start_byte); + WARN_ON_ONCE(start_byte > scan_end_byte); + + /* + * We find the end of this contiguous cached data range by + * seeking from start_byte to the beginning of the next hole. + */ + data_end = mapping_seek_hole_data(inode->i_mapping, start_byte, + scan_end_byte, SEEK_HOLE); + if (WARN_ON_ONCE(data_end < 0)) + return; + + /* + * If we race with post-direct I/O invalidation of the page cache, + * there might be no data left at start_byte. + */ + if (data_end == start_byte) + continue; + + WARN_ON_ONCE(data_end < start_byte); + WARN_ON_ONCE(data_end > scan_end_byte); + + iomap_write_delalloc_scan(inode, &punch_start_byte, start_byte, + data_end, iomap, punch); + + /* The next data search starts at the end of this one. */ + start_byte = data_end; + } + + if (punch_start_byte < end_byte) + punch(inode, punch_start_byte, end_byte - punch_start_byte, + iomap); +} +EXPORT_SYMBOL_GPL(iomap_write_delalloc_release); + +static int iomap_unshare_iter(struct iomap_iter *iter, + const struct iomap_write_ops *write_ops) +{ + struct iomap *iomap = &iter->iomap; + u64 bytes = iomap_length(iter); + int status; + + if (!iomap_want_unshare_iter(iter)) + return iomap_iter_advance(iter, bytes); + + do { + struct folio *folio; + size_t offset; + bool ret; + + bytes = min_t(u64, SIZE_MAX, bytes); + status = iomap_write_begin(iter, write_ops, &folio, &offset, + &bytes); + if (unlikely(status)) + return status; + if (iomap->flags & IOMAP_F_STALE) + break; + + ret = iomap_write_end(iter, bytes, bytes, folio); + __iomap_put_folio(iter, write_ops, bytes, folio); + if (WARN_ON_ONCE(!ret)) + return -EIO; + + cond_resched(); + + balance_dirty_pages_ratelimited(iter->inode->i_mapping); + + status = iomap_iter_advance(iter, bytes); + if (status) + break; + } while ((bytes = iomap_length(iter)) > 0); + + return status; +} + +int +iomap_file_unshare(struct inode *inode, loff_t pos, loff_t len, + const struct iomap_ops *ops, + const struct iomap_write_ops *write_ops) +{ + struct iomap_iter iter = { + .inode = inode, + .pos = pos, + .flags = IOMAP_WRITE | IOMAP_UNSHARE, + }; + loff_t size = i_size_read(inode); + int ret; + + if (pos < 0 || pos >= size) + return 0; + + iter.len = min(len, size - pos); + while ((ret = iomap_iter(&iter, ops)) > 0) + iter.status = iomap_unshare_iter(&iter, write_ops); + return ret; +} +EXPORT_SYMBOL_GPL(iomap_file_unshare); + +/* + * Flush the remaining range of the iter and mark the current mapping stale. + * This is used when zero range sees an unwritten mapping that may have had + * dirty pagecache over it. + */ +static inline int iomap_zero_iter_flush_and_stale(struct iomap_iter *i) +{ + struct address_space *mapping = i->inode->i_mapping; + loff_t end = i->pos + i->len - 1; + + i->iomap.flags |= IOMAP_F_STALE; + return filemap_write_and_wait_range(mapping, i->pos, end); +} + +static int iomap_zero_iter(struct iomap_iter *iter, bool *did_zero, + const struct iomap_write_ops *write_ops) +{ + u64 bytes = iomap_length(iter); + bool zeroed = false; + int status; + + do { + struct folio *folio; + size_t offset; + bool ret; + + balance_dirty_pages_ratelimited(iter->inode->i_mapping); + + bytes = min_t(u64, SIZE_MAX, bytes); + status = iomap_write_begin(iter, write_ops, &folio, &offset, + &bytes); + if (status) + return status; + if (iter->iomap.flags & IOMAP_F_STALE) + break; + + /* a NULL folio means we're done with a folio batch */ + if (!folio) { + status = iomap_iter_advance_full(iter); + if (status) + return status; + break; + } + + /* warn about zeroing folios beyond eof that won't write back */ + WARN_ON_ONCE(folio_pos(folio) > iter->inode->i_size); + + trace_iomap_zero_iter(iter->inode, folio_pos(folio) + offset, + bytes); + + folio_zero_range(folio, offset, bytes); + zeroed = true; + folio_mark_accessed(folio); + + ret = iomap_write_end(iter, bytes, bytes, folio); + __iomap_put_folio(iter, write_ops, bytes, folio); + if (WARN_ON_ONCE(!ret)) + return -EIO; + + status = iomap_iter_advance(iter, bytes); + if (status) + return status; + } while ((bytes = iomap_length(iter)) > 0); + + if (did_zero && zeroed) + *did_zero = true; + return status; +} + +/** + * iomap_fill_dirty_folios - fill a folio batch with dirty folios + * @iter: Iteration structure + * @start: Start offset of range. Updated based on lookup progress. + * @end: End offset of range + * @iomap_flags: Flags to set on the associated iomap to track the batch. + * + * Returns the folio count directly. Also returns the associated control flag if + * the the batch lookup is performed and the expected offset of a subsequent + * lookup via out params. The caller is responsible to set the flag on the + * associated iomap. + */ +unsigned int +iomap_fill_dirty_folios( + struct iomap_iter *iter, + loff_t *start, + loff_t end, + unsigned int *iomap_flags) +{ + struct address_space *mapping = iter->inode->i_mapping; + pgoff_t pstart = *start >> PAGE_SHIFT; + pgoff_t pend = (end - 1) >> PAGE_SHIFT; + unsigned int count; + + if (!iter->fbatch) { + *start = end; + return 0; + } + + count = filemap_get_folios_dirty(mapping, &pstart, pend, iter->fbatch); + *start = (pstart << PAGE_SHIFT); + *iomap_flags |= IOMAP_F_FOLIO_BATCH; + return count; +} +EXPORT_SYMBOL_GPL(iomap_fill_dirty_folios); + +int +iomap_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero, + const struct iomap_ops *ops, + const struct iomap_write_ops *write_ops, void *private) +{ + struct folio_batch fbatch; + struct iomap_iter iter = { + .inode = inode, + .pos = pos, + .len = len, + .flags = IOMAP_ZERO, + .private = private, + .fbatch = &fbatch, + }; + struct address_space *mapping = inode->i_mapping; + int ret; + bool range_dirty; + + folio_batch_init(&fbatch); + + /* + * To avoid an unconditional flush, check pagecache state and only flush + * if dirty and the fs returns a mapping that might convert on + * writeback. + */ + range_dirty = filemap_range_needs_writeback(mapping, iter.pos, + iter.pos + iter.len - 1); + while ((ret = iomap_iter(&iter, ops)) > 0) { + const struct iomap *srcmap = iomap_iter_srcmap(&iter); + + if (!(iter.iomap.flags & IOMAP_F_FOLIO_BATCH) && + (srcmap->type == IOMAP_HOLE || + srcmap->type == IOMAP_UNWRITTEN)) { + s64 status; + + if (range_dirty && srcmap->type == IOMAP_UNWRITTEN) { + range_dirty = false; + status = iomap_zero_iter_flush_and_stale(&iter); + } else { + status = iomap_iter_advance_full(&iter); + } + iter.status = status; + continue; + } + + iter.status = iomap_zero_iter(&iter, did_zero, write_ops); + } + return ret; +} +EXPORT_SYMBOL_GPL(iomap_zero_range); + +int +iomap_truncate_page(struct inode *inode, loff_t pos, bool *did_zero, + const struct iomap_ops *ops, + const struct iomap_write_ops *write_ops, void *private) +{ + unsigned int blocksize = i_blocksize(inode); + unsigned int off = pos & (blocksize - 1); + + /* Block boundary? Nothing to do */ + if (!off) + return 0; + return iomap_zero_range(inode, pos, blocksize - off, did_zero, ops, + write_ops, private); +} +EXPORT_SYMBOL_GPL(iomap_truncate_page); + +static int iomap_folio_mkwrite_iter(struct iomap_iter *iter, + struct folio *folio) +{ + loff_t length = iomap_length(iter); + int ret; + + if (iter->iomap.flags & IOMAP_F_BUFFER_HEAD) { + ret = __block_write_begin_int(folio, iter->pos, length, NULL, + &iter->iomap); + if (ret) + return ret; + block_commit_write(folio, 0, length); + } else { + WARN_ON_ONCE(!folio_test_uptodate(folio)); + folio_mark_dirty(folio); + } + + return iomap_iter_advance(iter, length); +} + +vm_fault_t iomap_page_mkwrite(struct vm_fault *vmf, const struct iomap_ops *ops, + void *private) +{ + struct iomap_iter iter = { + .inode = file_inode(vmf->vma->vm_file), + .flags = IOMAP_WRITE | IOMAP_FAULT, + .private = private, + }; + struct folio *folio = page_folio(vmf->page); + ssize_t ret; + + folio_lock(folio); + ret = folio_mkwrite_check_truncate(folio, iter.inode); + if (ret < 0) + goto out_unlock; + iter.pos = folio_pos(folio); + iter.len = ret; + while ((ret = iomap_iter(&iter, ops)) > 0) + iter.status = iomap_folio_mkwrite_iter(&iter, folio); + + if (ret < 0) + goto out_unlock; + folio_wait_stable(folio); + return VM_FAULT_LOCKED; +out_unlock: + folio_unlock(folio); + return vmf_fs_error(ret); +} +EXPORT_SYMBOL_GPL(iomap_page_mkwrite); + +static void iomap_writeback_init(struct inode *inode, struct folio *folio) +{ + struct iomap_folio_state *ifs = folio->private; + + WARN_ON_ONCE(i_blocks_per_folio(inode, folio) > 1 && !ifs); + if (ifs) { + WARN_ON_ONCE(atomic_read(&ifs->write_bytes_pending) != 0); + /* + * Set this to the folio size. After processing the folio for + * writeback in iomap_writeback_folio(), we'll subtract any + * ranges not written back. + * + * We do this because otherwise, we would have to atomically + * increment ifs->write_bytes_pending every time a range in the + * folio needs to be written back. + */ + atomic_set(&ifs->write_bytes_pending, folio_size(folio)); + } +} + +void iomap_finish_folio_write(struct inode *inode, struct folio *folio, + size_t len) +{ + struct iomap_folio_state *ifs = folio->private; + + WARN_ON_ONCE(i_blocks_per_folio(inode, folio) > 1 && !ifs); + WARN_ON_ONCE(ifs && atomic_read(&ifs->write_bytes_pending) <= 0); + + if (!ifs || atomic_sub_and_test(len, &ifs->write_bytes_pending)) + folio_end_writeback(folio); +} +EXPORT_SYMBOL_GPL(iomap_finish_folio_write); + +static int iomap_writeback_range(struct iomap_writepage_ctx *wpc, + struct folio *folio, u64 pos, u32 rlen, u64 end_pos, + size_t *bytes_submitted) +{ + do { + ssize_t ret; + + ret = wpc->ops->writeback_range(wpc, folio, pos, rlen, end_pos); + if (WARN_ON_ONCE(ret == 0 || ret > rlen)) + return -EIO; + if (ret < 0) + return ret; + rlen -= ret; + pos += ret; + + /* + * Holes are not written back by ->writeback_range, so track + * if we did handle anything that is not a hole here. + */ + if (wpc->iomap.type != IOMAP_HOLE) + *bytes_submitted += ret; + } while (rlen); + + return 0; +} + +/* + * Check interaction of the folio with the file end. + * + * If the folio is entirely beyond i_size, return false. If it straddles + * i_size, adjust end_pos and zero all data beyond i_size. Don't skip fsverity + * folios as those are beyond i_size. + */ +static bool iomap_writeback_handle_eof(struct folio *folio, + struct iomap_writepage_ctx *wpc, u64 *end_pos) +{ + struct inode *inode = wpc->inode; + u64 isize = i_size_read(inode); + + if (wpc->iomap.flags & IOMAP_F_FSVERITY) { + WARN_ON_ONCE(folio_pos(folio) < isize); + return true; + } + + if (*end_pos > isize) { + size_t poff = offset_in_folio(folio, isize); + pgoff_t end_index = isize >> PAGE_SHIFT; + + /* + * If the folio is entirely ouside of i_size, skip it. + * + * This can happen due to a truncate operation that is in + * progress and in that case truncate will finish it off once + * we've dropped the folio lock. + * + * Note that the pgoff_t used for end_index is an unsigned long. + * If the given offset is greater than 16TB on a 32-bit system, + * then if we checked if the folio is fully outside i_size with + * "if (folio->index >= end_index + 1)", "end_index + 1" would + * overflow and evaluate to 0. Hence this folio would be + * redirtied and written out repeatedly, which would result in + * an infinite loop; the user program performing this operation + * would hang. Instead, we can detect this situation by + * checking if the folio is totally beyond i_size or if its + * offset is just equal to the EOF. + */ + if (folio->index > end_index || + (folio->index == end_index && poff == 0)) + return false; + + /* + * The folio straddles i_size. + * + * It must be zeroed out on each and every writepage invocation + * because it may be mmapped: + * + * A file is mapped in multiples of the page size. For a + * file that is not a multiple of the page size, the + * remaining memory is zeroed when mapped, and writes to that + * region are not written out to the file. + * + * Also adjust the end_pos to the end of file and skip writeback + * for all blocks entirely beyond i_size. + */ + folio_zero_segment(folio, poff, folio_size(folio)); + *end_pos = isize; + } + + return true; +} + +int iomap_writeback_folio(struct iomap_writepage_ctx *wpc, struct folio *folio) +{ + struct iomap_folio_state *ifs = folio->private; + struct inode *inode = wpc->inode; + u64 pos = folio_pos(folio); + u64 end_pos = pos + folio_size(folio); + u64 end_aligned = 0; + loff_t orig_pos = pos; + size_t bytes_submitted = 0; + int error = 0; + u32 rlen; + + WARN_ON_ONCE(!folio_test_locked(folio)); + WARN_ON_ONCE(folio_test_dirty(folio)); + WARN_ON_ONCE(folio_test_writeback(folio)); + + trace_iomap_writeback_folio(inode, pos, folio_size(folio)); + + if (!iomap_writeback_handle_eof(folio, wpc, &end_pos)) + return 0; + WARN_ON_ONCE(end_pos <= pos); + + if (i_blocks_per_folio(inode, folio) > 1) { + if (!ifs) { + ifs = ifs_alloc(inode, folio, 0); + iomap_set_range_dirty(folio, 0, end_pos - pos); + } + + iomap_writeback_init(inode, folio); + } + + /* + * Set the writeback bit ASAP, as the I/O completion for the single + * block per folio case happen hit as soon as we're submitting the bio. + */ + folio_start_writeback(folio); + + /* + * Walk through the folio to find dirty areas to write back. + */ + end_aligned = round_up(end_pos, i_blocksize(inode)); + while ((rlen = iomap_find_dirty_range(folio, &pos, end_aligned))) { + error = iomap_writeback_range(wpc, folio, pos, rlen, end_pos, + &bytes_submitted); + if (error) + break; + pos += rlen; + } + + if (bytes_submitted) + wpc->nr_folios++; + if (error && pos > orig_pos) + fserror_report_io(inode, FSERR_BUFFERED_WRITE, orig_pos, 0, + error, GFP_NOFS); + + /* + * We can have dirty bits set past end of file in page_mkwrite path + * while mapping the last partial folio. Hence it's better to clear + * all the dirty bits in the folio here. + */ + iomap_clear_range_dirty(folio, 0, folio_size(folio)); + + /* + * Usually the writeback bit is cleared by the I/O completion handler. + * But we may end up either not actually writing any blocks, or (when + * there are multiple blocks in a folio) all I/O might have finished + * already at this point. In that case we need to clear the writeback + * bit ourselves right after unlocking the page. + */ + if (ifs) { + /* + * Subtract any bytes that were initially accounted to + * write_bytes_pending but skipped for writeback. + */ + size_t bytes_not_submitted = folio_size(folio) - + bytes_submitted; + + if (bytes_not_submitted) + iomap_finish_folio_write(inode, folio, + bytes_not_submitted); + } else if (!bytes_submitted) { + folio_end_writeback(folio); + } + + mapping_set_error(inode->i_mapping, error); + return error; +} +EXPORT_SYMBOL_GPL(iomap_writeback_folio); + +int +iomap_writepages(struct iomap_writepage_ctx *wpc) +{ + struct address_space *mapping = wpc->inode->i_mapping; + struct folio *folio = NULL; + int error; + + /* + * Writeback from reclaim context should never happen except in the case + * of a VM regression so warn about it and refuse to write the data. + */ + if (WARN_ON_ONCE((current->flags & (PF_MEMALLOC | PF_KSWAPD)) == + PF_MEMALLOC)) + return -EIO; + + while ((folio = writeback_iter(mapping, wpc->wbc, folio, &error))) { + error = iomap_writeback_folio(wpc, folio); + folio_unlock(folio); + } + + /* + * If @error is non-zero, it means that we have a situation where some + * part of the submission process has failed after we've marked pages + * for writeback. + * + * We cannot cancel the writeback directly in that case, so always call + * ->writeback_submit to run the I/O completion handler to clear the + * writeback bit and let the file system proess the errors. + */ + if (wpc->wb_ctx) + return wpc->ops->writeback_submit(wpc, error); + return error; +} +EXPORT_SYMBOL_GPL(iomap_writepages); diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c new file mode 100644 index 000000000..8b4039d16 --- /dev/null +++ b/fs/iomap/direct-io.c @@ -0,0 +1,1088 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2010 Red Hat, Inc. + * Copyright (c) 2016-2025 Christoph Hellwig. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" +#include "trace.h" + +#include "../internal.h" + +/* + * Private flags for iomap_dio, must not overlap with the public ones in + * iomap.h: + */ +#define IOMAP_DIO_NO_INVALIDATE (1U << 26) +#define IOMAP_DIO_COMP_WORK (1U << 27) +#define IOMAP_DIO_WRITE_THROUGH (1U << 28) +#define IOMAP_DIO_NEED_SYNC (1U << 29) +#define IOMAP_DIO_WRITE (1U << 30) +#define IOMAP_DIO_USER_BACKED (1U << 31) + +struct iomap_dio { + struct kiocb *iocb; + const struct iomap_dio_ops *dops; + loff_t i_size; + loff_t size; + atomic_t ref; + unsigned flags; + int error; + size_t done_before; + bool wait_for_completion; + + union { + /* used during submission and for synchronous completion: */ + struct { + struct iov_iter *iter; + struct task_struct *waiter; + } submit; + + /* used for aio completion: */ + struct { + struct work_struct work; + } aio; + }; +}; + +static struct bio *iomap_dio_alloc_bio(const struct iomap_iter *iter, + struct iomap_dio *dio, unsigned short nr_vecs, blk_opf_t opf) +{ + if (dio->dops && dio->dops->bio_set) + return bio_alloc_bioset(iter->iomap.bdev, nr_vecs, opf, + GFP_KERNEL, dio->dops->bio_set); + return bio_alloc(iter->iomap.bdev, nr_vecs, opf, GFP_KERNEL); +} + +static void iomap_dio_submit_bio(const struct iomap_iter *iter, + struct iomap_dio *dio, struct bio *bio, loff_t pos) +{ + struct kiocb *iocb = dio->iocb; + + atomic_inc(&dio->ref); + + /* Sync dio can't be polled reliably */ + if ((iocb->ki_flags & IOCB_HIPRI) && !is_sync_kiocb(iocb)) { + bio->bi_opf |= REQ_POLLED; + WRITE_ONCE(iocb->private, bio); + } + + if (dio->dops && dio->dops->submit_io) { + dio->dops->submit_io(iter, bio, pos); + } else { + WARN_ON_ONCE(iter->iomap.flags & IOMAP_F_ANON_WRITE); + blk_crypto_submit_bio(bio); + } +} + +static inline enum fserror_type iomap_dio_err_type(const struct iomap_dio *dio) +{ + if (dio->flags & IOMAP_DIO_WRITE) + return FSERR_DIRECTIO_WRITE; + return FSERR_DIRECTIO_READ; +} + +static inline bool should_report_dio_fserror(int error) +{ + switch (error) { + case 0: + case -EAGAIN: + case -ENOTBLK: + /* don't send fsnotify for success or magic retry codes */ + return false; + default: + return true; + } +} + +ssize_t iomap_dio_complete(struct iomap_dio *dio) +{ + const struct iomap_dio_ops *dops = dio->dops; + struct kiocb *iocb = dio->iocb; + loff_t offset = iocb->ki_pos; + ssize_t ret = dio->error; + + if (dops && dops->end_io) + ret = dops->end_io(iocb, dio->size, ret, dio->flags); + if (should_report_dio_fserror(dio->error)) + fserror_report_io(file_inode(iocb->ki_filp), + iomap_dio_err_type(dio), offset, dio->size, + dio->error, GFP_NOFS); + + if (likely(!ret)) { + ret = dio->size; + /* check for short read */ + if (offset + ret > dio->i_size && + !(dio->flags & IOMAP_DIO_WRITE)) + ret = dio->i_size - offset; + } + + /* + * Try again to invalidate clean pages which might have been cached by + * non-direct readahead, or faulted in by get_user_pages() if the source + * of the write was an mmap'ed region of the file we're writing. Either + * one is a pretty crazy thing to do, so we don't support it 100%. If + * this invalidation fails, tough, the write still worked... + * + * And this page cache invalidation has to be after ->end_io(), as some + * filesystems convert unwritten extents to real allocations in + * ->end_io() when necessary, otherwise a racing buffer read would cache + * zeros from unwritten extents. + */ + if (!dio->error && dio->size && (dio->flags & IOMAP_DIO_WRITE) && + !(dio->flags & IOMAP_DIO_NO_INVALIDATE)) + kiocb_invalidate_post_direct_write(iocb, dio->size); + + inode_dio_end(file_inode(iocb->ki_filp)); + + if (ret > 0) { + iocb->ki_pos += ret; + + /* + * If this is a DSYNC write, make sure we push it to stable + * storage now that we've written data. + */ + if (dio->flags & IOMAP_DIO_NEED_SYNC) + ret = generic_write_sync(iocb, ret); + if (ret > 0) + ret += dio->done_before; + } + trace_iomap_dio_complete(iocb, dio->error, ret); + kfree(dio); + return ret; +} +EXPORT_SYMBOL_GPL(iomap_dio_complete); + +static void iomap_dio_complete_work(struct work_struct *work) +{ + struct iomap_dio *dio = container_of(work, struct iomap_dio, aio.work); + struct kiocb *iocb = dio->iocb; + + iocb->ki_complete(iocb, iomap_dio_complete(dio)); +} + +/* + * Set an error in the dio if none is set yet. We have to use cmpxchg + * as the submission context and the completion context(s) can race to + * update the error. + */ +static inline void iomap_dio_set_error(struct iomap_dio *dio, int ret) +{ + cmpxchg(&dio->error, 0, ret); +} + +/* + * Called when dio->ref reaches zero from an I/O completion. + */ +static void iomap_dio_done(struct iomap_dio *dio) +{ + struct kiocb *iocb = dio->iocb; + + if (dio->wait_for_completion) { + /* + * Synchronous I/O, task itself will handle any completion work + * that needs after IO. All we need to do is wake the task. + */ + struct task_struct *waiter = dio->submit.waiter; + + WRITE_ONCE(dio->submit.waiter, NULL); + blk_wake_io_task(waiter); + return; + } + + /* + * Always run error completions in user context. These are not + * performance critical and some code relies on taking sleeping locks + * for error handling. + */ + if (dio->error) + dio->flags |= IOMAP_DIO_COMP_WORK; + + /* + * Never invalidate pages from this context to avoid deadlocks with + * buffered I/O completions when called from the ioend workqueue, + * or avoid sleeping when called directly from ->bi_end_io. + * Tough luck if you hit the tiny race with someone dirtying the range + * right between this check and the actual completion. + */ + if ((dio->flags & IOMAP_DIO_WRITE) && + !(dio->flags & IOMAP_DIO_COMP_WORK)) { + if (dio->iocb->ki_filp->f_mapping->nrpages) + dio->flags |= IOMAP_DIO_COMP_WORK; + else + dio->flags |= IOMAP_DIO_NO_INVALIDATE; + } + + if (dio->flags & IOMAP_DIO_COMP_WORK) { + struct inode *inode = file_inode(iocb->ki_filp); + + /* + * Async DIO completion that requires filesystem level + * completion work gets punted to a work queue to complete as + * the operation may require more IO to be issued to finalise + * filesystem metadata changes or guarantee data integrity. + */ + INIT_WORK(&dio->aio.work, iomap_dio_complete_work); + queue_work(inode->i_sb->s_dio_done_wq, &dio->aio.work); + return; + } + + WRITE_ONCE(iocb->private, NULL); + iomap_dio_complete_work(&dio->aio.work); +} + +static void __iomap_dio_bio_end_io(struct bio *bio, bool inline_completion) +{ + struct iomap_dio *dio = bio->bi_private; + + if (bio_integrity(bio)) + fs_bio_integrity_free(bio); + + if (dio->flags & IOMAP_DIO_BOUNCE) { + bio_iov_iter_unbounce(bio, !!dio->error, + dio->flags & IOMAP_DIO_USER_BACKED); + bio_put(bio); + } else if (dio->flags & IOMAP_DIO_USER_BACKED) { + bio_check_pages_dirty(bio); + } else { + bio_release_pages(bio, false); + bio_put(bio); + } + + /* Do not touch bio below, we just gave up our reference. */ + + if (atomic_dec_and_test(&dio->ref)) { + /* + * Avoid another context switch for the completion when already + * called from the ioend completion workqueue. + */ + if (inline_completion) + dio->flags &= ~IOMAP_DIO_COMP_WORK; + iomap_dio_done(dio); + } +} + +void iomap_dio_bio_end_io(struct bio *bio) +{ + struct iomap_dio *dio = bio->bi_private; + + if (bio->bi_status) + iomap_dio_set_error(dio, blk_status_to_errno(bio->bi_status)); + __iomap_dio_bio_end_io(bio, false); +} +EXPORT_SYMBOL_GPL(iomap_dio_bio_end_io); + +u32 iomap_finish_ioend_direct(struct iomap_ioend *ioend) +{ + struct iomap_dio *dio = ioend->io_bio.bi_private; + u32 vec_count = ioend->io_bio.bi_vcnt; + + if (ioend->io_error) + iomap_dio_set_error(dio, ioend->io_error); + __iomap_dio_bio_end_io(&ioend->io_bio, true); + + /* + * Return the number of bvecs completed as even direct I/O completions + * do significant per-folio work and we'll still want to give up the + * CPU after a lot of completions. + */ + return vec_count; +} + +static int iomap_dio_zero(const struct iomap_iter *iter, struct iomap_dio *dio, + loff_t pos, unsigned len) +{ + struct inode *inode = file_inode(dio->iocb->ki_filp); + struct bio *bio; + struct folio *zero_folio = largest_zero_folio(); + int nr_vecs = max(1, i_blocksize(inode) / folio_size(zero_folio)); + + if (!len) + return 0; + + /* + * This limit shall never be reached as most filesystems have a + * maximum blocksize of 64k. + */ + if (WARN_ON_ONCE(nr_vecs > BIO_MAX_VECS)) + return -EINVAL; + + bio = iomap_dio_alloc_bio(iter, dio, nr_vecs, + REQ_OP_WRITE | REQ_SYNC | REQ_IDLE); + fscrypt_set_bio_crypt_ctx(bio, inode, pos, GFP_KERNEL); + bio->bi_iter.bi_sector = iomap_sector(&iter->iomap, pos); + bio->bi_private = dio; + bio->bi_end_io = iomap_dio_bio_end_io; + + while (len > 0) { + unsigned int io_len = min(len, folio_size(zero_folio)); + + bio_add_folio_nofail(bio, zero_folio, io_len, 0); + len -= io_len; + } + iomap_dio_submit_bio(iter, dio, bio, pos); + + return 0; +} + +static ssize_t iomap_dio_bio_iter_one(struct iomap_iter *iter, + struct iomap_dio *dio, loff_t pos, unsigned int alignment, + blk_opf_t op) +{ + unsigned int nr_vecs; + struct bio *bio; + ssize_t ret; + + if (dio->flags & IOMAP_DIO_BOUNCE) + nr_vecs = bio_iov_bounce_nr_vecs(dio->submit.iter, op); + else + nr_vecs = bio_iov_vecs_to_alloc(dio->submit.iter, BIO_MAX_VECS); + + bio = iomap_dio_alloc_bio(iter, dio, nr_vecs, op); + fscrypt_set_bio_crypt_ctx(bio, iter->inode, pos, GFP_KERNEL); + bio->bi_iter.bi_sector = iomap_sector(&iter->iomap, pos); + bio->bi_write_hint = iter->inode->i_write_hint; + bio->bi_ioprio = dio->iocb->ki_ioprio; + bio->bi_private = dio; + bio->bi_end_io = iomap_dio_bio_end_io; + + + if (dio->flags & IOMAP_DIO_BOUNCE) + ret = bio_iov_iter_bounce(bio, dio->submit.iter, + iomap_max_bio_size(&iter->iomap), alignment); + else + ret = bio_iov_iter_get_pages(bio, dio->submit.iter, + bdev_dma_alignment(bio->bi_bdev), + alignment - 1); + if (unlikely(ret)) + goto out_put_bio; + ret = bio->bi_iter.bi_size; + + /* + * An atomic write bio must cover the complete length. If it doesn't, + * error out. + */ + if ((op & REQ_ATOMIC) && WARN_ON_ONCE(ret != iomap_length(iter))) { + ret = -EINVAL; + goto out_bio_release_pages; + } + + if (iter->iomap.flags & IOMAP_F_INTEGRITY) { + if (dio->flags & IOMAP_DIO_WRITE) + fs_bio_integrity_generate(bio); + else + fs_bio_integrity_alloc(bio); + } + + if (dio->flags & IOMAP_DIO_WRITE) + task_io_account_write(ret); + else if ((dio->flags & IOMAP_DIO_USER_BACKED) && + !(dio->flags & IOMAP_DIO_BOUNCE)) + bio_set_pages_dirty(bio); + + /* + * We can only poll for single bio I/Os. + */ + if (iov_iter_count(dio->submit.iter)) + dio->iocb->ki_flags &= ~IOCB_HIPRI; + iomap_dio_submit_bio(iter, dio, bio, pos); + return ret; + +out_bio_release_pages: + if (dio->flags & IOMAP_DIO_BOUNCE) + bio_iov_iter_unbounce(bio, true, false); + else + bio_release_pages(bio, false); +out_put_bio: + bio_put(bio); + return ret; +} + +static inline unsigned int iomap_dio_alignment(struct inode *inode, + struct block_device *bdev, unsigned int dio_flags) +{ + if (dio_flags & IOMAP_DIO_FSBLOCK_ALIGNED) + return i_blocksize(inode); + return bdev_logical_block_size(bdev); +} + +static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio) +{ + const struct iomap *iomap = &iter->iomap; + struct inode *inode = iter->inode; + unsigned int fs_block_size = i_blocksize(inode), pad; + const loff_t length = iomap_length(iter); + loff_t pos = iter->pos; + blk_opf_t bio_opf = REQ_SYNC | REQ_IDLE; + bool need_zeroout = false; + u64 copied = 0; + size_t orig_count; + unsigned int alignment; + ssize_t ret = 0; + + /* + * File systems that write out of place and always allocate new blocks + * need each bio to be block aligned as that's the unit of allocation. + */ + alignment = iomap_dio_alignment(inode, iomap->bdev, dio->flags); + + if ((pos | length) & (alignment - 1)) + return -EINVAL; + + if (dio->flags & IOMAP_DIO_WRITE) { + bool need_completion_work = true; + + switch (iomap->type) { + case IOMAP_MAPPED: + /* + * Directly mapped I/O does not inherently need to do + * work at I/O completion time. But there are various + * cases below where this will get set again. + */ + need_completion_work = false; + break; + case IOMAP_UNWRITTEN: + dio->flags |= IOMAP_DIO_UNWRITTEN; + need_zeroout = true; + break; + default: + break; + } + + if (iomap->flags & IOMAP_F_ATOMIC_BIO) { + /* + * Ensure that the mapping covers the full write + * length, otherwise it won't be submitted as a single + * bio, which is required to use hardware atomics. + */ + if (length != iter->len) + return -EINVAL; + bio_opf |= REQ_ATOMIC; + } + + if (iomap->flags & IOMAP_F_SHARED) { + /* + * Unsharing of needs to update metadata at I/O + * completion time. + */ + need_completion_work = true; + dio->flags |= IOMAP_DIO_COW; + } + + if (iomap->flags & IOMAP_F_NEW) { + /* + * Newly allocated blocks might need recording in + * metadata at I/O completion time. + */ + need_completion_work = true; + need_zeroout = true; + } + + /* + * Use a FUA write if we need datasync semantics and this is a + * pure overwrite that doesn't require any metadata updates. + * + * This allows us to avoid cache flushes on I/O completion. + */ + if (dio->flags & IOMAP_DIO_WRITE_THROUGH) { + if (!need_completion_work && + !(iomap->flags & IOMAP_F_DIRTY) && + (!bdev_write_cache(iomap->bdev) || + bdev_fua(iomap->bdev))) + bio_opf |= REQ_FUA; + else + dio->flags &= ~IOMAP_DIO_WRITE_THROUGH; + } + + /* + * We can only do inline completion for pure overwrites that + * don't require additional I/O at completion time. + * + * This rules out writes that need zeroing or metdata updates to + * convert unwritten or shared extents. + * + * Writes that extend i_size are also not supported, but this is + * handled in __iomap_dio_rw(). + */ + if (need_completion_work) + dio->flags |= IOMAP_DIO_COMP_WORK; + + bio_opf |= REQ_OP_WRITE; + } else { + bio_opf |= REQ_OP_READ; + } + + /* + * Save the original count and trim the iter to just the extent we + * are operating on right now. The iter will be re-expanded once + * we are done. + */ + orig_count = iov_iter_count(dio->submit.iter); + iov_iter_truncate(dio->submit.iter, length); + + if (!iov_iter_count(dio->submit.iter)) + goto out; + + /* + * The rules for polled IO completions follow the guidelines as the + * ones we set for inline and deferred completions. If none of those + * are available for this IO, clear the polled flag. + */ + if (dio->flags & IOMAP_DIO_COMP_WORK) + dio->iocb->ki_flags &= ~IOCB_HIPRI; + + if (need_zeroout) { + /* zero out from the start of the block to the write offset */ + pad = pos & (fs_block_size - 1); + + ret = iomap_dio_zero(iter, dio, pos - pad, pad); + if (ret) + goto out; + } + + do { + /* + * If completions already occurred and reported errors, give up now and + * don't bother submitting more bios. + */ + if (unlikely(data_race(dio->error))) + goto out; + + ret = iomap_dio_bio_iter_one(iter, dio, pos, alignment, bio_opf); + if (unlikely(ret < 0)) { + /* + * We have to stop part way through an IO. We must fall + * through to the sub-block tail zeroing here, otherwise + * this short IO may expose stale data in the tail of + * the block we haven't written data to. + */ + break; + } + dio->size += ret; + copied += ret; + pos += ret; + ret = 0; + } while (iov_iter_count(dio->submit.iter)); + + /* + * We need to zeroout the tail of a sub-block write if the extent type + * requires zeroing or the write extends beyond EOF. If we don't zero + * the block tail in the latter case, we can expose stale data via mmap + * reads of the EOF block. + */ + if (need_zeroout || + ((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode))) { + /* zero out from the end of the write to the end of the block */ + pad = pos & (fs_block_size - 1); + if (pad) + ret = iomap_dio_zero(iter, dio, pos, + fs_block_size - pad); + } +out: + /* Undo iter limitation to current extent */ + iov_iter_reexpand(dio->submit.iter, orig_count - copied); + if (copied) + return iomap_iter_advance(iter, copied); + return ret; +} + +static int iomap_dio_hole_iter(struct iomap_iter *iter, struct iomap_dio *dio) +{ + loff_t length = iov_iter_zero(iomap_length(iter), dio->submit.iter); + + dio->size += length; + if (!length) + return -EFAULT; + return iomap_iter_advance(iter, length); +} + +static int iomap_dio_inline_iter(struct iomap_iter *iomi, struct iomap_dio *dio) +{ + const struct iomap *iomap = &iomi->iomap; + struct iov_iter *iter = dio->submit.iter; + void *inline_data = iomap_inline_data(iomap, iomi->pos); + loff_t length = iomap_length(iomi); + loff_t pos = iomi->pos; + u64 copied; + + if (WARN_ON_ONCE(!inline_data)) + return -EIO; + + if (dio->flags & IOMAP_DIO_WRITE) { + loff_t size = iomi->inode->i_size; + + if (pos > size) + memset(iomap_inline_data(iomap, size), 0, pos - size); + copied = copy_from_iter(inline_data, length, iter); + if (copied) { + if (pos + copied > size) + i_size_write(iomi->inode, pos + copied); + mark_inode_dirty(iomi->inode); + } + } else { + copied = copy_to_iter(inline_data, length, iter); + } + dio->size += copied; + if (!copied) + return -EFAULT; + return iomap_iter_advance(iomi, copied); +} + +static int iomap_dio_iter(struct iomap_iter *iter, struct iomap_dio *dio) +{ + switch (iter->iomap.type) { + case IOMAP_HOLE: + if (WARN_ON_ONCE(dio->flags & IOMAP_DIO_WRITE)) + return -EIO; + return iomap_dio_hole_iter(iter, dio); + case IOMAP_UNWRITTEN: + if (!(dio->flags & IOMAP_DIO_WRITE)) + return iomap_dio_hole_iter(iter, dio); + return iomap_dio_bio_iter(iter, dio); + case IOMAP_MAPPED: + return iomap_dio_bio_iter(iter, dio); + case IOMAP_INLINE: + return iomap_dio_inline_iter(iter, dio); + case IOMAP_DELALLOC: + /* + * DIO is not serialised against mmap() access at all, and so + * if the page_mkwrite occurs between the writeback and the + * iomap_iter() call in the DIO path, then it will see the + * DELALLOC block that the page-mkwrite allocated. + */ + pr_warn_ratelimited("Direct I/O collision with buffered writes! File: %pD4 Comm: %.20s\n", + dio->iocb->ki_filp, current->comm); + return -EIO; + default: + WARN_ON_ONCE(1); + return -EIO; + } +} + +/* + * iomap_dio_rw() always completes O_[D]SYNC writes regardless of whether the IO + * is being issued as AIO or not. This allows us to optimise pure data writes + * to use REQ_FUA rather than requiring generic_write_sync() to issue a + * REQ_FLUSH post write. This is slightly tricky because a single request here + * can be mapped into multiple disjoint IOs and only a subset of the IOs issued + * may be pure data writes. In that case, we still need to do a full data sync + * completion. + * + * When page faults are disabled and @dio_flags includes IOMAP_DIO_PARTIAL, + * __iomap_dio_rw can return a partial result if it encounters a non-resident + * page in @iter after preparing a transfer. In that case, the non-resident + * pages can be faulted in and the request resumed with @done_before set to the + * number of bytes previously transferred. The request will then complete with + * the correct total number of bytes transferred; this is essential for + * completing partial requests asynchronously. + * + * Returns -ENOTBLK In case of a page invalidation invalidation failure for + * writes. The callers needs to fall back to buffered I/O in this case. + */ +struct iomap_dio * +__iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, + const struct iomap_ops *ops, const struct iomap_dio_ops *dops, + unsigned int dio_flags, void *private, size_t done_before) +{ + struct inode *inode = file_inode(iocb->ki_filp); + struct iomap_iter iomi = { + .inode = inode, + .pos = iocb->ki_pos, + .len = iov_iter_count(iter), + .flags = IOMAP_DIRECT, + .private = private, + }; + bool wait_for_completion = + is_sync_kiocb(iocb) || (dio_flags & IOMAP_DIO_FORCE_WAIT); + struct blk_plug plug; + struct iomap_dio *dio; + loff_t ret = 0; + + trace_iomap_dio_rw_begin(iocb, iter, dio_flags, done_before); + + if (!iomi.len) + return NULL; + + dio = kmalloc_obj(*dio); + if (!dio) + return ERR_PTR(-ENOMEM); + + dio->iocb = iocb; + atomic_set(&dio->ref, 1); + dio->size = 0; + dio->i_size = i_size_read(inode); + dio->dops = dops; + dio->error = 0; + dio->flags = dio_flags & (IOMAP_DIO_FSBLOCK_ALIGNED | IOMAP_DIO_BOUNCE); + dio->done_before = done_before; + + dio->submit.iter = iter; + dio->submit.waiter = current; + + if (iocb->ki_flags & IOCB_NOWAIT) + iomi.flags |= IOMAP_NOWAIT; + + if (iov_iter_rw(iter) == READ) { + if (iomi.pos >= dio->i_size) + goto out_free_dio; + + if (user_backed_iter(iter)) + dio->flags |= IOMAP_DIO_USER_BACKED; + + ret = kiocb_write_and_wait(iocb, iomi.len); + if (ret) + goto out_free_dio; + } else { + iomi.flags |= IOMAP_WRITE; + dio->flags |= IOMAP_DIO_WRITE; + + if (dio_flags & IOMAP_DIO_OVERWRITE_ONLY) { + ret = -EAGAIN; + if (iomi.pos >= dio->i_size || + iomi.pos + iomi.len > dio->i_size) + goto out_free_dio; + iomi.flags |= IOMAP_OVERWRITE_ONLY; + } + + if (iocb->ki_flags & IOCB_ATOMIC) + iomi.flags |= IOMAP_ATOMIC; + + /* for data sync or sync, we need sync completion processing */ + if (iocb_is_dsync(iocb)) { + dio->flags |= IOMAP_DIO_NEED_SYNC; + + /* + * For datasync only writes, we optimistically try using + * WRITE_THROUGH for this IO. This flag requires either + * FUA writes through the device's write cache, or a + * normal write to a device without a volatile write + * cache. For the former, Any non-FUA write that occurs + * will clear this flag, hence we know before completion + * whether a cache flush is necessary. + */ + if (!(iocb->ki_flags & IOCB_SYNC)) + dio->flags |= IOMAP_DIO_WRITE_THROUGH; + } + + /* + * i_size updates must to happen from process context. + */ + if (iomi.pos + iomi.len > dio->i_size) + dio->flags |= IOMAP_DIO_COMP_WORK; + + /* + * Try to invalidate cache pages for the range we are writing. + * If this invalidation fails, let the caller fall back to + * buffered I/O. + */ + ret = kiocb_invalidate_pages(iocb, iomi.len); + if (ret) { + if (ret != -EAGAIN) { + trace_iomap_dio_invalidate_fail(inode, iomi.pos, + iomi.len); + if (iocb->ki_flags & IOCB_ATOMIC) { + /* + * folio invalidation failed, maybe + * this is transient, unlock and see if + * the caller tries again. + */ + ret = -EAGAIN; + } else { + /* fall back to buffered write */ + ret = -ENOTBLK; + } + } + goto out_free_dio; + } + } + + if (!wait_for_completion && !inode->i_sb->s_dio_done_wq) { + ret = sb_init_dio_done_wq(inode->i_sb); + if (ret < 0) + goto out_free_dio; + } + + inode_dio_begin(inode); + + blk_start_plug(&plug); + while ((ret = iomap_iter(&iomi, ops)) > 0) { + iomi.status = iomap_dio_iter(&iomi, dio); + + /* + * We can only poll for single bio I/Os. + */ + iocb->ki_flags &= ~IOCB_HIPRI; + } + + blk_finish_plug(&plug); + + /* + * We only report that we've read data up to i_size. + * Revert iter to a state corresponding to that as some callers (such + * as the splice code) rely on it. + */ + if (iov_iter_rw(iter) == READ && iomi.pos >= dio->i_size) + iov_iter_revert(iter, iomi.pos - dio->i_size); + + if (ret == -EFAULT && dio->size && (dio_flags & IOMAP_DIO_PARTIAL)) { + if (!(iocb->ki_flags & IOCB_NOWAIT)) + wait_for_completion = true; + ret = 0; + } + + /* magic error code to fall back to buffered I/O */ + if (ret == -ENOTBLK) { + wait_for_completion = true; + ret = 0; + } + if (ret < 0) + iomap_dio_set_error(dio, ret); + + /* + * If all the writes we issued were already written through to the + * media, we don't need to flush the cache on IO completion. Clear the + * sync flag for this case. + * + * Otherwise clear the inline completion flag if any sync work is + * needed, as that needs to be performed from process context. + */ + if (dio->flags & IOMAP_DIO_WRITE_THROUGH) + dio->flags &= ~IOMAP_DIO_NEED_SYNC; + else if (dio->flags & IOMAP_DIO_NEED_SYNC) + dio->flags |= IOMAP_DIO_COMP_WORK; + + /* + * We are about to drop our additional submission reference, which + * might be the last reference to the dio. There are three different + * ways we can progress here: + * + * (a) If this is the last reference we will always complete and free + * the dio ourselves. + * (b) If this is not the last reference, and we serve an asynchronous + * iocb, we must never touch the dio after the decrement, the + * I/O completion handler will complete and free it. + * (c) If this is not the last reference, but we serve a synchronous + * iocb, the I/O completion handler will wake us up on the drop + * of the final reference, and we will complete and free it here + * after we got woken by the I/O completion handler. + */ + dio->wait_for_completion = wait_for_completion; + if (!atomic_dec_and_test(&dio->ref)) { + if (!wait_for_completion) { + trace_iomap_dio_rw_queued(inode, iomi.pos, iomi.len); + return ERR_PTR(-EIOCBQUEUED); + } + + for (;;) { + set_current_state(TASK_UNINTERRUPTIBLE); + if (!READ_ONCE(dio->submit.waiter)) + break; + + blk_io_schedule(); + } + __set_current_state(TASK_RUNNING); + } + + return dio; + +out_free_dio: + kfree(dio); + if (ret) + return ERR_PTR(ret); + return NULL; +} +EXPORT_SYMBOL_GPL(__iomap_dio_rw); + +ssize_t +iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, + const struct iomap_ops *ops, const struct iomap_dio_ops *dops, + unsigned int dio_flags, void *private, size_t done_before) +{ + struct iomap_dio *dio; + + dio = __iomap_dio_rw(iocb, iter, ops, dops, dio_flags, private, + done_before); + if (IS_ERR_OR_NULL(dio)) + return PTR_ERR_OR_ZERO(dio); + return iomap_dio_complete(dio); +} +EXPORT_SYMBOL_GPL(iomap_dio_rw); + +struct iomap_dio_simple { + struct kiocb *iocb; + size_t size; + unsigned int dio_flags; + struct work_struct work; + /* + * Align @bio to a cacheline boundary so that, combined with the + * front_pad passed to bioset_init(), the bio sits at the start of + * a cacheline in memory returned by the (HWCACHE-aligned) bio + * slab. This keeps the hot fields block layer touches on submit + * and completion (bi_iter, bi_status, ...) within a single line. + */ + struct bio bio ____cacheline_aligned_in_smp; +}; + +static struct bio_set iomap_dio_simple_pool; + +static ssize_t iomap_dio_simple_complete(struct iomap_dio_simple *sr) +{ + struct bio *bio = &sr->bio; + struct kiocb *iocb = sr->iocb; + struct inode *inode = file_inode(iocb->ki_filp); + ssize_t ret; + + if (unlikely(bio->bi_status)) { + ret = blk_status_to_errno(bio->bi_status); + if (should_report_dio_fserror(ret)) + fserror_report_io(inode, FSERR_DIRECTIO_READ, + iocb->ki_pos, sr->size, ret, + GFP_NOFS); + } else { + ret = sr->size; + iocb->ki_pos += ret; + } + + if (sr->dio_flags & IOMAP_DIO_USER_BACKED) { + bio_check_pages_dirty(bio); + } else { + bio_release_pages(bio, false); + bio_put(bio); + } + inode_dio_end(inode); + trace_iomap_dio_complete(iocb, ret < 0 ? ret : 0, ret); + return ret; +} + +static void iomap_dio_simple_complete_work(struct work_struct *work) +{ + struct iomap_dio_simple *sr = + container_of(work, struct iomap_dio_simple, work); + struct kiocb *iocb = sr->iocb; + + WRITE_ONCE(iocb->private, NULL); + iocb->ki_complete(iocb, iomap_dio_simple_complete(sr)); +} + +static void iomap_dio_simple_end_io(struct bio *bio) +{ + struct iomap_dio_simple *sr = + container_of(bio, struct iomap_dio_simple, bio); + struct kiocb *iocb = sr->iocb; + + if (unlikely(sr->bio.bi_status)) { + struct inode *inode = file_inode(iocb->ki_filp); + + INIT_WORK(&sr->work, iomap_dio_simple_complete_work); + queue_work(inode->i_sb->s_dio_done_wq, &sr->work); + return; + } + + WRITE_ONCE(iocb->private, NULL); + iocb->ki_complete(iocb, iomap_dio_simple_complete(sr)); +} + +ssize_t __iomap_dio_read_simple(struct kiocb *iocb, struct iov_iter *iter, + struct iomap_iter *iomi) +{ + gfp_t gfp = (iomi->flags & IOMAP_NOWAIT) ? GFP_NOWAIT : GFP_KERNEL; + struct iomap_dio_simple *sr; + unsigned int alignment; + struct bio *bio; + ssize_t ret; + + if (iomi->iomap.type != IOMAP_MAPPED || + iomi->iomap.offset + iomi->iomap.length < iomi->pos + iomi->len || + (iomi->iomap.flags & IOMAP_F_INTEGRITY)) { + ret = -ENOTBLK; + goto out_dio_end; + } + + alignment = iomap_dio_alignment(iomi->inode, iomi->iomap.bdev, 0); + if ((iomi->pos | iomi->len) & (alignment - 1)) { + ret = -EINVAL; + goto out_dio_end; + } + + if (unlikely(!iomi->inode->i_sb->s_dio_done_wq && + !is_sync_kiocb(iocb))) { + ret = sb_init_dio_done_wq(iomi->inode->i_sb); + if (ret < 0) + goto out_dio_end; + } + + trace_iomap_dio_rw_begin(iocb, iter, 0, 0); + + bio = bio_alloc_bioset(iomi->iomap.bdev, + bio_iov_vecs_to_alloc(iter, BIO_MAX_VECS), + REQ_OP_READ, gfp, &iomap_dio_simple_pool); + if (!bio) { + ret = -EAGAIN; + goto out_dio_end; + } + sr = container_of(bio, struct iomap_dio_simple, bio); + sr->iocb = iocb; + sr->dio_flags = 0; + + bio->bi_iter.bi_sector = iomap_sector(&iomi->iomap, iomi->pos); + bio->bi_ioprio = iocb->ki_ioprio; + + ret = bio_iov_iter_get_pages(bio, iter, + bdev_dma_alignment(bio->bi_bdev), + alignment - 1); + if (unlikely(ret)) + goto out_bio_put; + + if (bio->bi_iter.bi_size != iomi->len) { + iov_iter_revert(iter, bio->bi_iter.bi_size); + ret = -ENOTBLK; + goto out_bio_release_pages; + } + + sr->size = bio->bi_iter.bi_size; + if (user_backed_iter(iter)) { + bio_set_pages_dirty(bio); + sr->dio_flags |= IOMAP_DIO_USER_BACKED; + } + + if (iocb->ki_flags & IOCB_NOWAIT) + bio->bi_opf |= REQ_NOWAIT; + + if (is_sync_kiocb(iocb)) { + submit_bio_wait(bio); + return iomap_dio_simple_complete(sr); + } + + if ((iocb->ki_flags & IOCB_HIPRI)) { + bio->bi_opf |= REQ_POLLED; + WRITE_ONCE(iocb->private, bio); + } + bio->bi_end_io = iomap_dio_simple_end_io; + submit_bio(bio); + trace_iomap_dio_rw_queued(iomi->inode, iocb->ki_pos, iomi->len); + return -EIOCBQUEUED; + +out_bio_release_pages: + bio_release_pages(bio, false); +out_bio_put: + bio_put(bio); +out_dio_end: + inode_dio_end(iomi->inode); + return ret; +} +EXPORT_SYMBOL_GPL(__iomap_dio_read_simple); + +static int __init iomap_dio_init(void) +{ + return bioset_init(&iomap_dio_simple_pool, 4, + offsetof(struct iomap_dio_simple, bio), + BIOSET_NEED_BVECS | BIOSET_PERCPU_CACHE); +} +fs_initcall(iomap_dio_init); diff --git a/fs/iomap/fiemap.c b/fs/iomap/fiemap.c new file mode 100644 index 000000000..d11dadff8 --- /dev/null +++ b/fs/iomap/fiemap.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2016-2021 Christoph Hellwig. + */ +#include +#include +#include + +static int iomap_to_fiemap(struct fiemap_extent_info *fi, + const struct iomap *iomap, u32 flags) +{ + switch (iomap->type) { + case IOMAP_HOLE: + /* skip holes */ + return 0; + case IOMAP_DELALLOC: + flags |= FIEMAP_EXTENT_DELALLOC | FIEMAP_EXTENT_UNKNOWN; + break; + case IOMAP_MAPPED: + break; + case IOMAP_UNWRITTEN: + flags |= FIEMAP_EXTENT_UNWRITTEN; + break; + case IOMAP_INLINE: + flags |= FIEMAP_EXTENT_DATA_INLINE; + break; + } + + if (iomap->flags & IOMAP_F_MERGED) + flags |= FIEMAP_EXTENT_MERGED; + if (iomap->flags & IOMAP_F_SHARED) + flags |= FIEMAP_EXTENT_SHARED; + + return fiemap_fill_next_extent(fi, iomap->offset, + iomap->addr != IOMAP_NULL_ADDR ? iomap->addr : 0, + iomap->length, flags); +} + +static int iomap_fiemap_iter(struct iomap_iter *iter, + struct fiemap_extent_info *fi, struct iomap *prev) +{ + int ret; + + if (iter->iomap.type == IOMAP_HOLE) + goto advance; + + ret = iomap_to_fiemap(fi, prev, 0); + *prev = iter->iomap; + if (ret < 0) + return ret; + if (ret == 1) /* extent array full */ + return 0; + +advance: + return iomap_iter_advance_full(iter); +} + +int iomap_fiemap(struct inode *inode, struct fiemap_extent_info *fi, + u64 start, u64 len, const struct iomap_ops *ops) +{ + struct iomap_iter iter = { + .inode = inode, + .pos = start, + .len = len, + .flags = IOMAP_REPORT, + }; + struct iomap prev = { + .type = IOMAP_HOLE, + }; + int ret; + + ret = fiemap_prep(inode, fi, start, &iter.len, 0); + if (ret) + return ret; + + while ((ret = iomap_iter(&iter, ops)) > 0) + iter.status = iomap_fiemap_iter(&iter, fi, &prev); + + if (prev.type != IOMAP_HOLE) { + ret = iomap_to_fiemap(fi, &prev, FIEMAP_EXTENT_LAST); + if (ret < 0) + return ret; + } + + /* inode with no (attribute) mapping will give ENOENT */ + if (ret < 0 && ret != -ENOENT) + return ret; + return 0; +} +EXPORT_SYMBOL_GPL(iomap_fiemap); + +/* legacy ->bmap interface. 0 is the error return (!) */ +sector_t +iomap_bmap(struct address_space *mapping, sector_t bno, + const struct iomap_ops *ops) +{ + struct iomap_iter iter = { + .inode = mapping->host, + .pos = (loff_t)bno << mapping->host->i_blkbits, + .len = i_blocksize(mapping->host), + .flags = IOMAP_REPORT, + }; + const unsigned int blkshift = mapping->host->i_blkbits - SECTOR_SHIFT; + int ret; + + if (filemap_write_and_wait(mapping)) + return 0; + + bno = 0; + while ((ret = iomap_iter(&iter, ops)) > 0) { + if (iter.iomap.type == IOMAP_MAPPED) + bno = iomap_sector(&iter.iomap, iter.pos) >> blkshift; + /* leave iter.status unset to abort loop */ + } + if (ret) + return 0; + + return bno; +} +EXPORT_SYMBOL_GPL(iomap_bmap); diff --git a/fs/iomap/internal.h b/fs/iomap/internal.h new file mode 100644 index 000000000..74e898b19 --- /dev/null +++ b/fs/iomap/internal.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _IOMAP_INTERNAL_H +#define _IOMAP_INTERNAL_H 1 + +#define IOEND_BATCH_SIZE 4096 + +/* + * Normally we can build bios as big as the data structure supports. + * + * But for integrity protected I/O we need to respect the maximum size of the + * single contiguous allocation for the integrity buffer. + */ +static inline size_t iomap_max_bio_size(const struct iomap *iomap) +{ + if (iomap->flags & IOMAP_F_INTEGRITY) + return max_integrity_io_size(bdev_limits(iomap->bdev)); + return BIO_MAX_SIZE; +} + +u32 iomap_finish_ioend_buffered_read(struct iomap_ioend *ioend); +u32 iomap_finish_ioend_direct(struct iomap_ioend *ioend); + +#ifdef CONFIG_BLOCK +int iomap_bio_read_folio_range_sync(const struct iomap_iter *iter, + struct folio *folio, loff_t pos, size_t len); +#else +static inline int iomap_bio_read_folio_range_sync(const struct iomap_iter *iter, + struct folio *folio, loff_t pos, size_t len) +{ + WARN_ON_ONCE(1); + return -EIO; +} +#endif /* CONFIG_BLOCK */ + +#endif /* _IOMAP_INTERNAL_H */ diff --git a/fs/iomap/ioend.c b/fs/iomap/ioend.c new file mode 100644 index 000000000..7bbbb417f --- /dev/null +++ b/fs/iomap/ioend.c @@ -0,0 +1,536 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2016-2025 Christoph Hellwig. + */ +#include +#include +#include +#include +#include +#include +#include "internal.h" +#include "trace.h" + +struct bio_set iomap_ioend_bioset; +EXPORT_SYMBOL_GPL(iomap_ioend_bioset); +static struct bio_set iomap_ioend_split_bioset; + +struct iomap_ioend *iomap_init_ioend(struct inode *inode, + struct bio *bio, loff_t file_offset, u16 ioend_flags) +{ + struct iomap_ioend *ioend = iomap_ioend_from_bio(bio); + + atomic_set(&ioend->io_remaining, 1); + ioend->io_error = 0; + ioend->io_parent = NULL; + INIT_LIST_HEAD(&ioend->io_list); + ioend->io_flags = ioend_flags; + ioend->io_inode = inode; + ioend->io_offset = file_offset; + ioend->io_size = bio->bi_iter.bi_size; + ioend->io_sector = bio->bi_iter.bi_sector; + ioend->io_vi = NULL; + ioend->io_private = NULL; + return ioend; +} +EXPORT_SYMBOL_GPL(iomap_init_ioend); + +/* + * We're now finished for good with this ioend structure. Update the folio + * state, release holds on bios, and finally free up memory. Do not use the + * ioend after this. + */ +static u32 iomap_finish_ioend_buffered_write(struct iomap_ioend *ioend) +{ + struct inode *inode = ioend->io_inode; + struct bio *bio = &ioend->io_bio; + struct folio_iter fi; + u32 folio_count = 0; + + if (ioend->io_error) { + mapping_set_error(inode->i_mapping, ioend->io_error); + if (!bio_flagged(bio, BIO_QUIET)) { + pr_err_ratelimited( +"%s: writeback error on inode %llu, offset %lld, sector %llu", + inode->i_sb->s_id, inode->i_ino, + ioend->io_offset, ioend->io_sector); + } + } + + /* walk all folios in bio, ending page IO on them */ + bio_for_each_folio_all(fi, bio) { + if (ioend->io_error) + fserror_report_io(inode, FSERR_BUFFERED_WRITE, + folio_pos(fi.folio) + fi.offset, + fi.length, ioend->io_error, + GFP_ATOMIC); + iomap_finish_folio_write(inode, fi.folio, fi.length); + folio_count++; + } + + if (bio_integrity(bio)) + fs_bio_integrity_free(bio); + bio_put(bio); /* frees the ioend */ + return folio_count; +} + +static DEFINE_SPINLOCK(failed_ioend_lock); +static LIST_HEAD(failed_ioend_list); + +static void +iomap_fail_ioends( + struct work_struct *work) +{ + struct iomap_ioend *ioend; + struct list_head tmp; + unsigned long flags; + + spin_lock_irqsave(&failed_ioend_lock, flags); + list_replace_init(&failed_ioend_list, &tmp); + spin_unlock_irqrestore(&failed_ioend_lock, flags); + + while ((ioend = list_first_entry_or_null(&tmp, struct iomap_ioend, + io_list))) { + list_del_init(&ioend->io_list); + iomap_finish_ioend_buffered_write(ioend); + cond_resched(); + } +} + +static DECLARE_WORK(failed_ioend_work, iomap_fail_ioends); + +static void iomap_fail_ioend_buffered(struct iomap_ioend *ioend) +{ + unsigned long flags; + + /* + * Bounce I/O errors to a workqueue to avoid nested i_lock acquisitions + * in the fserror code. The caller no longer owns the ioend reference + * after the spinlock drops. + */ + spin_lock_irqsave(&failed_ioend_lock, flags); + if (list_empty(&failed_ioend_list)) + WARN_ON_ONCE(!schedule_work(&failed_ioend_work)); + list_add_tail(&ioend->io_list, &failed_ioend_list); + spin_unlock_irqrestore(&failed_ioend_lock, flags); +} + +static void ioend_writeback_end_bio(struct bio *bio) +{ + struct iomap_ioend *ioend = iomap_ioend_from_bio(bio); + + ioend->io_error = blk_status_to_errno(bio->bi_status); + if (ioend->io_error) { + iomap_fail_ioend_buffered(ioend); + return; + } + + iomap_finish_ioend_buffered_write(ioend); +} + +/* + * We cannot cancel the ioend directly in case of an error, so call the bio end + * I/O handler with the error status here to run the normal I/O completion + * handler. + */ +int iomap_ioend_writeback_submit(struct iomap_writepage_ctx *wpc, int error) +{ + struct iomap_ioend *ioend = wpc->wb_ctx; + + if (!ioend->io_bio.bi_end_io) + ioend->io_bio.bi_end_io = ioend_writeback_end_bio; + + if (WARN_ON_ONCE(wpc->iomap.flags & IOMAP_F_ANON_WRITE)) + error = -EIO; + + if (error) { + ioend->io_bio.bi_status = errno_to_blk_status(error); + bio_endio(&ioend->io_bio); + return error; + } + + if (wpc->iomap.flags & IOMAP_F_INTEGRITY) + fs_bio_integrity_generate(&ioend->io_bio); + submit_bio(&ioend->io_bio); + return 0; +} +EXPORT_SYMBOL_GPL(iomap_ioend_writeback_submit); + +static struct iomap_ioend *iomap_alloc_ioend(struct iomap_writepage_ctx *wpc, + loff_t pos, u16 ioend_flags) +{ + struct bio *bio; + + bio = bio_alloc_bioset(wpc->iomap.bdev, BIO_MAX_VECS, + REQ_OP_WRITE | wbc_to_write_flags(wpc->wbc), + GFP_NOFS, &iomap_ioend_bioset); + bio->bi_iter.bi_sector = iomap_sector(&wpc->iomap, pos); + bio->bi_write_hint = wpc->inode->i_write_hint; + wbc_init_bio(wpc->wbc, bio); + wpc->nr_folios = 0; + return iomap_init_ioend(wpc->inode, bio, pos, ioend_flags); +} + +static bool iomap_can_add_to_ioend(struct iomap_writepage_ctx *wpc, loff_t pos, + unsigned int map_len, u16 ioend_flags) +{ + struct iomap_ioend *ioend = wpc->wb_ctx; + + if (ioend->io_bio.bi_iter.bi_size > + iomap_max_bio_size(&wpc->iomap) - map_len) + return false; + if (ioend_flags & IOMAP_IOEND_BOUNDARY) + return false; + if ((ioend_flags & IOMAP_IOEND_NOMERGE_FLAGS) != + (ioend->io_flags & IOMAP_IOEND_NOMERGE_FLAGS)) + return false; + if (pos != ioend->io_offset + ioend->io_size) + return false; + if (!(wpc->iomap.flags & IOMAP_F_ANON_WRITE) && + iomap_sector(&wpc->iomap, pos) != bio_end_sector(&ioend->io_bio)) + return false; + /* + * Limit ioend bio chain lengths to minimise IO completion latency. This + * also prevents long tight loops ending page writeback on all the + * folios in the ioend. + */ + if (wpc->nr_folios >= IOEND_BATCH_SIZE) + return false; + return true; +} + +/* + * Test to see if we have an existing ioend structure that we could append to + * first; otherwise finish off the current ioend and start another. + * + * If a new ioend is created and cached, the old ioend is submitted to the block + * layer instantly. Batching optimisations are provided by higher level block + * plugging. + * + * At the end of a writeback pass, there will be a cached ioend remaining on the + * writepage context that the caller will need to submit. + */ +ssize_t iomap_add_to_ioend(struct iomap_writepage_ctx *wpc, struct folio *folio, + loff_t pos, loff_t end_pos, unsigned int dirty_len) +{ + struct iomap_ioend *ioend = wpc->wb_ctx; + size_t poff = offset_in_folio(folio, pos); + unsigned int ioend_flags = 0; + unsigned int map_len = min_t(u64, dirty_len, + wpc->iomap.offset + wpc->iomap.length - pos); + int error; + + trace_iomap_add_to_ioend(wpc->inode, pos, dirty_len, &wpc->iomap); + + WARN_ON_ONCE(!folio->private && map_len < dirty_len); + + switch (wpc->iomap.type) { + case IOMAP_UNWRITTEN: + ioend_flags |= IOMAP_IOEND_UNWRITTEN; + break; + case IOMAP_MAPPED: + break; + case IOMAP_HOLE: + return map_len; + default: + WARN_ON_ONCE(1); + return -EIO; + } + + if (wpc->iomap.flags & IOMAP_F_SHARED) + ioend_flags |= IOMAP_IOEND_SHARED; + if (pos == wpc->iomap.offset && (wpc->iomap.flags & IOMAP_F_BOUNDARY)) + ioend_flags |= IOMAP_IOEND_BOUNDARY; + + if (!ioend || !iomap_can_add_to_ioend(wpc, pos, map_len, ioend_flags)) { +new_ioend: + if (ioend) { + error = wpc->ops->writeback_submit(wpc, 0); + if (error) + return error; + } + wpc->wb_ctx = ioend = iomap_alloc_ioend(wpc, pos, ioend_flags); + } + + if (!bio_add_folio(&ioend->io_bio, folio, map_len, poff)) + goto new_ioend; + + if (folio_test_dropbehind(folio)) + bio_set_flag(&ioend->io_bio, BIO_COMPLETE_IN_TASK); + + /* + * Clamp io_offset and io_size to the incore EOF so that ondisk + * file size updates in the ioend completion are byte-accurate. + * This avoids recovering files with zeroed tail regions when + * writeback races with appending writes: + * + * Thread 1: Thread 2: + * ------------ ----------- + * write [A, A+B] + * update inode size to A+B + * submit I/O [A, A+BS] + * write [A+B, A+B+C] + * update inode size to A+B+C + * + * + * + * After reboot: + * 1) with A+B+C < A+BS, the file has zero padding in range + * [A+B, A+B+C] + * + * |< Block Size (BS) >| + * |DDDDDDDDDDDD0000000000000| + * ^ ^ ^ + * A A+B A+B+C + * (EOF) + * + * 2) with A+B+C > A+BS, the file has zero padding in range + * [A+B, A+BS] + * + * |< Block Size (BS) >|< Block Size (BS) >| + * |DDDDDDDDDDDD0000000000000|00000000000000000000000000| + * ^ ^ ^ ^ + * A A+B A+BS A+B+C + * (EOF) + * + * D = Valid Data + * 0 = Zero Padding + * + * Note that this defeats the ability to chain the ioends of + * appending writes. + */ + ioend->io_size += map_len; + if (ioend->io_offset + ioend->io_size > end_pos) { + if (ioend->io_offset >= end_pos) + ioend->io_size = 0; + else + ioend->io_size = end_pos - ioend->io_offset; + } + + wbc_account_cgroup_owner(wpc->wbc, folio, map_len); + return map_len; +} +EXPORT_SYMBOL_GPL(iomap_add_to_ioend); + +static u32 iomap_finish_ioend(struct iomap_ioend *ioend, int error) +{ + if (ioend->io_parent) { + struct bio *bio = &ioend->io_bio; + + ioend = ioend->io_parent; + bio_put(bio); + } + + if (error) + cmpxchg(&ioend->io_error, 0, error); + + if (!atomic_dec_and_test(&ioend->io_remaining)) + return 0; + + if (!ioend->io_error && + bio_integrity(&ioend->io_bio) && + bio_op(&ioend->io_bio) == REQ_OP_READ) { + ioend->io_error = fs_bio_integrity_verify(&ioend->io_bio, + ioend->io_sector, ioend->io_size); + } + + if (ioend->io_flags & IOMAP_IOEND_DIRECT) + return iomap_finish_ioend_direct(ioend); + if (bio_op(&ioend->io_bio) == REQ_OP_READ) + return iomap_finish_ioend_buffered_read(ioend); + return iomap_finish_ioend_buffered_write(ioend); +} + +/* + * Ioend completion routine for merged bios. This can only be called from task + * contexts as merged ioends can be of unbound length. Hence we have to break up + * the writeback completions into manageable chunks to avoid long scheduler + * holdoffs. We aim to keep scheduler holdoffs down below 10ms so that we get + * good batch processing throughput without creating adverse scheduler latency + * conditions. + */ +void iomap_finish_ioends(struct iomap_ioend *ioend, int error) +{ + struct list_head tmp; + u32 completions; + + might_sleep(); + + list_replace_init(&ioend->io_list, &tmp); + completions = iomap_finish_ioend(ioend, error); + + while (!list_empty(&tmp)) { + if (completions > IOEND_BATCH_SIZE * 8) { + cond_resched(); + completions = 0; + } + ioend = list_first_entry(&tmp, struct iomap_ioend, io_list); + list_del_init(&ioend->io_list); + completions += iomap_finish_ioend(ioend, error); + } +} +EXPORT_SYMBOL_GPL(iomap_finish_ioends); + +/* + * We can merge two adjacent ioends if they have the same set of work to do. + */ +static bool iomap_ioend_can_merge(struct iomap_ioend *ioend, + struct iomap_ioend *next) +{ + /* + * There is no point in merging reads as there is no completion + * processing that can be easily batched up for them. + */ + if (bio_op(&ioend->io_bio) == REQ_OP_READ || + bio_op(&next->io_bio) == REQ_OP_READ) + return false; + + if (ioend->io_bio.bi_status != next->io_bio.bi_status) + return false; + if (ioend->io_private != next->io_private) + return false; + if (next->io_flags & IOMAP_IOEND_BOUNDARY) + return false; + if ((ioend->io_flags & IOMAP_IOEND_NOMERGE_FLAGS) != + (next->io_flags & IOMAP_IOEND_NOMERGE_FLAGS)) + return false; + if (ioend->io_offset + ioend->io_size != next->io_offset) + return false; + /* + * Do not merge physically discontiguous ioends. The filesystem + * completion functions will have to iterate the physical + * discontiguities even if we merge the ioends at a logical level, so + * we don't gain anything by merging physical discontiguities here. + * + * We cannot use bio->bi_iter.bi_sector here as it is modified during + * submission so does not point to the start sector of the bio at + * completion. + */ + if (ioend->io_sector + (ioend->io_size >> SECTOR_SHIFT) != + next->io_sector) + return false; + return true; +} + +void iomap_ioend_try_merge(struct iomap_ioend *ioend, + struct list_head *more_ioends) +{ + struct iomap_ioend *next; + + INIT_LIST_HEAD(&ioend->io_list); + + while ((next = list_first_entry_or_null(more_ioends, struct iomap_ioend, + io_list))) { + if (!iomap_ioend_can_merge(ioend, next)) + break; + list_move_tail(&next->io_list, &ioend->io_list); + ioend->io_size += next->io_size; + } +} +EXPORT_SYMBOL_GPL(iomap_ioend_try_merge); + +static int iomap_ioend_compare(void *priv, const struct list_head *a, + const struct list_head *b) +{ + struct iomap_ioend *ia = container_of(a, struct iomap_ioend, io_list); + struct iomap_ioend *ib = container_of(b, struct iomap_ioend, io_list); + + if (ia->io_offset < ib->io_offset) + return -1; + if (ia->io_offset > ib->io_offset) + return 1; + return 0; +} + +void iomap_sort_ioends(struct list_head *ioend_list) +{ + list_sort(NULL, ioend_list, iomap_ioend_compare); +} +EXPORT_SYMBOL_GPL(iomap_sort_ioends); + +/* + * Split up to the first @max_len bytes from @ioend if the ioend covers more + * than @max_len bytes. + * + * If @is_append is set, the split will be based on the hardware limits for + * REQ_OP_ZONE_APPEND commands and can be less than @max_len if the hardware + * limits don't allow the entire @max_len length. + * + * The bio embedded into @ioend must be a REQ_OP_WRITE because the block layer + * does not allow splitting REQ_OP_ZONE_APPEND bios. The file systems has to + * switch the operation after this call, but before submitting the bio. + */ +struct iomap_ioend *iomap_split_ioend(struct iomap_ioend *ioend, + unsigned int max_len, bool is_append) +{ + struct bio *bio = &ioend->io_bio; + struct iomap_ioend *split_ioend; + unsigned int nr_segs; + int sector_offset; + struct bio *split; + + if (is_append) { + struct queue_limits *lim = bdev_limits(bio->bi_bdev); + + max_len = min(max_len, + lim->max_zone_append_sectors << SECTOR_SHIFT); + + sector_offset = bio_split_rw_at(bio, lim, &nr_segs, max_len); + if (unlikely(sector_offset < 0)) + return ERR_PTR(sector_offset); + if (!sector_offset) + return NULL; + } else { + if (bio->bi_iter.bi_size <= max_len) + return NULL; + sector_offset = max_len >> SECTOR_SHIFT; + } + + /* ensure the split ioend is still block size aligned */ + sector_offset = ALIGN_DOWN(sector_offset << SECTOR_SHIFT, + i_blocksize(ioend->io_inode)) >> SECTOR_SHIFT; + + split = bio_split(bio, sector_offset, GFP_NOFS, + &iomap_ioend_split_bioset); + if (IS_ERR(split)) + return ERR_CAST(split); + split->bi_private = bio->bi_private; + split->bi_end_io = bio->bi_end_io; + + split_ioend = iomap_init_ioend(ioend->io_inode, split, ioend->io_offset, + ioend->io_flags); + split_ioend->io_parent = ioend; + + atomic_inc(&ioend->io_remaining); + ioend->io_offset += split_ioend->io_size; + ioend->io_size -= split_ioend->io_size; + + split_ioend->io_sector = ioend->io_sector; + if (!is_append) + ioend->io_sector += (split_ioend->io_size >> SECTOR_SHIFT); + return split_ioend; +} +EXPORT_SYMBOL_GPL(iomap_split_ioend); + +static int __init iomap_ioend_init(void) +{ + const unsigned int nr_mempool_entries = 4 * (PAGE_SIZE / SECTOR_SIZE); + int error; + + error = bioset_init(&iomap_ioend_bioset, nr_mempool_entries, + offsetof(struct iomap_ioend, io_bio), + BIOSET_NEED_BVECS); + if (error) + return error; + error = bioset_init(&iomap_ioend_split_bioset, nr_mempool_entries, + offsetof(struct iomap_ioend, io_bio), + BIOSET_NEED_BVECS); + if (error) + goto out_exit_ioend_bioset; + return 0; + +out_exit_ioend_bioset: + bioset_exit(&iomap_ioend_bioset); + return error; +} +fs_initcall(iomap_ioend_init); diff --git a/fs/iomap/iter.c b/fs/iomap/iter.c new file mode 100644 index 000000000..c445a38b6 --- /dev/null +++ b/fs/iomap/iter.c @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2010 Red Hat, Inc. + * Copyright (c) 2016-2021 Christoph Hellwig. + */ +#include +#include "trace.h" + +/* + * Release the iter folio batch. Note that the iomap flag is meant to control + * the I/O path for the mapping and may not be set in error situations. + */ +static inline void iomap_iter_clean_fbatch(const struct iomap_iter *iter, + struct iomap *iomap) +{ + if (!iter->fbatch) + return; + iomap->flags &= ~IOMAP_F_FOLIO_BATCH; + if (folio_batch_count(iter->fbatch)) { + folio_batch_release(iter->fbatch); + folio_batch_reinit(iter->fbatch); + } +} + +/* Advance the current iterator position and decrement the remaining length */ +int iomap_iter_advance(struct iomap_iter *iter, u64 count) +{ + if (WARN_ON_ONCE(count > iomap_length(iter))) + return -EIO; + iter->pos += count; + iter->len -= count; + return 0; +} + +static inline void iomap_iter_done(struct iomap_iter *iter) +{ + WARN_ON_ONCE(iter->iomap.offset > iter->pos); + WARN_ON_ONCE(iter->iomap.length == 0); + WARN_ON_ONCE(iter->iomap.offset + iter->iomap.length <= iter->pos); + WARN_ON_ONCE(iter->iomap.flags & IOMAP_F_STALE); + + iter->iter_start_pos = iter->pos; + + trace_iomap_iter_dstmap(iter->inode, &iter->iomap); + if (iter->srcmap.type != IOMAP_HOLE) + trace_iomap_iter_srcmap(iter->inode, &iter->srcmap); +} + +/** + * iomap_iter_continue - decide whether iteration should continue + * @iter: iteration structure + * @iomap: the mapping that was just processed + * @srcmap: the source mapping that was just processed + * + * Helper normally called via iomap_iter_next(). Called after the previous + * mapping has been finished to determine whether there is more of the file + * range left to process. + * + * Returns 1 if there is more work to do, in which case @iomap and @srcmap are + * cleared so the caller can produce the next mapping; zero if the range is + * fully consumed; or a negative errno on error. + */ +int iomap_iter_continue(const struct iomap_iter *iter, struct iomap *iomap, + struct iomap *srcmap, int ret) +{ + const bool stale = iomap->flags & IOMAP_F_STALE; + const ssize_t advanced = iter->pos - iter->iter_start_pos; + + if (ret < 0 && !advanced) + return ret; + + /* + * Use iter->len to determine whether to continue onto the next mapping. + * Explicitly terminate on error status or if the current iter has not + * advanced at all (i.e. no work was done for some reason) unless the + * mapping has been marked stale and needs to be reprocessed. + */ + if (WARN_ON_ONCE(iter->status > 0)) + /* detect old return semantics where this would advance */ + ret = -EIO; + else if (iter->status < 0) + ret = iter->status; + else if (iter->len == 0 || (!advanced && !stale)) + ret = 0; + else + ret = 1; + + iomap_iter_clean_fbatch(iter, iomap); + + if (ret <= 0) + return ret; + + memset(iomap, 0, sizeof(*iomap)); + memset(srcmap, 0, sizeof(*srcmap)); + + return ret; +} +EXPORT_SYMBOL_GPL(iomap_iter_continue); + +/** + * iomap_iter - iterate over ranges in a file + * @iter: iteration structure + * @ops: iomap ops provided by the filesystem + * + * Iterate over filesystem-provided space mappings for the provided file range. + * + * This function handles cleanup of resources acquired for iteration when the + * filesystem indicates there are no more space mappings, which means that this + * function must be called in a loop that continues as long it returns a + * positive value. If 0 or a negative value is returned, the caller must not + * return to the loop body. Within a loop body, there are two ways to break out + * of the loop body: leave @iter.status unchanged, or set it to a negative + * errno. + */ +int iomap_iter(struct iomap_iter *iter, const struct iomap_ops *ops) +{ + int ret; + + trace_iomap_iter(iter, ops, _RET_IP_); + + if (ops->iomap_next) + ret = ops->iomap_next(iter, &iter->iomap, &iter->srcmap); + else + ret = iomap_iter_next(iter, &iter->iomap, &iter->srcmap, + ops->iomap_begin, ops->iomap_end); + + iter->status = 0; + if (ret > 0) + iomap_iter_done(iter); + else if (ret < 0) + iomap_iter_clean_fbatch(iter, &iter->iomap); + + return ret; +} diff --git a/fs/iomap/seek.c b/fs/iomap/seek.c new file mode 100644 index 000000000..6cbc587c9 --- /dev/null +++ b/fs/iomap/seek.c @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2017 Red Hat, Inc. + * Copyright (c) 2018-2021 Christoph Hellwig. + */ +#include +#include + +static int iomap_seek_hole_iter(struct iomap_iter *iter, + loff_t *hole_pos) +{ + loff_t length = iomap_length(iter); + + switch (iter->iomap.type) { + case IOMAP_UNWRITTEN: + *hole_pos = mapping_seek_hole_data(iter->inode->i_mapping, + iter->pos, iter->pos + length, SEEK_HOLE); + if (*hole_pos == iter->pos + length) + return iomap_iter_advance(iter, length); + return 0; + case IOMAP_HOLE: + *hole_pos = iter->pos; + return 0; + default: + return iomap_iter_advance(iter, length); + } +} + +loff_t +iomap_seek_hole(struct inode *inode, loff_t pos, const struct iomap_ops *ops) +{ + loff_t size = i_size_read(inode); + struct iomap_iter iter = { + .inode = inode, + .pos = pos, + .flags = IOMAP_REPORT, + }; + int ret; + + /* Nothing to be found before or beyond the end of the file. */ + if (pos < 0 || pos >= size) + return -ENXIO; + + iter.len = size - pos; + while ((ret = iomap_iter(&iter, ops)) > 0) + iter.status = iomap_seek_hole_iter(&iter, &pos); + if (ret < 0) + return ret; + if (iter.len) /* found hole before EOF */ + return pos; + return size; +} +EXPORT_SYMBOL_GPL(iomap_seek_hole); + +static int iomap_seek_data_iter(struct iomap_iter *iter, + loff_t *hole_pos) +{ + loff_t length = iomap_length(iter); + + switch (iter->iomap.type) { + case IOMAP_HOLE: + return iomap_iter_advance(iter, length); + case IOMAP_UNWRITTEN: + *hole_pos = mapping_seek_hole_data(iter->inode->i_mapping, + iter->pos, iter->pos + length, SEEK_DATA); + if (*hole_pos < 0) + return iomap_iter_advance(iter, length); + return 0; + default: + *hole_pos = iter->pos; + return 0; + } +} + +loff_t +iomap_seek_data(struct inode *inode, loff_t pos, const struct iomap_ops *ops) +{ + loff_t size = i_size_read(inode); + struct iomap_iter iter = { + .inode = inode, + .pos = pos, + .flags = IOMAP_REPORT, + }; + int ret; + + /* Nothing to be found before or beyond the end of the file. */ + if (pos < 0 || pos >= size) + return -ENXIO; + + iter.len = size - pos; + while ((ret = iomap_iter(&iter, ops)) > 0) + iter.status = iomap_seek_data_iter(&iter, &pos); + if (ret < 0) + return ret; + if (iter.len) /* found data before EOF */ + return pos; + /* We've reached the end of the file without finding data */ + return -ENXIO; +} +EXPORT_SYMBOL_GPL(iomap_seek_data); diff --git a/fs/iomap/swapfile.c b/fs/iomap/swapfile.c new file mode 100644 index 000000000..0db77c449 --- /dev/null +++ b/fs/iomap/swapfile.c @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2018 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include +#include + +/* Swapfile activation */ + +struct iomap_swapfile_info { + struct iomap iomap; /* accumulated iomap */ + struct swap_info_struct *sis; + uint64_t lowest_ppage; /* lowest physical addr seen (pages) */ + uint64_t highest_ppage; /* highest physical addr seen (pages) */ + unsigned long nr_pages; /* number of pages collected */ + int nr_extents; /* extent count */ + struct file *file; +}; + +/* + * Collect physical extents for this swap file. Physical extents reported to + * the swap code must be trimmed to align to a page boundary. The logical + * offset within the file is irrelevant since the swapfile code maps logical + * page numbers of the swap device to the physical page-aligned extents. + */ +static int iomap_swapfile_add_extent(struct iomap_swapfile_info *isi) +{ + struct iomap *iomap = &isi->iomap; + unsigned long nr_pages; + unsigned long max_pages; + uint64_t first_ppage; + uint64_t first_ppage_reported; + uint64_t next_ppage; + int error; + + if (unlikely(isi->nr_pages >= isi->sis->max)) + return 0; + max_pages = isi->sis->max - isi->nr_pages; + + /* + * Round the start up and the end down so that the physical + * extent aligns to a page boundary. + */ + first_ppage = ALIGN(iomap->addr, PAGE_SIZE) >> PAGE_SHIFT; + next_ppage = ALIGN_DOWN(iomap->addr + iomap->length, PAGE_SIZE) >> + PAGE_SHIFT; + + /* Skip too-short physical extents. */ + if (first_ppage >= next_ppage) + return 0; + nr_pages = next_ppage - first_ppage; + nr_pages = min(nr_pages, max_pages); + + /* + * Calculate how much swap space we're adding; the first page contains + * the swap header and doesn't count. The mm still wants that first + * page fed to add_swap_extent, however. + */ + first_ppage_reported = first_ppage; + if (iomap->offset == 0) + first_ppage_reported++; + if (isi->lowest_ppage > first_ppage_reported) + isi->lowest_ppage = first_ppage_reported; + if (isi->highest_ppage < (next_ppage - 1)) + isi->highest_ppage = next_ppage - 1; + + /* Add extent, set up for the next call. */ + error = add_swap_extent(isi->sis, isi->nr_pages, nr_pages, first_ppage); + if (error < 0) + return error; + isi->nr_extents += error; + isi->nr_pages += nr_pages; + return 0; +} + +static int iomap_swapfile_fail(struct iomap_swapfile_info *isi, const char *str) +{ + char *buf, *p = ERR_PTR(-ENOMEM); + + buf = kmalloc(PATH_MAX, GFP_KERNEL); + if (buf) + p = file_path(isi->file, buf, PATH_MAX); + pr_err("swapon: file %s %s\n", IS_ERR(p) ? "" : p, str); + kfree(buf); + return -EINVAL; +} + +/* + * Accumulate iomaps for this swap file. We have to accumulate iomaps because + * swap only cares about contiguous page-aligned physical extents and makes no + * distinction between written and unwritten extents. + */ +static int iomap_swapfile_iter(struct iomap_iter *iter, + struct iomap *iomap, struct iomap_swapfile_info *isi) +{ + switch (iomap->type) { + case IOMAP_MAPPED: + case IOMAP_UNWRITTEN: + /* Only real or unwritten extents. */ + break; + case IOMAP_INLINE: + /* No inline data. */ + return iomap_swapfile_fail(isi, "is inline"); + default: + return iomap_swapfile_fail(isi, "has unallocated extents"); + } + + /* No uncommitted metadata or shared blocks. */ + if (iomap->flags & IOMAP_F_DIRTY) + return iomap_swapfile_fail(isi, "is not committed"); + if (iomap->flags & IOMAP_F_SHARED) + return iomap_swapfile_fail(isi, "has shared extents"); + + /* Only one bdev per swap file. */ + if (iomap->bdev != isi->sis->bdev) + return iomap_swapfile_fail(isi, "outside the main device"); + + if (isi->iomap.length == 0) { + /* No accumulated extent, so just store it. */ + memcpy(&isi->iomap, iomap, sizeof(isi->iomap)); + } else if (isi->iomap.addr + isi->iomap.length == iomap->addr) { + /* Append this to the accumulated extent. */ + isi->iomap.length += iomap->length; + } else { + /* Otherwise, add the retained iomap and store this one. */ + int error = iomap_swapfile_add_extent(isi); + if (error) + return error; + memcpy(&isi->iomap, iomap, sizeof(isi->iomap)); + } + + return iomap_iter_advance_full(iter); +} + +/* + * Iterate a swap file's iomaps to construct physical extents that can be + * passed to the swapfile subsystem. + */ +int iomap_swapfile_activate(struct swap_info_struct *sis, + struct file *swap_file, sector_t *pagespan, + const struct iomap_ops *ops) +{ + struct inode *inode = swap_file->f_mapping->host; + struct iomap_iter iter = { + .inode = inode, + .pos = 0, + .len = ALIGN_DOWN(i_size_read(inode), PAGE_SIZE), + .flags = IOMAP_REPORT, + }; + struct iomap_swapfile_info isi = { + .sis = sis, + .lowest_ppage = (sector_t)-1ULL, + .file = swap_file, + }; + int ret; + + /* + * Persist all file mapping metadata so that we won't have any + * IOMAP_F_DIRTY iomaps. + */ + ret = vfs_fsync(swap_file, 1); + if (ret) + return ret; + + while ((ret = iomap_iter(&iter, ops)) > 0) + iter.status = iomap_swapfile_iter(&iter, &iter.iomap, &isi); + if (ret < 0) + return ret; + + if (isi.iomap.length) { + ret = iomap_swapfile_add_extent(&isi); + if (ret) + return ret; + } + + /* + * If this swapfile doesn't contain even a single page-aligned + * contiguous range of blocks, reject this useless swapfile to + * prevent confusion later on. + */ + if (isi.nr_pages == 0) { + pr_warn("swapon: Cannot find a single usable page in file.\n"); + return -EINVAL; + } + + *pagespan = 1 + isi.highest_ppage - isi.lowest_ppage; + sis->max = isi.nr_pages; + sis->pages = isi.nr_pages - 1; + return isi.nr_extents; +} +EXPORT_SYMBOL_GPL(iomap_swapfile_activate); diff --git a/fs/iomap/trace.c b/fs/iomap/trace.c new file mode 100644 index 000000000..da217246b --- /dev/null +++ b/fs/iomap/trace.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2019 Christoph Hellwig + */ +#include + +/* + * We include this last to have the helpers above available for the trace + * event implementations. + */ +#define CREATE_TRACE_POINTS +#include "trace.h" diff --git a/fs/iomap/trace.h b/fs/iomap/trace.h new file mode 100644 index 000000000..e1ea8392c --- /dev/null +++ b/fs/iomap/trace.h @@ -0,0 +1,333 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2009-2021 Christoph Hellwig + * + * NOTE: none of these tracepoints shall be considered a stable kernel ABI + * as they can change at any time. + * + * Current conventions for printing numbers measuring specific units: + * + * offset: byte offset into a subcomponent of a file operation + * pos: file offset, in bytes + * length: length of a file operation, in bytes + * ino: inode number + * + * Numbers describing space allocations should be formatted in hexadecimal. + */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM iomap + +#if !defined(_IOMAP_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _IOMAP_TRACE_H + +#include + +struct inode; + +DECLARE_EVENT_CLASS(iomap_readpage_class, + TP_PROTO(struct inode *inode, int nr_pages), + TP_ARGS(inode, nr_pages), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u64, ino) + __field(int, nr_pages) + ), + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->nr_pages = nr_pages; + ), + TP_printk("dev %d:%d ino 0x%llx nr_pages %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->nr_pages) +) + +#define DEFINE_READPAGE_EVENT(name) \ +DEFINE_EVENT(iomap_readpage_class, name, \ + TP_PROTO(struct inode *inode, int nr_pages), \ + TP_ARGS(inode, nr_pages)) +DEFINE_READPAGE_EVENT(iomap_readpage); +DEFINE_READPAGE_EVENT(iomap_readahead); + +DECLARE_EVENT_CLASS(iomap_range_class, + TP_PROTO(struct inode *inode, loff_t off, u64 len), + TP_ARGS(inode, off, len), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u64, ino) + __field(loff_t, size) + __field(loff_t, offset) + __field(u64, length) + ), + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->size = i_size_read(inode); + __entry->offset = off; + __entry->length = len; + ), + TP_printk("dev %d:%d ino 0x%llx size 0x%llx offset 0x%llx length 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->size, + __entry->offset, + __entry->length) +) + +#define DEFINE_RANGE_EVENT(name) \ +DEFINE_EVENT(iomap_range_class, name, \ + TP_PROTO(struct inode *inode, loff_t off, u64 len),\ + TP_ARGS(inode, off, len)) +DEFINE_RANGE_EVENT(iomap_writeback_folio); +DEFINE_RANGE_EVENT(iomap_release_folio); +DEFINE_RANGE_EVENT(iomap_invalidate_folio); +DEFINE_RANGE_EVENT(iomap_dio_invalidate_fail); +DEFINE_RANGE_EVENT(iomap_dio_rw_queued); +DEFINE_RANGE_EVENT(iomap_zero_iter); + +#define IOMAP_TYPE_STRINGS \ + { IOMAP_HOLE, "HOLE" }, \ + { IOMAP_DELALLOC, "DELALLOC" }, \ + { IOMAP_MAPPED, "MAPPED" }, \ + { IOMAP_UNWRITTEN, "UNWRITTEN" }, \ + { IOMAP_INLINE, "INLINE" } + +#define IOMAP_FLAGS_STRINGS \ + { IOMAP_WRITE, "WRITE" }, \ + { IOMAP_ZERO, "ZERO" }, \ + { IOMAP_REPORT, "REPORT" }, \ + { IOMAP_FAULT, "FAULT" }, \ + { IOMAP_DIRECT, "DIRECT" }, \ + { IOMAP_NOWAIT, "NOWAIT" }, \ + { IOMAP_OVERWRITE_ONLY, "OVERWRITE_ONLY" }, \ + { IOMAP_UNSHARE, "UNSHARE" }, \ + { IOMAP_DAX, "DAX" }, \ + { IOMAP_ATOMIC, "ATOMIC" }, \ + { IOMAP_DONTCACHE, "DONTCACHE" } + +#define IOMAP_F_FLAGS_STRINGS \ + { IOMAP_F_NEW, "NEW" }, \ + { IOMAP_F_DIRTY, "DIRTY" }, \ + { IOMAP_F_SHARED, "SHARED" }, \ + { IOMAP_F_MERGED, "MERGED" }, \ + { IOMAP_F_BUFFER_HEAD, "BH" }, \ + { IOMAP_F_XATTR, "XATTR" }, \ + { IOMAP_F_BOUNDARY, "BOUNDARY" }, \ + { IOMAP_F_ANON_WRITE, "ANON_WRITE" }, \ + { IOMAP_F_ATOMIC_BIO, "ATOMIC_BIO" }, \ + { IOMAP_F_PRIVATE, "PRIVATE" }, \ + { IOMAP_F_SIZE_CHANGED, "SIZE_CHANGED" }, \ + { IOMAP_F_STALE, "STALE" }, \ + { IOMAP_F_FSVERITY, "FSVERITY" }, \ + { IOMAP_F_ZERO_TAIL, "ZERO TAIL" } + + +#define IOMAP_DIO_STRINGS \ + {IOMAP_DIO_FORCE_WAIT, "DIO_FORCE_WAIT" }, \ + {IOMAP_DIO_OVERWRITE_ONLY, "DIO_OVERWRITE_ONLY" }, \ + {IOMAP_DIO_PARTIAL, "DIO_PARTIAL" }, \ + {IOMAP_DIO_FSBLOCK_ALIGNED, "DIO_FSBLOCK_ALIGNED" } + +DECLARE_EVENT_CLASS(iomap_class, + TP_PROTO(struct inode *inode, struct iomap *iomap), + TP_ARGS(inode, iomap), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u64, ino) + __field(u64, addr) + __field(loff_t, offset) + __field(u64, length) + __field(u16, type) + __field(u16, flags) + __field(dev_t, bdev) + ), + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->addr = iomap->addr; + __entry->offset = iomap->offset; + __entry->length = iomap->length; + __entry->type = iomap->type; + __entry->flags = iomap->flags; + __entry->bdev = iomap->bdev ? iomap->bdev->bd_dev : 0; + ), + TP_printk("dev %d:%d ino 0x%llx bdev %d:%d addr 0x%llx offset 0x%llx " + "length 0x%llx type %s (0x%x) flags %s (0x%x)", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + MAJOR(__entry->bdev), MINOR(__entry->bdev), + __entry->addr, + __entry->offset, + __entry->length, + __print_symbolic(__entry->type, IOMAP_TYPE_STRINGS), + __entry->type, + __print_flags(__entry->flags, "|", IOMAP_F_FLAGS_STRINGS), + __entry->flags) +) + +#define DEFINE_IOMAP_EVENT(name) \ +DEFINE_EVENT(iomap_class, name, \ + TP_PROTO(struct inode *inode, struct iomap *iomap), \ + TP_ARGS(inode, iomap)) +DEFINE_IOMAP_EVENT(iomap_iter_dstmap); +DEFINE_IOMAP_EVENT(iomap_iter_srcmap); + +TRACE_EVENT(iomap_add_to_ioend, + TP_PROTO(struct inode *inode, u64 pos, unsigned int dirty_len, + struct iomap *iomap), + TP_ARGS(inode, pos, dirty_len, iomap), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u64, ino) + __field(u64, pos) + __field(u64, dirty_len) + __field(u64, addr) + __field(loff_t, offset) + __field(u64, length) + __field(u16, type) + __field(u16, flags) + __field(dev_t, bdev) + ), + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->pos = pos; + __entry->dirty_len = dirty_len; + __entry->addr = iomap->addr; + __entry->offset = iomap->offset; + __entry->length = iomap->length; + __entry->type = iomap->type; + __entry->flags = iomap->flags; + __entry->bdev = iomap->bdev ? iomap->bdev->bd_dev : 0; + ), + TP_printk("dev %d:%d ino 0x%llx bdev %d:%d pos 0x%llx dirty len 0x%llx " + "addr 0x%llx offset 0x%llx length 0x%llx type %s (0x%x) flags %s (0x%x)", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + MAJOR(__entry->bdev), MINOR(__entry->bdev), + __entry->pos, + __entry->dirty_len, + __entry->addr, + __entry->offset, + __entry->length, + __print_symbolic(__entry->type, IOMAP_TYPE_STRINGS), + __entry->type, + __print_flags(__entry->flags, "|", IOMAP_F_FLAGS_STRINGS), + __entry->flags) +); + +TRACE_EVENT(iomap_iter, + TP_PROTO(struct iomap_iter *iter, const void *ops, + unsigned long caller), + TP_ARGS(iter, ops, caller), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u64, ino) + __field(loff_t, pos) + __field(u64, length) + __field(int, status) + __field(unsigned int, flags) + __field(const void *, ops) + __field(unsigned long, caller) + ), + TP_fast_assign( + __entry->dev = iter->inode->i_sb->s_dev; + __entry->ino = iter->inode->i_ino; + __entry->pos = iter->pos; + __entry->length = iomap_length(iter); + __entry->status = iter->status; + __entry->flags = iter->flags; + __entry->ops = ops; + __entry->caller = caller; + ), + TP_printk("dev %d:%d ino 0x%llx pos 0x%llx length 0x%llx status %d flags %s (0x%x) ops %ps caller %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->pos, + __entry->length, + __entry->status, + __print_flags(__entry->flags, "|", IOMAP_FLAGS_STRINGS), + __entry->flags, + __entry->ops, + (void *)__entry->caller) +); + +TRACE_EVENT(iomap_dio_rw_begin, + TP_PROTO(struct kiocb *iocb, struct iov_iter *iter, + unsigned int dio_flags, size_t done_before), + TP_ARGS(iocb, iter, dio_flags, done_before), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u64, ino) + __field(loff_t, isize) + __field(loff_t, pos) + __field(size_t, count) + __field(size_t, done_before) + __field(int, ki_flags) + __field(unsigned int, dio_flags) + __field(bool, aio) + ), + TP_fast_assign( + __entry->dev = file_inode(iocb->ki_filp)->i_sb->s_dev; + __entry->ino = file_inode(iocb->ki_filp)->i_ino; + __entry->isize = file_inode(iocb->ki_filp)->i_size; + __entry->pos = iocb->ki_pos; + __entry->count = iov_iter_count(iter); + __entry->done_before = done_before; + __entry->ki_flags = iocb->ki_flags; + __entry->dio_flags = dio_flags; + __entry->aio = !is_sync_kiocb(iocb); + ), + TP_printk("dev %d:%d ino 0x%llx size 0x%llx offset 0x%llx length 0x%zx done_before 0x%zx flags %s dio_flags %s aio %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->isize, + __entry->pos, + __entry->count, + __entry->done_before, + __print_flags(__entry->ki_flags, "|", TRACE_IOCB_STRINGS), + __print_flags(__entry->dio_flags, "|", IOMAP_DIO_STRINGS), + __entry->aio) +); + +TRACE_EVENT(iomap_dio_complete, + TP_PROTO(struct kiocb *iocb, int error, ssize_t ret), + TP_ARGS(iocb, error, ret), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u64, ino) + __field(loff_t, isize) + __field(loff_t, pos) + __field(int, ki_flags) + __field(bool, aio) + __field(int, error) + __field(ssize_t, ret) + ), + TP_fast_assign( + __entry->dev = file_inode(iocb->ki_filp)->i_sb->s_dev; + __entry->ino = file_inode(iocb->ki_filp)->i_ino; + __entry->isize = file_inode(iocb->ki_filp)->i_size; + __entry->pos = iocb->ki_pos; + __entry->ki_flags = iocb->ki_flags; + __entry->aio = !is_sync_kiocb(iocb); + __entry->error = error; + __entry->ret = ret; + ), + TP_printk("dev %d:%d ino 0x%llx size 0x%llx offset 0x%llx flags %s aio %d error %d ret %zd", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->isize, + __entry->pos, + __print_flags(__entry->ki_flags, "|", TRACE_IOCB_STRINGS), + __entry->aio, + __entry->error, + __entry->ret) +); + +#endif /* _IOMAP_TRACE_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE trace +#include diff --git a/fs/isofs/Kconfig b/fs/isofs/Kconfig new file mode 100644 index 000000000..51434f2a4 --- /dev/null +++ b/fs/isofs/Kconfig @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: GPL-2.0-only +config ISO9660_FS + tristate "ISO 9660 CDROM file system support" + select BUFFER_HEAD + help + This is the standard file system used on CD-ROMs. It was previously + known as "High Sierra File System" and is called "hsfs" on other + Unix systems. The so-called Rock-Ridge extensions which allow for + long Unix filenames and symbolic links are also supported by this + driver. If you have a CD-ROM drive and want to do more with it than + just listen to audio CDs and watch its LEDs, say Y (and read + and the CD-ROM-HOWTO, + available from ), thereby + enlarging your kernel by about 27 KB; otherwise say N. + + To compile this file system support as a module, choose M here: the + module will be called isofs. + +config JOLIET + bool "Microsoft Joliet CDROM extensions" + depends on ISO9660_FS + select NLS + help + Joliet is a Microsoft extension for the ISO 9660 CD-ROM file system + which allows for long filenames in unicode format (unicode is the + new 16 bit character code, successor to ASCII, which encodes the + characters of almost all languages of the world; see + for more information). Say Y here if you + want to be able to read Joliet CD-ROMs under Linux. + +config ZISOFS + bool "Transparent decompression extension" + depends on ISO9660_FS + select ZLIB_INFLATE + help + This is a Linux-specific extension to RockRidge which lets you store + data in compressed form on a CD-ROM and have it transparently + decompressed when the CD-ROM is accessed. See + for the tools + necessary to create such a filesystem. Say Y here if you want to be + able to read such compressed CD-ROMs. diff --git a/fs/isofs/Makefile b/fs/isofs/Makefile new file mode 100644 index 000000000..b25bc542a --- /dev/null +++ b/fs/isofs/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the Linux isofs filesystem routines. +# + +obj-$(CONFIG_ISO9660_FS) += isofs.o + +isofs-y := namei.o inode.o dir.o util.o rock.o export.o +isofs-$(CONFIG_JOLIET) += joliet.o +isofs-$(CONFIG_ZISOFS) += compress.o diff --git a/fs/isofs/compress.c b/fs/isofs/compress.c new file mode 100644 index 000000000..f9869d62b --- /dev/null +++ b/fs/isofs/compress.c @@ -0,0 +1,386 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* -*- linux-c -*- ------------------------------------------------------- * + * + * Copyright 2001 H. Peter Anvin - All Rights Reserved + * + * ----------------------------------------------------------------------- */ + +/* + * linux/fs/isofs/compress.c + * + * Transparent decompression of files on an iso9660 filesystem + */ + +#include +#include +#include + +#include +#include +#include + +#include "isofs.h" +#include "zisofs.h" + +/* This should probably be global. */ +static char zisofs_sink_page[PAGE_SIZE]; + +/* + * This contains the zlib memory allocation and the mutex for the + * allocation; this avoids failures at block-decompression time. + */ +static void *zisofs_zlib_workspace; +static DEFINE_MUTEX(zisofs_zlib_lock); + +/* + * Read data of @inode from @block_start to @block_end and uncompress + * to one zisofs block. Store the data in the @pages array with @pcount + * entries. Start storing at offset @poffset of the first page. + */ +static loff_t zisofs_uncompress_block(struct inode *inode, loff_t block_start, + loff_t block_end, int pcount, + struct page **pages, unsigned poffset, + int *errp) +{ + unsigned int zisofs_block_shift = ISOFS_I(inode)->i_format_parm[1]; + unsigned int bufsize = ISOFS_BUFFER_SIZE(inode); + unsigned int bufshift = ISOFS_BUFFER_BITS(inode); + unsigned int bufmask = bufsize - 1; + int i, block_size = block_end - block_start; + z_stream stream = { .total_out = 0, + .avail_in = 0, + .avail_out = 0, }; + int zerr; + int needblocks = (block_size + (block_start & bufmask) + bufmask) + >> bufshift; + int haveblocks; + blkcnt_t blocknum; + struct buffer_head **bhs; + int curbh, curpage; + + if (block_size > deflateBound(1UL << zisofs_block_shift)) { + *errp = -EIO; + return 0; + } + /* Empty block? */ + if (block_size == 0) { + for ( i = 0 ; i < pcount ; i++ ) { + unsigned int off = i ? 0 : poffset; + + if (!pages[i]) + continue; + memzero_page(pages[i], off, PAGE_SIZE - off); + SetPageUptodate(pages[i]); + } + return (((loff_t)pcount) << PAGE_SHIFT) - poffset; + } + + /* Because zlib is not thread-safe, do all the I/O at the top. */ + blocknum = block_start >> bufshift; + bhs = kzalloc_objs(*bhs, needblocks + 1); + if (!bhs) { + *errp = -ENOMEM; + return 0; + } + haveblocks = isofs_get_blocks(inode, blocknum, bhs, needblocks); + bh_read_batch(haveblocks, bhs); + + curbh = 0; + curpage = 0; + /* + * First block is special since it may be fractional. We also wait for + * it before grabbing the zlib mutex; odds are that the subsequent + * blocks are going to come in in short order so we don't hold the zlib + * mutex longer than necessary. + */ + + if (!bhs[0]) + goto b_eio; + + wait_on_buffer(bhs[0]); + if (!buffer_uptodate(bhs[0])) { + *errp = -EIO; + goto b_eio; + } + + stream.workspace = zisofs_zlib_workspace; + mutex_lock(&zisofs_zlib_lock); + + zerr = zlib_inflateInit(&stream); + if (zerr != Z_OK) { + if (zerr == Z_MEM_ERROR) + *errp = -ENOMEM; + else + *errp = -EIO; + printk(KERN_DEBUG "zisofs: zisofs_inflateInit returned %d\n", + zerr); + goto z_eio; + } + + while (curpage < pcount && curbh < haveblocks && + zerr != Z_STREAM_END) { + if (!stream.avail_out) { + if (pages[curpage]) { + stream.next_out = kmap_local_page(pages[curpage]) + + poffset; + stream.avail_out = PAGE_SIZE - poffset; + poffset = 0; + } else { + stream.next_out = (void *)&zisofs_sink_page; + stream.avail_out = PAGE_SIZE; + } + } + if (!stream.avail_in) { + wait_on_buffer(bhs[curbh]); + if (!buffer_uptodate(bhs[curbh])) { + *errp = -EIO; + break; + } + stream.next_in = bhs[curbh]->b_data + + (block_start & bufmask); + stream.avail_in = min_t(unsigned, bufsize - + (block_start & bufmask), + block_size); + block_size -= stream.avail_in; + block_start = 0; + } + + while (stream.avail_out && stream.avail_in) { + zerr = zlib_inflate(&stream, Z_SYNC_FLUSH); + if (zerr == Z_BUF_ERROR && stream.avail_in == 0) + break; + if (zerr == Z_STREAM_END) + break; + if (zerr != Z_OK) { + /* EOF, error, or trying to read beyond end of input */ + if (zerr == Z_MEM_ERROR) + *errp = -ENOMEM; + else { + printk(KERN_DEBUG + "zisofs: zisofs_inflate returned" + " %d, inode = %llu," + " page idx = %d, bh idx = %d," + " avail_in = %ld," + " avail_out = %ld\n", + zerr, inode->i_ino, curpage, + curbh, stream.avail_in, + stream.avail_out); + *errp = -EIO; + } + goto inflate_out; + } + } + + if (!stream.avail_out) { + /* This page completed */ + if (pages[curpage]) { + flush_dcache_page(pages[curpage]); + SetPageUptodate(pages[curpage]); + } + if (stream.next_out != (unsigned char *)zisofs_sink_page) { + kunmap_local(stream.next_out); + stream.next_out = NULL; + } + curpage++; + } + if (!stream.avail_in) + curbh++; + } +inflate_out: + zlib_inflateEnd(&stream); + if (stream.next_out && stream.next_out != (unsigned char *)zisofs_sink_page) + kunmap_local(stream.next_out); + +z_eio: + mutex_unlock(&zisofs_zlib_lock); + +b_eio: + for (i = 0; i < haveblocks; i++) + brelse(bhs[i]); + kfree(bhs); + return stream.total_out; +} + +/* + * Uncompress data so that pages[full_page] is fully uptodate and possibly + * fills in other pages if we have data for them. + */ +static int zisofs_fill_pages(struct inode *inode, int full_page, int pcount, + struct page **pages) +{ + loff_t start_off, end_off; + loff_t block_start, block_end; + unsigned int header_size = ISOFS_I(inode)->i_format_parm[0]; + unsigned int zisofs_block_shift = ISOFS_I(inode)->i_format_parm[1]; + unsigned int blockptr; + loff_t poffset = 0; + blkcnt_t cstart_block, cend_block; + struct buffer_head *bh; + unsigned int blkbits = ISOFS_BUFFER_BITS(inode); + unsigned int blksize = 1 << blkbits; + int err; + loff_t ret; + + BUG_ON(!pages[full_page]); + + /* + * We want to read at least 'full_page' page. Because we have to + * uncompress the whole compression block anyway, fill the surrounding + * pages with the data we have anyway... + */ + start_off = page_offset(pages[full_page]); + end_off = min_t(loff_t, start_off + PAGE_SIZE, inode->i_size); + + cstart_block = start_off >> zisofs_block_shift; + cend_block = (end_off + (1 << zisofs_block_shift) - 1) + >> zisofs_block_shift; + + WARN_ON(start_off - (full_page << PAGE_SHIFT) != + ((cstart_block << zisofs_block_shift) & PAGE_MASK)); + + /* Find the pointer to this specific chunk */ + /* Note: we're not using isonum_731() here because the data is known aligned */ + /* Note: header_size is in 32-bit words (4 bytes) */ + blockptr = (header_size + cstart_block) << 2; + bh = isofs_bread(inode, blockptr >> blkbits); + if (!bh) + return -EIO; + block_start = le32_to_cpu(*(__le32 *) + (bh->b_data + (blockptr & (blksize - 1)))); + + while (cstart_block < cend_block && pcount > 0) { + /* Load end of the compressed block in the file */ + blockptr += 4; + /* Traversed to next block? */ + if (!(blockptr & (blksize - 1))) { + brelse(bh); + + bh = isofs_bread(inode, blockptr >> blkbits); + if (!bh) + return -EIO; + } + block_end = le32_to_cpu(*(__le32 *) + (bh->b_data + (blockptr & (blksize - 1)))); + if (block_start > block_end) { + brelse(bh); + return -EIO; + } + err = 0; + ret = zisofs_uncompress_block(inode, block_start, block_end, + pcount, pages, poffset, &err); + poffset += ret; + pages += poffset >> PAGE_SHIFT; + pcount -= poffset >> PAGE_SHIFT; + full_page -= poffset >> PAGE_SHIFT; + poffset &= ~PAGE_MASK; + + if (err) { + brelse(bh); + /* + * Did we finish reading the page we really wanted + * to read? + */ + if (full_page < 0) + return 0; + return err; + } + + block_start = block_end; + cstart_block++; + } + + if (poffset && *pages) { + memzero_page(*pages, poffset, PAGE_SIZE - poffset); + SetPageUptodate(*pages); + } + brelse(bh); + return 0; +} + +/* + * When decompressing, we typically obtain more than one page + * per reference. We inject the additional pages into the page + * cache as a form of readahead. + */ +static int zisofs_read_folio(struct file *file, struct folio *folio) +{ + struct inode *inode = file_inode(file); + struct address_space *mapping = inode->i_mapping; + int err; + int i, pcount, full_page; + unsigned int zisofs_block_shift = ISOFS_I(inode)->i_format_parm[1]; + unsigned int zisofs_pages_per_cblock = + PAGE_SHIFT <= zisofs_block_shift ? + (1 << (zisofs_block_shift - PAGE_SHIFT)) : 0; + struct page **pages; + pgoff_t index = folio->index, end_index; + + end_index = (inode->i_size + PAGE_SIZE - 1) >> PAGE_SHIFT; + /* + * If this folio is wholly outside i_size we just return zero; + * do_generic_file_read() will handle this for us + */ + if (index >= end_index) { + folio_end_read(folio, true); + return 0; + } + + if (PAGE_SHIFT <= zisofs_block_shift) { + /* We have already been given one page, this is the one + we must do. */ + full_page = index & (zisofs_pages_per_cblock - 1); + pcount = min_t(int, zisofs_pages_per_cblock, + end_index - (index & ~(zisofs_pages_per_cblock - 1))); + index -= full_page; + } else { + full_page = 0; + pcount = 1; + } + pages = kzalloc_objs(*pages, + max_t(unsigned int, zisofs_pages_per_cblock, 1)); + if (!pages) { + folio_unlock(folio); + return -ENOMEM; + } + pages[full_page] = &folio->page; + + for (i = 0; i < pcount; i++, index++) { + if (i != full_page) + pages[i] = grab_cache_page_nowait(mapping, index); + } + + err = zisofs_fill_pages(inode, full_page, pcount, pages); + + /* Release any residual pages, do not SetPageUptodate */ + for (i = 0; i < pcount; i++) { + if (pages[i]) { + flush_dcache_page(pages[i]); + unlock_page(pages[i]); + if (i != full_page) + put_page(pages[i]); + } + } + + /* At this point, err contains 0 or -EIO depending on the "critical" page */ + kfree(pages); + return err; +} + +const struct address_space_operations zisofs_aops = { + .read_folio = zisofs_read_folio, + /* No bmap operation supported */ +}; + +int __init zisofs_init(void) +{ + zisofs_zlib_workspace = vmalloc(zlib_inflate_workspacesize()); + if ( !zisofs_zlib_workspace ) + return -ENOMEM; + + return 0; +} + +void zisofs_cleanup(void) +{ + vfree(zisofs_zlib_workspace); +} diff --git a/fs/isofs/dir.c b/fs/isofs/dir.c new file mode 100644 index 000000000..c7ca7603e --- /dev/null +++ b/fs/isofs/dir.c @@ -0,0 +1,280 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/isofs/dir.c + * + * (C) 1992, 1993, 1994 Eric Youngdale Modified for ISO 9660 filesystem. + * + * (C) 1991 Linus Torvalds - minix filesystem + * + * Steve Beynon : Missing last directory entries fixed + * (stephen@askone.demon.co.uk) : 21st June 1996 + * + * isofs directory handling functions + */ +#include +#include +#include +#include "isofs.h" +#include + +int isofs_name_translate(struct iso_directory_record *de, char *new, struct inode *inode) +{ + char * old = de->name; + int len = de->name_len[0]; + int i; + + for (i = 0; i < len; i++) { + unsigned char c = old[i]; + if (!c) + break; + + if (c >= 'A' && c <= 'Z') + c |= 0x20; /* lower case */ + + /* Drop trailing '.;1' (ISO 9660:1988 7.5.1 requires period) */ + if (c == '.' && i == len - 3 && old[i + 1] == ';' && old[i + 2] == '1') + break; + + /* Drop trailing ';1' */ + if (c == ';' && i == len - 2 && old[i + 1] == '1') + break; + + /* Convert remaining ';' to '.' */ + /* Also '/' to '.' (broken Acorn-generated ISO9660 images) */ + if (c == ';' || c == '/') + c = '.'; + + new[i] = c; + } + return i; +} + +/* Acorn extensions written by Matthew Wilcox 1998 */ +int get_acorn_filename(struct iso_directory_record *de, + char *retname, struct inode *inode) +{ + int std; + unsigned char *chr; + int retnamlen = isofs_name_translate(de, retname, inode); + + if (retnamlen == 0) + return 0; + std = sizeof(struct iso_directory_record) + de->name_len[0]; + if (std & 1) + std++; + if (de->length[0] - std != 32) + return retnamlen; + chr = ((unsigned char *) de) + std; + if (strncmp(chr, "ARCHIMEDES", 10)) + return retnamlen; + if ((*retname == '_') && ((chr[19] & 1) == 1)) + *retname = '!'; + if (((de->flags[0] & 2) == 0) && (chr[13] == 0xff) + && ((chr[12] & 0xf0) == 0xf0)) { + retname[retnamlen] = ','; + sprintf(retname+retnamlen+1, "%3.3x", + ((chr[12] & 0xf) << 8) | chr[11]); + retnamlen += 4; + } + return retnamlen; +} + +/* + * This should _really_ be cleaned up some day.. + */ +static int do_isofs_readdir(struct inode *inode, struct file *file, + struct dir_context *ctx, + char *tmpname) +{ + unsigned long bufsize = ISOFS_BUFFER_SIZE(inode); + unsigned char bufbits = ISOFS_BUFFER_BITS(inode); + unsigned long block, offset, block_saved, offset_saved; + unsigned long inode_number = 0; /* Quiet GCC */ + struct buffer_head *bh = NULL; + int len; + int map; + int first_de = 1; + char *p = NULL; /* Quiet GCC */ + struct iso_directory_record *de; + struct isofs_sb_info *sbi = ISOFS_SB(inode->i_sb); + + offset = ctx->pos & (bufsize - 1); + block = ctx->pos >> bufbits; + + while (ctx->pos < inode->i_size) { + int de_len; + + if (!bh) { + bh = isofs_bread(inode, block); + if (!bh) + return 0; + } + + de = (struct iso_directory_record *) (bh->b_data + offset); + + de_len = *(unsigned char *)de; + + /* + * If the length byte is zero, we should move on to the next + * CDROM sector. If we are at the end of the directory, we + * kick out of the while loop. + */ + + if (de_len == 0) { + brelse(bh); + bh = NULL; + ctx->pos = (ctx->pos + ISOFS_BLOCK_SIZE) & ~(ISOFS_BLOCK_SIZE - 1); + block = ctx->pos >> bufbits; + offset = 0; + continue; + } + + block_saved = block; + offset_saved = offset; + offset += de_len; + + if (!isofs_dir_record_valid(de, offset_saved, bufsize)) { + printk(KERN_NOTICE "iso9660: Corrupted directory entry" + " in block %lu of inode %llu\n", block, + inode->i_ino); + brelse(bh); + return -EIO; + } + + if (first_de) { + isofs_normalize_block_and_offset(de, + &block_saved, + &offset_saved); + inode_number = isofs_get_ino(block_saved, + offset_saved, bufbits); + } + + if (de->flags[-sbi->s_high_sierra] & 0x80) { + first_de = 0; + ctx->pos += de_len; + continue; + } + first_de = 1; + + /* Handle the case of the '.' directory */ + if (de->name_len[0] == 1 && de->name[0] == 0) { + if (!dir_emit_dot(file, ctx)) + break; + ctx->pos += de_len; + continue; + } + + len = 0; + + /* Handle the case of the '..' directory */ + if (de->name_len[0] == 1 && de->name[0] == 1) { + if (!dir_emit_dotdot(file, ctx)) + break; + ctx->pos += de_len; + continue; + } + + /* Handle everything else. Do name translation if there + is no Rock Ridge NM field. */ + + /* + * Do not report hidden files if so instructed, or associated + * files unless instructed to do so + */ + if ((sbi->s_hide && (de->flags[-sbi->s_high_sierra] & 1)) || + (!sbi->s_showassoc && + (de->flags[-sbi->s_high_sierra] & 4))) { + ctx->pos += de_len; + continue; + } + + map = 1; + if (sbi->s_rock) { + len = get_rock_ridge_filename(de, tmpname, inode); + if (len != 0) { /* may be -1 */ + p = tmpname; + map = 0; + } + } + if (map) { +#ifdef CONFIG_JOLIET + if (sbi->s_joliet_level) { + len = get_joliet_filename(de, tmpname, inode); + p = tmpname; + } else +#endif + if (sbi->s_mapping == 'a') { + len = get_acorn_filename(de, tmpname, inode); + p = tmpname; + } else + if (sbi->s_mapping == 'n') { + len = isofs_name_translate(de, tmpname, inode); + p = tmpname; + } else { + p = de->name; + len = de->name_len[0]; + } + } + if (len > 0) { + if (!dir_emit(ctx, p, len, inode_number, DT_UNKNOWN)) + break; + } + ctx->pos += de_len; + } + if (bh) + brelse(bh); + return 0; +} + +/* + * Handle allocation of temporary space for name translation and + * handling split directory entries.. The real work is done by + * "do_isofs_readdir()". + */ +static int isofs_readdir(struct file *file, struct dir_context *ctx) +{ + int result; + char *tmpname; + struct inode *inode = file_inode(file); + + tmpname = kmalloc(1024, GFP_KERNEL); + if (tmpname == NULL) + return -ENOMEM; + + result = do_isofs_readdir(inode, file, ctx, tmpname); + + kfree(tmpname); + return result; +} + +int isofs_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + struct isofs_sb_info *sbi = ISOFS_SB(dentry->d_sb); + + if (sbi->s_check == 'r') { + fa->fsx_xflags |= FS_XFLAG_CASEFOLD; + fa->flags |= FS_CASEFOLD_FL; + } + if (!sbi->s_joliet_level && !sbi->s_rock && + (sbi->s_mapping == 'n' || sbi->s_mapping == 'a')) + fa->fsx_xflags |= FS_XFLAG_CASENONPRESERVING; + return 0; +} + +const struct file_operations isofs_dir_operations = +{ + .llseek = generic_file_llseek, + .read = generic_read_dir, + .iterate_shared = isofs_readdir, + .setlease = generic_setlease, +}; + +/* + * directories can handle most operations... + */ +const struct inode_operations isofs_dir_inode_operations = +{ + .lookup = isofs_lookup, + .fileattr_get = isofs_fileattr_get, +}; + diff --git a/fs/isofs/export.c b/fs/isofs/export.c new file mode 100644 index 000000000..4f7fa1d50 --- /dev/null +++ b/fs/isofs/export.c @@ -0,0 +1,201 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/isofs/export.c + * + * (C) 2004 Paul Serice - The new inode scheme requires switching + * from iget() to iget5_locked() which means + * the NFS export operations have to be hand + * coded because the default routines rely on + * iget(). + * + * The following files are helpful: + * + * Documentation/filesystems/nfs/exporting.rst + * fs/exportfs/expfs.c. + */ + +#include "isofs.h" + +static struct dentry * +isofs_export_iget(struct super_block *sb, + unsigned long block, + unsigned long offset, + __u32 generation) +{ + struct inode *inode; + + if (block == 0 || block >= ISOFS_SB(sb)->s_nzones) + return ERR_PTR(-ESTALE); + inode = isofs_iget(sb, block, offset); + if (IS_ERR(inode)) + return ERR_CAST(inode); + if (generation && inode->i_generation != generation) { + iput(inode); + return ERR_PTR(-ESTALE); + } + return d_obtain_alias(inode); +} + +/* This function is surprisingly simple. The trick is understanding + * that "child" is always a directory. So, to find its parent, you + * simply need to find its ".." entry, normalize its block and offset, + * and return the underlying inode. See the comments for + * isofs_normalize_block_and_offset(). */ +static struct dentry *isofs_export_get_parent(struct dentry *child) +{ + unsigned long parent_block = 0; + unsigned long parent_offset = 0; + struct inode *child_inode = d_inode(child); + struct iso_inode_info *e_child_inode = ISOFS_I(child_inode); + struct iso_directory_record *de = NULL; + struct buffer_head * bh = NULL; + struct dentry *rv = NULL; + + /* "child" must always be a directory. */ + if (!S_ISDIR(child_inode->i_mode)) { + printk(KERN_ERR "isofs: isofs_export_get_parent(): " + "child is not a directory!\n"); + rv = ERR_PTR(-EACCES); + goto out; + } + + /* It is an invariant that the directory offset is zero. If + * it is not zero, it means the directory failed to be + * normalized for some reason. */ + if (e_child_inode->i_iget5_offset != 0) { + printk(KERN_ERR "isofs: isofs_export_get_parent(): " + "child directory not normalized!\n"); + rv = ERR_PTR(-EACCES); + goto out; + } + + /* The child inode has been normalized such that its + * i_iget5_block value points to the "." entry. Fortunately, + * the ".." entry is located in the same block. */ + parent_block = e_child_inode->i_iget5_block; + + /* Get the block in question. */ + bh = sb_bread(child_inode->i_sb, parent_block); + if (bh == NULL) { + rv = ERR_PTR(-EACCES); + goto out; + } + + /* This is the "." entry. */ + de = (struct iso_directory_record*)bh->b_data; + if (!isofs_dir_record_valid(de, 0, child_inode->i_sb->s_blocksize) || + isonum_711(de->name_len) != 1 || de->name[0] != 0) { + printk(KERN_ERR "isofs: Unable to find the \".\" directory for NFS.\n"); + rv = ERR_PTR(-EACCES); + goto out; + } + + /* The ".." entry is always the second entry. */ + parent_offset = (unsigned long)isonum_711(de->length); + de = (struct iso_directory_record*)(bh->b_data + parent_offset); + + /* Verify it is in fact the ".." entry. */ + if (!isofs_dir_record_valid(de, parent_offset, + child_inode->i_sb->s_blocksize) || + isonum_711(de->name_len) != 1 || de->name[0] != 1) { + printk(KERN_ERR "isofs: Unable to find the \"..\" " + "directory for NFS.\n"); + rv = ERR_PTR(-EACCES); + goto out; + } + + /* Normalize */ + isofs_normalize_block_and_offset(de, &parent_block, &parent_offset); + + rv = d_obtain_alias(isofs_iget(child_inode->i_sb, parent_block, + parent_offset)); + out: + if (bh) + brelse(bh); + return rv; +} + +static int +isofs_export_encode_fh(struct inode *inode, + __u32 *fh32, + int *max_len, + struct inode *parent) +{ + struct iso_inode_info * ei = ISOFS_I(inode); + int len = *max_len; + int type = 1; + __u16 *fh16 = (__u16*)fh32; + + /* + * WARNING: max_len is 5 for NFSv2. Because of this + * limitation, we use the lower 16 bits of fh32[1] to hold the + * offset of the inode and the upper 16 bits of fh32[1] to + * hold the offset of the parent. + */ + if (parent && (len < 5)) { + *max_len = 5; + return FILEID_INVALID; + } else if (len < 3) { + *max_len = 3; + return FILEID_INVALID; + } + + len = 3; + fh32[0] = ei->i_iget5_block; + fh16[2] = (__u16)ei->i_iget5_offset; /* fh16 [sic] */ + fh16[3] = 0; /* avoid leaking uninitialized data */ + fh32[2] = inode->i_generation; + if (parent) { + struct iso_inode_info *eparent; + eparent = ISOFS_I(parent); + fh32[3] = eparent->i_iget5_block; + fh16[3] = (__u16)eparent->i_iget5_offset; /* fh16 [sic] */ + fh32[4] = parent->i_generation; + len = 5; + type = 2; + } + *max_len = len; + return type; +} + +struct isofs_fid { + u32 block; + u16 offset; + u16 parent_offset; + u32 generation; + u32 parent_block; + u32 parent_generation; +}; + +static struct dentry *isofs_fh_to_dentry(struct super_block *sb, + struct fid *fid, int fh_len, int fh_type) +{ + struct isofs_fid *ifid = (struct isofs_fid *)fid; + + if (fh_len < 3 || fh_type > 2) + return NULL; + + return isofs_export_iget(sb, ifid->block, ifid->offset, + ifid->generation); +} + +static struct dentry *isofs_fh_to_parent(struct super_block *sb, + struct fid *fid, int fh_len, int fh_type) +{ + struct isofs_fid *ifid = (struct isofs_fid *)fid; + + if (fh_len < 2 || fh_type != 2) + return NULL; + + return isofs_export_iget(sb, + fh_len > 3 ? ifid->parent_block : 0, + ifid->parent_offset, + fh_len > 4 ? ifid->parent_generation : 0); +} + +const struct export_operations isofs_export_ops = { + .encode_fh = isofs_export_encode_fh, + .fh_to_dentry = isofs_fh_to_dentry, + .fh_to_parent = isofs_fh_to_parent, + .get_parent = isofs_export_get_parent, +}; diff --git a/fs/isofs/inode.c b/fs/isofs/inode.c new file mode 100644 index 000000000..337836a0a --- /dev/null +++ b/fs/isofs/inode.c @@ -0,0 +1,1637 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/isofs/inode.c + * + * (C) 1991 Linus Torvalds - minix filesystem + * 1992, 1993, 1994 Eric Youngdale Modified for ISO 9660 filesystem. + * 1994 Eberhard Mönkeberg - multi session handling. + * 1995 Mark Dobie - allow mounting of some weird VideoCDs and PhotoCDs. + * 1997 Gordon Chaffee - Joliet CDs + * 1998 Eric Lammerts - ISO 9660 Level 3 + * 2004 Paul Serice - Inode Support pushed out from 4GB to 128GB + * 2004 Paul Serice - NFS Export Operations + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "isofs.h" +#include "zisofs.h" + +/* max tz offset is 13 hours */ +#define MAX_TZ_OFFSET (52*15*60) + +#define BEQUIET + +static int isofs_hashi(const struct dentry *parent, struct qstr *qstr); +static int isofs_dentry_cmpi(const struct dentry *dentry, + unsigned int len, const char *str, const struct qstr *name); + +#ifdef CONFIG_JOLIET +static int isofs_hashi_ms(const struct dentry *parent, struct qstr *qstr); +static int isofs_hash_ms(const struct dentry *parent, struct qstr *qstr); +static int isofs_dentry_cmpi_ms(const struct dentry *dentry, + unsigned int len, const char *str, const struct qstr *name); +static int isofs_dentry_cmp_ms(const struct dentry *dentry, + unsigned int len, const char *str, const struct qstr *name); +#endif + +static void isofs_put_super(struct super_block *sb) +{ + struct isofs_sb_info *sbi = ISOFS_SB(sb); + +#ifdef CONFIG_JOLIET + unload_nls(sbi->s_nls_iocharset); +#endif + + kfree(sbi); + sb->s_fs_info = NULL; + return; +} + +static int isofs_read_inode(struct inode *, int relocated); +static int isofs_statfs (struct dentry *, struct kstatfs *); +static int isofs_show_options(struct seq_file *, struct dentry *); + +static struct kmem_cache *isofs_inode_cachep; + +static struct inode *isofs_alloc_inode(struct super_block *sb) +{ + struct iso_inode_info *ei; + ei = alloc_inode_sb(sb, isofs_inode_cachep, GFP_KERNEL); + if (!ei) + return NULL; + return &ei->vfs_inode; +} + +static void isofs_free_inode(struct inode *inode) +{ + kmem_cache_free(isofs_inode_cachep, ISOFS_I(inode)); +} + +static void init_once(void *foo) +{ + struct iso_inode_info *ei = foo; + + inode_init_once(&ei->vfs_inode); +} + +static int __init init_inodecache(void) +{ + isofs_inode_cachep = kmem_cache_create("isofs_inode_cache", + sizeof(struct iso_inode_info), + 0, (SLAB_RECLAIM_ACCOUNT| + SLAB_ACCOUNT), + init_once); + if (!isofs_inode_cachep) + return -ENOMEM; + return 0; +} + +static void destroy_inodecache(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(isofs_inode_cachep); +} + +static int isofs_reconfigure(struct fs_context *fc) +{ + sync_filesystem(fc->root->d_sb); + if (!(fc->sb_flags & SB_RDONLY)) + return -EROFS; + return 0; +} + +static const struct super_operations isofs_sops = { + .alloc_inode = isofs_alloc_inode, + .free_inode = isofs_free_inode, + .put_super = isofs_put_super, + .statfs = isofs_statfs, + .show_options = isofs_show_options, +}; + + +static const struct dentry_operations isofs_dentry_ops[] = { + { + .d_hash = isofs_hashi, + .d_compare = isofs_dentry_cmpi, + }, +#ifdef CONFIG_JOLIET + { + .d_hash = isofs_hash_ms, + .d_compare = isofs_dentry_cmp_ms, + }, + { + .d_hash = isofs_hashi_ms, + .d_compare = isofs_dentry_cmpi_ms, + }, +#endif +}; + +struct isofs_options{ + unsigned int rock:1; + unsigned int joliet:1; + unsigned int cruft:1; + unsigned int hide:1; + unsigned int showassoc:1; + unsigned int nocompress:1; + unsigned int overriderockperm:1; + unsigned int uid_set:1; + unsigned int gid_set:1; + unsigned char map; + unsigned char check; + unsigned int blocksize; + umode_t fmode; + umode_t dmode; + kgid_t gid; + kuid_t uid; + char *iocharset; + /* LVE */ + s32 session; + s32 sbsector; +}; + +/* + * Compute the hash for the isofs name corresponding to the dentry. + */ +static int +isofs_hashi_common(const struct dentry *dentry, struct qstr *qstr, int ms) +{ + const char *name; + int len; + char c; + unsigned long hash; + + len = qstr->len; + name = qstr->name; + if (ms) { + while (len && name[len-1] == '.') + len--; + } + + hash = init_name_hash(dentry); + while (len--) { + c = tolower(*name++); + hash = partial_name_hash(c, hash); + } + qstr->hash = end_name_hash(hash); + + return 0; +} + +/* + * Compare of two isofs names. + */ +static int isofs_dentry_cmp_common( + unsigned int len, const char *str, + const struct qstr *name, int ms, int ci) +{ + int alen, blen; + + /* A filename cannot end in '.' or we treat it like it has none */ + alen = name->len; + blen = len; + if (ms) { + while (alen && name->name[alen-1] == '.') + alen--; + while (blen && str[blen-1] == '.') + blen--; + } + if (alen == blen) { + if (ci) { + if (strncasecmp(name->name, str, alen) == 0) + return 0; + } else { + if (strncmp(name->name, str, alen) == 0) + return 0; + } + } + return 1; +} + +static int +isofs_hashi(const struct dentry *dentry, struct qstr *qstr) +{ + return isofs_hashi_common(dentry, qstr, 0); +} + +static int +isofs_dentry_cmpi(const struct dentry *dentry, + unsigned int len, const char *str, const struct qstr *name) +{ + return isofs_dentry_cmp_common(len, str, name, 0, 1); +} + +#ifdef CONFIG_JOLIET +/* + * Compute the hash for the isofs name corresponding to the dentry. + */ +static int +isofs_hash_common(const struct dentry *dentry, struct qstr *qstr, int ms) +{ + const char *name; + int len; + + len = qstr->len; + name = qstr->name; + if (ms) { + while (len && name[len-1] == '.') + len--; + } + + qstr->hash = full_name_hash(dentry, name, len); + + return 0; +} + +static int +isofs_hash_ms(const struct dentry *dentry, struct qstr *qstr) +{ + return isofs_hash_common(dentry, qstr, 1); +} + +static int +isofs_hashi_ms(const struct dentry *dentry, struct qstr *qstr) +{ + return isofs_hashi_common(dentry, qstr, 1); +} + +static int +isofs_dentry_cmp_ms(const struct dentry *dentry, + unsigned int len, const char *str, const struct qstr *name) +{ + return isofs_dentry_cmp_common(len, str, name, 1, 0); +} + +static int +isofs_dentry_cmpi_ms(const struct dentry *dentry, + unsigned int len, const char *str, const struct qstr *name) +{ + return isofs_dentry_cmp_common(len, str, name, 1, 1); +} +#endif + +enum { + Opt_block, Opt_check, Opt_cruft, Opt_gid, Opt_ignore, Opt_iocharset, + Opt_map, Opt_mode, Opt_nojoliet, Opt_norock, Opt_sb, Opt_session, + Opt_uid, Opt_unhide, Opt_utf8, Opt_err, Opt_nocompress, Opt_hide, + Opt_showassoc, Opt_dmode, Opt_overriderockperm, +}; + +static const struct constant_table isofs_param_map[] = { + {"acorn", 'a'}, + {"a", 'a'}, + {"normal", 'n'}, + {"n", 'n'}, + {"off", 'o'}, + {"o", 'o'}, + {} +}; + +static const struct constant_table isofs_param_check[] = { + {"relaxed", 'r'}, + {"r", 'r'}, + {"strict", 's'}, + {"s", 's'}, + {} +}; + +static const struct fs_parameter_spec isofs_param_spec[] = { + fsparam_flag ("norock", Opt_norock), + fsparam_flag ("nojoliet", Opt_nojoliet), + fsparam_flag ("unhide", Opt_unhide), + fsparam_flag ("hide", Opt_hide), + fsparam_flag ("showassoc", Opt_showassoc), + fsparam_flag ("cruft", Opt_cruft), + fsparam_flag ("utf8", Opt_utf8), + fsparam_string ("iocharset", Opt_iocharset), + fsparam_enum ("map", Opt_map, isofs_param_map), + fsparam_u32 ("session", Opt_session), + fsparam_u32 ("sbsector", Opt_sb), + fsparam_enum ("check", Opt_check, isofs_param_check), + fsparam_uid ("uid", Opt_uid), + fsparam_gid ("gid", Opt_gid), + /* Note: mode/dmode historically accepted %u not strictly %o */ + fsparam_u32 ("mode", Opt_mode), + fsparam_u32 ("dmode", Opt_dmode), + fsparam_flag ("overriderockperm", Opt_overriderockperm), + fsparam_u32 ("block", Opt_block), + fsparam_string ("conv", Opt_ignore), + fsparam_flag ("nocompress", Opt_nocompress), + {} +}; + +static int isofs_parse_param(struct fs_context *fc, + struct fs_parameter *param) +{ + struct isofs_options *popt = fc->fs_private; + struct fs_parse_result result; + int opt; + unsigned int n; + + /* There are no remountable options */ + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE) + return 0; + + opt = fs_parse(fc, isofs_param_spec, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_norock: + popt->rock = 0; + break; + case Opt_nojoliet: + popt->joliet = 0; + break; + case Opt_hide: + popt->hide = 1; + break; + case Opt_unhide: + case Opt_showassoc: + popt->showassoc = 1; + break; + case Opt_cruft: + popt->cruft = 1; + break; +#ifdef CONFIG_JOLIET + case Opt_utf8: + kfree(popt->iocharset); + popt->iocharset = kstrdup("utf8", GFP_KERNEL); + if (!popt->iocharset) + return -ENOMEM; + break; + case Opt_iocharset: + kfree(popt->iocharset); + popt->iocharset = kstrdup(param->string, GFP_KERNEL); + if (!popt->iocharset) + return -ENOMEM; + break; +#endif + case Opt_map: + popt->map = result.uint_32; + break; + case Opt_session: + n = result.uint_32; + /* + * Track numbers are supposed to be in range 1-99, the + * mount option starts indexing at 0. + */ + if (n >= 99) + return -EINVAL; + popt->session = n + 1; + break; + case Opt_sb: + popt->sbsector = result.uint_32; + break; + case Opt_check: + popt->check = result.uint_32; + break; + case Opt_ignore: + break; + case Opt_uid: + popt->uid = result.uid; + popt->uid_set = 1; + break; + case Opt_gid: + popt->gid = result.gid; + popt->gid_set = 1; + break; + case Opt_mode: + popt->fmode = result.uint_32; + break; + case Opt_dmode: + popt->dmode = result.uint_32; + break; + case Opt_overriderockperm: + popt->overriderockperm = 1; + break; + case Opt_block: + n = result.uint_32; + if (n != 512 && n != 1024 && n != 2048) + return -EINVAL; + popt->blocksize = n; + break; + case Opt_nocompress: + popt->nocompress = 1; + break; + default: + return -EINVAL; + } + return 0; +} + +/* + * Display the mount options in /proc/mounts. + */ +static int isofs_show_options(struct seq_file *m, struct dentry *root) +{ + struct isofs_sb_info *sbi = ISOFS_SB(root->d_sb); + + if (!sbi->s_rock) seq_puts(m, ",norock"); + else if (!sbi->s_joliet_level) seq_puts(m, ",nojoliet"); + if (sbi->s_cruft) seq_puts(m, ",cruft"); + if (sbi->s_hide) seq_puts(m, ",hide"); + if (sbi->s_nocompress) seq_puts(m, ",nocompress"); + if (sbi->s_overriderockperm) seq_puts(m, ",overriderockperm"); + if (sbi->s_showassoc) seq_puts(m, ",showassoc"); + + if (sbi->s_check) seq_printf(m, ",check=%c", sbi->s_check); + if (sbi->s_mapping) seq_printf(m, ",map=%c", sbi->s_mapping); + if (sbi->s_session != 255) seq_printf(m, ",session=%u", sbi->s_session - 1); + if (sbi->s_sbsector != -1) seq_printf(m, ",sbsector=%u", sbi->s_sbsector); + + if (root->d_sb->s_blocksize != 1024) + seq_printf(m, ",blocksize=%lu", root->d_sb->s_blocksize); + + if (sbi->s_uid_set) + seq_printf(m, ",uid=%u", + from_kuid_munged(&init_user_ns, sbi->s_uid)); + if (sbi->s_gid_set) + seq_printf(m, ",gid=%u", + from_kgid_munged(&init_user_ns, sbi->s_gid)); + + if (sbi->s_dmode != ISOFS_INVALID_MODE) + seq_printf(m, ",dmode=%o", sbi->s_dmode); + if (sbi->s_fmode != ISOFS_INVALID_MODE) + seq_printf(m, ",fmode=%o", sbi->s_fmode); + +#ifdef CONFIG_JOLIET + if (sbi->s_nls_iocharset) + seq_printf(m, ",iocharset=%s", sbi->s_nls_iocharset->charset); + else + seq_puts(m, ",iocharset=utf8"); +#endif + return 0; +} + +/* + * look if the driver can tell the multi session redirection value + * + * don't change this if you don't know what you do, please! + * Multisession is legal only with XA disks. + * A non-XA disk with more than one volume descriptor may do it right, but + * usually is written in a nowhere standardized "multi-partition" manner. + * Multisession uses absolute addressing (solely the first frame of the whole + * track is #0), multi-partition uses relative addressing (each first frame of + * each track is #0), and a track is not a session. + * + * A broken CDwriter software or drive firmware does not set new standards, + * at least not if conflicting with the existing ones. + * + * emoenke@gwdg.de + */ +#define WE_OBEY_THE_WRITTEN_STANDARDS 1 + +static unsigned int isofs_get_last_session(struct super_block *sb, s32 session) +{ + struct cdrom_device_info *cdi = disk_to_cdi(sb->s_bdev->bd_disk); + unsigned int vol_desc_start = 0; + + if (session > 0) { + struct cdrom_tocentry te; + + if (!cdi) + return 0; + + te.cdte_track = session; + te.cdte_format = CDROM_LBA; + if (cdrom_read_tocentry(cdi, &te) == 0) { + printk(KERN_DEBUG "ISOFS: Session %d start %d type %d\n", + session, te.cdte_addr.lba, + te.cdte_ctrl & CDROM_DATA_TRACK); + if ((te.cdte_ctrl & CDROM_DATA_TRACK) == 4) + return te.cdte_addr.lba; + } + + printk(KERN_ERR "ISOFS: Invalid session number or type of track\n"); + } + + if (cdi) { + struct cdrom_multisession ms_info; + + ms_info.addr_format = CDROM_LBA; + if (cdrom_multisession(cdi, &ms_info) == 0) { +#if WE_OBEY_THE_WRITTEN_STANDARDS + /* necessary for a valid ms_info.addr */ + if (ms_info.xa_flag) +#endif + vol_desc_start = ms_info.addr.lba; + } + } + + return vol_desc_start; +} + +/* + * Check if root directory is empty (has less than 3 files). + * + * Used to detect broken CDs where ISO root directory is empty but Joliet root + * directory is OK. If such CD has Rock Ridge extensions, they will be disabled + * (and Joliet used instead) or else no files would be visible. + */ +static bool rootdir_empty(struct super_block *sb, unsigned long block) +{ + int offset = 0, files = 0, de_len; + struct iso_directory_record *de; + struct buffer_head *bh; + + bh = sb_bread(sb, block); + if (!bh) + return true; + while (files < 3) { + de = (struct iso_directory_record *) (bh->b_data + offset); + de_len = *(unsigned char *) de; + if (de_len == 0) + break; + files++; + offset += de_len; + } + brelse(bh); + return files < 3; +} + +/* + * Initialize the superblock and read the root inode. + */ +static int isofs_fill_super(struct super_block *s, struct fs_context *fc) +{ + struct buffer_head *bh = NULL, *pri_bh = NULL; + struct hs_primary_descriptor *h_pri = NULL; + struct iso_primary_descriptor *pri = NULL; + struct iso_supplementary_descriptor *sec = NULL; + struct iso_directory_record *rootp; + struct inode *inode; + struct isofs_options *opt = fc->fs_private; + struct isofs_sb_info *sbi; + unsigned long first_data_zone; + int joliet_level = 0; + int iso_blknum, block; + int orig_zonesize; + int table, error = -EINVAL; + unsigned int vol_desc_start; + int silent = fc->sb_flags & SB_SILENT; + + sbi = kzalloc_obj(*sbi); + if (!sbi) + return -ENOMEM; + s->s_fs_info = sbi; + + /* + * First of all, get the hardware blocksize for this device. + * If we don't know what it is, or the hardware blocksize is + * larger than the blocksize the user specified, then use + * that value. + */ + /* + * What if bugger tells us to go beyond page size? + */ + if (bdev_logical_block_size(s->s_bdev) > 2048) { + printk(KERN_WARNING + "ISOFS: unsupported/invalid hardware sector size %d\n", + bdev_logical_block_size(s->s_bdev)); + goto out_freesbi; + } + opt->blocksize = sb_min_blocksize(s, opt->blocksize); + if (!opt->blocksize) { + printk(KERN_ERR + "ISOFS: unable to set blocksize\n"); + goto out_freesbi; + } + + sbi->s_high_sierra = 0; /* default is iso9660 */ + sbi->s_session = opt->session; + sbi->s_sbsector = opt->sbsector; + + vol_desc_start = (opt->sbsector != -1) ? + opt->sbsector : isofs_get_last_session(s, opt->session); + + for (iso_blknum = vol_desc_start+16; + iso_blknum < vol_desc_start+100; iso_blknum++) { + struct hs_volume_descriptor *hdp; + struct iso_volume_descriptor *vdp; + + block = iso_blknum << (ISOFS_BLOCK_BITS - s->s_blocksize_bits); + if (!(bh = sb_bread(s, block))) + goto out_no_read; + + vdp = (struct iso_volume_descriptor *)bh->b_data; + hdp = (struct hs_volume_descriptor *)bh->b_data; + + /* + * Due to the overlapping physical location of the descriptors, + * ISO CDs can match hdp->id==HS_STANDARD_ID as well. To ensure + * proper identification in this case, we first check for ISO. + */ + if (strncmp (vdp->id, ISO_STANDARD_ID, sizeof vdp->id) == 0) { + if (isonum_711(vdp->type) == ISO_VD_END) + break; + if (isonum_711(vdp->type) == ISO_VD_PRIMARY) { + if (!pri) { + pri = (struct iso_primary_descriptor *)vdp; + /* Save the buffer in case we need it ... */ + pri_bh = bh; + bh = NULL; + } + } +#ifdef CONFIG_JOLIET + else if (isonum_711(vdp->type) == ISO_VD_SUPPLEMENTARY) { + sec = (struct iso_supplementary_descriptor *)vdp; + if (sec->escape[0] == 0x25 && sec->escape[1] == 0x2f) { + if (opt->joliet) { + if (sec->escape[2] == 0x40) + joliet_level = 1; + else if (sec->escape[2] == 0x43) + joliet_level = 2; + else if (sec->escape[2] == 0x45) + joliet_level = 3; + + printk(KERN_DEBUG "ISO 9660 Extensions: " + "Microsoft Joliet Level %d\n", + joliet_level); + } + goto root_found; + } else { + /* Unknown supplementary volume descriptor */ + sec = NULL; + } + } +#endif + } else { + if (strncmp (hdp->id, HS_STANDARD_ID, sizeof hdp->id) == 0) { + if (isonum_711(hdp->type) != ISO_VD_PRIMARY) + goto out_freebh; + + sbi->s_high_sierra = 1; + opt->rock = 0; + h_pri = (struct hs_primary_descriptor *)vdp; + goto root_found; + } + } + + /* Just skip any volume descriptors we don't recognize */ + + brelse(bh); + bh = NULL; + } + /* + * If we fall through, either no volume descriptor was found, + * or else we passed a primary descriptor looking for others. + */ + if (!pri) + goto out_unknown_format; + brelse(bh); + bh = pri_bh; + pri_bh = NULL; + +root_found: + /* We don't support read-write mounts */ + if (!sb_rdonly(s)) { + error = -EACCES; + goto out_freebh; + } + + if (joliet_level && (!pri || !opt->rock)) { + /* This is the case of Joliet with the norock mount flag. + * A disc with both Joliet and Rock Ridge is handled later + */ + pri = (struct iso_primary_descriptor *) sec; + } + + if(sbi->s_high_sierra){ + rootp = (struct iso_directory_record *) h_pri->root_directory_record; + sbi->s_nzones = isonum_733(h_pri->volume_space_size); + sbi->s_log_zone_size = isonum_723(h_pri->logical_block_size); + sbi->s_max_size = isonum_733(h_pri->volume_space_size); + } else { + if (!pri) + goto out_freebh; + rootp = (struct iso_directory_record *) pri->root_directory_record; + sbi->s_nzones = isonum_733(pri->volume_space_size); + sbi->s_log_zone_size = isonum_723(pri->logical_block_size); + sbi->s_max_size = isonum_733(pri->volume_space_size); + } + + sbi->s_ninodes = 0; /* No way to figure this out easily */ + + orig_zonesize = sbi->s_log_zone_size; + /* + * If the zone size is smaller than the hardware sector size, + * this is a fatal error. This would occur if the disc drive + * had sectors that were 2048 bytes, but the filesystem had + * blocks that were 512 bytes (which should only very rarely + * happen.) + */ + if (orig_zonesize < opt->blocksize) + goto out_bad_size; + + /* RDE: convert log zone size to bit shift */ + switch (sbi->s_log_zone_size) { + case 512: sbi->s_log_zone_size = 9; break; + case 1024: sbi->s_log_zone_size = 10; break; + case 2048: sbi->s_log_zone_size = 11; break; + + default: + goto out_bad_zone_size; + } + + s->s_magic = ISOFS_SUPER_MAGIC; + + /* + * With multi-extent files, file size is only limited by the maximum + * size of a file system, which is 8 TB. + */ + s->s_maxbytes = 0x80000000000LL; + + /* ECMA-119 timestamp from 1900/1/1 with tz offset */ + s->s_time_min = mktime64(1900, 1, 1, 0, 0, 0) - MAX_TZ_OFFSET; + s->s_time_max = mktime64(U8_MAX+1900, 12, 31, 23, 59, 59) + MAX_TZ_OFFSET; + + /* Set this for reference. Its not currently used except on write + which we don't have .. */ + + first_data_zone = isonum_733(rootp->extent) + + isonum_711(rootp->ext_attr_length); + sbi->s_firstdatazone = first_data_zone; +#ifndef BEQUIET + printk(KERN_DEBUG "ISOFS: Max size:%ld Log zone size:%ld\n", + sbi->s_max_size, 1UL << sbi->s_log_zone_size); + printk(KERN_DEBUG "ISOFS: First datazone:%ld\n", sbi->s_firstdatazone); + if(sbi->s_high_sierra) + printk(KERN_DEBUG "ISOFS: Disc in High Sierra format.\n"); +#endif + + /* + * If the Joliet level is set, we _may_ decide to use the + * secondary descriptor, but can't be sure until after we + * read the root inode. But before reading the root inode + * we may need to change the device blocksize, and would + * rather release the old buffer first. So, we cache the + * first_data_zone value from the secondary descriptor. + */ + if (joliet_level) { + pri = (struct iso_primary_descriptor *) sec; + rootp = (struct iso_directory_record *) + pri->root_directory_record; + first_data_zone = isonum_733(rootp->extent) + + isonum_711(rootp->ext_attr_length); + } + + /* + * We're all done using the volume descriptor, and may need + * to change the device blocksize, so release the buffer now. + */ + brelse(pri_bh); + brelse(bh); + + /* + * Force the blocksize to 512 for 512 byte sectors. The file + * read primitives really get it wrong in a bad way if we don't + * do this. + * + * Note - we should never be setting the blocksize to something + * less than the hardware sector size for the device. If we + * do, we would end up having to read larger buffers and split + * out portions to satisfy requests. + * + * Note2- the idea here is that we want to deal with the optimal + * zonesize in the filesystem. If we have it set to something less, + * then we have horrible problems with trying to piece together + * bits of adjacent blocks in order to properly read directory + * entries. By forcing the blocksize in this way, we ensure + * that we will never be required to do this. + */ + if (!sb_set_blocksize(s, orig_zonesize)) + goto out_freesbi; + + sbi->s_nls_iocharset = NULL; + +#ifdef CONFIG_JOLIET + if (joliet_level) { + char *p = opt->iocharset ? opt->iocharset : CONFIG_NLS_DEFAULT; + if (strcmp(p, "utf8") != 0) { + sbi->s_nls_iocharset = opt->iocharset ? + load_nls(opt->iocharset) : load_nls_default(); + if (!sbi->s_nls_iocharset) + goto out_freesbi; + } + } +#endif + s->s_op = &isofs_sops; + s->s_export_op = &isofs_export_ops; + sbi->s_mapping = opt->map; + sbi->s_rock = (opt->rock ? 2 : 0); + sbi->s_rock_offset = -1; /* initial offset, will guess until SP is found*/ + sbi->s_cruft = opt->cruft; + sbi->s_hide = opt->hide; + sbi->s_showassoc = opt->showassoc; + sbi->s_uid = opt->uid; + sbi->s_gid = opt->gid; + sbi->s_uid_set = opt->uid_set; + sbi->s_gid_set = opt->gid_set; + sbi->s_nocompress = opt->nocompress; + sbi->s_overriderockperm = opt->overriderockperm; + /* + * It would be incredibly stupid to allow people to mark every file + * on the disk as suid, so we merely allow them to set the default + * permissions. + */ + if (opt->fmode != ISOFS_INVALID_MODE) + sbi->s_fmode = opt->fmode & 0777; + else + sbi->s_fmode = ISOFS_INVALID_MODE; + if (opt->dmode != ISOFS_INVALID_MODE) + sbi->s_dmode = opt->dmode & 0777; + else + sbi->s_dmode = ISOFS_INVALID_MODE; + + /* + * Read the root inode, which _may_ result in changing + * the s_rock flag. Once we have the final s_rock value, + * we then decide whether to use the Joliet descriptor. + */ + inode = isofs_iget(s, sbi->s_firstdatazone, 0); + + /* + * Fix for broken CDs with a corrupt root inode but a correct Joliet + * root directory. + */ + if (IS_ERR(inode)) { + if (joliet_level && sbi->s_firstdatazone != first_data_zone) { + printk(KERN_NOTICE + "ISOFS: root inode is unusable. " + "Disabling Rock Ridge and switching to Joliet."); + sbi->s_rock = 0; + inode = NULL; + } else { + goto out_no_root; + } + } + + /* + * Fix for broken CDs with Rock Ridge and empty ISO root directory but + * correct Joliet root directory. + */ + if (sbi->s_rock == 1 && joliet_level && + rootdir_empty(s, sbi->s_firstdatazone)) { + printk(KERN_NOTICE + "ISOFS: primary root directory is empty. " + "Disabling Rock Ridge and switching to Joliet."); + sbi->s_rock = 0; + } + + /* + * If this disk has both Rock Ridge and Joliet on it, then we + * want to use Rock Ridge by default. This can be overridden + * by using the norock mount option. There is still one other + * possibility that is not taken into account: a Rock Ridge + * CD with Unicode names. Until someone sees such a beast, it + * will not be supported. + */ + if (sbi->s_rock == 1) { + joliet_level = 0; + } else if (joliet_level) { + sbi->s_rock = 0; + if (sbi->s_firstdatazone != first_data_zone) { + sbi->s_firstdatazone = first_data_zone; + printk(KERN_DEBUG + "ISOFS: changing to secondary root\n"); + iput(inode); + inode = isofs_iget(s, sbi->s_firstdatazone, 0); + if (IS_ERR(inode)) + goto out_no_root; + } + } + + if (opt->check == 'u') { + /* Only Joliet is case insensitive by default */ + if (joliet_level) + opt->check = 'r'; + else + opt->check = 's'; + } + sbi->s_joliet_level = joliet_level; + + /* Make sure the root inode is a directory */ + if (!S_ISDIR(inode->i_mode)) { + printk(KERN_WARNING + "isofs_fill_super: root inode is not a directory. " + "Corrupted media?\n"); + goto out_iput; + } + + table = 0; + if (joliet_level) + table += 2; + if (opt->check == 'r') + table++; + sbi->s_check = opt->check; + + if (table) + set_default_d_op(s, &isofs_dentry_ops[table - 1]); + + /* get the root dentry */ + s->s_root = d_make_root(inode); + if (!(s->s_root)) { + error = -ENOMEM; + goto out_no_inode; + } + + return 0; + + /* + * Display error messages and free resources. + */ +out_iput: + iput(inode); + goto out_no_inode; +out_no_root: + error = PTR_ERR(inode); + if (error != -ENOMEM) + printk(KERN_WARNING "%s: get root inode failed\n", __func__); +out_no_inode: +#ifdef CONFIG_JOLIET + unload_nls(sbi->s_nls_iocharset); +#endif + goto out_freesbi; +out_no_read: + printk(KERN_WARNING "%s: bread failed, dev=%s, iso_blknum=%d, block=%d\n", + __func__, s->s_id, iso_blknum, block); + goto out_freebh; +out_bad_zone_size: + printk(KERN_WARNING "ISOFS: Bad logical zone size %ld\n", + sbi->s_log_zone_size); + goto out_freebh; +out_bad_size: + printk(KERN_WARNING "ISOFS: Logical zone size(%d) < hardware blocksize(%u)\n", + orig_zonesize, opt->blocksize); + goto out_freebh; +out_unknown_format: + if (!silent) + printk(KERN_WARNING "ISOFS: Unable to identify CD-ROM format.\n"); + +out_freebh: + brelse(bh); + brelse(pri_bh); +out_freesbi: + kfree(sbi); + s->s_fs_info = NULL; + return error; +} + +static int isofs_statfs (struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + u64 id = huge_encode_dev(sb->s_bdev->bd_dev); + + buf->f_type = ISOFS_SUPER_MAGIC; + buf->f_bsize = sb->s_blocksize; + buf->f_blocks = (ISOFS_SB(sb)->s_nzones + << (ISOFS_SB(sb)->s_log_zone_size - sb->s_blocksize_bits)); + buf->f_bfree = 0; + buf->f_bavail = 0; + buf->f_files = ISOFS_SB(sb)->s_ninodes; + buf->f_ffree = 0; + buf->f_fsid = u64_to_fsid(id); + buf->f_namelen = NAME_MAX; + return 0; +} + +/* + * Get a set of blocks; filling in buffer_heads if already allocated + * or getblk() if they are not. Returns the number of blocks inserted + * (-ve == error.) + */ +int isofs_get_blocks(struct inode *inode, sector_t iblock, + struct buffer_head **bh, unsigned long nblocks) +{ + unsigned long b_off = iblock; + unsigned offset, sect_size; + unsigned int firstext; + unsigned long nextblk, nextoff; + int section, rv, error; + struct iso_inode_info *ei = ISOFS_I(inode); + + error = -EIO; + rv = 0; + if (iblock != b_off) { + printk(KERN_DEBUG "%s: block number too large\n", __func__); + goto abort; + } + + + offset = 0; + firstext = ei->i_first_extent; + sect_size = ei->i_section_size >> ISOFS_BUFFER_BITS(inode); + nextblk = ei->i_next_section_block; + nextoff = ei->i_next_section_offset; + section = 0; + + while (nblocks) { + /* If we are *way* beyond the end of the file, print a message. + * Access beyond the end of the file up to the next page boundary + * is normal, however because of the way the page cache works. + * In this case, we just return 0 so that we can properly fill + * the page with useless information without generating any + * I/O errors. + */ + if (b_off > ((inode->i_size + PAGE_SIZE - 1) >> ISOFS_BUFFER_BITS(inode))) { + printk(KERN_DEBUG "%s: block >= EOF (%lu, %llu)\n", + __func__, b_off, + (unsigned long long)inode->i_size); + goto abort; + } + + /* On the last section, nextblk == 0, section size is likely to + * exceed sect_size by a partial block, and access beyond the + * end of the file will reach beyond the section size, too. + */ + while (nextblk && (b_off >= (offset + sect_size))) { + struct inode *ninode; + + offset += sect_size; + ninode = isofs_iget(inode->i_sb, nextblk, nextoff); + if (IS_ERR(ninode)) { + error = PTR_ERR(ninode); + goto abort; + } + firstext = ISOFS_I(ninode)->i_first_extent; + sect_size = ISOFS_I(ninode)->i_section_size >> ISOFS_BUFFER_BITS(ninode); + nextblk = ISOFS_I(ninode)->i_next_section_block; + nextoff = ISOFS_I(ninode)->i_next_section_offset; + iput(ninode); + + if (++section > 100) { + printk(KERN_DEBUG "%s: More than 100 file sections ?!?" + " aborting...\n", __func__); + printk(KERN_DEBUG "%s: block=%lu firstext=%u sect_size=%u " + "nextblk=%lu nextoff=%lu\n", __func__, + b_off, firstext, (unsigned) sect_size, + nextblk, nextoff); + goto abort; + } + } + + if (*bh) { + map_bh(*bh, inode->i_sb, firstext + b_off - offset); + } else { + *bh = sb_getblk(inode->i_sb, firstext+b_off-offset); + if (!*bh) + goto abort; + } + bh++; /* Next buffer head */ + b_off++; /* Next buffer offset */ + nblocks--; + rv++; + } + + error = 0; +abort: + return rv != 0 ? rv : error; +} + +/* + * Used by the standard interfaces. + */ +static int isofs_get_block(struct inode *inode, sector_t iblock, + struct buffer_head *bh_result, int create) +{ + int ret; + + if (create) { + printk(KERN_DEBUG "%s: Kernel tries to allocate a block\n", __func__); + return -EROFS; + } + + ret = isofs_get_blocks(inode, iblock, &bh_result, 1); + return ret < 0 ? ret : 0; +} + +static int isofs_bmap(struct inode *inode, sector_t block) +{ + struct buffer_head dummy; + int error; + + dummy.b_state = 0; + dummy.b_blocknr = -1000; + error = isofs_get_block(inode, block, &dummy, 0); + if (!error) + return dummy.b_blocknr; + return 0; +} + +struct buffer_head *isofs_bread(struct inode *inode, sector_t block) +{ + sector_t blknr = isofs_bmap(inode, block); + if (!blknr) + return NULL; + return sb_bread(inode->i_sb, blknr); +} + +static int isofs_read_folio(struct file *file, struct folio *folio) +{ + return mpage_read_folio(folio, isofs_get_block); +} + +static void isofs_readahead(struct readahead_control *rac) +{ + mpage_readahead(rac, isofs_get_block); +} + +static sector_t _isofs_bmap(struct address_space *mapping, sector_t block) +{ + return generic_block_bmap(mapping,block,isofs_get_block); +} + +static const struct address_space_operations isofs_aops = { + .read_folio = isofs_read_folio, + .readahead = isofs_readahead, + .bmap = _isofs_bmap +}; + +static int isofs_read_level3_size(struct inode *inode) +{ + unsigned long bufsize = ISOFS_BUFFER_SIZE(inode); + int high_sierra = ISOFS_SB(inode->i_sb)->s_high_sierra; + struct buffer_head *bh = NULL; + unsigned long block, offset, block_saved, offset_saved; + int i = 0; + int more_entries = 0; + struct iso_directory_record *tmpde = NULL; + struct iso_inode_info *ei = ISOFS_I(inode); + + inode->i_size = 0; + + /* The first 16 blocks are reserved as the System Area. Thus, + * no inodes can appear in block 0. We use this to flag that + * this is the last section. */ + ei->i_next_section_block = 0; + ei->i_next_section_offset = 0; + + block = ei->i_iget5_block; + offset = ei->i_iget5_offset; + + do { + struct iso_directory_record *de; + unsigned int de_len; + + if (!bh) { + bh = sb_bread(inode->i_sb, block); + if (!bh) + goto out_noread; + } + de = (struct iso_directory_record *) (bh->b_data + offset); + de_len = *(unsigned char *) de; + + if (de_len == 0) { + brelse(bh); + bh = NULL; + ++block; + offset = 0; + continue; + } + + block_saved = block; + offset_saved = offset; + offset += de_len; + + /* Make sure we have a full directory entry */ + if (offset >= bufsize) { + int slop = bufsize - offset + de_len; + if (!tmpde) { + tmpde = kmalloc(256, GFP_KERNEL); + if (!tmpde) + goto out_nomem; + } + memcpy(tmpde, de, slop); + offset &= bufsize - 1; + block++; + brelse(bh); + bh = NULL; + if (offset) { + bh = sb_bread(inode->i_sb, block); + if (!bh) + goto out_noread; + memcpy((void *)tmpde+slop, bh->b_data, offset); + } + de = tmpde; + } + + inode->i_size += isonum_733(de->size); + if (i == 1) { + ei->i_next_section_block = block_saved; + ei->i_next_section_offset = offset_saved; + } + + more_entries = de->flags[-high_sierra] & 0x80; + + i++; + if (i > 100) + goto out_toomany; + } while (more_entries); +out: + kfree(tmpde); + brelse(bh); + return 0; + +out_nomem: + brelse(bh); + return -ENOMEM; + +out_noread: + printk(KERN_INFO "ISOFS: unable to read i-node block %lu\n", block); + kfree(tmpde); + return -EIO; + +out_toomany: + printk(KERN_INFO "%s: More than 100 file sections ?!?, aborting...\n" + "isofs_read_level3_size: inode=%llu\n", + __func__, inode->i_ino); + goto out; +} + +static int isofs_read_inode(struct inode *inode, int relocated) +{ + struct super_block *sb = inode->i_sb; + struct isofs_sb_info *sbi = ISOFS_SB(sb); + unsigned long bufsize = ISOFS_BUFFER_SIZE(inode); + unsigned long block; + int high_sierra = sbi->s_high_sierra; + struct buffer_head *bh; + struct iso_directory_record *de; + struct iso_directory_record *tmpde = NULL; + unsigned int de_len; + unsigned long offset; + struct iso_inode_info *ei = ISOFS_I(inode); + int ret = -EIO; + struct timespec64 ts; + + block = ei->i_iget5_block; + bh = sb_bread(inode->i_sb, block); + if (!bh) + goto out_badread; + + offset = ei->i_iget5_offset; + + de = (struct iso_directory_record *) (bh->b_data + offset); + de_len = *(unsigned char *) de; + if (de_len < sizeof(struct iso_directory_record)) + goto fail; + + if (offset + de_len > bufsize) { + int frag1 = bufsize - offset; + + tmpde = kmalloc(de_len, GFP_KERNEL); + if (!tmpde) { + ret = -ENOMEM; + goto fail; + } + memcpy(tmpde, bh->b_data + offset, frag1); + brelse(bh); + bh = sb_bread(inode->i_sb, ++block); + if (!bh) + goto out_badread; + memcpy((char *)tmpde+frag1, bh->b_data, de_len - frag1); + de = tmpde; + } + + inode->i_ino = isofs_get_ino(ei->i_iget5_block, + ei->i_iget5_offset, + ISOFS_BUFFER_BITS(inode)); + + /* Assume it is a normal-format file unless told otherwise */ + ei->i_file_format = isofs_file_normal; + + if (de->flags[-high_sierra] & 2) { + if (sbi->s_dmode != ISOFS_INVALID_MODE) + inode->i_mode = S_IFDIR | sbi->s_dmode; + else + inode->i_mode = S_IFDIR | S_IRUGO | S_IXUGO; + set_nlink(inode, 1); /* + * Set to 1. We know there are 2, but + * the find utility tries to optimize + * if it is 2, and it screws up. It is + * easier to give 1 which tells find to + * do it the hard way. + */ + } else { + if (sbi->s_fmode != ISOFS_INVALID_MODE) { + inode->i_mode = S_IFREG | sbi->s_fmode; + } else { + /* + * Set default permissions: r-x for all. The disc + * could be shared with DOS machines so virtually + * anything could be a valid executable. + */ + inode->i_mode = S_IFREG | S_IRUGO | S_IXUGO; + } + set_nlink(inode, 1); + } + inode->i_uid = sbi->s_uid; + inode->i_gid = sbi->s_gid; + inode->i_blocks = 0; + + ei->i_format_parm[0] = 0; + ei->i_format_parm[1] = 0; + ei->i_format_parm[2] = 0; + + ei->i_section_size = isonum_733(de->size); + if (de->flags[-high_sierra] & 0x80) { + ret = isofs_read_level3_size(inode); + if (ret < 0) + goto fail; + ret = -EIO; + } else { + ei->i_next_section_block = 0; + ei->i_next_section_offset = 0; + inode->i_size = isonum_733(de->size); + } + + /* + * Some dipshit decided to store some other bit of information + * in the high byte of the file length. Truncate size in case + * this CDROM was mounted with the cruft option. + */ + + if (sbi->s_cruft) + inode->i_size &= 0x00ffffff; + + if (de->interleave[0]) { + printk(KERN_DEBUG "ISOFS: Interleaved files not (yet) supported.\n"); + inode->i_size = 0; + } + + /* I have no idea what file_unit_size is used for, so + we will flag it for now */ + if (de->file_unit_size[0] != 0) { + printk(KERN_DEBUG "ISOFS: File unit size != 0 for ISO file (%llu).\n", + inode->i_ino); + } + + /* I have no idea what other flag bits are used for, so + we will flag it for now */ +#ifdef DEBUG + if((de->flags[-high_sierra] & ~2)!= 0){ + printk(KERN_DEBUG "ISOFS: Unusual flag settings for ISO file " + "(%llu %x).\n", + inode->i_ino, de->flags[-high_sierra]); + } +#endif + ts = iso_date(de->date, high_sierra ? ISO_DATE_HIGH_SIERRA : 0); + inode_set_ctime_to_ts(inode, ts); + inode_set_atime_to_ts(inode, ts); + inode_set_mtime_to_ts(inode, ts); + + ei->i_first_extent = (isonum_733(de->extent) + + isonum_711(de->ext_attr_length)); + + /* Set the number of blocks for stat() - should be done before RR */ + inode->i_blocks = (inode->i_size + 511) >> 9; + + /* + * Now test for possible Rock Ridge extensions which will override + * some of these numbers in the inode structure. + */ + + if (!high_sierra) { + parse_rock_ridge_inode(de, inode, relocated); + /* if we want uid/gid set, override the rock ridge setting */ + if (sbi->s_uid_set) + inode->i_uid = sbi->s_uid; + if (sbi->s_gid_set) + inode->i_gid = sbi->s_gid; + } + /* Now set final access rights if overriding rock ridge setting */ + if (S_ISDIR(inode->i_mode) && sbi->s_overriderockperm && + sbi->s_dmode != ISOFS_INVALID_MODE) + inode->i_mode = S_IFDIR | sbi->s_dmode; + if (S_ISREG(inode->i_mode) && sbi->s_overriderockperm && + sbi->s_fmode != ISOFS_INVALID_MODE) + inode->i_mode = S_IFREG | sbi->s_fmode; + + /* Install the inode operations vector */ + if (S_ISREG(inode->i_mode)) { + inode->i_fop = &generic_ro_fops; + switch (ei->i_file_format) { +#ifdef CONFIG_ZISOFS + case isofs_file_compressed: + inode->i_data.a_ops = &zisofs_aops; + break; +#endif + default: + inode->i_data.a_ops = &isofs_aops; + break; + } + } else if (S_ISDIR(inode->i_mode)) { + inode->i_op = &isofs_dir_inode_operations; + inode->i_fop = &isofs_dir_operations; + } else if (S_ISLNK(inode->i_mode)) { + inode->i_op = &page_symlink_inode_operations; + inode_nohighmem(inode); + inode->i_data.a_ops = &isofs_symlink_aops; + } else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) || + S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) { + /* XXX - parse_rock_ridge_inode() had already set i_rdev. */ + init_special_inode(inode, inode->i_mode, inode->i_rdev); + } else { + printk(KERN_DEBUG "ISOFS: Invalid file type 0%04o for inode %llu.\n", + inode->i_mode, inode->i_ino); + ret = -EIO; + goto fail; + } + + ret = 0; +out: + kfree(tmpde); + brelse(bh); + return ret; + +out_badread: + printk(KERN_WARNING "ISOFS: unable to read i-node block\n"); +fail: + goto out; +} + +struct isofs_iget5_callback_data { + unsigned long block; + unsigned long offset; +}; + +static int isofs_iget5_test(struct inode *ino, void *data) +{ + struct iso_inode_info *i = ISOFS_I(ino); + struct isofs_iget5_callback_data *d = + (struct isofs_iget5_callback_data*)data; + return (i->i_iget5_block == d->block) + && (i->i_iget5_offset == d->offset); +} + +static int isofs_iget5_set(struct inode *ino, void *data) +{ + struct iso_inode_info *i = ISOFS_I(ino); + struct isofs_iget5_callback_data *d = + (struct isofs_iget5_callback_data*)data; + i->i_iget5_block = d->block; + i->i_iget5_offset = d->offset; + return 0; +} + +/* Store, in the inode's containing structure, the block and block + * offset that point to the underlying meta-data for the inode. The + * code below is otherwise similar to the iget() code in + * include/linux/fs.h */ +struct inode *__isofs_iget(struct super_block *sb, + unsigned long block, + unsigned long offset, + int relocated) +{ + unsigned long hashval; + struct inode *inode; + struct isofs_iget5_callback_data data; + long ret; + + if (offset >= 1ul << sb->s_blocksize_bits) + return ERR_PTR(-EINVAL); + + data.block = block; + data.offset = offset; + + hashval = (block << sb->s_blocksize_bits) | offset; + + inode = iget5_locked(sb, hashval, &isofs_iget5_test, + &isofs_iget5_set, &data); + + if (!inode) + return ERR_PTR(-ENOMEM); + + if (inode_state_read_once(inode) & I_NEW) { + ret = isofs_read_inode(inode, relocated); + if (ret < 0) { + iget_failed(inode); + inode = ERR_PTR(ret); + } else { + unlock_new_inode(inode); + } + } + + return inode; +} + +static int isofs_get_tree(struct fs_context *fc) +{ + return get_tree_bdev(fc, isofs_fill_super); +} + +static void isofs_free_fc(struct fs_context *fc) +{ + struct isofs_options *opt = fc->fs_private; + + kfree(opt->iocharset); + kfree(opt); +} + +static const struct fs_context_operations isofs_context_ops = { + .parse_param = isofs_parse_param, + .get_tree = isofs_get_tree, + .reconfigure = isofs_reconfigure, + .free = isofs_free_fc, +}; + +static int isofs_init_fs_context(struct fs_context *fc) +{ + struct isofs_options *opt; + + opt = kzalloc_obj(*opt); + if (!opt) + return -ENOMEM; + + opt->map = 'n'; + opt->rock = 1; + opt->joliet = 1; + opt->cruft = 0; + opt->hide = 0; + opt->showassoc = 0; + opt->check = 'u'; /* unset */ + opt->nocompress = 0; + opt->blocksize = 1024; + opt->fmode = opt->dmode = ISOFS_INVALID_MODE; + opt->uid_set = 0; + opt->gid_set = 0; + opt->gid = GLOBAL_ROOT_GID; + opt->uid = GLOBAL_ROOT_UID; + opt->iocharset = NULL; + opt->overriderockperm = 0; + opt->session = -1; + opt->sbsector = -1; + + fc->fs_private = opt; + fc->ops = &isofs_context_ops; + + return 0; +} + +static struct file_system_type iso9660_fs_type = { + .owner = THIS_MODULE, + .name = "iso9660", + .kill_sb = kill_block_super, + .fs_flags = FS_REQUIRES_DEV, + .init_fs_context = isofs_init_fs_context, + .parameters = isofs_param_spec, +}; +MODULE_ALIAS_FS("iso9660"); +MODULE_ALIAS("iso9660"); + +static int __init init_iso9660_fs(void) +{ + int err = init_inodecache(); + if (err) + goto out; +#ifdef CONFIG_ZISOFS + err = zisofs_init(); + if (err) + goto out1; +#endif + err = register_filesystem(&iso9660_fs_type); + if (err) + goto out2; + return 0; +out2: +#ifdef CONFIG_ZISOFS + zisofs_cleanup(); +out1: +#endif + destroy_inodecache(); +out: + return err; +} + +static void __exit exit_iso9660_fs(void) +{ + unregister_filesystem(&iso9660_fs_type); +#ifdef CONFIG_ZISOFS + zisofs_cleanup(); +#endif + destroy_inodecache(); +} + +module_init(init_iso9660_fs) +module_exit(exit_iso9660_fs) +MODULE_DESCRIPTION("ISO 9660 CDROM file system support"); +MODULE_LICENSE("GPL"); diff --git a/fs/isofs/isofs.h b/fs/isofs/isofs.h new file mode 100644 index 000000000..dacb9cdae --- /dev/null +++ b/fs/isofs/isofs.h @@ -0,0 +1,209 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include +#include +#include +#include + +enum isofs_file_format { + isofs_file_normal = 0, + isofs_file_sparse = 1, + isofs_file_compressed = 2, +}; + +/* + * iso fs inode data in memory + */ +struct iso_inode_info { + unsigned long i_iget5_block; + unsigned long i_iget5_offset; + unsigned int i_first_extent; + unsigned char i_file_format; + unsigned char i_format_parm[3]; + unsigned long i_next_section_block; + unsigned long i_next_section_offset; + off_t i_section_size; + struct inode vfs_inode; +}; + +/* + * iso9660 super-block data in memory + */ +struct isofs_sb_info { + unsigned long s_ninodes; + unsigned long s_nzones; + unsigned long s_firstdatazone; + unsigned long s_log_zone_size; + unsigned long s_max_size; + + int s_rock_offset; /* offset of SUSP fields within SU area */ + s32 s_sbsector; + unsigned char s_joliet_level; + unsigned char s_mapping; + unsigned char s_check; + unsigned char s_session; + unsigned int s_high_sierra:1; + unsigned int s_rock:2; + unsigned int s_cruft:1; /* Broken disks with high byte of length + * containing junk */ + unsigned int s_nocompress:1; + unsigned int s_hide:1; + unsigned int s_showassoc:1; + unsigned int s_overriderockperm:1; + unsigned int s_uid_set:1; + unsigned int s_gid_set:1; + + umode_t s_fmode; + umode_t s_dmode; + kgid_t s_gid; + kuid_t s_uid; + struct nls_table *s_nls_iocharset; /* Native language support table */ +}; + +#define ISOFS_INVALID_MODE ((umode_t) -1) + +static inline struct isofs_sb_info *ISOFS_SB(struct super_block *sb) +{ + return sb->s_fs_info; +} + +static inline struct iso_inode_info *ISOFS_I(struct inode *inode) +{ + return container_of(inode, struct iso_inode_info, vfs_inode); +} + +static inline int isonum_711(u8 *p) +{ + return *p; +} +static inline int isonum_712(s8 *p) +{ + return *p; +} +static inline unsigned int isonum_721(u8 *p) +{ + return get_unaligned_le16(p); +} +static inline unsigned int isonum_722(u8 *p) +{ + return get_unaligned_be16(p); +} +static inline unsigned int isonum_723(u8 *p) +{ + /* Ignore bigendian datum due to broken mastering programs */ + return get_unaligned_le16(p); +} +static inline unsigned int isonum_731(u8 *p) +{ + return get_unaligned_le32(p); +} +static inline unsigned int isonum_732(u8 *p) +{ + return get_unaligned_be32(p); +} +static inline unsigned int isonum_733(u8 *p) +{ + /* Ignore bigendian datum due to broken mastering programs */ + return get_unaligned_le32(p); +} +#define ISO_DATE_HIGH_SIERRA (1 << 0) +#define ISO_DATE_LONG_FORM (1 << 1) +struct timespec64 iso_date(u8 *p, int flags); + +struct inode; /* To make gcc happy */ + +extern int parse_rock_ridge_inode(struct iso_directory_record *, struct inode *, int relocated); +extern int get_rock_ridge_filename(struct iso_directory_record *, char *, struct inode *); +extern int isofs_name_translate(struct iso_directory_record *, char *, struct inode *); +bool isofs_dir_record_valid(struct iso_directory_record *de, + unsigned long offset, + unsigned long bufsize); + +int get_joliet_filename(struct iso_directory_record *, unsigned char *, struct inode *); +int get_acorn_filename(struct iso_directory_record *, char *, struct inode *); + +extern struct dentry *isofs_lookup(struct inode *, struct dentry *, unsigned int flags); +extern struct buffer_head *isofs_bread(struct inode *, sector_t); +extern int isofs_get_blocks(struct inode *, sector_t, struct buffer_head **, unsigned long); + +struct inode *__isofs_iget(struct super_block *sb, + unsigned long block, + unsigned long offset, + int relocated); + +static inline struct inode *isofs_iget(struct super_block *sb, + unsigned long block, + unsigned long offset) +{ + return __isofs_iget(sb, block, offset, 0); +} + +static inline struct inode *isofs_iget_reloc(struct super_block *sb, + unsigned long block, + unsigned long offset) +{ + return __isofs_iget(sb, block, offset, 1); +} + +/* Because the inode number is no longer relevant to finding the + * underlying meta-data for an inode, we are free to choose a more + * convenient 32-bit number as the inode number. The inode numbering + * scheme was recommended by Sergey Vlasov and Eric Lammerts. */ +static inline unsigned long isofs_get_ino(unsigned long block, + unsigned long offset, + unsigned long bufbits) +{ + return (block << (bufbits - 5)) | (offset >> 5); +} + +/* Every directory can have many redundant directory entries scattered + * throughout the directory tree. First there is the directory entry + * with the name of the directory stored in the parent directory. + * Then, there is the "." directory entry stored in the directory + * itself. Finally, there are possibly many ".." directory entries + * stored in all the subdirectories. + * + * In order for the NFS get_parent() method to work and for the + * general consistency of the dcache, we need to make sure the + * "i_iget5_block" and "i_iget5_offset" all point to exactly one of + * the many redundant entries for each directory. We normalize the + * block and offset by always making them point to the "." directory. + * + * Notice that we do not use the entry for the directory with the name + * that is located in the parent directory. Even though choosing this + * first directory is more natural, it is much easier to find the "." + * entry in the NFS get_parent() method because it is implicitly + * encoded in the "extent + ext_attr_length" fields of _all_ the + * redundant entries for the directory. Thus, it can always be + * reached regardless of which directory entry you have in hand. + * + * This works because the "." entry is simply the first directory + * record when you start reading the file that holds all the directory + * records, and this file starts at "extent + ext_attr_length" blocks. + * Because the "." entry is always the first entry listed in the + * directories file, the normalized "offset" value is always 0. + * + * You should pass the directory entry in "de". On return, "block" + * and "offset" will hold normalized values. Only directories are + * affected making it safe to call even for non-directory file + * types. */ +static inline void +isofs_normalize_block_and_offset(struct iso_directory_record* de, + unsigned long *block, + unsigned long *offset) +{ + /* Only directories are normalized. */ + if (de->flags[0] & 2) { + *offset = 0; + *block = (unsigned long)isonum_733(de->extent) + + (unsigned long)isonum_711(de->ext_attr_length); + } +} + +struct file_kattr; +int isofs_fileattr_get(struct dentry *dentry, struct file_kattr *fa); + +extern const struct inode_operations isofs_dir_inode_operations; +extern const struct file_operations isofs_dir_operations; +extern const struct address_space_operations isofs_symlink_aops; +extern const struct export_operations isofs_export_ops; diff --git a/fs/isofs/joliet.c b/fs/isofs/joliet.c new file mode 100644 index 000000000..c0f04a1e7 --- /dev/null +++ b/fs/isofs/joliet.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/isofs/joliet.c + * + * (C) 1996 Gordon Chaffee + * + * Joliet: Microsoft's Unicode extensions to iso9660 + */ + +#include +#include +#include "isofs.h" + +/* + * Convert Unicode 16 to UTF-8 or ASCII. + */ +static int +uni16_to_x8(unsigned char *ascii, __be16 *uni, int len, struct nls_table *nls) +{ + __be16 *ip, ch; + unsigned char *op; + + ip = uni; + op = ascii; + + while ((ch = get_unaligned(ip)) && len) { + int llen; + llen = nls->uni2char(be16_to_cpu(ch), op, NLS_MAX_CHARSET_SIZE); + if (llen > 0) + op += llen; + else + *op++ = '?'; + ip++; + + len--; + } + *op = 0; + return (op - ascii); +} + +int +get_joliet_filename(struct iso_directory_record * de, unsigned char *outname, struct inode * inode) +{ + struct nls_table *nls; + unsigned char len = 0; + + nls = ISOFS_SB(inode->i_sb)->s_nls_iocharset; + + if (!nls) { + len = utf16s_to_utf8s((const wchar_t *) de->name, + de->name_len[0] >> 1, UTF16_BIG_ENDIAN, + outname, PAGE_SIZE); + } else { + len = uni16_to_x8(outname, (__be16 *) de->name, + de->name_len[0] >> 1, nls); + } + if ((len > 2) && (outname[len-2] == ';') && (outname[len-1] == '1')) + len -= 2; + + /* + * Windows doesn't like periods at the end of a name, + * so neither do we + */ + while (len >= 2 && (outname[len-1] == '.')) + len--; + + return len; +} diff --git a/fs/isofs/namei.c b/fs/isofs/namei.c new file mode 100644 index 000000000..010682f59 --- /dev/null +++ b/fs/isofs/namei.c @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/isofs/namei.c + * + * (C) 1992 Eric Youngdale Modified for ISO 9660 filesystem. + * + * (C) 1991 Linus Torvalds - minix filesystem + */ + +#include +#include "isofs.h" + +bool isofs_dir_record_valid(struct iso_directory_record *de, + unsigned long offset, + unsigned long bufsize) +{ + unsigned int len; + unsigned int name_len; + unsigned long min_len = offsetof(struct iso_directory_record, name); + + if (offset > bufsize || bufsize - offset < min_len) + return false; + + len = isonum_711(de->length); + name_len = isonum_711(de->name_len); + if (len < min_len || name_len > len - min_len) + return false; + if (len > bufsize - offset) + return false; + return true; +} + +static int +isofs_cmp(struct dentry *dentry, const char *compare, int dlen) +{ + if (likely(!dentry->d_op)) + return dentry->d_name.len != dlen || memcmp(dentry->d_name.name, compare, dlen); + return dentry->d_op->d_compare(NULL, dentry->d_name.len, dentry->d_name.name, + &QSTR_LEN(compare, dlen)); +} + +/* + * isofs_find_entry() + * + * finds an entry in the specified directory with the wanted name. It + * returns the inode number of the found entry, or 0 on error. + */ +static unsigned long +isofs_find_entry(struct inode *dir, struct dentry *dentry, + unsigned long *block_rv, unsigned long *offset_rv, + char *tmpname) +{ + unsigned long bufsize = ISOFS_BUFFER_SIZE(dir); + unsigned char bufbits = ISOFS_BUFFER_BITS(dir); + unsigned long block, f_pos, offset, block_saved, offset_saved; + struct buffer_head *bh = NULL; + struct isofs_sb_info *sbi = ISOFS_SB(dir->i_sb); + + if (!ISOFS_I(dir)->i_first_extent) + return 0; + + f_pos = 0; + offset = 0; + block = 0; + + while (f_pos < dir->i_size) { + struct iso_directory_record *de; + int de_len, match, i, dlen; + char *dpnt; + + if (!bh) { + bh = isofs_bread(dir, block); + if (!bh) + return 0; + } + + de = (struct iso_directory_record *) (bh->b_data + offset); + + de_len = *(unsigned char *) de; + if (!de_len) { + brelse(bh); + bh = NULL; + f_pos = (f_pos + ISOFS_BLOCK_SIZE) & ~(ISOFS_BLOCK_SIZE - 1); + block = f_pos >> bufbits; + offset = 0; + continue; + } + + block_saved = bh->b_blocknr; + offset_saved = offset; + offset += de_len; + f_pos += de_len; + + if (!isofs_dir_record_valid(de, offset_saved, bufsize)) { + printk(KERN_NOTICE "iso9660: Corrupted directory entry" + " in block %lu of inode %llu\n", block, + dir->i_ino); + brelse(bh); + return 0; + } + dlen = de->name_len[0]; + dpnt = de->name; + + if (sbi->s_rock && + ((i = get_rock_ridge_filename(de, tmpname, dir)))) { + dlen = i; /* possibly -1 */ + dpnt = tmpname; +#ifdef CONFIG_JOLIET + } else if (sbi->s_joliet_level) { + dlen = get_joliet_filename(de, tmpname, dir); + dpnt = tmpname; +#endif + } else if (sbi->s_mapping == 'a') { + dlen = get_acorn_filename(de, tmpname, dir); + dpnt = tmpname; + } else if (sbi->s_mapping == 'n') { + dlen = isofs_name_translate(de, tmpname, dir); + dpnt = tmpname; + } + + /* + * Skip hidden or associated files unless hide or showassoc, + * respectively, is set + */ + match = 0; + if (dlen > 0 && + (!sbi->s_hide || + (!(de->flags[-sbi->s_high_sierra] & 1))) && + (sbi->s_showassoc || + (!(de->flags[-sbi->s_high_sierra] & 4)))) { + if (dpnt && (dlen > 1 || dpnt[0] > 1)) + match = (isofs_cmp(dentry, dpnt, dlen) == 0); + } + if (match) { + isofs_normalize_block_and_offset(de, + &block_saved, + &offset_saved); + *block_rv = block_saved; + *offset_rv = offset_saved; + brelse(bh); + return 1; + } + } + brelse(bh); + return 0; +} + +struct dentry *isofs_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) +{ + int found; + unsigned long block; + unsigned long offset; + struct inode *inode; + char *tmpname; + + tmpname = kmalloc(1024, GFP_USER); + if (!tmpname) + return ERR_PTR(-ENOMEM); + + found = isofs_find_entry(dir, dentry, &block, &offset, tmpname); + kfree(tmpname); + + inode = found ? isofs_iget(dir->i_sb, block, offset) : NULL; + + return d_splice_alias(inode, dentry); +} diff --git a/fs/isofs/rock.c b/fs/isofs/rock.c new file mode 100644 index 000000000..2628f31bd --- /dev/null +++ b/fs/isofs/rock.c @@ -0,0 +1,830 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/isofs/rock.c + * + * (C) 1992, 1993 Eric Youngdale + * + * Rock Ridge Extensions to iso9660 + */ + +#include +#include + +#include "isofs.h" +#include "rock.h" + +/* + * These functions are designed to read the system areas of a directory record + * and extract relevant information. There are different functions provided + * depending upon what information we need at the time. One function fills + * out an inode structure, a second one extracts a filename, a third one + * returns a symbolic link name, and a fourth one returns the extent number + * for the file. + */ + +#define SIG(A,B) ((A) | ((B) << 8)) /* isonum_721() */ + +struct rock_state { + void *buffer; + unsigned char *chr; + int len; + int cont_size; + int cont_extent; + int cont_offset; + int cont_loops; + struct inode *inode; +}; + +/* + * This is a way of ensuring that we have something in the system + * use fields that is compatible with Rock Ridge. Return zero on success. + */ + +static int check_sp(struct rock_ridge *rr, struct inode *inode) +{ + if (rr->u.SP.magic[0] != 0xbe) + return -1; + if (rr->u.SP.magic[1] != 0xef) + return -1; + ISOFS_SB(inode->i_sb)->s_rock_offset = rr->u.SP.skip; + return 0; +} + +static void setup_rock_ridge(struct iso_directory_record *de, + struct inode *inode, struct rock_state *rs) +{ + rs->len = sizeof(struct iso_directory_record) + de->name_len[0]; + if (rs->len & 1) + (rs->len)++; + rs->chr = (unsigned char *)de + rs->len; + rs->len = *((unsigned char *)de) - rs->len; + if (rs->len < 0) + rs->len = 0; + + if (ISOFS_SB(inode->i_sb)->s_rock_offset != -1) { + rs->len -= ISOFS_SB(inode->i_sb)->s_rock_offset; + rs->chr += ISOFS_SB(inode->i_sb)->s_rock_offset; + if (rs->len < 0) + rs->len = 0; + } +} + +static void init_rock_state(struct rock_state *rs, struct inode *inode) +{ + memset(rs, 0, sizeof(*rs)); + rs->inode = inode; +} + +/* Maximum number of Rock Ridge continuation entries */ +#define RR_MAX_CE_ENTRIES 32 + +/* + * Returns 0 if the caller should continue scanning, 1 if the scan must end + * and -ve on error. + */ +static int rock_continue(struct rock_state *rs) +{ + int ret = 1; + int blocksize = 1 << rs->inode->i_blkbits; + const int min_de_size = offsetof(struct rock_ridge, u); + + kfree(rs->buffer); + rs->buffer = NULL; + + if ((unsigned)rs->cont_offset > blocksize - min_de_size || + (unsigned)rs->cont_size > blocksize || + (unsigned)(rs->cont_offset + rs->cont_size) > blocksize) { + printk(KERN_NOTICE "rock: corrupted directory entry. " + "extent=%d, offset=%d, size=%d\n", + rs->cont_extent, rs->cont_offset, rs->cont_size); + ret = -EIO; + goto out; + } + + if ((unsigned)rs->cont_extent >= ISOFS_SB(rs->inode->i_sb)->s_nzones) { + printk(KERN_NOTICE "rock: corrupted directory entry. " + "extent=%u out of volume (nzones=%lu)\n", + (unsigned)rs->cont_extent, + ISOFS_SB(rs->inode->i_sb)->s_nzones); + ret = -EIO; + goto out; + } + + if (rs->cont_extent) { + struct buffer_head *bh; + + rs->buffer = kmalloc(rs->cont_size, GFP_KERNEL); + if (!rs->buffer) { + ret = -ENOMEM; + goto out; + } + ret = -EIO; + if (++rs->cont_loops >= RR_MAX_CE_ENTRIES) + goto out; + bh = sb_bread(rs->inode->i_sb, rs->cont_extent); + if (bh) { + memcpy(rs->buffer, bh->b_data + rs->cont_offset, + rs->cont_size); + put_bh(bh); + rs->chr = rs->buffer; + rs->len = rs->cont_size; + rs->cont_extent = 0; + rs->cont_size = 0; + rs->cont_offset = 0; + return 0; + } + printk("Unable to read rock-ridge attributes\n"); + } +out: + kfree(rs->buffer); + rs->buffer = NULL; + return ret; +} + +/* + * We think there's a record of type `sig' at rs->chr. Parse the signature + * and make sure that there's really room for a record of that type. + */ +static int rock_check_overflow(struct rock_state *rs, int sig) +{ + int len; + + switch (sig) { + case SIG('S', 'P'): + len = sizeof(struct SU_SP_s); + break; + case SIG('C', 'E'): + len = sizeof(struct SU_CE_s); + break; + case SIG('E', 'R'): + len = sizeof(struct SU_ER_s); + break; + case SIG('R', 'R'): + len = sizeof(struct RR_RR_s); + break; + case SIG('P', 'X'): + len = sizeof(struct RR_PX_s); + break; + case SIG('P', 'N'): + len = sizeof(struct RR_PN_s); + break; + case SIG('S', 'L'): + len = sizeof(struct RR_SL_s); + break; + case SIG('N', 'M'): + len = sizeof(struct RR_NM_s); + break; + case SIG('C', 'L'): + len = sizeof(struct RR_CL_s); + break; + case SIG('P', 'L'): + len = sizeof(struct RR_PL_s); + break; + case SIG('T', 'F'): + len = sizeof(struct RR_TF_s); + break; + case SIG('Z', 'F'): + len = sizeof(struct RR_ZF_s); + break; + default: + len = 0; + break; + } + len += offsetof(struct rock_ridge, u); + if (len > rs->len) { + printk(KERN_NOTICE "rock: directory entry would overflow " + "storage\n"); + printk(KERN_NOTICE "rock: sig=0x%02x, size=%d, remaining=%d\n", + sig, len, rs->len); + return -EIO; + } + return 0; +} + +/* + * return length of name field; 0: not found, -1: to be ignored + */ +int get_rock_ridge_filename(struct iso_directory_record *de, + char *retname, struct inode *inode) +{ + struct rock_state rs; + struct rock_ridge *rr; + int sig; + int retnamlen = 0; + int truncate = 0; + int ret = 0; + char *p; + int len; + + if (!ISOFS_SB(inode->i_sb)->s_rock) + return 0; + *retname = 0; + + init_rock_state(&rs, inode); + setup_rock_ridge(de, inode, &rs); +repeat: + + while (rs.len > 2) { /* There may be one byte for padding somewhere */ + rr = (struct rock_ridge *)rs.chr; + /* + * Ignore rock ridge info if rr->len is out of range, but + * don't return -EIO because that would make the file + * invisible. + */ + if (rr->len < 3) + goto out; /* Something got screwed up here */ + sig = isonum_721(rs.chr); + if (rock_check_overflow(&rs, sig)) + goto eio; + rs.chr += rr->len; + rs.len -= rr->len; + /* + * As above, just ignore the rock ridge info if rr->len + * is bogus. + */ + if (rs.len < 0) + goto out; /* Something got screwed up here */ + + switch (sig) { + case SIG('R', 'R'): + if ((rr->u.RR.flags[0] & RR_NM) == 0) + goto out; + break; + case SIG('S', 'P'): + if (check_sp(rr, inode)) + goto out; + break; + case SIG('C', 'E'): + rs.cont_extent = isonum_733(rr->u.CE.extent); + rs.cont_offset = isonum_733(rr->u.CE.offset); + rs.cont_size = isonum_733(rr->u.CE.size); + break; + case SIG('N', 'M'): + if (truncate) + break; + if (rr->len < 5) + break; + /* + * If the flags are 2 or 4, this indicates '.' or '..'. + * We don't want to do anything with this, because it + * screws up the code that calls us. We don't really + * care anyways, since we can just use the non-RR + * name. + */ + if (rr->u.NM.flags & 6) + break; + + if (rr->u.NM.flags & ~1) { + printk("Unsupported NM flag settings (%d)\n", + rr->u.NM.flags); + break; + } + len = rr->len - 5; + if (retnamlen + len > NAME_MAX) { + truncate = 1; + break; + } + p = memchr(rr->u.NM.name, '\0', len); + if (unlikely(p)) + len = p - rr->u.NM.name; + memcpy(retname + retnamlen, rr->u.NM.name, len); + retnamlen += len; + retname[retnamlen] = '\0'; + break; + case SIG('R', 'E'): + kfree(rs.buffer); + return -1; + default: + break; + } + } + ret = rock_continue(&rs); + if (ret == 0) + goto repeat; + if (ret == 1) + return retnamlen; /* If 0, this file did not have a NM field */ +out: + kfree(rs.buffer); + return ret; +eio: + ret = -EIO; + goto out; +} + +#define RR_REGARD_XA 1 +#define RR_RELOC_DE 2 + +static int +parse_rock_ridge_inode_internal(struct iso_directory_record *de, + struct inode *inode, int flags) +{ + int symlink_len = 0; + int cnt, sig; + unsigned int reloc_block; + struct inode *reloc; + struct rock_ridge *rr; + int rootflag; + struct rock_state rs; + int ret = 0; + + if (!ISOFS_SB(inode->i_sb)->s_rock) + return 0; + + init_rock_state(&rs, inode); + setup_rock_ridge(de, inode, &rs); + if (flags & RR_REGARD_XA) { + rs.chr += 14; + rs.len -= 14; + if (rs.len < 0) + rs.len = 0; + } + +repeat: + while (rs.len > 2) { /* There may be one byte for padding somewhere */ + rr = (struct rock_ridge *)rs.chr; + /* + * Ignore rock ridge info if rr->len is out of range, but + * don't return -EIO because that would make the file + * invisible. + */ + if (rr->len < 3) + goto out; /* Something got screwed up here */ + sig = isonum_721(rs.chr); + if (rock_check_overflow(&rs, sig)) + goto eio; + rs.chr += rr->len; + rs.len -= rr->len; + /* + * As above, just ignore the rock ridge info if rr->len + * is bogus. + */ + if (rs.len < 0) + goto out; /* Something got screwed up here */ + + switch (sig) { +#ifndef CONFIG_ZISOFS /* No flag for SF or ZF */ + case SIG('R', 'R'): + if ((rr->u.RR.flags[0] & + (RR_PX | RR_TF | RR_SL | RR_CL)) == 0) + goto out; + break; +#endif + case SIG('S', 'P'): + if (check_sp(rr, inode)) + goto out; + break; + case SIG('C', 'E'): + rs.cont_extent = isonum_733(rr->u.CE.extent); + rs.cont_offset = isonum_733(rr->u.CE.offset); + rs.cont_size = isonum_733(rr->u.CE.size); + break; + case SIG('E', 'R'): + /* Invalid length of ER tag id? */ + if (rr->u.ER.len_id + offsetof(struct rock_ridge, u.ER.data) > rr->len) + goto out; + ISOFS_SB(inode->i_sb)->s_rock = 1; + printk(KERN_DEBUG "ISO 9660 Extensions: "); + { + int p; + for (p = 0; p < rr->u.ER.len_id; p++) + printk(KERN_CONT "%c", rr->u.ER.data[p]); + } + printk(KERN_CONT "\n"); + break; + case SIG('P', 'X'): + inode->i_mode = isonum_733(rr->u.PX.mode); + set_nlink(inode, isonum_733(rr->u.PX.n_links)); + i_uid_write(inode, isonum_733(rr->u.PX.uid)); + i_gid_write(inode, isonum_733(rr->u.PX.gid)); + break; + case SIG('P', 'N'): + { + int high, low; + high = isonum_733(rr->u.PN.dev_high); + low = isonum_733(rr->u.PN.dev_low); + /* + * The Rock Ridge standard specifies that if + * sizeof(dev_t) <= 4, then the high field is + * unused, and the device number is completely + * stored in the low field. Some writers may + * ignore this subtlety, + * and as a result we test to see if the entire + * device number is + * stored in the low field, and use that. + */ + if ((low & ~0xff) && high == 0) { + inode->i_rdev = + MKDEV(low >> 8, low & 0xff); + } else { + inode->i_rdev = + MKDEV(high, low); + } + } + break; + case SIG('T', 'F'): { + int flags, size, slen; + + flags = rr->u.TF.flags & TF_LONG_FORM ? ISO_DATE_LONG_FORM : 0; + size = rr->u.TF.flags & TF_LONG_FORM ? 17 : 7; + slen = rr->len - 5; + /* + * Some RRIP writers incorrectly place ctime in the + * TF_CREATE field. Try to handle this correctly for + * either case. + */ + /* Rock ridge never appears on a High Sierra disk */ + cnt = 0; + if ((rr->u.TF.flags & TF_CREATE) && size <= slen) { + inode_set_ctime_to_ts(inode, + iso_date(rr->u.TF.data + size * cnt++, flags)); + slen -= size; + } + if ((rr->u.TF.flags & TF_MODIFY) && size <= slen) { + inode_set_mtime_to_ts(inode, + iso_date(rr->u.TF.data + size * cnt++, flags)); + slen -= size; + } + if ((rr->u.TF.flags & TF_ACCESS) && size <= slen) { + inode_set_atime_to_ts(inode, + iso_date(rr->u.TF.data + size * cnt++, flags)); + slen -= size; + } + if ((rr->u.TF.flags & TF_ATTRIBUTES) && size <= slen) { + inode_set_ctime_to_ts(inode, + iso_date(rr->u.TF.data + size * cnt++, flags)); + slen -= size; + } + break; + } + case SIG('S', 'L'): + { + int slen; + struct SL_component *slp; + struct SL_component *oldslp; + slen = rr->len - 5; + slp = &rr->u.SL.link; + inode->i_size = symlink_len; + while (slen > 1) { + rootflag = 0; + /* keep the component within the SL record */ + if (slp->len + 2 > slen) + goto eio; + switch (slp->flags & ~1) { + case 0: + inode->i_size += + slp->len; + break; + case 2: + inode->i_size += 1; + break; + case 4: + inode->i_size += 2; + break; + case 8: + rootflag = 1; + inode->i_size += 1; + break; + default: + printk("Symlink component flag " + "not implemented\n"); + } + slen -= slp->len + 2; + oldslp = slp; + slp = (struct SL_component *) + (((char *)slp) + slp->len + 2); + + if (slen < 2) { + if (((rr->u.SL. + flags & 1) != 0) + && + ((oldslp-> + flags & 1) == 0)) + inode->i_size += + 1; + break; + } + + /* + * If this component record isn't + * continued, then append a '/'. + */ + if (!rootflag + && (oldslp->flags & 1) == 0) + inode->i_size += 1; + } + } + symlink_len = inode->i_size; + break; + case SIG('R', 'E'): + printk(KERN_WARNING "Attempt to read inode for " + "relocated directory\n"); + goto out; + case SIG('C', 'L'): + if (flags & RR_RELOC_DE) { + printk(KERN_ERR + "ISOFS: Recursive directory relocation " + "is not supported\n"); + goto eio; + } + reloc_block = isonum_733(rr->u.CL.location); + if (reloc_block == ISOFS_I(inode)->i_iget5_block && + ISOFS_I(inode)->i_iget5_offset == 0) { + printk(KERN_ERR + "ISOFS: Directory relocation points to " + "itself\n"); + goto eio; + } + ISOFS_I(inode)->i_first_extent = reloc_block; + reloc = isofs_iget_reloc(inode->i_sb, reloc_block, 0); + if (IS_ERR(reloc)) { + ret = PTR_ERR(reloc); + goto out; + } + inode->i_mode = reloc->i_mode; + set_nlink(inode, reloc->i_nlink); + inode->i_uid = reloc->i_uid; + inode->i_gid = reloc->i_gid; + inode->i_rdev = reloc->i_rdev; + inode->i_size = reloc->i_size; + inode->i_blocks = reloc->i_blocks; + inode_set_atime_to_ts(inode, inode_get_atime(reloc)); + inode_set_ctime_to_ts(inode, inode_get_ctime(reloc)); + inode_set_mtime_to_ts(inode, inode_get_mtime(reloc)); + iput(reloc); + break; +#ifdef CONFIG_ZISOFS + case SIG('Z', 'F'): { + int algo; + + if (ISOFS_SB(inode->i_sb)->s_nocompress) + break; + algo = isonum_721(rr->u.ZF.algorithm); + if (algo == SIG('p', 'z')) { + int block_shift = + isonum_711(&rr->u.ZF.parms[1]); + if (block_shift > 17) { + printk(KERN_WARNING "isofs: " + "Can't handle ZF block " + "size of 2^%d\n", + block_shift); + } else { + /* + * Note: we don't change + * i_blocks here + */ + ISOFS_I(inode)->i_file_format = + isofs_file_compressed; + /* + * Parameters to compression + * algorithm (header size, + * block size) + */ + ISOFS_I(inode)->i_format_parm[0] = + isonum_711(&rr->u.ZF.parms[0]); + ISOFS_I(inode)->i_format_parm[1] = + isonum_711(&rr->u.ZF.parms[1]); + inode->i_size = + isonum_733(rr->u.ZF. + real_size); + } + } else { + printk(KERN_WARNING + "isofs: Unknown ZF compression " + "algorithm: %c%c\n", + rr->u.ZF.algorithm[0], + rr->u.ZF.algorithm[1]); + } + break; + } +#endif + default: + break; + } + } + ret = rock_continue(&rs); + if (ret == 0) + goto repeat; + if (ret == 1) + ret = 0; +out: + kfree(rs.buffer); + return ret; +eio: + ret = -EIO; + goto out; +} + +static char *get_symlink_chunk(char *rpnt, struct rock_ridge *rr, char *plimit) +{ + int slen; + int rootflag; + struct SL_component *oldslp; + struct SL_component *slp; + slen = rr->len - 5; + slp = &rr->u.SL.link; + while (slen > 1) { + rootflag = 0; + /* + * A component is slp->len + 2 bytes (a two-byte header plus + * len bytes of text). If it does not fit in the bytes left in + * the SL record the record is malformed: fail like the plimit + * checks below so readlink() returns -EIO, not a truncated path. + */ + if (slp->len + 2 > slen) + return NULL; + switch (slp->flags & ~1) { + case 0: + if (slp->len > plimit - rpnt) + return NULL; + memcpy(rpnt, slp->text, slp->len); + rpnt += slp->len; + break; + case 2: + if (rpnt >= plimit) + return NULL; + *rpnt++ = '.'; + break; + case 4: + if (2 > plimit - rpnt) + return NULL; + *rpnt++ = '.'; + *rpnt++ = '.'; + break; + case 8: + if (rpnt >= plimit) + return NULL; + rootflag = 1; + *rpnt++ = '/'; + break; + default: + printk("Symlink component flag not implemented (%d)\n", + slp->flags); + } + slen -= slp->len + 2; + oldslp = slp; + slp = (struct SL_component *)((char *)slp + slp->len + 2); + + if (slen < 2) { + /* + * If there is another SL record, and this component + * record isn't continued, then add a slash. + */ + if ((!rootflag) && (rr->u.SL.flags & 1) && + !(oldslp->flags & 1)) { + if (rpnt >= plimit) + return NULL; + *rpnt++ = '/'; + } + break; + } + + /* + * If this component record isn't continued, then append a '/'. + */ + if (!rootflag && !(oldslp->flags & 1)) { + if (rpnt >= plimit) + return NULL; + *rpnt++ = '/'; + } + } + return rpnt; +} + +int parse_rock_ridge_inode(struct iso_directory_record *de, struct inode *inode, + int relocated) +{ + int flags = relocated ? RR_RELOC_DE : 0; + int result = parse_rock_ridge_inode_internal(de, inode, flags); + + /* + * if rockridge flag was reset and we didn't look for attributes + * behind eventual XA attributes, have a look there + */ + if ((ISOFS_SB(inode->i_sb)->s_rock_offset == -1) + && (ISOFS_SB(inode->i_sb)->s_rock == 2)) { + result = parse_rock_ridge_inode_internal(de, inode, + flags | RR_REGARD_XA); + } + return result; +} + +/* + * read_folio() for symlinks: reads symlink contents into the folio and either + * makes it uptodate and returns 0 or returns error (-EIO) + */ +static int rock_ridge_symlink_read_folio(struct file *file, struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + struct iso_inode_info *ei = ISOFS_I(inode); + struct isofs_sb_info *sbi = ISOFS_SB(inode->i_sb); + char *link = folio_address(folio); + unsigned long bufsize = ISOFS_BUFFER_SIZE(inode); + struct buffer_head *bh; + char *rpnt = link; + unsigned char *pnt; + struct iso_directory_record *raw_de; + unsigned long block, offset; + int sig; + struct rock_ridge *rr; + struct rock_state rs; + int ret; + + if (!sbi->s_rock) + goto error; + + init_rock_state(&rs, inode); + block = ei->i_iget5_block; + bh = sb_bread(inode->i_sb, block); + if (!bh) + goto out_noread; + + offset = ei->i_iget5_offset; + pnt = (unsigned char *)bh->b_data + offset; + + raw_de = (struct iso_directory_record *)pnt; + + /* + * If we go past the end of the buffer, there is some sort of error. + */ + if (offset + *pnt > bufsize) + goto out_bad_span; + + /* + * Now test for possible Rock Ridge extensions which will override + * some of these numbers in the inode structure. + */ + + setup_rock_ridge(raw_de, inode, &rs); + +repeat: + while (rs.len > 2) { /* There may be one byte for padding somewhere */ + rr = (struct rock_ridge *)rs.chr; + if (rr->len < 3) + goto out; /* Something got screwed up here */ + sig = isonum_721(rs.chr); + if (rock_check_overflow(&rs, sig)) + goto out; + rs.chr += rr->len; + rs.len -= rr->len; + if (rs.len < 0) + goto out; /* corrupted isofs */ + + switch (sig) { + case SIG('R', 'R'): + if ((rr->u.RR.flags[0] & RR_SL) == 0) + goto out; + break; + case SIG('S', 'P'): + if (check_sp(rr, inode)) + goto out; + break; + case SIG('S', 'L'): + rpnt = get_symlink_chunk(rpnt, rr, + link + (PAGE_SIZE - 1)); + if (rpnt == NULL) + goto out; + break; + case SIG('C', 'E'): + /* This tells is if there is a continuation record */ + rs.cont_extent = isonum_733(rr->u.CE.extent); + rs.cont_offset = isonum_733(rr->u.CE.offset); + rs.cont_size = isonum_733(rr->u.CE.size); + break; + default: + break; + } + } + ret = rock_continue(&rs); + if (ret == 0) + goto repeat; + if (ret < 0) + goto fail; + + if (rpnt == link) + goto fail; + brelse(bh); + *rpnt = '\0'; + ret = 0; +end: + folio_end_read(folio, ret == 0); + return ret; + + /* error exit from macro */ +out: + kfree(rs.buffer); + goto fail; +out_noread: + printk("unable to read i-node block"); + goto fail; +out_bad_span: + printk("symlink spans iso9660 blocks\n"); +fail: + brelse(bh); +error: + ret = -EIO; + goto end; +} + +const struct address_space_operations isofs_symlink_aops = { + .read_folio = rock_ridge_symlink_read_folio +}; diff --git a/fs/isofs/rock.h b/fs/isofs/rock.h new file mode 100644 index 000000000..c0856fa9b --- /dev/null +++ b/fs/isofs/rock.h @@ -0,0 +1,119 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * These structs are used by the system-use-sharing protocol, in which the + * Rock Ridge extensions are embedded. It is quite possible that other + * extensions are present on the disk, and this is fine as long as they + * all use SUSP + */ + +struct SU_SP_s { + __u8 magic[2]; + __u8 skip; +} __attribute__ ((packed)); + +struct SU_CE_s { + __u8 extent[8]; + __u8 offset[8]; + __u8 size[8]; +}; + +struct SU_ER_s { + __u8 len_id; + __u8 len_des; + __u8 len_src; + __u8 ext_ver; + __u8 data[]; +} __attribute__ ((packed)); + +struct RR_RR_s { + __u8 flags[1]; +} __attribute__ ((packed)); + +struct RR_PX_s { + __u8 mode[8]; + __u8 n_links[8]; + __u8 uid[8]; + __u8 gid[8]; +}; + +struct RR_PN_s { + __u8 dev_high[8]; + __u8 dev_low[8]; +}; + +struct SL_component { + __u8 flags; + __u8 len; + __u8 text[] __counted_by(len); +} __attribute__ ((packed)); + +struct RR_SL_s { + __u8 flags; + struct SL_component link; +} __attribute__ ((packed)); + +struct RR_NM_s { + __u8 flags; + char name[]; +} __attribute__ ((packed)); + +struct RR_CL_s { + __u8 location[8]; +}; + +struct RR_PL_s { + __u8 location[8]; +}; + +struct RR_TF_s { + __u8 flags; + __u8 data[]; +} __attribute__ ((packed)); + +/* Linux-specific extension for transparent decompression */ +struct RR_ZF_s { + __u8 algorithm[2]; + __u8 parms[2]; + __u8 real_size[8]; +}; + +/* + * These are the bits and their meanings for flags in the TF structure. + */ +#define TF_CREATE 1 +#define TF_MODIFY 2 +#define TF_ACCESS 4 +#define TF_ATTRIBUTES 8 +#define TF_BACKUP 16 +#define TF_EXPIRATION 32 +#define TF_EFFECTIVE 64 +#define TF_LONG_FORM 128 + +struct rock_ridge { + __u8 signature[2]; + __u8 len; + __u8 version; + union { + struct SU_SP_s SP; + struct SU_CE_s CE; + struct SU_ER_s ER; + struct RR_RR_s RR; + struct RR_PX_s PX; + struct RR_PN_s PN; + struct RR_SL_s SL; + struct RR_NM_s NM; + struct RR_CL_s CL; + struct RR_PL_s PL; + struct RR_TF_s TF; + struct RR_ZF_s ZF; + } u; +}; + +#define RR_PX 1 /* POSIX attributes */ +#define RR_PN 2 /* POSIX devices */ +#define RR_SL 4 /* Symbolic link */ +#define RR_NM 8 /* Alternate Name */ +#define RR_CL 16 /* Child link */ +#define RR_PL 32 /* Parent link */ +#define RR_RE 64 /* Relocation directory */ +#define RR_TF 128 /* Timestamps */ diff --git a/fs/isofs/util.c b/fs/isofs/util.c new file mode 100644 index 000000000..42f479da0 --- /dev/null +++ b/fs/isofs/util.c @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/isofs/util.c + */ + +#include +#include "isofs.h" + +/* + * We have to convert from a MM/DD/YY format to the Unix ctime format. + * We have to take into account leap years and all of that good stuff. + * Unfortunately, the kernel does not have the information on hand to + * take into account daylight savings time, but it shouldn't matter. + * The time stored should be localtime (with or without DST in effect), + * and the timezone offset should hold the offset required to get back + * to GMT. Thus we should always be correct. + */ + +struct timespec64 iso_date(u8 *p, int flags) +{ + int year, month, day, hour, minute, second, tz; + struct timespec64 ts; + + if (flags & ISO_DATE_LONG_FORM) { + year = (p[0] - '0') * 1000 + + (p[1] - '0') * 100 + + (p[2] - '0') * 10 + + (p[3] - '0') - 1900; + month = ((p[4] - '0') * 10 + (p[5] - '0')); + day = ((p[6] - '0') * 10 + (p[7] - '0')); + hour = ((p[8] - '0') * 10 + (p[9] - '0')); + minute = ((p[10] - '0') * 10 + (p[11] - '0')); + second = ((p[12] - '0') * 10 + (p[13] - '0')); + ts.tv_nsec = ((p[14] - '0') * 10 + (p[15] - '0')) * 10000000; + tz = p[16]; + } else { + year = p[0]; + month = p[1]; + day = p[2]; + hour = p[3]; + minute = p[4]; + second = p[5]; + ts.tv_nsec = 0; + /* High sierra has no time zone */ + tz = flags & ISO_DATE_HIGH_SIERRA ? 0 : p[6]; + } + + if (year < 0) { + ts.tv_sec = 0; + } else { + ts.tv_sec = mktime64(year+1900, month, day, hour, minute, second); + + /* sign extend */ + if (tz & 0x80) + tz |= (-1 << 8); + + /* + * The timezone offset is unreliable on some disks, + * so we make a sanity check. In no case is it ever + * more than 13 hours from GMT, which is 52*15min. + * The time is always stored in localtime with the + * timezone offset being what get added to GMT to + * get to localtime. Thus we need to subtract the offset + * to get to true GMT, which is what we store the time + * as internally. On the local system, the user may set + * their timezone any way they wish, of course, so GMT + * gets converted back to localtime on the receiving + * system. + * + * NOTE: mkisofs in versions prior to mkisofs-1.10 had + * the sign wrong on the timezone offset. This has now + * been corrected there too, but if you are getting screwy + * results this may be the explanation. If enough people + * complain, a user configuration option could be added + * to add the timezone offset in with the wrong sign + * for 'compatibility' with older discs, but I cannot see how + * it will matter that much. + * + * Thanks to kuhlmav@elec.canterbury.ac.nz (Volker Kuhlmann) + * for pointing out the sign error. + */ + if (-52 <= tz && tz <= 52) + ts.tv_sec -= tz * 15 * 60; + } + return ts; +} diff --git a/fs/isofs/zisofs.h b/fs/isofs/zisofs.h new file mode 100644 index 000000000..70f1a218f --- /dev/null +++ b/fs/isofs/zisofs.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* ----------------------------------------------------------------------- * + * + * Copyright 2001 H. Peter Anvin - All Rights Reserved + * + * ----------------------------------------------------------------------- */ + +/* + * Prototypes for functions exported from the compressed isofs subsystem + */ + +#ifdef CONFIG_ZISOFS +extern const struct address_space_operations zisofs_aops; +extern int __init zisofs_init(void); +extern void zisofs_cleanup(void); +#endif diff --git a/fs/jbd2/Kconfig b/fs/jbd2/Kconfig new file mode 100644 index 000000000..9c19e1512 --- /dev/null +++ b/fs/jbd2/Kconfig @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: GPL-2.0-only +config JBD2 + tristate + select CRC32 + help + This is a generic journaling layer for block devices that support + both 32-bit and 64-bit block numbers. It is currently used by + the ext4 and OCFS2 filesystems, but it could also be used to add + journal support to other file systems or block devices such + as RAID or LVM. + + If you are using ext4 or OCFS2, you need to say Y here. + If you are not using ext4 or OCFS2 then you will + probably want to say N. + + To compile this device as a module, choose M here. The module will be + called jbd2. If you are compiling ext4 or OCFS2 into the kernel, + you cannot compile this code as a module. + +config JBD2_DEBUG + bool "JBD2 (ext4) debugging support" + depends on JBD2 + help + If you are using the ext4 journaled file system (or + potentially any other filesystem/device using JBD2), this option + allows you to enable debugging output while the system is running, + in order to help track down any problems you are having. + By default, the debugging output will be turned off. + + If you select Y here, then you will be able to turn on debugging + with "echo N > /sys/module/jbd2/parameters/jbd2_debug", where N is a + number between 1 and 5. The higher the number, the more debugging + output is generated. To turn debugging off again, do + "echo 0 > /sys/module/jbd2/parameters/jbd2_debug". diff --git a/fs/jbd2/Makefile b/fs/jbd2/Makefile new file mode 100644 index 000000000..126b4da6c --- /dev/null +++ b/fs/jbd2/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the linux journaling routines. +# + +obj-$(CONFIG_JBD2) += jbd2.o + +jbd2-objs := transaction.o commit.o recovery.o checkpoint.o revoke.o journal.o diff --git a/fs/jbd2/checkpoint.c b/fs/jbd2/checkpoint.c new file mode 100644 index 000000000..513273712 --- /dev/null +++ b/fs/jbd2/checkpoint.c @@ -0,0 +1,731 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * linux/fs/jbd2/checkpoint.c + * + * Written by Stephen C. Tweedie , 1999 + * + * Copyright 1999 Red Hat Software --- All Rights Reserved + * + * Checkpoint routines for the generic filesystem journaling code. + * Part of the ext2fs journaling system. + * + * Checkpointing is the process of ensuring that a section of the log is + * committed fully to disk, so that that portion of the log can be + * reused. + */ + +#include +#include +#include +#include +#include +#include +#include + +/* + * Unlink a buffer from a transaction checkpoint list. + * + * Called with j_list_lock held. + */ +static inline void __buffer_unlink(struct journal_head *jh) +{ + transaction_t *transaction = jh->b_cp_transaction; + + jh->b_cpnext->b_cpprev = jh->b_cpprev; + jh->b_cpprev->b_cpnext = jh->b_cpnext; + if (transaction->t_checkpoint_list == jh) { + transaction->t_checkpoint_list = jh->b_cpnext; + if (transaction->t_checkpoint_list == jh) + transaction->t_checkpoint_list = NULL; + } +} + +/* + * __jbd2_log_wait_for_space: wait until there is space in the journal. + * + * Called under j-state_lock *only*. It will be unlocked if we have to wait + * for a checkpoint to free up some space in the log. + */ +void __jbd2_log_wait_for_space(journal_t *journal) +__acquires(&journal->j_state_lock) +__releases(&journal->j_state_lock) +{ + int nblocks, space_left; + /* assert_spin_locked(&journal->j_state_lock); */ + + nblocks = journal->j_max_transaction_buffers; + while (jbd2_log_space_left(journal) < nblocks) { + write_unlock(&journal->j_state_lock); + mutex_lock_io(&journal->j_checkpoint_mutex); + + /* + * Test again, another process may have checkpointed while we + * were waiting for the checkpoint lock. If there are no + * transactions ready to be checkpointed, try to recover + * journal space by calling cleanup_journal_tail(), and if + * that doesn't work, by waiting for the currently committing + * transaction to complete. If there is absolutely no way + * to make progress, this is either a BUG or corrupted + * filesystem, so abort the journal and leave a stack + * trace for forensic evidence. + */ + write_lock(&journal->j_state_lock); + if (journal->j_flags & JBD2_ABORT) { + mutex_unlock(&journal->j_checkpoint_mutex); + return; + } + spin_lock(&journal->j_list_lock); + space_left = jbd2_log_space_left(journal); + if (space_left < nblocks) { + int chkpt = journal->j_checkpoint_transactions != NULL; + tid_t tid = 0; + bool has_transaction = false; + + if (journal->j_committing_transaction) { + tid = journal->j_committing_transaction->t_tid; + has_transaction = true; + } + spin_unlock(&journal->j_list_lock); + write_unlock(&journal->j_state_lock); + if (chkpt) { + jbd2_log_do_checkpoint(journal); + } else if (jbd2_cleanup_journal_tail(journal) <= 0) { + /* + * We were able to recover space or the + * journal was aborted due to an error. + */ + ; + } else if (has_transaction) { + /* + * jbd2_journal_commit_transaction() may want + * to take the checkpoint_mutex if JBD2_FLUSHED + * is set. So we need to temporarily drop it. + */ + mutex_unlock(&journal->j_checkpoint_mutex); + jbd2_log_wait_commit(journal, tid); + write_lock(&journal->j_state_lock); + continue; + } else { + printk(KERN_ERR "%s: needed %d blocks and " + "only had %d space available\n", + __func__, nblocks, space_left); + printk(KERN_ERR "%s: no way to get more " + "journal space in %s\n", __func__, + journal->j_devname); + WARN_ON(1); + jbd2_journal_abort(journal, -ENOSPC); + } + write_lock(&journal->j_state_lock); + } else { + spin_unlock(&journal->j_list_lock); + } + mutex_unlock(&journal->j_checkpoint_mutex); + } +} + +static void +__flush_batch(journal_t *journal, int *batch_count) +{ + int i; + struct blk_plug plug; + + blk_start_plug(&plug); + for (i = 0; i < *batch_count; i++) + write_dirty_buffer(journal->j_chkpt_bhs[i], JBD2_JOURNAL_REQ_FLAGS); + blk_finish_plug(&plug); + + for (i = 0; i < *batch_count; i++) { + struct buffer_head *bh = journal->j_chkpt_bhs[i]; + BUFFER_TRACE(bh, "brelse"); + __brelse(bh); + journal->j_chkpt_bhs[i] = NULL; + } + *batch_count = 0; +} + +/* + * Perform an actual checkpoint. We take the first transaction on the + * list of transactions to be checkpointed and send all its buffers + * to disk. We submit larger chunks of data at once. + * + * The journal should be locked before calling this function. + * Called with j_checkpoint_mutex held. + */ +int jbd2_log_do_checkpoint(journal_t *journal) +{ + struct journal_head *jh; + struct buffer_head *bh; + transaction_t *transaction; + tid_t this_tid; + int result, batch_count = 0; + + jbd2_debug(1, "Start checkpoint\n"); + + /* + * First thing: if there are any transactions in the log which + * don't need checkpointing, just eliminate them from the + * journal straight away. + */ + result = jbd2_cleanup_journal_tail(journal); + trace_jbd2_checkpoint(journal, result); + jbd2_debug(1, "cleanup_journal_tail returned %d\n", result); + if (result <= 0) + return result; + + /* + * OK, we need to start writing disk blocks. Take one transaction + * and write it. + */ + spin_lock(&journal->j_list_lock); + if (!journal->j_checkpoint_transactions) + goto out; + transaction = journal->j_checkpoint_transactions; + if (transaction->t_chp_stats.cs_chp_time == 0) + transaction->t_chp_stats.cs_chp_time = jiffies; + this_tid = transaction->t_tid; +restart: + /* + * If someone cleaned up this transaction while we slept, we're + * done (maybe it's a new transaction, but it fell at the same + * address). + */ + if (journal->j_checkpoint_transactions != transaction || + transaction->t_tid != this_tid) + goto out; + + /* checkpoint all of the transaction's buffers */ + while (transaction->t_checkpoint_list) { + jh = transaction->t_checkpoint_list; + bh = jh2bh(jh); + + if (jh->b_transaction != NULL) { + transaction_t *t = jh->b_transaction; + tid_t tid = t->t_tid; + + transaction->t_chp_stats.cs_forced_to_close++; + spin_unlock(&journal->j_list_lock); + if (unlikely(journal->j_flags & JBD2_UNMOUNT)) + /* + * The journal thread is dead; so + * starting and waiting for a commit + * to finish will cause us to wait for + * a _very_ long time. + */ + printk(KERN_ERR + "JBD2: %s: Waiting for Godot: block %llu\n", + journal->j_devname, (unsigned long long) bh->b_blocknr); + + if (batch_count) + __flush_batch(journal, &batch_count); + jbd2_log_start_commit(journal, tid); + /* + * jbd2_journal_commit_transaction() may want + * to take the checkpoint_mutex if JBD2_FLUSHED + * is set, jbd2_update_log_tail() called by + * jbd2_journal_commit_transaction() may also take + * checkpoint_mutex. So we need to temporarily + * drop it. + */ + mutex_unlock(&journal->j_checkpoint_mutex); + jbd2_log_wait_commit(journal, tid); + mutex_lock_io(&journal->j_checkpoint_mutex); + spin_lock(&journal->j_list_lock); + goto restart; + } + if (!trylock_buffer(bh)) { + /* + * The buffer is locked, it may be writing back, or + * flushing out in the last couple of cycles, or + * re-adding into a new transaction, need to check + * it again until it's unlocked. + */ + get_bh(bh); + spin_unlock(&journal->j_list_lock); + wait_on_buffer(bh); + /* the journal_head may have gone by now */ + BUFFER_TRACE(bh, "brelse"); + __brelse(bh); + goto retry; + } else if (!buffer_dirty(bh)) { + unlock_buffer(bh); + BUFFER_TRACE(bh, "remove from checkpoint"); + /* + * If the transaction was released or the checkpoint + * list was empty, we're done. + */ + if (__jbd2_journal_remove_checkpoint(jh) || + !transaction->t_checkpoint_list) + goto out; + } else { + unlock_buffer(bh); + /* + * We are about to write the buffer, it could be + * raced by some other transaction shrink or buffer + * re-log logic once we release the j_list_lock, + * leave it on the checkpoint list and check status + * again to make sure it's clean. + */ + BUFFER_TRACE(bh, "queue"); + get_bh(bh); + if (WARN_ON_ONCE(buffer_jwrite(bh))) { + put_bh(bh); /* drop the ref we just took */ + spin_unlock(&journal->j_list_lock); + /* Clean up any previously batched buffers */ + if (batch_count) + __flush_batch(journal, &batch_count); + jbd2_journal_abort(journal, -EFSCORRUPTED); + return -EFSCORRUPTED; + } + journal->j_chkpt_bhs[batch_count++] = bh; + transaction->t_chp_stats.cs_written++; + transaction->t_checkpoint_list = jh->b_cpnext; + } + + if ((batch_count == JBD2_NR_BATCH) || + need_resched() || spin_needbreak(&journal->j_list_lock) || + jh2bh(transaction->t_checkpoint_list) == journal->j_chkpt_bhs[0]) + goto unlock_and_flush; + } + + if (batch_count) { + unlock_and_flush: + spin_unlock(&journal->j_list_lock); + retry: + if (batch_count) + __flush_batch(journal, &batch_count); + cond_resched(); + spin_lock(&journal->j_list_lock); + goto restart; + } + +out: + spin_unlock(&journal->j_list_lock); + result = jbd2_cleanup_journal_tail(journal); + + return (result < 0) ? result : 0; +} + +/* + * Check the list of checkpoint transactions for the journal to see if + * we have already got rid of any since the last update of the log tail + * in the journal superblock. If so, we can instantly roll the + * superblock forward to remove those transactions from the log. + * + * Return <0 on error, 0 on success, 1 if there was nothing to clean up. + * + * Called with the journal lock held. + * + * This is the only part of the journaling code which really needs to be + * aware of transaction aborts. Checkpointing involves writing to the + * main filesystem area rather than to the journal, so it can proceed + * even in abort state, but we must not update the super block if + * checkpointing may have failed. Otherwise, we would lose some metadata + * buffers which should be written-back to the filesystem. + */ + +int jbd2_cleanup_journal_tail(journal_t *journal) +{ + tid_t first_tid; + unsigned long blocknr; + + if (is_journal_aborted(journal)) + return -EIO; + + if (!jbd2_journal_get_log_tail(journal, &first_tid, &blocknr)) + return 1; + if (WARN_ON_ONCE(blocknr == 0)) { + jbd2_journal_abort(journal, -EFSCORRUPTED); + return -EFSCORRUPTED; + } + + /* + * We need to make sure that any blocks that were recently written out + * --- perhaps by jbd2_log_do_checkpoint() --- are flushed out before + * we drop the transactions from the journal. It's unlikely this will + * be necessary, especially with an appropriately sized journal, but we + * need this to guarantee correctness. Fortunately + * jbd2_cleanup_journal_tail() doesn't get called all that often. + */ + if (journal->j_flags & JBD2_BARRIER) + blkdev_issue_flush(journal->j_fs_dev); + + return __jbd2_update_log_tail(journal, first_tid, blocknr); +} + + +/* Checkpoint list management */ + +/* + * journal_shrink_one_cp_list + * + * Find written-back checkpoint buffers in the given list and try to release + * them. If 'nr_to_scan' is set, scan at most that many buffers. If the whole + * transaction is released, set the 'released' parameter. Return the number of + * released checkpointed buffers. + * + * Called with j_list_lock held. + */ +static unsigned long journal_shrink_one_cp_list(struct journal_head *jh, + enum jbd2_shrink_type type, + unsigned long *nr_to_scan, + bool *released) +{ + struct journal_head *last_jh; + struct journal_head *next_jh = jh; + unsigned long nr_freed = 0; + int ret; + + *released = false; + if (!jh || (nr_to_scan && !*nr_to_scan)) + return 0; + + last_jh = jh->b_cpprev; + do { + jh = next_jh; + next_jh = jh->b_cpnext; + if (nr_to_scan) + (*nr_to_scan)--; + + if (type == JBD2_SHRINK_DESTROY) { + ret = __jbd2_journal_remove_checkpoint(jh); + } else { + ret = jbd2_journal_try_remove_checkpoint(jh); + if (ret < 0) { + if (type == JBD2_SHRINK_BUSY_SKIP) + goto next; + break; + } + } + + nr_freed++; + if (ret) { + *released = true; + break; + } + +next: + if (need_resched()) + break; + } while (jh != last_jh && (!nr_to_scan || *nr_to_scan)); + + return nr_freed; +} + +/* + * jbd2_journal_shrink_checkpoint_list + * + * Find 'nr_to_scan' written-back checkpoint buffers in the journal + * and try to release them. Return the number of released checkpointed + * buffers. + * + * Called with j_list_lock held. + */ +unsigned long jbd2_journal_shrink_checkpoint_list(journal_t *journal, + unsigned long *nr_to_scan) +{ + transaction_t *transaction, *last_transaction, *next_transaction; + bool __maybe_unused released; + tid_t first_tid = 0, last_tid = 0, next_tid = 0; + tid_t tid = 0; + unsigned long nr_freed = 0; + bool first_set = false; + +again: + spin_lock(&journal->j_list_lock); + if (!journal->j_checkpoint_transactions) { + spin_unlock(&journal->j_list_lock); + goto out; + } + + /* + * Get next shrink transaction, resume previous scan or start + * over again. If some others do checkpoint and drop transaction + * from the checkpoint list, we ignore saved j_shrink_transaction + * and start over unconditionally. + */ + if (journal->j_shrink_transaction) + transaction = journal->j_shrink_transaction; + else + transaction = journal->j_checkpoint_transactions; + + if (!first_set) { + first_tid = transaction->t_tid; + first_set = true; + } + last_transaction = journal->j_checkpoint_transactions->t_cpprev; + next_transaction = transaction; + last_tid = last_transaction->t_tid; + do { + transaction = next_transaction; + next_transaction = transaction->t_cpnext; + tid = transaction->t_tid; + + nr_freed += journal_shrink_one_cp_list(transaction->t_checkpoint_list, + JBD2_SHRINK_BUSY_SKIP, + nr_to_scan, &released); + if (*nr_to_scan == 0) + break; + if (need_resched() || spin_needbreak(&journal->j_list_lock)) + break; + } while (transaction != last_transaction); + + if (transaction != last_transaction) { + journal->j_shrink_transaction = next_transaction; + next_tid = next_transaction->t_tid; + } else { + journal->j_shrink_transaction = NULL; + next_tid = 0; + } + + spin_unlock(&journal->j_list_lock); + cond_resched(); + + if (*nr_to_scan && journal->j_shrink_transaction) + goto again; +out: + trace_jbd2_shrink_checkpoint_list(journal, first_tid, tid, last_tid, + nr_freed, next_tid); + + return nr_freed; +} + +/* + * journal_clean_checkpoint_list + * + * Find all the written-back checkpoint buffers in the journal and release them. + * If 'type' is JBD2_SHRINK_DESTROY, release all buffers unconditionally. If + * 'type' is JBD2_SHRINK_BUSY_STOP, will stop release buffers if encounters a + * busy buffer. To avoid wasting CPU cycles scanning the buffer list in some + * cases, don't pass JBD2_SHRINK_BUSY_SKIP 'type' for this function. + * + * Called with j_list_lock held. + */ +void __jbd2_journal_clean_checkpoint_list(journal_t *journal, + enum jbd2_shrink_type type) +{ + transaction_t *transaction, *last_transaction, *next_transaction; + bool released; + + WARN_ON_ONCE(type == JBD2_SHRINK_BUSY_SKIP); + + transaction = journal->j_checkpoint_transactions; + if (!transaction) + return; + + last_transaction = transaction->t_cpprev; + next_transaction = transaction; + do { + transaction = next_transaction; + next_transaction = transaction->t_cpnext; + journal_shrink_one_cp_list(transaction->t_checkpoint_list, + type, NULL, &released); + /* + * This function only frees up some memory if possible so we + * dont have an obligation to finish processing. Bail out if + * preemption requested: + */ + if (need_resched()) + return; + /* + * Stop scanning if we couldn't free the transaction. This + * avoids pointless scanning of transactions which still + * weren't checkpointed. + */ + if (!released) + return; + } while (transaction != last_transaction); +} + +/* + * Remove buffers from all checkpoint lists as journal is aborted and we just + * need to free memory + */ +void jbd2_journal_destroy_checkpoint(journal_t *journal) +{ + /* + * We loop because __jbd2_journal_clean_checkpoint_list() may abort + * early due to a need of rescheduling. + */ + while (1) { + spin_lock(&journal->j_list_lock); + if (!journal->j_checkpoint_transactions) { + spin_unlock(&journal->j_list_lock); + break; + } + __jbd2_journal_clean_checkpoint_list(journal, JBD2_SHRINK_DESTROY); + spin_unlock(&journal->j_list_lock); + cond_resched(); + } +} + +/* + * journal_remove_checkpoint: called after a buffer has been committed + * to disk (either by being write-back flushed to disk, or being + * committed to the log). + * + * We cannot safely clean a transaction out of the log until all of the + * buffer updates committed in that transaction have safely been stored + * elsewhere on disk. To achieve this, all of the buffers in a + * transaction need to be maintained on the transaction's checkpoint + * lists until they have been rewritten, at which point this function is + * called to remove the buffer from the existing transaction's + * checkpoint lists. + * + * The function returns 1 if it frees the transaction, 0 otherwise. + * The function can free jh and bh. + * + * This function is called with j_list_lock held. + */ +int __jbd2_journal_remove_checkpoint(struct journal_head *jh) +{ + struct transaction_chp_stats_s *stats; + transaction_t *transaction; + journal_t *journal; + + JBUFFER_TRACE(jh, "entry"); + + transaction = jh->b_cp_transaction; + if (!transaction) { + JBUFFER_TRACE(jh, "not on transaction"); + return 0; + } + journal = transaction->t_journal; + + JBUFFER_TRACE(jh, "removing from transaction"); + + __buffer_unlink(jh); + jh->b_cp_transaction = NULL; + percpu_counter_dec(&journal->j_checkpoint_jh_count); + jbd2_journal_put_journal_head(jh); + + /* Is this transaction empty? */ + if (transaction->t_checkpoint_list) + return 0; + + /* + * There is one special case to worry about: if we have just pulled the + * buffer off a running or committing transaction's checkpoing list, + * then even if the checkpoint list is empty, the transaction obviously + * cannot be dropped! + * + * The locking here around t_state is a bit sleazy. + * See the comment at the end of jbd2_journal_commit_transaction(). + */ + if (transaction->t_state != T_FINISHED) + return 0; + + /* + * OK, that was the last buffer for the transaction, we can now + * safely remove this transaction from the log. + */ + stats = &transaction->t_chp_stats; + if (stats->cs_chp_time) + stats->cs_chp_time = jbd2_time_diff(stats->cs_chp_time, + jiffies); + trace_jbd2_checkpoint_stats(journal->j_fs_dev->bd_dev, + transaction->t_tid, stats); + + __jbd2_journal_drop_transaction(journal, transaction); + jbd2_journal_free_transaction(transaction); + return 1; +} + +/* + * Check the checkpoint buffer and try to remove it from the checkpoint + * list if it's clean. Returns -EBUSY if it is not clean, returns 1 if + * it frees the transaction, 0 otherwise. + * + * This function is called with j_list_lock held. + */ +int jbd2_journal_try_remove_checkpoint(struct journal_head *jh) +{ + struct buffer_head *bh = jh2bh(jh); + + if (jh->b_transaction) + return -EBUSY; + if (!trylock_buffer(bh)) + return -EBUSY; + if (buffer_dirty(bh)) { + unlock_buffer(bh); + return -EBUSY; + } + unlock_buffer(bh); + + /* + * Buffer is clean and the IO has finished (we held the buffer + * lock) so the checkpoint is done. We can safely remove the + * buffer from this transaction. + */ + JBUFFER_TRACE(jh, "remove from checkpoint list"); + return __jbd2_journal_remove_checkpoint(jh); +} + +/* + * journal_insert_checkpoint: put a committed buffer onto a checkpoint + * list so that we know when it is safe to clean the transaction out of + * the log. + * + * Called with the journal locked. + * Called with j_list_lock held. + */ +void __jbd2_journal_insert_checkpoint(struct journal_head *jh, + transaction_t *transaction) +{ + JBUFFER_TRACE(jh, "entry"); + J_ASSERT_JH(jh, buffer_dirty(jh2bh(jh)) || buffer_jbddirty(jh2bh(jh))); + J_ASSERT_JH(jh, jh->b_cp_transaction == NULL); + + /* Get reference for checkpointing transaction */ + jbd2_journal_grab_journal_head(jh2bh(jh)); + jh->b_cp_transaction = transaction; + + if (!transaction->t_checkpoint_list) { + jh->b_cpnext = jh->b_cpprev = jh; + } else { + jh->b_cpnext = transaction->t_checkpoint_list; + jh->b_cpprev = transaction->t_checkpoint_list->b_cpprev; + jh->b_cpprev->b_cpnext = jh; + jh->b_cpnext->b_cpprev = jh; + } + transaction->t_checkpoint_list = jh; + percpu_counter_inc(&transaction->t_journal->j_checkpoint_jh_count); +} + +/* + * We've finished with this transaction structure: adios... + * + * The transaction must have no links except for the checkpoint by this + * point. + * + * Called with the journal locked. + * Called with j_list_lock held. + */ + +void __jbd2_journal_drop_transaction(journal_t *journal, transaction_t *transaction) +{ + assert_spin_locked(&journal->j_list_lock); + + journal->j_shrink_transaction = NULL; + if (transaction->t_cpnext) { + transaction->t_cpnext->t_cpprev = transaction->t_cpprev; + transaction->t_cpprev->t_cpnext = transaction->t_cpnext; + if (journal->j_checkpoint_transactions == transaction) + journal->j_checkpoint_transactions = + transaction->t_cpnext; + if (journal->j_checkpoint_transactions == transaction) + journal->j_checkpoint_transactions = NULL; + } + + J_ASSERT(transaction->t_state == T_FINISHED); + J_ASSERT(transaction->t_buffers == NULL); + J_ASSERT(transaction->t_forget == NULL); + J_ASSERT(transaction->t_shadow_list == NULL); + J_ASSERT(transaction->t_checkpoint_list == NULL); + J_ASSERT(atomic_read(&transaction->t_updates) == 0); + J_ASSERT(journal->j_committing_transaction != transaction); + J_ASSERT(journal->j_running_transaction != transaction); + + trace_jbd2_drop_transaction(journal, transaction); + + jbd2_debug(1, "Dropping transaction %d, all done\n", transaction->t_tid); +} diff --git a/fs/jbd2/commit.c b/fs/jbd2/commit.c new file mode 100644 index 000000000..3029cb6f6 --- /dev/null +++ b/fs/jbd2/commit.c @@ -0,0 +1,1186 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * linux/fs/jbd2/commit.c + * + * Written by Stephen C. Tweedie , 1998 + * + * Copyright 1998 Red Hat corp --- All Rights Reserved + * + * Journal commit routines for the generic filesystem journaling code; + * part of the ext2fs journaling system. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * IO end handler for temporary buffer_heads handling writes to the journal. + */ +static void journal_end_buffer_io_sync(struct bio *bio) +{ + struct buffer_head *bh; + bool uptodate = bio_endio_bh(bio, &bh); + struct buffer_head *orig_bh = bh->b_private; + + BUFFER_TRACE(bh, ""); + if (uptodate) + set_buffer_uptodate(bh); + else + clear_buffer_uptodate(bh); + if (orig_bh) { + clear_and_wake_up_bit(BH_Shadow, &orig_bh->b_state); + } + unlock_buffer(bh); +} + +/* + * When an ext4 file is truncated, it is possible that some pages are not + * successfully freed, because they are attached to a committing transaction. + * After the transaction commits, these pages are left on the LRU, with no + * ->mapping, and with attached buffers. These pages are trivially reclaimable + * by the VM, but their apparent absence upsets the VM accounting, and it makes + * the numbers in /proc/meminfo look odd. + * + * So here, we have a buffer which has just come off the forget list. Look to + * see if we can strip all buffers from the backing page. + * + * Called under j_list_lock. The caller provided us with a ref against the + * buffer, and we drop that here. + */ +static void release_buffer_page(struct buffer_head *bh) +{ + struct folio *folio; + + if (buffer_dirty(bh)) + goto nope; + if (atomic_read(&bh->b_count) != 1) + goto nope; + folio = bh->b_folio; + if (folio->mapping) + goto nope; + + /* OK, it's a truncated page */ + if (!folio_trylock(folio)) + goto nope; + + folio_get(folio); + __brelse(bh); + try_to_free_buffers(folio); + folio_unlock(folio); + folio_put(folio); + return; + +nope: + __brelse(bh); +} + +static void jbd2_commit_block_csum_set(journal_t *j, struct buffer_head *bh) +{ + struct commit_header *h; + __u32 csum; + + if (!jbd2_journal_has_csum_v2or3(j)) + return; + + h = (struct commit_header *)(bh->b_data); + h->h_chksum_type = 0; + h->h_chksum_size = 0; + h->h_chksum[0] = 0; + csum = jbd2_chksum(j->j_csum_seed, bh->b_data, j->j_blocksize); + h->h_chksum[0] = cpu_to_be32(csum); +} + +/* + * Done it all: now submit the commit record. We should have + * cleaned up our previous buffers by now, so if we are in abort + * mode we can now just skip the rest of the journal write + * entirely. + * + * Returns 1 if the journal needs to be aborted or 0 on success + */ +static int journal_submit_commit_record(journal_t *journal, + transaction_t *commit_transaction, + struct buffer_head **cbh, + __u32 crc32_sum) +{ + struct commit_header *tmp; + struct buffer_head *bh; + struct timespec64 now; + blk_opf_t write_flags = REQ_OP_WRITE | JBD2_JOURNAL_REQ_FLAGS; + + *cbh = NULL; + + if (is_journal_aborted(journal)) + return 0; + + bh = jbd2_journal_get_descriptor_buffer(commit_transaction, + JBD2_COMMIT_BLOCK); + if (!bh) + return 1; + + tmp = (struct commit_header *)bh->b_data; + ktime_get_coarse_real_ts64(&now); + tmp->h_commit_sec = cpu_to_be64(now.tv_sec); + tmp->h_commit_nsec = cpu_to_be32(now.tv_nsec); + + if (jbd2_has_feature_checksum(journal)) { + tmp->h_chksum_type = JBD2_CRC32_CHKSUM; + tmp->h_chksum_size = JBD2_CRC32_CHKSUM_SIZE; + tmp->h_chksum[0] = cpu_to_be32(crc32_sum); + } + jbd2_commit_block_csum_set(journal, bh); + + BUFFER_TRACE(bh, "submit commit block"); + lock_buffer(bh); + clear_buffer_dirty(bh); + set_buffer_uptodate(bh); + + if (journal->j_flags & JBD2_BARRIER && + !jbd2_has_feature_async_commit(journal)) + write_flags |= REQ_PREFLUSH | REQ_FUA; + + bh_submit(bh, write_flags, journal_end_buffer_io_sync); + *cbh = bh; + return 0; +} + +/* + * This function along with journal_submit_commit_record + * allows to write the commit record asynchronously. + */ +static int journal_wait_on_commit_record(journal_t *journal, + struct buffer_head *bh) +{ + int ret = 0; + + clear_buffer_dirty(bh); + wait_on_buffer(bh); + + if (unlikely(!buffer_uptodate(bh))) + ret = -EIO; + put_bh(bh); /* One for getblk() */ + + return ret; +} + +/* Send all the data buffers related to an inode */ +int jbd2_submit_inode_data(journal_t *journal, struct jbd2_inode *jinode) +{ + unsigned long flags; + + if (!jinode) + return 0; + + flags = READ_ONCE(jinode->i_flags); + if (!(flags & JI_WRITE_DATA)) + return 0; + + trace_jbd2_submit_inode_data(jinode->i_vfs_inode); + return journal->j_submit_inode_data_buffers(jinode); + +} +EXPORT_SYMBOL(jbd2_submit_inode_data); + +int jbd2_wait_inode_data(journal_t *journal, struct jbd2_inode *jinode) +{ + struct address_space *mapping; + struct inode *inode; + unsigned long flags; + loff_t start_byte, end_byte; + + if (!jinode) + return 0; + + flags = READ_ONCE(jinode->i_flags); + if (!(flags & JI_WAIT_DATA)) + return 0; + + inode = jinode->i_vfs_inode; + if (!inode) + return 0; + + mapping = inode->i_mapping; + if (!mapping) + return 0; + + if (!jbd2_jinode_get_dirty_range(jinode, &start_byte, &end_byte)) + return 0; + return filemap_fdatawait_range_keep_errors( + mapping, start_byte, end_byte); +} +EXPORT_SYMBOL(jbd2_wait_inode_data); + +/* + * Submit all the data buffers of inode associated with the transaction to + * disk. + * + * We are in a committing transaction. Therefore no new inode can be added to + * our inode list. We use JI_COMMIT_RUNNING flag to protect inode we currently + * operate on from being released while we write out pages. + */ +static int journal_submit_data_buffers(journal_t *journal, + transaction_t *commit_transaction) +{ + struct jbd2_inode *jinode; + int err, ret = 0; + + spin_lock(&journal->j_list_lock); + list_for_each_entry(jinode, &commit_transaction->t_inode_list, i_list) { + if (!(jinode->i_flags & JI_WRITE_DATA)) + continue; + WRITE_ONCE(jinode->i_flags, + jinode->i_flags | JI_COMMIT_RUNNING); + spin_unlock(&journal->j_list_lock); + /* submit the inode data buffers. */ + trace_jbd2_submit_inode_data(jinode->i_vfs_inode); + if (journal->j_submit_inode_data_buffers) { + err = journal->j_submit_inode_data_buffers(jinode); + if (!ret) + ret = err; + } + spin_lock(&journal->j_list_lock); + J_ASSERT(jinode->i_transaction == commit_transaction); + WRITE_ONCE(jinode->i_flags, + jinode->i_flags & ~JI_COMMIT_RUNNING); + smp_mb(); + wake_up_bit(&jinode->i_flags, __JI_COMMIT_RUNNING); + } + spin_unlock(&journal->j_list_lock); + return ret; +} + +int jbd2_journal_finish_inode_data_buffers(struct jbd2_inode *jinode) +{ + struct address_space *mapping = jinode->i_vfs_inode->i_mapping; + loff_t start_byte, end_byte; + + if (!jbd2_jinode_get_dirty_range(jinode, &start_byte, &end_byte)) + return 0; + + return filemap_fdatawait_range_keep_errors(mapping, + start_byte, end_byte); +} + +/* + * Wait for data submitted for writeout, refile inodes to proper + * transaction if needed. + * + */ +static int journal_finish_inode_data_buffers(journal_t *journal, + transaction_t *commit_transaction) +{ + struct jbd2_inode *jinode, *next_i; + int err, ret = 0; + + /* For locking, see the comment in journal_submit_data_buffers() */ + spin_lock(&journal->j_list_lock); + list_for_each_entry(jinode, &commit_transaction->t_inode_list, i_list) { + if (!(jinode->i_flags & JI_WAIT_DATA)) + continue; + WRITE_ONCE(jinode->i_flags, jinode->i_flags | JI_COMMIT_RUNNING); + spin_unlock(&journal->j_list_lock); + /* wait for the inode data buffers writeout. */ + if (journal->j_finish_inode_data_buffers) { + err = journal->j_finish_inode_data_buffers(jinode); + if (!ret) + ret = err; + } + cond_resched(); + spin_lock(&journal->j_list_lock); + WRITE_ONCE(jinode->i_flags, jinode->i_flags & ~JI_COMMIT_RUNNING); + smp_mb(); + wake_up_bit(&jinode->i_flags, __JI_COMMIT_RUNNING); + } + + /* Now refile inode to proper lists */ + list_for_each_entry_safe(jinode, next_i, + &commit_transaction->t_inode_list, i_list) { + list_del(&jinode->i_list); + if (jinode->i_next_transaction) { + jinode->i_transaction = jinode->i_next_transaction; + jinode->i_next_transaction = NULL; + list_add(&jinode->i_list, + &jinode->i_transaction->t_inode_list); + } else { + jinode->i_transaction = NULL; + WRITE_ONCE(jinode->i_dirty_start_page, 0); + WRITE_ONCE(jinode->i_dirty_end_page, 0); + } + } + spin_unlock(&journal->j_list_lock); + + return ret; +} + +static __u32 jbd2_checksum_data(__u32 crc32_sum, struct buffer_head *bh) +{ + char *addr; + __u32 checksum; + + addr = kmap_local_folio(bh->b_folio, bh_offset(bh)); + checksum = crc32_be(crc32_sum, addr, bh->b_size); + kunmap_local(addr); + + return checksum; +} + +static void write_tag_block(journal_t *j, journal_block_tag_t *tag, + unsigned long long block) +{ + tag->t_blocknr = cpu_to_be32(block & (u32)~0); + if (jbd2_has_feature_64bit(j)) + tag->t_blocknr_high = cpu_to_be32((block >> 31) >> 1); +} + +static void jbd2_block_tag_csum_set(journal_t *j, journal_block_tag_t *tag, + struct buffer_head *bh, __u32 sequence) +{ + journal_block_tag3_t *tag3 = (journal_block_tag3_t *)tag; + __u8 *addr; + __u32 csum32; + __be32 seq; + + if (!jbd2_journal_has_csum_v2or3(j)) + return; + + seq = cpu_to_be32(sequence); + addr = kmap_local_folio(bh->b_folio, bh_offset(bh)); + csum32 = jbd2_chksum(j->j_csum_seed, (__u8 *)&seq, sizeof(seq)); + csum32 = jbd2_chksum(csum32, addr, bh->b_size); + kunmap_local(addr); + + if (jbd2_has_feature_csum3(j)) + tag3->t_checksum = cpu_to_be32(csum32); + else + tag->t_checksum = cpu_to_be16(csum32); +} +/* + * jbd2_journal_commit_transaction + * + * The primary function for committing a transaction to the log. This + * function is called by the journal thread to begin a complete commit. + */ +void jbd2_journal_commit_transaction(journal_t *journal) +{ + struct transaction_stats_s stats; + transaction_t *commit_transaction; + struct journal_head *jh; + struct buffer_head *descriptor; + struct buffer_head **wbuf = journal->j_wbuf; + int bufs; + int escape; + int err; + unsigned long long blocknr; + ktime_t start_time; + u64 commit_time; + char *tagp = NULL; + journal_block_tag_t *tag = NULL; + int space_left = 0; + int first_tag = 0; + int tag_flag; + int i; + int tag_bytes = journal_tag_bytes(journal); + struct buffer_head *cbh = NULL; /* For transactional checksums */ + __u32 crc32_sum = ~0; + struct blk_plug plug; + /* Tail of the journal */ + unsigned long first_block; + tid_t first_tid; + int update_tail; + int csum_size = 0; + LIST_HEAD(io_bufs); + LIST_HEAD(log_bufs); + + if (jbd2_journal_has_csum_v2or3(journal)) + csum_size = sizeof(struct jbd2_journal_block_tail); + + /* + * First job: lock down the current transaction and wait for + * all outstanding updates to complete. + */ + + /* Do we need to erase the effects of a prior jbd2_journal_flush? */ + if (journal->j_flags & JBD2_FLUSHED) { + jbd2_debug(3, "super block updated\n"); + mutex_lock_io(&journal->j_checkpoint_mutex); + /* + * We hold j_checkpoint_mutex so tail cannot change under us. + * We don't need any special data guarantees for writing sb + * since journal is empty and it is ok for write to be + * flushed only with transaction commit. + */ + jbd2_journal_update_sb_log_tail(journal, + journal->j_tail_sequence, + journal->j_tail, 0); + mutex_unlock(&journal->j_checkpoint_mutex); + } else { + jbd2_debug(3, "superblock not updated\n"); + } + + J_ASSERT(journal->j_running_transaction != NULL); + J_ASSERT(journal->j_committing_transaction == NULL); + + write_lock(&journal->j_state_lock); + journal->j_flags |= JBD2_FULL_COMMIT_ONGOING; + while (journal->j_flags & JBD2_FAST_COMMIT_ONGOING) { + DEFINE_WAIT(wait); + + prepare_to_wait(&journal->j_fc_wait, &wait, + TASK_UNINTERRUPTIBLE); + write_unlock(&journal->j_state_lock); + schedule(); + write_lock(&journal->j_state_lock); + finish_wait(&journal->j_fc_wait, &wait); + /* + * TODO: by blocking fast commits here, we are increasing + * fsync() latency slightly. Strictly speaking, we don't need + * to block fast commits until the transaction enters T_FLUSH + * state. So an optimization is possible where we block new fast + * commits here and wait for existing ones to complete + * just before we enter T_FLUSH. That way, the existing fast + * commits and this full commit can proceed parallely. + */ + } + write_unlock(&journal->j_state_lock); + + commit_transaction = journal->j_running_transaction; + + trace_jbd2_start_commit(journal, commit_transaction); + jbd2_debug(1, "JBD2: starting commit of transaction %d\n", + commit_transaction->t_tid); + + write_lock(&journal->j_state_lock); + journal->j_fc_off = 0; + J_ASSERT(commit_transaction->t_state == T_RUNNING); + commit_transaction->t_state = T_LOCKED; + + trace_jbd2_commit_locking(journal, commit_transaction); + stats.run.rs_wait = commit_transaction->t_max_wait; + stats.run.rs_request_delay = 0; + stats.run.rs_locked = jiffies; + if (commit_transaction->t_requested) + stats.run.rs_request_delay = + jbd2_time_diff(commit_transaction->t_requested, + stats.run.rs_locked); + stats.run.rs_running = jbd2_time_diff(commit_transaction->t_start, + stats.run.rs_locked); + + // waits for any t_updates to finish + jbd2_journal_wait_updates(journal); + + commit_transaction->t_state = T_SWITCH; + + J_ASSERT (atomic_read(&commit_transaction->t_outstanding_credits) <= + journal->j_max_transaction_buffers); + + /* + * First thing we are allowed to do is to discard any remaining + * BJ_Reserved buffers. Note, it is _not_ permissible to assume + * that there are no such buffers: if a large filesystem + * operation like a truncate needs to split itself over multiple + * transactions, then it may try to do a jbd2_journal_restart() while + * there are still BJ_Reserved buffers outstanding. These must + * be released cleanly from the current transaction. + * + * In this case, the filesystem must still reserve write access + * again before modifying the buffer in the new transaction, but + * we do not require it to remember exactly which old buffers it + * has reserved. This is consistent with the existing behaviour + * that multiple jbd2_journal_get_write_access() calls to the same + * buffer are perfectly permissible. + * We use journal->j_state_lock here to serialize processing of + * t_reserved_list with eviction of buffers from journal_unmap_buffer(). + */ + while (commit_transaction->t_reserved_list) { + jh = commit_transaction->t_reserved_list; + JBUFFER_TRACE(jh, "reserved, unused: refile"); + /* + * A jbd2_journal_get_undo_access()+jbd2_journal_release_buffer() may + * leave undo-committed data. + */ + if (jh->b_committed_data) { + spin_lock(&jh->b_state_lock); + kfree(jh->b_committed_data); + jh->b_committed_data = NULL; + spin_unlock(&jh->b_state_lock); + } + jbd2_journal_refile_buffer(journal, jh); + } + + write_unlock(&journal->j_state_lock); + /* + * Now try to drop any written-back buffers from the journal's + * checkpoint lists. We do this *before* commit because it potentially + * frees some memory + */ + spin_lock(&journal->j_list_lock); + __jbd2_journal_clean_checkpoint_list(journal, JBD2_SHRINK_BUSY_STOP); + spin_unlock(&journal->j_list_lock); + + jbd2_debug(3, "JBD2: commit phase 1\n"); + + /* + * Clear revoked flag to reflect there is no revoked buffers + * in the next transaction which is going to be started. + */ + jbd2_clear_buffer_revoked_flags(journal); + + /* + * Switch to a new revoke table. + */ + jbd2_journal_switch_revoke_table(journal); + + write_lock(&journal->j_state_lock); + /* + * Reserved credits cannot be claimed anymore, free them + */ + atomic_sub(atomic_read(&journal->j_reserved_credits), + &commit_transaction->t_outstanding_credits); + + trace_jbd2_commit_flushing(journal, commit_transaction); + stats.run.rs_flushing = jiffies; + stats.run.rs_locked = jbd2_time_diff(stats.run.rs_locked, + stats.run.rs_flushing); + + commit_transaction->t_state = T_FLUSH; + journal->j_committing_transaction = commit_transaction; + journal->j_running_transaction = NULL; + start_time = ktime_get(); + commit_transaction->t_log_start = journal->j_head; + wake_up_all(&journal->j_wait_transaction_locked); + write_unlock(&journal->j_state_lock); + + jbd2_debug(3, "JBD2: commit phase 2a\n"); + + /* + * Now start flushing things to disk, in the order they appear + * on the transaction lists. Data blocks go first. + */ + err = journal_submit_data_buffers(journal, commit_transaction); + if (err) + jbd2_journal_abort(journal, err); + + blk_start_plug(&plug); + jbd2_journal_write_revoke_records(commit_transaction, &log_bufs); + + jbd2_debug(3, "JBD2: commit phase 2b\n"); + + /* + * Way to go: we have now written out all of the data for a + * transaction! Now comes the tricky part: we need to write out + * metadata. Loop over the transaction's entire buffer list: + */ + write_lock(&journal->j_state_lock); + commit_transaction->t_state = T_COMMIT; + write_unlock(&journal->j_state_lock); + + trace_jbd2_commit_logging(journal, commit_transaction); + stats.run.rs_logging = jiffies; + stats.run.rs_flushing = jbd2_time_diff(stats.run.rs_flushing, + stats.run.rs_logging); + stats.run.rs_blocks = commit_transaction->t_nr_buffers; + stats.run.rs_blocks_logged = 0; + + J_ASSERT(commit_transaction->t_nr_buffers <= + atomic_read(&commit_transaction->t_outstanding_credits)); + + bufs = 0; + descriptor = NULL; + while (commit_transaction->t_buffers) { + + /* Find the next buffer to be journaled... */ + + jh = commit_transaction->t_buffers; + + /* If we're in abort mode, we just un-journal the buffer and + release it. */ + + if (is_journal_aborted(journal)) { + clear_buffer_jbddirty(jh2bh(jh)); + JBUFFER_TRACE(jh, "journal is aborting: refile"); + jbd2_buffer_abort_trigger(jh, + jh->b_frozen_data ? + jh->b_frozen_triggers : + jh->b_triggers); + jbd2_journal_refile_buffer(journal, jh); + /* If that was the last one, we need to clean up + * any descriptor buffers which may have been + * already allocated, even if we are now + * aborting. */ + if (!commit_transaction->t_buffers) + goto start_journal_io; + continue; + } + + /* Make sure we have a descriptor block in which to + record the metadata buffer. */ + + if (!descriptor) { + J_ASSERT (bufs == 0); + + jbd2_debug(4, "JBD2: get descriptor\n"); + + descriptor = jbd2_journal_get_descriptor_buffer( + commit_transaction, + JBD2_DESCRIPTOR_BLOCK); + if (!descriptor) { + jbd2_journal_abort(journal, -EIO); + continue; + } + + jbd2_debug(4, "JBD2: got buffer %llu (%p)\n", + (unsigned long long)descriptor->b_blocknr, + descriptor->b_data); + tagp = &descriptor->b_data[sizeof(journal_header_t)]; + space_left = descriptor->b_size - + sizeof(journal_header_t); + first_tag = 1; + set_buffer_jwrite(descriptor); + set_buffer_dirty(descriptor); + wbuf[bufs++] = descriptor; + + /* Record it so that we can wait for IO + completion later */ + BUFFER_TRACE(descriptor, "ph3: file as descriptor"); + jbd2_file_log_bh(&log_bufs, descriptor); + } + + /* Where is the buffer to be written? */ + + err = jbd2_journal_next_log_block(journal, &blocknr); + /* If the block mapping failed, just abandon the buffer + and repeat this loop: we'll fall into the + refile-on-abort condition above. */ + if (err) { + jbd2_journal_abort(journal, err); + continue; + } + + /* + * start_this_handle() uses t_outstanding_credits to determine + * the free space in the log. + */ + atomic_dec(&commit_transaction->t_outstanding_credits); + + /* Bump b_count to prevent truncate from stumbling over + the shadowed buffer! @@@ This can go if we ever get + rid of the shadow pairing of buffers. */ + atomic_inc(&jh2bh(jh)->b_count); + + /* + * Make a temporary IO buffer with which to write it out + * (this will requeue the metadata buffer to BJ_Shadow). + */ + set_bit(BH_JWrite, &jh2bh(jh)->b_state); + JBUFFER_TRACE(jh, "ph3: write metadata"); + escape = jbd2_journal_write_metadata_buffer(commit_transaction, + jh, &wbuf[bufs], blocknr); + jbd2_file_log_bh(&io_bufs, wbuf[bufs]); + + /* Record the new block's tag in the current descriptor + buffer */ + + tag_flag = 0; + if (escape) + tag_flag |= JBD2_FLAG_ESCAPE; + if (!first_tag) + tag_flag |= JBD2_FLAG_SAME_UUID; + + tag = (journal_block_tag_t *) tagp; + write_tag_block(journal, tag, jh2bh(jh)->b_blocknr); + tag->t_flags = cpu_to_be16(tag_flag); + jbd2_block_tag_csum_set(journal, tag, wbuf[bufs], + commit_transaction->t_tid); + tagp += tag_bytes; + space_left -= tag_bytes; + bufs++; + + if (first_tag) { + memcpy (tagp, journal->j_uuid, 16); + tagp += 16; + space_left -= 16; + first_tag = 0; + } + + /* If there's no more to do, or if the descriptor is full, + let the IO rip! */ + + if (bufs == journal->j_wbufsize || + commit_transaction->t_buffers == NULL || + space_left < tag_bytes + 16 + csum_size) { + + jbd2_debug(4, "JBD2: Submit %d IOs\n", bufs); + + /* Write an end-of-descriptor marker before + submitting the IOs. "tag" still points to + the last tag we set up. */ + + tag->t_flags |= cpu_to_be16(JBD2_FLAG_LAST_TAG); +start_journal_io: + if (descriptor) + jbd2_descriptor_block_csum_set(journal, + descriptor); + + for (i = 0; i < bufs; i++) { + struct buffer_head *bh = wbuf[i]; + + /* + * Compute checksum. + */ + if (jbd2_has_feature_checksum(journal)) { + crc32_sum = + jbd2_checksum_data(crc32_sum, bh); + } + + lock_buffer(bh); + clear_buffer_dirty(bh); + set_buffer_uptodate(bh); + bh_submit(bh, + REQ_OP_WRITE | JBD2_JOURNAL_REQ_FLAGS, + journal_end_buffer_io_sync); + } + cond_resched(); + + /* Force a new descriptor to be generated next + time round the loop. */ + descriptor = NULL; + bufs = 0; + } + } + + err = journal_finish_inode_data_buffers(journal, commit_transaction); + if (err) { + printk(KERN_WARNING + "JBD2: Detected IO errors %d while flushing file data on %s\n", + err, journal->j_devname); + err = 0; + } + + /* + * Get current oldest transaction in the log before we issue flush + * to the filesystem device. After the flush we can be sure that + * blocks of all older transactions are checkpointed to persistent + * storage and we will be safe to update journal start in the + * superblock with the numbers we get here. + */ + update_tail = + jbd2_journal_get_log_tail(journal, &first_tid, &first_block); + + write_lock(&journal->j_state_lock); + if (update_tail) { + long freed = first_block - journal->j_tail; + + if (first_block < journal->j_tail) + freed += journal->j_last - journal->j_first; + /* Update tail only if we free significant amount of space */ + if (freed < journal->j_max_transaction_buffers) + update_tail = 0; + } + J_ASSERT(commit_transaction->t_state == T_COMMIT); + commit_transaction->t_state = T_COMMIT_DFLUSH; + write_unlock(&journal->j_state_lock); + + /* + * If the journal is not located on the file system device, + * then we must flush the file system device before we issue + * the commit record and update the journal tail sequence. + */ + if ((commit_transaction->t_need_data_flush || update_tail) && + (journal->j_fs_dev != journal->j_dev) && + (journal->j_flags & JBD2_BARRIER)) + blkdev_issue_flush(journal->j_fs_dev); + + /* Done it all: now write the commit record asynchronously. */ + if (jbd2_has_feature_async_commit(journal)) { + err = journal_submit_commit_record(journal, commit_transaction, + &cbh, crc32_sum); + if (err) + jbd2_journal_abort(journal, err); + } + + blk_finish_plug(&plug); + + /* Lo and behold: we have just managed to send a transaction to + the log. Before we can commit it, wait for the IO so far to + complete. Control buffers being written are on the + transaction's t_log_list queue, and metadata buffers are on + the io_bufs list. + + Wait for the buffers in reverse order. That way we are + less likely to be woken up until all IOs have completed, and + so we incur less scheduling load. + */ + + jbd2_debug(3, "JBD2: commit phase 3\n"); + + while (!list_empty(&io_bufs)) { + struct buffer_head *bh = list_entry(io_bufs.prev, + struct buffer_head, + b_assoc_buffers); + + wait_on_buffer(bh); + cond_resched(); + + if (unlikely(!buffer_uptodate(bh))) + err = -EIO; + jbd2_unfile_log_bh(bh); + stats.run.rs_blocks_logged++; + + /* + * The list contains temporary buffer heads created by + * jbd2_journal_write_metadata_buffer(). + */ + BUFFER_TRACE(bh, "dumping temporary bh"); + __brelse(bh); + J_ASSERT_BH(bh, atomic_read(&bh->b_count) == 0); + free_buffer_head(bh); + + /* We also have to refile the corresponding shadowed buffer */ + jh = commit_transaction->t_shadow_list->b_tprev; + bh = jh2bh(jh); + clear_buffer_jwrite(bh); + J_ASSERT_BH(bh, buffer_jbddirty(bh)); + J_ASSERT_BH(bh, !buffer_shadow(bh)); + + /* The metadata is now released for reuse, but we need + to remember it against this transaction so that when + we finally commit, we can do any checkpointing + required. */ + JBUFFER_TRACE(jh, "file as BJ_Forget"); + jbd2_journal_file_buffer(jh, commit_transaction, BJ_Forget); + JBUFFER_TRACE(jh, "brelse shadowed buffer"); + __brelse(bh); + } + + J_ASSERT (commit_transaction->t_shadow_list == NULL); + + jbd2_debug(3, "JBD2: commit phase 4\n"); + + /* Here we wait for the revoke record and descriptor record buffers */ + while (!list_empty(&log_bufs)) { + struct buffer_head *bh; + + bh = list_entry(log_bufs.prev, struct buffer_head, b_assoc_buffers); + wait_on_buffer(bh); + cond_resched(); + + if (unlikely(!buffer_uptodate(bh))) + err = -EIO; + + BUFFER_TRACE(bh, "ph5: control buffer writeout done: unfile"); + clear_buffer_jwrite(bh); + jbd2_unfile_log_bh(bh); + stats.run.rs_blocks_logged++; + __brelse(bh); /* One for getblk */ + /* AKPM: bforget here */ + } + + if (err) + jbd2_journal_abort(journal, err); + + jbd2_debug(3, "JBD2: commit phase 5\n"); + write_lock(&journal->j_state_lock); + J_ASSERT(commit_transaction->t_state == T_COMMIT_DFLUSH); + commit_transaction->t_state = T_COMMIT_JFLUSH; + write_unlock(&journal->j_state_lock); + + if (!jbd2_has_feature_async_commit(journal)) { + err = journal_submit_commit_record(journal, commit_transaction, + &cbh, crc32_sum); + if (err) + jbd2_journal_abort(journal, err); + } + if (cbh) + err = journal_wait_on_commit_record(journal, cbh); + stats.run.rs_blocks_logged++; + if (jbd2_has_feature_async_commit(journal) && + journal->j_flags & JBD2_BARRIER) { + blkdev_issue_flush(journal->j_dev); + } + + if (err) + jbd2_journal_abort(journal, err); + + WARN_ON_ONCE( + atomic_read(&commit_transaction->t_outstanding_credits) < 0); + + /* + * Now disk caches for filesystem device are flushed so we are safe to + * erase checkpointed transactions from the log by updating journal + * superblock. + */ + if (update_tail) + jbd2_update_log_tail(journal, first_tid, first_block); + + /* End of a transaction! Finally, we can do checkpoint + processing: any buffers committed as a result of this + transaction can be removed from any checkpoint list it was on + before. */ + + jbd2_debug(3, "JBD2: commit phase 6\n"); + + J_ASSERT(list_empty(&commit_transaction->t_inode_list)); + J_ASSERT(commit_transaction->t_buffers == NULL); + J_ASSERT(commit_transaction->t_checkpoint_list == NULL); + J_ASSERT(commit_transaction->t_shadow_list == NULL); + +restart_loop: + /* + * As there are other places (journal_unmap_buffer()) adding buffers + * to this list we have to be careful and hold the j_list_lock. + */ + spin_lock(&journal->j_list_lock); + while (commit_transaction->t_forget) { + transaction_t *cp_transaction; + struct buffer_head *bh; + int try_to_free = 0; + bool drop_ref; + + jh = commit_transaction->t_forget; + spin_unlock(&journal->j_list_lock); + bh = jh2bh(jh); + /* + * Get a reference so that bh cannot be freed before we are + * done with it. + */ + get_bh(bh); + spin_lock(&jh->b_state_lock); + J_ASSERT_JH(jh, jh->b_transaction == commit_transaction); + + /* + * If there is undo-protected committed data against + * this buffer, then we can remove it now. If it is a + * buffer needing such protection, the old frozen_data + * field now points to a committed version of the + * buffer, so rotate that field to the new committed + * data. + * + * Otherwise, we can just throw away the frozen data now. + * + * We also know that the frozen data has already fired + * its triggers if they exist, so we can clear that too. + */ + if (jh->b_committed_data) { + kfree(jh->b_committed_data); + jh->b_committed_data = NULL; + if (jh->b_frozen_data) { + jh->b_committed_data = jh->b_frozen_data; + jh->b_frozen_data = NULL; + jh->b_frozen_triggers = NULL; + } + } else if (jh->b_frozen_data) { + kfree(jh->b_frozen_data); + jh->b_frozen_data = NULL; + jh->b_frozen_triggers = NULL; + } + + spin_lock(&journal->j_list_lock); + cp_transaction = jh->b_cp_transaction; + if (cp_transaction) { + JBUFFER_TRACE(jh, "remove from old cp transaction"); + cp_transaction->t_chp_stats.cs_dropped++; + __jbd2_journal_remove_checkpoint(jh); + } + + /* Only re-checkpoint the buffer_head if it is marked + * dirty. If the buffer was added to the BJ_Forget list + * by jbd2_journal_forget, it may no longer be dirty and + * there's no point in keeping a checkpoint record for + * it. */ + + /* + * A buffer which has been freed while still being journaled + * by a previous transaction, refile the buffer to BJ_Forget of + * the running transaction. If the just committed transaction + * contains "add to orphan" operation, we can completely + * invalidate the buffer now. We are rather through in that + * since the buffer may be still accessible when blocksize < + * pagesize and it is attached to the last partial page. + */ + if (buffer_freed(bh) && !jh->b_next_transaction) { + struct address_space *mapping; + + clear_buffer_freed(bh); + clear_buffer_jbddirty(bh); + + /* + * Block device buffers need to stay mapped all the + * time, so it is enough to clear buffer_jbddirty and + * buffer_freed bits. For the file mapping buffers (i.e. + * journalled data) we need to unmap buffer and clear + * more bits. We also need to be careful about the check + * because the data page mapping can get cleared under + * our hands. Note that if mapping == NULL, we don't + * need to make buffer unmapped because the page is + * already detached from the mapping and buffers cannot + * get reused. + */ + mapping = READ_ONCE(bh->b_folio->mapping); + if (mapping && !sb_is_blkdev_sb(mapping->host->i_sb)) { + clear_buffer_mapped(bh); + clear_buffer_new(bh); + clear_buffer_req(bh); + bh->b_bdev = NULL; + } + } + + if (buffer_jbddirty(bh)) { + JBUFFER_TRACE(jh, "add to new checkpointing trans"); + __jbd2_journal_insert_checkpoint(jh, commit_transaction); + if (is_journal_aborted(journal)) + clear_buffer_jbddirty(bh); + } else { + J_ASSERT_BH(bh, !buffer_dirty(bh)); + /* + * The buffer on BJ_Forget list and not jbddirty means + * it has been freed by this transaction and hence it + * could not have been reallocated until this + * transaction has committed. *BUT* it could be + * reallocated once we have written all the data to + * disk and before we process the buffer on BJ_Forget + * list. + */ + if (!jh->b_next_transaction) + try_to_free = 1; + } + JBUFFER_TRACE(jh, "refile or unfile buffer"); + drop_ref = __jbd2_journal_refile_buffer(jh); + spin_unlock(&jh->b_state_lock); + if (drop_ref) + jbd2_journal_put_journal_head(jh); + if (try_to_free) + release_buffer_page(bh); /* Drops bh reference */ + else + __brelse(bh); + cond_resched_lock(&journal->j_list_lock); + } + spin_unlock(&journal->j_list_lock); + /* + * This is a bit sleazy. We use j_list_lock to protect transition + * of a transaction into T_FINISHED state and calling + * __jbd2_journal_drop_transaction(). Otherwise we could race with + * other checkpointing code processing the transaction... + */ + write_lock(&journal->j_state_lock); + spin_lock(&journal->j_list_lock); + /* + * Now recheck if some buffers did not get attached to the transaction + * while the lock was dropped... + */ + if (commit_transaction->t_forget) { + spin_unlock(&journal->j_list_lock); + write_unlock(&journal->j_state_lock); + goto restart_loop; + } + + /* Add the transaction to the checkpoint list + * __journal_remove_checkpoint() can not destroy transaction + * under us because it is not marked as T_FINISHED yet */ + if (journal->j_checkpoint_transactions == NULL) { + journal->j_checkpoint_transactions = commit_transaction; + commit_transaction->t_cpnext = commit_transaction; + commit_transaction->t_cpprev = commit_transaction; + } else { + commit_transaction->t_cpnext = + journal->j_checkpoint_transactions; + commit_transaction->t_cpprev = + commit_transaction->t_cpnext->t_cpprev; + commit_transaction->t_cpnext->t_cpprev = + commit_transaction; + commit_transaction->t_cpprev->t_cpnext = + commit_transaction; + } + spin_unlock(&journal->j_list_lock); + + /* Done with this transaction! */ + + jbd2_debug(3, "JBD2: commit phase 7\n"); + + J_ASSERT(commit_transaction->t_state == T_COMMIT_JFLUSH); + + commit_transaction->t_start = jiffies; + stats.run.rs_logging = jbd2_time_diff(stats.run.rs_logging, + commit_transaction->t_start); + + /* + * File the transaction statistics + */ + stats.ts_tid = commit_transaction->t_tid; + stats.run.rs_handle_count = + atomic_read(&commit_transaction->t_handle_count); + trace_jbd2_run_stats(journal->j_fs_dev->bd_dev, + commit_transaction->t_tid, &stats.run); + stats.ts_requested = (commit_transaction->t_requested) ? 1 : 0; + + commit_transaction->t_state = T_COMMIT_CALLBACK; + J_ASSERT(commit_transaction == journal->j_committing_transaction); + WRITE_ONCE(journal->j_commit_sequence, commit_transaction->t_tid); + journal->j_committing_transaction = NULL; + commit_time = ktime_to_ns(ktime_sub(ktime_get(), start_time)); + + /* + * weight the commit time higher than the average time so we don't + * react too strongly to vast changes in the commit time + */ + if (likely(journal->j_average_commit_time)) + journal->j_average_commit_time = (commit_time + + journal->j_average_commit_time*3) / 4; + else + journal->j_average_commit_time = commit_time; + + write_unlock(&journal->j_state_lock); + + if (journal->j_commit_callback) + journal->j_commit_callback(journal, commit_transaction); + if (journal->j_fc_cleanup_callback) + journal->j_fc_cleanup_callback(journal, 1, commit_transaction->t_tid); + + trace_jbd2_end_commit(journal, commit_transaction); + jbd2_debug(1, "JBD2: commit %d complete, head %d\n", + journal->j_commit_sequence, journal->j_tail_sequence); + + write_lock(&journal->j_state_lock); + journal->j_flags &= ~JBD2_FULL_COMMIT_ONGOING; + journal->j_flags &= ~JBD2_FAST_COMMIT_ONGOING; + spin_lock(&journal->j_list_lock); + commit_transaction->t_state = T_FINISHED; + /* Check if the transaction can be dropped now that we are finished */ + if (commit_transaction->t_checkpoint_list == NULL) { + __jbd2_journal_drop_transaction(journal, commit_transaction); + jbd2_journal_free_transaction(commit_transaction); + } + spin_unlock(&journal->j_list_lock); + write_unlock(&journal->j_state_lock); + wake_up(&journal->j_wait_done_commit); + wake_up(&journal->j_fc_wait); + + /* + * Calculate overall stats + */ + spin_lock(&journal->j_history_lock); + journal->j_stats.ts_tid++; + journal->j_stats.ts_requested += stats.ts_requested; + journal->j_stats.run.rs_wait += stats.run.rs_wait; + journal->j_stats.run.rs_request_delay += stats.run.rs_request_delay; + journal->j_stats.run.rs_running += stats.run.rs_running; + journal->j_stats.run.rs_locked += stats.run.rs_locked; + journal->j_stats.run.rs_flushing += stats.run.rs_flushing; + journal->j_stats.run.rs_logging += stats.run.rs_logging; + journal->j_stats.run.rs_handle_count += stats.run.rs_handle_count; + journal->j_stats.run.rs_blocks += stats.run.rs_blocks; + journal->j_stats.run.rs_blocks_logged += stats.run.rs_blocks_logged; + spin_unlock(&journal->j_history_lock); +} diff --git a/fs/jbd2/journal.c b/fs/jbd2/journal.c new file mode 100644 index 000000000..00f5a98f3 --- /dev/null +++ b/fs/jbd2/journal.c @@ -0,0 +1,3066 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * linux/fs/jbd2/journal.c + * + * Written by Stephen C. Tweedie , 1998 + * + * Copyright 1998 Red Hat corp --- All Rights Reserved + * + * Generic filesystem journal-writing code; part of the ext2fs + * journaling system. + * + * This file manages journals: areas of disk reserved for logging + * transactional updates. This includes the kernel journaling thread + * which is responsible for scheduling updates to the log. + * + * We do not actually manage the physical storage of the journal in this + * file: that is left to a per-journal policy function, which allows us + * to store the journal within a filesystem-specified area for ext2 + * journaling (ext2 can use a reserved inode for storing the log). + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CREATE_TRACE_POINTS +#include + +#include +#include + +#ifdef CONFIG_JBD2_DEBUG +static ushort jbd2_journal_enable_debug __read_mostly; + +module_param_named(jbd2_debug, jbd2_journal_enable_debug, ushort, 0644); +MODULE_PARM_DESC(jbd2_debug, "Debugging level for jbd2"); +#endif + +EXPORT_SYMBOL(jbd2_journal_extend); +EXPORT_SYMBOL(jbd2_journal_stop); +EXPORT_SYMBOL(jbd2_journal_lock_updates); +EXPORT_SYMBOL(jbd2_journal_unlock_updates); +EXPORT_SYMBOL(jbd2_journal_get_write_access); +EXPORT_SYMBOL(jbd2_journal_get_create_access); +EXPORT_SYMBOL(jbd2_journal_get_undo_access); +EXPORT_SYMBOL(jbd2_journal_set_triggers); +EXPORT_SYMBOL(jbd2_journal_dirty_metadata); +EXPORT_SYMBOL(jbd2_journal_forget); +EXPORT_SYMBOL(jbd2_journal_flush); +EXPORT_SYMBOL(jbd2_journal_revoke); + +EXPORT_SYMBOL(jbd2_journal_init_dev); +EXPORT_SYMBOL(jbd2_journal_init_inode); +EXPORT_SYMBOL(jbd2_journal_check_used_features); +EXPORT_SYMBOL(jbd2_journal_check_available_features); +EXPORT_SYMBOL(jbd2_journal_set_features); +EXPORT_SYMBOL(jbd2_journal_load); +EXPORT_SYMBOL(jbd2_journal_destroy); +EXPORT_SYMBOL(jbd2_journal_abort); +EXPORT_SYMBOL(jbd2_journal_errno); +EXPORT_SYMBOL(jbd2_journal_ack_err); +EXPORT_SYMBOL(jbd2_journal_clear_err); +EXPORT_SYMBOL(jbd2_log_wait_commit); +EXPORT_SYMBOL(jbd2_journal_start_commit); +EXPORT_SYMBOL(jbd2_journal_force_commit_nested); +EXPORT_SYMBOL(jbd2_journal_wipe); +EXPORT_SYMBOL(jbd2_journal_blocks_per_folio); +EXPORT_SYMBOL(jbd2_journal_invalidate_folio); +EXPORT_SYMBOL(jbd2_journal_try_to_free_buffers); +EXPORT_SYMBOL(jbd2_journal_force_commit); +EXPORT_SYMBOL(jbd2_journal_inode_ranged_write); +EXPORT_SYMBOL(jbd2_journal_inode_ranged_wait); +EXPORT_SYMBOL(jbd2_journal_finish_inode_data_buffers); +EXPORT_SYMBOL(jbd2_journal_init_jbd_inode); +EXPORT_SYMBOL(jbd2_journal_release_jbd_inode); +EXPORT_SYMBOL(jbd2_journal_begin_ordered_truncate); +EXPORT_SYMBOL(jbd2_inode_cache); +EXPORT_SYMBOL(jbd2_handle_cache); + +#ifdef CONFIG_JBD2_DEBUG +void __jbd2_debug(int level, const char *file, const char *func, + unsigned int line, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + if (level > jbd2_journal_enable_debug) + return; + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + printk(KERN_DEBUG "%s: (%s, %u): %pV", file, func, line, &vaf); + va_end(args); +} +#endif + +/* Checksumming functions */ +static __be32 jbd2_superblock_csum(journal_superblock_t *sb) +{ + __u32 csum; + __be32 old_csum; + + old_csum = sb->s_checksum; + sb->s_checksum = 0; + csum = jbd2_chksum(~0, (char *)sb, sizeof(journal_superblock_t)); + sb->s_checksum = old_csum; + + return cpu_to_be32(csum); +} + +/* + * Helper function used to manage commit timeouts + */ + +static void commit_timeout(struct timer_list *t) +{ + journal_t *journal = timer_container_of(journal, t, j_commit_timer); + + wake_up_process(journal->j_task); +} + +/* + * kjournald2: The main thread function used to manage a logging device + * journal. + * + * This kernel thread is responsible for two things: + * + * 1) COMMIT: Every so often we need to commit the current state of the + * filesystem to disk. The journal thread is responsible for writing + * all of the metadata buffers to disk. If a fast commit is ongoing + * journal thread waits until it's done and then continues from + * there on. + * + * 2) CHECKPOINT: We cannot reuse a used section of the log file until all + * of the data in that part of the log has been rewritten elsewhere on + * the disk. Flushing these old buffers to reclaim space in the log is + * known as checkpointing, and this thread is responsible for that job. + */ + +static int kjournald2(void *arg) +{ + journal_t *journal = arg; + transaction_t *transaction; + + /* + * Set up an interval timer which can be used to trigger a commit wakeup + * after the commit interval expires + */ + timer_setup(&journal->j_commit_timer, commit_timeout, 0); + + set_freezable(); + + /* Record that the journal thread is running */ + journal->j_task = current; + wake_up(&journal->j_wait_done_commit); + + /* + * Make sure that no allocations from this kernel thread will ever + * recurse to the fs layer because we are responsible for the + * transaction commit and any fs involvement might get stuck waiting for + * the trasn. commit. + */ + memalloc_nofs_save(); + + /* + * And now, wait forever for commit wakeup events. + */ + write_lock(&journal->j_state_lock); + +loop: + if (journal->j_flags & JBD2_UNMOUNT) + goto end_loop; + + jbd2_debug(1, "commit_sequence=%u, commit_request=%u\n", + journal->j_commit_sequence, journal->j_commit_request); + + if (journal->j_commit_sequence != journal->j_commit_request) { + jbd2_debug(1, "OK, requests differ\n"); + write_unlock(&journal->j_state_lock); + timer_delete_sync(&journal->j_commit_timer); + jbd2_journal_commit_transaction(journal); + write_lock(&journal->j_state_lock); + goto loop; + } + + wake_up(&journal->j_wait_done_commit); + if (freezing(current)) { + /* + * The simpler the better. Flushing journal isn't a + * good idea, because that depends on threads that may + * be already stopped. + */ + jbd2_debug(1, "Now suspending kjournald2\n"); + write_unlock(&journal->j_state_lock); + try_to_freeze(); + write_lock(&journal->j_state_lock); + } else { + /* + * We assume on resume that commits are already there, + * so we don't sleep + */ + DEFINE_WAIT(wait); + + prepare_to_wait(&journal->j_wait_commit, &wait, + TASK_INTERRUPTIBLE); + transaction = journal->j_running_transaction; + if (transaction == NULL || + time_before(jiffies, transaction->t_expires)) { + write_unlock(&journal->j_state_lock); + schedule(); + write_lock(&journal->j_state_lock); + } + finish_wait(&journal->j_wait_commit, &wait); + } + + jbd2_debug(1, "kjournald2 wakes\n"); + + /* + * Were we woken up by a commit wakeup event? + */ + transaction = journal->j_running_transaction; + if (transaction && time_after_eq(jiffies, transaction->t_expires)) { + journal->j_commit_request = transaction->t_tid; + jbd2_debug(1, "woke because of timeout\n"); + } + goto loop; + +end_loop: + timer_delete_sync(&journal->j_commit_timer); + journal->j_task = NULL; + wake_up(&journal->j_wait_done_commit); + jbd2_debug(1, "Journal thread exiting.\n"); + write_unlock(&journal->j_state_lock); + return 0; +} + +static int jbd2_journal_start_thread(journal_t *journal) +{ + struct task_struct *t; + + t = kthread_run(kjournald2, journal, "jbd2/%s", + journal->j_devname); + if (IS_ERR(t)) + return PTR_ERR(t); + + wait_event(journal->j_wait_done_commit, journal->j_task != NULL); + return 0; +} + +static void journal_kill_thread(journal_t *journal) +{ + write_lock(&journal->j_state_lock); + journal->j_flags |= JBD2_UNMOUNT; + + while (journal->j_task) { + write_unlock(&journal->j_state_lock); + wake_up(&journal->j_wait_commit); + wait_event(journal->j_wait_done_commit, journal->j_task == NULL); + write_lock(&journal->j_state_lock); + } + write_unlock(&journal->j_state_lock); +} + +static inline bool jbd2_data_needs_escaping(char *data) +{ + return *((__be32 *)data) == cpu_to_be32(JBD2_MAGIC_NUMBER); +} + +static inline void jbd2_data_do_escape(char *data) +{ + *((unsigned int *)data) = 0; +} + +/* + * jbd2_journal_write_metadata_buffer: write a metadata buffer to the journal. + * + * Writes a metadata buffer to a given disk block. The actual IO is not + * performed but a new buffer_head is constructed which labels the data + * to be written with the correct destination disk block. + * + * Any magic-number escaping which needs to be done will cause a + * copy-out here. If the buffer happens to start with the + * JBD2_MAGIC_NUMBER, then we can't write it to the log directly: the + * magic number is only written to the log for descripter blocks. In + * this case, we copy the data and replace the first word with 0, and we + * return a result code which indicates that this buffer needs to be + * marked as an escaped buffer in the corresponding log descriptor + * block. The missing word can then be restored when the block is read + * during recovery. + * + * If the source buffer has already been modified by a new transaction + * since we took the last commit snapshot, we use the frozen copy of + * that data for IO. If we end up using the existing buffer_head's data + * for the write, then we have to make sure nobody modifies it while the + * IO is in progress. do_get_write_access() handles this. + * + * The function returns a pointer to the buffer_head to be used for IO. + * + * + * Return value: + * =0: Finished OK without escape + * =1: Finished OK with escape + */ + +int jbd2_journal_write_metadata_buffer(transaction_t *transaction, + struct journal_head *jh_in, + struct buffer_head **bh_out, + sector_t blocknr) +{ + int do_escape = 0; + struct buffer_head *new_bh; + struct folio *new_folio; + unsigned int new_offset; + struct buffer_head *bh_in = jh2bh(jh_in); + journal_t *journal = transaction->t_journal; + + /* + * The buffer really shouldn't be locked: only the current committing + * transaction is allowed to write it, so nobody else is allowed + * to do any IO. + * + * akpm: except if we're journalling data, and write() output is + * also part of a shared mapping, and another thread has + * decided to launch a writepage() against this buffer. + */ + J_ASSERT_BH(bh_in, buffer_jbddirty(bh_in)); + + new_bh = alloc_buffer_head(GFP_NOFS|__GFP_NOFAIL); + + /* keep subsequent assertions sane */ + atomic_set(&new_bh->b_count, 1); + + spin_lock(&jh_in->b_state_lock); + /* + * If a new transaction has already done a buffer copy-out, then + * we use that version of the data for the commit. + */ + if (jh_in->b_frozen_data) { + new_folio = virt_to_folio(jh_in->b_frozen_data); + new_offset = offset_in_folio(new_folio, jh_in->b_frozen_data); + do_escape = jbd2_data_needs_escaping(jh_in->b_frozen_data); + if (do_escape) + jbd2_data_do_escape(jh_in->b_frozen_data); + } else { + char *tmp; + char *mapped_data; + + new_folio = bh_in->b_folio; + new_offset = offset_in_folio(new_folio, bh_in->b_data); + mapped_data = kmap_local_folio(new_folio, new_offset); + /* + * Fire data frozen trigger if data already wasn't frozen. Do + * this before checking for escaping, as the trigger may modify + * the magic offset. If a copy-out happens afterwards, it will + * have the correct data in the buffer. + */ + jbd2_buffer_frozen_trigger(jh_in, mapped_data, + jh_in->b_triggers); + do_escape = jbd2_data_needs_escaping(mapped_data); + kunmap_local(mapped_data); + /* + * Do we need to do a data copy? + */ + if (!do_escape) + goto escape_done; + + spin_unlock(&jh_in->b_state_lock); + tmp = kmalloc(bh_in->b_size, GFP_NOFS | __GFP_NOFAIL); + spin_lock(&jh_in->b_state_lock); + if (jh_in->b_frozen_data) { + kfree(tmp); + goto copy_done; + } + + jh_in->b_frozen_data = tmp; + memcpy_from_folio(tmp, new_folio, new_offset, bh_in->b_size); + /* + * This isn't strictly necessary, as we're using frozen + * data for the escaping, but it keeps consistency with + * b_frozen_data usage. + */ + jh_in->b_frozen_triggers = jh_in->b_triggers; + +copy_done: + new_folio = virt_to_folio(jh_in->b_frozen_data); + new_offset = offset_in_folio(new_folio, jh_in->b_frozen_data); + jbd2_data_do_escape(jh_in->b_frozen_data); + } + +escape_done: + folio_set_bh(new_bh, new_folio, new_offset); + new_bh->b_size = bh_in->b_size; + new_bh->b_bdev = journal->j_dev; + new_bh->b_blocknr = blocknr; + new_bh->b_private = bh_in; + set_buffer_mapped(new_bh); + set_buffer_dirty(new_bh); + + *bh_out = new_bh; + + /* + * The to-be-written buffer needs to get moved to the io queue, + * and the original buffer whose contents we are shadowing or + * copying is moved to the transaction's shadow queue. + */ + JBUFFER_TRACE(jh_in, "file as BJ_Shadow"); + spin_lock(&journal->j_list_lock); + __jbd2_journal_file_buffer(jh_in, transaction, BJ_Shadow); + spin_unlock(&journal->j_list_lock); + set_buffer_shadow(bh_in); + spin_unlock(&jh_in->b_state_lock); + + return do_escape; +} + +/* + * Allocation code for the journal file. Manage the space left in the + * journal, so that we can begin checkpointing when appropriate. + */ + +/* + * Called with j_state_lock locked for writing. + * Returns true if a transaction commit was started. + */ +static int __jbd2_log_start_commit(journal_t *journal, tid_t target) +{ + /* Return if the txn has already requested to be committed */ + if (journal->j_commit_request == target) + return 0; + + /* + * The only transaction we can possibly wait upon is the + * currently running transaction (if it exists). Otherwise, + * the target tid must be an old one. + */ + if (journal->j_running_transaction && + journal->j_running_transaction->t_tid == target) { + /* + * We want a new commit: OK, mark the request and wakeup the + * commit thread. We do _not_ do the commit ourselves. + */ + + journal->j_commit_request = target; + jbd2_debug(1, "JBD2: requesting commit %u/%u\n", + journal->j_commit_request, + journal->j_commit_sequence); + journal->j_running_transaction->t_requested = jiffies; + wake_up(&journal->j_wait_commit); + return 1; + } else if (!tid_geq(journal->j_commit_request, target)) + /* This should never happen, but if it does, preserve + the evidence before kjournald goes into a loop and + increments j_commit_sequence beyond all recognition. */ + WARN_ONCE(1, "JBD2: bad log_start_commit: %u %u %u %u\n", + journal->j_commit_request, + journal->j_commit_sequence, + target, journal->j_running_transaction ? + journal->j_running_transaction->t_tid : 0); + return 0; +} + +int jbd2_log_start_commit(journal_t *journal, tid_t tid) +{ + int ret; + + write_lock(&journal->j_state_lock); + ret = __jbd2_log_start_commit(journal, tid); + write_unlock(&journal->j_state_lock); + return ret; +} + +/* + * Force and wait any uncommitted transactions. We can only force the running + * transaction if we don't have an active handle, otherwise, we will deadlock. + * Returns: <0 in case of error, + * 0 if nothing to commit, + * 1 if transaction was successfully committed. + */ +static int __jbd2_journal_force_commit(journal_t *journal) +{ + transaction_t *transaction = NULL; + tid_t tid; + int need_to_start = 0, ret = 0; + + read_lock(&journal->j_state_lock); + if (journal->j_running_transaction && !current->journal_info) { + transaction = journal->j_running_transaction; + if (!tid_geq(journal->j_commit_request, transaction->t_tid)) + need_to_start = 1; + } else if (journal->j_committing_transaction) + transaction = journal->j_committing_transaction; + + if (!transaction) { + /* Nothing to commit */ + read_unlock(&journal->j_state_lock); + return 0; + } + tid = transaction->t_tid; + read_unlock(&journal->j_state_lock); + if (need_to_start) + jbd2_log_start_commit(journal, tid); + ret = jbd2_log_wait_commit(journal, tid); + if (!ret) + ret = 1; + + return ret; +} + +/** + * jbd2_journal_force_commit_nested - Force and wait upon a commit if the + * calling process is not within transaction. + * + * @journal: journal to force + * Returns true if progress was made. + * + * This is used for forcing out undo-protected data which contains + * bitmaps, when the fs is running out of space. + */ +int jbd2_journal_force_commit_nested(journal_t *journal) +{ + int ret; + + ret = __jbd2_journal_force_commit(journal); + return ret > 0; +} + +/** + * jbd2_journal_force_commit() - force any uncommitted transactions + * @journal: journal to force + * + * Caller want unconditional commit. We can only force the running transaction + * if we don't have an active handle, otherwise, we will deadlock. + */ +int jbd2_journal_force_commit(journal_t *journal) +{ + int ret; + + J_ASSERT(!current->journal_info); + ret = __jbd2_journal_force_commit(journal); + if (ret > 0) + ret = 0; + return ret; +} + +/* + * Start a commit of the current running transaction (if any). Returns true + * if a transaction is going to be committed (or is currently already + * committing), and fills its tid in at *ptid + */ +int jbd2_journal_start_commit(journal_t *journal, tid_t *ptid) +{ + int ret = 0; + + write_lock(&journal->j_state_lock); + if (journal->j_running_transaction) { + tid_t tid = journal->j_running_transaction->t_tid; + + __jbd2_log_start_commit(journal, tid); + /* There's a running transaction and we've just made sure + * it's commit has been scheduled. */ + if (ptid) + *ptid = tid; + ret = 1; + } else if (journal->j_committing_transaction) { + /* + * If commit has been started, then we have to wait for + * completion of that transaction. + */ + if (ptid) + *ptid = journal->j_committing_transaction->t_tid; + ret = 1; + } + write_unlock(&journal->j_state_lock); + return ret; +} + +/* + * Return 1 if a given transaction has not yet sent barrier request + * connected with a transaction commit. If 0 is returned, transaction + * may or may not have sent the barrier. Used to avoid sending barrier + * twice in common cases. + */ +int jbd2_trans_will_send_data_barrier(journal_t *journal, tid_t tid) +{ + int ret = 0; + transaction_t *commit_trans, *running_trans; + + if (!(journal->j_flags & JBD2_BARRIER)) + return 0; + read_lock(&journal->j_state_lock); + /* Transaction already committed? */ + if (tid_geq(journal->j_commit_sequence, tid)) + goto out; + commit_trans = journal->j_committing_transaction; + if (!commit_trans || commit_trans->t_tid != tid) { + running_trans = journal->j_running_transaction; + /* + * The query transaction hasn't started committing, + * it must still be running. + */ + if (WARN_ON_ONCE(!running_trans || + running_trans->t_tid != tid)) + goto out; + + running_trans->t_need_data_flush = 1; + ret = 1; + goto out; + } + /* + * Transaction is being committed and we already proceeded to + * submitting a flush to fs partition? + */ + if (journal->j_fs_dev != journal->j_dev) { + if (!commit_trans->t_need_data_flush || + commit_trans->t_state >= T_COMMIT_DFLUSH) + goto out; + } else { + if (commit_trans->t_state >= T_COMMIT_JFLUSH) + goto out; + } + ret = 1; +out: + read_unlock(&journal->j_state_lock); + return ret; +} +EXPORT_SYMBOL(jbd2_trans_will_send_data_barrier); + +/* + * Wait for a specified commit to complete. + * The caller may not hold the journal lock. + */ +int jbd2_log_wait_commit(journal_t *journal, tid_t tid) +{ + int err = 0; + + read_lock(&journal->j_state_lock); +#ifdef CONFIG_PROVE_LOCKING + /* + * Some callers make sure transaction is already committing and in that + * case we cannot block on open handles anymore. So don't warn in that + * case. + */ + if (tid_gt(tid, journal->j_commit_sequence) && + (!journal->j_committing_transaction || + journal->j_committing_transaction->t_tid != tid)) { + read_unlock(&journal->j_state_lock); + jbd2_might_wait_for_commit(journal); + read_lock(&journal->j_state_lock); + } +#endif +#ifdef CONFIG_JBD2_DEBUG + if (!tid_geq(journal->j_commit_request, tid)) { + printk(KERN_ERR + "%s: error: j_commit_request=%u, tid=%u\n", + __func__, journal->j_commit_request, tid); + } +#endif + while (tid_gt(tid, journal->j_commit_sequence)) { + jbd2_debug(1, "JBD2: want %u, j_commit_sequence=%u\n", + tid, journal->j_commit_sequence); + read_unlock(&journal->j_state_lock); + wake_up(&journal->j_wait_commit); + wait_event(journal->j_wait_done_commit, + !tid_gt(tid, journal->j_commit_sequence)); + read_lock(&journal->j_state_lock); + } + read_unlock(&journal->j_state_lock); + + if (unlikely(is_journal_aborted(journal))) + err = -EIO; + return err; +} + +/* + * Start a fast commit. If there's an ongoing fast or full commit wait for + * it to complete. Returns 0 if a new fast commit was started. Returns -EALREADY + * if a fast commit is not needed, either because there's an already a commit + * going on or this tid has already been committed. Returns -EINVAL if no jbd2 + * commit has yet been performed. + */ +int jbd2_fc_begin_commit(journal_t *journal, tid_t tid) +{ + if (unlikely(is_journal_aborted(journal))) + return -EIO; + /* + * Fast commits only allowed if at least one full commit has + * been processed. + */ + if (!journal->j_stats.ts_tid) + return -EINVAL; + + write_lock(&journal->j_state_lock); + if (tid_geq(journal->j_commit_sequence, tid)) { + write_unlock(&journal->j_state_lock); + return -EALREADY; + } + + if (journal->j_flags & JBD2_FULL_COMMIT_ONGOING || + (journal->j_flags & JBD2_FAST_COMMIT_ONGOING)) { + DEFINE_WAIT(wait); + + prepare_to_wait(&journal->j_fc_wait, &wait, + TASK_UNINTERRUPTIBLE); + write_unlock(&journal->j_state_lock); + schedule(); + finish_wait(&journal->j_fc_wait, &wait); + return -EALREADY; + } + journal->j_flags |= JBD2_FAST_COMMIT_ONGOING; + write_unlock(&journal->j_state_lock); + + return 0; +} +EXPORT_SYMBOL(jbd2_fc_begin_commit); + +/* + * Stop a fast commit. If fallback is set, this function starts commit of + * TID tid before any other fast commit can start. + */ +static int __jbd2_fc_end_commit(journal_t *journal, tid_t tid, bool fallback) +{ + if (journal->j_fc_cleanup_callback) + journal->j_fc_cleanup_callback(journal, 0, tid); + write_lock(&journal->j_state_lock); + journal->j_flags &= ~JBD2_FAST_COMMIT_ONGOING; + if (fallback) + journal->j_flags |= JBD2_FULL_COMMIT_ONGOING; + write_unlock(&journal->j_state_lock); + wake_up(&journal->j_fc_wait); + if (fallback) + return jbd2_complete_transaction(journal, tid); + return 0; +} + +int jbd2_fc_end_commit(journal_t *journal) +{ + return __jbd2_fc_end_commit(journal, 0, false); +} +EXPORT_SYMBOL(jbd2_fc_end_commit); + +int jbd2_fc_end_commit_fallback(journal_t *journal) +{ + tid_t tid; + + read_lock(&journal->j_state_lock); + tid = journal->j_running_transaction ? + journal->j_running_transaction->t_tid : 0; + read_unlock(&journal->j_state_lock); + return __jbd2_fc_end_commit(journal, tid, true); +} +EXPORT_SYMBOL(jbd2_fc_end_commit_fallback); + +/* Return 1 when transaction with given tid has already committed. */ +int jbd2_transaction_committed(journal_t *journal, tid_t tid) +{ + return tid_geq(READ_ONCE(journal->j_commit_sequence), tid); +} +EXPORT_SYMBOL(jbd2_transaction_committed); + +/* + * When this function returns the transaction corresponding to tid + * will be completed. If the transaction has currently running, start + * committing that transaction before waiting for it to complete. If + * the transaction id is stale, it is by definition already completed, + * so just return SUCCESS. + */ +int jbd2_complete_transaction(journal_t *journal, tid_t tid) +{ + int need_to_wait = 1; + + read_lock(&journal->j_state_lock); + if (journal->j_running_transaction && + journal->j_running_transaction->t_tid == tid) { + if (journal->j_commit_request != tid) { + /* transaction not yet started, so request it */ + read_unlock(&journal->j_state_lock); + jbd2_log_start_commit(journal, tid); + goto wait_commit; + } + } else if (!(journal->j_committing_transaction && + journal->j_committing_transaction->t_tid == tid)) + need_to_wait = 0; + read_unlock(&journal->j_state_lock); + if (!need_to_wait) + return 0; +wait_commit: + return jbd2_log_wait_commit(journal, tid); +} +EXPORT_SYMBOL(jbd2_complete_transaction); + +/* + * Log buffer allocation routines: + */ + +int jbd2_journal_next_log_block(journal_t *journal, unsigned long long *retp) +{ + unsigned long blocknr; + + write_lock(&journal->j_state_lock); + J_ASSERT(journal->j_free > 1); + + blocknr = journal->j_head; + journal->j_head++; + journal->j_free--; + if (journal->j_head == journal->j_last) + journal->j_head = journal->j_first; + write_unlock(&journal->j_state_lock); + return jbd2_journal_bmap(journal, blocknr, retp); +} + +/* Map one fast commit buffer for use by the file system */ +int jbd2_fc_get_buf(journal_t *journal, struct buffer_head **bh_out) +{ + unsigned long long pblock; + unsigned long blocknr; + int ret = 0; + struct buffer_head *bh; + int fc_off; + + *bh_out = NULL; + + if (journal->j_fc_off + journal->j_fc_first >= journal->j_fc_last) + return -EINVAL; + + fc_off = journal->j_fc_off; + blocknr = journal->j_fc_first + fc_off; + journal->j_fc_off++; + ret = jbd2_journal_bmap(journal, blocknr, &pblock); + if (ret) + return ret; + + bh = __getblk(journal->j_dev, pblock, journal->j_blocksize); + if (!bh) + return -ENOMEM; + + journal->j_fc_wbuf[fc_off] = bh; + + *bh_out = bh; + + return 0; +} +EXPORT_SYMBOL(jbd2_fc_get_buf); + +/* + * Wait on fast commit buffers that were allocated by jbd2_fc_get_buf + * for completion. + */ +int jbd2_fc_wait_bufs(journal_t *journal, int num_blks) +{ + struct buffer_head *bh; + int i, j_fc_off; + + j_fc_off = journal->j_fc_off; + + /* + * Wait in reverse order to minimize chances of us being woken up before + * all IOs have completed + */ + for (i = j_fc_off - 1; i >= j_fc_off - num_blks; i--) { + bh = journal->j_fc_wbuf[i]; + wait_on_buffer(bh); + /* + * Update j_fc_off so jbd2_fc_release_bufs can release remain + * buffer head. + */ + if (unlikely(!buffer_uptodate(bh))) { + journal->j_fc_off = i + 1; + return -EIO; + } + put_bh(bh); + journal->j_fc_wbuf[i] = NULL; + } + + return 0; +} +EXPORT_SYMBOL(jbd2_fc_wait_bufs); + +void jbd2_fc_release_bufs(journal_t *journal) +{ + struct buffer_head *bh; + int i, j_fc_off; + + j_fc_off = journal->j_fc_off; + + for (i = j_fc_off - 1; i >= 0; i--) { + bh = journal->j_fc_wbuf[i]; + if (!bh) + break; + put_bh(bh); + journal->j_fc_wbuf[i] = NULL; + } +} +EXPORT_SYMBOL(jbd2_fc_release_bufs); + +/* + * Conversion of logical to physical block numbers for the journal + * + * On external journals the journal blocks are identity-mapped, so + * this is a no-op. If needed, we can use j_blk_offset - everything is + * ready. + */ +int jbd2_journal_bmap(journal_t *journal, unsigned long blocknr, + unsigned long long *retp) +{ + int err = 0; + unsigned long long ret; + sector_t block = blocknr; + + if (journal->j_bmap) { + err = journal->j_bmap(journal, &block); + if (err == 0) + *retp = block; + } else if (journal->j_inode) { + ret = bmap(journal->j_inode, &block); + + if (ret || !block) { + printk(KERN_ALERT "%s: journal block not found " + "at offset %lu on %s\n", + __func__, blocknr, journal->j_devname); + jbd2_journal_abort(journal, ret ? ret : -EFSCORRUPTED); + err = -EIO; + } else { + *retp = block; + } + + } else { + *retp = blocknr; /* +journal->j_blk_offset */ + } + return err; +} + +/* + * We play buffer_head aliasing tricks to write data/metadata blocks to + * the journal without copying their contents, but for journal + * descriptor blocks we do need to generate bona fide buffers. + * + * After the caller of jbd2_journal_get_descriptor_buffer() has finished modifying + * the buffer's contents they really should run flush_dcache_folio(bh->b_folio). + * But we don't bother doing that, so there will be coherency problems with + * mmaps of blockdevs which hold live JBD-controlled filesystems. + */ +struct buffer_head * +jbd2_journal_get_descriptor_buffer(transaction_t *transaction, int type) +{ + journal_t *journal = transaction->t_journal; + struct buffer_head *bh; + unsigned long long blocknr; + journal_header_t *header; + int err; + + err = jbd2_journal_next_log_block(journal, &blocknr); + + if (err) + return NULL; + + bh = __getblk(journal->j_dev, blocknr, journal->j_blocksize); + if (!bh) + return NULL; + atomic_dec(&transaction->t_outstanding_credits); + lock_buffer(bh); + memset(bh->b_data, 0, journal->j_blocksize); + header = (journal_header_t *)bh->b_data; + header->h_magic = cpu_to_be32(JBD2_MAGIC_NUMBER); + header->h_blocktype = cpu_to_be32(type); + header->h_sequence = cpu_to_be32(transaction->t_tid); + set_buffer_uptodate(bh); + unlock_buffer(bh); + BUFFER_TRACE(bh, "return this buffer"); + return bh; +} + +void jbd2_descriptor_block_csum_set(journal_t *j, struct buffer_head *bh) +{ + struct jbd2_journal_block_tail *tail; + __u32 csum; + + if (!jbd2_journal_has_csum_v2or3(j)) + return; + + tail = (struct jbd2_journal_block_tail *)(bh->b_data + j->j_blocksize - + sizeof(struct jbd2_journal_block_tail)); + tail->t_checksum = 0; + csum = jbd2_chksum(j->j_csum_seed, bh->b_data, j->j_blocksize); + tail->t_checksum = cpu_to_be32(csum); +} + +/* + * Return tid of the oldest transaction in the journal and block in the journal + * where the transaction starts. + * + * If the journal is now empty, return which will be the next transaction ID + * we will write and where will that transaction start. + * + * The return value is 0 if journal tail cannot be pushed any further, 1 if + * it can. + */ +int jbd2_journal_get_log_tail(journal_t *journal, tid_t *tid, + unsigned long *block) +{ + transaction_t *transaction; + int ret; + + read_lock(&journal->j_state_lock); + spin_lock(&journal->j_list_lock); + transaction = journal->j_checkpoint_transactions; + if (transaction) { + *tid = transaction->t_tid; + *block = transaction->t_log_start; + } else if ((transaction = journal->j_committing_transaction) != NULL) { + *tid = transaction->t_tid; + *block = transaction->t_log_start; + } else if ((transaction = journal->j_running_transaction) != NULL) { + *tid = transaction->t_tid; + *block = journal->j_head; + } else { + *tid = journal->j_transaction_sequence; + *block = journal->j_head; + } + ret = tid_gt(*tid, journal->j_tail_sequence); + spin_unlock(&journal->j_list_lock); + read_unlock(&journal->j_state_lock); + + return ret; +} + +/* + * Update information in journal structure and in on disk journal superblock + * about log tail. This function does not check whether information passed in + * really pushes log tail further. It's responsibility of the caller to make + * sure provided log tail information is valid (e.g. by holding + * j_checkpoint_mutex all the time between computing log tail and calling this + * function as is the case with jbd2_cleanup_journal_tail()). + * + * Requires j_checkpoint_mutex + */ +int __jbd2_update_log_tail(journal_t *journal, tid_t tid, unsigned long block) +{ + unsigned long freed; + int ret; + + BUG_ON(!mutex_is_locked(&journal->j_checkpoint_mutex)); + + /* + * We cannot afford for write to remain in drive's caches since as + * soon as we update j_tail, next transaction can start reusing journal + * space and if we lose sb update during power failure we'd replay + * old transaction with possibly newly overwritten data. + */ + ret = jbd2_journal_update_sb_log_tail(journal, tid, block, REQ_FUA); + if (ret) + goto out; + + write_lock(&journal->j_state_lock); + freed = block - journal->j_tail; + if (block < journal->j_tail) + freed += journal->j_last - journal->j_first; + + trace_jbd2_update_log_tail(journal, tid, block, freed); + jbd2_debug(1, + "Cleaning journal tail from %u to %u (offset %lu), " + "freeing %lu\n", + journal->j_tail_sequence, tid, block, freed); + + journal->j_free += freed; + journal->j_tail_sequence = tid; + journal->j_tail = block; + write_unlock(&journal->j_state_lock); + +out: + return ret; +} + +/* + * This is a variation of __jbd2_update_log_tail which checks for validity of + * provided log tail and locks j_checkpoint_mutex. So it is safe against races + * with other threads updating log tail. + */ +void jbd2_update_log_tail(journal_t *journal, tid_t tid, unsigned long block) +{ + mutex_lock_io(&journal->j_checkpoint_mutex); + if (tid_gt(tid, journal->j_tail_sequence)) + __jbd2_update_log_tail(journal, tid, block); + mutex_unlock(&journal->j_checkpoint_mutex); +} + +struct jbd2_stats_proc_session { + journal_t *journal; + struct transaction_stats_s *stats; + int start; + int max; +}; + +static void *jbd2_seq_info_start(struct seq_file *seq, loff_t *pos) +{ + return *pos ? NULL : SEQ_START_TOKEN; +} + +static void *jbd2_seq_info_next(struct seq_file *seq, void *v, loff_t *pos) +{ + (*pos)++; + return NULL; +} + +static int jbd2_seq_info_show(struct seq_file *seq, void *v) +{ + struct jbd2_stats_proc_session *s = seq->private; + + if (v != SEQ_START_TOKEN) + return 0; + seq_printf(seq, "%lu transactions (%lu requested), " + "each up to %u blocks\n", + s->stats->ts_tid, s->stats->ts_requested, + s->journal->j_max_transaction_buffers); + if (s->stats->ts_tid == 0) + return 0; + seq_printf(seq, "average: \n %ums waiting for transaction\n", + jiffies_to_msecs(s->stats->run.rs_wait / s->stats->ts_tid)); + seq_printf(seq, " %ums request delay\n", + (s->stats->ts_requested == 0) ? 0 : + jiffies_to_msecs(s->stats->run.rs_request_delay / + s->stats->ts_requested)); + seq_printf(seq, " %ums running transaction\n", + jiffies_to_msecs(s->stats->run.rs_running / s->stats->ts_tid)); + seq_printf(seq, " %ums transaction was being locked\n", + jiffies_to_msecs(s->stats->run.rs_locked / s->stats->ts_tid)); + seq_printf(seq, " %ums flushing data (in ordered mode)\n", + jiffies_to_msecs(s->stats->run.rs_flushing / s->stats->ts_tid)); + seq_printf(seq, " %ums logging transaction\n", + jiffies_to_msecs(s->stats->run.rs_logging / s->stats->ts_tid)); + seq_printf(seq, " %lluus average transaction commit time\n", + div_u64(s->journal->j_average_commit_time, 1000)); + seq_printf(seq, " %lu handles per transaction\n", + s->stats->run.rs_handle_count / s->stats->ts_tid); + seq_printf(seq, " %lu blocks per transaction\n", + s->stats->run.rs_blocks / s->stats->ts_tid); + seq_printf(seq, " %lu logged blocks per transaction\n", + s->stats->run.rs_blocks_logged / s->stats->ts_tid); + return 0; +} + +static void jbd2_seq_info_stop(struct seq_file *seq, void *v) +{ +} + +static const struct seq_operations jbd2_seq_info_ops = { + .start = jbd2_seq_info_start, + .next = jbd2_seq_info_next, + .stop = jbd2_seq_info_stop, + .show = jbd2_seq_info_show, +}; + +static int jbd2_seq_info_open(struct inode *inode, struct file *file) +{ + journal_t *journal = pde_data(inode); + struct jbd2_stats_proc_session *s; + int rc, size; + + s = kmalloc_obj(*s); + if (s == NULL) + return -ENOMEM; + size = sizeof(struct transaction_stats_s); + s->stats = kmalloc(size, GFP_KERNEL); + if (s->stats == NULL) { + kfree(s); + return -ENOMEM; + } + spin_lock(&journal->j_history_lock); + memcpy(s->stats, &journal->j_stats, size); + s->journal = journal; + spin_unlock(&journal->j_history_lock); + + rc = seq_open(file, &jbd2_seq_info_ops); + if (rc == 0) { + struct seq_file *m = file->private_data; + m->private = s; + } else { + kfree(s->stats); + kfree(s); + } + return rc; + +} + +static int jbd2_seq_info_release(struct inode *inode, struct file *file) +{ + struct seq_file *seq = file->private_data; + struct jbd2_stats_proc_session *s = seq->private; + kfree(s->stats); + kfree(s); + return seq_release(inode, file); +} + +static const struct proc_ops jbd2_info_proc_ops = { + .proc_open = jbd2_seq_info_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = jbd2_seq_info_release, +}; + +static struct proc_dir_entry *proc_jbd2_stats; + +static void jbd2_stats_proc_init(journal_t *journal) +{ + journal->j_proc_entry = proc_mkdir(journal->j_devname, proc_jbd2_stats); + if (journal->j_proc_entry) { + proc_create_data("info", S_IRUGO, journal->j_proc_entry, + &jbd2_info_proc_ops, journal); + } +} + +static void jbd2_stats_proc_exit(journal_t *journal) +{ + remove_proc_entry("info", journal->j_proc_entry); + remove_proc_entry(journal->j_devname, proc_jbd2_stats); +} + +/* Minimum size of descriptor tag */ +static int jbd2_min_tag_size(void) +{ + /* + * Tag with 32-bit block numbers does not use last four bytes of the + * structure + */ + return sizeof(journal_block_tag_t) - 4; +} + +/** + * jbd2_journal_shrink_scan() + * @shrink: shrinker to work on + * @sc: reclaim request to process + * + * Scan the checkpointed buffer on the checkpoint list and release the + * journal_head. + */ +static unsigned long jbd2_journal_shrink_scan(struct shrinker *shrink, + struct shrink_control *sc) +{ + journal_t *journal = shrink->private_data; + unsigned long nr_to_scan = sc->nr_to_scan; + unsigned long nr_shrunk; + unsigned long count; + + count = percpu_counter_read_positive(&journal->j_checkpoint_jh_count); + trace_jbd2_shrink_scan_enter(journal, sc->nr_to_scan, count); + + nr_shrunk = jbd2_journal_shrink_checkpoint_list(journal, &nr_to_scan); + + count = percpu_counter_read_positive(&journal->j_checkpoint_jh_count); + trace_jbd2_shrink_scan_exit(journal, nr_to_scan, nr_shrunk, count); + + return nr_shrunk; +} + +/** + * jbd2_journal_shrink_count() + * @shrink: shrinker to work on + * @sc: reclaim request to process + * + * Count the number of checkpoint buffers on the checkpoint list. + */ +static unsigned long jbd2_journal_shrink_count(struct shrinker *shrink, + struct shrink_control *sc) +{ + journal_t *journal = shrink->private_data; + unsigned long count; + + count = percpu_counter_read_positive(&journal->j_checkpoint_jh_count); + trace_jbd2_shrink_count(journal, sc->nr_to_scan, count); + + return count; +} + +/* + * If the journal init or create aborts, we need to mark the journal + * superblock as being NULL to prevent the journal destroy from writing + * back a bogus superblock. + */ +static void journal_fail_superblock(journal_t *journal) +{ + struct buffer_head *bh = journal->j_sb_buffer; + brelse(bh); + journal->j_sb_buffer = NULL; +} + +/* + * Check the superblock for a given journal, performing initial + * validation of the format. + */ +static int journal_check_superblock(journal_t *journal) +{ + journal_superblock_t *sb = journal->j_superblock; + int num_fc_blks; + int err = -EINVAL; + + if (sb->s_header.h_magic != cpu_to_be32(JBD2_MAGIC_NUMBER) || + sb->s_blocksize != cpu_to_be32(journal->j_blocksize)) { + printk(KERN_WARNING "JBD2: no valid journal superblock found\n"); + return err; + } + + if (be32_to_cpu(sb->s_header.h_blocktype) != JBD2_SUPERBLOCK_V1 && + be32_to_cpu(sb->s_header.h_blocktype) != JBD2_SUPERBLOCK_V2) { + printk(KERN_WARNING "JBD2: unrecognised superblock format ID\n"); + return err; + } + + if (be32_to_cpu(sb->s_maxlen) > journal->j_total_len) { + printk(KERN_WARNING "JBD2: journal file too short\n"); + return err; + } + + if (be32_to_cpu(sb->s_first) == 0 || + be32_to_cpu(sb->s_first) >= journal->j_total_len) { + printk(KERN_WARNING + "JBD2: Invalid start block of journal: %u\n", + be32_to_cpu(sb->s_first)); + return err; + } + + /* + * If this is a V2 superblock, then we have to check the + * features flags on it. + */ + if (!jbd2_format_support_feature(journal)) + return 0; + + if ((sb->s_feature_ro_compat & + ~cpu_to_be32(JBD2_KNOWN_ROCOMPAT_FEATURES)) || + (sb->s_feature_incompat & + ~cpu_to_be32(JBD2_KNOWN_INCOMPAT_FEATURES))) { + printk(KERN_WARNING "JBD2: Unrecognised features on journal\n"); + return err; + } + + num_fc_blks = jbd2_has_feature_fast_commit(journal) ? + jbd2_journal_get_num_fc_blks(sb) : 0; + if (be32_to_cpu(sb->s_maxlen) < JBD2_MIN_JOURNAL_BLOCKS || + be32_to_cpu(sb->s_maxlen) - JBD2_MIN_JOURNAL_BLOCKS < num_fc_blks) { + printk(KERN_ERR "JBD2: journal file too short %u,%d\n", + be32_to_cpu(sb->s_maxlen), num_fc_blks); + return err; + } + + if (jbd2_has_feature_csum2(journal) && + jbd2_has_feature_csum3(journal)) { + /* Can't have checksum v2 and v3 at the same time! */ + printk(KERN_ERR "JBD2: Can't enable checksumming v2 and v3 " + "at the same time!\n"); + return err; + } + + if (jbd2_journal_has_csum_v2or3(journal) && + jbd2_has_feature_checksum(journal)) { + /* Can't have checksum v1 and v2 on at the same time! */ + printk(KERN_ERR "JBD2: Can't enable checksumming v1 and v2/3 " + "at the same time!\n"); + return err; + } + + if (jbd2_journal_has_csum_v2or3(journal)) { + if (sb->s_checksum_type != JBD2_CRC32C_CHKSUM) { + printk(KERN_ERR "JBD2: Unknown checksum type\n"); + return err; + } + + /* Check superblock checksum */ + if (sb->s_checksum != jbd2_superblock_csum(sb)) { + printk(KERN_ERR "JBD2: journal checksum error\n"); + err = -EFSBADCRC; + return err; + } + } + + return 0; +} + +static int journal_revoke_records_per_block(journal_t *journal) +{ + int record_size; + int space = journal->j_blocksize - sizeof(jbd2_journal_revoke_header_t); + + if (jbd2_has_feature_64bit(journal)) + record_size = 8; + else + record_size = 4; + + if (jbd2_journal_has_csum_v2or3(journal)) + space -= sizeof(struct jbd2_journal_block_tail); + return space / record_size; +} + +static int jbd2_journal_get_max_txn_bufs(journal_t *journal) +{ + return (journal->j_total_len - journal->j_fc_wbufsize) / 3; +} + +/* + * Base amount of descriptor blocks we reserve for each transaction. + */ +static int jbd2_descriptor_blocks_per_trans(journal_t *journal) +{ + int tag_space = journal->j_blocksize - sizeof(journal_header_t); + int tags_per_block; + + /* Subtract UUID */ + tag_space -= 16; + if (jbd2_journal_has_csum_v2or3(journal)) + tag_space -= sizeof(struct jbd2_journal_block_tail); + /* Commit code leaves a slack space of 16 bytes at the end of block */ + tags_per_block = (tag_space - 16) / journal_tag_bytes(journal); + /* + * Revoke descriptors are accounted separately so we need to reserve + * space for commit block and normal transaction descriptor blocks. + */ + return 1 + DIV_ROUND_UP(jbd2_journal_get_max_txn_bufs(journal), + tags_per_block); +} + +/* + * Initialize number of blocks each transaction reserves for its bookkeeping + * and maximum number of blocks a transaction can use. This needs to be called + * after the journal size and the fastcommit area size are initialized. + */ +static void jbd2_journal_init_transaction_limits(journal_t *journal) +{ + journal->j_revoke_records_per_block = + journal_revoke_records_per_block(journal); + journal->j_transaction_overhead_buffers = + jbd2_descriptor_blocks_per_trans(journal); + journal->j_max_transaction_buffers = + jbd2_journal_get_max_txn_bufs(journal); +} + +/* + * Load the on-disk journal superblock and read the key fields into the + * journal_t. + */ +static int journal_load_superblock(journal_t *journal) +{ + int err; + struct buffer_head *bh; + journal_superblock_t *sb; + + bh = getblk_unmovable(journal->j_dev, journal->j_blk_offset, + journal->j_blocksize); + if (bh) + err = bh_read(bh, 0); + if (!bh || err < 0) { + pr_err("%s: Cannot read journal superblock\n", __func__); + brelse(bh); + return -EIO; + } + + journal->j_sb_buffer = bh; + sb = (journal_superblock_t *)bh->b_data; + journal->j_superblock = sb; + err = journal_check_superblock(journal); + if (err) { + journal_fail_superblock(journal); + return err; + } + + journal->j_tail_sequence = be32_to_cpu(sb->s_sequence); + journal->j_tail = be32_to_cpu(sb->s_start); + journal->j_first = be32_to_cpu(sb->s_first); + journal->j_errno = be32_to_cpu(sb->s_errno); + journal->j_last = be32_to_cpu(sb->s_maxlen); + + if (be32_to_cpu(sb->s_maxlen) < journal->j_total_len) + journal->j_total_len = be32_to_cpu(sb->s_maxlen); + /* Precompute checksum seed for all metadata */ + if (jbd2_journal_has_csum_v2or3(journal)) + journal->j_csum_seed = jbd2_chksum(~0, sb->s_uuid, + sizeof(sb->s_uuid)); + /* After journal features are set, we can compute transaction limits */ + jbd2_journal_init_transaction_limits(journal); + + if (jbd2_has_feature_fast_commit(journal)) { + journal->j_fc_last = be32_to_cpu(sb->s_maxlen); + journal->j_last = journal->j_fc_last - + jbd2_journal_get_num_fc_blks(sb); + journal->j_fc_first = journal->j_last + 1; + journal->j_fc_off = 0; + } + + return 0; +} + + +/* + * Management for journal control blocks: functions to create and + * destroy journal_t structures, and to initialise and read existing + * journal blocks from disk. */ + +/* The journal_init_common() function creates and fills a journal_t object + * in memory. It calls journal_load_superblock() to load the on-disk journal + * superblock and initialize the journal_t object. + */ + +static journal_t *journal_init_common(struct block_device *bdev, + struct block_device *fs_dev, + unsigned long long start, int len, int blocksize) +{ + journal_t *journal; + int err; + int n; + + journal = kzalloc_obj(*journal); + if (!journal) + return ERR_PTR(-ENOMEM); + + lockdep_register_key(&journal->jbd2_trans_commit_key); + journal->j_blocksize = blocksize; + journal->j_dev = bdev; + journal->j_fs_dev = fs_dev; + journal->j_blk_offset = start; + journal->j_total_len = len; + jbd2_init_fs_dev_write_error(journal); + + err = journal_load_superblock(journal); + if (err) + goto err_cleanup; + + init_waitqueue_head(&journal->j_wait_transaction_locked); + init_waitqueue_head(&journal->j_wait_done_commit); + init_waitqueue_head(&journal->j_wait_commit); + init_waitqueue_head(&journal->j_wait_updates); + init_waitqueue_head(&journal->j_wait_reserved); + init_waitqueue_head(&journal->j_fc_wait); + mutex_init(&journal->j_abort_mutex); + mutex_init(&journal->j_barrier); + mutex_init(&journal->j_checkpoint_mutex); + spin_lock_init(&journal->j_revoke_lock); + spin_lock_init(&journal->j_list_lock); + spin_lock_init(&journal->j_history_lock); + rwlock_init(&journal->j_state_lock); + + journal->j_commit_interval = (HZ * JBD2_DEFAULT_MAX_COMMIT_AGE); + journal->j_min_batch_time = 0; + journal->j_max_batch_time = 15000; /* 15ms */ + atomic_set(&journal->j_reserved_credits, 0); + lockdep_init_map(&journal->j_trans_commit_map, "jbd2_handle", + &journal->jbd2_trans_commit_key, 0); + + /* The journal is marked for error until we succeed with recovery! */ + journal->j_flags = JBD2_ABORT; + + /* Set up a default-sized revoke table for the new mount. */ + err = jbd2_journal_init_revoke(journal, JOURNAL_REVOKE_DEFAULT_HASH); + if (err) + goto err_cleanup; + + /* + * journal descriptor can store up to n blocks, we need enough + * buffers to write out full descriptor block. + */ + err = -ENOMEM; + n = journal->j_blocksize / jbd2_min_tag_size(); + journal->j_wbufsize = n; + journal->j_fc_wbuf = NULL; + journal->j_wbuf = kmalloc_objs(struct buffer_head *, n); + if (!journal->j_wbuf) + goto err_cleanup; + + err = percpu_counter_init(&journal->j_checkpoint_jh_count, 0, + GFP_KERNEL); + if (err) + goto err_cleanup; + + journal->j_shrink_transaction = NULL; + + journal->j_shrinker = shrinker_alloc(0, "jbd2-journal:(%u:%u)", + MAJOR(bdev->bd_dev), + MINOR(bdev->bd_dev)); + if (!journal->j_shrinker) { + err = -ENOMEM; + goto err_cleanup; + } + + journal->j_shrinker->scan_objects = jbd2_journal_shrink_scan; + journal->j_shrinker->count_objects = jbd2_journal_shrink_count; + journal->j_shrinker->private_data = journal; + + shrinker_register(journal->j_shrinker); + + return journal; + +err_cleanup: + percpu_counter_destroy(&journal->j_checkpoint_jh_count); + kfree(journal->j_wbuf); + jbd2_journal_destroy_revoke(journal); + journal_fail_superblock(journal); + lockdep_unregister_key(&journal->jbd2_trans_commit_key); + kfree(journal); + return ERR_PTR(err); +} + +/* jbd2_journal_init_dev and jbd2_journal_init_inode: + * + * Create a journal structure assigned some fixed set of disk blocks to + * the journal. We don't actually touch those disk blocks yet, but we + * need to set up all of the mapping information to tell the journaling + * system where the journal blocks are. + * + */ + +/** + * journal_t * jbd2_journal_init_dev() - creates and initialises a journal structure + * @bdev: Block device on which to create the journal + * @fs_dev: Device which hold journalled filesystem for this journal. + * @start: Block nr Start of journal. + * @len: Length of the journal in blocks. + * @blocksize: blocksize of journalling device + * + * Returns: a newly created journal_t * + * + * jbd2_journal_init_dev creates a journal which maps a fixed contiguous + * range of blocks on an arbitrary block device. + * + */ +journal_t *jbd2_journal_init_dev(struct block_device *bdev, + struct block_device *fs_dev, + unsigned long long start, int len, int blocksize) +{ + journal_t *journal; + + journal = journal_init_common(bdev, fs_dev, start, len, blocksize); + if (IS_ERR(journal)) + return ERR_CAST(journal); + + snprintf(journal->j_devname, sizeof(journal->j_devname), + "%pg", journal->j_dev); + strreplace(journal->j_devname, '/', '!'); + jbd2_stats_proc_init(journal); + + return journal; +} + +/** + * journal_t * jbd2_journal_init_inode () - creates a journal which maps to a inode. + * @inode: An inode to create the journal in + * + * jbd2_journal_init_inode creates a journal which maps an on-disk inode as + * the journal. The inode must exist already, must support bmap() and + * must have all data blocks preallocated. + */ +journal_t *jbd2_journal_init_inode(struct inode *inode) +{ + journal_t *journal; + sector_t blocknr; + int err = 0; + + blocknr = 0; + err = bmap(inode, &blocknr); + if (err || !blocknr) { + pr_err("%s: Cannot locate journal superblock\n", __func__); + return err ? ERR_PTR(err) : ERR_PTR(-EINVAL); + } + + jbd2_debug(1, "JBD2: inode %s/%llu, size %lld, bits %d, blksize %ld\n", + inode->i_sb->s_id, inode->i_ino, (long long) inode->i_size, + inode->i_sb->s_blocksize_bits, inode->i_sb->s_blocksize); + + journal = journal_init_common(inode->i_sb->s_bdev, inode->i_sb->s_bdev, + blocknr, inode->i_size >> inode->i_sb->s_blocksize_bits, + inode->i_sb->s_blocksize); + if (IS_ERR(journal)) + return ERR_CAST(journal); + + journal->j_inode = inode; + snprintf(journal->j_devname, sizeof(journal->j_devname), + "%pg-%llu", journal->j_dev, journal->j_inode->i_ino); + strreplace(journal->j_devname, '/', '!'); + jbd2_stats_proc_init(journal); + + return journal; +} + +/* + * Given a journal_t structure, initialise the various fields for + * startup of a new journaling session. We use this both when creating + * a journal, and after recovering an old journal to reset it for + * subsequent use. + */ + +static int journal_reset(journal_t *journal) +{ + journal_superblock_t *sb = journal->j_superblock; + unsigned long long first, last; + + first = be32_to_cpu(sb->s_first); + last = be32_to_cpu(sb->s_maxlen); + if (first + JBD2_MIN_JOURNAL_BLOCKS > last + 1) { + printk(KERN_ERR "JBD2: Journal too short (blocks %llu-%llu).\n", + first, last); + journal_fail_superblock(journal); + return -EINVAL; + } + + journal->j_first = first; + journal->j_last = last; + + if (journal->j_head != 0 && journal->j_flags & JBD2_CYCLE_RECORD) { + /* + * Disable the cycled recording mode if the journal head block + * number is not correct. + */ + if (journal->j_head < first || journal->j_head >= last) { + printk(KERN_WARNING "JBD2: Incorrect Journal head block %lu, " + "disable journal_cycle_record\n", + journal->j_head); + journal->j_head = journal->j_first; + } + } else { + journal->j_head = journal->j_first; + } + journal->j_tail = journal->j_head; + journal->j_free = journal->j_last - journal->j_first; + + journal->j_tail_sequence = journal->j_transaction_sequence; + journal->j_commit_sequence = journal->j_transaction_sequence - 1; + journal->j_commit_request = journal->j_commit_sequence; + + /* + * Now that journal recovery is done, turn fast commits off here. This + * way, if fast commit was enabled before the crash but if now FS has + * disabled it, we don't enable fast commits. + */ + jbd2_clear_feature_fast_commit(journal); + + /* + * As a special case, if the on-disk copy is already marked as needing + * no recovery (s_start == 0), then we can safely defer the superblock + * update until the next commit by setting JBD2_FLUSHED. This avoids + * attempting a write to a potential-readonly device. + */ + if (sb->s_start == 0) { + jbd2_debug(1, "JBD2: Skipping superblock update on recovered sb " + "(start %ld, seq %u, errno %d)\n", + journal->j_tail, journal->j_tail_sequence, + journal->j_errno); + journal->j_flags |= JBD2_FLUSHED; + } else { + /* Lock here to make assertions happy... */ + mutex_lock_io(&journal->j_checkpoint_mutex); + /* + * Update log tail information. We use REQ_FUA since new + * transaction will start reusing journal space and so we + * must make sure information about current log tail is on + * disk before that. + */ + jbd2_journal_update_sb_log_tail(journal, + journal->j_tail_sequence, + journal->j_tail, REQ_FUA); + mutex_unlock(&journal->j_checkpoint_mutex); + } + return jbd2_journal_start_thread(journal); +} + +/* + * This function expects that the caller will have locked the journal + * buffer head, and will return with it unlocked + */ +static int jbd2_write_superblock(journal_t *journal, blk_opf_t write_flags) +{ + struct buffer_head *bh = journal->j_sb_buffer; + journal_superblock_t *sb = journal->j_superblock; + int ret = 0; + + /* Buffer got discarded which means block device got invalidated */ + if (!buffer_mapped(bh)) { + unlock_buffer(bh); + return -EIO; + } + + /* + * Always set high priority flags to exempt from block layer's + * QOS policies, e.g. writeback throttle. + */ + write_flags |= JBD2_JOURNAL_REQ_FLAGS; + if (!(journal->j_flags & JBD2_BARRIER)) + write_flags &= ~(REQ_FUA | REQ_PREFLUSH); + + trace_jbd2_write_superblock(journal, write_flags); + + if (buffer_write_io_error(bh)) { + /* + * Oh, dear. A previous attempt to write the journal + * superblock failed. This could happen because the + * USB device was yanked out. Or it could happen to + * be a transient write error and maybe the block will + * be remapped. Nothing we can do but to retry the + * write and hope for the best. + */ + printk(KERN_ERR "JBD2: previous I/O error detected " + "for journal superblock update for %s.\n", + journal->j_devname); + clear_buffer_write_io_error(bh); + set_buffer_uptodate(bh); + } + if (jbd2_journal_has_csum_v2or3(journal)) + sb->s_checksum = jbd2_superblock_csum(sb); + bh_submit(bh, REQ_OP_WRITE | write_flags, bh_end_write); + wait_on_buffer(bh); + if (buffer_write_io_error(bh)) { + clear_buffer_write_io_error(bh); + set_buffer_uptodate(bh); + ret = -EIO; + } + if (ret) { + printk(KERN_ERR "JBD2: I/O error when updating journal superblock for %s.\n", + journal->j_devname); + if (!is_journal_aborted(journal)) + jbd2_journal_abort(journal, ret); + } + + return ret; +} + +/** + * jbd2_journal_update_sb_log_tail() - Update log tail in journal sb on disk. + * @journal: The journal to update. + * @tail_tid: TID of the new transaction at the tail of the log + * @tail_block: The first block of the transaction at the tail of the log + * @write_flags: Flags for the journal sb write operation + * + * Update a journal's superblock information about log tail and write it to + * disk, waiting for the IO to complete. + */ +int jbd2_journal_update_sb_log_tail(journal_t *journal, tid_t tail_tid, + unsigned long tail_block, + blk_opf_t write_flags) +{ + journal_superblock_t *sb = journal->j_superblock; + int ret; + + if (is_journal_aborted(journal)) + return -EIO; + ret = jbd2_check_fs_dev_write_error(journal); + if (ret) { + jbd2_journal_abort(journal, ret); + return -EIO; + } + + BUG_ON(!mutex_is_locked(&journal->j_checkpoint_mutex)); + jbd2_debug(1, "JBD2: updating superblock (start %lu, seq %u)\n", + tail_block, tail_tid); + + lock_buffer(journal->j_sb_buffer); + sb->s_sequence = cpu_to_be32(tail_tid); + sb->s_start = cpu_to_be32(tail_block); + + ret = jbd2_write_superblock(journal, write_flags); + if (ret) + goto out; + + /* Log is no longer empty */ + write_lock(&journal->j_state_lock); + journal->j_flags &= ~JBD2_FLUSHED; + write_unlock(&journal->j_state_lock); + +out: + return ret; +} + +/** + * jbd2_mark_journal_empty() - Mark on disk journal as empty. + * @journal: The journal to update. + * @write_flags: Flags for the journal sb write operation + * + * Update a journal's dynamic superblock fields to show that journal is empty. + * Write updated superblock to disk waiting for IO to complete. + */ +static void jbd2_mark_journal_empty(journal_t *journal, blk_opf_t write_flags) +{ + journal_superblock_t *sb = journal->j_superblock; + bool had_fast_commit = false; + + BUG_ON(!mutex_is_locked(&journal->j_checkpoint_mutex)); + lock_buffer(journal->j_sb_buffer); + if (sb->s_start == 0) { /* Is it already empty? */ + unlock_buffer(journal->j_sb_buffer); + return; + } + + jbd2_debug(1, "JBD2: Marking journal as empty (seq %u)\n", + journal->j_tail_sequence); + + sb->s_sequence = cpu_to_be32(journal->j_tail_sequence); + sb->s_start = cpu_to_be32(0); + sb->s_head = cpu_to_be32(journal->j_head); + if (jbd2_has_feature_fast_commit(journal)) { + /* + * When journal is clean, no need to commit fast commit flag and + * make file system incompatible with older kernels. + */ + jbd2_clear_feature_fast_commit(journal); + had_fast_commit = true; + } + + jbd2_write_superblock(journal, write_flags); + + if (had_fast_commit) + jbd2_set_feature_fast_commit(journal); + + /* Log is empty */ + write_lock(&journal->j_state_lock); + journal->j_flags |= JBD2_FLUSHED; + write_unlock(&journal->j_state_lock); +} + +/** + * __jbd2_journal_erase() - Discard or zeroout journal blocks (excluding superblock) + * @journal: The journal to erase. + * @flags: A discard/zeroout request is sent for each physically contigous + * region of the journal. Either JBD2_JOURNAL_FLUSH_DISCARD or + * JBD2_JOURNAL_FLUSH_ZEROOUT must be set to determine which operation + * to perform. + * + * Note: JBD2_JOURNAL_FLUSH_ZEROOUT attempts to use hardware offload. Zeroes + * will be explicitly written if no hardware offload is available, see + * blkdev_issue_zeroout for more details. + */ +static int __jbd2_journal_erase(journal_t *journal, unsigned int flags) +{ + int err = 0; + unsigned long block, log_offset; /* logical */ + unsigned long long phys_block, block_start, block_stop; /* physical */ + loff_t byte_start, byte_stop, byte_count; + + /* flags must be set to either discard or zeroout */ + if ((flags & ~JBD2_JOURNAL_FLUSH_VALID) || !flags || + ((flags & JBD2_JOURNAL_FLUSH_DISCARD) && + (flags & JBD2_JOURNAL_FLUSH_ZEROOUT))) + return -EINVAL; + + if ((flags & JBD2_JOURNAL_FLUSH_DISCARD) && + !bdev_max_discard_sectors(journal->j_dev)) + return -EOPNOTSUPP; + + /* + * lookup block mapping and issue discard/zeroout for each + * contiguous region + */ + log_offset = be32_to_cpu(journal->j_superblock->s_first); + block_start = ~0ULL; + for (block = log_offset; block < journal->j_total_len; block++) { + err = jbd2_journal_bmap(journal, block, &phys_block); + if (err) { + pr_err("JBD2: bad block at offset %lu", block); + return err; + } + + if (block_start == ~0ULL) + block_stop = block_start = phys_block; + + /* + * last block not contiguous with current block, + * process last contiguous region and return to this block on + * next loop + */ + if (phys_block != block_stop) { + block--; + } else { + block_stop++; + /* + * if this isn't the last block of journal, + * no need to process now because next block may also + * be part of this contiguous region + */ + if (block != journal->j_total_len - 1) + continue; + } + + /* + * end of contiguous region or this is last block of journal, + * take care of the region + */ + byte_start = block_start * journal->j_blocksize; + byte_stop = block_stop * journal->j_blocksize; + byte_count = (block_stop - block_start) * journal->j_blocksize; + + truncate_inode_pages_range(journal->j_dev->bd_mapping, + byte_start, byte_stop - 1); + + if (flags & JBD2_JOURNAL_FLUSH_DISCARD) { + err = blkdev_issue_discard(journal->j_dev, + byte_start >> SECTOR_SHIFT, + byte_count >> SECTOR_SHIFT, + GFP_NOFS); + } else if (flags & JBD2_JOURNAL_FLUSH_ZEROOUT) { + err = blkdev_issue_zeroout(journal->j_dev, + byte_start >> SECTOR_SHIFT, + byte_count >> SECTOR_SHIFT, + GFP_NOFS, 0); + } + + if (unlikely(err != 0)) { + pr_err("JBD2: (error %d) unable to wipe journal at physical blocks [%llu, %llu)", + err, block_start, block_stop); + return err; + } + + /* reset start and stop after processing a region */ + block_start = ~0ULL; + } + + return blkdev_issue_flush(journal->j_dev); +} + +/** + * jbd2_journal_update_sb_errno() - Update error in the journal. + * @journal: The journal to update. + * + * Update a journal's errno. Write updated superblock to disk waiting for IO + * to complete. + */ +void jbd2_journal_update_sb_errno(journal_t *journal) +{ + journal_superblock_t *sb = journal->j_superblock; + int errcode; + + lock_buffer(journal->j_sb_buffer); + errcode = journal->j_errno; + if (errcode == -ESHUTDOWN) + errcode = 0; + jbd2_debug(1, "JBD2: updating superblock error (errno %d)\n", errcode); + sb->s_errno = cpu_to_be32(errcode); + + jbd2_write_superblock(journal, REQ_FUA); +} +EXPORT_SYMBOL(jbd2_journal_update_sb_errno); + +/** + * jbd2_journal_load() - Read journal from disk. + * @journal: Journal to act on. + * + * Given a journal_t structure which tells us which disk blocks contain + * a journal, read the journal from disk to initialise the in-memory + * structures. + */ +int jbd2_journal_load(journal_t *journal) +{ + int err; + + /* Let the recovery code check whether it needs to recover any + * data from the journal. */ + err = jbd2_journal_recover(journal); + if (err) { + pr_warn("JBD2: journal recovery failed\n"); + return err; + } + + if (journal->j_failed_commit) { + printk(KERN_ERR "JBD2: journal transaction %u on %s " + "is corrupt.\n", journal->j_failed_commit, + journal->j_devname); + return -EFSCORRUPTED; + } + /* + * clear JBD2_ABORT flag initialized in journal_init_common + * here to update log tail information with the newest seq. + */ + journal->j_flags &= ~JBD2_ABORT; + + /* OK, we've finished with the dynamic journal bits: + * reinitialise the dynamic contents of the superblock in memory + * and reset them on disk. */ + err = journal_reset(journal); + if (err) { + pr_warn("JBD2: journal reset failed\n"); + return err; + } + + journal->j_flags |= JBD2_LOADED; + return 0; +} + +/** + * jbd2_journal_destroy() - Release a journal_t structure. + * @journal: Journal to act on. + * + * Release a journal_t structure once it is no longer in use by the + * journaled object. + * Return <0 if we couldn't clean up the journal. + */ +int jbd2_journal_destroy(journal_t *journal) +{ + int err = 0; + + /* Wait for the commit thread to wake up and die. */ + journal_kill_thread(journal); + + /* Force a final log commit */ + if (journal->j_running_transaction) + jbd2_journal_commit_transaction(journal); + + /* Force any old transactions to disk */ + + /* Totally anal locking here... */ + spin_lock(&journal->j_list_lock); + while (journal->j_checkpoint_transactions != NULL) { + spin_unlock(&journal->j_list_lock); + mutex_lock_io(&journal->j_checkpoint_mutex); + err = jbd2_log_do_checkpoint(journal); + mutex_unlock(&journal->j_checkpoint_mutex); + /* + * If checkpointing failed, just free the buffers to avoid + * looping forever + */ + if (err) { + jbd2_journal_destroy_checkpoint(journal); + spin_lock(&journal->j_list_lock); + break; + } + spin_lock(&journal->j_list_lock); + } + + J_ASSERT(journal->j_running_transaction == NULL); + J_ASSERT(journal->j_committing_transaction == NULL); + J_ASSERT(journal->j_checkpoint_transactions == NULL); + spin_unlock(&journal->j_list_lock); + + /* + * OK, all checkpoint transactions have been checked, now check the + * writeback errseq of fs dev and abort the journal if some buffer + * failed to write back to the original location, otherwise the + * filesystem may become inconsistent. + */ + if (!is_journal_aborted(journal)) { + int ret = jbd2_check_fs_dev_write_error(journal); + if (ret) + jbd2_journal_abort(journal, ret); + } + + if (journal->j_sb_buffer) { + if (!is_journal_aborted(journal)) { + mutex_lock_io(&journal->j_checkpoint_mutex); + + write_lock(&journal->j_state_lock); + journal->j_tail_sequence = + ++journal->j_transaction_sequence; + write_unlock(&journal->j_state_lock); + + jbd2_mark_journal_empty(journal, REQ_PREFLUSH | REQ_FUA); + mutex_unlock(&journal->j_checkpoint_mutex); + } else + err = -EIO; + brelse(journal->j_sb_buffer); + } + + if (journal->j_shrinker) { + percpu_counter_destroy(&journal->j_checkpoint_jh_count); + shrinker_free(journal->j_shrinker); + } + if (journal->j_proc_entry) + jbd2_stats_proc_exit(journal); + iput(journal->j_inode); + if (journal->j_revoke) + jbd2_journal_destroy_revoke(journal); + kfree(journal->j_fc_wbuf); + kfree(journal->j_wbuf); + lockdep_unregister_key(&journal->jbd2_trans_commit_key); + kfree(journal); + + return err; +} + + +/** + * jbd2_journal_check_used_features() - Check if features specified are used. + * @journal: Journal to check. + * @compat: bitmask of compatible features + * @ro: bitmask of features that force read-only mount + * @incompat: bitmask of incompatible features + * + * Check whether the journal uses all of a given set of + * features. Return true (non-zero) if it does. + **/ + +int jbd2_journal_check_used_features(journal_t *journal, unsigned long compat, + unsigned long ro, unsigned long incompat) +{ + journal_superblock_t *sb; + + if (!compat && !ro && !incompat) + return 1; + if (!jbd2_format_support_feature(journal)) + return 0; + + sb = journal->j_superblock; + + if (((be32_to_cpu(sb->s_feature_compat) & compat) == compat) && + ((be32_to_cpu(sb->s_feature_ro_compat) & ro) == ro) && + ((be32_to_cpu(sb->s_feature_incompat) & incompat) == incompat)) + return 1; + + return 0; +} + +/** + * jbd2_journal_check_available_features() - Check feature set in journalling layer + * @journal: Journal to check. + * @compat: bitmask of compatible features + * @ro: bitmask of features that force read-only mount + * @incompat: bitmask of incompatible features + * + * Check whether the journaling code supports the use of + * all of a given set of features on this journal. Return true + * (non-zero) if it can. */ + +int jbd2_journal_check_available_features(journal_t *journal, unsigned long compat, + unsigned long ro, unsigned long incompat) +{ + if (!compat && !ro && !incompat) + return 1; + + if (!jbd2_format_support_feature(journal)) + return 0; + + if ((compat & JBD2_KNOWN_COMPAT_FEATURES) == compat && + (ro & JBD2_KNOWN_ROCOMPAT_FEATURES) == ro && + (incompat & JBD2_KNOWN_INCOMPAT_FEATURES) == incompat) + return 1; + + return 0; +} + +static int +jbd2_journal_initialize_fast_commit(journal_t *journal) +{ + journal_superblock_t *sb = journal->j_superblock; + unsigned long long num_fc_blks; + + num_fc_blks = jbd2_journal_get_num_fc_blks(sb); + if (num_fc_blks > journal->j_last) + return -EFSCORRUPTED; + if (journal->j_last - num_fc_blks < JBD2_MIN_JOURNAL_BLOCKS) + return -ENOSPC; + + /* Are we called twice? */ + WARN_ON(journal->j_fc_wbuf != NULL); + journal->j_fc_wbuf = kmalloc_objs(struct buffer_head *, num_fc_blks); + if (!journal->j_fc_wbuf) + return -ENOMEM; + + journal->j_fc_wbufsize = num_fc_blks; + journal->j_fc_last = journal->j_last; + journal->j_last = journal->j_fc_last - num_fc_blks; + journal->j_fc_first = journal->j_last + 1; + journal->j_fc_off = 0; + journal->j_free = journal->j_last - journal->j_first; + + return 0; +} + +/** + * jbd2_journal_set_features() - Mark a given journal feature in the superblock + * @journal: Journal to act on. + * @compat: bitmask of compatible features + * @ro: bitmask of features that force read-only mount + * @incompat: bitmask of incompatible features + * + * Mark a given journal feature as present on the + * superblock. Returns true if the requested features could be set. + * + */ + +int jbd2_journal_set_features(journal_t *journal, unsigned long compat, + unsigned long ro, unsigned long incompat) +{ +#define INCOMPAT_FEATURE_ON(f) \ + ((incompat & (f)) && !(sb->s_feature_incompat & cpu_to_be32(f))) +#define COMPAT_FEATURE_ON(f) \ + ((compat & (f)) && !(sb->s_feature_compat & cpu_to_be32(f))) + journal_superblock_t *sb; + + if (jbd2_journal_check_used_features(journal, compat, ro, incompat)) + return 1; + + if (!jbd2_journal_check_available_features(journal, compat, ro, incompat)) + return 0; + + /* If enabling v2 checksums, turn on v3 instead */ + if (incompat & JBD2_FEATURE_INCOMPAT_CSUM_V2) { + incompat &= ~JBD2_FEATURE_INCOMPAT_CSUM_V2; + incompat |= JBD2_FEATURE_INCOMPAT_CSUM_V3; + } + + /* Asking for checksumming v3 and v1? Only give them v3. */ + if (incompat & JBD2_FEATURE_INCOMPAT_CSUM_V3 && + compat & JBD2_FEATURE_COMPAT_CHECKSUM) + compat &= ~JBD2_FEATURE_COMPAT_CHECKSUM; + + jbd2_debug(1, "Setting new features 0x%lx/0x%lx/0x%lx\n", + compat, ro, incompat); + + sb = journal->j_superblock; + + if (incompat & JBD2_FEATURE_INCOMPAT_FAST_COMMIT) { + if (jbd2_journal_initialize_fast_commit(journal)) { + pr_err("JBD2: Cannot enable fast commits.\n"); + return 0; + } + } + + lock_buffer(journal->j_sb_buffer); + + /* If enabling v3 checksums, update superblock and precompute seed */ + if (INCOMPAT_FEATURE_ON(JBD2_FEATURE_INCOMPAT_CSUM_V3)) { + sb->s_checksum_type = JBD2_CRC32C_CHKSUM; + sb->s_feature_compat &= + ~cpu_to_be32(JBD2_FEATURE_COMPAT_CHECKSUM); + journal->j_csum_seed = jbd2_chksum(~0, sb->s_uuid, + sizeof(sb->s_uuid)); + } + + /* If enabling v1 checksums, downgrade superblock */ + if (COMPAT_FEATURE_ON(JBD2_FEATURE_COMPAT_CHECKSUM)) + sb->s_feature_incompat &= + ~cpu_to_be32(JBD2_FEATURE_INCOMPAT_CSUM_V2 | + JBD2_FEATURE_INCOMPAT_CSUM_V3); + + sb->s_feature_compat |= cpu_to_be32(compat); + sb->s_feature_ro_compat |= cpu_to_be32(ro); + sb->s_feature_incompat |= cpu_to_be32(incompat); + /* + * Update the checksum now so that it is valid even for read-only + * filesystems where jbd2_write_superblock() doesn't get called. + */ + if (jbd2_journal_has_csum_v2or3(journal)) + sb->s_checksum = jbd2_superblock_csum(sb); + unlock_buffer(journal->j_sb_buffer); + jbd2_journal_init_transaction_limits(journal); + + return 1; +#undef COMPAT_FEATURE_ON +#undef INCOMPAT_FEATURE_ON +} + +/* + * jbd2_journal_clear_features() - Clear a given journal feature in the + * superblock + * @journal: Journal to act on. + * @compat: bitmask of compatible features + * @ro: bitmask of features that force read-only mount + * @incompat: bitmask of incompatible features + * + * Clear a given journal feature as present on the + * superblock. + */ +void jbd2_journal_clear_features(journal_t *journal, unsigned long compat, + unsigned long ro, unsigned long incompat) +{ + journal_superblock_t *sb; + + jbd2_debug(1, "Clear features 0x%lx/0x%lx/0x%lx\n", + compat, ro, incompat); + + sb = journal->j_superblock; + + lock_buffer(journal->j_sb_buffer); + sb->s_feature_compat &= ~cpu_to_be32(compat); + sb->s_feature_ro_compat &= ~cpu_to_be32(ro); + sb->s_feature_incompat &= ~cpu_to_be32(incompat); + /* + * Update the checksum now so that it is valid even for read-only + * filesystems where jbd2_write_superblock() doesn't get called. + */ + if (jbd2_journal_has_csum_v2or3(journal)) + sb->s_checksum = jbd2_superblock_csum(sb); + unlock_buffer(journal->j_sb_buffer); + jbd2_journal_init_transaction_limits(journal); +} +EXPORT_SYMBOL(jbd2_journal_clear_features); + +/** + * jbd2_journal_flush() - Flush journal + * @journal: Journal to act on. + * @flags: optional operation on the journal blocks after the flush (see below) + * + * Flush all data for a given journal to disk and empty the journal. + * Filesystems can use this when remounting readonly to ensure that + * recovery does not need to happen on remount. Optionally, a discard or zeroout + * can be issued on the journal blocks after flushing. + * + * flags: + * JBD2_JOURNAL_FLUSH_DISCARD: issues discards for the journal blocks + * JBD2_JOURNAL_FLUSH_ZEROOUT: issues zeroouts for the journal blocks + */ +int jbd2_journal_flush(journal_t *journal, unsigned int flags) +{ + int err = 0; + transaction_t *transaction = NULL; + + write_lock(&journal->j_state_lock); + + /* Force everything buffered to the log... */ + if (journal->j_running_transaction) { + transaction = journal->j_running_transaction; + __jbd2_log_start_commit(journal, transaction->t_tid); + } else if (journal->j_committing_transaction) + transaction = journal->j_committing_transaction; + + /* Wait for the log commit to complete... */ + if (transaction) { + tid_t tid = transaction->t_tid; + + write_unlock(&journal->j_state_lock); + jbd2_log_wait_commit(journal, tid); + } else { + write_unlock(&journal->j_state_lock); + } + + /* ...and flush everything in the log out to disk. */ + spin_lock(&journal->j_list_lock); + while (!err && journal->j_checkpoint_transactions != NULL) { + spin_unlock(&journal->j_list_lock); + mutex_lock_io(&journal->j_checkpoint_mutex); + err = jbd2_log_do_checkpoint(journal); + mutex_unlock(&journal->j_checkpoint_mutex); + spin_lock(&journal->j_list_lock); + } + spin_unlock(&journal->j_list_lock); + + if (is_journal_aborted(journal)) + return -EIO; + + mutex_lock_io(&journal->j_checkpoint_mutex); + if (!err) { + err = jbd2_cleanup_journal_tail(journal); + if (err < 0) { + mutex_unlock(&journal->j_checkpoint_mutex); + goto out; + } + err = 0; + } + + /* Finally, mark the journal as really needing no recovery. + * This sets s_start==0 in the underlying superblock, which is + * the magic code for a fully-recovered superblock. Any future + * commits of data to the journal will restore the current + * s_start value. */ + jbd2_mark_journal_empty(journal, REQ_FUA); + + if (flags) + err = __jbd2_journal_erase(journal, flags); + + mutex_unlock(&journal->j_checkpoint_mutex); + write_lock(&journal->j_state_lock); + J_ASSERT(!journal->j_running_transaction); + J_ASSERT(!journal->j_committing_transaction); + J_ASSERT(!journal->j_checkpoint_transactions); + J_ASSERT(journal->j_head == journal->j_tail); + J_ASSERT(journal->j_tail_sequence == journal->j_transaction_sequence); + write_unlock(&journal->j_state_lock); +out: + return err; +} + +/** + * jbd2_journal_wipe() - Wipe journal contents + * @journal: Journal to act on. + * @write: flag (see below) + * + * Wipe out all of the contents of a journal, safely. This will produce + * a warning if the journal contains any valid recovery information. + * Must be called between journal_init_*() and jbd2_journal_load(). + * + * If 'write' is non-zero, then we wipe out the journal on disk; otherwise + * we merely suppress recovery. + */ + +int jbd2_journal_wipe(journal_t *journal, int write) +{ + int err; + + J_ASSERT (!(journal->j_flags & JBD2_LOADED)); + + if (!journal->j_tail) + return 0; + + printk(KERN_WARNING "JBD2: %s recovery information on journal\n", + write ? "Clearing" : "Ignoring"); + + err = jbd2_journal_skip_recovery(journal); + if (write) { + /* Lock to make assertions happy... */ + mutex_lock_io(&journal->j_checkpoint_mutex); + jbd2_mark_journal_empty(journal, REQ_FUA); + mutex_unlock(&journal->j_checkpoint_mutex); + } + + return err; +} + +/** + * jbd2_journal_abort () - Shutdown the journal immediately. + * @journal: the journal to shutdown. + * @errno: an error number to record in the journal indicating + * the reason for the shutdown. + * + * Perform a complete, immediate shutdown of the ENTIRE + * journal (not of a single transaction). This operation cannot be + * undone without closing and reopening the journal. + * + * The jbd2_journal_abort function is intended to support higher level error + * recovery mechanisms such as the ext2/ext3 remount-readonly error + * mode. + * + * Journal abort has very specific semantics. Any existing dirty, + * unjournaled buffers in the main filesystem will still be written to + * disk by bdflush, but the journaling mechanism will be suspended + * immediately and no further transaction commits will be honoured. + * + * Any dirty, journaled buffers will be written back to disk without + * hitting the journal. Atomicity cannot be guaranteed on an aborted + * filesystem, but we _do_ attempt to leave as much data as possible + * behind for fsck to use for cleanup. + * + * Any attempt to get a new transaction handle on a journal which is in + * ABORT state will just result in an -EROFS error return. A + * jbd2_journal_stop on an existing handle will return -EIO if we have + * entered abort state during the update. + * + * Recursive transactions are not disturbed by journal abort until the + * final jbd2_journal_stop, which will receive the -EIO error. + * + * Finally, the jbd2_journal_abort call allows the caller to supply an errno + * which will be recorded (if possible) in the journal superblock. This + * allows a client to record failure conditions in the middle of a + * transaction without having to complete the transaction to record the + * failure to disk. ext3_error, for example, now uses this + * functionality. + * + */ + +void jbd2_journal_abort(journal_t *journal, int errno) +{ + transaction_t *transaction; + + /* + * Lock the aborting procedure until everything is done, this avoid + * races between filesystem's error handling flow (e.g. ext4_abort()), + * ensure panic after the error info is written into journal's + * superblock. + */ + mutex_lock(&journal->j_abort_mutex); + /* + * ESHUTDOWN always takes precedence because a file system check + * caused by any other journal abort error is not required after + * a shutdown triggered. + */ + write_lock(&journal->j_state_lock); + if (journal->j_flags & JBD2_ABORT) { + int old_errno = journal->j_errno; + + write_unlock(&journal->j_state_lock); + if (old_errno != -ESHUTDOWN && errno == -ESHUTDOWN) { + journal->j_errno = errno; + jbd2_journal_update_sb_errno(journal); + } + mutex_unlock(&journal->j_abort_mutex); + return; + } + + /* + * Mark the abort as occurred and start current running transaction + * to release all journaled buffer. + */ + pr_err("Aborting journal on device %s.\n", journal->j_devname); + + journal->j_flags |= JBD2_ABORT; + journal->j_errno = errno; + transaction = journal->j_running_transaction; + if (transaction) + __jbd2_log_start_commit(journal, transaction->t_tid); + write_unlock(&journal->j_state_lock); + + /* + * Record errno to the journal super block, so that fsck and jbd2 + * layer could realise that a filesystem check is needed. + */ + jbd2_journal_update_sb_errno(journal); + mutex_unlock(&journal->j_abort_mutex); +} + +/** + * jbd2_journal_errno() - returns the journal's error state. + * @journal: journal to examine. + * + * This is the errno number set with jbd2_journal_abort(), the last + * time the journal was mounted - if the journal was stopped + * without calling abort this will be 0. + * + * If the journal has been aborted on this mount time -EROFS will + * be returned. + */ +int jbd2_journal_errno(journal_t *journal) +{ + int err; + + read_lock(&journal->j_state_lock); + if (journal->j_flags & JBD2_ABORT) + err = -EROFS; + else + err = journal->j_errno; + read_unlock(&journal->j_state_lock); + return err; +} + +/** + * jbd2_journal_clear_err() - clears the journal's error state + * @journal: journal to act on. + * + * An error must be cleared or acked to take a FS out of readonly + * mode. + */ +int jbd2_journal_clear_err(journal_t *journal) +{ + int err = 0; + + write_lock(&journal->j_state_lock); + if (journal->j_flags & JBD2_ABORT) + err = -EROFS; + else + journal->j_errno = 0; + write_unlock(&journal->j_state_lock); + return err; +} + +/** + * jbd2_journal_ack_err() - Ack journal err. + * @journal: journal to act on. + * + * An error must be cleared or acked to take a FS out of readonly + * mode. + */ +void jbd2_journal_ack_err(journal_t *journal) +{ + write_lock(&journal->j_state_lock); + if (journal->j_errno) + journal->j_flags |= JBD2_ACK_ERR; + write_unlock(&journal->j_state_lock); +} + +int jbd2_journal_blocks_per_folio(struct inode *inode) +{ + return 1 << (PAGE_SHIFT + mapping_max_folio_order(inode->i_mapping) - + inode->i_sb->s_blocksize_bits); +} + +/* + * helper functions to deal with 32 or 64bit block numbers. + */ +size_t journal_tag_bytes(journal_t *journal) +{ + size_t sz; + + if (jbd2_has_feature_csum3(journal)) + return sizeof(journal_block_tag3_t); + + sz = sizeof(journal_block_tag_t); + + if (jbd2_has_feature_csum2(journal)) + sz += sizeof(__u16); + + if (jbd2_has_feature_64bit(journal)) + return sz; + else + return sz - sizeof(__u32); +} + +/* + * Journal_head storage management + */ +static struct kmem_cache *jbd2_journal_head_cache; +#ifdef CONFIG_JBD2_DEBUG +static atomic_t nr_journal_heads = ATOMIC_INIT(0); +#endif + +static int __init jbd2_journal_init_journal_head_cache(void) +{ + J_ASSERT(!jbd2_journal_head_cache); + jbd2_journal_head_cache = kmem_cache_create("jbd2_journal_head", + sizeof(struct journal_head), + 0, /* offset */ + SLAB_TEMPORARY | SLAB_TYPESAFE_BY_RCU, + NULL); /* ctor */ + if (!jbd2_journal_head_cache) { + printk(KERN_EMERG "JBD2: no memory for journal_head cache\n"); + return -ENOMEM; + } + return 0; +} + +static void jbd2_journal_destroy_journal_head_cache(void) +{ + kmem_cache_destroy(jbd2_journal_head_cache); + jbd2_journal_head_cache = NULL; +} + +/* + * journal_head splicing and dicing + */ +static struct journal_head *journal_alloc_journal_head(void) +{ + struct journal_head *ret; + +#ifdef CONFIG_JBD2_DEBUG + atomic_inc(&nr_journal_heads); +#endif + ret = kmem_cache_zalloc(jbd2_journal_head_cache, GFP_NOFS); + if (!ret) { + jbd2_debug(1, "out of memory for journal_head\n"); + pr_notice_ratelimited("ENOMEM in %s, retrying.\n", __func__); + ret = kmem_cache_zalloc(jbd2_journal_head_cache, + GFP_NOFS | __GFP_NOFAIL); + } + spin_lock_init(&ret->b_state_lock); + return ret; +} + +static void journal_free_journal_head(struct journal_head *jh) +{ +#ifdef CONFIG_JBD2_DEBUG + atomic_dec(&nr_journal_heads); + memset(jh, JBD2_POISON_FREE, sizeof(*jh)); +#endif + kmem_cache_free(jbd2_journal_head_cache, jh); +} + +/* + * A journal_head is attached to a buffer_head whenever JBD has an + * interest in the buffer. + * + * Whenever a buffer has an attached journal_head, its ->b_state:BH_JBD bit + * is set. This bit is tested in core kernel code where we need to take + * JBD-specific actions. Testing the zeroness of ->b_private is not reliable + * there. + * + * When a buffer has its BH_JBD bit set, its ->b_count is elevated by one. + * + * When a buffer has its BH_JBD bit set it is immune from being released by + * core kernel code, mainly via ->b_count. + * + * A journal_head is detached from its buffer_head when the journal_head's + * b_jcount reaches zero. Running transaction (b_transaction) and checkpoint + * transaction (b_cp_transaction) hold their references to b_jcount. + * + * Various places in the kernel want to attach a journal_head to a buffer_head + * _before_ attaching the journal_head to a transaction. To protect the + * journal_head in this situation, jbd2_journal_add_journal_head elevates the + * journal_head's b_jcount refcount by one. The caller must call + * jbd2_journal_put_journal_head() to undo this. + * + * So the typical usage would be: + * + * (Attach a journal_head if needed. Increments b_jcount) + * struct journal_head *jh = jbd2_journal_add_journal_head(bh); + * ... + * (Get another reference for transaction) + * jbd2_journal_grab_journal_head(bh); + * jh->b_transaction = xxx; + * (Put original reference) + * jbd2_journal_put_journal_head(jh); + */ + +/* + * Give a buffer_head a journal_head. + * + * May sleep. + */ +struct journal_head *jbd2_journal_add_journal_head(struct buffer_head *bh) +{ + struct journal_head *jh; + struct journal_head *new_jh = NULL; + +repeat: + if (!buffer_jbd(bh)) + new_jh = journal_alloc_journal_head(); + + jbd_lock_bh_journal_head(bh); + if (buffer_jbd(bh)) { + jh = bh2jh(bh); + } else { + J_ASSERT_BH(bh, + (atomic_read(&bh->b_count) > 0) || + (bh->b_folio && bh->b_folio->mapping)); + + if (!new_jh) { + jbd_unlock_bh_journal_head(bh); + goto repeat; + } + + jh = new_jh; + new_jh = NULL; /* We consumed it */ + set_buffer_jbd(bh); + bh->b_private = jh; + jh->b_bh = bh; + get_bh(bh); + BUFFER_TRACE(bh, "added journal_head"); + } + jh->b_jcount++; + jbd_unlock_bh_journal_head(bh); + if (new_jh) + journal_free_journal_head(new_jh); + return bh->b_private; +} + +/* + * Grab a ref against this buffer_head's journal_head. If it ended up not + * having a journal_head, return NULL + */ +struct journal_head *jbd2_journal_grab_journal_head(struct buffer_head *bh) +{ + struct journal_head *jh = NULL; + + jbd_lock_bh_journal_head(bh); + if (buffer_jbd(bh)) { + jh = bh2jh(bh); + jh->b_jcount++; + } + jbd_unlock_bh_journal_head(bh); + return jh; +} +EXPORT_SYMBOL(jbd2_journal_grab_journal_head); + +static void __journal_remove_journal_head(struct buffer_head *bh) +{ + struct journal_head *jh = bh2jh(bh); + + J_ASSERT_JH(jh, jh->b_transaction == NULL); + J_ASSERT_JH(jh, jh->b_next_transaction == NULL); + J_ASSERT_JH(jh, jh->b_cp_transaction == NULL); + J_ASSERT_JH(jh, jh->b_jlist == BJ_None); + J_ASSERT_BH(bh, buffer_jbd(bh)); + J_ASSERT_BH(bh, jh2bh(jh) == bh); + BUFFER_TRACE(bh, "remove journal_head"); + + /* Unlink before dropping the lock */ + bh->b_private = NULL; + jh->b_bh = NULL; /* debug, really */ + clear_buffer_jbd(bh); +} + +static void journal_release_journal_head(struct journal_head *jh) +{ + if (jh->b_frozen_data) { + printk(KERN_WARNING "%s: freeing b_frozen_data\n", __func__); + kfree(jh->b_frozen_data); + } + if (jh->b_committed_data) { + printk(KERN_WARNING "%s: freeing b_committed_data\n", __func__); + kfree(jh->b_committed_data); + } + journal_free_journal_head(jh); +} + +/* + * Drop a reference on the passed journal_head. If it fell to zero then + * release the journal_head from the buffer_head. + */ +void jbd2_journal_put_journal_head(struct journal_head *jh) +{ + struct buffer_head *bh = jh2bh(jh); + + jbd_lock_bh_journal_head(bh); + J_ASSERT_JH(jh, jh->b_jcount > 0); + --jh->b_jcount; + if (!jh->b_jcount) { + __journal_remove_journal_head(bh); + jbd_unlock_bh_journal_head(bh); + journal_release_journal_head(jh); + __brelse(bh); + } else { + jbd_unlock_bh_journal_head(bh); + } +} +EXPORT_SYMBOL(jbd2_journal_put_journal_head); + +/* + * Initialize jbd inode head + */ +void jbd2_journal_init_jbd_inode(struct jbd2_inode *jinode, struct inode *inode) +{ + jinode->i_transaction = NULL; + jinode->i_next_transaction = NULL; + jinode->i_vfs_inode = inode; + jinode->i_flags = 0; + jinode->i_dirty_start_page = 0; + jinode->i_dirty_end_page = 0; + INIT_LIST_HEAD(&jinode->i_list); +} + +/* + * Function to be called before we start removing inode from memory (i.e., + * clear_inode() is a fine place to be called from). It removes inode from + * transaction's lists. + */ +void jbd2_journal_release_jbd_inode(journal_t *journal, + struct jbd2_inode *jinode) +{ + if (!journal) + return; +restart: + spin_lock(&journal->j_list_lock); + /* Is commit writing out inode - we have to wait */ + if (jinode->i_flags & JI_COMMIT_RUNNING) { + wait_queue_head_t *wq; + DEFINE_WAIT_BIT(wait, &jinode->i_flags, __JI_COMMIT_RUNNING); + wq = bit_waitqueue(&jinode->i_flags, __JI_COMMIT_RUNNING); + prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE); + spin_unlock(&journal->j_list_lock); + schedule(); + finish_wait(wq, &wait.wq_entry); + goto restart; + } + + if (jinode->i_transaction) { + list_del(&jinode->i_list); + jinode->i_transaction = NULL; + } + spin_unlock(&journal->j_list_lock); +} + + +#ifdef CONFIG_PROC_FS + +#define JBD2_STATS_PROC_NAME "fs/jbd2" + +static void __init jbd2_create_jbd_stats_proc_entry(void) +{ + proc_jbd2_stats = proc_mkdir(JBD2_STATS_PROC_NAME, NULL); +} + +static void __exit jbd2_remove_jbd_stats_proc_entry(void) +{ + if (proc_jbd2_stats) + remove_proc_entry(JBD2_STATS_PROC_NAME, NULL); +} + +#else + +#define jbd2_create_jbd_stats_proc_entry() do {} while (0) +#define jbd2_remove_jbd_stats_proc_entry() do {} while (0) + +#endif + +struct kmem_cache *jbd2_handle_cache, *jbd2_inode_cache; + +static int __init jbd2_journal_init_inode_cache(void) +{ + J_ASSERT(!jbd2_inode_cache); + jbd2_inode_cache = KMEM_CACHE(jbd2_inode, 0); + if (!jbd2_inode_cache) { + pr_emerg("JBD2: failed to create inode cache\n"); + return -ENOMEM; + } + return 0; +} + +static int __init jbd2_journal_init_handle_cache(void) +{ + J_ASSERT(!jbd2_handle_cache); + jbd2_handle_cache = KMEM_CACHE(jbd2_journal_handle, SLAB_TEMPORARY); + if (!jbd2_handle_cache) { + printk(KERN_EMERG "JBD2: failed to create handle cache\n"); + return -ENOMEM; + } + return 0; +} + +static void jbd2_journal_destroy_inode_cache(void) +{ + kmem_cache_destroy(jbd2_inode_cache); + jbd2_inode_cache = NULL; +} + +static void jbd2_journal_destroy_handle_cache(void) +{ + kmem_cache_destroy(jbd2_handle_cache); + jbd2_handle_cache = NULL; +} + +/* + * Module startup and shutdown + */ + +static int __init journal_init_caches(void) +{ + int ret; + + ret = jbd2_journal_init_revoke_record_cache(); + if (ret == 0) + ret = jbd2_journal_init_revoke_table_cache(); + if (ret == 0) + ret = jbd2_journal_init_journal_head_cache(); + if (ret == 0) + ret = jbd2_journal_init_handle_cache(); + if (ret == 0) + ret = jbd2_journal_init_inode_cache(); + if (ret == 0) + ret = jbd2_journal_init_transaction_cache(); + return ret; +} + +static void jbd2_journal_destroy_caches(void) +{ + jbd2_journal_destroy_revoke_record_cache(); + jbd2_journal_destroy_revoke_table_cache(); + jbd2_journal_destroy_journal_head_cache(); + jbd2_journal_destroy_handle_cache(); + jbd2_journal_destroy_inode_cache(); + jbd2_journal_destroy_transaction_cache(); +} + +static int __init journal_init(void) +{ + int ret; + + BUILD_BUG_ON(sizeof(struct journal_superblock_s) != 1024); + + ret = journal_init_caches(); + if (ret == 0) { + jbd2_create_jbd_stats_proc_entry(); + } else { + jbd2_journal_destroy_caches(); + } + return ret; +} + +static void __exit journal_exit(void) +{ +#ifdef CONFIG_JBD2_DEBUG + int n = atomic_read(&nr_journal_heads); + if (n) + printk(KERN_ERR "JBD2: leaked %d journal_heads!\n", n); +#endif + jbd2_remove_jbd_stats_proc_entry(); + jbd2_journal_destroy_caches(); +} + +MODULE_DESCRIPTION("Generic filesystem journal-writing module"); +MODULE_LICENSE("GPL"); +module_init(journal_init); +module_exit(journal_exit); diff --git a/fs/jbd2/recovery.c b/fs/jbd2/recovery.c new file mode 100644 index 000000000..cac8c2cd4 --- /dev/null +++ b/fs/jbd2/recovery.c @@ -0,0 +1,996 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * linux/fs/jbd2/recovery.c + * + * Written by Stephen C. Tweedie , 1999 + * + * Copyright 1999-2000 Red Hat Software --- All Rights Reserved + * + * Journal recovery routines for the generic filesystem journaling code; + * part of the ext2fs journaling system. + */ + +#ifndef __KERNEL__ +#include "jfs_user.h" +#else +#include +#include +#include +#include +#include +#include +#include +#endif + +/* + * Maintain information about the progress of the recovery job, so that + * the different passes can carry information between them. + */ +struct recovery_info +{ + tid_t start_transaction; + tid_t end_transaction; + unsigned long head_block; + + int nr_replays; + int nr_revokes; + int nr_revoke_hits; +}; + +static int do_one_pass(journal_t *journal, + struct recovery_info *info, enum passtype pass); +static int scan_revoke_records(journal_t *, enum passtype, struct buffer_head *, + tid_t, struct recovery_info *); + +#ifdef __KERNEL__ + +/* Release readahead buffers after use */ +static void journal_brelse_array(struct buffer_head *b[], int n) +{ + while (--n >= 0) + brelse (b[n]); +} + + +/* + * When reading from the journal, we are going through the block device + * layer directly and so there is no readahead being done for us. We + * need to implement any readahead ourselves if we want it to happen at + * all. Recovery is basically one long sequential read, so make sure we + * do the IO in reasonably large chunks. + * + * This is not so critical that we need to be enormously clever about + * the readahead size, though. 128K is a purely arbitrary, good-enough + * fixed value. + */ + +#define MAXBUF 8 +static void do_readahead(journal_t *journal, unsigned int start) +{ + unsigned int max, nbufs, next; + unsigned long long blocknr; + struct buffer_head *bh; + + struct buffer_head * bufs[MAXBUF]; + + /* Do up to 128K of readahead */ + max = start + (128 * 1024 / journal->j_blocksize); + if (max > journal->j_total_len) + max = journal->j_total_len; + + /* Do the readahead itself. We'll submit MAXBUF buffer_heads at + * a time to the block device IO layer. */ + + nbufs = 0; + + for (next = start; next < max; next++) { + int err = jbd2_journal_bmap(journal, next, &blocknr); + + if (err) { + printk(KERN_ERR "JBD2: bad block at offset %u\n", + next); + goto failed; + } + + bh = __getblk(journal->j_dev, blocknr, journal->j_blocksize); + if (!bh) + goto failed; + + if (!buffer_uptodate(bh) && !buffer_locked(bh)) { + bufs[nbufs++] = bh; + if (nbufs == MAXBUF) { + bh_readahead_batch(nbufs, bufs, 0); + journal_brelse_array(bufs, nbufs); + nbufs = 0; + } + } else + brelse(bh); + } + + if (nbufs) + bh_readahead_batch(nbufs, bufs, 0); + +failed: + if (nbufs) + journal_brelse_array(bufs, nbufs); +} + +#endif /* __KERNEL__ */ + + +/* + * Read a block from the journal + */ + +static int jread(struct buffer_head **bhp, journal_t *journal, + unsigned int offset) +{ + int err; + unsigned long long blocknr; + struct buffer_head *bh; + + *bhp = NULL; + + if (offset >= journal->j_total_len) { + printk(KERN_ERR "JBD2: corrupted journal superblock\n"); + return -EFSCORRUPTED; + } + + err = jbd2_journal_bmap(journal, offset, &blocknr); + + if (err) { + printk(KERN_ERR "JBD2: bad block at offset %u\n", + offset); + return err; + } + + bh = __getblk(journal->j_dev, blocknr, journal->j_blocksize); + if (!bh) + return -ENOMEM; + + if (!buffer_uptodate(bh)) { + /* + * If this is a brand new buffer, start readahead. + * Otherwise, we assume we are already reading it. + */ + bool need_readahead = !buffer_req(bh); + + bh_read_nowait(bh, 0); + if (need_readahead) + do_readahead(journal, offset); + wait_on_buffer(bh); + } + + if (!buffer_uptodate(bh)) { + printk(KERN_ERR "JBD2: Failed to read block at offset %u\n", + offset); + brelse(bh); + return -EIO; + } + + *bhp = bh; + return 0; +} + +static int jbd2_descriptor_block_csum_verify(journal_t *j, void *buf) +{ + struct jbd2_journal_block_tail *tail; + __be32 provided; + __u32 calculated; + + if (!jbd2_journal_has_csum_v2or3(j)) + return 1; + + tail = (struct jbd2_journal_block_tail *)((char *)buf + + j->j_blocksize - sizeof(struct jbd2_journal_block_tail)); + provided = tail->t_checksum; + tail->t_checksum = 0; + calculated = jbd2_chksum(j->j_csum_seed, buf, j->j_blocksize); + tail->t_checksum = provided; + + return provided == cpu_to_be32(calculated); +} + +/* + * Count the number of in-use tags in a journal descriptor block. + */ + +static int count_tags(journal_t *journal, struct buffer_head *bh) +{ + char * tagp; + journal_block_tag_t tag; + int nr = 0, size = journal->j_blocksize; + int tag_bytes = journal_tag_bytes(journal); + + if (jbd2_journal_has_csum_v2or3(journal)) + size -= sizeof(struct jbd2_journal_block_tail); + + tagp = &bh->b_data[sizeof(journal_header_t)]; + + while ((tagp - bh->b_data + tag_bytes) <= size) { + memcpy(&tag, tagp, sizeof(tag)); + + nr++; + tagp += tag_bytes; + if (!(tag.t_flags & cpu_to_be16(JBD2_FLAG_SAME_UUID))) + tagp += 16; + + if (tag.t_flags & cpu_to_be16(JBD2_FLAG_LAST_TAG)) + break; + } + + return nr; +} + + +/* Make sure we wrap around the log correctly! */ +#define wrap(journal, var) \ +do { \ + if (var >= (journal)->j_last) \ + var -= ((journal)->j_last - (journal)->j_first); \ +} while (0) + +static int fc_do_one_pass(journal_t *journal, + struct recovery_info *info, enum passtype pass) +{ + unsigned int expected_commit_id = info->end_transaction; + unsigned long next_fc_block; + struct buffer_head *bh; + int err = 0; + + next_fc_block = journal->j_fc_first; + if (!journal->j_fc_replay_callback) + return 0; + + while (next_fc_block <= journal->j_fc_last) { + jbd2_debug(3, "Fast commit replay: next block %ld\n", + next_fc_block); + err = jread(&bh, journal, next_fc_block); + if (err) { + jbd2_debug(3, "Fast commit replay: read error\n"); + break; + } + + err = journal->j_fc_replay_callback(journal, bh, pass, + next_fc_block - journal->j_fc_first, + expected_commit_id); + brelse(bh); + next_fc_block++; + if (err < 0 || err == JBD2_FC_REPLAY_STOP) + break; + err = 0; + } + + if (err) + jbd2_debug(3, "Fast commit replay failed, err = %d\n", err); + + return err; +} + +/** + * jbd2_journal_recover - recovers a on-disk journal + * @journal: the journal to recover + * + * The primary function for recovering the log contents when mounting a + * journaled device. + * + * Recovery is done in three passes. In the first pass, we look for the + * end of the log. In the second, we assemble the list of revoke + * blocks. In the third and final pass, we replay any un-revoked blocks + * in the log. + */ +int jbd2_journal_recover(journal_t *journal) +{ + int err, err2; + struct recovery_info info; + + memset(&info, 0, sizeof(info)); + + /* + * The journal superblock's s_start field (the current log head) + * is always zero if, and only if, the journal was cleanly + * unmounted. We use its in-memory version j_tail here because + * jbd2_journal_wipe() could have updated it without updating journal + * superblock. + */ + if (!journal->j_tail) { + journal_superblock_t *sb = journal->j_superblock; + + jbd2_debug(1, "No recovery required, last transaction %d, head block %u\n", + be32_to_cpu(sb->s_sequence), be32_to_cpu(sb->s_head)); + journal->j_transaction_sequence = be32_to_cpu(sb->s_sequence) + 1; + journal->j_head = be32_to_cpu(sb->s_head); + return 0; + } + + err = do_one_pass(journal, &info, PASS_SCAN); + if (!err) + err = do_one_pass(journal, &info, PASS_REVOKE); + if (!err) + err = do_one_pass(journal, &info, PASS_REPLAY); + + jbd2_debug(1, "JBD2: recovery, exit status %d, " + "recovered transactions %u to %u\n", + err, info.start_transaction, info.end_transaction); + jbd2_debug(1, "JBD2: Replayed %d and revoked %d/%d blocks\n", + info.nr_replays, info.nr_revoke_hits, info.nr_revokes); + + /* Restart the log at the next transaction ID, thus invalidating + * any existing commit records in the log. */ + journal->j_transaction_sequence = ++info.end_transaction; + journal->j_head = info.head_block; + jbd2_debug(1, "JBD2: last transaction %d, head block %lu\n", + journal->j_transaction_sequence, journal->j_head); + + jbd2_journal_clear_revoke(journal); + /* Free revoke table allocated for replay */ + if (journal->j_revoke != journal->j_revoke_table[0] && + journal->j_revoke != journal->j_revoke_table[1]) { + jbd2_journal_destroy_revoke_table(journal->j_revoke); + journal->j_revoke = journal->j_revoke_table[1]; + } + err2 = sync_blockdev(journal->j_fs_dev); + if (!err) + err = err2; + err2 = jbd2_check_fs_dev_write_error(journal); + if (!err) + err = err2; + /* Make sure all replayed data is on permanent storage */ + if (journal->j_flags & JBD2_BARRIER) { + err2 = blkdev_issue_flush(journal->j_fs_dev); + if (!err) + err = err2; + } + return err; +} + +/** + * jbd2_journal_skip_recovery - Start journal and wipe exiting records + * @journal: journal to startup + * + * Locate any valid recovery information from the journal and set up the + * journal structures in memory to ignore it (presumably because the + * caller has evidence that it is out of date). + * This function doesn't appear to be exported.. + * + * We perform one pass over the journal to allow us to tell the user how + * much recovery information is being erased, and to let us initialise + * the journal transaction sequence numbers to the next unused ID. + */ +int jbd2_journal_skip_recovery(journal_t *journal) +{ + int err; + + struct recovery_info info; + + memset (&info, 0, sizeof(info)); + + err = do_one_pass(journal, &info, PASS_SCAN); + + if (err) { + printk(KERN_ERR "JBD2: error %d scanning journal\n", err); + ++journal->j_transaction_sequence; + journal->j_head = journal->j_first; + } else { +#ifdef CONFIG_JBD2_DEBUG + int dropped = info.end_transaction - + be32_to_cpu(journal->j_superblock->s_sequence); + jbd2_debug(1, + "JBD2: ignoring %d transaction%s from the journal.\n", + dropped, str_plural(dropped)); +#endif + journal->j_transaction_sequence = ++info.end_transaction; + journal->j_head = info.head_block; + } + + journal->j_tail = 0; + return err; +} + +static inline unsigned long long read_tag_block(journal_t *journal, + journal_block_tag_t *tag) +{ + unsigned long long block = be32_to_cpu(tag->t_blocknr); + if (jbd2_has_feature_64bit(journal)) + block |= (u64)be32_to_cpu(tag->t_blocknr_high) << 32; + return block; +} + +/* + * calc_chksums calculates the checksums for the blocks described in the + * descriptor block. + */ +static int calc_chksums(journal_t *journal, struct buffer_head *bh, + unsigned long *next_log_block, __u32 *crc32_sum) +{ + int i, num_blks, err; + unsigned long io_block; + struct buffer_head *obh; + + num_blks = count_tags(journal, bh); + /* Calculate checksum of the descriptor block. */ + *crc32_sum = crc32_be(*crc32_sum, (void *)bh->b_data, bh->b_size); + + for (i = 0; i < num_blks; i++) { + io_block = (*next_log_block)++; + wrap(journal, *next_log_block); + err = jread(&obh, journal, io_block); + if (err) { + printk(KERN_ERR "JBD2: IO error %d recovering block " + "%lu in log\n", err, io_block); + return 1; + } else { + *crc32_sum = crc32_be(*crc32_sum, (void *)obh->b_data, + obh->b_size); + } + put_bh(obh); + } + return 0; +} + +static int jbd2_commit_block_csum_verify(journal_t *j, void *buf) +{ + struct commit_header *h; + __be32 provided; + __u32 calculated; + + if (!jbd2_journal_has_csum_v2or3(j)) + return 1; + + h = buf; + provided = h->h_chksum[0]; + h->h_chksum[0] = 0; + calculated = jbd2_chksum(j->j_csum_seed, buf, j->j_blocksize); + h->h_chksum[0] = provided; + + return provided == cpu_to_be32(calculated); +} + +static bool jbd2_commit_block_csum_verify_partial(journal_t *j, void *buf) +{ + struct commit_header *h; + __be32 provided; + __u32 calculated; + void *tmpbuf; + + tmpbuf = kzalloc(j->j_blocksize, GFP_KERNEL); + if (!tmpbuf) + return false; + + memcpy(tmpbuf, buf, sizeof(struct commit_header)); + h = tmpbuf; + provided = h->h_chksum[0]; + h->h_chksum[0] = 0; + calculated = jbd2_chksum(j->j_csum_seed, tmpbuf, j->j_blocksize); + kfree(tmpbuf); + + return provided == cpu_to_be32(calculated); +} + +static int jbd2_block_tag_csum_verify(journal_t *j, journal_block_tag_t *tag, + journal_block_tag3_t *tag3, + void *buf, __u32 sequence) +{ + __u32 csum32; + __be32 seq; + + if (!jbd2_journal_has_csum_v2or3(j)) + return 1; + + seq = cpu_to_be32(sequence); + csum32 = jbd2_chksum(j->j_csum_seed, (__u8 *)&seq, sizeof(seq)); + csum32 = jbd2_chksum(csum32, buf, j->j_blocksize); + + if (jbd2_has_feature_csum3(j)) + return tag3->t_checksum == cpu_to_be32(csum32); + else + return tag->t_checksum == cpu_to_be16(csum32); +} + +static __always_inline int jbd2_do_replay(journal_t *journal, + struct recovery_info *info, + struct buffer_head *bh, + unsigned long *next_log_block, + unsigned int next_commit_ID) +{ + char *tagp; + int flags; + int ret = 0; + int tag_bytes = journal_tag_bytes(journal); + int descr_csum_size = 0; + unsigned long io_block; + journal_block_tag_t tag; + struct buffer_head *obh; + struct buffer_head *nbh; + + if (jbd2_journal_has_csum_v2or3(journal)) + descr_csum_size = sizeof(struct jbd2_journal_block_tail); + + tagp = &bh->b_data[sizeof(journal_header_t)]; + while (tagp - bh->b_data + tag_bytes <= + journal->j_blocksize - descr_csum_size) { + int err; + + memcpy(&tag, tagp, sizeof(tag)); + flags = be16_to_cpu(tag.t_flags); + + io_block = (*next_log_block)++; + wrap(journal, *next_log_block); + err = jread(&obh, journal, io_block); + if (err) { + /* Recover what we can, but report failure at the end. */ + ret = err; + pr_err("JBD2: IO error %d recovering block %lu in log\n", + err, io_block); + } else { + unsigned long long blocknr; + + J_ASSERT(obh != NULL); + blocknr = read_tag_block(journal, &tag); + + /* If the block has been revoked, then we're all done here. */ + if (jbd2_journal_test_revoke(journal, blocknr, + next_commit_ID)) { + brelse(obh); + ++info->nr_revoke_hits; + goto skip_write; + } + + /* Look for block corruption */ + if (!jbd2_block_tag_csum_verify(journal, &tag, + (journal_block_tag3_t *)tagp, + obh->b_data, next_commit_ID)) { + brelse(obh); + ret = -EFSBADCRC; + pr_err("JBD2: Invalid checksum recovering data block %llu in journal block %lu\n", + blocknr, io_block); + goto skip_write; + } + + /* Find a buffer for the new data being restored */ + nbh = __getblk(journal->j_fs_dev, blocknr, + journal->j_blocksize); + if (nbh == NULL) { + pr_err("JBD2: Out of memory during recovery.\n"); + brelse(obh); + return -ENOMEM; + } + + lock_buffer(nbh); + memcpy(nbh->b_data, obh->b_data, journal->j_blocksize); + if (flags & JBD2_FLAG_ESCAPE) { + *((__be32 *)nbh->b_data) = + cpu_to_be32(JBD2_MAGIC_NUMBER); + } + + BUFFER_TRACE(nbh, "marking dirty"); + set_buffer_uptodate(nbh); + mark_buffer_dirty(nbh); + BUFFER_TRACE(nbh, "marking uptodate"); + ++info->nr_replays; + unlock_buffer(nbh); + brelse(obh); + brelse(nbh); + } + +skip_write: + tagp += tag_bytes; + if (!(flags & JBD2_FLAG_SAME_UUID)) + tagp += 16; + + if (flags & JBD2_FLAG_LAST_TAG) + break; + } + + return ret; +} + +static int do_one_pass(journal_t *journal, + struct recovery_info *info, enum passtype pass) +{ + unsigned int first_commit_ID, next_commit_ID; + unsigned long next_log_block, head_block; + int err, success = 0; + journal_superblock_t * sb; + journal_header_t * tmp; + struct buffer_head *bh = NULL; + unsigned int sequence; + int blocktype; + __u32 crc32_sum = ~0; /* Transactional Checksums */ + bool need_check_commit_time = false; + __u64 last_trans_commit_time = 0, commit_time; + + /* + * First thing is to establish what we expect to find in the log + * (in terms of transaction IDs), and where (in terms of log + * block offsets): query the superblock. + */ + + sb = journal->j_superblock; + next_commit_ID = be32_to_cpu(sb->s_sequence); + next_log_block = be32_to_cpu(sb->s_start); + head_block = next_log_block; + + first_commit_ID = next_commit_ID; + if (pass == PASS_SCAN) + info->start_transaction = first_commit_ID; + else if (pass == PASS_REVOKE) { + /* + * Would the default revoke table have too long hash chains + * during replay? + */ + if (info->nr_revokes > JOURNAL_REVOKE_DEFAULT_HASH * 16) { + unsigned int hash_size; + + /* + * Aim for average chain length of 8, limit at 1M + * entries to avoid problems with malicious + * filesystems. + */ + hash_size = min(roundup_pow_of_two(info->nr_revokes / 8), + 1U << 20); + journal->j_revoke = + jbd2_journal_init_revoke_table(hash_size); + if (!journal->j_revoke) { + printk(KERN_ERR + "JBD2: failed to allocate revoke table for replay with %u entries. " + "Journal replay may be slow.\n", hash_size); + journal->j_revoke = journal->j_revoke_table[1]; + } + } + } + + jbd2_debug(1, "Starting recovery pass %d\n", pass); + + /* + * Now we walk through the log, transaction by transaction, + * making sure that each transaction has a commit block in the + * expected place. Each complete transaction gets replayed back + * into the main filesystem. + */ + + while (1) { + cond_resched(); + + /* If we already know where to stop the log traversal, + * check right now that we haven't gone past the end of + * the log. */ + + if (pass != PASS_SCAN) + if (tid_geq(next_commit_ID, info->end_transaction)) + break; + + jbd2_debug(2, "Scanning for sequence ID %u at %lu/%lu\n", + next_commit_ID, next_log_block, journal->j_last); + + /* Skip over each chunk of the transaction looking + * either the next descriptor block or the final commit + * record. */ + + jbd2_debug(3, "JBD2: checking block %ld\n", next_log_block); + brelse(bh); + bh = NULL; + err = jread(&bh, journal, next_log_block); + if (err) + goto failed; + + next_log_block++; + wrap(journal, next_log_block); + + /* What kind of buffer is it? + * + * If it is a descriptor block, check that it has the + * expected sequence number. Otherwise, we're all done + * here. */ + + tmp = (journal_header_t *)bh->b_data; + + if (tmp->h_magic != cpu_to_be32(JBD2_MAGIC_NUMBER)) + break; + + blocktype = be32_to_cpu(tmp->h_blocktype); + sequence = be32_to_cpu(tmp->h_sequence); + jbd2_debug(3, "Found magic %d, sequence %d\n", + blocktype, sequence); + + if (sequence != next_commit_ID) + break; + + /* OK, we have a valid descriptor block which matches + * all of the sequence number checks. What are we going + * to do with it? That depends on the pass... */ + + switch(blocktype) { + case JBD2_DESCRIPTOR_BLOCK: + /* Verify checksum first */ + if (!jbd2_descriptor_block_csum_verify(journal, + bh->b_data)) { + /* + * PASS_SCAN can see stale blocks due to lazy + * journal init. Don't error out on those yet. + */ + if (pass != PASS_SCAN) { + pr_err("JBD2: Invalid checksum recovering block %lu in log\n", + next_log_block); + err = -EFSBADCRC; + goto failed; + } + need_check_commit_time = true; + jbd2_debug(1, + "invalid descriptor block found in %lu\n", + next_log_block); + } + + /* If it is a valid descriptor block, replay it + * in pass REPLAY; if journal_checksums enabled, then + * calculate checksums in PASS_SCAN, otherwise, + * just skip over the blocks it describes. */ + if (pass != PASS_REPLAY) { + if (pass == PASS_SCAN && + jbd2_has_feature_checksum(journal) && + !info->end_transaction) { + if (calc_chksums(journal, bh, + &next_log_block, + &crc32_sum)) + break; + continue; + } + next_log_block += count_tags(journal, bh); + wrap(journal, next_log_block); + continue; + } + + /* + * A descriptor block: we can now write all of the + * data blocks. Yay, useful work is finally getting + * done here! + */ + err = jbd2_do_replay(journal, info, bh, &next_log_block, + next_commit_ID); + if (err) { + if (err == -ENOMEM) + goto failed; + success = err; + } + + continue; + + case JBD2_COMMIT_BLOCK: + if (pass != PASS_SCAN) { + next_commit_ID++; + continue; + } + + /* How to differentiate between interrupted commit + * and journal corruption ? + * + * {nth transaction} + * Checksum Verification Failed + * | + * ____________________ + * | | + * async_commit sync_commit + * | | + * | GO TO NEXT "Journal Corruption" + * | TRANSACTION + * | + * {(n+1)th transanction} + * | + * _______|______________ + * | | + * Commit block found Commit block not found + * | | + * "Journal Corruption" | + * _____________|_________ + * | | + * nth trans corrupt OR nth trans + * and (n+1)th interrupted interrupted + * before commit block + * could reach the disk. + * (Cannot find the difference in above + * mentioned conditions. Hence assume + * "Interrupted Commit".) + */ + commit_time = be64_to_cpu( + ((struct commit_header *)bh->b_data)->h_commit_sec); + /* + * If need_check_commit_time is set, it means we are in + * PASS_SCAN and csum verify failed before. If + * commit_time is increasing, it's the same journal, + * otherwise it is stale journal block, just end this + * recovery. + */ + if (need_check_commit_time) { + if (commit_time >= last_trans_commit_time) { + pr_err("JBD2: Invalid checksum found in transaction %u\n", + next_commit_ID); + err = -EFSBADCRC; + goto failed; + } + ignore_crc_mismatch: + /* + * It likely does not belong to same journal, + * just end this recovery with success. + */ + jbd2_debug(1, "JBD2: Invalid checksum ignored in transaction %u, likely stale data\n", + next_commit_ID); + goto done; + } + + /* + * Found an expected commit block: if checksums + * are present, verify them in PASS_SCAN; else not + * much to do other than move on to the next sequence + * number. + */ + if (jbd2_has_feature_checksum(journal)) { + struct commit_header *cbh = + (struct commit_header *)bh->b_data; + unsigned found_chksum = + be32_to_cpu(cbh->h_chksum[0]); + + if (info->end_transaction) { + journal->j_failed_commit = + info->end_transaction; + break; + } + + /* Neither checksum match nor unused? */ + if (!((crc32_sum == found_chksum && + cbh->h_chksum_type == + JBD2_CRC32_CHKSUM && + cbh->h_chksum_size == + JBD2_CRC32_CHKSUM_SIZE) || + (cbh->h_chksum_type == 0 && + cbh->h_chksum_size == 0 && + found_chksum == 0))) + goto chksum_error; + + crc32_sum = ~0; + goto chksum_ok; + } + + if (jbd2_commit_block_csum_verify(journal, bh->b_data)) + goto chksum_ok; + + if (jbd2_commit_block_csum_verify_partial(journal, + bh->b_data)) { + pr_notice("JBD2: Find incomplete commit block in transaction %u block %lu\n", + next_commit_ID, next_log_block); + goto chksum_ok; + } + +chksum_error: + if (commit_time < last_trans_commit_time) + goto ignore_crc_mismatch; + info->end_transaction = next_commit_ID; + info->head_block = head_block; + + if (!jbd2_has_feature_async_commit(journal)) { + journal->j_failed_commit = next_commit_ID; + break; + } + +chksum_ok: + last_trans_commit_time = commit_time; + head_block = next_log_block; + next_commit_ID++; + continue; + + case JBD2_REVOKE_BLOCK: + /* + * If we aren't in the SCAN or REVOKE pass, then we can + * just skip over this block. + */ + if (pass != PASS_REVOKE && pass != PASS_SCAN) + continue; + + /* + * Check revoke block crc in pass_scan, if csum verify + * failed, check commit block time later. + */ + if (pass == PASS_SCAN && + !jbd2_descriptor_block_csum_verify(journal, + bh->b_data)) { + jbd2_debug(1, "JBD2: invalid revoke block found in %lu\n", + next_log_block); + need_check_commit_time = true; + } + + err = scan_revoke_records(journal, pass, bh, + next_commit_ID, info); + if (err) + goto failed; + continue; + + default: + jbd2_debug(3, "Unrecognised magic %d, end of scan.\n", + blocktype); + goto done; + } + } + + done: + brelse(bh); + /* + * We broke out of the log scan loop: either we came to the + * known end of the log or we found an unexpected block in the + * log. If the latter happened, then we know that the "current" + * transaction marks the end of the valid log. + */ + + if (pass == PASS_SCAN) { + if (!info->end_transaction) + info->end_transaction = next_commit_ID; + if (!info->head_block) + info->head_block = head_block; + } else { + /* It's really bad news if different passes end up at + * different places (but possible due to IO errors). */ + if (info->end_transaction != next_commit_ID) { + printk(KERN_ERR "JBD2: recovery pass %d ended at " + "transaction %u, expected %u\n", + pass, next_commit_ID, info->end_transaction); + if (!success) + success = -EIO; + } + } + + if (jbd2_has_feature_fast_commit(journal) && pass != PASS_REVOKE) { + err = fc_do_one_pass(journal, info, pass); + if (err) + success = err; + } + + return success; + + failed: + brelse(bh); + return err; +} + +/* Scan a revoke record, marking all blocks mentioned as revoked. */ + +static int scan_revoke_records(journal_t *journal, enum passtype pass, + struct buffer_head *bh, tid_t sequence, + struct recovery_info *info) +{ + jbd2_journal_revoke_header_t *header; + int offset, max; + unsigned csum_size = 0; + __u32 rcount; + int record_len = 4; + + header = (jbd2_journal_revoke_header_t *) bh->b_data; + offset = sizeof(jbd2_journal_revoke_header_t); + rcount = be32_to_cpu(header->r_count); + + if (jbd2_journal_has_csum_v2or3(journal)) + csum_size = sizeof(struct jbd2_journal_block_tail); + if (rcount > journal->j_blocksize - csum_size) + return -EINVAL; + max = rcount; + + if (jbd2_has_feature_64bit(journal)) + record_len = 8; + + if (pass == PASS_SCAN) { + info->nr_revokes += (max - offset) / record_len; + return 0; + } + + while (offset + record_len <= max) { + unsigned long long blocknr; + int err; + + if (record_len == 4) + blocknr = be32_to_cpu(* ((__be32 *) (bh->b_data+offset))); + else + blocknr = be64_to_cpu(* ((__be64 *) (bh->b_data+offset))); + offset += record_len; + err = jbd2_journal_set_revoke(journal, blocknr, sequence); + if (err) + return err; + } + return 0; +} diff --git a/fs/jbd2/revoke.c b/fs/jbd2/revoke.c new file mode 100644 index 000000000..e4c2fbd38 --- /dev/null +++ b/fs/jbd2/revoke.c @@ -0,0 +1,745 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * linux/fs/jbd2/revoke.c + * + * Written by Stephen C. Tweedie , 2000 + * + * Copyright 2000 Red Hat corp --- All Rights Reserved + * + * Journal revoke routines for the generic filesystem journaling code; + * part of the ext2fs journaling system. + * + * Revoke is the mechanism used to prevent old log records for deleted + * metadata from being replayed on top of newer data using the same + * blocks. The revoke mechanism is used in two separate places: + * + * + Commit: during commit we write the entire list of the current + * transaction's revoked blocks to the journal + * + * + Recovery: during recovery we record the transaction ID of all + * revoked blocks. If there are multiple revoke records in the log + * for a single block, only the last one counts, and if there is a log + * entry for a block beyond the last revoke, then that log entry still + * gets replayed. + * + * We can get interactions between revokes and new log data within a + * single transaction: + * + * Block is revoked and then journaled: + * The desired end result is the journaling of the new block, so we + * cancel the revoke before the transaction commits. + * + * Block is journaled and then revoked: + * The revoke must take precedence over the write of the block, so we + * need either to cancel the journal entry or to write the revoke + * later in the log than the log block. In this case, we choose the + * latter: journaling a block cancels any revoke record for that block + * in the current transaction, so any revoke for that block in the + * transaction must have happened after the block was journaled and so + * the revoke must take precedence. + * + * Block is revoked and then written as data: + * The data write is allowed to succeed, but the revoke is _not_ + * cancelled. We still need to prevent old log records from + * overwriting the new data. We don't even need to clear the revoke + * bit here. + * + * We cache revoke status of a buffer in the current transaction in b_states + * bits. As the name says, revokevalid flag indicates that the cached revoke + * status of a buffer is valid and we can rely on the cached status. + * + * Revoke information on buffers is a tri-state value: + * + * RevokeValid clear: no cached revoke status, need to look it up + * RevokeValid set, Revoked clear: + * buffer has not been revoked, and cancel_revoke + * need do nothing. + * RevokeValid set, Revoked set: + * buffer has been revoked. + * + * Locking rules: + * We keep two hash tables of revoke records. One hashtable belongs to the + * running transaction (is pointed to by journal->j_revoke), the other one + * belongs to the committing transaction. Accesses to the second hash table + * happen only from the kjournald and no other thread touches this table. Also + * journal_switch_revoke_table() which switches which hashtable belongs to the + * running and which to the committing transaction is called only from + * kjournald. Therefore we need no locks when accessing the hashtable belonging + * to the committing transaction. + * + * All users operating on the hash table belonging to the running transaction + * have a handle to the transaction. Therefore they are safe from kjournald + * switching hash tables under them. For operations on the lists of entries in + * the hash table j_revoke_lock is used. + * + * Finally, also replay code uses the hash tables but at this moment no one else + * can touch them (filesystem isn't mounted yet) and hence no locking is + * needed. + */ + +#ifndef __KERNEL__ +#include "jfs_user.h" +#else +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +static struct kmem_cache *jbd2_revoke_record_cache; +static struct kmem_cache *jbd2_revoke_table_cache; + +/* Each revoke record represents one single revoked block. During + journal replay, this involves recording the transaction ID of the + last transaction to revoke this block. */ + +struct jbd2_revoke_record_s +{ + struct list_head hash; + tid_t sequence; /* Used for recovery only */ + unsigned long long blocknr; +}; + + +/* The revoke table is just a simple hash table of revoke records. */ +struct jbd2_revoke_table_s +{ + /* It is conceivable that we might want a larger hash table + * for recovery. Must be a power of two. */ + int hash_size; + int hash_shift; + struct list_head *hash_table; +}; + + +#ifdef __KERNEL__ +static void write_one_revoke_record(transaction_t *, + struct list_head *, + struct buffer_head **, int *, + struct jbd2_revoke_record_s *); +static void flush_descriptor(journal_t *, struct buffer_head *, int); +#endif + +/* Utility functions to maintain the revoke table */ + +static inline int hash(journal_t *journal, unsigned long long block) +{ + return hash_64(block, journal->j_revoke->hash_shift); +} + +static int insert_revoke_hash(journal_t *journal, unsigned long long blocknr, + tid_t seq) +{ + struct list_head *hash_list; + struct jbd2_revoke_record_s *record; + gfp_t gfp_mask = GFP_NOFS; + + if (journal_oom_retry) + gfp_mask |= __GFP_NOFAIL; + record = kmem_cache_alloc(jbd2_revoke_record_cache, gfp_mask); + if (!record) + return -ENOMEM; + + record->sequence = seq; + record->blocknr = blocknr; + hash_list = &journal->j_revoke->hash_table[hash(journal, blocknr)]; + spin_lock(&journal->j_revoke_lock); + list_add(&record->hash, hash_list); + spin_unlock(&journal->j_revoke_lock); + return 0; +} + +/* Find a revoke record in the journal's hash table. */ + +static struct jbd2_revoke_record_s *find_revoke_record(journal_t *journal, + unsigned long long blocknr) +{ + struct list_head *hash_list; + struct jbd2_revoke_record_s *record; + + hash_list = &journal->j_revoke->hash_table[hash(journal, blocknr)]; + + spin_lock(&journal->j_revoke_lock); + record = (struct jbd2_revoke_record_s *) hash_list->next; + while (&(record->hash) != hash_list) { + if (record->blocknr == blocknr) { + spin_unlock(&journal->j_revoke_lock); + return record; + } + record = (struct jbd2_revoke_record_s *) record->hash.next; + } + spin_unlock(&journal->j_revoke_lock); + return NULL; +} + +void jbd2_journal_destroy_revoke_record_cache(void) +{ + kmem_cache_destroy(jbd2_revoke_record_cache); + jbd2_revoke_record_cache = NULL; +} + +void jbd2_journal_destroy_revoke_table_cache(void) +{ + kmem_cache_destroy(jbd2_revoke_table_cache); + jbd2_revoke_table_cache = NULL; +} + +int __init jbd2_journal_init_revoke_record_cache(void) +{ + J_ASSERT(!jbd2_revoke_record_cache); + jbd2_revoke_record_cache = KMEM_CACHE(jbd2_revoke_record_s, + SLAB_HWCACHE_ALIGN|SLAB_TEMPORARY); + + if (!jbd2_revoke_record_cache) { + pr_emerg("JBD2: failed to create revoke_record cache\n"); + return -ENOMEM; + } + return 0; +} + +int __init jbd2_journal_init_revoke_table_cache(void) +{ + J_ASSERT(!jbd2_revoke_table_cache); + jbd2_revoke_table_cache = KMEM_CACHE(jbd2_revoke_table_s, + SLAB_TEMPORARY); + if (!jbd2_revoke_table_cache) { + pr_emerg("JBD2: failed to create revoke_table cache\n"); + return -ENOMEM; + } + return 0; +} + +struct jbd2_revoke_table_s *jbd2_journal_init_revoke_table(int hash_size) +{ + int shift = 0; + int tmp = hash_size; + struct jbd2_revoke_table_s *table; + + table = kmem_cache_alloc(jbd2_revoke_table_cache, GFP_KERNEL); + if (!table) + goto out; + + while((tmp >>= 1UL) != 0UL) + shift++; + + table->hash_size = hash_size; + table->hash_shift = shift; + table->hash_table = + kvmalloc_objs(struct list_head, hash_size); + if (!table->hash_table) { + kmem_cache_free(jbd2_revoke_table_cache, table); + table = NULL; + goto out; + } + + for (tmp = 0; tmp < hash_size; tmp++) + INIT_LIST_HEAD(&table->hash_table[tmp]); + +out: + return table; +} + +void jbd2_journal_destroy_revoke_table(struct jbd2_revoke_table_s *table) +{ + int i; + struct list_head *hash_list; + + for (i = 0; i < table->hash_size; i++) { + hash_list = &table->hash_table[i]; + J_ASSERT(list_empty(hash_list)); + } + + kvfree(table->hash_table); + kmem_cache_free(jbd2_revoke_table_cache, table); +} + +/* Initialise the revoke table for a given journal to a given size. */ +int jbd2_journal_init_revoke(journal_t *journal, int hash_size) +{ + J_ASSERT(journal->j_revoke_table[0] == NULL); + J_ASSERT(is_power_of_2(hash_size)); + + journal->j_revoke_table[0] = jbd2_journal_init_revoke_table(hash_size); + if (!journal->j_revoke_table[0]) + goto fail0; + + journal->j_revoke_table[1] = jbd2_journal_init_revoke_table(hash_size); + if (!journal->j_revoke_table[1]) + goto fail1; + + journal->j_revoke = journal->j_revoke_table[1]; + + spin_lock_init(&journal->j_revoke_lock); + + return 0; + +fail1: + jbd2_journal_destroy_revoke_table(journal->j_revoke_table[0]); + journal->j_revoke_table[0] = NULL; +fail0: + return -ENOMEM; +} + +/* Destroy a journal's revoke table. The table must already be empty! */ +void jbd2_journal_destroy_revoke(journal_t *journal) +{ + journal->j_revoke = NULL; + if (journal->j_revoke_table[0]) + jbd2_journal_destroy_revoke_table(journal->j_revoke_table[0]); + if (journal->j_revoke_table[1]) + jbd2_journal_destroy_revoke_table(journal->j_revoke_table[1]); +} + + +#ifdef __KERNEL__ + +/* + * jbd2_journal_revoke: revoke a given buffer_head from the journal. This + * prevents the block from being replayed during recovery if we take a + * crash after this current transaction commits. Any subsequent + * metadata writes of the buffer in this transaction cancel the + * revoke. + * + * Note that this call may block --- it is up to the caller to make + * sure that there are no further calls to journal_write_metadata + * before the revoke is complete. In ext3, this implies calling the + * revoke before clearing the block bitmap when we are deleting + * metadata. + * + * Revoke performs a jbd2_journal_forget on any buffer_head passed in as a + * parameter, but does _not_ forget the buffer_head if the bh was only + * found implicitly. + * + * bh_in may not be a journalled buffer - it may have come off + * the hash tables without an attached journal_head. + * + * If bh_in is non-zero, jbd2_journal_revoke() will decrement its b_count + * by one. + */ + +int jbd2_journal_revoke(handle_t *handle, unsigned long long blocknr, + struct buffer_head *bh_in) +{ + struct buffer_head *bh = NULL; + journal_t *journal; + struct block_device *bdev; + int err; + + might_sleep(); + if (bh_in) + BUFFER_TRACE(bh_in, "enter"); + + journal = handle->h_transaction->t_journal; + if (!jbd2_journal_set_features(journal, 0, 0, JBD2_FEATURE_INCOMPAT_REVOKE)){ + J_ASSERT (!"Cannot set revoke feature!"); + return -EINVAL; + } + + bdev = journal->j_fs_dev; + bh = bh_in; + + if (!bh) { + bh = __find_get_block_nonatomic(bdev, blocknr, + journal->j_blocksize); + if (bh) + BUFFER_TRACE(bh, "found on hash"); + } +#ifdef JBD2_EXPENSIVE_CHECKING + else { + struct buffer_head *bh2; + + /* If there is a different buffer_head lying around in + * memory anywhere... */ + bh2 = __find_get_block_nonatomic(bdev, blocknr, + journal->j_blocksize); + if (bh2) { + /* ... and it has RevokeValid status... */ + if (bh2 != bh && buffer_revokevalid(bh2)) + /* ...then it better be revoked too, + * since it's illegal to create a revoke + * record against a buffer_head which is + * not marked revoked --- that would + * risk missing a subsequent revoke + * cancel. */ + J_ASSERT_BH(bh2, buffer_revoked(bh2)); + put_bh(bh2); + } + } +#endif + + if (WARN_ON_ONCE(handle->h_revoke_credits <= 0)) { + if (!bh_in) + brelse(bh); + return -EIO; + } + /* We really ought not ever to revoke twice in a row without + first having the revoke cancelled: it's illegal to free a + block twice without allocating it in between! */ + if (bh) { + if (!J_EXPECT_BH(bh, !buffer_revoked(bh), + "inconsistent data on disk")) { + if (!bh_in) + brelse(bh); + return -EIO; + } + set_buffer_revoked(bh); + set_buffer_revokevalid(bh); + if (bh_in) { + BUFFER_TRACE(bh_in, "call jbd2_journal_forget"); + jbd2_journal_forget(handle, bh_in); + } else { + BUFFER_TRACE(bh, "call brelse"); + __brelse(bh); + } + } + handle->h_revoke_credits--; + + jbd2_debug(2, "insert revoke for block %llu, bh_in=%p\n",blocknr, bh_in); + err = insert_revoke_hash(journal, blocknr, + handle->h_transaction->t_tid); + BUFFER_TRACE(bh_in, "exit"); + return err; +} + +/* + * Cancel an outstanding revoke. For use only internally by the + * journaling code (called from jbd2_journal_get_write_access). + * + * We trust buffer_revoked() on the buffer if the buffer is already + * being journaled: if there is no revoke pending on the buffer, then we + * don't do anything here. + * + * This would break if it were possible for a buffer to be revoked and + * discarded, and then reallocated within the same transaction. In such + * a case we would have lost the revoked bit, but when we arrived here + * the second time we would still have a pending revoke to cancel. So, + * do not trust the Revoked bit on buffers unless RevokeValid is also + * set. + */ +void jbd2_journal_cancel_revoke(handle_t *handle, struct journal_head *jh) +{ + struct jbd2_revoke_record_s *record; + journal_t *journal = handle->h_transaction->t_journal; + int need_cancel; + struct buffer_head *bh = jh2bh(jh); + struct address_space *bh_mapping = bh->b_folio->mapping; + + jbd2_debug(4, "journal_head %p, cancelling revoke\n", jh); + + /* Is the existing Revoke bit valid? If so, we trust it, and + * only perform the full cancel if the revoke bit is set. If + * not, we can't trust the revoke bit, and we need to do the + * full search for a revoke record. */ + if (test_set_buffer_revokevalid(bh)) { + need_cancel = test_clear_buffer_revoked(bh); + } else { + need_cancel = 1; + clear_buffer_revoked(bh); + } + + if (need_cancel) { + record = find_revoke_record(journal, bh->b_blocknr); + if (record) { + jbd2_debug(4, "cancelled existing revoke on " + "blocknr %llu\n", (unsigned long long)bh->b_blocknr); + spin_lock(&journal->j_revoke_lock); + list_del(&record->hash); + spin_unlock(&journal->j_revoke_lock); + kmem_cache_free(jbd2_revoke_record_cache, record); + } + } + +#ifdef JBD2_EXPENSIVE_CHECKING + /* There better not be one left behind by now! */ + record = find_revoke_record(journal, bh->b_blocknr); + J_ASSERT_JH(jh, record == NULL); +#endif + + /* Finally, have we just cleared revoke on an unhashed + * buffer_head? If so, we'd better make sure we clear the + * revoked status on any hashed alias too, otherwise the revoke + * state machine will get very upset later on. */ + if (need_cancel && !sb_is_blkdev_sb(bh_mapping->host->i_sb)) { + struct buffer_head *bh2; + + bh2 = __find_get_block_nonatomic(bh->b_bdev, bh->b_blocknr, + bh->b_size); + if (bh2) { + WARN_ON_ONCE(bh2 == bh); + clear_buffer_revoked(bh2); + __brelse(bh2); + } + } +} + +/* + * jbd2_clear_buffer_revoked_flags clears revoked flag of buffers in + * revoke table to reflect there is no revoked buffers in the next + * transaction which is going to be started. + */ +void jbd2_clear_buffer_revoked_flags(journal_t *journal) +{ + struct jbd2_revoke_table_s *revoke = journal->j_revoke; + int i = 0; + + for (i = 0; i < revoke->hash_size; i++) { + struct list_head *hash_list; + struct list_head *list_entry; + hash_list = &revoke->hash_table[i]; + + list_for_each(list_entry, hash_list) { + struct jbd2_revoke_record_s *record; + struct buffer_head *bh; + record = (struct jbd2_revoke_record_s *)list_entry; + bh = __find_get_block_nonatomic(journal->j_fs_dev, + record->blocknr, + journal->j_blocksize); + if (bh) { + clear_buffer_revoked(bh); + __brelse(bh); + } + } + } +} + +/* jbd2_journal_switch_revoke_table table select j_revoke for next + * transaction we do not want to suspend any processing until all + * revokes are written -bzzz + */ +void jbd2_journal_switch_revoke_table(journal_t *journal) +{ + int i; + + if (journal->j_revoke == journal->j_revoke_table[0]) + journal->j_revoke = journal->j_revoke_table[1]; + else + journal->j_revoke = journal->j_revoke_table[0]; + + for (i = 0; i < journal->j_revoke->hash_size; i++) + INIT_LIST_HEAD(&journal->j_revoke->hash_table[i]); +} + +/* + * Write revoke records to the journal for all entries in the current + * revoke hash, deleting the entries as we go. + */ +void jbd2_journal_write_revoke_records(transaction_t *transaction, + struct list_head *log_bufs) +{ + journal_t *journal = transaction->t_journal; + struct buffer_head *descriptor; + struct jbd2_revoke_record_s *record; + struct jbd2_revoke_table_s *revoke; + struct list_head *hash_list; + int i, offset, count; + + descriptor = NULL; + offset = 0; + count = 0; + + /* select revoke table for committing transaction */ + revoke = journal->j_revoke == journal->j_revoke_table[0] ? + journal->j_revoke_table[1] : journal->j_revoke_table[0]; + + for (i = 0; i < revoke->hash_size; i++) { + hash_list = &revoke->hash_table[i]; + + while (!list_empty(hash_list)) { + record = (struct jbd2_revoke_record_s *) + hash_list->next; + write_one_revoke_record(transaction, log_bufs, + &descriptor, &offset, record); + count++; + list_del(&record->hash); + kmem_cache_free(jbd2_revoke_record_cache, record); + } + } + if (descriptor) + flush_descriptor(journal, descriptor, offset); + jbd2_debug(1, "Wrote %d revoke records\n", count); +} + +/* + * Write out one revoke record. We need to create a new descriptor + * block if the old one is full or if we have not already created one. + */ + +static void write_one_revoke_record(transaction_t *transaction, + struct list_head *log_bufs, + struct buffer_head **descriptorp, + int *offsetp, + struct jbd2_revoke_record_s *record) +{ + journal_t *journal = transaction->t_journal; + int csum_size = 0; + struct buffer_head *descriptor; + int sz, offset; + + /* If we are already aborting, this all becomes a noop. We + still need to go round the loop in + jbd2_journal_write_revoke_records in order to free all of the + revoke records: only the IO to the journal is omitted. */ + if (is_journal_aborted(journal)) + return; + + descriptor = *descriptorp; + offset = *offsetp; + + /* Do we need to leave space at the end for a checksum? */ + if (jbd2_journal_has_csum_v2or3(journal)) + csum_size = sizeof(struct jbd2_journal_block_tail); + + if (jbd2_has_feature_64bit(journal)) + sz = 8; + else + sz = 4; + + /* Make sure we have a descriptor with space left for the record */ + if (descriptor) { + if (offset + sz > journal->j_blocksize - csum_size) { + flush_descriptor(journal, descriptor, offset); + descriptor = NULL; + } + } + + if (!descriptor) { + descriptor = jbd2_journal_get_descriptor_buffer(transaction, + JBD2_REVOKE_BLOCK); + if (!descriptor) + return; + + /* Record it so that we can wait for IO completion later */ + BUFFER_TRACE(descriptor, "file in log_bufs"); + jbd2_file_log_bh(log_bufs, descriptor); + + offset = sizeof(jbd2_journal_revoke_header_t); + *descriptorp = descriptor; + } + + if (jbd2_has_feature_64bit(journal)) + * ((__be64 *)(&descriptor->b_data[offset])) = + cpu_to_be64(record->blocknr); + else + * ((__be32 *)(&descriptor->b_data[offset])) = + cpu_to_be32(record->blocknr); + offset += sz; + + *offsetp = offset; +} + +/* + * Flush a revoke descriptor out to the journal. If we are aborting, + * this is a noop; otherwise we are generating a buffer which needs to + * be waited for during commit, so it has to go onto the appropriate + * journal buffer list. + */ + +static void flush_descriptor(journal_t *journal, + struct buffer_head *descriptor, + int offset) +{ + jbd2_journal_revoke_header_t *header; + + if (is_journal_aborted(journal)) + return; + + header = (jbd2_journal_revoke_header_t *)descriptor->b_data; + header->r_count = cpu_to_be32(offset); + jbd2_descriptor_block_csum_set(journal, descriptor); + + set_buffer_jwrite(descriptor); + BUFFER_TRACE(descriptor, "write"); + set_buffer_dirty(descriptor); + write_dirty_buffer(descriptor, JBD2_JOURNAL_REQ_FLAGS); +} +#endif + +/* + * Revoke support for recovery. + * + * Recovery needs to be able to: + * + * record all revoke records, including the tid of the latest instance + * of each revoke in the journal + * + * check whether a given block in a given transaction should be replayed + * (ie. has not been revoked by a revoke record in that or a subsequent + * transaction) + * + * empty the revoke table after recovery. + */ + +/* + * First, setting revoke records. We create a new revoke record for + * every block ever revoked in the log as we scan it for recovery, and + * we update the existing records if we find multiple revokes for a + * single block. + */ + +int jbd2_journal_set_revoke(journal_t *journal, + unsigned long long blocknr, + tid_t sequence) +{ + struct jbd2_revoke_record_s *record; + + record = find_revoke_record(journal, blocknr); + if (record) { + /* If we have multiple occurrences, only record the + * latest sequence number in the hashed record */ + if (tid_gt(sequence, record->sequence)) + record->sequence = sequence; + return 0; + } + return insert_revoke_hash(journal, blocknr, sequence); +} + +/* + * Test revoke records. For a given block referenced in the log, has + * that block been revoked? A revoke record with a given transaction + * sequence number revokes all blocks in that transaction and earlier + * ones, but later transactions still need replayed. + */ + +int jbd2_journal_test_revoke(journal_t *journal, + unsigned long long blocknr, + tid_t sequence) +{ + struct jbd2_revoke_record_s *record; + + record = find_revoke_record(journal, blocknr); + if (!record) + return 0; + if (tid_gt(sequence, record->sequence)) + return 0; + return 1; +} + +/* + * Finally, once recovery is over, we need to clear the revoke table so + * that it can be reused by the running filesystem. + */ + +void jbd2_journal_clear_revoke(journal_t *journal) +{ + int i; + struct list_head *hash_list; + struct jbd2_revoke_record_s *record; + struct jbd2_revoke_table_s *revoke; + + revoke = journal->j_revoke; + + for (i = 0; i < revoke->hash_size; i++) { + hash_list = &revoke->hash_table[i]; + while (!list_empty(hash_list)) { + record = (struct jbd2_revoke_record_s*) hash_list->next; + list_del(&record->hash); + kmem_cache_free(jbd2_revoke_record_cache, record); + } + } +} diff --git a/fs/jbd2/transaction.c b/fs/jbd2/transaction.c new file mode 100644 index 000000000..5cc7d097b --- /dev/null +++ b/fs/jbd2/transaction.c @@ -0,0 +1,2836 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * linux/fs/jbd2/transaction.c + * + * Written by Stephen C. Tweedie , 1998 + * + * Copyright 1998 Red Hat corp --- All Rights Reserved + * + * Generic filesystem transaction handling code; part of the ext2fs + * journaling system. + * + * This file manages transactions (compound commits managed by the + * journaling code) and handles (individual atomic operations by the + * filesystem). + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +static void __jbd2_journal_temp_unlink_buffer(struct journal_head *jh); +static void __jbd2_journal_unfile_buffer(struct journal_head *jh); + +static struct kmem_cache *transaction_cache; +int __init jbd2_journal_init_transaction_cache(void) +{ + J_ASSERT(!transaction_cache); + transaction_cache = kmem_cache_create("jbd2_transaction_s", + sizeof(transaction_t), + 0, + SLAB_HWCACHE_ALIGN|SLAB_TEMPORARY, + NULL); + if (!transaction_cache) { + pr_emerg("JBD2: failed to create transaction cache\n"); + return -ENOMEM; + } + return 0; +} + +void jbd2_journal_destroy_transaction_cache(void) +{ + kmem_cache_destroy(transaction_cache); + transaction_cache = NULL; +} + +void jbd2_journal_free_transaction(transaction_t *transaction) +{ + if (unlikely(ZERO_OR_NULL_PTR(transaction))) + return; + kmem_cache_free(transaction_cache, transaction); +} + +/* + * jbd2_get_transaction: obtain a new transaction_t object. + * + * Simply initialise a new transaction. Initialize it in + * RUNNING state and add it to the current journal (which should not + * have an existing running transaction: we only make a new transaction + * once we have started to commit the old one). + * + * Preconditions: + * The journal MUST be locked. We don't perform atomic mallocs on the + * new transaction and we can't block without protecting against other + * processes trying to touch the journal while it is in transition. + * + */ + +static void jbd2_get_transaction(journal_t *journal, + transaction_t *transaction) +{ + transaction->t_journal = journal; + transaction->t_state = T_RUNNING; + transaction->t_start_time = ktime_get(); + transaction->t_tid = journal->j_transaction_sequence++; + transaction->t_expires = jiffies + journal->j_commit_interval; + atomic_set(&transaction->t_updates, 0); + atomic_set(&transaction->t_outstanding_credits, + journal->j_transaction_overhead_buffers + + atomic_read(&journal->j_reserved_credits)); + atomic_set(&transaction->t_outstanding_revokes, 0); + atomic_set(&transaction->t_handle_count, 0); + INIT_LIST_HEAD(&transaction->t_inode_list); + + /* Set up the commit timer for the new transaction. */ + journal->j_commit_timer.expires = round_jiffies_up(transaction->t_expires); + add_timer(&journal->j_commit_timer); + + J_ASSERT(journal->j_running_transaction == NULL); + journal->j_running_transaction = transaction; + transaction->t_max_wait = 0; + transaction->t_start = jiffies; + transaction->t_requested = 0; +} + +/* + * Handle management. + * + * A handle_t is an object which represents a single atomic update to a + * filesystem, and which tracks all of the modifications which form part + * of that one update. + */ + +/* + * t_max_wait is carefully updated here with use of atomic compare exchange. + * Note that there could be multiplre threads trying to do this simultaneously + * hence using cmpxchg to avoid any use of locks in this case. + */ +static inline void update_t_max_wait(transaction_t *transaction, + unsigned long ts) +{ + unsigned long oldts, newts; + + if (time_after(transaction->t_start, ts)) { + newts = jbd2_time_diff(ts, transaction->t_start); + oldts = READ_ONCE(transaction->t_max_wait); + while (oldts < newts) + oldts = cmpxchg(&transaction->t_max_wait, oldts, newts); + } +} + +/* + * Wait until running transaction passes to T_FLUSH state and new transaction + * can thus be started. Also starts the commit if needed. The function expects + * running transaction to exist and releases j_state_lock. + */ +static void wait_transaction_locked(journal_t *journal) + __releases(journal->j_state_lock) +{ + DEFINE_WAIT(wait); + int need_to_start; + tid_t tid = journal->j_running_transaction->t_tid; + + prepare_to_wait_exclusive(&journal->j_wait_transaction_locked, &wait, + TASK_UNINTERRUPTIBLE); + need_to_start = !tid_geq(journal->j_commit_request, tid); + read_unlock(&journal->j_state_lock); + if (need_to_start) + jbd2_log_start_commit(journal, tid); + jbd2_might_wait_for_commit(journal); + schedule(); + finish_wait(&journal->j_wait_transaction_locked, &wait); +} + +/* + * Wait until running transaction transitions from T_SWITCH to T_FLUSH + * state and new transaction can thus be started. The function releases + * j_state_lock. + */ +static void wait_transaction_switching(journal_t *journal) + __releases(journal->j_state_lock) +{ + DEFINE_WAIT(wait); + + if (WARN_ON(!journal->j_running_transaction || + journal->j_running_transaction->t_state != T_SWITCH)) { + read_unlock(&journal->j_state_lock); + return; + } + prepare_to_wait_exclusive(&journal->j_wait_transaction_locked, &wait, + TASK_UNINTERRUPTIBLE); + read_unlock(&journal->j_state_lock); + /* + * We don't call jbd2_might_wait_for_commit() here as there's no + * waiting for outstanding handles happening anymore in T_SWITCH state + * and handling of reserved handles actually relies on that for + * correctness. + */ + schedule(); + finish_wait(&journal->j_wait_transaction_locked, &wait); +} + +static void sub_reserved_credits(journal_t *journal, int blocks) +{ + atomic_sub(blocks, &journal->j_reserved_credits); + wake_up(&journal->j_wait_reserved); +} + +/* Maximum number of blocks for user transaction payload */ +static int jbd2_max_user_trans_buffers(journal_t *journal) +{ + return journal->j_max_transaction_buffers - + journal->j_transaction_overhead_buffers; +} + +/* + * Wait until we can add credits for handle to the running transaction. Called + * with j_state_lock held for reading. Returns 0 if handle joined the running + * transaction. Returns 1 if we had to wait, j_state_lock is dropped, and + * caller must retry. + * + * Note: because j_state_lock may be dropped depending on the return + * value, we need to fake out sparse so ti doesn't complain about a + * locking imbalance. Callers of add_transaction_credits will need to + * make a similar accomodation. + */ +static int add_transaction_credits(journal_t *journal, int blocks, + int rsv_blocks) +__must_hold(&journal->j_state_lock) +{ + transaction_t *t = journal->j_running_transaction; + int needed; + int total = blocks + rsv_blocks; + + /* + * If the current transaction is locked down for commit, wait + * for the lock to be released. + */ + if (t->t_state != T_RUNNING) { + WARN_ON_ONCE(t->t_state >= T_FLUSH); + wait_transaction_locked(journal); + __acquire(&journal->j_state_lock); /* fake out sparse */ + return 1; + } + + /* + * If there is not enough space left in the log to write all + * potential buffers requested by this operation, we need to + * stall pending a log checkpoint to free some more log space. + */ + needed = atomic_add_return(total, &t->t_outstanding_credits); + if (needed > journal->j_max_transaction_buffers) { + /* + * If the current transaction is already too large, + * then start to commit it: we can then go back and + * attach this handle to a new transaction. + */ + atomic_sub(total, &t->t_outstanding_credits); + + /* + * Is the number of reserved credits in the current transaction too + * big to fit this handle? Wait until reserved credits are freed. + */ + if (atomic_read(&journal->j_reserved_credits) + total > + jbd2_max_user_trans_buffers(journal)) { + read_unlock(&journal->j_state_lock); + jbd2_might_wait_for_commit(journal); + wait_event(journal->j_wait_reserved, + atomic_read(&journal->j_reserved_credits) + total <= + jbd2_max_user_trans_buffers(journal)); + __acquire(&journal->j_state_lock); /* fake out sparse */ + return 1; + } + + wait_transaction_locked(journal); + __acquire(&journal->j_state_lock); /* fake out sparse */ + return 1; + } + + /* + * The commit code assumes that it can get enough log space + * without forcing a checkpoint. This is *critical* for + * correctness: a checkpoint of a buffer which is also + * associated with a committing transaction creates a deadlock, + * so commit simply cannot force through checkpoints. + * + * We must therefore ensure the necessary space in the journal + * *before* starting to dirty potentially checkpointed buffers + * in the new transaction. + */ + if (jbd2_log_space_left(journal) < journal->j_max_transaction_buffers) { + atomic_sub(total, &t->t_outstanding_credits); + read_unlock(&journal->j_state_lock); + jbd2_might_wait_for_commit(journal); + write_lock(&journal->j_state_lock); + if (jbd2_log_space_left(journal) < + journal->j_max_transaction_buffers) + __jbd2_log_wait_for_space(journal); + write_unlock(&journal->j_state_lock); + __acquire(&journal->j_state_lock); /* fake out sparse */ + return 1; + } + + /* No reservation? We are done... */ + if (!rsv_blocks) + return 0; + + needed = atomic_add_return(rsv_blocks, &journal->j_reserved_credits); + /* We allow at most half of a transaction to be reserved */ + if (needed > jbd2_max_user_trans_buffers(journal) / 2) { + sub_reserved_credits(journal, rsv_blocks); + atomic_sub(total, &t->t_outstanding_credits); + read_unlock(&journal->j_state_lock); + jbd2_might_wait_for_commit(journal); + wait_event(journal->j_wait_reserved, + atomic_read(&journal->j_reserved_credits) + rsv_blocks + <= jbd2_max_user_trans_buffers(journal) / 2); + __acquire(&journal->j_state_lock); /* fake out sparse */ + return 1; + } + return 0; +} + +/* + * start_this_handle: Given a handle, deal with any locking or stalling + * needed to make sure that there is enough journal space for the handle + * to begin. Attach the handle to a transaction and set up the + * transaction's buffer credits. + */ + +static int start_this_handle(journal_t *journal, handle_t *handle, + gfp_t gfp_mask) +{ + transaction_t *transaction, *new_transaction = NULL; + int blocks = handle->h_total_credits; + int rsv_blocks = 0; + unsigned long ts = jiffies; + + if (handle->h_rsv_handle) + rsv_blocks = handle->h_rsv_handle->h_total_credits; + + /* + * Limit the number of reserved credits to 1/2 of maximum transaction + * size and limit the number of total credits to not exceed maximum + * transaction size per operation. + */ + if (rsv_blocks > jbd2_max_user_trans_buffers(journal) / 2 || + rsv_blocks + blocks > jbd2_max_user_trans_buffers(journal)) { + printk(KERN_ERR "JBD2: %s wants too many credits " + "credits:%d rsv_credits:%d max:%d\n", + current->comm, blocks, rsv_blocks, + jbd2_max_user_trans_buffers(journal)); + WARN_ON(1); + return -ENOSPC; + } + +alloc_transaction: + /* + * This check is racy but it is just an optimization of allocating new + * transaction early if there are high chances we'll need it. If we + * guess wrong, we'll retry or free unused transaction. + */ + if (!data_race(journal->j_running_transaction)) { + /* + * If __GFP_FS is not present, then we may be being called from + * inside the fs writeback layer, so we MUST NOT fail. + */ + if ((gfp_mask & __GFP_FS) == 0) + gfp_mask |= __GFP_NOFAIL; + new_transaction = kmem_cache_zalloc(transaction_cache, + gfp_mask); + if (!new_transaction) + return -ENOMEM; + } + + jbd2_debug(3, "New handle %p going live.\n", handle); + + /* + * We need to hold j_state_lock until t_updates has been incremented, + * for proper journal barrier handling + */ +repeat: + read_lock(&journal->j_state_lock); + BUG_ON(journal->j_flags & JBD2_UNMOUNT); + if (is_journal_aborted(journal) || + (journal->j_errno != 0 && !(journal->j_flags & JBD2_ACK_ERR))) { + read_unlock(&journal->j_state_lock); + jbd2_journal_free_transaction(new_transaction); + return -EROFS; + } + + /* + * Wait on the journal's transaction barrier if necessary. Specifically + * we allow reserved handles to proceed because otherwise commit could + * deadlock on page writeback not being able to complete. + */ + if (!handle->h_reserved && journal->j_barrier_count) { + read_unlock(&journal->j_state_lock); + wait_event(journal->j_wait_transaction_locked, + journal->j_barrier_count == 0); + goto repeat; + } + + if (!journal->j_running_transaction) { + read_unlock(&journal->j_state_lock); + if (!new_transaction) + goto alloc_transaction; + write_lock(&journal->j_state_lock); + if (!journal->j_running_transaction && + (handle->h_reserved || !journal->j_barrier_count)) { + jbd2_get_transaction(journal, new_transaction); + new_transaction = NULL; + } + write_unlock(&journal->j_state_lock); + goto repeat; + } + + transaction = journal->j_running_transaction; + + if (!handle->h_reserved) { + /* We may have dropped j_state_lock - restart in that case */ + if (add_transaction_credits(journal, blocks, rsv_blocks)) { + /* + * add_transaction_credits releases + * j_state_lock on a non-zero return + */ + __release(&journal->j_state_lock); + goto repeat; + } + } else { + /* + * We have handle reserved so we are allowed to join T_LOCKED + * transaction and we don't have to check for transaction size + * and journal space. But we still have to wait while running + * transaction is being switched to a committing one as it + * won't wait for any handles anymore. + */ + if (transaction->t_state == T_SWITCH) { + wait_transaction_switching(journal); + goto repeat; + } + sub_reserved_credits(journal, blocks); + handle->h_reserved = 0; + } + + /* OK, account for the buffers that this operation expects to + * use and add the handle to the running transaction. + */ + update_t_max_wait(transaction, ts); + handle->h_transaction = transaction; + handle->h_requested_credits = blocks; + handle->h_revoke_credits_requested = handle->h_revoke_credits; + handle->h_start_jiffies = jiffies; + atomic_inc(&transaction->t_updates); + atomic_inc(&transaction->t_handle_count); + jbd2_debug(4, "Handle %p given %d credits (total %d, free %lu)\n", + handle, blocks, + atomic_read(&transaction->t_outstanding_credits), + jbd2_log_space_left(journal)); + read_unlock(&journal->j_state_lock); + current->journal_info = handle; + + rwsem_acquire_read(&journal->j_trans_commit_map, 0, 1, _THIS_IP_); + jbd2_journal_free_transaction(new_transaction); + /* + * Ensure that no allocations done while the transaction is open are + * going to recurse back to the fs layer. + */ + handle->saved_alloc_context = memalloc_nofs_save(); + return 0; +} + +/* Allocate a new handle. This should probably be in a slab... */ +static handle_t *new_handle(int nblocks) +{ + handle_t *handle = jbd2_alloc_handle(GFP_NOFS); + if (!handle) + return NULL; + handle->h_total_credits = nblocks; + handle->h_ref = 1; + + return handle; +} + +handle_t *jbd2__journal_start(journal_t *journal, int nblocks, int rsv_blocks, + int revoke_records, gfp_t gfp_mask, + unsigned int type, unsigned int line_no) +{ + handle_t *handle = journal_current_handle(); + int err; + + if (!journal) + return ERR_PTR(-EROFS); + + if (handle) { + if (WARN_ON_ONCE(handle->h_transaction->t_journal != journal)) + return ERR_PTR(-EINVAL); + handle->h_ref++; + return handle; + } + + nblocks += DIV_ROUND_UP(revoke_records, + journal->j_revoke_records_per_block); + handle = new_handle(nblocks); + if (!handle) + return ERR_PTR(-ENOMEM); + if (rsv_blocks) { + handle_t *rsv_handle; + + rsv_handle = new_handle(rsv_blocks); + if (!rsv_handle) { + jbd2_free_handle(handle); + return ERR_PTR(-ENOMEM); + } + rsv_handle->h_reserved = 1; + rsv_handle->h_journal = journal; + handle->h_rsv_handle = rsv_handle; + } + handle->h_revoke_credits = revoke_records; + + err = start_this_handle(journal, handle, gfp_mask); + if (err < 0) { + if (handle->h_rsv_handle) + jbd2_free_handle(handle->h_rsv_handle); + jbd2_free_handle(handle); + return ERR_PTR(err); + } + handle->h_type = type; + handle->h_line_no = line_no; + trace_jbd2_handle_start(journal->j_fs_dev->bd_dev, + handle->h_transaction->t_tid, type, + line_no, nblocks); + + return handle; +} +EXPORT_SYMBOL(jbd2__journal_start); + + +/** + * jbd2_journal_start() - Obtain a new handle. + * @journal: Journal to start transaction on. + * @nblocks: number of block buffer we might modify + * + * We make sure that the transaction can guarantee at least nblocks of + * modified buffers in the log. We block until the log can guarantee + * that much space. Additionally, if rsv_blocks > 0, we also create another + * handle with rsv_blocks reserved blocks in the journal. This handle is + * stored in h_rsv_handle. It is not attached to any particular transaction + * and thus doesn't block transaction commit. If the caller uses this reserved + * handle, it has to set h_rsv_handle to NULL as otherwise jbd2_journal_stop() + * on the parent handle will dispose the reserved one. Reserved handle has to + * be converted to a normal handle using jbd2_journal_start_reserved() before + * it can be used. + * + * Return a pointer to a newly allocated handle, or an ERR_PTR() value + * on failure. + */ +handle_t *jbd2_journal_start(journal_t *journal, int nblocks) +{ + return jbd2__journal_start(journal, nblocks, 0, 0, GFP_NOFS, 0, 0); +} +EXPORT_SYMBOL(jbd2_journal_start); + +static void __jbd2_journal_unreserve_handle(handle_t *handle, transaction_t *t) +{ + journal_t *journal = handle->h_journal; + + WARN_ON(!handle->h_reserved); + sub_reserved_credits(journal, handle->h_total_credits); + if (t) + atomic_sub(handle->h_total_credits, &t->t_outstanding_credits); +} + +void jbd2_journal_free_reserved(handle_t *handle) +{ + journal_t *journal = handle->h_journal; + + /* Get j_state_lock to pin running transaction if it exists */ + read_lock(&journal->j_state_lock); + __jbd2_journal_unreserve_handle(handle, journal->j_running_transaction); + read_unlock(&journal->j_state_lock); + jbd2_free_handle(handle); +} +EXPORT_SYMBOL(jbd2_journal_free_reserved); + +/** + * jbd2_journal_start_reserved() - start reserved handle + * @handle: handle to start + * @type: for handle statistics + * @line_no: for handle statistics + * + * Start handle that has been previously reserved with jbd2_journal_reserve(). + * This attaches @handle to the running transaction (or creates one if there's + * not transaction running). Unlike jbd2_journal_start() this function cannot + * block on journal commit, checkpointing, or similar stuff. It can block on + * memory allocation or frozen journal though. + * + * Return 0 on success, non-zero on error - handle is freed in that case. + */ +int jbd2_journal_start_reserved(handle_t *handle, unsigned int type, + unsigned int line_no) +{ + journal_t *journal = handle->h_journal; + int ret = -EIO; + + if (WARN_ON(!handle->h_reserved)) { + /* Someone passed in normal handle? Just stop it. */ + jbd2_journal_stop(handle); + return ret; + } + /* + * Usefulness of mixing of reserved and unreserved handles is + * questionable. So far nobody seems to need it so just error out. + */ + if (WARN_ON(current->journal_info)) { + jbd2_journal_free_reserved(handle); + return ret; + } + + handle->h_journal = NULL; + /* + * GFP_NOFS is here because callers are likely from writeback or + * similarly constrained call sites + */ + ret = start_this_handle(journal, handle, GFP_NOFS); + if (ret < 0) { + handle->h_journal = journal; + jbd2_journal_free_reserved(handle); + return ret; + } + handle->h_type = type; + handle->h_line_no = line_no; + trace_jbd2_handle_start(journal->j_fs_dev->bd_dev, + handle->h_transaction->t_tid, type, + line_no, handle->h_total_credits); + return 0; +} +EXPORT_SYMBOL(jbd2_journal_start_reserved); + +/** + * jbd2_journal_extend() - extend buffer credits. + * @handle: handle to 'extend' + * @nblocks: nr blocks to try to extend by. + * @revoke_records: number of revoke records to try to extend by. + * + * Some transactions, such as large extends and truncates, can be done + * atomically all at once or in several stages. The operation requests + * a credit for a number of buffer modifications in advance, but can + * extend its credit if it needs more. + * + * jbd2_journal_extend tries to give the running handle more buffer credits. + * It does not guarantee that allocation - this is a best-effort only. + * The calling process MUST be able to deal cleanly with a failure to + * extend here. + * + * Return 0 on success, non-zero on failure. + * + * return code < 0 implies an error + * return code > 0 implies normal transaction-full status. + */ +int jbd2_journal_extend(handle_t *handle, int nblocks, int revoke_records) +{ + transaction_t *transaction = handle->h_transaction; + journal_t *journal; + int result; + int wanted; + + if (is_handle_aborted(handle)) + return -EROFS; + journal = transaction->t_journal; + + result = 1; + + read_lock(&journal->j_state_lock); + + /* Don't extend a locked-down transaction! */ + if (transaction->t_state != T_RUNNING) { + jbd2_debug(3, "denied handle %p %d blocks: " + "transaction not running\n", handle, nblocks); + goto error_out; + } + + nblocks += DIV_ROUND_UP( + handle->h_revoke_credits_requested + revoke_records, + journal->j_revoke_records_per_block) - + DIV_ROUND_UP( + handle->h_revoke_credits_requested, + journal->j_revoke_records_per_block); + wanted = atomic_add_return(nblocks, + &transaction->t_outstanding_credits); + + if (wanted > journal->j_max_transaction_buffers) { + jbd2_debug(3, "denied handle %p %d blocks: " + "transaction too large\n", handle, nblocks); + atomic_sub(nblocks, &transaction->t_outstanding_credits); + goto error_out; + } + + trace_jbd2_handle_extend(journal->j_fs_dev->bd_dev, + transaction->t_tid, + handle->h_type, handle->h_line_no, + handle->h_total_credits, + nblocks); + + handle->h_total_credits += nblocks; + handle->h_requested_credits += nblocks; + handle->h_revoke_credits += revoke_records; + handle->h_revoke_credits_requested += revoke_records; + result = 0; + + jbd2_debug(3, "extended handle %p by %d\n", handle, nblocks); +error_out: + read_unlock(&journal->j_state_lock); + return result; +} + +static void stop_this_handle(handle_t *handle) +{ + transaction_t *transaction = handle->h_transaction; + journal_t *journal = transaction->t_journal; + int revokes; + + J_ASSERT(journal_current_handle() == handle); + J_ASSERT(atomic_read(&transaction->t_updates) > 0); + current->journal_info = NULL; + /* + * Subtract necessary revoke descriptor blocks from handle credits. We + * take care to account only for revoke descriptor blocks the + * transaction will really need as large sequences of transactions with + * small numbers of revokes are relatively common. + */ + revokes = handle->h_revoke_credits_requested - handle->h_revoke_credits; + if (revokes) { + int t_revokes, revoke_descriptors; + int rr_per_blk = journal->j_revoke_records_per_block; + + WARN_ON_ONCE(DIV_ROUND_UP(revokes, rr_per_blk) + > handle->h_total_credits); + t_revokes = atomic_add_return(revokes, + &transaction->t_outstanding_revokes); + revoke_descriptors = + DIV_ROUND_UP(t_revokes, rr_per_blk) - + DIV_ROUND_UP(t_revokes - revokes, rr_per_blk); + handle->h_total_credits -= revoke_descriptors; + } + atomic_sub(handle->h_total_credits, + &transaction->t_outstanding_credits); + if (handle->h_rsv_handle) + __jbd2_journal_unreserve_handle(handle->h_rsv_handle, + transaction); + if (atomic_dec_and_test(&transaction->t_updates)) + wake_up(&journal->j_wait_updates); + + rwsem_release(&journal->j_trans_commit_map, _THIS_IP_); + /* + * Scope of the GFP_NOFS context is over here and so we can restore the + * original alloc context. + */ + memalloc_nofs_restore(handle->saved_alloc_context); +} + +/** + * jbd2__journal_restart() - restart a handle . + * @handle: handle to restart + * @nblocks: nr credits requested + * @revoke_records: number of revoke record credits requested + * @gfp_mask: memory allocation flags (for start_this_handle) + * + * Restart a handle for a multi-transaction filesystem + * operation. + * + * If the jbd2_journal_extend() call above fails to grant new buffer credits + * to a running handle, a call to jbd2_journal_restart will commit the + * handle's transaction so far and reattach the handle to a new + * transaction capable of guaranteeing the requested number of + * credits. We preserve reserved handle if there's any attached to the + * passed in handle. + */ +int jbd2__journal_restart(handle_t *handle, int nblocks, int revoke_records, + gfp_t gfp_mask) +{ + transaction_t *transaction = handle->h_transaction; + journal_t *journal; + tid_t tid; + int need_to_start; + int ret; + + /* If we've had an abort of any type, don't even think about + * actually doing the restart! */ + if (is_handle_aborted(handle)) + return 0; + journal = transaction->t_journal; + tid = transaction->t_tid; + + /* + * First unlink the handle from its current transaction, and start the + * commit on that. + */ + jbd2_debug(2, "restarting handle %p\n", handle); + stop_this_handle(handle); + handle->h_transaction = NULL; + + /* + * TODO: If we use READ_ONCE / WRITE_ONCE for j_commit_request we can + * get rid of pointless j_state_lock traffic like this. + */ + read_lock(&journal->j_state_lock); + need_to_start = !tid_geq(journal->j_commit_request, tid); + read_unlock(&journal->j_state_lock); + if (need_to_start) + jbd2_log_start_commit(journal, tid); + handle->h_total_credits = nblocks + + DIV_ROUND_UP(revoke_records, + journal->j_revoke_records_per_block); + handle->h_revoke_credits = revoke_records; + ret = start_this_handle(journal, handle, gfp_mask); + trace_jbd2_handle_restart(journal->j_fs_dev->bd_dev, + ret ? 0 : handle->h_transaction->t_tid, + handle->h_type, handle->h_line_no, + handle->h_total_credits); + return ret; +} +EXPORT_SYMBOL(jbd2__journal_restart); + + +int jbd2_journal_restart(handle_t *handle, int nblocks) +{ + return jbd2__journal_restart(handle, nblocks, 0, GFP_NOFS); +} +EXPORT_SYMBOL(jbd2_journal_restart); + +/* + * Waits for any outstanding t_updates to finish. + * This is called with write j_state_lock held. + */ +void jbd2_journal_wait_updates(journal_t *journal) +{ + DEFINE_WAIT(wait); + + while (1) { + /* + * Note that the running transaction can get freed under us if + * this transaction is getting committed in + * jbd2_journal_commit_transaction() -> + * jbd2_journal_free_transaction(). This can only happen when we + * release j_state_lock -> schedule() -> acquire j_state_lock. + * Hence we should everytime retrieve new j_running_transaction + * value (after j_state_lock release acquire cycle), else it may + * lead to use-after-free of old freed transaction. + */ + transaction_t *transaction = journal->j_running_transaction; + + if (!transaction) + break; + + prepare_to_wait(&journal->j_wait_updates, &wait, + TASK_UNINTERRUPTIBLE); + if (!atomic_read(&transaction->t_updates)) { + finish_wait(&journal->j_wait_updates, &wait); + break; + } + write_unlock(&journal->j_state_lock); + schedule(); + finish_wait(&journal->j_wait_updates, &wait); + write_lock(&journal->j_state_lock); + } +} + +/** + * jbd2_journal_lock_updates () - establish a transaction barrier. + * @journal: Journal to establish a barrier on. + * + * This locks out any further updates from being started, and blocks + * until all existing updates have completed, returning only once the + * journal is in a quiescent state with no updates running. + * + * The journal lock should not be held on entry. + */ +void jbd2_journal_lock_updates(journal_t *journal) +{ + jbd2_might_wait_for_commit(journal); + + write_lock(&journal->j_state_lock); + ++journal->j_barrier_count; + + /* Wait until there are no reserved handles */ + if (atomic_read(&journal->j_reserved_credits)) { + write_unlock(&journal->j_state_lock); + wait_event(journal->j_wait_reserved, + atomic_read(&journal->j_reserved_credits) == 0); + write_lock(&journal->j_state_lock); + } + + /* Wait until there are no running t_updates */ + jbd2_journal_wait_updates(journal); + + write_unlock(&journal->j_state_lock); + + /* + * We have now established a barrier against other normal updates, but + * we also need to barrier against other jbd2_journal_lock_updates() calls + * to make sure that we serialise special journal-locked operations + * too. + */ + mutex_lock(&journal->j_barrier); +} + +/** + * jbd2_journal_unlock_updates () - release barrier + * @journal: Journal to release the barrier on. + * + * Release a transaction barrier obtained with jbd2_journal_lock_updates(). + * + * Should be called without the journal lock held. + */ +void jbd2_journal_unlock_updates (journal_t *journal) +{ + J_ASSERT(journal->j_barrier_count != 0); + + mutex_unlock(&journal->j_barrier); + write_lock(&journal->j_state_lock); + --journal->j_barrier_count; + write_unlock(&journal->j_state_lock); + wake_up_all(&journal->j_wait_transaction_locked); +} + +static void warn_dirty_buffer(struct buffer_head *bh) +{ + printk(KERN_WARNING + "JBD2: Spotted dirty metadata buffer (dev = %pg, blocknr = %llu). " + "There's a risk of filesystem corruption in case of system " + "crash.\n", + bh->b_bdev, (unsigned long long)bh->b_blocknr); +} + +/* Call t_frozen trigger and copy buffer data into jh->b_frozen_data. */ +static void jbd2_freeze_jh_data(struct journal_head *jh) +{ + char *source; + struct buffer_head *bh = jh2bh(jh); + + J_EXPECT_JH(jh, buffer_uptodate(bh), "Possible IO failure.\n"); + source = kmap_local_folio(bh->b_folio, bh_offset(bh)); + /* Fire data frozen trigger just before we copy the data */ + jbd2_buffer_frozen_trigger(jh, source, jh->b_triggers); + memcpy(jh->b_frozen_data, source, bh->b_size); + kunmap_local(source); + + /* + * Now that the frozen data is saved off, we need to store any matching + * triggers. + */ + jh->b_frozen_triggers = jh->b_triggers; +} + +/* + * If the buffer is already part of the current transaction, then there + * is nothing we need to do. If it is already part of a prior + * transaction which we are still committing to disk, then we need to + * make sure that we do not overwrite the old copy: we do copy-out to + * preserve the copy going to disk. We also account the buffer against + * the handle's metadata buffer credits (unless the buffer is already + * part of the transaction, that is). + * + */ +static int +do_get_write_access(handle_t *handle, struct journal_head *jh, + int force_copy) +{ + struct buffer_head *bh; + transaction_t *transaction = handle->h_transaction; + journal_t *journal; + int error; + char *frozen_buffer = NULL; + unsigned long start_lock, time_lock; + + journal = transaction->t_journal; + + jbd2_debug(5, "journal_head %p, force_copy %d\n", jh, force_copy); + + JBUFFER_TRACE(jh, "entry"); +repeat: + bh = jh2bh(jh); + + /* @@@ Need to check for errors here at some point. */ + + start_lock = jiffies; + lock_buffer(bh); + spin_lock(&jh->b_state_lock); + + /* If it takes too long to lock the buffer, trace it */ + time_lock = jbd2_time_diff(start_lock, jiffies); + if (time_lock > HZ/10) + trace_jbd2_lock_buffer_stall(bh->b_bdev->bd_dev, + jiffies_to_msecs(time_lock)); + + /* We now hold the buffer lock so it is safe to query the buffer + * state. Is the buffer dirty? + * + * If so, there are two possibilities. The buffer may be + * non-journaled, and undergoing a quite legitimate writeback. + * Otherwise, it is journaled, and we don't expect dirty buffers + * in that state (the buffers should be marked JBD_Dirty + * instead.) So either the IO is being done under our own + * control and this is a bug, or it's a third party IO such as + * dump(8) (which may leave the buffer scheduled for read --- + * ie. locked but not dirty) or tune2fs (which may actually have + * the buffer dirtied, ugh.) */ + + if (buffer_dirty(bh) && jh->b_transaction) { + warn_dirty_buffer(bh); + /* + * We need to clean the dirty flag and we must do it under the + * buffer lock to be sure we don't race with running write-out. + */ + JBUFFER_TRACE(jh, "Journalling dirty buffer"); + clear_buffer_dirty(bh); + /* + * The buffer is going to be added to BJ_Reserved list now and + * nothing guarantees jbd2_journal_dirty_metadata() will be + * ever called for it. So we need to set jbddirty bit here to + * make sure the buffer is dirtied and written out when the + * journaling machinery is done with it. + */ + set_buffer_jbddirty(bh); + } + + error = -EROFS; + if (is_handle_aborted(handle)) { + spin_unlock(&jh->b_state_lock); + unlock_buffer(bh); + goto out; + } + error = 0; + + /* + * The buffer is already part of this transaction if b_transaction or + * b_next_transaction points to it + */ + if (jh->b_transaction == transaction || + jh->b_next_transaction == transaction) { + unlock_buffer(bh); + goto done; + } + + /* + * this is the first time this transaction is touching this buffer, + * reset the modified flag + */ + jh->b_modified = 0; + + /* + * If the buffer is not journaled right now, we need to make sure it + * doesn't get written to disk before the caller actually commits the + * new data + */ + if (!jh->b_transaction) { + JBUFFER_TRACE(jh, "no transaction"); + if (WARN_ON_ONCE(jh->b_next_transaction)) { + spin_unlock(&jh->b_state_lock); + unlock_buffer(bh); + error = -EINVAL; + jbd2_journal_abort(journal, error); + goto out; + } + JBUFFER_TRACE(jh, "file as BJ_Reserved"); + /* + * Make sure all stores to jh (b_modified, b_frozen_data) are + * visible before attaching it to the running transaction. + * Paired with barrier in jbd2_write_access_granted() + */ + smp_wmb(); + spin_lock(&journal->j_list_lock); + if (test_clear_buffer_dirty(bh)) { + /* + * Execute buffer dirty clearing and jh->b_transaction + * assignment under journal->j_list_lock locked to + * prevent bh being removed from checkpoint list if + * the buffer is in an intermediate state (not dirty + * and jh->b_transaction is NULL). + */ + JBUFFER_TRACE(jh, "Journalling dirty buffer"); + set_buffer_jbddirty(bh); + } + __jbd2_journal_file_buffer(jh, transaction, BJ_Reserved); + spin_unlock(&journal->j_list_lock); + unlock_buffer(bh); + goto done; + } + unlock_buffer(bh); + + /* + * If there is already a copy-out version of this buffer, then we don't + * need to make another one + */ + if (jh->b_frozen_data) { + JBUFFER_TRACE(jh, "has frozen data"); + if (WARN_ON_ONCE(jh->b_next_transaction)) { + spin_unlock(&jh->b_state_lock); + error = -EINVAL; + jbd2_journal_abort(journal, error); + goto out; + } + goto attach_next; + } + + JBUFFER_TRACE(jh, "owned by older transaction"); + if (WARN_ON_ONCE(jh->b_next_transaction || + jh->b_transaction != + journal->j_committing_transaction)) { + pr_err("JBD2: %s: assertion failure: b_next_transaction=%p b_transaction=%p j_committing_transaction=%p\n", + journal->j_devname, jh->b_next_transaction, + jh->b_transaction, journal->j_committing_transaction); + spin_unlock(&jh->b_state_lock); + error = -EINVAL; + jbd2_journal_abort(journal, error); + goto out; + } + + /* + * There is one case we have to be very careful about. If the + * committing transaction is currently writing this buffer out to disk + * and has NOT made a copy-out, then we cannot modify the buffer + * contents at all right now. The essence of copy-out is that it is + * the extra copy, not the primary copy, which gets journaled. If the + * primary copy is already going to disk then we cannot do copy-out + * here. + */ + if (buffer_shadow(bh)) { + JBUFFER_TRACE(jh, "on shadow: sleep"); + spin_unlock(&jh->b_state_lock); + wait_on_bit_io(&bh->b_state, BH_Shadow, TASK_UNINTERRUPTIBLE); + goto repeat; + } + + /* + * Only do the copy if the currently-owning transaction still needs it. + * If buffer isn't on BJ_Metadata list, the committing transaction is + * past that stage (here we use the fact that BH_Shadow is set under + * bh_state lock together with refiling to BJ_Shadow list and at this + * point we know the buffer doesn't have BH_Shadow set). + * + * Subtle point, though: if this is a get_undo_access, then we will be + * relying on the frozen_data to contain the new value of the + * committed_data record after the transaction, so we HAVE to force the + * frozen_data copy in that case. + */ + if (jh->b_jlist == BJ_Metadata || force_copy) { + JBUFFER_TRACE(jh, "generate frozen data"); + if (!frozen_buffer) { + JBUFFER_TRACE(jh, "allocate memory for buffer"); + spin_unlock(&jh->b_state_lock); + frozen_buffer = kmalloc(jh2bh(jh)->b_size, + GFP_NOFS | __GFP_NOFAIL); + goto repeat; + } + jh->b_frozen_data = frozen_buffer; + frozen_buffer = NULL; + jbd2_freeze_jh_data(jh); + } +attach_next: + /* + * Make sure all stores to jh (b_modified, b_frozen_data) are visible + * before attaching it to the running transaction. Paired with barrier + * in jbd2_write_access_granted() + */ + smp_wmb(); + jh->b_next_transaction = transaction; + +done: + spin_unlock(&jh->b_state_lock); + + /* + * If we are about to journal a buffer, then any revoke pending on it is + * no longer valid + */ + jbd2_journal_cancel_revoke(handle, jh); + +out: + if (unlikely(frozen_buffer)) /* It's usually NULL */ + kfree(frozen_buffer); + + JBUFFER_TRACE(jh, "exit"); + return error; +} + +/* Fast check whether buffer is already attached to the required transaction */ +static bool jbd2_write_access_granted(handle_t *handle, struct buffer_head *bh, + bool undo) +{ + struct journal_head *jh; + bool ret = false; + + /* Dirty buffers require special handling... */ + if (buffer_dirty(bh)) + return false; + + /* + * RCU protects us from dereferencing freed pages. So the checks we do + * are guaranteed not to oops. However the jh slab object can get freed + * & reallocated while we work with it. So we have to be careful. When + * we see jh attached to the running transaction, we know it must stay + * so until the transaction is committed. Thus jh won't be freed and + * will be attached to the same bh while we run. However it can + * happen jh gets freed, reallocated, and attached to the transaction + * just after we get pointer to it from bh. So we have to be careful + * and recheck jh still belongs to our bh before we return success. + */ + rcu_read_lock(); + if (!buffer_jbd(bh)) + goto out; + /* This should be bh2jh() but that doesn't work with inline functions */ + jh = READ_ONCE(bh->b_private); + if (!jh) + goto out; + /* For undo access buffer must have data copied */ + if (undo && !jh->b_committed_data) + goto out; + if (READ_ONCE(jh->b_transaction) != handle->h_transaction && + READ_ONCE(jh->b_next_transaction) != handle->h_transaction) + goto out; + /* + * There are two reasons for the barrier here: + * 1) Make sure to fetch b_bh after we did previous checks so that we + * detect when jh went through free, realloc, attach to transaction + * while we were checking. Paired with implicit barrier in that path. + * 2) So that access to bh done after jbd2_write_access_granted() + * doesn't get reordered and see inconsistent state of concurrent + * do_get_write_access(). + */ + smp_mb(); + if (unlikely(jh->b_bh != bh)) + goto out; + ret = true; +out: + rcu_read_unlock(); + return ret; +} + +/** + * jbd2_journal_get_write_access() - notify intent to modify a buffer + * for metadata (not data) update. + * @handle: transaction to add buffer modifications to + * @bh: bh to be used for metadata writes + * + * Returns: error code or 0 on success. + * + * In full data journalling mode the buffer may be of type BJ_AsyncData, + * because we're ``write()ing`` a buffer which is also part of a shared mapping. + */ + +int jbd2_journal_get_write_access(handle_t *handle, struct buffer_head *bh) +{ + struct journal_head *jh; + journal_t *journal; + int rc; + + if (is_handle_aborted(handle)) + return -EROFS; + + journal = handle->h_transaction->t_journal; + rc = jbd2_check_fs_dev_write_error(journal); + if (rc) { + /* + * If the fs dev has writeback errors, it may have failed + * to async write out metadata buffers in the background. + * In this case, we could read old data from disk and write + * it out again, which may lead to on-disk filesystem + * inconsistency. Aborting journal can avoid it happen. + */ + jbd2_journal_abort(journal, rc); + return -EIO; + } + + if (jbd2_write_access_granted(handle, bh, false)) + return 0; + + jh = jbd2_journal_add_journal_head(bh); + /* We do not want to get caught playing with fields which the + * log thread also manipulates. Make sure that the buffer + * completes any outstanding IO before proceeding. */ + rc = do_get_write_access(handle, jh, 0); + jbd2_journal_put_journal_head(jh); + return rc; +} + + +/* + * When the user wants to journal a newly created buffer_head + * (ie. getblk() returned a new buffer and we are going to populate it + * manually rather than reading off disk), then we need to keep the + * buffer_head locked until it has been completely filled with new + * data. In this case, we should be able to make the assertion that + * the bh is not already part of an existing transaction. + * + * The buffer should already be locked by the caller by this point. + * There is no lock ranking violation: it was a newly created, + * unlocked buffer beforehand. */ + +/** + * jbd2_journal_get_create_access () - notify intent to use newly created bh + * @handle: transaction to new buffer to + * @bh: new buffer. + * + * Call this if you create a new bh. + */ +int jbd2_journal_get_create_access(handle_t *handle, struct buffer_head *bh) +{ + transaction_t *transaction = handle->h_transaction; + journal_t *journal; + struct journal_head *jh = jbd2_journal_add_journal_head(bh); + int err; + + jbd2_debug(5, "journal_head %p\n", jh); + err = -EROFS; + if (is_handle_aborted(handle)) + goto out; + journal = transaction->t_journal; + err = 0; + + JBUFFER_TRACE(jh, "entry"); + /* + * The buffer may already belong to this transaction due to pre-zeroing + * in the filesystem's new_block code. It may also be on the previous, + * committing transaction's lists, but it HAS to be in Forget state in + * that case: the transaction must have deleted the buffer for it to be + * reused here. + * In the case of file system data inconsistency, for example, if the + * block bitmap of a referenced block is not set, it can lead to the + * situation where a block being committed is allocated and used again. + * As a result, the following condition will not be satisfied, so here + * we directly trigger a JBD abort instead of immediately invoking + * bugon. + */ + spin_lock(&jh->b_state_lock); + if (!(jh->b_transaction == transaction || jh->b_transaction == NULL || + (jh->b_transaction == journal->j_committing_transaction && + jh->b_jlist == BJ_Forget)) || jh->b_next_transaction != NULL) { + err = -EROFS; + spin_unlock(&jh->b_state_lock); + jbd2_journal_abort(journal, err); + goto out; + } + + if (WARN_ON_ONCE(!buffer_locked(jh2bh(jh)))) { + err = -EINVAL; + spin_unlock(&jh->b_state_lock); + jbd2_journal_abort(journal, err); + goto out; + } + + if (jh->b_transaction == NULL) { + /* + * Previous jbd2_journal_forget() could have left the buffer + * with jbddirty bit set because it was being committed. When + * the commit finished, we've filed the buffer for + * checkpointing and marked it dirty. Now we are reallocating + * the buffer so the transaction freeing it must have + * committed and so it's safe to clear the dirty bit. + */ + clear_buffer_dirty(jh2bh(jh)); + /* first access by this transaction */ + jh->b_modified = 0; + + JBUFFER_TRACE(jh, "file as BJ_Reserved"); + spin_lock(&journal->j_list_lock); + __jbd2_journal_file_buffer(jh, transaction, BJ_Reserved); + spin_unlock(&journal->j_list_lock); + } else if (jh->b_transaction == journal->j_committing_transaction) { + /* first access by this transaction */ + jh->b_modified = 0; + + JBUFFER_TRACE(jh, "set next transaction"); + spin_lock(&journal->j_list_lock); + jh->b_next_transaction = transaction; + spin_unlock(&journal->j_list_lock); + } + spin_unlock(&jh->b_state_lock); + + /* + * akpm: I added this. ext3_alloc_branch can pick up new indirect + * blocks which contain freed but then revoked metadata. We need + * to cancel the revoke in case we end up freeing it yet again + * and the reallocating as data - this would cause a second revoke, + * which hits an assertion error. + */ + JBUFFER_TRACE(jh, "cancelling revoke"); + jbd2_journal_cancel_revoke(handle, jh); +out: + jbd2_journal_put_journal_head(jh); + return err; +} + +/** + * jbd2_journal_get_undo_access() - Notify intent to modify metadata with + * non-rewindable consequences + * @handle: transaction + * @bh: buffer to undo + * + * Sometimes there is a need to distinguish between metadata which has + * been committed to disk and that which has not. The ext3fs code uses + * this for freeing and allocating space, we have to make sure that we + * do not reuse freed space until the deallocation has been committed, + * since if we overwrote that space we would make the delete + * un-rewindable in case of a crash. + * + * To deal with that, jbd2_journal_get_undo_access requests write access to a + * buffer for parts of non-rewindable operations such as delete + * operations on the bitmaps. The journaling code must keep a copy of + * the buffer's contents prior to the undo_access call until such time + * as we know that the buffer has definitely been committed to disk. + * + * We never need to know which transaction the committed data is part + * of, buffers touched here are guaranteed to be dirtied later and so + * will be committed to a new transaction in due course, at which point + * we can discard the old committed data pointer. + * + * Returns error number or 0 on success. + */ +int jbd2_journal_get_undo_access(handle_t *handle, struct buffer_head *bh) +{ + int err; + struct journal_head *jh; + char *committed_data = NULL; + + if (is_handle_aborted(handle)) + return -EROFS; + + if (jbd2_write_access_granted(handle, bh, true)) + return 0; + + jh = jbd2_journal_add_journal_head(bh); + JBUFFER_TRACE(jh, "entry"); + + /* + * Do this first --- it can drop the journal lock, so we want to + * make sure that obtaining the committed_data is done + * atomically wrt. completion of any outstanding commits. + */ + err = do_get_write_access(handle, jh, 1); + if (err) + goto out; + +repeat: + if (!jh->b_committed_data) + committed_data = kmalloc(jh2bh(jh)->b_size, + GFP_NOFS|__GFP_NOFAIL); + + spin_lock(&jh->b_state_lock); + if (!jh->b_committed_data) { + /* Copy out the current buffer contents into the + * preserved, committed copy. */ + JBUFFER_TRACE(jh, "generate b_committed data"); + if (!committed_data) { + spin_unlock(&jh->b_state_lock); + goto repeat; + } + + jh->b_committed_data = committed_data; + committed_data = NULL; + memcpy(jh->b_committed_data, bh->b_data, bh->b_size); + } + spin_unlock(&jh->b_state_lock); +out: + jbd2_journal_put_journal_head(jh); + if (unlikely(committed_data)) + kfree(committed_data); + return err; +} + +/** + * jbd2_journal_set_triggers() - Add triggers for commit writeout + * @bh: buffer to trigger on + * @type: struct jbd2_buffer_trigger_type containing the trigger(s). + * + * Set any triggers on this journal_head. This is always safe, because + * triggers for a committing buffer will be saved off, and triggers for + * a running transaction will match the buffer in that transaction. + * + * Call with NULL to clear the triggers. + */ +void jbd2_journal_set_triggers(struct buffer_head *bh, + struct jbd2_buffer_trigger_type *type) +{ + struct journal_head *jh = jbd2_journal_grab_journal_head(bh); + + if (WARN_ON_ONCE(!jh)) + return; + jh->b_triggers = type; + jbd2_journal_put_journal_head(jh); +} + +void jbd2_buffer_frozen_trigger(struct journal_head *jh, void *mapped_data, + struct jbd2_buffer_trigger_type *triggers) +{ + struct buffer_head *bh = jh2bh(jh); + + if (!triggers || !triggers->t_frozen) + return; + + triggers->t_frozen(triggers, bh, mapped_data, bh->b_size); +} + +void jbd2_buffer_abort_trigger(struct journal_head *jh, + struct jbd2_buffer_trigger_type *triggers) +{ + if (!triggers || !triggers->t_abort) + return; + + triggers->t_abort(triggers, jh2bh(jh)); +} + +/** + * jbd2_journal_dirty_metadata() - mark a buffer as containing dirty metadata + * @handle: transaction to add buffer to. + * @bh: buffer to mark + * + * mark dirty metadata which needs to be journaled as part of the current + * transaction. + * + * The buffer must have previously had jbd2_journal_get_write_access() + * called so that it has a valid journal_head attached to the buffer + * head. + * + * The buffer is placed on the transaction's metadata list and is marked + * as belonging to the transaction. + * + * Returns error number or 0 on success. + * + * Special care needs to be taken if the buffer already belongs to the + * current committing transaction (in which case we should have frozen + * data present for that commit). In that case, we don't relink the + * buffer: that only gets done when the old transaction finally + * completes its commit. + */ +int jbd2_journal_dirty_metadata(handle_t *handle, struct buffer_head *bh) +{ + transaction_t *transaction; + journal_t *journal; + struct journal_head *jh; + int ret = 0; + + if (is_handle_aborted(handle)) + return -EROFS; + if (!buffer_jbd(bh)) + return -EUCLEAN; + + transaction = handle->h_transaction; + journal = transaction->t_journal; + + /* + * We don't grab jh reference here since the buffer must be part + * of the running transaction. + */ + jh = bh2jh(bh); + jbd2_debug(5, "journal_head %p\n", jh); + JBUFFER_TRACE(jh, "entry"); + + /* + * This and the following assertions are unreliable since we may see jh + * in inconsistent state unless we grab bh_state lock. But this is + * crucial to catch bugs so let's do a reliable check until the + * lockless handling is fully proven. + */ + if (data_race(jh->b_transaction != transaction && + jh->b_next_transaction != transaction)) { + spin_lock(&jh->b_state_lock); + if (WARN_ON_ONCE(jh->b_transaction != transaction && + jh->b_next_transaction != transaction)) { + pr_err("JBD2: %s: assertion failure: b_transaction=%p transaction=%p b_next_transaction=%p\n", + journal->j_devname, jh->b_transaction, + transaction, jh->b_next_transaction); + ret = -EINVAL; + goto out_unlock_bh; + } + spin_unlock(&jh->b_state_lock); + } + if (data_race(jh->b_modified == 1)) { + /* If it's in our transaction it must be in BJ_Metadata list. */ + if (data_race(jh->b_transaction == transaction && + jh->b_jlist != BJ_Metadata)) { + spin_lock(&jh->b_state_lock); + if (WARN_ON_ONCE(jh->b_transaction == transaction && + jh->b_jlist != BJ_Metadata)) { + pr_err("JBD2: assertion failure: h_type=%u h_line_no=%u block_no=%llu jlist=%u\n", + handle->h_type, handle->h_line_no, + (unsigned long long) bh->b_blocknr, + jh->b_jlist); + ret = -EINVAL; + goto out_unlock_bh; + } + spin_unlock(&jh->b_state_lock); + } + goto out; + } + + spin_lock(&jh->b_state_lock); + + if (is_handle_aborted(handle)) { + /* + * Check journal aborting with @jh->b_state_lock locked, + * since 'jh->b_transaction' could be replaced with + * 'jh->b_next_transaction' during old transaction + * committing if journal aborted, which may fail + * assertion on 'jh->b_frozen_data == NULL'. + */ + ret = -EROFS; + goto out_unlock_bh; + } + + if (jh->b_modified == 0) { + /* + * This buffer's got modified and becoming part + * of the transaction. This needs to be done + * once a transaction -bzzz + */ + if (WARN_ON_ONCE(jbd2_handle_buffer_credits(handle) <= 0)) { + ret = -ENOSPC; + goto out_unlock_bh; + } + jh->b_modified = 1; + handle->h_total_credits--; + } + + /* + * fastpath, to avoid expensive locking. If this buffer is already + * on the running transaction's metadata list there is nothing to do. + * Nobody can take it off again because there is a handle open. + * I _think_ we're OK here with SMP barriers - a mistaken decision will + * result in this test being false, so we go in and take the locks. + */ + if (jh->b_transaction == transaction && jh->b_jlist == BJ_Metadata) { + JBUFFER_TRACE(jh, "fastpath"); + if (unlikely(jh->b_transaction != + journal->j_running_transaction)) { + printk(KERN_ERR "JBD2: %s: " + "jh->b_transaction (%llu, %p, %u) != " + "journal->j_running_transaction (%p, %u)\n", + journal->j_devname, + (unsigned long long) bh->b_blocknr, + jh->b_transaction, + jh->b_transaction ? jh->b_transaction->t_tid : 0, + journal->j_running_transaction, + journal->j_running_transaction ? + journal->j_running_transaction->t_tid : 0); + ret = -EINVAL; + } + goto out_unlock_bh; + } + + set_buffer_jbddirty(bh); + + /* + * Metadata already on the current transaction list doesn't + * need to be filed. Metadata on another transaction's list must + * be committing, and will be refiled once the commit completes: + * leave it alone for now. + */ + if (jh->b_transaction != transaction) { + JBUFFER_TRACE(jh, "already on other transaction"); + if (unlikely(((jh->b_transaction != + journal->j_committing_transaction)) || + (jh->b_next_transaction != transaction))) { + printk(KERN_ERR "jbd2_journal_dirty_metadata: %s: " + "bad jh for block %llu: " + "transaction (%p, %u), " + "jh->b_transaction (%p, %u), " + "jh->b_next_transaction (%p, %u), jlist %u\n", + journal->j_devname, + (unsigned long long) bh->b_blocknr, + transaction, transaction->t_tid, + jh->b_transaction, + jh->b_transaction ? + jh->b_transaction->t_tid : 0, + jh->b_next_transaction, + jh->b_next_transaction ? + jh->b_next_transaction->t_tid : 0, + jh->b_jlist); + WARN_ON(1); + ret = -EINVAL; + } + /* And this case is illegal: we can't reuse another + * transaction's data buffer, ever. */ + goto out_unlock_bh; + } + + /* That test should have eliminated the following case: */ + if (WARN_ON_ONCE(jh->b_frozen_data)) { + ret = -EINVAL; + goto out_unlock_bh; + } + + JBUFFER_TRACE(jh, "file as BJ_Metadata"); + spin_lock(&journal->j_list_lock); + __jbd2_journal_file_buffer(jh, transaction, BJ_Metadata); + spin_unlock(&journal->j_list_lock); +out_unlock_bh: + spin_unlock(&jh->b_state_lock); +out: + JBUFFER_TRACE(jh, "exit"); + return ret; +} + +/** + * jbd2_journal_forget() - bforget() for potentially-journaled buffers. + * @handle: transaction handle + * @bh: bh to 'forget' + * + * We can only do the bforget if there are no commits pending against the + * buffer. If the buffer is dirty in the current running transaction we + * can safely unlink it. + * + * bh may not be a journalled buffer at all - it may be a non-JBD + * buffer which came off the hashtable. Check for this. + * + * Decrements bh->b_count by one. + * + * Allow this call even if the handle has aborted --- it may be part of + * the caller's cleanup after an abort. + */ +int jbd2_journal_forget(handle_t *handle, struct buffer_head *bh) +{ + transaction_t *transaction = handle->h_transaction; + journal_t *journal; + struct journal_head *jh; + int drop_reserve = 0; + int err = 0; + int was_modified = 0; + int wait_for_writeback = 0; + int abort_journal = 0; + + if (is_handle_aborted(handle)) + return -EROFS; + journal = transaction->t_journal; + + BUFFER_TRACE(bh, "entry"); + + jh = jbd2_journal_grab_journal_head(bh); + if (!jh) { + __bforget(bh); + return 0; + } + + spin_lock(&jh->b_state_lock); + + /* Critical error: attempting to delete a bitmap buffer, maybe? + * Don't do any jbd operations, and return an error. */ + if (!J_EXPECT_JH(jh, !jh->b_committed_data, + "inconsistent data on disk")) { + err = -EIO; + goto drop; + } + + /* keep track of whether or not this transaction modified us */ + was_modified = jh->b_modified; + + /* + * The buffer's going from the transaction, we must drop + * all references -bzzz + */ + jh->b_modified = 0; + + if (jh->b_transaction == transaction) { + if (WARN_ON_ONCE(jh->b_frozen_data)) { + err = -EINVAL; + abort_journal = 1; + goto drop; + } + + /* If we are forgetting a buffer which is already part + * of this transaction, then we can just drop it from + * the transaction immediately. */ + clear_buffer_dirty(bh); + clear_buffer_jbddirty(bh); + + JBUFFER_TRACE(jh, "belongs to current transaction: unfile"); + + /* + * we only want to drop a reference if this transaction + * modified the buffer + */ + if (was_modified) + drop_reserve = 1; + + /* + * We are no longer going to journal this buffer. + * However, the commit of this transaction is still + * important to the buffer: the delete that we are now + * processing might obsolete an old log entry, so by + * committing, we can satisfy the buffer's checkpoint. + * + * So, if we have a checkpoint on the buffer, we should + * now refile the buffer on our BJ_Forget list so that + * we know to remove the checkpoint after we commit. + */ + + spin_lock(&journal->j_list_lock); + if (jh->b_cp_transaction) { + __jbd2_journal_temp_unlink_buffer(jh); + __jbd2_journal_file_buffer(jh, transaction, BJ_Forget); + } else { + __jbd2_journal_unfile_buffer(jh); + jbd2_journal_put_journal_head(jh); + } + spin_unlock(&journal->j_list_lock); + } else if (jh->b_transaction) { + if (WARN_ON_ONCE(jh->b_transaction != journal->j_committing_transaction)) { + err = -EINVAL; + abort_journal = 1; + goto drop; + } + /* However, if the buffer is still owned by a prior + * (committing) transaction, we can't drop it yet... */ + JBUFFER_TRACE(jh, "belongs to older transaction"); + /* ... but we CAN drop it from the new transaction through + * marking the buffer as freed and set j_next_transaction to + * the new transaction, so that not only the commit code + * knows it should clear dirty bits when it is done with the + * buffer, but also the buffer can be checkpointed only + * after the new transaction commits. */ + + set_buffer_freed(bh); + + if (!jh->b_next_transaction) { + spin_lock(&journal->j_list_lock); + jh->b_next_transaction = transaction; + spin_unlock(&journal->j_list_lock); + } else { + if (WARN_ON_ONCE(jh->b_next_transaction != transaction)) { + err = -EINVAL; + abort_journal = 1; + goto drop; + } + + /* + * only drop a reference if this transaction modified + * the buffer + */ + if (was_modified) + drop_reserve = 1; + } + } else { + /* + * Finally, if the buffer is not belongs to any + * transaction, we can just drop it now if it has no + * checkpoint. + */ + spin_lock(&journal->j_list_lock); + if (!jh->b_cp_transaction) { + JBUFFER_TRACE(jh, "belongs to none transaction"); + spin_unlock(&journal->j_list_lock); + goto drop; + } + + /* + * Otherwise, if the buffer has been written to disk, + * it is safe to remove the checkpoint and drop it. + */ + if (jbd2_journal_try_remove_checkpoint(jh) >= 0) { + spin_unlock(&journal->j_list_lock); + goto drop; + } + + /* + * The buffer has not yet been written to disk. We should + * either clear the buffer or ensure that the ongoing I/O + * is completed, and attach this buffer to current + * transaction so that the buffer can be checkpointed only + * after the current transaction commits. + */ + clear_buffer_dirty(bh); + wait_for_writeback = 1; + __jbd2_journal_file_buffer(jh, transaction, BJ_Forget); + spin_unlock(&journal->j_list_lock); + } +drop: + __brelse(bh); + spin_unlock(&jh->b_state_lock); + if (abort_journal) + jbd2_journal_abort(journal, err); + if (wait_for_writeback) + wait_on_buffer(bh); + jbd2_journal_put_journal_head(jh); + if (drop_reserve) { + /* no need to reserve log space for this block -bzzz */ + handle->h_total_credits++; + } + return err; +} + +/** + * jbd2_journal_stop() - complete a transaction + * @handle: transaction to complete. + * + * All done for a particular handle. + * + * There is not much action needed here. We just return any remaining + * buffer credits to the transaction and remove the handle. The only + * complication is that we need to start a commit operation if the + * filesystem is marked for synchronous update. + * + * jbd2_journal_stop itself will not usually return an error, but it may + * do so in unusual circumstances. In particular, expect it to + * return -EIO if a jbd2_journal_abort has been executed since the + * transaction began. + */ +int jbd2_journal_stop(handle_t *handle) +{ + transaction_t *transaction = handle->h_transaction; + journal_t *journal; + int err = 0, wait_for_commit = 0; + tid_t tid; + pid_t pid; + + if (--handle->h_ref > 0) { + jbd2_debug(4, "h_ref %d -> %d\n", handle->h_ref + 1, + handle->h_ref); + if (is_handle_aborted(handle)) + return -EIO; + return 0; + } + if (!transaction) { + /* + * Handle is already detached from the transaction so there is + * nothing to do other than free the handle. + */ + memalloc_nofs_restore(handle->saved_alloc_context); + goto free_and_exit; + } + journal = transaction->t_journal; + tid = transaction->t_tid; + + if (is_handle_aborted(handle)) + err = -EIO; + + jbd2_debug(4, "Handle %p going down\n", handle); + trace_jbd2_handle_stats(journal->j_fs_dev->bd_dev, + tid, handle->h_type, handle->h_line_no, + jiffies - handle->h_start_jiffies, + handle->h_sync, handle->h_requested_credits, + (handle->h_requested_credits - + handle->h_total_credits)); + + /* + * Implement synchronous transaction batching. If the handle + * was synchronous, don't force a commit immediately. Let's + * yield and let another thread piggyback onto this + * transaction. Keep doing that while new threads continue to + * arrive. It doesn't cost much - we're about to run a commit + * and sleep on IO anyway. Speeds up many-threaded, many-dir + * operations by 30x or more... + * + * We try and optimize the sleep time against what the + * underlying disk can do, instead of having a static sleep + * time. This is useful for the case where our storage is so + * fast that it is more optimal to go ahead and force a flush + * and wait for the transaction to be committed than it is to + * wait for an arbitrary amount of time for new writers to + * join the transaction. We achieve this by measuring how + * long it takes to commit a transaction, and compare it with + * how long this transaction has been running, and if run time + * < commit time then we sleep for the delta and commit. This + * greatly helps super fast disks that would see slowdowns as + * more threads started doing fsyncs. + * + * But don't do this if this process was the most recent one + * to perform a synchronous write. We do this to detect the + * case where a single process is doing a stream of sync + * writes. No point in waiting for joiners in that case. + * + * Setting max_batch_time to 0 disables this completely. + */ + pid = current->pid; + if (handle->h_sync && journal->j_last_sync_writer != pid && + journal->j_max_batch_time) { + u64 commit_time, trans_time; + + journal->j_last_sync_writer = pid; + + read_lock(&journal->j_state_lock); + commit_time = journal->j_average_commit_time; + read_unlock(&journal->j_state_lock); + + trans_time = ktime_to_ns(ktime_sub(ktime_get(), + transaction->t_start_time)); + + commit_time = max_t(u64, commit_time, + 1000*journal->j_min_batch_time); + commit_time = min_t(u64, commit_time, + 1000*journal->j_max_batch_time); + + if (trans_time < commit_time) { + ktime_t expires = ktime_add_ns(ktime_get(), + commit_time); + set_current_state(TASK_UNINTERRUPTIBLE); + schedule_hrtimeout(&expires, HRTIMER_MODE_ABS); + } + } + + if (handle->h_sync) + transaction->t_synchronous_commit = 1; + + /* + * If the handle is marked SYNC, we need to set another commit + * going! We also want to force a commit if the transaction is too + * old now. + */ + if (handle->h_sync || + time_after_eq(jiffies, transaction->t_expires)) { + /* Do this even for aborted journals: an abort still + * completes the commit thread, it just doesn't write + * anything to disk. */ + + jbd2_debug(2, "transaction too old, requesting commit for " + "handle %p\n", handle); + /* This is non-blocking */ + jbd2_log_start_commit(journal, tid); + + /* + * Special case: JBD2_SYNC synchronous updates require us + * to wait for the commit to complete. + */ + if (handle->h_sync && !(current->flags & PF_MEMALLOC)) + wait_for_commit = 1; + } + + /* + * Once stop_this_handle() drops t_updates, the transaction could start + * committing on us and eventually disappear. So we must not + * dereference transaction pointer again after calling + * stop_this_handle(). + */ + stop_this_handle(handle); + + if (wait_for_commit) + err = jbd2_log_wait_commit(journal, tid); + +free_and_exit: + if (handle->h_rsv_handle) + jbd2_free_handle(handle->h_rsv_handle); + jbd2_free_handle(handle); + return err; +} + +/* + * + * List management code snippets: various functions for manipulating the + * transaction buffer lists. + * + */ + +/* + * Append a buffer to a transaction list, given the transaction's list head + * pointer. + * + * j_list_lock is held. + * + * jh->b_state_lock is held. + */ + +static inline void +__blist_add_buffer(struct journal_head **list, struct journal_head *jh) +{ + if (!*list) { + jh->b_tnext = jh->b_tprev = jh; + *list = jh; + } else { + /* Insert at the tail of the list to preserve order */ + struct journal_head *first = *list, *last = first->b_tprev; + jh->b_tprev = last; + jh->b_tnext = first; + last->b_tnext = first->b_tprev = jh; + } +} + +/* + * Remove a buffer from a transaction list, given the transaction's list + * head pointer. + * + * Called with j_list_lock held, and the journal may not be locked. + * + * jh->b_state_lock is held. + */ + +static inline void +__blist_del_buffer(struct journal_head **list, struct journal_head *jh) +{ + if (*list == jh) { + *list = jh->b_tnext; + if (*list == jh) + *list = NULL; + } + jh->b_tprev->b_tnext = jh->b_tnext; + jh->b_tnext->b_tprev = jh->b_tprev; +} + +/* + * Remove a buffer from the appropriate transaction list. + * + * Note that this function can *change* the value of + * bh->b_transaction->t_buffers, t_forget, t_shadow_list, t_log_list or + * t_reserved_list. If the caller is holding onto a copy of one of these + * pointers, it could go bad. Generally the caller needs to re-read the + * pointer from the transaction_t. + * + * Called under j_list_lock. + */ +static void __jbd2_journal_temp_unlink_buffer(struct journal_head *jh) +{ + struct journal_head **list = NULL; + transaction_t *transaction; + struct buffer_head *bh = jh2bh(jh); + + lockdep_assert_held(&jh->b_state_lock); + transaction = jh->b_transaction; + if (transaction) + assert_spin_locked(&transaction->t_journal->j_list_lock); + + J_ASSERT_JH(jh, jh->b_jlist < BJ_Types); + if (jh->b_jlist != BJ_None) + J_ASSERT_JH(jh, transaction != NULL); + + switch (jh->b_jlist) { + case BJ_None: + return; + case BJ_Metadata: + transaction->t_nr_buffers--; + J_ASSERT_JH(jh, transaction->t_nr_buffers >= 0); + list = &transaction->t_buffers; + break; + case BJ_Forget: + list = &transaction->t_forget; + break; + case BJ_Shadow: + list = &transaction->t_shadow_list; + break; + case BJ_Reserved: + list = &transaction->t_reserved_list; + break; + } + + __blist_del_buffer(list, jh); + jh->b_jlist = BJ_None; + if (transaction && is_journal_aborted(transaction->t_journal)) + clear_buffer_jbddirty(bh); + else if (test_clear_buffer_jbddirty(bh)) + mark_buffer_dirty(bh); /* Expose it to the VM */ +} + +/* + * Remove buffer from all transactions. The caller is responsible for dropping + * the jh reference that belonged to the transaction. + * + * Called with bh_state lock and j_list_lock + */ +static void __jbd2_journal_unfile_buffer(struct journal_head *jh) +{ + J_ASSERT_JH(jh, jh->b_transaction != NULL); + J_ASSERT_JH(jh, jh->b_next_transaction == NULL); + + __jbd2_journal_temp_unlink_buffer(jh); + jh->b_transaction = NULL; +} + +/** + * jbd2_journal_try_to_free_buffers() - try to free page buffers. + * @journal: journal for operation + * @folio: Folio to detach data from. + * + * For all the buffers on this page, + * if they are fully written out ordered data, move them onto BUF_CLEAN + * so try_to_free_buffers() can reap them. + * + * This function returns non-zero if we wish try_to_free_buffers() + * to be called. We do this if the page is releasable by try_to_free_buffers(). + * We also do it if the page has locked or dirty buffers and the caller wants + * us to perform sync or async writeout. + * + * This complicates JBD locking somewhat. We aren't protected by the + * BKL here. We wish to remove the buffer from its committing or + * running transaction's ->t_datalist via __jbd2_journal_unfile_buffer. + * + * This may *change* the value of transaction_t->t_datalist, so anyone + * who looks at t_datalist needs to lock against this function. + * + * Even worse, someone may be doing a jbd2_journal_dirty_data on this + * buffer. So we need to lock against that. jbd2_journal_dirty_data() + * will come out of the lock with the buffer dirty, which makes it + * ineligible for release here. + * + * Who else is affected by this? hmm... Really the only contender + * is do_get_write_access() - it could be looking at the buffer while + * journal_try_to_free_buffer() is changing its state. But that + * cannot happen because we never reallocate freed data as metadata + * while the data is part of a transaction. Yes? + * + * Return false on failure, true on success + */ +bool jbd2_journal_try_to_free_buffers(journal_t *journal, struct folio *folio) +{ + struct buffer_head *head; + struct buffer_head *bh; + bool ret = false; + + if (WARN_ON_ONCE(!folio_test_locked(folio))) + return false; + + head = folio_buffers(folio); + bh = head; + do { + struct journal_head *jh; + + /* + * We take our own ref against the journal_head here to avoid + * having to add tons of locking around each instance of + * jbd2_journal_put_journal_head(). + */ + jh = jbd2_journal_grab_journal_head(bh); + if (!jh) + continue; + + spin_lock(&jh->b_state_lock); + if (!jh->b_transaction && !jh->b_next_transaction) { + spin_lock(&journal->j_list_lock); + /* Remove written-back checkpointed metadata buffer */ + if (jh->b_cp_transaction != NULL) + jbd2_journal_try_remove_checkpoint(jh); + spin_unlock(&journal->j_list_lock); + } + spin_unlock(&jh->b_state_lock); + jbd2_journal_put_journal_head(jh); + if (buffer_jbd(bh)) + goto busy; + } while ((bh = bh->b_this_page) != head); + + ret = try_to_free_buffers(folio); +busy: + return ret; +} + +/* + * This buffer is no longer needed. If it is on an older transaction's + * checkpoint list we need to record it on this transaction's forget list + * to pin this buffer (and hence its checkpointing transaction) down until + * this transaction commits. If the buffer isn't on a checkpoint list, we + * release it. + * Returns non-zero if JBD no longer has an interest in the buffer. + * + * Called under j_list_lock. + * + * Called under jh->b_state_lock. + */ +static int __dispose_buffer(struct journal_head *jh, transaction_t *transaction) +{ + int may_free = 1; + struct buffer_head *bh = jh2bh(jh); + + if (jh->b_cp_transaction) { + JBUFFER_TRACE(jh, "on running+cp transaction"); + __jbd2_journal_temp_unlink_buffer(jh); + /* + * We don't want to write the buffer anymore, clear the + * bit so that we don't confuse checks in + * __jbd2_journal_file_buffer + */ + clear_buffer_dirty(bh); + __jbd2_journal_file_buffer(jh, transaction, BJ_Forget); + may_free = 0; + } else { + JBUFFER_TRACE(jh, "on running transaction"); + __jbd2_journal_unfile_buffer(jh); + jbd2_journal_put_journal_head(jh); + } + return may_free; +} + +/* + * jbd2_journal_invalidate_folio + * + * This code is tricky. It has a number of cases to deal with. + * + * There are two invariants which this code relies on: + * + * i_size must be updated on disk before we start calling invalidate_folio + * on the data. + * + * This is done in ext3 by defining an ext3_setattr method which + * updates i_size before truncate gets going. By maintaining this + * invariant, we can be sure that it is safe to throw away any buffers + * attached to the current transaction: once the transaction commits, + * we know that the data will not be needed. + * + * Note however that we can *not* throw away data belonging to the + * previous, committing transaction! + * + * Any disk blocks which *are* part of the previous, committing + * transaction (and which therefore cannot be discarded immediately) are + * not going to be reused in the new running transaction + * + * The bitmap committed_data images guarantee this: any block which is + * allocated in one transaction and removed in the next will be marked + * as in-use in the committed_data bitmap, so cannot be reused until + * the next transaction to delete the block commits. This means that + * leaving committing buffers dirty is quite safe: the disk blocks + * cannot be reallocated to a different file and so buffer aliasing is + * not possible. + * + * + * The above applies mainly to ordered data mode. In writeback mode we + * don't make guarantees about the order in which data hits disk --- in + * particular we don't guarantee that new dirty data is flushed before + * transaction commit --- so it is always safe just to discard data + * immediately in that mode. --sct + */ + +/* + * The journal_unmap_buffer helper function returns zero if the buffer + * concerned remains pinned as an anonymous buffer belonging to an older + * transaction. + * + * We're outside-transaction here. Either or both of j_running_transaction + * and j_committing_transaction may be NULL. + */ +static int journal_unmap_buffer(journal_t *journal, struct buffer_head *bh, + int partial_page) +{ + transaction_t *transaction; + struct journal_head *jh; + int may_free = 1; + + BUFFER_TRACE(bh, "entry"); + + /* + * It is safe to proceed here without the j_list_lock because the + * buffers cannot be stolen by try_to_free_buffers as long as we are + * holding the page lock. --sct + */ + + jh = jbd2_journal_grab_journal_head(bh); + if (!jh) + goto zap_buffer_unlocked; + + /* OK, we have data buffer in journaled mode */ + write_lock(&journal->j_state_lock); + spin_lock(&jh->b_state_lock); + spin_lock(&journal->j_list_lock); + + /* + * We cannot remove the buffer from checkpoint lists until the + * transaction adding inode to orphan list (let's call it T) + * is committed. Otherwise if the transaction changing the + * buffer would be cleaned from the journal before T is + * committed, a crash will cause that the correct contents of + * the buffer will be lost. On the other hand we have to + * clear the buffer dirty bit at latest at the moment when the + * transaction marking the buffer as freed in the filesystem + * structures is committed because from that moment on the + * block can be reallocated and used by a different page. + * Since the block hasn't been freed yet but the inode has + * already been added to orphan list, it is safe for us to add + * the buffer to BJ_Forget list of the newest transaction. + * + * Also we have to clear buffer_mapped flag of a truncated buffer + * because the buffer_head may be attached to the page straddling + * i_size (can happen only when blocksize < pagesize) and thus the + * buffer_head can be reused when the file is extended again. So we end + * up keeping around invalidated buffers attached to transactions' + * BJ_Forget list just to stop checkpointing code from cleaning up + * the transaction this buffer was modified in. + */ + transaction = jh->b_transaction; + if (transaction == NULL) { + /* First case: not on any transaction. If it + * has no checkpoint link, then we can zap it: + * it's a writeback-mode buffer so we don't care + * if it hits disk safely. */ + if (!jh->b_cp_transaction) { + JBUFFER_TRACE(jh, "not on any transaction: zap"); + goto zap_buffer; + } + + if (!buffer_dirty(bh)) { + /* bdflush has written it. We can drop it now */ + __jbd2_journal_remove_checkpoint(jh); + goto zap_buffer; + } + + /* OK, it must be in the journal but still not + * written fully to disk: it's metadata or + * journaled data... */ + + if (journal->j_running_transaction) { + /* ... and once the current transaction has + * committed, the buffer won't be needed any + * longer. */ + JBUFFER_TRACE(jh, "checkpointed: add to BJ_Forget"); + may_free = __dispose_buffer(jh, + journal->j_running_transaction); + goto zap_buffer; + } else { + /* There is no currently-running transaction. So the + * orphan record which we wrote for this file must have + * passed into commit. We must attach this buffer to + * the committing transaction, if it exists. */ + if (journal->j_committing_transaction) { + JBUFFER_TRACE(jh, "give to committing trans"); + may_free = __dispose_buffer(jh, + journal->j_committing_transaction); + goto zap_buffer; + } else { + /* The orphan record's transaction has + * committed. We can cleanse this buffer */ + clear_buffer_jbddirty(bh); + __jbd2_journal_remove_checkpoint(jh); + goto zap_buffer; + } + } + } else if (transaction == journal->j_committing_transaction) { + JBUFFER_TRACE(jh, "on committing transaction"); + /* + * The buffer is committing, we simply cannot touch + * it. If the page is straddling i_size we have to wait + * for commit and try again. + */ + if (partial_page) { + spin_unlock(&journal->j_list_lock); + spin_unlock(&jh->b_state_lock); + write_unlock(&journal->j_state_lock); + jbd2_journal_put_journal_head(jh); + /* Already zapped buffer? Nothing to do... */ + if (!bh->b_bdev) + return 0; + return -EBUSY; + } + /* + * OK, buffer won't be reachable after truncate. We just clear + * b_modified to not confuse transaction credit accounting, and + * set j_next_transaction to the running transaction (if there + * is one) and mark buffer as freed so that commit code knows + * it should clear dirty bits when it is done with the buffer. + */ + set_buffer_freed(bh); + if (journal->j_running_transaction && buffer_jbddirty(bh)) + jh->b_next_transaction = journal->j_running_transaction; + jh->b_modified = 0; + spin_unlock(&journal->j_list_lock); + spin_unlock(&jh->b_state_lock); + write_unlock(&journal->j_state_lock); + jbd2_journal_put_journal_head(jh); + return 0; + } else { + /* Good, the buffer belongs to the running transaction. + * We are writing our own transaction's data, not any + * previous one's, so it is safe to throw it away + * (remember that we expect the filesystem to have set + * i_size already for this truncate so recovery will not + * expose the disk blocks we are discarding here.) */ + J_ASSERT_JH(jh, transaction == journal->j_running_transaction); + JBUFFER_TRACE(jh, "on running transaction"); + may_free = __dispose_buffer(jh, transaction); + } + +zap_buffer: + /* + * This is tricky. Although the buffer is truncated, it may be reused + * if blocksize < pagesize and it is attached to the page straddling + * EOF. Since the buffer might have been added to BJ_Forget list of the + * running transaction, journal_get_write_access() won't clear + * b_modified and credit accounting gets confused. So clear b_modified + * here. + */ + jh->b_modified = 0; + spin_unlock(&journal->j_list_lock); + spin_unlock(&jh->b_state_lock); + write_unlock(&journal->j_state_lock); + jbd2_journal_put_journal_head(jh); +zap_buffer_unlocked: + clear_buffer_dirty(bh); + J_ASSERT_BH(bh, !buffer_jbddirty(bh)); + clear_buffer_mapped(bh); + clear_buffer_req(bh); + clear_buffer_new(bh); + clear_buffer_delay(bh); + clear_buffer_unwritten(bh); + bh->b_bdev = NULL; + return may_free; +} + +/** + * jbd2_journal_invalidate_folio() + * @journal: journal to use for flush... + * @folio: folio to flush + * @offset: start of the range to invalidate + * @length: length of the range to invalidate + * + * Reap page buffers containing data after in the specified range in page. + * Can return -EBUSY if buffers are part of the committing transaction and + * the page is straddling i_size. Caller then has to wait for current commit + * and try again. + */ +int jbd2_journal_invalidate_folio(journal_t *journal, struct folio *folio, + size_t offset, size_t length) +{ + struct buffer_head *head, *bh, *next; + unsigned int stop = offset + length; + unsigned int curr_off = 0; + int partial_page = (offset || length < folio_size(folio)); + int may_free = 1; + int ret = 0; + + if (!folio_test_locked(folio)) + BUG(); + head = folio_buffers(folio); + if (!head) + return 0; + + BUG_ON(stop > folio_size(folio) || stop < length); + + /* We will potentially be playing with lists other than just the + * data lists (especially for journaled data mode), so be + * cautious in our locking. */ + + bh = head; + do { + unsigned int next_off = curr_off + bh->b_size; + next = bh->b_this_page; + + if (next_off > stop) + return 0; + + if (offset <= curr_off) { + /* This block is wholly outside the truncation point */ + lock_buffer(bh); + ret = journal_unmap_buffer(journal, bh, partial_page); + unlock_buffer(bh); + if (ret < 0) + return ret; + may_free &= ret; + } + curr_off = next_off; + bh = next; + + } while (bh != head); + + if (!partial_page) { + if (may_free && try_to_free_buffers(folio)) + J_ASSERT(!folio_buffers(folio)); + } + return 0; +} + +/* + * File a buffer on the given transaction list. + */ +void __jbd2_journal_file_buffer(struct journal_head *jh, + transaction_t *transaction, int jlist) +{ + struct journal_head **list = NULL; + int was_dirty = 0; + struct buffer_head *bh = jh2bh(jh); + + lockdep_assert_held(&jh->b_state_lock); + assert_spin_locked(&transaction->t_journal->j_list_lock); + + J_ASSERT_JH(jh, jh->b_jlist < BJ_Types); + J_ASSERT_JH(jh, jh->b_transaction == transaction || + jh->b_transaction == NULL); + + if (jh->b_transaction && jh->b_jlist == jlist) + return; + + if (jlist == BJ_Metadata || jlist == BJ_Reserved || + jlist == BJ_Shadow || jlist == BJ_Forget) { + /* + * For metadata buffers, we track dirty bit in buffer_jbddirty + * instead of buffer_dirty. We should not see a dirty bit set + * here because we clear it in do_get_write_access but e.g. + * tune2fs can modify the sb and set the dirty bit at any time + * so we try to gracefully handle that. + */ + if (buffer_dirty(bh)) + warn_dirty_buffer(bh); + if (test_clear_buffer_dirty(bh) || + test_clear_buffer_jbddirty(bh)) + was_dirty = 1; + } + + if (jh->b_transaction) + __jbd2_journal_temp_unlink_buffer(jh); + else + jbd2_journal_grab_journal_head(bh); + jh->b_transaction = transaction; + + switch (jlist) { + case BJ_None: + J_ASSERT_JH(jh, !jh->b_committed_data); + J_ASSERT_JH(jh, !jh->b_frozen_data); + return; + case BJ_Metadata: + transaction->t_nr_buffers++; + list = &transaction->t_buffers; + break; + case BJ_Forget: + list = &transaction->t_forget; + break; + case BJ_Shadow: + list = &transaction->t_shadow_list; + break; + case BJ_Reserved: + list = &transaction->t_reserved_list; + break; + } + + __blist_add_buffer(list, jh); + jh->b_jlist = jlist; + + if (was_dirty) + set_buffer_jbddirty(bh); +} + +void jbd2_journal_file_buffer(struct journal_head *jh, + transaction_t *transaction, int jlist) +{ + spin_lock(&jh->b_state_lock); + spin_lock(&transaction->t_journal->j_list_lock); + __jbd2_journal_file_buffer(jh, transaction, jlist); + spin_unlock(&transaction->t_journal->j_list_lock); + spin_unlock(&jh->b_state_lock); +} + +/* + * Remove a buffer from its current buffer list in preparation for + * dropping it from its current transaction entirely. If the buffer has + * already started to be used by a subsequent transaction, refile the + * buffer on that transaction's metadata list. + * + * Called under j_list_lock + * Called under jh->b_state_lock + * + * When this function returns true, there's no next transaction to refile to + * and the caller has to drop jh reference through + * jbd2_journal_put_journal_head(). + */ +bool __jbd2_journal_refile_buffer(struct journal_head *jh) +{ + int was_dirty, jlist; + struct buffer_head *bh = jh2bh(jh); + + lockdep_assert_held(&jh->b_state_lock); + if (jh->b_transaction) + assert_spin_locked(&jh->b_transaction->t_journal->j_list_lock); + + /* If the buffer is now unused, just drop it. */ + if (jh->b_next_transaction == NULL) { + __jbd2_journal_unfile_buffer(jh); + return true; + } + + /* + * It has been modified by a later transaction: add it to the new + * transaction's metadata list. + */ + + was_dirty = test_clear_buffer_jbddirty(bh); + __jbd2_journal_temp_unlink_buffer(jh); + + /* + * b_transaction must be set, otherwise the new b_transaction won't + * be holding jh reference + */ + J_ASSERT_JH(jh, jh->b_transaction != NULL); + + /* + * We set b_transaction here because b_next_transaction will inherit + * our jh reference and thus __jbd2_journal_file_buffer() must not + * take a new one. + */ + WRITE_ONCE(jh->b_transaction, jh->b_next_transaction); + WRITE_ONCE(jh->b_next_transaction, NULL); + if (buffer_freed(bh)) + jlist = BJ_Forget; + else if (jh->b_modified) + jlist = BJ_Metadata; + else + jlist = BJ_Reserved; + __jbd2_journal_file_buffer(jh, jh->b_transaction, jlist); + J_ASSERT_JH(jh, jh->b_transaction->t_state == T_RUNNING); + + if (was_dirty) + set_buffer_jbddirty(bh); + return false; +} + +/* + * __jbd2_journal_refile_buffer() with necessary locking added. We take our + * bh reference so that we can safely unlock bh. + * + * The jh and bh may be freed by this call. + */ +void jbd2_journal_refile_buffer(journal_t *journal, struct journal_head *jh) +{ + bool drop; + + spin_lock(&jh->b_state_lock); + spin_lock(&journal->j_list_lock); + drop = __jbd2_journal_refile_buffer(jh); + spin_unlock(&jh->b_state_lock); + spin_unlock(&journal->j_list_lock); + if (drop) + jbd2_journal_put_journal_head(jh); +} + +/* + * File inode in the inode list of the handle's transaction + */ +static int jbd2_journal_file_inode(handle_t *handle, struct jbd2_inode *jinode, + unsigned long flags, loff_t start_byte, loff_t end_byte) +{ + transaction_t *transaction = handle->h_transaction; + journal_t *journal; + pgoff_t start_page, end_page; + int err = 0; + int abort_transaction = 0; + + if (is_handle_aborted(handle)) + return -EROFS; + journal = transaction->t_journal; + + jbd2_debug(4, "Adding inode %llu, tid:%d\n", jinode->i_vfs_inode->i_ino, + transaction->t_tid); + + start_page = (pgoff_t)(start_byte >> PAGE_SHIFT); + end_page = (pgoff_t)(end_byte >> PAGE_SHIFT) + 1; + + spin_lock(&journal->j_list_lock); + WRITE_ONCE(jinode->i_flags, jinode->i_flags | flags); + + if (jinode->i_dirty_start_page != jinode->i_dirty_end_page) { + WRITE_ONCE(jinode->i_dirty_start_page, + min(jinode->i_dirty_start_page, start_page)); + WRITE_ONCE(jinode->i_dirty_end_page, + max(jinode->i_dirty_end_page, end_page)); + } else { + /* Publish a new non-empty range by making end visible first. */ + WRITE_ONCE(jinode->i_dirty_end_page, end_page); + WRITE_ONCE(jinode->i_dirty_start_page, start_page); + } + + /* Is inode already attached where we need it? */ + if (jinode->i_transaction == transaction || + jinode->i_next_transaction == transaction) + goto done; + + /* + * We only ever set this variable to 1 so the test is safe. Since + * t_need_data_flush is likely to be set, we do the test to save some + * cacheline bouncing + */ + if (!transaction->t_need_data_flush) + transaction->t_need_data_flush = 1; + /* On some different transaction's list - should be + * the committing one */ + if (jinode->i_transaction) { + if (WARN_ON_ONCE(jinode->i_next_transaction || + jinode->i_transaction != + journal->j_committing_transaction)) { + pr_err("JBD2: %s: assertion failure: i_next_transaction=%p i_transaction=%p j_committing_transaction=%p\n", + journal->j_devname, jinode->i_next_transaction, + jinode->i_transaction, + journal->j_committing_transaction); + err = -EINVAL; + abort_transaction = 1; + goto done; + } + jinode->i_next_transaction = transaction; + goto done; + } + /* Not on any transaction list... */ + if (WARN_ON_ONCE(jinode->i_next_transaction)) { + err = -EINVAL; + abort_transaction = 1; + goto done; + } + jinode->i_transaction = transaction; + list_add(&jinode->i_list, &transaction->t_inode_list); +done: + spin_unlock(&journal->j_list_lock); + if (abort_transaction) + jbd2_journal_abort(journal, err); + return err; +} + +int jbd2_journal_inode_ranged_write(handle_t *handle, + struct jbd2_inode *jinode, loff_t start_byte, loff_t length) +{ + return jbd2_journal_file_inode(handle, jinode, + JI_WRITE_DATA | JI_WAIT_DATA, start_byte, + start_byte + length - 1); +} + +int jbd2_journal_inode_ranged_wait(handle_t *handle, struct jbd2_inode *jinode, + loff_t start_byte, loff_t length) +{ + return jbd2_journal_file_inode(handle, jinode, JI_WAIT_DATA, + start_byte, start_byte + length - 1); +} + +/* + * File truncate and transaction commit interact with each other in a + * non-trivial way. If a transaction writing data block A is + * committing, we cannot discard the data by truncate until we have + * written them. Otherwise if we crashed after the transaction with + * write has committed but before the transaction with truncate has + * committed, we could see stale data in block A. This function is a + * helper to solve this problem. It starts writeout of the truncated + * part in case it is in the committing transaction. + * + * Filesystem code must call this function when inode is journaled in + * ordered mode before truncation happens and after the inode has been + * placed on orphan list with the new inode size. The second condition + * avoids the race that someone writes new data and we start + * committing the transaction after this function has been called but + * before a transaction for truncate is started (and furthermore it + * allows us to optimize the case where the addition to orphan list + * happens in the same transaction as write --- we don't have to write + * any data in such case). + */ +int jbd2_journal_begin_ordered_truncate(journal_t *journal, + struct jbd2_inode *jinode, + loff_t new_size) +{ + transaction_t *inode_trans, *commit_trans; + int ret = 0; + + /* This is a quick check to avoid locking if not necessary */ + if (!READ_ONCE(jinode->i_transaction)) + goto out; + /* Locks are here just to force reading of recent values, it is + * enough that the transaction was not committing before we started + * a transaction adding the inode to orphan list */ + read_lock(&journal->j_state_lock); + commit_trans = journal->j_committing_transaction; + read_unlock(&journal->j_state_lock); + spin_lock(&journal->j_list_lock); + inode_trans = jinode->i_transaction; + spin_unlock(&journal->j_list_lock); + if (inode_trans == commit_trans) { + ret = filemap_fdatawrite_range(jinode->i_vfs_inode->i_mapping, + new_size, LLONG_MAX); + if (ret) + jbd2_journal_abort(journal, ret); + } +out: + return ret; +} diff --git a/fs/jffs2/Kconfig b/fs/jffs2/Kconfig new file mode 100644 index 000000000..560187d61 --- /dev/null +++ b/fs/jffs2/Kconfig @@ -0,0 +1,186 @@ +# SPDX-License-Identifier: GPL-2.0-only +config JFFS2_FS + tristate "Journalling Flash File System v2 (JFFS2) support" + select CRC32 + depends on MTD + help + JFFS2 is the second generation of the Journalling Flash File System + for use on diskless embedded devices. It provides improved wear + levelling, compression and support for hard links. You cannot use + this on normal block devices, only on 'MTD' devices. + + Further information on the design and implementation of JFFS2 is + available at . + +config JFFS2_FS_DEBUG + int "JFFS2 debugging verbosity (0 = quiet, 2 = noisy)" + depends on JFFS2_FS + default "0" + help + This controls the amount of debugging messages produced by the JFFS2 + code. Set it to zero for use in production systems. For evaluation, + testing and debugging, it's advisable to set it to one. This will + enable a few assertions and will print debugging messages at the + KERN_DEBUG loglevel, where they won't normally be visible. Level 2 + is unlikely to be useful - it enables extra debugging in certain + areas which at one point needed debugging, but when the bugs were + located and fixed, the detailed messages were relegated to level 2. + + If reporting bugs, please try to have available a full dump of the + messages at debug level 1 while the misbehaviour was occurring. + +config JFFS2_FS_WRITEBUFFER + bool "JFFS2 write-buffering support" + depends on JFFS2_FS + default y + help + This enables the write-buffering support in JFFS2. + + This functionality is required to support JFFS2 on the following + types of flash devices: + - NAND flash + - NOR flash with transparent ECC + - DataFlash + +config JFFS2_FS_WBUF_VERIFY + bool "Verify JFFS2 write-buffer reads" + depends on JFFS2_FS_WRITEBUFFER + default n + help + This causes JFFS2 to read back every page written through the + write-buffer, and check for errors. + +config JFFS2_SUMMARY + bool "JFFS2 summary support" + depends on JFFS2_FS + default n + help + This feature makes it possible to use summary information + for faster filesystem mount. + + The summary information can be inserted into a filesystem image + by the utility 'sumtool'. + + If unsure, say 'N'. + +config JFFS2_FS_XATTR + bool "JFFS2 XATTR support" + depends on JFFS2_FS + default n + help + Extended attributes are name:value pairs associated with inodes by + the kernel or by users (see the attr(5) manual page for details). + + If unsure, say N. + +config JFFS2_FS_POSIX_ACL + bool "JFFS2 POSIX Access Control Lists" + depends on JFFS2_FS_XATTR + default y + select FS_POSIX_ACL + help + Posix Access Control Lists (ACLs) support permissions for users and + groups beyond the owner/group/world scheme. + + If you don't know what Access Control Lists are, say N + +config JFFS2_FS_SECURITY + bool "JFFS2 Security Labels" + depends on JFFS2_FS_XATTR + default y + help + Security labels support alternative access control models + implemented by security modules like SELinux. This option + enables an extended attribute handler for file security + labels in the jffs2 filesystem. + + If you are not using a security module that requires using + extended attributes for file security labels, say N. + +config JFFS2_COMPRESSION_OPTIONS + bool "Advanced compression options for JFFS2" + depends on JFFS2_FS + default n + help + Enabling this option allows you to explicitly choose which + compression modules, if any, are enabled in JFFS2. Removing + compressors can mean you cannot read existing file systems, + and enabling experimental compressors can mean that you + write a file system which cannot be read by a standard kernel. + + If unsure, you should _definitely_ say 'N'. + +config JFFS2_ZLIB + bool "JFFS2 ZLIB compression support" if JFFS2_COMPRESSION_OPTIONS + select ZLIB_INFLATE + select ZLIB_DEFLATE + depends on JFFS2_FS + default y + help + Zlib is designed to be a free, general-purpose, legally unencumbered, + lossless data-compression library for use on virtually any computer + hardware and operating system. See for + further information. + + Say 'Y' if unsure. + +config JFFS2_LZO + bool "JFFS2 LZO compression support" if JFFS2_COMPRESSION_OPTIONS + select LZO_COMPRESS + select LZO_DECOMPRESS + depends on JFFS2_FS + default n + help + minilzo-based compression. Generally works better than Zlib. + + This feature was added in July, 2007. Say 'N' if you need + compatibility with older bootloaders or kernels. + +config JFFS2_RTIME + bool "JFFS2 RTIME compression support" if JFFS2_COMPRESSION_OPTIONS + depends on JFFS2_FS + default y + help + Rtime does manage to recompress already-compressed data. Say 'Y' if unsure. + +config JFFS2_RUBIN + bool "JFFS2 RUBIN compression support" if JFFS2_COMPRESSION_OPTIONS + depends on JFFS2_FS + default n + help + RUBINMIPS and DYNRUBIN compressors. Say 'N' if unsure. + +choice + prompt "JFFS2 default compression mode" + default JFFS2_CMODE_PRIORITY + depends on JFFS2_COMPRESSION_OPTIONS + depends on JFFS2_FS + help + You can set here the default compression mode of JFFS2 from + the available compression modes. Don't touch if unsure. + +config JFFS2_CMODE_NONE + bool "no compression" + help + Uses no compression. + +config JFFS2_CMODE_PRIORITY + bool "priority" + help + Tries the compressors in a predefined order and chooses the first + successful one. + +config JFFS2_CMODE_SIZE + bool "size" + help + Tries all compressors and chooses the one which has the smallest + result. + +config JFFS2_CMODE_FAVOURLZO + bool "Favour LZO" + help + Tries all compressors and chooses the one which has the smallest + result but gives some preference to LZO (which has faster + decompression) at the expense of size. + +endchoice diff --git a/fs/jffs2/LICENCE b/fs/jffs2/LICENCE new file mode 100644 index 000000000..562885908 --- /dev/null +++ b/fs/jffs2/LICENCE @@ -0,0 +1,30 @@ +The files in this directory and elsewhere which refer to this LICENCE +file are part of JFFS2, the Journalling Flash File System v2. + + Copyright © 2001-2007 Red Hat, Inc. and others + +JFFS2 is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2 or (at your option) any later +version. + +JFFS2 is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +for more details. + +You should have received a copy of the GNU General Public License along +with JFFS2; if not, write to the Free Software Foundation, Inc., +59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +As a special exception, if other files instantiate templates or use +macros or inline functions from these files, or you compile these +files and link them with other works to produce a work based on these +files, these files do not by themselves cause the resulting work to be +covered by the GNU General Public License. However the source code for +these files must still be made available in accordance with section (3) +of the GNU General Public License. + +This exception does not invalidate any other reasons why a work based on +this file might be covered by the GNU General Public License. + diff --git a/fs/jffs2/Makefile b/fs/jffs2/Makefile new file mode 100644 index 000000000..5294969d5 --- /dev/null +++ b/fs/jffs2/Makefile @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the Linux Journalling Flash File System v2 (JFFS2) +# +# + +obj-$(CONFIG_JFFS2_FS) += jffs2.o + +jffs2-y := compr.o dir.o file.o ioctl.o nodelist.o malloc.o +jffs2-y += read.o nodemgmt.o readinode.o write.o scan.o gc.o +jffs2-y += symlink.o build.o erase.o background.o fs.o writev.o +jffs2-y += super.o debug.o + +jffs2-$(CONFIG_JFFS2_FS_WRITEBUFFER) += wbuf.o +jffs2-$(CONFIG_JFFS2_FS_XATTR) += xattr.o xattr_trusted.o xattr_user.o +jffs2-$(CONFIG_JFFS2_FS_SECURITY) += security.o +jffs2-$(CONFIG_JFFS2_FS_POSIX_ACL) += acl.o +jffs2-$(CONFIG_JFFS2_RUBIN) += compr_rubin.o +jffs2-$(CONFIG_JFFS2_RTIME) += compr_rtime.o +jffs2-$(CONFIG_JFFS2_ZLIB) += compr_zlib.o +jffs2-$(CONFIG_JFFS2_LZO) += compr_lzo.o +jffs2-$(CONFIG_JFFS2_SUMMARY) += summary.o diff --git a/fs/jffs2/README.Locking b/fs/jffs2/README.Locking new file mode 100644 index 000000000..8918ac905 --- /dev/null +++ b/fs/jffs2/README.Locking @@ -0,0 +1,169 @@ + + JFFS2 LOCKING DOCUMENTATION + --------------------------- + +This document attempts to describe the existing locking rules for +JFFS2. It is not expected to remain perfectly up to date, but ought to +be fairly close. + + + alloc_sem + --------- + +The alloc_sem is a per-filesystem mutex, used primarily to ensure +contiguous allocation of space on the medium. It is automatically +obtained during space allocations (jffs2_reserve_space()) and freed +upon write completion (jffs2_complete_reservation()). Note that +the garbage collector will obtain this right at the beginning of +jffs2_garbage_collect_pass() and release it at the end, thereby +preventing any other write activity on the file system during a +garbage collect pass. + +When writing new nodes, the alloc_sem must be held until the new nodes +have been properly linked into the data structures for the inode to +which they belong. This is for the benefit of NAND flash - adding new +nodes to an inode may obsolete old ones, and by holding the alloc_sem +until this happens we ensure that any data in the write-buffer at the +time this happens are part of the new node, not just something that +was written afterwards. Hence, we can ensure the newly-obsoleted nodes +don't actually get erased until the write-buffer has been flushed to +the medium. + +With the introduction of NAND flash support and the write-buffer, +the alloc_sem is also used to protect the wbuf-related members of the +jffs2_sb_info structure. Atomically reading the wbuf_len member to see +if the wbuf is currently holding any data is permitted, though. + +Ordering constraints: See f->sem. + + + File Mutex f->sem + --------------------- + +This is the JFFS2-internal equivalent of the inode mutex i->i_sem. +It protects the contents of the jffs2_inode_info private inode data, +including the linked list of node fragments (but see the notes below on +erase_completion_lock), etc. + +The reason that the i_sem itself isn't used for this purpose is to +avoid deadlocks with garbage collection -- the VFS will lock the i_sem +before calling a function which may need to allocate space. The +allocation may trigger garbage-collection, which may need to move a +node belonging to the inode which was locked in the first place by the +VFS. If the garbage collection code were to attempt to lock the i_sem +of the inode from which it's garbage-collecting a physical node, this +lead to deadlock, unless we played games with unlocking the i_sem +before calling the space allocation functions. + +Instead of playing such games, we just have an extra internal +mutex, which is obtained by the garbage collection code and also +by the normal file system code _after_ allocation of space. + +Ordering constraints: + + 1. Never attempt to allocate space or lock alloc_sem with + any f->sem held. + 2. Never attempt to lock two file mutexes in one thread. + No ordering rules have been made for doing so. + 3. Never lock a page cache page with f->sem held. + + + erase_completion_lock spinlock + ------------------------------ + +This is used to serialise access to the eraseblock lists, to the +per-eraseblock lists of physical jffs2_raw_node_ref structures, and +(NB) the per-inode list of physical nodes. The latter is a special +case - see below. + +As the MTD API no longer permits erase-completion callback functions +to be called from bottom-half (timer) context (on the basis that nobody +ever actually implemented such a thing), it's now sufficient to use +a simple spin_lock() rather than spin_lock_bh(). + +Note that the per-inode list of physical nodes (f->nodes) is a special +case. Any changes to _valid_ nodes (i.e. ->flash_offset & 1 == 0) in +the list are protected by the file mutex f->sem. But the erase code +may remove _obsolete_ nodes from the list while holding only the +erase_completion_lock. So you can walk the list only while holding the +erase_completion_lock, and can drop the lock temporarily mid-walk as +long as the pointer you're holding is to a _valid_ node, not an +obsolete one. + +The erase_completion_lock is also used to protect the c->gc_task +pointer when the garbage collection thread exits. The code to kill the +GC thread locks it, sends the signal, then unlocks it - while the GC +thread itself locks it, zeroes c->gc_task, then unlocks on the exit path. + + + inocache_lock spinlock + ---------------------- + +This spinlock protects the hashed list (c->inocache_list) of the +in-core jffs2_inode_cache objects (each inode in JFFS2 has the +correspondent jffs2_inode_cache object). So, the inocache_lock +has to be locked while walking the c->inocache_list hash buckets. + +This spinlock also covers allocation of new inode numbers, which is +currently just '++->highest_ino++', but might one day get more complicated +if we need to deal with wrapping after 4 milliard inode numbers are used. + +Note, the f->sem guarantees that the correspondent jffs2_inode_cache +will not be removed. So, it is allowed to access it without locking +the inocache_lock spinlock. + +Ordering constraints: + + If both erase_completion_lock and inocache_lock are needed, the + c->erase_completion has to be acquired first. + + + erase_free_sem + -------------- + +This mutex is only used by the erase code which frees obsolete node +references and the jffs2_garbage_collect_deletion_dirent() function. +The latter function on NAND flash must read _obsolete_ nodes to +determine whether the 'deletion dirent' under consideration can be +discarded or whether it is still required to show that an inode has +been unlinked. Because reading from the flash may sleep, the +erase_completion_lock cannot be held, so an alternative, more +heavyweight lock was required to prevent the erase code from freeing +the jffs2_raw_node_ref structures in question while the garbage +collection code is looking at them. + +Suggestions for alternative solutions to this problem would be welcomed. + + + wbuf_sem + -------- + +This read/write semaphore protects against concurrent access to the +write-behind buffer ('wbuf') used for flash chips where we must write +in blocks. It protects both the contents of the wbuf and the metadata +which indicates which flash region (if any) is currently covered by +the buffer. + +Ordering constraints: + Lock wbuf_sem last, after the alloc_sem or and f->sem. + + + c->xattr_sem + ------------ + +This read/write semaphore protects against concurrent access to the +xattr related objects which include stuff in superblock and ic->xref. +In read-only path, write-semaphore is too much exclusion. It's enough +by read-semaphore. But you must hold write-semaphore when updating, +creating or deleting any xattr related object. + +Once xattr_sem released, there would be no assurance for the existence +of those objects. Thus, a series of processes is often required to retry, +when updating such a object is necessary under holding read semaphore. +For example, do_jffs2_getxattr() holds read-semaphore to scan xref and +xdatum at first. But it retries this process with holding write-semaphore +after release read-semaphore, if it's necessary to load name/value pair +from medium. + +Ordering constraints: + Lock xattr_sem last, after the alloc_sem. diff --git a/fs/jffs2/acl.c b/fs/jffs2/acl.c new file mode 100644 index 000000000..f0f8a4f57 --- /dev/null +++ b/fs/jffs2/acl.c @@ -0,0 +1,312 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2006 NEC Corporation + * + * Created by KaiGai Kohei + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "nodelist.h" + +static size_t jffs2_acl_size(int count) +{ + if (count <= 4) { + return sizeof(struct jffs2_acl_header) + + count * sizeof(struct jffs2_acl_entry_short); + } else { + return sizeof(struct jffs2_acl_header) + + 4 * sizeof(struct jffs2_acl_entry_short) + + (count - 4) * sizeof(struct jffs2_acl_entry); + } +} + +static int jffs2_acl_count(size_t size) +{ + size_t s; + + size -= sizeof(struct jffs2_acl_header); + if (size < 4 * sizeof(struct jffs2_acl_entry_short)) { + if (size % sizeof(struct jffs2_acl_entry_short)) + return -1; + return size / sizeof(struct jffs2_acl_entry_short); + } else { + s = size - 4 * sizeof(struct jffs2_acl_entry_short); + if (s % sizeof(struct jffs2_acl_entry)) + return -1; + return s / sizeof(struct jffs2_acl_entry) + 4; + } +} + +static struct posix_acl *jffs2_acl_from_medium(void *value, size_t size) +{ + void *end = value + size; + struct jffs2_acl_header *header = value; + struct jffs2_acl_entry *entry; + struct posix_acl *acl; + uint32_t ver; + int i, count; + + if (!value) + return NULL; + if (size < sizeof(struct jffs2_acl_header)) + return ERR_PTR(-EINVAL); + ver = je32_to_cpu(header->a_version); + if (ver != JFFS2_ACL_VERSION) { + JFFS2_WARNING("Invalid ACL version. (=%u)\n", ver); + return ERR_PTR(-EINVAL); + } + + value += sizeof(struct jffs2_acl_header); + count = jffs2_acl_count(size); + if (count < 0) + return ERR_PTR(-EINVAL); + if (count == 0) + return NULL; + + acl = posix_acl_alloc(count, GFP_KERNEL); + if (!acl) + return ERR_PTR(-ENOMEM); + + for (i=0; i < count; i++) { + entry = value; + if (value + sizeof(struct jffs2_acl_entry_short) > end) + goto fail; + acl->a_entries[i].e_tag = je16_to_cpu(entry->e_tag); + acl->a_entries[i].e_perm = je16_to_cpu(entry->e_perm); + switch (acl->a_entries[i].e_tag) { + case ACL_USER_OBJ: + case ACL_GROUP_OBJ: + case ACL_MASK: + case ACL_OTHER: + value += sizeof(struct jffs2_acl_entry_short); + break; + + case ACL_USER: + value += sizeof(struct jffs2_acl_entry); + if (value > end) + goto fail; + acl->a_entries[i].e_uid = + make_kuid(&init_user_ns, + je32_to_cpu(entry->e_id)); + break; + case ACL_GROUP: + value += sizeof(struct jffs2_acl_entry); + if (value > end) + goto fail; + acl->a_entries[i].e_gid = + make_kgid(&init_user_ns, + je32_to_cpu(entry->e_id)); + break; + + default: + goto fail; + } + } + if (value != end) + goto fail; + return acl; + fail: + posix_acl_release(acl); + return ERR_PTR(-EINVAL); +} + +static void *jffs2_acl_to_medium(const struct posix_acl *acl, size_t *size) +{ + struct jffs2_acl_header *header; + struct jffs2_acl_entry *entry; + void *e; + size_t i; + + *size = jffs2_acl_size(acl->a_count); + header = kmalloc_flex(*header, a_entries, acl->a_count); + if (!header) + return ERR_PTR(-ENOMEM); + header->a_version = cpu_to_je32(JFFS2_ACL_VERSION); + e = header + 1; + for (i=0; i < acl->a_count; i++) { + const struct posix_acl_entry *acl_e = &acl->a_entries[i]; + entry = e; + entry->e_tag = cpu_to_je16(acl_e->e_tag); + entry->e_perm = cpu_to_je16(acl_e->e_perm); + switch(acl_e->e_tag) { + case ACL_USER: + entry->e_id = cpu_to_je32( + from_kuid(&init_user_ns, acl_e->e_uid)); + e += sizeof(struct jffs2_acl_entry); + break; + case ACL_GROUP: + entry->e_id = cpu_to_je32( + from_kgid(&init_user_ns, acl_e->e_gid)); + e += sizeof(struct jffs2_acl_entry); + break; + + case ACL_USER_OBJ: + case ACL_GROUP_OBJ: + case ACL_MASK: + case ACL_OTHER: + e += sizeof(struct jffs2_acl_entry_short); + break; + + default: + goto fail; + } + } + return header; + fail: + kfree(header); + return ERR_PTR(-EINVAL); +} + +struct posix_acl *jffs2_get_acl(struct inode *inode, int type, bool rcu) +{ + struct posix_acl *acl; + char *value = NULL; + int rc, xprefix; + + if (rcu) + return ERR_PTR(-ECHILD); + + switch (type) { + case ACL_TYPE_ACCESS: + xprefix = JFFS2_XPREFIX_ACL_ACCESS; + break; + case ACL_TYPE_DEFAULT: + xprefix = JFFS2_XPREFIX_ACL_DEFAULT; + break; + default: + BUG(); + } + rc = do_jffs2_getxattr(inode, xprefix, "", NULL, 0); + if (rc > 0) { + value = kmalloc(rc, GFP_KERNEL); + if (!value) + return ERR_PTR(-ENOMEM); + rc = do_jffs2_getxattr(inode, xprefix, "", value, rc); + } + if (rc > 0) { + acl = jffs2_acl_from_medium(value, rc); + } else if (rc == -ENODATA || rc == -ENOSYS) { + acl = NULL; + } else { + acl = ERR_PTR(rc); + } + kfree(value); + return acl; +} + +static int __jffs2_set_acl(struct inode *inode, int xprefix, struct posix_acl *acl) +{ + char *value = NULL; + size_t size = 0; + int rc; + + if (acl) { + value = jffs2_acl_to_medium(acl, &size); + if (IS_ERR(value)) + return PTR_ERR(value); + } + rc = do_jffs2_setxattr(inode, xprefix, "", value, size, 0); + if (!value && rc == -ENODATA) + rc = 0; + kfree(value); + + return rc; +} + +int jffs2_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type) +{ + int rc, xprefix; + struct inode *inode = d_inode(dentry); + + switch (type) { + case ACL_TYPE_ACCESS: + xprefix = JFFS2_XPREFIX_ACL_ACCESS; + if (acl) { + umode_t mode; + + rc = posix_acl_update_mode(&nop_mnt_idmap, inode, &mode, + &acl); + if (rc) + return rc; + if (inode->i_mode != mode) { + struct iattr attr; + + attr.ia_valid = ATTR_MODE | ATTR_CTIME; + attr.ia_mode = mode; + attr.ia_ctime = current_time(inode); + rc = jffs2_do_setattr(inode, &attr); + if (rc < 0) + return rc; + } + } + break; + case ACL_TYPE_DEFAULT: + xprefix = JFFS2_XPREFIX_ACL_DEFAULT; + if (!S_ISDIR(inode->i_mode)) + return acl ? -EACCES : 0; + break; + default: + return -EINVAL; + } + rc = __jffs2_set_acl(inode, xprefix, acl); + if (!rc) + set_cached_acl(inode, type, acl); + return rc; +} + +int jffs2_init_acl_pre(struct inode *dir_i, struct inode *inode, umode_t *i_mode) +{ + struct posix_acl *default_acl, *acl; + int rc; + + cache_no_acl(inode); + + rc = posix_acl_create(dir_i, i_mode, &default_acl, &acl); + if (rc) + return rc; + + if (default_acl) { + set_cached_acl(inode, ACL_TYPE_DEFAULT, default_acl); + posix_acl_release(default_acl); + } + if (acl) { + set_cached_acl(inode, ACL_TYPE_ACCESS, acl); + posix_acl_release(acl); + } + return 0; +} + +int jffs2_init_acl_post(struct inode *inode) +{ + int rc; + + if (inode->i_default_acl) { + rc = __jffs2_set_acl(inode, JFFS2_XPREFIX_ACL_DEFAULT, inode->i_default_acl); + if (rc) + return rc; + } + + if (inode->i_acl) { + rc = __jffs2_set_acl(inode, JFFS2_XPREFIX_ACL_ACCESS, inode->i_acl); + if (rc) + return rc; + } + + return 0; +} diff --git a/fs/jffs2/acl.h b/fs/jffs2/acl.h new file mode 100644 index 000000000..e976b8cb8 --- /dev/null +++ b/fs/jffs2/acl.h @@ -0,0 +1,43 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2006 NEC Corporation + * + * Created by KaiGai Kohei + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +struct jffs2_acl_entry { + jint16_t e_tag; + jint16_t e_perm; + jint32_t e_id; +}; + +struct jffs2_acl_entry_short { + jint16_t e_tag; + jint16_t e_perm; +}; + +struct jffs2_acl_header { + jint32_t a_version; + struct jffs2_acl_entry a_entries[]; +}; + +#ifdef CONFIG_JFFS2_FS_POSIX_ACL + +struct posix_acl *jffs2_get_acl(struct inode *inode, int type, bool rcu); +int jffs2_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type); +extern int jffs2_init_acl_pre(struct inode *, struct inode *, umode_t *); +extern int jffs2_init_acl_post(struct inode *); + +#else + +#define jffs2_get_acl (NULL) +#define jffs2_set_acl (NULL) +#define jffs2_init_acl_pre(dir_i,inode,mode) (0) +#define jffs2_init_acl_post(inode) (0) + +#endif /* CONFIG_JFFS2_FS_POSIX_ACL */ diff --git a/fs/jffs2/background.c b/fs/jffs2/background.c new file mode 100644 index 000000000..bb0ee1a59 --- /dev/null +++ b/fs/jffs2/background.c @@ -0,0 +1,165 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * Copyright © 2004-2010 David Woodhouse + * + * Created by David Woodhouse + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include "nodelist.h" + + +static int jffs2_garbage_collect_thread(void *); + +void jffs2_garbage_collect_trigger(struct jffs2_sb_info *c) +{ + assert_spin_locked(&c->erase_completion_lock); + if (c->gc_task && jffs2_thread_should_wake(c)) + send_sig(SIGHUP, c->gc_task, 1); +} + +/* This must only ever be called when no GC thread is currently running */ +int jffs2_start_garbage_collect_thread(struct jffs2_sb_info *c) +{ + struct task_struct *tsk; + int ret = 0; + + BUG_ON(c->gc_task); + + init_completion(&c->gc_thread_start); + init_completion(&c->gc_thread_exit); + + tsk = kthread_run(jffs2_garbage_collect_thread, c, "jffs2_gcd_mtd%d", c->mtd->index); + if (IS_ERR(tsk)) { + pr_warn("fork failed for JFFS2 garbage collect thread: %pe\n", + tsk); + complete(&c->gc_thread_exit); + ret = PTR_ERR(tsk); + } else { + /* Wait for it... */ + jffs2_dbg(1, "Garbage collect thread is pid %d\n", tsk->pid); + wait_for_completion(&c->gc_thread_start); + ret = tsk->pid; + } + + return ret; +} + +void jffs2_stop_garbage_collect_thread(struct jffs2_sb_info *c) +{ + int wait = 0; + spin_lock(&c->erase_completion_lock); + if (c->gc_task) { + jffs2_dbg(1, "Killing GC task %d\n", c->gc_task->pid); + send_sig(SIGKILL, c->gc_task, 1); + wait = 1; + } + spin_unlock(&c->erase_completion_lock); + if (wait) + wait_for_completion(&c->gc_thread_exit); +} + +static int jffs2_garbage_collect_thread(void *_c) +{ + struct jffs2_sb_info *c = _c; + sigset_t hupmask; + + siginitset(&hupmask, sigmask(SIGHUP)); + allow_signal(SIGKILL); + allow_signal(SIGSTOP); + allow_signal(SIGHUP); + + c->gc_task = current; + complete(&c->gc_thread_start); + + set_user_nice(current, 10); + + set_freezable(); + for (;;) { + sigprocmask(SIG_UNBLOCK, &hupmask, NULL); + again: + spin_lock(&c->erase_completion_lock); + if (!jffs2_thread_should_wake(c)) { + set_current_state (TASK_INTERRUPTIBLE); + spin_unlock(&c->erase_completion_lock); + jffs2_dbg(1, "%s(): sleeping...\n", __func__); + schedule(); + } else { + spin_unlock(&c->erase_completion_lock); + } + /* Problem - immediately after bootup, the GCD spends a lot + * of time in places like jffs2_kill_fragtree(); so much so + * that userspace processes (like gdm and X) are starved + * despite plenty of cond_resched()s and renicing. Yield() + * doesn't help, either (presumably because userspace and GCD + * are generally competing for a higher latency resource - + * disk). + * This forces the GCD to slow the hell down. Pulling an + * inode in with read_inode() is much preferable to having + * the GC thread get there first. */ + schedule_timeout_interruptible(msecs_to_jiffies(50)); + + if (kthread_should_stop()) { + jffs2_dbg(1, "%s(): kthread_stop() called\n", __func__); + goto die; + } + + /* Put_super will send a SIGKILL and then wait on the sem. + */ + while (signal_pending(current) || freezing(current)) { + unsigned long signr; + + if (try_to_freeze()) + goto again; + + signr = kernel_dequeue_signal(); + + switch(signr) { + case SIGSTOP: + jffs2_dbg(1, "%s(): SIGSTOP received\n", + __func__); + kernel_signal_stop(); + break; + + case SIGKILL: + jffs2_dbg(1, "%s(): SIGKILL received\n", + __func__); + goto die; + + case SIGHUP: + jffs2_dbg(1, "%s(): SIGHUP received\n", + __func__); + break; + default: + jffs2_dbg(1, "%s(): signal %ld received\n", + __func__, signr); + } + } + /* We don't want SIGHUP to interrupt us. STOP and KILL are OK though. */ + sigprocmask(SIG_BLOCK, &hupmask, NULL); + + jffs2_dbg(1, "%s(): pass\n", __func__); + if (jffs2_garbage_collect_pass(c) == -ENOSPC) { + pr_notice("No space for garbage collection. Aborting GC thread\n"); + goto die; + } + } + die: + spin_lock(&c->erase_completion_lock); + c->gc_task = NULL; + spin_unlock(&c->erase_completion_lock); + kthread_complete_and_exit(&c->gc_thread_exit, 0); +} diff --git a/fs/jffs2/build.c b/fs/jffs2/build.c new file mode 100644 index 000000000..6ae9d6fef --- /dev/null +++ b/fs/jffs2/build.c @@ -0,0 +1,434 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * Copyright © 2004-2010 David Woodhouse + * + * Created by David Woodhouse + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include /* kvfree() */ +#include "nodelist.h" + +static void jffs2_build_remove_unlinked_inode(struct jffs2_sb_info *, + struct jffs2_inode_cache *, struct jffs2_full_dirent **); + +static inline struct jffs2_inode_cache * +first_inode_chain(int *i, struct jffs2_sb_info *c) +{ + for (; *i < c->inocache_hashsize; (*i)++) { + if (c->inocache_list[*i]) + return c->inocache_list[*i]; + } + return NULL; +} + +static inline struct jffs2_inode_cache * +next_inode(int *i, struct jffs2_inode_cache *ic, struct jffs2_sb_info *c) +{ + /* More in this chain? */ + if (ic->next) + return ic->next; + (*i)++; + return first_inode_chain(i, c); +} + +#define for_each_inode(i, c, ic) \ + for (i = 0, ic = first_inode_chain(&i, (c)); \ + ic; \ + ic = next_inode(&i, ic, (c))) + + +static void jffs2_build_inode_pass1(struct jffs2_sb_info *c, + struct jffs2_inode_cache *ic, + int *dir_hardlinks) +{ + struct jffs2_full_dirent *fd; + + dbg_fsbuild("building directory inode #%u\n", ic->ino); + + /* For each child, increase nlink */ + for(fd = ic->scan_dents; fd; fd = fd->next) { + struct jffs2_inode_cache *child_ic; + if (!fd->ino) + continue; + + /* we can get high latency here with huge directories */ + + child_ic = jffs2_get_ino_cache(c, fd->ino); + if (!child_ic) { + dbg_fsbuild("child \"%s\" (ino #%u) of dir ino #%u doesn't exist!\n", + fd->name, fd->ino, ic->ino); + jffs2_mark_node_obsolete(c, fd->raw); + /* Clear the ic/raw union so it doesn't cause problems later. */ + fd->ic = NULL; + continue; + } + + /* From this point, fd->raw is no longer used so we can set fd->ic */ + fd->ic = child_ic; + child_ic->pino_nlink++; + /* If we appear (at this stage) to have hard-linked directories, + * set a flag to trigger a scan later */ + if (fd->type == DT_DIR) { + child_ic->flags |= INO_FLAGS_IS_DIR; + if (child_ic->pino_nlink > 1) + *dir_hardlinks = 1; + } + + dbg_fsbuild("increased nlink for child \"%s\" (ino #%u)\n", fd->name, fd->ino); + /* Can't free scan_dents so far. We might need them in pass 2 */ + } +} + +/* Scan plan: + - Scan physical nodes. Build map of inodes/dirents. Allocate inocaches as we go + - Scan directory tree from top down, setting nlink in inocaches + - Scan inocaches for inodes with nlink==0 +*/ +static int jffs2_build_filesystem(struct jffs2_sb_info *c) +{ + int ret, i, dir_hardlinks = 0; + struct jffs2_inode_cache *ic; + struct jffs2_full_dirent *fd; + struct jffs2_full_dirent *dead_fds = NULL; + + dbg_fsbuild("build FS data structures\n"); + + /* First, scan the medium and build all the inode caches with + lists of physical nodes */ + + c->flags |= JFFS2_SB_FLAG_SCANNING; + ret = jffs2_scan_medium(c); + c->flags &= ~JFFS2_SB_FLAG_SCANNING; + if (ret) + goto exit; + + dbg_fsbuild("scanned flash completely\n"); + jffs2_dbg_dump_block_lists_nolock(c); + + dbg_fsbuild("pass 1 starting\n"); + c->flags |= JFFS2_SB_FLAG_BUILDING; + /* Now scan the directory tree, increasing nlink according to every dirent found. */ + for_each_inode(i, c, ic) { + if (ic->scan_dents) { + jffs2_build_inode_pass1(c, ic, &dir_hardlinks); + cond_resched(); + } + } + + dbg_fsbuild("pass 1 complete\n"); + + /* Next, scan for inodes with nlink == 0 and remove them. If + they were directories, then decrement the nlink of their + children too, and repeat the scan. As that's going to be + a fairly uncommon occurrence, it's not so evil to do it this + way. Recursion bad. */ + dbg_fsbuild("pass 2 starting\n"); + + for_each_inode(i, c, ic) { + if (ic->pino_nlink) + continue; + + jffs2_build_remove_unlinked_inode(c, ic, &dead_fds); + cond_resched(); + } + + dbg_fsbuild("pass 2a starting\n"); + + while (dead_fds) { + fd = dead_fds; + dead_fds = fd->next; + + ic = jffs2_get_ino_cache(c, fd->ino); + + if (ic) + jffs2_build_remove_unlinked_inode(c, ic, &dead_fds); + jffs2_free_full_dirent(fd); + } + + dbg_fsbuild("pass 2a complete\n"); + + if (dir_hardlinks) { + /* If we detected directory hardlinks earlier, *hopefully* + * they are gone now because some of the links were from + * dead directories which still had some old dirents lying + * around and not yet garbage-collected, but which have + * been discarded above. So clear the pino_nlink field + * in each directory, so that the final scan below can + * print appropriate warnings. */ + for_each_inode(i, c, ic) { + if (ic->flags & INO_FLAGS_IS_DIR) + ic->pino_nlink = 0; + } + } + dbg_fsbuild("freeing temporary data structures\n"); + + /* Finally, we can scan again and free the dirent structs */ + for_each_inode(i, c, ic) { + while(ic->scan_dents) { + fd = ic->scan_dents; + ic->scan_dents = fd->next; + /* We do use the pino_nlink field to count nlink of + * directories during fs build, so set it to the + * parent ino# now. Now that there's hopefully only + * one. */ + if (fd->type == DT_DIR) { + if (!fd->ic) { + /* We'll have complained about it and marked the coresponding + raw node obsolete already. Just skip it. */ + continue; + } + + /* We *have* to have set this in jffs2_build_inode_pass1() */ + BUG_ON(!(fd->ic->flags & INO_FLAGS_IS_DIR)); + + /* We clear ic->pino_nlink ∀ directories' ic *only* if dir_hardlinks + * is set. Otherwise, we know this should never trigger anyway, so + * we don't do the check. And ic->pino_nlink still contains the nlink + * value (which is 1). */ + if (dir_hardlinks && fd->ic->pino_nlink) { + JFFS2_ERROR("child dir \"%s\" (ino #%u) of dir ino #%u is also hard linked from dir ino #%u\n", + fd->name, fd->ino, ic->ino, fd->ic->pino_nlink); + /* Should we unlink it from its previous parent? */ + } + + /* For directories, ic->pino_nlink holds that parent inode # */ + fd->ic->pino_nlink = ic->ino; + } + jffs2_free_full_dirent(fd); + } + ic->scan_dents = NULL; + cond_resched(); + } + ret = jffs2_build_xattr_subsystem(c); + if (ret) + goto exit; + + c->flags &= ~JFFS2_SB_FLAG_BUILDING; + + dbg_fsbuild("FS build complete\n"); + + /* Rotate the lists by some number to ensure wear levelling */ + jffs2_rotate_lists(c); + + ret = 0; + +exit: + if (ret) { + for_each_inode(i, c, ic) { + while(ic->scan_dents) { + fd = ic->scan_dents; + ic->scan_dents = fd->next; + jffs2_free_full_dirent(fd); + } + } + jffs2_clear_xattr_subsystem(c); + } + + return ret; +} + +static void jffs2_build_remove_unlinked_inode(struct jffs2_sb_info *c, + struct jffs2_inode_cache *ic, + struct jffs2_full_dirent **dead_fds) +{ + struct jffs2_raw_node_ref *raw; + struct jffs2_full_dirent *fd; + + dbg_fsbuild("removing ino #%u with nlink == zero.\n", ic->ino); + + raw = ic->nodes; + while (raw != (void *)ic) { + struct jffs2_raw_node_ref *next = raw->next_in_ino; + dbg_fsbuild("obsoleting node at 0x%08x\n", ref_offset(raw)); + jffs2_mark_node_obsolete(c, raw); + raw = next; + } + + if (ic->scan_dents) { + int whinged = 0; + dbg_fsbuild("inode #%u was a directory which may have children...\n", ic->ino); + + while(ic->scan_dents) { + struct jffs2_inode_cache *child_ic; + + fd = ic->scan_dents; + ic->scan_dents = fd->next; + + if (!fd->ino) { + /* It's a deletion dirent. Ignore it */ + dbg_fsbuild("child \"%s\" is a deletion dirent, skipping...\n", fd->name); + jffs2_free_full_dirent(fd); + continue; + } + if (!whinged) + whinged = 1; + + dbg_fsbuild("removing child \"%s\", ino #%u\n", fd->name, fd->ino); + + child_ic = jffs2_get_ino_cache(c, fd->ino); + if (!child_ic) { + dbg_fsbuild("cannot remove child \"%s\", ino #%u, because it doesn't exist\n", + fd->name, fd->ino); + jffs2_free_full_dirent(fd); + continue; + } + + /* Reduce nlink of the child. If it's now zero, stick it on the + dead_fds list to be cleaned up later. Else just free the fd */ + child_ic->pino_nlink--; + + if (!child_ic->pino_nlink) { + dbg_fsbuild("inode #%u (\"%s\") now has no links; adding to dead_fds list.\n", + fd->ino, fd->name); + fd->next = *dead_fds; + *dead_fds = fd; + } else { + dbg_fsbuild("inode #%u (\"%s\") has now got nlink %d. Ignoring.\n", + fd->ino, fd->name, child_ic->pino_nlink); + jffs2_free_full_dirent(fd); + } + } + } + + /* + We don't delete the inocache from the hash list and free it yet. + The erase code will do that, when all the nodes are completely gone. + */ +} + +static void jffs2_calc_trigger_levels(struct jffs2_sb_info *c) +{ + uint32_t size; + + /* Deletion should almost _always_ be allowed. We're fairly + buggered once we stop allowing people to delete stuff + because there's not enough free space... */ + c->resv_blocks_deletion = 2; + + /* Be conservative about how much space we need before we allow writes. + On top of that which is required for deletia, require an extra 2% + of the medium to be available, for overhead caused by nodes being + split across blocks, etc. */ + + size = c->flash_size / 50; /* 2% of flash size */ + size += c->nr_blocks * 100; /* And 100 bytes per eraseblock */ + size += c->sector_size - 1; /* ... and round up */ + + c->resv_blocks_write = c->resv_blocks_deletion + (size / c->sector_size); + + /* When do we let the GC thread run in the background */ + + c->resv_blocks_gctrigger = c->resv_blocks_write + 1; + + /* When do we allow garbage collection to merge nodes to make + long-term progress at the expense of short-term space exhaustion? */ + c->resv_blocks_gcmerge = c->resv_blocks_deletion + 1; + + /* When do we allow garbage collection to eat from bad blocks rather + than actually making progress? */ + c->resv_blocks_gcbad = 0;//c->resv_blocks_deletion + 2; + + /* What number of 'very dirty' eraseblocks do we allow before we + trigger the GC thread even if we don't _need_ the space. When we + can't mark nodes obsolete on the medium, the old dirty nodes cause + performance problems because we have to inspect and discard them. */ + c->vdirty_blocks_gctrigger = c->resv_blocks_gctrigger; + if (jffs2_can_mark_obsolete(c)) + c->vdirty_blocks_gctrigger *= 10; + + /* If there's less than this amount of dirty space, don't bother + trying to GC to make more space. It'll be a fruitless task */ + c->nospc_dirty_size = c->sector_size + (c->flash_size / 100); + + dbg_fsbuild("trigger levels (size %d KiB, block size %d KiB, %d blocks)\n", + c->flash_size / 1024, c->sector_size / 1024, c->nr_blocks); + dbg_fsbuild("Blocks required to allow deletion: %d (%d KiB)\n", + c->resv_blocks_deletion, c->resv_blocks_deletion*c->sector_size/1024); + dbg_fsbuild("Blocks required to allow writes: %d (%d KiB)\n", + c->resv_blocks_write, c->resv_blocks_write*c->sector_size/1024); + dbg_fsbuild("Blocks required to quiesce GC thread: %d (%d KiB)\n", + c->resv_blocks_gctrigger, c->resv_blocks_gctrigger*c->sector_size/1024); + dbg_fsbuild("Blocks required to allow GC merges: %d (%d KiB)\n", + c->resv_blocks_gcmerge, c->resv_blocks_gcmerge*c->sector_size/1024); + dbg_fsbuild("Blocks required to GC bad blocks: %d (%d KiB)\n", + c->resv_blocks_gcbad, c->resv_blocks_gcbad*c->sector_size/1024); + dbg_fsbuild("Amount of dirty space required to GC: %d bytes\n", + c->nospc_dirty_size); + dbg_fsbuild("Very dirty blocks before GC triggered: %d\n", + c->vdirty_blocks_gctrigger); +} + +int jffs2_do_mount_fs(struct jffs2_sb_info *c) +{ + int ret; + int i; + int size; + + c->free_size = c->flash_size; + c->nr_blocks = c->flash_size / c->sector_size; + size = sizeof(struct jffs2_eraseblock) * c->nr_blocks; +#ifndef __ECOS + if (jffs2_blocks_use_vmalloc(c)) + c->blocks = vzalloc(size); + else +#endif + c->blocks = kzalloc(size, GFP_KERNEL); + if (!c->blocks) + return -ENOMEM; + + for (i=0; inr_blocks; i++) { + INIT_LIST_HEAD(&c->blocks[i].list); + c->blocks[i].offset = i * c->sector_size; + c->blocks[i].free_size = c->sector_size; + } + + INIT_LIST_HEAD(&c->clean_list); + INIT_LIST_HEAD(&c->very_dirty_list); + INIT_LIST_HEAD(&c->dirty_list); + INIT_LIST_HEAD(&c->erasable_list); + INIT_LIST_HEAD(&c->erasing_list); + INIT_LIST_HEAD(&c->erase_checking_list); + INIT_LIST_HEAD(&c->erase_pending_list); + INIT_LIST_HEAD(&c->erasable_pending_wbuf_list); + INIT_LIST_HEAD(&c->erase_complete_list); + INIT_LIST_HEAD(&c->free_list); + INIT_LIST_HEAD(&c->bad_list); + INIT_LIST_HEAD(&c->bad_used_list); + c->highest_ino = 1; + c->summary = NULL; + + ret = jffs2_sum_init(c); + if (ret) + goto out_free; + + if (jffs2_build_filesystem(c)) { + dbg_fsbuild("build_fs failed\n"); + jffs2_free_ino_caches(c); + jffs2_free_raw_node_refs(c); + ret = -EIO; + goto out_sum_exit; + } + + jffs2_calc_trigger_levels(c); + + return 0; + + out_sum_exit: + jffs2_sum_exit(c); + out_free: + kvfree(c->blocks); + + return ret; +} diff --git a/fs/jffs2/compr.c b/fs/jffs2/compr.c new file mode 100644 index 000000000..764f19dec --- /dev/null +++ b/fs/jffs2/compr.c @@ -0,0 +1,426 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * Copyright © 2004-2010 David Woodhouse + * Copyright © 2004 Ferenc Havasi , + * University of Szeged, Hungary + * + * Created by Arjan van de Ven + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include "compr.h" + +static DEFINE_SPINLOCK(jffs2_compressor_list_lock); + +/* Available compressors are on this list */ +static LIST_HEAD(jffs2_compressor_list); + +/* Actual compression mode */ +static int jffs2_compression_mode = JFFS2_COMPR_MODE_PRIORITY; + +/* Statistics for blocks stored without compression */ +static uint32_t none_stat_compr_blocks=0,none_stat_decompr_blocks=0,none_stat_compr_size=0; + + +/* + * Return 1 to use this compression + */ +static int jffs2_is_best_compression(struct jffs2_compressor *this, + struct jffs2_compressor *best, uint32_t size, uint32_t bestsize) +{ + switch (jffs2_compression_mode) { + case JFFS2_COMPR_MODE_SIZE: + if (bestsize > size) + return 1; + return 0; + case JFFS2_COMPR_MODE_FAVOURLZO: + if ((this->compr == JFFS2_COMPR_LZO) && (bestsize > size)) + return 1; + if ((best->compr != JFFS2_COMPR_LZO) && (bestsize > size)) + return 1; + if ((this->compr == JFFS2_COMPR_LZO) && (bestsize > (size * FAVOUR_LZO_PERCENT / 100))) + return 1; + if ((bestsize * FAVOUR_LZO_PERCENT / 100) > size) + return 1; + + return 0; + } + /* Shouldn't happen */ + return 0; +} + +/* + * jffs2_selected_compress: + * @compr: Explicit compression type to use (ie, JFFS2_COMPR_ZLIB). + * If 0, just take the first available compression mode. + * @data_in: Pointer to uncompressed data + * @cpage_out: Pointer to returned pointer to buffer for compressed data + * @datalen: On entry, holds the amount of data available for compression. + * On exit, expected to hold the amount of data actually compressed. + * @cdatalen: On entry, holds the amount of space available for compressed + * data. On exit, expected to hold the actual size of the compressed + * data. + * + * Returns: the compression type used. Zero is used to show that the data + * could not be compressed; probably because we couldn't find the requested + * compression mode. + */ +static int jffs2_selected_compress(u8 compr, unsigned char *data_in, + unsigned char **cpage_out, u32 *datalen, u32 *cdatalen) +{ + struct jffs2_compressor *this; + int err, ret = JFFS2_COMPR_NONE; + uint32_t orig_slen, orig_dlen; + char *output_buf; + + output_buf = kmalloc(*cdatalen, GFP_KERNEL); + if (!output_buf) { + pr_warn("No memory for compressor allocation. Compression failed.\n"); + return ret; + } + orig_slen = *datalen; + orig_dlen = *cdatalen; + spin_lock(&jffs2_compressor_list_lock); + list_for_each_entry(this, &jffs2_compressor_list, list) { + /* Skip decompress-only and disabled modules */ + if (!this->compress || this->disabled) + continue; + + /* Skip if not the desired compression type */ + if (compr && (compr != this->compr)) + continue; + + /* + * Either compression type was unspecified, or we found our + * compressor; either way, we're good to go. + */ + this->usecount++; + spin_unlock(&jffs2_compressor_list_lock); + + *datalen = orig_slen; + *cdatalen = orig_dlen; + err = this->compress(data_in, output_buf, datalen, cdatalen); + + spin_lock(&jffs2_compressor_list_lock); + this->usecount--; + if (!err) { + /* Success */ + ret = this->compr; + this->stat_compr_blocks++; + this->stat_compr_orig_size += *datalen; + this->stat_compr_new_size += *cdatalen; + break; + } + } + spin_unlock(&jffs2_compressor_list_lock); + if (ret == JFFS2_COMPR_NONE) + kfree(output_buf); + else + *cpage_out = output_buf; + + return ret; +} + +/* jffs2_compress: + * @data_in: Pointer to uncompressed data + * @cpage_out: Pointer to returned pointer to buffer for compressed data + * @datalen: On entry, holds the amount of data available for compression. + * On exit, expected to hold the amount of data actually compressed. + * @cdatalen: On entry, holds the amount of space available for compressed + * data. On exit, expected to hold the actual size of the compressed + * data. + * + * Returns: Lower byte to be stored with data indicating compression type used. + * Zero is used to show that the data could not be compressed - the + * compressed version was actually larger than the original. + * Upper byte will be used later. (soon) + * + * If the cdata buffer isn't large enough to hold all the uncompressed data, + * jffs2_compress should compress as much as will fit, and should set + * *datalen accordingly to show the amount of data which were compressed. + */ +uint16_t jffs2_compress(struct jffs2_sb_info *c, struct jffs2_inode_info *f, + unsigned char *data_in, unsigned char **cpage_out, + uint32_t *datalen, uint32_t *cdatalen) +{ + int ret = JFFS2_COMPR_NONE; + int mode, compr_ret; + struct jffs2_compressor *this, *best=NULL; + unsigned char *output_buf = NULL, *tmp_buf; + uint32_t orig_slen, orig_dlen; + uint32_t best_slen=0, best_dlen=0; + + if (c->mount_opts.override_compr) + mode = c->mount_opts.compr; + else + mode = jffs2_compression_mode; + + switch (mode) { + case JFFS2_COMPR_MODE_NONE: + break; + case JFFS2_COMPR_MODE_PRIORITY: + ret = jffs2_selected_compress(0, data_in, cpage_out, datalen, + cdatalen); + break; + case JFFS2_COMPR_MODE_SIZE: + case JFFS2_COMPR_MODE_FAVOURLZO: + orig_slen = *datalen; + orig_dlen = *cdatalen; + spin_lock(&jffs2_compressor_list_lock); + list_for_each_entry(this, &jffs2_compressor_list, list) { + /* Skip decompress-only backwards-compatibility and disabled modules */ + if ((!this->compress)||(this->disabled)) + continue; + /* Allocating memory for output buffer if necessary */ + if ((this->compr_buf_size < orig_slen) && (this->compr_buf)) { + spin_unlock(&jffs2_compressor_list_lock); + kfree(this->compr_buf); + spin_lock(&jffs2_compressor_list_lock); + this->compr_buf_size=0; + this->compr_buf=NULL; + } + if (!this->compr_buf) { + spin_unlock(&jffs2_compressor_list_lock); + tmp_buf = kmalloc(orig_slen, GFP_KERNEL); + spin_lock(&jffs2_compressor_list_lock); + if (!tmp_buf) { + pr_warn("No memory for compressor allocation. (%d bytes)\n", + orig_slen); + continue; + } + else { + this->compr_buf = tmp_buf; + this->compr_buf_size = orig_slen; + } + } + this->usecount++; + spin_unlock(&jffs2_compressor_list_lock); + *datalen = orig_slen; + *cdatalen = orig_dlen; + compr_ret = this->compress(data_in, this->compr_buf, datalen, cdatalen); + spin_lock(&jffs2_compressor_list_lock); + this->usecount--; + if (!compr_ret) { + if (((!best_dlen) || jffs2_is_best_compression(this, best, *cdatalen, best_dlen)) + && (*cdatalen < *datalen)) { + best_dlen = *cdatalen; + best_slen = *datalen; + best = this; + } + } + } + if (best_dlen) { + *cdatalen = best_dlen; + *datalen = best_slen; + output_buf = best->compr_buf; + best->compr_buf = NULL; + best->compr_buf_size = 0; + best->stat_compr_blocks++; + best->stat_compr_orig_size += best_slen; + best->stat_compr_new_size += best_dlen; + ret = best->compr; + *cpage_out = output_buf; + } + spin_unlock(&jffs2_compressor_list_lock); + break; + case JFFS2_COMPR_MODE_FORCELZO: + ret = jffs2_selected_compress(JFFS2_COMPR_LZO, data_in, + cpage_out, datalen, cdatalen); + break; + case JFFS2_COMPR_MODE_FORCEZLIB: + ret = jffs2_selected_compress(JFFS2_COMPR_ZLIB, data_in, + cpage_out, datalen, cdatalen); + break; + default: + pr_err("unknown compression mode\n"); + } + + if (ret == JFFS2_COMPR_NONE) { + *cpage_out = data_in; + *datalen = *cdatalen; + none_stat_compr_blocks++; + none_stat_compr_size += *datalen; + } + return ret; +} + +int jffs2_decompress(struct jffs2_sb_info *c, struct jffs2_inode_info *f, + uint16_t comprtype, unsigned char *cdata_in, + unsigned char *data_out, uint32_t cdatalen, uint32_t datalen) +{ + struct jffs2_compressor *this; + int ret; + + /* Older code had a bug where it would write non-zero 'usercompr' + fields. Deal with it. */ + if ((comprtype & 0xff) <= JFFS2_COMPR_ZLIB) + comprtype &= 0xff; + + switch (comprtype & 0xff) { + case JFFS2_COMPR_NONE: + /* This should be special-cased elsewhere, but we might as well deal with it */ + memcpy(data_out, cdata_in, datalen); + none_stat_decompr_blocks++; + break; + case JFFS2_COMPR_ZERO: + memset(data_out, 0, datalen); + break; + default: + spin_lock(&jffs2_compressor_list_lock); + list_for_each_entry(this, &jffs2_compressor_list, list) { + if (comprtype == this->compr) { + this->usecount++; + spin_unlock(&jffs2_compressor_list_lock); + ret = this->decompress(cdata_in, data_out, cdatalen, datalen); + spin_lock(&jffs2_compressor_list_lock); + if (ret) { + pr_warn("Decompressor \"%s\" returned %d\n", + this->name, ret); + } + else { + this->stat_decompr_blocks++; + } + this->usecount--; + spin_unlock(&jffs2_compressor_list_lock); + return ret; + } + } + pr_warn("compression type 0x%02x not available\n", comprtype); + spin_unlock(&jffs2_compressor_list_lock); + return -EIO; + } + return 0; +} + +int jffs2_register_compressor(struct jffs2_compressor *comp) +{ + struct jffs2_compressor *this; + + if (!comp->name) { + pr_warn("NULL compressor name at registering JFFS2 compressor. Failed.\n"); + return -1; + } + comp->compr_buf_size=0; + comp->compr_buf=NULL; + comp->usecount=0; + comp->stat_compr_orig_size=0; + comp->stat_compr_new_size=0; + comp->stat_compr_blocks=0; + comp->stat_decompr_blocks=0; + jffs2_dbg(1, "Registering JFFS2 compressor \"%s\"\n", comp->name); + + spin_lock(&jffs2_compressor_list_lock); + + list_for_each_entry(this, &jffs2_compressor_list, list) { + if (this->priority < comp->priority) { + list_add(&comp->list, this->list.prev); + goto out; + } + } + list_add_tail(&comp->list, &jffs2_compressor_list); +out: + D2(list_for_each_entry(this, &jffs2_compressor_list, list) { + printk(KERN_DEBUG "Compressor \"%s\", prio %d\n", this->name, this->priority); + }) + + spin_unlock(&jffs2_compressor_list_lock); + + return 0; +} + +int jffs2_unregister_compressor(struct jffs2_compressor *comp) +{ + D2(struct jffs2_compressor *this); + + jffs2_dbg(1, "Unregistering JFFS2 compressor \"%s\"\n", comp->name); + + spin_lock(&jffs2_compressor_list_lock); + + if (comp->usecount) { + spin_unlock(&jffs2_compressor_list_lock); + pr_warn("Compressor module is in use. Unregister failed.\n"); + return -1; + } + list_del(&comp->list); + + D2(list_for_each_entry(this, &jffs2_compressor_list, list) { + printk(KERN_DEBUG "Compressor \"%s\", prio %d\n", this->name, this->priority); + }) + spin_unlock(&jffs2_compressor_list_lock); + return 0; +} + +void jffs2_free_comprbuf(unsigned char *comprbuf, unsigned char *orig) +{ + if (orig != comprbuf) + kfree(comprbuf); +} + +int __init jffs2_compressors_init(void) +{ + int ret = 0; +/* Registering compressors */ + ret = jffs2_zlib_init(); + if (ret) + goto exit; + ret = jffs2_rtime_init(); + if (ret) + goto exit_zlib; + ret = jffs2_rubinmips_init(); + if (ret) + goto exit_rtime; + ret = jffs2_dynrubin_init(); + if (ret) + goto exit_runinmips; + ret = jffs2_lzo_init(); + if (ret) + goto exit_dynrubin; + + +/* Setting default compression mode */ +#ifdef CONFIG_JFFS2_CMODE_NONE + jffs2_compression_mode = JFFS2_COMPR_MODE_NONE; + jffs2_dbg(1, "default compression mode: none\n"); +#else +#ifdef CONFIG_JFFS2_CMODE_SIZE + jffs2_compression_mode = JFFS2_COMPR_MODE_SIZE; + jffs2_dbg(1, "default compression mode: size\n"); +#else +#ifdef CONFIG_JFFS2_CMODE_FAVOURLZO + jffs2_compression_mode = JFFS2_COMPR_MODE_FAVOURLZO; + jffs2_dbg(1, "default compression mode: favourlzo\n"); +#else + jffs2_dbg(1, "default compression mode: priority\n"); +#endif +#endif +#endif + return 0; + +exit_dynrubin: + jffs2_dynrubin_exit(); +exit_runinmips: + jffs2_rubinmips_exit(); +exit_rtime: + jffs2_rtime_exit(); +exit_zlib: + jffs2_zlib_exit(); +exit: + return ret; +} + +int jffs2_compressors_exit(void) +{ +/* Unregistering compressors */ + jffs2_lzo_exit(); + jffs2_dynrubin_exit(); + jffs2_rubinmips_exit(); + jffs2_rtime_exit(); + jffs2_zlib_exit(); + return 0; +} diff --git a/fs/jffs2/compr.h b/fs/jffs2/compr.h new file mode 100644 index 000000000..3716b6b79 --- /dev/null +++ b/fs/jffs2/compr.h @@ -0,0 +1,119 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2004 Ferenc Havasi , + * University of Szeged, Hungary + * Copyright © 2004-2010 David Woodhouse + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#ifndef __JFFS2_COMPR_H__ +#define __JFFS2_COMPR_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "jffs2_fs_i.h" +#include "jffs2_fs_sb.h" +#include "nodelist.h" + +#define JFFS2_RUBINMIPS_PRIORITY 10 +#define JFFS2_DYNRUBIN_PRIORITY 20 +#define JFFS2_LZARI_PRIORITY 30 +#define JFFS2_RTIME_PRIORITY 50 +#define JFFS2_ZLIB_PRIORITY 60 +#define JFFS2_LZO_PRIORITY 80 + + +#define JFFS2_RUBINMIPS_DISABLED /* RUBINs will be used only */ +#define JFFS2_DYNRUBIN_DISABLED /* for decompression */ + +#define JFFS2_COMPR_MODE_NONE 0 +#define JFFS2_COMPR_MODE_PRIORITY 1 +#define JFFS2_COMPR_MODE_SIZE 2 +#define JFFS2_COMPR_MODE_FAVOURLZO 3 +#define JFFS2_COMPR_MODE_FORCELZO 4 +#define JFFS2_COMPR_MODE_FORCEZLIB 5 + +#define FAVOUR_LZO_PERCENT 80 + +struct jffs2_compressor { + struct list_head list; + int priority; /* used by prirority comr. mode */ + char *name; + char compr; /* JFFS2_COMPR_XXX */ + int (*compress)(unsigned char *data_in, unsigned char *cpage_out, + uint32_t *srclen, uint32_t *destlen); + int (*decompress)(unsigned char *cdata_in, unsigned char *data_out, + uint32_t cdatalen, uint32_t datalen); + int usecount; + int disabled; /* if set the compressor won't compress */ + unsigned char *compr_buf; /* used by size compr. mode */ + uint32_t compr_buf_size; /* used by size compr. mode */ + uint32_t stat_compr_orig_size; + uint32_t stat_compr_new_size; + uint32_t stat_compr_blocks; + uint32_t stat_decompr_blocks; +}; + +int jffs2_register_compressor(struct jffs2_compressor *comp); +int jffs2_unregister_compressor(struct jffs2_compressor *comp); + +int jffs2_compressors_init(void); +int jffs2_compressors_exit(void); + +uint16_t jffs2_compress(struct jffs2_sb_info *c, struct jffs2_inode_info *f, + unsigned char *data_in, unsigned char **cpage_out, + uint32_t *datalen, uint32_t *cdatalen); + +int jffs2_decompress(struct jffs2_sb_info *c, struct jffs2_inode_info *f, + uint16_t comprtype, unsigned char *cdata_in, + unsigned char *data_out, uint32_t cdatalen, uint32_t datalen); + +void jffs2_free_comprbuf(unsigned char *comprbuf, unsigned char *orig); + +/* Compressor modules */ +/* These functions will be called by jffs2_compressors_init/exit */ + +#ifdef CONFIG_JFFS2_RUBIN +int jffs2_rubinmips_init(void); +void jffs2_rubinmips_exit(void); +int jffs2_dynrubin_init(void); +void jffs2_dynrubin_exit(void); +#else +static inline int jffs2_rubinmips_init(void) { return 0; } +static inline void jffs2_rubinmips_exit(void) {} +static inline int jffs2_dynrubin_init(void) { return 0; } +static inline void jffs2_dynrubin_exit(void) {} +#endif +#ifdef CONFIG_JFFS2_RTIME +extern int jffs2_rtime_init(void); +extern void jffs2_rtime_exit(void); +#else +static inline int jffs2_rtime_init(void) { return 0; } +static inline void jffs2_rtime_exit(void) {} +#endif +#ifdef CONFIG_JFFS2_ZLIB +extern int jffs2_zlib_init(void); +extern void jffs2_zlib_exit(void); +#else +static inline int jffs2_zlib_init(void) { return 0; } +static inline void jffs2_zlib_exit(void) {} +#endif +#ifdef CONFIG_JFFS2_LZO +extern int jffs2_lzo_init(void); +extern void jffs2_lzo_exit(void); +#else +static inline int jffs2_lzo_init(void) { return 0; } +static inline void jffs2_lzo_exit(void) {} +#endif + +#endif /* __JFFS2_COMPR_H__ */ diff --git a/fs/jffs2/compr_lzo.c b/fs/jffs2/compr_lzo.c new file mode 100644 index 000000000..c553bd650 --- /dev/null +++ b/fs/jffs2/compr_lzo.c @@ -0,0 +1,110 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2007 Nokia Corporation. All rights reserved. + * Copyright © 2004-2010 David Woodhouse + * + * Created by Richard Purdie + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#include +#include +#include +#include +#include +#include "compr.h" + +static void *lzo_mem; +static void *lzo_compress_buf; +static DEFINE_MUTEX(deflate_mutex); /* for lzo_mem and lzo_compress_buf */ + +static void free_workspace(void) +{ + vfree(lzo_mem); + vfree(lzo_compress_buf); +} + +static int __init alloc_workspace(void) +{ + lzo_mem = vmalloc(LZO1X_MEM_COMPRESS); + lzo_compress_buf = vmalloc(lzo1x_worst_compress(PAGE_SIZE)); + + if (!lzo_mem || !lzo_compress_buf) { + free_workspace(); + return -ENOMEM; + } + + return 0; +} + +static int jffs2_lzo_compress(unsigned char *data_in, unsigned char *cpage_out, + uint32_t *sourcelen, uint32_t *dstlen) +{ + size_t compress_size; + int ret; + + mutex_lock(&deflate_mutex); + ret = lzo1x_1_compress(data_in, *sourcelen, lzo_compress_buf, &compress_size, lzo_mem); + if (ret != LZO_E_OK) + goto fail; + + if (compress_size > *dstlen) + goto fail; + + memcpy(cpage_out, lzo_compress_buf, compress_size); + mutex_unlock(&deflate_mutex); + + *dstlen = compress_size; + return 0; + + fail: + mutex_unlock(&deflate_mutex); + return -1; +} + +static int jffs2_lzo_decompress(unsigned char *data_in, unsigned char *cpage_out, + uint32_t srclen, uint32_t destlen) +{ + size_t dl = destlen; + int ret; + + ret = lzo1x_decompress_safe(data_in, srclen, cpage_out, &dl); + + if (ret != LZO_E_OK || dl != destlen) + return -1; + + return 0; +} + +static struct jffs2_compressor jffs2_lzo_comp = { + .priority = JFFS2_LZO_PRIORITY, + .name = "lzo", + .compr = JFFS2_COMPR_LZO, + .compress = &jffs2_lzo_compress, + .decompress = &jffs2_lzo_decompress, + .disabled = 0, +}; + +int __init jffs2_lzo_init(void) +{ + int ret; + + ret = alloc_workspace(); + if (ret < 0) + return ret; + + ret = jffs2_register_compressor(&jffs2_lzo_comp); + if (ret) + free_workspace(); + + return ret; +} + +void jffs2_lzo_exit(void) +{ + jffs2_unregister_compressor(&jffs2_lzo_comp); + free_workspace(); +} diff --git a/fs/jffs2/compr_rtime.c b/fs/jffs2/compr_rtime.c new file mode 100644 index 000000000..3bd9d2f3b --- /dev/null +++ b/fs/jffs2/compr_rtime.c @@ -0,0 +1,136 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * Copyright © 2004-2010 David Woodhouse + * + * Created by Arjan van de Ven + * + * For licensing information, see the file 'LICENCE' in this directory. + * + * + * + * Very simple lz77-ish encoder. + * + * Theory of operation: Both encoder and decoder have a list of "last + * occurrences" for every possible source-value; after sending the + * first source-byte, the second byte indicated the "run" length of + * matches + * + * The algorithm is intended to only send "whole bytes", no bit-messing. + * + */ + +#include +#include +#include +#include +#include +#include "compr.h" + +/* _compress returns the compressed size, -1 if bigger */ +static int jffs2_rtime_compress(unsigned char *data_in, + unsigned char *cpage_out, + uint32_t *sourcelen, uint32_t *dstlen) +{ + unsigned short positions[256]; + int outpos = 0; + int pos=0; + + if (*dstlen <= 3) + return -1; + + memset(positions,0,sizeof(positions)); + + while (pos < (*sourcelen) && outpos <= (*dstlen)-2) { + int backpos, runlen=0; + unsigned char value; + + value = data_in[pos]; + + cpage_out[outpos++] = data_in[pos++]; + + backpos = positions[value]; + positions[value]=pos; + + while ((backpos < pos) && (pos < (*sourcelen)) && + (data_in[pos]==data_in[backpos++]) && (runlen<255)) { + pos++; + runlen++; + } + cpage_out[outpos++] = runlen; + } + + if (outpos >= pos) { + /* We failed */ + return -1; + } + + /* Tell the caller how much we managed to compress, and how much space it took */ + *sourcelen = pos; + *dstlen = outpos; + return 0; +} + + +static int jffs2_rtime_decompress(unsigned char *data_in, + unsigned char *cpage_out, + uint32_t srclen, uint32_t destlen) +{ + unsigned short positions[256]; + int outpos = 0; + int pos=0; + + memset(positions,0,sizeof(positions)); + + while (outpos destlen) { + return 1; + } + if (backoffs + repeat >= outpos) { + while(repeat) { + cpage_out[outpos++] = cpage_out[backoffs++]; + repeat--; + } + } else { + memcpy(&cpage_out[outpos],&cpage_out[backoffs],repeat); + outpos+=repeat; + } + } + } + return 0; +} + +static struct jffs2_compressor jffs2_rtime_comp = { + .priority = JFFS2_RTIME_PRIORITY, + .name = "rtime", + .compr = JFFS2_COMPR_RTIME, + .compress = &jffs2_rtime_compress, + .decompress = &jffs2_rtime_decompress, +#ifdef JFFS2_RTIME_DISABLED + .disabled = 1, +#else + .disabled = 0, +#endif +}; + +int jffs2_rtime_init(void) +{ + return jffs2_register_compressor(&jffs2_rtime_comp); +} + +void jffs2_rtime_exit(void) +{ + jffs2_unregister_compressor(&jffs2_rtime_comp); +} diff --git a/fs/jffs2/compr_rubin.c b/fs/jffs2/compr_rubin.c new file mode 100644 index 000000000..9854253d0 --- /dev/null +++ b/fs/jffs2/compr_rubin.c @@ -0,0 +1,447 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * Copyright © 2004-2010 David Woodhouse + * + * Created by Arjan van de Ven + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include "compr.h" + + +#define RUBIN_REG_SIZE 16 +#define UPPER_BIT_RUBIN (((long) 1)<<(RUBIN_REG_SIZE-1)) +#define LOWER_BITS_RUBIN ((((long) 1)<<(RUBIN_REG_SIZE-1))-1) + + +#define BIT_DIVIDER_MIPS 1043 +static int bits_mips[8] = { 277, 249, 290, 267, 229, 341, 212, 241}; + +struct pushpull { + unsigned char *buf; + unsigned int buflen; + unsigned int ofs; + unsigned int reserve; +}; + +struct rubin_state { + unsigned long p; + unsigned long q; + unsigned long rec_q; + long bit_number; + struct pushpull pp; + int bit_divider; + int bits[8]; +}; + +static inline void init_pushpull(struct pushpull *pp, char *buf, + unsigned buflen, unsigned ofs, + unsigned reserve) +{ + pp->buf = buf; + pp->buflen = buflen; + pp->ofs = ofs; + pp->reserve = reserve; +} + +static inline int pushbit(struct pushpull *pp, int bit, int use_reserved) +{ + if (pp->ofs >= pp->buflen - (use_reserved?0:pp->reserve)) + return -ENOSPC; + + if (bit) + pp->buf[pp->ofs >> 3] |= (1<<(7-(pp->ofs & 7))); + else + pp->buf[pp->ofs >> 3] &= ~(1<<(7-(pp->ofs & 7))); + + pp->ofs++; + + return 0; +} + +static inline int pushedbits(struct pushpull *pp) +{ + return pp->ofs; +} + +static inline int pullbit(struct pushpull *pp) +{ + int bit; + + bit = (pp->buf[pp->ofs >> 3] >> (7-(pp->ofs & 7))) & 1; + + pp->ofs++; + return bit; +} + + +static void init_rubin(struct rubin_state *rs, int div, int *bits) +{ + int c; + + rs->q = 0; + rs->p = (long) (2 * UPPER_BIT_RUBIN); + rs->bit_number = (long) 0; + rs->bit_divider = div; + + for (c=0; c<8; c++) + rs->bits[c] = bits[c]; +} + + +static int encode(struct rubin_state *rs, long A, long B, int symbol) +{ + + long i0, i1; + int ret; + + while ((rs->q >= UPPER_BIT_RUBIN) || + ((rs->p + rs->q) <= UPPER_BIT_RUBIN)) { + rs->bit_number++; + + ret = pushbit(&rs->pp, (rs->q & UPPER_BIT_RUBIN) ? 1 : 0, 0); + if (ret) + return ret; + rs->q &= LOWER_BITS_RUBIN; + rs->q <<= 1; + rs->p <<= 1; + } + i0 = A * rs->p / (A + B); + if (i0 <= 0) + i0 = 1; + + if (i0 >= rs->p) + i0 = rs->p - 1; + + i1 = rs->p - i0; + + if (symbol == 0) + rs->p = i0; + else { + rs->p = i1; + rs->q += i0; + } + return 0; +} + + +static void end_rubin(struct rubin_state *rs) +{ + + int i; + + for (i = 0; i < RUBIN_REG_SIZE; i++) { + pushbit(&rs->pp, (UPPER_BIT_RUBIN & rs->q) ? 1 : 0, 1); + rs->q &= LOWER_BITS_RUBIN; + rs->q <<= 1; + } +} + + +static void init_decode(struct rubin_state *rs, int div, int *bits) +{ + init_rubin(rs, div, bits); + + /* behalve lower */ + rs->rec_q = 0; + + for (rs->bit_number = 0; rs->bit_number++ < RUBIN_REG_SIZE; + rs->rec_q = rs->rec_q * 2 + (long) (pullbit(&rs->pp))) + ; +} + +static void __do_decode(struct rubin_state *rs, unsigned long p, + unsigned long q) +{ + register unsigned long lower_bits_rubin = LOWER_BITS_RUBIN; + unsigned long rec_q; + int c, bits = 0; + + /* + * First, work out how many bits we need from the input stream. + * Note that we have already done the initial check on this + * loop prior to calling this function. + */ + do { + bits++; + q &= lower_bits_rubin; + q <<= 1; + p <<= 1; + } while ((q >= UPPER_BIT_RUBIN) || ((p + q) <= UPPER_BIT_RUBIN)); + + rs->p = p; + rs->q = q; + + rs->bit_number += bits; + + /* + * Now get the bits. We really want this to be "get n bits". + */ + rec_q = rs->rec_q; + do { + c = pullbit(&rs->pp); + rec_q &= lower_bits_rubin; + rec_q <<= 1; + rec_q += c; + } while (--bits); + rs->rec_q = rec_q; +} + +static int decode(struct rubin_state *rs, long A, long B) +{ + unsigned long p = rs->p, q = rs->q; + long i0, threshold; + int symbol; + + if (q >= UPPER_BIT_RUBIN || ((p + q) <= UPPER_BIT_RUBIN)) + __do_decode(rs, p, q); + + i0 = A * rs->p / (A + B); + if (i0 <= 0) + i0 = 1; + + if (i0 >= rs->p) + i0 = rs->p - 1; + + threshold = rs->q + i0; + symbol = rs->rec_q >= threshold; + if (rs->rec_q >= threshold) { + rs->q += i0; + i0 = rs->p - i0; + } + + rs->p = i0; + + return symbol; +} + + + +static int out_byte(struct rubin_state *rs, unsigned char byte) +{ + int i, ret; + struct rubin_state rs_copy; + rs_copy = *rs; + + for (i=0; i<8; i++) { + ret = encode(rs, rs->bit_divider-rs->bits[i], + rs->bits[i], byte & 1); + if (ret) { + /* Failed. Restore old state */ + *rs = rs_copy; + return ret; + } + byte >>= 1 ; + } + return 0; +} + +static int in_byte(struct rubin_state *rs) +{ + int i, result = 0, bit_divider = rs->bit_divider; + + for (i = 0; i < 8; i++) + result |= decode(rs, bit_divider - rs->bits[i], + rs->bits[i]) << i; + + return result; +} + + + +static int rubin_do_compress(int bit_divider, int *bits, unsigned char *data_in, + unsigned char *cpage_out, uint32_t *sourcelen, + uint32_t *dstlen) + { + int outpos = 0; + int pos=0; + struct rubin_state rs; + + init_pushpull(&rs.pp, cpage_out, *dstlen * 8, 0, 32); + + init_rubin(&rs, bit_divider, bits); + + while (pos < (*sourcelen) && !out_byte(&rs, data_in[pos])) + pos++; + + end_rubin(&rs); + + /* Tell the caller how much we managed to compress, + * and how much space it took */ + + outpos = (pushedbits(&rs.pp)+7)/8; + + if (outpos >= pos) + return -1; /* We didn't actually compress */ + *sourcelen = pos; + *dstlen = outpos; + return 0; +} +#if 0 +/* _compress returns the compressed size, -1 if bigger */ +int jffs2_rubinmips_compress(unsigned char *data_in, unsigned char *cpage_out, + uint32_t *sourcelen, uint32_t *dstlen) +{ + return rubin_do_compress(BIT_DIVIDER_MIPS, bits_mips, data_in, + cpage_out, sourcelen, dstlen); +} +#endif +static int jffs2_dynrubin_compress(unsigned char *data_in, + unsigned char *cpage_out, + uint32_t *sourcelen, uint32_t *dstlen) +{ + int bits[8]; + unsigned char histo[256]; + int i; + int ret; + uint32_t mysrclen, mydstlen; + + mysrclen = *sourcelen; + mydstlen = *dstlen - 8; + + if (*dstlen <= 12) + return -1; + + memset(histo, 0, 256); + for (i=0; i 255) bits[i] = 255; + cpage_out[i] = bits[i]; + } + + ret = rubin_do_compress(256, bits, data_in, cpage_out+8, &mysrclen, + &mydstlen); + if (ret) + return ret; + + /* Add back the 8 bytes we took for the probabilities */ + mydstlen += 8; + + if (mysrclen <= mydstlen) { + /* We compressed */ + return -1; + } + + *sourcelen = mysrclen; + *dstlen = mydstlen; + return 0; +} + +static void rubin_do_decompress(int bit_divider, int *bits, + unsigned char *cdata_in, + unsigned char *page_out, uint32_t srclen, + uint32_t destlen) +{ + int outpos = 0; + struct rubin_state rs; + + init_pushpull(&rs.pp, cdata_in, srclen, 0, 0); + init_decode(&rs, bit_divider, bits); + + while (outpos < destlen) + page_out[outpos++] = in_byte(&rs); +} + + +static int jffs2_rubinmips_decompress(unsigned char *data_in, + unsigned char *cpage_out, + uint32_t sourcelen, uint32_t dstlen) +{ + rubin_do_decompress(BIT_DIVIDER_MIPS, bits_mips, data_in, + cpage_out, sourcelen, dstlen); + return 0; +} + +static int jffs2_dynrubin_decompress(unsigned char *data_in, + unsigned char *cpage_out, + uint32_t sourcelen, uint32_t dstlen) +{ + int bits[8]; + int c; + + for (c=0; c<8; c++) + bits[c] = data_in[c]; + + rubin_do_decompress(256, bits, data_in+8, cpage_out, sourcelen-8, + dstlen); + return 0; +} + +static struct jffs2_compressor jffs2_rubinmips_comp = { + .priority = JFFS2_RUBINMIPS_PRIORITY, + .name = "rubinmips", + .compr = JFFS2_COMPR_DYNRUBIN, + .compress = NULL, /*&jffs2_rubinmips_compress,*/ + .decompress = &jffs2_rubinmips_decompress, +#ifdef JFFS2_RUBINMIPS_DISABLED + .disabled = 1, +#else + .disabled = 0, +#endif +}; + +int jffs2_rubinmips_init(void) +{ + return jffs2_register_compressor(&jffs2_rubinmips_comp); +} + +void jffs2_rubinmips_exit(void) +{ + jffs2_unregister_compressor(&jffs2_rubinmips_comp); +} + +static struct jffs2_compressor jffs2_dynrubin_comp = { + .priority = JFFS2_DYNRUBIN_PRIORITY, + .name = "dynrubin", + .compr = JFFS2_COMPR_RUBINMIPS, + .compress = jffs2_dynrubin_compress, + .decompress = &jffs2_dynrubin_decompress, +#ifdef JFFS2_DYNRUBIN_DISABLED + .disabled = 1, +#else + .disabled = 0, +#endif +}; + +int jffs2_dynrubin_init(void) +{ + return jffs2_register_compressor(&jffs2_dynrubin_comp); +} + +void jffs2_dynrubin_exit(void) +{ + jffs2_unregister_compressor(&jffs2_dynrubin_comp); +} diff --git a/fs/jffs2/compr_zlib.c b/fs/jffs2/compr_zlib.c new file mode 100644 index 000000000..5698dae5d --- /dev/null +++ b/fs/jffs2/compr_zlib.c @@ -0,0 +1,222 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * Copyright © 2004-2010 David Woodhouse + * + * Created by David Woodhouse + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#if !defined(__KERNEL__) && !defined(__ECOS) +#error "The userspace support got too messy and was removed. Update your mkfs.jffs2" +#endif + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include "nodelist.h" +#include "compr.h" + + /* Plan: call deflate() with avail_in == *sourcelen, + avail_out = *dstlen - 12 and flush == Z_FINISH. + If it doesn't manage to finish, call it again with + avail_in == 0 and avail_out set to the remaining 12 + bytes for it to clean up. + Q: Is 12 bytes sufficient? + */ +#define STREAM_END_SPACE 12 + +static DEFINE_MUTEX(deflate_mutex); +static DEFINE_MUTEX(inflate_mutex); +static z_stream inf_strm, def_strm; + +#ifdef __KERNEL__ /* Linux-only */ +#include +#include +#include + +static int __init alloc_workspaces(void) +{ + def_strm.workspace = vmalloc(zlib_deflate_workspacesize(MAX_WBITS, + MAX_MEM_LEVEL)); + if (!def_strm.workspace) + return -ENOMEM; + + jffs2_dbg(1, "Allocated %d bytes for deflate workspace\n", + zlib_deflate_workspacesize(MAX_WBITS, MAX_MEM_LEVEL)); + inf_strm.workspace = vmalloc(zlib_inflate_workspacesize()); + if (!inf_strm.workspace) { + vfree(def_strm.workspace); + return -ENOMEM; + } + jffs2_dbg(1, "Allocated %d bytes for inflate workspace\n", + zlib_inflate_workspacesize()); + return 0; +} + +static void free_workspaces(void) +{ + vfree(def_strm.workspace); + vfree(inf_strm.workspace); +} +#else +#define alloc_workspaces() (0) +#define free_workspaces() do { } while(0) +#endif /* __KERNEL__ */ + +static int jffs2_zlib_compress(unsigned char *data_in, + unsigned char *cpage_out, + uint32_t *sourcelen, uint32_t *dstlen) +{ + int ret; + + if (*dstlen <= STREAM_END_SPACE) + return -1; + + mutex_lock(&deflate_mutex); + + if (Z_OK != zlib_deflateInit(&def_strm, 3)) { + pr_warn("deflateInit failed\n"); + mutex_unlock(&deflate_mutex); + return -1; + } + + def_strm.next_in = data_in; + def_strm.total_in = 0; + + def_strm.next_out = cpage_out; + def_strm.total_out = 0; + + while (def_strm.total_out < *dstlen - STREAM_END_SPACE && def_strm.total_in < *sourcelen) { + def_strm.avail_out = *dstlen - (def_strm.total_out + STREAM_END_SPACE); + def_strm.avail_in = min_t(unsigned long, + (*sourcelen-def_strm.total_in), def_strm.avail_out); + jffs2_dbg(1, "calling deflate with avail_in %ld, avail_out %ld\n", + def_strm.avail_in, def_strm.avail_out); + ret = zlib_deflate(&def_strm, Z_PARTIAL_FLUSH); + jffs2_dbg(1, "deflate returned with avail_in %ld, avail_out %ld, total_in %ld, total_out %ld\n", + def_strm.avail_in, def_strm.avail_out, + def_strm.total_in, def_strm.total_out); + if (ret != Z_OK) { + jffs2_dbg(1, "deflate in loop returned %d\n", ret); + zlib_deflateEnd(&def_strm); + mutex_unlock(&deflate_mutex); + return -1; + } + } + def_strm.avail_out += STREAM_END_SPACE; + def_strm.avail_in = 0; + ret = zlib_deflate(&def_strm, Z_FINISH); + zlib_deflateEnd(&def_strm); + + if (ret != Z_STREAM_END) { + jffs2_dbg(1, "final deflate returned %d\n", ret); + ret = -1; + goto out; + } + + if (def_strm.total_out >= def_strm.total_in) { + jffs2_dbg(1, "zlib compressed %ld bytes into %ld; failing\n", + def_strm.total_in, def_strm.total_out); + ret = -1; + goto out; + } + + jffs2_dbg(1, "zlib compressed %ld bytes into %ld\n", + def_strm.total_in, def_strm.total_out); + + *dstlen = def_strm.total_out; + *sourcelen = def_strm.total_in; + ret = 0; + out: + mutex_unlock(&deflate_mutex); + return ret; +} + +static int jffs2_zlib_decompress(unsigned char *data_in, + unsigned char *cpage_out, + uint32_t srclen, uint32_t destlen) +{ + int ret; + int wbits = MAX_WBITS; + + mutex_lock(&inflate_mutex); + + inf_strm.next_in = data_in; + inf_strm.avail_in = srclen; + inf_strm.total_in = 0; + + inf_strm.next_out = cpage_out; + inf_strm.avail_out = destlen; + inf_strm.total_out = 0; + + /* If it's deflate, and it's got no preset dictionary, then + we can tell zlib to skip the adler32 check. */ + if (srclen > 2 && !(data_in[1] & PRESET_DICT) && + ((data_in[0] & 0x0f) == Z_DEFLATED) && + !(((data_in[0]<<8) + data_in[1]) % 31)) { + + jffs2_dbg(2, "inflate skipping adler32\n"); + wbits = -((data_in[0] >> 4) + 8); + inf_strm.next_in += 2; + inf_strm.avail_in -= 2; + } else { + /* Let this remain D1 for now -- it should never happen */ + jffs2_dbg(1, "inflate not skipping adler32\n"); + } + + + if (Z_OK != zlib_inflateInit2(&inf_strm, wbits)) { + pr_warn("inflateInit failed\n"); + mutex_unlock(&inflate_mutex); + return 1; + } + + while((ret = zlib_inflate(&inf_strm, Z_FINISH)) == Z_OK) + ; + if (ret != Z_STREAM_END) { + pr_notice("inflate returned %d\n", ret); + } + zlib_inflateEnd(&inf_strm); + mutex_unlock(&inflate_mutex); + return 0; +} + +static struct jffs2_compressor jffs2_zlib_comp = { + .priority = JFFS2_ZLIB_PRIORITY, + .name = "zlib", + .compr = JFFS2_COMPR_ZLIB, + .compress = &jffs2_zlib_compress, + .decompress = &jffs2_zlib_decompress, +#ifdef JFFS2_ZLIB_DISABLED + .disabled = 1, +#else + .disabled = 0, +#endif +}; + +int __init jffs2_zlib_init(void) +{ + int ret; + + ret = alloc_workspaces(); + if (ret) + return ret; + + ret = jffs2_register_compressor(&jffs2_zlib_comp); + if (ret) + free_workspaces(); + + return ret; +} + +void jffs2_zlib_exit(void) +{ + jffs2_unregister_compressor(&jffs2_zlib_comp); + free_workspaces(); +} diff --git a/fs/jffs2/debug.c b/fs/jffs2/debug.c new file mode 100644 index 000000000..0925caab2 --- /dev/null +++ b/fs/jffs2/debug.c @@ -0,0 +1,866 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * Copyright © 2004-2010 David Woodhouse + * + * Created by David Woodhouse + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include "nodelist.h" +#include "debug.h" + +#ifdef JFFS2_DBG_SANITY_CHECKS + +void +__jffs2_dbg_acct_sanity_check_nolock(struct jffs2_sb_info *c, + struct jffs2_eraseblock *jeb) +{ + if (unlikely(jeb && jeb->used_size + jeb->dirty_size + + jeb->free_size + jeb->wasted_size + + jeb->unchecked_size != c->sector_size)) { + JFFS2_ERROR("eeep, space accounting for block at 0x%08x is screwed.\n", jeb->offset); + JFFS2_ERROR("free %#08x + dirty %#08x + used %#08x + wasted %#08x + unchecked %#08x != total %#08x.\n", + jeb->free_size, jeb->dirty_size, jeb->used_size, + jeb->wasted_size, jeb->unchecked_size, c->sector_size); + BUG(); + } + + if (unlikely(c->used_size + c->dirty_size + c->free_size + c->erasing_size + c->bad_size + + c->wasted_size + c->unchecked_size != c->flash_size)) { + JFFS2_ERROR("eeep, space accounting superblock info is screwed.\n"); + JFFS2_ERROR("free %#08x + dirty %#08x + used %#08x + erasing %#08x + bad %#08x + wasted %#08x + unchecked %#08x != total %#08x.\n", + c->free_size, c->dirty_size, c->used_size, c->erasing_size, c->bad_size, + c->wasted_size, c->unchecked_size, c->flash_size); + BUG(); + } +} + +void +__jffs2_dbg_acct_sanity_check(struct jffs2_sb_info *c, + struct jffs2_eraseblock *jeb) +{ + spin_lock(&c->erase_completion_lock); + jffs2_dbg_acct_sanity_check_nolock(c, jeb); + spin_unlock(&c->erase_completion_lock); +} + +#endif /* JFFS2_DBG_SANITY_CHECKS */ + +#ifdef JFFS2_DBG_PARANOIA_CHECKS +/* + * Check the fragtree. + */ +void +__jffs2_dbg_fragtree_paranoia_check(struct jffs2_inode_info *f) +{ + mutex_lock(&f->sem); + __jffs2_dbg_fragtree_paranoia_check_nolock(f); + mutex_unlock(&f->sem); +} + +void +__jffs2_dbg_fragtree_paranoia_check_nolock(struct jffs2_inode_info *f) +{ + struct jffs2_node_frag *frag; + int bitched = 0; + + for (frag = frag_first(&f->fragtree); frag; frag = frag_next(frag)) { + struct jffs2_full_dnode *fn = frag->node; + + if (!fn || !fn->raw) + continue; + + if (ref_flags(fn->raw) == REF_PRISTINE) { + if (fn->frags > 1) { + JFFS2_ERROR("REF_PRISTINE node at 0x%08x had %d frags. Tell dwmw2.\n", + ref_offset(fn->raw), fn->frags); + bitched = 1; + } + + /* A hole node which isn't multi-page should be garbage-collected + and merged anyway, so we just check for the frag size here, + rather than mucking around with actually reading the node + and checking the compression type, which is the real way + to tell a hole node. */ + if (frag->ofs & (PAGE_SIZE-1) && frag_prev(frag) + && frag_prev(frag)->size < PAGE_SIZE && frag_prev(frag)->node) { + JFFS2_ERROR("REF_PRISTINE node at 0x%08x had a previous non-hole frag in the same page. Tell dwmw2.\n", + ref_offset(fn->raw)); + bitched = 1; + } + + if ((frag->ofs+frag->size) & (PAGE_SIZE-1) && frag_next(frag) + && frag_next(frag)->size < PAGE_SIZE && frag_next(frag)->node) { + JFFS2_ERROR("REF_PRISTINE node at 0x%08x (%08x-%08x) had a following non-hole frag in the same page. Tell dwmw2.\n", + ref_offset(fn->raw), frag->ofs, frag->ofs+frag->size); + bitched = 1; + } + } + } + + if (bitched) { + JFFS2_ERROR("fragtree is corrupted.\n"); + __jffs2_dbg_dump_fragtree_nolock(f); + BUG(); + } +} + +/* + * Check if the flash contains all 0xFF before we start writing. + */ +void +__jffs2_dbg_prewrite_paranoia_check(struct jffs2_sb_info *c, + uint32_t ofs, int len) +{ + size_t retlen; + int ret, i; + unsigned char *buf; + + buf = kmalloc(len, GFP_KERNEL); + if (!buf) + return; + + ret = jffs2_flash_read(c, ofs, len, &retlen, buf); + if (ret || (retlen != len)) { + JFFS2_WARNING("read %d bytes failed or short. ret %d, retlen %zd.\n", + len, ret, retlen); + kfree(buf); + return; + } + + ret = 0; + for (i = 0; i < len; i++) + if (buf[i] != 0xff) + ret = 1; + + if (ret) { + JFFS2_ERROR("argh, about to write node to %#08x on flash, but there are data already there. The first corrupted byte is at %#08x offset.\n", + ofs, ofs + i); + __jffs2_dbg_dump_buffer(buf, len, ofs); + kfree(buf); + BUG(); + } + + kfree(buf); +} + +static void __jffs2_dbg_superblock_counts(struct jffs2_sb_info *c) +{ + struct jffs2_eraseblock *jeb; + uint32_t free = 0, dirty = 0, used = 0, wasted = 0, + erasing = 0, bad = 0, unchecked = 0; + int nr_counted = 0; + int dump = 0; + + if (c->gcblock) { + nr_counted++; + free += c->gcblock->free_size; + dirty += c->gcblock->dirty_size; + used += c->gcblock->used_size; + wasted += c->gcblock->wasted_size; + unchecked += c->gcblock->unchecked_size; + } + if (c->nextblock) { + nr_counted++; + free += c->nextblock->free_size; + dirty += c->nextblock->dirty_size; + used += c->nextblock->used_size; + wasted += c->nextblock->wasted_size; + unchecked += c->nextblock->unchecked_size; + } + list_for_each_entry(jeb, &c->clean_list, list) { + nr_counted++; + free += jeb->free_size; + dirty += jeb->dirty_size; + used += jeb->used_size; + wasted += jeb->wasted_size; + unchecked += jeb->unchecked_size; + } + list_for_each_entry(jeb, &c->very_dirty_list, list) { + nr_counted++; + free += jeb->free_size; + dirty += jeb->dirty_size; + used += jeb->used_size; + wasted += jeb->wasted_size; + unchecked += jeb->unchecked_size; + } + list_for_each_entry(jeb, &c->dirty_list, list) { + nr_counted++; + free += jeb->free_size; + dirty += jeb->dirty_size; + used += jeb->used_size; + wasted += jeb->wasted_size; + unchecked += jeb->unchecked_size; + } + list_for_each_entry(jeb, &c->erasable_list, list) { + nr_counted++; + free += jeb->free_size; + dirty += jeb->dirty_size; + used += jeb->used_size; + wasted += jeb->wasted_size; + unchecked += jeb->unchecked_size; + } + list_for_each_entry(jeb, &c->erasable_pending_wbuf_list, list) { + nr_counted++; + free += jeb->free_size; + dirty += jeb->dirty_size; + used += jeb->used_size; + wasted += jeb->wasted_size; + unchecked += jeb->unchecked_size; + } + list_for_each_entry(jeb, &c->erase_pending_list, list) { + nr_counted++; + free += jeb->free_size; + dirty += jeb->dirty_size; + used += jeb->used_size; + wasted += jeb->wasted_size; + unchecked += jeb->unchecked_size; + } + list_for_each_entry(jeb, &c->free_list, list) { + nr_counted++; + free += jeb->free_size; + dirty += jeb->dirty_size; + used += jeb->used_size; + wasted += jeb->wasted_size; + unchecked += jeb->unchecked_size; + } + list_for_each_entry(jeb, &c->bad_used_list, list) { + nr_counted++; + free += jeb->free_size; + dirty += jeb->dirty_size; + used += jeb->used_size; + wasted += jeb->wasted_size; + unchecked += jeb->unchecked_size; + } + + list_for_each_entry(jeb, &c->erasing_list, list) { + nr_counted++; + erasing += c->sector_size; + } + list_for_each_entry(jeb, &c->erase_checking_list, list) { + nr_counted++; + erasing += c->sector_size; + } + list_for_each_entry(jeb, &c->erase_complete_list, list) { + nr_counted++; + erasing += c->sector_size; + } + list_for_each_entry(jeb, &c->bad_list, list) { + nr_counted++; + bad += c->sector_size; + } + +#define check(sz) \ +do { \ + if (sz != c->sz##_size) { \ + pr_warn("%s_size mismatch counted 0x%x, c->%s_size 0x%x\n", \ + #sz, sz, #sz, c->sz##_size); \ + dump = 1; \ + } \ +} while (0) + + check(free); + check(dirty); + check(used); + check(wasted); + check(unchecked); + check(bad); + check(erasing); + +#undef check + + if (nr_counted != c->nr_blocks) { + pr_warn("%s counted only 0x%x blocks of 0x%x. Where are the others?\n", + __func__, nr_counted, c->nr_blocks); + dump = 1; + } + + if (dump) { + __jffs2_dbg_dump_block_lists_nolock(c); + BUG(); + } +} + +/* + * Check the space accounting and node_ref list correctness for the JFFS2 erasable block 'jeb'. + */ +void +__jffs2_dbg_acct_paranoia_check(struct jffs2_sb_info *c, + struct jffs2_eraseblock *jeb) +{ + spin_lock(&c->erase_completion_lock); + __jffs2_dbg_acct_paranoia_check_nolock(c, jeb); + spin_unlock(&c->erase_completion_lock); +} + +void +__jffs2_dbg_acct_paranoia_check_nolock(struct jffs2_sb_info *c, + struct jffs2_eraseblock *jeb) +{ + uint32_t my_used_size = 0; + uint32_t my_unchecked_size = 0; + uint32_t my_dirty_size = 0; + struct jffs2_raw_node_ref *ref2 = jeb->first_node; + + while (ref2) { + uint32_t totlen = ref_totlen(c, jeb, ref2); + + if (ref_offset(ref2) < jeb->offset || + ref_offset(ref2) > jeb->offset + c->sector_size) { + JFFS2_ERROR("node_ref %#08x shouldn't be in block at %#08x.\n", + ref_offset(ref2), jeb->offset); + goto error; + + } + if (ref_flags(ref2) == REF_UNCHECKED) + my_unchecked_size += totlen; + else if (!ref_obsolete(ref2)) + my_used_size += totlen; + else + my_dirty_size += totlen; + + if ((!ref_next(ref2)) != (ref2 == jeb->last_node)) { + JFFS2_ERROR("node_ref for node at %#08x (mem %p) has next at %#08x (mem %p), last_node is at %#08x (mem %p).\n", + ref_offset(ref2), ref2, ref_offset(ref_next(ref2)), ref_next(ref2), + ref_offset(jeb->last_node), jeb->last_node); + goto error; + } + ref2 = ref_next(ref2); + } + + if (my_used_size != jeb->used_size) { + JFFS2_ERROR("Calculated used size %#08x != stored used size %#08x.\n", + my_used_size, jeb->used_size); + goto error; + } + + if (my_unchecked_size != jeb->unchecked_size) { + JFFS2_ERROR("Calculated unchecked size %#08x != stored unchecked size %#08x.\n", + my_unchecked_size, jeb->unchecked_size); + goto error; + } + +#if 0 + /* This should work when we implement ref->__totlen elemination */ + if (my_dirty_size != jeb->dirty_size + jeb->wasted_size) { + JFFS2_ERROR("Calculated dirty+wasted size %#08x != stored dirty + wasted size %#08x\n", + my_dirty_size, jeb->dirty_size + jeb->wasted_size); + goto error; + } + + if (jeb->free_size == 0 + && my_used_size + my_unchecked_size + my_dirty_size != c->sector_size) { + JFFS2_ERROR("The sum of all nodes in block (%#x) != size of block (%#x)\n", + my_used_size + my_unchecked_size + my_dirty_size, + c->sector_size); + goto error; + } +#endif + + if (!(c->flags & (JFFS2_SB_FLAG_BUILDING|JFFS2_SB_FLAG_SCANNING))) + __jffs2_dbg_superblock_counts(c); + + return; + +error: + __jffs2_dbg_dump_node_refs_nolock(c, jeb); + __jffs2_dbg_dump_jeb_nolock(jeb); + __jffs2_dbg_dump_block_lists_nolock(c); + BUG(); + +} +#endif /* JFFS2_DBG_PARANOIA_CHECKS */ + +#if defined(JFFS2_DBG_DUMPS) || defined(JFFS2_DBG_PARANOIA_CHECKS) +/* + * Dump the node_refs of the 'jeb' JFFS2 eraseblock. + */ +void +__jffs2_dbg_dump_node_refs(struct jffs2_sb_info *c, + struct jffs2_eraseblock *jeb) +{ + spin_lock(&c->erase_completion_lock); + __jffs2_dbg_dump_node_refs_nolock(c, jeb); + spin_unlock(&c->erase_completion_lock); +} + +void +__jffs2_dbg_dump_node_refs_nolock(struct jffs2_sb_info *c, + struct jffs2_eraseblock *jeb) +{ + struct jffs2_raw_node_ref *ref; + int i = 0; + + printk(JFFS2_DBG_MSG_PREFIX " Dump node_refs of the eraseblock %#08x\n", jeb->offset); + if (!jeb->first_node) { + printk(JFFS2_DBG_MSG_PREFIX " no nodes in the eraseblock %#08x\n", jeb->offset); + return; + } + + printk(JFFS2_DBG); + for (ref = jeb->first_node; ; ref = ref_next(ref)) { + printk("%#08x", ref_offset(ref)); +#ifdef TEST_TOTLEN + printk("(%x)", ref->__totlen); +#endif + if (ref_next(ref)) + printk("->"); + else + break; + if (++i == 4) { + i = 0; + printk("\n" JFFS2_DBG); + } + } + printk("\n"); +} + +/* + * Dump an eraseblock's space accounting. + */ +void +__jffs2_dbg_dump_jeb(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb) +{ + spin_lock(&c->erase_completion_lock); + __jffs2_dbg_dump_jeb_nolock(jeb); + spin_unlock(&c->erase_completion_lock); +} + +void +__jffs2_dbg_dump_jeb_nolock(struct jffs2_eraseblock *jeb) +{ + if (!jeb) + return; + + printk(JFFS2_DBG_MSG_PREFIX " dump space accounting for the eraseblock at %#08x:\n", + jeb->offset); + + printk(JFFS2_DBG "used_size: %#08x\n", jeb->used_size); + printk(JFFS2_DBG "dirty_size: %#08x\n", jeb->dirty_size); + printk(JFFS2_DBG "wasted_size: %#08x\n", jeb->wasted_size); + printk(JFFS2_DBG "unchecked_size: %#08x\n", jeb->unchecked_size); + printk(JFFS2_DBG "free_size: %#08x\n", jeb->free_size); +} + +void +__jffs2_dbg_dump_block_lists(struct jffs2_sb_info *c) +{ + spin_lock(&c->erase_completion_lock); + __jffs2_dbg_dump_block_lists_nolock(c); + spin_unlock(&c->erase_completion_lock); +} + +void +__jffs2_dbg_dump_block_lists_nolock(struct jffs2_sb_info *c) +{ + printk(JFFS2_DBG_MSG_PREFIX " dump JFFS2 blocks lists:\n"); + + printk(JFFS2_DBG "flash_size: %#08x\n", c->flash_size); + printk(JFFS2_DBG "used_size: %#08x\n", c->used_size); + printk(JFFS2_DBG "dirty_size: %#08x\n", c->dirty_size); + printk(JFFS2_DBG "wasted_size: %#08x\n", c->wasted_size); + printk(JFFS2_DBG "unchecked_size: %#08x\n", c->unchecked_size); + printk(JFFS2_DBG "free_size: %#08x\n", c->free_size); + printk(JFFS2_DBG "erasing_size: %#08x\n", c->erasing_size); + printk(JFFS2_DBG "bad_size: %#08x\n", c->bad_size); + printk(JFFS2_DBG "sector_size: %#08x\n", c->sector_size); + printk(JFFS2_DBG "jffs2_reserved_blocks size: %#08x\n", + c->sector_size * c->resv_blocks_write); + + if (c->nextblock) + printk(JFFS2_DBG "nextblock: %#08x (used %#08x, dirty %#08x, wasted %#08x, unchecked %#08x, free %#08x)\n", + c->nextblock->offset, c->nextblock->used_size, + c->nextblock->dirty_size, c->nextblock->wasted_size, + c->nextblock->unchecked_size, c->nextblock->free_size); + else + printk(JFFS2_DBG "nextblock: NULL\n"); + + if (c->gcblock) + printk(JFFS2_DBG "gcblock: %#08x (used %#08x, dirty %#08x, wasted %#08x, unchecked %#08x, free %#08x)\n", + c->gcblock->offset, c->gcblock->used_size, c->gcblock->dirty_size, + c->gcblock->wasted_size, c->gcblock->unchecked_size, c->gcblock->free_size); + else + printk(JFFS2_DBG "gcblock: NULL\n"); + + if (list_empty(&c->clean_list)) { + printk(JFFS2_DBG "clean_list: empty\n"); + } else { + struct list_head *this; + int numblocks = 0; + uint32_t dirty = 0; + + list_for_each(this, &c->clean_list) { + struct jffs2_eraseblock *jeb = list_entry(this, struct jffs2_eraseblock, list); + numblocks ++; + dirty += jeb->wasted_size; + if (!(jeb->used_size == 0 && jeb->dirty_size == 0 && jeb->wasted_size == 0)) { + printk(JFFS2_DBG "clean_list: %#08x (used %#08x, dirty %#08x, wasted %#08x, unchecked %#08x, free %#08x)\n", + jeb->offset, jeb->used_size, jeb->dirty_size, jeb->wasted_size, + jeb->unchecked_size, jeb->free_size); + } + } + + printk (JFFS2_DBG "Contains %d blocks with total wasted size %u, average wasted size: %u\n", + numblocks, dirty, dirty / numblocks); + } + + if (list_empty(&c->very_dirty_list)) { + printk(JFFS2_DBG "very_dirty_list: empty\n"); + } else { + struct list_head *this; + int numblocks = 0; + uint32_t dirty = 0; + + list_for_each(this, &c->very_dirty_list) { + struct jffs2_eraseblock *jeb = list_entry(this, struct jffs2_eraseblock, list); + + numblocks ++; + dirty += jeb->dirty_size; + if (!(jeb->used_size == 0 && jeb->dirty_size == 0 && jeb->wasted_size == 0)) { + printk(JFFS2_DBG "very_dirty_list: %#08x (used %#08x, dirty %#08x, wasted %#08x, unchecked %#08x, free %#08x)\n", + jeb->offset, jeb->used_size, jeb->dirty_size, jeb->wasted_size, + jeb->unchecked_size, jeb->free_size); + } + } + + printk (JFFS2_DBG "Contains %d blocks with total dirty size %u, average dirty size: %u\n", + numblocks, dirty, dirty / numblocks); + } + + if (list_empty(&c->dirty_list)) { + printk(JFFS2_DBG "dirty_list: empty\n"); + } else { + struct list_head *this; + int numblocks = 0; + uint32_t dirty = 0; + + list_for_each(this, &c->dirty_list) { + struct jffs2_eraseblock *jeb = list_entry(this, struct jffs2_eraseblock, list); + + numblocks ++; + dirty += jeb->dirty_size; + if (!(jeb->used_size == 0 && jeb->dirty_size == 0 && jeb->wasted_size == 0)) { + printk(JFFS2_DBG "dirty_list: %#08x (used %#08x, dirty %#08x, wasted %#08x, unchecked %#08x, free %#08x)\n", + jeb->offset, jeb->used_size, jeb->dirty_size, jeb->wasted_size, + jeb->unchecked_size, jeb->free_size); + } + } + + printk (JFFS2_DBG "contains %d blocks with total dirty size %u, average dirty size: %u\n", + numblocks, dirty, dirty / numblocks); + } + + if (list_empty(&c->erasable_list)) { + printk(JFFS2_DBG "erasable_list: empty\n"); + } else { + struct list_head *this; + + list_for_each(this, &c->erasable_list) { + struct jffs2_eraseblock *jeb = list_entry(this, struct jffs2_eraseblock, list); + + if (!(jeb->used_size == 0 && jeb->dirty_size == 0 && jeb->wasted_size == 0)) { + printk(JFFS2_DBG "erasable_list: %#08x (used %#08x, dirty %#08x, wasted %#08x, unchecked %#08x, free %#08x)\n", + jeb->offset, jeb->used_size, jeb->dirty_size, jeb->wasted_size, + jeb->unchecked_size, jeb->free_size); + } + } + } + + if (list_empty(&c->erasing_list)) { + printk(JFFS2_DBG "erasing_list: empty\n"); + } else { + struct list_head *this; + + list_for_each(this, &c->erasing_list) { + struct jffs2_eraseblock *jeb = list_entry(this, struct jffs2_eraseblock, list); + + if (!(jeb->used_size == 0 && jeb->dirty_size == 0 && jeb->wasted_size == 0)) { + printk(JFFS2_DBG "erasing_list: %#08x (used %#08x, dirty %#08x, wasted %#08x, unchecked %#08x, free %#08x)\n", + jeb->offset, jeb->used_size, jeb->dirty_size, jeb->wasted_size, + jeb->unchecked_size, jeb->free_size); + } + } + } + if (list_empty(&c->erase_checking_list)) { + printk(JFFS2_DBG "erase_checking_list: empty\n"); + } else { + struct list_head *this; + + list_for_each(this, &c->erase_checking_list) { + struct jffs2_eraseblock *jeb = list_entry(this, struct jffs2_eraseblock, list); + + if (!(jeb->used_size == 0 && jeb->dirty_size == 0 && jeb->wasted_size == 0)) { + printk(JFFS2_DBG "erase_checking_list: %#08x (used %#08x, dirty %#08x, wasted %#08x, unchecked %#08x, free %#08x)\n", + jeb->offset, jeb->used_size, jeb->dirty_size, jeb->wasted_size, + jeb->unchecked_size, jeb->free_size); + } + } + } + + if (list_empty(&c->erase_pending_list)) { + printk(JFFS2_DBG "erase_pending_list: empty\n"); + } else { + struct list_head *this; + + list_for_each(this, &c->erase_pending_list) { + struct jffs2_eraseblock *jeb = list_entry(this, struct jffs2_eraseblock, list); + + if (!(jeb->used_size == 0 && jeb->dirty_size == 0 && jeb->wasted_size == 0)) { + printk(JFFS2_DBG "erase_pending_list: %#08x (used %#08x, dirty %#08x, wasted %#08x, unchecked %#08x, free %#08x)\n", + jeb->offset, jeb->used_size, jeb->dirty_size, jeb->wasted_size, + jeb->unchecked_size, jeb->free_size); + } + } + } + + if (list_empty(&c->erasable_pending_wbuf_list)) { + printk(JFFS2_DBG "erasable_pending_wbuf_list: empty\n"); + } else { + struct list_head *this; + + list_for_each(this, &c->erasable_pending_wbuf_list) { + struct jffs2_eraseblock *jeb = list_entry(this, struct jffs2_eraseblock, list); + + if (!(jeb->used_size == 0 && jeb->dirty_size == 0 && jeb->wasted_size == 0)) { + printk(JFFS2_DBG "erasable_pending_wbuf_list: %#08x (used %#08x, dirty %#08x, wasted %#08x, unchecked %#08x, free %#08x)\n", + jeb->offset, jeb->used_size, jeb->dirty_size, jeb->wasted_size, + jeb->unchecked_size, jeb->free_size); + } + } + } + + if (list_empty(&c->free_list)) { + printk(JFFS2_DBG "free_list: empty\n"); + } else { + struct list_head *this; + + list_for_each(this, &c->free_list) { + struct jffs2_eraseblock *jeb = list_entry(this, struct jffs2_eraseblock, list); + + if (!(jeb->used_size == 0 && jeb->dirty_size == 0 && jeb->wasted_size == 0)) { + printk(JFFS2_DBG "free_list: %#08x (used %#08x, dirty %#08x, wasted %#08x, unchecked %#08x, free %#08x)\n", + jeb->offset, jeb->used_size, jeb->dirty_size, jeb->wasted_size, + jeb->unchecked_size, jeb->free_size); + } + } + } + + if (list_empty(&c->bad_list)) { + printk(JFFS2_DBG "bad_list: empty\n"); + } else { + struct list_head *this; + + list_for_each(this, &c->bad_list) { + struct jffs2_eraseblock *jeb = list_entry(this, struct jffs2_eraseblock, list); + + if (!(jeb->used_size == 0 && jeb->dirty_size == 0 && jeb->wasted_size == 0)) { + printk(JFFS2_DBG "bad_list: %#08x (used %#08x, dirty %#08x, wasted %#08x, unchecked %#08x, free %#08x)\n", + jeb->offset, jeb->used_size, jeb->dirty_size, jeb->wasted_size, + jeb->unchecked_size, jeb->free_size); + } + } + } + + if (list_empty(&c->bad_used_list)) { + printk(JFFS2_DBG "bad_used_list: empty\n"); + } else { + struct list_head *this; + + list_for_each(this, &c->bad_used_list) { + struct jffs2_eraseblock *jeb = list_entry(this, struct jffs2_eraseblock, list); + + if (!(jeb->used_size == 0 && jeb->dirty_size == 0 && jeb->wasted_size == 0)) { + printk(JFFS2_DBG "bad_used_list: %#08x (used %#08x, dirty %#08x, wasted %#08x, unchecked %#08x, free %#08x)\n", + jeb->offset, jeb->used_size, jeb->dirty_size, jeb->wasted_size, + jeb->unchecked_size, jeb->free_size); + } + } + } +} + +void +__jffs2_dbg_dump_fragtree(struct jffs2_inode_info *f) +{ + mutex_lock(&f->sem); + jffs2_dbg_dump_fragtree_nolock(f); + mutex_unlock(&f->sem); +} + +void +__jffs2_dbg_dump_fragtree_nolock(struct jffs2_inode_info *f) +{ + struct jffs2_node_frag *this = frag_first(&f->fragtree); + uint32_t lastofs = 0; + int buggy = 0; + + printk(JFFS2_DBG_MSG_PREFIX " dump fragtree of ino #%u\n", f->inocache->ino); + while(this) { + if (this->node) + printk(JFFS2_DBG "frag %#04x-%#04x: %#08x(%d) on flash (*%p), left (%p), right (%p), parent (%p)\n", + this->ofs, this->ofs+this->size, ref_offset(this->node->raw), + ref_flags(this->node->raw), this, frag_left(this), frag_right(this), + frag_parent(this)); + else + printk(JFFS2_DBG "frag %#04x-%#04x: hole (*%p). left (%p), right (%p), parent (%p)\n", + this->ofs, this->ofs+this->size, this, frag_left(this), + frag_right(this), frag_parent(this)); + if (this->ofs != lastofs) + buggy = 1; + lastofs = this->ofs + this->size; + this = frag_next(this); + } + + if (f->metadata) + printk(JFFS2_DBG "metadata at 0x%08x\n", ref_offset(f->metadata->raw)); + + if (buggy) { + JFFS2_ERROR("frag tree got a hole in it.\n"); + BUG(); + } +} + +#define JFFS2_BUFDUMP_BYTES_PER_LINE 32 +void +__jffs2_dbg_dump_buffer(unsigned char *buf, int len, uint32_t offs) +{ + int skip; + int i; + + printk(JFFS2_DBG_MSG_PREFIX " dump from offset %#08x to offset %#08x (%x bytes).\n", + offs, offs + len, len); + i = skip = offs % JFFS2_BUFDUMP_BYTES_PER_LINE; + offs = offs & ~(JFFS2_BUFDUMP_BYTES_PER_LINE - 1); + + if (skip != 0) + printk(JFFS2_DBG "%#08x: ", offs); + + while (skip--) + printk(" "); + + while (i < len) { + if ((i % JFFS2_BUFDUMP_BYTES_PER_LINE) == 0 && i != len -1) { + if (i != 0) + printk("\n"); + offs += JFFS2_BUFDUMP_BYTES_PER_LINE; + printk(JFFS2_DBG "%0#8x: ", offs); + } + + printk("%02x ", buf[i]); + + i += 1; + } + + printk("\n"); +} + +/* + * Dump a JFFS2 node. + */ +void +__jffs2_dbg_dump_node(struct jffs2_sb_info *c, uint32_t ofs) +{ + union jffs2_node_union node; + int len = sizeof(union jffs2_node_union); + size_t retlen; + uint32_t crc; + int ret; + + printk(JFFS2_DBG_MSG_PREFIX " dump node at offset %#08x.\n", ofs); + + ret = jffs2_flash_read(c, ofs, len, &retlen, (unsigned char *)&node); + if (ret || (retlen != len)) { + JFFS2_ERROR("read %d bytes failed or short. ret %d, retlen %zd.\n", + len, ret, retlen); + return; + } + + printk(JFFS2_DBG "magic:\t%#04x\n", je16_to_cpu(node.u.magic)); + printk(JFFS2_DBG "nodetype:\t%#04x\n", je16_to_cpu(node.u.nodetype)); + printk(JFFS2_DBG "totlen:\t%#08x\n", je32_to_cpu(node.u.totlen)); + printk(JFFS2_DBG "hdr_crc:\t%#08x\n", je32_to_cpu(node.u.hdr_crc)); + + crc = crc32(0, &node.u, sizeof(node.u) - 4); + if (crc != je32_to_cpu(node.u.hdr_crc)) { + JFFS2_ERROR("wrong common header CRC.\n"); + return; + } + + if (je16_to_cpu(node.u.magic) != JFFS2_MAGIC_BITMASK && + je16_to_cpu(node.u.magic) != JFFS2_OLD_MAGIC_BITMASK) + { + JFFS2_ERROR("wrong node magic: %#04x instead of %#04x.\n", + je16_to_cpu(node.u.magic), JFFS2_MAGIC_BITMASK); + return; + } + + switch(je16_to_cpu(node.u.nodetype)) { + + case JFFS2_NODETYPE_INODE: + + printk(JFFS2_DBG "the node is inode node\n"); + printk(JFFS2_DBG "ino:\t%#08x\n", je32_to_cpu(node.i.ino)); + printk(JFFS2_DBG "version:\t%#08x\n", je32_to_cpu(node.i.version)); + printk(JFFS2_DBG "mode:\t%#08x\n", node.i.mode.m); + printk(JFFS2_DBG "uid:\t%#04x\n", je16_to_cpu(node.i.uid)); + printk(JFFS2_DBG "gid:\t%#04x\n", je16_to_cpu(node.i.gid)); + printk(JFFS2_DBG "isize:\t%#08x\n", je32_to_cpu(node.i.isize)); + printk(JFFS2_DBG "atime:\t%#08x\n", je32_to_cpu(node.i.atime)); + printk(JFFS2_DBG "mtime:\t%#08x\n", je32_to_cpu(node.i.mtime)); + printk(JFFS2_DBG "ctime:\t%#08x\n", je32_to_cpu(node.i.ctime)); + printk(JFFS2_DBG "offset:\t%#08x\n", je32_to_cpu(node.i.offset)); + printk(JFFS2_DBG "csize:\t%#08x\n", je32_to_cpu(node.i.csize)); + printk(JFFS2_DBG "dsize:\t%#08x\n", je32_to_cpu(node.i.dsize)); + printk(JFFS2_DBG "compr:\t%#02x\n", node.i.compr); + printk(JFFS2_DBG "usercompr:\t%#02x\n", node.i.usercompr); + printk(JFFS2_DBG "flags:\t%#04x\n", je16_to_cpu(node.i.flags)); + printk(JFFS2_DBG "data_crc:\t%#08x\n", je32_to_cpu(node.i.data_crc)); + printk(JFFS2_DBG "node_crc:\t%#08x\n", je32_to_cpu(node.i.node_crc)); + + crc = crc32(0, &node.i, sizeof(node.i) - 8); + if (crc != je32_to_cpu(node.i.node_crc)) { + JFFS2_ERROR("wrong node header CRC.\n"); + return; + } + break; + + case JFFS2_NODETYPE_DIRENT: + + printk(JFFS2_DBG "the node is dirent node\n"); + printk(JFFS2_DBG "pino:\t%#08x\n", je32_to_cpu(node.d.pino)); + printk(JFFS2_DBG "version:\t%#08x\n", je32_to_cpu(node.d.version)); + printk(JFFS2_DBG "ino:\t%#08x\n", je32_to_cpu(node.d.ino)); + printk(JFFS2_DBG "mctime:\t%#08x\n", je32_to_cpu(node.d.mctime)); + printk(JFFS2_DBG "nsize:\t%#02x\n", node.d.nsize); + printk(JFFS2_DBG "type:\t%#02x\n", node.d.type); + printk(JFFS2_DBG "node_crc:\t%#08x\n", je32_to_cpu(node.d.node_crc)); + printk(JFFS2_DBG "name_crc:\t%#08x\n", je32_to_cpu(node.d.name_crc)); + + node.d.name[node.d.nsize] = '\0'; + printk(JFFS2_DBG "name:\t\"%s\"\n", node.d.name); + + crc = crc32(0, &node.d, sizeof(node.d) - 8); + if (crc != je32_to_cpu(node.d.node_crc)) { + JFFS2_ERROR("wrong node header CRC.\n"); + return; + } + break; + + default: + printk(JFFS2_DBG "node type is unknown\n"); + break; + } +} +#endif /* JFFS2_DBG_DUMPS || JFFS2_DBG_PARANOIA_CHECKS */ diff --git a/fs/jffs2/debug.h b/fs/jffs2/debug.h new file mode 100644 index 000000000..40e203b6e --- /dev/null +++ b/fs/jffs2/debug.h @@ -0,0 +1,276 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * Copyright © 2004-2010 David Woodhouse + * + * Created by David Woodhouse + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#ifndef _JFFS2_DEBUG_H_ +#define _JFFS2_DEBUG_H_ + +#include +#include + +#ifndef CONFIG_JFFS2_FS_DEBUG +#define CONFIG_JFFS2_FS_DEBUG 0 +#endif + +#if CONFIG_JFFS2_FS_DEBUG > 0 +/* Enable "paranoia" checks and dumps */ +#define JFFS2_DBG_PARANOIA_CHECKS +#define JFFS2_DBG_DUMPS + +/* + * By defining/undefining the below macros one may select debugging messages + * fro specific JFFS2 subsystems. + */ +#define JFFS2_DBG_READINODE_MESSAGES +#define JFFS2_DBG_FRAGTREE_MESSAGES +#define JFFS2_DBG_DENTLIST_MESSAGES +#define JFFS2_DBG_NODEREF_MESSAGES +#define JFFS2_DBG_INOCACHE_MESSAGES +#define JFFS2_DBG_SUMMARY_MESSAGES +#define JFFS2_DBG_FSBUILD_MESSAGES +#endif + +#if CONFIG_JFFS2_FS_DEBUG > 1 +#define JFFS2_DBG_FRAGTREE2_MESSAGES +#define JFFS2_DBG_READINODE2_MESSAGES +#define JFFS2_DBG_MEMALLOC_MESSAGES +#endif + +/* Sanity checks are supposed to be light-weight and enabled by default */ +#define JFFS2_DBG_SANITY_CHECKS + +/* + * Dx() are mainly used for debugging messages, they must go away and be + * superseded by nicer dbg_xxx() macros... + */ +#if CONFIG_JFFS2_FS_DEBUG > 0 +#define DEBUG +#define D1(x) x +#else +#define D1(x) +#endif + +#if CONFIG_JFFS2_FS_DEBUG > 1 +#define D2(x) x +#else +#define D2(x) +#endif + +#define jffs2_dbg(level, fmt, ...) \ +do { \ + if (CONFIG_JFFS2_FS_DEBUG >= level) \ + pr_debug(fmt, ##__VA_ARGS__); \ +} while (0) + +/* The prefixes of JFFS2 messages */ +#define JFFS2_DBG KERN_DEBUG +#define JFFS2_DBG_PREFIX "[JFFS2 DBG]" +#define JFFS2_DBG_MSG_PREFIX JFFS2_DBG JFFS2_DBG_PREFIX + +/* JFFS2 message macros */ +#define JFFS2_ERROR(fmt, ...) \ + pr_err("error: (%d) %s: " fmt, \ + task_pid_nr(current), __func__, ##__VA_ARGS__) + +#define JFFS2_WARNING(fmt, ...) \ + pr_warn("warning: (%d) %s: " fmt, \ + task_pid_nr(current), __func__, ##__VA_ARGS__) + +#define JFFS2_NOTICE(fmt, ...) \ + pr_notice("notice: (%d) %s: " fmt, \ + task_pid_nr(current), __func__, ##__VA_ARGS__) + +#define JFFS2_DEBUG(fmt, ...) \ + printk(KERN_DEBUG "[JFFS2 DBG] (%d) %s: " fmt, \ + task_pid_nr(current), __func__, ##__VA_ARGS__) + +/* + * We split our debugging messages on several parts, depending on the JFFS2 + * subsystem the message belongs to. + */ +/* Read inode debugging messages */ +#ifdef JFFS2_DBG_READINODE_MESSAGES +#define dbg_readinode(fmt, ...) JFFS2_DEBUG(fmt, ##__VA_ARGS__) +#else +#define dbg_readinode(fmt, ...) no_printk(fmt, ##__VA_ARGS__) +#endif +#ifdef JFFS2_DBG_READINODE2_MESSAGES +#define dbg_readinode2(fmt, ...) JFFS2_DEBUG(fmt, ##__VA_ARGS__) +#else +#define dbg_readinode2(fmt, ...) no_printk(fmt, ##__VA_ARGS__) +#endif + +/* Fragtree build debugging messages */ +#ifdef JFFS2_DBG_FRAGTREE_MESSAGES +#define dbg_fragtree(fmt, ...) JFFS2_DEBUG(fmt, ##__VA_ARGS__) +#else +#define dbg_fragtree(fmt, ...) no_printk(fmt, ##__VA_ARGS__) +#endif +#ifdef JFFS2_DBG_FRAGTREE2_MESSAGES +#define dbg_fragtree2(fmt, ...) JFFS2_DEBUG(fmt, ##__VA_ARGS__) +#else +#define dbg_fragtree2(fmt, ...) no_printk(fmt, ##__VA_ARGS__) +#endif + +/* Directory entry list manilulation debugging messages */ +#ifdef JFFS2_DBG_DENTLIST_MESSAGES +#define dbg_dentlist(fmt, ...) JFFS2_DEBUG(fmt, ##__VA_ARGS__) +#else +#define dbg_dentlist(fmt, ...) no_printk(fmt, ##__VA_ARGS__) +#endif + +/* Print the messages about manipulating node_refs */ +#ifdef JFFS2_DBG_NODEREF_MESSAGES +#define dbg_noderef(fmt, ...) JFFS2_DEBUG(fmt, ##__VA_ARGS__) +#else +#define dbg_noderef(fmt, ...) no_printk(fmt, ##__VA_ARGS__) +#endif + +/* Manipulations with the list of inodes (JFFS2 inocache) */ +#ifdef JFFS2_DBG_INOCACHE_MESSAGES +#define dbg_inocache(fmt, ...) JFFS2_DEBUG(fmt, ##__VA_ARGS__) +#else +#define dbg_inocache(fmt, ...) no_printk(fmt, ##__VA_ARGS__) +#endif + +/* Summary debugging messages */ +#ifdef JFFS2_DBG_SUMMARY_MESSAGES +#define dbg_summary(fmt, ...) JFFS2_DEBUG(fmt, ##__VA_ARGS__) +#else +#define dbg_summary(fmt, ...) no_printk(fmt, ##__VA_ARGS__) +#endif + +/* File system build messages */ +#ifdef JFFS2_DBG_FSBUILD_MESSAGES +#define dbg_fsbuild(fmt, ...) JFFS2_DEBUG(fmt, ##__VA_ARGS__) +#else +#define dbg_fsbuild(fmt, ...) no_printk(fmt, ##__VA_ARGS__) +#endif + +/* Watch the object allocations */ +#ifdef JFFS2_DBG_MEMALLOC_MESSAGES +#define dbg_memalloc(fmt, ...) JFFS2_DEBUG(fmt, ##__VA_ARGS__) +#else +#define dbg_memalloc(fmt, ...) no_printk(fmt, ##__VA_ARGS__) +#endif + +/* Watch the XATTR subsystem */ +#ifdef JFFS2_DBG_XATTR_MESSAGES +#define dbg_xattr(fmt, ...) JFFS2_DEBUG(fmt, ##__VA_ARGS__) +#else +#define dbg_xattr(fmt, ...) no_printk(fmt, ##__VA_ARGS__) +#endif + +/* "Sanity" checks */ +void +__jffs2_dbg_acct_sanity_check_nolock(struct jffs2_sb_info *c, + struct jffs2_eraseblock *jeb); +void +__jffs2_dbg_acct_sanity_check(struct jffs2_sb_info *c, + struct jffs2_eraseblock *jeb); + +/* "Paranoia" checks */ +void +__jffs2_dbg_fragtree_paranoia_check(struct jffs2_inode_info *f); +void +__jffs2_dbg_fragtree_paranoia_check_nolock(struct jffs2_inode_info *f); +void +__jffs2_dbg_acct_paranoia_check(struct jffs2_sb_info *c, + struct jffs2_eraseblock *jeb); +void +__jffs2_dbg_acct_paranoia_check_nolock(struct jffs2_sb_info *c, + struct jffs2_eraseblock *jeb); +void +__jffs2_dbg_prewrite_paranoia_check(struct jffs2_sb_info *c, + uint32_t ofs, int len); + +/* "Dump" functions */ +void +__jffs2_dbg_dump_jeb(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb); +void +__jffs2_dbg_dump_jeb_nolock(struct jffs2_eraseblock *jeb); +void +__jffs2_dbg_dump_block_lists(struct jffs2_sb_info *c); +void +__jffs2_dbg_dump_block_lists_nolock(struct jffs2_sb_info *c); +void +__jffs2_dbg_dump_node_refs(struct jffs2_sb_info *c, + struct jffs2_eraseblock *jeb); +void +__jffs2_dbg_dump_node_refs_nolock(struct jffs2_sb_info *c, + struct jffs2_eraseblock *jeb); +void +__jffs2_dbg_dump_fragtree(struct jffs2_inode_info *f); +void +__jffs2_dbg_dump_fragtree_nolock(struct jffs2_inode_info *f); +void +__jffs2_dbg_dump_buffer(unsigned char *buf, int len, uint32_t offs); +void +__jffs2_dbg_dump_node(struct jffs2_sb_info *c, uint32_t ofs); + +#ifdef JFFS2_DBG_PARANOIA_CHECKS +#define jffs2_dbg_fragtree_paranoia_check(f) \ + __jffs2_dbg_fragtree_paranoia_check(f) +#define jffs2_dbg_fragtree_paranoia_check_nolock(f) \ + __jffs2_dbg_fragtree_paranoia_check_nolock(f) +#define jffs2_dbg_acct_paranoia_check(c, jeb) \ + __jffs2_dbg_acct_paranoia_check(c,jeb) +#define jffs2_dbg_acct_paranoia_check_nolock(c, jeb) \ + __jffs2_dbg_acct_paranoia_check_nolock(c,jeb) +#define jffs2_dbg_prewrite_paranoia_check(c, ofs, len) \ + __jffs2_dbg_prewrite_paranoia_check(c, ofs, len) +#else +#define jffs2_dbg_fragtree_paranoia_check(f) +#define jffs2_dbg_fragtree_paranoia_check_nolock(f) +#define jffs2_dbg_acct_paranoia_check(c, jeb) +#define jffs2_dbg_acct_paranoia_check_nolock(c, jeb) +#define jffs2_dbg_prewrite_paranoia_check(c, ofs, len) +#endif /* !JFFS2_PARANOIA_CHECKS */ + +#ifdef JFFS2_DBG_DUMPS +#define jffs2_dbg_dump_jeb(c, jeb) \ + __jffs2_dbg_dump_jeb(c, jeb); +#define jffs2_dbg_dump_jeb_nolock(jeb) \ + __jffs2_dbg_dump_jeb_nolock(jeb); +#define jffs2_dbg_dump_block_lists(c) \ + __jffs2_dbg_dump_block_lists(c) +#define jffs2_dbg_dump_block_lists_nolock(c) \ + __jffs2_dbg_dump_block_lists_nolock(c) +#define jffs2_dbg_dump_fragtree(f) \ + __jffs2_dbg_dump_fragtree(f); +#define jffs2_dbg_dump_fragtree_nolock(f) \ + __jffs2_dbg_dump_fragtree_nolock(f); +#define jffs2_dbg_dump_buffer(buf, len, offs) \ + __jffs2_dbg_dump_buffer(*buf, len, offs); +#define jffs2_dbg_dump_node(c, ofs) \ + __jffs2_dbg_dump_node(c, ofs); +#else +#define jffs2_dbg_dump_jeb(c, jeb) +#define jffs2_dbg_dump_jeb_nolock(jeb) +#define jffs2_dbg_dump_block_lists(c) +#define jffs2_dbg_dump_block_lists_nolock(c) +#define jffs2_dbg_dump_fragtree(f) +#define jffs2_dbg_dump_fragtree_nolock(f) +#define jffs2_dbg_dump_buffer(buf, len, offs) +#define jffs2_dbg_dump_node(c, ofs) +#endif /* !JFFS2_DBG_DUMPS */ + +#ifdef JFFS2_DBG_SANITY_CHECKS +#define jffs2_dbg_acct_sanity_check(c, jeb) \ + __jffs2_dbg_acct_sanity_check(c, jeb) +#define jffs2_dbg_acct_sanity_check_nolock(c, jeb) \ + __jffs2_dbg_acct_sanity_check_nolock(c, jeb) +#else +#define jffs2_dbg_acct_sanity_check(c, jeb) +#define jffs2_dbg_acct_sanity_check_nolock(c, jeb) +#endif /* !JFFS2_DBG_SANITY_CHECKS */ + +#endif /* _JFFS2_DEBUG_H_ */ diff --git a/fs/jffs2/dir.c b/fs/jffs2/dir.c new file mode 100644 index 000000000..656c92086 --- /dev/null +++ b/fs/jffs2/dir.c @@ -0,0 +1,889 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * Copyright © 2004-2010 David Woodhouse + * + * Created by David Woodhouse + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include "jffs2_fs_i.h" +#include "jffs2_fs_sb.h" +#include +#include "nodelist.h" + +static int jffs2_readdir (struct file *, struct dir_context *); + +static int jffs2_create (struct mnt_idmap *, struct inode *, + struct dentry *, umode_t); +static struct dentry *jffs2_lookup (struct inode *,struct dentry *, + unsigned int); +static int jffs2_link (struct dentry *,struct inode *,struct dentry *); +static int jffs2_unlink (struct inode *,struct dentry *); +static int jffs2_symlink (struct mnt_idmap *, struct inode *, + struct dentry *, const char *); +static struct dentry *jffs2_mkdir (struct mnt_idmap *, struct inode *,struct dentry *, + umode_t); +static int jffs2_rmdir (struct inode *,struct dentry *); +static int jffs2_mknod (struct mnt_idmap *, struct inode *,struct dentry *, + umode_t,dev_t); +static int jffs2_rename (struct mnt_idmap *, struct inode *, + struct dentry *, struct inode *, struct dentry *, + unsigned int); + +const struct file_operations jffs2_dir_operations = +{ + .read = generic_read_dir, + .iterate_shared=jffs2_readdir, + .unlocked_ioctl=jffs2_ioctl, + .fsync = jffs2_fsync, + .llseek = generic_file_llseek, + .setlease = generic_setlease, +}; + + +const struct inode_operations jffs2_dir_inode_operations = +{ + .create = jffs2_create, + .lookup = jffs2_lookup, + .link = jffs2_link, + .unlink = jffs2_unlink, + .symlink = jffs2_symlink, + .mkdir = jffs2_mkdir, + .rmdir = jffs2_rmdir, + .mknod = jffs2_mknod, + .rename = jffs2_rename, + .get_inode_acl = jffs2_get_acl, + .set_acl = jffs2_set_acl, + .setattr = jffs2_setattr, + .listxattr = jffs2_listxattr, +}; + +/***********************************************************************/ + + +/* We keep the dirent list sorted in increasing order of name hash, + and we use the same hash function as the dentries. Makes this + nice and simple +*/ +static struct dentry *jffs2_lookup(struct inode *dir_i, struct dentry *target, + unsigned int flags) +{ + struct jffs2_inode_info *dir_f; + struct jffs2_full_dirent *fd = NULL, *fd_list; + uint32_t ino = 0; + struct inode *inode = NULL; + unsigned int nhash; + + jffs2_dbg(1, "jffs2_lookup()\n"); + + if (target->d_name.len > JFFS2_MAX_NAME_LEN) + return ERR_PTR(-ENAMETOOLONG); + + dir_f = JFFS2_INODE_INFO(dir_i); + + /* The 'nhash' on the fd_list is not the same as the dentry hash */ + nhash = full_name_hash(NULL, target->d_name.name, target->d_name.len); + + mutex_lock(&dir_f->sem); + + /* NB: The 2.2 backport will need to explicitly check for '.' and '..' here */ + for (fd_list = dir_f->dents; fd_list && fd_list->nhash <= nhash; fd_list = fd_list->next) { + if (fd_list->nhash == nhash && + (!fd || fd_list->version > fd->version) && + strlen(fd_list->name) == target->d_name.len && + !strncmp(fd_list->name, target->d_name.name, target->d_name.len)) { + fd = fd_list; + } + } + if (fd) + ino = fd->ino; + mutex_unlock(&dir_f->sem); + if (ino) { + inode = jffs2_iget(dir_i->i_sb, ino); + if (IS_ERR(inode)) + pr_warn("iget() failed for ino #%u\n", ino); + } + + return d_splice_alias(inode, target); +} + +/***********************************************************************/ + + +static int jffs2_readdir(struct file *file, struct dir_context *ctx) +{ + struct inode *inode = file_inode(file); + struct jffs2_inode_info *f = JFFS2_INODE_INFO(inode); + struct jffs2_full_dirent *fd; + unsigned long curofs = 1; + + jffs2_dbg(1, "jffs2_readdir() for dir_i #%llu\n", inode->i_ino); + + if (!dir_emit_dots(file, ctx)) + return 0; + + mutex_lock(&f->sem); + for (fd = f->dents; fd; fd = fd->next) { + curofs++; + /* First loop: curofs = 2; pos = 2 */ + if (curofs < ctx->pos) { + jffs2_dbg(2, "Skipping dirent: \"%s\", ino #%u, type %d, because curofs %ld < offset %ld\n", + fd->name, fd->ino, fd->type, curofs, (unsigned long)ctx->pos); + continue; + } + if (!fd->ino) { + jffs2_dbg(2, "Skipping deletion dirent \"%s\"\n", + fd->name); + ctx->pos++; + continue; + } + jffs2_dbg(2, "Dirent %ld: \"%s\", ino #%u, type %d\n", + (unsigned long)ctx->pos, fd->name, fd->ino, fd->type); + if (!dir_emit(ctx, fd->name, strlen(fd->name), fd->ino, fd->type)) + break; + ctx->pos++; + } + mutex_unlock(&f->sem); + return 0; +} + +/***********************************************************************/ + + +static int jffs2_create(struct mnt_idmap *idmap, struct inode *dir_i, + struct dentry *dentry, umode_t mode) +{ + struct jffs2_raw_inode *ri; + struct jffs2_inode_info *f, *dir_f; + struct jffs2_sb_info *c; + struct inode *inode; + int ret; + + ri = jffs2_alloc_raw_inode(); + if (!ri) + return -ENOMEM; + + c = JFFS2_SB_INFO(dir_i->i_sb); + + jffs2_dbg(1, "%s()\n", __func__); + + inode = jffs2_new_inode(dir_i, mode, ri); + + if (IS_ERR(inode)) { + jffs2_dbg(1, "jffs2_new_inode() failed\n"); + jffs2_free_raw_inode(ri); + return PTR_ERR(inode); + } + + inode->i_op = &jffs2_file_inode_operations; + inode->i_fop = &jffs2_file_operations; + inode->i_mapping->a_ops = &jffs2_file_address_operations; + inode->i_mapping->nrpages = 0; + + f = JFFS2_INODE_INFO(inode); + dir_f = JFFS2_INODE_INFO(dir_i); + + /* jffs2_do_create() will want to lock it, _after_ reserving + space and taking c-alloc_sem. If we keep it locked here, + lockdep gets unhappy (although it's a false positive; + nothing else will be looking at this inode yet so there's + no chance of AB-BA deadlock involving its f->sem). */ + mutex_unlock(&f->sem); + + ret = jffs2_do_create(c, dir_f, f, ri, &dentry->d_name); + if (ret) + goto fail; + + inode_set_mtime_to_ts(dir_i, + inode_set_ctime_to_ts(dir_i, ITIME(je32_to_cpu(ri->ctime)))); + + jffs2_free_raw_inode(ri); + + jffs2_dbg(1, "%s(): Created ino #%llu with mode %o, nlink %d(%d). nrpages %ld\n", + __func__, inode->i_ino, inode->i_mode, inode->i_nlink, + f->inocache->pino_nlink, inode->i_mapping->nrpages); + + d_instantiate_new(dentry, inode); + return 0; + + fail: + iget_failed(inode); + jffs2_free_raw_inode(ri); + return ret; +} + +/***********************************************************************/ + + +static int jffs2_unlink(struct inode *dir_i, struct dentry *dentry) +{ + struct jffs2_sb_info *c = JFFS2_SB_INFO(dir_i->i_sb); + struct jffs2_inode_info *dir_f = JFFS2_INODE_INFO(dir_i); + struct jffs2_inode_info *dead_f = JFFS2_INODE_INFO(d_inode(dentry)); + int ret; + uint32_t now = JFFS2_NOW(); + + ret = jffs2_do_unlink(c, dir_f, dentry->d_name.name, + dentry->d_name.len, dead_f, now); + if (dead_f->inocache) + set_nlink(d_inode(dentry), dead_f->inocache->pino_nlink); + if (!ret) + inode_set_mtime_to_ts(dir_i, + inode_set_ctime_to_ts(dir_i, ITIME(now))); + return ret; +} +/***********************************************************************/ + + +static int jffs2_link (struct dentry *old_dentry, struct inode *dir_i, struct dentry *dentry) +{ + struct jffs2_sb_info *c = JFFS2_SB_INFO(old_dentry->d_sb); + struct jffs2_inode_info *f = JFFS2_INODE_INFO(d_inode(old_dentry)); + struct jffs2_inode_info *dir_f = JFFS2_INODE_INFO(dir_i); + int ret; + uint8_t type; + uint32_t now; + + /* Don't let people make hard links to bad inodes. */ + if (!f->inocache) + return -EIO; + + if (d_is_dir(old_dentry)) + return -EPERM; + + /* XXX: This is ugly */ + type = (d_inode(old_dentry)->i_mode & S_IFMT) >> 12; + if (!type) type = DT_REG; + + now = JFFS2_NOW(); + ret = jffs2_do_link(c, dir_f, f->inocache->ino, type, dentry->d_name.name, dentry->d_name.len, now); + + if (!ret) { + mutex_lock(&f->sem); + set_nlink(d_inode(old_dentry), ++f->inocache->pino_nlink); + mutex_unlock(&f->sem); + d_instantiate(dentry, d_inode(old_dentry)); + inode_set_mtime_to_ts(dir_i, + inode_set_ctime_to_ts(dir_i, ITIME(now))); + ihold(d_inode(old_dentry)); + } + return ret; +} + +/***********************************************************************/ + +static int jffs2_symlink (struct mnt_idmap *idmap, struct inode *dir_i, + struct dentry *dentry, const char *target) +{ + struct jffs2_inode_info *f, *dir_f; + struct jffs2_sb_info *c; + struct inode *inode; + struct jffs2_raw_inode *ri; + struct jffs2_raw_dirent *rd; + struct jffs2_full_dnode *fn; + struct jffs2_full_dirent *fd; + int namelen; + uint32_t alloclen; + int ret, targetlen = strlen(target); + + /* FIXME: If you care. We'd need to use frags for the target + if it grows much more than this */ + if (targetlen > 254) + return -ENAMETOOLONG; + + ri = jffs2_alloc_raw_inode(); + + if (!ri) + return -ENOMEM; + + c = JFFS2_SB_INFO(dir_i->i_sb); + + /* Try to reserve enough space for both node and dirent. + * Just the node will do for now, though + */ + namelen = dentry->d_name.len; + ret = jffs2_reserve_space(c, sizeof(*ri) + targetlen, &alloclen, + ALLOC_NORMAL, JFFS2_SUMMARY_INODE_SIZE); + + if (ret) { + jffs2_free_raw_inode(ri); + return ret; + } + + inode = jffs2_new_inode(dir_i, S_IFLNK | S_IRWXUGO, ri); + + if (IS_ERR(inode)) { + jffs2_free_raw_inode(ri); + jffs2_complete_reservation(c); + return PTR_ERR(inode); + } + + inode->i_op = &jffs2_symlink_inode_operations; + + f = JFFS2_INODE_INFO(inode); + + inode->i_size = targetlen; + ri->isize = ri->dsize = ri->csize = cpu_to_je32(inode->i_size); + ri->totlen = cpu_to_je32(sizeof(*ri) + inode->i_size); + ri->hdr_crc = cpu_to_je32(crc32(0, ri, sizeof(struct jffs2_unknown_node)-4)); + + ri->compr = JFFS2_COMPR_NONE; + ri->data_crc = cpu_to_je32(crc32(0, target, targetlen)); + ri->node_crc = cpu_to_je32(crc32(0, ri, sizeof(*ri)-8)); + + fn = jffs2_write_dnode(c, f, ri, target, targetlen, ALLOC_NORMAL); + + jffs2_free_raw_inode(ri); + + if (IS_ERR(fn)) { + /* Eeek. Wave bye bye */ + mutex_unlock(&f->sem); + jffs2_complete_reservation(c); + ret = PTR_ERR(fn); + goto fail; + } + + /* We use f->target field to store the target path. */ + f->target = kmemdup(target, targetlen + 1, GFP_KERNEL); + if (!f->target) { + pr_warn("Can't allocate %d bytes of memory\n", targetlen + 1); + mutex_unlock(&f->sem); + jffs2_complete_reservation(c); + ret = -ENOMEM; + goto fail; + } + inode->i_link = f->target; + + jffs2_dbg(1, "%s(): symlink's target '%s' cached\n", + __func__, (char *)f->target); + + /* No data here. Only a metadata node, which will be + obsoleted by the first data write + */ + f->metadata = fn; + mutex_unlock(&f->sem); + + jffs2_complete_reservation(c); + + ret = jffs2_init_security(inode, dir_i, &dentry->d_name); + if (ret) + goto fail; + + ret = jffs2_init_acl_post(inode); + if (ret) + goto fail; + + ret = jffs2_reserve_space(c, sizeof(*rd)+namelen, &alloclen, + ALLOC_NORMAL, JFFS2_SUMMARY_DIRENT_SIZE(namelen)); + if (ret) + goto fail; + + rd = jffs2_alloc_raw_dirent(); + if (!rd) { + /* Argh. Now we treat it like a normal delete */ + jffs2_complete_reservation(c); + ret = -ENOMEM; + goto fail; + } + + dir_f = JFFS2_INODE_INFO(dir_i); + mutex_lock(&dir_f->sem); + + rd->magic = cpu_to_je16(JFFS2_MAGIC_BITMASK); + rd->nodetype = cpu_to_je16(JFFS2_NODETYPE_DIRENT); + rd->totlen = cpu_to_je32(sizeof(*rd) + namelen); + rd->hdr_crc = cpu_to_je32(crc32(0, rd, sizeof(struct jffs2_unknown_node)-4)); + + rd->pino = cpu_to_je32(dir_i->i_ino); + rd->version = cpu_to_je32(++dir_f->highest_version); + rd->ino = cpu_to_je32(inode->i_ino); + rd->mctime = cpu_to_je32(JFFS2_NOW()); + rd->nsize = namelen; + rd->type = DT_LNK; + rd->node_crc = cpu_to_je32(crc32(0, rd, sizeof(*rd)-8)); + rd->name_crc = cpu_to_je32(crc32(0, dentry->d_name.name, namelen)); + + fd = jffs2_write_dirent(c, dir_f, rd, dentry->d_name.name, namelen, ALLOC_NORMAL); + + if (IS_ERR(fd)) { + /* dirent failed to write. Delete the inode normally + as if it were the final unlink() */ + jffs2_complete_reservation(c); + jffs2_free_raw_dirent(rd); + mutex_unlock(&dir_f->sem); + ret = PTR_ERR(fd); + goto fail; + } + + inode_set_mtime_to_ts(dir_i, + inode_set_ctime_to_ts(dir_i, ITIME(je32_to_cpu(rd->mctime)))); + + jffs2_free_raw_dirent(rd); + + /* Link the fd into the inode's list, obsoleting an old + one if necessary. */ + jffs2_add_fd_to_list(c, fd, &dir_f->dents); + + mutex_unlock(&dir_f->sem); + jffs2_complete_reservation(c); + + d_instantiate_new(dentry, inode); + return 0; + + fail: + iget_failed(inode); + return ret; +} + + +static struct dentry *jffs2_mkdir (struct mnt_idmap *idmap, struct inode *dir_i, + struct dentry *dentry, umode_t mode) +{ + struct jffs2_inode_info *f, *dir_f; + struct jffs2_sb_info *c; + struct inode *inode; + struct jffs2_raw_inode *ri; + struct jffs2_raw_dirent *rd; + struct jffs2_full_dnode *fn; + struct jffs2_full_dirent *fd; + int namelen; + uint32_t alloclen; + int ret; + + ri = jffs2_alloc_raw_inode(); + if (!ri) + return ERR_PTR(-ENOMEM); + + c = JFFS2_SB_INFO(dir_i->i_sb); + + /* Try to reserve enough space for both node and dirent. + * Just the node will do for now, though + */ + namelen = dentry->d_name.len; + ret = jffs2_reserve_space(c, sizeof(*ri), &alloclen, ALLOC_NORMAL, + JFFS2_SUMMARY_INODE_SIZE); + + if (ret) { + jffs2_free_raw_inode(ri); + return ERR_PTR(ret); + } + + inode = jffs2_new_inode(dir_i, mode, ri); + + if (IS_ERR(inode)) { + jffs2_free_raw_inode(ri); + jffs2_complete_reservation(c); + return ERR_CAST(inode); + } + + inode->i_op = &jffs2_dir_inode_operations; + inode->i_fop = &jffs2_dir_operations; + + f = JFFS2_INODE_INFO(inode); + + /* Directories get nlink 2 at start */ + set_nlink(inode, 2); + /* but ic->pino_nlink is the parent ino# */ + f->inocache->pino_nlink = dir_i->i_ino; + + ri->data_crc = cpu_to_je32(0); + ri->node_crc = cpu_to_je32(crc32(0, ri, sizeof(*ri)-8)); + + fn = jffs2_write_dnode(c, f, ri, NULL, 0, ALLOC_NORMAL); + + jffs2_free_raw_inode(ri); + + if (IS_ERR(fn)) { + /* Eeek. Wave bye bye */ + mutex_unlock(&f->sem); + jffs2_complete_reservation(c); + ret = PTR_ERR(fn); + goto fail; + } + /* No data here. Only a metadata node, which will be + obsoleted by the first data write + */ + f->metadata = fn; + mutex_unlock(&f->sem); + + jffs2_complete_reservation(c); + + ret = jffs2_init_security(inode, dir_i, &dentry->d_name); + if (ret) + goto fail; + + ret = jffs2_init_acl_post(inode); + if (ret) + goto fail; + + ret = jffs2_reserve_space(c, sizeof(*rd)+namelen, &alloclen, + ALLOC_NORMAL, JFFS2_SUMMARY_DIRENT_SIZE(namelen)); + if (ret) + goto fail; + + rd = jffs2_alloc_raw_dirent(); + if (!rd) { + /* Argh. Now we treat it like a normal delete */ + jffs2_complete_reservation(c); + ret = -ENOMEM; + goto fail; + } + + dir_f = JFFS2_INODE_INFO(dir_i); + mutex_lock(&dir_f->sem); + + rd->magic = cpu_to_je16(JFFS2_MAGIC_BITMASK); + rd->nodetype = cpu_to_je16(JFFS2_NODETYPE_DIRENT); + rd->totlen = cpu_to_je32(sizeof(*rd) + namelen); + rd->hdr_crc = cpu_to_je32(crc32(0, rd, sizeof(struct jffs2_unknown_node)-4)); + + rd->pino = cpu_to_je32(dir_i->i_ino); + rd->version = cpu_to_je32(++dir_f->highest_version); + rd->ino = cpu_to_je32(inode->i_ino); + rd->mctime = cpu_to_je32(JFFS2_NOW()); + rd->nsize = namelen; + rd->type = DT_DIR; + rd->node_crc = cpu_to_je32(crc32(0, rd, sizeof(*rd)-8)); + rd->name_crc = cpu_to_je32(crc32(0, dentry->d_name.name, namelen)); + + fd = jffs2_write_dirent(c, dir_f, rd, dentry->d_name.name, namelen, ALLOC_NORMAL); + + if (IS_ERR(fd)) { + /* dirent failed to write. Delete the inode normally + as if it were the final unlink() */ + jffs2_complete_reservation(c); + jffs2_free_raw_dirent(rd); + mutex_unlock(&dir_f->sem); + ret = PTR_ERR(fd); + goto fail; + } + + inode_set_mtime_to_ts(dir_i, + inode_set_ctime_to_ts(dir_i, ITIME(je32_to_cpu(rd->mctime)))); + inc_nlink(dir_i); + + jffs2_free_raw_dirent(rd); + + /* Link the fd into the inode's list, obsoleting an old + one if necessary. */ + jffs2_add_fd_to_list(c, fd, &dir_f->dents); + + mutex_unlock(&dir_f->sem); + jffs2_complete_reservation(c); + + d_instantiate_new(dentry, inode); + return NULL; + + fail: + iget_failed(inode); + return ERR_PTR(ret); +} + +static int jffs2_rmdir (struct inode *dir_i, struct dentry *dentry) +{ + struct jffs2_sb_info *c = JFFS2_SB_INFO(dir_i->i_sb); + struct jffs2_inode_info *dir_f = JFFS2_INODE_INFO(dir_i); + struct jffs2_inode_info *f = JFFS2_INODE_INFO(d_inode(dentry)); + struct jffs2_full_dirent *fd; + int ret; + uint32_t now = JFFS2_NOW(); + + mutex_lock(&f->sem); + for (fd = f->dents ; fd; fd = fd->next) { + if (fd->ino) { + mutex_unlock(&f->sem); + return -ENOTEMPTY; + } + } + mutex_unlock(&f->sem); + + ret = jffs2_do_unlink(c, dir_f, dentry->d_name.name, + dentry->d_name.len, f, now); + if (!ret) { + inode_set_mtime_to_ts(dir_i, + inode_set_ctime_to_ts(dir_i, ITIME(now))); + clear_nlink(d_inode(dentry)); + drop_nlink(dir_i); + } + return ret; +} + +static int jffs2_mknod (struct mnt_idmap *idmap, struct inode *dir_i, + struct dentry *dentry, umode_t mode, dev_t rdev) +{ + struct jffs2_inode_info *f, *dir_f; + struct jffs2_sb_info *c; + struct inode *inode; + struct jffs2_raw_inode *ri; + struct jffs2_raw_dirent *rd; + struct jffs2_full_dnode *fn; + struct jffs2_full_dirent *fd; + int namelen; + union jffs2_device_node dev; + int devlen = 0; + uint32_t alloclen; + int ret; + + ri = jffs2_alloc_raw_inode(); + if (!ri) + return -ENOMEM; + + c = JFFS2_SB_INFO(dir_i->i_sb); + + if (S_ISBLK(mode) || S_ISCHR(mode)) + devlen = jffs2_encode_dev(&dev, rdev); + + /* Try to reserve enough space for both node and dirent. + * Just the node will do for now, though + */ + namelen = dentry->d_name.len; + ret = jffs2_reserve_space(c, sizeof(*ri) + devlen, &alloclen, + ALLOC_NORMAL, JFFS2_SUMMARY_INODE_SIZE); + + if (ret) { + jffs2_free_raw_inode(ri); + return ret; + } + + inode = jffs2_new_inode(dir_i, mode, ri); + + if (IS_ERR(inode)) { + jffs2_free_raw_inode(ri); + jffs2_complete_reservation(c); + return PTR_ERR(inode); + } + inode->i_op = &jffs2_file_inode_operations; + init_special_inode(inode, inode->i_mode, rdev); + + f = JFFS2_INODE_INFO(inode); + + ri->dsize = ri->csize = cpu_to_je32(devlen); + ri->totlen = cpu_to_je32(sizeof(*ri) + devlen); + ri->hdr_crc = cpu_to_je32(crc32(0, ri, sizeof(struct jffs2_unknown_node)-4)); + + ri->compr = JFFS2_COMPR_NONE; + ri->data_crc = cpu_to_je32(crc32(0, &dev, devlen)); + ri->node_crc = cpu_to_je32(crc32(0, ri, sizeof(*ri)-8)); + + fn = jffs2_write_dnode(c, f, ri, (char *)&dev, devlen, ALLOC_NORMAL); + + jffs2_free_raw_inode(ri); + + if (IS_ERR(fn)) { + /* Eeek. Wave bye bye */ + mutex_unlock(&f->sem); + jffs2_complete_reservation(c); + ret = PTR_ERR(fn); + goto fail; + } + /* No data here. Only a metadata node, which will be + obsoleted by the first data write + */ + f->metadata = fn; + mutex_unlock(&f->sem); + + jffs2_complete_reservation(c); + + ret = jffs2_init_security(inode, dir_i, &dentry->d_name); + if (ret) + goto fail; + + ret = jffs2_init_acl_post(inode); + if (ret) + goto fail; + + ret = jffs2_reserve_space(c, sizeof(*rd)+namelen, &alloclen, + ALLOC_NORMAL, JFFS2_SUMMARY_DIRENT_SIZE(namelen)); + if (ret) + goto fail; + + rd = jffs2_alloc_raw_dirent(); + if (!rd) { + /* Argh. Now we treat it like a normal delete */ + jffs2_complete_reservation(c); + ret = -ENOMEM; + goto fail; + } + + dir_f = JFFS2_INODE_INFO(dir_i); + mutex_lock(&dir_f->sem); + + rd->magic = cpu_to_je16(JFFS2_MAGIC_BITMASK); + rd->nodetype = cpu_to_je16(JFFS2_NODETYPE_DIRENT); + rd->totlen = cpu_to_je32(sizeof(*rd) + namelen); + rd->hdr_crc = cpu_to_je32(crc32(0, rd, sizeof(struct jffs2_unknown_node)-4)); + + rd->pino = cpu_to_je32(dir_i->i_ino); + rd->version = cpu_to_je32(++dir_f->highest_version); + rd->ino = cpu_to_je32(inode->i_ino); + rd->mctime = cpu_to_je32(JFFS2_NOW()); + rd->nsize = namelen; + + /* XXX: This is ugly. */ + rd->type = (mode & S_IFMT) >> 12; + + rd->node_crc = cpu_to_je32(crc32(0, rd, sizeof(*rd)-8)); + rd->name_crc = cpu_to_je32(crc32(0, dentry->d_name.name, namelen)); + + fd = jffs2_write_dirent(c, dir_f, rd, dentry->d_name.name, namelen, ALLOC_NORMAL); + + if (IS_ERR(fd)) { + /* dirent failed to write. Delete the inode normally + as if it were the final unlink() */ + jffs2_complete_reservation(c); + jffs2_free_raw_dirent(rd); + mutex_unlock(&dir_f->sem); + ret = PTR_ERR(fd); + goto fail; + } + + inode_set_mtime_to_ts(dir_i, + inode_set_ctime_to_ts(dir_i, ITIME(je32_to_cpu(rd->mctime)))); + + jffs2_free_raw_dirent(rd); + + /* Link the fd into the inode's list, obsoleting an old + one if necessary. */ + jffs2_add_fd_to_list(c, fd, &dir_f->dents); + + mutex_unlock(&dir_f->sem); + jffs2_complete_reservation(c); + + d_instantiate_new(dentry, inode); + return 0; + + fail: + iget_failed(inode); + return ret; +} + +static int jffs2_rename (struct mnt_idmap *idmap, + struct inode *old_dir_i, struct dentry *old_dentry, + struct inode *new_dir_i, struct dentry *new_dentry, + unsigned int flags) +{ + int ret; + struct jffs2_sb_info *c = JFFS2_SB_INFO(old_dir_i->i_sb); + struct jffs2_inode_info *victim_f = NULL; + uint8_t type; + uint32_t now; + + if (flags & ~RENAME_NOREPLACE) + return -EINVAL; + + /* The VFS will check for us and prevent trying to rename a + * file over a directory and vice versa, but if it's a directory, + * the VFS can't check whether the victim is empty. The filesystem + * needs to do that for itself. + */ + if (d_really_is_positive(new_dentry)) { + victim_f = JFFS2_INODE_INFO(d_inode(new_dentry)); + if (d_is_dir(new_dentry)) { + struct jffs2_full_dirent *fd; + + mutex_lock(&victim_f->sem); + for (fd = victim_f->dents; fd; fd = fd->next) { + if (fd->ino) { + mutex_unlock(&victim_f->sem); + return -ENOTEMPTY; + } + } + mutex_unlock(&victim_f->sem); + } + } + + /* XXX: We probably ought to alloc enough space for + both nodes at the same time. Writing the new link, + then getting -ENOSPC, is quite bad :) + */ + + /* Make a hard link */ + + /* XXX: This is ugly */ + type = (d_inode(old_dentry)->i_mode & S_IFMT) >> 12; + if (!type) type = DT_REG; + + now = JFFS2_NOW(); + ret = jffs2_do_link(c, JFFS2_INODE_INFO(new_dir_i), + d_inode(old_dentry)->i_ino, type, + new_dentry->d_name.name, new_dentry->d_name.len, now); + + if (ret) + return ret; + + if (victim_f) { + /* There was a victim. Kill it off nicely */ + if (d_is_dir(new_dentry)) + clear_nlink(d_inode(new_dentry)); + else + drop_nlink(d_inode(new_dentry)); + /* Don't oops if the victim was a dirent pointing to an + inode which didn't exist. */ + if (victim_f->inocache) { + mutex_lock(&victim_f->sem); + if (d_is_dir(new_dentry)) + victim_f->inocache->pino_nlink = 0; + else + victim_f->inocache->pino_nlink--; + mutex_unlock(&victim_f->sem); + } + } + + /* If it was a directory we moved, and there was no victim, + increase i_nlink on its new parent */ + if (d_is_dir(old_dentry) && !victim_f) + inc_nlink(new_dir_i); + + /* Unlink the original */ + ret = jffs2_do_unlink(c, JFFS2_INODE_INFO(old_dir_i), + old_dentry->d_name.name, old_dentry->d_name.len, NULL, now); + + /* We don't touch inode->i_nlink */ + + if (ret) { + /* Oh shit. We really ought to make a single node which can do both atomically */ + struct jffs2_inode_info *f = JFFS2_INODE_INFO(d_inode(old_dentry)); + mutex_lock(&f->sem); + inc_nlink(d_inode(old_dentry)); + if (f->inocache && !d_is_dir(old_dentry)) + f->inocache->pino_nlink++; + mutex_unlock(&f->sem); + + pr_notice("%s(): Link succeeded, unlink failed (err %d). You now have a hard link\n", + __func__, ret); + /* + * We can't keep the target in dcache after that. + * For one thing, we can't afford dentry aliases for directories. + * For another, if there was a victim, we _can't_ set new inode + * for that sucker and we have to trigger mount eviction - the + * caller won't do it on its own since we are returning an error. + */ + d_invalidate(new_dentry); + inode_set_mtime_to_ts(new_dir_i, + inode_set_ctime_to_ts(new_dir_i, ITIME(now))); + return ret; + } + + if (d_is_dir(old_dentry)) + drop_nlink(old_dir_i); + + inode_set_mtime_to_ts(old_dir_i, + inode_set_ctime_to_ts(old_dir_i, ITIME(now))); + inode_set_mtime_to_ts(new_dir_i, + inode_set_ctime_to_ts(new_dir_i, ITIME(now))); + + return 0; +} + diff --git a/fs/jffs2/erase.c b/fs/jffs2/erase.c new file mode 100644 index 000000000..a99ee831b --- /dev/null +++ b/fs/jffs2/erase.c @@ -0,0 +1,487 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * Copyright © 2004-2010 David Woodhouse + * + * Created by David Woodhouse + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include "nodelist.h" + +static void jffs2_erase_failed(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, uint32_t bad_offset); +static void jffs2_erase_succeeded(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb); +static void jffs2_mark_erased_block(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb); + +static void jffs2_erase_block(struct jffs2_sb_info *c, + struct jffs2_eraseblock *jeb) +{ + int ret; + uint32_t bad_offset; +#ifdef __ECOS + ret = jffs2_flash_erase(c, jeb); + if (!ret) { + jffs2_erase_succeeded(c, jeb); + return; + } + bad_offset = jeb->offset; +#else /* Linux */ + struct erase_info *instr; + + jffs2_dbg(1, "%s(): erase block %#08x (range %#08x-%#08x)\n", + __func__, + jeb->offset, jeb->offset, jeb->offset + c->sector_size); + instr = kzalloc_obj(struct erase_info); + if (!instr) { + pr_warn("kzalloc for struct erase_info in jffs2_erase_block failed. Refiling block for later\n"); + mutex_lock(&c->erase_free_sem); + spin_lock(&c->erase_completion_lock); + list_move(&jeb->list, &c->erase_pending_list); + c->erasing_size -= c->sector_size; + c->dirty_size += c->sector_size; + jeb->dirty_size = c->sector_size; + spin_unlock(&c->erase_completion_lock); + mutex_unlock(&c->erase_free_sem); + return; + } + + instr->addr = jeb->offset; + instr->len = c->sector_size; + + ret = mtd_erase(c->mtd, instr); + if (!ret) { + jffs2_erase_succeeded(c, jeb); + kfree(instr); + return; + } + + bad_offset = instr->fail_addr; + kfree(instr); +#endif /* __ECOS */ + + if (ret == -ENOMEM || ret == -EAGAIN) { + /* Erase failed immediately. Refile it on the list */ + jffs2_dbg(1, "Erase at 0x%08x failed: %d. Refiling on erase_pending_list\n", + jeb->offset, ret); + mutex_lock(&c->erase_free_sem); + spin_lock(&c->erase_completion_lock); + list_move(&jeb->list, &c->erase_pending_list); + c->erasing_size -= c->sector_size; + c->dirty_size += c->sector_size; + jeb->dirty_size = c->sector_size; + spin_unlock(&c->erase_completion_lock); + mutex_unlock(&c->erase_free_sem); + return; + } + + if (ret == -EROFS) + pr_warn("Erase at 0x%08x failed immediately: -EROFS. Is the sector locked?\n", + jeb->offset); + else + pr_warn("Erase at 0x%08x failed immediately: errno %d\n", + jeb->offset, ret); + + jffs2_erase_failed(c, jeb, bad_offset); +} + +int jffs2_erase_pending_blocks(struct jffs2_sb_info *c, int count) +{ + struct jffs2_eraseblock *jeb; + int work_done = 0; + + mutex_lock(&c->erase_free_sem); + + spin_lock(&c->erase_completion_lock); + + while (!list_empty(&c->erase_complete_list) || + !list_empty(&c->erase_pending_list)) { + + if (!list_empty(&c->erase_complete_list)) { + jeb = list_entry(c->erase_complete_list.next, struct jffs2_eraseblock, list); + list_move(&jeb->list, &c->erase_checking_list); + spin_unlock(&c->erase_completion_lock); + mutex_unlock(&c->erase_free_sem); + jffs2_mark_erased_block(c, jeb); + + work_done++; + if (!--count) { + jffs2_dbg(1, "Count reached. jffs2_erase_pending_blocks leaving\n"); + goto done; + } + + } else if (!list_empty(&c->erase_pending_list)) { + jeb = list_entry(c->erase_pending_list.next, struct jffs2_eraseblock, list); + jffs2_dbg(1, "Starting erase of pending block 0x%08x\n", + jeb->offset); + list_del(&jeb->list); + c->erasing_size += c->sector_size; + c->wasted_size -= jeb->wasted_size; + c->free_size -= jeb->free_size; + c->used_size -= jeb->used_size; + c->dirty_size -= jeb->dirty_size; + jeb->wasted_size = jeb->used_size = jeb->dirty_size = jeb->free_size = 0; + jffs2_free_jeb_node_refs(c, jeb); + list_add(&jeb->list, &c->erasing_list); + spin_unlock(&c->erase_completion_lock); + mutex_unlock(&c->erase_free_sem); + + jffs2_erase_block(c, jeb); + + } else { + BUG(); + } + + /* Be nice */ + cond_resched(); + mutex_lock(&c->erase_free_sem); + spin_lock(&c->erase_completion_lock); + } + + spin_unlock(&c->erase_completion_lock); + mutex_unlock(&c->erase_free_sem); + done: + jffs2_dbg(1, "jffs2_erase_pending_blocks completed\n"); + return work_done; +} + +static void jffs2_erase_succeeded(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb) +{ + jffs2_dbg(1, "Erase completed successfully at 0x%08x\n", jeb->offset); + mutex_lock(&c->erase_free_sem); + spin_lock(&c->erase_completion_lock); + list_move_tail(&jeb->list, &c->erase_complete_list); + /* Wake the GC thread to mark them clean */ + jffs2_garbage_collect_trigger(c); + spin_unlock(&c->erase_completion_lock); + mutex_unlock(&c->erase_free_sem); + wake_up(&c->erase_wait); +} + +static void jffs2_erase_failed(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, uint32_t bad_offset) +{ + /* For NAND, if the failure did not occur at the device level for a + specific physical page, don't bother updating the bad block table. */ + if (jffs2_cleanmarker_oob(c) && (bad_offset != (uint32_t)MTD_FAIL_ADDR_UNKNOWN)) { + /* We had a device-level failure to erase. Let's see if we've + failed too many times. */ + if (!jffs2_write_nand_badblock(c, jeb, bad_offset)) { + /* We'd like to give this block another try. */ + mutex_lock(&c->erase_free_sem); + spin_lock(&c->erase_completion_lock); + list_move(&jeb->list, &c->erase_pending_list); + c->erasing_size -= c->sector_size; + c->dirty_size += c->sector_size; + jeb->dirty_size = c->sector_size; + spin_unlock(&c->erase_completion_lock); + mutex_unlock(&c->erase_free_sem); + return; + } + } + + mutex_lock(&c->erase_free_sem); + spin_lock(&c->erase_completion_lock); + c->erasing_size -= c->sector_size; + c->bad_size += c->sector_size; + list_move(&jeb->list, &c->bad_list); + c->nr_erasing_blocks--; + spin_unlock(&c->erase_completion_lock); + mutex_unlock(&c->erase_free_sem); + wake_up(&c->erase_wait); +} + +/* Hmmm. Maybe we should accept the extra space it takes and make + this a standard doubly-linked list? */ +static inline void jffs2_remove_node_refs_from_ino_list(struct jffs2_sb_info *c, + struct jffs2_raw_node_ref *ref, struct jffs2_eraseblock *jeb) +{ + struct jffs2_inode_cache *ic = NULL; + struct jffs2_raw_node_ref **prev; + + prev = &ref->next_in_ino; + + /* Walk the inode's list once, removing any nodes from this eraseblock */ + while (1) { + if (!(*prev)->next_in_ino) { + /* We're looking at the jffs2_inode_cache, which is + at the end of the linked list. Stash it and continue + from the beginning of the list */ + ic = (struct jffs2_inode_cache *)(*prev); + prev = &ic->nodes; + continue; + } + + if (SECTOR_ADDR((*prev)->flash_offset) == jeb->offset) { + /* It's in the block we're erasing */ + struct jffs2_raw_node_ref *this; + + this = *prev; + *prev = this->next_in_ino; + this->next_in_ino = NULL; + + if (this == ref) + break; + + continue; + } + /* Not to be deleted. Skip */ + prev = &((*prev)->next_in_ino); + } + + /* PARANOIA */ + if (!ic) { + JFFS2_WARNING("inode_cache/xattr_datum/xattr_ref" + " not found in remove_node_refs()!!\n"); + return; + } + + jffs2_dbg(1, "Removed nodes in range 0x%08x-0x%08x from ino #%u\n", + jeb->offset, jeb->offset + c->sector_size, ic->ino); + + D2({ + int i=0; + struct jffs2_raw_node_ref *this; + printk(KERN_DEBUG "After remove_node_refs_from_ino_list: \n"); + + this = ic->nodes; + + printk(KERN_DEBUG); + while(this) { + pr_cont("0x%08x(%d)->", + ref_offset(this), ref_flags(this)); + if (++i == 5) { + printk(KERN_DEBUG); + i=0; + } + this = this->next_in_ino; + } + pr_cont("\n"); + }); + + switch (ic->class) { +#ifdef CONFIG_JFFS2_FS_XATTR + case RAWNODE_CLASS_XATTR_DATUM: + jffs2_release_xattr_datum(c, (struct jffs2_xattr_datum *)ic); + break; + case RAWNODE_CLASS_XATTR_REF: + jffs2_release_xattr_ref(c, (struct jffs2_xattr_ref *)ic); + break; +#endif + default: + if (ic->nodes == (void *)ic && ic->pino_nlink == 0) + jffs2_del_ino_cache(c, ic); + } +} + +void jffs2_free_jeb_node_refs(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb) +{ + struct jffs2_raw_node_ref *block, *ref; + jffs2_dbg(1, "Freeing all node refs for eraseblock offset 0x%08x\n", + jeb->offset); + + block = ref = jeb->first_node; + + while (ref) { + if (ref->flash_offset == REF_LINK_NODE) { + ref = ref->next_in_ino; + jffs2_free_refblock(block); + block = ref; + continue; + } + if (ref->flash_offset != REF_EMPTY_NODE && ref->next_in_ino) + jffs2_remove_node_refs_from_ino_list(c, ref, jeb); + /* else it was a non-inode node or already removed, so don't bother */ + + ref++; + } + jeb->first_node = jeb->last_node = NULL; +} + +static int jffs2_block_check_erase(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, uint32_t *bad_offset) +{ + void *ebuf; + uint32_t ofs; + size_t retlen; + int ret; + unsigned long *wordebuf; + + ret = mtd_point(c->mtd, jeb->offset, c->sector_size, &retlen, + &ebuf, NULL); + if (ret != -EOPNOTSUPP) { + if (ret) { + jffs2_dbg(1, "MTD point failed %d\n", ret); + goto do_flash_read; + } + if (retlen < c->sector_size) { + /* Don't muck about if it won't let us point to the whole erase sector */ + jffs2_dbg(1, "MTD point returned len too short: 0x%zx\n", + retlen); + mtd_unpoint(c->mtd, jeb->offset, retlen); + goto do_flash_read; + } + wordebuf = ebuf-sizeof(*wordebuf); + retlen /= sizeof(*wordebuf); + do { + if (*++wordebuf != ~0) + break; + } while(--retlen); + mtd_unpoint(c->mtd, jeb->offset, c->sector_size); + if (retlen) { + *bad_offset = jeb->offset + c->sector_size - retlen * sizeof(*wordebuf); + pr_warn("Newly-erased block contained word 0x%lx at offset 0x%08x\n", + *wordebuf, *bad_offset); + return -EIO; + } + return 0; + } + do_flash_read: + ebuf = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!ebuf) { + pr_warn("Failed to allocate page buffer for verifying erase at 0x%08x. Refiling\n", + jeb->offset); + return -EAGAIN; + } + + jffs2_dbg(1, "Verifying erase at 0x%08x\n", jeb->offset); + + for (ofs = jeb->offset; ofs < jeb->offset + c->sector_size; ) { + uint32_t readlen = min((uint32_t)PAGE_SIZE, jeb->offset + c->sector_size - ofs); + int i; + + *bad_offset = ofs; + + ret = mtd_read(c->mtd, ofs, readlen, &retlen, ebuf); + if (ret) { + pr_warn("Read of newly-erased block at 0x%08x failed: %d. Putting on bad_list\n", + ofs, ret); + ret = -EIO; + goto fail; + } + if (retlen != readlen) { + pr_warn("Short read from newly-erased block at 0x%08x. Wanted %d, got %zd\n", + ofs, readlen, retlen); + ret = -EIO; + goto fail; + } + for (i=0; ioffset); + bad_offset = jeb->offset; + + /* Cleanmarker in oob area or no cleanmarker at all ? */ + if (jffs2_cleanmarker_oob(c) || c->cleanmarker_size == 0) { + + if (jffs2_cleanmarker_oob(c)) { + if (jffs2_write_nand_cleanmarker(c, jeb)) + goto filebad; + } + } else { + + struct kvec vecs[1]; + struct jffs2_unknown_node marker = { + .magic = cpu_to_je16(JFFS2_MAGIC_BITMASK), + .nodetype = cpu_to_je16(JFFS2_NODETYPE_CLEANMARKER), + .totlen = cpu_to_je32(c->cleanmarker_size) + }; + + ret = jffs2_prealloc_raw_node_refs(c, jeb, 1); + if (ret) + goto filebad; + + marker.hdr_crc = cpu_to_je32(crc32(0, &marker, sizeof(struct jffs2_unknown_node)-4)); + + vecs[0].iov_base = (unsigned char *) ▮ + vecs[0].iov_len = sizeof(marker); + ret = jffs2_flash_direct_writev(c, vecs, 1, jeb->offset, &retlen); + + if (ret || retlen != sizeof(marker)) { + if (ret) + pr_warn("Write clean marker to block at 0x%08x failed: %d\n", + jeb->offset, ret); + else + pr_warn("Short write to newly-erased block at 0x%08x: Wanted %zd, got %zd\n", + jeb->offset, sizeof(marker), retlen); + + goto filebad; + } + } + /* Everything else got zeroed before the erase */ + jeb->free_size = c->sector_size; + + mutex_lock(&c->erase_free_sem); + spin_lock(&c->erase_completion_lock); + + c->erasing_size -= c->sector_size; + c->free_size += c->sector_size; + + /* Account for cleanmarker now, if it's in-band */ + if (c->cleanmarker_size && !jffs2_cleanmarker_oob(c)) + jffs2_link_node_ref(c, jeb, jeb->offset | REF_NORMAL, c->cleanmarker_size, NULL); + + list_move_tail(&jeb->list, &c->free_list); + c->nr_erasing_blocks--; + c->nr_free_blocks++; + + jffs2_dbg_acct_sanity_check_nolock(c, jeb); + jffs2_dbg_acct_paranoia_check_nolock(c, jeb); + + spin_unlock(&c->erase_completion_lock); + mutex_unlock(&c->erase_free_sem); + wake_up(&c->erase_wait); + return; + +filebad: + jffs2_erase_failed(c, jeb, bad_offset); + return; + +refile: + /* Stick it back on the list from whence it came and come back later */ + mutex_lock(&c->erase_free_sem); + spin_lock(&c->erase_completion_lock); + jffs2_garbage_collect_trigger(c); + list_move(&jeb->list, &c->erase_complete_list); + spin_unlock(&c->erase_completion_lock); + mutex_unlock(&c->erase_free_sem); + return; +} diff --git a/fs/jffs2/file.c b/fs/jffs2/file.c new file mode 100644 index 000000000..1e18d3a79 --- /dev/null +++ b/fs/jffs2/file.c @@ -0,0 +1,336 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * Copyright © 2004-2010 David Woodhouse + * + * Created by David Woodhouse + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include "nodelist.h" + +static int jffs2_write_end(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct folio *folio, void *fsdata); +static int jffs2_write_begin(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, + struct folio **foliop, void **fsdata); +static int jffs2_read_folio(struct file *filp, struct folio *folio); + +int jffs2_fsync(struct file *filp, loff_t start, loff_t end, int datasync) +{ + struct inode *inode = filp->f_mapping->host; + struct jffs2_sb_info *c = JFFS2_SB_INFO(inode->i_sb); + int ret; + + ret = file_write_and_wait_range(filp, start, end); + if (ret) + return ret; + + inode_lock(inode); + /* Trigger GC to flush any pending writes for this inode */ + jffs2_flush_wbuf_gc(c, inode->i_ino); + inode_unlock(inode); + + return 0; +} + +const struct file_operations jffs2_file_operations = +{ + .llseek = generic_file_llseek, + .open = generic_file_open, + .read_iter = generic_file_read_iter, + .write_iter = generic_file_write_iter, + .unlocked_ioctl=jffs2_ioctl, + .mmap_prepare = generic_file_readonly_mmap_prepare, + .fsync = jffs2_fsync, + .splice_read = filemap_splice_read, + .splice_write = iter_file_splice_write, + .setlease = generic_setlease, +}; + +/* jffs2_file_inode_operations */ + +const struct inode_operations jffs2_file_inode_operations = +{ + .get_inode_acl = jffs2_get_acl, + .set_acl = jffs2_set_acl, + .setattr = jffs2_setattr, + .listxattr = jffs2_listxattr, +}; + +const struct address_space_operations jffs2_file_address_operations = +{ + .read_folio = jffs2_read_folio, + .write_begin = jffs2_write_begin, + .write_end = jffs2_write_end, +}; + +static int jffs2_do_readpage_nolock(struct inode *inode, struct folio *folio) +{ + struct jffs2_inode_info *f = JFFS2_INODE_INFO(inode); + struct jffs2_sb_info *c = JFFS2_SB_INFO(inode->i_sb); + unsigned char *kaddr; + int ret; + + jffs2_dbg(2, "%s(): ino #%llu, page at offset 0x%lx\n", + __func__, inode->i_ino, folio->index << PAGE_SHIFT); + + BUG_ON(!folio_test_locked(folio)); + + kaddr = kmap_local_folio(folio, 0); + ret = jffs2_read_inode_range(c, f, kaddr, folio->index << PAGE_SHIFT, + PAGE_SIZE); + kunmap_local(kaddr); + + if (!ret) + folio_mark_uptodate(folio); + + flush_dcache_folio(folio); + + jffs2_dbg(2, "readpage finished\n"); + return ret; +} + +int __jffs2_read_folio(struct file *file, struct folio *folio) +{ + int ret = jffs2_do_readpage_nolock(folio->mapping->host, folio); + folio_unlock(folio); + return ret; +} + +static int jffs2_read_folio(struct file *file, struct folio *folio) +{ + struct jffs2_inode_info *f = JFFS2_INODE_INFO(folio->mapping->host); + int ret; + + mutex_lock(&f->sem); + ret = __jffs2_read_folio(file, folio); + mutex_unlock(&f->sem); + return ret; +} + +static int jffs2_write_begin(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, + struct folio **foliop, void **fsdata) +{ + struct folio *folio; + struct inode *inode = mapping->host; + struct jffs2_inode_info *f = JFFS2_INODE_INFO(inode); + struct jffs2_sb_info *c = JFFS2_SB_INFO(inode->i_sb); + pgoff_t index = pos >> PAGE_SHIFT; + int ret = 0; + + jffs2_dbg(1, "%s()\n", __func__); + + if (pos > inode->i_size) { + /* Make new hole frag from old EOF to new position */ + struct jffs2_raw_inode ri; + struct jffs2_full_dnode *fn; + uint32_t alloc_len; + + jffs2_dbg(1, "Writing new hole frag 0x%x-0x%x between current EOF and new position\n", + (unsigned int)inode->i_size, (uint32_t)pos); + + ret = jffs2_reserve_space(c, sizeof(ri), &alloc_len, + ALLOC_NORMAL, JFFS2_SUMMARY_INODE_SIZE); + if (ret) + goto out_err; + + mutex_lock(&f->sem); + memset(&ri, 0, sizeof(ri)); + + ri.magic = cpu_to_je16(JFFS2_MAGIC_BITMASK); + ri.nodetype = cpu_to_je16(JFFS2_NODETYPE_INODE); + ri.totlen = cpu_to_je32(sizeof(ri)); + ri.hdr_crc = cpu_to_je32(crc32(0, &ri, sizeof(struct jffs2_unknown_node)-4)); + + ri.ino = cpu_to_je32(f->inocache->ino); + ri.version = cpu_to_je32(++f->highest_version); + ri.mode = cpu_to_jemode(inode->i_mode); + ri.uid = cpu_to_je16(i_uid_read(inode)); + ri.gid = cpu_to_je16(i_gid_read(inode)); + ri.isize = cpu_to_je32((uint32_t)pos); + ri.atime = ri.ctime = ri.mtime = cpu_to_je32(JFFS2_NOW()); + ri.offset = cpu_to_je32(inode->i_size); + ri.dsize = cpu_to_je32((uint32_t)pos - inode->i_size); + ri.csize = cpu_to_je32(0); + ri.compr = JFFS2_COMPR_ZERO; + ri.node_crc = cpu_to_je32(crc32(0, &ri, sizeof(ri)-8)); + ri.data_crc = cpu_to_je32(0); + + fn = jffs2_write_dnode(c, f, &ri, NULL, 0, ALLOC_NORMAL); + + if (IS_ERR(fn)) { + ret = PTR_ERR(fn); + jffs2_complete_reservation(c); + mutex_unlock(&f->sem); + goto out_err; + } + ret = jffs2_add_full_dnode_to_inode(c, f, fn); + if (f->metadata) { + jffs2_mark_node_obsolete(c, f->metadata->raw); + jffs2_free_full_dnode(f->metadata); + f->metadata = NULL; + } + if (ret) { + jffs2_dbg(1, "Eep. add_full_dnode_to_inode() failed in write_begin, returned %d\n", + ret); + jffs2_mark_node_obsolete(c, fn->raw); + jffs2_free_full_dnode(fn); + jffs2_complete_reservation(c); + mutex_unlock(&f->sem); + goto out_err; + } + jffs2_complete_reservation(c); + inode->i_size = pos; + mutex_unlock(&f->sem); + } + + /* + * While getting a page and reading data in, lock c->alloc_sem until + * the page is Uptodate. Otherwise GC task may attempt to read the same + * page in read_cache_page(), which causes a deadlock. + */ + mutex_lock(&c->alloc_sem); + folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN, + mapping_gfp_mask(mapping)); + if (IS_ERR(folio)) { + ret = PTR_ERR(folio); + goto release_sem; + } + *foliop = folio; + + /* + * Read in the folio if it wasn't already present. Cannot optimize away + * the whole folio write case until jffs2_write_end can handle the + * case of a short-copy. + */ + if (!folio_test_uptodate(folio)) { + mutex_lock(&f->sem); + ret = jffs2_do_readpage_nolock(inode, folio); + mutex_unlock(&f->sem); + if (ret) { + folio_unlock(folio); + folio_put(folio); + goto release_sem; + } + } + jffs2_dbg(1, "end write_begin(). folio->flags %lx\n", folio->flags.f); + +release_sem: + mutex_unlock(&c->alloc_sem); +out_err: + return ret; +} + +static int jffs2_write_end(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct folio *folio, void *fsdata) +{ + /* Actually commit the write from the page cache page we're looking at. + * For now, we write the full page out each time. It sucks, but it's simple + */ + struct inode *inode = mapping->host; + struct jffs2_inode_info *f = JFFS2_INODE_INFO(inode); + struct jffs2_sb_info *c = JFFS2_SB_INFO(inode->i_sb); + struct jffs2_raw_inode *ri; + unsigned start = pos & (PAGE_SIZE - 1); + unsigned end = start + copied; + unsigned aligned_start = start & ~3; + int ret = 0; + uint32_t writtenlen = 0; + void *buf; + + jffs2_dbg(1, "%s(): ino #%llu, page at 0x%llx, range %d-%d, flags %lx\n", + __func__, inode->i_ino, folio_pos(folio), + start, end, folio->flags.f); + + /* We need to avoid deadlock with page_cache_read() in + jffs2_garbage_collect_pass(). So the folio must be + up to date to prevent page_cache_read() from trying + to re-lock it. */ + BUG_ON(!folio_test_uptodate(folio)); + + if (end == PAGE_SIZE) { + /* When writing out the end of a page, write out the + _whole_ page. This helps to reduce the number of + nodes in files which have many short writes, like + syslog files. */ + aligned_start = 0; + } + + ri = jffs2_alloc_raw_inode(); + + if (!ri) { + jffs2_dbg(1, "%s(): Allocation of raw inode failed\n", + __func__); + folio_unlock(folio); + folio_put(folio); + return -ENOMEM; + } + + /* Set the fields that the generic jffs2_write_inode_range() code can't find */ + ri->ino = cpu_to_je32(inode->i_ino); + ri->mode = cpu_to_jemode(inode->i_mode); + ri->uid = cpu_to_je16(i_uid_read(inode)); + ri->gid = cpu_to_je16(i_gid_read(inode)); + ri->isize = cpu_to_je32((uint32_t)inode->i_size); + ri->atime = ri->ctime = ri->mtime = cpu_to_je32(JFFS2_NOW()); + + buf = kmap_local_folio(folio, aligned_start); + ret = jffs2_write_inode_range(c, f, ri, buf, + folio_pos(folio) + aligned_start, + end - aligned_start, &writtenlen); + kunmap_local(buf); + + if (ret) + mapping_set_error(mapping, ret); + + /* Adjust writtenlen for the padding we did, so we don't confuse our caller */ + writtenlen -= min(writtenlen, (start - aligned_start)); + + if (writtenlen) { + if (inode->i_size < pos + writtenlen) { + inode->i_size = pos + writtenlen; + inode->i_blocks = (inode->i_size + 511) >> 9; + + inode_set_mtime_to_ts(inode, + inode_set_ctime_to_ts(inode, ITIME(je32_to_cpu(ri->ctime)))); + } + } + + jffs2_free_raw_inode(ri); + + if (start+writtenlen < end) { + /* generic_file_write has written more to the page cache than we've + actually written to the medium. Mark the page !Uptodate so that + it gets reread */ + jffs2_dbg(1, "%s(): Not all bytes written. Marking page !uptodate\n", + __func__); + folio_clear_uptodate(folio); + } + + jffs2_dbg(1, "%s() returning %d\n", + __func__, writtenlen > 0 ? writtenlen : ret); + folio_unlock(folio); + folio_put(folio); + return writtenlen > 0 ? writtenlen : ret; +} diff --git a/fs/jffs2/fs.c b/fs/jffs2/fs.c new file mode 100644 index 000000000..6ada8369a --- /dev/null +++ b/fs/jffs2/fs.c @@ -0,0 +1,742 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * Copyright © 2004-2010 David Woodhouse + * + * Created by David Woodhouse + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "nodelist.h" + +static int jffs2_flash_setup(struct jffs2_sb_info *c); + +int jffs2_do_setattr (struct inode *inode, struct iattr *iattr) +{ + struct jffs2_full_dnode *old_metadata, *new_metadata; + struct jffs2_inode_info *f = JFFS2_INODE_INFO(inode); + struct jffs2_sb_info *c = JFFS2_SB_INFO(inode->i_sb); + struct jffs2_raw_inode *ri; + union jffs2_device_node dev; + unsigned char *mdata = NULL; + int mdatalen = 0; + unsigned int ivalid; + uint32_t alloclen; + int ret; + int alloc_type = ALLOC_NORMAL; + + jffs2_dbg(1, "%s(): ino #%llu\n", __func__, inode->i_ino); + + /* Special cases - we don't want more than one data node + for these types on the medium at any time. So setattr + must read the original data associated with the node + (i.e. the device numbers or the target name) and write + it out again with the appropriate data attached */ + if (S_ISBLK(inode->i_mode) || S_ISCHR(inode->i_mode)) { + /* For these, we don't actually need to read the old node */ + mdatalen = jffs2_encode_dev(&dev, inode->i_rdev); + mdata = (char *)&dev; + jffs2_dbg(1, "%s(): Writing %d bytes of kdev_t\n", + __func__, mdatalen); + } else if (S_ISLNK(inode->i_mode)) { + mutex_lock(&f->sem); + mdatalen = f->metadata->size; + mdata = kmalloc(f->metadata->size, GFP_USER); + if (!mdata) { + mutex_unlock(&f->sem); + return -ENOMEM; + } + ret = jffs2_read_dnode(c, f, f->metadata, mdata, 0, mdatalen); + if (ret) { + mutex_unlock(&f->sem); + kfree(mdata); + return ret; + } + mutex_unlock(&f->sem); + jffs2_dbg(1, "%s(): Writing %d bytes of symlink target\n", + __func__, mdatalen); + } + + ri = jffs2_alloc_raw_inode(); + if (!ri) { + if (S_ISLNK(inode->i_mode)) + kfree(mdata); + return -ENOMEM; + } + + ret = jffs2_reserve_space(c, sizeof(*ri) + mdatalen, &alloclen, + ALLOC_NORMAL, JFFS2_SUMMARY_INODE_SIZE); + if (ret) { + jffs2_free_raw_inode(ri); + if (S_ISLNK(inode->i_mode)) + kfree(mdata); + return ret; + } + mutex_lock(&f->sem); + ivalid = iattr->ia_valid; + + ri->magic = cpu_to_je16(JFFS2_MAGIC_BITMASK); + ri->nodetype = cpu_to_je16(JFFS2_NODETYPE_INODE); + ri->totlen = cpu_to_je32(sizeof(*ri) + mdatalen); + ri->hdr_crc = cpu_to_je32(crc32(0, ri, sizeof(struct jffs2_unknown_node)-4)); + + ri->ino = cpu_to_je32(inode->i_ino); + ri->version = cpu_to_je32(++f->highest_version); + + ri->uid = cpu_to_je16((ivalid & ATTR_UID)? + from_kuid(&init_user_ns, iattr->ia_uid):i_uid_read(inode)); + ri->gid = cpu_to_je16((ivalid & ATTR_GID)? + from_kgid(&init_user_ns, iattr->ia_gid):i_gid_read(inode)); + + if (ivalid & ATTR_MODE) + ri->mode = cpu_to_jemode(iattr->ia_mode); + else + ri->mode = cpu_to_jemode(inode->i_mode); + + + ri->isize = cpu_to_je32((ivalid & ATTR_SIZE)?iattr->ia_size:inode->i_size); + ri->atime = cpu_to_je32(I_SEC((ivalid & ATTR_ATIME)?iattr->ia_atime:inode_get_atime(inode))); + ri->mtime = cpu_to_je32(I_SEC((ivalid & ATTR_MTIME)?iattr->ia_mtime:inode_get_mtime(inode))); + ri->ctime = cpu_to_je32(I_SEC((ivalid & ATTR_CTIME)?iattr->ia_ctime:inode_get_ctime(inode))); + + ri->offset = cpu_to_je32(0); + ri->csize = ri->dsize = cpu_to_je32(mdatalen); + ri->compr = JFFS2_COMPR_NONE; + if (ivalid & ATTR_SIZE && inode->i_size < iattr->ia_size) { + /* It's an extension. Make it a hole node */ + ri->compr = JFFS2_COMPR_ZERO; + ri->dsize = cpu_to_je32(iattr->ia_size - inode->i_size); + ri->offset = cpu_to_je32(inode->i_size); + } else if (ivalid & ATTR_SIZE && !iattr->ia_size) { + /* For truncate-to-zero, treat it as deletion because + it'll always be obsoleting all previous nodes */ + alloc_type = ALLOC_DELETION; + } + ri->node_crc = cpu_to_je32(crc32(0, ri, sizeof(*ri)-8)); + if (mdatalen) + ri->data_crc = cpu_to_je32(crc32(0, mdata, mdatalen)); + else + ri->data_crc = cpu_to_je32(0); + + new_metadata = jffs2_write_dnode(c, f, ri, mdata, mdatalen, alloc_type); + if (S_ISLNK(inode->i_mode)) + kfree(mdata); + + if (IS_ERR(new_metadata)) { + jffs2_complete_reservation(c); + jffs2_free_raw_inode(ri); + mutex_unlock(&f->sem); + return PTR_ERR(new_metadata); + } + /* It worked. Update the inode */ + inode_set_atime_to_ts(inode, ITIME(je32_to_cpu(ri->atime))); + inode_set_ctime_to_ts(inode, ITIME(je32_to_cpu(ri->ctime))); + inode_set_mtime_to_ts(inode, ITIME(je32_to_cpu(ri->mtime))); + inode->i_mode = jemode_to_cpu(ri->mode); + i_uid_write(inode, je16_to_cpu(ri->uid)); + i_gid_write(inode, je16_to_cpu(ri->gid)); + + + old_metadata = f->metadata; + + if (ivalid & ATTR_SIZE && inode->i_size > iattr->ia_size) + jffs2_truncate_fragtree (c, &f->fragtree, iattr->ia_size); + + if (ivalid & ATTR_SIZE && inode->i_size < iattr->ia_size) { + jffs2_add_full_dnode_to_inode(c, f, new_metadata); + inode->i_size = iattr->ia_size; + inode->i_blocks = (inode->i_size + 511) >> 9; + f->metadata = NULL; + } else { + f->metadata = new_metadata; + } + if (old_metadata) { + jffs2_mark_node_obsolete(c, old_metadata->raw); + jffs2_free_full_dnode(old_metadata); + } + jffs2_free_raw_inode(ri); + + mutex_unlock(&f->sem); + jffs2_complete_reservation(c); + + /* We have to do the truncate_setsize() without f->sem held, since + some pages may be locked and waiting for it in read_folio(). + We are protected from a simultaneous write() extending i_size + back past iattr->ia_size, because do_truncate() holds the + generic inode semaphore. */ + if (ivalid & ATTR_SIZE && inode->i_size > iattr->ia_size) { + truncate_setsize(inode, iattr->ia_size); + inode->i_blocks = (inode->i_size + 511) >> 9; + } + + return 0; +} + +int jffs2_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *iattr) +{ + struct inode *inode = d_inode(dentry); + int rc; + + rc = setattr_prepare(&nop_mnt_idmap, dentry, iattr); + if (rc) + return rc; + + rc = jffs2_do_setattr(inode, iattr); + if (!rc && (iattr->ia_valid & ATTR_MODE)) + rc = posix_acl_chmod(&nop_mnt_idmap, dentry, inode->i_mode); + + return rc; +} + +int jffs2_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct jffs2_sb_info *c = JFFS2_SB_INFO(dentry->d_sb); + unsigned long avail; + + buf->f_type = JFFS2_SUPER_MAGIC; + buf->f_bsize = 1 << PAGE_SHIFT; + buf->f_blocks = c->flash_size >> PAGE_SHIFT; + buf->f_files = 0; + buf->f_ffree = 0; + buf->f_namelen = JFFS2_MAX_NAME_LEN; + buf->f_fsid.val[0] = JFFS2_SUPER_MAGIC; + buf->f_fsid.val[1] = c->mtd->index; + + spin_lock(&c->erase_completion_lock); + avail = c->dirty_size + c->free_size; + if (avail > c->sector_size * c->resv_blocks_write) + avail -= c->sector_size * c->resv_blocks_write; + else + avail = 0; + spin_unlock(&c->erase_completion_lock); + + buf->f_bavail = buf->f_bfree = avail >> PAGE_SHIFT; + + return 0; +} + + +void jffs2_evict_inode (struct inode *inode) +{ + /* We can forget about this inode for now - drop all + * the nodelists associated with it, etc. + */ + struct jffs2_sb_info *c = JFFS2_SB_INFO(inode->i_sb); + struct jffs2_inode_info *f = JFFS2_INODE_INFO(inode); + + jffs2_dbg(1, "%s(): ino #%llu mode %o\n", + __func__, inode->i_ino, inode->i_mode); + truncate_inode_pages_final(&inode->i_data); + clear_inode(inode); + jffs2_do_clear_inode(c, f); +} + +struct inode *jffs2_iget(struct super_block *sb, unsigned long ino) +{ + struct jffs2_inode_info *f; + struct jffs2_sb_info *c; + struct jffs2_raw_inode latest_node; + union jffs2_device_node jdev; + struct inode *inode; + dev_t rdev = 0; + int ret; + + jffs2_dbg(1, "%s(): ino == %lu\n", __func__, ino); + + inode = iget_locked(sb, ino); + if (!inode) + return ERR_PTR(-ENOMEM); + if (!(inode_state_read_once(inode) & I_NEW)) + return inode; + + f = JFFS2_INODE_INFO(inode); + c = JFFS2_SB_INFO(inode->i_sb); + + jffs2_init_inode_info(f); + mutex_lock(&f->sem); + + ret = jffs2_do_read_inode(c, f, inode->i_ino, &latest_node); + if (ret) + goto error; + + inode->i_mode = jemode_to_cpu(latest_node.mode); + i_uid_write(inode, je16_to_cpu(latest_node.uid)); + i_gid_write(inode, je16_to_cpu(latest_node.gid)); + inode->i_size = je32_to_cpu(latest_node.isize); + inode_set_atime_to_ts(inode, ITIME(je32_to_cpu(latest_node.atime))); + inode_set_mtime_to_ts(inode, ITIME(je32_to_cpu(latest_node.mtime))); + inode_set_ctime_to_ts(inode, ITIME(je32_to_cpu(latest_node.ctime))); + + set_nlink(inode, f->inocache->pino_nlink); + + inode->i_blocks = (inode->i_size + 511) >> 9; + + switch (inode->i_mode & S_IFMT) { + + case S_IFLNK: + inode->i_op = &jffs2_symlink_inode_operations; + inode->i_link = f->target; + break; + + case S_IFDIR: + { + struct jffs2_full_dirent *fd; + set_nlink(inode, 2); /* parent and '.' */ + + for (fd=f->dents; fd; fd = fd->next) { + if (fd->type == DT_DIR && fd->ino) + inc_nlink(inode); + } + /* Root dir gets i_nlink 3 for some reason */ + if (inode->i_ino == 1) + inc_nlink(inode); + + inode->i_op = &jffs2_dir_inode_operations; + inode->i_fop = &jffs2_dir_operations; + break; + } + case S_IFREG: + inode->i_op = &jffs2_file_inode_operations; + inode->i_fop = &jffs2_file_operations; + inode->i_mapping->a_ops = &jffs2_file_address_operations; + inode->i_mapping->nrpages = 0; + break; + + case S_IFBLK: + case S_IFCHR: + /* Read the device numbers from the media */ + if (f->metadata->size != sizeof(jdev.old_id) && + f->metadata->size != sizeof(jdev.new_id)) { + pr_notice("Device node has strange size %d\n", + f->metadata->size); + goto error_io; + } + jffs2_dbg(1, "Reading device numbers from flash\n"); + ret = jffs2_read_dnode(c, f, f->metadata, (char *)&jdev, 0, f->metadata->size); + if (ret < 0) { + /* Eep */ + pr_notice("Read device numbers for inode %llu failed\n", + inode->i_ino); + goto error; + } + if (f->metadata->size == sizeof(jdev.old_id)) + rdev = old_decode_dev(je16_to_cpu(jdev.old_id)); + else + rdev = new_decode_dev(je32_to_cpu(jdev.new_id)); + fallthrough; + + case S_IFSOCK: + case S_IFIFO: + inode->i_op = &jffs2_file_inode_operations; + init_special_inode(inode, inode->i_mode, rdev); + break; + + default: + pr_warn("%s(): Bogus i_mode %o for ino %llu\n", + __func__, inode->i_mode, inode->i_ino); + } + + mutex_unlock(&f->sem); + + jffs2_dbg(1, "jffs2_read_inode() returning\n"); + unlock_new_inode(inode); + return inode; + +error_io: + ret = -EIO; +error: + mutex_unlock(&f->sem); + iget_failed(inode); + return ERR_PTR(ret); +} + +void jffs2_dirty_inode(struct inode *inode, int flags) +{ + struct iattr iattr; + + if (!(inode_state_read_once(inode) & I_DIRTY_DATASYNC)) { + jffs2_dbg(2, "%s(): not calling setattr() for ino #%llu\n", + __func__, inode->i_ino); + return; + } + + jffs2_dbg(1, "%s(): calling setattr() for ino #%llu\n", + __func__, inode->i_ino); + + iattr.ia_valid = ATTR_MODE|ATTR_UID|ATTR_GID|ATTR_ATIME|ATTR_MTIME|ATTR_CTIME; + iattr.ia_mode = inode->i_mode; + iattr.ia_uid = inode->i_uid; + iattr.ia_gid = inode->i_gid; + iattr.ia_atime = inode_get_atime(inode); + iattr.ia_mtime = inode_get_mtime(inode); + iattr.ia_ctime = inode_get_ctime(inode); + + jffs2_do_setattr(inode, &iattr); +} + +int jffs2_do_remount_fs(struct super_block *sb, struct fs_context *fc) +{ + struct jffs2_sb_info *c = JFFS2_SB_INFO(sb); + + if (c->flags & JFFS2_SB_FLAG_RO && !sb_rdonly(sb)) + return -EROFS; + + /* We stop if it was running, then restart if it needs to. + This also catches the case where it was stopped and this + is just a remount to restart it. + Flush the writebuffer, if necessary, else we loose it */ + if (!sb_rdonly(sb)) { + jffs2_stop_garbage_collect_thread(c); + mutex_lock(&c->alloc_sem); + jffs2_flush_wbuf_pad(c); + mutex_unlock(&c->alloc_sem); + } + + if (!(fc->sb_flags & SB_RDONLY)) + jffs2_start_garbage_collect_thread(c); + + fc->sb_flags |= SB_NOATIME; + return 0; +} + +/* jffs2_new_inode: allocate a new inode and inocache, add it to the hash, + fill in the raw_inode while you're at it. */ +struct inode *jffs2_new_inode (struct inode *dir_i, umode_t mode, struct jffs2_raw_inode *ri) +{ + struct inode *inode; + struct super_block *sb = dir_i->i_sb; + struct jffs2_sb_info *c; + struct jffs2_inode_info *f; + int ret; + + jffs2_dbg(1, "%s(): dir_i %llu, mode 0x%x\n", + __func__, dir_i->i_ino, mode); + + c = JFFS2_SB_INFO(sb); + + inode = new_inode(sb); + + if (!inode) + return ERR_PTR(-ENOMEM); + + f = JFFS2_INODE_INFO(inode); + jffs2_init_inode_info(f); + mutex_lock(&f->sem); + + memset(ri, 0, sizeof(*ri)); + /* Set OS-specific defaults for new inodes */ + ri->uid = cpu_to_je16(from_kuid(&init_user_ns, current_fsuid())); + + if (dir_i->i_mode & S_ISGID) { + ri->gid = cpu_to_je16(i_gid_read(dir_i)); + if (S_ISDIR(mode)) + mode |= S_ISGID; + } else { + ri->gid = cpu_to_je16(from_kgid(&init_user_ns, current_fsgid())); + } + + /* POSIX ACLs have to be processed now, at least partly. + The umask is only applied if there's no default ACL */ + ret = jffs2_init_acl_pre(dir_i, inode, &mode); + if (ret) { + mutex_unlock(&f->sem); + make_bad_inode(inode); + iput(inode); + return ERR_PTR(ret); + } + ret = jffs2_do_new_inode (c, f, mode, ri); + if (ret) { + mutex_unlock(&f->sem); + make_bad_inode(inode); + iput(inode); + return ERR_PTR(ret); + } + set_nlink(inode, 1); + inode->i_ino = je32_to_cpu(ri->ino); + inode->i_mode = jemode_to_cpu(ri->mode); + i_gid_write(inode, je16_to_cpu(ri->gid)); + i_uid_write(inode, je16_to_cpu(ri->uid)); + simple_inode_init_ts(inode); + ri->atime = ri->mtime = ri->ctime = cpu_to_je32(I_SEC(inode_get_mtime(inode))); + + inode->i_blocks = 0; + inode->i_size = 0; + + if (insert_inode_locked(inode) < 0) { + mutex_unlock(&f->sem); + make_bad_inode(inode); + iput(inode); + return ERR_PTR(-EINVAL); + } + + return inode; +} + +static int calculate_inocache_hashsize(uint32_t flash_size) +{ + /* + * Pick a inocache hash size based on the size of the medium. + * Count how many megabytes we're dealing with, apply a hashsize twice + * that size, but rounding down to the usual big powers of 2. And keep + * to sensible bounds. + */ + + int size_mb = flash_size / 1024 / 1024; + int hashsize = (size_mb * 2) & ~0x3f; + + if (hashsize < INOCACHE_HASHSIZE_MIN) + return INOCACHE_HASHSIZE_MIN; + if (hashsize > INOCACHE_HASHSIZE_MAX) + return INOCACHE_HASHSIZE_MAX; + + return hashsize; +} + +int jffs2_do_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct jffs2_sb_info *c; + struct inode *root_i; + int ret; + size_t blocks; + + c = JFFS2_SB_INFO(sb); + + /* Do not support the MLC nand */ + if (c->mtd->type == MTD_MLCNANDFLASH) + return -EINVAL; + +#ifndef CONFIG_JFFS2_FS_WRITEBUFFER + if (c->mtd->type == MTD_NANDFLASH) { + errorf(fc, "Cannot operate on NAND flash unless jffs2 NAND support is compiled in"); + return -EINVAL; + } + if (c->mtd->type == MTD_DATAFLASH) { + errorf(fc, "Cannot operate on DataFlash unless jffs2 DataFlash support is compiled in"); + return -EINVAL; + } +#endif + + c->flash_size = c->mtd->size; + c->sector_size = c->mtd->erasesize; + blocks = c->flash_size / c->sector_size; + + /* + * Size alignment check + */ + if ((c->sector_size * blocks) != c->flash_size) { + c->flash_size = c->sector_size * blocks; + infof(fc, "Flash size not aligned to erasesize, reducing to %dKiB", + c->flash_size / 1024); + } + + if (c->flash_size < 5*c->sector_size) { + errorf(fc, "Too few erase blocks (%d)", + c->flash_size / c->sector_size); + return -EINVAL; + } + + c->cleanmarker_size = sizeof(struct jffs2_unknown_node); + + /* NAND (or other bizarre) flash... do setup accordingly */ + ret = jffs2_flash_setup(c); + if (ret) + return ret; + + c->inocache_hashsize = calculate_inocache_hashsize(c->flash_size); + c->inocache_list = kzalloc_objs(struct jffs2_inode_cache *, + c->inocache_hashsize); + if (!c->inocache_list) { + ret = -ENOMEM; + goto out_wbuf; + } + + jffs2_init_xattr_subsystem(c); + + if ((ret = jffs2_do_mount_fs(c))) + goto out_inohash; + + jffs2_dbg(1, "%s(): Getting root inode\n", __func__); + root_i = jffs2_iget(sb, 1); + if (IS_ERR(root_i)) { + jffs2_dbg(1, "get root inode failed\n"); + ret = PTR_ERR(root_i); + goto out_root; + } + + ret = -ENOMEM; + + jffs2_dbg(1, "%s(): d_make_root()\n", __func__); + sb->s_root = d_make_root(root_i); + if (!sb->s_root) + goto out_root; + + sb->s_maxbytes = 0xFFFFFFFF; + sb->s_blocksize = PAGE_SIZE; + sb->s_blocksize_bits = PAGE_SHIFT; + sb->s_magic = JFFS2_SUPER_MAGIC; + sb->s_time_min = 0; + sb->s_time_max = U32_MAX; + + if (!sb_rdonly(sb)) + jffs2_start_garbage_collect_thread(c); + return 0; + +out_root: + jffs2_free_ino_caches(c); + jffs2_free_raw_node_refs(c); + kvfree(c->blocks); + jffs2_clear_xattr_subsystem(c); + jffs2_sum_exit(c); + out_inohash: + kfree(c->inocache_list); + out_wbuf: + jffs2_flash_cleanup(c); + + return ret; +} + +void jffs2_gc_release_inode(struct jffs2_sb_info *c, + struct jffs2_inode_info *f) +{ + iput(OFNI_EDONI_2SFFJ(f)); +} + +struct jffs2_inode_info *jffs2_gc_fetch_inode(struct jffs2_sb_info *c, + int inum, int unlinked) +{ + struct inode *inode; + struct jffs2_inode_cache *ic; + + if (unlinked) { + /* The inode has zero nlink but its nodes weren't yet marked + obsolete. This has to be because we're still waiting for + the final (close() and) iput() to happen. + + There's a possibility that the final iput() could have + happened while we were contemplating. In order to ensure + that we don't cause a new read_inode() (which would fail) + for the inode in question, we use ilookup() in this case + instead of iget(). + + The nlink can't _become_ zero at this point because we're + holding the alloc_sem, and jffs2_do_unlink() would also + need that while decrementing nlink on any inode. + */ + inode = ilookup(OFNI_BS_2SFFJ(c), inum); + if (!inode) { + jffs2_dbg(1, "ilookup() failed for ino #%u; inode is probably deleted.\n", + inum); + + spin_lock(&c->inocache_lock); + ic = jffs2_get_ino_cache(c, inum); + if (!ic) { + jffs2_dbg(1, "Inode cache for ino #%u is gone\n", + inum); + spin_unlock(&c->inocache_lock); + return NULL; + } + if (ic->state != INO_STATE_CHECKEDABSENT) { + /* Wait for progress. Don't just loop */ + jffs2_dbg(1, "Waiting for ino #%u in state %d\n", + ic->ino, ic->state); + sleep_on_spinunlock(&c->inocache_wq, &c->inocache_lock); + } else { + spin_unlock(&c->inocache_lock); + } + + return NULL; + } + } else { + /* Inode has links to it still; they're not going away because + jffs2_do_unlink() would need the alloc_sem and we have it. + Just iget() it, and if read_inode() is necessary that's OK. + */ + inode = jffs2_iget(OFNI_BS_2SFFJ(c), inum); + if (IS_ERR(inode)) + return ERR_CAST(inode); + } + if (is_bad_inode(inode)) { + pr_notice("Eep. read_inode() failed for ino #%u. unlinked %d\n", + inum, unlinked); + /* NB. This will happen again. We need to do something appropriate here. */ + iput(inode); + return ERR_PTR(-EIO); + } + + return JFFS2_INODE_INFO(inode); +} + +static int jffs2_flash_setup(struct jffs2_sb_info *c) { + int ret = 0; + + if (jffs2_cleanmarker_oob(c)) { + /* NAND flash... do setup accordingly */ + ret = jffs2_nand_flash_setup(c); + if (ret) + return ret; + } + + /* and Dataflash */ + if (jffs2_dataflash(c)) { + ret = jffs2_dataflash_setup(c); + if (ret) + return ret; + } + + /* and Intel "Sibley" flash */ + if (jffs2_nor_wbuf_flash(c)) { + ret = jffs2_nor_wbuf_flash_setup(c); + if (ret) + return ret; + } + + /* and an UBI volume */ + if (jffs2_ubivol(c)) { + ret = jffs2_ubivol_setup(c); + if (ret) + return ret; + } + + return ret; +} + +void jffs2_flash_cleanup(struct jffs2_sb_info *c) { + + if (jffs2_cleanmarker_oob(c)) { + jffs2_nand_flash_cleanup(c); + } + + /* and DataFlash */ + if (jffs2_dataflash(c)) { + jffs2_dataflash_cleanup(c); + } + + /* and Intel "Sibley" flash */ + if (jffs2_nor_wbuf_flash(c)) { + jffs2_nor_wbuf_flash_cleanup(c); + } + + /* and an UBI volume */ + if (jffs2_ubivol(c)) { + jffs2_ubivol_cleanup(c); + } +} diff --git a/fs/jffs2/gc.c b/fs/jffs2/gc.c new file mode 100644 index 000000000..1b833bbff --- /dev/null +++ b/fs/jffs2/gc.c @@ -0,0 +1,1405 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * Copyright © 2004-2010 David Woodhouse + * + * Created by David Woodhouse + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include "nodelist.h" +#include "compr.h" + +static int jffs2_garbage_collect_pristine(struct jffs2_sb_info *c, + struct jffs2_inode_cache *ic, + struct jffs2_raw_node_ref *raw); +static int jffs2_garbage_collect_metadata(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, + struct jffs2_inode_info *f, struct jffs2_full_dnode *fd); +static int jffs2_garbage_collect_dirent(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, + struct jffs2_inode_info *f, struct jffs2_full_dirent *fd); +static int jffs2_garbage_collect_deletion_dirent(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, + struct jffs2_inode_info *f, struct jffs2_full_dirent *fd); +static int jffs2_garbage_collect_hole(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, + struct jffs2_inode_info *f, struct jffs2_full_dnode *fn, + uint32_t start, uint32_t end); +static int jffs2_garbage_collect_dnode(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, + struct jffs2_inode_info *f, struct jffs2_full_dnode *fn, + uint32_t start, uint32_t end); +static int jffs2_garbage_collect_live(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, + struct jffs2_raw_node_ref *raw, struct jffs2_inode_info *f); + +/* Called with erase_completion_lock held */ +static struct jffs2_eraseblock *jffs2_find_gc_block(struct jffs2_sb_info *c) +{ + struct jffs2_eraseblock *ret; + struct list_head *nextlist = NULL; + int n = jiffies % 128; + + /* Pick an eraseblock to garbage collect next. This is where we'll + put the clever wear-levelling algorithms. Eventually. */ + /* We possibly want to favour the dirtier blocks more when the + number of free blocks is low. */ +again: + if (!list_empty(&c->bad_used_list) && c->nr_free_blocks > c->resv_blocks_gcbad) { + jffs2_dbg(1, "Picking block from bad_used_list to GC next\n"); + nextlist = &c->bad_used_list; + } else if (n < 50 && !list_empty(&c->erasable_list)) { + /* Note that most of them will have gone directly to be erased. + So don't favour the erasable_list _too_ much. */ + jffs2_dbg(1, "Picking block from erasable_list to GC next\n"); + nextlist = &c->erasable_list; + } else if (n < 110 && !list_empty(&c->very_dirty_list)) { + /* Most of the time, pick one off the very_dirty list */ + jffs2_dbg(1, "Picking block from very_dirty_list to GC next\n"); + nextlist = &c->very_dirty_list; + } else if (n < 126 && !list_empty(&c->dirty_list)) { + jffs2_dbg(1, "Picking block from dirty_list to GC next\n"); + nextlist = &c->dirty_list; + } else if (!list_empty(&c->clean_list)) { + jffs2_dbg(1, "Picking block from clean_list to GC next\n"); + nextlist = &c->clean_list; + } else if (!list_empty(&c->dirty_list)) { + jffs2_dbg(1, "Picking block from dirty_list to GC next (clean_list was empty)\n"); + + nextlist = &c->dirty_list; + } else if (!list_empty(&c->very_dirty_list)) { + jffs2_dbg(1, "Picking block from very_dirty_list to GC next (clean_list and dirty_list were empty)\n"); + nextlist = &c->very_dirty_list; + } else if (!list_empty(&c->erasable_list)) { + jffs2_dbg(1, "Picking block from erasable_list to GC next (clean_list and {very_,}dirty_list were empty)\n"); + + nextlist = &c->erasable_list; + } else if (!list_empty(&c->erasable_pending_wbuf_list)) { + /* There are blocks are waiting for the wbuf sync */ + jffs2_dbg(1, "Synching wbuf in order to reuse erasable_pending_wbuf_list blocks\n"); + spin_unlock(&c->erase_completion_lock); + jffs2_flush_wbuf_pad(c); + spin_lock(&c->erase_completion_lock); + goto again; + } else { + /* Eep. All were empty */ + jffs2_dbg(1, "No clean, dirty _or_ erasable blocks to GC from! Where are they all?\n"); + return NULL; + } + + ret = list_entry(nextlist->next, struct jffs2_eraseblock, list); + list_del(&ret->list); + c->gcblock = ret; + ret->gc_node = ret->first_node; + if (!ret->gc_node) { + pr_warn("Eep. ret->gc_node for block at 0x%08x is NULL\n", + ret->offset); + BUG(); + } + + /* Have we accidentally picked a clean block with wasted space ? */ + if (ret->wasted_size) { + jffs2_dbg(1, "Converting wasted_size %08x to dirty_size\n", + ret->wasted_size); + ret->dirty_size += ret->wasted_size; + c->wasted_size -= ret->wasted_size; + c->dirty_size += ret->wasted_size; + ret->wasted_size = 0; + } + + return ret; +} + +/* jffs2_garbage_collect_pass + * Make a single attempt to progress GC. Move one node, and possibly + * start erasing one eraseblock. + */ +int jffs2_garbage_collect_pass(struct jffs2_sb_info *c) +{ + struct jffs2_inode_info *f; + struct jffs2_inode_cache *ic; + struct jffs2_eraseblock *jeb; + struct jffs2_raw_node_ref *raw; + uint32_t gcblock_dirty; + int ret = 0, inum, nlink; + int xattr = 0; + + if (mutex_lock_interruptible(&c->alloc_sem)) + return -EINTR; + + + for (;;) { + /* We can't start doing GC until we've finished checking + the node CRCs etc. */ + int bucket, want_ino; + + spin_lock(&c->erase_completion_lock); + if (!c->unchecked_size) + break; + spin_unlock(&c->erase_completion_lock); + + if (!xattr) + xattr = jffs2_verify_xattr(c); + + spin_lock(&c->inocache_lock); + /* Instead of doing the inodes in numeric order, doing a lookup + * in the hash for each possible number, just walk the hash + * buckets of *existing* inodes. This means that we process + * them out-of-order, but it can be a lot faster if there's + * a sparse inode# space. Which there often is. */ + want_ino = c->check_ino; + for (bucket = c->check_ino % c->inocache_hashsize ; bucket < c->inocache_hashsize; bucket++) { + for (ic = c->inocache_list[bucket]; ic; ic = ic->next) { + if (ic->ino < want_ino) + continue; + + if (ic->state != INO_STATE_CHECKEDABSENT && + ic->state != INO_STATE_PRESENT) + goto got_next; /* with inocache_lock held */ + + jffs2_dbg(1, "Skipping ino #%u already checked\n", + ic->ino); + } + want_ino = 0; + } + + /* Point c->check_ino past the end of the last bucket. */ + c->check_ino = ((c->highest_ino + c->inocache_hashsize + 1) & + ~c->inocache_hashsize) - 1; + + spin_unlock(&c->inocache_lock); + + pr_crit("Checked all inodes but still 0x%x bytes of unchecked space?\n", + c->unchecked_size); + jffs2_dbg_dump_block_lists_nolock(c); + mutex_unlock(&c->alloc_sem); + return -ENOSPC; + + got_next: + /* For next time round the loop, we want c->checked_ino to indicate + * the *next* one we want to check. And since we're walking the + * buckets rather than doing it sequentially, it's: */ + c->check_ino = ic->ino + c->inocache_hashsize; + + if (!ic->pino_nlink) { + jffs2_dbg(1, "Skipping check of ino #%d with nlink/pino zero\n", + ic->ino); + spin_unlock(&c->inocache_lock); + jffs2_xattr_delete_inode(c, ic); + continue; + } + switch(ic->state) { + case INO_STATE_CHECKEDABSENT: + case INO_STATE_PRESENT: + spin_unlock(&c->inocache_lock); + continue; + + case INO_STATE_GC: + case INO_STATE_CHECKING: + pr_warn("Inode #%u is in state %d during CRC check phase!\n", + ic->ino, ic->state); + spin_unlock(&c->inocache_lock); + BUG(); + + case INO_STATE_READING: + /* We need to wait for it to finish, lest we move on + and trigger the BUG() above while we haven't yet + finished checking all its nodes */ + jffs2_dbg(1, "Waiting for ino #%u to finish reading\n", + ic->ino); + /* We need to come back again for the _same_ inode. We've + made no progress in this case, but that should be OK */ + c->check_ino = ic->ino; + + mutex_unlock(&c->alloc_sem); + sleep_on_spinunlock(&c->inocache_wq, &c->inocache_lock); + return 0; + + default: + BUG(); + + case INO_STATE_UNCHECKED: + ; + } + ic->state = INO_STATE_CHECKING; + spin_unlock(&c->inocache_lock); + + jffs2_dbg(1, "%s(): triggering inode scan of ino#%u\n", + __func__, ic->ino); + + ret = jffs2_do_crccheck_inode(c, ic); + if (ret) + pr_warn("Returned error for crccheck of ino #%u. Expect badness...\n", + ic->ino); + + jffs2_set_inocache_state(c, ic, INO_STATE_CHECKEDABSENT); + mutex_unlock(&c->alloc_sem); + return ret; + } + + /* If there are any blocks which need erasing, erase them now */ + if (!list_empty(&c->erase_complete_list) || + !list_empty(&c->erase_pending_list)) { + spin_unlock(&c->erase_completion_lock); + mutex_unlock(&c->alloc_sem); + jffs2_dbg(1, "%s(): erasing pending blocks\n", __func__); + if (jffs2_erase_pending_blocks(c, 1)) + return 0; + + jffs2_dbg(1, "No progress from erasing block; doing GC anyway\n"); + mutex_lock(&c->alloc_sem); + spin_lock(&c->erase_completion_lock); + } + + /* First, work out which block we're garbage-collecting */ + jeb = c->gcblock; + + if (!jeb) + jeb = jffs2_find_gc_block(c); + + if (!jeb) { + /* Couldn't find a free block. But maybe we can just erase one and make 'progress'? */ + if (c->nr_erasing_blocks) { + spin_unlock(&c->erase_completion_lock); + mutex_unlock(&c->alloc_sem); + return -EAGAIN; + } + jffs2_dbg(1, "Couldn't find erase block to garbage collect!\n"); + spin_unlock(&c->erase_completion_lock); + mutex_unlock(&c->alloc_sem); + return -EIO; + } + + jffs2_dbg(1, "GC from block %08x, used_size %08x, dirty_size %08x, free_size %08x\n", + jeb->offset, jeb->used_size, jeb->dirty_size, jeb->free_size); + D1(if (c->nextblock) + printk(KERN_DEBUG "Nextblock at %08x, used_size %08x, dirty_size %08x, wasted_size %08x, free_size %08x\n", c->nextblock->offset, c->nextblock->used_size, c->nextblock->dirty_size, c->nextblock->wasted_size, c->nextblock->free_size)); + + if (!jeb->used_size) { + mutex_unlock(&c->alloc_sem); + goto eraseit; + } + + raw = jeb->gc_node; + gcblock_dirty = jeb->dirty_size; + + while(ref_obsolete(raw)) { + jffs2_dbg(1, "Node at 0x%08x is obsolete... skipping\n", + ref_offset(raw)); + raw = ref_next(raw); + if (unlikely(!raw)) { + pr_warn("eep. End of raw list while still supposedly nodes to GC\n"); + pr_warn("erase block at 0x%08x. free_size 0x%08x, dirty_size 0x%08x, used_size 0x%08x\n", + jeb->offset, jeb->free_size, + jeb->dirty_size, jeb->used_size); + jeb->gc_node = raw; + spin_unlock(&c->erase_completion_lock); + mutex_unlock(&c->alloc_sem); + BUG(); + } + } + jeb->gc_node = raw; + + jffs2_dbg(1, "Going to garbage collect node at 0x%08x\n", + ref_offset(raw)); + + if (!raw->next_in_ino) { + /* Inode-less node. Clean marker, snapshot or something like that */ + spin_unlock(&c->erase_completion_lock); + if (ref_flags(raw) == REF_PRISTINE) { + /* It's an unknown node with JFFS2_FEATURE_RWCOMPAT_COPY */ + jffs2_garbage_collect_pristine(c, NULL, raw); + } else { + /* Just mark it obsolete */ + jffs2_mark_node_obsolete(c, raw); + } + mutex_unlock(&c->alloc_sem); + goto eraseit_lock; + } + + ic = jffs2_raw_ref_to_ic(raw); + +#ifdef CONFIG_JFFS2_FS_XATTR + /* When 'ic' refers xattr_datum/xattr_ref, this node is GCed as xattr. + * We can decide whether this node is inode or xattr by ic->class. */ + if (ic->class == RAWNODE_CLASS_XATTR_DATUM + || ic->class == RAWNODE_CLASS_XATTR_REF) { + spin_unlock(&c->erase_completion_lock); + + if (ic->class == RAWNODE_CLASS_XATTR_DATUM) { + ret = jffs2_garbage_collect_xattr_datum(c, (struct jffs2_xattr_datum *)ic, raw); + } else { + ret = jffs2_garbage_collect_xattr_ref(c, (struct jffs2_xattr_ref *)ic, raw); + } + goto test_gcnode; + } +#endif + + /* We need to hold the inocache. Either the erase_completion_lock or + the inocache_lock are sufficient; we trade down since the inocache_lock + causes less contention. */ + spin_lock(&c->inocache_lock); + + spin_unlock(&c->erase_completion_lock); + + jffs2_dbg(1, "%s(): collecting from block @0x%08x. Node @0x%08x(%d), ino #%u\n", + __func__, jeb->offset, ref_offset(raw), ref_flags(raw), + ic->ino); + + /* Three possibilities: + 1. Inode is already in-core. We must iget it and do proper + updating to its fragtree, etc. + 2. Inode is not in-core, node is REF_PRISTINE. We lock the + inocache to prevent a read_inode(), copy the node intact. + 3. Inode is not in-core, node is not pristine. We must iget() + and take the slow path. + */ + + switch(ic->state) { + case INO_STATE_CHECKEDABSENT: + /* It's been checked, but it's not currently in-core. + We can just copy any pristine nodes, but have + to prevent anyone else from doing read_inode() while + we're at it, so we set the state accordingly */ + if (ref_flags(raw) == REF_PRISTINE) + ic->state = INO_STATE_GC; + else { + jffs2_dbg(1, "Ino #%u is absent but node not REF_PRISTINE. Reading.\n", + ic->ino); + } + break; + + case INO_STATE_PRESENT: + /* It's in-core. GC must iget() it. */ + break; + + case INO_STATE_UNCHECKED: + case INO_STATE_CHECKING: + case INO_STATE_GC: + /* Should never happen. We should have finished checking + by the time we actually start doing any GC, and since + we're holding the alloc_sem, no other garbage collection + can happen. + */ + pr_crit("Inode #%u already in state %d in jffs2_garbage_collect_pass()!\n", + ic->ino, ic->state); + mutex_unlock(&c->alloc_sem); + spin_unlock(&c->inocache_lock); + BUG(); + + case INO_STATE_READING: + /* Someone's currently trying to read it. We must wait for + them to finish and then go through the full iget() route + to do the GC. However, sometimes read_inode() needs to get + the alloc_sem() (for marking nodes invalid) so we must + drop the alloc_sem before sleeping. */ + + mutex_unlock(&c->alloc_sem); + jffs2_dbg(1, "%s(): waiting for ino #%u in state %d\n", + __func__, ic->ino, ic->state); + sleep_on_spinunlock(&c->inocache_wq, &c->inocache_lock); + /* And because we dropped the alloc_sem we must start again from the + beginning. Ponder chance of livelock here -- we're returning success + without actually making any progress. + + Q: What are the chances that the inode is back in INO_STATE_READING + again by the time we next enter this function? And that this happens + enough times to cause a real delay? + + A: Small enough that I don't care :) + */ + return 0; + } + + /* OK. Now if the inode is in state INO_STATE_GC, we are going to copy the + node intact, and we don't have to muck about with the fragtree etc. + because we know it's not in-core. If it _was_ in-core, we go through + all the iget() crap anyway */ + + if (ic->state == INO_STATE_GC) { + spin_unlock(&c->inocache_lock); + + ret = jffs2_garbage_collect_pristine(c, ic, raw); + + spin_lock(&c->inocache_lock); + ic->state = INO_STATE_CHECKEDABSENT; + wake_up(&c->inocache_wq); + + if (ret != -EBADFD) { + spin_unlock(&c->inocache_lock); + goto test_gcnode; + } + + /* Fall through if it wanted us to, with inocache_lock held */ + } + + /* Prevent the fairly unlikely race where the gcblock is + entirely obsoleted by the final close of a file which had + the only valid nodes in the block, followed by erasure, + followed by freeing of the ic because the erased block(s) + held _all_ the nodes of that inode.... never been seen but + it's vaguely possible. */ + + inum = ic->ino; + nlink = ic->pino_nlink; + spin_unlock(&c->inocache_lock); + + f = jffs2_gc_fetch_inode(c, inum, !nlink); + if (IS_ERR(f)) { + ret = PTR_ERR(f); + goto release_sem; + } + if (!f) { + ret = 0; + goto release_sem; + } + + ret = jffs2_garbage_collect_live(c, jeb, raw, f); + + jffs2_gc_release_inode(c, f); + + test_gcnode: + if (jeb->dirty_size == gcblock_dirty && !ref_obsolete(jeb->gc_node)) { + /* Eep. This really should never happen. GC is broken */ + pr_err("Error garbage collecting node at %08x!\n", + ref_offset(jeb->gc_node)); + ret = -ENOSPC; + } + release_sem: + mutex_unlock(&c->alloc_sem); + + eraseit_lock: + /* If we've finished this block, start it erasing */ + spin_lock(&c->erase_completion_lock); + + eraseit: + if (c->gcblock && !c->gcblock->used_size) { + jffs2_dbg(1, "Block at 0x%08x completely obsoleted by GC. Moving to erase_pending_list\n", + c->gcblock->offset); + /* We're GC'ing an empty block? */ + list_add_tail(&c->gcblock->list, &c->erase_pending_list); + c->gcblock = NULL; + c->nr_erasing_blocks++; + jffs2_garbage_collect_trigger(c); + } + spin_unlock(&c->erase_completion_lock); + + return ret; +} + +static int jffs2_garbage_collect_live(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, + struct jffs2_raw_node_ref *raw, struct jffs2_inode_info *f) +{ + struct jffs2_node_frag *frag; + struct jffs2_full_dnode *fn = NULL; + struct jffs2_full_dirent *fd; + uint32_t start = 0, end = 0, nrfrags = 0; + int ret = 0; + + mutex_lock(&f->sem); + + /* Now we have the lock for this inode. Check that it's still the one at the head + of the list. */ + + spin_lock(&c->erase_completion_lock); + + if (c->gcblock != jeb) { + spin_unlock(&c->erase_completion_lock); + jffs2_dbg(1, "GC block is no longer gcblock. Restart\n"); + goto upnout; + } + if (ref_obsolete(raw)) { + spin_unlock(&c->erase_completion_lock); + jffs2_dbg(1, "node to be GC'd was obsoleted in the meantime.\n"); + /* They'll call again */ + goto upnout; + } + spin_unlock(&c->erase_completion_lock); + + /* OK. Looks safe. And nobody can get us now because we have the semaphore. Move the block */ + if (f->metadata && f->metadata->raw == raw) { + fn = f->metadata; + ret = jffs2_garbage_collect_metadata(c, jeb, f, fn); + goto upnout; + } + + /* FIXME. Read node and do lookup? */ + for (frag = frag_first(&f->fragtree); frag; frag = frag_next(frag)) { + if (frag->node && frag->node->raw == raw) { + fn = frag->node; + end = frag->ofs + frag->size; + if (!nrfrags++) + start = frag->ofs; + if (nrfrags == frag->node->frags) + break; /* We've found them all */ + } + } + if (fn) { + if (ref_flags(raw) == REF_PRISTINE) { + ret = jffs2_garbage_collect_pristine(c, f->inocache, raw); + if (!ret) { + /* Urgh. Return it sensibly. */ + frag->node->raw = f->inocache->nodes; + } + if (ret != -EBADFD) + goto upnout; + } + /* We found a datanode. Do the GC */ + if((start >> PAGE_SHIFT) < ((end-1) >> PAGE_SHIFT)) { + /* It crosses a page boundary. Therefore, it must be a hole. */ + ret = jffs2_garbage_collect_hole(c, jeb, f, fn, start, end); + } else { + /* It could still be a hole. But we GC the page this way anyway */ + ret = jffs2_garbage_collect_dnode(c, jeb, f, fn, start, end); + } + goto upnout; + } + + /* Wasn't a dnode. Try dirent */ + for (fd = f->dents; fd; fd=fd->next) { + if (fd->raw == raw) + break; + } + + if (fd && fd->ino) { + ret = jffs2_garbage_collect_dirent(c, jeb, f, fd); + } else if (fd) { + ret = jffs2_garbage_collect_deletion_dirent(c, jeb, f, fd); + } else { + pr_warn("Raw node at 0x%08x wasn't in node lists for ino #%u\n", + ref_offset(raw), f->inocache->ino); + if (ref_obsolete(raw)) { + pr_warn("But it's obsolete so we don't mind too much\n"); + } else { + jffs2_dbg_dump_node(c, ref_offset(raw)); + BUG(); + } + } + upnout: + mutex_unlock(&f->sem); + + return ret; +} + +static int jffs2_garbage_collect_pristine(struct jffs2_sb_info *c, + struct jffs2_inode_cache *ic, + struct jffs2_raw_node_ref *raw) +{ + union jffs2_node_union *node; + size_t retlen; + int ret; + uint32_t phys_ofs, alloclen; + uint32_t crc, rawlen; + int retried = 0; + + jffs2_dbg(1, "Going to GC REF_PRISTINE node at 0x%08x\n", + ref_offset(raw)); + + alloclen = rawlen = ref_totlen(c, c->gcblock, raw); + + /* Ask for a small amount of space (or the totlen if smaller) because we + don't want to force wastage of the end of a block if splitting would + work. */ + if (ic && alloclen > sizeof(struct jffs2_raw_inode) + JFFS2_MIN_DATA_LEN) + alloclen = sizeof(struct jffs2_raw_inode) + JFFS2_MIN_DATA_LEN; + + ret = jffs2_reserve_space_gc(c, alloclen, &alloclen, rawlen); + /* 'rawlen' is not the exact summary size; it is only an upper estimation */ + + if (ret) + return ret; + + if (alloclen < rawlen) { + /* Doesn't fit untouched. We'll go the old route and split it */ + return -EBADFD; + } + + node = kmalloc(rawlen, GFP_KERNEL); + if (!node) + return -ENOMEM; + + ret = jffs2_flash_read(c, ref_offset(raw), rawlen, &retlen, (char *)node); + if (!ret && retlen != rawlen) + ret = -EIO; + if (ret) + goto out_node; + + crc = crc32(0, node, sizeof(struct jffs2_unknown_node)-4); + if (je32_to_cpu(node->u.hdr_crc) != crc) { + pr_warn("Header CRC failed on REF_PRISTINE node at 0x%08x: Read 0x%08x, calculated 0x%08x\n", + ref_offset(raw), je32_to_cpu(node->u.hdr_crc), crc); + goto bail; + } + + switch(je16_to_cpu(node->u.nodetype)) { + case JFFS2_NODETYPE_INODE: + crc = crc32(0, node, sizeof(node->i)-8); + if (je32_to_cpu(node->i.node_crc) != crc) { + pr_warn("Node CRC failed on REF_PRISTINE data node at 0x%08x: Read 0x%08x, calculated 0x%08x\n", + ref_offset(raw), je32_to_cpu(node->i.node_crc), + crc); + goto bail; + } + + if (je32_to_cpu(node->i.dsize)) { + crc = crc32(0, node->i.data, je32_to_cpu(node->i.csize)); + if (je32_to_cpu(node->i.data_crc) != crc) { + pr_warn("Data CRC failed on REF_PRISTINE data node at 0x%08x: Read 0x%08x, calculated 0x%08x\n", + ref_offset(raw), + je32_to_cpu(node->i.data_crc), crc); + goto bail; + } + } + break; + + case JFFS2_NODETYPE_DIRENT: + crc = crc32(0, node, sizeof(node->d)-8); + if (je32_to_cpu(node->d.node_crc) != crc) { + pr_warn("Node CRC failed on REF_PRISTINE dirent node at 0x%08x: Read 0x%08x, calculated 0x%08x\n", + ref_offset(raw), + je32_to_cpu(node->d.node_crc), crc); + goto bail; + } + + if (strnlen(node->d.name, node->d.nsize) != node->d.nsize) { + pr_warn("Name in dirent node at 0x%08x contains zeroes\n", + ref_offset(raw)); + goto bail; + } + + if (node->d.nsize) { + crc = crc32(0, node->d.name, node->d.nsize); + if (je32_to_cpu(node->d.name_crc) != crc) { + pr_warn("Name CRC failed on REF_PRISTINE dirent node at 0x%08x: Read 0x%08x, calculated 0x%08x\n", + ref_offset(raw), + je32_to_cpu(node->d.name_crc), crc); + goto bail; + } + } + break; + default: + /* If it's inode-less, we don't _know_ what it is. Just copy it intact */ + if (ic) { + pr_warn("Unknown node type for REF_PRISTINE node at 0x%08x: 0x%04x\n", + ref_offset(raw), je16_to_cpu(node->u.nodetype)); + goto bail; + } + } + + /* OK, all the CRCs are good; this node can just be copied as-is. */ + retry: + phys_ofs = write_ofs(c); + + ret = jffs2_flash_write(c, phys_ofs, rawlen, &retlen, (char *)node); + + if (ret || (retlen != rawlen)) { + pr_notice("Write of %d bytes at 0x%08x failed. returned %d, retlen %zd\n", + rawlen, phys_ofs, ret, retlen); + if (retlen) { + jffs2_add_physical_node_ref(c, phys_ofs | REF_OBSOLETE, rawlen, NULL); + } else { + pr_notice("Not marking the space at 0x%08x as dirty because the flash driver returned retlen zero\n", + phys_ofs); + } + if (!retried) { + /* Try to reallocate space and retry */ + uint32_t dummy; + struct jffs2_eraseblock *jeb = &c->blocks[phys_ofs / c->sector_size]; + + retried = 1; + + jffs2_dbg(1, "Retrying failed write of REF_PRISTINE node.\n"); + + jffs2_dbg_acct_sanity_check(c,jeb); + jffs2_dbg_acct_paranoia_check(c, jeb); + + ret = jffs2_reserve_space_gc(c, rawlen, &dummy, rawlen); + /* this is not the exact summary size of it, + it is only an upper estimation */ + + if (!ret) { + jffs2_dbg(1, "Allocated space at 0x%08x to retry failed write.\n", + phys_ofs); + + jffs2_dbg_acct_sanity_check(c,jeb); + jffs2_dbg_acct_paranoia_check(c, jeb); + + goto retry; + } + jffs2_dbg(1, "Failed to allocate space to retry failed write: %d!\n", + ret); + } + + if (!ret) + ret = -EIO; + goto out_node; + } + jffs2_add_physical_node_ref(c, phys_ofs | REF_PRISTINE, rawlen, ic); + + jffs2_mark_node_obsolete(c, raw); + jffs2_dbg(1, "WHEEE! GC REF_PRISTINE node at 0x%08x succeeded\n", + ref_offset(raw)); + + out_node: + kfree(node); + return ret; + bail: + ret = -EBADFD; + goto out_node; +} + +static int jffs2_garbage_collect_metadata(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, + struct jffs2_inode_info *f, struct jffs2_full_dnode *fn) +{ + struct jffs2_full_dnode *new_fn; + struct jffs2_raw_inode ri; + struct jffs2_node_frag *last_frag; + union jffs2_device_node dev; + char *mdata = NULL; + int mdatalen = 0; + uint32_t alloclen, ilen; + int ret; + + if (S_ISBLK(JFFS2_F_I_MODE(f)) || + S_ISCHR(JFFS2_F_I_MODE(f)) ) { + /* For these, we don't actually need to read the old node */ + mdatalen = jffs2_encode_dev(&dev, JFFS2_F_I_RDEV(f)); + mdata = (char *)&dev; + jffs2_dbg(1, "%s(): Writing %d bytes of kdev_t\n", + __func__, mdatalen); + } else if (S_ISLNK(JFFS2_F_I_MODE(f))) { + mdatalen = fn->size; + mdata = kmalloc(fn->size, GFP_KERNEL); + if (!mdata) { + pr_warn("kmalloc of mdata failed in jffs2_garbage_collect_metadata()\n"); + return -ENOMEM; + } + ret = jffs2_read_dnode(c, f, fn, mdata, 0, mdatalen); + if (ret) { + pr_warn("read of old metadata failed in jffs2_garbage_collect_metadata(): %d\n", + ret); + kfree(mdata); + return ret; + } + jffs2_dbg(1, "%s(): Writing %d bites of symlink target\n", + __func__, mdatalen); + + } + + ret = jffs2_reserve_space_gc(c, sizeof(ri) + mdatalen, &alloclen, + JFFS2_SUMMARY_INODE_SIZE); + if (ret) { + pr_warn("jffs2_reserve_space_gc of %zd bytes for garbage_collect_metadata failed: %d\n", + sizeof(ri) + mdatalen, ret); + goto out; + } + + last_frag = frag_last(&f->fragtree); + if (last_frag) + /* Fetch the inode length from the fragtree rather then + * from i_size since i_size may have not been updated yet */ + ilen = last_frag->ofs + last_frag->size; + else + ilen = JFFS2_F_I_SIZE(f); + + memset(&ri, 0, sizeof(ri)); + ri.magic = cpu_to_je16(JFFS2_MAGIC_BITMASK); + ri.nodetype = cpu_to_je16(JFFS2_NODETYPE_INODE); + ri.totlen = cpu_to_je32(sizeof(ri) + mdatalen); + ri.hdr_crc = cpu_to_je32(crc32(0, &ri, sizeof(struct jffs2_unknown_node)-4)); + + ri.ino = cpu_to_je32(f->inocache->ino); + ri.version = cpu_to_je32(++f->highest_version); + ri.mode = cpu_to_jemode(JFFS2_F_I_MODE(f)); + ri.uid = cpu_to_je16(JFFS2_F_I_UID(f)); + ri.gid = cpu_to_je16(JFFS2_F_I_GID(f)); + ri.isize = cpu_to_je32(ilen); + ri.atime = cpu_to_je32(JFFS2_F_I_ATIME(f)); + ri.ctime = cpu_to_je32(JFFS2_F_I_CTIME(f)); + ri.mtime = cpu_to_je32(JFFS2_F_I_MTIME(f)); + ri.offset = cpu_to_je32(0); + ri.csize = cpu_to_je32(mdatalen); + ri.dsize = cpu_to_je32(mdatalen); + ri.compr = JFFS2_COMPR_NONE; + ri.node_crc = cpu_to_je32(crc32(0, &ri, sizeof(ri)-8)); + ri.data_crc = cpu_to_je32(crc32(0, mdata, mdatalen)); + + new_fn = jffs2_write_dnode(c, f, &ri, mdata, mdatalen, ALLOC_GC); + + if (IS_ERR(new_fn)) { + pr_warn("Error writing new dnode: %ld\n", PTR_ERR(new_fn)); + ret = PTR_ERR(new_fn); + goto out; + } + jffs2_mark_node_obsolete(c, fn->raw); + jffs2_free_full_dnode(fn); + f->metadata = new_fn; + out: + if (S_ISLNK(JFFS2_F_I_MODE(f))) + kfree(mdata); + return ret; +} + +static int jffs2_garbage_collect_dirent(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, + struct jffs2_inode_info *f, struct jffs2_full_dirent *fd) +{ + struct jffs2_full_dirent *new_fd; + struct jffs2_raw_dirent rd; + uint32_t alloclen; + int ret; + + rd.magic = cpu_to_je16(JFFS2_MAGIC_BITMASK); + rd.nodetype = cpu_to_je16(JFFS2_NODETYPE_DIRENT); + rd.nsize = strlen(fd->name); + rd.totlen = cpu_to_je32(sizeof(rd) + rd.nsize); + rd.hdr_crc = cpu_to_je32(crc32(0, &rd, sizeof(struct jffs2_unknown_node)-4)); + + rd.pino = cpu_to_je32(f->inocache->ino); + rd.version = cpu_to_je32(++f->highest_version); + rd.ino = cpu_to_je32(fd->ino); + /* If the times on this inode were set by explicit utime() they can be different, + so refrain from splatting them. */ + if (JFFS2_F_I_MTIME(f) == JFFS2_F_I_CTIME(f)) + rd.mctime = cpu_to_je32(JFFS2_F_I_MTIME(f)); + else + rd.mctime = cpu_to_je32(0); + rd.type = fd->type; + rd.node_crc = cpu_to_je32(crc32(0, &rd, sizeof(rd)-8)); + rd.name_crc = cpu_to_je32(crc32(0, fd->name, rd.nsize)); + + ret = jffs2_reserve_space_gc(c, sizeof(rd)+rd.nsize, &alloclen, + JFFS2_SUMMARY_DIRENT_SIZE(rd.nsize)); + if (ret) { + pr_warn("jffs2_reserve_space_gc of %zd bytes for garbage_collect_dirent failed: %d\n", + sizeof(rd)+rd.nsize, ret); + return ret; + } + new_fd = jffs2_write_dirent(c, f, &rd, fd->name, rd.nsize, ALLOC_GC); + + if (IS_ERR(new_fd)) { + pr_warn("jffs2_write_dirent in garbage_collect_dirent failed: %ld\n", + PTR_ERR(new_fd)); + return PTR_ERR(new_fd); + } + jffs2_add_fd_to_list(c, new_fd, &f->dents); + return 0; +} + +static int jffs2_garbage_collect_deletion_dirent(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, + struct jffs2_inode_info *f, struct jffs2_full_dirent *fd) +{ + struct jffs2_full_dirent **fdp = &f->dents; + int found = 0; + + /* On a medium where we can't actually mark nodes obsolete + pernamently, such as NAND flash, we need to work out + whether this deletion dirent is still needed to actively + delete a 'real' dirent with the same name that's still + somewhere else on the flash. */ + if (!jffs2_can_mark_obsolete(c)) { + struct jffs2_raw_dirent *rd; + struct jffs2_raw_node_ref *raw; + int ret; + size_t retlen; + int name_len = strlen(fd->name); + uint32_t name_crc = crc32(0, fd->name, name_len); + uint32_t rawlen = ref_totlen(c, jeb, fd->raw); + + rd = kmalloc(rawlen, GFP_KERNEL); + if (!rd) + return -ENOMEM; + + /* Prevent the erase code from nicking the obsolete node refs while + we're looking at them. I really don't like this extra lock but + can't see any alternative. Suggestions on a postcard to... */ + mutex_lock(&c->erase_free_sem); + + for (raw = f->inocache->nodes; raw != (void *)f->inocache; raw = raw->next_in_ino) { + + cond_resched(); + + /* We only care about obsolete ones */ + if (!(ref_obsolete(raw))) + continue; + + /* Any dirent with the same name is going to have the same length... */ + if (ref_totlen(c, NULL, raw) != rawlen) + continue; + + /* Doesn't matter if there's one in the same erase block. We're going to + delete it too at the same time. */ + if (SECTOR_ADDR(raw->flash_offset) == SECTOR_ADDR(fd->raw->flash_offset)) + continue; + + jffs2_dbg(1, "Check potential deletion dirent at %08x\n", + ref_offset(raw)); + + /* This is an obsolete node belonging to the same directory, and it's of the right + length. We need to take a closer look...*/ + ret = jffs2_flash_read(c, ref_offset(raw), rawlen, &retlen, (char *)rd); + if (ret) { + pr_warn("%s(): Read error (%d) reading obsolete node at %08x\n", + __func__, ret, ref_offset(raw)); + /* If we can't read it, we don't need to continue to obsolete it. Continue */ + continue; + } + if (retlen != rawlen) { + pr_warn("%s(): Short read (%zd not %u) reading header from obsolete node at %08x\n", + __func__, retlen, rawlen, + ref_offset(raw)); + continue; + } + + if (je16_to_cpu(rd->nodetype) != JFFS2_NODETYPE_DIRENT) + continue; + + /* If the name CRC doesn't match, skip */ + if (je32_to_cpu(rd->name_crc) != name_crc) + continue; + + /* If the name length doesn't match, or it's another deletion dirent, skip */ + if (rd->nsize != name_len || !je32_to_cpu(rd->ino)) + continue; + + /* OK, check the actual name now */ + if (memcmp(rd->name, fd->name, name_len)) + continue; + + /* OK. The name really does match. There really is still an older node on + the flash which our deletion dirent obsoletes. So we have to write out + a new deletion dirent to replace it */ + mutex_unlock(&c->erase_free_sem); + + jffs2_dbg(1, "Deletion dirent at %08x still obsoletes real dirent \"%s\" at %08x for ino #%u\n", + ref_offset(fd->raw), fd->name, + ref_offset(raw), je32_to_cpu(rd->ino)); + kfree(rd); + + return jffs2_garbage_collect_dirent(c, jeb, f, fd); + } + + mutex_unlock(&c->erase_free_sem); + kfree(rd); + } + + /* FIXME: If we're deleting a dirent which contains the current mtime and ctime, + we should update the metadata node with those times accordingly */ + + /* No need for it any more. Just mark it obsolete and remove it from the list */ + while (*fdp) { + if ((*fdp) == fd) { + found = 1; + *fdp = fd->next; + break; + } + fdp = &(*fdp)->next; + } + if (!found) { + pr_warn("Deletion dirent \"%s\" not found in list for ino #%u\n", + fd->name, f->inocache->ino); + } + jffs2_mark_node_obsolete(c, fd->raw); + jffs2_free_full_dirent(fd); + return 0; +} + +static int jffs2_garbage_collect_hole(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, + struct jffs2_inode_info *f, struct jffs2_full_dnode *fn, + uint32_t start, uint32_t end) +{ + struct jffs2_raw_inode ri; + struct jffs2_node_frag *frag; + struct jffs2_full_dnode *new_fn; + uint32_t alloclen, ilen; + int ret; + + jffs2_dbg(1, "Writing replacement hole node for ino #%u from offset 0x%x to 0x%x\n", + f->inocache->ino, start, end); + + memset(&ri, 0, sizeof(ri)); + + if(fn->frags > 1) { + size_t readlen; + uint32_t crc; + /* It's partially obsoleted by a later write. So we have to + write it out again with the _same_ version as before */ + ret = jffs2_flash_read(c, ref_offset(fn->raw), sizeof(ri), &readlen, (char *)&ri); + if (readlen != sizeof(ri) || ret) { + pr_warn("Node read failed in jffs2_garbage_collect_hole. Ret %d, retlen %zd. Data will be lost by writing new hole node\n", + ret, readlen); + goto fill; + } + if (je16_to_cpu(ri.nodetype) != JFFS2_NODETYPE_INODE) { + pr_warn("%s(): Node at 0x%08x had node type 0x%04x instead of JFFS2_NODETYPE_INODE(0x%04x)\n", + __func__, ref_offset(fn->raw), + je16_to_cpu(ri.nodetype), JFFS2_NODETYPE_INODE); + return -EIO; + } + if (je32_to_cpu(ri.totlen) != sizeof(ri)) { + pr_warn("%s(): Node at 0x%08x had totlen 0x%x instead of expected 0x%zx\n", + __func__, ref_offset(fn->raw), + je32_to_cpu(ri.totlen), sizeof(ri)); + return -EIO; + } + crc = crc32(0, &ri, sizeof(ri)-8); + if (crc != je32_to_cpu(ri.node_crc)) { + pr_warn("%s: Node at 0x%08x had CRC 0x%08x which doesn't match calculated CRC 0x%08x\n", + __func__, ref_offset(fn->raw), + je32_to_cpu(ri.node_crc), crc); + /* FIXME: We could possibly deal with this by writing new holes for each frag */ + pr_warn("Data in the range 0x%08x to 0x%08x of inode #%u will be lost\n", + start, end, f->inocache->ino); + goto fill; + } + if (ri.compr != JFFS2_COMPR_ZERO) { + pr_warn("%s(): Node 0x%08x wasn't a hole node!\n", + __func__, ref_offset(fn->raw)); + pr_warn("Data in the range 0x%08x to 0x%08x of inode #%u will be lost\n", + start, end, f->inocache->ino); + goto fill; + } + } else { + fill: + ri.magic = cpu_to_je16(JFFS2_MAGIC_BITMASK); + ri.nodetype = cpu_to_je16(JFFS2_NODETYPE_INODE); + ri.totlen = cpu_to_je32(sizeof(ri)); + ri.hdr_crc = cpu_to_je32(crc32(0, &ri, sizeof(struct jffs2_unknown_node)-4)); + + ri.ino = cpu_to_je32(f->inocache->ino); + ri.version = cpu_to_je32(++f->highest_version); + ri.offset = cpu_to_je32(start); + ri.dsize = cpu_to_je32(end - start); + ri.csize = cpu_to_je32(0); + ri.compr = JFFS2_COMPR_ZERO; + } + + frag = frag_last(&f->fragtree); + if (frag) + /* Fetch the inode length from the fragtree rather then + * from i_size since i_size may have not been updated yet */ + ilen = frag->ofs + frag->size; + else + ilen = JFFS2_F_I_SIZE(f); + + ri.mode = cpu_to_jemode(JFFS2_F_I_MODE(f)); + ri.uid = cpu_to_je16(JFFS2_F_I_UID(f)); + ri.gid = cpu_to_je16(JFFS2_F_I_GID(f)); + ri.isize = cpu_to_je32(ilen); + ri.atime = cpu_to_je32(JFFS2_F_I_ATIME(f)); + ri.ctime = cpu_to_je32(JFFS2_F_I_CTIME(f)); + ri.mtime = cpu_to_je32(JFFS2_F_I_MTIME(f)); + ri.data_crc = cpu_to_je32(0); + ri.node_crc = cpu_to_je32(crc32(0, &ri, sizeof(ri)-8)); + + ret = jffs2_reserve_space_gc(c, sizeof(ri), &alloclen, + JFFS2_SUMMARY_INODE_SIZE); + if (ret) { + pr_warn("jffs2_reserve_space_gc of %zd bytes for garbage_collect_hole failed: %d\n", + sizeof(ri), ret); + return ret; + } + new_fn = jffs2_write_dnode(c, f, &ri, NULL, 0, ALLOC_GC); + + if (IS_ERR(new_fn)) { + pr_warn("Error writing new hole node: %ld\n", PTR_ERR(new_fn)); + return PTR_ERR(new_fn); + } + if (je32_to_cpu(ri.version) == f->highest_version) { + jffs2_add_full_dnode_to_inode(c, f, new_fn); + if (f->metadata) { + jffs2_mark_node_obsolete(c, f->metadata->raw); + jffs2_free_full_dnode(f->metadata); + f->metadata = NULL; + } + return 0; + } + + /* + * We should only get here in the case where the node we are + * replacing had more than one frag, so we kept the same version + * number as before. (Except in case of error -- see 'goto fill;' + * above.) + */ + D1(if(unlikely(fn->frags <= 1)) { + pr_warn("%s(): Replacing fn with %d frag(s) but new ver %d != highest_version %d of ino #%d\n", + __func__, fn->frags, je32_to_cpu(ri.version), + f->highest_version, je32_to_cpu(ri.ino)); + }); + + /* This is a partially-overlapped hole node. Mark it REF_NORMAL not REF_PRISTINE */ + mark_ref_normal(new_fn->raw); + + for (frag = jffs2_lookup_node_frag(&f->fragtree, fn->ofs); + frag; frag = frag_next(frag)) { + if (frag->ofs > fn->size + fn->ofs) + break; + if (frag->node == fn) { + frag->node = new_fn; + new_fn->frags++; + fn->frags--; + } + } + if (fn->frags) { + pr_warn("%s(): Old node still has frags!\n", __func__); + BUG(); + } + if (!new_fn->frags) { + pr_warn("%s(): New node has no frags!\n", __func__); + BUG(); + } + + jffs2_mark_node_obsolete(c, fn->raw); + jffs2_free_full_dnode(fn); + + return 0; +} + +static int jffs2_garbage_collect_dnode(struct jffs2_sb_info *c, struct jffs2_eraseblock *orig_jeb, + struct jffs2_inode_info *f, struct jffs2_full_dnode *fn, + uint32_t start, uint32_t end) +{ + struct inode *inode = OFNI_EDONI_2SFFJ(f); + struct jffs2_full_dnode *new_fn; + struct jffs2_raw_inode ri; + uint32_t alloclen, offset, orig_end, orig_start; + int ret = 0; + unsigned char *comprbuf = NULL, *writebuf; + struct folio *folio; + unsigned char *pg_ptr; + + memset(&ri, 0, sizeof(ri)); + + jffs2_dbg(1, "Writing replacement dnode for ino #%u from offset 0x%x to 0x%x\n", + f->inocache->ino, start, end); + + orig_end = end; + orig_start = start; + + if (c->nr_free_blocks + c->nr_erasing_blocks > c->resv_blocks_gcmerge) { + /* Attempt to do some merging. But only expand to cover logically + adjacent frags if the block containing them is already considered + to be dirty. Otherwise we end up with GC just going round in + circles dirtying the nodes it already wrote out, especially + on NAND where we have small eraseblocks and hence a much higher + chance of nodes having to be split to cross boundaries. */ + + struct jffs2_node_frag *frag; + uint32_t min, max; + + min = start & ~(PAGE_SIZE-1); + max = min + PAGE_SIZE; + + frag = jffs2_lookup_node_frag(&f->fragtree, start); + + /* BUG_ON(!frag) but that'll happen anyway... */ + + BUG_ON(frag->ofs != start); + + /* First grow down... */ + while((frag = frag_prev(frag)) && frag->ofs >= min) { + + /* If the previous frag doesn't even reach the beginning, there's + excessive fragmentation. Just merge. */ + if (frag->ofs > min) { + jffs2_dbg(1, "Expanding down to cover partial frag (0x%x-0x%x)\n", + frag->ofs, frag->ofs+frag->size); + start = frag->ofs; + continue; + } + /* OK. This frag holds the first byte of the page. */ + if (!frag->node || !frag->node->raw) { + jffs2_dbg(1, "First frag in page is hole (0x%x-0x%x). Not expanding down.\n", + frag->ofs, frag->ofs+frag->size); + break; + } else { + + /* OK, it's a frag which extends to the beginning of the page. Does it live + in a block which is still considered clean? If so, don't obsolete it. + If not, cover it anyway. */ + + struct jffs2_raw_node_ref *raw = frag->node->raw; + struct jffs2_eraseblock *jeb; + + jeb = &c->blocks[raw->flash_offset / c->sector_size]; + + if (jeb == c->gcblock) { + jffs2_dbg(1, "Expanding down to cover frag (0x%x-0x%x) in gcblock at %08x\n", + frag->ofs, + frag->ofs + frag->size, + ref_offset(raw)); + start = frag->ofs; + break; + } + if (!ISDIRTY(jeb->dirty_size + jeb->wasted_size)) { + jffs2_dbg(1, "Not expanding down to cover frag (0x%x-0x%x) in clean block %08x\n", + frag->ofs, + frag->ofs + frag->size, + jeb->offset); + break; + } + + jffs2_dbg(1, "Expanding down to cover frag (0x%x-0x%x) in dirty block %08x\n", + frag->ofs, + frag->ofs + frag->size, + jeb->offset); + start = frag->ofs; + break; + } + } + + /* ... then up */ + + /* Find last frag which is actually part of the node we're to GC. */ + frag = jffs2_lookup_node_frag(&f->fragtree, end-1); + + while((frag = frag_next(frag)) && frag->ofs+frag->size <= max) { + + /* If the previous frag doesn't even reach the beginning, there's lots + of fragmentation. Just merge. */ + if (frag->ofs+frag->size < max) { + jffs2_dbg(1, "Expanding up to cover partial frag (0x%x-0x%x)\n", + frag->ofs, frag->ofs+frag->size); + end = frag->ofs + frag->size; + continue; + } + + if (!frag->node || !frag->node->raw) { + jffs2_dbg(1, "Last frag in page is hole (0x%x-0x%x). Not expanding up.\n", + frag->ofs, frag->ofs+frag->size); + break; + } else { + + /* OK, it's a frag which extends to the beginning of the page. Does it live + in a block which is still considered clean? If so, don't obsolete it. + If not, cover it anyway. */ + + struct jffs2_raw_node_ref *raw = frag->node->raw; + struct jffs2_eraseblock *jeb; + + jeb = &c->blocks[raw->flash_offset / c->sector_size]; + + if (jeb == c->gcblock) { + jffs2_dbg(1, "Expanding up to cover frag (0x%x-0x%x) in gcblock at %08x\n", + frag->ofs, + frag->ofs + frag->size, + ref_offset(raw)); + end = frag->ofs + frag->size; + break; + } + if (!ISDIRTY(jeb->dirty_size + jeb->wasted_size)) { + jffs2_dbg(1, "Not expanding up to cover frag (0x%x-0x%x) in clean block %08x\n", + frag->ofs, + frag->ofs + frag->size, + jeb->offset); + break; + } + + jffs2_dbg(1, "Expanding up to cover frag (0x%x-0x%x) in dirty block %08x\n", + frag->ofs, + frag->ofs + frag->size, + jeb->offset); + end = frag->ofs + frag->size; + break; + } + } + jffs2_dbg(1, "Expanded dnode to write from (0x%x-0x%x) to (0x%x-0x%x)\n", + orig_start, orig_end, start, end); + + D1(BUG_ON(end > frag_last(&f->fragtree)->ofs + frag_last(&f->fragtree)->size)); + BUG_ON(end < orig_end); + BUG_ON(start > orig_start); + } + + /* The rules state that we must obtain the folio lock *before* f->sem, so + * drop f->sem temporarily. Since we also hold c->alloc_sem, nothing's + * actually going to *change* so we're safe; we only allow reading. + * + * It is important to note that jffs2_write_begin() will ensure that its + * folio is marked uptodate before allocating space. That means that if we + * end up here trying to GC the *same* folio that jffs2_write_begin() is + * trying to write out, read_cache_folio() will not deadlock. */ + mutex_unlock(&f->sem); + folio = read_cache_folio(inode->i_mapping, start >> PAGE_SHIFT, + __jffs2_read_folio, NULL); + if (IS_ERR(folio)) { + pr_warn("read_cache_folio() returned error: %ld\n", + PTR_ERR(folio)); + mutex_lock(&f->sem); + return PTR_ERR(folio); + } + + pg_ptr = kmap_local_folio(folio, 0); + mutex_lock(&f->sem); + + offset = start; + while(offset < orig_end) { + uint32_t datalen; + uint32_t cdatalen; + uint16_t comprtype = JFFS2_COMPR_NONE; + + ret = jffs2_reserve_space_gc(c, sizeof(ri) + JFFS2_MIN_DATA_LEN, + &alloclen, JFFS2_SUMMARY_INODE_SIZE); + + if (ret) { + pr_warn("jffs2_reserve_space_gc of %zd bytes for garbage_collect_dnode failed: %d\n", + sizeof(ri) + JFFS2_MIN_DATA_LEN, ret); + break; + } + cdatalen = min_t(uint32_t, alloclen - sizeof(ri), end - offset); + datalen = end - offset; + + writebuf = pg_ptr + (offset & (PAGE_SIZE -1)); + + comprtype = jffs2_compress(c, f, writebuf, &comprbuf, &datalen, &cdatalen); + + ri.magic = cpu_to_je16(JFFS2_MAGIC_BITMASK); + ri.nodetype = cpu_to_je16(JFFS2_NODETYPE_INODE); + ri.totlen = cpu_to_je32(sizeof(ri) + cdatalen); + ri.hdr_crc = cpu_to_je32(crc32(0, &ri, sizeof(struct jffs2_unknown_node)-4)); + + ri.ino = cpu_to_je32(f->inocache->ino); + ri.version = cpu_to_je32(++f->highest_version); + ri.mode = cpu_to_jemode(JFFS2_F_I_MODE(f)); + ri.uid = cpu_to_je16(JFFS2_F_I_UID(f)); + ri.gid = cpu_to_je16(JFFS2_F_I_GID(f)); + ri.isize = cpu_to_je32(JFFS2_F_I_SIZE(f)); + ri.atime = cpu_to_je32(JFFS2_F_I_ATIME(f)); + ri.ctime = cpu_to_je32(JFFS2_F_I_CTIME(f)); + ri.mtime = cpu_to_je32(JFFS2_F_I_MTIME(f)); + ri.offset = cpu_to_je32(offset); + ri.csize = cpu_to_je32(cdatalen); + ri.dsize = cpu_to_je32(datalen); + ri.compr = comprtype & 0xff; + ri.usercompr = (comprtype >> 8) & 0xff; + ri.node_crc = cpu_to_je32(crc32(0, &ri, sizeof(ri)-8)); + ri.data_crc = cpu_to_je32(crc32(0, comprbuf, cdatalen)); + + new_fn = jffs2_write_dnode(c, f, &ri, comprbuf, cdatalen, ALLOC_GC); + + jffs2_free_comprbuf(comprbuf, writebuf); + + if (IS_ERR(new_fn)) { + pr_warn("Error writing new dnode: %ld\n", + PTR_ERR(new_fn)); + ret = PTR_ERR(new_fn); + break; + } + ret = jffs2_add_full_dnode_to_inode(c, f, new_fn); + offset += datalen; + if (f->metadata) { + jffs2_mark_node_obsolete(c, f->metadata->raw); + jffs2_free_full_dnode(f->metadata); + f->metadata = NULL; + } + } + + folio_release_kmap(folio, pg_ptr); + return ret; +} diff --git a/fs/jffs2/ioctl.c b/fs/jffs2/ioctl.c new file mode 100644 index 000000000..859a598af --- /dev/null +++ b/fs/jffs2/ioctl.c @@ -0,0 +1,22 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * Copyright © 2004-2010 David Woodhouse + * + * Created by David Woodhouse + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#include +#include "nodelist.h" + +long jffs2_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + /* Later, this will provide for lsattr.jffs2 and chattr.jffs2, which + will include compression support etc. */ + return -ENOTTY; +} + diff --git a/fs/jffs2/jffs2_fs_i.h b/fs/jffs2/jffs2_fs_i.h new file mode 100644 index 000000000..93a295153 --- /dev/null +++ b/fs/jffs2/jffs2_fs_i.h @@ -0,0 +1,56 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * Copyright © 2004-2010 David Woodhouse + * + * Created by David Woodhouse + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#ifndef _JFFS2_FS_I +#define _JFFS2_FS_I + +#include +#include +#include + +struct jffs2_inode_info { + /* We need an internal mutex similar to inode->i_rwsem. + Unfortunately, we can't used the existing one, because + either the GC would deadlock, or we'd have to release it + before letting GC proceed. Or we'd have to put ugliness + into the GC code so it didn't attempt to obtain the i_rwsem + for the inode(s) which are already locked */ + struct mutex sem; + + /* The highest (datanode) version number used for this ino */ + uint32_t highest_version; + + /* List of data fragments which make up the file */ + struct rb_root fragtree; + + /* There may be one datanode which isn't referenced by any of the + above fragments, if it contains a metadata update but no actual + data - or if this is a directory inode */ + /* This also holds the _only_ dnode for symlinks/device nodes, + etc. */ + struct jffs2_full_dnode *metadata; + + /* Directory entries */ + struct jffs2_full_dirent *dents; + + /* The target path if this is the inode of a symlink */ + unsigned char *target; + + /* Some stuff we just have to keep in-core at all times, for each inode. */ + struct jffs2_inode_cache *inocache; + + uint16_t flags; + uint8_t usercompr; + struct inode vfs_inode; +}; + +#endif /* _JFFS2_FS_I */ diff --git a/fs/jffs2/jffs2_fs_sb.h b/fs/jffs2/jffs2_fs_sb.h new file mode 100644 index 000000000..5a7091746 --- /dev/null +++ b/fs/jffs2/jffs2_fs_sb.h @@ -0,0 +1,163 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * Copyright © 2004-2010 David Woodhouse + * + * Created by David Woodhouse + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#ifndef _JFFS2_FS_SB +#define _JFFS2_FS_SB + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define JFFS2_SB_FLAG_RO 1 +#define JFFS2_SB_FLAG_SCANNING 2 /* Flash scanning is in progress */ +#define JFFS2_SB_FLAG_BUILDING 4 /* File system building is in progress */ + +struct jffs2_inodirty; + +struct jffs2_mount_opts { + bool override_compr; + unsigned int compr; + + /* The size of the reserved pool. The reserved pool is the JFFS2 flash + * space which may only be used by root cannot be used by the other + * users. This is implemented simply by means of not allowing the + * latter users to write to the file system if the amount if the + * available space is less then 'rp_size'. */ + bool set_rp_size; + unsigned int rp_size; +}; + +/* A struct for the overall file system control. Pointers to + jffs2_sb_info structs are named `c' in the source code. + Nee jffs_control +*/ +struct jffs2_sb_info { + struct mtd_info *mtd; + + uint32_t highest_ino; + uint32_t check_ino; /* *NEXT* inode to be checked */ + + unsigned int flags; + + struct task_struct *gc_task; /* GC task struct */ + struct completion gc_thread_start; /* GC thread start completion */ + struct completion gc_thread_exit; /* GC thread exit completion port */ + + struct mutex alloc_sem; /* Used to protect all the following + fields, and also to protect against + out-of-order writing of nodes. And GC. */ + uint32_t cleanmarker_size; /* Size of an _inline_ CLEANMARKER + (i.e. zero for OOB CLEANMARKER */ + + uint32_t flash_size; + uint32_t used_size; + uint32_t dirty_size; + uint32_t wasted_size; + uint32_t free_size; + uint32_t erasing_size; + uint32_t bad_size; + uint32_t sector_size; + uint32_t unchecked_size; + + uint32_t nr_free_blocks; + uint32_t nr_erasing_blocks; + + /* Number of free blocks there must be before we... */ + uint8_t resv_blocks_write; /* ... allow a normal filesystem write */ + uint8_t resv_blocks_deletion; /* ... allow a normal filesystem deletion */ + uint8_t resv_blocks_gctrigger; /* ... wake up the GC thread */ + uint8_t resv_blocks_gcbad; /* ... pick a block from the bad_list to GC */ + uint8_t resv_blocks_gcmerge; /* ... merge pages when garbage collecting */ + /* Number of 'very dirty' blocks before we trigger immediate GC */ + uint8_t vdirty_blocks_gctrigger; + + uint32_t nospc_dirty_size; + + uint32_t nr_blocks; + struct jffs2_eraseblock *blocks; /* The whole array of blocks. Used for getting blocks + * from the offset (blocks[ofs / sector_size]) */ + struct jffs2_eraseblock *nextblock; /* The block we're currently filling */ + + struct jffs2_eraseblock *gcblock; /* The block we're currently garbage-collecting */ + + struct list_head clean_list; /* Blocks 100% full of clean data */ + struct list_head very_dirty_list; /* Blocks with lots of dirty space */ + struct list_head dirty_list; /* Blocks with some dirty space */ + struct list_head erasable_list; /* Blocks which are completely dirty, and need erasing */ + struct list_head erasable_pending_wbuf_list; /* Blocks which need erasing but only after the current wbuf is flushed */ + struct list_head erasing_list; /* Blocks which are currently erasing */ + struct list_head erase_checking_list; /* Blocks which are being checked and marked */ + struct list_head erase_pending_list; /* Blocks which need erasing now */ + struct list_head erase_complete_list; /* Blocks which are erased and need the clean marker written to them */ + struct list_head free_list; /* Blocks which are free and ready to be used */ + struct list_head bad_list; /* Bad blocks. */ + struct list_head bad_used_list; /* Bad blocks with valid data in. */ + + spinlock_t erase_completion_lock; /* Protect free_list and erasing_list + against erase completion handler */ + wait_queue_head_t erase_wait; /* For waiting for erases to complete */ + + wait_queue_head_t inocache_wq; + int inocache_hashsize; + struct jffs2_inode_cache **inocache_list; + spinlock_t inocache_lock; + + /* Sem to allow jffs2_garbage_collect_deletion_dirent to + drop the erase_completion_lock while it's holding a pointer + to an obsoleted node. I don't like this. Alternatives welcomed. */ + struct mutex erase_free_sem; + + uint32_t wbuf_pagesize; /* 0 for NOR and other flashes with no wbuf */ + +#ifdef CONFIG_JFFS2_FS_WBUF_VERIFY + unsigned char *wbuf_verify; /* read-back buffer for verification */ +#endif +#ifdef CONFIG_JFFS2_FS_WRITEBUFFER + unsigned char *wbuf; /* Write-behind buffer for NAND flash */ + uint32_t wbuf_ofs; + uint32_t wbuf_len; + struct jffs2_inodirty *wbuf_inodes; + struct rw_semaphore wbuf_sem; /* Protects the write buffer */ + + struct delayed_work wbuf_dwork; /* write-buffer write-out work */ + + unsigned char *oobbuf; + int oobavail; /* How many bytes are available for JFFS2 in OOB */ +#endif + + struct jffs2_summary *summary; /* Summary information */ + struct jffs2_mount_opts mount_opts; + +#ifdef CONFIG_JFFS2_FS_XATTR +#define XATTRINDEX_HASHSIZE (57) + uint32_t highest_xid; + uint32_t highest_xseqno; + struct list_head xattrindex[XATTRINDEX_HASHSIZE]; + struct list_head xattr_unchecked; + struct list_head xattr_dead_list; + struct jffs2_xattr_ref *xref_dead_list; + struct jffs2_xattr_ref *xref_temp; + struct rw_semaphore xattr_sem; + uint32_t xdatum_mem_usage; + uint32_t xdatum_mem_threshold; +#endif + /* OS-private pointer for getting back to master superblock info */ + void *os_priv; +}; + +#endif /* _JFFS2_FS_SB */ diff --git a/fs/jffs2/malloc.c b/fs/jffs2/malloc.c new file mode 100644 index 000000000..411de8b36 --- /dev/null +++ b/fs/jffs2/malloc.c @@ -0,0 +1,299 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * + * Created by David Woodhouse + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include "nodelist.h" + +/* These are initialised to NULL in the kernel startup code. + If you're porting to other operating systems, beware */ +static struct kmem_cache *full_dnode_slab; +static struct kmem_cache *raw_dirent_slab; +static struct kmem_cache *raw_inode_slab; +static struct kmem_cache *tmp_dnode_info_slab; +static struct kmem_cache *raw_node_ref_slab; +static struct kmem_cache *node_frag_slab; +static struct kmem_cache *inode_cache_slab; +#ifdef CONFIG_JFFS2_FS_XATTR +static struct kmem_cache *xattr_datum_cache; +static struct kmem_cache *xattr_ref_cache; +#endif + +int __init jffs2_create_slab_caches(void) +{ + full_dnode_slab = KMEM_CACHE(jffs2_full_dnode, 0); + if (!full_dnode_slab) + goto err; + + raw_dirent_slab = KMEM_CACHE(jffs2_raw_dirent, SLAB_HWCACHE_ALIGN); + if (!raw_dirent_slab) + goto err; + + raw_inode_slab = KMEM_CACHE(jffs2_raw_inode, SLAB_HWCACHE_ALIGN); + if (!raw_inode_slab) + goto err; + + tmp_dnode_info_slab = KMEM_CACHE(jffs2_tmp_dnode_info, 0); + if (!tmp_dnode_info_slab) + goto err; + + raw_node_ref_slab = kmem_cache_create("jffs2_refblock", + sizeof(struct jffs2_raw_node_ref) * (REFS_PER_BLOCK + 1), + 0, 0, NULL); + if (!raw_node_ref_slab) + goto err; + + node_frag_slab = KMEM_CACHE(jffs2_node_frag, 0); + if (!node_frag_slab) + goto err; + + inode_cache_slab = KMEM_CACHE(jffs2_inode_cache, 0); + if (!inode_cache_slab) + goto err; + +#ifdef CONFIG_JFFS2_FS_XATTR + xattr_datum_cache = KMEM_CACHE(jffs2_xattr_datum, 0); + if (!xattr_datum_cache) + goto err; + + xattr_ref_cache = KMEM_CACHE(jffs2_xattr_ref, 0); + if (!xattr_ref_cache) + goto err; +#endif + + return 0; + err: + jffs2_destroy_slab_caches(); + return -ENOMEM; +} + +void jffs2_destroy_slab_caches(void) +{ + kmem_cache_destroy(full_dnode_slab); + kmem_cache_destroy(raw_dirent_slab); + kmem_cache_destroy(raw_inode_slab); + kmem_cache_destroy(tmp_dnode_info_slab); + kmem_cache_destroy(raw_node_ref_slab); + kmem_cache_destroy(node_frag_slab); + kmem_cache_destroy(inode_cache_slab); +#ifdef CONFIG_JFFS2_FS_XATTR + kmem_cache_destroy(xattr_datum_cache); + kmem_cache_destroy(xattr_ref_cache); +#endif +} + +struct jffs2_full_dirent *jffs2_alloc_full_dirent(int namesize) +{ + struct jffs2_full_dirent *ret; + ret = kmalloc(sizeof(struct jffs2_full_dirent) + namesize, GFP_KERNEL); + dbg_memalloc("%p\n", ret); + return ret; +} + +void jffs2_free_full_dirent(struct jffs2_full_dirent *x) +{ + dbg_memalloc("%p\n", x); + kfree(x); +} + +struct jffs2_full_dnode *jffs2_alloc_full_dnode(void) +{ + struct jffs2_full_dnode *ret; + ret = kmem_cache_alloc(full_dnode_slab, GFP_KERNEL); + dbg_memalloc("%p\n", ret); + return ret; +} + +void jffs2_free_full_dnode(struct jffs2_full_dnode *x) +{ + dbg_memalloc("%p\n", x); + kmem_cache_free(full_dnode_slab, x); +} + +struct jffs2_raw_dirent *jffs2_alloc_raw_dirent(void) +{ + struct jffs2_raw_dirent *ret; + ret = kmem_cache_alloc(raw_dirent_slab, GFP_KERNEL); + dbg_memalloc("%p\n", ret); + return ret; +} + +void jffs2_free_raw_dirent(struct jffs2_raw_dirent *x) +{ + dbg_memalloc("%p\n", x); + kmem_cache_free(raw_dirent_slab, x); +} + +struct jffs2_raw_inode *jffs2_alloc_raw_inode(void) +{ + struct jffs2_raw_inode *ret; + ret = kmem_cache_alloc(raw_inode_slab, GFP_KERNEL); + dbg_memalloc("%p\n", ret); + return ret; +} + +void jffs2_free_raw_inode(struct jffs2_raw_inode *x) +{ + dbg_memalloc("%p\n", x); + kmem_cache_free(raw_inode_slab, x); +} + +struct jffs2_tmp_dnode_info *jffs2_alloc_tmp_dnode_info(void) +{ + struct jffs2_tmp_dnode_info *ret; + ret = kmem_cache_alloc(tmp_dnode_info_slab, GFP_KERNEL); + dbg_memalloc("%p\n", + ret); + return ret; +} + +void jffs2_free_tmp_dnode_info(struct jffs2_tmp_dnode_info *x) +{ + dbg_memalloc("%p\n", x); + kmem_cache_free(tmp_dnode_info_slab, x); +} + +static struct jffs2_raw_node_ref *jffs2_alloc_refblock(void) +{ + struct jffs2_raw_node_ref *ret; + + ret = kmem_cache_alloc(raw_node_ref_slab, GFP_KERNEL); + if (ret) { + int i = 0; + for (i=0; i < REFS_PER_BLOCK; i++) { + ret[i].flash_offset = REF_EMPTY_NODE; + ret[i].next_in_ino = NULL; + } + ret[i].flash_offset = REF_LINK_NODE; + ret[i].next_in_ino = NULL; + } + return ret; +} + +int jffs2_prealloc_raw_node_refs(struct jffs2_sb_info *c, + struct jffs2_eraseblock *jeb, int nr) +{ + struct jffs2_raw_node_ref **p, *ref; + int i = nr; + + dbg_memalloc("%d\n", nr); + + p = &jeb->last_node; + ref = *p; + + dbg_memalloc("Reserving %d refs for block @0x%08x\n", nr, jeb->offset); + + /* If jeb->last_node is really a valid node then skip over it */ + if (ref && ref->flash_offset != REF_EMPTY_NODE) + ref++; + + while (i) { + if (!ref) { + dbg_memalloc("Allocating new refblock linked from %p\n", p); + ref = *p = jffs2_alloc_refblock(); + if (!ref) + return -ENOMEM; + } + if (ref->flash_offset == REF_LINK_NODE) { + p = &ref->next_in_ino; + ref = *p; + continue; + } + i--; + ref++; + } + jeb->allocated_refs = nr; + + dbg_memalloc("Reserved %d refs for block @0x%08x, last_node is %p (%08x,%p)\n", + nr, jeb->offset, jeb->last_node, jeb->last_node->flash_offset, + jeb->last_node->next_in_ino); + + return 0; +} + +void jffs2_free_refblock(struct jffs2_raw_node_ref *x) +{ + dbg_memalloc("%p\n", x); + kmem_cache_free(raw_node_ref_slab, x); +} + +struct jffs2_node_frag *jffs2_alloc_node_frag(void) +{ + struct jffs2_node_frag *ret; + ret = kmem_cache_alloc(node_frag_slab, GFP_KERNEL); + dbg_memalloc("%p\n", ret); + return ret; +} + +void jffs2_free_node_frag(struct jffs2_node_frag *x) +{ + dbg_memalloc("%p\n", x); + kmem_cache_free(node_frag_slab, x); +} + +struct jffs2_inode_cache *jffs2_alloc_inode_cache(void) +{ + struct jffs2_inode_cache *ret; + ret = kmem_cache_alloc(inode_cache_slab, GFP_KERNEL); + dbg_memalloc("%p\n", ret); + return ret; +} + +void jffs2_free_inode_cache(struct jffs2_inode_cache *x) +{ + dbg_memalloc("%p\n", x); + kmem_cache_free(inode_cache_slab, x); +} + +#ifdef CONFIG_JFFS2_FS_XATTR +struct jffs2_xattr_datum *jffs2_alloc_xattr_datum(void) +{ + struct jffs2_xattr_datum *xd; + xd = kmem_cache_zalloc(xattr_datum_cache, GFP_KERNEL); + dbg_memalloc("%p\n", xd); + if (!xd) + return NULL; + + xd->class = RAWNODE_CLASS_XATTR_DATUM; + xd->node = (void *)xd; + INIT_LIST_HEAD(&xd->xindex); + return xd; +} + +void jffs2_free_xattr_datum(struct jffs2_xattr_datum *xd) +{ + dbg_memalloc("%p\n", xd); + kmem_cache_free(xattr_datum_cache, xd); +} + +struct jffs2_xattr_ref *jffs2_alloc_xattr_ref(void) +{ + struct jffs2_xattr_ref *ref; + ref = kmem_cache_zalloc(xattr_ref_cache, GFP_KERNEL); + dbg_memalloc("%p\n", ref); + if (!ref) + return NULL; + + ref->class = RAWNODE_CLASS_XATTR_REF; + ref->node = (void *)ref; + return ref; +} + +void jffs2_free_xattr_ref(struct jffs2_xattr_ref *ref) +{ + dbg_memalloc("%p\n", ref); + kmem_cache_free(xattr_ref_cache, ref); +} +#endif diff --git a/fs/jffs2/nodelist.c b/fs/jffs2/nodelist.c new file mode 100644 index 000000000..b86c78d17 --- /dev/null +++ b/fs/jffs2/nodelist.c @@ -0,0 +1,755 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * + * Created by David Woodhouse + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include "nodelist.h" + +static void jffs2_obsolete_node_frag(struct jffs2_sb_info *c, + struct jffs2_node_frag *this); + +void jffs2_add_fd_to_list(struct jffs2_sb_info *c, struct jffs2_full_dirent *new, struct jffs2_full_dirent **list) +{ + struct jffs2_full_dirent **prev = list; + + dbg_dentlist("add dirent \"%s\", ino #%u\n", new->name, new->ino); + + while ((*prev) && (*prev)->nhash <= new->nhash) { + if ((*prev)->nhash == new->nhash && !strcmp((*prev)->name, new->name)) { + /* Duplicate. Free one */ + if (new->version < (*prev)->version) { + dbg_dentlist("Eep! Marking new dirent node obsolete, old is \"%s\", ino #%u\n", + (*prev)->name, (*prev)->ino); + jffs2_mark_node_obsolete(c, new->raw); + jffs2_free_full_dirent(new); + } else { + dbg_dentlist("marking old dirent \"%s\", ino #%u obsolete\n", + (*prev)->name, (*prev)->ino); + new->next = (*prev)->next; + /* It may have been a 'placeholder' deletion dirent, + if jffs2_can_mark_obsolete() (see jffs2_do_unlink()) */ + if ((*prev)->raw) + jffs2_mark_node_obsolete(c, ((*prev)->raw)); + jffs2_free_full_dirent(*prev); + *prev = new; + } + return; + } + prev = &((*prev)->next); + } + new->next = *prev; + *prev = new; +} + +uint32_t jffs2_truncate_fragtree(struct jffs2_sb_info *c, struct rb_root *list, uint32_t size) +{ + struct jffs2_node_frag *frag = jffs2_lookup_node_frag(list, size); + + dbg_fragtree("truncating fragtree to 0x%08x bytes\n", size); + + /* We know frag->ofs <= size. That's what lookup does for us */ + if (frag && frag->ofs != size) { + if (frag->ofs+frag->size > size) { + frag->size = size - frag->ofs; + } + frag = frag_next(frag); + } + while (frag && frag->ofs >= size) { + struct jffs2_node_frag *next = frag_next(frag); + + frag_erase(frag, list); + jffs2_obsolete_node_frag(c, frag); + frag = next; + } + + if (size == 0) + return 0; + + frag = frag_last(list); + + /* Sanity check for truncation to longer than we started with... */ + if (!frag) + return 0; + if (frag->ofs + frag->size < size) + return frag->ofs + frag->size; + + /* If the last fragment starts at the RAM page boundary, it is + * REF_PRISTINE irrespective of its size. */ + if (frag->node && (frag->ofs & (PAGE_SIZE - 1)) == 0) { + dbg_fragtree2("marking the last fragment 0x%08x-0x%08x REF_PRISTINE.\n", + frag->ofs, frag->ofs + frag->size); + frag->node->raw->flash_offset = ref_offset(frag->node->raw) | REF_PRISTINE; + } + return size; +} + +static void jffs2_obsolete_node_frag(struct jffs2_sb_info *c, + struct jffs2_node_frag *this) +{ + if (this->node) { + this->node->frags--; + if (!this->node->frags) { + /* The node has no valid frags left. It's totally obsoleted */ + dbg_fragtree2("marking old node @0x%08x (0x%04x-0x%04x) obsolete\n", + ref_offset(this->node->raw), this->node->ofs, this->node->ofs+this->node->size); + jffs2_mark_node_obsolete(c, this->node->raw); + jffs2_free_full_dnode(this->node); + } else { + dbg_fragtree2("marking old node @0x%08x (0x%04x-0x%04x) REF_NORMAL. frags is %d\n", + ref_offset(this->node->raw), this->node->ofs, this->node->ofs+this->node->size, this->node->frags); + mark_ref_normal(this->node->raw); + } + + } + jffs2_free_node_frag(this); +} + +static void jffs2_fragtree_insert(struct jffs2_node_frag *newfrag, struct jffs2_node_frag *base) +{ + struct rb_node *parent = &base->rb; + struct rb_node **link = &parent; + + dbg_fragtree2("insert frag (0x%04x-0x%04x)\n", newfrag->ofs, newfrag->ofs + newfrag->size); + + while (*link) { + parent = *link; + base = rb_entry(parent, struct jffs2_node_frag, rb); + + if (newfrag->ofs > base->ofs) + link = &base->rb.rb_right; + else if (newfrag->ofs < base->ofs) + link = &base->rb.rb_left; + else { + JFFS2_ERROR("duplicate frag at %08x (%p,%p)\n", newfrag->ofs, newfrag, base); + BUG(); + } + } + + rb_link_node(&newfrag->rb, &base->rb, link); +} + +/* + * Allocate and initializes a new fragment. + */ +static struct jffs2_node_frag * new_fragment(struct jffs2_full_dnode *fn, uint32_t ofs, uint32_t size) +{ + struct jffs2_node_frag *newfrag; + + newfrag = jffs2_alloc_node_frag(); + if (likely(newfrag)) { + newfrag->ofs = ofs; + newfrag->size = size; + newfrag->node = fn; + } else { + JFFS2_ERROR("cannot allocate a jffs2_node_frag object\n"); + } + + return newfrag; +} + +/* + * Called when there is no overlapping fragment exist. Inserts a hole before the new + * fragment and inserts the new fragment to the fragtree. + */ +static int no_overlapping_node(struct jffs2_sb_info *c, struct rb_root *root, + struct jffs2_node_frag *newfrag, + struct jffs2_node_frag *this, uint32_t lastend) +{ + if (lastend < newfrag->node->ofs) { + /* put a hole in before the new fragment */ + struct jffs2_node_frag *holefrag; + + holefrag= new_fragment(NULL, lastend, newfrag->node->ofs - lastend); + if (unlikely(!holefrag)) { + jffs2_free_node_frag(newfrag); + return -ENOMEM; + } + + if (this) { + /* By definition, the 'this' node has no right-hand child, + because there are no frags with offset greater than it. + So that's where we want to put the hole */ + dbg_fragtree2("add hole frag %#04x-%#04x on the right of the new frag.\n", + holefrag->ofs, holefrag->ofs + holefrag->size); + rb_link_node(&holefrag->rb, &this->rb, &this->rb.rb_right); + } else { + dbg_fragtree2("Add hole frag %#04x-%#04x to the root of the tree.\n", + holefrag->ofs, holefrag->ofs + holefrag->size); + rb_link_node(&holefrag->rb, NULL, &root->rb_node); + } + rb_insert_color(&holefrag->rb, root); + this = holefrag; + } + + if (this) { + /* By definition, the 'this' node has no right-hand child, + because there are no frags with offset greater than it. + So that's where we want to put new fragment */ + dbg_fragtree2("add the new node at the right\n"); + rb_link_node(&newfrag->rb, &this->rb, &this->rb.rb_right); + } else { + dbg_fragtree2("insert the new node at the root of the tree\n"); + rb_link_node(&newfrag->rb, NULL, &root->rb_node); + } + rb_insert_color(&newfrag->rb, root); + + return 0; +} + +/* Doesn't set inode->i_size */ +static int jffs2_add_frag_to_fragtree(struct jffs2_sb_info *c, struct rb_root *root, struct jffs2_node_frag *newfrag) +{ + struct jffs2_node_frag *this; + uint32_t lastend; + + /* Skip all the nodes which are completed before this one starts */ + this = jffs2_lookup_node_frag(root, newfrag->node->ofs); + + if (this) { + dbg_fragtree2("lookup gave frag 0x%04x-0x%04x; phys 0x%08x (*%p)\n", + this->ofs, this->ofs+this->size, this->node?(ref_offset(this->node->raw)):0xffffffff, this); + lastend = this->ofs + this->size; + } else { + dbg_fragtree2("lookup gave no frag\n"); + lastend = 0; + } + + /* See if we ran off the end of the fragtree */ + if (lastend <= newfrag->ofs) { + /* We did */ + + /* Check if 'this' node was on the same page as the new node. + If so, both 'this' and the new node get marked REF_NORMAL so + the GC can take a look. + */ + if (lastend && (lastend-1) >> PAGE_SHIFT == newfrag->ofs >> PAGE_SHIFT) { + if (this->node) + mark_ref_normal(this->node->raw); + mark_ref_normal(newfrag->node->raw); + } + + return no_overlapping_node(c, root, newfrag, this, lastend); + } + + if (this->node) + dbg_fragtree2("dealing with frag %u-%u, phys %#08x(%d).\n", + this->ofs, this->ofs + this->size, + ref_offset(this->node->raw), ref_flags(this->node->raw)); + else + dbg_fragtree2("dealing with hole frag %u-%u.\n", + this->ofs, this->ofs + this->size); + + /* OK. 'this' is pointing at the first frag that newfrag->ofs at least partially obsoletes, + * - i.e. newfrag->ofs < this->ofs+this->size && newfrag->ofs >= this->ofs + */ + if (newfrag->ofs > this->ofs) { + /* This node isn't completely obsoleted. The start of it remains valid */ + + /* Mark the new node and the partially covered node REF_NORMAL -- let + the GC take a look at them */ + mark_ref_normal(newfrag->node->raw); + if (this->node) + mark_ref_normal(this->node->raw); + + if (this->ofs + this->size > newfrag->ofs + newfrag->size) { + /* The new node splits 'this' frag into two */ + struct jffs2_node_frag *newfrag2; + + if (this->node) + dbg_fragtree2("split old frag 0x%04x-0x%04x, phys 0x%08x\n", + this->ofs, this->ofs+this->size, ref_offset(this->node->raw)); + else + dbg_fragtree2("split old hole frag 0x%04x-0x%04x\n", + this->ofs, this->ofs+this->size); + + /* New second frag pointing to this's node */ + newfrag2 = new_fragment(this->node, newfrag->ofs + newfrag->size, + this->ofs + this->size - newfrag->ofs - newfrag->size); + if (unlikely(!newfrag2)) + return -ENOMEM; + if (this->node) + this->node->frags++; + + /* Adjust size of original 'this' */ + this->size = newfrag->ofs - this->ofs; + + /* Now, we know there's no node with offset + greater than this->ofs but smaller than + newfrag2->ofs or newfrag->ofs, for obvious + reasons. So we can do a tree insert from + 'this' to insert newfrag, and a tree insert + from newfrag to insert newfrag2. */ + jffs2_fragtree_insert(newfrag, this); + rb_insert_color(&newfrag->rb, root); + + jffs2_fragtree_insert(newfrag2, newfrag); + rb_insert_color(&newfrag2->rb, root); + + return 0; + } + /* New node just reduces 'this' frag in size, doesn't split it */ + this->size = newfrag->ofs - this->ofs; + + /* Again, we know it lives down here in the tree */ + jffs2_fragtree_insert(newfrag, this); + rb_insert_color(&newfrag->rb, root); + } else { + /* New frag starts at the same point as 'this' used to. Replace + it in the tree without doing a delete and insertion */ + dbg_fragtree2("inserting newfrag (*%p),%d-%d in before 'this' (*%p),%d-%d\n", + newfrag, newfrag->ofs, newfrag->ofs+newfrag->size, this, this->ofs, this->ofs+this->size); + + rb_replace_node(&this->rb, &newfrag->rb, root); + + if (newfrag->ofs + newfrag->size >= this->ofs+this->size) { + dbg_fragtree2("obsoleting node frag %p (%x-%x)\n", this, this->ofs, this->ofs+this->size); + jffs2_obsolete_node_frag(c, this); + } else { + this->ofs += newfrag->size; + this->size -= newfrag->size; + + jffs2_fragtree_insert(this, newfrag); + rb_insert_color(&this->rb, root); + return 0; + } + } + /* OK, now we have newfrag added in the correct place in the tree, but + frag_next(newfrag) may be a fragment which is overlapped by it + */ + while ((this = frag_next(newfrag)) && newfrag->ofs + newfrag->size >= this->ofs + this->size) { + /* 'this' frag is obsoleted completely. */ + dbg_fragtree2("obsoleting node frag %p (%x-%x) and removing from tree\n", + this, this->ofs, this->ofs+this->size); + rb_erase(&this->rb, root); + jffs2_obsolete_node_frag(c, this); + } + /* Now we're pointing at the first frag which isn't totally obsoleted by + the new frag */ + + if (!this || newfrag->ofs + newfrag->size == this->ofs) + return 0; + + /* Still some overlap but we don't need to move it in the tree */ + this->size = (this->ofs + this->size) - (newfrag->ofs + newfrag->size); + this->ofs = newfrag->ofs + newfrag->size; + + /* And mark them REF_NORMAL so the GC takes a look at them */ + if (this->node) + mark_ref_normal(this->node->raw); + mark_ref_normal(newfrag->node->raw); + + return 0; +} + +/* + * Given an inode, probably with existing tree of fragments, add the new node + * to the fragment tree. + */ +int jffs2_add_full_dnode_to_inode(struct jffs2_sb_info *c, struct jffs2_inode_info *f, struct jffs2_full_dnode *fn) +{ + int ret; + struct jffs2_node_frag *newfrag; + + if (unlikely(!fn->size)) + return 0; + + newfrag = new_fragment(fn, fn->ofs, fn->size); + if (unlikely(!newfrag)) + return -ENOMEM; + newfrag->node->frags = 1; + + dbg_fragtree("adding node %#04x-%#04x @0x%08x on flash, newfrag *%p\n", + fn->ofs, fn->ofs+fn->size, ref_offset(fn->raw), newfrag); + + ret = jffs2_add_frag_to_fragtree(c, &f->fragtree, newfrag); + if (unlikely(ret)) + return ret; + + /* If we now share a page with other nodes, mark either previous + or next node REF_NORMAL, as appropriate. */ + if (newfrag->ofs & (PAGE_SIZE-1)) { + struct jffs2_node_frag *prev = frag_prev(newfrag); + + mark_ref_normal(fn->raw); + /* If we don't start at zero there's _always_ a previous */ + if (prev->node) + mark_ref_normal(prev->node->raw); + } + + if ((newfrag->ofs+newfrag->size) & (PAGE_SIZE-1)) { + struct jffs2_node_frag *next = frag_next(newfrag); + + if (next) { + mark_ref_normal(fn->raw); + if (next->node) + mark_ref_normal(next->node->raw); + } + } + jffs2_dbg_fragtree_paranoia_check_nolock(f); + + return 0; +} + +void jffs2_set_inocache_state(struct jffs2_sb_info *c, struct jffs2_inode_cache *ic, int state) +{ + spin_lock(&c->inocache_lock); + ic->state = state; + wake_up(&c->inocache_wq); + spin_unlock(&c->inocache_lock); +} + +/* During mount, this needs no locking. During normal operation, its + callers want to do other stuff while still holding the inocache_lock. + Rather than introducing special case get_ino_cache functions or + callbacks, we just let the caller do the locking itself. */ + +struct jffs2_inode_cache *jffs2_get_ino_cache(struct jffs2_sb_info *c, uint32_t ino) +{ + struct jffs2_inode_cache *ret; + + ret = c->inocache_list[ino % c->inocache_hashsize]; + while (ret && ret->ino < ino) { + ret = ret->next; + } + + if (ret && ret->ino != ino) + ret = NULL; + + return ret; +} + +void jffs2_add_ino_cache (struct jffs2_sb_info *c, struct jffs2_inode_cache *new) +{ + struct jffs2_inode_cache **prev; + + spin_lock(&c->inocache_lock); + if (!new->ino) + new->ino = ++c->highest_ino; + + dbg_inocache("add %p (ino #%u)\n", new, new->ino); + + prev = &c->inocache_list[new->ino % c->inocache_hashsize]; + + while ((*prev) && (*prev)->ino < new->ino) { + prev = &(*prev)->next; + } + new->next = *prev; + *prev = new; + + spin_unlock(&c->inocache_lock); +} + +void jffs2_del_ino_cache(struct jffs2_sb_info *c, struct jffs2_inode_cache *old) +{ + struct jffs2_inode_cache **prev; + +#ifdef CONFIG_JFFS2_FS_XATTR + BUG_ON(old->xref); +#endif + dbg_inocache("del %p (ino #%u)\n", old, old->ino); + spin_lock(&c->inocache_lock); + + prev = &c->inocache_list[old->ino % c->inocache_hashsize]; + + while ((*prev) && (*prev)->ino < old->ino) { + prev = &(*prev)->next; + } + if ((*prev) == old) { + *prev = old->next; + } + + /* Free it now unless it's in READING or CLEARING state, which + are the transitions upon read_inode() and clear_inode(). The + rest of the time we know nobody else is looking at it, and + if it's held by read_inode() or clear_inode() they'll free it + for themselves. */ + if (old->state != INO_STATE_READING && old->state != INO_STATE_CLEARING) + jffs2_free_inode_cache(old); + + spin_unlock(&c->inocache_lock); +} + +void jffs2_free_ino_caches(struct jffs2_sb_info *c) +{ + int i; + struct jffs2_inode_cache *this, *next; + + for (i=0; i < c->inocache_hashsize; i++) { + this = c->inocache_list[i]; + while (this) { + next = this->next; + jffs2_xattr_free_inode(c, this); + jffs2_free_inode_cache(this); + this = next; + } + c->inocache_list[i] = NULL; + } +} + +void jffs2_free_raw_node_refs(struct jffs2_sb_info *c) +{ + int i; + struct jffs2_raw_node_ref *this, *next; + + for (i=0; inr_blocks; i++) { + this = c->blocks[i].first_node; + while (this) { + if (this[REFS_PER_BLOCK].flash_offset == REF_LINK_NODE) + next = this[REFS_PER_BLOCK].next_in_ino; + else + next = NULL; + + jffs2_free_refblock(this); + this = next; + } + c->blocks[i].first_node = c->blocks[i].last_node = NULL; + } +} + +struct jffs2_node_frag *jffs2_lookup_node_frag(struct rb_root *fragtree, uint32_t offset) +{ + /* The common case in lookup is that there will be a node + which precisely matches. So we go looking for that first */ + struct rb_node *next; + struct jffs2_node_frag *prev = NULL; + struct jffs2_node_frag *frag = NULL; + + dbg_fragtree2("root %p, offset %d\n", fragtree, offset); + + next = fragtree->rb_node; + + while(next) { + frag = rb_entry(next, struct jffs2_node_frag, rb); + + if (frag->ofs + frag->size <= offset) { + /* Remember the closest smaller match on the way down */ + if (!prev || frag->ofs > prev->ofs) + prev = frag; + next = frag->rb.rb_right; + } else if (frag->ofs > offset) { + next = frag->rb.rb_left; + } else { + return frag; + } + } + + /* Exact match not found. Go back up looking at each parent, + and return the closest smaller one */ + + if (prev) + dbg_fragtree2("no match. Returning frag %#04x-%#04x, closest previous\n", + prev->ofs, prev->ofs+prev->size); + else + dbg_fragtree2("returning NULL, empty fragtree\n"); + + return prev; +} + +/* Pass 'c' argument to indicate that nodes should be marked obsolete as + they're killed. */ +void jffs2_kill_fragtree(struct rb_root *root, struct jffs2_sb_info *c) +{ + struct jffs2_node_frag *frag, *next; + + dbg_fragtree("killing\n"); + rbtree_postorder_for_each_entry_safe(frag, next, root, rb) { + if (frag->node && !(--frag->node->frags)) { + /* Not a hole, and it's the final remaining frag + of this node. Free the node */ + if (c) + jffs2_mark_node_obsolete(c, frag->node->raw); + + jffs2_free_full_dnode(frag->node); + } + + jffs2_free_node_frag(frag); + cond_resched(); + } +} + +struct jffs2_raw_node_ref *jffs2_link_node_ref(struct jffs2_sb_info *c, + struct jffs2_eraseblock *jeb, + uint32_t ofs, uint32_t len, + struct jffs2_inode_cache *ic) +{ + struct jffs2_raw_node_ref *ref; + + BUG_ON(!jeb->allocated_refs); + jeb->allocated_refs--; + + ref = jeb->last_node; + + dbg_noderef("Last node at %p is (%08x,%p)\n", ref, ref->flash_offset, + ref->next_in_ino); + + while (ref->flash_offset != REF_EMPTY_NODE) { + if (ref->flash_offset == REF_LINK_NODE) + ref = ref->next_in_ino; + else + ref++; + } + + dbg_noderef("New ref is %p (%08x becomes %08x,%p) len 0x%x\n", ref, + ref->flash_offset, ofs, ref->next_in_ino, len); + + ref->flash_offset = ofs; + + if (!jeb->first_node) { + jeb->first_node = ref; + BUG_ON(ref_offset(ref) != jeb->offset); + } else if (unlikely(ref_offset(ref) != jeb->offset + c->sector_size - jeb->free_size)) { + uint32_t last_len = ref_totlen(c, jeb, jeb->last_node); + + JFFS2_ERROR("Adding new ref %p at (0x%08x-0x%08x) not immediately after previous (0x%08x-0x%08x)\n", + ref, ref_offset(ref), ref_offset(ref)+len, + ref_offset(jeb->last_node), + ref_offset(jeb->last_node)+last_len); + BUG(); + } + jeb->last_node = ref; + + if (ic) { + ref->next_in_ino = ic->nodes; + ic->nodes = ref; + } else { + ref->next_in_ino = NULL; + } + + switch(ref_flags(ref)) { + case REF_UNCHECKED: + c->unchecked_size += len; + jeb->unchecked_size += len; + break; + + case REF_NORMAL: + case REF_PRISTINE: + c->used_size += len; + jeb->used_size += len; + break; + + case REF_OBSOLETE: + c->dirty_size += len; + jeb->dirty_size += len; + break; + } + c->free_size -= len; + jeb->free_size -= len; + +#ifdef TEST_TOTLEN + /* Set (and test) __totlen field... for now */ + ref->__totlen = len; + ref_totlen(c, jeb, ref); +#endif + return ref; +} + +/* No locking, no reservation of 'ref'. Do not use on a live file system */ +int jffs2_scan_dirty_space(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, + uint32_t size) +{ + if (!size) + return 0; + if (unlikely(size > jeb->free_size)) { + pr_crit("Dirty space 0x%x larger then free_size 0x%x (wasted 0x%x)\n", + size, jeb->free_size, jeb->wasted_size); + BUG(); + } + /* REF_EMPTY_NODE is !obsolete, so that works OK */ + if (jeb->last_node && ref_obsolete(jeb->last_node)) { +#ifdef TEST_TOTLEN + jeb->last_node->__totlen += size; +#endif + c->dirty_size += size; + c->free_size -= size; + jeb->dirty_size += size; + jeb->free_size -= size; + } else { + uint32_t ofs = jeb->offset + c->sector_size - jeb->free_size; + ofs |= REF_OBSOLETE; + + jffs2_link_node_ref(c, jeb, ofs, size, NULL); + } + + return 0; +} + +/* Calculate totlen from surrounding nodes or eraseblock */ +static inline uint32_t __ref_totlen(struct jffs2_sb_info *c, + struct jffs2_eraseblock *jeb, + struct jffs2_raw_node_ref *ref) +{ + uint32_t ref_end; + struct jffs2_raw_node_ref *next_ref = ref_next(ref); + + if (next_ref) + ref_end = ref_offset(next_ref); + else { + if (!jeb) + jeb = &c->blocks[ref->flash_offset / c->sector_size]; + + /* Last node in block. Use free_space */ + if (unlikely(ref != jeb->last_node)) { + pr_crit("ref %p @0x%08x is not jeb->last_node (%p @0x%08x)\n", + ref, ref_offset(ref), jeb->last_node, + jeb->last_node ? + ref_offset(jeb->last_node) : 0); + BUG(); + } + ref_end = jeb->offset + c->sector_size - jeb->free_size; + } + return ref_end - ref_offset(ref); +} + +uint32_t __jffs2_ref_totlen(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, + struct jffs2_raw_node_ref *ref) +{ + uint32_t ret; + + ret = __ref_totlen(c, jeb, ref); + +#ifdef TEST_TOTLEN + if (unlikely(ret != ref->__totlen)) { + if (!jeb) + jeb = &c->blocks[ref->flash_offset / c->sector_size]; + + pr_crit("Totlen for ref at %p (0x%08x-0x%08x) miscalculated as 0x%x instead of %x\n", + ref, ref_offset(ref), ref_offset(ref) + ref->__totlen, + ret, ref->__totlen); + if (ref_next(ref)) { + pr_crit("next %p (0x%08x-0x%08x)\n", + ref_next(ref), ref_offset(ref_next(ref)), + ref_offset(ref_next(ref)) + ref->__totlen); + } else + pr_crit("No next ref. jeb->last_node is %p\n", + jeb->last_node); + + pr_crit("jeb->wasted_size %x, dirty_size %x, used_size %x, free_size %x\n", + jeb->wasted_size, jeb->dirty_size, jeb->used_size, + jeb->free_size); + +#if defined(JFFS2_DBG_DUMPS) || defined(JFFS2_DBG_PARANOIA_CHECKS) + __jffs2_dbg_dump_node_refs_nolock(c, jeb); +#endif + + WARN_ON(1); + + ret = ref->__totlen; + } +#endif /* TEST_TOTLEN */ + return ret; +} diff --git a/fs/jffs2/nodelist.h b/fs/jffs2/nodelist.h new file mode 100644 index 000000000..2e98fa277 --- /dev/null +++ b/fs/jffs2/nodelist.h @@ -0,0 +1,484 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * + * Created by David Woodhouse + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#ifndef __JFFS2_NODELIST_H__ +#define __JFFS2_NODELIST_H__ + +#include +#include +#include +#include "jffs2_fs_sb.h" +#include "jffs2_fs_i.h" +#include "xattr.h" +#include "acl.h" +#include "summary.h" + +#ifdef __ECOS +#include "os-ecos.h" +#else +#include "os-linux.h" +#endif + +#define JFFS2_NATIVE_ENDIAN + +/* Note we handle mode bits conversion from JFFS2 (i.e. Linux) to/from + whatever OS we're actually running on here too. */ + +#if defined(JFFS2_NATIVE_ENDIAN) +#define cpu_to_je16(x) ((jint16_t){x}) +#define cpu_to_je32(x) ((jint32_t){x}) +#define cpu_to_jemode(x) ((jmode_t){os_to_jffs2_mode(x)}) + +#define constant_cpu_to_je16(x) ((jint16_t){x}) +#define constant_cpu_to_je32(x) ((jint32_t){x}) + +#define je16_to_cpu(x) ((x).v16) +#define je32_to_cpu(x) ((x).v32) +#define jemode_to_cpu(x) (jffs2_to_os_mode((x).m)) +#elif defined(JFFS2_BIG_ENDIAN) +#define cpu_to_je16(x) ((jint16_t){cpu_to_be16(x)}) +#define cpu_to_je32(x) ((jint32_t){cpu_to_be32(x)}) +#define cpu_to_jemode(x) ((jmode_t){cpu_to_be32(os_to_jffs2_mode(x))}) + +#define constant_cpu_to_je16(x) ((jint16_t){__constant_cpu_to_be16(x)}) +#define constant_cpu_to_je32(x) ((jint32_t){__constant_cpu_to_be32(x)}) + +#define je16_to_cpu(x) (be16_to_cpu(x.v16)) +#define je32_to_cpu(x) (be32_to_cpu(x.v32)) +#define jemode_to_cpu(x) (be32_to_cpu(jffs2_to_os_mode((x).m))) +#elif defined(JFFS2_LITTLE_ENDIAN) +#define cpu_to_je16(x) ((jint16_t){cpu_to_le16(x)}) +#define cpu_to_je32(x) ((jint32_t){cpu_to_le32(x)}) +#define cpu_to_jemode(x) ((jmode_t){cpu_to_le32(os_to_jffs2_mode(x))}) + +#define constant_cpu_to_je16(x) ((jint16_t){__constant_cpu_to_le16(x)}) +#define constant_cpu_to_je32(x) ((jint32_t){__constant_cpu_to_le32(x)}) + +#define je16_to_cpu(x) (le16_to_cpu(x.v16)) +#define je32_to_cpu(x) (le32_to_cpu(x.v32)) +#define jemode_to_cpu(x) (le32_to_cpu(jffs2_to_os_mode((x).m))) +#else +#error wibble +#endif + +/* The minimal node header size */ +#define JFFS2_MIN_NODE_HEADER sizeof(struct jffs2_raw_dirent) + +/* + This is all we need to keep in-core for each raw node during normal + operation. As and when we do read_inode on a particular inode, we can + scan the nodes which are listed for it and build up a proper map of + which nodes are currently valid. JFFSv1 always used to keep that whole + map in core for each inode. +*/ +struct jffs2_raw_node_ref +{ + struct jffs2_raw_node_ref *next_in_ino; /* Points to the next raw_node_ref + for this object. If this _is_ the last, it points to the inode_cache, + xattr_ref or xattr_datum instead. The common part of those structures + has NULL in the first word. See jffs2_raw_ref_to_ic() below */ + uint32_t flash_offset; +#undef TEST_TOTLEN +#ifdef TEST_TOTLEN + uint32_t __totlen; /* This may die; use ref_totlen(c, jeb, ) below */ +#endif +}; + +#define REF_LINK_NODE ((int32_t)-1) +#define REF_EMPTY_NODE ((int32_t)-2) + +/* Use blocks of about 256 bytes */ +#define REFS_PER_BLOCK ((255/sizeof(struct jffs2_raw_node_ref))-1) + +static inline struct jffs2_raw_node_ref *ref_next(struct jffs2_raw_node_ref *ref) +{ + ref++; + + /* Link to another block of refs */ + if (ref->flash_offset == REF_LINK_NODE) { + ref = ref->next_in_ino; + if (!ref) + return ref; + } + + /* End of chain */ + if (ref->flash_offset == REF_EMPTY_NODE) + return NULL; + + return ref; +} + +static inline struct jffs2_inode_cache *jffs2_raw_ref_to_ic(struct jffs2_raw_node_ref *raw) +{ + while(raw->next_in_ino) + raw = raw->next_in_ino; + + /* NB. This can be a jffs2_xattr_datum or jffs2_xattr_ref and + not actually a jffs2_inode_cache. Check ->class */ + return ((struct jffs2_inode_cache *)raw); +} + + /* flash_offset & 3 always has to be zero, because nodes are + always aligned at 4 bytes. So we have a couple of extra bits + to play with, which indicate the node's status; see below: */ +#define REF_UNCHECKED 0 /* We haven't yet checked the CRC or built its inode */ +#define REF_OBSOLETE 1 /* Obsolete, can be completely ignored */ +#define REF_PRISTINE 2 /* Completely clean. GC without looking */ +#define REF_NORMAL 3 /* Possibly overlapped. Read the page and write again on GC */ +#define ref_flags(ref) ((ref)->flash_offset & 3) +#define ref_offset(ref) ((ref)->flash_offset & ~3) +#define ref_obsolete(ref) (((ref)->flash_offset & 3) == REF_OBSOLETE) +#define mark_ref_normal(ref) do { (ref)->flash_offset = ref_offset(ref) | REF_NORMAL; } while(0) + +/* Dirent nodes should be REF_PRISTINE only if they are not a deletion + dirent. Deletion dirents should be REF_NORMAL so that GC gets to + throw them away when appropriate */ +#define dirent_node_state(rd) ( (je32_to_cpu((rd)->ino)?REF_PRISTINE:REF_NORMAL) ) + +/* NB: REF_PRISTINE for an inode-less node (ref->next_in_ino == NULL) indicates + it is an unknown node of type JFFS2_NODETYPE_RWCOMPAT_COPY, so it'll get + copied. If you need to do anything different to GC inode-less nodes, then + you need to modify gc.c accordingly. */ + +/* For each inode in the filesystem, we need to keep a record of + nlink, because it would be a PITA to scan the whole directory tree + at read_inode() time to calculate it, and to keep sufficient information + in the raw_node_ref (basically both parent and child inode number for + dirent nodes) would take more space than this does. We also keep + a pointer to the first physical node which is part of this inode, too. +*/ +struct jffs2_inode_cache { + /* First part of structure is shared with other objects which + can terminate the raw node refs' next_in_ino list -- which + currently struct jffs2_xattr_datum and struct jffs2_xattr_ref. */ + + struct jffs2_full_dirent *scan_dents; /* Used during scan to hold + temporary lists of dirents, and later must be set to + NULL to mark the end of the raw_node_ref->next_in_ino + chain. */ + struct jffs2_raw_node_ref *nodes; + uint8_t class; /* It's used for identification */ + + /* end of shared structure */ + + uint8_t flags; + uint16_t state; + uint32_t ino; + struct jffs2_inode_cache *next; +#ifdef CONFIG_JFFS2_FS_XATTR + struct jffs2_xattr_ref *xref; +#endif + uint32_t pino_nlink; /* Directories store parent inode + here; other inodes store nlink. + Zero always means that it's + completely unlinked. */ +}; + +/* Inode states for 'state' above. We need the 'GC' state to prevent + someone from doing a read_inode() while we're moving a 'REF_PRISTINE' + node without going through all the iget() nonsense */ +#define INO_STATE_UNCHECKED 0 /* CRC checks not yet done */ +#define INO_STATE_CHECKING 1 /* CRC checks in progress */ +#define INO_STATE_PRESENT 2 /* In core */ +#define INO_STATE_CHECKEDABSENT 3 /* Checked, cleared again */ +#define INO_STATE_GC 4 /* GCing a 'pristine' node */ +#define INO_STATE_READING 5 /* In read_inode() */ +#define INO_STATE_CLEARING 6 /* In clear_inode() */ + +#define INO_FLAGS_XATTR_CHECKED 0x01 /* has no duplicate xattr_ref */ +#define INO_FLAGS_IS_DIR 0x02 /* is a directory */ + +#define RAWNODE_CLASS_INODE_CACHE 0 +#define RAWNODE_CLASS_XATTR_DATUM 1 +#define RAWNODE_CLASS_XATTR_REF 2 + +#define INOCACHE_HASHSIZE_MIN 128 +#define INOCACHE_HASHSIZE_MAX 1024 + +#define write_ofs(c) ((c)->nextblock->offset + (c)->sector_size - (c)->nextblock->free_size) + +/* + Larger representation of a raw node, kept in-core only when the + struct inode for this particular ino is instantiated. +*/ + +struct jffs2_full_dnode +{ + struct jffs2_raw_node_ref *raw; + uint32_t ofs; /* The offset to which the data of this node belongs */ + uint32_t size; + uint32_t frags; /* Number of fragments which currently refer + to this node. When this reaches zero, + the node is obsolete. */ +}; + +/* + Even larger representation of a raw node, kept in-core only while + we're actually building up the original map of which nodes go where, + in read_inode() +*/ +struct jffs2_tmp_dnode_info +{ + struct rb_node rb; + struct jffs2_full_dnode *fn; + uint32_t version; + uint32_t data_crc; + uint32_t partial_crc; + uint32_t csize; + uint16_t overlapped; +}; + +/* Temporary data structure used during readinode. */ +struct jffs2_readinode_info +{ + struct rb_root tn_root; + struct jffs2_tmp_dnode_info *mdata_tn; + uint32_t highest_version; + uint32_t latest_mctime; + uint32_t mctime_ver; + struct jffs2_full_dirent *fds; + struct jffs2_raw_node_ref *latest_ref; +}; + +struct jffs2_full_dirent +{ + union { + struct jffs2_raw_node_ref *raw; + struct jffs2_inode_cache *ic; /* Just during part of build */ + }; + struct jffs2_full_dirent *next; + uint32_t version; + uint32_t ino; /* == zero for unlink */ + unsigned int nhash; + unsigned char type; + unsigned char name[]; +}; + +/* + Fragments - used to build a map of which raw node to obtain + data from for each part of the ino +*/ +struct jffs2_node_frag +{ + struct rb_node rb; + struct jffs2_full_dnode *node; /* NULL for holes */ + uint32_t size; + uint32_t ofs; /* The offset to which this fragment belongs */ +}; + +struct jffs2_eraseblock +{ + struct list_head list; + int bad_count; + uint32_t offset; /* of this block in the MTD */ + + uint32_t unchecked_size; + uint32_t used_size; + uint32_t dirty_size; + uint32_t wasted_size; + uint32_t free_size; /* Note that sector_size - free_size + is the address of the first free space */ + uint32_t allocated_refs; + struct jffs2_raw_node_ref *first_node; + struct jffs2_raw_node_ref *last_node; + + struct jffs2_raw_node_ref *gc_node; /* Next node to be garbage collected */ +}; + +static inline int jffs2_blocks_use_vmalloc(struct jffs2_sb_info *c) +{ + return ((c->flash_size / c->sector_size) * sizeof (struct jffs2_eraseblock)) > (128 * 1024); +} + +#define ref_totlen(a, b, c) __jffs2_ref_totlen((a), (b), (c)) + +#define ALLOC_NORMAL 0 /* Normal allocation */ +#define ALLOC_DELETION 1 /* Deletion node. Best to allow it */ +#define ALLOC_GC 2 /* Space requested for GC. Give it or die */ +#define ALLOC_NORETRY 3 /* For jffs2_write_dnode: On failure, return -EAGAIN instead of retrying */ + +/* How much dirty space before it goes on the very_dirty_list */ +#define VERYDIRTY(c, size) ((size) >= ((c)->sector_size / 2)) + +/* check if dirty space is more than 255 Byte */ +#define ISDIRTY(size) ((size) > sizeof (struct jffs2_raw_inode) + JFFS2_MIN_DATA_LEN) + +#define PAD(x) (((x)+3)&~3) + +static inline int jffs2_encode_dev(union jffs2_device_node *jdev, dev_t rdev) +{ + if (old_valid_dev(rdev)) { + jdev->old_id = cpu_to_je16(old_encode_dev(rdev)); + return sizeof(jdev->old_id); + } else { + jdev->new_id = cpu_to_je32(new_encode_dev(rdev)); + return sizeof(jdev->new_id); + } +} + +static inline struct jffs2_node_frag *frag_first(struct rb_root *root) +{ + struct rb_node *node = rb_first(root); + + if (!node) + return NULL; + + return rb_entry(node, struct jffs2_node_frag, rb); +} + +static inline struct jffs2_node_frag *frag_last(struct rb_root *root) +{ + struct rb_node *node = rb_last(root); + + if (!node) + return NULL; + + return rb_entry(node, struct jffs2_node_frag, rb); +} + +#define frag_next(frag) rb_entry(rb_next(&(frag)->rb), struct jffs2_node_frag, rb) +#define frag_prev(frag) rb_entry(rb_prev(&(frag)->rb), struct jffs2_node_frag, rb) +#define frag_parent(frag) rb_entry(rb_parent(&(frag)->rb), struct jffs2_node_frag, rb) +#define frag_left(frag) rb_entry((frag)->rb.rb_left, struct jffs2_node_frag, rb) +#define frag_right(frag) rb_entry((frag)->rb.rb_right, struct jffs2_node_frag, rb) +#define frag_erase(frag, list) rb_erase(&frag->rb, list) + +#define tn_next(tn) rb_entry(rb_next(&(tn)->rb), struct jffs2_tmp_dnode_info, rb) +#define tn_prev(tn) rb_entry(rb_prev(&(tn)->rb), struct jffs2_tmp_dnode_info, rb) +#define tn_parent(tn) rb_entry(rb_parent(&(tn)->rb), struct jffs2_tmp_dnode_info, rb) +#define tn_left(tn) rb_entry((tn)->rb.rb_left, struct jffs2_tmp_dnode_info, rb) +#define tn_right(tn) rb_entry((tn)->rb.rb_right, struct jffs2_tmp_dnode_info, rb) +#define tn_erase(tn, list) rb_erase(&tn->rb, list) +#define tn_last(list) rb_entry(rb_last(list), struct jffs2_tmp_dnode_info, rb) +#define tn_first(list) rb_entry(rb_first(list), struct jffs2_tmp_dnode_info, rb) + +/* nodelist.c */ +void jffs2_add_fd_to_list(struct jffs2_sb_info *c, struct jffs2_full_dirent *new, struct jffs2_full_dirent **list); +void jffs2_set_inocache_state(struct jffs2_sb_info *c, struct jffs2_inode_cache *ic, int state); +struct jffs2_inode_cache *jffs2_get_ino_cache(struct jffs2_sb_info *c, uint32_t ino); +void jffs2_add_ino_cache (struct jffs2_sb_info *c, struct jffs2_inode_cache *new); +void jffs2_del_ino_cache(struct jffs2_sb_info *c, struct jffs2_inode_cache *old); +void jffs2_free_ino_caches(struct jffs2_sb_info *c); +void jffs2_free_raw_node_refs(struct jffs2_sb_info *c); +struct jffs2_node_frag *jffs2_lookup_node_frag(struct rb_root *fragtree, uint32_t offset); +void jffs2_kill_fragtree(struct rb_root *root, struct jffs2_sb_info *c_delete); +int jffs2_add_full_dnode_to_inode(struct jffs2_sb_info *c, struct jffs2_inode_info *f, struct jffs2_full_dnode *fn); +uint32_t jffs2_truncate_fragtree (struct jffs2_sb_info *c, struct rb_root *list, uint32_t size); +struct jffs2_raw_node_ref *jffs2_link_node_ref(struct jffs2_sb_info *c, + struct jffs2_eraseblock *jeb, + uint32_t ofs, uint32_t len, + struct jffs2_inode_cache *ic); +extern uint32_t __jffs2_ref_totlen(struct jffs2_sb_info *c, + struct jffs2_eraseblock *jeb, + struct jffs2_raw_node_ref *ref); + +/* nodemgmt.c */ +int jffs2_thread_should_wake(struct jffs2_sb_info *c); +int jffs2_reserve_space(struct jffs2_sb_info *c, uint32_t minsize, + uint32_t *len, int prio, uint32_t sumsize); +int jffs2_reserve_space_gc(struct jffs2_sb_info *c, uint32_t minsize, + uint32_t *len, uint32_t sumsize); +struct jffs2_raw_node_ref *jffs2_add_physical_node_ref(struct jffs2_sb_info *c, + uint32_t ofs, uint32_t len, + struct jffs2_inode_cache *ic); +void jffs2_complete_reservation(struct jffs2_sb_info *c); +void jffs2_mark_node_obsolete(struct jffs2_sb_info *c, struct jffs2_raw_node_ref *raw); + +/* write.c */ +int jffs2_do_new_inode(struct jffs2_sb_info *c, struct jffs2_inode_info *f, uint32_t mode, struct jffs2_raw_inode *ri); + +struct jffs2_full_dnode *jffs2_write_dnode(struct jffs2_sb_info *c, struct jffs2_inode_info *f, + struct jffs2_raw_inode *ri, const unsigned char *data, + uint32_t datalen, int alloc_mode); +struct jffs2_full_dirent *jffs2_write_dirent(struct jffs2_sb_info *c, struct jffs2_inode_info *f, + struct jffs2_raw_dirent *rd, const unsigned char *name, + uint32_t namelen, int alloc_mode); +int jffs2_write_inode_range(struct jffs2_sb_info *c, struct jffs2_inode_info *f, + struct jffs2_raw_inode *ri, unsigned char *buf, + uint32_t offset, uint32_t writelen, uint32_t *retlen); +int jffs2_do_create(struct jffs2_sb_info *c, struct jffs2_inode_info *dir_f, struct jffs2_inode_info *f, + struct jffs2_raw_inode *ri, const struct qstr *qstr); +int jffs2_do_unlink(struct jffs2_sb_info *c, struct jffs2_inode_info *dir_f, const char *name, + int namelen, struct jffs2_inode_info *dead_f, uint32_t time); +int jffs2_do_link(struct jffs2_sb_info *c, struct jffs2_inode_info *dir_f, uint32_t ino, + uint8_t type, const char *name, int namelen, uint32_t time); + + +/* readinode.c */ +int jffs2_do_read_inode(struct jffs2_sb_info *c, struct jffs2_inode_info *f, + uint32_t ino, struct jffs2_raw_inode *latest_node); +int jffs2_do_crccheck_inode(struct jffs2_sb_info *c, struct jffs2_inode_cache *ic); +void jffs2_do_clear_inode(struct jffs2_sb_info *c, struct jffs2_inode_info *f); + +/* malloc.c */ +int jffs2_create_slab_caches(void); +void jffs2_destroy_slab_caches(void); + +struct jffs2_full_dirent *jffs2_alloc_full_dirent(int namesize); +void jffs2_free_full_dirent(struct jffs2_full_dirent *); +struct jffs2_full_dnode *jffs2_alloc_full_dnode(void); +void jffs2_free_full_dnode(struct jffs2_full_dnode *); +struct jffs2_raw_dirent *jffs2_alloc_raw_dirent(void); +void jffs2_free_raw_dirent(struct jffs2_raw_dirent *); +struct jffs2_raw_inode *jffs2_alloc_raw_inode(void); +void jffs2_free_raw_inode(struct jffs2_raw_inode *); +struct jffs2_tmp_dnode_info *jffs2_alloc_tmp_dnode_info(void); +void jffs2_free_tmp_dnode_info(struct jffs2_tmp_dnode_info *); +int jffs2_prealloc_raw_node_refs(struct jffs2_sb_info *c, + struct jffs2_eraseblock *jeb, int nr); +void jffs2_free_refblock(struct jffs2_raw_node_ref *); +struct jffs2_node_frag *jffs2_alloc_node_frag(void); +void jffs2_free_node_frag(struct jffs2_node_frag *); +struct jffs2_inode_cache *jffs2_alloc_inode_cache(void); +void jffs2_free_inode_cache(struct jffs2_inode_cache *); +#ifdef CONFIG_JFFS2_FS_XATTR +struct jffs2_xattr_datum *jffs2_alloc_xattr_datum(void); +void jffs2_free_xattr_datum(struct jffs2_xattr_datum *); +struct jffs2_xattr_ref *jffs2_alloc_xattr_ref(void); +void jffs2_free_xattr_ref(struct jffs2_xattr_ref *); +#endif + +/* gc.c */ +int jffs2_garbage_collect_pass(struct jffs2_sb_info *c); + +/* read.c */ +int jffs2_read_dnode(struct jffs2_sb_info *c, struct jffs2_inode_info *f, + struct jffs2_full_dnode *fd, unsigned char *buf, + int ofs, int len); +int jffs2_read_inode_range(struct jffs2_sb_info *c, struct jffs2_inode_info *f, + unsigned char *buf, uint32_t offset, uint32_t len); +char *jffs2_getlink(struct jffs2_sb_info *c, struct jffs2_inode_info *f); + +/* scan.c */ +int jffs2_scan_medium(struct jffs2_sb_info *c); +void jffs2_rotate_lists(struct jffs2_sb_info *c); +struct jffs2_inode_cache *jffs2_scan_make_ino_cache(struct jffs2_sb_info *c, uint32_t ino); +int jffs2_scan_classify_jeb(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb); +int jffs2_scan_dirty_space(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, uint32_t size); + +/* build.c */ +int jffs2_do_mount_fs(struct jffs2_sb_info *c); + +/* erase.c */ +int jffs2_erase_pending_blocks(struct jffs2_sb_info *c, int count); +void jffs2_free_jeb_node_refs(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb); + +#ifdef CONFIG_JFFS2_FS_WRITEBUFFER +/* wbuf.c */ +int jffs2_flush_wbuf_gc(struct jffs2_sb_info *c, uint32_t ino); +int jffs2_flush_wbuf_pad(struct jffs2_sb_info *c); +int jffs2_check_nand_cleanmarker(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb); +int jffs2_write_nand_cleanmarker(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb); +#endif + +#include "debug.h" + +#endif /* __JFFS2_NODELIST_H__ */ diff --git a/fs/jffs2/nodemgmt.c b/fs/jffs2/nodemgmt.c new file mode 100644 index 000000000..3fb9f9807 --- /dev/null +++ b/fs/jffs2/nodemgmt.c @@ -0,0 +1,890 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * + * Created by David Woodhouse + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include "nodelist.h" +#include "debug.h" + +/* + * Check whether the user is allowed to write. + */ +static int jffs2_rp_can_write(struct jffs2_sb_info *c) +{ + uint32_t avail; + struct jffs2_mount_opts *opts = &c->mount_opts; + + avail = c->dirty_size + c->free_size + c->unchecked_size + + c->erasing_size - c->resv_blocks_write * c->sector_size + - c->nospc_dirty_size; + + if (avail < 2 * opts->rp_size) + jffs2_dbg(1, "rpsize %u, dirty_size %u, free_size %u, " + "erasing_size %u, unchecked_size %u, " + "nr_erasing_blocks %u, avail %u, resrv %u\n", + opts->rp_size, c->dirty_size, c->free_size, + c->erasing_size, c->unchecked_size, + c->nr_erasing_blocks, avail, c->nospc_dirty_size); + + if (avail > opts->rp_size) + return 1; + + /* Always allow root */ + if (capable(CAP_SYS_RESOURCE)) + return 1; + + jffs2_dbg(1, "forbid writing\n"); + return 0; +} + +static int jffs2_do_reserve_space(struct jffs2_sb_info *c, uint32_t minsize, + uint32_t *len, uint32_t sumsize); + +/** + * jffs2_reserve_space - request physical space to write nodes to flash + * @c: superblock info + * @minsize: Minimum acceptable size of allocation + * @len: Returned value of allocation length + * @prio: Allocation type - ALLOC_{NORMAL,DELETION} + * @sumsize: summary size requested or JFFS2_SUMMARY_NOSUM_SIZE for no summary + * + * Requests a block of physical space on the flash. + * + * Returns: %0 for success and puts 'len' into the appropriate place, + * or returns -ENOSPC or other error if appropriate. + * Doesn't return len since that's already returned in @len. + * + * If it returns %0, jffs2_reserve_space() also downs the per-filesystem + * allocation semaphore, to prevent more than one allocation from being + * active at any time. The semaphore is later released by jffs2_commit_allocation(). + * + * jffs2_reserve_space() may trigger garbage collection in order to make room + * for the requested allocation. + */ + +int jffs2_reserve_space(struct jffs2_sb_info *c, uint32_t minsize, + uint32_t *len, int prio, uint32_t sumsize) +{ + int ret = -EAGAIN; + int blocksneeded = c->resv_blocks_write; + /* align it */ + minsize = PAD(minsize); + + jffs2_dbg(1, "%s(): Requested 0x%x bytes\n", __func__, minsize); + mutex_lock(&c->alloc_sem); + + jffs2_dbg(1, "%s(): alloc sem got\n", __func__); + + spin_lock(&c->erase_completion_lock); + + /* + * Check if the free space is greater then size of the reserved pool. + * If not, only allow root to proceed with writing. + */ + if (prio != ALLOC_DELETION && !jffs2_rp_can_write(c)) { + ret = -ENOSPC; + goto out; + } + + /* this needs a little more thought (true :)) */ + while(ret == -EAGAIN) { + while(c->nr_free_blocks + c->nr_erasing_blocks < blocksneeded) { + uint32_t dirty, avail; + + /* calculate real dirty size + * dirty_size contains blocks on erase_pending_list + * those blocks are counted in c->nr_erasing_blocks. + * If one block is actually erased, it is not longer counted as dirty_space + * but it is counted in c->nr_erasing_blocks, so we add it and subtract it + * with c->nr_erasing_blocks * c->sector_size again. + * Blocks on erasable_list are counted as dirty_size, but not in c->nr_erasing_blocks + * This helps us to force gc and pick eventually a clean block to spread the load. + * We add unchecked_size here, as we hopefully will find some space to use. + * This will affect the sum only once, as gc first finishes checking + * of nodes. + */ + dirty = c->dirty_size + c->erasing_size - c->nr_erasing_blocks * c->sector_size + c->unchecked_size; + if (dirty < c->nospc_dirty_size) { + if (prio == ALLOC_DELETION && c->nr_free_blocks + c->nr_erasing_blocks >= c->resv_blocks_deletion) { + jffs2_dbg(1, "%s(): Low on dirty space to GC, but it's a deletion. Allowing...\n", + __func__); + break; + } + jffs2_dbg(1, "dirty size 0x%08x + unchecked_size 0x%08x < nospc_dirty_size 0x%08x, returning -ENOSPC\n", + dirty, c->unchecked_size, + c->sector_size); + + spin_unlock(&c->erase_completion_lock); + mutex_unlock(&c->alloc_sem); + return -ENOSPC; + } + + /* Calc possibly available space. Possibly available means that we + * don't know, if unchecked size contains obsoleted nodes, which could give us some + * more usable space. This will affect the sum only once, as gc first finishes checking + * of nodes. + + Return -ENOSPC, if the maximum possibly available space is less or equal than + * blocksneeded * sector_size. + * This blocks endless gc looping on a filesystem, which is nearly full, even if + * the check above passes. + */ + avail = c->free_size + c->dirty_size + c->erasing_size + c->unchecked_size; + if ( (avail / c->sector_size) <= blocksneeded) { + if (prio == ALLOC_DELETION && c->nr_free_blocks + c->nr_erasing_blocks >= c->resv_blocks_deletion) { + jffs2_dbg(1, "%s(): Low on possibly available space, but it's a deletion. Allowing...\n", + __func__); + break; + } + + jffs2_dbg(1, "max. available size 0x%08x < blocksneeded * sector_size 0x%08x, returning -ENOSPC\n", + avail, blocksneeded * c->sector_size); + spin_unlock(&c->erase_completion_lock); + mutex_unlock(&c->alloc_sem); + return -ENOSPC; + } + + mutex_unlock(&c->alloc_sem); + + jffs2_dbg(1, "Triggering GC pass. nr_free_blocks %d, nr_erasing_blocks %d, free_size 0x%08x, dirty_size 0x%08x, wasted_size 0x%08x, used_size 0x%08x, erasing_size 0x%08x, bad_size 0x%08x (total 0x%08x of 0x%08x)\n", + c->nr_free_blocks, c->nr_erasing_blocks, + c->free_size, c->dirty_size, c->wasted_size, + c->used_size, c->erasing_size, c->bad_size, + c->free_size + c->dirty_size + + c->wasted_size + c->used_size + + c->erasing_size + c->bad_size, + c->flash_size); + spin_unlock(&c->erase_completion_lock); + + ret = jffs2_garbage_collect_pass(c); + + if (ret == -EAGAIN) { + spin_lock(&c->erase_completion_lock); + if (c->nr_erasing_blocks && + list_empty(&c->erase_pending_list) && + list_empty(&c->erase_complete_list)) { + DECLARE_WAITQUEUE(wait, current); + set_current_state(TASK_UNINTERRUPTIBLE); + add_wait_queue(&c->erase_wait, &wait); + jffs2_dbg(1, "%s waiting for erase to complete\n", + __func__); + spin_unlock(&c->erase_completion_lock); + + schedule(); + remove_wait_queue(&c->erase_wait, &wait); + } else + spin_unlock(&c->erase_completion_lock); + } else if (ret) + return ret; + + cond_resched(); + + if (signal_pending(current)) + return -EINTR; + + mutex_lock(&c->alloc_sem); + spin_lock(&c->erase_completion_lock); + } + + ret = jffs2_do_reserve_space(c, minsize, len, sumsize); + if (ret) { + jffs2_dbg(1, "%s(): ret is %d\n", __func__, ret); + } + } + +out: + spin_unlock(&c->erase_completion_lock); + if (!ret) + ret = jffs2_prealloc_raw_node_refs(c, c->nextblock, 1); + if (ret) + mutex_unlock(&c->alloc_sem); + return ret; +} + +int jffs2_reserve_space_gc(struct jffs2_sb_info *c, uint32_t minsize, + uint32_t *len, uint32_t sumsize) +{ + int ret; + minsize = PAD(minsize); + + jffs2_dbg(1, "%s(): Requested 0x%x bytes\n", __func__, minsize); + + while (true) { + spin_lock(&c->erase_completion_lock); + ret = jffs2_do_reserve_space(c, minsize, len, sumsize); + if (ret) { + jffs2_dbg(1, "%s(): looping, ret is %d\n", + __func__, ret); + } + spin_unlock(&c->erase_completion_lock); + + if (ret == -EAGAIN) + cond_resched(); + else + break; + } + if (!ret) + ret = jffs2_prealloc_raw_node_refs(c, c->nextblock, 1); + + return ret; +} + + +/* Classify nextblock (clean, dirty of verydirty) and force to select an other one */ + +static void jffs2_close_nextblock(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb) +{ + + if (c->nextblock == NULL) { + jffs2_dbg(1, "%s(): Erase block at 0x%08x has already been placed in a list\n", + __func__, jeb->offset); + return; + } + /* Check, if we have a dirty block now, or if it was dirty already */ + if (ISDIRTY (jeb->wasted_size + jeb->dirty_size)) { + c->dirty_size += jeb->wasted_size; + c->wasted_size -= jeb->wasted_size; + jeb->dirty_size += jeb->wasted_size; + jeb->wasted_size = 0; + if (VERYDIRTY(c, jeb->dirty_size)) { + jffs2_dbg(1, "Adding full erase block at 0x%08x to very_dirty_list (free 0x%08x, dirty 0x%08x, used 0x%08x\n", + jeb->offset, jeb->free_size, jeb->dirty_size, + jeb->used_size); + list_add_tail(&jeb->list, &c->very_dirty_list); + } else { + jffs2_dbg(1, "Adding full erase block at 0x%08x to dirty_list (free 0x%08x, dirty 0x%08x, used 0x%08x\n", + jeb->offset, jeb->free_size, jeb->dirty_size, + jeb->used_size); + list_add_tail(&jeb->list, &c->dirty_list); + } + } else { + jffs2_dbg(1, "Adding full erase block at 0x%08x to clean_list (free 0x%08x, dirty 0x%08x, used 0x%08x\n", + jeb->offset, jeb->free_size, jeb->dirty_size, + jeb->used_size); + list_add_tail(&jeb->list, &c->clean_list); + } + c->nextblock = NULL; + +} + +/* Select a new jeb for nextblock */ + +static int jffs2_find_nextblock(struct jffs2_sb_info *c) +{ + struct list_head *next; + + /* Take the next block off the 'free' list */ + + if (list_empty(&c->free_list)) { + + if (!c->nr_erasing_blocks && + !list_empty(&c->erasable_list)) { + struct jffs2_eraseblock *ejeb; + + ejeb = list_entry(c->erasable_list.next, struct jffs2_eraseblock, list); + list_move_tail(&ejeb->list, &c->erase_pending_list); + c->nr_erasing_blocks++; + jffs2_garbage_collect_trigger(c); + jffs2_dbg(1, "%s(): Triggering erase of erasable block at 0x%08x\n", + __func__, ejeb->offset); + } + + if (!c->nr_erasing_blocks && + !list_empty(&c->erasable_pending_wbuf_list)) { + jffs2_dbg(1, "%s(): Flushing write buffer\n", + __func__); + /* c->nextblock is NULL, no update to c->nextblock allowed */ + spin_unlock(&c->erase_completion_lock); + jffs2_flush_wbuf_pad(c); + spin_lock(&c->erase_completion_lock); + /* Have another go. It'll be on the erasable_list now */ + return -EAGAIN; + } + + if (!c->nr_erasing_blocks) { + /* Ouch. We're in GC, or we wouldn't have got here. + And there's no space left. At all. */ + pr_crit("Argh. No free space left for GC. nr_erasing_blocks is %d. nr_free_blocks is %d. (erasableempty: %s, erasingempty: %s, erasependingempty: %s)\n", + c->nr_erasing_blocks, c->nr_free_blocks, + str_yes_no(list_empty(&c->erasable_list)), + str_yes_no(list_empty(&c->erasing_list)), + str_yes_no(list_empty(&c->erase_pending_list))); + return -ENOSPC; + } + + spin_unlock(&c->erase_completion_lock); + /* Don't wait for it; just erase one right now */ + jffs2_erase_pending_blocks(c, 1); + spin_lock(&c->erase_completion_lock); + + /* An erase may have failed, decreasing the + amount of free space available. So we must + restart from the beginning */ + return -EAGAIN; + } + + next = c->free_list.next; + list_del(next); + c->nextblock = list_entry(next, struct jffs2_eraseblock, list); + c->nr_free_blocks--; + + jffs2_sum_reset_collected(c->summary); /* reset collected summary */ + +#ifdef CONFIG_JFFS2_FS_WRITEBUFFER + /* adjust write buffer offset, else we get a non contiguous write bug */ + if (!(c->wbuf_ofs % c->sector_size) && !c->wbuf_len) + c->wbuf_ofs = 0xffffffff; +#endif + + jffs2_dbg(1, "%s(): new nextblock = 0x%08x\n", + __func__, c->nextblock->offset); + + return 0; +} + +/* Called with alloc sem _and_ erase_completion_lock */ +static int jffs2_do_reserve_space(struct jffs2_sb_info *c, uint32_t minsize, + uint32_t *len, uint32_t sumsize) +{ + struct jffs2_eraseblock *jeb = c->nextblock; + uint32_t reserved_size; /* for summary information at the end of the jeb */ + int ret; + + restart: + reserved_size = 0; + + if (jffs2_sum_active() && (sumsize != JFFS2_SUMMARY_NOSUM_SIZE)) { + /* NOSUM_SIZE means not to generate summary */ + + if (jeb) { + reserved_size = PAD(sumsize + c->summary->sum_size + JFFS2_SUMMARY_FRAME_SIZE); + dbg_summary("minsize=%d , jeb->free=%d ," + "summary->size=%d , sumsize=%d\n", + minsize, jeb->free_size, + c->summary->sum_size, sumsize); + } + + /* Is there enough space for writing out the current node, or we have to + write out summary information now, close this jeb and select new nextblock? */ + if (jeb && (PAD(minsize) + PAD(c->summary->sum_size + sumsize + + JFFS2_SUMMARY_FRAME_SIZE) > jeb->free_size)) { + + /* Has summary been disabled for this jeb? */ + if (jffs2_sum_is_disabled(c->summary)) { + sumsize = JFFS2_SUMMARY_NOSUM_SIZE; + goto restart; + } + + /* Writing out the collected summary information */ + dbg_summary("generating summary for 0x%08x.\n", jeb->offset); + ret = jffs2_sum_write_sumnode(c); + + if (ret) + return ret; + + if (jffs2_sum_is_disabled(c->summary)) { + /* jffs2_write_sumnode() couldn't write out the summary information + diabling summary for this jeb and free the collected information + */ + sumsize = JFFS2_SUMMARY_NOSUM_SIZE; + goto restart; + } + + jffs2_close_nextblock(c, jeb); + jeb = NULL; + /* keep always valid value in reserved_size */ + reserved_size = PAD(sumsize + c->summary->sum_size + JFFS2_SUMMARY_FRAME_SIZE); + } + } else { + if (jeb && minsize > jeb->free_size) { + uint32_t waste; + + /* Skip the end of this block and file it as having some dirty space */ + /* If there's a pending write to it, flush now */ + + if (jffs2_wbuf_dirty(c)) { + spin_unlock(&c->erase_completion_lock); + jffs2_dbg(1, "%s(): Flushing write buffer\n", + __func__); + jffs2_flush_wbuf_pad(c); + spin_lock(&c->erase_completion_lock); + jeb = c->nextblock; + goto restart; + } + + spin_unlock(&c->erase_completion_lock); + + ret = jffs2_prealloc_raw_node_refs(c, jeb, 1); + + /* Just lock it again and continue. Nothing much can change because + we hold c->alloc_sem anyway. In fact, it's not entirely clear why + we hold c->erase_completion_lock in the majority of this function... + but that's a question for another (more caffeine-rich) day. */ + spin_lock(&c->erase_completion_lock); + + if (ret) + return ret; + + waste = jeb->free_size; + jffs2_link_node_ref(c, jeb, + (jeb->offset + c->sector_size - waste) | REF_OBSOLETE, + waste, NULL); + /* FIXME: that made it count as dirty. Convert to wasted */ + jeb->dirty_size -= waste; + c->dirty_size -= waste; + jeb->wasted_size += waste; + c->wasted_size += waste; + + jffs2_close_nextblock(c, jeb); + jeb = NULL; + } + } + + if (!jeb) { + + ret = jffs2_find_nextblock(c); + if (ret) + return ret; + + jeb = c->nextblock; + + if (jeb->free_size != c->sector_size - c->cleanmarker_size) { + pr_warn("Eep. Block 0x%08x taken from free_list had free_size of 0x%08x!!\n", + jeb->offset, jeb->free_size); + goto restart; + } + } + /* OK, jeb (==c->nextblock) is now pointing at a block which definitely has + enough space */ + *len = jeb->free_size - reserved_size; + + if (c->cleanmarker_size && jeb->used_size == c->cleanmarker_size && + !jeb->first_node->next_in_ino) { + /* Only node in it beforehand was a CLEANMARKER node (we think). + So mark it obsolete now that there's going to be another node + in the block. This will reduce used_size to zero but We've + already set c->nextblock so that jffs2_mark_node_obsolete() + won't try to refile it to the dirty_list. + */ + spin_unlock(&c->erase_completion_lock); + jffs2_mark_node_obsolete(c, jeb->first_node); + spin_lock(&c->erase_completion_lock); + } + + jffs2_dbg(1, "%s(): Giving 0x%x bytes at 0x%x\n", + __func__, + *len, jeb->offset + (c->sector_size - jeb->free_size)); + return 0; +} + +/** + * jffs2_add_physical_node_ref - add a physical node reference to the list + * @c: superblock info + * @ofs: offset in the block + * @len: length of this physical node + * @ic: inode cache pointer + * + * Should only be used to report nodes for which space has been allocated + * by jffs2_reserve_space. + * + * Must be called with the alloc_sem held. + * + * Returns: pointer to new node on success or -errno code on error + */ + +struct jffs2_raw_node_ref *jffs2_add_physical_node_ref(struct jffs2_sb_info *c, + uint32_t ofs, uint32_t len, + struct jffs2_inode_cache *ic) +{ + struct jffs2_eraseblock *jeb; + struct jffs2_raw_node_ref *new; + + jeb = &c->blocks[ofs / c->sector_size]; + + jffs2_dbg(1, "%s(): Node at 0x%x(%d), size 0x%x\n", + __func__, ofs & ~3, ofs & 3, len); +#if 1 + /* Allow non-obsolete nodes only to be added at the end of c->nextblock, + if c->nextblock is set. Note that wbuf.c will file obsolete nodes + even after refiling c->nextblock */ + if ((c->nextblock || ((ofs & 3) != REF_OBSOLETE)) + && (jeb != c->nextblock || (ofs & ~3) != jeb->offset + (c->sector_size - jeb->free_size))) { + pr_warn("argh. node added in wrong place at 0x%08x(%d)\n", + ofs & ~3, ofs & 3); + if (c->nextblock) + pr_warn("nextblock 0x%08x", c->nextblock->offset); + else + pr_warn("No nextblock"); + pr_cont(", expected at %08x\n", + jeb->offset + (c->sector_size - jeb->free_size)); + return ERR_PTR(-EINVAL); + } +#endif + spin_lock(&c->erase_completion_lock); + + new = jffs2_link_node_ref(c, jeb, ofs, len, ic); + + if (!jeb->free_size && !jeb->dirty_size && !ISDIRTY(jeb->wasted_size)) { + /* If it lives on the dirty_list, jffs2_reserve_space will put it there */ + jffs2_dbg(1, "Adding full erase block at 0x%08x to clean_list (free 0x%08x, dirty 0x%08x, used 0x%08x\n", + jeb->offset, jeb->free_size, jeb->dirty_size, + jeb->used_size); + if (jffs2_wbuf_dirty(c)) { + /* Flush the last write in the block if it's outstanding */ + spin_unlock(&c->erase_completion_lock); + jffs2_flush_wbuf_pad(c); + spin_lock(&c->erase_completion_lock); + } + + list_add_tail(&jeb->list, &c->clean_list); + c->nextblock = NULL; + } + jffs2_dbg_acct_sanity_check_nolock(c,jeb); + jffs2_dbg_acct_paranoia_check_nolock(c, jeb); + + spin_unlock(&c->erase_completion_lock); + + return new; +} + + +void jffs2_complete_reservation(struct jffs2_sb_info *c) +{ + jffs2_dbg(1, "jffs2_complete_reservation()\n"); + spin_lock(&c->erase_completion_lock); + jffs2_garbage_collect_trigger(c); + spin_unlock(&c->erase_completion_lock); + mutex_unlock(&c->alloc_sem); +} + +static inline int on_list(struct list_head *obj, struct list_head *head) +{ + struct list_head *this; + + list_for_each(this, head) { + if (this == obj) { + jffs2_dbg(1, "%p is on list at %p\n", obj, head); + return 1; + + } + } + return 0; +} + +void jffs2_mark_node_obsolete(struct jffs2_sb_info *c, struct jffs2_raw_node_ref *ref) +{ + struct jffs2_eraseblock *jeb; + int blocknr; + struct jffs2_unknown_node n; + int ret, addedsize; + size_t retlen; + uint32_t freed_len; + + if(unlikely(!ref)) { + pr_notice("EEEEEK. jffs2_mark_node_obsolete called with NULL node\n"); + return; + } + if (ref_obsolete(ref)) { + jffs2_dbg(1, "%s(): called with already obsolete node at 0x%08x\n", + __func__, ref_offset(ref)); + return; + } + blocknr = ref->flash_offset / c->sector_size; + if (blocknr >= c->nr_blocks) { + pr_notice("raw node at 0x%08x is off the end of device!\n", + ref->flash_offset); + BUG(); + } + jeb = &c->blocks[blocknr]; + + if (jffs2_can_mark_obsolete(c) && !jffs2_is_readonly(c) && + !(c->flags & (JFFS2_SB_FLAG_SCANNING | JFFS2_SB_FLAG_BUILDING))) { + /* Hm. This may confuse static lock analysis. If any of the above + three conditions is false, we're going to return from this + function without actually obliterating any nodes or freeing + any jffs2_raw_node_refs. So we don't need to stop erases from + happening, or protect against people holding an obsolete + jffs2_raw_node_ref without the erase_completion_lock. */ + mutex_lock(&c->erase_free_sem); + } + + spin_lock(&c->erase_completion_lock); + + freed_len = ref_totlen(c, jeb, ref); + + if (ref_flags(ref) == REF_UNCHECKED) { + D1(if (unlikely(jeb->unchecked_size < freed_len)) { + pr_notice("raw unchecked node of size 0x%08x freed from erase block %d at 0x%08x, but unchecked_size was already 0x%08x\n", + freed_len, blocknr, + ref->flash_offset, jeb->used_size); + BUG(); + }) + jffs2_dbg(1, "Obsoleting previously unchecked node at 0x%08x of len %x\n", + ref_offset(ref), freed_len); + jeb->unchecked_size -= freed_len; + c->unchecked_size -= freed_len; + } else { + D1(if (unlikely(jeb->used_size < freed_len)) { + pr_notice("raw node of size 0x%08x freed from erase block %d at 0x%08x, but used_size was already 0x%08x\n", + freed_len, blocknr, + ref->flash_offset, jeb->used_size); + BUG(); + }) + jffs2_dbg(1, "Obsoleting node at 0x%08x of len %#x: ", + ref_offset(ref), freed_len); + jeb->used_size -= freed_len; + c->used_size -= freed_len; + } + + // Take care, that wasted size is taken into concern + if ((jeb->dirty_size || ISDIRTY(jeb->wasted_size + freed_len)) && jeb != c->nextblock) { + jffs2_dbg(1, "Dirtying\n"); + addedsize = freed_len; + jeb->dirty_size += freed_len; + c->dirty_size += freed_len; + + /* Convert wasted space to dirty, if not a bad block */ + if (jeb->wasted_size) { + if (on_list(&jeb->list, &c->bad_used_list)) { + jffs2_dbg(1, "Leaving block at %08x on the bad_used_list\n", + jeb->offset); + addedsize = 0; /* To fool the refiling code later */ + } else { + jffs2_dbg(1, "Converting %d bytes of wasted space to dirty in block at %08x\n", + jeb->wasted_size, jeb->offset); + addedsize += jeb->wasted_size; + jeb->dirty_size += jeb->wasted_size; + c->dirty_size += jeb->wasted_size; + c->wasted_size -= jeb->wasted_size; + jeb->wasted_size = 0; + } + } + } else { + jffs2_dbg(1, "Wasting\n"); + addedsize = 0; + jeb->wasted_size += freed_len; + c->wasted_size += freed_len; + } + ref->flash_offset = ref_offset(ref) | REF_OBSOLETE; + + jffs2_dbg_acct_sanity_check_nolock(c, jeb); + jffs2_dbg_acct_paranoia_check_nolock(c, jeb); + + if (c->flags & JFFS2_SB_FLAG_SCANNING) { + /* Flash scanning is in progress. Don't muck about with the block + lists because they're not ready yet, and don't actually + obliterate nodes that look obsolete. If they weren't + marked obsolete on the flash at the time they _became_ + obsolete, there was probably a reason for that. */ + spin_unlock(&c->erase_completion_lock); + /* We didn't lock the erase_free_sem */ + return; + } + + if (jeb == c->nextblock) { + jffs2_dbg(2, "Not moving nextblock 0x%08x to dirty/erase_pending list\n", + jeb->offset); + } else if (!jeb->used_size && !jeb->unchecked_size) { + if (jeb == c->gcblock) { + jffs2_dbg(1, "gcblock at 0x%08x completely dirtied. Clearing gcblock...\n", + jeb->offset); + c->gcblock = NULL; + } else { + jffs2_dbg(1, "Eraseblock at 0x%08x completely dirtied. Removing from (dirty?) list...\n", + jeb->offset); + list_del(&jeb->list); + } + if (jffs2_wbuf_dirty(c)) { + jffs2_dbg(1, "...and adding to erasable_pending_wbuf_list\n"); + list_add_tail(&jeb->list, &c->erasable_pending_wbuf_list); + } else { + if (jiffies & 127) { + /* Most of the time, we just erase it immediately. Otherwise we + spend ages scanning it on mount, etc. */ + jffs2_dbg(1, "...and adding to erase_pending_list\n"); + list_add_tail(&jeb->list, &c->erase_pending_list); + c->nr_erasing_blocks++; + jffs2_garbage_collect_trigger(c); + } else { + /* Sometimes, however, we leave it elsewhere so it doesn't get + immediately reused, and we spread the load a bit. */ + jffs2_dbg(1, "...and adding to erasable_list\n"); + list_add_tail(&jeb->list, &c->erasable_list); + } + } + jffs2_dbg(1, "Done OK\n"); + } else if (jeb == c->gcblock) { + jffs2_dbg(2, "Not moving gcblock 0x%08x to dirty_list\n", + jeb->offset); + } else if (ISDIRTY(jeb->dirty_size) && !ISDIRTY(jeb->dirty_size - addedsize)) { + jffs2_dbg(1, "Eraseblock at 0x%08x is freshly dirtied. Removing from clean list...\n", + jeb->offset); + list_del(&jeb->list); + jffs2_dbg(1, "...and adding to dirty_list\n"); + list_add_tail(&jeb->list, &c->dirty_list); + } else if (VERYDIRTY(c, jeb->dirty_size) && + !VERYDIRTY(c, jeb->dirty_size - addedsize)) { + jffs2_dbg(1, "Eraseblock at 0x%08x is now very dirty. Removing from dirty list...\n", + jeb->offset); + list_del(&jeb->list); + jffs2_dbg(1, "...and adding to very_dirty_list\n"); + list_add_tail(&jeb->list, &c->very_dirty_list); + } else { + jffs2_dbg(1, "Eraseblock at 0x%08x not moved anywhere. (free 0x%08x, dirty 0x%08x, used 0x%08x)\n", + jeb->offset, jeb->free_size, jeb->dirty_size, + jeb->used_size); + } + + spin_unlock(&c->erase_completion_lock); + + if (!jffs2_can_mark_obsolete(c) || jffs2_is_readonly(c) || + (c->flags & JFFS2_SB_FLAG_BUILDING)) { + /* We didn't lock the erase_free_sem */ + return; + } + + /* The erase_free_sem is locked, and has been since before we marked the node obsolete + and potentially put its eraseblock onto the erase_pending_list. Thus, we know that + the block hasn't _already_ been erased, and that 'ref' itself hasn't been freed yet + by jffs2_free_jeb_node_refs() in erase.c. Which is nice. */ + + jffs2_dbg(1, "obliterating obsoleted node at 0x%08x\n", + ref_offset(ref)); + ret = jffs2_flash_read(c, ref_offset(ref), sizeof(n), &retlen, (char *)&n); + if (ret) { + pr_warn("Read error reading from obsoleted node at 0x%08x: %d\n", + ref_offset(ref), ret); + goto out_erase_sem; + } + if (retlen != sizeof(n)) { + pr_warn("Short read from obsoleted node at 0x%08x: %zd\n", + ref_offset(ref), retlen); + goto out_erase_sem; + } + if (PAD(je32_to_cpu(n.totlen)) != PAD(freed_len)) { + pr_warn("Node totlen on flash (0x%08x) != totlen from node ref (0x%08x)\n", + je32_to_cpu(n.totlen), freed_len); + goto out_erase_sem; + } + if (!(je16_to_cpu(n.nodetype) & JFFS2_NODE_ACCURATE)) { + jffs2_dbg(1, "Node at 0x%08x was already marked obsolete (nodetype 0x%04x)\n", + ref_offset(ref), je16_to_cpu(n.nodetype)); + goto out_erase_sem; + } + /* XXX FIXME: This is ugly now */ + n.nodetype = cpu_to_je16(je16_to_cpu(n.nodetype) & ~JFFS2_NODE_ACCURATE); + ret = jffs2_flash_write(c, ref_offset(ref), sizeof(n), &retlen, (char *)&n); + if (ret) { + pr_warn("Write error in obliterating obsoleted node at 0x%08x: %d\n", + ref_offset(ref), ret); + goto out_erase_sem; + } + if (retlen != sizeof(n)) { + pr_warn("Short write in obliterating obsoleted node at 0x%08x: %zd\n", + ref_offset(ref), retlen); + goto out_erase_sem; + } + + /* Nodes which have been marked obsolete no longer need to be + associated with any inode. Remove them from the per-inode list. + + Note we can't do this for NAND at the moment because we need + obsolete dirent nodes to stay on the lists, because of the + horridness in jffs2_garbage_collect_deletion_dirent(). Also + because we delete the inocache, and on NAND we need that to + stay around until all the nodes are actually erased, in order + to stop us from giving the same inode number to another newly + created inode. */ + if (ref->next_in_ino) { + struct jffs2_inode_cache *ic; + struct jffs2_raw_node_ref **p; + + spin_lock(&c->erase_completion_lock); + + ic = jffs2_raw_ref_to_ic(ref); + for (p = &ic->nodes; (*p) != ref; p = &((*p)->next_in_ino)) + ; + + *p = ref->next_in_ino; + ref->next_in_ino = NULL; + + switch (ic->class) { +#ifdef CONFIG_JFFS2_FS_XATTR + case RAWNODE_CLASS_XATTR_DATUM: + jffs2_release_xattr_datum(c, (struct jffs2_xattr_datum *)ic); + break; + case RAWNODE_CLASS_XATTR_REF: + jffs2_release_xattr_ref(c, (struct jffs2_xattr_ref *)ic); + break; +#endif + default: + if (ic->nodes == (void *)ic && ic->pino_nlink == 0) + jffs2_del_ino_cache(c, ic); + break; + } + spin_unlock(&c->erase_completion_lock); + } + + out_erase_sem: + mutex_unlock(&c->erase_free_sem); +} + +int jffs2_thread_should_wake(struct jffs2_sb_info *c) +{ + int ret = 0; + uint32_t dirty; + int nr_very_dirty = 0; + struct jffs2_eraseblock *jeb; + + if (!list_empty(&c->erase_complete_list) || + !list_empty(&c->erase_pending_list)) + return 1; + + if (c->unchecked_size) { + jffs2_dbg(1, "jffs2_thread_should_wake(): unchecked_size %d, check_ino #%d\n", + c->unchecked_size, c->check_ino); + return 1; + } + + /* dirty_size contains blocks on erase_pending_list + * those blocks are counted in c->nr_erasing_blocks. + * If one block is actually erased, it is not longer counted as dirty_space + * but it is counted in c->nr_erasing_blocks, so we add it and subtract it + * with c->nr_erasing_blocks * c->sector_size again. + * Blocks on erasable_list are counted as dirty_size, but not in c->nr_erasing_blocks + * This helps us to force gc and pick eventually a clean block to spread the load. + */ + dirty = c->dirty_size + c->erasing_size - c->nr_erasing_blocks * c->sector_size; + + if (c->nr_free_blocks + c->nr_erasing_blocks < c->resv_blocks_gctrigger && + (dirty > c->nospc_dirty_size)) + ret = 1; + + list_for_each_entry(jeb, &c->very_dirty_list, list) { + nr_very_dirty++; + if (nr_very_dirty == c->vdirty_blocks_gctrigger) { + ret = 1; + /* In debug mode, actually go through and count them all */ + D1(continue); + break; + } + } + + jffs2_dbg(1, "%s(): nr_free_blocks %d, nr_erasing_blocks %d, dirty_size 0x%x, vdirty_blocks %d: %s\n", + __func__, c->nr_free_blocks, c->nr_erasing_blocks, + c->dirty_size, nr_very_dirty, str_yes_no(ret)); + + return ret; +} diff --git a/fs/jffs2/os-linux.h b/fs/jffs2/os-linux.h new file mode 100644 index 000000000..86ab014a3 --- /dev/null +++ b/fs/jffs2/os-linux.h @@ -0,0 +1,197 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * + * Created by David Woodhouse + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#ifndef __JFFS2_OS_LINUX_H__ +#define __JFFS2_OS_LINUX_H__ + +/* JFFS2 uses Linux mode bits natively -- no need for conversion */ +#define os_to_jffs2_mode(x) (x) +#define jffs2_to_os_mode(x) (x) + +struct kstatfs; +struct kvec; + +#define JFFS2_INODE_INFO(i) (container_of(i, struct jffs2_inode_info, vfs_inode)) +#define OFNI_EDONI_2SFFJ(f) (&(f)->vfs_inode) +#define JFFS2_SB_INFO(sb) (sb->s_fs_info) +#define OFNI_BS_2SFFJ(c) ((struct super_block *)c->os_priv) + + +#define JFFS2_F_I_SIZE(f) (OFNI_EDONI_2SFFJ(f)->i_size) +#define JFFS2_F_I_MODE(f) (OFNI_EDONI_2SFFJ(f)->i_mode) +#define JFFS2_F_I_UID(f) (i_uid_read(OFNI_EDONI_2SFFJ(f))) +#define JFFS2_F_I_GID(f) (i_gid_read(OFNI_EDONI_2SFFJ(f))) +#define JFFS2_F_I_RDEV(f) (OFNI_EDONI_2SFFJ(f)->i_rdev) + +#define JFFS2_CLAMP_TIME(t) ((uint32_t)clamp_t(time64_t, (t), 0, U32_MAX)) +#define ITIME(sec) ((struct timespec64){sec, 0}) +#define JFFS2_NOW() JFFS2_CLAMP_TIME(ktime_get_real_seconds()) +#define I_SEC(tv) JFFS2_CLAMP_TIME((tv).tv_sec) +#define JFFS2_F_I_CTIME(f) I_SEC(inode_get_ctime(OFNI_EDONI_2SFFJ(f))) +#define JFFS2_F_I_MTIME(f) I_SEC(inode_get_mtime(OFNI_EDONI_2SFFJ(f))) +#define JFFS2_F_I_ATIME(f) I_SEC(inode_get_atime(OFNI_EDONI_2SFFJ(f))) +#define sleep_on_spinunlock(wq, s) \ + do { \ + DECLARE_WAITQUEUE(__wait, current); \ + add_wait_queue((wq), &__wait); \ + set_current_state(TASK_UNINTERRUPTIBLE); \ + spin_unlock(s); \ + schedule(); \ + remove_wait_queue((wq), &__wait); \ + } while(0) + +static inline void jffs2_init_inode_info(struct jffs2_inode_info *f) +{ + f->highest_version = 0; + f->fragtree = RB_ROOT; + f->metadata = NULL; + f->dents = NULL; + f->target = NULL; + f->flags = 0; + f->usercompr = 0; +} + + +#define jffs2_is_readonly(c) (OFNI_BS_2SFFJ(c)->s_flags & SB_RDONLY) + +#define SECTOR_ADDR(x) ( (((unsigned long)(x) / c->sector_size) * c->sector_size) ) +#ifndef CONFIG_JFFS2_FS_WRITEBUFFER + + +#ifdef CONFIG_JFFS2_SUMMARY +#define jffs2_can_mark_obsolete(c) (0) +#else +#define jffs2_can_mark_obsolete(c) (1) +#endif + +#define jffs2_is_writebuffered(c) (0) +#define jffs2_cleanmarker_oob(c) (0) +#define jffs2_write_nand_cleanmarker(c,jeb) (-EIO) + +#define jffs2_flash_write(c, ofs, len, retlen, buf) jffs2_flash_direct_write(c, ofs, len, retlen, buf) +#define jffs2_flash_read(c, ofs, len, retlen, buf) (mtd_read((c)->mtd, ofs, len, retlen, buf)) +#define jffs2_flush_wbuf_pad(c) ({ do{} while(0); (void)(c), 0; }) +#define jffs2_flush_wbuf_gc(c, i) ({ do{} while(0); (void)(c), (void) i, 0; }) +#define jffs2_write_nand_badblock(c,jeb,bad_offset) (1) +#define jffs2_nand_flash_setup(c) (0) +#define jffs2_nand_flash_cleanup(c) do {} while(0) +#define jffs2_wbuf_dirty(c) (0) +#define jffs2_flash_writev(a,b,c,d,e,f) jffs2_flash_direct_writev(a,b,c,d,e) +#define jffs2_wbuf_timeout NULL +#define jffs2_wbuf_process NULL +#define jffs2_dataflash(c) (0) +#define jffs2_dataflash_setup(c) (0) +#define jffs2_dataflash_cleanup(c) do {} while (0) +#define jffs2_nor_wbuf_flash(c) (0) +#define jffs2_nor_wbuf_flash_setup(c) (0) +#define jffs2_nor_wbuf_flash_cleanup(c) do {} while (0) +#define jffs2_ubivol(c) (0) +#define jffs2_ubivol_setup(c) (0) +#define jffs2_ubivol_cleanup(c) do {} while (0) +#define jffs2_dirty_trigger(c) do {} while (0) + +#else /* NAND and/or ECC'd NOR support present */ + +#define jffs2_is_writebuffered(c) (c->wbuf != NULL) + +#ifdef CONFIG_JFFS2_SUMMARY +#define jffs2_can_mark_obsolete(c) (0) +#else +#define jffs2_can_mark_obsolete(c) (c->mtd->flags & (MTD_BIT_WRITEABLE)) +#endif + +#define jffs2_cleanmarker_oob(c) (c->mtd->type == MTD_NANDFLASH) + +#define jffs2_wbuf_dirty(c) (!!(c)->wbuf_len) + +/* wbuf.c */ +int jffs2_flash_writev(struct jffs2_sb_info *c, const struct kvec *vecs, unsigned long count, loff_t to, size_t *retlen, uint32_t ino); +int jffs2_flash_write(struct jffs2_sb_info *c, loff_t ofs, size_t len, size_t *retlen, const u_char *buf); +int jffs2_flash_read(struct jffs2_sb_info *c, loff_t ofs, size_t len, size_t *retlen, u_char *buf); +int jffs2_check_oob_empty(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb,int mode); +int jffs2_check_nand_cleanmarker(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb); +int jffs2_write_nand_cleanmarker(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb); +int jffs2_write_nand_badblock(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, uint32_t bad_offset); +void jffs2_wbuf_timeout(unsigned long data); +void jffs2_wbuf_process(void *data); +int jffs2_flush_wbuf_gc(struct jffs2_sb_info *c, uint32_t ino); +int jffs2_flush_wbuf_pad(struct jffs2_sb_info *c); +int jffs2_nand_flash_setup(struct jffs2_sb_info *c); +void jffs2_nand_flash_cleanup(struct jffs2_sb_info *c); + +#define jffs2_dataflash(c) (c->mtd->type == MTD_DATAFLASH) +int jffs2_dataflash_setup(struct jffs2_sb_info *c); +void jffs2_dataflash_cleanup(struct jffs2_sb_info *c); +#define jffs2_ubivol(c) (c->mtd->type == MTD_UBIVOLUME) +int jffs2_ubivol_setup(struct jffs2_sb_info *c); +void jffs2_ubivol_cleanup(struct jffs2_sb_info *c); + +#define jffs2_nor_wbuf_flash(c) (c->mtd->type == MTD_NORFLASH && ! (c->mtd->flags & MTD_BIT_WRITEABLE)) +int jffs2_nor_wbuf_flash_setup(struct jffs2_sb_info *c); +void jffs2_nor_wbuf_flash_cleanup(struct jffs2_sb_info *c); +void jffs2_dirty_trigger(struct jffs2_sb_info *c); + +#endif /* WRITEBUFFER */ + +/* background.c */ +int jffs2_start_garbage_collect_thread(struct jffs2_sb_info *c); +void jffs2_stop_garbage_collect_thread(struct jffs2_sb_info *c); +void jffs2_garbage_collect_trigger(struct jffs2_sb_info *c); + +/* dir.c */ +extern const struct file_operations jffs2_dir_operations; +extern const struct inode_operations jffs2_dir_inode_operations; + +/* file.c */ +extern const struct file_operations jffs2_file_operations; +extern const struct inode_operations jffs2_file_inode_operations; +extern const struct address_space_operations jffs2_file_address_operations; +int jffs2_fsync(struct file *, loff_t, loff_t, int); +int __jffs2_read_folio(struct file *file, struct folio *folio); + +/* ioctl.c */ +long jffs2_ioctl(struct file *, unsigned int, unsigned long); + +/* symlink.c */ +extern const struct inode_operations jffs2_symlink_inode_operations; + +/* fs.c */ +int jffs2_setattr (struct mnt_idmap *, struct dentry *, struct iattr *); +int jffs2_do_setattr (struct inode *, struct iattr *); +struct inode *jffs2_iget(struct super_block *, unsigned long); +void jffs2_evict_inode (struct inode *); +void jffs2_dirty_inode(struct inode *inode, int flags); +struct inode *jffs2_new_inode (struct inode *dir_i, umode_t mode, + struct jffs2_raw_inode *ri); +int jffs2_statfs (struct dentry *, struct kstatfs *); +int jffs2_do_remount_fs(struct super_block *sb, struct fs_context *fc); +int jffs2_do_fill_super(struct super_block *sb, struct fs_context *fc); +void jffs2_gc_release_inode(struct jffs2_sb_info *c, + struct jffs2_inode_info *f); +struct jffs2_inode_info *jffs2_gc_fetch_inode(struct jffs2_sb_info *c, + int inum, int unlinked); + +unsigned char *jffs2_gc_fetch_page(struct jffs2_sb_info *c, + struct jffs2_inode_info *f, + unsigned long offset, + unsigned long *priv); +void jffs2_flash_cleanup(struct jffs2_sb_info *c); + + +/* writev.c */ +int jffs2_flash_direct_writev(struct jffs2_sb_info *c, const struct kvec *vecs, + unsigned long count, loff_t to, size_t *retlen); +int jffs2_flash_direct_write(struct jffs2_sb_info *c, loff_t ofs, size_t len, + size_t *retlen, const u_char *buf); + +#endif /* __JFFS2_OS_LINUX_H__ */ + + diff --git a/fs/jffs2/read.c b/fs/jffs2/read.c new file mode 100644 index 000000000..0b042b1fc --- /dev/null +++ b/fs/jffs2/read.c @@ -0,0 +1,228 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * + * Created by David Woodhouse + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include "nodelist.h" +#include "compr.h" + +int jffs2_read_dnode(struct jffs2_sb_info *c, struct jffs2_inode_info *f, + struct jffs2_full_dnode *fd, unsigned char *buf, + int ofs, int len) +{ + struct jffs2_raw_inode *ri; + size_t readlen; + uint32_t crc; + unsigned char *decomprbuf = NULL; + unsigned char *readbuf = NULL; + int ret = 0; + + ri = jffs2_alloc_raw_inode(); + if (!ri) + return -ENOMEM; + + ret = jffs2_flash_read(c, ref_offset(fd->raw), sizeof(*ri), &readlen, (char *)ri); + if (ret) { + jffs2_free_raw_inode(ri); + pr_warn("Error reading node from 0x%08x: %d\n", + ref_offset(fd->raw), ret); + return ret; + } + if (readlen != sizeof(*ri)) { + jffs2_free_raw_inode(ri); + pr_warn("Short read from 0x%08x: wanted 0x%zx bytes, got 0x%zx\n", + ref_offset(fd->raw), sizeof(*ri), readlen); + return -EIO; + } + crc = crc32(0, ri, sizeof(*ri)-8); + + jffs2_dbg(1, "Node read from %08x: node_crc %08x, calculated CRC %08x. dsize %x, csize %x, offset %x, buf %p\n", + ref_offset(fd->raw), je32_to_cpu(ri->node_crc), + crc, je32_to_cpu(ri->dsize), je32_to_cpu(ri->csize), + je32_to_cpu(ri->offset), buf); + if (crc != je32_to_cpu(ri->node_crc)) { + pr_warn("Node CRC %08x != calculated CRC %08x for node at %08x\n", + je32_to_cpu(ri->node_crc), crc, ref_offset(fd->raw)); + ret = -EIO; + goto out_ri; + } + /* There was a bug where we wrote hole nodes out with csize/dsize + swapped. Deal with it */ + if (ri->compr == JFFS2_COMPR_ZERO && !je32_to_cpu(ri->dsize) && + je32_to_cpu(ri->csize)) { + ri->dsize = ri->csize; + ri->csize = cpu_to_je32(0); + } + + D1(if(ofs + len > je32_to_cpu(ri->dsize)) { + pr_warn("jffs2_read_dnode() asked for %d bytes at %d from %d-byte node\n", + len, ofs, je32_to_cpu(ri->dsize)); + ret = -EINVAL; + goto out_ri; + }); + + + if (ri->compr == JFFS2_COMPR_ZERO) { + memset(buf, 0, len); + goto out_ri; + } + + /* Cases: + Reading whole node and it's uncompressed - read directly to buffer provided, check CRC. + Reading whole node and it's compressed - read into comprbuf, check CRC and decompress to buffer provided + Reading partial node and it's uncompressed - read into readbuf, check CRC, and copy + Reading partial node and it's compressed - read into readbuf, check checksum, decompress to decomprbuf and copy + */ + if (ri->compr == JFFS2_COMPR_NONE && len == je32_to_cpu(ri->dsize)) { + readbuf = buf; + } else { + readbuf = kmalloc(je32_to_cpu(ri->csize), GFP_KERNEL); + if (!readbuf) { + ret = -ENOMEM; + goto out_ri; + } + } + if (ri->compr != JFFS2_COMPR_NONE) { + if (len < je32_to_cpu(ri->dsize)) { + decomprbuf = kmalloc(je32_to_cpu(ri->dsize), GFP_KERNEL); + if (!decomprbuf) { + ret = -ENOMEM; + goto out_readbuf; + } + } else { + decomprbuf = buf; + } + } else { + decomprbuf = readbuf; + } + + jffs2_dbg(2, "Read %d bytes to %p\n", je32_to_cpu(ri->csize), + readbuf); + ret = jffs2_flash_read(c, (ref_offset(fd->raw)) + sizeof(*ri), + je32_to_cpu(ri->csize), &readlen, readbuf); + + if (!ret && readlen != je32_to_cpu(ri->csize)) + ret = -EIO; + if (ret) + goto out_decomprbuf; + + crc = crc32(0, readbuf, je32_to_cpu(ri->csize)); + if (crc != je32_to_cpu(ri->data_crc)) { + pr_warn("Data CRC %08x != calculated CRC %08x for node at %08x\n", + je32_to_cpu(ri->data_crc), crc, ref_offset(fd->raw)); + ret = -EIO; + goto out_decomprbuf; + } + jffs2_dbg(2, "Data CRC matches calculated CRC %08x\n", crc); + if (ri->compr != JFFS2_COMPR_NONE) { + jffs2_dbg(2, "Decompress %d bytes from %p to %d bytes at %p\n", + je32_to_cpu(ri->csize), readbuf, + je32_to_cpu(ri->dsize), decomprbuf); + ret = jffs2_decompress(c, f, ri->compr | (ri->usercompr << 8), readbuf, decomprbuf, je32_to_cpu(ri->csize), je32_to_cpu(ri->dsize)); + if (ret) { + pr_warn("Error: jffs2_decompress returned %d\n", ret); + goto out_decomprbuf; + } + } + + if (len < je32_to_cpu(ri->dsize)) { + memcpy(buf, decomprbuf+ofs, len); + } + out_decomprbuf: + if(decomprbuf != buf && decomprbuf != readbuf) + kfree(decomprbuf); + out_readbuf: + if(readbuf != buf) + kfree(readbuf); + out_ri: + jffs2_free_raw_inode(ri); + + return ret; +} + +int jffs2_read_inode_range(struct jffs2_sb_info *c, struct jffs2_inode_info *f, + unsigned char *buf, uint32_t offset, uint32_t len) +{ + uint32_t end = offset + len; + struct jffs2_node_frag *frag; + int ret; + + jffs2_dbg(1, "%s(): ino #%u, range 0x%08x-0x%08x\n", + __func__, f->inocache->ino, offset, offset + len); + + frag = jffs2_lookup_node_frag(&f->fragtree, offset); + + /* XXX FIXME: Where a single physical node actually shows up in two + frags, we read it twice. Don't do that. */ + /* Now we're pointing at the first frag which overlaps our page + * (or perhaps is before it, if we've been asked to read off the + * end of the file). */ + while(offset < end) { + jffs2_dbg(2, "%s(): offset %d, end %d\n", + __func__, offset, end); + if (unlikely(!frag || frag->ofs > offset || + frag->ofs + frag->size <= offset)) { + uint32_t holesize = end - offset; + if (frag && frag->ofs > offset) { + jffs2_dbg(1, "Eep. Hole in ino #%u fraglist. frag->ofs = 0x%08x, offset = 0x%08x\n", + f->inocache->ino, frag->ofs, offset); + holesize = min(holesize, frag->ofs - offset); + } + jffs2_dbg(1, "Filling non-frag hole from %d-%d\n", + offset, offset + holesize); + memset(buf, 0, holesize); + buf += holesize; + offset += holesize; + continue; + } else if (unlikely(!frag->node)) { + uint32_t holeend = min(end, frag->ofs + frag->size); + jffs2_dbg(1, "Filling frag hole from %d-%d (frag 0x%x 0x%x)\n", + offset, holeend, frag->ofs, + frag->ofs + frag->size); + memset(buf, 0, holeend - offset); + buf += holeend - offset; + offset = holeend; + frag = frag_next(frag); + continue; + } else { + uint32_t readlen; + uint32_t fragofs; /* offset within the frag to start reading */ + + fragofs = offset - frag->ofs; + readlen = min(frag->size - fragofs, end - offset); + jffs2_dbg(1, "Reading %d-%d from node at 0x%08x (%d)\n", + frag->ofs+fragofs, + frag->ofs + fragofs+readlen, + ref_offset(frag->node->raw), + ref_flags(frag->node->raw)); + ret = jffs2_read_dnode(c, f, frag->node, buf, fragofs + frag->ofs - frag->node->ofs, readlen); + jffs2_dbg(2, "node read done\n"); + if (ret) { + jffs2_dbg(1, "%s(): error %d\n", + __func__, ret); + memset(buf, 0, readlen); + return ret; + } + buf += readlen; + offset += readlen; + frag = frag_next(frag); + jffs2_dbg(2, "node read was OK. Looping\n"); + } + } + return 0; +} + diff --git a/fs/jffs2/readinode.c b/fs/jffs2/readinode.c new file mode 100644 index 000000000..1caabff9d --- /dev/null +++ b/fs/jffs2/readinode.c @@ -0,0 +1,1447 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * + * Created by David Woodhouse + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include "nodelist.h" + +/* + * Check the data CRC of the node. + * + * Returns: 0 if the data CRC is correct; + * 1 - if incorrect; + * error code if an error occurred. + */ +static int check_node_data(struct jffs2_sb_info *c, struct jffs2_tmp_dnode_info *tn) +{ + struct jffs2_raw_node_ref *ref = tn->fn->raw; + int err = 0, pointed = 0; + struct jffs2_eraseblock *jeb; + unsigned char *buffer; + uint32_t crc, ofs, len; + size_t retlen; + + BUG_ON(tn->csize == 0); + + /* Calculate how many bytes were already checked */ + ofs = ref_offset(ref) + sizeof(struct jffs2_raw_inode); + len = tn->csize; + + if (jffs2_is_writebuffered(c)) { + int adj = ofs % c->wbuf_pagesize; + if (likely(adj)) + adj = c->wbuf_pagesize - adj; + + if (adj >= tn->csize) { + dbg_readinode("no need to check node at %#08x, data length %u, data starts at %#08x - it has already been checked.\n", + ref_offset(ref), tn->csize, ofs); + goto adj_acc; + } + + ofs += adj; + len -= adj; + } + + dbg_readinode("check node at %#08x, data length %u, partial CRC %#08x, correct CRC %#08x, data starts at %#08x, start checking from %#08x - %u bytes.\n", + ref_offset(ref), tn->csize, tn->partial_crc, tn->data_crc, ofs - len, ofs, len); + +#ifndef __ECOS + /* TODO: instead, incapsulate point() stuff to jffs2_flash_read(), + * adding and jffs2_flash_read_end() interface. */ + err = mtd_point(c->mtd, ofs, len, &retlen, (void **)&buffer, NULL); + if (!err && retlen < len) { + JFFS2_WARNING("MTD point returned len too short: %zu instead of %u.\n", retlen, tn->csize); + mtd_unpoint(c->mtd, ofs, retlen); + } else if (err) { + if (err != -EOPNOTSUPP) + JFFS2_WARNING("MTD point failed: error code %d.\n", err); + } else + pointed = 1; /* successfully pointed to device */ +#endif + + if (!pointed) { + buffer = kmalloc(len, GFP_KERNEL); + if (unlikely(!buffer)) + return -ENOMEM; + + /* TODO: this is very frequent pattern, make it a separate + * routine */ + err = jffs2_flash_read(c, ofs, len, &retlen, buffer); + if (err) { + JFFS2_ERROR("can not read %d bytes from 0x%08x, error code: %d.\n", len, ofs, err); + goto free_out; + } + + if (retlen != len) { + JFFS2_ERROR("short read at %#08x: %zd instead of %d.\n", ofs, retlen, len); + err = -EIO; + goto free_out; + } + } + + /* Continue calculating CRC */ + crc = crc32(tn->partial_crc, buffer, len); + if(!pointed) + kfree(buffer); +#ifndef __ECOS + else + mtd_unpoint(c->mtd, ofs, len); +#endif + + if (crc != tn->data_crc) { + JFFS2_NOTICE("wrong data CRC in data node at 0x%08x: read %#08x, calculated %#08x.\n", + ref_offset(ref), tn->data_crc, crc); + return 1; + } + +adj_acc: + jeb = &c->blocks[ref->flash_offset / c->sector_size]; + len = ref_totlen(c, jeb, ref); + /* If it should be REF_NORMAL, it'll get marked as such when + we build the fragtree, shortly. No need to worry about GC + moving it while it's marked REF_PRISTINE -- GC won't happen + till we've finished checking every inode anyway. */ + ref->flash_offset |= REF_PRISTINE; + /* + * Mark the node as having been checked and fix the + * accounting accordingly. + */ + spin_lock(&c->erase_completion_lock); + jeb->used_size += len; + jeb->unchecked_size -= len; + c->used_size += len; + c->unchecked_size -= len; + jffs2_dbg_acct_paranoia_check_nolock(c, jeb); + spin_unlock(&c->erase_completion_lock); + + return 0; + +free_out: + if(!pointed) + kfree(buffer); +#ifndef __ECOS + else + mtd_unpoint(c->mtd, ofs, len); +#endif + return err; +} + +/* + * Helper function for jffs2_add_older_frag_to_fragtree(). + * + * Checks the node if we are in the checking stage. + */ +static int check_tn_node(struct jffs2_sb_info *c, struct jffs2_tmp_dnode_info *tn) +{ + int ret; + + BUG_ON(ref_obsolete(tn->fn->raw)); + + /* We only check the data CRC of unchecked nodes */ + if (ref_flags(tn->fn->raw) != REF_UNCHECKED) + return 0; + + dbg_readinode("check node %#04x-%#04x, phys offs %#08x\n", + tn->fn->ofs, tn->fn->ofs + tn->fn->size, ref_offset(tn->fn->raw)); + + ret = check_node_data(c, tn); + if (unlikely(ret < 0)) { + JFFS2_ERROR("check_node_data() returned error: %d.\n", + ret); + } else if (unlikely(ret > 0)) { + dbg_readinode("CRC error, mark it obsolete.\n"); + jffs2_mark_node_obsolete(c, tn->fn->raw); + } + + return ret; +} + +static struct jffs2_tmp_dnode_info *jffs2_lookup_tn(struct rb_root *tn_root, uint32_t offset) +{ + struct rb_node *next; + struct jffs2_tmp_dnode_info *tn = NULL; + + dbg_readinode("root %p, offset %d\n", tn_root, offset); + + next = tn_root->rb_node; + + while (next) { + tn = rb_entry(next, struct jffs2_tmp_dnode_info, rb); + + if (tn->fn->ofs < offset) + next = tn->rb.rb_right; + else if (tn->fn->ofs >= offset) + next = tn->rb.rb_left; + else + break; + } + + return tn; +} + + +static void jffs2_kill_tn(struct jffs2_sb_info *c, struct jffs2_tmp_dnode_info *tn) +{ + jffs2_mark_node_obsolete(c, tn->fn->raw); + jffs2_free_full_dnode(tn->fn); + jffs2_free_tmp_dnode_info(tn); +} +/* + * This function is used when we read an inode. Data nodes arrive in + * arbitrary order -- they may be older or newer than the nodes which + * are already in the tree. Where overlaps occur, the older node can + * be discarded as long as the newer passes the CRC check. We don't + * bother to keep track of holes in this rbtree, and neither do we deal + * with frags -- we can have multiple entries starting at the same + * offset, and the one with the smallest length will come first in the + * ordering. + * + * Returns 0 if the node was handled (including marking it obsolete) + * < 0 an if error occurred + */ +static int jffs2_add_tn_to_tree(struct jffs2_sb_info *c, + struct jffs2_readinode_info *rii, + struct jffs2_tmp_dnode_info *tn) +{ + uint32_t fn_end = tn->fn->ofs + tn->fn->size; + struct jffs2_tmp_dnode_info *this, *ptn; + + dbg_readinode("insert fragment %#04x-%#04x, ver %u at %08x\n", tn->fn->ofs, fn_end, tn->version, ref_offset(tn->fn->raw)); + + /* If a node has zero dsize, we only have to keep it if it might be the + node with highest version -- i.e. the one which will end up as f->metadata. + Note that such nodes won't be REF_UNCHECKED since there are no data to + check anyway. */ + if (!tn->fn->size) { + if (rii->mdata_tn) { + if (rii->mdata_tn->version < tn->version) { + /* We had a candidate mdata node already */ + dbg_readinode("kill old mdata with ver %d\n", rii->mdata_tn->version); + jffs2_kill_tn(c, rii->mdata_tn); + } else { + dbg_readinode("kill new mdata with ver %d (older than existing %d\n", + tn->version, rii->mdata_tn->version); + jffs2_kill_tn(c, tn); + return 0; + } + } + rii->mdata_tn = tn; + dbg_readinode("keep new mdata with ver %d\n", tn->version); + return 0; + } + + /* Find the earliest node which _may_ be relevant to this one */ + this = jffs2_lookup_tn(&rii->tn_root, tn->fn->ofs); + if (this) { + /* If the node is coincident with another at a lower address, + back up until the other node is found. It may be relevant */ + while (this->overlapped) { + ptn = tn_prev(this); + if (!ptn) { + /* + * We killed a node which set the overlapped + * flags during the scan. Fix it up. + */ + this->overlapped = 0; + break; + } + this = ptn; + } + dbg_readinode("'this' found %#04x-%#04x (%s)\n", this->fn->ofs, this->fn->ofs + this->fn->size, this->fn ? "data" : "hole"); + } + + while (this) { + if (this->fn->ofs > fn_end) + break; + dbg_readinode("Ponder this ver %d, 0x%x-0x%x\n", + this->version, this->fn->ofs, this->fn->size); + + if (this->version == tn->version) { + /* Version number collision means REF_PRISTINE GC. Accept either of them + as long as the CRC is correct. Check the one we have already... */ + if (!check_tn_node(c, this)) { + /* The one we already had was OK. Keep it and throw away the new one */ + dbg_readinode("Like old node. Throw away new\n"); + jffs2_kill_tn(c, tn); + return 0; + } else { + /* Who cares if the new one is good; keep it for now anyway. */ + dbg_readinode("Like new node. Throw away old\n"); + rb_replace_node(&this->rb, &tn->rb, &rii->tn_root); + jffs2_kill_tn(c, this); + /* Same overlapping from in front and behind */ + return 0; + } + } + if (this->version < tn->version && + this->fn->ofs >= tn->fn->ofs && + this->fn->ofs + this->fn->size <= fn_end) { + /* New node entirely overlaps 'this' */ + if (check_tn_node(c, tn)) { + dbg_readinode("new node bad CRC\n"); + jffs2_kill_tn(c, tn); + return 0; + } + /* ... and is good. Kill 'this' and any subsequent nodes which are also overlapped */ + while (this && this->fn->ofs + this->fn->size <= fn_end) { + struct jffs2_tmp_dnode_info *next = tn_next(this); + if (this->version < tn->version) { + tn_erase(this, &rii->tn_root); + dbg_readinode("Kill overlapped ver %d, 0x%x-0x%x\n", + this->version, this->fn->ofs, + this->fn->ofs+this->fn->size); + jffs2_kill_tn(c, this); + } + this = next; + } + dbg_readinode("Done killing overlapped nodes\n"); + continue; + } + if (this->version > tn->version && + this->fn->ofs <= tn->fn->ofs && + this->fn->ofs+this->fn->size >= fn_end) { + /* New node entirely overlapped by 'this' */ + if (!check_tn_node(c, this)) { + dbg_readinode("Good CRC on old node. Kill new\n"); + jffs2_kill_tn(c, tn); + return 0; + } + /* ... but 'this' was bad. Replace it... */ + dbg_readinode("Bad CRC on old overlapping node. Kill it\n"); + tn_erase(this, &rii->tn_root); + jffs2_kill_tn(c, this); + break; + } + + this = tn_next(this); + } + + /* We neither completely obsoleted nor were completely + obsoleted by an earlier node. Insert into the tree */ + { + struct rb_node *parent; + struct rb_node **link = &rii->tn_root.rb_node; + struct jffs2_tmp_dnode_info *insert_point = NULL; + + while (*link) { + parent = *link; + insert_point = rb_entry(parent, struct jffs2_tmp_dnode_info, rb); + if (tn->fn->ofs > insert_point->fn->ofs) + link = &insert_point->rb.rb_right; + else if (tn->fn->ofs < insert_point->fn->ofs || + tn->fn->size < insert_point->fn->size) + link = &insert_point->rb.rb_left; + else + link = &insert_point->rb.rb_right; + } + rb_link_node(&tn->rb, &insert_point->rb, link); + rb_insert_color(&tn->rb, &rii->tn_root); + } + + /* If there's anything behind that overlaps us, note it */ + this = tn_prev(tn); + if (this) { + while (1) { + if (this->fn->ofs + this->fn->size > tn->fn->ofs) { + dbg_readinode("Node is overlapped by %p (v %d, 0x%x-0x%x)\n", + this, this->version, this->fn->ofs, + this->fn->ofs+this->fn->size); + tn->overlapped = 1; + break; + } + if (!this->overlapped) + break; + + ptn = tn_prev(this); + if (!ptn) { + /* + * We killed a node which set the overlapped + * flags during the scan. Fix it up. + */ + this->overlapped = 0; + break; + } + this = ptn; + } + } + + /* If the new node overlaps anything ahead, note it */ + this = tn_next(tn); + while (this && this->fn->ofs < fn_end) { + this->overlapped = 1; + dbg_readinode("Node ver %d, 0x%x-0x%x is overlapped\n", + this->version, this->fn->ofs, + this->fn->ofs+this->fn->size); + this = tn_next(this); + } + return 0; +} + +/* Trivial function to remove the last node in the tree. Which by definition + has no right-hand child — so can be removed just by making its left-hand + child (if any) take its place under its parent. Since this is only done + when we're consuming the whole tree, there's no need to use rb_erase() + and let it worry about adjusting colours and balancing the tree. That + would just be a waste of time. */ +static void eat_last(struct rb_root *root, struct rb_node *node) +{ + struct rb_node *parent = rb_parent(node); + struct rb_node **link; + + /* LAST! */ + BUG_ON(node->rb_right); + + if (!parent) + link = &root->rb_node; + else if (node == parent->rb_left) + link = &parent->rb_left; + else + link = &parent->rb_right; + + *link = node->rb_left; + if (node->rb_left) + node->rb_left->__rb_parent_color = node->__rb_parent_color; +} + +/* We put the version tree in reverse order, so we can use the same eat_last() + function that we use to consume the tmpnode tree (tn_root). */ +static void ver_insert(struct rb_root *ver_root, struct jffs2_tmp_dnode_info *tn) +{ + struct rb_node **link = &ver_root->rb_node; + struct rb_node *parent = NULL; + struct jffs2_tmp_dnode_info *this_tn; + + while (*link) { + parent = *link; + this_tn = rb_entry(parent, struct jffs2_tmp_dnode_info, rb); + + if (tn->version > this_tn->version) + link = &parent->rb_left; + else + link = &parent->rb_right; + } + dbg_readinode("Link new node at %p (root is %p)\n", link, ver_root); + rb_link_node(&tn->rb, parent, link); + rb_insert_color(&tn->rb, ver_root); +} + +/* Build final, normal fragtree from tn tree. It doesn't matter which order + we add nodes to the real fragtree, as long as they don't overlap. And + having thrown away the majority of overlapped nodes as we went, there + really shouldn't be many sets of nodes which do overlap. If we start at + the end, we can use the overlap markers -- we can just eat nodes which + aren't overlapped, and when we encounter nodes which _do_ overlap we + sort them all into a temporary tree in version order before replaying them. */ +static int jffs2_build_inode_fragtree(struct jffs2_sb_info *c, + struct jffs2_inode_info *f, + struct jffs2_readinode_info *rii) +{ + struct jffs2_tmp_dnode_info *pen, *last, *this; + struct rb_root ver_root = RB_ROOT; + uint32_t high_ver = 0; + + if (rii->mdata_tn) { + dbg_readinode("potential mdata is ver %d at %p\n", rii->mdata_tn->version, rii->mdata_tn); + high_ver = rii->mdata_tn->version; + rii->latest_ref = rii->mdata_tn->fn->raw; + } +#ifdef JFFS2_DBG_READINODE_MESSAGES + this = tn_last(&rii->tn_root); + while (this) { + dbg_readinode("tn %p ver %d range 0x%x-0x%x ov %d\n", this, this->version, this->fn->ofs, + this->fn->ofs+this->fn->size, this->overlapped); + this = tn_prev(this); + } +#endif + pen = tn_last(&rii->tn_root); + while ((last = pen)) { + pen = tn_prev(last); + + eat_last(&rii->tn_root, &last->rb); + ver_insert(&ver_root, last); + + if (unlikely(last->overlapped)) { + if (pen) + continue; + /* + * We killed a node which set the overlapped + * flags during the scan. Fix it up. + */ + last->overlapped = 0; + } + + /* Now we have a bunch of nodes in reverse version + order, in the tree at ver_root. Most of the time, + there'll actually be only one node in the 'tree', + in fact. */ + this = tn_last(&ver_root); + + while (this) { + struct jffs2_tmp_dnode_info *vers_next; + int ret; + vers_next = tn_prev(this); + eat_last(&ver_root, &this->rb); + if (check_tn_node(c, this)) { + dbg_readinode("node ver %d, 0x%x-0x%x failed CRC\n", + this->version, this->fn->ofs, + this->fn->ofs+this->fn->size); + jffs2_kill_tn(c, this); + } else { + if (this->version > high_ver) { + /* Note that this is different from the other + highest_version, because this one is only + counting _valid_ nodes which could give the + latest inode metadata */ + high_ver = this->version; + rii->latest_ref = this->fn->raw; + } + dbg_readinode("Add %p (v %d, 0x%x-0x%x, ov %d) to fragtree\n", + this, this->version, this->fn->ofs, + this->fn->ofs+this->fn->size, this->overlapped); + + ret = jffs2_add_full_dnode_to_inode(c, f, this->fn); + if (ret) { + /* Free the nodes in vers_root; let the caller + deal with the rest */ + JFFS2_ERROR("Add node to tree failed %d\n", ret); + while (1) { + vers_next = tn_prev(this); + if (check_tn_node(c, this)) + jffs2_mark_node_obsolete(c, this->fn->raw); + jffs2_free_full_dnode(this->fn); + jffs2_free_tmp_dnode_info(this); + this = vers_next; + if (!this) + break; + eat_last(&ver_root, &vers_next->rb); + } + return ret; + } + jffs2_free_tmp_dnode_info(this); + } + this = vers_next; + } + } + return 0; +} + +static void jffs2_free_tmp_dnode_info_list(struct rb_root *list) +{ + struct jffs2_tmp_dnode_info *tn, *next; + + rbtree_postorder_for_each_entry_safe(tn, next, list, rb) { + jffs2_free_full_dnode(tn->fn); + jffs2_free_tmp_dnode_info(tn); + } + + *list = RB_ROOT; +} + +static void jffs2_free_full_dirent_list(struct jffs2_full_dirent *fd) +{ + struct jffs2_full_dirent *next; + + while (fd) { + next = fd->next; + jffs2_free_full_dirent(fd); + fd = next; + } +} + +/* Returns first valid node after 'ref'. May return 'ref' */ +static struct jffs2_raw_node_ref *jffs2_first_valid_node(struct jffs2_raw_node_ref *ref) +{ + while (ref && ref->next_in_ino) { + if (!ref_obsolete(ref)) + return ref; + dbg_noderef("node at 0x%08x is obsoleted. Ignoring.\n", ref_offset(ref)); + ref = ref->next_in_ino; + } + return NULL; +} + +/* + * Helper function for jffs2_get_inode_nodes(). + * It is called every time an directory entry node is found. + * + * Returns: 0 on success; + * negative error code on failure. + */ +static inline int read_direntry(struct jffs2_sb_info *c, struct jffs2_raw_node_ref *ref, + struct jffs2_raw_dirent *rd, size_t read, + struct jffs2_readinode_info *rii) +{ + struct jffs2_full_dirent *fd; + uint32_t crc; + + /* Obsoleted. This cannot happen, surely? dwmw2 20020308 */ + BUG_ON(ref_obsolete(ref)); + + crc = crc32(0, rd, sizeof(*rd) - 8); + if (unlikely(crc != je32_to_cpu(rd->node_crc))) { + JFFS2_NOTICE("header CRC failed on dirent node at %#08x: read %#08x, calculated %#08x\n", + ref_offset(ref), je32_to_cpu(rd->node_crc), crc); + jffs2_mark_node_obsolete(c, ref); + return 0; + } + + /* If we've never checked the CRCs on this node, check them now */ + if (ref_flags(ref) == REF_UNCHECKED) { + struct jffs2_eraseblock *jeb; + int len; + + /* Sanity check */ + if (unlikely(PAD((rd->nsize + sizeof(*rd))) != PAD(je32_to_cpu(rd->totlen)))) { + JFFS2_ERROR("illegal nsize in node at %#08x: nsize %#02x, totlen %#04x\n", + ref_offset(ref), rd->nsize, je32_to_cpu(rd->totlen)); + jffs2_mark_node_obsolete(c, ref); + return 0; + } + + jeb = &c->blocks[ref->flash_offset / c->sector_size]; + len = ref_totlen(c, jeb, ref); + + spin_lock(&c->erase_completion_lock); + jeb->used_size += len; + jeb->unchecked_size -= len; + c->used_size += len; + c->unchecked_size -= len; + ref->flash_offset = ref_offset(ref) | dirent_node_state(rd); + spin_unlock(&c->erase_completion_lock); + } + + fd = jffs2_alloc_full_dirent(rd->nsize + 1); + if (unlikely(!fd)) + return -ENOMEM; + + fd->raw = ref; + fd->version = je32_to_cpu(rd->version); + fd->ino = je32_to_cpu(rd->ino); + fd->type = rd->type; + + if (fd->version > rii->highest_version) + rii->highest_version = fd->version; + + /* Pick out the mctime of the latest dirent */ + if(fd->version > rii->mctime_ver && je32_to_cpu(rd->mctime)) { + rii->mctime_ver = fd->version; + rii->latest_mctime = je32_to_cpu(rd->mctime); + } + + /* + * Copy as much of the name as possible from the raw + * dirent we've already read from the flash. + */ + if (read > sizeof(*rd)) + memcpy(&fd->name[0], &rd->name[0], + min_t(uint32_t, rd->nsize, (read - sizeof(*rd)) )); + + /* Do we need to copy any more of the name directly from the flash? */ + if (rd->nsize + sizeof(*rd) > read) { + /* FIXME: point() */ + int err; + int already = read - sizeof(*rd); + + err = jffs2_flash_read(c, (ref_offset(ref)) + read, + rd->nsize - already, &read, &fd->name[already]); + if (unlikely(read != rd->nsize - already) && likely(!err)) { + jffs2_free_full_dirent(fd); + JFFS2_ERROR("short read: wanted %d bytes, got %zd\n", + rd->nsize - already, read); + return -EIO; + } + + if (unlikely(err)) { + JFFS2_ERROR("read remainder of name: error %d\n", err); + jffs2_free_full_dirent(fd); + return -EIO; + } + +#ifdef CONFIG_JFFS2_SUMMARY + /* + * we use CONFIG_JFFS2_SUMMARY because without it, we + * have checked it while mounting + */ + crc = crc32(0, fd->name, rd->nsize); + if (unlikely(crc != je32_to_cpu(rd->name_crc))) { + JFFS2_NOTICE("name CRC failed on dirent node at" + "%#08x: read %#08x,calculated %#08x\n", + ref_offset(ref), je32_to_cpu(rd->node_crc), crc); + jffs2_mark_node_obsolete(c, ref); + jffs2_free_full_dirent(fd); + return 0; + } +#endif + } + + fd->nhash = full_name_hash(NULL, fd->name, rd->nsize); + fd->next = NULL; + fd->name[rd->nsize] = '\0'; + + /* + * Wheee. We now have a complete jffs2_full_dirent structure, with + * the name in it and everything. Link it into the list + */ + jffs2_add_fd_to_list(c, fd, &rii->fds); + + return 0; +} + +/* + * Helper function for jffs2_get_inode_nodes(). + * It is called every time an inode node is found. + * + * Returns: 0 on success (possibly after marking a bad node obsolete); + * negative error code on failure. + */ +static inline int read_dnode(struct jffs2_sb_info *c, struct jffs2_raw_node_ref *ref, + struct jffs2_raw_inode *rd, int rdlen, + struct jffs2_readinode_info *rii) +{ + struct jffs2_tmp_dnode_info *tn; + uint32_t len, csize; + int ret = 0; + uint32_t crc; + + /* Obsoleted. This cannot happen, surely? dwmw2 20020308 */ + BUG_ON(ref_obsolete(ref)); + + crc = crc32(0, rd, sizeof(*rd) - 8); + if (unlikely(crc != je32_to_cpu(rd->node_crc))) { + JFFS2_NOTICE("node CRC failed on dnode at %#08x: read %#08x, calculated %#08x\n", + ref_offset(ref), je32_to_cpu(rd->node_crc), crc); + jffs2_mark_node_obsolete(c, ref); + return 0; + } + + tn = jffs2_alloc_tmp_dnode_info(); + if (!tn) { + JFFS2_ERROR("failed to allocate tn (%zu bytes).\n", sizeof(*tn)); + return -ENOMEM; + } + + tn->partial_crc = 0; + csize = je32_to_cpu(rd->csize); + + /* If we've never checked the CRCs on this node, check them now */ + if (ref_flags(ref) == REF_UNCHECKED) { + + /* Sanity checks */ + if (unlikely(je32_to_cpu(rd->offset) > je32_to_cpu(rd->isize)) || + unlikely(PAD(je32_to_cpu(rd->csize) + sizeof(*rd)) != PAD(je32_to_cpu(rd->totlen)))) { + JFFS2_WARNING("inode node header CRC is corrupted at %#08x\n", ref_offset(ref)); + jffs2_dbg_dump_node(c, ref_offset(ref)); + jffs2_mark_node_obsolete(c, ref); + goto free_out; + } + + if (jffs2_is_writebuffered(c) && csize != 0) { + /* At this point we are supposed to check the data CRC + * of our unchecked node. But thus far, we do not + * know whether the node is valid or obsolete. To + * figure this out, we need to walk all the nodes of + * the inode and build the inode fragtree. We don't + * want to spend time checking data of nodes which may + * later be found to be obsolete. So we put off the full + * data CRC checking until we have read all the inode + * nodes and have started building the fragtree. + * + * The fragtree is being built starting with nodes + * having the highest version number, so we'll be able + * to detect whether a node is valid (i.e., it is not + * overlapped by a node with higher version) or not. + * And we'll be able to check only those nodes, which + * are not obsolete. + * + * Of course, this optimization only makes sense in case + * of NAND flashes (or other flashes with + * !jffs2_can_mark_obsolete()), since on NOR flashes + * nodes are marked obsolete physically. + * + * Since NAND flashes (or other flashes with + * jffs2_is_writebuffered(c)) are anyway read by + * fractions of c->wbuf_pagesize, and we have just read + * the node header, it is likely that the starting part + * of the node data is also read when we read the + * header. So we don't mind to check the CRC of the + * starting part of the data of the node now, and check + * the second part later (in jffs2_check_node_data()). + * Of course, we will not need to re-read and re-check + * the NAND page which we have just read. This is why we + * read the whole NAND page at jffs2_get_inode_nodes(), + * while we needed only the node header. + */ + unsigned char *buf; + + /* 'buf' will point to the start of data */ + buf = (unsigned char *)rd + sizeof(*rd); + /* len will be the read data length */ + len = min_t(uint32_t, rdlen - sizeof(*rd), csize); + tn->partial_crc = crc32(0, buf, len); + + dbg_readinode("Calculates CRC (%#08x) for %d bytes, csize %d\n", tn->partial_crc, len, csize); + + /* If we actually calculated the whole data CRC + * and it is wrong, drop the node. */ + if (len >= csize && unlikely(tn->partial_crc != je32_to_cpu(rd->data_crc))) { + JFFS2_NOTICE("wrong data CRC in data node at 0x%08x: read %#08x, calculated %#08x.\n", + ref_offset(ref), tn->partial_crc, je32_to_cpu(rd->data_crc)); + jffs2_mark_node_obsolete(c, ref); + goto free_out; + } + + } else if (csize == 0) { + /* + * We checked the header CRC. If the node has no data, adjust + * the space accounting now. For other nodes this will be done + * later either when the node is marked obsolete or when its + * data is checked. + */ + struct jffs2_eraseblock *jeb; + + dbg_readinode("the node has no data.\n"); + jeb = &c->blocks[ref->flash_offset / c->sector_size]; + len = ref_totlen(c, jeb, ref); + + spin_lock(&c->erase_completion_lock); + jeb->used_size += len; + jeb->unchecked_size -= len; + c->used_size += len; + c->unchecked_size -= len; + ref->flash_offset = ref_offset(ref) | REF_NORMAL; + spin_unlock(&c->erase_completion_lock); + } + } + + tn->fn = jffs2_alloc_full_dnode(); + if (!tn->fn) { + JFFS2_ERROR("alloc fn failed\n"); + ret = -ENOMEM; + goto free_out; + } + + tn->version = je32_to_cpu(rd->version); + tn->fn->ofs = je32_to_cpu(rd->offset); + tn->data_crc = je32_to_cpu(rd->data_crc); + tn->csize = csize; + tn->fn->raw = ref; + tn->overlapped = 0; + + if (tn->version > rii->highest_version) + rii->highest_version = tn->version; + + /* There was a bug where we wrote hole nodes out with + csize/dsize swapped. Deal with it */ + if (rd->compr == JFFS2_COMPR_ZERO && !je32_to_cpu(rd->dsize) && csize) + tn->fn->size = csize; + else // normal case... + tn->fn->size = je32_to_cpu(rd->dsize); + + dbg_readinode2("dnode @%08x: ver %u, offset %#04x, dsize %#04x, csize %#04x\n", + ref_offset(ref), je32_to_cpu(rd->version), + je32_to_cpu(rd->offset), je32_to_cpu(rd->dsize), csize); + + ret = jffs2_add_tn_to_tree(c, rii, tn); + + if (ret) { + jffs2_free_full_dnode(tn->fn); + free_out: + jffs2_free_tmp_dnode_info(tn); + return ret; + } +#ifdef JFFS2_DBG_READINODE2_MESSAGES + dbg_readinode2("After adding ver %d:\n", je32_to_cpu(rd->version)); + tn = tn_first(&rii->tn_root); + while (tn) { + dbg_readinode2("%p: v %d r 0x%x-0x%x ov %d\n", + tn, tn->version, tn->fn->ofs, + tn->fn->ofs+tn->fn->size, tn->overlapped); + tn = tn_next(tn); + } +#endif + return 0; +} + +/* + * Helper function for jffs2_get_inode_nodes(). + * It is called every time an unknown node is found. + * + * Returns: 0 on success; + * negative error code on failure. + */ +static inline int read_unknown(struct jffs2_sb_info *c, struct jffs2_raw_node_ref *ref, struct jffs2_unknown_node *un) +{ + /* We don't mark unknown nodes as REF_UNCHECKED */ + if (ref_flags(ref) == REF_UNCHECKED) { + JFFS2_ERROR("REF_UNCHECKED but unknown node at %#08x\n", + ref_offset(ref)); + JFFS2_ERROR("Node is {%04x,%04x,%08x,%08x}. Please report this error.\n", + je16_to_cpu(un->magic), je16_to_cpu(un->nodetype), + je32_to_cpu(un->totlen), je32_to_cpu(un->hdr_crc)); + jffs2_mark_node_obsolete(c, ref); + return 0; + } + + un->nodetype = cpu_to_je16(JFFS2_NODE_ACCURATE | je16_to_cpu(un->nodetype)); + + switch(je16_to_cpu(un->nodetype) & JFFS2_COMPAT_MASK) { + + case JFFS2_FEATURE_INCOMPAT: + JFFS2_ERROR("unknown INCOMPAT nodetype %#04X at %#08x\n", + je16_to_cpu(un->nodetype), ref_offset(ref)); + /* EEP */ + BUG(); + break; + + case JFFS2_FEATURE_ROCOMPAT: + JFFS2_ERROR("unknown ROCOMPAT nodetype %#04X at %#08x\n", + je16_to_cpu(un->nodetype), ref_offset(ref)); + BUG_ON(!(c->flags & JFFS2_SB_FLAG_RO)); + break; + + case JFFS2_FEATURE_RWCOMPAT_COPY: + JFFS2_NOTICE("unknown RWCOMPAT_COPY nodetype %#04X at %#08x\n", + je16_to_cpu(un->nodetype), ref_offset(ref)); + break; + + case JFFS2_FEATURE_RWCOMPAT_DELETE: + JFFS2_NOTICE("unknown RWCOMPAT_DELETE nodetype %#04X at %#08x\n", + je16_to_cpu(un->nodetype), ref_offset(ref)); + jffs2_mark_node_obsolete(c, ref); + return 0; + } + + return 0; +} + +/* + * Helper function for jffs2_get_inode_nodes(). + * The function detects whether more data should be read and reads it if yes. + * + * Returns: 0 on success; + * negative error code on failure. + */ +static int read_more(struct jffs2_sb_info *c, struct jffs2_raw_node_ref *ref, + int needed_len, int *rdlen, unsigned char *buf) +{ + int err, to_read = needed_len - *rdlen; + size_t retlen; + uint32_t offs; + + if (jffs2_is_writebuffered(c)) { + int rem = to_read % c->wbuf_pagesize; + + if (rem) + to_read += c->wbuf_pagesize - rem; + } + + /* We need to read more data */ + offs = ref_offset(ref) + *rdlen; + + dbg_readinode("read more %d bytes\n", to_read); + + err = jffs2_flash_read(c, offs, to_read, &retlen, buf + *rdlen); + if (err) { + JFFS2_ERROR("can not read %d bytes from 0x%08x, " + "error code: %d.\n", to_read, offs, err); + return err; + } + + if (retlen < to_read) { + JFFS2_ERROR("short read at %#08x: %zu instead of %d.\n", + offs, retlen, to_read); + return -EIO; + } + + *rdlen += to_read; + return 0; +} + +/* Get tmp_dnode_info and full_dirent for all non-obsolete nodes associated + with this ino. Perform a preliminary ordering on data nodes, throwing away + those which are completely obsoleted by newer ones. The naïve approach we + use to take of just returning them _all_ in version order will cause us to + run out of memory in certain degenerate cases. */ +static int jffs2_get_inode_nodes(struct jffs2_sb_info *c, struct jffs2_inode_info *f, + struct jffs2_readinode_info *rii) +{ + struct jffs2_raw_node_ref *ref, *valid_ref; + unsigned char *buf = NULL; + union jffs2_node_union *node; + size_t retlen; + int len, err; + + rii->mctime_ver = 0; + + dbg_readinode("ino #%u\n", f->inocache->ino); + + /* FIXME: in case of NOR and available ->point() this + * needs to be fixed. */ + len = sizeof(union jffs2_node_union) + c->wbuf_pagesize; + buf = kmalloc(len, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + spin_lock(&c->erase_completion_lock); + valid_ref = jffs2_first_valid_node(f->inocache->nodes); + if (!valid_ref && f->inocache->ino != 1) + JFFS2_WARNING("Eep. No valid nodes for ino #%u.\n", f->inocache->ino); + while (valid_ref) { + /* We can hold a pointer to a non-obsolete node without the spinlock, + but _obsolete_ nodes may disappear at any time, if the block + they're in gets erased. So if we mark 'ref' obsolete while we're + not holding the lock, it can go away immediately. For that reason, + we find the next valid node first, before processing 'ref'. + */ + ref = valid_ref; + valid_ref = jffs2_first_valid_node(ref->next_in_ino); + spin_unlock(&c->erase_completion_lock); + + cond_resched(); + + /* + * At this point we don't know the type of the node we're going + * to read, so we do not know the size of its header. In order + * to minimize the amount of flash IO we assume the header is + * of size = JFFS2_MIN_NODE_HEADER. + */ + len = JFFS2_MIN_NODE_HEADER; + if (jffs2_is_writebuffered(c)) { + int end, rem; + + /* + * We are about to read JFFS2_MIN_NODE_HEADER bytes, + * but this flash has some minimal I/O unit. It is + * possible that we'll need to read more soon, so read + * up to the next min. I/O unit, in order not to + * re-read the same min. I/O unit twice. + */ + end = ref_offset(ref) + len; + rem = end % c->wbuf_pagesize; + if (rem) + end += c->wbuf_pagesize - rem; + len = end - ref_offset(ref); + } + + dbg_readinode("read %d bytes at %#08x(%d).\n", len, ref_offset(ref), ref_flags(ref)); + + /* FIXME: point() */ + err = jffs2_flash_read(c, ref_offset(ref), len, &retlen, buf); + if (err) { + JFFS2_ERROR("can not read %d bytes from 0x%08x, error code: %d.\n", len, ref_offset(ref), err); + goto free_out; + } + + if (retlen < len) { + JFFS2_ERROR("short read at %#08x: %zu instead of %d.\n", ref_offset(ref), retlen, len); + err = -EIO; + goto free_out; + } + + node = (union jffs2_node_union *)buf; + + /* No need to mask in the valid bit; it shouldn't be invalid */ + if (je32_to_cpu(node->u.hdr_crc) != crc32(0, node, sizeof(node->u)-4)) { + JFFS2_NOTICE("Node header CRC failed at %#08x. {%04x,%04x,%08x,%08x}\n", + ref_offset(ref), je16_to_cpu(node->u.magic), + je16_to_cpu(node->u.nodetype), + je32_to_cpu(node->u.totlen), + je32_to_cpu(node->u.hdr_crc)); + jffs2_dbg_dump_node(c, ref_offset(ref)); + jffs2_mark_node_obsolete(c, ref); + goto cont; + } + if (je16_to_cpu(node->u.magic) != JFFS2_MAGIC_BITMASK) { + /* Not a JFFS2 node, whinge and move on */ + JFFS2_NOTICE("Wrong magic bitmask 0x%04x in node header at %#08x.\n", + je16_to_cpu(node->u.magic), ref_offset(ref)); + jffs2_mark_node_obsolete(c, ref); + goto cont; + } + + switch (je16_to_cpu(node->u.nodetype)) { + + case JFFS2_NODETYPE_DIRENT: + + if (JFFS2_MIN_NODE_HEADER < sizeof(struct jffs2_raw_dirent) && + len < sizeof(struct jffs2_raw_dirent)) { + err = read_more(c, ref, sizeof(struct jffs2_raw_dirent), &len, buf); + if (unlikely(err)) + goto free_out; + } + + err = read_direntry(c, ref, &node->d, retlen, rii); + if (unlikely(err)) + goto free_out; + + break; + + case JFFS2_NODETYPE_INODE: + + if (JFFS2_MIN_NODE_HEADER < sizeof(struct jffs2_raw_inode) && + len < sizeof(struct jffs2_raw_inode)) { + err = read_more(c, ref, sizeof(struct jffs2_raw_inode), &len, buf); + if (unlikely(err)) + goto free_out; + } + + err = read_dnode(c, ref, &node->i, len, rii); + if (unlikely(err)) + goto free_out; + + break; + + default: + if (JFFS2_MIN_NODE_HEADER < sizeof(struct jffs2_unknown_node) && + len < sizeof(struct jffs2_unknown_node)) { + err = read_more(c, ref, sizeof(struct jffs2_unknown_node), &len, buf); + if (unlikely(err)) + goto free_out; + } + + err = read_unknown(c, ref, &node->u); + if (unlikely(err)) + goto free_out; + + } + cont: + spin_lock(&c->erase_completion_lock); + } + + spin_unlock(&c->erase_completion_lock); + kfree(buf); + + f->highest_version = rii->highest_version; + + dbg_readinode("nodes of inode #%u were read, the highest version is %u, latest_mctime %u, mctime_ver %u.\n", + f->inocache->ino, rii->highest_version, rii->latest_mctime, + rii->mctime_ver); + return 0; + + free_out: + jffs2_free_tmp_dnode_info_list(&rii->tn_root); + jffs2_free_full_dirent_list(rii->fds); + rii->fds = NULL; + kfree(buf); + return err; +} + +static int jffs2_do_read_inode_internal(struct jffs2_sb_info *c, + struct jffs2_inode_info *f, + struct jffs2_raw_inode *latest_node) +{ + struct jffs2_readinode_info rii; + uint32_t crc, new_size; + size_t retlen; + int ret; + + dbg_readinode("ino #%u pino/nlink is %d\n", f->inocache->ino, + f->inocache->pino_nlink); + + memset(&rii, 0, sizeof(rii)); + + /* Grab all nodes relevant to this ino */ + ret = jffs2_get_inode_nodes(c, f, &rii); + + if (ret) { + JFFS2_ERROR("cannot read nodes for ino %u, returned error is %d\n", f->inocache->ino, ret); + if (f->inocache->state == INO_STATE_READING) + jffs2_set_inocache_state(c, f->inocache, INO_STATE_CHECKEDABSENT); + return ret; + } + + ret = jffs2_build_inode_fragtree(c, f, &rii); + if (ret) { + JFFS2_ERROR("Failed to build final fragtree for inode #%u: error %d\n", + f->inocache->ino, ret); + if (f->inocache->state == INO_STATE_READING) + jffs2_set_inocache_state(c, f->inocache, INO_STATE_CHECKEDABSENT); + jffs2_free_tmp_dnode_info_list(&rii.tn_root); + /* FIXME: We could at least crc-check them all */ + if (rii.mdata_tn) { + jffs2_free_full_dnode(rii.mdata_tn->fn); + jffs2_free_tmp_dnode_info(rii.mdata_tn); + rii.mdata_tn = NULL; + } + return ret; + } + + if (rii.mdata_tn) { + if (rii.mdata_tn->fn->raw == rii.latest_ref) { + f->metadata = rii.mdata_tn->fn; + jffs2_free_tmp_dnode_info(rii.mdata_tn); + } else { + jffs2_kill_tn(c, rii.mdata_tn); + } + rii.mdata_tn = NULL; + } + + f->dents = rii.fds; + + jffs2_dbg_fragtree_paranoia_check_nolock(f); + + if (unlikely(!rii.latest_ref)) { + /* No data nodes for this inode. */ + if (f->inocache->ino != 1) { + JFFS2_WARNING("no data nodes found for ino #%u\n", f->inocache->ino); + if (!rii.fds) { + if (f->inocache->state == INO_STATE_READING) + jffs2_set_inocache_state(c, f->inocache, INO_STATE_CHECKEDABSENT); + return -EIO; + } + JFFS2_NOTICE("but it has children so we fake some modes for it\n"); + } + latest_node->mode = cpu_to_jemode(S_IFDIR|S_IRUGO|S_IWUSR|S_IXUGO); + latest_node->version = cpu_to_je32(0); + latest_node->atime = latest_node->ctime = latest_node->mtime = cpu_to_je32(0); + latest_node->isize = cpu_to_je32(0); + latest_node->gid = cpu_to_je16(0); + latest_node->uid = cpu_to_je16(0); + if (f->inocache->state == INO_STATE_READING) + jffs2_set_inocache_state(c, f->inocache, INO_STATE_PRESENT); + return 0; + } + + ret = jffs2_flash_read(c, ref_offset(rii.latest_ref), sizeof(*latest_node), &retlen, (void *)latest_node); + if (ret || retlen != sizeof(*latest_node)) { + JFFS2_ERROR("failed to read from flash: error %d, %zd of %zd bytes read\n", + ret, retlen, sizeof(*latest_node)); + /* FIXME: If this fails, there seems to be a memory leak. Find it. */ + return ret ? ret : -EIO; + } + + crc = crc32(0, latest_node, sizeof(*latest_node)-8); + if (crc != je32_to_cpu(latest_node->node_crc)) { + JFFS2_ERROR("CRC failed for read_inode of inode %u at physical location 0x%x\n", + f->inocache->ino, ref_offset(rii.latest_ref)); + return -EIO; + } + + switch(jemode_to_cpu(latest_node->mode) & S_IFMT) { + case S_IFDIR: + if (rii.mctime_ver > je32_to_cpu(latest_node->version)) { + /* The times in the latest_node are actually older than + mctime in the latest dirent. Cheat. */ + latest_node->ctime = latest_node->mtime = cpu_to_je32(rii.latest_mctime); + } + break; + + + case S_IFREG: + /* If it was a regular file, truncate it to the latest node's isize */ + new_size = jffs2_truncate_fragtree(c, &f->fragtree, je32_to_cpu(latest_node->isize)); + if (new_size != je32_to_cpu(latest_node->isize)) { + JFFS2_WARNING("Truncating ino #%u to %d bytes failed because it only had %d bytes to start with!\n", + f->inocache->ino, je32_to_cpu(latest_node->isize), new_size); + latest_node->isize = cpu_to_je32(new_size); + } + break; + + case S_IFLNK: + /* Hack to work around broken isize in old symlink code. + Remove this when dwmw2 comes to his senses and stops + symlinks from being an entirely gratuitous special + case. */ + if (!je32_to_cpu(latest_node->isize)) + latest_node->isize = latest_node->dsize; + + if (f->inocache->state != INO_STATE_CHECKING) { + /* Symlink's inode data is the target path. Read it and + * keep in RAM to facilitate quick follow symlink + * operation. */ + uint32_t csize = je32_to_cpu(latest_node->csize); + if (csize > JFFS2_MAX_NAME_LEN) + return -ENAMETOOLONG; + f->target = kmalloc(csize + 1, GFP_KERNEL); + if (!f->target) { + JFFS2_ERROR("can't allocate %u bytes of memory for the symlink target path cache\n", csize); + return -ENOMEM; + } + + ret = jffs2_flash_read(c, ref_offset(rii.latest_ref) + sizeof(*latest_node), + csize, &retlen, (char *)f->target); + + if (ret || retlen != csize) { + if (retlen != csize) + ret = -EIO; + kfree(f->target); + f->target = NULL; + return ret; + } + + f->target[csize] = '\0'; + dbg_readinode("symlink's target '%s' cached\n", f->target); + } + + fallthrough; + + case S_IFBLK: + case S_IFCHR: + /* Certain inode types should have only one data node, and it's + kept as the metadata node */ + if (f->metadata) { + JFFS2_ERROR("Argh. Special inode #%u with mode 0%o had metadata node\n", + f->inocache->ino, jemode_to_cpu(latest_node->mode)); + return -EIO; + } + if (!frag_first(&f->fragtree)) { + JFFS2_ERROR("Argh. Special inode #%u with mode 0%o has no fragments\n", + f->inocache->ino, jemode_to_cpu(latest_node->mode)); + return -EIO; + } + /* ASSERT: f->fraglist != NULL */ + if (frag_next(frag_first(&f->fragtree))) { + JFFS2_ERROR("Argh. Special inode #%u with mode 0x%x had more than one node\n", + f->inocache->ino, jemode_to_cpu(latest_node->mode)); + /* FIXME: Deal with it - check crc32, check for duplicate node, check times and discard the older one */ + return -EIO; + } + /* OK. We're happy */ + f->metadata = frag_first(&f->fragtree)->node; + jffs2_free_node_frag(frag_first(&f->fragtree)); + f->fragtree = RB_ROOT; + break; + } + if (f->inocache->state == INO_STATE_READING) + jffs2_set_inocache_state(c, f->inocache, INO_STATE_PRESENT); + + return 0; +} + +/* Scan the list of all nodes present for this ino, build map of versions, etc. */ +int jffs2_do_read_inode(struct jffs2_sb_info *c, struct jffs2_inode_info *f, + uint32_t ino, struct jffs2_raw_inode *latest_node) +{ + dbg_readinode("read inode #%u\n", ino); + + retry_inocache: + spin_lock(&c->inocache_lock); + f->inocache = jffs2_get_ino_cache(c, ino); + + if (f->inocache) { + /* Check its state. We may need to wait before we can use it */ + switch(f->inocache->state) { + case INO_STATE_UNCHECKED: + case INO_STATE_CHECKEDABSENT: + f->inocache->state = INO_STATE_READING; + break; + + case INO_STATE_CHECKING: + case INO_STATE_GC: + /* If it's in either of these states, we need + to wait for whoever's got it to finish and + put it back. */ + dbg_readinode("waiting for ino #%u in state %d\n", ino, f->inocache->state); + sleep_on_spinunlock(&c->inocache_wq, &c->inocache_lock); + goto retry_inocache; + + case INO_STATE_READING: + case INO_STATE_PRESENT: + /* Eep. This should never happen. It can + happen if Linux calls read_inode() again + before clear_inode() has finished though. */ + JFFS2_ERROR("Eep. Trying to read_inode #%u when it's already in state %d!\n", ino, f->inocache->state); + /* Fail. That's probably better than allowing it to succeed */ + f->inocache = NULL; + break; + + default: + BUG(); + } + } + spin_unlock(&c->inocache_lock); + + if (!f->inocache && ino == 1) { + /* Special case - no root inode on medium */ + f->inocache = jffs2_alloc_inode_cache(); + if (!f->inocache) { + JFFS2_ERROR("cannot allocate inocache for root inode\n"); + return -ENOMEM; + } + dbg_readinode("creating inocache for root inode\n"); + memset(f->inocache, 0, sizeof(struct jffs2_inode_cache)); + f->inocache->ino = f->inocache->pino_nlink = 1; + f->inocache->nodes = (struct jffs2_raw_node_ref *)f->inocache; + f->inocache->state = INO_STATE_READING; + jffs2_add_ino_cache(c, f->inocache); + } + if (!f->inocache) { + JFFS2_ERROR("requested to read a nonexistent ino %u\n", ino); + return -ENOENT; + } + + return jffs2_do_read_inode_internal(c, f, latest_node); +} + +int jffs2_do_crccheck_inode(struct jffs2_sb_info *c, struct jffs2_inode_cache *ic) +{ + struct jffs2_raw_inode n; + struct jffs2_inode_info *f = kzalloc_obj(*f); + int ret; + + if (!f) + return -ENOMEM; + + mutex_init(&f->sem); + mutex_lock(&f->sem); + f->inocache = ic; + + ret = jffs2_do_read_inode_internal(c, f, &n); + mutex_unlock(&f->sem); + jffs2_do_clear_inode(c, f); + jffs2_xattr_do_crccheck_inode(c, ic); + kfree (f); + return ret; +} + +void jffs2_do_clear_inode(struct jffs2_sb_info *c, struct jffs2_inode_info *f) +{ + struct jffs2_full_dirent *fd, *fds; + int deleted; + + jffs2_xattr_delete_inode(c, f->inocache); + mutex_lock(&f->sem); + deleted = f->inocache && !f->inocache->pino_nlink; + + if (f->inocache && f->inocache->state != INO_STATE_CHECKING) + jffs2_set_inocache_state(c, f->inocache, INO_STATE_CLEARING); + + if (f->metadata) { + if (deleted) + jffs2_mark_node_obsolete(c, f->metadata->raw); + jffs2_free_full_dnode(f->metadata); + } + + jffs2_kill_fragtree(&f->fragtree, deleted?c:NULL); + + fds = f->dents; + while(fds) { + fd = fds; + fds = fd->next; + jffs2_free_full_dirent(fd); + } + + if (f->inocache && f->inocache->state != INO_STATE_CHECKING) { + jffs2_set_inocache_state(c, f->inocache, INO_STATE_CHECKEDABSENT); + if (f->inocache->nodes == (void *)f->inocache) + jffs2_del_ino_cache(c, f->inocache); + } + + mutex_unlock(&f->sem); +} diff --git a/fs/jffs2/scan.c b/fs/jffs2/scan.c new file mode 100644 index 000000000..06e494797 --- /dev/null +++ b/fs/jffs2/scan.c @@ -0,0 +1,1184 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * + * Created by David Woodhouse + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include "nodelist.h" +#include "summary.h" +#include "debug.h" + +#define DEFAULT_EMPTY_SCAN_SIZE 256 + +#define noisy_printk(noise, fmt, ...) \ +do { \ + if (*(noise)) { \ + pr_notice(fmt, ##__VA_ARGS__); \ + (*(noise))--; \ + if (!(*(noise))) \ + pr_notice("Further such events for this erase block will not be printed\n"); \ + } \ +} while (0) + +static uint32_t pseudo_random; + +static int jffs2_scan_eraseblock (struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, + unsigned char *buf, uint32_t buf_size, struct jffs2_summary *s); + +/* These helper functions _must_ increase ofs and also do the dirty/used space accounting. + * Returning an error will abort the mount - bad checksums etc. should just mark the space + * as dirty. + */ +static int jffs2_scan_inode_node(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, + struct jffs2_raw_inode *ri, uint32_t ofs, struct jffs2_summary *s); +static int jffs2_scan_dirent_node(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, + struct jffs2_raw_dirent *rd, uint32_t ofs, struct jffs2_summary *s); + +static inline int min_free(struct jffs2_sb_info *c) +{ + uint32_t min = 2 * sizeof(struct jffs2_raw_inode); +#ifdef CONFIG_JFFS2_FS_WRITEBUFFER + if (!jffs2_can_mark_obsolete(c) && min < c->wbuf_pagesize) + return c->wbuf_pagesize; +#endif + return min; + +} + +static inline uint32_t EMPTY_SCAN_SIZE(uint32_t sector_size) { + if (sector_size < DEFAULT_EMPTY_SCAN_SIZE) + return sector_size; + else + return DEFAULT_EMPTY_SCAN_SIZE; +} + +static int file_dirty(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb) +{ + int ret; + + if ((ret = jffs2_prealloc_raw_node_refs(c, jeb, 1))) + return ret; + if ((ret = jffs2_scan_dirty_space(c, jeb, jeb->free_size))) + return ret; + /* Turned wasted size into dirty, since we apparently + think it's recoverable now. */ + jeb->dirty_size += jeb->wasted_size; + c->dirty_size += jeb->wasted_size; + c->wasted_size -= jeb->wasted_size; + jeb->wasted_size = 0; + if (VERYDIRTY(c, jeb->dirty_size)) { + list_add(&jeb->list, &c->very_dirty_list); + } else { + list_add(&jeb->list, &c->dirty_list); + } + return 0; +} + +int jffs2_scan_medium(struct jffs2_sb_info *c) +{ + int i, ret; + uint32_t empty_blocks = 0, bad_blocks = 0; + unsigned char *flashbuf = NULL; + uint32_t buf_size = 0; + struct jffs2_summary *s = NULL; /* summary info collected by the scan process */ +#ifndef __ECOS + size_t pointlen, try_size; + + ret = mtd_point(c->mtd, 0, c->mtd->size, &pointlen, + (void **)&flashbuf, NULL); + if (!ret && pointlen < c->mtd->size) { + /* Don't muck about if it won't let us point to the whole flash */ + jffs2_dbg(1, "MTD point returned len too short: 0x%zx\n", + pointlen); + mtd_unpoint(c->mtd, 0, pointlen); + flashbuf = NULL; + } + if (ret && ret != -EOPNOTSUPP) + jffs2_dbg(1, "MTD point failed %d\n", ret); +#endif + if (!flashbuf) { + /* For NAND it's quicker to read a whole eraseblock at a time, + apparently */ + if (jffs2_cleanmarker_oob(c)) + try_size = c->sector_size; + else + try_size = PAGE_SIZE; + + jffs2_dbg(1, "Trying to allocate readbuf of %zu " + "bytes\n", try_size); + + flashbuf = mtd_kmalloc_up_to(c->mtd, &try_size); + if (!flashbuf) + return -ENOMEM; + + jffs2_dbg(1, "Allocated readbuf of %zu bytes\n", + try_size); + + buf_size = (uint32_t)try_size; + } + + if (jffs2_sum_active()) { + s = kzalloc_obj(struct jffs2_summary); + if (!s) { + JFFS2_WARNING("Can't allocate memory for summary\n"); + ret = -ENOMEM; + goto out_buf; + } + } + + for (i=0; inr_blocks; i++) { + struct jffs2_eraseblock *jeb = &c->blocks[i]; + + cond_resched(); + + /* reset summary info for next eraseblock scan */ + jffs2_sum_reset_collected(s); + + ret = jffs2_scan_eraseblock(c, jeb, buf_size?flashbuf:(flashbuf+jeb->offset), + buf_size, s); + + if (ret < 0) + goto out; + + jffs2_dbg_acct_paranoia_check_nolock(c, jeb); + + /* Now decide which list to put it on */ + switch(ret) { + case BLK_STATE_ALLFF: + /* + * Empty block. Since we can't be sure it + * was entirely erased, we just queue it for erase + * again. It will be marked as such when the erase + * is complete. Meanwhile we still count it as empty + * for later checks. + */ + empty_blocks++; + list_add(&jeb->list, &c->erase_pending_list); + c->nr_erasing_blocks++; + break; + + case BLK_STATE_CLEANMARKER: + /* Only a CLEANMARKER node is valid */ + if (!jeb->dirty_size) { + /* It's actually free */ + list_add(&jeb->list, &c->free_list); + c->nr_free_blocks++; + } else { + /* Dirt */ + jffs2_dbg(1, "Adding all-dirty block at 0x%08x to erase_pending_list\n", + jeb->offset); + list_add(&jeb->list, &c->erase_pending_list); + c->nr_erasing_blocks++; + } + break; + + case BLK_STATE_CLEAN: + /* Full (or almost full) of clean data. Clean list */ + list_add(&jeb->list, &c->clean_list); + break; + + case BLK_STATE_PARTDIRTY: + /* Some data, but not full. Dirty list. */ + /* We want to remember the block with most free space + and stick it in the 'nextblock' position to start writing to it. */ + if (jeb->free_size > min_free(c) && + (!c->nextblock || c->nextblock->free_size < jeb->free_size)) { + /* Better candidate for the next writes to go to */ + if (c->nextblock) { + ret = file_dirty(c, c->nextblock); + if (ret) + goto out; + /* deleting summary information of the old nextblock */ + jffs2_sum_reset_collected(c->summary); + } + /* update collected summary information for the current nextblock */ + jffs2_sum_move_collected(c, s); + jffs2_dbg(1, "%s(): new nextblock = 0x%08x\n", + __func__, jeb->offset); + c->nextblock = jeb; + } else { + ret = file_dirty(c, jeb); + if (ret) + goto out; + } + break; + + case BLK_STATE_ALLDIRTY: + /* Nothing valid - not even a clean marker. Needs erasing. */ + /* For now we just put it on the erasing list. We'll start the erases later */ + jffs2_dbg(1, "Erase block at 0x%08x is not formatted. It will be erased\n", + jeb->offset); + list_add(&jeb->list, &c->erase_pending_list); + c->nr_erasing_blocks++; + break; + + case BLK_STATE_BADBLOCK: + jffs2_dbg(1, "Block at 0x%08x is bad\n", jeb->offset); + list_add(&jeb->list, &c->bad_list); + c->bad_size += c->sector_size; + c->free_size -= c->sector_size; + bad_blocks++; + break; + default: + pr_warn("%s(): unknown block state\n", __func__); + BUG(); + } + } + + /* Nextblock dirty is always seen as wasted, because we cannot recycle it now */ + if (c->nextblock && (c->nextblock->dirty_size)) { + c->nextblock->wasted_size += c->nextblock->dirty_size; + c->wasted_size += c->nextblock->dirty_size; + c->dirty_size -= c->nextblock->dirty_size; + c->nextblock->dirty_size = 0; + } +#ifdef CONFIG_JFFS2_FS_WRITEBUFFER + if (!jffs2_can_mark_obsolete(c) && c->wbuf_pagesize && c->nextblock && (c->nextblock->free_size % c->wbuf_pagesize)) { + /* If we're going to start writing into a block which already + contains data, and the end of the data isn't page-aligned, + skip a little and align it. */ + + uint32_t skip = c->nextblock->free_size % c->wbuf_pagesize; + + jffs2_dbg(1, "%s(): Skipping %d bytes in nextblock to ensure page alignment\n", + __func__, skip); + ret = jffs2_prealloc_raw_node_refs(c, c->nextblock, 1); + if (ret) + goto out; + jffs2_scan_dirty_space(c, c->nextblock, skip); + } +#endif + if (c->nr_erasing_blocks) { + if (!c->used_size && !c->unchecked_size && + ((c->nr_free_blocks+empty_blocks+bad_blocks) != c->nr_blocks || bad_blocks == c->nr_blocks)) { + pr_notice("Cowardly refusing to erase blocks on filesystem with no valid JFFS2 nodes\n"); + pr_notice("empty_blocks %d, bad_blocks %d, c->nr_blocks %d\n", + empty_blocks, bad_blocks, c->nr_blocks); + ret = -EIO; + goto out; + } + spin_lock(&c->erase_completion_lock); + jffs2_garbage_collect_trigger(c); + spin_unlock(&c->erase_completion_lock); + } + ret = 0; + out: + jffs2_sum_reset_collected(s); + kfree(s); + out_buf: + if (buf_size) + kfree(flashbuf); +#ifndef __ECOS + else + mtd_unpoint(c->mtd, 0, c->mtd->size); +#endif + return ret; +} + +static int jffs2_fill_scan_buf(struct jffs2_sb_info *c, void *buf, + uint32_t ofs, uint32_t len) +{ + int ret; + size_t retlen; + + ret = jffs2_flash_read(c, ofs, len, &retlen, buf); + if (ret) { + jffs2_dbg(1, "mtd->read(0x%x bytes from 0x%x) returned %d\n", + len, ofs, ret); + return ret; + } + if (retlen < len) { + jffs2_dbg(1, "Read at 0x%x gave only 0x%zx bytes\n", + ofs, retlen); + return -EIO; + } + return 0; +} + +int jffs2_scan_classify_jeb(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb) +{ + if ((jeb->used_size + jeb->unchecked_size) == PAD(c->cleanmarker_size) && !jeb->dirty_size + && (!jeb->first_node || !ref_next(jeb->first_node)) ) + return BLK_STATE_CLEANMARKER; + + /* move blocks with max 4 byte dirty space to cleanlist */ + else if (!ISDIRTY(c->sector_size - (jeb->used_size + jeb->unchecked_size))) { + c->dirty_size -= jeb->dirty_size; + c->wasted_size += jeb->dirty_size; + jeb->wasted_size += jeb->dirty_size; + jeb->dirty_size = 0; + return BLK_STATE_CLEAN; + } else if (jeb->used_size || jeb->unchecked_size) + return BLK_STATE_PARTDIRTY; + else + return BLK_STATE_ALLDIRTY; +} + +#ifdef CONFIG_JFFS2_FS_XATTR +static int jffs2_scan_xattr_node(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, + struct jffs2_raw_xattr *rx, uint32_t ofs, + struct jffs2_summary *s) +{ + struct jffs2_xattr_datum *xd; + uint32_t xid, version, totlen, crc; + int err; + + crc = crc32(0, rx, sizeof(struct jffs2_raw_xattr) - 4); + if (crc != je32_to_cpu(rx->node_crc)) { + JFFS2_WARNING("node CRC failed at %#08x, read=%#08x, calc=%#08x\n", + ofs, je32_to_cpu(rx->node_crc), crc); + if ((err = jffs2_scan_dirty_space(c, jeb, je32_to_cpu(rx->totlen)))) + return err; + return 0; + } + + xid = je32_to_cpu(rx->xid); + version = je32_to_cpu(rx->version); + + totlen = PAD(sizeof(struct jffs2_raw_xattr) + + rx->name_len + 1 + je16_to_cpu(rx->value_len)); + if (totlen != je32_to_cpu(rx->totlen)) { + JFFS2_WARNING("node length mismatch at %#08x, read=%u, calc=%u\n", + ofs, je32_to_cpu(rx->totlen), totlen); + if ((err = jffs2_scan_dirty_space(c, jeb, je32_to_cpu(rx->totlen)))) + return err; + return 0; + } + + xd = jffs2_setup_xattr_datum(c, xid, version); + if (IS_ERR(xd)) + return PTR_ERR(xd); + + if (xd->version > version) { + struct jffs2_raw_node_ref *raw + = jffs2_link_node_ref(c, jeb, ofs | REF_PRISTINE, totlen, NULL); + raw->next_in_ino = xd->node->next_in_ino; + xd->node->next_in_ino = raw; + } else { + xd->version = version; + xd->xprefix = rx->xprefix; + xd->name_len = rx->name_len; + xd->value_len = je16_to_cpu(rx->value_len); + xd->data_crc = je32_to_cpu(rx->data_crc); + + jffs2_link_node_ref(c, jeb, ofs | REF_PRISTINE, totlen, (void *)xd); + } + + if (jffs2_sum_active()) + jffs2_sum_add_xattr_mem(s, rx, ofs - jeb->offset); + dbg_xattr("scanning xdatum at %#08x (xid=%u, version=%u)\n", + ofs, xd->xid, xd->version); + return 0; +} + +static int jffs2_scan_xref_node(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, + struct jffs2_raw_xref *rr, uint32_t ofs, + struct jffs2_summary *s) +{ + struct jffs2_xattr_ref *ref; + uint32_t crc; + int err; + + crc = crc32(0, rr, sizeof(*rr) - 4); + if (crc != je32_to_cpu(rr->node_crc)) { + JFFS2_WARNING("node CRC failed at %#08x, read=%#08x, calc=%#08x\n", + ofs, je32_to_cpu(rr->node_crc), crc); + if ((err = jffs2_scan_dirty_space(c, jeb, PAD(je32_to_cpu(rr->totlen))))) + return err; + return 0; + } + + if (PAD(sizeof(struct jffs2_raw_xref)) != je32_to_cpu(rr->totlen)) { + JFFS2_WARNING("node length mismatch at %#08x, read=%u, calc=%zd\n", + ofs, je32_to_cpu(rr->totlen), + PAD(sizeof(struct jffs2_raw_xref))); + if ((err = jffs2_scan_dirty_space(c, jeb, je32_to_cpu(rr->totlen)))) + return err; + return 0; + } + + ref = jffs2_alloc_xattr_ref(); + if (!ref) + return -ENOMEM; + + /* BEFORE jffs2_build_xattr_subsystem() called, + * and AFTER xattr_ref is marked as a dead xref, + * ref->xid is used to store 32bit xid, xd is not used + * ref->ino is used to store 32bit inode-number, ic is not used + * Thoes variables are declared as union, thus using those + * are exclusive. In a similar way, ref->next is temporarily + * used to chain all xattr_ref object. It's re-chained to + * jffs2_inode_cache in jffs2_build_xattr_subsystem() correctly. + */ + ref->ino = je32_to_cpu(rr->ino); + ref->xid = je32_to_cpu(rr->xid); + ref->xseqno = je32_to_cpu(rr->xseqno); + if (ref->xseqno > c->highest_xseqno) + c->highest_xseqno = (ref->xseqno & ~XREF_DELETE_MARKER); + ref->next = c->xref_temp; + c->xref_temp = ref; + + jffs2_link_node_ref(c, jeb, ofs | REF_PRISTINE, PAD(je32_to_cpu(rr->totlen)), (void *)ref); + + if (jffs2_sum_active()) + jffs2_sum_add_xref_mem(s, rr, ofs - jeb->offset); + dbg_xattr("scan xref at %#08x (xid=%u, ino=%u)\n", + ofs, ref->xid, ref->ino); + return 0; +} +#endif + +/* Called with 'buf_size == 0' if buf is in fact a pointer _directly_ into + the flash, XIP-style */ +static int jffs2_scan_eraseblock (struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, + unsigned char *buf, uint32_t buf_size, struct jffs2_summary *s) { + struct jffs2_unknown_node *node; + struct jffs2_unknown_node crcnode; + uint32_t ofs, prevofs, max_ofs; + uint32_t hdr_crc, buf_ofs, buf_len; + int err; + int noise = 0; + + +#ifdef CONFIG_JFFS2_FS_WRITEBUFFER + int cleanmarkerfound = 0; +#endif + + ofs = jeb->offset; + prevofs = jeb->offset - 1; + + jffs2_dbg(1, "%s(): Scanning block at 0x%x\n", __func__, ofs); + +#ifdef CONFIG_JFFS2_FS_WRITEBUFFER + if (jffs2_cleanmarker_oob(c)) { + int ret; + + if (mtd_block_isbad(c->mtd, jeb->offset)) + return BLK_STATE_BADBLOCK; + + ret = jffs2_check_nand_cleanmarker(c, jeb); + jffs2_dbg(2, "jffs_check_nand_cleanmarker returned %d\n", ret); + + /* Even if it's not found, we still scan to see + if the block is empty. We use this information + to decide whether to erase it or not. */ + switch (ret) { + case 0: cleanmarkerfound = 1; break; + case 1: break; + default: return ret; + } + } +#endif + + if (jffs2_sum_active()) { + struct jffs2_sum_marker *sm; + void *sumptr = NULL; + uint32_t sumlen; + + if (!buf_size) { + /* XIP case. Just look, point at the summary if it's there */ + sm = (void *)buf + c->sector_size - sizeof(*sm); + if (je32_to_cpu(sm->magic) == JFFS2_SUM_MAGIC) { + sumptr = buf + je32_to_cpu(sm->offset); + sumlen = c->sector_size - je32_to_cpu(sm->offset); + } + } else { + /* If NAND flash, read a whole page of it. Else just the end */ + if (c->wbuf_pagesize) + buf_len = c->wbuf_pagesize; + else + buf_len = sizeof(*sm); + + /* Read as much as we want into the _end_ of the preallocated buffer */ + err = jffs2_fill_scan_buf(c, buf + buf_size - buf_len, + jeb->offset + c->sector_size - buf_len, + buf_len); + if (err) + return err; + + sm = (void *)buf + buf_size - sizeof(*sm); + if (je32_to_cpu(sm->magic) == JFFS2_SUM_MAGIC) { + sumlen = c->sector_size - je32_to_cpu(sm->offset); + sumptr = buf + buf_size - sumlen; + + /* sm->offset maybe wrong but MAGIC maybe right */ + if (sumlen > c->sector_size) + goto full_scan; + + /* Now, make sure the summary itself is available */ + if (sumlen > buf_size) { + /* Need to kmalloc for this. */ + sumptr = kmalloc(sumlen, GFP_KERNEL); + if (!sumptr) + return -ENOMEM; + memcpy(sumptr + sumlen - buf_len, buf + buf_size - buf_len, buf_len); + } + if (buf_len < sumlen) { + /* Need to read more so that the entire summary node is present */ + err = jffs2_fill_scan_buf(c, sumptr, + jeb->offset + c->sector_size - sumlen, + sumlen - buf_len); + if (err) { + if (sumlen > buf_size) + kfree(sumptr); + return err; + } + } + } + + } + + if (sumptr) { + err = jffs2_sum_scan_sumnode(c, jeb, sumptr, sumlen, &pseudo_random); + + if (buf_size && sumlen > buf_size) + kfree(sumptr); + /* If it returns with a real error, bail. + If it returns positive, that's a block classification + (i.e. BLK_STATE_xxx) so return that too. + If it returns zero, fall through to full scan. */ + if (err) + return err; + } + } + +full_scan: + buf_ofs = jeb->offset; + + if (!buf_size) { + /* This is the XIP case -- we're reading _directly_ from the flash chip */ + buf_len = c->sector_size; + } else { + buf_len = EMPTY_SCAN_SIZE(c->sector_size); + err = jffs2_fill_scan_buf(c, buf, buf_ofs, buf_len); + if (err) + return err; + } + + /* We temporarily use 'ofs' as a pointer into the buffer/jeb */ + ofs = 0; + max_ofs = EMPTY_SCAN_SIZE(c->sector_size); + /* Scan only EMPTY_SCAN_SIZE of 0xFF before declaring it's empty */ + while(ofs < max_ofs && *(uint32_t *)(&buf[ofs]) == 0xFFFFFFFF) + ofs += 4; + + if (ofs == max_ofs) { +#ifdef CONFIG_JFFS2_FS_WRITEBUFFER + if (jffs2_cleanmarker_oob(c)) { + /* scan oob, take care of cleanmarker */ + int ret = jffs2_check_oob_empty(c, jeb, cleanmarkerfound); + jffs2_dbg(2, "jffs2_check_oob_empty returned %d\n", + ret); + switch (ret) { + case 0: return cleanmarkerfound ? BLK_STATE_CLEANMARKER : BLK_STATE_ALLFF; + case 1: return BLK_STATE_ALLDIRTY; + default: return ret; + } + } +#endif + jffs2_dbg(1, "Block at 0x%08x is empty (erased)\n", + jeb->offset); + if (c->cleanmarker_size == 0) + return BLK_STATE_CLEANMARKER; /* don't bother with re-erase */ + else + return BLK_STATE_ALLFF; /* OK to erase if all blocks are like this */ + } + if (ofs) { + jffs2_dbg(1, "Free space at %08x ends at %08x\n", jeb->offset, + jeb->offset + ofs); + if ((err = jffs2_prealloc_raw_node_refs(c, jeb, 1))) + return err; + if ((err = jffs2_scan_dirty_space(c, jeb, ofs))) + return err; + } + + /* Now ofs is a complete physical flash offset as it always was... */ + ofs += jeb->offset; + + noise = 10; + + dbg_summary("no summary found in jeb 0x%08x. Apply original scan.\n",jeb->offset); + +scan_more: + while(ofs < jeb->offset + c->sector_size) { + + jffs2_dbg_acct_paranoia_check_nolock(c, jeb); + + /* Make sure there are node refs available for use */ + err = jffs2_prealloc_raw_node_refs(c, jeb, 2); + if (err) + return err; + + cond_resched(); + + if (ofs & 3) { + pr_warn("Eep. ofs 0x%08x not word-aligned!\n", ofs); + ofs = PAD(ofs); + continue; + } + if (ofs == prevofs) { + pr_warn("ofs 0x%08x has already been seen. Skipping\n", + ofs); + if ((err = jffs2_scan_dirty_space(c, jeb, 4))) + return err; + ofs += 4; + continue; + } + prevofs = ofs; + + if (jeb->offset + c->sector_size < ofs + sizeof(*node)) { + jffs2_dbg(1, "Fewer than %zd bytes left to end of block. (%x+%x<%x+%zx) Not reading\n", + sizeof(struct jffs2_unknown_node), + jeb->offset, c->sector_size, ofs, + sizeof(*node)); + if ((err = jffs2_scan_dirty_space(c, jeb, (jeb->offset + c->sector_size)-ofs))) + return err; + break; + } + + if (buf_ofs + buf_len < ofs + sizeof(*node)) { + buf_len = min_t(uint32_t, buf_size, jeb->offset + c->sector_size - ofs); + jffs2_dbg(1, "Fewer than %zd bytes (node header) left to end of buf. Reading 0x%x at 0x%08x\n", + sizeof(struct jffs2_unknown_node), + buf_len, ofs); + err = jffs2_fill_scan_buf(c, buf, ofs, buf_len); + if (err) + return err; + buf_ofs = ofs; + } + + node = (struct jffs2_unknown_node *)&buf[ofs-buf_ofs]; + + if (*(uint32_t *)(&buf[ofs-buf_ofs]) == 0xffffffff) { + uint32_t inbuf_ofs; + uint32_t empty_start, scan_end; + + empty_start = ofs; + ofs += 4; + scan_end = min_t(uint32_t, EMPTY_SCAN_SIZE(c->sector_size)/8, buf_len); + + jffs2_dbg(1, "Found empty flash at 0x%08x\n", ofs); + more_empty: + inbuf_ofs = ofs - buf_ofs; + while (inbuf_ofs < scan_end) { + if (unlikely(*(uint32_t *)(&buf[inbuf_ofs]) != 0xffffffff)) { + pr_warn("Empty flash at 0x%08x ends at 0x%08x\n", + empty_start, ofs); + if ((err = jffs2_scan_dirty_space(c, jeb, ofs-empty_start))) + return err; + goto scan_more; + } + + inbuf_ofs+=4; + ofs += 4; + } + /* Ran off end. */ + jffs2_dbg(1, "Empty flash to end of buffer at 0x%08x\n", + ofs); + + /* If we're only checking the beginning of a block with a cleanmarker, + bail now */ + if (buf_ofs == jeb->offset && jeb->used_size == PAD(c->cleanmarker_size) && + c->cleanmarker_size && !jeb->dirty_size && !ref_next(jeb->first_node)) { + jffs2_dbg(1, "%d bytes at start of block seems clean... assuming all clean\n", + EMPTY_SCAN_SIZE(c->sector_size)); + return BLK_STATE_CLEANMARKER; + } + if (!buf_size && (scan_end != buf_len)) {/* XIP/point case */ + scan_end = buf_len; + goto more_empty; + } + + /* See how much more there is to read in this eraseblock... */ + buf_len = min_t(uint32_t, buf_size, jeb->offset + c->sector_size - ofs); + if (!buf_len) { + /* No more to read. Break out of main loop without marking + this range of empty space as dirty (because it's not) */ + jffs2_dbg(1, "Empty flash at %08x runs to end of block. Treating as free_space\n", + empty_start); + break; + } + /* point never reaches here */ + scan_end = buf_len; + jffs2_dbg(1, "Reading another 0x%x at 0x%08x\n", + buf_len, ofs); + err = jffs2_fill_scan_buf(c, buf, ofs, buf_len); + if (err) + return err; + buf_ofs = ofs; + goto more_empty; + } + + if (ofs == jeb->offset && je16_to_cpu(node->magic) == KSAMTIB_CIGAM_2SFFJ) { + pr_warn("Magic bitmask is backwards at offset 0x%08x. Wrong endian filesystem?\n", + ofs); + if ((err = jffs2_scan_dirty_space(c, jeb, 4))) + return err; + ofs += 4; + continue; + } + if (je16_to_cpu(node->magic) == JFFS2_DIRTY_BITMASK) { + jffs2_dbg(1, "Dirty bitmask at 0x%08x\n", ofs); + if ((err = jffs2_scan_dirty_space(c, jeb, 4))) + return err; + ofs += 4; + continue; + } + if (je16_to_cpu(node->magic) == JFFS2_OLD_MAGIC_BITMASK) { + pr_warn("Old JFFS2 bitmask found at 0x%08x\n", ofs); + pr_warn("You cannot use older JFFS2 filesystems with newer kernels\n"); + if ((err = jffs2_scan_dirty_space(c, jeb, 4))) + return err; + ofs += 4; + continue; + } + if (je16_to_cpu(node->magic) != JFFS2_MAGIC_BITMASK) { + /* OK. We're out of possibilities. Whinge and move on */ + noisy_printk(&noise, "%s(): Magic bitmask 0x%04x not found at 0x%08x: 0x%04x instead\n", + __func__, + JFFS2_MAGIC_BITMASK, ofs, + je16_to_cpu(node->magic)); + if ((err = jffs2_scan_dirty_space(c, jeb, 4))) + return err; + ofs += 4; + continue; + } + /* We seem to have a node of sorts. Check the CRC */ + crcnode.magic = node->magic; + crcnode.nodetype = cpu_to_je16( je16_to_cpu(node->nodetype) | JFFS2_NODE_ACCURATE); + crcnode.totlen = node->totlen; + hdr_crc = crc32(0, &crcnode, sizeof(crcnode)-4); + + if (hdr_crc != je32_to_cpu(node->hdr_crc)) { + noisy_printk(&noise, "%s(): Node at 0x%08x {0x%04x, 0x%04x, 0x%08x) has invalid CRC 0x%08x (calculated 0x%08x)\n", + __func__, + ofs, je16_to_cpu(node->magic), + je16_to_cpu(node->nodetype), + je32_to_cpu(node->totlen), + je32_to_cpu(node->hdr_crc), + hdr_crc); + if ((err = jffs2_scan_dirty_space(c, jeb, 4))) + return err; + ofs += 4; + continue; + } + + if (ofs + je32_to_cpu(node->totlen) > jeb->offset + c->sector_size) { + /* Eep. Node goes over the end of the erase block. */ + pr_warn("Node at 0x%08x with length 0x%08x would run over the end of the erase block\n", + ofs, je32_to_cpu(node->totlen)); + pr_warn("Perhaps the file system was created with the wrong erase size?\n"); + if ((err = jffs2_scan_dirty_space(c, jeb, 4))) + return err; + ofs += 4; + continue; + } + + if (!(je16_to_cpu(node->nodetype) & JFFS2_NODE_ACCURATE)) { + /* Wheee. This is an obsoleted node */ + jffs2_dbg(2, "Node at 0x%08x is obsolete. Skipping\n", + ofs); + if ((err = jffs2_scan_dirty_space(c, jeb, PAD(je32_to_cpu(node->totlen))))) + return err; + ofs += PAD(je32_to_cpu(node->totlen)); + continue; + } + + switch(je16_to_cpu(node->nodetype)) { + case JFFS2_NODETYPE_INODE: + if (buf_ofs + buf_len < ofs + sizeof(struct jffs2_raw_inode)) { + buf_len = min_t(uint32_t, buf_size, jeb->offset + c->sector_size - ofs); + jffs2_dbg(1, "Fewer than %zd bytes (inode node) left to end of buf. Reading 0x%x at 0x%08x\n", + sizeof(struct jffs2_raw_inode), + buf_len, ofs); + err = jffs2_fill_scan_buf(c, buf, ofs, buf_len); + if (err) + return err; + buf_ofs = ofs; + node = (void *)buf; + } + err = jffs2_scan_inode_node(c, jeb, (void *)node, ofs, s); + if (err) return err; + ofs += PAD(je32_to_cpu(node->totlen)); + break; + + case JFFS2_NODETYPE_DIRENT: + if (buf_ofs + buf_len < ofs + je32_to_cpu(node->totlen)) { + buf_len = min_t(uint32_t, buf_size, jeb->offset + c->sector_size - ofs); + jffs2_dbg(1, "Fewer than %d bytes (dirent node) left to end of buf. Reading 0x%x at 0x%08x\n", + je32_to_cpu(node->totlen), buf_len, + ofs); + err = jffs2_fill_scan_buf(c, buf, ofs, buf_len); + if (err) + return err; + buf_ofs = ofs; + node = (void *)buf; + } + err = jffs2_scan_dirent_node(c, jeb, (void *)node, ofs, s); + if (err) return err; + ofs += PAD(je32_to_cpu(node->totlen)); + break; + +#ifdef CONFIG_JFFS2_FS_XATTR + case JFFS2_NODETYPE_XATTR: + if (buf_ofs + buf_len < ofs + je32_to_cpu(node->totlen)) { + buf_len = min_t(uint32_t, buf_size, jeb->offset + c->sector_size - ofs); + jffs2_dbg(1, "Fewer than %d bytes (xattr node) left to end of buf. Reading 0x%x at 0x%08x\n", + je32_to_cpu(node->totlen), buf_len, + ofs); + err = jffs2_fill_scan_buf(c, buf, ofs, buf_len); + if (err) + return err; + buf_ofs = ofs; + node = (void *)buf; + } + err = jffs2_scan_xattr_node(c, jeb, (void *)node, ofs, s); + if (err) + return err; + ofs += PAD(je32_to_cpu(node->totlen)); + break; + case JFFS2_NODETYPE_XREF: + if (buf_ofs + buf_len < ofs + je32_to_cpu(node->totlen)) { + buf_len = min_t(uint32_t, buf_size, jeb->offset + c->sector_size - ofs); + jffs2_dbg(1, "Fewer than %d bytes (xref node) left to end of buf. Reading 0x%x at 0x%08x\n", + je32_to_cpu(node->totlen), buf_len, + ofs); + err = jffs2_fill_scan_buf(c, buf, ofs, buf_len); + if (err) + return err; + buf_ofs = ofs; + node = (void *)buf; + } + err = jffs2_scan_xref_node(c, jeb, (void *)node, ofs, s); + if (err) + return err; + ofs += PAD(je32_to_cpu(node->totlen)); + break; +#endif /* CONFIG_JFFS2_FS_XATTR */ + + case JFFS2_NODETYPE_CLEANMARKER: + jffs2_dbg(1, "CLEANMARKER node found at 0x%08x\n", ofs); + if (je32_to_cpu(node->totlen) != c->cleanmarker_size) { + pr_notice("CLEANMARKER node found at 0x%08x has totlen 0x%x != normal 0x%x\n", + ofs, je32_to_cpu(node->totlen), + c->cleanmarker_size); + if ((err = jffs2_scan_dirty_space(c, jeb, PAD(sizeof(struct jffs2_unknown_node))))) + return err; + ofs += PAD(sizeof(struct jffs2_unknown_node)); + } else if (jeb->first_node) { + pr_notice("CLEANMARKER node found at 0x%08x, not first node in block (0x%08x)\n", + ofs, jeb->offset); + if ((err = jffs2_scan_dirty_space(c, jeb, PAD(sizeof(struct jffs2_unknown_node))))) + return err; + ofs += PAD(sizeof(struct jffs2_unknown_node)); + } else { + jffs2_link_node_ref(c, jeb, ofs | REF_NORMAL, c->cleanmarker_size, NULL); + + ofs += PAD(c->cleanmarker_size); + } + break; + + case JFFS2_NODETYPE_PADDING: + if (jffs2_sum_active()) + jffs2_sum_add_padding_mem(s, je32_to_cpu(node->totlen)); + if ((err = jffs2_scan_dirty_space(c, jeb, PAD(je32_to_cpu(node->totlen))))) + return err; + ofs += PAD(je32_to_cpu(node->totlen)); + break; + + default: + switch (je16_to_cpu(node->nodetype) & JFFS2_COMPAT_MASK) { + case JFFS2_FEATURE_ROCOMPAT: + pr_notice("Read-only compatible feature node (0x%04x) found at offset 0x%08x\n", + je16_to_cpu(node->nodetype), ofs); + c->flags |= JFFS2_SB_FLAG_RO; + if (!(jffs2_is_readonly(c))) + return -EROFS; + if ((err = jffs2_scan_dirty_space(c, jeb, PAD(je32_to_cpu(node->totlen))))) + return err; + ofs += PAD(je32_to_cpu(node->totlen)); + break; + + case JFFS2_FEATURE_INCOMPAT: + pr_notice("Incompatible feature node (0x%04x) found at offset 0x%08x\n", + je16_to_cpu(node->nodetype), ofs); + return -EINVAL; + + case JFFS2_FEATURE_RWCOMPAT_DELETE: + jffs2_dbg(1, "Unknown but compatible feature node (0x%04x) found at offset 0x%08x\n", + je16_to_cpu(node->nodetype), ofs); + if ((err = jffs2_scan_dirty_space(c, jeb, PAD(je32_to_cpu(node->totlen))))) + return err; + ofs += PAD(je32_to_cpu(node->totlen)); + break; + + case JFFS2_FEATURE_RWCOMPAT_COPY: { + jffs2_dbg(1, "Unknown but compatible feature node (0x%04x) found at offset 0x%08x\n", + je16_to_cpu(node->nodetype), ofs); + + jffs2_link_node_ref(c, jeb, ofs | REF_PRISTINE, PAD(je32_to_cpu(node->totlen)), NULL); + + /* We can't summarise nodes we don't grok */ + jffs2_sum_disable_collecting(s); + ofs += PAD(je32_to_cpu(node->totlen)); + break; + } + } + } + } + + if (jffs2_sum_active()) { + if (PAD(s->sum_size + JFFS2_SUMMARY_FRAME_SIZE) > jeb->free_size) { + dbg_summary("There is not enough space for " + "summary information, disabling for this jeb!\n"); + jffs2_sum_disable_collecting(s); + } + } + + jffs2_dbg(1, "Block at 0x%08x: free 0x%08x, dirty 0x%08x, unchecked 0x%08x, used 0x%08x, wasted 0x%08x\n", + jeb->offset, jeb->free_size, jeb->dirty_size, + jeb->unchecked_size, jeb->used_size, jeb->wasted_size); + + /* mark_node_obsolete can add to wasted !! */ + if (jeb->wasted_size) { + jeb->dirty_size += jeb->wasted_size; + c->dirty_size += jeb->wasted_size; + c->wasted_size -= jeb->wasted_size; + jeb->wasted_size = 0; + } + + return jffs2_scan_classify_jeb(c, jeb); +} + +struct jffs2_inode_cache *jffs2_scan_make_ino_cache(struct jffs2_sb_info *c, uint32_t ino) +{ + struct jffs2_inode_cache *ic; + + ic = jffs2_get_ino_cache(c, ino); + if (ic) + return ic; + + if (ino > c->highest_ino) + c->highest_ino = ino; + + ic = jffs2_alloc_inode_cache(); + if (!ic) { + pr_notice("%s(): allocation of inode cache failed\n", __func__); + return NULL; + } + memset(ic, 0, sizeof(*ic)); + + ic->ino = ino; + ic->nodes = (void *)ic; + jffs2_add_ino_cache(c, ic); + if (ino == 1) + ic->pino_nlink = 1; + return ic; +} + +static int jffs2_scan_inode_node(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, + struct jffs2_raw_inode *ri, uint32_t ofs, struct jffs2_summary *s) +{ + struct jffs2_inode_cache *ic; + uint32_t crc, ino = je32_to_cpu(ri->ino); + + jffs2_dbg(1, "%s(): Node at 0x%08x\n", __func__, ofs); + + /* We do very little here now. Just check the ino# to which we should attribute + this node; we can do all the CRC checking etc. later. There's a tradeoff here -- + we used to scan the flash once only, reading everything we want from it into + memory, then building all our in-core data structures and freeing the extra + information. Now we allow the first part of the mount to complete a lot quicker, + but we have to go _back_ to the flash in order to finish the CRC checking, etc. + Which means that the _full_ amount of time to get to proper write mode with GC + operational may actually be _longer_ than before. Sucks to be me. */ + + /* Check the node CRC in any case. */ + crc = crc32(0, ri, sizeof(*ri)-8); + if (crc != je32_to_cpu(ri->node_crc)) { + pr_notice("%s(): CRC failed on node at 0x%08x: Read 0x%08x, calculated 0x%08x\n", + __func__, ofs, je32_to_cpu(ri->node_crc), crc); + /* + * We believe totlen because the CRC on the node + * _header_ was OK, just the node itself failed. + */ + return jffs2_scan_dirty_space(c, jeb, + PAD(je32_to_cpu(ri->totlen))); + } + + ic = jffs2_get_ino_cache(c, ino); + if (!ic) { + ic = jffs2_scan_make_ino_cache(c, ino); + if (!ic) + return -ENOMEM; + } + + /* Wheee. It worked */ + jffs2_link_node_ref(c, jeb, ofs | REF_UNCHECKED, PAD(je32_to_cpu(ri->totlen)), ic); + + jffs2_dbg(1, "Node is ino #%u, version %d. Range 0x%x-0x%x\n", + je32_to_cpu(ri->ino), je32_to_cpu(ri->version), + je32_to_cpu(ri->offset), + je32_to_cpu(ri->offset)+je32_to_cpu(ri->dsize)); + + pseudo_random += je32_to_cpu(ri->version); + + if (jffs2_sum_active()) { + jffs2_sum_add_inode_mem(s, ri, ofs - jeb->offset); + } + + return 0; +} + +static int jffs2_scan_dirent_node(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, + struct jffs2_raw_dirent *rd, uint32_t ofs, struct jffs2_summary *s) +{ + struct jffs2_full_dirent *fd; + struct jffs2_inode_cache *ic; + uint32_t checkedlen; + uint32_t crc; + int err; + + jffs2_dbg(1, "%s(): Node at 0x%08x\n", __func__, ofs); + + /* We don't get here unless the node is still valid, so we don't have to + mask in the ACCURATE bit any more. */ + crc = crc32(0, rd, sizeof(*rd)-8); + + if (crc != je32_to_cpu(rd->node_crc)) { + pr_notice("%s(): Node CRC failed on node at 0x%08x: Read 0x%08x, calculated 0x%08x\n", + __func__, ofs, je32_to_cpu(rd->node_crc), crc); + /* We believe totlen because the CRC on the node _header_ was OK, just the node itself failed. */ + if ((err = jffs2_scan_dirty_space(c, jeb, PAD(je32_to_cpu(rd->totlen))))) + return err; + return 0; + } + + pseudo_random += je32_to_cpu(rd->version); + + /* Should never happen. Did. (OLPC trac #4184)*/ + checkedlen = strnlen(rd->name, rd->nsize); + if (checkedlen < rd->nsize) { + pr_err("Dirent at %08x has zeroes in name. Truncating to %d chars\n", + ofs, checkedlen); + } + fd = jffs2_alloc_full_dirent(checkedlen+1); + if (!fd) { + return -ENOMEM; + } + memcpy(&fd->name, rd->name, checkedlen); + fd->name[checkedlen] = 0; + + crc = crc32(0, fd->name, checkedlen); + if (crc != je32_to_cpu(rd->name_crc)) { + pr_notice("%s(): Name CRC failed on node at 0x%08x: Read 0x%08x, calculated 0x%08x\n", + __func__, ofs, je32_to_cpu(rd->name_crc), crc); + jffs2_dbg(1, "Name for which CRC failed is (now) '%s', ino #%d\n", + fd->name, je32_to_cpu(rd->ino)); + jffs2_free_full_dirent(fd); + /* FIXME: Why do we believe totlen? */ + /* We believe totlen because the CRC on the node _header_ was OK, just the name failed. */ + if ((err = jffs2_scan_dirty_space(c, jeb, PAD(je32_to_cpu(rd->totlen))))) + return err; + return 0; + } + ic = jffs2_scan_make_ino_cache(c, je32_to_cpu(rd->pino)); + if (!ic) { + jffs2_free_full_dirent(fd); + return -ENOMEM; + } + + fd->raw = jffs2_link_node_ref(c, jeb, ofs | dirent_node_state(rd), + PAD(je32_to_cpu(rd->totlen)), ic); + + fd->next = NULL; + fd->version = je32_to_cpu(rd->version); + fd->ino = je32_to_cpu(rd->ino); + fd->nhash = full_name_hash(NULL, fd->name, checkedlen); + fd->type = rd->type; + jffs2_add_fd_to_list(c, fd, &ic->scan_dents); + + if (jffs2_sum_active()) { + jffs2_sum_add_dirent_mem(s, rd, ofs - jeb->offset); + } + + return 0; +} + +static int count_list(struct list_head *l) +{ + uint32_t count = 0; + struct list_head *tmp; + + list_for_each(tmp, l) { + count++; + } + return count; +} + +/* Note: This breaks if list_empty(head). I don't care. You + might, if you copy this code and use it elsewhere :) */ +static void rotate_list(struct list_head *head, uint32_t count) +{ + struct list_head *n = head->next; + + list_del(head); + while(count--) { + n = n->next; + } + list_add(head, n); +} + +void jffs2_rotate_lists(struct jffs2_sb_info *c) +{ + uint32_t x; + uint32_t rotateby; + + x = count_list(&c->clean_list); + if (x) { + rotateby = pseudo_random % x; + rotate_list((&c->clean_list), rotateby); + } + + x = count_list(&c->very_dirty_list); + if (x) { + rotateby = pseudo_random % x; + rotate_list((&c->very_dirty_list), rotateby); + } + + x = count_list(&c->dirty_list); + if (x) { + rotateby = pseudo_random % x; + rotate_list((&c->dirty_list), rotateby); + } + + x = count_list(&c->erasable_list); + if (x) { + rotateby = pseudo_random % x; + rotate_list((&c->erasable_list), rotateby); + } + + if (c->nr_erasing_blocks) { + rotateby = pseudo_random % c->nr_erasing_blocks; + rotate_list((&c->erase_pending_list), rotateby); + } + + if (c->nr_free_blocks) { + rotateby = pseudo_random % c->nr_free_blocks; + rotate_list((&c->free_list), rotateby); + } +} diff --git a/fs/jffs2/security.c b/fs/jffs2/security.c new file mode 100644 index 000000000..437f3a2c1 --- /dev/null +++ b/fs/jffs2/security.c @@ -0,0 +1,73 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2006 NEC Corporation + * + * Created by KaiGai Kohei + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "nodelist.h" + +/* ---- Initial Security Label(s) Attachment callback --- */ +static int jffs2_initxattrs(struct inode *inode, + const struct xattr *xattr_array, void *fs_info) +{ + const struct xattr *xattr; + int err = 0; + + for (xattr = xattr_array; xattr->name != NULL; xattr++) { + err = do_jffs2_setxattr(inode, JFFS2_XPREFIX_SECURITY, + xattr->name, xattr->value, + xattr->value_len, 0); + if (err < 0) + break; + } + return err; +} + +/* ---- Initial Security Label(s) Attachment ----------- */ +int jffs2_init_security(struct inode *inode, struct inode *dir, + const struct qstr *qstr) +{ + return security_inode_init_security(inode, dir, qstr, + &jffs2_initxattrs, NULL); +} + +/* ---- XATTR Handler for "security.*" ----------------- */ +static int jffs2_security_getxattr(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + return do_jffs2_getxattr(inode, JFFS2_XPREFIX_SECURITY, + name, buffer, size); +} + +static int jffs2_security_setxattr(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *buffer, + size_t size, int flags) +{ + return do_jffs2_setxattr(inode, JFFS2_XPREFIX_SECURITY, + name, buffer, size, flags); +} + +const struct xattr_handler jffs2_security_xattr_handler = { + .prefix = XATTR_SECURITY_PREFIX, + .set = jffs2_security_setxattr, + .get = jffs2_security_getxattr +}; diff --git a/fs/jffs2/summary.c b/fs/jffs2/summary.c new file mode 100644 index 000000000..4521a7723 --- /dev/null +++ b/fs/jffs2/summary.c @@ -0,0 +1,882 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2004 Ferenc Havasi , + * Zoltan Sogor , + * Patrik Kluba , + * University of Szeged, Hungary + * 2006 KaiGai Kohei + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include "nodelist.h" +#include "debug.h" + +int jffs2_sum_init(struct jffs2_sb_info *c) +{ + uint32_t sum_size = min_t(uint32_t, c->sector_size, MAX_SUMMARY_SIZE); + + c->summary = kzalloc_obj(struct jffs2_summary); + + if (!c->summary) { + JFFS2_WARNING("Can't allocate memory for summary information!\n"); + return -ENOMEM; + } + + c->summary->sum_buf = kmalloc(sum_size, GFP_KERNEL); + + if (!c->summary->sum_buf) { + JFFS2_WARNING("Can't allocate buffer for writing out summary information!\n"); + kfree(c->summary); + return -ENOMEM; + } + + dbg_summary("returned successfully\n"); + + return 0; +} + +void jffs2_sum_exit(struct jffs2_sb_info *c) +{ + dbg_summary("called\n"); + + jffs2_sum_disable_collecting(c->summary); + + kfree(c->summary->sum_buf); + c->summary->sum_buf = NULL; + + kfree(c->summary); + c->summary = NULL; +} + +static int jffs2_sum_add_mem(struct jffs2_summary *s, union jffs2_sum_mem *item) +{ + if (!s->sum_list_head) + s->sum_list_head = (union jffs2_sum_mem *) item; + if (s->sum_list_tail) + s->sum_list_tail->u.next = (union jffs2_sum_mem *) item; + s->sum_list_tail = (union jffs2_sum_mem *) item; + + switch (je16_to_cpu(item->u.nodetype)) { + case JFFS2_NODETYPE_INODE: + s->sum_size += JFFS2_SUMMARY_INODE_SIZE; + s->sum_num++; + dbg_summary("inode (%u) added to summary\n", + je32_to_cpu(item->i.inode)); + break; + case JFFS2_NODETYPE_DIRENT: + s->sum_size += JFFS2_SUMMARY_DIRENT_SIZE(item->d.nsize); + s->sum_num++; + dbg_summary("dirent (%u) added to summary\n", + je32_to_cpu(item->d.ino)); + break; +#ifdef CONFIG_JFFS2_FS_XATTR + case JFFS2_NODETYPE_XATTR: + s->sum_size += JFFS2_SUMMARY_XATTR_SIZE; + s->sum_num++; + dbg_summary("xattr (xid=%u, version=%u) added to summary\n", + je32_to_cpu(item->x.xid), je32_to_cpu(item->x.version)); + break; + case JFFS2_NODETYPE_XREF: + s->sum_size += JFFS2_SUMMARY_XREF_SIZE; + s->sum_num++; + dbg_summary("xref added to summary\n"); + break; +#endif + default: + JFFS2_WARNING("UNKNOWN node type %u\n", + je16_to_cpu(item->u.nodetype)); + return 1; + } + return 0; +} + + +/* The following 3 functions are called from scan.c to collect summary info for not closed jeb */ + +int jffs2_sum_add_padding_mem(struct jffs2_summary *s, uint32_t size) +{ + dbg_summary("called with %u\n", size); + s->sum_padded += size; + return 0; +} + +int jffs2_sum_add_inode_mem(struct jffs2_summary *s, struct jffs2_raw_inode *ri, + uint32_t ofs) +{ + struct jffs2_sum_inode_mem *temp = kmalloc_obj(struct jffs2_sum_inode_mem); + + if (!temp) + return -ENOMEM; + + temp->nodetype = ri->nodetype; + temp->inode = ri->ino; + temp->version = ri->version; + temp->offset = cpu_to_je32(ofs); /* relative offset from the beginning of the jeb */ + temp->totlen = ri->totlen; + temp->next = NULL; + + return jffs2_sum_add_mem(s, (union jffs2_sum_mem *)temp); +} + +int jffs2_sum_add_dirent_mem(struct jffs2_summary *s, struct jffs2_raw_dirent *rd, + uint32_t ofs) +{ + struct jffs2_sum_dirent_mem *temp = + kmalloc(sizeof(struct jffs2_sum_dirent_mem) + rd->nsize, GFP_KERNEL); + + if (!temp) + return -ENOMEM; + + temp->nodetype = rd->nodetype; + temp->totlen = rd->totlen; + temp->offset = cpu_to_je32(ofs); /* relative from the beginning of the jeb */ + temp->pino = rd->pino; + temp->version = rd->version; + temp->ino = rd->ino; + temp->nsize = rd->nsize; + temp->type = rd->type; + temp->next = NULL; + + memcpy(temp->name, rd->name, rd->nsize); + + return jffs2_sum_add_mem(s, (union jffs2_sum_mem *)temp); +} + +#ifdef CONFIG_JFFS2_FS_XATTR +int jffs2_sum_add_xattr_mem(struct jffs2_summary *s, struct jffs2_raw_xattr *rx, uint32_t ofs) +{ + struct jffs2_sum_xattr_mem *temp; + + temp = kmalloc_obj(struct jffs2_sum_xattr_mem); + if (!temp) + return -ENOMEM; + + temp->nodetype = rx->nodetype; + temp->xid = rx->xid; + temp->version = rx->version; + temp->offset = cpu_to_je32(ofs); + temp->totlen = rx->totlen; + temp->next = NULL; + + return jffs2_sum_add_mem(s, (union jffs2_sum_mem *)temp); +} + +int jffs2_sum_add_xref_mem(struct jffs2_summary *s, struct jffs2_raw_xref *rr, uint32_t ofs) +{ + struct jffs2_sum_xref_mem *temp; + + temp = kmalloc_obj(struct jffs2_sum_xref_mem); + if (!temp) + return -ENOMEM; + + temp->nodetype = rr->nodetype; + temp->offset = cpu_to_je32(ofs); + temp->next = NULL; + + return jffs2_sum_add_mem(s, (union jffs2_sum_mem *)temp); +} +#endif +/* Cleanup every collected summary information */ + +static void jffs2_sum_clean_collected(struct jffs2_summary *s) +{ + union jffs2_sum_mem *temp; + + if (!s->sum_list_head) { + dbg_summary("already empty\n"); + } + while (s->sum_list_head) { + temp = s->sum_list_head; + s->sum_list_head = s->sum_list_head->u.next; + kfree(temp); + } + s->sum_list_tail = NULL; + s->sum_padded = 0; + s->sum_num = 0; +} + +void jffs2_sum_reset_collected(struct jffs2_summary *s) +{ + dbg_summary("called\n"); + jffs2_sum_clean_collected(s); + s->sum_size = 0; +} + +void jffs2_sum_disable_collecting(struct jffs2_summary *s) +{ + dbg_summary("called\n"); + jffs2_sum_clean_collected(s); + s->sum_size = JFFS2_SUMMARY_NOSUM_SIZE; +} + +int jffs2_sum_is_disabled(struct jffs2_summary *s) +{ + return (s->sum_size == JFFS2_SUMMARY_NOSUM_SIZE); +} + +/* Move the collected summary information into sb (called from scan.c) */ + +void jffs2_sum_move_collected(struct jffs2_sb_info *c, struct jffs2_summary *s) +{ + dbg_summary("oldsize=0x%x oldnum=%u => newsize=0x%x newnum=%u\n", + c->summary->sum_size, c->summary->sum_num, + s->sum_size, s->sum_num); + + c->summary->sum_size = s->sum_size; + c->summary->sum_num = s->sum_num; + c->summary->sum_padded = s->sum_padded; + c->summary->sum_list_head = s->sum_list_head; + c->summary->sum_list_tail = s->sum_list_tail; + + s->sum_list_head = s->sum_list_tail = NULL; +} + +/* Called from wbuf.c to collect writed node info */ + +int jffs2_sum_add_kvec(struct jffs2_sb_info *c, const struct kvec *invecs, + unsigned long count, uint32_t ofs) +{ + union jffs2_node_union *node; + struct jffs2_eraseblock *jeb; + + if (c->summary->sum_size == JFFS2_SUMMARY_NOSUM_SIZE) { + dbg_summary("Summary is disabled for this jeb! Skipping summary info!\n"); + return 0; + } + + node = invecs[0].iov_base; + jeb = &c->blocks[ofs / c->sector_size]; + ofs -= jeb->offset; + + switch (je16_to_cpu(node->u.nodetype)) { + case JFFS2_NODETYPE_INODE: { + struct jffs2_sum_inode_mem *temp = + kmalloc_obj(struct jffs2_sum_inode_mem); + + if (!temp) + goto no_mem; + + temp->nodetype = node->i.nodetype; + temp->inode = node->i.ino; + temp->version = node->i.version; + temp->offset = cpu_to_je32(ofs); + temp->totlen = node->i.totlen; + temp->next = NULL; + + return jffs2_sum_add_mem(c->summary, (union jffs2_sum_mem *)temp); + } + + case JFFS2_NODETYPE_DIRENT: { + struct jffs2_sum_dirent_mem *temp = + kmalloc(sizeof(struct jffs2_sum_dirent_mem) + node->d.nsize, GFP_KERNEL); + + if (!temp) + goto no_mem; + + temp->nodetype = node->d.nodetype; + temp->totlen = node->d.totlen; + temp->offset = cpu_to_je32(ofs); + temp->pino = node->d.pino; + temp->version = node->d.version; + temp->ino = node->d.ino; + temp->nsize = node->d.nsize; + temp->type = node->d.type; + temp->next = NULL; + + switch (count) { + case 1: + memcpy(temp->name,node->d.name,node->d.nsize); + break; + + case 2: + memcpy(temp->name,invecs[1].iov_base,node->d.nsize); + break; + + default: + BUG(); /* impossible count value */ + break; + } + + return jffs2_sum_add_mem(c->summary, (union jffs2_sum_mem *)temp); + } +#ifdef CONFIG_JFFS2_FS_XATTR + case JFFS2_NODETYPE_XATTR: { + struct jffs2_sum_xattr_mem *temp; + temp = kmalloc_obj(struct jffs2_sum_xattr_mem); + if (!temp) + goto no_mem; + + temp->nodetype = node->x.nodetype; + temp->xid = node->x.xid; + temp->version = node->x.version; + temp->totlen = node->x.totlen; + temp->offset = cpu_to_je32(ofs); + temp->next = NULL; + + return jffs2_sum_add_mem(c->summary, (union jffs2_sum_mem *)temp); + } + case JFFS2_NODETYPE_XREF: { + struct jffs2_sum_xref_mem *temp; + temp = kmalloc_obj(struct jffs2_sum_xref_mem); + if (!temp) + goto no_mem; + temp->nodetype = node->r.nodetype; + temp->offset = cpu_to_je32(ofs); + temp->next = NULL; + + return jffs2_sum_add_mem(c->summary, (union jffs2_sum_mem *)temp); + } +#endif + case JFFS2_NODETYPE_PADDING: + dbg_summary("node PADDING\n"); + c->summary->sum_padded += je32_to_cpu(node->u.totlen); + break; + + case JFFS2_NODETYPE_CLEANMARKER: + dbg_summary("node CLEANMARKER\n"); + break; + + case JFFS2_NODETYPE_SUMMARY: + dbg_summary("node SUMMARY\n"); + break; + + default: + /* If you implement a new node type you should also implement + summary support for it or disable summary. + */ + BUG(); + break; + } + + return 0; + +no_mem: + JFFS2_WARNING("MEMORY ALLOCATION ERROR!"); + return -ENOMEM; +} + +static struct jffs2_raw_node_ref *sum_link_node_ref(struct jffs2_sb_info *c, + struct jffs2_eraseblock *jeb, + uint32_t ofs, uint32_t len, + struct jffs2_inode_cache *ic) +{ + /* If there was a gap, mark it dirty */ + if ((ofs & ~3) > c->sector_size - jeb->free_size) { + /* Ew. Summary doesn't actually tell us explicitly about dirty space */ + jffs2_scan_dirty_space(c, jeb, (ofs & ~3) - (c->sector_size - jeb->free_size)); + } + + return jffs2_link_node_ref(c, jeb, jeb->offset + ofs, len, ic); +} + +/* Process the stored summary information - helper function for jffs2_sum_scan_sumnode() */ + +static int jffs2_sum_process_sum_data(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, + struct jffs2_raw_summary *summary, uint32_t *pseudo_random) +{ + struct jffs2_inode_cache *ic; + struct jffs2_full_dirent *fd; + void *sp; + int i, ino; + int err; + + sp = summary->sum; + + for (i=0; isum_num); i++) { + dbg_summary("processing summary index %d\n", i); + + cond_resched(); + + /* Make sure there's a spare ref for dirty space */ + err = jffs2_prealloc_raw_node_refs(c, jeb, 2); + if (err) + return err; + + switch (je16_to_cpu(((struct jffs2_sum_unknown_flash *)sp)->nodetype)) { + case JFFS2_NODETYPE_INODE: { + struct jffs2_sum_inode_flash *spi; + spi = sp; + + ino = je32_to_cpu(spi->inode); + + dbg_summary("Inode at 0x%08x-0x%08x\n", + jeb->offset + je32_to_cpu(spi->offset), + jeb->offset + je32_to_cpu(spi->offset) + je32_to_cpu(spi->totlen)); + + ic = jffs2_scan_make_ino_cache(c, ino); + if (!ic) { + JFFS2_NOTICE("scan_make_ino_cache failed\n"); + return -ENOMEM; + } + + sum_link_node_ref(c, jeb, je32_to_cpu(spi->offset) | REF_UNCHECKED, + PAD(je32_to_cpu(spi->totlen)), ic); + + *pseudo_random += je32_to_cpu(spi->version); + + sp += JFFS2_SUMMARY_INODE_SIZE; + + break; + } + + case JFFS2_NODETYPE_DIRENT: { + struct jffs2_sum_dirent_flash *spd; + int checkedlen; + spd = sp; + + dbg_summary("Dirent at 0x%08x-0x%08x\n", + jeb->offset + je32_to_cpu(spd->offset), + jeb->offset + je32_to_cpu(spd->offset) + je32_to_cpu(spd->totlen)); + + + /* This should never happen, but https://dev.laptop.org/ticket/4184 */ + checkedlen = strnlen(spd->name, spd->nsize); + if (!checkedlen) { + pr_err("Dirent at %08x has zero at start of name. Aborting mount.\n", + jeb->offset + + je32_to_cpu(spd->offset)); + return -EIO; + } + if (checkedlen < spd->nsize) { + pr_err("Dirent at %08x has zeroes in name. Truncating to %d chars\n", + jeb->offset + + je32_to_cpu(spd->offset), + checkedlen); + } + + + fd = jffs2_alloc_full_dirent(checkedlen+1); + if (!fd) + return -ENOMEM; + + memcpy(&fd->name, spd->name, checkedlen); + fd->name[checkedlen] = 0; + + ic = jffs2_scan_make_ino_cache(c, je32_to_cpu(spd->pino)); + if (!ic) { + jffs2_free_full_dirent(fd); + return -ENOMEM; + } + + fd->raw = sum_link_node_ref(c, jeb, je32_to_cpu(spd->offset) | REF_UNCHECKED, + PAD(je32_to_cpu(spd->totlen)), ic); + + fd->next = NULL; + fd->version = je32_to_cpu(spd->version); + fd->ino = je32_to_cpu(spd->ino); + fd->nhash = full_name_hash(NULL, fd->name, checkedlen); + fd->type = spd->type; + + jffs2_add_fd_to_list(c, fd, &ic->scan_dents); + + *pseudo_random += je32_to_cpu(spd->version); + + sp += JFFS2_SUMMARY_DIRENT_SIZE(spd->nsize); + + break; + } +#ifdef CONFIG_JFFS2_FS_XATTR + case JFFS2_NODETYPE_XATTR: { + struct jffs2_xattr_datum *xd; + struct jffs2_sum_xattr_flash *spx; + + spx = (struct jffs2_sum_xattr_flash *)sp; + dbg_summary("xattr at %#08x-%#08x (xid=%u, version=%u)\n", + jeb->offset + je32_to_cpu(spx->offset), + jeb->offset + je32_to_cpu(spx->offset) + je32_to_cpu(spx->totlen), + je32_to_cpu(spx->xid), je32_to_cpu(spx->version)); + + xd = jffs2_setup_xattr_datum(c, je32_to_cpu(spx->xid), + je32_to_cpu(spx->version)); + if (IS_ERR(xd)) + return PTR_ERR(xd); + if (xd->version > je32_to_cpu(spx->version)) { + /* node is not the newest one */ + struct jffs2_raw_node_ref *raw + = sum_link_node_ref(c, jeb, je32_to_cpu(spx->offset) | REF_UNCHECKED, + PAD(je32_to_cpu(spx->totlen)), NULL); + raw->next_in_ino = xd->node->next_in_ino; + xd->node->next_in_ino = raw; + } else { + xd->version = je32_to_cpu(spx->version); + sum_link_node_ref(c, jeb, je32_to_cpu(spx->offset) | REF_UNCHECKED, + PAD(je32_to_cpu(spx->totlen)), (void *)xd); + } + *pseudo_random += je32_to_cpu(spx->xid); + sp += JFFS2_SUMMARY_XATTR_SIZE; + + break; + } + case JFFS2_NODETYPE_XREF: { + struct jffs2_xattr_ref *ref; + struct jffs2_sum_xref_flash *spr; + + spr = (struct jffs2_sum_xref_flash *)sp; + dbg_summary("xref at %#08x-%#08x\n", + jeb->offset + je32_to_cpu(spr->offset), + jeb->offset + je32_to_cpu(spr->offset) + + (uint32_t)PAD(sizeof(struct jffs2_raw_xref))); + + ref = jffs2_alloc_xattr_ref(); + if (!ref) { + JFFS2_NOTICE("allocation of xattr_datum failed\n"); + return -ENOMEM; + } + ref->next = c->xref_temp; + c->xref_temp = ref; + + sum_link_node_ref(c, jeb, je32_to_cpu(spr->offset) | REF_UNCHECKED, + PAD(sizeof(struct jffs2_raw_xref)), (void *)ref); + + *pseudo_random += ref->node->flash_offset; + sp += JFFS2_SUMMARY_XREF_SIZE; + + break; + } +#endif + default : { + uint16_t nodetype = je16_to_cpu(((struct jffs2_sum_unknown_flash *)sp)->nodetype); + JFFS2_WARNING("Unsupported node type %x found in summary! Exiting...\n", nodetype); + if ((nodetype & JFFS2_COMPAT_MASK) == JFFS2_FEATURE_INCOMPAT) + return -EIO; + + /* For compatible node types, just fall back to the full scan */ + c->wasted_size -= jeb->wasted_size; + c->free_size += c->sector_size - jeb->free_size; + c->used_size -= jeb->used_size; + c->dirty_size -= jeb->dirty_size; + jeb->wasted_size = jeb->used_size = jeb->dirty_size = 0; + jeb->free_size = c->sector_size; + + jffs2_free_jeb_node_refs(c, jeb); + return -ENOTRECOVERABLE; + } + } + } + return 0; +} + +/* Process the summary node - called from jffs2_scan_eraseblock() */ +int jffs2_sum_scan_sumnode(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, + struct jffs2_raw_summary *summary, uint32_t sumsize, + uint32_t *pseudo_random) +{ + struct jffs2_unknown_node crcnode; + int ret, ofs; + uint32_t crc; + + ofs = c->sector_size - sumsize; + + dbg_summary("summary found for 0x%08x at 0x%08x (0x%x bytes)\n", + jeb->offset, jeb->offset + ofs, sumsize); + + /* OK, now check for node validity and CRC */ + crcnode.magic = cpu_to_je16(JFFS2_MAGIC_BITMASK); + crcnode.nodetype = cpu_to_je16(JFFS2_NODETYPE_SUMMARY); + crcnode.totlen = summary->totlen; + crc = crc32(0, &crcnode, sizeof(crcnode)-4); + + if (je32_to_cpu(summary->hdr_crc) != crc) { + dbg_summary("Summary node header is corrupt (bad CRC or " + "no summary at all)\n"); + goto crc_err; + } + + if (je32_to_cpu(summary->totlen) != sumsize) { + dbg_summary("Summary node is corrupt (wrong erasesize?)\n"); + goto crc_err; + } + + crc = crc32(0, summary, sizeof(struct jffs2_raw_summary)-8); + + if (je32_to_cpu(summary->node_crc) != crc) { + dbg_summary("Summary node is corrupt (bad CRC)\n"); + goto crc_err; + } + + crc = crc32(0, summary->sum, sumsize - sizeof(struct jffs2_raw_summary)); + + if (je32_to_cpu(summary->sum_crc) != crc) { + dbg_summary("Summary node data is corrupt (bad CRC)\n"); + goto crc_err; + } + + if ( je32_to_cpu(summary->cln_mkr) ) { + + dbg_summary("Summary : CLEANMARKER node \n"); + + ret = jffs2_prealloc_raw_node_refs(c, jeb, 1); + if (ret) + return ret; + + if (je32_to_cpu(summary->cln_mkr) != c->cleanmarker_size) { + dbg_summary("CLEANMARKER node has totlen 0x%x != normal 0x%x\n", + je32_to_cpu(summary->cln_mkr), c->cleanmarker_size); + if ((ret = jffs2_scan_dirty_space(c, jeb, PAD(je32_to_cpu(summary->cln_mkr))))) + return ret; + } else if (jeb->first_node) { + dbg_summary("CLEANMARKER node not first node in block " + "(0x%08x)\n", jeb->offset); + if ((ret = jffs2_scan_dirty_space(c, jeb, PAD(je32_to_cpu(summary->cln_mkr))))) + return ret; + } else { + jffs2_link_node_ref(c, jeb, jeb->offset | REF_NORMAL, + je32_to_cpu(summary->cln_mkr), NULL); + } + } + + ret = jffs2_sum_process_sum_data(c, jeb, summary, pseudo_random); + /* -ENOTRECOVERABLE isn't a fatal error -- it means we should do a full + scan of this eraseblock. So return zero */ + if (ret == -ENOTRECOVERABLE) + return 0; + if (ret) + return ret; /* real error */ + + /* for PARANOIA_CHECK */ + ret = jffs2_prealloc_raw_node_refs(c, jeb, 2); + if (ret) + return ret; + + sum_link_node_ref(c, jeb, ofs | REF_NORMAL, sumsize, NULL); + + if (unlikely(jeb->free_size)) { + JFFS2_WARNING("Free size 0x%x bytes in eraseblock @0x%08x with summary?\n", + jeb->free_size, jeb->offset); + jeb->wasted_size += jeb->free_size; + c->wasted_size += jeb->free_size; + c->free_size -= jeb->free_size; + jeb->free_size = 0; + } + + return jffs2_scan_classify_jeb(c, jeb); + +crc_err: + JFFS2_WARNING("Summary node crc error, skipping summary information.\n"); + + return 0; +} + +/* Write summary data to flash - helper function for jffs2_sum_write_sumnode() */ + +static int jffs2_sum_write_data(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, + uint32_t infosize, uint32_t datasize, int padsize) +{ + struct jffs2_raw_summary isum; + union jffs2_sum_mem *temp; + struct jffs2_sum_marker *sm; + struct kvec vecs[2]; + uint32_t sum_ofs; + void *wpage; + int ret; + size_t retlen; + + if (padsize + datasize > MAX_SUMMARY_SIZE) { + /* It won't fit in the buffer. Abort summary for this jeb */ + jffs2_sum_disable_collecting(c->summary); + + JFFS2_WARNING("Summary too big (%d data, %d pad) in eraseblock at %08x\n", + datasize, padsize, jeb->offset); + /* Non-fatal */ + return 0; + } + /* Is there enough space for summary? */ + if (padsize < 0) { + /* don't try to write out summary for this jeb */ + jffs2_sum_disable_collecting(c->summary); + + JFFS2_WARNING("Not enough space for summary, padsize = %d\n", + padsize); + /* Non-fatal */ + return 0; + } + + memset(c->summary->sum_buf, 0xff, datasize); + memset(&isum, 0, sizeof(isum)); + + isum.magic = cpu_to_je16(JFFS2_MAGIC_BITMASK); + isum.nodetype = cpu_to_je16(JFFS2_NODETYPE_SUMMARY); + isum.totlen = cpu_to_je32(infosize); + isum.hdr_crc = cpu_to_je32(crc32(0, &isum, sizeof(struct jffs2_unknown_node) - 4)); + isum.padded = cpu_to_je32(c->summary->sum_padded); + isum.cln_mkr = cpu_to_je32(c->cleanmarker_size); + isum.sum_num = cpu_to_je32(c->summary->sum_num); + wpage = c->summary->sum_buf; + + while (c->summary->sum_num) { + temp = c->summary->sum_list_head; + + switch (je16_to_cpu(temp->u.nodetype)) { + case JFFS2_NODETYPE_INODE: { + struct jffs2_sum_inode_flash *sino_ptr = wpage; + + sino_ptr->nodetype = temp->i.nodetype; + sino_ptr->inode = temp->i.inode; + sino_ptr->version = temp->i.version; + sino_ptr->offset = temp->i.offset; + sino_ptr->totlen = temp->i.totlen; + + wpage += JFFS2_SUMMARY_INODE_SIZE; + + break; + } + + case JFFS2_NODETYPE_DIRENT: { + struct jffs2_sum_dirent_flash *sdrnt_ptr = wpage; + + sdrnt_ptr->nodetype = temp->d.nodetype; + sdrnt_ptr->totlen = temp->d.totlen; + sdrnt_ptr->offset = temp->d.offset; + sdrnt_ptr->pino = temp->d.pino; + sdrnt_ptr->version = temp->d.version; + sdrnt_ptr->ino = temp->d.ino; + sdrnt_ptr->nsize = temp->d.nsize; + sdrnt_ptr->type = temp->d.type; + + memcpy(sdrnt_ptr->name, temp->d.name, + temp->d.nsize); + + wpage += JFFS2_SUMMARY_DIRENT_SIZE(temp->d.nsize); + + break; + } +#ifdef CONFIG_JFFS2_FS_XATTR + case JFFS2_NODETYPE_XATTR: { + struct jffs2_sum_xattr_flash *sxattr_ptr = wpage; + + temp = c->summary->sum_list_head; + sxattr_ptr->nodetype = temp->x.nodetype; + sxattr_ptr->xid = temp->x.xid; + sxattr_ptr->version = temp->x.version; + sxattr_ptr->offset = temp->x.offset; + sxattr_ptr->totlen = temp->x.totlen; + + wpage += JFFS2_SUMMARY_XATTR_SIZE; + break; + } + case JFFS2_NODETYPE_XREF: { + struct jffs2_sum_xref_flash *sxref_ptr = wpage; + + temp = c->summary->sum_list_head; + sxref_ptr->nodetype = temp->r.nodetype; + sxref_ptr->offset = temp->r.offset; + + wpage += JFFS2_SUMMARY_XREF_SIZE; + break; + } +#endif + default : { + if ((je16_to_cpu(temp->u.nodetype) & JFFS2_COMPAT_MASK) + == JFFS2_FEATURE_RWCOMPAT_COPY) { + dbg_summary("Writing unknown RWCOMPAT_COPY node type %x\n", + je16_to_cpu(temp->u.nodetype)); + jffs2_sum_disable_collecting(c->summary); + /* The above call removes the list, nothing more to do */ + goto bail_rwcompat; + } else { + BUG(); /* unknown node in summary information */ + } + } + } + + c->summary->sum_list_head = temp->u.next; + kfree(temp); + + c->summary->sum_num--; + } + bail_rwcompat: + + jffs2_sum_reset_collected(c->summary); + + wpage += padsize; + + sm = wpage; + sm->offset = cpu_to_je32(c->sector_size - jeb->free_size); + sm->magic = cpu_to_je32(JFFS2_SUM_MAGIC); + + isum.sum_crc = cpu_to_je32(crc32(0, c->summary->sum_buf, datasize)); + isum.node_crc = cpu_to_je32(crc32(0, &isum, sizeof(isum) - 8)); + + vecs[0].iov_base = &isum; + vecs[0].iov_len = sizeof(isum); + vecs[1].iov_base = c->summary->sum_buf; + vecs[1].iov_len = datasize; + + sum_ofs = jeb->offset + c->sector_size - jeb->free_size; + + dbg_summary("writing out data to flash to pos : 0x%08x\n", sum_ofs); + + ret = jffs2_flash_writev(c, vecs, 2, sum_ofs, &retlen, 0); + + if (ret || (retlen != infosize)) { + + JFFS2_WARNING("Write of %u bytes at 0x%08x failed. returned %d, retlen %zd\n", + infosize, sum_ofs, ret, retlen); + + if (retlen) { + /* Waste remaining space */ + spin_lock(&c->erase_completion_lock); + jffs2_link_node_ref(c, jeb, sum_ofs | REF_OBSOLETE, infosize, NULL); + spin_unlock(&c->erase_completion_lock); + } + + c->summary->sum_size = JFFS2_SUMMARY_NOSUM_SIZE; + + return 0; + } + + spin_lock(&c->erase_completion_lock); + jffs2_link_node_ref(c, jeb, sum_ofs | REF_NORMAL, infosize, NULL); + spin_unlock(&c->erase_completion_lock); + + return 0; +} + +/* Write out summary information - called from jffs2_do_reserve_space */ + +int jffs2_sum_write_sumnode(struct jffs2_sb_info *c) + __must_hold(&c->erase_completion_block) +{ + int datasize, infosize, padsize; + struct jffs2_eraseblock *jeb; + int ret = 0; + + dbg_summary("called\n"); + + spin_unlock(&c->erase_completion_lock); + + jeb = c->nextblock; + ret = jffs2_prealloc_raw_node_refs(c, jeb, 1); + + if (ret) + goto out; + + if (!c->summary->sum_num || !c->summary->sum_list_head) { + JFFS2_WARNING("Empty summary info!!!\n"); + BUG(); + } + + datasize = c->summary->sum_size + sizeof(struct jffs2_sum_marker); + infosize = sizeof(struct jffs2_raw_summary) + datasize; + padsize = jeb->free_size - infosize; + infosize += padsize; + datasize += padsize; + + ret = jffs2_sum_write_data(c, jeb, infosize, datasize, padsize); + +out: + spin_lock(&c->erase_completion_lock); + return ret; +} diff --git a/fs/jffs2/summary.h b/fs/jffs2/summary.h new file mode 100644 index 000000000..36d9a1280 --- /dev/null +++ b/fs/jffs2/summary.h @@ -0,0 +1,213 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2004 Ferenc Havasi , + * Zoltan Sogor , + * Patrik Kluba , + * University of Szeged, Hungary + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#ifndef JFFS2_SUMMARY_H +#define JFFS2_SUMMARY_H + +/* Limit summary size to 64KiB so that we can kmalloc it. If the summary + is larger than that, we have to just ditch it and avoid using summary + for the eraseblock in question... and it probably doesn't hurt us much + anyway. */ +#define MAX_SUMMARY_SIZE 65536 + +#include +#include + +#define BLK_STATE_ALLFF 0 +#define BLK_STATE_CLEAN 1 +#define BLK_STATE_PARTDIRTY 2 +#define BLK_STATE_CLEANMARKER 3 +#define BLK_STATE_ALLDIRTY 4 +#define BLK_STATE_BADBLOCK 5 + +#define JFFS2_SUMMARY_NOSUM_SIZE 0xffffffff +#define JFFS2_SUMMARY_INODE_SIZE (sizeof(struct jffs2_sum_inode_flash)) +#define JFFS2_SUMMARY_DIRENT_SIZE(x) (sizeof(struct jffs2_sum_dirent_flash) + (x)) +#define JFFS2_SUMMARY_XATTR_SIZE (sizeof(struct jffs2_sum_xattr_flash)) +#define JFFS2_SUMMARY_XREF_SIZE (sizeof(struct jffs2_sum_xref_flash)) + +/* Summary structures used on flash */ + +struct jffs2_sum_unknown_flash +{ + jint16_t nodetype; /* node type */ +}; + +struct jffs2_sum_inode_flash +{ + jint16_t nodetype; /* node type */ + jint32_t inode; /* inode number */ + jint32_t version; /* inode version */ + jint32_t offset; /* offset on jeb */ + jint32_t totlen; /* record length */ +} __attribute__((packed)); + +struct jffs2_sum_dirent_flash +{ + jint16_t nodetype; /* == JFFS_NODETYPE_DIRENT */ + jint32_t totlen; /* record length */ + jint32_t offset; /* offset on jeb */ + jint32_t pino; /* parent inode */ + jint32_t version; /* dirent version */ + jint32_t ino; /* == zero for unlink */ + uint8_t nsize; /* dirent name size */ + uint8_t type; /* dirent type */ + uint8_t name[]; /* dirent name */ +} __attribute__((packed)); + +struct jffs2_sum_xattr_flash +{ + jint16_t nodetype; /* == JFFS2_NODETYPE_XATR */ + jint32_t xid; /* xattr identifier */ + jint32_t version; /* version number */ + jint32_t offset; /* offset on jeb */ + jint32_t totlen; /* node length */ +} __attribute__((packed)); + +struct jffs2_sum_xref_flash +{ + jint16_t nodetype; /* == JFFS2_NODETYPE_XREF */ + jint32_t offset; /* offset on jeb */ +} __attribute__((packed)); + +union jffs2_sum_flash +{ + struct jffs2_sum_unknown_flash u; + struct jffs2_sum_inode_flash i; + struct jffs2_sum_dirent_flash d; + struct jffs2_sum_xattr_flash x; + struct jffs2_sum_xref_flash r; +}; + +/* Summary structures used in the memory */ + +struct jffs2_sum_unknown_mem +{ + union jffs2_sum_mem *next; + jint16_t nodetype; /* node type */ +}; + +struct jffs2_sum_inode_mem +{ + union jffs2_sum_mem *next; + jint16_t nodetype; /* node type */ + jint32_t inode; /* inode number */ + jint32_t version; /* inode version */ + jint32_t offset; /* offset on jeb */ + jint32_t totlen; /* record length */ +} __attribute__((packed)); + +struct jffs2_sum_dirent_mem +{ + union jffs2_sum_mem *next; + jint16_t nodetype; /* == JFFS_NODETYPE_DIRENT */ + jint32_t totlen; /* record length */ + jint32_t offset; /* ofset on jeb */ + jint32_t pino; /* parent inode */ + jint32_t version; /* dirent version */ + jint32_t ino; /* == zero for unlink */ + uint8_t nsize; /* dirent name size */ + uint8_t type; /* dirent type */ + uint8_t name[]; /* dirent name */ +} __attribute__((packed)); + +struct jffs2_sum_xattr_mem +{ + union jffs2_sum_mem *next; + jint16_t nodetype; + jint32_t xid; + jint32_t version; + jint32_t offset; + jint32_t totlen; +} __attribute__((packed)); + +struct jffs2_sum_xref_mem +{ + union jffs2_sum_mem *next; + jint16_t nodetype; + jint32_t offset; +} __attribute__((packed)); + +union jffs2_sum_mem +{ + struct jffs2_sum_unknown_mem u; + struct jffs2_sum_inode_mem i; + struct jffs2_sum_dirent_mem d; + struct jffs2_sum_xattr_mem x; + struct jffs2_sum_xref_mem r; +}; + +/* Summary related information stored in superblock */ + +struct jffs2_summary +{ + uint32_t sum_size; /* collected summary information for nextblock */ + uint32_t sum_num; + uint32_t sum_padded; + union jffs2_sum_mem *sum_list_head; + union jffs2_sum_mem *sum_list_tail; + + jint32_t *sum_buf; /* buffer for writing out summary */ +}; + +/* Summary marker is stored at the end of every sumarized erase block */ + +struct jffs2_sum_marker +{ + jint32_t offset; /* offset of the summary node in the jeb */ + jint32_t magic; /* == JFFS2_SUM_MAGIC */ +}; + +#define JFFS2_SUMMARY_FRAME_SIZE (sizeof(struct jffs2_raw_summary) + sizeof(struct jffs2_sum_marker)) + +#ifdef CONFIG_JFFS2_SUMMARY /* SUMMARY SUPPORT ENABLED */ + +#define jffs2_sum_active() (1) +int jffs2_sum_init(struct jffs2_sb_info *c); +void jffs2_sum_exit(struct jffs2_sb_info *c); +void jffs2_sum_disable_collecting(struct jffs2_summary *s); +int jffs2_sum_is_disabled(struct jffs2_summary *s); +void jffs2_sum_reset_collected(struct jffs2_summary *s); +void jffs2_sum_move_collected(struct jffs2_sb_info *c, struct jffs2_summary *s); +int jffs2_sum_add_kvec(struct jffs2_sb_info *c, const struct kvec *invecs, + unsigned long count, uint32_t to); +int jffs2_sum_write_sumnode(struct jffs2_sb_info *c); +int jffs2_sum_add_padding_mem(struct jffs2_summary *s, uint32_t size); +int jffs2_sum_add_inode_mem(struct jffs2_summary *s, struct jffs2_raw_inode *ri, uint32_t ofs); +int jffs2_sum_add_dirent_mem(struct jffs2_summary *s, struct jffs2_raw_dirent *rd, uint32_t ofs); +int jffs2_sum_add_xattr_mem(struct jffs2_summary *s, struct jffs2_raw_xattr *rx, uint32_t ofs); +int jffs2_sum_add_xref_mem(struct jffs2_summary *s, struct jffs2_raw_xref *rr, uint32_t ofs); +int jffs2_sum_scan_sumnode(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, + struct jffs2_raw_summary *summary, uint32_t sumlen, + uint32_t *pseudo_random); + +#else /* SUMMARY DISABLED */ + +#define jffs2_sum_active() (0) +#define jffs2_sum_init(a) (0) +#define jffs2_sum_exit(a) do { } while (0) +#define jffs2_sum_disable_collecting(a) +#define jffs2_sum_is_disabled(a) (0) +#define jffs2_sum_reset_collected(a) do { } while (0) +#define jffs2_sum_add_kvec(a,b,c,d) (0) +#define jffs2_sum_move_collected(a,b) do { } while (0) +#define jffs2_sum_write_sumnode(a) (0) +#define jffs2_sum_add_padding_mem(a,b) do { } while (0) +#define jffs2_sum_add_inode_mem(a,b,c) do { } while (0) +#define jffs2_sum_add_dirent_mem(a,b,c) do { } while (0) +#define jffs2_sum_add_xattr_mem(a,b,c) do { } while (0) +#define jffs2_sum_add_xref_mem(a,b,c) do { } while (0) +#define jffs2_sum_scan_sumnode(a,b,c,d,e) (0) + +#endif /* CONFIG_JFFS2_SUMMARY */ + +#endif /* JFFS2_SUMMARY_H */ diff --git a/fs/jffs2/super.c b/fs/jffs2/super.c new file mode 100644 index 000000000..81396a092 --- /dev/null +++ b/fs/jffs2/super.c @@ -0,0 +1,443 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * + * Created by David Woodhouse + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "compr.h" +#include "nodelist.h" + +static void jffs2_put_super(struct super_block *); + +static struct kmem_cache *jffs2_inode_cachep; + +static struct inode *jffs2_alloc_inode(struct super_block *sb) +{ + struct jffs2_inode_info *f; + + f = alloc_inode_sb(sb, jffs2_inode_cachep, GFP_KERNEL); + if (!f) + return NULL; + return &f->vfs_inode; +} + +static void jffs2_free_inode(struct inode *inode) +{ + struct jffs2_inode_info *f = JFFS2_INODE_INFO(inode); + + kfree(f->target); + kmem_cache_free(jffs2_inode_cachep, f); +} + +static void jffs2_i_init_once(void *foo) +{ + struct jffs2_inode_info *f = foo; + + mutex_init(&f->sem); + f->target = NULL; + inode_init_once(&f->vfs_inode); +} + +static const char *jffs2_compr_name(unsigned int compr) +{ + switch (compr) { + case JFFS2_COMPR_MODE_NONE: + return "none"; +#ifdef CONFIG_JFFS2_LZO + case JFFS2_COMPR_MODE_FORCELZO: + return "lzo"; +#endif +#ifdef CONFIG_JFFS2_ZLIB + case JFFS2_COMPR_MODE_FORCEZLIB: + return "zlib"; +#endif + default: + /* should never happen; programmer error */ + WARN_ON(1); + return ""; + } +} + +static int jffs2_show_options(struct seq_file *s, struct dentry *root) +{ + struct jffs2_sb_info *c = JFFS2_SB_INFO(root->d_sb); + struct jffs2_mount_opts *opts = &c->mount_opts; + + if (opts->override_compr) + seq_printf(s, ",compr=%s", jffs2_compr_name(opts->compr)); + if (opts->set_rp_size) + seq_printf(s, ",rp_size=%u", opts->rp_size / 1024); + + return 0; +} + +static int jffs2_sync_fs(struct super_block *sb, int wait) +{ + struct jffs2_sb_info *c = JFFS2_SB_INFO(sb); + +#ifdef CONFIG_JFFS2_FS_WRITEBUFFER + if (jffs2_is_writebuffered(c)) + cancel_delayed_work_sync(&c->wbuf_dwork); +#endif + + mutex_lock(&c->alloc_sem); + jffs2_flush_wbuf_pad(c); + mutex_unlock(&c->alloc_sem); + return 0; +} + +static struct inode *jffs2_nfs_get_inode(struct super_block *sb, uint64_t ino, + uint32_t generation) +{ + /* We don't care about i_generation. We'll destroy the flash + before we start re-using inode numbers anyway. And even + if that wasn't true, we'd have other problems...*/ + return jffs2_iget(sb, ino); +} + +static struct dentry *jffs2_fh_to_dentry(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + return generic_fh_to_dentry(sb, fid, fh_len, fh_type, + jffs2_nfs_get_inode); +} + +static struct dentry *jffs2_fh_to_parent(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + return generic_fh_to_parent(sb, fid, fh_len, fh_type, + jffs2_nfs_get_inode); +} + +static struct dentry *jffs2_get_parent(struct dentry *child) +{ + struct jffs2_inode_info *f; + uint32_t pino; + + BUG_ON(!d_is_dir(child)); + + f = JFFS2_INODE_INFO(d_inode(child)); + + pino = f->inocache->pino_nlink; + + JFFS2_DEBUG("Parent of directory ino #%u is #%u\n", + f->inocache->ino, pino); + + return d_obtain_alias(jffs2_iget(child->d_sb, pino)); +} + +static const struct export_operations jffs2_export_ops = { + .encode_fh = generic_encode_ino32_fh, + .get_parent = jffs2_get_parent, + .fh_to_dentry = jffs2_fh_to_dentry, + .fh_to_parent = jffs2_fh_to_parent, +}; + +/* + * JFFS2 mount options. + * + * Opt_source: The source device + * Opt_override_compr: override default compressor + * Opt_rp_size: size of reserved pool in KiB + */ +enum { + Opt_override_compr, + Opt_rp_size, +}; + +static const struct constant_table jffs2_param_compr[] = { + {"none", JFFS2_COMPR_MODE_NONE }, +#ifdef CONFIG_JFFS2_LZO + {"lzo", JFFS2_COMPR_MODE_FORCELZO }, +#endif +#ifdef CONFIG_JFFS2_ZLIB + {"zlib", JFFS2_COMPR_MODE_FORCEZLIB }, +#endif + {} +}; + +static const struct fs_parameter_spec jffs2_fs_parameters[] = { + fsparam_enum ("compr", Opt_override_compr, jffs2_param_compr), + fsparam_u32 ("rp_size", Opt_rp_size), + {} +}; + +static int jffs2_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct fs_parse_result result; + struct jffs2_sb_info *c = fc->s_fs_info; + int opt; + + opt = fs_parse(fc, jffs2_fs_parameters, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_override_compr: + c->mount_opts.compr = result.uint_32; + c->mount_opts.override_compr = true; + break; + case Opt_rp_size: + if (result.uint_32 > UINT_MAX / 1024) + return invalf(fc, "jffs2: rp_size unrepresentable"); + c->mount_opts.rp_size = result.uint_32 * 1024; + c->mount_opts.set_rp_size = true; + break; + default: + return -EINVAL; + } + + return 0; +} + +static inline void jffs2_update_mount_opts(struct fs_context *fc) +{ + struct jffs2_sb_info *new_c = fc->s_fs_info; + struct jffs2_sb_info *c = JFFS2_SB_INFO(fc->root->d_sb); + + mutex_lock(&c->alloc_sem); + if (new_c->mount_opts.override_compr) { + c->mount_opts.override_compr = new_c->mount_opts.override_compr; + c->mount_opts.compr = new_c->mount_opts.compr; + } + if (new_c->mount_opts.set_rp_size) { + c->mount_opts.set_rp_size = new_c->mount_opts.set_rp_size; + c->mount_opts.rp_size = new_c->mount_opts.rp_size; + } + mutex_unlock(&c->alloc_sem); +} + +static int jffs2_reconfigure(struct fs_context *fc) +{ + struct super_block *sb = fc->root->d_sb; + + sync_filesystem(sb); + jffs2_update_mount_opts(fc); + + return jffs2_do_remount_fs(sb, fc); +} + +static const struct super_operations jffs2_super_operations = +{ + .alloc_inode = jffs2_alloc_inode, + .free_inode = jffs2_free_inode, + .put_super = jffs2_put_super, + .statfs = jffs2_statfs, + .evict_inode = jffs2_evict_inode, + .dirty_inode = jffs2_dirty_inode, + .show_options = jffs2_show_options, + .sync_fs = jffs2_sync_fs, +}; + +/* + * fill in the superblock + */ +static int jffs2_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct jffs2_sb_info *c = sb->s_fs_info; + + jffs2_dbg(1, "jffs2_get_sb_mtd():" + " New superblock for device %d (\"%s\")\n", + sb->s_mtd->index, sb->s_mtd->name); + + c->mtd = sb->s_mtd; + c->os_priv = sb; + + if (c->mount_opts.rp_size > c->mtd->size) + return invalf(fc, "jffs2: Too large reserve pool specified, max is %llu KB", + c->mtd->size / 1024); + + /* Initialize JFFS2 superblock locks, the further initialization will + * be done later */ + mutex_init(&c->alloc_sem); + mutex_init(&c->erase_free_sem); + init_waitqueue_head(&c->erase_wait); + init_waitqueue_head(&c->inocache_wq); + spin_lock_init(&c->erase_completion_lock); + spin_lock_init(&c->inocache_lock); + + sb->s_op = &jffs2_super_operations; + sb->s_export_op = &jffs2_export_ops; + sb->s_flags = sb->s_flags | SB_NOATIME; + sb->s_xattr = jffs2_xattr_handlers; +#ifdef CONFIG_JFFS2_FS_POSIX_ACL + sb->s_flags |= SB_POSIXACL; +#endif + return jffs2_do_fill_super(sb, fc); +} + +static int jffs2_get_tree(struct fs_context *fc) +{ + return get_tree_mtd(fc, jffs2_fill_super); +} + +static void jffs2_free_fc(struct fs_context *fc) +{ + kfree(fc->s_fs_info); +} + +static const struct fs_context_operations jffs2_context_ops = { + .free = jffs2_free_fc, + .parse_param = jffs2_parse_param, + .get_tree = jffs2_get_tree, + .reconfigure = jffs2_reconfigure, +}; + +static int jffs2_init_fs_context(struct fs_context *fc) +{ + struct jffs2_sb_info *ctx; + + ctx = kzalloc_obj(struct jffs2_sb_info); + if (!ctx) + return -ENOMEM; + + fc->s_fs_info = ctx; + fc->ops = &jffs2_context_ops; + return 0; +} + +static void jffs2_put_super (struct super_block *sb) +{ + struct jffs2_sb_info *c = JFFS2_SB_INFO(sb); + + jffs2_dbg(2, "%s()\n", __func__); + + mutex_lock(&c->alloc_sem); + jffs2_flush_wbuf_pad(c); + mutex_unlock(&c->alloc_sem); + + jffs2_sum_exit(c); + + jffs2_free_ino_caches(c); + jffs2_free_raw_node_refs(c); + kvfree(c->blocks); + jffs2_flash_cleanup(c); + kfree(c->inocache_list); + jffs2_clear_xattr_subsystem(c); + mtd_sync(c->mtd); + jffs2_dbg(1, "%s(): returning\n", __func__); +} + +static void jffs2_kill_sb(struct super_block *sb) +{ + struct jffs2_sb_info *c = JFFS2_SB_INFO(sb); + if (c && !sb_rdonly(sb)) + jffs2_stop_garbage_collect_thread(c); + kill_mtd_super(sb); + kfree(c); +} + +static struct file_system_type jffs2_fs_type = { + .owner = THIS_MODULE, + .name = "jffs2", + .init_fs_context = jffs2_init_fs_context, + .parameters = jffs2_fs_parameters, + .kill_sb = jffs2_kill_sb, +}; +MODULE_ALIAS_FS("jffs2"); + +static int __init init_jffs2_fs(void) +{ + int ret; + + /* Paranoia checks for on-medium structures. If we ask GCC + to pack them with __attribute__((packed)) then it _also_ + assumes that they're not aligned -- so it emits crappy + code on some architectures. Ideally we want an attribute + which means just 'no padding', without the alignment + thing. But GCC doesn't have that -- we have to just + hope the structs are the right sizes, instead. */ + BUILD_BUG_ON(sizeof(struct jffs2_unknown_node) != 12); + BUILD_BUG_ON(sizeof(struct jffs2_raw_dirent) != 40); + BUILD_BUG_ON(sizeof(struct jffs2_raw_inode) != 68); + BUILD_BUG_ON(sizeof(struct jffs2_raw_summary) != 32); + + pr_info("version 2.2." +#ifdef CONFIG_JFFS2_FS_WRITEBUFFER + " (NAND)" +#endif +#ifdef CONFIG_JFFS2_SUMMARY + " (SUMMARY) " +#endif + " © 2001-2006 Red Hat, Inc.\n"); + + jffs2_inode_cachep = kmem_cache_create("jffs2_i", + sizeof(struct jffs2_inode_info), + 0, (SLAB_RECLAIM_ACCOUNT| + SLAB_ACCOUNT), + jffs2_i_init_once); + if (!jffs2_inode_cachep) { + pr_err("error: Failed to initialise inode cache\n"); + return -ENOMEM; + } + ret = jffs2_compressors_init(); + if (ret) { + pr_err("error: Failed to initialise compressors\n"); + goto out; + } + ret = jffs2_create_slab_caches(); + if (ret) { + pr_err("error: Failed to initialise slab caches\n"); + goto out_compressors; + } + ret = register_filesystem(&jffs2_fs_type); + if (ret) { + pr_err("error: Failed to register filesystem\n"); + goto out_slab; + } + return 0; + + out_slab: + jffs2_destroy_slab_caches(); + out_compressors: + jffs2_compressors_exit(); + out: + kmem_cache_destroy(jffs2_inode_cachep); + return ret; +} + +static void __exit exit_jffs2_fs(void) +{ + unregister_filesystem(&jffs2_fs_type); + jffs2_destroy_slab_caches(); + jffs2_compressors_exit(); + + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(jffs2_inode_cachep); +} + +module_init(init_jffs2_fs); +module_exit(exit_jffs2_fs); + +MODULE_DESCRIPTION("The Journalling Flash File System, v2"); +MODULE_AUTHOR("Red Hat, Inc."); +MODULE_LICENSE("GPL"); // Actually dual-licensed, but it doesn't matter for + // the sake of this tag. It's Free Software. diff --git a/fs/jffs2/symlink.c b/fs/jffs2/symlink.c new file mode 100644 index 000000000..d2fa138a8 --- /dev/null +++ b/fs/jffs2/symlink.c @@ -0,0 +1,19 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * + * Created by David Woodhouse + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#include "nodelist.h" + +const struct inode_operations jffs2_symlink_inode_operations = +{ + .get_link = simple_get_link, + .setattr = jffs2_setattr, + .listxattr = jffs2_listxattr, +}; diff --git a/fs/jffs2/wbuf.c b/fs/jffs2/wbuf.c new file mode 100644 index 000000000..3b7803c75 --- /dev/null +++ b/fs/jffs2/wbuf.c @@ -0,0 +1,1350 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * Copyright © 2004 Thomas Gleixner + * + * Created by David Woodhouse + * Modified debugged and enhanced by Thomas Gleixner + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nodelist.h" + +/* For testing write failures */ +#undef BREAKME +#undef BREAKMEHEADER + +#ifdef BREAKME +static unsigned char *brokenbuf; +#endif + +#define PAGE_DIV(x) ( ((unsigned long)(x) / (unsigned long)(c->wbuf_pagesize)) * (unsigned long)(c->wbuf_pagesize) ) +#define PAGE_MOD(x) ( (unsigned long)(x) % (unsigned long)(c->wbuf_pagesize) ) + +/* max. erase failures before we mark a block bad */ +#define MAX_ERASE_FAILURES 2 + +struct jffs2_inodirty { + uint32_t ino; + struct jffs2_inodirty *next; +}; + +static struct jffs2_inodirty inodirty_nomem; + +static int jffs2_wbuf_pending_for_ino(struct jffs2_sb_info *c, uint32_t ino) +{ + struct jffs2_inodirty *this = c->wbuf_inodes; + + /* If a malloc failed, consider _everything_ dirty */ + if (this == &inodirty_nomem) + return 1; + + /* If ino == 0, _any_ non-GC writes mean 'yes' */ + if (this && !ino) + return 1; + + /* Look to see if the inode in question is pending in the wbuf */ + while (this) { + if (this->ino == ino) + return 1; + this = this->next; + } + return 0; +} + +static void jffs2_clear_wbuf_ino_list(struct jffs2_sb_info *c) +{ + struct jffs2_inodirty *this; + + this = c->wbuf_inodes; + + if (this != &inodirty_nomem) { + while (this) { + struct jffs2_inodirty *next = this->next; + kfree(this); + this = next; + } + } + c->wbuf_inodes = NULL; +} + +static void jffs2_wbuf_dirties_inode(struct jffs2_sb_info *c, uint32_t ino) +{ + struct jffs2_inodirty *new; + + /* Schedule delayed write-buffer write-out */ + jffs2_dirty_trigger(c); + + if (jffs2_wbuf_pending_for_ino(c, ino)) + return; + + new = kmalloc_obj(*new); + if (!new) { + jffs2_dbg(1, "No memory to allocate inodirty. Fallback to all considered dirty\n"); + jffs2_clear_wbuf_ino_list(c); + c->wbuf_inodes = &inodirty_nomem; + return; + } + new->ino = ino; + new->next = c->wbuf_inodes; + c->wbuf_inodes = new; + return; +} + +static inline void jffs2_refile_wbuf_blocks(struct jffs2_sb_info *c) +{ + struct list_head *this, *next; + static int n; + + if (list_empty(&c->erasable_pending_wbuf_list)) + return; + + list_for_each_safe(this, next, &c->erasable_pending_wbuf_list) { + struct jffs2_eraseblock *jeb = list_entry(this, struct jffs2_eraseblock, list); + + jffs2_dbg(1, "Removing eraseblock at 0x%08x from erasable_pending_wbuf_list...\n", + jeb->offset); + list_del(this); + if ((jiffies + (n++)) & 127) { + /* Most of the time, we just erase it immediately. Otherwise we + spend ages scanning it on mount, etc. */ + jffs2_dbg(1, "...and adding to erase_pending_list\n"); + list_add_tail(&jeb->list, &c->erase_pending_list); + c->nr_erasing_blocks++; + jffs2_garbage_collect_trigger(c); + } else { + /* Sometimes, however, we leave it elsewhere so it doesn't get + immediately reused, and we spread the load a bit. */ + jffs2_dbg(1, "...and adding to erasable_list\n"); + list_add_tail(&jeb->list, &c->erasable_list); + } + } +} + +#define REFILE_NOTEMPTY 0 +#define REFILE_ANYWAY 1 + +static void jffs2_block_refile(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, int allow_empty) +{ + jffs2_dbg(1, "About to refile bad block at %08x\n", jeb->offset); + + /* File the existing block on the bad_used_list.... */ + if (c->nextblock == jeb) + c->nextblock = NULL; + else /* Not sure this should ever happen... need more coffee */ + list_del(&jeb->list); + if (jeb->first_node) { + jffs2_dbg(1, "Refiling block at %08x to bad_used_list\n", + jeb->offset); + list_add(&jeb->list, &c->bad_used_list); + } else { + BUG_ON(allow_empty == REFILE_NOTEMPTY); + /* It has to have had some nodes or we couldn't be here */ + jffs2_dbg(1, "Refiling block at %08x to erase_pending_list\n", + jeb->offset); + list_add(&jeb->list, &c->erase_pending_list); + c->nr_erasing_blocks++; + jffs2_garbage_collect_trigger(c); + } + + if (!jffs2_prealloc_raw_node_refs(c, jeb, 1)) { + uint32_t oldfree = jeb->free_size; + + jffs2_link_node_ref(c, jeb, + (jeb->offset+c->sector_size-oldfree) | REF_OBSOLETE, + oldfree, NULL); + /* convert to wasted */ + c->wasted_size += oldfree; + jeb->wasted_size += oldfree; + c->dirty_size -= oldfree; + jeb->dirty_size -= oldfree; + } + + jffs2_dbg_dump_block_lists_nolock(c); + jffs2_dbg_acct_sanity_check_nolock(c,jeb); + jffs2_dbg_acct_paranoia_check_nolock(c, jeb); +} + +static struct jffs2_raw_node_ref **jffs2_incore_replace_raw(struct jffs2_sb_info *c, + struct jffs2_inode_info *f, + struct jffs2_raw_node_ref *raw, + union jffs2_node_union *node) +{ + struct jffs2_node_frag *frag; + struct jffs2_full_dirent *fd; + + dbg_noderef("incore_replace_raw: node at %p is {%04x,%04x}\n", + node, je16_to_cpu(node->u.magic), je16_to_cpu(node->u.nodetype)); + + BUG_ON(je16_to_cpu(node->u.magic) != 0x1985 && + je16_to_cpu(node->u.magic) != 0); + + switch (je16_to_cpu(node->u.nodetype)) { + case JFFS2_NODETYPE_INODE: + if (f->metadata && f->metadata->raw == raw) { + dbg_noderef("Will replace ->raw in f->metadata at %p\n", f->metadata); + return &f->metadata->raw; + } + frag = jffs2_lookup_node_frag(&f->fragtree, je32_to_cpu(node->i.offset)); + BUG_ON(!frag); + /* Find a frag which refers to the full_dnode we want to modify */ + while (!frag->node || frag->node->raw != raw) { + frag = frag_next(frag); + BUG_ON(!frag); + } + dbg_noderef("Will replace ->raw in full_dnode at %p\n", frag->node); + return &frag->node->raw; + + case JFFS2_NODETYPE_DIRENT: + for (fd = f->dents; fd; fd = fd->next) { + if (fd->raw == raw) { + dbg_noderef("Will replace ->raw in full_dirent at %p\n", fd); + return &fd->raw; + } + } + BUG(); + + default: + dbg_noderef("Don't care about replacing raw for nodetype %x\n", + je16_to_cpu(node->u.nodetype)); + break; + } + return NULL; +} + +#ifdef CONFIG_JFFS2_FS_WBUF_VERIFY +static int jffs2_verify_write(struct jffs2_sb_info *c, unsigned char *buf, + uint32_t ofs) +{ + int ret; + size_t retlen; + char *eccstr; + + ret = mtd_read(c->mtd, ofs, c->wbuf_pagesize, &retlen, c->wbuf_verify); + if (ret && ret != -EUCLEAN && ret != -EBADMSG) { + pr_warn("%s(): Read back of page at %08x failed: %d\n", + __func__, c->wbuf_ofs, ret); + return ret; + } else if (retlen != c->wbuf_pagesize) { + pr_warn("%s(): Read back of page at %08x gave short read: %zd not %d\n", + __func__, ofs, retlen, c->wbuf_pagesize); + return -EIO; + } + if (!memcmp(buf, c->wbuf_verify, c->wbuf_pagesize)) + return 0; + + if (ret == -EUCLEAN) + eccstr = "corrected"; + else if (ret == -EBADMSG) + eccstr = "correction failed"; + else + eccstr = "OK or unused"; + + pr_warn("Write verify error (ECC %s) at %08x. Wrote:\n", + eccstr, c->wbuf_ofs); + print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, 16, 1, + c->wbuf, c->wbuf_pagesize, 0); + + pr_warn("Read back:\n"); + print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, 16, 1, + c->wbuf_verify, c->wbuf_pagesize, 0); + + return -EIO; +} +#else +#define jffs2_verify_write(c,b,o) (0) +#endif + +/* Recover from failure to write wbuf. Recover the nodes up to the + * wbuf, not the one which we were starting to try to write. */ + +static void jffs2_wbuf_recover(struct jffs2_sb_info *c) +{ + struct jffs2_eraseblock *jeb, *new_jeb; + struct jffs2_raw_node_ref *raw, *next, *first_raw = NULL; + size_t retlen; + int ret; + int nr_refile = 0; + unsigned char *buf; + uint32_t start, end, ofs, len; + + jeb = &c->blocks[c->wbuf_ofs / c->sector_size]; + + spin_lock(&c->erase_completion_lock); + if (c->wbuf_ofs % c->mtd->erasesize) + jffs2_block_refile(c, jeb, REFILE_NOTEMPTY); + else + jffs2_block_refile(c, jeb, REFILE_ANYWAY); + spin_unlock(&c->erase_completion_lock); + + BUG_ON(!ref_obsolete(jeb->last_node)); + + /* Find the first node to be recovered, by skipping over every + node which ends before the wbuf starts, or which is obsolete. */ + for (next = raw = jeb->first_node; next; raw = next) { + next = ref_next(raw); + + if (ref_obsolete(raw) || + (next && ref_offset(next) <= c->wbuf_ofs)) { + dbg_noderef("Skipping node at 0x%08x(%d)-0x%08x which is either before 0x%08x or obsolete\n", + ref_offset(raw), ref_flags(raw), + (ref_offset(raw) + ref_totlen(c, jeb, raw)), + c->wbuf_ofs); + continue; + } + dbg_noderef("First node to be recovered is at 0x%08x(%d)-0x%08x\n", + ref_offset(raw), ref_flags(raw), + (ref_offset(raw) + ref_totlen(c, jeb, raw))); + + first_raw = raw; + break; + } + + if (!first_raw) { + /* All nodes were obsolete. Nothing to recover. */ + jffs2_dbg(1, "No non-obsolete nodes to be recovered. Just filing block bad\n"); + c->wbuf_len = 0; + return; + } + + start = ref_offset(first_raw); + end = ref_offset(jeb->last_node); + nr_refile = 1; + + /* Count the number of refs which need to be copied */ + while ((raw = ref_next(raw)) != jeb->last_node) + nr_refile++; + + dbg_noderef("wbuf recover %08x-%08x (%d bytes in %d nodes)\n", + start, end, end - start, nr_refile); + + buf = NULL; + if (start < c->wbuf_ofs) { + /* First affected node was already partially written. + * Attempt to reread the old data into our buffer. */ + + buf = kmalloc(end - start, GFP_KERNEL); + if (!buf) { + pr_crit("Malloc failure in wbuf recovery. Data loss ensues.\n"); + + goto read_failed; + } + + /* Do the read... */ + ret = mtd_read(c->mtd, start, c->wbuf_ofs - start, &retlen, + buf); + + /* ECC recovered ? */ + if ((ret == -EUCLEAN || ret == -EBADMSG) && + (retlen == c->wbuf_ofs - start)) + ret = 0; + + if (ret || retlen != c->wbuf_ofs - start) { + pr_crit("Old data are already lost in wbuf recovery. Data loss ensues.\n"); + + kfree(buf); + buf = NULL; + read_failed: + first_raw = ref_next(first_raw); + nr_refile--; + while (first_raw && ref_obsolete(first_raw)) { + first_raw = ref_next(first_raw); + nr_refile--; + } + + /* If this was the only node to be recovered, give up */ + if (!first_raw) { + c->wbuf_len = 0; + return; + } + + /* It wasn't. Go on and try to recover nodes complete in the wbuf */ + start = ref_offset(first_raw); + dbg_noderef("wbuf now recover %08x-%08x (%d bytes in %d nodes)\n", + start, end, end - start, nr_refile); + + } else { + /* Read succeeded. Copy the remaining data from the wbuf */ + memcpy(buf + (c->wbuf_ofs - start), c->wbuf, end - c->wbuf_ofs); + } + } + /* OK... we're to rewrite (end-start) bytes of data from first_raw onwards. + Either 'buf' contains the data, or we find it in the wbuf */ + + /* ... and get an allocation of space from a shiny new block instead */ + ret = jffs2_reserve_space_gc(c, end-start, &len, JFFS2_SUMMARY_NOSUM_SIZE); + if (ret) { + pr_warn("Failed to allocate space for wbuf recovery. Data loss ensues.\n"); + kfree(buf); + return; + } + + /* The summary is not recovered, so it must be disabled for this erase block */ + jffs2_sum_disable_collecting(c->summary); + + ret = jffs2_prealloc_raw_node_refs(c, c->nextblock, nr_refile); + if (ret) { + pr_warn("Failed to allocate node refs for wbuf recovery. Data loss ensues.\n"); + kfree(buf); + return; + } + + ofs = write_ofs(c); + + if (end-start >= c->wbuf_pagesize) { + /* Need to do another write immediately, but it's possible + that this is just because the wbuf itself is completely + full, and there's nothing earlier read back from the + flash. Hence 'buf' isn't necessarily what we're writing + from. */ + unsigned char *rewrite_buf = buf?:c->wbuf; + uint32_t towrite = (end-start) - ((end-start)%c->wbuf_pagesize); + + jffs2_dbg(1, "Write 0x%x bytes at 0x%08x in wbuf recover\n", + towrite, ofs); + +#ifdef BREAKMEHEADER + static int breakme; + if (breakme++ == 20) { + pr_notice("Faking write error at 0x%08x\n", ofs); + breakme = 0; + mtd_write(c->mtd, ofs, towrite, &retlen, brokenbuf); + ret = -EIO; + } else +#endif + ret = mtd_write(c->mtd, ofs, towrite, &retlen, + rewrite_buf); + + if (ret || retlen != towrite || jffs2_verify_write(c, rewrite_buf, ofs)) { + /* Argh. We tried. Really we did. */ + pr_crit("Recovery of wbuf failed due to a second write error\n"); + kfree(buf); + + if (retlen) + jffs2_add_physical_node_ref(c, ofs | REF_OBSOLETE, ref_totlen(c, jeb, first_raw), NULL); + + return; + } + pr_notice("Recovery of wbuf succeeded to %08x\n", ofs); + + c->wbuf_len = (end - start) - towrite; + c->wbuf_ofs = ofs + towrite; + memmove(c->wbuf, rewrite_buf + towrite, c->wbuf_len); + /* Don't muck about with c->wbuf_inodes. False positives are harmless. */ + } else { + /* OK, now we're left with the dregs in whichever buffer we're using */ + if (buf) { + memcpy(c->wbuf, buf, end-start); + } else { + memmove(c->wbuf, c->wbuf + (start - c->wbuf_ofs), end - start); + } + c->wbuf_ofs = ofs; + c->wbuf_len = end - start; + } + + /* Now sort out the jffs2_raw_node_refs, moving them from the old to the next block */ + new_jeb = &c->blocks[ofs / c->sector_size]; + + spin_lock(&c->erase_completion_lock); + for (raw = first_raw; raw != jeb->last_node; raw = ref_next(raw)) { + uint32_t rawlen = ref_totlen(c, jeb, raw); + struct jffs2_inode_cache *ic; + struct jffs2_raw_node_ref *new_ref; + struct jffs2_raw_node_ref **adjust_ref = NULL; + struct jffs2_inode_info *f = NULL; + + jffs2_dbg(1, "Refiling block of %08x at %08x(%d) to %08x\n", + rawlen, ref_offset(raw), ref_flags(raw), ofs); + + ic = jffs2_raw_ref_to_ic(raw); + + /* Ick. This XATTR mess should be fixed shortly... */ + if (ic && ic->class == RAWNODE_CLASS_XATTR_DATUM) { + struct jffs2_xattr_datum *xd = (void *)ic; + BUG_ON(xd->node != raw); + adjust_ref = &xd->node; + raw->next_in_ino = NULL; + ic = NULL; + } else if (ic && ic->class == RAWNODE_CLASS_XATTR_REF) { + struct jffs2_xattr_datum *xr = (void *)ic; + BUG_ON(xr->node != raw); + adjust_ref = &xr->node; + raw->next_in_ino = NULL; + ic = NULL; + } else if (ic && ic->class == RAWNODE_CLASS_INODE_CACHE) { + struct jffs2_raw_node_ref **p = &ic->nodes; + + /* Remove the old node from the per-inode list */ + while (*p && *p != (void *)ic) { + if (*p == raw) { + (*p) = (raw->next_in_ino); + raw->next_in_ino = NULL; + break; + } + p = &((*p)->next_in_ino); + } + + if (ic->state == INO_STATE_PRESENT && !ref_obsolete(raw)) { + /* If it's an in-core inode, then we have to adjust any + full_dirent or full_dnode structure to point to the + new version instead of the old */ + f = jffs2_gc_fetch_inode(c, ic->ino, !ic->pino_nlink); + if (IS_ERR(f)) { + /* Should never happen; it _must_ be present */ + JFFS2_ERROR("Failed to iget() ino #%u, err %ld\n", + ic->ino, PTR_ERR(f)); + BUG(); + } + /* We don't lock f->sem. There's a number of ways we could + end up in here with it already being locked, and nobody's + going to modify it on us anyway because we hold the + alloc_sem. We're only changing one ->raw pointer too, + which we can get away with without upsetting readers. */ + adjust_ref = jffs2_incore_replace_raw(c, f, raw, + (void *)(buf?:c->wbuf) + (ref_offset(raw) - start)); + } else if (unlikely(ic->state != INO_STATE_PRESENT && + ic->state != INO_STATE_CHECKEDABSENT && + ic->state != INO_STATE_GC)) { + JFFS2_ERROR("Inode #%u is in strange state %d!\n", ic->ino, ic->state); + BUG(); + } + } + + new_ref = jffs2_link_node_ref(c, new_jeb, ofs | ref_flags(raw), rawlen, ic); + + if (adjust_ref) { + BUG_ON(*adjust_ref != raw); + *adjust_ref = new_ref; + } + if (f) + jffs2_gc_release_inode(c, f); + + if (!ref_obsolete(raw)) { + jeb->dirty_size += rawlen; + jeb->used_size -= rawlen; + c->dirty_size += rawlen; + c->used_size -= rawlen; + raw->flash_offset = ref_offset(raw) | REF_OBSOLETE; + BUG_ON(raw->next_in_ino); + } + ofs += rawlen; + } + + kfree(buf); + + /* Fix up the original jeb now it's on the bad_list */ + if (first_raw == jeb->first_node) { + jffs2_dbg(1, "Failing block at %08x is now empty. Moving to erase_pending_list\n", + jeb->offset); + list_move(&jeb->list, &c->erase_pending_list); + c->nr_erasing_blocks++; + jffs2_garbage_collect_trigger(c); + } + + jffs2_dbg_acct_sanity_check_nolock(c, jeb); + jffs2_dbg_acct_paranoia_check_nolock(c, jeb); + + jffs2_dbg_acct_sanity_check_nolock(c, new_jeb); + jffs2_dbg_acct_paranoia_check_nolock(c, new_jeb); + + spin_unlock(&c->erase_completion_lock); + + jffs2_dbg(1, "wbuf recovery completed OK. wbuf_ofs 0x%08x, len 0x%x\n", + c->wbuf_ofs, c->wbuf_len); + +} + +/* Meaning of pad argument: + 0: Do not pad. Probably pointless - we only ever use this when we can't pad anyway. + 1: Pad, do not adjust nextblock free_size + 2: Pad, adjust nextblock free_size +*/ +#define NOPAD 0 +#define PAD_NOACCOUNT 1 +#define PAD_ACCOUNTING 2 + +static int __jffs2_flush_wbuf(struct jffs2_sb_info *c, int pad) +{ + struct jffs2_eraseblock *wbuf_jeb; + int ret; + size_t retlen; + + /* Nothing to do if not write-buffering the flash. In particular, we shouldn't + call timer_delete() on the timer we never initialised. */ + if (!jffs2_is_writebuffered(c)) + return 0; + + if (!mutex_is_locked(&c->alloc_sem)) { + pr_crit("jffs2_flush_wbuf() called with alloc_sem not locked!\n"); + BUG(); + } + + if (!c->wbuf_len) /* already checked c->wbuf above */ + return 0; + + wbuf_jeb = &c->blocks[c->wbuf_ofs / c->sector_size]; + if (jffs2_prealloc_raw_node_refs(c, wbuf_jeb, c->nextblock->allocated_refs + 1)) + return -ENOMEM; + + /* claim remaining space on the page + this happens, if we have a change to a new block, + or if fsync forces us to flush the writebuffer. + if we have a switch to next page, we will not have + enough remaining space for this. + */ + if (pad ) { + c->wbuf_len = PAD(c->wbuf_len); + + /* Pad with JFFS2_DIRTY_BITMASK initially. this helps out ECC'd NOR + with 8 byte page size */ + memset(c->wbuf + c->wbuf_len, 0, c->wbuf_pagesize - c->wbuf_len); + + if ( c->wbuf_len + sizeof(struct jffs2_unknown_node) < c->wbuf_pagesize) { + struct jffs2_unknown_node *padnode = (void *)(c->wbuf + c->wbuf_len); + padnode->magic = cpu_to_je16(JFFS2_MAGIC_BITMASK); + padnode->nodetype = cpu_to_je16(JFFS2_NODETYPE_PADDING); + padnode->totlen = cpu_to_je32(c->wbuf_pagesize - c->wbuf_len); + padnode->hdr_crc = cpu_to_je32(crc32(0, padnode, sizeof(*padnode)-4)); + } + } + /* else jffs2_flash_writev has actually filled in the rest of the + buffer for us, and will deal with the node refs etc. later. */ + +#ifdef BREAKME + static int breakme; + if (breakme++ == 20) { + pr_notice("Faking write error at 0x%08x\n", c->wbuf_ofs); + breakme = 0; + mtd_write(c->mtd, c->wbuf_ofs, c->wbuf_pagesize, &retlen, + brokenbuf); + ret = -EIO; + } else +#endif + + ret = mtd_write(c->mtd, c->wbuf_ofs, c->wbuf_pagesize, + &retlen, c->wbuf); + + if (ret) { + pr_warn("jffs2_flush_wbuf(): Write failed with %d\n", ret); + goto wfail; + } else if (retlen != c->wbuf_pagesize) { + pr_warn("jffs2_flush_wbuf(): Write was short: %zd instead of %d\n", + retlen, c->wbuf_pagesize); + ret = -EIO; + goto wfail; + } else if ((ret = jffs2_verify_write(c, c->wbuf, c->wbuf_ofs))) { + wfail: + jffs2_wbuf_recover(c); + + return ret; + } + + /* Adjust free size of the block if we padded. */ + if (pad) { + uint32_t waste = c->wbuf_pagesize - c->wbuf_len; + + jffs2_dbg(1, "jffs2_flush_wbuf() adjusting free_size of %sblock at %08x\n", + (wbuf_jeb == c->nextblock) ? "next" : "", + wbuf_jeb->offset); + + /* wbuf_pagesize - wbuf_len is the amount of space that's to be + padded. If there is less free space in the block than that, + something screwed up */ + if (wbuf_jeb->free_size < waste) { + pr_crit("jffs2_flush_wbuf(): Accounting error. wbuf at 0x%08x has 0x%03x bytes, 0x%03x left.\n", + c->wbuf_ofs, c->wbuf_len, waste); + pr_crit("jffs2_flush_wbuf(): But free_size for block at 0x%08x is only 0x%08x\n", + wbuf_jeb->offset, wbuf_jeb->free_size); + BUG(); + } + + spin_lock(&c->erase_completion_lock); + + jffs2_link_node_ref(c, wbuf_jeb, (c->wbuf_ofs + c->wbuf_len) | REF_OBSOLETE, waste, NULL); + /* FIXME: that made it count as dirty. Convert to wasted */ + wbuf_jeb->dirty_size -= waste; + c->dirty_size -= waste; + wbuf_jeb->wasted_size += waste; + c->wasted_size += waste; + } else + spin_lock(&c->erase_completion_lock); + + /* Stick any now-obsoleted blocks on the erase_pending_list */ + jffs2_refile_wbuf_blocks(c); + jffs2_clear_wbuf_ino_list(c); + spin_unlock(&c->erase_completion_lock); + + memset(c->wbuf,0xff,c->wbuf_pagesize); + /* adjust write buffer offset, else we get a non contiguous write bug */ + c->wbuf_ofs += c->wbuf_pagesize; + c->wbuf_len = 0; + return 0; +} + +/* Trigger garbage collection to flush the write-buffer. + If ino arg is zero, do it if _any_ real (i.e. not GC) writes are + outstanding. If ino arg non-zero, do it only if a write for the + given inode is outstanding. */ +int jffs2_flush_wbuf_gc(struct jffs2_sb_info *c, uint32_t ino) +{ + uint32_t old_wbuf_ofs; + uint32_t old_wbuf_len; + int ret = 0; + + jffs2_dbg(1, "jffs2_flush_wbuf_gc() called for ino #%u...\n", ino); + + if (!c->wbuf) + return 0; + + mutex_lock(&c->alloc_sem); + if (!jffs2_wbuf_pending_for_ino(c, ino)) { + jffs2_dbg(1, "Ino #%d not pending in wbuf. Returning\n", ino); + mutex_unlock(&c->alloc_sem); + return 0; + } + + old_wbuf_ofs = c->wbuf_ofs; + old_wbuf_len = c->wbuf_len; + + if (c->unchecked_size) { + /* GC won't make any progress for a while */ + jffs2_dbg(1, "%s(): padding. Not finished checking\n", + __func__); + down_write(&c->wbuf_sem); + ret = __jffs2_flush_wbuf(c, PAD_ACCOUNTING); + /* retry flushing wbuf in case jffs2_wbuf_recover + left some data in the wbuf */ + if (ret) + ret = __jffs2_flush_wbuf(c, PAD_ACCOUNTING); + up_write(&c->wbuf_sem); + } else while (old_wbuf_len && + old_wbuf_ofs == c->wbuf_ofs) { + + mutex_unlock(&c->alloc_sem); + + jffs2_dbg(1, "%s(): calls gc pass\n", __func__); + + ret = jffs2_garbage_collect_pass(c); + if (ret) { + /* GC failed. Flush it with padding instead */ + mutex_lock(&c->alloc_sem); + down_write(&c->wbuf_sem); + ret = __jffs2_flush_wbuf(c, PAD_ACCOUNTING); + /* retry flushing wbuf in case jffs2_wbuf_recover + left some data in the wbuf */ + if (ret) + ret = __jffs2_flush_wbuf(c, PAD_ACCOUNTING); + up_write(&c->wbuf_sem); + break; + } + mutex_lock(&c->alloc_sem); + } + + jffs2_dbg(1, "%s(): ends...\n", __func__); + + mutex_unlock(&c->alloc_sem); + return ret; +} + +/* Pad write-buffer to end and write it, wasting space. */ +int jffs2_flush_wbuf_pad(struct jffs2_sb_info *c) +{ + int ret; + + if (!c->wbuf) + return 0; + + down_write(&c->wbuf_sem); + ret = __jffs2_flush_wbuf(c, PAD_NOACCOUNT); + /* retry - maybe wbuf recover left some data in wbuf. */ + if (ret) + ret = __jffs2_flush_wbuf(c, PAD_NOACCOUNT); + up_write(&c->wbuf_sem); + + return ret; +} + +static size_t jffs2_fill_wbuf(struct jffs2_sb_info *c, const uint8_t *buf, + size_t len) +{ + if (len && !c->wbuf_len && (len >= c->wbuf_pagesize)) + return 0; + + if (len > (c->wbuf_pagesize - c->wbuf_len)) + len = c->wbuf_pagesize - c->wbuf_len; + memcpy(c->wbuf + c->wbuf_len, buf, len); + c->wbuf_len += (uint32_t) len; + return len; +} + +int jffs2_flash_writev(struct jffs2_sb_info *c, const struct kvec *invecs, + unsigned long count, loff_t to, size_t *retlen, + uint32_t ino) +{ + struct jffs2_eraseblock *jeb; + size_t wbuf_retlen, donelen = 0; + uint32_t outvec_to = to; + int ret, invec; + + /* If not writebuffered flash, don't bother */ + if (!jffs2_is_writebuffered(c)) + return jffs2_flash_direct_writev(c, invecs, count, to, retlen); + + down_write(&c->wbuf_sem); + + /* If wbuf_ofs is not initialized, set it to target address */ + if (c->wbuf_ofs == 0xFFFFFFFF) { + c->wbuf_ofs = PAGE_DIV(to); + c->wbuf_len = PAGE_MOD(to); + memset(c->wbuf,0xff,c->wbuf_pagesize); + } + + /* + * Sanity checks on target address. It's permitted to write + * at PAD(c->wbuf_len+c->wbuf_ofs), and it's permitted to + * write at the beginning of a new erase block. Anything else, + * and you die. New block starts at xxx000c (0-b = block + * header) + */ + if (SECTOR_ADDR(to) != SECTOR_ADDR(c->wbuf_ofs)) { + /* It's a write to a new block */ + if (c->wbuf_len) { + jffs2_dbg(1, "%s(): to 0x%lx causes flush of wbuf at 0x%08x\n", + __func__, (unsigned long)to, c->wbuf_ofs); + ret = __jffs2_flush_wbuf(c, PAD_NOACCOUNT); + if (ret) + goto outerr; + } + /* set pointer to new block */ + c->wbuf_ofs = PAGE_DIV(to); + c->wbuf_len = PAGE_MOD(to); + } + + if (to != PAD(c->wbuf_ofs + c->wbuf_len)) { + /* We're not writing immediately after the writebuffer. Bad. */ + pr_crit("%s(): Non-contiguous write to %08lx\n", + __func__, (unsigned long)to); + if (c->wbuf_len) + pr_crit("wbuf was previously %08x-%08x\n", + c->wbuf_ofs, c->wbuf_ofs + c->wbuf_len); + BUG(); + } + + /* adjust alignment offset */ + if (c->wbuf_len != PAGE_MOD(to)) { + c->wbuf_len = PAGE_MOD(to); + /* take care of alignment to next page */ + if (!c->wbuf_len) { + c->wbuf_len = c->wbuf_pagesize; + ret = __jffs2_flush_wbuf(c, NOPAD); + if (ret) + goto outerr; + } + } + + for (invec = 0; invec < count; invec++) { + int vlen = invecs[invec].iov_len; + uint8_t *v = invecs[invec].iov_base; + + wbuf_retlen = jffs2_fill_wbuf(c, v, vlen); + + if (c->wbuf_len == c->wbuf_pagesize) { + ret = __jffs2_flush_wbuf(c, NOPAD); + if (ret) + goto outerr; + } + vlen -= wbuf_retlen; + outvec_to += wbuf_retlen; + donelen += wbuf_retlen; + v += wbuf_retlen; + + if (vlen >= c->wbuf_pagesize) { + ret = mtd_write(c->mtd, outvec_to, PAGE_DIV(vlen), + &wbuf_retlen, v); + if (ret < 0 || wbuf_retlen != PAGE_DIV(vlen)) + goto outfile; + + vlen -= wbuf_retlen; + outvec_to += wbuf_retlen; + c->wbuf_ofs = outvec_to; + donelen += wbuf_retlen; + v += wbuf_retlen; + } + + wbuf_retlen = jffs2_fill_wbuf(c, v, vlen); + if (c->wbuf_len == c->wbuf_pagesize) { + ret = __jffs2_flush_wbuf(c, NOPAD); + if (ret) + goto outerr; + } + + outvec_to += wbuf_retlen; + donelen += wbuf_retlen; + } + + /* + * If there's a remainder in the wbuf and it's a non-GC write, + * remember that the wbuf affects this ino + */ + *retlen = donelen; + + if (jffs2_sum_active()) { + int res = jffs2_sum_add_kvec(c, invecs, count, (uint32_t) to); + if (res) + return res; + } + + if (c->wbuf_len && ino) + jffs2_wbuf_dirties_inode(c, ino); + + ret = 0; + up_write(&c->wbuf_sem); + return ret; + +outfile: + /* + * At this point we have no problem, c->wbuf is empty. However + * refile nextblock to avoid writing again to same address. + */ + + spin_lock(&c->erase_completion_lock); + + jeb = &c->blocks[outvec_to / c->sector_size]; + jffs2_block_refile(c, jeb, REFILE_ANYWAY); + + spin_unlock(&c->erase_completion_lock); + +outerr: + *retlen = 0; + up_write(&c->wbuf_sem); + return ret; +} + +/* + * This is the entry for flash write. + * Check, if we work on NAND FLASH, if so build an kvec and write it via vritev +*/ +int jffs2_flash_write(struct jffs2_sb_info *c, loff_t ofs, size_t len, + size_t *retlen, const u_char *buf) +{ + struct kvec vecs[1]; + + if (!jffs2_is_writebuffered(c)) + return jffs2_flash_direct_write(c, ofs, len, retlen, buf); + + vecs[0].iov_base = (unsigned char *) buf; + vecs[0].iov_len = len; + return jffs2_flash_writev(c, vecs, 1, ofs, retlen, 0); +} + +/* + Handle readback from writebuffer and ECC failure return +*/ +int jffs2_flash_read(struct jffs2_sb_info *c, loff_t ofs, size_t len, size_t *retlen, u_char *buf) +{ + loff_t orbf = 0, owbf = 0, lwbf = 0; + int ret; + + if (!jffs2_is_writebuffered(c)) + return mtd_read(c->mtd, ofs, len, retlen, buf); + + /* Read flash */ + down_read(&c->wbuf_sem); + ret = mtd_read(c->mtd, ofs, len, retlen, buf); + + if ( (ret == -EBADMSG || ret == -EUCLEAN) && (*retlen == len) ) { + if (ret == -EBADMSG) + pr_warn("mtd->read(0x%zx bytes from 0x%llx) returned ECC error\n", + len, ofs); + /* + * We have the raw data without ECC correction in the buffer, + * maybe we are lucky and all data or parts are correct. We + * check the node. If data are corrupted node check will sort + * it out. We keep this block, it will fail on write or erase + * and the we mark it bad. Or should we do that now? But we + * should give him a chance. Maybe we had a system crash or + * power loss before the ecc write or a erase was completed. + * So we return success. :) + */ + ret = 0; + } + + /* if no writebuffer available or write buffer empty, return */ + if (!c->wbuf_pagesize || !c->wbuf_len) + goto exit; + + /* if we read in a different block, return */ + if (SECTOR_ADDR(ofs) != SECTOR_ADDR(c->wbuf_ofs)) + goto exit; + + if (ofs >= c->wbuf_ofs) { + owbf = (ofs - c->wbuf_ofs); /* offset in write buffer */ + if (owbf > c->wbuf_len) /* is read beyond write buffer ? */ + goto exit; + lwbf = c->wbuf_len - owbf; /* number of bytes to copy */ + if (lwbf > len) + lwbf = len; + } else { + orbf = (c->wbuf_ofs - ofs); /* offset in read buffer */ + if (orbf > len) /* is write beyond write buffer ? */ + goto exit; + lwbf = len - orbf; /* number of bytes to copy */ + if (lwbf > c->wbuf_len) + lwbf = c->wbuf_len; + } + if (lwbf > 0) + memcpy(buf+orbf,c->wbuf+owbf,lwbf); + +exit: + up_read(&c->wbuf_sem); + return ret; +} + +#define NR_OOB_SCAN_PAGES 4 + +/* For historical reasons we use only 8 bytes for OOB clean marker */ +#define OOB_CM_SIZE 8 + +static const struct jffs2_unknown_node oob_cleanmarker = +{ + .magic = constant_cpu_to_je16(JFFS2_MAGIC_BITMASK), + .nodetype = constant_cpu_to_je16(JFFS2_NODETYPE_CLEANMARKER), + .totlen = constant_cpu_to_je32(8) +}; + +/* + * Check, if the out of band area is empty. This function knows about the clean + * marker and if it is present in OOB, treats the OOB as empty anyway. + */ +int jffs2_check_oob_empty(struct jffs2_sb_info *c, + struct jffs2_eraseblock *jeb, int mode) +{ + int i, ret; + int cmlen = min_t(int, c->oobavail, OOB_CM_SIZE); + struct mtd_oob_ops ops = { }; + + ops.mode = MTD_OPS_AUTO_OOB; + ops.ooblen = NR_OOB_SCAN_PAGES * c->oobavail; + ops.oobbuf = c->oobbuf; + ops.len = ops.ooboffs = ops.retlen = ops.oobretlen = 0; + ops.datbuf = NULL; + + ret = mtd_read_oob(c->mtd, jeb->offset, &ops); + if ((ret && !mtd_is_bitflip(ret)) || ops.oobretlen != ops.ooblen) { + pr_err("cannot read OOB for EB at %08x, requested %zd bytes, read %zd bytes, error %d\n", + jeb->offset, ops.ooblen, ops.oobretlen, ret); + if (!ret || mtd_is_bitflip(ret)) + ret = -EIO; + return ret; + } + + for(i = 0; i < ops.ooblen; i++) { + if (mode && i < cmlen) + /* Yeah, we know about the cleanmarker */ + continue; + + if (ops.oobbuf[i] != 0xFF) { + jffs2_dbg(2, "Found %02x at %x in OOB for " + "%08x\n", ops.oobbuf[i], i, jeb->offset); + return 1; + } + } + + return 0; +} + +/* + * Check for a valid cleanmarker. + * Returns: 0 if a valid cleanmarker was found + * 1 if no cleanmarker was found + * negative error code if an error occurred + */ +int jffs2_check_nand_cleanmarker(struct jffs2_sb_info *c, + struct jffs2_eraseblock *jeb) +{ + struct mtd_oob_ops ops = { }; + int ret, cmlen = min_t(int, c->oobavail, OOB_CM_SIZE); + + ops.mode = MTD_OPS_AUTO_OOB; + ops.ooblen = cmlen; + ops.oobbuf = c->oobbuf; + ops.len = ops.ooboffs = ops.retlen = ops.oobretlen = 0; + ops.datbuf = NULL; + + ret = mtd_read_oob(c->mtd, jeb->offset, &ops); + if ((ret && !mtd_is_bitflip(ret)) || ops.oobretlen != ops.ooblen) { + pr_err("cannot read OOB for EB at %08x, requested %zd bytes, read %zd bytes, error %d\n", + jeb->offset, ops.ooblen, ops.oobretlen, ret); + if (!ret || mtd_is_bitflip(ret)) + ret = -EIO; + return ret; + } + + return !!memcmp(&oob_cleanmarker, c->oobbuf, cmlen); +} + +int jffs2_write_nand_cleanmarker(struct jffs2_sb_info *c, + struct jffs2_eraseblock *jeb) +{ + int ret; + struct mtd_oob_ops ops = { }; + int cmlen = min_t(int, c->oobavail, OOB_CM_SIZE); + + ops.mode = MTD_OPS_AUTO_OOB; + ops.ooblen = cmlen; + ops.oobbuf = (uint8_t *)&oob_cleanmarker; + ops.len = ops.ooboffs = ops.retlen = ops.oobretlen = 0; + ops.datbuf = NULL; + + ret = mtd_write_oob(c->mtd, jeb->offset, &ops); + if (ret || ops.oobretlen != ops.ooblen) { + pr_err("cannot write OOB for EB at %08x, requested %zd bytes, read %zd bytes, error %d\n", + jeb->offset, ops.ooblen, ops.oobretlen, ret); + if (!ret) + ret = -EIO; + return ret; + } + + return 0; +} + +/* + * On NAND we try to mark this block bad. If the block was erased more + * than MAX_ERASE_FAILURES we mark it finally bad. + * Don't care about failures. This block remains on the erase-pending + * or badblock list as long as nobody manipulates the flash with + * a bootloader or something like that. + */ + +int jffs2_write_nand_badblock(struct jffs2_sb_info *c, struct jffs2_eraseblock *jeb, uint32_t bad_offset) +{ + int ret; + + /* if the count is < max, we try to write the counter to the 2nd page oob area */ + if( ++jeb->bad_count < MAX_ERASE_FAILURES) + return 0; + + pr_warn("marking eraseblock at %08x as bad\n", bad_offset); + ret = mtd_block_markbad(c->mtd, bad_offset); + + if (ret) { + jffs2_dbg(1, "%s(): Write failed for block at %08x: error %d\n", + __func__, jeb->offset, ret); + return ret; + } + return 1; +} + +static struct jffs2_sb_info *work_to_sb(struct work_struct *work) +{ + struct delayed_work *dwork; + + dwork = to_delayed_work(work); + return container_of(dwork, struct jffs2_sb_info, wbuf_dwork); +} + +static void delayed_wbuf_sync(struct work_struct *work) +{ + struct jffs2_sb_info *c = work_to_sb(work); + struct super_block *sb = OFNI_BS_2SFFJ(c); + + if (!sb_rdonly(sb)) { + jffs2_dbg(1, "%s()\n", __func__); + jffs2_flush_wbuf_gc(c, 0); + } +} + +void jffs2_dirty_trigger(struct jffs2_sb_info *c) +{ + struct super_block *sb = OFNI_BS_2SFFJ(c); + unsigned long delay; + + if (sb_rdonly(sb)) + return; + + delay = msecs_to_jiffies(dirty_writeback_interval * 10); + if (queue_delayed_work(system_dfl_long_wq, &c->wbuf_dwork, delay)) + jffs2_dbg(1, "%s()\n", __func__); +} + +int jffs2_nand_flash_setup(struct jffs2_sb_info *c) +{ + if (!c->mtd->oobsize) + return 0; + + /* Cleanmarker is out-of-band, so inline size zero */ + c->cleanmarker_size = 0; + + if (c->mtd->oobavail == 0) { + pr_err("inconsistent device description\n"); + return -EINVAL; + } + + jffs2_dbg(1, "using OOB on NAND\n"); + + c->oobavail = c->mtd->oobavail; + + /* Initialise write buffer */ + init_rwsem(&c->wbuf_sem); + INIT_DELAYED_WORK(&c->wbuf_dwork, delayed_wbuf_sync); + c->wbuf_pagesize = c->mtd->writesize; + c->wbuf_ofs = 0xFFFFFFFF; + + c->wbuf = kmalloc(c->wbuf_pagesize, GFP_KERNEL); + if (!c->wbuf) + return -ENOMEM; + + c->oobbuf = kmalloc_array(NR_OOB_SCAN_PAGES, c->oobavail, GFP_KERNEL); + if (!c->oobbuf) { + kfree(c->wbuf); + return -ENOMEM; + } + +#ifdef CONFIG_JFFS2_FS_WBUF_VERIFY + c->wbuf_verify = kmalloc(c->wbuf_pagesize, GFP_KERNEL); + if (!c->wbuf_verify) { + kfree(c->oobbuf); + kfree(c->wbuf); + return -ENOMEM; + } +#endif + return 0; +} + +void jffs2_nand_flash_cleanup(struct jffs2_sb_info *c) +{ +#ifdef CONFIG_JFFS2_FS_WBUF_VERIFY + kfree(c->wbuf_verify); +#endif + kfree(c->wbuf); + kfree(c->oobbuf); +} + +int jffs2_dataflash_setup(struct jffs2_sb_info *c) { + c->cleanmarker_size = 0; /* No cleanmarkers needed */ + + /* Initialize write buffer */ + init_rwsem(&c->wbuf_sem); + INIT_DELAYED_WORK(&c->wbuf_dwork, delayed_wbuf_sync); + c->wbuf_pagesize = c->mtd->erasesize; + + /* Find a suitable c->sector_size + * - Not too much sectors + * - Sectors have to be at least 4 K + some bytes + * - All known dataflashes have erase sizes of 528 or 1056 + * - we take at least 8 eraseblocks and want to have at least 8K size + * - The concatenation should be a power of 2 + */ + + c->sector_size = 8 * c->mtd->erasesize; + + while (c->sector_size < 8192) { + c->sector_size *= 2; + } + + /* It may be necessary to adjust the flash size */ + c->flash_size = c->mtd->size; + + if ((c->flash_size % c->sector_size) != 0) { + c->flash_size = (c->flash_size / c->sector_size) * c->sector_size; + pr_warn("flash size adjusted to %dKiB\n", c->flash_size); + } + + c->wbuf_ofs = 0xFFFFFFFF; + c->wbuf = kmalloc(c->wbuf_pagesize, GFP_KERNEL); + if (!c->wbuf) + return -ENOMEM; + +#ifdef CONFIG_JFFS2_FS_WBUF_VERIFY + c->wbuf_verify = kmalloc(c->wbuf_pagesize, GFP_KERNEL); + if (!c->wbuf_verify) { + kfree(c->wbuf); + return -ENOMEM; + } +#endif + + pr_info("write-buffering enabled buffer (%d) erasesize (%d)\n", + c->wbuf_pagesize, c->sector_size); + + return 0; +} + +void jffs2_dataflash_cleanup(struct jffs2_sb_info *c) { +#ifdef CONFIG_JFFS2_FS_WBUF_VERIFY + kfree(c->wbuf_verify); +#endif + kfree(c->wbuf); +} + +int jffs2_nor_wbuf_flash_setup(struct jffs2_sb_info *c) { + /* Cleanmarker currently occupies whole programming regions, + * either one or 2 for 8Byte STMicro flashes. */ + c->cleanmarker_size = max(16u, c->mtd->writesize); + + /* Initialize write buffer */ + init_rwsem(&c->wbuf_sem); + INIT_DELAYED_WORK(&c->wbuf_dwork, delayed_wbuf_sync); + + c->wbuf_pagesize = c->mtd->writesize; + c->wbuf_ofs = 0xFFFFFFFF; + + c->wbuf = kmalloc(c->wbuf_pagesize, GFP_KERNEL); + if (!c->wbuf) + return -ENOMEM; + +#ifdef CONFIG_JFFS2_FS_WBUF_VERIFY + c->wbuf_verify = kmalloc(c->wbuf_pagesize, GFP_KERNEL); + if (!c->wbuf_verify) { + kfree(c->wbuf); + return -ENOMEM; + } +#endif + return 0; +} + +void jffs2_nor_wbuf_flash_cleanup(struct jffs2_sb_info *c) { +#ifdef CONFIG_JFFS2_FS_WBUF_VERIFY + kfree(c->wbuf_verify); +#endif + kfree(c->wbuf); +} + +int jffs2_ubivol_setup(struct jffs2_sb_info *c) { + c->cleanmarker_size = 0; + + if (c->mtd->writesize == 1) + /* We do not need write-buffer */ + return 0; + + init_rwsem(&c->wbuf_sem); + INIT_DELAYED_WORK(&c->wbuf_dwork, delayed_wbuf_sync); + + c->wbuf_pagesize = c->mtd->writesize; + c->wbuf_ofs = 0xFFFFFFFF; + c->wbuf = kmalloc(c->wbuf_pagesize, GFP_KERNEL); + if (!c->wbuf) + return -ENOMEM; + + pr_info("write-buffering enabled buffer (%d) erasesize (%d)\n", + c->wbuf_pagesize, c->sector_size); + + return 0; +} + +void jffs2_ubivol_cleanup(struct jffs2_sb_info *c) { + kfree(c->wbuf); +} diff --git a/fs/jffs2/write.c b/fs/jffs2/write.c new file mode 100644 index 000000000..cda9a3613 --- /dev/null +++ b/fs/jffs2/write.c @@ -0,0 +1,723 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * + * Created by David Woodhouse + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include "nodelist.h" +#include "compr.h" + + +int jffs2_do_new_inode(struct jffs2_sb_info *c, struct jffs2_inode_info *f, + uint32_t mode, struct jffs2_raw_inode *ri) +{ + struct jffs2_inode_cache *ic; + + ic = jffs2_alloc_inode_cache(); + if (!ic) { + return -ENOMEM; + } + + memset(ic, 0, sizeof(*ic)); + + f->inocache = ic; + f->inocache->pino_nlink = 1; /* Will be overwritten shortly for directories */ + f->inocache->nodes = (struct jffs2_raw_node_ref *)f->inocache; + f->inocache->state = INO_STATE_PRESENT; + + jffs2_add_ino_cache(c, f->inocache); + jffs2_dbg(1, "%s(): Assigned ino# %d\n", __func__, f->inocache->ino); + ri->ino = cpu_to_je32(f->inocache->ino); + + ri->magic = cpu_to_je16(JFFS2_MAGIC_BITMASK); + ri->nodetype = cpu_to_je16(JFFS2_NODETYPE_INODE); + ri->totlen = cpu_to_je32(PAD(sizeof(*ri))); + ri->hdr_crc = cpu_to_je32(crc32(0, ri, sizeof(struct jffs2_unknown_node)-4)); + ri->mode = cpu_to_jemode(mode); + + f->highest_version = 1; + ri->version = cpu_to_je32(f->highest_version); + + return 0; +} + +/* jffs2_write_dnode - given a raw_inode, allocate a full_dnode for it, + write it to the flash, link it into the existing inode/fragment list */ + +struct jffs2_full_dnode *jffs2_write_dnode(struct jffs2_sb_info *c, struct jffs2_inode_info *f, + struct jffs2_raw_inode *ri, const unsigned char *data, + uint32_t datalen, int alloc_mode) + +{ + struct jffs2_full_dnode *fn; + size_t retlen; + uint32_t flash_ofs; + struct kvec vecs[2]; + int ret; + int retried = 0; + unsigned long cnt = 2; + + D1(if(je32_to_cpu(ri->hdr_crc) != crc32(0, ri, sizeof(struct jffs2_unknown_node)-4)) { + pr_crit("Eep. CRC not correct in jffs2_write_dnode()\n"); + BUG(); + } + ); + vecs[0].iov_base = ri; + vecs[0].iov_len = sizeof(*ri); + vecs[1].iov_base = (unsigned char *)data; + vecs[1].iov_len = datalen; + + if (je32_to_cpu(ri->totlen) != sizeof(*ri) + datalen) { + pr_warn("%s(): ri->totlen (0x%08x) != sizeof(*ri) (0x%08zx) + datalen (0x%08x)\n", + __func__, je32_to_cpu(ri->totlen), + sizeof(*ri), datalen); + } + + fn = jffs2_alloc_full_dnode(); + if (!fn) + return ERR_PTR(-ENOMEM); + + /* check number of valid vecs */ + if (!datalen || !data) + cnt = 1; + retry: + flash_ofs = write_ofs(c); + + jffs2_dbg_prewrite_paranoia_check(c, flash_ofs, vecs[0].iov_len + vecs[1].iov_len); + + if ((alloc_mode!=ALLOC_GC) && (je32_to_cpu(ri->version) < f->highest_version)) { + BUG_ON(!retried); + jffs2_dbg(1, "%s(): dnode_version %d, highest version %d -> updating dnode\n", + __func__, + je32_to_cpu(ri->version), f->highest_version); + ri->version = cpu_to_je32(++f->highest_version); + ri->node_crc = cpu_to_je32(crc32(0, ri, sizeof(*ri)-8)); + } + + ret = jffs2_flash_writev(c, vecs, cnt, flash_ofs, &retlen, + (alloc_mode==ALLOC_GC)?0:f->inocache->ino); + + if (ret || (retlen != sizeof(*ri) + datalen)) { + pr_notice("Write of %zd bytes at 0x%08x failed. returned %d, retlen %zd\n", + sizeof(*ri) + datalen, flash_ofs, ret, retlen); + + /* Mark the space as dirtied */ + if (retlen) { + /* Don't change raw->size to match retlen. We may have + written the node header already, and only the data will + seem corrupted, in which case the scan would skip over + any node we write before the original intended end of + this node */ + jffs2_add_physical_node_ref(c, flash_ofs | REF_OBSOLETE, PAD(sizeof(*ri)+datalen), NULL); + } else { + pr_notice("Not marking the space at 0x%08x as dirty because the flash driver returned retlen zero\n", + flash_ofs); + } + if (!retried && alloc_mode != ALLOC_NORETRY) { + /* Try to reallocate space and retry */ + uint32_t dummy; + struct jffs2_eraseblock *jeb = &c->blocks[flash_ofs / c->sector_size]; + + retried = 1; + + jffs2_dbg(1, "Retrying failed write.\n"); + + jffs2_dbg_acct_sanity_check(c,jeb); + jffs2_dbg_acct_paranoia_check(c, jeb); + + if (alloc_mode == ALLOC_GC) { + ret = jffs2_reserve_space_gc(c, sizeof(*ri) + datalen, &dummy, + JFFS2_SUMMARY_INODE_SIZE); + } else { + /* Locking pain */ + mutex_unlock(&f->sem); + jffs2_complete_reservation(c); + + ret = jffs2_reserve_space(c, sizeof(*ri) + datalen, &dummy, + alloc_mode, JFFS2_SUMMARY_INODE_SIZE); + mutex_lock(&f->sem); + } + + if (!ret) { + flash_ofs = write_ofs(c); + jffs2_dbg(1, "Allocated space at 0x%08x to retry failed write.\n", + flash_ofs); + + jffs2_dbg_acct_sanity_check(c,jeb); + jffs2_dbg_acct_paranoia_check(c, jeb); + + goto retry; + } + jffs2_dbg(1, "Failed to allocate space to retry failed write: %d!\n", + ret); + } + /* Release the full_dnode which is now useless, and return */ + jffs2_free_full_dnode(fn); + return ERR_PTR(ret?ret:-EIO); + } + /* Mark the space used */ + /* If node covers at least a whole page, or if it starts at the + beginning of a page and runs to the end of the file, or if + it's a hole node, mark it REF_PRISTINE, else REF_NORMAL. + */ + if ((je32_to_cpu(ri->dsize) >= PAGE_SIZE) || + ( ((je32_to_cpu(ri->offset)&(PAGE_SIZE-1))==0) && + (je32_to_cpu(ri->dsize)+je32_to_cpu(ri->offset) == je32_to_cpu(ri->isize)))) { + flash_ofs |= REF_PRISTINE; + } else { + flash_ofs |= REF_NORMAL; + } + fn->raw = jffs2_add_physical_node_ref(c, flash_ofs, PAD(sizeof(*ri)+datalen), f->inocache); + if (IS_ERR(fn->raw)) { + void *hold_err = fn->raw; + /* Release the full_dnode which is now useless, and return */ + jffs2_free_full_dnode(fn); + return ERR_CAST(hold_err); + } + fn->ofs = je32_to_cpu(ri->offset); + fn->size = je32_to_cpu(ri->dsize); + fn->frags = 0; + + jffs2_dbg(1, "jffs2_write_dnode wrote node at 0x%08x(%d) with dsize 0x%x, csize 0x%x, node_crc 0x%08x, data_crc 0x%08x, totlen 0x%08x\n", + flash_ofs & ~3, flash_ofs & 3, je32_to_cpu(ri->dsize), + je32_to_cpu(ri->csize), je32_to_cpu(ri->node_crc), + je32_to_cpu(ri->data_crc), je32_to_cpu(ri->totlen)); + + if (retried) { + jffs2_dbg_acct_sanity_check(c,NULL); + } + + return fn; +} + +struct jffs2_full_dirent *jffs2_write_dirent(struct jffs2_sb_info *c, struct jffs2_inode_info *f, + struct jffs2_raw_dirent *rd, const unsigned char *name, + uint32_t namelen, int alloc_mode) +{ + struct jffs2_full_dirent *fd; + size_t retlen; + struct kvec vecs[2]; + uint32_t flash_ofs; + int retried = 0; + int ret; + + jffs2_dbg(1, "%s(ino #%u, name at *0x%p \"%s\"->ino #%u, name_crc 0x%08x)\n", + __func__, + je32_to_cpu(rd->pino), name, name, je32_to_cpu(rd->ino), + je32_to_cpu(rd->name_crc)); + + D1(if(je32_to_cpu(rd->hdr_crc) != crc32(0, rd, sizeof(struct jffs2_unknown_node)-4)) { + pr_crit("Eep. CRC not correct in jffs2_write_dirent()\n"); + BUG(); + }); + + if (strnlen(name, namelen) != namelen) { + /* This should never happen, but seems to have done on at least one + occasion: https://dev.laptop.org/ticket/4184 */ + pr_crit("Error in jffs2_write_dirent() -- name contains zero bytes!\n"); + pr_crit("Directory inode #%u, name at *0x%p \"%s\"->ino #%u, name_crc 0x%08x\n", + je32_to_cpu(rd->pino), name, name, je32_to_cpu(rd->ino), + je32_to_cpu(rd->name_crc)); + WARN_ON(1); + return ERR_PTR(-EIO); + } + + vecs[0].iov_base = rd; + vecs[0].iov_len = sizeof(*rd); + vecs[1].iov_base = (unsigned char *)name; + vecs[1].iov_len = namelen; + + fd = jffs2_alloc_full_dirent(namelen+1); + if (!fd) + return ERR_PTR(-ENOMEM); + + fd->version = je32_to_cpu(rd->version); + fd->ino = je32_to_cpu(rd->ino); + fd->nhash = full_name_hash(NULL, name, namelen); + fd->type = rd->type; + memcpy(fd->name, name, namelen); + fd->name[namelen]=0; + + retry: + flash_ofs = write_ofs(c); + + jffs2_dbg_prewrite_paranoia_check(c, flash_ofs, vecs[0].iov_len + vecs[1].iov_len); + + if ((alloc_mode!=ALLOC_GC) && (je32_to_cpu(rd->version) < f->highest_version)) { + BUG_ON(!retried); + jffs2_dbg(1, "%s(): dirent_version %d, highest version %d -> updating dirent\n", + __func__, + je32_to_cpu(rd->version), f->highest_version); + rd->version = cpu_to_je32(++f->highest_version); + fd->version = je32_to_cpu(rd->version); + rd->node_crc = cpu_to_je32(crc32(0, rd, sizeof(*rd)-8)); + } + + ret = jffs2_flash_writev(c, vecs, 2, flash_ofs, &retlen, + (alloc_mode==ALLOC_GC)?0:je32_to_cpu(rd->pino)); + if (ret || (retlen != sizeof(*rd) + namelen)) { + pr_notice("Write of %zd bytes at 0x%08x failed. returned %d, retlen %zd\n", + sizeof(*rd) + namelen, flash_ofs, ret, retlen); + /* Mark the space as dirtied */ + if (retlen) { + jffs2_add_physical_node_ref(c, flash_ofs | REF_OBSOLETE, PAD(sizeof(*rd)+namelen), NULL); + } else { + pr_notice("Not marking the space at 0x%08x as dirty because the flash driver returned retlen zero\n", + flash_ofs); + } + if (!retried) { + /* Try to reallocate space and retry */ + uint32_t dummy; + struct jffs2_eraseblock *jeb = &c->blocks[flash_ofs / c->sector_size]; + + retried = 1; + + jffs2_dbg(1, "Retrying failed write.\n"); + + jffs2_dbg_acct_sanity_check(c,jeb); + jffs2_dbg_acct_paranoia_check(c, jeb); + + if (alloc_mode == ALLOC_GC) { + ret = jffs2_reserve_space_gc(c, sizeof(*rd) + namelen, &dummy, + JFFS2_SUMMARY_DIRENT_SIZE(namelen)); + } else { + /* Locking pain */ + mutex_unlock(&f->sem); + jffs2_complete_reservation(c); + + ret = jffs2_reserve_space(c, sizeof(*rd) + namelen, &dummy, + alloc_mode, JFFS2_SUMMARY_DIRENT_SIZE(namelen)); + mutex_lock(&f->sem); + } + + if (!ret) { + flash_ofs = write_ofs(c); + jffs2_dbg(1, "Allocated space at 0x%08x to retry failed write\n", + flash_ofs); + jffs2_dbg_acct_sanity_check(c,jeb); + jffs2_dbg_acct_paranoia_check(c, jeb); + goto retry; + } + jffs2_dbg(1, "Failed to allocate space to retry failed write: %d!\n", + ret); + } + /* Release the full_dnode which is now useless, and return */ + jffs2_free_full_dirent(fd); + return ERR_PTR(ret?ret:-EIO); + } + /* Mark the space used */ + fd->raw = jffs2_add_physical_node_ref(c, flash_ofs | dirent_node_state(rd), + PAD(sizeof(*rd)+namelen), f->inocache); + if (IS_ERR(fd->raw)) { + void *hold_err = fd->raw; + /* Release the full_dirent which is now useless, and return */ + jffs2_free_full_dirent(fd); + return ERR_CAST(hold_err); + } + + if (retried) { + jffs2_dbg_acct_sanity_check(c,NULL); + } + + return fd; +} + +/* The OS-specific code fills in the metadata in the jffs2_raw_inode for us, so that + we don't have to go digging in struct inode or its equivalent. It should set: + mode, uid, gid, (starting)isize, atime, ctime, mtime */ +int jffs2_write_inode_range(struct jffs2_sb_info *c, struct jffs2_inode_info *f, + struct jffs2_raw_inode *ri, unsigned char *buf, + uint32_t offset, uint32_t writelen, uint32_t *retlen) +{ + int ret = 0; + uint32_t writtenlen = 0; + + jffs2_dbg(1, "%s(): Ino #%u, ofs 0x%x, len 0x%x\n", + __func__, f->inocache->ino, offset, writelen); + + while(writelen) { + struct jffs2_full_dnode *fn; + unsigned char *comprbuf = NULL; + uint16_t comprtype = JFFS2_COMPR_NONE; + uint32_t alloclen; + uint32_t datalen, cdatalen; + int retried = 0; + + retry: + jffs2_dbg(2, "jffs2_commit_write() loop: 0x%x to write to 0x%x\n", + writelen, offset); + + ret = jffs2_reserve_space(c, sizeof(*ri) + JFFS2_MIN_DATA_LEN, + &alloclen, ALLOC_NORMAL, JFFS2_SUMMARY_INODE_SIZE); + if (ret) { + jffs2_dbg(1, "jffs2_reserve_space returned %d\n", ret); + break; + } + mutex_lock(&f->sem); + datalen = min_t(uint32_t, writelen, + PAGE_SIZE - (offset & (PAGE_SIZE-1))); + cdatalen = min_t(uint32_t, alloclen - sizeof(*ri), datalen); + + comprtype = jffs2_compress(c, f, buf, &comprbuf, &datalen, &cdatalen); + + ri->magic = cpu_to_je16(JFFS2_MAGIC_BITMASK); + ri->nodetype = cpu_to_je16(JFFS2_NODETYPE_INODE); + ri->totlen = cpu_to_je32(sizeof(*ri) + cdatalen); + ri->hdr_crc = cpu_to_je32(crc32(0, ri, sizeof(struct jffs2_unknown_node)-4)); + + ri->ino = cpu_to_je32(f->inocache->ino); + ri->version = cpu_to_je32(++f->highest_version); + ri->isize = cpu_to_je32(max(je32_to_cpu(ri->isize), offset + datalen)); + ri->offset = cpu_to_je32(offset); + ri->csize = cpu_to_je32(cdatalen); + ri->dsize = cpu_to_je32(datalen); + ri->compr = comprtype & 0xff; + ri->usercompr = (comprtype >> 8 ) & 0xff; + ri->node_crc = cpu_to_je32(crc32(0, ri, sizeof(*ri)-8)); + ri->data_crc = cpu_to_je32(crc32(0, comprbuf, cdatalen)); + + fn = jffs2_write_dnode(c, f, ri, comprbuf, cdatalen, ALLOC_NORETRY); + + jffs2_free_comprbuf(comprbuf, buf); + + if (IS_ERR(fn)) { + ret = PTR_ERR(fn); + mutex_unlock(&f->sem); + jffs2_complete_reservation(c); + if (!retried) { + /* Write error to be retried */ + retried = 1; + jffs2_dbg(1, "Retrying node write in jffs2_write_inode_range()\n"); + goto retry; + } + break; + } + ret = jffs2_add_full_dnode_to_inode(c, f, fn); + if (f->metadata) { + jffs2_mark_node_obsolete(c, f->metadata->raw); + jffs2_free_full_dnode(f->metadata); + f->metadata = NULL; + } + if (ret) { + /* Eep */ + jffs2_dbg(1, "Eep. add_full_dnode_to_inode() failed in commit_write, returned %d\n", + ret); + jffs2_mark_node_obsolete(c, fn->raw); + jffs2_free_full_dnode(fn); + + mutex_unlock(&f->sem); + jffs2_complete_reservation(c); + break; + } + mutex_unlock(&f->sem); + jffs2_complete_reservation(c); + if (!datalen) { + pr_warn("Eep. We didn't actually write any data in jffs2_write_inode_range()\n"); + ret = -EIO; + break; + } + jffs2_dbg(1, "increasing writtenlen by %d\n", datalen); + writtenlen += datalen; + offset += datalen; + writelen -= datalen; + buf += datalen; + } + *retlen = writtenlen; + return ret; +} + +int jffs2_do_create(struct jffs2_sb_info *c, struct jffs2_inode_info *dir_f, + struct jffs2_inode_info *f, struct jffs2_raw_inode *ri, + const struct qstr *qstr) +{ + struct jffs2_raw_dirent *rd; + struct jffs2_full_dnode *fn; + struct jffs2_full_dirent *fd; + uint32_t alloclen; + int ret; + + /* Try to reserve enough space for both node and dirent. + * Just the node will do for now, though + */ + ret = jffs2_reserve_space(c, sizeof(*ri), &alloclen, ALLOC_NORMAL, + JFFS2_SUMMARY_INODE_SIZE); + jffs2_dbg(1, "%s(): reserved 0x%x bytes\n", __func__, alloclen); + if (ret) + return ret; + + mutex_lock(&f->sem); + + ri->data_crc = cpu_to_je32(0); + ri->node_crc = cpu_to_je32(crc32(0, ri, sizeof(*ri)-8)); + + fn = jffs2_write_dnode(c, f, ri, NULL, 0, ALLOC_NORMAL); + + jffs2_dbg(1, "jffs2_do_create created file with mode 0x%x\n", + jemode_to_cpu(ri->mode)); + + if (IS_ERR(fn)) { + jffs2_dbg(1, "jffs2_write_dnode() failed\n"); + /* Eeek. Wave bye bye */ + mutex_unlock(&f->sem); + jffs2_complete_reservation(c); + return PTR_ERR(fn); + } + /* No data here. Only a metadata node, which will be + obsoleted by the first data write + */ + f->metadata = fn; + + mutex_unlock(&f->sem); + jffs2_complete_reservation(c); + + ret = jffs2_init_security(&f->vfs_inode, &dir_f->vfs_inode, qstr); + if (ret) + return ret; + ret = jffs2_init_acl_post(&f->vfs_inode); + if (ret) + return ret; + + ret = jffs2_reserve_space(c, sizeof(*rd)+qstr->len, &alloclen, + ALLOC_NORMAL, JFFS2_SUMMARY_DIRENT_SIZE(qstr->len)); + + if (ret) { + /* Eep. */ + jffs2_dbg(1, "jffs2_reserve_space() for dirent failed\n"); + return ret; + } + + rd = jffs2_alloc_raw_dirent(); + if (!rd) { + /* Argh. Now we treat it like a normal delete */ + jffs2_complete_reservation(c); + return -ENOMEM; + } + + mutex_lock(&dir_f->sem); + + rd->magic = cpu_to_je16(JFFS2_MAGIC_BITMASK); + rd->nodetype = cpu_to_je16(JFFS2_NODETYPE_DIRENT); + rd->totlen = cpu_to_je32(sizeof(*rd) + qstr->len); + rd->hdr_crc = cpu_to_je32(crc32(0, rd, sizeof(struct jffs2_unknown_node)-4)); + + rd->pino = cpu_to_je32(dir_f->inocache->ino); + rd->version = cpu_to_je32(++dir_f->highest_version); + rd->ino = ri->ino; + rd->mctime = ri->ctime; + rd->nsize = qstr->len; + rd->type = DT_REG; + rd->node_crc = cpu_to_je32(crc32(0, rd, sizeof(*rd)-8)); + rd->name_crc = cpu_to_je32(crc32(0, qstr->name, qstr->len)); + + fd = jffs2_write_dirent(c, dir_f, rd, qstr->name, qstr->len, ALLOC_NORMAL); + + jffs2_free_raw_dirent(rd); + + if (IS_ERR(fd)) { + /* dirent failed to write. Delete the inode normally + as if it were the final unlink() */ + jffs2_complete_reservation(c); + mutex_unlock(&dir_f->sem); + return PTR_ERR(fd); + } + + /* Link the fd into the inode's list, obsoleting an old + one if necessary. */ + jffs2_add_fd_to_list(c, fd, &dir_f->dents); + + jffs2_complete_reservation(c); + mutex_unlock(&dir_f->sem); + + return 0; +} + + +int jffs2_do_unlink(struct jffs2_sb_info *c, struct jffs2_inode_info *dir_f, + const char *name, int namelen, struct jffs2_inode_info *dead_f, + uint32_t time) +{ + struct jffs2_raw_dirent *rd; + struct jffs2_full_dirent *fd; + uint32_t alloclen; + int ret; + + if (!jffs2_can_mark_obsolete(c)) { + /* We can't mark stuff obsolete on the medium. We need to write a deletion dirent */ + + rd = jffs2_alloc_raw_dirent(); + if (!rd) + return -ENOMEM; + + ret = jffs2_reserve_space(c, sizeof(*rd)+namelen, &alloclen, + ALLOC_DELETION, JFFS2_SUMMARY_DIRENT_SIZE(namelen)); + if (ret) { + jffs2_free_raw_dirent(rd); + return ret; + } + + mutex_lock(&dir_f->sem); + + /* Build a deletion node */ + rd->magic = cpu_to_je16(JFFS2_MAGIC_BITMASK); + rd->nodetype = cpu_to_je16(JFFS2_NODETYPE_DIRENT); + rd->totlen = cpu_to_je32(sizeof(*rd) + namelen); + rd->hdr_crc = cpu_to_je32(crc32(0, rd, sizeof(struct jffs2_unknown_node)-4)); + + rd->pino = cpu_to_je32(dir_f->inocache->ino); + rd->version = cpu_to_je32(++dir_f->highest_version); + rd->ino = cpu_to_je32(0); + rd->mctime = cpu_to_je32(time); + rd->nsize = namelen; + rd->type = DT_UNKNOWN; + rd->node_crc = cpu_to_je32(crc32(0, rd, sizeof(*rd)-8)); + rd->name_crc = cpu_to_je32(crc32(0, name, namelen)); + + fd = jffs2_write_dirent(c, dir_f, rd, name, namelen, ALLOC_DELETION); + + jffs2_free_raw_dirent(rd); + + if (IS_ERR(fd)) { + jffs2_complete_reservation(c); + mutex_unlock(&dir_f->sem); + return PTR_ERR(fd); + } + + /* File it. This will mark the old one obsolete. */ + jffs2_add_fd_to_list(c, fd, &dir_f->dents); + mutex_unlock(&dir_f->sem); + } else { + uint32_t nhash = full_name_hash(NULL, name, namelen); + + fd = dir_f->dents; + /* We don't actually want to reserve any space, but we do + want to be holding the alloc_sem when we write to flash */ + mutex_lock(&c->alloc_sem); + mutex_lock(&dir_f->sem); + + for (fd = dir_f->dents; fd; fd = fd->next) { + if (fd->nhash == nhash && + !memcmp(fd->name, name, namelen) && + !fd->name[namelen]) { + + jffs2_dbg(1, "Marking old dirent node (ino #%u) @%08x obsolete\n", + fd->ino, ref_offset(fd->raw)); + jffs2_mark_node_obsolete(c, fd->raw); + /* We don't want to remove it from the list immediately, + because that screws up getdents()/seek() semantics even + more than they're screwed already. Turn it into a + node-less deletion dirent instead -- a placeholder */ + fd->raw = NULL; + fd->ino = 0; + break; + } + } + mutex_unlock(&dir_f->sem); + } + + /* dead_f is NULL if this was a rename not a real unlink */ + /* Also catch the !f->inocache case, where there was a dirent + pointing to an inode which didn't exist. */ + if (dead_f && dead_f->inocache) { + + mutex_lock(&dead_f->sem); + + if (S_ISDIR(OFNI_EDONI_2SFFJ(dead_f)->i_mode)) { + while (dead_f->dents) { + /* There can be only deleted ones */ + fd = dead_f->dents; + + dead_f->dents = fd->next; + + if (fd->ino) { + pr_warn("Deleting inode #%u with active dentry \"%s\"->ino #%u\n", + dead_f->inocache->ino, + fd->name, fd->ino); + } else { + jffs2_dbg(1, "Removing deletion dirent for \"%s\" from dir ino #%u\n", + fd->name, + dead_f->inocache->ino); + } + if (fd->raw) + jffs2_mark_node_obsolete(c, fd->raw); + jffs2_free_full_dirent(fd); + } + dead_f->inocache->pino_nlink = 0; + } else + dead_f->inocache->pino_nlink--; + /* NB: Caller must set inode nlink if appropriate */ + mutex_unlock(&dead_f->sem); + } + + jffs2_complete_reservation(c); + + return 0; +} + + +int jffs2_do_link (struct jffs2_sb_info *c, struct jffs2_inode_info *dir_f, uint32_t ino, uint8_t type, const char *name, int namelen, uint32_t time) +{ + struct jffs2_raw_dirent *rd; + struct jffs2_full_dirent *fd; + uint32_t alloclen; + int ret; + + rd = jffs2_alloc_raw_dirent(); + if (!rd) + return -ENOMEM; + + ret = jffs2_reserve_space(c, sizeof(*rd)+namelen, &alloclen, + ALLOC_NORMAL, JFFS2_SUMMARY_DIRENT_SIZE(namelen)); + if (ret) { + jffs2_free_raw_dirent(rd); + return ret; + } + + mutex_lock(&dir_f->sem); + + /* Build a deletion node */ + rd->magic = cpu_to_je16(JFFS2_MAGIC_BITMASK); + rd->nodetype = cpu_to_je16(JFFS2_NODETYPE_DIRENT); + rd->totlen = cpu_to_je32(sizeof(*rd) + namelen); + rd->hdr_crc = cpu_to_je32(crc32(0, rd, sizeof(struct jffs2_unknown_node)-4)); + + rd->pino = cpu_to_je32(dir_f->inocache->ino); + rd->version = cpu_to_je32(++dir_f->highest_version); + rd->ino = cpu_to_je32(ino); + rd->mctime = cpu_to_je32(time); + rd->nsize = namelen; + + rd->type = type; + + rd->node_crc = cpu_to_je32(crc32(0, rd, sizeof(*rd)-8)); + rd->name_crc = cpu_to_je32(crc32(0, name, namelen)); + + fd = jffs2_write_dirent(c, dir_f, rd, name, namelen, ALLOC_NORMAL); + + jffs2_free_raw_dirent(rd); + + if (IS_ERR(fd)) { + jffs2_complete_reservation(c); + mutex_unlock(&dir_f->sem); + return PTR_ERR(fd); + } + + /* File it. This will mark the old one obsolete. */ + jffs2_add_fd_to_list(c, fd, &dir_f->dents); + + jffs2_complete_reservation(c); + mutex_unlock(&dir_f->sem); + + return 0; +} diff --git a/fs/jffs2/writev.c b/fs/jffs2/writev.c new file mode 100644 index 000000000..a1bda9dab --- /dev/null +++ b/fs/jffs2/writev.c @@ -0,0 +1,51 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2001-2007 Red Hat, Inc. + * + * Created by David Woodhouse + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#include +#include +#include "nodelist.h" + +int jffs2_flash_direct_writev(struct jffs2_sb_info *c, const struct kvec *vecs, + unsigned long count, loff_t to, size_t *retlen) +{ + if (!jffs2_is_writebuffered(c)) { + if (jffs2_sum_active()) { + int res; + res = jffs2_sum_add_kvec(c, vecs, count, (uint32_t) to); + if (res) { + return res; + } + } + } + + return mtd_writev(c->mtd, vecs, count, to, retlen); +} + +int jffs2_flash_direct_write(struct jffs2_sb_info *c, loff_t ofs, size_t len, + size_t *retlen, const u_char *buf) +{ + int ret; + ret = mtd_write(c->mtd, ofs, len, retlen, buf); + + if (jffs2_sum_active()) { + struct kvec vecs[1]; + int res; + + vecs[0].iov_base = (unsigned char *) buf; + vecs[0].iov_len = len; + + res = jffs2_sum_add_kvec(c, vecs, 1, (uint32_t) ofs); + if (res) { + return res; + } + } + return ret; +} diff --git a/fs/jffs2/xattr.c b/fs/jffs2/xattr.c new file mode 100644 index 000000000..6b4f68593 --- /dev/null +++ b/fs/jffs2/xattr.c @@ -0,0 +1,1353 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2006 NEC Corporation + * + * Created by KaiGai Kohei + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#define JFFS2_XATTR_IS_CORRUPTED 1 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "nodelist.h" +/* -------- xdatum related functions ---------------- + * xattr_datum_hashkey(xprefix, xname, xvalue, xsize) + * is used to calcurate xdatum hashkey. The reminder of hashkey into XATTRINDEX_HASHSIZE is + * the index of the xattr name/value pair cache (c->xattrindex). + * is_xattr_datum_unchecked(c, xd) + * returns 1, if xdatum contains any unchecked raw nodes. if all raw nodes are not + * unchecked, it returns 0. + * unload_xattr_datum(c, xd) + * is used to release xattr name/value pair and detach from c->xattrindex. + * reclaim_xattr_datum(c) + * is used to reclaim xattr name/value pairs on the xattr name/value pair cache when + * memory usage by cache is over c->xdatum_mem_threshold. Currently, this threshold + * is hard coded as 32KiB. + * do_verify_xattr_datum(c, xd) + * is used to load the xdatum informations without name/value pair from the medium. + * It's necessary once, because those informations are not collected during mounting + * process when EBS is enabled. + * 0 will be returned, if success. An negative return value means recoverable error, and + * positive return value means unrecoverable error. Thus, caller must remove this xdatum + * and xref when it returned positive value. + * do_load_xattr_datum(c, xd) + * is used to load name/value pair from the medium. + * The meanings of return value is same as do_verify_xattr_datum(). + * load_xattr_datum(c, xd) + * is used to be as a wrapper of do_verify_xattr_datum() and do_load_xattr_datum(). + * If xd need to call do_verify_xattr_datum() at first, it's called before calling + * do_load_xattr_datum(). The meanings of return value is same as do_verify_xattr_datum(). + * save_xattr_datum(c, xd) + * is used to write xdatum to medium. xd->version will be incremented. + * create_xattr_datum(c, xprefix, xname, xvalue, xsize) + * is used to create new xdatum and write to medium. + * unrefer_xattr_datum(c, xd) + * is used to delete a xdatum. When nobody refers this xdatum, JFFS2_XFLAGS_DEAD + * is set on xd->flags and chained xattr_dead_list or release it immediately. + * In the first case, the garbage collector release it later. + * -------------------------------------------------- */ +static uint32_t xattr_datum_hashkey(int xprefix, const char *xname, const char *xvalue, int xsize) +{ + int name_len = strlen(xname); + + return crc32(xprefix, xname, name_len) ^ crc32(xprefix, xvalue, xsize); +} + +static int is_xattr_datum_unchecked(struct jffs2_sb_info *c, struct jffs2_xattr_datum *xd) +{ + struct jffs2_raw_node_ref *raw; + int rc = 0; + + spin_lock(&c->erase_completion_lock); + for (raw=xd->node; raw != (void *)xd; raw=raw->next_in_ino) { + if (ref_flags(raw) == REF_UNCHECKED) { + rc = 1; + break; + } + } + spin_unlock(&c->erase_completion_lock); + return rc; +} + +static void unload_xattr_datum(struct jffs2_sb_info *c, struct jffs2_xattr_datum *xd) +{ + /* must be called under down_write(xattr_sem) */ + D1(dbg_xattr("%s: xid=%u, version=%u\n", __func__, xd->xid, xd->version)); + if (xd->xname) { + c->xdatum_mem_usage -= (xd->name_len + 1 + xd->value_len); + kfree(xd->xname); + } + + list_del_init(&xd->xindex); + xd->hashkey = 0; + xd->xname = NULL; + xd->xvalue = NULL; +} + +static void reclaim_xattr_datum(struct jffs2_sb_info *c) +{ + /* must be called under down_write(xattr_sem) */ + struct jffs2_xattr_datum *xd, *_xd; + uint32_t target, before; + static int index = 0; + int count; + + if (c->xdatum_mem_threshold > c->xdatum_mem_usage) + return; + + before = c->xdatum_mem_usage; + target = c->xdatum_mem_usage * 4 / 5; /* 20% reduction */ + for (count = 0; count < XATTRINDEX_HASHSIZE; count++) { + list_for_each_entry_safe(xd, _xd, &c->xattrindex[index], xindex) { + if (xd->flags & JFFS2_XFLAGS_HOT) { + xd->flags &= ~JFFS2_XFLAGS_HOT; + } else if (!(xd->flags & JFFS2_XFLAGS_BIND)) { + unload_xattr_datum(c, xd); + } + if (c->xdatum_mem_usage <= target) + goto out; + } + index = (index+1) % XATTRINDEX_HASHSIZE; + } + out: + JFFS2_NOTICE("xdatum_mem_usage from %u byte to %u byte (%u byte reclaimed)\n", + before, c->xdatum_mem_usage, before - c->xdatum_mem_usage); +} + +static int do_verify_xattr_datum(struct jffs2_sb_info *c, struct jffs2_xattr_datum *xd) +{ + /* must be called under down_write(xattr_sem) */ + struct jffs2_eraseblock *jeb; + struct jffs2_raw_node_ref *raw; + struct jffs2_raw_xattr rx; + size_t readlen; + uint32_t crc, offset, totlen; + int rc; + + spin_lock(&c->erase_completion_lock); + offset = ref_offset(xd->node); + if (ref_flags(xd->node) == REF_PRISTINE) + goto complete; + spin_unlock(&c->erase_completion_lock); + + rc = jffs2_flash_read(c, offset, sizeof(rx), &readlen, (char *)&rx); + if (rc || readlen != sizeof(rx)) { + JFFS2_WARNING("jffs2_flash_read()=%d, req=%zu, read=%zu at %#08x\n", + rc, sizeof(rx), readlen, offset); + return rc ? rc : -EIO; + } + crc = crc32(0, &rx, sizeof(rx) - 4); + if (crc != je32_to_cpu(rx.node_crc)) { + JFFS2_ERROR("node CRC failed at %#08x, read=%#08x, calc=%#08x\n", + offset, je32_to_cpu(rx.hdr_crc), crc); + xd->flags |= JFFS2_XFLAGS_INVALID; + return JFFS2_XATTR_IS_CORRUPTED; + } + totlen = PAD(sizeof(rx) + rx.name_len + 1 + je16_to_cpu(rx.value_len)); + if (je16_to_cpu(rx.magic) != JFFS2_MAGIC_BITMASK + || je16_to_cpu(rx.nodetype) != JFFS2_NODETYPE_XATTR + || je32_to_cpu(rx.totlen) != totlen + || je32_to_cpu(rx.xid) != xd->xid + || je32_to_cpu(rx.version) != xd->version) { + JFFS2_ERROR("inconsistent xdatum at %#08x, magic=%#04x/%#04x, " + "nodetype=%#04x/%#04x, totlen=%u/%u, xid=%u/%u, version=%u/%u\n", + offset, je16_to_cpu(rx.magic), JFFS2_MAGIC_BITMASK, + je16_to_cpu(rx.nodetype), JFFS2_NODETYPE_XATTR, + je32_to_cpu(rx.totlen), totlen, + je32_to_cpu(rx.xid), xd->xid, + je32_to_cpu(rx.version), xd->version); + xd->flags |= JFFS2_XFLAGS_INVALID; + return JFFS2_XATTR_IS_CORRUPTED; + } + xd->xprefix = rx.xprefix; + xd->name_len = rx.name_len; + xd->value_len = je16_to_cpu(rx.value_len); + xd->data_crc = je32_to_cpu(rx.data_crc); + + spin_lock(&c->erase_completion_lock); + complete: + for (raw=xd->node; raw != (void *)xd; raw=raw->next_in_ino) { + jeb = &c->blocks[ref_offset(raw) / c->sector_size]; + totlen = PAD(ref_totlen(c, jeb, raw)); + if (ref_flags(raw) == REF_UNCHECKED) { + c->unchecked_size -= totlen; c->used_size += totlen; + jeb->unchecked_size -= totlen; jeb->used_size += totlen; + } + raw->flash_offset = ref_offset(raw) | ((xd->node==raw) ? REF_PRISTINE : REF_NORMAL); + } + spin_unlock(&c->erase_completion_lock); + + /* unchecked xdatum is chained with c->xattr_unchecked */ + list_del_init(&xd->xindex); + + dbg_xattr("success on verifying xdatum (xid=%u, version=%u)\n", + xd->xid, xd->version); + + return 0; +} + +static int do_load_xattr_datum(struct jffs2_sb_info *c, struct jffs2_xattr_datum *xd) +{ + /* must be called under down_write(xattr_sem) */ + char *data; + size_t readlen; + uint32_t crc, length; + int i, ret, retry = 0; + + BUG_ON(ref_flags(xd->node) != REF_PRISTINE); + BUG_ON(!list_empty(&xd->xindex)); + retry: + length = xd->name_len + 1 + xd->value_len; + data = kmalloc(length, GFP_KERNEL); + if (!data) + return -ENOMEM; + + ret = jffs2_flash_read(c, ref_offset(xd->node)+sizeof(struct jffs2_raw_xattr), + length, &readlen, data); + + if (ret || length!=readlen) { + JFFS2_WARNING("jffs2_flash_read() returned %d, request=%d, readlen=%zu, at %#08x\n", + ret, length, readlen, ref_offset(xd->node)); + kfree(data); + return ret ? ret : -EIO; + } + + data[xd->name_len] = '\0'; + crc = crc32(0, data, length); + if (crc != xd->data_crc) { + JFFS2_WARNING("node CRC failed (JFFS2_NODETYPE_XATTR)" + " at %#08x, read: 0x%08x calculated: 0x%08x\n", + ref_offset(xd->node), xd->data_crc, crc); + kfree(data); + xd->flags |= JFFS2_XFLAGS_INVALID; + return JFFS2_XATTR_IS_CORRUPTED; + } + + xd->flags |= JFFS2_XFLAGS_HOT; + xd->xname = data; + xd->xvalue = data + xd->name_len+1; + + c->xdatum_mem_usage += length; + + xd->hashkey = xattr_datum_hashkey(xd->xprefix, xd->xname, xd->xvalue, xd->value_len); + i = xd->hashkey % XATTRINDEX_HASHSIZE; + list_add(&xd->xindex, &c->xattrindex[i]); + if (!retry) { + retry = 1; + reclaim_xattr_datum(c); + if (!xd->xname) + goto retry; + } + + dbg_xattr("success on loading xdatum (xid=%u, xprefix=%u, xname='%s')\n", + xd->xid, xd->xprefix, xd->xname); + + return 0; +} + +static int load_xattr_datum(struct jffs2_sb_info *c, struct jffs2_xattr_datum *xd) +{ + /* must be called under down_write(xattr_sem); + * rc < 0 : recoverable error, try again + * rc = 0 : success + * rc > 0 : Unrecoverable error, this node should be deleted. + */ + int rc = 0; + + BUG_ON(xd->flags & JFFS2_XFLAGS_DEAD); + if (xd->xname) + return 0; + if (xd->flags & JFFS2_XFLAGS_INVALID) + return JFFS2_XATTR_IS_CORRUPTED; + if (unlikely(is_xattr_datum_unchecked(c, xd))) + rc = do_verify_xattr_datum(c, xd); + if (!rc) + rc = do_load_xattr_datum(c, xd); + return rc; +} + +static int save_xattr_datum(struct jffs2_sb_info *c, struct jffs2_xattr_datum *xd) +{ + /* must be called under down_write(xattr_sem) */ + struct jffs2_raw_xattr rx; + struct kvec vecs[2]; + size_t length; + int rc, totlen; + uint32_t phys_ofs = write_ofs(c); + + BUG_ON(!xd->xname); + BUG_ON(xd->flags & (JFFS2_XFLAGS_DEAD|JFFS2_XFLAGS_INVALID)); + + vecs[0].iov_base = ℞ + vecs[0].iov_len = sizeof(rx); + vecs[1].iov_base = xd->xname; + vecs[1].iov_len = xd->name_len + 1 + xd->value_len; + totlen = vecs[0].iov_len + vecs[1].iov_len; + + /* Setup raw-xattr */ + memset(&rx, 0, sizeof(rx)); + rx.magic = cpu_to_je16(JFFS2_MAGIC_BITMASK); + rx.nodetype = cpu_to_je16(JFFS2_NODETYPE_XATTR); + rx.totlen = cpu_to_je32(PAD(totlen)); + rx.hdr_crc = cpu_to_je32(crc32(0, &rx, sizeof(struct jffs2_unknown_node) - 4)); + + rx.xid = cpu_to_je32(xd->xid); + rx.version = cpu_to_je32(++xd->version); + rx.xprefix = xd->xprefix; + rx.name_len = xd->name_len; + rx.value_len = cpu_to_je16(xd->value_len); + rx.data_crc = cpu_to_je32(crc32(0, vecs[1].iov_base, vecs[1].iov_len)); + rx.node_crc = cpu_to_je32(crc32(0, &rx, sizeof(struct jffs2_raw_xattr) - 4)); + + rc = jffs2_flash_writev(c, vecs, 2, phys_ofs, &length, 0); + if (rc || totlen != length) { + JFFS2_WARNING("jffs2_flash_writev()=%d, req=%u, wrote=%zu, at %#08x\n", + rc, totlen, length, phys_ofs); + rc = rc ? rc : -EIO; + if (length) + jffs2_add_physical_node_ref(c, phys_ofs | REF_OBSOLETE, PAD(totlen), NULL); + + return rc; + } + /* success */ + jffs2_add_physical_node_ref(c, phys_ofs | REF_PRISTINE, PAD(totlen), (void *)xd); + + dbg_xattr("success on saving xdatum (xid=%u, version=%u, xprefix=%u, xname='%s')\n", + xd->xid, xd->version, xd->xprefix, xd->xname); + + return 0; +} + +static struct jffs2_xattr_datum *create_xattr_datum(struct jffs2_sb_info *c, + int xprefix, const char *xname, + const char *xvalue, int xsize) +{ + /* must be called under down_write(xattr_sem) */ + struct jffs2_xattr_datum *xd; + uint32_t hashkey, name_len; + char *data; + int i, rc; + + /* Search xattr_datum has same xname/xvalue by index */ + hashkey = xattr_datum_hashkey(xprefix, xname, xvalue, xsize); + i = hashkey % XATTRINDEX_HASHSIZE; + list_for_each_entry(xd, &c->xattrindex[i], xindex) { + if (xd->hashkey==hashkey + && xd->xprefix==xprefix + && xd->value_len==xsize + && !strcmp(xd->xname, xname) + && !memcmp(xd->xvalue, xvalue, xsize)) { + atomic_inc(&xd->refcnt); + return xd; + } + } + + /* Not found, Create NEW XATTR-Cache */ + name_len = strlen(xname); + + xd = jffs2_alloc_xattr_datum(); + if (!xd) + return ERR_PTR(-ENOMEM); + + data = kmalloc(name_len + 1 + xsize, GFP_KERNEL); + if (!data) { + jffs2_free_xattr_datum(xd); + return ERR_PTR(-ENOMEM); + } + strcpy(data, xname); + memcpy(data + name_len + 1, xvalue, xsize); + + atomic_set(&xd->refcnt, 1); + xd->xid = ++c->highest_xid; + xd->flags |= JFFS2_XFLAGS_HOT; + xd->xprefix = xprefix; + + xd->hashkey = hashkey; + xd->xname = data; + xd->xvalue = data + name_len + 1; + xd->name_len = name_len; + xd->value_len = xsize; + xd->data_crc = crc32(0, data, xd->name_len + 1 + xd->value_len); + + rc = save_xattr_datum(c, xd); + if (rc) { + kfree(xd->xname); + jffs2_free_xattr_datum(xd); + return ERR_PTR(rc); + } + + /* Insert Hash Index */ + i = hashkey % XATTRINDEX_HASHSIZE; + list_add(&xd->xindex, &c->xattrindex[i]); + + c->xdatum_mem_usage += (xd->name_len + 1 + xd->value_len); + reclaim_xattr_datum(c); + + return xd; +} + +static void unrefer_xattr_datum(struct jffs2_sb_info *c, struct jffs2_xattr_datum *xd) +{ + /* must be called under down_write(xattr_sem) */ + if (atomic_dec_and_lock(&xd->refcnt, &c->erase_completion_lock)) { + unload_xattr_datum(c, xd); + xd->flags |= JFFS2_XFLAGS_DEAD; + if (xd->node == (void *)xd) { + BUG_ON(!(xd->flags & JFFS2_XFLAGS_INVALID)); + jffs2_free_xattr_datum(xd); + } else { + list_add(&xd->xindex, &c->xattr_dead_list); + } + spin_unlock(&c->erase_completion_lock); + + dbg_xattr("xdatum(xid=%u, version=%u) was removed.\n", + xd->xid, xd->version); + } +} + +/* -------- xref related functions ------------------ + * verify_xattr_ref(c, ref) + * is used to load xref information from medium. Because summary data does not + * contain xid/ino, it's necessary to verify once while mounting process. + * save_xattr_ref(c, ref) + * is used to write xref to medium. If delete marker is marked, it write + * a delete marker of xref into medium. + * create_xattr_ref(c, ic, xd) + * is used to create a new xref and write to medium. + * delete_xattr_ref(c, ref) + * is used to delete jffs2_xattr_ref. It marks xref XREF_DELETE_MARKER, + * and allows GC to reclaim those physical nodes. + * jffs2_xattr_delete_inode(c, ic) + * is called to remove xrefs related to obsolete inode when inode is unlinked. + * jffs2_xattr_free_inode(c, ic) + * is called to release xattr related objects when unmounting. + * check_xattr_ref_inode(c, ic) + * is used to confirm inode does not have duplicate xattr name/value pair. + * jffs2_xattr_do_crccheck_inode(c, ic) + * is used to force xattr data integrity check during the initial gc scan. + * -------------------------------------------------- */ +static int verify_xattr_ref(struct jffs2_sb_info *c, struct jffs2_xattr_ref *ref) +{ + struct jffs2_eraseblock *jeb; + struct jffs2_raw_node_ref *raw; + struct jffs2_raw_xref rr; + size_t readlen; + uint32_t crc, offset, totlen; + int rc; + + spin_lock(&c->erase_completion_lock); + if (ref_flags(ref->node) != REF_UNCHECKED) + goto complete; + offset = ref_offset(ref->node); + spin_unlock(&c->erase_completion_lock); + + rc = jffs2_flash_read(c, offset, sizeof(rr), &readlen, (char *)&rr); + if (rc || sizeof(rr) != readlen) { + JFFS2_WARNING("jffs2_flash_read()=%d, req=%zu, read=%zu, at %#08x\n", + rc, sizeof(rr), readlen, offset); + return rc ? rc : -EIO; + } + /* obsolete node */ + crc = crc32(0, &rr, sizeof(rr) - 4); + if (crc != je32_to_cpu(rr.node_crc)) { + JFFS2_ERROR("node CRC failed at %#08x, read=%#08x, calc=%#08x\n", + offset, je32_to_cpu(rr.node_crc), crc); + return JFFS2_XATTR_IS_CORRUPTED; + } + if (je16_to_cpu(rr.magic) != JFFS2_MAGIC_BITMASK + || je16_to_cpu(rr.nodetype) != JFFS2_NODETYPE_XREF + || je32_to_cpu(rr.totlen) != PAD(sizeof(rr))) { + JFFS2_ERROR("inconsistent xref at %#08x, magic=%#04x/%#04x, " + "nodetype=%#04x/%#04x, totlen=%u/%zu\n", + offset, je16_to_cpu(rr.magic), JFFS2_MAGIC_BITMASK, + je16_to_cpu(rr.nodetype), JFFS2_NODETYPE_XREF, + je32_to_cpu(rr.totlen), PAD(sizeof(rr))); + return JFFS2_XATTR_IS_CORRUPTED; + } + ref->ino = je32_to_cpu(rr.ino); + ref->xid = je32_to_cpu(rr.xid); + ref->xseqno = je32_to_cpu(rr.xseqno); + if (ref->xseqno > c->highest_xseqno) + c->highest_xseqno = (ref->xseqno & ~XREF_DELETE_MARKER); + + spin_lock(&c->erase_completion_lock); + complete: + for (raw=ref->node; raw != (void *)ref; raw=raw->next_in_ino) { + jeb = &c->blocks[ref_offset(raw) / c->sector_size]; + totlen = PAD(ref_totlen(c, jeb, raw)); + if (ref_flags(raw) == REF_UNCHECKED) { + c->unchecked_size -= totlen; c->used_size += totlen; + jeb->unchecked_size -= totlen; jeb->used_size += totlen; + } + raw->flash_offset = ref_offset(raw) | ((ref->node==raw) ? REF_PRISTINE : REF_NORMAL); + } + spin_unlock(&c->erase_completion_lock); + + dbg_xattr("success on verifying xref (ino=%u, xid=%u) at %#08x\n", + ref->ino, ref->xid, ref_offset(ref->node)); + return 0; +} + +static int save_xattr_ref(struct jffs2_sb_info *c, struct jffs2_xattr_ref *ref) +{ + /* must be called under down_write(xattr_sem) */ + struct jffs2_raw_xref rr; + size_t length; + uint32_t xseqno, phys_ofs = write_ofs(c); + int ret; + + rr.magic = cpu_to_je16(JFFS2_MAGIC_BITMASK); + rr.nodetype = cpu_to_je16(JFFS2_NODETYPE_XREF); + rr.totlen = cpu_to_je32(PAD(sizeof(rr))); + rr.hdr_crc = cpu_to_je32(crc32(0, &rr, sizeof(struct jffs2_unknown_node) - 4)); + + xseqno = (c->highest_xseqno += 2); + if (is_xattr_ref_dead(ref)) { + xseqno |= XREF_DELETE_MARKER; + rr.ino = cpu_to_je32(ref->ino); + rr.xid = cpu_to_je32(ref->xid); + } else { + rr.ino = cpu_to_je32(ref->ic->ino); + rr.xid = cpu_to_je32(ref->xd->xid); + } + rr.xseqno = cpu_to_je32(xseqno); + rr.node_crc = cpu_to_je32(crc32(0, &rr, sizeof(rr) - 4)); + + ret = jffs2_flash_write(c, phys_ofs, sizeof(rr), &length, (char *)&rr); + if (ret || sizeof(rr) != length) { + JFFS2_WARNING("jffs2_flash_write() returned %d, request=%zu, retlen=%zu, at %#08x\n", + ret, sizeof(rr), length, phys_ofs); + ret = ret ? ret : -EIO; + if (length) + jffs2_add_physical_node_ref(c, phys_ofs | REF_OBSOLETE, PAD(sizeof(rr)), NULL); + + return ret; + } + /* success */ + ref->xseqno = xseqno; + jffs2_add_physical_node_ref(c, phys_ofs | REF_PRISTINE, PAD(sizeof(rr)), (void *)ref); + + dbg_xattr("success on saving xref (ino=%u, xid=%u)\n", ref->ic->ino, ref->xd->xid); + + return 0; +} + +static struct jffs2_xattr_ref *create_xattr_ref(struct jffs2_sb_info *c, struct jffs2_inode_cache *ic, + struct jffs2_xattr_datum *xd) +{ + /* must be called under down_write(xattr_sem) */ + struct jffs2_xattr_ref *ref; + int ret; + + ref = jffs2_alloc_xattr_ref(); + if (!ref) + return ERR_PTR(-ENOMEM); + ref->ic = ic; + ref->xd = xd; + + ret = save_xattr_ref(c, ref); + if (ret) { + jffs2_free_xattr_ref(ref); + return ERR_PTR(ret); + } + + /* Chain to inode */ + ref->next = ic->xref; + ic->xref = ref; + + return ref; /* success */ +} + +static void delete_xattr_ref(struct jffs2_sb_info *c, struct jffs2_xattr_ref *ref) +{ + /* must be called under down_write(xattr_sem) */ + struct jffs2_xattr_datum *xd; + + xd = ref->xd; + ref->xseqno |= XREF_DELETE_MARKER; + ref->ino = ref->ic->ino; + ref->xid = ref->xd->xid; + spin_lock(&c->erase_completion_lock); + ref->next = c->xref_dead_list; + c->xref_dead_list = ref; + spin_unlock(&c->erase_completion_lock); + + dbg_xattr("xref(ino=%u, xid=%u, xseqno=%u) was removed.\n", + ref->ino, ref->xid, ref->xseqno); + + unrefer_xattr_datum(c, xd); +} + +void jffs2_xattr_delete_inode(struct jffs2_sb_info *c, struct jffs2_inode_cache *ic) +{ + /* It's called from jffs2_evict_inode() on inode removing. + When an inode with XATTR is removed, those XATTRs must be removed. */ + struct jffs2_xattr_ref *ref, *_ref; + + if (!ic || ic->pino_nlink > 0) + return; + + down_write(&c->xattr_sem); + for (ref = ic->xref; ref; ref = _ref) { + _ref = ref->next; + delete_xattr_ref(c, ref); + } + ic->xref = NULL; + up_write(&c->xattr_sem); +} + +void jffs2_xattr_free_inode(struct jffs2_sb_info *c, struct jffs2_inode_cache *ic) +{ + /* It's called from jffs2_free_ino_caches() until unmounting FS. */ + struct jffs2_xattr_datum *xd; + struct jffs2_xattr_ref *ref, *_ref; + + down_write(&c->xattr_sem); + for (ref = ic->xref; ref; ref = _ref) { + _ref = ref->next; + xd = ref->xd; + if (atomic_dec_and_test(&xd->refcnt)) { + unload_xattr_datum(c, xd); + jffs2_free_xattr_datum(xd); + } + jffs2_free_xattr_ref(ref); + } + ic->xref = NULL; + up_write(&c->xattr_sem); +} + +static int check_xattr_ref_inode(struct jffs2_sb_info *c, struct jffs2_inode_cache *ic) +{ + /* success of check_xattr_ref_inode() means that inode (ic) dose not have + * duplicate name/value pairs. If duplicate name/value pair would be found, + * one will be removed. + */ + struct jffs2_xattr_ref *ref, *cmp, **pref, **pcmp; + int rc = 0; + + if (likely(ic->flags & INO_FLAGS_XATTR_CHECKED)) + return 0; + down_write(&c->xattr_sem); + retry: + rc = 0; + for (ref=ic->xref, pref=&ic->xref; ref; pref=&ref->next, ref=ref->next) { + if (!ref->xd->xname) { + rc = load_xattr_datum(c, ref->xd); + if (unlikely(rc > 0)) { + *pref = ref->next; + delete_xattr_ref(c, ref); + goto retry; + } else if (unlikely(rc < 0)) + goto out; + } + for (cmp=ref->next, pcmp=&ref->next; cmp; pcmp=&cmp->next, cmp=cmp->next) { + if (!cmp->xd->xname) { + ref->xd->flags |= JFFS2_XFLAGS_BIND; + rc = load_xattr_datum(c, cmp->xd); + ref->xd->flags &= ~JFFS2_XFLAGS_BIND; + if (unlikely(rc > 0)) { + *pcmp = cmp->next; + delete_xattr_ref(c, cmp); + goto retry; + } else if (unlikely(rc < 0)) + goto out; + } + if (ref->xd->xprefix == cmp->xd->xprefix + && !strcmp(ref->xd->xname, cmp->xd->xname)) { + if (ref->xseqno > cmp->xseqno) { + *pcmp = cmp->next; + delete_xattr_ref(c, cmp); + } else { + *pref = ref->next; + delete_xattr_ref(c, ref); + } + goto retry; + } + } + } + ic->flags |= INO_FLAGS_XATTR_CHECKED; + out: + up_write(&c->xattr_sem); + + return rc; +} + +void jffs2_xattr_do_crccheck_inode(struct jffs2_sb_info *c, struct jffs2_inode_cache *ic) +{ + check_xattr_ref_inode(c, ic); +} + +/* -------- xattr subsystem functions --------------- + * jffs2_init_xattr_subsystem(c) + * is used to initialize semaphore and list_head, and some variables. + * jffs2_find_xattr_datum(c, xid) + * is used to lookup xdatum while scanning process. + * jffs2_clear_xattr_subsystem(c) + * is used to release any xattr related objects. + * jffs2_build_xattr_subsystem(c) + * is used to associate xdatum and xref while super block building process. + * jffs2_setup_xattr_datum(c, xid, version) + * is used to insert xdatum while scanning process. + * -------------------------------------------------- */ +void jffs2_init_xattr_subsystem(struct jffs2_sb_info *c) +{ + int i; + + for (i=0; i < XATTRINDEX_HASHSIZE; i++) + INIT_LIST_HEAD(&c->xattrindex[i]); + INIT_LIST_HEAD(&c->xattr_unchecked); + INIT_LIST_HEAD(&c->xattr_dead_list); + c->xref_dead_list = NULL; + c->xref_temp = NULL; + + init_rwsem(&c->xattr_sem); + c->highest_xid = 0; + c->highest_xseqno = 0; + c->xdatum_mem_usage = 0; + c->xdatum_mem_threshold = 32 * 1024; /* Default 32KB */ +} + +static struct jffs2_xattr_datum *jffs2_find_xattr_datum(struct jffs2_sb_info *c, uint32_t xid) +{ + struct jffs2_xattr_datum *xd; + int i = xid % XATTRINDEX_HASHSIZE; + + /* It's only used in scanning/building process. */ + BUG_ON(!(c->flags & (JFFS2_SB_FLAG_SCANNING|JFFS2_SB_FLAG_BUILDING))); + + list_for_each_entry(xd, &c->xattrindex[i], xindex) { + if (xd->xid==xid) + return xd; + } + return NULL; +} + +void jffs2_clear_xattr_subsystem(struct jffs2_sb_info *c) +{ + struct jffs2_xattr_datum *xd, *_xd; + struct jffs2_xattr_ref *ref, *_ref; + int i; + + for (ref=c->xref_temp; ref; ref = _ref) { + _ref = ref->next; + jffs2_free_xattr_ref(ref); + } + + for (ref=c->xref_dead_list; ref; ref = _ref) { + _ref = ref->next; + jffs2_free_xattr_ref(ref); + } + + for (i=0; i < XATTRINDEX_HASHSIZE; i++) { + list_for_each_entry_safe(xd, _xd, &c->xattrindex[i], xindex) { + list_del(&xd->xindex); + kfree(xd->xname); + jffs2_free_xattr_datum(xd); + } + } + + list_for_each_entry_safe(xd, _xd, &c->xattr_dead_list, xindex) { + list_del(&xd->xindex); + jffs2_free_xattr_datum(xd); + } + list_for_each_entry_safe(xd, _xd, &c->xattr_unchecked, xindex) { + list_del(&xd->xindex); + jffs2_free_xattr_datum(xd); + } +} + +#define XREF_TMPHASH_SIZE (128) +int jffs2_build_xattr_subsystem(struct jffs2_sb_info *c) +{ + struct jffs2_xattr_ref *ref, *_ref; + struct jffs2_xattr_ref **xref_tmphash; + struct jffs2_xattr_datum *xd, *_xd; + struct jffs2_inode_cache *ic; + struct jffs2_raw_node_ref *raw; + int i, xdatum_count = 0, xdatum_unchecked_count = 0, xref_count = 0; + int xdatum_orphan_count = 0, xref_orphan_count = 0, xref_dead_count = 0; + + BUG_ON(!(c->flags & JFFS2_SB_FLAG_BUILDING)); + + xref_tmphash = kzalloc_objs(struct jffs2_xattr_ref *, XREF_TMPHASH_SIZE); + if (!xref_tmphash) + return -ENOMEM; + + /* Phase.1 : Merge same xref */ + for (ref=c->xref_temp; ref; ref=_ref) { + struct jffs2_xattr_ref *tmp; + + _ref = ref->next; + if (ref_flags(ref->node) != REF_PRISTINE) { + if (verify_xattr_ref(c, ref)) { + BUG_ON(ref->node->next_in_ino != (void *)ref); + ref->node->next_in_ino = NULL; + jffs2_mark_node_obsolete(c, ref->node); + jffs2_free_xattr_ref(ref); + continue; + } + } + + i = (ref->ino ^ ref->xid) % XREF_TMPHASH_SIZE; + for (tmp=xref_tmphash[i]; tmp; tmp=tmp->next) { + if (tmp->ino == ref->ino && tmp->xid == ref->xid) + break; + } + if (tmp) { + raw = ref->node; + if (ref->xseqno > tmp->xseqno) { + tmp->xseqno = ref->xseqno; + raw->next_in_ino = tmp->node; + tmp->node = raw; + } else { + raw->next_in_ino = tmp->node->next_in_ino; + tmp->node->next_in_ino = raw; + } + jffs2_free_xattr_ref(ref); + continue; + } else { + ref->next = xref_tmphash[i]; + xref_tmphash[i] = ref; + } + } + c->xref_temp = NULL; + + /* Phase.2 : Bind xref with inode_cache and xattr_datum */ + for (i=0; i < XREF_TMPHASH_SIZE; i++) { + for (ref=xref_tmphash[i]; ref; ref=_ref) { + xref_count++; + _ref = ref->next; + if (is_xattr_ref_dead(ref)) { + ref->next = c->xref_dead_list; + c->xref_dead_list = ref; + xref_dead_count++; + continue; + } + /* At this point, ref->xid and ref->ino contain XID and inode number. + ref->xd and ref->ic are not valid yet. */ + xd = jffs2_find_xattr_datum(c, ref->xid); + ic = jffs2_get_ino_cache(c, ref->ino); + if (!xd || !ic || !ic->pino_nlink) { + dbg_xattr("xref(ino=%u, xid=%u, xseqno=%u) is orphan.\n", + ref->ino, ref->xid, ref->xseqno); + ref->xseqno |= XREF_DELETE_MARKER; + ref->next = c->xref_dead_list; + c->xref_dead_list = ref; + xref_orphan_count++; + continue; + } + ref->xd = xd; + ref->ic = ic; + atomic_inc(&xd->refcnt); + ref->next = ic->xref; + ic->xref = ref; + } + } + + /* Phase.3 : Link unchecked xdatum to xattr_unchecked list */ + for (i=0; i < XATTRINDEX_HASHSIZE; i++) { + list_for_each_entry_safe(xd, _xd, &c->xattrindex[i], xindex) { + xdatum_count++; + list_del_init(&xd->xindex); + if (!atomic_read(&xd->refcnt)) { + dbg_xattr("xdatum(xid=%u, version=%u) is orphan.\n", + xd->xid, xd->version); + xd->flags |= JFFS2_XFLAGS_DEAD; + list_add(&xd->xindex, &c->xattr_unchecked); + xdatum_orphan_count++; + continue; + } + if (is_xattr_datum_unchecked(c, xd)) { + dbg_xattr("unchecked xdatum(xid=%u, version=%u)\n", + xd->xid, xd->version); + list_add(&xd->xindex, &c->xattr_unchecked); + xdatum_unchecked_count++; + } + } + } + /* build complete */ + JFFS2_NOTICE("complete building xattr subsystem, %u of xdatum" + " (%u unchecked, %u orphan) and " + "%u of xref (%u dead, %u orphan) found.\n", + xdatum_count, xdatum_unchecked_count, xdatum_orphan_count, + xref_count, xref_dead_count, xref_orphan_count); + kfree(xref_tmphash); + return 0; +} + +struct jffs2_xattr_datum *jffs2_setup_xattr_datum(struct jffs2_sb_info *c, + uint32_t xid, uint32_t version) +{ + struct jffs2_xattr_datum *xd; + + xd = jffs2_find_xattr_datum(c, xid); + if (!xd) { + xd = jffs2_alloc_xattr_datum(); + if (!xd) + return ERR_PTR(-ENOMEM); + xd->xid = xid; + xd->version = version; + if (xd->xid > c->highest_xid) + c->highest_xid = xd->xid; + list_add_tail(&xd->xindex, &c->xattrindex[xid % XATTRINDEX_HASHSIZE]); + } + return xd; +} + +/* -------- xattr subsystem functions --------------- + * xprefix_to_handler(xprefix) + * is used to translate xprefix into xattr_handler. + * jffs2_listxattr(dentry, buffer, size) + * is an implementation of listxattr handler on jffs2. + * do_jffs2_getxattr(inode, xprefix, xname, buffer, size) + * is an implementation of getxattr handler on jffs2. + * do_jffs2_setxattr(inode, xprefix, xname, buffer, size, flags) + * is an implementation of setxattr handler on jffs2. + * -------------------------------------------------- */ +const struct xattr_handler * const jffs2_xattr_handlers[] = { + &jffs2_user_xattr_handler, +#ifdef CONFIG_JFFS2_FS_SECURITY + &jffs2_security_xattr_handler, +#endif + &jffs2_trusted_xattr_handler, + NULL +}; + +static const char *jffs2_xattr_prefix(int xprefix, struct dentry *dentry) +{ + const struct xattr_handler *ret = NULL; + + switch (xprefix) { + case JFFS2_XPREFIX_USER: + ret = &jffs2_user_xattr_handler; + break; +#ifdef CONFIG_JFFS2_FS_SECURITY + case JFFS2_XPREFIX_SECURITY: + ret = &jffs2_security_xattr_handler; + break; +#endif +#ifdef CONFIG_JFFS2_FS_POSIX_ACL + case JFFS2_XPREFIX_ACL_ACCESS: + ret = &nop_posix_acl_access; + break; + case JFFS2_XPREFIX_ACL_DEFAULT: + ret = &nop_posix_acl_default; + break; +#endif + case JFFS2_XPREFIX_TRUSTED: + ret = &jffs2_trusted_xattr_handler; + break; + default: + return NULL; + } + + if (!xattr_handler_can_list(ret, dentry)) + return NULL; + + return xattr_prefix(ret); +} + +ssize_t jffs2_listxattr(struct dentry *dentry, char *buffer, size_t size) +{ + struct inode *inode = d_inode(dentry); + struct jffs2_inode_info *f = JFFS2_INODE_INFO(inode); + struct jffs2_sb_info *c = JFFS2_SB_INFO(inode->i_sb); + struct jffs2_inode_cache *ic = f->inocache; + struct jffs2_xattr_ref *ref, **pref; + struct jffs2_xattr_datum *xd; + const char *prefix; + ssize_t prefix_len, len, rc; + int retry = 0; + + rc = check_xattr_ref_inode(c, ic); + if (unlikely(rc)) + return rc; + + down_read(&c->xattr_sem); + retry: + len = 0; + for (ref=ic->xref, pref=&ic->xref; ref; pref=&ref->next, ref=ref->next) { + BUG_ON(ref->ic != ic); + xd = ref->xd; + if (!xd->xname) { + /* xdatum is unchached */ + if (!retry) { + retry = 1; + up_read(&c->xattr_sem); + down_write(&c->xattr_sem); + goto retry; + } else { + rc = load_xattr_datum(c, xd); + if (unlikely(rc > 0)) { + *pref = ref->next; + delete_xattr_ref(c, ref); + goto retry; + } else if (unlikely(rc < 0)) + goto out; + } + } + + prefix = jffs2_xattr_prefix(xd->xprefix, dentry); + if (!prefix) + continue; + prefix_len = strlen(prefix); + rc = prefix_len + xd->name_len + 1; + + if (buffer) { + if (rc > size - len) { + rc = -ERANGE; + goto out; + } + memcpy(buffer, prefix, prefix_len); + buffer += prefix_len; + memcpy(buffer, xd->xname, xd->name_len); + buffer += xd->name_len; + *buffer++ = 0; + } + len += rc; + } + rc = len; + out: + if (!retry) { + up_read(&c->xattr_sem); + } else { + up_write(&c->xattr_sem); + } + return rc; +} + +int do_jffs2_getxattr(struct inode *inode, int xprefix, const char *xname, + char *buffer, size_t size) +{ + struct jffs2_inode_info *f = JFFS2_INODE_INFO(inode); + struct jffs2_sb_info *c = JFFS2_SB_INFO(inode->i_sb); + struct jffs2_inode_cache *ic = f->inocache; + struct jffs2_xattr_datum *xd; + struct jffs2_xattr_ref *ref, **pref; + int rc, retry = 0; + + rc = check_xattr_ref_inode(c, ic); + if (unlikely(rc)) + return rc; + + down_read(&c->xattr_sem); + retry: + for (ref=ic->xref, pref=&ic->xref; ref; pref=&ref->next, ref=ref->next) { + BUG_ON(ref->ic!=ic); + + xd = ref->xd; + if (xd->xprefix != xprefix) + continue; + if (!xd->xname) { + /* xdatum is unchached */ + if (!retry) { + retry = 1; + up_read(&c->xattr_sem); + down_write(&c->xattr_sem); + goto retry; + } else { + rc = load_xattr_datum(c, xd); + if (unlikely(rc > 0)) { + *pref = ref->next; + delete_xattr_ref(c, ref); + goto retry; + } else if (unlikely(rc < 0)) { + goto out; + } + } + } + if (!strcmp(xname, xd->xname)) { + rc = xd->value_len; + if (buffer) { + if (size < rc) { + rc = -ERANGE; + } else { + memcpy(buffer, xd->xvalue, rc); + } + } + goto out; + } + } + rc = -ENODATA; + out: + if (!retry) { + up_read(&c->xattr_sem); + } else { + up_write(&c->xattr_sem); + } + return rc; +} + +int do_jffs2_setxattr(struct inode *inode, int xprefix, const char *xname, + const char *buffer, size_t size, int flags) +{ + struct jffs2_inode_info *f = JFFS2_INODE_INFO(inode); + struct jffs2_sb_info *c = JFFS2_SB_INFO(inode->i_sb); + struct jffs2_inode_cache *ic = f->inocache; + struct jffs2_xattr_datum *xd; + struct jffs2_xattr_ref *ref, *newref, **pref; + uint32_t length, request; + int rc; + + rc = check_xattr_ref_inode(c, ic); + if (unlikely(rc)) + return rc; + + request = PAD(sizeof(struct jffs2_raw_xattr) + strlen(xname) + 1 + size); + if (request > c->sector_size - c->cleanmarker_size) + return -ERANGE; + + rc = jffs2_reserve_space(c, request, &length, + ALLOC_NORMAL, JFFS2_SUMMARY_XATTR_SIZE); + if (rc) { + JFFS2_WARNING("jffs2_reserve_space()=%d, request=%u\n", rc, request); + return rc; + } + + /* Find existing xattr */ + down_write(&c->xattr_sem); + retry: + for (ref=ic->xref, pref=&ic->xref; ref; pref=&ref->next, ref=ref->next) { + xd = ref->xd; + if (xd->xprefix != xprefix) + continue; + if (!xd->xname) { + rc = load_xattr_datum(c, xd); + if (unlikely(rc > 0)) { + *pref = ref->next; + delete_xattr_ref(c, ref); + goto retry; + } else if (unlikely(rc < 0)) + goto out; + } + if (!strcmp(xd->xname, xname)) { + if (flags & XATTR_CREATE) { + rc = -EEXIST; + goto out; + } + if (!buffer) { + ref->ino = ic->ino; + ref->xid = xd->xid; + ref->xseqno |= XREF_DELETE_MARKER; + rc = save_xattr_ref(c, ref); + if (!rc) { + *pref = ref->next; + spin_lock(&c->erase_completion_lock); + ref->next = c->xref_dead_list; + c->xref_dead_list = ref; + spin_unlock(&c->erase_completion_lock); + unrefer_xattr_datum(c, xd); + } else { + ref->ic = ic; + ref->xd = xd; + ref->xseqno &= ~XREF_DELETE_MARKER; + } + goto out; + } + goto found; + } + } + /* not found */ + if (flags & XATTR_REPLACE) { + rc = -ENODATA; + goto out; + } + if (!buffer) { + rc = -ENODATA; + goto out; + } + found: + xd = create_xattr_datum(c, xprefix, xname, buffer, size); + if (IS_ERR(xd)) { + rc = PTR_ERR(xd); + goto out; + } + up_write(&c->xattr_sem); + jffs2_complete_reservation(c); + + /* create xattr_ref */ + request = PAD(sizeof(struct jffs2_raw_xref)); + rc = jffs2_reserve_space(c, request, &length, + ALLOC_NORMAL, JFFS2_SUMMARY_XREF_SIZE); + down_write(&c->xattr_sem); + if (rc) { + JFFS2_WARNING("jffs2_reserve_space()=%d, request=%u\n", rc, request); + unrefer_xattr_datum(c, xd); + up_write(&c->xattr_sem); + return rc; + } + if (ref) + *pref = ref->next; + newref = create_xattr_ref(c, ic, xd); + if (IS_ERR(newref)) { + if (ref) { + ref->next = ic->xref; + ic->xref = ref; + } + rc = PTR_ERR(newref); + unrefer_xattr_datum(c, xd); + } else if (ref) { + delete_xattr_ref(c, ref); + } + out: + up_write(&c->xattr_sem); + jffs2_complete_reservation(c); + return rc; +} + +/* -------- garbage collector functions ------------- + * jffs2_garbage_collect_xattr_datum(c, xd, raw) + * is used to move xdatum into new node. + * jffs2_garbage_collect_xattr_ref(c, ref, raw) + * is used to move xref into new node. + * jffs2_verify_xattr(c) + * is used to call do_verify_xattr_datum() before garbage collecting. + * jffs2_release_xattr_datum(c, xd) + * is used to release an in-memory object of xdatum. + * jffs2_release_xattr_ref(c, ref) + * is used to release an in-memory object of xref. + * -------------------------------------------------- */ +int jffs2_garbage_collect_xattr_datum(struct jffs2_sb_info *c, struct jffs2_xattr_datum *xd, + struct jffs2_raw_node_ref *raw) +{ + uint32_t totlen, length, old_ofs; + int rc = 0; + + down_write(&c->xattr_sem); + if (xd->node != raw) + goto out; + if (xd->flags & (JFFS2_XFLAGS_DEAD|JFFS2_XFLAGS_INVALID)) + goto out; + + rc = load_xattr_datum(c, xd); + if (unlikely(rc)) { + rc = (rc > 0) ? 0 : rc; + goto out; + } + old_ofs = ref_offset(xd->node); + totlen = PAD(sizeof(struct jffs2_raw_xattr) + + xd->name_len + 1 + xd->value_len); + rc = jffs2_reserve_space_gc(c, totlen, &length, JFFS2_SUMMARY_XATTR_SIZE); + if (rc) { + JFFS2_WARNING("jffs2_reserve_space_gc()=%d, request=%u\n", rc, totlen); + goto out; + } + rc = save_xattr_datum(c, xd); + if (!rc) + dbg_xattr("xdatum (xid=%u, version=%u) GC'ed from %#08x to %08x\n", + xd->xid, xd->version, old_ofs, ref_offset(xd->node)); + out: + if (!rc) + jffs2_mark_node_obsolete(c, raw); + up_write(&c->xattr_sem); + return rc; +} + +int jffs2_garbage_collect_xattr_ref(struct jffs2_sb_info *c, struct jffs2_xattr_ref *ref, + struct jffs2_raw_node_ref *raw) +{ + uint32_t totlen, length, old_ofs; + int rc = 0; + + down_write(&c->xattr_sem); + BUG_ON(!ref->node); + + if (ref->node != raw) + goto out; + if (is_xattr_ref_dead(ref) && (raw->next_in_ino == (void *)ref)) + goto out; + + old_ofs = ref_offset(ref->node); + totlen = ref_totlen(c, c->gcblock, ref->node); + + rc = jffs2_reserve_space_gc(c, totlen, &length, JFFS2_SUMMARY_XREF_SIZE); + if (rc) { + JFFS2_WARNING("%s: jffs2_reserve_space_gc() = %d, request = %u\n", + __func__, rc, totlen); + goto out; + } + rc = save_xattr_ref(c, ref); + if (!rc) + dbg_xattr("xref (ino=%u, xid=%u) GC'ed from %#08x to %08x\n", + ref->ic->ino, ref->xd->xid, old_ofs, ref_offset(ref->node)); + out: + if (!rc) + jffs2_mark_node_obsolete(c, raw); + up_write(&c->xattr_sem); + return rc; +} + +int jffs2_verify_xattr(struct jffs2_sb_info *c) +{ + struct jffs2_xattr_datum *xd, *_xd; + struct jffs2_eraseblock *jeb; + struct jffs2_raw_node_ref *raw; + uint32_t totlen; + int rc; + + down_write(&c->xattr_sem); + list_for_each_entry_safe(xd, _xd, &c->xattr_unchecked, xindex) { + rc = do_verify_xattr_datum(c, xd); + if (rc < 0) + continue; + list_del_init(&xd->xindex); + spin_lock(&c->erase_completion_lock); + for (raw=xd->node; raw != (void *)xd; raw=raw->next_in_ino) { + if (ref_flags(raw) != REF_UNCHECKED) + continue; + jeb = &c->blocks[ref_offset(raw) / c->sector_size]; + totlen = PAD(ref_totlen(c, jeb, raw)); + c->unchecked_size -= totlen; c->used_size += totlen; + jeb->unchecked_size -= totlen; jeb->used_size += totlen; + raw->flash_offset = ref_offset(raw) + | ((xd->node == (void *)raw) ? REF_PRISTINE : REF_NORMAL); + } + if (xd->flags & JFFS2_XFLAGS_DEAD) + list_add(&xd->xindex, &c->xattr_dead_list); + spin_unlock(&c->erase_completion_lock); + } + up_write(&c->xattr_sem); + return list_empty(&c->xattr_unchecked) ? 1 : 0; +} + +void jffs2_release_xattr_datum(struct jffs2_sb_info *c, struct jffs2_xattr_datum *xd) +{ + /* must be called under spin_lock(&c->erase_completion_lock) */ + if (atomic_read(&xd->refcnt) || xd->node != (void *)xd) + return; + + list_del(&xd->xindex); + jffs2_free_xattr_datum(xd); +} + +void jffs2_release_xattr_ref(struct jffs2_sb_info *c, struct jffs2_xattr_ref *ref) +{ + /* must be called under spin_lock(&c->erase_completion_lock) */ + struct jffs2_xattr_ref *tmp, **ptmp; + + if (ref->node != (void *)ref) + return; + + for (tmp=c->xref_dead_list, ptmp=&c->xref_dead_list; tmp; ptmp=&tmp->next, tmp=tmp->next) { + if (ref == tmp) { + *ptmp = tmp->next; + break; + } + } + jffs2_free_xattr_ref(ref); +} diff --git a/fs/jffs2/xattr.h b/fs/jffs2/xattr.h new file mode 100644 index 000000000..7e7de093e --- /dev/null +++ b/fs/jffs2/xattr.h @@ -0,0 +1,127 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2006 NEC Corporation + * + * Created by KaiGai Kohei + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#ifndef _JFFS2_FS_XATTR_H_ +#define _JFFS2_FS_XATTR_H_ + +#include +#include + +#define JFFS2_XFLAGS_HOT (0x01) /* This datum is HOT */ +#define JFFS2_XFLAGS_BIND (0x02) /* This datum is not reclaimed */ +#define JFFS2_XFLAGS_DEAD (0x40) /* This datum is already dead */ +#define JFFS2_XFLAGS_INVALID (0x80) /* This datum contains crc error */ + +struct jffs2_xattr_datum +{ + void *always_null; + struct jffs2_raw_node_ref *node; + uint8_t class; + uint8_t flags; + uint16_t xprefix; /* see JFFS2_XATTR_PREFIX_* */ + + struct list_head xindex; /* chained from c->xattrindex[n] */ + atomic_t refcnt; /* # of xattr_ref refers this */ + uint32_t xid; + uint32_t version; + + uint32_t data_crc; + uint32_t hashkey; + char *xname; /* XATTR name without prefix */ + uint32_t name_len; /* length of xname */ + char *xvalue; /* XATTR value */ + uint32_t value_len; /* length of xvalue */ +}; + +struct jffs2_inode_cache; +struct jffs2_xattr_ref +{ + void *always_null; + struct jffs2_raw_node_ref *node; + uint8_t class; + uint8_t flags; /* Currently unused */ + u16 unused; + + uint32_t xseqno; + union { + struct jffs2_inode_cache *ic; /* reference to jffs2_inode_cache */ + uint32_t ino; /* only used in scanning/building */ + }; + union { + struct jffs2_xattr_datum *xd; /* reference to jffs2_xattr_datum */ + uint32_t xid; /* only used in sccanning/building */ + }; + struct jffs2_xattr_ref *next; /* chained from ic->xref_list */ +}; + +#define XREF_DELETE_MARKER (0x00000001) +static inline int is_xattr_ref_dead(struct jffs2_xattr_ref *ref) +{ + return ((ref->xseqno & XREF_DELETE_MARKER) != 0); +} + +#ifdef CONFIG_JFFS2_FS_XATTR + +extern void jffs2_init_xattr_subsystem(struct jffs2_sb_info *c); +extern int jffs2_build_xattr_subsystem(struct jffs2_sb_info *c); +extern void jffs2_clear_xattr_subsystem(struct jffs2_sb_info *c); + +extern struct jffs2_xattr_datum *jffs2_setup_xattr_datum(struct jffs2_sb_info *c, + uint32_t xid, uint32_t version); + +extern void jffs2_xattr_do_crccheck_inode(struct jffs2_sb_info *c, struct jffs2_inode_cache *ic); +extern void jffs2_xattr_delete_inode(struct jffs2_sb_info *c, struct jffs2_inode_cache *ic); +extern void jffs2_xattr_free_inode(struct jffs2_sb_info *c, struct jffs2_inode_cache *ic); + +extern int jffs2_garbage_collect_xattr_datum(struct jffs2_sb_info *c, struct jffs2_xattr_datum *xd, + struct jffs2_raw_node_ref *raw); +extern int jffs2_garbage_collect_xattr_ref(struct jffs2_sb_info *c, struct jffs2_xattr_ref *ref, + struct jffs2_raw_node_ref *raw); +extern int jffs2_verify_xattr(struct jffs2_sb_info *c); +extern void jffs2_release_xattr_datum(struct jffs2_sb_info *c, struct jffs2_xattr_datum *xd); +extern void jffs2_release_xattr_ref(struct jffs2_sb_info *c, struct jffs2_xattr_ref *ref); + +extern int do_jffs2_getxattr(struct inode *inode, int xprefix, const char *xname, + char *buffer, size_t size); +extern int do_jffs2_setxattr(struct inode *inode, int xprefix, const char *xname, + const char *buffer, size_t size, int flags); + +extern const struct xattr_handler * const jffs2_xattr_handlers[]; +extern const struct xattr_handler jffs2_user_xattr_handler; +extern const struct xattr_handler jffs2_trusted_xattr_handler; + +extern ssize_t jffs2_listxattr(struct dentry *, char *, size_t); + +#else + +#define jffs2_init_xattr_subsystem(c) +#define jffs2_build_xattr_subsystem(c) (0) +#define jffs2_clear_xattr_subsystem(c) + +#define jffs2_xattr_do_crccheck_inode(c, ic) +#define jffs2_xattr_delete_inode(c, ic) +#define jffs2_xattr_free_inode(c, ic) +#define jffs2_verify_xattr(c) (1) + +#define jffs2_xattr_handlers NULL +#define jffs2_listxattr NULL + +#endif /* CONFIG_JFFS2_FS_XATTR */ + +#ifdef CONFIG_JFFS2_FS_SECURITY +extern int jffs2_init_security(struct inode *inode, struct inode *dir, + const struct qstr *qstr); +extern const struct xattr_handler jffs2_security_xattr_handler; +#else +#define jffs2_init_security(inode,dir,qstr) (0) +#endif /* CONFIG_JFFS2_FS_SECURITY */ + +#endif /* _JFFS2_FS_XATTR_H_ */ diff --git a/fs/jffs2/xattr_trusted.c b/fs/jffs2/xattr_trusted.c new file mode 100644 index 000000000..b7c5da2d8 --- /dev/null +++ b/fs/jffs2/xattr_trusted.c @@ -0,0 +1,47 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2006 NEC Corporation + * + * Created by KaiGai Kohei + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#include +#include +#include +#include +#include +#include "nodelist.h" + +static int jffs2_trusted_getxattr(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + return do_jffs2_getxattr(inode, JFFS2_XPREFIX_TRUSTED, + name, buffer, size); +} + +static int jffs2_trusted_setxattr(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *buffer, + size_t size, int flags) +{ + return do_jffs2_setxattr(inode, JFFS2_XPREFIX_TRUSTED, + name, buffer, size, flags); +} + +static bool jffs2_trusted_listxattr(struct dentry *dentry) +{ + return capable(CAP_SYS_ADMIN); +} + +const struct xattr_handler jffs2_trusted_xattr_handler = { + .prefix = XATTR_TRUSTED_PREFIX, + .list = jffs2_trusted_listxattr, + .set = jffs2_trusted_setxattr, + .get = jffs2_trusted_getxattr +}; diff --git a/fs/jffs2/xattr_user.c b/fs/jffs2/xattr_user.c new file mode 100644 index 000000000..f64edce49 --- /dev/null +++ b/fs/jffs2/xattr_user.c @@ -0,0 +1,41 @@ +/* + * JFFS2 -- Journalling Flash File System, Version 2. + * + * Copyright © 2006 NEC Corporation + * + * Created by KaiGai Kohei + * + * For licensing information, see the file 'LICENCE' in this directory. + * + */ + +#include +#include +#include +#include +#include +#include "nodelist.h" + +static int jffs2_user_getxattr(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + return do_jffs2_getxattr(inode, JFFS2_XPREFIX_USER, + name, buffer, size); +} + +static int jffs2_user_setxattr(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *buffer, + size_t size, int flags) +{ + return do_jffs2_setxattr(inode, JFFS2_XPREFIX_USER, + name, buffer, size, flags); +} + +const struct xattr_handler jffs2_user_xattr_handler = { + .prefix = XATTR_USER_PREFIX, + .set = jffs2_user_setxattr, + .get = jffs2_user_getxattr +}; diff --git a/fs/jfs/Kconfig b/fs/jfs/Kconfig new file mode 100644 index 000000000..3728cf4d9 --- /dev/null +++ b/fs/jfs/Kconfig @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: GPL-2.0-only +config JFS_FS + tristate "JFS filesystem support" + select BUFFER_HEAD + select NLS + select NLS_UCS2_UTILS + select CRC32 + select LEGACY_DIRECT_IO + help + This is a port of IBM's Journaled Filesystem . More information is + available in the file . + + If you do not intend to use the JFS filesystem, say N. + +config JFS_POSIX_ACL + bool "JFS POSIX Access Control Lists" + depends on JFS_FS + select FS_POSIX_ACL + help + Posix Access Control Lists (ACLs) support permissions for users and + groups beyond the owner/group/world scheme. + + If you don't know what Access Control Lists are, say N + +config JFS_SECURITY + bool "JFS Security Labels" + depends on JFS_FS + help + Security labels support alternative access control models + implemented by security modules like SELinux. This option + enables an extended attribute handler for file security + labels in the jfs filesystem. + + If you are not using a security module that requires using + extended attributes for file security labels, say N. + +config JFS_DEBUG + bool "JFS debugging" + depends on JFS_FS + help + If you are experiencing any problems with the JFS filesystem, say + Y here. This will result in additional debugging messages to be + written to the system log. Under normal circumstances, this + results in very little overhead. + +config JFS_STATISTICS + bool "JFS statistics" + depends on JFS_FS + help + Enabling this option will cause statistics from the JFS file system + to be made available to the user in the /proc/fs/jfs/ directory. diff --git a/fs/jfs/Makefile b/fs/jfs/Makefile new file mode 100644 index 000000000..b769bbf8b --- /dev/null +++ b/fs/jfs/Makefile @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the Linux JFS filesystem routines. +# + +obj-$(CONFIG_JFS_FS) += jfs.o + +jfs-y := super.o file.o inode.o namei.o jfs_mount.o jfs_umount.o \ + jfs_xtree.o jfs_imap.o jfs_debug.o jfs_dmap.o \ + jfs_unicode.o jfs_dtree.o jfs_inode.o jfs_discard.o \ + jfs_extent.o symlink.o jfs_metapage.o \ + jfs_logmgr.o jfs_txnmgr.o \ + resize.o xattr.o ioctl.o + +jfs-$(CONFIG_JFS_POSIX_ACL) += acl.o diff --git a/fs/jfs/acl.c b/fs/jfs/acl.c new file mode 100644 index 000000000..16b71a23f --- /dev/null +++ b/fs/jfs/acl.c @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) International Business Machines Corp., 2002-2004 + * Copyright (C) Andreas Gruenbacher, 2001 + * Copyright (C) Linus Torvalds, 1991, 1992 + */ + +#include +#include +#include +#include +#include "jfs_incore.h" +#include "jfs_txnmgr.h" +#include "jfs_xattr.h" +#include "jfs_acl.h" + +struct posix_acl *jfs_get_acl(struct inode *inode, int type, bool rcu) +{ + struct posix_acl *acl; + char *ea_name; + int size; + char *value = NULL; + + if (rcu) + return ERR_PTR(-ECHILD); + + switch(type) { + case ACL_TYPE_ACCESS: + ea_name = XATTR_NAME_POSIX_ACL_ACCESS; + break; + case ACL_TYPE_DEFAULT: + ea_name = XATTR_NAME_POSIX_ACL_DEFAULT; + break; + default: + return ERR_PTR(-EINVAL); + } + + size = __jfs_getxattr(inode, ea_name, NULL, 0); + + if (size > 0) { + value = kmalloc(size, GFP_KERNEL); + if (!value) + return ERR_PTR(-ENOMEM); + size = __jfs_getxattr(inode, ea_name, value, size); + } + + if (size < 0) { + if (size == -ENODATA) + acl = NULL; + else + acl = ERR_PTR(size); + } else { + acl = posix_acl_from_xattr(&init_user_ns, value, size); + } + kfree(value); + return acl; +} + +static int __jfs_set_acl(tid_t tid, struct inode *inode, int type, + struct posix_acl *acl) +{ + char *ea_name; + int rc; + size_t size = 0; + char *value = NULL; + + switch (type) { + case ACL_TYPE_ACCESS: + ea_name = XATTR_NAME_POSIX_ACL_ACCESS; + break; + case ACL_TYPE_DEFAULT: + ea_name = XATTR_NAME_POSIX_ACL_DEFAULT; + break; + default: + return -EINVAL; + } + + if (acl) { + value = posix_acl_to_xattr(&init_user_ns, acl, &size, GFP_KERNEL); + if (!value) + return -ENOMEM; + } + rc = __jfs_setxattr(tid, inode, ea_name, value, size, 0); + kfree(value); + + if (!rc) + set_cached_acl(inode, type, acl); + + return rc; +} + +int jfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type) +{ + int rc; + tid_t tid; + int update_mode = 0; + struct inode *inode = d_inode(dentry); + umode_t mode = inode->i_mode; + + tid = txBegin(inode->i_sb, 0); + mutex_lock(&JFS_IP(inode)->commit_mutex); + if (type == ACL_TYPE_ACCESS && acl) { + rc = posix_acl_update_mode(&nop_mnt_idmap, inode, &mode, &acl); + if (rc) + goto end_tx; + if (mode != inode->i_mode) + update_mode = 1; + } + rc = __jfs_set_acl(tid, inode, type, acl); + if (!rc) { + if (update_mode) { + inode->i_mode = mode; + inode_set_ctime_current(inode); + mark_inode_dirty(inode); + } + rc = txCommit(tid, 1, &inode, 0); + } +end_tx: + txEnd(tid); + mutex_unlock(&JFS_IP(inode)->commit_mutex); + return rc; +} + +int jfs_init_acl(tid_t tid, struct inode *inode, struct inode *dir) +{ + struct posix_acl *default_acl, *acl; + int rc = 0; + + rc = posix_acl_create(dir, &inode->i_mode, &default_acl, &acl); + if (rc) + return rc; + + if (default_acl) { + rc = __jfs_set_acl(tid, inode, ACL_TYPE_DEFAULT, default_acl); + posix_acl_release(default_acl); + } else { + inode->i_default_acl = NULL; + } + + if (acl) { + if (!rc) + rc = __jfs_set_acl(tid, inode, ACL_TYPE_ACCESS, acl); + posix_acl_release(acl); + } else { + inode->i_acl = NULL; + } + + JFS_IP(inode)->mode2 = (JFS_IP(inode)->mode2 & 0xffff0000) | + inode->i_mode; + + return rc; +} diff --git a/fs/jfs/file.c b/fs/jfs/file.c new file mode 100644 index 000000000..246568cb9 --- /dev/null +++ b/fs/jfs/file.c @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) International Business Machines Corp., 2000-2002 + * Portions Copyright (C) Christoph Hellwig, 2001-2002 + */ + +#include +#include +#include +#include +#include +#include "jfs_incore.h" +#include "jfs_inode.h" +#include "jfs_dmap.h" +#include "jfs_txnmgr.h" +#include "jfs_xattr.h" +#include "jfs_acl.h" +#include "jfs_debug.h" + +int jfs_fsync(struct file *file, loff_t start, loff_t end, int datasync) +{ + struct inode *inode = file->f_mapping->host; + int rc = 0; + + rc = file_write_and_wait_range(file, start, end); + if (rc) + return rc; + + inode_lock(inode); + if (!(inode_state_read_once(inode) & I_DIRTY_ALL) || + (datasync && !(inode_state_read_once(inode) & I_DIRTY_DATASYNC))) { + /* Make sure committed changes hit the disk */ + jfs_flush_journal(JFS_SBI(inode->i_sb)->log, 1); + inode_unlock(inode); + return rc; + } + + rc |= jfs_commit_inode(inode, 1); + inode_unlock(inode); + + return rc ? -EIO : 0; +} + +static int jfs_open(struct inode *inode, struct file *file) +{ + int rc; + + if (S_ISREG(inode->i_mode) && inode->i_size < 0) + return -EIO; + + if ((rc = dquot_file_open(inode, file))) + return rc; + + /* + * We attempt to allow only one "active" file open per aggregate + * group. Otherwise, appending to files in parallel can cause + * fragmentation within the files. + * + * If the file is empty, it was probably just created and going + * to be written to. If it has a size, we'll hold off until the + * file is actually grown. + */ + if (S_ISREG(inode->i_mode) && file->f_mode & FMODE_WRITE && + (inode->i_size == 0)) { + struct jfs_inode_info *ji = JFS_IP(inode); + spin_lock_irq(&ji->ag_lock); + if (ji->active_ag == -1) { + struct jfs_sb_info *jfs_sb = JFS_SBI(inode->i_sb); + ji->active_ag = BLKTOAG(addressPXD(&ji->ixpxd), jfs_sb); + atomic_inc(&jfs_sb->bmap->db_active[ji->active_ag]); + } + spin_unlock_irq(&ji->ag_lock); + } + + return 0; +} +static int jfs_release(struct inode *inode, struct file *file) +{ + struct jfs_inode_info *ji = JFS_IP(inode); + + spin_lock_irq(&ji->ag_lock); + if (ji->active_ag != -1) { + struct bmap *bmap = JFS_SBI(inode->i_sb)->bmap; + atomic_dec(&bmap->db_active[ji->active_ag]); + ji->active_ag = -1; + } + spin_unlock_irq(&ji->ag_lock); + + return 0; +} + +int jfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *iattr) +{ + struct inode *inode = d_inode(dentry); + int rc; + + rc = setattr_prepare(&nop_mnt_idmap, dentry, iattr); + if (rc) + return rc; + + if (is_quota_modification(&nop_mnt_idmap, inode, iattr)) { + rc = dquot_initialize(inode); + if (rc) + return rc; + } + if ((iattr->ia_valid & ATTR_UID && !uid_eq(iattr->ia_uid, inode->i_uid)) || + (iattr->ia_valid & ATTR_GID && !gid_eq(iattr->ia_gid, inode->i_gid))) { + rc = dquot_transfer(&nop_mnt_idmap, inode, iattr); + if (rc) + return rc; + } + + if ((iattr->ia_valid & ATTR_SIZE) && + iattr->ia_size != i_size_read(inode)) { + inode_dio_wait(inode); + + rc = inode_newsize_ok(inode, iattr->ia_size); + if (rc) + return rc; + + truncate_setsize(inode, iattr->ia_size); + jfs_truncate(inode); + } + + setattr_copy(&nop_mnt_idmap, inode, iattr); + mark_inode_dirty(inode); + + if (iattr->ia_valid & ATTR_MODE) + rc = posix_acl_chmod(&nop_mnt_idmap, dentry, inode->i_mode); + return rc; +} + +const struct inode_operations jfs_file_inode_operations = { + .listxattr = jfs_listxattr, + .setattr = jfs_setattr, + .fileattr_get = jfs_fileattr_get, + .fileattr_set = jfs_fileattr_set, +#ifdef CONFIG_JFS_POSIX_ACL + .get_inode_acl = jfs_get_acl, + .set_acl = jfs_set_acl, +#endif +}; + +const struct file_operations jfs_file_operations = { + .open = jfs_open, + .llseek = generic_file_llseek, + .read_iter = generic_file_read_iter, + .write_iter = generic_file_write_iter, + .mmap_prepare = generic_file_mmap_prepare, + .splice_read = filemap_splice_read, + .splice_write = iter_file_splice_write, + .fsync = jfs_fsync, + .release = jfs_release, + .unlocked_ioctl = jfs_ioctl, + .compat_ioctl = compat_ptr_ioctl, + .setlease = generic_setlease, +}; diff --git a/fs/jfs/inode.c b/fs/jfs/inode.c new file mode 100644 index 000000000..c7914dbc9 --- /dev/null +++ b/fs/jfs/inode.c @@ -0,0 +1,422 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) International Business Machines Corp., 2000-2004 + * Portions Copyright (C) Christoph Hellwig, 2001-2002 + */ + +#include +#include +#include +#include +#include +#include +#include +#include "jfs_incore.h" +#include "jfs_inode.h" +#include "jfs_filsys.h" +#include "jfs_imap.h" +#include "jfs_extent.h" +#include "jfs_unicode.h" +#include "jfs_debug.h" +#include "jfs_dmap.h" + + +struct inode *jfs_iget(struct super_block *sb, unsigned long ino) +{ + struct inode *inode; + int ret; + + inode = iget_locked(sb, ino); + if (!inode) + return ERR_PTR(-ENOMEM); + if (!(inode_state_read_once(inode) & I_NEW)) + return inode; + + ret = diRead(inode); + if (ret < 0) { + iget_failed(inode); + return ERR_PTR(ret); + } + + if (S_ISREG(inode->i_mode)) { + inode->i_op = &jfs_file_inode_operations; + inode->i_fop = &jfs_file_operations; + inode->i_mapping->a_ops = &jfs_aops; + } else if (S_ISDIR(inode->i_mode)) { + inode->i_op = &jfs_dir_inode_operations; + inode->i_fop = &jfs_dir_operations; + } else if (S_ISLNK(inode->i_mode)) { + if (inode->i_size >= IDATASIZE) { + inode->i_op = &page_symlink_inode_operations; + inode_nohighmem(inode); + inode->i_mapping->a_ops = &jfs_aops; + } else { + inode->i_op = &jfs_fast_symlink_inode_operations; + inode->i_link = JFS_IP(inode)->i_inline; + /* + * The inline data should be null-terminated, but + * don't let on-disk corruption crash the kernel + */ + inode->i_link[inode->i_size] = '\0'; + } + } else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) || + S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) { + inode->i_op = &jfs_file_inode_operations; + init_special_inode(inode, inode->i_mode, inode->i_rdev); + } else { + printk(KERN_DEBUG "JFS: Invalid file type 0%04o for inode %llu.\n", + inode->i_mode, inode->i_ino); + iget_failed(inode); + return ERR_PTR(-EIO); + } + unlock_new_inode(inode); + return inode; +} + +/* + * Workhorse of both fsync & write_inode + */ +int jfs_commit_inode(struct inode *inode, int wait) +{ + int rc = 0; + tid_t tid; + static int noisy = 5; + + jfs_info("In jfs_commit_inode, inode = 0x%p", inode); + + /* + * Don't commit if inode has been committed since last being + * marked dirty, or if it has been deleted. + */ + if (inode->i_nlink == 0 || !test_cflag(COMMIT_Dirty, inode)) + return 0; + + if (isReadOnly(inode)) { + /* kernel allows writes to devices on read-only + * partitions and may think inode is dirty + */ + if (!special_file(inode->i_mode) && noisy) { + jfs_err("jfs_commit_inode(0x%p) called on read-only volume", + inode); + jfs_err("Is remount racy?"); + noisy--; + } + return 0; + } + + tid = txBegin(inode->i_sb, COMMIT_INODE); + mutex_lock(&JFS_IP(inode)->commit_mutex); + + /* + * Retest inode state after taking commit_mutex + */ + if (inode->i_nlink && test_cflag(COMMIT_Dirty, inode)) + rc = txCommit(tid, 1, &inode, wait ? COMMIT_SYNC : 0); + + txEnd(tid); + mutex_unlock(&JFS_IP(inode)->commit_mutex); + return rc; +} + +int jfs_write_inode(struct inode *inode, struct writeback_control *wbc) +{ + int wait = wbc->sync_mode == WB_SYNC_ALL; + + if (inode->i_nlink == 0) + return 0; + /* + * If COMMIT_DIRTY is not set, the inode isn't really dirty. + * It has been committed since the last change, but was still + * on the dirty inode list. + */ + if (!test_cflag(COMMIT_Dirty, inode)) { + /* Make sure committed changes hit the disk */ + jfs_flush_journal(JFS_SBI(inode->i_sb)->log, wait); + return 0; + } + + if (jfs_commit_inode(inode, wait)) { + jfs_err("jfs_write_inode: jfs_commit_inode failed!"); + return -EIO; + } else + return 0; +} + +void jfs_evict_inode(struct inode *inode) +{ + struct jfs_inode_info *ji = JFS_IP(inode); + + jfs_info("In jfs_evict_inode, inode = 0x%p", inode); + + if (!inode->i_nlink && !is_bad_inode(inode)) { + dquot_initialize(inode); + + truncate_inode_pages_final(&inode->i_data); + if (JFS_IP(inode)->fileset == FILESYSTEM_I) { + struct inode *ipimap = JFS_SBI(inode->i_sb)->ipimap; + + if (test_cflag(COMMIT_Freewmap, inode)) + jfs_free_zero_link(inode); + + if (ipimap && JFS_IP(ipimap)->i_imap) + diFree(inode); + + /* + * Free the inode from the quota allocation. + */ + dquot_free_inode(inode); + } + } else { + truncate_inode_pages_final(&inode->i_data); + } + clear_inode(inode); + dquot_drop(inode); + + BUG_ON(!list_empty(&ji->anon_inode_list)); + + spin_lock_irq(&ji->ag_lock); + if (ji->active_ag != -1) { + struct bmap *bmap = JFS_SBI(inode->i_sb)->bmap; + atomic_dec(&bmap->db_active[ji->active_ag]); + ji->active_ag = -1; + } + spin_unlock_irq(&ji->ag_lock); +} + +void jfs_dirty_inode(struct inode *inode, int flags) +{ + static int noisy = 5; + + if (isReadOnly(inode)) { + if (!special_file(inode->i_mode) && noisy) { + /* kernel allows writes to devices on read-only + * partitions and may try to mark inode dirty + */ + jfs_err("jfs_dirty_inode called on read-only volume"); + jfs_err("Is remount racy?"); + noisy--; + } + return; + } + + set_cflag(COMMIT_Dirty, inode); +} + +int jfs_get_block(struct inode *ip, sector_t lblock, + struct buffer_head *bh_result, int create) +{ + s64 lblock64 = lblock; + int rc = 0; + xad_t xad; + s64 xaddr; + int xflag; + s32 xlen = bh_result->b_size >> ip->i_blkbits; + + /* + * Take appropriate lock on inode + */ + if (create) + IWRITE_LOCK(ip, RDWRLOCK_NORMAL); + else + IREAD_LOCK(ip, RDWRLOCK_NORMAL); + + if (((lblock64 << ip->i_sb->s_blocksize_bits) < ip->i_size) && + (!xtLookup(ip, lblock64, xlen, &xflag, &xaddr, &xlen, 0)) && + xaddr) { + if (xflag & XAD_NOTRECORDED) { + if (!create) + /* + * Allocated but not recorded, read treats + * this as a hole + */ + goto unlock; + XADoffset(&xad, lblock64); + XADlength(&xad, xlen); + XADaddress(&xad, xaddr); + rc = extRecord(ip, &xad); + if (rc) + goto unlock; + set_buffer_new(bh_result); + } + + map_bh(bh_result, ip->i_sb, xaddr); + bh_result->b_size = xlen << ip->i_blkbits; + goto unlock; + } + if (!create) + goto unlock; + + /* + * Allocate a new block + */ + if ((rc = extHint(ip, lblock64 << ip->i_sb->s_blocksize_bits, &xad))) + goto unlock; + rc = extAlloc(ip, xlen, lblock64, &xad, false); + if (rc) + goto unlock; + + set_buffer_new(bh_result); + map_bh(bh_result, ip->i_sb, addressXAD(&xad)); + bh_result->b_size = lengthXAD(&xad) << ip->i_blkbits; + + unlock: + /* + * Release lock on inode + */ + if (create) + IWRITE_UNLOCK(ip); + else + IREAD_UNLOCK(ip); + return rc; +} + +static int jfs_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + return mpage_writepages(mapping, wbc, jfs_get_block); +} + +static int jfs_read_folio(struct file *file, struct folio *folio) +{ + return mpage_read_folio(folio, jfs_get_block); +} + +static void jfs_readahead(struct readahead_control *rac) +{ + mpage_readahead(rac, jfs_get_block); +} + +static void jfs_write_failed(struct address_space *mapping, loff_t to) +{ + struct inode *inode = mapping->host; + + if (to > inode->i_size) { + truncate_pagecache(inode, inode->i_size); + jfs_truncate(inode); + } +} + +static int jfs_write_begin(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, + struct folio **foliop, void **fsdata) +{ + int ret; + + ret = block_write_begin(mapping, pos, len, foliop, jfs_get_block); + if (unlikely(ret)) + jfs_write_failed(mapping, pos + len); + + return ret; +} + +static int jfs_write_end(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct folio *folio, void *fsdata) +{ + int ret; + + ret = generic_write_end(iocb, mapping, pos, len, copied, folio, fsdata); + if (ret < len) + jfs_write_failed(mapping, pos + len); + return ret; +} + +static sector_t jfs_bmap(struct address_space *mapping, sector_t block) +{ + return generic_block_bmap(mapping, block, jfs_get_block); +} + +static ssize_t jfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) +{ + struct file *file = iocb->ki_filp; + struct address_space *mapping = file->f_mapping; + struct inode *inode = file->f_mapping->host; + size_t count = iov_iter_count(iter); + ssize_t ret; + + ret = blockdev_direct_IO(iocb, inode, iter, jfs_get_block); + + /* + * In case of error extending write may have instantiated a few + * blocks outside i_size. Trim these off again. + */ + if (unlikely(iov_iter_rw(iter) == WRITE && ret < 0)) { + loff_t isize = i_size_read(inode); + loff_t end = iocb->ki_pos + count; + + if (end > isize) + jfs_write_failed(mapping, end); + } + + return ret; +} + +const struct address_space_operations jfs_aops = { + .dirty_folio = block_dirty_folio, + .invalidate_folio = block_invalidate_folio, + .read_folio = jfs_read_folio, + .readahead = jfs_readahead, + .writepages = jfs_writepages, + .write_begin = jfs_write_begin, + .write_end = jfs_write_end, + .bmap = jfs_bmap, + .direct_IO = jfs_direct_IO, + .migrate_folio = buffer_migrate_folio, +}; + +/* + * Guts of jfs_truncate. Called with locks already held. Can be called + * with directory for truncating directory index table. + */ +void jfs_truncate_nolock(struct inode *ip, loff_t length) +{ + loff_t newsize; + tid_t tid; + + ASSERT(length >= 0); + + if (test_cflag(COMMIT_Nolink, ip) || isReadOnly(ip)) { + xtTruncate(0, ip, length, COMMIT_WMAP); + return; + } + + do { + tid = txBegin(ip->i_sb, 0); + + /* + * The commit_mutex cannot be taken before txBegin. + * txBegin may block and there is a chance the inode + * could be marked dirty and need to be committed + * before txBegin unblocks + */ + mutex_lock(&JFS_IP(ip)->commit_mutex); + + newsize = xtTruncate(tid, ip, length, + COMMIT_TRUNCATE | COMMIT_PWMAP); + if (newsize < 0) { + txEnd(tid); + mutex_unlock(&JFS_IP(ip)->commit_mutex); + break; + } + + inode_set_mtime_to_ts(ip, inode_set_ctime_current(ip)); + mark_inode_dirty(ip); + + txCommit(tid, 1, &ip, 0); + txEnd(tid); + mutex_unlock(&JFS_IP(ip)->commit_mutex); + } while (newsize > length); /* Truncate isn't always atomic */ +} + +void jfs_truncate(struct inode *ip) +{ + jfs_info("jfs_truncate: size = 0x%lx", (ulong) ip->i_size); + + block_truncate_page(ip->i_mapping, ip->i_size, jfs_get_block); + + IWRITE_LOCK(ip, RDWRLOCK_NORMAL); + jfs_truncate_nolock(ip, ip->i_size); + IWRITE_UNLOCK(ip); +} diff --git a/fs/jfs/ioctl.c b/fs/jfs/ioctl.c new file mode 100644 index 000000000..563f148be --- /dev/null +++ b/fs/jfs/ioctl.c @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/jfs/ioctl.c + * + * Copyright (C) 2006 Herbert Poetzl + * adapted from Remy Card's ext2/ioctl.c + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "jfs_filsys.h" +#include "jfs_debug.h" +#include "jfs_incore.h" +#include "jfs_dinode.h" +#include "jfs_inode.h" +#include "jfs_dmap.h" +#include "jfs_discard.h" + +static struct { + long jfs_flag; + long ext2_flag; +} jfs_map[] = { + {JFS_NOATIME_FL, FS_NOATIME_FL}, + {JFS_DIRSYNC_FL, FS_DIRSYNC_FL}, + {JFS_SYNC_FL, FS_SYNC_FL}, + {JFS_SECRM_FL, FS_SECRM_FL}, + {JFS_UNRM_FL, FS_UNRM_FL}, + {JFS_APPEND_FL, FS_APPEND_FL}, + {JFS_IMMUTABLE_FL, FS_IMMUTABLE_FL}, + {0, 0}, +}; + +static long jfs_map_ext2(unsigned long flags, int from) +{ + int index=0; + long mapped=0; + + while (jfs_map[index].jfs_flag) { + if (from) { + if (jfs_map[index].ext2_flag & flags) + mapped |= jfs_map[index].jfs_flag; + } else { + if (jfs_map[index].jfs_flag & flags) + mapped |= jfs_map[index].ext2_flag; + } + index++; + } + return mapped; +} + +int jfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + struct jfs_inode_info *jfs_inode = JFS_IP(d_inode(dentry)); + unsigned int flags = jfs_inode->mode2 & JFS_FL_USER_VISIBLE; + + if (d_is_special(dentry)) + return -ENOTTY; + + fileattr_fill_flags(fa, jfs_map_ext2(flags, 0)); + + return 0; +} + +int jfs_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + struct jfs_inode_info *jfs_inode = JFS_IP(inode); + unsigned int flags; + + if (d_is_special(dentry)) + return -ENOTTY; + + if (fileattr_has_fsx(fa)) + return -EOPNOTSUPP; + + flags = jfs_map_ext2(fa->flags, 1); + if (!S_ISDIR(inode->i_mode)) + flags &= ~JFS_DIRSYNC_FL; + + /* Is it quota file? Do not allow user to mess with it */ + if (IS_NOQUOTA(inode)) + return -EPERM; + + flags = flags & JFS_FL_USER_MODIFIABLE; + flags |= jfs_inode->mode2 & ~JFS_FL_USER_MODIFIABLE; + jfs_inode->mode2 = flags; + + jfs_set_inode_flags(inode); + inode_set_ctime_current(inode); + mark_inode_dirty(inode); + + return 0; +} + +long jfs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + + switch (cmd) { + case FITRIM: + { + struct super_block *sb = inode->i_sb; + struct fstrim_range range; + s64 ret = 0; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (!bdev_max_discard_sectors(sb->s_bdev)) { + jfs_warn("FITRIM not supported on device"); + return -EOPNOTSUPP; + } + + if (copy_from_user(&range, (struct fstrim_range __user *)arg, + sizeof(range))) + return -EFAULT; + + range.minlen = max_t(unsigned int, range.minlen, + bdev_discard_granularity(sb->s_bdev)); + + ret = jfs_ioc_trim(inode, &range); + if (ret < 0) + return ret; + + if (copy_to_user((struct fstrim_range __user *)arg, &range, + sizeof(range))) + return -EFAULT; + + return 0; + } + + default: + return -ENOTTY; + } +} diff --git a/fs/jfs/jfs_acl.h b/fs/jfs/jfs_acl.h new file mode 100644 index 000000000..f892e54d0 --- /dev/null +++ b/fs/jfs/jfs_acl.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) International Business Machines Corp., 2002 + */ +#ifndef _H_JFS_ACL +#define _H_JFS_ACL + +#ifdef CONFIG_JFS_POSIX_ACL + +struct posix_acl *jfs_get_acl(struct inode *inode, int type, bool rcu); +int jfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type); +int jfs_init_acl(tid_t, struct inode *, struct inode *); + +#else + +static inline int jfs_init_acl(tid_t tid, struct inode *inode, + struct inode *dir) +{ + return 0; +} + +#endif +#endif /* _H_JFS_ACL */ diff --git a/fs/jfs/jfs_btree.h b/fs/jfs/jfs_btree.h new file mode 100644 index 000000000..ce055ef50 --- /dev/null +++ b/fs/jfs/jfs_btree.h @@ -0,0 +1,159 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) International Business Machines Corp., 2000-2004 + */ +#ifndef _H_JFS_BTREE +#define _H_JFS_BTREE + +/* + * jfs_btree.h: B+-tree + * + * JFS B+-tree (dtree and xtree) common definitions + */ + +/* + * basic btree page - btpage + * +struct btpage { + s64 next; right sibling bn + s64 prev; left sibling bn + + u8 flag; + u8 rsrvd[7]; type specific + s64 self; self address + + u8 entry[4064]; +}; */ + +/* btpaget_t flag */ +#define BT_TYPE 0x07 /* B+-tree index */ +#define BT_ROOT 0x01 /* root page */ +#define BT_LEAF 0x02 /* leaf page */ +#define BT_INTERNAL 0x04 /* internal page */ +#define BT_RIGHTMOST 0x10 /* rightmost page */ +#define BT_LEFTMOST 0x20 /* leftmost page */ +#define BT_SWAPPED 0x80 /* used by fsck for endian swapping */ + +/* btorder (in inode) */ +#define BT_RANDOM 0x0000 +#define BT_SEQUENTIAL 0x0001 +#define BT_LOOKUP 0x0010 +#define BT_INSERT 0x0020 +#define BT_DELETE 0x0040 + +/* + * btree page buffer cache access + */ +#define BT_IS_ROOT(MP) (((MP)->xflag & COMMIT_PAGE) == 0) + +/* get page from buffer page */ +#define BT_PAGE(IP, MP, TYPE, ROOT)\ + (BT_IS_ROOT(MP) ? (TYPE *)&JFS_IP(IP)->ROOT : (TYPE *)(MP)->data) + +/* get the page buffer and the page for specified block address */ +#define BT_GETPAGE(IP, BN, MP, TYPE, SIZE, P, RC, ROOT)\ +{\ + if ((BN) == 0)\ + {\ + MP = (struct metapage *)&JFS_IP(IP)->bxflag;\ + P = (TYPE *)&JFS_IP(IP)->ROOT;\ + RC = 0;\ + }\ + else\ + {\ + MP = read_metapage((IP), BN, SIZE, 1);\ + if (MP) {\ + RC = 0;\ + P = (MP)->data;\ + } else {\ + P = NULL;\ + jfs_err("bread failed!");\ + RC = -EIO;\ + }\ + }\ +} + +#define BT_MARK_DIRTY(MP, IP)\ +{\ + if (BT_IS_ROOT(MP))\ + mark_inode_dirty(IP);\ + else\ + mark_metapage_dirty(MP);\ +} + +/* put the page buffer */ +#define BT_PUTPAGE(MP)\ +{\ + if (! BT_IS_ROOT(MP)) \ + release_metapage(MP); \ +} + + +/* + * btree traversal stack + * + * record the path traversed during the search; + * top frame record the leaf page/entry selected. + */ +struct btframe { /* stack frame */ + s64 bn; /* 8: */ + s16 index; /* 2: */ + s16 lastindex; /* 2: unused */ + struct metapage *mp; /* 4/8: */ +}; /* (16/24) */ + +struct btstack { + struct btframe *top; + int nsplit; + struct btframe stack[MAXTREEHEIGHT]; +}; + +#define BT_CLR(btstack)\ + (btstack)->top = (btstack)->stack + +#define BT_STACK_FULL(btstack)\ + ( (btstack)->top == &((btstack)->stack[MAXTREEHEIGHT-1])) + +#define BT_PUSH(BTSTACK, BN, INDEX)\ +{\ + assert(!BT_STACK_FULL(BTSTACK));\ + (BTSTACK)->top->bn = BN;\ + (BTSTACK)->top->index = INDEX;\ + ++(BTSTACK)->top;\ +} + +#define BT_POP(btstack)\ + ( (btstack)->top == (btstack)->stack ? NULL : --(btstack)->top ) + +#define BT_STACK(btstack)\ + ( (btstack)->top == (btstack)->stack ? NULL : (btstack)->top ) + +static inline void BT_STACK_DUMP(struct btstack *btstack) +{ + int i; + printk("btstack dump:\n"); + for (i = 0; i < MAXTREEHEIGHT; i++) + printk(KERN_ERR "bn = %Lx, index = %d\n", + (long long)btstack->stack[i].bn, + btstack->stack[i].index); +} + +/* retrieve search results */ +#define BT_GETSEARCH(IP, LEAF, BN, MP, TYPE, P, INDEX, ROOT)\ +{\ + BN = (LEAF)->bn;\ + MP = (LEAF)->mp;\ + if (BN)\ + P = (TYPE *)MP->data;\ + else\ + P = (TYPE *)&JFS_IP(IP)->ROOT;\ + INDEX = (LEAF)->index;\ +} + +/* put the page buffer of search */ +#define BT_PUTSEARCH(BTSTACK)\ +{\ + if (! BT_IS_ROOT((BTSTACK)->top->mp))\ + release_metapage((BTSTACK)->top->mp);\ +} +#endif /* _H_JFS_BTREE */ diff --git a/fs/jfs/jfs_debug.c b/fs/jfs/jfs_debug.c new file mode 100644 index 000000000..44b62b3c3 --- /dev/null +++ b/fs/jfs/jfs_debug.c @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) International Business Machines Corp., 2000-2004 + * Portions Copyright (C) Christoph Hellwig, 2001-2002 + */ + +#include +#include +#include +#include +#include +#include +#include "jfs_incore.h" +#include "jfs_filsys.h" +#include "jfs_debug.h" + +#ifdef PROC_FS_JFS /* see jfs_debug.h */ + +#ifdef CONFIG_JFS_DEBUG +static int jfs_loglevel_proc_show(struct seq_file *m, void *v) +{ + seq_printf(m, "%d\n", jfsloglevel); + return 0; +} + +static int jfs_loglevel_proc_open(struct inode *inode, struct file *file) +{ + return single_open(file, jfs_loglevel_proc_show, NULL); +} + +static ssize_t jfs_loglevel_proc_write(struct file *file, + const char __user *buffer, size_t count, loff_t *ppos) +{ + char c; + + if (get_user(c, buffer)) + return -EFAULT; + + /* yes, I know this is an ASCIIism. --hch */ + if (c < '0' || c > '9') + return -EINVAL; + jfsloglevel = c - '0'; + return count; +} + +static const struct proc_ops jfs_loglevel_proc_ops = { + .proc_open = jfs_loglevel_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = jfs_loglevel_proc_write, +}; +#endif + +void jfs_proc_init(void) +{ + struct proc_dir_entry *base; + + base = proc_mkdir("fs/jfs", NULL); + if (!base) + return; + +#ifdef CONFIG_JFS_STATISTICS + proc_create_single("lmstats", 0, base, jfs_lmstats_proc_show); + proc_create_single("txstats", 0, base, jfs_txstats_proc_show); + proc_create_single("xtstat", 0, base, jfs_xtstat_proc_show); + proc_create_single("mpstat", 0, base, jfs_mpstat_proc_show); +#endif +#ifdef CONFIG_JFS_DEBUG + proc_create_single("TxAnchor", 0, base, jfs_txanchor_proc_show); + proc_create("loglevel", 0, base, &jfs_loglevel_proc_ops); +#endif +} + +void jfs_proc_clean(void) +{ + remove_proc_subtree("fs/jfs", NULL); +} + +#endif /* PROC_FS_JFS */ diff --git a/fs/jfs/jfs_debug.h b/fs/jfs/jfs_debug.h new file mode 100644 index 000000000..48e2150c0 --- /dev/null +++ b/fs/jfs/jfs_debug.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) International Business Machines Corp., 2000-2002 + * Portions Copyright (C) Christoph Hellwig, 2001-2002 + */ +#ifndef _H_JFS_DEBUG +#define _H_JFS_DEBUG + +/* + * jfs_debug.h + * + * global debug message, data structure/macro definitions + * under control of CONFIG_JFS_DEBUG, CONFIG_JFS_STATISTICS; + */ + +/* + * Create /proc/fs/jfs if procfs is enabled andeither + * CONFIG_JFS_DEBUG or CONFIG_JFS_STATISTICS is defined + */ +#if defined(CONFIG_PROC_FS) && (defined(CONFIG_JFS_DEBUG) || defined(CONFIG_JFS_STATISTICS)) +#define PROC_FS_JFS +extern void jfs_proc_init(void); +extern void jfs_proc_clean(void); +#endif + +/* + * assert with traditional printf/panic + */ +#define assert(p) do { \ + if (!(p)) { \ + printk(KERN_CRIT "BUG at %s:%d assert(%s)\n", \ + __FILE__, __LINE__, #p); \ + BUG(); \ + } \ +} while (0) + +/* + * debug ON + * -------- + */ +#ifdef CONFIG_JFS_DEBUG +#define ASSERT(p) assert(p) + +/* printk verbosity */ +#define JFS_LOGLEVEL_ERR 1 +#define JFS_LOGLEVEL_WARN 2 +#define JFS_LOGLEVEL_DEBUG 3 +#define JFS_LOGLEVEL_INFO 4 + +extern int jfsloglevel; + +int jfs_txanchor_proc_show(struct seq_file *m, void *v); + +/* information message: e.g., configuration, major event */ +#define jfs_info(fmt, arg...) do { \ + if (jfsloglevel >= JFS_LOGLEVEL_INFO) \ + printk(KERN_INFO fmt "\n", ## arg); \ +} while (0) + +/* debug message: ad hoc */ +#define jfs_debug(fmt, arg...) do { \ + if (jfsloglevel >= JFS_LOGLEVEL_DEBUG) \ + printk(KERN_DEBUG fmt "\n", ## arg); \ +} while (0) + +/* warn message: */ +#define jfs_warn(fmt, arg...) do { \ + if (jfsloglevel >= JFS_LOGLEVEL_WARN) \ + printk(KERN_WARNING fmt "\n", ## arg); \ +} while (0) + +/* error event message: e.g., i/o error */ +#define jfs_err(fmt, arg...) do { \ + if (jfsloglevel >= JFS_LOGLEVEL_ERR) \ + printk(KERN_ERR fmt "\n", ## arg); \ +} while (0) + +/* + * debug OFF + * --------- + */ +#else /* CONFIG_JFS_DEBUG */ +#define ASSERT(p) do {} while (0) +#define jfs_info(fmt, arg...) do {} while (0) +#define jfs_debug(fmt, arg...) do {} while (0) +#define jfs_warn(fmt, arg...) do {} while (0) +#define jfs_err(fmt, arg...) do {} while (0) +#endif /* CONFIG_JFS_DEBUG */ + +/* + * statistics + * ---------- + */ +#ifdef CONFIG_JFS_STATISTICS +int jfs_lmstats_proc_show(struct seq_file *m, void *v); +int jfs_txstats_proc_show(struct seq_file *m, void *v); +int jfs_mpstat_proc_show(struct seq_file *m, void *v); +int jfs_xtstat_proc_show(struct seq_file *m, void *v); + +#define INCREMENT(x) ((x)++) +#define DECREMENT(x) ((x)--) +#define HIGHWATERMARK(x,y) ((x) = max((x), (y))) +#else +#define INCREMENT(x) +#define DECREMENT(x) +#define HIGHWATERMARK(x,y) +#endif /* CONFIG_JFS_STATISTICS */ + +#endif /* _H_JFS_DEBUG */ diff --git a/fs/jfs/jfs_dinode.h b/fs/jfs/jfs_dinode.h new file mode 100644 index 000000000..603aae17a --- /dev/null +++ b/fs/jfs/jfs_dinode.h @@ -0,0 +1,169 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) International Business Machines Corp., 2000-2001 + */ +#ifndef _H_JFS_DINODE +#define _H_JFS_DINODE + +/* + * jfs_dinode.h: on-disk inode manager + */ + +#define INODESLOTSIZE 128 +#define L2INODESLOTSIZE 7 +#define log2INODESIZE 9 /* log2(bytes per dinode) */ + + +/* + * on-disk inode : 512 bytes + * + * note: align 64-bit fields on 8-byte boundary. + */ +struct dinode { + /* + * I. base area (128 bytes) + * ------------------------ + * + * define generic/POSIX attributes + */ + __le32 di_inostamp; /* 4: stamp to show inode belongs to fileset */ + __le32 di_fileset; /* 4: fileset number */ + __le32 di_number; /* 4: inode number, aka file serial number */ + __le32 di_gen; /* 4: inode generation number */ + + pxd_t di_ixpxd; /* 8: inode extent descriptor */ + + __le64 di_size; /* 8: size */ + __le64 di_nblocks; /* 8: number of blocks allocated */ + + __le32 di_nlink; /* 4: number of links to the object */ + + __le32 di_uid; /* 4: user id of owner */ + __le32 di_gid; /* 4: group id of owner */ + + __le32 di_mode; /* 4: attribute, format and permission */ + + struct timestruc_t di_atime; /* 8: time last data accessed */ + struct timestruc_t di_ctime; /* 8: time last status changed */ + struct timestruc_t di_mtime; /* 8: time last data modified */ + struct timestruc_t di_otime; /* 8: time created */ + + dxd_t di_acl; /* 16: acl descriptor */ + + dxd_t di_ea; /* 16: ea descriptor */ + + __le32 di_next_index; /* 4: Next available dir_table index */ + + __le32 di_acltype; /* 4: Type of ACL */ + + /* + * Extension Areas. + * + * Historically, the inode was partitioned into 4 128-byte areas, + * the last 3 being defined as unions which could have multiple + * uses. The first 96 bytes had been completely unused until + * an index table was added to the directory. It is now more + * useful to describe the last 3/4 of the inode as a single + * union. We would probably be better off redesigning the + * entire structure from scratch, but we don't want to break + * commonality with OS/2's JFS at this time. + */ + union { + struct { + /* + * This table contains the information needed to + * find a directory entry from a 32-bit index. + * If the index is small enough, the table is inline, + * otherwise, an x-tree root overlays this table + */ + struct dir_table_slot _table[12]; /* 96: inline */ + + dtroot_t _dtroot; /* 288: dtree root */ + } _dir; /* (384) */ +#define di_dirtable u._dir._table +#define di_dtroot u._dir._dtroot +#define di_parent di_dtroot.header.idotdot +#define di_DASD di_dtroot.header.DASD + + struct { + union { + u8 _data[96]; /* 96: unused */ + struct { + void *_imap; /* 4: unused */ + __le32 _gengen; /* 4: generator */ + } _imap; + } _u1; /* 96: */ +#define di_gengen u._file._u1._imap._gengen + + union { + xtroot_t _xtroot; + struct { + u8 unused[16]; /* 16: */ + dxd_t _dxd; /* 16: */ + union { + /* + * The fast symlink area + * is expected to overflow + * into _inlineea when + * needed (which will clear + * INLINEEA). + */ + struct { + union { + __le32 _rdev; /* 4: */ + u8 _fastsymlink[128]; + } _u; + u8 _inlineea[128]; + }; + u8 _inline_all[256]; + }; + } _special; + } _u2; + } _file; +#define di_xtroot u._file._u2._xtroot +#define di_dxd u._file._u2._special._dxd +#define di_btroot di_xtroot +#define di_inlinedata u._file._u2._special._u +#define di_rdev u._file._u2._special._u._rdev +#define di_fastsymlink u._file._u2._special._u._fastsymlink +#define di_inlineea u._file._u2._special._inlineea +#define di_inline_all u._file._u2._special._inline_all + } u; +}; + +/* extended mode bits (on-disk inode di_mode) */ +#define IFJOURNAL 0x00010000 /* journalled file */ +#define ISPARSE 0x00020000 /* sparse file enabled */ +#define INLINEEA 0x00040000 /* inline EA area free */ +#define ISWAPFILE 0x00800000 /* file open for pager swap space */ + +/* more extended mode bits: attributes for OS/2 */ +#define IREADONLY 0x02000000 /* no write access to file */ +#define IHIDDEN 0x04000000 /* hidden file */ +#define ISYSTEM 0x08000000 /* system file */ + +#define IDIRECTORY 0x20000000 /* directory (shadow of real bit) */ +#define IARCHIVE 0x40000000 /* file archive bit */ +#define INEWNAME 0x80000000 /* non-8.3 filename format */ + +#define IRASH 0x4E000000 /* mask for changeable attributes */ +#define ATTRSHIFT 25 /* bits to shift to move attribute + specification to mode position */ + +/* extended attributes for Linux */ + +#define JFS_NOATIME_FL 0x00080000 /* do not update atime */ + +#define JFS_DIRSYNC_FL 0x00100000 /* dirsync behaviour */ +#define JFS_SYNC_FL 0x00200000 /* Synchronous updates */ +#define JFS_SECRM_FL 0x00400000 /* Secure deletion */ +#define JFS_UNRM_FL 0x00800000 /* allow for undelete */ + +#define JFS_APPEND_FL 0x01000000 /* writes to file may only append */ +#define JFS_IMMUTABLE_FL 0x02000000 /* Immutable file */ + +#define JFS_FL_USER_VISIBLE 0x03F80000 +#define JFS_FL_USER_MODIFIABLE 0x03F80000 +#define JFS_FL_INHERIT 0x03C80000 + +#endif /*_H_JFS_DINODE */ diff --git a/fs/jfs/jfs_discard.c b/fs/jfs/jfs_discard.c new file mode 100644 index 000000000..4b660296c --- /dev/null +++ b/fs/jfs/jfs_discard.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) Tino Reichardt, 2012 + */ + +#include +#include +#include + +#include "jfs_incore.h" +#include "jfs_superblock.h" +#include "jfs_discard.h" +#include "jfs_dmap.h" +#include "jfs_debug.h" + + +/* + * NAME: jfs_issue_discard() + * + * FUNCTION: TRIM the specified block range on device, if supported + * + * PARAMETERS: + * ip - pointer to in-core inode + * blkno - starting block number to be trimmed (0..N) + * nblocks - number of blocks to be trimmed + * + * RETURN VALUES: + * none + * + * serialization: IREAD_LOCK(ipbmap) held on entry/exit; + */ +void jfs_issue_discard(struct inode *ip, u64 blkno, u64 nblocks) +{ + struct super_block *sb = ip->i_sb; + int r = 0; + + r = sb_issue_discard(sb, blkno, nblocks, GFP_NOFS, 0); + if (unlikely(r != 0)) { + jfs_err("JFS: sb_issue_discard(%p, %llu, %llu, GFP_NOFS, 0) = %d => failed!", + sb, (unsigned long long)blkno, + (unsigned long long)nblocks, r); + } + + jfs_info("JFS: sb_issue_discard(%p, %llu, %llu, GFP_NOFS, 0) = %d", + sb, (unsigned long long)blkno, + (unsigned long long)nblocks, r); + + return; +} + +/* + * NAME: jfs_ioc_trim() + * + * FUNCTION: attempt to discard (TRIM) all free blocks from the + * filesystem. + * + * PARAMETERS: + * ip - pointer to in-core inode; + * range - the range, given by user space + * + * RETURN VALUES: + * 0 - success + * -EIO - i/o error + */ +int jfs_ioc_trim(struct inode *ip, struct fstrim_range *range) +{ + struct inode *ipbmap = JFS_SBI(ip->i_sb)->ipbmap; + struct bmap *bmp; + struct super_block *sb = ipbmap->i_sb; + int agno, agno_end; + u64 start, end, minlen; + u64 trimmed = 0; + + /** + * convert byte values to block size of filesystem: + * start: First Byte to trim + * len: number of Bytes to trim from start + * minlen: minimum extent length in Bytes + */ + start = range->start >> sb->s_blocksize_bits; + end = start + (range->len >> sb->s_blocksize_bits) - 1; + minlen = range->minlen >> sb->s_blocksize_bits; + if (minlen == 0) + minlen = 1; + + down_read(&sb->s_umount); + bmp = JFS_SBI(ip->i_sb)->bmap; + + if (bmp == NULL || + minlen > bmp->db_agsize || + start >= bmp->db_mapsize || + range->len < sb->s_blocksize) { + up_read(&sb->s_umount); + return -EINVAL; + } + + if (end >= bmp->db_mapsize) + end = bmp->db_mapsize - 1; + + /** + * we trim all ag's within the range + */ + agno = BLKTOAG(start, JFS_SBI(ip->i_sb)); + agno_end = BLKTOAG(end, JFS_SBI(ip->i_sb)); + while (agno <= agno_end) { + trimmed += dbDiscardAG(ip, agno, minlen); + agno++; + } + + up_read(&sb->s_umount); + range->len = trimmed << sb->s_blocksize_bits; + + return 0; +} diff --git a/fs/jfs/jfs_discard.h b/fs/jfs/jfs_discard.h new file mode 100644 index 000000000..05215fa27 --- /dev/null +++ b/fs/jfs/jfs_discard.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) Tino Reichardt, 2012 + */ +#ifndef _H_JFS_DISCARD +#define _H_JFS_DISCARD + +struct fstrim_range; + +extern void jfs_issue_discard(struct inode *ip, u64 blkno, u64 nblocks); +extern int jfs_ioc_trim(struct inode *ip, struct fstrim_range *range); + +#endif /* _H_JFS_DISCARD */ diff --git a/fs/jfs/jfs_dmap.c b/fs/jfs/jfs_dmap.c new file mode 100644 index 000000000..a841cf21d --- /dev/null +++ b/fs/jfs/jfs_dmap.c @@ -0,0 +1,4182 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) International Business Machines Corp., 2000-2004 + * Portions Copyright (C) Tino Reichardt, 2012 + */ + +#include +#include +#include "jfs_incore.h" +#include "jfs_superblock.h" +#include "jfs_dmap.h" +#include "jfs_imap.h" +#include "jfs_lock.h" +#include "jfs_metapage.h" +#include "jfs_debug.h" +#include "jfs_discard.h" + +/* + * SERIALIZATION of the Block Allocation Map. + * + * the working state of the block allocation map is accessed in + * two directions: + * + * 1) allocation and free requests that start at the dmap + * level and move up through the dmap control pages (i.e. + * the vast majority of requests). + * + * 2) allocation requests that start at dmap control page + * level and work down towards the dmaps. + * + * the serialization scheme used here is as follows. + * + * requests which start at the bottom are serialized against each + * other through buffers and each requests holds onto its buffers + * as it works it way up from a single dmap to the required level + * of dmap control page. + * requests that start at the top are serialized against each other + * and request that start from the bottom by the multiple read/single + * write inode lock of the bmap inode. requests starting at the top + * take this lock in write mode while request starting at the bottom + * take the lock in read mode. a single top-down request may proceed + * exclusively while multiple bottoms-up requests may proceed + * simultaneously (under the protection of busy buffers). + * + * in addition to information found in dmaps and dmap control pages, + * the working state of the block allocation map also includes read/ + * write information maintained in the bmap descriptor (i.e. total + * free block count, allocation group level free block counts). + * a single exclusive lock (BMAP_LOCK) is used to guard this information + * in the face of multiple-bottoms up requests. + * (lock ordering: IREAD_LOCK, BMAP_LOCK); + * + * accesses to the persistent state of the block allocation map (limited + * to the persistent bitmaps in dmaps) is guarded by (busy) buffers. + */ + +#define BMAP_LOCK_INIT(bmp) mutex_init(&bmp->db_bmaplock) +#define BMAP_LOCK(bmp) mutex_lock(&bmp->db_bmaplock) +#define BMAP_UNLOCK(bmp) mutex_unlock(&bmp->db_bmaplock) + +/* + * forward references + */ +static void dbAllocBits(struct bmap * bmp, struct dmap * dp, s64 blkno, + int nblocks); +static void dbSplit(dmtree_t *tp, int leafno, int splitsz, int newval, bool is_ctl); +static int dbBackSplit(dmtree_t *tp, int leafno, bool is_ctl); +static int dbJoin(dmtree_t *tp, int leafno, int newval, bool is_ctl); +static void dbAdjTree(dmtree_t *tp, int leafno, int newval, bool is_ctl); +static int dbAdjCtl(struct bmap * bmp, s64 blkno, int newval, int alloc, + int level); +static int dbAllocAny(struct bmap * bmp, s64 nblocks, int l2nb, s64 * results); +static int dbAllocNext(struct bmap * bmp, struct dmap * dp, s64 blkno, + int nblocks); +static int dbAllocNear(struct bmap * bmp, struct dmap * dp, s64 blkno, + int nblocks, + int l2nb, s64 * results); +static int dbAllocDmap(struct bmap * bmp, struct dmap * dp, s64 blkno, + int nblocks); +static int dbAllocDmapLev(struct bmap * bmp, struct dmap * dp, int nblocks, + int l2nb, + s64 * results); +static int dbAllocAG(struct bmap * bmp, int agno, s64 nblocks, int l2nb, + s64 * results); +static int dbAllocCtl(struct bmap * bmp, s64 nblocks, int l2nb, s64 blkno, + s64 * results); +static int dbExtend(struct inode *ip, s64 blkno, s64 nblocks, s64 addnblocks); +static int dbFindBits(u32 word, int l2nb); +static int dbFindCtl(struct bmap * bmp, int l2nb, int level, s64 * blkno); +static int dbFindLeaf(dmtree_t *tp, int l2nb, int *leafidx, bool is_ctl); +static int dbFreeBits(struct bmap * bmp, struct dmap * dp, s64 blkno, + int nblocks); +static int dbFreeDmap(struct bmap * bmp, struct dmap * dp, s64 blkno, + int nblocks); +static int dbMaxBud(u8 * cp); +static int blkstol2(s64 nb); + +static int cntlz(u32 value); +static int cnttz(u32 word); + +static int dbAllocDmapBU(struct bmap * bmp, struct dmap * dp, s64 blkno, + int nblocks); +static int dbInitDmap(struct dmap * dp, s64 blkno, int nblocks); +static int dbInitDmapTree(struct dmap * dp); +static int dbInitTree(struct dmaptree * dtp); +static int dbInitDmapCtl(struct dmapctl * dcp, int level, int i); +static int dbGetL2AGSize(s64 nblocks); + +/* + * buddy table + * + * table used for determining buddy sizes within characters of + * dmap bitmap words. the characters themselves serve as indexes + * into the table, with the table elements yielding the maximum + * binary buddy of free bits within the character. + */ +static const s8 budtab[256] = { + 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + 2, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + 2, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + 2, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + 2, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + 2, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + 2, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, -1 +}; + +/* + * check_dmapctl - Validate integrity of a dmapctl structure + * @dcp: Pointer to the dmapctl structure to check + * + * Return: true if valid, false if corrupted + */ +static bool check_dmapctl(struct dmapctl *dcp) +{ + s8 budmin = dcp->budmin; + u32 nleafs, l2nleafs, leafidx, height; + int i; + + nleafs = le32_to_cpu(dcp->nleafs); + /* Check basic field ranges */ + if (unlikely(nleafs > LPERCTL)) { + jfs_err("dmapctl: invalid nleafs %u (max %u)", + nleafs, LPERCTL); + return false; + } + + l2nleafs = le32_to_cpu(dcp->l2nleafs); + if (unlikely(l2nleafs > L2LPERCTL)) { + jfs_err("dmapctl: invalid l2nleafs %u (max %u)", + l2nleafs, L2LPERCTL); + return false; + } + + /* Verify nleafs matches l2nleafs (must be power of two) */ + if (unlikely((1U << l2nleafs) != nleafs)) { + jfs_err("dmapctl: nleafs %u != 2^%u", + nleafs, l2nleafs); + return false; + } + + leafidx = le32_to_cpu(dcp->leafidx); + /* Check leaf index matches expected position */ + if (unlikely(leafidx != CTLLEAFIND)) { + jfs_err("dmapctl: invalid leafidx %u (expected %u)", + leafidx, CTLLEAFIND); + return false; + } + + height = le32_to_cpu(dcp->height); + /* Check tree height is within valid range */ + if (unlikely(height > (L2LPERCTL >> 1))) { + jfs_err("dmapctl: invalid height %u (max %u)", + height, L2LPERCTL >> 1); + return false; + } + + /* Check budmin is valid (cannot be NOFREE for non-empty tree) */ + if (budmin == NOFREE) { + if (unlikely(nleafs > 0)) { + jfs_err("dmapctl: budmin is NOFREE but nleafs %u", + nleafs); + return false; + } + } else if (unlikely(budmin < BUDMIN)) { + jfs_err("dmapctl: invalid budmin %d (min %d)", + budmin, BUDMIN); + return false; + } + + /* Check leaf nodes fit within stree array */ + if (unlikely(leafidx + nleafs > CTLTREESIZE)) { + jfs_err("dmapctl: leaf range exceeds stree size (end %u > %u)", + leafidx + nleafs, CTLTREESIZE); + return false; + } + + /* Check leaf nodes have valid values */ + for (i = leafidx; i < leafidx + nleafs; i++) { + s8 val = dcp->stree[i]; + + if (unlikely(val < NOFREE)) { + jfs_err("dmapctl: invalid leaf value %d at index %d", + val, i); + return false; + } else if (unlikely(val > 31)) { + jfs_err("dmapctl: leaf value %d too large at index %d", val, i); + return false; + } + } + + return true; +} + +/* + * NAME: dbMount() + * + * FUNCTION: initializate the block allocation map. + * + * memory is allocated for the in-core bmap descriptor and + * the in-core descriptor is initialized from disk. + * + * PARAMETERS: + * ipbmap - pointer to in-core inode for the block map. + * + * RETURN VALUES: + * 0 - success + * -ENOMEM - insufficient memory + * -EIO - i/o error + * -EINVAL - wrong bmap data + */ +int dbMount(struct inode *ipbmap) +{ + struct bmap *bmp; + struct dbmap_disk *dbmp_le; + struct metapage *mp; + int i, err; + + /* + * allocate/initialize the in-memory bmap descriptor + */ + /* allocate memory for the in-memory bmap descriptor */ + bmp = kmalloc_obj(struct bmap); + if (bmp == NULL) + return -ENOMEM; + + /* read the on-disk bmap descriptor. */ + mp = read_metapage(ipbmap, + BMAPBLKNO << JFS_SBI(ipbmap->i_sb)->l2nbperpage, + PSIZE, 0); + if (mp == NULL) { + err = -EIO; + goto err_kfree_bmp; + } + + /* copy the on-disk bmap descriptor to its in-memory version. */ + dbmp_le = (struct dbmap_disk *) mp->data; + bmp->db_mapsize = le64_to_cpu(dbmp_le->dn_mapsize); + bmp->db_nfree = le64_to_cpu(dbmp_le->dn_nfree); + bmp->db_l2nbperpage = le32_to_cpu(dbmp_le->dn_l2nbperpage); + bmp->db_numag = le32_to_cpu(dbmp_le->dn_numag); + bmp->db_maxlevel = le32_to_cpu(dbmp_le->dn_maxlevel); + bmp->db_maxag = le32_to_cpu(dbmp_le->dn_maxag); + bmp->db_agpref = le32_to_cpu(dbmp_le->dn_agpref); + bmp->db_aglevel = le32_to_cpu(dbmp_le->dn_aglevel); + bmp->db_agheight = le32_to_cpu(dbmp_le->dn_agheight); + bmp->db_agwidth = le32_to_cpu(dbmp_le->dn_agwidth); + bmp->db_agstart = le32_to_cpu(dbmp_le->dn_agstart); + bmp->db_agl2size = le32_to_cpu(dbmp_le->dn_agl2size); + + if ((bmp->db_l2nbperpage > L2PSIZE - L2MINBLOCKSIZE) || + (bmp->db_l2nbperpage < 0) || + !bmp->db_numag || (bmp->db_numag > MAXAG) || + (bmp->db_maxag >= MAXAG) || (bmp->db_maxag < 0) || + (bmp->db_agpref >= MAXAG) || (bmp->db_agpref < 0) || + (bmp->db_agheight < 0) || (bmp->db_agheight > (L2LPERCTL >> 1)) || + (bmp->db_agwidth < 1) || (bmp->db_agwidth > (LPERCTL / MAXAG)) || + (bmp->db_agwidth > (1 << (L2LPERCTL - (bmp->db_agheight << 1)))) || + (bmp->db_agstart < 0) || + (bmp->db_agstart > (CTLTREESIZE - 1 - bmp->db_agwidth * (MAXAG - 1))) || + (bmp->db_agl2size > L2MAXL2SIZE - L2MAXAG) || + (bmp->db_agl2size < 0) || + ((bmp->db_mapsize - 1) >> bmp->db_agl2size) > MAXAG) { + err = -EINVAL; + goto err_release_metapage; + } + + for (i = 0; i < MAXAG; i++) + bmp->db_agfree[i] = le64_to_cpu(dbmp_le->dn_agfree[i]); + bmp->db_agsize = le64_to_cpu(dbmp_le->dn_agsize); + bmp->db_maxfreebud = dbmp_le->dn_maxfreebud; + + /* release the buffer. */ + release_metapage(mp); + + /* bind the bmap inode and the bmap descriptor to each other. */ + bmp->db_ipbmap = ipbmap; + JFS_SBI(ipbmap->i_sb)->bmap = bmp; + + memset(bmp->db_active, 0, sizeof(bmp->db_active)); + + /* + * allocate/initialize the bmap lock + */ + BMAP_LOCK_INIT(bmp); + + return (0); + +err_release_metapage: + release_metapage(mp); +err_kfree_bmp: + kfree(bmp); + return err; +} + + +/* + * NAME: dbUnmount() + * + * FUNCTION: terminate the block allocation map in preparation for + * file system unmount. + * + * the in-core bmap descriptor is written to disk and + * the memory for this descriptor is freed. + * + * PARAMETERS: + * ipbmap - pointer to in-core inode for the block map. + * + * RETURN VALUES: + * 0 - success + * -EIO - i/o error + */ +int dbUnmount(struct inode *ipbmap, int mounterror) +{ + struct bmap *bmp = JFS_SBI(ipbmap->i_sb)->bmap; + + if (!(mounterror || isReadOnly(ipbmap))) + dbSync(ipbmap); + + /* + * Invalidate the page cache buffers + */ + truncate_inode_pages(ipbmap->i_mapping, 0); + + /* free the memory for the in-memory bmap. */ + kfree(bmp); + JFS_SBI(ipbmap->i_sb)->bmap = NULL; + + return (0); +} + +/* + * dbSync() + */ +int dbSync(struct inode *ipbmap) +{ + struct dbmap_disk *dbmp_le; + struct bmap *bmp = JFS_SBI(ipbmap->i_sb)->bmap; + struct metapage *mp; + int i; + + /* + * write bmap global control page + */ + /* get the buffer for the on-disk bmap descriptor. */ + mp = read_metapage(ipbmap, + BMAPBLKNO << JFS_SBI(ipbmap->i_sb)->l2nbperpage, + PSIZE, 0); + if (mp == NULL) { + jfs_err("dbSync: read_metapage failed!"); + return -EIO; + } + /* copy the in-memory version of the bmap to the on-disk version */ + dbmp_le = (struct dbmap_disk *) mp->data; + dbmp_le->dn_mapsize = cpu_to_le64(bmp->db_mapsize); + dbmp_le->dn_nfree = cpu_to_le64(bmp->db_nfree); + dbmp_le->dn_l2nbperpage = cpu_to_le32(bmp->db_l2nbperpage); + dbmp_le->dn_numag = cpu_to_le32(bmp->db_numag); + dbmp_le->dn_maxlevel = cpu_to_le32(bmp->db_maxlevel); + dbmp_le->dn_maxag = cpu_to_le32(bmp->db_maxag); + dbmp_le->dn_agpref = cpu_to_le32(bmp->db_agpref); + dbmp_le->dn_aglevel = cpu_to_le32(bmp->db_aglevel); + dbmp_le->dn_agheight = cpu_to_le32(bmp->db_agheight); + dbmp_le->dn_agwidth = cpu_to_le32(bmp->db_agwidth); + dbmp_le->dn_agstart = cpu_to_le32(bmp->db_agstart); + dbmp_le->dn_agl2size = cpu_to_le32(bmp->db_agl2size); + for (i = 0; i < MAXAG; i++) + dbmp_le->dn_agfree[i] = cpu_to_le64(bmp->db_agfree[i]); + dbmp_le->dn_agsize = cpu_to_le64(bmp->db_agsize); + dbmp_le->dn_maxfreebud = bmp->db_maxfreebud; + + /* write the buffer */ + write_metapage(mp); + + /* + * write out dirty pages of bmap + */ + filemap_write_and_wait(ipbmap->i_mapping); + + diWriteSpecial(ipbmap, 0); + + return (0); +} + +/* + * NAME: dbFree() + * + * FUNCTION: free the specified block range from the working block + * allocation map. + * + * the blocks will be free from the working map one dmap + * at a time. + * + * PARAMETERS: + * ip - pointer to in-core inode; + * blkno - starting block number to be freed. + * nblocks - number of blocks to be freed. + * + * RETURN VALUES: + * 0 - success + * -EIO - i/o error + */ +int dbFree(struct inode *ip, s64 blkno, s64 nblocks) +{ + struct metapage *mp; + struct dmap *dp; + int nb, rc; + s64 lblkno, rem; + struct inode *ipbmap = JFS_SBI(ip->i_sb)->ipbmap; + struct bmap *bmp = JFS_SBI(ip->i_sb)->bmap; + struct super_block *sb = ipbmap->i_sb; + + IREAD_LOCK(ipbmap, RDWRLOCK_DMAP); + + /* block to be freed better be within the mapsize. */ + if (unlikely((blkno == 0) || (blkno + nblocks > bmp->db_mapsize))) { + IREAD_UNLOCK(ipbmap); + printk(KERN_ERR "blkno = %Lx, nblocks = %Lx\n", + (unsigned long long) blkno, + (unsigned long long) nblocks); + jfs_error(ip->i_sb, "block to be freed is outside the map\n"); + return -EIO; + } + + /** + * TRIM the blocks, when mounted with discard option + */ + if (JFS_SBI(sb)->flag & JFS_DISCARD) + if (JFS_SBI(sb)->minblks_trim <= nblocks) + jfs_issue_discard(ipbmap, blkno, nblocks); + + /* + * free the blocks a dmap at a time. + */ + mp = NULL; + for (rem = nblocks; rem > 0; rem -= nb, blkno += nb) { + /* release previous dmap if any */ + if (mp) { + write_metapage(mp); + } + + /* get the buffer for the current dmap. */ + lblkno = BLKTODMAP(blkno, bmp->db_l2nbperpage); + mp = read_metapage(ipbmap, lblkno, PSIZE, 0); + if (mp == NULL) { + IREAD_UNLOCK(ipbmap); + return -EIO; + } + dp = (struct dmap *) mp->data; + + /* determine the number of blocks to be freed from + * this dmap. + */ + nb = min(rem, BPERDMAP - (blkno & (BPERDMAP - 1))); + + /* free the blocks. */ + if ((rc = dbFreeDmap(bmp, dp, blkno, nb))) { + jfs_error(ip->i_sb, "error in block map\n"); + release_metapage(mp); + IREAD_UNLOCK(ipbmap); + return (rc); + } + } + + /* write the last buffer. */ + if (mp) + write_metapage(mp); + + IREAD_UNLOCK(ipbmap); + + return (0); +} + + +/* + * NAME: dbUpdatePMap() + * + * FUNCTION: update the allocation state (free or allocate) of the + * specified block range in the persistent block allocation map. + * + * the blocks will be updated in the persistent map one + * dmap at a time. + * + * PARAMETERS: + * ipbmap - pointer to in-core inode for the block map. + * free - 'true' if block range is to be freed from the persistent + * map; 'false' if it is to be allocated. + * blkno - starting block number of the range. + * nblocks - number of contiguous blocks in the range. + * tblk - transaction block; + * + * RETURN VALUES: + * 0 - success + * -EIO - i/o error + */ +int +dbUpdatePMap(struct inode *ipbmap, + int free, s64 blkno, s64 nblocks, struct tblock * tblk) +{ + int nblks, dbitno, wbitno, rbits; + int word, nbits, nwords; + struct bmap *bmp = JFS_SBI(ipbmap->i_sb)->bmap; + s64 lblkno, rem, lastlblkno; + u32 mask; + struct dmap *dp; + struct metapage *mp; + struct jfs_log *log; + int lsn, difft, diffp; + unsigned long flags; + + /* the blocks better be within the mapsize. */ + if (blkno + nblocks > bmp->db_mapsize) { + printk(KERN_ERR "blkno = %Lx, nblocks = %Lx\n", + (unsigned long long) blkno, + (unsigned long long) nblocks); + jfs_error(ipbmap->i_sb, "blocks are outside the map\n"); + return -EIO; + } + + /* compute delta of transaction lsn from log syncpt */ + lsn = tblk->lsn; + log = (struct jfs_log *) JFS_SBI(tblk->sb)->log; + logdiff(difft, lsn, log); + + /* + * update the block state a dmap at a time. + */ + mp = NULL; + lastlblkno = 0; + for (rem = nblocks; rem > 0; rem -= nblks, blkno += nblks) { + /* get the buffer for the current dmap. */ + lblkno = BLKTODMAP(blkno, bmp->db_l2nbperpage); + if (lblkno != lastlblkno) { + if (mp) { + write_metapage(mp); + } + + mp = read_metapage(bmp->db_ipbmap, lblkno, PSIZE, + 0); + if (mp == NULL) + return -EIO; + metapage_wait_for_io(mp); + } + dp = (struct dmap *) mp->data; + + /* determine the bit number and word within the dmap of + * the starting block. also determine how many blocks + * are to be updated within this dmap. + */ + dbitno = blkno & (BPERDMAP - 1); + word = dbitno >> L2DBWORD; + nblks = min(rem, (s64)BPERDMAP - dbitno); + + /* update the bits of the dmap words. the first and last + * words may only have a subset of their bits updated. if + * this is the case, we'll work against that word (i.e. + * partial first and/or last) only in a single pass. a + * single pass will also be used to update all words that + * are to have all their bits updated. + */ + for (rbits = nblks; rbits > 0; + rbits -= nbits, dbitno += nbits) { + /* determine the bit number within the word and + * the number of bits within the word. + */ + wbitno = dbitno & (DBWORD - 1); + nbits = min(rbits, DBWORD - wbitno); + + /* check if only part of the word is to be updated. */ + if (nbits < DBWORD) { + /* update (free or allocate) the bits + * in this word. + */ + mask = + (ONES << (DBWORD - nbits) >> wbitno); + if (free) + dp->pmap[word] &= + cpu_to_le32(~mask); + else + dp->pmap[word] |= + cpu_to_le32(mask); + + word += 1; + } else { + /* one or more words are to have all + * their bits updated. determine how + * many words and how many bits. + */ + nwords = rbits >> L2DBWORD; + nbits = nwords << L2DBWORD; + + /* update (free or allocate) the bits + * in these words. + */ + if (free) + memset(&dp->pmap[word], 0, + nwords * 4); + else + memset(&dp->pmap[word], (int) ONES, + nwords * 4); + + word += nwords; + } + } + + /* + * update dmap lsn + */ + if (lblkno == lastlblkno) + continue; + + lastlblkno = lblkno; + + LOGSYNC_LOCK(log, flags); + if (mp->lsn != 0) { + /* inherit older/smaller lsn */ + logdiff(diffp, mp->lsn, log); + if (difft < diffp) { + mp->lsn = lsn; + + /* move bp after tblock in logsync list */ + list_move(&mp->synclist, &tblk->synclist); + } + + /* inherit younger/larger clsn */ + logdiff(difft, tblk->clsn, log); + logdiff(diffp, mp->clsn, log); + if (difft > diffp) + mp->clsn = tblk->clsn; + } else { + mp->log = log; + mp->lsn = lsn; + + /* insert bp after tblock in logsync list */ + log->count++; + list_add(&mp->synclist, &tblk->synclist); + + mp->clsn = tblk->clsn; + } + LOGSYNC_UNLOCK(log, flags); + } + + /* write the last buffer. */ + if (mp) { + write_metapage(mp); + } + + return (0); +} + + +/* + * NAME: dbNextAG() + * + * FUNCTION: find the preferred allocation group for new allocations. + * + * Within the allocation groups, we maintain a preferred + * allocation group which consists of a group with at least + * average free space. It is the preferred group that we target + * new inode allocation towards. The tie-in between inode + * allocation and block allocation occurs as we allocate the + * first (data) block of an inode and specify the inode (block) + * as the allocation hint for this block. + * + * We try to avoid having more than one open file growing in + * an allocation group, as this will lead to fragmentation. + * This differs from the old OS/2 method of trying to keep + * empty ags around for large allocations. + * + * PARAMETERS: + * ipbmap - pointer to in-core inode for the block map. + * + * RETURN VALUES: + * the preferred allocation group number. + */ +int dbNextAG(struct inode *ipbmap) +{ + s64 avgfree; + int agpref; + s64 hwm = 0; + int i; + int next_best = -1; + struct bmap *bmp = JFS_SBI(ipbmap->i_sb)->bmap; + + BMAP_LOCK(bmp); + + /* determine the average number of free blocks within the ags. */ + avgfree = (u32)bmp->db_nfree / bmp->db_numag; + + /* + * if the current preferred ag does not have an active allocator + * and has at least average freespace, return it + */ + agpref = bmp->db_agpref; + if ((atomic_read(&bmp->db_active[agpref]) == 0) && + (bmp->db_agfree[agpref] >= avgfree)) + goto unlock; + + /* From the last preferred ag, find the next one with at least + * average free space. + */ + for (i = 0 ; i < bmp->db_numag; i++, agpref++) { + if (agpref >= bmp->db_numag) + agpref = 0; + + if (atomic_read(&bmp->db_active[agpref])) + /* open file is currently growing in this ag */ + continue; + if (bmp->db_agfree[agpref] >= avgfree) { + /* Return this one */ + bmp->db_agpref = agpref; + goto unlock; + } else if (bmp->db_agfree[agpref] > hwm) { + /* Less than avg. freespace, but best so far */ + hwm = bmp->db_agfree[agpref]; + next_best = agpref; + } + } + + /* + * If no inactive ag was found with average freespace, use the + * next best + */ + if (next_best != -1) + bmp->db_agpref = next_best; + /* else leave db_agpref unchanged */ +unlock: + BMAP_UNLOCK(bmp); + + /* return the preferred group. + */ + return (bmp->db_agpref); +} + +/* + * NAME: dbAlloc() + * + * FUNCTION: attempt to allocate a specified number of contiguous free + * blocks from the working allocation block map. + * + * the block allocation policy uses hints and a multi-step + * approach. + * + * for allocation requests smaller than the number of blocks + * per dmap, we first try to allocate the new blocks + * immediately following the hint. if these blocks are not + * available, we try to allocate blocks near the hint. if + * no blocks near the hint are available, we next try to + * allocate within the same dmap as contains the hint. + * + * if no blocks are available in the dmap or the allocation + * request is larger than the dmap size, we try to allocate + * within the same allocation group as contains the hint. if + * this does not succeed, we finally try to allocate anywhere + * within the aggregate. + * + * we also try to allocate anywhere within the aggregate + * for allocation requests larger than the allocation group + * size or requests that specify no hint value. + * + * PARAMETERS: + * ip - pointer to in-core inode; + * hint - allocation hint. + * nblocks - number of contiguous blocks in the range. + * results - on successful return, set to the starting block number + * of the newly allocated contiguous range. + * + * RETURN VALUES: + * 0 - success + * -ENOSPC - insufficient disk resources + * -EIO - i/o error + */ +int dbAlloc(struct inode *ip, s64 hint, s64 nblocks, s64 * results) +{ + int rc, agno; + struct inode *ipbmap = JFS_SBI(ip->i_sb)->ipbmap; + struct bmap *bmp; + struct metapage *mp; + s64 lblkno, blkno; + struct dmap *dp; + int l2nb; + s64 mapSize; + int writers; + + /* assert that nblocks is valid */ + assert(nblocks > 0); + + /* get the log2 number of blocks to be allocated. + * if the number of blocks is not a log2 multiple, + * it will be rounded up to the next log2 multiple. + */ + l2nb = BLKSTOL2(nblocks); + + bmp = JFS_SBI(ip->i_sb)->bmap; + + mapSize = bmp->db_mapsize; + + /* the hint should be within the map */ + if (hint >= mapSize) { + jfs_error(ip->i_sb, "the hint is outside the map\n"); + return -EIO; + } + + /* if the number of blocks to be allocated is greater than the + * allocation group size, try to allocate anywhere. + */ + if (l2nb > bmp->db_agl2size) { + IWRITE_LOCK(ipbmap, RDWRLOCK_DMAP); + + rc = dbAllocAny(bmp, nblocks, l2nb, results); + + goto write_unlock; + } + + /* + * If no hint, let dbNextAG recommend an allocation group + */ + if (hint == 0) + goto pref_ag; + + /* we would like to allocate close to the hint. adjust the + * hint to the block following the hint since the allocators + * will start looking for free space starting at this point. + */ + blkno = hint + 1; + + if (blkno >= bmp->db_mapsize) + goto pref_ag; + + agno = blkno >> bmp->db_agl2size; + + /* check if blkno crosses over into a new allocation group. + * if so, check if we should allow allocations within this + * allocation group. + */ + if ((blkno & (bmp->db_agsize - 1)) == 0) + /* check if the AG is currently being written to. + * if so, call dbNextAG() to find a non-busy + * AG with sufficient free space. + */ + if (atomic_read(&bmp->db_active[agno])) + goto pref_ag; + + /* check if the allocation request size can be satisfied from a + * single dmap. if so, try to allocate from the dmap containing + * the hint using a tiered strategy. + */ + if (nblocks <= BPERDMAP) { + IREAD_LOCK(ipbmap, RDWRLOCK_DMAP); + + /* get the buffer for the dmap containing the hint. + */ + rc = -EIO; + lblkno = BLKTODMAP(blkno, bmp->db_l2nbperpage); + mp = read_metapage(ipbmap, lblkno, PSIZE, 0); + if (mp == NULL) + goto read_unlock; + + dp = (struct dmap *) mp->data; + + /* first, try to satisfy the allocation request with the + * blocks beginning at the hint. + */ + if ((rc = dbAllocNext(bmp, dp, blkno, (int) nblocks)) + != -ENOSPC) { + if (rc == 0) { + *results = blkno; + mark_metapage_dirty(mp); + } + + release_metapage(mp); + goto read_unlock; + } + + writers = atomic_read(&bmp->db_active[agno]); + if ((writers > 1) || + ((writers == 1) && (JFS_IP(ip)->active_ag != agno))) { + /* + * Someone else is writing in this allocation + * group. To avoid fragmenting, try another ag + */ + release_metapage(mp); + IREAD_UNLOCK(ipbmap); + goto pref_ag; + } + + /* next, try to satisfy the allocation request with blocks + * near the hint. + */ + if ((rc = + dbAllocNear(bmp, dp, blkno, (int) nblocks, l2nb, results)) + != -ENOSPC) { + if (rc == 0) + mark_metapage_dirty(mp); + + release_metapage(mp); + goto read_unlock; + } + + /* try to satisfy the allocation request with blocks within + * the same dmap as the hint. + */ + if ((rc = dbAllocDmapLev(bmp, dp, (int) nblocks, l2nb, results)) + != -ENOSPC) { + if (rc == 0) + mark_metapage_dirty(mp); + + release_metapage(mp); + goto read_unlock; + } + + release_metapage(mp); + IREAD_UNLOCK(ipbmap); + } + + /* try to satisfy the allocation request with blocks within + * the same allocation group as the hint. + */ + IWRITE_LOCK(ipbmap, RDWRLOCK_DMAP); + if ((rc = dbAllocAG(bmp, agno, nblocks, l2nb, results)) != -ENOSPC) + goto write_unlock; + + IWRITE_UNLOCK(ipbmap); + + + pref_ag: + /* + * Let dbNextAG recommend a preferred allocation group + */ + agno = dbNextAG(ipbmap); + IWRITE_LOCK(ipbmap, RDWRLOCK_DMAP); + + /* Try to allocate within this allocation group. if that fails, try to + * allocate anywhere in the map. + */ + if ((rc = dbAllocAG(bmp, agno, nblocks, l2nb, results)) == -ENOSPC) + rc = dbAllocAny(bmp, nblocks, l2nb, results); + + write_unlock: + IWRITE_UNLOCK(ipbmap); + + return (rc); + + read_unlock: + IREAD_UNLOCK(ipbmap); + + return (rc); +} + +/* + * NAME: dbReAlloc() + * + * FUNCTION: attempt to extend a current allocation by a specified + * number of blocks. + * + * this routine attempts to satisfy the allocation request + * by first trying to extend the existing allocation in + * place by allocating the additional blocks as the blocks + * immediately following the current allocation. if these + * blocks are not available, this routine will attempt to + * allocate a new set of contiguous blocks large enough + * to cover the existing allocation plus the additional + * number of blocks required. + * + * PARAMETERS: + * ip - pointer to in-core inode requiring allocation. + * blkno - starting block of the current allocation. + * nblocks - number of contiguous blocks within the current + * allocation. + * addnblocks - number of blocks to add to the allocation. + * results - on successful return, set to the starting block number + * of the existing allocation if the existing allocation + * was extended in place or to a newly allocated contiguous + * range if the existing allocation could not be extended + * in place. + * + * RETURN VALUES: + * 0 - success + * -ENOSPC - insufficient disk resources + * -EIO - i/o error + */ +int +dbReAlloc(struct inode *ip, + s64 blkno, s64 nblocks, s64 addnblocks, s64 * results) +{ + int rc; + + /* try to extend the allocation in place. + */ + if ((rc = dbExtend(ip, blkno, nblocks, addnblocks)) == 0) { + *results = blkno; + return (0); + } else { + if (rc != -ENOSPC) + return (rc); + } + + /* could not extend the allocation in place, so allocate a + * new set of blocks for the entire request (i.e. try to get + * a range of contiguous blocks large enough to cover the + * existing allocation plus the additional blocks.) + */ + return (dbAlloc + (ip, blkno + nblocks - 1, addnblocks + nblocks, results)); +} + + +/* + * NAME: dbExtend() + * + * FUNCTION: attempt to extend a current allocation by a specified + * number of blocks. + * + * this routine attempts to satisfy the allocation request + * by first trying to extend the existing allocation in + * place by allocating the additional blocks as the blocks + * immediately following the current allocation. + * + * PARAMETERS: + * ip - pointer to in-core inode requiring allocation. + * blkno - starting block of the current allocation. + * nblocks - number of contiguous blocks within the current + * allocation. + * addnblocks - number of blocks to add to the allocation. + * + * RETURN VALUES: + * 0 - success + * -ENOSPC - insufficient disk resources + * -EIO - i/o error + */ +static int dbExtend(struct inode *ip, s64 blkno, s64 nblocks, s64 addnblocks) +{ + struct jfs_sb_info *sbi = JFS_SBI(ip->i_sb); + s64 lblkno, lastblkno, extblkno; + uint rel_block; + struct metapage *mp; + struct dmap *dp; + int rc; + struct inode *ipbmap = sbi->ipbmap; + struct bmap *bmp; + + /* + * We don't want a non-aligned extent to cross a page boundary + */ + if (((rel_block = blkno & (sbi->nbperpage - 1))) && + (rel_block + nblocks + addnblocks > sbi->nbperpage)) + return -ENOSPC; + + /* get the last block of the current allocation */ + lastblkno = blkno + nblocks - 1; + + /* determine the block number of the block following + * the existing allocation. + */ + extblkno = lastblkno + 1; + + IREAD_LOCK(ipbmap, RDWRLOCK_DMAP); + + /* better be within the file system */ + bmp = sbi->bmap; + if (lastblkno < 0 || lastblkno >= bmp->db_mapsize) { + IREAD_UNLOCK(ipbmap); + jfs_error(ip->i_sb, "the block is outside the filesystem\n"); + return -EIO; + } + + /* we'll attempt to extend the current allocation in place by + * allocating the additional blocks as the blocks immediately + * following the current allocation. we only try to extend the + * current allocation in place if the number of additional blocks + * can fit into a dmap, the last block of the current allocation + * is not the last block of the file system, and the start of the + * inplace extension is not on an allocation group boundary. + */ + if (addnblocks > BPERDMAP || extblkno >= bmp->db_mapsize || + (extblkno & (bmp->db_agsize - 1)) == 0) { + IREAD_UNLOCK(ipbmap); + return -ENOSPC; + } + + /* get the buffer for the dmap containing the first block + * of the extension. + */ + lblkno = BLKTODMAP(extblkno, bmp->db_l2nbperpage); + mp = read_metapage(ipbmap, lblkno, PSIZE, 0); + if (mp == NULL) { + IREAD_UNLOCK(ipbmap); + return -EIO; + } + + dp = (struct dmap *) mp->data; + + /* try to allocate the blocks immediately following the + * current allocation. + */ + rc = dbAllocNext(bmp, dp, extblkno, (int) addnblocks); + + IREAD_UNLOCK(ipbmap); + + /* were we successful ? */ + if (rc == 0) + write_metapage(mp); + else + /* we were not successful */ + release_metapage(mp); + + return (rc); +} + + +/* + * NAME: dbAllocNext() + * + * FUNCTION: attempt to allocate the blocks of the specified block + * range within a dmap. + * + * PARAMETERS: + * bmp - pointer to bmap descriptor + * dp - pointer to dmap. + * blkno - starting block number of the range. + * nblocks - number of contiguous free blocks of the range. + * + * RETURN VALUES: + * 0 - success + * -ENOSPC - insufficient disk resources + * -EIO - i/o error + * + * serialization: IREAD_LOCK(ipbmap) held on entry/exit; + */ +static int dbAllocNext(struct bmap * bmp, struct dmap * dp, s64 blkno, + int nblocks) +{ + int dbitno, word, rembits, nb, nwords, wbitno, nw; + int l2size; + s8 *leaf; + u32 mask; + + if (dp->tree.leafidx != cpu_to_le32(LEAFIND)) { + jfs_error(bmp->db_ipbmap->i_sb, "Corrupt dmap page\n"); + return -EIO; + } + + /* pick up a pointer to the leaves of the dmap tree. + */ + leaf = dp->tree.stree + le32_to_cpu(dp->tree.leafidx); + + /* determine the bit number and word within the dmap of the + * starting block. + */ + dbitno = blkno & (BPERDMAP - 1); + word = dbitno >> L2DBWORD; + + /* check if the specified block range is contained within + * this dmap. + */ + if (dbitno + nblocks > BPERDMAP) + return -ENOSPC; + + /* check if the starting leaf indicates that anything + * is free. + */ + if (leaf[word] == NOFREE) + return -ENOSPC; + + /* check the dmaps words corresponding to block range to see + * if the block range is free. not all bits of the first and + * last words may be contained within the block range. if this + * is the case, we'll work against those words (i.e. partial first + * and/or last) on an individual basis (a single pass) and examine + * the actual bits to determine if they are free. a single pass + * will be used for all dmap words fully contained within the + * specified range. within this pass, the leaves of the dmap + * tree will be examined to determine if the blocks are free. a + * single leaf may describe the free space of multiple dmap + * words, so we may visit only a subset of the actual leaves + * corresponding to the dmap words of the block range. + */ + for (rembits = nblocks; rembits > 0; rembits -= nb, dbitno += nb) { + /* determine the bit number within the word and + * the number of bits within the word. + */ + wbitno = dbitno & (DBWORD - 1); + nb = min(rembits, DBWORD - wbitno); + + /* check if only part of the word is to be examined. + */ + if (nb < DBWORD) { + /* check if the bits are free. + */ + mask = (ONES << (DBWORD - nb) >> wbitno); + if ((mask & ~le32_to_cpu(dp->wmap[word])) != mask) + return -ENOSPC; + + word += 1; + } else { + /* one or more dmap words are fully contained + * within the block range. determine how many + * words and how many bits. + */ + nwords = rembits >> L2DBWORD; + nb = nwords << L2DBWORD; + + /* now examine the appropriate leaves to determine + * if the blocks are free. + */ + while (nwords > 0) { + /* does the leaf describe any free space ? + */ + if (leaf[word] < BUDMIN) + return -ENOSPC; + + /* determine the l2 number of bits provided + * by this leaf. + */ + l2size = + min_t(int, leaf[word], NLSTOL2BSZ(nwords)); + + /* determine how many words were handled. + */ + nw = BUDSIZE(l2size, BUDMIN); + + nwords -= nw; + word += nw; + } + } + } + + /* allocate the blocks. + */ + return (dbAllocDmap(bmp, dp, blkno, nblocks)); +} + + +/* + * NAME: dbAllocNear() + * + * FUNCTION: attempt to allocate a number of contiguous free blocks near + * a specified block (hint) within a dmap. + * + * starting with the dmap leaf that covers the hint, we'll + * check the next four contiguous leaves for sufficient free + * space. if sufficient free space is found, we'll allocate + * the desired free space. + * + * PARAMETERS: + * bmp - pointer to bmap descriptor + * dp - pointer to dmap. + * blkno - block number to allocate near. + * nblocks - actual number of contiguous free blocks desired. + * l2nb - log2 number of contiguous free blocks desired. + * results - on successful return, set to the starting block number + * of the newly allocated range. + * + * RETURN VALUES: + * 0 - success + * -ENOSPC - insufficient disk resources + * -EIO - i/o error + * + * serialization: IREAD_LOCK(ipbmap) held on entry/exit; + */ +static int +dbAllocNear(struct bmap * bmp, + struct dmap * dp, s64 blkno, int nblocks, int l2nb, s64 * results) +{ + int word, lword, rc; + s8 *leaf; + + if (dp->tree.leafidx != cpu_to_le32(LEAFIND)) { + jfs_error(bmp->db_ipbmap->i_sb, "Corrupt dmap page\n"); + return -EIO; + } + + leaf = dp->tree.stree + le32_to_cpu(dp->tree.leafidx); + + /* determine the word within the dmap that holds the hint + * (i.e. blkno). also, determine the last word in the dmap + * that we'll include in our examination. + */ + word = (blkno & (BPERDMAP - 1)) >> L2DBWORD; + lword = min(word + 4, LPERDMAP); + + /* examine the leaves for sufficient free space. + */ + for (; word < lword; word++) { + /* does the leaf describe sufficient free space ? + */ + if (leaf[word] < l2nb) + continue; + + /* determine the block number within the file system + * of the first block described by this dmap word. + */ + blkno = le64_to_cpu(dp->start) + (word << L2DBWORD); + + /* if not all bits of the dmap word are free, get the + * starting bit number within the dmap word of the required + * string of free bits and adjust the block number with the + * value. + */ + if (leaf[word] < BUDMIN) + blkno += + dbFindBits(le32_to_cpu(dp->wmap[word]), l2nb); + + /* allocate the blocks. + */ + if ((rc = dbAllocDmap(bmp, dp, blkno, nblocks)) == 0) + *results = blkno; + + return (rc); + } + + return -ENOSPC; +} + + +/* + * NAME: dbAllocAG() + * + * FUNCTION: attempt to allocate the specified number of contiguous + * free blocks within the specified allocation group. + * + * unless the allocation group size is equal to the number + * of blocks per dmap, the dmap control pages will be used to + * find the required free space, if available. we start the + * search at the highest dmap control page level which + * distinctly describes the allocation group's free space + * (i.e. the highest level at which the allocation group's + * free space is not mixed in with that of any other group). + * in addition, we start the search within this level at a + * height of the dmapctl dmtree at which the nodes distinctly + * describe the allocation group's free space. at this height, + * the allocation group's free space may be represented by 1 + * or two sub-trees, depending on the allocation group size. + * we search the top nodes of these subtrees left to right for + * sufficient free space. if sufficient free space is found, + * the subtree is searched to find the leftmost leaf that + * has free space. once we have made it to the leaf, we + * move the search to the next lower level dmap control page + * corresponding to this leaf. we continue down the dmap control + * pages until we find the dmap that contains or starts the + * sufficient free space and we allocate at this dmap. + * + * if the allocation group size is equal to the dmap size, + * we'll start at the dmap corresponding to the allocation + * group and attempt the allocation at this level. + * + * the dmap control page search is also not performed if the + * allocation group is completely free and we go to the first + * dmap of the allocation group to do the allocation. this is + * done because the allocation group may be part (not the first + * part) of a larger binary buddy system, causing the dmap + * control pages to indicate no free space (NOFREE) within + * the allocation group. + * + * PARAMETERS: + * bmp - pointer to bmap descriptor + * agno - allocation group number. + * nblocks - actual number of contiguous free blocks desired. + * l2nb - log2 number of contiguous free blocks desired. + * results - on successful return, set to the starting block number + * of the newly allocated range. + * + * RETURN VALUES: + * 0 - success + * -ENOSPC - insufficient disk resources + * -EIO - i/o error + * + * note: IWRITE_LOCK(ipmap) held on entry/exit; + */ +static int +dbAllocAG(struct bmap * bmp, int agno, s64 nblocks, int l2nb, s64 * results) +{ + struct metapage *mp; + struct dmapctl *dcp; + int rc, ti, i, k, m, n, agperlev; + s64 blkno, lblkno; + int budmin; + + /* allocation request should not be for more than the + * allocation group size. + */ + if (l2nb > bmp->db_agl2size) { + jfs_error(bmp->db_ipbmap->i_sb, + "allocation request is larger than the allocation group size\n"); + return -EIO; + } + + /* determine the starting block number of the allocation + * group. + */ + blkno = (s64) agno << bmp->db_agl2size; + + /* check if the allocation group size is the minimum allocation + * group size or if the allocation group is completely free. if + * the allocation group size is the minimum size of BPERDMAP (i.e. + * 1 dmap), there is no need to search the dmap control page (below) + * that fully describes the allocation group since the allocation + * group is already fully described by a dmap. in this case, we + * just call dbAllocCtl() to search the dmap tree and allocate the + * required space if available. + * + * if the allocation group is completely free, dbAllocCtl() is + * also called to allocate the required space. this is done for + * two reasons. first, it makes no sense searching the dmap control + * pages for free space when we know that free space exists. second, + * the dmap control pages may indicate that the allocation group + * has no free space if the allocation group is part (not the first + * part) of a larger binary buddy system. + */ + if (bmp->db_agsize == BPERDMAP + || bmp->db_agfree[agno] == bmp->db_agsize) { + rc = dbAllocCtl(bmp, nblocks, l2nb, blkno, results); + if ((rc == -ENOSPC) && + (bmp->db_agfree[agno] == bmp->db_agsize)) { + printk(KERN_ERR "blkno = %Lx, blocks = %Lx\n", + (unsigned long long) blkno, + (unsigned long long) nblocks); + jfs_error(bmp->db_ipbmap->i_sb, + "dbAllocCtl failed in free AG\n"); + } + return (rc); + } + + /* the buffer for the dmap control page that fully describes the + * allocation group. + */ + lblkno = BLKTOCTL(blkno, bmp->db_l2nbperpage, bmp->db_aglevel); + mp = read_metapage(bmp->db_ipbmap, lblkno, PSIZE, 0); + if (mp == NULL) + return -EIO; + dcp = (struct dmapctl *) mp->data; + budmin = dcp->budmin; + + if (unlikely(!check_dmapctl(dcp))) { + jfs_error(bmp->db_ipbmap->i_sb, "Corrupt dmapctl page\n"); + release_metapage(mp); + return -EIO; + } + + /* search the subtree(s) of the dmap control page that describes + * the allocation group, looking for sufficient free space. to begin, + * determine how many allocation groups are represented in a dmap + * control page at the control page level (i.e. L0, L1, L2) that + * fully describes an allocation group. next, determine the starting + * tree index of this allocation group within the control page. + */ + agperlev = + (1 << (L2LPERCTL - (bmp->db_agheight << 1))) / bmp->db_agwidth; + ti = bmp->db_agstart + bmp->db_agwidth * (agno & (agperlev - 1)); + + if (ti < 0 || ti >= le32_to_cpu(dcp->nleafs)) { + jfs_error(bmp->db_ipbmap->i_sb, "Corrupt dmapctl page\n"); + release_metapage(mp); + return -EIO; + } + + /* dmap control page trees fan-out by 4 and a single allocation + * group may be described by 1 or 2 subtrees within the ag level + * dmap control page, depending upon the ag size. examine the ag's + * subtrees for sufficient free space, starting with the leftmost + * subtree. + */ + for (i = 0; i < bmp->db_agwidth; i++, ti++) { + /* is there sufficient free space ? + */ + if (l2nb > dcp->stree[ti]) + continue; + + /* sufficient free space found in a subtree. now search down + * the subtree to find the leftmost leaf that describes this + * free space. + */ + for (k = bmp->db_agheight; k > 0; k--) { + for (n = 0, m = (ti << 2) + 1; n < 4; n++) { + if (l2nb <= dcp->stree[m + n]) { + ti = m + n; + break; + } + } + if (n == 4) { + jfs_error(bmp->db_ipbmap->i_sb, + "failed descending stree\n"); + release_metapage(mp); + return -EIO; + } + } + + /* determine the block number within the file system + * that corresponds to this leaf. + */ + if (bmp->db_aglevel == 2) + blkno = 0; + else if (bmp->db_aglevel == 1) + blkno &= ~(MAXL1SIZE - 1); + else /* bmp->db_aglevel == 0 */ + blkno &= ~(MAXL0SIZE - 1); + + blkno += + ((s64) (ti - le32_to_cpu(dcp->leafidx))) << budmin; + + /* release the buffer in preparation for going down + * the next level of dmap control pages. + */ + release_metapage(mp); + + /* check if we need to continue to search down the lower + * level dmap control pages. we need to if the number of + * blocks required is less than maximum number of blocks + * described at the next lower level. + */ + if (l2nb < budmin) { + + /* search the lower level dmap control pages to get + * the starting block number of the dmap that + * contains or starts off the free space. + */ + if ((rc = + dbFindCtl(bmp, l2nb, bmp->db_aglevel - 1, + &blkno))) { + if (rc == -ENOSPC) { + jfs_error(bmp->db_ipbmap->i_sb, + "control page inconsistent\n"); + return -EIO; + } + return (rc); + } + } + + /* allocate the blocks. + */ + rc = dbAllocCtl(bmp, nblocks, l2nb, blkno, results); + if (rc == -ENOSPC) { + jfs_error(bmp->db_ipbmap->i_sb, + "unable to allocate blocks\n"); + rc = -EIO; + } + return (rc); + } + + /* no space in the allocation group. release the buffer and + * return -ENOSPC. + */ + release_metapage(mp); + + return -ENOSPC; +} + + +/* + * NAME: dbAllocAny() + * + * FUNCTION: attempt to allocate the specified number of contiguous + * free blocks anywhere in the file system. + * + * dbAllocAny() attempts to find the sufficient free space by + * searching down the dmap control pages, starting with the + * highest level (i.e. L0, L1, L2) control page. if free space + * large enough to satisfy the desired free space is found, the + * desired free space is allocated. + * + * PARAMETERS: + * bmp - pointer to bmap descriptor + * nblocks - actual number of contiguous free blocks desired. + * l2nb - log2 number of contiguous free blocks desired. + * results - on successful return, set to the starting block number + * of the newly allocated range. + * + * RETURN VALUES: + * 0 - success + * -ENOSPC - insufficient disk resources + * -EIO - i/o error + * + * serialization: IWRITE_LOCK(ipbmap) held on entry/exit; + */ +static int dbAllocAny(struct bmap * bmp, s64 nblocks, int l2nb, s64 * results) +{ + int rc; + s64 blkno = 0; + + /* starting with the top level dmap control page, search + * down the dmap control levels for sufficient free space. + * if free space is found, dbFindCtl() returns the starting + * block number of the dmap that contains or starts off the + * range of free space. + */ + if ((rc = dbFindCtl(bmp, l2nb, bmp->db_maxlevel, &blkno))) + return (rc); + + /* allocate the blocks. + */ + rc = dbAllocCtl(bmp, nblocks, l2nb, blkno, results); + if (rc == -ENOSPC) { + jfs_error(bmp->db_ipbmap->i_sb, "unable to allocate blocks\n"); + return -EIO; + } + return (rc); +} + + +/* + * NAME: dbDiscardAG() + * + * FUNCTION: attempt to discard (TRIM) all free blocks of specific AG + * + * algorithm: + * 1) allocate blocks, as large as possible and save them + * while holding IWRITE_LOCK on ipbmap + * 2) trim all these saved block/length values + * 3) mark the blocks free again + * + * benefit: + * - we work only on one ag at some time, minimizing how long we + * need to lock ipbmap + * - reading / writing the fs is possible most time, even on + * trimming + * + * downside: + * - we write two times to the dmapctl and dmap pages + * - but for me, this seems the best way, better ideas? + * /TR 2012 + * + * PARAMETERS: + * ip - pointer to in-core inode + * agno - ag to trim + * minlen - minimum value of contiguous blocks + * + * RETURN VALUES: + * s64 - actual number of blocks trimmed + */ +s64 dbDiscardAG(struct inode *ip, int agno, s64 minlen) +{ + struct inode *ipbmap = JFS_SBI(ip->i_sb)->ipbmap; + struct bmap *bmp = JFS_SBI(ip->i_sb)->bmap; + s64 nblocks, blkno; + u64 trimmed = 0; + int rc, l2nb; + struct super_block *sb = ipbmap->i_sb; + + struct range2trim { + u64 blkno; + u64 nblocks; + } *totrim, *tt; + + /* max blkno / nblocks pairs to trim */ + int count = 0, range_cnt; + u64 max_ranges; + + /* prevent others from writing new stuff here, while trimming */ + IWRITE_LOCK(ipbmap, RDWRLOCK_DMAP); + + nblocks = bmp->db_agfree[agno]; + max_ranges = nblocks; + do_div(max_ranges, minlen); + range_cnt = min_t(u64, max_ranges + 1, 32 * 1024); + totrim = kmalloc_objs(struct range2trim, range_cnt, GFP_NOFS); + if (totrim == NULL) { + jfs_error(bmp->db_ipbmap->i_sb, "no memory for trim array\n"); + IWRITE_UNLOCK(ipbmap); + return 0; + } + + tt = totrim; + while (nblocks >= minlen) { + l2nb = BLKSTOL2(nblocks); + + /* 0 = okay, -EIO = fatal, -ENOSPC -> try smaller block */ + rc = dbAllocAG(bmp, agno, nblocks, l2nb, &blkno); + if (rc == 0) { + tt->blkno = blkno; + tt->nblocks = nblocks; + tt++; count++; + + /* the whole ag is free, trim now */ + if (bmp->db_agfree[agno] == 0) + break; + + /* give a hint for the next while */ + nblocks = bmp->db_agfree[agno]; + continue; + } else if (rc == -ENOSPC) { + /* search for next smaller log2 block */ + l2nb = BLKSTOL2(nblocks) - 1; + if (unlikely(l2nb < 0)) + break; + nblocks = 1LL << l2nb; + } else { + /* Trim any already allocated blocks */ + jfs_error(bmp->db_ipbmap->i_sb, "-EIO\n"); + break; + } + + /* check, if our trim array is full */ + if (unlikely(count >= range_cnt - 1)) + break; + } + IWRITE_UNLOCK(ipbmap); + + tt->nblocks = 0; /* mark the current end */ + for (tt = totrim; tt->nblocks != 0; tt++) { + /* when mounted with online discard, dbFree() will + * call jfs_issue_discard() itself */ + if (!(JFS_SBI(sb)->flag & JFS_DISCARD)) + jfs_issue_discard(ip, tt->blkno, tt->nblocks); + dbFree(ip, tt->blkno, tt->nblocks); + trimmed += tt->nblocks; + } + kfree(totrim); + + return trimmed; +} + +/* + * NAME: dbFindCtl() + * + * FUNCTION: starting at a specified dmap control page level and block + * number, search down the dmap control levels for a range of + * contiguous free blocks large enough to satisfy an allocation + * request for the specified number of free blocks. + * + * if sufficient contiguous free blocks are found, this routine + * returns the starting block number within a dmap page that + * contains or starts a range of contiqious free blocks that + * is sufficient in size. + * + * PARAMETERS: + * bmp - pointer to bmap descriptor + * level - starting dmap control page level. + * l2nb - log2 number of contiguous free blocks desired. + * *blkno - on entry, starting block number for conducting the search. + * on successful return, the first block within a dmap page + * that contains or starts a range of contiguous free blocks. + * + * RETURN VALUES: + * 0 - success + * -ENOSPC - insufficient disk resources + * -EIO - i/o error + * + * serialization: IWRITE_LOCK(ipbmap) held on entry/exit; + */ +static int dbFindCtl(struct bmap * bmp, int l2nb, int level, s64 * blkno) +{ + int rc, leafidx, lev; + s64 b, lblkno; + struct dmapctl *dcp; + int budmin; + struct metapage *mp; + + /* starting at the specified dmap control page level and block + * number, search down the dmap control levels for the starting + * block number of a dmap page that contains or starts off + * sufficient free blocks. + */ + for (lev = level, b = *blkno; lev >= 0; lev--) { + /* get the buffer of the dmap control page for the block + * number and level (i.e. L0, L1, L2). + */ + lblkno = BLKTOCTL(b, bmp->db_l2nbperpage, lev); + mp = read_metapage(bmp->db_ipbmap, lblkno, PSIZE, 0); + if (mp == NULL) + return -EIO; + dcp = (struct dmapctl *) mp->data; + budmin = dcp->budmin; + + if (unlikely(!check_dmapctl(dcp))) { + jfs_error(bmp->db_ipbmap->i_sb, + "Corrupt dmapctl page\n"); + release_metapage(mp); + return -EIO; + } + + /* search the tree within the dmap control page for + * sufficient free space. if sufficient free space is found, + * dbFindLeaf() returns the index of the leaf at which + * free space was found. + */ + rc = dbFindLeaf((dmtree_t *) dcp, l2nb, &leafidx, true); + + /* release the buffer. + */ + release_metapage(mp); + + /* space found ? + */ + if (rc) { + if (lev != level) { + jfs_error(bmp->db_ipbmap->i_sb, + "dmap inconsistent\n"); + return -EIO; + } + return -ENOSPC; + } + + /* adjust the block number to reflect the location within + * the dmap control page (i.e. the leaf) at which free + * space was found. + */ + b += (((s64) leafidx) << budmin); + + /* we stop the search at this dmap control page level if + * the number of blocks required is greater than or equal + * to the maximum number of blocks described at the next + * (lower) level. + */ + if (l2nb >= budmin) + break; + } + + *blkno = b; + return (0); +} + + +/* + * NAME: dbAllocCtl() + * + * FUNCTION: attempt to allocate a specified number of contiguous + * blocks starting within a specific dmap. + * + * this routine is called by higher level routines that search + * the dmap control pages above the actual dmaps for contiguous + * free space. the result of successful searches by these + * routines are the starting block numbers within dmaps, with + * the dmaps themselves containing the desired contiguous free + * space or starting a contiguous free space of desired size + * that is made up of the blocks of one or more dmaps. these + * calls should not fail due to insufficent resources. + * + * this routine is called in some cases where it is not known + * whether it will fail due to insufficient resources. more + * specifically, this occurs when allocating from an allocation + * group whose size is equal to the number of blocks per dmap. + * in this case, the dmap control pages are not examined prior + * to calling this routine (to save pathlength) and the call + * might fail. + * + * for a request size that fits within a dmap, this routine relies + * upon the dmap's dmtree to find the requested contiguous free + * space. for request sizes that are larger than a dmap, the + * requested free space will start at the first block of the + * first dmap (i.e. blkno). + * + * PARAMETERS: + * bmp - pointer to bmap descriptor + * nblocks - actual number of contiguous free blocks to allocate. + * l2nb - log2 number of contiguous free blocks to allocate. + * blkno - starting block number of the dmap to start the allocation + * from. + * results - on successful return, set to the starting block number + * of the newly allocated range. + * + * RETURN VALUES: + * 0 - success + * -ENOSPC - insufficient disk resources + * -EIO - i/o error + * + * serialization: IWRITE_LOCK(ipbmap) held on entry/exit; + */ +static int +dbAllocCtl(struct bmap * bmp, s64 nblocks, int l2nb, s64 blkno, s64 * results) +{ + int rc, nb; + s64 b, lblkno, n; + struct metapage *mp; + struct dmap *dp; + + /* check if the allocation request is confined to a single dmap. + */ + if (l2nb <= L2BPERDMAP) { + /* get the buffer for the dmap. + */ + lblkno = BLKTODMAP(blkno, bmp->db_l2nbperpage); + mp = read_metapage(bmp->db_ipbmap, lblkno, PSIZE, 0); + if (mp == NULL) + return -EIO; + dp = (struct dmap *) mp->data; + + if (dp->tree.budmin < 0) { + release_metapage(mp); + return -EIO; + } + + /* try to allocate the blocks. + */ + rc = dbAllocDmapLev(bmp, dp, (int) nblocks, l2nb, results); + if (rc == 0) + mark_metapage_dirty(mp); + + release_metapage(mp); + + return (rc); + } + + /* allocation request involving multiple dmaps. it must start on + * a dmap boundary. + */ + assert((blkno & (BPERDMAP - 1)) == 0); + + /* allocate the blocks dmap by dmap. + */ + for (n = nblocks, b = blkno; n > 0; n -= nb, b += nb) { + /* get the buffer for the dmap. + */ + lblkno = BLKTODMAP(b, bmp->db_l2nbperpage); + mp = read_metapage(bmp->db_ipbmap, lblkno, PSIZE, 0); + if (mp == NULL) { + rc = -EIO; + goto backout; + } + dp = (struct dmap *) mp->data; + + /* the dmap better be all free. + */ + if (dp->tree.stree[ROOT] != L2BPERDMAP) { + release_metapage(mp); + jfs_error(bmp->db_ipbmap->i_sb, + "the dmap is not all free\n"); + rc = -EIO; + goto backout; + } + + /* determine how many blocks to allocate from this dmap. + */ + nb = min_t(s64, n, BPERDMAP); + + /* allocate the blocks from the dmap. + */ + if ((rc = dbAllocDmap(bmp, dp, b, nb))) { + release_metapage(mp); + goto backout; + } + + /* write the buffer. + */ + write_metapage(mp); + } + + /* set the results (starting block number) and return. + */ + *results = blkno; + return (0); + + /* something failed in handling an allocation request involving + * multiple dmaps. we'll try to clean up by backing out any + * allocation that has already happened for this request. if + * we fail in backing out the allocation, we'll mark the file + * system to indicate that blocks have been leaked. + */ + backout: + + /* try to backout the allocations dmap by dmap. + */ + for (n = nblocks - n, b = blkno; n > 0; + n -= BPERDMAP, b += BPERDMAP) { + /* get the buffer for this dmap. + */ + lblkno = BLKTODMAP(b, bmp->db_l2nbperpage); + mp = read_metapage(bmp->db_ipbmap, lblkno, PSIZE, 0); + if (mp == NULL) { + /* could not back out. mark the file system + * to indicate that we have leaked blocks. + */ + jfs_error(bmp->db_ipbmap->i_sb, + "I/O Error: Block Leakage\n"); + continue; + } + dp = (struct dmap *) mp->data; + + /* free the blocks is this dmap. + */ + if (dbFreeDmap(bmp, dp, b, BPERDMAP)) { + /* could not back out. mark the file system + * to indicate that we have leaked blocks. + */ + release_metapage(mp); + jfs_error(bmp->db_ipbmap->i_sb, "Block Leakage\n"); + continue; + } + + /* write the buffer. + */ + write_metapage(mp); + } + + return (rc); +} + + +/* + * NAME: dbAllocDmapLev() + * + * FUNCTION: attempt to allocate a specified number of contiguous blocks + * from a specified dmap. + * + * this routine checks if the contiguous blocks are available. + * if so, nblocks of blocks are allocated; otherwise, ENOSPC is + * returned. + * + * PARAMETERS: + * mp - pointer to bmap descriptor + * dp - pointer to dmap to attempt to allocate blocks from. + * l2nb - log2 number of contiguous block desired. + * nblocks - actual number of contiguous block desired. + * results - on successful return, set to the starting block number + * of the newly allocated range. + * + * RETURN VALUES: + * 0 - success + * -ENOSPC - insufficient disk resources + * -EIO - i/o error + * + * serialization: IREAD_LOCK(ipbmap), e.g., from dbAlloc(), or + * IWRITE_LOCK(ipbmap), e.g., dbAllocCtl(), held on entry/exit; + */ +static int +dbAllocDmapLev(struct bmap * bmp, + struct dmap * dp, int nblocks, int l2nb, s64 * results) +{ + s64 blkno; + int leafidx, rc; + + /* can't be more than a dmaps worth of blocks */ + assert(l2nb <= L2BPERDMAP); + + /* search the tree within the dmap page for sufficient + * free space. if sufficient free space is found, dbFindLeaf() + * returns the index of the leaf at which free space was found. + */ + if (dbFindLeaf((dmtree_t *) &dp->tree, l2nb, &leafidx, false)) + return -ENOSPC; + + if (leafidx < 0) + return -EIO; + + /* determine the block number within the file system corresponding + * to the leaf at which free space was found. + */ + blkno = le64_to_cpu(dp->start) + (leafidx << L2DBWORD); + + /* if not all bits of the dmap word are free, get the starting + * bit number within the dmap word of the required string of free + * bits and adjust the block number with this value. + */ + if (dp->tree.stree[leafidx + LEAFIND] < BUDMIN) + blkno += dbFindBits(le32_to_cpu(dp->wmap[leafidx]), l2nb); + + /* allocate the blocks */ + if ((rc = dbAllocDmap(bmp, dp, blkno, nblocks)) == 0) + *results = blkno; + + return (rc); +} + + +/* + * NAME: dbAllocDmap() + * + * FUNCTION: adjust the disk allocation map to reflect the allocation + * of a specified block range within a dmap. + * + * this routine allocates the specified blocks from the dmap + * through a call to dbAllocBits(). if the allocation of the + * block range causes the maximum string of free blocks within + * the dmap to change (i.e. the value of the root of the dmap's + * dmtree), this routine will cause this change to be reflected + * up through the appropriate levels of the dmap control pages + * by a call to dbAdjCtl() for the L0 dmap control page that + * covers this dmap. + * + * PARAMETERS: + * bmp - pointer to bmap descriptor + * dp - pointer to dmap to allocate the block range from. + * blkno - starting block number of the block to be allocated. + * nblocks - number of blocks to be allocated. + * + * RETURN VALUES: + * 0 - success + * -EIO - i/o error + * + * serialization: IREAD_LOCK(ipbmap) or IWRITE_LOCK(ipbmap) held on entry/exit; + */ +static int dbAllocDmap(struct bmap * bmp, struct dmap * dp, s64 blkno, + int nblocks) +{ + s8 oldroot; + int rc; + + /* save the current value of the root (i.e. maximum free string) + * of the dmap tree. + */ + oldroot = dp->tree.stree[ROOT]; + + /* allocate the specified (blocks) bits */ + dbAllocBits(bmp, dp, blkno, nblocks); + + /* if the root has not changed, done. */ + if (dp->tree.stree[ROOT] == oldroot) + return (0); + + /* root changed. bubble the change up to the dmap control pages. + * if the adjustment of the upper level control pages fails, + * backout the bit allocation (thus making everything consistent). + */ + if ((rc = dbAdjCtl(bmp, blkno, dp->tree.stree[ROOT], 1, 0))) + dbFreeBits(bmp, dp, blkno, nblocks); + + return (rc); +} + + +/* + * NAME: dbFreeDmap() + * + * FUNCTION: adjust the disk allocation map to reflect the allocation + * of a specified block range within a dmap. + * + * this routine frees the specified blocks from the dmap through + * a call to dbFreeBits(). if the deallocation of the block range + * causes the maximum string of free blocks within the dmap to + * change (i.e. the value of the root of the dmap's dmtree), this + * routine will cause this change to be reflected up through the + * appropriate levels of the dmap control pages by a call to + * dbAdjCtl() for the L0 dmap control page that covers this dmap. + * + * PARAMETERS: + * bmp - pointer to bmap descriptor + * dp - pointer to dmap to free the block range from. + * blkno - starting block number of the block to be freed. + * nblocks - number of blocks to be freed. + * + * RETURN VALUES: + * 0 - success + * -EIO - i/o error + * + * serialization: IREAD_LOCK(ipbmap) or IWRITE_LOCK(ipbmap) held on entry/exit; + */ +static int dbFreeDmap(struct bmap * bmp, struct dmap * dp, s64 blkno, + int nblocks) +{ + s8 oldroot; + int rc = 0, word; + + /* save the current value of the root (i.e. maximum free string) + * of the dmap tree. + */ + oldroot = dp->tree.stree[ROOT]; + + /* free the specified (blocks) bits */ + rc = dbFreeBits(bmp, dp, blkno, nblocks); + + /* if error or the root has not changed, done. */ + if (rc || (dp->tree.stree[ROOT] == oldroot)) + return (rc); + + /* root changed. bubble the change up to the dmap control pages. + * if the adjustment of the upper level control pages fails, + * backout the deallocation. + */ + if ((rc = dbAdjCtl(bmp, blkno, dp->tree.stree[ROOT], 0, 0))) { + word = (blkno & (BPERDMAP - 1)) >> L2DBWORD; + + /* as part of backing out the deallocation, we will have + * to back split the dmap tree if the deallocation caused + * the freed blocks to become part of a larger binary buddy + * system. + */ + if (dp->tree.stree[word] == NOFREE) + dbBackSplit((dmtree_t *)&dp->tree, word, false); + + dbAllocBits(bmp, dp, blkno, nblocks); + } + + return (rc); +} + + +/* + * NAME: dbAllocBits() + * + * FUNCTION: allocate a specified block range from a dmap. + * + * this routine updates the dmap to reflect the working + * state allocation of the specified block range. it directly + * updates the bits of the working map and causes the adjustment + * of the binary buddy system described by the dmap's dmtree + * leaves to reflect the bits allocated. it also causes the + * dmap's dmtree, as a whole, to reflect the allocated range. + * + * PARAMETERS: + * bmp - pointer to bmap descriptor + * dp - pointer to dmap to allocate bits from. + * blkno - starting block number of the bits to be allocated. + * nblocks - number of bits to be allocated. + * + * RETURN VALUES: none + * + * serialization: IREAD_LOCK(ipbmap) or IWRITE_LOCK(ipbmap) held on entry/exit; + */ +static void dbAllocBits(struct bmap * bmp, struct dmap * dp, s64 blkno, + int nblocks) +{ + int dbitno, word, rembits, nb, nwords, wbitno, nw, agno; + dmtree_t *tp = (dmtree_t *) & dp->tree; + int size; + s8 *leaf; + + /* pick up a pointer to the leaves of the dmap tree */ + leaf = dp->tree.stree + LEAFIND; + + /* determine the bit number and word within the dmap of the + * starting block. + */ + dbitno = blkno & (BPERDMAP - 1); + word = dbitno >> L2DBWORD; + + /* block range better be within the dmap */ + assert(dbitno + nblocks <= BPERDMAP); + + /* allocate the bits of the dmap's words corresponding to the block + * range. not all bits of the first and last words may be contained + * within the block range. if this is the case, we'll work against + * those words (i.e. partial first and/or last) on an individual basis + * (a single pass), allocating the bits of interest by hand and + * updating the leaf corresponding to the dmap word. a single pass + * will be used for all dmap words fully contained within the + * specified range. within this pass, the bits of all fully contained + * dmap words will be marked as free in a single shot and the leaves + * will be updated. a single leaf may describe the free space of + * multiple dmap words, so we may update only a subset of the actual + * leaves corresponding to the dmap words of the block range. + */ + for (rembits = nblocks; rembits > 0; rembits -= nb, dbitno += nb) { + /* determine the bit number within the word and + * the number of bits within the word. + */ + wbitno = dbitno & (DBWORD - 1); + nb = min(rembits, DBWORD - wbitno); + + /* check if only part of a word is to be allocated. + */ + if (nb < DBWORD) { + /* allocate (set to 1) the appropriate bits within + * this dmap word. + */ + dp->wmap[word] |= cpu_to_le32(ONES << (DBWORD - nb) + >> wbitno); + + /* update the leaf for this dmap word. in addition + * to setting the leaf value to the binary buddy max + * of the updated dmap word, dbSplit() will split + * the binary system of the leaves if need be. + */ + dbSplit(tp, word, BUDMIN, + dbMaxBud((u8 *)&dp->wmap[word]), false); + + word += 1; + } else { + /* one or more dmap words are fully contained + * within the block range. determine how many + * words and allocate (set to 1) the bits of these + * words. + */ + nwords = rembits >> L2DBWORD; + memset(&dp->wmap[word], (int) ONES, nwords * 4); + + /* determine how many bits. + */ + nb = nwords << L2DBWORD; + + /* now update the appropriate leaves to reflect + * the allocated words. + */ + for (; nwords > 0; nwords -= nw) { + if (leaf[word] < BUDMIN) { + jfs_error(bmp->db_ipbmap->i_sb, + "leaf page corrupt\n"); + break; + } + + /* determine what the leaf value should be + * updated to as the minimum of the l2 number + * of bits being allocated and the l2 number + * of bits currently described by this leaf. + */ + size = min_t(int, leaf[word], + NLSTOL2BSZ(nwords)); + + /* update the leaf to reflect the allocation. + * in addition to setting the leaf value to + * NOFREE, dbSplit() will split the binary + * system of the leaves to reflect the current + * allocation (size). + */ + dbSplit(tp, word, size, NOFREE, false); + + /* get the number of dmap words handled */ + nw = BUDSIZE(size, BUDMIN); + word += nw; + } + } + } + + /* update the free count for this dmap */ + le32_add_cpu(&dp->nfree, -nblocks); + + BMAP_LOCK(bmp); + + /* if this allocation group is completely free, + * update the maximum allocation group number if this allocation + * group is the new max. + */ + agno = blkno >> bmp->db_agl2size; + if (agno > bmp->db_maxag) + bmp->db_maxag = agno; + + /* update the free count for the allocation group and map */ + bmp->db_agfree[agno] -= nblocks; + bmp->db_nfree -= nblocks; + + BMAP_UNLOCK(bmp); +} + + +/* + * NAME: dbFreeBits() + * + * FUNCTION: free a specified block range from a dmap. + * + * this routine updates the dmap to reflect the working + * state allocation of the specified block range. it directly + * updates the bits of the working map and causes the adjustment + * of the binary buddy system described by the dmap's dmtree + * leaves to reflect the bits freed. it also causes the dmap's + * dmtree, as a whole, to reflect the deallocated range. + * + * PARAMETERS: + * bmp - pointer to bmap descriptor + * dp - pointer to dmap to free bits from. + * blkno - starting block number of the bits to be freed. + * nblocks - number of bits to be freed. + * + * RETURN VALUES: 0 for success + * + * serialization: IREAD_LOCK(ipbmap) or IWRITE_LOCK(ipbmap) held on entry/exit; + */ +static int dbFreeBits(struct bmap * bmp, struct dmap * dp, s64 blkno, + int nblocks) +{ + int dbitno, word, rembits, nb, nwords, wbitno, nw, agno; + dmtree_t *tp = (dmtree_t *) & dp->tree; + int rc = 0; + int size; + + /* determine the bit number and word within the dmap of the + * starting block. + */ + dbitno = blkno & (BPERDMAP - 1); + word = dbitno >> L2DBWORD; + + /* block range better be within the dmap. + */ + assert(dbitno + nblocks <= BPERDMAP); + + /* free the bits of the dmaps words corresponding to the block range. + * not all bits of the first and last words may be contained within + * the block range. if this is the case, we'll work against those + * words (i.e. partial first and/or last) on an individual basis + * (a single pass), freeing the bits of interest by hand and updating + * the leaf corresponding to the dmap word. a single pass will be used + * for all dmap words fully contained within the specified range. + * within this pass, the bits of all fully contained dmap words will + * be marked as free in a single shot and the leaves will be updated. a + * single leaf may describe the free space of multiple dmap words, + * so we may update only a subset of the actual leaves corresponding + * to the dmap words of the block range. + * + * dbJoin() is used to update leaf values and will join the binary + * buddy system of the leaves if the new leaf values indicate this + * should be done. + */ + for (rembits = nblocks; rembits > 0; rembits -= nb, dbitno += nb) { + /* determine the bit number within the word and + * the number of bits within the word. + */ + wbitno = dbitno & (DBWORD - 1); + nb = min(rembits, DBWORD - wbitno); + + /* check if only part of a word is to be freed. + */ + if (nb < DBWORD) { + /* free (zero) the appropriate bits within this + * dmap word. + */ + dp->wmap[word] &= + cpu_to_le32(~(ONES << (DBWORD - nb) + >> wbitno)); + + /* update the leaf for this dmap word. + */ + rc = dbJoin(tp, word, + dbMaxBud((u8 *)&dp->wmap[word]), false); + if (rc) + return rc; + + word += 1; + } else { + /* one or more dmap words are fully contained + * within the block range. determine how many + * words and free (zero) the bits of these words. + */ + nwords = rembits >> L2DBWORD; + memset(&dp->wmap[word], 0, nwords * 4); + + /* determine how many bits. + */ + nb = nwords << L2DBWORD; + + /* now update the appropriate leaves to reflect + * the freed words. + */ + for (; nwords > 0; nwords -= nw) { + /* determine what the leaf value should be + * updated to as the minimum of the l2 number + * of bits being freed and the l2 (max) number + * of bits that can be described by this leaf. + */ + size = + min(LITOL2BSZ + (word, L2LPERDMAP, BUDMIN), + NLSTOL2BSZ(nwords)); + + /* update the leaf. + */ + rc = dbJoin(tp, word, size, false); + if (rc) + return rc; + + /* get the number of dmap words handled. + */ + nw = BUDSIZE(size, BUDMIN); + word += nw; + } + } + } + + /* update the free count for this dmap. + */ + le32_add_cpu(&dp->nfree, nblocks); + + BMAP_LOCK(bmp); + + /* update the free count for the allocation group and + * map. + */ + agno = blkno >> bmp->db_agl2size; + bmp->db_nfree += nblocks; + bmp->db_agfree[agno] += nblocks; + + /* check if this allocation group is not completely free and + * if it is currently the maximum (rightmost) allocation group. + * if so, establish the new maximum allocation group number by + * searching left for the first allocation group with allocation. + */ + if ((bmp->db_agfree[agno] == bmp->db_agsize && agno == bmp->db_maxag) || + (agno == bmp->db_numag - 1 && + bmp->db_agfree[agno] == (bmp-> db_mapsize & (BPERDMAP - 1)))) { + while (bmp->db_maxag > 0) { + bmp->db_maxag -= 1; + if (bmp->db_agfree[bmp->db_maxag] != + bmp->db_agsize) + break; + } + + /* re-establish the allocation group preference if the + * current preference is right of the maximum allocation + * group. + */ + if (bmp->db_agpref > bmp->db_maxag) + bmp->db_agpref = bmp->db_maxag; + } + + BMAP_UNLOCK(bmp); + + return 0; +} + + +/* + * NAME: dbAdjCtl() + * + * FUNCTION: adjust a dmap control page at a specified level to reflect + * the change in a lower level dmap or dmap control page's + * maximum string of free blocks (i.e. a change in the root + * of the lower level object's dmtree) due to the allocation + * or deallocation of a range of blocks with a single dmap. + * + * on entry, this routine is provided with the new value of + * the lower level dmap or dmap control page root and the + * starting block number of the block range whose allocation + * or deallocation resulted in the root change. this range + * is respresented by a single leaf of the current dmapctl + * and the leaf will be updated with this value, possibly + * causing a binary buddy system within the leaves to be + * split or joined. the update may also cause the dmapctl's + * dmtree to be updated. + * + * if the adjustment of the dmap control page, itself, causes its + * root to change, this change will be bubbled up to the next dmap + * control level by a recursive call to this routine, specifying + * the new root value and the next dmap control page level to + * be adjusted. + * PARAMETERS: + * bmp - pointer to bmap descriptor + * blkno - the first block of a block range within a dmap. it is + * the allocation or deallocation of this block range that + * requires the dmap control page to be adjusted. + * newval - the new value of the lower level dmap or dmap control + * page root. + * alloc - 'true' if adjustment is due to an allocation. + * level - current level of dmap control page (i.e. L0, L1, L2) to + * be adjusted. + * + * RETURN VALUES: + * 0 - success + * -EIO - i/o error + * + * serialization: IREAD_LOCK(ipbmap) or IWRITE_LOCK(ipbmap) held on entry/exit; + */ +static int +dbAdjCtl(struct bmap * bmp, s64 blkno, int newval, int alloc, int level) +{ + struct metapage *mp; + s8 oldroot; + int oldval; + s64 lblkno; + struct dmapctl *dcp; + int rc, leafno, ti; + + /* get the buffer for the dmap control page for the specified + * block number and control page level. + */ + lblkno = BLKTOCTL(blkno, bmp->db_l2nbperpage, level); + mp = read_metapage(bmp->db_ipbmap, lblkno, PSIZE, 0); + if (mp == NULL) + return -EIO; + dcp = (struct dmapctl *) mp->data; + + if (unlikely(!check_dmapctl(dcp))) { + jfs_error(bmp->db_ipbmap->i_sb, "Corrupt dmapctl page\n"); + release_metapage(mp); + return -EIO; + } + + /* determine the leaf number corresponding to the block and + * the index within the dmap control tree. + */ + leafno = BLKTOCTLLEAF(blkno, dcp->budmin); + ti = leafno + le32_to_cpu(dcp->leafidx); + + /* save the current leaf value and the current root level (i.e. + * maximum l2 free string described by this dmapctl). + */ + oldval = dcp->stree[ti]; + oldroot = dcp->stree[ROOT]; + + /* check if this is a control page update for an allocation. + * if so, update the leaf to reflect the new leaf value using + * dbSplit(); otherwise (deallocation), use dbJoin() to update + * the leaf with the new value. in addition to updating the + * leaf, dbSplit() will also split the binary buddy system of + * the leaves, if required, and bubble new values within the + * dmapctl tree, if required. similarly, dbJoin() will join + * the binary buddy system of leaves and bubble new values up + * the dmapctl tree as required by the new leaf value. + */ + if (alloc) { + /* check if we are in the middle of a binary buddy + * system. this happens when we are performing the + * first allocation out of an allocation group that + * is part (not the first part) of a larger binary + * buddy system. if we are in the middle, back split + * the system prior to calling dbSplit() which assumes + * that it is at the front of a binary buddy system. + */ + if (oldval == NOFREE) { + rc = dbBackSplit((dmtree_t *)dcp, leafno, true); + if (rc) { + release_metapage(mp); + return rc; + } + oldval = dcp->stree[ti]; + } + dbSplit((dmtree_t *) dcp, leafno, dcp->budmin, newval, true); + } else { + rc = dbJoin((dmtree_t *) dcp, leafno, newval, true); + if (rc) { + release_metapage(mp); + return rc; + } + } + + /* check if the root of the current dmap control page changed due + * to the update and if the current dmap control page is not at + * the current top level (i.e. L0, L1, L2) of the map. if so (i.e. + * root changed and this is not the top level), call this routine + * again (recursion) for the next higher level of the mapping to + * reflect the change in root for the current dmap control page. + */ + if (dcp->stree[ROOT] != oldroot) { + /* are we below the top level of the map. if so, + * bubble the root up to the next higher level. + */ + if (level < bmp->db_maxlevel) { + /* bubble up the new root of this dmap control page to + * the next level. + */ + if ((rc = + dbAdjCtl(bmp, blkno, dcp->stree[ROOT], alloc, + level + 1))) { + /* something went wrong in bubbling up the new + * root value, so backout the changes to the + * current dmap control page. + */ + if (alloc) { + dbJoin((dmtree_t *) dcp, leafno, + oldval, true); + } else { + /* the dbJoin() above might have + * caused a larger binary buddy system + * to form and we may now be in the + * middle of it. if this is the case, + * back split the buddies. + */ + if (dcp->stree[ti] == NOFREE) + dbBackSplit((dmtree_t *) + dcp, leafno, true); + dbSplit((dmtree_t *) dcp, leafno, + dcp->budmin, oldval, true); + } + + /* release the buffer and return the error. + */ + release_metapage(mp); + return (rc); + } + } else { + /* we're at the top level of the map. update + * the bmap control page to reflect the size + * of the maximum free buddy system. + */ + assert(level == bmp->db_maxlevel); + if (bmp->db_maxfreebud != oldroot) { + jfs_error(bmp->db_ipbmap->i_sb, + "the maximum free buddy is not the old root\n"); + } + bmp->db_maxfreebud = dcp->stree[ROOT]; + } + } + + /* write the buffer. + */ + write_metapage(mp); + + return (0); +} + + +/* + * NAME: dbSplit() + * + * FUNCTION: update the leaf of a dmtree with a new value, splitting + * the leaf from the binary buddy system of the dmtree's + * leaves, as required. + * + * PARAMETERS: + * tp - pointer to the tree containing the leaf. + * leafno - the number of the leaf to be updated. + * splitsz - the size the binary buddy system starting at the leaf + * must be split to, specified as the log2 number of blocks. + * newval - the new value for the leaf. + * + * RETURN VALUES: none + * + * serialization: IREAD_LOCK(ipbmap) or IWRITE_LOCK(ipbmap) held on entry/exit; + */ +static void dbSplit(dmtree_t *tp, int leafno, int splitsz, int newval, bool is_ctl) +{ + int budsz; + int cursz; + s8 *leaf = tp->dmt_stree + le32_to_cpu(tp->dmt_leafidx); + + /* check if the leaf needs to be split. + */ + if (leaf[leafno] > tp->dmt_budmin) { + /* the split occurs by cutting the buddy system in half + * at the specified leaf until we reach the specified + * size. pick up the starting split size (current size + * - 1 in l2) and the corresponding buddy size. + */ + cursz = leaf[leafno] - 1; + budsz = BUDSIZE(cursz, tp->dmt_budmin); + + /* split until we reach the specified size. + */ + while (cursz >= splitsz) { + /* update the buddy's leaf with its new value. + */ + dbAdjTree(tp, leafno ^ budsz, cursz, is_ctl); + + /* on to the next size and buddy. + */ + cursz -= 1; + budsz >>= 1; + } + } + + /* adjust the dmap tree to reflect the specified leaf's new + * value. + */ + dbAdjTree(tp, leafno, newval, is_ctl); +} + + +/* + * NAME: dbBackSplit() + * + * FUNCTION: back split the binary buddy system of dmtree leaves + * that hold a specified leaf until the specified leaf + * starts its own binary buddy system. + * + * the allocators typically perform allocations at the start + * of binary buddy systems and dbSplit() is used to accomplish + * any required splits. in some cases, however, allocation + * may occur in the middle of a binary system and requires a + * back split, with the split proceeding out from the middle of + * the system (less efficient) rather than the start of the + * system (more efficient). the cases in which a back split + * is required are rare and are limited to the first allocation + * within an allocation group which is a part (not first part) + * of a larger binary buddy system and a few exception cases + * in which a previous join operation must be backed out. + * + * PARAMETERS: + * tp - pointer to the tree containing the leaf. + * leafno - the number of the leaf to be updated. + * + * RETURN VALUES: none + * + * serialization: IREAD_LOCK(ipbmap) or IWRITE_LOCK(ipbmap) held on entry/exit; + */ +static int dbBackSplit(dmtree_t *tp, int leafno, bool is_ctl) +{ + int budsz, bud, w, bsz, size; + int cursz; + s8 *leaf = tp->dmt_stree + le32_to_cpu(tp->dmt_leafidx); + + /* leaf should be part (not first part) of a binary + * buddy system. + */ + assert(leaf[leafno] == NOFREE); + + /* the back split is accomplished by iteratively finding the leaf + * that starts the buddy system that contains the specified leaf and + * splitting that system in two. this iteration continues until + * the specified leaf becomes the start of a buddy system. + * + * determine maximum possible l2 size for the specified leaf. + */ + size = + LITOL2BSZ(leafno, le32_to_cpu(tp->dmt_l2nleafs), + tp->dmt_budmin); + + /* determine the number of leaves covered by this size. this + * is the buddy size that we will start with as we search for + * the buddy system that contains the specified leaf. + */ + budsz = BUDSIZE(size, tp->dmt_budmin); + + /* back split. + */ + while (leaf[leafno] == NOFREE) { + /* find the leftmost buddy leaf. + */ + for (w = leafno, bsz = budsz;; bsz <<= 1, + w = (w < bud) ? w : bud) { + if (bsz >= le32_to_cpu(tp->dmt_nleafs)) { + jfs_err("JFS: block map error in dbBackSplit"); + return -EIO; + } + + /* determine the buddy. + */ + bud = w ^ bsz; + + /* check if this buddy is the start of the system. + */ + if (leaf[bud] != NOFREE) { + /* split the leaf at the start of the + * system in two. + */ + cursz = leaf[bud] - 1; + dbSplit(tp, bud, cursz, cursz, is_ctl); + break; + } + } + } + + if (leaf[leafno] != size) { + jfs_err("JFS: wrong leaf value in dbBackSplit"); + return -EIO; + } + return 0; +} + + +/* + * NAME: dbJoin() + * + * FUNCTION: update the leaf of a dmtree with a new value, joining + * the leaf with other leaves of the dmtree into a multi-leaf + * binary buddy system, as required. + * + * PARAMETERS: + * tp - pointer to the tree containing the leaf. + * leafno - the number of the leaf to be updated. + * newval - the new value for the leaf. + * + * RETURN VALUES: none + */ +static int dbJoin(dmtree_t *tp, int leafno, int newval, bool is_ctl) +{ + int budsz, buddy; + s8 *leaf; + + /* can the new leaf value require a join with other leaves ? + */ + if (newval >= tp->dmt_budmin) { + /* pickup a pointer to the leaves of the tree. + */ + leaf = tp->dmt_stree + le32_to_cpu(tp->dmt_leafidx); + + /* try to join the specified leaf into a large binary + * buddy system. the join proceeds by attempting to join + * the specified leafno with its buddy (leaf) at new value. + * if the join occurs, we attempt to join the left leaf + * of the joined buddies with its buddy at new value + 1. + * we continue to join until we find a buddy that cannot be + * joined (does not have a value equal to the size of the + * last join) or until all leaves have been joined into a + * single system. + * + * get the buddy size (number of words covered) of + * the new value. + */ + budsz = BUDSIZE(newval, tp->dmt_budmin); + + /* try to join. + */ + while (budsz < le32_to_cpu(tp->dmt_nleafs)) { + /* get the buddy leaf. + */ + buddy = leafno ^ budsz; + + /* if the leaf's new value is greater than its + * buddy's value, we join no more. + */ + if (newval > leaf[buddy]) + break; + + /* It shouldn't be less */ + if (newval < leaf[buddy]) + return -EIO; + + /* check which (leafno or buddy) is the left buddy. + * the left buddy gets to claim the blocks resulting + * from the join while the right gets to claim none. + * the left buddy is also eligible to participate in + * a join at the next higher level while the right + * is not. + * + */ + if (leafno < buddy) { + /* leafno is the left buddy. + */ + dbAdjTree(tp, buddy, NOFREE, is_ctl); + } else { + /* buddy is the left buddy and becomes + * leafno. + */ + dbAdjTree(tp, leafno, NOFREE, is_ctl); + leafno = buddy; + } + + /* on to try the next join. + */ + newval += 1; + budsz <<= 1; + } + } + + /* update the leaf value. + */ + dbAdjTree(tp, leafno, newval, is_ctl); + + return 0; +} + + +/* + * NAME: dbAdjTree() + * + * FUNCTION: update a leaf of a dmtree with a new value, adjusting + * the dmtree, as required, to reflect the new leaf value. + * the combination of any buddies must already be done before + * this is called. + * + * PARAMETERS: + * tp - pointer to the tree to be adjusted. + * leafno - the number of the leaf to be updated. + * newval - the new value for the leaf. + * + * RETURN VALUES: none + */ +static void dbAdjTree(dmtree_t *tp, int leafno, int newval, bool is_ctl) +{ + int lp, pp, k; + int max, size; + + size = is_ctl ? CTLTREESIZE : TREESIZE; + + /* pick up the index of the leaf for this leafno. + */ + lp = leafno + le32_to_cpu(tp->dmt_leafidx); + + if (WARN_ON_ONCE(lp >= size || lp < 0)) + return; + + /* is the current value the same as the old value ? if so, + * there is nothing to do. + */ + if (tp->dmt_stree[lp] == newval) + return; + + /* set the new value. + */ + tp->dmt_stree[lp] = newval; + + /* bubble the new value up the tree as required. + */ + for (k = 0; k < le32_to_cpu(tp->dmt_height); k++) { + if (lp == 0) + break; + + /* get the index of the first leaf of the 4 leaf + * group containing the specified leaf (leafno). + */ + lp = ((lp - 1) & ~0x03) + 1; + + /* get the index of the parent of this 4 leaf group. + */ + pp = (lp - 1) >> 2; + + /* determine the maximum of the 4 leaves. + */ + max = TREEMAX(&tp->dmt_stree[lp]); + + /* if the maximum of the 4 is the same as the + * parent's value, we're done. + */ + if (tp->dmt_stree[pp] == max) + break; + + /* parent gets new value. + */ + tp->dmt_stree[pp] = max; + + /* parent becomes leaf for next go-round. + */ + lp = pp; + } +} + + +/* + * NAME: dbFindLeaf() + * + * FUNCTION: search a dmtree_t for sufficient free blocks, returning + * the index of a leaf describing the free blocks if + * sufficient free blocks are found. + * + * the search starts at the top of the dmtree_t tree and + * proceeds down the tree to the leftmost leaf with sufficient + * free space. + * + * PARAMETERS: + * tp - pointer to the tree to be searched. + * l2nb - log2 number of free blocks to search for. + * leafidx - return pointer to be set to the index of the leaf + * describing at least l2nb free blocks if sufficient + * free blocks are found. + * is_ctl - determines if the tree is of type ctl + * + * RETURN VALUES: + * 0 - success + * -ENOSPC - insufficient free blocks. + */ +static int dbFindLeaf(dmtree_t *tp, int l2nb, int *leafidx, bool is_ctl) +{ + int ti, n = 0, k, x = 0; + int max_size, max_idx; + + max_size = is_ctl ? CTLTREESIZE : TREESIZE; + max_idx = is_ctl ? LPERCTL : LPERDMAP; + + /* first check the root of the tree to see if there is + * sufficient free space. + */ + if (l2nb > tp->dmt_stree[ROOT]) + return -ENOSPC; + + /* sufficient free space available. now search down the tree + * starting at the next level for the leftmost leaf that + * describes sufficient free space. + */ + for (k = le32_to_cpu(tp->dmt_height), ti = 1; + k > 0; k--, ti = ((ti + n) << 2) + 1) { + /* search the four nodes at this level, starting from + * the left. + */ + for (x = ti, n = 0; n < 4; n++) { + /* sufficient free space found. move to the next + * level (or quit if this is the last level). + */ + if (x + n > max_size) + return -ENOSPC; + if (l2nb <= tp->dmt_stree[x + n]) + break; + } + + /* better have found something since the higher + * levels of the tree said it was here. + */ + assert(n < 4); + } + if (le32_to_cpu(tp->dmt_leafidx) >= max_idx) + return -ENOSPC; + + /* set the return to the leftmost leaf describing sufficient + * free space. + */ + *leafidx = x + n - le32_to_cpu(tp->dmt_leafidx); + + return (0); +} + + +/* + * NAME: dbFindBits() + * + * FUNCTION: find a specified number of binary buddy free bits within a + * dmap bitmap word value. + * + * this routine searches the bitmap value for (1 << l2nb) free + * bits at (1 << l2nb) alignments within the value. + * + * PARAMETERS: + * word - dmap bitmap word value. + * l2nb - number of free bits specified as a log2 number. + * + * RETURN VALUES: + * starting bit number of free bits. + */ +static int dbFindBits(u32 word, int l2nb) +{ + int bitno, nb; + u32 mask; + + /* get the number of bits. + */ + nb = 1 << l2nb; + assert(nb <= DBWORD); + + /* complement the word so we can use a mask (i.e. 0s represent + * free bits) and compute the mask. + */ + word = ~word; + mask = ONES << (DBWORD - nb); + + /* scan the word for nb free bits at nb alignments. + */ + for (bitno = 0; mask != 0; bitno += nb, mask = (mask >> nb)) { + if ((mask & word) == mask) + break; + } + + ASSERT(bitno < 32); + + /* return the bit number. + */ + return (bitno); +} + + +/* + * NAME: dbMaxBud(u8 *cp) + * + * FUNCTION: determine the largest binary buddy string of free + * bits within 32-bits of the map. + * + * PARAMETERS: + * cp - pointer to the 32-bit value. + * + * RETURN VALUES: + * largest binary buddy of free bits within a dmap word. + */ +static int dbMaxBud(u8 * cp) +{ + signed char tmp1, tmp2; + + /* check if the wmap word is all free. if so, the + * free buddy size is BUDMIN. + */ + if (*((uint *) cp) == 0) + return (BUDMIN); + + /* check if the wmap word is half free. if so, the + * free buddy size is BUDMIN-1. + */ + if (*((u16 *) cp) == 0 || *((u16 *) cp + 1) == 0) + return (BUDMIN - 1); + + /* not all free or half free. determine the free buddy + * size thru table lookup using quarters of the wmap word. + */ + tmp1 = max(budtab[cp[2]], budtab[cp[3]]); + tmp2 = max(budtab[cp[0]], budtab[cp[1]]); + return (max(tmp1, tmp2)); +} + + +/* + * NAME: cnttz(uint word) + * + * FUNCTION: determine the number of trailing zeros within a 32-bit + * value. + * + * PARAMETERS: + * value - 32-bit value to be examined. + * + * RETURN VALUES: + * count of trailing zeros + */ +static int cnttz(u32 word) +{ + int n; + + for (n = 0; n < 32; n++, word >>= 1) { + if (word & 0x01) + break; + } + + return (n); +} + + +/* + * NAME: cntlz(u32 value) + * + * FUNCTION: determine the number of leading zeros within a 32-bit + * value. + * + * PARAMETERS: + * value - 32-bit value to be examined. + * + * RETURN VALUES: + * count of leading zeros + */ +static int cntlz(u32 value) +{ + int n; + + for (n = 0; n < 32; n++, value <<= 1) { + if (value & HIGHORDER) + break; + } + return (n); +} + + +/* + * NAME: blkstol2(s64 nb) + * + * FUNCTION: convert a block count to its log2 value. if the block + * count is not a l2 multiple, it is rounded up to the next + * larger l2 multiple. + * + * PARAMETERS: + * nb - number of blocks + * + * RETURN VALUES: + * log2 number of blocks + */ +static int blkstol2(s64 nb) +{ + int l2nb; + s64 mask; /* meant to be signed */ + + mask = (s64) 1 << (64 - 1); + + /* count the leading bits. + */ + for (l2nb = 0; l2nb < 64; l2nb++, mask >>= 1) { + /* leading bit found. + */ + if (nb & mask) { + /* determine the l2 value. + */ + l2nb = (64 - 1) - l2nb; + + /* check if we need to round up. + */ + if (~mask & nb) + l2nb++; + + return (l2nb); + } + } + assert(0); + return 0; /* fix compiler warning */ +} + + +/* + * NAME: dbAllocBottomUp() + * + * FUNCTION: alloc the specified block range from the working block + * allocation map. + * + * the blocks will be alloc from the working map one dmap + * at a time. + * + * PARAMETERS: + * ip - pointer to in-core inode; + * blkno - starting block number to be freed. + * nblocks - number of blocks to be freed. + * + * RETURN VALUES: + * 0 - success + * -EIO - i/o error + */ +int dbAllocBottomUp(struct inode *ip, s64 blkno, s64 nblocks) +{ + struct metapage *mp; + struct dmap *dp; + int nb, rc; + s64 lblkno, rem; + struct inode *ipbmap = JFS_SBI(ip->i_sb)->ipbmap; + struct bmap *bmp = JFS_SBI(ip->i_sb)->bmap; + + IREAD_LOCK(ipbmap, RDWRLOCK_DMAP); + + /* block to be allocated better be within the mapsize. */ + ASSERT(nblocks <= bmp->db_mapsize - blkno); + + /* + * allocate the blocks a dmap at a time. + */ + mp = NULL; + for (rem = nblocks; rem > 0; rem -= nb, blkno += nb) { + /* release previous dmap if any */ + if (mp) { + write_metapage(mp); + } + + /* get the buffer for the current dmap. */ + lblkno = BLKTODMAP(blkno, bmp->db_l2nbperpage); + mp = read_metapage(ipbmap, lblkno, PSIZE, 0); + if (mp == NULL) { + IREAD_UNLOCK(ipbmap); + return -EIO; + } + dp = (struct dmap *) mp->data; + + /* determine the number of blocks to be allocated from + * this dmap. + */ + nb = min(rem, BPERDMAP - (blkno & (BPERDMAP - 1))); + + /* allocate the blocks. */ + if ((rc = dbAllocDmapBU(bmp, dp, blkno, nb))) { + release_metapage(mp); + IREAD_UNLOCK(ipbmap); + return (rc); + } + } + + /* write the last buffer. */ + write_metapage(mp); + + IREAD_UNLOCK(ipbmap); + + return (0); +} + + +static int dbAllocDmapBU(struct bmap * bmp, struct dmap * dp, s64 blkno, + int nblocks) +{ + int rc; + int dbitno, word, rembits, nb, nwords, wbitno, agno; + s8 oldroot; + struct dmaptree *tp = (struct dmaptree *) & dp->tree; + + /* save the current value of the root (i.e. maximum free string) + * of the dmap tree. + */ + oldroot = tp->stree[ROOT]; + + /* determine the bit number and word within the dmap of the + * starting block. + */ + dbitno = blkno & (BPERDMAP - 1); + word = dbitno >> L2DBWORD; + + /* block range better be within the dmap */ + assert(dbitno + nblocks <= BPERDMAP); + + /* allocate the bits of the dmap's words corresponding to the block + * range. not all bits of the first and last words may be contained + * within the block range. if this is the case, we'll work against + * those words (i.e. partial first and/or last) on an individual basis + * (a single pass), allocating the bits of interest by hand and + * updating the leaf corresponding to the dmap word. a single pass + * will be used for all dmap words fully contained within the + * specified range. within this pass, the bits of all fully contained + * dmap words will be marked as free in a single shot and the leaves + * will be updated. a single leaf may describe the free space of + * multiple dmap words, so we may update only a subset of the actual + * leaves corresponding to the dmap words of the block range. + */ + for (rembits = nblocks; rembits > 0; rembits -= nb, dbitno += nb) { + /* determine the bit number within the word and + * the number of bits within the word. + */ + wbitno = dbitno & (DBWORD - 1); + nb = min(rembits, DBWORD - wbitno); + + /* check if only part of a word is to be allocated. + */ + if (nb < DBWORD) { + /* allocate (set to 1) the appropriate bits within + * this dmap word. + */ + dp->wmap[word] |= cpu_to_le32(ONES << (DBWORD - nb) + >> wbitno); + + word++; + } else { + /* one or more dmap words are fully contained + * within the block range. determine how many + * words and allocate (set to 1) the bits of these + * words. + */ + nwords = rembits >> L2DBWORD; + memset(&dp->wmap[word], (int) ONES, nwords * 4); + + /* determine how many bits */ + nb = nwords << L2DBWORD; + word += nwords; + } + } + + /* update the free count for this dmap */ + le32_add_cpu(&dp->nfree, -nblocks); + + /* reconstruct summary tree */ + dbInitDmapTree(dp); + + BMAP_LOCK(bmp); + + /* if this allocation group is completely free, + * update the highest active allocation group number + * if this allocation group is the new max. + */ + agno = blkno >> bmp->db_agl2size; + if (agno > bmp->db_maxag) + bmp->db_maxag = agno; + + /* update the free count for the allocation group and map */ + bmp->db_agfree[agno] -= nblocks; + bmp->db_nfree -= nblocks; + + BMAP_UNLOCK(bmp); + + /* if the root has not changed, done. */ + if (tp->stree[ROOT] == oldroot) + return (0); + + /* root changed. bubble the change up to the dmap control pages. + * if the adjustment of the upper level control pages fails, + * backout the bit allocation (thus making everything consistent). + */ + if ((rc = dbAdjCtl(bmp, blkno, tp->stree[ROOT], 1, 0))) + dbFreeBits(bmp, dp, blkno, nblocks); + + return (rc); +} + + +/* + * NAME: dbExtendFS() + * + * FUNCTION: extend bmap from blkno for nblocks; + * dbExtendFS() updates bmap ready for dbAllocBottomUp(); + * + * L2 + * | + * L1---------------------------------L1 + * | | + * L0---------L0---------L0 L0---------L0---------L0 + * | | | | | | + * d0,...,dn d0,...,dn d0,...,dn d0,...,dn d0,...,dn d0,.,dm; + * L2L1L0d0,...,dnL0d0,...,dnL0d0,...,dnL1L0d0,...,dnL0d0,...,dnL0d0,..dm + * + * <---old---><----------------------------extend-----------------------> + */ +int dbExtendFS(struct inode *ipbmap, s64 blkno, s64 nblocks) +{ + struct jfs_sb_info *sbi = JFS_SBI(ipbmap->i_sb); + int nbperpage = sbi->nbperpage; + int i, i0 = true, j, j0 = true, k, n; + s64 newsize; + s64 p; + struct metapage *mp, *l2mp, *l1mp = NULL, *l0mp = NULL; + struct dmapctl *l2dcp, *l1dcp, *l0dcp; + struct dmap *dp; + s8 *l0leaf, *l1leaf, *l2leaf; + struct bmap *bmp = sbi->bmap; + int agno, l2agsize, oldl2agsize; + s64 ag_rem; + + newsize = blkno + nblocks; + + jfs_info("dbExtendFS: blkno:%Ld nblocks:%Ld newsize:%Ld", + (long long) blkno, (long long) nblocks, (long long) newsize); + + /* + * initialize bmap control page. + * + * all the data in bmap control page should exclude + * the mkfs hidden dmap page. + */ + + /* update mapsize */ + bmp->db_mapsize = newsize; + bmp->db_maxlevel = BMAPSZTOLEV(bmp->db_mapsize); + + /* compute new AG size */ + l2agsize = dbGetL2AGSize(newsize); + oldl2agsize = bmp->db_agl2size; + + bmp->db_agl2size = l2agsize; + bmp->db_agsize = (s64)1 << l2agsize; + + /* compute new number of AG */ + agno = bmp->db_numag; + bmp->db_numag = newsize >> l2agsize; + bmp->db_numag += ((u32) newsize % (u32) bmp->db_agsize) ? 1 : 0; + + /* + * reconfigure db_agfree[] + * from old AG configuration to new AG configuration; + * + * coalesce contiguous k (newAGSize/oldAGSize) AGs; + * i.e., (AGi, ..., AGj) where i = k*n and j = k*(n+1) - 1 to AGn; + * note: new AG size = old AG size * (2**x). + */ + if (l2agsize == oldl2agsize) + goto extend; + k = 1 << (l2agsize - oldl2agsize); + ag_rem = bmp->db_agfree[0]; /* save agfree[0] */ + for (i = 0, n = 0; i < agno; n++) { + bmp->db_agfree[n] = 0; /* init collection point */ + + /* coalesce contiguous k AGs; */ + for (j = 0; j < k && i < agno; j++, i++) { + /* merge AGi to AGn */ + bmp->db_agfree[n] += bmp->db_agfree[i]; + } + } + bmp->db_agfree[0] += ag_rem; /* restore agfree[0] */ + + for (; n < MAXAG; n++) + bmp->db_agfree[n] = 0; + + /* + * update highest active ag number + */ + + bmp->db_maxag = bmp->db_maxag / k; + + /* + * extend bmap + * + * update bit maps and corresponding level control pages; + * global control page db_nfree, db_agfree[agno], db_maxfreebud; + */ + extend: + /* get L2 page */ + p = BMAPBLKNO + nbperpage; /* L2 page */ + l2mp = read_metapage(ipbmap, p, PSIZE, 0); + if (!l2mp) { + jfs_error(ipbmap->i_sb, "L2 page could not be read\n"); + return -EIO; + } + l2dcp = (struct dmapctl *) l2mp->data; + if (unlikely(!check_dmapctl(l2dcp))) { + jfs_error(ipbmap->i_sb, "Corrupt dmapctl page\n"); + release_metapage(l2mp); + return -EIO; + } + + /* compute start L1 */ + k = blkno >> L2MAXL1SIZE; + l2leaf = l2dcp->stree + CTLLEAFIND + k; + p = BLKTOL1(blkno, sbi->l2nbperpage); /* L1 page */ + + /* + * extend each L1 in L2 + */ + for (; k < LPERCTL; k++, p += nbperpage) { + /* get L1 page */ + if (j0) { + /* read in L1 page: (blkno & (MAXL1SIZE - 1)) */ + l1mp = read_metapage(ipbmap, p, PSIZE, 0); + if (l1mp == NULL) + goto errout; + l1dcp = (struct dmapctl *) l1mp->data; + if (unlikely(!check_dmapctl(l1dcp))) { + jfs_error(ipbmap->i_sb, "Corrupt dmapctl page\n"); + goto errout; + } + + /* compute start L0 */ + j = (blkno & (MAXL1SIZE - 1)) >> L2MAXL0SIZE; + l1leaf = l1dcp->stree + CTLLEAFIND + j; + p = BLKTOL0(blkno, sbi->l2nbperpage); + j0 = false; + } else { + /* assign/init L1 page */ + l1mp = get_metapage(ipbmap, p, PSIZE, 0); + if (l1mp == NULL) + goto errout; + + l1dcp = (struct dmapctl *) l1mp->data; + if (unlikely(!check_dmapctl(l1dcp))) { + jfs_error(ipbmap->i_sb, "Corrupt dmapctl page\n"); + goto errout; + } + + /* compute start L0 */ + j = 0; + l1leaf = l1dcp->stree + CTLLEAFIND; + p += nbperpage; /* 1st L0 of L1.k */ + } + + /* + * extend each L0 in L1 + */ + for (; j < LPERCTL; j++) { + /* get L0 page */ + if (i0) { + /* read in L0 page: (blkno & (MAXL0SIZE - 1)) */ + + l0mp = read_metapage(ipbmap, p, PSIZE, 0); + if (l0mp == NULL) + goto errout; + l0dcp = (struct dmapctl *) l0mp->data; + if (unlikely(!check_dmapctl(l0dcp))) { + jfs_error(ipbmap->i_sb, "Corrupt dmapctl page\n"); + goto errout; + } + + /* compute start dmap */ + i = (blkno & (MAXL0SIZE - 1)) >> + L2BPERDMAP; + l0leaf = l0dcp->stree + CTLLEAFIND + i; + p = BLKTODMAP(blkno, + sbi->l2nbperpage); + i0 = false; + } else { + /* assign/init L0 page */ + l0mp = get_metapage(ipbmap, p, PSIZE, 0); + if (l0mp == NULL) + goto errout; + + l0dcp = (struct dmapctl *) l0mp->data; + if (unlikely(!check_dmapctl(l0dcp))) { + jfs_error(ipbmap->i_sb, "Corrupt dmapctl page\n"); + goto errout; + } + + /* compute start dmap */ + i = 0; + l0leaf = l0dcp->stree + CTLLEAFIND; + p += nbperpage; /* 1st dmap of L0.j */ + } + + /* + * extend each dmap in L0 + */ + for (; i < LPERCTL; i++) { + /* + * reconstruct the dmap page, and + * initialize corresponding parent L0 leaf + */ + if ((n = blkno & (BPERDMAP - 1))) { + /* read in dmap page: */ + mp = read_metapage(ipbmap, p, + PSIZE, 0); + if (mp == NULL) + goto errout; + n = min(nblocks, (s64)BPERDMAP - n); + } else { + /* assign/init dmap page */ + mp = read_metapage(ipbmap, p, + PSIZE, 0); + if (mp == NULL) + goto errout; + + n = min_t(s64, nblocks, BPERDMAP); + } + + dp = (struct dmap *) mp->data; + *l0leaf = dbInitDmap(dp, blkno, n); + + bmp->db_nfree += n; + agno = le64_to_cpu(dp->start) >> l2agsize; + bmp->db_agfree[agno] += n; + + write_metapage(mp); + + l0leaf++; + p += nbperpage; + + blkno += n; + nblocks -= n; + if (nblocks == 0) + break; + } /* for each dmap in a L0 */ + + /* + * build current L0 page from its leaves, and + * initialize corresponding parent L1 leaf + */ + *l1leaf = dbInitDmapCtl(l0dcp, 0, ++i); + write_metapage(l0mp); + l0mp = NULL; + + if (nblocks) + l1leaf++; /* continue for next L0 */ + else { + /* more than 1 L0 ? */ + if (j > 0) + break; /* build L1 page */ + else { + /* summarize in global bmap page */ + bmp->db_maxfreebud = *l1leaf; + release_metapage(l1mp); + release_metapage(l2mp); + goto finalize; + } + } + } /* for each L0 in a L1 */ + + /* + * build current L1 page from its leaves, and + * initialize corresponding parent L2 leaf + */ + *l2leaf = dbInitDmapCtl(l1dcp, 1, ++j); + write_metapage(l1mp); + l1mp = NULL; + + if (nblocks) + l2leaf++; /* continue for next L1 */ + else { + /* more than 1 L1 ? */ + if (k > 0) + break; /* build L2 page */ + else { + /* summarize in global bmap page */ + bmp->db_maxfreebud = *l2leaf; + release_metapage(l2mp); + goto finalize; + } + } + } /* for each L1 in a L2 */ + + jfs_error(ipbmap->i_sb, "function has not returned as expected\n"); +errout: + if (l0mp) + release_metapage(l0mp); + if (l1mp) + release_metapage(l1mp); + release_metapage(l2mp); + return -EIO; + + /* + * finalize bmap control page + */ +finalize: + + return 0; +} + + +/* + * dbFinalizeBmap() + */ +void dbFinalizeBmap(struct inode *ipbmap) +{ + struct bmap *bmp = JFS_SBI(ipbmap->i_sb)->bmap; + int actags, inactags, l2nl; + s64 ag_rem, actfree, inactfree, avgfree; + int i, n; + + /* + * finalize bmap control page + */ +//finalize: + /* + * compute db_agpref: preferred ag to allocate from + * (the leftmost ag with average free space in it); + */ +//agpref: + /* get the number of active ags and inactive ags */ + actags = bmp->db_maxag + 1; + inactags = bmp->db_numag - actags; + ag_rem = bmp->db_mapsize & (bmp->db_agsize - 1); /* ??? */ + + /* determine how many blocks are in the inactive allocation + * groups. in doing this, we must account for the fact that + * the rightmost group might be a partial group (i.e. file + * system size is not a multiple of the group size). + */ + inactfree = (inactags && ag_rem) ? + (((s64)inactags - 1) << bmp->db_agl2size) + ag_rem + : ((s64)inactags << bmp->db_agl2size); + + /* determine how many free blocks are in the active + * allocation groups plus the average number of free blocks + * within the active ags. + */ + actfree = bmp->db_nfree - inactfree; + avgfree = (u32) actfree / (u32) actags; + + /* if the preferred allocation group has not average free space. + * re-establish the preferred group as the leftmost + * group with average free space. + */ + if (bmp->db_agfree[bmp->db_agpref] < avgfree) { + for (bmp->db_agpref = 0; bmp->db_agpref < actags; + bmp->db_agpref++) { + if (bmp->db_agfree[bmp->db_agpref] >= avgfree) + break; + } + if (bmp->db_agpref >= bmp->db_numag) { + jfs_error(ipbmap->i_sb, + "cannot find ag with average freespace\n"); + } + } + + /* + * compute db_aglevel, db_agheight, db_width, db_agstart: + * an ag is covered in aglevel dmapctl summary tree, + * at agheight level height (from leaf) with agwidth number of nodes + * each, which starts at agstart index node of the smmary tree node + * array; + */ + bmp->db_aglevel = BMAPSZTOLEV(bmp->db_agsize); + l2nl = + bmp->db_agl2size - (L2BPERDMAP + bmp->db_aglevel * L2LPERCTL); + bmp->db_agheight = l2nl >> 1; + bmp->db_agwidth = 1 << (l2nl - (bmp->db_agheight << 1)); + for (i = 5 - bmp->db_agheight, bmp->db_agstart = 0, n = 1; i > 0; + i--) { + bmp->db_agstart += n; + n <<= 2; + } + +} + + +/* + * NAME: dbInitDmap()/ujfs_idmap_page() + * + * FUNCTION: initialize working/persistent bitmap of the dmap page + * for the specified number of blocks: + * + * at entry, the bitmaps had been initialized as free (ZEROS); + * The number of blocks will only account for the actually + * existing blocks. Blocks which don't actually exist in + * the aggregate will be marked as allocated (ONES); + * + * PARAMETERS: + * dp - pointer to page of map + * nblocks - number of blocks this page + * + * RETURNS: NONE + */ +static int dbInitDmap(struct dmap * dp, s64 Blkno, int nblocks) +{ + int blkno, w, b, r, nw, nb, i; + + /* starting block number within the dmap */ + blkno = Blkno & (BPERDMAP - 1); + + if (blkno == 0) { + dp->nblocks = dp->nfree = cpu_to_le32(nblocks); + dp->start = cpu_to_le64(Blkno); + + if (nblocks == BPERDMAP) { + memset(&dp->wmap[0], 0, LPERDMAP * 4); + memset(&dp->pmap[0], 0, LPERDMAP * 4); + goto initTree; + } + } else { + le32_add_cpu(&dp->nblocks, nblocks); + le32_add_cpu(&dp->nfree, nblocks); + } + + /* word number containing start block number */ + w = blkno >> L2DBWORD; + + /* + * free the bits corresponding to the block range (ZEROS): + * note: not all bits of the first and last words may be contained + * within the block range. + */ + for (r = nblocks; r > 0; r -= nb, blkno += nb) { + /* number of bits preceding range to be freed in the word */ + b = blkno & (DBWORD - 1); + /* number of bits to free in the word */ + nb = min(r, DBWORD - b); + + /* is partial word to be freed ? */ + if (nb < DBWORD) { + /* free (set to 0) from the bitmap word */ + dp->wmap[w] &= cpu_to_le32(~(ONES << (DBWORD - nb) + >> b)); + dp->pmap[w] &= cpu_to_le32(~(ONES << (DBWORD - nb) + >> b)); + + /* skip the word freed */ + w++; + } else { + /* free (set to 0) contiguous bitmap words */ + nw = r >> L2DBWORD; + memset(&dp->wmap[w], 0, nw * 4); + memset(&dp->pmap[w], 0, nw * 4); + + /* skip the words freed */ + nb = nw << L2DBWORD; + w += nw; + } + } + + /* + * mark bits following the range to be freed (non-existing + * blocks) as allocated (ONES) + */ + + if (blkno == BPERDMAP) + goto initTree; + + /* the first word beyond the end of existing blocks */ + w = blkno >> L2DBWORD; + + /* does nblocks fall on a 32-bit boundary ? */ + b = blkno & (DBWORD - 1); + if (b) { + /* mark a partial word allocated */ + dp->wmap[w] = dp->pmap[w] = cpu_to_le32(ONES >> b); + w++; + } + + /* set the rest of the words in the page to allocated (ONES) */ + for (i = w; i < LPERDMAP; i++) + dp->pmap[i] = dp->wmap[i] = cpu_to_le32(ONES); + + /* + * init tree + */ + initTree: + return (dbInitDmapTree(dp)); +} + + +/* + * NAME: dbInitDmapTree()/ujfs_complete_dmap() + * + * FUNCTION: initialize summary tree of the specified dmap: + * + * at entry, bitmap of the dmap has been initialized; + * + * PARAMETERS: + * dp - dmap to complete + * blkno - starting block number for this dmap + * treemax - will be filled in with max free for this dmap + * + * RETURNS: max free string at the root of the tree + */ +static int dbInitDmapTree(struct dmap * dp) +{ + struct dmaptree *tp; + s8 *cp; + int i; + + /* init fixed info of tree */ + tp = &dp->tree; + tp->nleafs = cpu_to_le32(LPERDMAP); + tp->l2nleafs = cpu_to_le32(L2LPERDMAP); + tp->leafidx = cpu_to_le32(LEAFIND); + tp->height = cpu_to_le32(4); + tp->budmin = BUDMIN; + + /* init each leaf from corresponding wmap word: + * note: leaf is set to NOFREE(-1) if all blocks of corresponding + * bitmap word are allocated. + */ + cp = tp->stree + le32_to_cpu(tp->leafidx); + for (i = 0; i < LPERDMAP; i++) + *cp++ = dbMaxBud((u8 *) & dp->wmap[i]); + + /* build the dmap's binary buddy summary tree */ + return (dbInitTree(tp)); +} + + +/* + * NAME: dbInitTree()/ujfs_adjtree() + * + * FUNCTION: initialize binary buddy summary tree of a dmap or dmapctl. + * + * at entry, the leaves of the tree has been initialized + * from corresponding bitmap word or root of summary tree + * of the child control page; + * configure binary buddy system at the leaf level, then + * bubble up the values of the leaf nodes up the tree. + * + * PARAMETERS: + * cp - Pointer to the root of the tree + * l2leaves- Number of leaf nodes as a power of 2 + * l2min - Number of blocks that can be covered by a leaf + * as a power of 2 + * + * RETURNS: max free string at the root of the tree + */ +static int dbInitTree(struct dmaptree * dtp) +{ + int l2max, l2free, bsize, nextb, i; + int child, parent, nparent; + s8 *tp, *cp, *cp1; + + tp = dtp->stree; + + /* Determine the maximum free string possible for the leaves */ + l2max = le32_to_cpu(dtp->l2nleafs) + dtp->budmin; + + /* + * configure the leaf level into binary buddy system + * + * Try to combine buddies starting with a buddy size of 1 + * (i.e. two leaves). At a buddy size of 1 two buddy leaves + * can be combined if both buddies have a maximum free of l2min; + * the combination will result in the left-most buddy leaf having + * a maximum free of l2min+1. + * After processing all buddies for a given size, process buddies + * at the next higher buddy size (i.e. current size * 2) and + * the next maximum free (current free + 1). + * This continues until the maximum possible buddy combination + * yields maximum free. + */ + for (l2free = dtp->budmin, bsize = 1; l2free < l2max; + l2free++, bsize = nextb) { + /* get next buddy size == current buddy pair size */ + nextb = bsize << 1; + + /* scan each adjacent buddy pair at current buddy size */ + for (i = 0, cp = tp + le32_to_cpu(dtp->leafidx); + i < le32_to_cpu(dtp->nleafs); + i += nextb, cp += nextb) { + /* coalesce if both adjacent buddies are max free */ + if (*cp == l2free && *(cp + bsize) == l2free) { + *cp = l2free + 1; /* left take right */ + *(cp + bsize) = -1; /* right give left */ + } + } + } + + /* + * bubble summary information of leaves up the tree. + * + * Starting at the leaf node level, the four nodes described by + * the higher level parent node are compared for a maximum free and + * this maximum becomes the value of the parent node. + * when all lower level nodes are processed in this fashion then + * move up to the next level (parent becomes a lower level node) and + * continue the process for that level. + */ + for (child = le32_to_cpu(dtp->leafidx), + nparent = le32_to_cpu(dtp->nleafs) >> 2; + nparent > 0; nparent >>= 2, child = parent) { + /* get index of 1st node of parent level */ + parent = (child - 1) >> 2; + + /* set the value of the parent node as the maximum + * of the four nodes of the current level. + */ + for (i = 0, cp = tp + child, cp1 = tp + parent; + i < nparent; i++, cp += 4, cp1++) + *cp1 = TREEMAX(cp); + } + + return (*tp); +} + + +/* + * dbInitDmapCtl() + * + * function: initialize dmapctl page + */ +static int dbInitDmapCtl(struct dmapctl * dcp, int level, int i) +{ /* start leaf index not covered by range */ + s8 *cp; + + dcp->nleafs = cpu_to_le32(LPERCTL); + dcp->l2nleafs = cpu_to_le32(L2LPERCTL); + dcp->leafidx = cpu_to_le32(CTLLEAFIND); + dcp->height = cpu_to_le32(5); + dcp->budmin = L2BPERDMAP + L2LPERCTL * level; + + /* + * initialize the leaves of current level that were not covered + * by the specified input block range (i.e. the leaves have no + * low level dmapctl or dmap). + */ + cp = &dcp->stree[CTLLEAFIND + i]; + for (; i < LPERCTL; i++) + *cp++ = NOFREE; + + /* build the dmap's binary buddy summary tree */ + return (dbInitTree((struct dmaptree *) dcp)); +} + + +/* + * NAME: dbGetL2AGSize()/ujfs_getagl2size() + * + * FUNCTION: Determine log2(allocation group size) from aggregate size + * + * PARAMETERS: + * nblocks - Number of blocks in aggregate + * + * RETURNS: log2(allocation group size) in aggregate blocks + */ +static int dbGetL2AGSize(s64 nblocks) +{ + s64 sz; + s64 m; + int l2sz; + + if (nblocks < BPERDMAP * MAXAG) + return (L2BPERDMAP); + + /* round up aggregate size to power of 2 */ + m = ((u64) 1 << (64 - 1)); + for (l2sz = 64; l2sz >= 0; l2sz--, m >>= 1) { + if (m & nblocks) + break; + } + + sz = (s64) 1 << l2sz; + if (sz < nblocks) + l2sz += 1; + + /* agsize = roundupSize/max_number_of_ag */ + return (l2sz - L2MAXAG); +} + + +/* + * NAME: dbMapFileSizeToMapSize() + * + * FUNCTION: compute number of blocks the block allocation map file + * can cover from the map file size; + * + * RETURNS: Number of blocks which can be covered by this block map file; + */ + +/* + * maximum number of map pages at each level including control pages + */ +#define MAXL0PAGES (1 + LPERCTL) +#define MAXL1PAGES (1 + LPERCTL * MAXL0PAGES) + +/* + * convert number of map pages to the zero origin top dmapctl level + */ +#define BMAPPGTOLEV(npages) \ + (((npages) <= 3 + MAXL0PAGES) ? 0 : \ + ((npages) <= 2 + MAXL1PAGES) ? 1 : 2) + +s64 dbMapFileSizeToMapSize(struct inode * ipbmap) +{ + struct super_block *sb = ipbmap->i_sb; + s64 nblocks; + s64 npages, ndmaps; + int level, i; + int complete, factor; + + nblocks = ipbmap->i_size >> JFS_SBI(sb)->l2bsize; + npages = nblocks >> JFS_SBI(sb)->l2nbperpage; + level = BMAPPGTOLEV(npages); + + /* At each level, accumulate the number of dmap pages covered by + * the number of full child levels below it; + * repeat for the last incomplete child level. + */ + ndmaps = 0; + npages--; /* skip the first global control page */ + /* skip higher level control pages above top level covered by map */ + npages -= (2 - level); + npages--; /* skip top level's control page */ + for (i = level; i >= 0; i--) { + factor = + (i == 2) ? MAXL1PAGES : ((i == 1) ? MAXL0PAGES : 1); + complete = (u32) npages / factor; + ndmaps += complete * ((i == 2) ? LPERCTL * LPERCTL : + ((i == 1) ? LPERCTL : 1)); + + /* pages in last/incomplete child */ + npages = (u32) npages % factor; + /* skip incomplete child's level control page */ + npages--; + } + + /* convert the number of dmaps into the number of blocks + * which can be covered by the dmaps; + */ + nblocks = ndmaps << L2BPERDMAP; + + return (nblocks); +} diff --git a/fs/jfs/jfs_dmap.h b/fs/jfs/jfs_dmap.h new file mode 100644 index 000000000..aa03a904d --- /dev/null +++ b/fs/jfs/jfs_dmap.h @@ -0,0 +1,303 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) International Business Machines Corp., 2000-2002 + */ +#ifndef _H_JFS_DMAP +#define _H_JFS_DMAP + +#include "jfs_txnmgr.h" + +#define BMAPVERSION 1 /* version number */ +#define TREESIZE (256+64+16+4+1) /* size of a dmap tree */ +#define LEAFIND (64+16+4+1) /* index of 1st leaf of a dmap tree */ +#define LPERDMAP 256 /* num leaves per dmap tree */ +#define L2LPERDMAP 8 /* l2 number of leaves per dmap tree */ +#define DBWORD 32 /* # of blks covered by a map word */ +#define L2DBWORD 5 /* l2 # of blks covered by a mword */ +#define BUDMIN L2DBWORD /* max free string in a map word */ +#define BPERDMAP (LPERDMAP * DBWORD) /* num of blks per dmap */ +#define L2BPERDMAP 13 /* l2 num of blks per dmap */ +#define CTLTREESIZE (1024+256+64+16+4+1) /* size of a dmapctl tree */ +#define CTLLEAFIND (256+64+16+4+1) /* idx of 1st leaf of a dmapctl tree */ +#define LPERCTL 1024 /* num of leaves per dmapctl tree */ +#define L2LPERCTL 10 /* l2 num of leaves per dmapctl tree */ +#define ROOT 0 /* index of the root of a tree */ +#define NOFREE ((s8) -1) /* no blocks free */ +#define MAXAG 128 /* max number of allocation groups */ +#define L2MAXAG 7 /* l2 max num of AG */ +#define L2MINAGSZ 25 /* l2 of minimum AG size in bytes */ +#define BMAPBLKNO 0 /* lblkno of bmap within the map */ + +/* + * maximum l2 number of disk blocks at the various dmapctl levels. + */ +#define L2MAXL0SIZE (L2BPERDMAP + 1 * L2LPERCTL) +#define L2MAXL1SIZE (L2BPERDMAP + 2 * L2LPERCTL) +#define L2MAXL2SIZE (L2BPERDMAP + 3 * L2LPERCTL) + +/* + * maximum number of disk blocks at the various dmapctl levels. + */ +#define MAXL0SIZE ((s64)1 << L2MAXL0SIZE) +#define MAXL1SIZE ((s64)1 << L2MAXL1SIZE) +#define MAXL2SIZE ((s64)1 << L2MAXL2SIZE) + +#define MAXMAPSIZE MAXL2SIZE /* maximum aggregate map size */ + +/* + * determine the maximum free string for four (lower level) nodes + * of the tree. + */ +static inline signed char TREEMAX(signed char *cp) +{ + signed char tmp1, tmp2; + + tmp1 = max(*(cp+2), *(cp+3)); + tmp2 = max(*(cp), *(cp+1)); + + return max(tmp1, tmp2); +} + +/* + * convert disk block number to the logical block number of the dmap + * describing the disk block. s is the log2(number of logical blocks per page) + * + * The calculation figures out how many logical pages are in front of the dmap. + * - the number of dmaps preceding it + * - the number of L0 pages preceding its L0 page + * - the number of L1 pages preceding its L1 page + * - 3 is added to account for the L2, L1, and L0 page for this dmap + * - 1 is added to account for the control page of the map. + */ +#define BLKTODMAP(b,s) \ + ((((b) >> 13) + ((b) >> 23) + ((b) >> 33) + 3 + 1) << (s)) + +/* + * convert disk block number to the logical block number of the LEVEL 0 + * dmapctl describing the disk block. s is the log2(number of logical blocks + * per page) + * + * The calculation figures out how many logical pages are in front of the L0. + * - the number of dmap pages preceding it + * - the number of L0 pages preceding it + * - the number of L1 pages preceding its L1 page + * - 2 is added to account for the L2, and L1 page for this L0 + * - 1 is added to account for the control page of the map. + */ +#define BLKTOL0(b,s) \ + (((((b) >> 23) << 10) + ((b) >> 23) + ((b) >> 33) + 2 + 1) << (s)) + +/* + * convert disk block number to the logical block number of the LEVEL 1 + * dmapctl describing the disk block. s is the log2(number of logical blocks + * per page) + * + * The calculation figures out how many logical pages are in front of the L1. + * - the number of dmap pages preceding it + * - the number of L0 pages preceding it + * - the number of L1 pages preceding it + * - 1 is added to account for the L2 page + * - 1 is added to account for the control page of the map. + */ +#define BLKTOL1(b,s) \ + (((((b) >> 33) << 20) + (((b) >> 33) << 10) + ((b) >> 33) + 1 + 1) << (s)) + +/* + * convert disk block number to the logical block number of the dmapctl + * at the specified level which describes the disk block. + */ +#define BLKTOCTL(b,s,l) \ + (((l) == 2) ? 1 : ((l) == 1) ? BLKTOL1((b),(s)) : BLKTOL0((b),(s))) + +/* + * convert aggregate map size to the zero origin dmapctl level of the + * top dmapctl. + */ +#define BMAPSZTOLEV(size) \ + (((size) <= MAXL0SIZE) ? 0 : ((size) <= MAXL1SIZE) ? 1 : 2) + +/* convert disk block number to allocation group number. + */ +#define BLKTOAG(b,sbi) ((b) >> ((sbi)->bmap->db_agl2size)) + +/* convert allocation group number to starting disk block + * number. + */ +#define AGTOBLK(a,ip) \ + ((s64)(a) << (JFS_SBI((ip)->i_sb)->bmap->db_agl2size)) + +/* + * dmap summary tree + * + * dmaptree must be consistent with dmapctl. + */ +struct dmaptree { + __le32 nleafs; /* 4: number of tree leafs */ + __le32 l2nleafs; /* 4: l2 number of tree leafs */ + __le32 leafidx; /* 4: index of first tree leaf */ + __le32 height; /* 4: height of the tree */ + s8 budmin; /* 1: min l2 tree leaf value to combine */ + s8 stree[TREESIZE]; /* TREESIZE: tree */ + u8 pad[2]; /* 2: pad to word boundary */ +}; /* - 360 - */ + +/* + * dmap page per 8K blocks bitmap + */ +struct dmap { + __le32 nblocks; /* 4: num blks covered by this dmap */ + __le32 nfree; /* 4: num of free blks in this dmap */ + __le64 start; /* 8: starting blkno for this dmap */ + struct dmaptree tree; /* 360: dmap tree */ + u8 pad[1672]; /* 1672: pad to 2048 bytes */ + __le32 wmap[LPERDMAP]; /* 1024: bits of the working map */ + __le32 pmap[LPERDMAP]; /* 1024: bits of the persistent map */ +}; /* - 4096 - */ + +/* + * disk map control page per level. + * + * dmapctl must be consistent with dmaptree. + */ +struct dmapctl { + __le32 nleafs; /* 4: number of tree leafs */ + __le32 l2nleafs; /* 4: l2 number of tree leafs */ + __le32 leafidx; /* 4: index of the first tree leaf */ + __le32 height; /* 4: height of tree */ + s8 budmin; /* 1: minimum l2 tree leaf value */ + s8 stree[CTLTREESIZE]; /* CTLTREESIZE: dmapctl tree */ + u8 pad[2714]; /* 2714: pad to 4096 */ +}; /* - 4096 - */ + +/* + * common definition for dmaptree within dmap and dmapctl + */ +typedef union dmtree { + struct dmaptree t1; + struct dmapctl t2; +} dmtree_t; + +/* macros for accessing fields within dmtree */ +#define dmt_nleafs t1.nleafs +#define dmt_l2nleafs t1.l2nleafs +#define dmt_leafidx t1.leafidx +#define dmt_height t1.height +#define dmt_budmin t1.budmin +#define dmt_stree t2.stree + +/* + * on-disk aggregate disk allocation map descriptor. + */ +struct dbmap_disk { + __le64 dn_mapsize; /* 8: number of blocks in aggregate */ + __le64 dn_nfree; /* 8: num free blks in aggregate map */ + __le32 dn_l2nbperpage; /* 4: number of blks per page */ + __le32 dn_numag; /* 4: total number of ags */ + __le32 dn_maxlevel; /* 4: number of active ags */ + __le32 dn_maxag; /* 4: max active alloc group number */ + __le32 dn_agpref; /* 4: preferred alloc group (hint) */ + __le32 dn_aglevel; /* 4: dmapctl level holding the AG */ + __le32 dn_agheight; /* 4: height in dmapctl of the AG */ + __le32 dn_agwidth; /* 4: width in dmapctl of the AG */ + __le32 dn_agstart; /* 4: start tree index at AG height */ + __le32 dn_agl2size; /* 4: l2 num of blks per alloc group */ + __le64 dn_agfree[MAXAG];/* 8*MAXAG: per AG free count */ + __le64 dn_agsize; /* 8: num of blks per alloc group */ + s8 dn_maxfreebud; /* 1: max free buddy system */ + u8 pad[3007]; /* 3007: pad to 4096 */ +}; /* - 4096 - */ + +struct dbmap { + s64 dn_mapsize; /* number of blocks in aggregate */ + s64 dn_nfree; /* num free blks in aggregate map */ + int dn_l2nbperpage; /* number of blks per page */ + int dn_numag; /* total number of ags */ + int dn_maxlevel; /* number of active ags */ + int dn_maxag; /* max active alloc group number */ + int dn_agpref; /* preferred alloc group (hint) */ + int dn_aglevel; /* dmapctl level holding the AG */ + int dn_agheight; /* height in dmapctl of the AG */ + int dn_agwidth; /* width in dmapctl of the AG */ + int dn_agstart; /* start tree index at AG height */ + int dn_agl2size; /* l2 num of blks per alloc group */ + s64 dn_agfree[MAXAG]; /* per AG free count */ + s64 dn_agsize; /* num of blks per alloc group */ + signed char dn_maxfreebud; /* max free buddy system */ +}; /* - 4096 - */ +/* + * in-memory aggregate disk allocation map descriptor. + */ +struct bmap { + struct dbmap db_bmap; /* on-disk aggregate map descriptor */ + struct inode *db_ipbmap; /* ptr to aggregate map incore inode */ + struct mutex db_bmaplock; /* aggregate map lock */ + atomic_t db_active[MAXAG]; /* count of active, open files in AG */ + u32 *db_DBmap; +}; + +/* macros for accessing fields within in-memory aggregate map descriptor */ +#define db_mapsize db_bmap.dn_mapsize +#define db_nfree db_bmap.dn_nfree +#define db_agfree db_bmap.dn_agfree +#define db_agsize db_bmap.dn_agsize +#define db_agl2size db_bmap.dn_agl2size +#define db_agwidth db_bmap.dn_agwidth +#define db_agheight db_bmap.dn_agheight +#define db_agstart db_bmap.dn_agstart +#define db_numag db_bmap.dn_numag +#define db_maxlevel db_bmap.dn_maxlevel +#define db_aglevel db_bmap.dn_aglevel +#define db_agpref db_bmap.dn_agpref +#define db_maxag db_bmap.dn_maxag +#define db_maxfreebud db_bmap.dn_maxfreebud +#define db_l2nbperpage db_bmap.dn_l2nbperpage + +/* + * macros for various conversions needed by the allocators. + * blkstol2(), cntlz(), and cnttz() are operating system dependent functions. + */ +/* convert number of blocks to log2 number of blocks, rounding up to + * the next log2 value if blocks is not a l2 multiple. + */ +#define BLKSTOL2(d) (blkstol2(d)) + +/* convert number of leafs to log2 leaf value */ +#define NLSTOL2BSZ(n) (31 - cntlz((n)) + BUDMIN) + +/* convert leaf index to log2 leaf value */ +#define LITOL2BSZ(n,m,b) ((((n) == 0) ? (m) : cnttz((n))) + (b)) + +/* convert a block number to a dmap control leaf index */ +#define BLKTOCTLLEAF(b,m) \ + (((b) & (((s64)1 << ((m) + L2LPERCTL)) - 1)) >> (m)) + +/* convert log2 leaf value to buddy size */ +#define BUDSIZE(s,m) (1 << ((s) - (m))) + +/* + * external references. + */ +extern int dbMount(struct inode *ipbmap); + +extern int dbUnmount(struct inode *ipbmap, int mounterror); + +extern int dbFree(struct inode *ipbmap, s64 blkno, s64 nblocks); + +extern int dbUpdatePMap(struct inode *ipbmap, + int free, s64 blkno, s64 nblocks, struct tblock * tblk); + +extern int dbNextAG(struct inode *ipbmap); + +extern int dbAlloc(struct inode *ipbmap, s64 hint, s64 nblocks, s64 * results); + +extern int dbReAlloc(struct inode *ipbmap, + s64 blkno, s64 nblocks, s64 addnblocks, s64 * results); + +extern int dbSync(struct inode *ipbmap); +extern int dbAllocBottomUp(struct inode *ip, s64 blkno, s64 nblocks); +extern int dbExtendFS(struct inode *ipbmap, s64 blkno, s64 nblocks); +extern void dbFinalizeBmap(struct inode *ipbmap); +extern s64 dbMapFileSizeToMapSize(struct inode *ipbmap); +extern s64 dbDiscardAG(struct inode *ip, int agno, s64 minlen); + +#endif /* _H_JFS_DMAP */ diff --git a/fs/jfs/jfs_dtree.c b/fs/jfs/jfs_dtree.c new file mode 100644 index 000000000..8ce6e4458 --- /dev/null +++ b/fs/jfs/jfs_dtree.c @@ -0,0 +1,4483 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) International Business Machines Corp., 2000-2004 + */ + +/* + * jfs_dtree.c: directory B+-tree manager + * + * B+-tree with variable length key directory: + * + * each directory page is structured as an array of 32-byte + * directory entry slots initialized as a freelist + * to avoid search/compaction of free space at insertion. + * when an entry is inserted, a number of slots are allocated + * from the freelist as required to store variable length data + * of the entry; when the entry is deleted, slots of the entry + * are returned to freelist. + * + * leaf entry stores full name as key and file serial number + * (aka inode number) as data. + * internal/router entry stores sufffix compressed name + * as key and simple extent descriptor as data. + * + * each directory page maintains a sorted entry index table + * which stores the start slot index of sorted entries + * to allow binary search on the table. + * + * directory starts as a root/leaf page in on-disk inode + * inline data area. + * when it becomes full, it starts a leaf of a external extent + * of length of 1 block. each time the first leaf becomes full, + * it is extended rather than split (its size is doubled), + * until its length becoms 4 KBytes, from then the extent is split + * with new 4 Kbyte extent when it becomes full + * to reduce external fragmentation of small directories. + * + * blah, blah, blah, for linear scan of directory in pieces by + * readdir(). + * + * + * case-insensitive directory file system + * + * names are stored in case-sensitive way in leaf entry. + * but stored, searched and compared in case-insensitive (uppercase) order + * (i.e., both search key and entry key are folded for search/compare): + * (note that case-sensitive order is BROKEN in storage, e.g., + * sensitive: Ad, aB, aC, aD -> insensitive: aB, aC, aD, Ad + * + * entries which folds to the same key makes up a equivalent class + * whose members are stored as contiguous cluster (may cross page boundary) + * but whose order is arbitrary and acts as duplicate, e.g., + * abc, Abc, aBc, abC) + * + * once match is found at leaf, requires scan forward/backward + * either for, in case-insensitive search, duplicate + * or for, in case-sensitive search, for exact match + * + * router entry must be created/stored in case-insensitive way + * in internal entry: + * (right most key of left page and left most key of right page + * are folded, and its suffix compression is propagated as router + * key in parent) + * (e.g., if split occurs and , trather than + * should be made the router key for the split) + * + * case-insensitive search: + * + * fold search key; + * + * case-insensitive search of B-tree: + * for internal entry, router key is already folded; + * for leaf entry, fold the entry key before comparison. + * + * if (leaf entry case-insensitive match found) + * if (next entry satisfies case-insensitive match) + * return EDUPLICATE; + * if (prev entry satisfies case-insensitive match) + * return EDUPLICATE; + * return match; + * else + * return no match; + * + * serialization: + * target directory inode lock is being held on entry/exit + * of all main directory service routines. + * + * log based recovery: + */ + +#include +#include +#include +#include "jfs_incore.h" +#include "jfs_superblock.h" +#include "jfs_filsys.h" +#include "jfs_metapage.h" +#include "jfs_dmap.h" +#include "jfs_unicode.h" +#include "jfs_debug.h" + +/* dtree split parameter */ +struct dtsplit { + struct metapage *mp; + s16 index; + s16 nslot; + struct component_name *key; + ddata_t *data; + struct pxdlist *pxdlist; +}; + +#define DT_PAGE(IP, MP) BT_PAGE(IP, MP, dtpage_t, i_dtroot) + +/* get page buffer for specified block address */ +#define DT_GETPAGE(IP, BN, MP, SIZE, P, RC) \ +do { \ + BT_GETPAGE(IP, BN, MP, dtpage_t, SIZE, P, RC, i_dtroot); \ + if (!(RC)) { \ + if ((BN) && !check_dtpage(P)) { \ + BT_PUTPAGE(MP); \ + jfs_error((IP)->i_sb, \ + "DT_GETPAGE: dtree page corrupt\n"); \ + MP = NULL; \ + RC = -EIO; \ + } \ + } \ +} while (0) + +/* for consistency */ +#define DT_PUTPAGE(MP) BT_PUTPAGE(MP) + +#define DT_GETSEARCH(IP, LEAF, BN, MP, P, INDEX) \ + BT_GETSEARCH(IP, LEAF, BN, MP, dtpage_t, P, INDEX, i_dtroot) + +/* + * forward references + */ +static int dtSplitUp(tid_t tid, struct inode *ip, + struct dtsplit * split, struct btstack * btstack); + +static int dtSplitPage(tid_t tid, struct inode *ip, struct dtsplit * split, + struct metapage ** rmpp, dtpage_t ** rpp, pxd_t * rxdp); + +static int dtExtendPage(tid_t tid, struct inode *ip, + struct dtsplit * split, struct btstack * btstack); + +static int dtSplitRoot(tid_t tid, struct inode *ip, + struct dtsplit * split, struct metapage ** rmpp); + +static int dtDeleteUp(tid_t tid, struct inode *ip, struct metapage * fmp, + dtpage_t * fp, struct btstack * btstack); + +static int dtRelink(tid_t tid, struct inode *ip, dtpage_t * p); + +static int dtReadFirst(struct inode *ip, struct btstack * btstack); + +static int dtReadNext(struct inode *ip, + loff_t * offset, struct btstack * btstack); + +static int dtCompare(struct component_name * key, dtpage_t * p, int si); + +static int ciCompare(struct component_name * key, dtpage_t * p, int si, + int flag); + +static void dtGetKey(dtpage_t * p, int i, struct component_name * key, + int flag); + +static int ciGetLeafPrefixKey(dtpage_t * lp, int li, dtpage_t * rp, + int ri, struct component_name * key, int flag); + +static void dtInsertEntry(dtpage_t * p, int index, struct component_name * key, + ddata_t * data, struct dt_lock **); + +static void dtMoveEntry(dtpage_t * sp, int si, dtpage_t * dp, + struct dt_lock ** sdtlock, struct dt_lock ** ddtlock, + int do_index); + +static void dtDeleteEntry(dtpage_t * p, int fi, struct dt_lock ** dtlock); + +static void dtTruncateEntry(dtpage_t * p, int ti, struct dt_lock ** dtlock); + +static void dtLinelockFreelist(dtpage_t * p, int m, struct dt_lock ** dtlock); + +#define ciToUpper(c) UniStrupr((c)->name) + +/* + * read_index_page() + * + * Reads a page of a directory's index table. + * Having metadata mapped into the directory inode's address space + * presents a multitude of problems. We avoid this by mapping to + * the absolute address space outside of the *_metapage routines + */ +static struct metapage *read_index_page(struct inode *inode, s64 blkno) +{ + int rc; + s64 xaddr; + int xflag; + s32 xlen; + + rc = xtLookup(inode, blkno, 1, &xflag, &xaddr, &xlen, 1); + if (rc || (xaddr == 0)) + return NULL; + + return read_metapage(inode, xaddr, PSIZE, 1); +} + +/* + * get_index_page() + * + * Same as get_index_page(), but get's a new page without reading + */ +static struct metapage *get_index_page(struct inode *inode, s64 blkno) +{ + int rc; + s64 xaddr; + int xflag; + s32 xlen; + + rc = xtLookup(inode, blkno, 1, &xflag, &xaddr, &xlen, 1); + if (rc || (xaddr == 0)) + return NULL; + + return get_metapage(inode, xaddr, PSIZE, 1); +} + +/* + * find_index() + * + * Returns dtree page containing directory table entry for specified + * index and pointer to its entry. + * + * mp must be released by caller. + */ +static struct dir_table_slot *find_index(struct inode *ip, u32 index, + struct metapage ** mp, s64 *lblock) +{ + struct jfs_inode_info *jfs_ip = JFS_IP(ip); + s64 blkno; + s64 offset; + int page_offset; + struct dir_table_slot *slot; + static int maxWarnings = 10; + + if (index < 2) { + if (maxWarnings) { + jfs_warn("find_entry called with index = %d", index); + maxWarnings--; + } + return NULL; + } + + if (index >= jfs_ip->next_index) { + jfs_warn("find_entry called with index >= next_index"); + return NULL; + } + + if (jfs_dirtable_inline(ip)) { + /* + * Inline directory table + */ + *mp = NULL; + slot = &jfs_ip->i_dirtable[index - 2]; + } else { + offset = (index - 2) * sizeof(struct dir_table_slot); + page_offset = offset & (PSIZE - 1); + blkno = ((offset + 1) >> L2PSIZE) << + JFS_SBI(ip->i_sb)->l2nbperpage; + + if (*mp && (*lblock != blkno)) { + release_metapage(*mp); + *mp = NULL; + } + if (!(*mp)) { + *lblock = blkno; + *mp = read_index_page(ip, blkno); + } + if (!(*mp)) { + jfs_err("free_index: error reading directory table"); + return NULL; + } + + slot = + (struct dir_table_slot *) ((char *) (*mp)->data + + page_offset); + } + return slot; +} + +static inline void lock_index(tid_t tid, struct inode *ip, struct metapage * mp, + u32 index) +{ + struct tlock *tlck; + struct linelock *llck; + struct lv *lv; + + tlck = txLock(tid, ip, mp, tlckDATA); + llck = (struct linelock *) tlck->lock; + + if (llck->index >= llck->maxcnt) + llck = txLinelock(llck); + lv = &llck->lv[llck->index]; + + /* + * Linelock slot size is twice the size of directory table + * slot size. 512 entries per page. + */ + lv->offset = ((index - 2) & 511) >> 1; + lv->length = 1; + llck->index++; +} + +/* + * add_index() + * + * Adds an entry to the directory index table. This is used to provide + * each directory entry with a persistent index in which to resume + * directory traversals + */ +static u32 add_index(tid_t tid, struct inode *ip, s64 bn, int slot) +{ + struct super_block *sb = ip->i_sb; + struct jfs_sb_info *sbi = JFS_SBI(sb); + struct jfs_inode_info *jfs_ip = JFS_IP(ip); + u64 blkno; + struct dir_table_slot *dirtab_slot; + u32 index; + struct linelock *llck; + struct lv *lv; + struct metapage *mp; + s64 offset; + uint page_offset; + struct tlock *tlck; + s64 xaddr; + + ASSERT(DO_INDEX(ip)); + + if (jfs_ip->next_index < 2) { + jfs_warn("add_index: next_index = %d. Resetting!", + jfs_ip->next_index); + jfs_ip->next_index = 2; + } + + index = jfs_ip->next_index++; + + if (index <= MAX_INLINE_DIRTABLE_ENTRY) { + /* + * i_size reflects size of index table, or 8 bytes per entry. + */ + ip->i_size = (loff_t) (index - 1) << 3; + + /* + * dir table fits inline within inode + */ + dirtab_slot = &jfs_ip->i_dirtable[index-2]; + dirtab_slot->flag = DIR_INDEX_VALID; + dirtab_slot->slot = slot; + DTSaddress(dirtab_slot, bn); + + set_cflag(COMMIT_Dirtable, ip); + + return index; + } + if (index == (MAX_INLINE_DIRTABLE_ENTRY + 1)) { + struct dir_table_slot temp_table[12]; + + /* + * It's time to move the inline table to an external + * page and begin to build the xtree + */ + if (dquot_alloc_block(ip, sbi->nbperpage)) + goto clean_up; + if (dbAlloc(ip, 0, sbi->nbperpage, &xaddr)) { + dquot_free_block(ip, sbi->nbperpage); + goto clean_up; + } + + /* + * Save the table, we're going to overwrite it with the + * xtree root + */ + memcpy(temp_table, &jfs_ip->i_dirtable, sizeof(temp_table)); + + /* + * Initialize empty x-tree + */ + xtInitRoot(tid, ip); + + /* + * Add the first block to the xtree + */ + if (xtInsert(tid, ip, 0, 0, sbi->nbperpage, &xaddr, 0)) { + /* This really shouldn't fail */ + jfs_warn("add_index: xtInsert failed!"); + memcpy(&jfs_ip->i_dirtable, temp_table, + sizeof (temp_table)); + dbFree(ip, xaddr, sbi->nbperpage); + dquot_free_block(ip, sbi->nbperpage); + goto clean_up; + } + ip->i_size = PSIZE; + + mp = get_index_page(ip, 0); + if (!mp) { + jfs_err("add_index: get_metapage failed!"); + xtTruncate(tid, ip, 0, COMMIT_PWMAP); + memcpy(&jfs_ip->i_dirtable, temp_table, + sizeof (temp_table)); + goto clean_up; + } + tlck = txLock(tid, ip, mp, tlckDATA); + llck = (struct linelock *) & tlck->lock; + ASSERT(llck->index == 0); + lv = &llck->lv[0]; + + lv->offset = 0; + lv->length = 6; /* tlckDATA slot size is 16 bytes */ + llck->index++; + + memcpy(mp->data, temp_table, sizeof(temp_table)); + + mark_metapage_dirty(mp); + release_metapage(mp); + + /* + * Logging is now directed by xtree tlocks + */ + clear_cflag(COMMIT_Dirtable, ip); + } + + offset = (index - 2) * sizeof(struct dir_table_slot); + page_offset = offset & (PSIZE - 1); + blkno = ((offset + 1) >> L2PSIZE) << sbi->l2nbperpage; + if (page_offset == 0) { + /* + * This will be the beginning of a new page + */ + xaddr = 0; + if (xtInsert(tid, ip, 0, blkno, sbi->nbperpage, &xaddr, 0)) { + jfs_warn("add_index: xtInsert failed!"); + goto clean_up; + } + ip->i_size += PSIZE; + + if ((mp = get_index_page(ip, blkno))) + memset(mp->data, 0, PSIZE); /* Just looks better */ + else + xtTruncate(tid, ip, offset, COMMIT_PWMAP); + } else + mp = read_index_page(ip, blkno); + + if (!mp) { + jfs_err("add_index: get/read_metapage failed!"); + goto clean_up; + } + + lock_index(tid, ip, mp, index); + + dirtab_slot = + (struct dir_table_slot *) ((char *) mp->data + page_offset); + dirtab_slot->flag = DIR_INDEX_VALID; + dirtab_slot->slot = slot; + DTSaddress(dirtab_slot, bn); + + mark_metapage_dirty(mp); + release_metapage(mp); + + return index; + + clean_up: + + jfs_ip->next_index--; + + return 0; +} + +/* + * free_index() + * + * Marks an entry to the directory index table as free. + */ +static void free_index(tid_t tid, struct inode *ip, u32 index, u32 next) +{ + struct dir_table_slot *dirtab_slot; + s64 lblock; + struct metapage *mp = NULL; + + dirtab_slot = find_index(ip, index, &mp, &lblock); + + if (!dirtab_slot) + return; + + dirtab_slot->flag = DIR_INDEX_FREE; + dirtab_slot->slot = dirtab_slot->addr1 = 0; + dirtab_slot->addr2 = cpu_to_le32(next); + + if (mp) { + lock_index(tid, ip, mp, index); + mark_metapage_dirty(mp); + release_metapage(mp); + } else + set_cflag(COMMIT_Dirtable, ip); +} + +/* + * modify_index() + * + * Changes an entry in the directory index table + */ +static void modify_index(tid_t tid, struct inode *ip, u32 index, s64 bn, + int slot, struct metapage ** mp, s64 *lblock) +{ + struct dir_table_slot *dirtab_slot; + + dirtab_slot = find_index(ip, index, mp, lblock); + + if (!dirtab_slot) + return; + + DTSaddress(dirtab_slot, bn); + dirtab_slot->slot = slot; + + if (*mp) { + lock_index(tid, ip, *mp, index); + mark_metapage_dirty(*mp); + } else + set_cflag(COMMIT_Dirtable, ip); +} + +/* + * read_index() + * + * reads a directory table slot + */ +static int read_index(struct inode *ip, u32 index, + struct dir_table_slot * dirtab_slot) +{ + s64 lblock; + struct metapage *mp = NULL; + struct dir_table_slot *slot; + + slot = find_index(ip, index, &mp, &lblock); + if (!slot) { + return -EIO; + } + + memcpy(dirtab_slot, slot, sizeof(struct dir_table_slot)); + + if (mp) + release_metapage(mp); + + return 0; +} + +/* + * dtSearch() + * + * function: + * Search for the entry with specified key + * + * parameter: + * + * return: 0 - search result on stack, leaf page pinned; + * errno - I/O error + */ +int dtSearch(struct inode *ip, struct component_name * key, ino_t * data, + struct btstack * btstack, int flag) +{ + int rc = 0; + int cmp = 1; /* init for empty page */ + s64 bn; + struct metapage *mp; + dtpage_t *p; + s8 *stbl; + int base, index, lim; + struct btframe *btsp; + pxd_t *pxd; + int psize = 288; /* initial in-line directory */ + ino_t inumber; + struct component_name ciKey; + struct super_block *sb = ip->i_sb; + + ciKey.name = kmalloc_array(JFS_NAME_MAX + 1, sizeof(wchar_t), + GFP_NOFS); + if (!ciKey.name) { + rc = -ENOMEM; + goto dtSearch_Exit2; + } + + + /* uppercase search key for c-i directory */ + UniStrcpy(ciKey.name, key->name); + ciKey.namlen = key->namlen; + + /* only uppercase if case-insensitive support is on */ + if ((JFS_SBI(sb)->mntflag & JFS_OS2) == JFS_OS2) { + ciToUpper(&ciKey); + } + BT_CLR(btstack); /* reset stack */ + + /* init level count for max pages to split */ + btstack->nsplit = 1; + + /* + * search down tree from root: + * + * between two consecutive entries of and of + * internal page, child page Pi contains entry with k, Ki <= K < Kj. + * + * if entry with search key K is not found + * internal page search find the entry with largest key Ki + * less than K which point to the child page to search; + * leaf page search find the entry with smallest key Kj + * greater than K so that the returned index is the position of + * the entry to be shifted right for insertion of new entry. + * for empty tree, search key is greater than any key of the tree. + * + * by convention, root bn = 0. + */ + for (bn = 0;;) { + /* get/pin the page to search */ + DT_GETPAGE(ip, bn, mp, psize, p, rc); + if (rc) + goto dtSearch_Exit1; + + /* get sorted entry table of the page */ + stbl = DT_GETSTBL(p); + + /* + * binary search with search key K on the current page. + */ + for (base = 0, lim = p->header.nextindex; lim; lim >>= 1) { + index = base + (lim >> 1); + + if (stbl[index] < 0) { + rc = -EIO; + goto out; + } + + if (p->header.flag & BT_LEAF) { + /* uppercase leaf name to compare */ + cmp = + ciCompare(&ciKey, p, stbl[index], + JFS_SBI(sb)->mntflag); + } else { + /* router key is in uppercase */ + + cmp = dtCompare(&ciKey, p, stbl[index]); + + + } + if (cmp == 0) { + /* + * search hit + */ + /* search hit - leaf page: + * return the entry found + */ + if (p->header.flag & BT_LEAF) { + inumber = le32_to_cpu( + ((struct ldtentry *) & p->slot[stbl[index]])->inumber); + + /* + * search for JFS_LOOKUP + */ + if (flag == JFS_LOOKUP) { + *data = inumber; + rc = 0; + goto out; + } + + /* + * search for JFS_CREATE + */ + if (flag == JFS_CREATE) { + *data = inumber; + rc = -EEXIST; + goto out; + } + + /* + * search for JFS_REMOVE or JFS_RENAME + */ + if ((flag == JFS_REMOVE || + flag == JFS_RENAME) && + *data != inumber) { + rc = -ESTALE; + goto out; + } + + /* + * JFS_REMOVE|JFS_FINDDIR|JFS_RENAME + */ + /* save search result */ + *data = inumber; + btsp = btstack->top; + btsp->bn = bn; + btsp->index = index; + btsp->mp = mp; + + rc = 0; + goto dtSearch_Exit1; + } + + /* search hit - internal page: + * descend/search its child page + */ + goto getChild; + } + + if (cmp > 0) { + base = index + 1; + --lim; + } + } + + /* + * search miss + * + * base is the smallest index with key (Kj) greater than + * search key (K) and may be zero or (maxindex + 1) index. + */ + /* + * search miss - leaf page + * + * return location of entry (base) where new entry with + * search key K is to be inserted. + */ + if (p->header.flag & BT_LEAF) { + /* + * search for JFS_LOOKUP, JFS_REMOVE, or JFS_RENAME + */ + if (flag == JFS_LOOKUP || flag == JFS_REMOVE || + flag == JFS_RENAME) { + rc = -ENOENT; + goto out; + } + + /* + * search for JFS_CREATE|JFS_FINDDIR: + * + * save search result + */ + *data = 0; + btsp = btstack->top; + btsp->bn = bn; + btsp->index = base; + btsp->mp = mp; + + rc = 0; + goto dtSearch_Exit1; + } + + /* + * search miss - internal page + * + * if base is non-zero, decrement base by one to get the parent + * entry of the child page to search. + */ + index = base ? base - 1 : base; + + /* + * go down to child page + */ + getChild: + /* update max. number of pages to split */ + if (BT_STACK_FULL(btstack)) { + /* Something's corrupted, mark filesystem dirty so + * chkdsk will fix it. + */ + jfs_error(sb, "stack overrun!\n"); + BT_STACK_DUMP(btstack); + rc = -EIO; + goto out; + } + btstack->nsplit++; + + /* push (bn, index) of the parent page/entry */ + BT_PUSH(btstack, bn, index); + + /* get the child page block number */ + pxd = (pxd_t *) & p->slot[stbl[index]]; + bn = addressPXD(pxd); + psize = lengthPXD(pxd) << JFS_SBI(ip->i_sb)->l2bsize; + + /* unpin the parent page */ + DT_PUTPAGE(mp); + } + + out: + DT_PUTPAGE(mp); + + dtSearch_Exit1: + + kfree(ciKey.name); + + dtSearch_Exit2: + + return rc; +} + + +/* + * dtInsert() + * + * function: insert an entry to directory tree + * + * parameter: + * + * return: 0 - success; + * errno - failure; + */ +int dtInsert(tid_t tid, struct inode *ip, + struct component_name * name, ino_t * fsn, struct btstack * btstack) +{ + int rc = 0; + struct metapage *mp; /* meta-page buffer */ + dtpage_t *p; /* base B+-tree index page */ + s64 bn; + int index; + struct dtsplit split; /* split information */ + ddata_t data; + struct dt_lock *dtlck; + int n; + struct tlock *tlck; + struct lv *lv; + + /* + * retrieve search result + * + * dtSearch() returns (leaf page pinned, index at which to insert). + * n.b. dtSearch() may return index of (maxindex + 1) of + * the full page. + */ + DT_GETSEARCH(ip, btstack->top, bn, mp, p, index); + if (p->header.freelist == 0) + return -EINVAL; + + /* + * insert entry for new key + */ + if (DO_INDEX(ip)) { + if (JFS_IP(ip)->next_index == DIREND) { + DT_PUTPAGE(mp); + return -EMLINK; + } + n = NDTLEAF(name->namlen); + data.leaf.tid = tid; + data.leaf.ip = ip; + } else { + n = NDTLEAF_LEGACY(name->namlen); + data.leaf.ip = NULL; /* signifies legacy directory format */ + } + data.leaf.ino = *fsn; + + /* + * leaf page does not have enough room for new entry: + * + * extend/split the leaf page; + * + * dtSplitUp() will insert the entry and unpin the leaf page. + */ + if (n > p->header.freecnt) { + split.mp = mp; + split.index = index; + split.nslot = n; + split.key = name; + split.data = &data; + rc = dtSplitUp(tid, ip, &split, btstack); + return rc; + } + + /* + * leaf page does have enough room for new entry: + * + * insert the new data entry into the leaf page; + */ + BT_MARK_DIRTY(mp, ip); + /* + * acquire a transaction lock on the leaf page + */ + tlck = txLock(tid, ip, mp, tlckDTREE | tlckENTRY); + dtlck = (struct dt_lock *) & tlck->lock; + ASSERT(dtlck->index == 0); + lv = & dtlck->lv[0]; + + /* linelock header */ + lv->offset = 0; + lv->length = 1; + dtlck->index++; + + dtInsertEntry(p, index, name, &data, &dtlck); + + /* linelock stbl of non-root leaf page */ + if (!(p->header.flag & BT_ROOT)) { + if (dtlck->index >= dtlck->maxcnt) + dtlck = (struct dt_lock *) txLinelock(dtlck); + lv = & dtlck->lv[dtlck->index]; + n = index >> L2DTSLOTSIZE; + lv->offset = p->header.stblindex + n; + lv->length = + ((p->header.nextindex - 1) >> L2DTSLOTSIZE) - n + 1; + dtlck->index++; + } + + /* unpin the leaf page */ + DT_PUTPAGE(mp); + + return 0; +} + + +/* + * dtSplitUp() + * + * function: propagate insertion bottom up; + * + * parameter: + * + * return: 0 - success; + * errno - failure; + * leaf page unpinned; + */ +static int dtSplitUp(tid_t tid, + struct inode *ip, struct dtsplit * split, struct btstack * btstack) +{ + struct jfs_sb_info *sbi = JFS_SBI(ip->i_sb); + int rc = 0; + struct metapage *smp; + dtpage_t *sp; /* split page */ + struct metapage *rmp; + dtpage_t *rp; /* new right page split from sp */ + pxd_t rpxd; /* new right page extent descriptor */ + struct metapage *lmp; + dtpage_t *lp; /* left child page */ + int skip; /* index of entry of insertion */ + struct btframe *parent; /* parent page entry on traverse stack */ + s64 xaddr, nxaddr; + int xlen, xsize; + struct pxdlist pxdlist; + pxd_t *pxd; + struct component_name key = { 0, NULL }; + ddata_t *data = split->data; + int n; + struct dt_lock *dtlck; + struct tlock *tlck; + struct lv *lv; + int quota_allocation = 0; + + /* get split page */ + smp = split->mp; + sp = DT_PAGE(ip, smp); + + key.name = kmalloc_array(JFS_NAME_MAX + 2, sizeof(wchar_t), GFP_NOFS); + if (!key.name) { + DT_PUTPAGE(smp); + rc = -ENOMEM; + goto dtSplitUp_Exit; + } + + /* + * split leaf page + * + * The split routines insert the new entry, and + * acquire txLock as appropriate. + */ + /* + * split root leaf page: + */ + if (sp->header.flag & BT_ROOT) { + /* + * allocate a single extent child page + */ + xlen = 1; + n = sbi->bsize >> L2DTSLOTSIZE; + n -= (n + 31) >> L2DTSLOTSIZE; /* stbl size */ + n -= DTROOTMAXSLOT - sp->header.freecnt; /* header + entries */ + if (n <= split->nslot) + xlen++; + if ((rc = dbAlloc(ip, 0, (s64) xlen, &xaddr))) { + DT_PUTPAGE(smp); + goto freeKeyName; + } + + pxdlist.maxnpxd = 1; + pxdlist.npxd = 0; + pxd = &pxdlist.pxd[0]; + PXDaddress(pxd, xaddr); + PXDlength(pxd, xlen); + split->pxdlist = &pxdlist; + rc = dtSplitRoot(tid, ip, split, &rmp); + + if (rc) + dbFree(ip, xaddr, xlen); + else + DT_PUTPAGE(rmp); + + DT_PUTPAGE(smp); + + if (!DO_INDEX(ip)) + ip->i_size = xlen << sbi->l2bsize; + + goto freeKeyName; + } + + /* + * extend first leaf page + * + * extend the 1st extent if less than buffer page size + * (dtExtendPage() reurns leaf page unpinned) + */ + pxd = &sp->header.self; + xlen = lengthPXD(pxd); + xsize = xlen << sbi->l2bsize; + if (xsize < PSIZE) { + xaddr = addressPXD(pxd); + n = xsize >> L2DTSLOTSIZE; + n -= (n + 31) >> L2DTSLOTSIZE; /* stbl size */ + if ((n + sp->header.freecnt) <= split->nslot) + n = xlen + (xlen << 1); + else + n = xlen; + + /* Allocate blocks to quota. */ + rc = dquot_alloc_block(ip, n); + if (rc) + goto extendOut; + quota_allocation += n; + + if ((rc = dbReAlloc(sbi->ipbmap, xaddr, (s64) xlen, + (s64) n, &nxaddr))) + goto extendOut; + + pxdlist.maxnpxd = 1; + pxdlist.npxd = 0; + pxd = &pxdlist.pxd[0]; + PXDaddress(pxd, nxaddr); + PXDlength(pxd, xlen + n); + split->pxdlist = &pxdlist; + if ((rc = dtExtendPage(tid, ip, split, btstack))) { + nxaddr = addressPXD(pxd); + if (xaddr != nxaddr) { + /* free relocated extent */ + xlen = lengthPXD(pxd); + dbFree(ip, nxaddr, (s64) xlen); + } else { + /* free extended delta */ + xlen = lengthPXD(pxd) - n; + xaddr = addressPXD(pxd) + xlen; + dbFree(ip, xaddr, (s64) n); + } + } else if (!DO_INDEX(ip)) + ip->i_size = lengthPXD(pxd) << sbi->l2bsize; + + + extendOut: + DT_PUTPAGE(smp); + goto freeKeyName; + } + + /* + * split leaf page into and a new right page . + * + * return pinned and its extent descriptor + */ + /* + * allocate new directory page extent and + * new index page(s) to cover page split(s) + * + * allocation hint: ? + */ + n = btstack->nsplit; + pxdlist.maxnpxd = pxdlist.npxd = 0; + xlen = sbi->nbperpage; + for (pxd = pxdlist.pxd; n > 0; n--, pxd++) { + if ((rc = dbAlloc(ip, 0, (s64) xlen, &xaddr)) == 0) { + PXDaddress(pxd, xaddr); + PXDlength(pxd, xlen); + pxdlist.maxnpxd++; + continue; + } + + DT_PUTPAGE(smp); + + /* undo allocation */ + goto splitOut; + } + + split->pxdlist = &pxdlist; + if ((rc = dtSplitPage(tid, ip, split, &rmp, &rp, &rpxd))) { + DT_PUTPAGE(smp); + + /* undo allocation */ + goto splitOut; + } + + if (!DO_INDEX(ip)) + ip->i_size += PSIZE; + + /* + * propagate up the router entry for the leaf page just split + * + * insert a router entry for the new page into the parent page, + * propagate the insert/split up the tree by walking back the stack + * of (bn of parent page, index of child page entry in parent page) + * that were traversed during the search for the page that split. + * + * the propagation of insert/split up the tree stops if the root + * splits or the page inserted into doesn't have to split to hold + * the new entry. + * + * the parent entry for the split page remains the same, and + * a new entry is inserted at its right with the first key and + * block number of the new right page. + * + * There are a maximum of 4 pages pinned at any time: + * two children, left parent and right parent (when the parent splits). + * keep the child pages pinned while working on the parent. + * make sure that all pins are released at exit. + */ + while ((parent = BT_POP(btstack)) != NULL) { + /* parent page specified by stack frame */ + + /* keep current child pages (, ) pinned */ + lmp = smp; + lp = sp; + + /* + * insert router entry in parent for new right child page + */ + /* get the parent page */ + DT_GETPAGE(ip, parent->bn, smp, PSIZE, sp, rc); + if (rc) { + DT_PUTPAGE(lmp); + DT_PUTPAGE(rmp); + goto splitOut; + } + + /* + * The new key entry goes ONE AFTER the index of parent entry, + * because the split was to the right. + */ + skip = parent->index + 1; + + /* + * compute the key for the router entry + * + * key suffix compression: + * for internal pages that have leaf pages as children, + * retain only what's needed to distinguish between + * the new entry and the entry on the page to its left. + * If the keys compare equal, retain the entire key. + * + * note that compression is performed only at computing + * router key at the lowest internal level. + * further compression of the key between pairs of higher + * level internal pages loses too much information and + * the search may fail. + * (e.g., two adjacent leaf pages of {a, ..., x} {xx, ...,} + * results in two adjacent parent entries (a)(xx). + * if split occurs between these two entries, and + * if compression is applied, the router key of parent entry + * of right page (x) will divert search for x into right + * subtree and miss x in the left subtree.) + * + * the entire key must be retained for the next-to-leftmost + * internal key at any level of the tree, or search may fail + * (e.g., ?) + */ + switch (rp->header.flag & BT_TYPE) { + case BT_LEAF: + /* + * compute the length of prefix for suffix compression + * between last entry of left page and first entry + * of right page + */ + if ((sp->header.flag & BT_ROOT && skip > 1) || + sp->header.prev != 0 || skip > 1) { + /* compute uppercase router prefix key */ + rc = ciGetLeafPrefixKey(lp, + lp->header.nextindex-1, + rp, 0, &key, + sbi->mntflag); + if (rc) { + DT_PUTPAGE(lmp); + DT_PUTPAGE(rmp); + DT_PUTPAGE(smp); + goto splitOut; + } + } else { + /* next to leftmost entry of + lowest internal level */ + + /* compute uppercase router key */ + dtGetKey(rp, 0, &key, sbi->mntflag); + key.name[key.namlen] = 0; + + if ((sbi->mntflag & JFS_OS2) == JFS_OS2) + ciToUpper(&key); + } + + n = NDTINTERNAL(key.namlen); + break; + + case BT_INTERNAL: + dtGetKey(rp, 0, &key, sbi->mntflag); + n = NDTINTERNAL(key.namlen); + break; + + default: + jfs_err("dtSplitUp(): UFO!"); + break; + } + + /* unpin left child page */ + DT_PUTPAGE(lmp); + + /* + * compute the data for the router entry + */ + data->xd = rpxd; /* child page xd */ + + /* + * parent page is full - split the parent page + */ + if (n > sp->header.freecnt) { + /* init for parent page split */ + split->mp = smp; + split->index = skip; /* index at insert */ + split->nslot = n; + split->key = &key; + /* split->data = data; */ + + /* unpin right child page */ + DT_PUTPAGE(rmp); + + /* The split routines insert the new entry, + * acquire txLock as appropriate. + * return pinned and its block number . + */ + rc = (sp->header.flag & BT_ROOT) ? + dtSplitRoot(tid, ip, split, &rmp) : + dtSplitPage(tid, ip, split, &rmp, &rp, &rpxd); + if (rc) { + DT_PUTPAGE(smp); + goto splitOut; + } + + /* smp and rmp are pinned */ + } + /* + * parent page is not full - insert router entry in parent page + */ + else { + BT_MARK_DIRTY(smp, ip); + /* + * acquire a transaction lock on the parent page + */ + tlck = txLock(tid, ip, smp, tlckDTREE | tlckENTRY); + dtlck = (struct dt_lock *) & tlck->lock; + ASSERT(dtlck->index == 0); + lv = & dtlck->lv[0]; + + /* linelock header */ + lv->offset = 0; + lv->length = 1; + dtlck->index++; + + /* linelock stbl of non-root parent page */ + if (!(sp->header.flag & BT_ROOT)) { + lv++; + n = skip >> L2DTSLOTSIZE; + lv->offset = sp->header.stblindex + n; + lv->length = + ((sp->header.nextindex - + 1) >> L2DTSLOTSIZE) - n + 1; + dtlck->index++; + } + + dtInsertEntry(sp, skip, &key, data, &dtlck); + + /* exit propagate up */ + break; + } + } + + /* unpin current split and its right page */ + DT_PUTPAGE(smp); + DT_PUTPAGE(rmp); + + /* + * free remaining extents allocated for split + */ + splitOut: + n = pxdlist.npxd; + pxd = &pxdlist.pxd[n]; + for (; n < pxdlist.maxnpxd; n++, pxd++) + dbFree(ip, addressPXD(pxd), (s64) lengthPXD(pxd)); + + freeKeyName: + kfree(key.name); + + /* Rollback quota allocation */ + if (rc && quota_allocation) + dquot_free_block(ip, quota_allocation); + + dtSplitUp_Exit: + + return rc; +} + + +/* + * dtSplitPage() + * + * function: Split a non-root page of a btree. + * + * parameter: + * + * return: 0 - success; + * errno - failure; + * return split and new page pinned; + */ +static int dtSplitPage(tid_t tid, struct inode *ip, struct dtsplit * split, + struct metapage ** rmpp, dtpage_t ** rpp, pxd_t * rpxdp) +{ + int rc = 0; + struct metapage *smp; + dtpage_t *sp; + struct metapage *rmp; + dtpage_t *rp; /* new right page allocated */ + s64 rbn; /* new right page block number */ + struct metapage *mp; + dtpage_t *p; + s64 nextbn; + struct pxdlist *pxdlist; + pxd_t *pxd; + int skip, nextindex, half, left, nxt, off, si; + struct ldtentry *ldtentry; + struct idtentry *idtentry; + u8 *stbl; + struct dtslot *f; + int fsi, stblsize; + int n; + struct dt_lock *sdtlck, *rdtlck; + struct tlock *tlck; + struct dt_lock *dtlck; + struct lv *slv, *rlv, *lv; + + /* get split page */ + smp = split->mp; + sp = DT_PAGE(ip, smp); + + /* + * allocate the new right page for the split + */ + pxdlist = split->pxdlist; + pxd = &pxdlist->pxd[pxdlist->npxd]; + pxdlist->npxd++; + rbn = addressPXD(pxd); + rmp = get_metapage(ip, rbn, PSIZE, 1); + if (rmp == NULL) + return -EIO; + + /* Allocate blocks to quota. */ + rc = dquot_alloc_block(ip, lengthPXD(pxd)); + if (rc) { + release_metapage(rmp); + return rc; + } + + jfs_info("dtSplitPage: ip:0x%p smp:0x%p rmp:0x%p", ip, smp, rmp); + + BT_MARK_DIRTY(rmp, ip); + /* + * acquire a transaction lock on the new right page + */ + tlck = txLock(tid, ip, rmp, tlckDTREE | tlckNEW); + rdtlck = (struct dt_lock *) & tlck->lock; + + rp = (dtpage_t *) rmp->data; + *rpp = rp; + rp->header.self = *pxd; + + BT_MARK_DIRTY(smp, ip); + /* + * acquire a transaction lock on the split page + * + * action: + */ + tlck = txLock(tid, ip, smp, tlckDTREE | tlckENTRY); + sdtlck = (struct dt_lock *) & tlck->lock; + + /* linelock header of split page */ + ASSERT(sdtlck->index == 0); + slv = & sdtlck->lv[0]; + slv->offset = 0; + slv->length = 1; + sdtlck->index++; + + /* + * initialize/update sibling pointers between sp and rp + */ + nextbn = le64_to_cpu(sp->header.next); + rp->header.next = cpu_to_le64(nextbn); + rp->header.prev = cpu_to_le64(addressPXD(&sp->header.self)); + sp->header.next = cpu_to_le64(rbn); + + /* + * initialize new right page + */ + rp->header.flag = sp->header.flag; + + /* compute sorted entry table at start of extent data area */ + rp->header.nextindex = 0; + rp->header.stblindex = 1; + + n = PSIZE >> L2DTSLOTSIZE; + rp->header.maxslot = n; + stblsize = (n + 31) >> L2DTSLOTSIZE; /* in unit of slot */ + + /* init freelist */ + fsi = rp->header.stblindex + stblsize; + rp->header.freelist = fsi; + rp->header.freecnt = rp->header.maxslot - fsi; + + /* + * sequential append at tail: append without split + * + * If splitting the last page on a level because of appending + * a entry to it (skip is maxentry), it's likely that the access is + * sequential. Adding an empty page on the side of the level is less + * work and can push the fill factor much higher than normal. + * If we're wrong it's no big deal, we'll just do the split the right + * way next time. + * (It may look like it's equally easy to do a similar hack for + * reverse sorted data, that is, split the tree left, + * but it's not. Be my guest.) + */ + if (nextbn == 0 && split->index == sp->header.nextindex) { + /* linelock header + stbl (first slot) of new page */ + rlv = & rdtlck->lv[rdtlck->index]; + rlv->offset = 0; + rlv->length = 2; + rdtlck->index++; + + /* + * initialize freelist of new right page + */ + f = &rp->slot[fsi]; + for (fsi++; fsi < rp->header.maxslot; f++, fsi++) + f->next = fsi; + f->next = -1; + + /* insert entry at the first entry of the new right page */ + dtInsertEntry(rp, 0, split->key, split->data, &rdtlck); + + goto out; + } + + /* + * non-sequential insert (at possibly middle page) + */ + + /* + * update prev pointer of previous right sibling page; + */ + if (nextbn != 0) { + DT_GETPAGE(ip, nextbn, mp, PSIZE, p, rc); + if (rc) { + discard_metapage(rmp); + return rc; + } + + BT_MARK_DIRTY(mp, ip); + /* + * acquire a transaction lock on the next page + */ + tlck = txLock(tid, ip, mp, tlckDTREE | tlckRELINK); + jfs_info("dtSplitPage: tlck = 0x%p, ip = 0x%p, mp=0x%p", + tlck, ip, mp); + dtlck = (struct dt_lock *) & tlck->lock; + + /* linelock header of previous right sibling page */ + lv = & dtlck->lv[dtlck->index]; + lv->offset = 0; + lv->length = 1; + dtlck->index++; + + p->header.prev = cpu_to_le64(rbn); + + DT_PUTPAGE(mp); + } + + /* + * split the data between the split and right pages. + */ + skip = split->index; + half = (PSIZE >> L2DTSLOTSIZE) >> 1; /* swag */ + left = 0; + + /* + * compute fill factor for split pages + * + * traces the next entry to move to rp + * traces the next entry to stay in sp + */ + stbl = (u8 *) & sp->slot[sp->header.stblindex]; + nextindex = sp->header.nextindex; + for (nxt = off = 0; nxt < nextindex; ++off) { + if (off == skip) + /* check for fill factor with new entry size */ + n = split->nslot; + else { + si = stbl[nxt]; + switch (sp->header.flag & BT_TYPE) { + case BT_LEAF: + ldtentry = (struct ldtentry *) & sp->slot[si]; + if (DO_INDEX(ip)) + n = NDTLEAF(ldtentry->namlen); + else + n = NDTLEAF_LEGACY(ldtentry-> + namlen); + break; + + case BT_INTERNAL: + idtentry = (struct idtentry *) & sp->slot[si]; + n = NDTINTERNAL(idtentry->namlen); + break; + + default: + break; + } + + ++nxt; /* advance to next entry to move in sp */ + } + + left += n; + if (left >= half) + break; + } + + /* poins to the 1st entry to move */ + + /* + * move entries to right page + * + * dtMoveEntry() initializes rp and reserves entry for insertion + * + * split page moved out entries are linelocked; + * new/right page moved in entries are linelocked; + */ + /* linelock header + stbl of new right page */ + rlv = & rdtlck->lv[rdtlck->index]; + rlv->offset = 0; + rlv->length = 5; + rdtlck->index++; + + dtMoveEntry(sp, nxt, rp, &sdtlck, &rdtlck, DO_INDEX(ip)); + + sp->header.nextindex = nxt; + + /* + * finalize freelist of new right page + */ + fsi = rp->header.freelist; + f = &rp->slot[fsi]; + for (fsi++; fsi < rp->header.maxslot; f++, fsi++) + f->next = fsi; + f->next = -1; + + /* + * Update directory index table for entries now in right page + */ + if ((rp->header.flag & BT_LEAF) && DO_INDEX(ip)) { + s64 lblock; + + mp = NULL; + stbl = DT_GETSTBL(rp); + for (n = 0; n < rp->header.nextindex; n++) { + ldtentry = (struct ldtentry *) & rp->slot[stbl[n]]; + modify_index(tid, ip, le32_to_cpu(ldtentry->index), + rbn, n, &mp, &lblock); + } + if (mp) + release_metapage(mp); + } + + /* + * the skipped index was on the left page, + */ + if (skip <= off) { + /* insert the new entry in the split page */ + dtInsertEntry(sp, skip, split->key, split->data, &sdtlck); + + /* linelock stbl of split page */ + if (sdtlck->index >= sdtlck->maxcnt) + sdtlck = (struct dt_lock *) txLinelock(sdtlck); + slv = & sdtlck->lv[sdtlck->index]; + n = skip >> L2DTSLOTSIZE; + slv->offset = sp->header.stblindex + n; + slv->length = + ((sp->header.nextindex - 1) >> L2DTSLOTSIZE) - n + 1; + sdtlck->index++; + } + /* + * the skipped index was on the right page, + */ + else { + /* adjust the skip index to reflect the new position */ + skip -= nxt; + + /* insert the new entry in the right page */ + dtInsertEntry(rp, skip, split->key, split->data, &rdtlck); + } + + out: + *rmpp = rmp; + *rpxdp = *pxd; + + return rc; +} + + +/* + * dtExtendPage() + * + * function: extend 1st/only directory leaf page + * + * parameter: + * + * return: 0 - success; + * errno - failure; + * return extended page pinned; + */ +static int dtExtendPage(tid_t tid, + struct inode *ip, struct dtsplit * split, struct btstack * btstack) +{ + struct super_block *sb = ip->i_sb; + int rc; + struct metapage *smp, *pmp, *mp; + dtpage_t *sp, *pp; + struct pxdlist *pxdlist; + pxd_t *pxd, *tpxd; + int xlen, xsize; + int newstblindex, newstblsize; + int oldstblindex, oldstblsize; + int fsi, last; + struct dtslot *f; + struct btframe *parent; + int n; + struct dt_lock *dtlck; + s64 xaddr, txaddr; + struct tlock *tlck; + struct pxd_lock *pxdlock; + struct lv *lv; + uint type; + struct ldtentry *ldtentry; + u8 *stbl; + + /* get page to extend */ + smp = split->mp; + sp = DT_PAGE(ip, smp); + + /* get parent/root page */ + parent = BT_POP(btstack); + DT_GETPAGE(ip, parent->bn, pmp, PSIZE, pp, rc); + if (rc) + return (rc); + + /* + * extend the extent + */ + pxdlist = split->pxdlist; + pxd = &pxdlist->pxd[pxdlist->npxd]; + pxdlist->npxd++; + + xaddr = addressPXD(pxd); + tpxd = &sp->header.self; + txaddr = addressPXD(tpxd); + /* in-place extension */ + if (xaddr == txaddr) { + type = tlckEXTEND; + } + /* relocation */ + else { + type = tlckNEW; + + /* save moved extent descriptor for later free */ + tlck = txMaplock(tid, ip, tlckDTREE | tlckRELOCATE); + pxdlock = (struct pxd_lock *) & tlck->lock; + pxdlock->flag = mlckFREEPXD; + pxdlock->pxd = sp->header.self; + pxdlock->index = 1; + + /* + * Update directory index table to reflect new page address + */ + if (DO_INDEX(ip)) { + s64 lblock; + + mp = NULL; + stbl = DT_GETSTBL(sp); + for (n = 0; n < sp->header.nextindex; n++) { + ldtentry = + (struct ldtentry *) & sp->slot[stbl[n]]; + modify_index(tid, ip, + le32_to_cpu(ldtentry->index), + xaddr, n, &mp, &lblock); + } + if (mp) + release_metapage(mp); + } + } + + /* + * extend the page + */ + sp->header.self = *pxd; + + jfs_info("dtExtendPage: ip:0x%p smp:0x%p sp:0x%p", ip, smp, sp); + + BT_MARK_DIRTY(smp, ip); + /* + * acquire a transaction lock on the extended/leaf page + */ + tlck = txLock(tid, ip, smp, tlckDTREE | type); + dtlck = (struct dt_lock *) & tlck->lock; + lv = & dtlck->lv[0]; + + /* update buffer extent descriptor of extended page */ + xlen = lengthPXD(pxd); + xsize = xlen << JFS_SBI(sb)->l2bsize; + + /* + * copy old stbl to new stbl at start of extended area + */ + oldstblindex = sp->header.stblindex; + oldstblsize = (sp->header.maxslot + 31) >> L2DTSLOTSIZE; + newstblindex = sp->header.maxslot; + n = xsize >> L2DTSLOTSIZE; + newstblsize = (n + 31) >> L2DTSLOTSIZE; + memcpy(&sp->slot[newstblindex], &sp->slot[oldstblindex], + sp->header.nextindex); + + /* + * in-line extension: linelock old area of extended page + */ + if (type == tlckEXTEND) { + /* linelock header */ + lv->offset = 0; + lv->length = 1; + dtlck->index++; + lv++; + + /* linelock new stbl of extended page */ + lv->offset = newstblindex; + lv->length = newstblsize; + } + /* + * relocation: linelock whole relocated area + */ + else { + lv->offset = 0; + lv->length = sp->header.maxslot + newstblsize; + } + + dtlck->index++; + + sp->header.maxslot = n; + sp->header.stblindex = newstblindex; + /* sp->header.nextindex remains the same */ + + /* + * add old stbl region at head of freelist + */ + fsi = oldstblindex; + f = &sp->slot[fsi]; + last = sp->header.freelist; + for (n = 0; n < oldstblsize; n++, fsi++, f++) { + f->next = last; + last = fsi; + } + sp->header.freelist = last; + sp->header.freecnt += oldstblsize; + + /* + * append free region of newly extended area at tail of freelist + */ + /* init free region of newly extended area */ + fsi = n = newstblindex + newstblsize; + f = &sp->slot[fsi]; + for (fsi++; fsi < sp->header.maxslot; f++, fsi++) + f->next = fsi; + f->next = -1; + + /* append new free region at tail of old freelist */ + fsi = sp->header.freelist; + if (fsi == -1) + sp->header.freelist = n; + else { + do { + f = &sp->slot[fsi]; + fsi = f->next; + } while (fsi != -1); + + f->next = n; + } + + sp->header.freecnt += sp->header.maxslot - n; + + /* + * insert the new entry + */ + dtInsertEntry(sp, split->index, split->key, split->data, &dtlck); + + BT_MARK_DIRTY(pmp, ip); + /* + * linelock any freeslots residing in old extent + */ + if (type == tlckEXTEND) { + n = sp->header.maxslot >> 2; + if (sp->header.freelist < n) + dtLinelockFreelist(sp, n, &dtlck); + } + + /* + * update parent entry on the parent/root page + */ + /* + * acquire a transaction lock on the parent/root page + */ + tlck = txLock(tid, ip, pmp, tlckDTREE | tlckENTRY); + dtlck = (struct dt_lock *) & tlck->lock; + lv = & dtlck->lv[dtlck->index]; + + /* linelock parent entry - 1st slot */ + lv->offset = 1; + lv->length = 1; + dtlck->index++; + + /* update the parent pxd for page extension */ + tpxd = (pxd_t *) & pp->slot[1]; + *tpxd = *pxd; + + DT_PUTPAGE(pmp); + return 0; +} + + +/* + * dtSplitRoot() + * + * function: + * split the full root page into + * original/root/split page and new right page + * i.e., root remains fixed in tree anchor (inode) and + * the root is copied to a single new right child page + * since root page << non-root page, and + * the split root page contains a single entry for the + * new right child page. + * + * parameter: + * + * return: 0 - success; + * errno - failure; + * return new page pinned; + */ +static int dtSplitRoot(tid_t tid, + struct inode *ip, struct dtsplit * split, struct metapage ** rmpp) +{ + struct super_block *sb = ip->i_sb; + struct metapage *smp; + dtroot_t *sp; + struct metapage *rmp; + dtpage_t *rp; + s64 rbn; + int xlen; + int xsize; + struct dtslot *f; + s8 *stbl; + int fsi, stblsize, n; + struct idtentry *s; + pxd_t *ppxd; + struct pxdlist *pxdlist; + pxd_t *pxd; + struct dt_lock *dtlck; + struct tlock *tlck; + struct lv *lv; + int rc; + + /* get split root page */ + smp = split->mp; + sp = &JFS_IP(ip)->i_dtroot; + + /* + * allocate/initialize a single (right) child page + * + * N.B. at first split, a one (or two) block to fit new entry + * is allocated; at subsequent split, a full page is allocated; + */ + pxdlist = split->pxdlist; + pxd = &pxdlist->pxd[pxdlist->npxd]; + pxdlist->npxd++; + rbn = addressPXD(pxd); + xlen = lengthPXD(pxd); + xsize = xlen << JFS_SBI(sb)->l2bsize; + rmp = get_metapage(ip, rbn, xsize, 1); + if (!rmp) + return -EIO; + + rp = rmp->data; + + /* Allocate blocks to quota. */ + rc = dquot_alloc_block(ip, lengthPXD(pxd)); + if (rc) { + release_metapage(rmp); + return rc; + } + + BT_MARK_DIRTY(rmp, ip); + /* + * acquire a transaction lock on the new right page + */ + tlck = txLock(tid, ip, rmp, tlckDTREE | tlckNEW); + dtlck = (struct dt_lock *) & tlck->lock; + + rp->header.flag = + (sp->header.flag & BT_LEAF) ? BT_LEAF : BT_INTERNAL; + rp->header.self = *pxd; + + /* initialize sibling pointers */ + rp->header.next = 0; + rp->header.prev = 0; + + /* + * move in-line root page into new right page extent + */ + /* linelock header + copied entries + new stbl (1st slot) in new page */ + ASSERT(dtlck->index == 0); + lv = & dtlck->lv[0]; + lv->offset = 0; + lv->length = 10; /* 1 + 8 + 1 */ + dtlck->index++; + + n = xsize >> L2DTSLOTSIZE; + rp->header.maxslot = n; + stblsize = (n + 31) >> L2DTSLOTSIZE; + + /* copy old stbl to new stbl at start of extended area */ + rp->header.stblindex = DTROOTMAXSLOT; + stbl = (s8 *) & rp->slot[DTROOTMAXSLOT]; + memcpy(stbl, sp->header.stbl, sp->header.nextindex); + rp->header.nextindex = sp->header.nextindex; + + /* copy old data area to start of new data area */ + memcpy(&rp->slot[1], &sp->slot[1], IDATASIZE); + + /* + * append free region of newly extended area at tail of freelist + */ + /* init free region of newly extended area */ + fsi = n = DTROOTMAXSLOT + stblsize; + f = &rp->slot[fsi]; + for (fsi++; fsi < rp->header.maxslot; f++, fsi++) + f->next = fsi; + f->next = -1; + + /* append new free region at tail of old freelist */ + fsi = sp->header.freelist; + if (fsi == -1) + rp->header.freelist = n; + else { + rp->header.freelist = fsi; + + do { + f = &rp->slot[fsi]; + fsi = f->next; + } while (fsi >= 0); + + f->next = n; + } + + rp->header.freecnt = sp->header.freecnt + rp->header.maxslot - n; + + /* + * Update directory index table for entries now in right page + */ + if ((rp->header.flag & BT_LEAF) && DO_INDEX(ip)) { + s64 lblock; + struct metapage *mp = NULL; + struct ldtentry *ldtentry; + + stbl = DT_GETSTBL(rp); + for (n = 0; n < rp->header.nextindex; n++) { + ldtentry = (struct ldtentry *) & rp->slot[stbl[n]]; + modify_index(tid, ip, le32_to_cpu(ldtentry->index), + rbn, n, &mp, &lblock); + } + if (mp) + release_metapage(mp); + } + /* + * insert the new entry into the new right/child page + * (skip index in the new right page will not change) + */ + dtInsertEntry(rp, split->index, split->key, split->data, &dtlck); + + /* + * reset parent/root page + * + * set the 1st entry offset to 0, which force the left-most key + * at any level of the tree to be less than any search key. + * + * The btree comparison code guarantees that the left-most key on any + * level of the tree is never used, so it doesn't need to be filled in. + */ + BT_MARK_DIRTY(smp, ip); + /* + * acquire a transaction lock on the root page (in-memory inode) + */ + tlck = txLock(tid, ip, smp, tlckDTREE | tlckNEW | tlckBTROOT); + dtlck = (struct dt_lock *) & tlck->lock; + + /* linelock root */ + ASSERT(dtlck->index == 0); + lv = & dtlck->lv[0]; + lv->offset = 0; + lv->length = DTROOTMAXSLOT; + dtlck->index++; + + /* update page header of root */ + if (sp->header.flag & BT_LEAF) { + sp->header.flag &= ~BT_LEAF; + sp->header.flag |= BT_INTERNAL; + } + + /* init the first entry */ + s = (struct idtentry *) & sp->slot[DTENTRYSTART]; + ppxd = (pxd_t *) s; + *ppxd = *pxd; + s->next = -1; + s->namlen = 0; + + stbl = sp->header.stbl; + stbl[0] = DTENTRYSTART; + sp->header.nextindex = 1; + + /* init freelist */ + fsi = DTENTRYSTART + 1; + f = &sp->slot[fsi]; + + /* init free region of remaining area */ + for (fsi++; fsi < DTROOTMAXSLOT; f++, fsi++) + f->next = fsi; + f->next = -1; + + sp->header.freelist = DTENTRYSTART + 1; + sp->header.freecnt = DTROOTMAXSLOT - (DTENTRYSTART + 1); + + *rmpp = rmp; + + return 0; +} + + +/* + * dtDelete() + * + * function: delete the entry(s) referenced by a key. + * + * parameter: + * + * return: + */ +int dtDelete(tid_t tid, + struct inode *ip, struct component_name * key, ino_t * ino, int flag) +{ + int rc = 0; + s64 bn; + struct metapage *mp, *imp; + dtpage_t *p; + int index; + struct btstack btstack; + struct dt_lock *dtlck; + struct tlock *tlck; + struct lv *lv; + int i; + struct ldtentry *ldtentry; + u8 *stbl; + u32 table_index, next_index; + struct metapage *nmp; + dtpage_t *np; + + /* + * search for the entry to delete: + * + * dtSearch() returns (leaf page pinned, index at which to delete). + */ + if ((rc = dtSearch(ip, key, ino, &btstack, flag))) + return rc; + + /* retrieve search result */ + DT_GETSEARCH(ip, btstack.top, bn, mp, p, index); + + /* + * We need to find put the index of the next entry into the + * directory index table in order to resume a readdir from this + * entry. + */ + if (DO_INDEX(ip)) { + stbl = DT_GETSTBL(p); + ldtentry = (struct ldtentry *) & p->slot[stbl[index]]; + table_index = le32_to_cpu(ldtentry->index); + if (index == (p->header.nextindex - 1)) { + /* + * Last entry in this leaf page + */ + if ((p->header.flag & BT_ROOT) + || (p->header.next == 0)) + next_index = -1; + else { + /* Read next leaf page */ + DT_GETPAGE(ip, le64_to_cpu(p->header.next), + nmp, PSIZE, np, rc); + if (rc) + next_index = -1; + else { + stbl = DT_GETSTBL(np); + ldtentry = + (struct ldtentry *) & np-> + slot[stbl[0]]; + next_index = + le32_to_cpu(ldtentry->index); + DT_PUTPAGE(nmp); + } + } + } else { + ldtentry = + (struct ldtentry *) & p->slot[stbl[index + 1]]; + next_index = le32_to_cpu(ldtentry->index); + } + free_index(tid, ip, table_index, next_index); + } + /* + * the leaf page becomes empty, delete the page + */ + if (p->header.nextindex == 1) { + /* delete empty page */ + rc = dtDeleteUp(tid, ip, mp, p, &btstack); + } + /* + * the leaf page has other entries remaining: + * + * delete the entry from the leaf page. + */ + else { + BT_MARK_DIRTY(mp, ip); + /* + * acquire a transaction lock on the leaf page + */ + tlck = txLock(tid, ip, mp, tlckDTREE | tlckENTRY); + dtlck = (struct dt_lock *) & tlck->lock; + + /* + * Do not assume that dtlck->index will be zero. During a + * rename within a directory, this transaction may have + * modified this page already when adding the new entry. + */ + + /* linelock header */ + if (dtlck->index >= dtlck->maxcnt) + dtlck = (struct dt_lock *) txLinelock(dtlck); + lv = & dtlck->lv[dtlck->index]; + lv->offset = 0; + lv->length = 1; + dtlck->index++; + + /* linelock stbl of non-root leaf page */ + if (!(p->header.flag & BT_ROOT)) { + if (dtlck->index >= dtlck->maxcnt) + dtlck = (struct dt_lock *) txLinelock(dtlck); + lv = & dtlck->lv[dtlck->index]; + i = index >> L2DTSLOTSIZE; + lv->offset = p->header.stblindex + i; + lv->length = + ((p->header.nextindex - 1) >> L2DTSLOTSIZE) - + i + 1; + dtlck->index++; + } + + /* free the leaf entry */ + dtDeleteEntry(p, index, &dtlck); + + /* + * Update directory index table for entries moved in stbl + */ + if (DO_INDEX(ip) && index < p->header.nextindex) { + s64 lblock; + + imp = NULL; + stbl = DT_GETSTBL(p); + for (i = index; i < p->header.nextindex; i++) { + ldtentry = + (struct ldtentry *) & p->slot[stbl[i]]; + modify_index(tid, ip, + le32_to_cpu(ldtentry->index), + bn, i, &imp, &lblock); + } + if (imp) + release_metapage(imp); + } + + DT_PUTPAGE(mp); + } + + return rc; +} + + +/* + * dtDeleteUp() + * + * function: + * free empty pages as propagating deletion up the tree + * + * parameter: + * + * return: + */ +static int dtDeleteUp(tid_t tid, struct inode *ip, + struct metapage * fmp, dtpage_t * fp, struct btstack * btstack) +{ + int rc = 0; + struct metapage *mp; + dtpage_t *p; + int index, nextindex; + int xlen; + struct btframe *parent; + struct dt_lock *dtlck; + struct tlock *tlck; + struct lv *lv; + struct pxd_lock *pxdlock; + int i; + + /* + * keep the root leaf page which has become empty + */ + if (BT_IS_ROOT(fmp)) { + /* + * reset the root + * + * dtInitRoot() acquires txlock on the root + */ + dtInitRoot(tid, ip, PARENT(ip)); + + DT_PUTPAGE(fmp); + + return 0; + } + + /* + * free the non-root leaf page + */ + /* + * acquire a transaction lock on the page + * + * write FREEXTENT|NOREDOPAGE log record + * N.B. linelock is overlaid as freed extent descriptor, and + * the buffer page is freed; + */ + tlck = txMaplock(tid, ip, tlckDTREE | tlckFREE); + pxdlock = (struct pxd_lock *) & tlck->lock; + pxdlock->flag = mlckFREEPXD; + pxdlock->pxd = fp->header.self; + pxdlock->index = 1; + + /* update sibling pointers */ + if ((rc = dtRelink(tid, ip, fp))) { + BT_PUTPAGE(fmp); + return rc; + } + + xlen = lengthPXD(&fp->header.self); + + /* Free quota allocation. */ + dquot_free_block(ip, xlen); + + /* free/invalidate its buffer page */ + discard_metapage(fmp); + + /* + * propagate page deletion up the directory tree + * + * If the delete from the parent page makes it empty, + * continue all the way up the tree. + * stop if the root page is reached (which is never deleted) or + * if the entry deletion does not empty the page. + */ + while ((parent = BT_POP(btstack)) != NULL) { + /* pin the parent page */ + DT_GETPAGE(ip, parent->bn, mp, PSIZE, p, rc); + if (rc) + return rc; + + /* + * free the extent of the child page deleted + */ + index = parent->index; + + /* + * delete the entry for the child page from parent + */ + nextindex = p->header.nextindex; + + /* + * the parent has the single entry being deleted: + * + * free the parent page which has become empty. + */ + if (nextindex == 1) { + /* + * keep the root internal page which has become empty + */ + if (p->header.flag & BT_ROOT) { + /* + * reset the root + * + * dtInitRoot() acquires txlock on the root + */ + dtInitRoot(tid, ip, PARENT(ip)); + + DT_PUTPAGE(mp); + + return 0; + } + /* + * free the parent page + */ + else { + /* + * acquire a transaction lock on the page + * + * write FREEXTENT|NOREDOPAGE log record + */ + tlck = + txMaplock(tid, ip, + tlckDTREE | tlckFREE); + pxdlock = (struct pxd_lock *) & tlck->lock; + pxdlock->flag = mlckFREEPXD; + pxdlock->pxd = p->header.self; + pxdlock->index = 1; + + /* update sibling pointers */ + if ((rc = dtRelink(tid, ip, p))) { + DT_PUTPAGE(mp); + return rc; + } + + xlen = lengthPXD(&p->header.self); + + /* Free quota allocation */ + dquot_free_block(ip, xlen); + + /* free/invalidate its buffer page */ + discard_metapage(mp); + + /* propagate up */ + continue; + } + } + + /* + * the parent has other entries remaining: + * + * delete the router entry from the parent page. + */ + BT_MARK_DIRTY(mp, ip); + /* + * acquire a transaction lock on the page + * + * action: router entry deletion + */ + tlck = txLock(tid, ip, mp, tlckDTREE | tlckENTRY); + dtlck = (struct dt_lock *) & tlck->lock; + + /* linelock header */ + if (dtlck->index >= dtlck->maxcnt) + dtlck = (struct dt_lock *) txLinelock(dtlck); + lv = & dtlck->lv[dtlck->index]; + lv->offset = 0; + lv->length = 1; + dtlck->index++; + + /* linelock stbl of non-root leaf page */ + if (!(p->header.flag & BT_ROOT)) { + if (dtlck->index < dtlck->maxcnt) + lv++; + else { + dtlck = (struct dt_lock *) txLinelock(dtlck); + lv = & dtlck->lv[0]; + } + i = index >> L2DTSLOTSIZE; + lv->offset = p->header.stblindex + i; + lv->length = + ((p->header.nextindex - 1) >> L2DTSLOTSIZE) - + i + 1; + dtlck->index++; + } + + /* free the router entry */ + dtDeleteEntry(p, index, &dtlck); + + /* reset key of new leftmost entry of level (for consistency) */ + if (index == 0 && + ((p->header.flag & BT_ROOT) || p->header.prev == 0)) + dtTruncateEntry(p, 0, &dtlck); + + /* unpin the parent page */ + DT_PUTPAGE(mp); + + /* exit propagation up */ + break; + } + + if (!DO_INDEX(ip)) + ip->i_size -= PSIZE; + + return 0; +} + +/* + * dtRelink() + * + * function: + * link around a freed page. + * + * parameter: + * fp: page to be freed + * + * return: + */ +static int dtRelink(tid_t tid, struct inode *ip, dtpage_t * p) +{ + int rc; + struct metapage *mp; + s64 nextbn, prevbn; + struct tlock *tlck; + struct dt_lock *dtlck; + struct lv *lv; + + nextbn = le64_to_cpu(p->header.next); + prevbn = le64_to_cpu(p->header.prev); + + /* update prev pointer of the next page */ + if (nextbn != 0) { + DT_GETPAGE(ip, nextbn, mp, PSIZE, p, rc); + if (rc) + return rc; + + BT_MARK_DIRTY(mp, ip); + /* + * acquire a transaction lock on the next page + * + * action: update prev pointer; + */ + tlck = txLock(tid, ip, mp, tlckDTREE | tlckRELINK); + jfs_info("dtRelink nextbn: tlck = 0x%p, ip = 0x%p, mp=0x%p", + tlck, ip, mp); + dtlck = (struct dt_lock *) & tlck->lock; + + /* linelock header */ + if (dtlck->index >= dtlck->maxcnt) + dtlck = (struct dt_lock *) txLinelock(dtlck); + lv = & dtlck->lv[dtlck->index]; + lv->offset = 0; + lv->length = 1; + dtlck->index++; + + p->header.prev = cpu_to_le64(prevbn); + DT_PUTPAGE(mp); + } + + /* update next pointer of the previous page */ + if (prevbn != 0) { + DT_GETPAGE(ip, prevbn, mp, PSIZE, p, rc); + if (rc) + return rc; + + BT_MARK_DIRTY(mp, ip); + /* + * acquire a transaction lock on the prev page + * + * action: update next pointer; + */ + tlck = txLock(tid, ip, mp, tlckDTREE | tlckRELINK); + jfs_info("dtRelink prevbn: tlck = 0x%p, ip = 0x%p, mp=0x%p", + tlck, ip, mp); + dtlck = (struct dt_lock *) & tlck->lock; + + /* linelock header */ + if (dtlck->index >= dtlck->maxcnt) + dtlck = (struct dt_lock *) txLinelock(dtlck); + lv = & dtlck->lv[dtlck->index]; + lv->offset = 0; + lv->length = 1; + dtlck->index++; + + p->header.next = cpu_to_le64(nextbn); + DT_PUTPAGE(mp); + } + + return 0; +} + + +/* + * dtInitRoot() + * + * initialize directory root (inline in inode) + */ +void dtInitRoot(tid_t tid, struct inode *ip, u32 idotdot) +{ + struct jfs_inode_info *jfs_ip = JFS_IP(ip); + dtroot_t *p; + int fsi; + struct dtslot *f; + struct tlock *tlck; + struct dt_lock *dtlck; + struct lv *lv; + u16 xflag_save; + + /* + * If this was previously an non-empty directory, we need to remove + * the old directory table. + */ + if (DO_INDEX(ip)) { + if (!jfs_dirtable_inline(ip)) { + struct tblock *tblk = tid_to_tblock(tid); + /* + * We're playing games with the tid's xflag. If + * we're removing a regular file, the file's xtree + * is committed with COMMIT_PMAP, but we always + * commit the directories xtree with COMMIT_PWMAP. + */ + xflag_save = tblk->xflag; + tblk->xflag = 0; + /* + * xtTruncate isn't guaranteed to fully truncate + * the xtree. The caller needs to check i_size + * after committing the transaction to see if + * additional truncation is needed. The + * COMMIT_Stale flag tells caller that we + * initiated the truncation. + */ + xtTruncate(tid, ip, 0, COMMIT_PWMAP); + set_cflag(COMMIT_Stale, ip); + + tblk->xflag = xflag_save; + } else + ip->i_size = 1; + + jfs_ip->next_index = 2; + } else + ip->i_size = IDATASIZE; + + /* + * acquire a transaction lock on the root + * + * action: directory initialization; + */ + tlck = txLock(tid, ip, (struct metapage *) & jfs_ip->bxflag, + tlckDTREE | tlckENTRY | tlckBTROOT); + dtlck = (struct dt_lock *) & tlck->lock; + + /* linelock root */ + ASSERT(dtlck->index == 0); + lv = & dtlck->lv[0]; + lv->offset = 0; + lv->length = DTROOTMAXSLOT; + dtlck->index++; + + p = &jfs_ip->i_dtroot; + + p->header.flag = DXD_INDEX | BT_ROOT | BT_LEAF; + + p->header.nextindex = 0; + + /* init freelist */ + fsi = 1; + f = &p->slot[fsi]; + + /* init data area of root */ + for (fsi++; fsi < DTROOTMAXSLOT; f++, fsi++) + f->next = fsi; + f->next = -1; + + p->header.freelist = 1; + p->header.freecnt = 8; + + /* init '..' entry */ + p->header.idotdot = cpu_to_le32(idotdot); + + return; +} + +/* + * add_missing_indices() + * + * function: Fix dtree page in which one or more entries has an invalid index. + * fsck.jfs should really fix this, but it currently does not. + * Called from jfs_readdir when bad index is detected. + */ +static int add_missing_indices(struct inode *inode, s64 bn) +{ + struct ldtentry *d; + struct dt_lock *dtlck; + int i; + uint index; + struct lv *lv; + struct metapage *mp; + dtpage_t *p; + int rc = 0; + s8 *stbl; + tid_t tid; + struct tlock *tlck; + + tid = txBegin(inode->i_sb, 0); + + DT_GETPAGE(inode, bn, mp, PSIZE, p, rc); + + if (rc) { + printk(KERN_ERR "DT_GETPAGE failed!\n"); + goto end; + } + BT_MARK_DIRTY(mp, inode); + + ASSERT(p->header.flag & BT_LEAF); + + tlck = txLock(tid, inode, mp, tlckDTREE | tlckENTRY); + if (BT_IS_ROOT(mp)) + tlck->type |= tlckBTROOT; + + dtlck = (struct dt_lock *) &tlck->lock; + + stbl = DT_GETSTBL(p); + for (i = 0; i < p->header.nextindex; i++) { + if (stbl[i] < 0) { + jfs_err("jfs: add_missing_indices: Invalid stbl[%d] = %d for inode %ld, block = %lld", + i, stbl[i], (long)inode->i_ino, (long long)bn); + rc = -EIO; + + DT_PUTPAGE(mp); + txAbort(tid, 0); + goto end; + } + + d = (struct ldtentry *) &p->slot[stbl[i]]; + index = le32_to_cpu(d->index); + if ((index < 2) || (index >= JFS_IP(inode)->next_index)) { + d->index = cpu_to_le32(add_index(tid, inode, bn, i)); + if (dtlck->index >= dtlck->maxcnt) + dtlck = (struct dt_lock *) txLinelock(dtlck); + lv = &dtlck->lv[dtlck->index]; + lv->offset = stbl[i]; + lv->length = 1; + dtlck->index++; + } + } + + DT_PUTPAGE(mp); + (void) txCommit(tid, 1, &inode, 0); +end: + txEnd(tid); + return rc; +} + +/* + * Buffer to hold directory entry info while traversing a dtree page + * before being fed to the filldir function + */ +struct jfs_dirent { + loff_t position; + int ino; + u16 name_len; + char name[]; +}; + +/* + * function to determine next variable-sized jfs_dirent in buffer + */ +static inline struct jfs_dirent *next_jfs_dirent(struct jfs_dirent *dirent) +{ + return (struct jfs_dirent *) + ((char *)dirent + + ((sizeof (struct jfs_dirent) + dirent->name_len + 1 + + sizeof (loff_t) - 1) & + ~(sizeof (loff_t) - 1))); +} + +/* + * jfs_readdir() + * + * function: read directory entries sequentially + * from the specified entry offset + * + * parameter: + * + * return: offset = (pn, index) of start entry + * of next jfs_readdir()/dtRead() + */ +int jfs_readdir(struct file *file, struct dir_context *ctx) +{ + struct inode *ip = file_inode(file); + struct nls_table *codepage = JFS_SBI(ip->i_sb)->nls_tab; + int rc = 0; + loff_t dtpos; /* legacy OS/2 style position */ + struct dtoffset { + s16 pn; + s16 index; + s32 unused; + } *dtoffset = (struct dtoffset *) &dtpos; + s64 bn; + struct metapage *mp; + dtpage_t *p; + int index; + s8 *stbl; + struct btstack btstack; + int i, next; + struct ldtentry *d; + struct dtslot *t; + int d_namleft, len, outlen; + void *dirent_buf; + char *name_ptr; + u32 dir_index; + int do_index = 0; + uint loop_count = 0; + struct jfs_dirent *jfs_dirent; + int jfs_dirents; + int overflow, fix_page, page_fixed = 0; + static int unique_pos = 2; /* If we can't fix broken index */ + + if (ctx->pos == DIREND) + return 0; + + if (DO_INDEX(ip)) { + /* + * persistent index is stored in directory entries. + * Special cases: 0 = . + * 1 = .. + * -1 = End of directory + */ + do_index = 1; + + dir_index = (u32) ctx->pos; + + /* + * NFSv4 reserves cookies 1 and 2 for . and .. so the value + * we return to the vfs is one greater than the one we use + * internally. + */ + if (dir_index) + dir_index--; + + if (dir_index > 1) { + struct dir_table_slot dirtab_slot; + + if (dtEmpty(ip) || + (dir_index >= JFS_IP(ip)->next_index)) { + /* Stale position. Directory has shrunk */ + ctx->pos = DIREND; + return 0; + } + repeat: + rc = read_index(ip, dir_index, &dirtab_slot); + if (rc) { + ctx->pos = DIREND; + return rc; + } + if (dirtab_slot.flag == DIR_INDEX_FREE) { + if (loop_count++ > JFS_IP(ip)->next_index) { + jfs_err("jfs_readdir detected infinite loop!"); + ctx->pos = DIREND; + return 0; + } + dir_index = le32_to_cpu(dirtab_slot.addr2); + if (dir_index == -1) { + ctx->pos = DIREND; + return 0; + } + goto repeat; + } + bn = addressDTS(&dirtab_slot); + index = dirtab_slot.slot; + DT_GETPAGE(ip, bn, mp, PSIZE, p, rc); + if (rc) { + ctx->pos = DIREND; + return 0; + } + if (p->header.flag & BT_INTERNAL) { + jfs_err("jfs_readdir: bad index table"); + DT_PUTPAGE(mp); + ctx->pos = DIREND; + return 0; + } + } else { + if (dir_index == 0) { + /* + * self "." + */ + ctx->pos = 1; + if (!dir_emit(ctx, ".", 1, ip->i_ino, DT_DIR)) + return 0; + } + /* + * parent ".." + */ + ctx->pos = 2; + if (!dir_emit(ctx, "..", 2, PARENT(ip), DT_DIR)) + return 0; + + /* + * Find first entry of left-most leaf + */ + if (dtEmpty(ip)) { + ctx->pos = DIREND; + return 0; + } + + if ((rc = dtReadFirst(ip, &btstack))) + return rc; + + DT_GETSEARCH(ip, btstack.top, bn, mp, p, index); + } + } else { + /* + * Legacy filesystem - OS/2 & Linux JFS < 0.3.6 + * + * pn = 0; index = 1: First entry "." + * pn = 0; index = 2: Second entry ".." + * pn > 0: Real entries, pn=1 -> leftmost page + * pn = index = -1: No more entries + */ + dtpos = ctx->pos; + if (dtpos < 2) { + /* build "." entry */ + ctx->pos = 1; + if (!dir_emit(ctx, ".", 1, ip->i_ino, DT_DIR)) + return 0; + dtoffset->index = 2; + ctx->pos = dtpos; + } + + if (dtoffset->pn == 0) { + if (dtoffset->index == 2) { + /* build ".." entry */ + if (!dir_emit(ctx, "..", 2, PARENT(ip), DT_DIR)) + return 0; + } else { + jfs_err("jfs_readdir called with invalid offset!"); + } + dtoffset->pn = 1; + dtoffset->index = 0; + ctx->pos = dtpos; + } + + if (dtEmpty(ip)) { + ctx->pos = DIREND; + return 0; + } + + if ((rc = dtReadNext(ip, &ctx->pos, &btstack))) { + jfs_err("jfs_readdir: unexpected rc = %d from dtReadNext", + rc); + ctx->pos = DIREND; + return 0; + } + /* get start leaf page and index */ + DT_GETSEARCH(ip, btstack.top, bn, mp, p, index); + + /* offset beyond directory eof ? */ + if (bn < 0) { + ctx->pos = DIREND; + return 0; + } + } + + dirent_buf = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (dirent_buf == 0) { + DT_PUTPAGE(mp); + jfs_warn("jfs_readdir: __get_free_page failed!"); + ctx->pos = DIREND; + return -ENOMEM; + } + + while (1) { + jfs_dirent = dirent_buf; + jfs_dirents = 0; + overflow = fix_page = 0; + + stbl = DT_GETSTBL(p); + + for (i = index; i < p->header.nextindex; i++) { + if (stbl[i] < 0) { + jfs_err("JFS: Invalid stbl[%d] = %d for inode %ld, block = %lld", + i, stbl[i], (long)ip->i_ino, (long long)bn); + kfree(dirent_buf); + DT_PUTPAGE(mp); + return -EIO; + } + + d = (struct ldtentry *) & p->slot[stbl[i]]; + + if (((long) jfs_dirent + d->namlen + 1) > + ((long)dirent_buf + PAGE_SIZE)) { + /* DBCS codepages could overrun dirent_buf */ + index = i; + overflow = 1; + break; + } + + d_namleft = d->namlen; + name_ptr = jfs_dirent->name; + jfs_dirent->ino = le32_to_cpu(d->inumber); + + if (do_index) { + len = min(d_namleft, DTLHDRDATALEN); + jfs_dirent->position = le32_to_cpu(d->index); + /* + * d->index should always be valid, but it + * isn't. fsck.jfs doesn't create the + * directory index for the lost+found + * directory. Rather than let it go, + * we can try to fix it. + */ + if ((jfs_dirent->position < 2) || + (jfs_dirent->position >= + JFS_IP(ip)->next_index)) { + if (!page_fixed && !isReadOnly(ip)) { + fix_page = 1; + /* + * setting overflow and setting + * index to i will cause the + * same page to be processed + * again starting here + */ + overflow = 1; + index = i; + break; + } + jfs_dirent->position = unique_pos++; + } + /* + * We add 1 to the index because we may + * use a value of 2 internally, and NFSv4 + * doesn't like that. + */ + jfs_dirent->position++; + } else { + jfs_dirent->position = dtpos; + len = min(d_namleft, DTLHDRDATALEN_LEGACY); + } + + /* copy the name of head/only segment */ + outlen = jfs_strfromUCS_le(name_ptr, d->name, len, + codepage); + jfs_dirent->name_len = outlen; + + /* copy name in the additional segment(s) */ + next = d->next; + while (next >= 0) { + t = (struct dtslot *) & p->slot[next]; + name_ptr += outlen; + d_namleft -= len; + /* Sanity Check */ + if (d_namleft == 0) { + jfs_error(ip->i_sb, + "JFS:Dtree error: ino = %ld, bn=%lld, index = %d\n", + (long)ip->i_ino, + (long long)bn, + i); + goto skip_one; + } + len = min(d_namleft, DTSLOTDATALEN); + outlen = jfs_strfromUCS_le(name_ptr, t->name, + len, codepage); + jfs_dirent->name_len += outlen; + + next = t->next; + } + + jfs_dirents++; + jfs_dirent = next_jfs_dirent(jfs_dirent); +skip_one: + if (!do_index) + dtoffset->index++; + } + + if (!overflow) { + /* Point to next leaf page */ + if (p->header.flag & BT_ROOT) + bn = 0; + else { + bn = le64_to_cpu(p->header.next); + index = 0; + /* update offset (pn:index) for new page */ + if (!do_index) { + dtoffset->pn++; + dtoffset->index = 0; + } + } + page_fixed = 0; + } + + /* unpin previous leaf page */ + DT_PUTPAGE(mp); + + jfs_dirent = dirent_buf; + while (jfs_dirents--) { + ctx->pos = jfs_dirent->position; + if (!dir_emit(ctx, jfs_dirent->name, + jfs_dirent->name_len, + jfs_dirent->ino, DT_UNKNOWN)) + goto out; + jfs_dirent = next_jfs_dirent(jfs_dirent); + } + + if (fix_page) { + if ((rc = add_missing_indices(ip, bn))) + goto out; + page_fixed = 1; + } + + if (!overflow && (bn == 0)) { + ctx->pos = DIREND; + break; + } + + DT_GETPAGE(ip, bn, mp, PSIZE, p, rc); + if (rc) { + kfree(dirent_buf); + return rc; + } + } + + out: + kfree(dirent_buf); + + return rc; +} + + +/* + * dtReadFirst() + * + * function: get the leftmost page of the directory + */ +static int dtReadFirst(struct inode *ip, struct btstack * btstack) +{ + int rc = 0; + s64 bn; + int psize = 288; /* initial in-line directory */ + struct metapage *mp; + dtpage_t *p; + s8 *stbl; + struct btframe *btsp; + pxd_t *xd; + + BT_CLR(btstack); /* reset stack */ + + /* + * descend leftmost path of the tree + * + * by convention, root bn = 0. + */ + for (bn = 0;;) { + DT_GETPAGE(ip, bn, mp, psize, p, rc); + if (rc) + return rc; + + /* + * leftmost leaf page + */ + if (p->header.flag & BT_LEAF) { + /* return leftmost entry */ + btsp = btstack->top; + btsp->bn = bn; + btsp->index = 0; + btsp->mp = mp; + + return 0; + } + + /* + * descend down to leftmost child page + */ + if (BT_STACK_FULL(btstack)) { + DT_PUTPAGE(mp); + jfs_error(ip->i_sb, "btstack overrun\n"); + BT_STACK_DUMP(btstack); + return -EIO; + } + /* push (bn, index) of the parent page/entry */ + BT_PUSH(btstack, bn, 0); + + /* get the leftmost entry */ + stbl = DT_GETSTBL(p); + + if (stbl[0] < 0) { + DT_PUTPAGE(mp); + jfs_error(ip->i_sb, "stbl[0] out of bound\n"); + return -EIO; + } + + xd = (pxd_t *) & p->slot[stbl[0]]; + + /* get the child page block address */ + bn = addressPXD(xd); + psize = lengthPXD(xd) << JFS_SBI(ip->i_sb)->l2bsize; + + /* unpin the parent page */ + DT_PUTPAGE(mp); + } +} + + +/* + * dtReadNext() + * + * function: get the page of the specified offset (pn:index) + * + * return: if (offset > eof), bn = -1; + * + * note: if index > nextindex of the target leaf page, + * start with 1st entry of next leaf page; + */ +static int dtReadNext(struct inode *ip, loff_t * offset, + struct btstack * btstack) +{ + int rc = 0; + struct dtoffset { + s16 pn; + s16 index; + s32 unused; + } *dtoffset = (struct dtoffset *) offset; + s64 bn; + struct metapage *mp; + dtpage_t *p; + int index; + int pn; + s8 *stbl; + struct btframe *btsp, *parent; + pxd_t *xd; + + /* + * get leftmost leaf page pinned + */ + if ((rc = dtReadFirst(ip, btstack))) + return rc; + + /* get leaf page */ + DT_GETSEARCH(ip, btstack->top, bn, mp, p, index); + + /* get the start offset (pn:index) */ + pn = dtoffset->pn - 1; /* Now pn = 0 represents leftmost leaf */ + index = dtoffset->index; + + /* start at leftmost page ? */ + if (pn == 0) { + /* offset beyond eof ? */ + if (index < p->header.nextindex) + goto out; + + if (p->header.flag & BT_ROOT) { + bn = -1; + goto out; + } + + /* start with 1st entry of next leaf page */ + dtoffset->pn++; + dtoffset->index = index = 0; + goto a; + } + + /* start at non-leftmost page: scan parent pages for large pn */ + if (p->header.flag & BT_ROOT) { + bn = -1; + goto out; + } + + /* start after next leaf page ? */ + if (pn > 1) + goto b; + + /* get leaf page pn = 1 */ + a: + bn = le64_to_cpu(p->header.next); + + /* unpin leaf page */ + DT_PUTPAGE(mp); + + /* offset beyond eof ? */ + if (bn == 0) { + bn = -1; + goto out; + } + + goto c; + + /* + * scan last internal page level to get target leaf page + */ + b: + /* unpin leftmost leaf page */ + DT_PUTPAGE(mp); + + /* get left most parent page */ + btsp = btstack->top; + parent = btsp - 1; + bn = parent->bn; + DT_GETPAGE(ip, bn, mp, PSIZE, p, rc); + if (rc) + return rc; + + /* scan parent pages at last internal page level */ + while (pn >= p->header.nextindex) { + pn -= p->header.nextindex; + + /* get next parent page address */ + bn = le64_to_cpu(p->header.next); + + /* unpin current parent page */ + DT_PUTPAGE(mp); + + /* offset beyond eof ? */ + if (bn == 0) { + bn = -1; + goto out; + } + + /* get next parent page */ + DT_GETPAGE(ip, bn, mp, PSIZE, p, rc); + if (rc) + return rc; + + /* update parent page stack frame */ + parent->bn = bn; + } + + /* get leaf page address */ + stbl = DT_GETSTBL(p); + xd = (pxd_t *) & p->slot[stbl[pn]]; + bn = addressPXD(xd); + + /* unpin parent page */ + DT_PUTPAGE(mp); + + /* + * get target leaf page + */ + c: + DT_GETPAGE(ip, bn, mp, PSIZE, p, rc); + if (rc) + return rc; + + /* + * leaf page has been completed: + * start with 1st entry of next leaf page + */ + if (index >= p->header.nextindex) { + bn = le64_to_cpu(p->header.next); + + /* unpin leaf page */ + DT_PUTPAGE(mp); + + /* offset beyond eof ? */ + if (bn == 0) { + bn = -1; + goto out; + } + + /* get next leaf page */ + DT_GETPAGE(ip, bn, mp, PSIZE, p, rc); + if (rc) + return rc; + + /* start with 1st entry of next leaf page */ + dtoffset->pn++; + dtoffset->index = 0; + } + + out: + /* return target leaf page pinned */ + btsp = btstack->top; + btsp->bn = bn; + btsp->index = dtoffset->index; + btsp->mp = mp; + + return 0; +} + + +/* + * dtCompare() + * + * function: compare search key with an internal entry + * + * return: + * < 0 if k is < record + * = 0 if k is = record + * > 0 if k is > record + */ +static int dtCompare(struct component_name * key, /* search key */ + dtpage_t * p, /* directory page */ + int si) +{ /* entry slot index */ + wchar_t *kname; + __le16 *name; + int klen, namlen, len, rc; + struct idtentry *ih; + struct dtslot *t; + + /* + * force the left-most key on internal pages, at any level of + * the tree, to be less than any search key. + * this obviates having to update the leftmost key on an internal + * page when the user inserts a new key in the tree smaller than + * anything that has been stored. + * + * (? if/when dtSearch() narrows down to 1st entry (index = 0), + * at any internal page at any level of the tree, + * it descends to child of the entry anyway - + * ? make the entry as min size dummy entry) + * + * if (e->index == 0 && h->prevpg == P_INVALID && !(h->flags & BT_LEAF)) + * return (1); + */ + + kname = key->name; + klen = key->namlen; + + ih = (struct idtentry *) & p->slot[si]; + si = ih->next; + name = ih->name; + namlen = ih->namlen; + len = min(namlen, DTIHDRDATALEN); + + /* compare with head/only segment */ + len = min(klen, len); + if ((rc = UniStrncmp_le(kname, name, len))) + return rc; + + klen -= len; + namlen -= len; + + /* compare with additional segment(s) */ + kname += len; + while (klen > 0 && namlen > 0) { + /* compare with next name segment */ + t = (struct dtslot *) & p->slot[si]; + len = min(namlen, DTSLOTDATALEN); + len = min(klen, len); + name = t->name; + if ((rc = UniStrncmp_le(kname, name, len))) + return rc; + + klen -= len; + namlen -= len; + kname += len; + si = t->next; + } + + return (klen - namlen); +} + + + + +/* + * ciCompare() + * + * function: compare search key with an (leaf/internal) entry + * + * return: + * < 0 if k is < record + * = 0 if k is = record + * > 0 if k is > record + */ +static int ciCompare(struct component_name * key, /* search key */ + dtpage_t * p, /* directory page */ + int si, /* entry slot index */ + int flag) +{ + wchar_t *kname, x; + __le16 *name; + int klen, namlen, len, rc; + struct ldtentry *lh; + struct idtentry *ih; + struct dtslot *t; + int i; + + /* + * force the left-most key on internal pages, at any level of + * the tree, to be less than any search key. + * this obviates having to update the leftmost key on an internal + * page when the user inserts a new key in the tree smaller than + * anything that has been stored. + * + * (? if/when dtSearch() narrows down to 1st entry (index = 0), + * at any internal page at any level of the tree, + * it descends to child of the entry anyway - + * ? make the entry as min size dummy entry) + * + * if (e->index == 0 && h->prevpg == P_INVALID && !(h->flags & BT_LEAF)) + * return (1); + */ + + kname = key->name; + klen = key->namlen; + + /* + * leaf page entry + */ + if (p->header.flag & BT_LEAF) { + lh = (struct ldtentry *) & p->slot[si]; + si = lh->next; + name = lh->name; + namlen = lh->namlen; + if (flag & JFS_DIR_INDEX) + len = min(namlen, DTLHDRDATALEN); + else + len = min(namlen, DTLHDRDATALEN_LEGACY); + } + /* + * internal page entry + */ + else { + ih = (struct idtentry *) & p->slot[si]; + si = ih->next; + name = ih->name; + namlen = ih->namlen; + len = min(namlen, DTIHDRDATALEN); + } + + /* compare with head/only segment */ + len = min(klen, len); + for (i = 0; i < len; i++, kname++, name++) { + /* only uppercase if case-insensitive support is on */ + if ((flag & JFS_OS2) == JFS_OS2) + x = UniToupper(le16_to_cpu(*name)); + else + x = le16_to_cpu(*name); + if ((rc = *kname - x)) + return rc; + } + + klen -= len; + namlen -= len; + + /* compare with additional segment(s) */ + while (klen > 0 && namlen > 0) { + /* compare with next name segment */ + t = (struct dtslot *) & p->slot[si]; + len = min(namlen, DTSLOTDATALEN); + len = min(klen, len); + name = t->name; + for (i = 0; i < len; i++, kname++, name++) { + /* only uppercase if case-insensitive support is on */ + if ((flag & JFS_OS2) == JFS_OS2) + x = UniToupper(le16_to_cpu(*name)); + else + x = le16_to_cpu(*name); + + if ((rc = *kname - x)) + return rc; + } + + klen -= len; + namlen -= len; + si = t->next; + } + + return (klen - namlen); +} + + +/* + * ciGetLeafPrefixKey() + * + * function: compute prefix of suffix compression + * from two adjacent leaf entries + * across page boundary + * + * return: non-zero on error + * + */ +static int ciGetLeafPrefixKey(dtpage_t * lp, int li, dtpage_t * rp, + int ri, struct component_name * key, int flag) +{ + int klen, namlen; + wchar_t *pl, *pr, *kname; + struct component_name lkey; + struct component_name rkey; + + lkey.name = kmalloc_array(JFS_NAME_MAX + 1, sizeof(wchar_t), + GFP_KERNEL); + if (lkey.name == NULL) + return -ENOMEM; + + rkey.name = kmalloc_array(JFS_NAME_MAX + 1, sizeof(wchar_t), + GFP_KERNEL); + if (rkey.name == NULL) { + kfree(lkey.name); + return -ENOMEM; + } + + /* get left and right key */ + dtGetKey(lp, li, &lkey, flag); + lkey.name[lkey.namlen] = 0; + + if ((flag & JFS_OS2) == JFS_OS2) + ciToUpper(&lkey); + + dtGetKey(rp, ri, &rkey, flag); + rkey.name[rkey.namlen] = 0; + + + if ((flag & JFS_OS2) == JFS_OS2) + ciToUpper(&rkey); + + /* compute prefix */ + klen = 0; + kname = key->name; + namlen = min(lkey.namlen, rkey.namlen); + for (pl = lkey.name, pr = rkey.name; + namlen; pl++, pr++, namlen--, klen++, kname++) { + *kname = *pr; + if (*pl != *pr) { + key->namlen = klen + 1; + goto free_names; + } + } + + /* l->namlen <= r->namlen since l <= r */ + if (lkey.namlen < rkey.namlen) { + *kname = *pr; + key->namlen = klen + 1; + } else /* l->namelen == r->namelen */ + key->namlen = klen; + +free_names: + kfree(lkey.name); + kfree(rkey.name); + return 0; +} + + + +/* + * dtGetKey() + * + * function: get key of the entry + */ +static void dtGetKey(dtpage_t * p, int i, /* entry index */ + struct component_name * key, int flag) +{ + int si; + s8 *stbl; + struct ldtentry *lh; + struct idtentry *ih; + struct dtslot *t; + int namlen, len; + wchar_t *kname; + __le16 *name; + + /* get entry */ + stbl = DT_GETSTBL(p); + si = stbl[i]; + if (p->header.flag & BT_LEAF) { + lh = (struct ldtentry *) & p->slot[si]; + si = lh->next; + namlen = lh->namlen; + name = lh->name; + if (flag & JFS_DIR_INDEX) + len = min(namlen, DTLHDRDATALEN); + else + len = min(namlen, DTLHDRDATALEN_LEGACY); + } else { + ih = (struct idtentry *) & p->slot[si]; + si = ih->next; + namlen = ih->namlen; + name = ih->name; + len = min(namlen, DTIHDRDATALEN); + } + + key->namlen = namlen; + kname = key->name; + + /* + * move head/only segment + */ + UniStrncpy_from_le(kname, name, len); + + /* + * move additional segment(s) + */ + while (si >= 0) { + /* get next segment */ + t = &p->slot[si]; + kname += len; + namlen -= len; + len = min(namlen, DTSLOTDATALEN); + UniStrncpy_from_le(kname, t->name, len); + + si = t->next; + } +} + + +/* + * dtInsertEntry() + * + * function: allocate free slot(s) and + * write a leaf/internal entry + * + * return: entry slot index + */ +static void dtInsertEntry(dtpage_t * p, int index, struct component_name * key, + ddata_t * data, struct dt_lock ** dtlock) +{ + struct dtslot *h, *t; + struct ldtentry *lh = NULL; + struct idtentry *ih = NULL; + int hsi, fsi, klen, len, nextindex; + wchar_t *kname; + __le16 *name; + s8 *stbl; + pxd_t *xd; + struct dt_lock *dtlck = *dtlock; + struct lv *lv; + int xsi, n; + s64 bn = 0; + struct metapage *mp = NULL; + + klen = key->namlen; + kname = key->name; + + /* allocate a free slot */ + hsi = fsi = p->header.freelist; + h = &p->slot[fsi]; + p->header.freelist = h->next; + --p->header.freecnt; + + /* open new linelock */ + if (dtlck->index >= dtlck->maxcnt) + dtlck = (struct dt_lock *) txLinelock(dtlck); + + lv = & dtlck->lv[dtlck->index]; + lv->offset = hsi; + + /* write head/only segment */ + if (p->header.flag & BT_LEAF) { + lh = (struct ldtentry *) h; + lh->next = h->next; + lh->inumber = cpu_to_le32(data->leaf.ino); + lh->namlen = klen; + name = lh->name; + if (data->leaf.ip) { + len = min(klen, DTLHDRDATALEN); + if (!(p->header.flag & BT_ROOT)) + bn = addressPXD(&p->header.self); + lh->index = cpu_to_le32(add_index(data->leaf.tid, + data->leaf.ip, + bn, index)); + } else + len = min(klen, DTLHDRDATALEN_LEGACY); + } else { + ih = (struct idtentry *) h; + ih->next = h->next; + xd = (pxd_t *) ih; + *xd = data->xd; + ih->namlen = klen; + name = ih->name; + len = min(klen, DTIHDRDATALEN); + } + + UniStrncpy_to_le(name, kname, len); + + n = 1; + xsi = hsi; + + /* write additional segment(s) */ + t = h; + klen -= len; + while (klen) { + /* get free slot */ + fsi = p->header.freelist; + t = &p->slot[fsi]; + p->header.freelist = t->next; + --p->header.freecnt; + + /* is next slot contiguous ? */ + if (fsi != xsi + 1) { + /* close current linelock */ + lv->length = n; + dtlck->index++; + + /* open new linelock */ + if (dtlck->index < dtlck->maxcnt) + lv++; + else { + dtlck = (struct dt_lock *) txLinelock(dtlck); + lv = & dtlck->lv[0]; + } + + lv->offset = fsi; + n = 0; + } + + kname += len; + len = min(klen, DTSLOTDATALEN); + UniStrncpy_to_le(t->name, kname, len); + + n++; + xsi = fsi; + klen -= len; + } + + /* close current linelock */ + lv->length = n; + dtlck->index++; + + *dtlock = dtlck; + + /* terminate last/only segment */ + if (h == t) { + /* single segment entry */ + if (p->header.flag & BT_LEAF) + lh->next = -1; + else + ih->next = -1; + } else + /* multi-segment entry */ + t->next = -1; + + /* if insert into middle, shift right succeeding entries in stbl */ + stbl = DT_GETSTBL(p); + nextindex = p->header.nextindex; + if (index < nextindex) { + memmove(stbl + index + 1, stbl + index, nextindex - index); + + if ((p->header.flag & BT_LEAF) && data->leaf.ip) { + s64 lblock; + + /* + * Need to update slot number for entries that moved + * in the stbl + */ + mp = NULL; + for (n = index + 1; n <= nextindex; n++) { + lh = (struct ldtentry *) & (p->slot[stbl[n]]); + modify_index(data->leaf.tid, data->leaf.ip, + le32_to_cpu(lh->index), bn, n, + &mp, &lblock); + } + if (mp) + release_metapage(mp); + } + } + + stbl[index] = hsi; + + /* advance next available entry index of stbl */ + ++p->header.nextindex; +} + + +/* + * dtMoveEntry() + * + * function: move entries from split/left page to new/right page + * + * nextindex of dst page and freelist/freecnt of both pages + * are updated. + */ +static void dtMoveEntry(dtpage_t * sp, int si, dtpage_t * dp, + struct dt_lock ** sdtlock, struct dt_lock ** ddtlock, + int do_index) +{ + int ssi, next; /* src slot index */ + int di; /* dst entry index */ + int dsi; /* dst slot index */ + s8 *sstbl, *dstbl; /* sorted entry table */ + int snamlen, len; + struct ldtentry *slh, *dlh = NULL; + struct idtentry *sih, *dih = NULL; + struct dtslot *h, *s, *d; + struct dt_lock *sdtlck = *sdtlock, *ddtlck = *ddtlock; + struct lv *slv, *dlv; + int xssi, ns, nd; + int sfsi; + + sstbl = (s8 *) & sp->slot[sp->header.stblindex]; + dstbl = (s8 *) & dp->slot[dp->header.stblindex]; + + dsi = dp->header.freelist; /* first (whole page) free slot */ + sfsi = sp->header.freelist; + + /* linelock destination entry slot */ + dlv = & ddtlck->lv[ddtlck->index]; + dlv->offset = dsi; + + /* linelock source entry slot */ + slv = & sdtlck->lv[sdtlck->index]; + slv->offset = sstbl[si]; + xssi = slv->offset - 1; + + /* + * move entries + */ + ns = nd = 0; + for (di = 0; si < sp->header.nextindex; si++, di++) { + ssi = sstbl[si]; + dstbl[di] = dsi; + + /* is next slot contiguous ? */ + if (ssi != xssi + 1) { + /* close current linelock */ + slv->length = ns; + sdtlck->index++; + + /* open new linelock */ + if (sdtlck->index < sdtlck->maxcnt) + slv++; + else { + sdtlck = (struct dt_lock *) txLinelock(sdtlck); + slv = & sdtlck->lv[0]; + } + + slv->offset = ssi; + ns = 0; + } + + /* + * move head/only segment of an entry + */ + /* get dst slot */ + h = d = &dp->slot[dsi]; + + /* get src slot and move */ + s = &sp->slot[ssi]; + if (sp->header.flag & BT_LEAF) { + /* get source entry */ + slh = (struct ldtentry *) s; + dlh = (struct ldtentry *) h; + snamlen = slh->namlen; + + if (do_index) { + len = min(snamlen, DTLHDRDATALEN); + dlh->index = slh->index; /* little-endian */ + } else + len = min(snamlen, DTLHDRDATALEN_LEGACY); + + memcpy(dlh, slh, 6 + len * 2); + + next = slh->next; + + /* update dst head/only segment next field */ + dsi++; + dlh->next = dsi; + } else { + sih = (struct idtentry *) s; + snamlen = sih->namlen; + + len = min(snamlen, DTIHDRDATALEN); + dih = (struct idtentry *) h; + memcpy(dih, sih, 10 + len * 2); + next = sih->next; + + dsi++; + dih->next = dsi; + } + + /* free src head/only segment */ + s->next = sfsi; + s->cnt = 1; + sfsi = ssi; + + ns++; + nd++; + xssi = ssi; + + /* + * move additional segment(s) of the entry + */ + snamlen -= len; + while ((ssi = next) >= 0) { + /* is next slot contiguous ? */ + if (ssi != xssi + 1) { + /* close current linelock */ + slv->length = ns; + sdtlck->index++; + + /* open new linelock */ + if (sdtlck->index < sdtlck->maxcnt) + slv++; + else { + sdtlck = + (struct dt_lock *) + txLinelock(sdtlck); + slv = & sdtlck->lv[0]; + } + + slv->offset = ssi; + ns = 0; + } + + /* get next source segment */ + s = &sp->slot[ssi]; + + /* get next destination free slot */ + d++; + + len = min(snamlen, DTSLOTDATALEN); + UniStrncpy_le(d->name, s->name, len); + + ns++; + nd++; + xssi = ssi; + + dsi++; + d->next = dsi; + + /* free source segment */ + next = s->next; + s->next = sfsi; + s->cnt = 1; + sfsi = ssi; + + snamlen -= len; + } /* end while */ + + /* terminate dst last/only segment */ + if (h == d) { + /* single segment entry */ + if (dp->header.flag & BT_LEAF) + dlh->next = -1; + else + dih->next = -1; + } else + /* multi-segment entry */ + d->next = -1; + } /* end for */ + + /* close current linelock */ + slv->length = ns; + sdtlck->index++; + *sdtlock = sdtlck; + + dlv->length = nd; + ddtlck->index++; + *ddtlock = ddtlck; + + /* update source header */ + sp->header.freelist = sfsi; + sp->header.freecnt += nd; + + /* update destination header */ + dp->header.nextindex = di; + + dp->header.freelist = dsi; + dp->header.freecnt -= nd; +} + + +/* + * dtDeleteEntry() + * + * function: free a (leaf/internal) entry + * + * log freelist header, stbl, and each segment slot of entry + * (even though last/only segment next field is modified, + * physical image logging requires all segment slots of + * the entry logged to avoid applying previous updates + * to the same slots) + */ +static void dtDeleteEntry(dtpage_t * p, int fi, struct dt_lock ** dtlock) +{ + int fsi; /* free entry slot index */ + s8 *stbl; + struct dtslot *t; + int si, freecnt; + struct dt_lock *dtlck = *dtlock; + struct lv *lv; + int xsi, n; + + /* get free entry slot index */ + stbl = DT_GETSTBL(p); + fsi = stbl[fi]; + + /* open new linelock */ + if (dtlck->index >= dtlck->maxcnt) + dtlck = (struct dt_lock *) txLinelock(dtlck); + lv = & dtlck->lv[dtlck->index]; + + lv->offset = fsi; + + /* get the head/only segment */ + t = &p->slot[fsi]; + if (p->header.flag & BT_LEAF) + si = ((struct ldtentry *) t)->next; + else + si = ((struct idtentry *) t)->next; + t->next = si; + t->cnt = 1; + + n = freecnt = 1; + xsi = fsi; + + /* find the last/only segment */ + while (si >= 0) { + /* is next slot contiguous ? */ + if (si != xsi + 1) { + /* close current linelock */ + lv->length = n; + dtlck->index++; + + /* open new linelock */ + if (dtlck->index < dtlck->maxcnt) + lv++; + else { + dtlck = (struct dt_lock *) txLinelock(dtlck); + lv = & dtlck->lv[0]; + } + + lv->offset = si; + n = 0; + } + + n++; + xsi = si; + freecnt++; + + t = &p->slot[si]; + t->cnt = 1; + si = t->next; + } + + /* close current linelock */ + lv->length = n; + dtlck->index++; + + *dtlock = dtlck; + + /* update freelist */ + t->next = p->header.freelist; + p->header.freelist = fsi; + p->header.freecnt += freecnt; + + /* if delete from middle, + * shift left the succedding entries in the stbl + */ + si = p->header.nextindex; + if (fi < si - 1) + memmove(&stbl[fi], &stbl[fi + 1], si - fi - 1); + + p->header.nextindex--; +} + + +/* + * dtTruncateEntry() + * + * function: truncate a (leaf/internal) entry + * + * log freelist header, stbl, and each segment slot of entry + * (even though last/only segment next field is modified, + * physical image logging requires all segment slots of + * the entry logged to avoid applying previous updates + * to the same slots) + */ +static void dtTruncateEntry(dtpage_t * p, int ti, struct dt_lock ** dtlock) +{ + int tsi; /* truncate entry slot index */ + s8 *stbl; + struct dtslot *t; + int si, freecnt; + struct dt_lock *dtlck = *dtlock; + struct lv *lv; + int fsi, xsi, n; + + /* get free entry slot index */ + stbl = DT_GETSTBL(p); + tsi = stbl[ti]; + + /* open new linelock */ + if (dtlck->index >= dtlck->maxcnt) + dtlck = (struct dt_lock *) txLinelock(dtlck); + lv = & dtlck->lv[dtlck->index]; + + lv->offset = tsi; + + /* get the head/only segment */ + t = &p->slot[tsi]; + ASSERT(p->header.flag & BT_INTERNAL); + ((struct idtentry *) t)->namlen = 0; + si = ((struct idtentry *) t)->next; + ((struct idtentry *) t)->next = -1; + + n = 1; + freecnt = 0; + fsi = si; + xsi = tsi; + + /* find the last/only segment */ + while (si >= 0) { + /* is next slot contiguous ? */ + if (si != xsi + 1) { + /* close current linelock */ + lv->length = n; + dtlck->index++; + + /* open new linelock */ + if (dtlck->index < dtlck->maxcnt) + lv++; + else { + dtlck = (struct dt_lock *) txLinelock(dtlck); + lv = & dtlck->lv[0]; + } + + lv->offset = si; + n = 0; + } + + n++; + xsi = si; + freecnt++; + + t = &p->slot[si]; + t->cnt = 1; + si = t->next; + } + + /* close current linelock */ + lv->length = n; + dtlck->index++; + + *dtlock = dtlck; + + /* update freelist */ + if (freecnt == 0) + return; + t->next = p->header.freelist; + p->header.freelist = fsi; + p->header.freecnt += freecnt; +} + + +/* + * dtLinelockFreelist() + */ +static void dtLinelockFreelist(dtpage_t * p, /* directory page */ + int m, /* max slot index */ + struct dt_lock ** dtlock) +{ + int fsi; /* free entry slot index */ + struct dtslot *t; + int si; + struct dt_lock *dtlck = *dtlock; + struct lv *lv; + int xsi, n; + + /* get free entry slot index */ + fsi = p->header.freelist; + + /* open new linelock */ + if (dtlck->index >= dtlck->maxcnt) + dtlck = (struct dt_lock *) txLinelock(dtlck); + lv = & dtlck->lv[dtlck->index]; + + lv->offset = fsi; + + n = 1; + xsi = fsi; + + t = &p->slot[fsi]; + si = t->next; + + /* find the last/only segment */ + while (si < m && si >= 0) { + /* is next slot contiguous ? */ + if (si != xsi + 1) { + /* close current linelock */ + lv->length = n; + dtlck->index++; + + /* open new linelock */ + if (dtlck->index < dtlck->maxcnt) + lv++; + else { + dtlck = (struct dt_lock *) txLinelock(dtlck); + lv = & dtlck->lv[0]; + } + + lv->offset = si; + n = 0; + } + + n++; + xsi = si; + + t = &p->slot[si]; + si = t->next; + } + + /* close current linelock */ + lv->length = n; + dtlck->index++; + + *dtlock = dtlck; +} + + +/* + * NAME: dtModify + * + * FUNCTION: Modify the inode number part of a directory entry + * + * PARAMETERS: + * tid - Transaction id + * ip - Inode of parent directory + * key - Name of entry to be modified + * orig_ino - Original inode number expected in entry + * new_ino - New inode number to put into entry + * flag - JFS_RENAME + * + * RETURNS: + * -ESTALE - If entry found does not match orig_ino passed in + * -ENOENT - If no entry can be found to match key + * 0 - If successfully modified entry + */ +int dtModify(tid_t tid, struct inode *ip, + struct component_name * key, ino_t * orig_ino, ino_t new_ino, int flag) +{ + int rc; + s64 bn; + struct metapage *mp; + dtpage_t *p; + int index; + struct btstack btstack; + struct tlock *tlck; + struct dt_lock *dtlck; + struct lv *lv; + s8 *stbl; + int entry_si; /* entry slot index */ + struct ldtentry *entry; + + /* + * search for the entry to modify: + * + * dtSearch() returns (leaf page pinned, index at which to modify). + */ + if ((rc = dtSearch(ip, key, orig_ino, &btstack, flag))) + return rc; + + /* retrieve search result */ + DT_GETSEARCH(ip, btstack.top, bn, mp, p, index); + + BT_MARK_DIRTY(mp, ip); + /* + * acquire a transaction lock on the leaf page of named entry + */ + tlck = txLock(tid, ip, mp, tlckDTREE | tlckENTRY); + dtlck = (struct dt_lock *) & tlck->lock; + + /* get slot index of the entry */ + stbl = DT_GETSTBL(p); + entry_si = stbl[index]; + + /* linelock entry */ + ASSERT(dtlck->index == 0); + lv = & dtlck->lv[0]; + lv->offset = entry_si; + lv->length = 1; + dtlck->index++; + + /* get the head/only segment */ + entry = (struct ldtentry *) & p->slot[entry_si]; + + /* substitute the inode number of the entry */ + entry->inumber = cpu_to_le32(new_ino); + + /* unpin the leaf page */ + DT_PUTPAGE(mp); + + return 0; +} + +bool check_dtroot(dtroot_t *p) +{ + DECLARE_BITMAP(bitmap, DTROOTMAXSLOT) = {0}; + int i; + + /* freecnt cannot be negative or exceed DTROOTMAXSLOT-1 + * (since slot[0] is occupied by the header). + */ + if (unlikely(p->header.freecnt < 0 || + p->header.freecnt > DTROOTMAXSLOT - 1)) { + jfs_err("Bad freecnt:%d in dtroot\n", p->header.freecnt); + return false; + } else if (p->header.freecnt == 0) { + /* No free slots: freelist must be -1 */ + if (unlikely(p->header.freelist != -1)) { + jfs_err("freecnt=0, but freelist=%d in dtroot\n", + p->header.freelist); + return false; + } + } else { + int fsi, i; + /* When there are free slots, freelist must be a valid slot index in + * 1~DTROOTMAXSLOT-1(since slot[0] is occupied by the header). + */ + if (unlikely(p->header.freelist < 1 || + p->header.freelist >= DTROOTMAXSLOT)) { + jfs_err("Bad freelist:%d in dtroot\n", p->header.freelist); + return false; + } + + /* Traverse the free list to check validity of all node indices */ + fsi = p->header.freelist; + for (i = 0; i < p->header.freecnt - 1; i++) { + /* Check for duplicate indices in the free list */ + if (unlikely(__test_and_set_bit(fsi, bitmap))) { + jfs_err("duplicate index%d in slot in dtroot\n", fsi); + return false; + } + fsi = p->slot[fsi].next; + + /* Ensure the next slot index in the free list is valid */ + if (unlikely(fsi < 1 || fsi >= DTROOTMAXSLOT)) { + jfs_err("Bad index:%d in slot in dtroot\n", fsi); + return false; + } + } + + /* The last node in the free list must terminate with next = -1 */ + if (unlikely(p->slot[fsi].next != -1)) { + jfs_err("Bad next:%d of the last slot in dtroot\n", + p->slot[fsi].next); + return false; + } + } + + /* Validate nextindex (next free entry index in stbl) + * stbl array has size 8 (indices 0~7). + * It may get set to 8 when the last free slot has been filled. + */ + if (unlikely(p->header.nextindex > ARRAY_SIZE(p->header.stbl))) { + jfs_err("Bad nextindex:%d in dtroot\n", p->header.nextindex); + return false; + } + + /* Validate index validity of stbl array (8 elements) + * Each entry in stbl is a slot index, with valid range: -1 (invalid) + * or 0~8 (slot[0]~slot[8]) + */ + for (i = 0; i < p->header.nextindex; i++) { + int idx = p->header.stbl[i]; + + if (unlikely(idx < 0 || idx >= 9)) { + jfs_err("Bad index:%d of stbl[%d] in dtroot\n", idx, i); + return false; /* stbl entry points out of slot array range */ + } + + /* Check for duplicate valid indices (skip check for idx=0) */ + if (unlikely(idx && __test_and_set_bit(idx, bitmap))) { + jfs_err("Duplicate index:%d in stbl in dtroot\n", idx); + return false; + } + } + + return true; +} + +bool check_dtpage(dtpage_t *p) +{ + DECLARE_BITMAP(bitmap, DTPAGEMAXSLOT) = {0}; + const int stblsize = ((PSIZE >> L2DTSLOTSIZE) + 31) >> L2DTSLOTSIZE; + int i; + + /* Validate maxslot (maximum number of slots in the page) + * dtpage_t slot array is defined to hold up to DTPAGEMAXSLOT (128) slots + */ + if (unlikely(p->header.maxslot != DTPAGEMAXSLOT)) { + jfs_err("Bad maxslot:%d in dtpage (expected %d)\n", + p->header.maxslot, DTPAGEMAXSLOT); + return false; + } + + /* freecnt cannot be negative or exceed DTPAGEMAXSLOT-1 + * (since slot[0] is occupied by the header). + */ + if (unlikely(p->header.freecnt < 0 || + p->header.freecnt > DTPAGEMAXSLOT - 1)) { + jfs_err("Bad freecnt:%d in dtpage\n", p->header.freecnt); + return false; + } else if (p->header.freecnt == 0) { + /* No free slots: freelist must be -1 */ + if (unlikely(p->header.freelist != -1)) { + jfs_err("freecnt=0 but freelist=%d in dtpage\n", + p->header.freelist); + return false; + } + } else { + int fsi; + + if (unlikely(p->header.freelist < 1)) { + jfs_err("Bad freelist:%d in dtpage\n", p->header.freelist); + return false; + } + + /* Traverse the free list to check validity of all node indices */ + fsi = p->header.freelist; + for (i = 0; i < p->header.freecnt - 1; i++) { + /* Check for duplicate indices in the free list */ + if (unlikely(__test_and_set_bit(fsi, bitmap))) { + jfs_err("duplicate index%d in slot in dtpage\n", fsi); + return false; + } + fsi = p->slot[fsi].next; + + /* Ensure the next slot index in the free list is valid */ + if (unlikely(fsi < 1 || fsi >= DTPAGEMAXSLOT)) { + jfs_err("Bad index:%d in slot in dtpage\n", fsi); + return false; + } + } + + /* The last node in the free list must terminate with next = -1 */ + if (unlikely(p->slot[fsi].next != -1)) { + jfs_err("Bad next:%d of the last slot in dtpage\n", + p->slot[fsi].next); + return false; + } + } + + /* stbl must be little then DTPAGEMAXSLOT */ + if (unlikely(p->header.stblindex >= DTPAGEMAXSLOT - stblsize)) { + jfs_err("Bad stblindex:%d in dtpage (stbl size %d)\n", + p->header.stblindex, stblsize); + return false; + } + + /* nextindex must be little then stblsize*32 */ + if (unlikely(p->header.nextindex > (stblsize << L2DTSLOTSIZE))) { + jfs_err("Bad nextindex:%d in dtpage (stbl size %d)\n", + p->header.nextindex, stblsize); + return false; + } + + /* Validate stbl entries + * Each entry is a slot index, valid range: -1 (invalid) or + * [0, nextindex-1] (valid data slots) + * (stblindex and higher slots are reserved for stbl itself) + */ + for (i = 0; i < p->header.nextindex; i++) { + int idx = DT_GETSTBL(p)[i]; + + /* Check if index is out of valid data slot range */ + if (unlikely(idx < 1 || idx >= DTPAGEMAXSLOT)) { + jfs_err("Bad stbl[%d] index:%d (stblindex %d) in dtpage\n", + i, idx, p->header.stblindex); + return false; + } + + /* Check for duplicate valid indices (skip -1) */ + if (unlikely(__test_and_set_bit(idx, bitmap))) { + jfs_err("Duplicate index:%d in stbl of dtpage\n", idx); + return false; + } + } + + return true; +} diff --git a/fs/jfs/jfs_dtree.h b/fs/jfs/jfs_dtree.h new file mode 100644 index 000000000..dfc87b669 --- /dev/null +++ b/fs/jfs/jfs_dtree.h @@ -0,0 +1,260 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) International Business Machines Corp., 2000-2002 + */ +#ifndef _H_JFS_DTREE +#define _H_JFS_DTREE + +/* + * jfs_dtree.h: directory B+-tree manager + */ + +#include "jfs_btree.h" + +typedef union { + struct { + tid_t tid; + struct inode *ip; + u32 ino; + } leaf; + pxd_t xd; +} ddata_t; + + +/* + * entry segment/slot + * + * an entry consists of type dependent head/only segment/slot and + * additional segments/slots linked vi next field; + * N.B. last/only segment of entry is terminated by next = -1; + */ +/* + * directory page slot + */ +struct dtslot { + s8 next; /* 1: */ + s8 cnt; /* 1: */ + __le16 name[15]; /* 30: */ +}; /* (32) */ + + +#define DATASLOTSIZE 16 +#define L2DATASLOTSIZE 4 +#define DTSLOTSIZE 32 +#define L2DTSLOTSIZE 5 +#define DTSLOTHDRSIZE 2 +#define DTSLOTDATASIZE 30 +#define DTSLOTDATALEN 15 + +/* + * internal node entry head/only segment + */ +struct idtentry { + pxd_t xd; /* 8: child extent descriptor */ + + s8 next; /* 1: */ + u8 namlen; /* 1: */ + __le16 name[11]; /* 22: 2-byte aligned */ +}; /* (32) */ + +#define DTIHDRSIZE 10 +#define DTIHDRDATALEN 11 + +/* compute number of slots for entry */ +#define NDTINTERNAL(klen) (DIV_ROUND_UP((4 + (klen)), 15)) + + +/* + * leaf node entry head/only segment + * + * For legacy filesystems, name contains 13 wchars -- no index field + */ +struct ldtentry { + __le32 inumber; /* 4: 4-byte aligned */ + s8 next; /* 1: */ + u8 namlen; /* 1: */ + __le16 name[11]; /* 22: 2-byte aligned */ + __le32 index; /* 4: index into dir_table */ +}; /* (32) */ + +#define DTLHDRSIZE 6 +#define DTLHDRDATALEN_LEGACY 13 /* Old (OS/2) format */ +#define DTLHDRDATALEN 11 + +/* + * dir_table used for directory traversal during readdir + */ + +/* + * Keep persistent index for directory entries + */ +#define DO_INDEX(INODE) (JFS_SBI((INODE)->i_sb)->mntflag & JFS_DIR_INDEX) + +/* + * Maximum entry in inline directory table + */ +#define MAX_INLINE_DIRTABLE_ENTRY 13 + +struct dir_table_slot { + u8 rsrvd; /* 1: */ + u8 flag; /* 1: 0 if free */ + u8 slot; /* 1: slot within leaf page of entry */ + u8 addr1; /* 1: upper 8 bits of leaf page address */ + __le32 addr2; /* 4: lower 32 bits of leaf page address -OR- + index of next entry when this entry was deleted */ +}; /* (8) */ + +/* + * flag values + */ +#define DIR_INDEX_VALID 1 +#define DIR_INDEX_FREE 0 + +#define DTSaddress(dir_table_slot, address64)\ +{\ + (dir_table_slot)->addr1 = ((u64)address64) >> 32;\ + (dir_table_slot)->addr2 = __cpu_to_le32((address64) & 0xffffffff);\ +} + +#define addressDTS(dts)\ + ( ((s64)((dts)->addr1)) << 32 | __le32_to_cpu((dts)->addr2) ) + +/* compute number of slots for entry */ +#define NDTLEAF_LEGACY(klen) (DIV_ROUND_UP((2 + (klen)), 15)) +#define NDTLEAF NDTINTERNAL + + +/* + * directory root page (in-line in on-disk inode): + * + * cf. dtpage_t below. + */ +typedef union { + struct { + struct dasd DASD; /* 16: DASD limit/usage info */ + + u8 flag; /* 1: */ + u8 nextindex; /* 1: next free entry in stbl */ + s8 freecnt; /* 1: free count */ + s8 freelist; /* 1: freelist header */ + + __le32 idotdot; /* 4: parent inode number */ + + s8 stbl[8]; /* 8: sorted entry index table */ + } header; /* (32) */ + + struct dtslot slot[9]; +} dtroot_t; + +#define PARENT(IP) \ + (le32_to_cpu(JFS_IP(IP)->i_dtroot.header.idotdot)) + +#define DTROOTMAXSLOT 9 + +#define dtEmpty(IP) (JFS_IP(IP)->i_dtroot.header.nextindex == 0) + + +/* + * directory regular page: + * + * entry slot array of 32 byte slot + * + * sorted entry slot index table (stbl): + * contiguous slots at slot specified by stblindex, + * 1-byte per entry + * 512 byte block: 16 entry tbl (1 slot) + * 1024 byte block: 32 entry tbl (1 slot) + * 2048 byte block: 64 entry tbl (2 slot) + * 4096 byte block: 128 entry tbl (4 slot) + * + * data area: + * 512 byte block: 16 - 2 = 14 slot + * 1024 byte block: 32 - 2 = 30 slot + * 2048 byte block: 64 - 3 = 61 slot + * 4096 byte block: 128 - 5 = 123 slot + * + * N.B. index is 0-based; index fields refer to slot index + * except nextindex which refers to entry index in stbl; + * end of entry stot list or freelist is marked with -1. + */ +typedef union { + struct { + __le64 next; /* 8: next sibling */ + __le64 prev; /* 8: previous sibling */ + + u8 flag; /* 1: */ + u8 nextindex; /* 1: next entry index in stbl */ + s8 freecnt; /* 1: */ + s8 freelist; /* 1: slot index of head of freelist */ + + u8 maxslot; /* 1: number of slots in page slot[] */ + u8 stblindex; /* 1: slot index of start of stbl */ + u8 rsrvd[2]; /* 2: */ + + pxd_t self; /* 8: self pxd */ + } header; /* (32) */ + + struct dtslot slot[128]; +} dtpage_t; + +#define DTPAGEMAXSLOT 128 + +#define DT8THPGNODEBYTES 512 +#define DT8THPGNODETSLOTS 1 +#define DT8THPGNODESLOTS 16 + +#define DTQTRPGNODEBYTES 1024 +#define DTQTRPGNODETSLOTS 1 +#define DTQTRPGNODESLOTS 32 + +#define DTHALFPGNODEBYTES 2048 +#define DTHALFPGNODETSLOTS 2 +#define DTHALFPGNODESLOTS 64 + +#define DTFULLPGNODEBYTES 4096 +#define DTFULLPGNODETSLOTS 4 +#define DTFULLPGNODESLOTS 128 + +#define DTENTRYSTART 1 + +/* get sorted entry table of the page */ +#define DT_GETSTBL(p) ( ((p)->header.flag & BT_ROOT) ?\ + ((dtroot_t *)(p))->header.stbl : \ + (s8 *)&(p)->slot[(p)->header.stblindex] ) + +/* + * Flags for dtSearch + */ +#define JFS_CREATE 1 +#define JFS_LOOKUP 2 +#define JFS_REMOVE 3 +#define JFS_RENAME 4 + +/* + * Maximum file offset for directories. + */ +#define DIREND INT_MAX + +/* + * external declarations + */ +extern void dtInitRoot(tid_t tid, struct inode *ip, u32 idotdot); + +extern int dtSearch(struct inode *ip, struct component_name * key, + ino_t * data, struct btstack * btstack, int flag); + +extern int dtInsert(tid_t tid, struct inode *ip, struct component_name * key, + ino_t * ino, struct btstack * btstack); + +extern int dtDelete(tid_t tid, struct inode *ip, struct component_name * key, + ino_t * data, int flag); + +extern int dtModify(tid_t tid, struct inode *ip, struct component_name * key, + ino_t * orig_ino, ino_t new_ino, int flag); + +extern int jfs_readdir(struct file *file, struct dir_context *ctx); + +extern bool check_dtroot(dtroot_t *p); + +extern bool check_dtpage(dtpage_t *p); +#endif /* !_H_JFS_DTREE */ diff --git a/fs/jfs/jfs_extent.c b/fs/jfs/jfs_extent.c new file mode 100644 index 000000000..46529bcc8 --- /dev/null +++ b/fs/jfs/jfs_extent.c @@ -0,0 +1,398 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) International Business Machines Corp., 2000-2004 + */ + +#include +#include +#include "jfs_incore.h" +#include "jfs_inode.h" +#include "jfs_superblock.h" +#include "jfs_dmap.h" +#include "jfs_extent.h" +#include "jfs_debug.h" + +/* + * forward references + */ +static int extBalloc(struct inode *, s64, s64 *, s64 *); +static s64 extRoundDown(s64 nb); + +#define DPD(a) (printk("(a): %d\n",(a))) +#define DPC(a) (printk("(a): %c\n",(a))) +#define DPL1(a) \ +{ \ + if ((a) >> 32) \ + printk("(a): %x%08x ",(a)); \ + else \ + printk("(a): %x ",(a) << 32); \ +} +#define DPL(a) \ +{ \ + if ((a) >> 32) \ + printk("(a): %x%08x\n",(a)); \ + else \ + printk("(a): %x\n",(a) << 32); \ +} + +#define DPD1(a) (printk("(a): %d ",(a))) +#define DPX(a) (printk("(a): %08x\n",(a))) +#define DPX1(a) (printk("(a): %08x ",(a))) +#define DPS(a) (printk("%s\n",(a))) +#define DPE(a) (printk("\nENTERING: %s\n",(a))) +#define DPE1(a) (printk("\nENTERING: %s",(a))) +#define DPS1(a) (printk(" %s ",(a))) + + +/* + * NAME: extAlloc() + * + * FUNCTION: allocate an extent for a specified page range within a + * file. + * + * PARAMETERS: + * ip - the inode of the file. + * xlen - requested extent length. + * pno - the starting page number with the file. + * xp - pointer to an xad. on entry, xad describes an + * extent that is used as an allocation hint if the + * xaddr of the xad is non-zero. on successful exit, + * the xad describes the newly allocated extent. + * abnr - bool indicating whether the newly allocated extent + * should be marked as allocated but not recorded. + * + * RETURN VALUES: + * 0 - success + * -EIO - i/o error. + * -ENOSPC - insufficient disk resources. + */ +int +extAlloc(struct inode *ip, s64 xlen, s64 pno, xad_t * xp, bool abnr) +{ + struct jfs_sb_info *sbi = JFS_SBI(ip->i_sb); + s64 nxlen, nxaddr, xoff, hint, xaddr = 0; + int rc; + int xflag; + + if (isReadOnly(ip)) { + jfs_error(ip->i_sb, "read-only filesystem\n"); + return -EIO; + } + + /* This blocks if we are low on resources */ + txBeginAnon(ip->i_sb); + + /* Avoid race with jfs_commit_inode() */ + mutex_lock(&JFS_IP(ip)->commit_mutex); + + /* validate extent length */ + if (xlen > MAXXLEN) + xlen = MAXXLEN; + + /* get the page's starting extent offset */ + xoff = pno << sbi->l2nbperpage; + + /* check if an allocation hint was provided */ + if ((hint = addressXAD(xp))) { + /* get the size of the extent described by the hint */ + nxlen = lengthXAD(xp); + + /* check if the hint is for the portion of the file + * immediately previous to the current allocation + * request and if hint extent has the same abnr + * value as the current request. if so, we can + * extend the hint extent to include the current + * extent if we can allocate the blocks immediately + * following the hint extent. + */ + if (offsetXAD(xp) + nxlen == xoff && + abnr == ((xp->flag & XAD_NOTRECORDED) ? true : false)) + xaddr = hint + nxlen; + + /* adjust the hint to the last block of the extent */ + hint += (nxlen - 1); + } + + /* allocate the disk blocks for the extent. initially, extBalloc() + * will try to allocate disk blocks for the requested size (xlen). + * if this fails (xlen contiguous free blocks not available), it'll + * try to allocate a smaller number of blocks (producing a smaller + * extent), with this smaller number of blocks consisting of the + * requested number of blocks rounded down to the next smaller + * power of 2 number (i.e. 16 -> 8). it'll continue to round down + * and retry the allocation until the number of blocks to allocate + * is smaller than the number of blocks per page. + */ + nxlen = xlen; + if ((rc = extBalloc(ip, hint ? hint : INOHINT(ip), &nxlen, &nxaddr))) { + mutex_unlock(&JFS_IP(ip)->commit_mutex); + return (rc); + } + + /* Allocate blocks to quota. */ + rc = dquot_alloc_block(ip, nxlen); + if (rc) { + dbFree(ip, nxaddr, (s64) nxlen); + mutex_unlock(&JFS_IP(ip)->commit_mutex); + return rc; + } + + /* determine the value of the extent flag */ + xflag = abnr ? XAD_NOTRECORDED : 0; + + /* if we can extend the hint extent to cover the current request, + * extend it. otherwise, insert a new extent to + * cover the current request. + */ + if (xaddr && xaddr == nxaddr) + rc = xtExtend(0, ip, xoff, (int) nxlen, 0); + else + rc = xtInsert(0, ip, xflag, xoff, (int) nxlen, &nxaddr, 0); + + /* if the extend or insert failed, + * free the newly allocated blocks and return the error. + */ + if (rc) { + dbFree(ip, nxaddr, nxlen); + dquot_free_block(ip, nxlen); + mutex_unlock(&JFS_IP(ip)->commit_mutex); + return (rc); + } + + /* set the results of the extent allocation */ + XADaddress(xp, nxaddr); + XADlength(xp, nxlen); + XADoffset(xp, xoff); + xp->flag = xflag; + + mark_inode_dirty(ip); + + mutex_unlock(&JFS_IP(ip)->commit_mutex); + /* + * COMMIT_SyncList flags an anonymous tlock on page that is on + * sync list. + * We need to commit the inode to get the page written to the disk. + */ + if (test_and_clear_cflag(COMMIT_Synclist,ip)) + jfs_commit_inode(ip, 0); + + return (0); +} + +/* + * NAME: extHint() + * + * FUNCTION: produce an extent allocation hint for a file offset. + * + * PARAMETERS: + * ip - the inode of the file. + * offset - file offset for which the hint is needed. + * xp - pointer to the xad that is to be filled in with + * the hint. + * + * RETURN VALUES: + * 0 - success + * -EIO - i/o error. + */ +int extHint(struct inode *ip, s64 offset, xad_t * xp) +{ + struct super_block *sb = ip->i_sb; + int nbperpage = JFS_SBI(sb)->nbperpage; + s64 prev; + int rc = 0; + s64 xaddr; + int xlen; + int xflag; + + /* init the hint as "no hint provided" */ + XADaddress(xp, 0); + + /* determine the starting extent offset of the page previous + * to the page containing the offset. + */ + prev = ((offset & ~POFFSET) >> JFS_SBI(sb)->l2bsize) - nbperpage; + + /* if the offset is in the first page of the file, no hint provided. + */ + if (prev < 0) + goto out; + + rc = xtLookup(ip, prev, nbperpage, &xflag, &xaddr, &xlen, 0); + + if ((rc == 0) && xlen) { + if (xlen != nbperpage) { + jfs_error(ip->i_sb, "corrupt xtree\n"); + rc = -EIO; + } + XADaddress(xp, xaddr); + XADlength(xp, xlen); + XADoffset(xp, prev); + /* + * only preserve the abnr flag within the xad flags + * of the returned hint. + */ + xp->flag = xflag & XAD_NOTRECORDED; + } else + rc = 0; + +out: + return (rc); +} + + +/* + * NAME: extRecord() + * + * FUNCTION: change a page with a file from not recorded to recorded. + * + * PARAMETERS: + * ip - inode of the file. + * cp - cbuf of the file page. + * + * RETURN VALUES: + * 0 - success + * -EIO - i/o error. + * -ENOSPC - insufficient disk resources. + */ +int extRecord(struct inode *ip, xad_t * xp) +{ + int rc; + + if (isReadOnly(ip)) { + jfs_error(ip->i_sb, "read-only filesystem\n"); + return -EIO; + } + + txBeginAnon(ip->i_sb); + + mutex_lock(&JFS_IP(ip)->commit_mutex); + + /* update the extent */ + rc = xtUpdate(0, ip, xp); + + mutex_unlock(&JFS_IP(ip)->commit_mutex); + return rc; +} + +/* + * NAME: extBalloc() + * + * FUNCTION: allocate disk blocks to form an extent. + * + * initially, we will try to allocate disk blocks for the + * requested size (nblocks). if this fails (nblocks + * contiguous free blocks not available), we'll try to allocate + * a smaller number of blocks (producing a smaller extent), with + * this smaller number of blocks consisting of the requested + * number of blocks rounded down to the next smaller power of 2 + * number (i.e. 16 -> 8). we'll continue to round down and + * retry the allocation until the number of blocks to allocate + * is smaller than the number of blocks per page. + * + * PARAMETERS: + * ip - the inode of the file. + * hint - disk block number to be used as an allocation hint. + * *nblocks - pointer to an s64 value. on entry, this value specifies + * the desired number of block to be allocated. on successful + * exit, this value is set to the number of blocks actually + * allocated. + * blkno - pointer to a block address that is filled in on successful + * return with the starting block number of the newly + * allocated block range. + * + * RETURN VALUES: + * 0 - success + * -EIO - i/o error. + * -ENOSPC - insufficient disk resources. + */ +static int +extBalloc(struct inode *ip, s64 hint, s64 * nblocks, s64 * blkno) +{ + struct jfs_inode_info *ji = JFS_IP(ip); + struct jfs_sb_info *sbi = JFS_SBI(ip->i_sb); + s64 nb, nblks, daddr, max; + int rc, nbperpage = sbi->nbperpage; + struct bmap *bmp = sbi->bmap; + int ag; + + /* get the number of blocks to initially attempt to allocate. + * we'll first try the number of blocks requested unless this + * number is greater than the maximum number of contiguous free + * blocks in the map. in that case, we'll start off with the + * maximum free. + */ + + /* give up if no space left */ + if (bmp->db_maxfreebud == -1) + return -ENOSPC; + + max = (s64) 1 << bmp->db_maxfreebud; + if (*nblocks >= max && *nblocks > nbperpage) + nb = nblks = (max > nbperpage) ? max : nbperpage; + else + nb = nblks = *nblocks; + + /* try to allocate blocks */ + while ((rc = dbAlloc(ip, hint, nb, &daddr)) != 0) { + /* if something other than an out of space error, + * stop and return this error. + */ + if (rc != -ENOSPC) + return (rc); + + /* decrease the allocation request size */ + nb = min(nblks, extRoundDown(nb)); + + /* give up if we cannot cover a page */ + if (nb < nbperpage) + return (rc); + } + + *nblocks = nb; + *blkno = daddr; + + if (S_ISREG(ip->i_mode) && (ji->fileset == FILESYSTEM_I)) { + ag = BLKTOAG(daddr, sbi); + spin_lock_irq(&ji->ag_lock); + if (ji->active_ag == -1) { + atomic_inc(&bmp->db_active[ag]); + ji->active_ag = ag; + } else if (ji->active_ag != ag) { + atomic_dec(&bmp->db_active[ji->active_ag]); + atomic_inc(&bmp->db_active[ag]); + ji->active_ag = ag; + } + spin_unlock_irq(&ji->ag_lock); + } + + return (0); +} + +/* + * NAME: extRoundDown() + * + * FUNCTION: round down a specified number of blocks to the next + * smallest power of 2 number. + * + * PARAMETERS: + * nb - the inode of the file. + * + * RETURN VALUES: + * next smallest power of 2 number. + */ +static s64 extRoundDown(s64 nb) +{ + int i; + u64 m, k; + + for (i = 0, m = (u64) 1 << 63; i < 64; i++, m >>= 1) { + if (m & nb) + break; + } + + i = 63 - i; + k = (u64) 1 << i; + k = ((k - 1) & nb) ? k : k >> 1; + + return (k); +} diff --git a/fs/jfs/jfs_extent.h b/fs/jfs/jfs_extent.h new file mode 100644 index 000000000..a0ee4ccea --- /dev/null +++ b/fs/jfs/jfs_extent.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) International Business Machines Corp., 2000-2001 + */ +#ifndef _H_JFS_EXTENT +#define _H_JFS_EXTENT + +/* get block allocation hint as location of disk inode */ +#define INOHINT(ip) \ + (addressPXD(&(JFS_IP(ip)->ixpxd)) + lengthPXD(&(JFS_IP(ip)->ixpxd)) - 1) + +extern int extAlloc(struct inode *, s64, s64, xad_t *, bool); +extern int extHint(struct inode *, s64, xad_t *); +extern int extRecord(struct inode *, xad_t *); + +#endif /* _H_JFS_EXTENT */ diff --git a/fs/jfs/jfs_filsys.h b/fs/jfs/jfs_filsys.h new file mode 100644 index 000000000..8794281f8 --- /dev/null +++ b/fs/jfs/jfs_filsys.h @@ -0,0 +1,276 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) International Business Machines Corp., 2000-2003 + */ +#ifndef _H_JFS_FILSYS +#define _H_JFS_FILSYS + +/* + * jfs_filsys.h + * + * file system (implementation-dependent) constants + * + * refer to for system wide implementation-dependent constants + */ + +/* + * file system option (superblock flag) + */ + +/* directory option */ +#define JFS_UNICODE 0x00000001 /* unicode name */ + +/* mount time flags for error handling */ +#define JFS_ERR_REMOUNT_RO 0x00000002 /* remount read-only */ +#define JFS_ERR_CONTINUE 0x00000004 /* continue */ +#define JFS_ERR_PANIC 0x00000008 /* panic */ +#define JFS_ERR_MASK (JFS_ERR_REMOUNT_RO|JFS_ERR_CONTINUE|JFS_ERR_PANIC) + +/* Quota support */ +#define JFS_USRQUOTA 0x00000010 +#define JFS_GRPQUOTA 0x00000020 + +/* mount time flag to disable journaling to disk */ +#define JFS_NOINTEGRITY 0x00000040 + +/* mount time flag to enable TRIM to ssd disks */ +#define JFS_DISCARD 0x00000080 + +/* commit option */ +#define JFS_COMMIT 0x00000f00 /* commit option mask */ +#define JFS_GROUPCOMMIT 0x00000100 /* group (of 1) commit */ +#define JFS_LAZYCOMMIT 0x00000200 /* lazy commit */ +#define JFS_TMPFS 0x00000400 /* temporary file system - + * do not log/commit: + * Never implemented + */ + +/* log logical volume option */ +#define JFS_INLINELOG 0x00000800 /* inline log within file system */ +#define JFS_INLINEMOVE 0x00001000 /* inline log being moved */ + +/* Secondary aggregate inode table */ +#define JFS_BAD_SAIT 0x00010000 /* current secondary ait is bad */ + +/* sparse regular file support */ +#define JFS_SPARSE 0x00020000 /* sparse regular file */ + +/* DASD Limits F226941 */ +#define JFS_DASD_ENABLED 0x00040000 /* DASD limits enabled */ +#define JFS_DASD_PRIME 0x00080000 /* Prime DASD usage on boot */ + +/* big endian flag */ +#define JFS_SWAP_BYTES 0x00100000 /* running on big endian computer */ + +/* Directory index */ +#define JFS_DIR_INDEX 0x00200000 /* Persistent index for */ + +/* platform options */ +#define JFS_LINUX 0x10000000 /* Linux support */ +#define JFS_DFS 0x20000000 /* DCE DFS LFS support */ +/* Never implemented */ + +#define JFS_OS2 0x40000000 /* OS/2 support */ +/* case-insensitive name/directory support */ + +#define JFS_AIX 0x80000000 /* AIX support */ + +/* + * buffer cache configuration + */ +/* page size */ +#ifdef PSIZE +#undef PSIZE +#endif +#define PSIZE 4096 /* page size (in byte) */ +#define L2PSIZE 12 /* log2(PSIZE) */ +#define POFFSET 4095 /* offset within page */ + +/* buffer page size */ +#define BPSIZE PSIZE + +/* + * fs fundamental size + * + * PSIZE >= file system block size >= PBSIZE >= DISIZE + */ +#define PBSIZE 512 /* physical block size (in byte) */ +#define L2PBSIZE 9 /* log2(PBSIZE) */ + +#define DISIZE 512 /* on-disk inode size (in byte) */ +#define L2DISIZE 9 /* log2(DISIZE) */ + +#define IDATASIZE 256 /* inode inline data size */ +#define IXATTRSIZE 128 /* inode inline extended attribute size */ + +#define XTPAGE_SIZE 4096 +#define log2_PAGESIZE 12 + +#define IAG_SIZE 4096 +#define IAG_EXTENT_SIZE 4096 +#define INOSPERIAG 4096 /* number of disk inodes per iag */ +#define L2INOSPERIAG 12 /* l2 number of disk inodes per iag */ +#define INOSPEREXT 32 /* number of disk inode per extent */ +#define L2INOSPEREXT 5 /* l2 number of disk inode per extent */ +#define IXSIZE (DISIZE * INOSPEREXT) /* inode extent size */ +#define INOSPERPAGE 8 /* number of disk inodes per 4K page */ +#define L2INOSPERPAGE 3 /* log2(INOSPERPAGE) */ + +#define IAGFREELIST_LWM 64 + +#define INODE_EXTENT_SIZE IXSIZE /* inode extent size */ +#define NUM_INODE_PER_EXTENT INOSPEREXT +#define NUM_INODE_PER_IAG INOSPERIAG + +#define MINBLOCKSIZE 512 +#define L2MINBLOCKSIZE 9 +#define MAXBLOCKSIZE 4096 +#define L2MAXBLOCKSIZE 12 +#define MAXFILESIZE ((s64)1 << 52) + +#define JFS_LINK_MAX 0xffffffff + +/* Minimum number of bytes supported for a JFS partition */ +#define MINJFS (0x1000000) +#define MINJFSTEXT "16" + +/* + * file system block size -> physical block size + */ +#define LBOFFSET(x) ((x) & (PBSIZE - 1)) +#define LBNUMBER(x) ((x) >> L2PBSIZE) +#define LBLK2PBLK(sb,b) ((b) << (sb->s_blocksize_bits - L2PBSIZE)) +#define PBLK2LBLK(sb,b) ((b) >> (sb->s_blocksize_bits - L2PBSIZE)) +/* size in byte -> last page number */ +#define SIZE2PN(size) ( ((s64)((size) - 1)) >> (L2PSIZE) ) +/* size in byte -> last file system block number */ +#define SIZE2BN(size, l2bsize) ( ((s64)((size) - 1)) >> (l2bsize) ) + +/* + * fixed physical block address (physical block size = 512 byte) + * + * NOTE: since we can't guarantee a physical block size of 512 bytes the use of + * these macros should be removed and the byte offset macros used instead. + */ +#define SUPER1_B 64 /* primary superblock */ +#define AIMAP_B (SUPER1_B + 8) /* 1st extent of aggregate inode map */ +#define AITBL_B (AIMAP_B + 16) /* + * 1st extent of aggregate inode table + */ +#define SUPER2_B (AITBL_B + 32) /* 2ndary superblock pbn */ +#define BMAP_B (SUPER2_B + 8) /* block allocation map */ + +/* + * SIZE_OF_SUPER defines the total amount of space reserved on disk for the + * superblock. This is not the same as the superblock structure, since all of + * this space is not currently being used. + */ +#define SIZE_OF_SUPER PSIZE + +/* + * SIZE_OF_AG_TABLE defines the amount of space reserved to hold the AG table + */ +#define SIZE_OF_AG_TABLE PSIZE + +/* + * SIZE_OF_MAP_PAGE defines the amount of disk space reserved for each page of + * the inode allocation map (to hold iag) + */ +#define SIZE_OF_MAP_PAGE PSIZE + +/* + * fixed byte offset address + */ +#define SUPER1_OFF 0x8000 /* primary superblock */ +#define AIMAP_OFF (SUPER1_OFF + SIZE_OF_SUPER) + /* + * Control page of aggregate inode map + * followed by 1st extent of map + */ +#define AITBL_OFF (AIMAP_OFF + (SIZE_OF_MAP_PAGE << 1)) + /* + * 1st extent of aggregate inode table + */ +#define SUPER2_OFF (AITBL_OFF + INODE_EXTENT_SIZE) + /* + * secondary superblock + */ +#define BMAP_OFF (SUPER2_OFF + SIZE_OF_SUPER) + /* + * block allocation map + */ + +/* + * The following macro is used to indicate the number of reserved disk blocks at + * the front of an aggregate, in terms of physical blocks. This value is + * currently defined to be 32K. This turns out to be the same as the primary + * superblock's address, since it directly follows the reserved blocks. + */ +#define AGGR_RSVD_BLOCKS SUPER1_B + +/* + * The following macro is used to indicate the number of reserved bytes at the + * front of an aggregate. This value is currently defined to be 32K. This + * turns out to be the same as the primary superblock's byte offset, since it + * directly follows the reserved blocks. + */ +#define AGGR_RSVD_BYTES SUPER1_OFF + +/* + * The following macro defines the byte offset for the first inode extent in + * the aggregate inode table. This allows us to find the self inode to find the + * rest of the table. Currently this value is 44K. + */ +#define AGGR_INODE_TABLE_START AITBL_OFF + +/* + * fixed reserved inode number + */ +/* aggregate inode */ +#define AGGR_RESERVED_I 0 /* aggregate inode (reserved) */ +#define AGGREGATE_I 1 /* aggregate inode map inode */ +#define BMAP_I 2 /* aggregate block allocation map inode */ +#define LOG_I 3 /* aggregate inline log inode */ +#define BADBLOCK_I 4 /* aggregate bad block inode */ +#define FILESYSTEM_I 16 /* 1st/only fileset inode in ait: + * fileset inode map inode + */ + +/* per fileset inode */ +#define FILESET_RSVD_I 0 /* fileset inode (reserved) */ +#define FILESET_EXT_I 1 /* fileset inode extension */ +#define ROOT_I 2 /* fileset root inode */ +#define ACL_I 3 /* fileset ACL inode */ + +#define FILESET_OBJECT_I 4 /* the first fileset inode available for a file + * or directory or link... + */ +#define FIRST_FILESET_INO 16 /* the first aggregate inode which describes + * an inode. (To fsck this is also the first + * inode in part 2 of the agg inode table.) + */ + +/* + * directory configuration + */ +#define JFS_NAME_MAX 255 +#define JFS_PATH_MAX BPSIZE + + +/* + * file system state (superblock state) + */ +#define FM_CLEAN 0x00000000 /* file system is unmounted and clean */ +#define FM_MOUNT 0x00000001 /* file system is mounted cleanly */ +#define FM_DIRTY 0x00000002 /* file system was not unmounted and clean + * when mounted or + * commit failure occurred while being mounted: + * fsck() must be run to repair + */ +#define FM_LOGREDO 0x00000004 /* log based recovery (logredo()) failed: + * fsck() must be run to repair + */ +#define FM_EXTENDFS 0x00000008 /* file system extendfs() in progress */ +#define FM_STATE_MAX 0x0000000f /* max value of s_state */ + +#endif /* _H_JFS_FILSYS */ diff --git a/fs/jfs/jfs_imap.c b/fs/jfs/jfs_imap.c new file mode 100644 index 000000000..b84ba4d7d --- /dev/null +++ b/fs/jfs/jfs_imap.c @@ -0,0 +1,3178 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) International Business Machines Corp., 2000-2004 + */ + +/* + * jfs_imap.c: inode allocation map manager + * + * Serialization: + * Each AG has a simple lock which is used to control the serialization of + * the AG level lists. This lock should be taken first whenever an AG + * level list will be modified or accessed. + * + * Each IAG is locked by obtaining the buffer for the IAG page. + * + * There is also a inode lock for the inode map inode. A read lock needs to + * be taken whenever an IAG is read from the map or the global level + * information is read. A write lock needs to be taken whenever the global + * level information is modified or an atomic operation needs to be used. + * + * If more than one IAG is read at one time, the read lock may not + * be given up until all of the IAG's are read. Otherwise, a deadlock + * may occur when trying to obtain the read lock while another thread + * holding the read lock is waiting on the IAG already being held. + * + * The control page of the inode map is read into memory by diMount(). + * Thereafter it should only be modified in memory and then it will be + * written out when the filesystem is unmounted by diUnmount(). + */ + +#include +#include +#include +#include +#include + +#include "jfs_incore.h" +#include "jfs_inode.h" +#include "jfs_filsys.h" +#include "jfs_dinode.h" +#include "jfs_dmap.h" +#include "jfs_imap.h" +#include "jfs_metapage.h" +#include "jfs_superblock.h" +#include "jfs_debug.h" + +/* + * imap locks + */ +/* iag free list lock */ +#define IAGFREE_LOCK_INIT(imap) mutex_init(&imap->im_freelock) +#define IAGFREE_LOCK(imap) mutex_lock(&imap->im_freelock) +#define IAGFREE_UNLOCK(imap) mutex_unlock(&imap->im_freelock) + +/* per ag iag list locks */ +#define AG_LOCK_INIT(imap,index) mutex_init(&(imap->im_aglock[index])) +#define AG_LOCK(imap,agno) mutex_lock(&imap->im_aglock[agno]) +#define AG_UNLOCK(imap,agno) mutex_unlock(&imap->im_aglock[agno]) + +/* + * forward references + */ +static int diAllocAG(struct inomap *, int, bool, struct inode *); +static int diAllocAny(struct inomap *, int, bool, struct inode *); +static int diAllocBit(struct inomap *, struct iag *, int); +static int diAllocExt(struct inomap *, int, struct inode *); +static int diAllocIno(struct inomap *, int, struct inode *); +static int diFindFree(u32, int); +static int diNewExt(struct inomap *, struct iag *, int); +static int diNewIAG(struct inomap *, int *, int, struct metapage **); +static void duplicateIXtree(struct super_block *, s64, int, s64 *); + +static int diIAGRead(struct inomap * imap, int, struct metapage **); +static int copy_from_dinode(struct dinode *, struct inode *); +static void copy_to_dinode(struct dinode *, struct inode *); + +/* + * NAME: diMount() + * + * FUNCTION: initialize the incore inode map control structures for + * a fileset or aggregate init time. + * + * the inode map's control structure (dinomap) is + * brought in from disk and placed in virtual memory. + * + * PARAMETERS: + * ipimap - pointer to inode map inode for the aggregate or fileset. + * + * RETURN VALUES: + * 0 - success + * -ENOMEM - insufficient free virtual memory. + * -EIO - i/o error. + */ +int diMount(struct inode *ipimap) +{ + struct inomap *imap; + struct metapage *mp; + int index; + struct dinomap_disk *dinom_le; + + /* + * allocate/initialize the in-memory inode map control structure + */ + /* allocate the in-memory inode map control structure. */ + imap = kzalloc_obj(struct inomap); + if (imap == NULL) + return -ENOMEM; + + /* read the on-disk inode map control structure. */ + + mp = read_metapage(ipimap, + IMAPBLKNO << JFS_SBI(ipimap->i_sb)->l2nbperpage, + PSIZE, 0); + if (mp == NULL) { + kfree(imap); + return -EIO; + } + + /* copy the on-disk version to the in-memory version. */ + dinom_le = (struct dinomap_disk *) mp->data; + imap->im_freeiag = le32_to_cpu(dinom_le->in_freeiag); + imap->im_nextiag = le32_to_cpu(dinom_le->in_nextiag); + atomic_set(&imap->im_numinos, le32_to_cpu(dinom_le->in_numinos)); + atomic_set(&imap->im_numfree, le32_to_cpu(dinom_le->in_numfree)); + imap->im_nbperiext = le32_to_cpu(dinom_le->in_nbperiext); + imap->im_l2nbperiext = le32_to_cpu(dinom_le->in_l2nbperiext); + for (index = 0; index < MAXAG; index++) { + imap->im_agctl[index].inofree = + le32_to_cpu(dinom_le->in_agctl[index].inofree); + imap->im_agctl[index].extfree = + le32_to_cpu(dinom_le->in_agctl[index].extfree); + imap->im_agctl[index].numinos = + le32_to_cpu(dinom_le->in_agctl[index].numinos); + imap->im_agctl[index].numfree = + le32_to_cpu(dinom_le->in_agctl[index].numfree); + } + + /* release the buffer. */ + release_metapage(mp); + + /* + * allocate/initialize inode allocation map locks + */ + /* allocate and init iag free list lock */ + IAGFREE_LOCK_INIT(imap); + + /* allocate and init ag list locks */ + for (index = 0; index < MAXAG; index++) { + AG_LOCK_INIT(imap, index); + } + + /* bind the inode map inode and inode map control structure + * to each other. + */ + imap->im_ipimap = ipimap; + JFS_IP(ipimap)->i_imap = imap; + + return (0); +} + + +/* + * NAME: diUnmount() + * + * FUNCTION: write to disk the incore inode map control structures for + * a fileset or aggregate at unmount time. + * + * PARAMETERS: + * ipimap - pointer to inode map inode for the aggregate or fileset. + * + * RETURN VALUES: + * 0 - success + * -ENOMEM - insufficient free virtual memory. + * -EIO - i/o error. + */ +int diUnmount(struct inode *ipimap, int mounterror) +{ + struct inomap *imap = JFS_IP(ipimap)->i_imap; + + /* + * update the on-disk inode map control structure + */ + + if (!(mounterror || isReadOnly(ipimap))) + diSync(ipimap); + + /* + * Invalidate the page cache buffers + */ + truncate_inode_pages(ipimap->i_mapping, 0); + + /* + * free in-memory control structure + */ + kfree(imap); + JFS_IP(ipimap)->i_imap = NULL; + + return (0); +} + + +/* + * diSync() + */ +int diSync(struct inode *ipimap) +{ + struct dinomap_disk *dinom_le; + struct inomap *imp = JFS_IP(ipimap)->i_imap; + struct metapage *mp; + int index; + + /* + * write imap global conrol page + */ + /* read the on-disk inode map control structure */ + mp = get_metapage(ipimap, + IMAPBLKNO << JFS_SBI(ipimap->i_sb)->l2nbperpage, + PSIZE, 0); + if (mp == NULL) { + jfs_err("diSync: get_metapage failed!"); + return -EIO; + } + + /* copy the in-memory version to the on-disk version */ + dinom_le = (struct dinomap_disk *) mp->data; + dinom_le->in_freeiag = cpu_to_le32(imp->im_freeiag); + dinom_le->in_nextiag = cpu_to_le32(imp->im_nextiag); + dinom_le->in_numinos = cpu_to_le32(atomic_read(&imp->im_numinos)); + dinom_le->in_numfree = cpu_to_le32(atomic_read(&imp->im_numfree)); + dinom_le->in_nbperiext = cpu_to_le32(imp->im_nbperiext); + dinom_le->in_l2nbperiext = cpu_to_le32(imp->im_l2nbperiext); + for (index = 0; index < MAXAG; index++) { + dinom_le->in_agctl[index].inofree = + cpu_to_le32(imp->im_agctl[index].inofree); + dinom_le->in_agctl[index].extfree = + cpu_to_le32(imp->im_agctl[index].extfree); + dinom_le->in_agctl[index].numinos = + cpu_to_le32(imp->im_agctl[index].numinos); + dinom_le->in_agctl[index].numfree = + cpu_to_le32(imp->im_agctl[index].numfree); + } + + /* write out the control structure */ + write_metapage(mp); + + /* + * write out dirty pages of imap + */ + filemap_write_and_wait(ipimap->i_mapping); + + diWriteSpecial(ipimap, 0); + + return (0); +} + + +/* + * NAME: diRead() + * + * FUNCTION: initialize an incore inode from disk. + * + * on entry, the specifed incore inode should itself + * specify the disk inode number corresponding to the + * incore inode (i.e. i_number should be initialized). + * + * this routine handles incore inode initialization for + * both "special" and "regular" inodes. special inodes + * are those required early in the mount process and + * require special handling since much of the file system + * is not yet initialized. these "special" inodes are + * identified by a NULL inode map inode pointer and are + * actually initialized by a call to diReadSpecial(). + * + * for regular inodes, the iag describing the disk inode + * is read from disk to determine the inode extent address + * for the disk inode. with the inode extent address in + * hand, the page of the extent that contains the disk + * inode is read and the disk inode is copied to the + * incore inode. + * + * PARAMETERS: + * ip - pointer to incore inode to be initialized from disk. + * + * RETURN VALUES: + * 0 - success + * -EIO - i/o error. + * -ENOMEM - insufficient memory + * + */ +int diRead(struct inode *ip) +{ + struct jfs_sb_info *sbi = JFS_SBI(ip->i_sb); + int iagno, ino, extno, rc, agno; + struct inode *ipimap; + struct dinode *dp; + struct iag *iagp; + struct metapage *mp; + s64 blkno, agstart; + struct inomap *imap; + int block_offset; + int inodes_left; + unsigned long pageno; + int rel_inode; + + jfs_info("diRead: ino = %llu", ip->i_ino); + + ipimap = sbi->ipimap; + JFS_IP(ip)->ipimap = ipimap; + + /* determine the iag number for this inode (number) */ + iagno = INOTOIAG(ip->i_ino); + + /* read the iag */ + IREAD_LOCK(ipimap, RDWRLOCK_IMAP); + imap = JFS_IP(ipimap)->i_imap; + rc = diIAGRead(imap, iagno, &mp); + IREAD_UNLOCK(ipimap); + if (rc) { + jfs_err("diRead: diIAGRead returned %d", rc); + return (rc); + } + + iagp = (struct iag *) mp->data; + + /* determine inode extent that holds the disk inode */ + ino = ip->i_ino & (INOSPERIAG - 1); + extno = ino >> L2INOSPEREXT; + + if ((lengthPXD(&iagp->inoext[extno]) != imap->im_nbperiext) || + (addressPXD(&iagp->inoext[extno]) == 0)) { + release_metapage(mp); + return -ESTALE; + } + + /* get disk block number of the page within the inode extent + * that holds the disk inode. + */ + blkno = INOPBLK(&iagp->inoext[extno], ino, sbi->l2nbperpage); + + /* get the ag for the iag */ + agstart = le64_to_cpu(iagp->agstart); + agno = BLKTOAG(agstart, JFS_SBI(ip->i_sb)); + + release_metapage(mp); + if (agno >= MAXAG || agno < 0) + return -EIO; + + rel_inode = (ino & (INOSPERPAGE - 1)); + pageno = blkno >> sbi->l2nbperpage; + + if ((block_offset = ((u32) blkno & (sbi->nbperpage - 1)))) { + /* + * OS/2 didn't always align inode extents on page boundaries + */ + inodes_left = + (sbi->nbperpage - block_offset) << sbi->l2niperblk; + + if (rel_inode < inodes_left) + rel_inode += block_offset << sbi->l2niperblk; + else { + pageno += 1; + rel_inode -= inodes_left; + } + } + + /* read the page of disk inode */ + mp = read_metapage(ipimap, pageno << sbi->l2nbperpage, PSIZE, 1); + if (!mp) { + jfs_err("diRead: read_metapage failed"); + return -EIO; + } + + /* locate the disk inode requested */ + dp = (struct dinode *) mp->data; + dp += rel_inode; + + if (ip->i_ino != le32_to_cpu(dp->di_number)) { + jfs_error(ip->i_sb, "i_ino != di_number\n"); + rc = -EIO; + } else if (le32_to_cpu(dp->di_nlink) == 0) + rc = -ESTALE; + else + /* copy the disk inode to the in-memory inode */ + rc = copy_from_dinode(dp, ip); + + release_metapage(mp); + + /* set the ag for the inode */ + JFS_IP(ip)->agstart = agstart; + JFS_IP(ip)->active_ag = -1; + + return (rc); +} + + +/* + * NAME: diReadSpecial() + * + * FUNCTION: initialize a 'special' inode from disk. + * + * this routines handles aggregate level inodes. The + * inode cache cannot differentiate between the + * aggregate inodes and the filesystem inodes, so we + * handle these here. We don't actually use the aggregate + * inode map, since these inodes are at a fixed location + * and in some cases the aggregate inode map isn't initialized + * yet. + * + * PARAMETERS: + * sb - filesystem superblock + * inum - aggregate inode number + * secondary - 1 if secondary aggregate inode table + * + * RETURN VALUES: + * new inode - success + * NULL - i/o error. + */ +struct inode *diReadSpecial(struct super_block *sb, ino_t inum, int secondary) +{ + struct jfs_sb_info *sbi = JFS_SBI(sb); + uint address; + struct dinode *dp; + struct inode *ip; + struct metapage *mp; + + ip = new_inode(sb); + if (ip == NULL) { + jfs_err("diReadSpecial: new_inode returned NULL!"); + return ip; + } + + if (secondary) { + address = addressPXD(&sbi->ait2) >> sbi->l2nbperpage; + JFS_IP(ip)->ipimap = sbi->ipaimap2; + } else { + address = AITBL_OFF >> L2PSIZE; + JFS_IP(ip)->ipimap = sbi->ipaimap; + } + + ASSERT(inum < INOSPEREXT); + + ip->i_ino = inum; + + address += inum >> 3; /* 8 inodes per 4K page */ + + /* read the page of fixed disk inode (AIT) in raw mode */ + mp = read_metapage(ip, address << sbi->l2nbperpage, PSIZE, 1); + if (mp == NULL) { + set_nlink(ip, 1); /* Don't want iput() deleting it */ + iput(ip); + return (NULL); + } + + /* get the pointer to the disk inode of interest */ + dp = (struct dinode *) (mp->data); + dp += inum % 8; /* 8 inodes per 4K page */ + + /* copy on-disk inode to in-memory inode */ + if ((copy_from_dinode(dp, ip) != 0) || (ip->i_nlink == 0)) { + /* handle bad return by returning NULL for ip */ + set_nlink(ip, 1); /* Don't want iput() deleting it */ + iput(ip); + /* release the page */ + release_metapage(mp); + return (NULL); + + } + + ip->i_mapping->a_ops = &jfs_metapage_aops; + mapping_set_gfp_mask(ip->i_mapping, GFP_NOFS); + + /* Allocations to metadata inodes should not affect quotas */ + ip->i_flags |= S_NOQUOTA; + + if ((inum == FILESYSTEM_I) && (JFS_IP(ip)->ipimap == sbi->ipaimap)) { + sbi->gengen = le32_to_cpu(dp->di_gengen); + sbi->inostamp = le32_to_cpu(dp->di_inostamp); + } + + /* release the page */ + release_metapage(mp); + + inode_fake_hash(ip); + + return (ip); +} + +/* + * NAME: diWriteSpecial() + * + * FUNCTION: Write the special inode to disk + * + * PARAMETERS: + * ip - special inode + * secondary - 1 if secondary aggregate inode table + * + * RETURN VALUES: none + */ + +void diWriteSpecial(struct inode *ip, int secondary) +{ + struct jfs_sb_info *sbi = JFS_SBI(ip->i_sb); + uint address; + struct dinode *dp; + ino_t inum = ip->i_ino; + struct metapage *mp; + + if (secondary) + address = addressPXD(&sbi->ait2) >> sbi->l2nbperpage; + else + address = AITBL_OFF >> L2PSIZE; + + ASSERT(inum < INOSPEREXT); + + address += inum >> 3; /* 8 inodes per 4K page */ + + /* read the page of fixed disk inode (AIT) in raw mode */ + mp = read_metapage(ip, address << sbi->l2nbperpage, PSIZE, 1); + if (mp == NULL) { + jfs_err("diWriteSpecial: failed to read aggregate inode extent!"); + return; + } + + /* get the pointer to the disk inode of interest */ + dp = (struct dinode *) (mp->data); + dp += inum % 8; /* 8 inodes per 4K page */ + + /* copy on-disk inode to in-memory inode */ + copy_to_dinode(dp, ip); + memcpy(&dp->di_xtroot, &JFS_IP(ip)->i_xtroot, 288); + + if (inum == FILESYSTEM_I) + dp->di_gengen = cpu_to_le32(sbi->gengen); + + /* write the page */ + write_metapage(mp); +} + +/* + * NAME: diFreeSpecial() + * + * FUNCTION: Free allocated space for special inode + */ +void diFreeSpecial(struct inode *ip) +{ + if (ip == NULL) { + jfs_err("diFreeSpecial called with NULL ip!"); + return; + } + filemap_write_and_wait(ip->i_mapping); + truncate_inode_pages(ip->i_mapping, 0); + iput(ip); +} + + + +/* + * NAME: diWrite() + * + * FUNCTION: write the on-disk inode portion of the in-memory inode + * to its corresponding on-disk inode. + * + * on entry, the specifed incore inode should itself + * specify the disk inode number corresponding to the + * incore inode (i.e. i_number should be initialized). + * + * the inode contains the inode extent address for the disk + * inode. with the inode extent address in hand, the + * page of the extent that contains the disk inode is + * read and the disk inode portion of the incore inode + * is copied to the disk inode. + * + * PARAMETERS: + * tid - transacation id + * ip - pointer to incore inode to be written to the inode extent. + * + * RETURN VALUES: + * 0 - success + * -EIO - i/o error. + */ +int diWrite(tid_t tid, struct inode *ip) +{ + struct jfs_sb_info *sbi = JFS_SBI(ip->i_sb); + struct jfs_inode_info *jfs_ip = JFS_IP(ip); + int rc = 0; + s32 ino; + struct dinode *dp; + s64 blkno; + int block_offset; + int inodes_left; + struct metapage *mp; + unsigned long pageno; + int rel_inode; + int dioffset; + struct inode *ipimap; + uint type; + lid_t lid; + struct tlock *ditlck, *tlck; + struct linelock *dilinelock, *ilinelock; + struct lv *lv; + int n; + + ipimap = jfs_ip->ipimap; + + ino = ip->i_ino & (INOSPERIAG - 1); + + if (!addressPXD(&(jfs_ip->ixpxd)) || + (lengthPXD(&(jfs_ip->ixpxd)) != + JFS_IP(ipimap)->i_imap->im_nbperiext)) { + jfs_error(ip->i_sb, "ixpxd invalid\n"); + return -EIO; + } + + /* + * read the page of disk inode containing the specified inode: + */ + /* compute the block address of the page */ + blkno = INOPBLK(&(jfs_ip->ixpxd), ino, sbi->l2nbperpage); + + rel_inode = (ino & (INOSPERPAGE - 1)); + pageno = blkno >> sbi->l2nbperpage; + + if ((block_offset = ((u32) blkno & (sbi->nbperpage - 1)))) { + /* + * OS/2 didn't always align inode extents on page boundaries + */ + inodes_left = + (sbi->nbperpage - block_offset) << sbi->l2niperblk; + + if (rel_inode < inodes_left) + rel_inode += block_offset << sbi->l2niperblk; + else { + pageno += 1; + rel_inode -= inodes_left; + } + } + /* read the page of disk inode */ + retry: + mp = read_metapage(ipimap, pageno << sbi->l2nbperpage, PSIZE, 1); + if (!mp) + return -EIO; + + /* get the pointer to the disk inode */ + dp = (struct dinode *) mp->data; + dp += rel_inode; + + dioffset = (ino & (INOSPERPAGE - 1)) << L2DISIZE; + + /* + * acquire transaction lock on the on-disk inode; + * N.B. tlock is acquired on ipimap not ip; + */ + if ((ditlck = + txLock(tid, ipimap, mp, tlckINODE | tlckENTRY)) == NULL) + goto retry; + dilinelock = (struct linelock *) & ditlck->lock; + + /* + * copy btree root from in-memory inode to on-disk inode + * + * (tlock is taken from inline B+-tree root in in-memory + * inode when the B+-tree root is updated, which is pointed + * by jfs_ip->blid as well as being on tx tlock list) + * + * further processing of btree root is based on the copy + * in in-memory inode, where txLog() will log from, and, + * for xtree root, txUpdateMap() will update map and reset + * XAD_NEW bit; + */ + + if (S_ISDIR(ip->i_mode) && (lid = jfs_ip->xtlid)) { + /* + * This is the special xtree inside the directory for storing + * the directory table + */ + xtroot_t *p, *xp; + xad_t *xad; + + jfs_ip->xtlid = 0; + tlck = lid_to_tlock(lid); + assert(tlck->type & tlckXTREE); + tlck->type |= tlckBTROOT; + tlck->mp = mp; + ilinelock = (struct linelock *) & tlck->lock; + + /* + * copy xtree root from inode to dinode: + */ + p = &jfs_ip->i_xtroot; + xp = (xtroot_t *) &dp->di_dirtable; + lv = ilinelock->lv; + for (n = 0; n < ilinelock->index; n++, lv++) { + memcpy(&xp->xad[lv->offset], &p->xad[lv->offset], + lv->length << L2XTSLOTSIZE); + } + + /* reset on-disk (metadata page) xtree XAD_NEW bit */ + xad = &xp->xad[XTENTRYSTART]; + for (n = XTENTRYSTART; + n < le16_to_cpu(xp->header.nextindex); n++, xad++) + if (xad->flag & (XAD_NEW | XAD_EXTENDED)) + xad->flag &= ~(XAD_NEW | XAD_EXTENDED); + } + + if ((lid = jfs_ip->blid) == 0) + goto inlineData; + jfs_ip->blid = 0; + + tlck = lid_to_tlock(lid); + type = tlck->type; + tlck->type |= tlckBTROOT; + tlck->mp = mp; + ilinelock = (struct linelock *) & tlck->lock; + + /* + * regular file: 16 byte (XAD slot) granularity + */ + if (type & tlckXTREE) { + xtroot_t *p, *xp; + xad_t *xad; + + /* + * copy xtree root from inode to dinode: + */ + p = &jfs_ip->i_xtroot; + xp = &dp->di_xtroot; + lv = ilinelock->lv; + for (n = 0; n < ilinelock->index; n++, lv++) { + memcpy(&xp->xad[lv->offset], &p->xad[lv->offset], + lv->length << L2XTSLOTSIZE); + } + + /* reset on-disk (metadata page) xtree XAD_NEW bit */ + xad = &xp->xad[XTENTRYSTART]; + for (n = XTENTRYSTART; + n < le16_to_cpu(xp->header.nextindex); n++, xad++) + if (xad->flag & (XAD_NEW | XAD_EXTENDED)) + xad->flag &= ~(XAD_NEW | XAD_EXTENDED); + } + /* + * directory: 32 byte (directory entry slot) granularity + */ + else if (type & tlckDTREE) { + dtpage_t *p, *xp; + + /* + * copy dtree root from inode to dinode: + */ + p = (dtpage_t *) &jfs_ip->i_dtroot; + xp = (dtpage_t *) & dp->di_dtroot; + lv = ilinelock->lv; + for (n = 0; n < ilinelock->index; n++, lv++) { + memcpy(&xp->slot[lv->offset], &p->slot[lv->offset], + lv->length << L2DTSLOTSIZE); + } + } else { + jfs_err("diWrite: UFO tlock"); + } + + inlineData: + /* + * copy inline symlink from in-memory inode to on-disk inode + */ + if (S_ISLNK(ip->i_mode) && ip->i_size < IDATASIZE) { + lv = & dilinelock->lv[dilinelock->index]; + lv->offset = (dioffset + 2 * 128) >> L2INODESLOTSIZE; + lv->length = 2; + memcpy(&dp->di_inline_all, jfs_ip->i_inline_all, IDATASIZE); + dilinelock->index++; + } + /* + * copy inline data from in-memory inode to on-disk inode: + * 128 byte slot granularity + */ + if (test_cflag(COMMIT_Inlineea, ip)) { + lv = & dilinelock->lv[dilinelock->index]; + lv->offset = (dioffset + 3 * 128) >> L2INODESLOTSIZE; + lv->length = 1; + memcpy(&dp->di_inlineea, jfs_ip->i_inline_ea, INODESLOTSIZE); + dilinelock->index++; + + clear_cflag(COMMIT_Inlineea, ip); + } + + /* + * lock/copy inode base: 128 byte slot granularity + */ + lv = & dilinelock->lv[dilinelock->index]; + lv->offset = dioffset >> L2INODESLOTSIZE; + copy_to_dinode(dp, ip); + if (test_and_clear_cflag(COMMIT_Dirtable, ip)) { + lv->length = 2; + memcpy(&dp->di_dirtable, &jfs_ip->i_dirtable, 96); + } else + lv->length = 1; + dilinelock->index++; + + /* release the buffer holding the updated on-disk inode. + * the buffer will be later written by commit processing. + */ + write_metapage(mp); + + return (rc); +} + + +/* + * NAME: diFree(ip) + * + * FUNCTION: free a specified inode from the inode working map + * for a fileset or aggregate. + * + * if the inode to be freed represents the first (only) + * free inode within the iag, the iag will be placed on + * the ag free inode list. + * + * freeing the inode will cause the inode extent to be + * freed if the inode is the only allocated inode within + * the extent. in this case all the disk resource backing + * up the inode extent will be freed. in addition, the iag + * will be placed on the ag extent free list if the extent + * is the first free extent in the iag. if freeing the + * extent also means that no free inodes will exist for + * the iag, the iag will also be removed from the ag free + * inode list. + * + * the iag describing the inode will be freed if the extent + * is to be freed and it is the only backed extent within + * the iag. in this case, the iag will be removed from the + * ag free extent list and ag free inode list and placed on + * the inode map's free iag list. + * + * a careful update approach is used to provide consistency + * in the face of updates to multiple buffers. under this + * approach, all required buffers are obtained before making + * any updates and are held until all updates are complete. + * + * PARAMETERS: + * ip - inode to be freed. + * + * RETURN VALUES: + * 0 - success + * -EIO - i/o error. + */ +int diFree(struct inode *ip) +{ + int rc; + ino_t inum = ip->i_ino; + struct iag *iagp, *aiagp, *biagp, *ciagp, *diagp; + struct metapage *mp, *amp, *bmp, *cmp, *dmp; + int iagno, ino, extno, bitno, sword, agno; + int back, fwd; + u32 bitmap, mask; + struct inode *ipimap = JFS_SBI(ip->i_sb)->ipimap; + struct inomap *imap = JFS_IP(ipimap)->i_imap; + pxd_t freepxd; + tid_t tid; + struct inode *iplist[3]; + struct tlock *tlck; + struct pxd_lock *pxdlock; + + /* + * This is just to suppress compiler warnings. The same logic that + * references these variables is used to initialize them. + */ + aiagp = biagp = ciagp = diagp = NULL; + + /* get the iag number containing the inode. + */ + iagno = INOTOIAG(inum); + + /* make sure that the iag is contained within + * the map. + */ + if (iagno >= imap->im_nextiag) { + print_hex_dump(KERN_ERR, "imap: ", DUMP_PREFIX_ADDRESS, 16, 4, + imap, 32, 0); + jfs_error(ip->i_sb, "inum = %d, iagno = %d, nextiag = %d\n", + (uint) inum, iagno, imap->im_nextiag); + return -EIO; + } + + /* get the allocation group for this ino. + */ + agno = BLKTOAG(JFS_IP(ip)->agstart, JFS_SBI(ip->i_sb)); + + /* Lock the AG specific inode map information + */ + AG_LOCK(imap, agno); + + /* Obtain read lock in imap inode. Don't release it until we have + * read all of the IAG's that we are going to. + */ + IREAD_LOCK(ipimap, RDWRLOCK_IMAP); + + /* read the iag. + */ + if ((rc = diIAGRead(imap, iagno, &mp))) { + IREAD_UNLOCK(ipimap); + AG_UNLOCK(imap, agno); + return (rc); + } + iagp = (struct iag *) mp->data; + + /* get the inode number and extent number of the inode within + * the iag and the inode number within the extent. + */ + ino = inum & (INOSPERIAG - 1); + extno = ino >> L2INOSPEREXT; + bitno = ino & (INOSPEREXT - 1); + mask = HIGHORDER >> bitno; + + if (!(le32_to_cpu(iagp->wmap[extno]) & mask)) { + jfs_error(ip->i_sb, "wmap shows inode already free\n"); + } + + if (!addressPXD(&iagp->inoext[extno])) { + release_metapage(mp); + IREAD_UNLOCK(ipimap); + AG_UNLOCK(imap, agno); + jfs_error(ip->i_sb, "invalid inoext\n"); + return -EIO; + } + + /* compute the bitmap for the extent reflecting the freed inode. + */ + bitmap = le32_to_cpu(iagp->wmap[extno]) & ~mask; + + if (imap->im_agctl[agno].numfree > imap->im_agctl[agno].numinos) { + release_metapage(mp); + IREAD_UNLOCK(ipimap); + AG_UNLOCK(imap, agno); + jfs_error(ip->i_sb, "numfree > numinos\n"); + return -EIO; + } + /* + * inode extent still has some inodes or below low water mark: + * keep the inode extent; + */ + if (bitmap || + imap->im_agctl[agno].numfree < 96 || + (imap->im_agctl[agno].numfree < 288 && + (((imap->im_agctl[agno].numfree * 100) / + imap->im_agctl[agno].numinos) <= 25))) { + /* if the iag currently has no free inodes (i.e., + * the inode being freed is the first free inode of iag), + * insert the iag at head of the inode free list for the ag. + */ + if (iagp->nfreeinos == 0) { + /* check if there are any iags on the ag inode + * free list. if so, read the first one so that + * we can link the current iag onto the list at + * the head. + */ + if ((fwd = imap->im_agctl[agno].inofree) >= 0) { + /* read the iag that currently is the head + * of the list. + */ + if ((rc = diIAGRead(imap, fwd, &))) { + IREAD_UNLOCK(ipimap); + AG_UNLOCK(imap, agno); + release_metapage(mp); + return (rc); + } + aiagp = (struct iag *) amp->data; + + /* make current head point back to the iag. + */ + aiagp->inofreeback = cpu_to_le32(iagno); + + write_metapage(amp); + } + + /* iag points forward to current head and iag + * becomes the new head of the list. + */ + iagp->inofreefwd = + cpu_to_le32(imap->im_agctl[agno].inofree); + iagp->inofreeback = cpu_to_le32(-1); + imap->im_agctl[agno].inofree = iagno; + } + IREAD_UNLOCK(ipimap); + + /* update the free inode summary map for the extent if + * freeing the inode means the extent will now have free + * inodes (i.e., the inode being freed is the first free + * inode of extent), + */ + if (iagp->wmap[extno] == cpu_to_le32(ONES)) { + sword = extno >> L2EXTSPERSUM; + bitno = extno & (EXTSPERSUM - 1); + iagp->inosmap[sword] &= + cpu_to_le32(~(HIGHORDER >> bitno)); + } + + /* update the bitmap. + */ + iagp->wmap[extno] = cpu_to_le32(bitmap); + + /* update the free inode counts at the iag, ag and + * map level. + */ + le32_add_cpu(&iagp->nfreeinos, 1); + imap->im_agctl[agno].numfree += 1; + atomic_inc(&imap->im_numfree); + + /* release the AG inode map lock + */ + AG_UNLOCK(imap, agno); + + /* write the iag */ + write_metapage(mp); + + return (0); + } + + + /* + * inode extent has become free and above low water mark: + * free the inode extent; + */ + + /* + * prepare to update iag list(s) (careful update step 1) + */ + amp = bmp = cmp = dmp = NULL; + fwd = back = -1; + + /* check if the iag currently has no free extents. if so, + * it will be placed on the head of the ag extent free list. + */ + if (iagp->nfreeexts == 0) { + /* check if the ag extent free list has any iags. + * if so, read the iag at the head of the list now. + * this (head) iag will be updated later to reflect + * the addition of the current iag at the head of + * the list. + */ + if ((fwd = imap->im_agctl[agno].extfree) >= 0) { + if ((rc = diIAGRead(imap, fwd, &))) + goto error_out; + aiagp = (struct iag *) amp->data; + } + } else { + /* iag has free extents. check if the addition of a free + * extent will cause all extents to be free within this + * iag. if so, the iag will be removed from the ag extent + * free list and placed on the inode map's free iag list. + */ + if (iagp->nfreeexts == cpu_to_le32(EXTSPERIAG - 1)) { + /* in preparation for removing the iag from the + * ag extent free list, read the iags preceding + * and following the iag on the ag extent free + * list. + */ + if ((fwd = le32_to_cpu(iagp->extfreefwd)) >= 0) { + if ((rc = diIAGRead(imap, fwd, &))) + goto error_out; + aiagp = (struct iag *) amp->data; + } + + if ((back = le32_to_cpu(iagp->extfreeback)) >= 0) { + if ((rc = diIAGRead(imap, back, &bmp))) + goto error_out; + biagp = (struct iag *) bmp->data; + } + } + } + + /* remove the iag from the ag inode free list if freeing + * this extent cause the iag to have no free inodes. + */ + if (iagp->nfreeinos == cpu_to_le32(INOSPEREXT - 1)) { + int inofreeback = le32_to_cpu(iagp->inofreeback); + int inofreefwd = le32_to_cpu(iagp->inofreefwd); + + /* in preparation for removing the iag from the + * ag inode free list, read the iags preceding + * and following the iag on the ag inode free + * list. before reading these iags, we must make + * sure that we already don't have them in hand + * from up above, since re-reading an iag (buffer) + * we are currently holding would cause a deadlock. + */ + if (inofreefwd >= 0) { + + if (inofreefwd == fwd) + ciagp = (struct iag *) amp->data; + else if (inofreefwd == back) + ciagp = (struct iag *) bmp->data; + else { + if ((rc = + diIAGRead(imap, inofreefwd, &cmp))) + goto error_out; + ciagp = (struct iag *) cmp->data; + } + assert(ciagp != NULL); + } + + if (inofreeback >= 0) { + if (inofreeback == fwd) + diagp = (struct iag *) amp->data; + else if (inofreeback == back) + diagp = (struct iag *) bmp->data; + else { + if ((rc = + diIAGRead(imap, inofreeback, &dmp))) + goto error_out; + diagp = (struct iag *) dmp->data; + } + assert(diagp != NULL); + } + } + + IREAD_UNLOCK(ipimap); + + /* + * invalidate any page of the inode extent freed from buffer cache; + */ + freepxd = iagp->inoext[extno]; + invalidate_pxd_metapages(ip, freepxd); + + /* + * update iag list(s) (careful update step 2) + */ + /* add the iag to the ag extent free list if this is the + * first free extent for the iag. + */ + if (iagp->nfreeexts == 0) { + if (fwd >= 0) + aiagp->extfreeback = cpu_to_le32(iagno); + + iagp->extfreefwd = + cpu_to_le32(imap->im_agctl[agno].extfree); + iagp->extfreeback = cpu_to_le32(-1); + imap->im_agctl[agno].extfree = iagno; + } else { + /* remove the iag from the ag extent list if all extents + * are now free and place it on the inode map iag free list. + */ + if (iagp->nfreeexts == cpu_to_le32(EXTSPERIAG - 1)) { + if (fwd >= 0) + aiagp->extfreeback = iagp->extfreeback; + + if (back >= 0) + biagp->extfreefwd = iagp->extfreefwd; + else + imap->im_agctl[agno].extfree = + le32_to_cpu(iagp->extfreefwd); + + iagp->extfreefwd = iagp->extfreeback = cpu_to_le32(-1); + + IAGFREE_LOCK(imap); + iagp->iagfree = cpu_to_le32(imap->im_freeiag); + imap->im_freeiag = iagno; + IAGFREE_UNLOCK(imap); + } + } + + /* remove the iag from the ag inode free list if freeing + * this extent causes the iag to have no free inodes. + */ + if (iagp->nfreeinos == cpu_to_le32(INOSPEREXT - 1)) { + if ((int) le32_to_cpu(iagp->inofreefwd) >= 0) + ciagp->inofreeback = iagp->inofreeback; + + if ((int) le32_to_cpu(iagp->inofreeback) >= 0) + diagp->inofreefwd = iagp->inofreefwd; + else + imap->im_agctl[agno].inofree = + le32_to_cpu(iagp->inofreefwd); + + iagp->inofreefwd = iagp->inofreeback = cpu_to_le32(-1); + } + + /* update the inode extent address and working map + * to reflect the free extent. + * the permanent map should have been updated already + * for the inode being freed. + */ + if (iagp->pmap[extno] != 0) { + jfs_error(ip->i_sb, "the pmap does not show inode free\n"); + } + iagp->wmap[extno] = 0; + PXDlength(&iagp->inoext[extno], 0); + PXDaddress(&iagp->inoext[extno], 0); + + /* update the free extent and free inode summary maps + * to reflect the freed extent. + * the inode summary map is marked to indicate no inodes + * available for the freed extent. + */ + sword = extno >> L2EXTSPERSUM; + bitno = extno & (EXTSPERSUM - 1); + mask = HIGHORDER >> bitno; + iagp->inosmap[sword] |= cpu_to_le32(mask); + iagp->extsmap[sword] &= cpu_to_le32(~mask); + + /* update the number of free inodes and number of free extents + * for the iag. + */ + le32_add_cpu(&iagp->nfreeinos, -(INOSPEREXT - 1)); + le32_add_cpu(&iagp->nfreeexts, 1); + + /* update the number of free inodes and backed inodes + * at the ag and inode map level. + */ + imap->im_agctl[agno].numfree -= (INOSPEREXT - 1); + imap->im_agctl[agno].numinos -= INOSPEREXT; + atomic_sub(INOSPEREXT - 1, &imap->im_numfree); + atomic_sub(INOSPEREXT, &imap->im_numinos); + + if (amp) + write_metapage(amp); + if (bmp) + write_metapage(bmp); + if (cmp) + write_metapage(cmp); + if (dmp) + write_metapage(dmp); + + /* + * start transaction to update block allocation map + * for the inode extent freed; + * + * N.B. AG_LOCK is released and iag will be released below, and + * other thread may allocate inode from/reusing the ixad freed + * BUT with new/different backing inode extent from the extent + * to be freed by the transaction; + */ + tid = txBegin(ipimap->i_sb, COMMIT_FORCE); + mutex_lock(&JFS_IP(ipimap)->commit_mutex); + + /* acquire tlock of the iag page of the freed ixad + * to force the page NOHOMEOK (even though no data is + * logged from the iag page) until NOREDOPAGE|FREEXTENT log + * for the free of the extent is committed; + * write FREEXTENT|NOREDOPAGE log record + * N.B. linelock is overlaid as freed extent descriptor; + */ + tlck = txLock(tid, ipimap, mp, tlckINODE | tlckFREE); + pxdlock = (struct pxd_lock *) & tlck->lock; + pxdlock->flag = mlckFREEPXD; + pxdlock->pxd = freepxd; + pxdlock->index = 1; + + write_metapage(mp); + + iplist[0] = ipimap; + + /* + * logredo needs the IAG number and IAG extent index in order + * to ensure that the IMap is consistent. The least disruptive + * way to pass these values through to the transaction manager + * is in the iplist array. + * + * It's not pretty, but it works. + */ + iplist[1] = (struct inode *) (size_t)iagno; + iplist[2] = (struct inode *) (size_t)extno; + + rc = txCommit(tid, 1, &iplist[0], COMMIT_FORCE); + + txEnd(tid); + mutex_unlock(&JFS_IP(ipimap)->commit_mutex); + + /* unlock the AG inode map information */ + AG_UNLOCK(imap, agno); + + return (0); + + error_out: + IREAD_UNLOCK(ipimap); + + if (amp) + release_metapage(amp); + if (bmp) + release_metapage(bmp); + if (cmp) + release_metapage(cmp); + if (dmp) + release_metapage(dmp); + + AG_UNLOCK(imap, agno); + + release_metapage(mp); + + return (rc); +} + +/* + * There are several places in the diAlloc* routines where we initialize + * the inode. + */ +static inline void +diInitInode(struct inode *ip, int iagno, int ino, int extno, struct iag * iagp) +{ + struct jfs_inode_info *jfs_ip = JFS_IP(ip); + + ip->i_ino = (iagno << L2INOSPERIAG) + ino; + jfs_ip->ixpxd = iagp->inoext[extno]; + jfs_ip->agstart = le64_to_cpu(iagp->agstart); + jfs_ip->active_ag = -1; +} + + +/* + * NAME: diAlloc(pip,dir,ip) + * + * FUNCTION: allocate a disk inode from the inode working map + * for a fileset or aggregate. + * + * PARAMETERS: + * pip - pointer to incore inode for the parent inode. + * dir - 'true' if the new disk inode is for a directory. + * ip - pointer to a new inode + * + * RETURN VALUES: + * 0 - success. + * -ENOSPC - insufficient disk resources. + * -EIO - i/o error. + */ +int diAlloc(struct inode *pip, bool dir, struct inode *ip) +{ + int rc, ino, iagno, addext, extno, bitno, sword; + int nwords, rem, i, agno, dn_numag; + u32 mask, inosmap, extsmap; + struct inode *ipimap; + struct metapage *mp; + ino_t inum; + struct iag *iagp; + struct inomap *imap; + + /* get the pointers to the inode map inode and the + * corresponding imap control structure. + */ + ipimap = JFS_SBI(pip->i_sb)->ipimap; + imap = JFS_IP(ipimap)->i_imap; + JFS_IP(ip)->ipimap = ipimap; + JFS_IP(ip)->fileset = FILESYSTEM_I; + + /* for a directory, the allocation policy is to start + * at the ag level using the preferred ag. + */ + if (dir) { + agno = dbNextAG(JFS_SBI(pip->i_sb)->ipbmap); + AG_LOCK(imap, agno); + goto tryag; + } + + /* for files, the policy starts off by trying to allocate from + * the same iag containing the parent disk inode: + * try to allocate the new disk inode close to the parent disk + * inode, using parent disk inode number + 1 as the allocation + * hint. (we use a left-to-right policy to attempt to avoid + * moving backward on the disk.) compute the hint within the + * file system and the iag. + */ + + /* get the ag number of this iag */ + agno = BLKTOAG(JFS_IP(pip)->agstart, JFS_SBI(pip->i_sb)); + dn_numag = JFS_SBI(pip->i_sb)->bmap->db_numag; + if (agno < 0 || agno > dn_numag || agno >= MAXAG) + return -EIO; + + if (atomic_read(&JFS_SBI(pip->i_sb)->bmap->db_active[agno])) { + /* + * There is an open file actively growing. We want to + * allocate new inodes from a different ag to avoid + * fragmentation problems. + */ + agno = dbNextAG(JFS_SBI(pip->i_sb)->ipbmap); + AG_LOCK(imap, agno); + goto tryag; + } + + inum = pip->i_ino + 1; + ino = inum & (INOSPERIAG - 1); + + /* back off the hint if it is outside of the iag */ + if (ino == 0) + inum = pip->i_ino; + + /* lock the AG inode map information */ + AG_LOCK(imap, agno); + + /* Get read lock on imap inode */ + IREAD_LOCK(ipimap, RDWRLOCK_IMAP); + + /* get the iag number and read the iag */ + iagno = INOTOIAG(inum); + if ((rc = diIAGRead(imap, iagno, &mp))) { + IREAD_UNLOCK(ipimap); + AG_UNLOCK(imap, agno); + return (rc); + } + iagp = (struct iag *) mp->data; + + /* determine if new inode extent is allowed to be added to the iag. + * new inode extent can be added to the iag if the ag + * has less than 32 free disk inodes and the iag has free extents. + */ + addext = (imap->im_agctl[agno].numfree < 32 && iagp->nfreeexts); + + /* + * try to allocate from the IAG + */ + /* check if the inode may be allocated from the iag + * (i.e. the inode has free inodes or new extent can be added). + */ + if (iagp->nfreeinos || addext) { + /* determine the extent number of the hint. + */ + extno = ino >> L2INOSPEREXT; + + /* check if the extent containing the hint has backed + * inodes. if so, try to allocate within this extent. + */ + if (addressPXD(&iagp->inoext[extno])) { + bitno = ino & (INOSPEREXT - 1); + if ((bitno = + diFindFree(le32_to_cpu(iagp->wmap[extno]), + bitno)) + < INOSPEREXT) { + ino = (extno << L2INOSPEREXT) + bitno; + + /* a free inode (bit) was found within this + * extent, so allocate it. + */ + rc = diAllocBit(imap, iagp, ino); + IREAD_UNLOCK(ipimap); + if (rc) { + assert(rc == -EIO); + } else { + /* set the results of the allocation + * and write the iag. + */ + diInitInode(ip, iagno, ino, extno, + iagp); + mark_metapage_dirty(mp); + } + release_metapage(mp); + + /* free the AG lock and return. + */ + AG_UNLOCK(imap, agno); + return (rc); + } + + if (!addext) + extno = + (extno == + EXTSPERIAG - 1) ? 0 : extno + 1; + } + + /* + * no free inodes within the extent containing the hint. + * + * try to allocate from the backed extents following + * hint or, if appropriate (i.e. addext is true), allocate + * an extent of free inodes at or following the extent + * containing the hint. + * + * the free inode and free extent summary maps are used + * here, so determine the starting summary map position + * and the number of words we'll have to examine. again, + * the approach is to allocate following the hint, so we + * might have to initially ignore prior bits of the summary + * map that represent extents prior to the extent containing + * the hint and later revisit these bits. + */ + bitno = extno & (EXTSPERSUM - 1); + nwords = (bitno == 0) ? SMAPSZ : SMAPSZ + 1; + sword = extno >> L2EXTSPERSUM; + + /* mask any prior bits for the starting words of the + * summary map. + */ + mask = (bitno == 0) ? 0 : (ONES << (EXTSPERSUM - bitno)); + inosmap = le32_to_cpu(iagp->inosmap[sword]) | mask; + extsmap = le32_to_cpu(iagp->extsmap[sword]) | mask; + + /* scan the free inode and free extent summary maps for + * free resources. + */ + for (i = 0; i < nwords; i++) { + /* check if this word of the free inode summary + * map describes an extent with free inodes. + */ + if (~inosmap) { + /* an extent with free inodes has been + * found. determine the extent number + * and the inode number within the extent. + */ + rem = diFindFree(inosmap, 0); + extno = (sword << L2EXTSPERSUM) + rem; + rem = diFindFree(le32_to_cpu(iagp->wmap[extno]), + 0); + if (rem >= INOSPEREXT) { + IREAD_UNLOCK(ipimap); + release_metapage(mp); + AG_UNLOCK(imap, agno); + jfs_error(ip->i_sb, + "can't find free bit in wmap\n"); + return -EIO; + } + + /* determine the inode number within the + * iag and allocate the inode from the + * map. + */ + ino = (extno << L2INOSPEREXT) + rem; + rc = diAllocBit(imap, iagp, ino); + IREAD_UNLOCK(ipimap); + if (rc) + assert(rc == -EIO); + else { + /* set the results of the allocation + * and write the iag. + */ + diInitInode(ip, iagno, ino, extno, + iagp); + mark_metapage_dirty(mp); + } + release_metapage(mp); + + /* free the AG lock and return. + */ + AG_UNLOCK(imap, agno); + return (rc); + + } + + /* check if we may allocate an extent of free + * inodes and whether this word of the free + * extents summary map describes a free extent. + */ + if (addext && ~extsmap) { + /* a free extent has been found. determine + * the extent number. + */ + rem = diFindFree(extsmap, 0); + extno = (sword << L2EXTSPERSUM) + rem; + + /* allocate an extent of free inodes. + */ + if ((rc = diNewExt(imap, iagp, extno))) { + /* if there is no disk space for a + * new extent, try to allocate the + * disk inode from somewhere else. + */ + if (rc == -ENOSPC) + break; + + assert(rc == -EIO); + } else { + /* set the results of the allocation + * and write the iag. + */ + diInitInode(ip, iagno, + extno << L2INOSPEREXT, + extno, iagp); + mark_metapage_dirty(mp); + } + release_metapage(mp); + /* free the imap inode & the AG lock & return. + */ + IREAD_UNLOCK(ipimap); + AG_UNLOCK(imap, agno); + return (rc); + } + + /* move on to the next set of summary map words. + */ + sword = (sword == SMAPSZ - 1) ? 0 : sword + 1; + inosmap = le32_to_cpu(iagp->inosmap[sword]); + extsmap = le32_to_cpu(iagp->extsmap[sword]); + } + } + /* unlock imap inode */ + IREAD_UNLOCK(ipimap); + + /* nothing doing in this iag, so release it. */ + release_metapage(mp); + + tryag: + /* + * try to allocate anywhere within the same AG as the parent inode. + */ + rc = diAllocAG(imap, agno, dir, ip); + + AG_UNLOCK(imap, agno); + + if (rc != -ENOSPC) + return (rc); + + /* + * try to allocate in any AG. + */ + return (diAllocAny(imap, agno, dir, ip)); +} + + +/* + * NAME: diAllocAG(imap,agno,dir,ip) + * + * FUNCTION: allocate a disk inode from the allocation group. + * + * this routine first determines if a new extent of free + * inodes should be added for the allocation group, with + * the current request satisfied from this extent. if this + * is the case, an attempt will be made to do just that. if + * this attempt fails or it has been determined that a new + * extent should not be added, an attempt is made to satisfy + * the request by allocating an existing (backed) free inode + * from the allocation group. + * + * PRE CONDITION: Already have the AG lock for this AG. + * + * PARAMETERS: + * imap - pointer to inode map control structure. + * agno - allocation group to allocate from. + * dir - 'true' if the new disk inode is for a directory. + * ip - pointer to the new inode to be filled in on successful return + * with the disk inode number allocated, its extent address + * and the start of the ag. + * + * RETURN VALUES: + * 0 - success. + * -ENOSPC - insufficient disk resources. + * -EIO - i/o error. + */ +static int +diAllocAG(struct inomap * imap, int agno, bool dir, struct inode *ip) +{ + int rc, addext, numfree, numinos; + + /* get the number of free and the number of backed disk + * inodes currently within the ag. + */ + numfree = imap->im_agctl[agno].numfree; + numinos = imap->im_agctl[agno].numinos; + + if (numfree > numinos) { + jfs_error(ip->i_sb, "numfree > numinos\n"); + return -EIO; + } + + /* determine if we should allocate a new extent of free inodes + * within the ag: for directory inodes, add a new extent + * if there are a small number of free inodes or number of free + * inodes is a small percentage of the number of backed inodes. + */ + if (dir) + addext = (numfree < 64 || + (numfree < 256 + && ((numfree * 100) / numinos) <= 20)); + else + addext = (numfree == 0); + + /* + * try to allocate a new extent of free inodes. + */ + if (addext) { + /* if free space is not available for this new extent, try + * below to allocate a free and existing (already backed) + * inode from the ag. + */ + if ((rc = diAllocExt(imap, agno, ip)) != -ENOSPC) + return (rc); + } + + /* + * try to allocate an existing free inode from the ag. + */ + return (diAllocIno(imap, agno, ip)); +} + + +/* + * NAME: diAllocAny(imap,agno,dir,iap) + * + * FUNCTION: allocate a disk inode from any other allocation group. + * + * this routine is called when an allocation attempt within + * the primary allocation group has failed. if attempts to + * allocate an inode from any allocation group other than the + * specified primary group. + * + * PARAMETERS: + * imap - pointer to inode map control structure. + * agno - primary allocation group (to avoid). + * dir - 'true' if the new disk inode is for a directory. + * ip - pointer to a new inode to be filled in on successful return + * with the disk inode number allocated, its extent address + * and the start of the ag. + * + * RETURN VALUES: + * 0 - success. + * -ENOSPC - insufficient disk resources. + * -EIO - i/o error. + */ +static int +diAllocAny(struct inomap * imap, int agno, bool dir, struct inode *ip) +{ + int ag, rc; + int maxag = JFS_SBI(imap->im_ipimap->i_sb)->bmap->db_maxag; + + + /* try to allocate from the ags following agno up to + * the maximum ag number. + */ + for (ag = agno + 1; ag <= maxag; ag++) { + AG_LOCK(imap, ag); + + rc = diAllocAG(imap, ag, dir, ip); + + AG_UNLOCK(imap, ag); + + if (rc != -ENOSPC) + return (rc); + } + + /* try to allocate from the ags in front of agno. + */ + for (ag = 0; ag < agno; ag++) { + AG_LOCK(imap, ag); + + rc = diAllocAG(imap, ag, dir, ip); + + AG_UNLOCK(imap, ag); + + if (rc != -ENOSPC) + return (rc); + } + + /* no free disk inodes. + */ + return -ENOSPC; +} + + +/* + * NAME: diAllocIno(imap,agno,ip) + * + * FUNCTION: allocate a disk inode from the allocation group's free + * inode list, returning an error if this free list is + * empty (i.e. no iags on the list). + * + * allocation occurs from the first iag on the list using + * the iag's free inode summary map to find the leftmost + * free inode in the iag. + * + * PRE CONDITION: Already have AG lock for this AG. + * + * PARAMETERS: + * imap - pointer to inode map control structure. + * agno - allocation group. + * ip - pointer to new inode to be filled in on successful return + * with the disk inode number allocated, its extent address + * and the start of the ag. + * + * RETURN VALUES: + * 0 - success. + * -ENOSPC - insufficient disk resources. + * -EIO - i/o error. + */ +static int diAllocIno(struct inomap * imap, int agno, struct inode *ip) +{ + int iagno, ino, rc, rem, extno, sword; + struct metapage *mp; + struct iag *iagp; + + /* check if there are iags on the ag's free inode list. + */ + if ((iagno = imap->im_agctl[agno].inofree) < 0) + return -ENOSPC; + + /* obtain read lock on imap inode */ + IREAD_LOCK(imap->im_ipimap, RDWRLOCK_IMAP); + + /* read the iag at the head of the list. + */ + if ((rc = diIAGRead(imap, iagno, &mp))) { + IREAD_UNLOCK(imap->im_ipimap); + return (rc); + } + iagp = (struct iag *) mp->data; + + /* better be free inodes in this iag if it is on the + * list. + */ + if (!iagp->nfreeinos) { + IREAD_UNLOCK(imap->im_ipimap); + release_metapage(mp); + jfs_error(ip->i_sb, "nfreeinos = 0, but iag on freelist\n"); + return -EIO; + } + + /* scan the free inode summary map to find an extent + * with free inodes. + */ + for (sword = 0;; sword++) { + if (sword >= SMAPSZ) { + IREAD_UNLOCK(imap->im_ipimap); + release_metapage(mp); + jfs_error(ip->i_sb, + "free inode not found in summary map\n"); + return -EIO; + } + + if (~iagp->inosmap[sword]) + break; + } + + /* found a extent with free inodes. determine + * the extent number. + */ + rem = diFindFree(le32_to_cpu(iagp->inosmap[sword]), 0); + if (rem >= EXTSPERSUM) { + IREAD_UNLOCK(imap->im_ipimap); + release_metapage(mp); + jfs_error(ip->i_sb, "no free extent found\n"); + return -EIO; + } + extno = (sword << L2EXTSPERSUM) + rem; + + /* find the first free inode in the extent. + */ + rem = diFindFree(le32_to_cpu(iagp->wmap[extno]), 0); + if (rem >= INOSPEREXT) { + IREAD_UNLOCK(imap->im_ipimap); + release_metapage(mp); + jfs_error(ip->i_sb, "free inode not found\n"); + return -EIO; + } + + /* compute the inode number within the iag. + */ + ino = (extno << L2INOSPEREXT) + rem; + + /* allocate the inode. + */ + rc = diAllocBit(imap, iagp, ino); + IREAD_UNLOCK(imap->im_ipimap); + if (rc) { + release_metapage(mp); + return (rc); + } + + /* set the results of the allocation and write the iag. + */ + diInitInode(ip, iagno, ino, extno, iagp); + write_metapage(mp); + + return (0); +} + + +/* + * NAME: diAllocExt(imap,agno,ip) + * + * FUNCTION: add a new extent of free inodes to an iag, allocating + * an inode from this extent to satisfy the current allocation + * request. + * + * this routine first tries to find an existing iag with free + * extents through the ag free extent list. if list is not + * empty, the head of the list will be selected as the home + * of the new extent of free inodes. otherwise (the list is + * empty), a new iag will be allocated for the ag to contain + * the extent. + * + * once an iag has been selected, the free extent summary map + * is used to locate a free extent within the iag and diNewExt() + * is called to initialize the extent, with initialization + * including the allocation of the first inode of the extent + * for the purpose of satisfying this request. + * + * PARAMETERS: + * imap - pointer to inode map control structure. + * agno - allocation group number. + * ip - pointer to new inode to be filled in on successful return + * with the disk inode number allocated, its extent address + * and the start of the ag. + * + * RETURN VALUES: + * 0 - success. + * -ENOSPC - insufficient disk resources. + * -EIO - i/o error. + */ +static int diAllocExt(struct inomap * imap, int agno, struct inode *ip) +{ + int rem, iagno, sword, extno, rc; + struct metapage *mp; + struct iag *iagp; + + /* check if the ag has any iags with free extents. if not, + * allocate a new iag for the ag. + */ + if ((iagno = imap->im_agctl[agno].extfree) < 0) { + /* If successful, diNewIAG will obtain the read lock on the + * imap inode. + */ + if ((rc = diNewIAG(imap, &iagno, agno, &mp))) { + return (rc); + } + iagp = (struct iag *) mp->data; + + /* set the ag number if this a brand new iag + */ + iagp->agstart = + cpu_to_le64(AGTOBLK(agno, imap->im_ipimap)); + } else { + /* read the iag. + */ + IREAD_LOCK(imap->im_ipimap, RDWRLOCK_IMAP); + if ((rc = diIAGRead(imap, iagno, &mp))) { + IREAD_UNLOCK(imap->im_ipimap); + jfs_error(ip->i_sb, "error reading iag\n"); + return rc; + } + iagp = (struct iag *) mp->data; + } + + /* using the free extent summary map, find a free extent. + */ + for (sword = 0;; sword++) { + if (sword >= SMAPSZ) { + release_metapage(mp); + IREAD_UNLOCK(imap->im_ipimap); + jfs_error(ip->i_sb, "free ext summary map not found\n"); + return -EIO; + } + if (~iagp->extsmap[sword]) + break; + } + + /* determine the extent number of the free extent. + */ + rem = diFindFree(le32_to_cpu(iagp->extsmap[sword]), 0); + if (rem >= EXTSPERSUM) { + release_metapage(mp); + IREAD_UNLOCK(imap->im_ipimap); + jfs_error(ip->i_sb, "free extent not found\n"); + return -EIO; + } + extno = (sword << L2EXTSPERSUM) + rem; + + /* initialize the new extent. + */ + rc = diNewExt(imap, iagp, extno); + IREAD_UNLOCK(imap->im_ipimap); + if (rc) { + /* something bad happened. if a new iag was allocated, + * place it back on the inode map's iag free list, and + * clear the ag number information. + */ + if (iagp->nfreeexts == cpu_to_le32(EXTSPERIAG)) { + IAGFREE_LOCK(imap); + iagp->iagfree = cpu_to_le32(imap->im_freeiag); + imap->im_freeiag = iagno; + IAGFREE_UNLOCK(imap); + } + write_metapage(mp); + return (rc); + } + + /* set the results of the allocation and write the iag. + */ + diInitInode(ip, iagno, extno << L2INOSPEREXT, extno, iagp); + + write_metapage(mp); + + return (0); +} + + +/* + * NAME: diAllocBit(imap,iagp,ino) + * + * FUNCTION: allocate a backed inode from an iag. + * + * this routine performs the mechanics of allocating a + * specified inode from a backed extent. + * + * if the inode to be allocated represents the last free + * inode within the iag, the iag will be removed from the + * ag free inode list. + * + * a careful update approach is used to provide consistency + * in the face of updates to multiple buffers. under this + * approach, all required buffers are obtained before making + * any updates and are held all are updates are complete. + * + * PRE CONDITION: Already have buffer lock on iagp. Already have AG lock on + * this AG. Must have read lock on imap inode. + * + * PARAMETERS: + * imap - pointer to inode map control structure. + * iagp - pointer to iag. + * ino - inode number to be allocated within the iag. + * + * RETURN VALUES: + * 0 - success. + * -ENOSPC - insufficient disk resources. + * -EIO - i/o error. + */ +static int diAllocBit(struct inomap * imap, struct iag * iagp, int ino) +{ + int extno, bitno, agno, sword, rc; + struct metapage *amp = NULL, *bmp = NULL; + struct iag *aiagp = NULL, *biagp = NULL; + u32 mask; + + /* check if this is the last free inode within the iag. + * if so, it will have to be removed from the ag free + * inode list, so get the iags preceding and following + * it on the list. + */ + if (iagp->nfreeinos == cpu_to_le32(1)) { + if ((int) le32_to_cpu(iagp->inofreefwd) >= 0) { + if ((rc = + diIAGRead(imap, le32_to_cpu(iagp->inofreefwd), + &))) + return (rc); + aiagp = (struct iag *) amp->data; + } + + if ((int) le32_to_cpu(iagp->inofreeback) >= 0) { + if ((rc = + diIAGRead(imap, + le32_to_cpu(iagp->inofreeback), + &bmp))) { + if (amp) + release_metapage(amp); + return (rc); + } + biagp = (struct iag *) bmp->data; + } + } + + /* get the ag number, extent number, inode number within + * the extent. + */ + agno = BLKTOAG(le64_to_cpu(iagp->agstart), JFS_SBI(imap->im_ipimap->i_sb)); + extno = ino >> L2INOSPEREXT; + bitno = ino & (INOSPEREXT - 1); + + /* compute the mask for setting the map. + */ + mask = HIGHORDER >> bitno; + + /* the inode should be free and backed. + */ + if (((le32_to_cpu(iagp->pmap[extno]) & mask) != 0) || + ((le32_to_cpu(iagp->wmap[extno]) & mask) != 0) || + (addressPXD(&iagp->inoext[extno]) == 0)) { + if (amp) + release_metapage(amp); + if (bmp) + release_metapage(bmp); + + jfs_error(imap->im_ipimap->i_sb, "iag inconsistent\n"); + return -EIO; + } + + /* mark the inode as allocated in the working map. + */ + iagp->wmap[extno] |= cpu_to_le32(mask); + + /* check if all inodes within the extent are now + * allocated. if so, update the free inode summary + * map to reflect this. + */ + if (iagp->wmap[extno] == cpu_to_le32(ONES)) { + sword = extno >> L2EXTSPERSUM; + bitno = extno & (EXTSPERSUM - 1); + iagp->inosmap[sword] |= cpu_to_le32(HIGHORDER >> bitno); + } + + /* if this was the last free inode in the iag, remove the + * iag from the ag free inode list. + */ + if (iagp->nfreeinos == cpu_to_le32(1)) { + if (amp) { + aiagp->inofreeback = iagp->inofreeback; + write_metapage(amp); + } + + if (bmp) { + biagp->inofreefwd = iagp->inofreefwd; + write_metapage(bmp); + } else { + imap->im_agctl[agno].inofree = + le32_to_cpu(iagp->inofreefwd); + } + iagp->inofreefwd = iagp->inofreeback = cpu_to_le32(-1); + } + + /* update the free inode count at the iag, ag, inode + * map levels. + */ + le32_add_cpu(&iagp->nfreeinos, -1); + imap->im_agctl[agno].numfree -= 1; + atomic_dec(&imap->im_numfree); + + return (0); +} + + +/* + * NAME: diNewExt(imap,iagp,extno) + * + * FUNCTION: initialize a new extent of inodes for an iag, allocating + * the first inode of the extent for use for the current + * allocation request. + * + * disk resources are allocated for the new extent of inodes + * and the inodes themselves are initialized to reflect their + * existence within the extent (i.e. their inode numbers and + * inode extent addresses are set) and their initial state + * (mode and link count are set to zero). + * + * if the iag is new, it is not yet on an ag extent free list + * but will now be placed on this list. + * + * if the allocation of the new extent causes the iag to + * have no free extent, the iag will be removed from the + * ag extent free list. + * + * if the iag has no free backed inodes, it will be placed + * on the ag free inode list, since the addition of the new + * extent will now cause it to have free inodes. + * + * a careful update approach is used to provide consistency + * (i.e. list consistency) in the face of updates to multiple + * buffers. under this approach, all required buffers are + * obtained before making any updates and are held until all + * updates are complete. + * + * PRE CONDITION: Already have buffer lock on iagp. Already have AG lock on + * this AG. Must have read lock on imap inode. + * + * PARAMETERS: + * imap - pointer to inode map control structure. + * iagp - pointer to iag. + * extno - extent number. + * + * RETURN VALUES: + * 0 - success. + * -ENOSPC - insufficient disk resources. + * -EIO - i/o error. + */ +static int diNewExt(struct inomap * imap, struct iag * iagp, int extno) +{ + int agno, iagno, fwd, back, freei = 0, sword, rc; + struct iag *aiagp = NULL, *biagp = NULL, *ciagp = NULL; + struct metapage *amp, *bmp, *cmp, *dmp; + struct inode *ipimap; + s64 blkno, hint; + int i, j; + u32 mask; + ino_t ino; + struct dinode *dp; + struct jfs_sb_info *sbi; + + /* better have free extents. + */ + if (!iagp->nfreeexts) { + jfs_error(imap->im_ipimap->i_sb, "no free extents\n"); + return -EIO; + } + + /* get the inode map inode. + */ + ipimap = imap->im_ipimap; + sbi = JFS_SBI(ipimap->i_sb); + + amp = bmp = cmp = NULL; + + /* get the ag and iag numbers for this iag. + */ + agno = BLKTOAG(le64_to_cpu(iagp->agstart), sbi); + if (agno >= MAXAG || agno < 0) + return -EIO; + + iagno = le32_to_cpu(iagp->iagnum); + + /* check if this is the last free extent within the + * iag. if so, the iag must be removed from the ag + * free extent list, so get the iags preceding and + * following the iag on this list. + */ + if (iagp->nfreeexts == cpu_to_le32(1)) { + if ((fwd = le32_to_cpu(iagp->extfreefwd)) >= 0) { + if ((rc = diIAGRead(imap, fwd, &))) + return (rc); + aiagp = (struct iag *) amp->data; + } + + if ((back = le32_to_cpu(iagp->extfreeback)) >= 0) { + if ((rc = diIAGRead(imap, back, &bmp))) + goto error_out; + biagp = (struct iag *) bmp->data; + } + } else { + /* the iag has free extents. if all extents are free + * (as is the case for a newly allocated iag), the iag + * must be added to the ag free extent list, so get + * the iag at the head of the list in preparation for + * adding this iag to this list. + */ + fwd = back = -1; + if (iagp->nfreeexts == cpu_to_le32(EXTSPERIAG)) { + if ((fwd = imap->im_agctl[agno].extfree) >= 0) { + if ((rc = diIAGRead(imap, fwd, &))) + goto error_out; + aiagp = (struct iag *) amp->data; + } + } + } + + /* check if the iag has no free inodes. if so, the iag + * will have to be added to the ag free inode list, so get + * the iag at the head of the list in preparation for + * adding this iag to this list. in doing this, we must + * check if we already have the iag at the head of + * the list in hand. + */ + if (iagp->nfreeinos == 0) { + freei = imap->im_agctl[agno].inofree; + + if (freei >= 0) { + if (freei == fwd) { + ciagp = aiagp; + } else if (freei == back) { + ciagp = biagp; + } else { + if ((rc = diIAGRead(imap, freei, &cmp))) + goto error_out; + ciagp = (struct iag *) cmp->data; + } + if (ciagp == NULL) { + jfs_error(imap->im_ipimap->i_sb, + "ciagp == NULL\n"); + rc = -EIO; + goto error_out; + } + } + } + + /* allocate disk space for the inode extent. + */ + if ((extno == 0) || (addressPXD(&iagp->inoext[extno - 1]) == 0)) + hint = ((s64) agno << sbi->bmap->db_agl2size) - 1; + else + hint = addressPXD(&iagp->inoext[extno - 1]) + + lengthPXD(&iagp->inoext[extno - 1]) - 1; + + if ((rc = dbAlloc(ipimap, hint, (s64) imap->im_nbperiext, &blkno))) + goto error_out; + + /* compute the inode number of the first inode within the + * extent. + */ + ino = (iagno << L2INOSPERIAG) + (extno << L2INOSPEREXT); + + /* initialize the inodes within the newly allocated extent a + * page at a time. + */ + for (i = 0; i < imap->im_nbperiext; i += sbi->nbperpage) { + /* get a buffer for this page of disk inodes. + */ + dmp = get_metapage(ipimap, blkno + i, PSIZE, 1); + if (dmp == NULL) { + rc = -EIO; + goto error_out; + } + dp = (struct dinode *) dmp->data; + + /* initialize the inode number, mode, link count and + * inode extent address. + */ + for (j = 0; j < INOSPERPAGE; j++, dp++, ino++) { + dp->di_inostamp = cpu_to_le32(sbi->inostamp); + dp->di_number = cpu_to_le32(ino); + dp->di_fileset = cpu_to_le32(FILESYSTEM_I); + dp->di_mode = 0; + dp->di_nlink = 0; + PXDaddress(&(dp->di_ixpxd), blkno); + PXDlength(&(dp->di_ixpxd), imap->im_nbperiext); + } + write_metapage(dmp); + } + + /* if this is the last free extent within the iag, remove the + * iag from the ag free extent list. + */ + if (iagp->nfreeexts == cpu_to_le32(1)) { + if (fwd >= 0) + aiagp->extfreeback = iagp->extfreeback; + + if (back >= 0) + biagp->extfreefwd = iagp->extfreefwd; + else + imap->im_agctl[agno].extfree = + le32_to_cpu(iagp->extfreefwd); + + iagp->extfreefwd = iagp->extfreeback = cpu_to_le32(-1); + } else { + /* if the iag has all free extents (newly allocated iag), + * add the iag to the ag free extent list. + */ + if (iagp->nfreeexts == cpu_to_le32(EXTSPERIAG)) { + if (fwd >= 0) + aiagp->extfreeback = cpu_to_le32(iagno); + + iagp->extfreefwd = cpu_to_le32(fwd); + iagp->extfreeback = cpu_to_le32(-1); + imap->im_agctl[agno].extfree = iagno; + } + } + + /* if the iag has no free inodes, add the iag to the + * ag free inode list. + */ + if (iagp->nfreeinos == 0) { + if (freei >= 0) + ciagp->inofreeback = cpu_to_le32(iagno); + + iagp->inofreefwd = + cpu_to_le32(imap->im_agctl[agno].inofree); + iagp->inofreeback = cpu_to_le32(-1); + imap->im_agctl[agno].inofree = iagno; + } + + /* initialize the extent descriptor of the extent. */ + PXDlength(&iagp->inoext[extno], imap->im_nbperiext); + PXDaddress(&iagp->inoext[extno], blkno); + + /* initialize the working and persistent map of the extent. + * the working map will be initialized such that + * it indicates the first inode of the extent is allocated. + */ + iagp->wmap[extno] = cpu_to_le32(HIGHORDER); + iagp->pmap[extno] = 0; + + /* update the free inode and free extent summary maps + * for the extent to indicate the extent has free inodes + * and no longer represents a free extent. + */ + sword = extno >> L2EXTSPERSUM; + mask = HIGHORDER >> (extno & (EXTSPERSUM - 1)); + iagp->extsmap[sword] |= cpu_to_le32(mask); + iagp->inosmap[sword] &= cpu_to_le32(~mask); + + /* update the free inode and free extent counts for the + * iag. + */ + le32_add_cpu(&iagp->nfreeinos, (INOSPEREXT - 1)); + le32_add_cpu(&iagp->nfreeexts, -1); + + /* update the free and backed inode counts for the ag. + */ + imap->im_agctl[agno].numfree += (INOSPEREXT - 1); + imap->im_agctl[agno].numinos += INOSPEREXT; + + /* update the free and backed inode counts for the inode map. + */ + atomic_add(INOSPEREXT - 1, &imap->im_numfree); + atomic_add(INOSPEREXT, &imap->im_numinos); + + /* write the iags. + */ + if (amp) + write_metapage(amp); + if (bmp) + write_metapage(bmp); + if (cmp) + write_metapage(cmp); + + return (0); + + error_out: + + /* release the iags. + */ + if (amp) + release_metapage(amp); + if (bmp) + release_metapage(bmp); + if (cmp) + release_metapage(cmp); + + return (rc); +} + + +/* + * NAME: diNewIAG(imap,iagnop,agno) + * + * FUNCTION: allocate a new iag for an allocation group. + * + * first tries to allocate the iag from the inode map + * iagfree list: + * if the list has free iags, the head of the list is removed + * and returned to satisfy the request. + * if the inode map's iag free list is empty, the inode map + * is extended to hold a new iag. this new iag is initialized + * and returned to satisfy the request. + * + * PARAMETERS: + * imap - pointer to inode map control structure. + * iagnop - pointer to an iag number set with the number of the + * newly allocated iag upon successful return. + * agno - allocation group number. + * bpp - Buffer pointer to be filled in with new IAG's buffer + * + * RETURN VALUES: + * 0 - success. + * -ENOSPC - insufficient disk resources. + * -EIO - i/o error. + * + * serialization: + * AG lock held on entry/exit; + * write lock on the map is held inside; + * read lock on the map is held on successful completion; + * + * note: new iag transaction: + * . synchronously write iag; + * . write log of xtree and inode of imap; + * . commit; + * . synchronous write of xtree (right to left, bottom to top); + * . at start of logredo(): init in-memory imap with one additional iag page; + * . at end of logredo(): re-read imap inode to determine + * new imap size; + */ +static int +diNewIAG(struct inomap * imap, int *iagnop, int agno, struct metapage ** mpp) +{ + int rc; + int iagno, i, xlen; + struct inode *ipimap; + struct super_block *sb; + struct jfs_sb_info *sbi; + struct metapage *mp; + struct iag *iagp; + s64 xaddr = 0; + s64 blkno; + tid_t tid; + struct inode *iplist[1]; + + /* pick up pointers to the inode map and mount inodes */ + ipimap = imap->im_ipimap; + sb = ipimap->i_sb; + sbi = JFS_SBI(sb); + + /* acquire the free iag lock */ + IAGFREE_LOCK(imap); + + /* if there are any iags on the inode map free iag list, + * allocate the iag from the head of the list. + */ + if (imap->im_freeiag >= 0) { + /* pick up the iag number at the head of the list */ + iagno = imap->im_freeiag; + + /* determine the logical block number of the iag */ + blkno = IAGTOLBLK(iagno, sbi->l2nbperpage); + } else { + /* no free iags. the inode map will have to be extented + * to include a new iag. + */ + + /* acquire inode map lock */ + IWRITE_LOCK(ipimap, RDWRLOCK_IMAP); + + if (ipimap->i_size >> L2PSIZE != imap->im_nextiag + 1) { + IWRITE_UNLOCK(ipimap); + IAGFREE_UNLOCK(imap); + jfs_error(imap->im_ipimap->i_sb, + "ipimap->i_size is wrong\n"); + return -EIO; + } + + + /* get the next available iag number */ + iagno = imap->im_nextiag; + + /* make sure that we have not exceeded the maximum inode + * number limit. + */ + if (iagno > (MAXIAGS - 1)) { + /* release the inode map lock */ + IWRITE_UNLOCK(ipimap); + + rc = -ENOSPC; + goto out; + } + + /* + * synchronously append new iag page. + */ + /* determine the logical address of iag page to append */ + blkno = IAGTOLBLK(iagno, sbi->l2nbperpage); + + /* Allocate extent for new iag page */ + xlen = sbi->nbperpage; + if ((rc = dbAlloc(ipimap, 0, (s64) xlen, &xaddr))) { + /* release the inode map lock */ + IWRITE_UNLOCK(ipimap); + + goto out; + } + + /* + * start transaction of update of the inode map + * addressing structure pointing to the new iag page; + */ + tid = txBegin(sb, COMMIT_FORCE); + mutex_lock(&JFS_IP(ipimap)->commit_mutex); + + /* update the inode map addressing structure to point to it */ + if ((rc = + xtInsert(tid, ipimap, 0, blkno, xlen, &xaddr, 0))) { + txEnd(tid); + mutex_unlock(&JFS_IP(ipimap)->commit_mutex); + /* Free the blocks allocated for the iag since it was + * not successfully added to the inode map + */ + dbFree(ipimap, xaddr, (s64) xlen); + + /* release the inode map lock */ + IWRITE_UNLOCK(ipimap); + + goto out; + } + + /* update the inode map's inode to reflect the extension */ + ipimap->i_size += PSIZE; + inode_add_bytes(ipimap, PSIZE); + + /* assign a buffer for the page */ + mp = get_metapage(ipimap, blkno, PSIZE, 0); + if (!mp) { + /* + * This is very unlikely since we just created the + * extent, but let's try to handle it correctly + */ + xtTruncate(tid, ipimap, ipimap->i_size - PSIZE, + COMMIT_PWMAP); + + txAbort(tid, 0); + txEnd(tid); + mutex_unlock(&JFS_IP(ipimap)->commit_mutex); + + /* release the inode map lock */ + IWRITE_UNLOCK(ipimap); + + rc = -EIO; + goto out; + } + iagp = (struct iag *) mp->data; + + /* init the iag */ + memset(iagp, 0, sizeof(struct iag)); + iagp->iagnum = cpu_to_le32(iagno); + iagp->inofreefwd = iagp->inofreeback = cpu_to_le32(-1); + iagp->extfreefwd = iagp->extfreeback = cpu_to_le32(-1); + iagp->iagfree = cpu_to_le32(-1); + iagp->nfreeinos = 0; + iagp->nfreeexts = cpu_to_le32(EXTSPERIAG); + + /* initialize the free inode summary map (free extent + * summary map initialization handled by bzero). + */ + for (i = 0; i < SMAPSZ; i++) + iagp->inosmap[i] = cpu_to_le32(ONES); + + /* + * Write and sync the metapage + */ + flush_metapage(mp); + + /* + * txCommit(COMMIT_FORCE) will synchronously write address + * index pages and inode after commit in careful update order + * of address index pages (right to left, bottom up); + */ + iplist[0] = ipimap; + rc = txCommit(tid, 1, &iplist[0], COMMIT_FORCE); + + txEnd(tid); + mutex_unlock(&JFS_IP(ipimap)->commit_mutex); + + duplicateIXtree(sb, blkno, xlen, &xaddr); + + /* update the next available iag number */ + imap->im_nextiag += 1; + + /* Add the iag to the iag free list so we don't lose the iag + * if a failure happens now. + */ + imap->im_freeiag = iagno; + + /* Until we have logredo working, we want the imap inode & + * control page to be up to date. + */ + diSync(ipimap); + + /* release the inode map lock */ + IWRITE_UNLOCK(ipimap); + } + + /* obtain read lock on map */ + IREAD_LOCK(ipimap, RDWRLOCK_IMAP); + + /* read the iag */ + if ((rc = diIAGRead(imap, iagno, &mp))) { + IREAD_UNLOCK(ipimap); + rc = -EIO; + goto out; + } + iagp = (struct iag *) mp->data; + + /* remove the iag from the iag free list */ + imap->im_freeiag = le32_to_cpu(iagp->iagfree); + iagp->iagfree = cpu_to_le32(-1); + + /* set the return iag number and buffer pointer */ + *iagnop = iagno; + *mpp = mp; + + out: + /* release the iag free lock */ + IAGFREE_UNLOCK(imap); + + return (rc); +} + +/* + * NAME: diIAGRead() + * + * FUNCTION: get the buffer for the specified iag within a fileset + * or aggregate inode map. + * + * PARAMETERS: + * imap - pointer to inode map control structure. + * iagno - iag number. + * bpp - point to buffer pointer to be filled in on successful + * exit. + * + * SERIALIZATION: + * must have read lock on imap inode + * (When called by diExtendFS, the filesystem is quiesced, therefore + * the read lock is unnecessary.) + * + * RETURN VALUES: + * 0 - success. + * -EIO - i/o error. + */ +static int diIAGRead(struct inomap * imap, int iagno, struct metapage ** mpp) +{ + struct inode *ipimap = imap->im_ipimap; + s64 blkno; + + /* compute the logical block number of the iag. */ + blkno = IAGTOLBLK(iagno, JFS_SBI(ipimap->i_sb)->l2nbperpage); + + /* read the iag. */ + *mpp = read_metapage(ipimap, blkno, PSIZE, 0); + if (*mpp == NULL) { + return -EIO; + } + + return (0); +} + +/* + * NAME: diFindFree() + * + * FUNCTION: find the first free bit in a word starting at + * the specified bit position. + * + * PARAMETERS: + * word - word to be examined. + * start - starting bit position. + * + * RETURN VALUES: + * bit position of first free bit in the word or 32 if + * no free bits were found. + */ +static int diFindFree(u32 word, int start) +{ + int bitno; + assert(start < 32); + /* scan the word for the first free bit. */ + for (word <<= start, bitno = start; bitno < 32; + bitno++, word <<= 1) { + if ((word & HIGHORDER) == 0) + break; + } + return (bitno); +} + +/* + * NAME: diUpdatePMap() + * + * FUNCTION: Update the persistent map in an IAG for the allocation or + * freeing of the specified inode. + * + * PRE CONDITIONS: Working map has already been updated for allocate. + * + * PARAMETERS: + * ipimap - Incore inode map inode + * inum - Number of inode to mark in permanent map + * is_free - If 'true' indicates inode should be marked freed, otherwise + * indicates inode should be marked allocated. + * + * RETURN VALUES: + * 0 for success + */ +int +diUpdatePMap(struct inode *ipimap, + unsigned long inum, bool is_free, struct tblock * tblk) +{ + int rc; + struct iag *iagp; + struct metapage *mp; + int iagno, ino, extno, bitno; + struct inomap *imap; + u32 mask; + struct jfs_log *log; + int lsn, difft, diffp; + unsigned long flags; + + imap = JFS_IP(ipimap)->i_imap; + /* get the iag number containing the inode */ + iagno = INOTOIAG(inum); + /* make sure that the iag is contained within the map */ + if (iagno >= imap->im_nextiag) { + jfs_error(ipimap->i_sb, "the iag is outside the map\n"); + return -EIO; + } + /* read the iag */ + IREAD_LOCK(ipimap, RDWRLOCK_IMAP); + rc = diIAGRead(imap, iagno, &mp); + IREAD_UNLOCK(ipimap); + if (rc) + return (rc); + metapage_wait_for_io(mp); + iagp = (struct iag *) mp->data; + /* get the inode number and extent number of the inode within + * the iag and the inode number within the extent. + */ + ino = inum & (INOSPERIAG - 1); + extno = ino >> L2INOSPEREXT; + bitno = ino & (INOSPEREXT - 1); + mask = HIGHORDER >> bitno; + /* + * mark the inode free in persistent map: + */ + if (is_free) { + /* The inode should have been allocated both in working + * map and in persistent map; + * the inode will be freed from working map at the release + * of last reference release; + */ + if (!(le32_to_cpu(iagp->wmap[extno]) & mask)) { + jfs_error(ipimap->i_sb, + "inode %ld not marked as allocated in wmap!\n", + inum); + } + if (!(le32_to_cpu(iagp->pmap[extno]) & mask)) { + jfs_error(ipimap->i_sb, + "inode %ld not marked as allocated in pmap!\n", + inum); + } + /* update the bitmap for the extent of the freed inode */ + iagp->pmap[extno] &= cpu_to_le32(~mask); + } + /* + * mark the inode allocated in persistent map: + */ + else { + /* The inode should be already allocated in the working map + * and should be free in persistent map; + */ + if (!(le32_to_cpu(iagp->wmap[extno]) & mask)) { + release_metapage(mp); + jfs_error(ipimap->i_sb, + "the inode is not allocated in the working map\n"); + return -EIO; + } + if ((le32_to_cpu(iagp->pmap[extno]) & mask) != 0) { + release_metapage(mp); + jfs_error(ipimap->i_sb, + "the inode is not free in the persistent map\n"); + return -EIO; + } + /* update the bitmap for the extent of the allocated inode */ + iagp->pmap[extno] |= cpu_to_le32(mask); + } + /* + * update iag lsn + */ + lsn = tblk->lsn; + log = JFS_SBI(tblk->sb)->log; + LOGSYNC_LOCK(log, flags); + if (mp->lsn != 0) { + /* inherit older/smaller lsn */ + logdiff(difft, lsn, log); + logdiff(diffp, mp->lsn, log); + if (difft < diffp) { + mp->lsn = lsn; + /* move mp after tblock in logsync list */ + list_move(&mp->synclist, &tblk->synclist); + } + /* inherit younger/larger clsn */ + assert(mp->clsn); + logdiff(difft, tblk->clsn, log); + logdiff(diffp, mp->clsn, log); + if (difft > diffp) + mp->clsn = tblk->clsn; + } else { + mp->log = log; + mp->lsn = lsn; + /* insert mp after tblock in logsync list */ + log->count++; + list_add(&mp->synclist, &tblk->synclist); + mp->clsn = tblk->clsn; + } + LOGSYNC_UNLOCK(log, flags); + write_metapage(mp); + return (0); +} + +/* + * diExtendFS() + * + * function: update imap for extendfs(); + * + * note: AG size has been increased s.t. each k old contiguous AGs are + * coalesced into a new AG; + */ +int diExtendFS(struct inode *ipimap, struct inode *ipbmap) +{ + int rc, rcx = 0; + struct inomap *imap = JFS_IP(ipimap)->i_imap; + struct iag *iagp = NULL, *hiagp = NULL; + struct bmap *mp = JFS_SBI(ipbmap->i_sb)->bmap; + struct metapage *bp, *hbp; + int i, n, head; + int numinos, xnuminos = 0, xnumfree = 0; + s64 agstart; + + jfs_info("diExtendFS: nextiag:%d numinos:%d numfree:%d", + imap->im_nextiag, atomic_read(&imap->im_numinos), + atomic_read(&imap->im_numfree)); + + /* + * reconstruct imap + * + * coalesce contiguous k (newAGSize/oldAGSize) AGs; + * i.e., (AGi, ..., AGj) where i = k*n and j = k*(n+1) - 1 to AGn; + * note: new AG size = old AG size * (2**x). + */ + + /* init per AG control information im_agctl[] */ + for (i = 0; i < MAXAG; i++) { + imap->im_agctl[i].inofree = -1; + imap->im_agctl[i].extfree = -1; + imap->im_agctl[i].numinos = 0; /* number of backed inodes */ + imap->im_agctl[i].numfree = 0; /* number of free backed inodes */ + } + + /* + * process each iag page of the map. + * + * rebuild AG Free Inode List, AG Free Inode Extent List; + */ + for (i = 0; i < imap->im_nextiag; i++) { + if ((rc = diIAGRead(imap, i, &bp))) { + rcx = rc; + continue; + } + iagp = (struct iag *) bp->data; + if (le32_to_cpu(iagp->iagnum) != i) { + release_metapage(bp); + jfs_error(ipimap->i_sb, "unexpected value of iagnum\n"); + return -EIO; + } + + /* leave free iag in the free iag list */ + if (iagp->nfreeexts == cpu_to_le32(EXTSPERIAG)) { + release_metapage(bp); + continue; + } + + agstart = le64_to_cpu(iagp->agstart); + n = agstart >> mp->db_agl2size; + iagp->agstart = cpu_to_le64((s64)n << mp->db_agl2size); + + /* compute backed inodes */ + numinos = (EXTSPERIAG - le32_to_cpu(iagp->nfreeexts)) + << L2INOSPEREXT; + if (numinos > 0) { + /* merge AG backed inodes */ + imap->im_agctl[n].numinos += numinos; + xnuminos += numinos; + } + + /* if any backed free inodes, insert at AG free inode list */ + if ((int) le32_to_cpu(iagp->nfreeinos) > 0) { + if ((head = imap->im_agctl[n].inofree) == -1) { + iagp->inofreefwd = cpu_to_le32(-1); + iagp->inofreeback = cpu_to_le32(-1); + } else { + if ((rc = diIAGRead(imap, head, &hbp))) { + rcx = rc; + goto nextiag; + } + hiagp = (struct iag *) hbp->data; + hiagp->inofreeback = iagp->iagnum; + iagp->inofreefwd = cpu_to_le32(head); + iagp->inofreeback = cpu_to_le32(-1); + write_metapage(hbp); + } + + imap->im_agctl[n].inofree = + le32_to_cpu(iagp->iagnum); + + /* merge AG backed free inodes */ + imap->im_agctl[n].numfree += + le32_to_cpu(iagp->nfreeinos); + xnumfree += le32_to_cpu(iagp->nfreeinos); + } + + /* if any free extents, insert at AG free extent list */ + if (le32_to_cpu(iagp->nfreeexts) > 0) { + if ((head = imap->im_agctl[n].extfree) == -1) { + iagp->extfreefwd = cpu_to_le32(-1); + iagp->extfreeback = cpu_to_le32(-1); + } else { + if ((rc = diIAGRead(imap, head, &hbp))) { + rcx = rc; + goto nextiag; + } + hiagp = (struct iag *) hbp->data; + hiagp->extfreeback = iagp->iagnum; + iagp->extfreefwd = cpu_to_le32(head); + iagp->extfreeback = cpu_to_le32(-1); + write_metapage(hbp); + } + + imap->im_agctl[n].extfree = + le32_to_cpu(iagp->iagnum); + } + + nextiag: + write_metapage(bp); + } + + if (xnuminos != atomic_read(&imap->im_numinos) || + xnumfree != atomic_read(&imap->im_numfree)) { + jfs_error(ipimap->i_sb, "numinos or numfree incorrect\n"); + return -EIO; + } + + return rcx; +} + + +/* + * duplicateIXtree() + * + * serialization: IWRITE_LOCK held on entry/exit + * + * note: shadow page with regular inode (rel.2); + */ +static void duplicateIXtree(struct super_block *sb, s64 blkno, + int xlen, s64 *xaddr) +{ + struct jfs_superblock *j_sb; + struct buffer_head *bh; + struct inode *ip; + tid_t tid; + + /* if AIT2 ipmap2 is bad, do not try to update it */ + if (JFS_SBI(sb)->mntflag & JFS_BAD_SAIT) /* s_flag */ + return; + ip = diReadSpecial(sb, FILESYSTEM_I, 1); + if (ip == NULL) { + JFS_SBI(sb)->mntflag |= JFS_BAD_SAIT; + if (readSuper(sb, &bh)) + return; + j_sb = (struct jfs_superblock *)bh->b_data; + j_sb->s_flag |= cpu_to_le32(JFS_BAD_SAIT); + + mark_buffer_dirty(bh); + sync_dirty_buffer(bh); + brelse(bh); + return; + } + + /* start transaction */ + tid = txBegin(sb, COMMIT_FORCE); + /* update the inode map addressing structure to point to it */ + if (xtInsert(tid, ip, 0, blkno, xlen, xaddr, 0)) { + JFS_SBI(sb)->mntflag |= JFS_BAD_SAIT; + txAbort(tid, 1); + goto cleanup; + + } + /* update the inode map's inode to reflect the extension */ + ip->i_size += PSIZE; + inode_add_bytes(ip, PSIZE); + txCommit(tid, 1, &ip, COMMIT_FORCE); + cleanup: + txEnd(tid); + diFreeSpecial(ip); +} + +/* + * NAME: copy_from_dinode() + * + * FUNCTION: Copies inode info from disk inode to in-memory inode + * + * RETURN VALUES: + * 0 - success + * -EINVAL - unexpected inode type + */ +static int copy_from_dinode(struct dinode * dip, struct inode *ip) +{ + struct jfs_inode_info *jfs_ip = JFS_IP(ip); + struct jfs_sb_info *sbi = JFS_SBI(ip->i_sb); + int fileset = le32_to_cpu(dip->di_fileset); + + switch (fileset) { + case AGGR_RESERVED_I: case AGGREGATE_I: case BMAP_I: + case LOG_I: case BADBLOCK_I: case FILESYSTEM_I: + break; + default: + return -EINVAL; + } + + jfs_ip->fileset = fileset; + jfs_ip->mode2 = le32_to_cpu(dip->di_mode); + jfs_set_inode_flags(ip); + + ip->i_mode = le32_to_cpu(dip->di_mode) & 0xffff; + if (sbi->umask != -1) { + ip->i_mode = (ip->i_mode & ~0777) | (0777 & ~sbi->umask); + /* For directories, add x permission if r is allowed by umask */ + if (S_ISDIR(ip->i_mode)) { + if (ip->i_mode & 0400) + ip->i_mode |= 0100; + if (ip->i_mode & 0040) + ip->i_mode |= 0010; + if (ip->i_mode & 0004) + ip->i_mode |= 0001; + } + } + set_nlink(ip, le32_to_cpu(dip->di_nlink)); + + jfs_ip->saved_uid = make_kuid(&init_user_ns, le32_to_cpu(dip->di_uid)); + if (!uid_valid(sbi->uid)) + ip->i_uid = jfs_ip->saved_uid; + else { + ip->i_uid = sbi->uid; + } + + jfs_ip->saved_gid = make_kgid(&init_user_ns, le32_to_cpu(dip->di_gid)); + if (!gid_valid(sbi->gid)) + ip->i_gid = jfs_ip->saved_gid; + else { + ip->i_gid = sbi->gid; + } + + ip->i_size = le64_to_cpu(dip->di_size); + inode_set_atime(ip, le32_to_cpu(dip->di_atime.tv_sec), + le32_to_cpu(dip->di_atime.tv_nsec)); + inode_set_mtime(ip, le32_to_cpu(dip->di_mtime.tv_sec), + le32_to_cpu(dip->di_mtime.tv_nsec)); + inode_set_ctime(ip, le32_to_cpu(dip->di_ctime.tv_sec), + le32_to_cpu(dip->di_ctime.tv_nsec)); + ip->i_blocks = LBLK2PBLK(ip->i_sb, le64_to_cpu(dip->di_nblocks)); + ip->i_generation = le32_to_cpu(dip->di_gen); + + jfs_ip->ixpxd = dip->di_ixpxd; /* in-memory pxd's are little-endian */ + jfs_ip->acl = dip->di_acl; /* as are dxd's */ + jfs_ip->ea = dip->di_ea; + jfs_ip->next_index = le32_to_cpu(dip->di_next_index); + jfs_ip->otime = le32_to_cpu(dip->di_otime.tv_sec); + jfs_ip->acltype = le32_to_cpu(dip->di_acltype); + + if (S_ISCHR(ip->i_mode) || S_ISBLK(ip->i_mode)) { + jfs_ip->dev = le32_to_cpu(dip->di_rdev); + ip->i_rdev = new_decode_dev(jfs_ip->dev); + } + + if (S_ISDIR(ip->i_mode)) { + memcpy(&jfs_ip->u.dir, &dip->u._dir, 384); + if (!check_dtroot(&jfs_ip->i_dtroot)) { + jfs_error(ip->i_sb, "Corrupt dtroot\n"); + return -EIO; + } + } else if (S_ISREG(ip->i_mode) || S_ISLNK(ip->i_mode)) { + memcpy(&jfs_ip->i_xtroot, &dip->di_xtroot, 288); + } else + memcpy(&jfs_ip->i_inline_ea, &dip->di_inlineea, 128); + + /* Zero the in-memory-only stuff */ + jfs_ip->cflag = 0; + jfs_ip->btindex = 0; + jfs_ip->btorder = 0; + jfs_ip->bxflag = 0; + jfs_ip->blid = 0; + jfs_ip->atlhead = 0; + jfs_ip->atltail = 0; + jfs_ip->xtlid = 0; + return (0); +} + +/* + * NAME: copy_to_dinode() + * + * FUNCTION: Copies inode info from in-memory inode to disk inode + */ +static void copy_to_dinode(struct dinode * dip, struct inode *ip) +{ + struct jfs_inode_info *jfs_ip = JFS_IP(ip); + struct jfs_sb_info *sbi = JFS_SBI(ip->i_sb); + + dip->di_fileset = cpu_to_le32(jfs_ip->fileset); + dip->di_inostamp = cpu_to_le32(sbi->inostamp); + dip->di_number = cpu_to_le32(ip->i_ino); + dip->di_gen = cpu_to_le32(ip->i_generation); + dip->di_size = cpu_to_le64(ip->i_size); + dip->di_nblocks = cpu_to_le64(PBLK2LBLK(ip->i_sb, ip->i_blocks)); + dip->di_nlink = cpu_to_le32(ip->i_nlink); + if (!uid_valid(sbi->uid)) + dip->di_uid = cpu_to_le32(i_uid_read(ip)); + else + dip->di_uid =cpu_to_le32(from_kuid(&init_user_ns, + jfs_ip->saved_uid)); + if (!gid_valid(sbi->gid)) + dip->di_gid = cpu_to_le32(i_gid_read(ip)); + else + dip->di_gid = cpu_to_le32(from_kgid(&init_user_ns, + jfs_ip->saved_gid)); + /* + * mode2 is only needed for storing the higher order bits. + * Trust i_mode for the lower order ones + */ + if (sbi->umask == -1) + dip->di_mode = cpu_to_le32((jfs_ip->mode2 & 0xffff0000) | + ip->i_mode); + else /* Leave the original permissions alone */ + dip->di_mode = cpu_to_le32(jfs_ip->mode2); + + dip->di_atime.tv_sec = cpu_to_le32(inode_get_atime_sec(ip)); + dip->di_atime.tv_nsec = cpu_to_le32(inode_get_atime_nsec(ip)); + dip->di_ctime.tv_sec = cpu_to_le32(inode_get_ctime_sec(ip)); + dip->di_ctime.tv_nsec = cpu_to_le32(inode_get_ctime_nsec(ip)); + dip->di_mtime.tv_sec = cpu_to_le32(inode_get_mtime_sec(ip)); + dip->di_mtime.tv_nsec = cpu_to_le32(inode_get_mtime_nsec(ip)); + dip->di_ixpxd = jfs_ip->ixpxd; /* in-memory pxd's are little-endian */ + dip->di_acl = jfs_ip->acl; /* as are dxd's */ + dip->di_ea = jfs_ip->ea; + dip->di_next_index = cpu_to_le32(jfs_ip->next_index); + dip->di_otime.tv_sec = cpu_to_le32(jfs_ip->otime); + dip->di_otime.tv_nsec = 0; + dip->di_acltype = cpu_to_le32(jfs_ip->acltype); + if (S_ISCHR(ip->i_mode) || S_ISBLK(ip->i_mode)) + dip->di_rdev = cpu_to_le32(jfs_ip->dev); +} diff --git a/fs/jfs/jfs_imap.h b/fs/jfs/jfs_imap.h new file mode 100644 index 000000000..dd7409feb --- /dev/null +++ b/fs/jfs/jfs_imap.h @@ -0,0 +1,162 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) International Business Machines Corp., 2000-2002 + */ +#ifndef _H_JFS_IMAP +#define _H_JFS_IMAP + +#include "jfs_txnmgr.h" + +/* + * jfs_imap.h: disk inode manager + */ + +#define EXTSPERIAG 128 /* number of disk inode extent per iag */ +#define IMAPBLKNO 0 /* lblkno of dinomap within inode map */ +#define SMAPSZ 4 /* number of words per summary map */ +#define EXTSPERSUM 32 /* number of extents per summary map entry */ +#define L2EXTSPERSUM 5 /* l2 number of extents per summary map */ +#define PGSPERIEXT 4 /* number of 4K pages per dinode extent */ +#define MAXIAGS ((1<<20)-1) /* maximum number of iags */ +#define MAXAG 128 /* maximum number of allocation groups */ + +#define AMAPSIZE 512 /* bytes in the IAG allocation maps */ +#define SMAPSIZE 16 /* bytes in the IAG summary maps */ + +/* convert inode number to iag number */ +#define INOTOIAG(ino) ((ino) >> L2INOSPERIAG) + +/* convert iag number to logical block number of the iag page */ +#define IAGTOLBLK(iagno,l2nbperpg) (((iagno) + 1) << (l2nbperpg)) + +/* get the starting block number of the 4K page of an inode extent + * that contains ino. + */ +#define INOPBLK(pxd,ino,l2nbperpg) (addressPXD((pxd)) + \ + ((((ino) & (INOSPEREXT-1)) >> L2INOSPERPAGE) << (l2nbperpg))) + +/* + * inode allocation map: + * + * inode allocation map consists of + * . the inode map control page and + * . inode allocation group pages (per 4096 inodes) + * which are addressed by standard JFS xtree. + */ +/* + * inode allocation group page (per 4096 inodes of an AG) + */ +struct iag { + __le64 agstart; /* 8: starting block of ag */ + __le32 iagnum; /* 4: inode allocation group number */ + __le32 inofreefwd; /* 4: ag inode free list forward */ + __le32 inofreeback; /* 4: ag inode free list back */ + __le32 extfreefwd; /* 4: ag inode extent free list forward */ + __le32 extfreeback; /* 4: ag inode extent free list back */ + __le32 iagfree; /* 4: iag free list */ + + /* summary map: 1 bit per inode extent */ + __le32 inosmap[SMAPSZ]; /* 16: sum map of mapwords w/ free inodes; + * note: this indicates free and backed + * inodes, if the extent is not backed the + * value will be 1. if the extent is + * backed but all inodes are being used the + * value will be 1. if the extent is + * backed but at least one of the inodes is + * free the value will be 0. + */ + __le32 extsmap[SMAPSZ]; /* 16: sum map of mapwords w/ free extents */ + __le32 nfreeinos; /* 4: number of free inodes */ + __le32 nfreeexts; /* 4: number of free extents */ + /* (72) */ + u8 pad[1976]; /* 1976: pad to 2048 bytes */ + /* allocation bit map: 1 bit per inode (0 - free, 1 - allocated) */ + __le32 wmap[EXTSPERIAG]; /* 512: working allocation map */ + __le32 pmap[EXTSPERIAG]; /* 512: persistent allocation map */ + pxd_t inoext[EXTSPERIAG]; /* 1024: inode extent addresses */ +}; /* (4096) */ + +/* + * per AG control information (in inode map control page) + */ +struct iagctl_disk { + __le32 inofree; /* 4: free inode list anchor */ + __le32 extfree; /* 4: free extent list anchor */ + __le32 numinos; /* 4: number of backed inodes */ + __le32 numfree; /* 4: number of free inodes */ +}; /* (16) */ + +struct iagctl { + int inofree; /* free inode list anchor */ + int extfree; /* free extent list anchor */ + int numinos; /* number of backed inodes */ + int numfree; /* number of free inodes */ +}; + +/* + * per fileset/aggregate inode map control page + */ +struct dinomap_disk { + __le32 in_freeiag; /* 4: free iag list anchor */ + __le32 in_nextiag; /* 4: next free iag number */ + __le32 in_numinos; /* 4: num of backed inodes */ + __le32 in_numfree; /* 4: num of free backed inodes */ + __le32 in_nbperiext; /* 4: num of blocks per inode extent */ + __le32 in_l2nbperiext; /* 4: l2 of in_nbperiext */ + __le32 in_diskblock; /* 4: for standalone test driver */ + __le32 in_maxag; /* 4: for standalone test driver */ + u8 pad[2016]; /* 2016: pad to 2048 */ + struct iagctl_disk in_agctl[MAXAG]; /* 2048: AG control information */ +}; /* (4096) */ + +struct dinomap { + int in_freeiag; /* free iag list anchor */ + int in_nextiag; /* next free iag number */ + int in_numinos; /* num of backed inodes */ + int in_numfree; /* num of free backed inodes */ + int in_nbperiext; /* num of blocks per inode extent */ + int in_l2nbperiext; /* l2 of in_nbperiext */ + int in_diskblock; /* for standalone test driver */ + int in_maxag; /* for standalone test driver */ + struct iagctl in_agctl[MAXAG]; /* AG control information */ +}; + +/* + * In-core inode map control page + */ +struct inomap { + struct dinomap im_imap; /* 4096: inode allocation control */ + struct inode *im_ipimap; /* 4: ptr to inode for imap */ + struct mutex im_freelock; /* 4: iag free list lock */ + struct mutex im_aglock[MAXAG]; /* 512: per AG locks */ + u32 *im_DBGdimap; + atomic_t im_numinos; /* num of backed inodes */ + atomic_t im_numfree; /* num of free backed inodes */ +}; + +#define im_freeiag im_imap.in_freeiag +#define im_nextiag im_imap.in_nextiag +#define im_agctl im_imap.in_agctl +#define im_nbperiext im_imap.in_nbperiext +#define im_l2nbperiext im_imap.in_l2nbperiext + +/* for standalone testdriver + */ +#define im_diskblock im_imap.in_diskblock +#define im_maxag im_imap.in_maxag + +extern int diFree(struct inode *); +extern int diAlloc(struct inode *, bool, struct inode *); +extern int diSync(struct inode *); +/* external references */ +extern int diUpdatePMap(struct inode *ipimap, unsigned long inum, + bool is_free, struct tblock * tblk); +extern int diExtendFS(struct inode *ipimap, struct inode *ipbmap); +extern int diMount(struct inode *); +extern int diUnmount(struct inode *, int); +extern int diRead(struct inode *); +extern struct inode *diReadSpecial(struct super_block *, ino_t, int); +extern void diWriteSpecial(struct inode *, int); +extern void diFreeSpecial(struct inode *); +extern int diWrite(tid_t tid, struct inode *); +#endif /* _H_JFS_IMAP */ diff --git a/fs/jfs/jfs_incore.h b/fs/jfs/jfs_incore.h new file mode 100644 index 000000000..5aaafedb8 --- /dev/null +++ b/fs/jfs/jfs_incore.h @@ -0,0 +1,226 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) International Business Machines Corp., 2000-2004 + * Portions Copyright (C) Christoph Hellwig, 2001-2002 + */ +#ifndef _H_JFS_INCORE +#define _H_JFS_INCORE + +#include +#include +#include +#include +#include + +#include "jfs_types.h" +#include "jfs_xtree.h" +#include "jfs_dtree.h" + +/* + * JFS magic number + */ +#define JFS_SUPER_MAGIC 0x3153464a /* "JFS1" */ + +/* + * JFS-private inode information + */ +struct jfs_inode_info { + int fileset; /* fileset number (always 16)*/ + uint mode2; /* jfs-specific mode */ + kuid_t saved_uid; /* saved for uid mount option */ + kgid_t saved_gid; /* saved for gid mount option */ + pxd_t ixpxd; /* inode extent descriptor */ + dxd_t acl; /* dxd describing acl */ + dxd_t ea; /* dxd describing ea */ + time64_t otime; /* time created */ + uint next_index; /* next available directory entry index */ + int acltype; /* Type of ACL */ + short btorder; /* access order */ + short btindex; /* btpage entry index*/ + struct inode *ipimap; /* inode map */ + unsigned long cflag; /* commit flags */ + u64 agstart; /* agstart of the containing IAG */ + u16 bxflag; /* xflag of pseudo buffer? */ + unchar pad; + signed char active_ag; /* ag currently allocating from */ + lid_t blid; /* lid of pseudo buffer? */ + lid_t atlhead; /* anonymous tlock list head */ + lid_t atltail; /* anonymous tlock list tail */ + spinlock_t ag_lock; /* protects active_ag */ + struct list_head anon_inode_list; /* inodes having anonymous txns */ + /* + * rdwrlock serializes xtree between reads & writes and synchronizes + * changes to special inodes. It's use would be redundant on + * directories since the i_mutex taken in the VFS is sufficient. + */ + struct rw_semaphore rdwrlock; + /* + * commit_mutex serializes transaction processing on an inode. + * It must be taken after beginning a transaction (txBegin), since + * dirty inodes may be committed while a new transaction on the + * inode is blocked in txBegin or TxBeginAnon + */ + struct mutex commit_mutex; + /* xattr_sem allows us to access the xattrs without taking i_mutex */ + struct rw_semaphore xattr_sem; + lid_t xtlid; /* lid of xtree lock on directory */ + union { + struct { + xtroot_t _xtroot; /* 288: xtree root */ + struct inomap *_imap; /* 4: inode map header */ + } file; + struct { + struct dir_table_slot _table[12]; /* 96: dir index */ + dtroot_t _dtroot; /* 288: dtree root */ + } dir; + struct { + unchar _unused[16]; /* 16: */ + dxd_t _dxd; /* 16: */ + /* _inline_sym may overflow into _inline_ea when needed */ + /* _inline_ea may overlay the last part of + * file._xtroot if maxentry = XTROOTINITSLOT + */ + union { + struct { + /* 128: inline symlink */ + unchar _inline_sym[128]; + /* 128: inline extended attr */ + unchar _inline_ea[128]; + }; + unchar _inline_all[256]; + }; + } link; + } u; +#ifdef CONFIG_QUOTA + struct dquot __rcu *i_dquot[MAXQUOTAS]; +#endif + u32 dev; /* will die when we get wide dev_t */ + struct inode vfs_inode; +}; +#define i_xtroot u.file._xtroot +#define i_imap u.file._imap +#define i_dirtable u.dir._table +#define i_dtroot u.dir._dtroot +#define i_inline u.link._inline_sym +#define i_inline_ea u.link._inline_ea +#define i_inline_all u.link._inline_all + +#define IREAD_LOCK(ip, subclass) \ + down_read_nested(&JFS_IP(ip)->rdwrlock, subclass) +#define IREAD_UNLOCK(ip) up_read(&JFS_IP(ip)->rdwrlock) +#define IWRITE_LOCK(ip, subclass) \ + down_write_nested(&JFS_IP(ip)->rdwrlock, subclass) +#define IWRITE_UNLOCK(ip) up_write(&JFS_IP(ip)->rdwrlock) + +/* + * cflag + */ +enum cflags { + COMMIT_Nolink, /* inode committed with zero link count */ + COMMIT_Inlineea, /* commit inode inline EA */ + COMMIT_Freewmap, /* free WMAP at iClose() */ + COMMIT_Dirty, /* Inode is really dirty */ + COMMIT_Dirtable, /* commit changes to di_dirtable */ + COMMIT_Stale, /* data extent is no longer valid */ + COMMIT_Synclist, /* metadata pages on group commit synclist */ +}; + +/* + * commit_mutex nesting subclasses: + */ +enum commit_mutex_class +{ + COMMIT_MUTEX_PARENT, + COMMIT_MUTEX_CHILD, + COMMIT_MUTEX_SECOND_PARENT, /* Renaming */ + COMMIT_MUTEX_VICTIM /* Inode being unlinked due to rename */ +}; + +/* + * rdwrlock subclasses: + * The dmap inode may be locked while a normal inode or the imap inode are + * locked. + */ +enum rdwrlock_class +{ + RDWRLOCK_NORMAL, + RDWRLOCK_IMAP, + RDWRLOCK_DMAP +}; + +#define set_cflag(flag, ip) set_bit(flag, &(JFS_IP(ip)->cflag)) +#define clear_cflag(flag, ip) clear_bit(flag, &(JFS_IP(ip)->cflag)) +#define test_cflag(flag, ip) test_bit(flag, &(JFS_IP(ip)->cflag)) +#define test_and_clear_cflag(flag, ip) \ + test_and_clear_bit(flag, &(JFS_IP(ip)->cflag)) +/* + * JFS-private superblock information. + */ +struct jfs_sb_info { + struct super_block *sb; /* Point back to vfs super block */ + unsigned long mntflag; /* aggregate attributes */ + struct inode *ipbmap; /* block map inode */ + struct inode *ipaimap; /* aggregate inode map inode */ + struct inode *ipaimap2; /* secondary aimap inode */ + struct inode *ipimap; /* aggregate inode map inode */ + struct jfs_log *log; /* log */ + struct list_head log_list; /* volumes associated with a journal */ + short bsize; /* logical block size */ + short l2bsize; /* log2 logical block size */ + short nbperpage; /* blocks per page */ + short l2nbperpage; /* log2 blocks per page */ + short l2niperblk; /* log2 inodes per page */ + dev_t logdev; /* external log device */ + uint aggregate; /* volume identifier in log record */ + pxd_t logpxd; /* pxd describing log */ + pxd_t fsckpxd; /* pxd describing fsck wkspc */ + pxd_t ait2; /* pxd describing AIT copy */ + uuid_t uuid; /* 128-bit uuid for volume */ + uuid_t loguuid; /* 128-bit uuid for log */ + /* + * commit_state is used for synchronization of the jfs_commit + * threads. It is protected by LAZY_LOCK(). + */ + int commit_state; /* commit state */ + /* Formerly in ipimap */ + uint gengen; /* inode generation generator*/ + uint inostamp; /* shows inode belongs to fileset*/ + + /* Formerly in ipbmap */ + struct bmap *bmap; /* incore bmap descriptor */ + struct nls_table *nls_tab; /* current codepage */ + struct inode *direct_inode; /* metadata inode */ + uint state; /* mount/recovery state */ + unsigned long flag; /* mount time flags */ + uint p_state; /* state prior to going no integrity */ + kuid_t uid; /* uid to override on-disk uid */ + kgid_t gid; /* gid to override on-disk gid */ + uint umask; /* umask to override on-disk umask */ + uint minblks_trim; /* minimum blocks, for online trim */ +}; + +/* jfs_sb_info commit_state */ +#define IN_LAZYCOMMIT 1 + +static inline struct jfs_inode_info *JFS_IP(struct inode *inode) +{ + return container_of(inode, struct jfs_inode_info, vfs_inode); +} + +static inline int jfs_dirtable_inline(struct inode *inode) +{ + return (JFS_IP(inode)->next_index <= (MAX_INLINE_DIRTABLE_ENTRY + 1)); +} + +static inline struct jfs_sb_info *JFS_SBI(struct super_block *sb) +{ + return sb->s_fs_info; +} + +static inline int isReadOnly(struct inode *inode) +{ + if (JFS_SBI(inode->i_sb)->log) + return 0; + return 1; +} +#endif /* _H_JFS_INCORE */ diff --git a/fs/jfs/jfs_inode.c b/fs/jfs/jfs_inode.c new file mode 100644 index 000000000..f10f295d1 --- /dev/null +++ b/fs/jfs/jfs_inode.c @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) International Business Machines Corp., 2000-2004 + */ + +#include +#include +#include "jfs_incore.h" +#include "jfs_inode.h" +#include "jfs_filsys.h" +#include "jfs_imap.h" +#include "jfs_dinode.h" +#include "jfs_debug.h" + + +void jfs_set_inode_flags(struct inode *inode) +{ + unsigned int flags = JFS_IP(inode)->mode2; + unsigned int new_fl = 0; + + if (flags & JFS_IMMUTABLE_FL) + new_fl |= S_IMMUTABLE; + if (flags & JFS_APPEND_FL) + new_fl |= S_APPEND; + if (flags & JFS_NOATIME_FL) + new_fl |= S_NOATIME; + if (flags & JFS_DIRSYNC_FL) + new_fl |= S_DIRSYNC; + if (flags & JFS_SYNC_FL) + new_fl |= S_SYNC; + inode_set_flags(inode, new_fl, S_IMMUTABLE | S_APPEND | S_NOATIME | + S_DIRSYNC | S_SYNC); +} + +/* + * NAME: ialloc() + * + * FUNCTION: Allocate a new inode + * + */ +struct inode *ialloc(struct inode *parent, umode_t mode) +{ + struct super_block *sb = parent->i_sb; + struct inode *inode; + struct jfs_inode_info *jfs_inode; + int rc; + + inode = new_inode(sb); + if (!inode) { + jfs_warn("ialloc: new_inode returned NULL!"); + return ERR_PTR(-ENOMEM); + } + + jfs_inode = JFS_IP(inode); + + rc = diAlloc(parent, S_ISDIR(mode), inode); + if (rc) { + jfs_warn("ialloc: diAlloc returned %d!", rc); + goto fail_put; + } + + if (insert_inode_locked(inode) < 0) { + rc = -EINVAL; + goto fail_put; + } + + inode_init_owner(&nop_mnt_idmap, inode, parent, mode); + /* + * New inodes need to save sane values on disk when + * uid & gid mount options are used + */ + jfs_inode->saved_uid = inode->i_uid; + jfs_inode->saved_gid = inode->i_gid; + + /* + * Allocate inode to quota. + */ + rc = dquot_initialize(inode); + if (rc) + goto fail_drop; + rc = dquot_alloc_inode(inode); + if (rc) + goto fail_drop; + + /* inherit flags from parent */ + jfs_inode->mode2 = JFS_IP(parent)->mode2 & JFS_FL_INHERIT; + + if (S_ISDIR(mode)) { + jfs_inode->mode2 |= IDIRECTORY; + jfs_inode->mode2 &= ~JFS_DIRSYNC_FL; + } + else { + jfs_inode->mode2 |= INLINEEA | ISPARSE; + if (S_ISLNK(mode)) + jfs_inode->mode2 &= ~(JFS_IMMUTABLE_FL|JFS_APPEND_FL); + } + jfs_inode->mode2 |= inode->i_mode; + + inode->i_blocks = 0; + simple_inode_init_ts(inode); + jfs_inode->otime = inode_get_ctime_sec(inode); + inode->i_generation = JFS_SBI(sb)->gengen++; + + jfs_inode->cflag = 0; + + /* Zero remaining fields */ + memset(&jfs_inode->acl, 0, sizeof(dxd_t)); + memset(&jfs_inode->ea, 0, sizeof(dxd_t)); + jfs_inode->next_index = 0; + jfs_inode->acltype = 0; + jfs_inode->btorder = 0; + jfs_inode->btindex = 0; + jfs_inode->bxflag = 0; + jfs_inode->blid = 0; + jfs_inode->atlhead = 0; + jfs_inode->atltail = 0; + jfs_inode->xtlid = 0; + jfs_set_inode_flags(inode); + + jfs_info("ialloc returns inode = 0x%p", inode); + + return inode; + +fail_drop: + dquot_drop(inode); + inode->i_flags |= S_NOQUOTA; + clear_nlink(inode); + discard_new_inode(inode); + return ERR_PTR(rc); + +fail_put: + iput(inode); + return ERR_PTR(rc); +} diff --git a/fs/jfs/jfs_inode.h b/fs/jfs/jfs_inode.h new file mode 100644 index 000000000..2c6c81c8c --- /dev/null +++ b/fs/jfs/jfs_inode.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) International Business Machines Corp., 2000-2001 + */ +#ifndef _H_JFS_INODE +#define _H_JFS_INODE + +struct fid; + +extern struct inode *ialloc(struct inode *, umode_t); +extern int jfs_fsync(struct file *, loff_t, loff_t, int); +extern int jfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa); +extern int jfs_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa); +extern long jfs_ioctl(struct file *, unsigned int, unsigned long); +extern struct inode *jfs_iget(struct super_block *, unsigned long); +extern int jfs_commit_inode(struct inode *, int); +extern int jfs_write_inode(struct inode *, struct writeback_control *); +extern void jfs_evict_inode(struct inode *); +extern void jfs_dirty_inode(struct inode *, int); +extern void jfs_truncate(struct inode *); +extern void jfs_truncate_nolock(struct inode *, loff_t); +extern void jfs_free_zero_link(struct inode *); +extern struct dentry *jfs_get_parent(struct dentry *dentry); +extern struct dentry *jfs_fh_to_dentry(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type); +extern struct dentry *jfs_fh_to_parent(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type); +extern void jfs_set_inode_flags(struct inode *); +extern int jfs_get_block(struct inode *, sector_t, struct buffer_head *, int); +extern int jfs_setattr(struct mnt_idmap *, struct dentry *, struct iattr *); + +extern const struct address_space_operations jfs_aops; +extern const struct inode_operations jfs_dir_inode_operations; +extern const struct file_operations jfs_dir_operations; +extern const struct inode_operations jfs_file_inode_operations; +extern const struct file_operations jfs_file_operations; +extern const struct inode_operations jfs_symlink_inode_operations; +extern const struct inode_operations jfs_fast_symlink_inode_operations; +extern const struct dentry_operations jfs_ci_dentry_operations; +#endif /* _H_JFS_INODE */ diff --git a/fs/jfs/jfs_lock.h b/fs/jfs/jfs_lock.h new file mode 100644 index 000000000..feb37dd9d --- /dev/null +++ b/fs/jfs/jfs_lock.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) International Business Machines Corp., 2000-2001 + * Portions Copyright (C) Christoph Hellwig, 2001-2002 + */ +#ifndef _H_JFS_LOCK +#define _H_JFS_LOCK + +#include +#include +#include + +/* + * jfs_lock.h + */ + +/* + * Conditional sleep where condition is protected by spinlock + * + * lock_cmd and unlock_cmd take and release the spinlock + */ +#define __SLEEP_COND(wq, cond, lock_cmd, unlock_cmd) \ +do { \ + DECLARE_WAITQUEUE(__wait, current); \ + \ + add_wait_queue(&wq, &__wait); \ + for (;;) { \ + set_current_state(TASK_UNINTERRUPTIBLE);\ + if (cond) \ + break; \ + unlock_cmd; \ + io_schedule(); \ + lock_cmd; \ + } \ + __set_current_state(TASK_RUNNING); \ + remove_wait_queue(&wq, &__wait); \ +} while (0) + +#endif /* _H_JFS_LOCK */ diff --git a/fs/jfs/jfs_logmgr.c b/fs/jfs/jfs_logmgr.c new file mode 100644 index 000000000..306165e61 --- /dev/null +++ b/fs/jfs/jfs_logmgr.c @@ -0,0 +1,2485 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) International Business Machines Corp., 2000-2004 + * Portions Copyright (C) Christoph Hellwig, 2001-2002 + */ + +/* + * jfs_logmgr.c: log manager + * + * for related information, see transaction manager (jfs_txnmgr.c), and + * recovery manager (jfs_logredo.c). + * + * note: for detail, RTFS. + * + * log buffer manager: + * special purpose buffer manager supporting log i/o requirements. + * per log serial pageout of logpage + * queuing i/o requests and redrive i/o at iodone + * maintain current logpage buffer + * no caching since append only + * appropriate jfs buffer cache buffers as needed + * + * group commit: + * transactions which wrote COMMIT records in the same in-memory + * log page during the pageout of previous/current log page(s) are + * committed together by the pageout of the page. + * + * TBD lazy commit: + * transactions are committed asynchronously when the log page + * containing it COMMIT is paged out when it becomes full; + * + * serialization: + * . a per log lock serialize log write. + * . a per log lock serialize group commit. + * . a per log lock serialize log open/close; + * + * TBD log integrity: + * careful-write (ping-pong) of last logpage to recover from crash + * in overwrite. + * detection of split (out-of-order) write of physical sectors + * of last logpage via timestamp at end of each sector + * with its mirror data array at trailer). + * + * alternatives: + * lsn - 64-bit monotonically increasing integer vs + * 32-bit lspn and page eor. + */ + +#include +#include +#include +#include +#include +#include /* for sync_blockdev() */ +#include +#include +#include +#include +#include +#include +#include +#include "jfs_incore.h" +#include "jfs_filsys.h" +#include "jfs_metapage.h" +#include "jfs_superblock.h" +#include "jfs_txnmgr.h" +#include "jfs_debug.h" + + +/* + * lbuf's ready to be redriven. Protected by log_redrive_lock (jfsIO thread) + */ +static struct lbuf *log_redrive_list; +static DEFINE_SPINLOCK(log_redrive_lock); + + + + +/* + * log group commit serialization (per log) + */ + +#define LOGGC_LOCK_INIT(log) spin_lock_init(&(log)->gclock) +#define LOGGC_LOCK(log) spin_lock_irq(&(log)->gclock) +#define LOGGC_UNLOCK(log) spin_unlock_irq(&(log)->gclock) +#define LOGGC_WAKEUP(tblk) wake_up_all(&(tblk)->gcwait) + +/* + * log sync serialization (per log) + */ +#define LOGSYNC_DELTA(logsize) min((logsize)/8, 128*LOGPSIZE) +#define LOGSYNC_BARRIER(logsize) ((logsize)/4) +/* +#define LOGSYNC_DELTA(logsize) min((logsize)/4, 256*LOGPSIZE) +#define LOGSYNC_BARRIER(logsize) ((logsize)/2) +*/ + + +/* + * log buffer cache synchronization + */ +static DEFINE_SPINLOCK(jfsLCacheLock); + +#define LCACHE_LOCK(flags) spin_lock_irqsave(&jfsLCacheLock, flags) +#define LCACHE_UNLOCK(flags) spin_unlock_irqrestore(&jfsLCacheLock, flags) + +/* + * See __SLEEP_COND in jfs_locks.h + */ +#define LCACHE_SLEEP_COND(wq, cond, flags) \ +do { \ + if (cond) \ + break; \ + __SLEEP_COND(wq, cond, LCACHE_LOCK(flags), LCACHE_UNLOCK(flags)); \ +} while (0) + +#define LCACHE_WAKEUP(event) wake_up(event) + + +/* + * lbuf buffer cache (lCache) control + */ +/* log buffer manager pageout control (cumulative, inclusive) */ +#define lbmREAD 0x0001 +#define lbmWRITE 0x0002 /* enqueue at tail of write queue; + * init pageout if at head of queue; + */ +#define lbmRELEASE 0x0004 /* remove from write queue + * at completion of pageout; + * do not free/recycle it yet: + * caller will free it; + */ +#define lbmSYNC 0x0008 /* do not return to freelist + * when removed from write queue; + */ +#define lbmFREE 0x0010 /* return to freelist + * at completion of pageout; + * the buffer may be recycled; + */ +#define lbmDONE 0x0020 +#define lbmERROR 0x0040 +#define lbmGC 0x0080 /* lbmIODone to perform post-GC processing + * of log page + */ +#define lbmDIRECT 0x0100 + +/* + * Global list of active external journals + */ +static LIST_HEAD(jfs_external_logs); +static struct jfs_log *dummy_log; +static DEFINE_MUTEX(jfs_log_mutex); + +/* + * forward references + */ +static int lmWriteRecord(struct jfs_log * log, struct tblock * tblk, + struct lrd * lrd, struct tlock * tlck); + +static int lmNextPage(struct jfs_log * log); +static int lmLogFileSystem(struct jfs_log * log, struct jfs_sb_info *sbi, + int activate); + +static int open_inline_log(struct super_block *sb); +static int open_dummy_log(struct super_block *sb); +static int lbmLogInit(struct jfs_log * log); +static void lbmLogShutdown(struct jfs_log * log); +static struct lbuf *lbmAllocate(struct jfs_log * log, int); +static void lbmFree(struct lbuf * bp); +static void lbmfree(struct lbuf * bp); +static int lbmRead(struct jfs_log * log, int pn, struct lbuf ** bpp); +static void lbmWrite(struct jfs_log * log, struct lbuf * bp, int flag, int cant_block); +static void lbmDirectWrite(struct jfs_log * log, struct lbuf * bp, int flag); +static int lbmIOWait(struct lbuf * bp, int flag); +static bio_end_io_t lbmIODone; +static void lbmStartIO(struct lbuf * bp); +static void lmGCwrite(struct jfs_log * log, int cant_block); +static int lmLogSync(struct jfs_log * log, int hard_sync); + + + +/* + * statistics + */ +#ifdef CONFIG_JFS_STATISTICS +static struct lmStat { + uint commit; /* # of commit */ + uint pagedone; /* # of page written */ + uint submitted; /* # of pages submitted */ + uint full_page; /* # of full pages submitted */ + uint partial_page; /* # of partial pages submitted */ +} lmStat; +#endif + +static void write_special_inodes(struct jfs_log *log, + int (*writer)(struct address_space *)) +{ + struct jfs_sb_info *sbi; + + list_for_each_entry(sbi, &log->sb_list, log_list) { + /* These pointers can be NULL before list_del during umount */ + if (sbi->ipbmap) + writer(sbi->ipbmap->i_mapping); + if (sbi->ipimap) + writer(sbi->ipimap->i_mapping); + if (sbi->direct_inode) + writer(sbi->direct_inode->i_mapping); + } +} + +/* + * NAME: lmLog() + * + * FUNCTION: write a log record; + * + * PARAMETER: + * + * RETURN: lsn - offset to the next log record to write (end-of-log); + * -1 - error; + * + * note: todo: log error handler + */ +int lmLog(struct jfs_log * log, struct tblock * tblk, struct lrd * lrd, + struct tlock * tlck) +{ + int lsn; + int diffp, difft; + struct metapage *mp = NULL; + unsigned long flags; + + jfs_info("lmLog: log:0x%p tblk:0x%p, lrd:0x%p tlck:0x%p", + log, tblk, lrd, tlck); + + LOG_LOCK(log); + + /* log by (out-of-transaction) JFS ? */ + if (tblk == NULL) + goto writeRecord; + + /* log from page ? */ + if (tlck == NULL || + tlck->type & tlckBTROOT || (mp = tlck->mp) == NULL) + goto writeRecord; + + /* + * initialize/update page/transaction recovery lsn + */ + lsn = log->lsn; + + LOGSYNC_LOCK(log, flags); + + /* + * initialize page lsn if first log write of the page + */ + if (mp->lsn == 0) { + mp->log = log; + mp->lsn = lsn; + log->count++; + + /* insert page at tail of logsynclist */ + list_add_tail(&mp->synclist, &log->synclist); + } + + /* + * initialize/update lsn of tblock of the page + * + * transaction inherits oldest lsn of pages associated + * with allocation/deallocation of resources (their + * log records are used to reconstruct allocation map + * at recovery time: inode for inode allocation map, + * B+-tree index of extent descriptors for block + * allocation map); + * allocation map pages inherit transaction lsn at + * commit time to allow forwarding log syncpt past log + * records associated with allocation/deallocation of + * resources only after persistent map of these map pages + * have been updated and propagated to home. + */ + /* + * initialize transaction lsn: + */ + if (tblk->lsn == 0) { + /* inherit lsn of its first page logged */ + tblk->lsn = mp->lsn; + log->count++; + + /* insert tblock after the page on logsynclist */ + list_add(&tblk->synclist, &mp->synclist); + } + /* + * update transaction lsn: + */ + else { + /* inherit oldest/smallest lsn of page */ + logdiff(diffp, mp->lsn, log); + logdiff(difft, tblk->lsn, log); + if (diffp < difft) { + /* update tblock lsn with page lsn */ + tblk->lsn = mp->lsn; + + /* move tblock after page on logsynclist */ + list_move(&tblk->synclist, &mp->synclist); + } + } + + LOGSYNC_UNLOCK(log, flags); + + /* + * write the log record + */ + writeRecord: + lsn = lmWriteRecord(log, tblk, lrd, tlck); + + /* + * forward log syncpt if log reached next syncpt trigger + */ + logdiff(diffp, lsn, log); + if (diffp >= log->nextsync) + lsn = lmLogSync(log, 0); + + /* update end-of-log lsn */ + log->lsn = lsn; + + LOG_UNLOCK(log); + + /* return end-of-log address */ + return lsn; +} + +/* + * NAME: lmWriteRecord() + * + * FUNCTION: move the log record to current log page + * + * PARAMETER: cd - commit descriptor + * + * RETURN: end-of-log address + * + * serialization: LOG_LOCK() held on entry/exit + */ +static int +lmWriteRecord(struct jfs_log * log, struct tblock * tblk, struct lrd * lrd, + struct tlock * tlck) +{ + int lsn = 0; /* end-of-log address */ + struct lbuf *bp; /* dst log page buffer */ + struct logpage *lp; /* dst log page */ + caddr_t dst; /* destination address in log page */ + int dstoffset; /* end-of-log offset in log page */ + int freespace; /* free space in log page */ + caddr_t p; /* src meta-data page */ + caddr_t src; + int srclen; + int nbytes; /* number of bytes to move */ + int i; + int len; + struct linelock *linelock; + struct lv *lv; + struct lvd *lvd; + int l2linesize; + + len = 0; + + /* retrieve destination log page to write */ + bp = (struct lbuf *) log->bp; + lp = (struct logpage *) bp->l_ldata; + dstoffset = log->eor; + + /* any log data to write ? */ + if (tlck == NULL) + goto moveLrd; + + /* + * move log record data + */ + /* retrieve source meta-data page to log */ + if (tlck->flag & tlckPAGELOCK) { + p = (caddr_t) (tlck->mp->data); + linelock = (struct linelock *) & tlck->lock; + } + /* retrieve source in-memory inode to log */ + else if (tlck->flag & tlckINODELOCK) { + if (tlck->type & tlckDTREE) + p = (caddr_t) &JFS_IP(tlck->ip)->i_dtroot; + else + p = (caddr_t) &JFS_IP(tlck->ip)->i_xtroot; + linelock = (struct linelock *) & tlck->lock; + } + else { + jfs_err("lmWriteRecord: UFO tlck:0x%p", tlck); + return 0; /* Probably should trap */ + } + l2linesize = linelock->l2linesize; + + moveData: + ASSERT(linelock->index <= linelock->maxcnt); + + lv = linelock->lv; + for (i = 0; i < linelock->index; i++, lv++) { + if (lv->length == 0) + continue; + + /* is page full ? */ + if (dstoffset >= LOGPSIZE - LOGPTLRSIZE) { + /* page become full: move on to next page */ + lmNextPage(log); + + bp = log->bp; + lp = (struct logpage *) bp->l_ldata; + dstoffset = LOGPHDRSIZE; + } + + /* + * move log vector data + */ + src = (u8 *) p + (lv->offset << l2linesize); + srclen = lv->length << l2linesize; + len += srclen; + while (srclen > 0) { + freespace = (LOGPSIZE - LOGPTLRSIZE) - dstoffset; + nbytes = min(freespace, srclen); + dst = (caddr_t) lp + dstoffset; + memcpy(dst, src, nbytes); + dstoffset += nbytes; + + /* is page not full ? */ + if (dstoffset < LOGPSIZE - LOGPTLRSIZE) + break; + + /* page become full: move on to next page */ + lmNextPage(log); + + bp = (struct lbuf *) log->bp; + lp = (struct logpage *) bp->l_ldata; + dstoffset = LOGPHDRSIZE; + + srclen -= nbytes; + src += nbytes; + } + + /* + * move log vector descriptor + */ + len += 4; + lvd = (struct lvd *) ((caddr_t) lp + dstoffset); + lvd->offset = cpu_to_le16(lv->offset); + lvd->length = cpu_to_le16(lv->length); + dstoffset += 4; + jfs_info("lmWriteRecord: lv offset:%d length:%d", + lv->offset, lv->length); + } + + if ((i = linelock->next)) { + linelock = (struct linelock *) lid_to_tlock(i); + goto moveData; + } + + /* + * move log record descriptor + */ + moveLrd: + lrd->length = cpu_to_le16(len); + + src = (caddr_t) lrd; + srclen = LOGRDSIZE; + + while (srclen > 0) { + freespace = (LOGPSIZE - LOGPTLRSIZE) - dstoffset; + nbytes = min(freespace, srclen); + dst = (caddr_t) lp + dstoffset; + memcpy(dst, src, nbytes); + + dstoffset += nbytes; + srclen -= nbytes; + + /* are there more to move than freespace of page ? */ + if (srclen) + goto pageFull; + + /* + * end of log record descriptor + */ + + /* update last log record eor */ + log->eor = dstoffset; + bp->l_eor = dstoffset; + lsn = (log->page << L2LOGPSIZE) + dstoffset; + + if (lrd->type & cpu_to_le16(LOG_COMMIT)) { + tblk->clsn = lsn; + jfs_info("wr: tclsn:0x%x, beor:0x%x", tblk->clsn, + bp->l_eor); + + INCREMENT(lmStat.commit); /* # of commit */ + + /* + * enqueue tblock for group commit: + * + * enqueue tblock of non-trivial/synchronous COMMIT + * at tail of group commit queue + * (trivial/asynchronous COMMITs are ignored by + * group commit.) + */ + LOGGC_LOCK(log); + + /* init tblock gc state */ + tblk->flag = tblkGC_QUEUE; + tblk->bp = log->bp; + tblk->pn = log->page; + tblk->eor = log->eor; + + /* enqueue transaction to commit queue */ + list_add_tail(&tblk->cqueue, &log->cqueue); + + LOGGC_UNLOCK(log); + } + + jfs_info("lmWriteRecord: lrd:0x%04x bp:0x%p pn:%d eor:0x%x", + le16_to_cpu(lrd->type), log->bp, log->page, dstoffset); + + /* page not full ? */ + if (dstoffset < LOGPSIZE - LOGPTLRSIZE) + return lsn; + + pageFull: + /* page become full: move on to next page */ + lmNextPage(log); + + bp = (struct lbuf *) log->bp; + lp = (struct logpage *) bp->l_ldata; + dstoffset = LOGPHDRSIZE; + src += nbytes; + } + + return lsn; +} + + +/* + * NAME: lmNextPage() + * + * FUNCTION: write current page and allocate next page. + * + * PARAMETER: log + * + * RETURN: 0 + * + * serialization: LOG_LOCK() held on entry/exit + */ +static int lmNextPage(struct jfs_log * log) +{ + struct logpage *lp; + int lspn; /* log sequence page number */ + int pn; /* current page number */ + struct lbuf *bp; + struct lbuf *nextbp; + struct tblock *tblk; + + /* get current log page number and log sequence page number */ + pn = log->page; + bp = log->bp; + lp = (struct logpage *) bp->l_ldata; + lspn = le32_to_cpu(lp->h.page); + + LOGGC_LOCK(log); + + /* + * write or queue the full page at the tail of write queue + */ + /* get the tail tblk on commit queue */ + if (list_empty(&log->cqueue)) + tblk = NULL; + else + tblk = list_entry(log->cqueue.prev, struct tblock, cqueue); + + /* every tblk who has COMMIT record on the current page, + * and has not been committed, must be on commit queue + * since tblk is queued at commit queueu at the time + * of writing its COMMIT record on the page before + * page becomes full (even though the tblk thread + * who wrote COMMIT record may have been suspended + * currently); + */ + + /* is page bound with outstanding tail tblk ? */ + if (tblk && tblk->pn == pn) { + /* mark tblk for end-of-page */ + tblk->flag |= tblkGC_EOP; + + if (log->cflag & logGC_PAGEOUT) { + /* if page is not already on write queue, + * just enqueue (no lbmWRITE to prevent redrive) + * buffer to wqueue to ensure correct serial order + * of the pages since log pages will be added + * continuously + */ + if (bp->l_wqnext == NULL) + lbmWrite(log, bp, 0, 0); + } else { + /* + * No current GC leader, initiate group commit + */ + log->cflag |= logGC_PAGEOUT; + lmGCwrite(log, 0); + } + } + /* page is not bound with outstanding tblk: + * init write or mark it to be redriven (lbmWRITE) + */ + else { + /* finalize the page */ + bp->l_ceor = bp->l_eor; + lp->h.eor = lp->t.eor = cpu_to_le16(bp->l_ceor); + lbmWrite(log, bp, lbmWRITE | lbmRELEASE | lbmFREE, 0); + } + LOGGC_UNLOCK(log); + + /* + * allocate/initialize next page + */ + /* if log wraps, the first data page of log is 2 + * (0 never used, 1 is superblock). + */ + log->page = (pn == log->size - 1) ? 2 : pn + 1; + log->eor = LOGPHDRSIZE; /* ? valid page empty/full at logRedo() */ + + /* allocate/initialize next log page buffer */ + nextbp = lbmAllocate(log, log->page); + nextbp->l_eor = log->eor; + log->bp = nextbp; + + /* initialize next log page */ + lp = (struct logpage *) nextbp->l_ldata; + lp->h.page = lp->t.page = cpu_to_le32(lspn + 1); + lp->h.eor = lp->t.eor = cpu_to_le16(LOGPHDRSIZE); + + return 0; +} + + +/* + * NAME: lmGroupCommit() + * + * FUNCTION: group commit + * initiate pageout of the pages with COMMIT in the order of + * page number - redrive pageout of the page at the head of + * pageout queue until full page has been written. + * + * RETURN: + * + * NOTE: + * LOGGC_LOCK serializes log group commit queue, and + * transaction blocks on the commit queue. + * N.B. LOG_LOCK is NOT held during lmGroupCommit(). + */ +int lmGroupCommit(struct jfs_log * log, struct tblock * tblk) +{ + int rc = 0; + + LOGGC_LOCK(log); + + /* group committed already ? */ + if (tblk->flag & tblkGC_COMMITTED) { + if (tblk->flag & tblkGC_ERROR) + rc = -EIO; + + LOGGC_UNLOCK(log); + return rc; + } + jfs_info("lmGroup Commit: tblk = 0x%p, gcrtc = %d", tblk, log->gcrtc); + + if (tblk->xflag & COMMIT_LAZY) + tblk->flag |= tblkGC_LAZY; + + if ((!(log->cflag & logGC_PAGEOUT)) && (!list_empty(&log->cqueue)) && + (!(tblk->xflag & COMMIT_LAZY) || test_bit(log_FLUSH, &log->flag) + || jfs_tlocks_low)) { + /* + * No pageout in progress + * + * start group commit as its group leader. + */ + log->cflag |= logGC_PAGEOUT; + + lmGCwrite(log, 0); + } + + if (tblk->xflag & COMMIT_LAZY) { + /* + * Lazy transactions can leave now + */ + LOGGC_UNLOCK(log); + return 0; + } + + /* lmGCwrite gives up LOGGC_LOCK, check again */ + + if (tblk->flag & tblkGC_COMMITTED) { + if (tblk->flag & tblkGC_ERROR) + rc = -EIO; + + LOGGC_UNLOCK(log); + return rc; + } + + /* upcount transaction waiting for completion + */ + log->gcrtc++; + tblk->flag |= tblkGC_READY; + + __SLEEP_COND(tblk->gcwait, (tblk->flag & tblkGC_COMMITTED), + LOGGC_LOCK(log), LOGGC_UNLOCK(log)); + + /* removed from commit queue */ + if (tblk->flag & tblkGC_ERROR) + rc = -EIO; + + LOGGC_UNLOCK(log); + return rc; +} + +/* + * NAME: lmGCwrite() + * + * FUNCTION: group commit write + * initiate write of log page, building a group of all transactions + * with commit records on that page. + * + * RETURN: None + * + * NOTE: + * LOGGC_LOCK must be held by caller. + * N.B. LOG_LOCK is NOT held during lmGroupCommit(). + */ +static void lmGCwrite(struct jfs_log * log, int cant_write) +{ + struct lbuf *bp; + struct logpage *lp; + int gcpn; /* group commit page number */ + struct tblock *tblk; + struct tblock *xtblk = NULL; + + /* + * build the commit group of a log page + * + * scan commit queue and make a commit group of all + * transactions with COMMIT records on the same log page. + */ + /* get the head tblk on the commit queue */ + gcpn = list_entry(log->cqueue.next, struct tblock, cqueue)->pn; + + list_for_each_entry(tblk, &log->cqueue, cqueue) { + if (tblk->pn != gcpn) + break; + + xtblk = tblk; + + /* state transition: (QUEUE, READY) -> COMMIT */ + tblk->flag |= tblkGC_COMMIT; + } + tblk = xtblk; /* last tblk of the page */ + + /* + * pageout to commit transactions on the log page. + */ + bp = (struct lbuf *) tblk->bp; + lp = (struct logpage *) bp->l_ldata; + /* is page already full ? */ + if (tblk->flag & tblkGC_EOP) { + /* mark page to free at end of group commit of the page */ + tblk->flag &= ~tblkGC_EOP; + tblk->flag |= tblkGC_FREE; + bp->l_ceor = bp->l_eor; + lp->h.eor = lp->t.eor = cpu_to_le16(bp->l_ceor); + lbmWrite(log, bp, lbmWRITE | lbmRELEASE | lbmGC, + cant_write); + INCREMENT(lmStat.full_page); + } + /* page is not yet full */ + else { + bp->l_ceor = tblk->eor; /* ? bp->l_ceor = bp->l_eor; */ + lp->h.eor = lp->t.eor = cpu_to_le16(bp->l_ceor); + lbmWrite(log, bp, lbmWRITE | lbmGC, cant_write); + INCREMENT(lmStat.partial_page); + } +} + +/* + * NAME: lmPostGC() + * + * FUNCTION: group commit post-processing + * Processes transactions after their commit records have been written + * to disk, redriving log I/O if necessary. + * + * RETURN: None + * + * NOTE: + * This routine is called a interrupt time by lbmIODone + */ +static void lmPostGC(struct lbuf * bp) +{ + unsigned long flags; + struct jfs_log *log = bp->l_log; + struct logpage *lp; + struct tblock *tblk, *temp; + + //LOGGC_LOCK(log); + spin_lock_irqsave(&log->gclock, flags); + /* + * current pageout of group commit completed. + * + * remove/wakeup transactions from commit queue who were + * group committed with the current log page + */ + list_for_each_entry_safe(tblk, temp, &log->cqueue, cqueue) { + if (!(tblk->flag & tblkGC_COMMIT)) + break; + /* if transaction was marked GC_COMMIT then + * it has been shipped in the current pageout + * and made it to disk - it is committed. + */ + + if (bp->l_flag & lbmERROR) + tblk->flag |= tblkGC_ERROR; + + /* remove it from the commit queue */ + list_del(&tblk->cqueue); + tblk->flag &= ~tblkGC_QUEUE; + + if (tblk == log->flush_tblk) { + /* we can stop flushing the log now */ + clear_bit(log_FLUSH, &log->flag); + log->flush_tblk = NULL; + } + + jfs_info("lmPostGC: tblk = 0x%p, flag = 0x%x", tblk, + tblk->flag); + + if (!(tblk->xflag & COMMIT_FORCE)) + /* + * Hand tblk over to lazy commit thread + */ + txLazyUnlock(tblk); + else { + /* state transition: COMMIT -> COMMITTED */ + tblk->flag |= tblkGC_COMMITTED; + + if (tblk->flag & tblkGC_READY) + log->gcrtc--; + + LOGGC_WAKEUP(tblk); + } + + /* was page full before pageout ? + * (and this is the last tblk bound with the page) + */ + if (tblk->flag & tblkGC_FREE) + lbmFree(bp); + /* did page become full after pageout ? + * (and this is the last tblk bound with the page) + */ + else if (tblk->flag & tblkGC_EOP) { + /* finalize the page */ + lp = (struct logpage *) bp->l_ldata; + bp->l_ceor = bp->l_eor; + lp->h.eor = lp->t.eor = cpu_to_le16(bp->l_eor); + jfs_info("lmPostGC: calling lbmWrite"); + lbmWrite(log, bp, lbmWRITE | lbmRELEASE | lbmFREE, + 1); + } + + } + + /* are there any transactions who have entered lnGroupCommit() + * (whose COMMITs are after that of the last log page written. + * They are waiting for new group commit (above at (SLEEP 1)) + * or lazy transactions are on a full (queued) log page, + * select the latest ready transaction as new group leader and + * wake her up to lead her group. + */ + if ((!list_empty(&log->cqueue)) && + ((log->gcrtc > 0) || (tblk->bp->l_wqnext != NULL) || + test_bit(log_FLUSH, &log->flag) || jfs_tlocks_low)) + /* + * Call lmGCwrite with new group leader + */ + lmGCwrite(log, 1); + + /* no transaction are ready yet (transactions are only just + * queued (GC_QUEUE) and not entered for group commit yet). + * the first transaction entering group commit + * will elect herself as new group leader. + */ + else + log->cflag &= ~logGC_PAGEOUT; + + //LOGGC_UNLOCK(log); + spin_unlock_irqrestore(&log->gclock, flags); + return; +} + +/* + * NAME: lmLogSync() + * + * FUNCTION: write log SYNCPT record for specified log + * if new sync address is available + * (normally the case if sync() is executed by back-ground + * process). + * calculate new value of i_nextsync which determines when + * this code is called again. + * + * PARAMETERS: log - log structure + * hard_sync - 1 to force all metadata to be written + * + * RETURN: 0 + * + * serialization: LOG_LOCK() held on entry/exit + */ +static int lmLogSync(struct jfs_log * log, int hard_sync) +{ + int logsize; + int written; /* written since last syncpt */ + int free; /* free space left available */ + int delta; /* additional delta to write normally */ + int more; /* additional write granted */ + struct lrd lrd; + int lsn; + struct logsyncblk *lp; + unsigned long flags; + + /* push dirty metapages out to disk */ + if (hard_sync) + write_special_inodes(log, filemap_fdatawrite); + else + write_special_inodes(log, filemap_flush); + + /* + * forward syncpt + */ + /* if last sync is same as last syncpt, + * invoke sync point forward processing to update sync. + */ + + if (log->sync == log->syncpt) { + LOGSYNC_LOCK(log, flags); + if (list_empty(&log->synclist)) + log->sync = log->lsn; + else { + lp = list_entry(log->synclist.next, + struct logsyncblk, synclist); + log->sync = lp->lsn; + } + LOGSYNC_UNLOCK(log, flags); + + } + + /* if sync is different from last syncpt, + * write a SYNCPT record with syncpt = sync. + * reset syncpt = sync + */ + if (log->sync != log->syncpt) { + lrd.logtid = 0; + lrd.backchain = 0; + lrd.type = cpu_to_le16(LOG_SYNCPT); + lrd.length = 0; + lrd.log.syncpt.sync = cpu_to_le32(log->sync); + lsn = lmWriteRecord(log, NULL, &lrd, NULL); + + log->syncpt = log->sync; + } else + lsn = log->lsn; + + /* + * setup next syncpt trigger (SWAG) + */ + logsize = log->logsize; + + logdiff(written, lsn, log); + free = logsize - written; + delta = LOGSYNC_DELTA(logsize); + more = min(free / 2, delta); + if (more < 2 * LOGPSIZE) { + jfs_warn("\n ... Log Wrap ... Log Wrap ... Log Wrap ...\n"); + /* + * log wrapping + * + * option 1 - panic ? No.! + * option 2 - shutdown file systems + * associated with log ? + * option 3 - extend log ? + * option 4 - second chance + * + * mark log wrapped, and continue. + * when all active transactions are completed, + * mark log valid for recovery. + * if crashed during invalid state, log state + * implies invalid log, forcing fsck(). + */ + /* mark log state log wrap in log superblock */ + /* log->state = LOGWRAP; */ + + /* reset sync point computation */ + log->syncpt = log->sync = lsn; + log->nextsync = delta; + } else + /* next syncpt trigger = written + more */ + log->nextsync = written + more; + + /* if number of bytes written from last sync point is more + * than 1/4 of the log size, stop new transactions from + * starting until all current transactions are completed + * by setting syncbarrier flag. + */ + if (!test_bit(log_SYNCBARRIER, &log->flag) && + (written > LOGSYNC_BARRIER(logsize)) && log->active) { + set_bit(log_SYNCBARRIER, &log->flag); + jfs_info("log barrier on: lsn=0x%x syncpt=0x%x", lsn, + log->syncpt); + /* + * We may have to initiate group commit + */ + jfs_flush_journal(log, 0); + } + + return lsn; +} + +/* + * NAME: jfs_syncpt + * + * FUNCTION: write log SYNCPT record for specified log + * + * PARAMETERS: log - log structure + * hard_sync - set to 1 to force metadata to be written + */ +void jfs_syncpt(struct jfs_log *log, int hard_sync) +{ LOG_LOCK(log); + if (!test_bit(log_QUIESCE, &log->flag)) + lmLogSync(log, hard_sync); + LOG_UNLOCK(log); +} + +/* + * NAME: lmLogOpen() + * + * FUNCTION: open the log on first open; + * insert filesystem in the active list of the log. + * + * PARAMETER: ipmnt - file system mount inode + * iplog - log inode (out) + * + * RETURN: + * + * serialization: + */ +int lmLogOpen(struct super_block *sb) +{ + int rc; + struct file *bdev_file; + struct jfs_log *log; + struct jfs_sb_info *sbi = JFS_SBI(sb); + + if (sbi->flag & JFS_NOINTEGRITY) + return open_dummy_log(sb); + + if (sbi->mntflag & JFS_INLINELOG) + return open_inline_log(sb); + + mutex_lock(&jfs_log_mutex); + list_for_each_entry(log, &jfs_external_logs, journal_list) { + if (file_bdev(log->bdev_file)->bd_dev == sbi->logdev) { + if (!uuid_equal(&log->uuid, &sbi->loguuid)) { + jfs_warn("wrong uuid on JFS journal"); + mutex_unlock(&jfs_log_mutex); + return -EINVAL; + } + /* + * add file system to log active file system list + */ + if ((rc = lmLogFileSystem(log, sbi, 1))) { + mutex_unlock(&jfs_log_mutex); + return rc; + } + goto journal_found; + } + } + + if (!(log = kzalloc_obj(struct jfs_log))) { + mutex_unlock(&jfs_log_mutex); + return -ENOMEM; + } + INIT_LIST_HEAD(&log->sb_list); + init_waitqueue_head(&log->syncwait); + + /* + * external log as separate logical volume + * + * file systems to log may have n-to-1 relationship; + */ + + bdev_file = bdev_file_open_by_dev(sbi->logdev, + BLK_OPEN_READ | BLK_OPEN_WRITE, log, NULL); + if (IS_ERR(bdev_file)) { + rc = PTR_ERR(bdev_file); + goto free; + } + + log->bdev_file = bdev_file; + uuid_copy(&log->uuid, &sbi->loguuid); + + /* + * initialize log: + */ + if ((rc = lmLogInit(log))) + goto close; + + list_add(&log->journal_list, &jfs_external_logs); + + /* + * add file system to log active file system list + */ + if ((rc = lmLogFileSystem(log, sbi, 1))) + goto shutdown; + +journal_found: + LOG_LOCK(log); + list_add(&sbi->log_list, &log->sb_list); + sbi->log = log; + LOG_UNLOCK(log); + + mutex_unlock(&jfs_log_mutex); + return 0; + + /* + * unwind on error + */ + shutdown: /* unwind lbmLogInit() */ + list_del(&log->journal_list); + lbmLogShutdown(log); + + close: /* close external log device */ + bdev_fput(bdev_file); + + free: /* free log descriptor */ + mutex_unlock(&jfs_log_mutex); + kfree(log); + + jfs_warn("lmLogOpen: exit(%d)", rc); + return rc; +} + +static int open_inline_log(struct super_block *sb) +{ + struct jfs_log *log; + int rc; + + if (!(log = kzalloc_obj(struct jfs_log))) + return -ENOMEM; + INIT_LIST_HEAD(&log->sb_list); + init_waitqueue_head(&log->syncwait); + + set_bit(log_INLINELOG, &log->flag); + log->bdev_file = sb->s_bdev_file; + log->base = addressPXD(&JFS_SBI(sb)->logpxd); + log->size = lengthPXD(&JFS_SBI(sb)->logpxd) >> + (L2LOGPSIZE - sb->s_blocksize_bits); + log->l2bsize = sb->s_blocksize_bits; + ASSERT(L2LOGPSIZE >= sb->s_blocksize_bits); + + /* + * initialize log. + */ + if ((rc = lmLogInit(log))) { + kfree(log); + jfs_warn("lmLogOpen: exit(%d)", rc); + return rc; + } + + list_add(&JFS_SBI(sb)->log_list, &log->sb_list); + JFS_SBI(sb)->log = log; + + return rc; +} + +static int open_dummy_log(struct super_block *sb) +{ + int rc; + + mutex_lock(&jfs_log_mutex); + if (!dummy_log) { + dummy_log = kzalloc_obj(struct jfs_log); + if (!dummy_log) { + mutex_unlock(&jfs_log_mutex); + return -ENOMEM; + } + INIT_LIST_HEAD(&dummy_log->sb_list); + init_waitqueue_head(&dummy_log->syncwait); + dummy_log->no_integrity = 1; + /* Make up some stuff */ + dummy_log->size = 1024; + rc = lmLogInit(dummy_log); + if (rc) { + kfree(dummy_log); + dummy_log = NULL; + mutex_unlock(&jfs_log_mutex); + return rc; + } + } + + LOG_LOCK(dummy_log); + list_add(&JFS_SBI(sb)->log_list, &dummy_log->sb_list); + JFS_SBI(sb)->log = dummy_log; + LOG_UNLOCK(dummy_log); + mutex_unlock(&jfs_log_mutex); + + return 0; +} + +/* + * NAME: lmLogInit() + * + * FUNCTION: log initialization at first log open. + * + * logredo() (or logformat()) should have been run previously. + * initialize the log from log superblock. + * set the log state in the superblock to LOGMOUNT and + * write SYNCPT log record. + * + * PARAMETER: log - log structure + * + * RETURN: 0 - if ok + * -EINVAL - bad log magic number or superblock dirty + * error returned from logwait() + * + * serialization: single first open thread + */ +int lmLogInit(struct jfs_log * log) +{ + int rc = 0; + struct lrd lrd; + struct logsuper *logsuper; + struct lbuf *bpsuper; + struct lbuf *bp; + struct logpage *lp; + int lsn = 0; + + jfs_info("lmLogInit: log:0x%p", log); + + /* initialize the group commit serialization lock */ + LOGGC_LOCK_INIT(log); + + /* allocate/initialize the log write serialization lock */ + LOG_LOCK_INIT(log); + + LOGSYNC_LOCK_INIT(log); + + INIT_LIST_HEAD(&log->synclist); + + INIT_LIST_HEAD(&log->cqueue); + log->flush_tblk = NULL; + + log->count = 0; + + /* + * initialize log i/o + */ + if ((rc = lbmLogInit(log))) + return rc; + + if (!test_bit(log_INLINELOG, &log->flag)) + log->l2bsize = L2LOGPSIZE; + + /* check for disabled journaling to disk */ + if (log->no_integrity) { + /* + * Journal pages will still be filled. When the time comes + * to actually do the I/O, the write is not done, and the + * endio routine is called directly. + */ + bp = lbmAllocate(log , 0); + log->bp = bp; + bp->l_pn = bp->l_eor = 0; + } else { + /* + * validate log superblock + */ + if ((rc = lbmRead(log, 1, &bpsuper))) + goto errout10; + + logsuper = (struct logsuper *) bpsuper->l_ldata; + + if (logsuper->magic != cpu_to_le32(LOGMAGIC)) { + jfs_warn("*** Log Format Error ! ***"); + rc = -EINVAL; + goto errout20; + } + + /* logredo() should have been run successfully. */ + if (logsuper->state != cpu_to_le32(LOGREDONE)) { + jfs_warn("*** Log Is Dirty ! ***"); + rc = -EINVAL; + goto errout20; + } + + /* initialize log from log superblock */ + if (test_bit(log_INLINELOG,&log->flag)) { + if (log->size != le32_to_cpu(logsuper->size)) { + rc = -EINVAL; + goto errout20; + } + jfs_info("lmLogInit: inline log:0x%p base:0x%Lx size:0x%x", + log, (unsigned long long)log->base, log->size); + } else { + if (!uuid_equal(&logsuper->uuid, &log->uuid)) { + jfs_warn("wrong uuid on JFS log device"); + rc = -EINVAL; + goto errout20; + } + log->size = le32_to_cpu(logsuper->size); + log->l2bsize = le32_to_cpu(logsuper->l2bsize); + jfs_info("lmLogInit: external log:0x%p base:0x%Lx size:0x%x", + log, (unsigned long long)log->base, log->size); + } + + log->page = le32_to_cpu(logsuper->end) / LOGPSIZE; + log->eor = le32_to_cpu(logsuper->end) - (LOGPSIZE * log->page); + + /* + * initialize for log append write mode + */ + /* establish current/end-of-log page/buffer */ + if ((rc = lbmRead(log, log->page, &bp))) + goto errout20; + + lp = (struct logpage *) bp->l_ldata; + + jfs_info("lmLogInit: lsn:0x%x page:%d eor:%d:%d", + le32_to_cpu(logsuper->end), log->page, log->eor, + le16_to_cpu(lp->h.eor)); + + log->bp = bp; + bp->l_pn = log->page; + bp->l_eor = log->eor; + + /* if current page is full, move on to next page */ + if (log->eor >= LOGPSIZE - LOGPTLRSIZE) + lmNextPage(log); + + /* + * initialize log syncpoint + */ + /* + * write the first SYNCPT record with syncpoint = 0 + * (i.e., log redo up to HERE !); + * remove current page from lbm write queue at end of pageout + * (to write log superblock update), but do not release to + * freelist; + */ + lrd.logtid = 0; + lrd.backchain = 0; + lrd.type = cpu_to_le16(LOG_SYNCPT); + lrd.length = 0; + lrd.log.syncpt.sync = 0; + lsn = lmWriteRecord(log, NULL, &lrd, NULL); + bp = log->bp; + bp->l_ceor = bp->l_eor; + lp = (struct logpage *) bp->l_ldata; + lp->h.eor = lp->t.eor = cpu_to_le16(bp->l_eor); + lbmWrite(log, bp, lbmWRITE | lbmSYNC, 0); + if ((rc = lbmIOWait(bp, 0))) + goto errout30; + + /* + * update/write superblock + */ + logsuper->state = cpu_to_le32(LOGMOUNT); + log->serial = le32_to_cpu(logsuper->serial) + 1; + logsuper->serial = cpu_to_le32(log->serial); + lbmDirectWrite(log, bpsuper, lbmWRITE | lbmRELEASE | lbmSYNC); + if ((rc = lbmIOWait(bpsuper, lbmFREE))) + goto errout30; + } + + /* initialize logsync parameters */ + log->logsize = (log->size - 2) << L2LOGPSIZE; + log->lsn = lsn; + log->syncpt = lsn; + log->sync = log->syncpt; + log->nextsync = LOGSYNC_DELTA(log->logsize); + + jfs_info("lmLogInit: lsn:0x%x syncpt:0x%x sync:0x%x", + log->lsn, log->syncpt, log->sync); + + /* + * initialize for lazy/group commit + */ + log->clsn = lsn; + + return 0; + + /* + * unwind on error + */ + errout30: /* release log page */ + log->wqueue = NULL; + bp->l_wqnext = NULL; + lbmFree(bp); + + errout20: /* release log superblock */ + lbmFree(bpsuper); + + errout10: /* unwind lbmLogInit() */ + lbmLogShutdown(log); + + jfs_warn("lmLogInit: exit(%d)", rc); + return rc; +} + + +/* + * NAME: lmLogClose() + * + * FUNCTION: remove file system from active list of log + * and close it on last close. + * + * PARAMETER: sb - superblock + * + * RETURN: errors from subroutines + * + * serialization: + */ +int lmLogClose(struct super_block *sb) +{ + struct jfs_sb_info *sbi = JFS_SBI(sb); + struct jfs_log *log = sbi->log; + struct file *bdev_file; + int rc = 0; + + jfs_info("lmLogClose: log:0x%p", log); + + mutex_lock(&jfs_log_mutex); + LOG_LOCK(log); + list_del(&sbi->log_list); + LOG_UNLOCK(log); + sbi->log = NULL; + + /* + * We need to make sure all of the "written" metapages + * actually make it to disk + */ + sync_blockdev(sb->s_bdev); + + if (test_bit(log_INLINELOG, &log->flag)) { + /* + * in-line log in host file system + */ + rc = lmLogShutdown(log); + kfree(log); + goto out; + } + + if (!log->no_integrity) + lmLogFileSystem(log, sbi, 0); + + if (!list_empty(&log->sb_list)) + goto out; + + /* + * TODO: ensure that the dummy_log is in a state to allow + * lbmLogShutdown to deallocate all the buffers and call + * kfree against dummy_log. For now, leave dummy_log & its + * buffers in memory, and resuse if another no-integrity mount + * is requested. + */ + if (log->no_integrity) + goto out; + + /* + * external log as separate logical volume + */ + list_del(&log->journal_list); + bdev_file = log->bdev_file; + rc = lmLogShutdown(log); + + bdev_fput(bdev_file); + + kfree(log); + + out: + mutex_unlock(&jfs_log_mutex); + jfs_info("lmLogClose: exit(%d)", rc); + return rc; +} + + +/* + * NAME: jfs_flush_journal() + * + * FUNCTION: initiate write of any outstanding transactions to the journal + * and optionally wait until they are all written to disk + * + * wait == 0 flush until latest txn is committed, don't wait + * wait == 1 flush until latest txn is committed, wait + * wait > 1 flush until all txn's are complete, wait + */ +void jfs_flush_journal(struct jfs_log *log, int wait) +{ + int i; + struct tblock *target = NULL; + + /* jfs_write_inode may call us during read-only mount */ + if (!log) + return; + + jfs_info("jfs_flush_journal: log:0x%p wait=%d", log, wait); + + LOGGC_LOCK(log); + + if (!list_empty(&log->cqueue)) { + /* + * This ensures that we will keep writing to the journal as long + * as there are unwritten commit records + */ + target = list_entry(log->cqueue.prev, struct tblock, cqueue); + + if (test_bit(log_FLUSH, &log->flag)) { + /* + * We're already flushing. + * if flush_tblk is NULL, we are flushing everything, + * so leave it that way. Otherwise, update it to the + * latest transaction + */ + if (log->flush_tblk) + log->flush_tblk = target; + } else { + /* Only flush until latest transaction is committed */ + log->flush_tblk = target; + set_bit(log_FLUSH, &log->flag); + + /* + * Initiate I/O on outstanding transactions + */ + if (!(log->cflag & logGC_PAGEOUT)) { + log->cflag |= logGC_PAGEOUT; + lmGCwrite(log, 0); + } + } + } + if ((wait > 1) || test_bit(log_SYNCBARRIER, &log->flag)) { + /* Flush until all activity complete */ + set_bit(log_FLUSH, &log->flag); + log->flush_tblk = NULL; + } + + if (wait && target && !(target->flag & tblkGC_COMMITTED)) { + DECLARE_WAITQUEUE(__wait, current); + + add_wait_queue(&target->gcwait, &__wait); + set_current_state(TASK_UNINTERRUPTIBLE); + LOGGC_UNLOCK(log); + schedule(); + LOGGC_LOCK(log); + remove_wait_queue(&target->gcwait, &__wait); + } + LOGGC_UNLOCK(log); + + if (wait < 2) + return; + + write_special_inodes(log, filemap_fdatawrite); + + /* + * If there was recent activity, we may need to wait + * for the lazycommit thread to catch up + */ + if ((!list_empty(&log->cqueue)) || !list_empty(&log->synclist)) { + for (i = 0; i < 200; i++) { /* Too much? */ + msleep(250); + write_special_inodes(log, filemap_fdatawrite); + if (list_empty(&log->cqueue) && + list_empty(&log->synclist)) + break; + } + } + assert(list_empty(&log->cqueue)); + +#ifdef CONFIG_JFS_DEBUG + if (!list_empty(&log->synclist)) { + struct logsyncblk *lp; + + printk(KERN_ERR "jfs_flush_journal: synclist not empty\n"); + list_for_each_entry(lp, &log->synclist, synclist) { + if (lp->xflag & COMMIT_PAGE) { + struct metapage *mp = (struct metapage *)lp; + print_hex_dump(KERN_ERR, "metapage: ", + DUMP_PREFIX_ADDRESS, 16, 4, + mp, sizeof(struct metapage), 0); + print_hex_dump(KERN_ERR, "page: ", + DUMP_PREFIX_ADDRESS, 16, + sizeof(long), mp->folio, + sizeof(struct page), 0); + } else + print_hex_dump(KERN_ERR, "tblock:", + DUMP_PREFIX_ADDRESS, 16, 4, + lp, sizeof(struct tblock), 0); + } + } +#else + WARN_ON(!list_empty(&log->synclist)); +#endif + clear_bit(log_FLUSH, &log->flag); +} + +/* + * NAME: lmLogShutdown() + * + * FUNCTION: log shutdown at last LogClose(). + * + * write log syncpt record. + * update super block to set redone flag to 0. + * + * PARAMETER: log - log inode + * + * RETURN: 0 - success + * + * serialization: single last close thread + */ +int lmLogShutdown(struct jfs_log * log) +{ + int rc; + struct lrd lrd; + int lsn; + struct logsuper *logsuper; + struct lbuf *bpsuper; + struct lbuf *bp; + struct logpage *lp; + + jfs_info("lmLogShutdown: log:0x%p", log); + + jfs_flush_journal(log, 2); + + /* + * write the last SYNCPT record with syncpoint = 0 + * (i.e., log redo up to HERE !) + */ + lrd.logtid = 0; + lrd.backchain = 0; + lrd.type = cpu_to_le16(LOG_SYNCPT); + lrd.length = 0; + lrd.log.syncpt.sync = 0; + + lsn = lmWriteRecord(log, NULL, &lrd, NULL); + bp = log->bp; + lp = (struct logpage *) bp->l_ldata; + lp->h.eor = lp->t.eor = cpu_to_le16(bp->l_eor); + lbmWrite(log, log->bp, lbmWRITE | lbmRELEASE | lbmSYNC, 0); + lbmIOWait(log->bp, lbmFREE); + log->bp = NULL; + + /* + * synchronous update log superblock + * mark log state as shutdown cleanly + * (i.e., Log does not need to be replayed). + */ + if ((rc = lbmRead(log, 1, &bpsuper))) + goto out; + + logsuper = (struct logsuper *) bpsuper->l_ldata; + logsuper->state = cpu_to_le32(LOGREDONE); + logsuper->end = cpu_to_le32(lsn); + lbmDirectWrite(log, bpsuper, lbmWRITE | lbmRELEASE | lbmSYNC); + rc = lbmIOWait(bpsuper, lbmFREE); + + jfs_info("lmLogShutdown: lsn:0x%x page:%d eor:%d", + lsn, log->page, log->eor); + + out: + /* + * shutdown per log i/o + */ + lbmLogShutdown(log); + + if (rc) { + jfs_warn("lmLogShutdown: exit(%d)", rc); + } + return rc; +} + + +/* + * NAME: lmLogFileSystem() + * + * FUNCTION: insert ( = true)/remove ( = false) + * file system into/from log active file system list. + * + * PARAMETE: log - pointer to logs inode. + * fsdev - kdev_t of filesystem. + * serial - pointer to returned log serial number + * activate - insert/remove device from active list. + * + * RETURN: 0 - success + * errors returned by vms_iowait(). + */ +static int lmLogFileSystem(struct jfs_log * log, struct jfs_sb_info *sbi, + int activate) +{ + int rc = 0; + int i; + struct logsuper *logsuper; + struct lbuf *bpsuper; + uuid_t *uuid = &sbi->uuid; + + /* + * insert/remove file system device to log active file system list. + */ + if ((rc = lbmRead(log, 1, &bpsuper))) + return rc; + + logsuper = (struct logsuper *) bpsuper->l_ldata; + if (activate) { + for (i = 0; i < MAX_ACTIVE; i++) + if (uuid_is_null(&logsuper->active[i].uuid)) { + uuid_copy(&logsuper->active[i].uuid, uuid); + sbi->aggregate = i; + break; + } + if (i == MAX_ACTIVE) { + jfs_warn("Too many file systems sharing journal!"); + lbmFree(bpsuper); + return -EMFILE; /* Is there a better rc? */ + } + } else { + for (i = 0; i < MAX_ACTIVE; i++) + if (uuid_equal(&logsuper->active[i].uuid, uuid)) { + uuid_copy(&logsuper->active[i].uuid, + &uuid_null); + break; + } + if (i == MAX_ACTIVE) { + jfs_warn("Somebody stomped on the journal!"); + lbmFree(bpsuper); + return -EIO; + } + + } + + /* + * synchronous write log superblock: + * + * write sidestream bypassing write queue: + * at file system mount, log super block is updated for + * activation of the file system before any log record + * (MOUNT record) of the file system, and at file system + * unmount, all meta data for the file system has been + * flushed before log super block is updated for deactivation + * of the file system. + */ + lbmDirectWrite(log, bpsuper, lbmWRITE | lbmRELEASE | lbmSYNC); + rc = lbmIOWait(bpsuper, lbmFREE); + + return rc; +} + +/* + * log buffer manager (lbm) + * ------------------------ + * + * special purpose buffer manager supporting log i/o requirements. + * + * per log write queue: + * log pageout occurs in serial order by fifo write queue and + * restricting to a single i/o in pregress at any one time. + * a circular singly-linked list + * (log->wrqueue points to the tail, and buffers are linked via + * bp->wrqueue field), and + * maintains log page in pageout ot waiting for pageout in serial pageout. + */ + +/* + * lbmLogInit() + * + * initialize per log I/O setup at lmLogInit() + */ +static int lbmLogInit(struct jfs_log * log) +{ /* log inode */ + int i; + struct lbuf *lbuf; + + jfs_info("lbmLogInit: log:0x%p", log); + + /* initialize current buffer cursor */ + log->bp = NULL; + + /* initialize log device write queue */ + log->wqueue = NULL; + + /* + * Each log has its own buffer pages allocated to it. These are + * not managed by the page cache. This ensures that a transaction + * writing to the log does not block trying to allocate a page from + * the page cache (for the log). This would be bad, since page + * allocation waits on the kswapd thread that may be committing inodes + * which would cause log activity. Was that clear? I'm trying to + * avoid deadlock here. + */ + init_waitqueue_head(&log->free_wait); + + log->lbuf_free = NULL; + + for (i = 0; i < LOGPAGES;) { + char *buffer; + uint offset; + struct page *page = alloc_page(GFP_KERNEL | __GFP_ZERO); + + if (!page) + goto error; + buffer = page_address(page); + for (offset = 0; offset < PAGE_SIZE; offset += LOGPSIZE) { + lbuf = kmalloc_obj(struct lbuf); + if (lbuf == NULL) { + if (offset == 0) + __free_page(page); + goto error; + } + if (offset) /* we already have one reference */ + get_page(page); + lbuf->l_offset = offset; + lbuf->l_ldata = buffer + offset; + lbuf->l_page = page; + lbuf->l_log = log; + init_waitqueue_head(&lbuf->l_ioevent); + + lbuf->l_freelist = log->lbuf_free; + log->lbuf_free = lbuf; + i++; + } + } + + return (0); + + error: + lbmLogShutdown(log); + return -ENOMEM; +} + + +/* + * lbmLogShutdown() + * + * finalize per log I/O setup at lmLogShutdown() + */ +static void lbmLogShutdown(struct jfs_log * log) +{ + struct lbuf *lbuf; + + jfs_info("lbmLogShutdown: log:0x%p", log); + + lbuf = log->lbuf_free; + while (lbuf) { + struct lbuf *next = lbuf->l_freelist; + __free_page(lbuf->l_page); + kfree(lbuf); + lbuf = next; + } +} + + +/* + * lbmAllocate() + * + * allocate an empty log buffer + */ +static struct lbuf *lbmAllocate(struct jfs_log * log, int pn) +{ + struct lbuf *bp; + unsigned long flags; + + /* + * recycle from log buffer freelist if any + */ + LCACHE_LOCK(flags); + LCACHE_SLEEP_COND(log->free_wait, (bp = log->lbuf_free), flags); + log->lbuf_free = bp->l_freelist; + LCACHE_UNLOCK(flags); + + bp->l_flag = 0; + + bp->l_wqnext = NULL; + bp->l_freelist = NULL; + + bp->l_pn = pn; + bp->l_blkno = log->base + (pn << (L2LOGPSIZE - log->l2bsize)); + bp->l_ceor = 0; + + return bp; +} + + +/* + * lbmFree() + * + * release a log buffer to freelist + */ +static void lbmFree(struct lbuf * bp) +{ + unsigned long flags; + + LCACHE_LOCK(flags); + + lbmfree(bp); + + LCACHE_UNLOCK(flags); +} + +static void lbmfree(struct lbuf * bp) +{ + struct jfs_log *log = bp->l_log; + + assert(bp->l_wqnext == NULL); + + /* + * return the buffer to head of freelist + */ + bp->l_freelist = log->lbuf_free; + log->lbuf_free = bp; + + wake_up(&log->free_wait); + return; +} + + +/* + * NAME: lbmRedrive + * + * FUNCTION: add a log buffer to the log redrive list + * + * PARAMETER: + * bp - log buffer + * + * NOTES: + * Takes log_redrive_lock. + */ +static inline void lbmRedrive(struct lbuf *bp) +{ + unsigned long flags; + + spin_lock_irqsave(&log_redrive_lock, flags); + bp->l_redrive_next = log_redrive_list; + log_redrive_list = bp; + spin_unlock_irqrestore(&log_redrive_lock, flags); + + wake_up_process(jfsIOthread); +} + + +/* + * lbmRead() + */ +static int lbmRead(struct jfs_log * log, int pn, struct lbuf ** bpp) +{ + struct bio *bio; + struct lbuf *bp; + + /* + * allocate a log buffer + */ + *bpp = bp = lbmAllocate(log, pn); + jfs_info("lbmRead: bp:0x%p pn:0x%x", bp, pn); + + bp->l_flag |= lbmREAD; + + bio = bio_alloc(file_bdev(log->bdev_file), 1, REQ_OP_READ, GFP_NOFS); + bio->bi_iter.bi_sector = bp->l_blkno << (log->l2bsize - 9); + __bio_add_page(bio, bp->l_page, LOGPSIZE, bp->l_offset); + BUG_ON(bio->bi_iter.bi_size != LOGPSIZE); + + bio->bi_end_io = lbmIODone; + bio->bi_private = bp; + /*check if journaling to disk has been disabled*/ + if (log->no_integrity) { + bio->bi_iter.bi_size = 0; + lbmIODone(bio); + } else { + submit_bio(bio); + } + + wait_event(bp->l_ioevent, (bp->l_flag != lbmREAD)); + + return 0; +} + + +/* + * lbmWrite() + * + * buffer at head of pageout queue stays after completion of + * partial-page pageout and redriven by explicit initiation of + * pageout by caller until full-page pageout is completed and + * released. + * + * device driver i/o done redrives pageout of new buffer at + * head of pageout queue when current buffer at head of pageout + * queue is released at the completion of its full-page pageout. + * + * LOGGC_LOCK() serializes lbmWrite() by lmNextPage() and lmGroupCommit(). + * LCACHE_LOCK() serializes xflag between lbmWrite() and lbmIODone() + */ +static void lbmWrite(struct jfs_log * log, struct lbuf * bp, int flag, + int cant_block) +{ + struct lbuf *tail; + unsigned long flags; + + jfs_info("lbmWrite: bp:0x%p flag:0x%x pn:0x%x", bp, flag, bp->l_pn); + + /* map the logical block address to physical block address */ + bp->l_blkno = + log->base + (bp->l_pn << (L2LOGPSIZE - log->l2bsize)); + + LCACHE_LOCK(flags); /* disable+lock */ + + /* + * initialize buffer for device driver + */ + bp->l_flag = flag; + + /* + * insert bp at tail of write queue associated with log + * + * (request is either for bp already/currently at head of queue + * or new bp to be inserted at tail) + */ + tail = log->wqueue; + + /* is buffer not already on write queue ? */ + if (bp->l_wqnext == NULL) { + /* insert at tail of wqueue */ + if (tail == NULL) { + log->wqueue = bp; + bp->l_wqnext = bp; + } else { + log->wqueue = bp; + bp->l_wqnext = tail->l_wqnext; + tail->l_wqnext = bp; + } + + tail = bp; + } + + /* is buffer at head of wqueue and for write ? */ + if ((bp != tail->l_wqnext) || !(flag & lbmWRITE)) { + LCACHE_UNLOCK(flags); /* unlock+enable */ + return; + } + + LCACHE_UNLOCK(flags); /* unlock+enable */ + + if (cant_block) + lbmRedrive(bp); + else if (flag & lbmSYNC) + lbmStartIO(bp); + else { + LOGGC_UNLOCK(log); + lbmStartIO(bp); + LOGGC_LOCK(log); + } +} + + +/* + * lbmDirectWrite() + * + * initiate pageout bypassing write queue for sidestream + * (e.g., log superblock) write; + */ +static void lbmDirectWrite(struct jfs_log * log, struct lbuf * bp, int flag) +{ + jfs_info("lbmDirectWrite: bp:0x%p flag:0x%x pn:0x%x", + bp, flag, bp->l_pn); + + /* + * initialize buffer for device driver + */ + bp->l_flag = flag | lbmDIRECT; + + /* map the logical block address to physical block address */ + bp->l_blkno = + log->base + (bp->l_pn << (L2LOGPSIZE - log->l2bsize)); + + /* + * initiate pageout of the page + */ + lbmStartIO(bp); +} + + +/* + * NAME: lbmStartIO() + * + * FUNCTION: Interface to DD strategy routine + * + * RETURN: none + * + * serialization: LCACHE_LOCK() is NOT held during log i/o; + */ +static void lbmStartIO(struct lbuf * bp) +{ + struct bio *bio; + struct jfs_log *log = bp->l_log; + struct block_device *bdev = NULL; + + jfs_info("lbmStartIO"); + + if (!log->no_integrity) + bdev = file_bdev(log->bdev_file); + + bio = bio_alloc(bdev, 1, REQ_OP_WRITE | REQ_SYNC, + GFP_NOFS); + bio->bi_iter.bi_sector = bp->l_blkno << (log->l2bsize - 9); + __bio_add_page(bio, bp->l_page, LOGPSIZE, bp->l_offset); + BUG_ON(bio->bi_iter.bi_size != LOGPSIZE); + + bio->bi_end_io = lbmIODone; + bio->bi_private = bp; + + /* check if journaling to disk has been disabled */ + if (log->no_integrity) { + bio->bi_iter.bi_size = 0; + lbmIODone(bio); + } else { + submit_bio(bio); + INCREMENT(lmStat.submitted); + } +} + + +/* + * lbmIOWait() + */ +static int lbmIOWait(struct lbuf * bp, int flag) +{ + unsigned long flags; + int rc = 0; + + jfs_info("lbmIOWait1: bp:0x%p flag:0x%x:0x%x", bp, bp->l_flag, flag); + + LCACHE_LOCK(flags); /* disable+lock */ + + LCACHE_SLEEP_COND(bp->l_ioevent, (bp->l_flag & lbmDONE), flags); + + rc = (bp->l_flag & lbmERROR) ? -EIO : 0; + + if (flag & lbmFREE) + lbmfree(bp); + + LCACHE_UNLOCK(flags); /* unlock+enable */ + + jfs_info("lbmIOWait2: bp:0x%p flag:0x%x:0x%x", bp, bp->l_flag, flag); + return rc; +} + +/* + * lbmIODone() + * + * executed at INTIODONE level + */ +static void lbmIODone(struct bio *bio) +{ + struct lbuf *bp = bio->bi_private; + struct lbuf *nextbp, *tail; + struct jfs_log *log; + unsigned long flags; + + /* + * get back jfs buffer bound to the i/o buffer + */ + jfs_info("lbmIODone: bp:0x%p flag:0x%x", bp, bp->l_flag); + + LCACHE_LOCK(flags); /* disable+lock */ + + if (bio->bi_status) { + bp->l_flag |= lbmERROR; + + jfs_err("lbmIODone: I/O error in JFS log"); + } + + bio_put(bio); + + /* + * pagein completion + */ + if (bp->l_flag & lbmREAD) { + bp->l_flag &= ~lbmREAD; + + /* wakeup I/O initiator */ + LCACHE_WAKEUP(&bp->l_ioevent); + + goto out; + } + + /* + * pageout completion + * + * the bp at the head of write queue has completed pageout. + * + * if single-commit/full-page pageout, remove the current buffer + * from head of pageout queue, and redrive pageout with + * the new buffer at head of pageout queue; + * otherwise, the partial-page pageout buffer stays at + * the head of pageout queue to be redriven for pageout + * by lmGroupCommit() until full-page pageout is completed. + */ + bp->l_flag &= ~lbmWRITE; + INCREMENT(lmStat.pagedone); + + /* update committed lsn */ + log = bp->l_log; + log->clsn = (bp->l_pn << L2LOGPSIZE) + bp->l_ceor; + + if (bp->l_flag & lbmDIRECT) { + LCACHE_WAKEUP(&bp->l_ioevent); + goto out; + } + + tail = log->wqueue; + + /* single element queue */ + if (bp == tail) { + /* remove head buffer of full-page pageout + * from log device write queue + */ + if (bp->l_flag & lbmRELEASE) { + log->wqueue = NULL; + bp->l_wqnext = NULL; + } + } + /* multi element queue */ + else { + /* remove head buffer of full-page pageout + * from log device write queue + */ + if (bp->l_flag & lbmRELEASE) { + nextbp = tail->l_wqnext = bp->l_wqnext; + bp->l_wqnext = NULL; + + /* + * redrive pageout of next page at head of write queue: + * redrive next page without any bound tblk + * (i.e., page w/o any COMMIT records), or + * first page of new group commit which has been + * queued after current page (subsequent pageout + * is performed synchronously, except page without + * any COMMITs) by lmGroupCommit() as indicated + * by lbmWRITE flag; + */ + if (nextbp->l_flag & lbmWRITE) { + /* + * We can't do the I/O at interrupt time. + * The jfsIO thread can do it + */ + lbmRedrive(nextbp); + } + } + } + + /* + * synchronous pageout: + * + * buffer has not necessarily been removed from write queue + * (e.g., synchronous write of partial-page with COMMIT): + * leave buffer for i/o initiator to dispose + */ + if (bp->l_flag & lbmSYNC) { + /* wakeup I/O initiator */ + LCACHE_WAKEUP(&bp->l_ioevent); + } + + /* + * Group Commit pageout: + */ + else if (bp->l_flag & lbmGC) { + LCACHE_UNLOCK(flags); + lmPostGC(bp); + LCACHE_LOCK(flags); /* disable+lock */ + } + + /* + * asynchronous pageout: + * + * buffer must have been removed from write queue: + * insert buffer at head of freelist where it can be recycled + */ + else { + assert(bp->l_flag & lbmRELEASE); + assert(bp->l_flag & lbmFREE); + lbmfree(bp); + } + +out: + bp->l_flag |= lbmDONE; + LCACHE_UNLOCK(flags); +} + +int jfsIOWait(void *arg) +{ + struct lbuf *bp; + + set_freezable(); + do { + spin_lock_irq(&log_redrive_lock); + while ((bp = log_redrive_list)) { + log_redrive_list = bp->l_redrive_next; + bp->l_redrive_next = NULL; + spin_unlock_irq(&log_redrive_lock); + lbmStartIO(bp); + spin_lock_irq(&log_redrive_lock); + } + + if (freezing(current)) { + spin_unlock_irq(&log_redrive_lock); + try_to_freeze(); + } else { + set_current_state(TASK_INTERRUPTIBLE); + spin_unlock_irq(&log_redrive_lock); + schedule(); + } + } while (!kthread_should_stop()); + + jfs_info("jfsIOWait being killed!"); + return 0; +} + +/* + * NAME: lmLogFormat()/jfs_logform() + * + * FUNCTION: format file system log + * + * PARAMETERS: + * log - volume log + * logAddress - start address of log space in FS block + * logSize - length of log space in FS block; + * + * RETURN: 0 - success + * -EIO - i/o error + * + * XXX: We're synchronously writing one page at a time. This needs to + * be improved by writing multiple pages at once. + */ +int lmLogFormat(struct jfs_log *log, s64 logAddress, int logSize) +{ + int rc = -EIO; + struct jfs_sb_info *sbi; + struct logsuper *logsuper; + struct logpage *lp; + int lspn; /* log sequence page number */ + struct lrd *lrd_ptr; + int npages = 0; + struct lbuf *bp; + + jfs_info("lmLogFormat: logAddress:%Ld logSize:%d", + (long long)logAddress, logSize); + + sbi = list_entry(log->sb_list.next, struct jfs_sb_info, log_list); + + /* allocate a log buffer */ + bp = lbmAllocate(log, 1); + + npages = logSize >> sbi->l2nbperpage; + + /* + * log space: + * + * page 0 - reserved; + * page 1 - log superblock; + * page 2 - log data page: A SYNC log record is written + * into this page at logform time; + * pages 3-N - log data page: set to empty log data pages; + */ + /* + * init log superblock: log page 1 + */ + logsuper = (struct logsuper *) bp->l_ldata; + + logsuper->magic = cpu_to_le32(LOGMAGIC); + logsuper->version = cpu_to_le32(LOGVERSION); + logsuper->state = cpu_to_le32(LOGREDONE); + logsuper->flag = cpu_to_le32(sbi->mntflag); /* ? */ + logsuper->size = cpu_to_le32(npages); + logsuper->bsize = cpu_to_le32(sbi->bsize); + logsuper->l2bsize = cpu_to_le32(sbi->l2bsize); + logsuper->end = cpu_to_le32(2 * LOGPSIZE + LOGPHDRSIZE + LOGRDSIZE); + + bp->l_flag = lbmWRITE | lbmSYNC | lbmDIRECT; + bp->l_blkno = logAddress + sbi->nbperpage; + lbmStartIO(bp); + if ((rc = lbmIOWait(bp, 0))) + goto exit; + + /* + * init pages 2 to npages-1 as log data pages: + * + * log page sequence number (lpsn) initialization: + * + * pn: 0 1 2 3 n-1 + * +-----+-----+=====+=====+===.....===+=====+ + * lspn: N-1 0 1 N-2 + * <--- N page circular file ----> + * + * the N (= npages-2) data pages of the log is maintained as + * a circular file for the log records; + * lpsn grows by 1 monotonically as each log page is written + * to the circular file of the log; + * and setLogpage() will not reset the page number even if + * the eor is equal to LOGPHDRSIZE. In order for binary search + * still work in find log end process, we have to simulate the + * log wrap situation at the log format time. + * The 1st log page written will have the highest lpsn. Then + * the succeeding log pages will have ascending order of + * the lspn starting from 0, ... (N-2) + */ + lp = (struct logpage *) bp->l_ldata; + /* + * initialize 1st log page to be written: lpsn = N - 1, + * write a SYNCPT log record is written to this page + */ + lp->h.page = lp->t.page = cpu_to_le32(npages - 3); + lp->h.eor = lp->t.eor = cpu_to_le16(LOGPHDRSIZE + LOGRDSIZE); + + lrd_ptr = (struct lrd *) &lp->data; + lrd_ptr->logtid = 0; + lrd_ptr->backchain = 0; + lrd_ptr->type = cpu_to_le16(LOG_SYNCPT); + lrd_ptr->length = 0; + lrd_ptr->log.syncpt.sync = 0; + + bp->l_blkno += sbi->nbperpage; + bp->l_flag = lbmWRITE | lbmSYNC | lbmDIRECT; + lbmStartIO(bp); + if ((rc = lbmIOWait(bp, 0))) + goto exit; + + /* + * initialize succeeding log pages: lpsn = 0, 1, ..., (N-2) + */ + for (lspn = 0; lspn < npages - 3; lspn++) { + lp->h.page = lp->t.page = cpu_to_le32(lspn); + lp->h.eor = lp->t.eor = cpu_to_le16(LOGPHDRSIZE); + + bp->l_blkno += sbi->nbperpage; + bp->l_flag = lbmWRITE | lbmSYNC | lbmDIRECT; + lbmStartIO(bp); + if ((rc = lbmIOWait(bp, 0))) + goto exit; + } + + rc = 0; +exit: + /* + * finalize log + */ + /* release the buffer */ + lbmFree(bp); + + return rc; +} + +#ifdef CONFIG_JFS_STATISTICS +int jfs_lmstats_proc_show(struct seq_file *m, void *v) +{ + seq_printf(m, + "JFS Logmgr stats\n" + "================\n" + "commits = %d\n" + "writes submitted = %d\n" + "writes completed = %d\n" + "full pages submitted = %d\n" + "partial pages submitted = %d\n", + lmStat.commit, + lmStat.submitted, + lmStat.pagedone, + lmStat.full_page, + lmStat.partial_page); + return 0; +} +#endif /* CONFIG_JFS_STATISTICS */ diff --git a/fs/jfs/jfs_logmgr.h b/fs/jfs/jfs_logmgr.h new file mode 100644 index 000000000..09e0ef6ae --- /dev/null +++ b/fs/jfs/jfs_logmgr.h @@ -0,0 +1,507 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) International Business Machines Corp., 2000-2004 + * Portions Copyright (C) Christoph Hellwig, 2001-2002 + */ +#ifndef _H_JFS_LOGMGR +#define _H_JFS_LOGMGR + +#include + +#include "jfs_filsys.h" +#include "jfs_lock.h" + +/* + * log manager configuration parameters + */ + +/* log page size */ +#define LOGPSIZE 4096 +#define L2LOGPSIZE 12 + +#define LOGPAGES 16 /* Log pages per mounted file system */ + +/* + * log logical volume + * + * a log is used to make the commit operation on journalled + * files within the same logical volume group atomic. + * a log is implemented with a logical volume. + * there is one log per logical volume group. + * + * block 0 of the log logical volume is not used (ipl etc). + * block 1 contains a log "superblock" and is used by logFormat(), + * lmLogInit(), lmLogShutdown(), and logRedo() to record status + * of the log but is not otherwise used during normal processing. + * blocks 2 - (N-1) are used to contain log records. + * + * when a volume group is varied-on-line, logRedo() must have + * been executed before the file systems (logical volumes) in + * the volume group can be mounted. + */ +/* + * log superblock (block 1 of logical volume) + */ +#define LOGSUPER_B 1 +#define LOGSTART_B 2 + +#define LOGMAGIC 0x87654321 +#define LOGVERSION 1 + +#define MAX_ACTIVE 128 /* Max active file systems sharing log */ + +struct logsuper { + __le32 magic; /* 4: log lv identifier */ + __le32 version; /* 4: version number */ + __le32 serial; /* 4: log open/mount counter */ + __le32 size; /* 4: size in number of LOGPSIZE blocks */ + __le32 bsize; /* 4: logical block size in byte */ + __le32 l2bsize; /* 4: log2 of bsize */ + + __le32 flag; /* 4: option */ + __le32 state; /* 4: state - see below */ + + __le32 end; /* 4: addr of last log record set by logredo */ + uuid_t uuid; /* 16: 128-bit journal uuid */ + char label[16]; /* 16: journal label */ + struct { + uuid_t uuid; + } active[MAX_ACTIVE]; /* 2048: active file systems list */ +}; + +/* log flag: commit option (see jfs_filsys.h) */ + +/* log state */ +#define LOGMOUNT 0 /* log mounted by lmLogInit() */ +#define LOGREDONE 1 /* log shutdown by lmLogShutdown(). + * log redo completed by logredo(). + */ +#define LOGWRAP 2 /* log wrapped */ +#define LOGREADERR 3 /* log read error detected in logredo() */ + + +/* + * log logical page + * + * (this comment should be rewritten !) + * the header and trailer structures (h,t) will normally have + * the same page and eor value. + * An exception to this occurs when a complete page write is not + * accomplished on a power failure. Since the hardware may "split write" + * sectors in the page, any out of order sequence may occur during powerfail + * and needs to be recognized during log replay. The xor value is + * an "exclusive or" of all log words in the page up to eor. This + * 32 bit eor is stored with the top 16 bits in the header and the + * bottom 16 bits in the trailer. logredo can easily recognize pages + * that were not completed by reconstructing this eor and checking + * the log page. + * + * Previous versions of the operating system did not allow split + * writes and detected partially written records in logredo by + * ordering the updates to the header, trailer, and the move of data + * into the logdata area. The order: (1) data is moved (2) header + * is updated (3) trailer is updated. In logredo, when the header + * differed from the trailer, the header and trailer were reconciled + * as follows: if h.page != t.page they were set to the smaller of + * the two and h.eor and t.eor set to 8 (i.e. empty page). if (only) + * h.eor != t.eor they were set to the smaller of their two values. + */ +struct logpage { + struct { /* header */ + __le32 page; /* 4: log sequence page number */ + __le16 rsrvd; /* 2: */ + __le16 eor; /* 2: end-of-log offset of lasrt record write */ + } h; + + __le32 data[LOGPSIZE / 4 - 4]; /* log record area */ + + struct { /* trailer */ + __le32 page; /* 4: normally the same as h.page */ + __le16 rsrvd; /* 2: */ + __le16 eor; /* 2: normally the same as h.eor */ + } t; +}; + +#define LOGPHDRSIZE 8 /* log page header size */ +#define LOGPTLRSIZE 8 /* log page trailer size */ + + +/* + * log record + * + * (this comment should be rewritten !) + * jfs uses only "after" log records (only a single writer is allowed + * in a page, pages are written to temporary paging space if + * they must be written to disk before commit, and i/o is + * scheduled for modified pages to their home location after + * the log records containing the after values and the commit + * record is written to the log on disk, undo discards the copy + * in main-memory.) + * + * a log record consists of a data area of variable length followed by + * a descriptor of fixed size LOGRDSIZE bytes. + * the data area is rounded up to an integral number of 4-bytes and + * must be no longer than LOGPSIZE. + * the descriptor is of size of multiple of 4-bytes and aligned on a + * 4-byte boundary. + * records are packed one after the other in the data area of log pages. + * (sometimes a DUMMY record is inserted so that at least one record ends + * on every page or the longest record is placed on at most two pages). + * the field eor in page header/trailer points to the byte following + * the last record on a page. + */ + +/* log record types */ +#define LOG_COMMIT 0x8000 +#define LOG_SYNCPT 0x4000 +#define LOG_MOUNT 0x2000 +#define LOG_REDOPAGE 0x0800 +#define LOG_NOREDOPAGE 0x0080 +#define LOG_NOREDOINOEXT 0x0040 +#define LOG_UPDATEMAP 0x0008 +#define LOG_NOREDOFILE 0x0001 + +/* REDOPAGE/NOREDOPAGE log record data type */ +#define LOG_INODE 0x0001 +#define LOG_XTREE 0x0002 +#define LOG_DTREE 0x0004 +#define LOG_BTROOT 0x0010 +#define LOG_EA 0x0020 +#define LOG_ACL 0x0040 +#define LOG_DATA 0x0080 +#define LOG_NEW 0x0100 +#define LOG_EXTEND 0x0200 +#define LOG_RELOCATE 0x0400 +#define LOG_DIR_XTREE 0x0800 /* Xtree is in directory inode */ + +/* UPDATEMAP log record descriptor type */ +#define LOG_ALLOCXADLIST 0x0080 +#define LOG_ALLOCPXDLIST 0x0040 +#define LOG_ALLOCXAD 0x0020 +#define LOG_ALLOCPXD 0x0010 +#define LOG_FREEXADLIST 0x0008 +#define LOG_FREEPXDLIST 0x0004 +#define LOG_FREEXAD 0x0002 +#define LOG_FREEPXD 0x0001 + + +struct lrd { + /* + * type independent area + */ + __le32 logtid; /* 4: log transaction identifier */ + __le32 backchain; /* 4: ptr to prev record of same transaction */ + __le16 type; /* 2: record type */ + __le16 length; /* 2: length of data in record (in byte) */ + __le32 aggregate; /* 4: file system lv/aggregate */ + /* (16) */ + + /* + * type dependent area (20) + */ + union { + + /* + * COMMIT: commit + * + * transaction commit: no type-dependent information; + */ + + /* + * REDOPAGE: after-image + * + * apply after-image; + * + * N.B. REDOPAGE, NOREDOPAGE, and UPDATEMAP must be same format; + */ + struct { + __le32 fileset; /* 4: fileset number */ + __le32 inode; /* 4: inode number */ + __le16 type; /* 2: REDOPAGE record type */ + __le16 l2linesize; /* 2: log2 of line size */ + pxd_t pxd; /* 8: on-disk page pxd */ + } redopage; /* (20) */ + + /* + * NOREDOPAGE: the page is freed + * + * do not apply after-image records which precede this record + * in the log with the same page block number to this page. + * + * N.B. REDOPAGE, NOREDOPAGE, and UPDATEMAP must be same format; + */ + struct { + __le32 fileset; /* 4: fileset number */ + __le32 inode; /* 4: inode number */ + __le16 type; /* 2: NOREDOPAGE record type */ + __le16 rsrvd; /* 2: reserved */ + pxd_t pxd; /* 8: on-disk page pxd */ + } noredopage; /* (20) */ + + /* + * UPDATEMAP: update block allocation map + * + * either in-line PXD, + * or out-of-line XADLIST; + * + * N.B. REDOPAGE, NOREDOPAGE, and UPDATEMAP must be same format; + */ + struct { + __le32 fileset; /* 4: fileset number */ + __le32 inode; /* 4: inode number */ + __le16 type; /* 2: UPDATEMAP record type */ + __le16 nxd; /* 2: number of extents */ + pxd_t pxd; /* 8: pxd */ + } updatemap; /* (20) */ + + /* + * NOREDOINOEXT: the inode extent is freed + * + * do not apply after-image records which precede this + * record in the log with the any of the 4 page block + * numbers in this inode extent. + * + * NOTE: The fileset and pxd fields MUST remain in + * the same fields in the REDOPAGE record format. + * + */ + struct { + __le32 fileset; /* 4: fileset number */ + __le32 iagnum; /* 4: IAG number */ + __le32 inoext_idx; /* 4: inode extent index */ + pxd_t pxd; /* 8: on-disk page pxd */ + } noredoinoext; /* (20) */ + + /* + * SYNCPT: log sync point + * + * replay log up to syncpt address specified; + */ + struct { + __le32 sync; /* 4: syncpt address (0 = here) */ + } syncpt; + + /* + * MOUNT: file system mount + * + * file system mount: no type-dependent information; + */ + + /* + * ? FREEXTENT: free specified extent(s) + * + * free specified extent(s) from block allocation map + * N.B.: nextents should be length of data/sizeof(xad_t) + */ + struct { + __le32 type; /* 4: FREEXTENT record type */ + __le32 nextent; /* 4: number of extents */ + + /* data: PXD or XAD list */ + } freextent; + + /* + * ? NOREDOFILE: this file is freed + * + * do not apply records which precede this record in the log + * with the same inode number. + * + * NOREDOFILE must be the first to be written at commit + * (last to be read in logredo()) - it prevents + * replay of preceding updates of all preceding generations + * of the inumber esp. the on-disk inode itself. + */ + struct { + __le32 fileset; /* 4: fileset number */ + __le32 inode; /* 4: inode number */ + } noredofile; + + /* + * ? NEWPAGE: + * + * metadata type dependent + */ + struct { + __le32 fileset; /* 4: fileset number */ + __le32 inode; /* 4: inode number */ + __le32 type; /* 4: NEWPAGE record type */ + pxd_t pxd; /* 8: on-disk page pxd */ + } newpage; + + /* + * ? DUMMY: filler + * + * no type-dependent information + */ + } log; +}; /* (36) */ + +#define LOGRDSIZE (sizeof(struct lrd)) + +/* + * line vector descriptor + */ +struct lvd { + __le16 offset; + __le16 length; +}; + + +/* + * log logical volume + */ +struct jfs_log { + + struct list_head sb_list;/* This is used to sync metadata + * before writing syncpt. + */ + struct list_head journal_list; /* Global list */ + struct file *bdev_file; /* 4: log lv pointer */ + int serial; /* 4: log mount serial number */ + + s64 base; /* @8: log extent address (inline log ) */ + int size; /* 4: log size in log page (in page) */ + int l2bsize; /* 4: log2 of bsize */ + + unsigned long flag; /* 4: flag */ + + struct lbuf *lbuf_free; /* 4: free lbufs */ + wait_queue_head_t free_wait; /* 4: */ + + /* log write */ + int logtid; /* 4: log tid */ + int page; /* 4: page number of eol page */ + int eor; /* 4: eor of last record in eol page */ + struct lbuf *bp; /* 4: current log page buffer */ + + struct mutex loglock; /* 4: log write serialization lock */ + + /* syncpt */ + int nextsync; /* 4: bytes to write before next syncpt */ + int active; /* 4: */ + wait_queue_head_t syncwait; /* 4: */ + + /* commit */ + uint cflag; /* 4: */ + struct list_head cqueue; /* FIFO commit queue */ + struct tblock *flush_tblk; /* tblk we're waiting on for flush */ + int gcrtc; /* 4: GC_READY transaction count */ + struct tblock *gclrt; /* 4: latest GC_READY transaction */ + spinlock_t gclock; /* 4: group commit lock */ + int logsize; /* 4: log data area size in byte */ + int lsn; /* 4: end-of-log */ + int clsn; /* 4: clsn */ + int syncpt; /* 4: addr of last syncpt record */ + int sync; /* 4: addr from last logsync() */ + struct list_head synclist; /* 8: logsynclist anchor */ + spinlock_t synclock; /* 4: synclist lock */ + struct lbuf *wqueue; /* 4: log pageout queue */ + int count; /* 4: count */ + uuid_t uuid; /* 16: 128-bit uuid of log device */ + + int no_integrity; /* 3: flag to disable journaling to disk */ +}; + +/* + * log read/write serialization (per log) + */ +#define LOG_LOCK_INIT(log) mutex_init(&(log)->loglock) +#define LOG_LOCK(log) mutex_lock(&((log)->loglock)) +#define LOG_UNLOCK(log) mutex_unlock(&((log)->loglock)) + +/* + * Log flag + */ +#define log_INLINELOG 1 +#define log_SYNCBARRIER 2 +#define log_QUIESCE 3 +#define log_FLUSH 4 + +/* + * group commit flag + */ +/* jfs_log */ +#define logGC_PAGEOUT 0x00000001 + +/* tblock/lbuf */ +#define tblkGC_QUEUE 0x0001 +#define tblkGC_READY 0x0002 +#define tblkGC_COMMIT 0x0004 +#define tblkGC_COMMITTED 0x0008 +#define tblkGC_EOP 0x0010 +#define tblkGC_FREE 0x0020 +#define tblkGC_LEADER 0x0040 +#define tblkGC_ERROR 0x0080 +#define tblkGC_LAZY 0x0100 // D230860 +#define tblkGC_UNLOCKED 0x0200 // D230860 + +/* + * log cache buffer header + */ +struct lbuf { + struct jfs_log *l_log; /* 4: log associated with buffer */ + + /* + * data buffer base area + */ + uint l_flag; /* 4: pageout control flags */ + + struct lbuf *l_wqnext; /* 4: write queue link */ + struct lbuf *l_freelist; /* 4: freelistlink */ + + int l_pn; /* 4: log page number */ + int l_eor; /* 4: log record eor */ + int l_ceor; /* 4: committed log record eor */ + + s64 l_blkno; /* 8: log page block number */ + caddr_t l_ldata; /* 4: data page */ + struct page *l_page; /* The page itself */ + uint l_offset; /* Offset of l_ldata within the page */ + + wait_queue_head_t l_ioevent; /* 4: i/o done event */ +}; + +/* Reuse l_freelist for redrive list */ +#define l_redrive_next l_freelist + +/* + * logsynclist block + * + * common logsyncblk prefix for jbuf_t and tblock + */ +struct logsyncblk { + u16 xflag; /* flags */ + u16 flag; /* only meaninful in tblock */ + lid_t lid; /* lock id */ + s32 lsn; /* log sequence number */ + struct list_head synclist; /* log sync list link */ +}; + +/* + * logsynclist serialization (per log) + */ + +#define LOGSYNC_LOCK_INIT(log) spin_lock_init(&(log)->synclock) +#define LOGSYNC_LOCK(log, flags) spin_lock_irqsave(&(log)->synclock, flags) +#define LOGSYNC_UNLOCK(log, flags) \ + spin_unlock_irqrestore(&(log)->synclock, flags) + +/* compute the difference in bytes of lsn from sync point */ +#define logdiff(diff, lsn, log)\ +{\ + diff = (lsn) - (log)->syncpt;\ + if (diff < 0)\ + diff += (log)->logsize;\ +} + +extern int lmLogOpen(struct super_block *sb); +extern int lmLogClose(struct super_block *sb); +extern int lmLogShutdown(struct jfs_log * log); +extern int lmLogInit(struct jfs_log * log); +extern int lmLogFormat(struct jfs_log *log, s64 logAddress, int logSize); +extern int lmGroupCommit(struct jfs_log *, struct tblock *); +extern int jfsIOWait(void *); +extern void jfs_flush_journal(struct jfs_log * log, int wait); +extern void jfs_syncpt(struct jfs_log *log, int hard_sync); + +#endif /* _H_JFS_LOGMGR */ diff --git a/fs/jfs/jfs_metapage.c b/fs/jfs/jfs_metapage.c new file mode 100644 index 000000000..41fe12e64 --- /dev/null +++ b/fs/jfs/jfs_metapage.c @@ -0,0 +1,953 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) International Business Machines Corp., 2000-2005 + * Portions Copyright (C) Christoph Hellwig, 2001-2002 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "jfs_incore.h" +#include "jfs_superblock.h" +#include "jfs_filsys.h" +#include "jfs_metapage.h" +#include "jfs_txnmgr.h" +#include "jfs_debug.h" + +#ifdef CONFIG_JFS_STATISTICS +static struct { + uint pagealloc; /* # of page allocations */ + uint pagefree; /* # of page frees */ + uint lockwait; /* # of sleeping lock_metapage() calls */ +} mpStat; +#endif + +#define metapage_locked(mp) test_bit(META_locked, &(mp)->flag) +#define trylock_metapage(mp) test_and_set_bit_lock(META_locked, &(mp)->flag) + +static inline void unlock_metapage(struct metapage *mp) +{ + clear_bit_unlock(META_locked, &mp->flag); + wake_up(&mp->wait); +} + +static inline void __lock_metapage(struct metapage *mp) +{ + DECLARE_WAITQUEUE(wait, current); + INCREMENT(mpStat.lockwait); + add_wait_queue_exclusive(&mp->wait, &wait); + do { + set_current_state(TASK_UNINTERRUPTIBLE); + if (metapage_locked(mp)) { + folio_unlock(mp->folio); + io_schedule(); + folio_lock(mp->folio); + } + } while (trylock_metapage(mp)); + __set_current_state(TASK_RUNNING); + remove_wait_queue(&mp->wait, &wait); +} + +/* + * Must have mp->folio locked + */ +static inline void lock_metapage(struct metapage *mp) +{ + if (trylock_metapage(mp)) + __lock_metapage(mp); +} + +#define METAPOOL_MIN_PAGES 32 +static struct kmem_cache *metapage_cache; +static mempool_t *metapage_mempool; + +#define MPS_PER_PAGE (PAGE_SIZE >> L2PSIZE) + +#if MPS_PER_PAGE > 1 + +struct meta_anchor { + int mp_count; + atomic_t io_count; + blk_status_t status; + struct metapage *mp[MPS_PER_PAGE]; +}; + +static inline struct metapage *folio_to_mp(struct folio *folio, int offset) +{ + struct meta_anchor *anchor = folio->private; + + if (!anchor) + return NULL; + return anchor->mp[offset >> L2PSIZE]; +} + +static inline int insert_metapage(struct folio *folio, struct metapage *mp) +{ + struct meta_anchor *a; + int index; + int l2mp_blocks; /* log2 blocks per metapage */ + + a = folio->private; + if (!a) { + a = kzalloc_obj(struct meta_anchor, GFP_NOFS); + if (!a) + return -ENOMEM; + folio_attach_private(folio, a); + kmap(&folio->page); + } + + if (mp) { + l2mp_blocks = L2PSIZE - folio->mapping->host->i_blkbits; + index = (mp->index >> l2mp_blocks) & (MPS_PER_PAGE - 1); + a->mp_count++; + a->mp[index] = mp; + } + + return 0; +} + +static inline void remove_metapage(struct folio *folio, struct metapage *mp) +{ + struct meta_anchor *a = folio->private; + int l2mp_blocks = L2PSIZE - folio->mapping->host->i_blkbits; + int index; + + index = (mp->index >> l2mp_blocks) & (MPS_PER_PAGE - 1); + + BUG_ON(a->mp[index] != mp); + + a->mp[index] = NULL; + if (--a->mp_count == 0) { + kfree(a); + folio_detach_private(folio); + kunmap(&folio->page); + } +} + +static inline void inc_io(struct folio *folio) +{ + struct meta_anchor *anchor = folio->private; + + atomic_inc(&anchor->io_count); +} + +static inline void dec_io(struct folio *folio, blk_status_t status, + void (*handler)(struct folio *, blk_status_t)) +{ + struct meta_anchor *anchor = folio->private; + + if (anchor->status == BLK_STS_OK) + anchor->status = status; + + if (atomic_dec_and_test(&anchor->io_count)) + handler(folio, anchor->status); +} + +#ifdef CONFIG_MIGRATION +static int __metapage_migrate_folio(struct address_space *mapping, + struct folio *dst, struct folio *src, + enum migrate_mode mode) +{ + struct meta_anchor *src_anchor = src->private; + struct metapage *mps[MPS_PER_PAGE] = {0}; + struct metapage *mp; + int i, rc; + + for (i = 0; i < MPS_PER_PAGE; i++) { + mp = src_anchor->mp[i]; + if (mp && metapage_locked(mp)) + return -EAGAIN; + } + + rc = filemap_migrate_folio(mapping, dst, src, mode); + if (rc) + return rc; + + for (i = 0; i < MPS_PER_PAGE; i++) { + mp = src_anchor->mp[i]; + if (!mp) + continue; + if (unlikely(insert_metapage(dst, mp))) { + /* If error, roll-back previosly inserted pages */ + for (int j = 0 ; j < i; j++) { + if (mps[j]) + remove_metapage(dst, mps[j]); + } + return -EAGAIN; + } + mps[i] = mp; + } + + /* Update the metapage and remove it from src */ + for (i = 0; i < MPS_PER_PAGE; i++) { + mp = mps[i]; + if (mp) { + int page_offset = mp->data - folio_address(src); + + mp->data = folio_address(dst) + page_offset; + mp->folio = dst; + remove_metapage(src, mp); + } + } + + return 0; +} +#endif /* CONFIG_MIGRATION */ + +#else + +static inline struct metapage *folio_to_mp(struct folio *folio, int offset) +{ + return folio->private; +} + +static inline int insert_metapage(struct folio *folio, struct metapage *mp) +{ + if (mp) { + folio_attach_private(folio, mp); + kmap(&folio->page); + } + return 0; +} + +static inline void remove_metapage(struct folio *folio, struct metapage *mp) +{ + folio_detach_private(folio); + kunmap(&folio->page); +} + +#define inc_io(folio) do {} while(0) +#define dec_io(folio, status, handler) handler(folio, status) + +#ifdef CONFIG_MIGRATION +static int __metapage_migrate_folio(struct address_space *mapping, + struct folio *dst, struct folio *src, + enum migrate_mode mode) +{ + struct metapage *mp; + int page_offset; + int rc; + + mp = folio_to_mp(src, 0); + if (metapage_locked(mp)) + return -EAGAIN; + + rc = filemap_migrate_folio(mapping, dst, src, mode); + if (rc) + return rc; + + if (unlikely(insert_metapage(dst, mp))) + return -EAGAIN; + + page_offset = mp->data - folio_address(src); + mp->data = folio_address(dst) + page_offset; + mp->folio = dst; + remove_metapage(src, mp); + + return 0; +} +#endif /* CONFIG_MIGRATION */ + +#endif + +static inline struct metapage *alloc_metapage(gfp_t gfp_mask) +{ + struct metapage *mp = mempool_alloc(metapage_mempool, gfp_mask); + + if (mp) { + mp->lid = 0; + mp->lsn = 0; + mp->data = NULL; + mp->clsn = 0; + mp->log = NULL; + init_waitqueue_head(&mp->wait); + INIT_LIST_HEAD(&mp->synclist); + } + return mp; +} + +static inline void free_metapage(struct metapage *mp) +{ + mempool_free(mp, metapage_mempool); +} + +int __init metapage_init(void) +{ + /* + * Allocate the metapage structures + */ + metapage_cache = kmem_cache_create("jfs_mp", sizeof(struct metapage), + 0, 0, NULL); + if (metapage_cache == NULL) + return -ENOMEM; + + metapage_mempool = mempool_create_slab_pool(METAPOOL_MIN_PAGES, + metapage_cache); + + if (metapage_mempool == NULL) { + kmem_cache_destroy(metapage_cache); + return -ENOMEM; + } + + return 0; +} + +void metapage_exit(void) +{ + mempool_destroy(metapage_mempool); + kmem_cache_destroy(metapage_cache); +} + +static inline void drop_metapage(struct folio *folio, struct metapage *mp) +{ + if (mp->count || mp->nohomeok || test_bit(META_dirty, &mp->flag) || + test_bit(META_io, &mp->flag)) + return; + remove_metapage(folio, mp); + INCREMENT(mpStat.pagefree); + free_metapage(mp); +} + +/* + * Metapage address space operations + */ + +static sector_t metapage_get_blocks(struct inode *inode, sector_t lblock, + int *len) +{ + int rc = 0; + int xflag; + s64 xaddr; + sector_t file_blocks = (inode->i_size + inode->i_sb->s_blocksize - 1) >> + inode->i_blkbits; + + if (lblock >= file_blocks) + return 0; + if (lblock + *len > file_blocks) + *len = file_blocks - lblock; + + if (inode->i_ino) { + rc = xtLookup(inode, (s64)lblock, *len, &xflag, &xaddr, len, 0); + if ((rc == 0) && *len) + lblock = (sector_t)xaddr; + else + lblock = 0; + } /* else no mapping */ + + return lblock; +} + +static void last_read_complete(struct folio *folio, blk_status_t status) +{ + if (status) + printk(KERN_ERR "Read error %d at %#llx\n", status, + folio_pos(folio)); + + folio_end_read(folio, status == 0); +} + +static void metapage_read_end_io(struct bio *bio) +{ + struct folio *folio = bio->bi_private; + + dec_io(folio, bio->bi_status, last_read_complete); + bio_put(bio); +} + +static void remove_from_logsync(struct metapage *mp) +{ + struct jfs_log *log = mp->log; + unsigned long flags; +/* + * This can race. Recheck that log hasn't been set to null, and after + * acquiring logsync lock, recheck lsn + */ + if (!log) + return; + + LOGSYNC_LOCK(log, flags); + if (mp->lsn) { + mp->log = NULL; + mp->lsn = 0; + mp->clsn = 0; + log->count--; + list_del_init(&mp->synclist); + } + LOGSYNC_UNLOCK(log, flags); +} + +static void last_write_complete(struct folio *folio, blk_status_t status) +{ + struct metapage *mp; + unsigned int offset; + + if (status) { + int err = blk_status_to_errno(status); + printk(KERN_ERR "metapage_write_end_io: I/O error\n"); + mapping_set_error(folio->mapping, err); + } + + for (offset = 0; offset < PAGE_SIZE; offset += PSIZE) { + mp = folio_to_mp(folio, offset); + if (mp && test_bit(META_io, &mp->flag)) { + if (mp->lsn) + remove_from_logsync(mp); + clear_bit(META_io, &mp->flag); + } + /* + * I'd like to call drop_metapage here, but I don't think it's + * safe unless I have the page locked + */ + } + folio_end_writeback(folio); +} + +static void metapage_write_end_io(struct bio *bio) +{ + struct folio *folio = bio->bi_private; + + BUG_ON(!folio->private); + + dec_io(folio, bio->bi_status, last_write_complete); + bio_put(bio); +} + +static int metapage_write_folio(struct folio *folio, + struct writeback_control *wbc) +{ + struct bio *bio = NULL; + int block_offset; /* block offset of mp within page */ + struct inode *inode = folio->mapping->host; + int blocks_per_mp = JFS_SBI(inode->i_sb)->nbperpage; + int len; + int xlen; + struct metapage *mp; + int redirty = 0; + sector_t lblock; + int nr_underway = 0; + sector_t pblock; + sector_t next_block = 0; + sector_t page_start; + unsigned long bio_bytes = 0; + unsigned long bio_offset = 0; + int offset; + int bad_blocks = 0; + + page_start = folio_pos(folio) >> inode->i_blkbits; + BUG_ON(!folio_test_locked(folio)); + BUG_ON(folio_test_writeback(folio)); + folio_start_writeback(folio); + + for (offset = 0; offset < PAGE_SIZE; offset += PSIZE) { + mp = folio_to_mp(folio, offset); + + if (!mp || !test_bit(META_dirty, &mp->flag)) + continue; + + if (mp->nohomeok && !test_bit(META_forcewrite, &mp->flag)) { + redirty = 1; + /* + * Make sure this page isn't blocked indefinitely. + * If the journal isn't undergoing I/O, push it + */ + if (mp->log && !(mp->log->cflag & logGC_PAGEOUT)) + jfs_flush_journal(mp->log, 0); + continue; + } + + clear_bit(META_dirty, &mp->flag); + set_bit(META_io, &mp->flag); + block_offset = offset >> inode->i_blkbits; + lblock = page_start + block_offset; + if (bio) { + if (xlen && lblock == next_block) { + /* Contiguous, in memory & on disk */ + len = min(xlen, blocks_per_mp); + xlen -= len; + bio_bytes += len << inode->i_blkbits; + continue; + } + /* Not contiguous */ + bio_add_folio_nofail(bio, folio, bio_bytes, bio_offset); + /* + * Increment counter before submitting i/o to keep + * count from hitting zero before we're through + */ + inc_io(folio); + if (!bio->bi_iter.bi_size) + goto dump_bio; + submit_bio(bio); + nr_underway++; + bio = NULL; + } else + inc_io(folio); + xlen = (folio_size(folio) - offset) >> inode->i_blkbits; + pblock = metapage_get_blocks(inode, lblock, &xlen); + if (!pblock) { + printk(KERN_ERR "JFS: metapage_get_blocks failed\n"); + /* + * We already called inc_io(), but can't cancel it + * with dec_io() until we're done with the page + */ + bad_blocks++; + continue; + } + len = min(xlen, (int)JFS_SBI(inode->i_sb)->nbperpage); + + bio = bio_alloc(inode->i_sb->s_bdev, 1, REQ_OP_WRITE, GFP_NOFS); + bio->bi_iter.bi_sector = pblock << (inode->i_blkbits - 9); + bio->bi_end_io = metapage_write_end_io; + bio->bi_private = folio; + + /* Don't call bio_add_page yet, we may add to this vec */ + bio_offset = offset; + bio_bytes = len << inode->i_blkbits; + + xlen -= len; + next_block = lblock + len; + } + if (bio) { + bio_add_folio_nofail(bio, folio, bio_bytes, bio_offset); + if (!bio->bi_iter.bi_size) + goto dump_bio; + + submit_bio(bio); + nr_underway++; + } + if (redirty) + folio_redirty_for_writepage(wbc, folio); + + folio_unlock(folio); + + if (bad_blocks) + goto err_out; + + if (nr_underway == 0) + folio_end_writeback(folio); + + return 0; +dump_bio: + print_hex_dump(KERN_ERR, "JFS: dump of bio: ", DUMP_PREFIX_ADDRESS, 16, + 4, bio, sizeof(*bio), 0); + bio_put(bio); + folio_unlock(folio); + dec_io(folio, BLK_STS_OK, last_write_complete); +err_out: + while (bad_blocks--) + dec_io(folio, BLK_STS_OK, last_write_complete); + return -EIO; +} + +static int metapage_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct blk_plug plug; + struct folio *folio = NULL; + int err; + + blk_start_plug(&plug); + while ((folio = writeback_iter(mapping, wbc, folio, &err))) + err = metapage_write_folio(folio, wbc); + blk_finish_plug(&plug); + + return err; +} + +static int metapage_read_folio(struct file *fp, struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + struct bio *bio = NULL; + int block_offset; + int blocks_per_page = i_blocks_per_folio(inode, folio); + sector_t page_start; /* address of page in fs blocks */ + sector_t pblock; + int xlen; + unsigned int len; + int offset; + + BUG_ON(!folio_test_locked(folio)); + page_start = folio_pos(folio) >> inode->i_blkbits; + + block_offset = 0; + while (block_offset < blocks_per_page) { + xlen = blocks_per_page - block_offset; + pblock = metapage_get_blocks(inode, page_start + block_offset, + &xlen); + if (pblock) { + if (!folio->private) + insert_metapage(folio, NULL); + inc_io(folio); + if (bio) + submit_bio(bio); + + bio = bio_alloc(inode->i_sb->s_bdev, 1, REQ_OP_READ, + GFP_NOFS); + bio->bi_iter.bi_sector = + pblock << (inode->i_blkbits - 9); + bio->bi_end_io = metapage_read_end_io; + bio->bi_private = folio; + len = xlen << inode->i_blkbits; + offset = block_offset << inode->i_blkbits; + bio_add_folio_nofail(bio, folio, len, offset); + block_offset += xlen; + } else + block_offset++; + } + if (bio) + submit_bio(bio); + else + folio_unlock(folio); + + return 0; +} + +static bool metapage_release_folio(struct folio *folio, gfp_t gfp_mask) +{ + struct metapage *mp; + bool ret = true; + int offset; + + for (offset = 0; offset < PAGE_SIZE; offset += PSIZE) { + mp = folio_to_mp(folio, offset); + + if (!mp) + continue; + + jfs_info("metapage_release_folio: mp = 0x%p", mp); + if (mp->count || mp->nohomeok || + test_bit(META_dirty, &mp->flag)) { + jfs_info("count = %ld, nohomeok = %d", mp->count, + mp->nohomeok); + ret = false; + continue; + } + if (mp->lsn) + remove_from_logsync(mp); + remove_metapage(folio, mp); + INCREMENT(mpStat.pagefree); + free_metapage(mp); + } + return ret; +} + +#ifdef CONFIG_MIGRATION +/* + * metapage_migrate_folio - Migration function for JFS metapages + */ +static int metapage_migrate_folio(struct address_space *mapping, + struct folio *dst, struct folio *src, + enum migrate_mode mode) +{ + int expected_count; + + if (!src->private) + return filemap_migrate_folio(mapping, dst, src, mode); + + /* Check whether page does not have extra refs before we do more work */ + expected_count = folio_expected_ref_count(src) + 1; + if (folio_ref_count(src) != expected_count) + return -EAGAIN; + return __metapage_migrate_folio(mapping, dst, src, mode); +} +#else +#define metapage_migrate_folio NULL +#endif /* CONFIG_MIGRATION */ + +static void metapage_invalidate_folio(struct folio *folio, size_t offset, + size_t length) +{ + BUG_ON(offset || length < folio_size(folio)); + + BUG_ON(folio_test_writeback(folio)); + + metapage_release_folio(folio, 0); +} + +const struct address_space_operations jfs_metapage_aops = { + .read_folio = metapage_read_folio, + .writepages = metapage_writepages, + .release_folio = metapage_release_folio, + .invalidate_folio = metapage_invalidate_folio, + .dirty_folio = filemap_dirty_folio, + .migrate_folio = metapage_migrate_folio, +}; + +struct metapage *__get_metapage(struct inode *inode, unsigned long lblock, + unsigned int size, int absolute, + unsigned long new) +{ + int l2BlocksPerPage; + int l2bsize; + struct address_space *mapping; + struct metapage *mp = NULL; + struct folio *folio; + unsigned long page_index; + unsigned long page_offset; + + jfs_info("__get_metapage: ino = %llu, lblock = 0x%lx, abs=%d", + inode->i_ino, lblock, absolute); + + l2bsize = inode->i_blkbits; + l2BlocksPerPage = PAGE_SHIFT - l2bsize; + page_index = lblock >> l2BlocksPerPage; + page_offset = (lblock - (page_index << l2BlocksPerPage)) << l2bsize; + if ((page_offset + size) > PAGE_SIZE) { + jfs_err("MetaData crosses page boundary!!"); + jfs_err("lblock = %lx, size = %d", lblock, size); + dump_stack(); + return NULL; + } + if (absolute) + mapping = JFS_SBI(inode->i_sb)->direct_inode->i_mapping; + else { + /* + * If an nfs client tries to read an inode that is larger + * than any existing inodes, we may try to read past the + * end of the inode map + */ + if ((lblock << inode->i_blkbits) >= inode->i_size) + return NULL; + mapping = inode->i_mapping; + } + + if (new && (PSIZE == PAGE_SIZE)) { + folio = filemap_grab_folio(mapping, page_index); + if (IS_ERR(folio)) { + jfs_err("filemap_grab_folio failed!"); + return NULL; + } + folio_mark_uptodate(folio); + } else { + folio = read_mapping_folio(mapping, page_index, NULL); + if (IS_ERR(folio)) { + jfs_err("read_mapping_page failed!"); + return NULL; + } + folio_lock(folio); + } + + mp = folio_to_mp(folio, page_offset); + if (mp) { + if (mp->logical_size != size) { + jfs_error(inode->i_sb, + "get_mp->logical_size != size\n"); + jfs_err("logical_size = %d, size = %d", + mp->logical_size, size); + dump_stack(); + goto unlock; + } + mp->count++; + lock_metapage(mp); + if (test_bit(META_discard, &mp->flag)) { + if (!new) { + jfs_error(inode->i_sb, + "using a discarded metapage\n"); + discard_metapage(mp); + goto unlock; + } + clear_bit(META_discard, &mp->flag); + } + } else { + INCREMENT(mpStat.pagealloc); + mp = alloc_metapage(GFP_NOFS); + if (!mp) + goto unlock; + mp->folio = folio; + mp->sb = inode->i_sb; + mp->flag = 0; + mp->xflag = COMMIT_PAGE; + mp->count = 1; + mp->nohomeok = 0; + mp->logical_size = size; + mp->data = folio_address(folio) + page_offset; + mp->index = lblock; + if (unlikely(insert_metapage(folio, mp))) { + free_metapage(mp); + goto unlock; + } + lock_metapage(mp); + } + + if (new) { + jfs_info("zeroing mp = 0x%p", mp); + memset(mp->data, 0, PSIZE); + } + + folio_unlock(folio); + jfs_info("__get_metapage: returning = 0x%p data = 0x%p", mp, mp->data); + return mp; + +unlock: + folio_unlock(folio); + return NULL; +} + +void grab_metapage(struct metapage * mp) +{ + jfs_info("grab_metapage: mp = 0x%p", mp); + folio_get(mp->folio); + folio_lock(mp->folio); + mp->count++; + lock_metapage(mp); + folio_unlock(mp->folio); +} + +static int metapage_write_one(struct folio *folio) +{ + struct address_space *mapping = folio->mapping; + struct writeback_control wbc = { + .sync_mode = WB_SYNC_ALL, + .nr_to_write = folio_nr_pages(folio), + }; + int ret = 0; + + BUG_ON(!folio_test_locked(folio)); + + folio_wait_writeback(folio); + + if (folio_clear_dirty_for_io(folio)) { + folio_get(folio); + ret = metapage_write_folio(folio, &wbc); + if (ret == 0) + folio_wait_writeback(folio); + folio_put(folio); + } else { + folio_unlock(folio); + } + + if (!ret) + ret = filemap_check_errors(mapping); + return ret; +} + +void force_metapage(struct metapage *mp) +{ + struct folio *folio = mp->folio; + jfs_info("force_metapage: mp = 0x%p", mp); + set_bit(META_forcewrite, &mp->flag); + clear_bit(META_sync, &mp->flag); + folio_get(folio); + folio_lock(folio); + folio_mark_dirty(folio); + if (metapage_write_one(folio)) + jfs_error(mp->sb, "metapage_write_one() failed\n"); + clear_bit(META_forcewrite, &mp->flag); + folio_put(folio); +} + +void hold_metapage(struct metapage *mp) +{ + folio_lock(mp->folio); +} + +void put_metapage(struct metapage *mp) +{ + if (mp->count || mp->nohomeok) { + /* Someone else will release this */ + folio_unlock(mp->folio); + return; + } + folio_get(mp->folio); + mp->count++; + lock_metapage(mp); + folio_unlock(mp->folio); + release_metapage(mp); +} + +void release_metapage(struct metapage * mp) +{ + struct folio *folio = mp->folio; + jfs_info("release_metapage: mp = 0x%p, flag = 0x%lx", mp, mp->flag); + + folio_lock(folio); + unlock_metapage(mp); + + assert(mp->count); + if (--mp->count || mp->nohomeok) { + folio_unlock(folio); + folio_put(folio); + return; + } + + if (test_bit(META_dirty, &mp->flag)) { + folio_mark_dirty(folio); + if (test_bit(META_sync, &mp->flag)) { + clear_bit(META_sync, &mp->flag); + if (metapage_write_one(folio)) + jfs_error(mp->sb, "metapage_write_one() failed\n"); + folio_lock(folio); + } + } else if (mp->lsn) /* discard_metapage doesn't remove it */ + remove_from_logsync(mp); + + /* Try to keep metapages from using up too much memory */ + drop_metapage(folio, mp); + + folio_unlock(folio); + folio_put(folio); +} + +void __invalidate_metapages(struct inode *ip, s64 addr, int len) +{ + sector_t lblock; + int l2BlocksPerPage = PAGE_SHIFT - ip->i_blkbits; + int BlocksPerPage = 1 << l2BlocksPerPage; + /* All callers are interested in block device's mapping */ + struct address_space *mapping = + JFS_SBI(ip->i_sb)->direct_inode->i_mapping; + struct metapage *mp; + unsigned int offset; + + /* + * Mark metapages to discard. They will eventually be + * released, but should not be written. + */ + for (lblock = addr & ~(BlocksPerPage - 1); lblock < addr + len; + lblock += BlocksPerPage) { + struct folio *folio = filemap_lock_folio(mapping, + lblock >> l2BlocksPerPage); + if (IS_ERR(folio)) + continue; + for (offset = 0; offset < PAGE_SIZE; offset += PSIZE) { + mp = folio_to_mp(folio, offset); + if (!mp) + continue; + if (mp->index < addr) + continue; + if (mp->index >= addr + len) + break; + + clear_bit(META_dirty, &mp->flag); + set_bit(META_discard, &mp->flag); + if (mp->lsn) + remove_from_logsync(mp); + } + folio_unlock(folio); + folio_put(folio); + } +} + +#ifdef CONFIG_JFS_STATISTICS +int jfs_mpstat_proc_show(struct seq_file *m, void *v) +{ + seq_printf(m, + "JFS Metapage statistics\n" + "=======================\n" + "page allocations = %d\n" + "page frees = %d\n" + "lock waits = %d\n", + mpStat.pagealloc, + mpStat.pagefree, + mpStat.lockwait); + return 0; +} +#endif diff --git a/fs/jfs/jfs_metapage.h b/fs/jfs/jfs_metapage.h new file mode 100644 index 000000000..2e5015c27 --- /dev/null +++ b/fs/jfs/jfs_metapage.h @@ -0,0 +1,142 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) International Business Machines Corp., 2000-2002 + * Portions Copyright (C) Christoph Hellwig, 2001-2002 + */ +#ifndef _H_JFS_METAPAGE +#define _H_JFS_METAPAGE + +#include + +struct metapage { + /* Common logsyncblk prefix (see jfs_logmgr.h) */ + u16 xflag; + u16 unused; + lid_t lid; + int lsn; + struct list_head synclist; + /* End of logsyncblk prefix */ + + unsigned long flag; /* See Below */ + unsigned long count; /* Reference count */ + void *data; /* Data pointer */ + sector_t index; /* block address of page */ + wait_queue_head_t wait; + + /* implementation */ + struct folio *folio; + struct super_block *sb; + unsigned int logical_size; + + /* Journal management */ + int clsn; + int nohomeok; + struct jfs_log *log; +}; + +/* metapage flag */ +#define META_locked 0 +#define META_dirty 2 +#define META_sync 3 +#define META_discard 4 +#define META_forcewrite 5 +#define META_io 6 + +#define mark_metapage_dirty(mp) set_bit(META_dirty, &(mp)->flag) + +/* function prototypes */ +extern int metapage_init(void); +extern void metapage_exit(void); +extern struct metapage *__get_metapage(struct inode *inode, + unsigned long lblock, unsigned int size, + int absolute, unsigned long new); + +#define read_metapage(inode, lblock, size, absolute)\ + __get_metapage(inode, lblock, size, absolute, false) + +#define get_metapage(inode, lblock, size, absolute)\ + __get_metapage(inode, lblock, size, absolute, true) + +extern void release_metapage(struct metapage *); +extern void grab_metapage(struct metapage *); +extern void force_metapage(struct metapage *); + +/* + * hold_metapage and put_metapage are used in conjunction. The page lock + * is not dropped between the two, so no other threads can get or release + * the metapage + */ +extern void hold_metapage(struct metapage *); +extern void put_metapage(struct metapage *); + +static inline void write_metapage(struct metapage *mp) +{ + set_bit(META_dirty, &mp->flag); + release_metapage(mp); +} + +static inline void flush_metapage(struct metapage *mp) +{ + set_bit(META_sync, &mp->flag); + write_metapage(mp); +} + +static inline void discard_metapage(struct metapage *mp) +{ + clear_bit(META_dirty, &mp->flag); + set_bit(META_discard, &mp->flag); + release_metapage(mp); +} + +static inline void metapage_nohomeok(struct metapage *mp) +{ + struct folio *folio = mp->folio; + folio_lock(folio); + if (!mp->nohomeok++) { + mark_metapage_dirty(mp); + folio_get(folio); + folio_wait_writeback(folio); + } + folio_unlock(folio); +} + +/* + * This serializes access to mp->lsn when metapages are added to logsynclist + * without setting nohomeok. i.e. updating imap & dmap + */ +static inline void metapage_wait_for_io(struct metapage *mp) +{ + if (test_bit(META_io, &mp->flag)) + folio_wait_writeback(mp->folio); +} + +/* + * This is called when already holding the metapage + */ +static inline void _metapage_homeok(struct metapage *mp) +{ + if (!--mp->nohomeok) + folio_put(mp->folio); +} + +static inline void metapage_homeok(struct metapage *mp) +{ + hold_metapage(mp); + _metapage_homeok(mp); + put_metapage(mp); +} + +extern const struct address_space_operations jfs_metapage_aops; + +/* + * This routines invalidate all pages for an extent. + */ +extern void __invalidate_metapages(struct inode *, s64, int); +#define invalidate_pxd_metapages(ip, pxd) \ + __invalidate_metapages((ip), addressPXD(&(pxd)), lengthPXD(&(pxd))) +#define invalidate_dxd_metapages(ip, dxd) \ + __invalidate_metapages((ip), addressDXD(&(dxd)), lengthDXD(&(dxd))) +#define invalidate_xad_metapages(ip, xad) \ + __invalidate_metapages((ip), addressXAD(&(xad)), lengthXAD(&(xad))) + +#endif /* _H_JFS_METAPAGE */ diff --git a/fs/jfs/jfs_mount.c b/fs/jfs/jfs_mount.c new file mode 100644 index 000000000..dac822f15 --- /dev/null +++ b/fs/jfs/jfs_mount.c @@ -0,0 +1,502 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) International Business Machines Corp., 2000-2004 + */ + +/* + * Module: jfs_mount.c + * + * note: file system in transition to aggregate/fileset: + * + * file system mount is interpreted as the mount of aggregate, + * if not already mounted, and mount of the single/only fileset in + * the aggregate; + * + * a file system/aggregate is represented by an internal inode + * (aka mount inode) initialized with aggregate superblock; + * each vfs represents a fileset, and points to its "fileset inode + * allocation map inode" (aka fileset inode): + * (an aggregate itself is structured recursively as a filset: + * an internal vfs is constructed and points to its "fileset inode + * allocation map inode" (aka aggregate inode) where each inode + * represents a fileset inode) so that inode number is mapped to + * on-disk inode in uniform way at both aggregate and fileset level; + * + * each vnode/inode of a fileset is linked to its vfs (to facilitate + * per fileset inode operations, e.g., unmount of a fileset, etc.); + * each inode points to the mount inode (to facilitate access to + * per aggregate information, e.g., block size, etc.) as well as + * its file set inode. + * + * aggregate + * ipmnt + * mntvfs -> fileset ipimap+ -> aggregate ipbmap -> aggregate ipaimap; + * fileset vfs -> vp(1) <-> ... <-> vp(n) <->vproot; + */ + +#include +#include +#include +#include + +#include "jfs_incore.h" +#include "jfs_filsys.h" +#include "jfs_superblock.h" +#include "jfs_dmap.h" +#include "jfs_imap.h" +#include "jfs_metapage.h" +#include "jfs_debug.h" + + +/* + * forward references + */ +static int chkSuper(struct super_block *); +static int logMOUNT(struct super_block *sb); + +/* + * NAME: jfs_mount(sb) + * + * FUNCTION: vfs_mount() + * + * PARAMETER: sb - super block + * + * RETURN: -EBUSY - device already mounted or open for write + * -EBUSY - cvrdvp already mounted; + * -EBUSY - mount table full + * -ENOTDIR- cvrdvp not directory on a device mount + * -ENXIO - device open failure + */ +int jfs_mount(struct super_block *sb) +{ + int rc = 0; /* Return code */ + struct jfs_sb_info *sbi = JFS_SBI(sb); + struct inode *ipaimap = NULL; + struct inode *ipaimap2 = NULL; + struct inode *ipimap = NULL; + struct inode *ipbmap = NULL; + + /* + * read/validate superblock + * (initialize mount inode from the superblock) + */ + if ((rc = chkSuper(sb))) { + goto out; + } + + ipaimap = diReadSpecial(sb, AGGREGATE_I, 0); + if (ipaimap == NULL) { + jfs_err("jfs_mount: Failed to read AGGREGATE_I"); + rc = -EIO; + goto out; + } + sbi->ipaimap = ipaimap; + + jfs_info("jfs_mount: ipaimap:0x%p", ipaimap); + + /* + * initialize aggregate inode allocation map + */ + if ((rc = diMount(ipaimap))) { + jfs_err("jfs_mount: diMount(ipaimap) failed w/rc = %d", rc); + goto err_ipaimap; + } + + /* + * open aggregate block allocation map + */ + ipbmap = diReadSpecial(sb, BMAP_I, 0); + if (ipbmap == NULL) { + rc = -EIO; + goto err_umount_ipaimap; + } + + jfs_info("jfs_mount: ipbmap:0x%p", ipbmap); + + sbi->ipbmap = ipbmap; + + /* + * initialize aggregate block allocation map + */ + if ((rc = dbMount(ipbmap))) { + jfs_err("jfs_mount: dbMount failed w/rc = %d", rc); + goto err_ipbmap; + } + + /* + * open the secondary aggregate inode allocation map + * + * This is a duplicate of the aggregate inode allocation map. + * + * hand craft a vfs in the same fashion as we did to read ipaimap. + * By adding INOSPEREXT (32) to the inode number, we are telling + * diReadSpecial that we are reading from the secondary aggregate + * inode table. This also creates a unique entry in the inode hash + * table. + */ + if ((sbi->mntflag & JFS_BAD_SAIT) == 0) { + ipaimap2 = diReadSpecial(sb, AGGREGATE_I, 1); + if (!ipaimap2) { + jfs_err("jfs_mount: Failed to read AGGREGATE_I"); + rc = -EIO; + goto err_umount_ipbmap; + } + sbi->ipaimap2 = ipaimap2; + + jfs_info("jfs_mount: ipaimap2:0x%p", ipaimap2); + + /* + * initialize secondary aggregate inode allocation map + */ + if ((rc = diMount(ipaimap2))) { + jfs_err("jfs_mount: diMount(ipaimap2) failed, rc = %d", + rc); + goto err_ipaimap2; + } + } else + /* Secondary aggregate inode table is not valid */ + sbi->ipaimap2 = NULL; + + /* + * mount (the only/single) fileset + */ + /* + * open fileset inode allocation map (aka fileset inode) + */ + ipimap = diReadSpecial(sb, FILESYSTEM_I, 0); + if (ipimap == NULL) { + jfs_err("jfs_mount: Failed to read FILESYSTEM_I"); + /* open fileset secondary inode allocation map */ + rc = -EIO; + goto err_umount_ipaimap2; + } + jfs_info("jfs_mount: ipimap:0x%p", ipimap); + + /* initialize fileset inode allocation map */ + if ((rc = diMount(ipimap))) { + jfs_err("jfs_mount: diMount failed w/rc = %d", rc); + goto err_ipimap; + } + + /* map further access of per fileset inodes by the fileset inode */ + sbi->ipimap = ipimap; + + return rc; + + /* + * unwind on error + */ +err_ipimap: + /* close fileset inode allocation map inode */ + diFreeSpecial(ipimap); +err_umount_ipaimap2: + /* close secondary aggregate inode allocation map */ + if (ipaimap2) + diUnmount(ipaimap2, 1); +err_ipaimap2: + /* close aggregate inodes */ + if (ipaimap2) + diFreeSpecial(ipaimap2); +err_umount_ipbmap: /* close aggregate block allocation map */ + dbUnmount(ipbmap, 1); +err_ipbmap: /* close aggregate inodes */ + diFreeSpecial(ipbmap); +err_umount_ipaimap: /* close aggregate inode allocation map */ + diUnmount(ipaimap, 1); +err_ipaimap: /* close aggregate inodes */ + diFreeSpecial(ipaimap); +out: + if (rc) + jfs_err("Mount JFS Failure: %d", rc); + + return rc; +} + +/* + * NAME: jfs_mount_rw(sb, remount) + * + * FUNCTION: Completes read-write mount, or remounts read-only volume + * as read-write + */ +int jfs_mount_rw(struct super_block *sb, int remount) +{ + struct jfs_sb_info *sbi = JFS_SBI(sb); + int rc; + + /* + * If we are re-mounting a previously read-only volume, we want to + * re-read the inode and block maps, since fsck.jfs may have updated + * them. + */ + if (remount) { + if (chkSuper(sb) || (sbi->state != FM_CLEAN)) + return -EINVAL; + + truncate_inode_pages(sbi->ipimap->i_mapping, 0); + truncate_inode_pages(sbi->ipbmap->i_mapping, 0); + + IWRITE_LOCK(sbi->ipimap, RDWRLOCK_IMAP); + diUnmount(sbi->ipimap, 1); + if ((rc = diMount(sbi->ipimap))) { + IWRITE_UNLOCK(sbi->ipimap); + jfs_err("jfs_mount_rw: diMount failed!"); + return rc; + } + IWRITE_UNLOCK(sbi->ipimap); + + dbUnmount(sbi->ipbmap, 1); + if ((rc = dbMount(sbi->ipbmap))) { + jfs_err("jfs_mount_rw: dbMount failed!"); + return rc; + } + } + + /* + * open/initialize log + */ + if ((rc = lmLogOpen(sb))) + return rc; + + /* + * update file system superblock; + */ + if ((rc = updateSuper(sb, FM_MOUNT))) { + jfs_err("jfs_mount: updateSuper failed w/rc = %d", rc); + lmLogClose(sb); + return rc; + } + + /* + * write MOUNT log record of the file system + */ + logMOUNT(sb); + + return rc; +} + +/* + * chkSuper() + * + * validate the superblock of the file system to be mounted and + * get the file system parameters. + * + * returns + * 0 with fragsize set if check successful + * error code if not successful + */ +static int chkSuper(struct super_block *sb) +{ + int rc = 0; + struct jfs_sb_info *sbi = JFS_SBI(sb); + struct jfs_superblock *j_sb; + struct buffer_head *bh; + int AIM_bytesize, AIT_bytesize; + int expected_AIM_bytesize, expected_AIT_bytesize; + s64 AIM_byte_addr, AIT_byte_addr, fsckwsp_addr; + s64 byte_addr_diff0, byte_addr_diff1; + s32 bsize; + + if ((rc = readSuper(sb, &bh))) + return rc; + j_sb = (struct jfs_superblock *)bh->b_data; + + /* + * validate superblock + */ + /* validate fs signature */ + if (strncmp(j_sb->s_magic, JFS_MAGIC, 4) || + le32_to_cpu(j_sb->s_version) > JFS_VERSION) { + rc = -EINVAL; + goto out; + } + + bsize = le32_to_cpu(j_sb->s_bsize); + if (bsize != PSIZE) { + jfs_err("Only 4K block size supported!"); + rc = -EINVAL; + goto out; + } + + jfs_info("superblock: flag:0x%08x state:0x%08x size:0x%Lx", + le32_to_cpu(j_sb->s_flag), le32_to_cpu(j_sb->s_state), + (unsigned long long) le64_to_cpu(j_sb->s_size)); + + /* validate the descriptors for Secondary AIM and AIT */ + if ((j_sb->s_flag & cpu_to_le32(JFS_BAD_SAIT)) != + cpu_to_le32(JFS_BAD_SAIT)) { + expected_AIM_bytesize = 2 * PSIZE; + AIM_bytesize = lengthPXD(&j_sb->s_aim2) * bsize; + expected_AIT_bytesize = 4 * PSIZE; + AIT_bytesize = lengthPXD(&j_sb->s_ait2) * bsize; + AIM_byte_addr = addressPXD(&j_sb->s_aim2) * bsize; + AIT_byte_addr = addressPXD(&j_sb->s_ait2) * bsize; + byte_addr_diff0 = AIT_byte_addr - AIM_byte_addr; + fsckwsp_addr = addressPXD(&j_sb->s_fsckpxd) * bsize; + byte_addr_diff1 = fsckwsp_addr - AIT_byte_addr; + if ((AIM_bytesize != expected_AIM_bytesize) || + (AIT_bytesize != expected_AIT_bytesize) || + (byte_addr_diff0 != AIM_bytesize) || + (byte_addr_diff1 <= AIT_bytesize)) + j_sb->s_flag |= cpu_to_le32(JFS_BAD_SAIT); + } + + if ((j_sb->s_flag & cpu_to_le32(JFS_GROUPCOMMIT)) != + cpu_to_le32(JFS_GROUPCOMMIT)) + j_sb->s_flag |= cpu_to_le32(JFS_GROUPCOMMIT); + + /* validate fs state */ + if (j_sb->s_state != cpu_to_le32(FM_CLEAN) && + !sb_rdonly(sb)) { + jfs_err("jfs_mount: Mount Failure: File System Dirty."); + rc = -EINVAL; + goto out; + } + + sbi->state = le32_to_cpu(j_sb->s_state); + sbi->mntflag = le32_to_cpu(j_sb->s_flag); + + /* + * JFS always does I/O by 4K pages. Don't tell the buffer cache + * that we use anything else (leave s_blocksize alone). + */ + sbi->bsize = bsize; + sbi->l2bsize = le16_to_cpu(j_sb->s_l2bsize); + + /* check some fields for possible corruption */ + if (sbi->l2bsize != ilog2((u32)bsize) || + j_sb->pad != 0 || + le32_to_cpu(j_sb->s_state) > FM_STATE_MAX) { + rc = -EINVAL; + jfs_err("jfs_mount: Mount Failure: superblock is corrupt!"); + goto out; + } + + /* + * For now, ignore s_pbsize, l2bfactor. All I/O going through buffer + * cache. + */ + sbi->nbperpage = PSIZE >> sbi->l2bsize; + sbi->l2nbperpage = L2PSIZE - sbi->l2bsize; + sbi->l2niperblk = sbi->l2bsize - L2DISIZE; + uuid_copy(&sbi->uuid, &j_sb->s_uuid); + + if (sbi->mntflag & JFS_INLINELOG) + sbi->logpxd = j_sb->s_logpxd; + else { + sbi->logdev = new_decode_dev(le32_to_cpu(j_sb->s_logdev)); + uuid_copy(&sbi->loguuid, &j_sb->s_loguuid); + } + sbi->fsckpxd = j_sb->s_fsckpxd; + sbi->ait2 = j_sb->s_ait2; + + out: + brelse(bh); + return rc; +} + + +/* + * updateSuper() + * + * update synchronously superblock if it is mounted read-write. + */ +int updateSuper(struct super_block *sb, uint state) +{ + struct jfs_superblock *j_sb; + struct jfs_sb_info *sbi = JFS_SBI(sb); + struct buffer_head *bh; + int rc; + + if (sbi->flag & JFS_NOINTEGRITY) { + if (state == FM_DIRTY) { + sbi->p_state = state; + return 0; + } else if (state == FM_MOUNT) { + sbi->p_state = sbi->state; + state = FM_DIRTY; + } else if (state == FM_CLEAN) { + state = sbi->p_state; + } else + jfs_err("updateSuper: bad state"); + } else if (sbi->state == FM_DIRTY) + return 0; + + if ((rc = readSuper(sb, &bh))) + return rc; + + j_sb = (struct jfs_superblock *)bh->b_data; + + j_sb->s_state = cpu_to_le32(state); + sbi->state = state; + + if (state == FM_MOUNT) { + /* record log's dev_t and mount serial number */ + j_sb->s_logdev = cpu_to_le32( + new_encode_dev(file_bdev(sbi->log->bdev_file)->bd_dev)); + j_sb->s_logserial = cpu_to_le32(sbi->log->serial); + } else if (state == FM_CLEAN) { + /* + * If this volume is shared with OS/2, OS/2 will need to + * recalculate DASD usage, since we don't deal with it. + */ + if (j_sb->s_flag & cpu_to_le32(JFS_DASD_ENABLED)) + j_sb->s_flag |= cpu_to_le32(JFS_DASD_PRIME); + } + + mark_buffer_dirty(bh); + sync_dirty_buffer(bh); + brelse(bh); + + return 0; +} + + +/* + * readSuper() + * + * read superblock by raw sector address + */ +int readSuper(struct super_block *sb, struct buffer_head **bpp) +{ + /* read in primary superblock */ + *bpp = sb_bread(sb, SUPER1_OFF >> sb->s_blocksize_bits); + if (*bpp) + return 0; + + /* read in secondary/replicated superblock */ + *bpp = sb_bread(sb, SUPER2_OFF >> sb->s_blocksize_bits); + if (*bpp) + return 0; + + return -EIO; +} + + +/* + * logMOUNT() + * + * function: write a MOUNT log record for file system. + * + * MOUNT record keeps logredo() from processing log records + * for this file system past this point in log. + * it is harmless if mount fails. + * + * note: MOUNT record is at aggregate level, not at fileset level, + * since log records of previous mounts of a fileset + * (e.g., AFTER record of extent allocation) have to be processed + * to update block allocation map at aggregate level. + */ +static int logMOUNT(struct super_block *sb) +{ + struct jfs_log *log = JFS_SBI(sb)->log; + struct lrd lrd; + + lrd.logtid = 0; + lrd.backchain = 0; + lrd.type = cpu_to_le16(LOG_MOUNT); + lrd.length = 0; + lrd.aggregate = cpu_to_le32(new_encode_dev(sb->s_bdev->bd_dev)); + lmLog(log, NULL, &lrd, NULL); + + return 0; +} diff --git a/fs/jfs/jfs_superblock.h b/fs/jfs/jfs_superblock.h new file mode 100644 index 000000000..93402c420 --- /dev/null +++ b/fs/jfs/jfs_superblock.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) International Business Machines Corp., 2000-2003 + */ +#ifndef _H_JFS_SUPERBLOCK +#define _H_JFS_SUPERBLOCK + +#include + +/* + * make the magic number something a human could read + */ +#define JFS_MAGIC "JFS1" /* Magic word */ + +#define JFS_VERSION 2 /* Version number: Version 2 */ + +#define LV_NAME_SIZE 11 /* MUST BE 11 for OS/2 boot sector */ + +/* + * aggregate superblock + * + * The name superblock is too close to super_block, so the name has been + * changed to jfs_superblock. The utilities are still using the old name. + */ +struct jfs_superblock { + char s_magic[4]; /* 4: magic number */ + __le32 s_version; /* 4: version number */ + + __le64 s_size; /* 8: aggregate size in hardware/LVM blocks; + * VFS: number of blocks + */ + __le32 s_bsize; /* 4: aggregate block size in bytes; + * VFS: fragment size + */ + __le16 s_l2bsize; /* 2: log2 of s_bsize */ + __le16 s_l2bfactor; /* 2: log2(s_bsize/hardware block size) */ + __le32 s_pbsize; /* 4: hardware/LVM block size in bytes */ + __le16 s_l2pbsize; /* 2: log2 of s_pbsize */ + __le16 pad; /* 2: padding necessary for alignment */ + + __le32 s_agsize; /* 4: allocation group size in aggr. blocks */ + + __le32 s_flag; /* 4: aggregate attributes: + * see jfs_filsys.h + */ + __le32 s_state; /* 4: mount/unmount/recovery state: + * see jfs_filsys.h + */ + __le32 s_compress; /* 4: > 0 if data compression */ + + pxd_t s_ait2; /* 8: first extent of secondary + * aggregate inode table + */ + + pxd_t s_aim2; /* 8: first extent of secondary + * aggregate inode map + */ + __le32 s_logdev; /* 4: device address of log */ + __le32 s_logserial; /* 4: log serial number at aggregate mount */ + pxd_t s_logpxd; /* 8: inline log extent */ + + pxd_t s_fsckpxd; /* 8: inline fsck work space extent */ + + struct timestruc_t s_time; /* 8: time last updated */ + + __le32 s_fsckloglen; /* 4: Number of filesystem blocks reserved for + * the fsck service log. + * N.B. These blocks are divided among the + * versions kept. This is not a per + * version size. + * N.B. These blocks are included in the + * length field of s_fsckpxd. + */ + s8 s_fscklog; /* 1: which fsck service log is most recent + * 0 => no service log data yet + * 1 => the first one + * 2 => the 2nd one + */ + char s_fpack[11]; /* 11: file system volume name + * N.B. This must be 11 bytes to + * conform with the OS/2 BootSector + * requirements + * Only used when s_version is 1 + */ + + /* extendfs() parameter under s_state & FM_EXTENDFS */ + __le64 s_xsize; /* 8: extendfs s_size */ + pxd_t s_xfsckpxd; /* 8: extendfs fsckpxd */ + pxd_t s_xlogpxd; /* 8: extendfs logpxd */ + uuid_t s_uuid; /* 16: 128-bit uuid for volume */ + char s_label[16]; /* 16: volume label */ + uuid_t s_loguuid; /* 16: 128-bit uuid for log device */ + +}; + +extern int readSuper(struct super_block *, struct buffer_head **); +extern int updateSuper(struct super_block *, uint); +__printf(2, 3) +extern void jfs_error(struct super_block *, const char *, ...); +extern int jfs_mount(struct super_block *); +extern int jfs_mount_rw(struct super_block *, int); +extern int jfs_umount(struct super_block *); +extern int jfs_umount_rw(struct super_block *); +extern int jfs_extendfs(struct super_block *, s64, int); + +extern struct task_struct *jfsIOthread; +extern struct task_struct *jfsSyncThread; + +#endif /*_H_JFS_SUPERBLOCK */ diff --git a/fs/jfs/jfs_txnmgr.c b/fs/jfs/jfs_txnmgr.c new file mode 100644 index 000000000..083dbbb0c --- /dev/null +++ b/fs/jfs/jfs_txnmgr.c @@ -0,0 +1,3021 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) International Business Machines Corp., 2000-2005 + * Portions Copyright (C) Christoph Hellwig, 2001-2002 + */ + +/* + * jfs_txnmgr.c: transaction manager + * + * notes: + * transaction starts with txBegin() and ends with txCommit() + * or txAbort(). + * + * tlock is acquired at the time of update; + * (obviate scan at commit time for xtree and dtree) + * tlock and mp points to each other; + * (no hashlist for mp -> tlock). + * + * special cases: + * tlock on in-memory inode: + * in-place tlock in the in-memory inode itself; + * converted to page lock by iWrite() at commit time. + * + * tlock during write()/mmap() under anonymous transaction (tid = 0): + * transferred (?) to transaction at commit time. + * + * use the page itself to update allocation maps + * (obviate intermediate replication of allocation/deallocation data) + * hold on to mp+lock thru update of maps + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "jfs_incore.h" +#include "jfs_inode.h" +#include "jfs_filsys.h" +#include "jfs_metapage.h" +#include "jfs_dinode.h" +#include "jfs_imap.h" +#include "jfs_dmap.h" +#include "jfs_superblock.h" +#include "jfs_debug.h" + +/* + * transaction management structures + */ +static struct { + int freetid; /* index of a free tid structure */ + int freelock; /* index first free lock word */ + wait_queue_head_t freewait; /* eventlist of free tblock */ + wait_queue_head_t freelockwait; /* eventlist of free tlock */ + wait_queue_head_t lowlockwait; /* eventlist of ample tlocks */ + int tlocksInUse; /* Number of tlocks in use */ + spinlock_t LazyLock; /* synchronize sync_queue & unlock_queue */ +/* struct tblock *sync_queue; * Transactions waiting for data sync */ + struct list_head unlock_queue; /* Txns waiting to be released */ + struct list_head anon_list; /* inodes having anonymous txns */ + struct list_head anon_list2; /* inodes having anonymous txns + that couldn't be sync'ed */ +} TxAnchor; + +int jfs_tlocks_low; /* Indicates low number of available tlocks */ + +#ifdef CONFIG_JFS_STATISTICS +static struct { + uint txBegin; + uint txBegin_barrier; + uint txBegin_lockslow; + uint txBegin_freetid; + uint txBeginAnon; + uint txBeginAnon_barrier; + uint txBeginAnon_lockslow; + uint txLockAlloc; + uint txLockAlloc_freelock; +} TxStat; +#endif + +static int nTxBlock = -1; /* number of transaction blocks */ +module_param(nTxBlock, int, 0); +MODULE_PARM_DESC(nTxBlock, + "Number of transaction blocks (max:65536)"); + +static int nTxLock = -1; /* number of transaction locks */ +module_param(nTxLock, int, 0); +MODULE_PARM_DESC(nTxLock, + "Number of transaction locks (max:65536)"); + +struct tblock *TxBlock; /* transaction block table */ +static int TxLockLWM; /* Low water mark for number of txLocks used */ +static int TxLockHWM; /* High water mark for number of txLocks used */ +static int TxLockVHWM; /* Very High water mark */ +struct tlock *TxLock; /* transaction lock table */ + +/* + * transaction management lock + */ +static DEFINE_SPINLOCK(jfsTxnLock); + +#define TXN_LOCK() spin_lock(&jfsTxnLock) +#define TXN_UNLOCK() spin_unlock(&jfsTxnLock) + +#define LAZY_LOCK_INIT() spin_lock_init(&TxAnchor.LazyLock) +#define LAZY_LOCK(flags) spin_lock_irqsave(&TxAnchor.LazyLock, flags) +#define LAZY_UNLOCK(flags) spin_unlock_irqrestore(&TxAnchor.LazyLock, flags) + +static DECLARE_WAIT_QUEUE_HEAD(jfs_commit_thread_wait); +static int jfs_commit_thread_waking; + +/* + * Retry logic exist outside these macros to protect from spurrious wakeups. + */ +static inline void TXN_SLEEP_DROP_LOCK(wait_queue_head_t * event) +{ + DECLARE_WAITQUEUE(wait, current); + + add_wait_queue(event, &wait); + set_current_state(TASK_UNINTERRUPTIBLE); + TXN_UNLOCK(); + io_schedule(); + remove_wait_queue(event, &wait); +} + +#define TXN_SLEEP(event)\ +{\ + TXN_SLEEP_DROP_LOCK(event);\ + TXN_LOCK();\ +} + +#define TXN_WAKEUP(event) wake_up_all(event) + +/* + * statistics + */ +static struct { + tid_t maxtid; /* 4: biggest tid ever used */ + lid_t maxlid; /* 4: biggest lid ever used */ + int ntid; /* 4: # of transactions performed */ + int nlid; /* 4: # of tlocks acquired */ + int waitlock; /* 4: # of tlock wait */ +} stattx; + +/* + * forward references + */ +static void diLog(struct jfs_log *log, struct tblock *tblk, struct lrd *lrd, + struct tlock *tlck, struct commit *cd); +static void dataLog(struct jfs_log *log, struct tblock *tblk, struct lrd *lrd, + struct tlock *tlck); +static void dtLog(struct jfs_log * log, struct tblock * tblk, struct lrd * lrd, + struct tlock * tlck); +static void mapLog(struct jfs_log * log, struct tblock * tblk, struct lrd * lrd, + struct tlock * tlck); +static void txAllocPMap(struct inode *ip, struct maplock * maplock, + struct tblock * tblk); +static void txForce(struct tblock * tblk); +static void txLog(struct jfs_log *log, struct tblock *tblk, + struct commit *cd); +static void txUpdateMap(struct tblock * tblk); +static void txRelease(struct tblock * tblk); +static void xtLog(struct jfs_log * log, struct tblock * tblk, struct lrd * lrd, + struct tlock * tlck); +static void LogSyncRelease(struct metapage * mp); + +/* + * transaction block/lock management + * --------------------------------- + */ + +/* + * Get a transaction lock from the free list. If the number in use is + * greater than the high water mark, wake up the sync daemon. This should + * free some anonymous transaction locks. (TXN_LOCK must be held.) + */ +static lid_t txLockAlloc(void) +{ + lid_t lid; + + INCREMENT(TxStat.txLockAlloc); + if (!TxAnchor.freelock) { + INCREMENT(TxStat.txLockAlloc_freelock); + } + + while (!(lid = TxAnchor.freelock)) + TXN_SLEEP(&TxAnchor.freelockwait); + TxAnchor.freelock = TxLock[lid].next; + HIGHWATERMARK(stattx.maxlid, lid); + if ((++TxAnchor.tlocksInUse > TxLockHWM) && (jfs_tlocks_low == 0)) { + jfs_info("txLockAlloc tlocks low"); + jfs_tlocks_low = 1; + wake_up_process(jfsSyncThread); + } + + return lid; +} + +static void txLockFree(lid_t lid) +{ + TxLock[lid].tid = 0; + TxLock[lid].next = TxAnchor.freelock; + TxAnchor.freelock = lid; + TxAnchor.tlocksInUse--; + if (jfs_tlocks_low && (TxAnchor.tlocksInUse < TxLockLWM)) { + jfs_info("txLockFree jfs_tlocks_low no more"); + jfs_tlocks_low = 0; + TXN_WAKEUP(&TxAnchor.lowlockwait); + } + TXN_WAKEUP(&TxAnchor.freelockwait); +} + +/* + * NAME: txInit() + * + * FUNCTION: initialize transaction management structures + * + * RETURN: + * + * serialization: single thread at jfs_init() + */ +int txInit(void) +{ + int k, size; + struct sysinfo si; + + /* Set defaults for nTxLock and nTxBlock if unset */ + + if (nTxLock == -1) { + if (nTxBlock == -1) { + /* Base default on memory size */ + si_meminfo(&si); + if (si.totalram > (256 * 1024)) /* 1 GB */ + nTxLock = 64 * 1024; + else + nTxLock = si.totalram >> 2; + } else if (nTxBlock > (8 * 1024)) + nTxLock = 64 * 1024; + else + nTxLock = nTxBlock << 3; + } + if (nTxBlock == -1) + nTxBlock = nTxLock >> 3; + + /* Verify tunable parameters */ + if (nTxBlock < 16) + nTxBlock = 16; /* No one should set it this low */ + if (nTxBlock > 65536) + nTxBlock = 65536; + if (nTxLock < 256) + nTxLock = 256; /* No one should set it this low */ + if (nTxLock > 65536) + nTxLock = 65536; + + printk(KERN_INFO "JFS: nTxBlock = %d, nTxLock = %d\n", + nTxBlock, nTxLock); + /* + * initialize transaction block (tblock) table + * + * transaction id (tid) = tblock index + * tid = 0 is reserved. + */ + TxLockLWM = (nTxLock * 4) / 10; + TxLockHWM = (nTxLock * 7) / 10; + TxLockVHWM = (nTxLock * 8) / 10; + + size = sizeof(struct tblock) * nTxBlock; + TxBlock = vmalloc(size); + if (TxBlock == NULL) + return -ENOMEM; + + for (k = 0; k < nTxBlock; k++) { + init_waitqueue_head(&TxBlock[k].gcwait); + init_waitqueue_head(&TxBlock[k].waitor); + INIT_LIST_HEAD(&TxBlock[k].synclist); + } + + for (k = 1; k < nTxBlock - 1; k++) { + TxBlock[k].next = k + 1; + } + TxBlock[k].next = 0; + + TxAnchor.freetid = 1; + init_waitqueue_head(&TxAnchor.freewait); + + stattx.maxtid = 1; /* statistics */ + + /* + * initialize transaction lock (tlock) table + * + * transaction lock id = tlock index + * tlock id = 0 is reserved. + */ + size = sizeof(struct tlock) * nTxLock; + TxLock = vmalloc(size); + if (TxLock == NULL) { + vfree(TxBlock); + return -ENOMEM; + } + + /* initialize tlock table */ + for (k = 1; k < nTxLock - 1; k++) + TxLock[k].next = k + 1; + TxLock[k].next = 0; + init_waitqueue_head(&TxAnchor.freelockwait); + init_waitqueue_head(&TxAnchor.lowlockwait); + + TxAnchor.freelock = 1; + TxAnchor.tlocksInUse = 0; + INIT_LIST_HEAD(&TxAnchor.anon_list); + INIT_LIST_HEAD(&TxAnchor.anon_list2); + + LAZY_LOCK_INIT(); + INIT_LIST_HEAD(&TxAnchor.unlock_queue); + + stattx.maxlid = 1; /* statistics */ + + return 0; +} + +/* + * NAME: txExit() + * + * FUNCTION: clean up when module is unloaded + */ +void txExit(void) +{ + vfree(TxLock); + TxLock = NULL; + vfree(TxBlock); + TxBlock = NULL; +} + +/* + * NAME: txBegin() + * + * FUNCTION: start a transaction. + * + * PARAMETER: sb - superblock + * flag - force for nested tx; + * + * RETURN: tid - transaction id + * + * note: flag force allows to start tx for nested tx + * to prevent deadlock on logsync barrier; + */ +tid_t txBegin(struct super_block *sb, int flag) +{ + tid_t t; + struct tblock *tblk; + struct jfs_log *log; + + jfs_info("txBegin: flag = 0x%x", flag); + log = JFS_SBI(sb)->log; + + if (!log) { + jfs_error(sb, "read-only filesystem\n"); + return 0; + } + + TXN_LOCK(); + + INCREMENT(TxStat.txBegin); + + retry: + if (!(flag & COMMIT_FORCE)) { + /* + * synchronize with logsync barrier + */ + if (test_bit(log_SYNCBARRIER, &log->flag) || + test_bit(log_QUIESCE, &log->flag)) { + INCREMENT(TxStat.txBegin_barrier); + TXN_SLEEP(&log->syncwait); + goto retry; + } + } + if (flag == 0) { + /* + * Don't begin transaction if we're getting starved for tlocks + * unless COMMIT_FORCE or COMMIT_INODE (which may ultimately + * free tlocks) + */ + if (TxAnchor.tlocksInUse > TxLockVHWM) { + INCREMENT(TxStat.txBegin_lockslow); + TXN_SLEEP(&TxAnchor.lowlockwait); + goto retry; + } + } + + /* + * allocate transaction id/block + */ + if ((t = TxAnchor.freetid) == 0) { + jfs_info("txBegin: waiting for free tid"); + INCREMENT(TxStat.txBegin_freetid); + TXN_SLEEP(&TxAnchor.freewait); + goto retry; + } + + tblk = tid_to_tblock(t); + + if ((tblk->next == 0) && !(flag & COMMIT_FORCE)) { + /* Don't let a non-forced transaction take the last tblk */ + jfs_info("txBegin: waiting for free tid"); + INCREMENT(TxStat.txBegin_freetid); + TXN_SLEEP(&TxAnchor.freewait); + goto retry; + } + + TxAnchor.freetid = tblk->next; + + /* + * initialize transaction + */ + + /* + * We can't zero the whole thing or we screw up another thread being + * awakened after sleeping on tblk->waitor + * + * memset(tblk, 0, sizeof(struct tblock)); + */ + tblk->next = tblk->last = tblk->xflag = tblk->flag = tblk->lsn = 0; + + tblk->sb = sb; + ++log->logtid; + tblk->logtid = log->logtid; + + ++log->active; + + HIGHWATERMARK(stattx.maxtid, t); /* statistics */ + INCREMENT(stattx.ntid); /* statistics */ + + TXN_UNLOCK(); + + jfs_info("txBegin: returning tid = %d", t); + + return t; +} + +/* + * NAME: txBeginAnon() + * + * FUNCTION: start an anonymous transaction. + * Blocks if logsync or available tlocks are low to prevent + * anonymous tlocks from depleting supply. + * + * PARAMETER: sb - superblock + * + * RETURN: none + */ +void txBeginAnon(struct super_block *sb) +{ + struct jfs_log *log; + + log = JFS_SBI(sb)->log; + + TXN_LOCK(); + INCREMENT(TxStat.txBeginAnon); + + retry: + /* + * synchronize with logsync barrier + */ + if (test_bit(log_SYNCBARRIER, &log->flag) || + test_bit(log_QUIESCE, &log->flag)) { + INCREMENT(TxStat.txBeginAnon_barrier); + TXN_SLEEP(&log->syncwait); + goto retry; + } + + /* + * Don't begin transaction if we're getting starved for tlocks + */ + if (TxAnchor.tlocksInUse > TxLockVHWM) { + INCREMENT(TxStat.txBeginAnon_lockslow); + TXN_SLEEP(&TxAnchor.lowlockwait); + goto retry; + } + TXN_UNLOCK(); +} + +/* + * txEnd() + * + * function: free specified transaction block. + * + * logsync barrier processing: + * + * serialization: + */ +void txEnd(tid_t tid) +{ + struct tblock *tblk = tid_to_tblock(tid); + struct jfs_log *log; + + jfs_info("txEnd: tid = %d", tid); + TXN_LOCK(); + + /* + * wakeup transactions waiting on the page locked + * by the current transaction + */ + TXN_WAKEUP(&tblk->waitor); + + log = JFS_SBI(tblk->sb)->log; + + /* + * Lazy commit thread can't free this guy until we mark it UNLOCKED, + * otherwise, we would be left with a transaction that may have been + * reused. + * + * Lazy commit thread will turn off tblkGC_LAZY before calling this + * routine. + */ + if (tblk->flag & tblkGC_LAZY) { + jfs_info("txEnd called w/lazy tid: %d, tblk = 0x%p", tid, tblk); + TXN_UNLOCK(); + + spin_lock_irq(&log->gclock); // LOGGC_LOCK + tblk->flag |= tblkGC_UNLOCKED; + spin_unlock_irq(&log->gclock); // LOGGC_UNLOCK + return; + } + + jfs_info("txEnd: tid: %d, tblk = 0x%p", tid, tblk); + + assert(tblk->next == 0); + + /* + * insert tblock back on freelist + */ + tblk->next = TxAnchor.freetid; + TxAnchor.freetid = tid; + + /* + * mark the tblock not active + */ + if (--log->active == 0) { + clear_bit(log_FLUSH, &log->flag); + + /* + * synchronize with logsync barrier + */ + if (test_bit(log_SYNCBARRIER, &log->flag)) { + TXN_UNLOCK(); + + /* write dirty metadata & forward log syncpt */ + jfs_syncpt(log, 1); + + jfs_info("log barrier off: 0x%x", log->lsn); + + /* enable new transactions start */ + clear_bit(log_SYNCBARRIER, &log->flag); + + /* wakeup all waitors for logsync barrier */ + TXN_WAKEUP(&log->syncwait); + + goto wakeup; + } + } + + TXN_UNLOCK(); +wakeup: + /* + * wakeup all waitors for a free tblock + */ + TXN_WAKEUP(&TxAnchor.freewait); +} + +/* + * txLock() + * + * function: acquire a transaction lock on the specified + * + * parameter: + * + * return: transaction lock id + * + * serialization: + */ +struct tlock *txLock(tid_t tid, struct inode *ip, struct metapage * mp, + int type) +{ + struct jfs_inode_info *jfs_ip = JFS_IP(ip); + int dir_xtree = 0; + lid_t lid; + tid_t xtid; + struct tlock *tlck; + struct xtlock *xtlck; + struct linelock *linelock; + xtpage_t *p; + struct tblock *tblk; + + TXN_LOCK(); + + if (S_ISDIR(ip->i_mode) && (type & tlckXTREE) && + !(mp->xflag & COMMIT_PAGE)) { + /* + * Directory inode is special. It can have both an xtree tlock + * and a dtree tlock associated with it. + */ + dir_xtree = 1; + lid = jfs_ip->xtlid; + } else + lid = mp->lid; + + /* is page not locked by a transaction ? */ + if (lid == 0) + goto allocateLock; + + jfs_info("txLock: tid:%d ip:0x%p mp:0x%p lid:%d", tid, ip, mp, lid); + + /* is page locked by the requester transaction ? */ + tlck = lid_to_tlock(lid); + if ((xtid = tlck->tid) == tid) { + TXN_UNLOCK(); + goto grantLock; + } + + /* + * is page locked by anonymous transaction/lock ? + * + * (page update without transaction (i.e., file write) is + * locked under anonymous transaction tid = 0: + * anonymous tlocks maintained on anonymous tlock list of + * the inode of the page and available to all anonymous + * transactions until txCommit() time at which point + * they are transferred to the transaction tlock list of + * the committing transaction of the inode) + */ + if (xtid == 0) { + tlck->tid = tid; + TXN_UNLOCK(); + tblk = tid_to_tblock(tid); + /* + * The order of the tlocks in the transaction is important + * (during truncate, child xtree pages must be freed before + * parent's tlocks change the working map). + * Take tlock off anonymous list and add to tail of + * transaction list + * + * Note: We really need to get rid of the tid & lid and + * use list_head's. This code is getting UGLY! + */ + if (jfs_ip->atlhead == lid) { + if (jfs_ip->atltail == lid) { + /* only anonymous txn. + * Remove from anon_list + */ + TXN_LOCK(); + list_del_init(&jfs_ip->anon_inode_list); + TXN_UNLOCK(); + } + jfs_ip->atlhead = tlck->next; + } else { + lid_t last; + for (last = jfs_ip->atlhead; + lid_to_tlock(last)->next != lid; + last = lid_to_tlock(last)->next) { + assert(last); + } + lid_to_tlock(last)->next = tlck->next; + if (jfs_ip->atltail == lid) + jfs_ip->atltail = last; + } + + /* insert the tlock at tail of transaction tlock list */ + + if (tblk->next) + lid_to_tlock(tblk->last)->next = lid; + else + tblk->next = lid; + tlck->next = 0; + tblk->last = lid; + + goto grantLock; + } + + goto waitLock; + + /* + * allocate a tlock + */ + allocateLock: + lid = txLockAlloc(); + tlck = lid_to_tlock(lid); + + /* + * initialize tlock + */ + tlck->tid = tid; + + TXN_UNLOCK(); + + /* mark tlock for meta-data page */ + if (mp->xflag & COMMIT_PAGE) { + + tlck->flag = tlckPAGELOCK; + + /* mark the page dirty and nohomeok */ + metapage_nohomeok(mp); + + jfs_info("locking mp = 0x%p, nohomeok = %d tid = %d tlck = 0x%p", + mp, mp->nohomeok, tid, tlck); + + /* if anonymous transaction, and buffer is on the group + * commit synclist, mark inode to show this. This will + * prevent the buffer from being marked nohomeok for too + * long a time. + */ + if ((tid == 0) && mp->lsn) + set_cflag(COMMIT_Synclist, ip); + } + /* mark tlock for in-memory inode */ + else + tlck->flag = tlckINODELOCK; + + if (S_ISDIR(ip->i_mode)) + tlck->flag |= tlckDIRECTORY; + + tlck->type = 0; + + /* bind the tlock and the page */ + tlck->ip = ip; + tlck->mp = mp; + if (dir_xtree) + jfs_ip->xtlid = lid; + else + mp->lid = lid; + + /* + * enqueue transaction lock to transaction/inode + */ + /* insert the tlock at tail of transaction tlock list */ + if (tid) { + tblk = tid_to_tblock(tid); + if (tblk->next) + lid_to_tlock(tblk->last)->next = lid; + else + tblk->next = lid; + tlck->next = 0; + tblk->last = lid; + } + /* anonymous transaction: + * insert the tlock at head of inode anonymous tlock list + */ + else { + tlck->next = jfs_ip->atlhead; + jfs_ip->atlhead = lid; + if (tlck->next == 0) { + /* This inode's first anonymous transaction */ + jfs_ip->atltail = lid; + TXN_LOCK(); + list_add_tail(&jfs_ip->anon_inode_list, + &TxAnchor.anon_list); + TXN_UNLOCK(); + } + } + + /* initialize type dependent area for linelock */ + linelock = (struct linelock *) & tlck->lock; + linelock->next = 0; + linelock->flag = tlckLINELOCK; + linelock->maxcnt = TLOCKSHORT; + linelock->index = 0; + + switch (type & tlckTYPE) { + case tlckDTREE: + linelock->l2linesize = L2DTSLOTSIZE; + break; + + case tlckXTREE: + linelock->l2linesize = L2XTSLOTSIZE; + + xtlck = (struct xtlock *) linelock; + xtlck->header.offset = 0; + xtlck->header.length = 2; + + if (type & tlckNEW) { + xtlck->lwm.offset = XTENTRYSTART; + } else { + if (mp->xflag & COMMIT_PAGE) + p = (xtpage_t *) mp->data; + else + p = (xtpage_t *) &jfs_ip->i_xtroot; + xtlck->lwm.offset = + le16_to_cpu(p->header.nextindex); + } + xtlck->lwm.length = 0; /* ! */ + xtlck->twm.offset = 0; + xtlck->hwm.offset = 0; + + xtlck->index = 2; + break; + + case tlckINODE: + linelock->l2linesize = L2INODESLOTSIZE; + break; + + case tlckDATA: + linelock->l2linesize = L2DATASLOTSIZE; + break; + + default: + jfs_err("UFO tlock:0x%p", tlck); + } + + /* + * update tlock vector + */ + grantLock: + tlck->type |= type; + + return tlck; + + /* + * page is being locked by another transaction: + */ + waitLock: + /* Only locks on ipimap or ipaimap should reach here */ + /* assert(jfs_ip->fileset == AGGREGATE_I); */ + if (jfs_ip->fileset != AGGREGATE_I) { + printk(KERN_ERR "txLock: trying to lock locked page!"); + print_hex_dump(KERN_ERR, "ip: ", DUMP_PREFIX_ADDRESS, 16, 4, + ip, sizeof(*ip), 0); + print_hex_dump(KERN_ERR, "mp: ", DUMP_PREFIX_ADDRESS, 16, 4, + mp, sizeof(*mp), 0); + print_hex_dump(KERN_ERR, "Locker's tblock: ", + DUMP_PREFIX_ADDRESS, 16, 4, tid_to_tblock(tid), + sizeof(struct tblock), 0); + print_hex_dump(KERN_ERR, "Tlock: ", DUMP_PREFIX_ADDRESS, 16, 4, + tlck, sizeof(*tlck), 0); + BUG(); + } + INCREMENT(stattx.waitlock); /* statistics */ + TXN_UNLOCK(); + release_metapage(mp); + TXN_LOCK(); + xtid = tlck->tid; /* reacquire after dropping TXN_LOCK */ + + jfs_info("txLock: in waitLock, tid = %d, xtid = %d, lid = %d", + tid, xtid, lid); + + /* Recheck everything since dropping TXN_LOCK */ + if (xtid && (tlck->mp == mp) && (mp->lid == lid)) + TXN_SLEEP_DROP_LOCK(&tid_to_tblock(xtid)->waitor); + else + TXN_UNLOCK(); + jfs_info("txLock: awakened tid = %d, lid = %d", tid, lid); + + return NULL; +} + +/* + * NAME: txRelease() + * + * FUNCTION: Release buffers associated with transaction locks, but don't + * mark homeok yet. The allows other transactions to modify + * buffers, but won't let them go to disk until commit record + * actually gets written. + * + * PARAMETER: + * tblk - + * + * RETURN: Errors from subroutines. + */ +static void txRelease(struct tblock * tblk) +{ + struct metapage *mp; + lid_t lid; + struct tlock *tlck; + + TXN_LOCK(); + + for (lid = tblk->next; lid; lid = tlck->next) { + tlck = lid_to_tlock(lid); + if ((mp = tlck->mp) != NULL && + (tlck->type & tlckBTROOT) == 0) { + assert(mp->xflag & COMMIT_PAGE); + mp->lid = 0; + } + } + + /* + * wakeup transactions waiting on a page locked + * by the current transaction + */ + TXN_WAKEUP(&tblk->waitor); + + TXN_UNLOCK(); +} + +/* + * NAME: txUnlock() + * + * FUNCTION: Initiates pageout of pages modified by tid in journalled + * objects and frees their lockwords. + */ +static void txUnlock(struct tblock * tblk) +{ + struct tlock *tlck; + struct linelock *linelock; + lid_t lid, next, llid, k; + struct metapage *mp; + struct jfs_log *log; + int difft, diffp; + unsigned long flags; + + jfs_info("txUnlock: tblk = 0x%p", tblk); + log = JFS_SBI(tblk->sb)->log; + + /* + * mark page under tlock homeok (its log has been written): + */ + for (lid = tblk->next; lid; lid = next) { + tlck = lid_to_tlock(lid); + next = tlck->next; + + jfs_info("unlocking lid = %d, tlck = 0x%p", lid, tlck); + + /* unbind page from tlock */ + if ((mp = tlck->mp) != NULL && + (tlck->type & tlckBTROOT) == 0) { + assert(mp->xflag & COMMIT_PAGE); + + /* hold buffer + */ + hold_metapage(mp); + + assert(mp->nohomeok > 0); + _metapage_homeok(mp); + + /* inherit younger/larger clsn */ + LOGSYNC_LOCK(log, flags); + if (mp->clsn) { + logdiff(difft, tblk->clsn, log); + logdiff(diffp, mp->clsn, log); + if (difft > diffp) + mp->clsn = tblk->clsn; + } else + mp->clsn = tblk->clsn; + LOGSYNC_UNLOCK(log, flags); + + assert(!(tlck->flag & tlckFREEPAGE)); + + put_metapage(mp); + } + + /* insert tlock, and linelock(s) of the tlock if any, + * at head of freelist + */ + TXN_LOCK(); + + llid = ((struct linelock *) & tlck->lock)->next; + while (llid) { + linelock = (struct linelock *) lid_to_tlock(llid); + k = linelock->next; + txLockFree(llid); + llid = k; + } + txLockFree(lid); + + TXN_UNLOCK(); + } + tblk->next = tblk->last = 0; + + /* + * remove tblock from logsynclist + * (allocation map pages inherited lsn of tblk and + * has been inserted in logsync list at txUpdateMap()) + */ + if (tblk->lsn) { + LOGSYNC_LOCK(log, flags); + log->count--; + list_del_init(&tblk->synclist); + LOGSYNC_UNLOCK(log, flags); + } +} + +/* + * txMaplock() + * + * function: allocate a transaction lock for freed page/entry; + * for freed page, maplock is used as xtlock/dtlock type; + */ +struct tlock *txMaplock(tid_t tid, struct inode *ip, int type) +{ + struct jfs_inode_info *jfs_ip = JFS_IP(ip); + lid_t lid; + struct tblock *tblk; + struct tlock *tlck; + struct maplock *maplock; + + TXN_LOCK(); + + /* + * allocate a tlock + */ + lid = txLockAlloc(); + tlck = lid_to_tlock(lid); + + /* + * initialize tlock + */ + tlck->tid = tid; + + /* bind the tlock and the object */ + tlck->flag = tlckINODELOCK; + if (S_ISDIR(ip->i_mode)) + tlck->flag |= tlckDIRECTORY; + tlck->ip = ip; + tlck->mp = NULL; + + tlck->type = type; + + /* + * enqueue transaction lock to transaction/inode + */ + /* insert the tlock at tail of transaction tlock list */ + if (tid) { + tblk = tid_to_tblock(tid); + if (tblk->next) + lid_to_tlock(tblk->last)->next = lid; + else + tblk->next = lid; + tlck->next = 0; + tblk->last = lid; + } + /* anonymous transaction: + * insert the tlock at head of inode anonymous tlock list + */ + else { + tlck->next = jfs_ip->atlhead; + jfs_ip->atlhead = lid; + if (tlck->next == 0) { + /* This inode's first anonymous transaction */ + jfs_ip->atltail = lid; + list_add_tail(&jfs_ip->anon_inode_list, + &TxAnchor.anon_list); + } + } + + TXN_UNLOCK(); + + /* initialize type dependent area for maplock */ + maplock = (struct maplock *) & tlck->lock; + maplock->next = 0; + maplock->maxcnt = 0; + maplock->index = 0; + + return tlck; +} + +/* + * txLinelock() + * + * function: allocate a transaction lock for log vector list + */ +struct linelock *txLinelock(struct linelock * tlock) +{ + lid_t lid; + struct tlock *tlck; + struct linelock *linelock; + + TXN_LOCK(); + + /* allocate a TxLock structure */ + lid = txLockAlloc(); + tlck = lid_to_tlock(lid); + + TXN_UNLOCK(); + + /* initialize linelock */ + linelock = (struct linelock *) tlck; + linelock->next = 0; + linelock->flag = tlckLINELOCK; + linelock->maxcnt = TLOCKLONG; + linelock->index = 0; + if (tlck->flag & tlckDIRECTORY) + linelock->flag |= tlckDIRECTORY; + + /* append linelock after tlock */ + linelock->next = tlock->next; + tlock->next = lid; + + return linelock; +} + +/* + * transaction commit management + * ----------------------------- + */ + +/* + * NAME: txCommit() + * + * FUNCTION: commit the changes to the objects specified in + * clist. For journalled segments only the + * changes of the caller are committed, ie by tid. + * for non-journalled segments the data are flushed to + * disk and then the change to the disk inode and indirect + * blocks committed (so blocks newly allocated to the + * segment will be made a part of the segment atomically). + * + * all of the segments specified in clist must be in + * one file system. no more than 6 segments are needed + * to handle all unix svcs. + * + * if the i_nlink field (i.e. disk inode link count) + * is zero, and the type of inode is a regular file or + * directory, or symbolic link , the inode is truncated + * to zero length. the truncation is committed but the + * VM resources are unaffected until it is closed (see + * iput and iclose). + * + * PARAMETER: + * + * RETURN: + * + * serialization: + * on entry the inode lock on each segment is assumed + * to be held. + * + * i/o error: + */ +int txCommit(tid_t tid, /* transaction identifier */ + int nip, /* number of inodes to commit */ + struct inode **iplist, /* list of inode to commit */ + int flag) +{ + int rc = 0; + struct commit cd; + struct jfs_log *log; + struct tblock *tblk; + struct lrd *lrd; + struct inode *ip; + struct jfs_inode_info *jfs_ip; + int k, n; + ino_t top; + struct super_block *sb; + + jfs_info("txCommit, tid = %d, flag = %d", tid, flag); + /* is read-only file system ? */ + if (isReadOnly(iplist[0])) { + rc = -EROFS; + goto TheEnd; + } + + sb = cd.sb = iplist[0]->i_sb; + cd.tid = tid; + + if (tid == 0) + tid = txBegin(sb, 0); + tblk = tid_to_tblock(tid); + + /* + * initialize commit structure + */ + log = JFS_SBI(sb)->log; + cd.log = log; + + /* initialize log record descriptor in commit */ + lrd = &cd.lrd; + lrd->logtid = cpu_to_le32(tblk->logtid); + lrd->backchain = 0; + + tblk->xflag |= flag; + + if ((flag & (COMMIT_FORCE | COMMIT_SYNC)) == 0) + tblk->xflag |= COMMIT_LAZY; + /* + * prepare non-journaled objects for commit + * + * flush data pages of non-journaled file + * to prevent the file getting non-initialized disk blocks + * in case of crash. + * (new blocks - ) + */ + cd.iplist = iplist; + cd.nip = nip; + + /* + * acquire transaction lock on (on-disk) inodes + * + * update on-disk inode from in-memory inode + * acquiring transaction locks for AFTER records + * on the on-disk inode of file object + * + * sort the inodes array by inode number in descending order + * to prevent deadlock when acquiring transaction lock + * of on-disk inodes on multiple on-disk inode pages by + * multiple concurrent transactions + */ + for (k = 0; k < cd.nip; k++) { + top = (cd.iplist[k])->i_ino; + for (n = k + 1; n < cd.nip; n++) { + ip = cd.iplist[n]; + if (ip->i_ino > top) { + top = ip->i_ino; + cd.iplist[n] = cd.iplist[k]; + cd.iplist[k] = ip; + } + } + + ip = cd.iplist[k]; + jfs_ip = JFS_IP(ip); + + /* + * BUGBUG - This code has temporarily been removed. The + * intent is to ensure that any file data is written before + * the metadata is committed to the journal. This prevents + * uninitialized data from appearing in a file after the + * journal has been replayed. (The uninitialized data + * could be sensitive data removed by another user.) + * + * The problem now is that we are holding the IWRITELOCK + * on the inode, and calling filemap_fdatawrite on an + * unmapped page will cause a deadlock in jfs_get_block. + * + * The long term solution is to pare down the use of + * IWRITELOCK. We are currently holding it too long. + * We could also be smarter about which data pages need + * to be written before the transaction is committed and + * when we don't need to worry about it at all. + * + * if ((!S_ISDIR(ip->i_mode)) + * && (tblk->flag & COMMIT_DELETE) == 0) + * filemap_write_and_wait(ip->i_mapping); + */ + + /* + * Mark inode as not dirty. It will still be on the dirty + * inode list, but we'll know not to commit it again unless + * it gets marked dirty again + */ + clear_cflag(COMMIT_Dirty, ip); + + /* inherit anonymous tlock(s) of inode */ + if (jfs_ip->atlhead) { + lid_to_tlock(jfs_ip->atltail)->next = tblk->next; + tblk->next = jfs_ip->atlhead; + if (!tblk->last) + tblk->last = jfs_ip->atltail; + jfs_ip->atlhead = jfs_ip->atltail = 0; + TXN_LOCK(); + list_del_init(&jfs_ip->anon_inode_list); + TXN_UNLOCK(); + } + + /* + * acquire transaction lock on on-disk inode page + * (become first tlock of the tblk's tlock list) + */ + if (((rc = diWrite(tid, ip)))) + goto out; + } + + /* + * write log records from transaction locks + * + * txUpdateMap() resets XAD_NEW in XAD. + */ + txLog(log, tblk, &cd); + + /* + * Ensure that inode isn't reused before + * lazy commit thread finishes processing + */ + if (tblk->xflag & COMMIT_DELETE) { + ihold(tblk->u.ip); + /* + * Avoid a rare deadlock + * + * If the inode is locked, we may be blocked in + * jfs_commit_inode. If so, we don't want the + * lazy_commit thread doing the last iput() on the inode + * since that may block on the locked inode. Instead, + * commit the transaction synchronously, so the last iput + * will be done by the calling thread (or later) + */ + /* + * I believe this code is no longer needed. Splitting I_LOCK + * into two bits, I_NEW and I_SYNC should prevent this + * deadlock as well. But since I don't have a JFS testload + * to verify this, only a trivial s/I_LOCK/I_SYNC/ was done. + * Joern + */ + if (inode_state_read_once(tblk->u.ip) & I_SYNC) + tblk->xflag &= ~COMMIT_LAZY; + } + + ASSERT((!(tblk->xflag & COMMIT_DELETE)) || + ((tblk->u.ip->i_nlink == 0) && + !test_cflag(COMMIT_Nolink, tblk->u.ip))); + + /* + * write COMMIT log record + */ + lrd->type = cpu_to_le16(LOG_COMMIT); + lrd->length = 0; + lmLog(log, tblk, lrd, NULL); + + lmGroupCommit(log, tblk); + + /* + * - transaction is now committed - + */ + + /* + * force pages in careful update + * (imap addressing structure update) + */ + if (flag & COMMIT_FORCE) + txForce(tblk); + + /* + * update allocation map. + * + * update inode allocation map and inode: + * free pager lock on memory object of inode if any. + * update block allocation map. + * + * txUpdateMap() resets XAD_NEW in XAD. + */ + if (tblk->xflag & COMMIT_FORCE) + txUpdateMap(tblk); + + /* + * free transaction locks and pageout/free pages + */ + txRelease(tblk); + + if ((tblk->flag & tblkGC_LAZY) == 0) + txUnlock(tblk); + + + /* + * reset in-memory object state + */ + for (k = 0; k < cd.nip; k++) { + ip = cd.iplist[k]; + jfs_ip = JFS_IP(ip); + + /* + * reset in-memory inode state + */ + jfs_ip->bxflag = 0; + jfs_ip->blid = 0; + } + + out: + if (rc != 0) + txAbort(tid, 1); + + TheEnd: + jfs_info("txCommit: tid = %d, returning %d", tid, rc); + return rc; +} + +/* + * NAME: txLog() + * + * FUNCTION: Writes AFTER log records for all lines modified + * by tid for segments specified by inodes in comdata. + * Code assumes only WRITELOCKS are recorded in lockwords. + * + * PARAMETERS: + * + * RETURN : + */ +static void txLog(struct jfs_log *log, struct tblock *tblk, struct commit *cd) +{ + struct inode *ip; + lid_t lid; + struct tlock *tlck; + struct lrd *lrd = &cd->lrd; + + /* + * write log record(s) for each tlock of transaction, + */ + for (lid = tblk->next; lid; lid = tlck->next) { + tlck = lid_to_tlock(lid); + + tlck->flag |= tlckLOG; + + /* initialize lrd common */ + ip = tlck->ip; + lrd->aggregate = cpu_to_le32(JFS_SBI(ip->i_sb)->aggregate); + lrd->log.redopage.fileset = cpu_to_le32(JFS_IP(ip)->fileset); + lrd->log.redopage.inode = cpu_to_le32(ip->i_ino); + + /* write log record of page from the tlock */ + switch (tlck->type & tlckTYPE) { + case tlckXTREE: + xtLog(log, tblk, lrd, tlck); + break; + + case tlckDTREE: + dtLog(log, tblk, lrd, tlck); + break; + + case tlckINODE: + diLog(log, tblk, lrd, tlck, cd); + break; + + case tlckMAP: + mapLog(log, tblk, lrd, tlck); + break; + + case tlckDATA: + dataLog(log, tblk, lrd, tlck); + break; + + default: + jfs_err("UFO tlock:0x%p", tlck); + } + } + + return; +} + +/* + * diLog() + * + * function: log inode tlock and format maplock to update bmap; + */ +static void diLog(struct jfs_log *log, struct tblock *tblk, struct lrd *lrd, + struct tlock *tlck, struct commit *cd) +{ + struct metapage *mp; + pxd_t *pxd; + struct pxd_lock *pxdlock; + + mp = tlck->mp; + + /* initialize as REDOPAGE record format */ + lrd->log.redopage.type = cpu_to_le16(LOG_INODE); + lrd->log.redopage.l2linesize = cpu_to_le16(L2INODESLOTSIZE); + + pxd = &lrd->log.redopage.pxd; + + /* + * inode after image + */ + if (tlck->type & tlckENTRY) { + /* log after-image for logredo(): */ + lrd->type = cpu_to_le16(LOG_REDOPAGE); + PXDaddress(pxd, mp->index); + PXDlength(pxd, + mp->logical_size >> tblk->sb->s_blocksize_bits); + lrd->backchain = cpu_to_le32(lmLog(log, tblk, lrd, tlck)); + + /* mark page as homeward bound */ + tlck->flag |= tlckWRITEPAGE; + } else if (tlck->type & tlckFREE) { + /* + * free inode extent + * + * (pages of the freed inode extent have been invalidated and + * a maplock for free of the extent has been formatted at + * txLock() time); + * + * the tlock had been acquired on the inode allocation map page + * (iag) that specifies the freed extent, even though the map + * page is not itself logged, to prevent pageout of the map + * page before the log; + */ + + /* log LOG_NOREDOINOEXT of the freed inode extent for + * logredo() to start NoRedoPage filters, and to update + * imap and bmap for free of the extent; + */ + lrd->type = cpu_to_le16(LOG_NOREDOINOEXT); + /* + * For the LOG_NOREDOINOEXT record, we need + * to pass the IAG number and inode extent + * index (within that IAG) from which the + * extent is being released. These have been + * passed to us in the iplist[1] and iplist[2]. + */ + lrd->log.noredoinoext.iagnum = + cpu_to_le32((u32) (size_t) cd->iplist[1]); + lrd->log.noredoinoext.inoext_idx = + cpu_to_le32((u32) (size_t) cd->iplist[2]); + + pxdlock = (struct pxd_lock *) & tlck->lock; + *pxd = pxdlock->pxd; + lrd->backchain = cpu_to_le32(lmLog(log, tblk, lrd, NULL)); + + /* update bmap */ + tlck->flag |= tlckUPDATEMAP; + + /* mark page as homeward bound */ + tlck->flag |= tlckWRITEPAGE; + } else + jfs_err("diLog: UFO type tlck:0x%p", tlck); + return; +} + +/* + * dataLog() + * + * function: log data tlock + */ +static void dataLog(struct jfs_log *log, struct tblock *tblk, struct lrd *lrd, + struct tlock *tlck) +{ + struct metapage *mp; + pxd_t *pxd; + + mp = tlck->mp; + + /* initialize as REDOPAGE record format */ + lrd->log.redopage.type = cpu_to_le16(LOG_DATA); + lrd->log.redopage.l2linesize = cpu_to_le16(L2DATASLOTSIZE); + + pxd = &lrd->log.redopage.pxd; + + /* log after-image for logredo(): */ + lrd->type = cpu_to_le16(LOG_REDOPAGE); + + if (jfs_dirtable_inline(tlck->ip)) { + /* + * The table has been truncated, we've must have deleted + * the last entry, so don't bother logging this + */ + mp->lid = 0; + grab_metapage(mp); + metapage_homeok(mp); + discard_metapage(mp); + tlck->mp = NULL; + return; + } + + PXDaddress(pxd, mp->index); + PXDlength(pxd, mp->logical_size >> tblk->sb->s_blocksize_bits); + + lrd->backchain = cpu_to_le32(lmLog(log, tblk, lrd, tlck)); + + /* mark page as homeward bound */ + tlck->flag |= tlckWRITEPAGE; + + return; +} + +/* + * dtLog() + * + * function: log dtree tlock and format maplock to update bmap; + */ +static void dtLog(struct jfs_log * log, struct tblock * tblk, struct lrd * lrd, + struct tlock * tlck) +{ + struct metapage *mp; + struct pxd_lock *pxdlock; + pxd_t *pxd; + + mp = tlck->mp; + + /* initialize as REDOPAGE/NOREDOPAGE record format */ + lrd->log.redopage.type = cpu_to_le16(LOG_DTREE); + lrd->log.redopage.l2linesize = cpu_to_le16(L2DTSLOTSIZE); + + pxd = &lrd->log.redopage.pxd; + + if (tlck->type & tlckBTROOT) + lrd->log.redopage.type |= cpu_to_le16(LOG_BTROOT); + + /* + * page extension via relocation: entry insertion; + * page extension in-place: entry insertion; + * new right page from page split, reinitialized in-line + * root from root page split: entry insertion; + */ + if (tlck->type & (tlckNEW | tlckEXTEND)) { + /* log after-image of the new page for logredo(): + * mark log (LOG_NEW) for logredo() to initialize + * freelist and update bmap for alloc of the new page; + */ + lrd->type = cpu_to_le16(LOG_REDOPAGE); + if (tlck->type & tlckEXTEND) + lrd->log.redopage.type |= cpu_to_le16(LOG_EXTEND); + else + lrd->log.redopage.type |= cpu_to_le16(LOG_NEW); + PXDaddress(pxd, mp->index); + PXDlength(pxd, + mp->logical_size >> tblk->sb->s_blocksize_bits); + lrd->backchain = cpu_to_le32(lmLog(log, tblk, lrd, tlck)); + + /* format a maplock for txUpdateMap() to update bPMAP for + * alloc of the new page; + */ + if (tlck->type & tlckBTROOT) + return; + tlck->flag |= tlckUPDATEMAP; + pxdlock = (struct pxd_lock *) & tlck->lock; + pxdlock->flag = mlckALLOCPXD; + pxdlock->pxd = *pxd; + + pxdlock->index = 1; + + /* mark page as homeward bound */ + tlck->flag |= tlckWRITEPAGE; + return; + } + + /* + * entry insertion/deletion, + * sibling page link update (old right page before split); + */ + if (tlck->type & (tlckENTRY | tlckRELINK)) { + /* log after-image for logredo(): */ + lrd->type = cpu_to_le16(LOG_REDOPAGE); + PXDaddress(pxd, mp->index); + PXDlength(pxd, + mp->logical_size >> tblk->sb->s_blocksize_bits); + lrd->backchain = cpu_to_le32(lmLog(log, tblk, lrd, tlck)); + + /* mark page as homeward bound */ + tlck->flag |= tlckWRITEPAGE; + return; + } + + /* + * page deletion: page has been invalidated + * page relocation: source extent + * + * a maplock for free of the page has been formatted + * at txLock() time); + */ + if (tlck->type & (tlckFREE | tlckRELOCATE)) { + /* log LOG_NOREDOPAGE of the deleted page for logredo() + * to start NoRedoPage filter and to update bmap for free + * of the deletd page + */ + lrd->type = cpu_to_le16(LOG_NOREDOPAGE); + pxdlock = (struct pxd_lock *) & tlck->lock; + *pxd = pxdlock->pxd; + lrd->backchain = cpu_to_le32(lmLog(log, tblk, lrd, NULL)); + + /* a maplock for txUpdateMap() for free of the page + * has been formatted at txLock() time; + */ + tlck->flag |= tlckUPDATEMAP; + } + return; +} + +/* + * xtLog() + * + * function: log xtree tlock and format maplock to update bmap; + */ +static void xtLog(struct jfs_log * log, struct tblock * tblk, struct lrd * lrd, + struct tlock * tlck) +{ + struct inode *ip; + struct metapage *mp; + xtpage_t *p; + struct xtlock *xtlck; + struct maplock *maplock; + struct xdlistlock *xadlock; + struct pxd_lock *pxdlock; + pxd_t *page_pxd; + int next, lwm, hwm; + + ip = tlck->ip; + mp = tlck->mp; + + /* initialize as REDOPAGE/NOREDOPAGE record format */ + lrd->log.redopage.type = cpu_to_le16(LOG_XTREE); + lrd->log.redopage.l2linesize = cpu_to_le16(L2XTSLOTSIZE); + + page_pxd = &lrd->log.redopage.pxd; + + if (tlck->type & tlckBTROOT) { + lrd->log.redopage.type |= cpu_to_le16(LOG_BTROOT); + p = (xtpage_t *) &JFS_IP(ip)->i_xtroot; + if (S_ISDIR(ip->i_mode)) + lrd->log.redopage.type |= + cpu_to_le16(LOG_DIR_XTREE); + } else + p = (xtpage_t *) mp->data; + next = le16_to_cpu(p->header.nextindex); + + xtlck = (struct xtlock *) & tlck->lock; + + maplock = (struct maplock *) & tlck->lock; + xadlock = (struct xdlistlock *) maplock; + + /* + * entry insertion/extension; + * sibling page link update (old right page before split); + */ + if (tlck->type & (tlckNEW | tlckGROW | tlckRELINK)) { + /* log after-image for logredo(): + * logredo() will update bmap for alloc of new/extended + * extents (XAD_NEW|XAD_EXTEND) of XAD[lwm:next) from + * after-image of XADlist; + * logredo() resets (XAD_NEW|XAD_EXTEND) flag when + * applying the after-image to the meta-data page. + */ + lrd->type = cpu_to_le16(LOG_REDOPAGE); + PXDaddress(page_pxd, mp->index); + PXDlength(page_pxd, + mp->logical_size >> tblk->sb->s_blocksize_bits); + lrd->backchain = cpu_to_le32(lmLog(log, tblk, lrd, tlck)); + + /* format a maplock for txUpdateMap() to update bPMAP + * for alloc of new/extended extents of XAD[lwm:next) + * from the page itself; + * txUpdateMap() resets (XAD_NEW|XAD_EXTEND) flag. + */ + lwm = xtlck->lwm.offset; + if (lwm == 0) + lwm = XTPAGEMAXSLOT; + + if (lwm == next) + goto out; + if (lwm > next) { + jfs_err("xtLog: lwm > next"); + goto out; + } + tlck->flag |= tlckUPDATEMAP; + xadlock->flag = mlckALLOCXADLIST; + xadlock->count = next - lwm; + if ((xadlock->count <= 4) && (tblk->xflag & COMMIT_LAZY)) { + int i; + pxd_t *pxd; + /* + * Lazy commit may allow xtree to be modified before + * txUpdateMap runs. Copy xad into linelock to + * preserve correct data. + * + * We can fit twice as may pxd's as xads in the lock + */ + xadlock->flag = mlckALLOCPXDLIST; + pxd = xadlock->xdlist = &xtlck->pxdlock; + for (i = 0; i < xadlock->count; i++) { + PXDaddress(pxd, addressXAD(&p->xad[lwm + i])); + PXDlength(pxd, lengthXAD(&p->xad[lwm + i])); + p->xad[lwm + i].flag &= + ~(XAD_NEW | XAD_EXTENDED); + pxd++; + } + } else { + /* + * xdlist will point to into inode's xtree, ensure + * that transaction is not committed lazily. + */ + xadlock->flag = mlckALLOCXADLIST; + xadlock->xdlist = &p->xad[lwm]; + tblk->xflag &= ~COMMIT_LAZY; + } + jfs_info("xtLog: alloc ip:0x%p mp:0x%p tlck:0x%p lwm:%d count:%d", + tlck->ip, mp, tlck, lwm, xadlock->count); + + maplock->index = 1; + + out: + /* mark page as homeward bound */ + tlck->flag |= tlckWRITEPAGE; + + return; + } + + /* + * page deletion: file deletion/truncation (ref. xtTruncate()) + * + * (page will be invalidated after log is written and bmap + * is updated from the page); + */ + if (tlck->type & tlckFREE) { + /* LOG_NOREDOPAGE log for NoRedoPage filter: + * if page free from file delete, NoRedoFile filter from + * inode image of zero link count will subsume NoRedoPage + * filters for each page; + * if page free from file truncattion, write NoRedoPage + * filter; + * + * upadte of block allocation map for the page itself: + * if page free from deletion and truncation, LOG_UPDATEMAP + * log for the page itself is generated from processing + * its parent page xad entries; + */ + /* if page free from file truncation, log LOG_NOREDOPAGE + * of the deleted page for logredo() to start NoRedoPage + * filter for the page; + */ + if (tblk->xflag & COMMIT_TRUNCATE) { + /* write NOREDOPAGE for the page */ + lrd->type = cpu_to_le16(LOG_NOREDOPAGE); + PXDaddress(page_pxd, mp->index); + PXDlength(page_pxd, + mp->logical_size >> tblk->sb-> + s_blocksize_bits); + lrd->backchain = + cpu_to_le32(lmLog(log, tblk, lrd, NULL)); + + if (tlck->type & tlckBTROOT) { + /* Empty xtree must be logged */ + lrd->type = cpu_to_le16(LOG_REDOPAGE); + lrd->backchain = + cpu_to_le32(lmLog(log, tblk, lrd, tlck)); + } + } + + /* init LOG_UPDATEMAP of the freed extents + * XAD[XTENTRYSTART:hwm) from the deleted page itself + * for logredo() to update bmap; + */ + lrd->type = cpu_to_le16(LOG_UPDATEMAP); + lrd->log.updatemap.type = cpu_to_le16(LOG_FREEXADLIST); + xtlck = (struct xtlock *) & tlck->lock; + hwm = xtlck->hwm.offset; + lrd->log.updatemap.nxd = + cpu_to_le16(hwm - XTENTRYSTART + 1); + /* reformat linelock for lmLog() */ + xtlck->header.offset = XTENTRYSTART; + xtlck->header.length = hwm - XTENTRYSTART + 1; + xtlck->index = 1; + lrd->backchain = cpu_to_le32(lmLog(log, tblk, lrd, tlck)); + + /* format a maplock for txUpdateMap() to update bmap + * to free extents of XAD[XTENTRYSTART:hwm) from the + * deleted page itself; + */ + tlck->flag |= tlckUPDATEMAP; + xadlock->count = hwm - XTENTRYSTART + 1; + if ((xadlock->count <= 4) && (tblk->xflag & COMMIT_LAZY)) { + int i; + pxd_t *pxd; + /* + * Lazy commit may allow xtree to be modified before + * txUpdateMap runs. Copy xad into linelock to + * preserve correct data. + * + * We can fit twice as may pxd's as xads in the lock + */ + xadlock->flag = mlckFREEPXDLIST; + pxd = xadlock->xdlist = &xtlck->pxdlock; + for (i = 0; i < xadlock->count; i++) { + PXDaddress(pxd, + addressXAD(&p->xad[XTENTRYSTART + i])); + PXDlength(pxd, + lengthXAD(&p->xad[XTENTRYSTART + i])); + pxd++; + } + } else { + /* + * xdlist will point to into inode's xtree, ensure + * that transaction is not committed lazily. + */ + xadlock->flag = mlckFREEXADLIST; + xadlock->xdlist = &p->xad[XTENTRYSTART]; + tblk->xflag &= ~COMMIT_LAZY; + } + jfs_info("xtLog: free ip:0x%p mp:0x%p count:%d lwm:2", + tlck->ip, mp, xadlock->count); + + maplock->index = 1; + + /* mark page as invalid */ + if (((tblk->xflag & COMMIT_PWMAP) || S_ISDIR(ip->i_mode)) + && !(tlck->type & tlckBTROOT)) + tlck->flag |= tlckFREEPAGE; + /* + else (tblk->xflag & COMMIT_PMAP) + ? release the page; + */ + return; + } + + /* + * page/entry truncation: file truncation (ref. xtTruncate()) + * + * |----------+------+------+---------------| + * | | | + * | | hwm - hwm before truncation + * | next - truncation point + * lwm - lwm before truncation + * header ? + */ + if (tlck->type & tlckTRUNCATE) { + pxd_t pxd; /* truncated extent of xad */ + int twm; + + /* + * For truncation the entire linelock may be used, so it would + * be difficult to store xad list in linelock itself. + * Therefore, we'll just force transaction to be committed + * synchronously, so that xtree pages won't be changed before + * txUpdateMap runs. + */ + tblk->xflag &= ~COMMIT_LAZY; + lwm = xtlck->lwm.offset; + if (lwm == 0) + lwm = XTPAGEMAXSLOT; + hwm = xtlck->hwm.offset; + twm = xtlck->twm.offset; + + /* + * write log records + */ + /* log after-image for logredo(): + * + * logredo() will update bmap for alloc of new/extended + * extents (XAD_NEW|XAD_EXTEND) of XAD[lwm:next) from + * after-image of XADlist; + * logredo() resets (XAD_NEW|XAD_EXTEND) flag when + * applying the after-image to the meta-data page. + */ + lrd->type = cpu_to_le16(LOG_REDOPAGE); + PXDaddress(page_pxd, mp->index); + PXDlength(page_pxd, + mp->logical_size >> tblk->sb->s_blocksize_bits); + lrd->backchain = cpu_to_le32(lmLog(log, tblk, lrd, tlck)); + + /* + * truncate entry XAD[twm == next - 1]: + */ + if (twm == next - 1) { + /* init LOG_UPDATEMAP for logredo() to update bmap for + * free of truncated delta extent of the truncated + * entry XAD[next - 1]: + * (xtlck->pxdlock = truncated delta extent); + */ + pxdlock = (struct pxd_lock *) & xtlck->pxdlock; + /* assert(pxdlock->type & tlckTRUNCATE); */ + lrd->type = cpu_to_le16(LOG_UPDATEMAP); + lrd->log.updatemap.type = cpu_to_le16(LOG_FREEPXD); + lrd->log.updatemap.nxd = cpu_to_le16(1); + lrd->log.updatemap.pxd = pxdlock->pxd; + pxd = pxdlock->pxd; /* save to format maplock */ + lrd->backchain = + cpu_to_le32(lmLog(log, tblk, lrd, NULL)); + } + + /* + * free entries XAD[next:hwm]: + */ + if (hwm >= next) { + /* init LOG_UPDATEMAP of the freed extents + * XAD[next:hwm] from the deleted page itself + * for logredo() to update bmap; + */ + lrd->type = cpu_to_le16(LOG_UPDATEMAP); + lrd->log.updatemap.type = + cpu_to_le16(LOG_FREEXADLIST); + xtlck = (struct xtlock *) & tlck->lock; + hwm = xtlck->hwm.offset; + lrd->log.updatemap.nxd = + cpu_to_le16(hwm - next + 1); + /* reformat linelock for lmLog() */ + xtlck->header.offset = next; + xtlck->header.length = hwm - next + 1; + xtlck->index = 1; + lrd->backchain = + cpu_to_le32(lmLog(log, tblk, lrd, tlck)); + } + + /* + * format maplock(s) for txUpdateMap() to update bmap + */ + maplock->index = 0; + + /* + * allocate entries XAD[lwm:next): + */ + if (lwm < next) { + /* format a maplock for txUpdateMap() to update bPMAP + * for alloc of new/extended extents of XAD[lwm:next) + * from the page itself; + * txUpdateMap() resets (XAD_NEW|XAD_EXTEND) flag. + */ + tlck->flag |= tlckUPDATEMAP; + xadlock->flag = mlckALLOCXADLIST; + xadlock->count = next - lwm; + xadlock->xdlist = &p->xad[lwm]; + + jfs_info("xtLog: alloc ip:0x%p mp:0x%p count:%d lwm:%d next:%d", + tlck->ip, mp, xadlock->count, lwm, next); + maplock->index++; + xadlock++; + } + + /* + * truncate entry XAD[twm == next - 1]: + */ + if (twm == next - 1) { + /* format a maplock for txUpdateMap() to update bmap + * to free truncated delta extent of the truncated + * entry XAD[next - 1]; + * (xtlck->pxdlock = truncated delta extent); + */ + tlck->flag |= tlckUPDATEMAP; + pxdlock = (struct pxd_lock *) xadlock; + pxdlock->flag = mlckFREEPXD; + pxdlock->count = 1; + pxdlock->pxd = pxd; + + jfs_info("xtLog: truncate ip:0x%p mp:0x%p count:%d hwm:%d", + ip, mp, pxdlock->count, hwm); + maplock->index++; + xadlock++; + } + + /* + * free entries XAD[next:hwm]: + */ + if (hwm >= next) { + /* format a maplock for txUpdateMap() to update bmap + * to free extents of XAD[next:hwm] from thedeleted + * page itself; + */ + tlck->flag |= tlckUPDATEMAP; + xadlock->flag = mlckFREEXADLIST; + xadlock->count = hwm - next + 1; + xadlock->xdlist = &p->xad[next]; + + jfs_info("xtLog: free ip:0x%p mp:0x%p count:%d next:%d hwm:%d", + tlck->ip, mp, xadlock->count, next, hwm); + maplock->index++; + } + + /* mark page as homeward bound */ + tlck->flag |= tlckWRITEPAGE; + } + return; +} + +/* + * mapLog() + * + * function: log from maplock of freed data extents; + */ +static void mapLog(struct jfs_log * log, struct tblock * tblk, struct lrd * lrd, + struct tlock * tlck) +{ + struct pxd_lock *pxdlock; + int i, nlock; + pxd_t *pxd; + + /* + * page relocation: free the source page extent + * + * a maplock for txUpdateMap() for free of the page + * has been formatted at txLock() time saving the src + * relocated page address; + */ + if (tlck->type & tlckRELOCATE) { + /* log LOG_NOREDOPAGE of the old relocated page + * for logredo() to start NoRedoPage filter; + */ + lrd->type = cpu_to_le16(LOG_NOREDOPAGE); + pxdlock = (struct pxd_lock *) & tlck->lock; + pxd = &lrd->log.redopage.pxd; + *pxd = pxdlock->pxd; + lrd->backchain = cpu_to_le32(lmLog(log, tblk, lrd, NULL)); + + /* (N.B. currently, logredo() does NOT update bmap + * for free of the page itself for (LOG_XTREE|LOG_NOREDOPAGE); + * if page free from relocation, LOG_UPDATEMAP log is + * specifically generated now for logredo() + * to update bmap for free of src relocated page; + * (new flag LOG_RELOCATE may be introduced which will + * inform logredo() to start NORedoPage filter and also + * update block allocation map at the same time, thus + * avoiding an extra log write); + */ + lrd->type = cpu_to_le16(LOG_UPDATEMAP); + lrd->log.updatemap.type = cpu_to_le16(LOG_FREEPXD); + lrd->log.updatemap.nxd = cpu_to_le16(1); + lrd->log.updatemap.pxd = pxdlock->pxd; + lrd->backchain = cpu_to_le32(lmLog(log, tblk, lrd, NULL)); + + /* a maplock for txUpdateMap() for free of the page + * has been formatted at txLock() time; + */ + tlck->flag |= tlckUPDATEMAP; + return; + } + /* + + * Otherwise it's not a relocate request + * + */ + else { + /* log LOG_UPDATEMAP for logredo() to update bmap for + * free of truncated/relocated delta extent of the data; + * e.g.: external EA extent, relocated/truncated extent + * from xtTailgate(); + */ + lrd->type = cpu_to_le16(LOG_UPDATEMAP); + pxdlock = (struct pxd_lock *) & tlck->lock; + nlock = pxdlock->index; + for (i = 0; i < nlock; i++, pxdlock++) { + if (pxdlock->flag & mlckALLOCPXD) + lrd->log.updatemap.type = + cpu_to_le16(LOG_ALLOCPXD); + else + lrd->log.updatemap.type = + cpu_to_le16(LOG_FREEPXD); + lrd->log.updatemap.nxd = cpu_to_le16(1); + lrd->log.updatemap.pxd = pxdlock->pxd; + lrd->backchain = + cpu_to_le32(lmLog(log, tblk, lrd, NULL)); + jfs_info("mapLog: xaddr:0x%lx xlen:0x%x", + (ulong) addressPXD(&pxdlock->pxd), + lengthPXD(&pxdlock->pxd)); + } + + /* update bmap */ + tlck->flag |= tlckUPDATEMAP; + } +} + +/* + * txEA() + * + * function: acquire maplock for EA/ACL extents or + * set COMMIT_INLINE flag; + */ +void txEA(tid_t tid, struct inode *ip, dxd_t * oldea, dxd_t * newea) +{ + struct tlock *tlck = NULL; + struct pxd_lock *maplock = NULL, *pxdlock = NULL; + + /* + * format maplock for alloc of new EA extent + */ + if (newea) { + /* Since the newea could be a completely zeroed entry we need to + * check for the two flags which indicate we should actually + * commit new EA data + */ + if (newea->flag & DXD_EXTENT) { + tlck = txMaplock(tid, ip, tlckMAP); + maplock = (struct pxd_lock *) & tlck->lock; + pxdlock = (struct pxd_lock *) maplock; + pxdlock->flag = mlckALLOCPXD; + PXDaddress(&pxdlock->pxd, addressDXD(newea)); + PXDlength(&pxdlock->pxd, lengthDXD(newea)); + pxdlock++; + maplock->index = 1; + } else if (newea->flag & DXD_INLINE) { + tlck = NULL; + + set_cflag(COMMIT_Inlineea, ip); + } + } + + /* + * format maplock for free of old EA extent + */ + if (!test_cflag(COMMIT_Nolink, ip) && oldea->flag & DXD_EXTENT) { + if (tlck == NULL) { + tlck = txMaplock(tid, ip, tlckMAP); + maplock = (struct pxd_lock *) & tlck->lock; + pxdlock = (struct pxd_lock *) maplock; + maplock->index = 0; + } + pxdlock->flag = mlckFREEPXD; + PXDaddress(&pxdlock->pxd, addressDXD(oldea)); + PXDlength(&pxdlock->pxd, lengthDXD(oldea)); + maplock->index++; + } +} + +/* + * txForce() + * + * function: synchronously write pages locked by transaction + * after txLog() but before txUpdateMap(); + */ +static void txForce(struct tblock * tblk) +{ + struct tlock *tlck; + lid_t lid, next; + struct metapage *mp; + + /* + * reverse the order of transaction tlocks in + * careful update order of address index pages + * (right to left, bottom up) + */ + tlck = lid_to_tlock(tblk->next); + lid = tlck->next; + tlck->next = 0; + while (lid) { + tlck = lid_to_tlock(lid); + next = tlck->next; + tlck->next = tblk->next; + tblk->next = lid; + lid = next; + } + + /* + * synchronously write the page, and + * hold the page for txUpdateMap(); + */ + for (lid = tblk->next; lid; lid = next) { + tlck = lid_to_tlock(lid); + next = tlck->next; + + if ((mp = tlck->mp) != NULL && + (tlck->type & tlckBTROOT) == 0) { + assert(mp->xflag & COMMIT_PAGE); + + if (tlck->flag & tlckWRITEPAGE) { + tlck->flag &= ~tlckWRITEPAGE; + + /* do not release page to freelist */ + force_metapage(mp); +#if 0 + /* + * The "right" thing to do here is to + * synchronously write the metadata. + * With the current implementation this + * is hard since write_metapage requires + * us to kunmap & remap the page. If we + * have tlocks pointing into the metadata + * pages, we don't want to do this. I think + * we can get by with synchronously writing + * the pages when they are released. + */ + assert(mp->nohomeok); + set_bit(META_dirty, &mp->flag); + set_bit(META_sync, &mp->flag); +#endif + } + } + } +} + +/* + * txUpdateMap() + * + * function: update persistent allocation map (and working map + * if appropriate); + * + * parameter: + */ +static void txUpdateMap(struct tblock * tblk) +{ + struct inode *ip; + struct inode *ipimap; + lid_t lid; + struct tlock *tlck; + struct maplock *maplock; + struct pxd_lock pxdlock; + int maptype; + int k, nlock; + struct metapage *mp = NULL; + + ipimap = JFS_SBI(tblk->sb)->ipimap; + + maptype = (tblk->xflag & COMMIT_PMAP) ? COMMIT_PMAP : COMMIT_PWMAP; + + + /* + * update block allocation map + * + * update allocation state in pmap (and wmap) and + * update lsn of the pmap page; + */ + /* + * scan each tlock/page of transaction for block allocation/free: + * + * for each tlock/page of transaction, update map. + * ? are there tlock for pmap and pwmap at the same time ? + */ + for (lid = tblk->next; lid; lid = tlck->next) { + tlck = lid_to_tlock(lid); + + if ((tlck->flag & tlckUPDATEMAP) == 0) + continue; + + if (tlck->flag & tlckFREEPAGE) { + /* + * Another thread may attempt to reuse freed space + * immediately, so we want to get rid of the metapage + * before anyone else has a chance to get it. + * Lock metapage, update maps, then invalidate + * the metapage. + */ + mp = tlck->mp; + ASSERT(mp->xflag & COMMIT_PAGE); + grab_metapage(mp); + } + + /* + * extent list: + * . in-line PXD list: + * . out-of-line XAD list: + */ + maplock = (struct maplock *) & tlck->lock; + nlock = maplock->index; + + for (k = 0; k < nlock; k++, maplock++) { + /* + * allocate blocks in persistent map: + * + * blocks have been allocated from wmap at alloc time; + */ + if (maplock->flag & mlckALLOC) { + txAllocPMap(ipimap, maplock, tblk); + } + /* + * free blocks in persistent and working map: + * blocks will be freed in pmap and then in wmap; + * + * ? tblock specifies the PMAP/PWMAP based upon + * transaction + * + * free blocks in persistent map: + * blocks will be freed from wmap at last reference + * release of the object for regular files; + * + * Alway free blocks from both persistent & working + * maps for directories + */ + else { /* (maplock->flag & mlckFREE) */ + + if (tlck->flag & tlckDIRECTORY) + txFreeMap(ipimap, maplock, + tblk, COMMIT_PWMAP); + else + txFreeMap(ipimap, maplock, + tblk, maptype); + } + } + if (tlck->flag & tlckFREEPAGE) { + if (!(tblk->flag & tblkGC_LAZY)) { + /* This is equivalent to txRelease */ + ASSERT(mp->lid == lid); + tlck->mp->lid = 0; + } + assert(mp->nohomeok == 1); + metapage_homeok(mp); + discard_metapage(mp); + tlck->mp = NULL; + } + } + /* + * update inode allocation map + * + * update allocation state in pmap and + * update lsn of the pmap page; + * update in-memory inode flag/state + * + * unlock mapper/write lock + */ + if (tblk->xflag & COMMIT_CREATE) { + diUpdatePMap(ipimap, tblk->ino, false, tblk); + /* update persistent block allocation map + * for the allocation of inode extent; + */ + pxdlock.flag = mlckALLOCPXD; + pxdlock.pxd = tblk->u.ixpxd; + pxdlock.index = 1; + txAllocPMap(ipimap, (struct maplock *) & pxdlock, tblk); + } else if (tblk->xflag & COMMIT_DELETE) { + ip = tblk->u.ip; + diUpdatePMap(ipimap, ip->i_ino, true, tblk); + iput(ip); + } +} + +/* + * txAllocPMap() + * + * function: allocate from persistent map; + * + * parameter: + * ipbmap - + * malock - + * xad list: + * pxd: + * + * maptype - + * allocate from persistent map; + * free from persistent map; + * (e.g., tmp file - free from working map at releae + * of last reference); + * free from persistent and working map; + * + * lsn - log sequence number; + */ +static void txAllocPMap(struct inode *ip, struct maplock * maplock, + struct tblock * tblk) +{ + struct inode *ipbmap = JFS_SBI(ip->i_sb)->ipbmap; + struct xdlistlock *xadlistlock; + xad_t *xad; + s64 xaddr; + int xlen; + struct pxd_lock *pxdlock; + struct xdlistlock *pxdlistlock; + pxd_t *pxd; + int n; + + /* + * allocate from persistent map; + */ + if (maplock->flag & mlckALLOCXADLIST) { + xadlistlock = (struct xdlistlock *) maplock; + xad = xadlistlock->xdlist; + for (n = 0; n < xadlistlock->count; n++, xad++) { + if (xad->flag & (XAD_NEW | XAD_EXTENDED)) { + xaddr = addressXAD(xad); + xlen = lengthXAD(xad); + dbUpdatePMap(ipbmap, false, xaddr, + (s64) xlen, tblk); + xad->flag &= ~(XAD_NEW | XAD_EXTENDED); + jfs_info("allocPMap: xaddr:0x%lx xlen:%d", + (ulong) xaddr, xlen); + } + } + } else if (maplock->flag & mlckALLOCPXD) { + pxdlock = (struct pxd_lock *) maplock; + xaddr = addressPXD(&pxdlock->pxd); + xlen = lengthPXD(&pxdlock->pxd); + dbUpdatePMap(ipbmap, false, xaddr, (s64) xlen, tblk); + jfs_info("allocPMap: xaddr:0x%lx xlen:%d", (ulong) xaddr, xlen); + } else { /* (maplock->flag & mlckALLOCPXDLIST) */ + + pxdlistlock = (struct xdlistlock *) maplock; + pxd = pxdlistlock->xdlist; + for (n = 0; n < pxdlistlock->count; n++, pxd++) { + xaddr = addressPXD(pxd); + xlen = lengthPXD(pxd); + dbUpdatePMap(ipbmap, false, xaddr, (s64) xlen, + tblk); + jfs_info("allocPMap: xaddr:0x%lx xlen:%d", + (ulong) xaddr, xlen); + } + } +} + +/* + * txFreeMap() + * + * function: free from persistent and/or working map; + * + * todo: optimization + */ +void txFreeMap(struct inode *ip, + struct maplock * maplock, struct tblock * tblk, int maptype) +{ + struct inode *ipbmap = JFS_SBI(ip->i_sb)->ipbmap; + struct xdlistlock *xadlistlock; + xad_t *xad; + s64 xaddr; + int xlen; + struct pxd_lock *pxdlock; + struct xdlistlock *pxdlistlock; + pxd_t *pxd; + int n; + + jfs_info("txFreeMap: tblk:0x%p maplock:0x%p maptype:0x%x", + tblk, maplock, maptype); + + /* + * free from persistent map; + */ + if (maptype == COMMIT_PMAP || maptype == COMMIT_PWMAP) { + if (maplock->flag & mlckFREEXADLIST) { + xadlistlock = (struct xdlistlock *) maplock; + xad = xadlistlock->xdlist; + for (n = 0; n < xadlistlock->count; n++, xad++) { + if (!(xad->flag & XAD_NEW)) { + xaddr = addressXAD(xad); + xlen = lengthXAD(xad); + dbUpdatePMap(ipbmap, true, xaddr, + (s64) xlen, tblk); + jfs_info("freePMap: xaddr:0x%lx xlen:%d", + (ulong) xaddr, xlen); + } + } + } else if (maplock->flag & mlckFREEPXD) { + pxdlock = (struct pxd_lock *) maplock; + xaddr = addressPXD(&pxdlock->pxd); + xlen = lengthPXD(&pxdlock->pxd); + dbUpdatePMap(ipbmap, true, xaddr, (s64) xlen, + tblk); + jfs_info("freePMap: xaddr:0x%lx xlen:%d", + (ulong) xaddr, xlen); + } else { /* (maplock->flag & mlckALLOCPXDLIST) */ + + pxdlistlock = (struct xdlistlock *) maplock; + pxd = pxdlistlock->xdlist; + for (n = 0; n < pxdlistlock->count; n++, pxd++) { + xaddr = addressPXD(pxd); + xlen = lengthPXD(pxd); + dbUpdatePMap(ipbmap, true, xaddr, + (s64) xlen, tblk); + jfs_info("freePMap: xaddr:0x%lx xlen:%d", + (ulong) xaddr, xlen); + } + } + } + + /* + * free from working map; + */ + if (maptype == COMMIT_PWMAP || maptype == COMMIT_WMAP) { + if (maplock->flag & mlckFREEXADLIST) { + xadlistlock = (struct xdlistlock *) maplock; + xad = xadlistlock->xdlist; + for (n = 0; n < xadlistlock->count; n++, xad++) { + xaddr = addressXAD(xad); + xlen = lengthXAD(xad); + dbFree(ip, xaddr, (s64) xlen); + xad->flag = 0; + jfs_info("freeWMap: xaddr:0x%lx xlen:%d", + (ulong) xaddr, xlen); + } + } else if (maplock->flag & mlckFREEPXD) { + pxdlock = (struct pxd_lock *) maplock; + xaddr = addressPXD(&pxdlock->pxd); + xlen = lengthPXD(&pxdlock->pxd); + dbFree(ip, xaddr, (s64) xlen); + jfs_info("freeWMap: xaddr:0x%lx xlen:%d", + (ulong) xaddr, xlen); + } else { /* (maplock->flag & mlckFREEPXDLIST) */ + + pxdlistlock = (struct xdlistlock *) maplock; + pxd = pxdlistlock->xdlist; + for (n = 0; n < pxdlistlock->count; n++, pxd++) { + xaddr = addressPXD(pxd); + xlen = lengthPXD(pxd); + dbFree(ip, xaddr, (s64) xlen); + jfs_info("freeWMap: xaddr:0x%lx xlen:%d", + (ulong) xaddr, xlen); + } + } + } +} + +/* + * txFreelock() + * + * function: remove tlock from inode anonymous locklist + */ +void txFreelock(struct inode *ip) +{ + struct jfs_inode_info *jfs_ip = JFS_IP(ip); + struct tlock *xtlck, *tlck; + lid_t xlid = 0, lid; + + if (!jfs_ip->atlhead) + return; + + TXN_LOCK(); + xtlck = (struct tlock *) &jfs_ip->atlhead; + + while ((lid = xtlck->next) != 0) { + tlck = lid_to_tlock(lid); + if (tlck->flag & tlckFREELOCK) { + xtlck->next = tlck->next; + txLockFree(lid); + } else { + xtlck = tlck; + xlid = lid; + } + } + + if (jfs_ip->atlhead) + jfs_ip->atltail = xlid; + else { + jfs_ip->atltail = 0; + /* + * If inode was on anon_list, remove it + */ + list_del_init(&jfs_ip->anon_inode_list); + } + TXN_UNLOCK(); +} + +/* + * txAbort() + * + * function: abort tx before commit; + * + * frees line-locks and segment locks for all + * segments in comdata structure. + * Optionally sets state of file-system to FM_DIRTY in super-block. + * log age of page-frames in memory for which caller has + * are reset to 0 (to avoid logwarap). + */ +void txAbort(tid_t tid, int dirty) +{ + lid_t lid, next; + struct metapage *mp; + struct tblock *tblk = tid_to_tblock(tid); + struct tlock *tlck; + + /* + * free tlocks of the transaction + */ + for (lid = tblk->next; lid; lid = next) { + tlck = lid_to_tlock(lid); + next = tlck->next; + mp = tlck->mp; + JFS_IP(tlck->ip)->xtlid = 0; + + if (mp) { + mp->lid = 0; + + /* + * reset lsn of page to avoid logwarap: + * + * (page may have been previously committed by another + * transaction(s) but has not been paged, i.e., + * it may be on logsync list even though it has not + * been logged for the current tx.) + */ + if (mp->xflag & COMMIT_PAGE && mp->lsn) + LogSyncRelease(mp); + } + /* insert tlock at head of freelist */ + TXN_LOCK(); + txLockFree(lid); + TXN_UNLOCK(); + } + + /* caller will free the transaction block */ + + tblk->next = tblk->last = 0; + + /* + * mark filesystem dirty + */ + if (dirty) + jfs_error(tblk->sb, "\n"); + + return; +} + +/* + * txLazyCommit(void) + * + * All transactions except those changing ipimap (COMMIT_FORCE) are + * processed by this routine. This insures that the inode and block + * allocation maps are updated in order. For synchronous transactions, + * let the user thread finish processing after txUpdateMap() is called. + */ +static void txLazyCommit(struct tblock * tblk) +{ + struct jfs_log *log; + + while (((tblk->flag & tblkGC_READY) == 0) && + ((tblk->flag & tblkGC_UNLOCKED) == 0)) { + /* We must have gotten ahead of the user thread + */ + jfs_info("jfs_lazycommit: tblk 0x%p not unlocked", tblk); + yield(); + } + + jfs_info("txLazyCommit: processing tblk 0x%p", tblk); + + txUpdateMap(tblk); + + log = (struct jfs_log *) JFS_SBI(tblk->sb)->log; + + spin_lock_irq(&log->gclock); // LOGGC_LOCK + + tblk->flag |= tblkGC_COMMITTED; + + if (tblk->flag & tblkGC_READY) + log->gcrtc--; + + wake_up_all(&tblk->gcwait); // LOGGC_WAKEUP + + /* + * Can't release log->gclock until we've tested tblk->flag + */ + if (tblk->flag & tblkGC_LAZY) { + spin_unlock_irq(&log->gclock); // LOGGC_UNLOCK + txUnlock(tblk); + tblk->flag &= ~tblkGC_LAZY; + txEnd(tblk - TxBlock); /* Convert back to tid */ + } else + spin_unlock_irq(&log->gclock); // LOGGC_UNLOCK + + jfs_info("txLazyCommit: done: tblk = 0x%p", tblk); +} + +/* + * jfs_lazycommit(void) + * + * To be run as a kernel daemon. If lbmIODone is called in an interrupt + * context, or where blocking is not wanted, this routine will process + * committed transactions from the unlock queue. + */ +int jfs_lazycommit(void *arg) +{ + int WorkDone; + struct tblock *tblk; + unsigned long flags; + struct jfs_sb_info *sbi; + + set_freezable(); + do { + LAZY_LOCK(flags); + jfs_commit_thread_waking = 0; /* OK to wake another thread */ + while (!list_empty(&TxAnchor.unlock_queue)) { + WorkDone = 0; + list_for_each_entry(tblk, &TxAnchor.unlock_queue, + cqueue) { + + sbi = JFS_SBI(tblk->sb); + /* + * For each volume, the transactions must be + * handled in order. If another commit thread + * is handling a tblk for this superblock, + * skip it + */ + if (sbi->commit_state & IN_LAZYCOMMIT) + continue; + + sbi->commit_state |= IN_LAZYCOMMIT; + WorkDone = 1; + + /* + * Remove transaction from queue + */ + list_del(&tblk->cqueue); + + LAZY_UNLOCK(flags); + txLazyCommit(tblk); + LAZY_LOCK(flags); + + sbi->commit_state &= ~IN_LAZYCOMMIT; + /* + * Don't continue in the for loop. (We can't + * anyway, it's unsafe!) We want to go back to + * the beginning of the list. + */ + break; + } + + /* If there was nothing to do, don't continue */ + if (!WorkDone) + break; + } + /* In case a wakeup came while all threads were active */ + jfs_commit_thread_waking = 0; + + if (freezing(current)) { + LAZY_UNLOCK(flags); + try_to_freeze(); + } else { + DECLARE_WAITQUEUE(wq, current); + + add_wait_queue(&jfs_commit_thread_wait, &wq); + set_current_state(TASK_INTERRUPTIBLE); + LAZY_UNLOCK(flags); + schedule(); + remove_wait_queue(&jfs_commit_thread_wait, &wq); + } + } while (!kthread_should_stop()); + + if (!list_empty(&TxAnchor.unlock_queue)) + jfs_err("jfs_lazycommit being killed w/pending transactions!"); + else + jfs_info("jfs_lazycommit being killed"); + return 0; +} + +void txLazyUnlock(struct tblock * tblk) +{ + unsigned long flags; + + LAZY_LOCK(flags); + + list_add_tail(&tblk->cqueue, &TxAnchor.unlock_queue); + /* + * Don't wake up a commit thread if there is already one servicing + * this superblock, or if the last one we woke up hasn't started yet. + */ + if (!(JFS_SBI(tblk->sb)->commit_state & IN_LAZYCOMMIT) && + !jfs_commit_thread_waking) { + jfs_commit_thread_waking = 1; + wake_up(&jfs_commit_thread_wait); + } + LAZY_UNLOCK(flags); +} + +static void LogSyncRelease(struct metapage * mp) +{ + struct jfs_log *log = mp->log; + + assert(mp->nohomeok); + assert(log); + metapage_homeok(mp); +} + +/* + * txQuiesce + * + * Block all new transactions and push anonymous transactions to + * completion + * + * This does almost the same thing as jfs_sync below. We don't + * worry about deadlocking when jfs_tlocks_low is set, since we would + * expect jfs_sync to get us out of that jam. + */ +void txQuiesce(struct super_block *sb) +{ + struct inode *ip; + struct jfs_inode_info *jfs_ip; + struct jfs_log *log = JFS_SBI(sb)->log; + tid_t tid; + + set_bit(log_QUIESCE, &log->flag); + + TXN_LOCK(); +restart: + while (!list_empty(&TxAnchor.anon_list)) { + jfs_ip = list_entry(TxAnchor.anon_list.next, + struct jfs_inode_info, + anon_inode_list); + ip = &jfs_ip->vfs_inode; + + /* + * inode will be removed from anonymous list + * when it is committed + */ + TXN_UNLOCK(); + tid = txBegin(ip->i_sb, COMMIT_INODE | COMMIT_FORCE); + mutex_lock(&jfs_ip->commit_mutex); + txCommit(tid, 1, &ip, 0); + txEnd(tid); + mutex_unlock(&jfs_ip->commit_mutex); + /* + * Just to be safe. I don't know how + * long we can run without blocking + */ + cond_resched(); + TXN_LOCK(); + } + + /* + * If jfs_sync is running in parallel, there could be some inodes + * on anon_list2. Let's check. + */ + if (!list_empty(&TxAnchor.anon_list2)) { + list_splice_init(&TxAnchor.anon_list2, &TxAnchor.anon_list); + goto restart; + } + TXN_UNLOCK(); + + /* + * We may need to kick off the group commit + */ + jfs_flush_journal(log, 0); +} + +/* + * txResume() + * + * Allows transactions to start again following txQuiesce + */ +void txResume(struct super_block *sb) +{ + struct jfs_log *log = JFS_SBI(sb)->log; + + clear_bit(log_QUIESCE, &log->flag); + TXN_WAKEUP(&log->syncwait); +} + +/* + * jfs_sync(void) + * + * To be run as a kernel daemon. This is awakened when tlocks run low. + * We write any inodes that have anonymous tlocks so they will become + * available. + */ +int jfs_sync(void *arg) +{ + struct inode *ip; + struct jfs_inode_info *jfs_ip; + tid_t tid; + + set_freezable(); + do { + /* + * write each inode on the anonymous inode list + */ + TXN_LOCK(); + while (jfs_tlocks_low && !list_empty(&TxAnchor.anon_list)) { + jfs_ip = list_entry(TxAnchor.anon_list.next, + struct jfs_inode_info, + anon_inode_list); + ip = &jfs_ip->vfs_inode; + + if (! igrab(ip)) { + /* + * Inode is being freed + */ + list_del_init(&jfs_ip->anon_inode_list); + } else if (mutex_trylock(&jfs_ip->commit_mutex)) { + /* + * inode will be removed from anonymous list + * when it is committed + */ + TXN_UNLOCK(); + tid = txBegin(ip->i_sb, COMMIT_INODE); + txCommit(tid, 1, &ip, 0); + txEnd(tid); + mutex_unlock(&jfs_ip->commit_mutex); + + iput(ip); + /* + * Just to be safe. I don't know how + * long we can run without blocking + */ + cond_resched(); + TXN_LOCK(); + } else { + /* We can't get the commit mutex. It may + * be held by a thread waiting for tlock's + * so let's not block here. Save it to + * put back on the anon_list. + */ + + /* Move from anon_list to anon_list2 */ + list_move(&jfs_ip->anon_inode_list, + &TxAnchor.anon_list2); + + TXN_UNLOCK(); + iput(ip); + TXN_LOCK(); + } + } + /* Add anon_list2 back to anon_list */ + list_splice_init(&TxAnchor.anon_list2, &TxAnchor.anon_list); + + if (freezing(current)) { + TXN_UNLOCK(); + try_to_freeze(); + } else { + set_current_state(TASK_INTERRUPTIBLE); + TXN_UNLOCK(); + schedule(); + } + } while (!kthread_should_stop()); + + jfs_info("jfs_sync being killed"); + return 0; +} + +#if defined(CONFIG_PROC_FS) && defined(CONFIG_JFS_DEBUG) +int jfs_txanchor_proc_show(struct seq_file *m, void *v) +{ + char *freewait; + char *freelockwait; + char *lowlockwait; + + freewait = + waitqueue_active(&TxAnchor.freewait) ? "active" : "empty"; + freelockwait = + waitqueue_active(&TxAnchor.freelockwait) ? "active" : "empty"; + lowlockwait = + waitqueue_active(&TxAnchor.lowlockwait) ? "active" : "empty"; + + seq_printf(m, + "JFS TxAnchor\n" + "============\n" + "freetid = %d\n" + "freewait = %s\n" + "freelock = %d\n" + "freelockwait = %s\n" + "lowlockwait = %s\n" + "tlocksInUse = %d\n" + "jfs_tlocks_low = %d\n" + "unlock_queue is %sempty\n", + TxAnchor.freetid, + freewait, + TxAnchor.freelock, + freelockwait, + lowlockwait, + TxAnchor.tlocksInUse, + jfs_tlocks_low, + list_empty(&TxAnchor.unlock_queue) ? "" : "not "); + return 0; +} +#endif + +#if defined(CONFIG_PROC_FS) && defined(CONFIG_JFS_STATISTICS) +int jfs_txstats_proc_show(struct seq_file *m, void *v) +{ + seq_printf(m, + "JFS TxStats\n" + "===========\n" + "calls to txBegin = %d\n" + "txBegin blocked by sync barrier = %d\n" + "txBegin blocked by tlocks low = %d\n" + "txBegin blocked by no free tid = %d\n" + "calls to txBeginAnon = %d\n" + "txBeginAnon blocked by sync barrier = %d\n" + "txBeginAnon blocked by tlocks low = %d\n" + "calls to txLockAlloc = %d\n" + "tLockAlloc blocked by no free lock = %d\n", + TxStat.txBegin, + TxStat.txBegin_barrier, + TxStat.txBegin_lockslow, + TxStat.txBegin_freetid, + TxStat.txBeginAnon, + TxStat.txBeginAnon_barrier, + TxStat.txBeginAnon_lockslow, + TxStat.txLockAlloc, + TxStat.txLockAlloc_freelock); + return 0; +} +#endif diff --git a/fs/jfs/jfs_txnmgr.h b/fs/jfs/jfs_txnmgr.h new file mode 100644 index 000000000..ba71eb5ce --- /dev/null +++ b/fs/jfs/jfs_txnmgr.h @@ -0,0 +1,298 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) International Business Machines Corp., 2000-2004 + */ +#ifndef _H_JFS_TXNMGR +#define _H_JFS_TXNMGR + +#include "jfs_logmgr.h" + +/* + * Hide implementation of TxBlock and TxLock + */ +#define tid_to_tblock(tid) (&TxBlock[tid]) + +#define lid_to_tlock(lid) (&TxLock[lid]) + +/* + * transaction block + */ +struct tblock { + /* + * tblock and jbuf_t common area: struct logsyncblk + * + * the following 5 fields are the same as struct logsyncblk + * which is common to tblock and jbuf to form logsynclist + */ + u16 xflag; /* tx commit type */ + u16 flag; /* tx commit state */ + lid_t dummy; /* Must keep structures common */ + s32 lsn; /* recovery lsn */ + struct list_head synclist; /* logsynclist link */ + + /* lock management */ + struct super_block *sb; /* super block */ + lid_t next; /* index of first tlock of tid */ + lid_t last; /* index of last tlock of tid */ + wait_queue_head_t waitor; /* tids waiting on this tid */ + + /* log management */ + u32 logtid; /* log transaction id */ + + /* commit management */ + struct list_head cqueue; /* commit queue list */ + s32 clsn; /* commit lsn */ + struct lbuf *bp; + s32 pn; /* commit record log page number */ + s32 eor; /* commit record eor */ + wait_queue_head_t gcwait; /* group commit event list: + * ready transactions wait on this + * event for group commit completion. + */ + union { + struct inode *ip; /* inode being deleted */ + pxd_t ixpxd; /* pxd of inode extent for created inode */ + } u; + u32 ino; /* inode number being created */ +}; + +extern struct tblock *TxBlock; /* transaction block table */ + +/* commit flags: tblk->xflag */ +#define COMMIT_SYNC 0x0001 /* synchronous commit */ +#define COMMIT_FORCE 0x0002 /* force pageout at end of commit */ +#define COMMIT_FLUSH 0x0004 /* init flush at end of commit */ +#define COMMIT_MAP 0x00f0 +#define COMMIT_PMAP 0x0010 /* update pmap */ +#define COMMIT_WMAP 0x0020 /* update wmap */ +#define COMMIT_PWMAP 0x0040 /* update pwmap */ +#define COMMIT_FREE 0x0f00 +#define COMMIT_DELETE 0x0100 /* inode delete */ +#define COMMIT_TRUNCATE 0x0200 /* file truncation */ +#define COMMIT_CREATE 0x0400 /* inode create */ +#define COMMIT_LAZY 0x0800 /* lazy commit */ +#define COMMIT_PAGE 0x1000 /* Identifies element as metapage */ +#define COMMIT_INODE 0x2000 /* Identifies element as inode */ + +/* group commit flags tblk->flag: see jfs_logmgr.h */ + +/* + * transaction lock + */ +struct tlock { + lid_t next; /* 2: index next lockword on tid locklist + * next lockword on freelist + */ + tid_t tid; /* 2: transaction id holding lock */ + + u16 flag; /* 2: lock control */ + u16 type; /* 2: log type */ + + struct metapage *mp; /* 4/8: object page buffer locked */ + struct inode *ip; /* 4/8: object */ + /* (16) */ + + s16 lock[24]; /* 48: overlay area */ +}; /* (64) */ + +extern struct tlock *TxLock; /* transaction lock table */ + +/* + * tlock flag + */ +/* txLock state */ +#define tlckPAGELOCK 0x8000 +#define tlckINODELOCK 0x4000 +#define tlckLINELOCK 0x2000 +#define tlckINLINELOCK 0x1000 +/* lmLog state */ +#define tlckLOG 0x0800 +/* updateMap state */ +#define tlckUPDATEMAP 0x0080 +#define tlckDIRECTORY 0x0040 +/* freeLock state */ +#define tlckFREELOCK 0x0008 +#define tlckWRITEPAGE 0x0004 +#define tlckFREEPAGE 0x0002 + +/* + * tlock type + */ +#define tlckTYPE 0xfe00 +#define tlckINODE 0x8000 +#define tlckXTREE 0x4000 +#define tlckDTREE 0x2000 +#define tlckMAP 0x1000 +#define tlckEA 0x0800 +#define tlckACL 0x0400 +#define tlckDATA 0x0200 +#define tlckBTROOT 0x0100 + +#define tlckOPERATION 0x00ff +#define tlckGROW 0x0001 /* file grow */ +#define tlckREMOVE 0x0002 /* file delete */ +#define tlckTRUNCATE 0x0004 /* file truncate */ +#define tlckRELOCATE 0x0008 /* file/directory relocate */ +#define tlckENTRY 0x0001 /* directory insert/delete */ +#define tlckEXTEND 0x0002 /* directory extend in-line */ +#define tlckSPLIT 0x0010 /* splited page */ +#define tlckNEW 0x0020 /* new page from split */ +#define tlckFREE 0x0040 /* free page */ +#define tlckRELINK 0x0080 /* update sibling pointer */ + +/* + * linelock for lmLog() + * + * note: linelock and its variations are overlaid + * at tlock.lock: watch for alignment; + */ +struct lv { + u8 offset; /* 1: */ + u8 length; /* 1: */ +}; /* (2) */ + +#define TLOCKSHORT 20 +#define TLOCKLONG 28 + +struct linelock { + lid_t next; /* 2: next linelock */ + + s8 maxcnt; /* 1: */ + s8 index; /* 1: */ + + u16 flag; /* 2: */ + u8 type; /* 1: */ + u8 l2linesize; /* 1: log2 of linesize */ + /* (8) */ + + struct lv lv[20]; /* 40: */ +}; /* (48) */ + +#define dt_lock linelock + +struct xtlock { + lid_t next; /* 2: */ + + s8 maxcnt; /* 1: */ + s8 index; /* 1: */ + + u16 flag; /* 2: */ + u8 type; /* 1: */ + u8 l2linesize; /* 1: log2 of linesize */ + /* (8) */ + + struct lv header; /* 2: */ + struct lv lwm; /* 2: low water mark */ + struct lv hwm; /* 2: high water mark */ + struct lv twm; /* 2: */ + /* (16) */ + + s32 pxdlock[8]; /* 32: */ +}; /* (48) */ + + +/* + * maplock for txUpdateMap() + * + * note: maplock and its variations are overlaid + * at tlock.lock/linelock: watch for alignment; + * N.B. next field may be set by linelock, and should not + * be modified by maplock; + * N.B. index of the first pxdlock specifies index of next + * free maplock (i.e., number of maplock) in the tlock; + */ +struct maplock { + lid_t next; /* 2: */ + + u8 maxcnt; /* 2: */ + u8 index; /* 2: next free maplock index */ + + u16 flag; /* 2: */ + u8 type; /* 1: */ + u8 count; /* 1: number of pxd/xad */ + /* (8) */ + + pxd_t pxd; /* 8: */ +}; /* (16): */ + +/* maplock flag */ +#define mlckALLOC 0x00f0 +#define mlckALLOCXADLIST 0x0080 +#define mlckALLOCPXDLIST 0x0040 +#define mlckALLOCXAD 0x0020 +#define mlckALLOCPXD 0x0010 +#define mlckFREE 0x000f +#define mlckFREEXADLIST 0x0008 +#define mlckFREEPXDLIST 0x0004 +#define mlckFREEXAD 0x0002 +#define mlckFREEPXD 0x0001 + +#define pxd_lock maplock + +struct xdlistlock { + lid_t next; /* 2: */ + + u8 maxcnt; /* 2: */ + u8 index; /* 2: */ + + u16 flag; /* 2: */ + u8 type; /* 1: */ + u8 count; /* 1: number of pxd/xad */ + /* (8) */ + + /* + * We need xdlist to be 64 bits (8 bytes), regardless of + * whether void * is 32 or 64 bits + */ + union { + void *_xdlist; /* pxd/xad list */ + s64 pad; /* 8: Force 64-bit xdlist size */ + } union64; +}; /* (16): */ + +#define xdlist union64._xdlist + +/* + * commit + * + * parameter to the commit manager routines + */ +struct commit { + tid_t tid; /* tid = index of tblock */ + int flag; /* flags */ + struct jfs_log *log; /* log */ + struct super_block *sb; /* superblock */ + + int nip; /* number of entries in iplist */ + struct inode **iplist; /* list of pointers to inodes */ + + /* log record descriptor on 64-bit boundary */ + struct lrd lrd; /* : log record descriptor */ +}; + +/* + * external declarations + */ +extern int jfs_tlocks_low; + +extern int txInit(void); +extern void txExit(void); +extern struct tlock *txLock(tid_t, struct inode *, struct metapage *, int); +extern struct tlock *txMaplock(tid_t, struct inode *, int); +extern int txCommit(tid_t, int, struct inode **, int); +extern tid_t txBegin(struct super_block *, int); +extern void txBeginAnon(struct super_block *); +extern void txEnd(tid_t); +extern void txAbort(tid_t, int); +extern struct linelock *txLinelock(struct linelock *); +extern void txFreeMap(struct inode *, struct maplock *, struct tblock *, int); +extern void txEA(tid_t, struct inode *, dxd_t *, dxd_t *); +extern void txFreelock(struct inode *); +extern int lmLog(struct jfs_log *, struct tblock *, struct lrd *, + struct tlock *); +extern void txQuiesce(struct super_block *); +extern void txResume(struct super_block *); +extern void txLazyUnlock(struct tblock *); +extern int jfs_lazycommit(void *); +extern int jfs_sync(void *); +#endif /* _H_JFS_TXNMGR */ diff --git a/fs/jfs/jfs_types.h b/fs/jfs/jfs_types.h new file mode 100644 index 000000000..3ff9f26bc --- /dev/null +++ b/fs/jfs/jfs_types.h @@ -0,0 +1,157 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) International Business Machines Corp., 2000-2004 + */ +#ifndef _H_JFS_TYPES +#define _H_JFS_TYPES + +/* + * jfs_types.h: + * + * basic type/utility definitions + * + * note: this header file must be the 1st include file + * of JFS include list in all JFS .c file. + */ + +#include +#include + +/* + * transaction and lock id's + * + * Don't change these without carefully considering the impact on the + * size and alignment of all of the linelock variants + */ +typedef u16 tid_t; +typedef u16 lid_t; + +/* + * Almost identical to Linux's timespec, but not quite + */ +struct timestruc_t { + __le32 tv_sec; + __le32 tv_nsec; +}; + +/* + * handy + */ + +#define LEFTMOSTONE 0x80000000 +#define HIGHORDER 0x80000000u /* high order bit on */ +#define ONES 0xffffffffu /* all bit on */ + +/* + * physical xd (pxd) + * + * The leftmost 24 bits of len_addr are the extent length. + * The rightmost 8 bits of len_addr are the most signficant bits of + * the extent address + */ +typedef struct { + __le32 len_addr; + __le32 addr2; +} pxd_t; + +/* xd_t field construction */ + +static inline void PXDlength(pxd_t *pxd, __u32 len) +{ + pxd->len_addr = (pxd->len_addr & cpu_to_le32(~0xffffff)) | + cpu_to_le32(len & 0xffffff); +} + +static inline void PXDaddress(pxd_t *pxd, __u64 addr) +{ + pxd->len_addr = (pxd->len_addr & cpu_to_le32(0xffffff)) | + cpu_to_le32((addr >> 32)<<24); + pxd->addr2 = cpu_to_le32(addr & 0xffffffff); +} + +/* xd_t field extraction */ +static inline __u32 lengthPXD(pxd_t *pxd) +{ + return le32_to_cpu((pxd)->len_addr) & 0xffffff; +} + +static inline __u64 addressPXD(pxd_t *pxd) +{ + __u64 n = le32_to_cpu(pxd->len_addr) & ~0xffffff; + return (n << 8) + le32_to_cpu(pxd->addr2); +} + +#define MAXTREEHEIGHT 8 +/* pxd list */ +struct pxdlist { + s16 maxnpxd; + s16 npxd; + pxd_t pxd[MAXTREEHEIGHT]; +}; + + +/* + * data extent descriptor (dxd) + */ +typedef struct { + __u8 flag; /* 1: flags */ + __u8 rsrvd[3]; + __le32 size; /* 4: size in byte */ + pxd_t loc; /* 8: address and length in unit of fsblksize */ +} dxd_t; /* - 16 - */ + +/* dxd_t flags */ +#define DXD_INDEX 0x80 /* B+-tree index */ +#define DXD_INLINE 0x40 /* in-line data extent */ +#define DXD_EXTENT 0x20 /* out-of-line single extent */ +#define DXD_FILE 0x10 /* out-of-line file (inode) */ +#define DXD_CORRUPT 0x08 /* Inconsistency detected */ + +/* dxd_t field construction + */ +#define DXDlength(dxd, len) PXDlength(&(dxd)->loc, len) +#define DXDaddress(dxd, addr) PXDaddress(&(dxd)->loc, addr) +#define lengthDXD(dxd) lengthPXD(&(dxd)->loc) +#define addressDXD(dxd) addressPXD(&(dxd)->loc) +#define DXDsize(dxd, size32) ((dxd)->size = cpu_to_le32(size32)) +#define sizeDXD(dxd) le32_to_cpu((dxd)->size) + +/* + * directory entry argument + */ +struct component_name { + int namlen; + wchar_t *name; +}; + + +/* + * DASD limit information - stored in directory inode + */ +struct dasd { + u8 thresh; /* Alert Threshold (in percent) */ + u8 delta; /* Alert Threshold delta (in percent) */ + u8 rsrvd1; + u8 limit_hi; /* DASD limit (in logical blocks) */ + __le32 limit_lo; /* DASD limit (in logical blocks) */ + u8 rsrvd2[3]; + u8 used_hi; /* DASD usage (in logical blocks) */ + __le32 used_lo; /* DASD usage (in logical blocks) */ +}; + +#define DASDLIMIT(dasdp) \ + (((u64)((dasdp)->limit_hi) << 32) + __le32_to_cpu((dasdp)->limit_lo)) +#define setDASDLIMIT(dasdp, limit)\ +{\ + (dasdp)->limit_hi = ((u64)limit) >> 32;\ + (dasdp)->limit_lo = __cpu_to_le32(limit);\ +} +#define DASDUSED(dasdp) \ + (((u64)((dasdp)->used_hi) << 32) + __le32_to_cpu((dasdp)->used_lo)) +#define setDASDUSED(dasdp, used)\ +{\ + (dasdp)->used_hi = ((u64)used) >> 32;\ + (dasdp)->used_lo = __cpu_to_le32(used);\ +} + +#endif /* !_H_JFS_TYPES */ diff --git a/fs/jfs/jfs_umount.c b/fs/jfs/jfs_umount.c new file mode 100644 index 000000000..18569f1ea --- /dev/null +++ b/fs/jfs/jfs_umount.c @@ -0,0 +1,163 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) International Business Machines Corp., 2000-2004 + */ + +/* + * jfs_umount.c + * + * note: file system in transition to aggregate/fileset: + * (ref. jfs_mount.c) + * + * file system unmount is interpreted as mount of the single/only + * fileset in the aggregate and, if unmount of the last fileset, + * as unmount of the aggerate; + */ + +#include +#include "jfs_incore.h" +#include "jfs_filsys.h" +#include "jfs_superblock.h" +#include "jfs_dmap.h" +#include "jfs_imap.h" +#include "jfs_logmgr.h" +#include "jfs_metapage.h" +#include "jfs_debug.h" + +/* + * NAME: jfs_umount(vfsp, flags, crp) + * + * FUNCTION: vfs_umount() + * + * PARAMETERS: vfsp - virtual file system pointer + * flags - unmount for shutdown + * crp - credential + * + * RETURN : EBUSY - device has open files + */ +int jfs_umount(struct super_block *sb) +{ + struct jfs_sb_info *sbi = JFS_SBI(sb); + struct inode *ipbmap = sbi->ipbmap; + struct inode *ipimap = sbi->ipimap; + struct inode *ipaimap = sbi->ipaimap; + struct inode *ipaimap2 = sbi->ipaimap2; + struct jfs_log *log; + int rc = 0; + + jfs_info("UnMount JFS: sb:0x%p", sb); + + /* + * update superblock and close log + * + * if mounted read-write and log based recovery was enabled + */ + if ((log = sbi->log)) + /* + * Wait for outstanding transactions to be written to log: + */ + jfs_flush_journal(log, 2); + + /* + * Hold log lock so write_special_inodes (lmLogSync) cannot see + * this sbi with a NULL inode pointer while iterating log->sb_list. + */ + if (log) + LOG_LOCK(log); + /* + * close fileset inode allocation map (aka fileset inode) + */ + diUnmount(ipimap, 0); + + diFreeSpecial(ipimap); + sbi->ipimap = NULL; + + /* + * close secondary aggregate inode allocation map + */ + if (ipaimap2) { + diUnmount(ipaimap2, 0); + diFreeSpecial(ipaimap2); + sbi->ipaimap2 = NULL; + } + + /* + * close aggregate inode allocation map + */ + diUnmount(ipaimap, 0); + diFreeSpecial(ipaimap); + sbi->ipaimap = NULL; + + /* + * close aggregate block allocation map + */ + dbUnmount(ipbmap, 0); + + diFreeSpecial(ipbmap); + sbi->ipbmap = NULL; + + /* + * Make sure all metadata makes it to disk before we mark + * the superblock as clean + */ + filemap_write_and_wait(sbi->direct_inode->i_mapping); + + if (log) + LOG_UNLOCK(log); + + /* + * ensure all file system file pages are propagated to their + * home blocks on disk (and their in-memory buffer pages are + * invalidated) BEFORE updating file system superblock state + * (to signify file system is unmounted cleanly, and thus in + * consistent state) and log superblock active file system + * list (to signify skip logredo()). + */ + if (log) { /* log = NULL if read-only mount */ + updateSuper(sb, FM_CLEAN); + + /* + * close log: + * + * remove file system from log active file system list. + */ + rc = lmLogClose(sb); + } + jfs_info("UnMount JFS Complete: rc = %d", rc); + return rc; +} + + +int jfs_umount_rw(struct super_block *sb) +{ + struct jfs_sb_info *sbi = JFS_SBI(sb); + struct jfs_log *log = sbi->log; + + if (!log) + return 0; + + /* + * close log: + * + * remove file system from log active file system list. + */ + jfs_flush_journal(log, 2); + + /* + * Make sure all metadata makes it to disk + */ + dbSync(sbi->ipbmap); + diSync(sbi->ipimap); + + /* + * Note that we have to do this even if sync_blockdev() will + * do exactly the same a few instructions later: We can't + * mark the superblock clean before everything is flushed to + * disk. + */ + filemap_write_and_wait(sbi->direct_inode->i_mapping); + + updateSuper(sb, FM_CLEAN); + + return lmLogClose(sb); +} diff --git a/fs/jfs/jfs_unicode.c b/fs/jfs/jfs_unicode.c new file mode 100644 index 000000000..0c1e90272 --- /dev/null +++ b/fs/jfs/jfs_unicode.c @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) International Business Machines Corp., 2000-2004 + */ + +#include +#include +#include "jfs_incore.h" +#include "jfs_filsys.h" +#include "jfs_unicode.h" +#include "jfs_debug.h" + +/* + * NAME: jfs_strfromUCS() + * + * FUNCTION: Convert little-endian unicode string to character string + * + */ +int jfs_strfromUCS_le(char *to, const __le16 * from, + int len, struct nls_table *codepage) +{ + int i; + int outlen = 0; + static int warn_again = 5; /* Only warn up to 5 times total */ + int warn = !!warn_again; /* once per string */ + + if (codepage) { + for (i = 0; (i < len) && from[i]; i++) { + int charlen; + charlen = + codepage->uni2char(le16_to_cpu(from[i]), + &to[outlen], + NLS_MAX_CHARSET_SIZE); + if (charlen > 0) + outlen += charlen; + else + to[outlen++] = '?'; + } + } else { + for (i = 0; (i < len) && from[i]; i++) { + if (unlikely(le16_to_cpu(from[i]) & 0xff00)) { + to[i] = '?'; + if (unlikely(warn)) { + warn--; + warn_again--; + printk(KERN_ERR + "non-latin1 character 0x%x found in JFS file name\n", + le16_to_cpu(from[i])); + printk(KERN_ERR + "mount with iocharset=utf8 to access\n"); + } + + } + else + to[i] = (char) (le16_to_cpu(from[i])); + } + outlen = i; + } + to[outlen] = 0; + return outlen; +} + +/* + * NAME: jfs_strtoUCS() + * + * FUNCTION: Convert character string to unicode string + * + */ +static int jfs_strtoUCS(wchar_t * to, const unsigned char *from, int len, + struct nls_table *codepage) +{ + int charlen; + int i; + + if (codepage) { + for (i = 0; len && *from; i++, from += charlen, len -= charlen) + { + charlen = codepage->char2uni(from, len, &to[i]); + if (charlen < 1) { + jfs_err("jfs_strtoUCS: char2uni returned %d.", + charlen); + jfs_err("charset = %s, char = 0x%x", + codepage->charset, *from); + return charlen; + } + } + } else { + for (i = 0; (i < len) && from[i]; i++) + to[i] = (wchar_t) from[i]; + } + + to[i] = 0; + return i; +} + +/* + * NAME: get_UCSname() + * + * FUNCTION: Allocate and translate to unicode string + * + */ +int get_UCSname(struct component_name * uniName, struct dentry *dentry) +{ + struct nls_table *nls_tab = JFS_SBI(dentry->d_sb)->nls_tab; + int length = dentry->d_name.len; + + if (length > JFS_NAME_MAX) + return -ENAMETOOLONG; + + uniName->name = + kmalloc_array(length + 1, sizeof(wchar_t), GFP_NOFS); + + if (uniName->name == NULL) + return -ENOMEM; + + uniName->namlen = jfs_strtoUCS(uniName->name, dentry->d_name.name, + length, nls_tab); + + if (uniName->namlen < 0) { + kfree(uniName->name); + return uniName->namlen; + } + + return 0; +} diff --git a/fs/jfs/jfs_unicode.h b/fs/jfs/jfs_unicode.h new file mode 100644 index 000000000..b6a78d4ae --- /dev/null +++ b/fs/jfs/jfs_unicode.h @@ -0,0 +1,136 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) International Business Machines Corp., 2000-2002 + * Portions Copyright (C) Christoph Hellwig, 2001-2002 + */ +#ifndef _H_JFS_UNICODE +#define _H_JFS_UNICODE + +#include +#include +#include "../nls/nls_ucs2_data.h" +#include "jfs_types.h" + +extern int get_UCSname(struct component_name *, struct dentry *); +extern int jfs_strfromUCS_le(char *, const __le16 *, int, struct nls_table *); + +#define free_UCSname(COMP) kfree((COMP)->name) + +/* + * UniStrcpy: Copy a string + */ +static inline wchar_t *UniStrcpy(wchar_t * ucs1, const wchar_t * ucs2) +{ + wchar_t *anchor = ucs1; /* save the start of result string */ + + while ((*ucs1++ = *ucs2++)); + return anchor; +} + + + +/* + * UniStrncpy: Copy length limited string with pad + */ +static inline __le16 *UniStrncpy_le(__le16 * ucs1, const __le16 * ucs2, + size_t n) +{ + __le16 *anchor = ucs1; + + while (n-- && *ucs2) /* Copy the strings */ + *ucs1++ = *ucs2++; + + n++; + while (n--) /* Pad with nulls */ + *ucs1++ = 0; + return anchor; +} + +/* + * UniStrncmp_le: Compare length limited string - native to little-endian + */ +static inline int UniStrncmp_le(const wchar_t * ucs1, const __le16 * ucs2, + size_t n) +{ + if (!n) + return 0; /* Null strings are equal */ + while ((*ucs1 == __le16_to_cpu(*ucs2)) && *ucs1 && --n) { + ucs1++; + ucs2++; + } + return (int) *ucs1 - (int) __le16_to_cpu(*ucs2); +} + +/* + * UniStrncpy_to_le: Copy length limited string with pad to little-endian + */ +static inline __le16 *UniStrncpy_to_le(__le16 * ucs1, const wchar_t * ucs2, + size_t n) +{ + __le16 *anchor = ucs1; + + while (n-- && *ucs2) /* Copy the strings */ + *ucs1++ = cpu_to_le16(*ucs2++); + + n++; + while (n--) /* Pad with nulls */ + *ucs1++ = 0; + return anchor; +} + +/* + * UniStrncpy_from_le: Copy length limited string with pad from little-endian + */ +static inline wchar_t *UniStrncpy_from_le(wchar_t * ucs1, const __le16 * ucs2, + size_t n) +{ + wchar_t *anchor = ucs1; + + while (n-- && *ucs2) /* Copy the strings */ + *ucs1++ = __le16_to_cpu(*ucs2++); + + n++; + while (n--) /* Pad with nulls */ + *ucs1++ = 0; + return anchor; +} + +/* + * UniToupper: Convert a unicode character to upper case + */ +static inline wchar_t UniToupper(wchar_t uc) +{ + const struct UniCaseRange *rp; + + if (uc < sizeof(NlsUniUpperTable)) { /* Latin characters */ + return uc + NlsUniUpperTable[uc]; /* Use base tables */ + } else { + rp = NlsUniUpperRange; /* Use range tables */ + while (rp->start) { + if (uc < rp->start) /* Before start of range */ + return uc; /* Uppercase = input */ + if (uc <= rp->end) /* In range */ + return uc + rp->table[uc - rp->start]; + rp++; /* Try next range */ + } + } + return uc; /* Past last range */ +} + + +/* + * UniStrupr: Upper case a unicode string + */ +static inline wchar_t *UniStrupr(wchar_t * upin) +{ + wchar_t *up; + + up = upin; + while (*up) { /* For all characters */ + *up = UniToupper(*up); + up++; + } + return upin; /* Return input pointer */ +} + +#endif /* !_H_JFS_UNICODE */ diff --git a/fs/jfs/jfs_xattr.h b/fs/jfs/jfs_xattr.h new file mode 100644 index 000000000..ec67d8554 --- /dev/null +++ b/fs/jfs/jfs_xattr.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) International Business Machines Corp., 2000-2002 + */ + +#ifndef H_JFS_XATTR +#define H_JFS_XATTR + +#include + +/* + * jfs_ea_list describe the on-disk format of the extended attributes. + * I know the null-terminator is redundant since namelen is stored, but + * I am maintaining compatibility with OS/2 where possible. + */ +struct jfs_ea { + u8 flag; /* Unused? */ + u8 namelen; /* Length of name */ + __le16 valuelen; /* Length of value */ + char name[]; /* Attribute name (includes null-terminator) */ +}; /* Value immediately follows name */ + +struct jfs_ea_list { + __le32 size; /* overall size */ + struct jfs_ea ea[]; /* Variable length list */ +}; + +/* Macros for defining maximum number of bytes supported for EAs */ +#define MAXEASIZE 65535 +#define MAXEALISTSIZE MAXEASIZE + +/* + * some macros for dealing with variable length EA lists. + */ +#define EA_SIZE(ea) \ + (sizeof (struct jfs_ea) + (ea)->namelen + 1 + \ + le16_to_cpu((ea)->valuelen)) +#define NEXT_EA(ea) ((struct jfs_ea *) (((char *) (ea)) + (EA_SIZE (ea)))) +#define FIRST_EA(ealist) ((ealist)->ea) +#define EALIST_SIZE(ealist) le32_to_cpu((ealist)->size) +#define END_EALIST(ealist) \ + ((struct jfs_ea *) (((char *) (ealist)) + EALIST_SIZE(ealist))) + +extern int __jfs_setxattr(tid_t, struct inode *, const char *, const void *, + size_t, int); +extern ssize_t __jfs_getxattr(struct inode *, const char *, void *, size_t); +extern ssize_t jfs_listxattr(struct dentry *, char *, size_t); + +extern const struct xattr_handler * const jfs_xattr_handlers[]; + +#ifdef CONFIG_JFS_SECURITY +extern int jfs_init_security(tid_t, struct inode *, struct inode *, + const struct qstr *); +#else +static inline int jfs_init_security(tid_t tid, struct inode *inode, + struct inode *dir, const struct qstr *qstr) +{ + return 0; +} +#endif + +#endif /* H_JFS_XATTR */ diff --git a/fs/jfs/jfs_xtree.c b/fs/jfs/jfs_xtree.c new file mode 100644 index 000000000..28c3cf960 --- /dev/null +++ b/fs/jfs/jfs_xtree.c @@ -0,0 +1,2930 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) International Business Machines Corp., 2000-2005 + */ +/* + * jfs_xtree.c: extent allocation descriptor B+-tree manager + */ + +#include +#include +#include +#include +#include "jfs_incore.h" +#include "jfs_filsys.h" +#include "jfs_metapage.h" +#include "jfs_dmap.h" +#include "jfs_dinode.h" +#include "jfs_superblock.h" +#include "jfs_debug.h" + +/* + * xtree local flag + */ +#define XT_INSERT 0x00000001 + +/* + * xtree key/entry comparison: extent offset + * + * return: + * -1: k < start of extent + * 0: start_of_extent <= k <= end_of_extent + * 1: k > end_of_extent + */ +#define XT_CMP(CMP, K, X, OFFSET64)\ +{\ + OFFSET64 = offsetXAD(X);\ + (CMP) = ((K) >= OFFSET64 + lengthXAD(X)) ? 1 :\ + ((K) < OFFSET64) ? -1 : 0;\ +} + +/* write a xad entry */ +#define XT_PUTENTRY(XAD, FLAG, OFF, LEN, ADDR)\ +{\ + (XAD)->flag = (FLAG);\ + XADoffset((XAD), (OFF));\ + XADlength((XAD), (LEN));\ + XADaddress((XAD), (ADDR));\ +} + +#define XT_PAGE(IP, MP) BT_PAGE(IP, MP, xtpage_t, i_xtroot) + +/* for consistency */ +#define XT_PUTPAGE(MP) BT_PUTPAGE(MP) + +#define XT_GETSEARCH(IP, LEAF, BN, MP, P, INDEX) \ + BT_GETSEARCH(IP, LEAF, BN, MP, xtpage_t, P, INDEX, i_xtroot) +/* xtree entry parameter descriptor */ +struct xtsplit { + struct metapage *mp; + s16 index; + u8 flag; + s64 off; + s64 addr; + int len; + struct pxdlist *pxdlist; +}; + + +/* + * statistics + */ +#ifdef CONFIG_JFS_STATISTICS +static struct { + uint search; + uint fastSearch; + uint split; +} xtStat; +#endif + + +/* + * forward references + */ +static int xtSearch(struct inode *ip, s64 xoff, s64 *next, int *cmpp, + struct btstack * btstack, int flag); + +static int xtSplitUp(tid_t tid, + struct inode *ip, + struct xtsplit * split, struct btstack * btstack); + +static int xtSplitPage(tid_t tid, struct inode *ip, struct xtsplit * split, + struct metapage ** rmpp, s64 * rbnp); + +static int xtSplitRoot(tid_t tid, struct inode *ip, + struct xtsplit * split, struct metapage ** rmpp); + +/* + * xt_getpage() + * + * function: get the page buffer for a specified block address. + * + * parameters: + * ip - pointer to the inode + * bn - block number (s64) of the xtree page to be retrieved; + * mp - pointer to a metapage pointer where the page buffer is returned; + * + * returns: + * A pointer to the xtree page (xtpage_t) on success, -EIO on error. + */ + +static inline xtpage_t *xt_getpage(struct inode *ip, s64 bn, struct metapage **mp) +{ + xtpage_t *p; + int rc; + + BT_GETPAGE(ip, bn, *mp, xtpage_t, PSIZE, p, rc, i_xtroot); + + if (rc) + return ERR_PTR(rc); + if ((le16_to_cpu(p->header.nextindex) < XTENTRYSTART) || + (le16_to_cpu(p->header.nextindex) > + le16_to_cpu(p->header.maxentry)) || + (le16_to_cpu(p->header.maxentry) > + ((bn == 0) ? XTROOTMAXSLOT : PSIZE >> L2XTSLOTSIZE))) { + jfs_error(ip->i_sb, "xt_getpage: xtree page corrupt\n"); + BT_PUTPAGE(*mp); + *mp = NULL; + return ERR_PTR(-EIO); + } + return p; +} + +/* + * xtLookup() + * + * function: map a single page into a physical extent; + */ +int xtLookup(struct inode *ip, s64 lstart, + s64 llen, int *pflag, s64 * paddr, s32 * plen, int no_check) +{ + int rc = 0; + struct btstack btstack; + int cmp; + s64 bn; + struct metapage *mp; + xtpage_t *p; + int index; + xad_t *xad; + s64 next, size, xoff, xend; + int xlen; + s64 xaddr; + + *paddr = 0; + *plen = llen; + + if (!no_check) { + /* is lookup offset beyond eof ? */ + size = ((u64) ip->i_size + (JFS_SBI(ip->i_sb)->bsize - 1)) >> + JFS_SBI(ip->i_sb)->l2bsize; + if (lstart >= size) + return 0; + } + + /* + * search for the xad entry covering the logical extent + */ +//search: + if ((rc = xtSearch(ip, lstart, &next, &cmp, &btstack, 0))) { + jfs_err("xtLookup: xtSearch returned %d", rc); + return rc; + } + + /* + * compute the physical extent covering logical extent + * + * N.B. search may have failed (e.g., hole in sparse file), + * and returned the index of the next entry. + */ + /* retrieve search result */ + XT_GETSEARCH(ip, btstack.top, bn, mp, p, index); + + /* is xad found covering start of logical extent ? + * lstart is a page start address, + * i.e., lstart cannot start in a hole; + */ + if (cmp) { + if (next) + *plen = min(next - lstart, llen); + goto out; + } + + /* + * lxd covered by xad + */ + xad = &p->xad[index]; + xoff = offsetXAD(xad); + xlen = lengthXAD(xad); + xend = xoff + xlen; + xaddr = addressXAD(xad); + + /* initialize new pxd */ + *pflag = xad->flag; + *paddr = xaddr + (lstart - xoff); + /* a page must be fully covered by an xad */ + *plen = min(xend - lstart, llen); + + out: + XT_PUTPAGE(mp); + + return rc; +} + +/* + * xtSearch() + * + * function: search for the xad entry covering specified offset. + * + * parameters: + * ip - file object; + * xoff - extent offset; + * nextp - address of next extent (if any) for search miss + * cmpp - comparison result: + * btstack - traverse stack; + * flag - search process flag (XT_INSERT); + * + * returns: + * btstack contains (bn, index) of search path traversed to the entry. + * *cmpp is set to result of comparison with the entry returned. + * the page containing the entry is pinned at exit. + */ +static int xtSearch(struct inode *ip, s64 xoff, s64 *nextp, + int *cmpp, struct btstack * btstack, int flag) +{ + struct jfs_inode_info *jfs_ip = JFS_IP(ip); + int cmp = 1; /* init for empty page */ + s64 bn; /* block number */ + struct metapage *mp; /* page buffer */ + xtpage_t *p; /* page */ + xad_t *xad; + int base, index, lim, btindex; + struct btframe *btsp; + int nsplit = 0; /* number of pages to split */ + s64 t64; + s64 next = 0; + + INCREMENT(xtStat.search); + + BT_CLR(btstack); + + btstack->nsplit = 0; + + /* + * search down tree from root: + * + * between two consecutive entries of and of + * internal page, child page Pi contains entry with k, Ki <= K < Kj. + * + * if entry with search key K is not found + * internal page search find the entry with largest key Ki + * less than K which point to the child page to search; + * leaf page search find the entry with smallest key Kj + * greater than K so that the returned index is the position of + * the entry to be shifted right for insertion of new entry. + * for empty tree, search key is greater than any key of the tree. + * + * by convention, root bn = 0. + */ + for (bn = 0;;) { + /* get/pin the page to search */ + p = xt_getpage(ip, bn, &mp); + if (IS_ERR(p)) + return PTR_ERR(p); + + /* try sequential access heuristics with the previous + * access entry in target leaf page: + * once search narrowed down into the target leaf, + * key must either match an entry in the leaf or + * key entry does not exist in the tree; + */ +//fastSearch: + if ((jfs_ip->btorder & BT_SEQUENTIAL) && + (p->header.flag & BT_LEAF) && + (index = jfs_ip->btindex) < + le16_to_cpu(p->header.nextindex)) { + xad = &p->xad[index]; + t64 = offsetXAD(xad); + if (xoff < t64 + lengthXAD(xad)) { + if (xoff >= t64) { + *cmpp = 0; + goto out; + } + + /* stop sequential access heuristics */ + goto binarySearch; + } else { /* (t64 + lengthXAD(xad)) <= xoff */ + + /* try next sequential entry */ + index++; + if (index < + le16_to_cpu(p->header.nextindex)) { + xad++; + t64 = offsetXAD(xad); + if (xoff < t64 + lengthXAD(xad)) { + if (xoff >= t64) { + *cmpp = 0; + goto out; + } + + /* miss: key falls between + * previous and this entry + */ + *cmpp = 1; + next = t64; + goto out; + } + + /* (xoff >= t64 + lengthXAD(xad)); + * matching entry may be further out: + * stop heuristic search + */ + /* stop sequential access heuristics */ + goto binarySearch; + } + + /* (index == p->header.nextindex); + * miss: key entry does not exist in + * the target leaf/tree + */ + *cmpp = 1; + goto out; + } + + /* + * if hit, return index of the entry found, and + * if miss, where new entry with search key is + * to be inserted; + */ + out: + /* compute number of pages to split */ + if (flag & XT_INSERT) { + if (p->header.nextindex == /* little-endian */ + p->header.maxentry) + nsplit++; + else + nsplit = 0; + btstack->nsplit = nsplit; + } + + /* save search result */ + btsp = btstack->top; + btsp->bn = bn; + btsp->index = index; + btsp->mp = mp; + + /* update sequential access heuristics */ + jfs_ip->btindex = index; + + if (nextp) + *nextp = next; + + INCREMENT(xtStat.fastSearch); + return 0; + } + + /* well, ... full search now */ + binarySearch: + lim = le16_to_cpu(p->header.nextindex) - XTENTRYSTART; + + /* + * binary search with search key K on the current page + */ + for (base = XTENTRYSTART; lim; lim >>= 1) { + index = base + (lim >> 1); + + XT_CMP(cmp, xoff, &p->xad[index], t64); + if (cmp == 0) { + /* + * search hit + */ + /* search hit - leaf page: + * return the entry found + */ + if (p->header.flag & BT_LEAF) { + *cmpp = cmp; + + /* compute number of pages to split */ + if (flag & XT_INSERT) { + if (p->header.nextindex == + p->header.maxentry) + nsplit++; + else + nsplit = 0; + btstack->nsplit = nsplit; + } + + /* save search result */ + btsp = btstack->top; + btsp->bn = bn; + btsp->index = index; + btsp->mp = mp; + + /* init sequential access heuristics */ + btindex = jfs_ip->btindex; + if (index == btindex || + index == btindex + 1) + jfs_ip->btorder = BT_SEQUENTIAL; + else + jfs_ip->btorder = BT_RANDOM; + jfs_ip->btindex = index; + + return 0; + } + /* search hit - internal page: + * descend/search its child page + */ + if (index < le16_to_cpu(p->header.nextindex)-1) + next = offsetXAD(&p->xad[index + 1]); + goto next; + } + + if (cmp > 0) { + base = index + 1; + --lim; + } + } + + /* + * search miss + * + * base is the smallest index with key (Kj) greater than + * search key (K) and may be zero or maxentry index. + */ + if (base < le16_to_cpu(p->header.nextindex)) + next = offsetXAD(&p->xad[base]); + /* + * search miss - leaf page: + * + * return location of entry (base) where new entry with + * search key K is to be inserted. + */ + if (p->header.flag & BT_LEAF) { + *cmpp = cmp; + + /* compute number of pages to split */ + if (flag & XT_INSERT) { + if (p->header.nextindex == + p->header.maxentry) + nsplit++; + else + nsplit = 0; + btstack->nsplit = nsplit; + } + + /* save search result */ + btsp = btstack->top; + btsp->bn = bn; + btsp->index = base; + btsp->mp = mp; + + /* init sequential access heuristics */ + btindex = jfs_ip->btindex; + if (base == btindex || base == btindex + 1) + jfs_ip->btorder = BT_SEQUENTIAL; + else + jfs_ip->btorder = BT_RANDOM; + jfs_ip->btindex = base; + + if (nextp) + *nextp = next; + + return 0; + } + + /* + * search miss - non-leaf page: + * + * if base is non-zero, decrement base by one to get the parent + * entry of the child page to search. + */ + index = base ? base - 1 : base; + + /* + * go down to child page + */ + next: + /* update number of pages to split */ + if (p->header.nextindex == p->header.maxentry) + nsplit++; + else + nsplit = 0; + + /* push (bn, index) of the parent page/entry */ + if (BT_STACK_FULL(btstack)) { + jfs_error(ip->i_sb, "stack overrun!\n"); + XT_PUTPAGE(mp); + return -EIO; + } + BT_PUSH(btstack, bn, index); + + /* get the child page block number */ + bn = addressXAD(&p->xad[index]); + + /* unpin the parent page */ + XT_PUTPAGE(mp); + } +} + +/* + * xtInsert() + * + * function: + * + * parameter: + * tid - transaction id; + * ip - file object; + * xflag - extent flag (XAD_NOTRECORDED): + * xoff - extent offset; + * xlen - extent length; + * xaddrp - extent address pointer (in/out): + * if (*xaddrp) + * caller allocated data extent at *xaddrp; + * else + * allocate data extent and return its xaddr; + * flag - + * + * return: + */ +int xtInsert(tid_t tid, /* transaction id */ + struct inode *ip, int xflag, s64 xoff, s32 xlen, s64 * xaddrp, + int flag) +{ + int rc = 0; + s64 xaddr, hint; + struct metapage *mp; /* meta-page buffer */ + xtpage_t *p; /* base B+-tree index page */ + s64 bn; + int index, nextindex; + struct btstack btstack; /* traverse stack */ + struct xtsplit split; /* split information */ + xad_t *xad; + int cmp; + s64 next; + struct tlock *tlck; + struct xtlock *xtlck; + + jfs_info("xtInsert: nxoff:0x%lx nxlen:0x%x", (ulong) xoff, xlen); + + /* + * search for the entry location at which to insert: + * + * xtFastSearch() and xtSearch() both returns (leaf page + * pinned, index at which to insert). + * n.b. xtSearch() may return index of maxentry of + * the full page. + */ + if ((rc = xtSearch(ip, xoff, &next, &cmp, &btstack, XT_INSERT))) + return rc; + + /* retrieve search result */ + XT_GETSEARCH(ip, btstack.top, bn, mp, p, index); + + /* This test must follow XT_GETSEARCH since mp must be valid if + * we branch to out: */ + if ((cmp == 0) || (next && (xlen > next - xoff))) { + rc = -EEXIST; + goto out; + } + + /* + * allocate data extent requested + * + * allocation hint: last xad + */ + if ((xaddr = *xaddrp) == 0) { + if (index > XTENTRYSTART) { + xad = &p->xad[index - 1]; + hint = addressXAD(xad) + lengthXAD(xad) - 1; + } else + hint = 0; + if ((rc = dquot_alloc_block(ip, xlen))) + goto out; + if ((rc = dbAlloc(ip, hint, (s64) xlen, &xaddr))) { + dquot_free_block(ip, xlen); + goto out; + } + } + + /* + * insert entry for new extent + */ + xflag |= XAD_NEW; + + /* + * if the leaf page is full, split the page and + * propagate up the router entry for the new page from split + * + * The xtSplitUp() will insert the entry and unpin the leaf page. + */ + nextindex = le16_to_cpu(p->header.nextindex); + if (nextindex == le16_to_cpu(p->header.maxentry)) { + split.mp = mp; + split.index = index; + split.flag = xflag; + split.off = xoff; + split.len = xlen; + split.addr = xaddr; + split.pxdlist = NULL; + if ((rc = xtSplitUp(tid, ip, &split, &btstack))) { + /* undo data extent allocation */ + if (*xaddrp == 0) { + dbFree(ip, xaddr, (s64) xlen); + dquot_free_block(ip, xlen); + } + return rc; + } + + *xaddrp = xaddr; + return 0; + } + + /* + * insert the new entry into the leaf page + */ + /* + * acquire a transaction lock on the leaf page; + * + * action: xad insertion/extension; + */ + BT_MARK_DIRTY(mp, ip); + + /* if insert into middle, shift right remaining entries. */ + if (index < nextindex) + memmove(&p->xad[index + 1], &p->xad[index], + (nextindex - index) * sizeof(xad_t)); + + /* insert the new entry: mark the entry NEW */ + xad = &p->xad[index]; + XT_PUTENTRY(xad, xflag, xoff, xlen, xaddr); + + /* advance next available entry index */ + le16_add_cpu(&p->header.nextindex, 1); + + /* Don't log it if there are no links to the file */ + if (!test_cflag(COMMIT_Nolink, ip)) { + tlck = txLock(tid, ip, mp, tlckXTREE | tlckGROW); + xtlck = (struct xtlock *) & tlck->lock; + xtlck->lwm.offset = + (xtlck->lwm.offset) ? min(index, + (int)xtlck->lwm.offset) : index; + xtlck->lwm.length = + le16_to_cpu(p->header.nextindex) - xtlck->lwm.offset; + } + + *xaddrp = xaddr; + + out: + /* unpin the leaf page */ + XT_PUTPAGE(mp); + + return rc; +} + + +/* + * xtSplitUp() + * + * function: + * split full pages as propagating insertion up the tree + * + * parameter: + * tid - transaction id; + * ip - file object; + * split - entry parameter descriptor; + * btstack - traverse stack from xtSearch() + * + * return: + */ +static int +xtSplitUp(tid_t tid, + struct inode *ip, struct xtsplit * split, struct btstack * btstack) +{ + int rc = 0; + struct metapage *smp; + xtpage_t *sp; /* split page */ + struct metapage *rmp; + s64 rbn; /* new right page block number */ + struct metapage *rcmp; + xtpage_t *rcp; /* right child page */ + s64 rcbn; /* right child page block number */ + int skip; /* index of entry of insertion */ + int nextindex; /* next available entry index of p */ + struct btframe *parent; /* parent page entry on traverse stack */ + xad_t *xad; + s64 xaddr; + int xlen; + int nsplit; /* number of pages split */ + struct pxdlist pxdlist; + pxd_t *pxd; + struct tlock *tlck; + struct xtlock *xtlck; + + smp = split->mp; + sp = XT_PAGE(ip, smp); + + /* is inode xtree root extension/inline EA area free ? */ + if ((sp->header.flag & BT_ROOT) && (!S_ISDIR(ip->i_mode)) && + (le16_to_cpu(sp->header.maxentry) < XTROOTMAXSLOT) && + (JFS_IP(ip)->mode2 & INLINEEA)) { + sp->header.maxentry = cpu_to_le16(XTROOTMAXSLOT); + JFS_IP(ip)->mode2 &= ~INLINEEA; + + BT_MARK_DIRTY(smp, ip); + /* + * acquire a transaction lock on the leaf page; + * + * action: xad insertion/extension; + */ + + /* if insert into middle, shift right remaining entries. */ + skip = split->index; + nextindex = le16_to_cpu(sp->header.nextindex); + if (skip < nextindex) + memmove(&sp->xad[skip + 1], &sp->xad[skip], + (nextindex - skip) * sizeof(xad_t)); + + /* insert the new entry: mark the entry NEW */ + xad = &sp->xad[skip]; + XT_PUTENTRY(xad, split->flag, split->off, split->len, + split->addr); + + /* advance next available entry index */ + le16_add_cpu(&sp->header.nextindex, 1); + + /* Don't log it if there are no links to the file */ + if (!test_cflag(COMMIT_Nolink, ip)) { + tlck = txLock(tid, ip, smp, tlckXTREE | tlckGROW); + xtlck = (struct xtlock *) & tlck->lock; + xtlck->lwm.offset = (xtlck->lwm.offset) ? + min(skip, (int)xtlck->lwm.offset) : skip; + xtlck->lwm.length = + le16_to_cpu(sp->header.nextindex) - + xtlck->lwm.offset; + } + + return 0; + } + + /* + * allocate new index blocks to cover index page split(s) + * + * allocation hint: ? + */ + if (split->pxdlist == NULL) { + nsplit = btstack->nsplit; + split->pxdlist = &pxdlist; + pxdlist.maxnpxd = pxdlist.npxd = 0; + pxd = &pxdlist.pxd[0]; + xlen = JFS_SBI(ip->i_sb)->nbperpage; + for (; nsplit > 0; nsplit--, pxd++) { + if ((rc = dbAlloc(ip, (s64) 0, (s64) xlen, &xaddr)) + == 0) { + PXDaddress(pxd, xaddr); + PXDlength(pxd, xlen); + + pxdlist.maxnpxd++; + + continue; + } + + /* undo allocation */ + + XT_PUTPAGE(smp); + return rc; + } + } + + /* + * Split leaf page into and a new right page . + * + * The split routines insert the new entry into the leaf page, + * and acquire txLock as appropriate. + * return pinned and its block number . + */ + rc = (sp->header.flag & BT_ROOT) ? + xtSplitRoot(tid, ip, split, &rmp) : + xtSplitPage(tid, ip, split, &rmp, &rbn); + + XT_PUTPAGE(smp); + + if (rc) + return -EIO; + /* + * propagate up the router entry for the leaf page just split + * + * insert a router entry for the new page into the parent page, + * propagate the insert/split up the tree by walking back the stack + * of (bn of parent page, index of child page entry in parent page) + * that were traversed during the search for the page that split. + * + * the propagation of insert/split up the tree stops if the root + * splits or the page inserted into doesn't have to split to hold + * the new entry. + * + * the parent entry for the split page remains the same, and + * a new entry is inserted at its right with the first key and + * block number of the new right page. + * + * There are a maximum of 3 pages pinned at any time: + * right child, left parent and right parent (when the parent splits) + * to keep the child page pinned while working on the parent. + * make sure that all pins are released at exit. + */ + while ((parent = BT_POP(btstack)) != NULL) { + /* parent page specified by stack frame */ + + /* keep current child pages pinned */ + rcmp = rmp; + rcbn = rbn; + rcp = XT_PAGE(ip, rcmp); + + /* + * insert router entry in parent for new right child page + */ + /* get/pin the parent page */ + sp = xt_getpage(ip, parent->bn, &smp); + if (IS_ERR(sp)) { + XT_PUTPAGE(rcmp); + return PTR_ERR(sp); + } + + /* + * The new key entry goes ONE AFTER the index of parent entry, + * because the split was to the right. + */ + skip = parent->index + 1; + + /* + * split or shift right remaining entries of the parent page + */ + nextindex = le16_to_cpu(sp->header.nextindex); + /* + * parent page is full - split the parent page + */ + if (nextindex == le16_to_cpu(sp->header.maxentry)) { + /* init for parent page split */ + split->mp = smp; + split->index = skip; /* index at insert */ + split->flag = XAD_NEW; + split->off = offsetXAD(&rcp->xad[XTENTRYSTART]); + split->len = JFS_SBI(ip->i_sb)->nbperpage; + split->addr = rcbn; + + /* unpin previous right child page */ + XT_PUTPAGE(rcmp); + + /* The split routines insert the new entry, + * and acquire txLock as appropriate. + * return pinned and its block number . + */ + rc = (sp->header.flag & BT_ROOT) ? + xtSplitRoot(tid, ip, split, &rmp) : + xtSplitPage(tid, ip, split, &rmp, &rbn); + if (rc) { + XT_PUTPAGE(smp); + return rc; + } + + XT_PUTPAGE(smp); + /* keep new child page pinned */ + } + /* + * parent page is not full - insert in parent page + */ + else { + /* + * insert router entry in parent for the right child + * page from the first entry of the right child page: + */ + /* + * acquire a transaction lock on the parent page; + * + * action: router xad insertion; + */ + BT_MARK_DIRTY(smp, ip); + + /* + * if insert into middle, shift right remaining entries + */ + if (skip < nextindex) + memmove(&sp->xad[skip + 1], &sp->xad[skip], + (nextindex - + skip) << L2XTSLOTSIZE); + + /* insert the router entry */ + xad = &sp->xad[skip]; + XT_PUTENTRY(xad, XAD_NEW, + offsetXAD(&rcp->xad[XTENTRYSTART]), + JFS_SBI(ip->i_sb)->nbperpage, rcbn); + + /* advance next available entry index. */ + le16_add_cpu(&sp->header.nextindex, 1); + + /* Don't log it if there are no links to the file */ + if (!test_cflag(COMMIT_Nolink, ip)) { + tlck = txLock(tid, ip, smp, + tlckXTREE | tlckGROW); + xtlck = (struct xtlock *) & tlck->lock; + xtlck->lwm.offset = (xtlck->lwm.offset) ? + min(skip, (int)xtlck->lwm.offset) : skip; + xtlck->lwm.length = + le16_to_cpu(sp->header.nextindex) - + xtlck->lwm.offset; + } + + /* unpin parent page */ + XT_PUTPAGE(smp); + + /* exit propagate up */ + break; + } + } + + /* unpin current right page */ + XT_PUTPAGE(rmp); + + return 0; +} + + +/* + * xtSplitPage() + * + * function: + * split a full non-root page into + * original/split/left page and new right page + * i.e., the original/split page remains as left page. + * + * parameter: + * int tid, + * struct inode *ip, + * struct xtsplit *split, + * struct metapage **rmpp, + * u64 *rbnp, + * + * return: + * Pointer to page in which to insert or NULL on error. + */ +static int +xtSplitPage(tid_t tid, struct inode *ip, + struct xtsplit * split, struct metapage ** rmpp, s64 * rbnp) +{ + int rc = 0; + struct metapage *smp; + xtpage_t *sp; + struct metapage *rmp; + xtpage_t *rp; /* new right page allocated */ + s64 rbn; /* new right page block number */ + struct metapage *mp; + xtpage_t *p; + s64 nextbn; + int skip, maxentry, middle, righthalf, n; + xad_t *xad; + struct pxdlist *pxdlist; + pxd_t *pxd; + struct tlock *tlck; + struct xtlock *sxtlck = NULL, *rxtlck = NULL; + int quota_allocation = 0; + + smp = split->mp; + sp = XT_PAGE(ip, smp); + + INCREMENT(xtStat.split); + + pxdlist = split->pxdlist; + pxd = &pxdlist->pxd[pxdlist->npxd]; + pxdlist->npxd++; + rbn = addressPXD(pxd); + + /* Allocate blocks to quota. */ + rc = dquot_alloc_block(ip, lengthPXD(pxd)); + if (rc) + goto clean_up; + + quota_allocation += lengthPXD(pxd); + + /* + * allocate the new right page for the split + */ + rmp = get_metapage(ip, rbn, PSIZE, 1); + if (rmp == NULL) { + rc = -EIO; + goto clean_up; + } + + jfs_info("xtSplitPage: ip:0x%p smp:0x%p rmp:0x%p", ip, smp, rmp); + + BT_MARK_DIRTY(rmp, ip); + /* + * action: new page; + */ + + rp = (xtpage_t *) rmp->data; + rp->header.self = *pxd; + rp->header.flag = sp->header.flag & BT_TYPE; + rp->header.maxentry = sp->header.maxentry; /* little-endian */ + rp->header.nextindex = cpu_to_le16(XTENTRYSTART); + + BT_MARK_DIRTY(smp, ip); + /* Don't log it if there are no links to the file */ + if (!test_cflag(COMMIT_Nolink, ip)) { + /* + * acquire a transaction lock on the new right page; + */ + tlck = txLock(tid, ip, rmp, tlckXTREE | tlckNEW); + rxtlck = (struct xtlock *) & tlck->lock; + rxtlck->lwm.offset = XTENTRYSTART; + /* + * acquire a transaction lock on the split page + */ + tlck = txLock(tid, ip, smp, tlckXTREE | tlckGROW); + sxtlck = (struct xtlock *) & tlck->lock; + } + + /* + * initialize/update sibling pointers of and + */ + nextbn = le64_to_cpu(sp->header.next); + rp->header.next = cpu_to_le64(nextbn); + rp->header.prev = cpu_to_le64(addressPXD(&sp->header.self)); + sp->header.next = cpu_to_le64(rbn); + + skip = split->index; + + /* + * sequential append at tail (after last entry of last page) + * + * if splitting the last page on a level because of appending + * a entry to it (skip is maxentry), it's likely that the access is + * sequential. adding an empty page on the side of the level is less + * work and can push the fill factor much higher than normal. + * if we're wrong it's no big deal - we will do the split the right + * way next time. + * (it may look like it's equally easy to do a similar hack for + * reverse sorted data, that is, split the tree left, but it's not. + * Be my guest.) + */ + if (nextbn == 0 && skip == le16_to_cpu(sp->header.maxentry)) { + /* + * acquire a transaction lock on the new/right page; + * + * action: xad insertion; + */ + /* insert entry at the first entry of the new right page */ + xad = &rp->xad[XTENTRYSTART]; + XT_PUTENTRY(xad, split->flag, split->off, split->len, + split->addr); + + rp->header.nextindex = cpu_to_le16(XTENTRYSTART + 1); + + if (!test_cflag(COMMIT_Nolink, ip)) { + /* rxtlck->lwm.offset = XTENTRYSTART; */ + rxtlck->lwm.length = 1; + } + + *rmpp = rmp; + *rbnp = rbn; + + jfs_info("xtSplitPage: sp:0x%p rp:0x%p", sp, rp); + return 0; + } + + /* + * non-sequential insert (at possibly middle page) + */ + + /* + * update previous pointer of old next/right page of + */ + if (nextbn != 0) { + p = xt_getpage(ip, nextbn, &mp); + if (IS_ERR(p)) { + XT_PUTPAGE(rmp); + return PTR_ERR(p); + } + + BT_MARK_DIRTY(mp, ip); + /* + * acquire a transaction lock on the next page; + * + * action:sibling pointer update; + */ + if (!test_cflag(COMMIT_Nolink, ip)) + tlck = txLock(tid, ip, mp, tlckXTREE | tlckRELINK); + + p->header.prev = cpu_to_le64(rbn); + + /* sibling page may have been updated previously, or + * it may be updated later; + */ + + XT_PUTPAGE(mp); + } + + /* + * split the data between the split and new/right pages + */ + maxentry = le16_to_cpu(sp->header.maxentry); + middle = maxentry >> 1; + righthalf = maxentry - middle; + + /* + * skip index in old split/left page - insert into left page: + */ + if (skip <= middle) { + /* move right half of split page to the new right page */ + memmove(&rp->xad[XTENTRYSTART], &sp->xad[middle], + righthalf << L2XTSLOTSIZE); + + /* shift right tail of left half to make room for new entry */ + if (skip < middle) + memmove(&sp->xad[skip + 1], &sp->xad[skip], + (middle - skip) << L2XTSLOTSIZE); + + /* insert new entry */ + xad = &sp->xad[skip]; + XT_PUTENTRY(xad, split->flag, split->off, split->len, + split->addr); + + /* update page header */ + sp->header.nextindex = cpu_to_le16(middle + 1); + if (!test_cflag(COMMIT_Nolink, ip)) { + sxtlck->lwm.offset = (sxtlck->lwm.offset) ? + min(skip, (int)sxtlck->lwm.offset) : skip; + } + + rp->header.nextindex = + cpu_to_le16(XTENTRYSTART + righthalf); + } + /* + * skip index in new right page - insert into right page: + */ + else { + /* move left head of right half to right page */ + n = skip - middle; + memmove(&rp->xad[XTENTRYSTART], &sp->xad[middle], + n << L2XTSLOTSIZE); + + /* insert new entry */ + n += XTENTRYSTART; + xad = &rp->xad[n]; + XT_PUTENTRY(xad, split->flag, split->off, split->len, + split->addr); + + /* move right tail of right half to right page */ + if (skip < maxentry) + memmove(&rp->xad[n + 1], &sp->xad[skip], + (maxentry - skip) << L2XTSLOTSIZE); + + /* update page header */ + sp->header.nextindex = cpu_to_le16(middle); + if (!test_cflag(COMMIT_Nolink, ip)) { + sxtlck->lwm.offset = (sxtlck->lwm.offset) ? + min(middle, (int)sxtlck->lwm.offset) : middle; + } + + rp->header.nextindex = cpu_to_le16(XTENTRYSTART + + righthalf + 1); + } + + if (!test_cflag(COMMIT_Nolink, ip)) { + sxtlck->lwm.length = le16_to_cpu(sp->header.nextindex) - + sxtlck->lwm.offset; + + /* rxtlck->lwm.offset = XTENTRYSTART; */ + rxtlck->lwm.length = le16_to_cpu(rp->header.nextindex) - + XTENTRYSTART; + } + + *rmpp = rmp; + *rbnp = rbn; + + jfs_info("xtSplitPage: sp:0x%p rp:0x%p", sp, rp); + return rc; + + clean_up: + + /* Rollback quota allocation. */ + if (quota_allocation) + dquot_free_block(ip, quota_allocation); + + return (rc); +} + + +/* + * xtSplitRoot() + * + * function: + * split the full root page into original/root/split page and new + * right page + * i.e., root remains fixed in tree anchor (inode) and the root is + * copied to a single new right child page since root page << + * non-root page, and the split root page contains a single entry + * for the new right child page. + * + * parameter: + * int tid, + * struct inode *ip, + * struct xtsplit *split, + * struct metapage **rmpp) + * + * return: + * Pointer to page in which to insert or NULL on error. + */ +static int +xtSplitRoot(tid_t tid, + struct inode *ip, struct xtsplit * split, struct metapage ** rmpp) +{ + xtpage_t *sp; + struct metapage *rmp; + xtpage_t *rp; + s64 rbn; + int skip, nextindex; + xad_t *xad; + pxd_t *pxd; + struct pxdlist *pxdlist; + struct tlock *tlck; + struct xtlock *xtlck; + int rc; + + sp = (xtpage_t *) &JFS_IP(ip)->i_xtroot; + + INCREMENT(xtStat.split); + + /* + * allocate a single (right) child page + */ + pxdlist = split->pxdlist; + pxd = &pxdlist->pxd[pxdlist->npxd]; + pxdlist->npxd++; + rbn = addressPXD(pxd); + rmp = get_metapage(ip, rbn, PSIZE, 1); + if (rmp == NULL) + return -EIO; + + /* Allocate blocks to quota. */ + rc = dquot_alloc_block(ip, lengthPXD(pxd)); + if (rc) { + release_metapage(rmp); + return rc; + } + + jfs_info("xtSplitRoot: ip:0x%p rmp:0x%p", ip, rmp); + + /* + * acquire a transaction lock on the new right page; + * + * action: new page; + */ + BT_MARK_DIRTY(rmp, ip); + + rp = (xtpage_t *) rmp->data; + rp->header.flag = + (sp->header.flag & BT_LEAF) ? BT_LEAF : BT_INTERNAL; + rp->header.self = *pxd; + rp->header.nextindex = cpu_to_le16(XTENTRYSTART); + rp->header.maxentry = cpu_to_le16(PSIZE >> L2XTSLOTSIZE); + + /* initialize sibling pointers */ + rp->header.next = 0; + rp->header.prev = 0; + + /* + * copy the in-line root page into new right page extent + */ + nextindex = le16_to_cpu(sp->header.maxentry); + memmove(&rp->xad[XTENTRYSTART], &sp->xad[XTENTRYSTART], + (nextindex - XTENTRYSTART) << L2XTSLOTSIZE); + + /* + * insert the new entry into the new right/child page + * (skip index in the new right page will not change) + */ + skip = split->index; + /* if insert into middle, shift right remaining entries */ + if (skip != nextindex) + memmove(&rp->xad[skip + 1], &rp->xad[skip], + (nextindex - skip) * sizeof(xad_t)); + + xad = &rp->xad[skip]; + XT_PUTENTRY(xad, split->flag, split->off, split->len, split->addr); + + /* update page header */ + rp->header.nextindex = cpu_to_le16(nextindex + 1); + + if (!test_cflag(COMMIT_Nolink, ip)) { + tlck = txLock(tid, ip, rmp, tlckXTREE | tlckNEW); + xtlck = (struct xtlock *) & tlck->lock; + xtlck->lwm.offset = XTENTRYSTART; + xtlck->lwm.length = le16_to_cpu(rp->header.nextindex) - + XTENTRYSTART; + } + + /* + * reset the root + * + * init root with the single entry for the new right page + * set the 1st entry offset to 0, which force the left-most key + * at any level of the tree to be less than any search key. + */ + /* + * acquire a transaction lock on the root page (in-memory inode); + * + * action: root split; + */ + BT_MARK_DIRTY(split->mp, ip); + + xad = &sp->xad[XTENTRYSTART]; + XT_PUTENTRY(xad, XAD_NEW, 0, JFS_SBI(ip->i_sb)->nbperpage, rbn); + + /* update page header of root */ + sp->header.flag &= ~BT_LEAF; + sp->header.flag |= BT_INTERNAL; + + sp->header.nextindex = cpu_to_le16(XTENTRYSTART + 1); + + if (!test_cflag(COMMIT_Nolink, ip)) { + tlck = txLock(tid, ip, split->mp, tlckXTREE | tlckGROW); + xtlck = (struct xtlock *) & tlck->lock; + xtlck->lwm.offset = XTENTRYSTART; + xtlck->lwm.length = 1; + } + + *rmpp = rmp; + + jfs_info("xtSplitRoot: sp:0x%p rp:0x%p", sp, rp); + return 0; +} + + +/* + * xtExtend() + * + * function: extend in-place; + * + * note: existing extent may or may not have been committed. + * caller is responsible for pager buffer cache update, and + * working block allocation map update; + * update pmap: alloc whole extended extent; + */ +int xtExtend(tid_t tid, /* transaction id */ + struct inode *ip, s64 xoff, /* delta extent offset */ + s32 xlen, /* delta extent length */ + int flag) +{ + int rc = 0; + int cmp; + struct metapage *mp; /* meta-page buffer */ + xtpage_t *p; /* base B+-tree index page */ + s64 bn; + int index, nextindex, len; + struct btstack btstack; /* traverse stack */ + struct xtsplit split; /* split information */ + xad_t *xad; + s64 xaddr; + struct tlock *tlck; + struct xtlock *xtlck = NULL; + + jfs_info("xtExtend: nxoff:0x%lx nxlen:0x%x", (ulong) xoff, xlen); + + /* there must exist extent to be extended */ + if ((rc = xtSearch(ip, xoff - 1, NULL, &cmp, &btstack, XT_INSERT))) + return rc; + + /* retrieve search result */ + XT_GETSEARCH(ip, btstack.top, bn, mp, p, index); + + if (cmp != 0) { + XT_PUTPAGE(mp); + jfs_error(ip->i_sb, "xtSearch did not find extent\n"); + return -EIO; + } + + /* extension must be contiguous */ + xad = &p->xad[index]; + if ((offsetXAD(xad) + lengthXAD(xad)) != xoff) { + XT_PUTPAGE(mp); + jfs_error(ip->i_sb, "extension is not contiguous\n"); + return -EIO; + } + + /* + * acquire a transaction lock on the leaf page; + * + * action: xad insertion/extension; + */ + BT_MARK_DIRTY(mp, ip); + if (!test_cflag(COMMIT_Nolink, ip)) { + tlck = txLock(tid, ip, mp, tlckXTREE | tlckGROW); + xtlck = (struct xtlock *) & tlck->lock; + } + + /* extend will overflow extent ? */ + xlen = lengthXAD(xad) + xlen; + if ((len = xlen - MAXXLEN) <= 0) + goto extendOld; + + /* + * extent overflow: insert entry for new extent + */ +//insertNew: + xoff = offsetXAD(xad) + MAXXLEN; + xaddr = addressXAD(xad) + MAXXLEN; + nextindex = le16_to_cpu(p->header.nextindex); + + /* + * if the leaf page is full, insert the new entry and + * propagate up the router entry for the new page from split + * + * The xtSplitUp() will insert the entry and unpin the leaf page. + */ + if (nextindex == le16_to_cpu(p->header.maxentry)) { + /* xtSpliUp() unpins leaf pages */ + split.mp = mp; + split.index = index + 1; + split.flag = XAD_NEW; + split.off = xoff; /* split offset */ + split.len = len; + split.addr = xaddr; + split.pxdlist = NULL; + if ((rc = xtSplitUp(tid, ip, &split, &btstack))) + return rc; + + /* get back old page */ + p = xt_getpage(ip, bn, &mp); + if (IS_ERR(p)) + return PTR_ERR(p); + /* + * if leaf root has been split, original root has been + * copied to new child page, i.e., original entry now + * resides on the new child page; + */ + if (p->header.flag & BT_INTERNAL) { + ASSERT(p->header.nextindex == + cpu_to_le16(XTENTRYSTART + 1)); + xad = &p->xad[XTENTRYSTART]; + bn = addressXAD(xad); + XT_PUTPAGE(mp); + + /* get new child page */ + p = xt_getpage(ip, bn, &mp); + if (IS_ERR(p)) + return PTR_ERR(p); + + BT_MARK_DIRTY(mp, ip); + if (!test_cflag(COMMIT_Nolink, ip)) { + tlck = txLock(tid, ip, mp, tlckXTREE|tlckGROW); + xtlck = (struct xtlock *) & tlck->lock; + } + } + } + /* + * insert the new entry into the leaf page + */ + else { + /* insert the new entry: mark the entry NEW */ + xad = &p->xad[index + 1]; + XT_PUTENTRY(xad, XAD_NEW, xoff, len, xaddr); + + /* advance next available entry index */ + le16_add_cpu(&p->header.nextindex, 1); + } + + /* get back old entry */ + xad = &p->xad[index]; + xlen = MAXXLEN; + + /* + * extend old extent + */ + extendOld: + XADlength(xad, xlen); + if (!(xad->flag & XAD_NEW)) + xad->flag |= XAD_EXTENDED; + + if (!test_cflag(COMMIT_Nolink, ip)) { + xtlck->lwm.offset = + (xtlck->lwm.offset) ? min(index, + (int)xtlck->lwm.offset) : index; + xtlck->lwm.length = + le16_to_cpu(p->header.nextindex) - xtlck->lwm.offset; + } + + /* unpin the leaf page */ + XT_PUTPAGE(mp); + + return rc; +} + +/* + * xtUpdate() + * + * function: update XAD; + * + * update extent for allocated_but_not_recorded or + * compressed extent; + * + * parameter: + * nxad - new XAD; + * logical extent of the specified XAD must be completely + * contained by an existing XAD; + */ +int xtUpdate(tid_t tid, struct inode *ip, xad_t * nxad) +{ /* new XAD */ + int rc = 0; + int cmp; + struct metapage *mp; /* meta-page buffer */ + xtpage_t *p; /* base B+-tree index page */ + s64 bn; + int index0, index, newindex, nextindex; + struct btstack btstack; /* traverse stack */ + struct xtsplit split; /* split information */ + xad_t *xad, *lxad, *rxad; + int xflag; + s64 nxoff, xoff; + int nxlen, xlen, lxlen, rxlen; + s64 nxaddr, xaddr; + struct tlock *tlck; + struct xtlock *xtlck = NULL; + int newpage = 0; + + /* there must exist extent to be tailgated */ + nxoff = offsetXAD(nxad); + nxlen = lengthXAD(nxad); + nxaddr = addressXAD(nxad); + + if ((rc = xtSearch(ip, nxoff, NULL, &cmp, &btstack, XT_INSERT))) + return rc; + + /* retrieve search result */ + XT_GETSEARCH(ip, btstack.top, bn, mp, p, index0); + + if (cmp != 0) { + XT_PUTPAGE(mp); + jfs_error(ip->i_sb, "Could not find extent\n"); + return -EIO; + } + + BT_MARK_DIRTY(mp, ip); + /* + * acquire tlock of the leaf page containing original entry + */ + if (!test_cflag(COMMIT_Nolink, ip)) { + tlck = txLock(tid, ip, mp, tlckXTREE | tlckGROW); + xtlck = (struct xtlock *) & tlck->lock; + } + + xad = &p->xad[index0]; + xflag = xad->flag; + xoff = offsetXAD(xad); + xlen = lengthXAD(xad); + xaddr = addressXAD(xad); + + /* nXAD must be completely contained within XAD */ + if ((xoff > nxoff) || + (nxoff + nxlen > xoff + xlen)) { + XT_PUTPAGE(mp); + jfs_error(ip->i_sb, + "nXAD in not completely contained within XAD\n"); + return -EIO; + } + + index = index0; + newindex = index + 1; + nextindex = le16_to_cpu(p->header.nextindex); + + if (xoff < nxoff) + goto coalesceRight; + + /* + * coalesce with left XAD + */ + /* is XAD first entry of page ? */ + if (index == XTENTRYSTART) + goto replace; + + /* is nXAD logically and physically contiguous with lXAD ? */ + lxad = &p->xad[index - 1]; + lxlen = lengthXAD(lxad); + if (!(lxad->flag & XAD_NOTRECORDED) && + (nxoff == offsetXAD(lxad) + lxlen) && + (nxaddr == addressXAD(lxad) + lxlen) && + (lxlen + nxlen < MAXXLEN)) { + /* extend right lXAD */ + index0 = index - 1; + XADlength(lxad, lxlen + nxlen); + + /* If we just merged two extents together, need to make sure the + * right extent gets logged. If the left one is marked XAD_NEW, + * then we know it will be logged. Otherwise, mark as + * XAD_EXTENDED + */ + if (!(lxad->flag & XAD_NEW)) + lxad->flag |= XAD_EXTENDED; + + if (xlen > nxlen) { + /* truncate XAD */ + XADoffset(xad, xoff + nxlen); + XADlength(xad, xlen - nxlen); + XADaddress(xad, xaddr + nxlen); + goto out; + } else { /* (xlen == nxlen) */ + + /* remove XAD */ + if (index < nextindex - 1) + memmove(&p->xad[index], &p->xad[index + 1], + (nextindex - index - + 1) << L2XTSLOTSIZE); + + p->header.nextindex = + cpu_to_le16(le16_to_cpu(p->header.nextindex) - + 1); + + index = index0; + newindex = index + 1; + nextindex = le16_to_cpu(p->header.nextindex); + xoff = nxoff = offsetXAD(lxad); + xlen = nxlen = lxlen + nxlen; + xaddr = nxaddr = addressXAD(lxad); + goto coalesceRight; + } + } + + /* + * replace XAD with nXAD + */ + replace: /* (nxoff == xoff) */ + if (nxlen == xlen) { + /* replace XAD with nXAD:recorded */ + *xad = *nxad; + xad->flag = xflag & ~XAD_NOTRECORDED; + + goto coalesceRight; + } else /* (nxlen < xlen) */ + goto updateLeft; + + /* + * coalesce with right XAD + */ + coalesceRight: /* (xoff <= nxoff) */ + /* is XAD last entry of page ? */ + if (newindex == nextindex) { + if (xoff == nxoff) + goto out; + goto updateRight; + } + + /* is nXAD logically and physically contiguous with rXAD ? */ + rxad = &p->xad[index + 1]; + rxlen = lengthXAD(rxad); + if (!(rxad->flag & XAD_NOTRECORDED) && + (nxoff + nxlen == offsetXAD(rxad)) && + (nxaddr + nxlen == addressXAD(rxad)) && + (rxlen + nxlen < MAXXLEN)) { + /* extend left rXAD */ + XADoffset(rxad, nxoff); + XADlength(rxad, rxlen + nxlen); + XADaddress(rxad, nxaddr); + + /* If we just merged two extents together, need to make sure + * the left extent gets logged. If the right one is marked + * XAD_NEW, then we know it will be logged. Otherwise, mark as + * XAD_EXTENDED + */ + if (!(rxad->flag & XAD_NEW)) + rxad->flag |= XAD_EXTENDED; + + if (xlen > nxlen) + /* truncate XAD */ + XADlength(xad, xlen - nxlen); + else { /* (xlen == nxlen) */ + + /* remove XAD */ + memmove(&p->xad[index], &p->xad[index + 1], + (nextindex - index - 1) << L2XTSLOTSIZE); + + p->header.nextindex = + cpu_to_le16(le16_to_cpu(p->header.nextindex) - + 1); + } + + goto out; + } else if (xoff == nxoff) + goto out; + + if (xoff >= nxoff) { + XT_PUTPAGE(mp); + jfs_error(ip->i_sb, "xoff >= nxoff\n"); + return -EIO; + } + + /* + * split XAD into (lXAD, nXAD): + * + * |---nXAD---> + * --|----------XAD----------|-- + * |-lXAD-| + */ + updateRight: /* (xoff < nxoff) */ + /* truncate old XAD as lXAD:not_recorded */ + xad = &p->xad[index]; + XADlength(xad, nxoff - xoff); + + /* insert nXAD:recorded */ + if (nextindex == le16_to_cpu(p->header.maxentry)) { + + /* xtSpliUp() unpins leaf pages */ + split.mp = mp; + split.index = newindex; + split.flag = xflag & ~XAD_NOTRECORDED; + split.off = nxoff; + split.len = nxlen; + split.addr = nxaddr; + split.pxdlist = NULL; + if ((rc = xtSplitUp(tid, ip, &split, &btstack))) + return rc; + + /* get back old page */ + p = xt_getpage(ip, bn, &mp); + if (IS_ERR(p)) + return PTR_ERR(p); + /* + * if leaf root has been split, original root has been + * copied to new child page, i.e., original entry now + * resides on the new child page; + */ + if (p->header.flag & BT_INTERNAL) { + ASSERT(p->header.nextindex == + cpu_to_le16(XTENTRYSTART + 1)); + xad = &p->xad[XTENTRYSTART]; + bn = addressXAD(xad); + XT_PUTPAGE(mp); + + /* get new child page */ + p = xt_getpage(ip, bn, &mp); + if (IS_ERR(p)) + return PTR_ERR(p); + + BT_MARK_DIRTY(mp, ip); + if (!test_cflag(COMMIT_Nolink, ip)) { + tlck = txLock(tid, ip, mp, tlckXTREE|tlckGROW); + xtlck = (struct xtlock *) & tlck->lock; + } + } else { + /* is nXAD on new page ? */ + if (newindex > + (le16_to_cpu(p->header.maxentry) >> 1)) { + newindex = + newindex - + le16_to_cpu(p->header.nextindex) + + XTENTRYSTART; + newpage = 1; + } + } + } else { + /* if insert into middle, shift right remaining entries */ + if (newindex < nextindex) + memmove(&p->xad[newindex + 1], &p->xad[newindex], + (nextindex - newindex) << L2XTSLOTSIZE); + + /* insert the entry */ + xad = &p->xad[newindex]; + *xad = *nxad; + xad->flag = xflag & ~XAD_NOTRECORDED; + + /* advance next available entry index. */ + p->header.nextindex = + cpu_to_le16(le16_to_cpu(p->header.nextindex) + 1); + } + + /* + * does nXAD force 3-way split ? + * + * |---nXAD--->| + * --|----------XAD-------------|-- + * |-lXAD-| |-rXAD -| + */ + if (nxoff + nxlen == xoff + xlen) + goto out; + + /* reorient nXAD as XAD for further split XAD into (nXAD, rXAD) */ + if (newpage) { + /* close out old page */ + if (!test_cflag(COMMIT_Nolink, ip)) { + xtlck->lwm.offset = (xtlck->lwm.offset) ? + min(index0, (int)xtlck->lwm.offset) : index0; + xtlck->lwm.length = + le16_to_cpu(p->header.nextindex) - + xtlck->lwm.offset; + } + + bn = le64_to_cpu(p->header.next); + XT_PUTPAGE(mp); + + /* get new right page */ + p = xt_getpage(ip, bn, &mp); + if (IS_ERR(p)) + return PTR_ERR(p); + + BT_MARK_DIRTY(mp, ip); + if (!test_cflag(COMMIT_Nolink, ip)) { + tlck = txLock(tid, ip, mp, tlckXTREE | tlckGROW); + xtlck = (struct xtlock *) & tlck->lock; + } + + index0 = index = newindex; + } else + index++; + + newindex = index + 1; + nextindex = le16_to_cpu(p->header.nextindex); + xlen = xlen - (nxoff - xoff); + xoff = nxoff; + xaddr = nxaddr; + + /* recompute split pages */ + if (nextindex == le16_to_cpu(p->header.maxentry)) { + XT_PUTPAGE(mp); + + if ((rc = xtSearch(ip, nxoff, NULL, &cmp, &btstack, XT_INSERT))) + return rc; + + /* retrieve search result */ + XT_GETSEARCH(ip, btstack.top, bn, mp, p, index0); + + if (cmp != 0) { + XT_PUTPAGE(mp); + jfs_error(ip->i_sb, "xtSearch failed\n"); + return -EIO; + } + + if (index0 != index) { + XT_PUTPAGE(mp); + jfs_error(ip->i_sb, "unexpected value of index\n"); + return -EIO; + } + } + + /* + * split XAD into (nXAD, rXAD) + * + * ---nXAD---| + * --|----------XAD----------|-- + * |-rXAD-| + */ + updateLeft: /* (nxoff == xoff) && (nxlen < xlen) */ + /* update old XAD with nXAD:recorded */ + xad = &p->xad[index]; + *xad = *nxad; + xad->flag = xflag & ~XAD_NOTRECORDED; + + /* insert rXAD:not_recorded */ + xoff = xoff + nxlen; + xlen = xlen - nxlen; + xaddr = xaddr + nxlen; + if (nextindex == le16_to_cpu(p->header.maxentry)) { +/* +printf("xtUpdate.updateLeft.split p:0x%p\n", p); +*/ + /* xtSpliUp() unpins leaf pages */ + split.mp = mp; + split.index = newindex; + split.flag = xflag; + split.off = xoff; + split.len = xlen; + split.addr = xaddr; + split.pxdlist = NULL; + if ((rc = xtSplitUp(tid, ip, &split, &btstack))) + return rc; + + /* get back old page */ + p = xt_getpage(ip, bn, &mp); + if (IS_ERR(p)) + return PTR_ERR(p); + + /* + * if leaf root has been split, original root has been + * copied to new child page, i.e., original entry now + * resides on the new child page; + */ + if (p->header.flag & BT_INTERNAL) { + ASSERT(p->header.nextindex == + cpu_to_le16(XTENTRYSTART + 1)); + xad = &p->xad[XTENTRYSTART]; + bn = addressXAD(xad); + XT_PUTPAGE(mp); + + /* get new child page */ + p = xt_getpage(ip, bn, &mp); + if (IS_ERR(p)) + return PTR_ERR(p); + + BT_MARK_DIRTY(mp, ip); + if (!test_cflag(COMMIT_Nolink, ip)) { + tlck = txLock(tid, ip, mp, tlckXTREE|tlckGROW); + xtlck = (struct xtlock *) & tlck->lock; + } + } + } else { + /* if insert into middle, shift right remaining entries */ + if (newindex < nextindex) + memmove(&p->xad[newindex + 1], &p->xad[newindex], + (nextindex - newindex) << L2XTSLOTSIZE); + + /* insert the entry */ + xad = &p->xad[newindex]; + XT_PUTENTRY(xad, xflag, xoff, xlen, xaddr); + + /* advance next available entry index. */ + p->header.nextindex = + cpu_to_le16(le16_to_cpu(p->header.nextindex) + 1); + } + + out: + if (!test_cflag(COMMIT_Nolink, ip)) { + xtlck->lwm.offset = (xtlck->lwm.offset) ? + min(index0, (int)xtlck->lwm.offset) : index0; + xtlck->lwm.length = le16_to_cpu(p->header.nextindex) - + xtlck->lwm.offset; + } + + /* unpin the leaf page */ + XT_PUTPAGE(mp); + + return rc; +} + + +/* + * xtAppend() + * + * function: grow in append mode from contiguous region specified ; + * + * parameter: + * tid - transaction id; + * ip - file object; + * xflag - extent flag: + * xoff - extent offset; + * maxblocks - max extent length; + * xlen - extent length (in/out); + * xaddrp - extent address pointer (in/out): + * flag - + * + * return: + */ +int xtAppend(tid_t tid, /* transaction id */ + struct inode *ip, int xflag, s64 xoff, s32 maxblocks, + s32 * xlenp, /* (in/out) */ + s64 * xaddrp, /* (in/out) */ + int flag) +{ + int rc = 0; + struct metapage *mp; /* meta-page buffer */ + xtpage_t *p; /* base B+-tree index page */ + s64 bn, xaddr; + int index, nextindex; + struct btstack btstack; /* traverse stack */ + struct xtsplit split; /* split information */ + xad_t *xad; + int cmp; + struct tlock *tlck; + struct xtlock *xtlck; + int nsplit, nblocks, xlen; + struct pxdlist pxdlist; + pxd_t *pxd; + s64 next; + + xaddr = *xaddrp; + xlen = *xlenp; + jfs_info("xtAppend: xoff:0x%lx maxblocks:%d xlen:%d xaddr:0x%lx", + (ulong) xoff, maxblocks, xlen, (ulong) xaddr); + + /* + * search for the entry location at which to insert: + * + * xtFastSearch() and xtSearch() both returns (leaf page + * pinned, index at which to insert). + * n.b. xtSearch() may return index of maxentry of + * the full page. + */ + if ((rc = xtSearch(ip, xoff, &next, &cmp, &btstack, XT_INSERT))) + return rc; + + /* retrieve search result */ + XT_GETSEARCH(ip, btstack.top, bn, mp, p, index); + + if (cmp == 0) { + rc = -EEXIST; + goto out; + } + + if (next) + xlen = min(xlen, (int)(next - xoff)); +//insert: + /* + * insert entry for new extent + */ + xflag |= XAD_NEW; + + /* + * if the leaf page is full, split the page and + * propagate up the router entry for the new page from split + * + * The xtSplitUp() will insert the entry and unpin the leaf page. + */ + nextindex = le16_to_cpu(p->header.nextindex); + if (nextindex < le16_to_cpu(p->header.maxentry)) + goto insertLeaf; + + /* + * allocate new index blocks to cover index page split(s) + */ + nsplit = btstack.nsplit; + split.pxdlist = &pxdlist; + pxdlist.maxnpxd = pxdlist.npxd = 0; + pxd = &pxdlist.pxd[0]; + nblocks = JFS_SBI(ip->i_sb)->nbperpage; + for (; nsplit > 0; nsplit--, pxd++, xaddr += nblocks, maxblocks -= nblocks) { + if ((rc = dbAllocBottomUp(ip, xaddr, (s64) nblocks)) == 0) { + PXDaddress(pxd, xaddr); + PXDlength(pxd, nblocks); + + pxdlist.maxnpxd++; + + continue; + } + + /* undo allocation */ + + goto out; + } + + xlen = min(xlen, maxblocks); + + /* + * allocate data extent requested + */ + if ((rc = dbAllocBottomUp(ip, xaddr, (s64) xlen))) + goto out; + + split.mp = mp; + split.index = index; + split.flag = xflag; + split.off = xoff; + split.len = xlen; + split.addr = xaddr; + if ((rc = xtSplitUp(tid, ip, &split, &btstack))) { + /* undo data extent allocation */ + dbFree(ip, *xaddrp, (s64) * xlenp); + + return rc; + } + + *xaddrp = xaddr; + *xlenp = xlen; + return 0; + + /* + * insert the new entry into the leaf page + */ + insertLeaf: + /* + * allocate data extent requested + */ + if ((rc = dbAllocBottomUp(ip, xaddr, (s64) xlen))) + goto out; + + BT_MARK_DIRTY(mp, ip); + /* + * acquire a transaction lock on the leaf page; + * + * action: xad insertion/extension; + */ + tlck = txLock(tid, ip, mp, tlckXTREE | tlckGROW); + xtlck = (struct xtlock *) & tlck->lock; + + /* insert the new entry: mark the entry NEW */ + xad = &p->xad[index]; + XT_PUTENTRY(xad, xflag, xoff, xlen, xaddr); + + /* advance next available entry index */ + le16_add_cpu(&p->header.nextindex, 1); + + xtlck->lwm.offset = + (xtlck->lwm.offset) ? min(index,(int) xtlck->lwm.offset) : index; + xtlck->lwm.length = le16_to_cpu(p->header.nextindex) - + xtlck->lwm.offset; + + *xaddrp = xaddr; + *xlenp = xlen; + + out: + /* unpin the leaf page */ + XT_PUTPAGE(mp); + + return rc; +} + +/* + * xtInitRoot() + * + * initialize file root (inline in inode) + */ +void xtInitRoot(tid_t tid, struct inode *ip) +{ + xtroot_t *p; + + /* + * acquire a transaction lock on the root + * + * action: + */ + txLock(tid, ip, (struct metapage *) &JFS_IP(ip)->bxflag, + tlckXTREE | tlckNEW); + p = &JFS_IP(ip)->i_xtroot; + + p->header.flag = DXD_INDEX | BT_ROOT | BT_LEAF; + p->header.nextindex = cpu_to_le16(XTENTRYSTART); + + if (S_ISDIR(ip->i_mode)) + p->header.maxentry = cpu_to_le16(XTROOTINITSLOT_DIR); + else { + p->header.maxentry = cpu_to_le16(XTROOTINITSLOT); + ip->i_size = 0; + } + + + return; +} + + +/* + * We can run into a deadlock truncating a file with a large number of + * xtree pages (large fragmented file). A robust fix would entail a + * reservation system where we would reserve a number of metadata pages + * and tlocks which we would be guaranteed without a deadlock. Without + * this, a partial fix is to limit number of metadata pages we will lock + * in a single transaction. Currently we will truncate the file so that + * no more than 50 leaf pages will be locked. The caller of xtTruncate + * will be responsible for ensuring that the current transaction gets + * committed, and that subsequent transactions are created to truncate + * the file further if needed. + */ +#define MAX_TRUNCATE_LEAVES 50 + +/* + * xtTruncate() + * + * function: + * traverse for truncation logging backward bottom up; + * terminate at the last extent entry at the current subtree + * root page covering new down size. + * truncation may occur within the last extent entry. + * + * parameter: + * int tid, + * struct inode *ip, + * s64 newsize, + * int type) {PWMAP, PMAP, WMAP; DELETE, TRUNCATE} + * + * return: + * + * note: + * PWMAP: + * 1. truncate (non-COMMIT_NOLINK file) + * by jfs_truncate() or jfs_open(O_TRUNC): + * xtree is updated; + * 2. truncate index table of directory when last entry removed + * map update via tlock at commit time; + * PMAP: + * Call xtTruncate_pmap instead + * WMAP: + * 1. remove (free zero link count) on last reference release + * (pmap has been freed at commit zero link count); + * 2. truncate (COMMIT_NOLINK file, i.e., tmp file): + * xtree is updated; + * map update directly at truncation time; + * + * if (DELETE) + * no LOG_NOREDOPAGE is required (NOREDOFILE is sufficient); + * else if (TRUNCATE) + * must write LOG_NOREDOPAGE for deleted index page; + * + * pages may already have been tlocked by anonymous transactions + * during file growth (i.e., write) before truncation; + * + * except last truncated entry, deleted entries remains as is + * in the page (nextindex is updated) for other use + * (e.g., log/update allocation map): this avoid copying the page + * info but delay free of pages; + * + */ +s64 xtTruncate(tid_t tid, struct inode *ip, s64 newsize, int flag) +{ + s64 teof; + struct metapage *mp; + xtpage_t *p; + s64 bn; + int index, nextindex; + xad_t *xad; + s64 xoff, xaddr; + int xlen, len, freexlen; + struct btstack btstack; + struct btframe *parent; + struct tblock *tblk = NULL; + struct tlock *tlck = NULL; + struct xtlock *xtlck = NULL; + struct xdlistlock xadlock; /* maplock for COMMIT_WMAP */ + struct pxd_lock *pxdlock; /* maplock for COMMIT_WMAP */ + s64 nfreed; + int freed, log; + int locked_leaves = 0; + + /* save object truncation type */ + if (tid) { + tblk = tid_to_tblock(tid); + tblk->xflag |= flag; + } + + nfreed = 0; + + flag &= COMMIT_MAP; + assert(flag != COMMIT_PMAP); + + if (flag == COMMIT_PWMAP) + log = 1; + else { + log = 0; + xadlock.flag = mlckFREEXADLIST; + xadlock.index = 1; + } + + /* + * if the newsize is not an integral number of pages, + * the file between newsize and next page boundary will + * be cleared. + * if truncating into a file hole, it will cause + * a full block to be allocated for the logical block. + */ + + /* + * release page blocks of truncated region + * + * free the data blocks from the leaf index blocks. + * delete the parent index entries corresponding to + * the freed child data/index blocks. + * free the index blocks themselves which aren't needed + * in new sized file. + * + * index blocks are updated only if the blocks are to be + * retained in the new sized file. + * if type is PMAP, the data and index pages are NOT + * freed, and the data and index blocks are NOT freed + * from working map. + * (this will allow continued access of data/index of + * temporary file (zerolink count file truncated to zero-length)). + */ + teof = (newsize + (JFS_SBI(ip->i_sb)->bsize - 1)) >> + JFS_SBI(ip->i_sb)->l2bsize; + + /* clear stack */ + BT_CLR(&btstack); + + /* + * start with root + * + * root resides in the inode + */ + bn = 0; + + /* + * first access of each page: + */ + getPage: + p = xt_getpage(ip, bn, &mp); + if (IS_ERR(p)) + return PTR_ERR(p); + + /* process entries backward from last index */ + index = le16_to_cpu(p->header.nextindex) - 1; + + + /* Since this is the rightmost page at this level, and we may have + * already freed a page that was formerly to the right, let's make + * sure that the next pointer is zero. + */ + if (p->header.next) { + if (log) + /* + * Make sure this change to the header is logged. + * If we really truncate this leaf, the flag + * will be changed to tlckTRUNCATE + */ + tlck = txLock(tid, ip, mp, tlckXTREE|tlckGROW); + BT_MARK_DIRTY(mp, ip); + p->header.next = 0; + } + + if (p->header.flag & BT_INTERNAL) + goto getChild; + + /* + * leaf page + */ + freed = 0; + + /* does region covered by leaf page precede Teof ? */ + xad = &p->xad[index]; + xoff = offsetXAD(xad); + xlen = lengthXAD(xad); + if (teof >= xoff + xlen) { + XT_PUTPAGE(mp); + goto getParent; + } + + /* (re)acquire tlock of the leaf page */ + if (log) { + if (++locked_leaves > MAX_TRUNCATE_LEAVES) { + /* + * We need to limit the size of the transaction + * to avoid exhausting pagecache & tlocks + */ + XT_PUTPAGE(mp); + newsize = (xoff + xlen) << JFS_SBI(ip->i_sb)->l2bsize; + goto getParent; + } + tlck = txLock(tid, ip, mp, tlckXTREE); + tlck->type = tlckXTREE | tlckTRUNCATE; + xtlck = (struct xtlock *) & tlck->lock; + xtlck->hwm.offset = le16_to_cpu(p->header.nextindex) - 1; + } + BT_MARK_DIRTY(mp, ip); + + /* + * scan backward leaf page entries + */ + for (; index >= XTENTRYSTART; index--) { + xad = &p->xad[index]; + xoff = offsetXAD(xad); + xlen = lengthXAD(xad); + xaddr = addressXAD(xad); + + /* + * The "data" for a directory is indexed by the block + * device's address space. This metadata must be invalidated + * here + */ + if (S_ISDIR(ip->i_mode) && (teof == 0)) + invalidate_xad_metapages(ip, *xad); + /* + * entry beyond eof: continue scan of current page + * xad + * ---|---=======-------> + * eof + */ + if (teof < xoff) { + nfreed += xlen; + continue; + } + + /* + * (xoff <= teof): last entry to be deleted from page; + * If other entries remain in page: keep and update the page. + */ + + /* + * eof == entry_start: delete the entry + * xad + * -------|=======-------> + * eof + * + */ + if (teof == xoff) { + nfreed += xlen; + + if (index == XTENTRYSTART) + break; + + nextindex = index; + } + /* + * eof within the entry: truncate the entry. + * xad + * -------===|===-------> + * eof + */ + else if (teof < xoff + xlen) { + /* update truncated entry */ + len = teof - xoff; + freexlen = xlen - len; + XADlength(xad, len); + + /* save pxd of truncated extent in tlck */ + xaddr += len; + if (log) { /* COMMIT_PWMAP */ + xtlck->lwm.offset = (xtlck->lwm.offset) ? + min(index, (int)xtlck->lwm.offset) : index; + xtlck->lwm.length = index + 1 - + xtlck->lwm.offset; + xtlck->twm.offset = index; + pxdlock = (struct pxd_lock *) & xtlck->pxdlock; + pxdlock->flag = mlckFREEPXD; + PXDaddress(&pxdlock->pxd, xaddr); + PXDlength(&pxdlock->pxd, freexlen); + } + /* free truncated extent */ + else { /* COMMIT_WMAP */ + + pxdlock = (struct pxd_lock *) & xadlock; + pxdlock->flag = mlckFREEPXD; + PXDaddress(&pxdlock->pxd, xaddr); + PXDlength(&pxdlock->pxd, freexlen); + txFreeMap(ip, pxdlock, NULL, COMMIT_WMAP); + + /* reset map lock */ + xadlock.flag = mlckFREEXADLIST; + } + + /* current entry is new last entry; */ + nextindex = index + 1; + + nfreed += freexlen; + } + /* + * eof beyond the entry: + * xad + * -------=======---|---> + * eof + */ + else { /* (xoff + xlen < teof) */ + + nextindex = index + 1; + } + + if (nextindex < le16_to_cpu(p->header.nextindex)) { + if (!log) { /* COMMIT_WAMP */ + xadlock.xdlist = &p->xad[nextindex]; + xadlock.count = + le16_to_cpu(p->header.nextindex) - + nextindex; + txFreeMap(ip, (struct maplock *) & xadlock, + NULL, COMMIT_WMAP); + } + p->header.nextindex = cpu_to_le16(nextindex); + } + + XT_PUTPAGE(mp); + + /* assert(freed == 0); */ + goto getParent; + } /* end scan of leaf page entries */ + + freed = 1; + + /* + * leaf page become empty: free the page if type != PMAP + */ + if (log) { /* COMMIT_PWMAP */ + /* txCommit() with tlckFREE: + * free data extents covered by leaf [XTENTRYSTART:hwm); + * invalidate leaf if COMMIT_PWMAP; + * if (TRUNCATE), will write LOG_NOREDOPAGE; + */ + tlck->type = tlckXTREE | tlckFREE; + } else { /* COMMIT_WAMP */ + + /* free data extents covered by leaf */ + xadlock.xdlist = &p->xad[XTENTRYSTART]; + xadlock.count = + le16_to_cpu(p->header.nextindex) - XTENTRYSTART; + txFreeMap(ip, (struct maplock *) & xadlock, NULL, COMMIT_WMAP); + } + + if (p->header.flag & BT_ROOT) { + p->header.flag &= ~BT_INTERNAL; + p->header.flag |= BT_LEAF; + p->header.nextindex = cpu_to_le16(XTENTRYSTART); + + XT_PUTPAGE(mp); /* debug */ + goto out; + } else { + if (log) { /* COMMIT_PWMAP */ + /* page will be invalidated at tx completion + */ + XT_PUTPAGE(mp); + } else { /* COMMIT_WMAP */ + + if (mp->lid) + lid_to_tlock(mp->lid)->flag |= tlckFREELOCK; + + /* invalidate empty leaf page */ + discard_metapage(mp); + } + } + + /* + * the leaf page become empty: delete the parent entry + * for the leaf page if the parent page is to be kept + * in the new sized file. + */ + + /* + * go back up to the parent page + */ + getParent: + /* pop/restore parent entry for the current child page */ + if ((parent = BT_POP(&btstack)) == NULL) + /* current page must have been root */ + goto out; + + /* get back the parent page */ + bn = parent->bn; + p = xt_getpage(ip, bn, &mp); + if (IS_ERR(p)) + return PTR_ERR(p); + + index = parent->index; + + /* + * child page was not empty: + */ + if (freed == 0) { + /* has any entry deleted from parent ? */ + if (index < le16_to_cpu(p->header.nextindex) - 1) { + /* (re)acquire tlock on the parent page */ + if (log) { /* COMMIT_PWMAP */ + /* txCommit() with tlckTRUNCATE: + * free child extents covered by parent [); + */ + tlck = txLock(tid, ip, mp, tlckXTREE); + xtlck = (struct xtlock *) & tlck->lock; + if (!(tlck->type & tlckTRUNCATE)) { + xtlck->hwm.offset = + le16_to_cpu(p->header. + nextindex) - 1; + tlck->type = + tlckXTREE | tlckTRUNCATE; + } + } else { /* COMMIT_WMAP */ + + /* free child extents covered by parent */ + xadlock.xdlist = &p->xad[index + 1]; + xadlock.count = + le16_to_cpu(p->header.nextindex) - + index - 1; + txFreeMap(ip, (struct maplock *) & xadlock, + NULL, COMMIT_WMAP); + } + BT_MARK_DIRTY(mp, ip); + + p->header.nextindex = cpu_to_le16(index + 1); + } + XT_PUTPAGE(mp); + goto getParent; + } + + /* + * child page was empty: + */ + nfreed += lengthXAD(&p->xad[index]); + + /* + * During working map update, child page's tlock must be handled + * before parent's. This is because the parent's tlock will cause + * the child's disk space to be marked available in the wmap, so + * it's important that the child page be released by that time. + * + * ToDo: tlocks should be on doubly-linked list, so we can + * quickly remove it and add it to the end. + */ + + /* + * Move parent page's tlock to the end of the tid's tlock list + */ + if (log && mp->lid && (tblk->last != mp->lid) && + lid_to_tlock(mp->lid)->tid) { + lid_t lid = mp->lid; + struct tlock *prev; + + tlck = lid_to_tlock(lid); + + if (tblk->next == lid) + tblk->next = tlck->next; + else { + for (prev = lid_to_tlock(tblk->next); + prev->next != lid; + prev = lid_to_tlock(prev->next)) { + assert(prev->next); + } + prev->next = tlck->next; + } + lid_to_tlock(tblk->last)->next = lid; + tlck->next = 0; + tblk->last = lid; + } + + /* + * parent page become empty: free the page + */ + if (index == XTENTRYSTART) { + if (log) { /* COMMIT_PWMAP */ + /* txCommit() with tlckFREE: + * free child extents covered by parent; + * invalidate parent if COMMIT_PWMAP; + */ + tlck = txLock(tid, ip, mp, tlckXTREE); + xtlck = (struct xtlock *) & tlck->lock; + xtlck->hwm.offset = + le16_to_cpu(p->header.nextindex) - 1; + tlck->type = tlckXTREE | tlckFREE; + } else { /* COMMIT_WMAP */ + + /* free child extents covered by parent */ + xadlock.xdlist = &p->xad[XTENTRYSTART]; + xadlock.count = + le16_to_cpu(p->header.nextindex) - + XTENTRYSTART; + txFreeMap(ip, (struct maplock *) & xadlock, NULL, + COMMIT_WMAP); + } + BT_MARK_DIRTY(mp, ip); + + if (p->header.flag & BT_ROOT) { + p->header.flag &= ~BT_INTERNAL; + p->header.flag |= BT_LEAF; + p->header.nextindex = cpu_to_le16(XTENTRYSTART); + if (le16_to_cpu(p->header.maxentry) == XTROOTMAXSLOT) { + /* + * Shrink root down to allow inline + * EA (otherwise fsck complains) + */ + p->header.maxentry = + cpu_to_le16(XTROOTINITSLOT); + JFS_IP(ip)->mode2 |= INLINEEA; + } + + XT_PUTPAGE(mp); /* debug */ + goto out; + } else { + if (log) { /* COMMIT_PWMAP */ + /* page will be invalidated at tx completion + */ + XT_PUTPAGE(mp); + } else { /* COMMIT_WMAP */ + + if (mp->lid) + lid_to_tlock(mp->lid)->flag |= + tlckFREELOCK; + + /* invalidate parent page */ + discard_metapage(mp); + } + + /* parent has become empty and freed: + * go back up to its parent page + */ + /* freed = 1; */ + goto getParent; + } + } + /* + * parent page still has entries for front region; + */ + else { + /* try truncate region covered by preceding entry + * (process backward) + */ + index--; + + /* go back down to the child page corresponding + * to the entry + */ + goto getChild; + } + + /* + * internal page: go down to child page of current entry + */ + getChild: + /* save current parent entry for the child page */ + if (BT_STACK_FULL(&btstack)) { + jfs_error(ip->i_sb, "stack overrun!\n"); + XT_PUTPAGE(mp); + return -EIO; + } + BT_PUSH(&btstack, bn, index); + + /* get child page */ + xad = &p->xad[index]; + bn = addressXAD(xad); + + /* + * first access of each internal entry: + */ + /* release parent page */ + XT_PUTPAGE(mp); + + /* process the child page */ + goto getPage; + + out: + /* + * update file resource stat + */ + /* set size + */ + if (S_ISDIR(ip->i_mode) && !newsize) + ip->i_size = 1; /* fsck hates zero-length directories */ + else + ip->i_size = newsize; + + /* update quota allocation to reflect freed blocks */ + dquot_free_block(ip, nfreed); + + /* + * free tlock of invalidated pages + */ + if (flag == COMMIT_WMAP) + txFreelock(ip); + + return newsize; +} + + +/* + * xtTruncate_pmap() + * + * function: + * Perform truncate to zero length for deleted file, leaving the + * xtree and working map untouched. This allows the file to + * be accessed via open file handles, while the delete of the file + * is committed to disk. + * + * parameter: + * tid_t tid, + * struct inode *ip, + * s64 committed_size) + * + * return: new committed size + * + * note: + * + * To avoid deadlock by holding too many transaction locks, the + * truncation may be broken up into multiple transactions. + * The committed_size keeps track of part of the file has been + * freed from the pmaps. + */ +s64 xtTruncate_pmap(tid_t tid, struct inode *ip, s64 committed_size) +{ + s64 bn; + struct btstack btstack; + int cmp; + int index; + int locked_leaves = 0; + struct metapage *mp; + xtpage_t *p; + struct btframe *parent; + int rc; + struct tblock *tblk; + struct tlock *tlck = NULL; + xad_t *xad; + int xlen; + s64 xoff; + struct xtlock *xtlck = NULL; + + /* save object truncation type */ + tblk = tid_to_tblock(tid); + tblk->xflag |= COMMIT_PMAP; + + /* clear stack */ + BT_CLR(&btstack); + + if (committed_size) { + xoff = (committed_size >> JFS_SBI(ip->i_sb)->l2bsize) - 1; + rc = xtSearch(ip, xoff, NULL, &cmp, &btstack, 0); + if (rc) + return rc; + + XT_GETSEARCH(ip, btstack.top, bn, mp, p, index); + + if (cmp != 0) { + XT_PUTPAGE(mp); + jfs_error(ip->i_sb, "did not find extent\n"); + return -EIO; + } + } else { + /* + * start with root + * + * root resides in the inode + */ + bn = 0; + + /* + * first access of each page: + */ + getPage: + p = xt_getpage(ip, bn, &mp); + if (IS_ERR(p)) + return PTR_ERR(p); + + /* process entries backward from last index */ + index = le16_to_cpu(p->header.nextindex) - 1; + + if (p->header.flag & BT_INTERNAL) + goto getChild; + } + + /* + * leaf page + */ + + if (++locked_leaves > MAX_TRUNCATE_LEAVES) { + /* + * We need to limit the size of the transaction + * to avoid exhausting pagecache & tlocks + */ + xad = &p->xad[index]; + xoff = offsetXAD(xad); + xlen = lengthXAD(xad); + XT_PUTPAGE(mp); + return (xoff + xlen) << JFS_SBI(ip->i_sb)->l2bsize; + } + tlck = txLock(tid, ip, mp, tlckXTREE); + tlck->type = tlckXTREE | tlckFREE; + xtlck = (struct xtlock *) & tlck->lock; + xtlck->hwm.offset = index; + + + XT_PUTPAGE(mp); + + /* + * go back up to the parent page + */ + getParent: + /* pop/restore parent entry for the current child page */ + if ((parent = BT_POP(&btstack)) == NULL) + /* current page must have been root */ + goto out; + + /* get back the parent page */ + bn = parent->bn; + p = xt_getpage(ip, bn, &mp); + if (IS_ERR(p)) + return PTR_ERR(p); + + index = parent->index; + + /* + * parent page become empty: free the page + */ + if (index == XTENTRYSTART) { + /* txCommit() with tlckFREE: + * free child extents covered by parent; + * invalidate parent if COMMIT_PWMAP; + */ + tlck = txLock(tid, ip, mp, tlckXTREE); + xtlck = (struct xtlock *) & tlck->lock; + xtlck->hwm.offset = le16_to_cpu(p->header.nextindex) - 1; + tlck->type = tlckXTREE | tlckFREE; + + XT_PUTPAGE(mp); + + if (p->header.flag & BT_ROOT) { + + goto out; + } else { + goto getParent; + } + } + /* + * parent page still has entries for front region; + */ + else + index--; + /* + * internal page: go down to child page of current entry + */ + getChild: + /* save current parent entry for the child page */ + if (BT_STACK_FULL(&btstack)) { + jfs_error(ip->i_sb, "stack overrun!\n"); + XT_PUTPAGE(mp); + return -EIO; + } + BT_PUSH(&btstack, bn, index); + + /* get child page */ + xad = &p->xad[index]; + bn = addressXAD(xad); + + /* + * first access of each internal entry: + */ + /* release parent page */ + XT_PUTPAGE(mp); + + /* process the child page */ + goto getPage; + + out: + + return 0; +} + +#ifdef CONFIG_JFS_STATISTICS +int jfs_xtstat_proc_show(struct seq_file *m, void *v) +{ + seq_printf(m, + "JFS Xtree statistics\n" + "====================\n" + "searches = %d\n" + "fast searches = %d\n" + "splits = %d\n", + xtStat.search, + xtStat.fastSearch, + xtStat.split); + return 0; +} +#endif diff --git a/fs/jfs/jfs_xtree.h b/fs/jfs/jfs_xtree.h new file mode 100644 index 000000000..0f6cf5a1c --- /dev/null +++ b/fs/jfs/jfs_xtree.h @@ -0,0 +1,113 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) International Business Machines Corp., 2000-2002 + */ +#ifndef _H_JFS_XTREE +#define _H_JFS_XTREE + +/* + * jfs_xtree.h: extent allocation descriptor B+-tree manager + */ + +#include "jfs_btree.h" + + +/* + * extent allocation descriptor (xad) + */ +typedef struct xad { + __u8 flag; /* 1: flag */ + __u8 rsvrd[2]; /* 2: reserved */ + __u8 off1; /* 1: offset in unit of fsblksize */ + __le32 off2; /* 4: offset in unit of fsblksize */ + pxd_t loc; /* 8: length and address in unit of fsblksize */ +} xad_t; /* (16) */ + +#define MAXXLEN ((1 << 24) - 1) + +#define XTSLOTSIZE 16 +#define L2XTSLOTSIZE 4 + +/* xad_t field construction */ +#define XADoffset(xad, offset64)\ +{\ + (xad)->off1 = ((u64)offset64) >> 32;\ + (xad)->off2 = __cpu_to_le32((offset64) & 0xffffffff);\ +} +#define XADaddress(xad, address64) PXDaddress(&(xad)->loc, address64) +#define XADlength(xad, length32) PXDlength(&(xad)->loc, length32) + +/* xad_t field extraction */ +#define offsetXAD(xad)\ + ( ((s64)((xad)->off1)) << 32 | __le32_to_cpu((xad)->off2)) +#define addressXAD(xad) addressPXD(&(xad)->loc) +#define lengthXAD(xad) lengthPXD(&(xad)->loc) + +/* xad list */ +struct xadlist { + s16 maxnxad; + s16 nxad; + xad_t *xad; +}; + +/* xad_t flags */ +#define XAD_NEW 0x01 /* new */ +#define XAD_EXTENDED 0x02 /* extended */ +#define XAD_COMPRESSED 0x04 /* compressed with recorded length */ +#define XAD_NOTRECORDED 0x08 /* allocated but not recorded */ +#define XAD_COW 0x10 /* copy-on-write */ + + +/* possible values for maxentry */ +#define XTROOTINITSLOT_DIR 6 +#define XTROOTINITSLOT 10 +#define XTROOTMAXSLOT 18 +#define XTPAGEMAXSLOT 256 +#define XTENTRYSTART 2 + +struct xtheader { + __le64 next; /* 8: */ + __le64 prev; /* 8: */ + + u8 flag; /* 1: */ + u8 rsrvd1; /* 1: */ + __le16 nextindex; /* 2: next index = number of entries */ + __le16 maxentry; /* 2: max number of entries */ + __le16 rsrvd2; /* 2: */ + + pxd_t self; /* 8: self */ +}; + +/* + * xtree root (in inode): + */ +typedef union { + struct xtheader header; + xad_t xad[XTROOTMAXSLOT]; /* 16 * maxentry: xad array */ +} xtroot_t; + +/* + * xtree page: + */ +typedef union { + struct xtheader header; + xad_t xad[XTPAGEMAXSLOT]; /* 16 * maxentry: xad array */ +} xtpage_t; + +/* + * external declaration + */ +extern int xtLookup(struct inode *ip, s64 lstart, s64 llen, + int *pflag, s64 * paddr, int *plen, int flag); +extern void xtInitRoot(tid_t tid, struct inode *ip); +extern int xtInsert(tid_t tid, struct inode *ip, + int xflag, s64 xoff, int xlen, s64 * xaddrp, int flag); +extern int xtExtend(tid_t tid, struct inode *ip, s64 xoff, int xlen, + int flag); +extern int xtUpdate(tid_t tid, struct inode *ip, struct xad *nxad); +extern s64 xtTruncate(tid_t tid, struct inode *ip, s64 newsize, int type); +extern s64 xtTruncate_pmap(tid_t tid, struct inode *ip, s64 committed_size); +extern int xtAppend(tid_t tid, + struct inode *ip, int xflag, s64 xoff, int maxblocks, + int *xlenp, s64 * xaddrp, int flag); +#endif /* !_H_JFS_XTREE */ diff --git a/fs/jfs/namei.c b/fs/jfs/namei.c new file mode 100644 index 000000000..8a36c218f --- /dev/null +++ b/fs/jfs/namei.c @@ -0,0 +1,1621 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) International Business Machines Corp., 2000-2004 + * Portions Copyright (C) Christoph Hellwig, 2001-2002 + */ + +#include +#include +#include +#include +#include +#include +#include "jfs_incore.h" +#include "jfs_superblock.h" +#include "jfs_inode.h" +#include "jfs_dinode.h" +#include "jfs_dmap.h" +#include "jfs_unicode.h" +#include "jfs_metapage.h" +#include "jfs_xattr.h" +#include "jfs_acl.h" +#include "jfs_debug.h" + +/* + * forward references + */ +const struct dentry_operations jfs_ci_dentry_operations; + +static s64 commitZeroLink(tid_t, struct inode *); + +/* + * NAME: free_ea_wmap(inode) + * + * FUNCTION: free uncommitted extended attributes from working map + * + */ +static inline void free_ea_wmap(struct inode *inode) +{ + dxd_t *ea = &JFS_IP(inode)->ea; + + if (ea->flag & DXD_EXTENT) { + /* free EA pages from cache */ + invalidate_dxd_metapages(inode, *ea); + dbFree(inode, addressDXD(ea), lengthDXD(ea)); + } + ea->flag = 0; +} + +/* + * NAME: jfs_create(dip, dentry, mode) + * + * FUNCTION: create a regular file in the parent directory + * with name = and mode = + * + * PARAMETER: dip - parent directory vnode + * dentry - dentry of new file + * mode - create mode (rwxrwxrwx). + * nd- nd struct + * + * RETURN: Errors from subroutines + * + */ +static int jfs_create(struct mnt_idmap *idmap, struct inode *dip, + struct dentry *dentry, umode_t mode) +{ + int rc = 0; + tid_t tid; /* transaction id */ + struct inode *ip = NULL; /* child directory inode */ + ino_t ino; + struct component_name dname; /* child directory name */ + struct btstack btstack; + struct inode *iplist[2]; + struct tblock *tblk; + + jfs_info("jfs_create: dip:0x%p name:%pd", dip, dentry); + + rc = dquot_initialize(dip); + if (rc) + goto out1; + + /* + * search parent directory for entry/freespace + * (dtSearch() returns parent directory page pinned) + */ + if ((rc = get_UCSname(&dname, dentry))) + goto out1; + + /* + * Either iAlloc() or txBegin() may block. Deadlock can occur if we + * block there while holding dtree page, so we allocate the inode & + * begin the transaction before we search the directory. + */ + ip = ialloc(dip, mode); + if (IS_ERR(ip)) { + rc = PTR_ERR(ip); + goto out2; + } + + tid = txBegin(dip->i_sb, 0); + + mutex_lock_nested(&JFS_IP(dip)->commit_mutex, COMMIT_MUTEX_PARENT); + mutex_lock_nested(&JFS_IP(ip)->commit_mutex, COMMIT_MUTEX_CHILD); + + rc = jfs_init_acl(tid, ip, dip); + if (rc) + goto out3; + + rc = jfs_init_security(tid, ip, dip, &dentry->d_name); + if (rc) { + txAbort(tid, 0); + goto out3; + } + + if ((rc = dtSearch(dip, &dname, &ino, &btstack, JFS_CREATE))) { + jfs_err("jfs_create: dtSearch returned %d", rc); + txAbort(tid, 0); + goto out3; + } + + tblk = tid_to_tblock(tid); + tblk->xflag |= COMMIT_CREATE; + tblk->ino = ip->i_ino; + tblk->u.ixpxd = JFS_IP(ip)->ixpxd; + + iplist[0] = dip; + iplist[1] = ip; + + /* + * initialize the child XAD tree root in-line in inode + */ + xtInitRoot(tid, ip); + + /* + * create entry in parent directory for child directory + * (dtInsert() releases parent directory page) + */ + ino = ip->i_ino; + if ((rc = dtInsert(tid, dip, &dname, &ino, &btstack))) { + if (rc == -EIO) { + jfs_err("jfs_create: dtInsert returned -EIO"); + txAbort(tid, 1); /* Marks Filesystem dirty */ + } else + txAbort(tid, 0); /* Filesystem full */ + goto out3; + } + + ip->i_op = &jfs_file_inode_operations; + ip->i_fop = &jfs_file_operations; + ip->i_mapping->a_ops = &jfs_aops; + + mark_inode_dirty(ip); + + inode_set_mtime_to_ts(dip, inode_set_ctime_current(dip)); + + mark_inode_dirty(dip); + + rc = txCommit(tid, 2, &iplist[0], 0); + + out3: + txEnd(tid); + mutex_unlock(&JFS_IP(ip)->commit_mutex); + mutex_unlock(&JFS_IP(dip)->commit_mutex); + if (rc) { + free_ea_wmap(ip); + clear_nlink(ip); + discard_new_inode(ip); + } else { + d_instantiate_new(dentry, ip); + } + + out2: + free_UCSname(&dname); + + out1: + + jfs_info("jfs_create: rc:%d", rc); + return rc; +} + + +/* + * NAME: jfs_mkdir(dip, dentry, mode) + * + * FUNCTION: create a child directory in the parent directory + * with name = and mode = + * + * PARAMETER: dip - parent directory vnode + * dentry - dentry of child directory + * mode - create mode (rwxrwxrwx). + * + * RETURN: ERR_PTR() of errors from subroutines. + * + * note: + * EACCES: user needs search+write permission on the parent directory + */ +static struct dentry *jfs_mkdir(struct mnt_idmap *idmap, struct inode *dip, + struct dentry *dentry, umode_t mode) +{ + int rc = 0; + tid_t tid; /* transaction id */ + struct inode *ip = NULL; /* child directory inode */ + ino_t ino; + struct component_name dname; /* child directory name */ + struct btstack btstack; + struct inode *iplist[2]; + struct tblock *tblk; + + jfs_info("jfs_mkdir: dip:0x%p name:%pd", dip, dentry); + + rc = dquot_initialize(dip); + if (rc) + goto out1; + + /* + * search parent directory for entry/freespace + * (dtSearch() returns parent directory page pinned) + */ + if ((rc = get_UCSname(&dname, dentry))) + goto out1; + + /* + * Either iAlloc() or txBegin() may block. Deadlock can occur if we + * block there while holding dtree page, so we allocate the inode & + * begin the transaction before we search the directory. + */ + ip = ialloc(dip, mode); + if (IS_ERR(ip)) { + rc = PTR_ERR(ip); + goto out2; + } + + tid = txBegin(dip->i_sb, 0); + + mutex_lock_nested(&JFS_IP(dip)->commit_mutex, COMMIT_MUTEX_PARENT); + mutex_lock_nested(&JFS_IP(ip)->commit_mutex, COMMIT_MUTEX_CHILD); + + rc = jfs_init_acl(tid, ip, dip); + if (rc) + goto out3; + + rc = jfs_init_security(tid, ip, dip, &dentry->d_name); + if (rc) { + txAbort(tid, 0); + goto out3; + } + + if ((rc = dtSearch(dip, &dname, &ino, &btstack, JFS_CREATE))) { + jfs_err("jfs_mkdir: dtSearch returned %d", rc); + txAbort(tid, 0); + goto out3; + } + + tblk = tid_to_tblock(tid); + tblk->xflag |= COMMIT_CREATE; + tblk->ino = ip->i_ino; + tblk->u.ixpxd = JFS_IP(ip)->ixpxd; + + iplist[0] = dip; + iplist[1] = ip; + + /* + * initialize the child directory in-line in inode + */ + dtInitRoot(tid, ip, dip->i_ino); + + /* + * create entry in parent directory for child directory + * (dtInsert() releases parent directory page) + */ + ino = ip->i_ino; + if ((rc = dtInsert(tid, dip, &dname, &ino, &btstack))) { + if (rc == -EIO) { + jfs_err("jfs_mkdir: dtInsert returned -EIO"); + txAbort(tid, 1); /* Marks Filesystem dirty */ + } else + txAbort(tid, 0); /* Filesystem full */ + goto out3; + } + + set_nlink(ip, 2); /* for '.' */ + ip->i_op = &jfs_dir_inode_operations; + ip->i_fop = &jfs_dir_operations; + + mark_inode_dirty(ip); + + /* update parent directory inode */ + inc_nlink(dip); /* for '..' from child directory */ + inode_set_mtime_to_ts(dip, inode_set_ctime_current(dip)); + mark_inode_dirty(dip); + + rc = txCommit(tid, 2, &iplist[0], 0); + + out3: + txEnd(tid); + mutex_unlock(&JFS_IP(ip)->commit_mutex); + mutex_unlock(&JFS_IP(dip)->commit_mutex); + if (rc) { + free_ea_wmap(ip); + clear_nlink(ip); + discard_new_inode(ip); + } else { + d_instantiate_new(dentry, ip); + } + + out2: + free_UCSname(&dname); + + + out1: + + jfs_info("jfs_mkdir: rc:%d", rc); + return rc ? ERR_PTR(rc) : NULL; +} + +/* + * NAME: jfs_rmdir(dip, dentry) + * + * FUNCTION: remove a link to child directory + * + * PARAMETER: dip - parent inode + * dentry - child directory dentry + * + * RETURN: -EINVAL - if name is . or .. + * -EINVAL - if . or .. exist but are invalid. + * errors from subroutines + * + * note: + * if other threads have the directory open when the last link + * is removed, the "." and ".." entries, if present, are removed before + * rmdir() returns and no new entries may be created in the directory, + * but the directory is not removed until the last reference to + * the directory is released (cf.unlink() of regular file). + */ +static int jfs_rmdir(struct inode *dip, struct dentry *dentry) +{ + int rc; + tid_t tid; /* transaction id */ + struct inode *ip = d_inode(dentry); + ino_t ino; + struct component_name dname; + struct inode *iplist[2]; + struct tblock *tblk; + + jfs_info("jfs_rmdir: dip:0x%p name:%pd", dip, dentry); + + /* Init inode for quota operations. */ + rc = dquot_initialize(dip); + if (rc) + goto out; + rc = dquot_initialize(ip); + if (rc) + goto out; + + /* directory must be empty to be removed */ + if (!dtEmpty(ip)) { + rc = -ENOTEMPTY; + goto out; + } + + if ((rc = get_UCSname(&dname, dentry))) { + goto out; + } + + tid = txBegin(dip->i_sb, 0); + + mutex_lock_nested(&JFS_IP(dip)->commit_mutex, COMMIT_MUTEX_PARENT); + mutex_lock_nested(&JFS_IP(ip)->commit_mutex, COMMIT_MUTEX_CHILD); + + iplist[0] = dip; + iplist[1] = ip; + + tblk = tid_to_tblock(tid); + tblk->xflag |= COMMIT_DELETE; + tblk->u.ip = ip; + + /* + * delete the entry of target directory from parent directory + */ + ino = ip->i_ino; + if ((rc = dtDelete(tid, dip, &dname, &ino, JFS_REMOVE))) { + jfs_err("jfs_rmdir: dtDelete returned %d", rc); + if (rc == -EIO) + txAbort(tid, 1); + txEnd(tid); + mutex_unlock(&JFS_IP(ip)->commit_mutex); + mutex_unlock(&JFS_IP(dip)->commit_mutex); + + goto out2; + } + + /* update parent directory's link count corresponding + * to ".." entry of the target directory deleted + */ + inode_set_mtime_to_ts(dip, inode_set_ctime_current(dip)); + inode_dec_link_count(dip); + + /* + * OS/2 could have created EA and/or ACL + */ + /* free EA from both persistent and working map */ + if (JFS_IP(ip)->ea.flag & DXD_EXTENT) { + /* free EA pages */ + txEA(tid, ip, &JFS_IP(ip)->ea, NULL); + } + JFS_IP(ip)->ea.flag = 0; + + /* free ACL from both persistent and working map */ + if (JFS_IP(ip)->acl.flag & DXD_EXTENT) { + /* free ACL pages */ + txEA(tid, ip, &JFS_IP(ip)->acl, NULL); + } + JFS_IP(ip)->acl.flag = 0; + + /* mark the target directory as deleted */ + clear_nlink(ip); + mark_inode_dirty(ip); + + rc = txCommit(tid, 2, &iplist[0], 0); + + txEnd(tid); + + mutex_unlock(&JFS_IP(ip)->commit_mutex); + mutex_unlock(&JFS_IP(dip)->commit_mutex); + + /* + * Truncating the directory index table is not guaranteed. It + * may need to be done iteratively + */ + if (test_cflag(COMMIT_Stale, dip)) { + if (dip->i_size > 1) + jfs_truncate_nolock(dip, 0); + + clear_cflag(COMMIT_Stale, dip); + } + + out2: + free_UCSname(&dname); + + out: + jfs_info("jfs_rmdir: rc:%d", rc); + return rc; +} + +/* + * NAME: jfs_unlink(dip, dentry) + * + * FUNCTION: remove a link to object named by + * from parent directory + * + * PARAMETER: dip - inode of parent directory + * dentry - dentry of object to be removed + * + * RETURN: errors from subroutines + * + * note: + * temporary file: if one or more processes have the file open + * when the last link is removed, the link will be removed before + * unlink() returns, but the removal of the file contents will be + * postponed until all references to the files are closed. + * + * JFS does NOT support unlink() on directories. + * + */ +static int jfs_unlink(struct inode *dip, struct dentry *dentry) +{ + int rc; + tid_t tid; /* transaction id */ + struct inode *ip = d_inode(dentry); + ino_t ino; + struct component_name dname; /* object name */ + struct inode *iplist[2]; + struct tblock *tblk; + s64 new_size = 0; + int commit_flag; + + jfs_info("jfs_unlink: dip:0x%p name:%pd", dip, dentry); + + /* Init inode for quota operations. */ + rc = dquot_initialize(dip); + if (rc) + goto out; + rc = dquot_initialize(ip); + if (rc) + goto out; + + if ((rc = get_UCSname(&dname, dentry))) + goto out; + + IWRITE_LOCK(ip, RDWRLOCK_NORMAL); + + tid = txBegin(dip->i_sb, 0); + + mutex_lock_nested(&JFS_IP(dip)->commit_mutex, COMMIT_MUTEX_PARENT); + mutex_lock_nested(&JFS_IP(ip)->commit_mutex, COMMIT_MUTEX_CHILD); + + iplist[0] = dip; + iplist[1] = ip; + + /* + * delete the entry of target file from parent directory + */ + ino = ip->i_ino; + if ((rc = dtDelete(tid, dip, &dname, &ino, JFS_REMOVE))) { + jfs_err("jfs_unlink: dtDelete returned %d", rc); + if (rc == -EIO) + txAbort(tid, 1); /* Marks FS Dirty */ + txEnd(tid); + mutex_unlock(&JFS_IP(ip)->commit_mutex); + mutex_unlock(&JFS_IP(dip)->commit_mutex); + IWRITE_UNLOCK(ip); + goto out1; + } + + ASSERT(ip->i_nlink); + + inode_set_mtime_to_ts(dip, + inode_set_ctime_to_ts(dip, inode_set_ctime_current(ip))); + mark_inode_dirty(dip); + + /* update target's inode */ + inode_dec_link_count(ip); + + /* + * commit zero link count object + */ + if (ip->i_nlink == 0) { + assert(!test_cflag(COMMIT_Nolink, ip)); + /* free block resources */ + if ((new_size = commitZeroLink(tid, ip)) < 0) { + txAbort(tid, 1); /* Marks FS Dirty */ + txEnd(tid); + mutex_unlock(&JFS_IP(ip)->commit_mutex); + mutex_unlock(&JFS_IP(dip)->commit_mutex); + IWRITE_UNLOCK(ip); + rc = new_size; + goto out1; + } + tblk = tid_to_tblock(tid); + tblk->xflag |= COMMIT_DELETE; + tblk->u.ip = ip; + } + + /* + * Incomplete truncate of file data can + * result in timing problems unless we synchronously commit the + * transaction. + */ + if (new_size) + commit_flag = COMMIT_SYNC; + else + commit_flag = 0; + + /* + * If xtTruncate was incomplete, commit synchronously to avoid + * timing complications + */ + rc = txCommit(tid, 2, &iplist[0], commit_flag); + + txEnd(tid); + + mutex_unlock(&JFS_IP(ip)->commit_mutex); + mutex_unlock(&JFS_IP(dip)->commit_mutex); + + while (new_size && (rc == 0)) { + tid = txBegin(dip->i_sb, 0); + mutex_lock(&JFS_IP(ip)->commit_mutex); + new_size = xtTruncate_pmap(tid, ip, new_size); + if (new_size < 0) { + txAbort(tid, 1); /* Marks FS Dirty */ + rc = new_size; + } else + rc = txCommit(tid, 2, &iplist[0], COMMIT_SYNC); + txEnd(tid); + mutex_unlock(&JFS_IP(ip)->commit_mutex); + } + + if (ip->i_nlink == 0) + set_cflag(COMMIT_Nolink, ip); + + IWRITE_UNLOCK(ip); + + /* + * Truncating the directory index table is not guaranteed. It + * may need to be done iteratively + */ + if (test_cflag(COMMIT_Stale, dip)) { + if (dip->i_size > 1) + jfs_truncate_nolock(dip, 0); + + clear_cflag(COMMIT_Stale, dip); + } + + out1: + free_UCSname(&dname); + out: + jfs_info("jfs_unlink: rc:%d", rc); + return rc; +} + +/* + * NAME: commitZeroLink() + * + * FUNCTION: for non-directory, called by jfs_remove(), + * truncate a regular file, directory or symbolic + * link to zero length. return 0 if type is not + * one of these. + * + * if the file is currently associated with a VM segment + * only permanent disk and inode map resources are freed, + * and neither the inode nor indirect blocks are modified + * so that the resources can be later freed in the work + * map by ctrunc1. + * if there is no VM segment on entry, the resources are + * freed in both work and permanent map. + * (? for temporary file - memory object is cached even + * after no reference: + * reference count > 0 - ) + * + * PARAMETERS: cd - pointer to commit data structure. + * current inode is the one to truncate. + * + * RETURN: Errors from subroutines + */ +static s64 commitZeroLink(tid_t tid, struct inode *ip) +{ + int filetype; + struct tblock *tblk; + + jfs_info("commitZeroLink: tid = %d, ip = 0x%p", tid, ip); + + filetype = ip->i_mode & S_IFMT; + switch (filetype) { + case S_IFREG: + break; + case S_IFLNK: + /* fast symbolic link */ + if (ip->i_size < IDATASIZE) { + ip->i_size = 0; + return 0; + } + break; + default: + assert(filetype != S_IFDIR); + return 0; + } + + set_cflag(COMMIT_Freewmap, ip); + + /* mark transaction of block map update type */ + tblk = tid_to_tblock(tid); + tblk->xflag |= COMMIT_PMAP; + + /* + * free EA + */ + if (JFS_IP(ip)->ea.flag & DXD_EXTENT) + /* acquire maplock on EA to be freed from block map */ + txEA(tid, ip, &JFS_IP(ip)->ea, NULL); + + /* + * free ACL + */ + if (JFS_IP(ip)->acl.flag & DXD_EXTENT) + /* acquire maplock on EA to be freed from block map */ + txEA(tid, ip, &JFS_IP(ip)->acl, NULL); + + /* + * free xtree/data (truncate to zero length): + * free xtree/data pages from cache if COMMIT_PWMAP, + * free xtree/data blocks from persistent block map, and + * free xtree/data blocks from working block map if COMMIT_PWMAP; + */ + if (ip->i_size) + return xtTruncate_pmap(tid, ip, 0); + + return 0; +} + + +/* + * NAME: jfs_free_zero_link() + * + * FUNCTION: for non-directory, called by iClose(), + * free resources of a file from cache and WORKING map + * for a file previously committed with zero link count + * while associated with a pager object, + * + * PARAMETER: ip - pointer to inode of file. + */ +void jfs_free_zero_link(struct inode *ip) +{ + int type; + + jfs_info("jfs_free_zero_link: ip = 0x%p", ip); + + /* return if not reg or symbolic link or if size is + * already ok. + */ + type = ip->i_mode & S_IFMT; + + switch (type) { + case S_IFREG: + break; + case S_IFLNK: + /* if its contained in inode nothing to do */ + if (ip->i_size < IDATASIZE) + return; + break; + default: + return; + } + + /* + * free EA + */ + if (JFS_IP(ip)->ea.flag & DXD_EXTENT) { + s64 xaddr = addressDXD(&JFS_IP(ip)->ea); + int xlen = lengthDXD(&JFS_IP(ip)->ea); + struct maplock maplock; /* maplock for COMMIT_WMAP */ + struct pxd_lock *pxdlock; /* maplock for COMMIT_WMAP */ + + /* free EA pages from cache */ + invalidate_dxd_metapages(ip, JFS_IP(ip)->ea); + + /* free EA extent from working block map */ + maplock.index = 1; + pxdlock = (struct pxd_lock *) & maplock; + pxdlock->flag = mlckFREEPXD; + PXDaddress(&pxdlock->pxd, xaddr); + PXDlength(&pxdlock->pxd, xlen); + txFreeMap(ip, pxdlock, NULL, COMMIT_WMAP); + } + + /* + * free ACL + */ + if (JFS_IP(ip)->acl.flag & DXD_EXTENT) { + s64 xaddr = addressDXD(&JFS_IP(ip)->acl); + int xlen = lengthDXD(&JFS_IP(ip)->acl); + struct maplock maplock; /* maplock for COMMIT_WMAP */ + struct pxd_lock *pxdlock; /* maplock for COMMIT_WMAP */ + + invalidate_dxd_metapages(ip, JFS_IP(ip)->acl); + + /* free ACL extent from working block map */ + maplock.index = 1; + pxdlock = (struct pxd_lock *) & maplock; + pxdlock->flag = mlckFREEPXD; + PXDaddress(&pxdlock->pxd, xaddr); + PXDlength(&pxdlock->pxd, xlen); + txFreeMap(ip, pxdlock, NULL, COMMIT_WMAP); + } + + /* + * free xtree/data (truncate to zero length): + * free xtree/data pages from cache, and + * free xtree/data blocks from working block map; + */ + if (ip->i_size) + xtTruncate(0, ip, 0, COMMIT_WMAP); +} + +/* + * NAME: jfs_link(vp, dvp, name, crp) + * + * FUNCTION: create a link to by the name = + * in the parent directory + * + * PARAMETER: vp - target object + * dvp - parent directory of new link + * name - name of new link to target object + * crp - credential + * + * RETURN: Errors from subroutines + * + * note: + * JFS does NOT support link() on directories (to prevent circular + * path in the directory hierarchy); + * EPERM: the target object is a directory, and either the caller + * does not have appropriate privileges or the implementation prohibits + * using link() on directories [XPG4.2]. + * + * JFS does NOT support links between file systems: + * EXDEV: target object and new link are on different file systems and + * implementation does not support links between file systems [XPG4.2]. + */ +static int jfs_link(struct dentry *old_dentry, + struct inode *dir, struct dentry *dentry) +{ + int rc; + tid_t tid; + struct inode *ip = d_inode(old_dentry); + ino_t ino; + struct component_name dname; + struct btstack btstack; + struct inode *iplist[2]; + + jfs_info("jfs_link: %pd %pd", old_dentry, dentry); + + rc = dquot_initialize(dir); + if (rc) + goto out; + + if (isReadOnly(ip)) { + jfs_error(ip->i_sb, "read-only filesystem\n"); + return -EROFS; + } + + tid = txBegin(ip->i_sb, 0); + + mutex_lock_nested(&JFS_IP(dir)->commit_mutex, COMMIT_MUTEX_PARENT); + mutex_lock_nested(&JFS_IP(ip)->commit_mutex, COMMIT_MUTEX_CHILD); + + /* + * scan parent directory for entry/freespace + */ + if ((rc = get_UCSname(&dname, dentry))) + goto out_tx; + + if ((rc = dtSearch(dir, &dname, &ino, &btstack, JFS_CREATE))) + goto free_dname; + + /* + * create entry for new link in parent directory + */ + ino = ip->i_ino; + if ((rc = dtInsert(tid, dir, &dname, &ino, &btstack))) + goto free_dname; + + /* update object inode */ + inc_nlink(ip); /* for new link */ + inode_set_ctime_current(ip); + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + mark_inode_dirty(dir); + ihold(ip); + + iplist[0] = ip; + iplist[1] = dir; + rc = txCommit(tid, 2, &iplist[0], 0); + + if (rc) { + drop_nlink(ip); /* never instantiated */ + iput(ip); + } else + d_instantiate(dentry, ip); + + free_dname: + free_UCSname(&dname); + + out_tx: + txEnd(tid); + + mutex_unlock(&JFS_IP(ip)->commit_mutex); + mutex_unlock(&JFS_IP(dir)->commit_mutex); + + out: + jfs_info("jfs_link: rc:%d", rc); + return rc; +} + +/* + * NAME: jfs_symlink(dip, dentry, name) + * + * FUNCTION: creates a symbolic link to by name + * in directory + * + * PARAMETER: dip - parent directory vnode + * dentry - dentry of symbolic link + * name - the path name of the existing object + * that will be the source of the link + * + * RETURN: errors from subroutines + * + * note: + * ENAMETOOLONG: pathname resolution of a symbolic link produced + * an intermediate result whose length exceeds PATH_MAX [XPG4.2] +*/ + +static int jfs_symlink(struct mnt_idmap *idmap, struct inode *dip, + struct dentry *dentry, const char *name) +{ + int rc; + tid_t tid; + ino_t ino = 0; + struct component_name dname; + u32 ssize; /* source pathname size */ + struct btstack btstack; + struct inode *ip; + s64 xlen = 0; + int bmask = 0, xsize; + s64 xaddr; + struct metapage *mp; + struct super_block *sb; + struct tblock *tblk; + + struct inode *iplist[2]; + + jfs_info("jfs_symlink: dip:0x%p name:%s", dip, name); + + rc = dquot_initialize(dip); + if (rc) + goto out1; + + ssize = strlen(name) + 1; + + /* + * search parent directory for entry/freespace + * (dtSearch() returns parent directory page pinned) + */ + + if ((rc = get_UCSname(&dname, dentry))) + goto out1; + + /* + * allocate on-disk/in-memory inode for symbolic link: + * (iAlloc() returns new, locked inode) + */ + ip = ialloc(dip, S_IFLNK | 0777); + if (IS_ERR(ip)) { + rc = PTR_ERR(ip); + goto out2; + } + + tid = txBegin(dip->i_sb, 0); + + mutex_lock_nested(&JFS_IP(dip)->commit_mutex, COMMIT_MUTEX_PARENT); + mutex_lock_nested(&JFS_IP(ip)->commit_mutex, COMMIT_MUTEX_CHILD); + + rc = jfs_init_security(tid, ip, dip, &dentry->d_name); + if (rc) + goto out3; + + tblk = tid_to_tblock(tid); + tblk->xflag |= COMMIT_CREATE; + tblk->ino = ip->i_ino; + tblk->u.ixpxd = JFS_IP(ip)->ixpxd; + + /* fix symlink access permission + * (dir_create() ANDs in the u.u_cmask, + * but symlinks really need to be 777 access) + */ + ip->i_mode |= 0777; + + /* + * write symbolic link target path name + */ + xtInitRoot(tid, ip); + + /* + * write source path name inline in on-disk inode (fast symbolic link) + */ + + if (ssize <= IDATASIZE) { + ip->i_op = &jfs_fast_symlink_inode_operations; + + ip->i_link = JFS_IP(ip)->i_inline_all; + memcpy(ip->i_link, name, ssize); + ip->i_size = ssize - 1; + + /* + * if symlink is > 128 bytes, we don't have the space to + * store inline extended attributes + */ + if (ssize > sizeof (JFS_IP(ip)->i_inline)) + JFS_IP(ip)->mode2 &= ~INLINEEA; + + jfs_info("jfs_symlink: fast symlink added ssize:%u name:%s ", + ssize, name); + } + /* + * write source path name in a single extent + */ + else { + jfs_info("jfs_symlink: allocate extent ip:0x%p", ip); + + ip->i_op = &jfs_symlink_inode_operations; + inode_nohighmem(ip); + ip->i_mapping->a_ops = &jfs_aops; + + /* + * even though the data of symlink object (source + * path name) is treated as non-journaled user data, + * it is read/written thru buffer cache for performance. + */ + sb = ip->i_sb; + bmask = JFS_SBI(sb)->bsize - 1; + xsize = (ssize + bmask) & ~bmask; + xaddr = 0; + xlen = xsize >> JFS_SBI(sb)->l2bsize; + if ((rc = xtInsert(tid, ip, 0, 0, xlen, &xaddr, 0))) { + txAbort(tid, 0); + goto out3; + } + ip->i_size = ssize - 1; + while (ssize) { + /* This is kind of silly since PATH_MAX == 4K */ + u32 copy_size = min_t(u32, ssize, PSIZE); + + mp = get_metapage(ip, xaddr, PSIZE, 1); + + if (mp == NULL) { + xtTruncate(tid, ip, 0, COMMIT_PWMAP); + rc = -EIO; + txAbort(tid, 0); + goto out3; + } + memcpy(mp->data, name, copy_size); + flush_metapage(mp); + ssize -= copy_size; + name += copy_size; + xaddr += JFS_SBI(sb)->nbperpage; + } + } + + /* + * create entry for symbolic link in parent directory + */ + rc = dtSearch(dip, &dname, &ino, &btstack, JFS_CREATE); + if (rc == 0) { + ino = ip->i_ino; + rc = dtInsert(tid, dip, &dname, &ino, &btstack); + } + if (rc) { + if (xlen) + xtTruncate(tid, ip, 0, COMMIT_PWMAP); + txAbort(tid, 0); + /* discard new inode */ + goto out3; + } + + mark_inode_dirty(ip); + + inode_set_mtime_to_ts(dip, inode_set_ctime_current(dip)); + mark_inode_dirty(dip); + /* + * commit update of parent directory and link object + */ + + iplist[0] = dip; + iplist[1] = ip; + rc = txCommit(tid, 2, &iplist[0], 0); + + out3: + txEnd(tid); + mutex_unlock(&JFS_IP(ip)->commit_mutex); + mutex_unlock(&JFS_IP(dip)->commit_mutex); + if (rc) { + free_ea_wmap(ip); + clear_nlink(ip); + discard_new_inode(ip); + } else { + d_instantiate_new(dentry, ip); + } + + out2: + free_UCSname(&dname); + + out1: + jfs_info("jfs_symlink: rc:%d", rc); + return rc; +} + + +/* + * NAME: jfs_rename + * + * FUNCTION: rename a file or directory + */ +static int jfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) +{ + struct btstack btstack; + ino_t ino; + struct component_name new_dname; + struct inode *new_ip; + struct component_name old_dname; + struct inode *old_ip; + int rc; + tid_t tid; + struct tlock *tlck; + struct dt_lock *dtlck; + struct lv *lv; + int ipcount; + struct inode *iplist[4]; + struct tblock *tblk; + s64 new_size = 0; + int commit_flag; + + if (flags & ~RENAME_NOREPLACE) + return -EINVAL; + + jfs_info("jfs_rename: %pd %pd", old_dentry, new_dentry); + + rc = dquot_initialize(old_dir); + if (rc) + goto out1; + rc = dquot_initialize(new_dir); + if (rc) + goto out1; + + old_ip = d_inode(old_dentry); + new_ip = d_inode(new_dentry); + + if ((rc = get_UCSname(&old_dname, old_dentry))) + goto out1; + + if ((rc = get_UCSname(&new_dname, new_dentry))) + goto out2; + + /* + * Make sure source inode number is what we think it is + */ + rc = dtSearch(old_dir, &old_dname, &ino, &btstack, JFS_LOOKUP); + if (rc || (ino != old_ip->i_ino)) { + rc = -ENOENT; + goto out3; + } + + /* + * Make sure dest inode number (if any) is what we think it is + */ + rc = dtSearch(new_dir, &new_dname, &ino, &btstack, JFS_LOOKUP); + if (!rc) { + if ((!new_ip) || (ino != new_ip->i_ino)) { + rc = -ESTALE; + goto out3; + } + } else if (rc != -ENOENT) + goto out3; + else if (new_ip) { + /* no entry exists, but one was expected */ + rc = -ESTALE; + goto out3; + } + + if (S_ISDIR(old_ip->i_mode)) { + if (new_ip) { + if (!dtEmpty(new_ip)) { + rc = -ENOTEMPTY; + goto out3; + } + } + } else if (new_ip) { + IWRITE_LOCK(new_ip, RDWRLOCK_NORMAL); + /* Init inode for quota operations. */ + rc = dquot_initialize(new_ip); + if (rc) + goto out_unlock; + } + + /* + * The real work starts here + */ + tid = txBegin(new_dir->i_sb, 0); + + /* + * How do we know the locking is safe from deadlocks? + * The vfs does the hard part for us. Any time we are taking nested + * commit_mutexes, the vfs already has i_mutex held on the parent. + * Here, the vfs has already taken i_mutex on both old_dir and new_dir. + */ + mutex_lock_nested(&JFS_IP(new_dir)->commit_mutex, COMMIT_MUTEX_PARENT); + mutex_lock_nested(&JFS_IP(old_ip)->commit_mutex, COMMIT_MUTEX_CHILD); + if (old_dir != new_dir) + mutex_lock_nested(&JFS_IP(old_dir)->commit_mutex, + COMMIT_MUTEX_SECOND_PARENT); + + if (new_ip) { + mutex_lock_nested(&JFS_IP(new_ip)->commit_mutex, + COMMIT_MUTEX_VICTIM); + /* + * Change existing directory entry to new inode number + */ + ino = new_ip->i_ino; + rc = dtModify(tid, new_dir, &new_dname, &ino, + old_ip->i_ino, JFS_RENAME); + if (rc) + goto out_tx; + drop_nlink(new_ip); + if (S_ISDIR(new_ip->i_mode)) { + drop_nlink(new_ip); + if (new_ip->i_nlink) { + mutex_unlock(&JFS_IP(new_ip)->commit_mutex); + if (old_dir != new_dir) + mutex_unlock(&JFS_IP(old_dir)->commit_mutex); + mutex_unlock(&JFS_IP(old_ip)->commit_mutex); + mutex_unlock(&JFS_IP(new_dir)->commit_mutex); + if (!S_ISDIR(old_ip->i_mode) && new_ip) + IWRITE_UNLOCK(new_ip); + jfs_error(new_ip->i_sb, + "new_ip->i_nlink != 0\n"); + return -EIO; + } + tblk = tid_to_tblock(tid); + tblk->xflag |= COMMIT_DELETE; + tblk->u.ip = new_ip; + } else if (new_ip->i_nlink == 0) { + assert(!test_cflag(COMMIT_Nolink, new_ip)); + /* free block resources */ + if ((new_size = commitZeroLink(tid, new_ip)) < 0) { + txAbort(tid, 1); /* Marks FS Dirty */ + rc = new_size; + goto out_tx; + } + tblk = tid_to_tblock(tid); + tblk->xflag |= COMMIT_DELETE; + tblk->u.ip = new_ip; + } else { + inode_set_ctime_current(new_ip); + mark_inode_dirty(new_ip); + } + } else { + /* + * Add new directory entry + */ + rc = dtSearch(new_dir, &new_dname, &ino, &btstack, + JFS_CREATE); + if (rc) { + jfs_err("jfs_rename didn't expect dtSearch to fail w/rc = %d", + rc); + goto out_tx; + } + + ino = old_ip->i_ino; + rc = dtInsert(tid, new_dir, &new_dname, &ino, &btstack); + if (rc) { + if (rc == -EIO) + jfs_err("jfs_rename: dtInsert returned -EIO"); + goto out_tx; + } + if (S_ISDIR(old_ip->i_mode) && old_dir != new_dir) + inc_nlink(new_dir); + } + /* + * Remove old directory entry + */ + + ino = old_ip->i_ino; + rc = dtDelete(tid, old_dir, &old_dname, &ino, JFS_REMOVE); + if (rc) { + jfs_err("jfs_rename did not expect dtDelete to return rc = %d", + rc); + txAbort(tid, 1); /* Marks Filesystem dirty */ + goto out_tx; + } + if (S_ISDIR(old_ip->i_mode)) { + if (new_ip || old_dir != new_dir) + drop_nlink(old_dir); + + if (old_dir != new_dir) { + /* + * Change inode number of parent for moved directory + */ + + JFS_IP(old_ip)->i_dtroot.header.idotdot = + cpu_to_le32(new_dir->i_ino); + + /* Linelock header of dtree */ + tlck = txLock(tid, old_ip, + (struct metapage *) &JFS_IP(old_ip)->bxflag, + tlckDTREE | tlckBTROOT | tlckRELINK); + dtlck = (struct dt_lock *) & tlck->lock; + ASSERT(dtlck->index == 0); + lv = & dtlck->lv[0]; + lv->offset = 0; + lv->length = 1; + dtlck->index++; + } + } + + /* + * Update ctime on changed/moved inodes & mark dirty + */ + inode_set_ctime_current(old_ip); + mark_inode_dirty(old_ip); + + inode_set_mtime_to_ts(new_dir, inode_set_ctime_current(new_dir)); + mark_inode_dirty(new_dir); + + /* Build list of inodes modified by this transaction */ + ipcount = 0; + iplist[ipcount++] = old_ip; + if (new_ip) + iplist[ipcount++] = new_ip; + iplist[ipcount++] = old_dir; + + if (old_dir != new_dir) { + iplist[ipcount++] = new_dir; + inode_set_mtime_to_ts(old_dir, + inode_set_ctime_current(old_dir)); + mark_inode_dirty(old_dir); + } + + /* + * Incomplete truncate of file data can + * result in timing problems unless we synchronously commit the + * transaction. + */ + if (new_size) + commit_flag = COMMIT_SYNC; + else + commit_flag = 0; + + rc = txCommit(tid, ipcount, iplist, commit_flag); + + out_tx: + txEnd(tid); + if (new_ip) + mutex_unlock(&JFS_IP(new_ip)->commit_mutex); + if (old_dir != new_dir) + mutex_unlock(&JFS_IP(old_dir)->commit_mutex); + mutex_unlock(&JFS_IP(old_ip)->commit_mutex); + mutex_unlock(&JFS_IP(new_dir)->commit_mutex); + + while (new_size && (rc == 0)) { + tid = txBegin(new_ip->i_sb, 0); + mutex_lock(&JFS_IP(new_ip)->commit_mutex); + new_size = xtTruncate_pmap(tid, new_ip, new_size); + if (new_size < 0) { + txAbort(tid, 1); + rc = new_size; + } else + rc = txCommit(tid, 1, &new_ip, COMMIT_SYNC); + txEnd(tid); + mutex_unlock(&JFS_IP(new_ip)->commit_mutex); + } + if (new_ip && (new_ip->i_nlink == 0)) + set_cflag(COMMIT_Nolink, new_ip); + /* + * Truncating the directory index table is not guaranteed. It + * may need to be done iteratively + */ + if (test_cflag(COMMIT_Stale, old_dir)) { + if (old_dir->i_size > 1) + jfs_truncate_nolock(old_dir, 0); + + clear_cflag(COMMIT_Stale, old_dir); + } + out_unlock: + if (new_ip && !S_ISDIR(new_ip->i_mode)) + IWRITE_UNLOCK(new_ip); + out3: + free_UCSname(&new_dname); + out2: + free_UCSname(&old_dname); + out1: + jfs_info("jfs_rename: returning %d", rc); + return rc; +} + + +/* + * NAME: jfs_mknod + * + * FUNCTION: Create a special file (device) + */ +static int jfs_mknod(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) +{ + struct jfs_inode_info *jfs_ip; + struct btstack btstack; + struct component_name dname; + ino_t ino; + struct inode *ip; + struct inode *iplist[2]; + int rc; + tid_t tid; + struct tblock *tblk; + + jfs_info("jfs_mknod: %pd", dentry); + + rc = dquot_initialize(dir); + if (rc) + goto out; + + if ((rc = get_UCSname(&dname, dentry))) + goto out; + + ip = ialloc(dir, mode); + if (IS_ERR(ip)) { + rc = PTR_ERR(ip); + goto out1; + } + jfs_ip = JFS_IP(ip); + + tid = txBegin(dir->i_sb, 0); + + mutex_lock_nested(&JFS_IP(dir)->commit_mutex, COMMIT_MUTEX_PARENT); + mutex_lock_nested(&JFS_IP(ip)->commit_mutex, COMMIT_MUTEX_CHILD); + + rc = jfs_init_acl(tid, ip, dir); + if (rc) + goto out3; + + rc = jfs_init_security(tid, ip, dir, &dentry->d_name); + if (rc) { + txAbort(tid, 0); + goto out3; + } + + if ((rc = dtSearch(dir, &dname, &ino, &btstack, JFS_CREATE))) { + txAbort(tid, 0); + goto out3; + } + + tblk = tid_to_tblock(tid); + tblk->xflag |= COMMIT_CREATE; + tblk->ino = ip->i_ino; + tblk->u.ixpxd = JFS_IP(ip)->ixpxd; + + ino = ip->i_ino; + if ((rc = dtInsert(tid, dir, &dname, &ino, &btstack))) { + txAbort(tid, 0); + goto out3; + } + + ip->i_op = &jfs_file_inode_operations; + jfs_ip->dev = new_encode_dev(rdev); + init_special_inode(ip, ip->i_mode, rdev); + + mark_inode_dirty(ip); + + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + + mark_inode_dirty(dir); + + iplist[0] = dir; + iplist[1] = ip; + rc = txCommit(tid, 2, iplist, 0); + + out3: + txEnd(tid); + mutex_unlock(&JFS_IP(ip)->commit_mutex); + mutex_unlock(&JFS_IP(dir)->commit_mutex); + if (rc) { + free_ea_wmap(ip); + clear_nlink(ip); + discard_new_inode(ip); + } else { + d_instantiate_new(dentry, ip); + } + + out1: + free_UCSname(&dname); + + out: + jfs_info("jfs_mknod: returning %d", rc); + return rc; +} + +static struct dentry *jfs_lookup(struct inode *dip, struct dentry *dentry, unsigned int flags) +{ + struct btstack btstack; + ino_t inum; + struct inode *ip; + struct component_name key; + int rc; + + jfs_info("jfs_lookup: name = %pd", dentry); + + if ((rc = get_UCSname(&key, dentry))) + return ERR_PTR(rc); + rc = dtSearch(dip, &key, &inum, &btstack, JFS_LOOKUP); + free_UCSname(&key); + if (rc == -ENOENT) { + ip = NULL; + } else if (rc) { + jfs_err("jfs_lookup: dtSearch returned %d", rc); + ip = ERR_PTR(rc); + } else { + ip = jfs_iget(dip->i_sb, inum); + if (IS_ERR(ip)) + jfs_err("jfs_lookup: iget failed on inum %d", (uint)inum); + } + + return d_splice_alias(ip, dentry); +} + +static struct inode *jfs_nfs_get_inode(struct super_block *sb, + u64 ino, u32 generation) +{ + struct inode *inode; + + if (ino == 0) + return ERR_PTR(-ESTALE); + inode = jfs_iget(sb, ino); + if (IS_ERR(inode)) + return ERR_CAST(inode); + + if (generation && inode->i_generation != generation) { + iput(inode); + return ERR_PTR(-ESTALE); + } + + return inode; +} + +struct dentry *jfs_fh_to_dentry(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + return generic_fh_to_dentry(sb, fid, fh_len, fh_type, + jfs_nfs_get_inode); +} + +struct dentry *jfs_fh_to_parent(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + return generic_fh_to_parent(sb, fid, fh_len, fh_type, + jfs_nfs_get_inode); +} + +struct dentry *jfs_get_parent(struct dentry *dentry) +{ + unsigned long parent_ino; + + parent_ino = + le32_to_cpu(JFS_IP(d_inode(dentry))->i_dtroot.header.idotdot); + + return d_obtain_alias(jfs_iget(dentry->d_sb, parent_ino)); +} + +const struct inode_operations jfs_dir_inode_operations = { + .create = jfs_create, + .lookup = jfs_lookup, + .link = jfs_link, + .unlink = jfs_unlink, + .symlink = jfs_symlink, + .mkdir = jfs_mkdir, + .rmdir = jfs_rmdir, + .mknod = jfs_mknod, + .rename = jfs_rename, + .listxattr = jfs_listxattr, + .setattr = jfs_setattr, + .fileattr_get = jfs_fileattr_get, + .fileattr_set = jfs_fileattr_set, +#ifdef CONFIG_JFS_POSIX_ACL + .get_inode_acl = jfs_get_acl, + .set_acl = jfs_set_acl, +#endif +}; + +WRAP_DIR_ITER(jfs_readdir) // FIXME! +const struct file_operations jfs_dir_operations = { + .read = generic_read_dir, + .iterate_shared = shared_jfs_readdir, + .fsync = jfs_fsync, + .unlocked_ioctl = jfs_ioctl, + .compat_ioctl = compat_ptr_ioctl, + .llseek = generic_file_llseek, + .setlease = generic_setlease, +}; + +static int jfs_ci_hash(const struct dentry *dir, struct qstr *this) +{ + unsigned long hash; + int i; + + hash = init_name_hash(dir); + for (i=0; i < this->len; i++) + hash = partial_name_hash(tolower(this->name[i]), hash); + this->hash = end_name_hash(hash); + + return 0; +} + +static int jfs_ci_compare(const struct dentry *dentry, + unsigned int len, const char *str, const struct qstr *name) +{ + int i, result = 1; + + if (len != name->len) + goto out; + for (i=0; i < len; i++) { + if (tolower(str[i]) != tolower(name->name[i])) + goto out; + } + result = 0; +out: + return result; +} + +static int jfs_ci_revalidate(struct inode *dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + /* + * This is not negative dentry. Always valid. + * + * Note, rename() to existing directory entry will have ->d_inode, + * and will use existing name which isn't specified name by user. + * + * We may be able to drop this positive dentry here. But dropping + * positive dentry isn't good idea. So it's unsupported like + * rename("filename", "FILENAME") for now. + */ + if (d_really_is_positive(dentry)) + return 1; + + /* + * This may be nfsd (or something), anyway, we can't see the + * intent of this. So, since this can be for creation, drop it. + */ + if (!flags) + return 0; + + /* + * Drop the negative dentry, in order to make sure to use the + * case sensitive name which is specified by user if this is + * for creation. + */ + if (flags & (LOOKUP_CREATE | LOOKUP_RENAME_TARGET)) + return 0; + return 1; +} + +const struct dentry_operations jfs_ci_dentry_operations = +{ + .d_hash = jfs_ci_hash, + .d_compare = jfs_ci_compare, + .d_revalidate = jfs_ci_revalidate, +}; diff --git a/fs/jfs/resize.c b/fs/jfs/resize.c new file mode 100644 index 000000000..8b9a72ae5 --- /dev/null +++ b/fs/jfs/resize.c @@ -0,0 +1,536 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) International Business Machines Corp., 2000-2004 +*/ + +#include +#include +#include +#include +#include "jfs_incore.h" +#include "jfs_filsys.h" +#include "jfs_metapage.h" +#include "jfs_dinode.h" +#include "jfs_imap.h" +#include "jfs_dmap.h" +#include "jfs_superblock.h" +#include "jfs_txnmgr.h" +#include "jfs_debug.h" + +#define BITSPERPAGE (PSIZE << 3) +#define L2MEGABYTE 20 +#define MEGABYTE (1 << L2MEGABYTE) +#define MEGABYTE32 (MEGABYTE << 5) + +/* convert block number to bmap file page number */ +#define BLKTODMAPN(b)\ + (((b) >> 13) + ((b) >> 23) + ((b) >> 33) + 3 + 1) + +/* + * jfs_extendfs() + * + * function: extend file system; + * + * |-------------------------------|----------|----------| + * file system space fsck inline log + * workspace space + * + * input: + * new LVSize: in LV blocks (required) + * new LogSize: in LV blocks (optional) + * new FSSize: in LV blocks (optional) + * + * new configuration: + * 1. set new LogSize as specified or default from new LVSize; + * 2. compute new FSCKSize from new LVSize; + * 3. set new FSSize as MIN(FSSize, LVSize-(LogSize+FSCKSize)) where + * assert(new FSSize >= old FSSize), + * i.e., file system must not be shrunk; + */ +int jfs_extendfs(struct super_block *sb, s64 newLVSize, int newLogSize) +{ + int rc = 0; + struct jfs_sb_info *sbi = JFS_SBI(sb); + struct inode *ipbmap = sbi->ipbmap; + struct inode *ipbmap2; + struct inode *ipimap = sbi->ipimap; + struct jfs_log *log = sbi->log; + struct bmap *bmp = sbi->bmap; + s64 newLogAddress, newFSCKAddress; + int newFSCKSize; + s64 newMapSize = 0, mapSize; + s64 XAddress, XSize, nblocks, xoff, xaddr, t64; + s64 oldLVSize; + s64 newFSSize; + s64 VolumeSize; + int newNpages = 0, nPages, newPage, xlen, t32; + int tid; + int log_formatted = 0; + struct inode *iplist[1]; + struct jfs_superblock *j_sb, *j_sb2; + s64 old_agsize; + int agsizechanged = 0; + struct buffer_head *bh, *bh2; + + /* If the volume hasn't grown, get out now */ + + if (sbi->mntflag & JFS_INLINELOG) + oldLVSize = addressPXD(&sbi->logpxd) + lengthPXD(&sbi->logpxd); + else + oldLVSize = addressPXD(&sbi->fsckpxd) + + lengthPXD(&sbi->fsckpxd); + + if (oldLVSize >= newLVSize) { + printk(KERN_WARNING + "jfs_extendfs: volume hasn't grown, returning\n"); + goto out; + } + + VolumeSize = sb_bdev_nr_blocks(sb); + if (VolumeSize) { + if (newLVSize > VolumeSize) { + printk(KERN_WARNING "jfs_extendfs: invalid size\n"); + rc = -EINVAL; + goto out; + } + } else { + /* check the device */ + bh = sb_bread(sb, newLVSize - 1); + if (!bh) { + printk(KERN_WARNING "jfs_extendfs: invalid size\n"); + rc = -EINVAL; + goto out; + } + bforget(bh); + } + + /* Can't extend write-protected drive */ + + if (isReadOnly(ipbmap)) { + printk(KERN_WARNING "jfs_extendfs: read-only file system\n"); + rc = -EROFS; + goto out; + } + + /* + * reconfigure LV spaces + * --------------------- + * + * validate new size, or, if not specified, determine new size + */ + + /* + * reconfigure inline log space: + */ + if ((sbi->mntflag & JFS_INLINELOG)) { + if (newLogSize == 0) { + /* + * no size specified: default to 1/256 of aggregate + * size; rounded up to a megabyte boundary; + */ + newLogSize = newLVSize >> 8; + t32 = (1 << (20 - sbi->l2bsize)) - 1; + newLogSize = (newLogSize + t32) & ~t32; + newLogSize = + min(newLogSize, MEGABYTE32 >> sbi->l2bsize); + } else { + /* + * convert the newLogSize to fs blocks. + * + * Since this is given in megabytes, it will always be + * an even number of pages. + */ + newLogSize = (newLogSize * MEGABYTE) >> sbi->l2bsize; + } + + } else + newLogSize = 0; + + newLogAddress = newLVSize - newLogSize; + + /* + * reconfigure fsck work space: + * + * configure it to the end of the logical volume regardless of + * whether file system extends to the end of the aggregate; + * Need enough 4k pages to cover: + * - 1 bit per block in aggregate rounded up to BPERDMAP boundary + * - 1 extra page to handle control page and intermediate level pages + * - 50 extra pages for the chkdsk service log + */ + t64 = ((newLVSize - newLogSize + BPERDMAP - 1) >> L2BPERDMAP) + << L2BPERDMAP; + t32 = DIV_ROUND_UP(t64, BITSPERPAGE) + 1 + 50; + newFSCKSize = t32 << sbi->l2nbperpage; + newFSCKAddress = newLogAddress - newFSCKSize; + + /* + * compute new file system space; + */ + newFSSize = newLVSize - newLogSize - newFSCKSize; + + /* file system cannot be shrunk */ + if (newFSSize < bmp->db_mapsize) { + rc = -EINVAL; + goto out; + } + + /* + * If we're expanding enough that the inline log does not overlap + * the old one, we can format the new log before we quiesce the + * filesystem. + */ + if ((sbi->mntflag & JFS_INLINELOG) && (newLogAddress > oldLVSize)) { + if ((rc = lmLogFormat(log, newLogAddress, newLogSize))) + goto out; + log_formatted = 1; + } + /* + * quiesce file system + * + * (prepare to move the inline log and to prevent map update) + * + * block any new transactions and wait for completion of + * all wip transactions and flush modified pages s.t. + * on-disk file system is in consistent state and + * log is not required for recovery. + */ + txQuiesce(sb); + + /* Reset size of direct inode */ + sbi->direct_inode->i_size = bdev_nr_bytes(sb->s_bdev); + + if (sbi->mntflag & JFS_INLINELOG) { + /* + * deactivate old inline log + */ + lmLogShutdown(log); + + /* + * mark on-disk super block for fs in transition; + * + * update on-disk superblock for the new space configuration + * of inline log space and fsck work space descriptors: + * N.B. FS descriptor is NOT updated; + * + * crash recovery: + * logredo(): if FM_EXTENDFS, return to fsck() for cleanup; + * fsck(): if FM_EXTENDFS, reformat inline log and fsck + * workspace from superblock inline log descriptor and fsck + * workspace descriptor; + */ + + /* read in superblock */ + if ((rc = readSuper(sb, &bh))) + goto error_out; + j_sb = (struct jfs_superblock *)bh->b_data; + + /* mark extendfs() in progress */ + j_sb->s_state |= cpu_to_le32(FM_EXTENDFS); + j_sb->s_xsize = cpu_to_le64(newFSSize); + PXDaddress(&j_sb->s_xfsckpxd, newFSCKAddress); + PXDlength(&j_sb->s_xfsckpxd, newFSCKSize); + PXDaddress(&j_sb->s_xlogpxd, newLogAddress); + PXDlength(&j_sb->s_xlogpxd, newLogSize); + + /* synchronously update superblock */ + mark_buffer_dirty(bh); + sync_dirty_buffer(bh); + brelse(bh); + + /* + * format new inline log synchronously; + * + * crash recovery: if log move in progress, + * reformat log and exit success; + */ + if (!log_formatted) + if ((rc = lmLogFormat(log, newLogAddress, newLogSize))) + goto error_out; + + /* + * activate new log + */ + log->base = newLogAddress; + log->size = newLogSize >> (L2LOGPSIZE - sb->s_blocksize_bits); + if ((rc = lmLogInit(log))) + goto error_out; + } + + /* + * extend block allocation map + * --------------------------- + * + * extendfs() for new extension, retry after crash recovery; + * + * note: both logredo() and fsck() rebuild map from + * the bitmap and configuration parameter from superblock + * (disregarding all other control information in the map); + * + * superblock: + * s_size: aggregate size in physical blocks; + */ + /* + * compute the new block allocation map configuration + * + * map dinode: + * di_size: map file size in byte; + * di_nblocks: number of blocks allocated for map file; + * di_mapsize: number of blocks in aggregate (covered by map); + * map control page: + * db_mapsize: number of blocks in aggregate (covered by map); + */ + newMapSize = newFSSize; + /* number of data pages of new bmap file: + * roundup new size to full dmap page boundary and + * add 1 extra dmap page for next extendfs() + */ + t64 = (newMapSize - 1) + BPERDMAP; + newNpages = BLKTODMAPN(t64) + 1; + + /* + * extend map from current map (WITHOUT growing mapfile) + * + * map new extension with unmapped part of the last partial + * dmap page, if applicable, and extra page(s) allocated + * at end of bmap by mkfs() or previous extendfs(); + */ + extendBmap: + /* compute number of blocks requested to extend */ + mapSize = bmp->db_mapsize; + XAddress = mapSize; /* eXtension Address */ + XSize = newMapSize - mapSize; /* eXtension Size */ + old_agsize = bmp->db_agsize; /* We need to know if this changes */ + + /* compute number of blocks that can be extended by current mapfile */ + t64 = dbMapFileSizeToMapSize(ipbmap); + if (mapSize > t64) { + printk(KERN_ERR "jfs_extendfs: mapSize (0x%Lx) > t64 (0x%Lx)\n", + (long long) mapSize, (long long) t64); + rc = -EIO; + goto error_out; + } + nblocks = min(t64 - mapSize, XSize); + + /* + * update map pages for new extension: + * + * update/init dmap and bubble up the control hierarchy + * incrementally fold up dmaps into upper levels; + * update bmap control page; + */ + if ((rc = dbExtendFS(ipbmap, XAddress, nblocks))) + goto error_out; + + agsizechanged |= (bmp->db_agsize != old_agsize); + + /* + * the map now has extended to cover additional nblocks: + * dn_mapsize = oldMapsize + nblocks; + */ + /* ipbmap->i_mapsize += nblocks; */ + XSize -= nblocks; + + /* + * grow map file to cover remaining extension + * and/or one extra dmap page for next extendfs(); + * + * allocate new map pages and its backing blocks, and + * update map file xtree + */ + /* compute number of data pages of current bmap file */ + nPages = ipbmap->i_size >> L2PSIZE; + + /* need to grow map file ? */ + if (nPages == newNpages) + goto finalizeBmap; + + /* + * grow bmap file for the new map pages required: + * + * allocate growth at the start of newly extended region; + * bmap file only grows sequentially, i.e., both data pages + * and possibly xtree index pages may grow in append mode, + * s.t. logredo() can reconstruct pre-extension state + * by washing away bmap file of pages outside s_size boundary; + */ + /* + * journal map file growth as if a regular file growth: + * (note: bmap is created with di_mode = IFJOURNAL|IFREG); + * + * journaling of bmap file growth is not required since + * logredo() do/can not use log records of bmap file growth + * but it provides careful write semantics, pmap update, etc.; + */ + /* synchronous write of data pages: bmap data pages are + * cached in meta-data cache, and not written out + * by txCommit(); + */ + rc = filemap_fdatawait(ipbmap->i_mapping); + if (rc) + goto error_out; + + rc = filemap_write_and_wait(ipbmap->i_mapping); + if (rc) + goto error_out; + + diWriteSpecial(ipbmap, 0); + + newPage = nPages; /* first new page number */ + xoff = newPage << sbi->l2nbperpage; + xlen = (newNpages - nPages) << sbi->l2nbperpage; + xlen = min(xlen, (int) nblocks) & ~(sbi->nbperpage - 1); + xaddr = XAddress; + + tid = txBegin(sb, COMMIT_FORCE); + + if ((rc = xtAppend(tid, ipbmap, 0, xoff, nblocks, &xlen, &xaddr, 0))) { + txEnd(tid); + goto error_out; + } + /* update bmap file size */ + ipbmap->i_size += xlen << sbi->l2bsize; + inode_add_bytes(ipbmap, xlen << sbi->l2bsize); + + iplist[0] = ipbmap; + rc = txCommit(tid, 1, &iplist[0], COMMIT_FORCE); + + txEnd(tid); + + if (rc) + goto error_out; + + /* + * map file has been grown now to cover extension to further out; + * di_size = new map file size; + * + * if huge extension, the previous extension based on previous + * map file size may not have been sufficient to cover whole extension + * (it could have been used up for new map pages), + * but the newly grown map file now covers lot bigger new free space + * available for further extension of map; + */ + /* any more blocks to extend ? */ + if (XSize) + goto extendBmap; + + finalizeBmap: + /* finalize bmap */ + dbFinalizeBmap(ipbmap); + + /* + * update inode allocation map + * --------------------------- + * + * move iag lists from old to new iag; + * agstart field is not updated for logredo() to reconstruct + * iag lists if system crash occurs. + * (computation of ag number from agstart based on agsize + * will correctly identify the new ag); + */ + /* if new AG size the same as old AG size, done! */ + if (agsizechanged) { + if ((rc = diExtendFS(ipimap, ipbmap))) + goto error_out; + + /* finalize imap */ + if ((rc = diSync(ipimap))) + goto error_out; + } + + /* + * finalize + * -------- + * + * extension is committed when on-disk super block is + * updated with new descriptors: logredo will recover + * crash before it to pre-extension state; + */ + + /* sync log to skip log replay of bmap file growth transaction; */ + /* lmLogSync(log, 1); */ + + /* + * synchronous write bmap global control page; + * for crash before completion of write + * logredo() will recover to pre-extendfs state; + * for crash after completion of write, + * logredo() will recover post-extendfs state; + */ + if ((rc = dbSync(ipbmap))) + goto error_out; + + /* + * copy primary bmap inode to secondary bmap inode + */ + + ipbmap2 = diReadSpecial(sb, BMAP_I, 1); + if (ipbmap2 == NULL) { + printk(KERN_ERR "jfs_extendfs: diReadSpecial(bmap) failed\n"); + goto error_out; + } + memcpy(&JFS_IP(ipbmap2)->i_xtroot, &JFS_IP(ipbmap)->i_xtroot, 288); + ipbmap2->i_size = ipbmap->i_size; + ipbmap2->i_blocks = ipbmap->i_blocks; + + diWriteSpecial(ipbmap2, 1); + diFreeSpecial(ipbmap2); + + /* + * update superblock + */ + if ((rc = readSuper(sb, &bh))) + goto error_out; + j_sb = (struct jfs_superblock *)bh->b_data; + + /* mark extendfs() completion */ + j_sb->s_state &= cpu_to_le32(~FM_EXTENDFS); + j_sb->s_size = cpu_to_le64(bmp->db_mapsize << + le16_to_cpu(j_sb->s_l2bfactor)); + j_sb->s_agsize = cpu_to_le32(bmp->db_agsize); + + /* update inline log space descriptor */ + if (sbi->mntflag & JFS_INLINELOG) { + PXDaddress(&(j_sb->s_logpxd), newLogAddress); + PXDlength(&(j_sb->s_logpxd), newLogSize); + } + + /* record log's mount serial number */ + j_sb->s_logserial = cpu_to_le32(log->serial); + + /* update fsck work space descriptor */ + PXDaddress(&(j_sb->s_fsckpxd), newFSCKAddress); + PXDlength(&(j_sb->s_fsckpxd), newFSCKSize); + j_sb->s_fscklog = 1; + /* sb->s_fsckloglen remains the same */ + + /* Update secondary superblock */ + bh2 = sb_bread(sb, SUPER2_OFF >> sb->s_blocksize_bits); + if (bh2) { + j_sb2 = (struct jfs_superblock *)bh2->b_data; + memcpy(j_sb2, j_sb, sizeof (struct jfs_superblock)); + + mark_buffer_dirty(bh); + sync_dirty_buffer(bh2); + brelse(bh2); + } + + /* write primary superblock */ + mark_buffer_dirty(bh); + sync_dirty_buffer(bh); + brelse(bh); + + goto resume; + + error_out: + jfs_error(sb, "\n"); + + resume: + /* + * resume file system transactions + */ + txResume(sb); + + out: + return rc; +} diff --git a/fs/jfs/super.c b/fs/jfs/super.c new file mode 100644 index 000000000..8180d83d3 --- /dev/null +++ b/fs/jfs/super.c @@ -0,0 +1,1069 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) International Business Machines Corp., 2000-2004 + * Portions Copyright (C) Christoph Hellwig, 2001-2002 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "jfs_incore.h" +#include "jfs_filsys.h" +#include "jfs_inode.h" +#include "jfs_metapage.h" +#include "jfs_superblock.h" +#include "jfs_dmap.h" +#include "jfs_imap.h" +#include "jfs_acl.h" +#include "jfs_debug.h" +#include "jfs_xattr.h" +#include "jfs_dinode.h" + +MODULE_DESCRIPTION("The Journaled Filesystem (JFS)"); +MODULE_AUTHOR("Steve Best/Dave Kleikamp/Barry Arndt, IBM"); +MODULE_LICENSE("GPL"); + +static struct kmem_cache *jfs_inode_cachep; + +static const struct super_operations jfs_super_operations; +static const struct export_operations jfs_export_operations; +static struct file_system_type jfs_fs_type; + +#define MAX_COMMIT_THREADS 64 +static int commit_threads; +module_param(commit_threads, int, 0); +MODULE_PARM_DESC(commit_threads, "Number of commit threads"); + +static struct task_struct *jfsCommitThread[MAX_COMMIT_THREADS]; +struct task_struct *jfsIOthread; +struct task_struct *jfsSyncThread; + +#ifdef CONFIG_JFS_DEBUG +int jfsloglevel = JFS_LOGLEVEL_WARN; +module_param(jfsloglevel, int, 0644); +MODULE_PARM_DESC(jfsloglevel, "Specify JFS loglevel (0, 1 or 2)"); +#endif + +static void jfs_handle_error(struct super_block *sb) +{ + struct jfs_sb_info *sbi = JFS_SBI(sb); + + if (sb_rdonly(sb)) + return; + + updateSuper(sb, FM_DIRTY); + + if (sbi->flag & JFS_ERR_PANIC) + panic("JFS (device %s): panic forced after error\n", + sb->s_id); + else if (sbi->flag & JFS_ERR_REMOUNT_RO) { + jfs_err("ERROR: (device %s): remounting filesystem as read-only", + sb->s_id); + sb->s_flags |= SB_RDONLY; + } + + /* nothing is done for continue beyond marking the superblock dirty */ +} + +void jfs_error(struct super_block *sb, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + pr_err("ERROR: (device %s): %ps: %pV\n", + sb->s_id, __builtin_return_address(0), &vaf); + + va_end(args); + + jfs_handle_error(sb); +} + +static struct inode *jfs_alloc_inode(struct super_block *sb) +{ + struct jfs_inode_info *jfs_inode; + + jfs_inode = alloc_inode_sb(sb, jfs_inode_cachep, GFP_NOFS); + if (!jfs_inode) + return NULL; +#ifdef CONFIG_QUOTA + memset(&jfs_inode->i_dquot, 0, sizeof(jfs_inode->i_dquot)); +#endif + return &jfs_inode->vfs_inode; +} + +static void jfs_free_inode(struct inode *inode) +{ + kmem_cache_free(jfs_inode_cachep, JFS_IP(inode)); +} + +static int jfs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct jfs_sb_info *sbi = JFS_SBI(dentry->d_sb); + s64 maxinodes; + struct inomap *imap = JFS_IP(sbi->ipimap)->i_imap; + + jfs_info("In jfs_statfs"); + buf->f_type = JFS_SUPER_MAGIC; + buf->f_bsize = sbi->bsize; + buf->f_blocks = sbi->bmap->db_mapsize; + buf->f_bfree = sbi->bmap->db_nfree; + buf->f_bavail = sbi->bmap->db_nfree; + /* + * If we really return the number of allocated & free inodes, some + * applications will fail because they won't see enough free inodes. + * We'll try to calculate some guess as to how many inodes we can + * really allocate + * + * buf->f_files = atomic_read(&imap->im_numinos); + * buf->f_ffree = atomic_read(&imap->im_numfree); + */ + maxinodes = min((s64) atomic_read(&imap->im_numinos) + + ((sbi->bmap->db_nfree >> imap->im_l2nbperiext) + << L2INOSPEREXT), (s64) 0xffffffffLL); + buf->f_files = maxinodes; + buf->f_ffree = maxinodes - (atomic_read(&imap->im_numinos) - + atomic_read(&imap->im_numfree)); + buf->f_fsid.val[0] = crc32_le(0, (char *)&sbi->uuid, + sizeof(sbi->uuid)/2); + buf->f_fsid.val[1] = crc32_le(0, + (char *)&sbi->uuid + sizeof(sbi->uuid)/2, + sizeof(sbi->uuid)/2); + + buf->f_namelen = JFS_NAME_MAX; + return 0; +} + +#ifdef CONFIG_QUOTA +static int jfs_quota_off(struct super_block *sb, int type); +static int jfs_quota_on(struct super_block *sb, int type, int format_id, + const struct path *path); + +static void jfs_quota_off_umount(struct super_block *sb) +{ + int type; + + for (type = 0; type < MAXQUOTAS; type++) + jfs_quota_off(sb, type); +} + +static const struct quotactl_ops jfs_quotactl_ops = { + .quota_on = jfs_quota_on, + .quota_off = jfs_quota_off, + .quota_sync = dquot_quota_sync, + .get_state = dquot_get_state, + .set_info = dquot_set_dqinfo, + .get_dqblk = dquot_get_dqblk, + .set_dqblk = dquot_set_dqblk, + .get_nextdqblk = dquot_get_next_dqblk, +}; +#else +static inline void jfs_quota_off_umount(struct super_block *sb) +{ +} +#endif + +static void jfs_put_super(struct super_block *sb) +{ + struct jfs_sb_info *sbi = JFS_SBI(sb); + int rc; + + jfs_info("In jfs_put_super"); + + jfs_quota_off_umount(sb); + + rc = jfs_umount(sb); + if (rc) + jfs_err("jfs_umount failed with return code %d", rc); + + unload_nls(sbi->nls_tab); + + truncate_inode_pages(sbi->direct_inode->i_mapping, 0); + iput(sbi->direct_inode); + + kfree(sbi); +} + +enum { + Opt_integrity, Opt_nointegrity, Opt_iocharset, Opt_resize, + Opt_resize_nosize, Opt_errors, Opt_ignore, Opt_err, Opt_quota, + Opt_usrquota, Opt_grpquota, Opt_uid, Opt_gid, Opt_umask, + Opt_discard, Opt_nodiscard, Opt_discard_minblk +}; + +static const struct constant_table jfs_param_errors[] = { + {"continue", JFS_ERR_CONTINUE}, + {"remount-ro", JFS_ERR_REMOUNT_RO}, + {"panic", JFS_ERR_PANIC}, + {} +}; + +static const struct fs_parameter_spec jfs_param_spec[] = { + fsparam_flag_no ("integrity", Opt_integrity), + fsparam_string ("iocharset", Opt_iocharset), + fsparam_u64 ("resize", Opt_resize), + fsparam_flag ("resize", Opt_resize_nosize), + fsparam_enum ("errors", Opt_errors, jfs_param_errors), + fsparam_flag ("quota", Opt_quota), + fsparam_flag ("noquota", Opt_ignore), + fsparam_flag ("usrquota", Opt_usrquota), + fsparam_flag ("grpquota", Opt_grpquota), + fsparam_uid ("uid", Opt_uid), + fsparam_gid ("gid", Opt_gid), + fsparam_u32oct ("umask", Opt_umask), + fsparam_flag ("discard", Opt_discard), + fsparam_u32 ("discard", Opt_discard_minblk), + fsparam_flag ("nodiscard", Opt_nodiscard), + {} +}; + +struct jfs_context { + int flag; + kuid_t uid; + kgid_t gid; + uint umask; + uint minblks_trim; + void *nls_map; + bool resize; + s64 newLVSize; +}; + +static int jfs_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct jfs_context *ctx = fc->fs_private; + int reconfigure = (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE); + struct fs_parse_result result; + struct nls_table *nls_map; + int opt; + + opt = fs_parse(fc, jfs_param_spec, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_integrity: + if (result.negated) + ctx->flag |= JFS_NOINTEGRITY; + else + ctx->flag &= ~JFS_NOINTEGRITY; + break; + case Opt_ignore: + /* Silently ignore the quota options */ + /* Don't do anything ;-) */ + break; + case Opt_iocharset: + if (ctx->nls_map && ctx->nls_map != (void *) -1) { + unload_nls(ctx->nls_map); + ctx->nls_map = NULL; + } + if (!strcmp(param->string, "none")) + ctx->nls_map = NULL; + else { + nls_map = load_nls(param->string); + if (!nls_map) { + pr_err("JFS: charset not found\n"); + return -EINVAL; + } + ctx->nls_map = nls_map; + } + break; + case Opt_resize: + if (!reconfigure) + return -EINVAL; + ctx->resize = true; + ctx->newLVSize = result.uint_64; + break; + case Opt_resize_nosize: + if (!reconfigure) + return -EINVAL; + ctx->resize = true; + break; + case Opt_errors: + ctx->flag &= ~JFS_ERR_MASK; + ctx->flag |= result.uint_32; + break; + +#ifdef CONFIG_QUOTA + case Opt_quota: + case Opt_usrquota: + ctx->flag |= JFS_USRQUOTA; + break; + case Opt_grpquota: + ctx->flag |= JFS_GRPQUOTA; + break; +#else + case Opt_usrquota: + case Opt_grpquota: + case Opt_quota: + pr_err("JFS: quota operations not supported\n"); + break; +#endif + case Opt_uid: + ctx->uid = result.uid; + break; + + case Opt_gid: + ctx->gid = result.gid; + break; + + case Opt_umask: + if (result.uint_32 & ~0777) { + pr_err("JFS: Invalid value of umask\n"); + return -EINVAL; + } + ctx->umask = result.uint_32; + break; + + case Opt_discard: + /* if set to 1, even copying files will cause + * trimming :O + * -> user has more control over the online trimming + */ + ctx->minblks_trim = 64; + ctx->flag |= JFS_DISCARD; + break; + + case Opt_nodiscard: + ctx->flag &= ~JFS_DISCARD; + break; + + case Opt_discard_minblk: + ctx->minblks_trim = result.uint_32; + ctx->flag |= JFS_DISCARD; + break; + + default: + return -EINVAL; + } + + return 0; +} + +static int jfs_reconfigure(struct fs_context *fc) +{ + struct jfs_context *ctx = fc->fs_private; + struct super_block *sb = fc->root->d_sb; + int readonly = fc->sb_flags & SB_RDONLY; + int rc = 0; + int flag = ctx->flag; + int ret; + + sync_filesystem(sb); + + /* Transfer results of parsing to the sbi */ + JFS_SBI(sb)->flag = ctx->flag; + JFS_SBI(sb)->uid = ctx->uid; + JFS_SBI(sb)->gid = ctx->gid; + JFS_SBI(sb)->umask = ctx->umask; + JFS_SBI(sb)->minblks_trim = ctx->minblks_trim; + if (ctx->nls_map != (void *) -1) { + unload_nls(JFS_SBI(sb)->nls_tab); + JFS_SBI(sb)->nls_tab = ctx->nls_map; + } + ctx->nls_map = NULL; + + if (ctx->resize) { + if (sb_rdonly(sb)) { + pr_err("JFS: resize requires volume to be mounted read-write\n"); + return -EROFS; + } + + if (!ctx->newLVSize) { + ctx->newLVSize = sb_bdev_nr_blocks(sb); + if (ctx->newLVSize == 0) + pr_err("JFS: Cannot determine volume size\n"); + } + + rc = jfs_extendfs(sb, ctx->newLVSize, 0); + if (rc) + return rc; + } + + if (sb_rdonly(sb) && !readonly) { + /* + * Invalidate any previously read metadata. fsck may have + * changed the on-disk data since we mounted r/o + */ + truncate_inode_pages(JFS_SBI(sb)->direct_inode->i_mapping, 0); + + JFS_SBI(sb)->flag = flag; + ret = jfs_mount_rw(sb, 1); + + /* mark the fs r/w for quota activity */ + sb->s_flags &= ~SB_RDONLY; + + dquot_resume(sb, -1); + return ret; + } + if (!sb_rdonly(sb) && readonly) { + rc = dquot_suspend(sb, -1); + if (rc < 0) + return rc; + rc = jfs_umount_rw(sb); + JFS_SBI(sb)->flag = flag; + return rc; + } + if ((JFS_SBI(sb)->flag & JFS_NOINTEGRITY) != (flag & JFS_NOINTEGRITY)) { + if (!sb_rdonly(sb)) { + rc = jfs_umount_rw(sb); + if (rc) + return rc; + + JFS_SBI(sb)->flag = flag; + ret = jfs_mount_rw(sb, 1); + return ret; + } + } + JFS_SBI(sb)->flag = flag; + + return 0; +} + +static int jfs_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct jfs_context *ctx = fc->fs_private; + int silent = fc->sb_flags & SB_SILENT; + struct jfs_sb_info *sbi; + struct inode *inode; + int rc; + int ret = -EINVAL; + + jfs_info("In jfs_read_super: s_flags=0x%lx", sb->s_flags); + + sbi = kzalloc_obj(struct jfs_sb_info); + if (!sbi) + return -ENOMEM; + + sb->s_fs_info = sbi; + sb->s_max_links = JFS_LINK_MAX; + sb->s_time_min = 0; + sb->s_time_max = U32_MAX; + sbi->sb = sb; + + /* Transfer results of parsing to the sbi */ + sbi->flag = ctx->flag; + sbi->uid = ctx->uid; + sbi->gid = ctx->gid; + sbi->umask = ctx->umask; + if (ctx->nls_map != (void *) -1) { + unload_nls(sbi->nls_tab); + sbi->nls_tab = ctx->nls_map; + } + ctx->nls_map = NULL; + + if (sbi->flag & JFS_DISCARD) { + if (!bdev_max_discard_sectors(sb->s_bdev)) { + pr_err("JFS: discard option not supported on device\n"); + sbi->flag &= ~JFS_DISCARD; + } else { + sbi->minblks_trim = ctx->minblks_trim; + } + } + +#ifdef CONFIG_JFS_POSIX_ACL + sb->s_flags |= SB_POSIXACL; +#endif + + if (ctx->resize) { + pr_err("resize option for remount only\n"); + goto out_unload; + } + + /* + * Initialize blocksize to 4K. + */ + if (!sb_set_blocksize(sb, PSIZE)) + goto out_unload; + + /* + * Set method vectors. + */ + sb->s_op = &jfs_super_operations; + sb->s_export_op = &jfs_export_operations; + sb->s_xattr = jfs_xattr_handlers; +#ifdef CONFIG_QUOTA + sb->dq_op = &dquot_operations; + sb->s_qcop = &jfs_quotactl_ops; + sb->s_quota_types = QTYPE_MASK_USR | QTYPE_MASK_GRP; +#endif + + /* + * Initialize direct-mapping inode/address-space + */ + inode = new_inode(sb); + if (inode == NULL) { + ret = -ENOMEM; + goto out_unload; + } + inode->i_size = bdev_nr_bytes(sb->s_bdev); + inode->i_mapping->a_ops = &jfs_metapage_aops; + inode_fake_hash(inode); + mapping_set_gfp_mask(inode->i_mapping, GFP_NOFS); + + sbi->direct_inode = inode; + + rc = jfs_mount(sb); + if (rc) { + if (!silent) + jfs_err("jfs_mount failed w/return code = %d", rc); + goto out_mount_failed; + } + if (sb_rdonly(sb)) + sbi->log = NULL; + else { + rc = jfs_mount_rw(sb, 0); + if (rc) { + if (!silent) { + jfs_err("jfs_mount_rw failed, return code = %d", + rc); + } + goto out_no_rw; + } + } + + sb->s_magic = JFS_SUPER_MAGIC; + + if (sbi->mntflag & JFS_OS2) + set_default_d_op(sb, &jfs_ci_dentry_operations); + + inode = jfs_iget(sb, ROOT_I); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + goto out_no_rw; + } + sb->s_root = d_make_root(inode); + if (!sb->s_root) + goto out_no_root; + + /* logical blocks are represented by 40 bits in pxd_t, etc. + * and page cache is indexed by long + */ + sb->s_maxbytes = min(((loff_t)sb->s_blocksize) << 40, MAX_LFS_FILESIZE); + sb->s_time_gran = 1; + return 0; + +out_no_root: + jfs_err("jfs_read_super: get root dentry failed"); + +out_no_rw: + rc = jfs_umount(sb); + if (rc) + jfs_err("jfs_umount failed with return code %d", rc); +out_mount_failed: + filemap_write_and_wait(sbi->direct_inode->i_mapping); + truncate_inode_pages(sbi->direct_inode->i_mapping, 0); + make_bad_inode(sbi->direct_inode); + iput(sbi->direct_inode); + sbi->direct_inode = NULL; +out_unload: + unload_nls(sbi->nls_tab); + kfree(sbi); + return ret; +} + +static int jfs_freeze(struct super_block *sb) +{ + struct jfs_sb_info *sbi = JFS_SBI(sb); + struct jfs_log *log = sbi->log; + int rc = 0; + + if (!sb_rdonly(sb)) { + txQuiesce(sb); + rc = lmLogShutdown(log); + if (rc) { + jfs_error(sb, "lmLogShutdown failed\n"); + + /* let operations fail rather than hang */ + txResume(sb); + + return rc; + } + rc = updateSuper(sb, FM_CLEAN); + if (rc) { + jfs_err("jfs_freeze: updateSuper failed"); + /* + * Don't fail here. Everything succeeded except + * marking the superblock clean, so there's really + * no harm in leaving it frozen for now. + */ + } + } + return 0; +} + +static int jfs_unfreeze(struct super_block *sb) +{ + struct jfs_sb_info *sbi = JFS_SBI(sb); + struct jfs_log *log = sbi->log; + int rc = 0; + + if (!sb_rdonly(sb)) { + rc = updateSuper(sb, FM_MOUNT); + if (rc) { + jfs_error(sb, "updateSuper failed\n"); + goto out; + } + rc = lmLogInit(log); + if (rc) + jfs_error(sb, "lmLogInit failed\n"); +out: + txResume(sb); + } + return rc; +} + +static int jfs_get_tree(struct fs_context *fc) +{ + return get_tree_bdev(fc, jfs_fill_super); +} + +static int jfs_sync_fs(struct super_block *sb, int wait) +{ + struct jfs_log *log = JFS_SBI(sb)->log; + + /* log == NULL indicates read-only mount */ + if (log) { + /* + * Write quota structures to quota file, sync_blockdev() will + * write them to disk later + */ + dquot_writeback_dquots(sb, -1); + jfs_flush_journal(log, wait); + jfs_syncpt(log, 0); + } + + return 0; +} + +static int jfs_show_options(struct seq_file *seq, struct dentry *root) +{ + struct jfs_sb_info *sbi = JFS_SBI(root->d_sb); + + if (uid_valid(sbi->uid)) + seq_printf(seq, ",uid=%d", from_kuid(&init_user_ns, sbi->uid)); + if (gid_valid(sbi->gid)) + seq_printf(seq, ",gid=%d", from_kgid(&init_user_ns, sbi->gid)); + if (sbi->umask != -1) + seq_printf(seq, ",umask=%03o", sbi->umask); + if (sbi->flag & JFS_NOINTEGRITY) + seq_puts(seq, ",nointegrity"); + if (sbi->flag & JFS_DISCARD) + seq_printf(seq, ",discard=%u", sbi->minblks_trim); + if (sbi->nls_tab) + seq_printf(seq, ",iocharset=%s", sbi->nls_tab->charset); + if (sbi->flag & JFS_ERR_CONTINUE) + seq_printf(seq, ",errors=continue"); + if (sbi->flag & JFS_ERR_PANIC) + seq_printf(seq, ",errors=panic"); + +#ifdef CONFIG_QUOTA + if (sbi->flag & JFS_USRQUOTA) + seq_puts(seq, ",usrquota"); + + if (sbi->flag & JFS_GRPQUOTA) + seq_puts(seq, ",grpquota"); +#endif + + return 0; +} + +#ifdef CONFIG_QUOTA + +/* Read data from quotafile - avoid pagecache and such because we cannot afford + * acquiring the locks... As quota files are never truncated and quota code + * itself serializes the operations (and no one else should touch the files) + * we don't have to be afraid of races */ +static ssize_t jfs_quota_read(struct super_block *sb, int type, char *data, + size_t len, loff_t off) +{ + struct inode *inode = sb_dqopt(sb)->files[type]; + sector_t blk = off >> sb->s_blocksize_bits; + int err = 0; + int offset = off & (sb->s_blocksize - 1); + int tocopy; + size_t toread; + struct buffer_head tmp_bh; + struct buffer_head *bh; + loff_t i_size = i_size_read(inode); + + if (off > i_size) + return 0; + if (off+len > i_size) + len = i_size-off; + toread = len; + while (toread > 0) { + tocopy = min_t(size_t, sb->s_blocksize - offset, toread); + + tmp_bh.b_state = 0; + tmp_bh.b_size = i_blocksize(inode); + err = jfs_get_block(inode, blk, &tmp_bh, 0); + if (err) + return err; + if (!buffer_mapped(&tmp_bh)) /* A hole? */ + memset(data, 0, tocopy); + else { + bh = sb_bread(sb, tmp_bh.b_blocknr); + if (!bh) + return -EIO; + memcpy(data, bh->b_data+offset, tocopy); + brelse(bh); + } + offset = 0; + toread -= tocopy; + data += tocopy; + blk++; + } + return len; +} + +/* Write to quotafile */ +static ssize_t jfs_quota_write(struct super_block *sb, int type, + const char *data, size_t len, loff_t off) +{ + struct inode *inode = sb_dqopt(sb)->files[type]; + sector_t blk = off >> sb->s_blocksize_bits; + int err = 0; + int offset = off & (sb->s_blocksize - 1); + int tocopy; + size_t towrite = len; + struct buffer_head tmp_bh; + struct buffer_head *bh; + + inode_lock(inode); + while (towrite > 0) { + tocopy = min_t(size_t, sb->s_blocksize - offset, towrite); + + tmp_bh.b_state = 0; + tmp_bh.b_size = i_blocksize(inode); + err = jfs_get_block(inode, blk, &tmp_bh, 1); + if (err) + goto out; + if (offset || tocopy != sb->s_blocksize) + bh = sb_bread(sb, tmp_bh.b_blocknr); + else + bh = sb_getblk(sb, tmp_bh.b_blocknr); + if (!bh) { + err = -EIO; + goto out; + } + lock_buffer(bh); + memcpy(bh->b_data+offset, data, tocopy); + flush_dcache_folio(bh->b_folio); + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + unlock_buffer(bh); + brelse(bh); + offset = 0; + towrite -= tocopy; + data += tocopy; + blk++; + } +out: + if (len == towrite) { + inode_unlock(inode); + return err; + } + if (inode->i_size < off+len-towrite) + i_size_write(inode, off+len-towrite); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + mark_inode_dirty(inode); + inode_unlock(inode); + return len - towrite; +} + +static struct dquot __rcu **jfs_get_dquots(struct inode *inode) +{ + return JFS_IP(inode)->i_dquot; +} + +static int jfs_quota_on(struct super_block *sb, int type, int format_id, + const struct path *path) +{ + int err; + struct inode *inode; + + err = dquot_quota_on(sb, type, format_id, path); + if (err) + return err; + + inode = d_inode(path->dentry); + inode_lock(inode); + JFS_IP(inode)->mode2 |= JFS_NOATIME_FL | JFS_IMMUTABLE_FL; + inode_set_flags(inode, S_NOATIME | S_IMMUTABLE, + S_NOATIME | S_IMMUTABLE); + inode_unlock(inode); + mark_inode_dirty(inode); + + return 0; +} + +static int jfs_quota_off(struct super_block *sb, int type) +{ + struct inode *inode = sb_dqopt(sb)->files[type]; + int err; + + if (!inode || !igrab(inode)) + goto out; + + err = dquot_quota_off(sb, type); + if (err) + goto out_put; + + inode_lock(inode); + JFS_IP(inode)->mode2 &= ~(JFS_NOATIME_FL | JFS_IMMUTABLE_FL); + inode_set_flags(inode, 0, S_NOATIME | S_IMMUTABLE); + inode_unlock(inode); + mark_inode_dirty(inode); +out_put: + iput(inode); + return err; +out: + return dquot_quota_off(sb, type); +} +#endif + +static const struct super_operations jfs_super_operations = { + .alloc_inode = jfs_alloc_inode, + .free_inode = jfs_free_inode, + .dirty_inode = jfs_dirty_inode, + .write_inode = jfs_write_inode, + .evict_inode = jfs_evict_inode, + .put_super = jfs_put_super, + .sync_fs = jfs_sync_fs, + .freeze_fs = jfs_freeze, + .unfreeze_fs = jfs_unfreeze, + .statfs = jfs_statfs, + .show_options = jfs_show_options, +#ifdef CONFIG_QUOTA + .quota_read = jfs_quota_read, + .quota_write = jfs_quota_write, + .get_dquots = jfs_get_dquots, +#endif +}; + +static const struct export_operations jfs_export_operations = { + .encode_fh = generic_encode_ino32_fh, + .fh_to_dentry = jfs_fh_to_dentry, + .fh_to_parent = jfs_fh_to_parent, + .get_parent = jfs_get_parent, +}; + +static void jfs_init_options(struct fs_context *fc, struct jfs_context *ctx) +{ + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE) { + struct super_block *sb = fc->root->d_sb; + + /* Copy over current option values and mount flags */ + ctx->uid = JFS_SBI(sb)->uid; + ctx->gid = JFS_SBI(sb)->gid; + ctx->umask = JFS_SBI(sb)->umask; + ctx->nls_map = (void *)-1; + ctx->minblks_trim = JFS_SBI(sb)->minblks_trim; + ctx->flag = JFS_SBI(sb)->flag; + + } else { + /* + * Initialize the mount flag and determine the default + * error handler + */ + ctx->flag = JFS_ERR_REMOUNT_RO; + ctx->uid = INVALID_UID; + ctx->gid = INVALID_GID; + ctx->umask = -1; + ctx->nls_map = (void *)-1; + } +} + +static void jfs_free_fc(struct fs_context *fc) +{ + struct jfs_context *ctx = fc->fs_private; + + if (ctx->nls_map != (void *) -1) + unload_nls(ctx->nls_map); + kfree(ctx); +} + +static const struct fs_context_operations jfs_context_ops = { + .parse_param = jfs_parse_param, + .get_tree = jfs_get_tree, + .reconfigure = jfs_reconfigure, + .free = jfs_free_fc, +}; + +static int jfs_init_fs_context(struct fs_context *fc) +{ + struct jfs_context *ctx; + + ctx = kzalloc_obj(*ctx); + if (!ctx) + return -ENOMEM; + + jfs_init_options(fc, ctx); + + fc->fs_private = ctx; + fc->ops = &jfs_context_ops; + + return 0; +} + +static struct file_system_type jfs_fs_type = { + .owner = THIS_MODULE, + .name = "jfs", + .kill_sb = kill_block_super, + .fs_flags = FS_REQUIRES_DEV, + .init_fs_context = jfs_init_fs_context, + .parameters = jfs_param_spec, +}; +MODULE_ALIAS_FS("jfs"); + +static void init_once(void *foo) +{ + struct jfs_inode_info *jfs_ip = (struct jfs_inode_info *) foo; + + memset(jfs_ip, 0, sizeof(struct jfs_inode_info)); + INIT_LIST_HEAD(&jfs_ip->anon_inode_list); + init_rwsem(&jfs_ip->rdwrlock); + mutex_init(&jfs_ip->commit_mutex); + init_rwsem(&jfs_ip->xattr_sem); + spin_lock_init(&jfs_ip->ag_lock); + jfs_ip->active_ag = -1; + inode_init_once(&jfs_ip->vfs_inode); +} + +static int __init init_jfs_fs(void) +{ + int i; + int rc; + + jfs_inode_cachep = + kmem_cache_create_usercopy("jfs_ip", sizeof(struct jfs_inode_info), + 0, SLAB_RECLAIM_ACCOUNT|SLAB_ACCOUNT, + offsetof(struct jfs_inode_info, i_inline_all), + sizeof_field(struct jfs_inode_info, i_inline_all), + init_once); + if (jfs_inode_cachep == NULL) + return -ENOMEM; + + /* + * Metapage initialization + */ + rc = metapage_init(); + if (rc) { + jfs_err("metapage_init failed w/rc = %d", rc); + goto free_slab; + } + + /* + * Transaction Manager initialization + */ + rc = txInit(); + if (rc) { + jfs_err("txInit failed w/rc = %d", rc); + goto free_metapage; + } + + /* + * I/O completion thread (endio) + */ + jfsIOthread = kthread_run(jfsIOWait, NULL, "jfsIO"); + if (IS_ERR(jfsIOthread)) { + rc = PTR_ERR(jfsIOthread); + jfs_err("init_jfs_fs: fork failed w/rc = %d", rc); + goto end_txmngr; + } + + if (commit_threads < 1) + commit_threads = num_online_cpus(); + if (commit_threads > MAX_COMMIT_THREADS) + commit_threads = MAX_COMMIT_THREADS; + + for (i = 0; i < commit_threads; i++) { + jfsCommitThread[i] = kthread_run(jfs_lazycommit, NULL, + "jfsCommit"); + if (IS_ERR(jfsCommitThread[i])) { + rc = PTR_ERR(jfsCommitThread[i]); + jfs_err("init_jfs_fs: fork failed w/rc = %d", rc); + commit_threads = i; + goto kill_committask; + } + } + + jfsSyncThread = kthread_run(jfs_sync, NULL, "jfsSync"); + if (IS_ERR(jfsSyncThread)) { + rc = PTR_ERR(jfsSyncThread); + jfs_err("init_jfs_fs: fork failed w/rc = %d", rc); + goto kill_committask; + } + +#ifdef PROC_FS_JFS + jfs_proc_init(); +#endif + + rc = register_filesystem(&jfs_fs_type); + if (!rc) + return 0; + +#ifdef PROC_FS_JFS + jfs_proc_clean(); +#endif + kthread_stop(jfsSyncThread); +kill_committask: + for (i = 0; i < commit_threads; i++) + kthread_stop(jfsCommitThread[i]); + kthread_stop(jfsIOthread); +end_txmngr: + txExit(); +free_metapage: + metapage_exit(); +free_slab: + kmem_cache_destroy(jfs_inode_cachep); + return rc; +} + +static void __exit exit_jfs_fs(void) +{ + int i; + + jfs_info("exit_jfs_fs called"); + + txExit(); + metapage_exit(); + + kthread_stop(jfsIOthread); + for (i = 0; i < commit_threads; i++) + kthread_stop(jfsCommitThread[i]); + kthread_stop(jfsSyncThread); +#ifdef PROC_FS_JFS + jfs_proc_clean(); +#endif + unregister_filesystem(&jfs_fs_type); + + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(jfs_inode_cachep); +} + +module_init(init_jfs_fs) +module_exit(exit_jfs_fs) diff --git a/fs/jfs/symlink.c b/fs/jfs/symlink.c new file mode 100644 index 000000000..a040719aa --- /dev/null +++ b/fs/jfs/symlink.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) Christoph Hellwig, 2001-2002 + */ + +#include +#include "jfs_incore.h" +#include "jfs_inode.h" +#include "jfs_xattr.h" + +const struct inode_operations jfs_fast_symlink_inode_operations = { + .get_link = simple_get_link, + .setattr = jfs_setattr, + .listxattr = jfs_listxattr, +}; + +const struct inode_operations jfs_symlink_inode_operations = { + .get_link = page_get_link, + .setattr = jfs_setattr, + .listxattr = jfs_listxattr, +}; + diff --git a/fs/jfs/xattr.c b/fs/jfs/xattr.c new file mode 100644 index 000000000..11d7f74d2 --- /dev/null +++ b/fs/jfs/xattr.c @@ -0,0 +1,1055 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) International Business Machines Corp., 2000-2004 + * Copyright (C) Christoph Hellwig, 2002 + */ + +#include +#include +#include +#include +#include +#include +#include +#include "jfs_incore.h" +#include "jfs_superblock.h" +#include "jfs_dmap.h" +#include "jfs_debug.h" +#include "jfs_dinode.h" +#include "jfs_extent.h" +#include "jfs_metapage.h" +#include "jfs_xattr.h" +#include "jfs_acl.h" + +/* + * jfs_xattr.c: extended attribute service + * + * Overall design -- + * + * Format: + * + * Extended attribute lists (jfs_ea_list) consist of an overall size (32 bit + * value) and a variable (0 or more) number of extended attribute + * entries. Each extended attribute entry (jfs_ea) is a double + * where is constructed from a null-terminated ascii string + * (1 ... 255 bytes in the name) and is arbitrary 8 bit data + * (1 ... 65535 bytes). The in-memory format is + * + * 0 1 2 4 4 + namelen + 1 + * +-------+--------+--------+----------------+-------------------+ + * | Flags | Name | Value | Name String \0 | Data . . . . | + * | | Length | Length | | | + * +-------+--------+--------+----------------+-------------------+ + * + * A jfs_ea_list then is structured as + * + * 0 4 4 + EA_SIZE(ea1) + * +------------+-------------------+--------------------+----- + * | Overall EA | First FEA Element | Second FEA Element | ..... + * | List Size | | | + * +------------+-------------------+--------------------+----- + * + * On-disk: + * + * FEALISTs are stored on disk using blocks allocated by dbAlloc() and + * written directly. An EA list may be in-lined in the inode if there is + * sufficient room available. + */ + +struct ea_buffer { + int flag; /* Indicates what storage xattr points to */ + int max_size; /* largest xattr that fits in current buffer */ + dxd_t new_ea; /* dxd to replace ea when modifying xattr */ + struct metapage *mp; /* metapage containing ea list */ + struct jfs_ea_list *xattr; /* buffer containing ea list */ +}; + +/* + * ea_buffer.flag values + */ +#define EA_INLINE 0x0001 +#define EA_EXTENT 0x0002 +#define EA_NEW 0x0004 +#define EA_MALLOC 0x0008 + + +/* + * Mapping of on-disk attribute names: for on-disk attribute names with an + * unknown prefix (not "system.", "user.", "security.", or "trusted."), the + * prefix "os2." is prepended. On the way back to disk, "os2." prefixes are + * stripped and we make sure that the remaining name does not start with one + * of the know prefixes. + */ + +static int is_known_namespace(const char *name) +{ + if (strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN) && + strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN) && + strncmp(name, XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN) && + strncmp(name, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN)) + return false; + + return true; +} + +static inline int name_size(struct jfs_ea *ea) +{ + if (is_known_namespace(ea->name)) + return ea->namelen; + else + return ea->namelen + XATTR_OS2_PREFIX_LEN; +} + +static inline int copy_name(char *buffer, struct jfs_ea *ea) +{ + int len = ea->namelen; + + if (!is_known_namespace(ea->name)) { + memcpy(buffer, XATTR_OS2_PREFIX, XATTR_OS2_PREFIX_LEN); + buffer += XATTR_OS2_PREFIX_LEN; + len += XATTR_OS2_PREFIX_LEN; + } + memcpy(buffer, ea->name, ea->namelen); + buffer[ea->namelen] = 0; + + return len; +} + +/* Forward references */ +static void ea_release(struct inode *inode, struct ea_buffer *ea_buf); + +/* + * NAME: ea_write_inline + * + * FUNCTION: Attempt to write an EA inline if area is available + * + * PRE CONDITIONS: + * Already verified that the specified EA is small enough to fit inline + * + * PARAMETERS: + * ip - Inode pointer + * ealist - EA list pointer + * size - size of ealist in bytes + * ea - dxd_t structure to be filled in with necessary EA information + * if we successfully copy the EA inline + * + * NOTES: + * Checks if the inode's inline area is available. If so, copies EA inline + * and sets fields appropriately. Otherwise, returns failure, EA will + * have to be put into an extent. + * + * RETURNS: 0 for successful copy to inline area; -1 if area not available + */ +static int ea_write_inline(struct inode *ip, struct jfs_ea_list *ealist, + int size, dxd_t * ea) +{ + struct jfs_inode_info *ji = JFS_IP(ip); + + /* + * Make sure we have an EA -- the NULL EA list is valid, but you + * can't copy it! + */ + if (ealist && size > sizeof (struct jfs_ea_list)) { + assert(size <= sizeof (ji->i_inline_ea)); + + /* + * See if the space is available or if it is already being + * used for an inline EA. + */ + if (!(ji->mode2 & INLINEEA) && !(ji->ea.flag & DXD_INLINE)) + return -EPERM; + + DXDsize(ea, size); + DXDlength(ea, 0); + DXDaddress(ea, 0); + memcpy(ji->i_inline_ea, ealist, size); + ea->flag = DXD_INLINE; + ji->mode2 &= ~INLINEEA; + } else { + ea->flag = 0; + DXDsize(ea, 0); + DXDlength(ea, 0); + DXDaddress(ea, 0); + + /* Free up INLINE area */ + if (ji->ea.flag & DXD_INLINE) + ji->mode2 |= INLINEEA; + } + + return 0; +} + +/* + * NAME: ea_write + * + * FUNCTION: Write an EA for an inode + * + * PRE CONDITIONS: EA has been verified + * + * PARAMETERS: + * ip - Inode pointer + * ealist - EA list pointer + * size - size of ealist in bytes + * ea - dxd_t structure to be filled in appropriately with where the + * EA was copied + * + * NOTES: Will write EA inline if able to, otherwise allocates blocks for an + * extent and synchronously writes it to those blocks. + * + * RETURNS: 0 for success; Anything else indicates failure + */ +static int ea_write(struct inode *ip, struct jfs_ea_list *ealist, int size, + dxd_t * ea) +{ + struct super_block *sb = ip->i_sb; + struct jfs_inode_info *ji = JFS_IP(ip); + struct jfs_sb_info *sbi = JFS_SBI(sb); + int nblocks; + s64 blkno; + int rc = 0, i; + char *cp; + s32 nbytes, nb; + s32 bytes_to_write; + struct metapage *mp; + + /* + * Quick check to see if this is an in-linable EA. Short EAs + * and empty EAs are all in-linable, provided the space exists. + */ + if (!ealist || size <= sizeof (ji->i_inline_ea)) { + if (!ea_write_inline(ip, ealist, size, ea)) + return 0; + } + + /* figure out how many blocks we need */ + nblocks = (size + (sb->s_blocksize - 1)) >> sb->s_blocksize_bits; + + /* Allocate new blocks to quota. */ + rc = dquot_alloc_block(ip, nblocks); + if (rc) + return rc; + + rc = dbAlloc(ip, INOHINT(ip), nblocks, &blkno); + if (rc) { + /*Rollback quota allocation. */ + dquot_free_block(ip, nblocks); + return rc; + } + + /* + * Now have nblocks worth of storage to stuff into the FEALIST. + * loop over the FEALIST copying data into the buffer one page at + * a time. + */ + cp = (char *) ealist; + nbytes = size; + for (i = 0; i < nblocks; i += sbi->nbperpage) { + /* + * Determine how many bytes for this request, and round up to + * the nearest aggregate block size + */ + nb = min(PSIZE, nbytes); + bytes_to_write = + ((((nb + sb->s_blocksize - 1)) >> sb->s_blocksize_bits)) + << sb->s_blocksize_bits; + + if (!(mp = get_metapage(ip, blkno + i, bytes_to_write, 1))) { + rc = -EIO; + goto failed; + } + + memcpy(mp->data, cp, nb); + + /* + * We really need a way to propagate errors for + * forced writes like this one. --hch + * + * (__write_metapage => release_metapage => flush_metapage) + */ +#ifdef _JFS_FIXME + if ((rc = flush_metapage(mp))) { + /* + * the write failed -- this means that the buffer + * is still assigned and the blocks are not being + * used. this seems like the best error recovery + * we can get ... + */ + goto failed; + } +#else + flush_metapage(mp); +#endif + + cp += PSIZE; + nbytes -= nb; + } + + ea->flag = DXD_EXTENT; + DXDsize(ea, le32_to_cpu(ealist->size)); + DXDlength(ea, nblocks); + DXDaddress(ea, blkno); + + /* Free up INLINE area */ + if (ji->ea.flag & DXD_INLINE) + ji->mode2 |= INLINEEA; + + return 0; + + failed: + /* Rollback quota allocation. */ + dquot_free_block(ip, nblocks); + + dbFree(ip, blkno, nblocks); + return rc; +} + +/* + * NAME: ea_read_inline + * + * FUNCTION: Read an inlined EA into user's buffer + * + * PARAMETERS: + * ip - Inode pointer + * ealist - Pointer to buffer to fill in with EA + * + * RETURNS: 0 + */ +static int ea_read_inline(struct inode *ip, struct jfs_ea_list *ealist) +{ + struct jfs_inode_info *ji = JFS_IP(ip); + int ea_size = sizeDXD(&ji->ea); + + if (ea_size == 0) { + ealist->size = 0; + return 0; + } + + /* Sanity Check */ + if ((sizeDXD(&ji->ea) > sizeof (ji->i_inline_ea))) + return -EIO; + if (le32_to_cpu(((struct jfs_ea_list *) &ji->i_inline_ea)->size) + != ea_size) + return -EIO; + + memcpy(ealist, ji->i_inline_ea, ea_size); + return 0; +} + +/* + * NAME: ea_read + * + * FUNCTION: copy EA data into user's buffer + * + * PARAMETERS: + * ip - Inode pointer + * ealist - Pointer to buffer to fill in with EA + * + * NOTES: If EA is inline calls ea_read_inline() to copy EA. + * + * RETURNS: 0 for success; other indicates failure + */ +static int ea_read(struct inode *ip, struct jfs_ea_list *ealist) +{ + struct super_block *sb = ip->i_sb; + struct jfs_inode_info *ji = JFS_IP(ip); + struct jfs_sb_info *sbi = JFS_SBI(sb); + int nblocks; + s64 blkno; + char *cp = (char *) ealist; + int i; + int nbytes, nb; + s32 bytes_to_read; + struct metapage *mp; + + /* quick check for in-line EA */ + if (ji->ea.flag & DXD_INLINE) + return ea_read_inline(ip, ealist); + + nbytes = sizeDXD(&ji->ea); + if (!nbytes) { + jfs_error(sb, "nbytes is 0\n"); + return -EIO; + } + + /* + * Figure out how many blocks were allocated when this EA list was + * originally written to disk. + */ + nblocks = lengthDXD(&ji->ea) << sbi->l2nbperpage; + blkno = addressDXD(&ji->ea) << sbi->l2nbperpage; + + /* + * I have found the disk blocks which were originally used to store + * the FEALIST. now i loop over each contiguous block copying the + * data into the buffer. + */ + for (i = 0; i < nblocks; i += sbi->nbperpage) { + /* + * Determine how many bytes for this request, and round up to + * the nearest aggregate block size + */ + nb = min(PSIZE, nbytes); + bytes_to_read = + ((((nb + sb->s_blocksize - 1)) >> sb->s_blocksize_bits)) + << sb->s_blocksize_bits; + + if (!(mp = read_metapage(ip, blkno + i, bytes_to_read, 1))) + return -EIO; + + memcpy(cp, mp->data, nb); + release_metapage(mp); + + cp += PSIZE; + nbytes -= nb; + } + + return 0; +} + +/* + * NAME: ea_get + * + * FUNCTION: Returns buffer containing existing extended attributes. + * The size of the buffer will be the larger of the existing + * attributes size, or min_size. + * + * The buffer, which may be inlined in the inode or in the + * page cache must be release by calling ea_release or ea_put + * + * PARAMETERS: + * inode - Inode pointer + * ea_buf - Structure to be populated with ealist and its metadata + * min_size- minimum size of buffer to be returned + * + * RETURNS: 0 for success; Other indicates failure + */ +static int ea_get(struct inode *inode, struct ea_buffer *ea_buf, int min_size) +{ + struct jfs_inode_info *ji = JFS_IP(inode); + struct super_block *sb = inode->i_sb; + int size; + int ea_size = sizeDXD(&ji->ea); + int blocks_needed, current_blocks; + s64 blkno; + int rc; + int quota_allocation = 0; + + memset(&ea_buf->new_ea, 0, sizeof(ea_buf->new_ea)); + + /* When fsck.jfs clears a bad ea, it doesn't clear the size */ + if (ji->ea.flag == 0) + ea_size = 0; + + if (ea_size == 0) { + if (min_size == 0) { + ea_buf->flag = 0; + ea_buf->max_size = 0; + ea_buf->xattr = NULL; + return 0; + } + if ((min_size <= sizeof (ji->i_inline_ea)) && + (ji->mode2 & INLINEEA)) { + ea_buf->flag = EA_INLINE | EA_NEW; + ea_buf->max_size = sizeof (ji->i_inline_ea); + ea_buf->xattr = (struct jfs_ea_list *) ji->i_inline_ea; + DXDlength(&ea_buf->new_ea, 0); + DXDaddress(&ea_buf->new_ea, 0); + ea_buf->new_ea.flag = DXD_INLINE; + DXDsize(&ea_buf->new_ea, min_size); + return 0; + } + current_blocks = 0; + } else if (ji->ea.flag & DXD_INLINE) { + if (min_size <= sizeof (ji->i_inline_ea)) { + ea_buf->flag = EA_INLINE; + ea_buf->max_size = sizeof (ji->i_inline_ea); + ea_buf->xattr = (struct jfs_ea_list *) ji->i_inline_ea; + goto size_check; + } + current_blocks = 0; + } else { + if (!(ji->ea.flag & DXD_EXTENT)) { + jfs_error(sb, "invalid ea.flag\n"); + return -EIO; + } + current_blocks = (ea_size + sb->s_blocksize - 1) >> + sb->s_blocksize_bits; + } + size = max(min_size, ea_size); + + if (size > PSIZE) { + /* + * To keep the rest of the code simple. Allocate a + * contiguous buffer to work with. Make the buffer large + * enough to make use of the whole extent. + */ + ea_buf->max_size = (size + sb->s_blocksize - 1) & + ~(sb->s_blocksize - 1); + + ea_buf->xattr = kmalloc(ea_buf->max_size, GFP_KERNEL); + if (ea_buf->xattr == NULL) + return -ENOMEM; + + ea_buf->flag = EA_MALLOC; + + if (ea_size == 0) + return 0; + + if ((rc = ea_read(inode, ea_buf->xattr))) { + kfree(ea_buf->xattr); + ea_buf->xattr = NULL; + return rc; + } + goto size_check; + } + blocks_needed = (min_size + sb->s_blocksize - 1) >> + sb->s_blocksize_bits; + + if (blocks_needed > current_blocks) { + /* Allocate new blocks to quota. */ + rc = dquot_alloc_block(inode, blocks_needed); + if (rc) + return -EDQUOT; + + quota_allocation = blocks_needed; + + rc = dbAlloc(inode, INOHINT(inode), (s64) blocks_needed, + &blkno); + if (rc) + goto clean_up; + + DXDlength(&ea_buf->new_ea, blocks_needed); + DXDaddress(&ea_buf->new_ea, blkno); + ea_buf->new_ea.flag = DXD_EXTENT; + DXDsize(&ea_buf->new_ea, min_size); + + ea_buf->flag = EA_EXTENT | EA_NEW; + + ea_buf->mp = get_metapage(inode, blkno, + blocks_needed << sb->s_blocksize_bits, + 1); + if (ea_buf->mp == NULL) { + dbFree(inode, blkno, (s64) blocks_needed); + rc = -EIO; + goto clean_up; + } + ea_buf->xattr = ea_buf->mp->data; + ea_buf->max_size = (min_size + sb->s_blocksize - 1) & + ~(sb->s_blocksize - 1); + if (ea_size == 0) + return 0; + if ((rc = ea_read(inode, ea_buf->xattr))) { + discard_metapage(ea_buf->mp); + dbFree(inode, blkno, (s64) blocks_needed); + goto clean_up; + } + goto size_check; + } + ea_buf->flag = EA_EXTENT; + ea_buf->mp = read_metapage(inode, addressDXD(&ji->ea), + lengthDXD(&ji->ea) << sb->s_blocksize_bits, + 1); + if (ea_buf->mp == NULL) { + rc = -EIO; + goto clean_up; + } + ea_buf->xattr = ea_buf->mp->data; + ea_buf->max_size = (ea_size + sb->s_blocksize - 1) & + ~(sb->s_blocksize - 1); + + size_check: + if (EALIST_SIZE(ea_buf->xattr) != ea_size) { + if (unlikely(EALIST_SIZE(ea_buf->xattr) > INT_MAX)) { + printk(KERN_ERR "ea_get: extended attribute size too large: %u > INT_MAX\n", + EALIST_SIZE(ea_buf->xattr)); + } else { + int size = clamp_t(int, ea_size, 0, EALIST_SIZE(ea_buf->xattr)); + + printk(KERN_ERR "ea_get: invalid extended attribute\n"); + print_hex_dump(KERN_ERR, "", DUMP_PREFIX_ADDRESS, 16, 1, + ea_buf->xattr, size, 1); + } + ea_release(inode, ea_buf); + rc = -EIO; + goto clean_up; + } + + return ea_size; + + clean_up: + /* Rollback quota allocation */ + if (quota_allocation) + dquot_free_block(inode, quota_allocation); + + return (rc); +} + +static void ea_release(struct inode *inode, struct ea_buffer *ea_buf) +{ + if (ea_buf->flag & EA_MALLOC) + kfree(ea_buf->xattr); + else if (ea_buf->flag & EA_EXTENT) { + assert(ea_buf->mp); + release_metapage(ea_buf->mp); + + if (ea_buf->flag & EA_NEW) + dbFree(inode, addressDXD(&ea_buf->new_ea), + lengthDXD(&ea_buf->new_ea)); + } +} + +static int ea_put(tid_t tid, struct inode *inode, struct ea_buffer *ea_buf, + int new_size) +{ + struct jfs_inode_info *ji = JFS_IP(inode); + unsigned long old_blocks, new_blocks; + int rc = 0; + + if (new_size == 0) { + ea_release(inode, ea_buf); + ea_buf = NULL; + } else if (ea_buf->flag & EA_INLINE) { + assert(new_size <= sizeof (ji->i_inline_ea)); + ji->mode2 &= ~INLINEEA; + ea_buf->new_ea.flag = DXD_INLINE; + DXDsize(&ea_buf->new_ea, new_size); + DXDaddress(&ea_buf->new_ea, 0); + DXDlength(&ea_buf->new_ea, 0); + } else if (ea_buf->flag & EA_MALLOC) { + rc = ea_write(inode, ea_buf->xattr, new_size, &ea_buf->new_ea); + kfree(ea_buf->xattr); + } else if (ea_buf->flag & EA_NEW) { + /* We have already allocated a new dxd */ + flush_metapage(ea_buf->mp); + } else { + /* ->xattr must point to original ea's metapage */ + rc = ea_write(inode, ea_buf->xattr, new_size, &ea_buf->new_ea); + discard_metapage(ea_buf->mp); + } + if (rc) + return rc; + + old_blocks = new_blocks = 0; + + if (ji->ea.flag & DXD_EXTENT) { + invalidate_dxd_metapages(inode, ji->ea); + old_blocks = lengthDXD(&ji->ea); + } + + if (ea_buf) { + txEA(tid, inode, &ji->ea, &ea_buf->new_ea); + if (ea_buf->new_ea.flag & DXD_EXTENT) { + new_blocks = lengthDXD(&ea_buf->new_ea); + if (ji->ea.flag & DXD_INLINE) + ji->mode2 |= INLINEEA; + } + ji->ea = ea_buf->new_ea; + } else { + txEA(tid, inode, &ji->ea, NULL); + if (ji->ea.flag & DXD_INLINE) + ji->mode2 |= INLINEEA; + ji->ea.flag = 0; + ji->ea.size = 0; + } + + /* If old blocks exist, they must be removed from quota allocation. */ + if (old_blocks) + dquot_free_block(inode, old_blocks); + + inode_set_ctime_current(inode); + + return 0; +} + +int __jfs_setxattr(tid_t tid, struct inode *inode, const char *name, + const void *value, size_t value_len, int flags) +{ + struct jfs_ea_list *ealist; + struct jfs_ea *ea, *old_ea = NULL, *next_ea = NULL; + struct ea_buffer ea_buf; + int old_ea_size = 0; + int xattr_size; + int new_size; + int namelen = strlen(name); + int found = 0; + int rc; + int length; + + down_write(&JFS_IP(inode)->xattr_sem); + + xattr_size = ea_get(inode, &ea_buf, 0); + if (xattr_size < 0) { + rc = xattr_size; + goto out; + } + + again: + ealist = (struct jfs_ea_list *) ea_buf.xattr; + new_size = sizeof (struct jfs_ea_list); + + if (xattr_size) { + for (ea = FIRST_EA(ealist); ea < END_EALIST(ealist); + ea = NEXT_EA(ea)) { + if ((namelen == ea->namelen) && + (memcmp(name, ea->name, namelen) == 0)) { + found = 1; + if (flags & XATTR_CREATE) { + rc = -EEXIST; + goto release; + } + old_ea = ea; + old_ea_size = EA_SIZE(ea); + next_ea = NEXT_EA(ea); + } else + new_size += EA_SIZE(ea); + } + } + + if (!found) { + if (flags & XATTR_REPLACE) { + rc = -ENODATA; + goto release; + } + if (value == NULL) { + rc = 0; + goto release; + } + } + if (value) + new_size += sizeof (struct jfs_ea) + namelen + 1 + value_len; + + if (new_size > ea_buf.max_size) { + /* + * We need to allocate more space for merged ea list. + * We should only have loop to again: once. + */ + ea_release(inode, &ea_buf); + xattr_size = ea_get(inode, &ea_buf, new_size); + if (xattr_size < 0) { + rc = xattr_size; + goto out; + } + goto again; + } + + /* Remove old ea of the same name */ + if (found) { + /* number of bytes following target EA */ + length = (char *) END_EALIST(ealist) - (char *) next_ea; + if (length > 0) + memmove(old_ea, next_ea, length); + xattr_size -= old_ea_size; + } + + /* Add new entry to the end */ + if (value) { + if (xattr_size == 0) + /* Completely new ea list */ + xattr_size = sizeof (struct jfs_ea_list); + + /* + * The size of EA value is limitted by on-disk format up to + * __le16, there would be an overflow if the size is equal + * to XATTR_SIZE_MAX (65536). In order to avoid this issue, + * we can pre-checkup the value size against USHRT_MAX, and + * return -E2BIG in this case, which is consistent with the + * VFS setxattr interface. + */ + if (value_len >= USHRT_MAX) { + rc = -E2BIG; + goto release; + } + + ea = (struct jfs_ea *) ((char *) ealist + xattr_size); + ea->flag = 0; + ea->namelen = namelen; + ea->valuelen = (cpu_to_le16(value_len)); + memcpy(ea->name, name, namelen); + ea->name[namelen] = 0; + if (value_len) + memcpy(&ea->name[namelen + 1], value, value_len); + xattr_size += EA_SIZE(ea); + } + + /* DEBUG - If we did this right, these number match */ + if (xattr_size != new_size) { + printk(KERN_ERR + "__jfs_setxattr: xattr_size = %d, new_size = %d\n", + xattr_size, new_size); + + rc = -EINVAL; + goto release; + } + + /* + * If we're left with an empty list, there's no ea + */ + if (new_size == sizeof (struct jfs_ea_list)) + new_size = 0; + + ealist->size = cpu_to_le32(new_size); + + rc = ea_put(tid, inode, &ea_buf, new_size); + + goto out; + release: + ea_release(inode, &ea_buf); + out: + up_write(&JFS_IP(inode)->xattr_sem); + + return rc; +} + +ssize_t __jfs_getxattr(struct inode *inode, const char *name, void *data, + size_t buf_size) +{ + struct jfs_ea_list *ealist; + struct jfs_ea *ea, *ealist_end; + struct ea_buffer ea_buf; + int xattr_size; + ssize_t size; + int namelen = strlen(name); + char *value; + + down_read(&JFS_IP(inode)->xattr_sem); + + xattr_size = ea_get(inode, &ea_buf, 0); + + if (xattr_size < 0) { + size = xattr_size; + goto out; + } + + if (xattr_size == 0) + goto not_found; + + ealist = (struct jfs_ea_list *) ea_buf.xattr; + ealist_end = END_EALIST(ealist); + + /* Find the named attribute */ + for (ea = FIRST_EA(ealist); ea < ealist_end; ea = NEXT_EA(ea)) { + if (unlikely(ea + 1 > ealist_end) || + unlikely(NEXT_EA(ea) > ealist_end)) { + size = -EUCLEAN; + goto release; + } + + if ((namelen == ea->namelen) && + memcmp(name, ea->name, namelen) == 0) { + /* Found it */ + size = le16_to_cpu(ea->valuelen); + if (!data) + goto release; + else if (size > buf_size) { + size = -ERANGE; + goto release; + } + value = ((char *) &ea->name) + ea->namelen + 1; + memcpy(data, value, size); + goto release; + } + } + not_found: + size = -ENODATA; + release: + ea_release(inode, &ea_buf); + out: + up_read(&JFS_IP(inode)->xattr_sem); + + return size; +} + +/* + * No special permissions are needed to list attributes except for trusted.* + */ +static inline int can_list(struct jfs_ea *ea) +{ + return (strncmp(ea->name, XATTR_TRUSTED_PREFIX, + XATTR_TRUSTED_PREFIX_LEN) || + capable(CAP_SYS_ADMIN)); +} + +ssize_t jfs_listxattr(struct dentry * dentry, char *data, size_t buf_size) +{ + struct inode *inode = d_inode(dentry); + char *buffer; + ssize_t size = 0; + int xattr_size; + struct jfs_ea_list *ealist; + struct jfs_ea *ea, *ealist_end; + struct ea_buffer ea_buf; + + down_read(&JFS_IP(inode)->xattr_sem); + + xattr_size = ea_get(inode, &ea_buf, 0); + if (xattr_size < 0) { + size = xattr_size; + goto out; + } + + if (xattr_size == 0) + goto release; + + ealist = (struct jfs_ea_list *) ea_buf.xattr; + ealist_end = END_EALIST(ealist); + + /* compute required size of list */ + for (ea = FIRST_EA(ealist); ea < ealist_end; ea = NEXT_EA(ea)) { + if (unlikely(ea + 1 > ealist_end) || + unlikely(NEXT_EA(ea) > ealist_end)) { + size = -EUCLEAN; + goto release; + } + + if (can_list(ea)) + size += name_size(ea) + 1; + } + + if (!data) + goto release; + + if (size > buf_size) { + size = -ERANGE; + goto release; + } + + /* Copy attribute names to buffer */ + buffer = data; + for (ea = FIRST_EA(ealist); ea < END_EALIST(ealist); ea = NEXT_EA(ea)) { + if (can_list(ea)) { + int namelen = copy_name(buffer, ea); + buffer += namelen + 1; + } + } + + release: + ea_release(inode, &ea_buf); + out: + up_read(&JFS_IP(inode)->xattr_sem); + return size; +} + +static int __jfs_xattr_set(struct inode *inode, const char *name, + const void *value, size_t size, int flags) +{ + struct jfs_inode_info *ji = JFS_IP(inode); + tid_t tid; + int rc; + + tid = txBegin(inode->i_sb, 0); + mutex_lock(&ji->commit_mutex); + rc = __jfs_setxattr(tid, inode, name, value, size, flags); + if (!rc) + rc = txCommit(tid, 1, &inode, 0); + txEnd(tid); + mutex_unlock(&ji->commit_mutex); + + return rc; +} + +static int jfs_xattr_get(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *value, size_t size) +{ + name = xattr_full_name(handler, name); + return __jfs_getxattr(inode, name, value, size); +} + +static int jfs_xattr_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +{ + name = xattr_full_name(handler, name); + return __jfs_xattr_set(inode, name, value, size, flags); +} + +static int jfs_xattr_get_os2(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *value, size_t size) +{ + if (is_known_namespace(name)) + return -EOPNOTSUPP; + return __jfs_getxattr(inode, name, value, size); +} + +static int jfs_xattr_set_os2(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +{ + if (is_known_namespace(name)) + return -EOPNOTSUPP; + return __jfs_xattr_set(inode, name, value, size, flags); +} + +static const struct xattr_handler jfs_user_xattr_handler = { + .prefix = XATTR_USER_PREFIX, + .get = jfs_xattr_get, + .set = jfs_xattr_set, +}; + +static const struct xattr_handler jfs_os2_xattr_handler = { + .prefix = XATTR_OS2_PREFIX, + .get = jfs_xattr_get_os2, + .set = jfs_xattr_set_os2, +}; + +static const struct xattr_handler jfs_security_xattr_handler = { + .prefix = XATTR_SECURITY_PREFIX, + .get = jfs_xattr_get, + .set = jfs_xattr_set, +}; + +static const struct xattr_handler jfs_trusted_xattr_handler = { + .prefix = XATTR_TRUSTED_PREFIX, + .get = jfs_xattr_get, + .set = jfs_xattr_set, +}; + +const struct xattr_handler * const jfs_xattr_handlers[] = { + &jfs_os2_xattr_handler, + &jfs_user_xattr_handler, + &jfs_security_xattr_handler, + &jfs_trusted_xattr_handler, + NULL, +}; + + +#ifdef CONFIG_JFS_SECURITY +static int jfs_initxattrs(struct inode *inode, const struct xattr *xattr_array, + void *fs_info) +{ + const struct xattr *xattr; + tid_t *tid = fs_info; + char *name; + int err = 0; + + for (xattr = xattr_array; xattr->name != NULL; xattr++) { + name = kmalloc(XATTR_SECURITY_PREFIX_LEN + + strlen(xattr->name) + 1, GFP_NOFS); + if (!name) { + err = -ENOMEM; + break; + } + strcpy(name, XATTR_SECURITY_PREFIX); + strcpy(name + XATTR_SECURITY_PREFIX_LEN, xattr->name); + + err = __jfs_setxattr(*tid, inode, name, + xattr->value, xattr->value_len, 0); + kfree(name); + if (err < 0) + break; + } + return err; +} + +int jfs_init_security(tid_t tid, struct inode *inode, struct inode *dir, + const struct qstr *qstr) +{ + return security_inode_init_security(inode, dir, qstr, + &jfs_initxattrs, &tid); +} +#endif diff --git a/fs/kernel_read_file.c b/fs/kernel_read_file.c new file mode 100644 index 000000000..9c2ba9240 --- /dev/null +++ b/fs/kernel_read_file.c @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include + +/** + * kernel_read_file() - read file contents into a kernel buffer + * + * @file: file to read from + * @offset: where to start reading from (see below). + * @buf: pointer to a "void *" buffer for reading into (if + * *@buf is NULL, a buffer will be allocated, and + * @buf_size will be ignored) + * @buf_size: size of buf, if already allocated. If @buf not + * allocated, this is the largest size to allocate. + * @file_size: if non-NULL, the full size of @file will be + * written here. + * @id: the kernel_read_file_id identifying the type of + * file contents being read (for LSMs to examine) + * + * @offset must be 0 unless both @buf and @file_size are non-NULL + * (i.e. the caller must be expecting to read partial file contents + * via an already-allocated @buf, in at most @buf_size chunks, and + * will be able to determine when the entire file was read by + * checking @file_size). This isn't a recommended way to read a + * file, though, since it is possible that the contents might + * change between calls to kernel_read_file(). + * + * Returns number of bytes read (no single read will be bigger + * than SSIZE_MAX), or negative on error. + * + */ +ssize_t kernel_read_file(struct file *file, loff_t offset, void **buf, + size_t buf_size, size_t *file_size, + enum kernel_read_file_id id) +{ + loff_t i_size, pos; + ssize_t copied; + void *allocated = NULL; + bool whole_file; + int ret; + + if (offset != 0 && (!*buf || !file_size)) + return -EINVAL; + + if (!S_ISREG(file_inode(file)->i_mode)) + return -EINVAL; + + ret = deny_write_access(file); + if (ret) + return ret; + + i_size = i_size_read(file_inode(file)); + if (i_size <= 0) { + ret = -EINVAL; + goto out; + } + /* The file is too big for sane activities. */ + if (i_size > SSIZE_MAX) { + ret = -EFBIG; + goto out; + } + /* The entire file cannot be read in one buffer. */ + if (!file_size && offset == 0 && i_size > buf_size) { + ret = -EFBIG; + goto out; + } + + whole_file = (offset == 0 && i_size <= buf_size); + ret = security_kernel_read_file(file, id, whole_file); + if (ret) + goto out; + + if (file_size) + *file_size = i_size; + + if (!*buf) + *buf = allocated = vmalloc(i_size); + if (!*buf) { + ret = -ENOMEM; + goto out; + } + + pos = offset; + copied = 0; + while (copied < buf_size) { + ssize_t bytes; + size_t wanted = min_t(size_t, buf_size - copied, + i_size - pos); + + bytes = kernel_read(file, *buf + copied, wanted, &pos); + if (bytes < 0) { + ret = bytes; + goto out_free; + } + + if (bytes == 0) + break; + copied += bytes; + } + + if (whole_file) { + if (pos != i_size) { + ret = -EIO; + goto out_free; + } + + ret = security_kernel_post_read_file(file, *buf, i_size, id); + } + +out_free: + if (ret < 0) { + if (allocated) { + vfree(*buf); + *buf = NULL; + } + } + +out: + allow_write_access(file); + return ret == 0 ? copied : ret; +} +EXPORT_SYMBOL_GPL(kernel_read_file); + +ssize_t kernel_read_file_from_path(const char *path, loff_t offset, void **buf, + size_t buf_size, size_t *file_size, + enum kernel_read_file_id id) +{ + struct file *file; + ssize_t ret; + + if (!path || !*path) + return -EINVAL; + + file = filp_open(path, O_RDONLY, 0); + if (IS_ERR(file)) + return PTR_ERR(file); + + ret = kernel_read_file(file, offset, buf, buf_size, file_size, id); + fput(file); + return ret; +} +EXPORT_SYMBOL_GPL(kernel_read_file_from_path); + +ssize_t kernel_read_file_from_path_initns(const char *path, loff_t offset, + void **buf, size_t buf_size, + size_t *file_size, + enum kernel_read_file_id id) +{ + struct file *file; + ssize_t ret; + + if (!path || !*path) + return -EINVAL; + + scoped_with_init_fs() + file = filp_open(path, O_RDONLY, 0); + if (IS_ERR(file)) + return PTR_ERR(file); + + ret = kernel_read_file(file, offset, buf, buf_size, file_size, id); + fput(file); + return ret; +} +EXPORT_SYMBOL_GPL(kernel_read_file_from_path_initns); + +ssize_t kernel_read_file_from_fd(int fd, loff_t offset, void **buf, + size_t buf_size, size_t *file_size, + enum kernel_read_file_id id) +{ + CLASS(fd, f)(fd); + + if (fd_empty(f) || !(fd_file(f)->f_mode & FMODE_READ)) + return -EBADF; + + return kernel_read_file(fd_file(f), offset, buf, buf_size, file_size, id); +} +EXPORT_SYMBOL_GPL(kernel_read_file_from_fd); diff --git a/fs/kernfs/Kconfig b/fs/kernfs/Kconfig new file mode 100644 index 000000000..e7f09105f --- /dev/null +++ b/fs/kernfs/Kconfig @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# KERNFS should be selected by its users +# + +config KERNFS + bool + default n diff --git a/fs/kernfs/Makefile b/fs/kernfs/Makefile new file mode 100644 index 000000000..4ca54ff54 --- /dev/null +++ b/fs/kernfs/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the kernfs pseudo filesystem +# + +obj-y := mount.o inode.o dir.o file.o symlink.o diff --git a/fs/kernfs/dir.c b/fs/kernfs/dir.c new file mode 100644 index 000000000..82bbaeb32 --- /dev/null +++ b/fs/kernfs/dir.c @@ -0,0 +1,2015 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * fs/kernfs/dir.c - kernfs directory implementation + * + * Copyright (c) 2001-3 Patrick Mochel + * Copyright (c) 2007 SUSE Linux Products GmbH + * Copyright (c) 2007, 2013 Tejun Heo + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kernfs-internal.h" + +/* + * Don't use rename_lock to piggy back on pr_cont_buf. We don't want to + * call pr_cont() while holding rename_lock. Because sometimes pr_cont() + * will perform wakeups when releasing console_sem. Holding rename_lock + * will introduce deadlock if the scheduler reads the kernfs_name in the + * wakeup path. + */ +static DEFINE_SPINLOCK(kernfs_pr_cont_lock); +static char kernfs_pr_cont_buf[PATH_MAX]; /* protected by pr_cont_lock */ + +#define rb_to_kn(X) rb_entry((X), struct kernfs_node, rb) + +static bool __kernfs_active(struct kernfs_node *kn) +{ + return atomic_read(&kn->active) >= 0; +} + +static bool kernfs_active(struct kernfs_node *kn) +{ + lockdep_assert_held(&kernfs_root(kn)->kernfs_rwsem); + return __kernfs_active(kn); +} + +static bool kernfs_lockdep(struct kernfs_node *kn) +{ +#ifdef CONFIG_DEBUG_LOCK_ALLOC + return kn->flags & KERNFS_LOCKDEP; +#else + return false; +#endif +} + +/* kernfs_node_depth - compute depth from @from to @to */ +static size_t kernfs_depth(struct kernfs_node *from, struct kernfs_node *to) +{ + size_t depth = 0; + + while (rcu_dereference(to->__parent) && to != from) { + depth++; + to = rcu_dereference(to->__parent); + } + return depth; +} + +static struct kernfs_node *kernfs_common_ancestor(struct kernfs_node *a, + struct kernfs_node *b) +{ + size_t da, db; + struct kernfs_root *ra = kernfs_root(a), *rb = kernfs_root(b); + + if (ra != rb) + return NULL; + + da = kernfs_depth(ra->kn, a); + db = kernfs_depth(rb->kn, b); + + while (da > db) { + a = rcu_dereference(a->__parent); + da--; + } + while (db > da) { + b = rcu_dereference(b->__parent); + db--; + } + + /* worst case b and a will be the same at root */ + while (b != a) { + b = rcu_dereference(b->__parent); + a = rcu_dereference(a->__parent); + } + + return a; +} + +/** + * kernfs_path_from_node_locked - find a pseudo-absolute path to @kn_to, + * where kn_from is treated as root of the path. + * @kn_from: kernfs node which should be treated as root for the path + * @kn_to: kernfs node to which path is needed + * @buf: buffer to copy the path into + * @buflen: size of @buf + * + * We need to handle couple of scenarios here: + * [1] when @kn_from is an ancestor of @kn_to at some level + * kn_from: /n1/n2/n3 + * kn_to: /n1/n2/n3/n4/n5 + * result: /n4/n5 + * + * [2] when @kn_from is on a different hierarchy and we need to find common + * ancestor between @kn_from and @kn_to. + * kn_from: /n1/n2/n3/n4 + * kn_to: /n1/n2/n5 + * result: /../../n5 + * OR + * kn_from: /n1/n2/n3/n4/n5 [depth=5] + * kn_to: /n1/n2/n3 [depth=3] + * result: /../.. + * + * [3] when @kn_to is %NULL result will be "(null)" + * + * Return: the length of the constructed path. If the path would have been + * greater than @buflen, @buf contains the truncated path with the trailing + * '\0'. On error, -errno is returned. + */ +static int kernfs_path_from_node_locked(struct kernfs_node *kn_to, + struct kernfs_node *kn_from, + char *buf, size_t buflen) +{ + struct kernfs_node *kn, *common; + const char parent_str[] = "/.."; + size_t depth_from, depth_to, len = 0; + ssize_t copied; + int i, j; + + if (!kn_to) + return strscpy(buf, "(null)", buflen); + + if (!kn_from) + kn_from = kernfs_root(kn_to)->kn; + + if (kn_from == kn_to) + return strscpy(buf, "/", buflen); + + common = kernfs_common_ancestor(kn_from, kn_to); + if (WARN_ON(!common)) + return -EINVAL; + + depth_to = kernfs_depth(common, kn_to); + depth_from = kernfs_depth(common, kn_from); + + buf[0] = '\0'; + + for (i = 0; i < depth_from; i++) { + copied = strscpy(buf + len, parent_str, buflen - len); + if (copied < 0) + return copied; + len += copied; + } + + /* Calculate how many bytes we need for the rest */ + for (i = depth_to - 1; i >= 0; i--) { + const char *name; + + for (kn = kn_to, j = 0; j < i; j++) + kn = rcu_dereference(kn->__parent); + + name = rcu_dereference(kn->name); + len += scnprintf(buf + len, buflen - len, "/%s", name); + } + + return len; +} + +/** + * kernfs_name - obtain the name of a given node + * @kn: kernfs_node of interest + * @buf: buffer to copy @kn's name into + * @buflen: size of @buf + * + * Copies the name of @kn into @buf of @buflen bytes. The behavior is + * similar to strscpy(). + * + * Fills buffer with "(null)" if @kn is %NULL. + * + * Return: the resulting length of @buf. If @buf isn't long enough, + * it's filled up to @buflen-1 and nul terminated, and returns -E2BIG. + * + * This function can be called from any context. + */ +int kernfs_name(struct kernfs_node *kn, char *buf, size_t buflen) +{ + struct kernfs_node *kn_parent; + + if (!kn) + return strscpy(buf, "(null)", buflen); + + guard(rcu)(); + /* + * KERNFS_ROOT_INVARIANT_PARENT is ignored here. The name is RCU freed and + * the parent is either existing or not. + */ + kn_parent = rcu_dereference(kn->__parent); + return strscpy(buf, kn_parent ? rcu_dereference(kn->name) : "/", buflen); +} + +/** + * kernfs_path_from_node - build path of node @to relative to @from. + * @from: parent kernfs_node relative to which we need to build the path + * @to: kernfs_node of interest + * @buf: buffer to copy @to's path into + * @buflen: size of @buf + * + * Builds @to's path relative to @from in @buf. @from and @to must + * be on the same kernfs-root. If @from is not parent of @to, then a relative + * path (which includes '..'s) as needed to reach from @from to @to is + * returned. + * + * Return: the length of the constructed path. If the path would have been + * greater than @buflen, @buf contains the truncated path with the trailing + * '\0'. On error, -errno is returned. + */ +int kernfs_path_from_node(struct kernfs_node *to, struct kernfs_node *from, + char *buf, size_t buflen) +{ + struct kernfs_root *root; + + guard(rcu)(); + if (to) { + root = kernfs_root(to); + if (!(root->flags & KERNFS_ROOT_INVARIANT_PARENT)) { + guard(read_lock_irqsave)(&root->kernfs_rename_lock); + return kernfs_path_from_node_locked(to, from, buf, buflen); + } + } + return kernfs_path_from_node_locked(to, from, buf, buflen); +} +EXPORT_SYMBOL_GPL(kernfs_path_from_node); + +/** + * pr_cont_kernfs_name - pr_cont name of a kernfs_node + * @kn: kernfs_node of interest + * + * This function can be called from any context. + */ +void pr_cont_kernfs_name(struct kernfs_node *kn) +{ + unsigned long flags; + + spin_lock_irqsave(&kernfs_pr_cont_lock, flags); + + kernfs_name(kn, kernfs_pr_cont_buf, sizeof(kernfs_pr_cont_buf)); + pr_cont("%s", kernfs_pr_cont_buf); + + spin_unlock_irqrestore(&kernfs_pr_cont_lock, flags); +} + +/** + * pr_cont_kernfs_path - pr_cont path of a kernfs_node + * @kn: kernfs_node of interest + * + * This function can be called from any context. + */ +void pr_cont_kernfs_path(struct kernfs_node *kn) +{ + unsigned long flags; + int sz; + + spin_lock_irqsave(&kernfs_pr_cont_lock, flags); + + sz = kernfs_path_from_node(kn, NULL, kernfs_pr_cont_buf, + sizeof(kernfs_pr_cont_buf)); + if (sz < 0) { + if (sz == -E2BIG) + pr_cont("(name too long)"); + else + pr_cont("(error)"); + goto out; + } + + pr_cont("%s", kernfs_pr_cont_buf); + +out: + spin_unlock_irqrestore(&kernfs_pr_cont_lock, flags); +} + +/** + * kernfs_get_parent - determine the parent node and pin it + * @kn: kernfs_node of interest + * + * Determines @kn's parent, pins and returns it. This function can be + * called from any context. + * + * Return: parent node of @kn + */ +struct kernfs_node *kernfs_get_parent(struct kernfs_node *kn) +{ + struct kernfs_node *parent; + struct kernfs_root *root; + unsigned long flags; + + root = kernfs_root(kn); + read_lock_irqsave(&root->kernfs_rename_lock, flags); + parent = kernfs_parent(kn); + kernfs_get(parent); + read_unlock_irqrestore(&root->kernfs_rename_lock, flags); + + return parent; +} + +/* + * kernfs_ns_id - return the namespace id for a given namespace + * @ns: namespace tag (may be NULL) + * + * Use the 64-bit namespace id instead of raw pointers for hashing + * and comparison to avoid leaking kernel addresses to userspace. + */ +static u64 kernfs_ns_id(const struct ns_common *ns) +{ + return ns ? ns->ns_id : 0; +} + +/** + * kernfs_name_hash - calculate hash of @ns + @name + * @name: Null terminated string to hash + * @ns: Namespace tag to hash + * + * Return: 31-bit hash of ns + name (so it fits in an off_t) + */ +static unsigned int kernfs_name_hash(const char *name, + const struct ns_common *ns) +{ + unsigned long hash = init_name_hash(kernfs_ns_id(ns)); + + while (*name) + hash = partial_name_hash(*name++, hash); + hash = end_name_hash(hash); + hash &= 0x7fffffffU; + /* Reserve hash numbers 0, 1 and INT_MAX for magic directory entries */ + if (hash < 2) + hash += 2; + if (hash >= INT_MAX) + hash = INT_MAX - 1; + return hash; +} + +static int kernfs_name_compare(unsigned int hash, const char *name, + const struct ns_common *ns, const struct kernfs_node *kn) +{ + u64 ns_id = kernfs_ns_id(ns); + u64 kn_ns_id = kernfs_ns_id(kn->ns); + + if (hash < kn->hash) + return -1; + if (hash > kn->hash) + return 1; + if (ns_id < kn_ns_id) + return -1; + if (ns_id > kn_ns_id) + return 1; + return strcmp(name, kernfs_rcu_name(kn)); +} + +static int kernfs_sd_compare(const struct kernfs_node *left, + const struct kernfs_node *right) +{ + return kernfs_name_compare(left->hash, kernfs_rcu_name(left), left->ns, right); +} + +/** + * kernfs_link_sibling - link kernfs_node into sibling rbtree + * @kn: kernfs_node of interest + * + * Link @kn into its sibling rbtree which starts from + * @kn->parent->dir.children. + * + * Locking: + * kernfs_rwsem held exclusive + * + * Return: + * %0 on success, -EEXIST on failure. + */ +static int kernfs_link_sibling(struct kernfs_node *kn) +{ + struct rb_node *parent = NULL; + struct kernfs_node *kn_parent; + struct rb_node **node; + + kn_parent = kernfs_parent(kn); + node = &kn_parent->dir.children.rb_node; + + while (*node) { + struct kernfs_node *pos; + int result; + + pos = rb_to_kn(*node); + parent = *node; + result = kernfs_sd_compare(kn, pos); + if (result < 0) + node = &pos->rb.rb_left; + else if (result > 0) + node = &pos->rb.rb_right; + else + return -EEXIST; + } + + /* add new node and rebalance the tree */ + rb_link_node(&kn->rb, parent, node); + rb_insert_color(&kn->rb, &kn_parent->dir.children); + + /* successfully added, account subdir number */ + down_write(&kernfs_root(kn)->kernfs_iattr_rwsem); + if (kernfs_type(kn) == KERNFS_DIR) + kn_parent->dir.subdirs++; + kernfs_inc_rev(kn_parent); + up_write(&kernfs_root(kn)->kernfs_iattr_rwsem); + + return 0; +} + +/** + * kernfs_unlink_sibling - unlink kernfs_node from sibling rbtree + * @kn: kernfs_node of interest + * + * Try to unlink @kn from its sibling rbtree which starts from + * kn->parent->dir.children. + * + * Return: %true if @kn was actually removed, + * %false if @kn wasn't on the rbtree. + * + * Locking: + * kernfs_rwsem held exclusive + */ +static bool kernfs_unlink_sibling(struct kernfs_node *kn) +{ + struct kernfs_node *kn_parent; + + if (RB_EMPTY_NODE(&kn->rb)) + return false; + + kn_parent = kernfs_parent(kn); + down_write(&kernfs_root(kn)->kernfs_iattr_rwsem); + if (kernfs_type(kn) == KERNFS_DIR) + kn_parent->dir.subdirs--; + kernfs_inc_rev(kn_parent); + up_write(&kernfs_root(kn)->kernfs_iattr_rwsem); + + rb_erase(&kn->rb, &kn_parent->dir.children); + RB_CLEAR_NODE(&kn->rb); + return true; +} + +/** + * kernfs_get_active - get an active reference to kernfs_node + * @kn: kernfs_node to get an active reference to + * + * Get an active reference of @kn. This function is noop if @kn + * is %NULL. + * + * Return: + * Pointer to @kn on success, %NULL on failure. + */ +struct kernfs_node *kernfs_get_active(struct kernfs_node *kn) +{ + if (unlikely(!kn)) + return NULL; + + if (!atomic_inc_unless_negative(&kn->active)) + return NULL; + + if (kernfs_lockdep(kn)) + rwsem_acquire_read(&kn->dep_map, 0, 1, _RET_IP_); + return kn; +} + +/** + * kernfs_put_active - put an active reference to kernfs_node + * @kn: kernfs_node to put an active reference to + * + * Put an active reference to @kn. This function is noop if @kn + * is %NULL. + */ +void kernfs_put_active(struct kernfs_node *kn) +{ + int v; + + if (unlikely(!kn)) + return; + + if (kernfs_lockdep(kn)) + rwsem_release(&kn->dep_map, _RET_IP_); + v = atomic_dec_return(&kn->active); + if (likely(v != KN_DEACTIVATED_BIAS)) + return; + + wake_up_all(&kernfs_root(kn)->deactivate_waitq); +} + +/** + * kernfs_drain - drain kernfs_node + * @kn: kernfs_node to drain + * @drop_supers: Set to true if this function is called with the + * kernfs_supers_rwsem locked. + * + * Drain existing usages and nuke all existing mmaps of @kn. Multiple + * removers may invoke this function concurrently on @kn and all will + * return after draining is complete. + */ +static void kernfs_drain(struct kernfs_node *kn, bool drop_supers) + __releases(&kernfs_root(kn)->kernfs_rwsem) + __acquires(&kernfs_root(kn)->kernfs_rwsem) +{ + struct kernfs_root *root = kernfs_root(kn); + + lockdep_assert_held_write(&root->kernfs_rwsem); + WARN_ON_ONCE(kernfs_active(kn)); + + /* + * Skip draining if already fully drained. This avoids draining and its + * lockdep annotations for nodes which have never been activated + * allowing embedding kernfs_remove() in create error paths without + * worrying about draining. + */ + if (atomic_read(&kn->active) == KN_DEACTIVATED_BIAS && + !kernfs_should_drain_open_files(kn)) + return; + + up_write(&root->kernfs_rwsem); + if (drop_supers) + up_read(&root->kernfs_supers_rwsem); + + if (kernfs_lockdep(kn)) { + rwsem_acquire(&kn->dep_map, 0, 0, _RET_IP_); + if (atomic_read(&kn->active) != KN_DEACTIVATED_BIAS) + lock_contended(&kn->dep_map, _RET_IP_); + } + + wait_event(root->deactivate_waitq, + atomic_read(&kn->active) == KN_DEACTIVATED_BIAS); + + if (kernfs_lockdep(kn)) { + lock_acquired(&kn->dep_map, _RET_IP_); + rwsem_release(&kn->dep_map, _RET_IP_); + } + + if (kernfs_should_drain_open_files(kn)) + kernfs_drain_open_files(kn); + + if (drop_supers) + down_read(&root->kernfs_supers_rwsem); + down_write(&root->kernfs_rwsem); +} + +/** + * kernfs_get - get a reference count on a kernfs_node + * @kn: the target kernfs_node + */ +void kernfs_get(struct kernfs_node *kn) +{ + if (kn) { + WARN_ON(!atomic_read(&kn->count)); + atomic_inc(&kn->count); + } +} +EXPORT_SYMBOL_GPL(kernfs_get); + +static void kernfs_free_rcu(struct rcu_head *rcu) +{ + struct kernfs_node *kn = container_of(rcu, struct kernfs_node, rcu); + + /* If the whole node goes away, then name can't be used outside */ + kfree_const(rcu_access_pointer(kn->name)); + + if (kn->iattr) + kmem_cache_free(kernfs_iattrs_cache, kn->iattr); + + kmem_cache_free(kernfs_node_cache, kn); +} + +/** + * kernfs_put - put a reference count on a kernfs_node + * @kn: the target kernfs_node + * + * Put a reference count of @kn and destroy it if it reached zero. + */ +void kernfs_put(struct kernfs_node *kn) +{ + struct kernfs_node *parent; + struct kernfs_root *root; + + if (!kn || !atomic_dec_and_test(&kn->count)) + return; + root = kernfs_root(kn); + repeat: + /* + * Moving/renaming is always done while holding reference. + * kn->parent won't change beneath us. + */ + parent = kernfs_parent(kn); + + if (atomic_read(&kn->active) != KN_DEACTIVATED_BIAS) { + guard(rcu)(); + WARN_ONCE(1, + "kernfs_put: %s/%s: released with incorrect active_ref %d\n", + parent ? rcu_dereference(parent->name) : "", + rcu_dereference(kn->name), atomic_read(&kn->active)); + } + + if (kernfs_type(kn) == KERNFS_LINK) + kernfs_put(kn->symlink.target_kn); + + if (kn->iattr) + simple_xattrs_free(&root->xa_cache, &kn->iattr->xattrs, NULL); + + spin_lock(&root->kernfs_idr_lock); + idr_remove(&root->ino_idr, (u32)kernfs_ino(kn)); + spin_unlock(&root->kernfs_idr_lock); + + call_rcu(&kn->rcu, kernfs_free_rcu); + + kn = parent; + if (kn) { + if (atomic_dec_and_test(&kn->count)) + goto repeat; + } else { + /* just released the root kn, free @root too */ + idr_destroy(&root->ino_idr); + simple_xattr_cache_cleanup(&root->xa_cache); + kfree_rcu(root, rcu); + } +} +EXPORT_SYMBOL_GPL(kernfs_put); + +/** + * kernfs_node_from_dentry - determine kernfs_node associated with a dentry + * @dentry: the dentry in question + * + * Return: the kernfs_node associated with @dentry. If @dentry is not a + * kernfs one, %NULL is returned. + * + * While the returned kernfs_node will stay accessible as long as @dentry + * is accessible, the returned node can be in any state and the caller is + * fully responsible for determining what's accessible. + */ +struct kernfs_node *kernfs_node_from_dentry(struct dentry *dentry) +{ + if (dentry->d_sb->s_op == &kernfs_sops) + return kernfs_dentry_node(dentry); + return NULL; +} + +static struct kernfs_node *__kernfs_new_node(struct kernfs_root *root, + struct kernfs_node *parent, + const char *name, umode_t mode, + kuid_t uid, kgid_t gid, + unsigned flags) +{ + struct kernfs_node *kn; + u32 id_highbits; + int ret; + + name = kstrdup_const(name, GFP_KERNEL); + if (!name) + return NULL; + + kn = kmem_cache_zalloc(kernfs_node_cache, GFP_KERNEL); + if (!kn) + goto err_out1; + + idr_preload(GFP_KERNEL); + spin_lock(&root->kernfs_idr_lock); + ret = idr_alloc_cyclic(&root->ino_idr, kn, 1, 0, GFP_ATOMIC); + if (ret >= 0 && ret < root->last_id_lowbits) + root->id_highbits++; + id_highbits = root->id_highbits; + root->last_id_lowbits = ret; + spin_unlock(&root->kernfs_idr_lock); + idr_preload_end(); + if (ret < 0) + goto err_out2; + + kn->id = (u64)id_highbits << 32 | ret; + + atomic_set(&kn->count, 1); + atomic_set(&kn->active, KN_DEACTIVATED_BIAS); + RB_CLEAR_NODE(&kn->rb); + + rcu_assign_pointer(kn->name, name); + kn->mode = mode; + kn->flags = flags; + + if (!uid_eq(uid, GLOBAL_ROOT_UID) || !gid_eq(gid, GLOBAL_ROOT_GID)) { + struct iattr iattr = { + .ia_valid = ATTR_UID | ATTR_GID, + .ia_uid = uid, + .ia_gid = gid, + }; + + ret = __kernfs_setattr(kn, &iattr); + if (ret < 0) + goto err_out3; + } + + if (parent) { + kernfs_get(parent); + rcu_assign_pointer(kn->__parent, parent); + + ret = security_kernfs_init_security(parent, kn); + if (ret) + goto err_out4; + } + + return kn; + + err_out4: + RCU_INIT_POINTER(kn->__parent, NULL); + kernfs_put(parent); + if (kn->iattr) { + simple_xattrs_free(&root->xa_cache, &kn->iattr->xattrs, NULL); + kmem_cache_free(kernfs_iattrs_cache, kn->iattr); + } + err_out3: + spin_lock(&root->kernfs_idr_lock); + idr_remove(&root->ino_idr, (u32)kernfs_ino(kn)); + spin_unlock(&root->kernfs_idr_lock); + err_out2: + kmem_cache_free(kernfs_node_cache, kn); + err_out1: + kfree_const(name); + return NULL; +} + +struct kernfs_node *kernfs_new_node(struct kernfs_node *parent, + const char *name, umode_t mode, + kuid_t uid, kgid_t gid, + unsigned flags) +{ + struct kernfs_node *kn; + + if (parent->mode & S_ISGID) { + /* this code block imitates inode_init_owner() for + * kernfs + */ + + if (parent->iattr) + gid = parent->iattr->ia_gid; + + if (flags & KERNFS_DIR) + mode |= S_ISGID; + } + + kn = __kernfs_new_node(kernfs_root(parent), parent, + name, mode, uid, gid, flags); + return kn; +} + +/* + * kernfs_find_and_get_node_by_id - get kernfs_node from node id + * @root: the kernfs root + * @id: the target node id + * + * @id's lower 32bits encode ino and upper gen. If the gen portion is + * zero, all generations are matched. + * + * Return: %NULL on failure, + * otherwise a kernfs node with reference counter incremented. + */ +struct kernfs_node *kernfs_find_and_get_node_by_id(struct kernfs_root *root, + u64 id) +{ + struct kernfs_node *kn; + ino_t ino = kernfs_id_ino(id); + u32 gen = kernfs_id_gen(id); + + rcu_read_lock(); + + kn = idr_find(&root->ino_idr, (u32)ino); + if (!kn) + goto err_unlock; + + if (sizeof(ino_t) >= sizeof(u64)) { + /* we looked up with the low 32bits, compare the whole */ + if (kernfs_ino(kn) != ino) + goto err_unlock; + } else { + /* 0 matches all generations */ + if (unlikely(gen && kernfs_gen(kn) != gen)) + goto err_unlock; + } + + /* + * We should fail if @kn has never been activated and guarantee success + * if the caller knows that @kn is active. Both can be achieved by + * __kernfs_active() which tests @kn->active without kernfs_rwsem. + */ + if (unlikely(!__kernfs_active(kn) || !atomic_inc_not_zero(&kn->count))) + goto err_unlock; + + rcu_read_unlock(); + return kn; +err_unlock: + rcu_read_unlock(); + return NULL; +} + +/** + * kernfs_add_one - add kernfs_node to parent without warning + * @kn: kernfs_node to be added + * + * The caller must already have initialized @kn->parent. This + * function increments nlink of the parent's inode if @kn is a + * directory and link into the children list of the parent. + * + * Return: + * %0 on success, -EEXIST if entry with the given name already + * exists. + */ +int kernfs_add_one(struct kernfs_node *kn) +{ + struct kernfs_root *root = kernfs_root(kn); + struct kernfs_iattrs *ps_iattr; + struct kernfs_node *parent; + bool has_ns; + int ret; + + down_write(&root->kernfs_rwsem); + parent = kernfs_parent(kn); + + ret = -EINVAL; + has_ns = kernfs_ns_enabled(parent); + if (WARN(has_ns != (bool)kn->ns, KERN_WARNING "kernfs: ns %s in '%s' for '%s'\n", + has_ns ? "required" : "invalid", + kernfs_rcu_name(parent), kernfs_rcu_name(kn))) + goto out_unlock; + + if (kernfs_type(parent) != KERNFS_DIR) + goto out_unlock; + + ret = -ENOENT; + if (parent->flags & (KERNFS_REMOVING | KERNFS_EMPTY_DIR)) + goto out_unlock; + + kn->hash = kernfs_name_hash(kernfs_rcu_name(kn), kn->ns); + + ret = kernfs_link_sibling(kn); + if (ret) + goto out_unlock; + + /* Update timestamps on the parent */ + down_write(&root->kernfs_iattr_rwsem); + + ps_iattr = parent->iattr; + if (ps_iattr) { + ktime_get_real_ts64(&ps_iattr->ia_ctime); + ps_iattr->ia_mtime = ps_iattr->ia_ctime; + } + + up_write(&root->kernfs_iattr_rwsem); + up_write(&root->kernfs_rwsem); + + /* + * Activate the new node unless CREATE_DEACTIVATED is requested. + * If not activated here, the kernfs user is responsible for + * activating the node with kernfs_activate(). A node which hasn't + * been activated is not visible to userland and its removal won't + * trigger deactivation. + */ + if (!(kernfs_root(kn)->flags & KERNFS_ROOT_CREATE_DEACTIVATED)) + kernfs_activate(kn); + return 0; + +out_unlock: + up_write(&root->kernfs_rwsem); + return ret; +} + +/** + * kernfs_find_ns - find kernfs_node with the given name + * @parent: kernfs_node to search under + * @name: name to look for + * @ns: the namespace tag to use + * + * Look for kernfs_node with name @name under @parent. + * + * Return: pointer to the found kernfs_node on success, %NULL on failure. + */ +static struct kernfs_node *kernfs_find_ns(struct kernfs_node *parent, + const unsigned char *name, + const struct ns_common *ns) +{ + struct rb_node *node = parent->dir.children.rb_node; + bool has_ns = kernfs_ns_enabled(parent); + unsigned int hash; + + lockdep_assert_held(&kernfs_root(parent)->kernfs_rwsem); + + if (has_ns != (bool)ns) { + WARN(1, KERN_WARNING "kernfs: ns %s in '%s' for '%s'\n", + has_ns ? "required" : "invalid", kernfs_rcu_name(parent), name); + return NULL; + } + + hash = kernfs_name_hash(name, ns); + while (node) { + struct kernfs_node *kn; + int result; + + kn = rb_to_kn(node); + result = kernfs_name_compare(hash, name, ns, kn); + if (result < 0) + node = node->rb_left; + else if (result > 0) + node = node->rb_right; + else + return kn; + } + return NULL; +} + +static struct kernfs_node *kernfs_walk_ns(struct kernfs_node *parent, + const unsigned char *path, + const struct ns_common *ns) +{ + ssize_t len; + char *p, *name; + + lockdep_assert_held_read(&kernfs_root(parent)->kernfs_rwsem); + + spin_lock_irq(&kernfs_pr_cont_lock); + + len = strscpy(kernfs_pr_cont_buf, path, sizeof(kernfs_pr_cont_buf)); + + if (len < 0) { + spin_unlock_irq(&kernfs_pr_cont_lock); + return NULL; + } + + p = kernfs_pr_cont_buf; + + while ((name = strsep(&p, "/")) && parent) { + if (*name == '\0') + continue; + parent = kernfs_find_ns(parent, name, ns); + } + + spin_unlock_irq(&kernfs_pr_cont_lock); + + return parent; +} + +/** + * kernfs_find_and_get_ns - find and get kernfs_node with the given name + * @parent: kernfs_node to search under + * @name: name to look for + * @ns: the namespace tag to use + * + * Look for kernfs_node with name @name under @parent and get a reference + * if found. This function may sleep. + * + * Return: pointer to the found kernfs_node on success, %NULL on failure. + */ +struct kernfs_node *kernfs_find_and_get_ns(struct kernfs_node *parent, + const char *name, + const struct ns_common *ns) +{ + struct kernfs_node *kn; + struct kernfs_root *root = kernfs_root(parent); + + down_read(&root->kernfs_rwsem); + kn = kernfs_find_ns(parent, name, ns); + kernfs_get(kn); + up_read(&root->kernfs_rwsem); + + return kn; +} +EXPORT_SYMBOL_GPL(kernfs_find_and_get_ns); + +/** + * kernfs_walk_and_get_ns - find and get kernfs_node with the given path + * @parent: kernfs_node to search under + * @path: path to look for + * @ns: the namespace tag to use + * + * Look for kernfs_node with path @path under @parent and get a reference + * if found. This function may sleep. + * + * Return: pointer to the found kernfs_node on success, %NULL on failure. + */ +struct kernfs_node *kernfs_walk_and_get_ns(struct kernfs_node *parent, + const char *path, + const struct ns_common *ns) +{ + struct kernfs_node *kn; + struct kernfs_root *root = kernfs_root(parent); + + down_read(&root->kernfs_rwsem); + kn = kernfs_walk_ns(parent, path, ns); + kernfs_get(kn); + up_read(&root->kernfs_rwsem); + + return kn; +} + +unsigned int kernfs_root_flags(struct kernfs_node *kn) +{ + return kernfs_root(kn)->flags; +} + +/** + * kernfs_create_root - create a new kernfs hierarchy + * @scops: optional syscall operations for the hierarchy + * @flags: KERNFS_ROOT_* flags + * @priv: opaque data associated with the new directory + * + * Return: the root of the new hierarchy on success, ERR_PTR() value on + * failure. + */ +struct kernfs_root *kernfs_create_root(struct kernfs_syscall_ops *scops, + unsigned int flags, void *priv) +{ + struct kernfs_root *root; + struct kernfs_node *kn; + + root = kzalloc_obj(*root); + if (!root) + return ERR_PTR(-ENOMEM); + + idr_init(&root->ino_idr); + spin_lock_init(&root->kernfs_idr_lock); + init_rwsem(&root->kernfs_rwsem); + init_rwsem(&root->kernfs_iattr_rwsem); + init_rwsem(&root->kernfs_supers_rwsem); + INIT_LIST_HEAD(&root->supers); + rwlock_init(&root->kernfs_rename_lock); + + /* + * On 64bit ino setups, id is ino. On 32bit, low 32bits are ino. + * High bits generation. The starting value for both ino and + * genenration is 1. Initialize upper 32bit allocation + * accordingly. + */ + if (sizeof(ino_t) >= sizeof(u64)) + root->id_highbits = 0; + else + root->id_highbits = 1; + + kn = __kernfs_new_node(root, NULL, "", S_IFDIR | S_IRUGO | S_IXUGO, + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, + KERNFS_DIR); + if (!kn) { + idr_destroy(&root->ino_idr); + kfree(root); + return ERR_PTR(-ENOMEM); + } + + kn->priv = priv; + kn->dir.root = root; + + root->syscall_ops = scops; + root->flags = flags; + root->kn = kn; + init_waitqueue_head(&root->deactivate_waitq); + + if (!(root->flags & KERNFS_ROOT_CREATE_DEACTIVATED)) + kernfs_activate(kn); + + return root; +} + +/** + * kernfs_destroy_root - destroy a kernfs hierarchy + * @root: root of the hierarchy to destroy + * + * Destroy the hierarchy anchored at @root by removing all existing + * directories and destroying @root. + */ +void kernfs_destroy_root(struct kernfs_root *root) +{ + /* + * kernfs_remove holds kernfs_rwsem from the root so the root + * shouldn't be freed during the operation. + */ + kernfs_get(root->kn); + kernfs_remove(root->kn); + kernfs_put(root->kn); /* will also free @root */ +} + +/** + * kernfs_root_to_node - return the kernfs_node associated with a kernfs_root + * @root: root to use to lookup + * + * Return: @root's kernfs_node + */ +struct kernfs_node *kernfs_root_to_node(struct kernfs_root *root) +{ + return root->kn; +} + +/** + * kernfs_create_dir_ns - create a directory + * @parent: parent in which to create a new directory + * @name: name of the new directory + * @mode: mode of the new directory + * @uid: uid of the new directory + * @gid: gid of the new directory + * @priv: opaque data associated with the new directory + * @ns: optional namespace tag of the directory + * + * Return: the created node on success, ERR_PTR() value on failure. + */ +struct kernfs_node *kernfs_create_dir_ns(struct kernfs_node *parent, + const char *name, umode_t mode, + kuid_t uid, kgid_t gid, + void *priv, + const struct ns_common *ns) +{ + struct kernfs_node *kn; + int rc; + + /* allocate */ + kn = kernfs_new_node(parent, name, mode | S_IFDIR, + uid, gid, KERNFS_DIR); + if (!kn) + return ERR_PTR(-ENOMEM); + + kn->dir.root = parent->dir.root; + kn->ns = ns; + kn->priv = priv; + + /* link in */ + rc = kernfs_add_one(kn); + if (!rc) + return kn; + + kernfs_put(kn); + return ERR_PTR(rc); +} + +/** + * kernfs_create_empty_dir - create an always empty directory + * @parent: parent in which to create a new directory + * @name: name of the new directory + * + * Return: the created node on success, ERR_PTR() value on failure. + */ +struct kernfs_node *kernfs_create_empty_dir(struct kernfs_node *parent, + const char *name) +{ + struct kernfs_node *kn; + int rc; + + /* allocate */ + kn = kernfs_new_node(parent, name, S_IRUGO|S_IXUGO|S_IFDIR, + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, KERNFS_DIR); + if (!kn) + return ERR_PTR(-ENOMEM); + + kn->flags |= KERNFS_EMPTY_DIR; + kn->dir.root = parent->dir.root; + kn->ns = NULL; + kn->priv = NULL; + + /* link in */ + rc = kernfs_add_one(kn); + if (!rc) + return kn; + + kernfs_put(kn); + return ERR_PTR(rc); +} + +static int kernfs_dop_revalidate(struct inode *dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + struct kernfs_node *kn, *parent; + struct kernfs_root *root; + + if (flags & LOOKUP_RCU) + return -ECHILD; + + /* Negative hashed dentry? */ + if (d_really_is_negative(dentry)) { + /* If the kernfs parent node has changed discard and + * proceed to ->lookup. + * + * There's nothing special needed here when getting the + * dentry parent, even if a concurrent rename is in + * progress. That's because the dentry is negative so + * it can only be the target of the rename and it will + * be doing a d_move() not a replace. Consequently the + * dentry d_parent won't change over the d_move(). + * + * Also kernfs negative dentries transitioning from + * negative to positive during revalidate won't happen + * because they are invalidated on containing directory + * changes and the lookup re-done so that a new positive + * dentry can be properly created. + */ + root = kernfs_root_from_sb(dentry->d_sb); + down_read(&root->kernfs_rwsem); + parent = kernfs_dentry_node(dentry->d_parent); + if (parent) { + if (kernfs_dir_changed(parent, dentry)) { + up_read(&root->kernfs_rwsem); + return 0; + } + } + up_read(&root->kernfs_rwsem); + + /* The kernfs parent node hasn't changed, leave the + * dentry negative and return success. + */ + return 1; + } + + kn = kernfs_dentry_node(dentry); + root = kernfs_root(kn); + down_read(&root->kernfs_rwsem); + + /* The kernfs node has been deactivated */ + if (!kernfs_active(kn)) + goto out_bad; + + parent = kernfs_parent(kn); + /* The kernfs node has been moved? */ + if (kernfs_dentry_node(dentry->d_parent) != parent) + goto out_bad; + + /* The kernfs node has been renamed */ + if (strcmp(dentry->d_name.name, kernfs_rcu_name(kn)) != 0) + goto out_bad; + + /* The kernfs node has been moved to a different namespace */ + if (parent && kernfs_ns_enabled(parent) && + kernfs_ns_id(kernfs_info(dentry->d_sb)->ns) != kernfs_ns_id(kn->ns)) + goto out_bad; + + up_read(&root->kernfs_rwsem); + return 1; +out_bad: + up_read(&root->kernfs_rwsem); + return 0; +} + +const struct dentry_operations kernfs_dops = { + .d_revalidate = kernfs_dop_revalidate, +}; + +static struct dentry *kernfs_iop_lookup(struct inode *dir, + struct dentry *dentry, + unsigned int flags) +{ + struct kernfs_node *parent = dir->i_private; + struct kernfs_node *kn; + struct kernfs_root *root; + struct inode *inode = NULL; + const struct ns_common *ns = NULL; + + root = kernfs_root(parent); + down_read(&root->kernfs_rwsem); + if (kernfs_ns_enabled(parent)) + ns = kernfs_info(dir->i_sb)->ns; + + kn = kernfs_find_ns(parent, dentry->d_name.name, ns); + /* attach dentry and inode */ + if (kn) { + /* Inactive nodes are invisible to the VFS so don't + * create a negative. + */ + if (!kernfs_active(kn)) { + up_read(&root->kernfs_rwsem); + return NULL; + } + inode = kernfs_get_inode(dir->i_sb, kn); + if (!inode) + inode = ERR_PTR(-ENOMEM); + } + /* + * Needed for negative dentry validation. + * The negative dentry can be created in kernfs_iop_lookup() + * or transforms from positive dentry in dentry_unlink_inode() + * called from vfs_rmdir(). + */ + if (!IS_ERR(inode)) + kernfs_set_rev(parent, dentry); + up_read(&root->kernfs_rwsem); + + /* instantiate and hash (possibly negative) dentry */ + return d_splice_alias(inode, dentry); +} + +static struct dentry *kernfs_iop_mkdir(struct mnt_idmap *idmap, + struct inode *dir, struct dentry *dentry, + umode_t mode) +{ + struct kernfs_node *parent = dir->i_private; + struct kernfs_syscall_ops *scops = kernfs_root(parent)->syscall_ops; + int ret; + + if (!scops || !scops->mkdir) + return ERR_PTR(-EPERM); + + if (!kernfs_get_active(parent)) + return ERR_PTR(-ENODEV); + + ret = scops->mkdir(parent, dentry->d_name.name, mode); + + kernfs_put_active(parent); + return ERR_PTR(ret); +} + +static int kernfs_iop_rmdir(struct inode *dir, struct dentry *dentry) +{ + struct kernfs_node *kn = kernfs_dentry_node(dentry); + struct kernfs_syscall_ops *scops = kernfs_root(kn)->syscall_ops; + int ret; + + if (!scops || !scops->rmdir) + return -EPERM; + + if (!kernfs_get_active(kn)) + return -ENODEV; + + ret = scops->rmdir(kn); + + kernfs_put_active(kn); + return ret; +} + +static int kernfs_iop_rename(struct mnt_idmap *idmap, + struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry, + unsigned int flags) +{ + struct kernfs_node *kn = kernfs_dentry_node(old_dentry); + struct kernfs_node *new_parent = new_dir->i_private; + struct kernfs_syscall_ops *scops = kernfs_root(kn)->syscall_ops; + int ret; + + if (flags) + return -EINVAL; + + if (!scops || !scops->rename) + return -EPERM; + + if (!kernfs_get_active(kn)) + return -ENODEV; + + if (!kernfs_get_active(new_parent)) { + kernfs_put_active(kn); + return -ENODEV; + } + + ret = scops->rename(kn, new_parent, new_dentry->d_name.name); + + kernfs_put_active(new_parent); + kernfs_put_active(kn); + return ret; +} + +const struct inode_operations kernfs_dir_iops = { + .lookup = kernfs_iop_lookup, + .permission = kernfs_iop_permission, + .setattr = kernfs_iop_setattr, + .getattr = kernfs_iop_getattr, + .listxattr = kernfs_iop_listxattr, + + .mkdir = kernfs_iop_mkdir, + .rmdir = kernfs_iop_rmdir, + .rename = kernfs_iop_rename, +}; + +static struct kernfs_node *kernfs_leftmost_descendant(struct kernfs_node *pos) +{ + struct kernfs_node *last; + + while (true) { + struct rb_node *rbn; + + last = pos; + + if (kernfs_type(pos) != KERNFS_DIR) + break; + + rbn = rb_first(&pos->dir.children); + if (!rbn) + break; + + pos = rb_to_kn(rbn); + } + + return last; +} + +/** + * kernfs_next_descendant_post - find the next descendant for post-order walk + * @pos: the current position (%NULL to initiate traversal) + * @root: kernfs_node whose descendants to walk + * + * Find the next descendant to visit for post-order traversal of @root's + * descendants. @root is included in the iteration and the last node to be + * visited. + * + * Return: the next descendant to visit or %NULL when done. + */ +static struct kernfs_node *kernfs_next_descendant_post(struct kernfs_node *pos, + struct kernfs_node *root) +{ + struct rb_node *rbn; + + lockdep_assert_held_write(&kernfs_root(root)->kernfs_rwsem); + + /* if first iteration, visit leftmost descendant which may be root */ + if (!pos) + return kernfs_leftmost_descendant(root); + + /* if we visited @root, we're done */ + if (pos == root) + return NULL; + + /* if there's an unvisited sibling, visit its leftmost descendant */ + rbn = rb_next(&pos->rb); + if (rbn) + return kernfs_leftmost_descendant(rb_to_kn(rbn)); + + /* no sibling left, visit parent */ + return kernfs_parent(pos); +} + +static void kernfs_activate_one(struct kernfs_node *kn) +{ + lockdep_assert_held_write(&kernfs_root(kn)->kernfs_rwsem); + + kn->flags |= KERNFS_ACTIVATED; + + if (kernfs_active(kn) || (kn->flags & (KERNFS_HIDDEN | KERNFS_REMOVING))) + return; + + WARN_ON_ONCE(rcu_access_pointer(kn->__parent) && RB_EMPTY_NODE(&kn->rb)); + WARN_ON_ONCE(atomic_read(&kn->active) != KN_DEACTIVATED_BIAS); + + atomic_sub(KN_DEACTIVATED_BIAS, &kn->active); +} + +/** + * kernfs_activate - activate a node which started deactivated + * @kn: kernfs_node whose subtree is to be activated + * + * If the root has KERNFS_ROOT_CREATE_DEACTIVATED set, a newly created node + * needs to be explicitly activated. A node which hasn't been activated + * isn't visible to userland and deactivation is skipped during its + * removal. This is useful to construct atomic init sequences where + * creation of multiple nodes should either succeed or fail atomically. + * + * The caller is responsible for ensuring that this function is not called + * after kernfs_remove*() is invoked on @kn. + */ +void kernfs_activate(struct kernfs_node *kn) +{ + struct kernfs_node *pos; + struct kernfs_root *root = kernfs_root(kn); + + down_write(&root->kernfs_rwsem); + + pos = NULL; + while ((pos = kernfs_next_descendant_post(pos, kn))) + kernfs_activate_one(pos); + + up_write(&root->kernfs_rwsem); +} + +/** + * kernfs_show - show or hide a node + * @kn: kernfs_node to show or hide + * @show: whether to show or hide + * + * If @show is %false, @kn is marked hidden and deactivated. A hidden node is + * ignored in future activaitons. If %true, the mark is removed and activation + * state is restored. This function won't implicitly activate a new node in a + * %KERNFS_ROOT_CREATE_DEACTIVATED root which hasn't been activated yet. + * + * To avoid recursion complexities, directories aren't supported for now. + */ +void kernfs_show(struct kernfs_node *kn, bool show) +{ + struct kernfs_root *root = kernfs_root(kn); + + if (WARN_ON_ONCE(kernfs_type(kn) == KERNFS_DIR)) + return; + + down_write(&root->kernfs_rwsem); + + if (show) { + kn->flags &= ~KERNFS_HIDDEN; + if (kn->flags & KERNFS_ACTIVATED) + kernfs_activate_one(kn); + } else { + kn->flags |= KERNFS_HIDDEN; + if (kernfs_active(kn)) + atomic_add(KN_DEACTIVATED_BIAS, &kn->active); + kernfs_drain(kn, false); + } + + up_write(&root->kernfs_rwsem); +} + +/* + * This function enables VFS to send fsnotify events for deletions. + * There is gap in this implementation for certain file removals due their + * unique nature in kernfs. Directory removals that trigger file removals occur + * through vfs_rmdir, which shrinks the dcache and emits fsnotify events after + * the rmdir operation; there is no issue here. However kernfs writes to + * particular files (e.g. cgroup.subtree_control) can also cause file removal, + * but vfs_write does not attempt to emit fsnotify events after the write + * operation, even if i_nlink counts are 0. As a usecase for monitoring this + * category of file removals is not known, they are left without having + * IN_DELETE or IN_DELETE_SELF events generated. + * Fanotify recursive monitoring also does not work for kernfs nodes that do not + * have inodes attached, as they are created on-demand in kernfs. + */ +static void kernfs_clear_inode_nlink(struct kernfs_node *kn) +{ + struct kernfs_root *root = kernfs_root(kn); + struct kernfs_super_info *info; + + lockdep_assert_held_read(&root->kernfs_supers_rwsem); + + list_for_each_entry(info, &root->supers, node) { + struct inode *inode = ilookup(info->sb, kernfs_ino(kn)); + + if (inode) { + clear_nlink(inode); + iput(inode); + } + } +} + +static void __kernfs_remove(struct kernfs_node *kn) +{ + struct kernfs_node *pos, *parent; + + /* Short-circuit if non-root @kn has already finished removal. */ + if (!kn) + return; + + lockdep_assert_held_read(&kernfs_root(kn)->kernfs_supers_rwsem); + lockdep_assert_held_write(&kernfs_root(kn)->kernfs_rwsem); + + /* + * This is for kernfs_remove_self() which plays with active ref + * after removal. + */ + if (kernfs_parent(kn) && RB_EMPTY_NODE(&kn->rb)) + return; + + pr_debug("kernfs %s: removing\n", kernfs_rcu_name(kn)); + + /* prevent new usage by marking all nodes removing and deactivating */ + down_write(&kernfs_root(kn)->kernfs_iattr_rwsem); + pos = NULL; + while ((pos = kernfs_next_descendant_post(pos, kn))) { + pos->flags |= KERNFS_REMOVING; + if (kernfs_active(pos)) + atomic_add(KN_DEACTIVATED_BIAS, &pos->active); + } + up_write(&kernfs_root(kn)->kernfs_iattr_rwsem); + + /* deactivate and unlink the subtree node-by-node */ + do { + pos = kernfs_leftmost_descendant(kn); + + /* + * kernfs_drain() may drop kernfs_rwsem temporarily and @pos's + * base ref could have been put by someone else by the time + * the function returns. Make sure it doesn't go away + * underneath us. + */ + kernfs_get(pos); + + kernfs_drain(pos, true); + parent = kernfs_parent(pos); + /* + * kernfs_unlink_sibling() succeeds once per node. Use it + * to decide who's responsible for cleanups. + */ + if (!parent || kernfs_unlink_sibling(pos)) { + struct kernfs_iattrs *ps_iattr = + parent ? parent->iattr : NULL; + + down_write(&kernfs_root(kn)->kernfs_iattr_rwsem); + + kernfs_clear_inode_nlink(pos); + + /* update timestamps on the parent */ + if (ps_iattr) { + ktime_get_real_ts64(&ps_iattr->ia_ctime); + ps_iattr->ia_mtime = ps_iattr->ia_ctime; + } + + up_write(&kernfs_root(kn)->kernfs_iattr_rwsem); + kernfs_put(pos); + } + + kernfs_put(pos); + } while (pos != kn); +} + +/** + * kernfs_remove - remove a kernfs_node recursively + * @kn: the kernfs_node to remove + * + * Remove @kn along with all its subdirectories and files. + */ +void kernfs_remove(struct kernfs_node *kn) +{ + struct kernfs_root *root; + + if (!kn) + return; + + root = kernfs_root(kn); + + down_read(&root->kernfs_supers_rwsem); + down_write(&root->kernfs_rwsem); + __kernfs_remove(kn); + up_write(&root->kernfs_rwsem); + up_read(&root->kernfs_supers_rwsem); +} + +/** + * kernfs_break_active_protection - break out of active protection + * @kn: the self kernfs_node + * + * The caller must be running off of a kernfs operation which is invoked + * with an active reference - e.g. one of kernfs_ops. Each invocation of + * this function must also be matched with an invocation of + * kernfs_unbreak_active_protection(). + * + * This function releases the active reference of @kn the caller is + * holding. Once this function is called, @kn may be removed at any point + * and the caller is solely responsible for ensuring that the objects it + * dereferences are accessible. + */ +void kernfs_break_active_protection(struct kernfs_node *kn) +{ + /* + * Take out ourself out of the active ref dependency chain. If + * we're called without an active ref, lockdep will complain. + */ + kernfs_put_active(kn); +} + +/** + * kernfs_unbreak_active_protection - undo kernfs_break_active_protection() + * @kn: the self kernfs_node + * + * If kernfs_break_active_protection() was called, this function must be + * invoked before finishing the kernfs operation. Note that while this + * function restores the active reference, it doesn't and can't actually + * restore the active protection - @kn may already or be in the process of + * being drained and removed. Once kernfs_break_active_protection() is + * invoked, that protection is irreversibly gone for the kernfs operation + * instance. + * + * While this function may be called at any point after + * kernfs_break_active_protection() is invoked, its most useful location + * would be right before the enclosing kernfs operation returns. + */ +void kernfs_unbreak_active_protection(struct kernfs_node *kn) +{ + /* + * @kn->active could be in any state; however, the increment we do + * here will be undone as soon as the enclosing kernfs operation + * finishes and this temporary bump can't break anything. If @kn + * is alive, nothing changes. If @kn is being deactivated, the + * soon-to-follow put will either finish deactivation or restore + * deactivated state. If @kn is already removed, the temporary + * bump is guaranteed to be gone before @kn is released. + */ + atomic_inc(&kn->active); + if (kernfs_lockdep(kn)) + rwsem_acquire(&kn->dep_map, 0, 1, _RET_IP_); +} + +/** + * kernfs_remove_self - remove a kernfs_node from its own method + * @kn: the self kernfs_node to remove + * + * The caller must be running off of a kernfs operation which is invoked + * with an active reference - e.g. one of kernfs_ops. This can be used to + * implement a file operation which deletes itself. + * + * For example, the "delete" file for a sysfs device directory can be + * implemented by invoking kernfs_remove_self() on the "delete" file + * itself. This function breaks the circular dependency of trying to + * deactivate self while holding an active ref itself. It isn't necessary + * to modify the usual removal path to use kernfs_remove_self(). The + * "delete" implementation can simply invoke kernfs_remove_self() on self + * before proceeding with the usual removal path. kernfs will ignore later + * kernfs_remove() on self. + * + * kernfs_remove_self() can be called multiple times concurrently on the + * same kernfs_node. Only the first one actually performs removal and + * returns %true. All others will wait until the kernfs operation which + * won self-removal finishes and return %false. Note that the losers wait + * for the completion of not only the winning kernfs_remove_self() but also + * the whole kernfs_ops which won the arbitration. This can be used to + * guarantee, for example, all concurrent writes to a "delete" file to + * finish only after the whole operation is complete. + * + * Return: %true if @kn is removed by this call, otherwise %false. + */ +bool kernfs_remove_self(struct kernfs_node *kn) +{ + bool ret; + struct kernfs_root *root = kernfs_root(kn); + + down_read(&root->kernfs_supers_rwsem); + down_write(&root->kernfs_rwsem); + kernfs_break_active_protection(kn); + + /* + * SUICIDAL is used to arbitrate among competing invocations. Only + * the first one will actually perform removal. When the removal + * is complete, SUICIDED is set and the active ref is restored + * while kernfs_rwsem for held exclusive. The ones which lost + * arbitration waits for SUICIDED && drained which can happen only + * after the enclosing kernfs operation which executed the winning + * instance of kernfs_remove_self() finished. + */ + if (!(kn->flags & KERNFS_SUICIDAL)) { + kn->flags |= KERNFS_SUICIDAL; + __kernfs_remove(kn); + kn->flags |= KERNFS_SUICIDED; + ret = true; + } else { + wait_queue_head_t *waitq = &kernfs_root(kn)->deactivate_waitq; + DEFINE_WAIT(wait); + + while (true) { + prepare_to_wait(waitq, &wait, TASK_UNINTERRUPTIBLE); + + if ((kn->flags & KERNFS_SUICIDED) && + atomic_read(&kn->active) == KN_DEACTIVATED_BIAS) + break; + + up_write(&root->kernfs_rwsem); + up_read(&root->kernfs_supers_rwsem); + schedule(); + down_read(&root->kernfs_supers_rwsem); + down_write(&root->kernfs_rwsem); + } + finish_wait(waitq, &wait); + WARN_ON_ONCE(!RB_EMPTY_NODE(&kn->rb)); + ret = false; + } + + /* + * This must be done while kernfs_rwsem held exclusive; otherwise, + * waiting for SUICIDED && deactivated could finish prematurely. + */ + kernfs_unbreak_active_protection(kn); + + up_write(&root->kernfs_rwsem); + up_read(&root->kernfs_supers_rwsem); + return ret; +} + +/** + * kernfs_remove_by_name_ns - find a kernfs_node by name and remove it + * @parent: parent of the target + * @name: name of the kernfs_node to remove + * @ns: namespace tag of the kernfs_node to remove + * + * Look for the kernfs_node with @name and @ns under @parent and remove it. + * + * Return: %0 on success, -ENOENT if such entry doesn't exist. + */ +int kernfs_remove_by_name_ns(struct kernfs_node *parent, const char *name, + const struct ns_common *ns) +{ + struct kernfs_node *kn; + struct kernfs_root *root; + + if (!parent) { + WARN(1, KERN_WARNING "kernfs: can not remove '%s', no directory\n", + name); + return -ENOENT; + } + + root = kernfs_root(parent); + down_read(&root->kernfs_supers_rwsem); + down_write(&root->kernfs_rwsem); + + kn = kernfs_find_ns(parent, name, ns); + if (kn) { + kernfs_get(kn); + __kernfs_remove(kn); + kernfs_put(kn); + } + + up_write(&root->kernfs_rwsem); + up_read(&root->kernfs_supers_rwsem); + + if (kn) + return 0; + else + return -ENOENT; +} + +/** + * kernfs_rename_ns - move and rename a kernfs_node + * @kn: target node + * @new_parent: new parent to put @sd under + * @new_name: new name + * @new_ns: new namespace tag + * + * Return: %0 on success, -errno on failure. + */ +int kernfs_rename_ns(struct kernfs_node *kn, struct kernfs_node *new_parent, + const char *new_name, const struct ns_common *new_ns) +{ + struct kernfs_node *old_parent; + struct kernfs_root *root; + const char *old_name; + int error; + + /* can't move or rename root */ + if (!rcu_access_pointer(kn->__parent)) + return -EINVAL; + + root = kernfs_root(kn); + down_write(&root->kernfs_rwsem); + + error = -ENOENT; + if (!kernfs_active(kn) || !kernfs_active(new_parent) || + (new_parent->flags & KERNFS_EMPTY_DIR)) + goto out; + + old_parent = kernfs_parent(kn); + if (root->flags & KERNFS_ROOT_INVARIANT_PARENT) { + error = -EINVAL; + if (WARN_ON_ONCE(old_parent != new_parent)) + goto out; + } + + error = 0; + old_name = kernfs_rcu_name(kn); + if (!new_name) + new_name = old_name; + if ((old_parent == new_parent) && + (kernfs_ns_id(kn->ns) == kernfs_ns_id(new_ns)) && + (strcmp(old_name, new_name) == 0)) + goto out; /* nothing to rename */ + + error = -EEXIST; + if (kernfs_find_ns(new_parent, new_name, new_ns)) + goto out; + + /* rename kernfs_node */ + if (strcmp(old_name, new_name) != 0) { + error = -ENOMEM; + new_name = kstrdup_const(new_name, GFP_KERNEL); + if (!new_name) + goto out; + } else { + new_name = NULL; + } + + /* + * Move to the appropriate place in the appropriate directories rbtree. + */ + kernfs_unlink_sibling(kn); + + /* rename_lock protects ->parent accessors */ + if (old_parent != new_parent) { + kernfs_get(new_parent); + write_lock_irq(&root->kernfs_rename_lock); + + rcu_assign_pointer(kn->__parent, new_parent); + + kn->ns = new_ns; + if (new_name) + rcu_assign_pointer(kn->name, new_name); + + write_unlock_irq(&root->kernfs_rename_lock); + kernfs_put(old_parent); + } else { + /* name assignment is RCU protected, parent is the same */ + kn->ns = new_ns; + if (new_name) + rcu_assign_pointer(kn->name, new_name); + } + + kn->hash = kernfs_name_hash(new_name ?: old_name, kn->ns); + kernfs_link_sibling(kn); + + if (new_name && !is_kernel_rodata((unsigned long)old_name)) + kfree_rcu_mightsleep(old_name); + + error = 0; + out: + up_write(&root->kernfs_rwsem); + return error; +} + +static int kernfs_dir_fop_release(struct inode *inode, struct file *filp) +{ + kernfs_put(filp->private_data); + return 0; +} + +static struct kernfs_node *kernfs_dir_pos(const struct ns_common *ns, + struct kernfs_node *parent, loff_t hash, struct kernfs_node *pos) +{ + if (pos) { + int valid = kernfs_active(pos) && + rcu_access_pointer(pos->__parent) == parent && + hash == pos->hash; + kernfs_put(pos); + if (!valid) + pos = NULL; + } + if (!pos && (hash > 1) && (hash < INT_MAX)) { + struct rb_node *node = parent->dir.children.rb_node; + u64 ns_id = kernfs_ns_id(ns); + while (node) { + pos = rb_to_kn(node); + + if (hash < pos->hash) + node = node->rb_left; + else if (hash > pos->hash) + node = node->rb_right; + else if (ns_id < kernfs_ns_id(pos->ns)) + node = node->rb_left; + else if (ns_id > kernfs_ns_id(pos->ns)) + node = node->rb_right; + else + break; + } + } + /* Skip over entries which are dying/dead or in the wrong namespace */ + while (pos && (!kernfs_active(pos) || + kernfs_ns_id(pos->ns) != kernfs_ns_id(ns))) { + struct rb_node *node = rb_next(&pos->rb); + if (!node) + pos = NULL; + else + pos = rb_to_kn(node); + } + return pos; +} + +static struct kernfs_node *kernfs_dir_next_pos(const struct ns_common *ns, + struct kernfs_node *parent, ino_t ino, struct kernfs_node *pos) +{ + pos = kernfs_dir_pos(ns, parent, ino, pos); + if (pos) { + do { + struct rb_node *node = rb_next(&pos->rb); + if (!node) + pos = NULL; + else + pos = rb_to_kn(node); + } while (pos && (!kernfs_active(pos) || + kernfs_ns_id(pos->ns) != kernfs_ns_id(ns))); + } + return pos; +} + +static int kernfs_fop_readdir(struct file *file, struct dir_context *ctx) +{ + struct dentry *dentry = file->f_path.dentry; + struct kernfs_node *parent = kernfs_dentry_node(dentry); + struct kernfs_node *pos = file->private_data; + struct kernfs_root *root; + const struct ns_common *ns = NULL; + + if (!dir_emit_dots(file, ctx)) + return 0; + + root = kernfs_root(parent); + down_read(&root->kernfs_rwsem); + + if (kernfs_ns_enabled(parent)) + ns = kernfs_info(dentry->d_sb)->ns; + + for (pos = kernfs_dir_pos(ns, parent, ctx->pos, pos); + pos; + pos = kernfs_dir_next_pos(ns, parent, ctx->pos, pos)) { + const char *name = kernfs_rcu_name(pos); + unsigned int type = fs_umode_to_dtype(pos->mode); + int len = strlen(name); + ino_t ino = kernfs_ino(pos); + + ctx->pos = pos->hash; + file->private_data = pos; + kernfs_get(pos); + + if (!dir_emit(ctx, name, len, ino, type)) { + up_read(&root->kernfs_rwsem); + return 0; + } + } + up_read(&root->kernfs_rwsem); + file->private_data = NULL; + ctx->pos = INT_MAX; + return 0; +} + +const struct file_operations kernfs_dir_fops = { + .read = generic_read_dir, + .iterate_shared = kernfs_fop_readdir, + .release = kernfs_dir_fop_release, + .llseek = generic_file_llseek, +}; diff --git a/fs/kernfs/file.c b/fs/kernfs/file.c new file mode 100644 index 000000000..8e0e90c93 --- /dev/null +++ b/fs/kernfs/file.c @@ -0,0 +1,1085 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * fs/kernfs/file.c - kernfs file implementation + * + * Copyright (c) 2001-3 Patrick Mochel + * Copyright (c) 2007 SUSE Linux Products GmbH + * Copyright (c) 2007, 2013 Tejun Heo + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kernfs-internal.h" + +struct kernfs_open_node { + struct rcu_head rcu_head; + atomic_t event; + wait_queue_head_t poll; + struct list_head files; /* goes through kernfs_open_file.list */ + unsigned int nr_mmapped; + unsigned int nr_to_release; +}; + +/* + * kernfs_notify() may be called from any context and bounces notifications + * through a work item. To minimize space overhead in kernfs_node, the + * pending queue is implemented as a singly linked list of kernfs_nodes. + * The list is terminated with the self pointer so that whether a + * kernfs_node is on the list or not can be determined by testing the next + * pointer for %NULL. + */ +#define KERNFS_NOTIFY_EOL ((void *)&kernfs_notify_list) + +static DEFINE_SPINLOCK(kernfs_notify_lock); +static struct kernfs_node *kernfs_notify_list = KERNFS_NOTIFY_EOL; + +/* Compatibility wrappers - use the common hashed node lock */ +static inline struct mutex *kernfs_open_file_mutex_ptr(struct kernfs_node *kn) +{ + return kernfs_node_lock_ptr(kn); +} + +static inline struct mutex *kernfs_open_file_mutex_lock(struct kernfs_node *kn) +{ + return kernfs_node_lock(kn); +} + +/** + * of_on - Get the kernfs_open_node of the specified kernfs_open_file + * @of: target kernfs_open_file + * + * Return: the kernfs_open_node of the kernfs_open_file + */ +static struct kernfs_open_node *of_on(struct kernfs_open_file *of) +{ + return rcu_dereference_protected(of->kn->attr.open, + !list_empty(&of->list)); +} + +/* Get active reference to kernfs node for an open file */ +static struct kernfs_open_file *kernfs_get_active_of(struct kernfs_open_file *of) +{ + /* Skip if file was already released */ + if (unlikely(of->released)) + return NULL; + + if (!kernfs_get_active(of->kn)) + return NULL; + + return of; +} + +static void kernfs_put_active_of(struct kernfs_open_file *of) +{ + return kernfs_put_active(of->kn); +} + +/** + * kernfs_deref_open_node_locked - Get kernfs_open_node corresponding to @kn + * + * @kn: target kernfs_node. + * + * Fetch and return ->attr.open of @kn when caller holds the + * kernfs_open_file_mutex_ptr(kn). + * + * Update of ->attr.open happens under kernfs_open_file_mutex_ptr(kn). So when + * the caller guarantees that this mutex is being held, other updaters can't + * change ->attr.open and this means that we can safely deref ->attr.open + * outside RCU read-side critical section. + * + * The caller needs to make sure that kernfs_open_file_mutex is held. + * + * Return: @kn->attr.open when kernfs_open_file_mutex is held. + */ +static struct kernfs_open_node * +kernfs_deref_open_node_locked(struct kernfs_node *kn) +{ + return rcu_dereference_protected(kn->attr.open, + lockdep_is_held(kernfs_open_file_mutex_ptr(kn))); +} + +static struct kernfs_open_file *kernfs_of(struct file *file) +{ + return ((struct seq_file *)file->private_data)->private; +} + +/* + * Determine the kernfs_ops for the given kernfs_node. This function must + * be called while holding an active reference. + */ +static const struct kernfs_ops *kernfs_ops(struct kernfs_node *kn) +{ + if (kn->flags & KERNFS_LOCKDEP) + lockdep_assert_held(kn); + return kn->attr.ops; +} + +/* + * As kernfs_seq_stop() is also called after kernfs_seq_start() or + * kernfs_seq_next() failure, it needs to distinguish whether it's stopping + * a seq_file iteration which is fully initialized with an active reference + * or an aborted kernfs_seq_start() due to get_active failure. The + * position pointer is the only context for each seq_file iteration and + * thus the stop condition should be encoded in it. As the return value is + * directly visible to userland, ERR_PTR(-ENODEV) is the only acceptable + * choice to indicate get_active failure. + * + * Unfortunately, this is complicated due to the optional custom seq_file + * operations which may return ERR_PTR(-ENODEV) too. kernfs_seq_stop() + * can't distinguish whether ERR_PTR(-ENODEV) is from get_active failure or + * custom seq_file operations and thus can't decide whether put_active + * should be performed or not only on ERR_PTR(-ENODEV). + * + * This is worked around by factoring out the custom seq_stop() and + * put_active part into kernfs_seq_stop_active(), skipping it from + * kernfs_seq_stop() if ERR_PTR(-ENODEV) while invoking it directly after + * custom seq_file operations fail with ERR_PTR(-ENODEV) - this ensures + * that kernfs_seq_stop_active() is skipped only after get_active failure. + */ +static void kernfs_seq_stop_active(struct seq_file *sf, void *v) +{ + struct kernfs_open_file *of = sf->private; + const struct kernfs_ops *ops = kernfs_ops(of->kn); + + if (ops->seq_stop) + ops->seq_stop(sf, v); + kernfs_put_active_of(of); +} + +static void *kernfs_seq_start(struct seq_file *sf, loff_t *ppos) +{ + struct kernfs_open_file *of = sf->private; + const struct kernfs_ops *ops; + + /* + * @of->mutex nests outside active ref and is primarily to ensure that + * the ops aren't called concurrently for the same open file. + */ + mutex_lock(&of->mutex); + if (!kernfs_get_active_of(of)) + return ERR_PTR(-ENODEV); + + ops = kernfs_ops(of->kn); + if (ops->seq_start) { + void *next = ops->seq_start(sf, ppos); + /* see the comment above kernfs_seq_stop_active() */ + if (next == ERR_PTR(-ENODEV)) + kernfs_seq_stop_active(sf, next); + return next; + } + return single_start(sf, ppos); +} + +static void *kernfs_seq_next(struct seq_file *sf, void *v, loff_t *ppos) +{ + struct kernfs_open_file *of = sf->private; + const struct kernfs_ops *ops = kernfs_ops(of->kn); + + if (ops->seq_next) { + void *next = ops->seq_next(sf, v, ppos); + /* see the comment above kernfs_seq_stop_active() */ + if (next == ERR_PTR(-ENODEV)) + kernfs_seq_stop_active(sf, next); + return next; + } else { + /* + * The same behavior and code as single_open(), always + * terminate after the initial read. + */ + ++*ppos; + return NULL; + } +} + +static void kernfs_seq_stop(struct seq_file *sf, void *v) +{ + struct kernfs_open_file *of = sf->private; + + if (v != ERR_PTR(-ENODEV)) + kernfs_seq_stop_active(sf, v); + mutex_unlock(&of->mutex); +} + +static int kernfs_seq_show(struct seq_file *sf, void *v) +{ + struct kernfs_open_file *of = sf->private; + + of->event = atomic_read(&of_on(of)->event); + + return of->kn->attr.ops->seq_show(sf, v); +} + +static const struct seq_operations kernfs_seq_ops = { + .start = kernfs_seq_start, + .next = kernfs_seq_next, + .stop = kernfs_seq_stop, + .show = kernfs_seq_show, +}; + +/* + * As reading a bin file can have side-effects, the exact offset and bytes + * specified in read(2) call should be passed to the read callback making + * it difficult to use seq_file. Implement simplistic custom buffering for + * bin files. + */ +static ssize_t kernfs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) +{ + struct kernfs_open_file *of = kernfs_of(iocb->ki_filp); + ssize_t len = min_t(size_t, iov_iter_count(iter), PAGE_SIZE); + const struct kernfs_ops *ops; + char *buf; + + buf = of->prealloc_buf; + if (buf) + mutex_lock(&of->prealloc_mutex); + else + buf = kmalloc(len, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + /* + * @of->mutex nests outside active ref and is used both to ensure that + * the ops aren't called concurrently for the same open file. + */ + mutex_lock(&of->mutex); + if (!kernfs_get_active_of(of)) { + len = -ENODEV; + mutex_unlock(&of->mutex); + goto out_free; + } + + of->event = atomic_read(&of_on(of)->event); + + ops = kernfs_ops(of->kn); + if (ops->read) + len = ops->read(of, buf, len, iocb->ki_pos); + else + len = -EINVAL; + + kernfs_put_active_of(of); + mutex_unlock(&of->mutex); + + if (len < 0) + goto out_free; + + if (copy_to_iter(buf, len, iter) != len) { + len = -EFAULT; + goto out_free; + } + + iocb->ki_pos += len; + + out_free: + if (buf == of->prealloc_buf) + mutex_unlock(&of->prealloc_mutex); + else + kfree(buf); + return len; +} + +static ssize_t kernfs_fop_read_iter(struct kiocb *iocb, struct iov_iter *iter) +{ + if (kernfs_of(iocb->ki_filp)->kn->flags & KERNFS_HAS_SEQ_SHOW) + return seq_read_iter(iocb, iter); + return kernfs_file_read_iter(iocb, iter); +} + +/* + * Copy data in from userland and pass it to the matching kernfs write + * operation. + * + * There is no easy way for us to know if userspace is only doing a partial + * write, so we don't support them. We expect the entire buffer to come on + * the first write. Hint: if you're writing a value, first read the file, + * modify only the value you're changing, then write entire buffer + * back. + */ +static ssize_t kernfs_fop_write_iter(struct kiocb *iocb, struct iov_iter *iter) +{ + struct kernfs_open_file *of = kernfs_of(iocb->ki_filp); + ssize_t len = iov_iter_count(iter); + const struct kernfs_ops *ops; + char *buf; + + if (of->atomic_write_len) { + if (len > of->atomic_write_len) + return -E2BIG; + } else { + len = min_t(size_t, len, PAGE_SIZE); + } + + buf = of->prealloc_buf; + if (buf) + mutex_lock(&of->prealloc_mutex); + else + buf = kmalloc(len + 1, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + if (copy_from_iter(buf, len, iter) != len) { + len = -EFAULT; + goto out_free; + } + buf[len] = '\0'; /* guarantee string termination */ + + /* + * @of->mutex nests outside active ref and is used both to ensure that + * the ops aren't called concurrently for the same open file. + */ + mutex_lock(&of->mutex); + if (!kernfs_get_active_of(of)) { + mutex_unlock(&of->mutex); + len = -ENODEV; + goto out_free; + } + + ops = kernfs_ops(of->kn); + if (ops->write) + len = ops->write(of, buf, len, iocb->ki_pos); + else + len = -EINVAL; + + kernfs_put_active_of(of); + mutex_unlock(&of->mutex); + + if (len > 0) + iocb->ki_pos += len; + +out_free: + if (buf == of->prealloc_buf) + mutex_unlock(&of->prealloc_mutex); + else + kfree(buf); + return len; +} + +static void kernfs_vma_open(struct vm_area_struct *vma) +{ + struct file *file = vma->vm_file; + struct kernfs_open_file *of = kernfs_of(file); + + if (!of->vm_ops) + return; + + if (!kernfs_get_active_of(of)) + return; + + if (of->vm_ops->open) + of->vm_ops->open(vma); + + kernfs_put_active_of(of); +} + +static vm_fault_t kernfs_vma_fault(struct vm_fault *vmf) +{ + struct file *file = vmf->vma->vm_file; + struct kernfs_open_file *of = kernfs_of(file); + vm_fault_t ret; + + if (!of->vm_ops) + return VM_FAULT_SIGBUS; + + if (!kernfs_get_active_of(of)) + return VM_FAULT_SIGBUS; + + ret = VM_FAULT_SIGBUS; + if (of->vm_ops->fault) + ret = of->vm_ops->fault(vmf); + + kernfs_put_active_of(of); + return ret; +} + +static vm_fault_t kernfs_vma_page_mkwrite(struct vm_fault *vmf) +{ + struct file *file = vmf->vma->vm_file; + struct kernfs_open_file *of = kernfs_of(file); + vm_fault_t ret; + + if (!of->vm_ops) + return VM_FAULT_SIGBUS; + + if (!kernfs_get_active_of(of)) + return VM_FAULT_SIGBUS; + + ret = 0; + if (of->vm_ops->page_mkwrite) + ret = of->vm_ops->page_mkwrite(vmf); + else + file_update_time(file); + + kernfs_put_active_of(of); + return ret; +} + +static int kernfs_vma_access(struct vm_area_struct *vma, unsigned long addr, + void *buf, int len, int write) +{ + struct file *file = vma->vm_file; + struct kernfs_open_file *of = kernfs_of(file); + int ret; + + if (!of->vm_ops) + return -EINVAL; + + if (!kernfs_get_active_of(of)) + return -EINVAL; + + ret = -EINVAL; + if (of->vm_ops->access) + ret = of->vm_ops->access(vma, addr, buf, len, write); + + kernfs_put_active_of(of); + return ret; +} + +static const struct vm_operations_struct kernfs_vm_ops = { + .open = kernfs_vma_open, + .fault = kernfs_vma_fault, + .page_mkwrite = kernfs_vma_page_mkwrite, + .access = kernfs_vma_access, +}; + +static int kernfs_fop_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct kernfs_open_file *of = kernfs_of(file); + const struct kernfs_ops *ops; + int rc; + + /* + * mmap path and of->mutex are prone to triggering spurious lockdep + * warnings and we don't want to add spurious locking dependency + * between the two. Check whether mmap is actually implemented + * without grabbing @of->mutex by testing HAS_MMAP flag. See the + * comment in kernfs_fop_open() for more details. + */ + if (!(of->kn->flags & KERNFS_HAS_MMAP)) + return -ENODEV; + + mutex_lock(&of->mutex); + + rc = -ENODEV; + if (!kernfs_get_active_of(of)) + goto out_unlock; + + ops = kernfs_ops(of->kn); + rc = ops->mmap(of, vma); + if (rc) + goto out_put; + + /* + * PowerPC's pci_mmap of legacy_mem uses shmem_zero_setup() + * to satisfy versions of X which crash if the mmap fails: that + * substitutes a new vm_file, and we don't then want bin_vm_ops. + */ + if (vma->vm_file != file) + goto out_put; + + rc = -EINVAL; + if (of->mmapped && of->vm_ops != vma->vm_ops) + goto out_put; + + /* + * It is not possible to successfully wrap close. + * So error if someone is trying to use close. + */ + if (vma->vm_ops && vma->vm_ops->close) + goto out_put; + + rc = 0; + if (!of->mmapped) { + of->mmapped = true; + of_on(of)->nr_mmapped++; + of->vm_ops = vma->vm_ops; + } + vma->vm_ops = &kernfs_vm_ops; +out_put: + kernfs_put_active_of(of); +out_unlock: + mutex_unlock(&of->mutex); + + return rc; +} + +/** + * kernfs_get_open_node - get or create kernfs_open_node + * @kn: target kernfs_node + * @of: kernfs_open_file for this instance of open + * + * If @kn->attr.open exists, increment its reference count; otherwise, + * create one. @of is chained to the files list. + * + * Locking: + * Kernel thread context (may sleep). + * + * Return: + * %0 on success, -errno on failure. + */ +static int kernfs_get_open_node(struct kernfs_node *kn, + struct kernfs_open_file *of) +{ + struct kernfs_open_node *on; + struct mutex *mutex; + + mutex = kernfs_open_file_mutex_lock(kn); + on = kernfs_deref_open_node_locked(kn); + + if (!on) { + /* not there, initialize a new one */ + on = kzalloc_obj(*on); + if (!on) { + mutex_unlock(mutex); + return -ENOMEM; + } + atomic_set(&on->event, 1); + init_waitqueue_head(&on->poll); + INIT_LIST_HEAD(&on->files); + rcu_assign_pointer(kn->attr.open, on); + } + + list_add_tail(&of->list, &on->files); + if (kn->flags & KERNFS_HAS_RELEASE) + on->nr_to_release++; + + mutex_unlock(mutex); + return 0; +} + +/** + * kernfs_unlink_open_file - Unlink @of from @kn. + * + * @kn: target kernfs_node + * @of: associated kernfs_open_file + * @open_failed: ->open() failed, cancel ->release() + * + * Unlink @of from list of @kn's associated open files. If list of + * associated open files becomes empty, disassociate and free + * kernfs_open_node. + * + * LOCKING: + * None. + */ +static void kernfs_unlink_open_file(struct kernfs_node *kn, + struct kernfs_open_file *of, + bool open_failed) +{ + struct kernfs_open_node *on; + struct mutex *mutex; + + mutex = kernfs_open_file_mutex_lock(kn); + + on = kernfs_deref_open_node_locked(kn); + if (!on) { + mutex_unlock(mutex); + return; + } + + if (of) { + if (kn->flags & KERNFS_HAS_RELEASE) { + WARN_ON_ONCE(of->released == open_failed); + if (open_failed) + on->nr_to_release--; + } + if (of->mmapped) + on->nr_mmapped--; + list_del(&of->list); + } + + if (list_empty(&on->files)) { + rcu_assign_pointer(kn->attr.open, NULL); + kfree_rcu(on, rcu_head); + } + + mutex_unlock(mutex); +} + +static int kernfs_fop_open(struct inode *inode, struct file *file) +{ + struct kernfs_node *kn = inode->i_private; + struct kernfs_root *root = kernfs_root(kn); + const struct kernfs_ops *ops; + struct kernfs_open_file *of; + bool has_read, has_write, has_mmap; + int error = -EACCES; + + if (!kernfs_get_active(kn)) + return -ENODEV; + + ops = kernfs_ops(kn); + + has_read = ops->seq_show || ops->read || ops->mmap; + has_write = ops->write || ops->mmap; + has_mmap = ops->mmap; + + /* see the flag definition for details */ + if (root->flags & KERNFS_ROOT_EXTRA_OPEN_PERM_CHECK) { + if ((file->f_mode & FMODE_WRITE) && + (!(inode->i_mode & S_IWUGO) || !has_write)) + goto err_out; + + if ((file->f_mode & FMODE_READ) && + (!(inode->i_mode & S_IRUGO) || !has_read)) + goto err_out; + } + + /* allocate a kernfs_open_file for the file */ + error = -ENOMEM; + of = kzalloc_obj(struct kernfs_open_file); + if (!of) + goto err_out; + + /* + * The following is done to give a different lockdep key to + * @of->mutex for files which implement mmap. This is a rather + * crude way to avoid false positive lockdep warning around + * mm->mmap_lock - mmap nests @of->mutex under mm->mmap_lock and + * reading /sys/block/sda/trace/act_mask grabs sr_mutex, under + * which mm->mmap_lock nests, while holding @of->mutex. As each + * open file has a separate mutex, it's okay as long as those don't + * happen on the same file. At this point, we can't easily give + * each file a separate locking class. Let's differentiate on + * whether the file has mmap or not for now. + * + * For similar reasons, writable and readonly files are given different + * lockdep key, because the writable file /sys/power/resume may call vfs + * lookup helpers for arbitrary paths and readonly files can be read by + * overlayfs from vfs helpers when sysfs is a lower layer of overalyfs. + * + * All three cases look the same. They're supposed to + * look that way and give @of->mutex different static lockdep keys. + */ + if (has_mmap) + mutex_init(&of->mutex); + else if (file->f_mode & FMODE_WRITE) + mutex_init(&of->mutex); + else + mutex_init(&of->mutex); + + of->kn = kn; + of->file = file; + + /* + * Write path needs to atomic_write_len outside active reference. + * Cache it in open_file. See kernfs_fop_write_iter() for details. + */ + of->atomic_write_len = ops->atomic_write_len; + + error = -EINVAL; + /* + * ->seq_show is incompatible with ->prealloc, + * as seq_read does its own allocation. + * ->read must be used instead. + */ + if (ops->prealloc && ops->seq_show) + goto err_free; + if (ops->prealloc) { + int len = of->atomic_write_len ?: PAGE_SIZE; + of->prealloc_buf = kmalloc(len + 1, GFP_KERNEL); + error = -ENOMEM; + if (!of->prealloc_buf) + goto err_free; + mutex_init(&of->prealloc_mutex); + } + + /* + * Always instantiate seq_file even if read access doesn't use + * seq_file or is not requested. This unifies private data access + * and readable regular files are the vast majority anyway. + */ + if (ops->seq_show) + error = seq_open(file, &kernfs_seq_ops); + else + error = seq_open(file, NULL); + if (error) + goto err_free; + + of->seq_file = file->private_data; + of->seq_file->private = of; + + /* seq_file clears PWRITE unconditionally, restore it if WRITE */ + if (file->f_mode & FMODE_WRITE) + file->f_mode |= FMODE_PWRITE; + + /* make sure we have open node struct */ + error = kernfs_get_open_node(kn, of); + if (error) + goto err_seq_release; + + if (ops->open) { + /* nobody has access to @of yet, skip @of->mutex */ + error = ops->open(of); + if (error) + goto err_put_node; + } + + /* open succeeded, put active references */ + kernfs_put_active(kn); + return 0; + +err_put_node: + kernfs_unlink_open_file(kn, of, true); +err_seq_release: + seq_release(inode, file); +err_free: + kfree(of->prealloc_buf); + kfree(of); +err_out: + kernfs_put_active(kn); + return error; +} + +/* used from release/drain to ensure that ->release() is called exactly once */ +static void kernfs_release_file(struct kernfs_node *kn, + struct kernfs_open_file *of) +{ + /* + * @of is guaranteed to have no other file operations in flight and + * we just want to synchronize release and drain paths. + * @kernfs_open_file_mutex_ptr(kn) is enough. @of->mutex can't be used + * here because drain path may be called from places which can + * cause circular dependency. + */ + lockdep_assert_held(kernfs_open_file_mutex_ptr(kn)); + + if (!of->released) { + /* + * A file is never detached without being released and we + * need to be able to release files which are deactivated + * and being drained. Don't use kernfs_ops(). + */ + kn->attr.ops->release(of); + of->released = true; + of_on(of)->nr_to_release--; + } +} + +static int kernfs_fop_release(struct inode *inode, struct file *filp) +{ + struct kernfs_node *kn = inode->i_private; + struct kernfs_open_file *of = kernfs_of(filp); + + if (kn->flags & KERNFS_HAS_RELEASE) { + struct mutex *mutex; + + mutex = kernfs_open_file_mutex_lock(kn); + kernfs_release_file(kn, of); + mutex_unlock(mutex); + } + + kernfs_unlink_open_file(kn, of, false); + seq_release(inode, filp); + kfree(of->prealloc_buf); + kfree(of); + + return 0; +} + +bool kernfs_should_drain_open_files(struct kernfs_node *kn) +{ + struct kernfs_open_node *on; + bool ret; + + /* + * @kn being deactivated guarantees that @kn->attr.open can't change + * beneath us making the lockless test below safe. + * Callers post kernfs_unbreak_active_protection may be counted in + * kn->active by now, do not WARN_ON because of them. + */ + + rcu_read_lock(); + on = rcu_dereference(kn->attr.open); + ret = on && (on->nr_mmapped || on->nr_to_release); + rcu_read_unlock(); + + return ret; +} + +void kernfs_drain_open_files(struct kernfs_node *kn) +{ + struct kernfs_open_node *on; + struct kernfs_open_file *of; + struct mutex *mutex; + + mutex = kernfs_open_file_mutex_lock(kn); + on = kernfs_deref_open_node_locked(kn); + if (!on) { + mutex_unlock(mutex); + return; + } + + list_for_each_entry(of, &on->files, list) { + struct inode *inode = file_inode(of->file); + + if (of->mmapped) { + unmap_mapping_range(inode->i_mapping, 0, 0, 1); + of->mmapped = false; + on->nr_mmapped--; + } + + if (kn->flags & KERNFS_HAS_RELEASE) + kernfs_release_file(kn, of); + } + + WARN_ON_ONCE(on->nr_mmapped || on->nr_to_release); + mutex_unlock(mutex); +} + +/* + * Kernfs attribute files are pollable. The idea is that you read + * the content and then you use 'poll' or 'select' to wait for + * the content to change. When the content changes (assuming the + * manager for the kobject supports notification), poll will + * return EPOLLERR|EPOLLPRI, and select will return the fd whether + * it is waiting for read, write, or exceptions. + * Once poll/select indicates that the value has changed, you + * need to close and re-open the file, or seek to 0 and read again. + * Reminder: this only works for attributes which actively support + * it, and it is not possible to test an attribute from userspace + * to see if it supports poll (Neither 'poll' nor 'select' return + * an appropriate error code). When in doubt, set a suitable timeout value. + */ +__poll_t kernfs_generic_poll(struct kernfs_open_file *of, poll_table *wait) +{ + struct kernfs_open_node *on = of_on(of); + + poll_wait(of->file, &on->poll, wait); + + if (of->event != atomic_read(&on->event)) + return DEFAULT_POLLMASK|EPOLLERR|EPOLLPRI; + + return DEFAULT_POLLMASK; +} + +static __poll_t kernfs_fop_poll(struct file *filp, poll_table *wait) +{ + struct kernfs_open_file *of = kernfs_of(filp); + struct kernfs_node *kn = kernfs_dentry_node(filp->f_path.dentry); + __poll_t ret; + + if (!kernfs_get_active_of(of)) + return DEFAULT_POLLMASK|EPOLLERR|EPOLLPRI; + + if (kn->attr.ops->poll) + ret = kn->attr.ops->poll(of, wait); + else + ret = kernfs_generic_poll(of, wait); + + kernfs_put_active_of(of); + return ret; +} + +static loff_t kernfs_fop_llseek(struct file *file, loff_t offset, int whence) +{ + struct kernfs_open_file *of = kernfs_of(file); + const struct kernfs_ops *ops; + loff_t ret; + + /* + * @of->mutex nests outside active ref and is primarily to ensure that + * the ops aren't called concurrently for the same open file. + */ + mutex_lock(&of->mutex); + if (!kernfs_get_active_of(of)) { + mutex_unlock(&of->mutex); + return -ENODEV; + } + + ops = kernfs_ops(of->kn); + if (ops->llseek) + ret = ops->llseek(of, offset, whence); + else + ret = generic_file_llseek(file, offset, whence); + + kernfs_put_active_of(of); + mutex_unlock(&of->mutex); + return ret; +} + +static void kernfs_notify_workfn(struct work_struct *work) +{ + struct kernfs_node *kn; + struct kernfs_super_info *info; + struct kernfs_root *root; +repeat: + /* pop one off the notify_list */ + spin_lock_irq(&kernfs_notify_lock); + kn = kernfs_notify_list; + if (kn == KERNFS_NOTIFY_EOL) { + spin_unlock_irq(&kernfs_notify_lock); + return; + } + kernfs_notify_list = kn->attr.notify_next; + kn->attr.notify_next = NULL; + spin_unlock_irq(&kernfs_notify_lock); + + root = kernfs_root(kn); + /* kick fsnotify */ + + down_read(&root->kernfs_supers_rwsem); + down_read(&root->kernfs_rwsem); + list_for_each_entry(info, &kernfs_root(kn)->supers, node) { + struct kernfs_node *parent; + struct inode *p_inode = NULL; + const char *kn_name; + struct inode *inode; + struct qstr name; + + /* + * We want fsnotify_modify() on @kn but as the + * modifications aren't originating from userland don't + * have the matching @file available. Look up the inodes + * and generate the events manually. + */ + inode = ilookup(info->sb, kernfs_ino(kn)); + if (!inode) + continue; + + kn_name = kernfs_rcu_name(kn); + name = QSTR(kn_name); + parent = kernfs_get_parent(kn); + if (parent) { + p_inode = ilookup(info->sb, kernfs_ino(parent)); + if (p_inode) { + fsnotify(FS_MODIFY | FS_EVENT_ON_CHILD, + inode, FSNOTIFY_EVENT_INODE, + p_inode, &name, inode, 0); + iput(p_inode); + } + + kernfs_put(parent); + } + + if (!p_inode) + fsnotify_inode(inode, FS_MODIFY); + + iput(inode); + } + + up_read(&root->kernfs_rwsem); + up_read(&root->kernfs_supers_rwsem); + kernfs_put(kn); + goto repeat; +} + +/** + * kernfs_notify - notify a kernfs file + * @kn: file to notify + * + * Notify @kn such that poll(2) on @kn wakes up. Maybe be called from any + * context. + */ +void kernfs_notify(struct kernfs_node *kn) +{ + static DECLARE_WORK(kernfs_notify_work, kernfs_notify_workfn); + unsigned long flags; + struct kernfs_open_node *on; + + if (WARN_ON(kernfs_type(kn) != KERNFS_FILE)) + return; + + /* kick poll immediately */ + rcu_read_lock(); + on = rcu_dereference(kn->attr.open); + if (on) { + atomic_inc(&on->event); + wake_up_interruptible(&on->poll); + } + rcu_read_unlock(); + + /* schedule work to kick fsnotify */ + spin_lock_irqsave(&kernfs_notify_lock, flags); + if (!kn->attr.notify_next) { + kernfs_get(kn); + kn->attr.notify_next = kernfs_notify_list; + kernfs_notify_list = kn; + schedule_work(&kernfs_notify_work); + } + spin_unlock_irqrestore(&kernfs_notify_lock, flags); +} +EXPORT_SYMBOL_GPL(kernfs_notify); + +const struct file_operations kernfs_file_fops = { + .read_iter = kernfs_fop_read_iter, + .write_iter = kernfs_fop_write_iter, + .llseek = kernfs_fop_llseek, + .mmap = kernfs_fop_mmap, + .open = kernfs_fop_open, + .release = kernfs_fop_release, + .poll = kernfs_fop_poll, + .fsync = noop_fsync, + .splice_read = copy_splice_read, + .splice_write = iter_file_splice_write, +}; + +/** + * __kernfs_create_file - kernfs internal function to create a file + * @parent: directory to create the file in + * @name: name of the file + * @mode: mode of the file + * @uid: uid of the file + * @gid: gid of the file + * @size: size of the file + * @ops: kernfs operations for the file + * @priv: private data for the file + * @ns: optional namespace tag of the file + * @key: lockdep key for the file's active_ref, %NULL to disable lockdep + * + * Return: the created node on success, ERR_PTR() value on error. + */ +struct kernfs_node *__kernfs_create_file(struct kernfs_node *parent, + const char *name, + umode_t mode, kuid_t uid, kgid_t gid, + loff_t size, + const struct kernfs_ops *ops, + void *priv, const struct ns_common *ns, + struct lock_class_key *key) +{ + struct kernfs_node *kn; + unsigned flags; + int rc; + + flags = KERNFS_FILE; + + kn = kernfs_new_node(parent, name, (mode & S_IALLUGO) | S_IFREG, + uid, gid, flags); + if (!kn) + return ERR_PTR(-ENOMEM); + + kn->attr.ops = ops; + kn->attr.size = size; + kn->ns = ns; + kn->priv = priv; + +#ifdef CONFIG_DEBUG_LOCK_ALLOC + if (key) { + lockdep_init_map(&kn->dep_map, "kn->active", key, 0); + kn->flags |= KERNFS_LOCKDEP; + } +#endif + + /* + * kn->attr.ops is accessible only while holding active ref. We + * need to know whether some ops are implemented outside active + * ref. Cache their existence in flags. + */ + if (ops->seq_show) + kn->flags |= KERNFS_HAS_SEQ_SHOW; + if (ops->mmap) + kn->flags |= KERNFS_HAS_MMAP; + if (ops->release) + kn->flags |= KERNFS_HAS_RELEASE; + + rc = kernfs_add_one(kn); + if (rc) { + kernfs_put(kn); + return ERR_PTR(rc); + } + return kn; +} diff --git a/fs/kernfs/inode.c b/fs/kernfs/inode.c new file mode 100644 index 000000000..237dcdd73 --- /dev/null +++ b/fs/kernfs/inode.c @@ -0,0 +1,406 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * fs/kernfs/inode.c - kernfs inode implementation + * + * Copyright (c) 2001-3 Patrick Mochel + * Copyright (c) 2007 SUSE Linux Products GmbH + * Copyright (c) 2007, 2013 Tejun Heo + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "kernfs-internal.h" + +static const struct inode_operations kernfs_iops = { + .permission = kernfs_iop_permission, + .setattr = kernfs_iop_setattr, + .getattr = kernfs_iop_getattr, + .listxattr = kernfs_iop_listxattr, +}; + +static struct kernfs_iattrs *__kernfs_iattrs(struct kernfs_node *kn, bool alloc) +{ + struct kernfs_iattrs *ret __free(kfree) = NULL; + struct kernfs_iattrs *attr; + + attr = READ_ONCE(kn->iattr); + if (attr || !alloc) + return attr; + + ret = kmem_cache_zalloc(kernfs_iattrs_cache, GFP_KERNEL); + if (!ret) + return NULL; + + INIT_LIST_HEAD_RCU(&ret->xattrs); + /* assign default attributes */ + ret->ia_uid = GLOBAL_ROOT_UID; + ret->ia_gid = GLOBAL_ROOT_GID; + + ktime_get_real_ts64(&ret->ia_atime); + ret->ia_mtime = ret->ia_atime; + ret->ia_ctime = ret->ia_atime; + + simple_xattr_limits_init(&ret->xattr_limits); + + /* If someone raced us, recognize it. */ + if (!try_cmpxchg(&kn->iattr, &attr, ret)) + return READ_ONCE(kn->iattr); + + return no_free_ptr(ret); +} + +static struct kernfs_iattrs *kernfs_iattrs(struct kernfs_node *kn) +{ + return __kernfs_iattrs(kn, true); +} + +static struct kernfs_iattrs *kernfs_iattrs_noalloc(struct kernfs_node *kn) +{ + return __kernfs_iattrs(kn, false); +} + +int __kernfs_setattr(struct kernfs_node *kn, const struct iattr *iattr) +{ + struct kernfs_iattrs *attrs; + unsigned int ia_valid = iattr->ia_valid; + + attrs = kernfs_iattrs(kn); + if (!attrs) + return -ENOMEM; + + if (ia_valid & ATTR_UID) + attrs->ia_uid = iattr->ia_uid; + if (ia_valid & ATTR_GID) + attrs->ia_gid = iattr->ia_gid; + if (ia_valid & ATTR_ATIME) + attrs->ia_atime = iattr->ia_atime; + if (ia_valid & ATTR_MTIME) + attrs->ia_mtime = iattr->ia_mtime; + if (ia_valid & ATTR_CTIME) + attrs->ia_ctime = iattr->ia_ctime; + if (ia_valid & ATTR_MODE) + kn->mode = iattr->ia_mode; + return 0; +} + +/** + * kernfs_setattr - set iattr on a node + * @kn: target node + * @iattr: iattr to set + * + * Return: %0 on success, -errno on failure. + */ +int kernfs_setattr(struct kernfs_node *kn, const struct iattr *iattr) +{ + int ret; + struct kernfs_root *root = kernfs_root(kn); + + down_write(&root->kernfs_iattr_rwsem); + ret = __kernfs_setattr(kn, iattr); + up_write(&root->kernfs_iattr_rwsem); + return ret; +} + +int kernfs_iop_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *iattr) +{ + struct inode *inode = d_inode(dentry); + struct kernfs_node *kn = inode->i_private; + struct kernfs_root *root; + int error; + + if (!kn) + return -EINVAL; + + root = kernfs_root(kn); + down_write(&root->kernfs_iattr_rwsem); + error = setattr_prepare(&nop_mnt_idmap, dentry, iattr); + if (error) + goto out; + + error = __kernfs_setattr(kn, iattr); + if (error) + goto out; + + /* this ignores size changes */ + setattr_copy(&nop_mnt_idmap, inode, iattr); + +out: + up_write(&root->kernfs_iattr_rwsem); + return error; +} + +ssize_t kernfs_iop_listxattr(struct dentry *dentry, char *buf, size_t size) +{ + struct kernfs_node *kn = kernfs_dentry_node(dentry); + struct kernfs_iattrs *attrs; + + attrs = kernfs_iattrs_noalloc(kn); + if (!attrs) + return 0; + + return simple_xattr_list(d_inode(dentry), &attrs->xattrs, buf, size); +} + +static inline void set_default_inode_attr(struct inode *inode, umode_t mode) +{ + inode->i_mode = mode; + simple_inode_init_ts(inode); +} + +static inline void set_inode_attr(struct inode *inode, + struct kernfs_iattrs *attrs) +{ + inode->i_uid = attrs->ia_uid; + inode->i_gid = attrs->ia_gid; + inode_set_atime_to_ts(inode, attrs->ia_atime); + inode_set_mtime_to_ts(inode, attrs->ia_mtime); + inode_set_ctime_to_ts(inode, attrs->ia_ctime); +} + +static void kernfs_refresh_inode(struct kernfs_node *kn, struct inode *inode) +{ + struct kernfs_iattrs *attrs; + + inode->i_mode = kn->mode; + attrs = kernfs_iattrs_noalloc(kn); + if (attrs) + /* + * kernfs_node has non-default attributes get them from + * persistent copy in kernfs_node. + */ + set_inode_attr(inode, attrs); + + if (kernfs_type(kn) == KERNFS_DIR && !(kn->flags & KERNFS_REMOVING)) + set_nlink(inode, kn->dir.subdirs + 2); +} + +int kernfs_iop_getattr(struct mnt_idmap *idmap, + const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int query_flags) +{ + struct inode *inode = d_inode(path->dentry); + struct kernfs_node *kn = inode->i_private; + struct kernfs_root *root = kernfs_root(kn); + + down_read(&root->kernfs_iattr_rwsem); + kernfs_refresh_inode(kn, inode); + generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat); + up_read(&root->kernfs_iattr_rwsem); + + return 0; +} + +static void kernfs_init_inode(struct kernfs_node *kn, struct inode *inode) +{ + kernfs_get(kn); + inode->i_private = kn; + inode->i_mapping->a_ops = &ram_aops; + inode->i_op = &kernfs_iops; + inode->i_generation = kernfs_gen(kn); + + set_default_inode_attr(inode, kn->mode); + kernfs_refresh_inode(kn, inode); + + /* initialize inode according to type */ + switch (kernfs_type(kn)) { + case KERNFS_DIR: + inode->i_op = &kernfs_dir_iops; + inode->i_fop = &kernfs_dir_fops; + if (kn->flags & KERNFS_EMPTY_DIR) + make_empty_dir_inode(inode); + break; + case KERNFS_FILE: + inode->i_size = kn->attr.size; + inode->i_fop = &kernfs_file_fops; + break; + case KERNFS_LINK: + inode->i_op = &kernfs_symlink_iops; + break; + default: + BUG(); + } + + unlock_new_inode(inode); +} + +/** + * kernfs_get_inode - get inode for kernfs_node + * @sb: super block + * @kn: kernfs_node to allocate inode for + * + * Get inode for @kn. If such inode doesn't exist, a new inode is + * allocated and basics are initialized. New inode is returned + * locked. + * + * Locking: + * Kernel thread context (may sleep). + * + * Return: + * Pointer to allocated inode on success, %NULL on failure. + */ +struct inode *kernfs_get_inode(struct super_block *sb, struct kernfs_node *kn) +{ + struct inode *inode; + + inode = iget_locked(sb, kernfs_ino(kn)); + if (inode && (inode_state_read_once(inode) & I_NEW)) + kernfs_init_inode(kn, inode); + + return inode; +} + +/* + * The kernfs_node serves as both an inode and a directory entry for + * kernfs. To prevent the kernfs inode numbers from being freed + * prematurely we take a reference to kernfs_node from the kernfs inode. A + * super_operations.evict_inode() implementation is needed to drop that + * reference upon inode destruction. + */ +void kernfs_evict_inode(struct inode *inode) +{ + struct kernfs_node *kn = inode->i_private; + + truncate_inode_pages_final(&inode->i_data); + clear_inode(inode); + kernfs_put(kn); +} + +int kernfs_iop_permission(struct mnt_idmap *idmap, + struct inode *inode, int mask) +{ + struct kernfs_node *kn; + struct kernfs_root *root; + int ret; + + if (mask & MAY_NOT_BLOCK) + return -ECHILD; + + kn = inode->i_private; + root = kernfs_root(kn); + + down_read(&root->kernfs_iattr_rwsem); + kernfs_refresh_inode(kn, inode); + ret = generic_permission(&nop_mnt_idmap, inode, mask); + up_read(&root->kernfs_iattr_rwsem); + + return ret; +} + +int kernfs_xattr_get(struct kernfs_node *kn, const char *name, + void *value, size_t size) +{ + struct kernfs_iattrs *attrs = kernfs_iattrs_noalloc(kn); + struct simple_xattr_cache *cache = &kernfs_root(kn)->xa_cache; + + if (!attrs) + return -ENODATA; + + return simple_xattr_get(cache, &attrs->xattrs, name, value, size); +} + +int kernfs_xattr_set(struct kernfs_node *kn, const char *name, + const void *value, size_t size, int flags) +{ + struct simple_xattr *old_xattr; + struct kernfs_iattrs *attrs; + struct simple_xattr_cache *cache = &kernfs_root(kn)->xa_cache; + + attrs = kernfs_iattrs(kn); + if (!attrs) + return -ENOMEM; + + /* + * Protect xattr modifications with the hashed per-node mutex. + * Multiple superblocks (with different namespaces) can share the same + * kernfs_node, so inode locking alone is insufficient. The hashed mutex + * ensures serialization of concurrent xattr operations on the same node, + * including the lazy allocation of the xattrs structure itself. + */ + CLASS(kernfs_node_lock, lock)(kn); + + old_xattr = simple_xattr_set(cache, &attrs->xattrs, name, value, size, flags); + if (IS_ERR(old_xattr)) + return PTR_ERR(old_xattr); + + simple_xattr_free_rcu(old_xattr); + return 0; +} + +static int kernfs_vfs_xattr_get(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *suffix, void *value, size_t size) +{ + const char *name = xattr_full_name(handler, suffix); + struct kernfs_node *kn = inode->i_private; + + return kernfs_xattr_get(kn, name, value, size); +} + +static int kernfs_vfs_xattr_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *suffix, const void *value, + size_t size, int flags) +{ + const char *name = xattr_full_name(handler, suffix); + struct kernfs_node *kn = inode->i_private; + + return kernfs_xattr_set(kn, name, value, size, flags); +} + +static int kernfs_vfs_user_xattr_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *suffix, const void *value, + size_t size, int flags) +{ + const char *full_name = xattr_full_name(handler, suffix); + struct kernfs_node *kn = inode->i_private; + struct kernfs_iattrs *attrs; + + if (!(kernfs_root(kn)->flags & KERNFS_ROOT_SUPPORT_USER_XATTR)) + return -EOPNOTSUPP; + + attrs = kernfs_iattrs(kn); + if (!attrs) + return -ENOMEM; + + /* See comment in kernfs_xattr_set() about locking. */ + CLASS(kernfs_node_lock, lock)(kn); + + return simple_xattr_set_limited(&kernfs_root(kn)->xa_cache, + &attrs->xattrs, &attrs->xattr_limits, + full_name, value, size, flags); +} + +static const struct xattr_handler kernfs_trusted_xattr_handler = { + .prefix = XATTR_TRUSTED_PREFIX, + .get = kernfs_vfs_xattr_get, + .set = kernfs_vfs_xattr_set, +}; + +static const struct xattr_handler kernfs_security_xattr_handler = { + .prefix = XATTR_SECURITY_PREFIX, + .get = kernfs_vfs_xattr_get, + .set = kernfs_vfs_xattr_set, +}; + +static const struct xattr_handler kernfs_user_xattr_handler = { + .prefix = XATTR_USER_PREFIX, + .get = kernfs_vfs_xattr_get, + .set = kernfs_vfs_user_xattr_set, +}; + +const struct xattr_handler * const kernfs_xattr_handlers[] = { + &kernfs_trusted_xattr_handler, + &kernfs_security_xattr_handler, + &kernfs_user_xattr_handler, + NULL +}; diff --git a/fs/kernfs/kernfs-internal.h b/fs/kernfs/kernfs-internal.h new file mode 100644 index 000000000..aa784b540 --- /dev/null +++ b/fs/kernfs/kernfs-internal.h @@ -0,0 +1,236 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * fs/kernfs/kernfs-internal.h - kernfs internal header file + * + * Copyright (c) 2001-3 Patrick Mochel + * Copyright (c) 2007 SUSE Linux Products GmbH + * Copyright (c) 2007, 2013 Tejun Heo + */ + +#ifndef __KERNFS_INTERNAL_H +#define __KERNFS_INTERNAL_H + +#include +#include +#include +#include +#include + +#include +#include + +struct kernfs_iattrs { + kuid_t ia_uid; + kgid_t ia_gid; + struct timespec64 ia_atime; + struct timespec64 ia_mtime; + struct timespec64 ia_ctime; + + struct list_head xattrs; + struct simple_xattr_limits xattr_limits; +}; + +struct kernfs_root { + /* published fields */ + struct kernfs_node *kn; + unsigned int flags; /* KERNFS_ROOT_* flags */ + + /* private fields, do not use outside kernfs proper */ + struct idr ino_idr; + spinlock_t kernfs_idr_lock; /* root->ino_idr */ + u32 last_id_lowbits; + u32 id_highbits; + struct kernfs_syscall_ops *syscall_ops; + + /* list of kernfs_super_info of this root, protected by kernfs_rwsem */ + struct list_head supers; + + wait_queue_head_t deactivate_waitq; + struct rw_semaphore kernfs_rwsem; + struct rw_semaphore kernfs_iattr_rwsem; + struct rw_semaphore kernfs_supers_rwsem; + + /* kn->parent and kn->name */ + rwlock_t kernfs_rename_lock; + + struct rcu_head rcu; + + struct simple_xattr_cache xa_cache; +}; + +/* +1 to avoid triggering overflow warning when negating it */ +#define KN_DEACTIVATED_BIAS (INT_MIN + 1) + +/* KERNFS_TYPE_MASK and types are defined in include/linux/kernfs.h */ + +/** + * kernfs_root - find out the kernfs_root a kernfs_node belongs to + * @kn: kernfs_node of interest + * + * Return: the kernfs_root @kn belongs to. + */ +static inline struct kernfs_root *kernfs_root(const struct kernfs_node *kn) +{ + const struct kernfs_node *knp; + /* if parent exists, it's always a dir; otherwise, @sd is a dir */ + guard(rcu)(); + knp = rcu_dereference(kn->__parent); + if (knp) + kn = knp; + return kn->dir.root; +} + +/* + * mount.c + */ +struct kernfs_super_info { + struct super_block *sb; + + /* + * The root associated with this super_block. Each super_block is + * identified by the root and ns it's associated with. + */ + struct kernfs_root *root; + + /* + * Each sb is associated with one namespace tag, currently the + * network namespace of the task which mounted this kernfs + * instance. If multiple tags become necessary, make the following + * an array and compare kernfs_node tag against every entry. + */ + const struct ns_common *ns; + + /* anchored at kernfs_root->supers, protected by kernfs_rwsem */ + struct list_head node; +}; +#define kernfs_info(SB) ((struct kernfs_super_info *)(SB->s_fs_info)) + +static inline bool kernfs_root_is_locked(const struct kernfs_node *kn) +{ + return lockdep_is_held(&kernfs_root(kn)->kernfs_rwsem); +} + +static inline bool kernfs_rename_is_locked(const struct kernfs_node *kn) +{ + return lockdep_is_held(&kernfs_root(kn)->kernfs_rename_lock); +} + +static inline const char *kernfs_rcu_name(const struct kernfs_node *kn) +{ + return rcu_dereference_check(kn->name, kernfs_root_is_locked(kn)); +} + +static inline struct kernfs_node *kernfs_parent(const struct kernfs_node *kn) +{ + /* + * The kernfs_node::__parent remains valid within a RCU section. The kn + * can be reparented (and renamed) which changes the entry. This can be + * avoided by locking kernfs_root::kernfs_rwsem or + * kernfs_root::kernfs_rename_lock. + * Both locks can be used to obtain a reference on __parent. Once the + * reference count reaches 0 then the node is about to be freed + * and can not be renamed (or become a different parent) anymore. + */ + return rcu_dereference_check(kn->__parent, + kernfs_root_is_locked(kn) || + kernfs_rename_is_locked(kn) || + !atomic_read(&kn->count)); +} + +static inline struct kernfs_node *kernfs_dentry_node(struct dentry *dentry) +{ + if (d_really_is_negative(dentry)) + return NULL; + return d_inode(dentry)->i_private; +} + +static inline void kernfs_set_rev(struct kernfs_node *parent, + struct dentry *dentry) +{ + dentry->d_time = parent->dir.rev; +} + +static inline void kernfs_inc_rev(struct kernfs_node *parent) +{ + parent->dir.rev++; +} + +static inline bool kernfs_dir_changed(struct kernfs_node *parent, + struct dentry *dentry) +{ + if (parent->dir.rev != dentry->d_time) + return true; + return false; +} + +extern const struct super_operations kernfs_sops; +extern struct kmem_cache *kernfs_node_cache, *kernfs_iattrs_cache; + +/* + * inode.c + */ +extern const struct xattr_handler * const kernfs_xattr_handlers[]; +void kernfs_evict_inode(struct inode *inode); +int kernfs_iop_permission(struct mnt_idmap *idmap, + struct inode *inode, int mask); +int kernfs_iop_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *iattr); +int kernfs_iop_getattr(struct mnt_idmap *idmap, + const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int query_flags); +ssize_t kernfs_iop_listxattr(struct dentry *dentry, char *buf, size_t size); +int __kernfs_setattr(struct kernfs_node *kn, const struct iattr *iattr); + +/* + * dir.c + */ +extern const struct dentry_operations kernfs_dops; +extern const struct file_operations kernfs_dir_fops; +extern const struct inode_operations kernfs_dir_iops; + +struct kernfs_node *kernfs_get_active(struct kernfs_node *kn); +void kernfs_put_active(struct kernfs_node *kn); +int kernfs_add_one(struct kernfs_node *kn); +struct kernfs_node *kernfs_new_node(struct kernfs_node *parent, + const char *name, umode_t mode, + kuid_t uid, kgid_t gid, + unsigned flags); + +/* + * file.c + */ +extern const struct file_operations kernfs_file_fops; + +bool kernfs_should_drain_open_files(struct kernfs_node *kn); +void kernfs_drain_open_files(struct kernfs_node *kn); + +/* + * symlink.c + */ +extern const struct inode_operations kernfs_symlink_iops; + +/* + * kernfs locks + */ +extern struct kernfs_global_locks *kernfs_locks; + +/* Hashed mutex helpers - protect per-node data structures */ +static inline struct mutex *kernfs_node_lock_ptr(struct kernfs_node *kn) +{ + int idx = hash_ptr(kn, NR_KERNFS_LOCK_BITS); + + return &kernfs_locks->node_mutex[idx]; +} + +static inline struct mutex *kernfs_node_lock(struct kernfs_node *kn) +{ + struct mutex *lock = kernfs_node_lock_ptr(kn); + + mutex_lock(lock); + return lock; +} + +DEFINE_CLASS(kernfs_node_lock, struct mutex *, + mutex_unlock(_T), kernfs_node_lock(kn), struct kernfs_node *kn) + +#endif /* __KERNFS_INTERNAL_H */ diff --git a/fs/kernfs/mount.c b/fs/kernfs/mount.c new file mode 100644 index 000000000..f183a9677 --- /dev/null +++ b/fs/kernfs/mount.c @@ -0,0 +1,472 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * fs/kernfs/mount.c - kernfs mount implementation + * + * Copyright (c) 2001-3 Patrick Mochel + * Copyright (c) 2007 SUSE Linux Products GmbH + * Copyright (c) 2007, 2013 Tejun Heo + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kernfs-internal.h" + +struct kmem_cache *kernfs_node_cache __ro_after_init; +struct kmem_cache *kernfs_iattrs_cache __ro_after_init; +struct kernfs_global_locks *kernfs_locks __ro_after_init; + +static int kernfs_sop_show_options(struct seq_file *sf, struct dentry *dentry) +{ + struct kernfs_root *root = kernfs_root(kernfs_dentry_node(dentry)); + struct kernfs_syscall_ops *scops = root->syscall_ops; + + if (scops && scops->show_options) + return scops->show_options(sf, root); + return 0; +} + +static int kernfs_sop_show_path(struct seq_file *sf, struct dentry *dentry) +{ + struct kernfs_node *node = kernfs_dentry_node(dentry); + struct kernfs_root *root = kernfs_root(node); + struct kernfs_syscall_ops *scops = root->syscall_ops; + + if (scops && scops->show_path) + return scops->show_path(sf, node, root); + + seq_dentry(sf, dentry, " \t\n\\"); + return 0; +} + +static int kernfs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + simple_statfs(dentry, buf); + buf->f_fsid = uuid_to_fsid(dentry->d_sb->s_uuid.b); + return 0; +} + +const struct super_operations kernfs_sops = { + .statfs = kernfs_statfs, + .drop_inode = inode_just_drop, + .evict_inode = kernfs_evict_inode, + + .show_options = kernfs_sop_show_options, + .show_path = kernfs_sop_show_path, + + /* + * sysfs is built on top of kernfs and sysfs provides the power + * management infrastructure to support suspend/hibernate by + * writing to various files in /sys/power/. As filesystems may + * be automatically frozen during suspend/hibernate implementing + * freeze/thaw support for kernfs generically will cause + * deadlocks as the suspending/hibernation initiating task will + * hold a VFS lock that it will then wait upon to be released. + * If freeze/thaw for kernfs is needed talk to the VFS. + */ + .freeze_fs = NULL, + .unfreeze_fs = NULL, + .freeze_super = NULL, + .thaw_super = NULL, +}; + +static int kernfs_encode_fh(struct inode *inode, __u32 *fh, int *max_len, + struct inode *parent) +{ + struct kernfs_node *kn = inode->i_private; + + if (*max_len < 2) { + *max_len = 2; + return FILEID_INVALID; + } + + *max_len = 2; + *(u64 *)fh = kn->id; + return FILEID_KERNFS; +} + +static struct dentry *__kernfs_fh_to_dentry(struct super_block *sb, + struct fid *fid, int fh_len, + int fh_type, bool get_parent) +{ + struct kernfs_super_info *info = kernfs_info(sb); + struct kernfs_node *kn; + struct inode *inode; + u64 id; + + if (fh_len < 2) + return NULL; + + switch (fh_type) { + case FILEID_KERNFS: + id = *(u64 *)fid; + break; + case FILEID_INO32_GEN: + case FILEID_INO32_GEN_PARENT: + /* + * blk_log_action() exposes "LOW32,HIGH32" pair without + * type and userland can call us with generic fid + * constructed from them. Combine it back to ID. See + * blk_log_action(). + */ + id = ((u64)fid->i32.gen << 32) | fid->i32.ino; + break; + default: + return NULL; + } + + kn = kernfs_find_and_get_node_by_id(info->root, id); + if (!kn) + return ERR_PTR(-ESTALE); + + if (get_parent) { + struct kernfs_node *parent; + + parent = kernfs_get_parent(kn); + kernfs_put(kn); + kn = parent; + if (!kn) + return ERR_PTR(-ESTALE); + } + + inode = kernfs_get_inode(sb, kn); + kernfs_put(kn); + return d_obtain_alias(inode); +} + +static struct dentry *kernfs_fh_to_dentry(struct super_block *sb, + struct fid *fid, int fh_len, + int fh_type) +{ + return __kernfs_fh_to_dentry(sb, fid, fh_len, fh_type, false); +} + +static struct dentry *kernfs_fh_to_parent(struct super_block *sb, + struct fid *fid, int fh_len, + int fh_type) +{ + return __kernfs_fh_to_dentry(sb, fid, fh_len, fh_type, true); +} + +static struct dentry *kernfs_get_parent_dentry(struct dentry *child) +{ + struct kernfs_node *kn = kernfs_dentry_node(child); + struct kernfs_root *root = kernfs_root(kn); + + guard(rwsem_read)(&root->kernfs_rwsem); + return d_obtain_alias(kernfs_get_inode(child->d_sb, kernfs_parent(kn))); +} + +static const struct export_operations kernfs_export_ops = { + .encode_fh = kernfs_encode_fh, + .fh_to_dentry = kernfs_fh_to_dentry, + .fh_to_parent = kernfs_fh_to_parent, + .get_parent = kernfs_get_parent_dentry, +}; + +/** + * kernfs_root_from_sb - determine kernfs_root associated with a super_block + * @sb: the super_block in question + * + * Return: the kernfs_root associated with @sb. If @sb is not a kernfs one, + * %NULL is returned. + */ +struct kernfs_root *kernfs_root_from_sb(struct super_block *sb) +{ + if (sb->s_op == &kernfs_sops) + return kernfs_info(sb)->root; + return NULL; +} + +/* + * find the next ancestor in the path down to @child, where @parent was the + * ancestor whose descendant we want to find. + * + * Say the path is /a/b/c/d. @child is d, @parent is %NULL. We return the root + * node. If @parent is b, then we return the node for c. + * Passing in d as @parent is not ok. + */ +static struct kernfs_node *find_next_ancestor(struct kernfs_node *child, + struct kernfs_node *parent) +{ + if (child == parent) { + pr_crit_once("BUG in find_next_ancestor: called with parent == child"); + return NULL; + } + + while (kernfs_parent(child) != parent) { + child = kernfs_parent(child); + if (!child) + return NULL; + } + + return child; +} + +/** + * kernfs_node_dentry - get a dentry for the given kernfs_node + * @kn: kernfs_node for which a dentry is needed + * @sb: the kernfs super_block + * + * Return: the dentry pointer + */ +struct dentry *kernfs_node_dentry(struct kernfs_node *kn, + struct super_block *sb) +{ + struct dentry *dentry; + struct kernfs_node *knparent; + struct kernfs_root *root; + + BUG_ON(sb->s_op != &kernfs_sops); + + dentry = dget(sb->s_root); + + /* Check if this is the root kernfs_node */ + if (!rcu_access_pointer(kn->__parent)) + return dentry; + + root = kernfs_root(kn); + /* + * As long as kn is valid, its parent can not vanish. This is cgroup's + * kn so it can't have its parent replaced. Therefore it is safe to use + * the ancestor node outside of the RCU or locked section. + */ + if (WARN_ON_ONCE(!(root->flags & KERNFS_ROOT_INVARIANT_PARENT))) + return ERR_PTR(-EINVAL); + scoped_guard(rcu) { + knparent = find_next_ancestor(kn, NULL); + } + if (WARN_ON(!knparent)) { + dput(dentry); + return ERR_PTR(-EINVAL); + } + + do { + struct dentry *dtmp; + struct kernfs_node *kntmp; + const char *name; + + if (kn == knparent) + return dentry; + + scoped_guard(rwsem_read, &root->kernfs_rwsem) { + kntmp = find_next_ancestor(kn, knparent); + if (WARN_ON(!kntmp)) { + dput(dentry); + return ERR_PTR(-EINVAL); + } + name = kstrdup(kernfs_rcu_name(kntmp), GFP_KERNEL); + } + if (!name) { + dput(dentry); + return ERR_PTR(-ENOMEM); + } + dtmp = lookup_noperm_positive_unlocked(&QSTR(name), dentry); + dput(dentry); + kfree(name); + if (IS_ERR(dtmp)) + return dtmp; + knparent = kntmp; + dentry = dtmp; + } while (true); +} + +static int kernfs_fill_super(struct super_block *sb, struct kernfs_fs_context *kfc) +{ + struct kernfs_super_info *info = kernfs_info(sb); + struct kernfs_root *kf_root = kfc->root; + struct inode *inode; + struct dentry *root; + + info->sb = sb; + /* Userspace would break if executables or devices appear on sysfs */ + sb->s_iflags |= SB_I_NOEXEC | SB_I_NODEV; + sb->s_blocksize = PAGE_SIZE; + sb->s_blocksize_bits = PAGE_SHIFT; + sb->s_magic = kfc->magic; + sb->s_op = &kernfs_sops; + sb->s_xattr = kernfs_xattr_handlers; + if (info->root->flags & KERNFS_ROOT_SUPPORT_EXPORTOP) + sb->s_export_op = &kernfs_export_ops; + sb->s_time_gran = 1; + sb->s_maxbytes = MAX_LFS_FILESIZE; + + /* sysfs dentries and inodes don't require IO to create */ + sb->s_shrink->seeks = 0; + + /* get root inode, initialize and unlock it */ + down_read(&kf_root->kernfs_rwsem); + inode = kernfs_get_inode(sb, info->root->kn); + up_read(&kf_root->kernfs_rwsem); + if (!inode) { + pr_debug("kernfs: could not get root inode\n"); + return -ENOMEM; + } + + /* instantiate and link root dentry */ + root = d_make_root(inode); + if (!root) { + pr_debug("%s: could not get root dentry!\n", __func__); + return -ENOMEM; + } + sb->s_root = root; + set_default_d_op(sb, &kernfs_dops); + return 0; +} + +static int kernfs_test_super(struct super_block *sb, struct fs_context *fc) +{ + struct kernfs_super_info *sb_info = kernfs_info(sb); + struct kernfs_super_info *info = fc->s_fs_info; + + return sb_info->root == info->root && sb_info->ns == info->ns; +} + +static int kernfs_set_super(struct super_block *sb, struct fs_context *fc) +{ + struct kernfs_fs_context *kfc = fc->fs_private; + + kfc->ns_tag = NULL; + return set_anon_super_fc(sb, fc); +} + +/** + * kernfs_super_ns - determine the namespace tag of a kernfs super_block + * @sb: super_block of interest + * + * Return: the namespace tag associated with kernfs super_block @sb. + */ +const struct ns_common *kernfs_super_ns(struct super_block *sb) +{ + struct kernfs_super_info *info = kernfs_info(sb); + + return info->ns; +} + +/** + * kernfs_get_tree - kernfs filesystem access/retrieval helper + * @fc: The filesystem context. + * + * This is to be called from each kernfs user's fs_context->ops->get_tree() + * implementation, which should set the specified ->@fs_type and ->@flags, and + * specify the hierarchy and namespace tag to mount via ->@root and ->@ns, + * respectively. + * + * Return: %0 on success, -errno on failure. + */ +int kernfs_get_tree(struct fs_context *fc) +{ + struct kernfs_fs_context *kfc = fc->fs_private; + struct super_block *sb; + struct kernfs_super_info *info; + int error; + + info = kzalloc_obj(*info); + if (!info) + return -ENOMEM; + + info->root = kfc->root; + info->ns = kfc->ns_tag; + INIT_LIST_HEAD(&info->node); + + fc->s_fs_info = info; + sb = sget_fc(fc, kernfs_test_super, kernfs_set_super); + if (IS_ERR(sb)) + return PTR_ERR(sb); + + if (!sb->s_root) { + struct kernfs_super_info *info = kernfs_info(sb); + struct kernfs_root *root = kfc->root; + + kfc->new_sb_created = true; + + error = kernfs_fill_super(sb, kfc); + if (error) { + deactivate_locked_super(sb); + return error; + } + sb->s_flags |= SB_ACTIVE; + + uuid_t uuid; + uuid_gen(&uuid); + super_set_uuid(sb, uuid.b, sizeof(uuid)); + + down_write(&root->kernfs_supers_rwsem); + list_add(&info->node, &info->root->supers); + up_write(&root->kernfs_supers_rwsem); + } + + fc->root = dget(sb->s_root); + return 0; +} + +void kernfs_free_fs_context(struct fs_context *fc) +{ + /* Note that we don't deal with kfc->ns_tag here. */ + kfree(fc->s_fs_info); + fc->s_fs_info = NULL; +} + +/** + * kernfs_kill_sb - kill_sb for kernfs + * @sb: super_block being killed + * + * This can be used directly for file_system_type->kill_sb(). If a kernfs + * user needs extra cleanup, it can implement its own kill_sb() and call + * this function at the end. + */ +void kernfs_kill_sb(struct super_block *sb) +{ + struct kernfs_super_info *info = kernfs_info(sb); + struct kernfs_root *root = info->root; + + down_write(&root->kernfs_supers_rwsem); + list_del(&info->node); + up_write(&root->kernfs_supers_rwsem); + + /* + * Remove the superblock from fs_supers/s_instances + * so we can't find it, before freeing kernfs_super_info. + */ + kill_anon_super(sb); + kfree(info); +} + +static void __init kernfs_mutex_init(void) +{ + int count; + + for (count = 0; count < NR_KERNFS_LOCKS; count++) + mutex_init(&kernfs_locks->node_mutex[count]); +} + +static void __init kernfs_lock_init(void) +{ + kernfs_locks = kmalloc_obj(struct kernfs_global_locks); + WARN_ON(!kernfs_locks); + + kernfs_mutex_init(); +} + +void __init kernfs_init(void) +{ + kernfs_node_cache = kmem_cache_create("kernfs_node_cache", + sizeof(struct kernfs_node), + 0, SLAB_PANIC, NULL); + + /* Creates slab cache for kernfs inode attributes */ + kernfs_iattrs_cache = kmem_cache_create("kernfs_iattrs_cache", + sizeof(struct kernfs_iattrs), + 0, SLAB_PANIC, NULL); + + kernfs_lock_init(); +} diff --git a/fs/kernfs/symlink.c b/fs/kernfs/symlink.c new file mode 100644 index 000000000..90e2b3221 --- /dev/null +++ b/fs/kernfs/symlink.c @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * fs/kernfs/symlink.c - kernfs symlink implementation + * + * Copyright (c) 2001-3 Patrick Mochel + * Copyright (c) 2007 SUSE Linux Products GmbH + * Copyright (c) 2007, 2013 Tejun Heo + */ + +#include +#include +#include + +#include "kernfs-internal.h" + +/** + * kernfs_create_link - create a symlink + * @parent: directory to create the symlink in + * @name: name of the symlink + * @target: target node for the symlink to point to + * + * Return: the created node on success, ERR_PTR() value on error. + * Ownership of the link matches ownership of the target. + */ +struct kernfs_node *kernfs_create_link(struct kernfs_node *parent, + const char *name, + struct kernfs_node *target) +{ + struct kernfs_node *kn; + int error; + kuid_t uid = GLOBAL_ROOT_UID; + kgid_t gid = GLOBAL_ROOT_GID; + + if (target->iattr) { + uid = target->iattr->ia_uid; + gid = target->iattr->ia_gid; + } + + kn = kernfs_new_node(parent, name, S_IFLNK|0777, uid, gid, KERNFS_LINK); + if (!kn) + return ERR_PTR(-ENOMEM); + + if (kernfs_ns_enabled(parent)) + kn->ns = target->ns; + kn->symlink.target_kn = target; + kernfs_get(target); /* ref owned by symlink */ + + error = kernfs_add_one(kn); + if (!error) + return kn; + + kernfs_put(kn); + return ERR_PTR(error); +} + +static int kernfs_get_target_path(struct kernfs_node *parent, + struct kernfs_node *target, char *path) +{ + struct kernfs_node *base, *kn; + char *s = path; + int len = 0; + + /* go up to the root, stop at the base */ + base = parent; + while (kernfs_parent(base)) { + kn = kernfs_parent(target); + while (kernfs_parent(kn) && base != kn) + kn = kernfs_parent(kn); + + if (base == kn) + break; + + if ((s - path) + 3 >= PATH_MAX) + return -ENAMETOOLONG; + + memcpy(s, "../", 4); + s += 3; + base = kernfs_parent(base); + } + + /* determine end of target string for reverse fillup */ + kn = target; + while (kernfs_parent(kn) && kn != base) { + len += strlen(kernfs_rcu_name(kn)) + 1; + kn = kernfs_parent(kn); + } + + /* check limits */ + if (len < 2) + return -EINVAL; + len--; + if ((s - path) + len >= PATH_MAX) + return -ENAMETOOLONG; + + /* reverse fillup of target string from target to base */ + kn = target; + while (kernfs_parent(kn) && kn != base) { + const char *name = kernfs_rcu_name(kn); + int slen = strlen(name); + + len -= slen; + memcpy(s + len, name, slen); + if (len) + s[--len] = '/'; + + kn = kernfs_parent(kn); + } + + return 0; +} + +static int kernfs_getlink(struct inode *inode, char *path) +{ + struct kernfs_node *kn = inode->i_private; + struct kernfs_node *parent; + struct kernfs_node *target = kn->symlink.target_kn; + struct kernfs_root *root = kernfs_root(kn); + int error; + + down_read(&root->kernfs_rwsem); + parent = kernfs_parent(kn); + error = kernfs_get_target_path(parent, target, path); + up_read(&root->kernfs_rwsem); + + return error; +} + +static const char *kernfs_iop_get_link(struct dentry *dentry, + struct inode *inode, + struct delayed_call *done) +{ + char *body; + int error; + + if (!dentry) + return ERR_PTR(-ECHILD); + body = kzalloc(PAGE_SIZE, GFP_KERNEL); + if (!body) + return ERR_PTR(-ENOMEM); + error = kernfs_getlink(inode, body); + if (unlikely(error < 0)) { + kfree(body); + return ERR_PTR(error); + } + set_delayed_call(done, kfree_link, body); + return body; +} + +const struct inode_operations kernfs_symlink_iops = { + .listxattr = kernfs_iop_listxattr, + .get_link = kernfs_iop_get_link, + .setattr = kernfs_iop_setattr, + .getattr = kernfs_iop_getattr, + .permission = kernfs_iop_permission, +}; diff --git a/fs/libfs.c b/fs/libfs.c new file mode 100644 index 000000000..27d7dc16f --- /dev/null +++ b/fs/libfs.c @@ -0,0 +1,2317 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * fs/libfs.c + * Library for filesystems writers. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "internal.h" + +int simple_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, + unsigned int query_flags) +{ + struct inode *inode = d_inode(path->dentry); + generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat); + stat->blocks = inode->i_mapping->nrpages << (PAGE_SHIFT - 9); + return 0; +} +EXPORT_SYMBOL(simple_getattr); + +int simple_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + u64 id = huge_encode_dev(dentry->d_sb->s_dev); + + buf->f_fsid = u64_to_fsid(id); + buf->f_type = dentry->d_sb->s_magic; + buf->f_bsize = PAGE_SIZE; + buf->f_namelen = NAME_MAX; + return 0; +} +EXPORT_SYMBOL(simple_statfs); + +/* + * Retaining negative dentries for an in-memory filesystem just wastes + * memory and lookup time: arrange for them to be deleted immediately. + */ +int always_delete_dentry(const struct dentry *dentry) +{ + return 1; +} +EXPORT_SYMBOL(always_delete_dentry); + +/* + * Lookup the data. This is trivial - if the dentry didn't already + * exist, we know it is negative. Set d_op to delete negative dentries. + */ +struct dentry *simple_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) +{ + if (dentry->d_name.len > NAME_MAX) + return ERR_PTR(-ENAMETOOLONG); + if (!dentry->d_op && !(dentry->d_flags & DCACHE_DONTCACHE)) { + spin_lock(&dentry->d_lock); + dentry->d_flags |= DCACHE_DONTCACHE; + spin_unlock(&dentry->d_lock); + } + if (IS_ENABLED(CONFIG_UNICODE) && IS_CASEFOLDED(dir)) + return NULL; + + return d_splice_alias(NULL, dentry); +} +EXPORT_SYMBOL(simple_lookup); + +int dcache_dir_open(struct inode *inode, struct file *file) +{ + file->private_data = d_alloc_cursor(file->f_path.dentry); + + return file->private_data ? 0 : -ENOMEM; +} +EXPORT_SYMBOL(dcache_dir_open); + +int dcache_dir_close(struct inode *inode, struct file *file) +{ + dput(file->private_data); + return 0; +} +EXPORT_SYMBOL(dcache_dir_close); + +/* parent is locked at least shared */ +/* + * Returns an element of siblings' list. + * We are looking for th positive after

; if + * found, dentry is grabbed and returned to caller. + * If no such element exists, NULL is returned. + */ +static struct dentry *scan_positives(struct dentry *cursor, + struct hlist_node **p, + loff_t count, + struct dentry *last) +{ + struct dentry *dentry = cursor->d_parent, *found = NULL; + + spin_lock(&dentry->d_lock); + while (*p) { + struct dentry *d = hlist_entry(*p, struct dentry, d_sib); + p = &d->d_sib.next; + // we must at least skip cursors, to avoid livelocks + if (d->d_flags & DCACHE_DENTRY_CURSOR) + continue; + if (simple_positive(d) && !--count) { + spin_lock_nested(&d->d_lock, DENTRY_D_LOCK_NESTED); + if (simple_positive(d)) + found = dget_dlock(d); + spin_unlock(&d->d_lock); + if (likely(found)) + break; + count = 1; + } + if (need_resched()) { + if (!hlist_unhashed(&cursor->d_sib)) + __hlist_del(&cursor->d_sib); + hlist_add_behind(&cursor->d_sib, &d->d_sib); + p = &cursor->d_sib.next; + spin_unlock(&dentry->d_lock); + cond_resched(); + spin_lock(&dentry->d_lock); + } + } + spin_unlock(&dentry->d_lock); + dput(last); + return found; +} + +loff_t dcache_dir_lseek(struct file *file, loff_t offset, int whence) +{ + struct dentry *dentry = file->f_path.dentry; + switch (whence) { + case 1: + offset += file->f_pos; + fallthrough; + case 0: + if (offset >= 0) + break; + fallthrough; + default: + return -EINVAL; + } + if (offset != file->f_pos) { + struct dentry *cursor = file->private_data; + struct dentry *to = NULL; + + inode_lock_shared(dentry->d_inode); + + if (offset > 2) + to = scan_positives(cursor, &dentry->d_children.first, + offset - 2, NULL); + spin_lock(&dentry->d_lock); + hlist_del_init(&cursor->d_sib); + if (to) + hlist_add_behind(&cursor->d_sib, &to->d_sib); + spin_unlock(&dentry->d_lock); + dput(to); + + file->f_pos = offset; + + inode_unlock_shared(dentry->d_inode); + } + return offset; +} +EXPORT_SYMBOL(dcache_dir_lseek); + +/* + * Directory is locked and all positive dentries in it are safe, since + * for ramfs-type trees they can't go away without unlink() or rmdir(), + * both impossible due to the lock on directory. + */ + +int dcache_readdir(struct file *file, struct dir_context *ctx) +{ + struct dentry *dentry = file->f_path.dentry; + struct dentry *cursor = file->private_data; + struct dentry *next = NULL; + struct hlist_node **p; + + if (!dir_emit_dots(file, ctx)) + return 0; + + if (ctx->pos == 2) + p = &dentry->d_children.first; + else + p = &cursor->d_sib.next; + + while ((next = scan_positives(cursor, p, 1, next)) != NULL) { + if (!dir_emit(ctx, next->d_name.name, next->d_name.len, + d_inode(next)->i_ino, + fs_umode_to_dtype(d_inode(next)->i_mode))) + break; + ctx->pos++; + p = &next->d_sib.next; + } + spin_lock(&dentry->d_lock); + hlist_del_init(&cursor->d_sib); + if (next) + hlist_add_before(&cursor->d_sib, &next->d_sib); + spin_unlock(&dentry->d_lock); + dput(next); + + return 0; +} +EXPORT_SYMBOL(dcache_readdir); + +ssize_t generic_read_dir(struct file *filp, char __user *buf, size_t siz, loff_t *ppos) +{ + return -EISDIR; +} +EXPORT_SYMBOL(generic_read_dir); + +const struct file_operations simple_dir_operations = { + .open = dcache_dir_open, + .release = dcache_dir_close, + .llseek = dcache_dir_lseek, + .read = generic_read_dir, + .iterate_shared = dcache_readdir, + .fsync = noop_fsync, +}; +EXPORT_SYMBOL(simple_dir_operations); + +const struct inode_operations simple_dir_inode_operations = { + .lookup = simple_lookup, +}; +EXPORT_SYMBOL(simple_dir_inode_operations); + +/* simple_offset_add() never assigns these to a dentry */ +enum { + DIR_OFFSET_FIRST = 2, /* Find first real entry */ + DIR_OFFSET_EOD = S32_MAX, +}; + +/* simple_offset_add() allocation range */ +enum { + DIR_OFFSET_MIN = DIR_OFFSET_FIRST + 1, + DIR_OFFSET_MAX = DIR_OFFSET_EOD - 1, +}; + +static void offset_set(struct dentry *dentry, long offset) +{ + dentry->d_fsdata = (void *)offset; +} + +static long dentry2offset(struct dentry *dentry) +{ + return (long)dentry->d_fsdata; +} + +static struct lock_class_key simple_offset_lock_class; + +/** + * simple_offset_init - initialize an offset_ctx + * @octx: directory offset map to be initialized + * + */ +void simple_offset_init(struct offset_ctx *octx) +{ + mt_init_flags(&octx->mt, MT_FLAGS_ALLOC_RANGE); + lockdep_set_class(&octx->mt.ma_lock, &simple_offset_lock_class); + octx->next_offset = DIR_OFFSET_MIN; +} + +/** + * simple_offset_add - Add an entry to a directory's offset map + * @octx: directory offset ctx to be updated + * @dentry: new dentry being added + * + * Returns zero on success. @octx and the dentry's offset are updated. + * Otherwise, a negative errno value is returned. + */ +int simple_offset_add(struct offset_ctx *octx, struct dentry *dentry) +{ + unsigned long offset; + int ret; + + if (dentry2offset(dentry) != 0) + return -EBUSY; + + ret = mtree_alloc_cyclic(&octx->mt, &offset, dentry, DIR_OFFSET_MIN, + DIR_OFFSET_MAX, &octx->next_offset, + GFP_KERNEL); + if (unlikely(ret < 0)) + return ret == -EBUSY ? -ENOSPC : ret; + + offset_set(dentry, offset); + return 0; +} + +static int simple_offset_replace(struct offset_ctx *octx, struct dentry *dentry, + long offset) +{ + int ret; + + ret = mtree_store(&octx->mt, offset, dentry, GFP_KERNEL); + if (ret) + return ret; + offset_set(dentry, offset); + return 0; +} + +/** + * simple_offset_remove - Remove an entry to a directory's offset map + * @octx: directory offset ctx to be updated + * @dentry: dentry being removed + * + */ +void simple_offset_remove(struct offset_ctx *octx, struct dentry *dentry) +{ + long offset; + + offset = dentry2offset(dentry); + if (offset == 0) + return; + + mtree_erase(&octx->mt, offset); + offset_set(dentry, 0); +} + +/** + * simple_offset_rename - handle directory offsets for rename + * @old_dir: parent directory of source entry + * @old_dentry: dentry of source entry + * @new_dir: parent_directory of destination entry + * @new_dentry: dentry of destination + * + * Caller provides appropriate serialization. + * + * User space expects the directory offset value of the replaced + * (new) directory entry to be unchanged after a rename. + * + * Caller must have grabbed a slot for new_dentry in the maple_tree + * associated with new_dir, even if dentry is negative. + */ +void simple_offset_rename(struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry) +{ + struct offset_ctx *old_ctx = old_dir->i_op->get_offset_ctx(old_dir); + struct offset_ctx *new_ctx = new_dir->i_op->get_offset_ctx(new_dir); + long new_offset = dentry2offset(new_dentry); + + if (WARN_ON(!new_offset)) + return; + + simple_offset_remove(old_ctx, old_dentry); + offset_set(new_dentry, 0); + WARN_ON(simple_offset_replace(new_ctx, old_dentry, new_offset)); +} + +/** + * simple_offset_rename_exchange - exchange rename with directory offsets + * @old_dir: parent of dentry being moved + * @old_dentry: dentry being moved + * @new_dir: destination parent + * @new_dentry: destination dentry + * + * This API preserves the directory offset values. Caller provides + * appropriate serialization. + * + * Returns zero on success. Otherwise a negative errno is returned and the + * rename is rolled back. + */ +int simple_offset_rename_exchange(struct inode *old_dir, + struct dentry *old_dentry, + struct inode *new_dir, + struct dentry *new_dentry) +{ + struct offset_ctx *old_ctx = old_dir->i_op->get_offset_ctx(old_dir); + struct offset_ctx *new_ctx = new_dir->i_op->get_offset_ctx(new_dir); + long old_index = dentry2offset(old_dentry); + long new_index = dentry2offset(new_dentry); + int ret; + + if (WARN_ON(!old_index || !new_index)) + return -EINVAL; + + ret = mtree_store(&new_ctx->mt, new_index, old_dentry, GFP_KERNEL); + if (WARN_ON(ret)) + return ret; + + ret = mtree_store(&old_ctx->mt, old_index, new_dentry, GFP_KERNEL); + if (WARN_ON(ret)) { + mtree_store(&new_ctx->mt, new_index, new_dentry, GFP_KERNEL); + return ret; + } + + offset_set(old_dentry, new_index); + offset_set(new_dentry, old_index); + simple_rename_exchange(old_dir, old_dentry, new_dir, new_dentry); + return 0; +} + +/** + * simple_offset_destroy - Release offset map + * @octx: directory offset ctx that is about to be destroyed + * + * During fs teardown (eg. umount), a directory's offset map might still + * contain entries. xa_destroy() cleans out anything that remains. + */ +void simple_offset_destroy(struct offset_ctx *octx) +{ + mtree_destroy(&octx->mt); +} + +/** + * offset_dir_llseek - Advance the read position of a directory descriptor + * @file: an open directory whose position is to be updated + * @offset: a byte offset + * @whence: enumerator describing the starting position for this update + * + * SEEK_END, SEEK_DATA, and SEEK_HOLE are not supported for directories. + * + * Returns the updated read position if successful; otherwise a + * negative errno is returned and the read position remains unchanged. + */ +static loff_t offset_dir_llseek(struct file *file, loff_t offset, int whence) +{ + switch (whence) { + case SEEK_CUR: + offset += file->f_pos; + fallthrough; + case SEEK_SET: + if (offset >= 0) + break; + fallthrough; + default: + return -EINVAL; + } + + return vfs_setpos(file, offset, LONG_MAX); +} + +static struct dentry *find_positive_dentry(struct dentry *parent, + struct dentry *dentry, + bool next) +{ + struct dentry *found = NULL; + + spin_lock(&parent->d_lock); + if (next) + dentry = d_next_sibling(dentry); + else if (!dentry) + dentry = d_first_child(parent); + hlist_for_each_entry_from(dentry, d_sib) { + if (!simple_positive(dentry)) + continue; + spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); + if (simple_positive(dentry)) + found = dget_dlock(dentry); + spin_unlock(&dentry->d_lock); + if (likely(found)) + break; + } + spin_unlock(&parent->d_lock); + return found; +} + +static noinline_for_stack struct dentry * +offset_dir_lookup(struct dentry *parent, loff_t offset) +{ + struct inode *inode = d_inode(parent); + struct offset_ctx *octx = inode->i_op->get_offset_ctx(inode); + struct dentry *child, *found = NULL; + + MA_STATE(mas, &octx->mt, offset, offset); + + if (offset == DIR_OFFSET_FIRST) + found = find_positive_dentry(parent, NULL, false); + else { + rcu_read_lock(); + child = mas_find_rev(&mas, DIR_OFFSET_MIN); + found = find_positive_dentry(parent, child, false); + rcu_read_unlock(); + } + return found; +} + +static bool offset_dir_emit(struct dir_context *ctx, struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + + return dir_emit(ctx, dentry->d_name.name, dentry->d_name.len, + inode->i_ino, fs_umode_to_dtype(inode->i_mode)); +} + +static void offset_iterate_dir(struct file *file, struct dir_context *ctx) +{ + struct dentry *dir = file->f_path.dentry; + struct dentry *dentry; + + dentry = offset_dir_lookup(dir, ctx->pos); + if (!dentry) + goto out_eod; + while (true) { + struct dentry *next; + + ctx->pos = dentry2offset(dentry); + if (!offset_dir_emit(ctx, dentry)) + break; + + next = find_positive_dentry(dir, dentry, true); + dput(dentry); + + if (!next) + goto out_eod; + dentry = next; + } + dput(dentry); + return; + +out_eod: + ctx->pos = DIR_OFFSET_EOD; +} + +/** + * offset_readdir - Emit entries starting at offset @ctx->pos + * @file: an open directory to iterate over + * @ctx: directory iteration context + * + * Caller must hold @file's i_rwsem to prevent insertion or removal of + * entries during this call. + * + * On entry, @ctx->pos contains an offset that represents the first entry + * to be read from the directory. + * + * The operation continues until there are no more entries to read, or + * until the ctx->actor indicates there is no more space in the caller's + * output buffer. + * + * On return, @ctx->pos contains an offset that will read the next entry + * in this directory when offset_readdir() is called again with @ctx. + * Caller places this value in the d_off field of the last entry in the + * user's buffer. + * + * Return values: + * %0 - Complete + */ +static int offset_readdir(struct file *file, struct dir_context *ctx) +{ + struct dentry *dir = file->f_path.dentry; + + lockdep_assert_held(&d_inode(dir)->i_rwsem); + + if (!dir_emit_dots(file, ctx)) + return 0; + if (ctx->pos != DIR_OFFSET_EOD) + offset_iterate_dir(file, ctx); + return 0; +} + +const struct file_operations simple_offset_dir_operations = { + .llseek = offset_dir_llseek, + .iterate_shared = offset_readdir, + .read = generic_read_dir, + .fsync = noop_fsync, + .setlease = generic_setlease, +}; + +struct dentry *find_next_child(struct dentry *parent, struct dentry *prev) +{ + struct dentry *child = NULL, *d; + + spin_lock(&parent->d_lock); + d = prev ? d_next_sibling(prev) : d_first_child(parent); + hlist_for_each_entry_from(d, d_sib) { + if (simple_positive(d)) { + spin_lock_nested(&d->d_lock, DENTRY_D_LOCK_NESTED); + if (simple_positive(d)) + child = dget_dlock(d); + spin_unlock(&d->d_lock); + if (likely(child)) + break; + } + } + spin_unlock(&parent->d_lock); + dput(prev); + return child; +} +EXPORT_SYMBOL(find_next_child); + +static void __simple_recursive_removal(struct dentry *dentry, + void (*callback)(struct dentry *), + bool locked) +{ + struct dentry *this = dget(dentry); + while (true) { + struct dentry *victim = NULL, *child; + struct inode *inode = this->d_inode; + + inode_lock_nested(inode, I_MUTEX_CHILD); + if (d_is_dir(this)) + inode->i_flags |= S_DEAD; + while ((child = find_next_child(this, victim)) == NULL) { + // kill and ascend + // update metadata while it's still locked + inode_set_ctime_current(inode); + clear_nlink(inode); + inode_unlock(inode); + victim = this; + this = this->d_parent; + inode = this->d_inode; + if (!locked || victim != dentry) + inode_lock_nested(inode, I_MUTEX_CHILD); + if (simple_positive(victim)) { + d_invalidate(victim); // avoid lost mounts + if (callback) + callback(victim); + fsnotify_delete(inode, d_inode(victim), victim); + d_make_discardable(victim); + } + if (victim == dentry) { + inode_set_mtime_to_ts(inode, + inode_set_ctime_current(inode)); + if (d_is_dir(dentry)) + drop_nlink(inode); + if (!locked) + inode_unlock(inode); + dput(dentry); + return; + } + } + inode_unlock(inode); + this = child; + } +} + +void simple_recursive_removal(struct dentry *dentry, + void (*callback)(struct dentry *)) +{ + return __simple_recursive_removal(dentry, callback, false); +} +EXPORT_SYMBOL(simple_recursive_removal); + +void simple_remove_by_name(struct dentry *parent, const char *name, + void (*callback)(struct dentry *)) +{ + struct dentry *dentry; + + dentry = lookup_noperm_positive_unlocked(&QSTR(name), parent); + if (!IS_ERR(dentry)) { + simple_recursive_removal(dentry, callback); + dput(dentry); // paired with lookup_noperm_positive_unlocked() + } +} +EXPORT_SYMBOL(simple_remove_by_name); + +/* caller holds parent directory with I_MUTEX_PARENT */ +void locked_recursive_removal(struct dentry *dentry, + void (*callback)(struct dentry *)) +{ + return __simple_recursive_removal(dentry, callback, true); +} +EXPORT_SYMBOL(locked_recursive_removal); + +static const struct super_operations simple_super_operations = { + .statfs = simple_statfs, +}; + +static int pseudo_fs_fill_super(struct super_block *s, struct fs_context *fc) +{ + struct pseudo_fs_context *ctx = fc->fs_private; + struct inode *root; + + s->s_maxbytes = MAX_LFS_FILESIZE; + s->s_blocksize = PAGE_SIZE; + s->s_blocksize_bits = PAGE_SHIFT; + s->s_magic = ctx->magic; + s->s_op = ctx->ops ?: &simple_super_operations; + s->s_export_op = ctx->eops; + s->s_xattr = ctx->xattr; + s->s_time_gran = 1; + s->s_d_flags |= ctx->s_d_flags; + root = new_inode(s); + if (!root) + return -ENOMEM; + + /* + * since this is the first inode, make it number 1. New inodes created + * after this must take care not to collide with it (by passing + * max_reserved of 1 to iunique). + */ + root->i_ino = 1; + root->i_mode = S_IFDIR | S_IRUSR | S_IWUSR; + simple_inode_init_ts(root); + s->s_root = d_make_root(root); + if (!s->s_root) + return -ENOMEM; + set_default_d_op(s, ctx->dops); + return 0; +} + +static int pseudo_fs_get_tree(struct fs_context *fc) +{ + return get_tree_nodev(fc, pseudo_fs_fill_super); +} + +static void pseudo_fs_free(struct fs_context *fc) +{ + kfree(fc->fs_private); +} + +static const struct fs_context_operations pseudo_fs_context_ops = { + .free = pseudo_fs_free, + .get_tree = pseudo_fs_get_tree, +}; + +/* + * Common helper for pseudo-filesystems (sockfs, pipefs, bdev - stuff that + * will never be mountable) + */ +struct pseudo_fs_context *init_pseudo(struct fs_context *fc, + unsigned long magic) +{ + struct pseudo_fs_context *ctx; + + ctx = kzalloc_obj(struct pseudo_fs_context); + if (likely(ctx)) { + ctx->magic = magic; + fc->fs_private = ctx; + fc->ops = &pseudo_fs_context_ops; + fc->sb_flags |= SB_NOUSER; + fc->s_iflags |= SB_I_NOEXEC | SB_I_NODEV; + fc->global = true; + } + return ctx; +} +EXPORT_SYMBOL(init_pseudo); + +int simple_open(struct inode *inode, struct file *file) +{ + if (inode->i_private) + file->private_data = inode->i_private; + return 0; +} +EXPORT_SYMBOL(simple_open); + +int simple_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry) +{ + struct inode *inode = d_inode(old_dentry); + + inode_set_mtime_to_ts(dir, + inode_set_ctime_to_ts(dir, inode_set_ctime_current(inode))); + inc_nlink(inode); + ihold(inode); + d_make_persistent(dentry, inode); + return 0; +} +EXPORT_SYMBOL(simple_link); + +int simple_empty(struct dentry *dentry) +{ + struct dentry *child; + int ret = 0; + + spin_lock(&dentry->d_lock); + hlist_for_each_entry(child, &dentry->d_children, d_sib) { + spin_lock_nested(&child->d_lock, DENTRY_D_LOCK_NESTED); + if (simple_positive(child)) { + spin_unlock(&child->d_lock); + goto out; + } + spin_unlock(&child->d_lock); + } + ret = 1; +out: + spin_unlock(&dentry->d_lock); + return ret; +} +EXPORT_SYMBOL(simple_empty); + +void __simple_unlink(struct inode *dir, struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + + inode_set_mtime_to_ts(dir, + inode_set_ctime_to_ts(dir, inode_set_ctime_current(inode))); + drop_nlink(inode); +} +EXPORT_SYMBOL(__simple_unlink); + +void __simple_rmdir(struct inode *dir, struct dentry *dentry) +{ + drop_nlink(d_inode(dentry)); + __simple_unlink(dir, dentry); + drop_nlink(dir); +} +EXPORT_SYMBOL(__simple_rmdir); + +int simple_unlink(struct inode *dir, struct dentry *dentry) +{ + __simple_unlink(dir, dentry); + d_make_discardable(dentry); + return 0; +} +EXPORT_SYMBOL(simple_unlink); + +int simple_rmdir(struct inode *dir, struct dentry *dentry) +{ + if (!simple_empty(dentry)) + return -ENOTEMPTY; + + __simple_rmdir(dir, dentry); + d_make_discardable(dentry); + return 0; +} +EXPORT_SYMBOL(simple_rmdir); + +/** + * simple_rename_timestamp - update the various inode timestamps for rename + * @old_dir: old parent directory + * @old_dentry: dentry that is being renamed + * @new_dir: new parent directory + * @new_dentry: target for rename + * + * POSIX mandates that the old and new parent directories have their ctime and + * mtime updated, and that inodes of @old_dentry and @new_dentry (if any), have + * their ctime updated. + */ +void simple_rename_timestamp(struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry) +{ + struct inode *newino = d_inode(new_dentry); + + inode_set_mtime_to_ts(old_dir, inode_set_ctime_current(old_dir)); + if (new_dir != old_dir) + inode_set_mtime_to_ts(new_dir, + inode_set_ctime_current(new_dir)); + inode_set_ctime_current(d_inode(old_dentry)); + if (newino) + inode_set_ctime_current(newino); +} +EXPORT_SYMBOL_GPL(simple_rename_timestamp); + +int simple_rename_exchange(struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry) +{ + bool old_is_dir = d_is_dir(old_dentry); + bool new_is_dir = d_is_dir(new_dentry); + + if (old_dir != new_dir && old_is_dir != new_is_dir) { + if (old_is_dir) { + drop_nlink(old_dir); + inc_nlink(new_dir); + } else { + drop_nlink(new_dir); + inc_nlink(old_dir); + } + } + simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry); + return 0; +} +EXPORT_SYMBOL_GPL(simple_rename_exchange); + +int simple_rename(struct mnt_idmap *idmap, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) +{ + int they_are_dirs = d_is_dir(old_dentry); + + if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE)) + return -EINVAL; + + if (flags & RENAME_EXCHANGE) + return simple_rename_exchange(old_dir, old_dentry, new_dir, new_dentry); + + if (!simple_empty(new_dentry)) + return -ENOTEMPTY; + + if (d_really_is_positive(new_dentry)) { + simple_unlink(new_dir, new_dentry); + if (they_are_dirs) { + drop_nlink(d_inode(new_dentry)); + drop_nlink(old_dir); + } + } else if (they_are_dirs) { + drop_nlink(old_dir); + inc_nlink(new_dir); + } + + simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry); + return 0; +} +EXPORT_SYMBOL(simple_rename); + +/** + * simple_setattr - setattr for simple filesystem + * @idmap: idmap of the target mount + * @dentry: dentry + * @iattr: iattr structure + * + * Returns 0 on success, -error on failure. + * + * simple_setattr is a simple ->setattr implementation without a proper + * implementation of size changes. + * + * It can either be used for in-memory filesystems or special files + * on simple regular filesystems. Anything that needs to change on-disk + * or wire state on size changes needs its own setattr method. + */ +int simple_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *iattr) +{ + struct inode *inode = d_inode(dentry); + int error; + + error = setattr_prepare(idmap, dentry, iattr); + if (error) + return error; + + if (iattr->ia_valid & ATTR_SIZE) + truncate_setsize(inode, iattr->ia_size); + setattr_copy(idmap, inode, iattr); + mark_inode_dirty(inode); + return 0; +} +EXPORT_SYMBOL(simple_setattr); + +static int simple_read_folio(struct file *file, struct folio *folio) +{ + folio_zero_range(folio, 0, folio_size(folio)); + flush_dcache_folio(folio); + folio_mark_uptodate(folio); + folio_unlock(folio); + return 0; +} + +int simple_write_begin(const struct kiocb *iocb, struct address_space *mapping, + loff_t pos, unsigned len, + struct folio **foliop, void **fsdata) +{ + struct folio *folio; + + folio = __filemap_get_folio(mapping, pos / PAGE_SIZE, FGP_WRITEBEGIN, + mapping_gfp_mask(mapping)); + if (IS_ERR(folio)) + return PTR_ERR(folio); + + *foliop = folio; + + if (!folio_test_uptodate(folio) && (len != folio_size(folio))) { + size_t from = offset_in_folio(folio, pos); + + folio_zero_segments(folio, 0, from, + from + len, folio_size(folio)); + } + return 0; +} +EXPORT_SYMBOL(simple_write_begin); + +/** + * simple_write_end - .write_end helper for non-block-device FSes + * @iocb: kernel I/O control block + * @mapping: " + * @pos: " + * @len: " + * @copied: " + * @folio: " + * @fsdata: " + * + * simple_write_end does the minimum needed for updating a folio after + * writing is done. It has the same API signature as the .write_end of + * address_space_operations vector. So it can just be set onto .write_end for + * FSes that don't need any other processing. i_rwsem is assumed to be held + * exclusively. + * Block based filesystems should use generic_write_end(). + * NOTE: Even though i_size might get updated by this function, mark_inode_dirty + * is not called, so a filesystem that actually does store data in .write_inode + * should extend on what's done here with a call to mark_inode_dirty() in the + * case that i_size has changed. + * + * Use *ONLY* with simple_read_folio() + */ +static int simple_write_end(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct folio *folio, void *fsdata) +{ + struct inode *inode = folio->mapping->host; + loff_t last_pos = pos + copied; + + /* zero the stale part of the folio if we did a short copy */ + if (!folio_test_uptodate(folio)) { + if (copied < len) { + size_t from = offset_in_folio(folio, pos); + + folio_zero_range(folio, from + copied, len - copied); + } + folio_mark_uptodate(folio); + } + /* + * No need to use i_size_read() here, the i_size + * cannot change under us because we hold the i_rwsem. + */ + if (last_pos > inode->i_size) + i_size_write(inode, last_pos); + + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + + return copied; +} + +/* + * Provides ramfs-style behavior: data in the pagecache, but no writeback. + */ +const struct address_space_operations ram_aops = { + .read_folio = simple_read_folio, + .write_begin = simple_write_begin, + .write_end = simple_write_end, + .dirty_folio = noop_dirty_folio, +}; +EXPORT_SYMBOL(ram_aops); + +/* + * the inodes created here are not hashed. If you use iunique to generate + * unique inode values later for this filesystem, then you must take care + * to pass it an appropriate max_reserved value to avoid collisions. + */ +int simple_fill_super(struct super_block *s, unsigned long magic, + const struct tree_descr *files) +{ + struct inode *inode; + struct dentry *dentry; + int i; + + s->s_blocksize = PAGE_SIZE; + s->s_blocksize_bits = PAGE_SHIFT; + s->s_magic = magic; + s->s_op = &simple_super_operations; + s->s_time_gran = 1; + + inode = new_inode(s); + if (!inode) + return -ENOMEM; + /* + * because the root inode is 1, the files array must not contain an + * entry at index 1 + */ + inode->i_ino = 1; + inode->i_mode = S_IFDIR | 0755; + simple_inode_init_ts(inode); + inode->i_op = &simple_dir_inode_operations; + inode->i_fop = &simple_dir_operations; + set_nlink(inode, 2); + s->s_root = d_make_root(inode); + if (!s->s_root) + return -ENOMEM; + for (i = 0; !files->name || files->name[0]; i++, files++) { + if (!files->name) + continue; + + /* warn if it tries to conflict with the root inode */ + if (unlikely(i == 1)) + printk(KERN_WARNING "%s: %s passed in a files array" + "with an index of 1!\n", __func__, + s->s_type->name); + + dentry = d_alloc_name(s->s_root, files->name); + if (!dentry) + return -ENOMEM; + inode = new_inode(s); + if (!inode) { + dput(dentry); + return -ENOMEM; + } + inode->i_mode = S_IFREG | files->mode; + simple_inode_init_ts(inode); + inode->i_fop = files->ops; + inode->i_ino = i; + d_make_persistent(dentry, inode); + dput(dentry); + } + return 0; +} +EXPORT_SYMBOL(simple_fill_super); + +static DEFINE_SPINLOCK(pin_fs_lock); + +int simple_pin_fs(struct file_system_type *type, struct vfsmount **mount, int *count) +{ + struct vfsmount *mnt = NULL; + spin_lock(&pin_fs_lock); + if (unlikely(!*mount)) { + spin_unlock(&pin_fs_lock); + mnt = vfs_kern_mount(type, SB_KERNMOUNT, type->name, NULL); + if (IS_ERR(mnt)) + return PTR_ERR(mnt); + spin_lock(&pin_fs_lock); + if (!*mount) + *mount = mnt; + } + mntget(*mount); + ++*count; + spin_unlock(&pin_fs_lock); + mntput(mnt); + return 0; +} +EXPORT_SYMBOL(simple_pin_fs); + +void simple_release_fs(struct vfsmount **mount, int *count) +{ + struct vfsmount *mnt; + spin_lock(&pin_fs_lock); + mnt = *mount; + if (!--*count) + *mount = NULL; + spin_unlock(&pin_fs_lock); + mntput(mnt); +} +EXPORT_SYMBOL(simple_release_fs); + +/** + * simple_read_from_buffer - copy data from the buffer to user space + * @to: the user space buffer to read to + * @count: the maximum number of bytes to read + * @ppos: the current position in the buffer + * @from: the buffer to read from + * @available: the size of the buffer + * + * The simple_read_from_buffer() function reads up to @count bytes from the + * buffer @from at offset @ppos into the user space address starting at @to. + * + * On success, the number of bytes read is returned and the offset @ppos is + * advanced by this number, or negative value is returned on error. + **/ +ssize_t simple_read_from_buffer(void __user *to, size_t count, loff_t *ppos, + const void *from, size_t available) +{ + loff_t pos = *ppos; + size_t ret; + + if (pos < 0) + return -EINVAL; + if (pos >= available || !count) + return 0; + if (count > available - pos) + count = available - pos; + ret = copy_to_user(to, from + pos, count); + if (ret == count) + return -EFAULT; + count -= ret; + *ppos = pos + count; + return count; +} +EXPORT_SYMBOL(simple_read_from_buffer); + +/** + * simple_write_to_buffer - copy data from user space to the buffer + * @to: the buffer to write to + * @available: the size of the buffer + * @ppos: the current position in the buffer + * @from: the user space buffer to read from + * @count: the maximum number of bytes to read + * + * The simple_write_to_buffer() function reads up to @count bytes from the user + * space address starting at @from into the buffer @to at offset @ppos. + * + * On success, the number of bytes written is returned and the offset @ppos is + * advanced by this number, or negative value is returned on error. + **/ +ssize_t simple_write_to_buffer(void *to, size_t available, loff_t *ppos, + const void __user *from, size_t count) +{ + loff_t pos = *ppos; + size_t res; + + if (pos < 0) + return -EINVAL; + if (pos >= available || !count) + return 0; + if (count > available - pos) + count = available - pos; + res = copy_from_user(to + pos, from, count); + if (res == count) + return -EFAULT; + count -= res; + *ppos = pos + count; + return count; +} +EXPORT_SYMBOL(simple_write_to_buffer); + +/** + * memory_read_from_buffer - copy data from the buffer + * @to: the kernel space buffer to read to + * @count: the maximum number of bytes to read + * @ppos: the current position in the buffer + * @from: the buffer to read from + * @available: the size of the buffer + * + * The memory_read_from_buffer() function reads up to @count bytes from the + * buffer @from at offset @ppos into the kernel space address starting at @to. + * + * On success, the number of bytes read is returned and the offset @ppos is + * advanced by this number, or negative value is returned on error. + **/ +ssize_t memory_read_from_buffer(void *to, size_t count, loff_t *ppos, + const void *from, size_t available) +{ + loff_t pos = *ppos; + + if (pos < 0) + return -EINVAL; + if (pos >= available) + return 0; + if (count > available - pos) + count = available - pos; + memcpy(to, from + pos, count); + *ppos = pos + count; + + return count; +} +EXPORT_SYMBOL(memory_read_from_buffer); + +/* + * Transaction based IO. + * The file expects a single write which triggers the transaction, and then + * possibly a read which collects the result - which is stored in a + * file-local buffer. + */ + +void simple_transaction_set(struct file *file, size_t n) +{ + struct simple_transaction_argresp *ar = file->private_data; + + BUG_ON(n > SIMPLE_TRANSACTION_LIMIT); + + /* + * The barrier ensures that ar->size will really remain zero until + * ar->data is ready for reading. + */ + smp_mb(); + ar->size = n; +} +EXPORT_SYMBOL(simple_transaction_set); + +char *simple_transaction_get(struct file *file, const char __user *buf, size_t size) +{ + struct simple_transaction_argresp *ar; + static DEFINE_SPINLOCK(simple_transaction_lock); + + if (size > SIMPLE_TRANSACTION_LIMIT - 1) + return ERR_PTR(-EFBIG); + + ar = kzalloc(PAGE_SIZE, GFP_KERNEL); + if (!ar) + return ERR_PTR(-ENOMEM); + + spin_lock(&simple_transaction_lock); + + /* only one write allowed per open */ + if (file->private_data) { + spin_unlock(&simple_transaction_lock); + kfree(ar); + return ERR_PTR(-EBUSY); + } + + file->private_data = ar; + + spin_unlock(&simple_transaction_lock); + + if (copy_from_user(ar->data, buf, size)) + return ERR_PTR(-EFAULT); + + return ar->data; +} +EXPORT_SYMBOL(simple_transaction_get); + +ssize_t simple_transaction_read(struct file *file, char __user *buf, size_t size, loff_t *pos) +{ + struct simple_transaction_argresp *ar = file->private_data; + + if (!ar) + return 0; + return simple_read_from_buffer(buf, size, pos, ar->data, ar->size); +} +EXPORT_SYMBOL(simple_transaction_read); + +int simple_transaction_release(struct inode *inode, struct file *file) +{ + kfree(file->private_data); + return 0; +} +EXPORT_SYMBOL(simple_transaction_release); + +/* Simple attribute files */ + +struct simple_attr { + int (*get)(void *, u64 *); + int (*set)(void *, u64); + char get_buf[24]; /* enough to store a u64 and "\n\0" */ + char set_buf[24]; + void *data; + const char *fmt; /* format for read operation */ + struct mutex mutex; /* protects access to these buffers */ +}; + +/* simple_attr_open is called by an actual attribute open file operation + * to set the attribute specific access operations. */ +int simple_attr_open(struct inode *inode, struct file *file, + int (*get)(void *, u64 *), int (*set)(void *, u64), + const char *fmt) +{ + struct simple_attr *attr; + + attr = kzalloc_obj(*attr); + if (!attr) + return -ENOMEM; + + attr->get = get; + attr->set = set; + attr->data = inode->i_private; + attr->fmt = fmt; + mutex_init(&attr->mutex); + + file->private_data = attr; + + return nonseekable_open(inode, file); +} +EXPORT_SYMBOL_GPL(simple_attr_open); + +int simple_attr_release(struct inode *inode, struct file *file) +{ + kfree(file->private_data); + return 0; +} +EXPORT_SYMBOL_GPL(simple_attr_release); /* GPL-only? This? Really? */ + +/* read from the buffer that is filled with the get function */ +ssize_t simple_attr_read(struct file *file, char __user *buf, + size_t len, loff_t *ppos) +{ + struct simple_attr *attr; + size_t size; + ssize_t ret; + + attr = file->private_data; + + if (!attr->get) + return -EACCES; + + ret = mutex_lock_interruptible(&attr->mutex); + if (ret) + return ret; + + if (*ppos && attr->get_buf[0]) { + /* continued read */ + size = strlen(attr->get_buf); + } else { + /* first read */ + u64 val; + ret = attr->get(attr->data, &val); + if (ret) + goto out; + + size = scnprintf(attr->get_buf, sizeof(attr->get_buf), + attr->fmt, (unsigned long long)val); + } + + ret = simple_read_from_buffer(buf, len, ppos, attr->get_buf, size); +out: + mutex_unlock(&attr->mutex); + return ret; +} +EXPORT_SYMBOL_GPL(simple_attr_read); + +/* interpret the buffer as a number to call the set function with */ +static ssize_t simple_attr_write_xsigned(struct file *file, const char __user *buf, + size_t len, loff_t *ppos, bool is_signed) +{ + struct simple_attr *attr; + unsigned long long val; + size_t size; + ssize_t ret; + + attr = file->private_data; + if (!attr->set) + return -EACCES; + + ret = mutex_lock_interruptible(&attr->mutex); + if (ret) + return ret; + + ret = -EFAULT; + size = min(sizeof(attr->set_buf) - 1, len); + if (copy_from_user(attr->set_buf, buf, size)) + goto out; + + attr->set_buf[size] = '\0'; + if (is_signed) + ret = kstrtoll(attr->set_buf, 0, &val); + else + ret = kstrtoull(attr->set_buf, 0, &val); + if (ret) + goto out; + ret = attr->set(attr->data, val); + if (ret == 0) + ret = len; /* on success, claim we got the whole input */ +out: + mutex_unlock(&attr->mutex); + return ret; +} + +ssize_t simple_attr_write(struct file *file, const char __user *buf, + size_t len, loff_t *ppos) +{ + return simple_attr_write_xsigned(file, buf, len, ppos, false); +} +EXPORT_SYMBOL_GPL(simple_attr_write); + +ssize_t simple_attr_write_signed(struct file *file, const char __user *buf, + size_t len, loff_t *ppos) +{ + return simple_attr_write_xsigned(file, buf, len, ppos, true); +} +EXPORT_SYMBOL_GPL(simple_attr_write_signed); + +/** + * generic_encode_ino32_fh - generic export_operations->encode_fh function + * @inode: the object to encode + * @fh: where to store the file handle fragment + * @max_len: maximum length to store there (in 4 byte units) + * @parent: parent directory inode, if wanted + * + * This generic encode_fh function assumes that the 32 inode number + * is suitable for locating an inode, and that the generation number + * can be used to check that it is still valid. It places them in the + * filehandle fragment where export_decode_fh expects to find them. + */ +int generic_encode_ino32_fh(struct inode *inode, __u32 *fh, int *max_len, + struct inode *parent) +{ + struct fid *fid = (void *)fh; + int len = *max_len; + int type = FILEID_INO32_GEN; + + if (parent && (len < 4)) { + *max_len = 4; + return FILEID_INVALID; + } else if (len < 2) { + *max_len = 2; + return FILEID_INVALID; + } + + len = 2; + fid->i32.ino = inode->i_ino; + fid->i32.gen = inode->i_generation; + if (parent) { + fid->i32.parent_ino = parent->i_ino; + fid->i32.parent_gen = parent->i_generation; + len = 4; + type = FILEID_INO32_GEN_PARENT; + } + *max_len = len; + return type; +} +EXPORT_SYMBOL_GPL(generic_encode_ino32_fh); + +/** + * generic_fh_to_dentry - generic helper for the fh_to_dentry export operation + * @sb: filesystem to do the file handle conversion on + * @fid: file handle to convert + * @fh_len: length of the file handle in bytes + * @fh_type: type of file handle + * @get_inode: filesystem callback to retrieve inode + * + * This function decodes @fid as long as it has one of the well-known + * Linux filehandle types and calls @get_inode on it to retrieve the + * inode for the object specified in the file handle. + */ +struct dentry *generic_fh_to_dentry(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type, struct inode *(*get_inode) + (struct super_block *sb, u64 ino, u32 gen)) +{ + struct inode *inode = NULL; + + if (fh_len < 2) + return NULL; + + switch (fh_type) { + case FILEID_INO32_GEN: + case FILEID_INO32_GEN_PARENT: + inode = get_inode(sb, fid->i32.ino, fid->i32.gen); + break; + } + + return d_obtain_alias(inode); +} +EXPORT_SYMBOL_GPL(generic_fh_to_dentry); + +/** + * generic_fh_to_parent - generic helper for the fh_to_parent export operation + * @sb: filesystem to do the file handle conversion on + * @fid: file handle to convert + * @fh_len: length of the file handle in bytes + * @fh_type: type of file handle + * @get_inode: filesystem callback to retrieve inode + * + * This function decodes @fid as long as it has one of the well-known + * Linux filehandle types and calls @get_inode on it to retrieve the + * inode for the _parent_ object specified in the file handle if it + * is specified in the file handle, or NULL otherwise. + */ +struct dentry *generic_fh_to_parent(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type, struct inode *(*get_inode) + (struct super_block *sb, u64 ino, u32 gen)) +{ + struct inode *inode = NULL; + + if (fh_len <= 2) + return NULL; + + switch (fh_type) { + case FILEID_INO32_GEN_PARENT: + inode = get_inode(sb, fid->i32.parent_ino, + (fh_len > 3 ? fid->i32.parent_gen : 0)); + break; + } + + return d_obtain_alias(inode); +} +EXPORT_SYMBOL_GPL(generic_fh_to_parent); + +/** + * simple_fsync_noflush - generic fsync implementation for simple filesystems + * + * @file: file to synchronize + * @start: start offset in bytes + * @end: end offset in bytes (inclusive) + * @datasync: only synchronize essential metadata if true + * + * This function is an fsync handler for simple filesystems. It writes out + * dirty data, inode (if dirty), but does not issue a cache flush. + */ +int simple_fsync_noflush(struct file *file, loff_t start, loff_t end, + int datasync) +{ + struct inode *inode = file->f_mapping->host; + int err; + int ret = 0; + + err = file_write_and_wait_range(file, start, end); + if (err) + return err; + + if (!(inode_state_read_once(inode) & + (I_DIRTY_ALL | I_SYNC | I_METADATA_WRITEBACK))) + goto out; + if (datasync && + !(inode_state_read_once(inode) & + (I_DIRTY_DATASYNC | I_SYNC | I_METADATA_WRITEBACK))) + goto out; + + ret = sync_inode_metadata(inode, 1); +out: + /* check and advance again to catch errors after syncing out buffers */ + err = file_check_and_advance_wb_err(file); + if (ret == 0) + ret = err; + return ret; +} +EXPORT_SYMBOL(simple_fsync_noflush); + +/** + * simple_fsync - fsync implementation for simple filesystems with flush + * @file: file to synchronize + * @start: start offset in bytes + * @end: end offset in bytes (inclusive) + * @datasync: only synchronize essential metadata if true + * + * This function is an fsync handler for simple filesystems. It writes out + * dirty data, inode (if dirty), and issues a cache flush. + */ +int simple_fsync(struct file *file, loff_t start, loff_t end, int datasync) +{ + struct inode *inode = file->f_mapping->host; + int err; + + err = simple_fsync_noflush(file, start, end, datasync); + if (err) + return err; + return blkdev_issue_flush(inode->i_sb->s_bdev); +} +EXPORT_SYMBOL(simple_fsync); + +/** + * generic_check_addressable - Check addressability of file system + * @blocksize_bits: log of file system block size + * @num_blocks: number of blocks in file system + * + * Determine whether a file system with @num_blocks blocks (and a + * block size of 2**@blocksize_bits) is addressable by the sector_t + * and page cache of the system. Return 0 if so and -EFBIG otherwise. + */ +int generic_check_addressable(unsigned blocksize_bits, u64 num_blocks) +{ + u64 last_fs_block = num_blocks - 1; + u64 last_fs_page, max_bytes; + + if (check_shl_overflow(num_blocks, blocksize_bits, &max_bytes)) + return -EFBIG; + + last_fs_page = (max_bytes >> PAGE_SHIFT) - 1; + + if (unlikely(num_blocks == 0)) + return 0; + + if (blocksize_bits < 9) + return -EINVAL; + + if ((last_fs_block > (sector_t)(~0ULL) >> (blocksize_bits - 9)) || + (last_fs_page > (pgoff_t)(~0ULL))) { + return -EFBIG; + } + return 0; +} +EXPORT_SYMBOL(generic_check_addressable); + +/* + * No-op implementation of ->fsync for in-memory filesystems. + */ +int noop_fsync(struct file *file, loff_t start, loff_t end, int datasync) +{ + return 0; +} +EXPORT_SYMBOL(noop_fsync); + +ssize_t noop_direct_IO(struct kiocb *iocb, struct iov_iter *iter) +{ + /* + * iomap based filesystems support direct I/O without need for + * this callback. However, it still needs to be set in + * inode->a_ops so that open/fcntl know that direct I/O is + * generally supported. + */ + return -EINVAL; +} +EXPORT_SYMBOL_GPL(noop_direct_IO); + +/* Because kfree isn't assignment-compatible with void(void*) ;-/ */ +void kfree_link(void *p) +{ + kfree(p); +} +EXPORT_SYMBOL(kfree_link); + +struct inode *alloc_anon_inode(struct super_block *s) +{ + static const struct address_space_operations anon_aops = { + .dirty_folio = noop_dirty_folio, + }; + struct inode *inode = new_inode_pseudo(s); + + if (!inode) + return ERR_PTR(-ENOMEM); + + inode->i_ino = get_next_ino(); + inode->i_mapping->a_ops = &anon_aops; + + /* + * Mark the inode dirty from the very beginning, + * that way it will never be moved to the dirty + * list because mark_inode_dirty() will think + * that it already _is_ on the dirty list. + */ + inode_state_assign_raw(inode, I_DIRTY); + /* + * Historically anonymous inodes don't have a type at all and + * userspace has come to rely on this. + */ + inode->i_mode = S_IRUSR | S_IWUSR; + inode->i_uid = current_fsuid(); + inode->i_gid = current_fsgid(); + inode->i_flags |= S_PRIVATE | S_ANON_INODE; + simple_inode_init_ts(inode); + return inode; +} +EXPORT_SYMBOL(alloc_anon_inode); + +/** + * simple_get_link - generic helper to get the target of "fast" symlinks + * @dentry: not used here + * @inode: the symlink inode + * @done: not used here + * + * Generic helper for filesystems to use for symlink inodes where a pointer to + * the symlink target is stored in ->i_link. NOTE: this isn't normally called, + * since as an optimization the path lookup code uses any non-NULL ->i_link + * directly, without calling ->get_link(). But ->get_link() still must be set, + * to mark the inode_operations as being for a symlink. + * + * Return: the symlink target + */ +const char *simple_get_link(struct dentry *dentry, struct inode *inode, + struct delayed_call *done) +{ + return inode->i_link; +} +EXPORT_SYMBOL(simple_get_link); + +const struct inode_operations simple_symlink_inode_operations = { + .get_link = simple_get_link, +}; +EXPORT_SYMBOL(simple_symlink_inode_operations); + +/* + * Operations for a permanently empty directory. + */ +static struct dentry *empty_dir_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) +{ + return ERR_PTR(-ENOENT); +} + +static int empty_dir_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, struct iattr *attr) +{ + return -EPERM; +} + +static ssize_t empty_dir_listxattr(struct dentry *dentry, char *list, size_t size) +{ + return -EOPNOTSUPP; +} + +static const struct inode_operations empty_dir_inode_operations = { + .lookup = empty_dir_lookup, + .setattr = empty_dir_setattr, + .listxattr = empty_dir_listxattr, +}; + +static loff_t empty_dir_llseek(struct file *file, loff_t offset, int whence) +{ + /* An empty directory has two entries . and .. at offsets 0 and 1 */ + return generic_file_llseek_size(file, offset, whence, 2, 2); +} + +static int empty_dir_readdir(struct file *file, struct dir_context *ctx) +{ + dir_emit_dots(file, ctx); + return 0; +} + +static const struct file_operations empty_dir_operations = { + .llseek = empty_dir_llseek, + .read = generic_read_dir, + .iterate_shared = empty_dir_readdir, + .fsync = noop_fsync, +}; + + +void make_empty_dir_inode(struct inode *inode) +{ + set_nlink(inode, 2); + inode->i_mode = S_IFDIR | S_IRUGO | S_IXUGO; + inode->i_uid = GLOBAL_ROOT_UID; + inode->i_gid = GLOBAL_ROOT_GID; + inode->i_rdev = 0; + inode->i_size = 0; + inode->i_blkbits = PAGE_SHIFT; + inode->i_blocks = 0; + + inode->i_op = &empty_dir_inode_operations; + inode->i_opflags &= ~IOP_XATTR; + inode->i_fop = &empty_dir_operations; +} + +bool is_empty_dir_inode(struct inode *inode) +{ + return (inode->i_fop == &empty_dir_operations) && + (inode->i_op == &empty_dir_inode_operations); +} + +#if IS_ENABLED(CONFIG_UNICODE) +/** + * generic_ci_d_compare - generic d_compare implementation for casefolding filesystems + * @dentry: dentry whose name we are checking against + * @len: len of name of dentry + * @str: str pointer to name of dentry + * @name: Name to compare against + * + * Return: 0 if names match, 1 if mismatch, or -ERRNO + */ +int generic_ci_d_compare(const struct dentry *dentry, unsigned int len, + const char *str, const struct qstr *name) +{ + const struct dentry *parent; + const struct inode *dir; + union shortname_store strbuf; + struct qstr qstr; + + /* + * Attempt a case-sensitive match first. It is cheaper and + * should cover most lookups, including all the sane + * applications that expect a case-sensitive filesystem. + * + * This comparison is safe under RCU because the caller + * guarantees the consistency between str and len. See + * __d_lookup_rcu_op_compare() for details. + */ + if (len == name->len && !memcmp(str, name->name, len)) + return 0; + + parent = READ_ONCE(dentry->d_parent); + dir = READ_ONCE(parent->d_inode); + if (!dir || !IS_CASEFOLDED(dir)) + return 1; + + qstr.len = len; + qstr.name = str; + /* + * If the dentry name is stored in-line, then it may be concurrently + * modified by a rename. If this happens, the VFS will eventually retry + * the lookup, so it doesn't matter what ->d_compare() returns. + * However, it's unsafe to call utf8_strncasecmp() with an unstable + * string. Therefore, we have to copy the name into a temporary buffer. + * As above, len is guaranteed to match str, so the shortname case + * is exactly when str points to ->d_shortname. + */ + if (qstr.name == dentry->d_shortname.string) { + strbuf = dentry->d_shortname; // NUL is guaranteed to be in there + qstr.name = strbuf.string; + /* prevent compiler from optimizing out the temporary buffer */ + barrier(); + } + + return utf8_strncasecmp(dentry->d_sb->s_encoding, name, &qstr); +} +EXPORT_SYMBOL(generic_ci_d_compare); + +/** + * generic_ci_d_hash - generic d_hash implementation for casefolding filesystems + * @dentry: dentry of the parent directory + * @str: qstr of name whose hash we should fill in + * + * Return: 0 if hash was successful or unchanged, and -EINVAL on error + */ +int generic_ci_d_hash(const struct dentry *dentry, struct qstr *str) +{ + const struct inode *dir = READ_ONCE(dentry->d_inode); + struct super_block *sb = dentry->d_sb; + const struct unicode_map *um = sb->s_encoding; + int ret; + + if (!dir || !IS_CASEFOLDED(dir)) + return 0; + + ret = utf8_casefold_hash(um, dentry, str); + if (ret < 0 && sb_has_strict_encoding(sb)) + return -EINVAL; + return 0; +} +EXPORT_SYMBOL(generic_ci_d_hash); + +static const struct dentry_operations generic_ci_dentry_ops = { + .d_hash = generic_ci_d_hash, + .d_compare = generic_ci_d_compare, +#ifdef CONFIG_FS_ENCRYPTION + .d_revalidate = fscrypt_d_revalidate, +#endif +}; + +/** + * generic_ci_match() - Match a name (case-insensitively) with a dirent. + * This is a filesystem helper for comparison with directory entries. + * generic_ci_d_compare should be used in VFS' ->d_compare instead. + * + * @parent: Inode of the parent of the dirent under comparison + * @name: name under lookup. + * @folded_name: Optional pre-folded name under lookup + * @de_name: Dirent name. + * @de_name_len: dirent name length. + * + * Test whether a case-insensitive directory entry matches the filename + * being searched. If @folded_name is provided, it is used instead of + * recalculating the casefold of @name. + * + * Return: > 0 if the directory entry matches, 0 if it doesn't match, or + * < 0 on error. + */ +int generic_ci_match(const struct inode *parent, + const struct qstr *name, + const struct qstr *folded_name, + const u8 *de_name, u32 de_name_len) +{ + const struct super_block *sb = parent->i_sb; + const struct unicode_map *um = sb->s_encoding; + struct fscrypt_str decrypted_name = FSTR_INIT(NULL, de_name_len); + struct qstr dirent = QSTR_INIT(de_name, de_name_len); + int res = 0; + + if (IS_ENCRYPTED(parent)) { + const struct fscrypt_str encrypted_name = + FSTR_INIT((u8 *) de_name, de_name_len); + + if (WARN_ON_ONCE(!fscrypt_has_encryption_key(parent))) + return -EINVAL; + + decrypted_name.name = kmalloc(de_name_len, GFP_KERNEL); + if (!decrypted_name.name) + return -ENOMEM; + res = fscrypt_fname_disk_to_usr(parent, 0, 0, &encrypted_name, + &decrypted_name); + if (res < 0) { + kfree(decrypted_name.name); + return res; + } + dirent.name = decrypted_name.name; + dirent.len = decrypted_name.len; + } + + /* + * Attempt a case-sensitive match first. It is cheaper and + * should cover most lookups, including all the sane + * applications that expect a case-sensitive filesystem. + */ + + if (dirent.len == name->len && + !memcmp(name->name, dirent.name, dirent.len)) + goto out; + + if (folded_name->name) + res = utf8_strncasecmp_folded(um, folded_name, &dirent); + else + res = utf8_strncasecmp(um, name, &dirent); + +out: + kfree(decrypted_name.name); + if (res < 0 && sb_has_strict_encoding(sb)) { + pr_err_ratelimited("Directory contains filename that is invalid UTF-8"); + return 0; + } + return !res; +} +EXPORT_SYMBOL(generic_ci_match); +#endif + +#ifdef CONFIG_FS_ENCRYPTION +static const struct dentry_operations generic_encrypted_dentry_ops = { + .d_revalidate = fscrypt_d_revalidate, +}; +#endif + +/** + * generic_set_sb_d_ops - helper for choosing the set of + * filesystem-wide dentry operations for the enabled features + * @sb: superblock to be configured + * + * Filesystems supporting casefolding and/or fscrypt can call this + * helper at mount-time to configure default dentry_operations to the + * best set of dentry operations required for the enabled features. + * The helper must be called after these have been configured, but + * before the root dentry is created. + */ +void generic_set_sb_d_ops(struct super_block *sb) +{ +#if IS_ENABLED(CONFIG_UNICODE) + if (sb->s_encoding) { + set_default_d_op(sb, &generic_ci_dentry_ops); + return; + } +#endif +#ifdef CONFIG_FS_ENCRYPTION + if (sb->s_cop) { + set_default_d_op(sb, &generic_encrypted_dentry_ops); + return; + } +#endif +} +EXPORT_SYMBOL(generic_set_sb_d_ops); + +/** + * inode_maybe_inc_iversion - increments i_version + * @inode: inode with the i_version that should be updated + * @force: increment the counter even if it's not necessary? + * + * Every time the inode is modified, the i_version field must be seen to have + * changed by any observer. + * + * If "force" is set or the QUERIED flag is set, then ensure that we increment + * the value, and clear the queried flag. + * + * In the common case where neither is set, then we can return "false" without + * updating i_version. + * + * If this function returns false, and no other metadata has changed, then we + * can avoid logging the metadata. + */ +bool inode_maybe_inc_iversion(struct inode *inode, bool force) +{ + u64 cur, new; + + /* + * The i_version field is not strictly ordered with any other inode + * information, but the legacy inode_inc_iversion code used a spinlock + * to serialize increments. + * + * We add a full memory barrier to ensure that any de facto ordering + * with other state is preserved (either implicitly coming from cmpxchg + * or explicitly from smp_mb if we don't know upfront if we will execute + * the former). + * + * These barriers pair with inode_query_iversion(). + */ + cur = inode_peek_iversion_raw(inode); + if (!force && !(cur & I_VERSION_QUERIED)) { + smp_mb(); + cur = inode_peek_iversion_raw(inode); + } + + do { + /* If flag is clear then we needn't do anything */ + if (!force && !(cur & I_VERSION_QUERIED)) + return false; + + /* Since lowest bit is flag, add 2 to avoid it */ + new = (cur & ~I_VERSION_QUERIED) + I_VERSION_INCREMENT; + } while (!atomic64_try_cmpxchg(&inode->i_version, &cur, new)); + return true; +} +EXPORT_SYMBOL(inode_maybe_inc_iversion); + +/** + * inode_query_iversion - read i_version for later use + * @inode: inode from which i_version should be read + * + * Read the inode i_version counter. This should be used by callers that wish + * to store the returned i_version for later comparison. This will guarantee + * that a later query of the i_version will result in a different value if + * anything has changed. + * + * In this implementation, we fetch the current value, set the QUERIED flag and + * then try to swap it into place with a cmpxchg, if it wasn't already set. If + * that fails, we try again with the newly fetched value from the cmpxchg. + */ +u64 inode_query_iversion(struct inode *inode) +{ + u64 cur, new; + bool fenced = false; + + /* + * Memory barriers (implicit in cmpxchg, explicit in smp_mb) pair with + * inode_maybe_inc_iversion(), see that routine for more details. + */ + cur = inode_peek_iversion_raw(inode); + do { + /* If flag is already set, then no need to swap */ + if (cur & I_VERSION_QUERIED) { + if (!fenced) + smp_mb(); + break; + } + + fenced = true; + new = cur | I_VERSION_QUERIED; + } while (!atomic64_try_cmpxchg(&inode->i_version, &cur, new)); + return cur >> I_VERSION_QUERIED_SHIFT; +} +EXPORT_SYMBOL(inode_query_iversion); + +ssize_t direct_write_fallback(struct kiocb *iocb, struct iov_iter *iter, + ssize_t direct_written, ssize_t buffered_written) +{ + struct address_space *mapping = iocb->ki_filp->f_mapping; + loff_t pos = iocb->ki_pos - buffered_written; + loff_t end = iocb->ki_pos - 1; + int err; + + /* + * If the buffered write fallback returned an error, we want to return + * the number of bytes which were written by direct I/O, or the error + * code if that was zero. + * + * Note that this differs from normal direct-io semantics, which will + * return -EFOO even if some bytes were written. + */ + if (unlikely(buffered_written < 0)) { + if (direct_written) + return direct_written; + return buffered_written; + } + + /* + * We need to ensure that the page cache pages are written to disk and + * invalidated to preserve the expected O_DIRECT semantics. + */ + err = filemap_write_and_wait_range(mapping, pos, end); + if (err < 0) { + /* + * We don't know how much we wrote, so just return the number of + * bytes which were direct-written + */ + iocb->ki_pos -= buffered_written; + if (direct_written) + return direct_written; + return err; + } + invalidate_mapping_pages(mapping, pos >> PAGE_SHIFT, end >> PAGE_SHIFT); + return direct_written + buffered_written; +} +EXPORT_SYMBOL_GPL(direct_write_fallback); + +/** + * simple_inode_init_ts - initialize the timestamps for a new inode + * @inode: inode to be initialized + * + * When a new inode is created, most filesystems set the timestamps to the + * current time. Add a helper to do this. + */ +struct timespec64 simple_inode_init_ts(struct inode *inode) +{ + struct timespec64 ts = inode_set_ctime_current(inode); + + inode_set_atime_to_ts(inode, ts); + inode_set_mtime_to_ts(inode, ts); + return ts; +} +EXPORT_SYMBOL(simple_inode_init_ts); + +struct dentry *stashed_dentry_get(struct dentry **stashed) +{ + struct dentry *dentry; + + guard(rcu)(); + dentry = rcu_dereference(*stashed); + if (!dentry) + return NULL; + if (IS_ERR(dentry)) + return dentry; + if (!lockref_get_not_dead(&dentry->d_lockref)) + return NULL; + return dentry; +} + +static struct dentry *prepare_anon_dentry(struct dentry **stashed, + struct super_block *sb, + void *data) +{ + struct dentry *dentry; + struct inode *inode; + const struct stashed_operations *sops = sb->s_fs_info; + int ret; + + inode = new_inode_pseudo(sb); + if (!inode) { + sops->put_data(data); + return ERR_PTR(-ENOMEM); + } + + inode->i_flags |= S_IMMUTABLE; + inode->i_mode = S_IFREG; + simple_inode_init_ts(inode); + + ret = sops->init_inode(inode, data); + if (ret < 0) { + iput(inode); + return ERR_PTR(ret); + } + + /* Notice when this is changed. */ + WARN_ON_ONCE(!S_ISREG(inode->i_mode)); + + dentry = d_alloc_anon(sb); + if (!dentry) { + iput(inode); + return ERR_PTR(-ENOMEM); + } + + /* Store address of location where dentry's supposed to be stashed. */ + dentry->d_fsdata = stashed; + + /* @data is now owned by the fs */ + d_instantiate(dentry, inode); + return dentry; +} + +struct dentry *stash_dentry(struct dentry **stashed, struct dentry *dentry) +{ + guard(rcu)(); + for (;;) { + struct dentry *old; + + /* Assume any old dentry was cleared out. */ + old = cmpxchg(stashed, NULL, dentry); + if (likely(!old)) + return dentry; + + /* Check if somebody else installed a reusable dentry. */ + if (lockref_get_not_dead(&old->d_lockref)) + return old; + + /* There's an old dead dentry there, try to take it over. */ + if (likely(try_cmpxchg(stashed, &old, dentry))) + return dentry; + } +} + +/** + * path_from_stashed - create path from stashed or new dentry + * @stashed: where to retrieve or stash dentry + * @mnt: mnt of the filesystems to use + * @data: data to store in inode->i_private + * @path: path to create + * + * The function tries to retrieve a stashed dentry from @stashed. If the dentry + * is still valid then it will be reused. If the dentry isn't able the function + * will allocate a new dentry and inode. It will then check again whether it + * can reuse an existing dentry in case one has been added in the meantime or + * update @stashed with the newly added dentry. + * + * Special-purpose helper for nsfs and pidfs. + * + * Return: On success zero and on failure a negative error is returned. + */ +int path_from_stashed(struct dentry **stashed, struct vfsmount *mnt, void *data, + struct path *path) +{ + struct dentry *dentry, *res; + const struct stashed_operations *sops = mnt->mnt_sb->s_fs_info; + + /* See if dentry can be reused. */ + res = stashed_dentry_get(stashed); + if (IS_ERR(res)) + return PTR_ERR(res); + if (res) { + sops->put_data(data); + goto make_path; + } + + /* Allocate a new dentry. */ + dentry = prepare_anon_dentry(stashed, mnt->mnt_sb, data); + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + + /* Added a new dentry. @data is now owned by the filesystem. */ + if (sops->stash_dentry) + res = sops->stash_dentry(stashed, dentry); + else + res = stash_dentry(stashed, dentry); + if (IS_ERR(res)) { + dput(dentry); + return PTR_ERR(res); + } + if (res != dentry) + dput(dentry); + +make_path: + path->dentry = res; + path->mnt = mntget(mnt); + VFS_WARN_ON_ONCE(path->dentry->d_fsdata != stashed); + VFS_WARN_ON_ONCE(d_inode(path->dentry)->i_private != data); + return 0; +} + +void stashed_dentry_prune(struct dentry *dentry) +{ + struct dentry **stashed = dentry->d_fsdata; + struct inode *inode = d_inode(dentry); + + if (WARN_ON_ONCE(!stashed)) + return; + + if (!inode) + return; + + /* + * Only replace our own @dentry as someone else might've + * already cleared out @dentry and stashed their own + * dentry in there. + */ + cmpxchg(stashed, dentry, NULL); +} + +/** + * simple_start_creating - prepare to create a given name + * @parent: directory in which to prepare to create the name + * @name: the name to be created + * + * Required lock is taken and a lookup in performed prior to creating an + * object in a directory. No permission checking is performed. + * + * Returns: a negative dentry on which vfs_create() or similar may + * be attempted, or an error. + */ +struct dentry *simple_start_creating(struct dentry *parent, const char *name) +{ + struct qstr qname = QSTR(name); + int err; + + err = lookup_noperm_common(&qname, parent); + if (err) + return ERR_PTR(err); + return start_dirop(parent, &qname, LOOKUP_CREATE | LOOKUP_EXCL); +} +EXPORT_SYMBOL(simple_start_creating); + +/* parent must have been held exclusive since simple_start_creating() */ +void simple_done_creating(struct dentry *child) +{ + end_creating(child); +} +EXPORT_SYMBOL(simple_done_creating); diff --git a/fs/lockd/Makefile b/fs/lockd/Makefile new file mode 100644 index 000000000..3f9569d3b --- /dev/null +++ b/fs/lockd/Makefile @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the linux lock manager stuff +# + +ccflags-y += -I$(src) # needed for trace events + +obj-$(CONFIG_LOCKD) += lockd.o + +lockd-y := clntlock.o clntproc.o clntxdr.o host.o svc.o svclock.o \ + svcshare.o svcproc.o svcsubs.o mon.o trace.o netlink.o \ + nlm3xdr_gen.o +lockd-$(CONFIG_LOCKD_V4) += clnt4xdr.o svc4proc.o nlm4xdr_gen.o +lockd-$(CONFIG_PROC_FS) += procfs.o + +# +# XDR code generation (requires Python and additional packages) +# +# The generated *xdr_gen.{h,c} files are checked into git. Normal kernel +# builds do not require the xdrgen tool or its Python dependencies. +# +# Developers modifying .x files in Documentation/sunrpc/xdr/ should run +# "make xdrgen" to regenerate the affected files. +# +.PHONY: xdrgen + +XDRGEN = ../../tools/net/sunrpc/xdrgen/xdrgen + +XDRGEN_DEFINITIONS = ../../include/linux/sunrpc/xdrgen/nlm4.h \ + ../../include/linux/sunrpc/xdrgen/nlm3.h +XDRGEN_DECLARATIONS = nlm4xdr_gen.h nlm3xdr_gen.h +XDRGEN_SOURCE = nlm4xdr_gen.c nlm3xdr_gen.c + +xdrgen: $(XDRGEN_DEFINITIONS) $(XDRGEN_DECLARATIONS) $(XDRGEN_SOURCE) + +../../include/linux/sunrpc/xdrgen/nlm4.h: ../../Documentation/sunrpc/xdr/nlm4.x + $(XDRGEN) definitions $< > $@ + +../../include/linux/sunrpc/xdrgen/nlm3.h: ../../Documentation/sunrpc/xdr/nlm3.x + $(XDRGEN) definitions $< > $@ + +nlm4xdr_gen.h: ../../Documentation/sunrpc/xdr/nlm4.x + $(XDRGEN) declarations $< > $@ + +nlm3xdr_gen.h: ../../Documentation/sunrpc/xdr/nlm3.x + $(XDRGEN) declarations $< > $@ + +nlm4xdr_gen.c: ../../Documentation/sunrpc/xdr/nlm4.x + $(XDRGEN) source --peer server $< > $@ + +nlm3xdr_gen.c: ../../Documentation/sunrpc/xdr/nlm3.x + $(XDRGEN) source --peer server $< > $@ diff --git a/fs/lockd/clnt4xdr.c b/fs/lockd/clnt4xdr.c new file mode 100644 index 000000000..96a4a1e6a --- /dev/null +++ b/fs/lockd/clnt4xdr.c @@ -0,0 +1,582 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/lockd/clnt4xdr.c + * + * XDR functions to encode/decode NLM version 4 RPC arguments and results. + * + * NLM client-side only. + * + * Copyright (C) 2010, Oracle. All rights reserved. + */ + +#include +#include +#include +#include + +#include "lockd.h" + +#include + +#define NLMDBG_FACILITY NLMDBG_XDR + +#if (NLMCLNT_OHSIZE > XDR_MAX_NETOBJ) +# error "NLM host name cannot be larger than XDR_MAX_NETOBJ!" +#endif + +#if (NLMCLNT_OHSIZE > NLM_MAXSTRLEN) +# error "NLM host name cannot be larger than NLM's maximum string length!" +#endif + +/* + * Declare the space requirements for NLM arguments and replies as + * number of 32bit-words + */ +#define NLM4_void_sz (0) +#define NLM4_cookie_sz (1+(NLM_MAXCOOKIELEN>>2)) +#define NLM4_caller_sz (1+(NLMCLNT_OHSIZE>>2)) +#define NLM4_owner_sz (1+(NLMCLNT_OHSIZE>>2)) +#define NLM4_fhandle_sz (1+(NFS3_FHSIZE>>2)) +#define NLM4_lock_sz (5+NLM4_caller_sz+NLM4_owner_sz+NLM4_fhandle_sz) +#define NLM4_holder_sz (6+NLM4_owner_sz) + +#define NLM4_testargs_sz (NLM4_cookie_sz+1+NLM4_lock_sz) +#define NLM4_lockargs_sz (NLM4_cookie_sz+4+NLM4_lock_sz) +#define NLM4_cancargs_sz (NLM4_cookie_sz+2+NLM4_lock_sz) +#define NLM4_unlockargs_sz (NLM4_cookie_sz+NLM4_lock_sz) + +#define NLM4_testres_sz (NLM4_cookie_sz+1+NLM4_holder_sz) +#define NLM4_res_sz (NLM4_cookie_sz+1) +#define NLM4_norep_sz (0) + + +static s64 loff_t_to_s64(loff_t offset) +{ + s64 res; + + if (offset >= NLM4_OFFSET_MAX) + res = NLM4_OFFSET_MAX; + else if (offset <= -NLM4_OFFSET_MAX) + res = -NLM4_OFFSET_MAX; + else + res = offset; + return res; +} + +static void nlm4_compute_offsets(const struct lockd_lock *lock, + u64 *l_offset, u64 *l_len) +{ + const struct file_lock *fl = &lock->fl; + + *l_offset = loff_t_to_s64(fl->fl_start); + if (fl->fl_end == OFFSET_MAX) + *l_len = 0; + else + *l_len = loff_t_to_s64(fl->fl_end - fl->fl_start + 1); +} + +/* + * Encode/decode NLMv4 basic data types + * + * Basic NLMv4 data types are defined in Appendix II, section 6.1.4 + * of RFC 1813: "NFS Version 3 Protocol Specification" and in Chapter + * 10 of X/Open's "Protocols for Interworking: XNFS, Version 3W". + * + * Not all basic data types have their own encoding and decoding + * functions. For run-time efficiency, some data types are encoded + * or decoded inline. + */ + +static void encode_bool(struct xdr_stream *xdr, const int value) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, 4); + *p = value ? xdr_one : xdr_zero; +} + +static void encode_int32(struct xdr_stream *xdr, const s32 value) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, 4); + *p = cpu_to_be32(value); +} + +/* + * typedef opaque netobj + */ +static void encode_netobj(struct xdr_stream *xdr, + const u8 *data, const unsigned int length) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, 4 + length); + xdr_encode_opaque(p, data, length); +} + +static int decode_netobj(struct xdr_stream *xdr, + struct xdr_netobj *obj) +{ + ssize_t ret; + + ret = xdr_stream_decode_opaque_inline(xdr, (void *)&obj->data, + XDR_MAX_NETOBJ); + if (unlikely(ret < 0)) + return -EIO; + obj->len = ret; + return 0; +} + +/* + * netobj cookie; + */ +static void encode_cookie(struct xdr_stream *xdr, + const struct lockd_cookie *cookie) +{ + encode_netobj(xdr, (u8 *)&cookie->data, cookie->len); +} + +static int decode_cookie(struct xdr_stream *xdr, + struct lockd_cookie *cookie) +{ + u32 length; + __be32 *p; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(p == NULL)) + goto out_overflow; + length = be32_to_cpup(p++); + /* apparently HPUX can return empty cookies */ + if (length == 0) + goto out_hpux; + if (length > NLM_MAXCOOKIELEN) + goto out_size; + p = xdr_inline_decode(xdr, length); + if (unlikely(p == NULL)) + goto out_overflow; + cookie->len = length; + memcpy(cookie->data, p, length); + return 0; +out_hpux: + cookie->len = 4; + memset(cookie->data, 0, 4); + return 0; +out_size: + dprintk("NFS: returned cookie was too long: %u\n", length); + return -EIO; +out_overflow: + return -EIO; +} + +/* + * netobj fh; + */ +static void encode_fh(struct xdr_stream *xdr, const struct nfs_fh *fh) +{ + encode_netobj(xdr, (u8 *)&fh->data, fh->size); +} + +/* + * enum nlm4_stats { + * NLM4_GRANTED = 0, + * NLM4_DENIED = 1, + * NLM4_DENIED_NOLOCKS = 2, + * NLM4_BLOCKED = 3, + * NLM4_DENIED_GRACE_PERIOD = 4, + * NLM4_DEADLCK = 5, + * NLM4_ROFS = 6, + * NLM4_STALE_FH = 7, + * NLM4_FBIG = 8, + * NLM4_FAILED = 9 + * }; + * + * struct nlm4_stat { + * nlm4_stats stat; + * }; + * + * NB: we don't swap bytes for the NLM status values. The upper + * layers deal directly with the status value in network byte + * order. + */ +static void encode_nlm4_stat(struct xdr_stream *xdr, + const __be32 stat) +{ + __be32 *p; + + BUG_ON(be32_to_cpu(stat) > NLM_FAILED); + p = xdr_reserve_space(xdr, 4); + *p = stat; +} + +static int decode_nlm4_stat(struct xdr_stream *xdr, __be32 *stat) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(p == NULL)) + goto out_overflow; + if (unlikely(ntohl(*p) > ntohl(nlm4_failed))) + goto out_bad_xdr; + *stat = *p; + return 0; +out_bad_xdr: + dprintk("%s: server returned invalid nlm4_stats value: %u\n", + __func__, be32_to_cpup(p)); + return -EIO; +out_overflow: + return -EIO; +} + +/* + * struct nlm4_holder { + * bool exclusive; + * int32 svid; + * netobj oh; + * uint64 l_offset; + * uint64 l_len; + * }; + */ +static void encode_nlm4_holder(struct xdr_stream *xdr, + const struct lockd_res *result) +{ + const struct lockd_lock *lock = &result->lock; + u64 l_offset, l_len; + __be32 *p; + + encode_bool(xdr, lock->fl.c.flc_type == F_RDLCK); + encode_int32(xdr, lock->svid); + encode_netobj(xdr, lock->oh.data, lock->oh.len); + + p = xdr_reserve_space(xdr, 4 + 4); + nlm4_compute_offsets(lock, &l_offset, &l_len); + p = xdr_encode_hyper(p, l_offset); + xdr_encode_hyper(p, l_len); +} + +static int decode_nlm4_holder(struct xdr_stream *xdr, struct lockd_res *result) +{ + struct lockd_lock *lock = &result->lock; + struct file_lock *fl = &lock->fl; + u64 l_offset, l_len; + u32 exclusive; + int error; + __be32 *p; + + memset(lock, 0, sizeof(*lock)); + locks_init_lock(fl); + + p = xdr_inline_decode(xdr, 4 + 4); + if (unlikely(p == NULL)) + goto out_overflow; + exclusive = be32_to_cpup(p++); + lock->svid = be32_to_cpup(p); + fl->c.flc_pid = (pid_t)lock->svid; + + error = decode_netobj(xdr, &lock->oh); + if (unlikely(error)) + goto out; + + p = xdr_inline_decode(xdr, 8 + 8); + if (unlikely(p == NULL)) + goto out_overflow; + + fl->c.flc_flags = FL_POSIX; + fl->c.flc_type = exclusive != 0 ? F_WRLCK : F_RDLCK; + p = xdr_decode_hyper(p, &l_offset); + xdr_decode_hyper(p, &l_len); + lockd_set_file_lock_range4(fl, l_offset, l_len); + error = 0; +out: + return error; +out_overflow: + return -EIO; +} + +/* + * string caller_name; + */ +static void encode_caller_name(struct xdr_stream *xdr, const char *name) +{ + /* NB: client-side does not set lock->len */ + u32 length = strlen(name); + __be32 *p; + + p = xdr_reserve_space(xdr, 4 + length); + xdr_encode_opaque(p, name, length); +} + +/* + * struct nlm4_lock { + * string caller_name; + * netobj fh; + * netobj oh; + * int32 svid; + * uint64 l_offset; + * uint64 l_len; + * }; + */ +static void encode_nlm4_lock(struct xdr_stream *xdr, + const struct lockd_lock *lock) +{ + u64 l_offset, l_len; + __be32 *p; + + encode_caller_name(xdr, lock->caller); + encode_fh(xdr, &lock->fh); + encode_netobj(xdr, lock->oh.data, lock->oh.len); + + p = xdr_reserve_space(xdr, 4 + 8 + 8); + *p++ = cpu_to_be32(lock->svid); + + nlm4_compute_offsets(lock, &l_offset, &l_len); + p = xdr_encode_hyper(p, l_offset); + xdr_encode_hyper(p, l_len); +} + + +/* + * NLMv4 XDR encode functions + * + * NLMv4 argument types are defined in Appendix II of RFC 1813: + * "NFS Version 3 Protocol Specification" and Chapter 10 of X/Open's + * "Protocols for Interworking: XNFS, Version 3W". + */ + +/* + * struct nlm4_testargs { + * netobj cookie; + * bool exclusive; + * struct nlm4_lock alock; + * }; + */ +static void nlm4_xdr_enc_testargs(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct lockd_args *args = data; + const struct lockd_lock *lock = &args->lock; + + encode_cookie(xdr, &args->cookie); + encode_bool(xdr, lock->fl.c.flc_type == F_WRLCK); + encode_nlm4_lock(xdr, lock); +} + +/* + * struct nlm4_lockargs { + * netobj cookie; + * bool block; + * bool exclusive; + * struct nlm4_lock alock; + * bool reclaim; + * int state; + * }; + */ +static void nlm4_xdr_enc_lockargs(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct lockd_args *args = data; + const struct lockd_lock *lock = &args->lock; + + encode_cookie(xdr, &args->cookie); + encode_bool(xdr, args->block); + encode_bool(xdr, lock->fl.c.flc_type == F_WRLCK); + encode_nlm4_lock(xdr, lock); + encode_bool(xdr, args->reclaim); + encode_int32(xdr, args->state); +} + +/* + * struct nlm4_cancargs { + * netobj cookie; + * bool block; + * bool exclusive; + * struct nlm4_lock alock; + * }; + */ +static void nlm4_xdr_enc_cancargs(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct lockd_args *args = data; + const struct lockd_lock *lock = &args->lock; + + encode_cookie(xdr, &args->cookie); + encode_bool(xdr, args->block); + encode_bool(xdr, lock->fl.c.flc_type == F_WRLCK); + encode_nlm4_lock(xdr, lock); +} + +/* + * struct nlm4_unlockargs { + * netobj cookie; + * struct nlm4_lock alock; + * }; + */ +static void nlm4_xdr_enc_unlockargs(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct lockd_args *args = data; + const struct lockd_lock *lock = &args->lock; + + encode_cookie(xdr, &args->cookie); + encode_nlm4_lock(xdr, lock); +} + +/* + * struct nlm4_res { + * netobj cookie; + * nlm4_stat stat; + * }; + */ +static void nlm4_xdr_enc_res(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct lockd_res *result = data; + + encode_cookie(xdr, &result->cookie); + encode_nlm4_stat(xdr, result->status); +} + +/* + * union nlm4_testrply switch (nlm4_stats stat) { + * case NLM4_DENIED: + * struct nlm4_holder holder; + * default: + * void; + * }; + * + * struct nlm4_testres { + * netobj cookie; + * nlm4_testrply test_stat; + * }; + */ +static void nlm4_xdr_enc_testres(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct lockd_res *result = data; + + encode_cookie(xdr, &result->cookie); + encode_nlm4_stat(xdr, result->status); + if (result->status == nlm_lck_denied) + encode_nlm4_holder(xdr, result); +} + + +/* + * NLMv4 XDR decode functions + * + * NLMv4 argument types are defined in Appendix II of RFC 1813: + * "NFS Version 3 Protocol Specification" and Chapter 10 of X/Open's + * "Protocols for Interworking: XNFS, Version 3W". + */ + +/* + * union nlm4_testrply switch (nlm4_stats stat) { + * case NLM4_DENIED: + * struct nlm4_holder holder; + * default: + * void; + * }; + * + * struct nlm4_testres { + * netobj cookie; + * nlm4_testrply test_stat; + * }; + */ +static int decode_nlm4_testrply(struct xdr_stream *xdr, + struct lockd_res *result) +{ + int error; + + error = decode_nlm4_stat(xdr, &result->status); + if (unlikely(error)) + goto out; + if (result->status == nlm_lck_denied) + error = decode_nlm4_holder(xdr, result); +out: + return error; +} + +static int nlm4_xdr_dec_testres(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *data) +{ + struct lockd_res *result = data; + int error; + + error = decode_cookie(xdr, &result->cookie); + if (unlikely(error)) + goto out; + error = decode_nlm4_testrply(xdr, result); +out: + return error; +} + +/* + * struct nlm4_res { + * netobj cookie; + * nlm4_stat stat; + * }; + */ +static int nlm4_xdr_dec_res(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *data) +{ + struct lockd_res *result = data; + int error; + + error = decode_cookie(xdr, &result->cookie); + if (unlikely(error)) + goto out; + error = decode_nlm4_stat(xdr, &result->status); +out: + return error; +} + + +/* + * For NLM, a void procedure really returns nothing + */ +#define nlm4_xdr_dec_norep NULL + +#define PROC(proc, argtype, restype) \ +[NLMPROC_##proc] = { \ + .p_proc = NLMPROC_##proc, \ + .p_encode = nlm4_xdr_enc_##argtype, \ + .p_decode = nlm4_xdr_dec_##restype, \ + .p_arglen = NLM4_##argtype##_sz, \ + .p_replen = NLM4_##restype##_sz, \ + .p_statidx = NLMPROC_##proc, \ + .p_name = #proc, \ + } + +static const struct rpc_procinfo nlm4_procedures[] = { + PROC(TEST, testargs, testres), + PROC(LOCK, lockargs, res), + PROC(CANCEL, cancargs, res), + PROC(UNLOCK, unlockargs, res), + PROC(GRANTED, testargs, res), + PROC(TEST_MSG, testargs, norep), + PROC(LOCK_MSG, lockargs, norep), + PROC(CANCEL_MSG, cancargs, norep), + PROC(UNLOCK_MSG, unlockargs, norep), + PROC(GRANTED_MSG, testargs, norep), + PROC(TEST_RES, testres, norep), + PROC(LOCK_RES, res, norep), + PROC(CANCEL_RES, res, norep), + PROC(UNLOCK_RES, res, norep), + PROC(GRANTED_RES, res, norep), +}; + +static unsigned int nlm_version4_counts[ARRAY_SIZE(nlm4_procedures)]; +const struct rpc_version nlm_version4 = { + .number = 4, + .nrprocs = ARRAY_SIZE(nlm4_procedures), + .procs = nlm4_procedures, + .counts = nlm_version4_counts, +}; diff --git a/fs/lockd/clntlock.c b/fs/lockd/clntlock.c new file mode 100644 index 000000000..f797cc99f --- /dev/null +++ b/fs/lockd/clntlock.c @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/lockd/clntlock.c + * + * Lock handling for the client side NLM implementation + * + * Copyright (C) 1996, Olaf Kirch + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "lockd.h" +#include "trace.h" + +#define NLMDBG_FACILITY NLMDBG_CLIENT + +/* + * Local function prototypes + */ +static int reclaimer(void *ptr); + +/* + * The following functions handle blocking and granting from the + * client perspective. + */ + +static LIST_HEAD(nlm_blocked); +static DEFINE_SPINLOCK(nlm_blocked_lock); + +/** + * nlmclnt_init - Set up per-NFS mount point lockd data structures + * @nlm_init: pointer to arguments structure + * + * Returns pointer to an appropriate nlm_host struct, + * or an ERR_PTR value. + */ +struct nlm_host *nlmclnt_init(const struct nlmclnt_initdata *nlm_init) +{ + struct nlm_host *host; + u32 nlm_version = (nlm_init->nfs_version == 2) ? 1 : 4; + int status; + + status = lockd_up(nlm_init->net, nlm_init->cred); + if (status < 0) + return ERR_PTR(status); + + host = nlmclnt_lookup_host(nlm_init->address, nlm_init->addrlen, + nlm_init->protocol, nlm_version, + nlm_init->hostname, nlm_init->noresvport, + nlm_init->net, nlm_init->cred); + if (host == NULL) + goto out_nohost; + if (host->h_rpcclnt == NULL && nlm_bind_host(host) == NULL) + goto out_nobind; + + host->h_nlmclnt_ops = nlm_init->nlmclnt_ops; + return host; +out_nobind: + nlmclnt_release_host(host); +out_nohost: + lockd_down(nlm_init->net); + return ERR_PTR(-ENOLCK); +} +EXPORT_SYMBOL_GPL(nlmclnt_init); + +/** + * nlmclnt_done - Release resources allocated by nlmclnt_init() + * @host: nlm_host structure reserved by nlmclnt_init() + * + */ +void nlmclnt_done(struct nlm_host *host) +{ + struct net *net = host->net; + + nlmclnt_release_host(host); + lockd_down(net); +} +EXPORT_SYMBOL_GPL(nlmclnt_done); + +void nlmclnt_prepare_block(struct nlm_wait *block, struct nlm_host *host, struct file_lock *fl) +{ + block->b_host = host; + block->b_lock = fl; + init_waitqueue_head(&block->b_wait); + block->b_status = nlm_lck_blocked; +} + +struct rpc_clnt *nlmclnt_rpc_clnt(struct nlm_host *host) +{ + return host->h_rpcclnt; +} +EXPORT_SYMBOL_GPL(nlmclnt_rpc_clnt); + +/* + * Queue up a lock for blocking so that the GRANTED request can see it + */ +void nlmclnt_queue_block(struct nlm_wait *block) +{ + spin_lock(&nlm_blocked_lock); + list_add(&block->b_list, &nlm_blocked); + spin_unlock(&nlm_blocked_lock); +} + +/* + * Dequeue the block and return its final status + */ +__be32 nlmclnt_dequeue_block(struct nlm_wait *block) +{ + __be32 status; + + spin_lock(&nlm_blocked_lock); + list_del(&block->b_list); + status = block->b_status; + spin_unlock(&nlm_blocked_lock); + return status; +} + +/* + * Block on a lock + */ +int nlmclnt_wait(struct nlm_wait *block, struct nlm_rqst *req, long timeout) +{ + long ret; + + /* A borken server might ask us to block even if we didn't + * request it. Just say no! + */ + if (block == NULL) + return -EAGAIN; + + /* Go to sleep waiting for GRANT callback. Some servers seem + * to lose callbacks, however, so we're going to poll from + * time to time just to make sure. + * + * For now, the retry frequency is pretty high; normally + * a 1 minute timeout would do. See the comment before + * nlmclnt_lock for an explanation. + */ + ret = wait_event_interruptible_timeout(block->b_wait, + block->b_status != nlm_lck_blocked, + timeout); + if (ret < 0) + return -ERESTARTSYS; + /* Reset the lock status after a server reboot so we resend */ + if (block->b_status == nlm_lck_denied_grace_period) + block->b_status = nlm_lck_blocked; + return 0; +} + +/* + * The server lockd has called us back to tell us the lock was granted + */ +__be32 nlmclnt_grant(const struct sockaddr *addr, const struct lockd_lock *lock) +{ + const struct file_lock *fl = &lock->fl; + const struct nfs_fh *fh = &lock->fh; + struct nlm_wait *block; + __be32 res = nlm_lck_denied; + + /* + * Look up blocked request based on arguments. + * Warning: must not use cookie to match it! + */ + spin_lock(&nlm_blocked_lock); + list_for_each_entry(block, &nlm_blocked, b_list) { + struct file_lock *fl_blocked = block->b_lock; + + if (fl_blocked->fl_start != fl->fl_start) + continue; + if (fl_blocked->fl_end != fl->fl_end) + continue; + /* + * Careful! The NLM server will return the 32-bit "pid" that + * we put on the wire: in this case the lockowner "pid". + */ + if (fl_blocked->fl_u.nfs_fl.owner->pid != lock->svid) + continue; + if (!rpc_cmp_addr(nlm_addr(block->b_host), addr)) + continue; + if (nfs_compare_fh(NFS_FH(file_inode(fl_blocked->c.flc_file)), fh) != 0) + continue; + /* Alright, we found a lock. Set the return status + * and wake up the caller + */ + block->b_status = nlm_granted; + wake_up(&block->b_wait); + res = nlm_granted; + } + spin_unlock(&nlm_blocked_lock); + trace_nlmclnt_grant(lock, addr, svc_addr_len(addr), res); + return res; +} + +/* + * The following procedures deal with the recovery of locks after a + * server crash. + */ + +/* + * Reclaim all locks on server host. We do this by spawning a separate + * reclaimer thread. + */ +void +nlmclnt_recovery(struct nlm_host *host) +{ + struct task_struct *task; + + if (!host->h_reclaiming++) { + nlm_get_host(host); + task = kthread_run(reclaimer, host, "%s-reclaim", host->h_name); + if (IS_ERR(task)) + printk(KERN_ERR "lockd: unable to spawn reclaimer " + "thread. Locks for %s won't be reclaimed! " + "(%ld)\n", host->h_name, PTR_ERR(task)); + } +} + +static int +reclaimer(void *ptr) +{ + struct nlm_host *host = (struct nlm_host *) ptr; + struct nlm_wait *block; + struct nlm_rqst *req; + struct file_lock *fl, *next; + u32 nsmstate; + struct net *net = host->net; + + req = kmalloc_obj(*req); + if (!req) + return 0; + + allow_signal(SIGKILL); + + down_write(&host->h_rwsem); + lockd_up(net, NULL); /* note: this cannot fail as lockd is already running */ + + dprintk("lockd: reclaiming locks for host %s\n", host->h_name); + +restart: + nsmstate = host->h_nsmstate; + + /* Force a portmap getport - the peer's lockd will + * most likely end up on a different port. + */ + host->h_nextrebind = jiffies; + nlm_rebind_host(host); + + /* First, reclaim all locks that have been granted. */ + list_splice_init(&host->h_granted, &host->h_reclaim); + list_for_each_entry_safe(fl, next, &host->h_reclaim, fl_u.nfs_fl.list) { + list_del_init(&fl->fl_u.nfs_fl.list); + + /* + * sending this thread a SIGKILL will result in any unreclaimed + * locks being removed from the h_granted list. This means that + * the kernel will not attempt to reclaim them again if a new + * reclaimer thread is spawned for this host. + */ + if (signalled()) + continue; + if (nlmclnt_reclaim(host, fl, req) != 0) + continue; + list_add_tail(&fl->fl_u.nfs_fl.list, &host->h_granted); + if (host->h_nsmstate != nsmstate) { + /* Argh! The server rebooted again! */ + goto restart; + } + } + + host->h_reclaiming = 0; + up_write(&host->h_rwsem); + dprintk("NLM: done reclaiming locks for host %s\n", host->h_name); + + /* Now, wake up all processes that sleep on a blocked lock */ + spin_lock(&nlm_blocked_lock); + list_for_each_entry(block, &nlm_blocked, b_list) { + if (block->b_host == host) { + block->b_status = nlm_lck_denied_grace_period; + wake_up(&block->b_wait); + } + } + spin_unlock(&nlm_blocked_lock); + + /* Release host handle after use */ + nlmclnt_release_host(host); + lockd_down(net); + kfree(req); + return 0; +} diff --git a/fs/lockd/clntproc.c b/fs/lockd/clntproc.c new file mode 100644 index 000000000..f06faf577 --- /dev/null +++ b/fs/lockd/clntproc.c @@ -0,0 +1,889 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/lockd/clntproc.c + * + * RPC procedures for the client side NLM implementation + * + * Copyright (C) 1996, Olaf Kirch + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "lockd.h" +#include "trace.h" + +#define NLMDBG_FACILITY NLMDBG_CLIENT +#define NLMCLNT_GRACE_WAIT (5*HZ) +#define NLMCLNT_POLL_TIMEOUT (30*HZ) +#define NLMCLNT_MAX_RETRIES 3 + +static int nlmclnt_test(struct nlm_rqst *, struct file_lock *); +static int nlmclnt_lock(struct nlm_rqst *, struct file_lock *); +static int nlmclnt_unlock(struct nlm_rqst *, struct file_lock *); +static int nlm_stat_to_errno(__be32 stat); +static void nlmclnt_locks_init_private(struct file_lock *fl, struct nlm_host *host); +static int nlmclnt_cancel(struct nlm_host *, int , struct file_lock *); + +static const struct rpc_call_ops nlmclnt_unlock_ops; +static const struct rpc_call_ops nlmclnt_cancel_ops; + +/* + * Cookie counter for NLM requests + */ +static atomic_t nlm_cookie = ATOMIC_INIT(0x1234); + +void nlmclnt_next_cookie(struct lockd_cookie *c) +{ + u32 cookie = atomic_inc_return(&nlm_cookie); + + memcpy(c->data, &cookie, 4); + c->len=4; +} + +static struct nlm_lockowner * +nlmclnt_get_lockowner(struct nlm_lockowner *lockowner) +{ + refcount_inc(&lockowner->count); + return lockowner; +} + +static void nlmclnt_put_lockowner(struct nlm_lockowner *lockowner) +{ + if (!refcount_dec_and_lock(&lockowner->count, &lockowner->host->h_lock)) + return; + list_del(&lockowner->list); + spin_unlock(&lockowner->host->h_lock); + nlmclnt_release_host(lockowner->host); + kfree(lockowner); +} + +static inline int nlm_pidbusy(struct nlm_host *host, uint32_t pid) +{ + struct nlm_lockowner *lockowner; + list_for_each_entry(lockowner, &host->h_lockowners, list) { + if (lockowner->pid == pid) + return -EBUSY; + } + return 0; +} + +static inline uint32_t __nlm_alloc_pid(struct nlm_host *host) +{ + uint32_t res; + do { + res = host->h_pidcount++; + } while (nlm_pidbusy(host, res) < 0); + return res; +} + +static struct nlm_lockowner *__nlmclnt_find_lockowner(struct nlm_host *host, fl_owner_t owner) +{ + struct nlm_lockowner *lockowner; + list_for_each_entry(lockowner, &host->h_lockowners, list) { + if (lockowner->owner != owner) + continue; + return nlmclnt_get_lockowner(lockowner); + } + return NULL; +} + +static struct nlm_lockowner *nlmclnt_find_lockowner(struct nlm_host *host, fl_owner_t owner) +{ + struct nlm_lockowner *res, *new = NULL; + + spin_lock(&host->h_lock); + res = __nlmclnt_find_lockowner(host, owner); + if (res == NULL) { + spin_unlock(&host->h_lock); + new = kmalloc_obj(*new); + spin_lock(&host->h_lock); + res = __nlmclnt_find_lockowner(host, owner); + if (res == NULL && new != NULL) { + res = new; + refcount_set(&new->count, 1); + new->owner = owner; + new->pid = __nlm_alloc_pid(host); + new->host = nlm_get_host(host); + list_add(&new->list, &host->h_lockowners); + new = NULL; + } + } + spin_unlock(&host->h_lock); + kfree(new); + return res; +} + +/* + * Initialize arguments for TEST/LOCK/UNLOCK/CANCEL calls + */ +static void nlmclnt_setlockargs(struct nlm_rqst *req, struct file_lock *fl) +{ + struct lockd_args *argp = &req->a_args; + struct lockd_lock *lock = &argp->lock; + char *nodename = req->a_host->h_rpcclnt->cl_nodename; + + nlmclnt_next_cookie(&argp->cookie); + memcpy(&lock->fh, NFS_FH(file_inode(fl->c.flc_file)), + sizeof(struct nfs_fh)); + lock->caller = nodename; + lock->oh.data = req->a_owner; + lock->oh.len = snprintf(req->a_owner, sizeof(req->a_owner), "%u@%s", + (unsigned int)fl->fl_u.nfs_fl.owner->pid, + nodename); + lock->svid = fl->fl_u.nfs_fl.owner->pid; + lock->fl.fl_start = fl->fl_start; + lock->fl.fl_end = fl->fl_end; + lock->fl.c.flc_type = fl->c.flc_type; +} + +static void nlmclnt_release_lockargs(struct nlm_rqst *req) +{ + WARN_ON_ONCE(req->a_args.lock.fl.fl_ops != NULL); +} + +/** + * nlmclnt_proc - Perform a single client-side lock request + * @host: address of a valid nlm_host context representing the NLM server + * @cmd: fcntl-style file lock operation to perform + * @fl: address of arguments for the lock operation + * @data: address of data to be sent to callback operations + * + */ +int nlmclnt_proc(struct nlm_host *host, int cmd, struct file_lock *fl, void *data) +{ + struct nlm_rqst *call; + int status; + const struct nlmclnt_operations *nlmclnt_ops = host->h_nlmclnt_ops; + + call = nlm_alloc_call(host); + if (call == NULL) + return -ENOMEM; + + if (nlmclnt_ops && nlmclnt_ops->nlmclnt_alloc_call) + nlmclnt_ops->nlmclnt_alloc_call(data); + + nlmclnt_locks_init_private(fl, host); + if (!fl->fl_u.nfs_fl.owner) { + /* lockowner allocation has failed */ + nlmclnt_release_call(call); + return -ENOMEM; + } + /* Set up the argument struct */ + nlmclnt_setlockargs(call, fl); + call->a_callback_data = data; + + if (IS_SETLK(cmd) || IS_SETLKW(cmd)) { + if (fl->c.flc_type != F_UNLCK) { + call->a_args.block = IS_SETLKW(cmd) ? 1 : 0; + status = nlmclnt_lock(call, fl); + } else + status = nlmclnt_unlock(call, fl); + } else if (IS_GETLK(cmd)) + status = nlmclnt_test(call, fl); + else + status = -EINVAL; + fl->fl_ops->fl_release_private(fl); + fl->fl_ops = NULL; + + dprintk("lockd: clnt proc returns %d\n", status); + return status; +} +EXPORT_SYMBOL_GPL(nlmclnt_proc); + +/* + * Allocate an NLM RPC call struct + */ +struct nlm_rqst *nlm_alloc_call(struct nlm_host *host) +{ + struct nlm_rqst *call; + + for(;;) { + call = kzalloc_obj(*call); + if (call != NULL) { + refcount_set(&call->a_count, 1); + locks_init_lock(&call->a_args.lock.fl); + locks_init_lock(&call->a_res.lock.fl); + call->a_host = nlm_get_host(host); + return call; + } + if (signalled()) + break; + printk("nlm_alloc_call: failed, waiting for memory\n"); + schedule_timeout_interruptible(5*HZ); + } + return NULL; +} + +void nlmclnt_release_call(struct nlm_rqst *call) +{ + const struct nlmclnt_operations *nlmclnt_ops = call->a_host->h_nlmclnt_ops; + + if (!refcount_dec_and_test(&call->a_count)) + return; + if (nlmclnt_ops && nlmclnt_ops->nlmclnt_release_call) + nlmclnt_ops->nlmclnt_release_call(call->a_callback_data); + nlmclnt_release_host(call->a_host); + nlmclnt_release_lockargs(call); + kfree(call); +} + +static void nlmclnt_rpc_release(void *data) +{ + nlmclnt_release_call(data); +} + +static int nlm_wait_on_grace(wait_queue_head_t *queue) +{ + DEFINE_WAIT(wait); + int status = -EINTR; + + prepare_to_wait(queue, &wait, TASK_INTERRUPTIBLE); + if (!signalled ()) { + schedule_timeout(NLMCLNT_GRACE_WAIT); + try_to_freeze(); + if (!signalled ()) + status = 0; + } + finish_wait(queue, &wait); + return status; +} + +/* + * Generic NLM call + */ +static int +nlmclnt_call(const struct cred *cred, struct nlm_rqst *req, u32 proc) +{ + struct nlm_host *host = req->a_host; + struct rpc_clnt *clnt; + struct lockd_args *argp = &req->a_args; + struct lockd_res *resp = &req->a_res; + struct rpc_message msg = { + .rpc_argp = argp, + .rpc_resp = resp, + .rpc_cred = cred, + }; + int status; + + dprintk("lockd: call procedure %d on %s\n", + (int)proc, host->h_name); + + do { + if (host->h_reclaiming && !argp->reclaim) + goto in_grace_period; + + /* If we have no RPC client yet, create one. */ + if ((clnt = nlm_bind_host(host)) == NULL) + return -ENOLCK; + msg.rpc_proc = &clnt->cl_procinfo[proc]; + + /* Perform the RPC call. If an error occurs, try again */ + if ((status = rpc_call_sync(clnt, &msg, 0)) < 0) { + dprintk("lockd: rpc_call returned error %d\n", -status); + switch (status) { + case -EPROTONOSUPPORT: + status = -EINVAL; + break; + case -ECONNREFUSED: + case -ETIMEDOUT: + case -ENOTCONN: + nlm_rebind_host(host); + status = -EAGAIN; + break; + case -ERESTARTSYS: + return signalled () ? -EINTR : status; + default: + break; + } + break; + } else + if (resp->status == nlm_lck_denied_grace_period) { + dprintk("lockd: server in grace period\n"); + if (argp->reclaim) { + printk(KERN_WARNING + "lockd: spurious grace period reject?!\n"); + return -ENOLCK; + } + } else { + if (!argp->reclaim) { + /* We appear to be out of the grace period */ + wake_up_all(&host->h_gracewait); + } + dprintk("lockd: server returns status %d\n", + ntohl(resp->status)); + return 0; /* Okay, call complete */ + } + +in_grace_period: + /* + * The server has rebooted and appears to be in the grace + * period during which locks are only allowed to be + * reclaimed. + * We can only back off and try again later. + */ + status = nlm_wait_on_grace(&host->h_gracewait); + } while (status == 0); + + return status; +} + +/* + * Generic NLM call, async version. + */ +static struct rpc_task *__nlm_async_call(struct nlm_rqst *req, u32 proc, struct rpc_message *msg, const struct rpc_call_ops *tk_ops) +{ + struct nlm_host *host = req->a_host; + struct rpc_clnt *clnt; + struct rpc_task_setup task_setup_data = { + .rpc_message = msg, + .callback_ops = tk_ops, + .callback_data = req, + .flags = RPC_TASK_ASYNC, + }; + + dprintk("lockd: call procedure %d on %s (async)\n", + (int)proc, host->h_name); + + /* If we have no RPC client yet, create one. */ + clnt = nlm_bind_host(host); + if (clnt == NULL) + goto out_err; + msg->rpc_proc = &clnt->cl_procinfo[proc]; + task_setup_data.rpc_client = clnt; + + /* bootstrap and kick off the async RPC call */ + return rpc_run_task(&task_setup_data); +out_err: + tk_ops->rpc_release(req); + return ERR_PTR(-ENOLCK); +} + +static int nlm_do_async_call(struct nlm_rqst *req, u32 proc, struct rpc_message *msg, const struct rpc_call_ops *tk_ops) +{ + struct rpc_task *task; + + task = __nlm_async_call(req, proc, msg, tk_ops); + if (IS_ERR(task)) + return PTR_ERR(task); + rpc_put_task(task); + return 0; +} + +/* + * NLM asynchronous call. + */ +int nlm_async_call(struct nlm_rqst *req, u32 proc, const struct rpc_call_ops *tk_ops) +{ + struct rpc_message msg = { + .rpc_argp = &req->a_args, + .rpc_resp = &req->a_res, + }; + return nlm_do_async_call(req, proc, &msg, tk_ops); +} + +int nlm_async_reply(struct nlm_rqst *req, u32 proc, const struct rpc_call_ops *tk_ops) +{ + struct rpc_message msg = { + .rpc_argp = &req->a_res, + }; + return nlm_do_async_call(req, proc, &msg, tk_ops); +} + +/* + * NLM client asynchronous call. + * + * Note that although the calls are asynchronous, and are therefore + * guaranteed to complete, we still always attempt to wait for + * completion in order to be able to correctly track the lock + * state. + */ +static int nlmclnt_async_call(const struct cred *cred, struct nlm_rqst *req, u32 proc, const struct rpc_call_ops *tk_ops) +{ + struct rpc_message msg = { + .rpc_argp = &req->a_args, + .rpc_resp = &req->a_res, + .rpc_cred = cred, + }; + struct rpc_task *task; + int err; + + task = __nlm_async_call(req, proc, &msg, tk_ops); + if (IS_ERR(task)) + return PTR_ERR(task); + err = rpc_wait_for_completion_task(task); + rpc_put_task(task); + return err; +} + +/* + * TEST for the presence of a conflicting lock + */ +static int +nlmclnt_test(struct nlm_rqst *req, struct file_lock *fl) +{ + int status; + + status = nlmclnt_call(nfs_file_cred(fl->c.flc_file), req, + NLMPROC_TEST); + if (status < 0) + goto out; + + switch (req->a_res.status) { + case nlm_granted: + fl->c.flc_type = F_UNLCK; + break; + case nlm_lck_denied: + /* + * Report the conflicting lock back to the application. + */ + fl->fl_start = req->a_res.lock.fl.fl_start; + fl->fl_end = req->a_res.lock.fl.fl_end; + fl->c.flc_type = req->a_res.lock.fl.c.flc_type; + fl->c.flc_pid = -req->a_res.lock.fl.c.flc_pid; + break; + default: + status = nlm_stat_to_errno(req->a_res.status); + } +out: + trace_nlmclnt_test(&req->a_args.lock, + (const struct sockaddr *)&req->a_host->h_addr, + req->a_host->h_addrlen, req->a_res.status); + nlmclnt_release_call(req); + return status; +} + +static void nlmclnt_locks_copy_lock(struct file_lock *new, struct file_lock *fl) +{ + spin_lock(&fl->fl_u.nfs_fl.owner->host->h_lock); + new->fl_u.nfs_fl.state = fl->fl_u.nfs_fl.state; + new->fl_u.nfs_fl.owner = nlmclnt_get_lockowner(fl->fl_u.nfs_fl.owner); + list_add_tail(&new->fl_u.nfs_fl.list, &fl->fl_u.nfs_fl.owner->host->h_granted); + spin_unlock(&fl->fl_u.nfs_fl.owner->host->h_lock); +} + +static void nlmclnt_locks_release_private(struct file_lock *fl) +{ + spin_lock(&fl->fl_u.nfs_fl.owner->host->h_lock); + list_del(&fl->fl_u.nfs_fl.list); + spin_unlock(&fl->fl_u.nfs_fl.owner->host->h_lock); + nlmclnt_put_lockowner(fl->fl_u.nfs_fl.owner); +} + +static const struct file_lock_operations nlmclnt_lock_ops = { + .fl_copy_lock = nlmclnt_locks_copy_lock, + .fl_release_private = nlmclnt_locks_release_private, +}; + +static void nlmclnt_locks_init_private(struct file_lock *fl, struct nlm_host *host) +{ + fl->fl_u.nfs_fl.state = 0; + fl->fl_u.nfs_fl.owner = nlmclnt_find_lockowner(host, + fl->c.flc_owner); + INIT_LIST_HEAD(&fl->fl_u.nfs_fl.list); + fl->fl_ops = &nlmclnt_lock_ops; +} + +static int do_vfs_lock(struct file_lock *fl) +{ + return locks_lock_file_wait(fl->c.flc_file, fl); +} + +/* + * LOCK: Try to create a lock + * + * Programmer Harassment Alert + * + * When given a blocking lock request in a sync RPC call, the HPUX lockd + * will faithfully return LCK_BLOCKED but never cares to notify us when + * the lock could be granted. This way, our local process could hang + * around forever waiting for the callback. + * + * Solution A: Implement busy-waiting + * Solution B: Use the async version of the call (NLM_LOCK_{MSG,RES}) + * + * For now I am implementing solution A, because I hate the idea of + * re-implementing lockd for a third time in two months. The async + * calls shouldn't be too hard to do, however. + * + * This is one of the lovely things about standards in the NFS area: + * they're so soft and squishy you can't really blame HP for doing this. + */ +static int +nlmclnt_lock(struct nlm_rqst *req, struct file_lock *fl) +{ + const struct cred *cred = nfs_file_cred(fl->c.flc_file); + struct nlm_host *host = req->a_host; + struct lockd_res *resp = &req->a_res; + struct nlm_wait block; + unsigned char flags = fl->c.flc_flags; + unsigned char type; + __be32 b_status; + int status = -ENOLCK; + + if (nsm_monitor(host) < 0) + goto out; + req->a_args.state = nsm_local_state; + + fl->c.flc_flags |= FL_ACCESS; + status = do_vfs_lock(fl); + fl->c.flc_flags = flags; + if (status < 0) + goto out; + + nlmclnt_prepare_block(&block, host, fl); +again: + /* + * Initialise resp->status to a valid non-zero value, + * since 0 == nlm_lck_granted + */ + resp->status = nlm_lck_blocked; + + /* + * A GRANTED callback can come at any time -- even before the reply + * to the LOCK request arrives, so we queue the wait before + * requesting the lock. + */ + nlmclnt_queue_block(&block); + for (;;) { + /* Reboot protection */ + fl->fl_u.nfs_fl.state = host->h_state; + status = nlmclnt_call(cred, req, NLMPROC_LOCK); + if (status < 0) + break; + /* Did a reclaimer thread notify us of a server reboot? */ + if (resp->status == nlm_lck_denied_grace_period) + continue; + if (resp->status != nlm_lck_blocked) + break; + /* Wait on an NLM blocking lock */ + status = nlmclnt_wait(&block, req, NLMCLNT_POLL_TIMEOUT); + if (status < 0) + break; + if (block.b_status != nlm_lck_blocked) + break; + } + b_status = nlmclnt_dequeue_block(&block); + if (resp->status == nlm_lck_blocked) + resp->status = b_status; + + /* if we were interrupted while blocking, then cancel the lock request + * and exit + */ + if (resp->status == nlm_lck_blocked) { + if (!req->a_args.block) + goto out_unlock; + if (nlmclnt_cancel(host, req->a_args.block, fl) == 0) + goto out; + } + + if (resp->status == nlm_granted) { + down_read(&host->h_rwsem); + /* Check whether or not the server has rebooted */ + if (fl->fl_u.nfs_fl.state != host->h_state) { + up_read(&host->h_rwsem); + goto again; + } + /* Ensure the resulting lock will get added to granted list */ + fl->c.flc_flags |= FL_SLEEP; + if (do_vfs_lock(fl) < 0) + printk(KERN_WARNING "%s: VFS is out of sync with lock manager!\n", __func__); + up_read(&host->h_rwsem); + fl->c.flc_flags = flags; + status = 0; + } + if (status < 0) + goto out_unlock; + /* + * EAGAIN doesn't make sense for sleeping locks, and in some + * cases NLM_LCK_DENIED is returned for a permanent error. So + * turn it into an ENOLCK. + */ + if (resp->status == nlm_lck_denied && (flags & FL_SLEEP)) + status = -ENOLCK; + else + status = nlm_stat_to_errno(resp->status); +out: + trace_nlmclnt_lock(&req->a_args.lock, + (const struct sockaddr *)&req->a_host->h_addr, + req->a_host->h_addrlen, req->a_res.status); + nlmclnt_release_call(req); + return status; +out_unlock: + /* Fatal error: ensure that we remove the lock altogether */ + trace_nlmclnt_lock(&req->a_args.lock, + (const struct sockaddr *)&req->a_host->h_addr, + req->a_host->h_addrlen, req->a_res.status); + dprintk("lockd: lock attempt ended in fatal error.\n" + " Attempting to unlock.\n"); + type = fl->c.flc_type; + fl->c.flc_type = F_UNLCK; + down_read(&host->h_rwsem); + do_vfs_lock(fl); + up_read(&host->h_rwsem); + fl->c.flc_type = type; + fl->c.flc_flags = flags; + nlmclnt_async_call(cred, req, NLMPROC_UNLOCK, &nlmclnt_unlock_ops); + return status; +} + +/* + * RECLAIM: Try to reclaim a lock + */ +int +nlmclnt_reclaim(struct nlm_host *host, struct file_lock *fl, + struct nlm_rqst *req) +{ + int status; + + memset(req, 0, sizeof(*req)); + locks_init_lock(&req->a_args.lock.fl); + locks_init_lock(&req->a_res.lock.fl); + req->a_host = host; + + /* Set up the argument struct */ + nlmclnt_setlockargs(req, fl); + req->a_args.reclaim = 1; + + status = nlmclnt_call(nfs_file_cred(fl->c.flc_file), req, + NLMPROC_LOCK); + if (status >= 0 && req->a_res.status == nlm_granted) + return 0; + + printk(KERN_WARNING "lockd: failed to reclaim lock for pid %d " + "(errno %d, status %d)\n", + fl->c.flc_pid, + status, ntohl(req->a_res.status)); + + /* + * FIXME: This is a serious failure. We can + * + * a. Ignore the problem + * b. Send the owning process some signal (Linux doesn't have + * SIGLOST, though...) + * c. Retry the operation + * + * Until someone comes up with a simple implementation + * for b or c, I'll choose option a. + */ + + return -ENOLCK; +} + +/* + * UNLOCK: remove an existing lock + */ +static int +nlmclnt_unlock(struct nlm_rqst *req, struct file_lock *fl) +{ + struct nlm_host *host = req->a_host; + struct lockd_res *resp = &req->a_res; + int status; + unsigned char flags = fl->c.flc_flags; + + /* + * Note: the server is supposed to either grant us the unlock + * request, or to deny it with NLM_LCK_DENIED_GRACE_PERIOD. In either + * case, we want to unlock. + */ + fl->c.flc_flags |= FL_EXISTS; + down_read(&host->h_rwsem); + status = do_vfs_lock(fl); + up_read(&host->h_rwsem); + fl->c.flc_flags = flags; + if (status == -ENOENT) { + status = 0; + goto out; + } + + refcount_inc(&req->a_count); + status = nlmclnt_async_call(nfs_file_cred(fl->c.flc_file), req, + NLMPROC_UNLOCK, &nlmclnt_unlock_ops); + if (status < 0) + goto out; + + if (resp->status == nlm_granted) + goto out; + + if (resp->status != nlm_lck_denied_nolocks) + printk("lockd: unexpected unlock status: %d\n", + ntohl(resp->status)); + /* What to do now? I'm out of my depth... */ + status = -ENOLCK; +out: + trace_nlmclnt_unlock(&req->a_args.lock, + (const struct sockaddr *)&req->a_host->h_addr, + req->a_host->h_addrlen, req->a_res.status); + nlmclnt_release_call(req); + return status; +} + +static void nlmclnt_unlock_prepare(struct rpc_task *task, void *data) +{ + struct nlm_rqst *req = data; + const struct nlmclnt_operations *nlmclnt_ops = req->a_host->h_nlmclnt_ops; + bool defer_call = false; + + if (nlmclnt_ops && nlmclnt_ops->nlmclnt_unlock_prepare) + defer_call = nlmclnt_ops->nlmclnt_unlock_prepare(task, req->a_callback_data); + + if (!defer_call) + rpc_call_start(task); +} + +static void nlmclnt_unlock_callback(struct rpc_task *task, void *data) +{ + struct nlm_rqst *req = data; + u32 status = ntohl(req->a_res.status); + + if (RPC_SIGNALLED(task)) + goto die; + + if (task->tk_status < 0) { + dprintk("lockd: unlock failed (err = %d)\n", -task->tk_status); + switch (task->tk_status) { + case -EACCES: + case -EIO: + goto die; + default: + goto retry_rebind; + } + } + if (status == NLM_LCK_DENIED_GRACE_PERIOD) { + rpc_delay(task, NLMCLNT_GRACE_WAIT); + goto retry_unlock; + } + if (status != NLM_LCK_GRANTED) + printk(KERN_WARNING "lockd: unexpected unlock status: %d\n", status); +die: + return; + retry_rebind: + nlm_rebind_host(req->a_host); + retry_unlock: + rpc_restart_call(task); +} + +static const struct rpc_call_ops nlmclnt_unlock_ops = { + .rpc_call_prepare = nlmclnt_unlock_prepare, + .rpc_call_done = nlmclnt_unlock_callback, + .rpc_release = nlmclnt_rpc_release, +}; + +/* + * Cancel a blocked lock request. + * We always use an async RPC call for this in order not to hang a + * process that has been Ctrl-C'ed. + */ +static int nlmclnt_cancel(struct nlm_host *host, int block, struct file_lock *fl) +{ + struct nlm_rqst *req; + int status; + + dprintk("lockd: blocking lock attempt was interrupted by a signal.\n" + " Attempting to cancel lock.\n"); + + req = nlm_alloc_call(host); + if (!req) + return -ENOMEM; + req->a_flags = RPC_TASK_ASYNC; + + nlmclnt_setlockargs(req, fl); + req->a_args.block = block; + + refcount_inc(&req->a_count); + status = nlmclnt_async_call(nfs_file_cred(fl->c.flc_file), req, + NLMPROC_CANCEL, &nlmclnt_cancel_ops); + if (status == 0 && req->a_res.status == nlm_lck_denied) + status = -ENOLCK; + nlmclnt_release_call(req); + return status; +} + +static void nlmclnt_cancel_callback(struct rpc_task *task, void *data) +{ + struct nlm_rqst *req = data; + u32 status = ntohl(req->a_res.status); + + if (RPC_SIGNALLED(task)) + goto die; + + if (task->tk_status < 0) { + dprintk("lockd: CANCEL call error %d, retrying.\n", + task->tk_status); + goto retry_cancel; + } + + switch (status) { + case NLM_LCK_GRANTED: + case NLM_LCK_DENIED_GRACE_PERIOD: + case NLM_LCK_DENIED: + /* Everything's good */ + break; + case NLM_LCK_DENIED_NOLOCKS: + dprintk("lockd: CANCEL failed (server has no locks)\n"); + goto retry_cancel; + default: + printk(KERN_NOTICE "lockd: weird return %d for CANCEL call\n", + status); + } + +die: + return; + +retry_cancel: + /* Don't ever retry more than 3 times */ + if (req->a_retries++ >= NLMCLNT_MAX_RETRIES) + goto die; + nlm_rebind_host(req->a_host); + rpc_restart_call(task); + rpc_delay(task, 30 * HZ); +} + +static const struct rpc_call_ops nlmclnt_cancel_ops = { + .rpc_call_done = nlmclnt_cancel_callback, + .rpc_release = nlmclnt_rpc_release, +}; + +/* + * Convert an NLM status code to a generic kernel errno + */ +static int +nlm_stat_to_errno(__be32 status) +{ + switch(ntohl(status)) { + case NLM_LCK_GRANTED: + return 0; + case NLM_LCK_DENIED: + return -EAGAIN; + case NLM_LCK_DENIED_NOLOCKS: + case NLM_LCK_DENIED_GRACE_PERIOD: + return -ENOLCK; + case NLM_LCK_BLOCKED: + printk(KERN_NOTICE "lockd: unexpected status NLM_BLOCKED\n"); + return -ENOLCK; +#ifdef CONFIG_LOCKD_V4 + case NLM_DEADLCK: + return -EDEADLK; + case NLM_ROFS: + return -EROFS; + case NLM_STALE_FH: + return -ESTALE; + case NLM_FBIG: + return -EOVERFLOW; + case NLM_FAILED: + return -ENOLCK; +#endif + } + printk(KERN_NOTICE "lockd: unexpected server status %d\n", + ntohl(status)); + return -ENOLCK; +} diff --git a/fs/lockd/clntxdr.c b/fs/lockd/clntxdr.c new file mode 100644 index 000000000..3789ecb0b --- /dev/null +++ b/fs/lockd/clntxdr.c @@ -0,0 +1,614 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/lockd/clntxdr.c + * + * XDR functions to encode/decode NLM version 1 and 3 RPC + * arguments and results. NLM version 2 is not specified + * by a standard, thus it is not implemented. + * + * NLM client-side only. + * + * Copyright (C) 2010, Oracle. All rights reserved. + */ + +#include +#include +#include +#include + +#include "lockd.h" + +#include + +#define NLMDBG_FACILITY NLMDBG_XDR + +#if (NLMCLNT_OHSIZE > XDR_MAX_NETOBJ) +# error "NLM host name cannot be larger than XDR_MAX_NETOBJ!" +#endif + +/* + * Declare the space requirements for NLM arguments and replies as + * number of 32bit-words + */ +#define NLM_cookie_sz (1+(NLM_MAXCOOKIELEN>>2)) +#define NLM_caller_sz (1+(NLMCLNT_OHSIZE>>2)) +#define NLM_owner_sz (1+(NLMCLNT_OHSIZE>>2)) +#define NLM_fhandle_sz (1+(NFS2_FHSIZE>>2)) +#define NLM_lock_sz (3+NLM_caller_sz+NLM_owner_sz+NLM_fhandle_sz) +#define NLM_holder_sz (4+NLM_owner_sz) + +#define NLM_testargs_sz (NLM_cookie_sz+1+NLM_lock_sz) +#define NLM_lockargs_sz (NLM_cookie_sz+4+NLM_lock_sz) +#define NLM_cancargs_sz (NLM_cookie_sz+2+NLM_lock_sz) +#define NLM_unlockargs_sz (NLM_cookie_sz+NLM_lock_sz) + +#define NLM_testres_sz (NLM_cookie_sz+1+NLM_holder_sz) +#define NLM_res_sz (NLM_cookie_sz+1) +#define NLM_norep_sz (0) + + +static s32 loff_t_to_s32(loff_t offset) +{ + s32 res; + + if (offset >= NLM_OFFSET_MAX) + res = NLM_OFFSET_MAX; + else if (offset <= -NLM_OFFSET_MAX) + res = -NLM_OFFSET_MAX; + else + res = offset; + return res; +} + +static void nlm_compute_offsets(const struct lockd_lock *lock, + u32 *l_offset, u32 *l_len) +{ + const struct file_lock *fl = &lock->fl; + + *l_offset = loff_t_to_s32(fl->fl_start); + if (fl->fl_end == OFFSET_MAX) + *l_len = 0; + else + *l_len = loff_t_to_s32(fl->fl_end - fl->fl_start + 1); +} + +/* + * Encode/decode NLMv3 basic data types + * + * Basic NLMv3 data types are not defined in an IETF standards + * document. X/Open has a description of these data types that + * is useful. See Chapter 10 of "Protocols for Interworking: + * XNFS, Version 3W". + * + * Not all basic data types have their own encoding and decoding + * functions. For run-time efficiency, some data types are encoded + * or decoded inline. + */ + +static void encode_bool(struct xdr_stream *xdr, const int value) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, 4); + *p = value ? xdr_one : xdr_zero; +} + +static void encode_int32(struct xdr_stream *xdr, const s32 value) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, 4); + *p = cpu_to_be32(value); +} + +/* + * typedef opaque netobj + */ +static void encode_netobj(struct xdr_stream *xdr, + const u8 *data, const unsigned int length) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, 4 + length); + xdr_encode_opaque(p, data, length); +} + +static int decode_netobj(struct xdr_stream *xdr, + struct xdr_netobj *obj) +{ + ssize_t ret; + + ret = xdr_stream_decode_opaque_inline(xdr, (void *)&obj->data, + XDR_MAX_NETOBJ); + if (unlikely(ret < 0)) + return -EIO; + obj->len = ret; + return 0; +} + +/* + * netobj cookie; + */ +static void encode_cookie(struct xdr_stream *xdr, + const struct lockd_cookie *cookie) +{ + encode_netobj(xdr, (u8 *)&cookie->data, cookie->len); +} + +static int decode_cookie(struct xdr_stream *xdr, + struct lockd_cookie *cookie) +{ + u32 length; + __be32 *p; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(p == NULL)) + goto out_overflow; + length = be32_to_cpup(p++); + /* apparently HPUX can return empty cookies */ + if (length == 0) + goto out_hpux; + if (length > NLM_MAXCOOKIELEN) + goto out_size; + p = xdr_inline_decode(xdr, length); + if (unlikely(p == NULL)) + goto out_overflow; + cookie->len = length; + memcpy(cookie->data, p, length); + return 0; +out_hpux: + cookie->len = 4; + memset(cookie->data, 0, 4); + return 0; +out_size: + dprintk("NFS: returned cookie was too long: %u\n", length); + return -EIO; +out_overflow: + return -EIO; +} + +/* + * netobj fh; + */ +static void encode_fh(struct xdr_stream *xdr, const struct nfs_fh *fh) +{ + encode_netobj(xdr, (u8 *)&fh->data, NFS2_FHSIZE); +} + +/* + * enum nlm_stats { + * LCK_GRANTED = 0, + * LCK_DENIED = 1, + * LCK_DENIED_NOLOCKS = 2, + * LCK_BLOCKED = 3, + * LCK_DENIED_GRACE_PERIOD = 4 + * }; + * + * + * struct nlm_stat { + * nlm_stats stat; + * }; + * + * NB: we don't swap bytes for the NLM status values. The upper + * layers deal directly with the status value in network byte + * order. + */ + +static void encode_nlm_stat(struct xdr_stream *xdr, + const __be32 stat) +{ + __be32 *p; + + WARN_ON_ONCE(be32_to_cpu(stat) > NLM_LCK_DENIED_GRACE_PERIOD); + p = xdr_reserve_space(xdr, 4); + *p = stat; +} + +static int decode_nlm_stat(struct xdr_stream *xdr, + __be32 *stat) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(p == NULL)) + goto out_overflow; + if (unlikely(ntohl(*p) > ntohl(nlm_lck_denied_grace_period))) + goto out_enum; + *stat = *p; + return 0; +out_enum: + dprintk("%s: server returned invalid nlm_stats value: %u\n", + __func__, be32_to_cpup(p)); + return -EIO; +out_overflow: + return -EIO; +} + +/* + * struct nlm_holder { + * bool exclusive; + * int uppid; + * netobj oh; + * unsigned l_offset; + * unsigned l_len; + * }; + */ +static void encode_nlm_holder(struct xdr_stream *xdr, + const struct lockd_res *result) +{ + const struct lockd_lock *lock = &result->lock; + u32 l_offset, l_len; + __be32 *p; + + encode_bool(xdr, lock->fl.c.flc_type == F_RDLCK); + encode_int32(xdr, lock->svid); + encode_netobj(xdr, lock->oh.data, lock->oh.len); + + p = xdr_reserve_space(xdr, 4 + 4); + nlm_compute_offsets(lock, &l_offset, &l_len); + *p++ = cpu_to_be32(l_offset); + *p = cpu_to_be32(l_len); +} + +static int decode_nlm_holder(struct xdr_stream *xdr, struct lockd_res *result) +{ + struct lockd_lock *lock = &result->lock; + struct file_lock *fl = &lock->fl; + u32 exclusive, l_offset, l_len; + int error; + __be32 *p; + s32 end; + + memset(lock, 0, sizeof(*lock)); + locks_init_lock(fl); + + p = xdr_inline_decode(xdr, 4 + 4); + if (unlikely(p == NULL)) + goto out_overflow; + exclusive = be32_to_cpup(p++); + lock->svid = be32_to_cpup(p); + fl->c.flc_pid = (pid_t)lock->svid; + + error = decode_netobj(xdr, &lock->oh); + if (unlikely(error)) + goto out; + + p = xdr_inline_decode(xdr, 4 + 4); + if (unlikely(p == NULL)) + goto out_overflow; + + fl->c.flc_flags = FL_POSIX; + fl->c.flc_type = exclusive != 0 ? F_WRLCK : F_RDLCK; + l_offset = be32_to_cpup(p++); + l_len = be32_to_cpup(p); + end = l_offset + l_len - 1; + + fl->fl_start = (loff_t)l_offset; + if (l_len == 0 || end < 0) + fl->fl_end = OFFSET_MAX; + else + fl->fl_end = (loff_t)end; + error = 0; +out: + return error; +out_overflow: + return -EIO; +} + +/* + * string caller_name; + */ +static void encode_caller_name(struct xdr_stream *xdr, const char *name) +{ + /* NB: client-side does not set lock->len */ + u32 length = strlen(name); + __be32 *p; + + p = xdr_reserve_space(xdr, 4 + length); + xdr_encode_opaque(p, name, length); +} + +/* + * struct nlm_lock { + * string caller_name; + * netobj fh; + * netobj oh; + * int uppid; + * unsigned l_offset; + * unsigned l_len; + * }; + */ +static void encode_nlm_lock(struct xdr_stream *xdr, + const struct lockd_lock *lock) +{ + u32 l_offset, l_len; + __be32 *p; + + encode_caller_name(xdr, lock->caller); + encode_fh(xdr, &lock->fh); + encode_netobj(xdr, lock->oh.data, lock->oh.len); + + p = xdr_reserve_space(xdr, 4 + 4 + 4); + *p++ = cpu_to_be32(lock->svid); + + nlm_compute_offsets(lock, &l_offset, &l_len); + *p++ = cpu_to_be32(l_offset); + *p = cpu_to_be32(l_len); +} + + +/* + * NLMv3 XDR encode functions + * + * NLMv3 argument types are defined in Chapter 10 of The Open Group's + * "Protocols for Interworking: XNFS, Version 3W". + */ + +/* + * struct nlm_testargs { + * netobj cookie; + * bool exclusive; + * struct nlm_lock alock; + * }; + */ +static void nlm_xdr_enc_testargs(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct lockd_args *args = data; + const struct lockd_lock *lock = &args->lock; + + encode_cookie(xdr, &args->cookie); + encode_bool(xdr, lock->fl.c.flc_type == F_WRLCK); + encode_nlm_lock(xdr, lock); +} + +/* + * struct nlm_lockargs { + * netobj cookie; + * bool block; + * bool exclusive; + * struct nlm_lock alock; + * bool reclaim; + * int state; + * }; + */ +static void nlm_xdr_enc_lockargs(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct lockd_args *args = data; + const struct lockd_lock *lock = &args->lock; + + encode_cookie(xdr, &args->cookie); + encode_bool(xdr, args->block); + encode_bool(xdr, lock->fl.c.flc_type == F_WRLCK); + encode_nlm_lock(xdr, lock); + encode_bool(xdr, args->reclaim); + encode_int32(xdr, args->state); +} + +/* + * struct nlm_cancargs { + * netobj cookie; + * bool block; + * bool exclusive; + * struct nlm_lock alock; + * }; + */ +static void nlm_xdr_enc_cancargs(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct lockd_args *args = data; + const struct lockd_lock *lock = &args->lock; + + encode_cookie(xdr, &args->cookie); + encode_bool(xdr, args->block); + encode_bool(xdr, lock->fl.c.flc_type == F_WRLCK); + encode_nlm_lock(xdr, lock); +} + +/* + * struct nlm_unlockargs { + * netobj cookie; + * struct nlm_lock alock; + * }; + */ +static void nlm_xdr_enc_unlockargs(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct lockd_args *args = data; + const struct lockd_lock *lock = &args->lock; + + encode_cookie(xdr, &args->cookie); + encode_nlm_lock(xdr, lock); +} + +/* + * struct nlm_res { + * netobj cookie; + * nlm_stat stat; + * }; + */ +static void nlm_xdr_enc_res(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct lockd_res *result = data; + + encode_cookie(xdr, &result->cookie); + encode_nlm_stat(xdr, result->status); +} + +/* + * union nlm_testrply switch (nlm_stats stat) { + * case LCK_DENIED: + * struct nlm_holder holder; + * default: + * void; + * }; + * + * struct nlm_testres { + * netobj cookie; + * nlm_testrply test_stat; + * }; + */ +static void encode_nlm_testrply(struct xdr_stream *xdr, + const struct lockd_res *result) +{ + if (result->status == nlm_lck_denied) + encode_nlm_holder(xdr, result); +} + +static void nlm_xdr_enc_testres(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct lockd_res *result = data; + + encode_cookie(xdr, &result->cookie); + encode_nlm_stat(xdr, result->status); + encode_nlm_testrply(xdr, result); +} + + +/* + * NLMv3 XDR decode functions + * + * NLMv3 result types are defined in Chapter 10 of The Open Group's + * "Protocols for Interworking: XNFS, Version 3W". + */ + +/* + * union nlm_testrply switch (nlm_stats stat) { + * case LCK_DENIED: + * struct nlm_holder holder; + * default: + * void; + * }; + * + * struct nlm_testres { + * netobj cookie; + * nlm_testrply test_stat; + * }; + */ +static int decode_nlm_testrply(struct xdr_stream *xdr, + struct lockd_res *result) +{ + int error; + + error = decode_nlm_stat(xdr, &result->status); + if (unlikely(error)) + goto out; + if (result->status == nlm_lck_denied) + error = decode_nlm_holder(xdr, result); +out: + return error; +} + +static int nlm_xdr_dec_testres(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *data) +{ + struct lockd_res *result = data; + int error; + + error = decode_cookie(xdr, &result->cookie); + if (unlikely(error)) + goto out; + error = decode_nlm_testrply(xdr, result); +out: + return error; +} + +/* + * struct nlm_res { + * netobj cookie; + * nlm_stat stat; + * }; + */ +static int nlm_xdr_dec_res(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *data) +{ + struct lockd_res *result = data; + int error; + + error = decode_cookie(xdr, &result->cookie); + if (unlikely(error)) + goto out; + error = decode_nlm_stat(xdr, &result->status); +out: + return error; +} + + +/* + * For NLM, a void procedure really returns nothing + */ +#define nlm_xdr_dec_norep NULL + +#define PROC(proc, argtype, restype) \ +[NLMPROC_##proc] = { \ + .p_proc = NLMPROC_##proc, \ + .p_encode = nlm_xdr_enc_##argtype, \ + .p_decode = nlm_xdr_dec_##restype, \ + .p_arglen = NLM_##argtype##_sz, \ + .p_replen = NLM_##restype##_sz, \ + .p_statidx = NLMPROC_##proc, \ + .p_name = #proc, \ + } + +static const struct rpc_procinfo nlm_procedures[] = { + PROC(TEST, testargs, testres), + PROC(LOCK, lockargs, res), + PROC(CANCEL, cancargs, res), + PROC(UNLOCK, unlockargs, res), + PROC(GRANTED, testargs, res), + PROC(TEST_MSG, testargs, norep), + PROC(LOCK_MSG, lockargs, norep), + PROC(CANCEL_MSG, cancargs, norep), + PROC(UNLOCK_MSG, unlockargs, norep), + PROC(GRANTED_MSG, testargs, norep), + PROC(TEST_RES, testres, norep), + PROC(LOCK_RES, res, norep), + PROC(CANCEL_RES, res, norep), + PROC(UNLOCK_RES, res, norep), + PROC(GRANTED_RES, res, norep), +}; + +static unsigned int nlm_version1_counts[ARRAY_SIZE(nlm_procedures)]; +static const struct rpc_version nlm_version1 = { + .number = 1, + .nrprocs = ARRAY_SIZE(nlm_procedures), + .procs = nlm_procedures, + .counts = nlm_version1_counts, +}; + +static unsigned int nlm_version3_counts[ARRAY_SIZE(nlm_procedures)]; +static const struct rpc_version nlm_version3 = { + .number = 3, + .nrprocs = ARRAY_SIZE(nlm_procedures), + .procs = nlm_procedures, + .counts = nlm_version3_counts, +}; + +static const struct rpc_version *nlm_versions[] = { + [1] = &nlm_version1, + [3] = &nlm_version3, +#ifdef CONFIG_LOCKD_V4 + [4] = &nlm_version4, +#endif +}; + +static struct rpc_stat nlm_rpc_stats; + +const struct rpc_program nlm_program = { + .name = "lockd", + .number = NLM_PROGRAM, + .nrvers = ARRAY_SIZE(nlm_versions), + .version = nlm_versions, + .stats = &nlm_rpc_stats, +}; diff --git a/fs/lockd/host.c b/fs/lockd/host.c new file mode 100644 index 000000000..d572cb275 --- /dev/null +++ b/fs/lockd/host.c @@ -0,0 +1,723 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/lockd/host.c + * + * Management for NLM peer hosts. The nlm_host struct is shared + * between client and server implementation. The only reason to + * do so is to reduce code bloat. + * + * Copyright (C) 1996, Olaf Kirch + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include "lockd.h" +#include "netns.h" + +#define NLMDBG_FACILITY NLMDBG_HOSTCACHE +#define NLM_HOST_NRHASH 32 +#define NLM_HOST_REBIND (60 * HZ) +#define NLM_HOST_EXPIRE (300 * HZ) +#define NLM_HOST_COLLECT (120 * HZ) + +static struct hlist_head nlm_server_hosts[NLM_HOST_NRHASH]; +static struct hlist_head nlm_client_hosts[NLM_HOST_NRHASH]; + +#define for_each_host(host, chain, table) \ + for ((chain) = (table); \ + (chain) < (table) + NLM_HOST_NRHASH; ++(chain)) \ + hlist_for_each_entry((host), (chain), h_hash) + +#define for_each_host_safe(host, next, chain, table) \ + for ((chain) = (table); \ + (chain) < (table) + NLM_HOST_NRHASH; ++(chain)) \ + hlist_for_each_entry_safe((host), (next), \ + (chain), h_hash) + +static unsigned long nrhosts; +static DEFINE_MUTEX(nlm_host_mutex); + +static void nlm_gc_hosts(struct net *net); + +struct nlm_lookup_host_info { + const int server; /* search for server|client */ + const struct sockaddr *sap; /* address to search for */ + const size_t salen; /* it's length */ + const unsigned short protocol; /* transport to search for*/ + const u32 version; /* NLM version to search for */ + const char *hostname; /* remote's hostname */ + const size_t hostname_len; /* it's length */ + const int noresvport; /* use non-priv port */ + struct net *net; /* network namespace to bind */ + const struct cred *cred; +}; + +/* + * Hash function must work well on big- and little-endian platforms + */ +static unsigned int __nlm_hash32(const __be32 n) +{ + unsigned int hash = (__force u32)n ^ ((__force u32)n >> 16); + return hash ^ (hash >> 8); +} + +static unsigned int __nlm_hash_addr4(const struct sockaddr *sap) +{ + const struct sockaddr_in *sin = (struct sockaddr_in *)sap; + return __nlm_hash32(sin->sin_addr.s_addr); +} + +static unsigned int __nlm_hash_addr6(const struct sockaddr *sap) +{ + const struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)sap; + const struct in6_addr addr = sin6->sin6_addr; + return __nlm_hash32(addr.s6_addr32[0]) ^ + __nlm_hash32(addr.s6_addr32[1]) ^ + __nlm_hash32(addr.s6_addr32[2]) ^ + __nlm_hash32(addr.s6_addr32[3]); +} + +static unsigned int nlm_hash_address(const struct sockaddr *sap) +{ + unsigned int hash; + + switch (sap->sa_family) { + case AF_INET: + hash = __nlm_hash_addr4(sap); + break; + case AF_INET6: + hash = __nlm_hash_addr6(sap); + break; + default: + hash = 0; + } + return hash & (NLM_HOST_NRHASH - 1); +} + +/* + * Allocate and initialize an nlm_host. Common to both client and server. + */ +static struct nlm_host *nlm_alloc_host(struct nlm_lookup_host_info *ni, + struct nsm_handle *nsm) +{ + struct nlm_host *host = NULL; + unsigned long now = jiffies; + + if (nsm != NULL) + refcount_inc(&nsm->sm_count); + else { + nsm = nsm_get_handle(ni->net, ni->sap, ni->salen, + ni->hostname, ni->hostname_len); + if (unlikely(nsm == NULL)) { + dprintk("lockd: %s failed; no nsm handle\n", + __func__); + goto out; + } + } + + host = kmalloc_obj(*host); + if (unlikely(host == NULL)) { + dprintk("lockd: %s failed; no memory\n", __func__); + nsm_release(nsm); + goto out; + } + + memcpy(nlm_addr(host), ni->sap, ni->salen); + host->h_addrlen = ni->salen; + rpc_set_port(nlm_addr(host), 0); + host->h_srcaddrlen = 0; + + host->h_rpcclnt = NULL; + host->h_name = nsm->sm_name; + host->h_version = ni->version; + host->h_proto = ni->protocol; + host->h_reclaiming = 0; + host->h_server = ni->server; + host->h_noresvport = ni->noresvport; + host->h_inuse = 0; + init_waitqueue_head(&host->h_gracewait); + init_rwsem(&host->h_rwsem); + host->h_state = 0; + host->h_nsmstate = 0; + host->h_pidcount = 0; + refcount_set(&host->h_count, 1); + mutex_init(&host->h_mutex); + host->h_nextrebind = now + NLM_HOST_REBIND; + host->h_expires = now + NLM_HOST_EXPIRE; + INIT_LIST_HEAD(&host->h_lockowners); + spin_lock_init(&host->h_lock); + INIT_LIST_HEAD(&host->h_granted); + INIT_LIST_HEAD(&host->h_reclaim); + host->h_nsmhandle = nsm; + host->h_addrbuf = nsm->sm_addrbuf; + host->net = ni->net; + host->h_cred = get_cred(ni->cred); + strscpy(host->nodename, utsname()->nodename, sizeof(host->nodename)); + +out: + return host; +} + +/* + * Destroy an nlm_host and free associated resources + * + * Caller must hold nlm_host_mutex. + */ +static void nlm_destroy_host_locked(struct nlm_host *host) +{ + struct rpc_clnt *clnt; + struct lockd_net *ln = net_generic(host->net, lockd_net_id); + + dprintk("lockd: destroy host %s\n", host->h_name); + + hlist_del_init(&host->h_hash); + + nsm_unmonitor(host); + nsm_release(host->h_nsmhandle); + + clnt = host->h_rpcclnt; + if (clnt != NULL) + rpc_shutdown_client(clnt); + put_cred(host->h_cred); + kfree(host); + + ln->nrhosts--; + nrhosts--; +} + +/** + * nlmclnt_lookup_host - Find an NLM host handle matching a remote server + * @sap: network address of server + * @salen: length of server address + * @protocol: transport protocol to use + * @version: NLM protocol version + * @hostname: '\0'-terminated hostname of server + * @noresvport: 1 if non-privileged port should be used + * @net: pointer to net namespace + * @cred: pointer to cred + * + * Returns an nlm_host structure that matches the passed-in + * [server address, transport protocol, NLM version, server hostname]. + * If one doesn't already exist in the host cache, a new handle is + * created and returned. + */ +struct nlm_host *nlmclnt_lookup_host(const struct sockaddr *sap, + const size_t salen, + const unsigned short protocol, + const u32 version, + const char *hostname, + int noresvport, + struct net *net, + const struct cred *cred) +{ + struct nlm_lookup_host_info ni = { + .server = 0, + .sap = sap, + .salen = salen, + .protocol = protocol, + .version = version, + .hostname = hostname, + .hostname_len = strlen(hostname), + .noresvport = noresvport, + .net = net, + .cred = cred, + }; + struct hlist_head *chain; + struct nlm_host *host; + struct nsm_handle *nsm = NULL; + struct lockd_net *ln = net_generic(net, lockd_net_id); + + dprintk("lockd: %s(host='%s', vers=%u, proto=%s)\n", __func__, + (hostname ? hostname : ""), version, + (protocol == IPPROTO_UDP ? "udp" : "tcp")); + + mutex_lock(&nlm_host_mutex); + + chain = &nlm_client_hosts[nlm_hash_address(sap)]; + hlist_for_each_entry(host, chain, h_hash) { + if (host->net != net) + continue; + if (!rpc_cmp_addr(nlm_addr(host), sap)) + continue; + + /* Same address. Share an NSM handle if we already have one */ + if (nsm == NULL) + nsm = host->h_nsmhandle; + + if (host->h_proto != protocol) + continue; + if (host->h_version != version) + continue; + + nlm_get_host(host); + dprintk("lockd: %s found host %s (%s)\n", __func__, + host->h_name, host->h_addrbuf); + goto out; + } + + host = nlm_alloc_host(&ni, nsm); + if (unlikely(host == NULL)) + goto out; + + hlist_add_head(&host->h_hash, chain); + ln->nrhosts++; + nrhosts++; + + dprintk("lockd: %s created host %s (%s)\n", __func__, + host->h_name, host->h_addrbuf); + +out: + mutex_unlock(&nlm_host_mutex); + return host; +} + +/** + * nlmclnt_release_host - release client nlm_host + * @host: nlm_host to release + * + */ +void nlmclnt_release_host(struct nlm_host *host) +{ + if (host == NULL) + return; + + dprintk("lockd: release client host %s\n", host->h_name); + + WARN_ON_ONCE(host->h_server); + + if (refcount_dec_and_mutex_lock(&host->h_count, &nlm_host_mutex)) { + WARN_ON_ONCE(!list_empty(&host->h_lockowners)); + WARN_ON_ONCE(!list_empty(&host->h_granted)); + WARN_ON_ONCE(!list_empty(&host->h_reclaim)); + + nlm_destroy_host_locked(host); + mutex_unlock(&nlm_host_mutex); + } +} + +/* Callback for rpc_cancel_tasks() - matches all tasks for cancellation */ +static bool nlmclnt_match_all(const struct rpc_task *task, const void *data) +{ + return true; +} + +/** + * nlmclnt_shutdown_rpc_clnt - safely shut down NLM client RPC operations + * @host: nlm_host to shut down + * + * Cancels outstanding RPC tasks and marks the client as shut down. + * Synchronizes with nlmclnt_release_host() via nlm_host_mutex to prevent + * races between shutdown and host destruction. Safe to call if h_rpcclnt + * is NULL or already shut down. + */ +void nlmclnt_shutdown_rpc_clnt(struct nlm_host *host) +{ + struct rpc_clnt *clnt; + + mutex_lock(&nlm_host_mutex); + clnt = host->h_rpcclnt; + if (clnt) { + clnt->cl_shutdown = 1; + rpc_cancel_tasks(clnt, -EIO, nlmclnt_match_all, NULL); + } + mutex_unlock(&nlm_host_mutex); +} +EXPORT_SYMBOL_GPL(nlmclnt_shutdown_rpc_clnt); + +/** + * nlmsvc_lookup_host - Find an NLM host handle matching a remote client + * @rqstp: incoming NLM request + * @hostname: name of client host + * @hostname_len: length of client hostname + * + * Returns an nlm_host structure that matches the [client address, + * transport protocol, NLM version, client hostname] of the passed-in + * NLM request. If one doesn't already exist in the host cache, a + * new handle is created and returned. + * + * Before possibly creating a new nlm_host, construct a sockaddr + * for a specific source address in case the local system has + * multiple network addresses. The family of the address in + * rq_daddr is guaranteed to be the same as the family of the + * address in rq_addr, so it's safe to use the same family for + * the source address. + */ +struct nlm_host *nlmsvc_lookup_host(const struct svc_rqst *rqstp, + const char *hostname, + const size_t hostname_len) +{ + struct hlist_head *chain; + struct nlm_host *host = NULL; + struct nsm_handle *nsm = NULL; + struct sockaddr *src_sap = svc_daddr(rqstp); + size_t src_len = rqstp->rq_daddrlen; + struct net *net = SVC_NET(rqstp); + struct nlm_lookup_host_info ni = { + .server = 1, + .sap = svc_addr(rqstp), + .salen = rqstp->rq_addrlen, + .protocol = rqstp->rq_prot, + .version = rqstp->rq_vers, + .hostname = hostname, + .hostname_len = hostname_len, + .net = net, + }; + struct lockd_net *ln = net_generic(net, lockd_net_id); + + dprintk("lockd: %s(host='%.*s', vers=%u, proto=%s)\n", __func__, + (int)hostname_len, hostname, rqstp->rq_vers, + (rqstp->rq_prot == IPPROTO_UDP ? "udp" : "tcp")); + + mutex_lock(&nlm_host_mutex); + + if (time_after_eq(jiffies, ln->next_gc)) + nlm_gc_hosts(net); + + chain = &nlm_server_hosts[nlm_hash_address(ni.sap)]; + hlist_for_each_entry(host, chain, h_hash) { + if (host->net != net) + continue; + if (!rpc_cmp_addr(nlm_addr(host), ni.sap)) + continue; + + /* Same address. Share an NSM handle if we already have one */ + if (nsm == NULL) + nsm = host->h_nsmhandle; + + if (host->h_proto != ni.protocol) + continue; + if (host->h_version != ni.version) + continue; + if (!rpc_cmp_addr(nlm_srcaddr(host), src_sap)) + continue; + + /* Move to head of hash chain. */ + hlist_del(&host->h_hash); + hlist_add_head(&host->h_hash, chain); + + nlm_get_host(host); + dprintk("lockd: %s found host %s (%s)\n", + __func__, host->h_name, host->h_addrbuf); + goto out; + } + + host = nlm_alloc_host(&ni, nsm); + if (unlikely(host == NULL)) + goto out; + + memcpy(nlm_srcaddr(host), src_sap, src_len); + host->h_srcaddrlen = src_len; + hlist_add_head(&host->h_hash, chain); + ln->nrhosts++; + nrhosts++; + + refcount_inc(&host->h_count); + + dprintk("lockd: %s created host %s (%s)\n", + __func__, host->h_name, host->h_addrbuf); + +out: + mutex_unlock(&nlm_host_mutex); + return host; +} + +/** + * nlmsvc_release_host - release server nlm_host + * @host: nlm_host to release + * + * Host is destroyed later in nlm_gc_host(). + */ +void nlmsvc_release_host(struct nlm_host *host) +{ + if (host == NULL) + return; + + dprintk("lockd: release server host %s\n", host->h_name); + + WARN_ON_ONCE(!host->h_server); + refcount_dec(&host->h_count); +} + +/* + * Create the NLM RPC client for an NLM peer + */ +struct rpc_clnt * +nlm_bind_host(struct nlm_host *host) +{ + struct rpc_clnt *clnt; + + dprintk("lockd: nlm_bind_host %s (%s)\n", + host->h_name, host->h_addrbuf); + + /* Lock host handle */ + mutex_lock(&host->h_mutex); + + /* If we've already created an RPC client, check whether + * RPC rebind is required + */ + if ((clnt = host->h_rpcclnt) != NULL) { + nlm_rebind_host(host); + } else { + unsigned long increment = nlm_timeout * HZ; + struct rpc_timeout timeparms = { + .to_initval = increment, + .to_increment = increment, + .to_maxval = increment * 6UL, + .to_retries = 5U, + }; + struct rpc_create_args args = { + .net = host->net, + .protocol = host->h_proto, + .address = nlm_addr(host), + .addrsize = host->h_addrlen, + .timeout = &timeparms, + .servername = host->h_name, + .program = &nlm_program, + .version = host->h_version, + .authflavor = RPC_AUTH_UNIX, + .flags = (RPC_CLNT_CREATE_NOPING | + RPC_CLNT_CREATE_AUTOBIND | + RPC_CLNT_CREATE_REUSEPORT), + .cred = host->h_cred, + }; + + /* + * lockd retries server side blocks automatically so we want + * those to be soft RPC calls. Client side calls need to be + * hard RPC tasks. + */ + if (!host->h_server) + args.flags |= RPC_CLNT_CREATE_HARDRTRY; + if (host->h_noresvport) + args.flags |= RPC_CLNT_CREATE_NONPRIVPORT; + if (host->h_srcaddrlen) + args.saddress = nlm_srcaddr(host); + + clnt = rpc_create(&args); + if (!IS_ERR(clnt)) + host->h_rpcclnt = clnt; + else { + printk("lockd: couldn't create RPC handle for %s\n", host->h_name); + clnt = NULL; + } + } + + mutex_unlock(&host->h_mutex); + return clnt; +} + +/** + * nlm_rebind_host - If needed, force a portmap lookup of the peer's lockd port + * @host: NLM host handle for peer + * + * This is not needed when using a connection-oriented protocol, such as TCP. + * The existing autobind mechanism is sufficient to force a rebind when + * required, e.g. on connection state transitions. + */ +void +nlm_rebind_host(struct nlm_host *host) +{ + if (host->h_proto != IPPROTO_UDP) + return; + + if (host->h_rpcclnt && time_after_eq(jiffies, host->h_nextrebind)) { + rpc_force_rebind(host->h_rpcclnt); + host->h_nextrebind = jiffies + NLM_HOST_REBIND; + } +} + +/* + * Increment NLM host count + */ +struct nlm_host * nlm_get_host(struct nlm_host *host) +{ + if (host) { + dprintk("lockd: get host %s\n", host->h_name); + refcount_inc(&host->h_count); + host->h_expires = jiffies + NLM_HOST_EXPIRE; + } + return host; +} + +static struct nlm_host *next_host_state(struct hlist_head *cache, + struct nsm_handle *nsm, + const struct lockd_reboot *info) +{ + struct nlm_host *host; + struct hlist_head *chain; + + mutex_lock(&nlm_host_mutex); + for_each_host(host, chain, cache) { + if (host->h_nsmhandle == nsm + && host->h_nsmstate != info->state) { + host->h_nsmstate = info->state; + host->h_state++; + + nlm_get_host(host); + mutex_unlock(&nlm_host_mutex); + return host; + } + } + + mutex_unlock(&nlm_host_mutex); + return NULL; +} + +/** + * nlm_host_rebooted - Release all resources held by rebooted host + * @net: network namespace + * @info: pointer to decoded results of NLM_SM_NOTIFY call + * + * We were notified that the specified host has rebooted. Release + * all resources held by that peer. + */ +void nlm_host_rebooted(const struct net *net, const struct lockd_reboot *info) +{ + struct nsm_handle *nsm; + struct nlm_host *host; + + nsm = nsm_reboot_lookup(net, info); + if (unlikely(nsm == NULL)) + return; + + /* Mark all hosts tied to this NSM state as having rebooted. + * We run the loop repeatedly, because we drop the host table + * lock for this. + * To avoid processing a host several times, we match the nsmstate. + */ + while ((host = next_host_state(nlm_server_hosts, nsm, info)) != NULL) { + nlmsvc_free_host_resources(host); + nlmsvc_release_host(host); + } + while ((host = next_host_state(nlm_client_hosts, nsm, info)) != NULL) { + nlmclnt_recovery(host); + nlmclnt_release_host(host); + } + + nsm_release(nsm); +} + +static void nlm_complain_hosts(struct net *net) +{ + struct hlist_head *chain; + struct nlm_host *host; + + if (net) { + struct lockd_net *ln = net_generic(net, lockd_net_id); + + if (ln->nrhosts == 0) + return; + pr_warn("lockd: couldn't shutdown host module for net %x!\n", + net->ns.inum); + dprintk("lockd: %lu hosts left in net %x:\n", ln->nrhosts, + net->ns.inum); + } else { + if (nrhosts == 0) + return; + printk(KERN_WARNING "lockd: couldn't shutdown host module!\n"); + dprintk("lockd: %lu hosts left:\n", nrhosts); + } + + for_each_host(host, chain, nlm_server_hosts) { + if (net && host->net != net) + continue; + dprintk(" %s (cnt %d use %d exp %ld net %x)\n", + host->h_name, refcount_read(&host->h_count), + host->h_inuse, host->h_expires, host->net->ns.inum); + } +} + +void +nlm_shutdown_hosts_net(struct net *net) +{ + struct hlist_head *chain; + struct nlm_host *host; + + mutex_lock(&nlm_host_mutex); + + /* First, make all hosts eligible for gc */ + dprintk("lockd: nuking all hosts in net %x...\n", + net ? net->ns.inum : 0); + for_each_host(host, chain, nlm_server_hosts) { + if (net && host->net != net) + continue; + host->h_expires = jiffies - 1; + if (host->h_rpcclnt) { + rpc_shutdown_client(host->h_rpcclnt); + host->h_rpcclnt = NULL; + } + nlmsvc_free_host_resources(host); + } + + /* Then, perform a garbage collection pass */ + nlm_gc_hosts(net); + nlm_complain_hosts(net); + mutex_unlock(&nlm_host_mutex); +} + +/* + * Shut down the hosts module. + * Note that this routine is called only at server shutdown time. + */ +void +nlm_shutdown_hosts(void) +{ + dprintk("lockd: shutting down host module\n"); + nlm_shutdown_hosts_net(NULL); +} + +/* + * Garbage collect any unused NLM hosts. + * This GC combines reference counting for async operations with + * mark & sweep for resources held by remote clients. + */ +static void +nlm_gc_hosts(struct net *net) +{ + struct hlist_head *chain; + struct hlist_node *next; + struct nlm_host *host; + + dprintk("lockd: host garbage collection for net %x\n", + net ? net->ns.inum : 0); + for_each_host(host, chain, nlm_server_hosts) { + if (net && host->net != net) + continue; + host->h_inuse = 0; + } + + /* Mark all hosts that hold locks, blocks or shares */ + nlmsvc_mark_resources(net); + + for_each_host_safe(host, next, chain, nlm_server_hosts) { + if (net && host->net != net) + continue; + if (host->h_inuse || time_before(jiffies, host->h_expires)) { + dprintk("nlm_gc_hosts skipping %s " + "(cnt %d use %d exp %ld net %x)\n", + host->h_name, refcount_read(&host->h_count), + host->h_inuse, host->h_expires, + host->net->ns.inum); + continue; + } + if (refcount_dec_if_one(&host->h_count)) + nlm_destroy_host_locked(host); + } + + if (net) { + struct lockd_net *ln = net_generic(net, lockd_net_id); + + ln->next_gc = jiffies + NLM_HOST_COLLECT; + } +} diff --git a/fs/lockd/lockd.h b/fs/lockd/lockd.h new file mode 100644 index 000000000..14cc952fe --- /dev/null +++ b/fs/lockd/lockd.h @@ -0,0 +1,483 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 1996 Olaf Kirch + */ + +#ifndef _LOCKD_LOCKD_H +#define _LOCKD_LOCKD_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "nlm.h" +#include +#include "xdr.h" +#include +#include + +/* + * Enable lockd debugging. + * Requires CONFIG_SUNRPC_DEBUG. + */ +#undef ifdebug +#if IS_ENABLED(CONFIG_SUNRPC_DEBUG) +# define ifdebug(flag) if (unlikely(nlm_debug & NLMDBG_##flag)) +#else +# define ifdebug(flag) if (0) +#endif + +#define NLMDBG_SVC 0x0001 +#define NLMDBG_CLIENT 0x0002 +#define NLMDBG_CLNTLOCK 0x0004 +#define NLMDBG_SVCLOCK 0x0008 +#define NLMDBG_MONITOR 0x0010 +#define NLMDBG_CLNTSUBS 0x0020 +#define NLMDBG_SVCSUBS 0x0040 +#define NLMDBG_HOSTCACHE 0x0080 +#define NLMDBG_XDR 0x0100 +#define NLMDBG_ALL 0x7fff + +/* + * Version string + */ +#define LOCKD_VERSION "0.5" + +/* + * Default timeout for RPC calls (seconds) + */ +#define LOCKD_DFLT_TIMEO 10 + +/* + * Number of leading bytes of nfs_fh.data that file_hash() + * digests when bucketing nlm_files[]. Sized for historical + * NFSv2 handles; nfs_fh.data must be initialized at least + * this far before lookup, regardless of fh.size. + */ +#define LOCKD_FH_HASH_SIZE 32 + +/* error codes new to NLMv4 */ +#define nlm4_deadlock cpu_to_be32(NLM_DEADLCK) +#define nlm4_rofs cpu_to_be32(NLM_ROFS) +#define nlm4_stale_fh cpu_to_be32(NLM_STALE_FH) +#define nlm4_fbig cpu_to_be32(NLM_FBIG) +#define nlm4_failed cpu_to_be32(NLM_FAILED) + +/* + * Internal-use status codes, not to be placed on the wire. + * Version handlers translate these to appropriate wire values. + */ +#define nlm__int__drop_reply cpu_to_be32(30000) +#define nlm__int__deadlock cpu_to_be32(30001) +#define nlm__int__stale_fh cpu_to_be32(30002) +#define nlm__int__failed cpu_to_be32(30003) + +/* + * Lockd host handle (used both by the client and server personality). + */ +struct nlm_host { + struct hlist_node h_hash; /* doubly linked list */ + struct sockaddr_storage h_addr; /* peer address */ + size_t h_addrlen; + struct sockaddr_storage h_srcaddr; /* our address (optional) */ + size_t h_srcaddrlen; + struct rpc_clnt *h_rpcclnt; /* RPC client to talk to peer */ + char *h_name; /* remote hostname */ + u32 h_version; /* interface version */ + unsigned short h_proto; /* transport proto */ + unsigned short h_reclaiming : 1, + h_server : 1, /* server side, not client side */ + h_noresvport : 1, + h_inuse : 1; + wait_queue_head_t h_gracewait; /* wait while reclaiming */ + struct rw_semaphore h_rwsem; /* Reboot recovery lock */ + u32 h_state; /* pseudo-state counter */ + u32 h_nsmstate; /* true remote NSM state */ + u32 h_pidcount; /* Pseudopids */ + refcount_t h_count; /* reference count */ + struct mutex h_mutex; /* mutex for pmap binding */ + unsigned long h_nextrebind; /* next portmap call */ + unsigned long h_expires; /* eligible for GC */ + struct list_head h_lockowners; /* Lockowners for the client */ + spinlock_t h_lock; + struct list_head h_granted; /* Locks in GRANTED state */ + struct list_head h_reclaim; /* Locks in RECLAIM state */ + struct nsm_handle *h_nsmhandle; /* NSM status handle */ + char *h_addrbuf; /* address eyecatcher */ + struct net *net; /* host net */ + const struct cred *h_cred; + char nodename[UNX_MAXNODENAME + 1]; + const struct nlmclnt_operations *h_nlmclnt_ops; /* Callback ops for NLM users */ +}; + +/* + * The largest string sm_addrbuf should hold is a full-size IPv6 address + * (no "::" anywhere) with a scope ID. The buffer size is computed to + * hold eight groups of colon-separated four-hex-digit numbers, a + * percent sign, a scope id (at most 32 bits, in decimal), and NUL. + */ +#define NSM_ADDRBUF ((8 * 4 + 7) + (1 + 10) + 1) + +struct nsm_handle { + struct list_head sm_link; + refcount_t sm_count; + char *sm_mon_name; + char *sm_name; + struct sockaddr_storage sm_addr; + size_t sm_addrlen; + unsigned int sm_monitored : 1, + sm_sticky : 1; /* don't unmonitor */ + struct nsm_private sm_priv; + char sm_addrbuf[NSM_ADDRBUF]; +}; + +/* + * Rigorous type checking on sockaddr type conversions + */ +static inline struct sockaddr *nlm_addr(const struct nlm_host *host) +{ + return (struct sockaddr *)&host->h_addr; +} + +static inline struct sockaddr *nlm_srcaddr(const struct nlm_host *host) +{ + return (struct sockaddr *)&host->h_srcaddr; +} + +/* + * Map an fl_owner_t into a unique 32-bit "pid" + */ +struct nlm_lockowner { + struct list_head list; + refcount_t count; + + struct nlm_host *host; + fl_owner_t owner; + uint32_t pid; +}; + +/* + * This is the representation of a blocked client lock. + */ +struct nlm_wait { + struct list_head b_list; /* linked list */ + wait_queue_head_t b_wait; /* where to wait on */ + struct nlm_host *b_host; + struct file_lock *b_lock; /* local file lock */ + __be32 b_status; /* grant callback status */ +}; + +/* + * Memory chunk for NLM client RPC request. + */ +#define NLMCLNT_OHSIZE ((__NEW_UTS_LEN) + 10u) +struct nlm_rqst { + refcount_t a_count; + unsigned int a_flags; /* initial RPC task flags */ + struct nlm_host * a_host; /* host handle */ + struct lockd_args a_args; /* arguments */ + struct lockd_res a_res; /* result */ + struct nlm_block * a_block; + unsigned int a_retries; /* Retry count */ + u8 a_owner[NLMCLNT_OHSIZE]; + void * a_callback_data; /* sent to nlmclnt_operations callbacks */ +}; + +struct lockd_share; + +/* + * This struct describes a file held open by lockd on behalf of + * an NFS client. + */ +struct nlm_file { + struct hlist_node f_list; /* linked list */ + struct nfs_fh f_handle; /* NFS file handle */ + struct file * f_file[2]; /* VFS file pointers, + indexed by O_ flags */ + struct lockd_share * f_shares; /* DOS shares */ + struct list_head f_blocks; /* blocked locks */ + unsigned int f_locks; /* guesstimate # of locks */ + unsigned int f_count; /* reference count */ + struct mutex f_mutex; /* avoid concurrent access */ +}; + +/* + * This is a server block (i.e. a lock requested by some client which + * couldn't be granted because of a conflicting lock). + */ +#define NLM_NEVER (~(unsigned long) 0) +/* timeout on non-blocking call: */ +#define NLM_TIMEOUT (7 * HZ) + +struct nlm_block { + struct kref b_count; /* Reference count */ + struct list_head b_list; /* linked list of all blocks */ + struct list_head b_flist; /* linked list (per file) */ + struct nlm_rqst * b_call; /* RPC args & callback info */ + struct svc_serv * b_daemon; /* NLM service */ + struct nlm_host * b_host; /* host handle for RPC clnt */ + unsigned long b_when; /* next re-xmit */ + unsigned int b_id; /* block id */ + unsigned char b_granted; /* VFS granted lock */ + struct nlm_file * b_file; /* file in question */ + struct cache_req * b_cache_req; /* deferred request handling */ + struct cache_deferred_req * b_deferred_req; + unsigned int b_flags; /* block flags */ +#define B_QUEUED 1 /* lock queued */ +#define B_GOT_CALLBACK 2 /* got lock or conflicting lock */ +#define B_TIMED_OUT 4 /* filesystem too slow to respond */ +}; + +/* + * Global variables + */ +extern const struct rpc_program nlm_program; +extern const struct svc_version nlmsvc_version1; +extern const struct svc_version nlmsvc_version3; +#ifdef CONFIG_LOCKD_V4 +extern const struct svc_version nlmsvc_version4; +#endif +extern int nlmsvc_grace_period; +extern unsigned long nlm_timeout; +extern bool nsm_use_hostnames; +extern u32 nsm_local_state; + +extern struct timer_list nlmsvc_retry; + +/* + * Lockd client functions + */ +struct nlm_rqst * nlm_alloc_call(struct nlm_host *host); +int nlm_async_call(struct nlm_rqst *, u32, const struct rpc_call_ops *); +int nlm_async_reply(struct nlm_rqst *, u32, const struct rpc_call_ops *); +void nlmclnt_release_call(struct nlm_rqst *); +void nlmclnt_prepare_block(struct nlm_wait *block, struct nlm_host *host, + struct file_lock *fl); +void nlmclnt_queue_block(struct nlm_wait *block); +__be32 nlmclnt_dequeue_block(struct nlm_wait *block); +int nlmclnt_wait(struct nlm_wait *block, struct nlm_rqst *req, long timeout); +__be32 nlmclnt_grant(const struct sockaddr *addr, + const struct lockd_lock *lock); +void nlmclnt_recovery(struct nlm_host *); +int nlmclnt_reclaim(struct nlm_host *, struct file_lock *, + struct nlm_rqst *); +void nlmclnt_next_cookie(struct lockd_cookie *); + +#ifdef CONFIG_LOCKD_V4 +extern const struct rpc_version nlm_version4; +#endif + +/* + * Host cache + */ +struct nlm_host *nlmclnt_lookup_host(const struct sockaddr *sap, + const size_t salen, + const unsigned short protocol, + const u32 version, + const char *hostname, + int noresvport, + struct net *net, + const struct cred *cred); +void nlmclnt_release_host(struct nlm_host *); +struct nlm_host *nlmsvc_lookup_host(const struct svc_rqst *rqstp, + const char *hostname, + const size_t hostname_len); +void nlmsvc_release_host(struct nlm_host *); +struct rpc_clnt * nlm_bind_host(struct nlm_host *); +void nlm_rebind_host(struct nlm_host *); +struct nlm_host * nlm_get_host(struct nlm_host *); +void nlm_shutdown_hosts(void); +void nlm_shutdown_hosts_net(struct net *net); +void nlm_host_rebooted(const struct net *net, + const struct lockd_reboot *); + +/* + * Host monitoring + */ +int nsm_monitor(const struct nlm_host *host); +void nsm_unmonitor(const struct nlm_host *host); + +struct nsm_handle *nsm_get_handle(const struct net *net, + const struct sockaddr *sap, + const size_t salen, + const char *hostname, + const size_t hostname_len); +struct nsm_handle *nsm_reboot_lookup(const struct net *net, + const struct lockd_reboot *info); +void nsm_release(struct nsm_handle *nsm); + +/* + * This is used in garbage collection and resource reclaim + * A return value != 0 means destroy the lock/block/share + */ +typedef int (*nlm_host_match_fn_t)(void *owner, struct nlm_host *ref); + +/* + * Server-side lock handling + */ +int lock_to_openmode(struct file_lock *); +__be32 nlmsvc_lock(struct svc_rqst *, struct nlm_file *, + struct nlm_host *, struct lockd_lock *, int, + struct lockd_cookie *, int); +__be32 nlmsvc_unlock(struct net *net, struct nlm_file *, struct lockd_lock *); +__be32 nlmsvc_testlock(struct svc_rqst *rqstp, struct nlm_file *file, + struct nlm_host *host, struct lockd_lock *lock, + struct lockd_lock *conflock); +__be32 nlmsvc_cancel_blocked(struct net *net, struct nlm_file *, struct lockd_lock *); +void nlmsvc_retry_blocked(struct svc_rqst *rqstp); +void nlmsvc_traverse_blocks(struct nlm_host *, struct nlm_file *, + nlm_host_match_fn_t match); +void nlmsvc_grant_reply(struct lockd_cookie *, __be32); +void nlmsvc_release_call(struct nlm_rqst *); +void nlmsvc_locks_init_private(struct file_lock *, struct nlm_host *, pid_t); +int nlmsvc_dispatch(struct svc_rqst *rqstp); + +/* + * File handling for the server personality + */ +__be32 nlm_lookup_file(struct svc_rqst *, struct nlm_file **, + struct lockd_lock *, int); +void nlm_release_file(struct nlm_file *); +void nlmsvc_put_lockowner(struct nlm_lockowner *); +void nlmsvc_release_lockowner(struct lockd_lock *); +void nlmsvc_mark_resources(struct net *); +void nlmsvc_free_host_resources(struct nlm_host *); +void nlmsvc_invalidate_all(void); + +static inline struct file *nlmsvc_file_file(const struct nlm_file *file) +{ + return file->f_file[O_RDONLY] ? + file->f_file[O_RDONLY] : file->f_file[O_WRONLY]; +} + +static inline struct inode *nlmsvc_file_inode(struct nlm_file *file) +{ + return file_inode(nlmsvc_file_file(file)); +} + +static inline bool +nlmsvc_file_cannot_lock(const struct nlm_file *file) +{ + return exportfs_cannot_lock(nlmsvc_file_file(file)->f_path.dentry->d_sb->s_export_op); +} + +static inline int __nlm_privileged_request4(const struct sockaddr *sap) +{ + const struct sockaddr_in *sin = (struct sockaddr_in *)sap; + + if (ntohs(sin->sin_port) > 1023) + return 0; + + return ipv4_is_loopback(sin->sin_addr.s_addr); +} + +#if IS_ENABLED(CONFIG_IPV6) +static inline int __nlm_privileged_request6(const struct sockaddr *sap) +{ + const struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)sap; + + if (ntohs(sin6->sin6_port) > 1023) + return 0; + + if (ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_MAPPED) + return ipv4_is_loopback(sin6->sin6_addr.s6_addr32[3]); + + return ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_LOOPBACK; +} +#else /* IS_ENABLED(CONFIG_IPV6) */ +static inline int __nlm_privileged_request6(const struct sockaddr *sap) +{ + return 0; +} +#endif /* IS_ENABLED(CONFIG_IPV6) */ + +/* + * Ensure incoming requests are from local privileged callers. + * + * Return TRUE if sender is local and is connecting via a privileged port; + * otherwise return FALSE. + */ +static inline int nlm_privileged_requester(const struct svc_rqst *rqstp) +{ + const struct sockaddr *sap = svc_addr(rqstp); + + switch (sap->sa_family) { + case AF_INET: + return __nlm_privileged_request4(sap); + case AF_INET6: + return __nlm_privileged_request6(sap); + default: + return 0; + } +} + +/* + * Compare two NLM locks. + * When the second lock is of type F_UNLCK, this acts like a wildcard. + */ +static inline int nlm_compare_locks(const struct file_lock *fl1, + const struct file_lock *fl2) +{ + return file_inode(fl1->c.flc_file) == file_inode(fl2->c.flc_file) + && fl1->c.flc_pid == fl2->c.flc_pid + && fl1->c.flc_owner == fl2->c.flc_owner + && fl1->fl_start == fl2->fl_start + && fl1->fl_end == fl2->fl_end + &&(fl1->c.flc_type == fl2->c.flc_type || fl2->c.flc_type == F_UNLCK); +} + +/** + * lockd_set_file_lock_range3 - set the byte range of a file_lock + * @fl: file_lock whose length fields are to be initialized + * @off: starting offset of the lock, in bytes + * @len: length of the byte range, in bytes, or zero + * + * NLMv3 uses a (start, length) representation for lock byte ranges, + * while the kernel's file_lock uses (start, end). Treat a length of + * zero or arithmetic overflow (end wrapping negative when the sum + * exceeds S32_MAX) as "lock to end of file." + */ +static inline void +lockd_set_file_lock_range3(struct file_lock *fl, u32 off, u32 len) +{ + s32 end = off + len - 1; + + fl->fl_start = off; + if (len == 0 || end < 0) + fl->fl_end = OFFSET_MAX; + else + fl->fl_end = end; +} + +/** + * lockd_set_file_lock_range4 - set the byte range of a file_lock + * @fl: file_lock whose length fields are to be initialized + * @off: starting offset of the lock, in bytes + * @len: length of the byte range, in bytes, or zero + * + * The NLMv4 protocol represents lock byte ranges as (start, length), + * where length zero means "lock to end of file." The kernel's file_lock + * structure uses (start, end) representation. Convert from NLMv4 format + * to file_lock format, clamping the starting offset and treating + * arithmetic overflow as "lock to EOF." + */ +static inline void +lockd_set_file_lock_range4(struct file_lock *fl, u64 off, u64 len) +{ + u64 clamped_off = (off > OFFSET_MAX) ? OFFSET_MAX : off; + s64 end = clamped_off + len - 1; + + fl->fl_start = clamped_off; + if (len == 0 || end < 0) + fl->fl_end = OFFSET_MAX; + else + fl->fl_end = end; +} + +extern const struct lock_manager_operations nlmsvc_lock_operations; + +#endif /* _LOCKD_LOCKD_H */ diff --git a/fs/lockd/mon.c b/fs/lockd/mon.c new file mode 100644 index 000000000..a8f5ac6f0 --- /dev/null +++ b/fs/lockd/mon.c @@ -0,0 +1,581 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/lockd/mon.c + * + * The kernel statd client. + * + * Copyright (C) 1996, Olaf Kirch + */ + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +#include "lockd.h" +#include "netns.h" + +#define NLMDBG_FACILITY NLMDBG_MONITOR +#define NSM_PROGRAM 100024 +#define NSM_VERSION 1 + +enum { + NSMPROC_NULL, + NSMPROC_STAT, + NSMPROC_MON, + NSMPROC_UNMON, + NSMPROC_UNMON_ALL, + NSMPROC_SIMU_CRASH, + NSMPROC_NOTIFY, +}; + +struct nsm_args { + struct nsm_private *priv; + u32 prog; /* RPC callback info */ + u32 vers; + u32 proc; + + char *mon_name; + const char *nodename; +}; + +struct nsm_res { + u32 status; + u32 state; +}; + +static const struct rpc_program nsm_program; +static DEFINE_SPINLOCK(nsm_lock); + +/* + * Local NSM state + */ +u32 __read_mostly nsm_local_state; +bool __read_mostly nsm_use_hostnames; + +static inline struct sockaddr *nsm_addr(const struct nsm_handle *nsm) +{ + return (struct sockaddr *)&nsm->sm_addr; +} + +static struct rpc_clnt *nsm_create(struct net *net, const char *nodename) +{ + struct sockaddr_in sin = { + .sin_family = AF_INET, + .sin_addr.s_addr = htonl(INADDR_LOOPBACK), + }; + struct rpc_create_args args = { + .net = net, + .protocol = XPRT_TRANSPORT_TCP, + .address = (struct sockaddr *)&sin, + .addrsize = sizeof(sin), + .servername = "rpc.statd", + .nodename = nodename, + .program = &nsm_program, + .version = NSM_VERSION, + .authflavor = RPC_AUTH_NULL, + .flags = RPC_CLNT_CREATE_NOPING, + .cred = current_cred(), + }; + + return rpc_create(&args); +} + +static int nsm_mon_unmon(struct nsm_handle *nsm, u32 proc, struct nsm_res *res, + const struct nlm_host *host) +{ + int status; + struct rpc_clnt *clnt; + struct nsm_args args = { + .priv = &nsm->sm_priv, + .prog = NLM_PROGRAM, + .vers = 3, + .proc = NLMPROC_NSM_NOTIFY, + .mon_name = nsm->sm_mon_name, + .nodename = host->nodename, + }; + struct rpc_message msg = { + .rpc_argp = &args, + .rpc_resp = res, + }; + + memset(res, 0, sizeof(*res)); + + clnt = nsm_create(host->net, host->nodename); + if (IS_ERR(clnt)) { + dprintk("lockd: failed to create NSM upcall transport, " + "status=%ld, net=%x\n", PTR_ERR(clnt), + host->net->ns.inum); + return PTR_ERR(clnt); + } + + msg.rpc_proc = &clnt->cl_procinfo[proc]; + status = rpc_call_sync(clnt, &msg, RPC_TASK_SOFTCONN); + if (status == -ECONNREFUSED) { + dprintk("lockd: NSM upcall RPC failed, status=%d, forcing rebind\n", + status); + rpc_force_rebind(clnt); + status = rpc_call_sync(clnt, &msg, RPC_TASK_SOFTCONN); + } + if (status < 0) + dprintk("lockd: NSM upcall RPC failed, status=%d\n", + status); + else + status = 0; + + rpc_shutdown_client(clnt); + return status; +} + +/** + * nsm_monitor - Notify a peer in case we reboot + * @host: pointer to nlm_host of peer to notify + * + * If this peer is not already monitored, this function sends an + * upcall to the local rpc.statd to record the name/address of + * the peer to notify in case we reboot. + * + * Returns zero if the peer is monitored by the local rpc.statd; + * otherwise a negative errno value is returned. + */ +int nsm_monitor(const struct nlm_host *host) +{ + struct nsm_handle *nsm = host->h_nsmhandle; + struct nsm_res res; + int status; + + dprintk("lockd: nsm_monitor(%s)\n", nsm->sm_name); + + if (nsm->sm_monitored) + return 0; + + /* + * Choose whether to record the caller_name or IP address of + * this peer in the local rpc.statd's database. + */ + nsm->sm_mon_name = nsm_use_hostnames ? nsm->sm_name : nsm->sm_addrbuf; + + status = nsm_mon_unmon(nsm, NSMPROC_MON, &res, host); + if (unlikely(res.status != 0)) + status = -EIO; + if (unlikely(status < 0)) { + pr_notice_ratelimited("lockd: cannot monitor %s\n", nsm->sm_name); + return status; + } + + nsm->sm_monitored = 1; + if (unlikely(nsm_local_state != res.state)) { + nsm_local_state = res.state; + dprintk("lockd: NSM state changed to %d\n", nsm_local_state); + } + return 0; +} + +/** + * nsm_unmonitor - Unregister peer notification + * @host: pointer to nlm_host of peer to stop monitoring + * + * If this peer is monitored, this function sends an upcall to + * tell the local rpc.statd not to send this peer a notification + * when we reboot. + */ +void nsm_unmonitor(const struct nlm_host *host) +{ + struct nsm_handle *nsm = host->h_nsmhandle; + struct nsm_res res; + int status; + + if (refcount_read(&nsm->sm_count) == 1 + && nsm->sm_monitored && !nsm->sm_sticky) { + dprintk("lockd: nsm_unmonitor(%s)\n", nsm->sm_name); + + status = nsm_mon_unmon(nsm, NSMPROC_UNMON, &res, host); + if (res.status != 0) + status = -EIO; + if (status < 0) + printk(KERN_NOTICE "lockd: cannot unmonitor %s\n", + nsm->sm_name); + else + nsm->sm_monitored = 0; + } +} + +static struct nsm_handle *nsm_lookup_hostname(const struct list_head *nsm_handles, + const char *hostname, const size_t len) +{ + struct nsm_handle *nsm; + + list_for_each_entry(nsm, nsm_handles, sm_link) + if (strlen(nsm->sm_name) == len && + memcmp(nsm->sm_name, hostname, len) == 0) + return nsm; + return NULL; +} + +static struct nsm_handle *nsm_lookup_addr(const struct list_head *nsm_handles, + const struct sockaddr *sap) +{ + struct nsm_handle *nsm; + + list_for_each_entry(nsm, nsm_handles, sm_link) + if (rpc_cmp_addr(nsm_addr(nsm), sap)) + return nsm; + return NULL; +} + +static struct nsm_handle *nsm_lookup_priv(const struct list_head *nsm_handles, + const struct nsm_private *priv) +{ + struct nsm_handle *nsm; + + list_for_each_entry(nsm, nsm_handles, sm_link) + if (memcmp(nsm->sm_priv.data, priv->data, + sizeof(priv->data)) == 0) + return nsm; + return NULL; +} + +/* + * Construct a unique cookie to match this nsm_handle to this monitored + * host. It is passed to the local rpc.statd via NSMPROC_MON, and + * returned via NLMPROC_SM_NOTIFY, in the "priv" field of these + * requests. + * + * The NSM protocol requires that these cookies be unique while the + * system is running. We prefer a stronger requirement of making them + * unique across reboots. If user space bugs cause a stale cookie to + * be sent to the kernel, it could cause the wrong host to lose its + * lock state if cookies were not unique across reboots. + * + * The cookies are exposed only to local user space via loopback. They + * do not appear on the physical network. If we want greater security + * for some reason, nsm_init_private() could perform a one-way hash to + * obscure the contents of the cookie. + */ +static void nsm_init_private(struct nsm_handle *nsm) +{ + u64 *p = (u64 *)&nsm->sm_priv.data; + s64 ns; + + ns = ktime_get_ns(); + put_unaligned(ns, p); + put_unaligned((unsigned long)nsm, p + 1); +} + +static struct nsm_handle *nsm_create_handle(const struct sockaddr *sap, + const size_t salen, + const char *hostname, + const size_t hostname_len) +{ + struct nsm_handle *new; + + if (!hostname) + return NULL; + + new = kzalloc(sizeof(*new) + hostname_len + 1, GFP_KERNEL); + if (unlikely(new == NULL)) + return NULL; + + refcount_set(&new->sm_count, 1); + new->sm_name = (char *)(new + 1); + memcpy(nsm_addr(new), sap, salen); + new->sm_addrlen = salen; + nsm_init_private(new); + + if (rpc_ntop(nsm_addr(new), new->sm_addrbuf, + sizeof(new->sm_addrbuf)) == 0) + (void)snprintf(new->sm_addrbuf, sizeof(new->sm_addrbuf), + "unsupported address family"); + memcpy(new->sm_name, hostname, hostname_len); + new->sm_name[hostname_len] = '\0'; + + return new; +} + +/** + * nsm_get_handle - Find or create a cached nsm_handle + * @net: network namespace + * @sap: pointer to socket address of handle to find + * @salen: length of socket address + * @hostname: pointer to C string containing hostname to find + * @hostname_len: length of C string + * + * Behavior is modulated by the global nsm_use_hostnames variable. + * + * Returns a cached nsm_handle after bumping its ref count, or + * returns a fresh nsm_handle if a handle that matches @sap and/or + * @hostname cannot be found in the handle cache. Returns NULL if + * an error occurs. + */ +struct nsm_handle *nsm_get_handle(const struct net *net, + const struct sockaddr *sap, + const size_t salen, const char *hostname, + const size_t hostname_len) +{ + struct nsm_handle *cached, *new = NULL; + struct lockd_net *ln = net_generic(net, lockd_net_id); + + if (hostname && memchr(hostname, '/', hostname_len) != NULL) { + if (printk_ratelimit()) { + printk(KERN_WARNING "Invalid hostname \"%.*s\" " + "in NFS lock request\n", + (int)hostname_len, hostname); + } + return NULL; + } + +retry: + spin_lock(&nsm_lock); + + if (nsm_use_hostnames && hostname != NULL) + cached = nsm_lookup_hostname(&ln->nsm_handles, + hostname, hostname_len); + else + cached = nsm_lookup_addr(&ln->nsm_handles, sap); + + if (cached != NULL) { + refcount_inc(&cached->sm_count); + spin_unlock(&nsm_lock); + kfree(new); + dprintk("lockd: found nsm_handle for %s (%s), " + "cnt %d\n", cached->sm_name, + cached->sm_addrbuf, + refcount_read(&cached->sm_count)); + return cached; + } + + if (new != NULL) { + list_add(&new->sm_link, &ln->nsm_handles); + spin_unlock(&nsm_lock); + dprintk("lockd: created nsm_handle for %s (%s)\n", + new->sm_name, new->sm_addrbuf); + return new; + } + + spin_unlock(&nsm_lock); + + new = nsm_create_handle(sap, salen, hostname, hostname_len); + if (unlikely(new == NULL)) + return NULL; + goto retry; +} + +/** + * nsm_reboot_lookup - match NLMPROC_SM_NOTIFY arguments to an nsm_handle + * @net: network namespace + * @info: pointer to NLMPROC_SM_NOTIFY arguments + * + * Returns a matching nsm_handle if found in the nsm cache. The returned + * nsm_handle's reference count is bumped. Otherwise returns NULL if some + * error occurred. + */ +struct nsm_handle *nsm_reboot_lookup(const struct net *net, + const struct lockd_reboot *info) +{ + struct nsm_handle *cached; + struct lockd_net *ln = net_generic(net, lockd_net_id); + + spin_lock(&nsm_lock); + + cached = nsm_lookup_priv(&ln->nsm_handles, &info->priv); + if (unlikely(cached == NULL)) { + spin_unlock(&nsm_lock); + dprintk("lockd: never saw rebooted peer '%.*s' before\n", + info->len, info->mon); + return cached; + } + + refcount_inc(&cached->sm_count); + spin_unlock(&nsm_lock); + + dprintk("lockd: host %s (%s) rebooted, cnt %d\n", + cached->sm_name, cached->sm_addrbuf, + refcount_read(&cached->sm_count)); + return cached; +} + +/** + * nsm_release - Release an NSM handle + * @nsm: pointer to handle to be released + * + */ +void nsm_release(struct nsm_handle *nsm) +{ + if (refcount_dec_and_lock(&nsm->sm_count, &nsm_lock)) { + list_del(&nsm->sm_link); + spin_unlock(&nsm_lock); + dprintk("lockd: destroyed nsm_handle for %s (%s)\n", + nsm->sm_name, nsm->sm_addrbuf); + kfree(nsm); + } +} + +/* + * XDR functions for NSM. + * + * See https://www.opengroup.org/ for details on the Network + * Status Monitor wire protocol. + */ + +static void encode_nsm_string(struct xdr_stream *xdr, const char *string) +{ + const u32 len = strlen(string); + __be32 *p; + + p = xdr_reserve_space(xdr, 4 + len); + xdr_encode_opaque(p, string, len); +} + +/* + * "mon_name" specifies the host to be monitored. + */ +static void encode_mon_name(struct xdr_stream *xdr, const struct nsm_args *argp) +{ + encode_nsm_string(xdr, argp->mon_name); +} + +/* + * The "my_id" argument specifies the hostname and RPC procedure + * to be called when the status manager receives notification + * (via the NLMPROC_SM_NOTIFY call) that the state of host "mon_name" + * has changed. + */ +static void encode_my_id(struct xdr_stream *xdr, const struct nsm_args *argp) +{ + __be32 *p; + + encode_nsm_string(xdr, argp->nodename); + p = xdr_reserve_space(xdr, 4 + 4 + 4); + *p++ = cpu_to_be32(argp->prog); + *p++ = cpu_to_be32(argp->vers); + *p = cpu_to_be32(argp->proc); +} + +/* + * The "mon_id" argument specifies the non-private arguments + * of an NSMPROC_MON or NSMPROC_UNMON call. + */ +static void encode_mon_id(struct xdr_stream *xdr, const struct nsm_args *argp) +{ + encode_mon_name(xdr, argp); + encode_my_id(xdr, argp); +} + +/* + * The "priv" argument may contain private information required + * by the NSMPROC_MON call. This information will be supplied in the + * NLMPROC_SM_NOTIFY call. + */ +static void encode_priv(struct xdr_stream *xdr, const struct nsm_args *argp) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, SM_PRIV_SIZE); + xdr_encode_opaque_fixed(p, argp->priv->data, SM_PRIV_SIZE); +} + +static void nsm_xdr_enc_mon(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *argp) +{ + encode_mon_id(xdr, argp); + encode_priv(xdr, argp); +} + +static void nsm_xdr_enc_unmon(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *argp) +{ + encode_mon_id(xdr, argp); +} + +static int nsm_xdr_dec_stat_res(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nsm_res *resp = data; + __be32 *p; + + p = xdr_inline_decode(xdr, 4 + 4); + if (unlikely(p == NULL)) + return -EIO; + resp->status = be32_to_cpup(p++); + resp->state = be32_to_cpup(p); + + dprintk("lockd: %s status %d state %d\n", + __func__, resp->status, resp->state); + return 0; +} + +static int nsm_xdr_dec_stat(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nsm_res *resp = data; + __be32 *p; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(p == NULL)) + return -EIO; + resp->state = be32_to_cpup(p); + + dprintk("lockd: %s state %d\n", __func__, resp->state); + return 0; +} + +#define SM_my_name_sz (1+XDR_QUADLEN(SM_MAXSTRLEN)) +#define SM_my_id_sz (SM_my_name_sz+3) +#define SM_mon_name_sz (1+XDR_QUADLEN(SM_MAXSTRLEN)) +#define SM_mon_id_sz (SM_mon_name_sz+SM_my_id_sz) +#define SM_priv_sz (XDR_QUADLEN(SM_PRIV_SIZE)) +#define SM_mon_sz (SM_mon_id_sz+SM_priv_sz) +#define SM_monres_sz 2 +#define SM_unmonres_sz 1 + +static const struct rpc_procinfo nsm_procedures[] = { +[NSMPROC_MON] = { + .p_proc = NSMPROC_MON, + .p_encode = nsm_xdr_enc_mon, + .p_decode = nsm_xdr_dec_stat_res, + .p_arglen = SM_mon_sz, + .p_replen = SM_monres_sz, + .p_statidx = NSMPROC_MON, + .p_name = "MONITOR", + }, +[NSMPROC_UNMON] = { + .p_proc = NSMPROC_UNMON, + .p_encode = nsm_xdr_enc_unmon, + .p_decode = nsm_xdr_dec_stat, + .p_arglen = SM_mon_id_sz, + .p_replen = SM_unmonres_sz, + .p_statidx = NSMPROC_UNMON, + .p_name = "UNMONITOR", + }, +}; + +static unsigned int nsm_version1_counts[ARRAY_SIZE(nsm_procedures)]; +static const struct rpc_version nsm_version1 = { + .number = 1, + .nrprocs = ARRAY_SIZE(nsm_procedures), + .procs = nsm_procedures, + .counts = nsm_version1_counts, +}; + +static const struct rpc_version *nsm_version[] = { + [1] = &nsm_version1, +}; + +static struct rpc_stat nsm_stats; + +static const struct rpc_program nsm_program = { + .name = "statd", + .number = NSM_PROGRAM, + .nrvers = ARRAY_SIZE(nsm_version), + .version = nsm_version, + .stats = &nsm_stats +}; diff --git a/fs/lockd/netlink.c b/fs/lockd/netlink.c new file mode 100644 index 000000000..880c42b4f --- /dev/null +++ b/fs/lockd/netlink.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) +/* Do not edit directly, auto-generated from: */ +/* Documentation/netlink/specs/lockd.yaml */ +/* YNL-GEN kernel source */ +/* To regenerate run: tools/net/ynl/ynl-regen.sh */ + +#include +#include + +#include "netlink.h" + +#include + +/* LOCKD_CMD_SERVER_SET - do */ +static const struct nla_policy lockd_server_set_nl_policy[LOCKD_A_SERVER_UDP_PORT + 1] = { + [LOCKD_A_SERVER_GRACETIME] = { .type = NLA_U32, }, + [LOCKD_A_SERVER_TCP_PORT] = { .type = NLA_U16, }, + [LOCKD_A_SERVER_UDP_PORT] = { .type = NLA_U16, }, +}; + +/* Ops table for lockd */ +static const struct genl_split_ops lockd_nl_ops[] = { + { + .cmd = LOCKD_CMD_SERVER_SET, + .doit = lockd_nl_server_set_doit, + .policy = lockd_server_set_nl_policy, + .maxattr = LOCKD_A_SERVER_UDP_PORT, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = LOCKD_CMD_SERVER_GET, + .doit = lockd_nl_server_get_doit, + .flags = GENL_CMD_CAP_DO, + }, +}; + +struct genl_family lockd_nl_family __ro_after_init = { + .name = LOCKD_FAMILY_NAME, + .version = LOCKD_FAMILY_VERSION, + .netnsok = true, + .parallel_ops = true, + .module = THIS_MODULE, + .split_ops = lockd_nl_ops, + .n_split_ops = ARRAY_SIZE(lockd_nl_ops), +}; diff --git a/fs/lockd/netlink.h b/fs/lockd/netlink.h new file mode 100644 index 000000000..d8408f077 --- /dev/null +++ b/fs/lockd/netlink.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */ +/* Do not edit directly, auto-generated from: */ +/* Documentation/netlink/specs/lockd.yaml */ +/* YNL-GEN kernel header */ +/* To regenerate run: tools/net/ynl/ynl-regen.sh */ + +#ifndef _LINUX_LOCKD_GEN_H +#define _LINUX_LOCKD_GEN_H + +#include +#include + +#include + +int lockd_nl_server_set_doit(struct sk_buff *skb, struct genl_info *info); +int lockd_nl_server_get_doit(struct sk_buff *skb, struct genl_info *info); + +extern struct genl_family lockd_nl_family; + +#endif /* _LINUX_LOCKD_GEN_H */ diff --git a/fs/lockd/netns.h b/fs/lockd/netns.h new file mode 100644 index 000000000..88e8e2a97 --- /dev/null +++ b/fs/lockd/netns.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LOCKD_NETNS_H__ +#define __LOCKD_NETNS_H__ + +#include +#include +#include + +struct lockd_net { + unsigned int nlmsvc_users; + unsigned long next_gc; + unsigned long nrhosts; + u32 gracetime; + u16 tcp_port; + u16 udp_port; + + struct delayed_work grace_period_end; + struct lock_manager lockd_manager; + + struct list_head nsm_handles; +}; + +extern unsigned int lockd_net_id; + +#endif diff --git a/fs/lockd/nlm.h b/fs/lockd/nlm.h new file mode 100644 index 000000000..47be65d01 --- /dev/null +++ b/fs/lockd/nlm.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Declarations for the Network Lock Manager protocol. + * + * Copyright (C) 1996, Olaf Kirch + */ + +#ifndef _LOCKD_NLM_H +#define _LOCKD_NLM_H + + +/* Maximum file offset in file_lock.fl_end */ +# define NLM_OFFSET_MAX ((s32) 0x7fffffff) +# define NLM4_OFFSET_MAX ((s64) ((~(u64)0) >> 1)) + +/* Return states for NLM */ +enum { + NLM_LCK_GRANTED = 0, + NLM_LCK_DENIED = 1, + NLM_LCK_DENIED_NOLOCKS = 2, + NLM_LCK_BLOCKED = 3, + NLM_LCK_DENIED_GRACE_PERIOD = 4, +#ifdef CONFIG_LOCKD_V4 + NLM_DEADLCK = 5, + NLM_ROFS = 6, + NLM_STALE_FH = 7, + NLM_FBIG = 8, + NLM_FAILED = 9, +#endif +}; + +#define NLM_PROGRAM 100021 + +#define NLMPROC_NULL 0 +#define NLMPROC_TEST 1 +#define NLMPROC_LOCK 2 +#define NLMPROC_CANCEL 3 +#define NLMPROC_UNLOCK 4 +#define NLMPROC_GRANTED 5 +#define NLMPROC_TEST_MSG 6 +#define NLMPROC_LOCK_MSG 7 +#define NLMPROC_CANCEL_MSG 8 +#define NLMPROC_UNLOCK_MSG 9 +#define NLMPROC_GRANTED_MSG 10 +#define NLMPROC_TEST_RES 11 +#define NLMPROC_LOCK_RES 12 +#define NLMPROC_CANCEL_RES 13 +#define NLMPROC_UNLOCK_RES 14 +#define NLMPROC_GRANTED_RES 15 +#define NLMPROC_NSM_NOTIFY 16 /* statd callback */ +#define NLMPROC_SHARE 20 +#define NLMPROC_UNSHARE 21 +#define NLMPROC_NM_LOCK 22 +#define NLMPROC_FREE_ALL 23 + +#endif /* _LOCKD_NLM_H */ diff --git a/fs/lockd/nlm3xdr_gen.c b/fs/lockd/nlm3xdr_gen.c new file mode 100644 index 000000000..4b6e3ee7a --- /dev/null +++ b/fs/lockd/nlm3xdr_gen.c @@ -0,0 +1,686 @@ +// SPDX-License-Identifier: GPL-2.0 +// Generated by xdrgen. Manual edits will be lost. +// XDR specification file: ../../Documentation/sunrpc/xdr/nlm3.x +// XDR specification modification time: Mon Jun 29 20:42:29 2026 + +#include + +#include "nlm3xdr_gen.h" + +static bool __maybe_unused +xdrgen_decode_netobj(struct xdr_stream *xdr, netobj *ptr) +{ + return xdrgen_decode_opaque(xdr, ptr, MAXNETOBJ_SZ); +} + +static bool __maybe_unused +xdrgen_decode_nlm_stats(struct xdr_stream *xdr, nlm_stats *ptr) +{ + __be32 raw; + u32 val; + + if (xdr_stream_decode_be32(xdr, &raw) < 0) + return false; + val = be32_to_cpu(raw); + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case LCK_GRANTED: + case LCK_DENIED: + case LCK_DENIED_NOLOCKS: + case LCK_BLOCKED: + case LCK_DENIED_GRACE_PERIOD: + break; + default: + return false; + } + *ptr = raw; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_stat(struct xdr_stream *xdr, struct nlm_stat *ptr) +{ + if (!xdrgen_decode_nlm_stats(xdr, &ptr->stat)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_res(struct xdr_stream *xdr, struct nlm_res *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_nlm_stat(xdr, &ptr->stat)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_holder(struct xdr_stream *xdr, struct nlm_holder *ptr) +{ + if (!xdrgen_decode_bool(xdr, &ptr->exclusive)) + return false; + if (!xdrgen_decode_int(xdr, &ptr->uppid)) + return false; + if (!xdrgen_decode_netobj(xdr, &ptr->oh)) + return false; + if (!xdrgen_decode_unsigned_int(xdr, &ptr->l_offset)) + return false; + if (!xdrgen_decode_unsigned_int(xdr, &ptr->l_len)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_testrply(struct xdr_stream *xdr, struct nlm_testrply *ptr) +{ + if (!xdrgen_decode_nlm_stats(xdr, &ptr->stat)) + return false; + switch (ptr->stat) { + case __constant_cpu_to_be32(LCK_DENIED): + if (!xdrgen_decode_nlm_holder(xdr, &ptr->u.holder)) + return false; + break; + default: + break; + } + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_testres(struct xdr_stream *xdr, struct nlm_testres *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_nlm_testrply(xdr, &ptr->test_stat)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_lock(struct xdr_stream *xdr, struct nlm_lock *ptr) +{ + if (!xdrgen_decode_string(xdr, (string *)ptr, LM_MAXSTRLEN)) + return false; + if (!xdrgen_decode_netobj(xdr, &ptr->fh)) + return false; + if (!xdrgen_decode_netobj(xdr, &ptr->oh)) + return false; + if (!xdrgen_decode_int(xdr, &ptr->uppid)) + return false; + if (!xdrgen_decode_unsigned_int(xdr, &ptr->l_offset)) + return false; + if (!xdrgen_decode_unsigned_int(xdr, &ptr->l_len)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_lockargs(struct xdr_stream *xdr, struct nlm_lockargs *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->block)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->exclusive)) + return false; + if (!xdrgen_decode_nlm_lock(xdr, &ptr->alock)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->reclaim)) + return false; + if (!xdrgen_decode_int(xdr, &ptr->state)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_cancargs(struct xdr_stream *xdr, struct nlm_cancargs *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->block)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->exclusive)) + return false; + if (!xdrgen_decode_nlm_lock(xdr, &ptr->alock)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_testargs(struct xdr_stream *xdr, struct nlm_testargs *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->exclusive)) + return false; + if (!xdrgen_decode_nlm_lock(xdr, &ptr->alock)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_unlockargs(struct xdr_stream *xdr, struct nlm_unlockargs *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_nlm_lock(xdr, &ptr->alock)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_fsh_mode(struct xdr_stream *xdr, fsh_mode *ptr) +{ + u32 val; + + if (xdr_stream_decode_u32(xdr, &val) < 0) + return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case fsm_DN: + case fsm_DR: + case fsm_DW: + case fsm_DRW: + break; + default: + return false; + } + *ptr = val; + return true; +} + +static bool __maybe_unused +xdrgen_decode_fsh_access(struct xdr_stream *xdr, fsh_access *ptr) +{ + u32 val; + + if (xdr_stream_decode_u32(xdr, &val) < 0) + return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case fsa_NONE: + case fsa_R: + case fsa_W: + case fsa_RW: + break; + default: + return false; + } + *ptr = val; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_share(struct xdr_stream *xdr, struct nlm_share *ptr) +{ + if (!xdrgen_decode_string(xdr, (string *)ptr, LM_MAXSTRLEN)) + return false; + if (!xdrgen_decode_netobj(xdr, &ptr->fh)) + return false; + if (!xdrgen_decode_netobj(xdr, &ptr->oh)) + return false; + if (!xdrgen_decode_fsh_mode(xdr, &ptr->mode)) + return false; + if (!xdrgen_decode_fsh_access(xdr, &ptr->access)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_shareargs(struct xdr_stream *xdr, struct nlm_shareargs *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_nlm_share(xdr, &ptr->share)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->reclaim)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_shareres(struct xdr_stream *xdr, struct nlm_shareres *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_nlm_stats(xdr, &ptr->stat)) + return false; + if (!xdrgen_decode_int(xdr, &ptr->sequence)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_notify(struct xdr_stream *xdr, struct nlm_notify *ptr) +{ + if (!xdrgen_decode_string(xdr, (string *)ptr, LM_MAXNAMELEN)) + return false; + if (!xdrgen_decode_long(xdr, &ptr->state)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm_notifyargs(struct xdr_stream *xdr, struct nlm_notifyargs *ptr) +{ + if (!xdrgen_decode_nlm_notify(xdr, &ptr->notify)) + return false; + if (xdr_stream_decode_opaque_fixed(xdr, ptr->private, SM_PRIV_SIZE) < 0) + return false; + return true; +} + +/** + * nlm_svc_decode_nlm_testargs - Decode a nlm_testargs argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm_svc_decode_nlm_testargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm_testargs *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm_testargs(xdr, argp); +} + +/** + * nlm_svc_decode_nlm_lockargs - Decode a nlm_lockargs argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm_svc_decode_nlm_lockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm_lockargs *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm_lockargs(xdr, argp); +} + +/** + * nlm_svc_decode_nlm_cancargs - Decode a nlm_cancargs argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm_svc_decode_nlm_cancargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm_cancargs *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm_cancargs(xdr, argp); +} + +/** + * nlm_svc_decode_nlm_unlockargs - Decode a nlm_unlockargs argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm_svc_decode_nlm_unlockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm_unlockargs *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm_unlockargs(xdr, argp); +} + +/** + * nlm_svc_decode_nlm_testres - Decode a nlm_testres argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm_svc_decode_nlm_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm_testres *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm_testres(xdr, argp); +} + +/** + * nlm_svc_decode_nlm_res - Decode a nlm_res argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm_svc_decode_nlm_res(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm_res *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm_res(xdr, argp); +} + +/** + * nlm_svc_decode_nlm_notifyargs - Decode a nlm_notifyargs argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm_svc_decode_nlm_notifyargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm_notifyargs *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm_notifyargs(xdr, argp); +} + +/** + * nlm_svc_decode_nlm_shareargs - Decode a nlm_shareargs argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm_svc_decode_nlm_shareargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm_shareargs *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm_shareargs(xdr, argp); +} + +/** + * nlm_svc_decode_nlm_notify - Decode a nlm_notify argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm_svc_decode_nlm_notify(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm_notify *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm_notify(xdr, argp); +} + +static bool __maybe_unused +xdrgen_encode_netobj(struct xdr_stream *xdr, const netobj value) +{ + return xdr_stream_encode_opaque(xdr, value.data, value.len) >= 0; +} + +static bool __maybe_unused +xdrgen_encode_nlm_stats(struct xdr_stream *xdr, nlm_stats value) +{ + return xdr_stream_encode_be32(xdr, value) == XDR_UNIT; +} + +static bool __maybe_unused +xdrgen_encode_nlm_stat(struct xdr_stream *xdr, const struct nlm_stat *value) +{ + if (!xdrgen_encode_nlm_stats(xdr, value->stat)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm_res(struct xdr_stream *xdr, const struct nlm_res *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_nlm_stat(xdr, &value->stat)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm_holder(struct xdr_stream *xdr, const struct nlm_holder *value) +{ + if (!xdrgen_encode_bool(xdr, value->exclusive)) + return false; + if (!xdrgen_encode_int(xdr, value->uppid)) + return false; + if (!xdrgen_encode_netobj(xdr, value->oh)) + return false; + if (!xdrgen_encode_unsigned_int(xdr, value->l_offset)) + return false; + if (!xdrgen_encode_unsigned_int(xdr, value->l_len)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm_testrply(struct xdr_stream *xdr, const struct nlm_testrply *ptr) +{ + if (!xdrgen_encode_nlm_stats(xdr, ptr->stat)) + return false; + switch (ptr->stat) { + case __constant_cpu_to_be32(LCK_DENIED): + if (!xdrgen_encode_nlm_holder(xdr, &ptr->u.holder)) + return false; + break; + default: + break; + } + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm_testres(struct xdr_stream *xdr, const struct nlm_testres *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_nlm_testrply(xdr, &value->test_stat)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm_lock(struct xdr_stream *xdr, const struct nlm_lock *value) +{ + if (value->caller_name.len > LM_MAXSTRLEN) + return false; + if (xdr_stream_encode_opaque(xdr, value->caller_name.data, value->caller_name.len) < 0) + return false; + if (!xdrgen_encode_netobj(xdr, value->fh)) + return false; + if (!xdrgen_encode_netobj(xdr, value->oh)) + return false; + if (!xdrgen_encode_int(xdr, value->uppid)) + return false; + if (!xdrgen_encode_unsigned_int(xdr, value->l_offset)) + return false; + if (!xdrgen_encode_unsigned_int(xdr, value->l_len)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm_lockargs(struct xdr_stream *xdr, const struct nlm_lockargs *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_bool(xdr, value->block)) + return false; + if (!xdrgen_encode_bool(xdr, value->exclusive)) + return false; + if (!xdrgen_encode_nlm_lock(xdr, &value->alock)) + return false; + if (!xdrgen_encode_bool(xdr, value->reclaim)) + return false; + if (!xdrgen_encode_int(xdr, value->state)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm_cancargs(struct xdr_stream *xdr, const struct nlm_cancargs *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_bool(xdr, value->block)) + return false; + if (!xdrgen_encode_bool(xdr, value->exclusive)) + return false; + if (!xdrgen_encode_nlm_lock(xdr, &value->alock)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm_testargs(struct xdr_stream *xdr, const struct nlm_testargs *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_bool(xdr, value->exclusive)) + return false; + if (!xdrgen_encode_nlm_lock(xdr, &value->alock)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm_unlockargs(struct xdr_stream *xdr, const struct nlm_unlockargs *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_nlm_lock(xdr, &value->alock)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_fsh_mode(struct xdr_stream *xdr, fsh_mode value) +{ + return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; +} + +static bool __maybe_unused +xdrgen_encode_fsh_access(struct xdr_stream *xdr, fsh_access value) +{ + return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; +} + +static bool __maybe_unused +xdrgen_encode_nlm_share(struct xdr_stream *xdr, const struct nlm_share *value) +{ + if (value->caller_name.len > LM_MAXSTRLEN) + return false; + if (xdr_stream_encode_opaque(xdr, value->caller_name.data, value->caller_name.len) < 0) + return false; + if (!xdrgen_encode_netobj(xdr, value->fh)) + return false; + if (!xdrgen_encode_netobj(xdr, value->oh)) + return false; + if (!xdrgen_encode_fsh_mode(xdr, value->mode)) + return false; + if (!xdrgen_encode_fsh_access(xdr, value->access)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm_shareargs(struct xdr_stream *xdr, const struct nlm_shareargs *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_nlm_share(xdr, &value->share)) + return false; + if (!xdrgen_encode_bool(xdr, value->reclaim)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm_shareres(struct xdr_stream *xdr, const struct nlm_shareres *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_nlm_stats(xdr, value->stat)) + return false; + if (!xdrgen_encode_int(xdr, value->sequence)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm_notify(struct xdr_stream *xdr, const struct nlm_notify *value) +{ + if (value->name.len > LM_MAXNAMELEN) + return false; + if (xdr_stream_encode_opaque(xdr, value->name.data, value->name.len) < 0) + return false; + if (!xdrgen_encode_long(xdr, value->state)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm_notifyargs(struct xdr_stream *xdr, const struct nlm_notifyargs *value) +{ + if (!xdrgen_encode_nlm_notify(xdr, &value->notify)) + return false; + if (xdr_stream_encode_opaque_fixed(xdr, value->private, SM_PRIV_SIZE) < 0) + return false; + return true; +} + +/** + * nlm_svc_encode_nlm_testres - Encode a nlm_testres result + * @rqstp: RPC transaction context + * @xdr: target XDR data stream + * + * Return values: + * %true: procedure results encoded successfully + * %false: encode failed + */ +bool nlm_svc_encode_nlm_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm_testres *resp = rqstp->rq_resp; + + return xdrgen_encode_nlm_testres(xdr, resp); +} + +/** + * nlm_svc_encode_nlm_res - Encode a nlm_res result + * @rqstp: RPC transaction context + * @xdr: target XDR data stream + * + * Return values: + * %true: procedure results encoded successfully + * %false: encode failed + */ +bool nlm_svc_encode_nlm_res(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm_res *resp = rqstp->rq_resp; + + return xdrgen_encode_nlm_res(xdr, resp); +} + +/** + * nlm_svc_encode_nlm_shareres - Encode a nlm_shareres result + * @rqstp: RPC transaction context + * @xdr: target XDR data stream + * + * Return values: + * %true: procedure results encoded successfully + * %false: encode failed + */ +bool nlm_svc_encode_nlm_shareres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm_shareres *resp = rqstp->rq_resp; + + return xdrgen_encode_nlm_shareres(xdr, resp); +} diff --git a/fs/lockd/nlm3xdr_gen.h b/fs/lockd/nlm3xdr_gen.h new file mode 100644 index 000000000..bdbfc26ba --- /dev/null +++ b/fs/lockd/nlm3xdr_gen.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Generated by xdrgen. Manual edits will be lost. */ +/* XDR specification file: ../../Documentation/sunrpc/xdr/nlm3.x */ +/* XDR specification modification time: Mon Jun 29 20:42:29 2026 */ + +#ifndef _LINUX_XDRGEN_NLM3_DECL_H +#define _LINUX_XDRGEN_NLM3_DECL_H + +#include + +#include +#include +#include +#include + +bool nlm_svc_decode_nlm_testargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm_svc_decode_nlm_lockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm_svc_decode_nlm_cancargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm_svc_decode_nlm_unlockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm_svc_decode_nlm_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm_svc_decode_nlm_res(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm_svc_decode_nlm_notifyargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm_svc_decode_nlm_shareargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm_svc_decode_nlm_notify(struct svc_rqst *rqstp, struct xdr_stream *xdr); + +bool nlm_svc_encode_nlm_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm_svc_encode_nlm_res(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm_svc_encode_nlm_shareres(struct svc_rqst *rqstp, struct xdr_stream *xdr); + +#endif /* _LINUX_XDRGEN_NLM3_DECL_H */ diff --git a/fs/lockd/nlm4xdr_gen.c b/fs/lockd/nlm4xdr_gen.c new file mode 100644 index 000000000..f98d74ee3 --- /dev/null +++ b/fs/lockd/nlm4xdr_gen.c @@ -0,0 +1,739 @@ +// SPDX-License-Identifier: GPL-2.0 +// Generated by xdrgen. Manual edits will be lost. +// XDR specification file: ../../Documentation/sunrpc/xdr/nlm4.x +// XDR specification modification time: Mon Jun 29 20:42:29 2026 + +#include + +#include "nlm4xdr_gen.h" + +static bool __maybe_unused +xdrgen_decode_netobj(struct xdr_stream *xdr, netobj *ptr) +{ + return xdrgen_decode_opaque(xdr, ptr, MAXNETOBJ_SZ); +} + +static bool __maybe_unused +xdrgen_decode_fsh4_mode(struct xdr_stream *xdr, fsh4_mode *ptr) +{ + u32 val; + + if (xdr_stream_decode_u32(xdr, &val) < 0) + return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case fsm_DN: + case fsm_DR: + case fsm_DW: + case fsm_DRW: + break; + default: + return false; + } + *ptr = val; + return true; +} + +static bool __maybe_unused +xdrgen_decode_fsh4_access(struct xdr_stream *xdr, fsh4_access *ptr) +{ + u32 val; + + if (xdr_stream_decode_u32(xdr, &val) < 0) + return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case fsa_NONE: + case fsa_R: + case fsa_W: + case fsa_RW: + break; + default: + return false; + } + *ptr = val; + return true; +} + +static bool __maybe_unused +xdrgen_decode_uint64(struct xdr_stream *xdr, uint64 *ptr) +{ + return xdrgen_decode_unsigned_hyper(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_int64(struct xdr_stream *xdr, int64 *ptr) +{ + return xdrgen_decode_hyper(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_uint32(struct xdr_stream *xdr, uint32 *ptr) +{ + return xdrgen_decode_unsigned_long(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_int32(struct xdr_stream *xdr, int32 *ptr) +{ + return xdrgen_decode_long(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_nlm4_stats(struct xdr_stream *xdr, nlm4_stats *ptr) +{ + __be32 raw; + u32 val; + + if (xdr_stream_decode_be32(xdr, &raw) < 0) + return false; + val = be32_to_cpu(raw); + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case NLM4_GRANTED: + case NLM4_DENIED: + case NLM4_DENIED_NOLOCKS: + case NLM4_BLOCKED: + case NLM4_DENIED_GRACE_PERIOD: + case NLM4_DEADLCK: + case NLM4_ROFS: + case NLM4_STALE_FH: + case NLM4_FBIG: + case NLM4_FAILED: + break; + default: + return false; + } + *ptr = raw; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_holder(struct xdr_stream *xdr, struct nlm4_holder *ptr) +{ + if (!xdrgen_decode_bool(xdr, &ptr->exclusive)) + return false; + if (!xdrgen_decode_int32(xdr, &ptr->svid)) + return false; + if (!xdrgen_decode_netobj(xdr, &ptr->oh)) + return false; + if (!xdrgen_decode_uint64(xdr, &ptr->l_offset)) + return false; + if (!xdrgen_decode_uint64(xdr, &ptr->l_len)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_testrply(struct xdr_stream *xdr, struct nlm4_testrply *ptr) +{ + if (!xdrgen_decode_nlm4_stats(xdr, &ptr->stat)) + return false; + switch (ptr->stat) { + case __constant_cpu_to_be32(NLM4_DENIED): + if (!xdrgen_decode_nlm4_holder(xdr, &ptr->u.holder)) + return false; + break; + default: + break; + } + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_stat(struct xdr_stream *xdr, struct nlm4_stat *ptr) +{ + if (!xdrgen_decode_nlm4_stats(xdr, &ptr->stat)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_res(struct xdr_stream *xdr, struct nlm4_res *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_nlm4_stat(xdr, &ptr->stat)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_testres(struct xdr_stream *xdr, struct nlm4_testres *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_nlm4_testrply(xdr, &ptr->stat)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_lock(struct xdr_stream *xdr, struct nlm4_lock *ptr) +{ + if (!xdrgen_decode_string(xdr, (string *)ptr, LM_MAXSTRLEN)) + return false; + if (!xdrgen_decode_netobj(xdr, &ptr->fh)) + return false; + if (!xdrgen_decode_netobj(xdr, &ptr->oh)) + return false; + if (!xdrgen_decode_int32(xdr, &ptr->svid)) + return false; + if (!xdrgen_decode_uint64(xdr, &ptr->l_offset)) + return false; + if (!xdrgen_decode_uint64(xdr, &ptr->l_len)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_lockargs(struct xdr_stream *xdr, struct nlm4_lockargs *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->block)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->exclusive)) + return false; + if (!xdrgen_decode_nlm4_lock(xdr, &ptr->alock)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->reclaim)) + return false; + if (!xdrgen_decode_int32(xdr, &ptr->state)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_cancargs(struct xdr_stream *xdr, struct nlm4_cancargs *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->block)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->exclusive)) + return false; + if (!xdrgen_decode_nlm4_lock(xdr, &ptr->alock)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_testargs(struct xdr_stream *xdr, struct nlm4_testargs *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->exclusive)) + return false; + if (!xdrgen_decode_nlm4_lock(xdr, &ptr->alock)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_unlockargs(struct xdr_stream *xdr, struct nlm4_unlockargs *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_nlm4_lock(xdr, &ptr->alock)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_share(struct xdr_stream *xdr, struct nlm4_share *ptr) +{ + if (!xdrgen_decode_string(xdr, (string *)ptr, LM_MAXSTRLEN)) + return false; + if (!xdrgen_decode_netobj(xdr, &ptr->fh)) + return false; + if (!xdrgen_decode_netobj(xdr, &ptr->oh)) + return false; + if (!xdrgen_decode_fsh4_mode(xdr, &ptr->mode)) + return false; + if (!xdrgen_decode_fsh4_access(xdr, &ptr->access)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_shareargs(struct xdr_stream *xdr, struct nlm4_shareargs *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_nlm4_share(xdr, &ptr->share)) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->reclaim)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_shareres(struct xdr_stream *xdr, struct nlm4_shareres *ptr) +{ + if (!xdrgen_decode_netobj(xdr, &ptr->cookie)) + return false; + if (!xdrgen_decode_nlm4_stats(xdr, &ptr->stat)) + return false; + if (!xdrgen_decode_int32(xdr, &ptr->sequence)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_notify(struct xdr_stream *xdr, struct nlm4_notify *ptr) +{ + if (!xdrgen_decode_string(xdr, (string *)ptr, LM_MAXNAMELEN)) + return false; + if (!xdrgen_decode_int32(xdr, &ptr->state)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nlm4_notifyargs(struct xdr_stream *xdr, struct nlm4_notifyargs *ptr) +{ + if (!xdrgen_decode_nlm4_notify(xdr, &ptr->notify)) + return false; + if (xdr_stream_decode_opaque_fixed(xdr, ptr->private, SM_PRIV_SIZE) < 0) + return false; + return true; +} + +/** + * nlm4_svc_decode_nlm4_testargs - Decode a nlm4_testargs argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm4_svc_decode_nlm4_testargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm4_testargs *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm4_testargs(xdr, argp); +} + +/** + * nlm4_svc_decode_nlm4_lockargs - Decode a nlm4_lockargs argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm4_svc_decode_nlm4_lockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm4_lockargs *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm4_lockargs(xdr, argp); +} + +/** + * nlm4_svc_decode_nlm4_cancargs - Decode a nlm4_cancargs argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm4_svc_decode_nlm4_cancargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm4_cancargs *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm4_cancargs(xdr, argp); +} + +/** + * nlm4_svc_decode_nlm4_unlockargs - Decode a nlm4_unlockargs argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm4_svc_decode_nlm4_unlockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm4_unlockargs *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm4_unlockargs(xdr, argp); +} + +/** + * nlm4_svc_decode_nlm4_testres - Decode a nlm4_testres argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm4_svc_decode_nlm4_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm4_testres *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm4_testres(xdr, argp); +} + +/** + * nlm4_svc_decode_nlm4_res - Decode a nlm4_res argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm4_svc_decode_nlm4_res(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm4_res *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm4_res(xdr, argp); +} + +/** + * nlm4_svc_decode_nlm4_notifyargs - Decode a nlm4_notifyargs argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm4_svc_decode_nlm4_notifyargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm4_notifyargs *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm4_notifyargs(xdr, argp); +} + +/** + * nlm4_svc_decode_nlm4_shareargs - Decode a nlm4_shareargs argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm4_svc_decode_nlm4_shareargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm4_shareargs *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm4_shareargs(xdr, argp); +} + +/** + * nlm4_svc_decode_nlm4_notify - Decode a nlm4_notify argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +bool nlm4_svc_decode_nlm4_notify(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm4_notify *argp = rqstp->rq_argp; + + return xdrgen_decode_nlm4_notify(xdr, argp); +} + +static bool __maybe_unused +xdrgen_encode_netobj(struct xdr_stream *xdr, const netobj value) +{ + return xdr_stream_encode_opaque(xdr, value.data, value.len) >= 0; +} + +static bool __maybe_unused +xdrgen_encode_fsh4_mode(struct xdr_stream *xdr, fsh4_mode value) +{ + return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; +} + +static bool __maybe_unused +xdrgen_encode_fsh4_access(struct xdr_stream *xdr, fsh4_access value) +{ + return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; +} + +static bool __maybe_unused +xdrgen_encode_uint64(struct xdr_stream *xdr, const uint64 value) +{ + return xdrgen_encode_unsigned_hyper(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_int64(struct xdr_stream *xdr, const int64 value) +{ + return xdrgen_encode_hyper(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_uint32(struct xdr_stream *xdr, const uint32 value) +{ + return xdrgen_encode_unsigned_long(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_int32(struct xdr_stream *xdr, const int32 value) +{ + return xdrgen_encode_long(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_nlm4_stats(struct xdr_stream *xdr, nlm4_stats value) +{ + return xdr_stream_encode_be32(xdr, value) == XDR_UNIT; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_holder(struct xdr_stream *xdr, const struct nlm4_holder *value) +{ + if (!xdrgen_encode_bool(xdr, value->exclusive)) + return false; + if (!xdrgen_encode_int32(xdr, value->svid)) + return false; + if (!xdrgen_encode_netobj(xdr, value->oh)) + return false; + if (!xdrgen_encode_uint64(xdr, value->l_offset)) + return false; + if (!xdrgen_encode_uint64(xdr, value->l_len)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_testrply(struct xdr_stream *xdr, const struct nlm4_testrply *ptr) +{ + if (!xdrgen_encode_nlm4_stats(xdr, ptr->stat)) + return false; + switch (ptr->stat) { + case __constant_cpu_to_be32(NLM4_DENIED): + if (!xdrgen_encode_nlm4_holder(xdr, &ptr->u.holder)) + return false; + break; + default: + break; + } + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_stat(struct xdr_stream *xdr, const struct nlm4_stat *value) +{ + if (!xdrgen_encode_nlm4_stats(xdr, value->stat)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_res(struct xdr_stream *xdr, const struct nlm4_res *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_nlm4_stat(xdr, &value->stat)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_testres(struct xdr_stream *xdr, const struct nlm4_testres *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_nlm4_testrply(xdr, &value->stat)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_lock(struct xdr_stream *xdr, const struct nlm4_lock *value) +{ + if (value->caller_name.len > LM_MAXSTRLEN) + return false; + if (xdr_stream_encode_opaque(xdr, value->caller_name.data, value->caller_name.len) < 0) + return false; + if (!xdrgen_encode_netobj(xdr, value->fh)) + return false; + if (!xdrgen_encode_netobj(xdr, value->oh)) + return false; + if (!xdrgen_encode_int32(xdr, value->svid)) + return false; + if (!xdrgen_encode_uint64(xdr, value->l_offset)) + return false; + if (!xdrgen_encode_uint64(xdr, value->l_len)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_lockargs(struct xdr_stream *xdr, const struct nlm4_lockargs *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_bool(xdr, value->block)) + return false; + if (!xdrgen_encode_bool(xdr, value->exclusive)) + return false; + if (!xdrgen_encode_nlm4_lock(xdr, &value->alock)) + return false; + if (!xdrgen_encode_bool(xdr, value->reclaim)) + return false; + if (!xdrgen_encode_int32(xdr, value->state)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_cancargs(struct xdr_stream *xdr, const struct nlm4_cancargs *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_bool(xdr, value->block)) + return false; + if (!xdrgen_encode_bool(xdr, value->exclusive)) + return false; + if (!xdrgen_encode_nlm4_lock(xdr, &value->alock)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_testargs(struct xdr_stream *xdr, const struct nlm4_testargs *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_bool(xdr, value->exclusive)) + return false; + if (!xdrgen_encode_nlm4_lock(xdr, &value->alock)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_unlockargs(struct xdr_stream *xdr, const struct nlm4_unlockargs *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_nlm4_lock(xdr, &value->alock)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_share(struct xdr_stream *xdr, const struct nlm4_share *value) +{ + if (value->caller_name.len > LM_MAXSTRLEN) + return false; + if (xdr_stream_encode_opaque(xdr, value->caller_name.data, value->caller_name.len) < 0) + return false; + if (!xdrgen_encode_netobj(xdr, value->fh)) + return false; + if (!xdrgen_encode_netobj(xdr, value->oh)) + return false; + if (!xdrgen_encode_fsh4_mode(xdr, value->mode)) + return false; + if (!xdrgen_encode_fsh4_access(xdr, value->access)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_shareargs(struct xdr_stream *xdr, const struct nlm4_shareargs *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_nlm4_share(xdr, &value->share)) + return false; + if (!xdrgen_encode_bool(xdr, value->reclaim)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_shareres(struct xdr_stream *xdr, const struct nlm4_shareres *value) +{ + if (!xdrgen_encode_netobj(xdr, value->cookie)) + return false; + if (!xdrgen_encode_nlm4_stats(xdr, value->stat)) + return false; + if (!xdrgen_encode_int32(xdr, value->sequence)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_notify(struct xdr_stream *xdr, const struct nlm4_notify *value) +{ + if (value->name.len > LM_MAXNAMELEN) + return false; + if (xdr_stream_encode_opaque(xdr, value->name.data, value->name.len) < 0) + return false; + if (!xdrgen_encode_int32(xdr, value->state)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nlm4_notifyargs(struct xdr_stream *xdr, const struct nlm4_notifyargs *value) +{ + if (!xdrgen_encode_nlm4_notify(xdr, &value->notify)) + return false; + if (xdr_stream_encode_opaque_fixed(xdr, value->private, SM_PRIV_SIZE) < 0) + return false; + return true; +} + +/** + * nlm4_svc_encode_nlm4_testres - Encode a nlm4_testres result + * @rqstp: RPC transaction context + * @xdr: target XDR data stream + * + * Return values: + * %true: procedure results encoded successfully + * %false: encode failed + */ +bool nlm4_svc_encode_nlm4_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm4_testres *resp = rqstp->rq_resp; + + return xdrgen_encode_nlm4_testres(xdr, resp); +} + +/** + * nlm4_svc_encode_nlm4_res - Encode a nlm4_res result + * @rqstp: RPC transaction context + * @xdr: target XDR data stream + * + * Return values: + * %true: procedure results encoded successfully + * %false: encode failed + */ +bool nlm4_svc_encode_nlm4_res(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm4_res *resp = rqstp->rq_resp; + + return xdrgen_encode_nlm4_res(xdr, resp); +} + +/** + * nlm4_svc_encode_nlm4_shareres - Encode a nlm4_shareres result + * @rqstp: RPC transaction context + * @xdr: target XDR data stream + * + * Return values: + * %true: procedure results encoded successfully + * %false: encode failed + */ +bool nlm4_svc_encode_nlm4_shareres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nlm4_shareres *resp = rqstp->rq_resp; + + return xdrgen_encode_nlm4_shareres(xdr, resp); +} diff --git a/fs/lockd/nlm4xdr_gen.h b/fs/lockd/nlm4xdr_gen.h new file mode 100644 index 000000000..1a72c26ad --- /dev/null +++ b/fs/lockd/nlm4xdr_gen.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Generated by xdrgen. Manual edits will be lost. */ +/* XDR specification file: ../../Documentation/sunrpc/xdr/nlm4.x */ +/* XDR specification modification time: Mon Jun 29 20:42:29 2026 */ + +#ifndef _LINUX_XDRGEN_NLM4_DECL_H +#define _LINUX_XDRGEN_NLM4_DECL_H + +#include + +#include +#include +#include +#include + +bool nlm4_svc_decode_nlm4_testargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm4_svc_decode_nlm4_lockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm4_svc_decode_nlm4_cancargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm4_svc_decode_nlm4_unlockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm4_svc_decode_nlm4_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm4_svc_decode_nlm4_res(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm4_svc_decode_nlm4_notifyargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm4_svc_decode_nlm4_shareargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm4_svc_decode_nlm4_notify(struct svc_rqst *rqstp, struct xdr_stream *xdr); + +bool nlm4_svc_encode_nlm4_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm4_svc_encode_nlm4_res(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nlm4_svc_encode_nlm4_shareres(struct svc_rqst *rqstp, struct xdr_stream *xdr); + +#endif /* _LINUX_XDRGEN_NLM4_DECL_H */ diff --git a/fs/lockd/procfs.c b/fs/lockd/procfs.c new file mode 100644 index 000000000..a01f08c8c --- /dev/null +++ b/fs/lockd/procfs.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Procfs support for lockd + * + * Copyright (c) 2014 Jeff Layton + */ + +#include +#include +#include +#include +#include + +#include "netns.h" +#include "procfs.h" + +/* + * We only allow strings that start with 'Y', 'y', or '1'. + */ +static ssize_t +nlm_end_grace_write(struct file *file, const char __user *buf, size_t size, + loff_t *pos) +{ + char *data; + struct lockd_net *ln = net_generic(current->nsproxy->net_ns, + lockd_net_id); + + if (size < 1) + return -EINVAL; + + data = simple_transaction_get(file, buf, size); + if (IS_ERR(data)) + return PTR_ERR(data); + + switch(data[0]) { + case 'Y': + case 'y': + case '1': + locks_end_grace(&ln->lockd_manager); + break; + default: + return -EINVAL; + } + + return size; +} + +static ssize_t +nlm_end_grace_read(struct file *file, char __user *buf, size_t size, + loff_t *pos) +{ + struct lockd_net *ln = net_generic(current->nsproxy->net_ns, + lockd_net_id); + char resp[3]; + + resp[0] = list_empty(&ln->lockd_manager.list) ? 'Y' : 'N'; + resp[1] = '\n'; + resp[2] = '\0'; + + return simple_read_from_buffer(buf, size, pos, resp, sizeof(resp)); +} + +static const struct proc_ops lockd_end_grace_proc_ops = { + .proc_write = nlm_end_grace_write, + .proc_read = nlm_end_grace_read, + .proc_lseek = default_llseek, + .proc_release = simple_transaction_release, +}; + +int __init +lockd_create_procfs(void) +{ + struct proc_dir_entry *entry; + + entry = proc_mkdir("fs/lockd", NULL); + if (!entry) + return -ENOMEM; + entry = proc_create("nlm_end_grace", S_IRUGO|S_IWUSR, entry, + &lockd_end_grace_proc_ops); + if (!entry) { + remove_proc_entry("fs/lockd", NULL); + return -ENOMEM; + } + return 0; +} + +void __exit +lockd_remove_procfs(void) +{ + remove_proc_entry("fs/lockd/nlm_end_grace", NULL); + remove_proc_entry("fs/lockd", NULL); +} diff --git a/fs/lockd/procfs.h b/fs/lockd/procfs.h new file mode 100644 index 000000000..ba9a82f4c --- /dev/null +++ b/fs/lockd/procfs.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Procfs support for lockd + * + * Copyright (c) 2014 Jeff Layton + */ +#ifndef _LOCKD_PROCFS_H +#define _LOCKD_PROCFS_H + +#if IS_ENABLED(CONFIG_PROC_FS) +int lockd_create_procfs(void); +void lockd_remove_procfs(void); +#else +static inline int +lockd_create_procfs(void) +{ + return 0; +} + +static inline void +lockd_remove_procfs(void) +{ + return; +} +#endif /* IS_ENABLED(CONFIG_PROC_FS) */ + +#endif /* _LOCKD_PROCFS_H */ diff --git a/fs/lockd/share.h b/fs/lockd/share.h new file mode 100644 index 000000000..a12b6c454 --- /dev/null +++ b/fs/lockd/share.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * DOS share management for lockd. + * + * Copyright (C) 1996, Olaf Kirch + */ + +#ifndef _LOCKD_SHARE_H +#define _LOCKD_SHARE_H + +#include + +/* Synthetic svid for lockowner lookup during share operations */ +#define LOCKD_SHARE_SVID (~(u32)0) + +/* One bit per (access, deny) pair; index = (access << 2) | deny */ +#define LOCKD_FSH_BIT(a, d) BIT(((a) << 2) | (d)) + +/* + * DOS share for a specific file + */ +struct lockd_share { + struct lockd_share * s_next; /* linked list */ + struct nlm_host * s_host; /* client host */ + struct nlm_file * s_file; /* shared file */ + struct xdr_netobj s_owner; /* owner handle */ + u32 s_access; /* access mode */ + u32 s_mode; /* deny mode */ + u16 s_access_deny_bmap; /* held (access, deny) pairs */ +}; + +__be32 nlmsvc_share_file(struct nlm_host *host, struct nlm_file *file, + struct xdr_netobj *oh, u32 access, u32 mode); +__be32 nlmsvc_unshare_file(struct nlm_host *host, struct nlm_file *file, + struct xdr_netobj *oh, u32 access, u32 mode); +void nlmsvc_traverse_shares(struct nlm_host *, struct nlm_file *, + nlm_host_match_fn_t); + +#endif /* _LOCKD_SHARE_H */ diff --git a/fs/lockd/svc.c b/fs/lockd/svc.c new file mode 100644 index 000000000..ee90e7430 --- /dev/null +++ b/fs/lockd/svc.c @@ -0,0 +1,791 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/lockd/svc.c + * + * This is the central lockd service. + * + * FIXME: Separate the lockd NFS server functionality from the lockd NFS + * client functionality. Oh why didn't Sun create two separate + * services in the first place? + * + * Authors: Olaf Kirch (okir@monad.swb.de) + * + * Copyright (C) 1995, 1996 Olaf Kirch + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "lockd.h" +#include "netns.h" +#include "procfs.h" +#include "netlink.h" + +#define NLMDBG_FACILITY NLMDBG_SVC + +static struct svc_program nlmsvc_program; + +const struct nlmsvc_binding __rcu *nlmsvc_ops; +EXPORT_SYMBOL_GPL(nlmsvc_ops); + +static DEFINE_MUTEX(nlmsvc_mutex); +static unsigned int nlmsvc_users; +static struct svc_serv *nlmsvc_serv; + +static void nlmsvc_request_retry(struct timer_list *tl) +{ + svc_wake_up(nlmsvc_serv); +} +DEFINE_TIMER(nlmsvc_retry, nlmsvc_request_retry); + +unsigned int lockd_net_id; + +/* + * These can be set at insmod time (useful for NFS as root filesystem), + * and also changed through the sysctl interface. -- Jamie Lokier, Aug 2003 + */ +static unsigned long nlm_grace_period; +unsigned long nlm_timeout = LOCKD_DFLT_TIMEO; +static int nlm_udpport, nlm_tcpport; + +/* + * Constants needed for the sysctl interface. + */ +static const unsigned long nlm_grace_period_min = 0; +static const unsigned long nlm_grace_period_max = 240; +static const unsigned long nlm_timeout_min = 3; +static const unsigned long nlm_timeout_max = 20; + +#ifdef CONFIG_SYSCTL +static const int nlm_port_min = 0, nlm_port_max = 65535; +static struct ctl_table_header * nlm_sysctl_table; +#endif + +static unsigned long get_lockd_grace_period(struct net *net) +{ + struct lockd_net *ln = net_generic(net, lockd_net_id); + + /* Return the net-ns specific grace period, if there is one */ + if (ln->gracetime) + return ln->gracetime * HZ; + + /* Note: nlm_timeout should always be nonzero */ + if (nlm_grace_period) + return roundup(nlm_grace_period, nlm_timeout) * HZ; + else + return nlm_timeout * 5 * HZ; +} + +static void grace_ender(struct work_struct *grace) +{ + struct delayed_work *dwork = to_delayed_work(grace); + struct lockd_net *ln = container_of(dwork, struct lockd_net, + grace_period_end); + + locks_end_grace(&ln->lockd_manager); +} + +static void set_grace_period(struct net *net) +{ + unsigned long grace_period = get_lockd_grace_period(net); + struct lockd_net *ln = net_generic(net, lockd_net_id); + + locks_start_grace(net, &ln->lockd_manager); + cancel_delayed_work_sync(&ln->grace_period_end); + schedule_delayed_work(&ln->grace_period_end, grace_period); +} + +/* + * This is the lockd kernel thread + */ +static int +lockd(void *vrqstp) +{ + struct svc_rqst *rqstp = vrqstp; + struct net *net = &init_net; + struct lockd_net *ln = net_generic(net, lockd_net_id); + + svc_thread_init_status(rqstp, 0); + + /* try_to_freeze() is called from svc_recv() */ + set_freezable(); + + dprintk("NFS locking service started (ver " LOCKD_VERSION ").\n"); + + /* + * The main request loop. We don't terminate until the last + * NFS mount or NFS daemon has gone away. + */ + while (!svc_thread_should_stop(rqstp)) { + nlmsvc_retry_blocked(rqstp); + svc_recv(rqstp, 0); + } + if (rcu_access_pointer(nlmsvc_ops)) + nlmsvc_invalidate_all(); + nlm_shutdown_hosts(); + cancel_delayed_work_sync(&ln->grace_period_end); + locks_end_grace(&ln->lockd_manager); + + dprintk("lockd_down: service stopped\n"); + + svc_exit_thread(rqstp); + return 0; +} + +static int create_lockd_listener(struct svc_serv *serv, const char *name, + struct net *net, const int family, + const unsigned short port, + const struct cred *cred) +{ + struct svc_xprt *xprt; + + xprt = svc_find_xprt(serv, name, net, family, 0); + if (xprt == NULL) + return svc_xprt_create(serv, name, net, family, port, + SVC_SOCK_DEFAULTS, cred); + svc_xprt_put(xprt); + return 0; +} + +static int create_lockd_family(struct svc_serv *serv, struct net *net, + const int family, const struct cred *cred) +{ + struct lockd_net *ln = net_generic(net, lockd_net_id); + int err; + + err = create_lockd_listener(serv, "udp", net, family, + ln->udp_port ? ln->udp_port : nlm_udpport, cred); + if (err < 0) + return err; + + return create_lockd_listener(serv, "tcp", net, family, + ln->tcp_port ? ln->tcp_port : nlm_tcpport, cred); +} + +/* + * Ensure there are active UDP and TCP listeners for lockd. + * + * Even if we have only TCP NFS mounts and/or TCP NFSDs, some + * local services (such as rpc.statd) still require UDP, and + * some NFS servers do not yet support NLM over TCP. + * + * Returns zero if all listeners are available; otherwise a + * negative errno value is returned. + */ +static int make_socks(struct svc_serv *serv, struct net *net, + const struct cred *cred) +{ + static int warned; + int err; + + err = create_lockd_family(serv, net, PF_INET, cred); + if (err < 0) + goto out_err; + + err = create_lockd_family(serv, net, PF_INET6, cred); + if (err < 0 && err != -EAFNOSUPPORT) + goto out_err; + + warned = 0; + return 0; + +out_err: + if (warned++ == 0) + printk(KERN_WARNING + "lockd_up: makesock failed, error=%d\n", err); + svc_xprt_destroy_all(serv, net, true); + return err; +} + +static int lockd_up_net(struct svc_serv *serv, struct net *net, + const struct cred *cred) +{ + struct lockd_net *ln = net_generic(net, lockd_net_id); + int error; + + if (ln->nlmsvc_users++) + return 0; + + error = svc_bind(serv, net); + if (error) + goto err_bind; + + error = make_socks(serv, net, cred); + if (error < 0) + goto err_bind; + set_grace_period(net); + dprintk("%s: per-net data created; net=%x\n", __func__, net->ns.inum); + return 0; + +err_bind: + ln->nlmsvc_users--; + return error; +} + +static void lockd_down_net(struct svc_serv *serv, struct net *net) +{ + struct lockd_net *ln = net_generic(net, lockd_net_id); + + if (ln->nlmsvc_users) { + if (--ln->nlmsvc_users == 0) { + nlm_shutdown_hosts_net(net); + cancel_delayed_work_sync(&ln->grace_period_end); + locks_end_grace(&ln->lockd_manager); + svc_xprt_destroy_all(serv, net, true); + } + } else { + pr_err("%s: no users! net=%x\n", + __func__, net->ns.inum); + BUG(); + } +} + +static int lockd_inetaddr_event(struct notifier_block *this, + unsigned long event, void *ptr) +{ + struct in_ifaddr *ifa = (struct in_ifaddr *)ptr; + struct sockaddr_in sin; + + if (event != NETDEV_DOWN) + goto out; + + if (nlmsvc_serv) { + dprintk("lockd_inetaddr_event: removed %pI4\n", + &ifa->ifa_local); + sin.sin_family = AF_INET; + sin.sin_addr.s_addr = ifa->ifa_local; + svc_age_temp_xprts_now(nlmsvc_serv, (struct sockaddr *)&sin); + } + +out: + return NOTIFY_DONE; +} + +static struct notifier_block lockd_inetaddr_notifier = { + .notifier_call = lockd_inetaddr_event, +}; + +#if IS_ENABLED(CONFIG_IPV6) +static int lockd_inet6addr_event(struct notifier_block *this, + unsigned long event, void *ptr) +{ + struct inet6_ifaddr *ifa = (struct inet6_ifaddr *)ptr; + struct sockaddr_in6 sin6; + + if (event != NETDEV_DOWN) + goto out; + + if (nlmsvc_serv) { + dprintk("lockd_inet6addr_event: removed %pI6\n", &ifa->addr); + sin6.sin6_family = AF_INET6; + sin6.sin6_addr = ifa->addr; + if (ipv6_addr_type(&sin6.sin6_addr) & IPV6_ADDR_LINKLOCAL) + sin6.sin6_scope_id = ifa->idev->dev->ifindex; + svc_age_temp_xprts_now(nlmsvc_serv, (struct sockaddr *)&sin6); + } + +out: + return NOTIFY_DONE; +} + +static struct notifier_block lockd_inet6addr_notifier = { + .notifier_call = lockd_inet6addr_event, +}; +#endif + +static int lockd_get(void) +{ + struct svc_serv *serv; + unsigned int bufsize; + int error; + + if (nlmsvc_serv) { + nlmsvc_users++; + return 0; + } + + /* + * Sanity check: if there's no pid, + * we should be the first user ... + */ + if (nlmsvc_users) + printk(KERN_WARNING + "lockd_up: no pid, %d users??\n", nlmsvc_users); + +#ifdef CONFIG_LOCKD_V4 + bufsize = 1024 + max3(nlmsvc_version1.vs_xdrsize, + nlmsvc_version3.vs_xdrsize, + nlmsvc_version4.vs_xdrsize); +#else + bufsize = 1024 + max(nlmsvc_version1.vs_xdrsize, + nlmsvc_version3.vs_xdrsize); +#endif + serv = svc_create(&nlmsvc_program, bufsize, lockd); + if (!serv) { + printk(KERN_WARNING "lockd_up: create service failed\n"); + return -ENOMEM; + } + + error = svc_set_num_threads(serv, 0, 1); + if (error < 0) { + svc_destroy(&serv); + return error; + } + + nlmsvc_serv = serv; + register_inetaddr_notifier(&lockd_inetaddr_notifier); +#if IS_ENABLED(CONFIG_IPV6) + register_inet6addr_notifier(&lockd_inet6addr_notifier); +#endif + dprintk("lockd_up: service created\n"); + nlmsvc_users++; + return 0; +} + +static void lockd_put(void) +{ + if (WARN(nlmsvc_users <= 0, "lockd_down: no users!\n")) + return; + if (--nlmsvc_users) + return; + + unregister_inetaddr_notifier(&lockd_inetaddr_notifier); +#if IS_ENABLED(CONFIG_IPV6) + unregister_inet6addr_notifier(&lockd_inet6addr_notifier); +#endif + + svc_set_num_threads(nlmsvc_serv, 0, 0); + timer_delete_sync(&nlmsvc_retry); + svc_destroy(&nlmsvc_serv); + dprintk("lockd_down: service destroyed\n"); +} + +/* + * Bring up the lockd process if it's not already up. + */ +int lockd_up(struct net *net, const struct cred *cred) +{ + int error; + + mutex_lock(&nlmsvc_mutex); + + error = lockd_get(); + if (error) + goto err; + + error = lockd_up_net(nlmsvc_serv, net, cred); + if (error < 0) { + lockd_put(); + goto err; + } + +err: + mutex_unlock(&nlmsvc_mutex); + return error; +} +EXPORT_SYMBOL_GPL(lockd_up); + +/* + * Decrement the user count and bring down lockd if we're the last. + */ +void +lockd_down(struct net *net) +{ + mutex_lock(&nlmsvc_mutex); + lockd_down_net(nlmsvc_serv, net); + lockd_put(); + mutex_unlock(&nlmsvc_mutex); +} +EXPORT_SYMBOL_GPL(lockd_down); + +#ifdef CONFIG_SYSCTL + +/* + * Sysctl parameters (same as module parameters, different interface). + */ + +static const struct ctl_table nlm_sysctls[] = { + { + .procname = "nlm_grace_period", + .data = &nlm_grace_period, + .maxlen = sizeof(unsigned long), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + .extra1 = (unsigned long *) &nlm_grace_period_min, + .extra2 = (unsigned long *) &nlm_grace_period_max, + }, + { + .procname = "nlm_timeout", + .data = &nlm_timeout, + .maxlen = sizeof(unsigned long), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + .extra1 = (unsigned long *) &nlm_timeout_min, + .extra2 = (unsigned long *) &nlm_timeout_max, + }, + { + .procname = "nlm_udpport", + .data = &nlm_udpport, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = (int *) &nlm_port_min, + .extra2 = (int *) &nlm_port_max, + }, + { + .procname = "nlm_tcpport", + .data = &nlm_tcpport, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = (int *) &nlm_port_min, + .extra2 = (int *) &nlm_port_max, + }, + { + .procname = "nsm_use_hostnames", + .data = &nsm_use_hostnames, + .maxlen = sizeof(bool), + .mode = 0644, + .proc_handler = proc_dobool, + }, + { + .procname = "nsm_local_state", + .data = &nsm_local_state, + .maxlen = sizeof(nsm_local_state), + .mode = 0644, + .proc_handler = proc_douintvec, + .extra1 = SYSCTL_ZERO, + }, +}; + +#endif /* CONFIG_SYSCTL */ + +/* + * Module (and sysfs) parameters. + */ + +#define param_set_min_max(name, type, which_strtol, min, max) \ +static int param_set_##name(const char *val, const struct kernel_param *kp) \ +{ \ + char *endp; \ + __typeof__(type) num = which_strtol(val, &endp, 0); \ + if (endp == val || *endp || num < (min) || num > (max)) \ + return -EINVAL; \ + *((type *) kp->arg) = num; \ + return 0; \ +} + +static inline int is_callback(u32 proc) +{ + return proc == NLMPROC_GRANTED + || proc == NLMPROC_GRANTED_MSG + || proc == NLMPROC_TEST_RES + || proc == NLMPROC_LOCK_RES + || proc == NLMPROC_CANCEL_RES + || proc == NLMPROC_UNLOCK_RES + || proc == NLMPROC_NSM_NOTIFY; +} + + +static enum svc_auth_status lockd_authenticate(struct svc_rqst *rqstp) +{ + rqstp->rq_client = NULL; + switch (rqstp->rq_authop->flavour) { + case RPC_AUTH_NULL: + case RPC_AUTH_UNIX: + rqstp->rq_auth_stat = rpc_auth_ok; + if (rqstp->rq_proc == 0) + return SVC_OK; + if (is_callback(rqstp->rq_proc)) { + /* Leave it to individual procedures to + * call nlmsvc_lookup_host(rqstp) + */ + return SVC_OK; + } + return svc_set_client(rqstp); + } + rqstp->rq_auth_stat = rpc_autherr_badcred; + return SVC_DENIED; +} + + +param_set_min_max(port, int, simple_strtol, 0, 65535) +param_set_min_max(grace_period, unsigned long, simple_strtoul, + nlm_grace_period_min, nlm_grace_period_max) +param_set_min_max(timeout, unsigned long, simple_strtoul, + nlm_timeout_min, nlm_timeout_max) + +MODULE_AUTHOR("Olaf Kirch "); +MODULE_DESCRIPTION("NFS file locking service version " LOCKD_VERSION "."); +MODULE_LICENSE("GPL"); + +module_param_call(nlm_grace_period, param_set_grace_period, param_get_ulong, + &nlm_grace_period, 0644); +module_param_call(nlm_timeout, param_set_timeout, param_get_ulong, + &nlm_timeout, 0644); +module_param_call(nlm_udpport, param_set_port, param_get_int, + &nlm_udpport, 0644); +module_param_call(nlm_tcpport, param_set_port, param_get_int, + &nlm_tcpport, 0644); +module_param(nsm_use_hostnames, bool, 0644); + +static int lockd_init_net(struct net *net) +{ + struct lockd_net *ln = net_generic(net, lockd_net_id); + + INIT_DELAYED_WORK(&ln->grace_period_end, grace_ender); + INIT_LIST_HEAD(&ln->lockd_manager.list); + ln->lockd_manager.block_opens = false; + INIT_LIST_HEAD(&ln->nsm_handles); + return 0; +} + +static void lockd_exit_net(struct net *net) +{ + struct lockd_net *ln = net_generic(net, lockd_net_id); + + WARN_ONCE(!list_empty(&ln->lockd_manager.list), + "net %x %s: lockd_manager.list is not empty\n", + net->ns.inum, __func__); + WARN_ONCE(!list_empty(&ln->nsm_handles), + "net %x %s: nsm_handles list is not empty\n", + net->ns.inum, __func__); + WARN_ONCE(delayed_work_pending(&ln->grace_period_end), + "net %x %s: grace_period_end was not cancelled\n", + net->ns.inum, __func__); +} + +static struct pernet_operations lockd_net_ops = { + .init = lockd_init_net, + .exit = lockd_exit_net, + .id = &lockd_net_id, + .size = sizeof(struct lockd_net), +}; + + +/* + * Initialising and terminating the module. + */ + +static int __init init_nlm(void) +{ + int err; + +#ifdef CONFIG_SYSCTL + err = -ENOMEM; + nlm_sysctl_table = register_sysctl("fs/nfs", nlm_sysctls); + if (nlm_sysctl_table == NULL) + goto err_sysctl; +#endif + err = register_pernet_subsys(&lockd_net_ops); + if (err) + goto err_pernet; + + err = genl_register_family(&lockd_nl_family); + if (err) + goto err_netlink; + + err = lockd_create_procfs(); + if (err) + goto err_procfs; + + return 0; + +err_procfs: + genl_unregister_family(&lockd_nl_family); +err_netlink: + unregister_pernet_subsys(&lockd_net_ops); +err_pernet: +#ifdef CONFIG_SYSCTL + unregister_sysctl_table(nlm_sysctl_table); +err_sysctl: +#endif + return err; +} + +static void __exit exit_nlm(void) +{ + /* FIXME: delete all NLM clients */ + nlm_shutdown_hosts(); + genl_unregister_family(&lockd_nl_family); + lockd_remove_procfs(); + unregister_pernet_subsys(&lockd_net_ops); +#ifdef CONFIG_SYSCTL + unregister_sysctl_table(nlm_sysctl_table); +#endif +} + +module_init(init_nlm); +module_exit(exit_nlm); + +/** + * nlmsvc_dispatch - Process an NLM Request + * @rqstp: incoming request + * + * Return values: + * %0: Processing complete; do not send a Reply + * %1: Processing complete; send Reply in rqstp->rq_res + */ +int nlmsvc_dispatch(struct svc_rqst *rqstp) +{ + const struct svc_procedure *procp = rqstp->rq_procinfo; + __be32 *statp = rqstp->rq_accept_statp; + + if (!procp->pc_decode(rqstp, &rqstp->rq_arg_stream)) + goto out_decode_err; + + *statp = procp->pc_func(rqstp); + if (*statp == rpc_drop_reply) + return 0; + if (*statp != rpc_success) + return 1; + + if (!procp->pc_encode(rqstp, &rqstp->rq_res_stream)) + goto out_encode_err; + + return 1; + +out_decode_err: + *statp = rpc_garbage_args; + return 1; + +out_encode_err: + *statp = rpc_system_err; + return 1; +} + +/* + * Define NLM program and procedures + */ +static const struct svc_version *nlmsvc_version[] = { + [1] = &nlmsvc_version1, + [3] = &nlmsvc_version3, +#ifdef CONFIG_LOCKD_V4 + [4] = &nlmsvc_version4, +#endif +}; + +#define NLM_NRVERS ARRAY_SIZE(nlmsvc_version) +static struct svc_program nlmsvc_program = { + .pg_prog = NLM_PROGRAM, /* program number */ + .pg_nvers = NLM_NRVERS, /* number of entries in nlmsvc_version */ + .pg_vers = nlmsvc_version, /* version table */ + .pg_name = "lockd", /* service name */ + .pg_class = "nfsd", /* share authentication with nfsd */ + .pg_authenticate = &lockd_authenticate, /* export authentication */ + .pg_init_request = svc_generic_init_request, + .pg_rpcbind_set = svc_generic_rpcbind_set, +}; + +/** + * lockd_nl_server_set_doit - set the lockd server parameters via netlink + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * This updates the per-net values. When updating the values in the init_net + * namespace, also update the "legacy" global values. + * + * Return 0 on success or a negative errno. + */ +int lockd_nl_server_set_doit(struct sk_buff *skb, struct genl_info *info) +{ + struct net *net = genl_info_net(info); + struct lockd_net *ln = net_generic(net, lockd_net_id); + const struct nlattr *attr; + + if (GENL_REQ_ATTR_CHECK(info, LOCKD_A_SERVER_GRACETIME)) + return -EINVAL; + + if (info->attrs[LOCKD_A_SERVER_GRACETIME] || + info->attrs[LOCKD_A_SERVER_TCP_PORT] || + info->attrs[LOCKD_A_SERVER_UDP_PORT]) { + attr = info->attrs[LOCKD_A_SERVER_GRACETIME]; + if (attr) { + u32 gracetime = nla_get_u32(attr); + + if (gracetime > nlm_grace_period_max) + return -EINVAL; + + ln->gracetime = gracetime; + + if (net == &init_net) + nlm_grace_period = gracetime; + } + + attr = info->attrs[LOCKD_A_SERVER_TCP_PORT]; + if (attr) { + ln->tcp_port = nla_get_u16(attr); + if (net == &init_net) + nlm_tcpport = ln->tcp_port; + } + + attr = info->attrs[LOCKD_A_SERVER_UDP_PORT]; + if (attr) { + ln->udp_port = nla_get_u16(attr); + if (net == &init_net) + nlm_udpport = ln->udp_port; + } + } + return 0; +} + +/** + * lockd_nl_server_get_doit - get lockd server parameters via netlink + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Return 0 on success or a negative errno. + */ +int lockd_nl_server_get_doit(struct sk_buff *skb, struct genl_info *info) +{ + struct net *net = genl_info_net(info); + struct lockd_net *ln = net_generic(net, lockd_net_id); + void *hdr; + int err; + + skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); + if (!skb) + return -ENOMEM; + + hdr = genlmsg_iput(skb, info); + if (!hdr) { + err = -EMSGSIZE; + goto err_free_msg; + } + + err = nla_put_u32(skb, LOCKD_A_SERVER_GRACETIME, ln->gracetime) || + nla_put_u16(skb, LOCKD_A_SERVER_TCP_PORT, ln->tcp_port) || + nla_put_u16(skb, LOCKD_A_SERVER_UDP_PORT, ln->udp_port); + if (err) + goto err_free_msg; + + genlmsg_end(skb, hdr); + + return genlmsg_reply(skb, info); +err_free_msg: + nlmsg_free(skb); + + return err; +} diff --git a/fs/lockd/svc4proc.c b/fs/lockd/svc4proc.c new file mode 100644 index 000000000..03c5554ca --- /dev/null +++ b/fs/lockd/svc4proc.c @@ -0,0 +1,1432 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/lockd/svc4proc.c + * + * Lockd server procedures. We don't implement the NLM_*_RES + * procedures because we don't use the async procedures. + * + * Copyright (C) 1996, Olaf Kirch + */ + +#include +#include +#include + +#include "lockd.h" + +/* + * xdr.h defines SM_MAXSTRLEN and SM_PRIV_SIZE as macros. + * nlm4xdr_gen.h defines them as enum constants. Undefine the + * macros to allow the xdrgen enum definitions to be used. + */ +#undef SM_MAXSTRLEN +#undef SM_PRIV_SIZE + +#include "share.h" +#include "nlm4xdr_gen.h" + +/* + * Wrapper structures combine xdrgen types with legacy lockd_lock. + * The xdrgen field must be first so the structure can be cast + * to its XDR type for the RPC dispatch layer. + */ +struct nlm4_testargs_wrapper { + struct nlm4_testargs xdrgen; + struct lockd_lock lock; +}; + +static_assert(offsetof(struct nlm4_testargs_wrapper, xdrgen) == 0); + +struct nlm4_lockargs_wrapper { + struct nlm4_lockargs xdrgen; + struct lockd_cookie cookie; + struct lockd_lock lock; +}; + +static_assert(offsetof(struct nlm4_lockargs_wrapper, xdrgen) == 0); + +struct nlm4_cancargs_wrapper { + struct nlm4_cancargs xdrgen; + struct lockd_lock lock; +}; + +static_assert(offsetof(struct nlm4_cancargs_wrapper, xdrgen) == 0); + +struct nlm4_unlockargs_wrapper { + struct nlm4_unlockargs xdrgen; + struct lockd_lock lock; +}; + +static_assert(offsetof(struct nlm4_unlockargs_wrapper, xdrgen) == 0); + +struct nlm4_notifyargs_wrapper { + struct nlm4_notifyargs xdrgen; + struct lockd_reboot reboot; +}; + +static_assert(offsetof(struct nlm4_notifyargs_wrapper, xdrgen) == 0); + +struct nlm4_notify_wrapper { + struct nlm4_notify xdrgen; +}; + +static_assert(offsetof(struct nlm4_notify_wrapper, xdrgen) == 0); + +struct nlm4_testres_wrapper { + struct nlm4_testres xdrgen; + struct lockd_lock lock; +}; + +struct nlm4_shareargs_wrapper { + struct nlm4_shareargs xdrgen; + struct lockd_lock lock; +}; + +static_assert(offsetof(struct nlm4_shareargs_wrapper, xdrgen) == 0); + +static_assert(offsetof(struct nlm4_testres_wrapper, xdrgen) == 0); + +struct nlm4_res_wrapper { + struct nlm4_res xdrgen; + struct lockd_cookie cookie; +}; + +static_assert(offsetof(struct nlm4_res_wrapper, xdrgen) == 0); + +struct nlm4_shareres_wrapper { + struct nlm4_shareres xdrgen; +}; + +static_assert(offsetof(struct nlm4_shareres_wrapper, xdrgen) == 0); + +static __be32 +nlm4_netobj_to_cookie(struct lockd_cookie *cookie, netobj *object) +{ + if (object->len > NLM_MAXCOOKIELEN) + return nlm_lck_denied_nolocks; + cookie->len = object->len; + memcpy(cookie->data, object->data, object->len); + return nlm_granted; +} + +static __be32 +nlm4_lock_to_lockd_lock(struct lockd_lock *lock, struct nlm4_lock *alock) +{ + if (alock->fh.len > NFS_MAXFHSIZE) + return nlm_lck_denied; + lock->fh.size = alock->fh.len; + memcpy(lock->fh.data, alock->fh.data, alock->fh.len); + lock->oh.len = alock->oh.len; + lock->oh.data = alock->oh.data; + lock->svid = alock->svid; + lockd_set_file_lock_range4(&lock->fl, alock->l_offset, alock->l_len); + return nlm_granted; +} + +static struct nlm_host * +nlm4svc_lookup_host(struct svc_rqst *rqstp, string caller, bool monitored) +{ + struct nlm_host *host; + + if (!rcu_access_pointer(nlmsvc_ops)) + return NULL; + host = nlmsvc_lookup_host(rqstp, caller.data, caller.len); + if (!host) + return NULL; + if (monitored && nsm_monitor(host) < 0) { + nlmsvc_release_host(host); + return NULL; + } + return host; +} + +static __be32 +nlm4svc_lookup_file(struct svc_rqst *rqstp, struct nlm_host *host, + struct lockd_lock *lock, struct nlm_file **filp, + struct nlm4_lock *xdr_lock, unsigned char type) +{ + bool is_test = (rqstp->rq_proc == NLMPROC4_TEST || + rqstp->rq_proc == NLMPROC4_TEST_MSG); + struct file_lock *fl = &lock->fl; + struct nlm_file *file = NULL; + int mode; + __be32 error; + + if (xdr_lock->fh.len > NFS_MAXFHSIZE) + return nlm_lck_denied_nolocks; + lock->fh.size = xdr_lock->fh.len; + memcpy(lock->fh.data, xdr_lock->fh.data, xdr_lock->fh.len); + if (xdr_lock->fh.len < LOCKD_FH_HASH_SIZE) + memset(lock->fh.data + xdr_lock->fh.len, 0, + LOCKD_FH_HASH_SIZE - xdr_lock->fh.len); + + lock->oh.len = xdr_lock->oh.len; + lock->oh.data = xdr_lock->oh.data; + + lock->svid = xdr_lock->svid; + lock->lock_start = xdr_lock->l_offset; + lock->lock_len = xdr_lock->l_len; + + if (lock->lock_start > OFFSET_MAX || + (lock->lock_len && ((lock->lock_len - 1) > (OFFSET_MAX - lock->lock_start)))) + return nlm4_fbig; + + locks_init_lock(fl); + fl->c.flc_type = type; + lockd_set_file_lock_range4(fl, lock->lock_start, lock->lock_len); + + mode = is_test ? O_RDWR : lock_to_openmode(fl); + error = nlm_lookup_file(rqstp, &file, lock, mode); + switch (error) { + case nlm_granted: + break; + case nlm__int__stale_fh: + return nlm4_stale_fh; + case nlm__int__failed: + return nlm4_failed; + default: + return error; + } + *filp = file; + + fl->c.flc_flags = FL_POSIX; + fl->c.flc_file = is_test ? nlmsvc_file_file(file) + : file->f_file[mode]; + fl->c.flc_pid = current->tgid; + fl->fl_lmops = &nlmsvc_lock_operations; + nlmsvc_locks_init_private(fl, host, (pid_t)lock->svid); + if (!fl->c.flc_owner) + return nlm_lck_denied_nolocks; + + return nlm_granted; +} + +/** + * nlm4svc_proc_null - NULL: Test for presence of service + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully + * + * RPC synopsis: + * void NLMPROC4_NULL(void) = 0; + */ +static __be32 +nlm4svc_proc_null(struct svc_rqst *rqstp) +{ + return rpc_success; +} + +/** + * nlm4svc_proc_test - TEST: Check for conflicting lock + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * + * RPC synopsis: + * nlm4_testres NLMPROC4_TEST(nlm4_testargs) = 1; + * + * Permissible procedure status codes: + * %NLM4_GRANTED: The server would be able to grant the + * requested lock. + * %NLM4_DENIED: The requested lock conflicted with existing + * lock reservations for the file. + * %NLM4_DENIED_NOLOCKS: The server could not allocate the resources + * needed to process the request. + * %NLM4_DENIED_GRACE_PERIOD: The server has recently restarted and is + * re-establishing existing locks, and is not + * yet ready to accept normal service requests. + * + * The Linux NLM server implementation also returns: + * %NLM4_STALE_FH: The request specified an invalid file handle. + * %NLM4_FBIG: The request specified a length or offset + * that exceeds the range supported by the + * server. + * %NLM4_FAILED: The request failed for an unspecified reason. + */ +static __be32 nlm4svc_proc_test(struct svc_rqst *rqstp) +{ + struct nlm4_testargs_wrapper *argp = rqstp->rq_argp; + unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK; + struct nlm4_testres_wrapper *resp = rqstp->rq_resp; + struct nlm_file *file = NULL; + struct nlm_host *host; + + resp->xdrgen.cookie = argp->xdrgen.cookie; + + resp->xdrgen.stat.stat = nlm_lck_denied_nolocks; + host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + goto out; + + resp->xdrgen.stat.stat = nlm4svc_lookup_file(rqstp, host, &argp->lock, + &file, &argp->xdrgen.alock, + type); + if (resp->xdrgen.stat.stat) + goto out; + + resp->xdrgen.stat.stat = nlmsvc_testlock(rqstp, file, host, + &argp->lock, &resp->lock); + nlmsvc_release_lockowner(&argp->lock); + + if (resp->xdrgen.stat.stat == nlm_lck_denied) { + struct lockd_lock *conf = &resp->lock; + struct nlm4_holder *holder = &resp->xdrgen.stat.u.holder; + + holder->exclusive = (conf->fl.c.flc_type != F_RDLCK); + holder->svid = conf->svid; + holder->oh.len = conf->oh.len; + holder->oh.data = conf->oh.data; + holder->l_offset = conf->fl.fl_start; + if (conf->fl.fl_end == OFFSET_MAX) + holder->l_len = 0; + else + holder->l_len = conf->fl.fl_end - conf->fl.fl_start + 1; + } + +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->xdrgen.stat.stat == nlm__int__drop_reply ? + rpc_drop_reply : rpc_success; +} + +static __be32 +nlm4svc_do_lock(struct svc_rqst *rqstp, bool monitored) +{ + struct nlm4_lockargs_wrapper *argp = rqstp->rq_argp; + unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK; + struct nlm4_res_wrapper *resp = rqstp->rq_resp; + struct nlm_file *file = NULL; + struct nlm_host *host = NULL; + + resp->xdrgen.cookie = argp->xdrgen.cookie; + + resp->xdrgen.stat.stat = nlm4_netobj_to_cookie(&argp->cookie, + &argp->xdrgen.cookie); + if (resp->xdrgen.stat.stat) + goto out; + + resp->xdrgen.stat.stat = nlm_lck_denied_nolocks; + host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, + monitored); + if (!host) + goto out; + + resp->xdrgen.stat.stat = nlm4svc_lookup_file(rqstp, host, &argp->lock, + &file, &argp->xdrgen.alock, + type); + if (resp->xdrgen.stat.stat) + goto out; + + resp->xdrgen.stat.stat = nlmsvc_lock(rqstp, file, host, &argp->lock, + argp->xdrgen.block, &argp->cookie, + argp->xdrgen.reclaim); + if (resp->xdrgen.stat.stat == nlm__int__deadlock) + resp->xdrgen.stat.stat = nlm4_deadlock; + + nlmsvc_release_lockowner(&argp->lock); + +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->xdrgen.stat.stat == nlm__int__drop_reply ? + rpc_drop_reply : rpc_success; +} + +/** + * nlm4svc_proc_lock - LOCK: Establish a monitored lock + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * + * RPC synopsis: + * nlm4_res NLMPROC4_LOCK(nlm4_lockargs) = 2; + * + * Permissible procedure status codes: + * %NLM4_GRANTED: The requested lock was granted. + * %NLM4_DENIED: The requested lock conflicted with existing + * lock reservations for the file. + * %NLM4_DENIED_NOLOCKS: The server could not allocate the resources + * needed to process the request. + * %NLM4_BLOCKED: The blocking request cannot be granted + * immediately. The server will send an + * NLMPROC4_GRANTED callback to the client when + * the lock can be granted. + * %NLM4_DENIED_GRACE_PERIOD: The server has recently restarted and is + * re-establishing existing locks, and is not + * yet ready to accept normal service requests. + * + * The Linux NLM server implementation also returns: + * %NLM4_DEADLCK: The request could not be granted and + * blocking would cause a deadlock. + * %NLM4_STALE_FH: The request specified an invalid file handle. + * %NLM4_FBIG: The request specified a length or offset + * that exceeds the range supported by the + * server. + * %NLM4_FAILED: The request failed for an unspecified reason. + */ +static __be32 +nlm4svc_proc_lock(struct svc_rqst *rqstp) +{ + return nlm4svc_do_lock(rqstp, true); +} + +/** + * nlm4svc_proc_cancel - CANCEL: Cancel an outstanding blocked lock request + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully + * %rpc_drop_reply: Do not send an RPC reply + * + * RPC synopsis: + * nlm4_res NLMPROC4_CANCEL(nlm4_cancargs) = 3; + * + * Permissible procedure status codes: + * %NLM4_LCK_GRANTED: The requested lock was canceled. + * %NLM4_LCK_DENIED: There was no lock to cancel. + * %NLM4_DENIED_GRACE_PERIOD: The server has recently restarted and is + * re-establishing existing locks, and is not + * yet ready to accept normal service requests. + * + * The Linux NLM server implementation also returns: + * %NLM4_DENIED_NOLOCKS: A needed resource could not be allocated. + * %NLM4_STALE_FH: The request specified an invalid file handle. + * %NLM4_FBIG: The request specified a length or offset + * that exceeds the range supported by the + * server. + * %NLM4_FAILED: The request failed for an unspecified reason. + */ +static __be32 +nlm4svc_proc_cancel(struct svc_rqst *rqstp) +{ + struct nlm4_cancargs_wrapper *argp = rqstp->rq_argp; + unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK; + struct nlm4_res_wrapper *resp = rqstp->rq_resp; + struct net *net = SVC_NET(rqstp); + struct nlm_host *host = NULL; + struct nlm_file *file = NULL; + + resp->xdrgen.cookie = argp->xdrgen.cookie; + + resp->xdrgen.stat.stat = nlm_lck_denied_grace_period; + if (locks_in_grace(net)) + goto out; + + resp->xdrgen.stat.stat = nlm_lck_denied_nolocks; + host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + goto out; + + resp->xdrgen.stat.stat = nlm4svc_lookup_file(rqstp, host, &argp->lock, + &file, &argp->xdrgen.alock, + type); + if (resp->xdrgen.stat.stat) + goto out; + + resp->xdrgen.stat.stat = nlmsvc_cancel_blocked(net, file, &argp->lock); + nlmsvc_release_lockowner(&argp->lock); + +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->xdrgen.stat.stat == nlm__int__drop_reply ? + rpc_drop_reply : rpc_success; +} + +/** + * nlm4svc_proc_unlock - UNLOCK: Remove a lock + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * + * RPC synopsis: + * nlm4_res NLMPROC4_UNLOCK(nlm4_unlockargs) = 4; + * + * Permissible procedure status codes: + * %NLM4_GRANTED: The requested lock was released. + * %NLM4_DENIED_GRACE_PERIOD: The server has recently restarted and is + * re-establishing existing locks, and is not + * yet ready to accept normal service requests. + * + * The Linux NLM server implementation also returns: + * %NLM4_DENIED_NOLOCKS: A needed resource could not be allocated. + * %NLM4_STALE_FH: The request specified an invalid file handle. + * %NLM4_FBIG: The request specified a length or offset + * that exceeds the range supported by the + * server. + * %NLM4_FAILED: The request failed for an unspecified reason. + */ +static __be32 +nlm4svc_proc_unlock(struct svc_rqst *rqstp) +{ + struct nlm4_unlockargs_wrapper *argp = rqstp->rq_argp; + struct nlm4_res_wrapper *resp = rqstp->rq_resp; + struct net *net = SVC_NET(rqstp); + struct nlm_host *host = NULL; + struct nlm_file *file = NULL; + + resp->xdrgen.cookie = argp->xdrgen.cookie; + + resp->xdrgen.stat.stat = nlm_lck_denied_grace_period; + if (locks_in_grace(net)) + goto out; + + resp->xdrgen.stat.stat = nlm_lck_denied_nolocks; + host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + goto out; + + resp->xdrgen.stat.stat = nlm4svc_lookup_file(rqstp, host, &argp->lock, + &file, &argp->xdrgen.alock, + F_UNLCK); + if (resp->xdrgen.stat.stat) + goto out; + + resp->xdrgen.stat.stat = nlmsvc_unlock(net, file, &argp->lock); + nlmsvc_release_lockowner(&argp->lock); + +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->xdrgen.stat.stat == nlm__int__drop_reply ? + rpc_drop_reply : rpc_success; +} + +/** + * nlm4svc_proc_granted - GRANTED: Server grants a previously blocked lock + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * + * RPC synopsis: + * nlm4_res NLMPROC4_GRANTED(nlm4_testargs) = 5; + * + * Permissible procedure status codes: + * %NLM4_GRANTED: The granted lock was accepted. + * %NLM4_DENIED: The procedure failed, possibly due to + * internal resource constraints. + * %NLM4_DENIED_GRACE_PERIOD: The client host recently restarted and + * its NLM is re-establishing existing locks, + * so it is not yet ready to accept callbacks. + */ +static __be32 +nlm4svc_proc_granted(struct svc_rqst *rqstp) +{ + struct nlm4_testargs_wrapper *argp = rqstp->rq_argp; + struct nlm4_res_wrapper *resp = rqstp->rq_resp; + + resp->xdrgen.cookie = argp->xdrgen.cookie; + + resp->xdrgen.stat.stat = nlm4_lock_to_lockd_lock(&argp->lock, + &argp->xdrgen.alock); + if (resp->xdrgen.stat.stat) + goto out; + + resp->xdrgen.stat.stat = nlmclnt_grant(svc_addr(rqstp), &argp->lock); + +out: + return rpc_success; +} + +/* + * This is the generic lockd callback for async RPC calls + */ +static void nlm4svc_callback_exit(struct rpc_task *task, void *data) +{ +} + +static void nlm4svc_callback_release(void *data) +{ + nlmsvc_release_call(data); +} + +static const struct rpc_call_ops nlm4svc_callback_ops = { + .rpc_call_done = nlm4svc_callback_exit, + .rpc_release = nlm4svc_callback_release, +}; + +/* + * Dispatch an async callback RPC to a client with a pre-resolved host. + * Caller provides a reference to @host; this function takes ownership + * and releases it via nlmsvc_release_host() before returning. + */ +static __be32 +nlm4svc_callback(struct svc_rqst *rqstp, struct nlm_host *host, u32 proc, + __be32 (*func)(struct svc_rqst *, struct lockd_res *)) +{ + struct nlm_rqst *call; + __be32 stat; + + call = nlm_alloc_call(host); + nlmsvc_release_host(host); + if (call == NULL) + return rpc_system_err; + + stat = func(rqstp, &call->a_res); + if (stat != 0) { + nlmsvc_release_call(call); + return stat; + } + + call->a_flags = RPC_TASK_ASYNC; + if (nlm_async_reply(call, proc, &nlm4svc_callback_ops) < 0) + return rpc_system_err; + return rpc_success; +} + +static __be32 +__nlm4svc_proc_test_msg(struct svc_rqst *rqstp, struct lockd_res *resp) +{ + struct nlm4_testargs_wrapper *argp = rqstp->rq_argp; + unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK; + struct nlm_lockowner *owner; + struct nlm_file *file = NULL; + struct nlm_host *host = NULL; + + resp->status = nlm_lck_denied_nolocks; + if (nlm4_netobj_to_cookie(&resp->cookie, &argp->xdrgen.cookie)) + goto out; + + host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + goto out; + + resp->status = nlm4svc_lookup_file(rqstp, host, &argp->lock, + &file, &argp->xdrgen.alock, type); + if (resp->status) + goto out; + + owner = argp->lock.fl.c.flc_owner; + resp->status = nlmsvc_testlock(rqstp, file, host, &argp->lock, + &resp->lock); + nlmsvc_put_lockowner(owner); + +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->status == nlm__int__drop_reply ? rpc_drop_reply : rpc_success; +} + +/** + * nlm4svc_proc_test_msg - TEST_MSG: Check for conflicting lock + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_system_err: RPC execution failed. + * + * RPC synopsis: + * void NLMPROC4_TEST_MSG(nlm4_testargs) = 6; + * + * The response to this request is delivered via the TEST_RES procedure. + */ +static __be32 nlm4svc_proc_test_msg(struct svc_rqst *rqstp) +{ + struct nlm4_testargs_wrapper *argp = rqstp->rq_argp; + struct nlm_host *host; + + host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + return rpc_system_err; + + return nlm4svc_callback(rqstp, host, NLMPROC4_TEST_RES, + __nlm4svc_proc_test_msg); +} + +static __be32 +__nlm4svc_proc_lock_msg(struct svc_rqst *rqstp, struct lockd_res *resp) +{ + struct nlm4_lockargs_wrapper *argp = rqstp->rq_argp; + unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK; + struct nlm_file *file = NULL; + struct nlm_host *host = NULL; + + resp->status = nlm_lck_denied_nolocks; + if (nlm4_netobj_to_cookie(&resp->cookie, &argp->xdrgen.cookie)) + goto out; + + host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, true); + if (!host) + goto out; + + resp->status = nlm4svc_lookup_file(rqstp, host, &argp->lock, + &file, &argp->xdrgen.alock, type); + if (resp->status) + goto out; + + resp->status = nlmsvc_lock(rqstp, file, host, &argp->lock, + argp->xdrgen.block, &resp->cookie, + argp->xdrgen.reclaim); + if (resp->status == nlm__int__deadlock) + resp->status = nlm4_deadlock; + nlmsvc_release_lockowner(&argp->lock); + +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->status == nlm__int__drop_reply ? + rpc_drop_reply : rpc_success; +} + +/** + * nlm4svc_proc_lock_msg - LOCK_MSG: Establish a monitored lock + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_system_err: RPC execution failed. + * + * RPC synopsis: + * void NLMPROC4_LOCK_MSG(nlm4_lockargs) = 7; + * + * The response to this request is delivered via the LOCK_RES procedure. + */ +static __be32 nlm4svc_proc_lock_msg(struct svc_rqst *rqstp) +{ + struct nlm4_lockargs_wrapper *argp = rqstp->rq_argp; + struct nlm_host *host; + + host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + return rpc_system_err; + + return nlm4svc_callback(rqstp, host, NLMPROC4_LOCK_RES, + __nlm4svc_proc_lock_msg); +} + +static __be32 +__nlm4svc_proc_cancel_msg(struct svc_rqst *rqstp, struct lockd_res *resp) +{ + struct nlm4_cancargs_wrapper *argp = rqstp->rq_argp; + unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK; + struct net *net = SVC_NET(rqstp); + struct nlm_file *file = NULL; + struct nlm_host *host = NULL; + + resp->status = nlm_lck_denied_nolocks; + if (nlm4_netobj_to_cookie(&resp->cookie, &argp->xdrgen.cookie)) + goto out; + + resp->status = nlm_lck_denied_grace_period; + if (locks_in_grace(net)) + goto out; + + resp->status = nlm_lck_denied_nolocks; + host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + goto out; + + resp->status = nlm4svc_lookup_file(rqstp, host, &argp->lock, + &file, &argp->xdrgen.alock, type); + if (resp->status) + goto out; + + resp->status = nlmsvc_cancel_blocked(net, file, &argp->lock); + nlmsvc_release_lockowner(&argp->lock); + +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->status == nlm__int__drop_reply ? + rpc_drop_reply : rpc_success; +} + +/** + * nlm4svc_proc_cancel_msg - CANCEL_MSG: Cancel an outstanding lock request + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_system_err: RPC execution failed. + * + * RPC synopsis: + * void NLMPROC4_CANCEL_MSG(nlm4_cancargs) = 8; + * + * The response to this request is delivered via the CANCEL_RES procedure. + */ +static __be32 nlm4svc_proc_cancel_msg(struct svc_rqst *rqstp) +{ + struct nlm4_cancargs_wrapper *argp = rqstp->rq_argp; + struct nlm_host *host; + + host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + return rpc_system_err; + + return nlm4svc_callback(rqstp, host, NLMPROC4_CANCEL_RES, + __nlm4svc_proc_cancel_msg); +} + +static __be32 +__nlm4svc_proc_unlock_msg(struct svc_rqst *rqstp, struct lockd_res *resp) +{ + struct nlm4_unlockargs_wrapper *argp = rqstp->rq_argp; + struct net *net = SVC_NET(rqstp); + struct nlm_file *file = NULL; + struct nlm_host *host = NULL; + + resp->status = nlm_lck_denied_nolocks; + if (nlm4_netobj_to_cookie(&resp->cookie, &argp->xdrgen.cookie)) + goto out; + + resp->status = nlm_lck_denied_grace_period; + if (locks_in_grace(net)) + goto out; + + resp->status = nlm_lck_denied_nolocks; + host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + goto out; + + resp->status = nlm4svc_lookup_file(rqstp, host, &argp->lock, + &file, &argp->xdrgen.alock, F_UNLCK); + if (resp->status) + goto out; + + resp->status = nlmsvc_unlock(net, file, &argp->lock); + nlmsvc_release_lockowner(&argp->lock); + +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->status == nlm__int__drop_reply ? + rpc_drop_reply : rpc_success; +} + +/** + * nlm4svc_proc_unlock_msg - UNLOCK_MSG: Remove an existing lock + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_system_err: RPC execution failed. + * + * RPC synopsis: + * void NLMPROC4_UNLOCK_MSG(nlm4_unlockargs) = 9; + * + * The response to this request is delivered via the UNLOCK_RES procedure. + */ +static __be32 nlm4svc_proc_unlock_msg(struct svc_rqst *rqstp) +{ + struct nlm4_unlockargs_wrapper *argp = rqstp->rq_argp; + struct nlm_host *host; + + host = nlm4svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + return rpc_system_err; + + return nlm4svc_callback(rqstp, host, NLMPROC4_UNLOCK_RES, + __nlm4svc_proc_unlock_msg); +} + +static __be32 +__nlm4svc_proc_granted_msg(struct svc_rqst *rqstp, struct lockd_res *resp) +{ + struct nlm4_testargs_wrapper *argp = rqstp->rq_argp; + + resp->status = nlm_lck_denied; + if (nlm4_netobj_to_cookie(&resp->cookie, &argp->xdrgen.cookie)) + goto out; + + if (nlm4_lock_to_lockd_lock(&argp->lock, &argp->xdrgen.alock)) + goto out; + + resp->status = nlmclnt_grant(svc_addr(rqstp), &argp->lock); + +out: + return rpc_success; +} + +/** + * nlm4svc_proc_granted_msg - GRANTED_MSG: Blocked lock has been granted + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_system_err: RPC execution failed. + * + * RPC synopsis: + * void NLMPROC4_GRANTED_MSG(nlm4_testargs) = 10; + * + * The response to this request is delivered via the GRANTED_RES procedure. + */ +static __be32 nlm4svc_proc_granted_msg(struct svc_rqst *rqstp) +{ + struct nlm4_testargs_wrapper *argp = rqstp->rq_argp; + struct nlm_host *host; + + host = nlmsvc_lookup_host(rqstp, argp->xdrgen.alock.caller_name.data, + argp->xdrgen.alock.caller_name.len); + if (!host) + return rpc_system_err; + + return nlm4svc_callback(rqstp, host, NLMPROC4_GRANTED_RES, + __nlm4svc_proc_granted_msg); +} + +/** + * nlm4svc_proc_granted_res - GRANTED_RES: Lock Granted result + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * + * RPC synopsis: + * void NLMPROC4_GRANTED_RES(nlm4_res) = 15; + */ +static __be32 nlm4svc_proc_granted_res(struct svc_rqst *rqstp) +{ + struct nlm4_res_wrapper *argp = rqstp->rq_argp; + + if (!rcu_access_pointer(nlmsvc_ops)) + return rpc_success; + + if (nlm4_netobj_to_cookie(&argp->cookie, &argp->xdrgen.cookie)) + return rpc_success; + nlmsvc_grant_reply(&argp->cookie, argp->xdrgen.stat.stat); + + return rpc_success; +} + +/** + * nlm4svc_proc_sm_notify - SM_NOTIFY: Peer has rebooted + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_system_err: RPC execution failed. + * + * The SM_NOTIFY procedure is a private callback from Linux statd and is + * not part of the official NLM protocol. + * + * RPC synopsis: + * void NLMPROC4_SM_NOTIFY(nlm4_notifyargs) = 16; + */ +static __be32 nlm4svc_proc_sm_notify(struct svc_rqst *rqstp) +{ + struct nlm4_notifyargs_wrapper *argp = rqstp->rq_argp; + struct lockd_reboot *reboot = &argp->reboot; + + if (!nlm_privileged_requester(rqstp)) { + char buf[RPC_MAX_ADDRBUFLEN]; + + pr_warn("lockd: rejected NSM callback from %s\n", + svc_print_addr(rqstp, buf, sizeof(buf))); + return rpc_system_err; + } + + reboot->len = argp->xdrgen.notify.name.len; + reboot->mon = (char *)argp->xdrgen.notify.name.data; + reboot->state = argp->xdrgen.notify.state; + memcpy(&reboot->priv.data, argp->xdrgen.private, + sizeof(reboot->priv.data)); + + nlm_host_rebooted(SVC_NET(rqstp), reboot); + + return rpc_success; +} + +/** + * nlm4svc_proc_unused - stub for unused procedures + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_proc_unavail: Program can't support procedure. + */ +static __be32 nlm4svc_proc_unused(struct svc_rqst *rqstp) +{ + return rpc_proc_unavail; +} + +/** + * nlm4svc_proc_share - SHARE: Open a file using DOS file-sharing modes + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * + * RPC synopsis: + * nlm4_shareres NLMPROC4_SHARE(nlm4_shareargs) = 20; + * + * Permissible procedure status codes: + * %NLM4_GRANTED: The requested share lock was granted. + * %NLM4_DENIED: The requested lock conflicted with existing + * lock reservations for the file. + * %NLM4_DENIED_GRACE_PERIOD: The server has recently restarted and is + * re-establishing existing locks, and is not + * yet ready to accept normal service requests. + * + * The Linux NLM server implementation also returns: + * %NLM4_DENIED_NOLOCKS: A needed resource could not be allocated. + * %NLM4_STALE_FH: The request specified an invalid file handle. + * %NLM4_FBIG: The request specified a length or offset + * that exceeds the range supported by the + * server. + * %NLM4_FAILED: The request failed for an unspecified reason. + */ +static __be32 nlm4svc_proc_share(struct svc_rqst *rqstp) +{ + struct nlm4_shareargs_wrapper *argp = rqstp->rq_argp; + struct nlm4_shareres_wrapper *resp = rqstp->rq_resp; + struct lockd_lock *lock = &argp->lock; + struct nlm_host *host = NULL; + struct nlm_file *file = NULL; + struct nlm4_lock xdr_lock = { + .fh = argp->xdrgen.share.fh, + .oh = argp->xdrgen.share.oh, + .svid = LOCKD_SHARE_SVID, + }; + + resp->xdrgen.cookie = argp->xdrgen.cookie; + + resp->xdrgen.stat = nlm_lck_denied_grace_period; + if (locks_in_grace(SVC_NET(rqstp)) && !argp->xdrgen.reclaim) + goto out; + + resp->xdrgen.stat = nlm_lck_denied_nolocks; + host = nlm4svc_lookup_host(rqstp, argp->xdrgen.share.caller_name, true); + if (!host) + goto out; + + resp->xdrgen.stat = nlm4svc_lookup_file(rqstp, host, lock, &file, + &xdr_lock, F_RDLCK); + if (resp->xdrgen.stat) + goto out; + + resp->xdrgen.stat = nlmsvc_share_file(host, file, &lock->oh, + argp->xdrgen.share.access, + argp->xdrgen.share.mode); + + nlmsvc_release_lockowner(lock); + +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->xdrgen.stat == nlm__int__drop_reply ? + rpc_drop_reply : rpc_success; +} + +/** + * nlm4svc_proc_unshare - UNSHARE: Release a share reservation + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * + * RPC synopsis: + * nlm4_shareres NLMPROC4_UNSHARE(nlm4_shareargs) = 21; + * + * Permissible procedure status codes: + * %NLM4_GRANTED: The share reservation was released. + * %NLM4_DENIED_GRACE_PERIOD: The server has recently restarted and is + * re-establishing existing locks, and is not + * yet ready to accept normal service requests. + * + * The Linux NLM server implementation also returns: + * %NLM4_DENIED_NOLOCKS: A needed resource could not be allocated. + * %NLM4_STALE_FH: The request specified an invalid file handle. + * %NLM4_FBIG: The request specified a length or offset + * that exceeds the range supported by the + * server. + * %NLM4_FAILED: The request failed for an unspecified reason. + */ +static __be32 nlm4svc_proc_unshare(struct svc_rqst *rqstp) +{ + struct nlm4_shareargs_wrapper *argp = rqstp->rq_argp; + struct nlm4_shareres_wrapper *resp = rqstp->rq_resp; + struct lockd_lock *lock = &argp->lock; + struct nlm4_lock xdr_lock = { + .fh = argp->xdrgen.share.fh, + .oh = argp->xdrgen.share.oh, + .svid = LOCKD_SHARE_SVID, + }; + struct nlm_host *host = NULL; + struct nlm_file *file = NULL; + + resp->xdrgen.cookie = argp->xdrgen.cookie; + + resp->xdrgen.stat = nlm_lck_denied_grace_period; + if (locks_in_grace(SVC_NET(rqstp))) + goto out; + + resp->xdrgen.stat = nlm_lck_denied_nolocks; + host = nlm4svc_lookup_host(rqstp, argp->xdrgen.share.caller_name, true); + if (!host) + goto out; + + resp->xdrgen.stat = nlm4svc_lookup_file(rqstp, host, lock, &file, + &xdr_lock, F_RDLCK); + if (resp->xdrgen.stat) + goto out; + + resp->xdrgen.stat = nlmsvc_unshare_file(host, file, &lock->oh, + argp->xdrgen.share.access, + argp->xdrgen.share.mode); + + nlmsvc_release_lockowner(lock); + +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->xdrgen.stat == nlm__int__drop_reply ? + rpc_drop_reply : rpc_success; +} + +/** + * nlm4svc_proc_nm_lock - NM_LOCK: Establish a non-monitored lock + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * + * RPC synopsis: + * nlm4_res NLMPROC4_NM_LOCK(nlm4_lockargs) = 22; + * + * Permissible procedure status codes: + * %NLM4_GRANTED: The requested lock was granted. + * %NLM4_DENIED: The requested lock conflicted with existing + * lock reservations for the file. + * %NLM4_DENIED_NOLOCKS: The server could not allocate the resources + * needed to process the request. + * %NLM4_BLOCKED: The blocking request cannot be granted + * immediately. The server will send an + * NLMPROC4_GRANTED callback to the client when + * the lock can be granted. + * %NLM4_DENIED_GRACE_PERIOD: The server has recently restarted and is + * re-establishing existing locks, and is not + * yet ready to accept normal service requests. + * + * The Linux NLM server implementation also returns: + * %NLM4_DEADLCK: The request could not be granted and + * blocking would cause a deadlock. + * %NLM4_STALE_FH: The request specified an invalid file handle. + * %NLM4_FBIG: The request specified a length or offset + * that exceeds the range supported by the + * server. + * %NLM4_FAILED: The request failed for an unspecified reason. + */ +static __be32 nlm4svc_proc_nm_lock(struct svc_rqst *rqstp) +{ + return nlm4svc_do_lock(rqstp, false); +} + +/** + * nlm4svc_proc_free_all - FREE_ALL: Discard client's lock and share state + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * + * RPC synopsis: + * void NLMPROC4_FREE_ALL(nlm4_notify) = 23; + */ +static __be32 nlm4svc_proc_free_all(struct svc_rqst *rqstp) +{ + struct nlm4_notify_wrapper *argp = rqstp->rq_argp; + struct nlm_host *host; + + host = nlm4svc_lookup_host(rqstp, argp->xdrgen.name, false); + if (!host) + goto out; + + nlmsvc_free_host_resources(host); + + nlmsvc_release_host(host); + +out: + return rpc_success; +} + + +/* + * NLMv4 Server procedures. + */ + +static const struct svc_procedure nlm4svc_procedures[24] = { + [NLMPROC4_NULL] = { + .pc_func = nlm4svc_proc_null, + .pc_decode = xdrgen_svc_decode_void, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = XDR_void, + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "NULL", + }, + [NLMPROC4_TEST] = { + .pc_func = nlm4svc_proc_test, + .pc_decode = nlm4_svc_decode_nlm4_testargs, + .pc_encode = nlm4_svc_encode_nlm4_testres, + .pc_argsize = sizeof(struct nlm4_testargs_wrapper), + .pc_argzero = 0, + .pc_ressize = sizeof(struct nlm4_testres_wrapper), + .pc_xdrressize = NLM4_nlm4_testres_sz, + .pc_name = "TEST", + }, + [NLMPROC4_LOCK] = { + .pc_func = nlm4svc_proc_lock, + .pc_decode = nlm4_svc_decode_nlm4_lockargs, + .pc_encode = nlm4_svc_encode_nlm4_res, + .pc_argsize = sizeof(struct nlm4_lockargs_wrapper), + .pc_argzero = 0, + .pc_ressize = sizeof(struct nlm4_res_wrapper), + .pc_xdrressize = NLM4_nlm4_res_sz, + .pc_name = "LOCK", + }, + [NLMPROC4_CANCEL] = { + .pc_func = nlm4svc_proc_cancel, + .pc_decode = nlm4_svc_decode_nlm4_cancargs, + .pc_encode = nlm4_svc_encode_nlm4_res, + .pc_argsize = sizeof(struct nlm4_cancargs_wrapper), + .pc_argzero = 0, + .pc_ressize = sizeof(struct nlm4_res_wrapper), + .pc_xdrressize = NLM4_nlm4_res_sz, + .pc_name = "CANCEL", + }, + [NLMPROC4_UNLOCK] = { + .pc_func = nlm4svc_proc_unlock, + .pc_decode = nlm4_svc_decode_nlm4_unlockargs, + .pc_encode = nlm4_svc_encode_nlm4_res, + .pc_argsize = sizeof(struct nlm4_unlockargs_wrapper), + .pc_argzero = 0, + .pc_ressize = sizeof(struct nlm4_res_wrapper), + .pc_xdrressize = NLM4_nlm4_res_sz, + .pc_name = "UNLOCK", + }, + [NLMPROC4_GRANTED] = { + .pc_func = nlm4svc_proc_granted, + .pc_decode = nlm4_svc_decode_nlm4_testargs, + .pc_encode = nlm4_svc_encode_nlm4_res, + .pc_argsize = sizeof(struct nlm4_testargs_wrapper), + .pc_argzero = 0, + .pc_ressize = sizeof(struct nlm4_res_wrapper), + .pc_xdrressize = NLM4_nlm4_res_sz, + .pc_name = "GRANTED", + }, + [NLMPROC4_TEST_MSG] = { + .pc_func = nlm4svc_proc_test_msg, + .pc_decode = nlm4_svc_decode_nlm4_testargs, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = sizeof(struct nlm4_testargs_wrapper), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "TEST_MSG", + }, + [NLMPROC4_LOCK_MSG] = { + .pc_func = nlm4svc_proc_lock_msg, + .pc_decode = nlm4_svc_decode_nlm4_lockargs, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = sizeof(struct nlm4_lockargs_wrapper), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "LOCK_MSG", + }, + [NLMPROC4_CANCEL_MSG] = { + .pc_func = nlm4svc_proc_cancel_msg, + .pc_decode = nlm4_svc_decode_nlm4_cancargs, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = sizeof(struct nlm4_cancargs_wrapper), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "CANCEL_MSG", + }, + [NLMPROC4_UNLOCK_MSG] = { + .pc_func = nlm4svc_proc_unlock_msg, + .pc_decode = nlm4_svc_decode_nlm4_unlockargs, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = sizeof(struct nlm4_unlockargs_wrapper), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "UNLOCK_MSG", + }, + [NLMPROC4_GRANTED_MSG] = { + .pc_func = nlm4svc_proc_granted_msg, + .pc_decode = nlm4_svc_decode_nlm4_testargs, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = sizeof(struct nlm4_testargs_wrapper), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "GRANTED_MSG", + }, + [NLMPROC4_TEST_RES] = { + .pc_func = nlm4svc_proc_null, + .pc_decode = nlm4_svc_decode_nlm4_testres, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = sizeof(struct nlm4_testres), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "TEST_RES", + }, + [NLMPROC4_LOCK_RES] = { + .pc_func = nlm4svc_proc_null, + .pc_decode = nlm4_svc_decode_nlm4_res, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = sizeof(struct nlm4_res), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "LOCK_RES", + }, + [NLMPROC4_CANCEL_RES] = { + .pc_func = nlm4svc_proc_null, + .pc_decode = nlm4_svc_decode_nlm4_res, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = sizeof(struct nlm4_res), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "CANCEL_RES", + }, + [NLMPROC4_UNLOCK_RES] = { + .pc_func = nlm4svc_proc_null, + .pc_decode = nlm4_svc_decode_nlm4_res, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = sizeof(struct nlm4_res), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "UNLOCK_RES", + }, + [NLMPROC4_GRANTED_RES] = { + .pc_func = nlm4svc_proc_granted_res, + .pc_decode = nlm4_svc_decode_nlm4_res, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = sizeof(struct nlm4_res_wrapper), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "GRANTED_RES", + }, + [NLMPROC4_SM_NOTIFY] = { + .pc_func = nlm4svc_proc_sm_notify, + .pc_decode = nlm4_svc_decode_nlm4_notifyargs, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = sizeof(struct nlm4_notifyargs_wrapper), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "SM_NOTIFY", + }, + [17] = { + .pc_func = nlm4svc_proc_unused, + .pc_decode = xdrgen_svc_decode_void, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = 0, + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "UNUSED", + }, + [18] = { + .pc_func = nlm4svc_proc_unused, + .pc_decode = xdrgen_svc_decode_void, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = 0, + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "UNUSED", + }, + [19] = { + .pc_func = nlm4svc_proc_unused, + .pc_decode = xdrgen_svc_decode_void, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = 0, + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "UNUSED", + }, + [NLMPROC4_SHARE] = { + .pc_func = nlm4svc_proc_share, + .pc_decode = nlm4_svc_decode_nlm4_shareargs, + .pc_encode = nlm4_svc_encode_nlm4_shareres, + .pc_argsize = sizeof(struct nlm4_shareargs_wrapper), + .pc_argzero = 0, + .pc_ressize = sizeof(struct nlm4_shareres_wrapper), + .pc_xdrressize = NLM4_nlm4_shareres_sz, + .pc_name = "SHARE", + }, + [NLMPROC4_UNSHARE] = { + .pc_func = nlm4svc_proc_unshare, + .pc_decode = nlm4_svc_decode_nlm4_shareargs, + .pc_encode = nlm4_svc_encode_nlm4_shareres, + .pc_argsize = sizeof(struct nlm4_shareargs_wrapper), + .pc_argzero = 0, + .pc_ressize = sizeof(struct nlm4_shareres_wrapper), + .pc_xdrressize = NLM4_nlm4_shareres_sz, + .pc_name = "UNSHARE", + }, + [NLMPROC4_NM_LOCK] = { + .pc_func = nlm4svc_proc_nm_lock, + .pc_decode = nlm4_svc_decode_nlm4_lockargs, + .pc_encode = nlm4_svc_encode_nlm4_res, + .pc_argsize = sizeof(struct nlm4_lockargs_wrapper), + .pc_argzero = 0, + .pc_ressize = sizeof(struct nlm4_res_wrapper), + .pc_xdrressize = NLM4_nlm4_res_sz, + .pc_name = "NM_LOCK", + }, + [NLMPROC4_FREE_ALL] = { + .pc_func = nlm4svc_proc_free_all, + .pc_decode = nlm4_svc_decode_nlm4_notify, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = sizeof(struct nlm4_notify_wrapper), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "FREE_ALL", + }, +}; + +/* + * Storage requirements for XDR arguments and results + */ +union nlm4svc_xdrstore { + struct nlm4_testargs_wrapper testargs; + struct nlm4_lockargs_wrapper lockargs; + struct nlm4_cancargs_wrapper cancargs; + struct nlm4_unlockargs_wrapper unlockargs; + struct nlm4_notifyargs_wrapper notifyargs; + struct nlm4_shareargs_wrapper shareargs; + struct nlm4_notify_wrapper notify; + struct nlm4_testres_wrapper testres; + struct nlm4_res_wrapper res; + struct nlm4_shareres_wrapper shareres; +}; + +const struct svc_version nlmsvc_version4 = { + .vs_vers = 4, + .vs_nproc = ARRAY_SIZE(nlm4svc_procedures), + .vs_proc = nlm4svc_procedures, + .vs_dispatch = nlmsvc_dispatch, + .vs_xdrsize = sizeof(union nlm4svc_xdrstore), +}; diff --git a/fs/lockd/svclock.c b/fs/lockd/svclock.c new file mode 100644 index 000000000..e628b5d35 --- /dev/null +++ b/fs/lockd/svclock.c @@ -0,0 +1,1042 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/lockd/svclock.c + * + * Handling of server-side locks, mostly of the blocked variety. + * This is the ugliest part of lockd because we tread on very thin ice. + * GRANT and CANCEL calls may get stuck, meet in mid-flight, etc. + * IMNSHO introducing the grant callback into the NLM protocol was one + * of the worst ideas Sun ever had. Except maybe for the idea of doing + * NFS file locking at all. + * + * I'm trying hard to avoid race conditions by protecting most accesses + * to a file's list of blocked locks through a semaphore. The global + * list of blocked locks is not protected in this fashion however. + * Therefore, some functions (such as the RPC callback for the async grant + * call) move blocked locks towards the head of the list *while some other + * process might be traversing it*. This should not be a problem in + * practice, because this will only cause functions traversing the list + * to visit some blocks twice. + * + * Copyright (C) 1996, Olaf Kirch + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "lockd.h" + +#define NLMDBG_FACILITY NLMDBG_SVCLOCK + +static void nlmsvc_release_block(struct nlm_block *block); +static void nlmsvc_insert_block(struct nlm_block *block, unsigned long); +static void nlmsvc_remove_block(struct nlm_block *block); + +static int nlmsvc_setgrantargs(struct nlm_rqst *call, struct lockd_lock *lock); +static void nlmsvc_freegrantargs(struct nlm_rqst *call); +static const struct rpc_call_ops nlmsvc_grant_ops; + +/* + * The list of blocked locks to retry + */ +static LIST_HEAD(nlm_blocked); +static DEFINE_SPINLOCK(nlm_blocked_lock); + +/* + * Insert a blocked lock into the global list + */ +static void +nlmsvc_insert_block_locked(struct nlm_block *block, unsigned long when) +{ + struct nlm_block *b; + struct list_head *pos; + + dprintk("lockd: nlmsvc_insert_block(%p, %ld)\n", block, when); + if (list_empty(&block->b_list)) { + kref_get(&block->b_count); + } else { + list_del_init(&block->b_list); + } + + pos = &nlm_blocked; + if (when != NLM_NEVER) { + if ((when += jiffies) == NLM_NEVER) + when ++; + list_for_each(pos, &nlm_blocked) { + b = list_entry(pos, struct nlm_block, b_list); + if (time_after(b->b_when,when) || b->b_when == NLM_NEVER) + break; + } + /* On normal exit from the loop, pos == &nlm_blocked, + * so we will be adding to the end of the list - good + */ + } + + list_add_tail(&block->b_list, pos); + block->b_when = when; +} + +static void nlmsvc_insert_block(struct nlm_block *block, unsigned long when) +{ + spin_lock(&nlm_blocked_lock); + nlmsvc_insert_block_locked(block, when); + spin_unlock(&nlm_blocked_lock); +} + +/* + * Remove a block from the global list + */ +static inline void +nlmsvc_remove_block(struct nlm_block *block) +{ + spin_lock(&nlm_blocked_lock); + if (!list_empty(&block->b_list)) { + list_del_init(&block->b_list); + spin_unlock(&nlm_blocked_lock); + nlmsvc_release_block(block); + return; + } + spin_unlock(&nlm_blocked_lock); +} + +/* + * Find a block for a given lock + */ +static struct nlm_block * +nlmsvc_lookup_block(struct nlm_file *file, struct lockd_lock *lock) +{ + struct nlm_block *block; + struct file_lock *fl; + + dprintk("lockd: nlmsvc_lookup_block f=%p pd=%d %Ld-%Ld ty=%d\n", + file, lock->fl.c.flc_pid, + (long long)lock->fl.fl_start, + (long long)lock->fl.fl_end, + lock->fl.c.flc_type); + spin_lock(&nlm_blocked_lock); + list_for_each_entry(block, &nlm_blocked, b_list) { + fl = &block->b_call->a_args.lock.fl; + dprintk("lockd: check f=%p pd=%d %Ld-%Ld ty=%d cookie=%*phN\n", + block->b_file, fl->c.flc_pid, + (long long)fl->fl_start, + (long long)fl->fl_end, fl->c.flc_type, + block->b_call->a_args.cookie.len, + block->b_call->a_args.cookie.data); + if (block->b_file == file && nlm_compare_locks(fl, &lock->fl)) { + kref_get(&block->b_count); + spin_unlock(&nlm_blocked_lock); + return block; + } + } + spin_unlock(&nlm_blocked_lock); + + return NULL; +} + +static int lockd_cookie_match(struct lockd_cookie *a, struct lockd_cookie *b) +{ + if (a->len != b->len) + return 0; + if (memcmp(a->data, b->data, a->len)) + return 0; + return 1; +} + +/* + * Find a block with a given NLM cookie. + */ +static inline struct nlm_block * +nlmsvc_find_block(struct lockd_cookie *cookie) +{ + struct nlm_block *block; + + spin_lock(&nlm_blocked_lock); + list_for_each_entry(block, &nlm_blocked, b_list) { + if (lockd_cookie_match(&block->b_call->a_args.cookie, cookie)) + goto found; + } + spin_unlock(&nlm_blocked_lock); + + return NULL; + +found: + dprintk("nlmsvc_find_block(%*phN): block=%p\n", + cookie->len, cookie->data, block); + kref_get(&block->b_count); + spin_unlock(&nlm_blocked_lock); + return block; +} + +/* + * Create a block and initialize it. + * + * Note: we explicitly set the cookie of the grant reply to that of + * the blocked lock request. The spec explicitly mentions that the client + * should _not_ rely on the callback containing the same cookie as the + * request, but (as I found out later) that's because some implementations + * do just this. Never mind the standards comittees, they support our + * logging industries. + * + * 10 years later: I hope we can safely ignore these old and broken + * clients by now. Let's fix this so we can uniquely identify an incoming + * GRANTED_RES message by cookie, without having to rely on the client's IP + * address. --okir + */ +static struct nlm_block * +nlmsvc_create_block(struct svc_rqst *rqstp, struct nlm_host *host, + struct nlm_file *file, struct lockd_lock *lock, + struct lockd_cookie *cookie) +{ + struct nlm_block *block; + struct nlm_rqst *call = NULL; + + call = nlm_alloc_call(host); + if (call == NULL) + return NULL; + + /* Allocate memory for block, and initialize arguments */ + block = kzalloc_obj(*block); + if (block == NULL) + goto failed; + kref_init(&block->b_count); + INIT_LIST_HEAD(&block->b_list); + INIT_LIST_HEAD(&block->b_flist); + + if (!nlmsvc_setgrantargs(call, lock)) + goto failed_free; + + /* Set notifier function for VFS, and init args */ + call->a_args.lock.fl.c.flc_flags |= FL_SLEEP; + call->a_args.lock.fl.fl_lmops = &nlmsvc_lock_operations; + nlmclnt_next_cookie(&call->a_args.cookie); + + dprintk("lockd: created block %p...\n", block); + + /* Create and initialize the block */ + block->b_daemon = rqstp->rq_server; + block->b_host = host; + block->b_file = file; + file->f_count++; + + /* Add to file's list of blocks */ + list_add(&block->b_flist, &file->f_blocks); + + /* Set up RPC arguments for callback */ + block->b_call = call; + call->a_flags = RPC_TASK_ASYNC; + call->a_block = block; + + return block; + +failed_free: + kfree(block); +failed: + nlmsvc_release_call(call); + return NULL; +} + +/* + * Delete a block. + * It is the caller's responsibility to check whether the file + * can be closed hereafter. + */ +static int nlmsvc_unlink_block(struct nlm_block *block) +{ + int status; + dprintk("lockd: unlinking block %p...\n", block); + + /* Remove block from list */ + status = locks_delete_block(&block->b_call->a_args.lock.fl); + nlmsvc_remove_block(block); + return status; +} + +static void nlmsvc_free_block(struct kref *kref) +{ + struct nlm_block *block = container_of(kref, struct nlm_block, b_count); + struct nlm_file *file = block->b_file; + + dprintk("lockd: freeing block %p...\n", block); + + /* Remove block from file's list of blocks */ + list_del_init(&block->b_flist); + mutex_unlock(&file->f_mutex); + + nlmsvc_freegrantargs(block->b_call); + nlmsvc_release_call(block->b_call); + nlm_release_file(block->b_file); + kfree(block); +} + +static void nlmsvc_release_block(struct nlm_block *block) +{ + if (block != NULL) + kref_put_mutex(&block->b_count, nlmsvc_free_block, &block->b_file->f_mutex); +} + +/* + * Loop over all blocks and delete blocks held by + * a matching host. + */ +void nlmsvc_traverse_blocks(struct nlm_host *host, + struct nlm_file *file, + nlm_host_match_fn_t match) +{ + struct nlm_block *block, *next; + +restart: + mutex_lock(&file->f_mutex); + spin_lock(&nlm_blocked_lock); + list_for_each_entry_safe(block, next, &file->f_blocks, b_flist) { + if (!match(block->b_host, host)) + continue; + /* Do not destroy blocks that are not on + * the global retry list - why? */ + if (list_empty(&block->b_list)) + continue; + kref_get(&block->b_count); + spin_unlock(&nlm_blocked_lock); + mutex_unlock(&file->f_mutex); + nlmsvc_unlink_block(block); + nlmsvc_release_block(block); + goto restart; + } + spin_unlock(&nlm_blocked_lock); + mutex_unlock(&file->f_mutex); +} + +static struct nlm_lockowner * +nlmsvc_get_lockowner(struct nlm_lockowner *lockowner) +{ + refcount_inc(&lockowner->count); + return lockowner; +} + +void nlmsvc_put_lockowner(struct nlm_lockowner *lockowner) +{ + if (!refcount_dec_and_lock(&lockowner->count, &lockowner->host->h_lock)) + return; + list_del(&lockowner->list); + spin_unlock(&lockowner->host->h_lock); + nlmsvc_release_host(lockowner->host); + kfree(lockowner); +} + +static struct nlm_lockowner *__nlmsvc_find_lockowner(struct nlm_host *host, pid_t pid) +{ + struct nlm_lockowner *lockowner; + list_for_each_entry(lockowner, &host->h_lockowners, list) { + if (lockowner->pid != pid) + continue; + return nlmsvc_get_lockowner(lockowner); + } + return NULL; +} + +static struct nlm_lockowner *nlmsvc_find_lockowner(struct nlm_host *host, pid_t pid) +{ + struct nlm_lockowner *res, *new = NULL; + + spin_lock(&host->h_lock); + res = __nlmsvc_find_lockowner(host, pid); + + if (res == NULL) { + spin_unlock(&host->h_lock); + new = kmalloc_obj(*res); + spin_lock(&host->h_lock); + res = __nlmsvc_find_lockowner(host, pid); + if (res == NULL && new != NULL) { + res = new; + /* fs/locks.c will manage the refcount through lock_ops */ + refcount_set(&new->count, 1); + new->pid = pid; + new->host = nlm_get_host(host); + list_add(&new->list, &host->h_lockowners); + new = NULL; + } + } + + spin_unlock(&host->h_lock); + kfree(new); + return res; +} + +void +nlmsvc_release_lockowner(struct lockd_lock *lock) +{ + if (lock->fl.c.flc_owner) + nlmsvc_put_lockowner(lock->fl.c.flc_owner); +} + +void nlmsvc_locks_init_private(struct file_lock *fl, struct nlm_host *host, + pid_t pid) +{ + fl->c.flc_owner = nlmsvc_find_lockowner(host, pid); +} + +/* + * Initialize arguments for GRANTED call. The nlm_rqst structure + * has been cleared already. + */ +static int nlmsvc_setgrantargs(struct nlm_rqst *call, struct lockd_lock *lock) +{ + locks_copy_lock(&call->a_args.lock.fl, &lock->fl); + memcpy(&call->a_args.lock.fh, &lock->fh, sizeof(call->a_args.lock.fh)); + call->a_args.lock.caller = utsname()->nodename; + call->a_args.lock.oh.len = lock->oh.len; + + /* set default data area */ + call->a_args.lock.oh.data = call->a_owner; + call->a_args.lock.svid = ((struct nlm_lockowner *) lock->fl.c.flc_owner)->pid; + + if (lock->oh.len > NLMCLNT_OHSIZE) { + void *data = kmalloc(lock->oh.len, GFP_KERNEL); + if (!data) + return 0; + call->a_args.lock.oh.data = (u8 *) data; + } + + memcpy(call->a_args.lock.oh.data, lock->oh.data, lock->oh.len); + return 1; +} + +static void nlmsvc_freegrantargs(struct nlm_rqst *call) +{ + if (call->a_args.lock.oh.data != call->a_owner) + kfree(call->a_args.lock.oh.data); + + locks_release_private(&call->a_args.lock.fl); +} + +/* + * Deferred lock request handling for non-blocking lock + */ +static __be32 +nlmsvc_defer_lock_rqst(struct svc_rqst *rqstp, struct nlm_block *block) +{ + __be32 status = nlm_lck_denied_nolocks; + + block->b_flags |= B_QUEUED; + + nlmsvc_insert_block(block, NLM_TIMEOUT); + + block->b_cache_req = &rqstp->rq_chandle; + if (rqstp->rq_chandle.defer) { + block->b_deferred_req = + rqstp->rq_chandle.defer(block->b_cache_req); + if (block->b_deferred_req != NULL) + status = nlm__int__drop_reply; + } + dprintk("lockd: nlmsvc_defer_lock_rqst block %p flags %d status %d\n", + block, block->b_flags, ntohl(status)); + + return status; +} + +/* + * Attempt to establish a lock, and if it can't be granted, block it + * if required. + */ +__be32 +nlmsvc_lock(struct svc_rqst *rqstp, struct nlm_file *file, + struct nlm_host *host, struct lockd_lock *lock, int wait, + struct lockd_cookie *cookie, int reclaim) +{ + struct inode *inode __maybe_unused = nlmsvc_file_inode(file); + struct nlm_block *block = NULL; + int error; + int mode; + int async_block = 0; + __be32 ret; + + dprintk("lockd: nlmsvc_lock(%s/%llu, ty=%d, pi=%d, %Ld-%Ld, bl=%d)\n", + inode->i_sb->s_id, inode->i_ino, + lock->fl.c.flc_type, + lock->fl.c.flc_pid, + (long long)lock->fl.fl_start, + (long long)lock->fl.fl_end, + wait); + + if (nlmsvc_file_cannot_lock(file)) + return nlm_lck_denied_nolocks; + + if (!locks_can_async_lock(nlmsvc_file_file(file)->f_op)) { + async_block = wait; + wait = 0; + } + + /* Lock file against concurrent access */ + mutex_lock(&file->f_mutex); + /* Get existing block (in case client is busy-waiting) + * or create new block + */ + block = nlmsvc_lookup_block(file, lock); + if (block == NULL) { + block = nlmsvc_create_block(rqstp, host, file, lock, cookie); + ret = nlm_lck_denied_nolocks; + if (block == NULL) + goto out; + lock = &block->b_call->a_args.lock; + } else + lock->fl.c.flc_flags &= ~FL_SLEEP; + + if (block->b_flags & B_QUEUED) { + dprintk("lockd: nlmsvc_lock deferred block %p flags %d\n", + block, block->b_flags); + if (block->b_granted) { + nlmsvc_unlink_block(block); + ret = nlm_granted; + goto out; + } + if (block->b_flags & B_TIMED_OUT) { + nlmsvc_unlink_block(block); + ret = nlm_lck_denied; + goto out; + } + ret = nlm__int__drop_reply; + goto out; + } + + if (locks_in_grace(SVC_NET(rqstp)) && !reclaim) { + ret = nlm_lck_denied_grace_period; + goto out; + } + if (reclaim && !locks_in_grace(SVC_NET(rqstp))) { + ret = nlm_lck_denied_grace_period; + goto out; + } + + spin_lock(&nlm_blocked_lock); + /* + * If this is a lock request for an already pending + * lock request we return nlm_lck_blocked without calling + * vfs_lock_file() again. Otherwise we have two pending + * requests on the underlaying ->lock() implementation but + * only one nlm_block to being granted by lm_grant(). + */ + if (locks_can_async_lock(nlmsvc_file_file(file)->f_op) && + !list_empty(&block->b_list)) { + spin_unlock(&nlm_blocked_lock); + ret = nlm_lck_blocked; + goto out; + } + + /* Append to list of blocked */ + nlmsvc_insert_block_locked(block, NLM_NEVER); + spin_unlock(&nlm_blocked_lock); + + if (!wait) + lock->fl.c.flc_flags &= ~FL_SLEEP; + mode = lock_to_openmode(&lock->fl); + error = vfs_lock_file(file->f_file[mode], F_SETLK, &lock->fl, NULL); + lock->fl.c.flc_flags &= ~FL_SLEEP; + + dprintk("lockd: vfs_lock_file returned %d\n", error); + switch (error) { + case 0: + nlmsvc_remove_block(block); + ret = nlm_granted; + goto out; + case -EAGAIN: + if (!wait) + nlmsvc_remove_block(block); + ret = async_block ? nlm_lck_blocked : nlm_lck_denied; + goto out; + case FILE_LOCK_DEFERRED: + if (wait) + break; + /* Filesystem lock operation is in progress + Add it to the queue waiting for callback */ + ret = nlmsvc_defer_lock_rqst(rqstp, block); + goto out; + case -EDEADLK: + nlmsvc_remove_block(block); + ret = nlm__int__deadlock; + goto out; + default: /* includes ENOLCK */ + nlmsvc_remove_block(block); + ret = nlm_lck_denied_nolocks; + goto out; + } + + ret = nlm_lck_blocked; +out: + mutex_unlock(&file->f_mutex); + nlmsvc_release_block(block); + dprintk("lockd: nlmsvc_lock returned %u\n", ret); + return ret; +} + +/* + * Test for presence of a conflicting lock. + */ +__be32 +nlmsvc_testlock(struct svc_rqst *rqstp, struct nlm_file *file, + struct nlm_host *host, struct lockd_lock *lock, + struct lockd_lock *conflock) +{ + int error; + __be32 ret; + + dprintk("lockd: nlmsvc_testlock(%s/%llu, ty=%d, %Ld-%Ld)\n", + nlmsvc_file_inode(file)->i_sb->s_id, + nlmsvc_file_inode(file)->i_ino, + lock->fl.c.flc_type, + (long long)lock->fl.fl_start, + (long long)lock->fl.fl_end); + + if (nlmsvc_file_cannot_lock(file)) + return nlm_lck_denied_nolocks; + + if (locks_in_grace(SVC_NET(rqstp))) { + ret = nlm_lck_denied_grace_period; + goto out; + } + + locks_init_lock(&conflock->fl); + /* vfs_test_lock only uses start, end, and owner, but tests flc_file */ + conflock->fl.c.flc_file = lock->fl.c.flc_file; + conflock->fl.fl_start = lock->fl.fl_start; + conflock->fl.fl_end = lock->fl.fl_end; + conflock->fl.c.flc_owner = lock->fl.c.flc_owner; + error = vfs_test_lock(lock->fl.c.flc_file, &conflock->fl); + if (error) { + ret = nlm_lck_denied_nolocks; + goto out; + } + + if (conflock->fl.c.flc_type == F_UNLCK) { + ret = nlm_granted; + goto out; + } + + dprintk("lockd: conflicting lock(ty=%d, %Ld-%Ld)\n", + conflock->fl.c.flc_type, (long long)conflock->fl.fl_start, + (long long)conflock->fl.fl_end); + conflock->caller = "somehost"; /* FIXME */ + conflock->len = strlen(conflock->caller); + conflock->oh.len = 0; /* don't return OH info */ + conflock->svid = conflock->fl.c.flc_pid; + locks_release_private(&conflock->fl); + + ret = nlm_lck_denied; +out: + return ret; +} + +/* + * Remove a lock. + * This implies a CANCEL call: We send a GRANT_MSG, the client replies + * with a GRANT_RES call which gets lost, and calls UNLOCK immediately + * afterwards. In this case the block will still be there, and hence + * must be removed. + */ +__be32 +nlmsvc_unlock(struct net *net, struct nlm_file *file, struct lockd_lock *lock) +{ + int error = 0; + + dprintk("lockd: nlmsvc_unlock(%s/%llu, pi=%d, %Ld-%Ld)\n", + nlmsvc_file_inode(file)->i_sb->s_id, + nlmsvc_file_inode(file)->i_ino, + lock->fl.c.flc_pid, + (long long)lock->fl.fl_start, + (long long)lock->fl.fl_end); + + if (nlmsvc_file_cannot_lock(file)) + return nlm_lck_denied_nolocks; + + /* First, cancel any lock that might be there */ + nlmsvc_cancel_blocked(net, file, lock); + + lock->fl.c.flc_type = F_UNLCK; + lock->fl.c.flc_file = file->f_file[O_RDONLY]; + if (lock->fl.c.flc_file) + error = vfs_lock_file(lock->fl.c.flc_file, F_SETLK, + &lock->fl, NULL); + lock->fl.c.flc_file = file->f_file[O_WRONLY]; + if (lock->fl.c.flc_file) + error |= vfs_lock_file(lock->fl.c.flc_file, F_SETLK, + &lock->fl, NULL); + + return (error < 0)? nlm_lck_denied_nolocks : nlm_granted; +} + +/* + * Cancel a previously blocked request. + * + * A cancel request always overrides any grant that may currently + * be in progress. + * The calling procedure must check whether the file can be closed. + */ +__be32 +nlmsvc_cancel_blocked(struct net *net, struct nlm_file *file, struct lockd_lock *lock) +{ + struct nlm_block *block; + int status = 0; + int mode; + + dprintk("lockd: nlmsvc_cancel(%s/%llu, pi=%d, %Ld-%Ld)\n", + nlmsvc_file_inode(file)->i_sb->s_id, + nlmsvc_file_inode(file)->i_ino, + lock->fl.c.flc_pid, + (long long)lock->fl.fl_start, + (long long)lock->fl.fl_end); + + if (nlmsvc_file_cannot_lock(file)) + return nlm_lck_denied_nolocks; + + if (locks_in_grace(net)) + return nlm_lck_denied_grace_period; + + mutex_lock(&file->f_mutex); + block = nlmsvc_lookup_block(file, lock); + mutex_unlock(&file->f_mutex); + if (block != NULL) { + struct file_lock *fl = &block->b_call->a_args.lock.fl; + + mode = lock_to_openmode(fl); + vfs_cancel_lock(block->b_file->f_file[mode], fl); + status = nlmsvc_unlink_block(block); + nlmsvc_release_block(block); + } + return status ? nlm_lck_denied : nlm_granted; +} + +/* + * This is a callback from the filesystem for VFS file lock requests. + * It will be used if lm_grant is defined and the filesystem can not + * respond to the request immediately. + * For SETLK or SETLKW request it will get the local posix lock. + * In all cases it will move the block to the head of nlm_blocked q where + * nlmsvc_retry_blocked() can send back a reply for SETLKW or revisit the + * deferred rpc for GETLK and SETLK. + */ +static void +nlmsvc_update_deferred_block(struct nlm_block *block, int result) +{ + block->b_flags |= B_GOT_CALLBACK; + if (result == 0) + block->b_granted = 1; + else + block->b_flags |= B_TIMED_OUT; +} + +static int nlmsvc_grant_deferred(struct file_lock *fl, int result) +{ + struct nlm_block *block; + int rc = -ENOENT; + + spin_lock(&nlm_blocked_lock); + list_for_each_entry(block, &nlm_blocked, b_list) { + if (nlm_compare_locks(&block->b_call->a_args.lock.fl, fl)) { + dprintk("lockd: nlmsvc_notify_blocked block %p flags %d\n", + block, block->b_flags); + if (block->b_flags & B_QUEUED) { + if (block->b_flags & B_TIMED_OUT) { + rc = -ENOLCK; + break; + } + nlmsvc_update_deferred_block(block, result); + } else if (result == 0) + block->b_granted = 1; + + nlmsvc_insert_block_locked(block, 0); + svc_wake_up(block->b_daemon); + rc = 0; + break; + } + } + spin_unlock(&nlm_blocked_lock); + if (rc == -ENOENT) + printk(KERN_WARNING "lockd: grant for unknown block\n"); + return rc; +} + +/* + * Unblock a blocked lock request. This is a callback invoked from the + * VFS layer when a lock on which we blocked is removed. + * + * This function doesn't grant the blocked lock instantly, but rather moves + * the block to the head of nlm_blocked where it can be picked up by lockd. + */ +static void +nlmsvc_notify_blocked(struct file_lock *fl) +{ + struct nlm_block *block; + + dprintk("lockd: VFS unblock notification for block %p\n", fl); + spin_lock(&nlm_blocked_lock); + list_for_each_entry(block, &nlm_blocked, b_list) { + if (nlm_compare_locks(&block->b_call->a_args.lock.fl, fl)) { + nlmsvc_insert_block_locked(block, 0); + spin_unlock(&nlm_blocked_lock); + svc_wake_up(block->b_daemon); + return; + } + } + spin_unlock(&nlm_blocked_lock); + printk(KERN_WARNING "lockd: notification for unknown block!\n"); +} + +static fl_owner_t nlmsvc_get_owner(fl_owner_t owner) +{ + return nlmsvc_get_lockowner(owner); +} + +static void nlmsvc_put_owner(fl_owner_t owner) +{ + nlmsvc_put_lockowner(owner); +} + +const struct lock_manager_operations nlmsvc_lock_operations = { + .lm_notify = nlmsvc_notify_blocked, + .lm_grant = nlmsvc_grant_deferred, + .lm_get_owner = nlmsvc_get_owner, + .lm_put_owner = nlmsvc_put_owner, +}; + +/* + * Try to claim a lock that was previously blocked. + * + * Note that we use both the RPC_GRANTED_MSG call _and_ an async + * RPC thread when notifying the client. This seems like overkill... + * Here's why: + * - we don't want to use a synchronous RPC thread, otherwise + * we might find ourselves hanging on a dead portmapper. + * - Some lockd implementations (e.g. HP) don't react to + * RPC_GRANTED calls; they seem to insist on RPC_GRANTED_MSG calls. + */ +static void +nlmsvc_grant_blocked(struct nlm_block *block) +{ + struct nlm_file *file = block->b_file; + struct lockd_lock *lock = &block->b_call->a_args.lock; + int mode; + int error; + loff_t fl_start, fl_end; + + dprintk("lockd: grant blocked lock %p\n", block); + + kref_get(&block->b_count); + + /* Unlink block request from list */ + nlmsvc_unlink_block(block); + + /* If b_granted is true this means we've been here before. + * Just retry the grant callback, possibly refreshing the RPC + * binding */ + if (block->b_granted) { + nlm_rebind_host(block->b_host); + goto callback; + } + + /* Try the lock operation again */ + /* vfs_lock_file() can mangle fl_start and fl_end, but we need + * them unchanged for the GRANT_MSG + */ + lock->fl.c.flc_flags |= FL_SLEEP; + fl_start = lock->fl.fl_start; + fl_end = lock->fl.fl_end; + mode = lock_to_openmode(&lock->fl); + error = vfs_lock_file(file->f_file[mode], F_SETLK, &lock->fl, NULL); + lock->fl.c.flc_flags &= ~FL_SLEEP; + lock->fl.fl_start = fl_start; + lock->fl.fl_end = fl_end; + + switch (error) { + case 0: + break; + case FILE_LOCK_DEFERRED: + dprintk("lockd: lock still blocked error %d\n", error); + nlmsvc_insert_block(block, NLM_NEVER); + nlmsvc_release_block(block); + return; + default: + printk(KERN_WARNING "lockd: unexpected error %d in %s!\n", + -error, __func__); + nlmsvc_insert_block(block, 10 * HZ); + nlmsvc_release_block(block); + return; + } + +callback: + /* Lock was granted by VFS. */ + dprintk("lockd: GRANTing blocked lock.\n"); + block->b_granted = 1; + + /* keep block on the list, but don't reattempt until the RPC + * completes or the submission fails + */ + nlmsvc_insert_block(block, NLM_NEVER); + + /* Call the client -- use a soft RPC task since nlmsvc_retry_blocked + * will queue up a new one if this one times out + */ + error = nlm_async_call(block->b_call, NLMPROC_GRANTED_MSG, + &nlmsvc_grant_ops); + + /* RPC submission failed, wait a bit and retry */ + if (error < 0) + nlmsvc_insert_block(block, 10 * HZ); +} + +/* + * This is the callback from the RPC layer when the NLM_GRANTED_MSG + * RPC call has succeeded or timed out. + * Like all RPC callbacks, it is invoked by the rpciod process, so it + * better not sleep. Therefore, we put the blocked lock on the nlm_blocked + * chain once more in order to have it removed by lockd itself (which can + * then sleep on the file semaphore without disrupting e.g. the nfs client). + */ +static void nlmsvc_grant_callback(struct rpc_task *task, void *data) +{ + struct nlm_rqst *call = data; + struct nlm_block *block = call->a_block; + unsigned long timeout; + + dprintk("lockd: GRANT_MSG RPC callback\n"); + + spin_lock(&nlm_blocked_lock); + /* if the block is not on a list at this point then it has + * been invalidated. Don't try to requeue it. + * + * FIXME: it's possible that the block is removed from the list + * after this check but before the nlmsvc_insert_block. In that + * case it will be added back. Perhaps we need better locking + * for nlm_blocked? + */ + if (list_empty(&block->b_list)) + goto out; + + /* Technically, we should down the file semaphore here. Since we + * move the block towards the head of the queue only, no harm + * can be done, though. */ + if (task->tk_status < 0) { + /* RPC error: Re-insert for retransmission */ + timeout = 10 * HZ; + } else { + /* Call was successful, now wait for client callback */ + timeout = 60 * HZ; + } + nlmsvc_insert_block_locked(block, timeout); + svc_wake_up(block->b_daemon); +out: + spin_unlock(&nlm_blocked_lock); +} + +/* + * FIXME: nlmsvc_release_block() grabs a mutex. This is not allowed for an + * .rpc_release rpc_call_op + */ +static void nlmsvc_grant_release(void *data) +{ + struct nlm_rqst *call = data; + nlmsvc_release_block(call->a_block); +} + +static const struct rpc_call_ops nlmsvc_grant_ops = { + .rpc_call_done = nlmsvc_grant_callback, + .rpc_release = nlmsvc_grant_release, +}; + +/* + * We received a GRANT_RES callback. Try to find the corresponding + * block. + */ +void +nlmsvc_grant_reply(struct lockd_cookie *cookie, __be32 status) +{ + struct nlm_block *block; + struct file_lock *fl; + int error; + + dprintk("grant_reply: looking for cookie %x, s=%d\n", + *(unsigned int *)(cookie->data), status); + if (!(block = nlmsvc_find_block(cookie))) + return; + + switch (status) { + case nlm_lck_denied_grace_period: + /* Try again in a couple of seconds */ + nlmsvc_insert_block(block, 10 * HZ); + break; + case nlm_lck_denied: + /* Client doesn't want it, just unlock it */ + nlmsvc_unlink_block(block); + fl = &block->b_call->a_args.lock.fl; + fl->c.flc_type = F_UNLCK; + error = vfs_lock_file(fl->c.flc_file, F_SETLK, fl, NULL); + if (error) + pr_warn("lockd: unable to unlock lock rejected by client!\n"); + break; + default: + /* + * Either it was accepted or the status makes no sense + * just unlink it either way. + */ + nlmsvc_unlink_block(block); + } + nlmsvc_release_block(block); +} + +/* Helper function to handle retry of a deferred block. + * If it is a blocking lock, call grant_blocked. + * For a non-blocking lock or test lock, revisit the request. + */ +static void +retry_deferred_block(struct nlm_block *block) +{ + if (!(block->b_flags & B_GOT_CALLBACK)) + block->b_flags |= B_TIMED_OUT; + nlmsvc_insert_block(block, NLM_TIMEOUT); + dprintk("revisit block %p flags %d\n", block, block->b_flags); + if (block->b_deferred_req) { + block->b_deferred_req->revisit(block->b_deferred_req, 0); + block->b_deferred_req = NULL; + } +} + +/* + * Retry all blocked locks that have been notified. This is where lockd + * picks up locks that can be granted, or grant notifications that must + * be retransmitted. + */ +void +nlmsvc_retry_blocked(struct svc_rqst *rqstp) +{ + unsigned long timeout = MAX_SCHEDULE_TIMEOUT; + struct nlm_block *block; + + spin_lock(&nlm_blocked_lock); + while (!list_empty(&nlm_blocked) && !svc_thread_should_stop(rqstp)) { + block = list_entry(nlm_blocked.next, struct nlm_block, b_list); + + if (block->b_when == NLM_NEVER) + break; + if (time_after(block->b_when, jiffies)) { + timeout = block->b_when - jiffies; + break; + } + spin_unlock(&nlm_blocked_lock); + + dprintk("nlmsvc_retry_blocked(%p, when=%ld)\n", + block, block->b_when); + if (block->b_flags & B_QUEUED) { + dprintk("nlmsvc_retry_blocked delete block (%p, granted=%d, flags=%d)\n", + block, block->b_granted, block->b_flags); + retry_deferred_block(block); + } else + nlmsvc_grant_blocked(block); + spin_lock(&nlm_blocked_lock); + } + spin_unlock(&nlm_blocked_lock); + + if (timeout < MAX_SCHEDULE_TIMEOUT) + mod_timer(&nlmsvc_retry, jiffies + timeout); +} diff --git a/fs/lockd/svcproc.c b/fs/lockd/svcproc.c new file mode 100644 index 000000000..a8b5195c2 --- /dev/null +++ b/fs/lockd/svcproc.c @@ -0,0 +1,1453 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/lockd/svcproc.c + * + * Lockd server procedures. We don't implement the NLM_*_RES + * procedures because we don't use the async procedures. + * + * Copyright (C) 1996, Olaf Kirch + */ + +#include +#include +#include + +#include "lockd.h" + +/* + * xdr.h defines SM_PRIV_SIZE as a macro. nlm3xdr_gen.h defines + * it as an enum constant. Undefine the macro before including + * the generated header. + */ +#undef SM_PRIV_SIZE + +#include "share.h" +#include "nlm3xdr_gen.h" + +/* + * Size of an NFSv2 file handle, in bytes, which is 32. + * Defined locally to avoid including uapi/linux/nfs2.h. + */ +#define NLM3_FHSIZE 32 + +/* + * Wrapper structures combine xdrgen types with legacy lockd_lock. + * The xdrgen field must be first so the structure can be cast + * to its XDR type for the RPC dispatch layer. + */ +struct nlm_testargs_wrapper { + struct nlm_testargs xdrgen; + struct lockd_lock lock; +}; + +static_assert(offsetof(struct nlm_testargs_wrapper, xdrgen) == 0); + +struct nlm_testres_wrapper { + struct nlm_testres xdrgen; + struct lockd_lock lock; +}; + +static_assert(offsetof(struct nlm_testres_wrapper, xdrgen) == 0); + +struct nlm_lockargs_wrapper { + struct nlm_lockargs xdrgen; + struct lockd_cookie cookie; + struct lockd_lock lock; +}; + +static_assert(offsetof(struct nlm_lockargs_wrapper, xdrgen) == 0); + +struct nlm_res_wrapper { + struct nlm_res xdrgen; + struct lockd_cookie cookie; +}; + +static_assert(offsetof(struct nlm_res_wrapper, xdrgen) == 0); + +struct nlm_cancargs_wrapper { + struct nlm_cancargs xdrgen; + struct lockd_lock lock; +}; + +static_assert(offsetof(struct nlm_cancargs_wrapper, xdrgen) == 0); + +struct nlm_unlockargs_wrapper { + struct nlm_unlockargs xdrgen; + struct lockd_lock lock; +}; + +static_assert(offsetof(struct nlm_unlockargs_wrapper, xdrgen) == 0); + +struct nlm_notifyargs_wrapper { + struct nlm_notifyargs xdrgen; + struct lockd_reboot reboot; +}; + +static_assert(offsetof(struct nlm_notifyargs_wrapper, xdrgen) == 0); + +struct nlm_shareargs_wrapper { + struct nlm_shareargs xdrgen; + struct lockd_lock lock; +}; + +static_assert(offsetof(struct nlm_shareargs_wrapper, xdrgen) == 0); + +struct nlm_shareres_wrapper { + struct nlm_shareres xdrgen; +}; + +static_assert(offsetof(struct nlm_shareres_wrapper, xdrgen) == 0); + +struct nlm_notify_wrapper { + struct nlm_notify xdrgen; +}; + +static_assert(offsetof(struct nlm_notify_wrapper, xdrgen) == 0); + +static __be32 +nlm_netobj_to_cookie(struct lockd_cookie *cookie, netobj *object) +{ + if (object->len > NLM_MAXCOOKIELEN) + return nlm_lck_denied_nolocks; + cookie->len = object->len; + memcpy(cookie->data, object->data, object->len); + return nlm_granted; +} + +static __be32 +nlm_lock_to_lockd_lock(struct lockd_lock *lock, struct nlm_lock *alock) +{ + if (alock->fh.len != NLM3_FHSIZE) + return nlm_lck_denied; + lock->fh.size = alock->fh.len; + memcpy(lock->fh.data, alock->fh.data, alock->fh.len); + lock->oh.len = alock->oh.len; + lock->oh.data = alock->oh.data; + lock->svid = alock->uppid; + lockd_set_file_lock_range3(&lock->fl, alock->l_offset, alock->l_len); + return nlm_granted; +} + +static struct nlm_host * +nlm3svc_lookup_host(struct svc_rqst *rqstp, string caller, bool monitored) +{ + struct nlm_host *host; + + if (!rcu_access_pointer(nlmsvc_ops)) + return NULL; + host = nlmsvc_lookup_host(rqstp, caller.data, caller.len); + if (!host) + return NULL; + if (monitored && nsm_monitor(host) < 0) { + nlmsvc_release_host(host); + return NULL; + } + return host; +} + +static __be32 +nlm3svc_lookup_file(struct svc_rqst *rqstp, struct nlm_host *host, + struct lockd_lock *lock, struct nlm_file **filp, + struct nlm_lock *xdr_lock, unsigned char type) +{ + bool is_test = (rqstp->rq_proc == NLMPROC_TEST || + rqstp->rq_proc == NLMPROC_TEST_MSG); + struct file_lock *fl = &lock->fl; + struct nlm_file *file = NULL; + __be32 error; + int mode; + + if (xdr_lock->fh.len != NLM3_FHSIZE) + return nlm_lck_denied_nolocks; + lock->fh.size = xdr_lock->fh.len; + memcpy(lock->fh.data, xdr_lock->fh.data, xdr_lock->fh.len); + + lock->oh.len = xdr_lock->oh.len; + lock->oh.data = xdr_lock->oh.data; + + lock->svid = xdr_lock->uppid; + lock->lock_start = xdr_lock->l_offset; + lock->lock_len = xdr_lock->l_len; + + locks_init_lock(fl); + fl->c.flc_type = type; + lockd_set_file_lock_range3(fl, lock->lock_start, lock->lock_len); + + mode = lock_to_openmode(fl); + if (is_test) + mode = O_RDWR; + + error = nlm_lookup_file(rqstp, &file, lock, mode); + switch (error) { + case nlm_granted: + break; + case nlm__int__stale_fh: + case nlm__int__failed: + return nlm_lck_denied_nolocks; + default: + return error; + } + *filp = file; + + fl->c.flc_flags = FL_POSIX; + if (is_test) + fl->c.flc_file = nlmsvc_file_file(file); + else + fl->c.flc_file = file->f_file[mode]; + fl->c.flc_pid = current->tgid; + fl->fl_lmops = &nlmsvc_lock_operations; + nlmsvc_locks_init_private(fl, host, (pid_t)lock->svid); + if (!fl->c.flc_owner) + return nlm_lck_denied_nolocks; + + return nlm_granted; +} + +static inline __be32 cast_status(__be32 status) +{ + switch (status) { + case nlm_granted: + case nlm_lck_denied: + case nlm_lck_denied_nolocks: + case nlm_lck_blocked: + case nlm_lck_denied_grace_period: + case nlm__int__drop_reply: + break; + case nlm__int__deadlock: + status = nlm_lck_denied; + break; + default: + pr_warn_once("lockd: unhandled internal status %u\n", + be32_to_cpu(status)); + status = nlm_lck_denied_nolocks; + break; + } + return status; +} + +/** + * nlmsvc_proc_null - NULL: Test for presence of service + * @rqstp: RPC transaction context + * + * Return: + * %rpc_success: RPC executed successfully + * + * RPC synopsis: + * void NLM_NULL(void) = 0; + */ +static __be32 nlmsvc_proc_null(struct svc_rqst *rqstp) +{ + return rpc_success; +} + +/** + * nlmsvc_proc_test - TEST: Check for conflicting lock + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * + * RPC synopsis: + * nlm_testres NLM_TEST(nlm_testargs) = 1; + * + * Permissible procedure status codes: + * %LCK_GRANTED: The server would be able to grant the + * requested lock. + * %LCK_DENIED: The requested lock conflicted with existing + * lock reservations for the file. + * %LCK_DENIED_NOLOCKS: The server could not allocate the resources + * needed to process the request. + * %LCK_DENIED_GRACE_PERIOD: The server has recently restarted and is + * re-establishing existing locks, and is not + * yet ready to accept normal service requests. + */ +static __be32 nlmsvc_proc_test(struct svc_rqst *rqstp) +{ + struct nlm_testargs_wrapper *argp = rqstp->rq_argp; + unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK; + struct nlm_testres_wrapper *resp = rqstp->rq_resp; + struct nlm_file *file = NULL; + struct nlm_host *host; + + resp->xdrgen.cookie = argp->xdrgen.cookie; + + resp->xdrgen.test_stat.stat = nlm_lck_denied_nolocks; + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + goto out; + + resp->xdrgen.test_stat.stat = + nlm3svc_lookup_file(rqstp, host, &argp->lock, &file, + &argp->xdrgen.alock, type); + if (resp->xdrgen.test_stat.stat) + goto out; + + resp->xdrgen.test_stat.stat = + cast_status(nlmsvc_testlock(rqstp, file, host, &argp->lock, + &resp->lock)); + nlmsvc_release_lockowner(&argp->lock); + + if (resp->xdrgen.test_stat.stat == nlm_lck_denied) { + struct lockd_lock *conf = &resp->lock; + struct nlm_holder *holder = &resp->xdrgen.test_stat.u.holder; + + holder->exclusive = (conf->fl.c.flc_type != F_RDLCK); + holder->uppid = conf->svid; + holder->oh.len = conf->oh.len; + holder->oh.data = conf->oh.data; + holder->l_offset = min_t(loff_t, conf->fl.fl_start, + NLM_OFFSET_MAX); + if (conf->fl.fl_end == OFFSET_MAX) + holder->l_len = 0; + else + holder->l_len = min_t(loff_t, + conf->fl.fl_end - + conf->fl.fl_start + 1, + NLM_OFFSET_MAX); + } + +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->xdrgen.test_stat.stat == nlm__int__drop_reply ? + rpc_drop_reply : rpc_success; +} + +static __be32 +nlmsvc_do_lock(struct svc_rqst *rqstp, bool monitored) +{ + struct nlm_lockargs_wrapper *argp = rqstp->rq_argp; + unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK; + struct nlm_res_wrapper *resp = rqstp->rq_resp; + struct nlm_file *file = NULL; + struct nlm_host *host = NULL; + + resp->xdrgen.cookie = argp->xdrgen.cookie; + + resp->xdrgen.stat.stat = nlm_netobj_to_cookie(&argp->cookie, + &argp->xdrgen.cookie); + if (resp->xdrgen.stat.stat) + goto out; + + resp->xdrgen.stat.stat = nlm_lck_denied_nolocks; + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, + monitored); + if (!host) + goto out; + + resp->xdrgen.stat.stat = nlm3svc_lookup_file(rqstp, host, &argp->lock, + &file, &argp->xdrgen.alock, + type); + if (resp->xdrgen.stat.stat) + goto out; + + resp->xdrgen.stat.stat = cast_status(nlmsvc_lock(rqstp, file, host, + &argp->lock, + argp->xdrgen.block, + &argp->cookie, + argp->xdrgen.reclaim)); + + nlmsvc_release_lockowner(&argp->lock); + +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->xdrgen.stat.stat == nlm__int__drop_reply ? + rpc_drop_reply : rpc_success; +} + +/** + * nlmsvc_proc_lock - LOCK: Establish a monitored lock + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * + * RPC synopsis: + * nlm_res NLM_LOCK(nlm_lockargs) = 2; + * + * Permissible procedure status codes: + * %LCK_GRANTED: The requested lock was granted. + * %LCK_DENIED: The requested lock conflicted with existing + * lock reservations for the file. + * %LCK_DENIED_NOLOCKS: The server could not allocate the resources + * needed to process the request. + * %LCK_BLOCKED: The blocking request cannot be granted + * immediately. The server will send an + * NLM_GRANTED callback to the client when + * the lock can be granted. + * %LCK_DENIED_GRACE_PERIOD: The server has recently restarted and is + * re-establishing existing locks, and is not + * yet ready to accept normal service requests. + */ +static __be32 +nlmsvc_proc_lock(struct svc_rqst *rqstp) +{ + return nlmsvc_do_lock(rqstp, true); +} + +/** + * nlmsvc_proc_cancel - CANCEL: Cancel an outstanding blocked lock request + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * + * RPC synopsis: + * nlm_res NLM_CANCEL(nlm_cancargs) = 3; + * + * Permissible procedure status codes: + * %LCK_GRANTED: The requested lock was canceled. + * %LCK_DENIED: There was no lock to cancel. + * %LCK_DENIED_GRACE_PERIOD: The server has recently restarted and is + * re-establishing existing locks, and is not + * yet ready to accept normal service requests. + * + * The Linux NLM server implementation also returns: + * %LCK_DENIED_NOLOCKS: A needed resource could not be allocated. + */ +static __be32 +nlmsvc_proc_cancel(struct svc_rqst *rqstp) +{ + struct nlm_cancargs_wrapper *argp = rqstp->rq_argp; + unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK; + struct nlm_res_wrapper *resp = rqstp->rq_resp; + struct net *net = SVC_NET(rqstp); + struct nlm_host *host = NULL; + struct nlm_file *file = NULL; + + resp->xdrgen.cookie = argp->xdrgen.cookie; + + resp->xdrgen.stat.stat = nlm_lck_denied_grace_period; + if (locks_in_grace(net)) + goto out; + + resp->xdrgen.stat.stat = nlm_lck_denied_nolocks; + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + goto out; + + resp->xdrgen.stat.stat = nlm3svc_lookup_file(rqstp, host, &argp->lock, + &file, &argp->xdrgen.alock, + type); + if (resp->xdrgen.stat.stat) + goto out; + + resp->xdrgen.stat.stat = nlmsvc_cancel_blocked(net, file, &argp->lock); + nlmsvc_release_lockowner(&argp->lock); + +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->xdrgen.stat.stat == nlm__int__drop_reply ? + rpc_drop_reply : rpc_success; +} + +/** + * nlmsvc_proc_unlock - UNLOCK: Remove a lock + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * + * RPC synopsis: + * nlm_res NLM_UNLOCK(nlm_unlockargs) = 4; + * + * Permissible procedure status codes: + * %LCK_GRANTED: The requested lock was released. + * %LCK_DENIED_GRACE_PERIOD: The server has recently restarted and is + * re-establishing existing locks, and is not + * yet ready to accept normal service requests. + * + * The Linux NLM server implementation also returns: + * %LCK_DENIED_NOLOCKS: A needed resource could not be allocated. + */ +static __be32 +nlmsvc_proc_unlock(struct svc_rqst *rqstp) +{ + struct nlm_unlockargs_wrapper *argp = rqstp->rq_argp; + struct nlm_res_wrapper *resp = rqstp->rq_resp; + struct net *net = SVC_NET(rqstp); + struct nlm_host *host = NULL; + struct nlm_file *file = NULL; + + resp->xdrgen.cookie = argp->xdrgen.cookie; + + resp->xdrgen.stat.stat = nlm_lck_denied_grace_period; + if (locks_in_grace(net)) + goto out; + + resp->xdrgen.stat.stat = nlm_lck_denied_nolocks; + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + goto out; + + resp->xdrgen.stat.stat = nlm3svc_lookup_file(rqstp, host, &argp->lock, + &file, &argp->xdrgen.alock, + F_UNLCK); + if (resp->xdrgen.stat.stat) + goto out; + + resp->xdrgen.stat.stat = nlmsvc_unlock(net, file, &argp->lock); + nlmsvc_release_lockowner(&argp->lock); + +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->xdrgen.stat.stat == nlm__int__drop_reply ? + rpc_drop_reply : rpc_success; +} + +/** + * nlmsvc_proc_granted - GRANTED: Blocked lock has been granted + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * + * RPC synopsis: + * nlm_res NLM_GRANTED(nlm_testargs) = 5; + * + * Permissible procedure status codes: + * %LCK_GRANTED: The granted lock was accepted. + * %LCK_DENIED: The procedure failed, possibly due to + * internal resource constraints. + * %LCK_DENIED_GRACE_PERIOD: The client host recently restarted and + * its NLM is re-establishing existing locks, + * so it is not yet ready to accept callbacks. + */ +static __be32 +nlmsvc_proc_granted(struct svc_rqst *rqstp) +{ + struct nlm_testargs_wrapper *argp = rqstp->rq_argp; + struct nlm_res_wrapper *resp = rqstp->rq_resp; + + resp->xdrgen.cookie = argp->xdrgen.cookie; + + resp->xdrgen.stat.stat = nlm_lock_to_lockd_lock(&argp->lock, + &argp->xdrgen.alock); + if (resp->xdrgen.stat.stat) + goto out; + + resp->xdrgen.stat.stat = nlmclnt_grant(svc_addr(rqstp), &argp->lock); + +out: + return rpc_success; +} + +/* + * This is the generic lockd callback for async RPC calls + */ +static void nlmsvc_callback_exit(struct rpc_task *task, void *data) +{ +} + +void nlmsvc_release_call(struct nlm_rqst *call) +{ + if (!refcount_dec_and_test(&call->a_count)) + return; + nlmsvc_release_host(call->a_host); + kfree(call); +} + +static void nlmsvc_callback_release(void *data) +{ + nlmsvc_release_call(data); +} + +static const struct rpc_call_ops nlmsvc_callback_ops = { + .rpc_call_done = nlmsvc_callback_exit, + .rpc_release = nlmsvc_callback_release, +}; + +/* + * `Async' versions of the above service routines. They aren't really, + * because we send the callback before the reply proper. I hope this + * doesn't break any clients. + */ +static __be32 +nlmsvc_callback(struct svc_rqst *rqstp, struct nlm_host *host, u32 proc, + __be32 (*func)(struct svc_rqst *, struct lockd_res *)) +{ + struct nlm_rqst *call; + __be32 stat; + + call = nlm_alloc_call(host); + nlmsvc_release_host(host); + if (call == NULL) + return rpc_system_err; + + stat = func(rqstp, &call->a_res); + if (stat != 0) { + nlmsvc_release_call(call); + return stat; + } + + call->a_flags = RPC_TASK_ASYNC; + if (nlm_async_reply(call, proc, &nlmsvc_callback_ops) < 0) + return rpc_system_err; + return rpc_success; +} + +static __be32 +__nlmsvc_proc_test_msg(struct svc_rqst *rqstp, struct lockd_res *resp) +{ + struct nlm_testargs_wrapper *argp = rqstp->rq_argp; + unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK; + struct nlm_lockowner *owner; + struct nlm_file *file = NULL; + struct nlm_host *host = NULL; + + resp->status = nlm_lck_denied_nolocks; + if (nlm_netobj_to_cookie(&resp->cookie, &argp->xdrgen.cookie)) + goto out; + + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + goto out; + + resp->status = nlm3svc_lookup_file(rqstp, host, &argp->lock, + &file, &argp->xdrgen.alock, type); + if (resp->status) + goto out; + + owner = argp->lock.fl.c.flc_owner; + resp->status = cast_status(nlmsvc_testlock(rqstp, file, host, + &argp->lock, &resp->lock)); + nlmsvc_put_lockowner(owner); + +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->status == nlm__int__drop_reply ? rpc_drop_reply : rpc_success; +} + +/** + * nlmsvc_proc_test_msg - TEST_MSG: Check for conflicting lock + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * %rpc_garbage_args: The request arguments are malformed. + * %rpc_system_err: RPC execution failed. + * + * RPC synopsis: + * void NLMPROC_TEST_MSG(nlm_testargs) = 6; + * + * The response to this request is delivered via the TEST_RES procedure. + */ +static __be32 nlmsvc_proc_test_msg(struct svc_rqst *rqstp) +{ + struct nlm_testargs_wrapper *argp = rqstp->rq_argp; + struct nlm_host *host; + + if (argp->xdrgen.cookie.len > NLM_MAXCOOKIELEN) + return rpc_garbage_args; + + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + return rpc_system_err; + + return nlmsvc_callback(rqstp, host, NLMPROC_TEST_RES, + __nlmsvc_proc_test_msg); +} + +static __be32 +__nlmsvc_proc_lock_msg(struct svc_rqst *rqstp, struct lockd_res *resp) +{ + struct nlm_lockargs_wrapper *argp = rqstp->rq_argp; + unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK; + struct nlm_file *file = NULL; + struct nlm_host *host = NULL; + + resp->status = nlm_lck_denied_nolocks; + if (nlm_netobj_to_cookie(&resp->cookie, &argp->xdrgen.cookie)) + goto out; + + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, true); + if (!host) + goto out; + + resp->status = nlm3svc_lookup_file(rqstp, host, &argp->lock, + &file, &argp->xdrgen.alock, type); + if (resp->status) + goto out; + + resp->status = cast_status(nlmsvc_lock(rqstp, file, host, &argp->lock, + argp->xdrgen.block, &resp->cookie, + argp->xdrgen.reclaim)); + nlmsvc_release_lockowner(&argp->lock); + +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->status == nlm__int__drop_reply ? rpc_drop_reply : rpc_success; +} + +/** + * nlmsvc_proc_lock_msg - LOCK_MSG: Establish a monitored lock + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * %rpc_garbage_args: The request arguments are malformed. + * %rpc_system_err: RPC execution failed. + * + * RPC synopsis: + * void NLMPROC_LOCK_MSG(nlm_lockargs) = 7; + * + * The response to this request is delivered via the LOCK_RES procedure. + */ +static __be32 nlmsvc_proc_lock_msg(struct svc_rqst *rqstp) +{ + struct nlm_lockargs_wrapper *argp = rqstp->rq_argp; + struct nlm_host *host; + + if (argp->xdrgen.cookie.len > NLM_MAXCOOKIELEN) + return rpc_garbage_args; + + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + return rpc_system_err; + + return nlmsvc_callback(rqstp, host, NLMPROC_LOCK_RES, + __nlmsvc_proc_lock_msg); +} + +static __be32 +__nlmsvc_proc_cancel_msg(struct svc_rqst *rqstp, struct lockd_res *resp) +{ + struct nlm_cancargs_wrapper *argp = rqstp->rq_argp; + unsigned char type = argp->xdrgen.exclusive ? F_WRLCK : F_RDLCK; + struct net *net = SVC_NET(rqstp); + struct nlm_file *file = NULL; + struct nlm_host *host = NULL; + + resp->status = nlm_lck_denied_nolocks; + if (nlm_netobj_to_cookie(&resp->cookie, &argp->xdrgen.cookie)) + goto out; + + resp->status = nlm_lck_denied_grace_period; + if (locks_in_grace(net)) + goto out; + + resp->status = nlm_lck_denied_nolocks; + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + goto out; + + resp->status = nlm3svc_lookup_file(rqstp, host, &argp->lock, + &file, &argp->xdrgen.alock, type); + if (resp->status) + goto out; + + resp->status = nlmsvc_cancel_blocked(net, file, &argp->lock); + nlmsvc_release_lockowner(&argp->lock); + +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->status == nlm__int__drop_reply ? rpc_drop_reply : rpc_success; +} + +/** + * nlmsvc_proc_cancel_msg - CANCEL_MSG: Cancel an outstanding lock request + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * %rpc_garbage_args: The request arguments are malformed. + * %rpc_system_err: RPC execution failed. + * + * RPC synopsis: + * void NLMPROC_CANCEL_MSG(nlm_cancargs) = 8; + * + * The response to this request is delivered via the CANCEL_RES procedure. + */ +static __be32 nlmsvc_proc_cancel_msg(struct svc_rqst *rqstp) +{ + struct nlm_cancargs_wrapper *argp = rqstp->rq_argp; + struct nlm_host *host; + + if (argp->xdrgen.cookie.len > NLM_MAXCOOKIELEN) + return rpc_garbage_args; + + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + return rpc_system_err; + + return nlmsvc_callback(rqstp, host, NLMPROC_CANCEL_RES, + __nlmsvc_proc_cancel_msg); +} + +static __be32 +__nlmsvc_proc_unlock_msg(struct svc_rqst *rqstp, struct lockd_res *resp) +{ + struct nlm_unlockargs_wrapper *argp = rqstp->rq_argp; + struct net *net = SVC_NET(rqstp); + struct nlm_file *file = NULL; + struct nlm_host *host = NULL; + + resp->status = nlm_lck_denied_nolocks; + if (nlm_netobj_to_cookie(&resp->cookie, &argp->xdrgen.cookie)) + goto out; + + resp->status = nlm_lck_denied_grace_period; + if (locks_in_grace(net)) + goto out; + + resp->status = nlm_lck_denied_nolocks; + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + goto out; + + resp->status = nlm3svc_lookup_file(rqstp, host, &argp->lock, + &file, &argp->xdrgen.alock, F_UNLCK); + if (resp->status) + goto out; + + resp->status = nlmsvc_unlock(net, file, &argp->lock); + nlmsvc_release_lockowner(&argp->lock); + +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->status == nlm__int__drop_reply ? rpc_drop_reply : rpc_success; +} + +/** + * nlmsvc_proc_unlock_msg - UNLOCK_MSG: Remove an existing lock + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * %rpc_garbage_args: The request arguments are malformed. + * %rpc_system_err: RPC execution failed. + * + * RPC synopsis: + * void NLMPROC_UNLOCK_MSG(nlm_unlockargs) = 9; + * + * The response to this request is delivered via the UNLOCK_RES procedure. + */ +static __be32 nlmsvc_proc_unlock_msg(struct svc_rqst *rqstp) +{ + struct nlm_unlockargs_wrapper *argp = rqstp->rq_argp; + struct nlm_host *host; + + if (argp->xdrgen.cookie.len > NLM_MAXCOOKIELEN) + return rpc_garbage_args; + + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.alock.caller_name, false); + if (!host) + return rpc_system_err; + + return nlmsvc_callback(rqstp, host, NLMPROC_UNLOCK_RES, + __nlmsvc_proc_unlock_msg); +} + +static __be32 +__nlmsvc_proc_granted_msg(struct svc_rqst *rqstp, struct lockd_res *resp) +{ + struct nlm_testargs_wrapper *argp = rqstp->rq_argp; + + resp->status = nlm_lck_denied; + if (nlm_netobj_to_cookie(&resp->cookie, &argp->xdrgen.cookie)) + goto out; + + if (nlm_lock_to_lockd_lock(&argp->lock, &argp->xdrgen.alock)) + goto out; + + resp->status = nlmclnt_grant(svc_addr(rqstp), &argp->lock); + +out: + return rpc_success; +} + +/** + * nlmsvc_proc_granted_msg - GRANTED_MSG: Blocked lock has been granted + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_garbage_args: The request arguments are malformed. + * %rpc_system_err: RPC execution failed. + * + * RPC synopsis: + * void NLMPROC_GRANTED_MSG(nlm_testargs) = 10; + * + * The response to this request is delivered via the GRANTED_RES procedure. + */ +static __be32 nlmsvc_proc_granted_msg(struct svc_rqst *rqstp) +{ + struct nlm_testargs_wrapper *argp = rqstp->rq_argp; + struct nlm_host *host; + + if (argp->xdrgen.cookie.len > NLM_MAXCOOKIELEN) + return rpc_garbage_args; + + host = nlmsvc_lookup_host(rqstp, argp->xdrgen.alock.caller_name.data, + argp->xdrgen.alock.caller_name.len); + if (!host) + return rpc_system_err; + + return nlmsvc_callback(rqstp, host, NLMPROC_GRANTED_RES, + __nlmsvc_proc_granted_msg); +} + +/** + * nlmsvc_proc_granted_res - GRANTED_RES: Lock Granted result + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * + * RPC synopsis: + * void NLMPROC_GRANTED_RES(nlm_res) = 15; + */ +static __be32 nlmsvc_proc_granted_res(struct svc_rqst *rqstp) +{ + struct nlm_res_wrapper *argp = rqstp->rq_argp; + + if (!rcu_access_pointer(nlmsvc_ops)) + return rpc_success; + + if (nlm_netobj_to_cookie(&argp->cookie, &argp->xdrgen.cookie)) + return rpc_success; + nlmsvc_grant_reply(&argp->cookie, argp->xdrgen.stat.stat); + + return rpc_success; +} + +/** + * nlmsvc_proc_sm_notify - SM_NOTIFY: Peer has rebooted + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_system_err: RPC execution failed. + * + * The SM_NOTIFY procedure is a private callback from Linux statd and is + * not part of the official NLM protocol. + * + * RPC synopsis: + * void NLM_SM_NOTIFY(nlm_notifyargs) = 16; + */ +static __be32 nlmsvc_proc_sm_notify(struct svc_rqst *rqstp) +{ + struct nlm_notifyargs_wrapper *argp = rqstp->rq_argp; + struct lockd_reboot *reboot = &argp->reboot; + + if (!nlm_privileged_requester(rqstp)) { + char buf[RPC_MAX_ADDRBUFLEN]; + + pr_warn("lockd: rejected NSM callback from %s\n", + svc_print_addr(rqstp, buf, sizeof(buf))); + return rpc_system_err; + } + + reboot->len = argp->xdrgen.notify.name.len; + reboot->mon = (char *)argp->xdrgen.notify.name.data; + reboot->state = argp->xdrgen.notify.state; + memcpy(&reboot->priv.data, argp->xdrgen.private, + sizeof(reboot->priv.data)); + + nlm_host_rebooted(SVC_NET(rqstp), reboot); + + return rpc_success; +} + +/** + * nlmsvc_proc_unused - stub for unused procedures + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_proc_unavail: Program can't support procedure. + */ +static __be32 nlmsvc_proc_unused(struct svc_rqst *rqstp) +{ + return rpc_proc_unavail; +} + +/** + * nlmsvc_proc_share - SHARE: Open a file using DOS file-sharing modes + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * + * RPC synopsis: + * nlm_shareres NLM_SHARE(nlm_shareargs) = 20; + * + * Permissible procedure status codes: + * %LCK_GRANTED: The requested share lock was granted. + * %LCK_DENIED: The requested lock conflicted with existing + * lock reservations for the file. + * %LCK_DENIED_GRACE_PERIOD: The server has recently restarted and is + * re-establishing existing locks, and is not + * yet ready to accept normal service requests. + * + * The Linux NLM server implementation also returns: + * %LCK_DENIED_NOLOCKS: A needed resource could not be allocated. + */ +static __be32 nlmsvc_proc_share(struct svc_rqst *rqstp) +{ + struct nlm_shareargs_wrapper *argp = rqstp->rq_argp; + struct nlm_shareres_wrapper *resp = rqstp->rq_resp; + struct lockd_lock *lock = &argp->lock; + struct nlm_host *host = NULL; + struct nlm_file *file = NULL; + struct nlm_lock xdr_lock = { + .fh = argp->xdrgen.share.fh, + .oh = argp->xdrgen.share.oh, + .uppid = LOCKD_SHARE_SVID, + }; + + resp->xdrgen.cookie = argp->xdrgen.cookie; + + resp->xdrgen.stat = nlm_lck_denied_grace_period; + if (locks_in_grace(SVC_NET(rqstp)) && !argp->xdrgen.reclaim) + goto out; + + resp->xdrgen.stat = nlm_lck_denied_nolocks; + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.share.caller_name, false); + if (!host) + goto out; + + resp->xdrgen.stat = nlm3svc_lookup_file(rqstp, host, lock, &file, + &xdr_lock, F_RDLCK); + if (resp->xdrgen.stat) + goto out; + + resp->xdrgen.stat = nlmsvc_share_file(host, file, &lock->oh, + argp->xdrgen.share.access, + argp->xdrgen.share.mode); + + nlmsvc_release_lockowner(lock); + +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->xdrgen.stat == nlm__int__drop_reply ? + rpc_drop_reply : rpc_success; +} + +/** + * nlmsvc_proc_unshare - UNSHARE: Release a share reservation + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * + * RPC synopsis: + * nlm_shareres NLM_UNSHARE(nlm_shareargs) = 21; + * + * Permissible procedure status codes: + * %LCK_GRANTED: The share reservation was released. + * %LCK_DENIED_GRACE_PERIOD: The server has recently restarted and is + * re-establishing existing locks, and is not + * yet ready to accept normal service requests. + * + * The Linux NLM server implementation also returns: + * %LCK_DENIED_NOLOCKS: A needed resource could not be allocated. + */ +static __be32 nlmsvc_proc_unshare(struct svc_rqst *rqstp) +{ + struct nlm_shareargs_wrapper *argp = rqstp->rq_argp; + struct nlm_shareres_wrapper *resp = rqstp->rq_resp; + struct lockd_lock *lock = &argp->lock; + struct nlm_lock xdr_lock = { + .fh = argp->xdrgen.share.fh, + .oh = argp->xdrgen.share.oh, + .uppid = LOCKD_SHARE_SVID, + }; + struct nlm_host *host = NULL; + struct nlm_file *file = NULL; + + resp->xdrgen.cookie = argp->xdrgen.cookie; + + resp->xdrgen.stat = nlm_lck_denied_grace_period; + if (locks_in_grace(SVC_NET(rqstp))) + goto out; + + resp->xdrgen.stat = nlm_lck_denied_nolocks; + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.share.caller_name, false); + if (!host) + goto out; + + resp->xdrgen.stat = nlm3svc_lookup_file(rqstp, host, lock, &file, + &xdr_lock, F_RDLCK); + if (resp->xdrgen.stat) + goto out; + + resp->xdrgen.stat = nlmsvc_unshare_file(host, file, &lock->oh, + argp->xdrgen.share.access, + argp->xdrgen.share.mode); + + nlmsvc_release_lockowner(lock); + +out: + if (file) + nlm_release_file(file); + nlmsvc_release_host(host); + return resp->xdrgen.stat == nlm__int__drop_reply ? + rpc_drop_reply : rpc_success; +} + +/** + * nlmsvc_proc_nm_lock - NM_LOCK: Establish a non-monitored lock + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * %rpc_drop_reply: Do not send an RPC reply. + * + * RPC synopsis: + * nlm_res NLM_NM_LOCK(nlm_lockargs) = 22; + * + * Permissible procedure status codes: + * %LCK_GRANTED: The requested lock was granted. + * %LCK_DENIED: The requested lock conflicted with existing + * lock reservations for the file. + * %LCK_DENIED_NOLOCKS: The server could not allocate the resources + * needed to process the request. + * %LCK_BLOCKED: The blocking request cannot be granted + * immediately. The server will send an + * NLM_GRANTED callback to the client when + * the lock can be granted. + * %LCK_DENIED_GRACE_PERIOD: The server has recently restarted and is + * re-establishing existing locks, and is not + * yet ready to accept normal service requests. + */ +static __be32 nlmsvc_proc_nm_lock(struct svc_rqst *rqstp) +{ + return nlmsvc_do_lock(rqstp, false); +} + +/** + * nlmsvc_proc_free_all - FREE_ALL: Discard client's lock and share state + * @rqstp: RPC transaction context + * + * Returns: + * %rpc_success: RPC executed successfully. + * + * RPC synopsis: + * void NLMPROC_FREE_ALL(nlm_notify) = 23; + */ +static __be32 nlmsvc_proc_free_all(struct svc_rqst *rqstp) +{ + struct nlm_notify_wrapper *argp = rqstp->rq_argp; + struct nlm_host *host; + + host = nlm3svc_lookup_host(rqstp, argp->xdrgen.name, false); + if (!host) + goto out; + + nlmsvc_free_host_resources(host); + + nlmsvc_release_host(host); + +out: + return rpc_success; +} + +/* + * NLMv3 Server procedures. + */ + +static const struct svc_procedure nlmsvc_procedures[24] = { + [NLM_NULL] = { + .pc_func = nlmsvc_proc_null, + .pc_decode = xdrgen_svc_decode_void, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = XDR_void, + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "NULL", + }, + [NLM_TEST] = { + .pc_func = nlmsvc_proc_test, + .pc_decode = nlm_svc_decode_nlm_testargs, + .pc_encode = nlm_svc_encode_nlm_testres, + .pc_argsize = sizeof(struct nlm_testargs_wrapper), + .pc_argzero = 0, + .pc_ressize = sizeof(struct nlm_testres_wrapper), + .pc_xdrressize = NLM3_nlm_testres_sz, + .pc_name = "TEST", + }, + [NLM_LOCK] = { + .pc_func = nlmsvc_proc_lock, + .pc_decode = nlm_svc_decode_nlm_lockargs, + .pc_encode = nlm_svc_encode_nlm_res, + .pc_argsize = sizeof(struct nlm_lockargs_wrapper), + .pc_argzero = 0, + .pc_ressize = sizeof(struct nlm_res_wrapper), + .pc_xdrressize = NLM3_nlm_res_sz, + .pc_name = "LOCK", + }, + [NLM_CANCEL] = { + .pc_func = nlmsvc_proc_cancel, + .pc_decode = nlm_svc_decode_nlm_cancargs, + .pc_encode = nlm_svc_encode_nlm_res, + .pc_argsize = sizeof(struct nlm_cancargs_wrapper), + .pc_argzero = 0, + .pc_ressize = sizeof(struct nlm_res_wrapper), + .pc_xdrressize = NLM3_nlm_res_sz, + .pc_name = "CANCEL", + }, + [NLM_UNLOCK] = { + .pc_func = nlmsvc_proc_unlock, + .pc_decode = nlm_svc_decode_nlm_unlockargs, + .pc_encode = nlm_svc_encode_nlm_res, + .pc_argsize = sizeof(struct nlm_unlockargs_wrapper), + .pc_argzero = 0, + .pc_ressize = sizeof(struct nlm_res_wrapper), + .pc_xdrressize = NLM3_nlm_res_sz, + .pc_name = "UNLOCK", + }, + [NLM_GRANTED] = { + .pc_func = nlmsvc_proc_granted, + .pc_decode = nlm_svc_decode_nlm_testargs, + .pc_encode = nlm_svc_encode_nlm_res, + .pc_argsize = sizeof(struct nlm_testargs_wrapper), + .pc_argzero = 0, + .pc_ressize = sizeof(struct nlm_res_wrapper), + .pc_xdrressize = NLM3_nlm_res_sz, + .pc_name = "GRANTED", + }, + [NLM_TEST_MSG] = { + .pc_func = nlmsvc_proc_test_msg, + .pc_decode = nlm_svc_decode_nlm_testargs, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = sizeof(struct nlm_testargs_wrapper), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "TEST_MSG", + }, + [NLM_LOCK_MSG] = { + .pc_func = nlmsvc_proc_lock_msg, + .pc_decode = nlm_svc_decode_nlm_lockargs, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = sizeof(struct nlm_lockargs_wrapper), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "LOCK_MSG", + }, + [NLM_CANCEL_MSG] = { + .pc_func = nlmsvc_proc_cancel_msg, + .pc_decode = nlm_svc_decode_nlm_cancargs, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = sizeof(struct nlm_cancargs_wrapper), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "CANCEL_MSG", + }, + [NLM_UNLOCK_MSG] = { + .pc_func = nlmsvc_proc_unlock_msg, + .pc_decode = nlm_svc_decode_nlm_unlockargs, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = sizeof(struct nlm_unlockargs_wrapper), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "UNLOCK_MSG", + }, + [NLM_GRANTED_MSG] = { + .pc_func = nlmsvc_proc_granted_msg, + .pc_decode = nlm_svc_decode_nlm_testargs, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = sizeof(struct nlm_testargs_wrapper), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "GRANTED_MSG", + }, + [NLM_TEST_RES] = { + .pc_func = nlmsvc_proc_null, + .pc_decode = nlm_svc_decode_nlm_testres, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = sizeof(struct nlm_testres), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "TEST_RES", + }, + [NLM_LOCK_RES] = { + .pc_func = nlmsvc_proc_null, + .pc_decode = nlm_svc_decode_nlm_res, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = sizeof(struct nlm_res), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "LOCK_RES", + }, + [NLM_CANCEL_RES] = { + .pc_func = nlmsvc_proc_null, + .pc_decode = nlm_svc_decode_nlm_res, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = sizeof(struct nlm_res), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "CANCEL_RES", + }, + [NLM_UNLOCK_RES] = { + .pc_func = nlmsvc_proc_null, + .pc_decode = nlm_svc_decode_nlm_res, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = sizeof(struct nlm_res), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "UNLOCK_RES", + }, + [NLM_GRANTED_RES] = { + .pc_func = nlmsvc_proc_granted_res, + .pc_decode = nlm_svc_decode_nlm_res, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = sizeof(struct nlm_res_wrapper), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "GRANTED_RES", + }, + [NLM_SM_NOTIFY] = { + .pc_func = nlmsvc_proc_sm_notify, + .pc_decode = nlm_svc_decode_nlm_notifyargs, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = sizeof(struct nlm_notifyargs_wrapper), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "SM_NOTIFY", + }, + [17] = { + .pc_func = nlmsvc_proc_unused, + .pc_decode = xdrgen_svc_decode_void, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = 0, + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "UNUSED", + }, + [18] = { + .pc_func = nlmsvc_proc_unused, + .pc_decode = xdrgen_svc_decode_void, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = 0, + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "UNUSED", + }, + [19] = { + .pc_func = nlmsvc_proc_unused, + .pc_decode = xdrgen_svc_decode_void, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = 0, + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "UNUSED", + }, + [NLM_SHARE] = { + .pc_func = nlmsvc_proc_share, + .pc_decode = nlm_svc_decode_nlm_shareargs, + .pc_encode = nlm_svc_encode_nlm_shareres, + .pc_argsize = sizeof(struct nlm_shareargs_wrapper), + .pc_argzero = 0, + .pc_ressize = sizeof(struct nlm_shareres_wrapper), + .pc_xdrressize = NLM3_nlm_shareres_sz, + .pc_name = "SHARE", + }, + [NLM_UNSHARE] = { + .pc_func = nlmsvc_proc_unshare, + .pc_decode = nlm_svc_decode_nlm_shareargs, + .pc_encode = nlm_svc_encode_nlm_shareres, + .pc_argsize = sizeof(struct nlm_shareargs_wrapper), + .pc_argzero = 0, + .pc_ressize = sizeof(struct nlm_shareres_wrapper), + .pc_xdrressize = NLM3_nlm_shareres_sz, + .pc_name = "UNSHARE", + }, + [NLM_NM_LOCK] = { + .pc_func = nlmsvc_proc_nm_lock, + .pc_decode = nlm_svc_decode_nlm_lockargs, + .pc_encode = nlm_svc_encode_nlm_res, + .pc_argsize = sizeof(struct nlm_lockargs_wrapper), + .pc_argzero = 0, + .pc_ressize = sizeof(struct nlm_res_wrapper), + .pc_xdrressize = NLM3_nlm_res_sz, + .pc_name = "NM_LOCK", + }, + [NLM_FREE_ALL] = { + .pc_func = nlmsvc_proc_free_all, + .pc_decode = nlm_svc_decode_nlm_notify, + .pc_encode = xdrgen_svc_encode_void, + .pc_argsize = sizeof(struct nlm_notify_wrapper), + .pc_argzero = 0, + .pc_ressize = 0, + .pc_xdrressize = XDR_void, + .pc_name = "FREE_ALL", + }, +}; + +/* + * Storage requirements for XDR arguments and results + */ +union nlmsvc_xdrstore { + struct nlm_testargs_wrapper testargs; + struct nlm_lockargs_wrapper lockargs; + struct nlm_cancargs_wrapper cancargs; + struct nlm_unlockargs_wrapper unlockargs; + struct nlm_notifyargs_wrapper notifyargs; + struct nlm_shareargs_wrapper shareargs; + struct nlm_notify_wrapper notify; + struct nlm_testres_wrapper testres; + struct nlm_res_wrapper res; + struct nlm_shareres_wrapper shareres; +}; + +/* + * NLMv1 defines only procedures 1 - 15. Linux lockd also implements + * procedures 0 (NULL) and 16 (SM_NOTIFY). + */ +const struct svc_version nlmsvc_version1 = { + .vs_vers = 1, + .vs_nproc = 17, + .vs_proc = nlmsvc_procedures, + .vs_dispatch = nlmsvc_dispatch, + .vs_xdrsize = sizeof(union nlmsvc_xdrstore), +}; + +const struct svc_version nlmsvc_version3 = { + .vs_vers = 3, + .vs_nproc = ARRAY_SIZE(nlmsvc_procedures), + .vs_proc = nlmsvc_procedures, + .vs_dispatch = nlmsvc_dispatch, + .vs_xdrsize = sizeof(union nlmsvc_xdrstore), +}; diff --git a/fs/lockd/svcshare.c b/fs/lockd/svcshare.c new file mode 100644 index 000000000..a58b7035b --- /dev/null +++ b/fs/lockd/svcshare.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/lockd/svcshare.c + * + * Management of DOS shares. + * + * Copyright (C) 1996 Olaf Kirch + */ + +#include +#include +#include +#include + +#include +#include + +#include "lockd.h" +#include "share.h" + +static inline int +nlm_cmp_owner(struct lockd_share *share, struct xdr_netobj *oh) +{ + return share->s_owner.len == oh->len + && !memcmp(share->s_owner.data, oh->data, oh->len); +} + +/* + * Recompute s_access / s_mode as the union of every (access, deny) pair + * whose bit is currently set in s_access_deny_bmap. + */ +static void nlm_recompute_share(struct lockd_share *share) +{ + u32 new_access = 0, new_mode = 0; + unsigned int i; + + for (i = 0; i < 16; i++) { + if (share->s_access_deny_bmap & BIT(i)) { + new_access |= i >> 2; + new_mode |= i & 3; + } + } + share->s_access = new_access; + share->s_mode = new_mode; +} + +/** + * nlmsvc_share_file - create a share + * @host: Network client peer + * @file: File to be shared + * @oh: Share owner handle + * @access: Requested access mode + * @mode: Requested file sharing mode + * + * Returns an NLM status code. + */ +__be32 +nlmsvc_share_file(struct nlm_host *host, struct nlm_file *file, + struct xdr_netobj *oh, u32 access, u32 mode) +{ + struct lockd_share *share; + u8 *ohdata; + + if (nlmsvc_file_cannot_lock(file)) + return nlm_lck_denied_nolocks; + + for (share = file->f_shares; share; share = share->s_next) { + if (share->s_host == host && nlm_cmp_owner(share, oh)) + goto update; + if ((access & share->s_mode) || (mode & share->s_access)) + return nlm_lck_denied; + } + + share = kmalloc(sizeof(*share) + oh->len, GFP_KERNEL); + if (share == NULL) + return nlm_lck_denied_nolocks; + + /* Copy owner handle */ + ohdata = (u8 *) (share + 1); + memcpy(ohdata, oh->data, oh->len); + + share->s_file = file; + share->s_host = host; + share->s_owner.data = ohdata; + share->s_owner.len = oh->len; + share->s_access_deny_bmap = 0; + share->s_next = file->f_shares; + file->f_shares = share; + +update: + share->s_access_deny_bmap |= LOCKD_FSH_BIT(access, mode); + nlm_recompute_share(share); + return nlm_granted; +} + +/** + * nlmsvc_unshare_file - delete a share + * @host: Network client peer + * @file: File to be unshared + * @oh: Share owner handle + * @access: Access mode of the SHARE being released + * @mode: Deny mode of the SHARE being released + * + * Returns an NLM status code. + */ +__be32 +nlmsvc_unshare_file(struct nlm_host *host, struct nlm_file *file, + struct xdr_netobj *oh, u32 access, u32 mode) +{ + struct lockd_share *share, **shpp; + + if (nlmsvc_file_cannot_lock(file)) + return nlm_lck_denied_nolocks; + + for (shpp = &file->f_shares; (share = *shpp) != NULL; + shpp = &share->s_next) { + if (share->s_host == host && nlm_cmp_owner(share, oh)) { + share->s_access_deny_bmap &= ~LOCKD_FSH_BIT(access, mode); + nlm_recompute_share(share); + if (!share->s_access_deny_bmap) { + *shpp = share->s_next; + kfree(share); + } + return nlm_granted; + } + } + + /* X/Open spec says return success even if there was no + * corresponding share. */ + return nlm_granted; +} + +/* + * Traverse all shares for a given file, and delete + * those owned by the given (type of) host + */ +void nlmsvc_traverse_shares(struct nlm_host *host, struct nlm_file *file, + nlm_host_match_fn_t match) +{ + struct lockd_share *share, **shpp; + + shpp = &file->f_shares; + while ((share = *shpp) != NULL) { + if (match(share->s_host, host)) { + *shpp = share->s_next; + kfree(share); + continue; + } + shpp = &share->s_next; + } +} diff --git a/fs/lockd/svcsubs.c b/fs/lockd/svcsubs.c new file mode 100644 index 000000000..ffab76278 --- /dev/null +++ b/fs/lockd/svcsubs.c @@ -0,0 +1,578 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/lockd/svcsubs.c + * + * Various support routines for the NLM server. + * + * Copyright (C) 1996, Olaf Kirch + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "lockd.h" +#include "share.h" + +#define NLMDBG_FACILITY NLMDBG_SVCSUBS + + +/* + * Global file hash table + */ +#define FILE_HASH_BITS 7 +#define FILE_NRHASH (1<data; + + /* print the first 32 bytes of the fh */ + dprintk("lockd: %s (%08x %08x %08x %08x %08x %08x %08x %08x)\n", + msg, fhp[0], fhp[1], fhp[2], fhp[3], + fhp[4], fhp[5], fhp[6], fhp[7]); +} + +static inline void nlm_debug_print_file(char *msg, struct nlm_file *file) +{ + struct inode *inode = nlmsvc_file_inode(file); + + dprintk("lockd: %s %s/%llu\n", + msg, inode->i_sb->s_id, inode->i_ino); +} +#else +static inline void nlm_debug_print_fh(char *msg, struct nfs_fh *f) +{ + return; +} + +static inline void nlm_debug_print_file(char *msg, struct nlm_file *file) +{ + return; +} +#endif + +static inline unsigned int file_hash(struct nfs_fh *f) +{ + unsigned int tmp=0; + int i; + for (i = 0; i < LOCKD_FH_HASH_SIZE; i++) + tmp += f->data[i]; + return tmp & (FILE_NRHASH - 1); +} + +int lock_to_openmode(struct file_lock *lock) +{ + return lock_is_write(lock) ? O_WRONLY : O_RDONLY; +} + +/* + * Open the file. Note that if we're reexporting, for example, + * this could block the lockd thread for a while. + * + * We have to make sure we have the right credential to open + * the file. + * + * @mode is O_RDONLY, O_WRONLY, or O_RDWR. O_RDWR means success + * is achieved with EITHER O_RDONLY or O_WRONLY; it does not + * require both. + */ +static __be32 nlm_do_fopen(struct svc_rqst *rqstp, + struct nlm_file *file, int mode) +{ + const struct nlmsvc_binding *ops; + __be32 nlmerr = nlm__int__failed; + __be32 deferred = 0; + int error; + int m; + + rcu_read_lock(); + ops = rcu_dereference(nlmsvc_ops); + if (!ops || !try_module_get(ops->owner)) { + rcu_read_unlock(); + return nlm__int__failed; + } + rcu_read_unlock(); + + for (m = O_RDONLY; m <= O_WRONLY; m++) { + struct file **fp = &file->f_file[m]; + + if (mode != O_RDWR && mode != m) + continue; + if (*fp) { + module_put(ops->owner); + return nlm_granted; + } + + error = ops->fopen(rqstp, &file->f_handle, fp, m); + if (!error) { + module_put(ops->owner); + return nlm_granted; + } + + dprintk("lockd: open failed (errno %d)\n", error); + switch (error) { + case -EWOULDBLOCK: + nlmerr = nlm__int__drop_reply; + deferred = nlmerr; + break; + case -ESTALE: + nlmerr = nlm__int__stale_fh; + break; + default: + nlmerr = nlm__int__failed; + break; + } + } + + module_put(ops->owner); + return deferred ? deferred : nlmerr; +} + +/* + * Lookup file info. If it doesn't exist, create a file info struct + * and open a (VFS) file for the given inode. + */ +__be32 +nlm_lookup_file(struct svc_rqst *rqstp, struct nlm_file **result, + struct lockd_lock *lock, int mode) +{ + struct nlm_file *file; + unsigned int hash; + __be32 nfserr; + + nlm_debug_print_fh("nlm_lookup_file", &lock->fh); + + hash = file_hash(&lock->fh); + + /* Lock file table */ + mutex_lock(&nlm_file_mutex); + + hlist_for_each_entry(file, &nlm_files[hash], f_list) + if (!nfs_compare_fh(&file->f_handle, &lock->fh)) { + mutex_lock(&file->f_mutex); + nfserr = nlm_do_fopen(rqstp, file, mode); + mutex_unlock(&file->f_mutex); + if (nfserr) + goto out_unlock; + goto found; + } + nlm_debug_print_fh("creating file for", &lock->fh); + + nfserr = nlm_lck_denied_nolocks; + file = kzalloc_obj(*file); + if (!file) + goto out_free; + + memcpy(&file->f_handle, &lock->fh, sizeof(struct nfs_fh)); + mutex_init(&file->f_mutex); + INIT_HLIST_NODE(&file->f_list); + INIT_LIST_HEAD(&file->f_blocks); + + nfserr = nlm_do_fopen(rqstp, file, mode); + if (nfserr) + goto out_free; + + hlist_add_head(&file->f_list, &nlm_files[hash]); + +found: + dprintk("lockd: found file %p (count %d)\n", file, file->f_count); + *result = file; + file->f_count++; + +out_unlock: + mutex_unlock(&nlm_file_mutex); + return nfserr; + +out_free: + kfree(file); + goto out_unlock; +} + +/* + * Release the struct file references held by a nlm_file. + */ +static void nlm_release_files(struct nlm_file *file) +{ + const struct nlmsvc_binding *ops; + bool have_ops; + + rcu_read_lock(); + ops = rcu_dereference(nlmsvc_ops); + have_ops = ops && try_module_get(ops->owner); + rcu_read_unlock(); + + if (have_ops) { + if (file->f_file[O_RDONLY]) + ops->fclose(file->f_file[O_RDONLY]); + if (file->f_file[O_WRONLY]) + ops->fclose(file->f_file[O_WRONLY]); + module_put(ops->owner); + } else { + if (file->f_file[O_RDONLY]) + fput(file->f_file[O_RDONLY]); + if (file->f_file[O_WRONLY]) + fput(file->f_file[O_WRONLY]); + } +} + +/* + * Delete a file after having released all locks, blocks and shares + */ +static inline void +nlm_delete_file(struct nlm_file *file) +{ + nlm_debug_print_file("closing file", file); + if (!hlist_unhashed(&file->f_list)) { + hlist_del(&file->f_list); + nlm_release_files(file); + kfree(file); + } else { + printk(KERN_WARNING "lockd: attempt to release unknown file!\n"); + } +} + +static int nlm_unlock_files(struct nlm_file *file, const struct file_lock *fl) +{ + struct file_lock lock; + + locks_init_lock(&lock); + lock.c.flc_type = F_UNLCK; + lock.fl_start = 0; + lock.fl_end = OFFSET_MAX; + lock.c.flc_owner = fl->c.flc_owner; + lock.c.flc_pid = fl->c.flc_pid; + lock.c.flc_flags = FL_POSIX; + + lock.c.flc_file = file->f_file[O_RDONLY]; + if (lock.c.flc_file && vfs_lock_file(lock.c.flc_file, F_SETLK, &lock, NULL)) + goto out_err; + lock.c.flc_file = file->f_file[O_WRONLY]; + if (lock.c.flc_file && vfs_lock_file(lock.c.flc_file, F_SETLK, &lock, NULL)) + goto out_err; + return 0; +out_err: + pr_warn("lockd: unlock failure in %s:%d\n", __FILE__, __LINE__); + return 1; +} + +/* + * Loop over all locks on the given file and perform the specified + * action. + */ +static int +nlm_traverse_locks(struct nlm_host *host, struct nlm_file *file, + nlm_host_match_fn_t match) +{ + struct inode *inode = nlmsvc_file_inode(file); + struct file_lock *fl; + struct file_lock_context *flctx = locks_inode_context(inode); + struct nlm_host *lockhost; + + if (!flctx || list_empty_careful(&flctx->flc_posix)) + return 0; +again: + file->f_locks = 0; + spin_lock(&flctx->flc_lock); + for_each_file_lock(fl, &flctx->flc_posix) { + if (fl->fl_lmops != &nlmsvc_lock_operations) + continue; + + /* update current lock count */ + file->f_locks++; + + lockhost = ((struct nlm_lockowner *) fl->c.flc_owner)->host; + if (match(lockhost, host)) { + + spin_unlock(&flctx->flc_lock); + if (nlm_unlock_files(file, fl)) + return 1; + goto again; + } + } + spin_unlock(&flctx->flc_lock); + + return 0; +} + +static int +nlmsvc_always_match(void *dummy1, struct nlm_host *dummy2) +{ + return 1; +} + +/* + * Inspect a single file + */ +static inline int +nlm_inspect_file(struct nlm_host *host, struct nlm_file *file, nlm_host_match_fn_t match) +{ + nlmsvc_traverse_blocks(host, file, match); + nlmsvc_traverse_shares(host, file, match); + return nlm_traverse_locks(host, file, match); +} + +/* + * Quick check whether there are still any locks, blocks or + * shares on a given file. + */ +static inline int +nlm_file_inuse(struct nlm_file *file) +{ + struct inode *inode = nlmsvc_file_inode(file); + struct file_lock *fl; + struct file_lock_context *flctx = locks_inode_context(inode); + + if (file->f_count || !list_empty(&file->f_blocks) || file->f_shares) + return 1; + + if (flctx && !list_empty_careful(&flctx->flc_posix)) { + spin_lock(&flctx->flc_lock); + for_each_file_lock(fl, &flctx->flc_posix) { + if (fl->fl_lmops == &nlmsvc_lock_operations) { + spin_unlock(&flctx->flc_lock); + return 1; + } + } + spin_unlock(&flctx->flc_lock); + } + file->f_locks = 0; + return 0; +} + +static void nlm_file_release(struct nlm_file *file) +{ + if (!nlm_file_inuse(file)) + nlm_delete_file(file); +} + +/* + * Loop over all files in the file table. + */ +static int +nlm_traverse_files(void *data, nlm_host_match_fn_t match, + int (*is_failover_file)(void *data, struct nlm_file *file)) +{ + struct nlm_file *file, *next; + int i, ret = 0; + + mutex_lock(&nlm_file_mutex); + for (i = 0; i < FILE_NRHASH; i++) { + file = hlist_entry_safe(nlm_files[i].first, + struct nlm_file, f_list); + if (file) + file->f_count++; + while (file) { + /* + * Pin the next neighbour before we drop the mutex + * for nlm_inspect_file(); a concurrent + * nlm_release_file() under the same mutex would + * otherwise be free to unlink and kfree it during + * the unlock window, leaving us to dereference a + * freed slab when we walked to next afterwards. + */ + next = hlist_entry_safe(file->f_list.next, + struct nlm_file, f_list); + if (next) + next->f_count++; + + if (!is_failover_file || is_failover_file(data, file)) { + mutex_unlock(&nlm_file_mutex); + + if (nlm_inspect_file(data, file, match)) + ret = 1; + + mutex_lock(&nlm_file_mutex); + } + + file->f_count--; + nlm_file_release(file); + file = next; + } + } + mutex_unlock(&nlm_file_mutex); + return ret; +} + +/* + * Release file. If there are no more remote locks on this file, + * close it and free the handle. + * + * Note that we can't do proper reference counting without major + * contortions because the code in fs/locks.c creates, deletes and + * splits locks without notification. Our only way is to walk the + * entire lock list each time we remove a lock. + */ +void +nlm_release_file(struct nlm_file *file) +{ + dprintk("lockd: nlm_release_file(%p, ct = %d)\n", + file, file->f_count); + + /* Lock file table */ + mutex_lock(&nlm_file_mutex); + + /* If there are no more locks etc, delete the file */ + if (--file->f_count == 0 && !nlm_file_inuse(file)) + nlm_delete_file(file); + + mutex_unlock(&nlm_file_mutex); +} + +/* + * Helpers function for resource traversal + * + * nlmsvc_mark_host: + * used by the garbage collector; simply sets h_inuse only for those + * hosts, which passed network check. + * Always returns 0. + * + * nlmsvc_same_host: + * returns 1 iff the two hosts match. Used to release + * all resources bound to a specific host. + * + * nlmsvc_is_client: + * returns 1 iff the host is a client. + * Used by nlmsvc_invalidate_all + */ + +static int +nlmsvc_mark_host(void *data, struct nlm_host *hint) +{ + struct nlm_host *host = data; + + if ((hint->net == NULL) || + (host->net == hint->net)) + host->h_inuse = 1; + return 0; +} + +static int +nlmsvc_same_host(void *data, struct nlm_host *other) +{ + struct nlm_host *host = data; + + return host == other; +} + +static int +nlmsvc_is_client(void *data, struct nlm_host *dummy) +{ + struct nlm_host *host = data; + + if (host->h_server) { + /* we are destroying locks even though the client + * hasn't asked us too, so don't unmonitor the + * client + */ + if (host->h_nsmhandle) + host->h_nsmhandle->sm_sticky = 1; + return 1; + } else + return 0; +} + +/* + * Mark all hosts that still hold resources + */ +void +nlmsvc_mark_resources(struct net *net) +{ + struct nlm_host hint; + + dprintk("lockd: %s for net %x\n", __func__, net ? net->ns.inum : 0); + hint.net = net; + nlm_traverse_files(&hint, nlmsvc_mark_host, NULL); +} + +/* + * Release all resources held by the given client + */ +void +nlmsvc_free_host_resources(struct nlm_host *host) +{ + dprintk("lockd: nlmsvc_free_host_resources\n"); + + if (nlm_traverse_files(host, nlmsvc_same_host, NULL)) { + printk(KERN_WARNING + "lockd: couldn't remove all locks held by %s\n", + host->h_name); + BUG(); + } +} + +/** + * nlmsvc_invalidate_all - remove all locks held for clients + * + * Release all locks held by NFS clients. + * + */ +void +nlmsvc_invalidate_all(void) +{ + /* + * Previously, the code would call + * nlmsvc_free_host_resources for each client in + * turn, which is about as inefficient as it gets. + * Now we just do it once in nlm_traverse_files. + */ + nlm_traverse_files(NULL, nlmsvc_is_client, NULL); +} + + +static int +nlmsvc_match_sb(void *datap, struct nlm_file *file) +{ + struct super_block *sb = datap; + + return sb == nlmsvc_file_inode(file)->i_sb; +} + +/** + * nlmsvc_unlock_all_by_sb - release locks held on this file system + * @sb: super block + * + * Release all locks held by clients accessing this file system. + */ +int +nlmsvc_unlock_all_by_sb(struct super_block *sb) +{ + int ret; + + ret = nlm_traverse_files(sb, nlmsvc_always_match, nlmsvc_match_sb); + return ret ? -EIO : 0; +} +EXPORT_SYMBOL_GPL(nlmsvc_unlock_all_by_sb); + +static int +nlmsvc_match_ip(void *datap, struct nlm_host *host) +{ + return rpc_cmp_addr(nlm_srcaddr(datap), (struct sockaddr *)host); +} + +/** + * nlmsvc_unlock_all_by_ip - release local locks by IP address + * @server_addr: server's IP address as seen by clients + * + * Release all locks held by clients accessing this host + * via the passed in IP address. + */ +int +nlmsvc_unlock_all_by_ip(struct sockaddr *server_addr) +{ + int ret; + + ret = nlm_traverse_files(server_addr, nlmsvc_match_ip, NULL); + return ret ? -EIO : 0; +} +EXPORT_SYMBOL_GPL(nlmsvc_unlock_all_by_ip); diff --git a/fs/lockd/trace.c b/fs/lockd/trace.c new file mode 100644 index 000000000..d9a6ff6e6 --- /dev/null +++ b/fs/lockd/trace.c @@ -0,0 +1,3 @@ +// SPDX-License-Identifier: GPL-2.0 +#define CREATE_TRACE_POINTS +#include "trace.h" diff --git a/fs/lockd/trace.h b/fs/lockd/trace.h new file mode 100644 index 000000000..a11d04e8c --- /dev/null +++ b/fs/lockd/trace.h @@ -0,0 +1,107 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM lockd + +#if !defined(_TRACE_LOCKD_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_LOCKD_H + +#include +#include +#include + +#include "lockd.h" + +#ifdef CONFIG_LOCKD_V4 +#define NLM_STATUS_LIST \ + nlm_status_code(LCK_GRANTED) \ + nlm_status_code(LCK_DENIED) \ + nlm_status_code(LCK_DENIED_NOLOCKS) \ + nlm_status_code(LCK_BLOCKED) \ + nlm_status_code(LCK_DENIED_GRACE_PERIOD) \ + nlm_status_code(DEADLCK) \ + nlm_status_code(ROFS) \ + nlm_status_code(STALE_FH) \ + nlm_status_code(FBIG) \ + nlm_status_code_end(FAILED) +#else +#define NLM_STATUS_LIST \ + nlm_status_code(LCK_GRANTED) \ + nlm_status_code(LCK_DENIED) \ + nlm_status_code(LCK_DENIED_NOLOCKS) \ + nlm_status_code(LCK_BLOCKED) \ + nlm_status_code_end(LCK_DENIED_GRACE_PERIOD) +#endif + +#undef nlm_status_code +#undef nlm_status_code_end +#define nlm_status_code(x) TRACE_DEFINE_ENUM(NLM_##x); +#define nlm_status_code_end(x) TRACE_DEFINE_ENUM(NLM_##x); + +NLM_STATUS_LIST + +#undef nlm_status_code +#undef nlm_status_code_end +#define nlm_status_code(x) { NLM_##x, #x }, +#define nlm_status_code_end(x) { NLM_##x, #x } + +#define show_nlm_status(x) __print_symbolic(x, NLM_STATUS_LIST) + +DECLARE_EVENT_CLASS(nlmclnt_lock_event, + TP_PROTO( + const struct lockd_lock *lock, + const struct sockaddr *addr, + unsigned int addrlen, + __be32 status + ), + + TP_ARGS(lock, addr, addrlen, status), + + TP_STRUCT__entry( + __field(u32, oh) + __field(u32, svid) + __field(u32, fh) + __field(unsigned long, status) + __field(loff_t, start) + __field(loff_t, end) + __sockaddr(addr, addrlen) + ), + + TP_fast_assign( + __entry->oh = ~crc32_le(0xffffffff, lock->oh.data, lock->oh.len); + __entry->svid = lock->svid; + __entry->fh = nfs_fhandle_hash(&lock->fh); + __entry->start = lock->fl.fl_start; + __entry->end = lock->fl.fl_end; + __entry->status = be32_to_cpu(status); + __assign_sockaddr(addr, addr, addrlen); + ), + + TP_printk( + "addr=%pISpc oh=0x%08x svid=0x%08x fh=0x%08x start=%lld end=%lld status=%s", + __get_sockaddr(addr), __entry->oh, __entry->svid, + __entry->fh, __entry->start, __entry->end, + show_nlm_status(__entry->status) + ) +); + +#define DEFINE_NLMCLNT_EVENT(name) \ + DEFINE_EVENT(nlmclnt_lock_event, name, \ + TP_PROTO( \ + const struct lockd_lock *lock, \ + const struct sockaddr *addr, \ + unsigned int addrlen, \ + __be32 status \ + ), \ + TP_ARGS(lock, addr, addrlen, status)) + +DEFINE_NLMCLNT_EVENT(nlmclnt_test); +DEFINE_NLMCLNT_EVENT(nlmclnt_lock); +DEFINE_NLMCLNT_EVENT(nlmclnt_unlock); +DEFINE_NLMCLNT_EVENT(nlmclnt_grant); + +#endif /* _TRACE_LOCKD_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE trace +#include diff --git a/fs/lockd/xdr.h b/fs/lockd/xdr.h new file mode 100644 index 000000000..a1126cca9 --- /dev/null +++ b/fs/lockd/xdr.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * XDR types for the NLM protocol + * + * Copyright (C) 1996 Olaf Kirch + */ + +#ifndef _LOCKD_XDR_H +#define _LOCKD_XDR_H + +#include +#include +#include +#include + +#define SM_MAXSTRLEN 1024 +#define SM_PRIV_SIZE 16 + +struct nsm_private { + unsigned char data[SM_PRIV_SIZE]; +}; + +#define NLM_MAXCOOKIELEN 32 +#define NLM_MAXSTRLEN 1024 + +#define nlm_granted cpu_to_be32(NLM_LCK_GRANTED) +#define nlm_lck_denied cpu_to_be32(NLM_LCK_DENIED) +#define nlm_lck_denied_nolocks cpu_to_be32(NLM_LCK_DENIED_NOLOCKS) +#define nlm_lck_blocked cpu_to_be32(NLM_LCK_BLOCKED) +#define nlm_lck_denied_grace_period cpu_to_be32(NLM_LCK_DENIED_GRACE_PERIOD) + +/* Lock info passed via NLM */ +struct lockd_lock { + char * caller; + unsigned int len; /* length of "caller" */ + struct nfs_fh fh; + struct xdr_netobj oh; + u32 svid; + u64 lock_start; + u64 lock_len; + struct file_lock fl; +}; + +/* + * NLM cookies. Technically they can be 1K, but Linux only uses 8 bytes. + * FreeBSD uses 16, Apple Mac OS X 10.3 uses 20. Therefore we set it to + * 32 bytes. + */ + +struct lockd_cookie { + unsigned char data[NLM_MAXCOOKIELEN]; + unsigned int len; +}; + +/* + * Generic lockd arguments for all but sm_notify + */ +struct lockd_args { + struct lockd_cookie cookie; + struct lockd_lock lock; + u32 block; + u32 reclaim; + u32 state; +}; + +/* + * Generic lockd result + */ +struct lockd_res { + struct lockd_cookie cookie; + __be32 status; + struct lockd_lock lock; +}; + +/* + * statd callback when client has rebooted + */ +struct lockd_reboot { + char *mon; + unsigned int len; + u32 state; + struct nsm_private priv; +}; + +#endif /* _LOCKD_XDR_H */ diff --git a/fs/locks.c b/fs/locks.c new file mode 100644 index 000000000..6e4ff7fce --- /dev/null +++ b/fs/locks.c @@ -0,0 +1,3163 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/locks.c + * + * We implement four types of file locks: BSD locks, posix locks, open + * file description locks, and leases. For details about BSD locks, + * see the flock(2) man page; for details about the other three, see + * fcntl(2). + * + * + * Locking conflicts and dependencies: + * If multiple threads attempt to lock the same byte (or flock the same file) + * only one can be granted the lock, and other must wait their turn. + * The first lock has been "applied" or "granted", the others are "waiting" + * and are "blocked" by the "applied" lock.. + * + * Waiting and applied locks are all kept in trees whose properties are: + * + * - the root of a tree may be an applied or waiting lock. + * - every other node in the tree is a waiting lock that + * conflicts with every ancestor of that node. + * + * Every such tree begins life as a waiting singleton which obviously + * satisfies the above properties. + * + * The only ways we modify trees preserve these properties: + * + * 1. We may add a new leaf node, but only after first verifying that it + * conflicts with all of its ancestors. + * 2. We may remove the root of a tree, creating a new singleton + * tree from the root and N new trees rooted in the immediate + * children. + * 3. If the root of a tree is not currently an applied lock, we may + * apply it (if possible). + * 4. We may upgrade the root of the tree (either extend its range, + * or upgrade its entire range from read to write). + * + * When an applied lock is modified in a way that reduces or downgrades any + * part of its range, we remove all its children (2 above). This particularly + * happens when a lock is unlocked. + * + * For each of those child trees we "wake up" the thread which is + * waiting for the lock so it can continue handling as follows: if the + * root of the tree applies, we do so (3). If it doesn't, it must + * conflict with some applied lock. We remove (wake up) all of its children + * (2), and add it is a new leaf to the tree rooted in the applied + * lock (1). We then repeat the process recursively with those + * children. + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CREATE_TRACE_POINTS +#include + +#include + +static struct file_lock *file_lock(struct file_lock_core *flc) +{ + return container_of(flc, struct file_lock, c); +} + +static struct file_lease *file_lease(struct file_lock_core *flc) +{ + return container_of(flc, struct file_lease, c); +} + +static bool lease_breaking(struct file_lease *fl) +{ + return fl->c.flc_flags & (FL_UNLOCK_PENDING | FL_DOWNGRADE_PENDING); +} + +static int target_leasetype(struct file_lease *fl) +{ + if (fl->c.flc_flags & FL_UNLOCK_PENDING) + return F_UNLCK; + if (fl->c.flc_flags & FL_DOWNGRADE_PENDING) + return F_RDLCK; + return fl->c.flc_type; +} + +static int leases_enable = 1; +static int lease_break_time = 45; + +#ifdef CONFIG_SYSCTL +static const struct ctl_table locks_sysctls[] = { + { + .procname = "leases-enable", + .data = &leases_enable, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#ifdef CONFIG_MMU + { + .procname = "lease-break-time", + .data = &lease_break_time, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#endif /* CONFIG_MMU */ +}; + +static int __init init_fs_locks_sysctls(void) +{ + register_sysctl_init("fs", locks_sysctls); + return 0; +} +early_initcall(init_fs_locks_sysctls); +#endif /* CONFIG_SYSCTL */ + +/* + * The global file_lock_list is only used for displaying /proc/locks, so we + * keep a list on each CPU, with each list protected by its own spinlock. + * Global serialization is done using file_rwsem. + * + * Note that alterations to the list also require that the relevant flc_lock is + * held. + */ +struct file_lock_list_struct { + spinlock_t lock; + struct hlist_head hlist; +}; +static DEFINE_PER_CPU(struct file_lock_list_struct, file_lock_list); +DEFINE_STATIC_PERCPU_RWSEM(file_rwsem); + + +/* + * The blocked_hash is used to find POSIX lock loops for deadlock detection. + * It is protected by blocked_lock_lock. + * + * We hash locks by lockowner in order to optimize searching for the lock a + * particular lockowner is waiting on. + * + * FIXME: make this value scale via some heuristic? We generally will want more + * buckets when we have more lockowners holding locks, but that's a little + * difficult to determine without knowing what the workload will look like. + */ +#define BLOCKED_HASH_BITS 7 +static DEFINE_HASHTABLE(blocked_hash, BLOCKED_HASH_BITS); + +/* + * This lock protects the blocked_hash. Generally, if you're accessing it, you + * want to be holding this lock. + * + * In addition, it also protects the fl->fl_blocked_requests list, and the + * fl->fl_blocker pointer for file_lock structures that are acting as lock + * requests (in contrast to those that are acting as records of acquired locks). + * + * Note that when we acquire this lock in order to change the above fields, + * we often hold the flc_lock as well. In certain cases, when reading the fields + * protected by this lock, we can skip acquiring it iff we already hold the + * flc_lock. + */ +static DEFINE_SPINLOCK(blocked_lock_lock); + +static struct kmem_cache *flctx_cache __ro_after_init; +static struct kmem_cache *filelock_cache __ro_after_init; +static struct kmem_cache *filelease_cache __ro_after_init; + +static struct file_lock_context * +locks_get_lock_context(struct inode *inode, int type) +{ + struct file_lock_context *ctx; + + ctx = locks_inode_context(inode); + if (likely(ctx) || type == F_UNLCK) + goto out; + + ctx = kmem_cache_alloc(flctx_cache, GFP_KERNEL); + if (!ctx) + goto out; + + spin_lock_init(&ctx->flc_lock); + INIT_LIST_HEAD(&ctx->flc_flock); + INIT_LIST_HEAD(&ctx->flc_posix); + INIT_LIST_HEAD(&ctx->flc_lease); + + /* + * Assign the pointer if it's not already assigned. If it is, then + * free the context we just allocated. + */ + spin_lock(&inode->i_lock); + if (!(inode->i_opflags & IOP_FLCTX)) { + VFS_BUG_ON_INODE(inode->i_flctx, inode); + WRITE_ONCE(inode->i_flctx, ctx); + /* + * Paired with locks_inode_context(). + */ + smp_store_release(&inode->i_opflags, inode->i_opflags | IOP_FLCTX); + spin_unlock(&inode->i_lock); + } else { + VFS_BUG_ON_INODE(!inode->i_flctx, inode); + spin_unlock(&inode->i_lock); + kmem_cache_free(flctx_cache, ctx); + ctx = locks_inode_context(inode); + } +out: + trace_locks_get_lock_context(inode, type, ctx); + return ctx; +} + +static void +locks_dump_ctx_list(struct list_head *list, char *list_type) +{ + struct file_lock_core *flc; + + list_for_each_entry(flc, list, flc_list) + pr_warn("%s: fl_owner=%p fl_flags=0x%x fl_type=0x%x fl_pid=%u\n", + list_type, flc->flc_owner, flc->flc_flags, + flc->flc_type, flc->flc_pid); +} + +static void +locks_check_ctx_lists(struct inode *inode) +{ + struct file_lock_context *ctx = inode->i_flctx; + + if (unlikely(!list_empty(&ctx->flc_flock) || + !list_empty(&ctx->flc_posix) || + !list_empty(&ctx->flc_lease))) { + pr_warn("Leaked locks on dev=0x%x:0x%x ino=0x%llx:\n", + MAJOR(inode->i_sb->s_dev), MINOR(inode->i_sb->s_dev), + inode->i_ino); + locks_dump_ctx_list(&ctx->flc_flock, "FLOCK"); + locks_dump_ctx_list(&ctx->flc_posix, "POSIX"); + locks_dump_ctx_list(&ctx->flc_lease, "LEASE"); + } +} + +static void +locks_check_ctx_file_list(struct file *filp, struct list_head *list, char *list_type) +{ + struct file_lock_core *flc; + struct inode *inode = file_inode(filp); + + list_for_each_entry(flc, list, flc_list) + if (flc->flc_file == filp) + pr_warn("Leaked %s lock on dev=0x%x:0x%x ino=0x%llx " + " fl_owner=%p fl_flags=0x%x fl_type=0x%x fl_pid=%u\n", + list_type, MAJOR(inode->i_sb->s_dev), + MINOR(inode->i_sb->s_dev), inode->i_ino, + flc->flc_owner, flc->flc_flags, + flc->flc_type, flc->flc_pid); +} + +void +locks_free_lock_context(struct inode *inode) +{ + struct file_lock_context *ctx = locks_inode_context(inode); + + if (unlikely(ctx)) { + locks_check_ctx_lists(inode); + kmem_cache_free(flctx_cache, ctx); + } +} + +static void locks_init_lock_heads(struct file_lock_core *flc) +{ + INIT_HLIST_NODE(&flc->flc_link); + INIT_LIST_HEAD(&flc->flc_list); + INIT_LIST_HEAD(&flc->flc_blocked_requests); + INIT_LIST_HEAD(&flc->flc_blocked_member); + init_waitqueue_head(&flc->flc_wait); +} + +/* Allocate an empty lock structure. */ +struct file_lock *locks_alloc_lock(void) +{ + struct file_lock *fl = kmem_cache_zalloc(filelock_cache, GFP_KERNEL); + + if (fl) + locks_init_lock_heads(&fl->c); + + return fl; +} +EXPORT_SYMBOL_GPL(locks_alloc_lock); + +/* Allocate an empty lock structure. */ +struct file_lease *locks_alloc_lease(void) +{ + struct file_lease *fl = kmem_cache_zalloc(filelease_cache, GFP_KERNEL); + + if (fl) + locks_init_lock_heads(&fl->c); + + return fl; +} +EXPORT_SYMBOL_GPL(locks_alloc_lease); + +void locks_release_private(struct file_lock *fl) +{ + struct file_lock_core *flc = &fl->c; + + BUG_ON(waitqueue_active(&flc->flc_wait)); + BUG_ON(!list_empty(&flc->flc_list)); + BUG_ON(!list_empty(&flc->flc_blocked_requests)); + BUG_ON(!list_empty(&flc->flc_blocked_member)); + BUG_ON(!hlist_unhashed(&flc->flc_link)); + + if (fl->fl_ops) { + if (fl->fl_ops->fl_release_private) + fl->fl_ops->fl_release_private(fl); + fl->fl_ops = NULL; + } + + if (fl->fl_lmops) { + if (fl->fl_lmops->lm_put_owner) { + fl->fl_lmops->lm_put_owner(flc->flc_owner); + flc->flc_owner = NULL; + } + fl->fl_lmops = NULL; + } +} +EXPORT_SYMBOL_GPL(locks_release_private); + +/** + * locks_owner_has_blockers - Check for blocking lock requests + * @flctx: file lock context + * @owner: lock owner + * + * Return values: + * %true: @owner has at least one blocker + * %false: @owner has no blockers + */ +bool locks_owner_has_blockers(struct file_lock_context *flctx, fl_owner_t owner) +{ + struct file_lock_core *flc; + + spin_lock(&flctx->flc_lock); + list_for_each_entry(flc, &flctx->flc_posix, flc_list) { + if (flc->flc_owner != owner) + continue; + if (!list_empty(&flc->flc_blocked_requests)) { + spin_unlock(&flctx->flc_lock); + return true; + } + } + spin_unlock(&flctx->flc_lock); + return false; +} +EXPORT_SYMBOL_GPL(locks_owner_has_blockers); + +/* Free a lock which is not in use. */ +void locks_free_lock(struct file_lock *fl) +{ + locks_release_private(fl); + kmem_cache_free(filelock_cache, fl); +} +EXPORT_SYMBOL(locks_free_lock); + +/* Free a lease which is not in use. */ +void locks_free_lease(struct file_lease *fl) +{ + kmem_cache_free(filelease_cache, fl); +} +EXPORT_SYMBOL(locks_free_lease); + +static void +locks_dispose_list(struct list_head *dispose) +{ + struct file_lock_core *flc; + + while (!list_empty(dispose)) { + flc = list_first_entry(dispose, struct file_lock_core, flc_list); + list_del_init(&flc->flc_list); + locks_free_lock(file_lock(flc)); + } +} + +static void +lease_dispose_list(struct list_head *dispose) +{ + struct file_lock_core *flc; + + while (!list_empty(dispose)) { + flc = list_first_entry(dispose, struct file_lock_core, flc_list); + list_del_init(&flc->flc_list); + locks_free_lease(file_lease(flc)); + } +} + +void locks_init_lock(struct file_lock *fl) +{ + memset(fl, 0, sizeof(struct file_lock)); + locks_init_lock_heads(&fl->c); +} +EXPORT_SYMBOL(locks_init_lock); + +void locks_init_lease(struct file_lease *fl) +{ + memset(fl, 0, sizeof(*fl)); + locks_init_lock_heads(&fl->c); +} +EXPORT_SYMBOL(locks_init_lease); + +/* + * Initialize a new lock from an existing file_lock structure. + */ +void locks_copy_conflock(struct file_lock *new, struct file_lock *fl) +{ + new->c.flc_owner = fl->c.flc_owner; + new->c.flc_pid = fl->c.flc_pid; + new->c.flc_file = NULL; + new->c.flc_flags = fl->c.flc_flags; + new->c.flc_type = fl->c.flc_type; + new->fl_start = fl->fl_start; + new->fl_end = fl->fl_end; + new->fl_lmops = fl->fl_lmops; + new->fl_ops = NULL; + + if (fl->fl_lmops) { + if (fl->fl_lmops->lm_get_owner) + fl->fl_lmops->lm_get_owner(fl->c.flc_owner); + } +} +EXPORT_SYMBOL(locks_copy_conflock); + +void locks_copy_lock(struct file_lock *new, struct file_lock *fl) +{ + /* "new" must be a freshly-initialized lock */ + WARN_ON_ONCE(new->fl_ops); + + locks_copy_conflock(new, fl); + + new->c.flc_file = fl->c.flc_file; + new->fl_ops = fl->fl_ops; + + if (fl->fl_ops) { + if (fl->fl_ops->fl_copy_lock) + fl->fl_ops->fl_copy_lock(new, fl); + } +} +EXPORT_SYMBOL(locks_copy_lock); + +static void locks_move_blocks(struct file_lock *new, struct file_lock *fl) +{ + struct file_lock *f; + + /* + * As ctx->flc_lock is held, new requests cannot be added to + * ->flc_blocked_requests, so we don't need a lock to check if it + * is empty. + */ + if (list_empty(&fl->c.flc_blocked_requests)) + return; + spin_lock(&blocked_lock_lock); + list_splice_init(&fl->c.flc_blocked_requests, + &new->c.flc_blocked_requests); + list_for_each_entry(f, &new->c.flc_blocked_requests, + c.flc_blocked_member) + f->c.flc_blocker = &new->c; + spin_unlock(&blocked_lock_lock); +} + +static inline int flock_translate_cmd(int cmd) { + switch (cmd) { + case LOCK_SH: + return F_RDLCK; + case LOCK_EX: + return F_WRLCK; + case LOCK_UN: + return F_UNLCK; + } + return -EINVAL; +} + +/* Fill in a file_lock structure with an appropriate FLOCK lock. */ +static void flock_make_lock(struct file *filp, struct file_lock *fl, int type) +{ + locks_init_lock(fl); + + fl->c.flc_file = filp; + fl->c.flc_owner = filp; + fl->c.flc_pid = current->tgid; + fl->c.flc_flags = FL_FLOCK; + fl->c.flc_type = type; + fl->fl_end = OFFSET_MAX; +} + +static int assign_type(struct file_lock_core *flc, int type) +{ + switch (type) { + case F_RDLCK: + case F_WRLCK: + case F_UNLCK: + flc->flc_type = type; + break; + default: + return -EINVAL; + } + return 0; +} + +static int flock64_to_posix_lock(struct file *filp, struct file_lock *fl, + struct flock64 *l) +{ + switch (l->l_whence) { + case SEEK_SET: + fl->fl_start = 0; + break; + case SEEK_CUR: + fl->fl_start = filp->f_pos; + break; + case SEEK_END: + fl->fl_start = i_size_read(file_inode(filp)); + break; + default: + return -EINVAL; + } + if (l->l_start > OFFSET_MAX - fl->fl_start) + return -EOVERFLOW; + fl->fl_start += l->l_start; + if (fl->fl_start < 0) + return -EINVAL; + + /* POSIX-1996 leaves the case l->l_len < 0 undefined; + POSIX-2001 defines it. */ + if (l->l_len > 0) { + if (l->l_len - 1 > OFFSET_MAX - fl->fl_start) + return -EOVERFLOW; + fl->fl_end = fl->fl_start + (l->l_len - 1); + + } else if (l->l_len < 0) { + if (fl->fl_start + l->l_len < 0) + return -EINVAL; + fl->fl_end = fl->fl_start - 1; + fl->fl_start += l->l_len; + } else + fl->fl_end = OFFSET_MAX; + + fl->c.flc_owner = current->files; + fl->c.flc_pid = current->tgid; + fl->c.flc_file = filp; + fl->c.flc_flags = FL_POSIX; + fl->fl_ops = NULL; + fl->fl_lmops = NULL; + + return assign_type(&fl->c, l->l_type); +} + +/* Verify a "struct flock" and copy it to a "struct file_lock" as a POSIX + * style lock. + */ +static int flock_to_posix_lock(struct file *filp, struct file_lock *fl, + struct flock *l) +{ + struct flock64 ll = { + .l_type = l->l_type, + .l_whence = l->l_whence, + .l_start = l->l_start, + .l_len = l->l_len, + }; + + return flock64_to_posix_lock(filp, fl, &ll); +} + +/* default lease lock manager operations */ +static bool +lease_break_callback(struct file_lease *fl) +{ + kill_fasync(&fl->fl_fasync, SIGIO, POLL_MSG); + return false; +} + +static void +lease_setup(struct file_lease *fl, void **priv) +{ + struct file *filp = fl->c.flc_file; + struct fasync_struct *fa = *priv; + + /* + * fasync_insert_entry() returns the old entry if any. If there was no + * old entry, then it used "priv" and inserted it into the fasync list. + * Clear the pointer to indicate that it shouldn't be freed. + */ + if (!fasync_insert_entry(fa->fa_fd, filp, &fl->fl_fasync, fa)) + *priv = NULL; + + __f_setown(filp, task_pid(current), PIDTYPE_TGID, 0); +} + +/** + * lease_open_conflict - see if the given file points to an inode that has + * an existing open that would conflict with the + * desired lease. + * @filp: file to check + * @arg: type of lease that we're trying to acquire + * + * Check to see if there's an existing open fd on this file that would + * conflict with the lease we're trying to set. + */ +static int +lease_open_conflict(struct file *filp, const int arg) +{ + struct inode *inode = file_inode(filp); + int self_wcount = 0, self_rcount = 0; + + if (arg == F_RDLCK) + return inode_is_open_for_write(inode) ? -EAGAIN : 0; + else if (arg != F_WRLCK) + return 0; + + /* + * Make sure that only read/write count is from lease requestor. + * Note that this will result in denying write leases when i_writecount + * is negative, which is what we want. (We shouldn't grant write leases + * on files open for execution.) + */ + if (filp->f_mode & FMODE_WRITE) + self_wcount = 1; + else if (filp->f_mode & FMODE_READ) + self_rcount = 1; + + if (atomic_read(&inode->i_writecount) != self_wcount || + atomic_read(&inode->i_readcount) != self_rcount) + return -EAGAIN; + + return 0; +} + +static const struct lease_manager_operations lease_manager_ops = { + .lm_break = lease_break_callback, + .lm_change = lease_modify, + .lm_setup = lease_setup, + .lm_open_conflict = lease_open_conflict, +}; + +/* + * Initialize a lease, use the default lock manager operations + */ +static int lease_init(struct file *filp, unsigned int flags, int type, struct file_lease *fl) +{ + if (assign_type(&fl->c, type) != 0) + return -EINVAL; + + fl->c.flc_owner = filp; + fl->c.flc_pid = current->tgid; + + fl->c.flc_file = filp; + fl->c.flc_flags = flags; + fl->fl_lmops = &lease_manager_ops; + return 0; +} + +/* Allocate a file_lock initialised to this type of lease */ +static struct file_lease *lease_alloc(struct file *filp, unsigned int flags, int type) +{ + struct file_lease *fl = locks_alloc_lease(); + int error = -ENOMEM; + + if (fl == NULL) + return ERR_PTR(error); + + error = lease_init(filp, flags, type, fl); + if (error) { + locks_free_lease(fl); + return ERR_PTR(error); + } + return fl; +} + +/* Check if two locks overlap each other. + */ +static inline int locks_overlap(struct file_lock *fl1, struct file_lock *fl2) +{ + return ((fl1->fl_end >= fl2->fl_start) && + (fl2->fl_end >= fl1->fl_start)); +} + +/* + * Check whether two locks have the same owner. + */ +static int posix_same_owner(struct file_lock_core *fl1, struct file_lock_core *fl2) +{ + return fl1->flc_owner == fl2->flc_owner; +} + +/* Must be called with the flc_lock held! */ +static void locks_insert_global_locks(struct file_lock_core *flc) +{ + struct file_lock_list_struct *fll = this_cpu_ptr(&file_lock_list); + + percpu_rwsem_assert_held(&file_rwsem); + + spin_lock(&fll->lock); + flc->flc_link_cpu = smp_processor_id(); + hlist_add_head(&flc->flc_link, &fll->hlist); + spin_unlock(&fll->lock); +} + +/* Must be called with the flc_lock held! */ +static void locks_delete_global_locks(struct file_lock_core *flc) +{ + struct file_lock_list_struct *fll; + + percpu_rwsem_assert_held(&file_rwsem); + + /* + * Avoid taking lock if already unhashed. This is safe since this check + * is done while holding the flc_lock, and new insertions into the list + * also require that it be held. + */ + if (hlist_unhashed(&flc->flc_link)) + return; + + fll = per_cpu_ptr(&file_lock_list, flc->flc_link_cpu); + spin_lock(&fll->lock); + hlist_del_init(&flc->flc_link); + spin_unlock(&fll->lock); +} + +static unsigned long +posix_owner_key(struct file_lock_core *flc) +{ + return (unsigned long) flc->flc_owner; +} + +static void locks_insert_global_blocked(struct file_lock_core *waiter) +{ + lockdep_assert_held(&blocked_lock_lock); + + hash_add(blocked_hash, &waiter->flc_link, posix_owner_key(waiter)); +} + +static void locks_delete_global_blocked(struct file_lock_core *waiter) +{ + lockdep_assert_held(&blocked_lock_lock); + + hash_del(&waiter->flc_link); +} + +/* Remove waiter from blocker's block list. + * When blocker ends up pointing to itself then the list is empty. + * + * Must be called with blocked_lock_lock held. + */ +static void __locks_unlink_block(struct file_lock_core *waiter) +{ + locks_delete_global_blocked(waiter); + list_del_init(&waiter->flc_blocked_member); +} + +static void __locks_wake_up_blocks(struct file_lock_core *blocker) +{ + while (!list_empty(&blocker->flc_blocked_requests)) { + struct file_lock_core *waiter; + struct file_lock *fl; + + waiter = list_first_entry(&blocker->flc_blocked_requests, + struct file_lock_core, flc_blocked_member); + + fl = file_lock(waiter); + __locks_unlink_block(waiter); + if ((waiter->flc_flags & (FL_POSIX | FL_FLOCK)) && + fl->fl_lmops && fl->fl_lmops->lm_notify) + fl->fl_lmops->lm_notify(fl); + else + locks_wake_up_waiter(waiter); + + /* + * The setting of flc_blocker to NULL marks the "done" + * point in deleting a block. Paired with acquire at the top + * of locks_delete_block(). + */ + smp_store_release(&waiter->flc_blocker, NULL); + } +} + +static int __locks_delete_block(struct file_lock_core *waiter) +{ + int status = -ENOENT; + + /* + * If fl_blocker is NULL, it won't be set again as this thread "owns" + * the lock and is the only one that might try to claim the lock. + * + * We use acquire/release to manage fl_blocker so that we can + * optimize away taking the blocked_lock_lock in many cases. + * + * The smp_load_acquire guarantees two things: + * + * 1/ that fl_blocked_requests can be tested locklessly. If something + * was recently added to that list it must have been in a locked region + * *before* the locked region when fl_blocker was set to NULL. + * + * 2/ that no other thread is accessing 'waiter', so it is safe to free + * it. __locks_wake_up_blocks is careful not to touch waiter after + * fl_blocker is released. + * + * If a lockless check of fl_blocker shows it to be NULL, we know that + * no new locks can be inserted into its fl_blocked_requests list, and + * can avoid doing anything further if the list is empty. + */ + if (!smp_load_acquire(&waiter->flc_blocker) && + list_empty(&waiter->flc_blocked_requests)) + return status; + + spin_lock(&blocked_lock_lock); + if (waiter->flc_blocker) + status = 0; + __locks_wake_up_blocks(waiter); + __locks_unlink_block(waiter); + + /* + * The setting of fl_blocker to NULL marks the "done" point in deleting + * a block. Paired with acquire at the top of this function. + */ + smp_store_release(&waiter->flc_blocker, NULL); + spin_unlock(&blocked_lock_lock); + return status; +} + +/** + * locks_delete_block - stop waiting for a file lock + * @waiter: the lock which was waiting + * + * lockd/nfsd need to disconnect the lock while working on it. + */ +int locks_delete_block(struct file_lock *waiter) +{ + return __locks_delete_block(&waiter->c); +} +EXPORT_SYMBOL(locks_delete_block); + +/* Insert waiter into blocker's block list. + * We use a circular list so that processes can be easily woken up in + * the order they blocked. The documentation doesn't require this but + * it seems like the reasonable thing to do. + * + * Must be called with both the flc_lock and blocked_lock_lock held. The + * fl_blocked_requests list itself is protected by the blocked_lock_lock, + * but by ensuring that the flc_lock is also held on insertions we can avoid + * taking the blocked_lock_lock in some cases when we see that the + * fl_blocked_requests list is empty. + * + * Rather than just adding to the list, we check for conflicts with any existing + * waiters, and add beneath any waiter that blocks the new waiter. + * Thus wakeups don't happen until needed. + */ +static void __locks_insert_block(struct file_lock_core *blocker, + struct file_lock_core *waiter, + bool conflict(struct file_lock_core *, + struct file_lock_core *)) +{ + struct file_lock_core *flc; + + BUG_ON(!list_empty(&waiter->flc_blocked_member)); +new_blocker: + list_for_each_entry(flc, &blocker->flc_blocked_requests, flc_blocked_member) + if (conflict(flc, waiter)) { + blocker = flc; + goto new_blocker; + } + waiter->flc_blocker = blocker; + list_add_tail(&waiter->flc_blocked_member, + &blocker->flc_blocked_requests); + + if ((blocker->flc_flags & (FL_POSIX|FL_OFDLCK)) == FL_POSIX) + locks_insert_global_blocked(waiter); + + /* The requests in waiter->flc_blocked are known to conflict with + * waiter, but might not conflict with blocker, or the requests + * and lock which block it. So they all need to be woken. + */ + __locks_wake_up_blocks(waiter); +} + +/* Must be called with flc_lock held. */ +static void locks_insert_block(struct file_lock_core *blocker, + struct file_lock_core *waiter, + bool conflict(struct file_lock_core *, + struct file_lock_core *)) +{ + spin_lock(&blocked_lock_lock); + __locks_insert_block(blocker, waiter, conflict); + spin_unlock(&blocked_lock_lock); +} + +/* + * Wake up processes blocked waiting for blocker. + * + * Must be called with the inode->flc_lock held! + */ +static void locks_wake_up_blocks(struct file_lock_core *blocker) +{ + /* + * Avoid taking global lock if list is empty. This is safe since new + * blocked requests are only added to the list under the flc_lock, and + * the flc_lock is always held here. Note that removal from the + * fl_blocked_requests list does not require the flc_lock, so we must + * recheck list_empty() after acquiring the blocked_lock_lock. + */ + if (list_empty(&blocker->flc_blocked_requests)) + return; + + spin_lock(&blocked_lock_lock); + __locks_wake_up_blocks(blocker); + spin_unlock(&blocked_lock_lock); +} + +static void +locks_insert_lock_ctx(struct file_lock_core *fl, struct list_head *before) +{ + list_add_tail(&fl->flc_list, before); + locks_insert_global_locks(fl); +} + +static void +locks_unlink_lock_ctx(struct file_lock_core *fl) +{ + locks_delete_global_locks(fl); + list_del_init(&fl->flc_list); + locks_wake_up_blocks(fl); +} + +static void +locks_delete_lock_ctx(struct file_lock_core *fl, struct list_head *dispose) +{ + locks_unlink_lock_ctx(fl); + if (dispose) + list_add(&fl->flc_list, dispose); + else + locks_free_lock(file_lock(fl)); +} + +/* Determine if lock sys_fl blocks lock caller_fl. Common functionality + * checks for shared/exclusive status of overlapping locks. + */ +static bool locks_conflict(struct file_lock_core *caller_flc, + struct file_lock_core *sys_flc) +{ + if (sys_flc->flc_type == F_WRLCK) + return true; + if (caller_flc->flc_type == F_WRLCK) + return true; + return false; +} + +/* Determine if lock sys_fl blocks lock caller_fl. POSIX specific + * checking before calling the locks_conflict(). + */ +static bool posix_locks_conflict(struct file_lock_core *caller_flc, + struct file_lock_core *sys_flc) +{ + struct file_lock *caller_fl = file_lock(caller_flc); + struct file_lock *sys_fl = file_lock(sys_flc); + + /* POSIX locks owned by the same process do not conflict with + * each other. + */ + if (posix_same_owner(caller_flc, sys_flc)) + return false; + + /* Check whether they overlap */ + if (!locks_overlap(caller_fl, sys_fl)) + return false; + + return locks_conflict(caller_flc, sys_flc); +} + +/* Determine if lock sys_fl blocks lock caller_fl. Used on xx_GETLK + * path so checks for additional GETLK-specific things like F_UNLCK. + */ +static bool posix_test_locks_conflict(struct file_lock *caller_fl, + struct file_lock *sys_fl) +{ + struct file_lock_core *caller = &caller_fl->c; + struct file_lock_core *sys = &sys_fl->c; + + /* F_UNLCK checks any locks on the same fd. */ + if (lock_is_unlock(caller_fl)) { + if (!posix_same_owner(caller, sys)) + return false; + return locks_overlap(caller_fl, sys_fl); + } + return posix_locks_conflict(caller, sys); +} + +/* Determine if lock sys_fl blocks lock caller_fl. FLOCK specific + * checking before calling the locks_conflict(). + */ +static bool flock_locks_conflict(struct file_lock_core *caller_flc, + struct file_lock_core *sys_flc) +{ + /* FLOCK locks referring to the same filp do not conflict with + * each other. + */ + if (caller_flc->flc_file == sys_flc->flc_file) + return false; + + return locks_conflict(caller_flc, sys_flc); +} + +void +posix_test_lock(struct file *filp, struct file_lock *fl) +{ + struct file_lock *cfl; + struct file_lock_context *ctx; + struct inode *inode = file_inode(filp); + void *owner; + void (*func)(void); + + ctx = locks_inode_context(inode); + if (!ctx || list_empty_careful(&ctx->flc_posix)) { + fl->c.flc_type = F_UNLCK; + return; + } + +retry: + spin_lock(&ctx->flc_lock); + list_for_each_entry(cfl, &ctx->flc_posix, c.flc_list) { + if (!posix_test_locks_conflict(fl, cfl)) + continue; + if (cfl->fl_lmops && cfl->fl_lmops->lm_lock_expirable + && (*cfl->fl_lmops->lm_lock_expirable)(cfl)) { + owner = cfl->fl_lmops->lm_mod_owner; + func = cfl->fl_lmops->lm_expire_lock; + __module_get(owner); + spin_unlock(&ctx->flc_lock); + (*func)(); + module_put(owner); + goto retry; + } + locks_copy_conflock(fl, cfl); + goto out; + } + fl->c.flc_type = F_UNLCK; +out: + spin_unlock(&ctx->flc_lock); + return; +} +EXPORT_SYMBOL(posix_test_lock); + +/* + * Deadlock detection: + * + * We attempt to detect deadlocks that are due purely to posix file + * locks. + * + * We assume that a task can be waiting for at most one lock at a time. + * So for any acquired lock, the process holding that lock may be + * waiting on at most one other lock. That lock in turns may be held by + * someone waiting for at most one other lock. Given a requested lock + * caller_fl which is about to wait for a conflicting lock block_fl, we + * follow this chain of waiters to ensure we are not about to create a + * cycle. + * + * Since we do this before we ever put a process to sleep on a lock, we + * are ensured that there is never a cycle; that is what guarantees that + * the while() loop in posix_locks_deadlock() eventually completes. + * + * Note: the above assumption may not be true when handling lock + * requests from a broken NFS client. It may also fail in the presence + * of tasks (such as posix threads) sharing the same open file table. + * To handle those cases, we just bail out after a few iterations. + * + * For FL_OFDLCK locks, the owner is the filp, not the files_struct. + * Because the owner is not even nominally tied to a thread of + * execution, the deadlock detection below can't reasonably work well. Just + * skip it for those. + * + * In principle, we could do a more limited deadlock detection on FL_OFDLCK + * locks that just checks for the case where two tasks are attempting to + * upgrade from read to write locks on the same inode. + */ + +#define MAX_DEADLK_ITERATIONS 10 + +/* Find a lock that the owner of the given @blocker is blocking on. */ +static struct file_lock_core *what_owner_is_waiting_for(struct file_lock_core *blocker) +{ + struct file_lock_core *flc; + + hash_for_each_possible(blocked_hash, flc, flc_link, posix_owner_key(blocker)) { + if (posix_same_owner(flc, blocker)) { + while (flc->flc_blocker) + flc = flc->flc_blocker; + return flc; + } + } + return NULL; +} + +/* Must be called with the blocked_lock_lock held! */ +static bool posix_locks_deadlock(struct file_lock *caller_fl, + struct file_lock *block_fl) +{ + struct file_lock_core *caller = &caller_fl->c; + struct file_lock_core *blocker = &block_fl->c; + int i = 0; + + lockdep_assert_held(&blocked_lock_lock); + + /* + * This deadlock detector can't reasonably detect deadlocks with + * FL_OFDLCK locks, since they aren't owned by a process, per-se. + */ + if (caller->flc_flags & FL_OFDLCK) + return false; + + while ((blocker = what_owner_is_waiting_for(blocker))) { + if (i++ > MAX_DEADLK_ITERATIONS) + return false; + if (posix_same_owner(caller, blocker)) + return true; + } + return false; +} + +/* Try to create a FLOCK lock on filp. We always insert new FLOCK locks + * after any leases, but before any posix locks. + * + * Note that if called with an FL_EXISTS argument, the caller may determine + * whether or not a lock was successfully freed by testing the return + * value for -ENOENT. + */ +static int flock_lock_inode(struct inode *inode, struct file_lock *request) +{ + struct file_lock *new_fl = NULL; + struct file_lock *fl; + struct file_lock_context *ctx; + int error = 0; + bool found = false; + LIST_HEAD(dispose); + + ctx = locks_get_lock_context(inode, request->c.flc_type); + if (!ctx) { + if (request->c.flc_type != F_UNLCK) + return -ENOMEM; + return (request->c.flc_flags & FL_EXISTS) ? -ENOENT : 0; + } + + if (!(request->c.flc_flags & FL_ACCESS) && (request->c.flc_type != F_UNLCK)) { + new_fl = locks_alloc_lock(); + if (!new_fl) + return -ENOMEM; + } + + percpu_down_read(&file_rwsem); + spin_lock(&ctx->flc_lock); + if (request->c.flc_flags & FL_ACCESS) + goto find_conflict; + + list_for_each_entry(fl, &ctx->flc_flock, c.flc_list) { + if (request->c.flc_file != fl->c.flc_file) + continue; + if (request->c.flc_type == fl->c.flc_type) + goto out; + found = true; + locks_delete_lock_ctx(&fl->c, &dispose); + break; + } + + if (lock_is_unlock(request)) { + if ((request->c.flc_flags & FL_EXISTS) && !found) + error = -ENOENT; + goto out; + } + +find_conflict: + list_for_each_entry(fl, &ctx->flc_flock, c.flc_list) { + if (!flock_locks_conflict(&request->c, &fl->c)) + continue; + error = -EAGAIN; + if (!(request->c.flc_flags & FL_SLEEP)) + goto out; + error = FILE_LOCK_DEFERRED; + locks_insert_block(&fl->c, &request->c, flock_locks_conflict); + goto out; + } + if (request->c.flc_flags & FL_ACCESS) + goto out; + locks_copy_lock(new_fl, request); + locks_move_blocks(new_fl, request); + locks_insert_lock_ctx(&new_fl->c, &ctx->flc_flock); + new_fl = NULL; + error = 0; + +out: + spin_unlock(&ctx->flc_lock); + percpu_up_read(&file_rwsem); + if (new_fl) + locks_free_lock(new_fl); + locks_dispose_list(&dispose); + trace_flock_lock_inode(inode, request, error); + return error; +} + +static int posix_lock_inode(struct inode *inode, struct file_lock *request, + struct file_lock *conflock) +{ + struct file_lock *fl, *tmp; + struct file_lock *new_fl = NULL; + struct file_lock *new_fl2 = NULL; + struct file_lock *left = NULL; + struct file_lock *right = NULL; + struct file_lock_context *ctx; + int error; + bool added = false; + LIST_HEAD(dispose); + void *owner; + void (*func)(void); + + ctx = locks_get_lock_context(inode, request->c.flc_type); + if (!ctx) + return lock_is_unlock(request) ? 0 : -ENOMEM; + + /* + * We may need two file_lock structures for this operation, + * so we get them in advance to avoid races. + * + * In some cases we can be sure, that no new locks will be needed + */ + if (!(request->c.flc_flags & FL_ACCESS) && + (request->c.flc_type != F_UNLCK || + request->fl_start != 0 || request->fl_end != OFFSET_MAX)) { + new_fl = locks_alloc_lock(); + new_fl2 = locks_alloc_lock(); + } + +retry: + percpu_down_read(&file_rwsem); + spin_lock(&ctx->flc_lock); + /* + * New lock request. Walk all POSIX locks and look for conflicts. If + * there are any, either return error or put the request on the + * blocker's list of waiters and the global blocked_hash. + */ + if (request->c.flc_type != F_UNLCK) { + list_for_each_entry(fl, &ctx->flc_posix, c.flc_list) { + if (!posix_locks_conflict(&request->c, &fl->c)) + continue; + if (fl->fl_lmops && fl->fl_lmops->lm_lock_expirable + && (*fl->fl_lmops->lm_lock_expirable)(fl)) { + owner = fl->fl_lmops->lm_mod_owner; + func = fl->fl_lmops->lm_expire_lock; + __module_get(owner); + spin_unlock(&ctx->flc_lock); + percpu_up_read(&file_rwsem); + (*func)(); + module_put(owner); + goto retry; + } + if (conflock) + locks_copy_conflock(conflock, fl); + error = -EAGAIN; + if (!(request->c.flc_flags & FL_SLEEP)) + goto out; + /* + * Deadlock detection and insertion into the blocked + * locks list must be done while holding the same lock! + */ + error = -EDEADLK; + spin_lock(&blocked_lock_lock); + /* + * Ensure that we don't find any locks blocked on this + * request during deadlock detection. + */ + __locks_wake_up_blocks(&request->c); + if (likely(!posix_locks_deadlock(request, fl))) { + error = FILE_LOCK_DEFERRED; + __locks_insert_block(&fl->c, &request->c, + posix_locks_conflict); + } + spin_unlock(&blocked_lock_lock); + goto out; + } + } + + /* If we're just looking for a conflict, we're done. */ + error = 0; + if (request->c.flc_flags & FL_ACCESS) + goto out; + + /* Find the first old lock with the same owner as the new lock */ + list_for_each_entry(fl, &ctx->flc_posix, c.flc_list) { + if (posix_same_owner(&request->c, &fl->c)) + break; + } + + /* Process locks with this owner. */ + list_for_each_entry_safe_from(fl, tmp, &ctx->flc_posix, c.flc_list) { + if (!posix_same_owner(&request->c, &fl->c)) + break; + + /* Detect adjacent or overlapping regions (if same lock type) */ + if (request->c.flc_type == fl->c.flc_type) { + /* In all comparisons of start vs end, use + * "start - 1" rather than "end + 1". If end + * is OFFSET_MAX, end + 1 will become negative. + */ + if (fl->fl_end < request->fl_start - 1) + continue; + /* If the next lock in the list has entirely bigger + * addresses than the new one, insert the lock here. + */ + if (fl->fl_start - 1 > request->fl_end) + break; + + /* If we come here, the new and old lock are of the + * same type and adjacent or overlapping. Make one + * lock yielding from the lower start address of both + * locks to the higher end address. + */ + if (fl->fl_start > request->fl_start) + fl->fl_start = request->fl_start; + else + request->fl_start = fl->fl_start; + if (fl->fl_end < request->fl_end) + fl->fl_end = request->fl_end; + else + request->fl_end = fl->fl_end; + if (added) { + locks_delete_lock_ctx(&fl->c, &dispose); + continue; + } + request = fl; + added = true; + } else { + /* Processing for different lock types is a bit + * more complex. + */ + if (fl->fl_end < request->fl_start) + continue; + if (fl->fl_start > request->fl_end) + break; + if (lock_is_unlock(request)) + added = true; + if (fl->fl_start < request->fl_start) + left = fl; + /* If the next lock in the list has a higher end + * address than the new one, insert the new one here. + */ + if (fl->fl_end > request->fl_end) { + right = fl; + break; + } + if (fl->fl_start >= request->fl_start) { + /* The new lock completely replaces an old + * one (This may happen several times). + */ + if (added) { + locks_delete_lock_ctx(&fl->c, &dispose); + continue; + } + /* + * Replace the old lock with new_fl, and + * remove the old one. It's safe to do the + * insert here since we know that we won't be + * using new_fl later, and that the lock is + * just replacing an existing lock. + */ + error = -ENOLCK; + if (!new_fl) + goto out; + locks_copy_lock(new_fl, request); + locks_move_blocks(new_fl, request); + request = new_fl; + new_fl = NULL; + locks_insert_lock_ctx(&request->c, + &fl->c.flc_list); + locks_delete_lock_ctx(&fl->c, &dispose); + added = true; + } + } + } + + /* + * The above code only modifies existing locks in case of merging or + * replacing. If new lock(s) need to be inserted all modifications are + * done below this, so it's safe yet to bail out. + */ + error = -ENOLCK; /* "no luck" */ + if (right && left == right && !new_fl2) + goto out; + + error = 0; + if (!added) { + if (lock_is_unlock(request)) { + if (request->c.flc_flags & FL_EXISTS) + error = -ENOENT; + goto out; + } + + if (!new_fl) { + error = -ENOLCK; + goto out; + } + locks_copy_lock(new_fl, request); + locks_move_blocks(new_fl, request); + locks_insert_lock_ctx(&new_fl->c, &fl->c.flc_list); + fl = new_fl; + new_fl = NULL; + } + if (right) { + if (left == right) { + /* The new lock breaks the old one in two pieces, + * so we have to use the second new lock. + */ + left = new_fl2; + new_fl2 = NULL; + locks_copy_lock(left, right); + locks_insert_lock_ctx(&left->c, &fl->c.flc_list); + } + right->fl_start = request->fl_end + 1; + locks_wake_up_blocks(&right->c); + } + if (left) { + left->fl_end = request->fl_start - 1; + locks_wake_up_blocks(&left->c); + } + out: + trace_posix_lock_inode(inode, request, error); + spin_unlock(&ctx->flc_lock); + percpu_up_read(&file_rwsem); + /* + * Free any unused locks. + */ + if (new_fl) + locks_free_lock(new_fl); + if (new_fl2) + locks_free_lock(new_fl2); + locks_dispose_list(&dispose); + + return error; +} + +/** + * posix_lock_file - Apply a POSIX-style lock to a file + * @filp: The file to apply the lock to + * @fl: The lock to be applied + * @conflock: Place to return a copy of the conflicting lock, if found. + * + * Add a POSIX style lock to a file. + * We merge adjacent & overlapping locks whenever possible. + * POSIX locks are sorted by owner task, then by starting address + * + * Note that if called with an FL_EXISTS argument, the caller may determine + * whether or not a lock was successfully freed by testing the return + * value for -ENOENT. + */ +int posix_lock_file(struct file *filp, struct file_lock *fl, + struct file_lock *conflock) +{ + return posix_lock_inode(file_inode(filp), fl, conflock); +} +EXPORT_SYMBOL(posix_lock_file); + +/** + * posix_lock_inode_wait - Apply a POSIX-style lock to a file + * @inode: inode of file to which lock request should be applied + * @fl: The lock to be applied + * + * Apply a POSIX style lock request to an inode. + */ +static int posix_lock_inode_wait(struct inode *inode, struct file_lock *fl) +{ + int error; + might_sleep (); + for (;;) { + error = posix_lock_inode(inode, fl, NULL); + if (error != FILE_LOCK_DEFERRED) + break; + error = wait_event_interruptible(fl->c.flc_wait, + list_empty(&fl->c.flc_blocked_member)); + if (error) + break; + } + locks_delete_block(fl); + return error; +} + +static void lease_clear_pending(struct file_lease *fl, int arg) +{ + switch (arg) { + case F_UNLCK: + fl->c.flc_flags &= ~FL_UNLOCK_PENDING; + fallthrough; + case F_RDLCK: + fl->c.flc_flags &= ~FL_DOWNGRADE_PENDING; + } +} + +/* We already had a lease on this file; just change its type */ +int lease_modify(struct file_lease *fl, int arg, struct list_head *dispose) +{ + int error = assign_type(&fl->c, arg); + + if (error) + return error; + lease_clear_pending(fl, arg); + locks_wake_up_blocks(&fl->c); + if (arg == F_UNLCK) { + struct file *filp = fl->c.flc_file; + + f_delown(filp); + file_f_owner(filp)->signum = 0; + fasync_helper(0, fl->c.flc_file, 0, &fl->fl_fasync); + if (fl->fl_fasync != NULL) { + printk(KERN_ERR "locks_delete_lock: fasync == %p\n", fl->fl_fasync); + fl->fl_fasync = NULL; + } + locks_delete_lock_ctx(&fl->c, dispose); + } + return 0; +} +EXPORT_SYMBOL(lease_modify); + +static bool past_time(unsigned long then) +{ + if (!then) + /* 0 is a special value meaning "this never expires": */ + return false; + return time_after(jiffies, then); +} + +static void time_out_leases(struct inode *inode, struct list_head *dispose) +{ + struct file_lock_context *ctx = inode->i_flctx; + struct file_lease *fl, *tmp; + bool remove; + + lockdep_assert_held(&ctx->flc_lock); + + list_for_each_entry_safe(fl, tmp, &ctx->flc_lease, c.flc_list) { + trace_time_out_leases(inode, fl); + if (past_time(fl->fl_downgrade_time)) + lease_modify(fl, F_RDLCK, dispose); + + remove = true; + if (past_time(fl->fl_break_time)) { + /* + * Consult the lease manager when a lease break times + * out to determine whether the lease should be disposed + * of. + */ + if (fl->fl_lmops && fl->fl_lmops->lm_breaker_timedout) + remove = fl->fl_lmops->lm_breaker_timedout(fl); + if (remove) + lease_modify(fl, F_UNLCK, dispose); + } + } +} + +static bool leases_conflict(struct file_lock_core *lc, struct file_lock_core *bc) +{ + bool rc; + struct file_lease *lease = file_lease(lc); + struct file_lease *breaker = file_lease(bc); + + if (lease->fl_lmops->lm_breaker_owns_lease + && lease->fl_lmops->lm_breaker_owns_lease(lease)) + return false; + if ((bc->flc_flags & FL_LAYOUT) != (lc->flc_flags & FL_LAYOUT)) { + rc = false; + goto trace; + } + if ((bc->flc_flags & FL_DELEG) && (lc->flc_flags & FL_LEASE)) { + rc = false; + goto trace; + } + + rc = locks_conflict(bc, lc); +trace: + trace_leases_conflict(rc, lease, breaker); + return rc; +} + +#define IGNORE_MASK (FL_IGN_DIR_CREATE | FL_IGN_DIR_DELETE | FL_IGN_DIR_RENAME) + +/** + * inode_lease_ignore_mask - return union of all ignored inode events for this inode + * @inode: inode of which to get ignore mask + * + * Walk the list of leases, and return the result of all of + * their FL_IGN_DIR_* bits or'ed together. + */ +u32 +inode_lease_ignore_mask(struct inode *inode) +{ + struct file_lock_context *ctx; + struct file_lock_core *flc; + u32 mask = 0; + + ctx = locks_inode_context(inode); + if (!ctx) + return 0; + + spin_lock(&ctx->flc_lock); + list_for_each_entry(flc, &ctx->flc_lease, flc_list) { + mask |= flc->flc_flags & IGNORE_MASK; + /* If we already have everything, we can stop */ + if (mask == IGNORE_MASK) + break; + } + spin_unlock(&ctx->flc_lock); + return mask; +} +EXPORT_SYMBOL_GPL(inode_lease_ignore_mask); + +static bool +ignore_dir_deleg_break(struct file_lease *fl, unsigned int flags) +{ + if ((flags & LEASE_BREAK_DIR_CREATE) && (fl->c.flc_flags & FL_IGN_DIR_CREATE)) + return true; + if ((flags & LEASE_BREAK_DIR_DELETE) && (fl->c.flc_flags & FL_IGN_DIR_DELETE)) + return true; + if ((flags & LEASE_BREAK_DIR_RENAME) && (fl->c.flc_flags & FL_IGN_DIR_RENAME)) + return true; + + return false; +} + +static unsigned int +break_lease_flags_to_type(unsigned int flags) +{ + if (flags & LEASE_BREAK_LEASE) + return FL_LEASE; + else if (flags & LEASE_BREAK_DELEG) + return FL_DELEG; + else if (flags & LEASE_BREAK_LAYOUT) + return FL_LAYOUT; + else + return 0; + +} + +static struct file_lease * +first_visible_lease(struct inode *inode, struct file_lease *new_fl, unsigned int flags) +{ + struct file_lock_context *ctx = locks_inode_context(inode); + struct file_lease *fl; + + lockdep_assert_held(&ctx->flc_lock); + + list_for_each_entry(fl, &ctx->flc_lease, c.flc_list) { + if (!leases_conflict(&fl->c, &new_fl->c)) + continue; + if (S_ISDIR(inode->i_mode) && ignore_dir_deleg_break(fl, flags)) + continue; + return fl; + } + return NULL; +} + + +/** + * __break_lease - revoke all outstanding leases on file + * @inode: the inode of the file to return + * @flags: LEASE_BREAK_* flags + * + * break_lease (inlined for speed) has checked there already is at least + * some kind of lock (maybe a lease) on this file. Leases and Delegations + * are broken on a call to open() or truncate(). Delegations are also + * broken on any event that would change the ctime. Directory delegations + * are broken whenever the directory changes (unless the delegation is set + * up to ignore the event). This function can block waiting for the lease + * break unless you specify LEASE_BREAK_NONBLOCK. + */ +int __break_lease(struct inode *inode, unsigned int flags) +{ + struct file_lease *new_fl, *fl, *tmp; + struct file_lock_context *ctx; + unsigned long break_time; + unsigned int type; + LIST_HEAD(dispose); + bool want_write = !(flags & LEASE_BREAK_OPEN_RDONLY); + int error = 0; + + trace_break_lease(inode, flags); + + type = break_lease_flags_to_type(flags); + if (!type) + return -EINVAL; + + new_fl = lease_alloc(NULL, type, want_write ? F_WRLCK : F_RDLCK); + if (IS_ERR(new_fl)) + return PTR_ERR(new_fl); + + /* typically we will check that ctx is non-NULL before calling */ + ctx = locks_inode_context(inode); + if (!ctx) { + WARN_ON_ONCE(1); + goto free_lock; + } + + percpu_down_read(&file_rwsem); + spin_lock(&ctx->flc_lock); + + time_out_leases(inode, &dispose); + + if (!first_visible_lease(inode, new_fl, flags)) + goto out; + + break_time = 0; + if (lease_break_time > 0) { + break_time = jiffies + lease_break_time * HZ; + if (break_time == 0) + break_time++; /* so that 0 means no break time */ + } + + list_for_each_entry_safe(fl, tmp, &ctx->flc_lease, c.flc_list) { + if (!leases_conflict(&fl->c, &new_fl->c)) + continue; + if (S_ISDIR(inode->i_mode) && ignore_dir_deleg_break(fl, flags)) + continue; + if (want_write) { + if (fl->c.flc_flags & FL_UNLOCK_PENDING) + continue; + fl->c.flc_flags |= FL_UNLOCK_PENDING; + fl->fl_break_time = break_time; + } else { + if (lease_breaking(fl)) + continue; + fl->c.flc_flags |= FL_DOWNGRADE_PENDING; + fl->fl_downgrade_time = break_time; + } + if (fl->fl_lmops->lm_break(fl)) + locks_delete_lock_ctx(&fl->c, &dispose); + } + + fl = first_visible_lease(inode, new_fl, flags); + if (!fl) + goto out; + + if (flags & LEASE_BREAK_NONBLOCK) { + trace_break_lease_noblock(inode, new_fl); + error = -EWOULDBLOCK; + goto out; + } + +restart: + break_time = fl->fl_break_time; + if (break_time != 0) { + if (time_after(jiffies, break_time)) { + fl->fl_break_time = jiffies + lease_break_time * HZ; + break_time = lease_break_time * HZ; + } else + break_time -= jiffies; + } else + break_time++; + locks_insert_block(&fl->c, &new_fl->c, leases_conflict); + trace_break_lease_block(inode, fl); + spin_unlock(&ctx->flc_lock); + percpu_up_read(&file_rwsem); + + lease_dispose_list(&dispose); + error = wait_event_interruptible_timeout(new_fl->c.flc_wait, + list_empty(&new_fl->c.flc_blocked_member), + break_time); + + percpu_down_read(&file_rwsem); + spin_lock(&ctx->flc_lock); + trace_break_lease_unblock(inode, NULL); + __locks_delete_block(&new_fl->c); + if (error >= 0) { + /* + * Wait for the next conflicting lease that has not been + * broken yet + */ + if (error == 0) + time_out_leases(inode, &dispose); + fl = first_visible_lease(inode, new_fl, flags); + if (fl) + goto restart; + error = 0; + } +out: + spin_unlock(&ctx->flc_lock); + percpu_up_read(&file_rwsem); + lease_dispose_list(&dispose); +free_lock: + locks_free_lease(new_fl); + return error; +} +EXPORT_SYMBOL(__break_lease); + +/** + * lease_get_mtime - update modified time of an inode with exclusive lease + * @inode: the inode + * @time: pointer to a timespec which contains the last modified time + * + * This is to force NFS clients to flush their caches for files with + * exclusive leases. The justification is that if someone has an + * exclusive lease, then they could be modifying it. + */ +void lease_get_mtime(struct inode *inode, struct timespec64 *time) +{ + bool has_lease = false; + struct file_lock_context *ctx; + struct file_lock_core *flc; + + ctx = locks_inode_context(inode); + if (ctx && !list_empty_careful(&ctx->flc_lease)) { + spin_lock(&ctx->flc_lock); + flc = list_first_entry_or_null(&ctx->flc_lease, + struct file_lock_core, flc_list); + if (flc && flc->flc_type == F_WRLCK) + has_lease = true; + spin_unlock(&ctx->flc_lock); + } + + if (has_lease) + *time = current_time(inode); +} +EXPORT_SYMBOL(lease_get_mtime); + +/** + * __fcntl_getlease - Enquire what lease is currently active + * @filp: the file + * @flavor: type of lease flags to check + * + * The value returned by this function will be one of + * (if no lease break is pending): + * + * %F_RDLCK to indicate a shared lease is held. + * + * %F_WRLCK to indicate an exclusive lease is held. + * + * %F_UNLCK to indicate no lease is held. + * + * (if a lease break is pending): + * + * %F_RDLCK to indicate an exclusive lease needs to be + * changed to a shared lease (or removed). + * + * %F_UNLCK to indicate the lease needs to be removed. + * + * XXX: sfr & willy disagree over whether F_INPROGRESS + * should be returned to userspace. + */ +static int __fcntl_getlease(struct file *filp, unsigned int flavor) +{ + struct file_lease *fl; + struct inode *inode = file_inode(filp); + struct file_lock_context *ctx; + int type = F_UNLCK; + LIST_HEAD(dispose); + + ctx = locks_inode_context(inode); + if (ctx && !list_empty_careful(&ctx->flc_lease)) { + percpu_down_read(&file_rwsem); + spin_lock(&ctx->flc_lock); + time_out_leases(inode, &dispose); + list_for_each_entry(fl, &ctx->flc_lease, c.flc_list) { + if (fl->c.flc_file != filp) + continue; + if (fl->c.flc_flags & flavor) + type = target_leasetype(fl); + break; + } + spin_unlock(&ctx->flc_lock); + percpu_up_read(&file_rwsem); + + lease_dispose_list(&dispose); + } + return type; +} + +int fcntl_getlease(struct file *filp) +{ + return __fcntl_getlease(filp, FL_LEASE); +} + +int fcntl_getdeleg(struct file *filp, struct delegation *deleg) +{ + if (deleg->d_flags != 0 || deleg->__pad != 0) + return -EINVAL; + deleg->d_type = __fcntl_getlease(filp, FL_DELEG); + return 0; +} + +static int +generic_add_lease(struct file *filp, int arg, struct file_lease **flp, void **priv) +{ + struct file_lease *fl, *my_fl = NULL, *lease; + struct inode *inode = file_inode(filp); + struct file_lock_context *ctx; + bool is_deleg = (*flp)->c.flc_flags & FL_DELEG; + int error; + LIST_HEAD(dispose); + + lease = *flp; + trace_generic_add_lease(inode, lease); + + error = file_f_owner_allocate(filp); + if (error) + return error; + + /* Note that arg is never F_UNLCK here */ + ctx = locks_get_lock_context(inode, arg); + if (!ctx) + return -ENOMEM; + + /* + * In the delegation case we need mutual exclusion with + * a number of operations that take the i_rwsem. We trylock + * because delegations are an optional optimization, and if + * there's some chance of a conflict--we'd rather not + * bother, maybe that's a sign this just isn't a good file to + * hand out a delegation on. + */ + if (is_deleg && !inode_trylock(inode)) + return -EAGAIN; + + percpu_down_read(&file_rwsem); + spin_lock(&ctx->flc_lock); + time_out_leases(inode, &dispose); + error = lease->fl_lmops->lm_open_conflict(filp, arg); + if (error) + goto out; + + /* + * At this point, we know that if there is an exclusive + * lease on this file, then we hold it on this filp + * (otherwise our open of this file would have blocked). + * And if we are trying to acquire an exclusive lease, + * then the file is not open by anyone (including us) + * except for this filp. + */ + error = -EAGAIN; + list_for_each_entry(fl, &ctx->flc_lease, c.flc_list) { + if (fl->c.flc_file == filp && + fl->c.flc_owner == lease->c.flc_owner) { + my_fl = fl; + continue; + } + + /* + * No exclusive leases if someone else has a lease on + * this file: + */ + if (arg == F_WRLCK) + goto out; + /* + * Modifying our existing lease is OK, but no getting a + * new lease if someone else is opening for write: + */ + if (fl->c.flc_flags & FL_UNLOCK_PENDING) + goto out; + } + + if (my_fl != NULL) { + lease = my_fl; + error = lease->fl_lmops->lm_change(lease, arg, &dispose); + if (error) + goto out; + goto out_setup; + } + + error = -EINVAL; + if (!leases_enable) + goto out; + + locks_insert_lock_ctx(&lease->c, &ctx->flc_lease); + /* + * The check in break_lease() is lockless. It's possible for another + * open to race in after we did the earlier check for a conflicting + * open but before the lease was inserted. Check again for a + * conflicting open and cancel the lease if there is one. + * + * We also add a barrier here to ensure that the insertion of the lock + * precedes these checks. + */ + smp_mb(); + error = lease->fl_lmops->lm_open_conflict(filp, arg); + if (error) { + locks_unlink_lock_ctx(&lease->c); + goto out; + } + +out_setup: + if (lease->fl_lmops->lm_setup) + lease->fl_lmops->lm_setup(lease, priv); +out: + spin_unlock(&ctx->flc_lock); + percpu_up_read(&file_rwsem); + lease_dispose_list(&dispose); + if (is_deleg) + inode_unlock(inode); + if (!error && !my_fl) + *flp = NULL; + return error; +} + +static int generic_delete_lease(struct file *filp, void *owner) +{ + int error = -EAGAIN; + struct file_lease *fl, *victim = NULL; + struct inode *inode = file_inode(filp); + struct file_lock_context *ctx; + LIST_HEAD(dispose); + + ctx = locks_inode_context(inode); + if (!ctx) { + trace_generic_delete_lease(inode, NULL); + return error; + } + + percpu_down_read(&file_rwsem); + spin_lock(&ctx->flc_lock); + list_for_each_entry(fl, &ctx->flc_lease, c.flc_list) { + if (fl->c.flc_file == filp && + fl->c.flc_owner == owner) { + victim = fl; + break; + } + } + trace_generic_delete_lease(inode, victim); + if (victim) + error = fl->fl_lmops->lm_change(victim, F_UNLCK, &dispose); + spin_unlock(&ctx->flc_lock); + percpu_up_read(&file_rwsem); + lease_dispose_list(&dispose); + return error; +} + +/** + * generic_setlease - sets a lease on an open file + * @filp: file pointer + * @arg: type of lease to obtain + * @flp: input - file_lock to use, output - file_lock inserted + * @priv: private data for lm_setup (may be NULL if lm_setup + * doesn't require it) + * + * The (input) flp->fl_lmops->lm_break function is required + * by break_lease(). + */ +int generic_setlease(struct file *filp, int arg, struct file_lease **flp, + void **priv) +{ + struct inode *inode = file_inode(filp); + + if (!S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode)) + return -EINVAL; + + switch (arg) { + case F_UNLCK: + return generic_delete_lease(filp, *priv); + case F_WRLCK: + if (S_ISDIR(inode->i_mode)) + return -EINVAL; + fallthrough; + case F_RDLCK: + if (!(*flp)->fl_lmops->lm_break) { + WARN_ON_ONCE(1); + return -ENOLCK; + } + + return generic_add_lease(filp, arg, flp, priv); + default: + return -EINVAL; + } +} +EXPORT_SYMBOL(generic_setlease); + +/* + * Kernel subsystems can register to be notified on any attempt to set + * a new lease with the lease_notifier_chain. This is used by (e.g.) nfsd + * to close files that it may have cached when there is an attempt to set a + * conflicting lease. + */ +static struct srcu_notifier_head lease_notifier_chain; + +static inline void +lease_notifier_chain_init(void) +{ + srcu_init_notifier_head(&lease_notifier_chain); +} + +static inline void +setlease_notifier(int arg, struct file_lease *lease) +{ + if (arg != F_UNLCK) + srcu_notifier_call_chain(&lease_notifier_chain, arg, lease); +} + +int lease_register_notifier(struct notifier_block *nb) +{ + return srcu_notifier_chain_register(&lease_notifier_chain, nb); +} +EXPORT_SYMBOL_GPL(lease_register_notifier); + +void lease_unregister_notifier(struct notifier_block *nb) +{ + srcu_notifier_chain_unregister(&lease_notifier_chain, nb); +} +EXPORT_SYMBOL_GPL(lease_unregister_notifier); + + +int +kernel_setlease(struct file *filp, int arg, struct file_lease **lease, void **priv) +{ + if (lease) + setlease_notifier(arg, *lease); + if (filp->f_op->setlease) + return filp->f_op->setlease(filp, arg, lease, priv); + return -EINVAL; +} +EXPORT_SYMBOL_GPL(kernel_setlease); + +/** + * vfs_setlease - sets a lease on an open file + * @filp: file pointer + * @arg: type of lease to obtain + * @lease: file_lock to use when adding a lease + * @priv: private info for lm_setup when adding a lease (may be + * NULL if lm_setup doesn't require it) + * + * Call this to establish a lease on the file. The "lease" argument is not + * used for F_UNLCK requests and may be NULL. For commands that set or alter + * an existing lease, the ``(*lease)->fl_lmops->lm_break`` operation must be + * set; if not, this function will return -ENOLCK (and generate a scary-looking + * stack trace). + * + * The "priv" pointer is passed directly to the lm_setup function as-is. It + * may be NULL if the lm_setup operation doesn't require it. + */ +int +vfs_setlease(struct file *filp, int arg, struct file_lease **lease, void **priv) +{ + struct inode *inode = file_inode(filp); + vfsuid_t vfsuid = i_uid_into_vfsuid(file_mnt_idmap(filp), inode); + int error; + + if ((!vfsuid_eq_kuid(vfsuid, current_fsuid())) && !capable(CAP_LEASE)) + return -EACCES; + error = security_file_lock(filp, arg); + if (error) + return error; + return kernel_setlease(filp, arg, lease, priv); +} +EXPORT_SYMBOL_GPL(vfs_setlease); + +static int do_fcntl_add_lease(unsigned int fd, struct file *filp, unsigned int flavor, int arg) +{ + struct file_lease *fl; + struct fasync_struct *new; + int error; + + fl = lease_alloc(filp, flavor, arg); + if (IS_ERR(fl)) + return PTR_ERR(fl); + + new = fasync_alloc(); + if (!new) { + locks_free_lease(fl); + return -ENOMEM; + } + new->fa_fd = fd; + + error = vfs_setlease(filp, arg, &fl, (void **)&new); + if (fl) + locks_free_lease(fl); + if (new) + fasync_free(new); + return error; +} + +/** + * fcntl_setlease - sets a lease on an open file + * @fd: open file descriptor + * @filp: file pointer + * @arg: type of lease to obtain + * + * Call this fcntl to establish a lease on the file. + * Note that you also need to call %F_SETSIG to + * receive a signal when the lease is broken. + */ +int fcntl_setlease(unsigned int fd, struct file *filp, int arg) +{ + if (S_ISDIR(file_inode(filp)->i_mode)) + return -EINVAL; + + if (arg == F_UNLCK) + return vfs_setlease(filp, F_UNLCK, NULL, (void **)&filp); + return do_fcntl_add_lease(fd, filp, FL_LEASE, arg); +} + +/** + * fcntl_setdeleg - sets a delegation on an open file + * @fd: open file descriptor + * @filp: file pointer + * @deleg: delegation request from userland + * + * Call this fcntl to establish a delegation on the file. + * Note that you also need to call %F_SETSIG to + * receive a signal when the lease is broken. + */ +int fcntl_setdeleg(unsigned int fd, struct file *filp, struct delegation *deleg) +{ + /* For now, no flags are supported */ + if (deleg->d_flags != 0 || deleg->__pad != 0) + return -EINVAL; + + if (deleg->d_type == F_UNLCK) + return vfs_setlease(filp, F_UNLCK, NULL, (void **)&filp); + return do_fcntl_add_lease(fd, filp, FL_DELEG, deleg->d_type); +} + +/** + * flock_lock_inode_wait - Apply a FLOCK-style lock to a file + * @inode: inode of the file to apply to + * @fl: The lock to be applied + * + * Apply a FLOCK style lock request to an inode. + */ +static int flock_lock_inode_wait(struct inode *inode, struct file_lock *fl) +{ + int error; + might_sleep(); + for (;;) { + error = flock_lock_inode(inode, fl); + if (error != FILE_LOCK_DEFERRED) + break; + error = wait_event_interruptible(fl->c.flc_wait, + list_empty(&fl->c.flc_blocked_member)); + if (error) + break; + } + locks_delete_block(fl); + return error; +} + +/** + * locks_lock_inode_wait - Apply a lock to an inode + * @inode: inode of the file to apply to + * @fl: The lock to be applied + * + * Apply a POSIX or FLOCK style lock request to an inode. + */ +int locks_lock_inode_wait(struct inode *inode, struct file_lock *fl) +{ + int res = 0; + switch (fl->c.flc_flags & (FL_POSIX|FL_FLOCK)) { + case FL_POSIX: + res = posix_lock_inode_wait(inode, fl); + break; + case FL_FLOCK: + res = flock_lock_inode_wait(inode, fl); + break; + default: + BUG(); + } + return res; +} +EXPORT_SYMBOL(locks_lock_inode_wait); + +/** + * sys_flock: - flock() system call. + * @fd: the file descriptor to lock. + * @cmd: the type of lock to apply. + * + * Apply a %FL_FLOCK style lock to an open file descriptor. + * The @cmd can be one of: + * + * - %LOCK_SH -- a shared lock. + * - %LOCK_EX -- an exclusive lock. + * - %LOCK_UN -- remove an existing lock. + * - %LOCK_MAND -- a 'mandatory' flock. (DEPRECATED) + * + * %LOCK_MAND support has been removed from the kernel. + */ +SYSCALL_DEFINE2(flock, unsigned int, fd, unsigned int, cmd) +{ + int can_sleep, error, type; + struct file_lock fl; + + /* + * LOCK_MAND locks were broken for a long time in that they never + * conflicted with one another and didn't prevent any sort of open, + * read or write activity. + * + * Just ignore these requests now, to preserve legacy behavior, but + * throw a warning to let people know that they don't actually work. + */ + if (cmd & LOCK_MAND) { + pr_warn_once("%s(%d): Attempt to set a LOCK_MAND lock via flock(2). This support has been removed and the request ignored.\n", current->comm, current->pid); + return 0; + } + + type = flock_translate_cmd(cmd & ~LOCK_NB); + if (type < 0) + return type; + + CLASS(fd, f)(fd); + if (fd_empty(f)) + return -EBADF; + + if (type != F_UNLCK && !(fd_file(f)->f_mode & (FMODE_READ | FMODE_WRITE))) + return -EBADF; + + flock_make_lock(fd_file(f), &fl, type); + + error = security_file_lock(fd_file(f), fl.c.flc_type); + if (error) + return error; + + can_sleep = !(cmd & LOCK_NB); + if (can_sleep) + fl.c.flc_flags |= FL_SLEEP; + + if (fd_file(f)->f_op->flock) + error = fd_file(f)->f_op->flock(fd_file(f), + (can_sleep) ? F_SETLKW : F_SETLK, + &fl); + else + error = locks_lock_file_wait(fd_file(f), &fl); + + locks_release_private(&fl); + return error; +} + +/** + * vfs_test_lock - test file byte range lock + * @filp: The file to test lock for + * @fl: The byte-range in the file to test; also used to hold result + * + * On entry, @fl does not contain a lock, but identifies a range (fl_start, fl_end) + * in the file (c.flc_file), and an owner (c.flc_owner) for whom existing locks + * should be ignored. c.flc_type and c.flc_flags are ignored. + * Both fl_lmops and fl_ops in @fl must be NULL. + * Returns -ERRNO on failure. Indicates presence of conflicting lock by + * setting fl->fl_type to something other than F_UNLCK. + * + * If vfs_test_lock() does find a lock and return it, the caller must + * use locks_free_lock() or locks_release_private() on the returned lock. + */ +int vfs_test_lock(struct file *filp, struct file_lock *fl) +{ + int error = 0; + + WARN_ON_ONCE(fl->fl_ops || fl->fl_lmops); + WARN_ON_ONCE(filp != fl->c.flc_file); + if (filp->f_op->lock) + error = filp->f_op->lock(filp, F_GETLK, fl); + else + posix_test_lock(filp, fl); + + /* + * We don't expect FILE_LOCK_DEFERRED and callers cannot + * handle it. + */ + if (WARN_ON_ONCE(error == FILE_LOCK_DEFERRED)) + error = -EIO; + + return error; +} +EXPORT_SYMBOL_GPL(vfs_test_lock); + +/** + * locks_translate_pid - translate a file_lock's fl_pid number into a namespace + * @fl: The file_lock who's fl_pid should be translated + * @ns: The namespace into which the pid should be translated + * + * Used to translate a fl_pid into a namespace virtual pid number + */ +static pid_t locks_translate_pid(struct file_lock_core *fl, struct pid_namespace *ns) +{ + pid_t vnr; + struct pid *pid; + + if (fl->flc_flags & FL_OFDLCK) + return -1; + + /* Remote locks report a negative pid value */ + if (fl->flc_pid <= 0) + return fl->flc_pid; + + /* + * If the flock owner process is dead and its pid has been already + * freed, the translation below won't work, but we still want to show + * flock owner pid number in init pidns. + */ + if (ns == &init_pid_ns) + return (pid_t) fl->flc_pid; + + rcu_read_lock(); + pid = find_pid_ns(fl->flc_pid, &init_pid_ns); + vnr = pid_nr_ns(pid, ns); + rcu_read_unlock(); + return vnr; +} + +static int posix_lock_to_flock(struct flock *flock, struct file_lock *fl) +{ + flock->l_pid = locks_translate_pid(&fl->c, task_active_pid_ns(current)); +#if BITS_PER_LONG == 32 + /* + * Make sure we can represent the posix lock via + * legacy 32bit flock. + */ + if (fl->fl_start > OFFT_OFFSET_MAX) + return -EOVERFLOW; + if (fl->fl_end != OFFSET_MAX && fl->fl_end > OFFT_OFFSET_MAX) + return -EOVERFLOW; +#endif + flock->l_start = fl->fl_start; + flock->l_len = fl->fl_end == OFFSET_MAX ? 0 : + fl->fl_end - fl->fl_start + 1; + flock->l_whence = 0; + flock->l_type = fl->c.flc_type; + return 0; +} + +#if BITS_PER_LONG == 32 +static void posix_lock_to_flock64(struct flock64 *flock, struct file_lock *fl) +{ + flock->l_pid = locks_translate_pid(&fl->c, task_active_pid_ns(current)); + flock->l_start = fl->fl_start; + flock->l_len = fl->fl_end == OFFSET_MAX ? 0 : + fl->fl_end - fl->fl_start + 1; + flock->l_whence = 0; + flock->l_type = fl->c.flc_type; +} +#endif + +/* Report the first existing lock that would conflict with l. + * This implements the F_GETLK command of fcntl(). + */ +int fcntl_getlk(struct file *filp, unsigned int cmd, struct flock *flock) +{ + struct file_lock *fl; + int error; + + fl = locks_alloc_lock(); + if (fl == NULL) + return -ENOMEM; + error = -EINVAL; + if (cmd != F_OFD_GETLK && flock->l_type != F_RDLCK + && flock->l_type != F_WRLCK) + goto out; + + error = flock_to_posix_lock(filp, fl, flock); + if (error) + goto out; + + if (cmd == F_OFD_GETLK) { + error = -EINVAL; + if (flock->l_pid != 0) + goto out; + + fl->c.flc_flags |= FL_OFDLCK; + fl->c.flc_owner = filp; + } + + error = vfs_test_lock(filp, fl); + if (error) + goto out; + + flock->l_type = fl->c.flc_type; + if (fl->c.flc_type != F_UNLCK) { + error = posix_lock_to_flock(flock, fl); + if (error) + goto out; + } +out: + locks_free_lock(fl); + return error; +} + +/** + * vfs_lock_file - file byte range lock + * @filp: The file to apply the lock to + * @cmd: type of locking operation (F_SETLK, F_GETLK, etc.) + * @fl: The lock to be applied + * @conf: Place to return a copy of the conflicting lock, if found. + * + * A caller that doesn't care about the conflicting lock may pass NULL + * as the final argument. + * + * If the filesystem defines a private ->lock() method, then @conf will + * be left unchanged; so a caller that cares should initialize it to + * some acceptable default. + * + * To avoid blocking kernel daemons, such as lockd, that need to acquire POSIX + * locks, the ->lock() interface may return asynchronously, before the lock has + * been granted or denied by the underlying filesystem, if (and only if) + * lm_grant is set. Additionally FOP_ASYNC_LOCK in file_operations fop_flags + * need to be set. + * + * Callers expecting ->lock() to return asynchronously will only use F_SETLK, + * not F_SETLKW; they will set FL_SLEEP if (and only if) the request is for a + * blocking lock. When ->lock() does return asynchronously, it must return + * FILE_LOCK_DEFERRED, and call ->lm_grant() when the lock request completes. + * If the request is for non-blocking lock the file system should return + * FILE_LOCK_DEFERRED then try to get the lock and call the callback routine + * with the result. If the request timed out the callback routine will return a + * nonzero return code and the file system should release the lock. The file + * system is also responsible to keep a corresponding posix lock when it + * grants a lock so the VFS can find out which locks are locally held and do + * the correct lock cleanup when required. + * The underlying filesystem must not drop the kernel lock or call + * ->lm_grant() before returning to the caller with a FILE_LOCK_DEFERRED + * return code. + */ +int vfs_lock_file(struct file *filp, unsigned int cmd, struct file_lock *fl, struct file_lock *conf) +{ + WARN_ON_ONCE(filp != fl->c.flc_file); + if (filp->f_op->lock) + return filp->f_op->lock(filp, cmd, fl); + else + return posix_lock_file(filp, fl, conf); +} +EXPORT_SYMBOL_GPL(vfs_lock_file); + +static int do_lock_file_wait(struct file *filp, unsigned int cmd, + struct file_lock *fl) +{ + int error; + + error = security_file_lock(filp, fl->c.flc_type); + if (error) + return error; + + for (;;) { + error = vfs_lock_file(filp, cmd, fl, NULL); + if (error != FILE_LOCK_DEFERRED) + break; + error = wait_event_interruptible(fl->c.flc_wait, + list_empty(&fl->c.flc_blocked_member)); + if (error) + break; + } + locks_delete_block(fl); + + return error; +} + +/* Ensure that fl->fl_file has compatible f_mode for F_SETLK calls */ +static int +check_fmode_for_setlk(struct file_lock *fl) +{ + switch (fl->c.flc_type) { + case F_RDLCK: + if (!(fl->c.flc_file->f_mode & FMODE_READ)) + return -EBADF; + break; + case F_WRLCK: + if (!(fl->c.flc_file->f_mode & FMODE_WRITE)) + return -EBADF; + } + return 0; +} + +/* Apply the lock described by l to an open file descriptor. + * This implements both the F_SETLK and F_SETLKW commands of fcntl(). + */ +int fcntl_setlk(unsigned int fd, struct file *filp, unsigned int cmd, + struct flock *flock) +{ + struct file_lock *file_lock = locks_alloc_lock(); + struct inode *inode = file_inode(filp); + struct file *f; + int error; + + if (file_lock == NULL) + return -ENOLCK; + + error = flock_to_posix_lock(filp, file_lock, flock); + if (error) + goto out; + + error = check_fmode_for_setlk(file_lock); + if (error) + goto out; + + /* + * If the cmd is requesting file-private locks, then set the + * FL_OFDLCK flag and override the owner. + */ + switch (cmd) { + case F_OFD_SETLK: + error = -EINVAL; + if (flock->l_pid != 0) + goto out; + + cmd = F_SETLK; + file_lock->c.flc_flags |= FL_OFDLCK; + file_lock->c.flc_owner = filp; + break; + case F_OFD_SETLKW: + error = -EINVAL; + if (flock->l_pid != 0) + goto out; + + cmd = F_SETLKW; + file_lock->c.flc_flags |= FL_OFDLCK; + file_lock->c.flc_owner = filp; + fallthrough; + case F_SETLKW: + file_lock->c.flc_flags |= FL_SLEEP; + } + + error = do_lock_file_wait(filp, cmd, file_lock); + + /* + * Detect close/fcntl races and recover by zapping all POSIX locks + * associated with this file and our files_struct, just like on + * filp_flush(). There is no need to do that when we're + * unlocking though, or for OFD locks. + */ + if (!error && file_lock->c.flc_type != F_UNLCK && + !(file_lock->c.flc_flags & FL_OFDLCK)) { + struct files_struct *files = current->files; + /* + * We need that spin_lock here - it prevents reordering between + * update of i_flctx->flc_posix and check for it done in + * close(). rcu_read_lock() wouldn't do. + */ + spin_lock(&files->file_lock); + f = files_lookup_fd_locked(files, fd); + spin_unlock(&files->file_lock); + if (f != filp) { + locks_remove_posix(filp, files); + error = -EBADF; + } + } +out: + trace_fcntl_setlk(inode, file_lock, error); + locks_free_lock(file_lock); + return error; +} + +#if BITS_PER_LONG == 32 +/* Report the first existing lock that would conflict with l. + * This implements the F_GETLK command of fcntl(). + */ +int fcntl_getlk64(struct file *filp, unsigned int cmd, struct flock64 *flock) +{ + struct file_lock *fl; + int error; + + fl = locks_alloc_lock(); + if (fl == NULL) + return -ENOMEM; + + error = -EINVAL; + if (cmd != F_OFD_GETLK && flock->l_type != F_RDLCK + && flock->l_type != F_WRLCK) + goto out; + + error = flock64_to_posix_lock(filp, fl, flock); + if (error) + goto out; + + if (cmd == F_OFD_GETLK) { + error = -EINVAL; + if (flock->l_pid != 0) + goto out; + + fl->c.flc_flags |= FL_OFDLCK; + fl->c.flc_owner = filp; + } + + error = vfs_test_lock(filp, fl); + if (error) + goto out; + + flock->l_type = fl->c.flc_type; + if (fl->c.flc_type != F_UNLCK) + posix_lock_to_flock64(flock, fl); + +out: + locks_free_lock(fl); + return error; +} + +/* Apply the lock described by l to an open file descriptor. + * This implements both the F_SETLK and F_SETLKW commands of fcntl(). + */ +int fcntl_setlk64(unsigned int fd, struct file *filp, unsigned int cmd, + struct flock64 *flock) +{ + struct file_lock *file_lock = locks_alloc_lock(); + struct file *f; + int error; + + if (file_lock == NULL) + return -ENOLCK; + + error = flock64_to_posix_lock(filp, file_lock, flock); + if (error) + goto out; + + error = check_fmode_for_setlk(file_lock); + if (error) + goto out; + + /* + * If the cmd is requesting file-private locks, then set the + * FL_OFDLCK flag and override the owner. + */ + switch (cmd) { + case F_OFD_SETLK: + error = -EINVAL; + if (flock->l_pid != 0) + goto out; + + cmd = F_SETLK64; + file_lock->c.flc_flags |= FL_OFDLCK; + file_lock->c.flc_owner = filp; + break; + case F_OFD_SETLKW: + error = -EINVAL; + if (flock->l_pid != 0) + goto out; + + cmd = F_SETLKW64; + file_lock->c.flc_flags |= FL_OFDLCK; + file_lock->c.flc_owner = filp; + fallthrough; + case F_SETLKW64: + file_lock->c.flc_flags |= FL_SLEEP; + } + + error = do_lock_file_wait(filp, cmd, file_lock); + + /* + * Detect close/fcntl races and recover by zapping all POSIX locks + * associated with this file and our files_struct, just like on + * filp_flush(). There is no need to do that when we're + * unlocking though, or for OFD locks. + */ + if (!error && file_lock->c.flc_type != F_UNLCK && + !(file_lock->c.flc_flags & FL_OFDLCK)) { + struct files_struct *files = current->files; + /* + * We need that spin_lock here - it prevents reordering between + * update of i_flctx->flc_posix and check for it done in + * close(). rcu_read_lock() wouldn't do. + */ + spin_lock(&files->file_lock); + f = files_lookup_fd_locked(files, fd); + spin_unlock(&files->file_lock); + if (f != filp) { + locks_remove_posix(filp, files); + error = -EBADF; + } + } +out: + locks_free_lock(file_lock); + return error; +} +#endif /* BITS_PER_LONG == 32 */ + +/* + * This function is called when the file is being removed + * from the task's fd array. POSIX locks belonging to this task + * are deleted at this time. + */ +void locks_remove_posix(struct file *filp, fl_owner_t owner) +{ + int error; + struct inode *inode = file_inode(filp); + struct file_lock lock; + struct file_lock_context *ctx; + + /* + * If there are no locks held on this file, we don't need to call + * posix_lock_file(). Another process could be setting a lock on this + * file at the same time, but we wouldn't remove that lock anyway. + */ + ctx = locks_inode_context(inode); + if (!ctx || list_empty(&ctx->flc_posix)) + return; + + locks_init_lock(&lock); + lock.c.flc_type = F_UNLCK; + lock.c.flc_flags = FL_POSIX | FL_CLOSE; + lock.fl_start = 0; + lock.fl_end = OFFSET_MAX; + lock.c.flc_owner = owner; + lock.c.flc_pid = current->tgid; + lock.c.flc_file = filp; + lock.fl_ops = NULL; + lock.fl_lmops = NULL; + + error = vfs_lock_file(filp, F_SETLK, &lock, NULL); + + if (lock.fl_ops && lock.fl_ops->fl_release_private) + lock.fl_ops->fl_release_private(&lock); + trace_locks_remove_posix(inode, &lock, error); +} +EXPORT_SYMBOL(locks_remove_posix); + +/* The i_flctx must be valid when calling into here */ +static void +locks_remove_flock(struct file *filp, struct file_lock_context *flctx) +{ + struct file_lock fl; + struct inode *inode = file_inode(filp); + + if (list_empty(&flctx->flc_flock)) + return; + + flock_make_lock(filp, &fl, F_UNLCK); + fl.c.flc_flags |= FL_CLOSE; + + if (filp->f_op->flock) + filp->f_op->flock(filp, F_SETLKW, &fl); + else + flock_lock_inode(inode, &fl); + + if (fl.fl_ops && fl.fl_ops->fl_release_private) + fl.fl_ops->fl_release_private(&fl); +} + +/* The i_flctx must be valid when calling into here */ +static void +locks_remove_lease(struct file *filp, struct file_lock_context *ctx) +{ + struct file_lease *fl, *tmp; + LIST_HEAD(dispose); + + if (list_empty(&ctx->flc_lease)) + return; + + percpu_down_read(&file_rwsem); + spin_lock(&ctx->flc_lock); + list_for_each_entry_safe(fl, tmp, &ctx->flc_lease, c.flc_list) + if (filp == fl->c.flc_file) + lease_modify(fl, F_UNLCK, &dispose); + spin_unlock(&ctx->flc_lock); + percpu_up_read(&file_rwsem); + + lease_dispose_list(&dispose); +} + +/* + * This function is called on the last close of an open file. + */ +void locks_remove_file(struct file *filp) +{ + struct file_lock_context *ctx; + + ctx = locks_inode_context(file_inode(filp)); + if (!ctx) + return; + + /* remove any OFD locks */ + locks_remove_posix(filp, filp); + + /* remove flock locks */ + locks_remove_flock(filp, ctx); + + /* remove any leases */ + locks_remove_lease(filp, ctx); + + spin_lock(&ctx->flc_lock); + locks_check_ctx_file_list(filp, &ctx->flc_posix, "POSIX"); + locks_check_ctx_file_list(filp, &ctx->flc_flock, "FLOCK"); + locks_check_ctx_file_list(filp, &ctx->flc_lease, "LEASE"); + spin_unlock(&ctx->flc_lock); +} + +/** + * vfs_cancel_lock - file byte range unblock lock + * @filp: The file to apply the unblock to + * @fl: The lock to be unblocked + * + * Used by lock managers to cancel blocked requests + */ +int vfs_cancel_lock(struct file *filp, struct file_lock *fl) +{ + WARN_ON_ONCE(filp != fl->c.flc_file); + if (filp->f_op->lock) + return filp->f_op->lock(filp, F_CANCELLK, fl); + return 0; +} +EXPORT_SYMBOL_GPL(vfs_cancel_lock); + +/** + * vfs_inode_has_locks - are any file locks held on @inode? + * @inode: inode to check for locks + * + * Return true if there are any FL_POSIX or FL_FLOCK locks currently + * set on @inode. + */ +bool vfs_inode_has_locks(struct inode *inode) +{ + struct file_lock_context *ctx; + bool ret; + + ctx = locks_inode_context(inode); + if (!ctx) + return false; + + spin_lock(&ctx->flc_lock); + ret = !list_empty(&ctx->flc_posix) || !list_empty(&ctx->flc_flock); + spin_unlock(&ctx->flc_lock); + return ret; +} +EXPORT_SYMBOL_GPL(vfs_inode_has_locks); + +#ifdef CONFIG_PROC_FS +#include +#include + +struct locks_iterator { + int li_cpu; + loff_t li_pos; +}; + +static void lock_get_status(struct seq_file *f, struct file_lock_core *flc, + loff_t id, char *pfx, int repeat) +{ + struct inode *inode = NULL; + unsigned int pid; + struct pid_namespace *proc_pidns = proc_pid_ns(file_inode(f->file)->i_sb); + int type = flc->flc_type; + struct file_lock *fl = file_lock(flc); + + pid = locks_translate_pid(flc, proc_pidns); + + /* + * If lock owner is dead (and pid is freed) or not visible in current + * pidns, zero is shown as a pid value. Check lock info from + * init_pid_ns to get saved lock pid value. + */ + if (flc->flc_file != NULL) + inode = file_inode(flc->flc_file); + + seq_printf(f, "%lld: ", id); + + if (repeat) + seq_printf(f, "%*s", repeat - 1 + (int)strlen(pfx), pfx); + + if (flc->flc_flags & FL_POSIX) { + if (flc->flc_flags & FL_ACCESS) + seq_puts(f, "ACCESS"); + else if (flc->flc_flags & FL_OFDLCK) + seq_puts(f, "OFDLCK"); + else + seq_puts(f, "POSIX "); + + seq_printf(f, " %s ", + (inode == NULL) ? "*NOINODE*" : "ADVISORY "); + } else if (flc->flc_flags & FL_FLOCK) { + seq_puts(f, "FLOCK ADVISORY "); + } else if (flc->flc_flags & (FL_LEASE|FL_DELEG|FL_LAYOUT)) { + struct file_lease *lease = file_lease(flc); + + type = target_leasetype(lease); + + if (flc->flc_flags & FL_DELEG) + seq_puts(f, "DELEG "); + else + seq_puts(f, "LEASE "); + + if (lease_breaking(lease)) + seq_puts(f, "BREAKING "); + else if (flc->flc_file) + seq_puts(f, "ACTIVE "); + else + seq_puts(f, "BREAKER "); + } else { + seq_puts(f, "UNKNOWN UNKNOWN "); + } + + seq_printf(f, "%s ", (type == F_WRLCK) ? "WRITE" : + (type == F_RDLCK) ? "READ" : "UNLCK"); + if (inode) { + /* userspace relies on this representation of dev_t */ + seq_printf(f, "%d %02x:%02x:%llu ", pid, + MAJOR(inode->i_sb->s_dev), + MINOR(inode->i_sb->s_dev), inode->i_ino); + } else { + seq_printf(f, "%d :0 ", pid); + } + if (flc->flc_flags & FL_POSIX) { + if (fl->fl_end == OFFSET_MAX) + seq_printf(f, "%Ld EOF\n", fl->fl_start); + else + seq_printf(f, "%Ld %Ld\n", fl->fl_start, fl->fl_end); + } else { + seq_puts(f, "0 EOF\n"); + } +} + +static struct file_lock_core *get_next_blocked_member(struct file_lock_core *node) +{ + struct file_lock_core *tmp; + + /* NULL node or root node */ + if (node == NULL || node->flc_blocker == NULL) + return NULL; + + /* Next member in the linked list could be itself */ + tmp = list_next_entry(node, flc_blocked_member); + if (list_entry_is_head(tmp, &node->flc_blocker->flc_blocked_requests, + flc_blocked_member) + || tmp == node) { + return NULL; + } + + return tmp; +} + +static int locks_show(struct seq_file *f, void *v) +{ + struct locks_iterator *iter = f->private; + struct file_lock_core *cur, *tmp; + struct pid_namespace *proc_pidns = proc_pid_ns(file_inode(f->file)->i_sb); + int level = 0; + + cur = hlist_entry(v, struct file_lock_core, flc_link); + + if (locks_translate_pid(cur, proc_pidns) == 0) + return 0; + + /* View this crossed linked list as a binary tree, the first member of flc_blocked_requests + * is the left child of current node, the next silibing in flc_blocked_member is the + * right child, we can alse get the parent of current node from flc_blocker, so this + * question becomes traversal of a binary tree + */ + while (cur != NULL) { + if (level) + lock_get_status(f, cur, iter->li_pos, "-> ", level); + else + lock_get_status(f, cur, iter->li_pos, "", level); + + if (!list_empty(&cur->flc_blocked_requests)) { + /* Turn left */ + cur = list_first_entry_or_null(&cur->flc_blocked_requests, + struct file_lock_core, + flc_blocked_member); + level++; + } else { + /* Turn right */ + tmp = get_next_blocked_member(cur); + /* Fall back to parent node */ + while (tmp == NULL && cur->flc_blocker != NULL) { + cur = cur->flc_blocker; + level--; + tmp = get_next_blocked_member(cur); + } + cur = tmp; + } + } + + return 0; +} + +static void __show_fd_locks(struct seq_file *f, + struct list_head *head, int *id, + struct file *filp, struct files_struct *files) +{ + struct file_lock_core *fl; + + list_for_each_entry(fl, head, flc_list) { + + if (filp != fl->flc_file) + continue; + if (fl->flc_owner != files && fl->flc_owner != filp) + continue; + + (*id)++; + seq_puts(f, "lock:\t"); + lock_get_status(f, fl, *id, "", 0); + } +} + +void show_fd_locks(struct seq_file *f, + struct file *filp, struct files_struct *files) +{ + struct inode *inode = file_inode(filp); + struct file_lock_context *ctx; + int id = 0; + + ctx = locks_inode_context(inode); + if (!ctx) + return; + + spin_lock(&ctx->flc_lock); + __show_fd_locks(f, &ctx->flc_flock, &id, filp, files); + __show_fd_locks(f, &ctx->flc_posix, &id, filp, files); + __show_fd_locks(f, &ctx->flc_lease, &id, filp, files); + spin_unlock(&ctx->flc_lock); +} + +static void *locks_start(struct seq_file *f, loff_t *pos) + __acquires(&blocked_lock_lock) +{ + struct locks_iterator *iter = f->private; + + iter->li_pos = *pos + 1; + percpu_down_write(&file_rwsem); + spin_lock(&blocked_lock_lock); + return seq_hlist_start_percpu(&file_lock_list.hlist, &iter->li_cpu, *pos); +} + +static void *locks_next(struct seq_file *f, void *v, loff_t *pos) +{ + struct locks_iterator *iter = f->private; + + ++iter->li_pos; + return seq_hlist_next_percpu(v, &file_lock_list.hlist, &iter->li_cpu, pos); +} + +static void locks_stop(struct seq_file *f, void *v) + __releases(&blocked_lock_lock) +{ + spin_unlock(&blocked_lock_lock); + percpu_up_write(&file_rwsem); +} + +static const struct seq_operations locks_seq_operations = { + .start = locks_start, + .next = locks_next, + .stop = locks_stop, + .show = locks_show, +}; + +static int __init proc_locks_init(void) +{ + proc_create_seq_private("locks", 0, NULL, &locks_seq_operations, + sizeof(struct locks_iterator), NULL); + return 0; +} +fs_initcall(proc_locks_init); +#endif + +static int __init filelock_init(void) +{ + int i; + + flctx_cache = kmem_cache_create("file_lock_ctx", + sizeof(struct file_lock_context), 0, SLAB_PANIC, NULL); + + filelock_cache = kmem_cache_create("file_lock_cache", + sizeof(struct file_lock), 0, SLAB_PANIC, NULL); + + filelease_cache = kmem_cache_create("file_lease_cache", + sizeof(struct file_lease), 0, SLAB_PANIC, NULL); + + for_each_possible_cpu(i) { + struct file_lock_list_struct *fll = per_cpu_ptr(&file_lock_list, i); + + spin_lock_init(&fll->lock); + INIT_HLIST_HEAD(&fll->hlist); + } + + lease_notifier_chain_init(); + return 0; +} +core_initcall(filelock_init); diff --git a/fs/mbcache.c b/fs/mbcache.c new file mode 100644 index 000000000..2a6319b40 --- /dev/null +++ b/fs/mbcache.c @@ -0,0 +1,444 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Mbcache is a simple key-value store. Keys need not be unique, however + * key-value pairs are expected to be unique (we use this fact in + * mb_cache_entry_delete_or_get()). + * + * Ext2 and ext4 use this cache for deduplication of extended attribute blocks. + * Ext4 also uses it for deduplication of xattr values stored in inodes. + * They use hash of data as a key and provide a value that may represent a + * block or inode number. That's why keys need not be unique (hash of different + * data may be the same). However user provided value always uniquely + * identifies a cache entry. + * + * We provide functions for creation and removal of entries, search by key, + * and a special "delete entry with given key-value pair" operation. Fixed + * size hash table is used for fast key lookups. + */ + +struct mb_cache { + /* Hash table of entries */ + struct hlist_bl_head *c_hash; + /* log2 of hash table size */ + int c_bucket_bits; + /* Maximum entries in cache to avoid degrading hash too much */ + unsigned long c_max_entries; + /* Protects c_list, c_entry_count */ + spinlock_t c_list_lock; + struct list_head c_list; + /* Number of entries in cache */ + unsigned long c_entry_count; + struct shrinker *c_shrink; + /* Work for shrinking when the cache has too many entries */ + struct work_struct c_shrink_work; +}; + +static struct kmem_cache *mb_entry_cache; + +static unsigned long mb_cache_shrink(struct mb_cache *cache, + unsigned long nr_to_scan); + +static inline struct hlist_bl_head *mb_cache_entry_head(struct mb_cache *cache, + u32 key) +{ + return &cache->c_hash[hash_32(key, cache->c_bucket_bits)]; +} + +/* + * Number of entries to reclaim synchronously when there are too many entries + * in cache + */ +#define SYNC_SHRINK_BATCH 64 + +/* + * mb_cache_entry_create - create entry in cache + * @cache - cache where the entry should be created + * @mask - gfp mask with which the entry should be allocated + * @key - key of the entry + * @value - value of the entry + * @reusable - is the entry reusable by others? + * + * Creates entry in @cache with key @key and value @value. The function returns + * -EBUSY if entry with the same key and value already exists in cache. + * Otherwise 0 is returned. + */ +int mb_cache_entry_create(struct mb_cache *cache, gfp_t mask, u32 key, + u64 value, bool reusable) +{ + struct mb_cache_entry *entry, *dup; + struct hlist_bl_node *dup_node; + struct hlist_bl_head *head; + + /* Schedule background reclaim if there are too many entries */ + if (cache->c_entry_count >= cache->c_max_entries) + schedule_work(&cache->c_shrink_work); + /* Do some sync reclaim if background reclaim cannot keep up */ + if (cache->c_entry_count >= 2*cache->c_max_entries) + mb_cache_shrink(cache, SYNC_SHRINK_BATCH); + + entry = kmem_cache_alloc(mb_entry_cache, mask); + if (!entry) + return -ENOMEM; + + INIT_LIST_HEAD(&entry->e_list); + /* + * We create entry with two references. One reference is kept by the + * hash table, the other reference is used to protect us from + * mb_cache_entry_delete_or_get() until the entry is fully setup. This + * avoids nesting of cache->c_list_lock into hash table bit locks which + * is problematic for RT. + */ + atomic_set(&entry->e_refcnt, 2); + entry->e_key = key; + entry->e_value = value; + entry->e_flags = 0; + if (reusable) + set_bit(MBE_REUSABLE_B, &entry->e_flags); + head = mb_cache_entry_head(cache, key); + hlist_bl_lock(head); + hlist_bl_for_each_entry(dup, dup_node, head, e_hash_list) { + if (dup->e_key == key && dup->e_value == value) { + hlist_bl_unlock(head); + kmem_cache_free(mb_entry_cache, entry); + return -EBUSY; + } + } + hlist_bl_add_head(&entry->e_hash_list, head); + hlist_bl_unlock(head); + spin_lock(&cache->c_list_lock); + list_add_tail(&entry->e_list, &cache->c_list); + cache->c_entry_count++; + spin_unlock(&cache->c_list_lock); + mb_cache_entry_put(cache, entry); + + return 0; +} +EXPORT_SYMBOL(mb_cache_entry_create); + +void __mb_cache_entry_free(struct mb_cache *cache, struct mb_cache_entry *entry) +{ + struct hlist_bl_head *head; + + head = mb_cache_entry_head(cache, entry->e_key); + hlist_bl_lock(head); + hlist_bl_del(&entry->e_hash_list); + hlist_bl_unlock(head); + kmem_cache_free(mb_entry_cache, entry); +} +EXPORT_SYMBOL(__mb_cache_entry_free); + +/* + * mb_cache_entry_wait_unused - wait to be the last user of the entry + * + * @entry - entry to work on + * + * Wait to be the last user of the entry. + */ +void mb_cache_entry_wait_unused(struct mb_cache_entry *entry) +{ + wait_var_event(&entry->e_refcnt, atomic_read(&entry->e_refcnt) <= 2); +} +EXPORT_SYMBOL(mb_cache_entry_wait_unused); + +static struct mb_cache_entry *__entry_find(struct mb_cache *cache, + struct mb_cache_entry *entry, + u32 key) +{ + struct mb_cache_entry *old_entry = entry; + struct hlist_bl_node *node; + struct hlist_bl_head *head; + + head = mb_cache_entry_head(cache, key); + hlist_bl_lock(head); + if (entry && !hlist_bl_unhashed(&entry->e_hash_list)) + node = entry->e_hash_list.next; + else + node = hlist_bl_first(head); + while (node) { + entry = hlist_bl_entry(node, struct mb_cache_entry, + e_hash_list); + if (entry->e_key == key && + test_bit(MBE_REUSABLE_B, &entry->e_flags) && + atomic_inc_not_zero(&entry->e_refcnt)) + goto out; + node = node->next; + } + entry = NULL; +out: + hlist_bl_unlock(head); + if (old_entry) + mb_cache_entry_put(cache, old_entry); + + return entry; +} + +/* + * mb_cache_entry_find_first - find the first reusable entry with the given key + * @cache: cache where we should search + * @key: key to look for + * + * Search in @cache for a reusable entry with key @key. Grabs reference to the + * first reusable entry found and returns the entry. + */ +struct mb_cache_entry *mb_cache_entry_find_first(struct mb_cache *cache, + u32 key) +{ + return __entry_find(cache, NULL, key); +} +EXPORT_SYMBOL(mb_cache_entry_find_first); + +/* + * mb_cache_entry_find_next - find next reusable entry with the same key + * @cache: cache where we should search + * @entry: entry to start search from + * + * Finds next reusable entry in the hash chain which has the same key as @entry. + * If @entry is unhashed (which can happen when deletion of entry races with the + * search), finds the first reusable entry in the hash chain. The function drops + * reference to @entry and returns with a reference to the found entry. + */ +struct mb_cache_entry *mb_cache_entry_find_next(struct mb_cache *cache, + struct mb_cache_entry *entry) +{ + return __entry_find(cache, entry, entry->e_key); +} +EXPORT_SYMBOL(mb_cache_entry_find_next); + +/* + * mb_cache_entry_get - get a cache entry by value (and key) + * @cache - cache we work with + * @key - key + * @value - value + */ +struct mb_cache_entry *mb_cache_entry_get(struct mb_cache *cache, u32 key, + u64 value) +{ + struct hlist_bl_node *node; + struct hlist_bl_head *head; + struct mb_cache_entry *entry; + + head = mb_cache_entry_head(cache, key); + hlist_bl_lock(head); + hlist_bl_for_each_entry(entry, node, head, e_hash_list) { + if (entry->e_key == key && entry->e_value == value && + atomic_inc_not_zero(&entry->e_refcnt)) + goto out; + } + entry = NULL; +out: + hlist_bl_unlock(head); + return entry; +} +EXPORT_SYMBOL(mb_cache_entry_get); + +/* mb_cache_entry_delete_or_get - remove a cache entry if it has no users + * @cache - cache we work with + * @key - key + * @value - value + * + * Remove entry from cache @cache with key @key and value @value. The removal + * happens only if the entry is unused. The function returns NULL in case the + * entry was successfully removed or there's no entry in cache. Otherwise the + * function grabs reference of the entry that we failed to delete because it + * still has users and return it. + */ +struct mb_cache_entry *mb_cache_entry_delete_or_get(struct mb_cache *cache, + u32 key, u64 value) +{ + struct mb_cache_entry *entry; + + entry = mb_cache_entry_get(cache, key, value); + if (!entry) + return NULL; + + /* + * Drop the ref we got from mb_cache_entry_get() and the initial hash + * ref if we are the last user + */ + if (atomic_cmpxchg(&entry->e_refcnt, 2, 0) != 2) + return entry; + + spin_lock(&cache->c_list_lock); + if (!list_empty(&entry->e_list)) + list_del_init(&entry->e_list); + cache->c_entry_count--; + spin_unlock(&cache->c_list_lock); + __mb_cache_entry_free(cache, entry); + return NULL; +} +EXPORT_SYMBOL(mb_cache_entry_delete_or_get); + +/* mb_cache_entry_touch - cache entry got used + * @cache - cache the entry belongs to + * @entry - entry that got used + * + * Marks entry as used to give hit higher chances of surviving in cache. + */ +void mb_cache_entry_touch(struct mb_cache *cache, + struct mb_cache_entry *entry) +{ + set_bit(MBE_REFERENCED_B, &entry->e_flags); +} +EXPORT_SYMBOL(mb_cache_entry_touch); + +static unsigned long mb_cache_count(struct shrinker *shrink, + struct shrink_control *sc) +{ + struct mb_cache *cache = shrink->private_data; + + return cache->c_entry_count; +} + +/* Shrink number of entries in cache */ +static unsigned long mb_cache_shrink(struct mb_cache *cache, + unsigned long nr_to_scan) +{ + struct mb_cache_entry *entry; + unsigned long shrunk = 0; + + spin_lock(&cache->c_list_lock); + while (nr_to_scan-- && !list_empty(&cache->c_list)) { + entry = list_first_entry(&cache->c_list, + struct mb_cache_entry, e_list); + /* Drop initial hash reference if there is no user */ + if (test_bit(MBE_REFERENCED_B, &entry->e_flags) || + atomic_cmpxchg(&entry->e_refcnt, 1, 0) != 1) { + clear_bit(MBE_REFERENCED_B, &entry->e_flags); + list_move_tail(&entry->e_list, &cache->c_list); + continue; + } + list_del_init(&entry->e_list); + cache->c_entry_count--; + spin_unlock(&cache->c_list_lock); + __mb_cache_entry_free(cache, entry); + shrunk++; + cond_resched(); + spin_lock(&cache->c_list_lock); + } + spin_unlock(&cache->c_list_lock); + + return shrunk; +} + +static unsigned long mb_cache_scan(struct shrinker *shrink, + struct shrink_control *sc) +{ + struct mb_cache *cache = shrink->private_data; + return mb_cache_shrink(cache, sc->nr_to_scan); +} + +/* We shrink 1/X of the cache when we have too many entries in it */ +#define SHRINK_DIVISOR 16 + +static void mb_cache_shrink_worker(struct work_struct *work) +{ + struct mb_cache *cache = container_of(work, struct mb_cache, + c_shrink_work); + mb_cache_shrink(cache, cache->c_max_entries / SHRINK_DIVISOR); +} + +/* + * mb_cache_create - create cache + * @bucket_bits: log2 of the hash table size + * + * Create cache for keys with 2^bucket_bits hash entries. + */ +struct mb_cache *mb_cache_create(int bucket_bits) +{ + struct mb_cache *cache; + unsigned long bucket_count = 1UL << bucket_bits; + unsigned long i; + + cache = kzalloc_obj(struct mb_cache); + if (!cache) + goto err_out; + cache->c_bucket_bits = bucket_bits; + cache->c_max_entries = bucket_count << 4; + INIT_LIST_HEAD(&cache->c_list); + spin_lock_init(&cache->c_list_lock); + cache->c_hash = kmalloc_objs(struct hlist_bl_head, bucket_count); + if (!cache->c_hash) { + kfree(cache); + goto err_out; + } + for (i = 0; i < bucket_count; i++) + INIT_HLIST_BL_HEAD(&cache->c_hash[i]); + + cache->c_shrink = shrinker_alloc(0, "mbcache-shrinker"); + if (!cache->c_shrink) { + kfree(cache->c_hash); + kfree(cache); + goto err_out; + } + + cache->c_shrink->count_objects = mb_cache_count; + cache->c_shrink->scan_objects = mb_cache_scan; + cache->c_shrink->private_data = cache; + + shrinker_register(cache->c_shrink); + + INIT_WORK(&cache->c_shrink_work, mb_cache_shrink_worker); + + return cache; + +err_out: + return NULL; +} +EXPORT_SYMBOL(mb_cache_create); + +/* + * mb_cache_destroy - destroy cache + * @cache: the cache to destroy + * + * Free all entries in cache and cache itself. Caller must make sure nobody + * (except shrinker) can reach @cache when calling this. + */ +void mb_cache_destroy(struct mb_cache *cache) +{ + struct mb_cache_entry *entry, *next; + + cancel_work_sync(&cache->c_shrink_work); + shrinker_free(cache->c_shrink); + + /* + * We don't bother with any locking. Cache must not be used at this + * point. + */ + list_for_each_entry_safe(entry, next, &cache->c_list, e_list) { + list_del(&entry->e_list); + WARN_ON(atomic_read(&entry->e_refcnt) != 1); + mb_cache_entry_put(cache, entry); + } + kfree(cache->c_hash); + kfree(cache); +} +EXPORT_SYMBOL(mb_cache_destroy); + +static int __init mbcache_init(void) +{ + mb_entry_cache = KMEM_CACHE(mb_cache_entry, SLAB_RECLAIM_ACCOUNT); + if (!mb_entry_cache) + return -ENOMEM; + return 0; +} + +static void __exit mbcache_exit(void) +{ + kmem_cache_destroy(mb_entry_cache); +} + +module_init(mbcache_init) +module_exit(mbcache_exit) + +MODULE_AUTHOR("Jan Kara "); +MODULE_DESCRIPTION("Meta block cache (for extended attributes)"); +MODULE_LICENSE("GPL"); diff --git a/fs/minix/Kconfig b/fs/minix/Kconfig new file mode 100644 index 000000000..90ddfad2a --- /dev/null +++ b/fs/minix/Kconfig @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: GPL-2.0-only +config MINIX_FS + tristate "Minix file system support" + depends on BLOCK + select BUFFER_HEAD + help + Minix is a simple operating system used in many classes about OS's. + The minix file system (method to organize files on a hard disk + partition or a floppy disk) was the original file system for Linux, + but has been superseded by the second extended file system ext2fs. + You don't want to use the minix file system on your hard disk + because of certain built-in restrictions, but it is sometimes found + on older Linux floppy disks. This option will enlarge your kernel + by about 28 KB. If unsure, say N. + + To compile this file system support as a module, choose M here: the + module will be called minix. Note that the file system of your root + partition (the one containing the directory /) cannot be compiled as + a module. + +config MINIX_FS_NATIVE_ENDIAN + def_bool MINIX_FS + depends on MICROBLAZE || MIPS || S390 || SUPERH || SPARC || XTENSA || (M68K && !MMU) + +config MINIX_FS_BIG_ENDIAN_16BIT_INDEXED + def_bool MINIX_FS + depends on M68K && MMU diff --git a/fs/minix/Makefile b/fs/minix/Makefile new file mode 100644 index 000000000..a2d3ab58d --- /dev/null +++ b/fs/minix/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the Linux minix filesystem routines. +# + +obj-$(CONFIG_MINIX_FS) += minix.o + +minix-objs := bitmap.o itree_v1.o itree_v2.o namei.o inode.o file.o dir.o diff --git a/fs/minix/bitmap.c b/fs/minix/bitmap.c new file mode 100644 index 000000000..abec43833 --- /dev/null +++ b/fs/minix/bitmap.c @@ -0,0 +1,269 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/minix/bitmap.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + */ + +/* + * Modified for 680x0 by Hamish Macdonald + * Fixed for 680x0 by Andreas Schwab + */ + +/* bitmap.c contains the code that handles the inode and block bitmaps */ + +#include "minix.h" +#include +#include +#include + +static DEFINE_SPINLOCK(bitmap_lock); + +/* + * bitmap consists of blocks filled with 16bit words + * bit set == busy, bit clear == free + * endianness is a mess, but for counting zero bits it really doesn't matter... + */ +static __u32 count_free(struct buffer_head *map[], unsigned blocksize, __u32 numbits) +{ + __u32 sum = 0; + unsigned blocks = DIV_ROUND_UP(numbits, blocksize * 8); + + while (blocks--) { + unsigned words = blocksize / 2; + __u16 *p = (__u16 *)(*map++)->b_data; + while (words--) + sum += 16 - hweight16(*p++); + } + + return sum; +} + +void minix_free_block(struct inode *inode, unsigned long block) +{ + struct super_block *sb = inode->i_sb; + struct minix_sb_info *sbi = minix_sb(sb); + struct buffer_head *bh; + int k = sb->s_blocksize_bits + 3; + unsigned long bit, zone; + + if (block < sbi->s_firstdatazone || block >= sbi->s_nzones) { + printk("Trying to free block not in datazone\n"); + return; + } + zone = block - sbi->s_firstdatazone + 1; + bit = zone & ((1<>= k; + if (zone >= sbi->s_zmap_blocks) { + printk("minix_free_block: nonexistent bitmap buffer\n"); + return; + } + bh = sbi->s_zmap[zone]; + spin_lock(&bitmap_lock); + if (!minix_test_and_clear_bit(bit, bh->b_data)) + printk("minix_free_block (%s:%lu): bit already cleared\n", + sb->s_id, block); + spin_unlock(&bitmap_lock); + mark_buffer_dirty(bh); + return; +} + +int minix_new_block(struct inode * inode) +{ + struct minix_sb_info *sbi = minix_sb(inode->i_sb); + int bits_per_zone = 8 * inode->i_sb->s_blocksize; + int i; + + for (i = 0; i < sbi->s_zmap_blocks; i++) { + struct buffer_head *bh = sbi->s_zmap[i]; + int j; + + spin_lock(&bitmap_lock); + j = minix_find_first_zero_bit(bh->b_data, bits_per_zone); + if (j < bits_per_zone) { + minix_set_bit(j, bh->b_data); + spin_unlock(&bitmap_lock); + mark_buffer_dirty(bh); + j += i * bits_per_zone + sbi->s_firstdatazone-1; + if (j < sbi->s_firstdatazone || j >= sbi->s_nzones) + break; + return j; + } + spin_unlock(&bitmap_lock); + } + return 0; +} + +unsigned long minix_count_free_blocks(struct super_block *sb) +{ + struct minix_sb_info *sbi = minix_sb(sb); + u32 bits = sbi->s_nzones - sbi->s_firstdatazone + 1; + + return (count_free(sbi->s_zmap, sb->s_blocksize, bits) + << sbi->s_log_zone_size); +} + +struct minix_inode * +minix_V1_raw_inode(struct super_block *sb, ino_t ino, struct buffer_head **bh) +{ + int block; + struct minix_sb_info *sbi = minix_sb(sb); + struct minix_inode *p; + + if (!ino || ino > sbi->s_ninodes) { + printk("Bad inode number on dev %s: %ld is out of range\n", + sb->s_id, (long)ino); + return NULL; + } + ino--; + block = 2 + sbi->s_imap_blocks + sbi->s_zmap_blocks + + ino / MINIX_INODES_PER_BLOCK; + *bh = sb_bread(sb, block); + if (!*bh) { + printk("Unable to read inode block\n"); + return NULL; + } + p = (void *)(*bh)->b_data; + return p + ino % MINIX_INODES_PER_BLOCK; +} + +struct minix2_inode * +minix_V2_raw_inode(struct super_block *sb, ino_t ino, struct buffer_head **bh) +{ + int block; + struct minix_sb_info *sbi = minix_sb(sb); + struct minix2_inode *p; + int minix2_inodes_per_block = sb->s_blocksize / sizeof(struct minix2_inode); + + *bh = NULL; + if (!ino || ino > sbi->s_ninodes) { + printk("Bad inode number on dev %s: %ld is out of range\n", + sb->s_id, (long)ino); + return NULL; + } + ino--; + block = 2 + sbi->s_imap_blocks + sbi->s_zmap_blocks + + ino / minix2_inodes_per_block; + *bh = sb_bread(sb, block); + if (!*bh) { + printk("Unable to read inode block\n"); + return NULL; + } + p = (void *)(*bh)->b_data; + return p + ino % minix2_inodes_per_block; +} + +/* Clear the link count and mode of a deleted inode on disk. */ + +static void minix_clear_inode(struct inode *inode) +{ + struct buffer_head *bh = NULL; + + if (INODE_VERSION(inode) == MINIX_V1) { + struct minix_inode *raw_inode; + raw_inode = minix_V1_raw_inode(inode->i_sb, inode->i_ino, &bh); + if (raw_inode) { + raw_inode->i_nlinks = 0; + raw_inode->i_mode = 0; + } + } else { + struct minix2_inode *raw_inode; + raw_inode = minix_V2_raw_inode(inode->i_sb, inode->i_ino, &bh); + if (raw_inode) { + raw_inode->i_nlinks = 0; + raw_inode->i_mode = 0; + } + } + if (bh) { + mark_buffer_dirty(bh); + brelse (bh); + } +} + +void minix_free_inode(struct inode * inode) +{ + struct super_block *sb = inode->i_sb; + struct minix_sb_info *sbi = minix_sb(inode->i_sb); + struct buffer_head *bh; + int k = sb->s_blocksize_bits + 3; + unsigned long ino, bit; + + ino = inode->i_ino; + if (ino < 1 || ino > sbi->s_ninodes) { + printk("minix_free_inode: inode 0 or nonexistent inode\n"); + return; + } + bit = ino & ((1<>= k; + if (ino >= sbi->s_imap_blocks) { + printk("minix_free_inode: nonexistent imap in superblock\n"); + return; + } + + minix_clear_inode(inode); /* clear on-disk copy */ + + bh = sbi->s_imap[ino]; + spin_lock(&bitmap_lock); + if (!minix_test_and_clear_bit(bit, bh->b_data)) + printk("minix_free_inode: bit %lu already cleared\n", bit); + spin_unlock(&bitmap_lock); + mark_buffer_dirty(bh); +} + +struct inode *minix_new_inode(const struct inode *dir, umode_t mode) +{ + struct super_block *sb = dir->i_sb; + struct minix_sb_info *sbi = minix_sb(sb); + struct inode *inode = new_inode(sb); + struct buffer_head * bh; + int bits_per_zone = 8 * sb->s_blocksize; + unsigned long j; + int i; + + if (!inode) + return ERR_PTR(-ENOMEM); + j = bits_per_zone; + bh = NULL; + spin_lock(&bitmap_lock); + for (i = 0; i < sbi->s_imap_blocks; i++) { + bh = sbi->s_imap[i]; + j = minix_find_first_zero_bit(bh->b_data, bits_per_zone); + if (j < bits_per_zone) + break; + } + if (!bh || j >= bits_per_zone) { + spin_unlock(&bitmap_lock); + iput(inode); + return ERR_PTR(-ENOSPC); + } + if (minix_test_and_set_bit(j, bh->b_data)) { /* shouldn't happen */ + spin_unlock(&bitmap_lock); + printk("minix_new_inode: bit already set\n"); + iput(inode); + return ERR_PTR(-ENOSPC); + } + spin_unlock(&bitmap_lock); + mark_buffer_dirty(bh); + j += i * bits_per_zone; + if (!j || j > sbi->s_ninodes) { + iput(inode); + return ERR_PTR(-EFSCORRUPTED); + } + inode_init_owner(&nop_mnt_idmap, inode, dir, mode); + inode->i_ino = j; + simple_inode_init_ts(inode); + inode->i_blocks = 0; + memset(&minix_i(inode)->u, 0, sizeof(minix_i(inode)->u)); + insert_inode_hash(inode); + mark_inode_dirty(inode); + + return inode; +} + +unsigned long minix_count_free_inodes(struct super_block *sb) +{ + struct minix_sb_info *sbi = minix_sb(sb); + u32 bits = sbi->s_ninodes + 1; + + return count_free(sbi->s_imap, sb->s_blocksize, bits); +} diff --git a/fs/minix/dir.c b/fs/minix/dir.c new file mode 100644 index 000000000..2ca16f849 --- /dev/null +++ b/fs/minix/dir.c @@ -0,0 +1,458 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/minix/dir.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * minix directory handling functions + * + * Updated to filesystem version 3 by Daniel Aragones + */ + +#include "minix.h" +#include +#include +#include + +typedef struct minix_dir_entry minix_dirent; +typedef struct minix3_dir_entry minix3_dirent; + +static int minix_readdir(struct file *, struct dir_context *); + +const struct file_operations minix_dir_operations = { + .llseek = generic_file_llseek, + .read = generic_read_dir, + .iterate_shared = minix_readdir, + .fsync = simple_fsync, +}; + +/* + * Return the offset into page `page_nr' of the last valid + * byte in that page, plus one. + */ +static unsigned +minix_last_byte(struct inode *inode, unsigned long page_nr) +{ + unsigned last_byte = PAGE_SIZE; + + if (page_nr == (inode->i_size >> PAGE_SHIFT)) + last_byte = inode->i_size & (PAGE_SIZE - 1); + return last_byte; +} + +static void dir_commit_chunk(struct folio *folio, loff_t pos, unsigned len) +{ + struct address_space *mapping = folio->mapping; + struct inode *dir = mapping->host; + + block_write_end(pos, len, len, folio); + + if (pos+len > dir->i_size) { + i_size_write(dir, pos+len); + mark_inode_dirty(dir); + } + folio_unlock(folio); +} + +static int minix_handle_dirsync(struct inode *dir) +{ + int err; + + err = filemap_write_and_wait(dir->i_mapping); + if (!err) + err = sync_inode_metadata(dir, 1); + return err; +} + +static void *dir_get_folio(struct inode *dir, unsigned long n, + struct folio **foliop) +{ + struct folio *folio = read_mapping_folio(dir->i_mapping, n, NULL); + + if (IS_ERR(folio)) + return ERR_CAST(folio); + *foliop = folio; + return kmap_local_folio(folio, 0); +} + +static inline void *minix_next_entry(void *de, struct minix_sb_info *sbi) +{ + return (void*)((char*)de + sbi->s_dirsize); +} + +static int minix_readdir(struct file *file, struct dir_context *ctx) +{ + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + struct minix_sb_info *sbi = minix_sb(sb); + unsigned chunk_size = sbi->s_dirsize; + unsigned long npages = dir_pages(inode); + unsigned long pos = ctx->pos; + unsigned offset; + unsigned long n; + + ctx->pos = pos = ALIGN(pos, chunk_size); + if (pos >= inode->i_size) + return 0; + + offset = pos & ~PAGE_MASK; + n = pos >> PAGE_SHIFT; + + for ( ; n < npages; n++, offset = 0) { + char *p, *kaddr, *limit; + struct folio *folio; + + kaddr = dir_get_folio(inode, n, &folio); + if (IS_ERR(kaddr)) + continue; + p = kaddr+offset; + limit = kaddr + minix_last_byte(inode, n) - chunk_size; + for ( ; p <= limit; p = minix_next_entry(p, sbi)) { + const char *name; + __u32 inumber; + if (sbi->s_version == MINIX_V3) { + minix3_dirent *de3 = (minix3_dirent *)p; + name = de3->name; + inumber = de3->inode; + } else { + minix_dirent *de = (minix_dirent *)p; + name = de->name; + inumber = de->inode; + } + if (inumber) { + unsigned l = strnlen(name, sbi->s_namelen); + if (!dir_emit(ctx, name, l, + inumber, DT_UNKNOWN)) { + folio_release_kmap(folio, p); + return 0; + } + } + ctx->pos += chunk_size; + } + folio_release_kmap(folio, kaddr); + } + return 0; +} + +static inline int namecompare(int len, int maxlen, + const char * name, const char * buffer) +{ + if (len < maxlen && buffer[len]) + return 0; + return !memcmp(name, buffer, len); +} + +/* + * minix_find_entry() + * + * finds an entry in the specified directory with the wanted name. + * It does NOT read the inode of the + * entry - you'll have to do that yourself if you want to. + * + * On Success folio_release_kmap() should be called on *foliop. + */ +minix_dirent *minix_find_entry(struct dentry *dentry, struct folio **foliop) +{ + const char * name = dentry->d_name.name; + int namelen = dentry->d_name.len; + struct inode * dir = d_inode(dentry->d_parent); + struct super_block * sb = dir->i_sb; + struct minix_sb_info * sbi = minix_sb(sb); + unsigned long n; + unsigned long npages = dir_pages(dir); + char *p; + + char *namx; + __u32 inumber; + + for (n = 0; n < npages; n++) { + char *kaddr, *limit; + + kaddr = dir_get_folio(dir, n, foliop); + if (IS_ERR(kaddr)) + continue; + + limit = kaddr + minix_last_byte(dir, n) - sbi->s_dirsize; + for (p = kaddr; p <= limit; p = minix_next_entry(p, sbi)) { + if (sbi->s_version == MINIX_V3) { + minix3_dirent *de3 = (minix3_dirent *)p; + namx = de3->name; + inumber = de3->inode; + } else { + minix_dirent *de = (minix_dirent *)p; + namx = de->name; + inumber = de->inode; + } + if (!inumber) + continue; + if (namecompare(namelen, sbi->s_namelen, name, namx)) + goto found; + } + folio_release_kmap(*foliop, kaddr); + } + return NULL; + +found: + return (minix_dirent *)p; +} + +int minix_add_link(struct dentry *dentry, struct inode *inode) +{ + struct inode *dir = d_inode(dentry->d_parent); + const char * name = dentry->d_name.name; + int namelen = dentry->d_name.len; + struct super_block * sb = dir->i_sb; + struct minix_sb_info * sbi = minix_sb(sb); + struct folio *folio = NULL; + unsigned long npages = dir_pages(dir); + unsigned long n; + char *kaddr, *p; + minix_dirent *de; + minix3_dirent *de3; + loff_t pos; + int err; + char *namx = NULL; + __u32 inumber; + + /* + * We take care of directory expansion in the same loop + * This code plays outside i_size, so it locks the page + * to protect that region. + */ + for (n = 0; n <= npages; n++) { + char *limit, *dir_end; + + kaddr = dir_get_folio(dir, n, &folio); + if (IS_ERR(kaddr)) + return PTR_ERR(kaddr); + folio_lock(folio); + dir_end = kaddr + minix_last_byte(dir, n); + limit = kaddr + PAGE_SIZE - sbi->s_dirsize; + for (p = kaddr; p <= limit; p = minix_next_entry(p, sbi)) { + de = (minix_dirent *)p; + de3 = (minix3_dirent *)p; + if (sbi->s_version == MINIX_V3) { + namx = de3->name; + inumber = de3->inode; + } else { + namx = de->name; + inumber = de->inode; + } + if (p == dir_end) { + /* We hit i_size */ + if (sbi->s_version == MINIX_V3) + de3->inode = 0; + else + de->inode = 0; + goto got_it; + } + if (!inumber) + goto got_it; + err = -EEXIST; + if (namecompare(namelen, sbi->s_namelen, name, namx)) + goto out_unlock; + } + folio_unlock(folio); + folio_release_kmap(folio, kaddr); + } + BUG(); + return -EINVAL; + +got_it: + pos = folio_pos(folio) + offset_in_folio(folio, p); + err = minix_prepare_chunk(folio, pos, sbi->s_dirsize); + if (err) + goto out_unlock; + memcpy (namx, name, namelen); + if (sbi->s_version == MINIX_V3) { + memset (namx + namelen, 0, sbi->s_dirsize - namelen - 4); + de3->inode = inode->i_ino; + } else { + memset (namx + namelen, 0, sbi->s_dirsize - namelen - 2); + de->inode = inode->i_ino; + } + dir_commit_chunk(folio, pos, sbi->s_dirsize); + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + mark_inode_dirty(dir); + err = minix_handle_dirsync(dir); +out_put: + folio_release_kmap(folio, kaddr); + return err; +out_unlock: + folio_unlock(folio); + goto out_put; +} + +int minix_delete_entry(struct minix_dir_entry *de, struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + loff_t pos = folio_pos(folio) + offset_in_folio(folio, de); + struct minix_sb_info *sbi = minix_sb(inode->i_sb); + unsigned len = sbi->s_dirsize; + int err; + + folio_lock(folio); + err = minix_prepare_chunk(folio, pos, len); + if (err) { + folio_unlock(folio); + return err; + } + if (sbi->s_version == MINIX_V3) + ((minix3_dirent *)de)->inode = 0; + else + de->inode = 0; + dir_commit_chunk(folio, pos, len); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + mark_inode_dirty(inode); + return minix_handle_dirsync(inode); +} + +int minix_make_empty(struct inode *inode, struct inode *dir) +{ + struct folio *folio = filemap_grab_folio(inode->i_mapping, 0); + struct minix_sb_info *sbi = minix_sb(inode->i_sb); + char *kaddr; + int err; + + if (IS_ERR(folio)) + return PTR_ERR(folio); + err = minix_prepare_chunk(folio, 0, 2 * sbi->s_dirsize); + if (err) { + folio_unlock(folio); + goto fail; + } + + kaddr = kmap_local_folio(folio, 0); + memset(kaddr, 0, folio_size(folio)); + + if (sbi->s_version == MINIX_V3) { + minix3_dirent *de3 = (minix3_dirent *)kaddr; + + de3->inode = inode->i_ino; + strcpy(de3->name, "."); + de3 = minix_next_entry(de3, sbi); + de3->inode = dir->i_ino; + strcpy(de3->name, ".."); + } else { + minix_dirent *de = (minix_dirent *)kaddr; + + de->inode = inode->i_ino; + strcpy(de->name, "."); + de = minix_next_entry(de, sbi); + de->inode = dir->i_ino; + strcpy(de->name, ".."); + } + kunmap_local(kaddr); + + dir_commit_chunk(folio, 0, 2 * sbi->s_dirsize); + err = minix_handle_dirsync(inode); +fail: + folio_put(folio); + return err; +} + +/* + * routine to check that the specified directory is empty (for rmdir) + */ +int minix_empty_dir(struct inode * inode) +{ + struct folio *folio = NULL; + unsigned long i, npages = dir_pages(inode); + struct minix_sb_info *sbi = minix_sb(inode->i_sb); + char *name, *kaddr; + __u32 inumber; + + for (i = 0; i < npages; i++) { + char *p, *limit; + + kaddr = dir_get_folio(inode, i, &folio); + if (IS_ERR(kaddr)) + continue; + + limit = kaddr + minix_last_byte(inode, i) - sbi->s_dirsize; + for (p = kaddr; p <= limit; p = minix_next_entry(p, sbi)) { + if (sbi->s_version == MINIX_V3) { + minix3_dirent *de3 = (minix3_dirent *)p; + name = de3->name; + inumber = de3->inode; + } else { + minix_dirent *de = (minix_dirent *)p; + name = de->name; + inumber = de->inode; + } + + if (inumber != 0) { + /* check for . and .. */ + if (name[0] != '.') + goto not_empty; + if (!name[1]) { + if (inumber != inode->i_ino) + goto not_empty; + } else if (name[1] != '.') + goto not_empty; + else if (name[2]) + goto not_empty; + } + } + folio_release_kmap(folio, kaddr); + } + return 1; + +not_empty: + folio_release_kmap(folio, kaddr); + return 0; +} + +/* Releases the page */ +int minix_set_link(struct minix_dir_entry *de, struct folio *folio, + struct inode *inode) +{ + struct inode *dir = folio->mapping->host; + struct minix_sb_info *sbi = minix_sb(dir->i_sb); + loff_t pos = folio_pos(folio) + offset_in_folio(folio, de); + int err; + + folio_lock(folio); + err = minix_prepare_chunk(folio, pos, sbi->s_dirsize); + if (err) { + folio_unlock(folio); + return err; + } + if (sbi->s_version == MINIX_V3) + ((minix3_dirent *)de)->inode = inode->i_ino; + else + de->inode = inode->i_ino; + dir_commit_chunk(folio, pos, sbi->s_dirsize); + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + mark_inode_dirty(dir); + return minix_handle_dirsync(dir); +} + +struct minix_dir_entry *minix_dotdot(struct inode *dir, struct folio **foliop) +{ + struct minix_sb_info *sbi = minix_sb(dir->i_sb); + struct minix_dir_entry *de = dir_get_folio(dir, 0, foliop); + + if (!IS_ERR(de)) + return minix_next_entry(de, sbi); + return NULL; +} + +ino_t minix_inode_by_name(struct dentry *dentry) +{ + struct folio *folio; + struct minix_dir_entry *de = minix_find_entry(dentry, &folio); + ino_t res = 0; + + if (de) { + struct inode *inode = folio->mapping->host; + struct minix_sb_info *sbi = minix_sb(inode->i_sb); + + if (sbi->s_version == MINIX_V3) + res = ((minix3_dirent *) de)->inode; + else + res = de->inode; + folio_release_kmap(folio, de); + } + return res; +} diff --git a/fs/minix/file.c b/fs/minix/file.c new file mode 100644 index 000000000..02aabbdb5 --- /dev/null +++ b/fs/minix/file.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/minix/file.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * minix regular file handling primitives + */ + +#include +#include "minix.h" + +/* + * We have mostly NULLs here: the current defaults are OK for + * the minix filesystem. + */ +const struct file_operations minix_file_operations = { + .llseek = generic_file_llseek, + .read_iter = generic_file_read_iter, + .write_iter = generic_file_write_iter, + .mmap_prepare = generic_file_mmap_prepare, + .fsync = simple_fsync, + .splice_read = filemap_splice_read, +}; + +static int minix_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, struct iattr *attr) +{ + struct inode *inode = d_inode(dentry); + int error; + + error = setattr_prepare(&nop_mnt_idmap, dentry, attr); + if (error) + return error; + + if ((attr->ia_valid & ATTR_SIZE) && + attr->ia_size != i_size_read(inode)) { + error = inode_newsize_ok(inode, attr->ia_size); + if (error) + return error; + + truncate_setsize(inode, attr->ia_size); + minix_truncate(inode); + } + + setattr_copy(&nop_mnt_idmap, inode, attr); + mark_inode_dirty(inode); + return 0; +} + +const struct inode_operations minix_file_inode_operations = { + .setattr = minix_setattr, + .getattr = minix_getattr, +}; diff --git a/fs/minix/inode.c b/fs/minix/inode.c new file mode 100644 index 000000000..daf83e4ff --- /dev/null +++ b/fs/minix/inode.c @@ -0,0 +1,789 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/minix/inode.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * Copyright (C) 1996 Gertjan van Wingerde + * Minix V2 fs support. + * + * Modified for 680x0 by Andreas Schwab + * Updated to filesystem version 3 by Daniel Aragones + */ + +#include +#include "minix.h" +#include +#include +#include +#include +#include +#include +#include +#include + +static int minix_write_inode(struct inode *inode, + struct writeback_control *wbc); +static int minix_sync_inode_metadata(struct inode *inode, + struct writeback_control *wbc); +static int minix_statfs(struct dentry *dentry, struct kstatfs *buf); + +void __minix_error_inode(struct inode *inode, const char *function, + unsigned int line, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + printk(KERN_CRIT "minix-fs error (device %s): %s:%d: " + "inode #%llu: comm %s: %pV\n", + inode->i_sb->s_id, function, line, inode->i_ino, + current->comm, &vaf); + va_end(args); +} + +static void minix_evict_inode(struct inode *inode) +{ + truncate_inode_pages_final(&inode->i_data); + if (!inode->i_nlink) { + inode->i_size = 0; + minix_truncate(inode); + } else { + mmb_sync(&minix_i(inode)->i_metadata_bhs); + } + mmb_invalidate(&minix_i(inode)->i_metadata_bhs); + clear_inode(inode); + if (!inode->i_nlink) + minix_free_inode(inode); +} + +static void minix_put_super(struct super_block *sb) +{ + int i; + struct minix_sb_info *sbi = minix_sb(sb); + + if (!sb_rdonly(sb)) { + if (sbi->s_version != MINIX_V3) /* s_state is now out from V3 sb */ + sbi->s_ms->s_state = sbi->s_mount_state; + mark_buffer_dirty(sbi->s_sbh); + } + for (i = 0; i < sbi->s_imap_blocks; i++) + brelse(sbi->s_imap[i]); + for (i = 0; i < sbi->s_zmap_blocks; i++) + brelse(sbi->s_zmap[i]); + brelse (sbi->s_sbh); + kfree(sbi->s_imap); + sb->s_fs_info = NULL; + kfree(sbi); +} + +static struct kmem_cache * minix_inode_cachep; + +static struct inode *minix_alloc_inode(struct super_block *sb) +{ + struct minix_inode_info *ei; + ei = alloc_inode_sb(sb, minix_inode_cachep, GFP_KERNEL); + if (!ei) + return NULL; + mmb_init(&ei->i_metadata_bhs, &ei->vfs_inode.i_data); + + return &ei->vfs_inode; +} + +static void minix_free_in_core_inode(struct inode *inode) +{ + kmem_cache_free(minix_inode_cachep, minix_i(inode)); +} + +static void init_once(void *foo) +{ + struct minix_inode_info *ei = (struct minix_inode_info *) foo; + + inode_init_once(&ei->vfs_inode); +} + +static int __init init_inodecache(void) +{ + minix_inode_cachep = kmem_cache_create("minix_inode_cache", + sizeof(struct minix_inode_info), + 0, (SLAB_RECLAIM_ACCOUNT| + SLAB_ACCOUNT), + init_once); + if (minix_inode_cachep == NULL) + return -ENOMEM; + return 0; +} + +static void destroy_inodecache(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(minix_inode_cachep); +} + +static const struct super_operations minix_sops = { + .alloc_inode = minix_alloc_inode, + .free_inode = minix_free_in_core_inode, + .write_inode = minix_write_inode, + .sync_inode_metadata = minix_sync_inode_metadata, + .evict_inode = minix_evict_inode, + .put_super = minix_put_super, + .statfs = minix_statfs, +}; + +static int minix_reconfigure(struct fs_context *fc) +{ + struct minix_super_block * ms; + struct super_block *sb = fc->root->d_sb; + struct minix_sb_info * sbi = sb->s_fs_info; + + sync_filesystem(sb); + ms = sbi->s_ms; + if ((bool)(fc->sb_flags & SB_RDONLY) == sb_rdonly(sb)) + return 0; + if (fc->sb_flags & SB_RDONLY) { + if (ms->s_state & MINIX_VALID_FS || + !(sbi->s_mount_state & MINIX_VALID_FS)) + return 0; + /* Mounting a rw partition read-only. */ + if (sbi->s_version != MINIX_V3) + ms->s_state = sbi->s_mount_state; + mark_buffer_dirty(sbi->s_sbh); + } else { + /* Mount a partition which is read-only, read-write. */ + if (sbi->s_version != MINIX_V3) { + sbi->s_mount_state = ms->s_state; + ms->s_state &= ~MINIX_VALID_FS; + } else { + sbi->s_mount_state = MINIX_VALID_FS; + } + mark_buffer_dirty(sbi->s_sbh); + + if (!(sbi->s_mount_state & MINIX_VALID_FS)) + printk("MINIX-fs warning: remounting unchecked fs, " + "running fsck is recommended\n"); + else if ((sbi->s_mount_state & MINIX_ERROR_FS)) + printk("MINIX-fs warning: remounting fs with errors, " + "running fsck is recommended\n"); + } + return 0; +} + +static bool minix_check_superblock(struct super_block *sb) +{ + struct minix_sb_info *sbi = minix_sb(sb); + unsigned long block; + + if (sbi->s_log_zone_size != 0) { + printk("minix-fs error: zone size must equal block size. " + "s_log_zone_size > 0 is not supported.\n"); + return false; + } + + if (sbi->s_ninodes < 1 || sbi->s_firstdatazone <= 4 || + sbi->s_firstdatazone >= sbi->s_nzones) + return false; + + /* Apparently minix can create filesystems that allocate more blocks for + * the bitmaps than needed. We simply ignore that, but verify it didn't + * create one with not enough blocks and bail out if so. + */ + block = minix_blocks_needed(sbi->s_ninodes, sb->s_blocksize); + if (sbi->s_imap_blocks < block) { + printk("MINIX-fs: file system does not have enough " + "imap blocks allocated. Refusing to mount.\n"); + return false; + } + + block = minix_blocks_needed( + (sbi->s_nzones - sbi->s_firstdatazone + 1), + sb->s_blocksize); + if (sbi->s_zmap_blocks < block) { + printk("MINIX-fs: file system does not have enough " + "zmap blocks allocated. Refusing to mount.\n"); + return false; + } + + /* + * s_max_size must not exceed the block mapping limitation. This check + * is only needed for V1 filesystems, since V2/V3 support an extra level + * of indirect blocks which places the limit well above U32_MAX. + */ + if (sbi->s_version == MINIX_V1 && + sb->s_maxbytes > (7 + 512 + 512*512) * BLOCK_SIZE) + return false; + + return true; +} + +static int minix_fill_super(struct super_block *s, struct fs_context *fc) +{ + struct buffer_head *bh; + struct buffer_head **map; + struct minix_super_block *ms; + struct minix3_super_block *m3s = NULL; + unsigned long i, block; + struct inode *root_inode; + struct minix_sb_info *sbi; + int ret = -EINVAL; + int silent = fc->sb_flags & SB_SILENT; + + sbi = kzalloc_obj(struct minix_sb_info); + if (!sbi) + return -ENOMEM; + s->s_fs_info = sbi; + + BUILD_BUG_ON(32 != sizeof (struct minix_inode)); + BUILD_BUG_ON(64 != sizeof(struct minix2_inode)); + + if (!sb_set_blocksize(s, BLOCK_SIZE)) + goto out_bad_hblock; + + if (!(bh = sb_bread(s, 1))) + goto out_bad_sb; + + ms = (struct minix_super_block *) bh->b_data; + sbi->s_ms = ms; + sbi->s_sbh = bh; + sbi->s_mount_state = ms->s_state; + sbi->s_ninodes = ms->s_ninodes; + sbi->s_nzones = ms->s_nzones; + sbi->s_imap_blocks = ms->s_imap_blocks; + sbi->s_zmap_blocks = ms->s_zmap_blocks; + sbi->s_firstdatazone = ms->s_firstdatazone; + sbi->s_log_zone_size = ms->s_log_zone_size; + s->s_maxbytes = ms->s_max_size; + s->s_magic = ms->s_magic; + if (s->s_magic == MINIX_SUPER_MAGIC) { + sbi->s_version = MINIX_V1; + sbi->s_dirsize = 16; + sbi->s_namelen = 14; + s->s_max_links = MINIX_LINK_MAX; + } else if (s->s_magic == MINIX_SUPER_MAGIC2) { + sbi->s_version = MINIX_V1; + sbi->s_dirsize = 32; + sbi->s_namelen = 30; + s->s_max_links = MINIX_LINK_MAX; + } else if (s->s_magic == MINIX2_SUPER_MAGIC) { + sbi->s_version = MINIX_V2; + sbi->s_nzones = ms->s_zones; + sbi->s_dirsize = 16; + sbi->s_namelen = 14; + s->s_max_links = MINIX2_LINK_MAX; + } else if (s->s_magic == MINIX2_SUPER_MAGIC2) { + sbi->s_version = MINIX_V2; + sbi->s_nzones = ms->s_zones; + sbi->s_dirsize = 32; + sbi->s_namelen = 30; + s->s_max_links = MINIX2_LINK_MAX; + } else if ( *(__u16 *)(bh->b_data + 24) == MINIX3_SUPER_MAGIC) { + m3s = (struct minix3_super_block *) bh->b_data; + s->s_magic = m3s->s_magic; + sbi->s_imap_blocks = m3s->s_imap_blocks; + sbi->s_zmap_blocks = m3s->s_zmap_blocks; + sbi->s_firstdatazone = m3s->s_firstdatazone; + sbi->s_log_zone_size = m3s->s_log_zone_size; + s->s_maxbytes = m3s->s_max_size; + sbi->s_ninodes = m3s->s_ninodes; + sbi->s_nzones = m3s->s_zones; + sbi->s_dirsize = 64; + sbi->s_namelen = 60; + sbi->s_version = MINIX_V3; + sbi->s_mount_state = MINIX_VALID_FS; + if (!sb_set_blocksize(s, m3s->s_blocksize)) + goto out_release; + s->s_max_links = MINIX2_LINK_MAX; + } else + goto out_no_fs; + + if (!minix_check_superblock(s)) + goto out_illegal_sb; + + /* + * Allocate the buffer map to keep the superblock small. + */ + i = (sbi->s_imap_blocks + sbi->s_zmap_blocks) * sizeof(bh); + map = kzalloc(i, GFP_KERNEL); + if (!map) + goto out_no_map; + sbi->s_imap = &map[0]; + sbi->s_zmap = &map[sbi->s_imap_blocks]; + + block=2; + for (i=0 ; i < sbi->s_imap_blocks ; i++) { + if (!(sbi->s_imap[i]=sb_bread(s, block))) + goto out_no_bitmap; + block++; + } + for (i=0 ; i < sbi->s_zmap_blocks ; i++) { + if (!(sbi->s_zmap[i]=sb_bread(s, block))) + goto out_no_bitmap; + block++; + } + + minix_set_bit(0,sbi->s_imap[0]->b_data); + minix_set_bit(0,sbi->s_zmap[0]->b_data); + + /* set up enough so that it can read an inode */ + s->s_op = &minix_sops; + s->s_time_min = 0; + s->s_time_max = U32_MAX; + root_inode = minix_iget(s, MINIX_ROOT_INO); + if (IS_ERR(root_inode)) { + ret = PTR_ERR(root_inode); + goto out_no_root; + } + + ret = -ENOMEM; + s->s_root = d_make_root(root_inode); + if (!s->s_root) + goto out_no_root; + + if (!sb_rdonly(s)) { + if (sbi->s_version != MINIX_V3) /* s_state is now out from V3 sb */ + ms->s_state &= ~MINIX_VALID_FS; + mark_buffer_dirty(bh); + } + if (!(sbi->s_mount_state & MINIX_VALID_FS)) + printk("MINIX-fs: mounting unchecked file system, " + "running fsck is recommended\n"); + else if (sbi->s_mount_state & MINIX_ERROR_FS) + printk("MINIX-fs: mounting file system with errors, " + "running fsck is recommended\n"); + + return 0; + +out_no_root: + if (!silent) + printk("MINIX-fs: get root inode failed\n"); + goto out_freemap; + +out_no_bitmap: + printk("MINIX-fs: bad superblock or unable to read bitmaps\n"); +out_freemap: + for (i = 0; i < sbi->s_imap_blocks; i++) + brelse(sbi->s_imap[i]); + for (i = 0; i < sbi->s_zmap_blocks; i++) + brelse(sbi->s_zmap[i]); + kfree(sbi->s_imap); + goto out_release; + +out_no_map: + ret = -ENOMEM; + if (!silent) + printk("MINIX-fs: can't allocate map\n"); + goto out_release; + +out_illegal_sb: + if (!silent) + printk("MINIX-fs: bad superblock\n"); + goto out_release; + +out_no_fs: + if (!silent) + printk("VFS: Can't find a Minix filesystem V1 | V2 | V3 " + "on device %s.\n", s->s_id); +out_release: + brelse(bh); + goto out; + +out_bad_hblock: + printk("MINIX-fs: blocksize too small for device\n"); + goto out; + +out_bad_sb: + printk("MINIX-fs: unable to read superblock\n"); +out: + s->s_fs_info = NULL; + kfree(sbi); + return ret; +} + +static int minix_get_tree(struct fs_context *fc) +{ + return get_tree_bdev(fc, minix_fill_super); +} + +static const struct fs_context_operations minix_context_ops = { + .get_tree = minix_get_tree, + .reconfigure = minix_reconfigure, +}; + +static int minix_init_fs_context(struct fs_context *fc) +{ + fc->ops = &minix_context_ops; + + return 0; +} + +static int minix_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + struct minix_sb_info *sbi = minix_sb(sb); + u64 id = huge_encode_dev(sb->s_bdev->bd_dev); + buf->f_type = sb->s_magic; + buf->f_bsize = sb->s_blocksize; + buf->f_blocks = (sbi->s_nzones - sbi->s_firstdatazone) << sbi->s_log_zone_size; + buf->f_bfree = minix_count_free_blocks(sb); + buf->f_bavail = buf->f_bfree; + buf->f_files = sbi->s_ninodes; + buf->f_ffree = minix_count_free_inodes(sb); + buf->f_namelen = sbi->s_namelen; + buf->f_fsid = u64_to_fsid(id); + + return 0; +} + +static int minix_get_block(struct inode *inode, sector_t block, + struct buffer_head *bh_result, int create) +{ + if (INODE_VERSION(inode) == MINIX_V1) + return V1_minix_get_block(inode, block, bh_result, create); + else + return V2_minix_get_block(inode, block, bh_result, create); +} + +static int minix_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + return mpage_writepages(mapping, wbc, minix_get_block); +} + +static int minix_read_folio(struct file *file, struct folio *folio) +{ + return block_read_full_folio(folio, minix_get_block); +} + +int minix_prepare_chunk(struct folio *folio, loff_t pos, unsigned len) +{ + return __block_write_begin(folio, pos, len, minix_get_block); +} + +static void minix_write_failed(struct address_space *mapping, loff_t to) +{ + struct inode *inode = mapping->host; + + if (to > inode->i_size) { + truncate_pagecache(inode, inode->i_size); + minix_truncate(inode); + } +} + +static int minix_write_begin(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, + struct folio **foliop, void **fsdata) +{ + int ret; + + ret = block_write_begin(mapping, pos, len, foliop, minix_get_block); + if (unlikely(ret)) + minix_write_failed(mapping, pos + len); + + return ret; +} + +static sector_t minix_bmap(struct address_space *mapping, sector_t block) +{ + return generic_block_bmap(mapping,block,minix_get_block); +} + +static const struct address_space_operations minix_aops = { + .dirty_folio = block_dirty_folio, + .invalidate_folio = block_invalidate_folio, + .read_folio = minix_read_folio, + .writepages = minix_writepages, + .write_begin = minix_write_begin, + .write_end = generic_write_end, + .migrate_folio = buffer_migrate_folio, + .bmap = minix_bmap, + .direct_IO = noop_direct_IO +}; + +static const struct inode_operations minix_symlink_inode_operations = { + .get_link = page_get_link, + .getattr = minix_getattr, +}; + +void minix_set_inode(struct inode *inode, dev_t rdev) +{ + if (S_ISREG(inode->i_mode)) { + inode->i_op = &minix_file_inode_operations; + inode->i_fop = &minix_file_operations; + inode->i_mapping->a_ops = &minix_aops; + } else if (S_ISDIR(inode->i_mode)) { + inode->i_op = &minix_dir_inode_operations; + inode->i_fop = &minix_dir_operations; + inode->i_mapping->a_ops = &minix_aops; + } else if (S_ISLNK(inode->i_mode)) { + inode->i_op = &minix_symlink_inode_operations; + inode_nohighmem(inode); + inode->i_mapping->a_ops = &minix_aops; + } else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) || + S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) { + init_special_inode(inode, inode->i_mode, rdev); + } else { + printk(KERN_DEBUG "MINIX-fs: Invalid file type 0%04o for inode %llu.\n", + inode->i_mode, inode->i_ino); + make_bad_inode(inode); + } +} + +/* + * The minix V1 function to read an inode. + */ +static struct inode *V1_minix_iget(struct inode *inode) +{ + struct buffer_head * bh; + struct minix_inode * raw_inode; + struct minix_inode_info *minix_inode = minix_i(inode); + int i; + + raw_inode = minix_V1_raw_inode(inode->i_sb, inode->i_ino, &bh); + if (!raw_inode) { + iget_failed(inode); + return ERR_PTR(-EIO); + } + if (raw_inode->i_nlinks == 0) { + printk("MINIX-fs: deleted inode referenced: %llu\n", + inode->i_ino); + brelse(bh); + iget_failed(inode); + return ERR_PTR(-ESTALE); + } + inode->i_mode = raw_inode->i_mode; + i_uid_write(inode, raw_inode->i_uid); + i_gid_write(inode, raw_inode->i_gid); + set_nlink(inode, raw_inode->i_nlinks); + inode->i_size = raw_inode->i_size; + inode_set_mtime_to_ts(inode, + inode_set_atime_to_ts(inode, inode_set_ctime(inode, raw_inode->i_time, 0))); + inode->i_blocks = 0; + for (i = 0; i < 9; i++) + minix_inode->u.i1_data[i] = raw_inode->i_zone[i]; + minix_set_inode(inode, old_decode_dev(raw_inode->i_zone[0])); + brelse(bh); + unlock_new_inode(inode); + return inode; +} + +/* + * The minix V2 function to read an inode. + */ +static struct inode *V2_minix_iget(struct inode *inode) +{ + struct buffer_head * bh; + struct minix2_inode * raw_inode; + struct minix_inode_info *minix_inode = minix_i(inode); + int i; + + raw_inode = minix_V2_raw_inode(inode->i_sb, inode->i_ino, &bh); + if (!raw_inode) { + iget_failed(inode); + return ERR_PTR(-EIO); + } + if (raw_inode->i_nlinks == 0) { + printk("MINIX-fs: deleted inode referenced: %llu\n", + inode->i_ino); + brelse(bh); + iget_failed(inode); + return ERR_PTR(-ESTALE); + } + inode->i_mode = raw_inode->i_mode; + i_uid_write(inode, raw_inode->i_uid); + i_gid_write(inode, raw_inode->i_gid); + set_nlink(inode, raw_inode->i_nlinks); + inode->i_size = raw_inode->i_size; + inode_set_mtime(inode, raw_inode->i_mtime, 0); + inode_set_atime(inode, raw_inode->i_atime, 0); + inode_set_ctime(inode, raw_inode->i_ctime, 0); + inode->i_blocks = 0; + for (i = 0; i < 10; i++) + minix_inode->u.i2_data[i] = raw_inode->i_zone[i]; + minix_set_inode(inode, old_decode_dev(raw_inode->i_zone[0])); + brelse(bh); + unlock_new_inode(inode); + return inode; +} + +/* + * The global function to read an inode. + */ +struct inode *minix_iget(struct super_block *sb, unsigned long ino) +{ + struct inode *inode; + + inode = iget_locked(sb, ino); + if (!inode) + return ERR_PTR(-ENOMEM); + if (!(inode_state_read_once(inode) & I_NEW)) + return inode; + + if (INODE_VERSION(inode) == MINIX_V1) + return V1_minix_iget(inode); + else + return V2_minix_iget(inode); +} + +/* + * The minix V1 function to synchronize an inode. + */ +static int V1_minix_update_inode(struct inode * inode) +{ + struct buffer_head * bh; + struct minix_inode * raw_inode; + struct minix_inode_info *minix_inode = minix_i(inode); + int i; + + raw_inode = minix_V1_raw_inode(inode->i_sb, inode->i_ino, &bh); + if (!raw_inode) + return -EIO; + raw_inode->i_mode = inode->i_mode; + raw_inode->i_uid = fs_high2lowuid(i_uid_read(inode)); + raw_inode->i_gid = fs_high2lowgid(i_gid_read(inode)); + raw_inode->i_nlinks = inode->i_nlink; + raw_inode->i_size = inode->i_size; + raw_inode->i_time = inode_get_mtime_sec(inode); + if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) + raw_inode->i_zone[0] = old_encode_dev(inode->i_rdev); + else for (i = 0; i < 9; i++) + raw_inode->i_zone[i] = minix_inode->u.i1_data[i]; + mark_buffer_dirty(bh); + brelse(bh); + set_inode_metadata_writeback(inode); + return 0; +} + +/* + * The minix V2 function to synchronize an inode. + */ +static int V2_minix_update_inode(struct inode * inode) +{ + struct buffer_head * bh; + struct minix2_inode * raw_inode; + struct minix_inode_info *minix_inode = minix_i(inode); + int i; + + raw_inode = minix_V2_raw_inode(inode->i_sb, inode->i_ino, &bh); + if (!raw_inode) + return -EIO; + raw_inode->i_mode = inode->i_mode; + raw_inode->i_uid = fs_high2lowuid(i_uid_read(inode)); + raw_inode->i_gid = fs_high2lowgid(i_gid_read(inode)); + raw_inode->i_nlinks = inode->i_nlink; + raw_inode->i_size = inode->i_size; + raw_inode->i_mtime = inode_get_mtime_sec(inode); + raw_inode->i_atime = inode_get_atime_sec(inode); + raw_inode->i_ctime = inode_get_ctime_sec(inode); + if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) + raw_inode->i_zone[0] = old_encode_dev(inode->i_rdev); + else for (i = 0; i < 10; i++) + raw_inode->i_zone[i] = minix_inode->u.i2_data[i]; + mark_buffer_dirty(bh); + brelse(bh); + set_inode_metadata_writeback(inode); + return 0; +} + +static int minix_write_inode(struct inode *inode, struct writeback_control *wbc) +{ + if (INODE_VERSION(inode) == MINIX_V1) + return V1_minix_update_inode(inode); + return V2_minix_update_inode(inode); +} + +static int minix_sync_inode_metadata(struct inode *inode, + struct writeback_control *wbc) +{ + int err = 0; + struct buffer_head *bh; + void *raw_inode; + + if (INODE_VERSION(inode) == MINIX_V1) + raw_inode = minix_V1_raw_inode(inode->i_sb, inode->i_ino, &bh); + else + raw_inode = minix_V2_raw_inode(inode->i_sb, inode->i_ino, &bh); + if (!raw_inode) + return -EIO; + err = mmb_sync(&minix_i(inode)->i_metadata_bhs); + if (err) + goto out; + sync_dirty_buffer(bh); + if (buffer_write_io_error(bh)) { + printk("IO error syncing minix inode [%s:%08llx]\n", + inode->i_sb->s_id, inode->i_ino); + err = -EIO; + } +out: + brelse(bh); + return err; +} + +int minix_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags) +{ + struct super_block *sb = path->dentry->d_sb; + struct inode *inode = d_inode(path->dentry); + + generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat); + if (INODE_VERSION(inode) == MINIX_V1) + stat->blocks = (BLOCK_SIZE / 512) * V1_minix_blocks(stat->size, sb); + else + stat->blocks = (sb->s_blocksize / 512) * V2_minix_blocks(stat->size, sb); + stat->blksize = sb->s_blocksize; + return 0; +} + +/* + * The function that is called for file truncation. + */ +void minix_truncate(struct inode * inode) +{ + if (!(S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode))) + return; + if (INODE_VERSION(inode) == MINIX_V1) + V1_minix_truncate(inode); + else + V2_minix_truncate(inode); +} + +static struct file_system_type minix_fs_type = { + .owner = THIS_MODULE, + .name = "minix", + .kill_sb = kill_block_super, + .fs_flags = FS_REQUIRES_DEV, + .init_fs_context = minix_init_fs_context, +}; +MODULE_ALIAS_FS("minix"); + +static int __init init_minix_fs(void) +{ + int err = init_inodecache(); + if (err) + goto out1; + err = register_filesystem(&minix_fs_type); + if (err) + goto out; + return 0; +out: + destroy_inodecache(); +out1: + return err; +} + +static void __exit exit_minix_fs(void) +{ + unregister_filesystem(&minix_fs_type); + destroy_inodecache(); +} + +module_init(init_minix_fs) +module_exit(exit_minix_fs) +MODULE_DESCRIPTION("Minix file system"); +MODULE_LICENSE("GPL"); + diff --git a/fs/minix/itree_common.c b/fs/minix/itree_common.c new file mode 100644 index 000000000..c3cd2c75a --- /dev/null +++ b/fs/minix/itree_common.c @@ -0,0 +1,374 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Generic part */ + +typedef struct { + block_t *p; + block_t key; + struct buffer_head *bh; +} Indirect; + +static DEFINE_RWLOCK(pointers_lock); + +static inline void add_chain(Indirect *p, struct buffer_head *bh, block_t *v) +{ + p->key = *(p->p = v); + p->bh = bh; +} + +static inline int verify_chain(Indirect *from, Indirect *to) +{ + while (from <= to && from->key == *from->p) + from++; + return (from > to); +} + +static inline block_t *block_end(struct buffer_head *bh) +{ + return (block_t *)((char*)bh->b_data + bh->b_size); +} + +static inline Indirect *get_branch(struct inode *inode, + int depth, + int *offsets, + Indirect chain[DEPTH], + int *err) +{ + struct super_block *sb = inode->i_sb; + Indirect *p = chain; + struct buffer_head *bh; + + *err = 0; + /* i_data is not going away, no lock needed */ + add_chain (chain, NULL, i_data(inode) + *offsets); + if (!p->key) + goto no_block; + while (--depth) { + bh = sb_bread(sb, block_to_cpu(p->key)); + if (!bh) + goto failure; + read_lock(&pointers_lock); + if (!verify_chain(chain, p)) + goto changed; + add_chain(++p, bh, (block_t *)bh->b_data + *++offsets); + read_unlock(&pointers_lock); + if (!p->key) + goto no_block; + } + return NULL; + +changed: + read_unlock(&pointers_lock); + brelse(bh); + *err = -EAGAIN; + goto no_block; +failure: + *err = -EIO; +no_block: + return p; +} + +static int alloc_branch(struct inode *inode, + int num, + int *offsets, + Indirect *branch) +{ + int n = 0; + int i; + int parent = minix_new_block(inode); + int err = -ENOSPC; + + branch[0].key = cpu_to_block(parent); + if (parent) for (n = 1; n < num; n++) { + struct buffer_head *bh; + /* Allocate the next block */ + int nr = minix_new_block(inode); + if (!nr) + break; + branch[n].key = cpu_to_block(nr); + bh = sb_getblk(inode->i_sb, parent); + if (!bh) { + minix_free_block(inode, nr); + err = -ENOMEM; + break; + } + lock_buffer(bh); + memset(bh->b_data, 0, bh->b_size); + branch[n].bh = bh; + branch[n].p = (block_t*) bh->b_data + offsets[n]; + *branch[n].p = branch[n].key; + set_buffer_uptodate(bh); + unlock_buffer(bh); + mmb_mark_buffer_dirty(bh, &minix_i(inode)->i_metadata_bhs); + parent = nr; + } + if (n == num) + return 0; + + /* Allocation failed, free what we already allocated */ + for (i = 1; i < n; i++) + bforget(branch[i].bh); + for (i = 0; i < n; i++) + minix_free_block(inode, block_to_cpu(branch[i].key)); + return err; +} + +static inline int splice_branch(struct inode *inode, + Indirect chain[DEPTH], + Indirect *where, + int num) +{ + int i; + + write_lock(&pointers_lock); + + /* Verify that place we are splicing to is still there and vacant */ + if (!verify_chain(chain, where-1) || *where->p) + goto changed; + + *where->p = where->key; + + write_unlock(&pointers_lock); + + /* We are done with atomic stuff, now do the rest of housekeeping */ + + inode_set_ctime_current(inode); + + /* had we spliced it onto indirect block? */ + if (where->bh) + mmb_mark_buffer_dirty(where->bh, + &minix_i(inode)->i_metadata_bhs); + + mark_inode_dirty(inode); + return 0; + +changed: + write_unlock(&pointers_lock); + for (i = 1; i < num; i++) + bforget(where[i].bh); + for (i = 0; i < num; i++) + minix_free_block(inode, block_to_cpu(where[i].key)); + return -EAGAIN; +} + +static int get_block(struct inode * inode, sector_t block, + struct buffer_head *bh, int create) +{ + int err = -EIO; + int offsets[DEPTH]; + Indirect chain[DEPTH]; + Indirect *partial; + int left; + int depth = block_to_path(inode, block, offsets); + + if (depth == 0) + goto out; + +reread: + partial = get_branch(inode, depth, offsets, chain, &err); + + /* Simplest case - block found, no allocation needed */ + if (!partial) { +got_it: + map_bh(bh, inode->i_sb, block_to_cpu(chain[depth-1].key)); + /* Clean up and exit */ + partial = chain+depth-1; /* the whole chain */ + goto cleanup; + } + + /* Next simple case - plain lookup or failed read of indirect block */ + if (!create || err == -EIO) { +cleanup: + while (partial > chain) { + brelse(partial->bh); + partial--; + } +out: + return err; + } + + /* + * Indirect block might be removed by truncate while we were + * reading it. Handling of that case (forget what we've got and + * reread) is taken out of the main path. + */ + if (err == -EAGAIN) + goto changed; + + left = (chain + depth) - partial; + err = alloc_branch(inode, left, offsets+(partial-chain), partial); + if (err) + goto cleanup; + + if (splice_branch(inode, chain, partial, left) < 0) + goto changed; + + set_buffer_new(bh); + goto got_it; + +changed: + while (partial > chain) { + brelse(partial->bh); + partial--; + } + goto reread; +} + +static inline int all_zeroes(block_t *p, block_t *q) +{ + while (p < q) + if (*p++) + return 0; + return 1; +} + +static Indirect *find_shared(struct inode *inode, + int depth, + int offsets[DEPTH], + Indirect chain[DEPTH], + block_t *top) +{ + Indirect *partial, *p; + int k, err; + + *top = 0; + for (k = depth; k > 1 && !offsets[k-1]; k--) + ; + partial = get_branch(inode, k, offsets, chain, &err); + + write_lock(&pointers_lock); + if (!partial) + partial = chain + k-1; + if (!partial->key && *partial->p) { + write_unlock(&pointers_lock); + goto no_top; + } + for (p=partial;p>chain && all_zeroes((block_t*)p->bh->b_data,p->p);p--) + ; + if (p == chain + k - 1 && p > chain) { + p->p--; + } else { + *top = *p->p; + *p->p = 0; + } + write_unlock(&pointers_lock); + + while(partial > p) + { + brelse(partial->bh); + partial--; + } +no_top: + return partial; +} + +static inline void free_data(struct inode *inode, block_t *p, block_t *q) +{ + unsigned long nr; + + for ( ; p < q ; p++) { + nr = block_to_cpu(*p); + if (nr) { + *p = 0; + minix_free_block(inode, nr); + } + } +} + +static void free_branches(struct inode *inode, block_t *p, block_t *q, int depth) +{ + struct buffer_head * bh; + unsigned long nr; + + if (depth--) { + for ( ; p < q ; p++) { + nr = block_to_cpu(*p); + if (!nr) + continue; + *p = 0; + bh = sb_bread(inode->i_sb, nr); + if (!bh) + continue; + free_branches(inode, (block_t*)bh->b_data, + block_end(bh), depth); + bforget(bh); + minix_free_block(inode, nr); + mark_inode_dirty(inode); + } + } else + free_data(inode, p, q); +} + +static inline void truncate (struct inode * inode) +{ + struct super_block *sb = inode->i_sb; + block_t *idata = i_data(inode); + int offsets[DEPTH]; + Indirect chain[DEPTH]; + Indirect *partial; + block_t nr = 0; + int n; + int first_whole; + long iblock; + + iblock = (inode->i_size + sb->s_blocksize -1) >> sb->s_blocksize_bits; + block_truncate_page(inode->i_mapping, inode->i_size, get_block); + + n = block_to_path(inode, iblock, offsets); + if (!n) + return; + + if (n == 1) { + free_data(inode, idata+offsets[0], idata + DIRECT); + first_whole = 0; + goto do_indirects; + } + + first_whole = offsets[0] + 1 - DIRECT; + partial = find_shared(inode, n, offsets, chain, &nr); + if (nr) { + if (partial == chain) + mark_inode_dirty(inode); + else + mmb_mark_buffer_dirty(partial->bh, + &minix_i(inode)->i_metadata_bhs); + free_branches(inode, &nr, &nr+1, (chain+n-1) - partial); + } + /* Clear the ends of indirect blocks on the shared branch */ + while (partial > chain) { + free_branches(inode, partial->p + 1, block_end(partial->bh), + (chain+n-1) - partial); + mmb_mark_buffer_dirty(partial->bh, + &minix_i(inode)->i_metadata_bhs); + brelse (partial->bh); + partial--; + } +do_indirects: + /* Kill the remaining (whole) subtrees */ + while (first_whole < DEPTH-1) { + nr = idata[DIRECT+first_whole]; + if (nr) { + idata[DIRECT+first_whole] = 0; + mark_inode_dirty(inode); + free_branches(inode, &nr, &nr+1, first_whole+1); + } + first_whole++; + } + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + mark_inode_dirty(inode); +} + +static inline unsigned nblocks(loff_t size, struct super_block *sb) +{ + int k = sb->s_blocksize_bits - 10; + unsigned blocks, res, direct = DIRECT, i = DEPTH; + blocks = (size + sb->s_blocksize - 1) >> (BLOCK_SIZE_BITS + k); + res = blocks; + while (--i && blocks > direct) { + blocks -= direct; + blocks += sb->s_blocksize/sizeof(block_t) - 1; + blocks /= sb->s_blocksize/sizeof(block_t); + res += blocks; + direct = 1; + } + return res; +} diff --git a/fs/minix/itree_v1.c b/fs/minix/itree_v1.c new file mode 100644 index 000000000..1fed90604 --- /dev/null +++ b/fs/minix/itree_v1.c @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "minix.h" + +enum {DEPTH = 3, DIRECT = 7}; /* Only double indirect */ + +typedef u16 block_t; /* 16 bit, host order */ + +static inline unsigned long block_to_cpu(block_t n) +{ + return n; +} + +static inline block_t cpu_to_block(unsigned long n) +{ + return n; +} + +static inline block_t *i_data(struct inode *inode) +{ + return (block_t *)minix_i(inode)->u.i1_data; +} + +static int block_to_path(struct inode * inode, long block, int offsets[DEPTH]) +{ + int n = 0; + + if (block < 0) { + printk("MINIX-fs: block_to_path: block %ld < 0 on dev %pg\n", + block, inode->i_sb->s_bdev); + return 0; + } + if ((u64)block * BLOCK_SIZE >= inode->i_sb->s_maxbytes) + return 0; + + if (block < 7) { + offsets[n++] = block; + } else if ((block -= 7) < 512) { + offsets[n++] = 7; + offsets[n++] = block; + } else { + block -= 512; + offsets[n++] = 8; + offsets[n++] = block>>9; + offsets[n++] = block & 511; + } + return n; +} + +#include "itree_common.c" + +int V1_minix_get_block(struct inode * inode, long block, + struct buffer_head *bh_result, int create) +{ + return get_block(inode, block, bh_result, create); +} + +void V1_minix_truncate(struct inode * inode) +{ + truncate(inode); +} + +unsigned V1_minix_blocks(loff_t size, struct super_block *sb) +{ + return nblocks(size, sb); +} diff --git a/fs/minix/itree_v2.c b/fs/minix/itree_v2.c new file mode 100644 index 000000000..9d00f31a2 --- /dev/null +++ b/fs/minix/itree_v2.c @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include "minix.h" + +enum {DIRECT = 7, DEPTH = 4}; /* Have triple indirect */ + +typedef u32 block_t; /* 32 bit, host order */ + +static inline unsigned long block_to_cpu(block_t n) +{ + return n; +} + +static inline block_t cpu_to_block(unsigned long n) +{ + return n; +} + +static inline block_t *i_data(struct inode *inode) +{ + return (block_t *)minix_i(inode)->u.i2_data; +} + +#define DIRCOUNT 7 +#define INDIRCOUNT(sb) (1 << ((sb)->s_blocksize_bits - 2)) + +static int block_to_path(struct inode * inode, long block, int offsets[DEPTH]) +{ + int n = 0; + struct super_block *sb = inode->i_sb; + + if (block < 0) { + printk("MINIX-fs: block_to_path: block %ld < 0 on dev %pg\n", + block, sb->s_bdev); + return 0; + } + if ((u64)block * (u64)sb->s_blocksize >= sb->s_maxbytes) + return 0; + + if (block < DIRCOUNT) { + offsets[n++] = block; + } else if ((block -= DIRCOUNT) < INDIRCOUNT(sb)) { + offsets[n++] = DIRCOUNT; + offsets[n++] = block; + } else if ((block -= INDIRCOUNT(sb)) < INDIRCOUNT(sb) * INDIRCOUNT(sb)) { + offsets[n++] = DIRCOUNT + 1; + offsets[n++] = block / INDIRCOUNT(sb); + offsets[n++] = block % INDIRCOUNT(sb); + } else { + block -= INDIRCOUNT(sb) * INDIRCOUNT(sb); + offsets[n++] = DIRCOUNT + 2; + offsets[n++] = (block / INDIRCOUNT(sb)) / INDIRCOUNT(sb); + offsets[n++] = (block / INDIRCOUNT(sb)) % INDIRCOUNT(sb); + offsets[n++] = block % INDIRCOUNT(sb); + } + return n; +} + +#include "itree_common.c" + +int V2_minix_get_block(struct inode * inode, long block, + struct buffer_head *bh_result, int create) +{ + return get_block(inode, block, bh_result, create); +} + +void V2_minix_truncate(struct inode * inode) +{ + truncate(inode); +} + +unsigned V2_minix_blocks(loff_t size, struct super_block *sb) +{ + return nblocks(size, sb); +} diff --git a/fs/minix/minix.h b/fs/minix/minix.h new file mode 100644 index 000000000..78722ce22 --- /dev/null +++ b/fs/minix/minix.h @@ -0,0 +1,180 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef FS_MINIX_H +#define FS_MINIX_H + +#include +#include +#include + +#define INODE_VERSION(inode) minix_sb(inode->i_sb)->s_version +#define MINIX_V1 0x0001 /* original minix fs */ +#define MINIX_V2 0x0002 /* minix V2 fs */ +#define MINIX_V3 0x0003 /* minix V3 fs */ + +/* + * minix fs inode data in memory + */ +struct minix_inode_info { + union { + __u16 i1_data[16]; + __u32 i2_data[16]; + } u; + struct mapping_metadata_bhs i_metadata_bhs; + struct inode vfs_inode; +}; + +/* + * minix super-block data in memory + */ +struct minix_sb_info { + unsigned long s_ninodes; + unsigned long s_nzones; + unsigned long s_imap_blocks; + unsigned long s_zmap_blocks; + unsigned long s_firstdatazone; + unsigned long s_log_zone_size; + int s_dirsize; + int s_namelen; + struct buffer_head ** s_imap; + struct buffer_head ** s_zmap; + struct buffer_head * s_sbh; + struct minix_super_block * s_ms; + unsigned short s_mount_state; + unsigned short s_version; +}; + +void __minix_error_inode(struct inode *inode, const char *function, + unsigned int line, const char *fmt, ...); + +struct inode *minix_iget(struct super_block *, unsigned long); +struct minix_inode *minix_V1_raw_inode(struct super_block *, ino_t, struct buffer_head **); +struct minix2_inode *minix_V2_raw_inode(struct super_block *, ino_t, struct buffer_head **); +struct inode *minix_new_inode(const struct inode *, umode_t); +void minix_free_inode(struct inode *inode); +unsigned long minix_count_free_inodes(struct super_block *sb); +int minix_new_block(struct inode *inode); +void minix_free_block(struct inode *inode, unsigned long block); +unsigned long minix_count_free_blocks(struct super_block *sb); +int minix_getattr(struct mnt_idmap *, const struct path *, + struct kstat *, u32, unsigned int); +int minix_prepare_chunk(struct folio *folio, loff_t pos, unsigned len); +struct mapping_metadata_bhs *minix_get_metadata_bhs(struct inode *inode); + +extern void V1_minix_truncate(struct inode *); +extern void V2_minix_truncate(struct inode *); +extern void minix_truncate(struct inode *); +extern void minix_set_inode(struct inode *, dev_t); +extern int V1_minix_get_block(struct inode *, long, struct buffer_head *, int); +extern int V2_minix_get_block(struct inode *, long, struct buffer_head *, int); +extern unsigned V1_minix_blocks(loff_t, struct super_block *); +extern unsigned V2_minix_blocks(loff_t, struct super_block *); + +struct minix_dir_entry *minix_find_entry(struct dentry *, struct folio **); +int minix_add_link(struct dentry*, struct inode*); +int minix_delete_entry(struct minix_dir_entry *, struct folio *); +int minix_make_empty(struct inode*, struct inode*); +int minix_empty_dir(struct inode*); +int minix_set_link(struct minix_dir_entry *de, struct folio *folio, + struct inode *inode); +struct minix_dir_entry *minix_dotdot(struct inode*, struct folio **); +ino_t minix_inode_by_name(struct dentry*); + +extern const struct inode_operations minix_file_inode_operations; +extern const struct inode_operations minix_dir_inode_operations; +extern const struct file_operations minix_file_operations; +extern const struct file_operations minix_dir_operations; + +static inline struct minix_sb_info *minix_sb(struct super_block *sb) +{ + return sb->s_fs_info; +} + +static inline struct minix_inode_info *minix_i(struct inode *inode) +{ + return container_of(inode, struct minix_inode_info, vfs_inode); +} + +static inline unsigned minix_blocks_needed(unsigned bits, unsigned blocksize) +{ + return DIV_ROUND_UP_POW2(bits, blocksize * 8); +} + +#if defined(CONFIG_MINIX_FS_NATIVE_ENDIAN) && \ + defined(CONFIG_MINIX_FS_BIG_ENDIAN_16BIT_INDEXED) + +#error Minix file system byte order broken + +#elif defined(CONFIG_MINIX_FS_NATIVE_ENDIAN) + +/* + * big-endian 32 or 64 bit indexed bitmaps on big-endian system or + * little-endian bitmaps on little-endian system + */ + +#define minix_test_and_set_bit(nr, addr) \ + __test_and_set_bit((nr), (unsigned long *)(addr)) +#define minix_set_bit(nr, addr) \ + __set_bit((nr), (unsigned long *)(addr)) +#define minix_test_and_clear_bit(nr, addr) \ + __test_and_clear_bit((nr), (unsigned long *)(addr)) +#define minix_test_bit(nr, addr) \ + test_bit((nr), (unsigned long *)(addr)) +#define minix_find_first_zero_bit(addr, size) \ + find_first_zero_bit((unsigned long *)(addr), (size)) + +#elif defined(CONFIG_MINIX_FS_BIG_ENDIAN_16BIT_INDEXED) + +/* + * big-endian 16bit indexed bitmaps + */ + +static inline int minix_find_first_zero_bit(const void *vaddr, unsigned size) +{ + const unsigned short *p = vaddr, *addr = vaddr; + unsigned short num; + + if (!size) + return 0; + + size >>= 4; + while (*p++ == 0xffff) { + if (--size == 0) + return (p - addr) << 4; + } + + num = *--p; + return ((p - addr) << 4) + ffz(num); +} + +#define minix_test_and_set_bit(nr, addr) \ + __test_and_set_bit((nr) ^ 16, (unsigned long *)(addr)) +#define minix_set_bit(nr, addr) \ + __set_bit((nr) ^ 16, (unsigned long *)(addr)) +#define minix_test_and_clear_bit(nr, addr) \ + __test_and_clear_bit((nr) ^ 16, (unsigned long *)(addr)) + +static inline int minix_test_bit(int nr, const void *vaddr) +{ + const unsigned short *p = vaddr; + return (p[nr >> 4] & (1U << (nr & 15))) != 0; +} + +#else + +/* + * little-endian bitmaps + */ + +#define minix_test_and_set_bit __test_and_set_bit_le +#define minix_set_bit __set_bit_le +#define minix_test_and_clear_bit __test_and_clear_bit_le +#define minix_test_bit test_bit_le +#define minix_find_first_zero_bit find_first_zero_bit_le + +#endif + +#define minix_error_inode(inode, fmt, ...) \ + __minix_error_inode((inode), __func__, __LINE__, \ + (fmt), ##__VA_ARGS__) + +#endif /* FS_MINIX_H */ diff --git a/fs/minix/namei.c b/fs/minix/namei.c new file mode 100644 index 000000000..5525ba367 --- /dev/null +++ b/fs/minix/namei.c @@ -0,0 +1,291 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/minix/namei.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + */ + +#include "minix.h" + +static int add_nondir(struct dentry *dentry, struct inode *inode) +{ + int err = minix_add_link(dentry, inode); + if (!err) { + d_instantiate(dentry, inode); + return 0; + } + inode_dec_link_count(inode); + iput(inode); + return err; +} + +static struct dentry *minix_lookup(struct inode * dir, struct dentry *dentry, unsigned int flags) +{ + struct inode * inode = NULL; + ino_t ino; + + if (dentry->d_name.len > minix_sb(dir->i_sb)->s_namelen) + return ERR_PTR(-ENAMETOOLONG); + + ino = minix_inode_by_name(dentry); + if (ino) + inode = minix_iget(dir->i_sb, ino); + return d_splice_alias(inode, dentry); +} + +static int minix_mknod(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) +{ + struct inode *inode; + + if (!old_valid_dev(rdev)) + return -EINVAL; + + inode = minix_new_inode(dir, mode); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + minix_set_inode(inode, rdev); + mark_inode_dirty(inode); + return add_nondir(dentry, inode); +} + +static int minix_tmpfile(struct mnt_idmap *idmap, struct inode *dir, + struct file *file, umode_t mode) +{ + struct inode *inode = minix_new_inode(dir, mode); + + if (IS_ERR(inode)) + return finish_open_simple(file, PTR_ERR(inode)); + minix_set_inode(inode, 0); + mark_inode_dirty(inode); + d_tmpfile(file, inode); + return finish_open_simple(file, 0); +} + +static int minix_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + return minix_mknod(&nop_mnt_idmap, dir, dentry, mode, 0); +} + +static int minix_symlink(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const char *symname) +{ + int i = strlen(symname)+1; + struct inode * inode; + int err; + + if (i > dir->i_sb->s_blocksize) + return -ENAMETOOLONG; + + inode = minix_new_inode(dir, S_IFLNK | 0777); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + minix_set_inode(inode, 0); + err = page_symlink(inode, symname, i); + if (unlikely(err)) { + inode_dec_link_count(inode); + iput(inode); + return err; + } + return add_nondir(dentry, inode); +} + +static int minix_link(struct dentry * old_dentry, struct inode * dir, + struct dentry *dentry) +{ + struct inode *inode = d_inode(old_dentry); + + inode_set_ctime_current(inode); + inode_inc_link_count(inode); + ihold(inode); + return add_nondir(dentry, inode); +} + +static struct dentry *minix_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct inode * inode; + int err; + + inode = minix_new_inode(dir, mode); + if (IS_ERR(inode)) + return ERR_CAST(inode); + + inode_inc_link_count(dir); + minix_set_inode(inode, 0); + inode_inc_link_count(inode); + + err = minix_make_empty(inode, dir); + if (err) + goto out_fail; + + err = minix_add_link(dentry, inode); + if (err) + goto out_fail; + + d_instantiate(dentry, inode); +out: + return ERR_PTR(err); + +out_fail: + inode_dec_link_count(inode); + inode_dec_link_count(inode); + iput(inode); + inode_dec_link_count(dir); + goto out; +} + +static int minix_unlink(struct inode * dir, struct dentry *dentry) +{ + struct inode * inode = d_inode(dentry); + struct folio *folio; + struct minix_dir_entry * de; + int err; + + if (inode->i_nlink == 0) { + minix_error_inode(inode, "inode has corrupted nlink"); + return -EFSCORRUPTED; + } + + de = minix_find_entry(dentry, &folio); + if (!de) + return -ENOENT; + err = minix_delete_entry(de, folio); + folio_release_kmap(folio, de); + + if (err) + return err; + inode_set_ctime_to_ts(inode, inode_get_ctime(dir)); + inode_dec_link_count(inode); + return 0; +} + +static int minix_rmdir(struct inode * dir, struct dentry *dentry) +{ + struct inode * inode = d_inode(dentry); + int err = -EFSCORRUPTED; + + if (dir->i_nlink <= 2) { + minix_error_inode(dir, "inode has corrupted nlink"); + goto out; + } + + err = -ENOTEMPTY; + if (!minix_empty_dir(inode)) + goto out; + + err = minix_unlink(dir, dentry); + if (!err) { + inode_dec_link_count(dir); + inode_dec_link_count(inode); + } + +out: + return err; +} + +static int minix_rename(struct mnt_idmap *idmap, + struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry, + unsigned int flags) +{ + struct inode * old_inode = d_inode(old_dentry); + struct inode * new_inode = d_inode(new_dentry); + struct folio * dir_folio = NULL; + struct minix_dir_entry * dir_de = NULL; + struct folio *old_folio; + struct minix_dir_entry * old_de; + int err = -ENOENT; + + if (flags & ~RENAME_NOREPLACE) + return -EINVAL; + + old_de = minix_find_entry(old_dentry, &old_folio); + if (!old_de) + goto out; + + if (S_ISDIR(old_inode->i_mode)) { + err = -EIO; + dir_de = minix_dotdot(old_inode, &dir_folio); + if (!dir_de) + goto out_old; + } + + if (new_inode) { + struct folio *new_folio; + struct minix_dir_entry * new_de; + + err = -ENOTEMPTY; + if (dir_de && !minix_empty_dir(new_inode)) + goto out_dir; + + err = -EFSCORRUPTED; + if (new_inode->i_nlink == 0 || (dir_de && new_inode->i_nlink != 2)) { + minix_error_inode(new_inode, "inode has corrupted nlink"); + goto out_dir; + } + + if (dir_de && old_dir->i_nlink <= 2) { + minix_error_inode(old_dir, "inode has corrupted nlink"); + goto out_dir; + } + + err = -ENOENT; + new_de = minix_find_entry(new_dentry, &new_folio); + if (!new_de) + goto out_dir; + err = minix_set_link(new_de, new_folio, old_inode); + folio_release_kmap(new_folio, new_de); + if (err) + goto out_dir; + inode_set_ctime_current(new_inode); + if (dir_de) + drop_nlink(new_inode); + inode_dec_link_count(new_inode); + } else { + err = minix_add_link(new_dentry, old_inode); + if (err) + goto out_dir; + if (dir_de) + inode_inc_link_count(new_dir); + } + + err = minix_delete_entry(old_de, old_folio); + if (err) + goto out_dir; + + mark_inode_dirty(old_inode); + + if (dir_de) { + err = minix_set_link(dir_de, dir_folio, new_dir); + if (!err) + inode_dec_link_count(old_dir); + } +out_dir: + if (dir_de) + folio_release_kmap(dir_folio, dir_de); +out_old: + folio_release_kmap(old_folio, old_de); +out: + return err; +} + +/* + * directories can handle most operations... + */ +const struct inode_operations minix_dir_inode_operations = { + .create = minix_create, + .lookup = minix_lookup, + .link = minix_link, + .unlink = minix_unlink, + .symlink = minix_symlink, + .mkdir = minix_mkdir, + .rmdir = minix_rmdir, + .mknod = minix_mknod, + .rename = minix_rename, + .getattr = minix_getattr, + .tmpfile = minix_tmpfile, +}; diff --git a/fs/mnt_idmapping.c b/fs/mnt_idmapping.c new file mode 100644 index 000000000..cb61fbdb5 --- /dev/null +++ b/fs/mnt_idmapping.c @@ -0,0 +1,389 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2022 Christian Brauner */ + +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +/* + * Outside of this file vfs{g,u}id_t are always created from k{g,u}id_t, + * never from raw values. These are just internal helpers. + */ +#define VFSUIDT_INIT_RAW(val) (vfsuid_t){ val } +#define VFSGIDT_INIT_RAW(val) (vfsgid_t){ val } + +struct mnt_idmap { + struct uid_gid_map uid_map; + struct uid_gid_map gid_map; + refcount_t count; +}; + +/* + * Carries the initial idmapping of 0:0:4294967295 which is an identity + * mapping. This means that {g,u}id 0 is mapped to {g,u}id 0, {g,u}id 1 is + * mapped to {g,u}id 1, [...], {g,u}id 1000 to {g,u}id 1000, [...]. + */ +struct mnt_idmap nop_mnt_idmap = { + .count = REFCOUNT_INIT(1), +}; +EXPORT_SYMBOL_GPL(nop_mnt_idmap); + +/* + * Carries the invalid idmapping of a full 0-4294967295 {g,u}id range. + * This means that all {g,u}ids are mapped to INVALID_VFS{G,U}ID. + */ +struct mnt_idmap invalid_mnt_idmap = { + .count = REFCOUNT_INIT(1), +}; +EXPORT_SYMBOL_GPL(invalid_mnt_idmap); + +/** + * initial_idmapping - check whether this is the initial mapping + * @ns: idmapping to check + * + * Check whether this is the initial mapping, mapping 0 to 0, 1 to 1, + * [...], 1000 to 1000 [...]. + * + * Return: true if this is the initial mapping, false if not. + */ +static inline bool initial_idmapping(const struct user_namespace *ns) +{ + return ns == &init_user_ns; +} + +/** + * make_vfsuid - map a filesystem kuid according to an idmapping + * @idmap: the mount's idmapping + * @fs_userns: the filesystem's idmapping + * @kuid : kuid to be mapped + * + * Take a @kuid and remap it from @fs_userns into @idmap. Use this + * function when preparing a @kuid to be reported to userspace. + * + * If initial_idmapping() determines that this is not an idmapped mount + * we can simply return @kuid unchanged. + * If initial_idmapping() tells us that the filesystem is not mounted with an + * idmapping we know the value of @kuid won't change when calling + * from_kuid() so we can simply retrieve the value via __kuid_val() + * directly. + * + * Return: @kuid mapped according to @idmap. + * If @kuid has no mapping in either @idmap or @fs_userns INVALID_UID is + * returned. + */ + +vfsuid_t make_vfsuid(struct mnt_idmap *idmap, + struct user_namespace *fs_userns, + kuid_t kuid) +{ + uid_t uid; + + if (idmap == &nop_mnt_idmap) + return VFSUIDT_INIT(kuid); + if (idmap == &invalid_mnt_idmap) + return INVALID_VFSUID; + if (initial_idmapping(fs_userns)) + uid = __kuid_val(kuid); + else + uid = from_kuid(fs_userns, kuid); + if (uid == (uid_t)-1) + return INVALID_VFSUID; + return VFSUIDT_INIT_RAW(map_id_down(&idmap->uid_map, uid)); +} +EXPORT_SYMBOL_GPL(make_vfsuid); + +/** + * make_vfsgid - map a filesystem kgid according to an idmapping + * @idmap: the mount's idmapping + * @fs_userns: the filesystem's idmapping + * @kgid : kgid to be mapped + * + * Take a @kgid and remap it from @fs_userns into @idmap. Use this + * function when preparing a @kgid to be reported to userspace. + * + * If initial_idmapping() determines that this is not an idmapped mount + * we can simply return @kgid unchanged. + * If initial_idmapping() tells us that the filesystem is not mounted with an + * idmapping we know the value of @kgid won't change when calling + * from_kgid() so we can simply retrieve the value via __kgid_val() + * directly. + * + * Return: @kgid mapped according to @idmap. + * If @kgid has no mapping in either @idmap or @fs_userns INVALID_GID is + * returned. + */ +vfsgid_t make_vfsgid(struct mnt_idmap *idmap, + struct user_namespace *fs_userns, kgid_t kgid) +{ + gid_t gid; + + if (idmap == &nop_mnt_idmap) + return VFSGIDT_INIT(kgid); + if (idmap == &invalid_mnt_idmap) + return INVALID_VFSGID; + if (initial_idmapping(fs_userns)) + gid = __kgid_val(kgid); + else + gid = from_kgid(fs_userns, kgid); + if (gid == (gid_t)-1) + return INVALID_VFSGID; + return VFSGIDT_INIT_RAW(map_id_down(&idmap->gid_map, gid)); +} +EXPORT_SYMBOL_GPL(make_vfsgid); + +/** + * from_vfsuid - map a vfsuid into the filesystem idmapping + * @idmap: the mount's idmapping + * @fs_userns: the filesystem's idmapping + * @vfsuid : vfsuid to be mapped + * + * Map @vfsuid into the filesystem idmapping. This function has to be used in + * order to e.g. write @vfsuid to inode->i_uid. + * + * Return: @vfsuid mapped into the filesystem idmapping + */ +kuid_t from_vfsuid(struct mnt_idmap *idmap, + struct user_namespace *fs_userns, vfsuid_t vfsuid) +{ + uid_t uid; + + if (idmap == &nop_mnt_idmap) + return AS_KUIDT(vfsuid); + if (idmap == &invalid_mnt_idmap) + return INVALID_UID; + uid = map_id_up(&idmap->uid_map, __vfsuid_val(vfsuid)); + if (uid == (uid_t)-1) + return INVALID_UID; + if (initial_idmapping(fs_userns)) + return KUIDT_INIT(uid); + return make_kuid(fs_userns, uid); +} +EXPORT_SYMBOL_GPL(from_vfsuid); + +/** + * from_vfsgid - map a vfsgid into the filesystem idmapping + * @idmap: the mount's idmapping + * @fs_userns: the filesystem's idmapping + * @vfsgid : vfsgid to be mapped + * + * Map @vfsgid into the filesystem idmapping. This function has to be used in + * order to e.g. write @vfsgid to inode->i_gid. + * + * Return: @vfsgid mapped into the filesystem idmapping + */ +kgid_t from_vfsgid(struct mnt_idmap *idmap, + struct user_namespace *fs_userns, vfsgid_t vfsgid) +{ + gid_t gid; + + if (idmap == &nop_mnt_idmap) + return AS_KGIDT(vfsgid); + if (idmap == &invalid_mnt_idmap) + return INVALID_GID; + gid = map_id_up(&idmap->gid_map, __vfsgid_val(vfsgid)); + if (gid == (gid_t)-1) + return INVALID_GID; + if (initial_idmapping(fs_userns)) + return KGIDT_INIT(gid); + return make_kgid(fs_userns, gid); +} +EXPORT_SYMBOL_GPL(from_vfsgid); + +#ifdef CONFIG_MULTIUSER +/** + * vfsgid_in_group_p() - check whether a vfsuid matches the caller's groups + * @vfsgid: the mnt gid to match + * + * This function can be used to determine whether @vfsuid matches any of the + * caller's groups. + * + * Return: 1 if vfsuid matches caller's groups, 0 if not. + */ +int vfsgid_in_group_p(vfsgid_t vfsgid) +{ + return in_group_p(AS_KGIDT(vfsgid)); +} +#else +int vfsgid_in_group_p(vfsgid_t vfsgid) +{ + return 1; +} +#endif +EXPORT_SYMBOL_GPL(vfsgid_in_group_p); + +static int copy_mnt_idmap(struct uid_gid_map *map_from, + struct uid_gid_map *map_to) +{ + struct uid_gid_extent *forward, *reverse; + u32 nr_extents = READ_ONCE(map_from->nr_extents); + /* Pairs with smp_wmb() when writing the idmapping. */ + smp_rmb(); + + /* + * Don't blindly copy @map_to into @map_from if nr_extents is + * smaller or equal to UID_GID_MAP_MAX_BASE_EXTENTS. Since we + * read @nr_extents someone could have written an idmapping and + * then we might end up with inconsistent data. So just don't do + * anything at all. + */ + if (nr_extents == 0) + return -EINVAL; + + /* + * Here we know that nr_extents is greater than zero which means + * a map has been written. Since idmappings can't be changed + * once they have been written we know that we can safely copy + * from @map_to into @map_from. + */ + + if (nr_extents <= UID_GID_MAP_MAX_BASE_EXTENTS) { + *map_to = *map_from; + return 0; + } + + forward = kmemdup_array(map_from->forward, nr_extents, + sizeof(struct uid_gid_extent), + GFP_KERNEL_ACCOUNT); + if (!forward) + return -ENOMEM; + + reverse = kmemdup_array(map_from->reverse, nr_extents, + sizeof(struct uid_gid_extent), + GFP_KERNEL_ACCOUNT); + if (!reverse) { + kfree(forward); + return -ENOMEM; + } + + /* + * The idmapping isn't exposed anywhere so we don't need to care + * about ordering between extent pointers and @nr_extents + * initialization. + */ + map_to->forward = forward; + map_to->reverse = reverse; + map_to->nr_extents = nr_extents; + return 0; +} + +static void free_mnt_idmap(struct mnt_idmap *idmap) +{ + if (idmap->uid_map.nr_extents > UID_GID_MAP_MAX_BASE_EXTENTS) { + kfree(idmap->uid_map.forward); + kfree(idmap->uid_map.reverse); + } + if (idmap->gid_map.nr_extents > UID_GID_MAP_MAX_BASE_EXTENTS) { + kfree(idmap->gid_map.forward); + kfree(idmap->gid_map.reverse); + } + kfree(idmap); +} + +struct mnt_idmap *alloc_mnt_idmap(struct user_namespace *mnt_userns) +{ + struct mnt_idmap *idmap; + int ret; + + idmap = kzalloc_obj(struct mnt_idmap, GFP_KERNEL_ACCOUNT); + if (!idmap) + return ERR_PTR(-ENOMEM); + + refcount_set(&idmap->count, 1); + ret = copy_mnt_idmap(&mnt_userns->uid_map, &idmap->uid_map); + if (!ret) + ret = copy_mnt_idmap(&mnt_userns->gid_map, &idmap->gid_map); + if (ret) { + free_mnt_idmap(idmap); + idmap = ERR_PTR(ret); + } + return idmap; +} + +/** + * mnt_idmap_get - get a reference to an idmapping + * @idmap: the idmap to bump the reference on + * + * If @idmap is not the @nop_mnt_idmap bump the reference count. + * + * Return: @idmap with reference count bumped if @not_mnt_idmap isn't passed. + */ +struct mnt_idmap *mnt_idmap_get(struct mnt_idmap *idmap) +{ + if (idmap != &nop_mnt_idmap && idmap != &invalid_mnt_idmap) + refcount_inc(&idmap->count); + + return idmap; +} +EXPORT_SYMBOL_GPL(mnt_idmap_get); + +/** + * mnt_idmap_put - put a reference to an idmapping + * @idmap: the idmap to put the reference on + * + * If this is a non-initial idmapping, put the reference count when a mount is + * released and free it if we're the last user. + */ +void mnt_idmap_put(struct mnt_idmap *idmap) +{ + if (idmap != &nop_mnt_idmap && idmap != &invalid_mnt_idmap && + refcount_dec_and_test(&idmap->count)) + free_mnt_idmap(idmap); +} +EXPORT_SYMBOL_GPL(mnt_idmap_put); + +int statmount_mnt_idmap(struct mnt_idmap *idmap, struct seq_file *seq, bool uid_map) +{ + struct uid_gid_map *map, *map_up; + u32 idx, nr_mappings; + + if (!is_valid_mnt_idmap(idmap)) + return 0; + + /* + * Idmappings are shown relative to the caller's idmapping. + * This is both the most intuitive and most useful solution. + */ + if (uid_map) { + map = &idmap->uid_map; + map_up = ¤t_user_ns()->uid_map; + } else { + map = &idmap->gid_map; + map_up = ¤t_user_ns()->gid_map; + } + + for (idx = 0, nr_mappings = 0; idx < map->nr_extents; idx++) { + uid_t lower; + struct uid_gid_extent *extent; + + if (map->nr_extents <= UID_GID_MAP_MAX_BASE_EXTENTS) + extent = &map->extent[idx]; + else + extent = &map->forward[idx]; + + /* + * Verify that the whole range of the mapping can be + * resolved in the caller's idmapping. If it cannot be + * resolved skip the mapping. + */ + lower = map_id_range_up(map_up, extent->lower_first, extent->count); + if (lower == (uid_t) -1) + continue; + + seq_printf(seq, "%u %u %u", extent->first, lower, extent->count); + if (seq_has_overflowed(seq)) + return -EAGAIN; + + seq->count++; /* mappings are separated by \0 */ + if (seq_has_overflowed(seq)) + return -EAGAIN; + + nr_mappings++; + } + + return nr_mappings; +} diff --git a/fs/mount.h b/fs/mount.h new file mode 100644 index 000000000..94fcc306d --- /dev/null +++ b/fs/mount.h @@ -0,0 +1,278 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include +#include +#include +#include + +extern struct file_system_type nullfs_fs_type; +extern struct list_head notify_list; + +struct mnt_namespace { + struct ns_common ns; + struct mount * root; + struct { + struct rb_root mounts; /* Protected by namespace_sem */ + struct rb_node *mnt_last_node; /* last (rightmost) mount in the rbtree */ + struct rb_node *mnt_first_node; /* first (leftmost) mount in the rbtree */ + }; + struct user_namespace *user_ns; + struct ucounts *ucounts; + wait_queue_head_t poll; + u64 seq_origin; /* Sequence number of origin mount namespace */ + u64 event; +#ifdef CONFIG_FSNOTIFY + __u32 n_fsnotify_mask; + struct fsnotify_mark_connector __rcu *n_fsnotify_marks; +#endif + struct hlist_head mnt_visible_mounts; /* SB_I_USERNS_VISIBLE mounts */ + unsigned int nr_mounts; /* # of mounts in the namespace */ + unsigned int pending_mounts; + refcount_t passive; /* number references not pinning @mounts */ + bool is_anon; +} __randomize_layout; + +struct mnt_pcp { + int mnt_count; + int mnt_writers; +}; + +struct mountpoint { + struct hlist_node m_hash; + struct dentry *m_dentry; + struct hlist_head m_list; +}; + +struct mount { + struct hlist_node mnt_hash; + struct mount *mnt_parent; + struct dentry *mnt_mountpoint; + struct vfsmount mnt; + union { + struct rb_node mnt_node; /* node in the ns->mounts rbtree */ + struct rcu_head mnt_rcu; + struct llist_node mnt_llist; + }; +#ifdef CONFIG_SMP + struct mnt_pcp __percpu *mnt_pcp; +#else + int mnt_count; + int mnt_writers; +#endif + struct list_head mnt_mounts; /* list of children, anchored here */ + struct list_head mnt_child; /* and going through their mnt_child */ + struct mount *mnt_next_for_sb; /* the next two fields are hlist_node, */ + struct mount * __aligned(1) *mnt_pprev_for_sb; + /* except that LSB of pprev is stolen */ +#define WRITE_HOLD 1 /* ... for use by mnt_hold_writers() */ + const char *mnt_devname; /* Name of device e.g. /dev/dsk/hda1 */ + struct list_head mnt_list; + struct list_head mnt_expire; /* link in fs-specific expiry list */ + struct list_head mnt_share; /* circular list of shared mounts */ + struct hlist_head mnt_slave_list;/* list of slave mounts */ + struct hlist_node mnt_slave; /* slave list entry */ + struct mount *mnt_master; /* slave is on master->mnt_slave_list */ + /* + * Containing namespace (active or deactivating, non-refcounted). + * Normally protected by namespace_sem. + * Can also be accessed locklessly under RCU. RCU readers can't rely on + * the namespace still being active, but implicitly hold a passive + * reference (because an RCU delay happens between a namespace being + * deactivated and the corresponding passive refcount drop). + */ + struct mnt_namespace *mnt_ns; + struct mountpoint *mnt_mp; /* where is it mounted */ + union { + struct hlist_node mnt_mp_list; /* list mounts with the same mountpoint */ + struct hlist_node mnt_umount; + }; +#ifdef CONFIG_FSNOTIFY + struct fsnotify_mark_connector __rcu *mnt_fsnotify_marks; + __u32 mnt_fsnotify_mask; + struct list_head to_notify; /* need to queue notification */ + struct mnt_namespace *prev_ns; /* previous namespace (NULL if none) */ +#endif + int mnt_t_flags; /* namespace_sem-protected flags */ + int mnt_id; /* mount identifier, reused */ + u64 mnt_id_unique; /* mount ID unique until reboot */ + int mnt_group_id; /* peer group identifier */ + int mnt_expiry_mark; /* true if marked for expiry */ + struct hlist_head mnt_pins; + struct hlist_head mnt_stuck_children; + struct hlist_node mnt_ns_visible; /* link in ns->mnt_visible_mounts */ + struct mount *overmount; /* mounted on ->mnt_root */ +} __randomize_layout; + +enum { + T_SHARED = 1, /* mount is shared */ + T_UNBINDABLE = 2, /* mount is unbindable */ + T_MARKED = 4, /* internal mark for propagate_... */ + T_UMOUNT_CANDIDATE = 8, /* for propagate_umount */ + + /* + * T_SHARED_MASK is the set of flags that should be cleared when a + * mount becomes shared. Currently, this is only the flag that says a + * mount cannot be bind mounted, since this is how we create a mount + * that shares events with another mount. If you add a new T_* + * flag, consider how it interacts with shared mounts. + */ + T_SHARED_MASK = T_UNBINDABLE, +}; + +#define MNT_NS_INTERNAL ERR_PTR(-EINVAL) /* distinct from any mnt_namespace */ + +static inline struct mount *real_mount(struct vfsmount *mnt) +{ + return container_of(mnt, struct mount, mnt); +} + +static inline int mnt_has_parent(const struct mount *mnt) +{ + return mnt != mnt->mnt_parent; +} + +static inline int is_mounted(struct vfsmount *mnt) +{ + /* neither detached nor internal? */ + return !IS_ERR_OR_NULL(real_mount(mnt)->mnt_ns); +} + +extern struct mount *__lookup_mnt(struct vfsmount *, struct dentry *); + +extern int __legitimize_mnt(struct vfsmount *, unsigned); + +static inline bool __path_is_mountpoint(const struct path *path) +{ + struct mount *m = __lookup_mnt(path->mnt, path->dentry); + return m && likely(!(m->mnt.mnt_flags & MNT_SYNC_UMOUNT)); +} + +extern void __detach_mounts(struct dentry *dentry); + +static inline void detach_mounts(struct dentry *dentry) +{ + if (!d_mountpoint(dentry)) + return; + __detach_mounts(dentry); +} + +static inline void get_mnt_ns(struct mnt_namespace *ns) +{ + ns_ref_inc(ns); +} + +extern seqlock_t mount_lock; + +DEFINE_LOCK_GUARD_0(mount_writer, write_seqlock(&mount_lock), + write_sequnlock(&mount_lock)) +DEFINE_LOCK_GUARD_0(mount_locked_reader, read_seqlock_excl(&mount_lock), + read_sequnlock_excl(&mount_lock)) + +struct proc_mounts { + struct mnt_namespace *ns; + struct path root; + int (*show)(struct seq_file *, struct vfsmount *); +}; + +extern const struct seq_operations mounts_op; + +extern bool __is_local_mountpoint(const struct dentry *dentry); +static inline bool is_local_mountpoint(const struct dentry *dentry) +{ + if (!d_mountpoint(dentry)) + return false; + + return __is_local_mountpoint(dentry); +} + +static inline bool is_anon_ns(struct mnt_namespace *ns) +{ + return ns->is_anon; +} + +static inline bool anon_ns_root(const struct mount *m) +{ + struct mnt_namespace *ns = READ_ONCE(m->mnt_ns); + + return !IS_ERR_OR_NULL(ns) && is_anon_ns(ns) && m == ns->root; +} + +static inline bool mnt_ns_attached(const struct mount *mnt) +{ + return !RB_EMPTY_NODE(&mnt->mnt_node); +} + +static inline bool mnt_ns_empty(const struct mnt_namespace *ns) +{ + return RB_EMPTY_ROOT(&ns->mounts); +} + +static inline void move_from_ns(struct mount *mnt) +{ + struct mnt_namespace *ns = mnt->mnt_ns; + WARN_ON(!mnt_ns_attached(mnt)); + if (ns->mnt_last_node == &mnt->mnt_node) + ns->mnt_last_node = rb_prev(&mnt->mnt_node); + if (ns->mnt_first_node == &mnt->mnt_node) + ns->mnt_first_node = rb_next(&mnt->mnt_node); + rb_erase(&mnt->mnt_node, &ns->mounts); + RB_CLEAR_NODE(&mnt->mnt_node); + if (!hlist_unhashed(&mnt->mnt_ns_visible)) + hlist_del_init(&mnt->mnt_ns_visible); +} + +bool has_locked_children(struct mount *mnt, struct dentry *dentry); +struct mnt_namespace *get_sequential_mnt_ns(struct mnt_namespace *mnt_ns, + bool previous); + +static inline struct mnt_namespace *to_mnt_ns(struct ns_common *ns) +{ + return container_of(ns, struct mnt_namespace, ns); +} + +#ifdef CONFIG_FSNOTIFY +static inline void mnt_notify_add(struct mount *m) +{ + /* Optimize the case where there are no watches */ + if ((m->mnt_ns && m->mnt_ns->n_fsnotify_marks) || + (m->prev_ns && m->prev_ns->n_fsnotify_marks)) + list_add_tail(&m->to_notify, ¬ify_list); + else + m->prev_ns = m->mnt_ns; +} +#else +static inline void mnt_notify_add(struct mount *m) +{ +} +#endif + +static inline struct mount *topmost_overmount(struct mount *m) +{ + while (m->overmount) + m = m->overmount; + return m; +} + +static inline bool __test_write_hold(struct mount * __aligned(1) *val) +{ + return (unsigned long)val & WRITE_HOLD; +} + +static inline bool test_write_hold(const struct mount *m) +{ + return __test_write_hold(m->mnt_pprev_for_sb); +} + +static inline void set_write_hold(struct mount *m) +{ + m->mnt_pprev_for_sb = (void *)((unsigned long)m->mnt_pprev_for_sb + | WRITE_HOLD); +} + +static inline void clear_write_hold(struct mount *m) +{ + m->mnt_pprev_for_sb = (void *)((unsigned long)m->mnt_pprev_for_sb + & ~WRITE_HOLD); +} + +struct mnt_namespace *mnt_ns_from_dentry(struct dentry *dentry); diff --git a/fs/mpage.c b/fs/mpage.c new file mode 100644 index 000000000..f02d8ebd7 --- /dev/null +++ b/fs/mpage.c @@ -0,0 +1,693 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/mpage.c + * + * Copyright (C) 2002, Linus Torvalds. + * + * Contains functions related to preparing and submitting BIOs which contain + * multiple pagecache pages. + * + * 15May2002 Andrew Morton + * Initial version + * 27Jun2002 axboe@suse.de + * use bio_add_page() to build bio's just the right size + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +/* + * I/O completion handler for multipage BIOs. + * + * The mpage code never puts partial pages into a BIO (except for end-of-file). + * If a page does not map to a contiguous run of blocks then it simply falls + * back to block_read_full_folio(). + * + * Why is this? If a page's completion depends on a number of different BIOs + * which can complete in any order (or at the same time) then determining the + * status of that page is hard. See end_buffer_async_read() for the details. + * There is no point in duplicating all that complexity. + */ +static void mpage_read_end_io(struct bio *bio) +{ + struct folio_iter fi; + int err = blk_status_to_errno(bio->bi_status); + + bio_for_each_folio_all(fi, bio) + folio_end_read(fi.folio, err == 0); + + bio_put(bio); +} + +static void mpage_write_end_io(struct bio *bio) +{ + struct folio_iter fi; + int err = blk_status_to_errno(bio->bi_status); + + bio_for_each_folio_all(fi, bio) { + if (err) + mapping_set_error(fi.folio->mapping, err); + folio_end_writeback(fi.folio); + } + + bio_put(bio); +} + +static struct bio *mpage_bio_submit_read(struct bio *bio) +{ + bio->bi_end_io = mpage_read_end_io; + guard_bio_eod(bio); + submit_bio(bio); + return NULL; +} + +static struct bio *mpage_bio_submit_write(struct bio *bio) +{ + bio->bi_end_io = mpage_write_end_io; + guard_bio_eod(bio); + submit_bio(bio); + return NULL; +} + +/* + * support function for mpage_readahead. The fs supplied get_block might + * return an up to date buffer. This is used to map that buffer into + * the page, which allows read_folio to avoid triggering a duplicate call + * to get_block. + * + * The idea is to avoid adding buffers to pages that don't already have + * them. So when the buffer is up to date and the page size == block size, + * this marks the page up to date instead of adding new buffers. + */ +static void map_buffer_to_folio(struct folio *folio, struct buffer_head *bh, + int page_block) +{ + struct inode *inode = folio->mapping->host; + struct buffer_head *page_bh, *head; + int block = 0; + + head = folio_buffers(folio); + if (!head) { + /* + * don't make any buffers if there is only one buffer on + * the folio and the folio just needs to be set up to date + */ + if (inode->i_blkbits == folio_shift(folio) && + buffer_uptodate(bh)) { + folio_mark_uptodate(folio); + return; + } + head = create_empty_buffers(folio, i_blocksize(inode), 0); + } + + page_bh = head; + do { + if (block == page_block) { + page_bh->b_state = bh->b_state; + page_bh->b_bdev = bh->b_bdev; + page_bh->b_blocknr = bh->b_blocknr; + break; + } + page_bh = page_bh->b_this_page; + block++; + } while (page_bh != head); +} + +struct mpage_readpage_args { + struct bio *bio; + struct folio *folio; + unsigned int nr_pages; + bool is_readahead; + sector_t last_block_in_bio; + struct buffer_head map_bh; + unsigned long first_logical_block; + get_block_t *get_block; +}; + +/* + * This is the worker routine which does all the work of mapping the disk + * blocks and constructs largest possible bios, submits them for IO if the + * blocks are not contiguous on the disk. + * + * We pass a buffer_head back and forth and use its buffer_mapped() flag to + * represent the validity of its disk mapping and to decide when to do the next + * get_block() call. + */ +static void do_mpage_readpage(struct mpage_readpage_args *args) +{ + struct folio *folio = args->folio; + struct inode *inode = folio->mapping->host; + const unsigned blkbits = inode->i_blkbits; + const unsigned blocks_per_folio = folio_size(folio) >> blkbits; + const unsigned blocksize = 1 << blkbits; + struct buffer_head *map_bh = &args->map_bh; + sector_t block_in_file; + sector_t last_block; + sector_t last_block_in_file; + sector_t first_block; + unsigned page_block; + unsigned first_hole = blocks_per_folio; + struct block_device *bdev = NULL; + int length; + int fully_mapped = 1; + blk_opf_t opf = REQ_OP_READ; + unsigned nblocks; + unsigned relative_block; + gfp_t gfp = mapping_gfp_constraint(folio->mapping, GFP_KERNEL); + + if (args->is_readahead) { + opf |= REQ_RAHEAD; + gfp |= __GFP_NORETRY | __GFP_NOWARN; + } + + if (folio_buffers(folio)) + goto confused; + + block_in_file = folio_pos(folio) >> blkbits; + last_block = block_in_file + ((args->nr_pages * PAGE_SIZE) >> blkbits); + last_block_in_file = (i_size_read(inode) + blocksize - 1) >> blkbits; + if (last_block > last_block_in_file) + last_block = last_block_in_file; + page_block = 0; + + /* + * Map blocks using the result from the previous get_blocks call first. + */ + nblocks = map_bh->b_size >> blkbits; + if (buffer_mapped(map_bh) && + block_in_file > args->first_logical_block && + block_in_file < (args->first_logical_block + nblocks)) { + unsigned map_offset = block_in_file - args->first_logical_block; + unsigned last = nblocks - map_offset; + + first_block = map_bh->b_blocknr + map_offset; + for (relative_block = 0; ; relative_block++) { + if (relative_block == last) { + clear_buffer_mapped(map_bh); + break; + } + if (page_block == blocks_per_folio) + break; + page_block++; + block_in_file++; + } + bdev = map_bh->b_bdev; + } + + /* + * Then do more get_blocks calls until we are done with this folio. + */ + map_bh->b_folio = folio; + while (page_block < blocks_per_folio) { + map_bh->b_state = 0; + map_bh->b_size = 0; + + if (block_in_file < last_block) { + map_bh->b_size = (last_block-block_in_file) << blkbits; + if (args->get_block(inode, block_in_file, map_bh, 0)) + goto confused; + args->first_logical_block = block_in_file; + } + + if (!buffer_mapped(map_bh)) { + fully_mapped = 0; + if (first_hole == blocks_per_folio) + first_hole = page_block; + page_block++; + block_in_file++; + continue; + } + + /* some filesystems will copy data into the page during + * the get_block call, in which case we don't want to + * read it again. map_buffer_to_folio copies the data + * we just collected from get_block into the folio's buffers + * so read_folio doesn't have to repeat the get_block call + */ + if (buffer_uptodate(map_bh)) { + map_buffer_to_folio(folio, map_bh, page_block); + goto confused; + } + + if (first_hole != blocks_per_folio) + goto confused; /* hole -> non-hole */ + + /* Contiguous blocks? */ + if (!page_block) + first_block = map_bh->b_blocknr; + else if (first_block + page_block != map_bh->b_blocknr) + goto confused; + nblocks = map_bh->b_size >> blkbits; + for (relative_block = 0; ; relative_block++) { + if (relative_block == nblocks) { + clear_buffer_mapped(map_bh); + break; + } else if (page_block == blocks_per_folio) + break; + page_block++; + block_in_file++; + } + bdev = map_bh->b_bdev; + } + + if (first_hole != blocks_per_folio) { + folio_zero_segment(folio, first_hole << blkbits, folio_size(folio)); + if (first_hole == 0) { + folio_mark_uptodate(folio); + folio_unlock(folio); + goto out; + } + } else if (fully_mapped) { + folio_set_mappedtodisk(folio); + } + + /* + * This folio will go to BIO. Do we need to send this BIO off first? + */ + if (args->bio && (args->last_block_in_bio != first_block - 1)) + args->bio = mpage_bio_submit_read(args->bio); + +alloc_new: + if (args->bio == NULL) { + args->bio = bio_alloc(bdev, bio_max_segs(args->nr_pages), opf, + gfp); + if (args->bio == NULL) + goto confused; + args->bio->bi_iter.bi_sector = first_block << (blkbits - 9); + } + + length = first_hole << blkbits; + if (!bio_add_folio(args->bio, folio, length, 0)) { + args->bio = mpage_bio_submit_read(args->bio); + goto alloc_new; + } + + relative_block = block_in_file - args->first_logical_block; + nblocks = map_bh->b_size >> blkbits; + if ((buffer_boundary(map_bh) && relative_block == nblocks) || + (first_hole != blocks_per_folio)) + args->bio = mpage_bio_submit_read(args->bio); + else + args->last_block_in_bio = first_block + blocks_per_folio - 1; +out: + return; + +confused: + if (args->bio) + args->bio = mpage_bio_submit_read(args->bio); + if (!folio_test_uptodate(folio)) + block_read_full_folio(folio, args->get_block); + else + folio_unlock(folio); + goto out; +} + +/** + * mpage_readahead - start reads against pages + * @rac: Describes which pages to read. + * @get_block: The filesystem's block mapper function. + * + * This function walks the pages and the blocks within each page, building and + * emitting large BIOs. + * + * If anything unusual happens, such as: + * + * - encountering a page which has buffers + * - encountering a page which has a non-hole after a hole + * - encountering a page with non-contiguous blocks + * + * then this code just gives up and calls the buffer_head-based read function. + * It does handle a page which has holes at the end - that is a common case: + * the end-of-file on blocksize < PAGE_SIZE setups. + * + * BH_Boundary explanation: + * + * There is a problem. The mpage read code assembles several pages, gets all + * their disk mappings, and then submits them all. That's fine, but obtaining + * the disk mappings may require I/O. Reads of indirect blocks, for example. + * + * So an mpage read of the first 16 blocks of an ext2 file will cause I/O to be + * submitted in the following order: + * + * 12 0 1 2 3 4 5 6 7 8 9 10 11 13 14 15 16 + * + * because the indirect block has to be read to get the mappings of blocks + * 13,14,15,16. Obviously, this impacts performance. + * + * So what we do it to allow the filesystem's get_block() function to set + * BH_Boundary when it maps block 11. BH_Boundary says: mapping of the block + * after this one will require I/O against a block which is probably close to + * this one. So you should push what I/O you have currently accumulated. + * + * This all causes the disk requests to be issued in the correct order. + */ +void mpage_readahead(struct readahead_control *rac, get_block_t get_block) +{ + struct folio *folio; + struct mpage_readpage_args args = { + .get_block = get_block, + .is_readahead = true, + }; + + while ((folio = readahead_folio(rac))) { + prefetchw(&folio->flags); + args.folio = folio; + args.nr_pages = readahead_count(rac); + do_mpage_readpage(&args); + /* + * If read ahead failed synchronously, it may cause by removed + * device, or some filesystem metadata error. + */ + if (!folio_test_locked(folio) && !folio_test_uptodate(folio)) + break; + } + if (args.bio) + mpage_bio_submit_read(args.bio); +} +EXPORT_SYMBOL(mpage_readahead); + +/* + * This isn't called much at all + */ +int mpage_read_folio(struct folio *folio, get_block_t get_block) +{ + struct mpage_readpage_args args = { + .folio = folio, + .nr_pages = folio_nr_pages(folio), + .get_block = get_block, + }; + + do_mpage_readpage(&args); + if (args.bio) + mpage_bio_submit_read(args.bio); + return 0; +} +EXPORT_SYMBOL(mpage_read_folio); + +/* + * Writing is not so simple. + * + * If the page has buffers then they will be used for obtaining the disk + * mapping. We only support pages which are fully mapped-and-dirty, with a + * special case for pages which are unmapped at the end: end-of-file. + * + * If the page has no buffers (preferred) then the page is mapped here. + * + * If all blocks are found to be contiguous then the page can go into the + * BIO. Otherwise fall back to the mapping's writepage(). + * + * FIXME: This code wants an estimate of how many pages are still to be + * written, so it can intelligently allocate a suitably-sized BIO. For now, + * just allocate full-size (16-page) BIOs. + */ + +struct mpage_data { + struct bio *bio; + sector_t last_block_in_bio; + get_block_t *get_block; +}; + +/* + * We have our BIO, so we can now mark the buffers clean. Make + * sure to only clean buffers which we know we'll be writing. + */ +static void clean_buffers(struct folio *folio, unsigned first_unmapped) +{ + unsigned buffer_counter = 0; + struct buffer_head *bh, *head = folio_buffers(folio); + + if (!head) + return; + bh = head; + + do { + if (buffer_counter++ == first_unmapped) + break; + clear_buffer_dirty(bh); + bh = bh->b_this_page; + } while (bh != head); + + /* + * we cannot drop the bh if the page is not uptodate or a concurrent + * read_folio would fail to serialize with the bh and it would read from + * disk before we reach the platter. + */ + if (buffer_heads_over_limit && folio_test_uptodate(folio)) + try_to_free_buffers(folio); +} + +static int mpage_write_folio(struct writeback_control *wbc, struct folio *folio, + struct mpage_data *mpd) +{ + struct bio *bio = mpd->bio; + struct address_space *mapping = folio->mapping; + struct inode *inode = mapping->host; + const unsigned blkbits = inode->i_blkbits; + const unsigned blocks_per_folio = folio_size(folio) >> blkbits; + sector_t last_block; + sector_t block_in_file; + sector_t first_block; + unsigned page_block; + unsigned first_unmapped = blocks_per_folio; + struct block_device *bdev = NULL; + int boundary = 0; + sector_t boundary_block = 0; + struct block_device *boundary_bdev = NULL; + size_t length; + struct buffer_head map_bh; + loff_t i_size = i_size_read(inode); + int ret = 0; + struct buffer_head *head = folio_buffers(folio); + + if (head) { + struct buffer_head *bh = head; + + /* If they're all mapped and dirty, do it */ + page_block = 0; + do { + BUG_ON(buffer_locked(bh)); + if (!buffer_mapped(bh)) { + /* + * unmapped dirty buffers are created by + * block_dirty_folio -> mmapped data + */ + if (buffer_dirty(bh)) + goto confused; + if (first_unmapped == blocks_per_folio) + first_unmapped = page_block; + continue; + } + + if (first_unmapped != blocks_per_folio) + goto confused; /* hole -> non-hole */ + + if (!buffer_dirty(bh) || !buffer_uptodate(bh)) + goto confused; + if (page_block) { + if (bh->b_blocknr != first_block + page_block) + goto confused; + } else { + first_block = bh->b_blocknr; + } + page_block++; + boundary = buffer_boundary(bh); + if (boundary) { + boundary_block = bh->b_blocknr; + boundary_bdev = bh->b_bdev; + } + bdev = bh->b_bdev; + } while ((bh = bh->b_this_page) != head); + + if (first_unmapped) + goto page_is_mapped; + + /* + * Page has buffers, but they are all unmapped. The page was + * created by pagein or read over a hole which was handled by + * block_read_full_folio(). If this address_space is also + * using mpage_readahead then this can rarely happen. + */ + goto confused; + } + + /* + * The page has no buffers: map it to disk + */ + BUG_ON(!folio_test_uptodate(folio)); + block_in_file = folio_pos(folio) >> blkbits; + /* + * Whole page beyond EOF? Skip allocating blocks to avoid leaking + * space. + */ + if (block_in_file >= (i_size + (1 << blkbits) - 1) >> blkbits) + goto page_is_mapped; + last_block = (i_size - 1) >> blkbits; + map_bh.b_folio = folio; + for (page_block = 0; page_block < blocks_per_folio; ) { + + map_bh.b_state = 0; + map_bh.b_size = 1 << blkbits; + if (mpd->get_block(inode, block_in_file, &map_bh, 1)) + goto confused; + if (!buffer_mapped(&map_bh)) + goto confused; + if (buffer_new(&map_bh)) + clean_bdev_bh_alias(&map_bh); + if (buffer_boundary(&map_bh)) { + boundary_block = map_bh.b_blocknr; + boundary_bdev = map_bh.b_bdev; + } + if (page_block) { + if (map_bh.b_blocknr != first_block + page_block) + goto confused; + } else { + first_block = map_bh.b_blocknr; + } + page_block++; + boundary = buffer_boundary(&map_bh); + bdev = map_bh.b_bdev; + if (block_in_file == last_block) + break; + block_in_file++; + } + BUG_ON(page_block == 0); + + first_unmapped = page_block; + +page_is_mapped: + /* Don't bother writing beyond EOF, truncate will discard the folio */ + if (folio_pos(folio) >= i_size) + goto confused; + length = folio_size(folio); + if (folio_pos(folio) + length > i_size) { + /* + * The page straddles i_size. It must be zeroed out on each + * and every writepage invocation because it may be mmapped. + * "A file is mapped in multiples of the page size. For a file + * that is not a multiple of the page size, the remaining memory + * is zeroed when mapped, and writes to that region are not + * written out to the file." + */ + length = i_size - folio_pos(folio); + folio_zero_segment(folio, length, folio_size(folio)); + } + + /* + * This page will go to BIO. Do we need to send this BIO off first? + */ + if (bio && mpd->last_block_in_bio != first_block - 1) + bio = mpage_bio_submit_write(bio); + +alloc_new: + if (bio == NULL) { + bio = bio_alloc(bdev, BIO_MAX_VECS, + REQ_OP_WRITE | wbc_to_write_flags(wbc), + GFP_NOFS); + bio->bi_iter.bi_sector = first_block << (blkbits - 9); + wbc_init_bio(wbc, bio); + bio->bi_write_hint = inode->i_write_hint; + } + + /* + * Must try to add the page before marking the buffer clean or + * the confused fail path above (OOM) will be very confused when + * it finds all bh marked clean (i.e. it will not write anything) + */ + wbc_account_cgroup_owner(wbc, folio, folio_size(folio)); + length = first_unmapped << blkbits; + if (!bio_add_folio(bio, folio, length, 0)) { + bio = mpage_bio_submit_write(bio); + goto alloc_new; + } + + clean_buffers(folio, first_unmapped); + + BUG_ON(folio_test_writeback(folio)); + folio_start_writeback(folio); + folio_unlock(folio); + if (boundary || (first_unmapped != blocks_per_folio)) { + bio = mpage_bio_submit_write(bio); + if (boundary_block) { + write_boundary_block(boundary_bdev, + boundary_block, 1 << blkbits); + } + } else { + mpd->last_block_in_bio = first_block + blocks_per_folio - 1; + } + goto out; + +confused: + if (bio) + bio = mpage_bio_submit_write(bio); + + /* + * The caller has a ref on the inode, so *mapping is stable + */ + ret = block_write_full_folio(folio, wbc, mpd->get_block); + mapping_set_error(mapping, ret); +out: + mpd->bio = bio; + return ret; +} + +/** + * __mpage_writepages - walk the list of dirty pages of the given address space + * & writepage() all of them + * @mapping: address space structure to write + * @wbc: subtract the number of written pages from *@wbc->nr_to_write + * @get_block: the filesystem's block mapper function. + * @write_folio: handler to call for each folio before calling + * mpage_write_folio() + * + * This is a library function, which implements the writepages() + * address_space_operation. It calls @write_folio handler for each folio. If + * the handler returns value > 0, it calls mpage_write_folio() to do the + * folio writeback. + */ +int +__mpage_writepages(struct address_space *mapping, + struct writeback_control *wbc, get_block_t get_block, + int (*write_folio)(struct folio *folio, + struct writeback_control *wbc)) +{ + struct mpage_data mpd = { + .get_block = get_block, + }; + struct folio *folio = NULL; + struct blk_plug plug; + int error; + + blk_start_plug(&plug); + while ((folio = writeback_iter(mapping, wbc, folio, &error))) { + if (write_folio) { + error = write_folio(folio, wbc); + /* + * == 0 means folio is handled, < 0 means error. In + * both cases hand back control to writeback_iter() + */ + if (error <= 0) + continue; + /* Let mpage_write_folio() handle the folio. */ + } + error = mpage_write_folio(wbc, folio, &mpd); + } + if (mpd.bio) + mpage_bio_submit_write(mpd.bio); + blk_finish_plug(&plug); + return error; +} +EXPORT_SYMBOL(__mpage_writepages); diff --git a/fs/namei.c b/fs/namei.c new file mode 100644 index 000000000..20a6534ea --- /dev/null +++ b/fs/namei.c @@ -0,0 +1,6581 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/namei.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + */ + +/* + * Some corrections by tytso. + */ + +/* [Feb 1997 T. Schoebel-Theuer] Complete rewrite of the pathname + * lookup logic. + */ +/* [Feb-Apr 2000, AV] Rewrite to the new namespace architecture. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "internal.h" +#include "mount.h" + +/* [Feb-1997 T. Schoebel-Theuer] + * Fundamental changes in the pathname lookup mechanisms (namei) + * were necessary because of omirr. The reason is that omirr needs + * to know the _real_ pathname, not the user-supplied one, in case + * of symlinks (and also when transname replacements occur). + * + * The new code replaces the old recursive symlink resolution with + * an iterative one (in case of non-nested symlink chains). It does + * this with calls to _follow_link(). + * As a side effect, dir_namei(), _namei() and follow_link() are now + * replaced with a single function lookup_dentry() that can handle all + * the special cases of the former code. + * + * With the new dcache, the pathname is stored at each inode, at least as + * long as the refcount of the inode is positive. As a side effect, the + * size of the dcache depends on the inode cache and thus is dynamic. + * + * [29-Apr-1998 C. Scott Ananian] Updated above description of symlink + * resolution to correspond with current state of the code. + * + * Note that the symlink resolution is not *completely* iterative. + * There is still a significant amount of tail- and mid- recursion in + * the algorithm. Also, note that _readlink() is not used in + * lookup_dentry(): lookup_dentry() on the result of _readlink() + * may return different results than _follow_link(). Many virtual + * filesystems (including /proc) exhibit this behavior. + */ + +/* [24-Feb-97 T. Schoebel-Theuer] Side effects caused by new implementation: + * New symlink semantics: when open() is called with flags O_CREAT | O_EXCL + * and the name already exists in form of a symlink, try to create the new + * name indicated by the symlink. The old code always complained that the + * name already exists, due to not following the symlink even if its target + * is nonexistent. The new semantics affects also mknod() and link() when + * the name is a symlink pointing to a non-existent name. + * + * I don't know which semantics is the right one, since I have no access + * to standards. But I found by trial that HP-UX 9.0 has the full "new" + * semantics implemented, while SunOS 4.1.1 and Solaris (SunOS 5.4) have the + * "old" one. Personally, I think the new semantics is much more logical. + * Note that "ln old new" where "new" is a symlink pointing to a non-existing + * file does succeed in both HP-UX and SunOs, but not in Solaris + * and in the old Linux semantics. + */ + +/* [16-Dec-97 Kevin Buhr] For security reasons, we change some symlink + * semantics. See the comments in "open_namei" and "do_link" below. + * + * [10-Sep-98 Alan Modra] Another symlink change. + */ + +/* [Feb-Apr 2000 AV] Complete rewrite. Rules for symlinks: + * inside the path - always follow. + * in the last component in creation/removal/renaming - never follow. + * if LOOKUP_FOLLOW passed - follow. + * if the pathname has trailing slashes - follow. + * otherwise - don't follow. + * (applied in that order). + * + * [Jun 2000 AV] Inconsistent behaviour of open() in case if flags==O_CREAT + * restored for 2.4. This is the last surviving part of old 4.2BSD bug. + * During the 2.4 we need to fix the userland stuff depending on it - + * hopefully we will be able to get rid of that wart in 2.5. So far only + * XEmacs seems to be relying on it... + */ +/* + * [Sep 2001 AV] Single-semaphore locking scheme (kudos to David Holland) + * implemented. Let's see if raised priority of ->s_vfs_rename_mutex gives + * any extra contention... + */ + +/* In order to reduce some races, while at the same time doing additional + * checking and hopefully speeding things up, we copy filenames to the + * kernel data space before using them.. + * + * POSIX.1 2.4: an empty pathname is invalid (ENOENT). + * PATH_MAX includes the nul terminator --RR. + */ + +/* SLAB cache for struct filename instances */ +static struct kmem_cache *__names_cache __ro_after_init; +#define names_cache runtime_const_ptr(__names_cache) + +/* + * Type of the last component on LOOKUP_PARENT + */ +enum last_type {LAST_NORM, LAST_ROOT, LAST_DOT, LAST_DOTDOT}; + +void __init filename_init(void) +{ + __names_cache = kmem_cache_create_usercopy("names_cache", sizeof(struct filename), 0, + SLAB_HWCACHE_ALIGN|SLAB_PANIC, offsetof(struct filename, iname), + EMBEDDED_NAME_MAX, NULL); + runtime_const_init(ptr, __names_cache); +} + +static inline struct filename *alloc_filename(void) +{ + return kmem_cache_alloc(names_cache, GFP_KERNEL); +} + +static inline void free_filename(struct filename *p) +{ + kmem_cache_free(names_cache, p); +} + +static inline void initname(struct filename *name) +{ + name->aname = NULL; + name->refcnt = 1; +} + +static int getname_long(struct filename *name, const char __user *filename) +{ + int len; + char *p __free(kfree) = kmalloc(PATH_MAX, GFP_KERNEL); + if (unlikely(!p)) + return -ENOMEM; + + memcpy(p, &name->iname, EMBEDDED_NAME_MAX); + len = strncpy_from_user(p + EMBEDDED_NAME_MAX, + filename + EMBEDDED_NAME_MAX, + PATH_MAX - EMBEDDED_NAME_MAX); + if (unlikely(len < 0)) + return len; + if (unlikely(len == PATH_MAX - EMBEDDED_NAME_MAX)) + return -ENAMETOOLONG; + name->name = no_free_ptr(p); + return 0; +} + +static struct filename * +do_getname(const char __user *filename, int flags, bool incomplete) +{ + struct filename *result; + char *kname; + int len; + + result = alloc_filename(); + if (unlikely(!result)) + return ERR_PTR(-ENOMEM); + + /* + * First, try to embed the struct filename inside the names_cache + * allocation + */ + kname = (char *)result->iname; + result->name = kname; + + len = strncpy_from_user(kname, filename, EMBEDDED_NAME_MAX); + /* + * Handle both empty path and copy failure in one go. + */ + if (unlikely(len <= 0)) { + /* The empty path is special. */ + if (!len && !(flags & LOOKUP_EMPTY)) + len = -ENOENT; + } + + /* + * Uh-oh. We have a name that's approaching PATH_MAX. Allocate a + * separate struct filename so we can dedicate the entire + * names_cache allocation for the pathname, and re-do the copy from + * userland. + */ + if (unlikely(len == EMBEDDED_NAME_MAX)) + len = getname_long(result, filename); + if (unlikely(len < 0)) { + free_filename(result); + return ERR_PTR(len); + } + + initname(result); + if (likely(!incomplete)) + audit_getname(result); + return result; +} + +struct filename * +getname_flags(const char __user *filename, int flags) +{ + return do_getname(filename, flags, false); +} + +struct filename *getname_uflags(const char __user *filename, int uflags) +{ + int flags = (uflags & AT_EMPTY_PATH) ? LOOKUP_EMPTY : 0; + + return getname_flags(filename, flags); +} + +struct filename *__getname_maybe_null(const char __user *pathname) +{ + char c; + + /* try to save on allocations; loss on um, though */ + if (get_user(c, pathname)) + return ERR_PTR(-EFAULT); + if (!c) + return NULL; + + CLASS(filename_flags, name)(pathname, LOOKUP_EMPTY); + /* empty pathname translates to NULL */ + if (!IS_ERR(name) && !(name->name[0])) + return NULL; + return no_free_ptr(name); +} + +static struct filename *do_getname_kernel(const char *filename, bool incomplete) +{ + struct filename *result; + int len = strlen(filename) + 1; + char *p; + + if (unlikely(len > PATH_MAX)) + return ERR_PTR(-ENAMETOOLONG); + + result = alloc_filename(); + if (unlikely(!result)) + return ERR_PTR(-ENOMEM); + + if (len <= EMBEDDED_NAME_MAX) { + p = (char *)result->iname; + memcpy(p, filename, len); + } else { + p = kmemdup(filename, len, GFP_KERNEL); + if (unlikely(!p)) { + free_filename(result); + return ERR_PTR(-ENOMEM); + } + } + result->name = p; + initname(result); + if (likely(!incomplete)) + audit_getname(result); + return result; +} + +struct filename *getname_kernel(const char *filename) +{ + return do_getname_kernel(filename, false); +} +EXPORT_SYMBOL(getname_kernel); + +void putname(struct filename *name) +{ + int refcnt; + + if (IS_ERR_OR_NULL(name)) + return; + + refcnt = name->refcnt; + if (unlikely(refcnt != 1)) { + if (WARN_ON_ONCE(!refcnt)) + return; + + name->refcnt--; + return; + } + + if (unlikely(name->name != name->iname)) + kfree(name->name); + free_filename(name); +} +EXPORT_SYMBOL(putname); + +static inline int __delayed_getname(struct delayed_filename *v, + const char __user *string, int flags) +{ + v->__incomplete_filename = do_getname(string, flags, true); + return PTR_ERR_OR_ZERO(v->__incomplete_filename); +} + +int delayed_getname(struct delayed_filename *v, const char __user *string) +{ + return __delayed_getname(v, string, 0); +} + +int delayed_getname_uflags(struct delayed_filename *v, const char __user *string, + int uflags) +{ + int flags = (uflags & AT_EMPTY_PATH) ? LOOKUP_EMPTY : 0; + return __delayed_getname(v, string, flags); +} + +int putname_to_delayed(struct delayed_filename *v, struct filename *name) +{ + if (likely(name->refcnt == 1)) { + v->__incomplete_filename = name; + return 0; + } + name->refcnt--; + v->__incomplete_filename = do_getname_kernel(name->name, true); + return PTR_ERR_OR_ZERO(v->__incomplete_filename); +} + +void dismiss_delayed_filename(struct delayed_filename *v) +{ + putname(no_free_ptr(v->__incomplete_filename)); +} + +struct filename *complete_getname(struct delayed_filename *v) +{ + struct filename *res = no_free_ptr(v->__incomplete_filename); + if (!IS_ERR(res)) + audit_getname(res); + return res; +} + +/** + * check_acl - perform ACL permission checking + * @idmap: idmap of the mount the inode was found from + * @inode: inode to check permissions on + * @mask: right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC ...) + * + * This function performs the ACL permission checking. Since this function + * retrieve POSIX acls it needs to know whether it is called from a blocking or + * non-blocking context and thus cares about the MAY_NOT_BLOCK bit. + * + * If the inode has been found through an idmapped mount the idmap of + * the vfsmount must be passed through @idmap. This function will then take + * care to map the inode according to @idmap before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply pass @nop_mnt_idmap. + */ +static int check_acl(struct mnt_idmap *idmap, + struct inode *inode, int mask) +{ +#ifdef CONFIG_FS_POSIX_ACL + struct posix_acl *acl; + + if (mask & MAY_NOT_BLOCK) { + acl = get_cached_acl_rcu(inode, ACL_TYPE_ACCESS); + if (!acl) + return -EAGAIN; + /* no ->get_inode_acl() calls in RCU mode... */ + if (is_uncached_acl(acl)) + return -ECHILD; + return posix_acl_permission(idmap, inode, acl, mask); + } + + acl = get_inode_acl(inode, ACL_TYPE_ACCESS); + if (IS_ERR(acl)) + return PTR_ERR(acl); + if (acl) { + int error = posix_acl_permission(idmap, inode, acl, mask); + posix_acl_release(acl); + return error; + } +#endif + + return -EAGAIN; +} + +/* + * Very quick optimistic "we know we have no ACL's" check. + * + * Note that this is purely for ACL_TYPE_ACCESS, and purely + * for the "we have cached that there are no ACLs" case. + * + * If this returns true, we know there are no ACLs. But if + * it returns false, we might still not have ACLs (it could + * be the is_uncached_acl() case). + */ +static inline bool no_acl_inode(struct inode *inode) +{ +#ifdef CONFIG_FS_POSIX_ACL + return likely(!READ_ONCE(inode->i_acl)); +#else + return true; +#endif +} + +/** + * acl_permission_check - perform basic UNIX permission checking + * @idmap: idmap of the mount the inode was found from + * @inode: inode to check permissions on + * @mask: right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC ...) + * + * This function performs the basic UNIX permission checking. Since this + * function may retrieve POSIX acls it needs to know whether it is called from a + * blocking or non-blocking context and thus cares about the MAY_NOT_BLOCK bit. + * + * If the inode has been found through an idmapped mount the idmap of + * the vfsmount must be passed through @idmap. This function will then take + * care to map the inode according to @idmap before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply pass @nop_mnt_idmap. + */ +static int acl_permission_check(struct mnt_idmap *idmap, + struct inode *inode, int mask) +{ + unsigned int mode = inode->i_mode; + vfsuid_t vfsuid; + + /* + * Common cheap case: everybody has the requested + * rights, and there are no ACLs to check. No need + * to do any owner/group checks in that case. + * + * - 'mask&7' is the requested permission bit set + * - multiplying by 0111 spreads them out to all of ugo + * - '& ~mode' looks for missing inode permission bits + * - the '!' is for "no missing permissions" + * + * After that, we just need to check that there are no + * ACL's on the inode - do the 'IS_POSIXACL()' check last + * because it will dereference the ->i_sb pointer and we + * want to avoid that if at all possible. + */ + if (!((mask & 7) * 0111 & ~mode)) { + if (no_acl_inode(inode)) + return 0; + if (!IS_POSIXACL(inode)) + return 0; + } + + /* Are we the owner? If so, ACL's don't matter */ + vfsuid = i_uid_into_vfsuid(idmap, inode); + if (likely(vfsuid_eq_kuid(vfsuid, current_fsuid()))) { + mask &= 7; + mode >>= 6; + return (mask & ~mode) ? -EACCES : 0; + } + + /* Do we have ACL's? */ + if (IS_POSIXACL(inode) && (mode & S_IRWXG)) { + int error = check_acl(idmap, inode, mask); + if (error != -EAGAIN) + return error; + } + + /* Only RWX matters for group/other mode bits */ + mask &= 7; + + /* + * Are the group permissions different from + * the other permissions in the bits we care + * about? Need to check group ownership if so. + */ + if (mask & (mode ^ (mode >> 3))) { + vfsgid_t vfsgid = i_gid_into_vfsgid(idmap, inode); + if (vfsgid_in_group_p(vfsgid)) + mode >>= 3; + } + + /* Bits in 'mode' clear that we require? */ + return (mask & ~mode) ? -EACCES : 0; +} + +/** + * generic_permission - check for access rights on a Posix-like filesystem + * @idmap: idmap of the mount the inode was found from + * @inode: inode to check access rights for + * @mask: right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC, + * %MAY_NOT_BLOCK ...) + * + * Used to check for read/write/execute permissions on a file. + * We use "fsuid" for this, letting us set arbitrary permissions + * for filesystem access without changing the "normal" uids which + * are used for other things. + * + * generic_permission is rcu-walk aware. It returns -ECHILD in case an rcu-walk + * request cannot be satisfied (eg. requires blocking or too much complexity). + * It would then be called again in ref-walk mode. + * + * If the inode has been found through an idmapped mount the idmap of + * the vfsmount must be passed through @idmap. This function will then take + * care to map the inode according to @idmap before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply pass @nop_mnt_idmap. + */ +int generic_permission(struct mnt_idmap *idmap, struct inode *inode, + int mask) +{ + int ret; + + /* + * Do the basic permission checks. + */ + ret = acl_permission_check(idmap, inode, mask); + if (ret != -EACCES) + return ret; + + if (S_ISDIR(inode->i_mode)) { + /* DACs are overridable for directories */ + if (!(mask & MAY_WRITE)) + if (capable_wrt_inode_uidgid(idmap, inode, + CAP_DAC_READ_SEARCH)) + return 0; + if (capable_wrt_inode_uidgid(idmap, inode, + CAP_DAC_OVERRIDE)) + return 0; + return -EACCES; + } + + /* + * Searching includes executable on directories, else just read. + */ + mask &= MAY_READ | MAY_WRITE | MAY_EXEC; + if (mask == MAY_READ) + if (capable_wrt_inode_uidgid(idmap, inode, + CAP_DAC_READ_SEARCH)) + return 0; + /* + * Read/write DACs are always overridable. + * Executable DACs are overridable when there is + * at least one exec bit set. + */ + if (!(mask & MAY_EXEC) || (inode->i_mode & S_IXUGO)) + if (capable_wrt_inode_uidgid(idmap, inode, + CAP_DAC_OVERRIDE)) + return 0; + + return -EACCES; +} +EXPORT_SYMBOL(generic_permission); + +/** + * do_inode_permission - UNIX permission checking + * @idmap: idmap of the mount the inode was found from + * @inode: inode to check permissions on + * @mask: right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC ...) + * + * We _really_ want to just do "generic_permission()" without + * even looking at the inode->i_op values. So we keep a cache + * flag in inode->i_opflags, that says "this has not special + * permission function, use the fast case". + */ +static inline int do_inode_permission(struct mnt_idmap *idmap, + struct inode *inode, int mask) +{ + if (unlikely(!(inode->i_opflags & IOP_FASTPERM))) { + if (likely(inode->i_op->permission)) + return inode->i_op->permission(idmap, inode, mask); + + /* This gets set once for the inode lifetime */ + spin_lock(&inode->i_lock); + inode->i_opflags |= IOP_FASTPERM; + spin_unlock(&inode->i_lock); + } + return generic_permission(idmap, inode, mask); +} + +/** + * sb_permission - Check superblock-level permissions + * @sb: Superblock of inode to check permission on + * @inode: Inode to check permission on + * @mask: Right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC) + * + * Separate out file-system wide checks from inode-specific permission checks. + * + * Note: lookup_inode_permission_may_exec() does not call here. If you add + * MAY_EXEC checks, adjust it. + */ +static int sb_permission(struct super_block *sb, struct inode *inode, int mask) +{ + if (mask & MAY_WRITE) { + umode_t mode = inode->i_mode; + + /* Nobody gets write access to a read-only fs. */ + if (sb_rdonly(sb) && (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode))) + return -EROFS; + } + return 0; +} + +/** + * inode_permission - Check for access rights to a given inode + * @idmap: idmap of the mount the inode was found from + * @inode: Inode to check permission on + * @mask: Right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC) + * + * Check for read/write/execute permissions on an inode. We use fs[ug]id for + * this, letting us set arbitrary permissions for filesystem access without + * changing the "normal" UIDs which are used for other things. + * + * When checking for MAY_APPEND, MAY_WRITE must also be set in @mask. + */ +int inode_permission(struct mnt_idmap *idmap, + struct inode *inode, int mask) +{ + int retval; + + retval = sb_permission(inode->i_sb, inode, mask); + if (unlikely(retval)) + return retval; + + if (mask & MAY_WRITE) { + /* + * Nobody gets write access to an immutable file. + */ + if (unlikely(IS_IMMUTABLE(inode))) + return -EPERM; + + /* + * Updating mtime will likely cause i_uid and i_gid to be + * written back improperly if their true value is unknown + * to the vfs. + */ + if (unlikely(HAS_UNMAPPED_ID(idmap, inode))) + return -EACCES; + } + + retval = do_inode_permission(idmap, inode, mask); + if (unlikely(retval)) + return retval; + + retval = devcgroup_inode_permission(inode, mask); + if (unlikely(retval)) + return retval; + + return security_inode_permission(inode, mask); +} +EXPORT_SYMBOL(inode_permission); + +/* + * lookup_inode_permission_may_exec - Check traversal right for given inode + * + * This is a special case routine for may_lookup() making assumptions specific + * to path traversal. Use inode_permission() if you are doing something else. + * + * Work is shaved off compared to inode_permission() as follows: + * - we know for a fact there is no MAY_WRITE to worry about + * - it is an invariant the inode is a directory + * + * Since majority of real-world traversal happens on inodes which grant it for + * everyone, we check it upfront and only resort to more expensive work if it + * fails. + * + * Filesystems which have their own ->permission hook and consequently miss out + * on IOP_FASTPERM can still get the optimization if they set IOP_FASTPERM_MAY_EXEC + * on their directory inodes. + */ +static __always_inline int lookup_inode_permission_may_exec(struct mnt_idmap *idmap, + struct inode *inode, int mask) +{ + /* Lookup already checked this to return -ENOTDIR */ + VFS_BUG_ON_INODE(!S_ISDIR(inode->i_mode), inode); + VFS_BUG_ON((mask & ~MAY_NOT_BLOCK) != 0); + + mask |= MAY_EXEC; + + if (unlikely(!(inode->i_opflags & (IOP_FASTPERM | IOP_FASTPERM_MAY_EXEC)))) + return inode_permission(idmap, inode, mask); + + if (unlikely(((inode->i_mode & 0111) != 0111) || !no_acl_inode(inode))) + return inode_permission(idmap, inode, mask); + + return security_inode_permission(inode, mask); +} + +/** + * path_get - get a reference to a path + * @path: path to get the reference to + * + * Given a path increment the reference count to the dentry and the vfsmount. + */ +void path_get(const struct path *path) +{ + mntget(path->mnt); + dget(path->dentry); +} +EXPORT_SYMBOL(path_get); + +/** + * path_put - put a reference to a path + * @path: path to put the reference to + * + * Given a path decrement the reference count to the dentry and the vfsmount. + */ +void path_put(const struct path *path) +{ + dput(path->dentry); + mntput(path->mnt); +} +EXPORT_SYMBOL(path_put); + +#define EMBEDDED_LEVELS 2 +struct nameidata { + struct path path; + struct qstr last; + struct path root; + struct inode *inode; /* path.dentry.d_inode */ + unsigned int flags, state; + unsigned seq, next_seq, m_seq, r_seq; + enum last_type last_type; + unsigned depth; + int total_link_count; + struct saved { + struct path link; + struct delayed_call done; + const char *name; + unsigned seq; + } *stack, internal[EMBEDDED_LEVELS]; + struct filename *name; + const char *pathname; + struct nameidata *saved; + unsigned root_seq; + int dfd; + vfsuid_t dir_vfsuid; + umode_t dir_mode; +} __randomize_layout; + +#define ND_ROOT_PRESET 1 +#define ND_ROOT_GRABBED 2 +#define ND_JUMPED 4 + +static void __set_nameidata(struct nameidata *p, int dfd, struct filename *name) +{ + struct nameidata *old = current->nameidata; + p->stack = p->internal; + p->depth = 0; + p->dfd = dfd; + p->name = name; + p->pathname = likely(name) ? name->name : ""; + p->path.mnt = NULL; + p->path.dentry = NULL; + p->total_link_count = old ? old->total_link_count : 0; + p->saved = old; + current->nameidata = p; +} + +static inline void set_nameidata(struct nameidata *p, int dfd, struct filename *name, + const struct path *root) +{ + __set_nameidata(p, dfd, name); + p->state = 0; + if (unlikely(root)) { + p->state = ND_ROOT_PRESET; + p->root = *root; + } +} + +static void restore_nameidata(void) +{ + struct nameidata *now = current->nameidata, *old = now->saved; + + current->nameidata = old; + if (old) + old->total_link_count = now->total_link_count; + if (now->stack != now->internal) + kfree(now->stack); +} + +static bool nd_alloc_stack(struct nameidata *nd) +{ + struct saved *p; + + p= kmalloc_objs(struct saved, MAXSYMLINKS, + nd->flags & LOOKUP_RCU ? GFP_ATOMIC : GFP_KERNEL); + if (unlikely(!p)) + return false; + memcpy(p, nd->internal, sizeof(nd->internal)); + nd->stack = p; + return true; +} + +/** + * path_connected - Verify that a dentry is below mnt.mnt_root + * @mnt: The mountpoint to check. + * @dentry: The dentry to check. + * + * Rename can sometimes move a file or directory outside of a bind + * mount, path_connected allows those cases to be detected. + */ +static bool path_connected(struct vfsmount *mnt, struct dentry *dentry) +{ + struct super_block *sb = mnt->mnt_sb; + + /* Bind mounts can have disconnected paths */ + if (mnt->mnt_root == sb->s_root) + return true; + + return is_subdir(dentry, mnt->mnt_root); +} + +static void drop_links(struct nameidata *nd) +{ + int i = nd->depth; + while (i--) { + struct saved *last = nd->stack + i; + do_delayed_call(&last->done); + clear_delayed_call(&last->done); + } +} + +static void leave_rcu(struct nameidata *nd) +{ + nd->flags &= ~LOOKUP_RCU; + nd->seq = nd->next_seq = 0; + rcu_read_unlock(); +} + +static void terminate_walk(struct nameidata *nd) +{ + if (unlikely(nd->depth)) + drop_links(nd); + if (!(nd->flags & LOOKUP_RCU)) { + int i; + path_put(&nd->path); + for (i = 0; i < nd->depth; i++) + path_put(&nd->stack[i].link); + if (nd->state & ND_ROOT_GRABBED) { + path_put(&nd->root); + nd->state &= ~ND_ROOT_GRABBED; + } + } else { + leave_rcu(nd); + } + nd->depth = 0; + nd->path.mnt = NULL; + nd->path.dentry = NULL; +} + +/* path_put is needed afterwards regardless of success or failure */ +static bool __legitimize_path(struct path *path, unsigned seq, unsigned mseq) +{ + int res = __legitimize_mnt(path->mnt, mseq); + if (unlikely(res)) { + if (res > 0) + path->mnt = NULL; + path->dentry = NULL; + return false; + } + if (unlikely(!lockref_get_not_dead(&path->dentry->d_lockref))) { + path->dentry = NULL; + return false; + } + return !read_seqcount_retry(&path->dentry->d_seq, seq); +} + +static inline bool legitimize_path(struct nameidata *nd, + struct path *path, unsigned seq) +{ + return __legitimize_path(path, seq, nd->m_seq); +} + +static bool legitimize_links(struct nameidata *nd) +{ + int i; + + VFS_BUG_ON(nd->flags & LOOKUP_CACHED); + + for (i = 0; i < nd->depth; i++) { + struct saved *last = nd->stack + i; + if (unlikely(!legitimize_path(nd, &last->link, last->seq))) { + drop_links(nd); + nd->depth = i + 1; + return false; + } + } + return true; +} + +static bool legitimize_root(struct nameidata *nd) +{ + /* Nothing to do if nd->root is zero or is managed by the VFS user. */ + if (!nd->root.mnt || (nd->state & ND_ROOT_PRESET)) + return true; + nd->state |= ND_ROOT_GRABBED; + return legitimize_path(nd, &nd->root, nd->root_seq); +} + +/* + * Path walking has 2 modes, rcu-walk and ref-walk (see + * Documentation/filesystems/path-lookup.txt). In situations when we can't + * continue in RCU mode, we attempt to drop out of rcu-walk mode and grab + * normal reference counts on dentries and vfsmounts to transition to ref-walk + * mode. Refcounts are grabbed at the last known good point before rcu-walk + * got stuck, so ref-walk may continue from there. If this is not successful + * (eg. a seqcount has changed), then failure is returned and it's up to caller + * to restart the path walk from the beginning in ref-walk mode. + */ + +/** + * try_to_unlazy - try to switch to ref-walk mode. + * @nd: nameidata pathwalk data + * Returns: true on success, false on failure + * + * try_to_unlazy attempts to legitimize the current nd->path and nd->root + * for ref-walk mode. + * Must be called from rcu-walk context. + * Nothing should touch nameidata between try_to_unlazy() failure and + * terminate_walk(). + */ +static bool try_to_unlazy(struct nameidata *nd) +{ + struct dentry *parent = nd->path.dentry; + + VFS_BUG_ON(!(nd->flags & LOOKUP_RCU)); + + if (unlikely(nd->flags & LOOKUP_CACHED)) { + drop_links(nd); + nd->depth = 0; + goto out1; + } + if (unlikely(nd->depth && !legitimize_links(nd))) + goto out1; + if (unlikely(!legitimize_path(nd, &nd->path, nd->seq))) + goto out; + if (unlikely(!legitimize_root(nd))) + goto out; + leave_rcu(nd); + BUG_ON(nd->inode != parent->d_inode); + return true; + +out1: + nd->path.mnt = NULL; + nd->path.dentry = NULL; +out: + leave_rcu(nd); + return false; +} + +/** + * try_to_unlazy_next - try to switch to ref-walk mode. + * @nd: nameidata pathwalk data + * @dentry: next dentry to step into + * Returns: true on success, false on failure + * + * Similar to try_to_unlazy(), but here we have the next dentry already + * picked by rcu-walk and want to legitimize that in addition to the current + * nd->path and nd->root for ref-walk mode. Must be called from rcu-walk context. + * Nothing should touch nameidata between try_to_unlazy_next() failure and + * terminate_walk(). + */ +static bool try_to_unlazy_next(struct nameidata *nd, struct dentry *dentry) +{ + int res; + + VFS_BUG_ON(!(nd->flags & LOOKUP_RCU)); + + if (unlikely(nd->flags & LOOKUP_CACHED)) { + drop_links(nd); + nd->depth = 0; + goto out2; + } + if (unlikely(nd->depth && !legitimize_links(nd))) + goto out2; + res = __legitimize_mnt(nd->path.mnt, nd->m_seq); + if (unlikely(res)) { + if (res > 0) + goto out2; + goto out1; + } + if (unlikely(!lockref_get_not_dead(&nd->path.dentry->d_lockref))) + goto out1; + + /* + * We need to move both the parent and the dentry from the RCU domain + * to be properly refcounted. And the sequence number in the dentry + * validates *both* dentry counters, since we checked the sequence + * number of the parent after we got the child sequence number. So we + * know the parent must still be valid if the child sequence number is + */ + if (unlikely(!lockref_get_not_dead(&dentry->d_lockref))) + goto out; + if (read_seqcount_retry(&dentry->d_seq, nd->next_seq)) + goto out_dput; + /* + * Sequence counts matched. Now make sure that the root is + * still valid and get it if required. + */ + if (unlikely(!legitimize_root(nd))) + goto out_dput; + leave_rcu(nd); + return true; + +out2: + nd->path.mnt = NULL; +out1: + nd->path.dentry = NULL; +out: + leave_rcu(nd); + return false; +out_dput: + leave_rcu(nd); + dput(dentry); + return false; +} + +static inline int d_revalidate(struct inode *dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + if (unlikely(dentry->d_flags & DCACHE_OP_REVALIDATE)) + return dentry->d_op->d_revalidate(dir, name, dentry, flags); + else + return 1; +} + +/** + * complete_walk - successful completion of path walk + * @nd: pointer nameidata + * + * If we had been in RCU mode, drop out of it and legitimize nd->path. + * Revalidate the final result, unless we'd already done that during + * the path walk or the filesystem doesn't ask for it. Return 0 on + * success, -error on failure. In case of failure caller does not + * need to drop nd->path. + */ +static int complete_walk(struct nameidata *nd) +{ + struct dentry *dentry = nd->path.dentry; + int status; + + if (nd->flags & LOOKUP_RCU) { + /* + * We don't want to zero nd->root for scoped-lookups or + * externally-managed nd->root. + */ + if (likely(!(nd->state & ND_ROOT_PRESET))) + if (likely(!(nd->flags & LOOKUP_IS_SCOPED))) + nd->root.mnt = NULL; + nd->flags &= ~LOOKUP_CACHED; + if (!try_to_unlazy(nd)) + return -ECHILD; + } + + if (unlikely(nd->flags & LOOKUP_IS_SCOPED)) { + /* + * While the guarantee of LOOKUP_IS_SCOPED is (roughly) "don't + * ever step outside the root during lookup" and should already + * be guaranteed by the rest of namei, we want to avoid a namei + * BUG resulting in userspace being given a path that was not + * scoped within the root at some point during the lookup. + * + * So, do a final sanity-check to make sure that in the + * worst-case scenario (a complete bypass of LOOKUP_IS_SCOPED) + * we won't silently return an fd completely outside of the + * requested root to userspace. + * + * Userspace could move the path outside the root after this + * check, but as discussed elsewhere this is not a concern (the + * resolved file was inside the root at some point). + */ + if (!path_is_under(&nd->path, &nd->root)) + return -EXDEV; + } + + if (likely(!(nd->state & ND_JUMPED))) + return 0; + + if (likely(!(dentry->d_flags & DCACHE_OP_WEAK_REVALIDATE))) + return 0; + + status = dentry->d_op->d_weak_revalidate(dentry, nd->flags); + if (status > 0) + return 0; + + if (!status) + status = -ESTALE; + + return status; +} + +static int set_root(struct nameidata *nd) +{ + struct fs_struct *fs = current->fs; + + /* + * Jumping to the real root in a scoped-lookup is a BUG in namei, but we + * still have to ensure it doesn't happen because it will cause a breakout + * from the dirfd. + */ + if (WARN_ON(nd->flags & LOOKUP_IS_SCOPED)) + return -ENOTRECOVERABLE; + + if (nd->flags & LOOKUP_RCU) { + unsigned seq; + + do { + seq = read_seqbegin(&fs->seq); + nd->root = fs->root; + nd->root_seq = __read_seqcount_begin(&nd->root.dentry->d_seq); + } while (read_seqretry(&fs->seq, seq)); + } else { + get_fs_root(fs, &nd->root); + nd->state |= ND_ROOT_GRABBED; + } + return 0; +} + +static int nd_jump_root(struct nameidata *nd) +{ + if (unlikely(nd->flags & LOOKUP_BENEATH)) + return -EXDEV; + if (unlikely(nd->flags & LOOKUP_NO_XDEV)) { + /* Absolute path arguments to path_init() are allowed. */ + if (nd->path.mnt != NULL && nd->path.mnt != nd->root.mnt) + return -EXDEV; + } + if (!nd->root.mnt) { + int error = set_root(nd); + if (unlikely(error)) + return error; + } + if (nd->flags & LOOKUP_RCU) { + struct dentry *d; + nd->path = nd->root; + d = nd->path.dentry; + nd->inode = d->d_inode; + nd->seq = nd->root_seq; + if (read_seqcount_retry(&d->d_seq, nd->seq)) + return -ECHILD; + } else { + path_put(&nd->path); + nd->path = nd->root; + path_get(&nd->path); + nd->inode = nd->path.dentry->d_inode; + } + nd->state |= ND_JUMPED; + return 0; +} + +/* + * Helper to directly jump to a known parsed path from ->get_link, + * caller must have taken a reference to path beforehand. + */ +int nd_jump_link(const struct path *path) +{ + int error = -ELOOP; + struct nameidata *nd = current->nameidata; + + if (unlikely(nd->flags & LOOKUP_NO_MAGICLINKS)) + goto err; + + error = -EXDEV; + if (unlikely(nd->flags & LOOKUP_NO_XDEV)) { + if (nd->path.mnt != path->mnt) + goto err; + } + /* Not currently safe for scoped-lookups. */ + if (unlikely(nd->flags & LOOKUP_IS_SCOPED)) + goto err; + + path_put(&nd->path); + nd->path = *path; + nd->inode = nd->path.dentry->d_inode; + nd->state |= ND_JUMPED; + return 0; + +err: + path_put(path); + return error; +} + +static inline void put_link(struct nameidata *nd) +{ + struct saved *last = nd->stack + --nd->depth; + do_delayed_call(&last->done); + if (!(nd->flags & LOOKUP_RCU)) + path_put(&last->link); +} + +static int sysctl_protected_symlinks __read_mostly; +static int sysctl_protected_hardlinks __read_mostly; +static int sysctl_protected_fifos __read_mostly; +static int sysctl_protected_regular __read_mostly; + +#ifdef CONFIG_SYSCTL +static const struct ctl_table namei_sysctls[] = { + { + .procname = "protected_symlinks", + .data = &sysctl_protected_symlinks, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "protected_hardlinks", + .data = &sysctl_protected_hardlinks, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, + { + .procname = "protected_fifos", + .data = &sysctl_protected_fifos, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_TWO, + }, + { + .procname = "protected_regular", + .data = &sysctl_protected_regular, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_TWO, + }, +}; + +static int __init init_fs_namei_sysctls(void) +{ + register_sysctl_init("fs", namei_sysctls); + return 0; +} +fs_initcall(init_fs_namei_sysctls); + +#endif /* CONFIG_SYSCTL */ + +/** + * may_follow_link - Check symlink following for unsafe situations + * @nd: nameidata pathwalk data + * @inode: Used for idmapping. + * + * In the case of the sysctl_protected_symlinks sysctl being enabled, + * CAP_DAC_OVERRIDE needs to be specifically ignored if the symlink is + * in a sticky world-writable directory. This is to protect privileged + * processes from failing races against path names that may change out + * from under them by way of other users creating malicious symlinks. + * It will permit symlinks to be followed only when outside a sticky + * world-writable directory, or when the uid of the symlink and follower + * match, or when the directory owner matches the symlink's owner. + * + * Returns 0 if following the symlink is allowed, -ve on error. + */ +static inline int may_follow_link(struct nameidata *nd, const struct inode *inode) +{ + struct mnt_idmap *idmap; + vfsuid_t vfsuid; + + if (!sysctl_protected_symlinks) + return 0; + + idmap = mnt_idmap(nd->path.mnt); + vfsuid = i_uid_into_vfsuid(idmap, inode); + /* Allowed if owner and follower match. */ + if (vfsuid_eq_kuid(vfsuid, current_fsuid())) + return 0; + + /* Allowed if parent directory not sticky and world-writable. */ + if ((nd->dir_mode & (S_ISVTX|S_IWOTH)) != (S_ISVTX|S_IWOTH)) + return 0; + + /* Allowed if parent directory and link owner match. */ + if (vfsuid_valid(nd->dir_vfsuid) && vfsuid_eq(nd->dir_vfsuid, vfsuid)) + return 0; + + if (nd->flags & LOOKUP_RCU) + return -ECHILD; + + audit_inode(nd->name, nd->stack[0].link.dentry, 0); + audit_log_path_denied(AUDIT_ANOM_LINK, "follow_link"); + return -EACCES; +} + +/** + * safe_hardlink_source - Check for safe hardlink conditions + * @idmap: idmap of the mount the inode was found from + * @inode: the source inode to hardlink from + * + * Return false if at least one of the following conditions: + * - inode is not a regular file + * - inode is setuid + * - inode is setgid and group-exec + * - access failure for read and write + * + * Otherwise returns true. + */ +static bool safe_hardlink_source(struct mnt_idmap *idmap, + struct inode *inode) +{ + umode_t mode = inode->i_mode; + + /* Special files should not get pinned to the filesystem. */ + if (!S_ISREG(mode)) + return false; + + /* Setuid files should not get pinned to the filesystem. */ + if (mode & S_ISUID) + return false; + + /* Executable setgid files should not get pinned to the filesystem. */ + if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP)) + return false; + + /* Hardlinking to unreadable or unwritable sources is dangerous. */ + if (inode_permission(idmap, inode, MAY_READ | MAY_WRITE)) + return false; + + return true; +} + +/** + * may_linkat - Check permissions for creating a hardlink + * @idmap: idmap of the mount the inode was found from + * @link: the source to hardlink from + * + * Block hardlink when all of: + * - sysctl_protected_hardlinks enabled + * - fsuid does not match inode + * - hardlink source is unsafe (see safe_hardlink_source() above) + * - not CAP_FOWNER in a namespace with the inode owner uid mapped + * + * If the inode has been found through an idmapped mount the idmap of + * the vfsmount must be passed through @idmap. This function will then take + * care to map the inode according to @idmap before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply pass @nop_mnt_idmap. + * + * Returns 0 if successful, -ve on error. + */ +int may_linkat(struct mnt_idmap *idmap, const struct path *link) +{ + struct inode *inode = link->dentry->d_inode; + + /* Inode writeback is not safe when the uid or gid are invalid. */ + if (!vfsuid_valid(i_uid_into_vfsuid(idmap, inode)) || + !vfsgid_valid(i_gid_into_vfsgid(idmap, inode))) + return -EOVERFLOW; + + if (!sysctl_protected_hardlinks) + return 0; + + /* Source inode owner (or CAP_FOWNER) can hardlink all they like, + * otherwise, it must be a safe source. + */ + if (safe_hardlink_source(idmap, inode) || + inode_owner_or_capable(idmap, inode)) + return 0; + + audit_log_path_denied(AUDIT_ANOM_LINK, "linkat"); + return -EPERM; +} + +/** + * may_create_in_sticky - Check whether an O_CREAT open in a sticky directory + * should be allowed, or not, on files that already + * exist. + * @idmap: idmap of the mount the inode was found from + * @nd: nameidata pathwalk data + * @inode: the inode of the file to open + * + * Block an O_CREAT open of a FIFO (or a regular file) when: + * - sysctl_protected_fifos (or sysctl_protected_regular) is enabled + * - the file already exists + * - we are in a sticky directory + * - we don't own the file + * - the owner of the directory doesn't own the file + * - the directory is world writable + * If the sysctl_protected_fifos (or sysctl_protected_regular) is set to 2 + * the directory doesn't have to be world writable: being group writable will + * be enough. + * + * If the inode has been found through an idmapped mount the idmap of + * the vfsmount must be passed through @idmap. This function will then take + * care to map the inode according to @idmap before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply pass @nop_mnt_idmap. + * + * Returns 0 if the open is allowed, -ve on error. + */ +static int may_create_in_sticky(struct mnt_idmap *idmap, struct nameidata *nd, + struct inode *const inode) +{ + umode_t dir_mode = nd->dir_mode; + vfsuid_t dir_vfsuid = nd->dir_vfsuid, i_vfsuid; + + if (likely(!(dir_mode & S_ISVTX))) + return 0; + + if (S_ISREG(inode->i_mode) && !sysctl_protected_regular) + return 0; + + if (S_ISFIFO(inode->i_mode) && !sysctl_protected_fifos) + return 0; + + i_vfsuid = i_uid_into_vfsuid(idmap, inode); + + if (vfsuid_eq(i_vfsuid, dir_vfsuid)) + return 0; + + if (vfsuid_eq_kuid(i_vfsuid, current_fsuid())) + return 0; + + if (likely(dir_mode & 0002)) { + audit_log_path_denied(AUDIT_ANOM_CREAT, "sticky_create"); + return -EACCES; + } + + if (dir_mode & 0020) { + if (sysctl_protected_fifos >= 2 && S_ISFIFO(inode->i_mode)) { + audit_log_path_denied(AUDIT_ANOM_CREAT, + "sticky_create_fifo"); + return -EACCES; + } + + if (sysctl_protected_regular >= 2 && S_ISREG(inode->i_mode)) { + audit_log_path_denied(AUDIT_ANOM_CREAT, + "sticky_create_regular"); + return -EACCES; + } + } + + return 0; +} + +/* + * follow_up - Find the mountpoint of path's vfsmount + * + * Given a path, find the mountpoint of its source file system. + * Replace @path with the path of the mountpoint in the parent mount. + * Up is towards /. + * + * Return 1 if we went up a level and 0 if we were already at the + * root. + */ +int follow_up(struct path *path) +{ + struct mount *mnt = real_mount(path->mnt); + struct mount *parent; + struct dentry *mountpoint; + + read_seqlock_excl(&mount_lock); + parent = mnt->mnt_parent; + if (parent == mnt) { + read_sequnlock_excl(&mount_lock); + return 0; + } + mntget(&parent->mnt); + mountpoint = dget(mnt->mnt_mountpoint); + read_sequnlock_excl(&mount_lock); + dput(path->dentry); + path->dentry = mountpoint; + mntput(path->mnt); + path->mnt = &parent->mnt; + return 1; +} +EXPORT_SYMBOL(follow_up); + +static bool choose_mountpoint_rcu(struct mount *m, const struct path *root, + struct path *path, unsigned *seqp) +{ + while (mnt_has_parent(m)) { + struct dentry *mountpoint = m->mnt_mountpoint; + + m = m->mnt_parent; + if (unlikely(root->dentry == mountpoint && + root->mnt == &m->mnt)) + break; + if (mountpoint != m->mnt.mnt_root) { + path->mnt = &m->mnt; + path->dentry = mountpoint; + *seqp = read_seqcount_begin(&mountpoint->d_seq); + return true; + } + } + return false; +} + +static bool choose_mountpoint(struct mount *m, const struct path *root, + struct path *path) +{ + bool found; + + rcu_read_lock(); + while (1) { + unsigned seq, mseq = read_seqbegin(&mount_lock); + + found = choose_mountpoint_rcu(m, root, path, &seq); + if (unlikely(!found)) { + if (!read_seqretry(&mount_lock, mseq)) + break; + } else { + if (likely(__legitimize_path(path, seq, mseq))) + break; + rcu_read_unlock(); + path_put(path); + rcu_read_lock(); + } + } + rcu_read_unlock(); + return found; +} + +/* + * Perform an automount + * - return -EISDIR to tell follow_managed() to stop and return the path we + * were called with. + */ +static int follow_automount(struct path *path, int *count, unsigned lookup_flags) +{ + struct dentry *dentry = path->dentry; + + /* We don't want to mount if someone's just doing a stat - + * unless they're stat'ing a directory and appended a '/' to + * the name. + * + * We do, however, want to mount if someone wants to open or + * create a file of any type under the mountpoint, wants to + * traverse through the mountpoint or wants to open the + * mounted directory. Also, autofs may mark negative dentries + * as being automount points. These will need the attentions + * of the daemon to instantiate them before they can be used. + */ + if (!(lookup_flags & (LOOKUP_PARENT | LOOKUP_DIRECTORY | + LOOKUP_OPEN | LOOKUP_CREATE | LOOKUP_AUTOMOUNT)) && + dentry->d_inode) + return -EISDIR; + + /* No need to trigger automounts if mountpoint crossing is disabled. */ + if (lookup_flags & LOOKUP_NO_XDEV) + return -EXDEV; + + if (count && (*count)++ >= MAXSYMLINKS) + return -ELOOP; + + return finish_automount(dentry->d_op->d_automount(path), path); +} + +/* + * mount traversal - out-of-line part. One note on ->d_flags accesses - + * dentries are pinned but not locked here, so negative dentry can go + * positive right under us. Use of smp_load_acquire() provides a barrier + * sufficient for ->d_inode and ->d_flags consistency. + */ +static int __traverse_mounts(struct path *path, unsigned flags, bool *jumped, + int *count, unsigned lookup_flags) +{ + struct vfsmount *mnt = path->mnt; + bool need_mntput = false; + int ret = 0; + + while (flags & DCACHE_MANAGED_DENTRY) { + /* Allow the filesystem to manage the transit without i_rwsem + * being held. */ + if (flags & DCACHE_MANAGE_TRANSIT) { + if (lookup_flags & LOOKUP_NO_XDEV) { + ret = -EXDEV; + break; + } + ret = path->dentry->d_op->d_manage(path, false); + flags = smp_load_acquire(&path->dentry->d_flags); + if (ret < 0) + break; + } + + if (flags & DCACHE_MOUNTED) { // something's mounted on it.. + struct vfsmount *mounted = lookup_mnt(path); + if (mounted) { // ... in our namespace + dput(path->dentry); + if (need_mntput) + mntput(path->mnt); + path->mnt = mounted; + path->dentry = dget(mounted->mnt_root); + // here we know it's positive + flags = path->dentry->d_flags; + need_mntput = true; + if (unlikely(lookup_flags & LOOKUP_NO_XDEV)) { + ret = -EXDEV; + break; + } + continue; + } + } + + if (!(flags & DCACHE_NEED_AUTOMOUNT)) + break; + + // uncovered automount point + ret = follow_automount(path, count, lookup_flags); + flags = smp_load_acquire(&path->dentry->d_flags); + if (ret < 0) + break; + } + + if (ret == -EISDIR) + ret = 0; + // possible if you race with several mount --move + if (need_mntput && path->mnt == mnt) + mntput(path->mnt); + if (!ret && unlikely(d_flags_negative(flags))) + ret = -ENOENT; + *jumped = need_mntput; + return ret; +} + +static inline int traverse_mounts(struct path *path, bool *jumped, + int *count, unsigned lookup_flags) +{ + unsigned flags = smp_load_acquire(&path->dentry->d_flags); + + /* fastpath */ + if (likely(!(flags & DCACHE_MANAGED_DENTRY))) { + *jumped = false; + if (unlikely(d_flags_negative(flags))) + return -ENOENT; + return 0; + } + return __traverse_mounts(path, flags, jumped, count, lookup_flags); +} + +int follow_down_one(struct path *path) +{ + struct vfsmount *mounted; + + mounted = lookup_mnt(path); + if (mounted) { + dput(path->dentry); + mntput(path->mnt); + path->mnt = mounted; + path->dentry = dget(mounted->mnt_root); + return 1; + } + return 0; +} +EXPORT_SYMBOL(follow_down_one); + +/* + * Follow down to the covering mount currently visible to userspace. At each + * point, the filesystem owning that dentry may be queried as to whether the + * caller is permitted to proceed or not. + */ +int follow_down(struct path *path, unsigned int flags) +{ + struct vfsmount *mnt = path->mnt; + bool jumped; + int ret = traverse_mounts(path, &jumped, NULL, flags); + + if (path->mnt != mnt) + mntput(mnt); + return ret; +} +EXPORT_SYMBOL(follow_down); + +/* + * Try to skip to top of mountpoint pile in rcuwalk mode. Fail if + * we meet a managed dentry that would need blocking. + */ +static bool __follow_mount_rcu(struct nameidata *nd, struct path *path) +{ + struct dentry *dentry = path->dentry; + unsigned int flags = dentry->d_flags; + + if (unlikely(nd->flags & LOOKUP_NO_XDEV)) + return false; + + for (;;) { + /* + * Don't forget we might have a non-mountpoint managed dentry + * that wants to block transit. + */ + if (unlikely(flags & DCACHE_MANAGE_TRANSIT)) { + int res = dentry->d_op->d_manage(path, true); + if (res) + return res == -EISDIR; + flags = dentry->d_flags; + } + + if (flags & DCACHE_MOUNTED) { + struct mount *mounted = __lookup_mnt(path->mnt, dentry); + if (mounted) { + path->mnt = &mounted->mnt; + dentry = path->dentry = mounted->mnt.mnt_root; + nd->state |= ND_JUMPED; + nd->next_seq = read_seqcount_begin(&dentry->d_seq); + flags = dentry->d_flags; + // makes sure that non-RCU pathwalk could reach + // this state. + if (read_seqretry(&mount_lock, nd->m_seq)) + return false; + continue; + } + if (read_seqretry(&mount_lock, nd->m_seq)) + return false; + } + return !(flags & DCACHE_NEED_AUTOMOUNT); + } +} + +static inline int handle_mounts(struct nameidata *nd, struct dentry *dentry, + struct path *path) +{ + bool jumped; + int ret; + + path->mnt = nd->path.mnt; + path->dentry = dentry; + if (nd->flags & LOOKUP_RCU) { + unsigned int seq = nd->next_seq; + if (likely(!d_managed(dentry))) + return 0; + if (likely(__follow_mount_rcu(nd, path))) + return 0; + // *path and nd->next_seq might've been clobbered + path->mnt = nd->path.mnt; + path->dentry = dentry; + nd->next_seq = seq; + if (unlikely(!try_to_unlazy_next(nd, dentry))) + return -ECHILD; + } + ret = traverse_mounts(path, &jumped, &nd->total_link_count, nd->flags); + if (jumped) + nd->state |= ND_JUMPED; + if (unlikely(ret)) { + dput(path->dentry); + if (path->mnt != nd->path.mnt) + mntput(path->mnt); + } + return ret; +} + +/* + * This looks up the name in dcache and possibly revalidates the found dentry. + * NULL is returned if the dentry does not exist in the cache. + */ +static struct dentry *lookup_dcache(const struct qstr *name, + struct dentry *dir, + unsigned int flags) +{ + struct dentry *dentry = d_lookup(dir, name); + if (dentry) { + int error = d_revalidate(dir->d_inode, name, dentry, flags); + if (unlikely(error <= 0)) { + if (!error) + d_invalidate(dentry); + dput(dentry); + return ERR_PTR(error); + } + } + return dentry; +} + +/* + * Parent directory has inode locked exclusive. This is one + * and only case when ->lookup() gets called on non in-lookup + * dentries - as the matter of fact, this only gets called + * when directory is guaranteed to have no in-lookup children + * at all. + * Will return -ENOENT if name isn't found and LOOKUP_CREATE wasn't passed. + * Will return -EEXIST if name is found and LOOKUP_EXCL was passed. + */ +static struct dentry *lookup_one_qstr_excl(const struct qstr *name, + struct dentry *base, unsigned int flags) +{ + struct dentry *dentry; + struct dentry *old; + struct inode *dir; + + dentry = lookup_dcache(name, base, flags); + if (dentry) + goto found; + + /* Don't create child dentry for a dead directory. */ + dir = base->d_inode; + if (unlikely(IS_DEADDIR(dir))) + return ERR_PTR(-ENOENT); + + dentry = d_alloc(base, name); + if (unlikely(!dentry)) + return ERR_PTR(-ENOMEM); + + old = dir->i_op->lookup(dir, dentry, flags); + if (unlikely(old)) { + dput(dentry); + dentry = old; + } +found: + if (IS_ERR(dentry)) + return dentry; + if (d_is_negative(dentry) && !(flags & LOOKUP_CREATE)) { + dput(dentry); + return ERR_PTR(-ENOENT); + } + if (d_is_positive(dentry) && (flags & LOOKUP_EXCL)) { + dput(dentry); + return ERR_PTR(-EEXIST); + } + return dentry; +} + +/** + * lookup_fast - do fast lockless (but racy) lookup of a dentry + * @nd: current nameidata + * + * Do a fast, but racy lookup in the dcache for the given dentry, and + * revalidate it. Returns a valid dentry pointer or NULL if one wasn't + * found. On error, an ERR_PTR will be returned. + * + * If this function returns a valid dentry and the walk is no longer + * lazy, the dentry will carry a reference that must later be put. If + * RCU mode is still in force, then this is not the case and the dentry + * must be legitimized before use. If this returns NULL, then the walk + * will no longer be in RCU mode. + */ +static struct dentry *lookup_fast(struct nameidata *nd) +{ + struct dentry *dentry, *parent = nd->path.dentry; + int status = 1; + + /* + * Rename seqlock is not required here because in the off chance + * of a false negative due to a concurrent rename, the caller is + * going to fall back to non-racy lookup. + */ + if (nd->flags & LOOKUP_RCU) { + dentry = __d_lookup_rcu(parent, &nd->last, &nd->next_seq); + if (unlikely(!dentry)) { + if (!try_to_unlazy(nd)) + return ERR_PTR(-ECHILD); + return NULL; + } + + /* + * This sequence count validates that the parent had no + * changes while we did the lookup of the dentry above. + */ + if (read_seqcount_retry(&parent->d_seq, nd->seq)) + return ERR_PTR(-ECHILD); + + status = d_revalidate(nd->inode, &nd->last, dentry, nd->flags); + if (likely(status > 0)) + return dentry; + if (!try_to_unlazy_next(nd, dentry)) + return ERR_PTR(-ECHILD); + if (status == -ECHILD) + /* we'd been told to redo it in non-rcu mode */ + status = d_revalidate(nd->inode, &nd->last, + dentry, nd->flags); + } else { + dentry = __d_lookup(parent, &nd->last); + if (unlikely(!dentry)) + return NULL; + status = d_revalidate(nd->inode, &nd->last, dentry, nd->flags); + } + if (unlikely(status <= 0)) { + if (!status) + d_invalidate(dentry); + dput(dentry); + return ERR_PTR(status); + } + return dentry; +} + +/* Fast lookup failed, do it the slow way */ +static struct dentry *__lookup_slow(const struct qstr *name, + struct dentry *dir, + unsigned int flags) +{ + struct dentry *dentry, *old; + struct inode *inode = dir->d_inode; + + /* Don't go there if it's already dead */ + if (unlikely(IS_DEADDIR(inode))) + return ERR_PTR(-ENOENT); +again: + dentry = d_alloc_parallel(dir, name); + if (IS_ERR(dentry)) + return dentry; + if (unlikely(!d_in_lookup(dentry))) { + int error = d_revalidate(inode, name, dentry, flags); + if (unlikely(error <= 0)) { + if (!error) { + d_invalidate(dentry); + dput(dentry); + goto again; + } + dput(dentry); + dentry = ERR_PTR(error); + } + } else { + old = inode->i_op->lookup(inode, dentry, flags); + d_lookup_done(dentry); + if (unlikely(old)) { + dput(dentry); + dentry = old; + } + } + return dentry; +} + +static noinline struct dentry *lookup_slow(const struct qstr *name, + struct dentry *dir, + unsigned int flags) +{ + struct inode *inode = dir->d_inode; + struct dentry *res; + inode_lock_shared(inode); + res = __lookup_slow(name, dir, flags); + inode_unlock_shared(inode); + return res; +} + +static struct dentry *lookup_slow_killable(const struct qstr *name, + struct dentry *dir, + unsigned int flags) +{ + struct inode *inode = dir->d_inode; + struct dentry *res; + + if (inode_lock_shared_killable(inode)) + return ERR_PTR(-EINTR); + res = __lookup_slow(name, dir, flags); + inode_unlock_shared(inode); + return res; +} + +static inline int may_lookup(struct mnt_idmap *idmap, + struct nameidata *restrict nd) +{ + int err, mask; + + mask = nd->flags & LOOKUP_RCU ? MAY_NOT_BLOCK : 0; + err = lookup_inode_permission_may_exec(idmap, nd->inode, mask); + if (likely(!err)) + return 0; + + // If we failed, and we weren't in LOOKUP_RCU, it's final + if (!(nd->flags & LOOKUP_RCU)) + return err; + + // Drop out of RCU mode to make sure it wasn't transient + if (!try_to_unlazy(nd)) + return -ECHILD; // redo it all non-lazy + + if (err != -ECHILD) // hard error + return err; + + return lookup_inode_permission_may_exec(idmap, nd->inode, 0); +} + +static int reserve_stack(struct nameidata *nd, struct path *link) +{ + if (unlikely(nd->total_link_count++ >= MAXSYMLINKS)) + return -ELOOP; + + if (likely(nd->depth != EMBEDDED_LEVELS)) + return 0; + if (likely(nd->stack != nd->internal)) + return 0; + if (likely(nd_alloc_stack(nd))) + return 0; + + if (nd->flags & LOOKUP_RCU) { + // we need to grab link before we do unlazy. And we can't skip + // unlazy even if we fail to grab the link - cleanup needs it + bool grabbed_link = legitimize_path(nd, link, nd->next_seq); + + if (!try_to_unlazy(nd) || !grabbed_link) + return -ECHILD; + + if (nd_alloc_stack(nd)) + return 0; + } + return -ENOMEM; +} + +enum {WALK_TRAILING = 1, WALK_MORE = 2, WALK_NOFOLLOW = 4}; + +static noinline const char *pick_link(struct nameidata *nd, struct path *link, + struct inode *inode, int flags) +{ + struct saved *last; + const char *res; + int error; + + if (nd->flags & LOOKUP_RCU) { + /* make sure that d_is_symlink from step_into_slowpath() matches the inode */ + if (read_seqcount_retry(&link->dentry->d_seq, nd->next_seq)) + return ERR_PTR(-ECHILD); + } else { + if (link->mnt == nd->path.mnt) + mntget(link->mnt); + } + + error = reserve_stack(nd, link); + if (unlikely(error)) { + if (!(nd->flags & LOOKUP_RCU)) + path_put(link); + return ERR_PTR(error); + } + last = nd->stack + nd->depth++; + last->link = *link; + clear_delayed_call(&last->done); + last->seq = nd->next_seq; + + if (flags & WALK_TRAILING) { + error = may_follow_link(nd, inode); + if (unlikely(error)) + return ERR_PTR(error); + } + + if (unlikely(nd->flags & LOOKUP_NO_SYMLINKS) || + unlikely(link->mnt->mnt_flags & MNT_NOSYMFOLLOW)) + return ERR_PTR(-ELOOP); + + if (unlikely(atime_needs_update(&last->link, inode))) { + if (nd->flags & LOOKUP_RCU) { + if (!try_to_unlazy(nd)) + return ERR_PTR(-ECHILD); + } + touch_atime(&last->link); + cond_resched(); + } + + error = security_inode_follow_link(link->dentry, inode, + nd->flags & LOOKUP_RCU); + if (unlikely(error)) + return ERR_PTR(error); + + res = READ_ONCE(inode->i_link); + if (!res) { + const char * (*get)(struct dentry *, struct inode *, + struct delayed_call *); + get = inode->i_op->get_link; + if (nd->flags & LOOKUP_RCU) { + res = get(NULL, inode, &last->done); + if (res == ERR_PTR(-ECHILD) && try_to_unlazy(nd)) + res = get(link->dentry, inode, &last->done); + } else { + res = get(link->dentry, inode, &last->done); + } + if (!res) + goto all_done; + if (IS_ERR(res)) + return res; + } + if (*res == '/') { + error = nd_jump_root(nd); + if (unlikely(error)) + return ERR_PTR(error); + while (unlikely(*++res == '/')) + ; + } + if (*res) + return res; +all_done: // pure jump + put_link(nd); + return NULL; +} + +/* + * Do we need to follow links? We _really_ want to be able + * to do this check without having to look at inode->i_op, + * so we keep a cache of "no, this doesn't need follow_link" + * for the common case. + * + * NOTE: dentry must be what nd->next_seq had been sampled from. + */ +static noinline const char *step_into_slowpath(struct nameidata *nd, int flags, + struct dentry *dentry) +{ + struct path path; + struct inode *inode; + int err; + + err = handle_mounts(nd, dentry, &path); + if (unlikely(err < 0)) + return ERR_PTR(err); + inode = path.dentry->d_inode; + if (likely(!d_is_symlink(path.dentry)) || + ((flags & WALK_TRAILING) && !(nd->flags & LOOKUP_FOLLOW)) || + (flags & WALK_NOFOLLOW)) { + /* not a symlink or should not follow */ + if (nd->flags & LOOKUP_RCU) { + if (read_seqcount_retry(&path.dentry->d_seq, nd->next_seq)) + return ERR_PTR(-ECHILD); + if (unlikely(!inode)) + return ERR_PTR(-ENOENT); + } else { + dput(nd->path.dentry); + if (nd->path.mnt != path.mnt) + mntput(nd->path.mnt); + } + nd->path = path; + nd->inode = inode; + nd->seq = nd->next_seq; + return NULL; + } + return pick_link(nd, &path, inode, flags); +} + +static __always_inline const char *step_into(struct nameidata *nd, int flags, + struct dentry *dentry) +{ + /* + * In the common case we are in rcu-walk and traversing over a non-mounted on + * directory (as opposed to e.g., a symlink). + * + * We can handle that and negative entries with the checks below. + */ + if (likely((nd->flags & LOOKUP_RCU) && + !d_managed(dentry) && !d_is_symlink(dentry))) { + struct inode *inode = dentry->d_inode; + if (read_seqcount_retry(&dentry->d_seq, nd->next_seq)) + return ERR_PTR(-ECHILD); + if (unlikely(!inode)) + return ERR_PTR(-ENOENT); + nd->path.dentry = dentry; + /* nd->path.mnt remains unchanged as no mount point was crossed */ + nd->inode = inode; + nd->seq = nd->next_seq; + return NULL; + } + return step_into_slowpath(nd, flags, dentry); +} + +static struct dentry *follow_dotdot_rcu(struct nameidata *nd) +{ + struct dentry *parent, *old; + + if (path_equal(&nd->path, &nd->root)) + goto in_root; + if (unlikely(nd->path.dentry == nd->path.mnt->mnt_root)) { + struct path path; + unsigned seq; + if (!choose_mountpoint_rcu(real_mount(nd->path.mnt), + &nd->root, &path, &seq)) + goto in_root; + if (unlikely(nd->flags & LOOKUP_NO_XDEV)) + return ERR_PTR(-ECHILD); + nd->path = path; + nd->inode = path.dentry->d_inode; + nd->seq = seq; + // makes sure that non-RCU pathwalk could reach this state + if (read_seqretry(&mount_lock, nd->m_seq)) + return ERR_PTR(-ECHILD); + /* we know that mountpoint was pinned */ + } + old = nd->path.dentry; + parent = old->d_parent; + nd->next_seq = read_seqcount_begin(&parent->d_seq); + // makes sure that non-RCU pathwalk could reach this state + if (read_seqcount_retry(&old->d_seq, nd->seq)) + return ERR_PTR(-ECHILD); + if (unlikely(!path_connected(nd->path.mnt, parent))) + return ERR_PTR(-ECHILD); + return parent; +in_root: + if (read_seqretry(&mount_lock, nd->m_seq)) + return ERR_PTR(-ECHILD); + if (unlikely(nd->flags & LOOKUP_BENEATH)) + return ERR_PTR(-ECHILD); + nd->next_seq = nd->seq; + return nd->path.dentry; +} + +static struct dentry *follow_dotdot(struct nameidata *nd) +{ + struct dentry *parent; + + if (path_equal(&nd->path, &nd->root)) + goto in_root; + if (unlikely(nd->path.dentry == nd->path.mnt->mnt_root)) { + struct path path; + + if (!choose_mountpoint(real_mount(nd->path.mnt), + &nd->root, &path)) + goto in_root; + path_put(&nd->path); + nd->path = path; + nd->inode = path.dentry->d_inode; + if (unlikely(nd->flags & LOOKUP_NO_XDEV)) + return ERR_PTR(-EXDEV); + } + /* rare case of legitimate dget_parent()... */ + parent = dget_parent(nd->path.dentry); + if (unlikely(!path_connected(nd->path.mnt, parent))) { + dput(parent); + return ERR_PTR(-ENOENT); + } + return parent; + +in_root: + if (unlikely(nd->flags & LOOKUP_BENEATH)) + return ERR_PTR(-EXDEV); + return dget(nd->path.dentry); +} + +static const char *handle_dots(struct nameidata *nd, enum last_type type) +{ + if (type == LAST_DOTDOT) { + const char *error = NULL; + struct dentry *parent; + + if (!nd->root.mnt) { + error = ERR_PTR(set_root(nd)); + if (unlikely(error)) + return error; + } + if (nd->flags & LOOKUP_RCU) + parent = follow_dotdot_rcu(nd); + else + parent = follow_dotdot(nd); + if (IS_ERR(parent)) + return ERR_CAST(parent); + error = step_into(nd, WALK_NOFOLLOW, parent); + if (unlikely(error)) + return error; + + if (unlikely(nd->flags & LOOKUP_IS_SCOPED)) { + /* + * If there was a racing rename or mount along our + * path, then we can't be sure that ".." hasn't jumped + * above nd->root (and so userspace should retry or use + * some fallback). + */ + smp_rmb(); + if (__read_seqcount_retry(&mount_lock.seqcount, nd->m_seq)) + return ERR_PTR(-EAGAIN); + if (__read_seqcount_retry(&rename_lock.seqcount, nd->r_seq)) + return ERR_PTR(-EAGAIN); + } + } + return NULL; +} + +static __always_inline const char *walk_component(struct nameidata *nd, int flags) +{ + struct dentry *dentry; + /* + * "." and ".." are special - ".." especially so because it has + * to be able to know about the current root directory and + * parent relationships. + */ + if (unlikely(nd->last_type != LAST_NORM)) { + if (unlikely(nd->depth) && !(flags & WALK_MORE)) + put_link(nd); + return handle_dots(nd, nd->last_type); + } + dentry = lookup_fast(nd); + if (IS_ERR(dentry)) + return ERR_CAST(dentry); + if (unlikely(!dentry)) { + dentry = lookup_slow(&nd->last, nd->path.dentry, nd->flags); + if (IS_ERR(dentry)) + return ERR_CAST(dentry); + } + if (unlikely(nd->depth) && !(flags & WALK_MORE)) + put_link(nd); + return step_into(nd, flags, dentry); +} + +/* + * We can do the critical dentry name comparison and hashing + * operations one word at a time, but we are limited to: + * + * - Architectures with fast unaligned word accesses. We could + * do a "get_unaligned()" if this helps and is sufficiently + * fast. + * + * - non-CONFIG_DEBUG_PAGEALLOC configurations (so that we + * do not trap on the (extremely unlikely) case of a page + * crossing operation. + * + * - Furthermore, we need an efficient 64-bit compile for the + * 64-bit case in order to generate the "number of bytes in + * the final mask". Again, that could be replaced with a + * efficient population count instruction or similar. + */ +#ifdef CONFIG_DCACHE_WORD_ACCESS + +#include + +#ifdef HASH_MIX + +/* Architecture provides HASH_MIX and fold_hash() in */ + +#elif defined(CONFIG_64BIT) +/* + * Register pressure in the mixing function is an issue, particularly + * on 32-bit x86, but almost any function requires one state value and + * one temporary. Instead, use a function designed for two state values + * and no temporaries. + * + * This function cannot create a collision in only two iterations, so + * we have two iterations to achieve avalanche. In those two iterations, + * we have six layers of mixing, which is enough to spread one bit's + * influence out to 2^6 = 64 state bits. + * + * Rotate constants are scored by considering either 64 one-bit input + * deltas or 64*63/2 = 2016 two-bit input deltas, and finding the + * probability of that delta causing a change to each of the 128 output + * bits, using a sample of random initial states. + * + * The Shannon entropy of the computed probabilities is then summed + * to produce a score. Ideally, any input change has a 50% chance of + * toggling any given output bit. + * + * Mixing scores (in bits) for (12,45): + * Input delta: 1-bit 2-bit + * 1 round: 713.3 42542.6 + * 2 rounds: 2753.7 140389.8 + * 3 rounds: 5954.1 233458.2 + * 4 rounds: 7862.6 256672.2 + * Perfect: 8192 258048 + * (64*128) (64*63/2 * 128) + */ +#define HASH_MIX(x, y, a) \ + ( x ^= (a), \ + y ^= x, x = rol64(x,12),\ + x += y, y = rol64(y,45),\ + y *= 9 ) + +/* + * Fold two longs into one 32-bit hash value. This must be fast, but + * latency isn't quite as critical, as there is a fair bit of additional + * work done before the hash value is used. + */ +static inline unsigned int fold_hash(unsigned long x, unsigned long y) +{ + y ^= x * GOLDEN_RATIO_64; + y *= GOLDEN_RATIO_64; + return y >> 32; +} + +#else /* 32-bit case */ + +/* + * Mixing scores (in bits) for (7,20): + * Input delta: 1-bit 2-bit + * 1 round: 330.3 9201.6 + * 2 rounds: 1246.4 25475.4 + * 3 rounds: 1907.1 31295.1 + * 4 rounds: 2042.3 31718.6 + * Perfect: 2048 31744 + * (32*64) (32*31/2 * 64) + */ +#define HASH_MIX(x, y, a) \ + ( x ^= (a), \ + y ^= x, x = rol32(x, 7),\ + x += y, y = rol32(y,20),\ + y *= 9 ) + +static inline unsigned int fold_hash(unsigned long x, unsigned long y) +{ + /* Use arch-optimized multiply if one exists */ + return __hash_32(y ^ __hash_32(x)); +} + +#endif + +/* + * Return the hash of a string of known length. This is carfully + * designed to match hash_name(), which is the more critical function. + * In particular, we must end by hashing a final word containing 0..7 + * payload bytes, to match the way that hash_name() iterates until it + * finds the delimiter after the name. + */ +unsigned int full_name_hash(const void *salt, const char *name, unsigned int len) +{ + unsigned long a, x = 0, y = (unsigned long)salt; + + for (;;) { + if (!len) + goto done; + a = load_unaligned_zeropad(name); + if (len < sizeof(unsigned long)) + break; + HASH_MIX(x, y, a); + name += sizeof(unsigned long); + len -= sizeof(unsigned long); + } + x ^= a & bytemask_from_count(len); +done: + return fold_hash(x, y); +} +EXPORT_SYMBOL(full_name_hash); + +/* Return the "hash_len" (hash and length) of a null-terminated string */ +u64 hashlen_string(const void *salt, const char *name) +{ + unsigned long a = 0, x = 0, y = (unsigned long)salt; + unsigned long adata, mask, len; + const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS; + + len = 0; + goto inside; + + do { + HASH_MIX(x, y, a); + len += sizeof(unsigned long); +inside: + a = load_unaligned_zeropad(name+len); + } while (!has_zero(a, &adata, &constants)); + + adata = prep_zero_mask(a, adata, &constants); + mask = create_zero_mask(adata); + x ^= a & zero_bytemask(mask); + + return hashlen_create(fold_hash(x, y), len + find_zero(mask)); +} +EXPORT_SYMBOL(hashlen_string); + +/* + * hash_name - Calculate the length and hash of the path component + * @nd: the path resolution state + * @name: the pathname to read the component from + * @lastword: if the component fits in a single word, LAST_WORD_IS_DOT, + * LAST_WORD_IS_DOTDOT, or some other value depending on whether the + * component is '.', '..', or something else. Otherwise, @lastword is 0. + * + * Returns: a pointer to the terminating '/' or NUL character in @name. + */ +static inline const char *hash_name(struct nameidata *nd, + const char *name, + unsigned long *lastword) +{ + unsigned long a, b, x, y = (unsigned long)nd->path.dentry; + unsigned long adata, bdata, mask, len; + const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS; + + /* + * The first iteration is special, because it can result in + * '.' and '..' and has no mixing other than the final fold. + */ + a = load_unaligned_zeropad(name); + b = a ^ REPEAT_BYTE('/'); + if (has_zero(a, &adata, &constants) | has_zero(b, &bdata, &constants)) { + adata = prep_zero_mask(a, adata, &constants); + bdata = prep_zero_mask(b, bdata, &constants); + mask = create_zero_mask(adata | bdata); + a &= zero_bytemask(mask); + *lastword = a; + len = find_zero(mask); + nd->last.hash = fold_hash(a, y); + nd->last.len = len; + return name + len; + } + + len = 0; + x = 0; + do { + HASH_MIX(x, y, a); + len += sizeof(unsigned long); + a = load_unaligned_zeropad(name+len); + b = a ^ REPEAT_BYTE('/'); + } while (!(has_zero(a, &adata, &constants) | has_zero(b, &bdata, &constants))); + + adata = prep_zero_mask(a, adata, &constants); + bdata = prep_zero_mask(b, bdata, &constants); + mask = create_zero_mask(adata | bdata); + a &= zero_bytemask(mask); + x ^= a; + len += find_zero(mask); + *lastword = 0; // Multi-word components cannot be DOT or DOTDOT + + nd->last.hash = fold_hash(x, y); + nd->last.len = len; + return name + len; +} + +/* + * Note that the 'last' word is always zero-masked, but + * was loaded as a possibly big-endian word. + */ +#ifdef __BIG_ENDIAN + #define LAST_WORD_IS_DOT (0x2eul << (BITS_PER_LONG-8)) + #define LAST_WORD_IS_DOTDOT (0x2e2eul << (BITS_PER_LONG-16)) +#endif + +#else /* !CONFIG_DCACHE_WORD_ACCESS: Slow, byte-at-a-time version */ + +/* Return the hash of a string of known length */ +unsigned int full_name_hash(const void *salt, const char *name, unsigned int len) +{ + unsigned long hash = init_name_hash(salt); + while (len--) + hash = partial_name_hash((unsigned char)*name++, hash); + return end_name_hash(hash); +} +EXPORT_SYMBOL(full_name_hash); + +/* Return the "hash_len" (hash and length) of a null-terminated string */ +u64 hashlen_string(const void *salt, const char *name) +{ + unsigned long hash = init_name_hash(salt); + unsigned long len = 0, c; + + c = (unsigned char)*name; + while (c) { + len++; + hash = partial_name_hash(c, hash); + c = (unsigned char)name[len]; + } + return hashlen_create(end_name_hash(hash), len); +} +EXPORT_SYMBOL(hashlen_string); + +/* + * We know there's a real path component here of at least + * one character. + */ +static inline const char *hash_name(struct nameidata *nd, const char *name, unsigned long *lastword) +{ + unsigned long hash = init_name_hash(nd->path.dentry); + unsigned long len = 0, c, last = 0; + + c = (unsigned char)*name; + do { + last = (last << 8) + c; + len++; + hash = partial_name_hash(c, hash); + c = (unsigned char)name[len]; + } while (c && c != '/'); + + // This is reliable for DOT or DOTDOT, since the component + // cannot contain NUL characters - top bits being zero means + // we cannot have had any other pathnames. + *lastword = last; + nd->last.hash = end_name_hash(hash); + nd->last.len = len; + return name + len; +} + +#endif + +#ifndef LAST_WORD_IS_DOT + #define LAST_WORD_IS_DOT 0x2e + #define LAST_WORD_IS_DOTDOT 0x2e2e +#endif + +/* + * Name resolution. + * This is the basic name resolution function, turning a pathname into + * the final dentry. We expect 'base' to be positive and a directory. + * + * Returns 0 and nd will have valid dentry and mnt on success. + * Returns error and drops reference to input namei data on failure. + */ +static int link_path_walk(const char *name, struct nameidata *nd) +{ + int depth = 0; // depth <= nd->depth + int err; + + nd->last_type = LAST_ROOT; + nd->flags |= LOOKUP_PARENT; + if (IS_ERR(name)) + return PTR_ERR(name); + if (*name == '/') { + do { + name++; + } while (unlikely(*name == '/')); + } + if (unlikely(!*name)) { + nd->dir_mode = 0; // short-circuit the 'hardening' idiocy + return 0; + } + + /* At this point we know we have a real path component. */ + for(;;) { + struct mnt_idmap *idmap; + const char *link; + unsigned long lastword; + + idmap = mnt_idmap(nd->path.mnt); + err = may_lookup(idmap, nd); + if (unlikely(err)) + return err; + + nd->last.name = name; + name = hash_name(nd, name, &lastword); + + switch(lastword) { + case LAST_WORD_IS_DOTDOT: + nd->last_type = LAST_DOTDOT; + nd->state |= ND_JUMPED; + break; + + case LAST_WORD_IS_DOT: + nd->last_type = LAST_DOT; + break; + + default: + nd->last_type = LAST_NORM; + nd->state &= ~ND_JUMPED; + + struct dentry *parent = nd->path.dentry; + if (unlikely(parent->d_flags & DCACHE_OP_HASH)) { + err = parent->d_op->d_hash(parent, &nd->last); + if (err < 0) + return err; + } + } + + if (!*name) + goto OK; + /* + * If it wasn't NUL, we know it was '/'. Skip that + * slash, and continue until no more slashes. + */ + do { + name++; + } while (unlikely(*name == '/')); + if (unlikely(!*name)) { +OK: + /* pathname or trailing symlink, done */ + if (likely(!depth)) { + nd->dir_vfsuid = i_uid_into_vfsuid(idmap, nd->inode); + nd->dir_mode = nd->inode->i_mode; + nd->flags &= ~LOOKUP_PARENT; + return 0; + } + /* last component of nested symlink */ + name = nd->stack[--depth].name; + link = walk_component(nd, 0); + } else { + /* not the last component */ + link = walk_component(nd, WALK_MORE); + } + if (unlikely(link)) { + if (IS_ERR(link)) + return PTR_ERR(link); + /* a symlink to follow */ + nd->stack[depth++].name = name; + name = link; + continue; + } + if (unlikely(!d_can_lookup(nd->path.dentry))) { + if (nd->flags & LOOKUP_RCU) { + if (!try_to_unlazy(nd)) + return -ECHILD; + } + return -ENOTDIR; + } + } +} + +/* must be paired with terminate_walk() */ +static const char *path_init(struct nameidata *nd, unsigned flags) +{ + int error; + const char *s = nd->pathname; + + /* LOOKUP_CACHED requires RCU, ask caller to retry */ + if (unlikely((flags & (LOOKUP_RCU | LOOKUP_CACHED)) == LOOKUP_CACHED)) + return ERR_PTR(-EAGAIN); + + if (unlikely(!*s)) + flags &= ~LOOKUP_RCU; + if (flags & LOOKUP_RCU) + rcu_read_lock(); + else + nd->seq = nd->next_seq = 0; + + nd->flags = flags; + nd->state |= ND_JUMPED; + + nd->m_seq = __read_seqcount_begin(&mount_lock.seqcount); + nd->r_seq = __read_seqcount_begin(&rename_lock.seqcount); + smp_rmb(); + + if (unlikely(nd->state & ND_ROOT_PRESET)) { + struct dentry *root = nd->root.dentry; + struct inode *inode = root->d_inode; + if (*s && unlikely(!d_can_lookup(root))) + return ERR_PTR(-ENOTDIR); + nd->path = nd->root; + nd->inode = inode; + if (flags & LOOKUP_RCU) { + nd->seq = read_seqcount_begin(&nd->path.dentry->d_seq); + nd->root_seq = nd->seq; + } else { + path_get(&nd->path); + } + return s; + } + + nd->root.mnt = NULL; + + /* Absolute pathname -- fetch the root (LOOKUP_IN_ROOT uses nd->dfd). */ + if (*s == '/' && likely(!(flags & LOOKUP_IN_ROOT))) { + error = nd_jump_root(nd); + if (unlikely(error)) + return ERR_PTR(error); + return s; + } + + /* Relative pathname -- get the starting-point it is relative to. */ + if (nd->dfd == AT_FDCWD) { + if (flags & LOOKUP_RCU) { + struct fs_struct *fs = current->fs; + unsigned seq; + + do { + seq = read_seqbegin(&fs->seq); + nd->path = fs->pwd; + nd->inode = nd->path.dentry->d_inode; + nd->seq = __read_seqcount_begin(&nd->path.dentry->d_seq); + } while (read_seqretry(&fs->seq, seq)); + } else { + get_fs_pwd(current->fs, &nd->path); + nd->inode = nd->path.dentry->d_inode; + } + } else { + /* Caller must check execute permissions on the starting path component */ + CLASS(fd_raw, f)(nd->dfd); + struct dentry *dentry; + + if (fd_empty(f)) + return ERR_PTR(-EBADF); + + if (flags & LOOKUP_LINKAT_EMPTY) { + if (fd_file(f)->f_cred != current_cred() && + !ns_capable(fd_file(f)->f_cred->user_ns, CAP_DAC_READ_SEARCH)) + return ERR_PTR(-ENOENT); + } + + dentry = fd_file(f)->f_path.dentry; + + if (*s && unlikely(!d_can_lookup(dentry))) + return ERR_PTR(-ENOTDIR); + + nd->path = fd_file(f)->f_path; + if (flags & LOOKUP_RCU) { + nd->inode = nd->path.dentry->d_inode; + nd->seq = read_seqcount_begin(&nd->path.dentry->d_seq); + } else { + path_get(&nd->path); + nd->inode = nd->path.dentry->d_inode; + } + } + + /* For scoped-lookups we need to set the root to the dirfd as well. */ + if (unlikely(flags & LOOKUP_IS_SCOPED)) { + nd->root = nd->path; + if (flags & LOOKUP_RCU) { + nd->root_seq = nd->seq; + } else { + path_get(&nd->root); + nd->state |= ND_ROOT_GRABBED; + } + } + return s; +} + +static inline const char *lookup_last(struct nameidata *nd) +{ + if (nd->last_type == LAST_NORM && nd->last.name[nd->last.len]) + nd->flags |= LOOKUP_FOLLOW | LOOKUP_DIRECTORY; + + return walk_component(nd, WALK_TRAILING); +} + +static int handle_lookup_down(struct nameidata *nd) +{ + if (!(nd->flags & LOOKUP_RCU)) + dget(nd->path.dentry); + nd->next_seq = nd->seq; + return PTR_ERR(step_into(nd, WALK_NOFOLLOW, nd->path.dentry)); +} + +/* Returns 0 and nd will be valid on success; Returns error, otherwise. */ +static int path_lookupat(struct nameidata *nd, unsigned flags, struct path *path) +{ + const char *s = path_init(nd, flags); + int err; + + if (unlikely(flags & LOOKUP_DOWN) && !IS_ERR(s)) { + err = handle_lookup_down(nd); + if (unlikely(err < 0)) + s = ERR_PTR(err); + } + + while (!(err = link_path_walk(s, nd)) && + (s = lookup_last(nd)) != NULL) + ; + if (!err && unlikely(nd->flags & LOOKUP_MOUNTPOINT)) { + err = handle_lookup_down(nd); + nd->state &= ~ND_JUMPED; // no d_weak_revalidate(), please... + } + if (!err) + err = complete_walk(nd); + + if (!err && nd->flags & LOOKUP_DIRECTORY) + if (!d_can_lookup(nd->path.dentry)) + err = -ENOTDIR; + if (!err) { + *path = nd->path; + nd->path.mnt = NULL; + nd->path.dentry = NULL; + } + terminate_walk(nd); + return err; +} + +int filename_lookup(int dfd, struct filename *name, unsigned flags, + struct path *path, const struct path *root) +{ + int retval; + struct nameidata nd; + if (IS_ERR(name)) + return PTR_ERR(name); + set_nameidata(&nd, dfd, name, root); + retval = path_lookupat(&nd, flags | LOOKUP_RCU, path); + if (unlikely(retval == -ECHILD)) + retval = path_lookupat(&nd, flags, path); + if (unlikely(retval == -ESTALE)) + retval = path_lookupat(&nd, flags | LOOKUP_REVAL, path); + + if (likely(!retval)) + audit_inode(name, path->dentry, + flags & LOOKUP_MOUNTPOINT ? AUDIT_INODE_NOEVAL : 0); + restore_nameidata(); + return retval; +} + +/* Returns 0 and nd will be valid on success; Returns error, otherwise. */ +static int path_parentat(struct nameidata *nd, unsigned flags, + struct path *parent) +{ + const char *s = path_init(nd, flags); + int err = link_path_walk(s, nd); + if (!err) + err = complete_walk(nd); + if (!err) { + *parent = nd->path; + nd->path.mnt = NULL; + nd->path.dentry = NULL; + } + terminate_walk(nd); + return err; +} + +/* Note: this does not consume "name" */ +static int __filename_parentat(int dfd, struct filename *name, + unsigned int flags, struct path *parent, + struct qstr *last, enum last_type *type, + const struct path *root) +{ + int retval; + struct nameidata nd; + + if (IS_ERR(name)) + return PTR_ERR(name); + set_nameidata(&nd, dfd, name, root); + retval = path_parentat(&nd, flags | LOOKUP_RCU, parent); + if (unlikely(retval == -ECHILD)) + retval = path_parentat(&nd, flags, parent); + if (unlikely(retval == -ESTALE)) + retval = path_parentat(&nd, flags | LOOKUP_REVAL, parent); + if (likely(!retval)) { + *last = nd.last; + *type = nd.last_type; + audit_inode(name, parent->dentry, AUDIT_INODE_PARENT); + } + restore_nameidata(); + return retval; +} + +static int filename_parentat(int dfd, struct filename *name, + unsigned int flags, struct path *parent, + struct qstr *last, enum last_type *type) +{ + return __filename_parentat(dfd, name, flags, parent, last, type, NULL); +} + +static struct dentry *__start_dirop(struct dentry *parent, struct qstr *name, + unsigned int lookup_flags, + unsigned int state) +{ + struct dentry *dentry; + struct inode *dir = d_inode(parent); + + if (state == TASK_KILLABLE) { + int ret = down_write_killable_nested(&dir->i_rwsem, + I_MUTEX_PARENT); + if (ret) + return ERR_PTR(ret); + } else { + inode_lock_nested(dir, I_MUTEX_PARENT); + } + dentry = lookup_one_qstr_excl(name, parent, lookup_flags); + if (IS_ERR(dentry)) + inode_unlock(dir); + return dentry; +} + +/** + * start_dirop - begin a create or remove dirop, performing locking and lookup + * @parent: the dentry of the parent in which the operation will occur + * @name: a qstr holding the name within that parent + * @lookup_flags: intent and other lookup flags. + * + * The lookup is performed and necessary locks are taken so that, on success, + * the returned dentry can be operated on safely. + * The qstr must already have the hash value calculated. + * + * Returns: a locked dentry, or an error. + * + */ +struct dentry *start_dirop(struct dentry *parent, struct qstr *name, + unsigned int lookup_flags) +{ + return __start_dirop(parent, name, lookup_flags, TASK_NORMAL); +} + +/** + * end_dirop - signal completion of a dirop + * @de: the dentry which was returned by start_dirop or similar. + * + * If the de is an error, nothing happens. Otherwise any lock taken to + * protect the dentry is dropped and the dentry itself is release (dput()). + */ +void end_dirop(struct dentry *de) +{ + if (!IS_ERR(de)) { + inode_unlock(de->d_parent->d_inode); + dput(de); + } +} +EXPORT_SYMBOL(end_dirop); + +/* does lookup, returns the object with parent locked */ +struct dentry *start_removing_path(const char *name, struct path *path) +{ + CLASS(filename_kernel, filename)(name); + struct path parent_path __free(path_put) = {}; + struct dentry *d; + struct qstr last; + enum last_type type; + int error; + + error = filename_parentat(AT_FDCWD, filename, 0, &parent_path, &last, + &type); + if (error) + return ERR_PTR(error); + if (unlikely(type != LAST_NORM)) + return ERR_PTR(-EINVAL); + /* don't fail immediately if it's r/o, at least try to report other errors */ + error = mnt_want_write(parent_path.mnt); + d = start_dirop(parent_path.dentry, &last, 0); + if (IS_ERR(d)) + goto drop; + if (error) + goto fail; + path->dentry = no_free_ptr(parent_path.dentry); + path->mnt = no_free_ptr(parent_path.mnt); + return d; + +fail: + end_dirop(d); + d = ERR_PTR(error); +drop: + if (!error) + mnt_drop_write(parent_path.mnt); + return d; +} + +/** + * kern_path_parent: lookup path returning parent and target + * @name: path name + * @path: path to store parent in + * + * The path @name should end with a normal component, not "." or ".." or "/". + * A lookup is performed and if successful the parent information + * is store in @parent and the dentry is returned. + * + * The dentry maybe negative, the parent will be positive. + * + * Returns: dentry or error. + */ +struct dentry *kern_path_parent(const char *name, struct path *path) +{ + struct path parent_path __free(path_put) = {}; + CLASS(filename_kernel, filename)(name); + struct dentry *d; + struct qstr last; + enum last_type type; + int error; + + error = filename_parentat(AT_FDCWD, filename, 0, &parent_path, &last, &type); + if (error) + return ERR_PTR(error); + if (unlikely(type != LAST_NORM)) + return ERR_PTR(-EINVAL); + + d = lookup_noperm_unlocked(&last, parent_path.dentry); + if (IS_ERR(d)) + return d; + path->dentry = no_free_ptr(parent_path.dentry); + path->mnt = no_free_ptr(parent_path.mnt); + return d; +} + +int kern_path(const char *name, unsigned int flags, struct path *path) +{ + CLASS(filename_kernel, filename)(name); + return filename_lookup(AT_FDCWD, filename, flags, path, NULL); +} +EXPORT_SYMBOL(kern_path); + +/** + * vfs_path_parent_lookup - lookup a parent path relative to a dentry-vfsmount pair + * @filename: filename structure + * @flags: lookup flags + * @parent: pointer to struct path to fill + * @last: last component + * @root: pointer to struct path of the base directory + */ +int vfs_path_parent_lookup(struct filename *filename, unsigned int flags, + struct path *parent, struct qstr *last, + const struct path *root) +{ + enum last_type type; + int err = __filename_parentat(AT_FDCWD, filename, flags, parent, last, + &type, root); + if (err) + return err; + if (unlikely(type != LAST_NORM)) { + path_put(parent); + return -EINVAL; + } + return 0; +} +EXPORT_SYMBOL(vfs_path_parent_lookup); + +/** + * vfs_path_lookup - lookup a file path relative to a dentry-vfsmount pair + * @dentry: pointer to dentry of the base directory + * @mnt: pointer to vfs mount of the base directory + * @name: pointer to file name + * @flags: lookup flags + * @path: pointer to struct path to fill + */ +int vfs_path_lookup(struct dentry *dentry, struct vfsmount *mnt, + const char *name, unsigned int flags, + struct path *path) +{ + CLASS(filename_kernel, filename)(name); + struct path root = {.mnt = mnt, .dentry = dentry}; + + /* the first argument of filename_lookup() is ignored with root */ + return filename_lookup(AT_FDCWD, filename, flags, path, &root); +} +EXPORT_SYMBOL(vfs_path_lookup); + +int lookup_noperm_common(struct qstr *qname, struct dentry *base) +{ + const char *name = qname->name; + u32 len = qname->len; + + qname->hash = full_name_hash(base, name, len); + if (!len) + return -EACCES; + + if (name_is_dot_dotdot(name, len)) + return -EACCES; + + while (len--) { + unsigned int c = *(const unsigned char *)name++; + if (c == '/' || c == '\0') + return -EACCES; + } + /* + * See if the low-level filesystem might want + * to use its own hash.. + */ + if (base->d_flags & DCACHE_OP_HASH) { + int err = base->d_op->d_hash(base, qname); + if (err < 0) + return err; + } + return 0; +} + +static int lookup_one_common(struct mnt_idmap *idmap, + struct qstr *qname, struct dentry *base) +{ + int err; + err = lookup_noperm_common(qname, base); + if (err < 0) + return err; + return inode_permission(idmap, base->d_inode, MAY_EXEC); +} + +/** + * try_lookup_noperm - filesystem helper to lookup single pathname component + * @name: qstr storing pathname component to lookup + * @base: base directory to lookup from + * + * Look up a dentry by name in the dcache, returning NULL if it does not + * currently exist or an error if there is a problem with the name. + * The function does not try to create a dentry and if one + * is found it doesn't try to revalidate it. + * + * Note that this routine is purely a helper for filesystem usage and should + * not be called by generic code. It does no permission checking. + * + * No locks need be held - only a counted reference to @base is needed. + * + * Returns: + * - ref-counted dentry on success, or + * - %NULL if name could not be found, or + * - ERR_PTR(-EACCES) if name is dot or dotdot or contains a slash or nul, or + * - ERR_PTR() if fs provide ->d_hash, and this returned an error. + */ +struct dentry *try_lookup_noperm(struct qstr *name, struct dentry *base) +{ + int err; + + err = lookup_noperm_common(name, base); + if (err) + return ERR_PTR(err); + + return d_lookup(base, name); +} +EXPORT_SYMBOL(try_lookup_noperm); + +/** + * lookup_noperm - filesystem helper to lookup single pathname component + * @name: qstr storing pathname component to lookup + * @base: base directory to lookup from + * + * Note that this routine is purely a helper for filesystem usage and should + * not be called by generic code. It does no permission checking. + * + * The caller must hold base->i_rwsem. + */ +struct dentry *lookup_noperm(struct qstr *name, struct dentry *base) +{ + struct dentry *dentry; + int err; + + WARN_ON_ONCE(!inode_is_locked(base->d_inode)); + + err = lookup_noperm_common(name, base); + if (err) + return ERR_PTR(err); + + dentry = lookup_dcache(name, base, 0); + return dentry ? dentry : __lookup_slow(name, base, 0); +} +EXPORT_SYMBOL(lookup_noperm); + +/** + * lookup_one - lookup single pathname component + * @idmap: idmap of the mount the lookup is performed from + * @name: qstr holding pathname component to lookup + * @base: base directory to lookup from + * + * This can be used for in-kernel filesystem clients such as file servers. + * + * The caller must hold base->i_rwsem. + */ +struct dentry *lookup_one(struct mnt_idmap *idmap, struct qstr *name, + struct dentry *base) +{ + struct dentry *dentry; + int err; + + WARN_ON_ONCE(!inode_is_locked(base->d_inode)); + + err = lookup_one_common(idmap, name, base); + if (err) + return ERR_PTR(err); + + dentry = lookup_dcache(name, base, 0); + return dentry ? dentry : __lookup_slow(name, base, 0); +} +EXPORT_SYMBOL(lookup_one); + +/** + * lookup_one_unlocked - lookup single pathname component + * @idmap: idmap of the mount the lookup is performed from + * @name: qstr olding pathname component to lookup + * @base: base directory to lookup from + * + * This can be used for in-kernel filesystem clients such as file servers. + * + * Unlike lookup_one, it should be called without the parent + * i_rwsem held, and will take the i_rwsem itself if necessary. + * + * Returns: - A dentry, possibly negative, or + * - same errors as try_lookup_noperm() or + * - ERR_PTR(-ENOENT) if parent has been removed, or + * - ERR_PTR(-EACCES) if parent directory is not searchable. + */ +struct dentry *lookup_one_unlocked(struct mnt_idmap *idmap, struct qstr *name, + struct dentry *base) +{ + int err; + struct dentry *ret; + + err = lookup_one_common(idmap, name, base); + if (err) + return ERR_PTR(err); + + ret = lookup_dcache(name, base, 0); + if (!ret) + ret = lookup_slow(name, base, 0); + return ret; +} +EXPORT_SYMBOL(lookup_one_unlocked); + +/** + * lookup_one_positive_killable - lookup single pathname component + * @idmap: idmap of the mount the lookup is performed from + * @name: qstr olding pathname component to lookup + * @base: base directory to lookup from + * + * This helper will yield ERR_PTR(-ENOENT) on negatives. The helper returns + * known positive or ERR_PTR(). This is what most of the users want. + * + * Note that pinned negative with unlocked parent _can_ become positive at any + * time, so callers of lookup_one_unlocked() need to be very careful; pinned + * positives have >d_inode stable, so this one avoids such problems. + * + * This can be used for in-kernel filesystem clients such as file servers. + * + * It should be called without the parent i_rwsem held, and will take + * the i_rwsem itself if necessary. If a fatal signal is pending or + * delivered, it will return %-EINTR if the lock is needed. + * + * Returns: A dentry, possibly negative, or + * - same errors as lookup_one_unlocked() or + * - ERR_PTR(-EINTR) if a fatal signal is pending. + */ +struct dentry *lookup_one_positive_killable(struct mnt_idmap *idmap, + struct qstr *name, + struct dentry *base) +{ + int err; + struct dentry *ret; + + err = lookup_one_common(idmap, name, base); + if (err) + return ERR_PTR(err); + + ret = lookup_dcache(name, base, 0); + if (!ret) + ret = lookup_slow_killable(name, base, 0); + if (!IS_ERR(ret) && d_flags_negative(smp_load_acquire(&ret->d_flags))) { + dput(ret); + ret = ERR_PTR(-ENOENT); + } + return ret; +} +EXPORT_SYMBOL(lookup_one_positive_killable); + +/** + * lookup_one_positive_unlocked - lookup single pathname component + * @idmap: idmap of the mount the lookup is performed from + * @name: qstr holding pathname component to lookup + * @base: base directory to lookup from + * + * This helper will yield ERR_PTR(-ENOENT) on negatives. The helper returns + * known positive or ERR_PTR(). This is what most of the users want. + * + * Note that pinned negative with unlocked parent _can_ become positive at any + * time, so callers of lookup_one_unlocked() need to be very careful; pinned + * positives have >d_inode stable, so this one avoids such problems. + * + * This can be used for in-kernel filesystem clients such as file servers. + * + * The helper should be called without i_rwsem held. + * + * Returns: A positive dentry, or + * - ERR_PTR(-ENOENT) if the name could not be found, or + * - same errors as lookup_one_unlocked(). + */ +struct dentry *lookup_one_positive_unlocked(struct mnt_idmap *idmap, + struct qstr *name, + struct dentry *base) +{ + struct dentry *ret = lookup_one_unlocked(idmap, name, base); + + if (!IS_ERR(ret) && d_flags_negative(smp_load_acquire(&ret->d_flags))) { + dput(ret); + ret = ERR_PTR(-ENOENT); + } + return ret; +} +EXPORT_SYMBOL(lookup_one_positive_unlocked); + +/** + * lookup_noperm_unlocked - filesystem helper to lookup single pathname component + * @name: pathname component to lookup + * @base: base directory to lookup from + * + * Note that this routine is purely a helper for filesystem usage and should + * not be called by generic code. It does no permission checking. + * + * Unlike lookup_noperm(), it should be called without the parent + * i_rwsem held, and will take the i_rwsem itself if necessary. + * + * Unlike try_lookup_noperm() it *does* revalidate the dentry if it already + * existed. + * + * Returns: A dentry, possibly negative, or + * - ERR_PTR(-ENOENT) if parent has been removed, or + * - same errors as try_lookup_noperm() + */ +struct dentry *lookup_noperm_unlocked(struct qstr *name, struct dentry *base) +{ + struct dentry *ret; + int err; + + err = lookup_noperm_common(name, base); + if (err) + return ERR_PTR(err); + + ret = lookup_dcache(name, base, 0); + if (!ret) + ret = lookup_slow(name, base, 0); + return ret; +} +EXPORT_SYMBOL(lookup_noperm_unlocked); + +/* + * Like lookup_noperm_unlocked(), except that it yields ERR_PTR(-ENOENT) + * on negatives. Returns known positive or ERR_PTR(); that's what + * most of the users want. Note that pinned negative with unlocked parent + * _can_ become positive at any time, so callers of lookup_noperm_unlocked() + * need to be very careful; pinned positives have ->d_inode stable, so + * this one avoids such problems. + * + * Returns: A positive dentry, or + * - ERR_PTR(-ENOENT) if name cannot be found or parent has been removed, or + * - same errors as try_lookup_noperm() + */ +struct dentry *lookup_noperm_positive_unlocked(struct qstr *name, + struct dentry *base) +{ + struct dentry *ret; + + ret = lookup_noperm_unlocked(name, base); + if (!IS_ERR(ret) && d_flags_negative(smp_load_acquire(&ret->d_flags))) { + dput(ret); + ret = ERR_PTR(-ENOENT); + } + return ret; +} +EXPORT_SYMBOL(lookup_noperm_positive_unlocked); + +/** + * start_creating - prepare to create a given name with permission checking + * @idmap: idmap of the mount + * @parent: directory in which to prepare to create the name + * @name: the name to be created + * + * Locks are taken and a lookup is performed prior to creating + * an object in a directory. Permission checking (MAY_EXEC) is performed + * against @idmap. + * + * If the name already exists, a positive dentry is returned, so + * behaviour is similar to O_CREAT without O_EXCL, which doesn't fail + * with -EEXIST. + * + * Returns: a negative or positive dentry, or an error. + */ +struct dentry *start_creating(struct mnt_idmap *idmap, struct dentry *parent, + struct qstr *name) +{ + int err = lookup_one_common(idmap, name, parent); + + if (err) + return ERR_PTR(err); + return start_dirop(parent, name, LOOKUP_CREATE); +} +EXPORT_SYMBOL(start_creating); + +/** + * start_removing - prepare to remove a given name with permission checking + * @idmap: idmap of the mount + * @parent: directory in which to find the name + * @name: the name to be removed + * + * Locks are taken and a lookup in performed prior to removing + * an object from a directory. Permission checking (MAY_EXEC) is performed + * against @idmap. + * + * If the name doesn't exist, an error is returned. + * + * end_removing() should be called when removal is complete, or aborted. + * + * Returns: a positive dentry, or an error. + */ +struct dentry *start_removing(struct mnt_idmap *idmap, struct dentry *parent, + struct qstr *name) +{ + int err = lookup_one_common(idmap, name, parent); + + if (err) + return ERR_PTR(err); + return start_dirop(parent, name, 0); +} +EXPORT_SYMBOL(start_removing); + +/** + * start_creating_killable - prepare to create a given name with permission checking + * @idmap: idmap of the mount + * @parent: directory in which to prepare to create the name + * @name: the name to be created + * + * Locks are taken and a lookup in performed prior to creating + * an object in a directory. Permission checking (MAY_EXEC) is performed + * against @idmap. + * + * If the name already exists, a positive dentry is returned. + * + * If a signal is received or was already pending, the function aborts + * with -EINTR; + * + * Returns: a negative or positive dentry, or an error. + */ +struct dentry *start_creating_killable(struct mnt_idmap *idmap, + struct dentry *parent, + struct qstr *name) +{ + int err = lookup_one_common(idmap, name, parent); + + if (err) + return ERR_PTR(err); + return __start_dirop(parent, name, LOOKUP_CREATE, TASK_KILLABLE); +} +EXPORT_SYMBOL(start_creating_killable); + +/** + * start_removing_killable - prepare to remove a given name with permission checking + * @idmap: idmap of the mount + * @parent: directory in which to find the name + * @name: the name to be removed + * + * Locks are taken and a lookup in performed prior to removing + * an object from a directory. Permission checking (MAY_EXEC) is performed + * against @idmap. + * + * If the name doesn't exist, an error is returned. + * + * end_removing() should be called when removal is complete, or aborted. + * + * If a signal is received or was already pending, the function aborts + * with -EINTR; + * + * Returns: a positive dentry, or an error. + */ +struct dentry *start_removing_killable(struct mnt_idmap *idmap, + struct dentry *parent, + struct qstr *name) +{ + int err = lookup_one_common(idmap, name, parent); + + if (err) + return ERR_PTR(err); + return __start_dirop(parent, name, 0, TASK_KILLABLE); +} +EXPORT_SYMBOL(start_removing_killable); + +/** + * start_creating_noperm - prepare to create a given name without permission checking + * @parent: directory in which to prepare to create the name + * @name: the name to be created + * + * Locks are taken and a lookup in performed prior to creating + * an object in a directory. + * + * If the name already exists, a positive dentry is returned. + * + * Returns: a negative or positive dentry, or an error. + */ +struct dentry *start_creating_noperm(struct dentry *parent, + struct qstr *name) +{ + int err = lookup_noperm_common(name, parent); + + if (err) + return ERR_PTR(err); + return start_dirop(parent, name, LOOKUP_CREATE); +} +EXPORT_SYMBOL(start_creating_noperm); + +/** + * start_removing_noperm - prepare to remove a given name without permission checking + * @parent: directory in which to find the name + * @name: the name to be removed + * + * Locks are taken and a lookup in performed prior to removing + * an object from a directory. + * + * If the name doesn't exist, an error is returned. + * + * end_removing() should be called when removal is complete, or aborted. + * + * Returns: a positive dentry, or an error. + */ +struct dentry *start_removing_noperm(struct dentry *parent, + struct qstr *name) +{ + int err = lookup_noperm_common(name, parent); + + if (err) + return ERR_PTR(err); + return start_dirop(parent, name, 0); +} +EXPORT_SYMBOL(start_removing_noperm); + +/** + * start_creating_dentry - prepare to create a given dentry + * @parent: directory from which dentry should be removed + * @child: the dentry to be removed + * + * A lock is taken to protect the dentry again other dirops and + * the validity of the dentry is checked: correct parent and still hashed. + * + * If the dentry is valid and negative a reference is taken and + * returned. If not an error is returned. + * + * end_creating() should be called when creation is complete, or aborted. + * + * Returns: the valid dentry, or an error. + */ +struct dentry *start_creating_dentry(struct dentry *parent, + struct dentry *child) +{ + inode_lock_nested(parent->d_inode, I_MUTEX_PARENT); + if (unlikely(IS_DEADDIR(parent->d_inode) || + child->d_parent != parent || + d_unhashed(child))) { + inode_unlock(parent->d_inode); + return ERR_PTR(-EINVAL); + } + if (d_is_positive(child)) { + inode_unlock(parent->d_inode); + return ERR_PTR(-EEXIST); + } + return dget(child); +} +EXPORT_SYMBOL(start_creating_dentry); + +/** + * start_removing_dentry - prepare to remove a given dentry + * @parent: directory from which dentry should be removed + * @child: the dentry to be removed + * + * A lock is taken to protect the dentry again other dirops and + * the validity of the dentry is checked: correct parent and still hashed. + * + * If the dentry is valid and positive, a reference is taken and + * returned. If not an error is returned. + * + * end_removing() should be called when removal is complete, or aborted. + * + * Returns: the valid dentry, or an error. + */ +struct dentry *start_removing_dentry(struct dentry *parent, + struct dentry *child) +{ + inode_lock_nested(parent->d_inode, I_MUTEX_PARENT); + if (unlikely(IS_DEADDIR(parent->d_inode) || + child->d_parent != parent || + d_unhashed(child))) { + inode_unlock(parent->d_inode); + return ERR_PTR(-EINVAL); + } + if (d_is_negative(child)) { + inode_unlock(parent->d_inode); + return ERR_PTR(-ENOENT); + } + return dget(child); +} +EXPORT_SYMBOL(start_removing_dentry); + +#ifdef CONFIG_UNIX98_PTYS +int path_pts(struct path *path) +{ + /* Find something mounted on "pts" in the same directory as + * the input path. + */ + struct dentry *parent = dget_parent(path->dentry); + struct dentry *child; + + if (unlikely(!path_connected(path->mnt, parent))) { + dput(parent); + return -ENOENT; + } + dput(path->dentry); + path->dentry = parent; + child = d_hash_and_lookup(parent, &QSTR("pts")); + if (IS_ERR_OR_NULL(child)) + return -ENOENT; + + path->dentry = child; + dput(parent); + follow_down(path, 0); + return 0; +} +#endif + +int user_path_at(int dfd, const char __user *name, unsigned flags, + struct path *path) +{ + CLASS(filename_flags, filename)(name, flags); + return filename_lookup(dfd, filename, flags, path, NULL); +} +EXPORT_SYMBOL(user_path_at); + +int __check_sticky(struct mnt_idmap *idmap, struct inode *dir, + struct inode *inode) +{ + kuid_t fsuid = current_fsuid(); + + if (vfsuid_eq_kuid(i_uid_into_vfsuid(idmap, inode), fsuid)) + return 0; + if (vfsuid_eq_kuid(i_uid_into_vfsuid(idmap, dir), fsuid)) + return 0; + return !capable_wrt_inode_uidgid(idmap, inode, CAP_FOWNER); +} +EXPORT_SYMBOL(__check_sticky); + +/* + * Check whether we can remove a link victim from directory dir, check + * whether the type of victim is right. + * 1. We can't do it if dir is read-only (done in permission()) + * 2. We should have write and exec permissions on dir + * 3. We can't remove anything from append-only dir + * 4. We can't do anything with immutable dir (done in permission()) + * 5. If the sticky bit on dir is set we should either + * a. be owner of dir, or + * b. be owner of victim, or + * c. have CAP_FOWNER capability + * 6. If the victim is append-only or immutable we can't do antyhing with + * links pointing to it. + * 7. If the victim has an unknown uid or gid we can't change the inode. + * 8. If we were asked to remove a directory and victim isn't one - ENOTDIR. + * 9. If we were asked to remove a non-directory and victim isn't one - EISDIR. + * 10. We can't remove a root or mountpoint. + * 11. We don't allow removal of NFS sillyrenamed files; it's handled by + * nfs_async_unlink(). + */ +int may_delete_dentry(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *victim, bool isdir) +{ + struct inode *inode = d_backing_inode(victim); + int error; + + if (d_is_negative(victim)) + return -ENOENT; + BUG_ON(!inode); + + BUG_ON(victim->d_parent->d_inode != dir); + + /* Inode writeback is not safe when the uid or gid are invalid. */ + if (!vfsuid_valid(i_uid_into_vfsuid(idmap, inode)) || + !vfsgid_valid(i_gid_into_vfsgid(idmap, inode))) + return -EOVERFLOW; + + audit_inode_child(dir, victim, AUDIT_TYPE_CHILD_DELETE); + + error = inode_permission(idmap, dir, MAY_WRITE | MAY_EXEC); + if (error) + return error; + if (IS_APPEND(dir)) + return -EPERM; + + if (check_sticky(idmap, dir, inode) || IS_APPEND(inode) || + IS_IMMUTABLE(inode) || IS_SWAPFILE(inode) || + HAS_UNMAPPED_ID(idmap, inode)) + return -EPERM; + if (isdir) { + if (!d_is_dir(victim)) + return -ENOTDIR; + if (IS_ROOT(victim)) + return -EBUSY; + } else if (d_is_dir(victim)) + return -EISDIR; + if (IS_DEADDIR(dir)) + return -ENOENT; + if (victim->d_flags & DCACHE_NFSFS_RENAMED) + return -EBUSY; + return 0; +} +EXPORT_SYMBOL(may_delete_dentry); + +/* Check whether we can create an object with dentry child in directory + * dir. + * 1. We can't do it if child already exists (open has special treatment for + * this case, but since we are inlined it's OK) + * 2. We can't do it if dir is read-only (done in permission()) + * 3. We can't do it if the fs can't represent the fsuid or fsgid. + * 4. We should have write and exec permissions on dir + * 5. We can't do it if dir is immutable (done in permission()) + */ +int may_create_dentry(struct mnt_idmap *idmap, + struct inode *dir, struct dentry *child) +{ + audit_inode_child(dir, child, AUDIT_TYPE_CHILD_CREATE); + if (child->d_inode) + return -EEXIST; + if (IS_DEADDIR(dir)) + return -ENOENT; + if (!fsuidgid_has_mapping(dir->i_sb, idmap)) + return -EOVERFLOW; + + return inode_permission(idmap, dir, MAY_WRITE | MAY_EXEC); +} +EXPORT_SYMBOL(may_create_dentry); + +// p1 != p2, both are on the same filesystem, ->s_vfs_rename_mutex is held +static struct dentry *lock_two_directories(struct dentry *p1, struct dentry *p2) +{ + struct dentry *p = p1, *q = p2, *r; + + while ((r = p->d_parent) != p2 && r != p) + p = r; + if (r == p2) { + // p is a child of p2 and an ancestor of p1 or p1 itself + inode_lock_nested(p2->d_inode, I_MUTEX_PARENT); + inode_lock_nested(p1->d_inode, I_MUTEX_PARENT2); + return p; + } + // p is the root of connected component that contains p1 + // p2 does not occur on the path from p to p1 + while ((r = q->d_parent) != p1 && r != p && r != q) + q = r; + if (r == p1) { + // q is a child of p1 and an ancestor of p2 or p2 itself + inode_lock_nested(p1->d_inode, I_MUTEX_PARENT); + inode_lock_nested(p2->d_inode, I_MUTEX_PARENT2); + return q; + } else if (likely(r == p)) { + // both p2 and p1 are descendents of p + inode_lock_nested(p1->d_inode, I_MUTEX_PARENT); + inode_lock_nested(p2->d_inode, I_MUTEX_PARENT2); + return NULL; + } else { // no common ancestor at the time we'd been called + mutex_unlock(&p1->d_sb->s_vfs_rename_mutex); + return ERR_PTR(-EXDEV); + } +} + +/* + * p1 and p2 should be directories on the same fs. + */ +static struct dentry *lock_rename(struct dentry *p1, struct dentry *p2) +{ + if (p1 == p2) { + inode_lock_nested(p1->d_inode, I_MUTEX_PARENT); + return NULL; + } + + mutex_lock(&p1->d_sb->s_vfs_rename_mutex); + return lock_two_directories(p1, p2); +} + +/* + * c1 and p2 should be on the same fs. + */ +static struct dentry *lock_rename_child(struct dentry *c1, struct dentry *p2) +{ + if (READ_ONCE(c1->d_parent) == p2) { + /* + * hopefully won't need to touch ->s_vfs_rename_mutex at all. + */ + inode_lock_nested(p2->d_inode, I_MUTEX_PARENT); + /* + * now that p2 is locked, nobody can move in or out of it, + * so the test below is safe. + */ + if (likely(c1->d_parent == p2)) + return NULL; + + /* + * c1 got moved out of p2 while we'd been taking locks; + * unlock and fall back to slow case. + */ + inode_unlock(p2->d_inode); + } + + mutex_lock(&c1->d_sb->s_vfs_rename_mutex); + /* + * nobody can move out of any directories on this fs. + */ + if (likely(c1->d_parent != p2)) + return lock_two_directories(c1->d_parent, p2); + + /* + * c1 got moved into p2 while we were taking locks; + * we need p2 locked and ->s_vfs_rename_mutex unlocked, + * for consistency with lock_rename(). + */ + inode_lock_nested(p2->d_inode, I_MUTEX_PARENT); + mutex_unlock(&c1->d_sb->s_vfs_rename_mutex); + return NULL; +} + +static void unlock_rename(struct dentry *p1, struct dentry *p2) +{ + inode_unlock(p1->d_inode); + if (p1 != p2) { + inode_unlock(p2->d_inode); + mutex_unlock(&p1->d_sb->s_vfs_rename_mutex); + } +} + +/** + * __start_renaming - lookup and lock names for rename + * @rd: rename data containing parents and flags, and + * for receiving found dentries + * @lookup_flags: extra flags to pass to ->lookup (e.g. LOOKUP_REVAL, + * LOOKUP_NO_SYMLINKS etc). + * @old_last: name of object in @rd.old_parent + * @new_last: name of object in @rd.new_parent + * + * Look up two names and ensure locks are in place for + * rename. + * + * On success the found dentries are stored in @rd.old_dentry, + * @rd.new_dentry and an extra ref is taken on @rd.old_parent. + * These references and the lock are dropped by end_renaming(). + * + * The passed in qstrs must have the hash calculated, and no permission + * checking is performed. + * + * Returns: zero or an error. + */ +static int +__start_renaming(struct renamedata *rd, int lookup_flags, + struct qstr *old_last, struct qstr *new_last) +{ + struct dentry *trap; + struct dentry *d1, *d2; + int target_flags = LOOKUP_RENAME_TARGET | LOOKUP_CREATE; + int err; + + if (rd->flags & RENAME_EXCHANGE) + target_flags = 0; + if (rd->flags & RENAME_NOREPLACE) + target_flags |= LOOKUP_EXCL; + + trap = lock_rename(rd->old_parent, rd->new_parent); + if (IS_ERR(trap)) + return PTR_ERR(trap); + + d1 = lookup_one_qstr_excl(old_last, rd->old_parent, + lookup_flags); + err = PTR_ERR(d1); + if (IS_ERR(d1)) + goto out_unlock; + + d2 = lookup_one_qstr_excl(new_last, rd->new_parent, + lookup_flags | target_flags); + err = PTR_ERR(d2); + if (IS_ERR(d2)) + goto out_dput_d1; + + if (d1 == trap) { + /* source is an ancestor of target */ + err = -EINVAL; + goto out_dput_d2; + } + + if (d2 == trap) { + /* target is an ancestor of source */ + if (rd->flags & RENAME_EXCHANGE) + err = -EINVAL; + else + err = -ENOTEMPTY; + goto out_dput_d2; + } + + rd->old_dentry = d1; + rd->new_dentry = d2; + dget(rd->old_parent); + return 0; + +out_dput_d2: + dput(d2); +out_dput_d1: + dput(d1); +out_unlock: + unlock_rename(rd->old_parent, rd->new_parent); + return err; +} + +/** + * start_renaming - lookup and lock names for rename with permission checking + * @rd: rename data containing parents and flags, and + * for receiving found dentries + * @lookup_flags: extra flags to pass to ->lookup (e.g. LOOKUP_REVAL, + * LOOKUP_NO_SYMLINKS etc). + * @old_last: name of object in @rd.old_parent + * @new_last: name of object in @rd.new_parent + * + * Look up two names and ensure locks are in place for + * rename. + * + * On success the found dentries are stored in @rd.old_dentry, + * @rd.new_dentry. Also the refcount on @rd->old_parent is increased. + * These references and the lock are dropped by end_renaming(). + * + * The passed in qstrs need not have the hash calculated, and basic + * eXecute permission checking is performed against @rd.mnt_idmap. + * + * Returns: zero or an error. + */ +int start_renaming(struct renamedata *rd, int lookup_flags, + struct qstr *old_last, struct qstr *new_last) +{ + int err; + + err = lookup_one_common(rd->mnt_idmap, old_last, rd->old_parent); + if (err) + return err; + err = lookup_one_common(rd->mnt_idmap, new_last, rd->new_parent); + if (err) + return err; + return __start_renaming(rd, lookup_flags, old_last, new_last); +} +EXPORT_SYMBOL(start_renaming); + +static int +__start_renaming_dentry(struct renamedata *rd, int lookup_flags, + struct dentry *old_dentry, struct qstr *new_last) +{ + struct dentry *trap; + struct dentry *d2; + int target_flags = LOOKUP_RENAME_TARGET | LOOKUP_CREATE; + int err; + + if (rd->flags & RENAME_EXCHANGE) + target_flags = 0; + if (rd->flags & RENAME_NOREPLACE) + target_flags |= LOOKUP_EXCL; + + /* Already have the dentry - need to be sure to lock the correct parent */ + trap = lock_rename_child(old_dentry, rd->new_parent); + if (IS_ERR(trap)) + return PTR_ERR(trap); + if (d_unhashed(old_dentry) || + (rd->old_parent && rd->old_parent != old_dentry->d_parent)) { + /* dentry was removed, or moved and explicit parent requested */ + err = -EINVAL; + goto out_unlock; + } + + d2 = lookup_one_qstr_excl(new_last, rd->new_parent, + lookup_flags | target_flags); + err = PTR_ERR(d2); + if (IS_ERR(d2)) + goto out_unlock; + + if (old_dentry == trap) { + /* source is an ancestor of target */ + err = -EINVAL; + goto out_dput_d2; + } + + if (d2 == trap) { + /* target is an ancestor of source */ + if (rd->flags & RENAME_EXCHANGE) + err = -EINVAL; + else + err = -ENOTEMPTY; + goto out_dput_d2; + } + + rd->old_dentry = dget(old_dentry); + rd->new_dentry = d2; + rd->old_parent = dget(old_dentry->d_parent); + return 0; + +out_dput_d2: + dput(d2); +out_unlock: + unlock_rename(old_dentry->d_parent, rd->new_parent); + return err; +} + +/** + * start_renaming_dentry - lookup and lock name for rename with permission checking + * @rd: rename data containing parents and flags, and + * for receiving found dentries + * @lookup_flags: extra flags to pass to ->lookup (e.g. LOOKUP_REVAL, + * LOOKUP_NO_SYMLINKS etc). + * @old_dentry: dentry of name to move + * @new_last: name of target in @rd.new_parent + * + * Look up target name and ensure locks are in place for + * rename. + * + * On success the found dentry is stored in @rd.new_dentry and + * @rd.old_parent is confirmed to be the parent of @old_dentry. If it + * was originally %NULL, it is set. In either case a reference is taken + * so that end_renaming() can have a stable reference to unlock. + * + * References and the lock can be dropped with end_renaming() + * + * The passed in qstr need not have the hash calculated, and basic + * eXecute permission checking is performed against @rd.mnt_idmap. + * + * Returns: zero or an error. + */ +int start_renaming_dentry(struct renamedata *rd, int lookup_flags, + struct dentry *old_dentry, struct qstr *new_last) +{ + int err; + + err = lookup_one_common(rd->mnt_idmap, new_last, rd->new_parent); + if (err) + return err; + return __start_renaming_dentry(rd, lookup_flags, old_dentry, new_last); +} +EXPORT_SYMBOL(start_renaming_dentry); + +/** + * start_renaming_two_dentries - Lock to dentries in given parents for rename + * @rd: rename data containing parent + * @old_dentry: dentry of name to move + * @new_dentry: dentry to move to + * + * Ensure locks are in place for rename and check parentage is still correct. + * + * On success the two dentries are stored in @rd.old_dentry and + * @rd.new_dentry and @rd.old_parent and @rd.new_parent are confirmed to + * be the parents of the dentries. + * + * References and the lock can be dropped with end_renaming() + * + * Returns: zero or an error. + */ +int +start_renaming_two_dentries(struct renamedata *rd, + struct dentry *old_dentry, struct dentry *new_dentry) +{ + struct dentry *trap; + int err; + + /* Already have the dentry - need to be sure to lock the correct parent */ + trap = lock_rename_child(old_dentry, rd->new_parent); + if (IS_ERR(trap)) + return PTR_ERR(trap); + err = -EINVAL; + if (d_unhashed(old_dentry) || + (rd->old_parent && rd->old_parent != old_dentry->d_parent)) + /* old_dentry was removed, or moved and explicit parent requested */ + goto out_unlock; + if (d_unhashed(new_dentry) || + rd->new_parent != new_dentry->d_parent) + /* new_dentry was removed or moved */ + goto out_unlock; + + if (old_dentry == trap) + /* source is an ancestor of target */ + goto out_unlock; + + if (new_dentry == trap) { + /* target is an ancestor of source */ + if (rd->flags & RENAME_EXCHANGE) + err = -EINVAL; + else + err = -ENOTEMPTY; + goto out_unlock; + } + + err = -EEXIST; + if (d_is_positive(new_dentry) && (rd->flags & RENAME_NOREPLACE)) + goto out_unlock; + + rd->old_dentry = dget(old_dentry); + rd->new_dentry = dget(new_dentry); + rd->old_parent = dget(old_dentry->d_parent); + return 0; + +out_unlock: + unlock_rename(old_dentry->d_parent, rd->new_parent); + return err; +} +EXPORT_SYMBOL(start_renaming_two_dentries); + +void end_renaming(struct renamedata *rd) +{ + unlock_rename(rd->old_parent, rd->new_parent); + dput(rd->old_dentry); + dput(rd->new_dentry); + dput(rd->old_parent); +} +EXPORT_SYMBOL(end_renaming); + +/** + * vfs_prepare_mode - prepare the mode to be used for a new inode + * @idmap: idmap of the mount the inode was found from + * @dir: parent directory of the new inode + * @mode: mode of the new inode + * @mask_perms: allowed permission by the vfs + * @type: type of file to be created + * + * This helper consolidates and enforces vfs restrictions on the @mode of a new + * object to be created. + * + * Umask stripping depends on whether the filesystem supports POSIX ACLs (see + * the kernel documentation for mode_strip_umask()). Moving umask stripping + * after setgid stripping allows the same ordering for both non-POSIX ACL and + * POSIX ACL supporting filesystems. + * + * Returns: mode to be passed to the filesystem + */ +static inline umode_t vfs_prepare_mode(struct mnt_idmap *idmap, + const struct inode *dir, umode_t mode, + umode_t mask_perms, umode_t type) +{ + mode = mode_strip_sgid(idmap, dir, mode); + mode = mode_strip_umask(dir, mode); + + /* + * Apply the vfs mandated allowed permission mask and set the type of + * file to be created before we call into the filesystem. + */ + mode &= (mask_perms & ~S_IFMT); + mode |= (type & S_IFMT); + + return mode; +} + +/** + * vfs_create - create new file + * @idmap: idmap of the mount the inode was found from + * @dentry: dentry of the child file + * @mode: mode of the child file + * @di: returns parent inode, if the inode is delegated. + * + * Create a new file. + * + * If the inode has been found through an idmapped mount the idmap of + * the vfsmount must be passed through @idmap. This function will then take + * care to map the inode according to @idmap before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply pass @nop_mnt_idmap. + */ +int vfs_create(struct mnt_idmap *idmap, struct dentry *dentry, umode_t mode, + struct delegated_inode *di) +{ + struct inode *dir = d_inode(dentry->d_parent); + int error; + + error = may_create_dentry(idmap, dir, dentry); + if (error) + return error; + + if (!dir->i_op->create) + return -EACCES; /* shouldn't it be ENOSYS? */ + + mode = vfs_prepare_mode(idmap, dir, mode, S_IALLUGO, S_IFREG); + error = security_inode_create(dir, dentry, mode); + if (error) + return error; + error = try_break_deleg(dir, LEASE_BREAK_DIR_CREATE, di); + if (error) + return error; + error = dir->i_op->create(idmap, dir, dentry, mode); + if (!error) + fsnotify_create(dir, dentry); + return error; +} +EXPORT_SYMBOL(vfs_create); + +int vfs_mkobj(struct dentry *dentry, umode_t mode, + int (*f)(struct dentry *, umode_t, void *), + void *arg) +{ + struct inode *dir = dentry->d_parent->d_inode; + int error = may_create_dentry(&nop_mnt_idmap, dir, dentry); + if (error) + return error; + + mode &= S_IALLUGO; + mode |= S_IFREG; + error = security_inode_create(dir, dentry, mode); + if (error) + return error; + error = f(dentry, mode, arg); + if (!error) + fsnotify_create(dir, dentry); + return error; +} +EXPORT_SYMBOL(vfs_mkobj); + +bool may_open_dev(const struct path *path) +{ + return !(path->mnt->mnt_flags & MNT_NODEV) && + !(path->mnt->mnt_sb->s_iflags & SB_I_NODEV); +} + +static int may_open(struct mnt_idmap *idmap, const struct path *path, + int acc_mode, int flag) +{ + struct dentry *dentry = path->dentry; + struct inode *inode = dentry->d_inode; + int error; + + if (!inode) + return -ENOENT; + + switch (inode->i_mode & S_IFMT) { + case S_IFLNK: + return -ELOOP; + case S_IFDIR: + if (acc_mode & MAY_WRITE) + return -EISDIR; + if (acc_mode & MAY_EXEC) + return -EACCES; + break; + case S_IFBLK: + case S_IFCHR: + if (!may_open_dev(path)) + return -EACCES; + fallthrough; + case S_IFIFO: + case S_IFSOCK: + if (acc_mode & MAY_EXEC) + return -EACCES; + flag &= ~O_TRUNC; + break; + case S_IFREG: + if ((acc_mode & MAY_EXEC) && path_noexec(path)) + return -EACCES; + break; + default: + VFS_BUG_ON_INODE(!IS_ANON_FILE(inode), inode); + } + + error = inode_permission(idmap, inode, MAY_OPEN | acc_mode); + if (error) + return error; + + /* + * An append-only file must be opened in append mode for writing. + */ + if (IS_APPEND(inode)) { + if ((flag & O_ACCMODE) != O_RDONLY && !(flag & O_APPEND)) + return -EPERM; + if (flag & O_TRUNC) + return -EPERM; + } + + /* O_NOATIME can only be set by the owner or superuser */ + if (flag & O_NOATIME && !inode_owner_or_capable(idmap, inode)) + return -EPERM; + + return 0; +} + +static int handle_truncate(struct mnt_idmap *idmap, struct file *filp) +{ + const struct path *path = &filp->f_path; + struct inode *inode = path->dentry->d_inode; + int error = get_write_access(inode); + if (error) + return error; + + error = security_file_truncate(filp); + if (!error) { + error = do_truncate(idmap, path->dentry, 0, + ATTR_MTIME|ATTR_CTIME|ATTR_OPEN, + filp); + } + put_write_access(inode); + return error; +} + +static inline int open_to_namei_flags(int flag) +{ + if ((flag & O_ACCMODE) == 3) + flag--; + return flag; +} + +static int may_o_create(struct mnt_idmap *idmap, + const struct path *dir, struct dentry *dentry, + umode_t mode) +{ + int error = security_path_mknod(dir, dentry, mode, 0); + if (error) + return error; + + if (!fsuidgid_has_mapping(dir->dentry->d_sb, idmap)) + return -EOVERFLOW; + + error = inode_permission(idmap, dir->dentry->d_inode, + MAY_WRITE | MAY_EXEC); + if (error) + return error; + + return security_inode_create(dir->dentry->d_inode, dentry, mode); +} + +/** + * atomic_open() - atomically look up, create and open a file + * @path: parent directory path + * @dentry: child to ->atomic_open() + * @file: file to attach child to + * @open_flag: open flags + * @mode: create mode + * @create_error: return value from may_o_create() + * + * Attempt to look up, create and open @dentry, which must be negative, in a + * single call into the filesystem. + * + * If a non-error dentry is returned then: when FMODE_OPENED is set, + * the file will have been attached to @file by the filesystem calling + * finish_open(). If FMODE_OPENED isn't set, the filesystem instead called + * finish_no_open() and the caller will need to perform the open themselves. + * + * FMODE_CREATED is set when the call to ->atomic_open() actually created + * the file. + * + * Returns: the opened or looked-up dentry, or ERR_PTR() on failure. The + * reference to @dentry is consumed in either case. + */ +static struct dentry *atomic_open(const struct path *path, struct dentry *dentry, + struct file *file, + int open_flag, umode_t mode, int create_error) +{ + struct dentry *const DENTRY_NOT_SET = (void *) -1UL; + struct inode *dir_inode = path->dentry->d_inode; + int error; + + file->__f_path.dentry = DENTRY_NOT_SET; + file->__f_path.mnt = path->mnt; + error = dir_inode->i_op->atomic_open(dir_inode, dentry, file, + open_to_namei_flags(open_flag), mode); + d_lookup_done(dentry); + + if (!error) { + if (file->f_mode & FMODE_OPENED) { + /* finish_open() called */ + struct dentry *opened = file->f_path.dentry; + + if (unlikely(opened != dentry)) { + dput(dentry); + dentry = dget(opened); + } + } else if (likely(file->f_path.dentry != DENTRY_NOT_SET)) { + /* finish_no_open() called */ + struct dentry *replaced = file->f_path.dentry; + + if (replaced) { + dput(dentry); + dentry = replaced; + } + if (unlikely(d_is_negative(dentry))) + error = -ENOENT; + } else { + const char *fsname = dentry->d_sb->s_type->name; + + WARN(1, "%s: ->atomic_open() left file->f_path.dentry unset!\n", + fsname); + error = -EIO; + } + } + + if (error) { + if (unlikely(create_error) && error == -ENOENT) { + /* + * Should have done a create, but errored before. + * Some filesystems return -ENOENT directly instead of + * calling finish_no_open() with a negative dentry; + * either way it should only mean the child doesn't exist, + * so a refused create is safe to record here. + */ + audit_inode_child(dir_inode, dentry, AUDIT_TYPE_CHILD_CREATE); + error = create_error; + } + dput(dentry); + dentry = ERR_PTR(error); + } + return dentry; +} + +/* + * Look up and maybe create and open the last component. + * + * Takes the parent inode lock itself, exclusive if O_CREAT was requested and + * shared otherwise, and drops it again before returning. The caller must not + * hold it. + * + * On success returns the dentry of the last component. If FMODE_OPENED is set + * on file->f_mode the file was also opened and attached to @file; otherwise + * only lookup and creation were performed and the caller has to open it. In + * the latter case the dentry may be negative if O_CREAT hadn't been specified. + * + * Returns ERR_PTR() on failure. + */ +static struct dentry *lookup_open(struct nameidata *nd, struct file *file, + const struct open_flags *op) +{ + struct delegated_inode delegated_inode = { }; + struct mnt_idmap *idmap; + struct dentry *dir = nd->path.dentry; + struct inode *dir_inode = dir->d_inode; + int open_flag; + struct dentry *dentry; + int error, create_error; + umode_t mode; + bool got_write; + +retry: + open_flag = op->open_flag; + got_write = false; + mode = op->mode; + create_error = 0; + + if (open_flag & (O_CREAT | O_TRUNC | O_WRONLY | O_RDWR)) { + got_write = !mnt_want_write(nd->path.mnt); + /* + * do _not_ fail yet - we might not need that or fail with + * a different error; we'll be dropping this one anyway. + */ + } + if (open_flag & O_CREAT) + inode_lock(dir_inode); + else + inode_lock_shared(dir_inode); + + if (unlikely(IS_DEADDIR(dir_inode))) { + dentry = ERR_PTR(-ENOENT); + goto out; + } + + file->f_mode &= ~FMODE_CREATED; + dentry = d_lookup(dir, &nd->last); + for (;;) { + if (!dentry) { + dentry = d_alloc_parallel(dir, &nd->last); + if (IS_ERR(dentry)) + goto out; + } + if (d_in_lookup(dentry)) + break; + + error = d_revalidate(dir_inode, &nd->last, dentry, nd->flags); + if (likely(error > 0)) + break; + if (error) + goto out_dput; + d_invalidate(dentry); + dput(dentry); + dentry = NULL; + } + if (dentry->d_inode) { + /* Cached positive dentry: will open in do_open(). */ + goto out; + } + + if (open_flag & O_CREAT) + audit_inode(nd->name, dir, AUDIT_INODE_PARENT); + + /* + * Checking write permission is tricky, bacuse we don't know if we are + * going to actually need it: O_CREAT opens should work as long as the + * file exists. But checking existence breaks atomicity. The trick is + * to check access and if not granted clear O_CREAT from the flags. + * + * Another problem is returing the "right" error value (e.g. for an + * O_EXCL open we want to return EEXIST not EROFS). + */ + if (unlikely(!got_write)) + open_flag &= ~O_TRUNC; + idmap = mnt_idmap(nd->path.mnt); + if (open_flag & O_CREAT) { + if (open_flag & O_EXCL) + open_flag &= ~O_TRUNC; + mode = vfs_prepare_mode(idmap, dir_inode, mode, mode, mode); + if (likely(got_write)) + create_error = may_o_create(idmap, &nd->path, + dentry, mode); + else + create_error = -EROFS; + } + if (create_error) + open_flag &= ~O_CREAT; + if (dir_inode->i_op->atomic_open) { + if (nd->flags & LOOKUP_DIRECTORY) + open_flag |= O_DIRECTORY; + dentry = atomic_open(&nd->path, dentry, file, open_flag, mode, + create_error); + goto out; + } + + if (d_in_lookup(dentry)) { + struct dentry *res = dir_inode->i_op->lookup(dir_inode, dentry, + nd->flags); + d_lookup_done(dentry); + if (unlikely(res)) { + if (IS_ERR(res)) { + error = PTR_ERR(res); + goto out_dput; + } + dput(dentry); + dentry = res; + } + } + if (dentry->d_inode || !(op->open_flag & O_CREAT)) { + /* + * No need to create a file. If lookup returned a positive + * dentry, the file will be opened in do_open(). + */ + goto out; + } + + /* Negative dentry with O_CREAT flag set */ + audit_inode_child(dir_inode, dentry, AUDIT_TYPE_CHILD_CREATE); + + if (unlikely(create_error)) { + /* should have done a create, but we already errored */ + error = create_error; + goto out_dput; + } + + error = try_break_deleg(dir_inode, LEASE_BREAK_DIR_CREATE, &delegated_inode); + if (error) + goto out_dput; + + file->f_mode |= FMODE_CREATED; + if (!dir_inode->i_op->create) { + error = -EACCES; + goto out_dput; + } + + error = dir_inode->i_op->create(idmap, dir_inode, dentry, mode); + if (error) + goto out_dput; +out: + if (!IS_ERR(dentry)) { + if (file->f_mode & FMODE_CREATED) + fsnotify_create(dir_inode, dentry); + if (file->f_mode & FMODE_OPENED) + fsnotify_open(file); + } + if ((open_flag & O_CREAT) || create_error) + inode_unlock(dir_inode); + else + inode_unlock_shared(dir_inode); + + if (got_write) + mnt_drop_write(nd->path.mnt); + + if (is_delegated(&delegated_inode)) { + /* Must have come through out_dput: dentry is an ERR_PTR() */ + error = break_deleg_wait(&delegated_inode); + + if (!error) + goto retry; + dentry = ERR_PTR(error); + } + + return dentry; + +out_dput: + dput(dentry); + dentry = ERR_PTR(error); + goto out; +} + +/** + * vfs_lookup_open - open and possibly create a regular file + * @parent: directory to contain file + * @last: final component of file name + * @open_flag: O_flags + * @mode: initial permissions for file + * + * Open a file after lookup and/or create. This provides similar + * functionality to open_last_lookups() for non-VFS users, particularly + * nfsd. + * It uses ->atomic_open or ->lookup / ->create / ->open as appropriate. + * + * If the fs object found is not a regular file then an error is returned. + * In some cases, related errors are repurposed so that the caller can + * determine the type of file found from the error. + * -EISDIR : a directory was found + * -ELOOP : a symlink was found + * -ENODEV : a block or character device special file was found + * -EFTYPE : any other non-regular file was found, such as FIFO or SOCK. + * or ->atomic_open responded to __O_REGULAR. + * + * Returns: the opened struct file, or an error. + */ +struct file *vfs_lookup_open(struct path *parent, struct qstr *last, + int open_flag, umode_t mode) +{ + struct file *file __free(fput) = NULL; + struct nameidata nd = {}; + struct open_flags op = {}; + struct dentry *dentry; + int error = 0; + + WARN_ONCE(mode & ~S_IALLUGO, "mode must only have permission bits"); + WARN_ONCE(open_flag & ~(O_ACCMODE|O_CREAT|O_EXCL|O_TRUNC|__O_REGULAR), + "open_flag has unsupported flags"); + + mode |= S_IFREG; + open_flag |= __O_REGULAR; + + error = lookup_noperm_common(last, parent->dentry); + if (error) + return ERR_PTR(error); + + file = alloc_empty_file(open_flag, current_cred()); + if (IS_ERR(file)) + return file; + + nd.path = *parent; + nd.last = *last; + nd.flags = LOOKUP_OPEN; + if (open_flag & O_CREAT) { + nd.flags |= LOOKUP_CREATE; + if (open_flag & O_EXCL) + nd.flags |= LOOKUP_EXCL; + } + op.open_flag = open_flag; + op.mode = mode; + dentry = lookup_open(&nd, file, &op); + + if (IS_ERR(dentry)) + return ERR_CAST(dentry); + + if (d_really_is_negative(dentry)) { + error = -ENOENT; + } else if (!(file->f_mode & FMODE_CREATED) && (open_flag & O_EXCL)) { + error = -EEXIST; + } else if ((dentry->d_inode->i_mode & S_IFMT) != S_IFREG) { + switch (dentry->d_inode->i_mode & S_IFMT) { + case S_IFDIR: + error = -EISDIR; + break; + case S_IFLNK: + error = -ELOOP; + break; + case S_IFBLK: + case S_IFCHR: + error = -ENODEV; + break; + case S_IFIFO: + case S_IFSOCK: + default: + error = -EFTYPE; + break; + } + } else if (!(file->f_mode & FMODE_OPENED)) { + nd.path.dentry = dentry; + error = vfs_open(&nd.path, file); + } + dput(dentry); + + if (error) + return ERR_PTR(error); + return no_free_ptr(file); +} +EXPORT_SYMBOL_FOR_MODULES(vfs_lookup_open, "nfsd"); + +static inline bool trailing_slashes(struct nameidata *nd) +{ + return (bool)nd->last.name[nd->last.len]; +} + +static struct dentry *lookup_fast_for_open(struct nameidata *nd, int open_flag) +{ + struct dentry *dentry; + + if (open_flag & O_CREAT) { + if (trailing_slashes(nd)) + return ERR_PTR(-EISDIR); + + /* Don't bother on an O_EXCL create */ + if (open_flag & O_EXCL) + return NULL; + } + + if (trailing_slashes(nd)) + nd->flags |= LOOKUP_FOLLOW | LOOKUP_DIRECTORY; + + dentry = lookup_fast(nd); + if (IS_ERR_OR_NULL(dentry)) + return dentry; + + if (open_flag & O_CREAT) { + /* Discard negative dentries. Need inode_lock to do the create */ + if (!dentry->d_inode) { + if (!(nd->flags & LOOKUP_RCU)) + dput(dentry); + dentry = NULL; + } + } + return dentry; +} + +static const char *open_last_lookups(struct nameidata *nd, + struct file *file, const struct open_flags *op) +{ + int open_flag = op->open_flag; + struct dentry *dentry; + const char *res; + + nd->flags |= op->intent; + + if (nd->last_type != LAST_NORM) { + if (nd->depth) + put_link(nd); + return handle_dots(nd, nd->last_type); + } + + /* We _can_ be in RCU mode here */ + dentry = lookup_fast_for_open(nd, open_flag); + if (IS_ERR(dentry)) + return ERR_CAST(dentry); + + if (likely(dentry)) + goto finish_lookup; + + if (!(open_flag & O_CREAT)) { + if (WARN_ON_ONCE(nd->flags & LOOKUP_RCU)) + return ERR_PTR(-ECHILD); + } else { + if (nd->flags & LOOKUP_RCU) { + if (!try_to_unlazy(nd)) + return ERR_PTR(-ECHILD); + } + } + + dentry = lookup_open(nd, file, op); + if (IS_ERR(dentry)) + return ERR_CAST(dentry); + + if (file->f_mode & (FMODE_OPENED | FMODE_CREATED)) { + dput(nd->path.dentry); + nd->path.dentry = dentry; + return NULL; + } + +finish_lookup: + if (nd->depth) + put_link(nd); + res = step_into(nd, WALK_TRAILING, dentry); + if (unlikely(res)) + nd->flags &= ~(LOOKUP_OPEN|LOOKUP_CREATE|LOOKUP_EXCL); + return res; +} + +/* + * Handle the last step of open() + */ +static int do_open(struct nameidata *nd, + struct file *file, const struct open_flags *op) +{ + struct mnt_idmap *idmap; + int open_flag = op->open_flag; + bool do_truncate; + int acc_mode; + int error; + + if (!(file->f_mode & (FMODE_OPENED | FMODE_CREATED))) { + error = complete_walk(nd); + if (error) + return error; + } + if (!(file->f_mode & FMODE_CREATED)) + audit_inode(nd->name, nd->path.dentry, 0); + idmap = mnt_idmap(nd->path.mnt); + if (open_flag & O_CREAT) { + if ((open_flag & O_EXCL) && !(file->f_mode & FMODE_CREATED)) + return -EEXIST; + if (d_is_dir(nd->path.dentry)) + return -EISDIR; + error = may_create_in_sticky(idmap, nd, + d_backing_inode(nd->path.dentry)); + if (unlikely(error)) + return error; + } + + if ((open_flag & __O_REGULAR) && !d_is_reg(nd->path.dentry)) + return -EFTYPE; + + if ((nd->flags & LOOKUP_DIRECTORY) && !d_can_lookup(nd->path.dentry)) + return -ENOTDIR; + + do_truncate = false; + acc_mode = op->acc_mode; + if (file->f_mode & FMODE_CREATED) { + /* Don't check for write permission, don't truncate */ + open_flag &= ~O_TRUNC; + acc_mode = 0; + } else if (d_is_reg(nd->path.dentry) && open_flag & O_TRUNC) { + error = mnt_want_write(nd->path.mnt); + if (error) + return error; + do_truncate = true; + } + error = may_open(idmap, &nd->path, acc_mode, open_flag); + if (!error && !(file->f_mode & FMODE_OPENED)) + error = vfs_open(&nd->path, file); + if (!error) + error = security_file_post_open(file, op->acc_mode); + if (!error && do_truncate) + error = handle_truncate(idmap, file); + if (unlikely(error > 0)) { + WARN_ON(1); + error = -EINVAL; + } + if (do_truncate) + mnt_drop_write(nd->path.mnt); + return error; +} + +/** + * vfs_tmpfile - create tmpfile + * @idmap: idmap of the mount the inode was found from + * @parentpath: pointer to the path of the base directory + * @file: file descriptor of the new tmpfile + * @mode: mode of the new tmpfile + * + * Create a temporary file. + * + * If the inode has been found through an idmapped mount the idmap of + * the vfsmount must be passed through @idmap. This function will then take + * care to map the inode according to @idmap before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply pass @nop_mnt_idmap. + */ +int vfs_tmpfile(struct mnt_idmap *idmap, + const struct path *parentpath, + struct file *file, umode_t mode) +{ + struct dentry *child; + struct inode *dir = d_inode(parentpath->dentry); + struct inode *inode; + int error; + int open_flag = file->f_flags; + + /* A tmpfile is I_LINKABLE, so guard its owner like may_o_create(). */ + if (!fsuidgid_has_mapping(dir->i_sb, idmap)) + return -EOVERFLOW; + + /* we want directory to be writable */ + error = inode_permission(idmap, dir, MAY_WRITE | MAY_EXEC); + if (error) + return error; + if (!dir->i_op->tmpfile) + return -EOPNOTSUPP; + child = d_alloc(parentpath->dentry, &slash_name); + if (unlikely(!child)) + return -ENOMEM; + file->__f_path.mnt = parentpath->mnt; + file->__f_path.dentry = child; + mode = vfs_prepare_mode(idmap, dir, mode, mode, mode); + error = dir->i_op->tmpfile(idmap, dir, file, mode); + dput(child); + if (file->f_mode & FMODE_OPENED) + fsnotify_open(file); + if (error) + return error; + /* Don't check for other permissions, the inode was just created */ + error = may_open(idmap, &file->f_path, 0, file->f_flags); + if (error) + return error; + inode = file_inode(file); + if (!(open_flag & O_EXCL)) { + spin_lock(&inode->i_lock); + inode_state_set(inode, I_LINKABLE); + spin_unlock(&inode->i_lock); + } + security_inode_post_create_tmpfile(idmap, inode); + return 0; +} + +/** + * kernel_tmpfile_open - open a tmpfile for kernel internal use + * @idmap: idmap of the mount the inode was found from + * @parentpath: path of the base directory + * @mode: mode of the new tmpfile + * @open_flag: flags + * @cred: credentials for open + * + * Create and open a temporary file. The file is not accounted in nr_files, + * hence this is only for kernel internal use, and must not be installed into + * file tables or such. + */ +struct file *kernel_tmpfile_open(struct mnt_idmap *idmap, + const struct path *parentpath, + umode_t mode, int open_flag, + const struct cred *cred) +{ + struct file *file; + int error; + + file = alloc_empty_file_noaccount(open_flag, cred); + if (IS_ERR(file)) + return file; + + error = vfs_tmpfile(idmap, parentpath, file, mode); + if (error) { + fput(file); + file = ERR_PTR(error); + } + return file; +} +EXPORT_SYMBOL(kernel_tmpfile_open); + +static int do_tmpfile(struct nameidata *nd, unsigned flags, + const struct open_flags *op, + struct file *file) +{ + struct path path; + int error = path_lookupat(nd, flags | LOOKUP_DIRECTORY, &path); + + if (unlikely(error)) + return error; + error = mnt_want_write(path.mnt); + if (unlikely(error)) + goto out; + error = vfs_tmpfile(mnt_idmap(path.mnt), &path, file, op->mode); + if (error) + goto out2; + audit_inode(nd->name, file->f_path.dentry, 0); +out2: + mnt_drop_write(path.mnt); +out: + path_put(&path); + return error; +} + +static int do_o_path(struct nameidata *nd, unsigned flags, struct file *file) +{ + struct path path; + int error = path_lookupat(nd, flags, &path); + if (!error) { + audit_inode(nd->name, path.dentry, 0); + error = vfs_open(&path, file); + path_put(&path); + } + return error; +} + +static struct file *path_openat(struct nameidata *nd, + const struct open_flags *op, unsigned flags) +{ + struct file *file; + int error; + + file = alloc_empty_file(op->open_flag, current_cred()); + if (IS_ERR(file)) + return file; + + if (unlikely(file->f_flags & __O_TMPFILE)) { + error = do_tmpfile(nd, flags, op, file); + } else if (unlikely(file->f_flags & O_PATH)) { + error = do_o_path(nd, flags, file); + } else { + const char *s = path_init(nd, flags); + while (!(error = link_path_walk(s, nd)) && + (s = open_last_lookups(nd, file, op)) != NULL) + ; + if (!error) + error = do_open(nd, file, op); + terminate_walk(nd); + } + if (likely(!error)) { + if (likely(file->f_mode & FMODE_OPENED)) + return file; + WARN_ON(1); + error = -EINVAL; + } + fput_close(file); + if (error == -EOPENSTALE) { + if (flags & LOOKUP_RCU) + error = -ECHILD; + else + error = -ESTALE; + } + return ERR_PTR(error); +} + +struct file *do_file_open(int dfd, struct filename *pathname, + const struct open_flags *op) +{ + struct nameidata nd; + int flags = op->lookup_flags; + struct file *filp; + + if (IS_ERR(pathname)) + return ERR_CAST(pathname); + set_nameidata(&nd, dfd, pathname, NULL); + filp = path_openat(&nd, op, flags | LOOKUP_RCU); + if (unlikely(filp == ERR_PTR(-ECHILD))) + filp = path_openat(&nd, op, flags); + if (unlikely(filp == ERR_PTR(-ESTALE))) + filp = path_openat(&nd, op, flags | LOOKUP_REVAL); + restore_nameidata(); + return filp; +} + +struct file *do_file_open_root(const struct path *root, + const char *name, const struct open_flags *op) +{ + struct nameidata nd; + struct file *file; + int flags = op->lookup_flags; + + if (d_is_symlink(root->dentry) && op->intent & LOOKUP_OPEN) + return ERR_PTR(-ELOOP); + + CLASS(filename_kernel, filename)(name); + if (IS_ERR(filename)) + return ERR_CAST(filename); + + set_nameidata(&nd, -1, filename, root); + file = path_openat(&nd, op, flags | LOOKUP_RCU); + if (unlikely(file == ERR_PTR(-ECHILD))) + file = path_openat(&nd, op, flags); + if (unlikely(file == ERR_PTR(-ESTALE))) + file = path_openat(&nd, op, flags | LOOKUP_REVAL); + restore_nameidata(); + return file; +} + +static struct dentry *filename_create(int dfd, struct filename *name, + struct path *path, unsigned int lookup_flags) +{ + struct dentry *dentry = ERR_PTR(-EEXIST); + struct qstr last; + bool want_dir = lookup_flags & LOOKUP_DIRECTORY; + unsigned int reval_flag = lookup_flags & LOOKUP_REVAL; + unsigned int create_flags = LOOKUP_CREATE | LOOKUP_EXCL; + enum last_type type; + int error; + + error = filename_parentat(dfd, name, reval_flag, path, &last, &type); + if (error) + return ERR_PTR(error); + + /* + * Yucky last component or no last component at all? + * (foo/., foo/.., /////) + */ + if (unlikely(type != LAST_NORM)) + goto out; + + /* don't fail immediately if it's r/o, at least try to report other errors */ + error = mnt_want_write(path->mnt); + /* + * Do the final lookup. Suppress 'create' if there is a trailing + * '/', and a directory wasn't requested. + */ + if (last.name[last.len] && !want_dir) + create_flags &= ~LOOKUP_CREATE; + dentry = start_dirop(path->dentry, &last, reval_flag | create_flags); + if (IS_ERR(dentry)) + goto out_drop_write; + + if (unlikely(error)) + goto fail; + + return dentry; +fail: + end_dirop(dentry); + dentry = ERR_PTR(error); +out_drop_write: + if (!error) + mnt_drop_write(path->mnt); +out: + path_put(path); + return dentry; +} + +struct dentry *start_creating_path(int dfd, const char *pathname, + struct path *path, unsigned int lookup_flags) +{ + CLASS(filename_kernel, filename)(pathname); + return filename_create(dfd, filename, path, lookup_flags); +} +EXPORT_SYMBOL(start_creating_path); + +/** + * end_creating_path - finish a code section started by start_creating_path() + * @path: the path instantiated by start_creating_path() + * @dentry: the dentry returned by start_creating_path() + * + * end_creating_path() will unlock and locks taken by start_creating_path() + * and drop an references that were taken. It should only be called + * if start_creating_path() returned a non-error. + * If vfs_mkdir() was called and it returned an error, that error *should* + * be passed to end_creating_path() together with the path. + */ +void end_creating_path(const struct path *path, struct dentry *dentry) +{ + end_creating(dentry); + mnt_drop_write(path->mnt); + path_put(path); +} +EXPORT_SYMBOL(end_creating_path); + +inline struct dentry *start_creating_user_path( + int dfd, const char __user *pathname, + struct path *path, unsigned int lookup_flags) +{ + CLASS(filename, filename)(pathname); + return filename_create(dfd, filename, path, lookup_flags); +} +EXPORT_SYMBOL(start_creating_user_path); + +/** + * dentry_create - Create and open a file + * @path: path to create + * @flags: O\_ flags + * @mode: mode bits for new file + * @cred: credentials to use + * + * Caller must hold the parent directory's lock, and have prepared + * a negative dentry, placed in @path->dentry, for the new file. + * + * Caller sets @path->mnt to the vfsmount of the filesystem where + * the new file is to be created. The parent directory and the + * negative dentry must reside on the same filesystem instance. + * + * On success, returns a ``struct file *``. Otherwise an ERR_PTR + * is returned. + */ +struct file *dentry_create(struct path *path, int flags, umode_t mode, + const struct cred *cred) +{ + struct file *file __free(fput) = NULL; + struct dentry *dentry = path->dentry; + struct dentry *orig_dentry = dentry; + struct dentry *dir = dentry->d_parent; + struct inode *dir_inode = d_inode(dir); + struct mnt_idmap *idmap; + int error, create_error; + + file = alloc_empty_file(flags, cred); + if (IS_ERR(file)) + return file; + + idmap = mnt_idmap(path->mnt); + + if (dir_inode->i_op->atomic_open) { + path->dentry = dir; + mode = vfs_prepare_mode(idmap, dir_inode, mode, S_IALLUGO, S_IFREG); + + create_error = may_o_create(idmap, path, dentry, mode); + if (create_error) + flags &= ~O_CREAT; + + /* atomic_open will dput(dentry) on error */ + dget(orig_dentry); + dentry = atomic_open(path, dentry, file, flags, mode, create_error); + error = PTR_ERR_OR_ZERO(dentry); + + if (IS_ERR(dentry)) + /* keep the original */ + dentry = orig_dentry; + else + /* Drop the extra reference */ + dput(orig_dentry); + + if (!error) { + if (file->f_mode & FMODE_CREATED) + fsnotify_create(dir->d_inode, dentry); + if (file->f_mode & FMODE_OPENED) + fsnotify_open(file); + } + + path->dentry = dentry; + + } else { + error = vfs_create(mnt_idmap(path->mnt), path->dentry, mode, NULL); + if (!error) + error = vfs_open(path, file); + } + if (unlikely(error)) + return ERR_PTR(error); + + return no_free_ptr(file); +} +EXPORT_SYMBOL(dentry_create); + +/** + * vfs_mknod - create device node or file + * @idmap: idmap of the mount the inode was found from + * @dir: inode of the parent directory + * @dentry: dentry of the child device node + * @mode: mode of the child device node + * @dev: device number of device to create + * @delegated_inode: returns parent inode, if the inode is delegated. + * + * Create a device node or file. + * + * If the inode has been found through an idmapped mount the idmap of + * the vfsmount must be passed through @idmap. This function will then take + * care to map the inode according to @idmap before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply pass @nop_mnt_idmap. + */ +int vfs_mknod(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t dev, + struct delegated_inode *delegated_inode) +{ + bool is_whiteout = S_ISCHR(mode) && dev == WHITEOUT_DEV; + int error = may_create_dentry(idmap, dir, dentry); + + if (error) + return error; + + if ((S_ISCHR(mode) || S_ISBLK(mode)) && !is_whiteout && + !capable(CAP_MKNOD)) + return -EPERM; + + if (!dir->i_op->mknod) + return -EPERM; + + mode = vfs_prepare_mode(idmap, dir, mode, mode, mode); + error = devcgroup_inode_mknod(mode, dev); + if (error) + return error; + + error = security_inode_mknod(dir, dentry, mode, dev); + if (error) + return error; + + error = try_break_deleg(dir, LEASE_BREAK_DIR_CREATE, delegated_inode); + if (error) + return error; + + error = dir->i_op->mknod(idmap, dir, dentry, mode, dev); + if (!error) + fsnotify_create(dir, dentry); + return error; +} +EXPORT_SYMBOL(vfs_mknod); + +static int may_mknod(umode_t mode) +{ + switch (mode & S_IFMT) { + case S_IFREG: + case S_IFCHR: + case S_IFBLK: + case S_IFIFO: + case S_IFSOCK: + case 0: /* zero mode translates to S_IFREG */ + return 0; + case S_IFDIR: + return -EPERM; + default: + return -EINVAL; + } +} + +int filename_mknodat(int dfd, struct filename *name, umode_t mode, + unsigned int dev) +{ + struct delegated_inode di = { }; + struct mnt_idmap *idmap; + struct dentry *dentry; + struct path path; + int error; + unsigned int lookup_flags = 0; + + error = may_mknod(mode); + if (error) + return error; +retry: + dentry = filename_create(dfd, name, &path, lookup_flags); + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + + error = security_path_mknod(&path, dentry, + mode_strip_umask(path.dentry->d_inode, mode), dev); + if (error) + goto out2; + + idmap = mnt_idmap(path.mnt); + switch (mode & S_IFMT) { + case 0: case S_IFREG: + error = vfs_create(idmap, dentry, mode, &di); + if (!error) + security_path_post_mknod(idmap, dentry); + break; + case S_IFCHR: case S_IFBLK: + error = vfs_mknod(idmap, path.dentry->d_inode, + dentry, mode, new_decode_dev(dev), &di); + break; + case S_IFIFO: case S_IFSOCK: + error = vfs_mknod(idmap, path.dentry->d_inode, + dentry, mode, 0, &di); + break; + } +out2: + end_creating_path(&path, dentry); + if (is_delegated(&di)) { + error = break_deleg_wait(&di); + if (!error) + goto retry; + } + if (retry_estale(error, lookup_flags)) { + lookup_flags |= LOOKUP_REVAL; + goto retry; + } + return error; +} + +SYSCALL_DEFINE4(mknodat, int, dfd, const char __user *, filename, umode_t, mode, + unsigned int, dev) +{ + CLASS(filename, name)(filename); + return filename_mknodat(dfd, name, mode, dev); +} + +SYSCALL_DEFINE3(mknod, const char __user *, filename, umode_t, mode, unsigned, dev) +{ + CLASS(filename, name)(filename); + return filename_mknodat(AT_FDCWD, name, mode, dev); +} + +/** + * vfs_mkdir - create directory returning correct dentry if possible + * @idmap: idmap of the mount the inode was found from + * @dir: inode of the parent directory + * @dentry: dentry of the child directory + * @mode: mode of the child directory + * @delegated_inode: returns parent inode, if the inode is delegated. + * + * Create a directory. + * + * If the inode has been found through an idmapped mount the idmap of + * the vfsmount must be passed through @idmap. This function will then take + * care to map the inode according to @idmap before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply pass @nop_mnt_idmap. + * + * In the event that the filesystem does not use the *@dentry but leaves it + * negative or unhashes it and possibly splices a different one returning it, + * the original dentry is dput() and the alternate is returned. + * + * In case of an error the dentry is dput() and an ERR_PTR() is returned. + */ +struct dentry *vfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, + struct delegated_inode *delegated_inode) +{ + int error; + unsigned max_links = dir->i_sb->s_max_links; + struct dentry *de; + + error = may_create_dentry(idmap, dir, dentry); + if (error) + goto err; + + error = -EPERM; + if (!dir->i_op->mkdir) + goto err; + + mode = vfs_prepare_mode(idmap, dir, mode, S_IRWXUGO | S_ISVTX, S_IFDIR); + error = security_inode_mkdir(dir, dentry, mode); + if (error) + goto err; + + error = -EMLINK; + if (max_links && dir->i_nlink >= max_links) + goto err; + + error = try_break_deleg(dir, LEASE_BREAK_DIR_CREATE, delegated_inode); + if (error) + goto err; + + de = dir->i_op->mkdir(idmap, dir, dentry, mode); + error = PTR_ERR(de); + if (IS_ERR(de)) + goto err; + if (de) { + dput(dentry); + dentry = de; + } + fsnotify_mkdir(dir, dentry); + return dentry; + +err: + end_creating(dentry); + return ERR_PTR(error); +} +EXPORT_SYMBOL(vfs_mkdir); + +int filename_mkdirat(int dfd, struct filename *name, umode_t mode) +{ + struct dentry *dentry; + struct path path; + int error; + unsigned int lookup_flags = LOOKUP_DIRECTORY; + struct delegated_inode delegated_inode = { }; + +retry: + dentry = filename_create(dfd, name, &path, lookup_flags); + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + + error = security_path_mkdir(&path, dentry, + mode_strip_umask(path.dentry->d_inode, mode)); + if (!error) { + dentry = vfs_mkdir(mnt_idmap(path.mnt), path.dentry->d_inode, + dentry, mode, &delegated_inode); + if (IS_ERR(dentry)) + error = PTR_ERR(dentry); + } + end_creating_path(&path, dentry); + if (is_delegated(&delegated_inode)) { + error = break_deleg_wait(&delegated_inode); + if (!error) + goto retry; + } + if (retry_estale(error, lookup_flags)) { + lookup_flags |= LOOKUP_REVAL; + goto retry; + } + return error; +} + +SYSCALL_DEFINE3(mkdirat, int, dfd, const char __user *, pathname, umode_t, mode) +{ + CLASS(filename, name)(pathname); + return filename_mkdirat(dfd, name, mode); +} + +SYSCALL_DEFINE2(mkdir, const char __user *, pathname, umode_t, mode) +{ + CLASS(filename, name)(pathname); + return filename_mkdirat(AT_FDCWD, name, mode); +} + +/** + * vfs_rmdir - remove directory + * @idmap: idmap of the mount the inode was found from + * @dir: inode of the parent directory + * @dentry: dentry of the child directory + * @delegated_inode: returns parent inode, if it's delegated. + * + * Remove a directory. + * + * If the inode has been found through an idmapped mount the idmap of + * the vfsmount must be passed through @idmap. This function will then take + * care to map the inode according to @idmap before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply pass @nop_mnt_idmap. + */ +int vfs_rmdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, struct delegated_inode *delegated_inode) +{ + int error = may_delete_dentry(idmap, dir, dentry, true); + + if (error) + return error; + + if (!dir->i_op->rmdir) + return -EPERM; + + dget(dentry); + inode_lock(dentry->d_inode); + + error = -EBUSY; + if (is_local_mountpoint(dentry) || + (dentry->d_inode->i_flags & S_KERNEL_FILE)) + goto out; + + error = security_inode_rmdir(dir, dentry); + if (error) + goto out; + + error = try_break_deleg(dir, LEASE_BREAK_DIR_DELETE, delegated_inode); + if (error) + goto out; + + error = dir->i_op->rmdir(dir, dentry); + if (error) + goto out; + + shrink_dcache_parent(dentry); + dentry->d_inode->i_flags |= S_DEAD; + dont_mount(dentry); + detach_mounts(dentry); + +out: + inode_unlock(dentry->d_inode); + dput(dentry); + if (!error) + d_delete_notify(dir, dentry); + return error; +} +EXPORT_SYMBOL(vfs_rmdir); + +int filename_rmdir(int dfd, struct filename *name) +{ + int error; + struct dentry *dentry; + struct path path; + struct qstr last; + enum last_type type; + unsigned int lookup_flags = 0; + struct delegated_inode delegated_inode = { }; +retry: + error = filename_parentat(dfd, name, lookup_flags, &path, &last, &type); + if (error) + return error; + + switch (type) { + case LAST_NORM: + break; + case LAST_DOTDOT: + error = -ENOTEMPTY; + goto exit2; + case LAST_DOT: + error = -EINVAL; + goto exit2; + case LAST_ROOT: + error = -EBUSY; + goto exit2; + } + + error = mnt_want_write(path.mnt); + if (error) + goto exit2; + + dentry = start_dirop(path.dentry, &last, lookup_flags); + error = PTR_ERR(dentry); + if (IS_ERR(dentry)) + goto exit3; + error = security_path_rmdir(&path, dentry); + if (error) + goto exit4; + error = vfs_rmdir(mnt_idmap(path.mnt), path.dentry->d_inode, + dentry, &delegated_inode); +exit4: + end_dirop(dentry); +exit3: + mnt_drop_write(path.mnt); +exit2: + path_put(&path); + if (is_delegated(&delegated_inode)) { + error = break_deleg_wait(&delegated_inode); + if (!error) + goto retry; + } + if (retry_estale(error, lookup_flags)) { + lookup_flags |= LOOKUP_REVAL; + goto retry; + } + return error; +} + +SYSCALL_DEFINE1(rmdir, const char __user *, pathname) +{ + CLASS(filename, name)(pathname); + return filename_rmdir(AT_FDCWD, name); +} + +/** + * vfs_unlink - unlink a filesystem object + * @idmap: idmap of the mount the inode was found from + * @dir: parent directory + * @dentry: victim + * @delegated_inode: returns victim inode, if the inode is delegated. + * + * The caller must hold dir->i_rwsem exclusively. + * + * If vfs_unlink discovers a delegation, it will return -EWOULDBLOCK and + * return a reference to the inode in delegated_inode. The caller + * should then break the delegation on that inode and retry. Because + * breaking a delegation may take a long time, the caller should drop + * dir->i_rwsem before doing so. + * + * Alternatively, a caller may pass NULL for delegated_inode. This may + * be appropriate for callers that expect the underlying filesystem not + * to be NFS exported. + * + * If the inode has been found through an idmapped mount the idmap of + * the vfsmount must be passed through @idmap. This function will then take + * care to map the inode according to @idmap before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply pass @nop_mnt_idmap. + */ +int vfs_unlink(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, struct delegated_inode *delegated_inode) +{ + struct inode *target = dentry->d_inode; + int error = may_delete_dentry(idmap, dir, dentry, false); + + if (error) + return error; + + if (!dir->i_op->unlink) + return -EPERM; + + inode_lock(target); + if (IS_SWAPFILE(target)) + error = -EPERM; + else if (is_local_mountpoint(dentry)) + error = -EBUSY; + else { + error = security_inode_unlink(dir, dentry); + if (!error) { + error = try_break_deleg(dir, LEASE_BREAK_DIR_DELETE, delegated_inode); + if (error) + goto out; + error = try_break_deleg(target, 0, delegated_inode); + if (error) + goto out; + error = dir->i_op->unlink(dir, dentry); + if (!error) { + dont_mount(dentry); + detach_mounts(dentry); + } + } + } +out: + inode_unlock(target); + + /* We don't d_delete() NFS sillyrenamed files--they still exist. */ + if (!error && dentry->d_flags & DCACHE_NFSFS_RENAMED) { + fsnotify_unlink(dir, dentry); + } else if (!error) { + fsnotify_link_count(target); + d_delete_notify(dir, dentry); + } + + return error; +} +EXPORT_SYMBOL(vfs_unlink); + +/* + * Make sure that the actual truncation of the file will occur outside its + * directory's i_rwsem. Truncate can take a long time if there is a lot of + * writeout happening, and we don't want to prevent access to the directory + * while waiting on the I/O. + */ +int filename_unlinkat(int dfd, struct filename *name) +{ + int error; + struct dentry *dentry; + struct path path; + struct qstr last; + enum last_type type; + struct inode *inode; + struct delegated_inode delegated_inode = { }; + unsigned int lookup_flags = 0; +retry: + error = filename_parentat(dfd, name, lookup_flags, &path, &last, &type); + if (error) + return error; + + error = -EISDIR; + if (type != LAST_NORM) + goto exit_path_put; + + error = mnt_want_write(path.mnt); + if (error) + goto exit_path_put; +retry_deleg: + dentry = start_dirop(path.dentry, &last, lookup_flags); + error = PTR_ERR(dentry); + if (IS_ERR(dentry)) + goto exit_drop_write; + + /* Why not before? Because we want correct error value */ + if (unlikely(last.name[last.len])) { + if (d_is_dir(dentry)) + error = -EISDIR; + else + error = -ENOTDIR; + end_dirop(dentry); + goto exit_drop_write; + } + inode = dentry->d_inode; + ihold(inode); + error = security_path_unlink(&path, dentry); + if (error) + goto exit_end_dirop; + error = vfs_unlink(mnt_idmap(path.mnt), path.dentry->d_inode, + dentry, &delegated_inode); +exit_end_dirop: + end_dirop(dentry); + iput(inode); /* truncate the inode here */ + if (is_delegated(&delegated_inode)) { + error = break_deleg_wait(&delegated_inode); + if (!error) + goto retry_deleg; + } +exit_drop_write: + mnt_drop_write(path.mnt); +exit_path_put: + path_put(&path); + if (retry_estale(error, lookup_flags)) { + lookup_flags |= LOOKUP_REVAL; + goto retry; + } + return error; +} + +SYSCALL_DEFINE3(unlinkat, int, dfd, const char __user *, pathname, int, flag) +{ + if ((flag & ~AT_REMOVEDIR) != 0) + return -EINVAL; + + CLASS(filename, name)(pathname); + if (flag & AT_REMOVEDIR) + return filename_rmdir(dfd, name); + return filename_unlinkat(dfd, name); +} + +SYSCALL_DEFINE1(unlink, const char __user *, pathname) +{ + CLASS(filename, name)(pathname); + return filename_unlinkat(AT_FDCWD, name); +} + +/** + * vfs_symlink - create symlink + * @idmap: idmap of the mount the inode was found from + * @dir: inode of the parent directory + * @dentry: dentry of the child symlink file + * @oldname: name of the file to link to + * @delegated_inode: returns victim inode, if the inode is delegated. + * + * Create a symlink. + * + * If the inode has been found through an idmapped mount the idmap of + * the vfsmount must be passed through @idmap. This function will then take + * care to map the inode according to @idmap before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply pass @nop_mnt_idmap. + */ +int vfs_symlink(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const char *oldname, + struct delegated_inode *delegated_inode) +{ + int error; + + error = may_create_dentry(idmap, dir, dentry); + if (error) + return error; + + if (!dir->i_op->symlink) + return -EPERM; + + error = security_inode_symlink(dir, dentry, oldname); + if (error) + return error; + + error = try_break_deleg(dir, LEASE_BREAK_DIR_CREATE, delegated_inode); + if (error) + return error; + + error = dir->i_op->symlink(idmap, dir, dentry, oldname); + if (!error) + fsnotify_create(dir, dentry); + return error; +} +EXPORT_SYMBOL(vfs_symlink); + +int filename_symlinkat(struct filename *from, int newdfd, struct filename *to) +{ + int error; + struct dentry *dentry; + struct path path; + unsigned int lookup_flags = 0; + struct delegated_inode delegated_inode = { }; + + if (IS_ERR(from)) + return PTR_ERR(from); + +retry: + dentry = filename_create(newdfd, to, &path, lookup_flags); + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + + error = security_path_symlink(&path, dentry, from->name); + if (!error) + error = vfs_symlink(mnt_idmap(path.mnt), path.dentry->d_inode, + dentry, from->name, &delegated_inode); + end_creating_path(&path, dentry); + if (is_delegated(&delegated_inode)) { + error = break_deleg_wait(&delegated_inode); + if (!error) + goto retry; + } + if (retry_estale(error, lookup_flags)) { + lookup_flags |= LOOKUP_REVAL; + goto retry; + } + return error; +} + +SYSCALL_DEFINE3(symlinkat, const char __user *, oldname, + int, newdfd, const char __user *, newname) +{ + CLASS(filename, old)(oldname); + CLASS(filename, new)(newname); + return filename_symlinkat(old, newdfd, new); +} + +SYSCALL_DEFINE2(symlink, const char __user *, oldname, const char __user *, newname) +{ + CLASS(filename, old)(oldname); + CLASS(filename, new)(newname); + return filename_symlinkat(old, AT_FDCWD, new); +} + +/** + * vfs_link - create a new link + * @old_dentry: object to be linked + * @idmap: idmap of the mount + * @dir: new parent + * @new_dentry: where to create the new link + * @delegated_inode: returns inode needing a delegation break + * + * The caller must hold dir->i_rwsem exclusively. + * + * If vfs_link discovers a delegation on the to-be-linked file in need + * of breaking, it will return -EWOULDBLOCK and return a reference to the + * inode in delegated_inode. The caller should then break the delegation + * and retry. Because breaking a delegation may take a long time, the + * caller should drop the i_rwsem before doing so. + * + * Alternatively, a caller may pass NULL for delegated_inode. This may + * be appropriate for callers that expect the underlying filesystem not + * to be NFS exported. + * + * If the inode has been found through an idmapped mount the idmap of + * the vfsmount must be passed through @idmap. This function will then take + * care to map the inode according to @idmap before checking permissions. + * On non-idmapped mounts or if permission checking is to be performed on the + * raw inode simply pass @nop_mnt_idmap. + */ +int vfs_link(struct dentry *old_dentry, struct mnt_idmap *idmap, + struct inode *dir, struct dentry *new_dentry, + struct delegated_inode *delegated_inode) +{ + struct inode *inode = old_dentry->d_inode; + unsigned max_links = dir->i_sb->s_max_links; + int error; + + if (!inode) + return -ENOENT; + + error = may_create_dentry(idmap, dir, new_dentry); + if (error) + return error; + + if (dir->i_sb != inode->i_sb) + return -EXDEV; + + /* + * A link to an append-only or immutable file cannot be created. + */ + if (IS_APPEND(inode) || IS_IMMUTABLE(inode)) + return -EPERM; + /* + * Updating the link count will likely cause i_uid and i_gid to + * be written back improperly if their true value is unknown to + * the vfs. + */ + if (HAS_UNMAPPED_ID(idmap, inode)) + return -EPERM; + if (!dir->i_op->link) + return -EPERM; + if (S_ISDIR(inode->i_mode)) + return -EPERM; + + error = security_inode_link(old_dentry, dir, new_dentry); + if (error) + return error; + + inode_lock(inode); + /* Make sure we don't allow creating hardlink to an unlinked file */ + if (inode->i_nlink == 0 && !(inode_state_read_once(inode) & I_LINKABLE)) + error = -ENOENT; + else if (max_links && inode->i_nlink >= max_links) + error = -EMLINK; + else { + error = try_break_deleg(dir, LEASE_BREAK_DIR_CREATE, delegated_inode); + if (!error) + error = try_break_deleg(inode, 0, delegated_inode); + if (!error) + error = dir->i_op->link(old_dentry, dir, new_dentry); + } + + if (!error && (inode_state_read_once(inode) & I_LINKABLE)) { + spin_lock(&inode->i_lock); + inode_state_clear(inode, I_LINKABLE); + spin_unlock(&inode->i_lock); + } + inode_unlock(inode); + if (!error) + fsnotify_link(dir, inode, new_dentry); + return error; +} +EXPORT_SYMBOL(vfs_link); + +/* + * Hardlinks are often used in delicate situations. We avoid + * security-related surprises by not following symlinks on the + * newname. --KAB + * + * We don't follow them on the oldname either to be compatible + * with linux 2.0, and to avoid hard-linking to directories + * and other special files. --ADM +*/ +int filename_linkat(int olddfd, struct filename *old, + int newdfd, struct filename *new, int flags) +{ + struct mnt_idmap *idmap; + struct dentry *new_dentry; + struct path old_path, new_path; + struct delegated_inode delegated_inode = { }; + int how = 0; + int error; + + if ((flags & ~(AT_SYMLINK_FOLLOW | AT_EMPTY_PATH)) != 0) + return -EINVAL; + /* + * To use null names we require CAP_DAC_READ_SEARCH or + * that the open-time creds of the dfd matches current. + * This ensures that not everyone will be able to create + * a hardlink using the passed file descriptor. + */ + if (flags & AT_EMPTY_PATH) + how |= LOOKUP_LINKAT_EMPTY; + + if (flags & AT_SYMLINK_FOLLOW) + how |= LOOKUP_FOLLOW; +retry: + error = filename_lookup(olddfd, old, how, &old_path, NULL); + if (error) + return error; + + new_dentry = filename_create(newdfd, new, &new_path, + (how & LOOKUP_REVAL)); + error = PTR_ERR(new_dentry); + if (IS_ERR(new_dentry)) + goto out_putpath; + + error = -EXDEV; + if (old_path.mnt != new_path.mnt) + goto out_dput; + idmap = mnt_idmap(new_path.mnt); + error = may_linkat(idmap, &old_path); + if (unlikely(error)) + goto out_dput; + error = security_path_link(old_path.dentry, &new_path, new_dentry); + if (error) + goto out_dput; + error = vfs_link(old_path.dentry, idmap, new_path.dentry->d_inode, + new_dentry, &delegated_inode); +out_dput: + end_creating_path(&new_path, new_dentry); + if (is_delegated(&delegated_inode)) { + error = break_deleg_wait(&delegated_inode); + if (!error) { + path_put(&old_path); + goto retry; + } + } + if (retry_estale(error, how)) { + path_put(&old_path); + how |= LOOKUP_REVAL; + goto retry; + } +out_putpath: + path_put(&old_path); + return error; +} + +SYSCALL_DEFINE5(linkat, int, olddfd, const char __user *, oldname, + int, newdfd, const char __user *, newname, int, flags) +{ + CLASS(filename_uflags, old)(oldname, flags); + CLASS(filename, new)(newname); + return filename_linkat(olddfd, old, newdfd, new, flags); +} + +SYSCALL_DEFINE2(link, const char __user *, oldname, const char __user *, newname) +{ + CLASS(filename, old)(oldname); + CLASS(filename, new)(newname); + return filename_linkat(AT_FDCWD, old, AT_FDCWD, new, 0); +} + +/** + * vfs_rename - rename a filesystem object + * @rd: pointer to &struct renamedata info + * + * The caller must hold multiple mutexes--see lock_rename()). + * + * If vfs_rename discovers a delegation in need of breaking at either + * the source or destination, it will return -EWOULDBLOCK and return a + * reference to the inode in delegated_inode. The caller should then + * break the delegation and retry. Because breaking a delegation may + * take a long time, the caller should drop all locks before doing + * so. + * + * Alternatively, a caller may pass NULL for delegated_inode. This may + * be appropriate for callers that expect the underlying filesystem not + * to be NFS exported. + * + * The worst of all namespace operations - renaming directory. "Perverted" + * doesn't even start to describe it. Somebody in UCB had a heck of a trip... + * Problems: + * + * a) we can get into loop creation. + * b) race potential - two innocent renames can create a loop together. + * That's where 4.4BSD screws up. Current fix: serialization on + * sb->s_vfs_rename_mutex. We might be more accurate, but that's another + * story. + * c) we may have to lock up to _four_ objects - parents and victim (if it exists), + * and source (if it's a non-directory or a subdirectory that moves to + * different parent). + * And that - after we got ->i_rwsem on parents (until then we don't know + * whether the target exists). Solution: try to be smart with locking + * order for inodes. We rely on the fact that tree topology may change + * only under ->s_vfs_rename_mutex _and_ that parent of the object we + * move will be locked. Thus we can rank directories by the tree + * (ancestors first) and rank all non-directories after them. + * That works since everybody except rename does "lock parent, lookup, + * lock child" and rename is under ->s_vfs_rename_mutex. + * HOWEVER, it relies on the assumption that any object with ->lookup() + * has no more than 1 dentry. If "hybrid" objects will ever appear, + * we'd better make sure that there's no link(2) for them. + * d) conversion from fhandle to dentry may come in the wrong moment - when + * we are removing the target. Solution: we will have to grab ->i_rwsem + * in the fhandle_to_dentry code. [FIXME - current nfsfh.c relies on + * ->i_rwsem on parents, which works but leads to some truly excessive + * locking]. + */ +int vfs_rename(struct renamedata *rd) +{ + int error; + struct inode *old_dir = d_inode(rd->old_parent); + struct inode *new_dir = d_inode(rd->new_parent); + struct dentry *old_dentry = rd->old_dentry; + struct dentry *new_dentry = rd->new_dentry; + struct delegated_inode *delegated_inode = rd->delegated_inode; + unsigned int flags = rd->flags; + bool is_dir = d_is_dir(old_dentry); + struct inode *source = old_dentry->d_inode; + struct inode *target = new_dentry->d_inode; + bool new_is_dir = false; + unsigned max_links = new_dir->i_sb->s_max_links; + struct name_snapshot old_name; + bool lock_old_subdir, lock_new_subdir; + + if (source == target) + return 0; + + error = may_delete_dentry(rd->mnt_idmap, old_dir, old_dentry, is_dir); + if (error) + return error; + + if (!target) { + error = may_create_dentry(rd->mnt_idmap, new_dir, new_dentry); + } else { + new_is_dir = d_is_dir(new_dentry); + + if (!(flags & RENAME_EXCHANGE)) + error = may_delete_dentry(rd->mnt_idmap, new_dir, + new_dentry, is_dir); + else + error = may_delete_dentry(rd->mnt_idmap, new_dir, + new_dentry, new_is_dir); + } + if (error) + return error; + + if (!old_dir->i_op->rename) + return -EPERM; + + /* + * If we are going to change the parent - check write permissions, + * we'll need to flip '..'. + */ + if (new_dir != old_dir) { + if (is_dir) { + error = inode_permission(rd->mnt_idmap, source, + MAY_WRITE); + if (error) + return error; + } + if ((flags & RENAME_EXCHANGE) && new_is_dir) { + error = inode_permission(rd->mnt_idmap, target, + MAY_WRITE); + if (error) + return error; + } + } + + error = security_inode_rename(old_dir, old_dentry, new_dir, new_dentry, + flags); + if (error) + return error; + + take_dentry_name_snapshot(&old_name, old_dentry); + dget(new_dentry); + /* + * Lock children. + * The source subdirectory needs to be locked on cross-directory + * rename or cross-directory exchange since its parent changes. + * The target subdirectory needs to be locked on cross-directory + * exchange due to parent change and on any rename due to becoming + * a victim. + * Non-directories need locking in all cases (for NFS reasons); + * they get locked after any subdirectories (in inode address order). + * + * NOTE: WE ONLY LOCK UNRELATED DIRECTORIES IN CROSS-DIRECTORY CASE. + * NEVER, EVER DO THAT WITHOUT ->s_vfs_rename_mutex. + */ + lock_old_subdir = new_dir != old_dir; + lock_new_subdir = new_dir != old_dir || !(flags & RENAME_EXCHANGE); + if (is_dir) { + if (lock_old_subdir) + inode_lock_nested(source, I_MUTEX_CHILD); + if (target && (!new_is_dir || lock_new_subdir)) + inode_lock(target); + } else if (new_is_dir) { + if (lock_new_subdir) + inode_lock_nested(target, I_MUTEX_CHILD); + inode_lock(source); + } else { + lock_two_nondirectories(source, target); + } + + error = -EPERM; + if (IS_SWAPFILE(source) || (target && IS_SWAPFILE(target))) + goto out; + + error = -EBUSY; + if (is_local_mountpoint(old_dentry) || is_local_mountpoint(new_dentry)) + goto out; + + if (max_links && new_dir != old_dir) { + error = -EMLINK; + if (is_dir && !new_is_dir && new_dir->i_nlink >= max_links) + goto out; + if ((flags & RENAME_EXCHANGE) && !is_dir && new_is_dir && + old_dir->i_nlink >= max_links) + goto out; + } + error = try_break_deleg(old_dir, + old_dir == new_dir ? LEASE_BREAK_DIR_RENAME : + LEASE_BREAK_DIR_DELETE, + delegated_inode); + if (error) + goto out; + if (new_dir != old_dir) { + error = try_break_deleg(new_dir, LEASE_BREAK_DIR_CREATE, delegated_inode); + if (error) + goto out; + } + if (!is_dir) { + error = try_break_deleg(source, 0, delegated_inode); + if (error) + goto out; + } + if (target && !new_is_dir) { + error = try_break_deleg(target, 0, delegated_inode); + if (error) + goto out; + } + error = old_dir->i_op->rename(rd->mnt_idmap, old_dir, old_dentry, + new_dir, new_dentry, flags); + if (error) + goto out; + + if (!(flags & RENAME_EXCHANGE) && target) { + if (is_dir) { + shrink_dcache_parent(new_dentry); + target->i_flags |= S_DEAD; + } + dont_mount(new_dentry); + detach_mounts(new_dentry); + } + if (!(old_dir->i_sb->s_type->fs_flags & FS_RENAME_DOES_D_MOVE)) { + if (!(flags & RENAME_EXCHANGE)) + d_move(old_dentry, new_dentry); + else + d_exchange(old_dentry, new_dentry); + } +out: + if (!is_dir || lock_old_subdir) + inode_unlock(source); + if (target && (!new_is_dir || lock_new_subdir)) + inode_unlock(target); + dput(new_dentry); + if (!error) { + fsnotify_move(old_dir, new_dir, &old_name.name, is_dir, + !(flags & RENAME_EXCHANGE) ? target : NULL, old_dentry); + if (flags & RENAME_EXCHANGE) { + fsnotify_move(new_dir, old_dir, &old_dentry->d_name, + new_is_dir, NULL, new_dentry); + } + } + release_dentry_name_snapshot(&old_name); + + return error; +} +EXPORT_SYMBOL(vfs_rename); + +int filename_renameat2(int olddfd, struct filename *from, + int newdfd, struct filename *to, unsigned int flags) +{ + struct renamedata rd; + struct path old_path, new_path; + struct qstr old_last, new_last; + enum last_type old_type, new_type; + struct delegated_inode delegated_inode = { }; + unsigned int lookup_flags = 0; + bool should_retry = false; + int error; + + if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) + return -EINVAL; + + if ((flags & (RENAME_NOREPLACE | RENAME_WHITEOUT)) && + (flags & RENAME_EXCHANGE)) + return -EINVAL; + +retry: + error = filename_parentat(olddfd, from, lookup_flags, &old_path, + &old_last, &old_type); + if (error) + return error; + + error = filename_parentat(newdfd, to, lookup_flags, &new_path, &new_last, + &new_type); + if (error) + goto exit1; + + error = -EXDEV; + if (old_path.mnt != new_path.mnt) + goto exit2; + + error = -EBUSY; + if (old_type != LAST_NORM) + goto exit2; + + if (flags & RENAME_NOREPLACE) + error = -EEXIST; + if (new_type != LAST_NORM) + goto exit2; + + error = mnt_want_write(old_path.mnt); + if (error) + goto exit2; + +retry_deleg: + rd.old_parent = old_path.dentry; + rd.mnt_idmap = mnt_idmap(old_path.mnt); + rd.new_parent = new_path.dentry; + rd.delegated_inode = &delegated_inode; + rd.flags = flags; + + error = __start_renaming(&rd, lookup_flags, &old_last, &new_last); + if (error) + goto exit_lock_rename; + + if (flags & RENAME_EXCHANGE) { + if (!d_is_dir(rd.new_dentry)) { + error = -ENOTDIR; + if (new_last.name[new_last.len]) + goto exit_unlock; + } + } + /* unless the source is a directory trailing slashes give -ENOTDIR */ + if (!d_is_dir(rd.old_dentry)) { + error = -ENOTDIR; + if (old_last.name[old_last.len]) + goto exit_unlock; + if (!(flags & RENAME_EXCHANGE) && new_last.name[new_last.len]) + goto exit_unlock; + } + + error = security_path_rename(&old_path, rd.old_dentry, + &new_path, rd.new_dentry, flags); + if (error) + goto exit_unlock; + + error = vfs_rename(&rd); +exit_unlock: + end_renaming(&rd); +exit_lock_rename: + if (is_delegated(&delegated_inode)) { + error = break_deleg_wait(&delegated_inode); + if (!error) + goto retry_deleg; + } + mnt_drop_write(old_path.mnt); +exit2: + if (retry_estale(error, lookup_flags)) + should_retry = true; + path_put(&new_path); +exit1: + path_put(&old_path); + if (should_retry) { + should_retry = false; + lookup_flags |= LOOKUP_REVAL; + goto retry; + } + return error; +} + +SYSCALL_DEFINE5(renameat2, int, olddfd, const char __user *, oldname, + int, newdfd, const char __user *, newname, unsigned int, flags) +{ + CLASS(filename, old)(oldname); + CLASS(filename, new)(newname); + return filename_renameat2(olddfd, old, newdfd, new, flags); +} + +SYSCALL_DEFINE4(renameat, int, olddfd, const char __user *, oldname, + int, newdfd, const char __user *, newname) +{ + CLASS(filename, old)(oldname); + CLASS(filename, new)(newname); + return filename_renameat2(olddfd, old, newdfd, new, 0); +} + +SYSCALL_DEFINE2(rename, const char __user *, oldname, const char __user *, newname) +{ + CLASS(filename, old)(oldname); + CLASS(filename, new)(newname); + return filename_renameat2(AT_FDCWD, old, AT_FDCWD, new, 0); +} + +int readlink_copy(char __user *buffer, int buflen, const char *link, int linklen) +{ + int copylen; + + copylen = linklen; + if (unlikely(copylen > (unsigned) buflen)) + copylen = buflen; + if (copy_to_user(buffer, link, copylen)) + copylen = -EFAULT; + return copylen; +} + +/** + * vfs_readlink - copy symlink body into userspace buffer + * @dentry: dentry on which to get symbolic link + * @buffer: user memory pointer + * @buflen: size of buffer + * + * Does not touch atime. That's up to the caller if necessary + * + * Does not call security hook. + */ +int vfs_readlink(struct dentry *dentry, char __user *buffer, int buflen) +{ + struct inode *inode = d_inode(dentry); + DEFINE_DELAYED_CALL(done); + const char *link; + int res; + + if (inode->i_opflags & IOP_CACHED_LINK) + return readlink_copy(buffer, buflen, inode->i_link, inode->i_linklen); + + if (unlikely(!(inode->i_opflags & IOP_DEFAULT_READLINK))) { + if (unlikely(inode->i_op->readlink)) + return inode->i_op->readlink(dentry, buffer, buflen); + + if (!d_is_symlink(dentry)) + return -EINVAL; + + spin_lock(&inode->i_lock); + inode->i_opflags |= IOP_DEFAULT_READLINK; + spin_unlock(&inode->i_lock); + } + + link = READ_ONCE(inode->i_link); + if (!link) { + link = inode->i_op->get_link(dentry, inode, &done); + if (IS_ERR(link)) + return PTR_ERR(link); + } + res = readlink_copy(buffer, buflen, link, strlen(link)); + do_delayed_call(&done); + return res; +} +EXPORT_SYMBOL(vfs_readlink); + +/** + * vfs_get_link - get symlink body + * @dentry: dentry on which to get symbolic link + * @done: caller needs to free returned data with this + * + * Calls security hook and i_op->get_link() on the supplied inode. + * + * It does not touch atime. That's up to the caller if necessary. + * + * Does not work on "special" symlinks like /proc/$$/fd/N + */ +const char *vfs_get_link(struct dentry *dentry, struct delayed_call *done) +{ + const char *res = ERR_PTR(-EINVAL); + struct inode *inode = d_inode(dentry); + + if (d_is_symlink(dentry)) { + res = ERR_PTR(security_inode_readlink(dentry)); + if (!res) + res = inode->i_op->get_link(dentry, inode, done); + } + return res; +} +EXPORT_SYMBOL(vfs_get_link); + +/* get the link contents into pagecache */ +static char *__page_get_link(struct dentry *dentry, struct inode *inode, + struct delayed_call *callback) +{ + struct folio *folio; + struct address_space *mapping = inode->i_mapping; + + if (!dentry) { + folio = filemap_get_folio(mapping, 0); + if (IS_ERR(folio)) + return ERR_PTR(-ECHILD); + if (!folio_test_uptodate(folio)) { + folio_put(folio); + return ERR_PTR(-ECHILD); + } + } else { + folio = read_mapping_folio(mapping, 0, NULL); + if (IS_ERR(folio)) + return ERR_CAST(folio); + } + set_delayed_call(callback, page_put_link, folio); + BUG_ON(mapping_gfp_mask(mapping) & __GFP_HIGHMEM); + return folio_address(folio); +} + +const char *page_get_link_raw(struct dentry *dentry, struct inode *inode, + struct delayed_call *callback) +{ + return __page_get_link(dentry, inode, callback); +} +EXPORT_SYMBOL_GPL(page_get_link_raw); + +/** + * page_get_link() - An implementation of the get_link inode_operation. + * @dentry: The directory entry which is the symlink. + * @inode: The inode for the symlink. + * @callback: Used to drop the reference to the symlink. + * + * Filesystems which store their symlinks in the page cache should use + * this to implement the get_link() member of their inode_operations. + * + * Return: A pointer to the NUL-terminated symlink. + */ +const char *page_get_link(struct dentry *dentry, struct inode *inode, + struct delayed_call *callback) +{ + char *kaddr = __page_get_link(dentry, inode, callback); + + if (!IS_ERR(kaddr)) + nd_terminate_link(kaddr, inode->i_size, PAGE_SIZE - 1); + return kaddr; +} +EXPORT_SYMBOL(page_get_link); + +/** + * page_put_link() - Drop the reference to the symlink. + * @arg: The folio which contains the symlink. + * + * This is used internally by page_get_link(). It is exported for use + * by filesystems which need to implement a variant of page_get_link() + * themselves. Despite the apparent symmetry, filesystems which use + * page_get_link() do not need to call page_put_link(). + * + * The argument, while it has a void pointer type, must be a pointer to + * the folio which was retrieved from the page cache. The delayed_call + * infrastructure is used to drop the reference count once the caller + * is done with the symlink. + */ +void page_put_link(void *arg) +{ + folio_put(arg); +} +EXPORT_SYMBOL(page_put_link); + +int page_readlink(struct dentry *dentry, char __user *buffer, int buflen) +{ + const char *link; + int res; + + DEFINE_DELAYED_CALL(done); + link = page_get_link(dentry, d_inode(dentry), &done); + res = PTR_ERR(link); + if (!IS_ERR(link)) + res = readlink_copy(buffer, buflen, link, strlen(link)); + do_delayed_call(&done); + return res; +} +EXPORT_SYMBOL(page_readlink); + +int page_symlink(struct inode *inode, const char *symname, int len) +{ + struct address_space *mapping = inode->i_mapping; + const struct address_space_operations *aops = mapping->a_ops; + bool nofs = !mapping_gfp_constraint(mapping, __GFP_FS); + struct folio *folio; + void *fsdata = NULL; + int err; + unsigned int flags; + +retry: + if (nofs) + flags = memalloc_nofs_save(); + err = aops->write_begin(NULL, mapping, 0, len-1, &folio, &fsdata); + if (nofs) + memalloc_nofs_restore(flags); + if (err) + goto fail; + + memcpy(folio_address(folio), symname, len - 1); + + err = aops->write_end(NULL, mapping, 0, len - 1, len - 1, + folio, fsdata); + if (err < 0) + goto fail; + if (err < len-1) + goto retry; + + mark_inode_dirty(inode); + return 0; +fail: + return err; +} +EXPORT_SYMBOL(page_symlink); + +const struct inode_operations page_symlink_inode_operations = { + .get_link = page_get_link, +}; +EXPORT_SYMBOL(page_symlink_inode_operations); diff --git a/fs/namespace.c b/fs/namespace.c new file mode 100644 index 000000000..1ecd96c91 --- /dev/null +++ b/fs/namespace.c @@ -0,0 +1,6566 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/namespace.c + * + * (C) Copyright Al Viro 2000, 2001 + * + * Based on code from fs/super.c, copyright Linus Torvalds and others. + * Heavily rewritten. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* init_rootfs */ +#include /* get_fs_root et.al. */ +#include /* fsnotify_vfsmount_delete */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "pnode.h" +#include "internal.h" + +/* Maximum number of mounts in a mount namespace */ +static unsigned int sysctl_mount_max __read_mostly = 100000; + +static unsigned int m_hash_mask __ro_after_init; +static unsigned int m_hash_shift __ro_after_init; +static unsigned int mp_hash_mask __ro_after_init; +static unsigned int mp_hash_shift __ro_after_init; + +static __initdata unsigned long mhash_entries; +static int __init set_mhash_entries(char *str) +{ + return kstrtoul(str, 0, &mhash_entries) == 0; +} +__setup("mhash_entries=", set_mhash_entries); + +static __initdata unsigned long mphash_entries; +static int __init set_mphash_entries(char *str) +{ + return kstrtoul(str, 0, &mphash_entries) == 0; +} +__setup("mphash_entries=", set_mphash_entries); + +static char * __initdata initramfs_options; +static int __init initramfs_options_setup(char *str) +{ + initramfs_options = str; + return 1; +} + +__setup("initramfs_options=", initramfs_options_setup); + +static u64 event; +static DEFINE_XARRAY_FLAGS(mnt_id_xa, XA_FLAGS_ALLOC); +static DEFINE_IDA(mnt_group_ida); + +/* Don't allow confusion with old 32bit mount ID */ +#define MNT_UNIQUE_ID_OFFSET (1ULL << 31) +static u64 mnt_id_ctr = MNT_UNIQUE_ID_OFFSET; + +static struct hlist_head *mount_hashtable __ro_after_init; +static struct hlist_head *mountpoint_hashtable __ro_after_init; +static struct kmem_cache *mnt_cache __ro_after_init; +static DECLARE_RWSEM(namespace_sem); +static HLIST_HEAD(unmounted); /* protected by namespace_sem */ +static LIST_HEAD(ex_mountpoints); /* protected by namespace_sem */ +static struct mnt_namespace *emptied_ns; /* protected by namespace_sem */ + +static inline void namespace_lock(void); +static void namespace_unlock(void); +DEFINE_LOCK_GUARD_0(namespace_excl, namespace_lock(), namespace_unlock()) +DEFINE_LOCK_GUARD_0(namespace_shared, down_read(&namespace_sem), + up_read(&namespace_sem)) + +DEFINE_FREE(mntput, struct vfsmount *, if (!IS_ERR(_T)) mntput(_T)) + +#ifdef CONFIG_FSNOTIFY +LIST_HEAD(notify_list); /* protected by namespace_sem */ +#endif + +enum mount_kattr_flags_t { + MOUNT_KATTR_RECURSE = (1 << 0), + MOUNT_KATTR_IDMAP_REPLACE = (1 << 1), +}; + +struct mount_kattr { + unsigned int attr_set; + unsigned int attr_clr; + unsigned int propagation; + unsigned int lookup_flags; + enum mount_kattr_flags_t kflags; + struct user_namespace *mnt_userns; + struct mnt_idmap *mnt_idmap; +}; + +/* /sys/fs */ +struct kobject *fs_kobj __ro_after_init; +EXPORT_SYMBOL_GPL(fs_kobj); + +/* + * vfsmount lock may be taken for read to prevent changes to the + * vfsmount hash, ie. during mountpoint lookups or walking back + * up the tree. + * + * It should be taken for write in all cases where the vfsmount + * tree or hash is modified or when a vfsmount structure is modified. + */ +__cacheline_aligned_in_smp DEFINE_SEQLOCK(mount_lock); + +static void mnt_ns_release(struct mnt_namespace *ns) +{ + /* keep alive for {list,stat}mount() */ + if (ns && refcount_dec_and_test(&ns->passive)) { + fsnotify_mntns_delete(ns); + put_user_ns(ns->user_ns); + kfree(ns); + } +} +DEFINE_FREE(mnt_ns_release, struct mnt_namespace *, + if (!IS_ERR(_T)) mnt_ns_release(_T)) + +static void mnt_ns_release_rcu(struct rcu_head *rcu) +{ + mnt_ns_release(container_of(rcu, struct mnt_namespace, ns.ns_rcu)); +} + +static void mnt_ns_tree_remove(struct mnt_namespace *ns) +{ + /* remove from global mount namespace list */ + if (ns_tree_active(ns)) + ns_tree_remove(ns); + + call_rcu(&ns->ns.ns_rcu, mnt_ns_release_rcu); +} + +/* + * Lookup a mount namespace by id and take a passive reference count. Taking a + * passive reference means the mount namespace can be emptied if e.g., the last + * task holding an active reference exits. To access the mounts of the + * namespace the @namespace_sem must first be acquired. If the namespace has + * already shut down before acquiring @namespace_sem, {list,stat}mount() will + * see that the mount rbtree of the namespace is empty. + * + * Note the lookup is lockless protected by a sequence counter. We only + * need to guard against false negatives as false positives aren't + * possible. So if we didn't find a mount namespace and the sequence + * counter has changed we need to retry. If the sequence counter is + * still the same we know the search actually failed. + */ +static struct mnt_namespace *lookup_mnt_ns(u64 mnt_ns_id) +{ + struct mnt_namespace *mnt_ns; + struct ns_common *ns; + + guard(rcu)(); + ns = ns_tree_lookup_rcu(mnt_ns_id, CLONE_NEWNS); + if (!ns) + return NULL; + + /* + * The last reference count is put with RCU delay so we can + * unconditonally acquire a reference here. + */ + mnt_ns = container_of(ns, struct mnt_namespace, ns); + refcount_inc(&mnt_ns->passive); + return mnt_ns; +} + +static inline void lock_mount_hash(void) +{ + write_seqlock(&mount_lock); +} + +static inline void unlock_mount_hash(void) +{ + write_sequnlock(&mount_lock); +} + +static inline struct hlist_head *m_hash(struct vfsmount *mnt, struct dentry *dentry) +{ + unsigned long tmp = ((unsigned long)mnt / L1_CACHE_BYTES); + tmp += ((unsigned long)dentry / L1_CACHE_BYTES); + tmp = tmp + (tmp >> m_hash_shift); + return &mount_hashtable[tmp & m_hash_mask]; +} + +static inline struct hlist_head *mp_hash(struct dentry *dentry) +{ + unsigned long tmp = ((unsigned long)dentry / L1_CACHE_BYTES); + tmp = tmp + (tmp >> mp_hash_shift); + return &mountpoint_hashtable[tmp & mp_hash_mask]; +} + +static int mnt_alloc_id(struct mount *mnt) +{ + int res; + + xa_lock(&mnt_id_xa); + res = __xa_alloc(&mnt_id_xa, &mnt->mnt_id, mnt, xa_limit_31b, GFP_KERNEL); + if (!res) + mnt->mnt_id_unique = ++mnt_id_ctr; + xa_unlock(&mnt_id_xa); + return res; +} + +static void mnt_free_id(struct mount *mnt) +{ + xa_erase(&mnt_id_xa, mnt->mnt_id); +} + +/* + * Allocate a new peer group ID + */ +static int mnt_alloc_group_id(struct mount *mnt) +{ + int res = ida_alloc_min(&mnt_group_ida, 1, GFP_KERNEL); + + if (res < 0) + return res; + mnt->mnt_group_id = res; + return 0; +} + +/* + * Release a peer group ID + */ +void mnt_release_group_id(struct mount *mnt) +{ + ida_free(&mnt_group_ida, mnt->mnt_group_id); + mnt->mnt_group_id = 0; +} + +/* + * vfsmount lock must be held for read + */ +static inline void mnt_add_count(struct mount *mnt, int n) +{ +#ifdef CONFIG_SMP + this_cpu_add(mnt->mnt_pcp->mnt_count, n); +#else + preempt_disable(); + mnt->mnt_count += n; + preempt_enable(); +#endif +} + +/* + * vfsmount lock must be held for write + */ +int mnt_get_count(struct mount *mnt) +{ +#ifdef CONFIG_SMP + int count = 0; + int cpu; + + for_each_possible_cpu(cpu) { + count += per_cpu_ptr(mnt->mnt_pcp, cpu)->mnt_count; + } + + return count; +#else + return mnt->mnt_count; +#endif +} + +static struct mount *alloc_vfsmnt(const char *name) +{ + struct mount *mnt = kmem_cache_zalloc(mnt_cache, GFP_KERNEL); + if (mnt) { + int err; + + err = mnt_alloc_id(mnt); + if (err) + goto out_free_cache; + + if (name) + mnt->mnt_devname = kstrdup_const(name, + GFP_KERNEL_ACCOUNT); + else + mnt->mnt_devname = "none"; + if (!mnt->mnt_devname) + goto out_free_id; + +#ifdef CONFIG_SMP + mnt->mnt_pcp = alloc_percpu(struct mnt_pcp); + if (!mnt->mnt_pcp) + goto out_free_devname; + + this_cpu_add(mnt->mnt_pcp->mnt_count, 1); +#else + mnt->mnt_count = 1; + mnt->mnt_writers = 0; +#endif + + INIT_HLIST_NODE(&mnt->mnt_hash); + INIT_LIST_HEAD(&mnt->mnt_child); + INIT_LIST_HEAD(&mnt->mnt_mounts); + INIT_LIST_HEAD(&mnt->mnt_list); + INIT_LIST_HEAD(&mnt->mnt_expire); + INIT_LIST_HEAD(&mnt->mnt_share); + INIT_HLIST_HEAD(&mnt->mnt_slave_list); + INIT_HLIST_NODE(&mnt->mnt_slave); + INIT_HLIST_NODE(&mnt->mnt_mp_list); + INIT_HLIST_HEAD(&mnt->mnt_stuck_children); + INIT_HLIST_NODE(&mnt->mnt_ns_visible); + RB_CLEAR_NODE(&mnt->mnt_node); + mnt->mnt.mnt_idmap = &nop_mnt_idmap; + } + return mnt; + +#ifdef CONFIG_SMP +out_free_devname: + kfree_const(mnt->mnt_devname); +#endif +out_free_id: + mnt_free_id(mnt); +out_free_cache: + kmem_cache_free(mnt_cache, mnt); + return NULL; +} + +/* + * Most r/o checks on a fs are for operations that take + * discrete amounts of time, like a write() or unlink(). + * We must keep track of when those operations start + * (for permission checks) and when they end, so that + * we can determine when writes are able to occur to + * a filesystem. + */ +/* + * __mnt_is_readonly: check whether a mount is read-only + * @mnt: the mount to check for its write status + * + * This shouldn't be used directly ouside of the VFS. + * It does not guarantee that the filesystem will stay + * r/w, just that it is right *now*. This can not and + * should not be used in place of IS_RDONLY(inode). + * mnt_want/drop_write() will _keep_ the filesystem + * r/w. + */ +bool __mnt_is_readonly(const struct vfsmount *mnt) +{ + return (mnt->mnt_flags & MNT_READONLY) || sb_rdonly(mnt->mnt_sb); +} +EXPORT_SYMBOL_GPL(__mnt_is_readonly); + +static inline void mnt_inc_writers(struct mount *mnt) +{ +#ifdef CONFIG_SMP + this_cpu_inc(mnt->mnt_pcp->mnt_writers); +#else + mnt->mnt_writers++; +#endif +} + +static inline void mnt_dec_writers(struct mount *mnt) +{ +#ifdef CONFIG_SMP + this_cpu_dec(mnt->mnt_pcp->mnt_writers); +#else + mnt->mnt_writers--; +#endif +} + +static unsigned int mnt_get_writers(struct mount *mnt) +{ +#ifdef CONFIG_SMP + unsigned int count = 0; + int cpu; + + for_each_possible_cpu(cpu) { + count += per_cpu_ptr(mnt->mnt_pcp, cpu)->mnt_writers; + } + + return count; +#else + return mnt->mnt_writers; +#endif +} + +static int mnt_is_readonly(const struct vfsmount *mnt) +{ + if (READ_ONCE(mnt->mnt_sb->s_readonly_remount)) + return 1; + /* + * The barrier pairs with the barrier in sb_start_ro_state_change() + * making sure if we don't see s_readonly_remount set yet, we also will + * not see any superblock / mount flag changes done by remount. + * It also pairs with the barrier in sb_end_ro_state_change() + * assuring that if we see s_readonly_remount already cleared, we will + * see the values of superblock / mount flags updated by remount. + */ + smp_rmb(); + return __mnt_is_readonly(mnt); +} + +/* + * Most r/o & frozen checks on a fs are for operations that take discrete + * amounts of time, like a write() or unlink(). We must keep track of when + * those operations start (for permission checks) and when they end, so that we + * can determine when writes are able to occur to a filesystem. + */ +/** + * mnt_get_write_access - get write access to a mount without freeze protection + * @m: the mount on which to take a write + * + * This tells the low-level filesystem that a write is about to be performed to + * it, and makes sure that writes are allowed (mnt it read-write) before + * returning success. This operation does not protect against filesystem being + * frozen. When the write operation is finished, mnt_put_write_access() must be + * called. This is effectively a refcount. + */ +int mnt_get_write_access(struct vfsmount *m) +{ + struct mount *mnt = real_mount(m); + int ret = 0; + + preempt_disable(); + mnt_inc_writers(mnt); + /* + * The store to mnt_inc_writers must be visible before we pass + * WRITE_HOLD loop below, so that the slowpath can see our + * incremented count after it has set WRITE_HOLD. + */ + smp_mb(); + might_lock(&mount_lock.lock); + while (__test_write_hold(READ_ONCE(mnt->mnt_pprev_for_sb))) { + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) { + cpu_relax(); + } else { + /* + * This prevents priority inversion, if the task + * setting WRITE_HOLD got preempted on a remote + * CPU, and it prevents life lock if the task setting + * WRITE_HOLD has a lower priority and is bound to + * the same CPU as the task that is spinning here. + */ + preempt_enable(); + read_seqlock_excl(&mount_lock); + read_sequnlock_excl(&mount_lock); + preempt_disable(); + } + } + /* + * The barrier pairs with the barrier sb_start_ro_state_change() making + * sure that if we see WRITE_HOLD cleared, we will also see + * s_readonly_remount set (or even SB_RDONLY / MNT_READONLY flags) in + * mnt_is_readonly() and bail in case we are racing with remount + * read-only. + */ + smp_rmb(); + if (mnt_is_readonly(m)) { + mnt_dec_writers(mnt); + ret = -EROFS; + } + preempt_enable(); + + return ret; +} +EXPORT_SYMBOL_GPL(mnt_get_write_access); + +/** + * mnt_want_write - get write access to a mount + * @m: the mount on which to take a write + * + * This tells the low-level filesystem that a write is about to be performed to + * it, and makes sure that writes are allowed (mount is read-write, filesystem + * is not frozen) before returning success. When the write operation is + * finished, mnt_drop_write() must be called. This is effectively a refcount. + */ +int mnt_want_write(struct vfsmount *m) +{ + int ret; + + sb_start_write(m->mnt_sb); + ret = mnt_get_write_access(m); + if (ret) + sb_end_write(m->mnt_sb); + return ret; +} +EXPORT_SYMBOL_GPL(mnt_want_write); + +/** + * mnt_get_write_access_file - get write access to a file's mount + * @file: the file who's mount on which to take a write + * + * This is like mnt_get_write_access, but if @file is already open for write it + * skips incrementing mnt_writers (since the open file already has a reference) + * and instead only does the check for emergency r/o remounts. This must be + * paired with mnt_put_write_access_file. + */ +int mnt_get_write_access_file(struct file *file) +{ + if (file->f_mode & FMODE_WRITER) { + /* + * Superblock may have become readonly while there are still + * writable fd's, e.g. due to a fs error with errors=remount-ro + */ + if (__mnt_is_readonly(file->f_path.mnt)) + return -EROFS; + return 0; + } + return mnt_get_write_access(file->f_path.mnt); +} + +/** + * mnt_want_write_file - get write access to a file's mount + * @file: the file who's mount on which to take a write + * + * This is like mnt_want_write, but if the file is already open for writing it + * skips incrementing mnt_writers (since the open file already has a reference) + * and instead only does the freeze protection and the check for emergency r/o + * remounts. This must be paired with mnt_drop_write_file. + */ +int mnt_want_write_file(struct file *file) +{ + int ret; + + sb_start_write(file_inode(file)->i_sb); + ret = mnt_get_write_access_file(file); + if (ret) + sb_end_write(file_inode(file)->i_sb); + return ret; +} +EXPORT_SYMBOL_GPL(mnt_want_write_file); + +/** + * mnt_put_write_access - give up write access to a mount + * @mnt: the mount on which to give up write access + * + * Tells the low-level filesystem that we are done + * performing writes to it. Must be matched with + * mnt_get_write_access() call above. + */ +void mnt_put_write_access(struct vfsmount *mnt) +{ + preempt_disable(); + mnt_dec_writers(real_mount(mnt)); + preempt_enable(); +} +EXPORT_SYMBOL_GPL(mnt_put_write_access); + +/** + * mnt_drop_write - give up write access to a mount + * @mnt: the mount on which to give up write access + * + * Tells the low-level filesystem that we are done performing writes to it and + * also allows filesystem to be frozen again. Must be matched with + * mnt_want_write() call above. + */ +void mnt_drop_write(struct vfsmount *mnt) +{ + mnt_put_write_access(mnt); + sb_end_write(mnt->mnt_sb); +} +EXPORT_SYMBOL_GPL(mnt_drop_write); + +void mnt_put_write_access_file(struct file *file) +{ + if (!(file->f_mode & FMODE_WRITER)) + mnt_put_write_access(file->f_path.mnt); +} + +void mnt_drop_write_file(struct file *file) +{ + mnt_put_write_access_file(file); + sb_end_write(file_inode(file)->i_sb); +} +EXPORT_SYMBOL(mnt_drop_write_file); + +/** + * mnt_hold_writers - prevent write access to the given mount + * @mnt: mnt to prevent write access to + * + * Prevents write access to @mnt if there are no active writers for @mnt. + * This function needs to be called and return successfully before changing + * properties of @mnt that need to remain stable for callers with write access + * to @mnt. + * + * After this functions has been called successfully callers must pair it with + * a call to mnt_unhold_writers() in order to stop preventing write access to + * @mnt. + * + * Context: This function expects to be in mount_locked_reader scope serializing + * setting WRITE_HOLD. + * Return: On success 0 is returned. + * On error, -EBUSY is returned. + */ +static inline int mnt_hold_writers(struct mount *mnt) +{ + set_write_hold(mnt); + /* + * After storing WRITE_HOLD, we'll read the counters. This store + * should be visible before we do. + */ + smp_mb(); + + /* + * With writers on hold, if this value is zero, then there are + * definitely no active writers (although held writers may subsequently + * increment the count, they'll have to wait, and decrement it after + * seeing MNT_READONLY). + * + * It is OK to have counter incremented on one CPU and decremented on + * another: the sum will add up correctly. The danger would be when we + * sum up each counter, if we read a counter before it is incremented, + * but then read another CPU's count which it has been subsequently + * decremented from -- we would see more decrements than we should. + * WRITE_HOLD protects against this scenario, because + * mnt_want_write first increments count, then smp_mb, then spins on + * WRITE_HOLD, so it can't be decremented by another CPU while + * we're counting up here. + */ + if (mnt_get_writers(mnt) > 0) + return -EBUSY; + + return 0; +} + +/** + * mnt_unhold_writers - stop preventing write access to the given mount + * @mnt: mnt to stop preventing write access to + * + * Stop preventing write access to @mnt allowing callers to gain write access + * to @mnt again. + * + * This function can only be called after a call to mnt_hold_writers(). + * + * Context: This function expects to be in the same mount_locked_reader scope + * as the matching mnt_hold_writers(). + */ +static inline void mnt_unhold_writers(struct mount *mnt) +{ + if (!test_write_hold(mnt)) + return; + /* + * MNT_READONLY must become visible before ~WRITE_HOLD, so writers + * that become unheld will see MNT_READONLY. + */ + smp_wmb(); + clear_write_hold(mnt); +} + +static inline void mnt_del_instance(struct mount *m) +{ + struct mount **p = m->mnt_pprev_for_sb; + struct mount *next = m->mnt_next_for_sb; + + if (next) + next->mnt_pprev_for_sb = p; + *p = next; +} + +static inline void mnt_add_instance(struct mount *m, struct super_block *s) +{ + struct mount *first = s->s_mounts; + + if (first) + first->mnt_pprev_for_sb = &m->mnt_next_for_sb; + m->mnt_next_for_sb = first; + m->mnt_pprev_for_sb = &s->s_mounts; + s->s_mounts = m; +} + +static int mnt_make_readonly(struct mount *mnt) +{ + int ret; + + ret = mnt_hold_writers(mnt); + if (!ret) + mnt->mnt.mnt_flags |= MNT_READONLY; + mnt_unhold_writers(mnt); + return ret; +} + +int sb_prepare_remount_readonly(struct super_block *sb) +{ + int err = 0; + + /* Racy optimization. Recheck the counter under WRITE_HOLD */ + if (atomic_long_read(&sb->s_remove_count)) + return -EBUSY; + + guard(mount_locked_reader)(); + + for (struct mount *m = sb->s_mounts; m; m = m->mnt_next_for_sb) { + if (!(m->mnt.mnt_flags & MNT_READONLY)) { + err = mnt_hold_writers(m); + if (err) + break; + } + } + if (!err && atomic_long_read(&sb->s_remove_count)) + err = -EBUSY; + + if (!err) + sb_start_ro_state_change(sb); + for (struct mount *m = sb->s_mounts; m; m = m->mnt_next_for_sb) { + if (test_write_hold(m)) + clear_write_hold(m); + } + + return err; +} + +static void free_vfsmnt(struct mount *mnt) +{ + mnt_idmap_put(mnt_idmap(&mnt->mnt)); + kfree_const(mnt->mnt_devname); +#ifdef CONFIG_SMP + free_percpu(mnt->mnt_pcp); +#endif + kmem_cache_free(mnt_cache, mnt); +} + +static void delayed_free_vfsmnt(struct rcu_head *head) +{ + free_vfsmnt(container_of(head, struct mount, mnt_rcu)); +} + +/* call under rcu_read_lock */ +int __legitimize_mnt(struct vfsmount *bastard, unsigned seq) +{ + struct mount *mnt; + if (read_seqretry(&mount_lock, seq)) + return 1; + if (bastard == NULL) + return 0; + mnt = real_mount(bastard); + mnt_add_count(mnt, 1); + smp_mb(); // see mntput_no_expire() and do_umount() + if (likely(!read_seqretry(&mount_lock, seq))) + return 0; + lock_mount_hash(); + if (unlikely(bastard->mnt_flags & (MNT_SYNC_UMOUNT | MNT_DOOMED))) { + mnt_add_count(mnt, -1); + unlock_mount_hash(); + return 1; + } + unlock_mount_hash(); + /* caller will mntput() */ + return -1; +} + +/* call under rcu_read_lock */ +static bool legitimize_mnt(struct vfsmount *bastard, unsigned seq) +{ + int res = __legitimize_mnt(bastard, seq); + if (likely(!res)) + return true; + if (unlikely(res < 0)) { + rcu_read_unlock(); + mntput(bastard); + rcu_read_lock(); + } + return false; +} + +/** + * __lookup_mnt - mount hash lookup + * @mnt: parent mount + * @dentry: dentry of mountpoint + * + * If @mnt has a child mount @c mounted on @dentry find and return it. + * Caller must either hold the spinlock component of @mount_lock or + * hold rcu_read_lock(), sample the seqcount component before the call + * and recheck it afterwards. + * + * Return: The child of @mnt mounted on @dentry or %NULL. + */ +struct mount *__lookup_mnt(struct vfsmount *mnt, struct dentry *dentry) +{ + struct hlist_head *head = m_hash(mnt, dentry); + struct mount *p; + + hlist_for_each_entry_rcu(p, head, mnt_hash) + if (&p->mnt_parent->mnt == mnt && p->mnt_mountpoint == dentry) + return p; + return NULL; +} + +/** + * lookup_mnt - Return the child mount mounted at given location + * @path: location in the namespace + * + * Acquires and returns a new reference to mount at given location + * or %NULL if nothing is mounted there. + */ +struct vfsmount *lookup_mnt(const struct path *path) +{ + struct mount *child_mnt; + struct vfsmount *m; + unsigned seq; + + rcu_read_lock(); + do { + seq = read_seqbegin(&mount_lock); + child_mnt = __lookup_mnt(path->mnt, path->dentry); + m = child_mnt ? &child_mnt->mnt : NULL; + } while (!legitimize_mnt(m, seq)); + rcu_read_unlock(); + return m; +} + +/* + * __is_local_mountpoint - Test to see if dentry is a mountpoint in the + * current mount namespace. + * + * The common case is dentries are not mountpoints at all and that + * test is handled inline. For the slow case when we are actually + * dealing with a mountpoint of some kind, walk through all of the + * mounts in the current mount namespace and test to see if the dentry + * is a mountpoint. + * + * The mount_hashtable is not usable in the context because we + * need to identify all mounts that may be in the current mount + * namespace not just a mount that happens to have some specified + * parent mount. + */ +bool __is_local_mountpoint(const struct dentry *dentry) +{ + struct mnt_namespace *ns = current->nsproxy->mnt_ns; + struct mount *mnt, *n; + + guard(namespace_shared)(); + + rbtree_postorder_for_each_entry_safe(mnt, n, &ns->mounts, mnt_node) + if (mnt->mnt_mountpoint == dentry) + return true; + + return false; +} + +struct pinned_mountpoint { + struct hlist_node node; + struct mountpoint *mp; + struct mount *parent; +}; + +static bool lookup_mountpoint(struct dentry *dentry, struct pinned_mountpoint *m) +{ + struct hlist_head *chain = mp_hash(dentry); + struct mountpoint *mp; + + hlist_for_each_entry(mp, chain, m_hash) { + if (mp->m_dentry == dentry) { + hlist_add_head(&m->node, &mp->m_list); + m->mp = mp; + return true; + } + } + return false; +} + +static int get_mountpoint(struct dentry *dentry, struct pinned_mountpoint *m) +{ + struct mountpoint *mp __free(kfree) = NULL; + bool found; + int ret; + + if (d_mountpoint(dentry)) { + /* might be worth a WARN_ON() */ + if (d_unlinked(dentry)) + return -ENOENT; +mountpoint: + read_seqlock_excl(&mount_lock); + found = lookup_mountpoint(dentry, m); + read_sequnlock_excl(&mount_lock); + if (found) + return 0; + } + + if (!mp) + mp = kmalloc_obj(struct mountpoint); + if (!mp) + return -ENOMEM; + + /* Exactly one processes may set d_mounted */ + ret = d_set_mounted(dentry); + + /* Someone else set d_mounted? */ + if (ret == -EBUSY) + goto mountpoint; + + /* The dentry is not available as a mountpoint? */ + if (ret) + return ret; + + /* Add the new mountpoint to the hash table */ + read_seqlock_excl(&mount_lock); + mp->m_dentry = dget(dentry); + hlist_add_head(&mp->m_hash, mp_hash(dentry)); + INIT_HLIST_HEAD(&mp->m_list); + hlist_add_head(&m->node, &mp->m_list); + m->mp = no_free_ptr(mp); + read_sequnlock_excl(&mount_lock); + return 0; +} + +/* + * vfsmount lock must be held. Additionally, the caller is responsible + * for serializing calls for given disposal list. + */ +static void maybe_free_mountpoint(struct mountpoint *mp, struct list_head *list) +{ + if (hlist_empty(&mp->m_list)) { + struct dentry *dentry = mp->m_dentry; + spin_lock(&dentry->d_lock); + dentry->d_flags &= ~DCACHE_MOUNTED; + spin_unlock(&dentry->d_lock); + dput_to_list(dentry, list); + hlist_del(&mp->m_hash); + kfree(mp); + } +} + +/* + * locks: mount_lock [read_seqlock_excl], namespace_sem [excl] + */ +static void unpin_mountpoint(struct pinned_mountpoint *m) +{ + if (m->mp) { + hlist_del(&m->node); + maybe_free_mountpoint(m->mp, &ex_mountpoints); + } +} + +static inline int check_mnt(const struct mount *mnt) +{ + return mnt->mnt_ns == current->nsproxy->mnt_ns; +} + +static inline bool check_anonymous_mnt(struct mount *mnt) +{ + u64 seq; + + if (!is_anon_ns(mnt->mnt_ns)) + return false; + + seq = mnt->mnt_ns->seq_origin; + return !seq || (seq == current->nsproxy->mnt_ns->ns.ns_id); +} + +/* + * vfsmount lock must be held for write + */ +static void touch_mnt_namespace(struct mnt_namespace *ns) +{ + if (ns) { + ns->event = ++event; + wake_up_interruptible(&ns->poll); + } +} + +/* + * vfsmount lock must be held for write + */ +static void __touch_mnt_namespace(struct mnt_namespace *ns) +{ + if (ns && ns->event != event) { + ns->event = event; + wake_up_interruptible(&ns->poll); + } +} + +/* + * locks: mount_lock[write_seqlock] + */ +static void __umount_mnt(struct mount *mnt, struct list_head *shrink_list) +{ + struct mountpoint *mp; + struct mount *parent = mnt->mnt_parent; + if (unlikely(parent->overmount == mnt)) + parent->overmount = NULL; + mnt->mnt_parent = mnt; + mnt->mnt_mountpoint = mnt->mnt.mnt_root; + list_del_init(&mnt->mnt_child); + hlist_del_init_rcu(&mnt->mnt_hash); + hlist_del_init(&mnt->mnt_mp_list); + mp = mnt->mnt_mp; + mnt->mnt_mp = NULL; + maybe_free_mountpoint(mp, shrink_list); +} + +/* + * locks: mount_lock[write_seqlock], namespace_sem[excl] (for ex_mountpoints) + */ +static void umount_mnt(struct mount *mnt) +{ + __umount_mnt(mnt, &ex_mountpoints); +} + +/* + * vfsmount lock must be held for write + */ +void mnt_set_mountpoint(struct mount *mnt, + struct mountpoint *mp, + struct mount *child_mnt) +{ + child_mnt->mnt_mountpoint = mp->m_dentry; + child_mnt->mnt_parent = mnt; + child_mnt->mnt_mp = mp; + hlist_add_head(&child_mnt->mnt_mp_list, &mp->m_list); +} + +static void make_visible(struct mount *mnt) +{ + struct mount *parent = mnt->mnt_parent; + if (unlikely(mnt->mnt_mountpoint == parent->mnt.mnt_root)) + parent->overmount = mnt; + hlist_add_head_rcu(&mnt->mnt_hash, + m_hash(&parent->mnt, mnt->mnt_mountpoint)); + list_add_tail(&mnt->mnt_child, &parent->mnt_mounts); +} + +/** + * attach_mnt - mount a mount, attach to @mount_hashtable and parent's + * list of child mounts + * @parent: the parent + * @mnt: the new mount + * @mp: the new mountpoint + * + * Mount @mnt at @mp on @parent. Then attach @mnt + * to @parent's child mount list and to @mount_hashtable. + * + * Note, when make_visible() is called @mnt->mnt_parent already points + * to the correct parent. + * + * Context: This function expects namespace_lock() and lock_mount_hash() + * to have been acquired in that order. + */ +static void attach_mnt(struct mount *mnt, struct mount *parent, + struct mountpoint *mp) +{ + mnt_set_mountpoint(parent, mp, mnt); + make_visible(mnt); +} + +void mnt_change_mountpoint(struct mount *parent, struct mountpoint *mp, struct mount *mnt) +{ + struct mountpoint *old_mp = mnt->mnt_mp; + + list_del_init(&mnt->mnt_child); + hlist_del_init(&mnt->mnt_mp_list); + hlist_del_init_rcu(&mnt->mnt_hash); + + attach_mnt(mnt, parent, mp); + + maybe_free_mountpoint(old_mp, &ex_mountpoints); +} + +static inline struct mount *node_to_mount(struct rb_node *node) +{ + return node ? rb_entry(node, struct mount, mnt_node) : NULL; +} + +static void mnt_add_to_ns(struct mnt_namespace *ns, struct mount *mnt) +{ + struct rb_node **link = &ns->mounts.rb_node; + struct rb_node *parent = NULL; + bool mnt_first_node = true, mnt_last_node = true; + + WARN_ON(mnt_ns_attached(mnt)); + WRITE_ONCE(mnt->mnt_ns, ns); + while (*link) { + parent = *link; + if (mnt->mnt_id_unique < node_to_mount(parent)->mnt_id_unique) { + link = &parent->rb_left; + mnt_last_node = false; + } else { + link = &parent->rb_right; + mnt_first_node = false; + } + } + + if (mnt_last_node) + ns->mnt_last_node = &mnt->mnt_node; + if (mnt_first_node) + ns->mnt_first_node = &mnt->mnt_node; + rb_link_node(&mnt->mnt_node, parent, link); + rb_insert_color(&mnt->mnt_node, &ns->mounts); + + if ((mnt->mnt.mnt_sb->s_type->fs_flags & FS_USERNS_MOUNT_RESTRICTED) && + mnt->mnt.mnt_root == mnt->mnt.mnt_sb->s_root) + hlist_add_head(&mnt->mnt_ns_visible, &ns->mnt_visible_mounts); + + mnt_notify_add(mnt); +} + +static struct mount *next_mnt(struct mount *p, struct mount *root) +{ + struct list_head *next = p->mnt_mounts.next; + if (next == &p->mnt_mounts) { + while (1) { + if (p == root) + return NULL; + next = p->mnt_child.next; + if (next != &p->mnt_parent->mnt_mounts) + break; + p = p->mnt_parent; + } + } + return list_entry(next, struct mount, mnt_child); +} + +static struct mount *skip_mnt_tree(struct mount *p) +{ + struct list_head *prev = p->mnt_mounts.prev; + while (prev != &p->mnt_mounts) { + p = list_entry(prev, struct mount, mnt_child); + prev = p->mnt_mounts.prev; + } + return p; +} + +/* + * vfsmount lock must be held for write + */ +static void commit_tree(struct mount *mnt) +{ + struct mnt_namespace *n = mnt->mnt_parent->mnt_ns; + + if (!mnt_ns_attached(mnt)) { + for (struct mount *m = mnt; m; m = next_mnt(m, mnt)) + mnt_add_to_ns(n, m); + n->nr_mounts += n->pending_mounts; + n->pending_mounts = 0; + } + + make_visible(mnt); + touch_mnt_namespace(n); +} + +static void setup_mnt(struct mount *m, struct dentry *root) +{ + struct super_block *s = root->d_sb; + + atomic_inc(&s->s_active); + m->mnt.mnt_sb = s; + m->mnt.mnt_root = dget(root); + m->mnt_mountpoint = m->mnt.mnt_root; + m->mnt_parent = m; + + guard(mount_locked_reader)(); + mnt_add_instance(m, s); +} + +/** + * vfs_create_mount - Create a mount for a configured superblock + * @fc: The configuration context with the superblock attached + * + * Create a mount to an already configured superblock. If necessary, the + * caller should invoke vfs_get_tree() before calling this. + * + * Note that this does not attach the mount to anything. + */ +struct vfsmount *vfs_create_mount(struct fs_context *fc) +{ + struct mount *mnt; + + if (!fc->root) + return ERR_PTR(-EINVAL); + + mnt = alloc_vfsmnt(fc->source); + if (!mnt) + return ERR_PTR(-ENOMEM); + + if (fc->sb_flags & SB_KERNMOUNT) + mnt->mnt.mnt_flags = MNT_INTERNAL; + + setup_mnt(mnt, fc->root); + + return &mnt->mnt; +} +EXPORT_SYMBOL(vfs_create_mount); + +struct vfsmount *fc_mount(struct fs_context *fc) +{ + int err = vfs_get_tree(fc); + if (!err) { + up_write(&fc->root->d_sb->s_umount); + return vfs_create_mount(fc); + } + return ERR_PTR(err); +} +EXPORT_SYMBOL(fc_mount); + +struct vfsmount *fc_mount_longterm(struct fs_context *fc) +{ + struct vfsmount *mnt = fc_mount(fc); + if (!IS_ERR(mnt)) + real_mount(mnt)->mnt_ns = MNT_NS_INTERNAL; + return mnt; +} +EXPORT_SYMBOL(fc_mount_longterm); + +struct vfsmount *vfs_kern_mount(struct file_system_type *type, + int flags, const char *name, + void *data) +{ + struct fs_context *fc; + struct vfsmount *mnt; + int ret = 0; + + if (!type) + return ERR_PTR(-EINVAL); + + fc = fs_context_for_mount(type, flags); + if (IS_ERR(fc)) + return ERR_CAST(fc); + + if (name) + ret = vfs_parse_fs_string(fc, "source", name); + if (!ret) + ret = parse_monolithic_mount_data(fc, data); + if (!ret) + mnt = fc_mount(fc); + else + mnt = ERR_PTR(ret); + + put_fs_context(fc); + return mnt; +} +EXPORT_SYMBOL_GPL(vfs_kern_mount); + +static struct mount *clone_mnt(struct mount *old, struct dentry *root, + int flag) +{ + struct mount *mnt; + int err; + + mnt = alloc_vfsmnt(old->mnt_devname); + if (!mnt) + return ERR_PTR(-ENOMEM); + + mnt->mnt.mnt_flags = READ_ONCE(old->mnt.mnt_flags) & + ~MNT_INTERNAL_FLAGS; + + if (flag & (CL_SLAVE | CL_PRIVATE)) + mnt->mnt_group_id = 0; /* not a peer of original */ + else + mnt->mnt_group_id = old->mnt_group_id; + + if ((flag & CL_MAKE_SHARED) && !mnt->mnt_group_id) { + err = mnt_alloc_group_id(mnt); + if (err) + goto out_free; + } + + if (mnt->mnt_group_id) + set_mnt_shared(mnt); + + mnt->mnt.mnt_idmap = mnt_idmap_get(mnt_idmap(&old->mnt)); + + setup_mnt(mnt, root); + + if (flag & CL_PRIVATE) // we are done with it + return mnt; + + if (peers(mnt, old)) + list_add(&mnt->mnt_share, &old->mnt_share); + + if ((flag & CL_SLAVE) && old->mnt_group_id) { + hlist_add_head(&mnt->mnt_slave, &old->mnt_slave_list); + mnt->mnt_master = old; + } else if (IS_MNT_SLAVE(old)) { + hlist_add_behind(&mnt->mnt_slave, &old->mnt_slave); + mnt->mnt_master = old->mnt_master; + } + return mnt; + + out_free: + mnt_free_id(mnt); + free_vfsmnt(mnt); + return ERR_PTR(err); +} + +static void cleanup_mnt(struct mount *mnt) +{ + struct hlist_node *p; + struct mount *m; + /* + * The warning here probably indicates that somebody messed + * up a mnt_want/drop_write() pair. If this happens, the + * filesystem was probably unable to make r/w->r/o transitions. + * The locking used to deal with mnt_count decrement provides barriers, + * so mnt_get_writers() below is safe. + */ + WARN_ON(mnt_get_writers(mnt)); + if (unlikely(mnt->mnt_pins.first)) + mnt_pin_kill(mnt); + hlist_for_each_entry_safe(m, p, &mnt->mnt_stuck_children, mnt_umount) { + hlist_del(&m->mnt_umount); + mntput(&m->mnt); + } + fsnotify_vfsmount_delete(&mnt->mnt); + dput(mnt->mnt.mnt_root); + deactivate_super(mnt->mnt.mnt_sb); + mnt_free_id(mnt); + call_rcu(&mnt->mnt_rcu, delayed_free_vfsmnt); +} + +static void __cleanup_mnt(struct rcu_head *head) +{ + cleanup_mnt(container_of(head, struct mount, mnt_rcu)); +} + +static LLIST_HEAD(delayed_mntput_list); +static void delayed_mntput(struct work_struct *unused) +{ + struct llist_node *node = llist_del_all(&delayed_mntput_list); + struct mount *m, *t; + + llist_for_each_entry_safe(m, t, node, mnt_llist) + cleanup_mnt(m); +} +static DECLARE_DELAYED_WORK(delayed_mntput_work, delayed_mntput); + +static void noinline mntput_no_expire_slowpath(struct mount *mnt) +{ + LIST_HEAD(list); + int count; + + VFS_BUG_ON(mnt->mnt_ns); + lock_mount_hash(); + /* + * make sure that if __legitimize_mnt() has not seen us grab + * mount_lock, we'll see their refcount increment here. + */ + smp_mb(); + mnt_add_count(mnt, -1); + count = mnt_get_count(mnt); + if (count != 0) { + WARN_ON(count < 0); + rcu_read_unlock(); + unlock_mount_hash(); + return; + } + if (unlikely(mnt->mnt.mnt_flags & MNT_DOOMED)) { + rcu_read_unlock(); + unlock_mount_hash(); + return; + } + mnt->mnt.mnt_flags |= MNT_DOOMED; + rcu_read_unlock(); + + mnt_del_instance(mnt); + if (unlikely(!list_empty(&mnt->mnt_expire))) + list_del(&mnt->mnt_expire); + + if (unlikely(!list_empty(&mnt->mnt_mounts))) { + struct mount *p, *tmp; + list_for_each_entry_safe(p, tmp, &mnt->mnt_mounts, mnt_child) { + __umount_mnt(p, &list); + hlist_add_head(&p->mnt_umount, &mnt->mnt_stuck_children); + } + } + unlock_mount_hash(); + shrink_dentry_list(&list); + + if (likely(!(mnt->mnt.mnt_flags & MNT_INTERNAL))) { + struct task_struct *task = current; + if (likely(!(task->flags & PF_KTHREAD))) { + init_task_work(&mnt->mnt_rcu, __cleanup_mnt); + if (!task_work_add(task, &mnt->mnt_rcu, TWA_RESUME)) + return; + } + if (llist_add(&mnt->mnt_llist, &delayed_mntput_list)) + schedule_delayed_work(&delayed_mntput_work, 1); + return; + } + cleanup_mnt(mnt); +} + +static void mntput_no_expire(struct mount *mnt) +{ + rcu_read_lock(); + if (likely(READ_ONCE(mnt->mnt_ns))) { + /* + * Since we don't do lock_mount_hash() here, + * ->mnt_ns can change under us. However, if it's + * non-NULL, then there's a reference that won't + * be dropped until after an RCU delay done after + * turning ->mnt_ns NULL. So if we observe it + * non-NULL under rcu_read_lock(), the reference + * we are dropping is not the final one. + */ + mnt_add_count(mnt, -1); + rcu_read_unlock(); + return; + } + mntput_no_expire_slowpath(mnt); +} + +void mntput(struct vfsmount *mnt) +{ + if (mnt) { + struct mount *m = real_mount(mnt); + /* avoid cacheline pingpong */ + if (unlikely(m->mnt_expiry_mark)) + WRITE_ONCE(m->mnt_expiry_mark, 0); + mntput_no_expire(m); + } +} +EXPORT_SYMBOL(mntput); + +struct vfsmount *mntget(struct vfsmount *mnt) +{ + if (mnt) + mnt_add_count(real_mount(mnt), 1); + return mnt; +} +EXPORT_SYMBOL(mntget); + +/* + * Make a mount point inaccessible to new lookups. + * Because there may still be current users, the caller MUST WAIT + * for an RCU grace period before destroying the mount point. + */ +void mnt_make_shortterm(struct vfsmount *mnt) +{ + if (mnt) + WRITE_ONCE(real_mount(mnt)->mnt_ns, NULL); +} + +/** + * path_is_mountpoint() - Check if path is a mount in the current namespace. + * @path: path to check + * + * d_mountpoint() can only be used reliably to establish if a dentry is + * not mounted in any namespace and that common case is handled inline. + * d_mountpoint() isn't aware of the possibility there may be multiple + * mounts using a given dentry in a different namespace. This function + * checks if the passed in path is a mountpoint rather than the dentry + * alone. + */ +bool path_is_mountpoint(const struct path *path) +{ + unsigned seq; + bool res; + + if (!d_mountpoint(path->dentry)) + return false; + + rcu_read_lock(); + do { + seq = read_seqbegin(&mount_lock); + res = __path_is_mountpoint(path); + } while (read_seqretry(&mount_lock, seq)); + rcu_read_unlock(); + + return res; +} +EXPORT_SYMBOL(path_is_mountpoint); + +struct vfsmount *mnt_clone_internal(const struct path *path) +{ + struct mount *p; + p = clone_mnt(real_mount(path->mnt), path->dentry, CL_PRIVATE); + if (IS_ERR(p)) + return ERR_CAST(p); + p->mnt.mnt_flags |= MNT_INTERNAL; + return &p->mnt; +} + +/* + * Returns the mount which either has the specified mnt_id, or has the next + * smallest id afer the specified one. + */ +static struct mount *mnt_find_id_at(struct mnt_namespace *ns, u64 mnt_id) +{ + struct rb_node *node = ns->mounts.rb_node; + struct mount *ret = NULL; + + while (node) { + struct mount *m = node_to_mount(node); + + if (mnt_id <= m->mnt_id_unique) { + ret = node_to_mount(node); + if (mnt_id == m->mnt_id_unique) + break; + node = node->rb_left; + } else { + node = node->rb_right; + } + } + return ret; +} + +/* + * Returns the mount which either has the specified mnt_id, or has the next + * greater id before the specified one. + */ +static struct mount *mnt_find_id_at_reverse(struct mnt_namespace *ns, u64 mnt_id) +{ + struct rb_node *node = ns->mounts.rb_node; + struct mount *ret = NULL; + + while (node) { + struct mount *m = node_to_mount(node); + + if (mnt_id >= m->mnt_id_unique) { + ret = node_to_mount(node); + if (mnt_id == m->mnt_id_unique) + break; + node = node->rb_right; + } else { + node = node->rb_left; + } + } + return ret; +} + +#ifdef CONFIG_PROC_FS + +/* iterator; we want it to have access to namespace_sem, thus here... */ +static void *m_start(struct seq_file *m, loff_t *pos) +{ + struct proc_mounts *p = m->private; + struct mount *mnt; + + down_read(&namespace_sem); + + mnt = mnt_find_id_at(p->ns, *pos); + if (mnt) + *pos = mnt->mnt_id_unique; + return mnt; +} + +static void *m_next(struct seq_file *m, void *v, loff_t *pos) +{ + struct mount *mnt = v; + struct rb_node *node = rb_next(&mnt->mnt_node); + + if (node) { + struct mount *next = node_to_mount(node); + *pos = next->mnt_id_unique; + return next; + } + + /* + * No more mounts. Set pos past current mount's ID so that if + * iteration restarts, mnt_find_id_at() returns NULL. + */ + *pos = mnt->mnt_id_unique + 1; + return NULL; +} + +static void m_stop(struct seq_file *m, void *v) +{ + up_read(&namespace_sem); +} + +static int m_show(struct seq_file *m, void *v) +{ + struct proc_mounts *p = m->private; + struct mount *r = v; + return p->show(m, &r->mnt); +} + +const struct seq_operations mounts_op = { + .start = m_start, + .next = m_next, + .stop = m_stop, + .show = m_show, +}; + +#endif /* CONFIG_PROC_FS */ + +/** + * may_umount_tree - check if a mount tree is busy + * @m: root of mount tree + * + * This is called to check if a tree of mounts has any + * open files, pwds, chroots or sub mounts that are + * busy. + */ +int may_umount_tree(struct vfsmount *m) +{ + struct mount *mnt = real_mount(m); + bool busy = false; + + /* write lock needed for mnt_get_count */ + lock_mount_hash(); + for (struct mount *p = mnt; p; p = next_mnt(p, mnt)) { + if (mnt_get_count(p) > (p == mnt ? 2 : 1)) { + busy = true; + break; + } + } + unlock_mount_hash(); + + return !busy; +} + +EXPORT_SYMBOL(may_umount_tree); + +/** + * may_umount - check if a mount point is busy + * @mnt: root of mount + * + * This is called to check if a mount point has any + * open files, pwds, chroots or sub mounts. If the + * mount has sub mounts this will return busy + * regardless of whether the sub mounts are busy. + * + * Doesn't take quota and stuff into account. IOW, in some cases it will + * give false negatives. The main reason why it's here is that we need + * a non-destructive way to look for easily umountable filesystems. + */ +int may_umount(struct vfsmount *mnt) +{ + int ret = 1; + down_read(&namespace_sem); + lock_mount_hash(); + if (propagate_mount_busy(real_mount(mnt), 2)) + ret = 0; + unlock_mount_hash(); + up_read(&namespace_sem); + return ret; +} + +EXPORT_SYMBOL(may_umount); + +#ifdef CONFIG_FSNOTIFY +static void mnt_notify(struct mount *p) +{ + if (!p->prev_ns && p->mnt_ns) { + fsnotify_mnt_attach(p->mnt_ns, &p->mnt); + } else if (p->prev_ns && !p->mnt_ns) { + fsnotify_mnt_detach(p->prev_ns, &p->mnt); + } else if (p->prev_ns == p->mnt_ns) { + fsnotify_mnt_move(p->mnt_ns, &p->mnt); + } else { + fsnotify_mnt_detach(p->prev_ns, &p->mnt); + fsnotify_mnt_attach(p->mnt_ns, &p->mnt); + } + p->prev_ns = p->mnt_ns; +} + +static void notify_mnt_list(void) +{ + struct mount *m, *tmp; + /* + * Notify about mounts that were added/reparented/detached/remain + * connected after unmount. + */ + list_for_each_entry_safe(m, tmp, ¬ify_list, to_notify) { + mnt_notify(m); + list_del_init(&m->to_notify); + } +} + +static bool need_notify_mnt_list(void) +{ + return !list_empty(¬ify_list); +} +#else +static void notify_mnt_list(void) +{ +} + +static bool need_notify_mnt_list(void) +{ + return false; +} +#endif + +static void free_mnt_ns(struct mnt_namespace *); +static void namespace_unlock(void) +{ + struct hlist_head head; + struct hlist_node *p; + struct mount *m; + struct mnt_namespace *ns = emptied_ns; + LIST_HEAD(list); + + hlist_move_list(&unmounted, &head); + list_splice_init(&ex_mountpoints, &list); + emptied_ns = NULL; + + if (need_notify_mnt_list()) { + /* + * No point blocking out concurrent readers while notifications + * are sent. This will also allow statmount()/listmount() to run + * concurrently. + */ + downgrade_write(&namespace_sem); + notify_mnt_list(); + up_read(&namespace_sem); + } else { + up_write(&namespace_sem); + } + if (unlikely(ns)) { + /* Make sure we notice when we leak mounts. */ + VFS_WARN_ON_ONCE(!mnt_ns_empty(ns)); + free_mnt_ns(ns); + } + + shrink_dentry_list(&list); + + if (likely(hlist_empty(&head))) + return; + + synchronize_rcu_expedited(); + + hlist_for_each_entry_safe(m, p, &head, mnt_umount) { + hlist_del(&m->mnt_umount); + mntput(&m->mnt); + } +} + +static inline void namespace_lock(void) +{ + down_write(&namespace_sem); +} + +enum umount_tree_flags { + UMOUNT_SYNC = 1, + UMOUNT_PROPAGATE = 2, + UMOUNT_CONNECTED = 4, +}; + +static bool disconnect_mount(struct mount *mnt, enum umount_tree_flags how) +{ + /* Leaving mounts connected is only valid for lazy umounts */ + if (how & UMOUNT_SYNC) + return true; + + /* A mount without a parent has nothing to be connected to */ + if (!mnt_has_parent(mnt)) + return true; + + /* Because the reference counting rules change when mounts are + * unmounted and connected, umounted mounts may not be + * connected to mounted mounts. + */ + if (!(mnt->mnt_parent->mnt.mnt_flags & MNT_UMOUNT)) + return true; + + /* Has it been requested that the mount remain connected? */ + if (how & UMOUNT_CONNECTED) + return false; + + /* Is the mount locked such that it needs to remain connected? */ + if (IS_MNT_LOCKED(mnt)) + return false; + + /* By default disconnect the mount */ + return true; +} + +/* + * mount_lock must be held + * namespace_sem must be held for write + */ +static void umount_tree(struct mount *mnt, enum umount_tree_flags how) +{ + LIST_HEAD(tmp_list); + struct mount *p; + + if (how & UMOUNT_PROPAGATE) + propagate_mount_unlock(mnt); + + /* Gather the mounts to umount */ + for (p = mnt; p; p = next_mnt(p, mnt)) { + p->mnt.mnt_flags |= MNT_UMOUNT; + if (mnt_ns_attached(p)) + move_from_ns(p); + list_add_tail(&p->mnt_list, &tmp_list); + } + + /* Hide the mounts from mnt_mounts */ + list_for_each_entry(p, &tmp_list, mnt_list) { + list_del_init(&p->mnt_child); + } + + /* Add propagated mounts to the tmp_list */ + if (how & UMOUNT_PROPAGATE) + propagate_umount(&tmp_list); + + bulk_make_private(&tmp_list); + + while (!list_empty(&tmp_list)) { + struct mnt_namespace *ns; + bool disconnect; + p = list_first_entry(&tmp_list, struct mount, mnt_list); + list_del_init(&p->mnt_expire); + list_del_init(&p->mnt_list); + ns = p->mnt_ns; + if (ns) { + ns->nr_mounts--; + __touch_mnt_namespace(ns); + } + WRITE_ONCE(p->mnt_ns, NULL); + if (how & UMOUNT_SYNC) + p->mnt.mnt_flags |= MNT_SYNC_UMOUNT; + + disconnect = disconnect_mount(p, how); + if (mnt_has_parent(p)) { + if (!disconnect) { + /* Don't forget about p */ + list_add_tail(&p->mnt_child, &p->mnt_parent->mnt_mounts); + } else { + umount_mnt(p); + } + } + if (disconnect) + hlist_add_head(&p->mnt_umount, &unmounted); + + /* + * At this point p->mnt_ns is NULL, notification will be queued + * only if + * + * - p->prev_ns is non-NULL *and* + * - p->prev_ns->n_fsnotify_marks is non-NULL + * + * This will preclude queuing the mount if this is a cleanup + * after a failed copy_tree() or destruction of an anonymous + * namespace, etc. + */ + mnt_notify_add(p); + } +} + +static void shrink_submounts(struct mount *mnt); + +static int do_umount_root(struct super_block *sb) +{ + int ret = 0; + + down_write(&sb->s_umount); + if (!sb_rdonly(sb)) { + struct fs_context *fc; + + fc = fs_context_for_reconfigure(sb->s_root, SB_RDONLY, + SB_RDONLY); + if (IS_ERR(fc)) { + ret = PTR_ERR(fc); + } else { + ret = parse_monolithic_mount_data(fc, NULL); + if (!ret) + ret = reconfigure_super(fc); + put_fs_context(fc); + } + } + up_write(&sb->s_umount); + return ret; +} + +static int do_umount(struct mount *mnt, int flags) +{ + struct super_block *sb = mnt->mnt.mnt_sb; + int retval; + + retval = security_sb_umount(&mnt->mnt, flags); + if (retval) + return retval; + + /* + * Allow userspace to request a mountpoint be expired rather than + * unmounting unconditionally. Unmount only happens if: + * (1) the mark is already set (the mark is cleared by mntput()) + * (2) the usage count == 1 [parent vfsmount] + 1 [sys_umount] + */ + if (flags & MNT_EXPIRE) { + if (&mnt->mnt == current->fs->root.mnt || + flags & (MNT_FORCE | MNT_DETACH)) + return -EINVAL; + + /* + * probably don't strictly need the lock here if we examined + * all race cases, but it's a slowpath. + */ + lock_mount_hash(); + if (!list_empty(&mnt->mnt_mounts) || mnt_get_count(mnt) != 2) { + unlock_mount_hash(); + return -EBUSY; + } + unlock_mount_hash(); + + if (!xchg(&mnt->mnt_expiry_mark, 1)) + return -EAGAIN; + } + + /* + * If we may have to abort operations to get out of this + * mount, and they will themselves hold resources we must + * allow the fs to do things. In the Unix tradition of + * 'Gee thats tricky lets do it in userspace' the umount_begin + * might fail to complete on the first run through as other tasks + * must return, and the like. Thats for the mount program to worry + * about for the moment. + */ + + if (flags & MNT_FORCE && sb->s_op->umount_begin) { + sb->s_op->umount_begin(sb); + } + + /* + * No sense to grab the lock for this test, but test itself looks + * somewhat bogus. Suggestions for better replacement? + * Ho-hum... In principle, we might treat that as umount + switch + * to rootfs. GC would eventually take care of the old vfsmount. + * Actually it makes sense, especially if rootfs would contain a + * /reboot - static binary that would close all descriptors and + * call reboot(9). Then init(8) could umount root and exec /reboot. + */ + if (&mnt->mnt == current->fs->root.mnt && !(flags & MNT_DETACH)) { + /* + * Special case for "unmounting" root ... + * we just try to remount it readonly. + */ + if (!ns_capable(sb->s_user_ns, CAP_SYS_ADMIN)) + return -EPERM; + return do_umount_root(sb); + } + + namespace_lock(); + lock_mount_hash(); + + /* Repeat the earlier racy checks, now that we are holding the locks */ + retval = -EINVAL; + if (!check_mnt(mnt)) + goto out; + + if (mnt->mnt.mnt_flags & MNT_LOCKED) + goto out; + + if (!mnt_has_parent(mnt)) /* not the absolute root */ + goto out; + + event++; + if (flags & MNT_DETACH) { + umount_tree(mnt, UMOUNT_PROPAGATE); + retval = 0; + } else { + smp_mb(); // paired with __legitimize_mnt() + shrink_submounts(mnt); + retval = -EBUSY; + if (!propagate_mount_busy(mnt, 2)) { + umount_tree(mnt, UMOUNT_PROPAGATE|UMOUNT_SYNC); + retval = 0; + } + } +out: + unlock_mount_hash(); + namespace_unlock(); + return retval; +} + +/* + * __detach_mounts - lazily unmount all mounts on the specified dentry + * + * During unlink, rmdir, and d_drop it is possible to loose the path + * to an existing mountpoint, and wind up leaking the mount. + * detach_mounts allows lazily unmounting those mounts instead of + * leaking them. + * + * The caller may hold dentry->d_inode->i_rwsem. + */ +void __detach_mounts(struct dentry *dentry) +{ + struct pinned_mountpoint mp = {}; + struct mount *mnt; + + guard(namespace_excl)(); + guard(mount_writer)(); + + if (!lookup_mountpoint(dentry, &mp)) + return; + + event++; + while (mp.node.next) { + mnt = hlist_entry(mp.node.next, struct mount, mnt_mp_list); + if (mnt->mnt.mnt_flags & MNT_UMOUNT) { + umount_mnt(mnt); + hlist_add_head(&mnt->mnt_umount, &unmounted); + } + else umount_tree(mnt, UMOUNT_CONNECTED); + } + unpin_mountpoint(&mp); +} + +/* + * Is the caller allowed to modify his namespace? + */ +bool may_mount(void) +{ + return ns_capable(current->nsproxy->mnt_ns->user_ns, CAP_SYS_ADMIN); +} + +static void warn_mandlock(void) +{ + pr_warn_once("=======================================================\n" + "WARNING: The mand mount option has been deprecated and\n" + " and is ignored by this kernel. Remove the mand\n" + " option from the mount to silence this warning.\n" + "=======================================================\n"); +} + +static int can_umount(const struct path *path, int flags) +{ + struct mount *mnt = real_mount(path->mnt); + struct super_block *sb = path->dentry->d_sb; + + if (!may_mount()) + return -EPERM; + if (!path_mounted(path)) + return -EINVAL; + if (!check_mnt(mnt)) + return -EINVAL; + if (mnt->mnt.mnt_flags & MNT_LOCKED) /* Check optimistically */ + return -EINVAL; + if (flags & MNT_FORCE && !ns_capable(sb->s_user_ns, CAP_SYS_ADMIN)) + return -EPERM; + return 0; +} + +// caller is responsible for flags being sane +int path_umount(const struct path *path, int flags) +{ + struct mount *mnt = real_mount(path->mnt); + int ret; + + ret = can_umount(path, flags); + if (!ret) + ret = do_umount(mnt, flags); + + /* we mustn't call path_put() as that would clear mnt_expiry_mark */ + dput(path->dentry); + mntput_no_expire(mnt); + return ret; +} + +static int ksys_umount(char __user *name, int flags) +{ + int lookup_flags = LOOKUP_MOUNTPOINT; + struct path path; + int ret; + + // basic validity checks done first + if (flags & ~(MNT_FORCE | MNT_DETACH | MNT_EXPIRE | UMOUNT_NOFOLLOW)) + return -EINVAL; + + if (!(flags & UMOUNT_NOFOLLOW)) + lookup_flags |= LOOKUP_FOLLOW; + ret = user_path_at(AT_FDCWD, name, lookup_flags, &path); + if (ret) + return ret; + return path_umount(&path, flags); +} + +SYSCALL_DEFINE2(umount, char __user *, name, int, flags) +{ + return ksys_umount(name, flags); +} + +#ifdef __ARCH_WANT_SYS_OLDUMOUNT + +/* + * The 2.0 compatible umount. No flags. + */ +SYSCALL_DEFINE1(oldumount, char __user *, name) +{ + return ksys_umount(name, 0); +} + +#endif + +static bool is_mnt_ns_file(struct dentry *dentry) +{ + struct ns_common *ns; + + /* Is this a proxy for a mount namespace? */ + if (dentry->d_op != &ns_dentry_operations) + return false; + + ns = d_inode(dentry)->i_private; + + return ns->ops == &mntns_operations; +} + +struct ns_common *from_mnt_ns(struct mnt_namespace *mnt) +{ + return &mnt->ns; +} + +struct mnt_namespace *get_sequential_mnt_ns(struct mnt_namespace *mntns, bool previous) +{ + struct ns_common *ns; + + guard(rcu)(); + + for (;;) { + ns = ns_tree_adjoined_rcu(mntns, previous); + if (IS_ERR(ns)) + return ERR_CAST(ns); + + mntns = to_mnt_ns(ns); + + /* + * The last passive reference count is put with RCU + * delay so accessing the mount namespace is not just + * safe but all relevant members are still valid. + */ + if (!ns_capable_noaudit(mntns->user_ns, CAP_SYS_ADMIN)) + continue; + + /* + * We need an active reference count as we're persisting + * the mount namespace and it might already be on its + * deathbed. + */ + if (!ns_ref_get(mntns)) + continue; + + return mntns; + } +} + +struct mnt_namespace *mnt_ns_from_dentry(struct dentry *dentry) +{ + if (!is_mnt_ns_file(dentry)) + return NULL; + + return to_mnt_ns(get_proc_ns(dentry->d_inode)); +} + +static bool mnt_ns_loop(struct dentry *dentry) +{ + /* Could bind mounting the mount namespace inode cause a + * mount namespace loop? + */ + struct mnt_namespace *mnt_ns = mnt_ns_from_dentry(dentry); + + if (!mnt_ns) + return false; + + return current->nsproxy->mnt_ns->ns.ns_id >= mnt_ns->ns.ns_id; +} + +struct mount *copy_tree(struct mount *src_root, struct dentry *dentry, + int flag) +{ + struct mount *res, *src_parent, *src_root_child, *src_mnt, + *dst_parent, *dst_mnt; + + if (!(flag & CL_COPY_UNBINDABLE) && IS_MNT_UNBINDABLE(src_root)) + return ERR_PTR(-EINVAL); + + if (!(flag & CL_COPY_MNT_NS_FILE) && is_mnt_ns_file(dentry)) + return ERR_PTR(-EINVAL); + + res = dst_mnt = clone_mnt(src_root, dentry, flag); + if (IS_ERR(dst_mnt)) + return dst_mnt; + + src_parent = src_root; + + list_for_each_entry(src_root_child, &src_root->mnt_mounts, mnt_child) { + if (!is_subdir(src_root_child->mnt_mountpoint, dentry)) + continue; + + for (src_mnt = src_root_child; src_mnt; + src_mnt = next_mnt(src_mnt, src_root_child)) { + if (!(flag & CL_COPY_UNBINDABLE) && + IS_MNT_UNBINDABLE(src_mnt)) { + if (src_mnt->mnt.mnt_flags & MNT_LOCKED) { + /* Both unbindable and locked. */ + dst_mnt = ERR_PTR(-EPERM); + goto out; + } else { + src_mnt = skip_mnt_tree(src_mnt); + continue; + } + } + if (!(flag & CL_COPY_MNT_NS_FILE) && + is_mnt_ns_file(src_mnt->mnt.mnt_root)) { + src_mnt = skip_mnt_tree(src_mnt); + continue; + } + while (src_parent != src_mnt->mnt_parent) { + src_parent = src_parent->mnt_parent; + dst_mnt = dst_mnt->mnt_parent; + } + + src_parent = src_mnt; + dst_parent = dst_mnt; + dst_mnt = clone_mnt(src_mnt, src_mnt->mnt.mnt_root, flag); + if (IS_ERR(dst_mnt)) + goto out; + lock_mount_hash(); + if (src_mnt->mnt.mnt_flags & MNT_LOCKED) + dst_mnt->mnt.mnt_flags |= MNT_LOCKED; + if (unlikely(flag & CL_EXPIRE)) { + /* stick the duplicate mount on the same expiry + * list as the original if that was on one */ + if (!list_empty(&src_mnt->mnt_expire)) + list_add(&dst_mnt->mnt_expire, + &src_mnt->mnt_expire); + } + attach_mnt(dst_mnt, dst_parent, src_parent->mnt_mp); + unlock_mount_hash(); + } + } + return res; + +out: + if (res) { + lock_mount_hash(); + umount_tree(res, UMOUNT_SYNC); + unlock_mount_hash(); + } + return dst_mnt; +} + +static inline bool extend_array(struct path **res, struct path **to_free, + unsigned n, unsigned *count, unsigned new_count) +{ + struct path *p; + + if (likely(n < *count)) + return true; + p = kmalloc_objs(struct path, new_count); + if (p && *count) + memcpy(p, *res, *count * sizeof(struct path)); + *count = new_count; + kfree(*to_free); + *to_free = *res = p; + return p; +} + +const struct path *collect_paths(const struct path *path, + struct path *prealloc, unsigned count) +{ + struct mount *root = real_mount(path->mnt); + struct mount *child; + struct path *res = prealloc, *to_free = NULL; + unsigned n = 0; + + guard(namespace_shared)(); + + if (!check_mnt(root)) + return ERR_PTR(-EINVAL); + if (!extend_array(&res, &to_free, 0, &count, 32)) + return ERR_PTR(-ENOMEM); + res[n++] = *path; + list_for_each_entry(child, &root->mnt_mounts, mnt_child) { + if (!is_subdir(child->mnt_mountpoint, path->dentry)) + continue; + for (struct mount *m = child; m; m = next_mnt(m, child)) { + if (!extend_array(&res, &to_free, n, &count, 2 * count)) + return ERR_PTR(-ENOMEM); + res[n].mnt = &m->mnt; + res[n].dentry = m->mnt.mnt_root; + n++; + } + } + if (!extend_array(&res, &to_free, n, &count, count + 1)) + return ERR_PTR(-ENOMEM); + memset(res + n, 0, (count - n) * sizeof(struct path)); + for (struct path *p = res; p->mnt; p++) + path_get(p); + return res; +} + +void drop_collected_paths(const struct path *paths, const struct path *prealloc) +{ + for (const struct path *p = paths; p->mnt; p++) + path_put(p); + if (paths != prealloc) + kfree(paths); +} + +static struct mnt_namespace *alloc_mnt_ns(struct user_namespace *, bool); + +void dissolve_on_fput(struct vfsmount *mnt) +{ + struct mount *m = real_mount(mnt); + + /* + * m used to be the root of anon namespace; if it still is one, + * we need to dissolve the mount tree and free that namespace. + * Let's try to avoid taking namespace_sem if we can determine + * that there's nothing to do without it - rcu_read_lock() is + * enough to make anon_ns_root() memory-safe and once m has + * left its namespace, it's no longer our concern, since it will + * never become a root of anon ns again. + */ + + scoped_guard(rcu) { + if (!anon_ns_root(m)) + return; + } + + scoped_guard(namespace_excl) { + if (!anon_ns_root(m)) + return; + + emptied_ns = m->mnt_ns; + lock_mount_hash(); + umount_tree(m, UMOUNT_CONNECTED); + unlock_mount_hash(); + } +} + +/* locks: namespace_shared && pinned(mnt) || mount_locked_reader */ +static bool __has_locked_children(struct mount *mnt, struct dentry *dentry) +{ + struct mount *child; + + list_for_each_entry(child, &mnt->mnt_mounts, mnt_child) { + if (!is_subdir(child->mnt_mountpoint, dentry)) + continue; + + if (child->mnt.mnt_flags & MNT_LOCKED) + return true; + } + return false; +} + +bool has_locked_children(struct mount *mnt, struct dentry *dentry) +{ + guard(mount_locked_reader)(); + return __has_locked_children(mnt, dentry); +} + +/* + * Check that there aren't references to earlier/same mount namespaces in the + * specified subtree. Such references can act as pins for mount namespaces + * that aren't checked by the mount-cycle checking code, thereby allowing + * cycles to be made. + * + * locks: mount_locked_reader || namespace_shared && pinned(subtree) + */ +static bool check_for_nsfs_mounts(struct mount *subtree) +{ + for (struct mount *p = subtree; p; p = next_mnt(p, subtree)) + if (mnt_ns_loop(p->mnt.mnt_root)) + return false; + return true; +} + +/** + * clone_private_mount - create a private clone of a path + * @path: path to clone + * + * This creates a new vfsmount, which will be the clone of @path. The new mount + * will not be attached anywhere in the namespace and will be private (i.e. + * changes to the originating mount won't be propagated into this). + * + * This assumes caller has called or done the equivalent of may_mount(). + * + * Release with mntput(). + */ +struct vfsmount *clone_private_mount(const struct path *path) +{ + struct mount *old_mnt = real_mount(path->mnt); + struct mount *new_mnt; + + guard(namespace_shared)(); + + if (IS_MNT_UNBINDABLE(old_mnt)) + return ERR_PTR(-EINVAL); + + /* + * Make sure the source mount is acceptable. + * Anything mounted in our mount namespace is allowed. + * Otherwise, it must be the root of an anonymous mount + * namespace, and we need to make sure no namespace + * loops get created. + */ + if (!check_mnt(old_mnt)) { + if (!anon_ns_root(old_mnt)) + return ERR_PTR(-EINVAL); + + if (!check_for_nsfs_mounts(old_mnt)) + return ERR_PTR(-EINVAL); + } + + if (!ns_capable(old_mnt->mnt_ns->user_ns, CAP_SYS_ADMIN)) + return ERR_PTR(-EPERM); + + if (__has_locked_children(old_mnt, path->dentry)) + return ERR_PTR(-EINVAL); + + new_mnt = clone_mnt(old_mnt, path->dentry, CL_PRIVATE); + if (IS_ERR(new_mnt)) + return ERR_PTR(-EINVAL); + + /* Longterm mount to be removed by kern_unmount*() */ + new_mnt->mnt_ns = MNT_NS_INTERNAL; + return &new_mnt->mnt; +} +EXPORT_SYMBOL_GPL(clone_private_mount); + +static void lock_mnt_tree(struct mount *mnt) +{ + struct mount *p; + + for (p = mnt; p; p = next_mnt(p, mnt)) { + int flags = p->mnt.mnt_flags; + /* Don't allow unprivileged users to change mount flags */ + flags |= MNT_LOCK_ATIME; + + if (flags & MNT_READONLY) + flags |= MNT_LOCK_READONLY; + + if (flags & MNT_NODEV) + flags |= MNT_LOCK_NODEV; + + if (flags & MNT_NOSUID) + flags |= MNT_LOCK_NOSUID; + + if (flags & MNT_NOEXEC) + flags |= MNT_LOCK_NOEXEC; + /* Don't allow unprivileged users to reveal what is under a mount */ + if (list_empty(&p->mnt_expire) && p != mnt) + flags |= MNT_LOCKED; + p->mnt.mnt_flags = flags; + } +} + +static void cleanup_group_ids(struct mount *mnt, struct mount *end) +{ + struct mount *p; + + for (p = mnt; p != end; p = next_mnt(p, mnt)) { + if (p->mnt_group_id && !IS_MNT_SHARED(p)) + mnt_release_group_id(p); + } +} + +static int invent_group_ids(struct mount *mnt, bool recurse) +{ + struct mount *p; + + for (p = mnt; p; p = recurse ? next_mnt(p, mnt) : NULL) { + if (!p->mnt_group_id) { + int err = mnt_alloc_group_id(p); + if (err) { + cleanup_group_ids(mnt, p); + return err; + } + } + } + + return 0; +} + +int count_mounts(struct mnt_namespace *ns, struct mount *mnt) +{ + unsigned int max = READ_ONCE(sysctl_mount_max); + unsigned int mounts = 0; + struct mount *p; + + if (ns->nr_mounts >= max) + return -ENOSPC; + max -= ns->nr_mounts; + if (ns->pending_mounts >= max) + return -ENOSPC; + max -= ns->pending_mounts; + + for (p = mnt; p; p = next_mnt(p, mnt)) + mounts++; + + if (mounts > max) + return -ENOSPC; + + ns->pending_mounts += mounts; + return 0; +} + +enum mnt_tree_flags_t { + MNT_TREE_BENEATH = BIT(0), + MNT_TREE_PROPAGATION = BIT(1), +}; + +/** + * attach_recursive_mnt - attach a source mount tree + * @source_mnt: mount tree to be attached + * @dest: the context for mounting at the place where the tree should go + * + * NOTE: in the table below explains the semantics when a source mount + * of a given type is attached to a destination mount of a given type. + * --------------------------------------------------------------------------- + * | BIND MOUNT OPERATION | + * |************************************************************************** + * | source-->| shared | private | slave | unbindable | + * | dest | | | | | + * | | | | | | | + * | v | | | | | + * |************************************************************************** + * | shared | shared (++) | shared (+) | shared(+++)| invalid | + * | | | | | | + * |non-shared| shared (+) | private | slave (*) | invalid | + * *************************************************************************** + * A bind operation clones the source mount and mounts the clone on the + * destination mount. + * + * (++) the cloned mount is propagated to all the mounts in the propagation + * tree of the destination mount and the cloned mount is added to + * the peer group of the source mount. + * (+) the cloned mount is created under the destination mount and is marked + * as shared. The cloned mount is added to the peer group of the source + * mount. + * (+++) the mount is propagated to all the mounts in the propagation tree + * of the destination mount and the cloned mount is made slave + * of the same master as that of the source mount. The cloned mount + * is marked as 'shared and slave'. + * (*) the cloned mount is made a slave of the same master as that of the + * source mount. + * + * --------------------------------------------------------------------------- + * | MOVE MOUNT OPERATION | + * |************************************************************************** + * | source-->| shared | private | slave | unbindable | + * | dest | | | | | + * | | | | | | | + * | v | | | | | + * |************************************************************************** + * | shared | shared (+) | shared (+) | shared(+++) | invalid | + * | | | | | | + * |non-shared| shared (+*) | private | slave (*) | unbindable | + * *************************************************************************** + * + * (+) the mount is moved to the destination. And is then propagated to + * all the mounts in the propagation tree of the destination mount. + * (+*) the mount is moved to the destination. + * (+++) the mount is moved to the destination and is then propagated to + * all the mounts belonging to the destination mount's propagation tree. + * the mount is marked as 'shared and slave'. + * (*) the mount continues to be a slave at the new location. + * + * if the source mount is a tree, the operations explained above is + * applied to each mount in the tree. + * Must be called without spinlocks held, since this function can sleep + * in allocations. + * + * Context: The function expects namespace_lock() to be held. + * Return: If @source_mnt was successfully attached 0 is returned. + * Otherwise a negative error code is returned. + */ +static int attach_recursive_mnt(struct mount *source_mnt, + const struct pinned_mountpoint *dest) +{ + struct user_namespace *user_ns = current->nsproxy->mnt_ns->user_ns; + struct mount *dest_mnt = dest->parent; + struct mountpoint *dest_mp = dest->mp; + HLIST_HEAD(tree_list); + struct mnt_namespace *ns = dest_mnt->mnt_ns; + struct pinned_mountpoint root = {}; + struct mountpoint *shorter = NULL; + struct mount *child, *p; + struct mount *top; + struct hlist_node *n; + int err = 0; + bool moving = mnt_has_parent(source_mnt); + + /* + * Preallocate a mountpoint in case the new mounts need to be + * mounted beneath mounts on the same mountpoint. + */ + for (top = source_mnt; unlikely(top->overmount); top = top->overmount) { + if (!shorter && is_mnt_ns_file(top->mnt.mnt_root)) + shorter = top->mnt_mp; + } + err = get_mountpoint(top->mnt.mnt_root, &root); + if (err) + return err; + + /* Is there space to add these mounts to the mount namespace? */ + if (!moving) { + err = count_mounts(ns, source_mnt); + if (err) + goto out; + } + + if (IS_MNT_SHARED(dest_mnt)) { + err = invent_group_ids(source_mnt, true); + if (err) + goto out; + err = propagate_mnt(dest_mnt, dest_mp, source_mnt, &tree_list); + } + lock_mount_hash(); + if (err) + goto out_cleanup_ids; + + if (IS_MNT_SHARED(dest_mnt)) { + for (p = source_mnt; p; p = next_mnt(p, source_mnt)) + set_mnt_shared(p); + } + + if (moving) { + umount_mnt(source_mnt); + mnt_notify_add(source_mnt); + /* if the mount is moved, it should no longer be expired + * automatically */ + list_del_init(&source_mnt->mnt_expire); + } else { + if (source_mnt->mnt_ns) { + /* move from anon - the caller will destroy */ + emptied_ns = source_mnt->mnt_ns; + for (p = source_mnt; p; p = next_mnt(p, source_mnt)) + move_from_ns(p); + } + } + + mnt_set_mountpoint(dest_mnt, dest_mp, source_mnt); + /* + * Now the original copy is in the same state as the secondaries - + * its root attached to mountpoint, but not hashed and all mounts + * in it are either in our namespace or in no namespace at all. + * Add the original to the list of copies and deal with the + * rest of work for all of them uniformly. + */ + hlist_add_head(&source_mnt->mnt_hash, &tree_list); + + hlist_for_each_entry_safe(child, n, &tree_list, mnt_hash) { + struct mount *q; + hlist_del_init(&child->mnt_hash); + /* Notice when we are propagating across user namespaces */ + if (child->mnt_parent->mnt_ns->user_ns != user_ns) + lock_mnt_tree(child); + q = __lookup_mnt(&child->mnt_parent->mnt, + child->mnt_mountpoint); + commit_tree(child); + if (q) { + struct mount *r = topmost_overmount(child); + struct mountpoint *mp = root.mp; + + if (unlikely(shorter) && child != source_mnt) + mp = shorter; + /* + * If @q was locked it was meant to hide + * whatever was under it. Let @child take over + * that job and lock it, then we can unlock @q. + * That'll allow another namespace to shed @q + * and reveal @child. Clearly, that mounter + * consented to this by not severing the mount + * relationship. Otherwise, what's the point. + */ + if (IS_MNT_LOCKED(q)) { + child->mnt.mnt_flags |= MNT_LOCKED; + q->mnt.mnt_flags &= ~MNT_LOCKED; + } + mnt_change_mountpoint(r, mp, q); + } + } + unpin_mountpoint(&root); + unlock_mount_hash(); + + return 0; + + out_cleanup_ids: + while (!hlist_empty(&tree_list)) { + child = hlist_entry(tree_list.first, struct mount, mnt_hash); + child->mnt_parent->mnt_ns->pending_mounts = 0; + umount_tree(child, UMOUNT_SYNC); + } + unlock_mount_hash(); + cleanup_group_ids(source_mnt, NULL); + out: + ns->pending_mounts = 0; + + read_seqlock_excl(&mount_lock); + unpin_mountpoint(&root); + read_sequnlock_excl(&mount_lock); + + return err; +} + +static inline struct mount *where_to_mount(const struct path *path, + struct dentry **dentry, + bool beneath) +{ + struct mount *m; + + if (unlikely(beneath)) { + m = topmost_overmount(real_mount(path->mnt)); + *dentry = m->mnt_mountpoint; + return m->mnt_parent; + } + m = __lookup_mnt(path->mnt, path->dentry); + if (unlikely(m)) { + m = topmost_overmount(m); + *dentry = m->mnt.mnt_root; + return m; + } + *dentry = path->dentry; + return real_mount(path->mnt); +} + +/** + * do_lock_mount - acquire environment for mounting + * @path: target path + * @res: context to set up + * @beneath: whether the intention is to mount beneath @path + * + * To mount something at given location, we need + * namespace_sem locked exclusive + * inode of dentry we are mounting on locked exclusive + * struct mountpoint for that dentry + * struct mount we are mounting on + * + * Results are stored in caller-supplied context (pinned_mountpoint); + * on success we have res->parent and res->mp pointing to parent and + * mountpoint respectively and res->node inserted into the ->m_list + * of the mountpoint, making sure the mountpoint won't disappear. + * On failure we have res->parent set to ERR_PTR(-E...), res->mp + * left NULL, res->node - empty. + * In case of success do_lock_mount returns with locks acquired (in + * proper order - inode lock nests outside of namespace_sem). + * + * Request to mount on overmounted location is treated as "mount on + * top of whatever's overmounting it"; request to mount beneath + * a location - "mount immediately beneath the topmost mount at that + * place". + * + * In all cases the location must not have been unmounted and the + * chosen mountpoint must be allowed to be mounted on. For "beneath" + * case we also require the location to be at the root of a mount + * that has something mounted on top of it (i.e. has an overmount). + */ +static void do_lock_mount(const struct path *path, + struct pinned_mountpoint *res, + bool beneath) +{ + int err; + + if (unlikely(beneath) && !path_mounted(path)) { + res->parent = ERR_PTR(-EINVAL); + return; + } + + do { + struct dentry *dentry, *d; + struct mount *m, *n; + + scoped_guard(mount_locked_reader) { + m = where_to_mount(path, &dentry, beneath); + if (&m->mnt != path->mnt) { + mntget(&m->mnt); + dget(dentry); + } + } + + inode_lock(dentry->d_inode); + namespace_lock(); + + // check if the chain of mounts (if any) has changed. + scoped_guard(mount_locked_reader) + n = where_to_mount(path, &d, beneath); + + if (unlikely(n != m || dentry != d)) + err = -EAGAIN; // something moved, retry + else if (unlikely(cant_mount(dentry) || !is_mounted(path->mnt))) + err = -ENOENT; // not to be mounted on + else if (beneath && &m->mnt == path->mnt && !m->overmount) + err = -EINVAL; + else + err = get_mountpoint(dentry, res); + + if (unlikely(err)) { + res->parent = ERR_PTR(err); + namespace_unlock(); + inode_unlock(dentry->d_inode); + } else { + res->parent = m; + } + /* + * Drop the temporary references. This is subtle - on success + * we are doing that under namespace_sem, which would normally + * be forbidden. However, in that case we are guaranteed that + * refcounts won't reach zero, since we know that path->mnt + * is mounted and thus all mounts reachable from it are pinned + * and stable, along with their mountpoints and roots. + */ + if (&m->mnt != path->mnt) { + dput(dentry); + mntput(&m->mnt); + } + } while (err == -EAGAIN); +} + +static void __unlock_mount(struct pinned_mountpoint *m) +{ + inode_unlock(m->mp->m_dentry->d_inode); + read_seqlock_excl(&mount_lock); + unpin_mountpoint(m); + read_sequnlock_excl(&mount_lock); + namespace_unlock(); +} + +static inline void unlock_mount(struct pinned_mountpoint *m) +{ + if (!IS_ERR(m->parent)) + __unlock_mount(m); +} + +static void lock_mount_exact(const struct path *path, + struct pinned_mountpoint *mp, bool copy_mount, + unsigned int copy_flags); + +#define LOCK_MOUNT_MAYBE_BENEATH(mp, path, beneath) \ + struct pinned_mountpoint mp __cleanup(unlock_mount) = {}; \ + do_lock_mount((path), &mp, (beneath)) +#define LOCK_MOUNT(mp, path) LOCK_MOUNT_MAYBE_BENEATH(mp, (path), false) +#define LOCK_MOUNT_EXACT(mp, path) \ + struct pinned_mountpoint mp __cleanup(unlock_mount) = {}; \ + lock_mount_exact((path), &mp, false, 0) +#define LOCK_MOUNT_EXACT_COPY(mp, path, copy_flags) \ + struct pinned_mountpoint mp __cleanup(unlock_mount) = {}; \ + lock_mount_exact((path), &mp, true, (copy_flags)) + +static int graft_tree(struct mount *mnt, const struct pinned_mountpoint *mp) +{ + if (mnt->mnt.mnt_sb->s_flags & SB_NOUSER) + return -EINVAL; + + if (d_is_dir(mp->mp->m_dentry) != + d_is_dir(mnt->mnt.mnt_root)) + return -ENOTDIR; + + return attach_recursive_mnt(mnt, mp); +} + +static int may_change_propagation(const struct mount *m) +{ + struct mnt_namespace *ns = m->mnt_ns; + + // it must be mounted in some namespace + if (IS_ERR_OR_NULL(ns)) // is_mounted() + return -EINVAL; + // and the caller must be admin in userns of that namespace + if (!ns_capable(ns->user_ns, CAP_SYS_ADMIN)) + return -EPERM; + return 0; +} + +/* + * Sanity check the flags to change_mnt_propagation. + */ + +static int flags_to_propagation_type(int ms_flags) +{ + int type = ms_flags & ~(MS_REC | MS_SILENT); + + /* Fail if any non-propagation flags are set */ + if (type & ~(MS_SHARED | MS_PRIVATE | MS_SLAVE | MS_UNBINDABLE)) + return 0; + /* Only one propagation flag should be set */ + if (!is_power_of_2(type)) + return 0; + return type; +} + +/* + * recursively change the type of the mountpoint. + */ +static int do_change_type(const struct path *path, int ms_flags) +{ + struct mount *m; + struct mount *mnt = real_mount(path->mnt); + int recurse = ms_flags & MS_REC; + int type; + int err; + + if (!path_mounted(path)) + return -EINVAL; + + type = flags_to_propagation_type(ms_flags); + if (!type) + return -EINVAL; + + guard(namespace_excl)(); + + err = may_change_propagation(mnt); + if (err) + return err; + + if (type == MS_SHARED) { + err = invent_group_ids(mnt, recurse); + if (err) + return err; + } + + for (m = mnt; m; m = (recurse ? next_mnt(m, mnt) : NULL)) + change_mnt_propagation(m, type); + + guard(mount_locked_reader)(); + touch_mnt_namespace(mnt->mnt_ns); + + return 0; +} + +/* may_copy_tree() - check if a mount tree can be copied + * @path: path to the mount tree to be copied + * + * This helper checks if the caller may copy the mount tree starting + * from @path->mnt. The caller may copy the mount tree under the + * following circumstances: + * + * (1) The caller is located in the mount namespace of the mount tree. + * This also implies that the mount does not belong to an anonymous + * mount namespace. + * (2) The caller tries to copy an nfs mount referring to a mount + * namespace, i.e., the caller is trying to copy a mount namespace + * entry from nsfs. + * (3) The caller tries to copy a pidfs mount referring to a pidfd. + * (4) The caller is trying to copy a mount tree that belongs to an + * anonymous mount namespace. + * + * For that to be safe, this helper enforces that the origin mount + * namespace the anonymous mount namespace was created from is the + * same as the caller's mount namespace by comparing the sequence + * numbers. + * + * This is not strictly necessary. The current semantics of the new + * mount api enforce that the caller must be located in the same + * mount namespace as the mount tree it interacts with. Using the + * origin sequence number preserves these semantics even for + * anonymous mount namespaces. However, one could envision extending + * the api to directly operate across mount namespace if needed. + * + * The ownership of a non-anonymous mount namespace such as the + * caller's cannot change. + * => We know that the caller's mount namespace is stable. + * + * If the origin sequence number of the anonymous mount namespace is + * the same as the sequence number of the caller's mount namespace. + * => The owning namespaces are the same. + * + * ==> The earlier capability check on the owning namespace of the + * caller's mount namespace ensures that the caller has the + * ability to copy the mount tree. + * + * Returns true if the mount tree can be copied, false otherwise. + */ +static inline bool may_copy_tree(const struct path *path) +{ + struct mount *mnt = real_mount(path->mnt); + const struct dentry_operations *d_op; + + if (check_mnt(mnt)) + return true; + + d_op = path->dentry->d_op; + if (d_op == &ns_dentry_operations) + return true; + + if (d_op == &pidfs_dentry_operations) + return true; + + if (!is_mounted(path->mnt)) + return false; + + return check_anonymous_mnt(mnt); +} + +static struct mount *__do_loopback(const struct path *old_path, + bool recurse, unsigned int copy_flags) +{ + struct mount *old = real_mount(old_path->mnt); + + if (IS_MNT_UNBINDABLE(old)) + return ERR_PTR(-EINVAL); + + if (!may_copy_tree(old_path)) + return ERR_PTR(-EINVAL); + + if (!recurse && __has_locked_children(old, old_path->dentry)) + return ERR_PTR(-EINVAL); + + if (recurse) + return copy_tree(old, old_path->dentry, copy_flags); + + return clone_mnt(old, old_path->dentry, copy_flags); +} + +/* + * do loopback mount. + */ +static int do_loopback(const struct path *path, const char *old_name, + int recurse) +{ + struct path old_path __free(path_put) = {}; + struct mount *mnt = NULL; + int err; + + if (!old_name || !*old_name) + return -EINVAL; + err = kern_path(old_name, LOOKUP_FOLLOW|LOOKUP_AUTOMOUNT, &old_path); + if (err) + return err; + + if (mnt_ns_loop(old_path.dentry)) + return -EINVAL; + + LOCK_MOUNT(mp, path); + if (IS_ERR(mp.parent)) + return PTR_ERR(mp.parent); + + if (!check_mnt(mp.parent)) + return -EINVAL; + + mnt = __do_loopback(&old_path, recurse, CL_COPY_MNT_NS_FILE); + if (IS_ERR(mnt)) + return PTR_ERR(mnt); + + err = graft_tree(mnt, &mp); + if (err) { + lock_mount_hash(); + umount_tree(mnt, UMOUNT_SYNC); + unlock_mount_hash(); + } + return err; +} + +static struct mnt_namespace *get_detached_copy(const struct path *path, unsigned int flags) +{ + struct mnt_namespace *ns, *mnt_ns = current->nsproxy->mnt_ns, *src_mnt_ns; + struct user_namespace *user_ns = mnt_ns->user_ns; + struct mount *mnt, *p; + + ns = alloc_mnt_ns(user_ns, true); + if (IS_ERR(ns)) + return ns; + + guard(namespace_excl)(); + + /* + * Record the sequence number of the source mount namespace. + * This needs to hold namespace_sem to ensure that the mount + * doesn't get attached. + */ + if (is_mounted(path->mnt)) { + src_mnt_ns = real_mount(path->mnt)->mnt_ns; + if (is_anon_ns(src_mnt_ns)) + ns->seq_origin = src_mnt_ns->seq_origin; + else + ns->seq_origin = src_mnt_ns->ns.ns_id; + } + + mnt = __do_loopback(path, (flags & AT_RECURSIVE), CL_COPY_MNT_NS_FILE); + if (IS_ERR(mnt)) { + emptied_ns = ns; + return ERR_CAST(mnt); + } + + for (p = mnt; p; p = next_mnt(p, mnt)) { + mnt_add_to_ns(ns, p); + ns->nr_mounts++; + } + ns->root = mnt; + return ns; +} + +static struct file *open_detached_copy(struct path *path, unsigned int flags) +{ + struct mnt_namespace *ns = get_detached_copy(path, flags); + struct file *file; + + if (IS_ERR(ns)) + return ERR_CAST(ns); + + mntput(path->mnt); + path->mnt = mntget(&ns->root->mnt); + file = dentry_open(path, O_PATH, current_cred()); + if (IS_ERR(file)) + dissolve_on_fput(path->mnt); + else + file->f_mode |= FMODE_NEED_UNMOUNT; + return file; +} + +enum mount_copy_flags_t { + MOUNT_COPY_RECURSIVE = (1 << 0), + MOUNT_COPY_NEW = (1 << 1), +}; + +static struct mnt_namespace *create_new_namespace(struct path *path, + enum mount_copy_flags_t flags) +{ + struct mnt_namespace *ns = current->nsproxy->mnt_ns; + struct user_namespace *user_ns = current_user_ns(); + struct mnt_namespace *new_ns; + struct mount *new_ns_root, *old_ns_root; + struct path to_path; + struct mount *mnt; + unsigned int copy_flags = 0; + bool locked = false, recurse = flags & MOUNT_COPY_RECURSIVE; + + if (unlikely(!d_can_lookup(path->dentry))) + return ERR_PTR(-ENOTDIR); + + if (user_ns != ns->user_ns) + copy_flags |= CL_SLAVE; + + new_ns = alloc_mnt_ns(user_ns, false); + if (IS_ERR(new_ns)) + return ERR_CAST(new_ns); + + old_ns_root = ns->root; + to_path.mnt = &old_ns_root->mnt; + to_path.dentry = old_ns_root->mnt.mnt_root; + + VFS_WARN_ON_ONCE(old_ns_root->mnt.mnt_sb->s_type != &nullfs_fs_type); + + LOCK_MOUNT_EXACT_COPY(mp, &to_path, copy_flags); + if (IS_ERR(mp.parent)) { + free_mnt_ns(new_ns); + return ERR_CAST(mp.parent); + } + new_ns_root = mp.parent; + + /* + * If the real rootfs had a locked mount on top of it somewhere + * in the stack, lock the new mount tree as well so it can't be + * exposed. + */ + mnt = old_ns_root; + while (mnt->overmount) { + mnt = mnt->overmount; + if (mnt->mnt.mnt_flags & MNT_LOCKED) + locked = true; + } + + /* + * We don't emulate unshare()ing a mount namespace. We stick to + * the restrictions of creating detached bind-mounts. It has a + * lot saner and simpler semantics. + */ + if (flags & MOUNT_COPY_NEW) + mnt = clone_mnt(real_mount(path->mnt), path->dentry, copy_flags); + else + mnt = __do_loopback(path, recurse, copy_flags); + scoped_guard(mount_writer) { + if (IS_ERR(mnt)) { + emptied_ns = new_ns; + umount_tree(new_ns_root, 0); + return ERR_CAST(mnt); + } + + if (locked) + mnt->mnt.mnt_flags |= MNT_LOCKED; + /* + * now mount the detached tree on top of the copy + * of the real rootfs we created. + */ + attach_mnt(mnt, new_ns_root, mp.mp); + if (user_ns != ns->user_ns) + lock_mnt_tree(new_ns_root); + } + + for (mnt = new_ns_root; mnt; mnt = next_mnt(mnt, new_ns_root)) { + mnt_add_to_ns(new_ns, mnt); + new_ns->nr_mounts++; + } + + new_ns->root = new_ns_root; + ns_tree_add_raw(new_ns); + return new_ns; +} + +static struct file *open_new_namespace(struct path *path, + enum mount_copy_flags_t flags) +{ + struct mnt_namespace *new_ns; + + new_ns = create_new_namespace(path, flags); + if (IS_ERR(new_ns)) + return ERR_CAST(new_ns); + return open_namespace_file(to_ns_common(new_ns)); +} + +static struct file *vfs_open_tree(int dfd, const char __user *filename, unsigned int flags) +{ + int ret; + struct path path __free(path_put) = {}; + int lookup_flags = LOOKUP_AUTOMOUNT | LOOKUP_FOLLOW; + + BUILD_BUG_ON(OPEN_TREE_CLOEXEC != O_CLOEXEC); + + if (flags & ~(AT_EMPTY_PATH | AT_NO_AUTOMOUNT | AT_RECURSIVE | + AT_SYMLINK_NOFOLLOW | OPEN_TREE_CLONE | + OPEN_TREE_CLOEXEC | OPEN_TREE_NAMESPACE)) + return ERR_PTR(-EINVAL); + + if ((flags & (AT_RECURSIVE | OPEN_TREE_CLONE | OPEN_TREE_NAMESPACE)) == + AT_RECURSIVE) + return ERR_PTR(-EINVAL); + + if (hweight32(flags & (OPEN_TREE_CLONE | OPEN_TREE_NAMESPACE)) > 1) + return ERR_PTR(-EINVAL); + + if (flags & AT_NO_AUTOMOUNT) + lookup_flags &= ~LOOKUP_AUTOMOUNT; + if (flags & AT_SYMLINK_NOFOLLOW) + lookup_flags &= ~LOOKUP_FOLLOW; + + /* + * If we create a new mount namespace with the cloned mount tree we + * just care about being privileged over our current user namespace. + * The new mount namespace will be owned by it. + */ + if ((flags & OPEN_TREE_NAMESPACE) && + !ns_capable(current_user_ns(), CAP_SYS_ADMIN)) + return ERR_PTR(-EPERM); + + if ((flags & OPEN_TREE_CLONE) && !may_mount()) + return ERR_PTR(-EPERM); + + CLASS(filename_uflags, name)(filename, flags); + ret = filename_lookup(dfd, name, lookup_flags, &path, NULL); + if (unlikely(ret)) + return ERR_PTR(ret); + + if (flags & OPEN_TREE_NAMESPACE) + return open_new_namespace(&path, (flags & AT_RECURSIVE) ? MOUNT_COPY_RECURSIVE : 0); + + if (flags & OPEN_TREE_CLONE) + return open_detached_copy(&path, flags); + + return dentry_open(&path, O_PATH, current_cred()); +} + +SYSCALL_DEFINE3(open_tree, int, dfd, const char __user *, filename, unsigned, flags) +{ + return FD_ADD(flags, vfs_open_tree(dfd, filename, flags)); +} + +/* + * Don't allow locked mount flags to be cleared. + * + * No locks need to be held here while testing the various MNT_LOCK + * flags because those flags can never be cleared once they are set. + */ +static bool can_change_locked_flags(struct mount *mnt, unsigned int mnt_flags) +{ + unsigned int fl = mnt->mnt.mnt_flags; + + if ((fl & MNT_LOCK_READONLY) && + !(mnt_flags & MNT_READONLY)) + return false; + + if ((fl & MNT_LOCK_NODEV) && + !(mnt_flags & MNT_NODEV)) + return false; + + if ((fl & MNT_LOCK_NOSUID) && + !(mnt_flags & MNT_NOSUID)) + return false; + + if ((fl & MNT_LOCK_NOEXEC) && + !(mnt_flags & MNT_NOEXEC)) + return false; + + if ((fl & MNT_LOCK_ATIME) && + ((fl & MNT_ATIME_MASK) != (mnt_flags & MNT_ATIME_MASK))) + return false; + + return true; +} + +static int change_mount_ro_state(struct mount *mnt, unsigned int mnt_flags) +{ + bool readonly_request = (mnt_flags & MNT_READONLY); + + if (readonly_request == __mnt_is_readonly(&mnt->mnt)) + return 0; + + if (readonly_request) + return mnt_make_readonly(mnt); + + mnt->mnt.mnt_flags &= ~MNT_READONLY; + return 0; +} + +static void set_mount_attributes(struct mount *mnt, unsigned int mnt_flags) +{ + mnt_flags |= mnt->mnt.mnt_flags & ~MNT_USER_SETTABLE_MASK; + mnt->mnt.mnt_flags = mnt_flags; + touch_mnt_namespace(mnt->mnt_ns); +} + +static void mnt_warn_timestamp_expiry(const struct path *mountpoint, + struct vfsmount *mnt) +{ + struct super_block *sb = mnt->mnt_sb; + + if (!__mnt_is_readonly(mnt) && + (!(sb->s_iflags & SB_I_TS_EXPIRY_WARNED)) && + (ktime_get_real_seconds() + TIME_UPTIME_SEC_MAX > sb->s_time_max)) { + char *buf, *mntpath; + + buf = __getname(); + if (buf) + mntpath = d_path(mountpoint, buf, PATH_MAX); + else + mntpath = ERR_PTR(-ENOMEM); + if (IS_ERR(mntpath)) + mntpath = "(unknown)"; + + pr_warn("%s filesystem being %s at %s supports timestamps until %ptTd (0x%llx)\n", + sb->s_type->name, + is_mounted(mnt) ? "remounted" : "mounted", + mntpath, &sb->s_time_max, + (unsigned long long)sb->s_time_max); + + sb->s_iflags |= SB_I_TS_EXPIRY_WARNED; + __putname(buf); + } +} + +/* + * Handle reconfiguration of the mountpoint only without alteration of the + * superblock it refers to. This is triggered by specifying MS_REMOUNT|MS_BIND + * to mount(2). + */ +static int do_reconfigure_mnt(const struct path *path, unsigned int mnt_flags) +{ + struct super_block *sb = path->mnt->mnt_sb; + struct mount *mnt = real_mount(path->mnt); + int ret; + + if (!check_mnt(mnt)) + return -EINVAL; + + if (!path_mounted(path)) + return -EINVAL; + + if (!can_change_locked_flags(mnt, mnt_flags)) + return -EPERM; + + /* + * We're only checking whether the superblock is read-only not + * changing it, so only take down_read(&sb->s_umount). + */ + down_read(&sb->s_umount); + lock_mount_hash(); + ret = change_mount_ro_state(mnt, mnt_flags); + if (ret == 0) + set_mount_attributes(mnt, mnt_flags); + unlock_mount_hash(); + up_read(&sb->s_umount); + + mnt_warn_timestamp_expiry(path, &mnt->mnt); + + return ret; +} + +/* + * change filesystem flags. dir should be a physical root of filesystem. + * If you've mounted a non-root directory somewhere and want to do remount + * on it - tough luck. + */ +static int do_remount(const struct path *path, int sb_flags, + int mnt_flags, void *data) +{ + int err; + struct super_block *sb = path->mnt->mnt_sb; + struct mount *mnt = real_mount(path->mnt); + struct fs_context *fc; + + if (!check_mnt(mnt)) + return -EINVAL; + + if (!path_mounted(path)) + return -EINVAL; + + if (!can_change_locked_flags(mnt, mnt_flags)) + return -EPERM; + + fc = fs_context_for_reconfigure(path->dentry, sb_flags, MS_RMT_MASK); + if (IS_ERR(fc)) + return PTR_ERR(fc); + + /* + * Indicate to the filesystem that the remount request is coming + * from the legacy mount system call. + */ + fc->oldapi = true; + + err = parse_monolithic_mount_data(fc, data); + if (!err) { + down_write(&sb->s_umount); + err = -EPERM; + if (ns_capable(sb->s_user_ns, CAP_SYS_ADMIN)) { + err = reconfigure_super(fc); + if (!err) { + lock_mount_hash(); + set_mount_attributes(mnt, mnt_flags); + unlock_mount_hash(); + } + } + up_write(&sb->s_umount); + } + + mnt_warn_timestamp_expiry(path, &mnt->mnt); + + put_fs_context(fc); + return err; +} + +static inline int tree_contains_unbindable(struct mount *mnt) +{ + struct mount *p; + for (p = mnt; p; p = next_mnt(p, mnt)) { + if (IS_MNT_UNBINDABLE(p)) + return 1; + } + return 0; +} + +static int do_set_group(const struct path *from_path, const struct path *to_path) +{ + struct mount *from = real_mount(from_path->mnt); + struct mount *to = real_mount(to_path->mnt); + int err; + + guard(namespace_excl)(); + + err = may_change_propagation(from); + if (err) + return err; + err = may_change_propagation(to); + if (err) + return err; + + /* To and From paths should be mount roots */ + if (!path_mounted(from_path)) + return -EINVAL; + if (!path_mounted(to_path)) + return -EINVAL; + + /* Setting sharing groups is only allowed across same superblock */ + if (from->mnt.mnt_sb != to->mnt.mnt_sb) + return -EINVAL; + + /* From mount root should be wider than To mount root */ + if (!is_subdir(to->mnt.mnt_root, from->mnt.mnt_root)) + return -EINVAL; + + /* From mount should not have locked children in place of To's root */ + if (__has_locked_children(from, to->mnt.mnt_root)) + return -EINVAL; + + /* Setting sharing groups is only allowed on private mounts */ + if (IS_MNT_SHARED(to) || IS_MNT_SLAVE(to)) + return -EINVAL; + + /* From should not be private */ + if (!IS_MNT_SHARED(from) && !IS_MNT_SLAVE(from)) + return -EINVAL; + + if (IS_MNT_SLAVE(from)) { + hlist_add_behind(&to->mnt_slave, &from->mnt_slave); + to->mnt_master = from->mnt_master; + } + + if (IS_MNT_SHARED(from)) { + to->mnt_group_id = from->mnt_group_id; + list_add(&to->mnt_share, &from->mnt_share); + set_mnt_shared(to); + } + + guard(mount_locked_reader)(); + touch_mnt_namespace(to->mnt_ns); + + return 0; +} + +/** + * path_overmounted - check if path is overmounted + * @path: path to check + * + * Check if path is overmounted, i.e., if there's a mount on top of + * @path->mnt with @path->dentry as mountpoint. + * + * Context: namespace_sem must be held at least shared. + * MUST NOT be called under lock_mount_hash() (there one should just + * call __lookup_mnt() and check if it returns NULL). + * Return: If path is overmounted true is returned, false if not. + */ +static inline bool path_overmounted(const struct path *path) +{ + unsigned seq = read_seqbegin(&mount_lock); + bool no_child; + + rcu_read_lock(); + no_child = !__lookup_mnt(path->mnt, path->dentry); + rcu_read_unlock(); + if (need_seqretry(&mount_lock, seq)) { + read_seqlock_excl(&mount_lock); + no_child = !__lookup_mnt(path->mnt, path->dentry); + read_sequnlock_excl(&mount_lock); + } + return unlikely(!no_child); +} + +/* + * Check if there is a possibly empty chain of descent from p1 to p2. + * Locks: namespace_sem (shared) or mount_lock (read_seqlock_excl). + */ +static bool mount_is_ancestor(const struct mount *p1, const struct mount *p2) +{ + while (p2 != p1 && mnt_has_parent(p2)) + p2 = p2->mnt_parent; + return p2 == p1; +} + +/** + * can_move_mount_beneath - check that we can mount beneath the top mount + * @mnt_from: mount we are trying to move + * @mnt_to: mount under which to mount + * @mp: mountpoint of @mnt_to + * + * - Make sure that the caller can unmount the topmost mount ensuring + * that the caller could reveal the underlying mountpoint. + * - Ensure that nothing has been mounted on top of @mnt_from before we + * grabbed @namespace_sem to avoid creating pointless shadow mounts. + * - Prevent mounting beneath a mount if the propagation relationship + * between the source mount, parent mount, and top mount would lead to + * nonsensical mount trees. + * + * Context: This function expects namespace_lock() to be held. + * Return: On success 0, and on error a negative error code is returned. + */ +static int can_move_mount_beneath(const struct mount *mnt_from, + const struct mount *mnt_to, + struct pinned_mountpoint *mp) +{ + struct mount *parent_mnt_to = mnt_to->mnt_parent; + + /* Avoid creating shadow mounts during mount propagation. */ + if (mnt_from->overmount) + return -EINVAL; + + if (mount_is_ancestor(mnt_to, mnt_from)) + return -EINVAL; + + /* + * If the parent mount propagates to the child mount this would + * mean mounting @mnt_from on @mnt_to->mnt_parent and then + * propagating a copy @c of @mnt_from on top of @mnt_to. This + * defeats the whole purpose of mounting beneath another mount. + */ + if (propagation_would_overmount(parent_mnt_to, mnt_to, mp->mp)) + return -EINVAL; + + /* + * If @mnt_to->mnt_parent propagates to @mnt_from this would + * mean propagating a copy @c of @mnt_from on top of @mnt_from. + * Afterwards @mnt_from would be mounted on top of + * @mnt_to->mnt_parent and @mnt_to would be unmounted from + * @mnt->mnt_parent and remounted on @mnt_from. But since @c is + * already mounted on @mnt_from, @mnt_to would ultimately be + * remounted on top of @c. Afterwards, @mnt_from would be + * covered by a copy @c of @mnt_from and @c would be covered by + * @mnt_from itself. This defeats the whole purpose of mounting + * @mnt_from beneath @mnt_to. + */ + if (check_mnt(mnt_from) && + propagation_would_overmount(parent_mnt_to, mnt_from, mp->mp)) + return -EINVAL; + + return 0; +} + +/* may_use_mount() - check if a mount tree can be used + * @mnt: vfsmount to be used + * + * This helper checks if the caller may use the mount tree starting + * from @path->mnt. The caller may use the mount tree under the + * following circumstances: + * + * (1) The caller is located in the mount namespace of the mount tree. + * This also implies that the mount does not belong to an anonymous + * mount namespace. + * (2) The caller is trying to use a mount tree that belongs to an + * anonymous mount namespace. + * + * For that to be safe, this helper enforces that the origin mount + * namespace the anonymous mount namespace was created from is the + * same as the caller's mount namespace by comparing the sequence + * numbers. + * + * The ownership of a non-anonymous mount namespace such as the + * caller's cannot change. + * => We know that the caller's mount namespace is stable. + * + * If the origin sequence number of the anonymous mount namespace is + * the same as the sequence number of the caller's mount namespace. + * => The owning namespaces are the same. + * + * ==> The earlier capability check on the owning namespace of the + * caller's mount namespace ensures that the caller has the + * ability to use the mount tree. + * + * Returns true if the mount tree can be used, false otherwise. + */ +static inline bool may_use_mount(struct mount *mnt) +{ + if (check_mnt(mnt)) + return true; + + /* + * Make sure that noone unmounted the target path or somehow + * managed to get their hands on something purely kernel + * internal. + */ + if (!is_mounted(&mnt->mnt)) + return false; + + return check_anonymous_mnt(mnt); +} + +static int do_move_mount(const struct path *old_path, + const struct path *new_path, + enum mnt_tree_flags_t flags) +{ + struct mount *old = real_mount(old_path->mnt); + int err; + bool beneath = flags & MNT_TREE_BENEATH; + + if (!path_mounted(old_path)) + return -EINVAL; + + if (d_is_dir(new_path->dentry) != d_is_dir(old_path->dentry)) + return -EINVAL; + + LOCK_MOUNT_MAYBE_BENEATH(mp, new_path, beneath); + if (IS_ERR(mp.parent)) + return PTR_ERR(mp.parent); + + if (check_mnt(old)) { + /* if the source is in our namespace... */ + /* ... it should be detachable from parent */ + if (!mnt_has_parent(old) || IS_MNT_LOCKED(old)) + return -EINVAL; + /* ... which should not be shared */ + if (IS_MNT_SHARED(old->mnt_parent)) + return -EINVAL; + /* ... and the target should be in our namespace */ + if (!check_mnt(mp.parent)) + return -EINVAL; + } else { + /* + * otherwise the source must be the root of some anon namespace. + */ + if (!anon_ns_root(old)) + return -EINVAL; + /* + * Bail out early if the target is within the same namespace - + * subsequent checks would've rejected that, but they lose + * some corner cases if we check it early. + */ + if (old->mnt_ns == mp.parent->mnt_ns) + return -EINVAL; + /* + * Target should be either in our namespace or in an acceptable + * anon namespace, sensu check_anonymous_mnt(). + */ + if (!may_use_mount(mp.parent)) + return -EINVAL; + } + + if (beneath) { + struct mount *over = real_mount(new_path->mnt); + + if (mp.parent != over->mnt_parent) + over = mp.parent->overmount; + err = can_move_mount_beneath(old, over, &mp); + if (err) + return err; + } + + /* + * Don't move a mount tree containing unbindable mounts to a destination + * mount which is shared. + */ + if (IS_MNT_SHARED(mp.parent) && tree_contains_unbindable(old)) + return -EINVAL; + if (!check_for_nsfs_mounts(old)) + return -ELOOP; + if (mount_is_ancestor(old, mp.parent)) + return -ELOOP; + + return attach_recursive_mnt(old, &mp); +} + +static int do_move_mount_old(const struct path *path, const char *old_name) +{ + struct path old_path __free(path_put) = {}; + int err; + + if (!old_name || !*old_name) + return -EINVAL; + + err = kern_path(old_name, LOOKUP_FOLLOW, &old_path); + if (err) + return err; + + return do_move_mount(&old_path, path, 0); +} + +/* + * add a mount into a namespace's mount tree + */ +static int do_add_mount(struct mount *newmnt, const struct pinned_mountpoint *mp, + int mnt_flags) +{ + struct mount *parent = mp->parent; + + if (IS_ERR(parent)) + return PTR_ERR(parent); + + mnt_flags &= ~MNT_INTERNAL_FLAGS; + + if (unlikely(!check_mnt(parent))) { + /* that's acceptable only for automounts done in private ns */ + if (!(mnt_flags & MNT_SHRINKABLE)) + return -EINVAL; + /* ... and for those we'd better have mountpoint still alive */ + if (!parent->mnt_ns) + return -EINVAL; + } + + /* Refuse the same filesystem on the same mount point */ + if (parent->mnt.mnt_sb == newmnt->mnt.mnt_sb && + parent->mnt.mnt_root == mp->mp->m_dentry) + return -EBUSY; + + if (d_is_symlink(newmnt->mnt.mnt_root)) + return -EINVAL; + + newmnt->mnt.mnt_flags = mnt_flags; + return graft_tree(newmnt, mp); +} + +static bool mount_too_revealing(const struct super_block *sb, int *new_mnt_flags); + +/* + * Create a new mount using a superblock configuration and request it + * be added to the namespace tree. + */ +static int do_new_mount_fc(struct fs_context *fc, const struct path *mountpoint, + unsigned int mnt_flags) +{ + struct super_block *sb; + struct vfsmount *mnt __free(mntput) = fc_mount(fc); + int error; + + if (IS_ERR(mnt)) + return PTR_ERR(mnt); + + sb = fc->root->d_sb; + error = security_sb_kern_mount(sb); + if (unlikely(error)) + return error; + + if (unlikely(mount_too_revealing(sb, &mnt_flags))) { + errorfcp(fc, "VFS", "Mount too revealing"); + return -EPERM; + } + + mnt_warn_timestamp_expiry(mountpoint, mnt); + + LOCK_MOUNT(mp, mountpoint); + error = do_add_mount(real_mount(mnt), &mp, mnt_flags); + if (!error) + retain_and_null_ptr(mnt); // consumed on success + return error; +} + +/* + * create a new mount for userspace and request it to be added into the + * namespace's tree + */ +static int do_new_mount(const struct path *path, const char *fstype, + int sb_flags, int mnt_flags, + const char *name, void *data) +{ + struct file_system_type *type; + struct fs_context *fc; + const char *subtype = NULL; + int err = 0; + + if (!fstype) + return -EINVAL; + + type = get_fs_type(fstype); + if (!type) + return -ENODEV; + + if (type->fs_flags & FS_HAS_SUBTYPE) { + subtype = strchr(fstype, '.'); + if (subtype) { + subtype++; + if (!*subtype) { + put_filesystem(type); + return -EINVAL; + } + } + } + + fc = fs_context_for_mount(type, sb_flags); + put_filesystem(type); + if (IS_ERR(fc)) + return PTR_ERR(fc); + + /* + * Indicate to the filesystem that the mount request is coming + * from the legacy mount system call. + */ + fc->oldapi = true; + + if (subtype) + err = vfs_parse_fs_string(fc, "subtype", subtype); + if (!err && name) + err = vfs_parse_fs_string(fc, "source", name); + if (!err) + err = parse_monolithic_mount_data(fc, data); + if (!err && !mount_capable(fc)) + err = -EPERM; + if (!err) + err = do_new_mount_fc(fc, path, mnt_flags); + + put_fs_context(fc); + return err; +} + +static void lock_mount_exact(const struct path *path, + struct pinned_mountpoint *mp, bool copy_mount, + unsigned int copy_flags) +{ + struct dentry *dentry = path->dentry; + int err; + + /* Assert that inode_lock() locked the correct inode. */ + VFS_WARN_ON_ONCE(copy_mount && !path_mounted(path)); + + inode_lock(dentry->d_inode); + namespace_lock(); + if (unlikely(cant_mount(dentry))) + err = -ENOENT; + else if (!copy_mount && path_overmounted(path)) + err = -EBUSY; + else + err = get_mountpoint(dentry, mp); + if (unlikely(err)) { + namespace_unlock(); + inode_unlock(dentry->d_inode); + mp->parent = ERR_PTR(err); + return; + } + + if (copy_mount) + mp->parent = clone_mnt(real_mount(path->mnt), dentry, copy_flags); + else + mp->parent = real_mount(path->mnt); + if (unlikely(IS_ERR(mp->parent))) + __unlock_mount(mp); +} + +int finish_automount(struct vfsmount *__m, const struct path *path) +{ + struct vfsmount *m __free(mntput) = __m; + struct mount *mnt; + int err; + + if (!m) + return 0; + if (IS_ERR(m)) + return PTR_ERR(m); + + mnt = real_mount(m); + + if (m->mnt_root == path->dentry) + return -ELOOP; + + /* + * we don't want to use LOCK_MOUNT() - in this case finding something + * that overmounts our mountpoint to be means "quitely drop what we've + * got", not "try to mount it on top". + */ + LOCK_MOUNT_EXACT(mp, path); + if (mp.parent == ERR_PTR(-EBUSY)) + return 0; + + err = do_add_mount(mnt, &mp, path->mnt->mnt_flags | MNT_SHRINKABLE); + if (likely(!err)) + retain_and_null_ptr(m); + return err; +} + +/** + * mnt_set_expiry - Put a mount on an expiration list + * @mnt: The mount to list. + * @expiry_list: The list to add the mount to. + */ +void mnt_set_expiry(struct vfsmount *mnt, struct list_head *expiry_list) +{ + guard(mount_locked_reader)(); + list_add_tail(&real_mount(mnt)->mnt_expire, expiry_list); +} +EXPORT_SYMBOL(mnt_set_expiry); + +/* + * process a list of expirable mountpoints with the intent of discarding any + * mountpoints that aren't in use and haven't been touched since last we came + * here + */ +void mark_mounts_for_expiry(struct list_head *mounts) +{ + struct mount *mnt, *next; + LIST_HEAD(graveyard); + + if (list_empty(mounts)) + return; + + guard(namespace_excl)(); + guard(mount_writer)(); + + /* extract from the expiration list every vfsmount that matches the + * following criteria: + * - already mounted + * - only referenced by its parent vfsmount + * - still marked for expiry (marked on the last call here; marks are + * cleared by mntput()) + */ + list_for_each_entry_safe(mnt, next, mounts, mnt_expire) { + if (!is_mounted(&mnt->mnt)) + continue; + if (!xchg(&mnt->mnt_expiry_mark, 1) || + propagate_mount_busy(mnt, 1)) + continue; + list_move(&mnt->mnt_expire, &graveyard); + } + while (!list_empty(&graveyard)) { + mnt = list_first_entry(&graveyard, struct mount, mnt_expire); + touch_mnt_namespace(mnt->mnt_ns); + umount_tree(mnt, UMOUNT_PROPAGATE|UMOUNT_SYNC); + } +} + +EXPORT_SYMBOL_GPL(mark_mounts_for_expiry); + +/* + * Ripoff of 'select_parent()' + * + * search the list of submounts for a given mountpoint, and move any + * shrinkable submounts to the 'graveyard' list. + */ +static int select_submounts(struct mount *parent, struct list_head *graveyard) +{ + struct mount *this_parent = parent; + struct list_head *next; + int found = 0; + +repeat: + next = this_parent->mnt_mounts.next; +resume: + while (next != &this_parent->mnt_mounts) { + struct list_head *tmp = next; + struct mount *mnt = list_entry(tmp, struct mount, mnt_child); + + next = tmp->next; + if (!(mnt->mnt.mnt_flags & MNT_SHRINKABLE)) + continue; + /* + * Descend a level if the d_mounts list is non-empty. + */ + if (!list_empty(&mnt->mnt_mounts)) { + this_parent = mnt; + goto repeat; + } + + if (!propagate_mount_busy(mnt, 1)) { + list_move_tail(&mnt->mnt_expire, graveyard); + found++; + } + } + /* + * All done at this level ... ascend and resume the search + */ + if (this_parent != parent) { + next = this_parent->mnt_child.next; + this_parent = this_parent->mnt_parent; + goto resume; + } + return found; +} + +/* + * process a list of expirable mountpoints with the intent of discarding any + * submounts of a specific parent mountpoint + * + * mount_lock must be held for write + */ +static void shrink_submounts(struct mount *mnt) +{ + LIST_HEAD(graveyard); + struct mount *m; + + /* extract submounts of 'mountpoint' from the expiration list */ + while (select_submounts(mnt, &graveyard)) { + while (!list_empty(&graveyard)) { + m = list_first_entry(&graveyard, struct mount, + mnt_expire); + touch_mnt_namespace(m->mnt_ns); + umount_tree(m, UMOUNT_PROPAGATE|UMOUNT_SYNC); + } + } +} + +static void *copy_mount_options(const void __user * data) +{ + char *copy; + unsigned left, offset; + + if (!data) + return NULL; + + copy = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!copy) + return ERR_PTR(-ENOMEM); + + left = copy_from_user(copy, data, PAGE_SIZE); + + /* + * Not all architectures have an exact copy_from_user(). Resort to + * byte at a time. + */ + offset = PAGE_SIZE - left; + while (left) { + char c; + if (get_user(c, (const char __user *)data + offset)) + break; + copy[offset] = c; + left--; + offset++; + } + + if (left == PAGE_SIZE) { + kfree(copy); + return ERR_PTR(-EFAULT); + } + + return copy; +} + +static char *copy_mount_string(const void __user *data) +{ + return data ? strndup_user(data, PATH_MAX) : NULL; +} + +/* + * Flags is a 32-bit value that allows up to 31 non-fs dependent flags to + * be given to the mount() call (ie: read-only, no-dev, no-suid etc). + * + * data is a (void *) that can point to any structure up to + * PAGE_SIZE-1 bytes, which can contain arbitrary fs-dependent + * information (or be NULL). + * + * Pre-0.97 versions of mount() didn't have a flags word. + * When the flags word was introduced its top half was required + * to have the magic value 0xC0ED, and this remained so until 2.4.0-test9. + * Therefore, if this magic number is present, it carries no information + * and must be discarded. + */ +int path_mount(const char *dev_name, const struct path *path, + const char *type_page, unsigned long flags, void *data_page) +{ + unsigned int mnt_flags = 0, sb_flags; + int ret; + + /* Discard magic */ + if ((flags & MS_MGC_MSK) == MS_MGC_VAL) + flags &= ~MS_MGC_MSK; + + /* Basic sanity checks */ + if (data_page) + ((char *)data_page)[PAGE_SIZE - 1] = 0; + + if (flags & MS_NOUSER) + return -EINVAL; + + ret = security_sb_mount(dev_name, path, type_page, flags, data_page); + if (ret) + return ret; + if (!may_mount()) + return -EPERM; + if (flags & SB_MANDLOCK) + warn_mandlock(); + + /* Default to relatime unless overriden */ + if (!(flags & MS_NOATIME)) + mnt_flags |= MNT_RELATIME; + + /* Separate the per-mountpoint flags */ + if (flags & MS_NOSUID) + mnt_flags |= MNT_NOSUID; + if (flags & MS_NODEV) + mnt_flags |= MNT_NODEV; + if (flags & MS_NOEXEC) + mnt_flags |= MNT_NOEXEC; + if (flags & MS_NOATIME) + mnt_flags |= MNT_NOATIME; + if (flags & MS_NODIRATIME) + mnt_flags |= MNT_NODIRATIME; + if (flags & MS_STRICTATIME) + mnt_flags &= ~(MNT_RELATIME | MNT_NOATIME); + if (flags & MS_RDONLY) + mnt_flags |= MNT_READONLY; + if (flags & MS_NOSYMFOLLOW) + mnt_flags |= MNT_NOSYMFOLLOW; + + /* The default atime for remount is preservation */ + if ((flags & MS_REMOUNT) && + ((flags & (MS_NOATIME | MS_NODIRATIME | MS_RELATIME | + MS_STRICTATIME)) == 0)) { + mnt_flags &= ~MNT_ATIME_MASK; + mnt_flags |= path->mnt->mnt_flags & MNT_ATIME_MASK; + } + + sb_flags = flags & (SB_RDONLY | + SB_SYNCHRONOUS | + SB_MANDLOCK | + SB_DIRSYNC | + SB_SILENT | + SB_POSIXACL | + SB_LAZYTIME | + SB_I_VERSION); + + if ((flags & (MS_REMOUNT | MS_BIND)) == (MS_REMOUNT | MS_BIND)) + return do_reconfigure_mnt(path, mnt_flags); + if (flags & MS_REMOUNT) + return do_remount(path, sb_flags, mnt_flags, data_page); + if (flags & MS_BIND) + return do_loopback(path, dev_name, flags & MS_REC); + if (flags & (MS_SHARED | MS_PRIVATE | MS_SLAVE | MS_UNBINDABLE)) + return do_change_type(path, flags); + if (flags & MS_MOVE) + return do_move_mount_old(path, dev_name); + + return do_new_mount(path, type_page, sb_flags, mnt_flags, dev_name, + data_page); +} + +int do_mount(const char *dev_name, const char __user *dir_name, + const char *type_page, unsigned long flags, void *data_page) +{ + struct path path __free(path_put) = {}; + int ret; + + ret = user_path_at(AT_FDCWD, dir_name, LOOKUP_FOLLOW, &path); + if (ret) + return ret; + return path_mount(dev_name, &path, type_page, flags, data_page); +} + +static struct ucounts *inc_mnt_namespaces(struct user_namespace *ns) +{ + return inc_ucount(ns, current_euid(), UCOUNT_MNT_NAMESPACES); +} + +static void dec_mnt_namespaces(struct ucounts *ucounts) +{ + dec_ucount(ucounts, UCOUNT_MNT_NAMESPACES); +} + +static void free_mnt_ns(struct mnt_namespace *ns) +{ + if (!is_anon_ns(ns)) + ns_common_free(ns); + dec_mnt_namespaces(ns->ucounts); + mnt_ns_tree_remove(ns); +} + +static struct mnt_namespace *alloc_mnt_ns(struct user_namespace *user_ns, bool anon) +{ + struct mnt_namespace *new_ns; + struct ucounts *ucounts; + int ret; + + ucounts = inc_mnt_namespaces(user_ns); + if (!ucounts) + return ERR_PTR(-ENOSPC); + + new_ns = kzalloc_obj(struct mnt_namespace, GFP_KERNEL_ACCOUNT); + if (!new_ns) { + dec_mnt_namespaces(ucounts); + return ERR_PTR(-ENOMEM); + } + + if (anon) + ret = ns_common_init_inum(new_ns, MNT_NS_ANON_INO); + else + ret = ns_common_init(new_ns); + if (ret) { + kfree(new_ns); + dec_mnt_namespaces(ucounts); + return ERR_PTR(ret); + } + ns_tree_gen_id(new_ns); + + new_ns->is_anon = anon; + refcount_set(&new_ns->passive, 1); + new_ns->mounts = RB_ROOT; + init_waitqueue_head(&new_ns->poll); + new_ns->user_ns = get_user_ns(user_ns); + new_ns->ucounts = ucounts; + return new_ns; +} + +__latent_entropy +struct mnt_namespace *copy_mnt_ns(u64 flags, struct mnt_namespace *ns, + struct user_namespace *user_ns, struct fs_struct *new_fs) +{ + struct mnt_namespace *new_ns; + struct path old_root __free(path_put) = {}; + struct path old_pwd __free(path_put) = {}; + struct mount *p, *q; + struct mount *old; + struct mount *new; + int copy_flags; + + BUG_ON(!ns); + + if (likely(!(flags & CLONE_NEWNS))) { + get_mnt_ns(ns); + return ns; + } + + old = ns->root; + + new_ns = alloc_mnt_ns(user_ns, false); + if (IS_ERR(new_ns)) + return new_ns; + + guard(namespace_excl)(); + + if (flags & CLONE_EMPTY_MNTNS) + copy_flags = 0; + else + copy_flags = CL_COPY_UNBINDABLE | CL_EXPIRE; + if (user_ns != ns->user_ns) + copy_flags |= CL_SLAVE; + + if (flags & CLONE_EMPTY_MNTNS) + new = clone_mnt(old, old->mnt.mnt_root, copy_flags); + else + new = copy_tree(old, old->mnt.mnt_root, copy_flags); + if (IS_ERR(new)) { + emptied_ns = new_ns; + return ERR_CAST(new); + } + if (user_ns != ns->user_ns) { + guard(mount_writer)(); + lock_mnt_tree(new); + } + new_ns->root = new; + + if (flags & CLONE_EMPTY_MNTNS) { + /* + * Empty mount namespace: only the root mount exists. + * Reset root and pwd to the cloned mount's root dentry. + */ + if (new_fs) { + old_root = new_fs->root; + old_pwd = new_fs->pwd; + + new_fs->root.mnt = mntget(&new->mnt); + new_fs->root.dentry = dget(new->mnt.mnt_root); + + new_fs->pwd.mnt = mntget(&new->mnt); + new_fs->pwd.dentry = dget(new->mnt.mnt_root); + } + mnt_add_to_ns(new_ns, new); + new_ns->nr_mounts++; + } else { + /* + * Full copy: walk old and new trees in parallel, switching + * the tsk->fs->* elements and marking new vfsmounts as + * belonging to new namespace. We have already acquired a + * private fs_struct, so tsk->fs->lock is not needed. + */ + p = old; + q = new; + while (p) { + mnt_add_to_ns(new_ns, q); + new_ns->nr_mounts++; + if (new_fs) { + if (&p->mnt == new_fs->root.mnt) { + old_root.mnt = new_fs->root.mnt; + new_fs->root.mnt = mntget(&q->mnt); + } + if (&p->mnt == new_fs->pwd.mnt) { + old_pwd.mnt = new_fs->pwd.mnt; + new_fs->pwd.mnt = mntget(&q->mnt); + } + } + p = next_mnt(p, old); + q = next_mnt(q, new); + if (!q) + break; + // an mntns binding we'd skipped? + while (p->mnt.mnt_root != q->mnt.mnt_root) + p = next_mnt(skip_mnt_tree(p), old); + } + } + ns_tree_add_raw(new_ns); + return new_ns; +} + +struct dentry *mount_subtree(struct vfsmount *m, const char *name) +{ + struct mount *mnt = real_mount(m); + struct mnt_namespace *ns; + struct super_block *s; + struct path path; + int err; + + ns = alloc_mnt_ns(&init_user_ns, true); + if (IS_ERR(ns)) { + mntput(m); + return ERR_CAST(ns); + } + ns->root = mnt; + ns->nr_mounts++; + mnt_add_to_ns(ns, mnt); + + err = vfs_path_lookup(m->mnt_root, m, + name, LOOKUP_FOLLOW|LOOKUP_AUTOMOUNT, &path); + + put_mnt_ns(ns); + + if (err) + return ERR_PTR(err); + + /* trade a vfsmount reference for active sb one */ + s = path.mnt->mnt_sb; + atomic_inc(&s->s_active); + mntput(path.mnt); + /* lock the sucker */ + down_write(&s->s_umount); + /* ... and return the root of (sub)tree on it */ + return path.dentry; +} +EXPORT_SYMBOL(mount_subtree); + +SYSCALL_DEFINE5(mount, char __user *, dev_name, char __user *, dir_name, + char __user *, type, unsigned long, flags, void __user *, data) +{ + int ret; + char *kernel_type; + char *kernel_dev; + void *options; + + kernel_type = copy_mount_string(type); + ret = PTR_ERR(kernel_type); + if (IS_ERR(kernel_type)) + goto out_type; + + kernel_dev = copy_mount_string(dev_name); + ret = PTR_ERR(kernel_dev); + if (IS_ERR(kernel_dev)) + goto out_dev; + + options = copy_mount_options(data); + ret = PTR_ERR(options); + if (IS_ERR(options)) + goto out_data; + + ret = do_mount(kernel_dev, dir_name, kernel_type, flags, options); + + kfree(options); +out_data: + kfree(kernel_dev); +out_dev: + kfree(kernel_type); +out_type: + return ret; +} + +#define FSMOUNT_VALID_FLAGS \ + (MOUNT_ATTR_RDONLY | MOUNT_ATTR_NOSUID | MOUNT_ATTR_NODEV | \ + MOUNT_ATTR_NOEXEC | MOUNT_ATTR__ATIME | MOUNT_ATTR_NODIRATIME | \ + MOUNT_ATTR_NOSYMFOLLOW) + +#define MOUNT_SETATTR_VALID_FLAGS (FSMOUNT_VALID_FLAGS | MOUNT_ATTR_IDMAP) + +#define MOUNT_SETATTR_PROPAGATION_FLAGS \ + (MS_UNBINDABLE | MS_PRIVATE | MS_SLAVE | MS_SHARED) + +static unsigned int attr_flags_to_mnt_flags(u64 attr_flags) +{ + unsigned int mnt_flags = 0; + + if (attr_flags & MOUNT_ATTR_RDONLY) + mnt_flags |= MNT_READONLY; + if (attr_flags & MOUNT_ATTR_NOSUID) + mnt_flags |= MNT_NOSUID; + if (attr_flags & MOUNT_ATTR_NODEV) + mnt_flags |= MNT_NODEV; + if (attr_flags & MOUNT_ATTR_NOEXEC) + mnt_flags |= MNT_NOEXEC; + if (attr_flags & MOUNT_ATTR_NODIRATIME) + mnt_flags |= MNT_NODIRATIME; + if (attr_flags & MOUNT_ATTR_NOSYMFOLLOW) + mnt_flags |= MNT_NOSYMFOLLOW; + + return mnt_flags; +} + +/* + * Create a kernel mount representation for a new, prepared superblock + * (specified by fs_fd) and attach to an open_tree-like file descriptor. + */ +SYSCALL_DEFINE3(fsmount, int, fs_fd, unsigned int, flags, + unsigned int, attr_flags) +{ + struct path new_path __free(path_put) = {}; + struct mnt_namespace *ns; + struct fs_context *fc; + struct vfsmount *new_mnt; + struct mount *mnt; + unsigned int mnt_flags = 0; + long ret; + + if ((flags & ~(FSMOUNT_CLOEXEC | FSMOUNT_NAMESPACE)) != 0) + return -EINVAL; + + if ((flags & FSMOUNT_NAMESPACE) && + !ns_capable(current_user_ns(), CAP_SYS_ADMIN)) + return -EPERM; + + if (!(flags & FSMOUNT_NAMESPACE) && !may_mount()) + return -EPERM; + + if (attr_flags & ~FSMOUNT_VALID_FLAGS) + return -EINVAL; + + mnt_flags = attr_flags_to_mnt_flags(attr_flags); + + switch (attr_flags & MOUNT_ATTR__ATIME) { + case MOUNT_ATTR_STRICTATIME: + break; + case MOUNT_ATTR_NOATIME: + mnt_flags |= MNT_NOATIME; + break; + case MOUNT_ATTR_RELATIME: + mnt_flags |= MNT_RELATIME; + break; + default: + return -EINVAL; + } + + CLASS(fd, f)(fs_fd); + if (fd_empty(f)) + return -EBADF; + + if (fd_file(f)->f_op != &fscontext_fops) + return -EINVAL; + + fc = fd_file(f)->private_data; + + ACQUIRE(mutex_intr, uapi_mutex)(&fc->uapi_mutex); + ret = ACQUIRE_ERR(mutex_intr, &uapi_mutex); + if (ret) + return ret; + + /* There must be a valid superblock or we can't mount it */ + ret = -EINVAL; + if (!fc->root) + return ret; + + ret = -EPERM; + if (mount_too_revealing(fc->root->d_sb, &mnt_flags)) { + errorfcp(fc, "VFS", "Mount too revealing"); + return ret; + } + + ret = -EBUSY; + if (fc->phase != FS_CONTEXT_AWAITING_MOUNT) + return ret; + + if (fc->sb_flags & SB_MANDLOCK) + warn_mandlock(); + + new_mnt = vfs_create_mount(fc); + if (IS_ERR(new_mnt)) + return PTR_ERR(new_mnt); + if (new_mnt->mnt_sb->s_flags & SB_NOUSER) { + mntput(new_mnt); + return -EINVAL; + } + new_mnt->mnt_flags = mnt_flags; + + new_path.dentry = dget(fc->root); + new_path.mnt = new_mnt; + + /* We've done the mount bit - now move the file context into more or + * less the same state as if we'd done an fspick(). We don't want to + * do any memory allocation or anything like that at this point as we + * don't want to have to handle any errors incurred. + */ + vfs_clean_context(fc); + + if (flags & FSMOUNT_NAMESPACE) + return FD_ADD((flags & FSMOUNT_CLOEXEC) ? O_CLOEXEC : 0, + open_new_namespace(&new_path, MOUNT_COPY_NEW)); + + ns = alloc_mnt_ns(current->nsproxy->mnt_ns->user_ns, true); + if (IS_ERR(ns)) + return PTR_ERR(ns); + mnt = real_mount(new_path.mnt); + ns->root = mnt; + ns->nr_mounts = 1; + mnt_add_to_ns(ns, mnt); + mntget(new_path.mnt); + + FD_PREPARE(fdf, (flags & FSMOUNT_CLOEXEC) ? O_CLOEXEC : 0, + dentry_open(&new_path, O_PATH, fc->cred)); + if (fdf.err) { + dissolve_on_fput(new_path.mnt); + return fdf.err; + } + + /* + * Attach to an apparent O_PATH fd with a note that we + * need to unmount it, not just simply put it. + */ + fd_prepare_file(fdf)->f_mode |= FMODE_NEED_UNMOUNT; + return fd_publish(fdf); +} + +static inline int vfs_move_mount(const struct path *from_path, + const struct path *to_path, + enum mnt_tree_flags_t mflags) +{ + int ret; + + ret = security_move_mount(from_path, to_path); + if (ret) + return ret; + + if (mflags & MNT_TREE_PROPAGATION) + return do_set_group(from_path, to_path); + + return do_move_mount(from_path, to_path, mflags); +} + +/* + * Move a mount from one place to another. In combination with + * fsopen()/fsmount() this is used to install a new mount and in combination + * with open_tree(OPEN_TREE_CLONE [| AT_RECURSIVE]) it can be used to copy + * a mount subtree. + * + * Note the flags value is a combination of MOVE_MOUNT_* flags. + */ +SYSCALL_DEFINE5(move_mount, + int, from_dfd, const char __user *, from_pathname, + int, to_dfd, const char __user *, to_pathname, + unsigned int, flags) +{ + struct path to_path __free(path_put) = {}; + struct path from_path __free(path_put) = {}; + unsigned int lflags, uflags; + enum mnt_tree_flags_t mflags = 0; + int ret = 0; + + if (!may_mount()) + return -EPERM; + + if (flags & ~MOVE_MOUNT__MASK) + return -EINVAL; + + if ((flags & (MOVE_MOUNT_BENEATH | MOVE_MOUNT_SET_GROUP)) == + (MOVE_MOUNT_BENEATH | MOVE_MOUNT_SET_GROUP)) + return -EINVAL; + + if (flags & MOVE_MOUNT_SET_GROUP) mflags |= MNT_TREE_PROPAGATION; + if (flags & MOVE_MOUNT_BENEATH) mflags |= MNT_TREE_BENEATH; + + uflags = 0; + if (flags & MOVE_MOUNT_T_EMPTY_PATH) + uflags = AT_EMPTY_PATH; + + CLASS(filename_maybe_null,to_name)(to_pathname, uflags); + if (!to_name && to_dfd >= 0) { + CLASS(fd_raw, f_to)(to_dfd); + if (fd_empty(f_to)) + return -EBADF; + + to_path = fd_file(f_to)->f_path; + path_get(&to_path); + } else { + lflags = 0; + if (flags & MOVE_MOUNT_T_SYMLINKS) + lflags |= LOOKUP_FOLLOW; + if (flags & MOVE_MOUNT_T_AUTOMOUNTS) + lflags |= LOOKUP_AUTOMOUNT; + ret = filename_lookup(to_dfd, to_name, lflags, &to_path, NULL); + if (ret) + return ret; + } + + uflags = 0; + if (flags & MOVE_MOUNT_F_EMPTY_PATH) + uflags = AT_EMPTY_PATH; + + CLASS(filename_maybe_null,from_name)(from_pathname, uflags); + if (!from_name && from_dfd >= 0) { + CLASS(fd_raw, f_from)(from_dfd); + if (fd_empty(f_from)) + return -EBADF; + + return vfs_move_mount(&fd_file(f_from)->f_path, &to_path, mflags); + } + + lflags = 0; + if (flags & MOVE_MOUNT_F_SYMLINKS) + lflags |= LOOKUP_FOLLOW; + if (flags & MOVE_MOUNT_F_AUTOMOUNTS) + lflags |= LOOKUP_AUTOMOUNT; + ret = filename_lookup(from_dfd, from_name, lflags, &from_path, NULL); + if (ret) + return ret; + + return vfs_move_mount(&from_path, &to_path, mflags); +} + +/* + * Return true if path is reachable from root + * + * locks: mount_locked_reader || namespace_shared && is_mounted(mnt) + */ +bool is_path_reachable(struct mount *mnt, struct dentry *dentry, + const struct path *root) +{ + while (&mnt->mnt != root->mnt && mnt_has_parent(mnt)) { + dentry = mnt->mnt_mountpoint; + mnt = mnt->mnt_parent; + } + return &mnt->mnt == root->mnt && is_subdir(dentry, root->dentry); +} + +bool path_is_under(const struct path *path1, const struct path *path2) +{ + guard(mount_locked_reader)(); + return is_path_reachable(real_mount(path1->mnt), path1->dentry, path2); +} +EXPORT_SYMBOL(path_is_under); + +int path_pivot_root(struct path *new, struct path *old) +{ + struct path root __free(path_put) = {}; + struct mount *new_mnt, *root_mnt, *old_mnt, *root_parent, *ex_parent; + int error; + + if (!may_mount()) + return -EPERM; + + error = security_sb_pivotroot(old, new); + if (error) + return error; + + get_fs_root(current->fs, &root); + + LOCK_MOUNT(old_mp, old); + old_mnt = old_mp.parent; + if (IS_ERR(old_mnt)) + return PTR_ERR(old_mnt); + + new_mnt = real_mount(new->mnt); + root_mnt = real_mount(root.mnt); + ex_parent = new_mnt->mnt_parent; + root_parent = root_mnt->mnt_parent; + if (IS_MNT_SHARED(old_mnt) || + IS_MNT_SHARED(ex_parent) || + IS_MNT_SHARED(root_parent)) + return -EINVAL; + if (!check_mnt(root_mnt) || !check_mnt(new_mnt)) + return -EINVAL; + if (new_mnt->mnt.mnt_flags & MNT_LOCKED) + return -EINVAL; + if (d_unlinked(new->dentry)) + return -ENOENT; + if (new_mnt == root_mnt || old_mnt == root_mnt) + return -EBUSY; /* loop, on the same file system */ + if (!path_mounted(&root)) + return -EINVAL; /* not a mountpoint */ + if (!mnt_has_parent(root_mnt)) + return -EINVAL; /* absolute root */ + if (!path_mounted(new)) + return -EINVAL; /* not a mountpoint */ + if (!mnt_has_parent(new_mnt)) + return -EINVAL; /* absolute root */ + /* make sure we can reach put_old from new_root */ + if (!is_path_reachable(old_mnt, old_mp.mp->m_dentry, new)) + return -EINVAL; + /* make certain new is below the root */ + if (!is_path_reachable(new_mnt, new->dentry, &root)) + return -EINVAL; + lock_mount_hash(); + umount_mnt(new_mnt); + if (root_mnt->mnt.mnt_flags & MNT_LOCKED) { + new_mnt->mnt.mnt_flags |= MNT_LOCKED; + root_mnt->mnt.mnt_flags &= ~MNT_LOCKED; + } + /* mount new_root on / */ + attach_mnt(new_mnt, root_parent, root_mnt->mnt_mp); + umount_mnt(root_mnt); + /* mount old root on put_old */ + attach_mnt(root_mnt, old_mnt, old_mp.mp); + touch_mnt_namespace(current->nsproxy->mnt_ns); + /* A moved mount should not expire automatically */ + list_del_init(&new_mnt->mnt_expire); + unlock_mount_hash(); + mnt_notify_add(root_mnt); + mnt_notify_add(new_mnt); + chroot_fs_refs(&root, new); + return 0; +} + +/* + * pivot_root Semantics: + * Moves the root file system of the current process to the directory put_old, + * makes new_root as the new root file system of the current process, and sets + * root/cwd of all processes which had them on the current root to new_root. + * + * Restrictions: + * The new_root and put_old must be directories, and must not be on the + * same file system as the current process root. The put_old must be + * underneath new_root, i.e. adding a non-zero number of /.. to the string + * pointed to by put_old must yield the same directory as new_root. No other + * file system may be mounted on put_old. After all, new_root is a mountpoint. + * + * The immutable nullfs filesystem is mounted as the true root of the VFS + * hierarchy. The mutable rootfs (tmpfs/ramfs) is layered on top of this, + * allowing pivot_root() to work normally from initramfs. + * + * Notes: + * - we don't move root/cwd if they are not at the root (reason: if something + * cared enough to change them, it's probably wrong to force them elsewhere) + * - it's okay to pick a root that isn't the root of a file system, e.g. + * /nfs/my_root where /nfs is the mount point. It must be a mountpoint, + * though, so you may need to say mount --bind /nfs/my_root /nfs/my_root + * first. + */ +SYSCALL_DEFINE2(pivot_root, const char __user *, new_root, + const char __user *, put_old) +{ + struct path new __free(path_put) = {}; + struct path old __free(path_put) = {}; + int error; + + error = user_path_at(AT_FDCWD, new_root, + LOOKUP_FOLLOW | LOOKUP_DIRECTORY, &new); + if (error) + return error; + + error = user_path_at(AT_FDCWD, put_old, + LOOKUP_FOLLOW | LOOKUP_DIRECTORY, &old); + if (error) + return error; + + return path_pivot_root(&new, &old); +} + +static unsigned int recalc_flags(struct mount_kattr *kattr, struct mount *mnt) +{ + unsigned int flags = mnt->mnt.mnt_flags; + + /* flags to clear */ + flags &= ~kattr->attr_clr; + /* flags to raise */ + flags |= kattr->attr_set; + + return flags; +} + +static int can_idmap_mount(const struct mount_kattr *kattr, struct mount *mnt) +{ + struct vfsmount *m = &mnt->mnt; + struct user_namespace *fs_userns = m->mnt_sb->s_user_ns; + + if (!kattr->mnt_idmap) + return 0; + + /* + * Creating an idmapped mount with the filesystem wide idmapping + * doesn't make sense so block that. We don't allow mushy semantics. + */ + if (kattr->mnt_userns == m->mnt_sb->s_user_ns) + return -EINVAL; + + /* + * We only allow an mount to change it's idmapping if it has + * never been accessible to userspace. + */ + if (!(kattr->kflags & MOUNT_KATTR_IDMAP_REPLACE) && is_idmapped_mnt(m)) + return -EPERM; + + /* The underlying filesystem doesn't support idmapped mounts yet. */ + if (!(m->mnt_sb->s_type->fs_flags & FS_ALLOW_IDMAP)) + return -EINVAL; + + /* The filesystem has turned off idmapped mounts. */ + if (m->mnt_sb->s_iflags & SB_I_NOIDMAP) + return -EINVAL; + + /* We're not controlling the superblock. */ + if (!ns_capable(fs_userns, CAP_SYS_ADMIN)) + return -EPERM; + + /* Mount has already been visible in the filesystem hierarchy. */ + if (!is_anon_ns(mnt->mnt_ns)) + return -EINVAL; + + return 0; +} + +/** + * mnt_allow_writers() - check whether the attribute change allows writers + * @kattr: the new mount attributes + * @mnt: the mount to which @kattr will be applied + * + * Check whether thew new mount attributes in @kattr allow concurrent writers. + * + * Return: true if writers need to be held, false if not + */ +static inline bool mnt_allow_writers(const struct mount_kattr *kattr, + const struct mount *mnt) +{ + return (!(kattr->attr_set & MNT_READONLY) || + (mnt->mnt.mnt_flags & MNT_READONLY)) && + !kattr->mnt_idmap; +} + +static int mount_setattr_prepare(struct mount_kattr *kattr, struct mount *mnt) +{ + struct mount *m; + int err; + + for (m = mnt; m; m = next_mnt(m, mnt)) { + if (!can_change_locked_flags(m, recalc_flags(kattr, m))) { + err = -EPERM; + break; + } + + err = can_idmap_mount(kattr, m); + if (err) + break; + + if (!mnt_allow_writers(kattr, m)) { + err = mnt_hold_writers(m); + if (err) { + m = next_mnt(m, mnt); + break; + } + } + + if (!(kattr->kflags & MOUNT_KATTR_RECURSE)) + return 0; + } + + if (err) { + /* undo all mnt_hold_writers() we'd done */ + for (struct mount *p = mnt; p != m; p = next_mnt(p, mnt)) + mnt_unhold_writers(p); + } + return err; +} + +static void do_idmap_mount(const struct mount_kattr *kattr, struct mount *mnt) +{ + struct mnt_idmap *old_idmap; + + if (!kattr->mnt_idmap) + return; + + old_idmap = mnt_idmap(&mnt->mnt); + + /* Pairs with smp_load_acquire() in mnt_idmap(). */ + smp_store_release(&mnt->mnt.mnt_idmap, mnt_idmap_get(kattr->mnt_idmap)); + mnt_idmap_put(old_idmap); +} + +static void mount_setattr_commit(struct mount_kattr *kattr, struct mount *mnt) +{ + struct mount *m; + + for (m = mnt; m; m = next_mnt(m, mnt)) { + unsigned int flags; + + do_idmap_mount(kattr, m); + flags = recalc_flags(kattr, m); + WRITE_ONCE(m->mnt.mnt_flags, flags); + + /* If we had to hold writers unblock them. */ + mnt_unhold_writers(m); + + if (kattr->propagation) + change_mnt_propagation(m, kattr->propagation); + if (!(kattr->kflags & MOUNT_KATTR_RECURSE)) + break; + } + touch_mnt_namespace(mnt->mnt_ns); +} + +static int do_mount_setattr(const struct path *path, struct mount_kattr *kattr) +{ + struct mount *mnt = real_mount(path->mnt); + int err = 0; + + if (!path_mounted(path)) + return -EINVAL; + + if (kattr->mnt_userns) { + struct mnt_idmap *mnt_idmap; + + mnt_idmap = alloc_mnt_idmap(kattr->mnt_userns); + if (IS_ERR(mnt_idmap)) + return PTR_ERR(mnt_idmap); + kattr->mnt_idmap = mnt_idmap; + } + + if (kattr->propagation) { + /* + * Only take namespace_lock() if we're actually changing + * propagation. + */ + namespace_lock(); + if (kattr->propagation == MS_SHARED) { + err = invent_group_ids(mnt, kattr->kflags & MOUNT_KATTR_RECURSE); + if (err) { + namespace_unlock(); + return err; + } + } + } + + err = -EINVAL; + lock_mount_hash(); + + if (!anon_ns_root(mnt) && !check_mnt(mnt)) + goto out; + + /* + * First, we get the mount tree in a shape where we can change mount + * properties without failure. If we succeeded to do so we commit all + * changes and if we failed we clean up. + */ + err = mount_setattr_prepare(kattr, mnt); + if (!err) + mount_setattr_commit(kattr, mnt); + +out: + unlock_mount_hash(); + + if (kattr->propagation) { + if (err) + cleanup_group_ids(mnt, NULL); + namespace_unlock(); + } + + return err; +} + +static int build_mount_idmapped(const struct mount_attr *attr, size_t usize, + struct mount_kattr *kattr) +{ + struct ns_common *ns; + struct user_namespace *mnt_userns; + + if (!((attr->attr_set | attr->attr_clr) & MOUNT_ATTR_IDMAP)) + return 0; + + if (attr->attr_clr & MOUNT_ATTR_IDMAP) { + /* + * We can only remove an idmapping if it's never been + * exposed to userspace. + */ + if (!(kattr->kflags & MOUNT_KATTR_IDMAP_REPLACE)) + return -EINVAL; + + /* + * Removal of idmappings is equivalent to setting + * nop_mnt_idmap. + */ + if (!(attr->attr_set & MOUNT_ATTR_IDMAP)) { + kattr->mnt_idmap = &nop_mnt_idmap; + return 0; + } + } + + if (attr->userns_fd > INT_MAX) + return -EINVAL; + + CLASS(fd, f)(attr->userns_fd); + if (fd_empty(f)) + return -EBADF; + + if (!proc_ns_file(fd_file(f))) + return -EINVAL; + + ns = get_proc_ns(file_inode(fd_file(f))); + if (ns->ns_type != CLONE_NEWUSER) + return -EINVAL; + + /* + * The initial idmapping cannot be used to create an idmapped + * mount. We use the initial idmapping as an indicator of a mount + * that is not idmapped. It can simply be passed into helpers that + * are aware of idmapped mounts as a convenient shortcut. A user + * can just create a dedicated identity mapping to achieve the same + * result. + */ + mnt_userns = container_of(ns, struct user_namespace, ns); + if (mnt_userns == &init_user_ns) + return -EPERM; + + /* We're not controlling the target namespace. */ + if (!ns_capable(mnt_userns, CAP_SYS_ADMIN)) + return -EPERM; + + kattr->mnt_userns = get_user_ns(mnt_userns); + return 0; +} + +static int build_mount_kattr(const struct mount_attr *attr, size_t usize, + struct mount_kattr *kattr) +{ + if (attr->propagation & ~MOUNT_SETATTR_PROPAGATION_FLAGS) + return -EINVAL; + if (hweight32(attr->propagation & MOUNT_SETATTR_PROPAGATION_FLAGS) > 1) + return -EINVAL; + kattr->propagation = attr->propagation; + + if ((attr->attr_set | attr->attr_clr) & ~MOUNT_SETATTR_VALID_FLAGS) + return -EINVAL; + + kattr->attr_set = attr_flags_to_mnt_flags(attr->attr_set); + kattr->attr_clr = attr_flags_to_mnt_flags(attr->attr_clr); + + /* + * Since the MOUNT_ATTR_ values are an enum, not a bitmap, + * users wanting to transition to a different atime setting cannot + * simply specify the atime setting in @attr_set, but must also + * specify MOUNT_ATTR__ATIME in the @attr_clr field. + * So ensure that MOUNT_ATTR__ATIME can't be partially set in + * @attr_clr and that @attr_set can't have any atime bits set if + * MOUNT_ATTR__ATIME isn't set in @attr_clr. + */ + if (attr->attr_clr & MOUNT_ATTR__ATIME) { + if ((attr->attr_clr & MOUNT_ATTR__ATIME) != MOUNT_ATTR__ATIME) + return -EINVAL; + + /* + * Clear all previous time settings as they are mutually + * exclusive. + */ + kattr->attr_clr |= MNT_RELATIME | MNT_NOATIME; + switch (attr->attr_set & MOUNT_ATTR__ATIME) { + case MOUNT_ATTR_RELATIME: + kattr->attr_set |= MNT_RELATIME; + break; + case MOUNT_ATTR_NOATIME: + kattr->attr_set |= MNT_NOATIME; + break; + case MOUNT_ATTR_STRICTATIME: + break; + default: + return -EINVAL; + } + } else { + if (attr->attr_set & MOUNT_ATTR__ATIME) + return -EINVAL; + } + + return build_mount_idmapped(attr, usize, kattr); +} + +static void finish_mount_kattr(struct mount_kattr *kattr) +{ + if (kattr->mnt_userns) { + put_user_ns(kattr->mnt_userns); + kattr->mnt_userns = NULL; + } + + if (kattr->mnt_idmap) + mnt_idmap_put(kattr->mnt_idmap); +} + +static int wants_mount_setattr(struct mount_attr __user *uattr, size_t usize, + struct mount_kattr *kattr) +{ + int ret; + struct mount_attr attr; + + BUILD_BUG_ON(sizeof(struct mount_attr) != MOUNT_ATTR_SIZE_VER0); + + if (unlikely(usize > PAGE_SIZE)) + return -E2BIG; + if (unlikely(usize < MOUNT_ATTR_SIZE_VER0)) + return -EINVAL; + + if (!may_mount()) + return -EPERM; + + ret = copy_struct_from_user(&attr, sizeof(attr), uattr, usize); + if (ret) + return ret; + + /* Don't bother walking through the mounts if this is a nop. */ + if (attr.attr_set == 0 && + attr.attr_clr == 0 && + attr.propagation == 0) + return 0; /* Tell caller to not bother. */ + + ret = build_mount_kattr(&attr, usize, kattr); + if (ret < 0) + return ret; + + return 1; +} + +SYSCALL_DEFINE5(mount_setattr, int, dfd, const char __user *, path, + unsigned int, flags, struct mount_attr __user *, uattr, + size_t, usize) +{ + int err; + struct path target; + struct mount_kattr kattr; + unsigned int lookup_flags = LOOKUP_AUTOMOUNT | LOOKUP_FOLLOW; + + if (flags & ~(AT_EMPTY_PATH | + AT_RECURSIVE | + AT_SYMLINK_NOFOLLOW | + AT_NO_AUTOMOUNT)) + return -EINVAL; + + if (flags & AT_NO_AUTOMOUNT) + lookup_flags &= ~LOOKUP_AUTOMOUNT; + if (flags & AT_SYMLINK_NOFOLLOW) + lookup_flags &= ~LOOKUP_FOLLOW; + + kattr = (struct mount_kattr) { + .lookup_flags = lookup_flags, + }; + + if (flags & AT_RECURSIVE) + kattr.kflags |= MOUNT_KATTR_RECURSE; + + err = wants_mount_setattr(uattr, usize, &kattr); + if (err <= 0) + return err; + + CLASS(filename_uflags, name)(path, flags); + err = filename_lookup(dfd, name, kattr.lookup_flags, &target, NULL); + if (!err) { + err = do_mount_setattr(&target, &kattr); + path_put(&target); + } + finish_mount_kattr(&kattr); + return err; +} + +SYSCALL_DEFINE5(open_tree_attr, int, dfd, const char __user *, filename, + unsigned, flags, struct mount_attr __user *, uattr, + size_t, usize) +{ + if (!uattr && usize) + return -EINVAL; + + FD_PREPARE(fdf, flags, vfs_open_tree(dfd, filename, flags)); + if (fdf.err) + return fdf.err; + + if (uattr) { + struct mount_kattr kattr = {}; + struct file *file = fd_prepare_file(fdf); + int ret; + + if (flags & OPEN_TREE_CLONE) + kattr.kflags = MOUNT_KATTR_IDMAP_REPLACE; + if (flags & AT_RECURSIVE) + kattr.kflags |= MOUNT_KATTR_RECURSE; + + ret = wants_mount_setattr(uattr, usize, &kattr); + if (ret > 0) { + ret = do_mount_setattr(&file->f_path, &kattr); + finish_mount_kattr(&kattr); + } + if (ret) + return ret; + } + + return fd_publish(fdf); +} + +int show_path(struct seq_file *m, struct dentry *root) +{ + if (root->d_sb->s_op->show_path) + return root->d_sb->s_op->show_path(m, root); + + seq_dentry(m, root, " \t\n\\"); + return 0; +} + +static struct vfsmount *lookup_mnt_in_ns(u64 id, struct mnt_namespace *ns) +{ + struct mount *mnt = mnt_find_id_at(ns, id); + + if (!mnt || mnt->mnt_id_unique != id) + return NULL; + + return &mnt->mnt; +} + +struct kstatmount { + struct statmount __user *buf; + size_t bufsize; + struct vfsmount *mnt; + struct mnt_idmap *idmap; + u64 mask; + struct path root; + struct seq_file seq; + + /* Must be last --ends in a flexible-array member. */ + struct statmount sm; +}; + +static u64 mnt_to_attr_flags(struct vfsmount *mnt) +{ + unsigned int mnt_flags = READ_ONCE(mnt->mnt_flags); + u64 attr_flags = 0; + + if (mnt_flags & MNT_READONLY) + attr_flags |= MOUNT_ATTR_RDONLY; + if (mnt_flags & MNT_NOSUID) + attr_flags |= MOUNT_ATTR_NOSUID; + if (mnt_flags & MNT_NODEV) + attr_flags |= MOUNT_ATTR_NODEV; + if (mnt_flags & MNT_NOEXEC) + attr_flags |= MOUNT_ATTR_NOEXEC; + if (mnt_flags & MNT_NODIRATIME) + attr_flags |= MOUNT_ATTR_NODIRATIME; + if (mnt_flags & MNT_NOSYMFOLLOW) + attr_flags |= MOUNT_ATTR_NOSYMFOLLOW; + + if (mnt_flags & MNT_NOATIME) + attr_flags |= MOUNT_ATTR_NOATIME; + else if (mnt_flags & MNT_RELATIME) + attr_flags |= MOUNT_ATTR_RELATIME; + else + attr_flags |= MOUNT_ATTR_STRICTATIME; + + if (is_idmapped_mnt(mnt)) + attr_flags |= MOUNT_ATTR_IDMAP; + + return attr_flags; +} + +static u64 mnt_to_propagation_flags(struct mount *m) +{ + u64 propagation = 0; + + if (IS_MNT_SHARED(m)) + propagation |= MS_SHARED; + if (IS_MNT_SLAVE(m)) + propagation |= MS_SLAVE; + if (IS_MNT_UNBINDABLE(m)) + propagation |= MS_UNBINDABLE; + if (!propagation) + propagation |= MS_PRIVATE; + + return propagation; +} + +u64 vfsmount_to_propagation_flags(struct vfsmount *mnt) +{ + return mnt_to_propagation_flags(real_mount(mnt)); +} +EXPORT_SYMBOL_GPL(vfsmount_to_propagation_flags); + +static void statmount_sb_basic(struct kstatmount *s) +{ + struct super_block *sb = s->mnt->mnt_sb; + + s->sm.mask |= STATMOUNT_SB_BASIC; + s->sm.sb_dev_major = MAJOR(sb->s_dev); + s->sm.sb_dev_minor = MINOR(sb->s_dev); + s->sm.sb_magic = sb->s_magic; + s->sm.sb_flags = sb->s_flags & (SB_RDONLY|SB_SYNCHRONOUS|SB_DIRSYNC|SB_LAZYTIME); +} + +static void statmount_mnt_basic(struct kstatmount *s) +{ + struct mount *m = real_mount(s->mnt); + + s->sm.mask |= STATMOUNT_MNT_BASIC; + s->sm.mnt_id = m->mnt_id_unique; + s->sm.mnt_parent_id = m->mnt_parent->mnt_id_unique; + s->sm.mnt_id_old = m->mnt_id; + s->sm.mnt_parent_id_old = m->mnt_parent->mnt_id; + s->sm.mnt_attr = mnt_to_attr_flags(&m->mnt); + s->sm.mnt_propagation = mnt_to_propagation_flags(m); + s->sm.mnt_peer_group = m->mnt_group_id; + s->sm.mnt_master = IS_MNT_SLAVE(m) ? m->mnt_master->mnt_group_id : 0; +} + +static void statmount_propagate_from(struct kstatmount *s) +{ + struct mount *m = real_mount(s->mnt); + + s->sm.mask |= STATMOUNT_PROPAGATE_FROM; + if (IS_MNT_SLAVE(m)) + s->sm.propagate_from = get_dominating_id(m, ¤t->fs->root); +} + +static int statmount_mnt_root(struct kstatmount *s, struct seq_file *seq) +{ + int ret; + size_t start = seq->count; + + ret = show_path(seq, s->mnt->mnt_root); + if (ret) + return ret; + + if (unlikely(seq_has_overflowed(seq))) + return -EAGAIN; + + /* + * Unescape the result. It would be better if supplied string was not + * escaped in the first place, but that's a pretty invasive change. + */ + seq->buf[seq->count] = '\0'; + seq->count = start; + seq_commit(seq, string_unescape_inplace(seq->buf + start, UNESCAPE_OCTAL)); + return 0; +} + +static int statmount_mnt_point(struct kstatmount *s, struct seq_file *seq) +{ + struct vfsmount *mnt = s->mnt; + struct path mnt_path = { .dentry = mnt->mnt_root, .mnt = mnt }; + int err; + + err = seq_path_root(seq, &mnt_path, &s->root, ""); + return err == SEQ_SKIP ? 0 : err; +} + +static int statmount_fs_type(struct kstatmount *s, struct seq_file *seq) +{ + struct super_block *sb = s->mnt->mnt_sb; + + seq_puts(seq, sb->s_type->name); + return 0; +} + +static void statmount_fs_subtype(struct kstatmount *s, struct seq_file *seq) +{ + struct super_block *sb = s->mnt->mnt_sb; + + if (sb->s_subtype) + seq_puts(seq, sb->s_subtype); +} + +static int statmount_sb_source(struct kstatmount *s, struct seq_file *seq) +{ + struct super_block *sb = s->mnt->mnt_sb; + struct mount *r = real_mount(s->mnt); + + if (sb->s_op->show_devname) { + size_t start = seq->count; + int ret; + + ret = sb->s_op->show_devname(seq, s->mnt->mnt_root); + if (ret) + return ret; + + if (unlikely(seq_has_overflowed(seq))) + return -EAGAIN; + + /* Unescape the result */ + seq->buf[seq->count] = '\0'; + seq->count = start; + seq_commit(seq, string_unescape_inplace(seq->buf + start, UNESCAPE_OCTAL)); + } else { + seq_puts(seq, r->mnt_devname); + } + return 0; +} + +static void statmount_mnt_ns_id(struct kstatmount *s, struct mnt_namespace *ns) +{ + s->sm.mask |= STATMOUNT_MNT_NS_ID; + s->sm.mnt_ns_id = ns->ns.ns_id; +} + +static int statmount_mnt_opts(struct kstatmount *s, struct seq_file *seq) +{ + struct vfsmount *mnt = s->mnt; + struct super_block *sb = mnt->mnt_sb; + size_t start = seq->count; + int err; + + err = security_sb_show_options(seq, sb); + if (err) + return err; + + if (sb->s_op->show_options) { + err = sb->s_op->show_options(seq, mnt->mnt_root); + if (err) + return err; + } + + if (unlikely(seq_has_overflowed(seq))) + return -EAGAIN; + + if (seq->count == start) + return 0; + + /* skip leading comma */ + memmove(seq->buf + start, seq->buf + start + 1, + seq->count - start - 1); + seq->count--; + + return 0; +} + +static inline int statmount_opt_process(struct seq_file *seq, size_t start) +{ + char *buf_end, *opt_end, *src, *dst; + int count = 0; + + if (unlikely(seq_has_overflowed(seq))) + return -EAGAIN; + + buf_end = seq->buf + seq->count; + dst = seq->buf + start; + src = dst + 1; /* skip initial comma */ + + if (src >= buf_end) { + seq->count = start; + return 0; + } + + *buf_end = '\0'; + for (; src < buf_end; src = opt_end + 1) { + opt_end = strchrnul(src, ','); + *opt_end = '\0'; + dst += string_unescape(src, dst, 0, UNESCAPE_OCTAL) + 1; + if (WARN_ON_ONCE(++count == INT_MAX)) + return -EOVERFLOW; + } + seq->count = dst - 1 - seq->buf; + return count; +} + +static int statmount_opt_array(struct kstatmount *s, struct seq_file *seq) +{ + struct vfsmount *mnt = s->mnt; + struct super_block *sb = mnt->mnt_sb; + size_t start = seq->count; + int err; + + if (!sb->s_op->show_options) + return 0; + + err = sb->s_op->show_options(seq, mnt->mnt_root); + if (err) + return err; + + err = statmount_opt_process(seq, start); + if (err < 0) + return err; + + s->sm.opt_num = err; + return 0; +} + +static int statmount_opt_sec_array(struct kstatmount *s, struct seq_file *seq) +{ + struct vfsmount *mnt = s->mnt; + struct super_block *sb = mnt->mnt_sb; + size_t start = seq->count; + int err; + + err = security_sb_show_options(seq, sb); + if (err) + return err; + + err = statmount_opt_process(seq, start); + if (err < 0) + return err; + + s->sm.opt_sec_num = err; + return 0; +} + +static inline int statmount_mnt_uidmap(struct kstatmount *s, struct seq_file *seq) +{ + int ret; + + ret = statmount_mnt_idmap(s->idmap, seq, true); + if (ret < 0) + return ret; + + s->sm.mnt_uidmap_num = ret; + /* + * Always raise STATMOUNT_MNT_UIDMAP even if there are no valid + * mappings. This allows userspace to distinguish between a + * non-idmapped mount and an idmapped mount where none of the + * individual mappings are valid in the caller's idmapping. + */ + if (is_valid_mnt_idmap(s->idmap)) + s->sm.mask |= STATMOUNT_MNT_UIDMAP; + return 0; +} + +static inline int statmount_mnt_gidmap(struct kstatmount *s, struct seq_file *seq) +{ + int ret; + + ret = statmount_mnt_idmap(s->idmap, seq, false); + if (ret < 0) + return ret; + + s->sm.mnt_gidmap_num = ret; + /* + * Always raise STATMOUNT_MNT_GIDMAP even if there are no valid + * mappings. This allows userspace to distinguish between a + * non-idmapped mount and an idmapped mount where none of the + * individual mappings are valid in the caller's idmapping. + */ + if (is_valid_mnt_idmap(s->idmap)) + s->sm.mask |= STATMOUNT_MNT_GIDMAP; + return 0; +} + +static int statmount_string(struct kstatmount *s, u64 flag) +{ + int ret = 0; + size_t kbufsize; + struct seq_file *seq = &s->seq; + struct statmount *sm = &s->sm; + u32 start, *offp; + + /* Reserve an empty string at the beginning for any unset offsets */ + if (!seq->count) + seq_putc(seq, 0); + + start = seq->count; + + switch (flag) { + case STATMOUNT_FS_TYPE: + offp = &sm->fs_type; + ret = statmount_fs_type(s, seq); + break; + case STATMOUNT_MNT_ROOT: + offp = &sm->mnt_root; + ret = statmount_mnt_root(s, seq); + break; + case STATMOUNT_MNT_POINT: + offp = &sm->mnt_point; + ret = statmount_mnt_point(s, seq); + break; + case STATMOUNT_MNT_OPTS: + offp = &sm->mnt_opts; + ret = statmount_mnt_opts(s, seq); + break; + case STATMOUNT_OPT_ARRAY: + offp = &sm->opt_array; + ret = statmount_opt_array(s, seq); + break; + case STATMOUNT_OPT_SEC_ARRAY: + offp = &sm->opt_sec_array; + ret = statmount_opt_sec_array(s, seq); + break; + case STATMOUNT_FS_SUBTYPE: + offp = &sm->fs_subtype; + statmount_fs_subtype(s, seq); + break; + case STATMOUNT_SB_SOURCE: + offp = &sm->sb_source; + ret = statmount_sb_source(s, seq); + break; + case STATMOUNT_MNT_UIDMAP: + offp = &sm->mnt_uidmap; + ret = statmount_mnt_uidmap(s, seq); + break; + case STATMOUNT_MNT_GIDMAP: + offp = &sm->mnt_gidmap; + ret = statmount_mnt_gidmap(s, seq); + break; + default: + WARN_ON_ONCE(true); + return -EINVAL; + } + + /* + * If nothing was emitted, return to avoid setting the flag + * and terminating the buffer. + */ + if (seq->count == start) + return ret; + if (unlikely(check_add_overflow(sizeof(*sm), seq->count, &kbufsize))) + return -EOVERFLOW; + if (kbufsize >= s->bufsize) + return -EOVERFLOW; + + /* signal a retry */ + if (unlikely(seq_has_overflowed(seq))) + return -EAGAIN; + + if (ret) + return ret; + + seq->buf[seq->count++] = '\0'; + sm->mask |= flag; + *offp = start; + return 0; +} + +static int copy_statmount_to_user(struct kstatmount *s) +{ + struct statmount *sm = &s->sm; + struct seq_file *seq = &s->seq; + char __user *str = ((char __user *)s->buf) + sizeof(*sm); + size_t copysize = min_t(size_t, s->bufsize, sizeof(*sm)); + + if (seq->count && copy_to_user(str, seq->buf, seq->count)) + return -EFAULT; + + /* Return the number of bytes copied to the buffer */ + sm->size = copysize + seq->count; + if (copy_to_user(s->buf, sm, copysize)) + return -EFAULT; + + return 0; +} + +static struct mount *listmnt_next(struct mount *curr, bool reverse) +{ + struct rb_node *node; + + if (reverse) + node = rb_prev(&curr->mnt_node); + else + node = rb_next(&curr->mnt_node); + + return node_to_mount(node); +} + +static int grab_requested_root(struct mnt_namespace *ns, struct path *root) +{ + struct mount *first, *child; + + rwsem_assert_held(&namespace_sem); + + /* We're looking at our own ns, just use get_fs_root. */ + if (ns == current->nsproxy->mnt_ns) { + get_fs_root(current->fs, root); + return 0; + } + + /* + * We have to find the first mount in our ns and use that, however it + * may not exist, so handle that properly. + */ + if (mnt_ns_empty(ns)) + return -ENOENT; + + first = ns->root; + for (child = node_to_mount(ns->mnt_first_node); child; + child = listmnt_next(child, false)) { + if (child != first && child->mnt_parent == first) + break; + } + if (!child) + return -ENOENT; + + root->mnt = mntget(&child->mnt); + root->dentry = dget(root->mnt->mnt_root); + return 0; +} + +/* This must be updated whenever a new flag is added */ +#define STATMOUNT_SUPPORTED (STATMOUNT_SB_BASIC | \ + STATMOUNT_MNT_BASIC | \ + STATMOUNT_PROPAGATE_FROM | \ + STATMOUNT_MNT_ROOT | \ + STATMOUNT_MNT_POINT | \ + STATMOUNT_FS_TYPE | \ + STATMOUNT_MNT_NS_ID | \ + STATMOUNT_MNT_OPTS | \ + STATMOUNT_FS_SUBTYPE | \ + STATMOUNT_SB_SOURCE | \ + STATMOUNT_OPT_ARRAY | \ + STATMOUNT_OPT_SEC_ARRAY | \ + STATMOUNT_SUPPORTED_MASK | \ + STATMOUNT_MNT_UIDMAP | \ + STATMOUNT_MNT_GIDMAP) + +/* locks: namespace_shared */ +static int do_statmount(struct kstatmount *s, u64 mnt_id, u64 mnt_ns_id, + struct file *mnt_file, struct mnt_namespace *ns) +{ + int err; + + if (mnt_file) { + WARN_ON_ONCE(ns != NULL); + + s->mnt = mnt_file->f_path.mnt; + ns = real_mount(s->mnt)->mnt_ns; + if (IS_ERR(ns)) + return PTR_ERR(ns); + if (!ns) + /* + * We can't set mount point and mnt_ns_id since we don't have a + * ns for the mount. This can happen if the mount is unmounted + * with MNT_DETACH. + */ + s->mask &= ~(STATMOUNT_MNT_POINT | STATMOUNT_MNT_NS_ID); + } else { + /* Has the namespace already been emptied? */ + if (mnt_ns_id && mnt_ns_empty(ns)) + return -ENOENT; + + s->mnt = lookup_mnt_in_ns(mnt_id, ns); + if (!s->mnt) + return -ENOENT; + } + + if (ns) { + err = grab_requested_root(ns, &s->root); + if (err) + return err; + + if (!mnt_file) { + struct mount *m; + /* + * Don't trigger audit denials. We just want to determine what + * mounts to show users. + */ + m = real_mount(s->mnt); + if (!is_path_reachable(m, m->mnt.mnt_root, &s->root) && + !ns_capable_noaudit(ns->user_ns, CAP_SYS_ADMIN)) + return -EPERM; + } + } + + err = security_sb_statfs(s->mnt->mnt_root); + if (err) + return err; + + /* + * Note that mount properties in mnt->mnt_flags, mnt->mnt_idmap + * can change concurrently as we only hold the read-side of the + * namespace semaphore and mount properties may change with only + * the mount lock held. + * + * We could sample the mount lock sequence counter to detect + * those changes and retry. But it's not worth it. Worst that + * happens is that the mnt->mnt_idmap pointer is already changed + * while mnt->mnt_flags isn't or vica versa. So what. + * + * Both mnt->mnt_flags and mnt->mnt_idmap are set and retrieved + * via READ_ONCE()/WRITE_ONCE() and guard against theoretical + * torn read/write. That's all we care about right now. + */ + s->idmap = mnt_idmap(s->mnt); + if (s->mask & STATMOUNT_MNT_BASIC) + statmount_mnt_basic(s); + + if (s->mask & STATMOUNT_SB_BASIC) + statmount_sb_basic(s); + + if (s->mask & STATMOUNT_PROPAGATE_FROM) + statmount_propagate_from(s); + + if (s->mask & STATMOUNT_FS_TYPE) + err = statmount_string(s, STATMOUNT_FS_TYPE); + + if (!err && s->mask & STATMOUNT_MNT_ROOT) + err = statmount_string(s, STATMOUNT_MNT_ROOT); + + if (!err && s->mask & STATMOUNT_MNT_POINT) + err = statmount_string(s, STATMOUNT_MNT_POINT); + + if (!err && s->mask & STATMOUNT_MNT_OPTS) + err = statmount_string(s, STATMOUNT_MNT_OPTS); + + if (!err && s->mask & STATMOUNT_OPT_ARRAY) + err = statmount_string(s, STATMOUNT_OPT_ARRAY); + + if (!err && s->mask & STATMOUNT_OPT_SEC_ARRAY) + err = statmount_string(s, STATMOUNT_OPT_SEC_ARRAY); + + if (!err && s->mask & STATMOUNT_FS_SUBTYPE) + err = statmount_string(s, STATMOUNT_FS_SUBTYPE); + + if (!err && s->mask & STATMOUNT_SB_SOURCE) + err = statmount_string(s, STATMOUNT_SB_SOURCE); + + if (!err && s->mask & STATMOUNT_MNT_UIDMAP) + err = statmount_string(s, STATMOUNT_MNT_UIDMAP); + + if (!err && s->mask & STATMOUNT_MNT_GIDMAP) + err = statmount_string(s, STATMOUNT_MNT_GIDMAP); + + if (!err && s->mask & STATMOUNT_MNT_NS_ID) + statmount_mnt_ns_id(s, ns); + + if (!err && s->mask & STATMOUNT_SUPPORTED_MASK) { + s->sm.mask |= STATMOUNT_SUPPORTED_MASK; + s->sm.supported_mask = STATMOUNT_SUPPORTED; + } + + if (err) + return err; + + /* Are there bits in the return mask not present in STATMOUNT_SUPPORTED? */ + WARN_ON_ONCE(~STATMOUNT_SUPPORTED & s->sm.mask); + + return 0; +} + +static inline bool retry_statmount(const long ret, size_t *seq_size) +{ + if (likely(ret != -EAGAIN)) + return false; + if (unlikely(check_mul_overflow(*seq_size, 2, seq_size))) + return false; + if (unlikely(*seq_size > MAX_RW_COUNT)) + return false; + return true; +} + +#define STATMOUNT_STRING_REQ (STATMOUNT_MNT_ROOT | STATMOUNT_MNT_POINT | \ + STATMOUNT_FS_TYPE | STATMOUNT_MNT_OPTS | \ + STATMOUNT_FS_SUBTYPE | STATMOUNT_SB_SOURCE | \ + STATMOUNT_OPT_ARRAY | STATMOUNT_OPT_SEC_ARRAY | \ + STATMOUNT_MNT_UIDMAP | STATMOUNT_MNT_GIDMAP) + +static int prepare_kstatmount(struct kstatmount *ks, struct mnt_id_req *kreq, + struct statmount __user *buf, size_t bufsize, + size_t seq_size) +{ + if (!access_ok(buf, bufsize)) + return -EFAULT; + + memset(ks, 0, sizeof(*ks)); + ks->mask = kreq->param; + ks->buf = buf; + ks->bufsize = bufsize; + + if (ks->mask & STATMOUNT_STRING_REQ) { + if (bufsize == sizeof(ks->sm)) + return -EOVERFLOW; + + ks->seq.buf = kvmalloc(seq_size, GFP_KERNEL_ACCOUNT); + if (!ks->seq.buf) + return -ENOMEM; + + ks->seq.size = seq_size; + } + + return 0; +} + +static int copy_mnt_id_req(const struct mnt_id_req __user *req, + struct mnt_id_req *kreq, unsigned int flags) +{ + int ret; + size_t usize; + + BUILD_BUG_ON(sizeof(struct mnt_id_req) != MNT_ID_REQ_SIZE_VER1); + + ret = get_user(usize, &req->size); + if (ret) + return -EFAULT; + if (unlikely(usize > PAGE_SIZE)) + return -E2BIG; + if (unlikely(usize < MNT_ID_REQ_SIZE_VER0)) + return -EINVAL; + memset(kreq, 0, sizeof(*kreq)); + ret = copy_struct_from_user(kreq, sizeof(*kreq), req, usize); + if (ret) + return ret; + + if (flags & STATMOUNT_BY_FD) { + if (kreq->mnt_id || kreq->mnt_ns_id) + return -EINVAL; + } else { + if (kreq->mnt_ns_fd != 0 && kreq->mnt_ns_id) + return -EINVAL; + /* The first valid unique mount id is MNT_UNIQUE_ID_OFFSET + 1. */ + if (kreq->mnt_id <= MNT_UNIQUE_ID_OFFSET) + return -EINVAL; + } + return 0; +} + +/* + * If the user requested a specific mount namespace id, look that up and return + * that, or if not simply grab a passive reference on our mount namespace and + * return that. + */ +static struct mnt_namespace *grab_requested_mnt_ns(const struct mnt_id_req *kreq) +{ + struct mnt_namespace *mnt_ns; + + if (kreq->mnt_ns_id) { + mnt_ns = lookup_mnt_ns(kreq->mnt_ns_id); + if (!mnt_ns) + return ERR_PTR(-ENOENT); + } else if (kreq->mnt_ns_fd) { + struct ns_common *ns; + + CLASS(fd, f)(kreq->mnt_ns_fd); + if (fd_empty(f)) + return ERR_PTR(-EBADF); + + if (!proc_ns_file(fd_file(f))) + return ERR_PTR(-EINVAL); + + ns = get_proc_ns(file_inode(fd_file(f))); + if (ns->ns_type != CLONE_NEWNS) + return ERR_PTR(-EINVAL); + + mnt_ns = to_mnt_ns(ns); + refcount_inc(&mnt_ns->passive); + } else { + mnt_ns = current->nsproxy->mnt_ns; + refcount_inc(&mnt_ns->passive); + } + + return mnt_ns; +} + +SYSCALL_DEFINE4(statmount, const struct mnt_id_req __user *, req, + struct statmount __user *, buf, size_t, bufsize, + unsigned int, flags) +{ + struct mnt_namespace *ns __free(mnt_ns_release) = NULL; + struct kstatmount *ks __free(kfree) = NULL; + struct file *mnt_file __free(fput) = NULL; + struct mnt_id_req kreq; + /* We currently support retrieval of 3 strings. */ + size_t seq_size = 3 * PATH_MAX; + int ret; + + if (flags & ~STATMOUNT_BY_FD) + return -EINVAL; + + ret = copy_mnt_id_req(req, &kreq, flags); + if (ret) + return ret; + + if (flags & STATMOUNT_BY_FD) { + mnt_file = fget_raw(kreq.mnt_fd); + if (!mnt_file) + return -EBADF; + /* do_statmount sets ns in case of STATMOUNT_BY_FD */ + } else { + ns = grab_requested_mnt_ns(&kreq); + if (IS_ERR(ns)) + return PTR_ERR(ns); + + if (kreq.mnt_ns_id && (ns != current->nsproxy->mnt_ns) && + !ns_capable_noaudit(ns->user_ns, CAP_SYS_ADMIN)) + return -EPERM; + } + + ks = kmalloc(sizeof(*ks), GFP_KERNEL_ACCOUNT); + if (!ks) + return -ENOMEM; + +retry: + ret = prepare_kstatmount(ks, &kreq, buf, bufsize, seq_size); + if (ret) + return ret; + + scoped_guard(namespace_shared) + ret = do_statmount(ks, kreq.mnt_id, kreq.mnt_ns_id, mnt_file, ns); + + if (!ret) + ret = copy_statmount_to_user(ks); + kvfree(ks->seq.buf); + path_put(&ks->root); + if (retry_statmount(ret, &seq_size)) + goto retry; + return ret; +} + +struct klistmount { + u64 last_mnt_id; + u64 mnt_parent_id; + u64 *kmnt_ids; + u32 nr_mnt_ids; + struct mnt_namespace *ns; + struct path root; +}; + +/* locks: namespace_shared */ +static ssize_t do_listmount(struct klistmount *kls, bool reverse) +{ + struct mnt_namespace *ns = kls->ns; + u64 mnt_parent_id = kls->mnt_parent_id; + u64 last_mnt_id = kls->last_mnt_id; + u64 *mnt_ids = kls->kmnt_ids; + size_t nr_mnt_ids = kls->nr_mnt_ids; + struct path orig; + struct mount *r, *first; + ssize_t ret; + + rwsem_assert_held(&namespace_sem); + + ret = grab_requested_root(ns, &kls->root); + if (ret) + return ret; + + if (mnt_parent_id == LSMT_ROOT) { + orig = kls->root; + } else { + orig.mnt = lookup_mnt_in_ns(mnt_parent_id, ns); + if (!orig.mnt) + return -ENOENT; + orig.dentry = orig.mnt->mnt_root; + } + + /* + * Don't trigger audit denials. We just want to determine what + * mounts to show users. + */ + if (!is_path_reachable(real_mount(orig.mnt), orig.dentry, &kls->root) && + !ns_capable_noaudit(ns->user_ns, CAP_SYS_ADMIN)) + return -EPERM; + + ret = security_sb_statfs(orig.dentry); + if (ret) + return ret; + + if (!last_mnt_id) { + if (reverse) + first = node_to_mount(ns->mnt_last_node); + else + first = node_to_mount(ns->mnt_first_node); + } else { + if (reverse) + first = mnt_find_id_at_reverse(ns, last_mnt_id - 1); + else + first = mnt_find_id_at(ns, last_mnt_id + 1); + } + + for (ret = 0, r = first; r && nr_mnt_ids; r = listmnt_next(r, reverse)) { + if (r->mnt_id_unique == mnt_parent_id) + continue; + if (!is_path_reachable(r, r->mnt.mnt_root, &orig)) + continue; + *mnt_ids = r->mnt_id_unique; + mnt_ids++; + nr_mnt_ids--; + ret++; + } + return ret; +} + +static void __free_klistmount_free(const struct klistmount *kls) +{ + path_put(&kls->root); + kvfree(kls->kmnt_ids); + mnt_ns_release(kls->ns); +} + +static inline int prepare_klistmount(struct klistmount *kls, struct mnt_id_req *kreq, + size_t nr_mnt_ids) +{ + u64 last_mnt_id = kreq->param; + struct mnt_namespace *ns; + + /* The first valid unique mount id is MNT_UNIQUE_ID_OFFSET + 1. */ + if (last_mnt_id != 0 && last_mnt_id <= MNT_UNIQUE_ID_OFFSET) + return -EINVAL; + + kls->last_mnt_id = last_mnt_id; + + kls->nr_mnt_ids = nr_mnt_ids; + kls->kmnt_ids = kvmalloc_array(nr_mnt_ids, sizeof(*kls->kmnt_ids), + GFP_KERNEL_ACCOUNT); + if (!kls->kmnt_ids) + return -ENOMEM; + + ns = grab_requested_mnt_ns(kreq); + if (IS_ERR(ns)) + return PTR_ERR(ns); + kls->ns = ns; + + kls->mnt_parent_id = kreq->mnt_id; + return 0; +} + +SYSCALL_DEFINE4(listmount, const struct mnt_id_req __user *, req, + u64 __user *, mnt_ids, size_t, nr_mnt_ids, unsigned int, flags) +{ + struct klistmount kls __free(klistmount_free) = {}; + const size_t maxcount = 1000000; + struct mnt_id_req kreq; + ssize_t ret; + + if (flags & ~LISTMOUNT_REVERSE) + return -EINVAL; + + /* + * If the mount namespace really has more than 1 million mounts the + * caller must iterate over the mount namespace (and reconsider their + * system design...). + */ + if (unlikely(nr_mnt_ids > maxcount)) + return -EOVERFLOW; + + if (!access_ok(mnt_ids, nr_mnt_ids * sizeof(*mnt_ids))) + return -EFAULT; + + ret = copy_mnt_id_req(req, &kreq, 0); + if (ret) + return ret; + + ret = prepare_klistmount(&kls, &kreq, nr_mnt_ids); + if (ret) + return ret; + + if (kreq.mnt_ns_id && (kls.ns != current->nsproxy->mnt_ns) && + !ns_capable_noaudit(kls.ns->user_ns, CAP_SYS_ADMIN)) + return -ENOENT; + + /* + * We only need to guard against mount topology changes as + * listmount() doesn't care about any mount properties. + */ + scoped_guard(namespace_shared) + ret = do_listmount(&kls, (flags & LISTMOUNT_REVERSE)); + if (ret <= 0) + return ret; + + if (copy_to_user(mnt_ids, kls.kmnt_ids, ret * sizeof(*mnt_ids))) + return -EFAULT; + + return ret; +} + +struct mnt_namespace init_mnt_ns = { + .ns = NS_COMMON_INIT(init_mnt_ns), + .user_ns = &init_user_ns, + .passive = REFCOUNT_INIT(1), + .mounts = RB_ROOT, + .poll = __WAIT_QUEUE_HEAD_INITIALIZER(init_mnt_ns.poll), +}; + +static void __init init_mount_tree(void) +{ + struct vfsmount *mnt, *nullfs_mnt; + struct mount *mnt_root; + struct path root; + + /* + * We create three mounts: + * + * (1) nullfs with mount id 1 + * (2) mutable rootfs with mount id 2 + * (3) private nullfs for kthreads (SB_KERNMOUNT) + * + * with (2) mounted on top of (1). The init_task's root and pwd + * are pointed at (3) so all kthreads start isolated in nullfs. + */ + nullfs_mnt = vfs_kern_mount(&nullfs_fs_type, 0, "nullfs", NULL); + if (IS_ERR(nullfs_mnt)) + panic("VFS: Failed to create nullfs"); + + mnt = vfs_kern_mount(&rootfs_fs_type, 0, "rootfs", initramfs_options); + if (IS_ERR(mnt)) + panic("Can't create rootfs"); + + VFS_WARN_ON_ONCE(real_mount(nullfs_mnt)->mnt_id != 1); + VFS_WARN_ON_ONCE(real_mount(mnt)->mnt_id != 2); + + /* The namespace root is the nullfs mnt. */ + mnt_root = real_mount(nullfs_mnt); + init_mnt_ns.root = mnt_root; + + /* Mount mutable rootfs on top of nullfs. */ + root.mnt = nullfs_mnt; + root.dentry = nullfs_mnt->mnt_root; + + LOCK_MOUNT_EXACT(mp, &root); + if (unlikely(IS_ERR(mp.parent))) + panic("VFS: Failed to mount rootfs on nullfs"); + scoped_guard(mount_writer) + attach_mnt(real_mount(mnt), mp.parent, mp.mp); + + pr_info("VFS: Finished mounting rootfs on nullfs\n"); + + /* + * We've dropped all locks here but that's fine. Not just are we + * the only task that's running, there's no other mount + * namespace in existence and the initial mount namespace is + * completely empty until we add the mounts we just created. + */ + for (struct mount *p = mnt_root; p; p = next_mnt(p, mnt_root)) { + mnt_add_to_ns(&init_mnt_ns, p); + init_mnt_ns.nr_mounts++; + } + + nullfs_mnt = kern_mount(&nullfs_fs_type); + if (IS_ERR(nullfs_mnt)) + panic("VFS: Failed to create private nullfs instance"); + root.mnt = nullfs_mnt; + root.dentry = nullfs_mnt->mnt_root; + + init_task.nsproxy->mnt_ns = &init_mnt_ns; + get_mnt_ns(&init_mnt_ns); + set_fs_pwd(current->fs, &root); + set_fs_root(current->fs, &root); + + ns_tree_add(&init_mnt_ns); +} + +void __init mnt_init(void) +{ + int err; + + mnt_cache = kmem_cache_create("mnt_cache", sizeof(struct mount), + 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL); + + mount_hashtable = alloc_large_system_hash("Mount-cache", + sizeof(struct hlist_head), + mhash_entries, 19, + HASH_ZERO, + &m_hash_shift, &m_hash_mask, 0, 0); + mountpoint_hashtable = alloc_large_system_hash("Mountpoint-cache", + sizeof(struct hlist_head), + mphash_entries, 19, + HASH_ZERO, + &mp_hash_shift, &mp_hash_mask, 0, 0); + + super_dev_init(); + + kernfs_init(); + + err = sysfs_init(); + if (err) + printk(KERN_WARNING "%s: sysfs_init error: %d\n", + __func__, err); + fs_kobj = kobject_create_and_add("fs", NULL); + if (!fs_kobj) + printk(KERN_WARNING "%s: kobj create error\n", __func__); + shmem_init(); + init_rootfs(); + init_mount_tree(); + failfs_init(); +} + +void put_mnt_ns(struct mnt_namespace *ns) +{ + if (!ns_ref_put(ns)) + return; + guard(namespace_excl)(); + emptied_ns = ns; + guard(mount_writer)(); + umount_tree(ns->root, UMOUNT_CONNECTED); +} + +struct vfsmount *kern_mount(struct file_system_type *type) +{ + struct vfsmount *mnt; + mnt = vfs_kern_mount(type, SB_KERNMOUNT, type->name, NULL); + if (!IS_ERR(mnt)) { + /* + * it is a longterm mount, don't release mnt until + * we unmount before file sys is unregistered + */ + real_mount(mnt)->mnt_ns = MNT_NS_INTERNAL; + } + return mnt; +} +EXPORT_SYMBOL_GPL(kern_mount); + +void kern_unmount(struct vfsmount *mnt) +{ + /* release long term mount so mount point can be released */ + if (!IS_ERR(mnt)) { + mnt_make_shortterm(mnt); + synchronize_rcu(); /* yecchhh... */ + mntput(mnt); + } +} +EXPORT_SYMBOL(kern_unmount); + +void kern_unmount_array(struct vfsmount *mnt[], unsigned int num) +{ + unsigned int i; + + for (i = 0; i < num; i++) + mnt_make_shortterm(mnt[i]); + synchronize_rcu_expedited(); + for (i = 0; i < num; i++) + mntput(mnt[i]); +} +EXPORT_SYMBOL(kern_unmount_array); + +bool our_mnt(struct vfsmount *mnt) +{ + return check_mnt(real_mount(mnt)); +} + +bool current_chrooted(void) +{ + /* Does the current process have a non-standard root */ + struct path fs_root __free(path_put) = {}; + struct mount *root; + + get_fs_root(current->fs, &fs_root); + + /* Find the namespace root */ + + guard(mount_locked_reader)(); + + root = topmost_overmount(current->nsproxy->mnt_ns->root); + + return fs_root.mnt != &root->mnt || !path_mounted(&fs_root); +} + +static bool mnt_already_visible(struct mnt_namespace *ns, + const struct super_block *sb, + int *new_mnt_flags) +{ + int new_flags = *new_mnt_flags; + struct mount *mnt; + + /* Don't acquire namespace semaphore without a good reason. */ + if (hlist_empty(&ns->mnt_visible_mounts)) + return false; + + guard(namespace_shared)(); + hlist_for_each_entry(mnt, &ns->mnt_visible_mounts, mnt_ns_visible) { + const struct super_block *sb_visible = mnt->mnt.mnt_sb; + struct mount *child; + int mnt_flags; + + if (sb_visible->s_type != sb->s_type) + continue; + + /* + * Restricted variants are not compatible with anything, even + * other restricted variants. + */ + if (sb_visible->s_iflags & SB_I_RESTRICTED_VARIANT) + continue; + + /* A local view of the mount flags */ + mnt_flags = mnt->mnt.mnt_flags; + + /* Don't miss readonly hidden in the superblock flags */ + if (sb_rdonly(mnt->mnt.mnt_sb)) + mnt_flags |= MNT_LOCK_READONLY; + + /* Verify the mount flags are equal to or more permissive + * than the proposed new mount. + */ + if ((mnt_flags & MNT_LOCK_READONLY) && + !(new_flags & MNT_READONLY)) + continue; + if ((mnt_flags & MNT_LOCK_ATIME) && + ((mnt_flags & MNT_ATIME_MASK) != (new_flags & MNT_ATIME_MASK))) + continue; + + /* This mount is not fully visible if there are any + * locked child mounts that cover anything except for + * empty directories. + */ + list_for_each_entry(child, &mnt->mnt_mounts, mnt_child) { + struct inode *inode = child->mnt_mountpoint->d_inode; + /* Only worry about locked mounts */ + if (!(child->mnt.mnt_flags & MNT_LOCKED)) + continue; + /* Is the directory permanently empty? */ + if (!is_empty_dir_inode(inode)) + goto next; + } + /* Preserve the locked attributes */ + *new_mnt_flags |= mnt_flags & (MNT_LOCK_READONLY | \ + MNT_LOCK_ATIME); + return true; + next: ; + } + return false; +} + +static bool mount_too_revealing(const struct super_block *sb, int *new_mnt_flags) +{ + const unsigned long required_iflags = SB_I_NOEXEC | SB_I_NODEV; + struct mnt_namespace *ns = current->nsproxy->mnt_ns; + unsigned long s_iflags; + + if (ns->user_ns == &init_user_ns) + return false; + + /* Can this filesystem be too revealing? */ + if (!(sb->s_type->fs_flags & FS_USERNS_MOUNT_RESTRICTED)) + return false; + + s_iflags = sb->s_iflags; + if ((s_iflags & required_iflags) != required_iflags) { + WARN_ONCE(1, "Expected s_iflags to contain 0x%lx\n", + required_iflags); + return true; + } + + /* + * Restricted variants don't need an already visible mount because they + * don't expose the full filesystem view. + */ + if (s_iflags & SB_I_RESTRICTED_VARIANT) + return false; + + return !mnt_already_visible(ns, sb, new_mnt_flags); +} + +bool mnt_may_suid(struct vfsmount *mnt) +{ + /* + * Foreign mounts (accessed via fchdir or through /proc + * symlinks) are always treated as if they are nosuid. This + * prevents namespaces from trusting potentially unsafe + * suid/sgid bits, file caps, or security labels that originate + * in other namespaces. + */ + return !(mnt->mnt_flags & MNT_NOSUID) && check_mnt(real_mount(mnt)) && + current_in_userns(mnt->mnt_sb->s_user_ns); +} + +static struct ns_common *mntns_get(struct task_struct *task) +{ + struct ns_common *ns = NULL; + struct nsproxy *nsproxy; + + task_lock(task); + nsproxy = task->nsproxy; + if (nsproxy) { + ns = &nsproxy->mnt_ns->ns; + get_mnt_ns(to_mnt_ns(ns)); + } + task_unlock(task); + + return ns; +} + +static void mntns_put(struct ns_common *ns) +{ + put_mnt_ns(to_mnt_ns(ns)); +} + +static int mntns_install(struct nsset *nsset, struct ns_common *ns) +{ + struct nsproxy *nsproxy = nsset->nsproxy; + struct fs_struct *fs = nsset->fs; + struct mnt_namespace *mnt_ns = to_mnt_ns(ns), *old_mnt_ns; + struct user_namespace *user_ns = nsset->cred->user_ns; + struct path root; + int err; + + if (!ns_capable(mnt_ns->user_ns, CAP_SYS_ADMIN) || + !ns_capable(user_ns, CAP_SYS_CHROOT) || + !ns_capable(user_ns, CAP_SYS_ADMIN)) + return -EPERM; + + if (is_anon_ns(mnt_ns)) + return -EINVAL; + + if (fs->users != 1) + return -EINVAL; + + get_mnt_ns(mnt_ns); + old_mnt_ns = nsproxy->mnt_ns; + nsproxy->mnt_ns = mnt_ns; + + /* Find the root */ + err = vfs_path_lookup(mnt_ns->root->mnt.mnt_root, &mnt_ns->root->mnt, + "/", LOOKUP_DOWN, &root); + if (err) { + /* revert to old namespace */ + nsproxy->mnt_ns = old_mnt_ns; + put_mnt_ns(mnt_ns); + return err; + } + + put_mnt_ns(old_mnt_ns); + + /* Update the pwd and root */ + set_fs_pwd(fs, &root); + set_fs_root(fs, &root); + + path_put(&root); + return 0; +} + +static struct user_namespace *mntns_owner(struct ns_common *ns) +{ + return to_mnt_ns(ns)->user_ns; +} + +const struct proc_ns_operations mntns_operations = { + .name = "mnt", + .get = mntns_get, + .put = mntns_put, + .install = mntns_install, + .owner = mntns_owner, +}; + +#ifdef CONFIG_SYSCTL +static const struct ctl_table fs_namespace_sysctls[] = { + { + .procname = "mount-max", + .data = &sysctl_mount_max, + .maxlen = sizeof(unsigned int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ONE, + }, +}; + +static int __init init_fs_namespace_sysctls(void) +{ + register_sysctl_init("fs", fs_namespace_sysctls); + return 0; +} +fs_initcall(init_fs_namespace_sysctls); + +#endif /* CONFIG_SYSCTL */ diff --git a/fs/netfs/Kconfig b/fs/netfs/Kconfig new file mode 100644 index 000000000..7701c037c --- /dev/null +++ b/fs/netfs/Kconfig @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config NETFS_SUPPORT + tristate + help + This option enables support for network filesystems, including + helpers for high-level buffered I/O, abstracting out read + segmentation, local caching and transparent huge page support. + +config NETFS_STATS + bool "Gather statistical information on local caching" + depends on NETFS_SUPPORT && PROC_FS + help + This option causes statistical information to be gathered on local + caching and exported through file: + + /proc/fs/fscache/stats + + The gathering of statistics adds a certain amount of overhead to + execution as there are a quite a few stats gathered, and on a + multi-CPU system these may be on cachelines that keep bouncing + between CPUs. On the other hand, the stats are very useful for + debugging purposes. Saying 'Y' here is recommended. + +config NETFS_DEBUG + bool "Enable dynamic debugging netfslib and FS-Cache" + depends on NETFS_SUPPORT + help + This permits debugging to be dynamically enabled in the local caching + management module. If this is set, the debugging output may be + enabled by setting bits in /sys/module/netfs/parameters/debug. + +config FSCACHE + bool "General filesystem local caching manager" + depends on NETFS_SUPPORT + help + This option enables a generic filesystem caching manager that can be + used by various network and other filesystems to cache data locally. + Different sorts of caches can be plugged in, depending on the + resources available. + + See Documentation/filesystems/caching/fscache.rst for more information. + +config FSCACHE_STATS + bool "Gather statistical information on local caching" + depends on FSCACHE && PROC_FS + select NETFS_STATS + help + This option causes statistical information to be gathered on local + caching and exported through file: + + /proc/fs/fscache/stats + + The gathering of statistics adds a certain amount of overhead to + execution as there are a quite a few stats gathered, and on a + multi-CPU system these may be on cachelines that keep bouncing + between CPUs. On the other hand, the stats are very useful for + debugging purposes. Saying 'Y' here is recommended. + + See Documentation/filesystems/caching/fscache.rst for more information. diff --git a/fs/netfs/Makefile b/fs/netfs/Makefile new file mode 100644 index 000000000..b43188d64 --- /dev/null +++ b/fs/netfs/Makefile @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: GPL-2.0 + +netfs-y := \ + buffered_read.o \ + buffered_write.o \ + direct_read.o \ + direct_write.o \ + iterator.o \ + locking.o \ + main.o \ + misc.o \ + objects.o \ + read_collect.o \ + read_pgpriv2.o \ + read_retry.o \ + read_single.o \ + rolling_buffer.o \ + write_collect.o \ + write_issue.o \ + write_retry.o + +netfs-$(CONFIG_NETFS_STATS) += stats.o + +netfs-$(CONFIG_FSCACHE) += \ + fscache_cache.o \ + fscache_cookie.o \ + fscache_io.o \ + fscache_main.o \ + fscache_volume.o + +ifeq ($(CONFIG_PROC_FS),y) +netfs-$(CONFIG_FSCACHE) += fscache_proc.o +endif +netfs-$(CONFIG_FSCACHE_STATS) += fscache_stats.o + +obj-$(CONFIG_NETFS_SUPPORT) += netfs.o diff --git a/fs/netfs/buffered_read.c b/fs/netfs/buffered_read.c new file mode 100644 index 000000000..7fdfa4f27 --- /dev/null +++ b/fs/netfs/buffered_read.c @@ -0,0 +1,840 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Network filesystem high-level buffered read support. + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include "internal.h" + +static void netfs_cache_expand_readahead(struct netfs_io_request *rreq, + unsigned long long *_start, + unsigned long long *_len, + unsigned long long i_size) +{ + struct netfs_cache_resources *cres = &rreq->cache_resources; + + if (cres->ops && cres->ops->expand_readahead) + cres->ops->expand_readahead(cres, _start, _len, i_size); +} + +static void netfs_rreq_expand(struct netfs_io_request *rreq, + struct readahead_control *ractl) +{ + /* Give the cache a chance to change the request parameters. The + * resultant request must contain the original region. + */ + netfs_cache_expand_readahead(rreq, &rreq->start, &rreq->len, rreq->i_size); + + /* Give the netfs a chance to change the request parameters. The + * resultant request must contain the original region. + */ + if (rreq->netfs_ops->expand_readahead) + rreq->netfs_ops->expand_readahead(rreq); + + /* Expand the request if the cache wants it to start earlier. Note + * that the expansion may get further extended if the VM wishes to + * insert THPs and the preferred start and/or end wind up in the middle + * of THPs. + * + * If this is the case, however, the THP size should be an integer + * multiple of the cache granule size, so we get a whole number of + * granules to deal with. + */ + if (rreq->start != readahead_pos(ractl) || + rreq->len != readahead_length(ractl)) { + readahead_expand(ractl, rreq->start, rreq->len); + rreq->start = readahead_pos(ractl); + rreq->len = readahead_length(ractl); + + trace_netfs_read(rreq, readahead_pos(ractl), readahead_length(ractl), + netfs_read_trace_expanded); + } +} + +/* + * Begin an operation, and fetch the stored zero point value from the cookie if + * available. + */ +static int netfs_begin_cache_read(struct netfs_io_request *rreq, struct netfs_inode *ctx) +{ + return fscache_begin_read_operation(&rreq->cache_resources, netfs_i_cookie(ctx)); +} + +/* + * netfs_prepare_read_iterator - Prepare the subreq iterator for I/O + * @subreq: The subrequest to be set up + * + * Prepare the I/O iterator representing the read buffer on a subrequest for + * the filesystem to use for I/O (it can be passed directly to a socket). This + * is intended to be called from the ->issue_read() method once the filesystem + * has trimmed the request to the size it wants. + * + * Returns the limited size if successful and -ENOMEM if insufficient memory + * available. + * + * [!] NOTE: This must be run in the same thread as ->issue_read() was called + * in as we access the readahead_control struct. + */ +static ssize_t netfs_prepare_read_iterator(struct netfs_io_subrequest *subreq, + struct readahead_control *ractl) +{ + struct netfs_io_request *rreq = subreq->rreq; + size_t rsize = subreq->len; + + if (subreq->source == NETFS_DOWNLOAD_FROM_SERVER) + rsize = umin(rsize, rreq->io_streams[0].sreq_max_len); + + if (ractl) { + /* If we don't have sufficient folios in the rolling buffer, + * extract a folioq's worth from the readahead region at a time + * into the buffer. Note that this acquires a ref on each page + * that we will need to release later - but we don't want to do + * that until after we've started the I/O. + */ + struct folio_batch put_batch; + + folio_batch_init(&put_batch); + while (rreq->submitted < subreq->start + rsize) { + ssize_t added; + + added = rolling_buffer_load_from_ra(&rreq->buffer, ractl, + &put_batch); + if (added < 0) { + folio_batch_release(&put_batch); + return added; + } + rreq->submitted += added; + } + folio_batch_release(&put_batch); + } + + subreq->len = rsize; + if (unlikely(rreq->io_streams[0].sreq_max_segs)) { + size_t limit = netfs_limit_iter(&rreq->buffer.iter, 0, rsize, + rreq->io_streams[0].sreq_max_segs); + + if (limit < rsize) { + subreq->len = limit; + trace_netfs_sreq(subreq, netfs_sreq_trace_limited); + } + } + + subreq->io_iter = rreq->buffer.iter; + + iov_iter_truncate(&subreq->io_iter, subreq->len); + rolling_buffer_advance(&rreq->buffer, subreq->len); + return subreq->len; +} + +static enum netfs_io_source netfs_cache_prepare_read(struct netfs_io_request *rreq, + struct netfs_io_subrequest *subreq, + loff_t i_size) +{ + struct netfs_cache_resources *cres = &rreq->cache_resources; + enum netfs_io_source source; + + if (!cres->ops) + return NETFS_DOWNLOAD_FROM_SERVER; + source = cres->ops->prepare_read(subreq, i_size); + trace_netfs_sreq(subreq, netfs_sreq_trace_prepare); + return source; + +} + +/* + * Issue a read against the cache. + * - Eats the caller's ref on subreq. + */ +static void netfs_read_cache_to_pagecache(struct netfs_io_request *rreq, + struct netfs_io_subrequest *subreq) +{ + struct netfs_cache_resources *cres = &rreq->cache_resources; + + netfs_stat(&netfs_n_rh_read); + cres->ops->read(cres, subreq->start, &subreq->io_iter, NETFS_READ_HOLE_IGNORE, + netfs_cache_read_terminated, subreq); +} + +void netfs_queue_read(struct netfs_io_request *rreq, + struct netfs_io_subrequest *subreq) +{ + struct netfs_io_stream *stream = &rreq->io_streams[0]; + + __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); + + /* We add to the end of the list whilst the collector may be walking + * the list. The collector only goes nextwards and uses the lock to + * remove entries off of the front. + */ + spin_lock(&rreq->lock); + /* Write IN_PROGRESS before pointer to new subreq */ + list_add_tail_release(&subreq->rreq_link, &stream->subrequests); + if (list_is_first(&subreq->rreq_link, &stream->subrequests)) { + if (!stream->active) { + stream->collected_to = subreq->start; + /* Store list pointers before active flag */ + smp_store_release(&stream->active, true); + } + } + + spin_unlock(&rreq->lock); +} + +static void netfs_issue_read(struct netfs_io_request *rreq, + struct netfs_io_subrequest *subreq) +{ + switch (subreq->source) { + case NETFS_DOWNLOAD_FROM_SERVER: + rreq->netfs_ops->issue_read(subreq); + break; + case NETFS_READ_FROM_CACHE: + netfs_read_cache_to_pagecache(rreq, subreq); + break; + default: + __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags); + subreq->error = 0; + iov_iter_zero(subreq->len, &subreq->io_iter); + subreq->transferred = subreq->len; + netfs_read_subreq_terminated(subreq); + break; + } +} + +/* + * Perform a read to the pagecache from a series of sources of different types, + * slicing up the region to be read according to available cache blocks and + * network rsize. + */ +static void netfs_read_to_pagecache(struct netfs_io_request *rreq, + struct readahead_control *ractl) +{ + unsigned long long start = rreq->start; + ssize_t size = rreq->len; + int ret = 0; + + do { + struct netfs_io_subrequest *subreq; + enum netfs_io_source source = NETFS_SOURCE_UNKNOWN; + ssize_t slice; + + subreq = netfs_alloc_subrequest(rreq); + if (!subreq) { + ret = -ENOMEM; + break; + } + + subreq->start = start; + subreq->len = size; + + netfs_queue_read(rreq, subreq); + + source = netfs_cache_prepare_read(rreq, subreq, rreq->i_size); + subreq->source = source; + if (source == NETFS_DOWNLOAD_FROM_SERVER) { + unsigned long long zero_point = netfs_read_zero_point(rreq->inode); + unsigned long long zp = umin(zero_point, rreq->i_size); + size_t len = subreq->len; + + if (unlikely(rreq->origin == NETFS_READ_SINGLE)) + zp = rreq->i_size; + if (subreq->start >= zp) { + subreq->source = source = NETFS_FILL_WITH_ZEROES; + goto fill_with_zeroes; + } + + if (len > zp - subreq->start) + len = zp - subreq->start; + if (len == 0) { + pr_err("ZERO-LEN READ: R=%08x[%x] l=%zx/%zx s=%llx z=%llx i=%llx", + rreq->debug_id, subreq->debug_index, + subreq->len, size, + subreq->start, zero_point, rreq->i_size); + netfs_cancel_read(subreq, ret); + break; + } + subreq->len = len; + + netfs_stat(&netfs_n_rh_download); + if (rreq->netfs_ops->prepare_read) { + ret = rreq->netfs_ops->prepare_read(subreq); + if (ret < 0) { + netfs_cancel_read(subreq, ret); + break; + } + trace_netfs_sreq(subreq, netfs_sreq_trace_prepare); + } + goto issue; + } + + fill_with_zeroes: + if (source == NETFS_FILL_WITH_ZEROES) { + subreq->source = NETFS_FILL_WITH_ZEROES; + trace_netfs_sreq(subreq, netfs_sreq_trace_submit); + netfs_stat(&netfs_n_rh_zero); + goto issue; + } + + if (source == NETFS_READ_FROM_CACHE) { + trace_netfs_sreq(subreq, netfs_sreq_trace_submit); + goto issue; + } + + pr_err("Unexpected read source %u\n", source); + WARN_ON_ONCE(1); + netfs_cancel_read(subreq, ret); + break; + + issue: + slice = netfs_prepare_read_iterator(subreq, ractl); + if (slice < 0) { + ret = slice; + netfs_cancel_read(subreq, ret); + break; + } + start += slice; + size -= slice; + if (size <= 0) { + smp_wmb(); /* Write lists before ALL_QUEUED. */ + set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); + } + + netfs_issue_read(rreq, subreq); + + if (test_bit(NETFS_RREQ_PAUSE, &rreq->flags)) + netfs_wait_for_paused_read(rreq); + if (test_bit(NETFS_RREQ_FAILED, &rreq->flags)) + break; + cond_resched(); + } while (size > 0); + + if (unlikely(size > 0)) { + smp_wmb(); /* Write lists before ALL_QUEUED. */ + set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); + netfs_wake_collector(rreq); + } + + /* Defer error return as we may need to wait for outstanding I/O. */ + cmpxchg(&rreq->error, 0, ret); +} + +/** + * netfs_readahead - Helper to manage a read request + * @ractl: The description of the readahead request + * + * Fulfil a readahead request by drawing data from the cache if possible, or + * the netfs if not. Space beyond the EOF is zero-filled. Multiple I/O + * requests from different sources will get munged together. If necessary, the + * readahead window can be expanded in either direction to a more convenient + * alighment for RPC efficiency or to make storage in the cache feasible. + * + * The calling netfs must initialise a netfs context contiguous to the vfs + * inode before calling this. + * + * This is usable whether or not caching is enabled. + */ +void netfs_readahead(struct readahead_control *ractl) +{ + struct netfs_io_request *rreq; + struct netfs_inode *ictx = netfs_inode(ractl->mapping->host); + unsigned long long start = readahead_pos(ractl); + size_t size = readahead_length(ractl); + int ret; + + rreq = netfs_alloc_request(ractl->mapping, ractl->file, start, size, + NETFS_READAHEAD); + if (IS_ERR(rreq)) + return; + + __set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &rreq->flags); + + ret = netfs_begin_cache_read(rreq, ictx); + if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS) + goto cleanup_free; + + netfs_stat(&netfs_n_rh_readahead); + trace_netfs_read(rreq, readahead_pos(ractl), readahead_length(ractl), + netfs_read_trace_readahead); + + netfs_rreq_expand(rreq, ractl); + + rreq->submitted = rreq->start; + if (rolling_buffer_init(&rreq->buffer, rreq->debug_id, ITER_DEST, rreq->gfp) < 0) + goto cleanup_free; + netfs_read_to_pagecache(rreq, ractl); + + return netfs_put_request(rreq, netfs_rreq_trace_put_return); + +cleanup_free: + return netfs_put_failed_request(rreq); +} +EXPORT_SYMBOL(netfs_readahead); + +/* + * Create a rolling buffer with a single occupying folio. + */ +static int netfs_create_singular_buffer(struct netfs_io_request *rreq, struct folio *folio, + unsigned int rollbuf_flags) +{ + ssize_t added; + + if (rolling_buffer_init(&rreq->buffer, rreq->debug_id, ITER_DEST, rreq->gfp) < 0) + return -ENOMEM; + + added = rolling_buffer_append(&rreq->buffer, folio, rollbuf_flags, rreq->gfp); + if (added < 0) + return added; + rreq->submitted = rreq->start + added; + return 0; +} + +/* + * Read into gaps in a folio partially filled by a streaming write. + */ +static int netfs_read_gaps(struct file *file, struct folio *folio) +{ + struct netfs_io_request *rreq; + struct address_space *mapping = folio->mapping; + struct netfs_group *group = netfs_folio_group(folio); + struct netfs_folio *finfo = netfs_folio_info(folio); + struct netfs_inode *ctx = netfs_inode(mapping->host); + struct folio *sink = NULL; + struct bio_vec *bvec; + unsigned int from = finfo->dirty_offset; + unsigned int to = from + finfo->dirty_len; + unsigned int off = 0, i = 0; + size_t flen = folio_size(folio); + size_t nr_bvec = flen / PAGE_SIZE + 2; + size_t part; + int ret; + + _enter("%lx", folio->index); + + rreq = netfs_alloc_request(mapping, file, folio_pos(folio), flen, NETFS_READ_GAPS); + if (IS_ERR(rreq)) { + ret = PTR_ERR(rreq); + goto alloc_error; + } + + ret = netfs_begin_cache_read(rreq, ctx); + if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS) + goto discard; + + netfs_stat(&netfs_n_rh_read_folio); + trace_netfs_read(rreq, rreq->start, rreq->len, netfs_read_trace_read_gaps); + + /* Fiddle the buffer so that a gap at the beginning and/or a gap at the + * end get copied to, but the middle is discarded. + */ + ret = -ENOMEM; + bvec = kmalloc_objs(*bvec, nr_bvec); + if (!bvec) + goto discard; + + sink = folio_alloc(GFP_KERNEL, 0); + if (!sink) { + kfree(bvec); + goto discard; + } + + trace_netfs_folio(folio, netfs_folio_trace_read_gaps); + + rreq->direct_bv = bvec; + rreq->direct_bv_count = nr_bvec; + if (from > 0) { + bvec_set_folio(&bvec[i++], folio, from, 0); + off = from; + } + while (off < to) { + part = min_t(size_t, to - off, PAGE_SIZE); + bvec_set_folio(&bvec[i++], sink, part, 0); + off += part; + } + if (to < flen) + bvec_set_folio(&bvec[i++], folio, flen - to, to); + iov_iter_bvec(&rreq->buffer.iter, ITER_DEST, bvec, i, rreq->len); + rreq->submitted = rreq->start + flen; + + netfs_read_to_pagecache(rreq, NULL); + + ret = netfs_wait_for_read(rreq); + if (ret >= 0) { + if (group) + folio_change_private(folio, group); + else + folio_detach_private(folio); + kfree(finfo); + trace_netfs_folio(folio, netfs_folio_trace_filled_gaps); + flush_dcache_folio(folio); + folio_mark_uptodate(folio); + } + + if (sink) + folio_put(sink); + folio_unlock(folio); + netfs_put_request(rreq, netfs_rreq_trace_put_return); + return ret < 0 ? ret : 0; + +discard: + netfs_put_failed_request(rreq); +alloc_error: + folio_unlock(folio); + return ret; +} + +/** + * netfs_read_folio - Helper to manage a read_folio request + * @file: The file to read from + * @folio: The folio to read + * + * Fulfil a read_folio request by drawing data from the cache if + * possible, or the netfs if not. Space beyond the EOF is zero-filled. + * Multiple I/O requests from different sources will get munged together. + * + * The calling netfs must initialise a netfs context contiguous to the vfs + * inode before calling this. + * + * This is usable whether or not caching is enabled. + */ +int netfs_read_folio(struct file *file, struct folio *folio) +{ + struct address_space *mapping = folio->mapping; + struct netfs_io_request *rreq; + struct netfs_inode *ctx = netfs_inode(mapping->host); + int ret; + + folio_wait_writeback(folio); + + if (folio_test_dirty(folio)) + return netfs_read_gaps(file, folio); + + _enter("%lx", folio->index); + + rreq = netfs_alloc_request(mapping, file, + folio_pos(folio), folio_size(folio), + NETFS_READPAGE); + if (IS_ERR(rreq)) { + ret = PTR_ERR(rreq); + goto alloc_error; + } + + ret = netfs_begin_cache_read(rreq, ctx); + if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS) + goto discard; + + netfs_stat(&netfs_n_rh_read_folio); + trace_netfs_read(rreq, rreq->start, rreq->len, netfs_read_trace_readpage); + + /* Set up the output buffer */ + ret = netfs_create_singular_buffer(rreq, folio, 0); + if (ret < 0) + goto discard; + + netfs_read_to_pagecache(rreq, NULL); + ret = netfs_wait_for_read(rreq); + netfs_put_request(rreq, netfs_rreq_trace_put_return); + return ret < 0 ? ret : 0; + +discard: + netfs_put_failed_request(rreq); +alloc_error: + folio_unlock(folio); + return ret; +} +EXPORT_SYMBOL(netfs_read_folio); + +/* + * Prepare a folio for writing without reading first + * @folio: The folio being prepared + * @pos: starting position for the write + * @len: length of write + * @always_fill: T if the folio should always be completely filled/cleared + * + * In some cases, write_begin doesn't need to read at all: + * - full folio write + * - write that lies in a folio that is completely beyond EOF + * - write that covers the folio from start to EOF or beyond it + * + * If any of these criteria are met, then zero out the unwritten parts + * of the folio and return true. Otherwise, return false. + */ +static bool netfs_skip_folio_read(struct folio *folio, loff_t pos, size_t len, + bool always_fill) +{ + struct inode *inode = folio_inode(folio); + loff_t i_size = i_size_read(inode); + size_t offset = offset_in_folio(folio, pos); + size_t plen = folio_size(folio); + + if (unlikely(always_fill)) { + if (pos - offset + len <= i_size) + return false; /* Page entirely before EOF */ + folio_zero_segment(folio, 0, plen); + folio_mark_uptodate(folio); + return true; + } + + /* Full folio write */ + if (offset == 0 && len >= plen) + return true; + + /* Page entirely beyond the end of the file */ + if (pos - offset >= i_size) + goto zero_out; + + /* Write that covers from the start of the folio to EOF or beyond */ + if (offset == 0 && (pos + len) >= i_size) + goto zero_out; + + return false; +zero_out: + folio_zero_segments(folio, 0, offset, offset + len, plen); + return true; +} + +/** + * netfs_write_begin - Helper to prepare for writing [DEPRECATED] + * @ctx: The netfs context + * @file: The file to read from + * @mapping: The mapping to read from + * @pos: File position at which the write will begin + * @len: The length of the write (may extend beyond the end of the folio chosen) + * @_folio: Where to put the resultant folio + * @_fsdata: Place for the netfs to store a cookie + * + * Pre-read data for a write-begin request by drawing data from the cache if + * possible, or the netfs if not. Space beyond the EOF is zero-filled. + * Multiple I/O requests from different sources will get munged together. + * + * The calling netfs must provide a table of operations, only one of which, + * issue_read, is mandatory. + * + * The check_write_begin() operation can be provided to check for and flush + * conflicting writes once the folio is grabbed and locked. It is passed a + * pointer to the fsdata cookie that gets returned to the VM to be passed to + * write_end. It is permitted to sleep. It should return 0 if the request + * should go ahead or it may return an error. It may also unlock and put the + * folio, provided it sets ``*foliop`` to NULL, in which case a return of 0 + * will cause the folio to be re-got and the process to be retried. + * + * The calling netfs must initialise a netfs context contiguous to the vfs + * inode before calling this. + * + * This is usable whether or not caching is enabled. + * + * Note that this should be considered deprecated and netfs_perform_write() + * used instead. + */ +int netfs_write_begin(struct netfs_inode *ctx, + struct file *file, struct address_space *mapping, + loff_t pos, unsigned int len, struct folio **_folio, + void **_fsdata) +{ + struct netfs_io_request *rreq; + struct folio *folio; + pgoff_t index = pos >> PAGE_SHIFT; + int ret; + +retry: + folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN, + mapping_gfp_mask(mapping)); + if (IS_ERR(folio)) + return PTR_ERR(folio); + + if (ctx->ops->check_write_begin) { + /* Allow the netfs (eg. ceph) to flush conflicts. */ + ret = ctx->ops->check_write_begin(file, pos, len, &folio, _fsdata); + if (ret < 0) { + trace_netfs_failure(NULL, NULL, ret, netfs_fail_check_write_begin); + goto error; + } + if (!folio) + goto retry; + } + + if (folio_test_uptodate(folio)) + goto have_folio; + + /* If the folio is beyond the EOF, we want to clear it - unless it's + * within the cache granule containing the EOF, in which case we need + * to preload the granule. + */ + if (!netfs_is_cache_maybe_enabled(ctx) && + netfs_skip_folio_read(folio, pos, len, false)) { + netfs_stat(&netfs_n_rh_write_zskip); + goto have_folio_no_wait; + } + + rreq = netfs_alloc_request(mapping, file, + folio_pos(folio), folio_size(folio), + NETFS_READ_FOR_WRITE); + if (IS_ERR(rreq)) { + ret = PTR_ERR(rreq); + goto error; + } + rreq->no_unlock_folio = folio; + __set_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags); + + ret = netfs_begin_cache_read(rreq, ctx); + if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS) + goto error_put; + + netfs_stat(&netfs_n_rh_write_begin); + trace_netfs_read(rreq, pos, len, netfs_read_trace_write_begin); + + /* Set up the output buffer */ + ret = netfs_create_singular_buffer(rreq, folio, 0); + if (ret < 0) + goto error_put; + + netfs_read_to_pagecache(rreq, NULL); + ret = netfs_wait_for_read(rreq); + netfs_put_request(rreq, netfs_rreq_trace_put_return); + if (ret < 0) + goto error; + +have_folio: + ret = folio_wait_private_2_killable(folio); + if (ret < 0) + goto error; +have_folio_no_wait: + *_folio = folio; + _leave(" = 0"); + return 0; + +error_put: + netfs_put_failed_request(rreq); +error: + if (folio) { + folio_unlock(folio); + folio_put(folio); + } + _leave(" = %d", ret); + return ret; +} +EXPORT_SYMBOL(netfs_write_begin); + +/* + * Preload the data into a folio we're proposing to write into. + */ +int netfs_prefetch_for_write(struct file *file, struct folio *folio, + size_t offset, size_t len) +{ + struct netfs_io_request *rreq; + struct address_space *mapping = folio->mapping; + struct netfs_inode *ctx = netfs_inode(mapping->host); + unsigned long long start = folio_pos(folio); + size_t flen = folio_size(folio); + int ret; + + _enter("%zx @%llx", flen, start); + + ret = -ENOMEM; + + rreq = netfs_alloc_request(mapping, file, start, flen, + NETFS_READ_FOR_WRITE); + if (IS_ERR(rreq)) { + ret = PTR_ERR(rreq); + goto error; + } + + rreq->no_unlock_folio = folio; + __set_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags); + ret = netfs_begin_cache_read(rreq, ctx); + if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS) + goto error_put; + + netfs_stat(&netfs_n_rh_write_begin); + trace_netfs_read(rreq, start, flen, netfs_read_trace_prefetch_for_write); + + /* Set up the output buffer */ + ret = netfs_create_singular_buffer(rreq, folio, NETFS_ROLLBUF_PAGECACHE_MARK); + if (ret < 0) + goto error_put; + + netfs_read_to_pagecache(rreq, NULL); + ret = netfs_wait_for_read(rreq); + netfs_put_request(rreq, netfs_rreq_trace_put_return); + return ret < 0 ? ret : 0; + +error_put: + netfs_put_failed_request(rreq); +error: + _leave(" = %d", ret); + return ret; +} + +/** + * netfs_buffered_read_iter - Filesystem buffered I/O read routine + * @iocb: kernel I/O control block + * @iter: destination for the data read + * + * This is the ->read_iter() routine for all filesystems that can use the page + * cache directly. + * + * The IOCB_NOWAIT flag in iocb->ki_flags indicates that -EAGAIN shall be + * returned when no data can be read without waiting for I/O requests to + * complete; it doesn't prevent readahead. + * + * The IOCB_NOIO flag in iocb->ki_flags indicates that no new I/O requests + * shall be made for the read or for readahead. When no data can be read, + * -EAGAIN shall be returned. When readahead would be triggered, a partial, + * possibly empty read shall be returned. + * + * Return: + * * number of bytes copied, even for partial reads + * * negative error code (or 0 if IOCB_NOIO) if nothing was read + */ +ssize_t netfs_buffered_read_iter(struct kiocb *iocb, struct iov_iter *iter) +{ + struct inode *inode = file_inode(iocb->ki_filp); + struct netfs_inode *ictx = netfs_inode(inode); + ssize_t ret; + + if (WARN_ON_ONCE((iocb->ki_flags & IOCB_DIRECT) || + test_bit(NETFS_ICTX_UNBUFFERED, &ictx->flags))) + return -EINVAL; + + ret = netfs_start_io_read(inode); + if (ret == 0) { + ret = filemap_read(iocb, iter, 0); + netfs_end_io_read(inode); + } + return ret; +} +EXPORT_SYMBOL(netfs_buffered_read_iter); + +/** + * netfs_file_read_iter - Generic filesystem read routine + * @iocb: kernel I/O control block + * @iter: destination for the data read + * + * This is the ->read_iter() routine for all filesystems that can use the page + * cache directly. + * + * The IOCB_NOWAIT flag in iocb->ki_flags indicates that -EAGAIN shall be + * returned when no data can be read without waiting for I/O requests to + * complete; it doesn't prevent readahead. + * + * The IOCB_NOIO flag in iocb->ki_flags indicates that no new I/O requests + * shall be made for the read or for readahead. When no data can be read, + * -EAGAIN shall be returned. When readahead would be triggered, a partial, + * possibly empty read shall be returned. + * + * Return: + * * number of bytes copied, even for partial reads + * * negative error code (or 0 if IOCB_NOIO) if nothing was read + */ +ssize_t netfs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) +{ + struct netfs_inode *ictx = netfs_inode(iocb->ki_filp->f_mapping->host); + + if ((iocb->ki_flags & IOCB_DIRECT) || + test_bit(NETFS_ICTX_UNBUFFERED, &ictx->flags)) + return netfs_unbuffered_read_iter(iocb, iter); + + return netfs_buffered_read_iter(iocb, iter); +} +EXPORT_SYMBOL(netfs_file_read_iter); diff --git a/fs/netfs/buffered_write.c b/fs/netfs/buffered_write.c new file mode 100644 index 000000000..2cdb68e6b --- /dev/null +++ b/fs/netfs/buffered_write.c @@ -0,0 +1,616 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Network filesystem high-level buffered write support. + * + * Copyright (C) 2023 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include "internal.h" + +/* + * Grab a folio for writing and lock it. Attempt to allocate as large a folio + * as possible to hold as much of the remaining length as possible in one go. + */ +static struct folio *netfs_grab_folio_for_write(struct address_space *mapping, + loff_t pos, size_t part) +{ + pgoff_t index = pos / PAGE_SIZE; + fgf_t fgp_flags = FGP_WRITEBEGIN; + + if (mapping_large_folio_support(mapping)) + fgp_flags |= fgf_set_order(pos % PAGE_SIZE + part); + + return __filemap_get_folio(mapping, index, fgp_flags, + mapping_gfp_mask(mapping)); +} + +/* + * Update i_size and estimate the update to i_blocks to reflect the additional + * data written into the pagecache until we can find out from the server what + * the values actually are. + */ +void netfs_update_i_size(struct netfs_inode *ctx, struct inode *inode, + loff_t pos, size_t copied) +{ + loff_t i_size, end = pos + copied; + blkcnt_t add; + size_t gap; + + if (end <= i_size_read(inode)) + return; + + if (ctx->ops->update_i_size) { + ctx->ops->update_i_size(inode, end); + return; + } + + spin_lock(&inode->i_lock); + + i_size = i_size_read(inode); + if (end > i_size) { + i_size_write(inode, end); +#if IS_ENABLED(CONFIG_FSCACHE) + fscache_update_cookie(ctx->cache, NULL, &end); +#endif + + gap = SECTOR_SIZE - (i_size & (SECTOR_SIZE - 1)); + if (copied > gap) { + add = DIV_ROUND_UP(copied - gap, SECTOR_SIZE); + + inode->i_blocks = min_t(blkcnt_t, + DIV_ROUND_UP(end, SECTOR_SIZE), + inode->i_blocks + add); + } + } + spin_unlock(&inode->i_lock); +} + +/** + * netfs_perform_write - Copy data into the pagecache. + * @iocb: The operation parameters + * @iter: The source buffer + * @netfs_group: Grouping for dirty folios (eg. ceph snaps). + * + * Copy data into pagecache folios attached to the inode specified by @iocb. + * The caller must hold appropriate inode locks. + * + * Dirty folios are tagged with a netfs_folio struct if they're not up to date + * to indicate the range modified. Dirty folios may also be tagged with a + * netfs-specific grouping such that data from an old group gets flushed before + * a new one is started. + */ +ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter, + struct netfs_group *netfs_group) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct address_space *mapping = inode->i_mapping; + struct netfs_inode *ctx = netfs_inode(inode); + struct writeback_control wbc = { + .sync_mode = WB_SYNC_NONE, + .for_sync = true, + .nr_to_write = LONG_MAX, + .range_start = iocb->ki_pos, + .range_end = iocb->ki_pos + iter->count, + }; + struct netfs_io_request *wreq = NULL; + struct folio *folio = NULL, *writethrough = NULL; + unsigned int bdp_flags = (iocb->ki_flags & IOCB_NOWAIT) ? BDP_ASYNC : 0; + ssize_t written = 0, ret, ret2; + loff_t pos = iocb->ki_pos; + size_t max_chunk = mapping_max_folio_size(mapping); + bool maybe_trouble = false; + + if (unlikely(iocb->ki_flags & (IOCB_DSYNC | IOCB_SYNC)) + ) { + wbc_attach_fdatawrite_inode(&wbc, mapping->host); + + ret = filemap_write_and_wait_range(mapping, pos, pos + iter->count); + if (ret < 0) { + wbc_detach_inode(&wbc); + goto out; + } + + wreq = netfs_begin_writethrough(iocb, iter->count); + if (IS_ERR(wreq)) { + wbc_detach_inode(&wbc); + ret = PTR_ERR(wreq); + wreq = NULL; + goto out; + } + if (!is_sync_kiocb(iocb)) + wreq->iocb = iocb; + netfs_stat(&netfs_n_wh_writethrough); + } else { + netfs_stat(&netfs_n_wh_buffered_write); + } + + do { + enum netfs_folio_trace trace; + struct netfs_folio *finfo; + struct netfs_group *group; + unsigned long long fpos; + size_t flen; + size_t offset; /* Offset into pagecache folio */ + size_t part; /* Bytes to write to folio */ + size_t copied; /* Bytes copied from user */ + void *priv; + + offset = pos & (max_chunk - 1); + part = min(max_chunk - offset, iov_iter_count(iter)); + + /* Bring in the user pages that we will copy from _first_ lest + * we hit a nasty deadlock on copying from the same page as + * we're writing to, without it being marked uptodate. + * + * Not only is this an optimisation, but it is also required to + * check that the address is actually valid, when atomic + * usercopies are used below. + * + * We rely on the page being held onto long enough by the LRU + * that we can grab it below if this causes it to be read. + */ + ret = -EFAULT; + if (unlikely(fault_in_iov_iter_readable(iter, part) == part)) + break; + + folio = netfs_grab_folio_for_write(mapping, pos, part); + if (IS_ERR(folio)) { + ret = PTR_ERR(folio); + break; + } + + flen = folio_size(folio); + fpos = folio_pos(folio); + offset = pos - fpos; + part = min_t(size_t, flen - offset, part); + + /* Wait for writeback to complete. The writeback engine owns + * the info in folio->private and may change it until it + * removes the WB mark. + */ + if (folio_get_private(folio) && + folio_wait_writeback_killable(folio)) { + ret = written ? -EINTR : -ERESTARTSYS; + goto error_folio_unlock; + } + + if (signal_pending(current)) { + ret = written ? -EINTR : -ERESTARTSYS; + goto error_folio_unlock; + } + + finfo = netfs_folio_info(folio); + group = netfs_folio_group(folio); + + /* If the requested group differs from the group set on the + * page, then we need to flush out the folio if it has a group + * set (ie. is non-NULL). Note that COPY_TO_CACHE is a special + * case, being a netfs annotation rather than an actual group. + * + * The filesystem isn't permitted to mix writes with groups and + * writes without groups as the NULL group is used to indicate + * that no group is set. + */ + if (unlikely(group != netfs_group) && + group != NETFS_FOLIO_COPY_TO_CACHE && + group) { + WARN_ON_ONCE(!netfs_group); + goto flush_content; + } + + /* Decide how we should modify a folio. We might be attempting + * to do write-streaming, as we don't want to a local RMW cycle + * if we can avoid it. If we're doing local caching or content + * crypto, we award that priority over avoiding RMW. If the + * file is open readably, then we let ->read_folio() fill in + * the gaps. + */ + if (folio_test_uptodate(folio)) { + if (mapping_writably_mapped(mapping)) + flush_dcache_folio(folio); + copied = copy_folio_from_iter_atomic(folio, offset, part, iter); + if (unlikely(copied == 0)) + goto copy_failed; + trace = netfs_folio_is_uptodate; + goto copied_uptodate; + } + + /* If the page is above the zero-point then we assume that the + * server would just return a block of zeros or a short read if + * we try to read it. + */ + if (fpos >= netfs_read_zero_point(inode)) { + folio_zero_segment(folio, 0, offset); + copied = copy_folio_from_iter_atomic(folio, offset, part, iter); + if (unlikely(copied == 0)) + goto copy_failed; + folio_zero_segment(folio, offset + copied, flen); + if (finfo) + trace = netfs_modify_and_clear_rm_finfo; + else + trace = netfs_modify_and_clear; + goto mark_uptodate; + } + + /* See if we can write a whole folio in one go. */ + if (!maybe_trouble && offset == 0 && part >= flen) { + copied = copy_folio_from_iter_atomic(folio, offset, part, iter); + if (likely(copied == part)) { + if (finfo) + trace = netfs_whole_folio_modify_filled; + else + trace = netfs_whole_folio_modify; + goto mark_uptodate; + } + if (copied == 0) + goto copy_failed; + if (!finfo || copied <= finfo->dirty_offset) { + maybe_trouble = true; + iov_iter_revert(iter, copied); + copied = 0; + folio_unlock(folio); + goto retry; + } + + /* We overwrote some existing dirty data, so we have to + * accept the partial write. + */ + finfo->dirty_len += finfo->dirty_offset; + if (finfo->dirty_len == flen) { + trace = netfs_whole_folio_modify_filled_efault; + goto mark_uptodate; + } + if (copied > finfo->dirty_len) + finfo->dirty_len = copied; + finfo->dirty_offset = 0; + trace = netfs_whole_folio_modify_efault; + goto copied; + } + + /* We don't want to do a streaming write on a file that loses + * caching service temporarily because the backing store got + * culled. + */ + if (netfs_is_cache_maybe_enabled(ctx)) { + if (finfo) { + netfs_stat(&netfs_n_wh_wstream_conflict); + goto flush_content; + } + ret = netfs_prefetch_for_write(file, folio, offset, part); + if (ret < 0) { + _debug("prefetch = %zd", ret); + goto error_folio_unlock; + } + /* Note that copy-to-cache may have been set. */ + + copied = copy_folio_from_iter_atomic(folio, offset, part, iter); + if (unlikely(copied == 0)) + goto copy_failed; + trace = netfs_just_prefetch; + goto copied_uptodate; + } + + /* Do a streaming write on a folio that has nothing in it yet. */ + if (!finfo) { + ret = -EIO; + if (WARN_ON(folio_get_private(folio))) + goto error_folio_unlock; + copied = copy_folio_from_iter_atomic(folio, offset, part, iter); + if (unlikely(copied == 0)) + goto copy_failed; + if (offset == 0 && copied == flen) { + trace = netfs_streaming_filled_page; + goto mark_uptodate; + } + + finfo = kzalloc_obj(*finfo); + if (!finfo) { + iov_iter_revert(iter, copied); + ret = -ENOMEM; + goto error_folio_unlock; + } + finfo->netfs_group = netfs_get_group(netfs_group); + finfo->dirty_offset = offset; + finfo->dirty_len = copied; + folio_attach_private(folio, (void *)((unsigned long)finfo | + NETFS_FOLIO_INFO)); + trace = netfs_streaming_write; + goto copied; + } + + /* We can continue a streaming write only if it continues on + * from the previous. If it overlaps, we must flush lest we + * suffer a partial copy and disjoint dirty regions. + */ + if (offset == finfo->dirty_offset + finfo->dirty_len) { + copied = copy_folio_from_iter_atomic(folio, offset, part, iter); + if (unlikely(copied == 0)) + goto copy_failed; + finfo->dirty_len += copied; + if (finfo->dirty_offset == 0 && finfo->dirty_len == flen) { + trace = netfs_streaming_cont_filled_page; + goto mark_uptodate; + } + trace = netfs_streaming_write_cont; + goto copied; + } + + /* Incompatible write; flush the folio and try again. */ + flush_content: + trace_netfs_folio(folio, netfs_flush_content); + folio_unlock(folio); + folio_put(folio); + ret = filemap_write_and_wait_range(mapping, fpos, fpos + flen - 1); + if (ret < 0) + goto out; + continue; + + /* Mark a folio as being up to data when we've filled it + * completely. If the folio has a group attached, then it must + * be the same group, otherwise we should have flushed it out + * above. We have to get rid of the netfs_folio struct if + * there was one. + */ + mark_uptodate: + folio_mark_uptodate(folio); + + copied_uptodate: + priv = folio_get_private(folio); + if (likely(priv == netfs_group)) { + /* Already set correctly; no change required. */ + } else if (priv == NETFS_FOLIO_COPY_TO_CACHE) { + if (!netfs_group) + folio_detach_private(folio); + else + folio_change_private(folio, netfs_get_group(netfs_group)); + } else if (!priv) { + folio_attach_private(folio, netfs_get_group(netfs_group)); + } else { + WARN_ON_ONCE(!finfo); + if (netfs_group) + /* finfo->netfs_group has a ref */ + folio_change_private(folio, netfs_group); + else + folio_detach_private(folio); + kfree(finfo); + } + + copied: + trace_netfs_folio(folio, trace); + flush_dcache_folio(folio); + + /* Update the inode size if we moved the EOF marker */ + netfs_update_i_size(ctx, inode, pos, copied); + pos += copied; + written += copied; + + if (likely(!wreq)) { + folio_mark_dirty(folio); + folio_unlock(folio); + } else { + netfs_advance_writethrough(wreq, &wbc, folio, copied, + offset + copied == flen, + &writethrough); + /* Folio unlocked */ + } + retry: + folio_put(folio); + folio = NULL; + + ret = balance_dirty_pages_ratelimited_flags(mapping, bdp_flags); + if (unlikely(ret < 0)) + break; + + cond_resched(); + } while (iov_iter_count(iter)); + +out: + if (likely(written)) { + /* Set indication that ctime and mtime got updated in case + * close is deferred. + */ + set_bit(NETFS_ICTX_MODIFIED_ATTR, &ctx->flags); + if (unlikely(ctx->ops->post_modify)) + ctx->ops->post_modify(inode); + } + + if (unlikely(wreq)) { + ret2 = netfs_end_writethrough(wreq, &wbc, writethrough); + wbc_detach_inode(&wbc); + if (ret2 == -EIOCBQUEUED) + return ret2; + if (ret == 0 && ret2 < 0) + ret = ret2; + } + + iocb->ki_pos += written; + _leave(" = %zd [%zd]", written, ret); + return written ? written : ret; + +copy_failed: + ret = -EFAULT; +error_folio_unlock: + folio_unlock(folio); + folio_put(folio); + goto out; +} +EXPORT_SYMBOL(netfs_perform_write); + +/** + * netfs_buffered_write_iter_locked - write data to a file + * @iocb: IO state structure (file, offset, etc.) + * @from: iov_iter with data to write + * @netfs_group: Grouping for dirty folios (eg. ceph snaps). + * + * This function does all the work needed for actually writing data to a + * file. It does all basic checks, removes SUID from the file, updates + * modification times and calls proper subroutines depending on whether we + * do direct IO or a standard buffered write. + * + * The caller must hold appropriate locks around this function and have called + * generic_write_checks() already. The caller is also responsible for doing + * any necessary syncing afterwards. + * + * This function does *not* take care of syncing data in case of O_SYNC write. + * A caller has to handle it. This is mainly due to the fact that we want to + * avoid syncing under i_rwsem. + * + * Return: + * * number of bytes written, even for truncated writes + * * negative error code if no data has been written at all + */ +ssize_t netfs_buffered_write_iter_locked(struct kiocb *iocb, struct iov_iter *from, + struct netfs_group *netfs_group) +{ + struct file *file = iocb->ki_filp; + ssize_t ret; + + trace_netfs_write_iter(iocb, from); + + ret = file_remove_privs(file); + if (ret) + return ret; + + ret = file_update_time(file); + if (ret) + return ret; + + return netfs_perform_write(iocb, from, netfs_group); +} +EXPORT_SYMBOL(netfs_buffered_write_iter_locked); + +/** + * netfs_file_write_iter - write data to a file + * @iocb: IO state structure + * @from: iov_iter with data to write + * + * Perform a write to a file, writing into the pagecache if possible and doing + * an unbuffered write instead if not. + * + * Return: + * * Negative error code if no data has been written at all of + * vfs_fsync_range() failed for a synchronous write + * * Number of bytes written, even for truncated writes + */ +ssize_t netfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file->f_mapping->host; + struct netfs_inode *ictx = netfs_inode(inode); + ssize_t ret; + + _enter("%llx,%zx,%llx", iocb->ki_pos, iov_iter_count(from), i_size_read(inode)); + + if (!iov_iter_count(from)) + return 0; + + if ((iocb->ki_flags & IOCB_DIRECT) || + test_bit(NETFS_ICTX_UNBUFFERED, &ictx->flags)) + return netfs_unbuffered_write_iter(iocb, from); + + ret = netfs_start_io_write(inode); + if (ret < 0) + return ret; + + ret = generic_write_checks(iocb, from); + if (ret > 0) + ret = netfs_buffered_write_iter_locked(iocb, from, NULL); + netfs_end_io_write(inode); + if (ret > 0) + ret = generic_write_sync(iocb, ret); + return ret; +} +EXPORT_SYMBOL(netfs_file_write_iter); + +/* + * Notification that a previously read-only page is about to become writable. + * The caller indicates the precise page that needs to be written to, but + * we only track group on a per-folio basis, so we block more often than + * we might otherwise. + */ +vm_fault_t netfs_page_mkwrite(struct vm_fault *vmf, struct netfs_group *netfs_group) +{ + struct netfs_group *group; + struct folio *folio = page_folio(vmf->page); + struct file *file = vmf->vma->vm_file; + struct address_space *mapping = file->f_mapping; + struct inode *inode = file_inode(file); + struct netfs_inode *ictx = netfs_inode(inode); + vm_fault_t ret = VM_FAULT_NOPAGE; + void *priv; + int err; + + _enter("%lx", folio->index); + + sb_start_pagefault(inode->i_sb); + + if (folio_lock_killable(folio) < 0) + goto out; + if (folio->mapping != mapping) + goto unlock; + if (folio_wait_writeback_killable(folio) < 0) + goto unlock; + + /* Can we see a streaming write here? */ + if (WARN_ON(!folio_test_uptodate(folio))) { + ret = VM_FAULT_SIGBUS; + goto unlock; + } + + group = netfs_folio_group(folio); + if (group && + group != netfs_group && + group != NETFS_FOLIO_COPY_TO_CACHE) { + folio_unlock(folio); + err = filemap_fdatawrite_range(mapping, + folio_pos(folio), + folio_next_pos(folio)); + switch (err) { + case 0: + ret = VM_FAULT_RETRY; + goto out; + case -ENOMEM: + ret = VM_FAULT_OOM; + goto out; + default: + ret = VM_FAULT_SIGBUS; + goto out; + } + } + + if (folio_test_dirty(folio)) + trace_netfs_folio(folio, netfs_folio_trace_mkwrite_plus); + else + trace_netfs_folio(folio, netfs_folio_trace_mkwrite); + + priv = folio_get_private(folio); + if (priv != netfs_group) { + if (!netfs_group && priv == NETFS_FOLIO_COPY_TO_CACHE) + folio_detach_private(folio); + else if (netfs_group && priv == NETFS_FOLIO_COPY_TO_CACHE) + folio_change_private(folio, netfs_get_group(netfs_group)); + else if (netfs_group && !priv) + folio_attach_private(folio, netfs_get_group(netfs_group)); + else + WARN_ON_ONCE(1); + } + + file_update_time(file); + set_bit(NETFS_ICTX_MODIFIED_ATTR, &ictx->flags); + if (ictx->ops->post_modify) + ictx->ops->post_modify(inode); + ret = VM_FAULT_LOCKED; +out: + sb_end_pagefault(inode->i_sb); + return ret; +unlock: + folio_unlock(folio); + goto out; +} +EXPORT_SYMBOL(netfs_page_mkwrite); diff --git a/fs/netfs/direct_read.c b/fs/netfs/direct_read.c new file mode 100644 index 000000000..6a8fb0d55 --- /dev/null +++ b/fs/netfs/direct_read.c @@ -0,0 +1,255 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Direct I/O support. + * + * Copyright (C) 2023 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +static void netfs_prepare_dio_read_iterator(struct netfs_io_subrequest *subreq) +{ + struct netfs_io_request *rreq = subreq->rreq; + size_t rsize; + + rsize = umin(subreq->len, rreq->io_streams[0].sreq_max_len); + subreq->len = rsize; + + if (unlikely(rreq->io_streams[0].sreq_max_segs)) { + size_t limit = netfs_limit_iter(&rreq->buffer.iter, 0, rsize, + rreq->io_streams[0].sreq_max_segs); + + if (limit < rsize) { + subreq->len = limit; + trace_netfs_sreq(subreq, netfs_sreq_trace_limited); + } + } + + trace_netfs_sreq(subreq, netfs_sreq_trace_prepare); + + subreq->io_iter = rreq->buffer.iter; + iov_iter_truncate(&subreq->io_iter, subreq->len); + iov_iter_advance(&rreq->buffer.iter, subreq->len); +} + +/* + * Perform a read to a buffer from the server, slicing up the region to be read + * according to the network rsize. + */ +static void netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq) +{ + unsigned long long start = rreq->start; + ssize_t size = rreq->len; + int ret; + + do { + struct netfs_io_subrequest *subreq; + ssize_t slice; + + subreq = netfs_alloc_subrequest(rreq); + if (!subreq) { + /* Stash the error in the request if there's not + * already an error set. + */ + cmpxchg(&rreq->error, 0, -ENOMEM); + break; + } + + subreq->source = NETFS_DOWNLOAD_FROM_SERVER; + subreq->start = start; + subreq->len = size; + + netfs_queue_read(rreq, subreq); + + netfs_stat(&netfs_n_rh_download); + if (rreq->netfs_ops->prepare_read) { + ret = rreq->netfs_ops->prepare_read(subreq); + if (ret < 0) { + netfs_cancel_read(subreq, ret); + break; + } + } + + netfs_prepare_dio_read_iterator(subreq); + slice = subreq->len; + size -= slice; + start += slice; + rreq->submitted += slice; + if (size <= 0) { + smp_wmb(); /* Write lists before ALL_QUEUED. */ + set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); + } + + rreq->netfs_ops->issue_read(subreq); + + if (test_bit(NETFS_RREQ_PAUSE, &rreq->flags)) + netfs_wait_for_paused_read(rreq); + if (test_bit(NETFS_RREQ_FAILED, &rreq->flags)) + break; + cond_resched(); + } while (size > 0); + + if (unlikely(size > 0)) { + smp_wmb(); /* Write lists before ALL_QUEUED. */ + set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); + netfs_wake_collector(rreq); + } +} + +/* + * Perform a read to an application buffer, bypassing the pagecache and the + * local disk cache. + */ +static ssize_t netfs_unbuffered_read(struct netfs_io_request *rreq, bool sync) +{ + ssize_t ret; + + _enter("R=%x %llx-%llx", + rreq->debug_id, rreq->start, rreq->start + rreq->len - 1); + + if (rreq->len == 0) { + pr_err("Zero-sized read [R=%x]\n", rreq->debug_id); + netfs_put_request(rreq, netfs_rreq_trace_put_discard); + return -EIO; + } + + // TODO: Use bounce buffer if requested + + inode_dio_begin(rreq->inode); + netfs_dispatch_unbuffered_reads(rreq); + + /* The collector will get run, even if we don't manage to submit any + * subreqs, so we shouldn't call inode_dio_end() here. + */ + + if (sync) + ret = netfs_wait_for_read(rreq); + else + ret = -EIOCBQUEUED; + + _leave(" = %zd", ret); + return ret; +} + +/** + * netfs_unbuffered_read_iter_locked - Perform an unbuffered or direct I/O read + * @iocb: The I/O control descriptor describing the read + * @iter: The output buffer (also specifies read length) + * + * Perform an unbuffered I/O or direct I/O from the file in @iocb to the + * output buffer. No use is made of the pagecache. + * + * The caller must hold any appropriate locks. + */ +ssize_t netfs_unbuffered_read_iter_locked(struct kiocb *iocb, struct iov_iter *iter) +{ + struct netfs_io_request *rreq; + ssize_t ret; + size_t orig_count = iov_iter_count(iter); + bool sync = is_sync_kiocb(iocb); + + _enter(""); + + if (!orig_count) + return 0; /* Don't update atime */ + + ret = kiocb_write_and_wait(iocb, orig_count); + if (ret < 0) + return ret; + file_accessed(iocb->ki_filp); + + rreq = netfs_alloc_request(iocb->ki_filp->f_mapping, iocb->ki_filp, + iocb->ki_pos, orig_count, + iocb->ki_flags & IOCB_DIRECT ? + NETFS_DIO_READ : NETFS_UNBUFFERED_READ); + if (IS_ERR(rreq)) + return PTR_ERR(rreq); + + netfs_stat(&netfs_n_rh_dio_read); + trace_netfs_read(rreq, rreq->start, rreq->len, netfs_read_trace_dio_read); + + /* If this is an async op, we have to keep track of the destination + * buffer for ourselves as the caller's iterator will be trashed when + * we return. + * + * In such a case, extract an iterator to represent as much of the the + * output buffer as we can manage. Note that the extraction might not + * be able to allocate a sufficiently large bvec array and may shorten + * the request. + */ + if (user_backed_iter(iter)) { + ret = netfs_extract_user_iter(iter, rreq->len, &rreq->buffer.iter, 0); + if (ret < 0) + goto error_put; + rreq->direct_bv = (struct bio_vec *)rreq->buffer.iter.bvec; + rreq->direct_bv_count = ret; + rreq->direct_bv_unpin = iov_iter_extract_will_pin(iter); + rreq->len = iov_iter_count(&rreq->buffer.iter); + } else { + rreq->buffer.iter = *iter; + rreq->len = orig_count; + rreq->direct_bv_unpin = false; + iov_iter_advance(iter, orig_count); + } + + // TODO: Set up bounce buffer if needed + + if (!sync) { + rreq->iocb = iocb; + __set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &rreq->flags); + } + + ret = netfs_unbuffered_read(rreq, sync); + if (ret < 0) + goto out; /* May be -EIOCBQUEUED */ + if (sync) { + // TODO: Copy from bounce buffer + iocb->ki_pos += rreq->transferred; + ret = rreq->transferred; + } + +out: + netfs_put_request(rreq, netfs_rreq_trace_put_return); + if (ret > 0) + orig_count -= ret; + return ret; + +error_put: + netfs_put_failed_request(rreq); + return ret; +} +EXPORT_SYMBOL(netfs_unbuffered_read_iter_locked); + +/** + * netfs_unbuffered_read_iter - Perform an unbuffered or direct I/O read + * @iocb: The I/O control descriptor describing the read + * @iter: The output buffer (also specifies read length) + * + * Perform an unbuffered I/O or direct I/O from the file in @iocb to the + * output buffer. No use is made of the pagecache. + */ +ssize_t netfs_unbuffered_read_iter(struct kiocb *iocb, struct iov_iter *iter) +{ + struct inode *inode = file_inode(iocb->ki_filp); + ssize_t ret; + + if (!iter->count) + return 0; /* Don't update atime */ + + ret = netfs_start_io_direct(inode); + if (ret == 0) { + ret = netfs_unbuffered_read_iter_locked(iocb, iter); + netfs_end_io_direct(inode); + } + return ret; +} +EXPORT_SYMBOL(netfs_unbuffered_read_iter); diff --git a/fs/netfs/direct_write.c b/fs/netfs/direct_write.c new file mode 100644 index 000000000..c16fbad28 --- /dev/null +++ b/fs/netfs/direct_write.c @@ -0,0 +1,387 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Unbuffered and direct write support. + * + * Copyright (C) 2023 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include "internal.h" + +/* + * Perform the cleanup rituals after an unbuffered write is complete. + */ +static void netfs_unbuffered_write_done(struct netfs_io_request *wreq) +{ + struct netfs_inode *ictx = netfs_inode(wreq->inode); + + _enter("R=%x", wreq->debug_id); + + /* Okay, declare that all I/O is complete. */ + trace_netfs_rreq(wreq, netfs_rreq_trace_write_done); + + if (!wreq->error) + netfs_update_i_size(ictx, &ictx->inode, wreq->start, wreq->transferred); + + if (wreq->origin == NETFS_DIO_WRITE && + wreq->mapping->nrpages) { + /* mmap may have got underfoot and we may now have folios + * locally covering the region we just wrote. Attempt to + * discard the folios, but leave in place any modified locally. + * ->write_iter() is prevented from interfering by the DIO + * counter. + */ + pgoff_t first = wreq->start >> PAGE_SHIFT; + pgoff_t last = (wreq->start + wreq->transferred - 1) >> PAGE_SHIFT; + + invalidate_inode_pages2_range(wreq->mapping, first, last); + } + + if (wreq->origin == NETFS_DIO_WRITE) + inode_dio_end(wreq->inode); + + _debug("finished"); + netfs_wake_rreq_flag(wreq, NETFS_RREQ_IN_PROGRESS, netfs_rreq_trace_wake_ip); + /* As we cleared NETFS_RREQ_IN_PROGRESS, we acquired its ref. */ + + if (wreq->iocb) { + size_t written = umin(wreq->transferred, wreq->len); + + wreq->iocb->ki_pos += written; + if (wreq->iocb->ki_complete) { + trace_netfs_rreq(wreq, netfs_rreq_trace_ki_complete); + wreq->iocb->ki_complete(wreq->iocb, wreq->error ?: written); + } + wreq->iocb = VFS_PTR_POISON; + } + + netfs_clear_subrequests(wreq); +} + +/* + * Collect the subrequest results of unbuffered write subrequests. + */ +static void netfs_unbuffered_write_collect(struct netfs_io_request *wreq, + struct netfs_io_stream *stream, + struct netfs_io_subrequest *subreq) +{ + trace_netfs_collect_sreq(wreq, subreq); + + spin_lock(&wreq->lock); + list_del_init(&subreq->rreq_link); + spin_unlock(&wreq->lock); + + wreq->transferred += subreq->transferred; + iov_iter_advance(&wreq->buffer.iter, subreq->transferred); + + stream->collected_to = subreq->start + subreq->transferred; + wreq->collected_to = stream->collected_to; + netfs_put_subrequest(subreq, netfs_sreq_trace_put_done); + + trace_netfs_collect_stream(wreq, stream); + trace_netfs_collect_state(wreq, wreq->collected_to, 0); +} + +/* + * Write data to the server without going through the pagecache and without + * writing it to the local cache. We dispatch the subrequests serially and + * wait for each to complete before dispatching the next, lest we leave a gap + * in the data written due to a failure such as ENOSPC. We could, however + * attempt to do preparation such as content encryption for the next subreq + * whilst the current is in progress. + */ +static int netfs_unbuffered_write(struct netfs_io_request *wreq) +{ + struct netfs_io_subrequest *subreq = NULL; + struct netfs_io_stream *stream = &wreq->io_streams[0]; + int ret; + + _enter("%llx", wreq->len); + + if (wreq->origin == NETFS_DIO_WRITE) + inode_dio_begin(wreq->inode); + + stream->collected_to = wreq->start; + + for (;;) { + bool retry = false; + + if (!subreq) { + netfs_prepare_write(wreq, stream, wreq->start + wreq->transferred); + subreq = stream->construct; + stream->construct = NULL; + } + + /* Check if (re-)preparation failed. */ + if (unlikely(test_bit(NETFS_SREQ_FAILED, &subreq->flags))) { + netfs_write_subrequest_terminated(subreq, subreq->error); + wreq->error = subreq->error; + break; + } + + iov_iter_truncate(&subreq->io_iter, wreq->len - wreq->transferred); + if (!iov_iter_count(&subreq->io_iter)) + break; + + subreq->len = netfs_limit_iter(&subreq->io_iter, 0, + stream->sreq_max_len, + stream->sreq_max_segs); + iov_iter_truncate(&subreq->io_iter, subreq->len); + stream->submit_extendable_to = subreq->len; + + trace_netfs_sreq(subreq, netfs_sreq_trace_submit); + stream->issue_write(subreq); + + /* Async, need to wait. */ + netfs_wait_for_in_progress_stream(wreq, stream); + + if (test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { + retry = true; + } else if (test_bit(NETFS_SREQ_FAILED, &subreq->flags)) { + ret = subreq->error; + wreq->error = ret; + netfs_see_subrequest(subreq, netfs_sreq_trace_see_failed); + subreq = NULL; + break; + } + ret = 0; + + if (!retry) { + netfs_unbuffered_write_collect(wreq, stream, subreq); + subreq = NULL; + if (wreq->transferred >= wreq->len) + break; + if (!wreq->iocb && signal_pending(current)) { + ret = wreq->transferred ? -EINTR : -ERESTARTSYS; + trace_netfs_rreq(wreq, netfs_rreq_trace_intr); + break; + } + continue; + } + + /* We need to retry the last subrequest, so first reset the + * iterator, taking into account what, if anything, we managed + * to transfer. + */ + subreq->error = -EAGAIN; + trace_netfs_sreq(subreq, netfs_sreq_trace_retry); + if (subreq->transferred > 0) { + iov_iter_advance(&wreq->buffer.iter, subreq->transferred); + wreq->transferred += subreq->transferred; + } + + if (stream->source == NETFS_UPLOAD_TO_SERVER && + wreq->netfs_ops->retry_request) + wreq->netfs_ops->retry_request(wreq, stream); + + __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); + __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); + __clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags); + __clear_bit(NETFS_SREQ_FAILED, &subreq->flags); + subreq->io_iter = wreq->buffer.iter; + subreq->start = wreq->start + wreq->transferred; + subreq->len = wreq->len - wreq->transferred; + subreq->transferred = 0; + subreq->retry_count += 1; + stream->sreq_max_len = UINT_MAX; + stream->sreq_max_segs = INT_MAX; + + netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); + + if (stream->prepare_write) + stream->prepare_write(subreq); + __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); + netfs_stat(&netfs_n_wh_retry_write_subreq); + } + + netfs_unbuffered_write_done(wreq); + _leave(" = %d", ret); + return ret; +} + +static void netfs_unbuffered_write_async(struct work_struct *work) +{ + struct netfs_io_request *wreq = container_of(work, struct netfs_io_request, work); + + netfs_unbuffered_write(wreq); + netfs_put_request(wreq, netfs_rreq_trace_put_complete); +} + +/* + * Perform an unbuffered write where we may have to do an RMW operation on an + * encrypted file. This can also be used for direct I/O writes. + */ +ssize_t netfs_unbuffered_write_iter_locked(struct kiocb *iocb, struct iov_iter *iter, + struct netfs_group *netfs_group) +{ + struct netfs_io_request *wreq; + unsigned long long start = iocb->ki_pos; + unsigned long long end = start + iov_iter_count(iter); + ssize_t ret, n; + size_t len = iov_iter_count(iter); + bool async = !is_sync_kiocb(iocb); + + _enter(""); + + /* We're going to need a bounce buffer if what we transmit is going to + * be different in some way to the source buffer, e.g. because it gets + * encrypted/compressed or because it needs expanding to a block size. + */ + // TODO + + _debug("uw %llx-%llx", start, end); + + wreq = netfs_create_write_req(iocb->ki_filp->f_mapping, iocb->ki_filp, start, + iocb->ki_flags & IOCB_DIRECT ? + NETFS_DIO_WRITE : NETFS_UNBUFFERED_WRITE); + if (IS_ERR(wreq)) + return PTR_ERR(wreq); + + wreq->io_streams[0].avail = true; + trace_netfs_write(wreq, (iocb->ki_flags & IOCB_DIRECT ? + netfs_write_trace_dio_write : + netfs_write_trace_unbuffered_write)); + + { + /* If this is an async op and we're not using a bounce buffer, + * we have to save the source buffer as the iterator is only + * good until we return. In such a case, extract an iterator + * to represent as much of the the output buffer as we can + * manage. Note that the extraction might not be able to + * allocate a sufficiently large bvec array and may shorten the + * request. + */ + if (user_backed_iter(iter)) { + n = netfs_extract_user_iter(iter, len, &wreq->buffer.iter, 0); + if (n < 0) { + ret = n; + goto error_put; + } + wreq->direct_bv = (struct bio_vec *)wreq->buffer.iter.bvec; + wreq->direct_bv_count = n; + wreq->direct_bv_unpin = iov_iter_extract_will_pin(iter); + } else { + /* If this is a kernel-generated async DIO request, + * assume that any resources the iterator points to + * (eg. a bio_vec array) will persist till the end of + * the op. + */ + wreq->buffer.iter = *iter; + } + + wreq->len = iov_iter_count(&wreq->buffer.iter); + } + + __set_bit(NETFS_RREQ_USE_IO_ITER, &wreq->flags); + + /* Copy the data into the bounce buffer and encrypt it. */ + // TODO + + /* Dispatch the write. */ + __set_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags); + + if (async) { + INIT_WORK(&wreq->work, netfs_unbuffered_write_async); + wreq->iocb = iocb; + queue_work(system_dfl_wq, &wreq->work); + ret = -EIOCBQUEUED; + } else { + ret = netfs_unbuffered_write(wreq); + if (ret < 0) { + _debug("begin = %zd", ret); + } else { + iocb->ki_pos += wreq->transferred; + ret = wreq->transferred ?: wreq->error; + } + + netfs_put_request(wreq, netfs_rreq_trace_put_complete); + } + + netfs_put_request(wreq, netfs_rreq_trace_put_return); + return ret; + +error_put: + netfs_put_failed_request(wreq); + return ret; +} +EXPORT_SYMBOL(netfs_unbuffered_write_iter_locked); + +/** + * netfs_unbuffered_write_iter - Unbuffered write to a file + * @iocb: IO state structure + * @from: iov_iter with data to write + * + * Do an unbuffered write to a file, writing the data directly to the server + * and not lodging the data in the pagecache. + * + * Return: + * * Negative error code if no data has been written at all of + * vfs_fsync_range() failed for a synchronous write + * * Number of bytes written, even for truncated writes + */ +ssize_t netfs_unbuffered_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct address_space *mapping = file->f_mapping; + struct inode *inode = mapping->host; + struct netfs_inode *ictx = netfs_inode(inode); + ssize_t ret; + loff_t pos = iocb->ki_pos; + unsigned long long end = pos + iov_iter_count(from) - 1; + + _enter("%llx,%zx,%llx", pos, iov_iter_count(from), i_size_read(inode)); + + if (!iov_iter_count(from)) + return 0; + + trace_netfs_write_iter(iocb, from); + netfs_stat(&netfs_n_wh_dio_write); + + ret = netfs_start_io_direct(inode); + if (ret < 0) + return ret; + ret = generic_write_checks(iocb, from); + if (ret <= 0) + goto out; + ret = file_remove_privs(file); + if (ret < 0) + goto out; + ret = file_update_time(file); + if (ret < 0) + goto out; + if (iocb->ki_flags & IOCB_NOWAIT) { + /* We could block if there are any pages in the range. */ + ret = -EAGAIN; + if (filemap_range_has_page(mapping, pos, end)) + if (filemap_invalidate_inode(inode, true, pos, end)) + goto out; + } else { + ret = filemap_write_and_wait_range(mapping, pos, end); + if (ret < 0) + goto out; + } + + /* + * After a write we want buffered reads to be sure to go to disk to get + * the new data. We invalidate clean cached page from the region we're + * about to write. We do this *before* the write so that we can return + * without clobbering -EIOCBQUEUED from ->direct_IO(). + */ + ret = filemap_invalidate_inode(inode, true, pos, end); + if (ret < 0) + goto out; + end = iocb->ki_pos + iov_iter_count(from); + spin_lock(&inode->i_lock); + if (end > ictx->_zero_point) + netfs_write_zero_point(inode, end); + spin_unlock(&inode->i_lock); + + fscache_invalidate(netfs_i_cookie(ictx), NULL, i_size_read(inode), + FSCACHE_INVAL_DIO_WRITE); + ret = netfs_unbuffered_write_iter_locked(iocb, from, NULL); +out: + netfs_end_io_direct(inode); + return ret; +} +EXPORT_SYMBOL(netfs_unbuffered_write_iter); diff --git a/fs/netfs/fscache_cache.c b/fs/netfs/fscache_cache.c new file mode 100644 index 000000000..15b54d133 --- /dev/null +++ b/fs/netfs/fscache_cache.c @@ -0,0 +1,429 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* FS-Cache cache handling + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define FSCACHE_DEBUG_LEVEL CACHE +#include +#include +#include "internal.h" + +static LIST_HEAD(fscache_caches); +DECLARE_RWSEM(fscache_addremove_sem); +EXPORT_SYMBOL(fscache_addremove_sem); +DECLARE_WAIT_QUEUE_HEAD(fscache_clearance_waiters); +EXPORT_SYMBOL(fscache_clearance_waiters); + +static atomic_t fscache_cache_debug_id; + +/* + * Allocate a cache cookie. + */ +static struct fscache_cache *fscache_alloc_cache(const char *name) +{ + struct fscache_cache *cache; + + cache = kzalloc_obj(*cache); + if (cache) { + if (name) { + cache->name = kstrdup(name, GFP_KERNEL); + if (!cache->name) { + kfree(cache); + return NULL; + } + } + refcount_set(&cache->ref, 1); + INIT_LIST_HEAD(&cache->cache_link); + cache->debug_id = atomic_inc_return(&fscache_cache_debug_id); + } + return cache; +} + +static bool fscache_get_cache_maybe(struct fscache_cache *cache, + enum fscache_cache_trace where) +{ + bool success; + int ref; + + success = __refcount_inc_not_zero(&cache->ref, &ref); + if (success) + trace_fscache_cache(cache->debug_id, ref + 1, where); + return success; +} + +/* + * Look up a cache cookie. + */ +struct fscache_cache *fscache_lookup_cache(const char *name, bool is_cache) +{ + struct fscache_cache *candidate, *cache, *unnamed = NULL; + + /* firstly check for the existence of the cache under read lock */ + down_read(&fscache_addremove_sem); + + list_for_each_entry(cache, &fscache_caches, cache_link) { + if (cache->name && name && strcmp(cache->name, name) == 0 && + fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) + goto got_cache_r; + if (!cache->name && !name && + fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) + goto got_cache_r; + } + + if (!name) { + list_for_each_entry(cache, &fscache_caches, cache_link) { + if (cache->name && + fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) + goto got_cache_r; + } + } + + up_read(&fscache_addremove_sem); + + /* the cache does not exist - create a candidate */ + candidate = fscache_alloc_cache(name); + if (!candidate) + return ERR_PTR(-ENOMEM); + + /* write lock, search again and add if still not present */ + down_write(&fscache_addremove_sem); + + list_for_each_entry(cache, &fscache_caches, cache_link) { + if (cache->name && name && strcmp(cache->name, name) == 0 && + fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) + goto got_cache_w; + if (!cache->name) { + unnamed = cache; + if (!name && + fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) + goto got_cache_w; + } + } + + if (unnamed && is_cache && + fscache_get_cache_maybe(unnamed, fscache_cache_get_acquire)) + goto use_unnamed_cache; + + if (!name) { + list_for_each_entry(cache, &fscache_caches, cache_link) { + if (cache->name && + fscache_get_cache_maybe(cache, fscache_cache_get_acquire)) + goto got_cache_w; + } + } + + list_add_tail(&candidate->cache_link, &fscache_caches); + trace_fscache_cache(candidate->debug_id, + refcount_read(&candidate->ref), + fscache_cache_new_acquire); + up_write(&fscache_addremove_sem); + return candidate; + +got_cache_r: + up_read(&fscache_addremove_sem); + return cache; +use_unnamed_cache: + cache = unnamed; + cache->name = candidate->name; + candidate->name = NULL; +got_cache_w: + up_write(&fscache_addremove_sem); + kfree(candidate->name); + kfree(candidate); + return cache; +} + +/** + * fscache_acquire_cache - Acquire a cache-level cookie. + * @name: The name of the cache. + * + * Get a cookie to represent an actual cache. If a name is given and there is + * a nameless cache record available, this will acquire that and set its name, + * directing all the volumes using it to this cache. + * + * The cache will be switched over to the preparing state if not currently in + * use, otherwise -EBUSY will be returned. + */ +struct fscache_cache *fscache_acquire_cache(const char *name) +{ + struct fscache_cache *cache; + + ASSERT(name); + cache = fscache_lookup_cache(name, true); + if (IS_ERR(cache)) + return cache; + + if (!fscache_set_cache_state_maybe(cache, + FSCACHE_CACHE_IS_NOT_PRESENT, + FSCACHE_CACHE_IS_PREPARING)) { + pr_warn("Cache tag %s in use\n", name); + fscache_put_cache(cache, fscache_cache_put_cache); + return ERR_PTR(-EBUSY); + } + + return cache; +} +EXPORT_SYMBOL(fscache_acquire_cache); + +/** + * fscache_put_cache - Release a cache-level cookie. + * @cache: The cache cookie to be released + * @where: An indication of where the release happened + * + * Release the caller's reference on a cache-level cookie. The @where + * indication should give information about the circumstances in which the call + * occurs and will be logged through a tracepoint. + */ +void fscache_put_cache(struct fscache_cache *cache, + enum fscache_cache_trace where) +{ + unsigned int debug_id; + bool zero; + int ref; + + if (IS_ERR_OR_NULL(cache)) + return; + + debug_id = cache->debug_id; + zero = __refcount_dec_and_test(&cache->ref, &ref); + trace_fscache_cache(debug_id, ref - 1, where); + + if (zero) { + down_write(&fscache_addremove_sem); + list_del_init(&cache->cache_link); + up_write(&fscache_addremove_sem); + kfree(cache->name); + kfree(cache); + } +} + +/** + * fscache_relinquish_cache - Reset cache state and release cookie + * @cache: The cache cookie to be released + * + * Reset the state of a cache and release the caller's reference on a cache + * cookie. + */ +void fscache_relinquish_cache(struct fscache_cache *cache) +{ + enum fscache_cache_trace where = + (cache->state == FSCACHE_CACHE_IS_PREPARING) ? + fscache_cache_put_prep_failed : + fscache_cache_put_relinquish; + + cache->ops = NULL; + cache->cache_priv = NULL; + fscache_set_cache_state(cache, FSCACHE_CACHE_IS_NOT_PRESENT); + fscache_put_cache(cache, where); +} +EXPORT_SYMBOL(fscache_relinquish_cache); + +/** + * fscache_add_cache - Declare a cache as being open for business + * @cache: The cache-level cookie representing the cache + * @ops: Table of cache operations to use + * @cache_priv: Private data for the cache record + * + * Add a cache to the system, making it available for netfs's to use. + * + * See Documentation/filesystems/caching/backend-api.rst for a complete + * description. + */ +int fscache_add_cache(struct fscache_cache *cache, + const struct fscache_cache_ops *ops, + void *cache_priv) +{ + int n_accesses; + + _enter("{%s,%s}", ops->name, cache->name); + + BUG_ON(fscache_cache_state(cache) != FSCACHE_CACHE_IS_PREPARING); + + /* Get a ref on the cache cookie and keep its n_accesses counter raised + * by 1 to prevent wakeups from transitioning it to 0 until we're + * withdrawing caching services from it. + */ + n_accesses = atomic_inc_return(&cache->n_accesses); + trace_fscache_access_cache(cache->debug_id, refcount_read(&cache->ref), + n_accesses, fscache_access_cache_pin); + + down_write(&fscache_addremove_sem); + + cache->ops = ops; + cache->cache_priv = cache_priv; + fscache_set_cache_state(cache, FSCACHE_CACHE_IS_ACTIVE); + + up_write(&fscache_addremove_sem); + pr_notice("Cache \"%s\" added (type %s)\n", cache->name, ops->name); + _leave(" = 0 [%s]", cache->name); + return 0; +} +EXPORT_SYMBOL(fscache_add_cache); + +/** + * fscache_begin_cache_access - Pin a cache so it can be accessed + * @cache: The cache-level cookie + * @why: An indication of the circumstances of the access for tracing + * + * Attempt to pin the cache to prevent it from going away whilst we're + * accessing it and returns true if successful. This works as follows: + * + * (1) If the cache tests as not live (state is not FSCACHE_CACHE_IS_ACTIVE), + * then we return false to indicate access was not permitted. + * + * (2) If the cache tests as live, then we increment the n_accesses count and + * then recheck the liveness, ending the access if it ceased to be live. + * + * (3) When we end the access, we decrement n_accesses and wake up the any + * waiters if it reaches 0. + * + * (4) Whilst the cache is caching, n_accesses is kept artificially + * incremented to prevent wakeups from happening. + * + * (5) When the cache is taken offline, the state is changed to prevent new + * accesses, n_accesses is decremented and we wait for n_accesses to + * become 0. + */ +bool fscache_begin_cache_access(struct fscache_cache *cache, enum fscache_access_trace why) +{ + int n_accesses; + + if (!fscache_cache_is_live(cache)) + return false; + + n_accesses = atomic_inc_return(&cache->n_accesses); + smp_mb__after_atomic(); /* Reread live flag after n_accesses */ + trace_fscache_access_cache(cache->debug_id, refcount_read(&cache->ref), + n_accesses, why); + if (!fscache_cache_is_live(cache)) { + fscache_end_cache_access(cache, fscache_access_unlive); + return false; + } + return true; +} + +/** + * fscache_end_cache_access - Unpin a cache at the end of an access. + * @cache: The cache-level cookie + * @why: An indication of the circumstances of the access for tracing + * + * Unpin a cache after we've accessed it. The @why indicator is merely + * provided for tracing purposes. + */ +void fscache_end_cache_access(struct fscache_cache *cache, enum fscache_access_trace why) +{ + int n_accesses; + + smp_mb__before_atomic(); + n_accesses = atomic_dec_return(&cache->n_accesses); + trace_fscache_access_cache(cache->debug_id, refcount_read(&cache->ref), + n_accesses, why); + if (n_accesses == 0) + wake_up_var(&cache->n_accesses); +} + +/** + * fscache_io_error - Note a cache I/O error + * @cache: The record describing the cache + * + * Note that an I/O error occurred in a cache and that it should no longer be + * used for anything. This also reports the error into the kernel log. + * + * See Documentation/filesystems/caching/backend-api.rst for a complete + * description. + */ +void fscache_io_error(struct fscache_cache *cache) +{ + if (fscache_set_cache_state_maybe(cache, + FSCACHE_CACHE_IS_ACTIVE, + FSCACHE_CACHE_GOT_IOERROR)) + pr_err("Cache '%s' stopped due to I/O error\n", + cache->name); +} +EXPORT_SYMBOL(fscache_io_error); + +/** + * fscache_withdraw_cache - Withdraw a cache from the active service + * @cache: The cache cookie + * + * Begin the process of withdrawing a cache from service. This stops new + * cache-level and volume-level accesses from taking place and waits for + * currently ongoing cache-level accesses to end. + */ +void fscache_withdraw_cache(struct fscache_cache *cache) +{ + int n_accesses; + + pr_notice("Withdrawing cache \"%s\" (%u objs)\n", + cache->name, atomic_read(&cache->object_count)); + + fscache_set_cache_state(cache, FSCACHE_CACHE_IS_WITHDRAWN); + + /* Allow wakeups on dec-to-0 */ + n_accesses = atomic_dec_return(&cache->n_accesses); + trace_fscache_access_cache(cache->debug_id, refcount_read(&cache->ref), + n_accesses, fscache_access_cache_unpin); + + wait_var_event(&cache->n_accesses, + atomic_read(&cache->n_accesses) == 0); +} +EXPORT_SYMBOL(fscache_withdraw_cache); + +#ifdef CONFIG_PROC_FS +static const char fscache_cache_states[NR__FSCACHE_CACHE_STATE] __nonstring = "-PAEW"; + +/* + * Generate a list of caches in /proc/fs/fscache/caches + */ +static int fscache_caches_seq_show(struct seq_file *m, void *v) +{ + struct fscache_cache *cache; + + if (v == &fscache_caches) { + seq_puts(m, + "CACHE REF VOLS OBJS ACCES S NAME\n" + "======== ===== ===== ===== ===== = ===============\n" + ); + return 0; + } + + cache = list_entry(v, struct fscache_cache, cache_link); + seq_printf(m, + "%08x %5d %5d %5d %5d %c %s\n", + cache->debug_id, + refcount_read(&cache->ref), + atomic_read(&cache->n_volumes), + atomic_read(&cache->object_count), + atomic_read(&cache->n_accesses), + fscache_cache_states[cache->state], + cache->name ?: "-"); + return 0; +} + +static void *fscache_caches_seq_start(struct seq_file *m, loff_t *_pos) + __acquires(fscache_addremove_sem) +{ + down_read(&fscache_addremove_sem); + return seq_list_start_head(&fscache_caches, *_pos); +} + +static void *fscache_caches_seq_next(struct seq_file *m, void *v, loff_t *_pos) +{ + return seq_list_next(v, &fscache_caches, _pos); +} + +static void fscache_caches_seq_stop(struct seq_file *m, void *v) + __releases(fscache_addremove_sem) +{ + up_read(&fscache_addremove_sem); +} + +const struct seq_operations fscache_caches_seq_ops = { + .start = fscache_caches_seq_start, + .next = fscache_caches_seq_next, + .stop = fscache_caches_seq_stop, + .show = fscache_caches_seq_show, +}; +#endif /* CONFIG_PROC_FS */ diff --git a/fs/netfs/fscache_cookie.c b/fs/netfs/fscache_cookie.c new file mode 100644 index 000000000..3d56fc734 --- /dev/null +++ b/fs/netfs/fscache_cookie.c @@ -0,0 +1,1184 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* netfs cookie management + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + * + * See Documentation/filesystems/caching/netfs-api.rst for more information on + * the netfs API. + */ + +#define FSCACHE_DEBUG_LEVEL COOKIE +#include +#include +#include "internal.h" + +struct kmem_cache *fscache_cookie_jar; + +static void fscache_cookie_lru_timed_out(struct timer_list *timer); +static void fscache_cookie_lru_worker(struct work_struct *work); +static void fscache_cookie_worker(struct work_struct *work); +static void fscache_unhash_cookie(struct fscache_cookie *cookie); +static void fscache_perform_invalidation(struct fscache_cookie *cookie); + +#define fscache_cookie_hash_shift 15 +static struct hlist_bl_head fscache_cookie_hash[1 << fscache_cookie_hash_shift]; +static LIST_HEAD(fscache_cookies); +static DEFINE_RWLOCK(fscache_cookies_lock); +static LIST_HEAD(fscache_cookie_lru); +static DEFINE_SPINLOCK(fscache_cookie_lru_lock); +DEFINE_TIMER(fscache_cookie_lru_timer, fscache_cookie_lru_timed_out); +static DECLARE_WORK(fscache_cookie_lru_work, fscache_cookie_lru_worker); +static const char fscache_cookie_states[FSCACHE_COOKIE_STATE__NR] __nonstring = "-LCAIFUWRD"; +static unsigned int fscache_lru_cookie_timeout = 10 * HZ; + +void fscache_print_cookie(struct fscache_cookie *cookie, char prefix) +{ + const u8 *k; + + pr_err("%c-cookie c=%08x [fl=%lx na=%u nA=%u s=%c]\n", + prefix, + cookie->debug_id, + cookie->flags, + atomic_read(&cookie->n_active), + atomic_read(&cookie->n_accesses), + fscache_cookie_states[cookie->state]); + pr_err("%c-cookie V=%08x [%s]\n", + prefix, + cookie->volume->debug_id, + cookie->volume->key); + + k = (cookie->key_len <= sizeof(cookie->inline_key)) ? + cookie->inline_key : cookie->key; + pr_err("%c-key=[%u] '%*phN'\n", prefix, cookie->key_len, cookie->key_len, k); +} + +static void fscache_free_cookie(struct fscache_cookie *cookie) +{ + if (WARN_ON_ONCE(!list_empty(&cookie->commit_link))) { + spin_lock(&fscache_cookie_lru_lock); + list_del_init(&cookie->commit_link); + spin_unlock(&fscache_cookie_lru_lock); + fscache_stat_d(&fscache_n_cookies_lru); + fscache_stat(&fscache_n_cookies_lru_removed); + } + + if (WARN_ON_ONCE(test_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags))) { + fscache_print_cookie(cookie, 'F'); + return; + } + + write_lock(&fscache_cookies_lock); + list_del(&cookie->proc_link); + write_unlock(&fscache_cookies_lock); + if (cookie->aux_len > sizeof(cookie->inline_aux)) + kfree(cookie->aux); + if (cookie->key_len > sizeof(cookie->inline_key)) + kfree(cookie->key); + fscache_stat_d(&fscache_n_cookies); + kmem_cache_free(fscache_cookie_jar, cookie); +} + +static void __fscache_queue_cookie(struct fscache_cookie *cookie) +{ + if (!queue_work(fscache_wq, &cookie->work)) + fscache_put_cookie(cookie, fscache_cookie_put_over_queued); +} + +static void fscache_queue_cookie(struct fscache_cookie *cookie, + enum fscache_cookie_trace where) +{ + fscache_get_cookie(cookie, where); + __fscache_queue_cookie(cookie); +} + +/* + * Initialise the access gate on a cookie by setting a flag to prevent the + * state machine from being queued when the access counter transitions to 0. + * We're only interested in this when we withdraw caching services from the + * cookie. + */ +static void fscache_init_access_gate(struct fscache_cookie *cookie) +{ + int n_accesses; + + n_accesses = atomic_read(&cookie->n_accesses); + trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), + n_accesses, fscache_access_cache_pin); + set_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags); +} + +/** + * fscache_end_cookie_access - Unpin a cache at the end of an access. + * @cookie: A data file cookie + * @why: An indication of the circumstances of the access for tracing + * + * Unpin a cache cookie after we've accessed it and bring a deferred + * relinquishment or withdrawal state into effect. + * + * The @why indicator is provided for tracing purposes. + */ +void fscache_end_cookie_access(struct fscache_cookie *cookie, + enum fscache_access_trace why) +{ + int n_accesses; + + smp_mb__before_atomic(); + n_accesses = atomic_dec_return(&cookie->n_accesses); + trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), + n_accesses, why); + if (n_accesses == 0 && + !test_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags)) + fscache_queue_cookie(cookie, fscache_cookie_get_end_access); +} +EXPORT_SYMBOL(fscache_end_cookie_access); + +/* + * Pin the cache behind a cookie so that we can access it. + */ +static void __fscache_begin_cookie_access(struct fscache_cookie *cookie, + enum fscache_access_trace why) +{ + int n_accesses; + + n_accesses = atomic_inc_return(&cookie->n_accesses); + smp_mb__after_atomic(); /* (Future) read state after is-caching. + * Reread n_accesses after is-caching + */ + trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), + n_accesses, why); +} + +/** + * fscache_begin_cookie_access - Pin a cache so data can be accessed + * @cookie: A data file cookie + * @why: An indication of the circumstances of the access for tracing + * + * Attempt to pin the cache to prevent it from going away whilst we're + * accessing data and returns true if successful. This works as follows: + * + * (1) If the cookie is not being cached (ie. FSCACHE_COOKIE_IS_CACHING is not + * set), we return false to indicate access was not permitted. + * + * (2) If the cookie is being cached, we increment its n_accesses count and + * then recheck the IS_CACHING flag, ending the access if it got cleared. + * + * (3) When we end the access, we decrement the cookie's n_accesses and wake + * up the any waiters if it reaches 0. + * + * (4) Whilst the cookie is actively being cached, its n_accesses is kept + * artificially incremented to prevent wakeups from happening. + * + * (5) When the cache is taken offline or if the cookie is culled, the flag is + * cleared to prevent new accesses, the cookie's n_accesses is decremented + * and we wait for it to become 0. + * + * The @why indicator are merely provided for tracing purposes. + */ +bool fscache_begin_cookie_access(struct fscache_cookie *cookie, + enum fscache_access_trace why) +{ + if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags)) + return false; + __fscache_begin_cookie_access(cookie, why); + if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags) || + !fscache_cache_is_live(cookie->volume->cache)) { + fscache_end_cookie_access(cookie, fscache_access_unlive); + return false; + } + return true; +} + +static inline void wake_up_cookie_state(struct fscache_cookie *cookie) +{ + /* Use a barrier to ensure that waiters see the state variable + * change, as spin_unlock doesn't guarantee a barrier. + * + * See comments over wake_up_bit() and waitqueue_active(). + */ + smp_mb(); + wake_up_var(&cookie->state); +} + +/* + * Change the state a cookie is at and wake up anyone waiting for that. Impose + * an ordering between the stuff stored in the cookie and the state member. + * Paired with fscache_cookie_state(). + */ +static void __fscache_set_cookie_state(struct fscache_cookie *cookie, + enum fscache_cookie_state state) +{ + smp_store_release(&cookie->state, state); +} + +static void fscache_set_cookie_state(struct fscache_cookie *cookie, + enum fscache_cookie_state state) +{ + spin_lock(&cookie->lock); + __fscache_set_cookie_state(cookie, state); + spin_unlock(&cookie->lock); + wake_up_cookie_state(cookie); +} + +/** + * fscache_cookie_lookup_negative - Note negative lookup + * @cookie: The cookie that was being looked up + * + * Note that some part of the metadata path in the cache doesn't exist and so + * we can release any waiting readers in the certain knowledge that there's + * nothing for them to actually read. + * + * This function uses no locking and must only be called from the state machine. + */ +void fscache_cookie_lookup_negative(struct fscache_cookie *cookie) +{ + set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); + fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_CREATING); +} +EXPORT_SYMBOL(fscache_cookie_lookup_negative); + +/** + * fscache_resume_after_invalidation - Allow I/O to resume after invalidation + * @cookie: The cookie that was invalidated + * + * Tell fscache that invalidation is sufficiently complete that I/O can be + * allowed again. + */ +void fscache_resume_after_invalidation(struct fscache_cookie *cookie) +{ + fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_ACTIVE); +} +EXPORT_SYMBOL(fscache_resume_after_invalidation); + +/** + * fscache_caching_failed - Report that a failure stopped caching on a cookie + * @cookie: The cookie that was affected + * + * Tell fscache that caching on a cookie needs to be stopped due to some sort + * of failure. + * + * This function uses no locking and must only be called from the state machine. + */ +void fscache_caching_failed(struct fscache_cookie *cookie) +{ + clear_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags); + fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_FAILED); + trace_fscache_cookie(cookie->debug_id, refcount_read(&cookie->ref), + fscache_cookie_failed); +} +EXPORT_SYMBOL(fscache_caching_failed); + +/* + * Set the index key in a cookie. The cookie struct has space for a 16-byte + * key plus length and hash, but if that's not big enough, it's instead a + * pointer to a buffer containing 3 bytes of hash, 1 byte of length and then + * the key data. + */ +static int fscache_set_key(struct fscache_cookie *cookie, + const void *index_key, size_t index_key_len) +{ + void *buf; + size_t buf_size; + + buf_size = round_up(index_key_len, sizeof(__le32)); + + if (index_key_len > sizeof(cookie->inline_key)) { + buf = kzalloc(buf_size, GFP_KERNEL); + if (!buf) + return -ENOMEM; + cookie->key = buf; + } else { + buf = cookie->inline_key; + } + + memcpy(buf, index_key, index_key_len); + cookie->key_hash = fscache_hash(cookie->volume->key_hash, + buf, buf_size); + return 0; +} + +static bool fscache_cookie_same(const struct fscache_cookie *a, + const struct fscache_cookie *b) +{ + const void *ka, *kb; + + if (a->key_hash != b->key_hash || + a->volume != b->volume || + a->key_len != b->key_len) + return false; + + if (a->key_len <= sizeof(a->inline_key)) { + ka = &a->inline_key; + kb = &b->inline_key; + } else { + ka = a->key; + kb = b->key; + } + return memcmp(ka, kb, a->key_len) == 0; +} + +static atomic_t fscache_cookie_debug_id = ATOMIC_INIT(1); + +/* + * Allocate a cookie. + */ +static struct fscache_cookie *fscache_alloc_cookie( + struct fscache_volume *volume, + u8 advice, + const void *index_key, size_t index_key_len, + const void *aux_data, size_t aux_data_len, + loff_t object_size) +{ + struct fscache_cookie *cookie; + + /* allocate and initialise a cookie */ + cookie = kmem_cache_zalloc(fscache_cookie_jar, GFP_KERNEL); + if (!cookie) + return NULL; + fscache_stat(&fscache_n_cookies); + + cookie->volume = volume; + cookie->advice = advice; + cookie->key_len = index_key_len; + cookie->aux_len = aux_data_len; + cookie->object_size = object_size; + if (object_size == 0) + __set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); + + if (fscache_set_key(cookie, index_key, index_key_len) < 0) + goto nomem; + + if (cookie->aux_len <= sizeof(cookie->inline_aux)) { + memcpy(cookie->inline_aux, aux_data, cookie->aux_len); + } else { + cookie->aux = kmemdup(aux_data, cookie->aux_len, GFP_KERNEL); + if (!cookie->aux) + goto nomem; + } + + refcount_set(&cookie->ref, 1); + cookie->debug_id = atomic_inc_return(&fscache_cookie_debug_id); + spin_lock_init(&cookie->lock); + INIT_LIST_HEAD(&cookie->commit_link); + INIT_WORK(&cookie->work, fscache_cookie_worker); + __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); + + write_lock(&fscache_cookies_lock); + list_add_tail(&cookie->proc_link, &fscache_cookies); + write_unlock(&fscache_cookies_lock); + fscache_see_cookie(cookie, fscache_cookie_new_acquire); + return cookie; + +nomem: + fscache_free_cookie(cookie); + return NULL; +} + +static inline bool fscache_cookie_is_dropped(struct fscache_cookie *cookie) +{ + return READ_ONCE(cookie->state) == FSCACHE_COOKIE_STATE_DROPPED; +} + +static void fscache_wait_on_collision(struct fscache_cookie *candidate, + struct fscache_cookie *wait_for) +{ + enum fscache_cookie_state *statep = &wait_for->state; + + wait_var_event_timeout(statep, fscache_cookie_is_dropped(wait_for), + 20 * HZ); + if (!fscache_cookie_is_dropped(wait_for)) { + pr_notice("Potential collision c=%08x old: c=%08x", + candidate->debug_id, wait_for->debug_id); + wait_var_event(statep, fscache_cookie_is_dropped(wait_for)); + } +} + +/* + * Attempt to insert the new cookie into the hash. If there's a collision, we + * wait for the old cookie to complete if it's being relinquished and an error + * otherwise. + */ +static bool fscache_hash_cookie(struct fscache_cookie *candidate) +{ + struct fscache_cookie *cursor, *wait_for = NULL; + struct hlist_bl_head *h; + struct hlist_bl_node *p; + unsigned int bucket; + + bucket = candidate->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1); + h = &fscache_cookie_hash[bucket]; + + hlist_bl_lock(h); + hlist_bl_for_each_entry(cursor, p, h, hash_link) { + if (fscache_cookie_same(candidate, cursor)) { + if (!test_bit(FSCACHE_COOKIE_RELINQUISHED, &cursor->flags)) + goto collision; + wait_for = fscache_get_cookie(cursor, + fscache_cookie_get_hash_collision); + break; + } + } + + fscache_get_volume(candidate->volume, fscache_volume_get_cookie); + atomic_inc(&candidate->volume->n_cookies); + hlist_bl_add_head(&candidate->hash_link, h); + set_bit(FSCACHE_COOKIE_IS_HASHED, &candidate->flags); + hlist_bl_unlock(h); + + if (wait_for) { + fscache_wait_on_collision(candidate, wait_for); + fscache_put_cookie(wait_for, fscache_cookie_put_hash_collision); + } + return true; + +collision: + trace_fscache_cookie(cursor->debug_id, refcount_read(&cursor->ref), + fscache_cookie_collision); + pr_err("Duplicate cookie detected\n"); + fscache_print_cookie(cursor, 'O'); + fscache_print_cookie(candidate, 'N'); + hlist_bl_unlock(h); + return false; +} + +/* + * Request a cookie to represent a data storage object within a volume. + * + * We never let on to the netfs about errors. We may set a negative cookie + * pointer, but that's okay + */ +struct fscache_cookie *__fscache_acquire_cookie( + struct fscache_volume *volume, + u8 advice, + const void *index_key, size_t index_key_len, + const void *aux_data, size_t aux_data_len, + loff_t object_size) +{ + struct fscache_cookie *cookie; + + _enter("V=%x", volume->debug_id); + + if (!index_key || !index_key_len || index_key_len > 255 || aux_data_len > 255) + return NULL; + if (!aux_data || !aux_data_len) { + aux_data = NULL; + aux_data_len = 0; + } + + fscache_stat(&fscache_n_acquires); + + cookie = fscache_alloc_cookie(volume, advice, + index_key, index_key_len, + aux_data, aux_data_len, + object_size); + if (!cookie) { + fscache_stat(&fscache_n_acquires_oom); + return NULL; + } + + if (!fscache_hash_cookie(cookie)) { + fscache_see_cookie(cookie, fscache_cookie_discard); + fscache_free_cookie(cookie); + return NULL; + } + + trace_fscache_acquire(cookie); + fscache_stat(&fscache_n_acquires_ok); + _leave(" = c=%08x", cookie->debug_id); + return cookie; +} +EXPORT_SYMBOL(__fscache_acquire_cookie); + +/* + * Prepare a cache object to be written to. + */ +static void fscache_prepare_to_write(struct fscache_cookie *cookie) +{ + cookie->volume->cache->ops->prepare_to_write(cookie); +} + +/* + * Look up a cookie in the cache. + */ +static void fscache_perform_lookup(struct fscache_cookie *cookie) +{ + enum fscache_access_trace trace = fscache_access_lookup_cookie_end_failed; + bool need_withdraw = false; + + _enter(""); + + if (!cookie->volume->cache_priv) { + fscache_create_volume(cookie->volume, true); + if (!cookie->volume->cache_priv) { + fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); + goto out; + } + } + + if (!cookie->volume->cache->ops->lookup_cookie(cookie)) { + if (cookie->state != FSCACHE_COOKIE_STATE_FAILED) + fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); + need_withdraw = true; + _leave(" [fail]"); + goto out; + } + + fscache_see_cookie(cookie, fscache_cookie_see_active); + spin_lock(&cookie->lock); + if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags)) + __fscache_set_cookie_state(cookie, + FSCACHE_COOKIE_STATE_INVALIDATING); + else + __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_ACTIVE); + spin_unlock(&cookie->lock); + wake_up_cookie_state(cookie); + trace = fscache_access_lookup_cookie_end; + +out: + fscache_end_cookie_access(cookie, trace); + if (need_withdraw) + fscache_withdraw_cookie(cookie); + fscache_end_volume_access(cookie->volume, cookie, trace); +} + +/* + * Begin the process of looking up a cookie. We offload the actual process to + * a worker thread. + */ +static bool fscache_begin_lookup(struct fscache_cookie *cookie, bool will_modify) +{ + if (will_modify) { + set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags); + set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags); + } + if (!fscache_begin_volume_access(cookie->volume, cookie, + fscache_access_lookup_cookie)) + return false; + + __fscache_begin_cookie_access(cookie, fscache_access_lookup_cookie); + __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_LOOKING_UP); + set_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags); + set_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags); + return true; +} + +/* + * Start using the cookie for I/O. This prevents the backing object from being + * reaped by VM pressure. + */ +void __fscache_use_cookie(struct fscache_cookie *cookie, bool will_modify) +{ + enum fscache_cookie_state state; + bool queue = false; + int n_active; + + _enter("c=%08x", cookie->debug_id); + + if (WARN(test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags), + "Trying to use relinquished cookie\n")) + return; + + spin_lock(&cookie->lock); + + n_active = atomic_inc_return(&cookie->n_active); + trace_fscache_active(cookie->debug_id, refcount_read(&cookie->ref), + n_active, atomic_read(&cookie->n_accesses), + will_modify ? + fscache_active_use_modify : fscache_active_use); + +again: + state = fscache_cookie_state(cookie); + switch (state) { + case FSCACHE_COOKIE_STATE_QUIESCENT: + queue = fscache_begin_lookup(cookie, will_modify); + break; + + case FSCACHE_COOKIE_STATE_LOOKING_UP: + case FSCACHE_COOKIE_STATE_CREATING: + if (will_modify) + set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags); + break; + case FSCACHE_COOKIE_STATE_ACTIVE: + case FSCACHE_COOKIE_STATE_INVALIDATING: + if (will_modify && + !test_and_set_bit(FSCACHE_COOKIE_LOCAL_WRITE, &cookie->flags)) { + set_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags); + queue = true; + } + /* + * We could race with cookie_lru which may set LRU_DISCARD bit + * but has yet to run the cookie state machine. If this happens + * and another thread tries to use the cookie, clear LRU_DISCARD + * so we don't end up withdrawing the cookie while in use. + */ + if (test_and_clear_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags)) + fscache_see_cookie(cookie, fscache_cookie_see_lru_discard_clear); + break; + + case FSCACHE_COOKIE_STATE_FAILED: + case FSCACHE_COOKIE_STATE_WITHDRAWING: + break; + + case FSCACHE_COOKIE_STATE_LRU_DISCARDING: + spin_unlock(&cookie->lock); + wait_var_event(&cookie->state, + fscache_cookie_state(cookie) != + FSCACHE_COOKIE_STATE_LRU_DISCARDING); + spin_lock(&cookie->lock); + goto again; + + case FSCACHE_COOKIE_STATE_DROPPED: + case FSCACHE_COOKIE_STATE_RELINQUISHING: + WARN(1, "Can't use cookie in state %u\n", state); + break; + } + + spin_unlock(&cookie->lock); + if (queue) + fscache_queue_cookie(cookie, fscache_cookie_get_use_work); + _leave(""); +} +EXPORT_SYMBOL(__fscache_use_cookie); + +static void fscache_unuse_cookie_locked(struct fscache_cookie *cookie) +{ + clear_bit(FSCACHE_COOKIE_DISABLED, &cookie->flags); + if (!test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags)) + return; + + cookie->unused_at = jiffies; + spin_lock(&fscache_cookie_lru_lock); + if (list_empty(&cookie->commit_link)) { + fscache_get_cookie(cookie, fscache_cookie_get_lru); + fscache_stat(&fscache_n_cookies_lru); + } + list_move_tail(&cookie->commit_link, &fscache_cookie_lru); + + spin_unlock(&fscache_cookie_lru_lock); + timer_reduce(&fscache_cookie_lru_timer, + jiffies + fscache_lru_cookie_timeout); +} + +/* + * Stop using the cookie for I/O. + */ +void __fscache_unuse_cookie(struct fscache_cookie *cookie, + const void *aux_data, const loff_t *object_size) +{ + unsigned int debug_id = cookie->debug_id; + unsigned int r = refcount_read(&cookie->ref); + unsigned int a = atomic_read(&cookie->n_accesses); + unsigned int c; + + if (aux_data || object_size) + __fscache_update_cookie(cookie, aux_data, object_size); + + /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */ + c = atomic_fetch_add_unless(&cookie->n_active, -1, 1); + if (c != 1) { + trace_fscache_active(debug_id, r, c - 1, a, fscache_active_unuse); + return; + } + + spin_lock(&cookie->lock); + r = refcount_read(&cookie->ref); + a = atomic_read(&cookie->n_accesses); + c = atomic_dec_return(&cookie->n_active); + trace_fscache_active(debug_id, r, c, a, fscache_active_unuse); + if (c == 0) + fscache_unuse_cookie_locked(cookie); + spin_unlock(&cookie->lock); +} +EXPORT_SYMBOL(__fscache_unuse_cookie); + +/* + * Perform work upon the cookie, such as committing its cache state, + * relinquishing it or withdrawing the backing cache. We're protected from the + * cache going away under us as object withdrawal must come through this + * non-reentrant work item. + */ +static void fscache_cookie_state_machine(struct fscache_cookie *cookie) +{ + enum fscache_cookie_state state; + bool wake = false; + + _enter("c=%x", cookie->debug_id); + +again: + spin_lock(&cookie->lock); +again_locked: + state = cookie->state; + switch (state) { + case FSCACHE_COOKIE_STATE_QUIESCENT: + /* The QUIESCENT state is jumped to the LOOKING_UP state by + * fscache_use_cookie(). + */ + + if (atomic_read(&cookie->n_accesses) == 0 && + test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) { + __fscache_set_cookie_state(cookie, + FSCACHE_COOKIE_STATE_RELINQUISHING); + wake = true; + goto again_locked; + } + break; + + case FSCACHE_COOKIE_STATE_LOOKING_UP: + spin_unlock(&cookie->lock); + fscache_init_access_gate(cookie); + fscache_perform_lookup(cookie); + goto again; + + case FSCACHE_COOKIE_STATE_INVALIDATING: + spin_unlock(&cookie->lock); + fscache_perform_invalidation(cookie); + goto again; + + case FSCACHE_COOKIE_STATE_ACTIVE: + if (test_and_clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags)) { + spin_unlock(&cookie->lock); + fscache_prepare_to_write(cookie); + spin_lock(&cookie->lock); + } + if (test_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags)) { + if (atomic_read(&cookie->n_accesses) != 0) + /* still being accessed: postpone it */ + break; + + __fscache_set_cookie_state(cookie, + FSCACHE_COOKIE_STATE_LRU_DISCARDING); + wake = true; + goto again_locked; + } + fallthrough; + + case FSCACHE_COOKIE_STATE_FAILED: + if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags)) + fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end); + + if (atomic_read(&cookie->n_accesses) != 0) + break; + if (test_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags)) { + __fscache_set_cookie_state(cookie, + FSCACHE_COOKIE_STATE_RELINQUISHING); + wake = true; + goto again_locked; + } + if (test_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags)) { + __fscache_set_cookie_state(cookie, + FSCACHE_COOKIE_STATE_WITHDRAWING); + wake = true; + goto again_locked; + } + break; + + case FSCACHE_COOKIE_STATE_LRU_DISCARDING: + case FSCACHE_COOKIE_STATE_RELINQUISHING: + case FSCACHE_COOKIE_STATE_WITHDRAWING: + if (cookie->cache_priv) { + spin_unlock(&cookie->lock); + cookie->volume->cache->ops->withdraw_cookie(cookie); + spin_lock(&cookie->lock); + } + + if (test_and_clear_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags)) + fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end); + + switch (state) { + case FSCACHE_COOKIE_STATE_RELINQUISHING: + fscache_see_cookie(cookie, fscache_cookie_see_relinquish); + fscache_unhash_cookie(cookie); + __fscache_set_cookie_state(cookie, + FSCACHE_COOKIE_STATE_DROPPED); + wake = true; + goto out; + case FSCACHE_COOKIE_STATE_LRU_DISCARDING: + fscache_see_cookie(cookie, fscache_cookie_see_lru_discard); + break; + case FSCACHE_COOKIE_STATE_WITHDRAWING: + fscache_see_cookie(cookie, fscache_cookie_see_withdraw); + break; + default: + BUG(); + } + + clear_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &cookie->flags); + clear_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags); + clear_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags); + clear_bit(FSCACHE_COOKIE_DO_PREP_TO_WRITE, &cookie->flags); + set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); + __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_QUIESCENT); + wake = true; + goto again_locked; + + case FSCACHE_COOKIE_STATE_DROPPED: + break; + + default: + WARN_ONCE(1, "Cookie %x in unexpected state %u\n", + cookie->debug_id, state); + break; + } + +out: + spin_unlock(&cookie->lock); + if (wake) + wake_up_cookie_state(cookie); + _leave(""); +} + +static void fscache_cookie_worker(struct work_struct *work) +{ + struct fscache_cookie *cookie = container_of(work, struct fscache_cookie, work); + + fscache_see_cookie(cookie, fscache_cookie_see_work); + fscache_cookie_state_machine(cookie); + fscache_put_cookie(cookie, fscache_cookie_put_work); +} + +/* + * Wait for the object to become inactive. The cookie's work item will be + * scheduled when someone transitions n_accesses to 0 - but if someone's + * already done that, schedule it anyway. + */ +static void __fscache_withdraw_cookie(struct fscache_cookie *cookie) +{ + int n_accesses; + bool unpinned; + + unpinned = test_and_clear_bit(FSCACHE_COOKIE_NO_ACCESS_WAKE, &cookie->flags); + + /* Need to read the access count after unpinning */ + n_accesses = atomic_read(&cookie->n_accesses); + if (unpinned) + trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), + n_accesses, fscache_access_cache_unpin); + if (n_accesses == 0) + fscache_queue_cookie(cookie, fscache_cookie_get_end_access); +} + +static void fscache_cookie_lru_do_one(struct fscache_cookie *cookie) +{ + fscache_see_cookie(cookie, fscache_cookie_see_lru_do_one); + + spin_lock(&cookie->lock); + if (cookie->state != FSCACHE_COOKIE_STATE_ACTIVE || + time_before(jiffies, cookie->unused_at + fscache_lru_cookie_timeout) || + atomic_read(&cookie->n_active) > 0) { + spin_unlock(&cookie->lock); + fscache_stat(&fscache_n_cookies_lru_removed); + } else { + set_bit(FSCACHE_COOKIE_DO_LRU_DISCARD, &cookie->flags); + spin_unlock(&cookie->lock); + fscache_stat(&fscache_n_cookies_lru_expired); + _debug("lru c=%x", cookie->debug_id); + __fscache_withdraw_cookie(cookie); + } + + fscache_put_cookie(cookie, fscache_cookie_put_lru); +} + +static void fscache_cookie_lru_worker(struct work_struct *work) +{ + struct fscache_cookie *cookie; + unsigned long unused_at; + + spin_lock(&fscache_cookie_lru_lock); + + while (!list_empty(&fscache_cookie_lru)) { + cookie = list_first_entry(&fscache_cookie_lru, + struct fscache_cookie, commit_link); + unused_at = cookie->unused_at + fscache_lru_cookie_timeout; + if (time_before(jiffies, unused_at)) { + timer_reduce(&fscache_cookie_lru_timer, unused_at); + break; + } + + list_del_init(&cookie->commit_link); + fscache_stat_d(&fscache_n_cookies_lru); + spin_unlock(&fscache_cookie_lru_lock); + fscache_cookie_lru_do_one(cookie); + spin_lock(&fscache_cookie_lru_lock); + } + + spin_unlock(&fscache_cookie_lru_lock); +} + +static void fscache_cookie_lru_timed_out(struct timer_list *timer) +{ + queue_work(fscache_wq, &fscache_cookie_lru_work); +} + +static void fscache_cookie_drop_from_lru(struct fscache_cookie *cookie) +{ + bool need_put = false; + + if (!list_empty(&cookie->commit_link)) { + spin_lock(&fscache_cookie_lru_lock); + if (!list_empty(&cookie->commit_link)) { + list_del_init(&cookie->commit_link); + fscache_stat_d(&fscache_n_cookies_lru); + fscache_stat(&fscache_n_cookies_lru_dropped); + need_put = true; + } + spin_unlock(&fscache_cookie_lru_lock); + if (need_put) + fscache_put_cookie(cookie, fscache_cookie_put_lru); + } +} + +/* + * Remove a cookie from the hash table. + */ +static void fscache_unhash_cookie(struct fscache_cookie *cookie) +{ + struct hlist_bl_head *h; + unsigned int bucket; + + bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1); + h = &fscache_cookie_hash[bucket]; + + hlist_bl_lock(h); + hlist_bl_del(&cookie->hash_link); + clear_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags); + hlist_bl_unlock(h); + fscache_stat(&fscache_n_relinquishes_dropped); +} + +static void fscache_drop_withdraw_cookie(struct fscache_cookie *cookie) +{ + fscache_cookie_drop_from_lru(cookie); + __fscache_withdraw_cookie(cookie); +} + +/** + * fscache_withdraw_cookie - Mark a cookie for withdrawal + * @cookie: The cookie to be withdrawn. + * + * Allow the cache backend to withdraw the backing for a cookie for its own + * reasons, even if that cookie is in active use. + */ +void fscache_withdraw_cookie(struct fscache_cookie *cookie) +{ + set_bit(FSCACHE_COOKIE_DO_WITHDRAW, &cookie->flags); + fscache_drop_withdraw_cookie(cookie); +} +EXPORT_SYMBOL(fscache_withdraw_cookie); + +/* + * Allow the netfs to release a cookie back to the cache. + * - the object will be marked as recyclable on disk if retire is true + */ +void __fscache_relinquish_cookie(struct fscache_cookie *cookie, bool retire) +{ + fscache_stat(&fscache_n_relinquishes); + if (retire) + fscache_stat(&fscache_n_relinquishes_retire); + + _enter("c=%08x{%d},%d", + cookie->debug_id, atomic_read(&cookie->n_active), retire); + + if (WARN(test_and_set_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags), + "Cookie c=%x already relinquished\n", cookie->debug_id)) + return; + + if (retire) + set_bit(FSCACHE_COOKIE_RETIRED, &cookie->flags); + trace_fscache_relinquish(cookie, retire); + + ASSERTCMP(atomic_read(&cookie->n_active), ==, 0); + ASSERTCMP(atomic_read(&cookie->volume->n_cookies), >, 0); + atomic_dec(&cookie->volume->n_cookies); + + if (test_bit(FSCACHE_COOKIE_HAS_BEEN_CACHED, &cookie->flags)) { + set_bit(FSCACHE_COOKIE_DO_RELINQUISH, &cookie->flags); + fscache_drop_withdraw_cookie(cookie); + } else { + fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_DROPPED); + fscache_unhash_cookie(cookie); + } + fscache_put_cookie(cookie, fscache_cookie_put_relinquish); +} +EXPORT_SYMBOL(__fscache_relinquish_cookie); + +/* + * Drop a reference to a cookie. + */ +void fscache_put_cookie(struct fscache_cookie *cookie, + enum fscache_cookie_trace where) +{ + struct fscache_volume *volume = cookie->volume; + unsigned int cookie_debug_id = cookie->debug_id; + bool zero; + int ref; + + zero = __refcount_dec_and_test(&cookie->ref, &ref); + trace_fscache_cookie(cookie_debug_id, ref - 1, where); + if (zero) { + fscache_free_cookie(cookie); + fscache_put_volume(volume, fscache_volume_put_cookie); + } +} +EXPORT_SYMBOL(fscache_put_cookie); + +/* + * Get a reference to a cookie. + */ +struct fscache_cookie *fscache_get_cookie(struct fscache_cookie *cookie, + enum fscache_cookie_trace where) +{ + int ref; + + __refcount_inc(&cookie->ref, &ref); + trace_fscache_cookie(cookie->debug_id, ref + 1, where); + return cookie; +} +EXPORT_SYMBOL(fscache_get_cookie); + +/* + * Ask the cache to effect invalidation of a cookie. + */ +static void fscache_perform_invalidation(struct fscache_cookie *cookie) +{ + if (!cookie->volume->cache->ops->invalidate_cookie(cookie)) + fscache_caching_failed(cookie); + fscache_end_cookie_access(cookie, fscache_access_invalidate_cookie_end); +} + +/* + * Invalidate an object. + */ +void __fscache_invalidate(struct fscache_cookie *cookie, + const void *aux_data, loff_t new_size, + unsigned int flags) +{ + bool is_caching; + + _enter("c=%x", cookie->debug_id); + + fscache_stat(&fscache_n_invalidates); + + if (WARN(test_bit(FSCACHE_COOKIE_RELINQUISHED, &cookie->flags), + "Trying to invalidate relinquished cookie\n")) + return; + + if ((flags & FSCACHE_INVAL_DIO_WRITE) && + test_and_set_bit(FSCACHE_COOKIE_DISABLED, &cookie->flags)) + return; + + spin_lock(&cookie->lock); + set_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags); + fscache_update_aux(cookie, aux_data, &new_size); + cookie->inval_counter++; + trace_fscache_invalidate(cookie, new_size); + + switch (cookie->state) { + case FSCACHE_COOKIE_STATE_INVALIDATING: /* is_still_valid will catch it */ + default: + spin_unlock(&cookie->lock); + _leave(" [no %u]", cookie->state); + return; + + case FSCACHE_COOKIE_STATE_LOOKING_UP: + if (!test_and_set_bit(FSCACHE_COOKIE_DO_INVALIDATE, &cookie->flags)) + __fscache_begin_cookie_access(cookie, fscache_access_invalidate_cookie); + fallthrough; + case FSCACHE_COOKIE_STATE_CREATING: + spin_unlock(&cookie->lock); + _leave(" [look %x]", cookie->inval_counter); + return; + + case FSCACHE_COOKIE_STATE_ACTIVE: + is_caching = fscache_begin_cookie_access( + cookie, fscache_access_invalidate_cookie); + if (is_caching) + __fscache_set_cookie_state(cookie, FSCACHE_COOKIE_STATE_INVALIDATING); + spin_unlock(&cookie->lock); + wake_up_cookie_state(cookie); + + if (is_caching) + fscache_queue_cookie(cookie, fscache_cookie_get_inval_work); + _leave(" [inv]"); + return; + } +} +EXPORT_SYMBOL(__fscache_invalidate); + +#ifdef CONFIG_PROC_FS +/* + * Generate a list of extant cookies in /proc/fs/fscache/cookies + */ +static int fscache_cookies_seq_show(struct seq_file *m, void *v) +{ + struct fscache_cookie *cookie; + unsigned int keylen = 0, auxlen = 0; + u8 *p; + + if (v == &fscache_cookies) { + seq_puts(m, + "COOKIE VOLUME REF ACT ACC S FL DEF \n" + "======== ======== === === === = == ================\n" + ); + return 0; + } + + cookie = list_entry(v, struct fscache_cookie, proc_link); + + seq_printf(m, + "%08x %08x %3d %3d %3d %c %02lx", + cookie->debug_id, + cookie->volume->debug_id, + refcount_read(&cookie->ref), + atomic_read(&cookie->n_active), + atomic_read(&cookie->n_accesses), + fscache_cookie_states[cookie->state], + cookie->flags); + + keylen = cookie->key_len; + auxlen = cookie->aux_len; + + if (keylen > 0 || auxlen > 0) { + seq_puts(m, " "); + p = keylen <= sizeof(cookie->inline_key) ? + cookie->inline_key : cookie->key; + for (; keylen > 0; keylen--) + seq_printf(m, "%02x", *p++); + if (auxlen > 0) { + seq_puts(m, ", "); + p = auxlen <= sizeof(cookie->inline_aux) ? + cookie->inline_aux : cookie->aux; + for (; auxlen > 0; auxlen--) + seq_printf(m, "%02x", *p++); + } + } + + seq_puts(m, "\n"); + return 0; +} + +static void *fscache_cookies_seq_start(struct seq_file *m, loff_t *_pos) + __acquires(fscache_cookies_lock) +{ + read_lock(&fscache_cookies_lock); + return seq_list_start_head(&fscache_cookies, *_pos); +} + +static void *fscache_cookies_seq_next(struct seq_file *m, void *v, loff_t *_pos) +{ + return seq_list_next(v, &fscache_cookies, _pos); +} + +static void fscache_cookies_seq_stop(struct seq_file *m, void *v) + __releases(rcu) +{ + read_unlock(&fscache_cookies_lock); +} + + +const struct seq_operations fscache_cookies_seq_ops = { + .start = fscache_cookies_seq_start, + .next = fscache_cookies_seq_next, + .stop = fscache_cookies_seq_stop, + .show = fscache_cookies_seq_show, +}; +#endif diff --git a/fs/netfs/fscache_internal.h b/fs/netfs/fscache_internal.h new file mode 100644 index 000000000..a09b948fc --- /dev/null +++ b/fs/netfs/fscache_internal.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Internal definitions for FS-Cache + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include "internal.h" + +#ifdef pr_fmt +#undef pr_fmt +#endif + +#define pr_fmt(fmt) "FS-Cache: " fmt diff --git a/fs/netfs/fscache_io.c b/fs/netfs/fscache_io.c new file mode 100644 index 000000000..37f05b4d3 --- /dev/null +++ b/fs/netfs/fscache_io.c @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Cache data I/O routines + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ +#define FSCACHE_DEBUG_LEVEL OPERATION +#include +#include +#include +#include +#include "internal.h" + +/** + * fscache_wait_for_operation - Wait for an object become accessible + * @cres: The cache resources for the operation being performed + * @want_state: The minimum state the object must be at + * + * See if the target cache object is at the specified minimum state of + * accessibility yet, and if not, wait for it. + */ +bool fscache_wait_for_operation(struct netfs_cache_resources *cres, + enum fscache_want_state want_state) +{ + struct fscache_cookie *cookie = fscache_cres_cookie(cres); + enum fscache_cookie_state state; + +again: + if (!fscache_cache_is_live(cookie->volume->cache)) { + _leave(" [broken]"); + return false; + } + + state = fscache_cookie_state(cookie); + _enter("c=%08x{%u},%x", cookie->debug_id, state, want_state); + + switch (state) { + case FSCACHE_COOKIE_STATE_CREATING: + case FSCACHE_COOKIE_STATE_INVALIDATING: + if (want_state == FSCACHE_WANT_PARAMS) + goto ready; /* There can be no content */ + fallthrough; + case FSCACHE_COOKIE_STATE_LOOKING_UP: + case FSCACHE_COOKIE_STATE_LRU_DISCARDING: + wait_var_event(&cookie->state, + fscache_cookie_state(cookie) != state); + goto again; + + case FSCACHE_COOKIE_STATE_ACTIVE: + goto ready; + case FSCACHE_COOKIE_STATE_DROPPED: + case FSCACHE_COOKIE_STATE_RELINQUISHING: + default: + _leave(" [not live]"); + return false; + } + +ready: + if (!cres->cache_priv2) + return cookie->volume->cache->ops->begin_operation(cres, want_state); + return true; +} +EXPORT_SYMBOL(fscache_wait_for_operation); + +/* + * Begin an I/O operation on the cache, waiting till we reach the right state. + * + * Attaches the resources required to the operation resources record. + */ +static int fscache_begin_operation(struct netfs_cache_resources *cres, + struct fscache_cookie *cookie, + enum fscache_want_state want_state, + enum fscache_access_trace why) +{ + enum fscache_cookie_state state; + long timeo; + bool once_only = false; + + cres->ops = NULL; + cres->cache_priv = cookie; + cres->cache_priv2 = NULL; + cres->debug_id = cookie->debug_id; + cres->inval_counter = cookie->inval_counter; + + if (!fscache_begin_cookie_access(cookie, why)) { + cres->cache_priv = NULL; + return -ENOBUFS; + } + +again: + spin_lock(&cookie->lock); + + state = fscache_cookie_state(cookie); + _enter("c=%08x{%u},%x", cookie->debug_id, state, want_state); + + switch (state) { + case FSCACHE_COOKIE_STATE_LOOKING_UP: + case FSCACHE_COOKIE_STATE_LRU_DISCARDING: + case FSCACHE_COOKIE_STATE_INVALIDATING: + goto wait_for_file_wrangling; + case FSCACHE_COOKIE_STATE_CREATING: + if (want_state == FSCACHE_WANT_PARAMS) + goto ready; /* There can be no content */ + goto wait_for_file_wrangling; + case FSCACHE_COOKIE_STATE_ACTIVE: + goto ready; + case FSCACHE_COOKIE_STATE_DROPPED: + case FSCACHE_COOKIE_STATE_RELINQUISHING: + WARN(1, "Can't use cookie in state %u\n", cookie->state); + goto not_live; + default: + goto not_live; + } + +ready: + spin_unlock(&cookie->lock); + if (!cookie->volume->cache->ops->begin_operation(cres, want_state)) + goto failed; + return 0; + +wait_for_file_wrangling: + spin_unlock(&cookie->lock); + trace_fscache_access(cookie->debug_id, refcount_read(&cookie->ref), + atomic_read(&cookie->n_accesses), + fscache_access_io_wait); + timeo = wait_var_event_timeout(&cookie->state, + fscache_cookie_state(cookie) != state, 20 * HZ); + if (timeo <= 1 && !once_only) { + pr_warn("%s: cookie state change wait timed out: cookie->state=%u state=%u", + __func__, fscache_cookie_state(cookie), state); + fscache_print_cookie(cookie, 'O'); + once_only = true; + } + goto again; + +not_live: + spin_unlock(&cookie->lock); +failed: + cres->cache_priv = NULL; + cres->ops = NULL; + fscache_end_cookie_access(cookie, fscache_access_io_not_live); + _leave(" = -ENOBUFS"); + return -ENOBUFS; +} + +int __fscache_begin_read_operation(struct netfs_cache_resources *cres, + struct fscache_cookie *cookie) +{ + return fscache_begin_operation(cres, cookie, FSCACHE_WANT_PARAMS, + fscache_access_io_read); +} +EXPORT_SYMBOL(__fscache_begin_read_operation); + +int __fscache_begin_write_operation(struct netfs_cache_resources *cres, + struct fscache_cookie *cookie) +{ + return fscache_begin_operation(cres, cookie, FSCACHE_WANT_PARAMS, + fscache_access_io_write); +} +EXPORT_SYMBOL(__fscache_begin_write_operation); + +struct fscache_write_request { + struct netfs_cache_resources cache_resources; + struct address_space *mapping; + loff_t start; + size_t len; + bool set_bits; + bool using_pgpriv2; + netfs_io_terminated_t term_func; + void *term_func_priv; +}; + +void __fscache_clear_page_bits(struct address_space *mapping, + loff_t start, size_t len) +{ + pgoff_t first = start / PAGE_SIZE; + pgoff_t last = (start + len - 1) / PAGE_SIZE; + struct page *page; + + if (len) { + XA_STATE(xas, &mapping->i_pages, first); + + rcu_read_lock(); + xas_for_each(&xas, page, last) { + folio_end_private_2(page_folio(page)); + } + rcu_read_unlock(); + } +} +EXPORT_SYMBOL(__fscache_clear_page_bits); + +/* + * Deal with the completion of writing the data to the cache. + */ +static void fscache_wreq_done(void *priv, ssize_t transferred_or_error) +{ + struct fscache_write_request *wreq = priv; + + if (wreq->using_pgpriv2) + fscache_clear_page_bits(wreq->mapping, wreq->start, wreq->len, + wreq->set_bits); + + if (wreq->term_func) + wreq->term_func(wreq->term_func_priv, transferred_or_error); + fscache_end_operation(&wreq->cache_resources); + kfree(wreq); +} + +void __fscache_write_to_cache(struct fscache_cookie *cookie, + struct address_space *mapping, + loff_t start, size_t len, loff_t i_size, + netfs_io_terminated_t term_func, + void *term_func_priv, + bool using_pgpriv2, bool cond) +{ + struct fscache_write_request *wreq; + struct netfs_cache_resources *cres; + struct iov_iter iter; + int ret = -ENOBUFS; + + if (len == 0) + goto abandon; + + _enter("%llx,%zx", start, len); + + wreq = kzalloc_obj(struct fscache_write_request, GFP_NOFS); + if (!wreq) + goto abandon; + wreq->mapping = mapping; + wreq->start = start; + wreq->len = len; + wreq->using_pgpriv2 = using_pgpriv2; + wreq->set_bits = cond; + wreq->term_func = term_func; + wreq->term_func_priv = term_func_priv; + + cres = &wreq->cache_resources; + if (fscache_begin_operation(cres, cookie, FSCACHE_WANT_WRITE, + fscache_access_io_write) < 0) + goto abandon_free; + + ret = cres->ops->prepare_write(cres, &start, &len, len, i_size, false); + if (ret < 0) + goto abandon_end; + + /* TODO: Consider clearing page bits now for space the write isn't + * covering. This is more complicated than it appears when THPs are + * taken into account. + */ + + iov_iter_xarray(&iter, ITER_SOURCE, &mapping->i_pages, start, len); + fscache_write(cres, start, &iter, fscache_wreq_done, wreq); + return; + +abandon_end: + return fscache_wreq_done(wreq, ret); +abandon_free: + kfree(wreq); +abandon: + if (using_pgpriv2) + fscache_clear_page_bits(mapping, start, len, cond); + if (term_func) + term_func(term_func_priv, ret); +} +EXPORT_SYMBOL(__fscache_write_to_cache); + +/* + * Change the size of a backing object. + */ +void __fscache_resize_cookie(struct fscache_cookie *cookie, loff_t new_size) +{ + struct netfs_cache_resources cres; + + trace_fscache_resize(cookie, new_size); + if (fscache_begin_operation(&cres, cookie, FSCACHE_WANT_WRITE, + fscache_access_io_resize) == 0) { + fscache_stat(&fscache_n_resizes); + set_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &cookie->flags); + + /* We cannot defer a resize as we need to do it inside the + * netfs's inode lock so that we're serialised with respect to + * writes. + */ + cookie->volume->cache->ops->resize_cookie(&cres, new_size); + fscache_end_operation(&cres); + } else { + fscache_stat(&fscache_n_resizes_null); + } +} +EXPORT_SYMBOL(__fscache_resize_cookie); diff --git a/fs/netfs/fscache_main.c b/fs/netfs/fscache_main.c new file mode 100644 index 000000000..49849005e --- /dev/null +++ b/fs/netfs/fscache_main.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* General filesystem local caching manager + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define FSCACHE_DEBUG_LEVEL CACHE +#include +#include +#include "internal.h" +#define CREATE_TRACE_POINTS +#include + +EXPORT_TRACEPOINT_SYMBOL(fscache_access_cache); +EXPORT_TRACEPOINT_SYMBOL(fscache_access_volume); +EXPORT_TRACEPOINT_SYMBOL(fscache_access); + +struct workqueue_struct *fscache_wq; +EXPORT_SYMBOL(fscache_wq); + +/* + * Mixing scores (in bits) for (7,20): + * Input delta: 1-bit 2-bit + * 1 round: 330.3 9201.6 + * 2 rounds: 1246.4 25475.4 + * 3 rounds: 1907.1 31295.1 + * 4 rounds: 2042.3 31718.6 + * Perfect: 2048 31744 + * (32*64) (32*31/2 * 64) + */ +#define HASH_MIX(x, y, a) \ + ( x ^= (a), \ + y ^= x, x = rol32(x, 7),\ + x += y, y = rol32(y,20),\ + y *= 9 ) + +static inline unsigned int fold_hash(unsigned long x, unsigned long y) +{ + /* Use arch-optimized multiply if one exists */ + return __hash_32(y ^ __hash_32(x)); +} + +/* + * Generate a hash. This is derived from full_name_hash(), but we want to be + * sure it is arch independent and that it doesn't change as bits of the + * computed hash value might appear on disk. The caller must guarantee that + * the source data is a multiple of four bytes in size. + */ +unsigned int fscache_hash(unsigned int salt, const void *data, size_t len) +{ + const __le32 *p = data; + unsigned int a, x = 0, y = salt, n = len / sizeof(__le32); + + for (; n; n--) { + a = le32_to_cpu(*p++); + HASH_MIX(x, y, a); + } + return fold_hash(x, y); +} + +/* + * initialise the fs caching module + */ +int __init fscache_init(void) +{ + int ret = -ENOMEM; + + fscache_wq = alloc_workqueue("fscache", WQ_UNBOUND | WQ_FREEZABLE, 0); + if (!fscache_wq) + goto error_wq; + + ret = fscache_proc_init(); + if (ret < 0) + goto error_proc; + + fscache_cookie_jar = kmem_cache_create("fscache_cookie_jar", + sizeof(struct fscache_cookie), + 0, 0, NULL); + if (!fscache_cookie_jar) { + pr_notice("Failed to allocate a cookie jar\n"); + ret = -ENOMEM; + goto error_cookie_jar; + } + + pr_notice("FS-Cache loaded\n"); + return 0; + +error_cookie_jar: + fscache_proc_cleanup(); +error_proc: + destroy_workqueue(fscache_wq); +error_wq: + return ret; +} + +/* + * clean up on module removal + */ +void __exit fscache_exit(void) +{ + _enter(""); + + kmem_cache_destroy(fscache_cookie_jar); + fscache_proc_cleanup(); + timer_shutdown_sync(&fscache_cookie_lru_timer); + destroy_workqueue(fscache_wq); + pr_notice("FS-Cache unloaded\n"); +} diff --git a/fs/netfs/fscache_proc.c b/fs/netfs/fscache_proc.c new file mode 100644 index 000000000..874d951bc --- /dev/null +++ b/fs/netfs/fscache_proc.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* FS-Cache statistics viewing interface + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define FSCACHE_DEBUG_LEVEL CACHE +#include +#include +#include +#include "internal.h" + +/* + * Add files to /proc/fs/netfs/. + */ +int __init fscache_proc_init(void) +{ + if (!proc_symlink("fs/fscache", NULL, "netfs")) + goto error_sym; + + if (!proc_create_seq("fs/netfs/caches", S_IFREG | 0444, NULL, + &fscache_caches_seq_ops)) + goto error; + + if (!proc_create_seq("fs/netfs/volumes", S_IFREG | 0444, NULL, + &fscache_volumes_seq_ops)) + goto error; + + if (!proc_create_seq("fs/netfs/cookies", S_IFREG | 0444, NULL, + &fscache_cookies_seq_ops)) + goto error; + return 0; + +error: + remove_proc_entry("fs/fscache", NULL); +error_sym: + return -ENOMEM; +} + +/* + * Clean up the /proc/fs/fscache symlink. + */ +void fscache_proc_cleanup(void) +{ + remove_proc_subtree("fs/fscache", NULL); +} diff --git a/fs/netfs/fscache_stats.c b/fs/netfs/fscache_stats.c new file mode 100644 index 000000000..add21abdf --- /dev/null +++ b/fs/netfs/fscache_stats.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* FS-Cache statistics + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define FSCACHE_DEBUG_LEVEL CACHE +#include +#include +#include "internal.h" + +/* + * operation counters + */ +atomic_t fscache_n_volumes; +atomic_t fscache_n_volumes_collision; +atomic_t fscache_n_volumes_nomem; +atomic_t fscache_n_cookies; +atomic_t fscache_n_cookies_lru; +atomic_t fscache_n_cookies_lru_expired; +atomic_t fscache_n_cookies_lru_removed; +atomic_t fscache_n_cookies_lru_dropped; + +atomic_t fscache_n_acquires; +atomic_t fscache_n_acquires_ok; +atomic_t fscache_n_acquires_oom; + +atomic_t fscache_n_invalidates; + +atomic_t fscache_n_updates; +EXPORT_SYMBOL(fscache_n_updates); + +atomic_t fscache_n_relinquishes; +atomic_t fscache_n_relinquishes_retire; +atomic_t fscache_n_relinquishes_dropped; + +atomic_t fscache_n_resizes; +atomic_t fscache_n_resizes_null; + +atomic_t fscache_n_read; +EXPORT_SYMBOL(fscache_n_read); +atomic_t fscache_n_write; +EXPORT_SYMBOL(fscache_n_write); +atomic_t fscache_n_no_write_space; +EXPORT_SYMBOL(fscache_n_no_write_space); +atomic_t fscache_n_no_create_space; +EXPORT_SYMBOL(fscache_n_no_create_space); +atomic_t fscache_n_culled; +EXPORT_SYMBOL(fscache_n_culled); +atomic_t fscache_n_dio_misfit; +EXPORT_SYMBOL(fscache_n_dio_misfit); + +/* + * display the general statistics + */ +int fscache_stats_show(struct seq_file *m) +{ + seq_puts(m, "-- FS-Cache statistics --\n"); + seq_printf(m, "Cookies: n=%d v=%d vcol=%u voom=%u\n", + atomic_read(&fscache_n_cookies), + atomic_read(&fscache_n_volumes), + atomic_read(&fscache_n_volumes_collision), + atomic_read(&fscache_n_volumes_nomem) + ); + + seq_printf(m, "Acquire: n=%u ok=%u oom=%u\n", + atomic_read(&fscache_n_acquires), + atomic_read(&fscache_n_acquires_ok), + atomic_read(&fscache_n_acquires_oom)); + + seq_printf(m, "LRU : n=%u exp=%u rmv=%u drp=%u at=%ld\n", + atomic_read(&fscache_n_cookies_lru), + atomic_read(&fscache_n_cookies_lru_expired), + atomic_read(&fscache_n_cookies_lru_removed), + atomic_read(&fscache_n_cookies_lru_dropped), + timer_pending(&fscache_cookie_lru_timer) ? + fscache_cookie_lru_timer.expires - jiffies : 0); + + seq_printf(m, "Invals : n=%u\n", + atomic_read(&fscache_n_invalidates)); + + seq_printf(m, "Updates: n=%u rsz=%u rsn=%u\n", + atomic_read(&fscache_n_updates), + atomic_read(&fscache_n_resizes), + atomic_read(&fscache_n_resizes_null)); + + seq_printf(m, "Relinqs: n=%u rtr=%u drop=%u\n", + atomic_read(&fscache_n_relinquishes), + atomic_read(&fscache_n_relinquishes_retire), + atomic_read(&fscache_n_relinquishes_dropped)); + + seq_printf(m, "NoSpace: nwr=%u ncr=%u cull=%u\n", + atomic_read(&fscache_n_no_write_space), + atomic_read(&fscache_n_no_create_space), + atomic_read(&fscache_n_culled)); + + seq_printf(m, "IO : rd=%u wr=%u mis=%u\n", + atomic_read(&fscache_n_read), + atomic_read(&fscache_n_write), + atomic_read(&fscache_n_dio_misfit)); + return 0; +} diff --git a/fs/netfs/fscache_volume.c b/fs/netfs/fscache_volume.c new file mode 100644 index 000000000..78412fb85 --- /dev/null +++ b/fs/netfs/fscache_volume.c @@ -0,0 +1,532 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Volume-level cache cookie handling. + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define FSCACHE_DEBUG_LEVEL COOKIE +#include +#include +#include "internal.h" + +#define fscache_volume_hash_shift 10 +static struct hlist_bl_head fscache_volume_hash[1 << fscache_volume_hash_shift]; +static atomic_t fscache_volume_debug_id; +static LIST_HEAD(fscache_volumes); + +static void fscache_create_volume_work(struct work_struct *work); + +struct fscache_volume *fscache_get_volume(struct fscache_volume *volume, + enum fscache_volume_trace where) +{ + int ref; + + __refcount_inc(&volume->ref, &ref); + trace_fscache_volume(volume->debug_id, ref + 1, where); + return volume; +} + +struct fscache_volume *fscache_try_get_volume(struct fscache_volume *volume, + enum fscache_volume_trace where) +{ + int ref; + + if (!__refcount_inc_not_zero(&volume->ref, &ref)) + return NULL; + + trace_fscache_volume(volume->debug_id, ref + 1, where); + return volume; +} +EXPORT_SYMBOL(fscache_try_get_volume); + +static void fscache_see_volume(struct fscache_volume *volume, + enum fscache_volume_trace where) +{ + int ref = refcount_read(&volume->ref); + + trace_fscache_volume(volume->debug_id, ref, where); +} + +/* + * Pin the cache behind a volume so that we can access it. + */ +static void __fscache_begin_volume_access(struct fscache_volume *volume, + struct fscache_cookie *cookie, + enum fscache_access_trace why) +{ + int n_accesses; + + n_accesses = atomic_inc_return(&volume->n_accesses); + smp_mb__after_atomic(); + trace_fscache_access_volume(volume->debug_id, cookie ? cookie->debug_id : 0, + refcount_read(&volume->ref), + n_accesses, why); +} + +/** + * fscache_begin_volume_access - Pin a cache so a volume can be accessed + * @volume: The volume cookie + * @cookie: A datafile cookie for a tracing reference (or NULL) + * @why: An indication of the circumstances of the access for tracing + * + * Attempt to pin the cache to prevent it from going away whilst we're + * accessing a volume and returns true if successful. This works as follows: + * + * (1) If the cache tests as not live (state is not FSCACHE_CACHE_IS_ACTIVE), + * then we return false to indicate access was not permitted. + * + * (2) If the cache tests as live, then we increment the volume's n_accesses + * count and then recheck the cache liveness, ending the access if it + * ceased to be live. + * + * (3) When we end the access, we decrement the volume's n_accesses and wake + * up the any waiters if it reaches 0. + * + * (4) Whilst the cache is caching, the volume's n_accesses is kept + * artificially incremented to prevent wakeups from happening. + * + * (5) When the cache is taken offline, the state is changed to prevent new + * accesses, the volume's n_accesses is decremented and we wait for it to + * become 0. + * + * The datafile @cookie and the @why indicator are merely provided for tracing + * purposes. + */ +bool fscache_begin_volume_access(struct fscache_volume *volume, + struct fscache_cookie *cookie, + enum fscache_access_trace why) +{ + if (!fscache_cache_is_live(volume->cache)) + return false; + __fscache_begin_volume_access(volume, cookie, why); + if (!fscache_cache_is_live(volume->cache)) { + fscache_end_volume_access(volume, cookie, fscache_access_unlive); + return false; + } + return true; +} + +/** + * fscache_end_volume_access - Unpin a cache at the end of an access. + * @volume: The volume cookie + * @cookie: A datafile cookie for a tracing reference (or NULL) + * @why: An indication of the circumstances of the access for tracing + * + * Unpin a cache volume after we've accessed it. The datafile @cookie and the + * @why indicator are merely provided for tracing purposes. + */ +void fscache_end_volume_access(struct fscache_volume *volume, + struct fscache_cookie *cookie, + enum fscache_access_trace why) +{ + int n_accesses; + + smp_mb__before_atomic(); + n_accesses = atomic_dec_return(&volume->n_accesses); + trace_fscache_access_volume(volume->debug_id, cookie ? cookie->debug_id : 0, + refcount_read(&volume->ref), + n_accesses, why); + if (n_accesses == 0) + wake_up_var(&volume->n_accesses); +} +EXPORT_SYMBOL(fscache_end_volume_access); + +static bool fscache_volume_same(const struct fscache_volume *a, + const struct fscache_volume *b) +{ + size_t klen; + + if (a->key_hash != b->key_hash || + a->cache != b->cache || + a->key[0] != b->key[0]) + return false; + + klen = round_up(a->key[0] + 1, sizeof(__le32)); + return memcmp(a->key, b->key, klen) == 0; +} + +static bool fscache_is_acquire_pending(struct fscache_volume *volume) +{ + return test_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &volume->flags); +} + +static void fscache_wait_on_volume_collision(struct fscache_volume *candidate, + unsigned int collidee_debug_id) +{ + wait_on_bit_timeout(&candidate->flags, FSCACHE_VOLUME_ACQUIRE_PENDING, + TASK_UNINTERRUPTIBLE, 20 * HZ); + if (fscache_is_acquire_pending(candidate)) { + pr_notice("Potential volume collision new=%08x old=%08x", + candidate->debug_id, collidee_debug_id); + fscache_stat(&fscache_n_volumes_collision); + wait_on_bit(&candidate->flags, FSCACHE_VOLUME_ACQUIRE_PENDING, + TASK_UNINTERRUPTIBLE); + } +} + +/* + * Attempt to insert the new volume into the hash. If there's a collision, we + * wait for the old volume to complete if it's being relinquished and an error + * otherwise. + */ +static bool fscache_hash_volume(struct fscache_volume *candidate) +{ + struct fscache_volume *cursor; + struct hlist_bl_head *h; + struct hlist_bl_node *p; + unsigned int bucket, collidee_debug_id = 0; + + bucket = candidate->key_hash & (ARRAY_SIZE(fscache_volume_hash) - 1); + h = &fscache_volume_hash[bucket]; + + hlist_bl_lock(h); + hlist_bl_for_each_entry(cursor, p, h, hash_link) { + if (fscache_volume_same(candidate, cursor)) { + if (!test_bit(FSCACHE_VOLUME_RELINQUISHED, &cursor->flags)) + goto collision; + fscache_see_volume(cursor, fscache_volume_get_hash_collision); + set_bit(FSCACHE_VOLUME_COLLIDED_WITH, &cursor->flags); + set_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &candidate->flags); + collidee_debug_id = cursor->debug_id; + break; + } + } + + hlist_bl_add_head(&candidate->hash_link, h); + hlist_bl_unlock(h); + + if (fscache_is_acquire_pending(candidate)) + fscache_wait_on_volume_collision(candidate, collidee_debug_id); + return true; + +collision: + fscache_see_volume(cursor, fscache_volume_collision); + hlist_bl_unlock(h); + return false; +} + +/* + * Allocate and initialise a volume representation cookie. + */ +static struct fscache_volume *fscache_alloc_volume(const char *volume_key, + const char *cache_name, + const void *coherency_data, + size_t coherency_len) +{ + struct fscache_volume *volume; + struct fscache_cache *cache; + size_t klen, hlen; + u8 *key; + + klen = strlen(volume_key); + if (klen > NAME_MAX) + return NULL; + + if (!coherency_data) + coherency_len = 0; + + cache = fscache_lookup_cache(cache_name, false); + if (IS_ERR(cache)) + return NULL; + + volume = kzalloc_flex(*volume, coherency, coherency_len); + if (!volume) + goto err_cache; + + volume->cache = cache; + volume->coherency_len = coherency_len; + if (coherency_data) + memcpy(volume->coherency, coherency_data, coherency_len); + INIT_LIST_HEAD(&volume->proc_link); + INIT_WORK(&volume->work, fscache_create_volume_work); + refcount_set(&volume->ref, 1); + spin_lock_init(&volume->lock); + + /* Stick the length on the front of the key and pad it out to make + * hashing easier. + */ + hlen = round_up(1 + klen + 1, sizeof(__le32)); + key = kzalloc(hlen, GFP_KERNEL); + if (!key) + goto err_vol; + key[0] = klen; + memcpy(key + 1, volume_key, klen); + + volume->key = key; + volume->key_hash = fscache_hash(0, key, hlen); + + volume->debug_id = atomic_inc_return(&fscache_volume_debug_id); + down_write(&fscache_addremove_sem); + atomic_inc(&cache->n_volumes); + list_add_tail(&volume->proc_link, &fscache_volumes); + fscache_see_volume(volume, fscache_volume_new_acquire); + fscache_stat(&fscache_n_volumes); + up_write(&fscache_addremove_sem); + _leave(" = v=%x", volume->debug_id); + return volume; + +err_vol: + kfree(volume); +err_cache: + fscache_put_cache(cache, fscache_cache_put_alloc_volume); + fscache_stat(&fscache_n_volumes_nomem); + return NULL; +} + +/* + * Create a volume's representation on disk. Have a volume ref and a cache + * access we have to release. + */ +static void fscache_create_volume_work(struct work_struct *work) +{ + const struct fscache_cache_ops *ops; + struct fscache_volume *volume = + container_of(work, struct fscache_volume, work); + + fscache_see_volume(volume, fscache_volume_see_create_work); + + ops = volume->cache->ops; + if (ops->acquire_volume) + ops->acquire_volume(volume); + fscache_end_cache_access(volume->cache, + fscache_access_acquire_volume_end); + + clear_and_wake_up_bit(FSCACHE_VOLUME_CREATING, &volume->flags); + fscache_put_volume(volume, fscache_volume_put_create_work); +} + +/* + * Dispatch a worker thread to create a volume's representation on disk. + */ +void fscache_create_volume(struct fscache_volume *volume, bool wait) +{ + if (test_and_set_bit(FSCACHE_VOLUME_CREATING, &volume->flags)) + goto maybe_wait; + if (volume->cache_priv) + goto no_wait; /* We raced */ + if (!fscache_begin_cache_access(volume->cache, + fscache_access_acquire_volume)) + goto no_wait; + + fscache_get_volume(volume, fscache_volume_get_create_work); + if (!schedule_work(&volume->work)) + fscache_put_volume(volume, fscache_volume_put_create_work); + +maybe_wait: + if (wait) { + fscache_see_volume(volume, fscache_volume_wait_create_work); + wait_on_bit(&volume->flags, FSCACHE_VOLUME_CREATING, + TASK_UNINTERRUPTIBLE); + } + return; +no_wait: + clear_and_wake_up_bit(FSCACHE_VOLUME_CREATING, &volume->flags); +} + +/* + * Acquire a volume representation cookie and link it to a (proposed) cache. + */ +struct fscache_volume *__fscache_acquire_volume(const char *volume_key, + const char *cache_name, + const void *coherency_data, + size_t coherency_len) +{ + struct fscache_volume *volume; + + volume = fscache_alloc_volume(volume_key, cache_name, + coherency_data, coherency_len); + if (!volume) + return ERR_PTR(-ENOMEM); + + if (!fscache_hash_volume(volume)) { + fscache_put_volume(volume, fscache_volume_put_hash_collision); + return ERR_PTR(-EBUSY); + } + + fscache_create_volume(volume, false); + return volume; +} +EXPORT_SYMBOL(__fscache_acquire_volume); + +static void fscache_wake_pending_volume(struct fscache_volume *volume, + struct hlist_bl_head *h) +{ + struct fscache_volume *cursor; + struct hlist_bl_node *p; + + hlist_bl_for_each_entry(cursor, p, h, hash_link) { + if (fscache_volume_same(cursor, volume)) { + fscache_see_volume(cursor, fscache_volume_see_hash_wake); + clear_and_wake_up_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, + &cursor->flags); + return; + } + } +} + +/* + * Remove a volume cookie from the hash table. + */ +static void fscache_unhash_volume(struct fscache_volume *volume) +{ + struct hlist_bl_head *h; + unsigned int bucket; + + bucket = volume->key_hash & (ARRAY_SIZE(fscache_volume_hash) - 1); + h = &fscache_volume_hash[bucket]; + + hlist_bl_lock(h); + hlist_bl_del(&volume->hash_link); + if (test_bit(FSCACHE_VOLUME_COLLIDED_WITH, &volume->flags)) + fscache_wake_pending_volume(volume, h); + hlist_bl_unlock(h); +} + +/* + * Drop a cache's volume attachments. + */ +static void fscache_free_volume(struct fscache_volume *volume) +{ + struct fscache_cache *cache = volume->cache; + + if (volume->cache_priv) { + __fscache_begin_volume_access(volume, NULL, + fscache_access_relinquish_volume); + if (volume->cache_priv) + cache->ops->free_volume(volume); + fscache_end_volume_access(volume, NULL, + fscache_access_relinquish_volume_end); + } + + down_write(&fscache_addremove_sem); + list_del_init(&volume->proc_link); + atomic_dec(&volume->cache->n_volumes); + up_write(&fscache_addremove_sem); + + if (!hlist_bl_unhashed(&volume->hash_link)) + fscache_unhash_volume(volume); + + trace_fscache_volume(volume->debug_id, 0, fscache_volume_free); + kfree(volume->key); + kfree(volume); + fscache_stat_d(&fscache_n_volumes); + fscache_put_cache(cache, fscache_cache_put_volume); +} + +/* + * Drop a reference to a volume cookie. + */ +void fscache_put_volume(struct fscache_volume *volume, + enum fscache_volume_trace where) +{ + if (volume) { + unsigned int debug_id = volume->debug_id; + bool zero; + int ref; + + zero = __refcount_dec_and_test(&volume->ref, &ref); + trace_fscache_volume(debug_id, ref - 1, where); + if (zero) + fscache_free_volume(volume); + } +} +EXPORT_SYMBOL(fscache_put_volume); + +/* + * Relinquish a volume representation cookie. + */ +void __fscache_relinquish_volume(struct fscache_volume *volume, + const void *coherency_data, + bool invalidate) +{ + if (WARN_ON(test_and_set_bit(FSCACHE_VOLUME_RELINQUISHED, &volume->flags))) + return; + + if (invalidate) { + set_bit(FSCACHE_VOLUME_INVALIDATE, &volume->flags); + } else if (coherency_data) { + memcpy(volume->coherency, coherency_data, volume->coherency_len); + } + + fscache_put_volume(volume, fscache_volume_put_relinquish); +} +EXPORT_SYMBOL(__fscache_relinquish_volume); + +/** + * fscache_withdraw_volume - Withdraw a volume from being cached + * @volume: Volume cookie + * + * Withdraw a cache volume from service, waiting for all accesses to complete + * before returning. + */ +void fscache_withdraw_volume(struct fscache_volume *volume) +{ + int n_accesses; + + _debug("withdraw V=%x", volume->debug_id); + + /* Allow wakeups on dec-to-0 */ + n_accesses = atomic_dec_return(&volume->n_accesses); + trace_fscache_access_volume(volume->debug_id, 0, + refcount_read(&volume->ref), + n_accesses, fscache_access_cache_unpin); + + wait_var_event(&volume->n_accesses, + atomic_read(&volume->n_accesses) == 0); +} +EXPORT_SYMBOL(fscache_withdraw_volume); + +#ifdef CONFIG_PROC_FS +/* + * Generate a list of volumes in /proc/fs/fscache/volumes + */ +static int fscache_volumes_seq_show(struct seq_file *m, void *v) +{ + struct fscache_volume *volume; + + if (v == &fscache_volumes) { + seq_puts(m, + "VOLUME REF nCOOK ACC FL CACHE KEY\n" + "======== ===== ===== === == =============== ================\n"); + return 0; + } + + volume = list_entry(v, struct fscache_volume, proc_link); + seq_printf(m, + "%08x %5d %5d %3d %02lx %-15.15s %s\n", + volume->debug_id, + refcount_read(&volume->ref), + atomic_read(&volume->n_cookies), + atomic_read(&volume->n_accesses), + volume->flags, + volume->cache->name ?: "-", + volume->key + 1); + return 0; +} + +static void *fscache_volumes_seq_start(struct seq_file *m, loff_t *_pos) + __acquires(&fscache_addremove_sem) +{ + down_read(&fscache_addremove_sem); + return seq_list_start_head(&fscache_volumes, *_pos); +} + +static void *fscache_volumes_seq_next(struct seq_file *m, void *v, loff_t *_pos) +{ + return seq_list_next(v, &fscache_volumes, _pos); +} + +static void fscache_volumes_seq_stop(struct seq_file *m, void *v) + __releases(&fscache_addremove_sem) +{ + up_read(&fscache_addremove_sem); +} + +const struct seq_operations fscache_volumes_seq_ops = { + .start = fscache_volumes_seq_start, + .next = fscache_volumes_seq_next, + .stop = fscache_volumes_seq_stop, + .show = fscache_volumes_seq_show, +}; +#endif /* CONFIG_PROC_FS */ diff --git a/fs/netfs/internal.h b/fs/netfs/internal.h new file mode 100644 index 000000000..420ee7b26 --- /dev/null +++ b/fs/netfs/internal.h @@ -0,0 +1,553 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Internal definitions for network filesystem support + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef pr_fmt +#undef pr_fmt +#endif + +#define pr_fmt(fmt) "netfs: " fmt + +/* + * buffered_read.c + */ +void netfs_queue_read(struct netfs_io_request *rreq, + struct netfs_io_subrequest *subreq); +void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error); +int netfs_prefetch_for_write(struct file *file, struct folio *folio, + size_t offset, size_t len); + +/* + * buffered_write.c + */ +void netfs_update_i_size(struct netfs_inode *ctx, struct inode *inode, + loff_t pos, size_t copied); + +/* + * main.c + */ +extern unsigned int netfs_debug; +extern struct list_head netfs_io_requests; +extern spinlock_t netfs_proc_lock; +extern mempool_t netfs_request_pool; +extern mempool_t netfs_subrequest_pool; +extern mempool_t netfs_folioq_pool; + +#ifdef CONFIG_PROC_FS +static inline void netfs_proc_add_rreq(struct netfs_io_request *rreq) +{ + spin_lock(&netfs_proc_lock); + list_add_tail_rcu(&rreq->proc_link, &netfs_io_requests); + spin_unlock(&netfs_proc_lock); +} +static inline void netfs_proc_del_rreq(struct netfs_io_request *rreq) +{ + if (!list_empty(&rreq->proc_link)) { + spin_lock(&netfs_proc_lock); + list_del_rcu(&rreq->proc_link); + spin_unlock(&netfs_proc_lock); + } +} +#else +static inline void netfs_proc_add_rreq(struct netfs_io_request *rreq) {} +static inline void netfs_proc_del_rreq(struct netfs_io_request *rreq) {} +#endif + +/* + * misc.c + */ +struct folio_queue *netfs_buffer_make_space(struct netfs_io_request *rreq, + enum netfs_folioq_trace trace); +void netfs_reset_iter(struct netfs_io_subrequest *subreq); +void netfs_wake_collector(struct netfs_io_request *rreq); +void netfs_subreq_clear_in_progress(struct netfs_io_subrequest *subreq); +void netfs_wait_for_in_progress_stream(struct netfs_io_request *rreq, + struct netfs_io_stream *stream); +ssize_t netfs_wait_for_read(struct netfs_io_request *rreq); +ssize_t netfs_wait_for_write(struct netfs_io_request *rreq); +void netfs_wait_for_paused_read(struct netfs_io_request *rreq); +void netfs_wait_for_paused_write(struct netfs_io_request *rreq); + +/* + * objects.c + */ +struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, + struct file *file, + loff_t start, size_t len, + enum netfs_io_origin origin); +void netfs_get_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what); +void netfs_clear_subrequests(struct netfs_io_request *rreq); +void netfs_put_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what); +void netfs_put_failed_request(struct netfs_io_request *rreq); +struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq); + +static inline void netfs_see_request(struct netfs_io_request *rreq, + enum netfs_rreq_ref_trace what) +{ + trace_netfs_rreq_ref(rreq->debug_id, refcount_read(&rreq->ref), what); +} + +static inline void netfs_see_subrequest(struct netfs_io_subrequest *subreq, + enum netfs_sreq_ref_trace what) +{ + trace_netfs_sreq_ref(subreq->rreq->debug_id, subreq->debug_index, + refcount_read(&subreq->ref), what); +} + +/* + * read_collect.c + */ +bool netfs_read_collection(struct netfs_io_request *rreq); +void netfs_read_collection_worker(struct work_struct *work); +void netfs_cancel_read(struct netfs_io_subrequest *subreq, int error); +void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error); + +/* + * read_pgpriv2.c + */ +void netfs_pgpriv2_copy_to_cache(struct netfs_io_request *rreq, struct folio *folio); +void netfs_pgpriv2_end_copy_to_cache(struct netfs_io_request *rreq); +bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *wreq); + +/* + * read_retry.c + */ +void netfs_retry_reads(struct netfs_io_request *rreq); +void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq); + +/* + * stats.c + */ +#ifdef CONFIG_NETFS_STATS +extern atomic_t netfs_n_rh_dio_read; +extern atomic_t netfs_n_rh_readahead; +extern atomic_t netfs_n_rh_read_folio; +extern atomic_t netfs_n_rh_read_single; +extern atomic_t netfs_n_rh_rreq; +extern atomic_t netfs_n_rh_sreq; +extern atomic_t netfs_n_rh_download; +extern atomic_t netfs_n_rh_download_done; +extern atomic_t netfs_n_rh_download_failed; +extern atomic_t netfs_n_rh_download_instead; +extern atomic_t netfs_n_rh_read; +extern atomic_t netfs_n_rh_read_done; +extern atomic_t netfs_n_rh_read_failed; +extern atomic_t netfs_n_rh_zero; +extern atomic_t netfs_n_rh_short_read; +extern atomic_t netfs_n_rh_write; +extern atomic_t netfs_n_rh_write_begin; +extern atomic_t netfs_n_rh_write_done; +extern atomic_t netfs_n_rh_write_failed; +extern atomic_t netfs_n_rh_write_zskip; +extern atomic_t netfs_n_rh_retry_read_req; +extern atomic_t netfs_n_rh_retry_read_subreq; +extern atomic_t netfs_n_wh_buffered_write; +extern atomic_t netfs_n_wh_writethrough; +extern atomic_t netfs_n_wh_dio_write; +extern atomic_t netfs_n_wh_writepages; +extern atomic_t netfs_n_wh_copy_to_cache; +extern atomic_t netfs_n_wh_wstream_conflict; +extern atomic_t netfs_n_wh_upload; +extern atomic_t netfs_n_wh_upload_done; +extern atomic_t netfs_n_wh_upload_failed; +extern atomic_t netfs_n_wh_write; +extern atomic_t netfs_n_wh_write_done; +extern atomic_t netfs_n_wh_write_failed; +extern atomic_t netfs_n_wh_retry_write_req; +extern atomic_t netfs_n_wh_retry_write_subreq; +extern atomic_t netfs_n_wb_lock_skip; +extern atomic_t netfs_n_wb_lock_wait; +extern atomic_t netfs_n_folioq; + +int netfs_stats_show(struct seq_file *m, void *v); + +static inline void netfs_stat(atomic_t *stat) +{ + atomic_inc(stat); +} + +static inline void netfs_stat_d(atomic_t *stat) +{ + atomic_dec(stat); +} + +#else +#define netfs_stat(x) do {} while(0) +#define netfs_stat_d(x) do {} while(0) +#endif + +/* + * write_collect.c + */ +int netfs_folio_written_back(struct folio *folio); +bool netfs_write_collection(struct netfs_io_request *wreq); +void netfs_write_collection_worker(struct work_struct *work); + +/* + * write_issue.c + */ +struct netfs_io_request *netfs_create_write_req(struct address_space *mapping, + struct file *file, + loff_t start, + enum netfs_io_origin origin); +void netfs_prepare_write(struct netfs_io_request *wreq, + struct netfs_io_stream *stream, + loff_t start); +void netfs_reissue_write(struct netfs_io_stream *stream, + struct netfs_io_subrequest *subreq, + struct iov_iter *source); +void netfs_issue_write(struct netfs_io_request *wreq, + struct netfs_io_stream *stream); +size_t netfs_advance_write(struct netfs_io_request *wreq, + struct netfs_io_stream *stream, + loff_t start, size_t len, bool to_eof); +struct netfs_io_request *netfs_begin_writethrough(struct kiocb *iocb, size_t len); +int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc, + struct folio *folio, size_t copied, bool to_page_end, + struct folio **writethrough_cache); +ssize_t netfs_end_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc, + struct folio *writethrough_cache); + +/* + * write_retry.c + */ +void netfs_retry_writes(struct netfs_io_request *wreq); + +/* + * Miscellaneous functions. + */ +static inline bool netfs_is_cache_enabled(struct netfs_inode *ctx) +{ +#if IS_ENABLED(CONFIG_FSCACHE) + struct fscache_cookie *cookie = ctx->cache; + + return fscache_cookie_valid(cookie) && cookie->cache_priv && + fscache_cookie_enabled(cookie); +#else + return false; +#endif +} + +static inline bool netfs_is_cache_maybe_enabled(struct netfs_inode *ctx) +{ +#if IS_ENABLED(CONFIG_FSCACHE) + struct fscache_cookie *cookie = ctx->cache; + + return fscache_cookie_valid(cookie) && + test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags); +#else + return false; +#endif +} + +/* + * Get a ref on a netfs group attached to a dirty page (e.g. a ceph snap). + */ +static inline struct netfs_group *netfs_get_group(struct netfs_group *netfs_group) +{ + if (netfs_group && netfs_group != NETFS_FOLIO_COPY_TO_CACHE) + refcount_inc(&netfs_group->ref); + return netfs_group; +} + +/* + * Dispose of a netfs group attached to a dirty page (e.g. a ceph snap). + */ +static inline void netfs_put_group(struct netfs_group *netfs_group) +{ + if (netfs_group && + netfs_group != NETFS_FOLIO_COPY_TO_CACHE && + refcount_dec_and_test(&netfs_group->ref)) + netfs_group->free(netfs_group); +} + +/* + * Dispose of a netfs group attached to a dirty page (e.g. a ceph snap). + */ +static inline void netfs_put_group_many(struct netfs_group *netfs_group, int nr) +{ + if (netfs_group && + netfs_group != NETFS_FOLIO_COPY_TO_CACHE && + refcount_sub_and_test(nr, &netfs_group->ref)) + netfs_group->free(netfs_group); +} + +/* + * Clear and wake up a NETFS_RREQ_* flag bit on a request. + */ +static inline void netfs_wake_rreq_flag(struct netfs_io_request *rreq, + unsigned int rreq_flag, + enum netfs_rreq_trace trace) +{ + if (test_bit(rreq_flag, &rreq->flags)) { + clear_bit_unlock(rreq_flag, &rreq->flags); + smp_mb__after_atomic(); /* Set flag before task state */ + trace_netfs_rreq(rreq, trace); + wake_up(&rreq->waitq); + } +} + +/* + * Test the NETFS_RREQ_IN_PROGRESS flag, inserting an appropriate barrier. + */ +static inline bool netfs_check_rreq_in_progress(const struct netfs_io_request *rreq) +{ + /* Order read of flags before read of anything else, such as error. */ + return test_bit_acquire(NETFS_RREQ_IN_PROGRESS, &rreq->flags); +} + +/* + * Test the NETFS_SREQ_IN_PROGRESS flag, inserting an appropriate barrier. + */ +static inline bool netfs_check_subreq_in_progress(const struct netfs_io_subrequest *subreq) +{ + /* Order read of flags before read of anything else, such as error. */ + return test_bit_acquire(NETFS_SREQ_IN_PROGRESS, &subreq->flags); +} + +/* + * fscache-cache.c + */ +#ifdef CONFIG_PROC_FS +extern const struct seq_operations fscache_caches_seq_ops; +#endif +bool fscache_begin_cache_access(struct fscache_cache *cache, enum fscache_access_trace why); +void fscache_end_cache_access(struct fscache_cache *cache, enum fscache_access_trace why); +struct fscache_cache *fscache_lookup_cache(const char *name, bool is_cache); +void fscache_put_cache(struct fscache_cache *cache, enum fscache_cache_trace where); + +static inline enum fscache_cache_state fscache_cache_state(const struct fscache_cache *cache) +{ + return smp_load_acquire(&cache->state); +} + +static inline bool fscache_cache_is_live(const struct fscache_cache *cache) +{ + return fscache_cache_state(cache) == FSCACHE_CACHE_IS_ACTIVE; +} + +static inline void fscache_set_cache_state(struct fscache_cache *cache, + enum fscache_cache_state new_state) +{ + smp_store_release(&cache->state, new_state); + +} + +static inline bool fscache_set_cache_state_maybe(struct fscache_cache *cache, + enum fscache_cache_state old_state, + enum fscache_cache_state new_state) +{ + return try_cmpxchg_release(&cache->state, &old_state, new_state); +} + +/* + * fscache-cookie.c + */ +extern struct kmem_cache *fscache_cookie_jar; +#ifdef CONFIG_PROC_FS +extern const struct seq_operations fscache_cookies_seq_ops; +#endif +extern struct timer_list fscache_cookie_lru_timer; + +extern void fscache_print_cookie(struct fscache_cookie *cookie, char prefix); +extern bool fscache_begin_cookie_access(struct fscache_cookie *cookie, + enum fscache_access_trace why); + +static inline void fscache_see_cookie(struct fscache_cookie *cookie, + enum fscache_cookie_trace where) +{ + trace_fscache_cookie(cookie->debug_id, refcount_read(&cookie->ref), + where); +} + +/* + * fscache-main.c + */ +extern unsigned int fscache_hash(unsigned int salt, const void *data, size_t len); +#ifdef CONFIG_FSCACHE +int __init fscache_init(void); +void __exit fscache_exit(void); +#else +static inline int fscache_init(void) { return 0; } +static inline void fscache_exit(void) {} +#endif + +/* + * fscache-proc.c + */ +#ifdef CONFIG_PROC_FS +extern int __init fscache_proc_init(void); +extern void fscache_proc_cleanup(void); +#else +#define fscache_proc_init() (0) +#define fscache_proc_cleanup() do {} while (0) +#endif + +/* + * fscache-stats.c + */ +#ifdef CONFIG_FSCACHE_STATS +extern atomic_t fscache_n_volumes; +extern atomic_t fscache_n_volumes_collision; +extern atomic_t fscache_n_volumes_nomem; +extern atomic_t fscache_n_cookies; +extern atomic_t fscache_n_cookies_lru; +extern atomic_t fscache_n_cookies_lru_expired; +extern atomic_t fscache_n_cookies_lru_removed; +extern atomic_t fscache_n_cookies_lru_dropped; + +extern atomic_t fscache_n_acquires; +extern atomic_t fscache_n_acquires_ok; +extern atomic_t fscache_n_acquires_oom; + +extern atomic_t fscache_n_invalidates; + +extern atomic_t fscache_n_relinquishes; +extern atomic_t fscache_n_relinquishes_retire; +extern atomic_t fscache_n_relinquishes_dropped; + +extern atomic_t fscache_n_resizes; +extern atomic_t fscache_n_resizes_null; + +static inline void fscache_stat(atomic_t *stat) +{ + atomic_inc(stat); +} + +static inline void fscache_stat_d(atomic_t *stat) +{ + atomic_dec(stat); +} + +#define __fscache_stat(stat) (stat) + +int fscache_stats_show(struct seq_file *m); +#else + +#define __fscache_stat(stat) (NULL) +#define fscache_stat(stat) do {} while (0) +#define fscache_stat_d(stat) do {} while (0) + +static inline int fscache_stats_show(struct seq_file *m) { return 0; } +#endif + +/* + * fscache-volume.c + */ +#ifdef CONFIG_PROC_FS +extern const struct seq_operations fscache_volumes_seq_ops; +#endif + +struct fscache_volume *fscache_get_volume(struct fscache_volume *volume, + enum fscache_volume_trace where); +bool fscache_begin_volume_access(struct fscache_volume *volume, + struct fscache_cookie *cookie, + enum fscache_access_trace why); +void fscache_create_volume(struct fscache_volume *volume, bool wait); + +/*****************************************************************************/ +/* + * debug tracing + */ +#define dbgprintk(FMT, ...) \ + printk("[%-6.6s] "FMT"\n", current->comm, ##__VA_ARGS__) + +#define kenter(FMT, ...) dbgprintk("==> %s("FMT")", __func__, ##__VA_ARGS__) +#define kleave(FMT, ...) dbgprintk("<== %s()"FMT"", __func__, ##__VA_ARGS__) +#define kdebug(FMT, ...) dbgprintk(FMT, ##__VA_ARGS__) + +#ifdef __KDEBUG +#define _enter(FMT, ...) kenter(FMT, ##__VA_ARGS__) +#define _leave(FMT, ...) kleave(FMT, ##__VA_ARGS__) +#define _debug(FMT, ...) kdebug(FMT, ##__VA_ARGS__) + +#elif defined(CONFIG_NETFS_DEBUG) +#define _enter(FMT, ...) \ +do { \ + if (netfs_debug) \ + kenter(FMT, ##__VA_ARGS__); \ +} while (0) + +#define _leave(FMT, ...) \ +do { \ + if (netfs_debug) \ + kleave(FMT, ##__VA_ARGS__); \ +} while (0) + +#define _debug(FMT, ...) \ +do { \ + if (netfs_debug) \ + kdebug(FMT, ##__VA_ARGS__); \ +} while (0) + +#else +#define _enter(FMT, ...) no_printk("==> %s("FMT")", __func__, ##__VA_ARGS__) +#define _leave(FMT, ...) no_printk("<== %s()"FMT"", __func__, ##__VA_ARGS__) +#define _debug(FMT, ...) no_printk(FMT, ##__VA_ARGS__) +#endif + +/* + * assertions + */ +#if 1 /* defined(__KDEBUGALL) */ + +#define ASSERT(X) \ +do { \ + if (unlikely(!(X))) { \ + pr_err("\n"); \ + pr_err("Assertion failed\n"); \ + BUG(); \ + } \ +} while (0) + +#define ASSERTCMP(X, OP, Y) \ +do { \ + if (unlikely(!((X) OP (Y)))) { \ + pr_err("\n"); \ + pr_err("Assertion failed\n"); \ + pr_err("%lx " #OP " %lx is false\n", \ + (unsigned long)(X), (unsigned long)(Y)); \ + BUG(); \ + } \ +} while (0) + +#define ASSERTIF(C, X) \ +do { \ + if (unlikely((C) && !(X))) { \ + pr_err("\n"); \ + pr_err("Assertion failed\n"); \ + BUG(); \ + } \ +} while (0) + +#define ASSERTIFCMP(C, X, OP, Y) \ +do { \ + if (unlikely((C) && !((X) OP (Y)))) { \ + pr_err("\n"); \ + pr_err("Assertion failed\n"); \ + pr_err("%lx " #OP " %lx is false\n", \ + (unsigned long)(X), (unsigned long)(Y)); \ + BUG(); \ + } \ +} while (0) + +#else + +#define ASSERT(X) do {} while (0) +#define ASSERTCMP(X, OP, Y) do {} while (0) +#define ASSERTIF(C, X) do {} while (0) +#define ASSERTIFCMP(C, X, OP, Y) do {} while (0) + +#endif /* assert or not */ diff --git a/fs/netfs/iterator.c b/fs/netfs/iterator.c new file mode 100644 index 000000000..b375567e0 --- /dev/null +++ b/fs/netfs/iterator.c @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Iterator helpers. + * + * Copyright (C) 2022 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include "internal.h" + +/** + * netfs_extract_user_iter - Extract the pages from a user iterator into a bvec + * @orig: The original iterator + * @orig_len: The amount of iterator to copy + * @new: The iterator to be set up + * @extraction_flags: Flags to qualify the request + * + * Extract the page fragments from the given amount of the source iterator and + * build up a second iterator that refers to all of those bits. This allows + * the original iterator to be disposed of. + * + * @extraction_flags can have ITER_ALLOW_P2PDMA set to request peer-to-peer DMA be + * allowed on the pages extracted. + * + * On success, the number of elements in the bvec is returned, the original + * iterator will have been advanced by the amount extracted. + * + * The iov_iter_extract_mode() function should be used to query how cleanup + * should be performed. + */ +ssize_t netfs_extract_user_iter(struct iov_iter *orig, size_t orig_len, + struct iov_iter *new, + iov_iter_extraction_t extraction_flags) +{ + struct bio_vec *bv = NULL; + struct page **pages; + unsigned int cur_npages; + unsigned int max_pages; + unsigned int npages = 0; + unsigned int i; + ssize_t ret = 0; + size_t count = orig_len, offset, len; + size_t bv_size, pg_size; + + if (WARN_ON_ONCE(!iter_is_ubuf(orig) && !iter_is_iovec(orig))) + return -EIO; + + max_pages = iov_iter_npages(orig, INT_MAX); + bv_size = array_size(max_pages, sizeof(*bv)); + bv = kvmalloc(bv_size, GFP_KERNEL); + if (!bv) + return -ENOMEM; + + /* Put the page list at the end of the bvec list storage. bvec + * elements are larger than page pointers, so as long as we work + * 0->last, we should be fine. + */ + pg_size = array_size(max_pages, sizeof(*pages)); + pages = (void *)bv + bv_size - pg_size; + + while (count && npages < max_pages) { + ret = iov_iter_extract_pages(orig, &pages, count, + max_pages - npages, extraction_flags, + &offset); + if (unlikely(ret <= 0)) { + ret = ret ?: -EIO; + break; + } + + if (WARN(ret > count, + "%s: extract_pages overrun %zd > %zu bytes\n", + __func__, ret, count)) { + ret = -EIO; + break; + } + + cur_npages = DIV_ROUND_UP(offset + ret, PAGE_SIZE); + if (WARN(cur_npages > max_pages - npages, + "%s: extract_pages overrun %u > %u pages\n", + __func__, npages + cur_npages, max_pages)) { + ret = -EIO; + break; + } + + count -= ret; + ret += offset; + + for (i = 0; i < cur_npages; i++) { + len = ret > PAGE_SIZE ? PAGE_SIZE : ret; + bvec_set_page(bv + npages + i, *pages++, len - offset, offset); + ret -= len; + offset = 0; + } + + npages += cur_npages; + } + + /* Note: Don't try to clean up after EIO. Either we got no pages, so + * nothing to clean up, or we got a buffer overrun, memory corruption + * and can't trust the stuff in the buffer (a WARN was emitted). + */ + + if (ret < 0 && (ret == -ENOMEM || npages == 0)) { + for (i = 0; i < npages; i++) + unpin_user_page(bv[i].bv_page); + kvfree(bv); + return ret; + } + + iov_iter_bvec(new, orig->data_source, bv, npages, orig_len - count); + return npages; +} +EXPORT_SYMBOL_GPL(netfs_extract_user_iter); + +/* + * Select the span of a bvec iterator we're going to use. Limit it by both maximum + * size and maximum number of segments. Returns the size of the span in bytes. + */ +static size_t netfs_limit_bvec(const struct iov_iter *iter, size_t start_offset, + size_t max_size, size_t max_segs) +{ + const struct bio_vec *bvecs = iter->bvec; + unsigned int nbv = iter->nr_segs, ix = 0, nsegs = 0; + size_t len, span = 0, n = iter->count; + size_t skip = iter->iov_offset + start_offset; + + if (WARN_ON(!iov_iter_is_bvec(iter)) || + WARN_ON(start_offset > n) || + n == 0) + return 0; + + while (n && ix < nbv && skip) { + len = bvecs[ix].bv_len; + if (skip < len) + break; + skip -= len; + n -= len; + ix++; + } + + while (n && ix < nbv) { + len = min3(n, bvecs[ix].bv_len - skip, max_size); + span += len; + nsegs++; + ix++; + if (span >= max_size || nsegs >= max_segs) + break; + skip = 0; + n -= len; + } + + return min(span, max_size); +} + +/* + * Select the span of a kvec iterator we're going to use. Limit it by both + * maximum size and maximum number of segments. Returns the size of the span + * in bytes. + */ +static size_t netfs_limit_kvec(const struct iov_iter *iter, size_t start_offset, + size_t max_size, size_t max_segs) +{ + const struct kvec *kvecs = iter->kvec; + unsigned int nkv = iter->nr_segs, ix = 0, nsegs = 0; + size_t len, span = 0, n = iter->count; + size_t skip = iter->iov_offset + start_offset; + + if (WARN_ON(!iov_iter_is_kvec(iter)) || + WARN_ON(start_offset > n) || + n == 0) + return 0; + + while (n && ix < nkv && skip) { + len = kvecs[ix].iov_len; + if (skip < len) + break; + skip -= len; + n -= len; + ix++; + } + + while (n && ix < nkv) { + len = min3(n, kvecs[ix].iov_len - skip, max_size); + span += len; + nsegs++; + ix++; + if (span >= max_size || nsegs >= max_segs) + break; + skip = 0; + n -= len; + } + + return min(span, max_size); +} + +/* + * Select the span of an xarray iterator we're going to use. Limit it by both + * maximum size and maximum number of segments. It is assumed that segments + * can be larger than a page in size, provided they're physically contiguous. + * Returns the size of the span in bytes. + */ +static size_t netfs_limit_xarray(const struct iov_iter *iter, size_t start_offset, + size_t max_size, size_t max_segs) +{ + struct folio *folio; + unsigned int nsegs = 0; + loff_t pos = iter->xarray_start + iter->iov_offset; + pgoff_t index = pos / PAGE_SIZE; + size_t span = 0, n = iter->count; + + XA_STATE(xas, iter->xarray, index); + + if (WARN_ON(!iov_iter_is_xarray(iter)) || + WARN_ON(start_offset > n) || + n == 0) + return 0; + max_size = min(max_size, n - start_offset); + + rcu_read_lock(); + xas_for_each(&xas, folio, ULONG_MAX) { + size_t offset, flen, len; + if (xas_retry(&xas, folio)) + continue; + if (WARN_ON(xa_is_value(folio))) + break; + if (WARN_ON(folio_test_hugetlb(folio))) + break; + + flen = folio_size(folio); + offset = offset_in_folio(folio, pos); + len = min(max_size, flen - offset); + span += len; + nsegs++; + if (span >= max_size || nsegs >= max_segs) + break; + } + + rcu_read_unlock(); + return min(span, max_size); +} + +/* + * Select the span of a folio queue iterator we're going to use. Limit it by + * both maximum size and maximum number of segments. Returns the size of the + * span in bytes. + */ +static size_t netfs_limit_folioq(const struct iov_iter *iter, size_t start_offset, + size_t max_size, size_t max_segs) +{ + const struct folio_queue *folioq = iter->folioq; + unsigned int nsegs = 0; + unsigned int slot = iter->folioq_slot; + size_t span = 0, n = iter->count; + + if (WARN_ON(!iov_iter_is_folioq(iter)) || + WARN_ON(start_offset > n) || + n == 0) + return 0; + max_size = umin(max_size, n - start_offset); + + if (slot >= folioq_nr_slots(folioq)) { + folioq = folioq->next; + slot = 0; + } + + start_offset += iter->iov_offset; + do { + size_t flen = folioq_folio_size(folioq, slot); + + if (start_offset < flen) { + span += flen - start_offset; + nsegs++; + start_offset = 0; + } else { + start_offset -= flen; + } + if (span >= max_size || nsegs >= max_segs) + break; + + slot++; + if (slot >= folioq_nr_slots(folioq)) { + folioq = folioq->next; + slot = 0; + } + } while (folioq); + + return umin(span, max_size); +} + +size_t netfs_limit_iter(const struct iov_iter *iter, size_t start_offset, + size_t max_size, size_t max_segs) +{ + if (iov_iter_is_folioq(iter)) + return netfs_limit_folioq(iter, start_offset, max_size, max_segs); + if (iov_iter_is_bvec(iter)) + return netfs_limit_bvec(iter, start_offset, max_size, max_segs); + if (iov_iter_is_xarray(iter)) + return netfs_limit_xarray(iter, start_offset, max_size, max_segs); + if (iov_iter_is_kvec(iter)) + return netfs_limit_kvec(iter, start_offset, max_size, max_segs); + BUG(); +} +EXPORT_SYMBOL(netfs_limit_iter); diff --git a/fs/netfs/locking.c b/fs/netfs/locking.c new file mode 100644 index 000000000..4e3be2b81 --- /dev/null +++ b/fs/netfs/locking.c @@ -0,0 +1,300 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * I/O and data path helper functionality. + * + * Borrowed from NFS Copyright (c) 2016 Trond Myklebust + */ + +#include +#include +#include "internal.h" + +struct netfs_wb_waiter { + struct list_head link; /* Link in ictx->wb_queue */ + struct task_struct *waiter; /* Waiter task; cleared when lock granted */ +}; + +/* + * inode_dio_wait_interruptible - wait for outstanding DIO requests to finish + * @inode: inode to wait for + * + * Waits for all pending direct I/O requests to finish so that we can + * proceed with a truncate or equivalent operation. + * + * Must be called under a lock that serializes taking new references + * to i_dio_count, usually by inode->i_mutex. + */ +static int netfs_inode_dio_wait_interruptible(struct inode *inode) +{ + if (inode_dio_finished(inode)) + return 0; + + inode_dio_wait_interruptible(inode); + return !inode_dio_finished(inode) ? -ERESTARTSYS : 0; +} + +/* Call with exclusively locked inode->i_rwsem */ +static int netfs_block_o_direct(struct netfs_inode *ictx) +{ + if (!test_bit(NETFS_ICTX_ODIRECT, &ictx->flags)) + return 0; + clear_bit(NETFS_ICTX_ODIRECT, &ictx->flags); + return netfs_inode_dio_wait_interruptible(&ictx->inode); +} + +/** + * netfs_start_io_read - declare the file is being used for buffered reads + * @inode: file inode + * + * Declare that a buffered read operation is about to start, and ensure + * that we block all direct I/O. + * On exit, the function ensures that the NETFS_ICTX_ODIRECT flag is unset, + * and holds a shared lock on inode->i_rwsem to ensure that the flag + * cannot be changed. + * In practice, this means that buffered read operations are allowed to + * execute in parallel, thanks to the shared lock, whereas direct I/O + * operations need to wait to grab an exclusive lock in order to set + * NETFS_ICTX_ODIRECT. + * Note that buffered writes and truncates both take a write lock on + * inode->i_rwsem, meaning that those are serialised w.r.t. the reads. + */ +int netfs_start_io_read(struct inode *inode) + __acquires(inode->i_rwsem) +{ + struct netfs_inode *ictx = netfs_inode(inode); + + /* Be an optimist! */ + if (down_read_interruptible(&inode->i_rwsem) < 0) + return -ERESTARTSYS; + if (test_bit(NETFS_ICTX_ODIRECT, &ictx->flags) == 0) + return 0; + up_read(&inode->i_rwsem); + + /* Slow path.... */ + if (down_write_killable(&inode->i_rwsem) < 0) + return -ERESTARTSYS; + if (netfs_block_o_direct(ictx) < 0) { + up_write(&inode->i_rwsem); + return -ERESTARTSYS; + } + downgrade_write(&inode->i_rwsem); + return 0; +} +EXPORT_SYMBOL(netfs_start_io_read); + +/** + * netfs_end_io_read - declare that the buffered read operation is done + * @inode: file inode + * + * Declare that a buffered read operation is done, and release the shared + * lock on inode->i_rwsem. + */ +void netfs_end_io_read(struct inode *inode) + __releases(inode->i_rwsem) +{ + up_read(&inode->i_rwsem); +} +EXPORT_SYMBOL(netfs_end_io_read); + +/** + * netfs_start_io_write - declare the file is being used for buffered writes + * @inode: file inode + * + * Declare that a buffered read operation is about to start, and ensure + * that we block all direct I/O. + */ +int netfs_start_io_write(struct inode *inode) + __acquires(inode->i_rwsem) +{ + struct netfs_inode *ictx = netfs_inode(inode); + + if (down_write_killable(&inode->i_rwsem) < 0) + return -ERESTARTSYS; + if (netfs_block_o_direct(ictx) < 0) { + up_write(&inode->i_rwsem); + return -ERESTARTSYS; + } + downgrade_write(&inode->i_rwsem); + return 0; +} +EXPORT_SYMBOL(netfs_start_io_write); + +/** + * netfs_end_io_write - declare that the buffered write operation is done + * @inode: file inode + * + * Declare that a buffered write operation is done, and release the + * lock on inode->i_rwsem. + */ +void netfs_end_io_write(struct inode *inode) + __releases(inode->i_rwsem) +{ + up_read(&inode->i_rwsem); +} +EXPORT_SYMBOL(netfs_end_io_write); + +/* Call with exclusively locked inode->i_rwsem */ +static int netfs_block_buffered(struct inode *inode) +{ + struct netfs_inode *ictx = netfs_inode(inode); + int ret; + + if (!test_bit(NETFS_ICTX_ODIRECT, &ictx->flags)) { + set_bit(NETFS_ICTX_ODIRECT, &ictx->flags); + if (inode->i_mapping->nrpages != 0) { + unmap_mapping_range(inode->i_mapping, 0, 0, 0); + ret = filemap_fdatawait(inode->i_mapping); + if (ret < 0) { + clear_bit(NETFS_ICTX_ODIRECT, &ictx->flags); + return ret; + } + } + } + return 0; +} + +/** + * netfs_start_io_direct - declare the file is being used for direct i/o + * @inode: file inode + * + * Declare that a direct I/O operation is about to start, and ensure + * that we block all buffered I/O. + * On exit, the function ensures that the NETFS_ICTX_ODIRECT flag is set, + * and holds a shared lock on inode->i_rwsem to ensure that the flag + * cannot be changed. + * In practice, this means that direct I/O operations are allowed to + * execute in parallel, thanks to the shared lock, whereas buffered I/O + * operations need to wait to grab an exclusive lock in order to clear + * NETFS_ICTX_ODIRECT. + * Note that buffered writes and truncates both take a write lock on + * inode->i_rwsem, meaning that those are serialised w.r.t. O_DIRECT. + */ +int netfs_start_io_direct(struct inode *inode) + __acquires(inode->i_rwsem) +{ + struct netfs_inode *ictx = netfs_inode(inode); + int ret; + + /* Be an optimist! */ + if (down_read_interruptible(&inode->i_rwsem) < 0) + return -ERESTARTSYS; + if (test_bit(NETFS_ICTX_ODIRECT, &ictx->flags) != 0) + return 0; + up_read(&inode->i_rwsem); + + /* Slow path.... */ + if (down_write_killable(&inode->i_rwsem) < 0) + return -ERESTARTSYS; + ret = netfs_block_buffered(inode); + if (ret < 0) { + up_write(&inode->i_rwsem); + return ret; + } + downgrade_write(&inode->i_rwsem); + return 0; +} +EXPORT_SYMBOL(netfs_start_io_direct); + +/** + * netfs_end_io_direct - declare that the direct i/o operation is done + * @inode: file inode + * + * Declare that a direct I/O operation is done, and release the shared + * lock on inode->i_rwsem. + */ +void netfs_end_io_direct(struct inode *inode) + __releases(inode->i_rwsem) +{ + up_read(&inode->i_rwsem); +} +EXPORT_SYMBOL(netfs_end_io_direct); + +/* + * Wait to have exclusive access to writeback. + */ +static bool netfs_wb_begin_wait(struct netfs_inode *ictx) +{ + struct netfs_wb_waiter waiter = {}; + struct task_struct *tsk = current; + bool got = false; + + spin_lock(&ictx->lock); + + if (test_and_set_bit_lock(NETFS_ICTX_WB_LOCK, &ictx->flags)) { + get_task_struct(tsk); + waiter.waiter = tsk; + list_add_tail(&waiter.link, &ictx->wb_queue); + } else { + got = true; + } + spin_unlock(&ictx->lock); + + if (!got) { + for (;;) { + set_current_state(TASK_UNINTERRUPTIBLE); + /* Read waiter before accessing inode state. */ + if (smp_load_acquire(&waiter.waiter) == NULL) + break; + schedule(); + } + } + __set_current_state(TASK_RUNNING); + return true; +} + +/** + * netfs_wb_begin - Begin writeback, waiting if need be + * @ictx: The inode to get writeback access on + * @nowait: Return failure immediately rather than waiting if true + * + * Begin writeback to an inode, waiting for exclusive access if @nowait is + * false. This prevents collection from being done out of order with respect + * to the issuance of write subrequests. + * + * Note that writeback may be ended in a different process (e.g. the collection + * function on a workqueue) than started it. + * + * Return: True if can proceed, false if denied. + */ +bool netfs_wb_begin(struct netfs_inode *ictx, bool nowait) +{ + if (!test_and_set_bit_lock(NETFS_ICTX_WB_LOCK, &ictx->flags)) + return true; + if (nowait) { + netfs_stat(&netfs_n_wb_lock_skip); + return false; + } + netfs_stat(&netfs_n_wb_lock_wait); + return netfs_wb_begin_wait(ictx); +} +EXPORT_SYMBOL(netfs_wb_begin); + +/* netfs_wb_end - End writeback + * @ictx: The inode we have writeback access to + * + * End writeback access on an inode, waking up the next writeback request. + */ +void netfs_wb_end(struct netfs_inode *ictx) +{ + struct netfs_wb_waiter *waiter; + struct task_struct *tsk; + + WARN_ON_ONCE(!test_bit(NETFS_ICTX_WB_LOCK, &ictx->flags)); + + spin_lock(&ictx->lock); + + waiter = list_first_entry_or_null(&ictx->wb_queue, struct netfs_wb_waiter, link); + if (waiter) { + list_del(&waiter->link); + tsk = waiter->waiter; + /* Write inode state before clearing waiter. */ + smp_store_release(&waiter->waiter, NULL); + wake_up_process(tsk); + put_task_struct(tsk); + } else { + clear_bit_unlock(NETFS_ICTX_WB_LOCK, &ictx->flags); + } + + spin_unlock(&ictx->lock); +} +EXPORT_SYMBOL(netfs_wb_end); diff --git a/fs/netfs/main.c b/fs/netfs/main.c new file mode 100644 index 000000000..927badf39 --- /dev/null +++ b/fs/netfs/main.c @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Miscellaneous bits for the netfs support library. + * + * Copyright (C) 2022 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include "internal.h" +#define CREATE_TRACE_POINTS +#include + +MODULE_DESCRIPTION("Network fs support"); +MODULE_AUTHOR("Red Hat, Inc."); +MODULE_LICENSE("GPL"); + +EXPORT_TRACEPOINT_SYMBOL(netfs_sreq); + +unsigned netfs_debug; +module_param_named(debug, netfs_debug, uint, S_IWUSR | S_IRUGO); +MODULE_PARM_DESC(netfs_debug, "Netfs support debugging mask"); + +static struct kmem_cache *netfs_request_slab; +static struct kmem_cache *netfs_subrequest_slab; +mempool_t netfs_request_pool; +mempool_t netfs_subrequest_pool; +mempool_t netfs_folioq_pool; + +#ifdef CONFIG_PROC_FS +LIST_HEAD(netfs_io_requests); +DEFINE_SPINLOCK(netfs_proc_lock); + +static const char *netfs_origins[nr__netfs_io_origin] = { + [NETFS_READAHEAD] = "RA", + [NETFS_READPAGE] = "RP", + [NETFS_READ_GAPS] = "RG", + [NETFS_READ_SINGLE] = "R1", + [NETFS_READ_FOR_WRITE] = "RW", + [NETFS_UNBUFFERED_READ] = "UR", + [NETFS_DIO_READ] = "DR", + [NETFS_WRITEBACK] = "WB", + [NETFS_WRITEBACK_SINGLE] = "W1", + [NETFS_WRITETHROUGH] = "WT", + [NETFS_UNBUFFERED_WRITE] = "UW", + [NETFS_DIO_WRITE] = "DW", + [NETFS_PGPRIV2_COPY_TO_CACHE] = "2C", +}; + +/* + * Generate a list of I/O requests in /proc/fs/netfs/requests + */ +static int netfs_requests_seq_show(struct seq_file *m, void *v) +{ + struct netfs_io_request *rreq; + + if (v == &netfs_io_requests) { + seq_puts(m, + "REQUEST OR REF FLAG ERR OPS COVERAGE\n" + "======== == === ==== ==== === =========\n" + ); + return 0; + } + + rreq = list_entry(v, struct netfs_io_request, proc_link); + seq_printf(m, + "%08x %s %3d %4lx %4ld %3d @%04llx %llx/%llx", + rreq->debug_id, + netfs_origins[rreq->origin], + refcount_read(&rreq->ref), + rreq->flags, + rreq->error, + 0, + rreq->start, rreq->submitted, rreq->len); + seq_putc(m, '\n'); + return 0; +} + +static void *netfs_requests_seq_start(struct seq_file *m, loff_t *_pos) + __acquires(rcu) +{ + rcu_read_lock(); + return seq_list_start_head(&netfs_io_requests, *_pos); +} + +static void *netfs_requests_seq_next(struct seq_file *m, void *v, loff_t *_pos) +{ + return seq_list_next(v, &netfs_io_requests, _pos); +} + +static void netfs_requests_seq_stop(struct seq_file *m, void *v) + __releases(rcu) +{ + rcu_read_unlock(); +} + +static const struct seq_operations netfs_requests_seq_ops = { + .start = netfs_requests_seq_start, + .next = netfs_requests_seq_next, + .stop = netfs_requests_seq_stop, + .show = netfs_requests_seq_show, +}; +#endif /* CONFIG_PROC_FS */ + +static int __init netfs_init(void) +{ + int ret = -ENOMEM; + + if (mempool_init_kmalloc_pool(&netfs_folioq_pool, 100, sizeof(struct folio_queue)) < 0) + goto error_folioq_pool; + + netfs_request_slab = kmem_cache_create("netfs_request", + sizeof(struct netfs_io_request), 0, + SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT, + NULL); + if (!netfs_request_slab) + goto error_req; + + if (mempool_init_slab_pool(&netfs_request_pool, 100, netfs_request_slab) < 0) + goto error_reqpool; + + netfs_subrequest_slab = kmem_cache_create("netfs_subrequest", + sizeof(struct netfs_io_subrequest) + 16, 0, + SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT, + NULL); + if (!netfs_subrequest_slab) + goto error_subreq; + + if (mempool_init_slab_pool(&netfs_subrequest_pool, 100, netfs_subrequest_slab) < 0) + goto error_subreqpool; + +#ifdef CONFIG_PROC_FS + if (!proc_mkdir("fs/netfs", NULL)) + goto error_proc; + if (!proc_create_seq("fs/netfs/requests", S_IFREG | 0444, NULL, + &netfs_requests_seq_ops)) + goto error_procfile; +#endif +#ifdef CONFIG_FSCACHE_STATS + if (!proc_create_single("fs/netfs/stats", S_IFREG | 0444, NULL, + netfs_stats_show)) + goto error_procfile; +#endif + + ret = fscache_init(); + if (ret < 0) + goto error_fscache; + return 0; + +error_fscache: +#ifdef CONFIG_PROC_FS +error_procfile: + remove_proc_subtree("fs/netfs", NULL); +error_proc: +#endif + mempool_exit(&netfs_subrequest_pool); +error_subreqpool: + kmem_cache_destroy(netfs_subrequest_slab); +error_subreq: + mempool_exit(&netfs_request_pool); +error_reqpool: + kmem_cache_destroy(netfs_request_slab); +error_req: + mempool_exit(&netfs_folioq_pool); +error_folioq_pool: + return ret; +} +fs_initcall(netfs_init); + +static void __exit netfs_exit(void) +{ + fscache_exit(); + remove_proc_subtree("fs/netfs", NULL); + mempool_exit(&netfs_subrequest_pool); + kmem_cache_destroy(netfs_subrequest_slab); + mempool_exit(&netfs_request_pool); + kmem_cache_destroy(netfs_request_slab); + mempool_exit(&netfs_folioq_pool); +} +module_exit(netfs_exit); diff --git a/fs/netfs/misc.c b/fs/netfs/misc.c new file mode 100644 index 000000000..5d554512e --- /dev/null +++ b/fs/netfs/misc.c @@ -0,0 +1,565 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Miscellaneous routines. + * + * Copyright (C) 2023 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include "internal.h" + +/** + * netfs_alloc_folioq_buffer - Allocate buffer space into a folio queue + * @mapping: Address space to set on the folio (or NULL). + * @_buffer: Pointer to the folio queue to add to (may point to a NULL; updated). + * @_cur_size: Current size of the buffer (updated). + * @size: Target size of the buffer. + * @gfp: The allocation constraints. + */ +int netfs_alloc_folioq_buffer(struct address_space *mapping, + struct folio_queue **_buffer, + size_t *_cur_size, ssize_t size, gfp_t gfp) +{ + struct folio_queue *tail = *_buffer, *p; + + size = round_up(size, PAGE_SIZE); + if (*_cur_size >= size) + return 0; + + if (tail) + while (tail->next) + tail = tail->next; + + do { + struct folio *folio; + int order = 0, slot; + + if (!tail || folioq_full(tail)) { + p = netfs_folioq_alloc(0, GFP_NOFS, netfs_trace_folioq_alloc_buffer); + if (!p) + return -ENOMEM; + if (tail) { + tail->next = p; + p->prev = tail; + } else { + *_buffer = p; + } + tail = p; + } + + if (size - *_cur_size > PAGE_SIZE) + order = umin(ilog2(size - *_cur_size) - PAGE_SHIFT, + MAX_PAGECACHE_ORDER); + + folio = folio_alloc(gfp, order); + if (!folio && order > 0) + folio = folio_alloc(gfp, 0); + if (!folio) + return -ENOMEM; + + folio->mapping = mapping; + folio->index = *_cur_size / PAGE_SIZE; + trace_netfs_folio(folio, netfs_folio_trace_alloc_buffer); + slot = folioq_append_mark(tail, folio); + *_cur_size += folioq_folio_size(tail, slot); + } while (*_cur_size < size); + + return 0; +} +EXPORT_SYMBOL(netfs_alloc_folioq_buffer); + +/** + * netfs_free_folioq_buffer - Free a folio queue. + * @fq: The start of the folio queue to free + * + * Free up a chain of folio_queues and, if marked, the marked folios they point + * to. + */ +void netfs_free_folioq_buffer(struct folio_queue *fq) +{ + struct folio_queue *next; + struct folio_batch fbatch; + + folio_batch_init(&fbatch); + + for (; fq; fq = next) { + for (int slot = 0; slot < folioq_count(fq); slot++) { + struct folio *folio = folioq_folio(fq, slot); + + if (!folio || + !folioq_is_marked(fq, slot)) + continue; + + trace_netfs_folio(folio, netfs_folio_trace_put); + if (folio_batch_add(&fbatch, folio)) + folio_batch_release(&fbatch); + } + + netfs_stat_d(&netfs_n_folioq); + next = fq->next; + kfree(fq); + } + + folio_batch_release(&fbatch); +} +EXPORT_SYMBOL(netfs_free_folioq_buffer); + +/* + * Reset the subrequest iterator to refer just to the region remaining to be + * read. The iterator may or may not have been advanced by socket ops or + * extraction ops to an extent that may or may not match the amount actually + * read. + */ +void netfs_reset_iter(struct netfs_io_subrequest *subreq) +{ + struct iov_iter *io_iter = &subreq->io_iter; + size_t remain = subreq->len - subreq->transferred; + + if (io_iter->count > remain) + iov_iter_advance(io_iter, io_iter->count - remain); + else if (io_iter->count < remain) + iov_iter_revert(io_iter, remain - io_iter->count); + iov_iter_truncate(&subreq->io_iter, remain); +} + +/** + * netfs_dirty_folio - Mark folio dirty and pin a cache object for writeback + * @mapping: The mapping the folio belongs to. + * @folio: The folio being dirtied. + * + * Set the dirty flag on a folio and pin an in-use cache object in memory so + * that writeback can later write to it. This is intended to be called from + * the filesystem's ->dirty_folio() method. + * + * Return: true if the dirty flag was set on the folio, false otherwise. + */ +bool netfs_dirty_folio(struct address_space *mapping, struct folio *folio) +{ + struct inode *inode = mapping->host; + struct netfs_inode *ictx = netfs_inode(inode); + struct fscache_cookie *cookie = netfs_i_cookie(ictx); + bool need_use = false; + + _enter(""); + + if (!filemap_dirty_folio(mapping, folio)) + return false; + if (!fscache_cookie_valid(cookie)) + return true; + + if (!(inode_state_read_once(inode) & I_PINNING_NETFS_WB)) { + spin_lock(&inode->i_lock); + if (!(inode_state_read(inode) & I_PINNING_NETFS_WB)) { + inode_state_set(inode, I_PINNING_NETFS_WB); + need_use = true; + } + spin_unlock(&inode->i_lock); + + if (need_use) + fscache_use_cookie(cookie, true); + } + return true; +} +EXPORT_SYMBOL(netfs_dirty_folio); + +/** + * netfs_unpin_writeback - Unpin writeback resources + * @inode: The inode on which the cookie resides + * @wbc: The writeback control + * + * Unpin the writeback resources pinned by netfs_dirty_folio(). This is + * intended to be called as/by the netfs's ->write_inode() method. + */ +int netfs_unpin_writeback(struct inode *inode, struct writeback_control *wbc) +{ + struct fscache_cookie *cookie = netfs_i_cookie(netfs_inode(inode)); + + if (wbc->unpinned_netfs_wb) + fscache_unuse_cookie(cookie, NULL, NULL); + return 0; +} +EXPORT_SYMBOL(netfs_unpin_writeback); + +/** + * netfs_clear_inode_writeback - Clear writeback resources pinned by an inode + * @inode: The inode to clean up + * @aux: Auxiliary data to apply to the inode + * + * Clear any writeback resources held by an inode when the inode is evicted. + * This must be called before clear_inode() is called. + */ +void netfs_clear_inode_writeback(struct inode *inode, const void *aux) +{ + struct fscache_cookie *cookie = netfs_i_cookie(netfs_inode(inode)); + + if (inode_state_read_once(inode) & I_PINNING_NETFS_WB) { + loff_t i_size = i_size_read(inode); + fscache_unuse_cookie(cookie, aux, &i_size); + } +} +EXPORT_SYMBOL(netfs_clear_inode_writeback); + +/** + * netfs_invalidate_folio - Invalidate or partially invalidate a folio + * @folio: Folio proposed for release + * @offset: Offset of the invalidated region + * @length: Length of the invalidated region + * + * Invalidate part or all of a folio for a network filesystem. The folio will + * be removed afterwards if the invalidated region covers the entire folio. + */ +void netfs_invalidate_folio(struct folio *folio, size_t offset, size_t length) +{ + struct netfs_folio *finfo; + struct inode *inode = folio_inode(folio); + struct netfs_inode *ctx = netfs_inode(inode); + size_t flen = folio_size(folio); + + _enter("{%lx},%zx,%zx", folio->index, offset, length); + + if (offset == 0 && length == flen) { + unsigned long long i_size, remote_i_size, zero_point; + unsigned long long fpos = folio_pos(folio), end; + + netfs_read_sizes(inode, &i_size, &remote_i_size, &zero_point); + end = umin(fpos + flen, i_size); + if (fpos < i_size && end > zero_point) { + spin_lock(&inode->i_lock); + end = umin(fpos + flen, inode->i_size); + if (fpos < i_size && end > ctx->_zero_point) + netfs_write_zero_point(inode, end); + spin_unlock(&inode->i_lock); + } + } + + folio_wait_private_2(folio); /* [DEPRECATED] */ + + if (!folio_test_private(folio)) + return; + + finfo = netfs_folio_info(folio); + + if (offset == 0 && length >= flen) + goto erase_completely; + + if (finfo) { + /* We have a partially uptodate page from a streaming write. */ + unsigned int fstart = finfo->dirty_offset; + unsigned int fend = fstart + finfo->dirty_len; + unsigned int iend = offset + length; + + if (offset >= fend) + return; + if (iend <= fstart) + return; + + /* The invalidation region overlaps the data. If the region + * covers the start of the data, we either move along the start + * or just erase the data entirely. + */ + if (offset <= fstart) { + if (iend >= fend) + goto erase_completely; + /* Move the start of the data. */ + finfo->dirty_len = fend - iend; + finfo->dirty_offset = iend; + trace_netfs_folio(folio, netfs_folio_trace_invalidate_front); + return; + } + + /* Reduce the length of the data if the invalidation region + * covers the tail part. + */ + if (iend >= fend) { + finfo->dirty_len = offset - fstart; + trace_netfs_folio(folio, netfs_folio_trace_invalidate_tail); + return; + } + + /* A partial write was split. The caller has already zeroed + * it, so just absorb the hole. + */ + trace_netfs_folio(folio, netfs_folio_trace_invalidate_middle); + } + return; + +erase_completely: + netfs_put_group(netfs_folio_group(folio)); + folio_detach_private(folio); + folio_clear_uptodate(folio); + folio_cancel_dirty(folio); + kfree(finfo); + trace_netfs_folio(folio, netfs_folio_trace_invalidate_all); +} +EXPORT_SYMBOL(netfs_invalidate_folio); + +/** + * netfs_release_folio - Try to release a folio + * @folio: Folio proposed for release + * @gfp: Flags qualifying the release + * + * Request release of a folio and clean up its private state if it's not busy. + * Returns true if the folio can now be released, false if not + */ +bool netfs_release_folio(struct folio *folio, gfp_t gfp) +{ + struct inode *inode = folio_inode(folio); + struct netfs_inode *ctx = netfs_inode(inode); + unsigned long long i_size, remote_i_size, zero_point, end; + + if (folio_test_dirty(folio)) + return false; + + netfs_read_sizes(inode, &i_size, &remote_i_size, &zero_point); + end = folio_next_pos(folio); + if (end > zero_point) { + spin_lock(&inode->i_lock); + end = umin(end, ctx->_remote_i_size); + if (end > ctx->_zero_point) + netfs_write_zero_point(inode, end); + spin_unlock(&inode->i_lock); + } + + if (folio_test_private(folio)) + return false; + if (unlikely(folio_test_private_2(folio))) { /* [DEPRECATED] */ + if (current_is_kswapd() || !(gfp & __GFP_FS)) + return false; + folio_wait_private_2(folio); + } + fscache_note_page_release(netfs_i_cookie(ctx)); + return true; +} +EXPORT_SYMBOL(netfs_release_folio); + +/* + * Wake the collection work item. + */ +void netfs_wake_collector(struct netfs_io_request *rreq) +{ + if (test_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &rreq->flags) && + !test_bit(NETFS_RREQ_RETRYING, &rreq->flags)) { + queue_work(system_dfl_wq, &rreq->work); + } else { + trace_netfs_rreq(rreq, netfs_rreq_trace_wake_queue); + wake_up(&rreq->waitq); + } +} + +/* + * Mark a subrequest as no longer being in progress and, if need be, wake the + * collector. + */ +void netfs_subreq_clear_in_progress(struct netfs_io_subrequest *subreq) +{ + struct netfs_io_request *rreq = subreq->rreq; + struct netfs_io_stream *stream = &rreq->io_streams[subreq->stream_nr]; + + clear_bit_unlock(NETFS_SREQ_IN_PROGRESS, &subreq->flags); + smp_mb__after_atomic(); /* Clear IN_PROGRESS before task state */ + + /* If we are at the head of the queue, wake up the collector. */ + if (list_is_first(&subreq->rreq_link, &stream->subrequests) || + test_bit(NETFS_RREQ_RETRYING, &rreq->flags)) + netfs_wake_collector(rreq); +} + +/* + * Wait for all outstanding I/O in a stream to quiesce. + */ +void netfs_wait_for_in_progress_stream(struct netfs_io_request *rreq, + struct netfs_io_stream *stream) +{ + struct netfs_io_subrequest *subreq; + DEFINE_WAIT(myself); + + list_for_each_entry(subreq, &stream->subrequests, rreq_link) { + smp_rmb(); /* Read ->next before IN_PROGRESS. */ + if (!netfs_check_subreq_in_progress(subreq)) + continue; + + trace_netfs_rreq(rreq, netfs_rreq_trace_wait_quiesce); + for (;;) { + prepare_to_wait(&rreq->waitq, &myself, TASK_UNINTERRUPTIBLE); + + if (!netfs_check_subreq_in_progress(subreq)) + break; + + trace_netfs_sreq(subreq, netfs_sreq_trace_wait_for); + schedule(); + } + } + + trace_netfs_rreq(rreq, netfs_rreq_trace_waited_quiesce); + finish_wait(&rreq->waitq, &myself); +} + +/* + * Perform collection in app thread if not offloaded to workqueue. + */ +static int netfs_collect_in_app(struct netfs_io_request *rreq, + bool (*collector)(struct netfs_io_request *rreq)) +{ + bool need_collect = false, inactive = true, done = true; + + if (!netfs_check_rreq_in_progress(rreq)) { + trace_netfs_rreq(rreq, netfs_rreq_trace_recollect); + return 1; /* Done */ + } + + for (int i = 0; i < NR_IO_STREAMS; i++) { + struct netfs_io_subrequest *subreq; + struct netfs_io_stream *stream = &rreq->io_streams[i]; + + if (!stream->active) + continue; + inactive = false; + trace_netfs_collect_stream(rreq, stream); + subreq = list_first_entry_or_null(&stream->subrequests, + struct netfs_io_subrequest, + rreq_link); + if (subreq && + (!netfs_check_subreq_in_progress(subreq) || + test_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags))) { + need_collect = true; + break; + } + if (subreq || !test_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags)) + done = false; + } + + if (!need_collect && !inactive && !done) + return 0; /* Sleep */ + + __set_current_state(TASK_RUNNING); + if (collector(rreq)) { + /* Drop the ref from the NETFS_RREQ_IN_PROGRESS flag. */ + netfs_put_request(rreq, netfs_rreq_trace_put_work_ip); + return 1; /* Done */ + } + + if (inactive) { + WARN(true, "Failed to collect inactive req R=%08x\n", + rreq->debug_id); + cond_resched(); + } + return 2; /* Again */ +} + +/* + * Wait for a request to complete, successfully or otherwise. + */ +static ssize_t netfs_wait_for_in_progress(struct netfs_io_request *rreq, + bool (*collector)(struct netfs_io_request *rreq)) +{ + DEFINE_WAIT(myself); + ssize_t ret; + + for (;;) { + prepare_to_wait(&rreq->waitq, &myself, TASK_UNINTERRUPTIBLE); + + if (!test_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &rreq->flags)) { + switch (netfs_collect_in_app(rreq, collector)) { + case 0: + break; + case 1: + goto all_collected; + case 2: + if (!netfs_check_rreq_in_progress(rreq)) + break; + cond_resched(); + continue; + } + } + + if (!netfs_check_rreq_in_progress(rreq)) + break; + + trace_netfs_rreq(rreq, netfs_rreq_trace_wait_ip); + schedule(); + } + +all_collected: + trace_netfs_rreq(rreq, netfs_rreq_trace_waited_ip); + finish_wait(&rreq->waitq, &myself); + + ret = rreq->error; + if (ret == 0) { + ret = rreq->transferred; + switch (rreq->origin) { + case NETFS_DIO_READ: + case NETFS_DIO_WRITE: + case NETFS_READ_SINGLE: + case NETFS_UNBUFFERED_READ: + case NETFS_UNBUFFERED_WRITE: + break; + default: + if (rreq->submitted < rreq->len) { + trace_netfs_failure(rreq, NULL, ret, netfs_fail_short_read); + ret = -EIO; + } + break; + } + } + + return ret; +} + +ssize_t netfs_wait_for_read(struct netfs_io_request *rreq) +{ + return netfs_wait_for_in_progress(rreq, netfs_read_collection); +} + +ssize_t netfs_wait_for_write(struct netfs_io_request *rreq) +{ + return netfs_wait_for_in_progress(rreq, netfs_write_collection); +} + +/* + * Wait for a paused operation to unpause or complete in some manner. + */ +static void netfs_wait_for_pause(struct netfs_io_request *rreq, + bool (*collector)(struct netfs_io_request *rreq)) +{ + DEFINE_WAIT(myself); + + for (;;) { + trace_netfs_rreq(rreq, netfs_rreq_trace_wait_pause); + prepare_to_wait(&rreq->waitq, &myself, TASK_UNINTERRUPTIBLE); + + if (!test_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &rreq->flags)) { + switch (netfs_collect_in_app(rreq, collector)) { + case 0: + break; + case 1: + goto all_collected; + case 2: + if (!netfs_check_rreq_in_progress(rreq) || + !test_bit(NETFS_RREQ_PAUSE, &rreq->flags)) + break; + cond_resched(); + continue; + } + } + + if (!netfs_check_rreq_in_progress(rreq) || + !test_bit(NETFS_RREQ_PAUSE, &rreq->flags)) + break; + + schedule(); + } + +all_collected: + trace_netfs_rreq(rreq, netfs_rreq_trace_waited_pause); + finish_wait(&rreq->waitq, &myself); +} + +void netfs_wait_for_paused_read(struct netfs_io_request *rreq) +{ + return netfs_wait_for_pause(rreq, netfs_read_collection); +} + +void netfs_wait_for_paused_write(struct netfs_io_request *rreq) +{ + return netfs_wait_for_pause(rreq, netfs_write_collection); +} diff --git a/fs/netfs/objects.c b/fs/netfs/objects.c new file mode 100644 index 000000000..01461a746 --- /dev/null +++ b/fs/netfs/objects.c @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Object lifetime handling and tracing. + * + * Copyright (C) 2022 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include "internal.h" + +static void netfs_free_request(struct work_struct *work); + +/* + * Allocate an I/O request and initialise it. + */ +struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, + struct file *file, + loff_t start, size_t len, + enum netfs_io_origin origin) +{ + static atomic_t debug_ids; + struct inode *inode = file ? file_inode(file) : mapping->host; + struct netfs_inode *ctx = netfs_inode(inode); + struct netfs_io_request *rreq; + mempool_t *mempool = ctx->ops->request_pool ?: &netfs_request_pool; + struct kmem_cache *cache = mempool->pool_data; + gfp_t gfp = GFP_KERNEL; + int ret; + + /* Writeback is part of memory reclaim and must not fail due to ENOMEM. */ + if (origin == NETFS_WRITEBACK || origin == NETFS_WRITEBACK_SINGLE) { + gfp = GFP_NOFS; /* Allows use of mempools. */ + + rreq = mempool_alloc(mempool, gfp); + } else { + rreq = mempool->alloc(gfp, mempool->pool_data); + if (!rreq) + return ERR_PTR(-ENOMEM); + } + + memset(rreq, 0, kmem_cache_size(cache)); + INIT_WORK(&rreq->cleanup_work, netfs_free_request); + rreq->gfp = gfp; + rreq->start = start; + rreq->len = len; + rreq->origin = origin; + rreq->netfs_ops = ctx->ops; + rreq->mapping = mapping; + rreq->inode = inode; + rreq->i_size = i_size_read(inode); + rreq->debug_id = atomic_inc_return(&debug_ids); + rreq->wsize = INT_MAX; + rreq->io_streams[0].sreq_max_len = ULONG_MAX; + rreq->io_streams[0].sreq_max_segs = 0; + spin_lock_init(&rreq->lock); + INIT_LIST_HEAD(&rreq->io_streams[0].subrequests); + INIT_LIST_HEAD(&rreq->io_streams[1].subrequests); + init_waitqueue_head(&rreq->waitq); + refcount_set(&rreq->ref, 2); + + if (origin == NETFS_READAHEAD || + origin == NETFS_READPAGE || + origin == NETFS_READ_GAPS || + origin == NETFS_READ_SINGLE || + origin == NETFS_READ_FOR_WRITE || + origin == NETFS_UNBUFFERED_READ || + origin == NETFS_DIO_READ) { + INIT_WORK(&rreq->work, netfs_read_collection_worker); + rreq->io_streams[0].avail = true; + } else { + INIT_WORK(&rreq->work, netfs_write_collection_worker); + } + + __set_bit(NETFS_RREQ_IN_PROGRESS, &rreq->flags); + if (rreq->netfs_ops->init_request) { + ret = rreq->netfs_ops->init_request(rreq, file); + if (ret < 0) { + mempool_free(rreq, rreq->netfs_ops->request_pool ?: &netfs_request_pool); + return ERR_PTR(ret); + } + } + + atomic_inc(&ctx->io_count); + trace_netfs_rreq_ref(rreq->debug_id, refcount_read(&rreq->ref), netfs_rreq_trace_new); + netfs_proc_add_rreq(rreq); + netfs_stat(&netfs_n_rh_rreq); + return rreq; +} + +void netfs_get_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what) +{ + int r; + + __refcount_inc(&rreq->ref, &r); + trace_netfs_rreq_ref(rreq->debug_id, r + 1, what); +} + +void netfs_clear_subrequests(struct netfs_io_request *rreq) +{ + struct netfs_io_subrequest *subreq; + struct netfs_io_stream *stream; + int s; + + for (s = 0; s < ARRAY_SIZE(rreq->io_streams); s++) { + stream = &rreq->io_streams[s]; + while (!list_empty(&stream->subrequests)) { + subreq = list_first_entry(&stream->subrequests, + struct netfs_io_subrequest, rreq_link); + list_del(&subreq->rreq_link); + netfs_put_subrequest(subreq, netfs_sreq_trace_put_clear); + } + } +} + +static void netfs_free_request_rcu(struct rcu_head *rcu) +{ + struct netfs_io_request *rreq = container_of(rcu, struct netfs_io_request, rcu); + + mempool_free(rreq, rreq->netfs_ops->request_pool ?: &netfs_request_pool); + netfs_stat_d(&netfs_n_rh_rreq); +} + +static void netfs_deinit_request(struct netfs_io_request *rreq) +{ + struct netfs_inode *ictx = netfs_inode(rreq->inode); + unsigned int i; + + trace_netfs_rreq(rreq, netfs_rreq_trace_free); + + /* Cancel/flush the result collection worker. That does not carry a + * ref of its own, so we must wait for it somewhere. + */ + cancel_work_sync(&rreq->work); + + netfs_proc_del_rreq(rreq); + netfs_clear_subrequests(rreq); + if (rreq->netfs_ops->free_request) + rreq->netfs_ops->free_request(rreq); + if (rreq->cache_resources.ops) + rreq->cache_resources.ops->end_operation(&rreq->cache_resources); + if (rreq->direct_bv) { + for (i = 0; i < rreq->direct_bv_count; i++) { + if (rreq->direct_bv[i].bv_page) { + if (rreq->direct_bv_unpin) + unpin_user_page(rreq->direct_bv[i].bv_page); + } + } + kvfree(rreq->direct_bv); + } + rolling_buffer_clear(&rreq->buffer); + + if (atomic_dec_and_test(&ictx->io_count)) + wake_up_var(&ictx->io_count); +} + +static void netfs_free_request(struct work_struct *work) +{ + struct netfs_io_request *rreq = + container_of(work, struct netfs_io_request, cleanup_work); + + netfs_deinit_request(rreq); + call_rcu(&rreq->rcu, netfs_free_request_rcu); +} + +void netfs_put_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what) +{ + unsigned int debug_id; + bool dead; + int r; + + if (rreq) { + debug_id = rreq->debug_id; + dead = __refcount_dec_and_test(&rreq->ref, &r); + trace_netfs_rreq_ref(debug_id, r - 1, what); + if (dead) + WARN_ON(!queue_work(system_dfl_wq, &rreq->cleanup_work)); + } +} + +/* + * Free a request (synchronously) that was just allocated but has + * failed before it could be submitted. + */ +void netfs_put_failed_request(struct netfs_io_request *rreq) +{ + int r = refcount_read(&rreq->ref); + + /* new requests have two references (see + * netfs_alloc_request(), and this function is only allowed on + * new request objects + */ + WARN_ON_ONCE(r != 2); + + trace_netfs_rreq_ref(rreq->debug_id, r, netfs_rreq_trace_put_failed); + netfs_free_request(&rreq->cleanup_work); +} + +/* + * Allocate and partially initialise an I/O request structure. + */ +struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq) +{ + struct netfs_io_subrequest *subreq; + mempool_t *mempool = rreq->netfs_ops->subrequest_pool ?: &netfs_subrequest_pool; + struct kmem_cache *cache = mempool->pool_data; + + if (rreq->gfp == GFP_KERNEL) + subreq = mempool->alloc(rreq->gfp, mempool->pool_data); + else + subreq = mempool_alloc(mempool, rreq->gfp); + if (!subreq) + return NULL; + + memset(subreq, 0, kmem_cache_size(cache)); + INIT_WORK(&subreq->work, NULL); + INIT_LIST_HEAD(&subreq->rreq_link); + refcount_set(&subreq->ref, 2); + subreq->rreq = rreq; + subreq->debug_index = atomic_inc_return(&rreq->subreq_counter); + netfs_get_request(rreq, netfs_rreq_trace_get_subreq); + netfs_stat(&netfs_n_rh_sreq); + return subreq; +} + +void netfs_get_subrequest(struct netfs_io_subrequest *subreq, + enum netfs_sreq_ref_trace what) +{ + int r; + + __refcount_inc(&subreq->ref, &r); + trace_netfs_sreq_ref(subreq->rreq->debug_id, subreq->debug_index, r + 1, + what); +} + +static void netfs_free_subrequest(struct netfs_io_subrequest *subreq) +{ + struct netfs_io_request *rreq = subreq->rreq; + + trace_netfs_sreq(subreq, netfs_sreq_trace_free); + if (rreq->netfs_ops->free_subrequest) + rreq->netfs_ops->free_subrequest(subreq); + mempool_free(subreq, rreq->netfs_ops->subrequest_pool ?: &netfs_subrequest_pool); + netfs_stat_d(&netfs_n_rh_sreq); + netfs_put_request(rreq, netfs_rreq_trace_put_subreq); +} + +void netfs_put_subrequest(struct netfs_io_subrequest *subreq, + enum netfs_sreq_ref_trace what) +{ + unsigned int debug_index = subreq->debug_index; + unsigned int debug_id = subreq->rreq->debug_id; + bool dead; + int r; + + dead = __refcount_dec_and_test(&subreq->ref, &r); + trace_netfs_sreq_ref(debug_id, debug_index, r - 1, what); + if (dead) + netfs_free_subrequest(subreq); +} diff --git a/fs/netfs/read_collect.c b/fs/netfs/read_collect.c new file mode 100644 index 000000000..23660a590 --- /dev/null +++ b/fs/netfs/read_collect.c @@ -0,0 +1,608 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Network filesystem read subrequest result collection, assessment and + * retrying. + * + * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include "internal.h" + +/* Notes made in the collector */ +#define HIT_PENDING 0x01 /* A front op was still pending */ +#define MADE_PROGRESS 0x04 /* Made progress cleaning up a stream or the folio set */ +#define BUFFERED 0x08 /* The pagecache needs cleaning up */ +#define NEED_RETRY 0x10 /* A front op requests retrying */ +#define COPY_TO_CACHE 0x40 /* Need to copy subrequest to cache */ +#define ABANDON_SREQ 0x80 /* Need to abandon untransferred part of subrequest */ + +/* + * Clear the unread part of an I/O request. + */ +static void netfs_clear_unread(struct netfs_io_subrequest *subreq) +{ + netfs_reset_iter(subreq); + WARN_ON_ONCE(subreq->len - subreq->transferred != iov_iter_count(&subreq->io_iter)); + iov_iter_zero(iov_iter_count(&subreq->io_iter), &subreq->io_iter); + if (subreq->start + subreq->transferred >= subreq->rreq->i_size) + __set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags); +} + +/* + * Flush, mark and unlock a folio that's now completely read. If we want to + * cache the folio, we set the group to NETFS_FOLIO_COPY_TO_CACHE, mark it + * dirty and let writeback handle it. + */ +static void netfs_unlock_read_folio(struct netfs_io_request *rreq, + struct folio_queue *folioq, + int slot) +{ + struct netfs_folio *finfo; + struct folio *folio = folioq_folio(folioq, slot); + + if (unlikely(folio_pos(folio) < rreq->abandon_to)) { + trace_netfs_folio(folio, netfs_folio_trace_abandon); + goto just_unlock; + } + + flush_dcache_folio(folio); + folio_mark_uptodate(folio); + + if (!test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)) { + finfo = netfs_folio_info(folio); + if (finfo) { + trace_netfs_folio(folio, netfs_folio_trace_filled_gaps); + if (finfo->netfs_group) + folio_change_private(folio, finfo->netfs_group); + else + folio_detach_private(folio); + kfree(finfo); + } + + if (test_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &rreq->flags)) { + if (!WARN_ON_ONCE(folio_get_private(folio) != NULL)) { + trace_netfs_folio(folio, netfs_folio_trace_copy_to_cache); + folio_attach_private(folio, NETFS_FOLIO_COPY_TO_CACHE); + folio_mark_dirty(folio); + } + } else { + trace_netfs_folio(folio, netfs_folio_trace_read_done); + } + + folioq_clear(folioq, slot); + } else { + // TODO: Use of PG_private_2 is deprecated. + if (test_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &rreq->flags)) + netfs_pgpriv2_copy_to_cache(rreq, folio); + } + +just_unlock: + if (folio == rreq->no_unlock_folio && + test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags)) { + _debug("no unlock"); + } else { + trace_netfs_folio(folio, netfs_folio_trace_read_unlock); + folio_unlock(folio); + } + + folioq_clear(folioq, slot); +} + +/* + * Unlock any folios we've finished with. + */ +static void netfs_read_unlock_folios(struct netfs_io_request *rreq, + unsigned int *notes) +{ + struct folio_queue *folioq = rreq->buffer.tail; + unsigned long long collected_to = rreq->collected_to; + unsigned int slot = rreq->buffer.first_tail_slot; + + if (rreq->cleaned_to >= rreq->collected_to) + return; + + // TODO: Begin decryption + + if (slot >= folioq_nr_slots(folioq)) { + folioq = rolling_buffer_delete_spent(&rreq->buffer); + if (!folioq) { + rreq->front_folio_order = 0; + return; + } + slot = 0; + } + + for (;;) { + struct folio *folio; + unsigned long long fpos, fend; + unsigned int order; + size_t fsize; + + if (*notes & COPY_TO_CACHE) + set_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &rreq->flags); + + folio = folioq_folio(folioq, slot); + if (WARN_ONCE(!folio_test_locked(folio), + "R=%08x: folio %lx is not locked\n", + rreq->debug_id, folio->index)) + trace_netfs_folio(folio, netfs_folio_trace_not_locked); + + order = folioq_folio_order(folioq, slot); + rreq->front_folio_order = order; + fsize = PAGE_SIZE << order; + fpos = folio_pos(folio); + fend = fpos + fsize; + + trace_netfs_collect_folio(rreq, folio, fend, collected_to); + + /* Unlock any folio we've transferred all of. */ + if (collected_to < fend) + break; + + netfs_unlock_read_folio(rreq, folioq, slot); + WRITE_ONCE(rreq->cleaned_to, fpos + fsize); + *notes |= MADE_PROGRESS; + + clear_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &rreq->flags); + + /* Clean up the head folioq. If we clear an entire folioq, then + * we can get rid of it provided it's not also the tail folioq + * being filled by the issuer. + */ + folioq_clear(folioq, slot); + slot++; + if (slot >= folioq_nr_slots(folioq)) { + folioq = rolling_buffer_delete_spent(&rreq->buffer); + if (!folioq) + goto done; + slot = 0; + trace_netfs_folioq(folioq, netfs_trace_folioq_read_progress); + } + + if (fpos + fsize >= collected_to) + break; + } + + rreq->buffer.tail = folioq; +done: + rreq->buffer.first_tail_slot = slot; +} + +/* + * Collect and assess the results of various read subrequests. We may need to + * retry some of the results. + * + * Note that we have a sequence of subrequests, which may be drawing on + * different sources and may or may not be the same size or starting position + * and may not even correspond in boundary alignment. + */ +static void netfs_collect_read_results(struct netfs_io_request *rreq) +{ + struct netfs_io_subrequest *front, *remove; + struct netfs_io_stream *stream = &rreq->io_streams[0]; + unsigned int notes; + + _enter("%llx-%llx", rreq->start, rreq->start + rreq->len); + trace_netfs_rreq(rreq, netfs_rreq_trace_collect); + trace_netfs_collect(rreq); + +reassess: + if (rreq->origin == NETFS_READAHEAD || + rreq->origin == NETFS_READPAGE || + rreq->origin == NETFS_READ_FOR_WRITE) + notes = BUFFERED; + else + notes = 0; + + /* Remove completed subrequests from the front of the stream and + * advance the completion point. We stop when we hit something that's + * in progress. The issuer thread may be adding stuff to the tail + * whilst we're doing this. + */ + front = list_first_entry_or_null_acquire(&stream->subrequests, + struct netfs_io_subrequest, rreq_link); + /* Read first subreq pointer before IN_PROGRESS flag. */ + + while (front) { + size_t transferred; + + trace_netfs_collect_sreq(rreq, front); + _debug("sreq [%x] %llx %zx/%zx", + front->debug_index, front->start, front->transferred, front->len); + + if (stream->collected_to < front->start) { + trace_netfs_collect_gap(rreq, stream, front->start, 'F'); + stream->collected_to = front->start; + } + + if (netfs_check_subreq_in_progress(front)) + notes |= HIT_PENDING; + smp_rmb(); /* Read counters after IN_PROGRESS flag. */ + transferred = READ_ONCE(front->transferred); + + /* If we can now collect the next folio, do so. We don't want + * to defer this as we have to decide whether we need to copy + * to the cache or not, and that may differ between adjacent + * subreqs. + */ + if (notes & BUFFERED) { + size_t fsize = PAGE_SIZE << rreq->front_folio_order; + + /* Clear the tail of a short read. */ + if (!(notes & HIT_PENDING) && + front->error == 0 && + transferred < front->len && + (test_bit(NETFS_SREQ_HIT_EOF, &front->flags) || + test_bit(NETFS_SREQ_CLEAR_TAIL, &front->flags))) { + netfs_clear_unread(front); + transferred = front->transferred = front->len; + trace_netfs_sreq(front, netfs_sreq_trace_clear); + } + + stream->collected_to = front->start + transferred; + rreq->collected_to = stream->collected_to; + + if (test_bit(NETFS_SREQ_COPY_TO_CACHE, &front->flags)) + notes |= COPY_TO_CACHE; + + if (test_bit(NETFS_SREQ_FAILED, &front->flags)) { + rreq->abandon_to = front->start + front->len; + front->transferred = front->len; + transferred = front->len; + trace_netfs_rreq(rreq, netfs_rreq_trace_set_abandon); + } + if (front->start + transferred >= rreq->cleaned_to + fsize || + test_bit(NETFS_SREQ_HIT_EOF, &front->flags)) + netfs_read_unlock_folios(rreq, ¬es); + } else { + stream->collected_to = front->start + transferred; + rreq->collected_to = stream->collected_to; + } + + /* Stall if the front is still undergoing I/O. */ + if (notes & HIT_PENDING) + break; + + if (test_bit(NETFS_SREQ_FAILED, &front->flags)) { + if (!stream->failed) { + stream->error = front->error; + rreq->error = front->error; + set_bit(NETFS_RREQ_FAILED, &rreq->flags); + stream->failed = true; + } + notes |= MADE_PROGRESS | ABANDON_SREQ; + } else if (test_bit(NETFS_SREQ_NEED_RETRY, &front->flags)) { + stream->need_retry = true; + notes |= NEED_RETRY | MADE_PROGRESS; + break; + } else if (test_bit(NETFS_RREQ_SHORT_TRANSFER, &rreq->flags)) { + notes |= MADE_PROGRESS; + } else { + if (!stream->failed) { + stream->transferred += transferred; + stream->transferred_valid = true; + } + if (front->transferred < front->len) + set_bit(NETFS_RREQ_SHORT_TRANSFER, &rreq->flags); + notes |= MADE_PROGRESS; + } + + /* Remove if completely consumed. */ + stream->source = front->source; + spin_lock(&rreq->lock); + + remove = front; + trace_netfs_sreq(front, + notes & ABANDON_SREQ ? + netfs_sreq_trace_abandoned : netfs_sreq_trace_consumed); + list_del_init(&front->rreq_link); + front = list_first_entry_or_null(&stream->subrequests, + struct netfs_io_subrequest, rreq_link); + spin_unlock(&rreq->lock); + netfs_put_subrequest(remove, + notes & ABANDON_SREQ ? + netfs_sreq_trace_put_abandon : + netfs_sreq_trace_put_done); + } + + trace_netfs_collect_stream(rreq, stream); + trace_netfs_collect_state(rreq, rreq->collected_to, notes); + + if (!(notes & BUFFERED)) + rreq->cleaned_to = rreq->collected_to; + + if (notes & NEED_RETRY) + goto need_retry; + if (notes & MADE_PROGRESS) { + netfs_wake_rreq_flag(rreq, NETFS_RREQ_PAUSE, netfs_rreq_trace_unpause); + //cond_resched(); + goto reassess; + } + +out: + _leave(" = %x", notes); + return; + +need_retry: + /* Okay... We're going to have to retry parts of the stream. Note + * that any partially completed op will have had any wholly transferred + * folios removed from it. + */ + _debug("retry"); + netfs_retry_reads(rreq); + goto out; +} + +/* + * Do page flushing and suchlike after DIO. + */ +static void netfs_rreq_assess_dio(struct netfs_io_request *rreq) +{ + unsigned int i; + + if (rreq->origin == NETFS_UNBUFFERED_READ || + rreq->origin == NETFS_DIO_READ) { + for (i = 0; i < rreq->direct_bv_count; i++) { + flush_dcache_page(rreq->direct_bv[i].bv_page); + // TODO: cifs marks pages in the destination buffer + // dirty under some circumstances after a read. Do we + // need to do that too? + set_page_dirty(rreq->direct_bv[i].bv_page); + } + } + + if (rreq->iocb) { + rreq->iocb->ki_pos += rreq->transferred; + if (rreq->iocb->ki_complete) { + trace_netfs_rreq(rreq, netfs_rreq_trace_ki_complete); + rreq->iocb->ki_complete( + rreq->iocb, rreq->error ? rreq->error : rreq->transferred); + } + } + if (rreq->netfs_ops->done) + rreq->netfs_ops->done(rreq); + if (rreq->origin == NETFS_UNBUFFERED_READ || + rreq->origin == NETFS_DIO_READ) + inode_dio_end(rreq->inode); +} + +/* + * Do processing after reading a monolithic single object. + */ +static void netfs_rreq_assess_single(struct netfs_io_request *rreq) +{ + struct netfs_io_stream *stream = &rreq->io_streams[0]; + + if (!rreq->error && stream->source == NETFS_DOWNLOAD_FROM_SERVER && + fscache_resources_valid(&rreq->cache_resources)) { + trace_netfs_rreq(rreq, netfs_rreq_trace_dirty); + netfs_single_mark_inode_dirty(rreq->inode); + } + + if (rreq->iocb) { + rreq->iocb->ki_pos += rreq->transferred; + if (rreq->iocb->ki_complete) { + trace_netfs_rreq(rreq, netfs_rreq_trace_ki_complete); + rreq->iocb->ki_complete( + rreq->iocb, rreq->error ? rreq->error : rreq->transferred); + } + } + if (rreq->netfs_ops->done) + rreq->netfs_ops->done(rreq); +} + +/* + * Perform the collection of subrequests and folios. + * + * Note that we're in normal kernel thread context at this point, possibly + * running on a workqueue. + */ +bool netfs_read_collection(struct netfs_io_request *rreq) +{ + struct netfs_io_stream *stream = &rreq->io_streams[0]; + + netfs_collect_read_results(rreq); + + /* We're done when the app thread has finished posting subreqs and the + * queue is empty. + */ + if (!test_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags)) + return false; + smp_rmb(); /* Read ALL_QUEUED before subreq lists. */ + + if (!list_empty(&stream->subrequests)) + return false; + + /* Okay, declare that all I/O is complete. */ + rreq->transferred = stream->transferred; + trace_netfs_rreq(rreq, netfs_rreq_trace_complete); + + //netfs_rreq_is_still_valid(rreq); + + switch (rreq->origin) { + case NETFS_UNBUFFERED_READ: + case NETFS_DIO_READ: + case NETFS_READ_GAPS: + netfs_rreq_assess_dio(rreq); + break; + case NETFS_READ_SINGLE: + netfs_rreq_assess_single(rreq); + break; + default: + break; + } + task_io_account_read(rreq->transferred); + + netfs_wake_rreq_flag(rreq, NETFS_RREQ_IN_PROGRESS, netfs_rreq_trace_wake_ip); + /* As we cleared NETFS_RREQ_IN_PROGRESS, we acquired its ref. */ + + trace_netfs_rreq(rreq, netfs_rreq_trace_done); + netfs_clear_subrequests(rreq); + netfs_unlock_abandoned_read_pages(rreq); + if (unlikely(rreq->copy_to_cache)) + netfs_pgpriv2_end_copy_to_cache(rreq); + return true; +} + +void netfs_read_collection_worker(struct work_struct *work) +{ + struct netfs_io_request *rreq = container_of(work, struct netfs_io_request, work); + + netfs_see_request(rreq, netfs_rreq_trace_see_work); + if (netfs_check_rreq_in_progress(rreq)) { + if (netfs_read_collection(rreq)) + /* Drop the ref from the IN_PROGRESS flag. */ + netfs_put_request(rreq, netfs_rreq_trace_put_work_ip); + else + netfs_see_request(rreq, netfs_rreq_trace_see_work_complete); + } +} + +/** + * netfs_read_subreq_progress - Note progress of a read operation. + * @subreq: The read request that has terminated. + * + * This tells the read side of netfs lib that a contributory I/O operation has + * made some progress and that it may be possible to unlock some folios. + * + * Before calling, the filesystem should update subreq->transferred to track + * the amount of data copied into the output buffer. + */ +void netfs_read_subreq_progress(struct netfs_io_subrequest *subreq) +{ + struct netfs_io_request *rreq = subreq->rreq; + struct netfs_io_stream *stream = &rreq->io_streams[0]; + size_t fsize = PAGE_SIZE << rreq->front_folio_order; + + trace_netfs_sreq(subreq, netfs_sreq_trace_progress); + + /* If we are at the head of the queue, wake up the collector, + * getting a ref to it if we were the ones to do so. + */ + if (subreq->start + subreq->transferred > rreq->cleaned_to + fsize && + (rreq->origin == NETFS_READAHEAD || + rreq->origin == NETFS_READPAGE || + rreq->origin == NETFS_READ_FOR_WRITE) && + list_is_first(&subreq->rreq_link, &stream->subrequests) + ) { + __set_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); + netfs_wake_collector(rreq); + } +} +EXPORT_SYMBOL(netfs_read_subreq_progress); + +/** + * netfs_read_subreq_terminated - Note the termination of an I/O operation. + * @subreq: The I/O request that has terminated. + * + * This tells the read helper that a contributory I/O operation has terminated, + * one way or another, and that it should integrate the results. + * + * The caller indicates the outcome of the operation through @subreq->error, + * supplying 0 to indicate a successful or retryable transfer (if + * NETFS_SREQ_NEED_RETRY is set) or a negative error code. The helper will + * look after reissuing I/O operations as appropriate and writing downloaded + * data to the cache. + * + * Before calling, the filesystem should update subreq->transferred to track + * the amount of data copied into the output buffer. + */ +void netfs_read_subreq_terminated(struct netfs_io_subrequest *subreq) +{ + struct netfs_io_request *rreq = subreq->rreq; + + switch (subreq->source) { + case NETFS_READ_FROM_CACHE: + netfs_stat(&netfs_n_rh_read_done); + break; + case NETFS_DOWNLOAD_FROM_SERVER: + netfs_stat(&netfs_n_rh_download_done); + break; + default: + break; + } + + /* Deal with retry requests, short reads and errors. If we retry + * but don't make progress, we abandon the attempt. + */ + if (!subreq->error && subreq->transferred < subreq->len) { + if (test_bit(NETFS_SREQ_HIT_EOF, &subreq->flags)) { + trace_netfs_sreq(subreq, netfs_sreq_trace_hit_eof); + } else if (test_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags)) { + trace_netfs_sreq(subreq, netfs_sreq_trace_need_clear); + } else if (test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { + trace_netfs_sreq(subreq, netfs_sreq_trace_need_retry); + } else if (test_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags)) { + __set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); + trace_netfs_sreq(subreq, netfs_sreq_trace_partial_read); + } else { + __set_bit(NETFS_SREQ_FAILED, &subreq->flags); + subreq->error = -ENODATA; + trace_netfs_sreq(subreq, netfs_sreq_trace_short); + } + } + + /* If need retry is set, error should not matter unless we hit too many + * retries. Pause the generation of new subreqs + */ + if (test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { + trace_netfs_rreq(rreq, netfs_rreq_trace_set_pause); + set_bit(NETFS_RREQ_PAUSE, &rreq->flags); + goto skip_error_checks; + } + + if (unlikely(subreq->error < 0)) { + trace_netfs_failure(rreq, subreq, subreq->error, netfs_fail_read); + if (subreq->source == NETFS_READ_FROM_CACHE) { + netfs_stat(&netfs_n_rh_read_failed); + __set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); + } else { + netfs_stat(&netfs_n_rh_download_failed); + __set_bit(NETFS_SREQ_FAILED, &subreq->flags); + } + trace_netfs_rreq(rreq, netfs_rreq_trace_set_pause); + set_bit(NETFS_RREQ_PAUSE, &rreq->flags); + } + +skip_error_checks: + trace_netfs_sreq(subreq, netfs_sreq_trace_terminated); + netfs_subreq_clear_in_progress(subreq); + netfs_put_subrequest(subreq, netfs_sreq_trace_put_terminated); +} +EXPORT_SYMBOL(netfs_read_subreq_terminated); + +/* + * Cancel a read subrequest due to preparation failure. + */ +void netfs_cancel_read(struct netfs_io_subrequest *subreq, int error) +{ + trace_netfs_sreq(subreq, netfs_sreq_trace_cancel); + subreq->error = error; + __set_bit(NETFS_SREQ_FAILED, &subreq->flags); + netfs_read_subreq_terminated(subreq); +} + +/* + * Handle termination of a read from the cache. + */ +void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error) +{ + struct netfs_io_subrequest *subreq = priv; + + if (transferred_or_error > 0) { + subreq->error = 0; + if (transferred_or_error > 0) { + subreq->transferred += transferred_or_error; + __set_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); + } + } else { + subreq->error = transferred_or_error; + } + netfs_read_subreq_terminated(subreq); +} diff --git a/fs/netfs/read_pgpriv2.c b/fs/netfs/read_pgpriv2.c new file mode 100644 index 000000000..c31190993 --- /dev/null +++ b/fs/netfs/read_pgpriv2.c @@ -0,0 +1,233 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Read with PG_private_2 [DEPRECATED]. + * + * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include "internal.h" + +/* + * [DEPRECATED] Copy a folio to the cache with PG_private_2 set. + */ +static void netfs_pgpriv2_copy_folio(struct netfs_io_request *creq, struct folio *folio) +{ + struct netfs_io_stream *cache = &creq->io_streams[1]; + size_t fsize = folio_size(folio), flen = fsize; + loff_t fpos = folio_pos(folio), i_size; + bool to_eof = false; + + _enter(""); + + /* netfs_perform_write() may shift i_size around the page or from out + * of the page to beyond it, but cannot move i_size into or through the + * page since we have it locked. + */ + i_size = i_size_read(creq->inode); + + if (fpos >= i_size) { + /* mmap beyond eof. */ + _debug("beyond eof"); + folio_end_private_2(folio); + return; + } + + if (fpos + fsize > creq->i_size) + creq->i_size = i_size; + + if (flen > i_size - fpos) { + flen = i_size - fpos; + to_eof = true; + } else if (flen == i_size - fpos) { + to_eof = true; + } + + _debug("folio %zx %zx", flen, fsize); + + trace_netfs_folio(folio, netfs_folio_trace_store_copy); + + /* Attach the folio to the rolling buffer. */ + if (rolling_buffer_append(&creq->buffer, folio, 0, creq->gfp) < 0) { + folio_end_private_2(folio); + clear_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &creq->flags); + return; + } + + cache->submit_extendable_to = fsize; + cache->submit_off = 0; + cache->submit_len = flen; + + /* Attach the folio to one or more subrequests. For a big folio, we + * could end up with thousands of subrequests if the wsize is small - + * but we might need to wait during the creation of subrequests for + * network resources (eg. SMB credits). + */ + do { + ssize_t part; + + creq->buffer.iter.iov_offset = cache->submit_off; + + atomic64_set(&creq->issued_to, fpos + cache->submit_off); + cache->submit_extendable_to = fsize - cache->submit_off; + part = netfs_advance_write(creq, cache, fpos + cache->submit_off, + cache->submit_len, to_eof); + cache->submit_off += part; + if (part > cache->submit_len) + cache->submit_len = 0; + else + cache->submit_len -= part; + } while (cache->submit_len > 0); + + creq->buffer.iter.iov_offset = 0; + rolling_buffer_advance(&creq->buffer, fsize); + atomic64_set(&creq->issued_to, fpos + fsize); + + if (flen < fsize) + netfs_issue_write(creq, cache); +} + +/* + * [DEPRECATED] Set up copying to the cache. + */ +static struct netfs_io_request *netfs_pgpriv2_begin_copy_to_cache( + struct netfs_io_request *rreq, struct folio *folio) +{ + struct netfs_io_request *creq; + + if (!fscache_resources_valid(&rreq->cache_resources)) + goto cancel; + + creq = netfs_create_write_req(rreq->mapping, NULL, folio_pos(folio), + NETFS_PGPRIV2_COPY_TO_CACHE); + if (IS_ERR(creq)) + goto cancel; + + if (!creq->io_streams[1].avail) + goto cancel_put; + + __set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &creq->flags); + trace_netfs_copy2cache(rreq, creq); + trace_netfs_write(creq, netfs_write_trace_copy_to_cache); + netfs_stat(&netfs_n_wh_copy_to_cache); + rreq->copy_to_cache = creq; + return creq; + +cancel_put: + netfs_put_failed_request(creq); +cancel: + rreq->copy_to_cache = ERR_PTR(-ENOBUFS); + clear_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &rreq->flags); + return ERR_PTR(-ENOBUFS); +} + +/* + * [DEPRECATED] Mark page as requiring copy-to-cache using PG_private_2 and add + * it to the copy write request. + */ +void netfs_pgpriv2_copy_to_cache(struct netfs_io_request *rreq, struct folio *folio) +{ + struct netfs_io_request *creq = rreq->copy_to_cache; + + if (!creq) + creq = netfs_pgpriv2_begin_copy_to_cache(rreq, folio); + if (IS_ERR(creq)) + return; + + trace_netfs_folio(folio, netfs_folio_trace_copy_to_cache); + folio_start_private_2(folio); + netfs_pgpriv2_copy_folio(creq, folio); +} + +/* + * [DEPRECATED] End writing to the cache, flushing out any outstanding writes. + */ +void netfs_pgpriv2_end_copy_to_cache(struct netfs_io_request *rreq) +{ + struct netfs_io_request *creq = rreq->copy_to_cache; + + if (IS_ERR_OR_NULL(creq)) + return; + + netfs_issue_write(creq, &creq->io_streams[1]); + smp_wmb(); /* Write lists before ALL_QUEUED. */ + set_bit(NETFS_RREQ_ALL_QUEUED, &creq->flags); + trace_netfs_rreq(rreq, netfs_rreq_trace_end_copy_to_cache); + if (list_empty_careful(&creq->io_streams[1].subrequests)) + netfs_wake_collector(creq); + + netfs_put_request(creq, netfs_rreq_trace_put_return); + creq->copy_to_cache = NULL; +} + +/* + * [DEPRECATED] Remove the PG_private_2 mark from any folios we've finished + * copying. + */ +bool netfs_pgpriv2_unlock_copied_folios(struct netfs_io_request *creq) +{ + struct folio_queue *folioq = creq->buffer.tail; + unsigned long long collected_to = creq->collected_to; + unsigned int slot = creq->buffer.first_tail_slot; + bool made_progress = false; + + if (slot >= folioq_nr_slots(folioq)) { + folioq = rolling_buffer_delete_spent(&creq->buffer); + slot = 0; + } + + for (;;) { + struct folio *folio; + unsigned long long fpos, fend; + size_t fsize, flen; + + folio = folioq_folio(folioq, slot); + if (WARN_ONCE(!folio_test_private_2(folio), + "R=%08x: folio %lx is not marked private_2\n", + creq->debug_id, folio->index)) + trace_netfs_folio(folio, netfs_folio_trace_not_under_wback); + + fpos = folio_pos(folio); + fsize = folio_size(folio); + flen = fsize; + + fend = min_t(unsigned long long, fpos + flen, creq->i_size); + + trace_netfs_collect_folio(creq, folio, fend, collected_to); + + /* Unlock any folio we've transferred all of. */ + if (collected_to < fend) + break; + + trace_netfs_folio(folio, netfs_folio_trace_end_copy); + folio_end_private_2(folio); + creq->cleaned_to = fpos + fsize; + made_progress = true; + + /* Clean up the head folioq. If we clear an entire folioq, then + * we can get rid of it provided it's not also the tail folioq + * being filled by the issuer. + */ + folioq_clear(folioq, slot); + slot++; + if (slot >= folioq_nr_slots(folioq)) { + folioq = rolling_buffer_delete_spent(&creq->buffer); + if (!folioq) + goto done; + slot = 0; + } + + if (fpos + fsize >= collected_to) + break; + } + + creq->buffer.tail = folioq; +done: + creq->buffer.first_tail_slot = slot; + return made_progress; +} diff --git a/fs/netfs/read_retry.c b/fs/netfs/read_retry.c new file mode 100644 index 000000000..2b42758e0 --- /dev/null +++ b/fs/netfs/read_retry.c @@ -0,0 +1,312 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Network filesystem read subrequest retrying. + * + * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include "internal.h" + +static void netfs_reissue_read(struct netfs_io_request *rreq, + struct netfs_io_subrequest *subreq) +{ + subreq->error = 0; + __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); + __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); + netfs_stat(&netfs_n_rh_retry_read_subreq); + subreq->rreq->netfs_ops->issue_read(subreq); +} + +/* + * Go through the list of failed/short reads, retrying all retryable ones. We + * need to switch failed cache reads to network downloads. + */ +static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) +{ + struct netfs_io_subrequest *subreq; + struct netfs_io_stream *stream = &rreq->io_streams[0]; + struct list_head *next; + + _enter("R=%x", rreq->debug_id); + + if (list_empty(&stream->subrequests)) + return; + + if (rreq->netfs_ops->retry_request) + rreq->netfs_ops->retry_request(rreq, NULL); + + /* If there's no renegotiation to do, just resend each retryable subreq + * up to the first permanently failed one. + */ + if (!rreq->netfs_ops->prepare_read && + !rreq->cache_resources.ops) { + list_for_each_entry(subreq, &stream->subrequests, rreq_link) { + if (test_bit(NETFS_SREQ_FAILED, &subreq->flags)) + break; + if (__test_and_clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { + __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); + subreq->retry_count++; + netfs_reset_iter(subreq); + netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); + netfs_reissue_read(rreq, subreq); + } + } + return; + } + + /* Okay, we need to renegotiate all the download requests and flip any + * failed cache reads over to being download requests and negotiate + * those also. All fully successful subreqs have been removed from the + * list and any spare data from those has been donated. + * + * What we do is decant the list and rebuild it one subreq at a time so + * that we don't end up with donations jumping over a gap we're busy + * populating with smaller subrequests. In the event that the subreq + * we just launched finishes before we insert the next subreq, it'll + * fill in rreq->prev_donated instead. + * + * Note: Alternatively, we could split the tail subrequest right before + * we reissue it and fix up the donations under lock. + */ + next = stream->subrequests.next; + + do { + struct netfs_io_subrequest *from, *to, *tmp; + struct iov_iter source; + unsigned long long start, len; + size_t part; + bool boundary = false, subreq_superfluous = false; + + /* Go through the subreqs and find the next span of contiguous + * buffer that we then rejig (cifs, for example, needs the + * rsize renegotiating) and reissue. + */ + from = list_entry(next, struct netfs_io_subrequest, rreq_link); + to = from; + start = from->start + from->transferred; + len = from->len - from->transferred; + + _debug("from R=%08x[%x] s=%llx ctl=%zx/%zx", + rreq->debug_id, from->debug_index, + from->start, from->transferred, from->len); + + if (test_bit(NETFS_SREQ_FAILED, &from->flags) || + !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags)) { + subreq = from; + goto abandon; + } + + for (;;) { + /* Read pointer to subreq before reading subreq state. */ + next = smp_load_acquire(&next->next); + if (next == &stream->subrequests) + break; + + subreq = list_entry(next, struct netfs_io_subrequest, rreq_link); + if (subreq->start + subreq->transferred != start + len || + test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags) || + !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) + break; + to = subreq; + len += to->len; + } + + _debug(" - range: %llx-%llx %llx", start, start + len - 1, len); + + /* Determine the set of buffers we're going to use. Each + * subreq gets a subset of a single overall contiguous buffer. + */ + netfs_reset_iter(from); + source = from->io_iter; + source.count = len; + + /* Work through the sublist. */ + subreq = from; + list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) { + if (!len) { + subreq_superfluous = true; + break; + } + subreq->source = NETFS_DOWNLOAD_FROM_SERVER; + subreq->start = start - subreq->transferred; + subreq->len = len + subreq->transferred; + __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); + __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); + subreq->retry_count++; + + trace_netfs_sreq(subreq, netfs_sreq_trace_retry); + + /* Renegotiate max_len (rsize) */ + stream->sreq_max_len = subreq->len; + if (rreq->netfs_ops->prepare_read && + rreq->netfs_ops->prepare_read(subreq) < 0) { + trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed); + __set_bit(NETFS_SREQ_FAILED, &subreq->flags); + goto abandon; + } + + part = umin(len, stream->sreq_max_len); + if (unlikely(stream->sreq_max_segs)) + part = netfs_limit_iter(&source, 0, part, stream->sreq_max_segs); + subreq->len = subreq->transferred + part; + subreq->io_iter = source; + iov_iter_truncate(&subreq->io_iter, part); + iov_iter_advance(&source, part); + len -= part; + start += part; + if (!len) { + if (boundary) + __set_bit(NETFS_SREQ_BOUNDARY, &subreq->flags); + } else { + __clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags); + } + + netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); + netfs_reissue_read(rreq, subreq); + if (subreq == to) { + subreq_superfluous = false; + break; + } + } + + /* If we managed to use fewer subreqs, we can discard the + * excess; if we used the same number, then we're done. + */ + if (!len) { + if (!subreq_superfluous) + continue; + list_for_each_entry_safe_from(subreq, tmp, + &stream->subrequests, rreq_link) { + trace_netfs_sreq(subreq, netfs_sreq_trace_superfluous); + spin_lock(&rreq->lock); + list_del(&subreq->rreq_link); + spin_unlock(&rreq->lock); + netfs_put_subrequest(subreq, netfs_sreq_trace_put_done); + if (subreq == to) + break; + } + subreq = NULL; + continue; + } + + /* We ran out of subrequests, so we need to allocate some more + * and insert them after. + */ + do { + subreq = netfs_alloc_subrequest(rreq); + if (!subreq) { + subreq = to; + goto abandon_after; + } + subreq->source = NETFS_DOWNLOAD_FROM_SERVER; + subreq->start = start; + subreq->len = len; + subreq->stream_nr = stream->stream_nr; + subreq->retry_count = 1; + + trace_netfs_sreq_ref(rreq->debug_id, subreq->debug_index, + refcount_read(&subreq->ref), + netfs_sreq_trace_new); + + spin_lock(&rreq->lock); + list_add(&subreq->rreq_link, &to->rreq_link); + spin_unlock(&rreq->lock); + to = subreq; + trace_netfs_sreq(subreq, netfs_sreq_trace_retry); + + stream->sreq_max_len = umin(len, rreq->rsize); + stream->sreq_max_segs = 0; + if (unlikely(stream->sreq_max_segs)) + part = netfs_limit_iter(&source, 0, part, stream->sreq_max_segs); + + netfs_stat(&netfs_n_rh_download); + if (rreq->netfs_ops->prepare_read(subreq) < 0) { + trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed); + __set_bit(NETFS_SREQ_FAILED, &subreq->flags); + goto abandon; + } + + part = umin(len, stream->sreq_max_len); + subreq->len = subreq->transferred + part; + subreq->io_iter = source; + iov_iter_truncate(&subreq->io_iter, part); + iov_iter_advance(&source, part); + + len -= part; + start += part; + if (!len && boundary) { + __set_bit(NETFS_SREQ_BOUNDARY, &to->flags); + boundary = false; + } + + netfs_reissue_read(rreq, subreq); + } while (len); + + } while (!list_is_head(next, &stream->subrequests)); + + return; + + /* If we hit an error, fail all remaining incomplete subrequests */ +abandon_after: + if (list_is_last(&subreq->rreq_link, &stream->subrequests)) + return; + subreq = list_next_entry(subreq, rreq_link); +abandon: + list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) { + if (!test_bit(NETFS_SREQ_FAILED, &subreq->flags) && + !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) + continue; + subreq->error = -ENOMEM; + __set_bit(NETFS_SREQ_FAILED, &subreq->flags); + __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); + } +} + +/* + * Retry reads. + */ +void netfs_retry_reads(struct netfs_io_request *rreq) +{ + struct netfs_io_stream *stream = &rreq->io_streams[0]; + + netfs_stat(&netfs_n_rh_retry_read_req); + + /* Wait for all outstanding I/O to quiesce before performing retries as + * we may need to renegotiate the I/O sizes. + */ + set_bit(NETFS_RREQ_RETRYING, &rreq->flags); + netfs_wait_for_in_progress_stream(rreq, stream); + clear_bit(NETFS_RREQ_RETRYING, &rreq->flags); + + trace_netfs_rreq(rreq, netfs_rreq_trace_resubmit); + netfs_retry_read_subrequests(rreq); +} + +/* + * Unlock any the pages that haven't been unlocked yet due to abandoned + * subrequests. + */ +void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq) +{ + struct folio_queue *p; + + for (p = rreq->buffer.tail; p; p = p->next) { + for (int slot = 0; slot < folioq_count(p); slot++) { + struct folio *folio = folioq_folio(p, slot); + + if (folio && !folioq_is_marked2(p, slot)) { + if (folio == rreq->no_unlock_folio && + test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, + &rreq->flags)) { + _debug("no unlock"); + } else { + trace_netfs_folio(folio, + netfs_folio_trace_abandon); + folio_unlock(folio); + } + } + } + } +} diff --git a/fs/netfs/read_single.c b/fs/netfs/read_single.c new file mode 100644 index 000000000..8833550d2 --- /dev/null +++ b/fs/netfs/read_single.c @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Single, monolithic object support (e.g. AFS directory). + * + * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +/** + * netfs_single_mark_inode_dirty - Mark a single, monolithic object inode dirty + * @inode: The inode to mark + * + * Mark an inode that contains a single, monolithic object as dirty so that its + * writepages op will get called. If set, the SINGLE_NO_UPLOAD flag indicates + * that the object will only be written to the cache and not uploaded (e.g. AFS + * directory contents). + */ +void netfs_single_mark_inode_dirty(struct inode *inode) +{ + struct netfs_inode *ictx = netfs_inode(inode); + bool cache_only = test_bit(NETFS_ICTX_SINGLE_NO_UPLOAD, &ictx->flags); + bool caching = fscache_cookie_enabled(netfs_i_cookie(netfs_inode(inode))); + + if (cache_only && !caching) + return; + + mark_inode_dirty(inode); + + if (caching && !(inode_state_read_once(inode) & I_PINNING_NETFS_WB)) { + bool need_use = false; + + spin_lock(&inode->i_lock); + if (!(inode_state_read(inode) & I_PINNING_NETFS_WB)) { + inode_state_set(inode, I_PINNING_NETFS_WB); + need_use = true; + } + spin_unlock(&inode->i_lock); + + if (need_use) + fscache_use_cookie(netfs_i_cookie(ictx), true); + } + +} +EXPORT_SYMBOL(netfs_single_mark_inode_dirty); + +static int netfs_single_begin_cache_read(struct netfs_io_request *rreq, struct netfs_inode *ctx) +{ + return fscache_begin_read_operation(&rreq->cache_resources, netfs_i_cookie(ctx)); +} + +static void netfs_single_cache_prepare_read(struct netfs_io_request *rreq, + struct netfs_io_subrequest *subreq) +{ + struct netfs_cache_resources *cres = &rreq->cache_resources; + + if (!cres->ops) { + subreq->source = NETFS_DOWNLOAD_FROM_SERVER; + return; + } + subreq->source = cres->ops->prepare_read(subreq, rreq->i_size); + trace_netfs_sreq(subreq, netfs_sreq_trace_prepare); + +} + +static void netfs_single_read_cache(struct netfs_io_request *rreq, + struct netfs_io_subrequest *subreq) +{ + struct netfs_cache_resources *cres = &rreq->cache_resources; + + _enter("R=%08x[%x]", rreq->debug_id, subreq->debug_index); + netfs_stat(&netfs_n_rh_read); + cres->ops->read(cres, subreq->start, &subreq->io_iter, NETFS_READ_HOLE_FAIL, + netfs_cache_read_terminated, subreq); +} + +/* + * Perform a read to a buffer from the cache or the server. Only a single + * subreq is permitted as the object must be fetched in a single transaction. + */ +static int netfs_single_dispatch_read(struct netfs_io_request *rreq) +{ + struct netfs_io_subrequest *subreq; + int ret = 0; + + subreq = netfs_alloc_subrequest(rreq); + if (!subreq) + return -ENOMEM; + + subreq->source = NETFS_SOURCE_UNKNOWN; + subreq->start = 0; + subreq->len = rreq->len; + subreq->io_iter = rreq->buffer.iter; + + netfs_queue_read(rreq, subreq); + + netfs_single_cache_prepare_read(rreq, subreq); + switch (subreq->source) { + case NETFS_DOWNLOAD_FROM_SERVER: + netfs_stat(&netfs_n_rh_download); + if (rreq->netfs_ops->prepare_read) { + ret = rreq->netfs_ops->prepare_read(subreq); + if (ret < 0) + goto cancel; + } + + smp_wmb(); /* Write lists before ALL_QUEUED. */ + set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); + rreq->netfs_ops->issue_read(subreq); + rreq->submitted += subreq->len; + break; + case NETFS_READ_FROM_CACHE: + smp_wmb(); /* Write lists before ALL_QUEUED. */ + set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); + trace_netfs_sreq(subreq, netfs_sreq_trace_submit); + netfs_single_read_cache(rreq, subreq); + rreq->submitted += subreq->len; + ret = 0; + break; + default: + pr_warn("Unexpected single-read source %u\n", subreq->source); + WARN_ON_ONCE(true); + ret = -EIO; + goto cancel; + } + + return ret; +cancel: + netfs_cancel_read(subreq, ret); + smp_wmb(); /* Write lists before ALL_QUEUED. */ + set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); + netfs_wake_collector(rreq); + return ret; +} + +/** + * netfs_read_single - Synchronously read a single blob of pages. + * @inode: The inode to read from. + * @file: The file we're using to read or NULL. + * @iter: The buffer we're reading into. + * + * Fulfil a read request for a single monolithic object by drawing data from + * the cache if possible, or the netfs if not. The buffer may be larger than + * the file content; unused beyond the EOF will be zero-filled. The content + * will be read with a single I/O request (though this may be retried). + * + * The calling netfs must initialise a netfs context contiguous to the vfs + * inode before calling this. + * + * This is usable whether or not caching is enabled. If caching is enabled, + * the data will be stored as a single object into the cache. + */ +ssize_t netfs_read_single(struct inode *inode, struct file *file, struct iov_iter *iter) +{ + struct netfs_io_request *rreq; + struct netfs_inode *ictx = netfs_inode(inode); + ssize_t ret; + + rreq = netfs_alloc_request(inode->i_mapping, file, 0, iov_iter_count(iter), + NETFS_READ_SINGLE); + if (IS_ERR(rreq)) + return PTR_ERR(rreq); + + ret = netfs_single_begin_cache_read(rreq, ictx); + if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS) + goto cleanup_free; + + netfs_stat(&netfs_n_rh_read_single); + trace_netfs_read(rreq, 0, rreq->len, netfs_read_trace_read_single); + + rreq->buffer.iter = *iter; + netfs_single_dispatch_read(rreq); + + ret = netfs_wait_for_read(rreq); + netfs_put_request(rreq, netfs_rreq_trace_put_return); + return ret; + +cleanup_free: + netfs_put_failed_request(rreq); + return ret; +} +EXPORT_SYMBOL(netfs_read_single); diff --git a/fs/netfs/rolling_buffer.c b/fs/netfs/rolling_buffer.c new file mode 100644 index 000000000..8c0026836 --- /dev/null +++ b/fs/netfs/rolling_buffer.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Rolling buffer helpers + * + * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include "internal.h" + +static atomic_t debug_ids; + +/** + * netfs_folioq_alloc - Allocate a folio_queue struct + * @rreq_id: Associated debugging ID for tracing purposes + * @gfp: Allocation constraints + * @trace: Trace tag to indicate the purpose of the allocation + * + * Allocate, initialise and account the folio_queue struct and log a trace line + * to mark the allocation. + */ +struct folio_queue *netfs_folioq_alloc(unsigned int rreq_id, gfp_t gfp, + unsigned int /*enum netfs_folioq_trace*/ trace) +{ + struct folio_queue *fq; + + if (gfp == GFP_KERNEL) + fq = netfs_folioq_pool.alloc(gfp, netfs_folioq_pool.pool_data); + else + fq = mempool_alloc(&netfs_folioq_pool, gfp); + if (fq) { + netfs_stat(&netfs_n_folioq); + folioq_init(fq, rreq_id); + fq->debug_id = atomic_inc_return(&debug_ids); + trace_netfs_folioq(fq, trace); + } + return fq; +} +EXPORT_SYMBOL(netfs_folioq_alloc); + +/** + * netfs_folioq_free - Free a folio_queue struct + * @folioq: The object to free + * @trace: Trace tag to indicate which free + * + * Free and unaccount the folio_queue struct. + */ +void netfs_folioq_free(struct folio_queue *folioq, + unsigned int /*enum netfs_trace_folioq*/ trace) +{ + trace_netfs_folioq(folioq, trace); + netfs_stat_d(&netfs_n_folioq); + mempool_free(folioq, &netfs_folioq_pool); +} +EXPORT_SYMBOL(netfs_folioq_free); + +/* + * Initialise a rolling buffer. We allocate an empty folio queue struct to so + * that the pointers can be independently driven by the producer and the + * consumer. + */ +int rolling_buffer_init(struct rolling_buffer *roll, unsigned int rreq_id, + unsigned int direction, gfp_t gfp) +{ + struct folio_queue *fq; + + fq = netfs_folioq_alloc(rreq_id, gfp, netfs_trace_folioq_rollbuf_init); + if (!fq) + return -ENOMEM; + + roll->head = fq; + roll->tail = fq; + iov_iter_folio_queue(&roll->iter, direction, fq, 0, 0, 0); + return 0; +} + +/* + * Add another folio_queue to a rolling buffer if there's no space left. + */ +int rolling_buffer_make_space(struct rolling_buffer *roll, gfp_t gfp) +{ + struct folio_queue *fq, *head = roll->head; + + if (!folioq_full(head)) + return 0; + + fq = netfs_folioq_alloc(head->rreq_id, gfp, netfs_trace_folioq_make_space); + if (!fq) + return -ENOMEM; + fq->prev = head; + + roll->head = fq; + if (folioq_full(head)) { + /* Make sure we don't leave the master iterator pointing to a + * block that might get immediately consumed. + */ + if (roll->iter.folioq == head && + roll->iter.folioq_slot == folioq_nr_slots(head)) { + roll->iter.folioq = fq; + roll->iter.folioq_slot = 0; + } + } + + /* Make sure the initialisation is stored before the next pointer. + * + * [!] NOTE: After we set head->next, the consumer is at liberty to + * immediately delete the old head. + */ + smp_store_release(&head->next, fq); + return 0; +} + +/* + * Decant the list of folios to read into a rolling buffer. + */ +ssize_t rolling_buffer_load_from_ra(struct rolling_buffer *roll, + struct readahead_control *ractl, + struct folio_batch *put_batch) +{ + struct folio_queue *fq; + struct page **vec; + int nr, ix, to; + ssize_t size = 0; + + if (rolling_buffer_make_space(roll, GFP_KERNEL) < 0) + return -ENOMEM; + + fq = roll->head; + vec = (struct page **)fq->vec.folios; + nr = __readahead_batch(ractl, vec + folio_batch_count(&fq->vec), + folio_batch_space(&fq->vec)); + ix = fq->vec.nr; + to = ix + nr; + fq->vec.nr = to; + for (; ix < to; ix++) { + struct folio *folio = folioq_folio(fq, ix); + unsigned int order = folio_order(folio); + + fq->orders[ix] = order; + size += PAGE_SIZE << order; + trace_netfs_folio(folio, netfs_folio_trace_read); + if (!folio_batch_add(put_batch, folio)) + folio_batch_release(put_batch); + } + WRITE_ONCE(roll->iter.count, roll->iter.count + size); + + /* Store the counter after setting the slot. */ + smp_store_release(&roll->next_head_slot, to); + return size; +} + +/* + * Append a folio to the rolling buffer. + */ +ssize_t rolling_buffer_append(struct rolling_buffer *roll, struct folio *folio, + unsigned int flags, gfp_t gfp) +{ + ssize_t size = folio_size(folio); + int slot; + + if (rolling_buffer_make_space(roll, gfp) < 0) + return -ENOMEM; + + slot = folioq_append(roll->head, folio); + if (flags & ROLLBUF_MARK_1) + folioq_mark(roll->head, slot); + if (flags & ROLLBUF_MARK_2) + folioq_mark2(roll->head, slot); + + WRITE_ONCE(roll->iter.count, roll->iter.count + size); + + /* Store the counter after setting the slot. */ + smp_store_release(&roll->next_head_slot, slot); + return size; +} + +/* + * Delete a spent buffer from a rolling queue and return the next in line. We + * don't return the last buffer to keep the pointers independent, but return + * NULL instead. + */ +struct folio_queue *rolling_buffer_delete_spent(struct rolling_buffer *roll) +{ + struct folio_queue *spent = roll->tail, *next = READ_ONCE(spent->next); + + if (!next) + return NULL; + next->prev = NULL; + netfs_folioq_free(spent, netfs_trace_folioq_delete); + roll->tail = next; + return next; +} + +/* + * Clear out a rolling queue. Folios that have mark 1 set are put. + */ +void rolling_buffer_clear(struct rolling_buffer *roll) +{ + struct folio_batch fbatch; + struct folio_queue *p; + + folio_batch_init(&fbatch); + + while ((p = roll->tail)) { + roll->tail = p->next; + for (int slot = 0; slot < folioq_count(p); slot++) { + struct folio *folio = folioq_folio(p, slot); + + if (!folio) + continue; + if (folioq_is_marked(p, slot)) { + trace_netfs_folio(folio, netfs_folio_trace_put); + if (!folio_batch_add(&fbatch, folio)) + folio_batch_release(&fbatch); + } + } + + netfs_folioq_free(p, netfs_trace_folioq_clear); + } + + folio_batch_release(&fbatch); +} diff --git a/fs/netfs/stats.c b/fs/netfs/stats.c new file mode 100644 index 000000000..ab6b916ad --- /dev/null +++ b/fs/netfs/stats.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Netfs support statistics + * + * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include "internal.h" + +atomic_t netfs_n_rh_dio_read; +atomic_t netfs_n_rh_readahead; +atomic_t netfs_n_rh_read_folio; +atomic_t netfs_n_rh_read_single; +atomic_t netfs_n_rh_rreq; +atomic_t netfs_n_rh_sreq; +atomic_t netfs_n_rh_download; +atomic_t netfs_n_rh_download_done; +atomic_t netfs_n_rh_download_failed; +atomic_t netfs_n_rh_download_instead; +atomic_t netfs_n_rh_read; +atomic_t netfs_n_rh_read_done; +atomic_t netfs_n_rh_read_failed; +atomic_t netfs_n_rh_zero; +atomic_t netfs_n_rh_short_read; +atomic_t netfs_n_rh_write; +atomic_t netfs_n_rh_write_begin; +atomic_t netfs_n_rh_write_done; +atomic_t netfs_n_rh_write_failed; +atomic_t netfs_n_rh_write_zskip; +atomic_t netfs_n_rh_retry_read_req; +atomic_t netfs_n_rh_retry_read_subreq; +atomic_t netfs_n_wh_buffered_write; +atomic_t netfs_n_wh_writethrough; +atomic_t netfs_n_wh_dio_write; +atomic_t netfs_n_wh_writepages; +atomic_t netfs_n_wh_copy_to_cache; +atomic_t netfs_n_wh_wstream_conflict; +atomic_t netfs_n_wh_upload; +atomic_t netfs_n_wh_upload_done; +atomic_t netfs_n_wh_upload_failed; +atomic_t netfs_n_wh_write; +atomic_t netfs_n_wh_write_done; +atomic_t netfs_n_wh_write_failed; +atomic_t netfs_n_wh_retry_write_req; +atomic_t netfs_n_wh_retry_write_subreq; +atomic_t netfs_n_wb_lock_skip; +atomic_t netfs_n_wb_lock_wait; +atomic_t netfs_n_folioq; + +int netfs_stats_show(struct seq_file *m, void *v) +{ + seq_printf(m, "Reads : DR=%u RA=%u RF=%u RS=%u WB=%u WBZ=%u\n", + atomic_read(&netfs_n_rh_dio_read), + atomic_read(&netfs_n_rh_readahead), + atomic_read(&netfs_n_rh_read_folio), + atomic_read(&netfs_n_rh_read_single), + atomic_read(&netfs_n_rh_write_begin), + atomic_read(&netfs_n_rh_write_zskip)); + seq_printf(m, "Writes : BW=%u WT=%u DW=%u WP=%u 2C=%u\n", + atomic_read(&netfs_n_wh_buffered_write), + atomic_read(&netfs_n_wh_writethrough), + atomic_read(&netfs_n_wh_dio_write), + atomic_read(&netfs_n_wh_writepages), + atomic_read(&netfs_n_wh_copy_to_cache)); + seq_printf(m, "ZeroOps: ZR=%u sh=%u sk=%u\n", + atomic_read(&netfs_n_rh_zero), + atomic_read(&netfs_n_rh_short_read), + atomic_read(&netfs_n_rh_write_zskip)); + seq_printf(m, "DownOps: DL=%u ds=%u df=%u di=%u\n", + atomic_read(&netfs_n_rh_download), + atomic_read(&netfs_n_rh_download_done), + atomic_read(&netfs_n_rh_download_failed), + atomic_read(&netfs_n_rh_download_instead)); + seq_printf(m, "CaRdOps: RD=%u rs=%u rf=%u\n", + atomic_read(&netfs_n_rh_read), + atomic_read(&netfs_n_rh_read_done), + atomic_read(&netfs_n_rh_read_failed)); + seq_printf(m, "UpldOps: UL=%u us=%u uf=%u\n", + atomic_read(&netfs_n_wh_upload), + atomic_read(&netfs_n_wh_upload_done), + atomic_read(&netfs_n_wh_upload_failed)); + seq_printf(m, "CaWrOps: WR=%u ws=%u wf=%u\n", + atomic_read(&netfs_n_wh_write), + atomic_read(&netfs_n_wh_write_done), + atomic_read(&netfs_n_wh_write_failed)); + seq_printf(m, "Retries: rq=%u rs=%u wq=%u ws=%u\n", + atomic_read(&netfs_n_rh_retry_read_req), + atomic_read(&netfs_n_rh_retry_read_subreq), + atomic_read(&netfs_n_wh_retry_write_req), + atomic_read(&netfs_n_wh_retry_write_subreq)); + seq_printf(m, "Objs : rr=%u sr=%u foq=%u wsc=%u\n", + atomic_read(&netfs_n_rh_rreq), + atomic_read(&netfs_n_rh_sreq), + atomic_read(&netfs_n_folioq), + atomic_read(&netfs_n_wh_wstream_conflict)); + seq_printf(m, "WbLock : skip=%u wait=%u\n", + atomic_read(&netfs_n_wb_lock_skip), + atomic_read(&netfs_n_wb_lock_wait)); + return fscache_stats_show(m); +} +EXPORT_SYMBOL(netfs_stats_show); diff --git a/fs/netfs/write_collect.c b/fs/netfs/write_collect.c new file mode 100644 index 000000000..210eb8f39 --- /dev/null +++ b/fs/netfs/write_collect.c @@ -0,0 +1,525 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Network filesystem write subrequest result collection, assessment + * and retrying. + * + * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include "internal.h" + +/* Notes made in the collector */ +#define HIT_PENDING 0x01 /* A front op was still pending */ +#define NEED_REASSESS 0x02 /* Need to loop round and reassess */ +#define MADE_PROGRESS 0x04 /* Made progress cleaning up a stream or the folio set */ +#define NEED_UNLOCK 0x08 /* The pagecache needs unlocking */ +#define NEED_RETRY 0x10 /* A front op requests retrying */ +#define SAW_FAILURE 0x20 /* One stream or hit a permanent failure */ + +static void netfs_dump_request(const struct netfs_io_request *rreq) +{ + pr_err("Request R=%08x r=%d fl=%lx or=%x e=%ld\n", + rreq->debug_id, refcount_read(&rreq->ref), rreq->flags, + rreq->origin, rreq->error); + pr_err(" st=%llx tsl=%zx/%llx/%llx\n", + rreq->start, rreq->transferred, rreq->submitted, rreq->len); + pr_err(" cci=%llx/%llx/%llx\n", + rreq->cleaned_to, rreq->collected_to, atomic64_read(&rreq->issued_to)); + pr_err(" iw=%pSR\n", rreq->netfs_ops->issue_write); + for (int i = 0; i < NR_IO_STREAMS; i++) { + const struct netfs_io_subrequest *sreq; + const struct netfs_io_stream *s = &rreq->io_streams[i]; + + pr_err(" str[%x] s=%x e=%d acnf=%u,%u,%u,%u\n", + s->stream_nr, s->source, s->error, + s->avail, s->active, s->need_retry, s->failed); + pr_err(" str[%x] ct=%llx t=%zx\n", + s->stream_nr, s->collected_to, s->transferred); + list_for_each_entry(sreq, &s->subrequests, rreq_link) { + pr_err(" sreq[%x:%x] sc=%u s=%llx t=%zx/%zx r=%d f=%lx\n", + sreq->stream_nr, sreq->debug_index, sreq->source, + sreq->start, sreq->transferred, sreq->len, + refcount_read(&sreq->ref), sreq->flags); + } + } +} + +/* + * Successful completion of write of a folio to the server and/or cache. Note + * that we are not allowed to lock the folio here on pain of deadlocking with + * truncate. + */ +int netfs_folio_written_back(struct folio *folio) +{ + enum netfs_folio_trace why = netfs_folio_trace_clear; + struct inode *inode = folio_inode(folio); + struct netfs_inode *ictx = netfs_inode(inode); + struct netfs_folio *finfo; + struct netfs_group *group = NULL; + int gcount = 0; + + if ((finfo = netfs_folio_info(folio))) { + /* Streaming writes cannot be redirtied whilst under writeback, + * so discard the streaming record. + */ + unsigned long long fend; + + fend = folio_pos(folio) + finfo->dirty_offset + finfo->dirty_len; + spin_lock(&ictx->inode.i_lock); + if (fend > ictx->_zero_point) + netfs_write_zero_point(inode, fend); + spin_unlock(&ictx->inode.i_lock); + + folio_detach_private(folio); + group = finfo->netfs_group; + gcount++; + kfree(finfo); + why = netfs_folio_trace_clear_s; + goto end_wb; + } + + if ((group = netfs_folio_group(folio))) { + if (group == NETFS_FOLIO_COPY_TO_CACHE) { + why = netfs_folio_trace_clear_cc; + folio_detach_private(folio); + goto end_wb; + } + + /* Need to detach the group pointer if the page didn't get + * redirtied. If it has been redirtied, then it must be within + * the same group. + */ + why = netfs_folio_trace_redirtied; + if (!folio_test_dirty(folio)) { + folio_detach_private(folio); + gcount++; + why = netfs_folio_trace_clear_g; + } + } + +end_wb: + trace_netfs_folio(folio, why); + folio_end_writeback(folio); + return gcount; +} + +/* + * Unlock any folios we've finished with. + */ +static void netfs_writeback_unlock_folios(struct netfs_io_request *wreq, + unsigned int *notes) +{ + struct folio_queue *folioq = wreq->buffer.tail; + unsigned long long collected_to = wreq->collected_to; + unsigned int slot = wreq->buffer.first_tail_slot; + + if (WARN_ON_ONCE(!folioq)) { + pr_err("[!] Writeback unlock found empty rolling buffer!\n"); + netfs_dump_request(wreq); + return; + } + + if (wreq->origin == NETFS_PGPRIV2_COPY_TO_CACHE) { + if (netfs_pgpriv2_unlock_copied_folios(wreq)) + *notes |= MADE_PROGRESS; + return; + } + + if (slot >= folioq_nr_slots(folioq)) { + folioq = rolling_buffer_delete_spent(&wreq->buffer); + if (!folioq) + return; + slot = 0; + } + + for (;;) { + struct folio *folio; + struct netfs_folio *finfo; + unsigned long long fpos, fend; + size_t fsize, flen; + + folio = folioq_folio(folioq, slot); + if (WARN_ONCE(!folio_test_writeback(folio), + "R=%08x: folio %lx is not under writeback\n", + wreq->debug_id, folio->index)) + trace_netfs_folio(folio, netfs_folio_trace_not_under_wback); + + fpos = folio_pos(folio); + fsize = folio_size(folio); + finfo = netfs_folio_info(folio); + flen = finfo ? finfo->dirty_offset + finfo->dirty_len : fsize; + + fend = min_t(unsigned long long, fpos + flen, wreq->i_size); + + trace_netfs_collect_folio(wreq, folio, fend, collected_to); + + /* Unlock any folio we've transferred all of. */ + if (collected_to < fend) + break; + + wreq->nr_group_rel += netfs_folio_written_back(folio); + wreq->cleaned_to = fpos + fsize; + *notes |= MADE_PROGRESS; + + /* Clean up the head folioq. If we clear an entire folioq, then + * we can get rid of it provided it's not also the tail folioq + * being filled by the issuer. + */ + folioq_clear(folioq, slot); + slot++; + if (slot >= folioq_nr_slots(folioq)) { + folioq = rolling_buffer_delete_spent(&wreq->buffer); + if (!folioq) + goto done; + slot = 0; + } + + if (fpos + fsize >= collected_to) + break; + } + + wreq->buffer.tail = folioq; +done: + wreq->buffer.first_tail_slot = slot; +} + +/* + * Collect and assess the results of various write subrequests. We may need to + * retry some of the results - or even do an RMW cycle for content crypto. + * + * Note that we have a number of parallel, overlapping lists of subrequests, + * one to the server and one to the local cache for example, which may not be + * the same size or starting position and may not even correspond in boundary + * alignment. + */ +static void netfs_collect_write_results(struct netfs_io_request *wreq) +{ + struct netfs_io_subrequest *front, *remove; + struct netfs_io_stream *stream; + unsigned long long collected_to, issued_to; + unsigned int notes; + int s; + + _enter("%llx-%llx", wreq->start, wreq->start + wreq->len); + trace_netfs_collect(wreq); + trace_netfs_rreq(wreq, netfs_rreq_trace_collect); + +reassess_streams: + issued_to = atomic64_read(&wreq->issued_to); + smp_rmb(); + collected_to = ULLONG_MAX; + if (wreq->origin == NETFS_WRITEBACK || + wreq->origin == NETFS_WRITETHROUGH || + wreq->origin == NETFS_PGPRIV2_COPY_TO_CACHE) + notes = NEED_UNLOCK; + else + notes = 0; + + /* Remove completed subrequests from the front of the streams and + * advance the completion point on each stream. We stop when we hit + * something that's in progress. The issuer thread may be adding stuff + * to the tail whilst we're doing this. + */ + for (s = 0; s < NR_IO_STREAMS; s++) { + stream = &wreq->io_streams[s]; + /* Read active flag before list pointers */ + if (!smp_load_acquire(&stream->active)) + continue; + + front = list_first_entry_or_null_acquire(&stream->subrequests, + struct netfs_io_subrequest, rreq_link); + /* Read first subreq pointer before IN_PROGRESS flag. */ + + while (front) { + trace_netfs_collect_sreq(wreq, front); + //_debug("sreq [%x] %llx %zx/%zx", + // front->debug_index, front->start, front->transferred, front->len); + + if (stream->collected_to < front->start) { + trace_netfs_collect_gap(wreq, stream, issued_to, 'F'); + stream->collected_to = front->start; + } + + /* Stall if the front is still undergoing I/O. */ + if (netfs_check_subreq_in_progress(front)) { + notes |= HIT_PENDING; + break; + } + smp_rmb(); /* Read counters after I-P flag. */ + + if (stream->failed) { + stream->collected_to = front->start + front->len; + notes |= MADE_PROGRESS | SAW_FAILURE; + goto cancel; + } + if (front->start + front->transferred > stream->collected_to) { + stream->collected_to = front->start + front->transferred; + stream->transferred = stream->collected_to - wreq->start; + stream->transferred_valid = true; + notes |= MADE_PROGRESS; + } + if (test_bit(NETFS_SREQ_FAILED, &front->flags)) { + stream->failed = true; + stream->error = front->error; + if (stream->source == NETFS_UPLOAD_TO_SERVER) + mapping_set_error(wreq->mapping, front->error); + notes |= NEED_REASSESS | SAW_FAILURE; + break; + } + if (front->transferred < front->len) { + stream->need_retry = true; + notes |= NEED_RETRY | MADE_PROGRESS; + break; + } + + cancel: + /* Remove if completely consumed. */ + spin_lock(&wreq->lock); + + remove = front; + list_del_init(&front->rreq_link); + front = list_first_entry_or_null(&stream->subrequests, + struct netfs_io_subrequest, rreq_link); + spin_unlock(&wreq->lock); + netfs_put_subrequest(remove, + notes & SAW_FAILURE ? + netfs_sreq_trace_put_cancel : + netfs_sreq_trace_put_done); + } + + /* If we have an empty stream, we need to jump it forward + * otherwise the collection point will never advance. + */ + if (!front && issued_to > stream->collected_to) { + trace_netfs_collect_gap(wreq, stream, issued_to, 'E'); + stream->collected_to = issued_to; + } + + if (stream->collected_to < collected_to) + collected_to = stream->collected_to; + } + + if (collected_to != ULLONG_MAX && collected_to > wreq->collected_to) + wreq->collected_to = collected_to; + + for (s = 0; s < NR_IO_STREAMS; s++) { + stream = &wreq->io_streams[s]; + if (stream->active) + trace_netfs_collect_stream(wreq, stream); + } + + trace_netfs_collect_state(wreq, wreq->collected_to, notes); + + /* Unlock any folios that we have now finished with. */ + if (notes & NEED_UNLOCK) { + if (wreq->cleaned_to < wreq->collected_to) + netfs_writeback_unlock_folios(wreq, ¬es); + } else { + wreq->cleaned_to = wreq->collected_to; + } + + // TODO: Discard encryption buffers + + if (notes & NEED_RETRY) + goto need_retry; + + if (notes & MADE_PROGRESS) { + netfs_wake_rreq_flag(wreq, NETFS_RREQ_PAUSE, netfs_rreq_trace_unpause); + //cond_resched(); + goto reassess_streams; + } + + if (notes & NEED_REASSESS) { + //cond_resched(); + goto reassess_streams; + } + +out: + netfs_put_group_many(wreq->group, wreq->nr_group_rel); + wreq->nr_group_rel = 0; + _leave(" = %x", notes); + return; + +need_retry: + /* Okay... We're going to have to retry one or both streams. Note + * that any partially completed op will have had any wholly transferred + * folios removed from it. + */ + _debug("retry"); + netfs_retry_writes(wreq); + goto out; +} + +/* + * Perform the collection of subrequests, folios and encryption buffers. + */ +bool netfs_write_collection(struct netfs_io_request *wreq) +{ + struct netfs_inode *ictx = netfs_inode(wreq->inode); + size_t transferred; + bool transferred_valid = false; + int s; + + _enter("R=%x", wreq->debug_id); + + netfs_collect_write_results(wreq); + + /* We're done when the app thread has finished posting subreqs and all + * the queues in all the streams are empty. + */ + if (!test_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags)) + return false; + smp_rmb(); /* Read ALL_QUEUED before lists. */ + + transferred = LONG_MAX; + for (s = 0; s < NR_IO_STREAMS; s++) { + struct netfs_io_stream *stream = &wreq->io_streams[s]; + if (!stream->active) + continue; + if (!list_empty(&stream->subrequests)) + return false; + if (stream->transferred_valid && + stream->transferred < transferred) { + transferred = stream->transferred; + transferred_valid = true; + } + } + + /* Okay, declare that all I/O is complete. */ + if (transferred_valid) + wreq->transferred = transferred; + trace_netfs_rreq(wreq, netfs_rreq_trace_write_done); + + if (wreq->io_streams[1].active && + wreq->io_streams[1].failed && + ictx->ops->invalidate_cache) { + /* Cache write failure doesn't prevent writeback completion + * unless we're in disconnected mode. + */ + ictx->ops->invalidate_cache(wreq); + } + + _debug("finished"); + netfs_wake_rreq_flag(wreq, NETFS_RREQ_IN_PROGRESS, netfs_rreq_trace_wake_ip); + /* As we cleared NETFS_RREQ_IN_PROGRESS, we acquired its ref. */ + + switch (wreq->origin) { + case NETFS_WRITEBACK: + case NETFS_WRITEBACK_SINGLE: + case NETFS_WRITETHROUGH: + netfs_wb_end(ictx); + break; + default: + break; + } + + if (wreq->iocb) { + size_t written = min(wreq->transferred, wreq->len); + wreq->iocb->ki_pos += written; + if (wreq->iocb->ki_complete) { + trace_netfs_rreq(wreq, netfs_rreq_trace_ki_complete); + wreq->iocb->ki_complete( + wreq->iocb, wreq->error ? wreq->error : written); + } + wreq->iocb = VFS_PTR_POISON; + } + + netfs_clear_subrequests(wreq); + return true; +} + +void netfs_write_collection_worker(struct work_struct *work) +{ + struct netfs_io_request *rreq = container_of(work, struct netfs_io_request, work); + + netfs_see_request(rreq, netfs_rreq_trace_see_work); + if (netfs_check_rreq_in_progress(rreq)) { + if (netfs_write_collection(rreq)) + /* Drop the ref from the IN_PROGRESS flag. */ + netfs_put_request(rreq, netfs_rreq_trace_put_work_ip); + else + netfs_see_request(rreq, netfs_rreq_trace_see_work_complete); + } +} + +/** + * netfs_write_subrequest_terminated - Note the termination of a write operation. + * @_op: The I/O request that has terminated. + * @transferred_or_error: The amount of data transferred or an error code. + * + * This tells the library that a contributory write I/O operation has + * terminated, one way or another, and that it should collect the results. + * + * The caller indicates in @transferred_or_error the outcome of the operation, + * supplying a positive value to indicate the number of bytes transferred or a + * negative error code. The library will look after reissuing I/O operations + * as appropriate and writing downloaded data to the cache. + * + * When this is called, ownership of the subrequest is transferred back to the + * library, along with a ref. + * + * Note that %_op is a void* so that the function can be passed to + * kiocb::term_func without the need for a casting wrapper. + */ +void netfs_write_subrequest_terminated(void *_op, ssize_t transferred_or_error) +{ + struct netfs_io_subrequest *subreq = _op; + struct netfs_io_request *wreq = subreq->rreq; + + _enter("%x[%x] %zd", wreq->debug_id, subreq->debug_index, transferred_or_error); + + switch (subreq->source) { + case NETFS_UPLOAD_TO_SERVER: + netfs_stat(&netfs_n_wh_upload_done); + break; + case NETFS_WRITE_TO_CACHE: + netfs_stat(&netfs_n_wh_write_done); + break; + default: + BUG(); + } + + if (IS_ERR_VALUE(transferred_or_error)) { + subreq->error = transferred_or_error; + /* if need retry is set, error should not matter */ + if (!test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { + set_bit(NETFS_SREQ_FAILED, &subreq->flags); + trace_netfs_failure(wreq, subreq, transferred_or_error, netfs_fail_write); + } + + switch (subreq->source) { + case NETFS_WRITE_TO_CACHE: + netfs_stat(&netfs_n_wh_write_failed); + break; + case NETFS_UPLOAD_TO_SERVER: + netfs_stat(&netfs_n_wh_upload_failed); + break; + default: + break; + } + trace_netfs_rreq(wreq, netfs_rreq_trace_set_pause); + set_bit(NETFS_RREQ_PAUSE, &wreq->flags); + } else { + if (WARN(transferred_or_error > subreq->len - subreq->transferred, + "Subreq excess write: R=%x[%x] %zd > %zu - %zu", + wreq->debug_id, subreq->debug_index, + transferred_or_error, subreq->len, subreq->transferred)) + transferred_or_error = subreq->len - subreq->transferred; + + subreq->error = 0; + subreq->transferred += transferred_or_error; + + if (subreq->transferred < subreq->len) + set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); + } + + trace_netfs_sreq(subreq, netfs_sreq_trace_terminated); + netfs_subreq_clear_in_progress(subreq); + netfs_put_subrequest(subreq, netfs_sreq_trace_put_terminated); +} +EXPORT_SYMBOL(netfs_write_subrequest_terminated); diff --git a/fs/netfs/write_issue.c b/fs/netfs/write_issue.c new file mode 100644 index 000000000..2d9cfcd43 --- /dev/null +++ b/fs/netfs/write_issue.c @@ -0,0 +1,906 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Network filesystem high-level (buffered) writeback. + * + * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + * + * + * To support network filesystems with local caching, we manage a situation + * that can be envisioned like the following: + * + * +---+---+-----+-----+---+----------+ + * Folios: | | | | | | | + * +---+---+-----+-----+---+----------+ + * + * +------+------+ +----+----+ + * Upload: | | |.....| | | + * (Stream 0) +------+------+ +----+----+ + * + * +------+------+------+------+------+ + * Cache: | | | | | | + * (Stream 1) +------+------+------+------+------+ + * + * Where we have a sequence of folios of varying sizes that we need to overlay + * with multiple parallel streams of I/O requests, where the I/O requests in a + * stream may also be of various sizes (in cifs, for example, the sizes are + * negotiated with the server; in something like ceph, they may represent the + * sizes of storage objects). + * + * The sequence in each stream may contain gaps and noncontiguous subrequests + * may be glued together into single vectored write RPCs. + */ + +#include +#include +#include +#include +#include "internal.h" + +/* + * Kill all dirty folios in the event of an unrecoverable error, starting with + * a locked folio we've already obtained from writeback_iter(). + */ +static void netfs_kill_dirty_pages(struct address_space *mapping, + struct writeback_control *wbc, + struct folio *folio) +{ + int error = 0; + + do { + enum netfs_folio_trace why = netfs_folio_trace_kill; + struct netfs_group *group = NULL; + struct netfs_folio *finfo = NULL; + void *priv; + + priv = folio_detach_private(folio); + if (priv) { + finfo = __netfs_folio_info(priv); + if (finfo) { + /* Kill folio from streaming write. */ + group = finfo->netfs_group; + why = netfs_folio_trace_kill_s; + } else { + group = priv; + if (group == NETFS_FOLIO_COPY_TO_CACHE) { + /* Kill copy-to-cache folio */ + why = netfs_folio_trace_kill_cc; + group = NULL; + } else { + /* Kill folio with group */ + why = netfs_folio_trace_kill_g; + } + } + } + + trace_netfs_folio(folio, why); + + folio_start_writeback(folio); + folio_unlock(folio); + folio_end_writeback(folio); + + netfs_put_group(group); + kfree(finfo); + + } while ((folio = writeback_iter(mapping, wbc, folio, &error))); +} + +/* + * Create a write request and set it up appropriately for the origin type. + */ +struct netfs_io_request *netfs_create_write_req(struct address_space *mapping, + struct file *file, + loff_t start, + enum netfs_io_origin origin) +{ + struct netfs_io_request *wreq; + struct netfs_inode *ictx; + bool is_cacheable = (origin == NETFS_WRITEBACK || + origin == NETFS_WRITEBACK_SINGLE || + origin == NETFS_WRITETHROUGH || + origin == NETFS_PGPRIV2_COPY_TO_CACHE); + + wreq = netfs_alloc_request(mapping, file, start, 0, origin); + if (IS_ERR(wreq)) + return wreq; + + _enter("R=%x", wreq->debug_id); + + ictx = netfs_inode(wreq->inode); + if (is_cacheable) + fscache_begin_write_operation(&wreq->cache_resources, netfs_i_cookie(ictx)); + if (rolling_buffer_init(&wreq->buffer, wreq->debug_id, ITER_SOURCE, wreq->gfp) < 0) + goto nomem; + + wreq->cleaned_to = wreq->start; + + wreq->io_streams[0].stream_nr = 0; + wreq->io_streams[0].source = NETFS_UPLOAD_TO_SERVER; + wreq->io_streams[0].prepare_write = ictx->ops->prepare_write; + wreq->io_streams[0].issue_write = ictx->ops->issue_write; + wreq->io_streams[0].collected_to = start; + wreq->io_streams[0].transferred = 0; + + wreq->io_streams[1].stream_nr = 1; + wreq->io_streams[1].source = NETFS_WRITE_TO_CACHE; + wreq->io_streams[1].collected_to = start; + wreq->io_streams[1].transferred = 0; + if (fscache_resources_valid(&wreq->cache_resources)) { + wreq->io_streams[1].avail = true; + wreq->io_streams[1].active = true; + wreq->io_streams[1].prepare_write = wreq->cache_resources.ops->prepare_write_subreq; + wreq->io_streams[1].issue_write = wreq->cache_resources.ops->issue_write; + } + + return wreq; +nomem: + netfs_put_failed_request(wreq); + return ERR_PTR(-ENOMEM); +} + +/** + * netfs_prepare_write_failed - Note write preparation failed + * @subreq: The subrequest to mark + * + * Mark a subrequest to note that preparation for write failed. + */ +void netfs_prepare_write_failed(struct netfs_io_subrequest *subreq) +{ + __set_bit(NETFS_SREQ_FAILED, &subreq->flags); + trace_netfs_sreq(subreq, netfs_sreq_trace_prep_failed); +} +EXPORT_SYMBOL(netfs_prepare_write_failed); + +/* + * Prepare a write subrequest. We need to allocate a new subrequest + * if we don't have one. + */ +void netfs_prepare_write(struct netfs_io_request *wreq, + struct netfs_io_stream *stream, + loff_t start) +{ + struct netfs_io_subrequest *subreq; + struct iov_iter *wreq_iter = &wreq->buffer.iter; + + /* Make sure we don't point the iterator at a used-up folio_queue + * struct being used as a placeholder to prevent the queue from + * collapsing. In such a case, extend the queue. + */ + if (iov_iter_is_folioq(wreq_iter) && + wreq_iter->folioq_slot >= folioq_nr_slots(wreq_iter->folioq)) + rolling_buffer_make_space(&wreq->buffer, wreq->gfp); + + subreq = netfs_alloc_subrequest(wreq); + subreq->source = stream->source; + subreq->start = start; + subreq->stream_nr = stream->stream_nr; + subreq->io_iter = *wreq_iter; + + _enter("R=%x[%x]", wreq->debug_id, subreq->debug_index); + + trace_netfs_sreq(subreq, netfs_sreq_trace_prepare); + + stream->sreq_max_len = UINT_MAX; + stream->sreq_max_segs = INT_MAX; + switch (stream->source) { + case NETFS_UPLOAD_TO_SERVER: + netfs_stat(&netfs_n_wh_upload); + stream->sreq_max_len = wreq->wsize; + break; + case NETFS_WRITE_TO_CACHE: + netfs_stat(&netfs_n_wh_write); + break; + default: + WARN_ON_ONCE(1); + break; + } + + if (stream->prepare_write) + stream->prepare_write(subreq); + + __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); + + /* We add to the end of the list whilst the collector may be walking + * the list. The collector only goes nextwards and uses the lock to + * remove entries off of the front. + */ + spin_lock(&wreq->lock); + /* Write IN_PROGRESS before pointer to new subreq */ + list_add_tail_release(&subreq->rreq_link, &stream->subrequests); + if (list_is_first(&subreq->rreq_link, &stream->subrequests)) { + if (!stream->active) { + stream->collected_to = subreq->start; + /* Write list pointers before active flag */ + smp_store_release(&stream->active, true); + } + } + + spin_unlock(&wreq->lock); + + stream->construct = subreq; +} + +/* + * Set the I/O iterator for the filesystem/cache to use and dispatch the I/O + * operation. The operation may be asynchronous and should call + * netfs_write_subrequest_terminated() when complete. + */ +static void netfs_do_issue_write(struct netfs_io_stream *stream, + struct netfs_io_subrequest *subreq) +{ + struct netfs_io_request *wreq = subreq->rreq; + + _enter("R=%x[%x],%zx", wreq->debug_id, subreq->debug_index, subreq->len); + + if (test_bit(NETFS_SREQ_FAILED, &subreq->flags)) + return netfs_write_subrequest_terminated(subreq, subreq->error); + + trace_netfs_sreq(subreq, netfs_sreq_trace_submit); + stream->issue_write(subreq); +} + +void netfs_reissue_write(struct netfs_io_stream *stream, + struct netfs_io_subrequest *subreq, + struct iov_iter *source) +{ + size_t size = subreq->len - subreq->transferred; + + // TODO: Use encrypted buffer + subreq->io_iter = *source; + iov_iter_advance(source, size); + iov_iter_truncate(&subreq->io_iter, size); + + subreq->retry_count++; + subreq->error = 0; + __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); + __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); + netfs_stat(&netfs_n_wh_retry_write_subreq); + netfs_do_issue_write(stream, subreq); +} + +void netfs_issue_write(struct netfs_io_request *wreq, + struct netfs_io_stream *stream) +{ + struct netfs_io_subrequest *subreq = stream->construct; + + if (!subreq) + return; + stream->construct = NULL; + subreq->io_iter.count = subreq->len; + netfs_do_issue_write(stream, subreq); +} + +/* + * Add data to the write subrequest, dispatching each as we fill it up or if it + * is discontiguous with the previous. We only fill one part at a time so that + * we can avoid overrunning the credits obtained (cifs) and try to parallelise + * content-crypto preparation with network writes. + */ +size_t netfs_advance_write(struct netfs_io_request *wreq, + struct netfs_io_stream *stream, + loff_t start, size_t len, bool to_eof) +{ + struct netfs_io_subrequest *subreq = stream->construct; + size_t part; + + if (!stream->avail) { + _leave("no write"); + return len; + } + + _enter("R=%x[%x]", wreq->debug_id, subreq ? subreq->debug_index : 0); + + if (subreq && start != subreq->start + subreq->len) { + netfs_issue_write(wreq, stream); + subreq = NULL; + } + + if (!stream->construct) + netfs_prepare_write(wreq, stream, start); + subreq = stream->construct; + + part = umin(stream->sreq_max_len - subreq->len, len); + _debug("part %zx/%zx %zx/%zx", subreq->len, stream->sreq_max_len, part, len); + subreq->len += part; + subreq->nr_segs++; + stream->submit_extendable_to -= part; + + if (subreq->len >= stream->sreq_max_len || + subreq->nr_segs >= stream->sreq_max_segs || + to_eof) { + netfs_issue_write(wreq, stream); + subreq = NULL; + } + + return part; +} + +/* + * Write some of a pending folio data back to the server. + */ +static int netfs_write_folio(struct netfs_io_request *wreq, + struct writeback_control *wbc, + struct folio *folio) +{ + struct netfs_io_stream *upload = &wreq->io_streams[0]; + struct netfs_io_stream *cache = &wreq->io_streams[1]; + struct netfs_io_stream *stream; + struct netfs_group *fgroup; /* TODO: Use this with ceph */ + struct netfs_folio *finfo; + size_t iter_off = 0; + size_t fsize = folio_size(folio), flen = fsize, foff = 0; + loff_t fpos = folio_pos(folio), i_size; + bool to_eof = false, streamw = false; + bool debug = false; + + _enter(""); + + if (rolling_buffer_make_space(&wreq->buffer, wreq->gfp) < 0) + return -ENOMEM; + + /* netfs_perform_write() may shift i_size around the page or from out + * of the page to beyond it, but cannot move i_size into or through the + * page since we have it locked. + */ + i_size = i_size_read(wreq->inode); + + if (fpos >= i_size) { + /* mmap beyond eof. */ + _debug("beyond eof"); + folio_start_writeback(folio); + folio_unlock(folio); + wreq->nr_group_rel += netfs_folio_written_back(folio); + netfs_put_group_many(wreq->group, wreq->nr_group_rel); + wreq->nr_group_rel = 0; + return 0; + } + + if (fpos + fsize > wreq->i_size) + wreq->i_size = i_size; + + fgroup = netfs_folio_group(folio); + finfo = netfs_folio_info(folio); + if (finfo) { + foff = finfo->dirty_offset; + flen = foff + finfo->dirty_len; + streamw = true; + } + + if (wreq->origin == NETFS_WRITETHROUGH) { + to_eof = false; + if (flen > i_size - fpos) + flen = i_size - fpos; + } else if (flen > i_size - fpos) { + flen = i_size - fpos; + if (!streamw) + folio_zero_segment(folio, flen, fsize); + to_eof = true; + } else if (flen == i_size - fpos) { + to_eof = true; + } + flen -= foff; + + _debug("folio %zx %zx %zx", foff, flen, fsize); + + /* Deal with discontinuities in the stream of dirty pages. These can + * arise from a number of sources: + * + * (1) Intervening non-dirty pages from random-access writes, multiple + * flushers writing back different parts simultaneously and manual + * syncing. + * + * (2) Partially-written pages from write-streaming. + * + * (3) Pages that belong to a different write-back group (eg. Ceph + * snapshots). + * + * (4) Actually-clean pages that were marked for write to the cache + * when they were read. Note that these appear as a special + * write-back group. + */ + if (fgroup == NETFS_FOLIO_COPY_TO_CACHE) { + netfs_issue_write(wreq, upload); + } else if (fgroup != wreq->group) { + /* We can't write this page to the server yet. */ + kdebug("wrong group"); + folio_redirty_for_writepage(wbc, folio); + folio_unlock(folio); + netfs_issue_write(wreq, upload); + netfs_issue_write(wreq, cache); + return 0; + } + + if (foff > 0) + netfs_issue_write(wreq, upload); + if (streamw) + netfs_issue_write(wreq, cache); + + folio_start_writeback(folio); + folio_unlock(folio); + + if (fgroup == NETFS_FOLIO_COPY_TO_CACHE) { + if (!cache->avail) { + trace_netfs_folio(folio, netfs_folio_trace_cancel_copy); + netfs_issue_write(wreq, upload); + netfs_folio_written_back(folio); + return 0; + } + trace_netfs_folio(folio, netfs_folio_trace_store_copy); + } else if (!upload->avail && !cache->avail) { + trace_netfs_folio(folio, netfs_folio_trace_cancel_store); + netfs_folio_written_back(folio); + return 0; + } else if (!upload->construct) { + trace_netfs_folio(folio, netfs_folio_trace_store); + } else { + trace_netfs_folio(folio, netfs_folio_trace_store_plus); + } + + /* Attach the folio to the rolling buffer. */ + rolling_buffer_append(&wreq->buffer, folio, 0, wreq->gfp); + + /* Move the submission point forward to allow for write-streaming data + * not starting at the front of the page. We don't do write-streaming + * with the cache as the cache requires DIO alignment. + * + * Also skip uploading for data that's been read and just needs copying + * to the cache. + */ + for (int s = 0; s < NR_IO_STREAMS; s++) { + stream = &wreq->io_streams[s]; + stream->submit_off = foff; + stream->submit_len = flen; + if (!stream->avail || + (stream->source == NETFS_WRITE_TO_CACHE && streamw) || + (stream->source == NETFS_UPLOAD_TO_SERVER && + fgroup == NETFS_FOLIO_COPY_TO_CACHE)) { + stream->submit_off = UINT_MAX; + stream->submit_len = 0; + } + } + + /* Attach the folio to one or more subrequests. For a big folio, we + * could end up with thousands of subrequests if the wsize is small - + * but we might need to wait during the creation of subrequests for + * network resources (eg. SMB credits). + */ + for (;;) { + ssize_t part; + size_t lowest_off = ULONG_MAX; + int choose_s = -1; + + /* Always add to the lowest-submitted stream first. */ + for (int s = 0; s < NR_IO_STREAMS; s++) { + stream = &wreq->io_streams[s]; + if (stream->submit_len > 0 && + stream->submit_off < lowest_off) { + lowest_off = stream->submit_off; + choose_s = s; + } + } + + if (choose_s < 0) + break; + stream = &wreq->io_streams[choose_s]; + + /* Advance the iterator(s). */ + if (stream->submit_off > iter_off) { + rolling_buffer_advance(&wreq->buffer, stream->submit_off - iter_off); + iter_off = stream->submit_off; + } + + atomic64_set(&wreq->issued_to, fpos + stream->submit_off); + stream->submit_extendable_to = fsize - stream->submit_off; + part = netfs_advance_write(wreq, stream, fpos + stream->submit_off, + stream->submit_len, to_eof); + stream->submit_off += part; + if (part > stream->submit_len) + stream->submit_len = 0; + else + stream->submit_len -= part; + if (part > 0) + debug = true; + } + + if (fsize > iter_off) + rolling_buffer_advance(&wreq->buffer, fsize - iter_off); + atomic64_set(&wreq->issued_to, fpos + fsize); + + if (!debug) + kdebug("R=%x: No submit", wreq->debug_id); + + if (foff + flen < fsize) + for (int s = 0; s < NR_IO_STREAMS; s++) + netfs_issue_write(wreq, &wreq->io_streams[s]); + + _leave(" = 0"); + return 0; +} + +/* + * End the issuing of writes, letting the collector know we're done. + */ +static void netfs_end_issue_write(struct netfs_io_request *wreq) +{ + bool needs_poke = true; + + smp_wmb(); /* Write subreq lists before ALL_QUEUED. */ + set_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags); + + for (int s = 0; s < NR_IO_STREAMS; s++) { + struct netfs_io_stream *stream = &wreq->io_streams[s]; + + if (!stream->active) + continue; + if (!list_empty(&stream->subrequests)) + needs_poke = false; + netfs_issue_write(wreq, stream); + } + + if (needs_poke) + netfs_wake_collector(wreq); +} + +/* + * Write some of the pending data back to the server + */ +int netfs_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct netfs_inode *ictx = netfs_inode(mapping->host); + struct netfs_io_request *wreq = NULL; + struct folio *folio; + int error = 0; + + if (!netfs_wb_begin(ictx, wbc->sync_mode == WB_SYNC_NONE)) + return 0; + + /* Need the first folio to be able to set up the op. */ + folio = writeback_iter(mapping, wbc, NULL, &error); + if (!folio) + goto out; + + wreq = netfs_create_write_req(mapping, NULL, folio_pos(folio), NETFS_WRITEBACK); + if (IS_ERR(wreq)) { + error = PTR_ERR(wreq); + goto couldnt_start; + } + + __set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &wreq->flags); + trace_netfs_write(wreq, netfs_write_trace_writeback); + netfs_stat(&netfs_n_wh_writepages); + + do { + _debug("wbiter %lx %llx", folio->index, atomic64_read(&wreq->issued_to)); + + /* It appears we don't have to handle cyclic writeback wrapping. */ + WARN_ON_ONCE(wreq && folio_pos(folio) < atomic64_read(&wreq->issued_to)); + + if (netfs_folio_group(folio) != NETFS_FOLIO_COPY_TO_CACHE && + unlikely(!test_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags))) { + set_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags); + wreq->netfs_ops->begin_writeback(wreq); + } + + error = netfs_write_folio(wreq, wbc, folio); + if (error == -ENOMEM) { + folio_redirty_for_writepage(wbc, folio); + folio_unlock(folio); + } + } while ((folio = writeback_iter(mapping, wbc, folio, &error))); + + netfs_end_issue_write(wreq); + netfs_wake_collector(wreq); + + netfs_put_request(wreq, netfs_rreq_trace_put_return); + _leave(" = %d", error); + return error; + +couldnt_start: + if (error == -ENOMEM) { + folio_redirty_for_writepage(wbc, folio); + folio_unlock(folio); + folio = writeback_iter(mapping, wbc, folio, &error); + WARN_ON_ONCE(folio != NULL); + } else { + netfs_kill_dirty_pages(mapping, wbc, folio); + } +out: + netfs_wb_end(ictx); + _leave(" = %d", error); + return error; +} +EXPORT_SYMBOL(netfs_writepages); + +/* + * Begin a write operation for writing through the pagecache. + */ +struct netfs_io_request *netfs_begin_writethrough(struct kiocb *iocb, size_t len) +{ + struct netfs_io_request *wreq = NULL; + struct netfs_inode *ictx = netfs_inode(file_inode(iocb->ki_filp)); + + netfs_wb_begin(ictx, false); + + wreq = netfs_create_write_req(iocb->ki_filp->f_mapping, iocb->ki_filp, + iocb->ki_pos, NETFS_WRITETHROUGH); + if (IS_ERR(wreq)) { + netfs_wb_end(ictx); + return wreq; + } + + wreq->io_streams[0].avail = true; + __set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &wreq->flags); + trace_netfs_write(wreq, netfs_write_trace_writethrough); + return wreq; +} + +/* + * Advance the state of the write operation used when writing through the + * pagecache. Data has been copied into the pagecache that we need to append + * to the request. If we've added more than wsize then we need to create a new + * subrequest. + */ +int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc, + struct folio *folio, size_t copied, bool to_page_end, + struct folio **writethrough_cache) +{ + int ret; + + _enter("R=%x ic=%zu ws=%u cp=%zu tp=%u", + wreq->debug_id, wreq->buffer.iter.count, wreq->wsize, copied, to_page_end); + + /* The folio is locked. */ + + if (*writethrough_cache != folio) { + if (*writethrough_cache) { + /* Did the folio get moved? */ + folio_put(*writethrough_cache); + *writethrough_cache = NULL; + } + /* We can make multiple writes to the folio... */ + if (wreq->len == 0) + trace_netfs_folio(folio, netfs_folio_trace_wthru); + else + trace_netfs_folio(folio, netfs_folio_trace_wthru_plus); + *writethrough_cache = folio; + folio_get(folio); + } + + wreq->len += copied; + + if (!to_page_end) { + folio_mark_dirty(folio); + folio_unlock(folio); + return 0; + } + + ret = netfs_write_folio(wreq, wbc, folio); + folio_put(*writethrough_cache); + *writethrough_cache = NULL; + wreq->submitted = wreq->len; + return ret; +} + +/* + * End a write operation used when writing through the pagecache. + */ +ssize_t netfs_end_writethrough(struct netfs_io_request *wreq, struct writeback_control *wbc, + struct folio *writethrough_cache) +{ + ssize_t ret; + + _enter("R=%x", wreq->debug_id); + + if (writethrough_cache) { + folio_lock(writethrough_cache); + netfs_write_folio(wreq, wbc, writethrough_cache); + folio_put(writethrough_cache); + wreq->submitted = wreq->len; + } + + netfs_end_issue_write(wreq); + + if (wreq->iocb) + ret = -EIOCBQUEUED; + else + ret = netfs_wait_for_write(wreq); + netfs_put_request(wreq, netfs_rreq_trace_put_return); + return ret; +} + +/* + * Write some of a pending folio data back to the server and/or the cache. + */ +static int netfs_write_folio_single(struct netfs_io_request *wreq, + struct folio *folio) +{ + struct netfs_io_stream *upload = &wreq->io_streams[0]; + struct netfs_io_stream *cache = &wreq->io_streams[1]; + struct netfs_io_stream *stream; + size_t iter_off = 0; + size_t fsize = folio_size(folio), flen; + loff_t fpos = folio_pos(folio); + ssize_t ret; + bool to_eof = false; + bool no_debug = false; + + _enter(""); + + flen = folio_size(folio); + if (flen > wreq->i_size - fpos) { + flen = wreq->i_size - fpos; + folio_zero_segment(folio, flen, fsize); + to_eof = true; + } else if (flen == wreq->i_size - fpos) { + to_eof = true; + } + + _debug("folio %zx/%zx", flen, fsize); + + if (!upload->avail && !cache->avail) { + trace_netfs_folio(folio, netfs_folio_trace_cancel_store); + return 0; + } + + if (!upload->construct) + trace_netfs_folio(folio, netfs_folio_trace_store); + else + trace_netfs_folio(folio, netfs_folio_trace_store_plus); + + /* Attach the folio to the rolling buffer. */ + folio_get(folio); + ret = rolling_buffer_append(&wreq->buffer, folio, NETFS_ROLLBUF_PUT_MARK, wreq->gfp); + if (ret < 0) { + folio_put(folio); + return ret; + } + + /* Move the submission point forward to allow for write-streaming data + * not starting at the front of the page. We don't do write-streaming + * with the cache as the cache requires DIO alignment. + * + * Also skip uploading for data that's been read and just needs copying + * to the cache. + */ + for (int s = 0; s < NR_IO_STREAMS; s++) { + stream = &wreq->io_streams[s]; + stream->submit_off = 0; + stream->submit_len = flen; + if (!stream->avail) { + stream->submit_off = UINT_MAX; + stream->submit_len = 0; + } + } + + /* Attach the folio to one or more subrequests. For a big folio, we + * could end up with thousands of subrequests if the wsize is small - + * but we might need to wait during the creation of subrequests for + * network resources (eg. SMB credits). + */ + for (;;) { + ssize_t part; + size_t lowest_off = ULONG_MAX; + int choose_s = -1; + + /* Always add to the lowest-submitted stream first. */ + for (int s = 0; s < NR_IO_STREAMS; s++) { + stream = &wreq->io_streams[s]; + if (stream->submit_len > 0 && + stream->submit_off < lowest_off) { + lowest_off = stream->submit_off; + choose_s = s; + } + } + + if (choose_s < 0) + break; + stream = &wreq->io_streams[choose_s]; + + /* Advance the iterator(s). */ + if (stream->submit_off > iter_off) { + rolling_buffer_advance(&wreq->buffer, stream->submit_off - iter_off); + iter_off = stream->submit_off; + } + + atomic64_set(&wreq->issued_to, fpos + stream->submit_off); + stream->submit_extendable_to = fsize - stream->submit_off; + part = netfs_advance_write(wreq, stream, fpos + stream->submit_off, + stream->submit_len, to_eof); + stream->submit_off += part; + if (part > stream->submit_len) + stream->submit_len = 0; + else + stream->submit_len -= part; + if (part > 0) + no_debug = true; + } + + wreq->buffer.iter.iov_offset = 0; + if (fsize > iter_off) + rolling_buffer_advance(&wreq->buffer, fsize - iter_off); + atomic64_set(&wreq->issued_to, fpos + fsize); + + if (!no_debug) + kdebug("R=%x: No submit", wreq->debug_id); + _leave(" = 0"); + return 0; +} + +/** + * netfs_writeback_single - Write back a monolithic payload + * @mapping: The mapping to write from + * @wbc: Hints from the VM + * @iter: Data to write, must be ITER_FOLIOQ. + * + * Write a monolithic, non-pagecache object back to the server and/or + * the cache. + * + * Return: 0 if successful; 1 if skipped due to lock conflict and WB_SYNC_NONE; + * or a negative error code. + */ +int netfs_writeback_single(struct address_space *mapping, + struct writeback_control *wbc, + struct iov_iter *iter) +{ + struct netfs_io_request *wreq; + struct netfs_inode *ictx = netfs_inode(mapping->host); + struct folio_queue *fq; + size_t size = iov_iter_count(iter); + int ret; + + if (WARN_ON_ONCE(!iov_iter_is_folioq(iter))) + return -EIO; + + if (!netfs_wb_begin(ictx, wbc->sync_mode == WB_SYNC_NONE)) { + /* The VFS will have undirtied the inode. */ + netfs_single_mark_inode_dirty(&ictx->inode); + return 1; + } + + wreq = netfs_create_write_req(mapping, NULL, 0, NETFS_WRITEBACK_SINGLE); + if (IS_ERR(wreq)) { + ret = PTR_ERR(wreq); + goto couldnt_start; + } + + __set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &wreq->flags); + trace_netfs_write(wreq, netfs_write_trace_writeback_single); + netfs_stat(&netfs_n_wh_writepages); + + if (__test_and_set_bit(NETFS_RREQ_UPLOAD_TO_SERVER, &wreq->flags)) + wreq->netfs_ops->begin_writeback(wreq); + + for (fq = (struct folio_queue *)iter->folioq; fq; fq = fq->next) { + for (int slot = 0; slot < folioq_count(fq); slot++) { + struct folio *folio = folioq_folio(fq, slot); + size_t part = umin(folioq_folio_size(fq, slot), size); + + _debug("wbiter %lx %llx", folio->index, atomic64_read(&wreq->issued_to)); + + ret = netfs_write_folio_single(wreq, folio); + if (ret < 0) + goto stop; + size -= part; + if (size <= 0) + goto stop; + } + } + +stop: + for (int s = 0; s < NR_IO_STREAMS; s++) + netfs_issue_write(wreq, &wreq->io_streams[s]); + smp_wmb(); /* Write lists before ALL_QUEUED. */ + set_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags); + + netfs_wake_collector(wreq); + + netfs_put_request(wreq, netfs_rreq_trace_put_return); + _leave(" = %d", ret); + return ret; + +couldnt_start: + netfs_wb_end(ictx); + _leave(" = %d", ret); + return ret; +} +EXPORT_SYMBOL(netfs_writeback_single); diff --git a/fs/netfs/write_retry.c b/fs/netfs/write_retry.c new file mode 100644 index 000000000..058bc7a16 --- /dev/null +++ b/fs/netfs/write_retry.c @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Network filesystem write retrying. + * + * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include "internal.h" + +/* + * Perform retries on the streams that need it. + */ +static void netfs_retry_write_stream(struct netfs_io_request *wreq, + struct netfs_io_stream *stream) +{ + struct list_head *next; + + _enter("R=%x[%x:]", wreq->debug_id, stream->stream_nr); + + if (list_empty(&stream->subrequests)) + return; + + if (stream->source == NETFS_UPLOAD_TO_SERVER && + wreq->netfs_ops->retry_request) + wreq->netfs_ops->retry_request(wreq, stream); + + if (unlikely(stream->failed)) + return; + + /* If there's no renegotiation to do, just resend each failed subreq. */ + if (!stream->prepare_write) { + struct netfs_io_subrequest *subreq; + + list_for_each_entry(subreq, &stream->subrequests, rreq_link) { + if (test_bit(NETFS_SREQ_FAILED, &subreq->flags)) + break; + if (__test_and_clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { + struct iov_iter source; + + netfs_reset_iter(subreq); + source = subreq->io_iter; + netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); + netfs_reissue_write(stream, subreq, &source); + } + } + return; + } + + next = stream->subrequests.next; + + do { + struct netfs_io_subrequest *subreq = NULL, *from, *to, *tmp; + struct iov_iter source; + unsigned long long start, len; + size_t part; + bool boundary = false; + + /* Go through the stream and find the next span of contiguous + * data that we then rejig (cifs, for example, needs the wsize + * renegotiating) and reissue. + */ + from = list_entry(next, struct netfs_io_subrequest, rreq_link); + to = from; + start = from->start + from->transferred; + len = from->len - from->transferred; + + if (test_bit(NETFS_SREQ_FAILED, &from->flags) || + !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags)) + return; + + for (;;) { + /* Read pointer to subreq before reading subreq state. */ + next = smp_load_acquire(&next->next); + if (next == &stream->subrequests) + break; + + subreq = list_entry(next, struct netfs_io_subrequest, rreq_link); + if (subreq->start + subreq->transferred != start + len || + test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags) || + !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) + break; + to = subreq; + len += to->len; + } + + /* Determine the set of buffers we're going to use. Each + * subreq gets a subset of a single overall contiguous buffer. + */ + netfs_reset_iter(from); + source = from->io_iter; + source.count = len; + + /* Work through the sublist. */ + subreq = from; + list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) { + if (!len) + break; + + subreq->start = start; + subreq->len = len; + __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); + trace_netfs_sreq(subreq, netfs_sreq_trace_retry); + + /* Renegotiate max_len (wsize) */ + stream->sreq_max_len = len; + stream->prepare_write(subreq); + + part = umin(len, stream->sreq_max_len); + if (unlikely(stream->sreq_max_segs)) + part = netfs_limit_iter(&source, 0, part, stream->sreq_max_segs); + subreq->len = part; + subreq->transferred = 0; + len -= part; + start += part; + if (len && subreq == to && + __test_and_clear_bit(NETFS_SREQ_BOUNDARY, &to->flags)) + boundary = true; + + netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); + netfs_reissue_write(stream, subreq, &source); + if (subreq == to) + break; + } + + /* If we managed to use fewer subreqs, we can discard the + * excess; if we used the same number, then we're done. + */ + if (!len) { + if (subreq == to) + continue; + list_for_each_entry_safe_from(subreq, tmp, + &stream->subrequests, rreq_link) { + trace_netfs_sreq(subreq, netfs_sreq_trace_discard); + spin_lock(&wreq->lock); + list_del(&subreq->rreq_link); + spin_unlock(&wreq->lock); + netfs_put_subrequest(subreq, netfs_sreq_trace_put_done); + if (subreq == to) + break; + } + continue; + } + + /* We ran out of subrequests, so we need to allocate some more + * and insert them after. + */ + do { + subreq = netfs_alloc_subrequest(wreq); + subreq->source = to->source; + subreq->start = start; + subreq->stream_nr = to->stream_nr; + subreq->retry_count = 1; + + trace_netfs_sreq_ref(wreq->debug_id, subreq->debug_index, + refcount_read(&subreq->ref), + netfs_sreq_trace_new); + trace_netfs_sreq(subreq, netfs_sreq_trace_split); + + spin_lock(&wreq->lock); + list_add(&subreq->rreq_link, &to->rreq_link); + spin_unlock(&wreq->lock); + to = subreq; + trace_netfs_sreq(subreq, netfs_sreq_trace_retry); + + stream->sreq_max_len = len; + stream->sreq_max_segs = INT_MAX; + switch (stream->source) { + case NETFS_UPLOAD_TO_SERVER: + netfs_stat(&netfs_n_wh_upload); + stream->sreq_max_len = umin(len, wreq->wsize); + break; + case NETFS_WRITE_TO_CACHE: + netfs_stat(&netfs_n_wh_write); + break; + default: + WARN_ON_ONCE(1); + } + + stream->prepare_write(subreq); + + part = umin(len, stream->sreq_max_len); + subreq->len = subreq->transferred + part; + len -= part; + start += part; + if (!len && boundary) { + __set_bit(NETFS_SREQ_BOUNDARY, &to->flags); + boundary = false; + } + + netfs_reissue_write(stream, subreq, &source); + if (!len) + break; + + } while (len); + + } while (!list_is_head(next, &stream->subrequests)); +} + +/* + * Perform retries on the streams that need it. If we're doing content + * encryption and the server copy changed due to a third-party write, we may + * need to do an RMW cycle and also rewrite the data to the cache. + */ +void netfs_retry_writes(struct netfs_io_request *wreq) +{ + struct netfs_io_stream *stream; + int s; + + netfs_stat(&netfs_n_wh_retry_write_req); + + /* Wait for all outstanding I/O to quiesce before performing retries as + * we may need to renegotiate the I/O sizes. + */ + set_bit(NETFS_RREQ_RETRYING, &wreq->flags); + for (s = 0; s < NR_IO_STREAMS; s++) { + stream = &wreq->io_streams[s]; + if (stream->active) + netfs_wait_for_in_progress_stream(wreq, stream); + } + clear_bit(NETFS_RREQ_RETRYING, &wreq->flags); + + // TODO: Enc: Fetch changed partial pages + // TODO: Enc: Reencrypt content if needed. + // TODO: Enc: Wind back transferred point. + // TODO: Enc: Mark cache pages for retry. + + for (s = 0; s < NR_IO_STREAMS; s++) { + stream = &wreq->io_streams[s]; + if (stream->need_retry) { + stream->need_retry = false; + netfs_retry_write_stream(wreq, stream); + } + } +} diff --git a/fs/nfs/Kconfig b/fs/nfs/Kconfig new file mode 100644 index 000000000..6bb30543e --- /dev/null +++ b/fs/nfs/Kconfig @@ -0,0 +1,217 @@ +# SPDX-License-Identifier: GPL-2.0-only +config NFS_FS + tristate "NFS client support" + depends on INET && FILE_LOCKING && MULTIUSER + select CRC32 + select LOCKD + select SUNRPC + select NFS_COMMON + select NFS_ACL_SUPPORT if NFS_V3_ACL + help + Choose Y here if you want to access files residing on other + computers using Sun's Network File System protocol. To compile + this file system support as a module, choose M here: the module + will be called nfs. + + To mount file systems exported by NFS servers, you also need to + install the user space mount.nfs command which can be found in + the Linux nfs-utils package, available from http://linux-nfs.org/. + Information about using the mount command is available in the + mount(8) man page. More detail about the Linux NFS client + implementation is available via the nfs(5) man page. + + Below you can choose which versions of the NFS protocol are + available in the kernel to mount NFS servers. Support for NFS + version 2 (RFC 1094) is always available when NFS_FS is selected. + + To configure a system which mounts its root file system via NFS + at boot time, say Y here, select "Kernel level IP + autoconfiguration" in the NETWORK menu, and select "Root file + system on NFS" below. You cannot compile this file system as a + module in this case. + + If unsure, say N. + +config NFS_V2 + tristate "NFS client support for NFS version 2" + depends on NFS_FS + default n + help + This option enables support for version 2 of the NFS protocol + (RFC 1094) in the kernel's NFS client. + + If unsure, say N. + +config NFS_V3 + tristate "NFS client support for NFS version 3" + depends on NFS_FS + default y + help + This option enables support for version 3 of the NFS protocol + (RFC 1813) in the kernel's NFS client. + + If unsure, say Y. + +config NFS_V3_ACL + bool "NFS client support for the NFSv3 ACL protocol extension" + depends on NFS_V3 + help + Some NFS servers support an auxiliary NFSv3 ACL protocol that + Sun added to Solaris but never became an official part of the + NFS version 3 protocol. This protocol extension allows + applications on NFS clients to manipulate POSIX Access Control + Lists on files residing on NFS servers. NFS servers enforce + ACLs on local files whether this protocol is available or not. + + Choose Y here if your NFS server supports the Solaris NFSv3 ACL + protocol extension and you want your NFS client to allow + applications to access and modify ACLs on files on the server. + + Most NFS servers don't support the Solaris NFSv3 ACL protocol + extension. You can choose N here or specify the "noacl" mount + option to prevent your NFS client from trying to use the NFSv3 + ACL protocol. + + If unsure, say N. + +config NFS_V4 + tristate "NFS client support for NFS version 4" + depends on NFS_FS + select KEYS + select SUNRPC_BACKCHANNEL + help + This option enables support for version 4.1 of the NFS protocol + (RFC 5661) in the kernel's NFS client. + + To mount NFS servers using NFSv4, you also need to install user + space programs which can be found in the Linux nfs-utils package, + available from http://linux-nfs.org/. + + If unsure, say N. + +config NFS_SWAP + bool "Provide swap over NFS support" + default n + depends on NFS_FS && SWAP + select SUNRPC_SWAP + help + This option enables swapon to work on files located on NFS mounts. + +config NFS_V4_0 + bool "NFS client support for NFSv4.0" + depends on NFS_V4 + default y + help + This option enables support for minor version 0 of the NFSv4 protocol + (RFC 3530) in the kernel's NFS client. + + If unsure, say N. + +config NFS_V4_2 + bool "NFS client support for NFSv4.2" + depends on NFS_V4 + help + This option enables support for minor version 2 of the NFSv4 protocol + in the kernel's NFS client. + + If unsure, say N. + +config PNFS_FILE_LAYOUT + tristate + depends on NFS_V4 + default NFS_V4 + +config PNFS_BLOCK + tristate + depends on NFS_V4 && BLK_DEV_DM + default NFS_V4 + +config PNFS_FLEXFILE_LAYOUT + tristate + depends on NFS_V4 + default NFS_V4 + +config NFS_V4_1_IMPLEMENTATION_ID_DOMAIN + string "NFSv4.1 Implementation ID Domain" + depends on NFS_V4 + default "kernel.org" + help + This option defines the domain portion of the implementation ID that + may be sent in the NFS exchange_id operation. The value must be in + the format of a DNS domain name and should be set to the DNS domain + name of the distribution. + If the NFS client is unchanged from the upstream kernel, this + option should be set to the default "kernel.org". + +config NFS_V4_1_MIGRATION + bool "NFSv4.1 client support for migration" + depends on NFS_V4 + default n + help + This option makes the NFS client advertise to NFSv4.1 servers that + it can support NFSv4 migration. + + The NFSv4.1 pieces of the Linux NFSv4 migration implementation are + still experimental. If you are not an NFSv4 developer, say N here. + +config NFS_V4_SECURITY_LABEL + bool + depends on NFS_V4_2 && SECURITY + default y + +config ROOT_NFS + bool "Root file system on NFS" + depends on NFS_FS=y && IP_PNP + help + If you want your system to mount its root file system via NFS, + choose Y here. This is common practice for managing systems + without local permanent storage. For details, read + . + + Most people say N here. + +config NFS_FSCACHE + bool "Provide NFS client caching support" + depends on NFS_FS + select NETFS_SUPPORT + select FSCACHE + help + Say Y here if you want NFS data to be cached locally on disc through + the general filesystem cache manager + +config NFS_USE_LEGACY_DNS + bool "Use the legacy NFS DNS resolver" + depends on NFS_V4 + help + The kernel now provides a method for translating a host name into an + IP address. Select Y here if you would rather use your own DNS + resolver script. + + If unsure, say N + +config NFS_USE_KERNEL_DNS + bool + depends on NFS_V4 && !NFS_USE_LEGACY_DNS + select DNS_RESOLVER + default y + +config NFS_DEBUG + bool + depends on NFS_FS && SUNRPC_DEBUG + default y + +config NFS_DISABLE_UDP_SUPPORT + bool "NFS: Disable NFS UDP protocol support" + depends on NFS_FS + default y + help + Choose Y here to disable the use of NFS over UDP. NFS over UDP + on modern networks (1Gb+) can lead to data corruption caused by + fragmentation during high loads. + +config NFS_V4_2_READ_PLUS + bool "NFS: Enable support for the NFSv4.2 READ_PLUS operation" + depends on NFS_V4_2 + default y + help + Choose Y here to enable use of the NFS v4.2 READ_PLUS operation. diff --git a/fs/nfs/Makefile b/fs/nfs/Makefile new file mode 100644 index 000000000..c895521f2 --- /dev/null +++ b/fs/nfs/Makefile @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the Linux nfs filesystem routines. +# + +obj-$(CONFIG_NFS_FS) += nfs.o + +CFLAGS_nfstrace.o += -I$(src) +nfs-y := client.o dir.o file.o getroot.o inode.o super.o \ + io.o direct.o pagelist.o read.o symlink.o unlink.o \ + write.o namespace.o mount_clnt.o nfstrace.o \ + export.o sysfs.o fs_context.o +nfs-$(CONFIG_ROOT_NFS) += nfsroot.o +nfs-$(CONFIG_SYSCTL) += sysctl.o +nfs-$(CONFIG_NFS_FSCACHE) += fscache.o +nfs-$(CONFIG_NFS_LOCALIO) += localio.o + +obj-$(CONFIG_NFS_V2) += nfsv2.o +nfsv2-y := nfs2super.o proc.o nfs2xdr.o + +obj-$(CONFIG_NFS_V3) += nfsv3.o +nfsv3-y := nfs3super.o nfs3client.o nfs3proc.o nfs3xdr.o +nfsv3-$(CONFIG_NFS_V3_ACL) += nfs3acl.o + +obj-$(CONFIG_NFS_V4) += nfsv4.o +CFLAGS_nfs4trace.o += -I$(src) +nfsv4-y := nfs4proc.o nfs4xdr.o nfs4state.o nfs4renewd.o nfs4super.o nfs4file.o \ + delegation.o nfs4idmap.o callback.o callback_xdr.o callback_proc.o \ + nfs4namespace.o nfs4getroot.o nfs4client.o nfs4session.o \ + dns_resolve.o nfs4trace.o pnfs.o pnfs_dev.o pnfs_nfs.o +nfsv4-$(CONFIG_NFS_USE_LEGACY_DNS) += cache_lib.o +nfsv4-$(CONFIG_SYSCTL) += nfs4sysctl.o +nfsv4-$(CONFIG_NFS_V4_0) += nfs40client.o nfs40proc.o +nfsv4-$(CONFIG_NFS_V4_2) += nfs42proc.o nfs42xattr.o + +obj-$(CONFIG_PNFS_FILE_LAYOUT) += filelayout/ +obj-$(CONFIG_PNFS_BLOCK) += blocklayout/ +obj-$(CONFIG_PNFS_FLEXFILE_LAYOUT) += flexfilelayout/ diff --git a/fs/nfs/blocklayout/Makefile b/fs/nfs/blocklayout/Makefile new file mode 100644 index 000000000..7668a1bfb --- /dev/null +++ b/fs/nfs/blocklayout/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the pNFS block layout driver kernel module +# +obj-$(CONFIG_PNFS_BLOCK) += blocklayoutdriver.o + +blocklayoutdriver-y += blocklayout.o dev.o extent_tree.o rpc_pipefs.o diff --git a/fs/nfs/blocklayout/blocklayout.c b/fs/nfs/blocklayout/blocklayout.c new file mode 100644 index 000000000..d54a141a8 --- /dev/null +++ b/fs/nfs/blocklayout/blocklayout.c @@ -0,0 +1,1047 @@ +/* + * linux/fs/nfs/blocklayout/blocklayout.c + * + * Module for the NFSv4.1 pNFS block layout driver. + * + * Copyright (c) 2006 The Regents of the University of Michigan. + * All rights reserved. + * + * Andy Adamson + * Fred Isaman + * + * permission is granted to use, copy, create derivative works and + * redistribute this software and such derivative works for any purpose, + * so long as the name of the university of michigan is not used in + * any advertising or publicity pertaining to the use or distribution + * of this software without specific, written prior authorization. if + * the above copyright notice or any other identification of the + * university of michigan is included in any copy of any portion of + * this software, then the disclaimer below must also be included. + * + * this software is provided as is, without representation from the + * university of michigan as to its fitness for any purpose, and without + * warranty by the university of michigan of any kind, either express + * or implied, including without limitation the implied warranties of + * merchantability and fitness for a particular purpose. the regents + * of the university of michigan shall not be liable for any damages, + * including special, indirect, incidental, or consequential damages, + * with respect to any claim arising out or in connection with the use + * of the software, even if it has been or is hereafter advised of the + * possibility of such damages. + */ + +#include +#include +#include +#include +#include /* struct bio */ +#include + +#include "../pnfs.h" +#include "../nfs4session.h" +#include "../internal.h" +#include "blocklayout.h" + +#define NFSDBG_FACILITY NFSDBG_PNFS_LD + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Andy Adamson "); +MODULE_DESCRIPTION("The NFSv4.1 pNFS Block layout driver"); + +static bool is_hole(struct pnfs_block_extent *be) +{ + switch (be->be_state) { + case PNFS_BLOCK_NONE_DATA: + return true; + case PNFS_BLOCK_INVALID_DATA: + return be->be_tag ? false : true; + default: + return false; + } +} + +/* The data we are handed might be spread across several bios. We need + * to track when the last one is finished. + */ +struct parallel_io { + struct kref refcnt; + void (*pnfs_callback) (void *data); + void *data; +}; + +static inline struct parallel_io *alloc_parallel(void *data) +{ + struct parallel_io *rv; + + rv = kmalloc_obj(*rv, GFP_NOFS); + if (rv) { + rv->data = data; + kref_init(&rv->refcnt); + } + return rv; +} + +static inline void get_parallel(struct parallel_io *p) +{ + kref_get(&p->refcnt); +} + +static void destroy_parallel(struct kref *kref) +{ + struct parallel_io *p = container_of(kref, struct parallel_io, refcnt); + + dprintk("%s enter\n", __func__); + p->pnfs_callback(p->data); + kfree(p); +} + +static inline void put_parallel(struct parallel_io *p) +{ + kref_put(&p->refcnt, destroy_parallel); +} + +static struct bio * +bl_submit_bio(struct bio *bio) +{ + if (bio) { + get_parallel(bio->bi_private); + dprintk("%s submitting %s bio %u@%llu\n", __func__, + bio_op(bio) == READ ? "read" : "write", + bio->bi_iter.bi_size, + (unsigned long long)bio->bi_iter.bi_sector); + submit_bio(bio); + } + return NULL; +} + +static bool offset_in_map(u64 offset, struct pnfs_block_dev_map *map) +{ + return offset >= map->start && offset < map->start + map->len; +} + +static struct bio * +do_add_page_to_bio(struct bio *bio, int npg, enum req_op op, sector_t isect, + struct page *page, struct pnfs_block_dev_map *map, + struct pnfs_block_extent *be, bio_end_io_t end_io, + struct parallel_io *par, unsigned int offset, int *len) +{ + struct pnfs_block_dev *dev = + container_of(be->be_device, struct pnfs_block_dev, node); + u64 disk_addr, end; + + dprintk("%s: npg %d rw %d isect %llu offset %u len %d\n", __func__, + npg, (__force u32)op, (unsigned long long)isect, offset, *len); + + /* translate to device offset */ + isect += be->be_v_offset; + isect -= be->be_f_offset; + + /* translate to physical disk offset */ + disk_addr = (u64)isect << SECTOR_SHIFT; + if (!offset_in_map(disk_addr, map)) { + if (!dev->map(dev, disk_addr, map) || !offset_in_map(disk_addr, map)) + return ERR_PTR(-EIO); + bio = bl_submit_bio(bio); + } + disk_addr += map->disk_offset; + disk_addr -= map->start; + + /* limit length to what the device mapping allows */ + end = disk_addr + *len; + if (end >= map->disk_offset + map->len) + *len = map->disk_offset + map->len - disk_addr; + +retry: + if (!bio) { + bio = bio_alloc(map->bdev, bio_max_segs(npg), op, GFP_NOIO); + bio->bi_iter.bi_sector = disk_addr >> SECTOR_SHIFT; + bio->bi_end_io = end_io; + bio->bi_private = par; + } + if (bio_add_page(bio, page, *len, offset) < *len) { + bio = bl_submit_bio(bio); + goto retry; + } + return bio; +} + +static void bl_mark_devices_unavailable(struct nfs_pgio_header *header, bool rw) +{ + struct pnfs_block_layout *bl = BLK_LSEG2EXT(header->lseg); + size_t bytes_left = header->args.count; + sector_t isect, extent_length = 0; + struct pnfs_block_extent be; + + isect = header->args.offset >> SECTOR_SHIFT; + bytes_left += header->args.offset - (isect << SECTOR_SHIFT); + + while (bytes_left > 0) { + if (!ext_tree_lookup(bl, isect, &be, rw)) + return; + extent_length = be.be_length - (isect - be.be_f_offset); + nfs4_mark_deviceid_unavailable(be.be_device); + isect += extent_length; + if (bytes_left > extent_length << SECTOR_SHIFT) + bytes_left -= extent_length << SECTOR_SHIFT; + else + bytes_left = 0; + } +} + +static void bl_end_io_read(struct bio *bio) +{ + struct parallel_io *par = bio->bi_private; + + if (bio->bi_status) { + struct nfs_pgio_header *header = par->data; + + if (!header->pnfs_error) + header->pnfs_error = -EIO; + pnfs_set_lo_fail(header->lseg); + bl_mark_devices_unavailable(header, false); + } + + bio_put(bio); + put_parallel(par); +} + +static void bl_read_cleanup(struct work_struct *work) +{ + struct rpc_task *task; + struct nfs_pgio_header *hdr; + dprintk("%s enter\n", __func__); + task = container_of(work, struct rpc_task, u.tk_work); + hdr = container_of(task, struct nfs_pgio_header, task); + pnfs_ld_read_done(hdr); +} + +static void +bl_end_par_io_read(void *data) +{ + struct nfs_pgio_header *hdr = data; + + hdr->task.tk_status = hdr->pnfs_error; + INIT_WORK(&hdr->task.u.tk_work, bl_read_cleanup); + schedule_work(&hdr->task.u.tk_work); +} + +static enum pnfs_try_status +bl_read_pagelist(struct nfs_pgio_header *header) +{ + struct pnfs_block_layout *bl = BLK_LSEG2EXT(header->lseg); + struct pnfs_block_dev_map map = { .start = NFS4_MAX_UINT64 }; + struct bio *bio = NULL; + struct pnfs_block_extent be; + sector_t isect, extent_length = 0; + struct parallel_io *par; + loff_t f_offset = header->args.offset; + size_t bytes_left = header->args.count; + unsigned int pg_offset = header->args.pgbase, pg_len; + struct page **pages = header->args.pages; + int pg_index = header->args.pgbase >> PAGE_SHIFT; + const bool is_dio = (header->dreq != NULL); + struct blk_plug plug; + int i; + + dprintk("%s enter nr_pages %u offset %lld count %u\n", __func__, + header->page_array.npages, f_offset, + (unsigned int)header->args.count); + + par = alloc_parallel(header); + if (!par) + return PNFS_NOT_ATTEMPTED; + par->pnfs_callback = bl_end_par_io_read; + + blk_start_plug(&plug); + + isect = (sector_t) (f_offset >> SECTOR_SHIFT); + /* Code assumes extents are page-aligned */ + for (i = pg_index; i < header->page_array.npages; i++) { + if (extent_length <= 0) { + /* We've used up the previous extent */ + bio = bl_submit_bio(bio); + + /* Get the next one */ + if (!ext_tree_lookup(bl, isect, &be, false)) { + header->pnfs_error = -EIO; + goto out; + } + extent_length = be.be_length - (isect - be.be_f_offset); + } + + if (is_dio) { + if (pg_offset + bytes_left > PAGE_SIZE) + pg_len = PAGE_SIZE - pg_offset; + else + pg_len = bytes_left; + } else { + BUG_ON(pg_offset != 0); + pg_len = PAGE_SIZE; + } + + if (is_hole(&be)) { + bio = bl_submit_bio(bio); + /* Fill hole w/ zeroes w/o accessing device */ + dprintk("%s Zeroing page for hole\n", __func__); + zero_user_segment(pages[i], pg_offset, pg_len); + + /* invalidate map */ + map.start = NFS4_MAX_UINT64; + } else { + bio = do_add_page_to_bio(bio, + header->page_array.npages - i, + REQ_OP_READ, + isect, pages[i], &map, &be, + bl_end_io_read, par, + pg_offset, &pg_len); + if (IS_ERR(bio)) { + header->pnfs_error = PTR_ERR(bio); + bio = NULL; + goto out; + } + } + isect += (pg_len >> SECTOR_SHIFT); + extent_length -= (pg_len >> SECTOR_SHIFT); + f_offset += pg_len; + bytes_left -= pg_len; + pg_offset = 0; + } + if ((isect << SECTOR_SHIFT) >= header->inode->i_size) { + header->res.eof = 1; + header->res.count = header->inode->i_size - header->args.offset; + } else { + header->res.count = (isect << SECTOR_SHIFT) - header->args.offset; + } +out: + bl_submit_bio(bio); + blk_finish_plug(&plug); + put_parallel(par); + return PNFS_ATTEMPTED; +} + +static void bl_end_io_write(struct bio *bio) +{ + struct parallel_io *par = bio->bi_private; + struct nfs_pgio_header *header = par->data; + + if (bio->bi_status) { + if (!header->pnfs_error) + header->pnfs_error = -EIO; + pnfs_set_lo_fail(header->lseg); + bl_mark_devices_unavailable(header, true); + } + bio_put(bio); + put_parallel(par); +} + +/* Function scheduled for call during bl_end_par_io_write, + * it marks sectors as written and extends the commitlist. + */ +static void bl_write_cleanup(struct work_struct *work) +{ + struct rpc_task *task = container_of(work, struct rpc_task, u.tk_work); + struct nfs_pgio_header *hdr = + container_of(task, struct nfs_pgio_header, task); + + dprintk("%s enter\n", __func__); + + if (likely(!hdr->pnfs_error)) { + struct pnfs_block_layout *bl = BLK_LSEG2EXT(hdr->lseg); + u64 start = hdr->args.offset & (loff_t)PAGE_MASK; + u64 end = (hdr->args.offset + hdr->args.count + + PAGE_SIZE - 1) & (loff_t)PAGE_MASK; + u64 lwb = hdr->args.offset + hdr->args.count; + + ext_tree_mark_written(bl, start >> SECTOR_SHIFT, + (end - start) >> SECTOR_SHIFT, lwb); + } + + pnfs_ld_write_done(hdr); +} + +/* Called when last of bios associated with a bl_write_pagelist call finishes */ +static void bl_end_par_io_write(void *data) +{ + struct nfs_pgio_header *hdr = data; + + hdr->task.tk_status = hdr->pnfs_error; + hdr->verf.committed = NFS_FILE_SYNC; + INIT_WORK(&hdr->task.u.tk_work, bl_write_cleanup); + schedule_work(&hdr->task.u.tk_work); +} + +static enum pnfs_try_status +bl_write_pagelist(struct nfs_pgio_header *header, int sync) +{ + struct pnfs_block_layout *bl = BLK_LSEG2EXT(header->lseg); + struct pnfs_block_dev_map map = { .start = NFS4_MAX_UINT64 }; + struct bio *bio = NULL; + struct pnfs_block_extent be; + sector_t isect, extent_length = 0; + struct parallel_io *par = NULL; + loff_t offset = header->args.offset; + struct page **pages = header->args.pages; + int pg_index = header->args.pgbase >> PAGE_SHIFT; + unsigned int pg_len; + struct blk_plug plug; + int i; + + dprintk("%s enter, %u@%lld\n", __func__, header->args.count, offset); + + /* At this point, header->page_aray is a (sequential) list of nfs_pages. + * We want to write each, and if there is an error set pnfs_error + * to have it redone using nfs. + */ + par = alloc_parallel(header); + if (!par) + return PNFS_NOT_ATTEMPTED; + par->pnfs_callback = bl_end_par_io_write; + + blk_start_plug(&plug); + + /* we always write out the whole page */ + offset = offset & (loff_t)PAGE_MASK; + isect = offset >> SECTOR_SHIFT; + + for (i = pg_index; i < header->page_array.npages; i++) { + if (extent_length <= 0) { + /* We've used up the previous extent */ + bio = bl_submit_bio(bio); + /* Get the next one */ + if (!ext_tree_lookup(bl, isect, &be, true)) { + header->pnfs_error = -EINVAL; + goto out; + } + + extent_length = be.be_length - (isect - be.be_f_offset); + } + + pg_len = PAGE_SIZE; + bio = do_add_page_to_bio(bio, header->page_array.npages - i, + REQ_OP_WRITE, isect, pages[i], &map, + &be, bl_end_io_write, par, 0, &pg_len); + if (IS_ERR(bio)) { + header->pnfs_error = PTR_ERR(bio); + bio = NULL; + goto out; + } + + offset += pg_len; + isect += (pg_len >> SECTOR_SHIFT); + extent_length -= (pg_len >> SECTOR_SHIFT); + } + + header->res.count = header->args.count; +out: + bl_submit_bio(bio); + blk_finish_plug(&plug); + put_parallel(par); + return PNFS_ATTEMPTED; +} + +static void bl_free_layout_hdr(struct pnfs_layout_hdr *lo) +{ + struct pnfs_block_layout *bl = BLK_LO2EXT(lo); + int err; + + dprintk("%s enter\n", __func__); + + err = ext_tree_remove(bl, true, 0, LLONG_MAX); + WARN_ON(err); + + kfree_rcu(bl, bl_layout.plh_rcu); +} + +static struct pnfs_layout_hdr *__bl_alloc_layout_hdr(struct inode *inode, + gfp_t gfp_flags, bool is_scsi_layout) +{ + struct pnfs_block_layout *bl; + + dprintk("%s enter\n", __func__); + bl = kzalloc_obj(*bl, gfp_flags); + if (!bl) + return NULL; + + bl->bl_ext_rw = RB_ROOT; + bl->bl_ext_ro = RB_ROOT; + spin_lock_init(&bl->bl_ext_lock); + + bl->bl_scsi_layout = is_scsi_layout; + return &bl->bl_layout; +} + +static struct pnfs_layout_hdr *bl_alloc_layout_hdr(struct inode *inode, + gfp_t gfp_flags) +{ + return __bl_alloc_layout_hdr(inode, gfp_flags, false); +} + +static struct pnfs_layout_hdr *sl_alloc_layout_hdr(struct inode *inode, + gfp_t gfp_flags) +{ + return __bl_alloc_layout_hdr(inode, gfp_flags, true); +} + +static void bl_free_lseg(struct pnfs_layout_segment *lseg) +{ + dprintk("%s enter\n", __func__); + kfree(lseg); +} + +/* Tracks info needed to ensure extents in layout obey constraints of spec */ +struct layout_verification { + u32 mode; /* R or RW */ + u64 start; /* Expected start of next non-COW extent */ + u64 inval; /* Start of INVAL coverage */ + u64 cowread; /* End of COW read coverage */ +}; + +/* Verify the extent meets the layout requirements of the pnfs-block draft, + * section 2.3.1. + */ +static int verify_extent(struct pnfs_block_extent *be, + struct layout_verification *lv) +{ + if (lv->mode == IOMODE_READ) { + if (be->be_state == PNFS_BLOCK_READWRITE_DATA || + be->be_state == PNFS_BLOCK_INVALID_DATA) + return -EIO; + if (be->be_f_offset != lv->start) + return -EIO; + lv->start += be->be_length; + return 0; + } + /* lv->mode == IOMODE_RW */ + if (be->be_state == PNFS_BLOCK_READWRITE_DATA) { + if (be->be_f_offset != lv->start) + return -EIO; + if (lv->cowread > lv->start) + return -EIO; + lv->start += be->be_length; + lv->inval = lv->start; + return 0; + } else if (be->be_state == PNFS_BLOCK_INVALID_DATA) { + if (be->be_f_offset != lv->start) + return -EIO; + lv->start += be->be_length; + return 0; + } else if (be->be_state == PNFS_BLOCK_READ_DATA) { + if (be->be_f_offset > lv->start) + return -EIO; + if (be->be_f_offset < lv->inval) + return -EIO; + if (be->be_f_offset < lv->cowread) + return -EIO; + /* It looks like you might want to min this with lv->start, + * but you really don't. + */ + lv->inval = lv->inval + be->be_length; + lv->cowread = be->be_f_offset + be->be_length; + return 0; + } else + return -EIO; +} + +static int decode_sector_number(__be32 **rp, sector_t *sp) +{ + uint64_t s; + + *rp = xdr_decode_hyper(*rp, &s); + if (s & 0x1ff) { + printk(KERN_WARNING "NFS: %s: sector not aligned\n", __func__); + return -1; + } + *sp = s >> SECTOR_SHIFT; + return 0; +} + +static struct nfs4_deviceid_node * +bl_find_get_deviceid(struct nfs_server *server, + const struct nfs4_deviceid *id, const struct cred *cred, + gfp_t gfp_mask) +{ + struct nfs4_deviceid_node *node; + int err = -ENODEV; + +retry: + node = nfs4_find_get_deviceid(server, id, cred, gfp_mask); + if (!node) + return ERR_PTR(-ENODEV); + + /* + * Devices that are marked unavailable are left in the cache with a + * timeout to avoid sending GETDEVINFO after every LAYOUTGET, or + * constantly attempting to register the device. Once marked as + * unavailable they must be deleted and never reused. + */ + if (test_bit(NFS_DEVICEID_UNAVAILABLE, &node->flags)) { + unsigned long end = jiffies; + unsigned long start = end - PNFS_DEVICE_RETRY_TIMEOUT; + + if (!time_in_range(node->timestamp_unavailable, start, end)) { + /* Uncork subsequent GETDEVINFO operations for this device */ + nfs4_delete_deviceid(node->ld, node->nfs_client, id); + goto retry; + } + goto out_put; + } + + if (!bl_register_dev(container_of(node, struct pnfs_block_dev, node))) { + /* + * If we cannot register, treat this device as transient: + * Make a negative cache entry for the device + */ + nfs4_mark_deviceid_unavailable(node); + goto out_put; + } + + return node; + +out_put: + nfs4_put_deviceid_node(node); + return ERR_PTR(err); +} + +static int +bl_alloc_extent(struct xdr_stream *xdr, struct pnfs_layout_hdr *lo, + struct layout_verification *lv, struct list_head *extents, + gfp_t gfp_mask) +{ + struct pnfs_block_extent *be; + struct nfs4_deviceid id; + int error; + __be32 *p; + + p = xdr_inline_decode(xdr, 28 + NFS4_DEVICEID4_SIZE); + if (!p) + return -EIO; + + be = kzalloc_obj(*be, GFP_NOFS); + if (!be) + return -ENOMEM; + + memcpy(&id, p, NFS4_DEVICEID4_SIZE); + p += XDR_QUADLEN(NFS4_DEVICEID4_SIZE); + + be->be_device = bl_find_get_deviceid(NFS_SERVER(lo->plh_inode), &id, + lo->plh_lc_cred, gfp_mask); + if (IS_ERR(be->be_device)) { + error = PTR_ERR(be->be_device); + goto out_free_be; + } + + /* + * The next three values are read in as bytes, but stored in the + * extent structure in 512-byte granularity. + */ + error = -EIO; + if (decode_sector_number(&p, &be->be_f_offset) < 0) + goto out_put_deviceid; + if (decode_sector_number(&p, &be->be_length) < 0) + goto out_put_deviceid; + if (decode_sector_number(&p, &be->be_v_offset) < 0) + goto out_put_deviceid; + be->be_state = be32_to_cpup(p++); + + error = verify_extent(be, lv); + if (error) { + dprintk("%s: extent verification failed\n", __func__); + goto out_put_deviceid; + } + + list_add_tail(&be->be_list, extents); + return 0; + +out_put_deviceid: + nfs4_put_deviceid_node(be->be_device); +out_free_be: + kfree(be); + return error; +} + +static struct pnfs_layout_segment * +bl_alloc_lseg(struct pnfs_layout_hdr *lo, struct nfs4_layoutget_res *lgr, + gfp_t gfp_mask) +{ + struct layout_verification lv = { + .mode = lgr->range.iomode, + .start = lgr->range.offset >> SECTOR_SHIFT, + .inval = lgr->range.offset >> SECTOR_SHIFT, + .cowread = lgr->range.offset >> SECTOR_SHIFT, + }; + struct pnfs_block_layout *bl = BLK_LO2EXT(lo); + struct pnfs_layout_segment *lseg; + struct xdr_buf buf; + struct xdr_stream xdr; + struct folio *scratch; + int status, i; + uint32_t count; + __be32 *p; + LIST_HEAD(extents); + + dprintk("---> %s\n", __func__); + + lseg = kzalloc_obj(*lseg, gfp_mask); + if (!lseg) + return ERR_PTR(-ENOMEM); + + status = -ENOMEM; + scratch = folio_alloc(gfp_mask, 0); + if (!scratch) + goto out; + + xdr_init_decode_pages(&xdr, &buf, + lgr->layoutp->pages, lgr->layoutp->len); + xdr_set_scratch_folio(&xdr, scratch); + + status = -EIO; + p = xdr_inline_decode(&xdr, 4); + if (unlikely(!p)) + goto out_free_scratch; + + count = be32_to_cpup(p++); + dprintk("%s: number of extents %d\n", __func__, count); + + /* + * Decode individual extents, putting them in temporary staging area + * until whole layout is decoded to make error recovery easier. + */ + for (i = 0; i < count; i++) { + status = bl_alloc_extent(&xdr, lo, &lv, &extents, gfp_mask); + if (status) + goto process_extents; + } + + if (lgr->range.offset + lgr->range.length != + lv.start << SECTOR_SHIFT) { + dprintk("%s Final length mismatch\n", __func__); + status = -EIO; + goto process_extents; + } + + if (lv.start < lv.cowread) { + dprintk("%s Final uncovered COW extent\n", __func__); + status = -EIO; + } + +process_extents: + while (!list_empty(&extents)) { + struct pnfs_block_extent *be = + list_first_entry(&extents, struct pnfs_block_extent, + be_list); + list_del(&be->be_list); + + if (!status) + status = ext_tree_insert(bl, be); + + if (status) { + nfs4_put_deviceid_node(be->be_device); + kfree(be); + } + } + +out_free_scratch: + folio_put(scratch); +out: + dprintk("%s returns %d\n", __func__, status); + switch (status) { + case -ENODEV: + /* Our extent block devices are unavailable */ + set_bit(NFS_LSEG_UNAVAILABLE, &lseg->pls_flags); + fallthrough; + case 0: + return lseg; + default: + kfree(lseg); + return ERR_PTR(status); + } +} + +static void +bl_return_range(struct pnfs_layout_hdr *lo, + struct pnfs_layout_range *range) +{ + struct pnfs_block_layout *bl = BLK_LO2EXT(lo); + sector_t offset = range->offset >> SECTOR_SHIFT, end; + + if (range->offset % 8) { + dprintk("%s: offset %lld not block size aligned\n", + __func__, range->offset); + return; + } + + if (range->length != NFS4_MAX_UINT64) { + if (range->length % 8) { + dprintk("%s: length %lld not block size aligned\n", + __func__, range->length); + return; + } + + end = offset + (range->length >> SECTOR_SHIFT); + } else { + end = round_down(NFS4_MAX_UINT64, PAGE_SIZE); + } + + ext_tree_remove(bl, range->iomode & IOMODE_RW, offset, end); +} + +static int +bl_prepare_layoutcommit(struct nfs4_layoutcommit_args *arg) +{ + return ext_tree_prepare_commit(arg); +} + +static void +bl_cleanup_layoutcommit(struct nfs4_layoutcommit_data *lcdata) +{ + ext_tree_mark_committed(&lcdata->args, lcdata->res.status); +} + +static int +bl_set_layoutdriver(struct nfs_server *server, const struct nfs_fh *fh) +{ + dprintk("%s enter\n", __func__); + + if (server->pnfs_blksize == 0) { + dprintk("%s Server did not return blksize\n", __func__); + return -EINVAL; + } + if (server->pnfs_blksize > PAGE_SIZE) { + printk(KERN_ERR "%s: pNFS blksize %d not supported.\n", + __func__, server->pnfs_blksize); + return -EINVAL; + } + + return 0; +} + +static bool +is_aligned_req(struct nfs_pageio_descriptor *pgio, + struct nfs_page *req, unsigned int alignment, bool is_write) +{ + /* + * Always accept buffered writes, higher layers take care of the + * right alignment. + */ + if (pgio->pg_dreq == NULL) + return true; + + if (!IS_ALIGNED(req->wb_offset, alignment)) + return false; + + if (IS_ALIGNED(req->wb_bytes, alignment)) + return true; + + if (is_write && + (req_offset(req) + req->wb_bytes == i_size_read(pgio->pg_inode))) { + /* + * If the write goes up to the inode size, just write + * the full page. Data past the inode size is + * guaranteed to be zeroed by the higher level client + * code, and this behaviour is mandated by RFC 5663 + * section 2.3.2. + */ + return true; + } + + return false; +} + +static void +bl_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req) +{ + if (!is_aligned_req(pgio, req, SECTOR_SIZE, false)) { + nfs_pageio_reset_read_mds(pgio); + return; + } + + pnfs_generic_pg_init_read(pgio, req); + + if (pgio->pg_lseg && + test_bit(NFS_LSEG_UNAVAILABLE, &pgio->pg_lseg->pls_flags)) { + pnfs_error_mark_layout_for_return(pgio->pg_inode, pgio->pg_lseg); + pnfs_set_lo_fail(pgio->pg_lseg); + nfs_pageio_reset_read_mds(pgio); + } +} + +/* + * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number + * of bytes (maximum @req->wb_bytes) that can be coalesced. + */ +static size_t +bl_pg_test_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *prev, + struct nfs_page *req) +{ + if (!is_aligned_req(pgio, req, SECTOR_SIZE, false)) + return 0; + return pnfs_generic_pg_test(pgio, prev, req); +} + +/* + * Return the number of contiguous bytes for a given inode + * starting at page frame idx. + */ +static u64 pnfs_num_cont_bytes(struct inode *inode, pgoff_t idx) +{ + struct address_space *mapping = inode->i_mapping; + pgoff_t end; + + /* Optimize common case that writes from 0 to end of file */ + end = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); + if (end != inode->i_mapping->nrpages) { + rcu_read_lock(); + end = page_cache_next_miss(mapping, idx + 1, ULONG_MAX); + rcu_read_unlock(); + } + + if (!end) + return i_size_read(inode) - (idx << PAGE_SHIFT); + else + return (end - idx) << PAGE_SHIFT; +} + +static void +bl_pg_init_write(struct nfs_pageio_descriptor *pgio, struct nfs_page *req) +{ + u64 wb_size; + + if (!is_aligned_req(pgio, req, PAGE_SIZE, true)) { + nfs_pageio_reset_write_mds(pgio); + return; + } + + if (pgio->pg_dreq == NULL) + wb_size = pnfs_num_cont_bytes(pgio->pg_inode, req->wb_index); + else + wb_size = nfs_dreq_bytes_left(pgio->pg_dreq, req_offset(req)); + + pnfs_generic_pg_init_write(pgio, req, wb_size); + + if (pgio->pg_lseg && + test_bit(NFS_LSEG_UNAVAILABLE, &pgio->pg_lseg->pls_flags)) { + + pnfs_error_mark_layout_for_return(pgio->pg_inode, pgio->pg_lseg); + pnfs_set_lo_fail(pgio->pg_lseg); + nfs_pageio_reset_write_mds(pgio); + } +} + +/* + * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number + * of bytes (maximum @req->wb_bytes) that can be coalesced. + */ +static size_t +bl_pg_test_write(struct nfs_pageio_descriptor *pgio, struct nfs_page *prev, + struct nfs_page *req) +{ + if (!is_aligned_req(pgio, req, PAGE_SIZE, true)) + return 0; + return pnfs_generic_pg_test(pgio, prev, req); +} + +static const struct nfs_pageio_ops bl_pg_read_ops = { + .pg_init = bl_pg_init_read, + .pg_test = bl_pg_test_read, + .pg_doio = pnfs_generic_pg_readpages, + .pg_cleanup = pnfs_generic_pg_cleanup, +}; + +static const struct nfs_pageio_ops bl_pg_write_ops = { + .pg_init = bl_pg_init_write, + .pg_test = bl_pg_test_write, + .pg_doio = pnfs_generic_pg_writepages, + .pg_cleanup = pnfs_generic_pg_cleanup, +}; + +static struct pnfs_layoutdriver_type blocklayout_type = { + .id = LAYOUT_BLOCK_VOLUME, + .name = "LAYOUT_BLOCK_VOLUME", + .owner = THIS_MODULE, + .flags = PNFS_LAYOUTRET_ON_SETATTR | + PNFS_LAYOUTRET_ON_ERROR | + PNFS_READ_WHOLE_PAGE, + .read_pagelist = bl_read_pagelist, + .write_pagelist = bl_write_pagelist, + .alloc_layout_hdr = bl_alloc_layout_hdr, + .free_layout_hdr = bl_free_layout_hdr, + .alloc_lseg = bl_alloc_lseg, + .free_lseg = bl_free_lseg, + .return_range = bl_return_range, + .prepare_layoutcommit = bl_prepare_layoutcommit, + .cleanup_layoutcommit = bl_cleanup_layoutcommit, + .set_layoutdriver = bl_set_layoutdriver, + .alloc_deviceid_node = bl_alloc_deviceid_node, + .free_deviceid_node = bl_free_deviceid_node, + .pg_read_ops = &bl_pg_read_ops, + .pg_write_ops = &bl_pg_write_ops, + .sync = pnfs_generic_sync, +}; + +static struct pnfs_layoutdriver_type scsilayout_type = { + .id = LAYOUT_SCSI, + .name = "LAYOUT_SCSI", + .owner = THIS_MODULE, + .flags = PNFS_LAYOUTRET_ON_SETATTR | + PNFS_LAYOUTRET_ON_ERROR | + PNFS_READ_WHOLE_PAGE, + .read_pagelist = bl_read_pagelist, + .write_pagelist = bl_write_pagelist, + .alloc_layout_hdr = sl_alloc_layout_hdr, + .free_layout_hdr = bl_free_layout_hdr, + .alloc_lseg = bl_alloc_lseg, + .free_lseg = bl_free_lseg, + .return_range = bl_return_range, + .prepare_layoutcommit = bl_prepare_layoutcommit, + .cleanup_layoutcommit = bl_cleanup_layoutcommit, + .set_layoutdriver = bl_set_layoutdriver, + .alloc_deviceid_node = bl_alloc_deviceid_node, + .free_deviceid_node = bl_free_deviceid_node, + .pg_read_ops = &bl_pg_read_ops, + .pg_write_ops = &bl_pg_write_ops, + .sync = pnfs_generic_sync, +}; + + +static int __init nfs4blocklayout_init(void) +{ + int ret; + + dprintk("%s: NFSv4 Block Layout Driver Registering...\n", __func__); + + ret = bl_init_pipefs(); + if (ret) + goto out; + + ret = pnfs_register_layoutdriver(&blocklayout_type); + if (ret) + goto out_cleanup_pipe; + + ret = pnfs_register_layoutdriver(&scsilayout_type); + if (ret) + goto out_unregister_block; + return 0; + +out_unregister_block: + pnfs_unregister_layoutdriver(&blocklayout_type); +out_cleanup_pipe: + bl_cleanup_pipefs(); +out: + return ret; +} + +static void __exit nfs4blocklayout_exit(void) +{ + dprintk("%s: NFSv4 Block Layout Driver Unregistering...\n", + __func__); + + pnfs_unregister_layoutdriver(&scsilayout_type); + pnfs_unregister_layoutdriver(&blocklayout_type); + bl_cleanup_pipefs(); +} + +MODULE_ALIAS("nfs-layouttype4-3"); +MODULE_ALIAS("nfs-layouttype4-5"); + +module_init(nfs4blocklayout_init); +module_exit(nfs4blocklayout_exit); diff --git a/fs/nfs/blocklayout/blocklayout.h b/fs/nfs/blocklayout/blocklayout.h new file mode 100644 index 000000000..6da40ca19 --- /dev/null +++ b/fs/nfs/blocklayout/blocklayout.h @@ -0,0 +1,204 @@ +/* + * linux/fs/nfs/blocklayout/blocklayout.h + * + * Module for the NFSv4.1 pNFS block layout driver. + * + * Copyright (c) 2006 The Regents of the University of Michigan. + * All rights reserved. + * + * Andy Adamson + * Fred Isaman + * + * permission is granted to use, copy, create derivative works and + * redistribute this software and such derivative works for any purpose, + * so long as the name of the university of michigan is not used in + * any advertising or publicity pertaining to the use or distribution + * of this software without specific, written prior authorization. if + * the above copyright notice or any other identification of the + * university of michigan is included in any copy of any portion of + * this software, then the disclaimer below must also be included. + * + * this software is provided as is, without representation from the + * university of michigan as to its fitness for any purpose, and without + * warranty by the university of michigan of any kind, either express + * or implied, including without limitation the implied warranties of + * merchantability and fitness for a particular purpose. the regents + * of the university of michigan shall not be liable for any damages, + * including special, indirect, incidental, or consequential damages, + * with respect to any claim arising out or in connection with the use + * of the software, even if it has been or is hereafter advised of the + * possibility of such damages. + */ +#ifndef FS_NFS_NFS4BLOCKLAYOUT_H +#define FS_NFS_NFS4BLOCKLAYOUT_H + +#include +#include +#include + +#include "../nfs4_fs.h" +#include "../pnfs.h" +#include "../netns.h" + +#define PAGE_CACHE_SECTORS (PAGE_SIZE >> SECTOR_SHIFT) +#define PAGE_CACHE_SECTOR_SHIFT (PAGE_SHIFT - SECTOR_SHIFT) +#define SECTOR_SIZE (1 << SECTOR_SHIFT) + +struct pnfs_block_dev; + +#define PNFS_BLOCK_MAX_UUIDS 4 +#define PNFS_BLOCK_MAX_DEVICES 64 + +/* + * Random upper cap for the uuid length to avoid unbounded allocation. + * Not actually limited by the protocol. + */ +#define PNFS_BLOCK_UUID_LEN 128 + +struct pnfs_block_volume { + enum pnfs_block_volume_type type; + union { + struct { + int len; + int nr_sigs; + struct { + u64 offset; + u32 sig_len; + u8 sig[PNFS_BLOCK_UUID_LEN]; + } sigs[PNFS_BLOCK_MAX_UUIDS]; + } simple; + struct { + u64 start; + u64 len; + u32 volume; + } slice; + struct { + u32 volumes_count; + u32 volumes[PNFS_BLOCK_MAX_DEVICES]; + } concat; + struct { + u64 chunk_size; + u32 volumes_count; + u32 volumes[PNFS_BLOCK_MAX_DEVICES]; + } stripe; + struct { + enum scsi_code_set code_set; + enum scsi_designator_type designator_type; + int designator_len; + u8 designator[256]; + u64 pr_key; + } scsi; + }; +}; + +struct pnfs_block_dev_map { + u64 start; + u64 len; + u64 disk_offset; + struct block_device *bdev; +}; + +struct pnfs_block_dev { + struct nfs4_deviceid_node node; + + u64 start; + u64 len; + + enum pnfs_block_volume_type type; + u32 nr_children; + struct pnfs_block_dev *children; + u64 chunk_size; + + struct file *bdev_file; + u64 disk_offset; + unsigned long flags; + + u64 pr_key; + + bool (*map)(struct pnfs_block_dev *dev, u64 offset, + struct pnfs_block_dev_map *map); +}; + +/* pnfs_block_dev flag bits */ +enum { + PNFS_BDEV_REGISTERED = 0, +}; + +/* sector_t fields are all in 512-byte sectors */ +struct pnfs_block_extent { + union { + struct rb_node be_node; + struct list_head be_list; + }; + struct nfs4_deviceid_node *be_device; + sector_t be_f_offset; /* the starting offset in the file */ + sector_t be_length; /* the size of the extent */ + sector_t be_v_offset; /* the starting offset in the volume */ + enum pnfs_block_extent_state be_state; /* the state of this extent */ +#define EXTENT_WRITTEN 1 +#define EXTENT_COMMITTING 2 + unsigned int be_tag; +}; + +struct pnfs_block_layout { + struct pnfs_layout_hdr bl_layout; + struct rb_root bl_ext_rw; + struct rb_root bl_ext_ro; + spinlock_t bl_ext_lock; /* Protects list manipulation */ + bool bl_scsi_layout; + u64 bl_lwb; +}; + +static inline struct pnfs_block_layout * +BLK_LO2EXT(struct pnfs_layout_hdr *lo) +{ + return container_of(lo, struct pnfs_block_layout, bl_layout); +} + +static inline struct pnfs_block_layout * +BLK_LSEG2EXT(struct pnfs_layout_segment *lseg) +{ + return BLK_LO2EXT(lseg->pls_layout); +} + +struct bl_pipe_msg { + struct rpc_pipe_msg msg; + wait_queue_head_t *bl_wq; +}; + +struct bl_msg_hdr { + u8 type; + u16 totallen; /* length of entire message, including hdr itself */ +}; + +#define BL_DEVICE_UMOUNT 0x0 /* Umount--delete devices */ +#define BL_DEVICE_MOUNT 0x1 /* Mount--create devices*/ +#define BL_DEVICE_REQUEST_INIT 0x0 /* Start request */ +#define BL_DEVICE_REQUEST_PROC 0x1 /* User level process succeeds */ +#define BL_DEVICE_REQUEST_ERR 0x2 /* User level process fails */ + +/* dev.c */ +bool bl_register_dev(struct pnfs_block_dev *d); +struct nfs4_deviceid_node *bl_alloc_deviceid_node(struct nfs_server *server, + struct pnfs_device *pdev, gfp_t gfp_mask); +void bl_free_deviceid_node(struct nfs4_deviceid_node *d); + +/* extent_tree.c */ +int ext_tree_insert(struct pnfs_block_layout *bl, + struct pnfs_block_extent *new); +int ext_tree_remove(struct pnfs_block_layout *bl, bool rw, sector_t start, + sector_t end); +int ext_tree_mark_written(struct pnfs_block_layout *bl, sector_t start, + sector_t len, u64 lwb); +bool ext_tree_lookup(struct pnfs_block_layout *bl, sector_t isect, + struct pnfs_block_extent *ret, bool rw); +int ext_tree_prepare_commit(struct nfs4_layoutcommit_args *arg); +void ext_tree_mark_committed(struct nfs4_layoutcommit_args *arg, int status); + +/* rpc_pipefs.c */ +dev_t bl_resolve_deviceid(struct nfs_server *server, + struct pnfs_block_volume *b, gfp_t gfp_mask); +int __init bl_init_pipefs(void); +void bl_cleanup_pipefs(void); + +#endif /* FS_NFS_NFS4BLOCKLAYOUT_H */ diff --git a/fs/nfs/blocklayout/dev.c b/fs/nfs/blocklayout/dev.c new file mode 100644 index 000000000..368d20daf --- /dev/null +++ b/fs/nfs/blocklayout/dev.c @@ -0,0 +1,599 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2014-2016 Christoph Hellwig. + */ +#include +#include +#include +#include +#include +#include +#include + +#include "blocklayout.h" +#include "../nfs4trace.h" + +#define NFSDBG_FACILITY NFSDBG_PNFS_LD + +static void bl_unregister_scsi(struct pnfs_block_dev *dev) +{ + struct block_device *bdev = file_bdev(dev->bdev_file); + const struct pr_ops *ops = bdev->bd_disk->fops->pr_ops; + int status; + + status = ops->pr_register(bdev, dev->pr_key, 0, false); + if (status) + trace_bl_pr_key_unreg_err(bdev, dev->pr_key, status); + else + trace_bl_pr_key_unreg(bdev, dev->pr_key); +} + +static bool bl_register_scsi(struct pnfs_block_dev *dev) +{ + struct block_device *bdev = file_bdev(dev->bdev_file); + const struct pr_ops *ops = bdev->bd_disk->fops->pr_ops; + int status; + + if (test_and_set_bit(PNFS_BDEV_REGISTERED, &dev->flags)) + return true; + + status = ops->pr_register(bdev, 0, dev->pr_key, true); + if (status) { + trace_bl_pr_key_reg_err(bdev, dev->pr_key, status); + return false; + } + trace_bl_pr_key_reg(bdev, dev->pr_key); + return true; +} + +static void bl_unregister_dev(struct pnfs_block_dev *dev) +{ + u32 i; + + if (dev->nr_children) { + for (i = 0; i < dev->nr_children; i++) + bl_unregister_dev(&dev->children[i]); + return; + } + + if (dev->type == PNFS_BLOCK_VOLUME_SCSI && + test_and_clear_bit(PNFS_BDEV_REGISTERED, &dev->flags)) + bl_unregister_scsi(dev); +} + +bool bl_register_dev(struct pnfs_block_dev *dev) +{ + u32 i; + + if (dev->nr_children) { + for (i = 0; i < dev->nr_children; i++) { + if (!bl_register_dev(&dev->children[i])) { + while (i > 0) + bl_unregister_dev(&dev->children[--i]); + return false; + } + } + return true; + } + + if (dev->type == PNFS_BLOCK_VOLUME_SCSI) + return bl_register_scsi(dev); + return true; +} + +static void +bl_free_device(struct pnfs_block_dev *dev) +{ + bl_unregister_dev(dev); + + if (dev->nr_children) { + int i; + + for (i = 0; i < dev->nr_children; i++) + bl_free_device(&dev->children[i]); + kfree(dev->children); + } else { + if (dev->bdev_file) + fput(dev->bdev_file); + } +} + +void +bl_free_deviceid_node(struct nfs4_deviceid_node *d) +{ + struct pnfs_block_dev *dev = + container_of(d, struct pnfs_block_dev, node); + + bl_free_device(dev); + kfree_rcu(dev, node.rcu); +} + +static int +nfs4_block_decode_volume(struct xdr_stream *xdr, struct pnfs_block_volume *b) +{ + __be32 *p; + int i; + + p = xdr_inline_decode(xdr, 4); + if (!p) + return -EIO; + b->type = be32_to_cpup(p++); + + switch (b->type) { + case PNFS_BLOCK_VOLUME_SIMPLE: + p = xdr_inline_decode(xdr, 4); + if (!p) + return -EIO; + b->simple.nr_sigs = be32_to_cpup(p++); + if (!b->simple.nr_sigs || b->simple.nr_sigs > PNFS_BLOCK_MAX_UUIDS) { + dprintk("Bad signature count: %d\n", b->simple.nr_sigs); + return -EIO; + } + + b->simple.len = 4 + 4; + for (i = 0; i < b->simple.nr_sigs; i++) { + p = xdr_inline_decode(xdr, 8 + 4); + if (!p) + return -EIO; + p = xdr_decode_hyper(p, &b->simple.sigs[i].offset); + b->simple.sigs[i].sig_len = be32_to_cpup(p++); + if (b->simple.sigs[i].sig_len > PNFS_BLOCK_UUID_LEN) { + pr_info("signature too long: %d\n", + b->simple.sigs[i].sig_len); + return -EIO; + } + + p = xdr_inline_decode(xdr, b->simple.sigs[i].sig_len); + if (!p) + return -EIO; + memcpy(&b->simple.sigs[i].sig, p, + b->simple.sigs[i].sig_len); + + b->simple.len += 8 + 4 + \ + (XDR_QUADLEN(b->simple.sigs[i].sig_len) << 2); + } + break; + case PNFS_BLOCK_VOLUME_SLICE: + p = xdr_inline_decode(xdr, 8 + 8 + 4); + if (!p) + return -EIO; + p = xdr_decode_hyper(p, &b->slice.start); + p = xdr_decode_hyper(p, &b->slice.len); + b->slice.volume = be32_to_cpup(p++); + break; + case PNFS_BLOCK_VOLUME_CONCAT: + p = xdr_inline_decode(xdr, 4); + if (!p) + return -EIO; + + b->concat.volumes_count = be32_to_cpup(p++); + if (b->concat.volumes_count > PNFS_BLOCK_MAX_DEVICES) { + dprintk("Too many volumes: %d\n", b->concat.volumes_count); + return -EIO; + } + + p = xdr_inline_decode(xdr, b->concat.volumes_count * 4); + if (!p) + return -EIO; + for (i = 0; i < b->concat.volumes_count; i++) + b->concat.volumes[i] = be32_to_cpup(p++); + break; + case PNFS_BLOCK_VOLUME_STRIPE: + p = xdr_inline_decode(xdr, 8 + 4); + if (!p) + return -EIO; + + p = xdr_decode_hyper(p, &b->stripe.chunk_size); + b->stripe.volumes_count = be32_to_cpup(p++); + if (b->stripe.volumes_count > PNFS_BLOCK_MAX_DEVICES) { + dprintk("Too many volumes: %d\n", b->stripe.volumes_count); + return -EIO; + } + + p = xdr_inline_decode(xdr, b->stripe.volumes_count * 4); + if (!p) + return -EIO; + for (i = 0; i < b->stripe.volumes_count; i++) + b->stripe.volumes[i] = be32_to_cpup(p++); + break; + case PNFS_BLOCK_VOLUME_SCSI: + p = xdr_inline_decode(xdr, 4 + 4 + 4); + if (!p) + return -EIO; + b->scsi.code_set = be32_to_cpup(p++); + b->scsi.designator_type = be32_to_cpup(p++); + b->scsi.designator_len = be32_to_cpup(p++); + p = xdr_inline_decode(xdr, b->scsi.designator_len); + if (!p) + return -EIO; + if (b->scsi.designator_len > 256) + return -EIO; + memcpy(&b->scsi.designator, p, b->scsi.designator_len); + p = xdr_inline_decode(xdr, 8); + if (!p) + return -EIO; + p = xdr_decode_hyper(p, &b->scsi.pr_key); + break; + default: + dprintk("unknown volume type!\n"); + return -EIO; + } + + return 0; +} + +static bool bl_map_simple(struct pnfs_block_dev *dev, u64 offset, + struct pnfs_block_dev_map *map) +{ + map->start = dev->start; + map->len = dev->len; + map->disk_offset = dev->disk_offset; + map->bdev = file_bdev(dev->bdev_file); + return true; +} + +static bool bl_map_concat(struct pnfs_block_dev *dev, u64 offset, + struct pnfs_block_dev_map *map) +{ + int i; + + for (i = 0; i < dev->nr_children; i++) { + struct pnfs_block_dev *child = &dev->children[i]; + + if (child->start > offset || + child->start + child->len <= offset) + continue; + + child->map(child, offset - child->start, map); + return true; + } + + dprintk("%s: ran off loop!\n", __func__); + return false; +} + +static bool bl_map_stripe(struct pnfs_block_dev *dev, u64 offset, + struct pnfs_block_dev_map *map) +{ + struct pnfs_block_dev *child; + u64 chunk; + u32 chunk_idx; + u64 disk_chunk; + u64 disk_offset; + + chunk = div_u64(offset, dev->chunk_size); + disk_chunk = div_u64_rem(chunk, dev->nr_children, &chunk_idx); + + if (chunk_idx >= dev->nr_children) { + dprintk("%s: invalid chunk idx %d (%lld/%lld)\n", + __func__, chunk_idx, offset, dev->chunk_size); + /* error, should not happen */ + return false; + } + + /* truncate offset to the beginning of the stripe */ + offset = chunk * dev->chunk_size; + + /* disk offset of the stripe */ + disk_offset = disk_chunk * dev->chunk_size; + + child = &dev->children[chunk_idx]; + child->map(child, disk_offset, map); + + map->start += offset; + map->disk_offset += disk_offset; + map->len = dev->chunk_size; + return true; +} + +static int +bl_parse_deviceid(struct nfs_server *server, struct pnfs_block_dev *d, + struct pnfs_block_volume *volumes, int idx, gfp_t gfp_mask); + + +static int +bl_parse_simple(struct nfs_server *server, struct pnfs_block_dev *d, + struct pnfs_block_volume *volumes, int idx, gfp_t gfp_mask) +{ + struct pnfs_block_volume *v = &volumes[idx]; + struct file *bdev_file; + dev_t dev; + + dev = bl_resolve_deviceid(server, v, gfp_mask); + if (!dev) + return -EIO; + + bdev_file = bdev_file_open_by_dev(dev, BLK_OPEN_READ | BLK_OPEN_WRITE, + NULL, NULL); + if (IS_ERR(bdev_file)) { + printk(KERN_WARNING "pNFS: failed to open device %d:%d (%ld)\n", + MAJOR(dev), MINOR(dev), PTR_ERR(bdev_file)); + return PTR_ERR(bdev_file); + } + d->bdev_file = bdev_file; + d->len = bdev_nr_bytes(file_bdev(bdev_file)); + d->map = bl_map_simple; + + printk(KERN_INFO "pNFS: using block device %s\n", + file_bdev(bdev_file)->bd_disk->disk_name); + return 0; +} + +static bool +bl_validate_designator(struct pnfs_block_volume *v) +{ + switch (v->scsi.designator_type) { + case PS_DESIGNATOR_EUI64: + if (v->scsi.code_set != PS_CODE_SET_BINARY) + return false; + + if (v->scsi.designator_len != 8 && + v->scsi.designator_len != 10 && + v->scsi.designator_len != 16) + return false; + + return true; + case PS_DESIGNATOR_NAA: + if (v->scsi.code_set != PS_CODE_SET_BINARY) + return false; + + if (v->scsi.designator_len != 8 && + v->scsi.designator_len != 16) + return false; + + return true; + case PS_DESIGNATOR_T10: + case PS_DESIGNATOR_NAME: + pr_err("pNFS: unsupported designator " + "(code set %d, type %d, len %d.\n", + v->scsi.code_set, + v->scsi.designator_type, + v->scsi.designator_len); + return false; + default: + pr_err("pNFS: invalid designator " + "(code set %d, type %d, len %d.\n", + v->scsi.code_set, + v->scsi.designator_type, + v->scsi.designator_len); + return false; + } +} + +static struct file * +bl_open_path(struct pnfs_block_volume *v, const char *prefix) +{ + struct file *bdev_file; + const char *devname __free(kfree) = NULL; + + devname = kasprintf(GFP_KERNEL, "/dev/disk/by-id/%s%*phN", + prefix, v->scsi.designator_len, v->scsi.designator); + if (!devname) + return ERR_PTR(-ENOMEM); + + if (tsk_is_kthread(current)) { + scoped_with_init_fs() + bdev_file = bdev_file_open_by_path(devname, + BLK_OPEN_READ | BLK_OPEN_WRITE, + NULL, NULL); + } else { + bdev_file = bdev_file_open_by_path(devname, + BLK_OPEN_READ | BLK_OPEN_WRITE, NULL, NULL); + } + if (IS_ERR(bdev_file)) { + dprintk("failed to open device %s (%ld)\n", + devname, PTR_ERR(bdev_file)); + } else { + pr_info("pNFS: using block device %s\n", + file_bdev(bdev_file)->bd_disk->disk_name); + } + + return bdev_file; +} + +static int +bl_parse_scsi(struct nfs_server *server, struct pnfs_block_dev *d, + struct pnfs_block_volume *volumes, int idx, gfp_t gfp_mask) +{ + struct pnfs_block_volume *v = &volumes[idx]; + struct block_device *bdev; + const struct pr_ops *ops; + struct file *bdev_file; + int error; + + if (!bl_validate_designator(v)) + return -EINVAL; + + /* + * Try to open the RH/Fedora specific dm-mpath udev path first, as the + * wwn- links will only point to the first discovered SCSI device there. + * On other distributions like Debian, the default SCSI by-id path will + * point to the dm-multipath device if one exists. + */ + bdev_file = bl_open_path(v, "dm-uuid-mpath-0x"); + if (IS_ERR(bdev_file)) + bdev_file = bl_open_path(v, "wwn-0x"); + if (IS_ERR(bdev_file)) + bdev_file = bl_open_path(v, "nvme-eui."); + if (IS_ERR(bdev_file)) { + pr_warn("pNFS: no device found for volume %*phN\n", + v->scsi.designator_len, v->scsi.designator); + return PTR_ERR(bdev_file); + } + d->bdev_file = bdev_file; + bdev = file_bdev(bdev_file); + + d->len = bdev_nr_bytes(bdev); + d->map = bl_map_simple; + d->pr_key = v->scsi.pr_key; + + if (d->len == 0) { + error = -ENODEV; + goto out_blkdev_put; + } + + ops = bdev->bd_disk->fops->pr_ops; + if (!ops) { + pr_err("pNFS: block device %s does not support reservations.", + bdev->bd_disk->disk_name); + error = -EINVAL; + goto out_blkdev_put; + } + + return 0; + +out_blkdev_put: + fput(d->bdev_file); + return error; +} + +static int +bl_parse_slice(struct nfs_server *server, struct pnfs_block_dev *d, + struct pnfs_block_volume *volumes, int idx, gfp_t gfp_mask) +{ + struct pnfs_block_volume *v = &volumes[idx]; + int ret; + + ret = bl_parse_deviceid(server, d, volumes, v->slice.volume, gfp_mask); + if (ret) + return ret; + + d->disk_offset = v->slice.start; + d->len = v->slice.len; + return 0; +} + +static int +bl_parse_concat(struct nfs_server *server, struct pnfs_block_dev *d, + struct pnfs_block_volume *volumes, int idx, gfp_t gfp_mask) +{ + struct pnfs_block_volume *v = &volumes[idx]; + u64 len = 0; + int ret, i; + + d->children = kzalloc_objs(struct pnfs_block_dev, + v->concat.volumes_count, gfp_mask); + if (!d->children) + return -ENOMEM; + + for (i = 0; i < v->concat.volumes_count; i++) { + ret = bl_parse_deviceid(server, &d->children[i], + volumes, v->concat.volumes[i], gfp_mask); + if (ret) + return ret; + + d->nr_children++; + d->children[i].start += len; + len += d->children[i].len; + } + + d->len = len; + d->map = bl_map_concat; + return 0; +} + +static int +bl_parse_stripe(struct nfs_server *server, struct pnfs_block_dev *d, + struct pnfs_block_volume *volumes, int idx, gfp_t gfp_mask) +{ + struct pnfs_block_volume *v = &volumes[idx]; + u64 len = 0; + int ret, i; + + d->children = kzalloc_objs(struct pnfs_block_dev, + v->stripe.volumes_count, gfp_mask); + if (!d->children) + return -ENOMEM; + + for (i = 0; i < v->stripe.volumes_count; i++) { + ret = bl_parse_deviceid(server, &d->children[i], + volumes, v->stripe.volumes[i], gfp_mask); + if (ret) + return ret; + + d->nr_children++; + len += d->children[i].len; + } + + d->len = len; + d->chunk_size = v->stripe.chunk_size; + d->map = bl_map_stripe; + return 0; +} + +static int +bl_parse_deviceid(struct nfs_server *server, struct pnfs_block_dev *d, + struct pnfs_block_volume *volumes, int idx, gfp_t gfp_mask) +{ + d->type = volumes[idx].type; + + switch (d->type) { + case PNFS_BLOCK_VOLUME_SIMPLE: + return bl_parse_simple(server, d, volumes, idx, gfp_mask); + case PNFS_BLOCK_VOLUME_SLICE: + return bl_parse_slice(server, d, volumes, idx, gfp_mask); + case PNFS_BLOCK_VOLUME_CONCAT: + return bl_parse_concat(server, d, volumes, idx, gfp_mask); + case PNFS_BLOCK_VOLUME_STRIPE: + return bl_parse_stripe(server, d, volumes, idx, gfp_mask); + case PNFS_BLOCK_VOLUME_SCSI: + return bl_parse_scsi(server, d, volumes, idx, gfp_mask); + default: + dprintk("unsupported volume type: %d\n", d->type); + return -EIO; + } +} + +struct nfs4_deviceid_node * +bl_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev, + gfp_t gfp_mask) +{ + struct nfs4_deviceid_node *node = NULL; + struct pnfs_block_volume *volumes; + struct pnfs_block_dev *top; + struct xdr_stream xdr; + struct xdr_buf buf; + struct folio *scratch; + int nr_volumes, ret, i; + __be32 *p; + + scratch = folio_alloc(gfp_mask, 0); + if (!scratch) + goto out; + + xdr_init_decode_pages(&xdr, &buf, pdev->pages, pdev->pglen); + xdr_set_scratch_folio(&xdr, scratch); + + p = xdr_inline_decode(&xdr, sizeof(__be32)); + if (!p) + goto out_free_scratch; + nr_volumes = be32_to_cpup(p++); + + volumes = kzalloc_objs(struct pnfs_block_volume, nr_volumes, gfp_mask); + if (!volumes) + goto out_free_scratch; + + for (i = 0; i < nr_volumes; i++) { + ret = nfs4_block_decode_volume(&xdr, &volumes[i]); + if (ret < 0) + goto out_free_volumes; + } + + top = kzalloc_obj(*top, gfp_mask); + if (!top) + goto out_free_volumes; + + ret = bl_parse_deviceid(server, top, volumes, nr_volumes - 1, gfp_mask); + + node = &top->node; + nfs4_init_deviceid_node(node, server, &pdev->dev_id); + if (ret) + nfs4_mark_deviceid_unavailable(node); + +out_free_volumes: + kfree(volumes); +out_free_scratch: + folio_put(scratch); +out: + return node; +} diff --git a/fs/nfs/blocklayout/extent_tree.c b/fs/nfs/blocklayout/extent_tree.c new file mode 100644 index 000000000..7393118c4 --- /dev/null +++ b/fs/nfs/blocklayout/extent_tree.c @@ -0,0 +1,726 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2014-2016 Christoph Hellwig. + */ + +#include + +#include "blocklayout.h" +#include "../nfs4trace.h" + +#define NFSDBG_FACILITY NFSDBG_PNFS_LD + +static inline struct pnfs_block_extent * +ext_node(struct rb_node *node) +{ + return rb_entry(node, struct pnfs_block_extent, be_node); +} + +static struct pnfs_block_extent * +ext_tree_first(struct rb_root *root) +{ + struct rb_node *node = rb_first(root); + return node ? ext_node(node) : NULL; +} + +static struct pnfs_block_extent * +ext_tree_prev(struct pnfs_block_extent *be) +{ + struct rb_node *node = rb_prev(&be->be_node); + return node ? ext_node(node) : NULL; +} + +static struct pnfs_block_extent * +ext_tree_next(struct pnfs_block_extent *be) +{ + struct rb_node *node = rb_next(&be->be_node); + return node ? ext_node(node) : NULL; +} + +static inline sector_t +ext_f_end(struct pnfs_block_extent *be) +{ + return be->be_f_offset + be->be_length; +} + +static struct pnfs_block_extent * +__ext_tree_search(struct rb_root *root, sector_t start) +{ + struct rb_node *node = root->rb_node; + struct pnfs_block_extent *be = NULL; + + while (node) { + be = ext_node(node); + if (start < be->be_f_offset) + node = node->rb_left; + else if (start >= ext_f_end(be)) + node = node->rb_right; + else + return be; + } + + if (be) { + if (start < be->be_f_offset) + return be; + + if (start >= ext_f_end(be)) + return ext_tree_next(be); + } + + return NULL; +} + +static bool +ext_can_merge(struct pnfs_block_extent *be1, struct pnfs_block_extent *be2) +{ + if (be1->be_state != be2->be_state) + return false; + if (be1->be_device != be2->be_device) + return false; + + if (be1->be_f_offset + be1->be_length != be2->be_f_offset) + return false; + + if (be1->be_state != PNFS_BLOCK_NONE_DATA && + (be1->be_v_offset + be1->be_length != be2->be_v_offset)) + return false; + + if (be1->be_state == PNFS_BLOCK_INVALID_DATA && + be1->be_tag != be2->be_tag) + return false; + + return true; +} + +static struct pnfs_block_extent * +ext_try_to_merge_left(struct rb_root *root, struct pnfs_block_extent *be) +{ + struct pnfs_block_extent *left = ext_tree_prev(be); + + if (left && ext_can_merge(left, be)) { + left->be_length += be->be_length; + rb_erase(&be->be_node, root); + nfs4_put_deviceid_node(be->be_device); + kfree(be); + return left; + } + + return be; +} + +static struct pnfs_block_extent * +ext_try_to_merge_right(struct rb_root *root, struct pnfs_block_extent *be) +{ + struct pnfs_block_extent *right = ext_tree_next(be); + + if (right && ext_can_merge(be, right)) { + be->be_length += right->be_length; + rb_erase(&right->be_node, root); + nfs4_put_deviceid_node(right->be_device); + kfree(right); + } + + return be; +} + +static void __ext_put_deviceids(struct list_head *head) +{ + struct pnfs_block_extent *be, *tmp; + + list_for_each_entry_safe(be, tmp, head, be_list) { + nfs4_put_deviceid_node(be->be_device); + kfree(be); + } +} + +static void +__ext_tree_insert(struct rb_root *root, + struct pnfs_block_extent *new, bool merge_ok) +{ + struct rb_node **p = &root->rb_node, *parent = NULL; + struct pnfs_block_extent *be; + + while (*p) { + parent = *p; + be = ext_node(parent); + + if (new->be_f_offset < be->be_f_offset) { + if (merge_ok && ext_can_merge(new, be)) { + be->be_f_offset = new->be_f_offset; + if (be->be_state != PNFS_BLOCK_NONE_DATA) + be->be_v_offset = new->be_v_offset; + be->be_length += new->be_length; + be = ext_try_to_merge_left(root, be); + goto free_new; + } + p = &(*p)->rb_left; + } else if (new->be_f_offset >= ext_f_end(be)) { + if (merge_ok && ext_can_merge(be, new)) { + be->be_length += new->be_length; + be = ext_try_to_merge_right(root, be); + goto free_new; + } + p = &(*p)->rb_right; + } else { + BUG(); + } + } + + rb_link_node(&new->be_node, parent, p); + rb_insert_color(&new->be_node, root); + return; +free_new: + nfs4_put_deviceid_node(new->be_device); + kfree(new); +} + +static int +__ext_tree_remove(struct rb_root *root, + sector_t start, sector_t end, struct list_head *tmp) +{ + struct pnfs_block_extent *be; + sector_t len1 = 0, len2 = 0; + sector_t orig_v_offset; + sector_t orig_len; + + be = __ext_tree_search(root, start); + if (!be) + return 0; + if (be->be_f_offset >= end) + return 0; + + orig_v_offset = be->be_v_offset; + orig_len = be->be_length; + + if (start > be->be_f_offset) + len1 = start - be->be_f_offset; + if (ext_f_end(be) > end) + len2 = ext_f_end(be) - end; + + if (len2 > 0) { + if (len1 > 0) { + struct pnfs_block_extent *new; + + new = kzalloc_obj(*new, GFP_ATOMIC); + if (!new) + return -ENOMEM; + + be->be_length = len1; + + new->be_f_offset = end; + if (be->be_state != PNFS_BLOCK_NONE_DATA) { + new->be_v_offset = + orig_v_offset + orig_len - len2; + } + new->be_length = len2; + new->be_state = be->be_state; + new->be_tag = be->be_tag; + new->be_device = nfs4_get_deviceid(be->be_device); + + __ext_tree_insert(root, new, true); + } else { + be->be_f_offset = end; + if (be->be_state != PNFS_BLOCK_NONE_DATA) { + be->be_v_offset = + orig_v_offset + orig_len - len2; + } + be->be_length = len2; + } + } else { + if (len1 > 0) { + be->be_length = len1; + be = ext_tree_next(be); + } + + while (be && ext_f_end(be) <= end) { + struct pnfs_block_extent *next = ext_tree_next(be); + + rb_erase(&be->be_node, root); + list_add_tail(&be->be_list, tmp); + be = next; + } + + if (be && be->be_f_offset < end) { + len1 = ext_f_end(be) - end; + be->be_f_offset = end; + if (be->be_state != PNFS_BLOCK_NONE_DATA) + be->be_v_offset += be->be_length - len1; + be->be_length = len1; + } + } + + return 0; +} + +int +ext_tree_insert(struct pnfs_block_layout *bl, struct pnfs_block_extent *new) +{ + struct pnfs_block_extent *be; + struct rb_root *root; + int err = 0; + + switch (new->be_state) { + case PNFS_BLOCK_READWRITE_DATA: + case PNFS_BLOCK_INVALID_DATA: + root = &bl->bl_ext_rw; + break; + case PNFS_BLOCK_READ_DATA: + case PNFS_BLOCK_NONE_DATA: + root = &bl->bl_ext_ro; + break; + default: + dprintk("invalid extent type\n"); + return -EINVAL; + } + + spin_lock(&bl->bl_ext_lock); +retry: + be = __ext_tree_search(root, new->be_f_offset); + if (!be || be->be_f_offset >= ext_f_end(new)) { + __ext_tree_insert(root, new, true); + } else if (new->be_f_offset >= be->be_f_offset) { + if (ext_f_end(new) <= ext_f_end(be)) { + nfs4_put_deviceid_node(new->be_device); + kfree(new); + } else { + sector_t new_len = ext_f_end(new) - ext_f_end(be); + sector_t diff = new->be_length - new_len; + + new->be_f_offset += diff; + new->be_v_offset += diff; + new->be_length = new_len; + goto retry; + } + } else if (ext_f_end(new) <= ext_f_end(be)) { + new->be_length = be->be_f_offset - new->be_f_offset; + __ext_tree_insert(root, new, true); + } else { + struct pnfs_block_extent *split; + sector_t new_len = ext_f_end(new) - ext_f_end(be); + sector_t diff = new->be_length - new_len; + + split = kmemdup(new, sizeof(*new), GFP_ATOMIC); + if (!split) { + err = -EINVAL; + goto out; + } + + split->be_length = be->be_f_offset - split->be_f_offset; + split->be_device = nfs4_get_deviceid(new->be_device); + __ext_tree_insert(root, split, true); + + new->be_f_offset += diff; + new->be_v_offset += diff; + new->be_length = new_len; + goto retry; + } +out: + spin_unlock(&bl->bl_ext_lock); + return err; +} + +static bool +__ext_tree_lookup(struct rb_root *root, sector_t isect, + struct pnfs_block_extent *ret) +{ + struct rb_node *node; + struct pnfs_block_extent *be; + + node = root->rb_node; + while (node) { + be = ext_node(node); + if (isect < be->be_f_offset) + node = node->rb_left; + else if (isect >= ext_f_end(be)) + node = node->rb_right; + else { + *ret = *be; + return true; + } + } + + return false; +} + +bool +ext_tree_lookup(struct pnfs_block_layout *bl, sector_t isect, + struct pnfs_block_extent *ret, bool rw) +{ + bool found = false; + + spin_lock(&bl->bl_ext_lock); + if (!rw) + found = __ext_tree_lookup(&bl->bl_ext_ro, isect, ret); + if (!found) + found = __ext_tree_lookup(&bl->bl_ext_rw, isect, ret); + spin_unlock(&bl->bl_ext_lock); + + return found; +} + +int ext_tree_remove(struct pnfs_block_layout *bl, bool rw, + sector_t start, sector_t end) +{ + int err, err2; + LIST_HEAD(tmp); + + spin_lock(&bl->bl_ext_lock); + err = __ext_tree_remove(&bl->bl_ext_ro, start, end, &tmp); + if (rw) { + err2 = __ext_tree_remove(&bl->bl_ext_rw, start, end, &tmp); + if (!err) + err = err2; + } + spin_unlock(&bl->bl_ext_lock); + + __ext_put_deviceids(&tmp); + return err; +} + +static int +ext_tree_split(struct rb_root *root, struct pnfs_block_extent *be, + sector_t split) +{ + struct pnfs_block_extent *new; + sector_t orig_len = be->be_length; + + new = kzalloc_obj(*new, GFP_ATOMIC); + if (!new) + return -ENOMEM; + + be->be_length = split - be->be_f_offset; + + new->be_f_offset = split; + if (be->be_state != PNFS_BLOCK_NONE_DATA) + new->be_v_offset = be->be_v_offset + be->be_length; + new->be_length = orig_len - be->be_length; + new->be_state = be->be_state; + new->be_tag = be->be_tag; + new->be_device = nfs4_get_deviceid(be->be_device); + + __ext_tree_insert(root, new, false); + return 0; +} + +int +ext_tree_mark_written(struct pnfs_block_layout *bl, sector_t start, + sector_t len, u64 lwb) +{ + struct rb_root *root = &bl->bl_ext_rw; + sector_t end = start + len; + struct pnfs_block_extent *be; + int err = 0; + LIST_HEAD(tmp); + + spin_lock(&bl->bl_ext_lock); + /* + * First remove all COW extents or holes from written to range. + */ + err = __ext_tree_remove(&bl->bl_ext_ro, start, end, &tmp); + if (err) + goto out; + + /* + * Then mark all invalid extents in the range as written to. + */ + for (be = __ext_tree_search(root, start); be; be = ext_tree_next(be)) { + if (be->be_f_offset >= end) + break; + + if (be->be_state != PNFS_BLOCK_INVALID_DATA || be->be_tag) + continue; + + if (be->be_f_offset < start) { + struct pnfs_block_extent *left = ext_tree_prev(be); + + if (left && ext_can_merge(left, be)) { + sector_t diff = start - be->be_f_offset; + + left->be_length += diff; + + be->be_f_offset += diff; + be->be_v_offset += diff; + be->be_length -= diff; + } else { + err = ext_tree_split(root, be, start); + if (err) + goto out; + } + } + + if (ext_f_end(be) > end) { + struct pnfs_block_extent *right = ext_tree_next(be); + + if (right && ext_can_merge(be, right)) { + sector_t diff = end - be->be_f_offset; + + be->be_length -= diff; + + right->be_f_offset -= diff; + right->be_v_offset -= diff; + right->be_length += diff; + } else { + err = ext_tree_split(root, be, end); + if (err) + goto out; + } + } + + if (be->be_f_offset >= start && ext_f_end(be) <= end) { + be->be_tag = EXTENT_WRITTEN; + be = ext_try_to_merge_left(root, be); + be = ext_try_to_merge_right(root, be); + } + } +out: + if (bl->bl_lwb < lwb) + bl->bl_lwb = lwb; + spin_unlock(&bl->bl_ext_lock); + + __ext_put_deviceids(&tmp); + return err; +} + +static size_t ext_tree_layoutupdate_size(struct pnfs_block_layout *bl, size_t count) +{ + if (bl->bl_scsi_layout) + return sizeof(__be32) + PNFS_SCSI_RANGE_SIZE * count; + else + return sizeof(__be32) + PNFS_BLOCK_EXTENT_SIZE * count; +} + +static void ext_tree_free_commitdata(struct nfs4_layoutcommit_args *arg, + size_t buffer_size) +{ + if (arg->layoutupdate_pages != &arg->layoutupdate_page) { + int nr_pages = DIV_ROUND_UP(buffer_size, PAGE_SIZE), i; + + for (i = 0; i < nr_pages; i++) + put_page(arg->layoutupdate_pages[i]); + vfree(arg->start_p); + kfree(arg->layoutupdate_pages); + } else { + put_page(arg->layoutupdate_page); + } +} + +static __be32 *encode_block_extent(struct pnfs_block_extent *be, __be32 *p) +{ + p = xdr_encode_opaque_fixed(p, be->be_device->deviceid.data, + NFS4_DEVICEID4_SIZE); + p = xdr_encode_hyper(p, be->be_f_offset << SECTOR_SHIFT); + p = xdr_encode_hyper(p, be->be_length << SECTOR_SHIFT); + p = xdr_encode_hyper(p, 0LL); + *p++ = cpu_to_be32(PNFS_BLOCK_READWRITE_DATA); + return p; +} + +static __be32 *encode_scsi_range(struct pnfs_block_extent *be, __be32 *p) +{ + p = xdr_encode_hyper(p, be->be_f_offset << SECTOR_SHIFT); + return xdr_encode_hyper(p, be->be_length << SECTOR_SHIFT); +} + +/** + * ext_tree_try_encode_commit - try to encode all extents into the buffer + * @bl: pointer to the layout + * @p: pointer to the output buffer + * @buffer_size: size of the output buffer + * @count: output pointer to the number of encoded extents + * @lastbyte: output pointer to the last written byte + * + * Return values: + * %0: Success, all required extents encoded, outputs are valid + * %-ENOSPC: Buffer too small, nothing encoded, outputs are invalid + */ +static int +ext_tree_try_encode_commit(struct pnfs_block_layout *bl, __be32 *p, + size_t buffer_size, size_t *count, __u64 *lastbyte) +{ + struct pnfs_block_extent *be; + + spin_lock(&bl->bl_ext_lock); + for (be = ext_tree_first(&bl->bl_ext_rw); be; be = ext_tree_next(be)) { + if (be->be_state != PNFS_BLOCK_INVALID_DATA || + be->be_tag != EXTENT_WRITTEN) + continue; + + (*count)++; + if (ext_tree_layoutupdate_size(bl, *count) > buffer_size) { + spin_unlock(&bl->bl_ext_lock); + return -ENOSPC; + } + } + for (be = ext_tree_first(&bl->bl_ext_rw); be; be = ext_tree_next(be)) { + if (be->be_state != PNFS_BLOCK_INVALID_DATA || + be->be_tag != EXTENT_WRITTEN) + continue; + + if (bl->bl_scsi_layout) + p = encode_scsi_range(be, p); + else + p = encode_block_extent(be, p); + be->be_tag = EXTENT_COMMITTING; + } + *lastbyte = (bl->bl_lwb != 0) ? bl->bl_lwb - 1 : U64_MAX; + bl->bl_lwb = 0; + spin_unlock(&bl->bl_ext_lock); + + return 0; +} + +/** + * ext_tree_encode_commit - encode as much as possible extents into the buffer + * @bl: pointer to the layout + * @p: pointer to the output buffer + * @buffer_size: size of the output buffer + * @count: output pointer to the number of encoded extents + * @lastbyte: output pointer to the last written byte + * + * Return values: + * %0: Success, all required extents encoded, outputs are valid + * %-ENOSPC: Buffer too small, some extents are encoded, outputs are valid + */ +static int +ext_tree_encode_commit(struct pnfs_block_layout *bl, __be32 *p, + size_t buffer_size, size_t *count, __u64 *lastbyte) +{ + struct pnfs_block_extent *be, *be_prev; + int ret = 0; + + spin_lock(&bl->bl_ext_lock); + for (be = ext_tree_first(&bl->bl_ext_rw); be; be = ext_tree_next(be)) { + if (be->be_state != PNFS_BLOCK_INVALID_DATA || + be->be_tag != EXTENT_WRITTEN) + continue; + + (*count)++; + if (ext_tree_layoutupdate_size(bl, *count) > buffer_size) { + (*count)--; + ret = -ENOSPC; + break; + } + + if (bl->bl_scsi_layout) + p = encode_scsi_range(be, p); + else + p = encode_block_extent(be, p); + be->be_tag = EXTENT_COMMITTING; + be_prev = be; + } + if (!ret) { + *lastbyte = (bl->bl_lwb != 0) ? bl->bl_lwb - 1 : U64_MAX; + bl->bl_lwb = 0; + } else { + *lastbyte = be_prev->be_f_offset + be_prev->be_length; + *lastbyte <<= SECTOR_SHIFT; + *lastbyte -= 1; + } + spin_unlock(&bl->bl_ext_lock); + + return ret; +} + +/** + * ext_tree_prepare_commit - encode extents that need to be committed + * @arg: layout commit data + * + * Return values: + * %0: Success, all required extents are encoded + * %-ENOSPC: Some extents are encoded, but not all, due to RPC size limit + * %-ENOMEM: Out of memory, extents not encoded + */ +int +ext_tree_prepare_commit(struct nfs4_layoutcommit_args *arg) +{ + struct pnfs_block_layout *bl = BLK_LO2EXT(NFS_I(arg->inode)->layout); + size_t count = 0, buffer_size = PAGE_SIZE; + __be32 *start_p; + int ret; + + arg->layoutupdate_page = alloc_page(GFP_NOFS); + if (!arg->layoutupdate_page) + return -ENOMEM; + start_p = page_address(arg->layoutupdate_page); + arg->layoutupdate_pages = &arg->layoutupdate_page; + + ret = ext_tree_try_encode_commit(bl, start_p + 1, buffer_size, + &count, &arg->lastbytewritten); + if (unlikely(ret)) { + ext_tree_free_commitdata(arg, buffer_size); + + buffer_size = NFS_SERVER(arg->inode)->wsize; + count = 0; + + arg->layoutupdate_pages = + kzalloc_objs(struct page *, + DIV_ROUND_UP(buffer_size, PAGE_SIZE), + GFP_NOFS); + if (!arg->layoutupdate_pages) + return -ENOMEM; + + start_p = __vmalloc(buffer_size, GFP_NOFS); + if (!start_p) { + kfree(arg->layoutupdate_pages); + return -ENOMEM; + } + + ret = ext_tree_encode_commit(bl, start_p + 1, buffer_size, + &count, &arg->lastbytewritten); + } + + *start_p = cpu_to_be32(count); + arg->layoutupdate_len = ext_tree_layoutupdate_size(bl, count); + + if (unlikely(arg->layoutupdate_pages != &arg->layoutupdate_page)) { + void *p = start_p, *end = p + arg->layoutupdate_len; + struct page *page = NULL; + int i = 0; + + arg->start_p = start_p; + for ( ; p < end; p += PAGE_SIZE) { + page = vmalloc_to_page(p); + arg->layoutupdate_pages[i++] = page; + get_page(page); + } + } + + trace_bl_ext_tree_prepare_commit(ret, count, + arg->lastbytewritten, !!ret); + return ret; +} + +void +ext_tree_mark_committed(struct nfs4_layoutcommit_args *arg, int status) +{ + struct pnfs_block_layout *bl = BLK_LO2EXT(NFS_I(arg->inode)->layout); + struct rb_root *root = &bl->bl_ext_rw; + struct pnfs_block_extent *be; + + dprintk("%s status %d\n", __func__, status); + + ext_tree_free_commitdata(arg, arg->layoutupdate_len); + + spin_lock(&bl->bl_ext_lock); + for (be = ext_tree_first(root); be; be = ext_tree_next(be)) { + if (be->be_state != PNFS_BLOCK_INVALID_DATA || + be->be_tag != EXTENT_COMMITTING) + continue; + + if (status) { + /* + * Mark as written and try again. + * + * XXX: some real error handling here wouldn't hurt.. + */ + be->be_tag = EXTENT_WRITTEN; + } else { + be->be_state = PNFS_BLOCK_READWRITE_DATA; + be->be_tag = 0; + } + + be = ext_try_to_merge_left(root, be); + be = ext_try_to_merge_right(root, be); + } + spin_unlock(&bl->bl_ext_lock); +} diff --git a/fs/nfs/blocklayout/rpc_pipefs.c b/fs/nfs/blocklayout/rpc_pipefs.c new file mode 100644 index 000000000..d526f5ba7 --- /dev/null +++ b/fs/nfs/blocklayout/rpc_pipefs.c @@ -0,0 +1,270 @@ +/* + * Copyright (c) 2006,2007 The Regents of the University of Michigan. + * All rights reserved. + * + * Andy Adamson + * Fred Isaman + * + * permission is granted to use, copy, create derivative works and + * redistribute this software and such derivative works for any purpose, + * so long as the name of the university of michigan is not used in + * any advertising or publicity pertaining to the use or distribution + * of this software without specific, written prior authorization. if + * the above copyright notice or any other identification of the + * university of michigan is included in any copy of any portion of + * this software, then the disclaimer below must also be included. + * + * this software is provided as is, without representation from the + * university of michigan as to its fitness for any purpose, and without + * warranty by the university of michigan of any kind, either express + * or implied, including without limitation the implied warranties of + * merchantability and fitness for a particular purpose. the regents + * of the university of michigan shall not be liable for any damages, + * including special, indirect, incidental, or consequential damages, + * with respect to any claim arising out or in connection with the use + * of the software, even if it has been or is hereafter advised of the + * possibility of such damages. + */ + +#include +#include + +#include "blocklayout.h" + +#define NFSDBG_FACILITY NFSDBG_PNFS_LD + +static void +nfs4_encode_simple(__be32 *p, struct pnfs_block_volume *b) +{ + int i; + + *p++ = cpu_to_be32(1); + *p++ = cpu_to_be32(b->type); + *p++ = cpu_to_be32(b->simple.nr_sigs); + for (i = 0; i < b->simple.nr_sigs; i++) { + p = xdr_encode_hyper(p, b->simple.sigs[i].offset); + p = xdr_encode_opaque(p, b->simple.sigs[i].sig, + b->simple.sigs[i].sig_len); + } +} + +dev_t +bl_resolve_deviceid(struct nfs_server *server, struct pnfs_block_volume *b, + gfp_t gfp_mask) +{ + struct net *net = server->nfs_client->cl_net; + struct nfs_net *nn = net_generic(net, nfs_net_id); + struct bl_dev_msg *reply = &nn->bl_mount_reply; + struct bl_pipe_msg bl_pipe_msg; + struct rpc_pipe_msg *msg = &bl_pipe_msg.msg; + struct bl_msg_hdr *bl_msg; + DECLARE_WAITQUEUE(wq, current); + dev_t dev = 0; + int rc; + + dprintk("%s CREATING PIPEFS MESSAGE\n", __func__); + + mutex_lock(&nn->bl_mutex); + bl_pipe_msg.bl_wq = &nn->bl_wq; + + b->simple.len += 4; /* single volume */ + if (b->simple.len > PAGE_SIZE) + goto out_unlock; + + memset(msg, 0, sizeof(*msg)); + msg->len = sizeof(*bl_msg) + b->simple.len; + msg->data = kzalloc(msg->len, gfp_mask); + if (!msg->data) + goto out_unlock; + + bl_msg = msg->data; + bl_msg->type = BL_DEVICE_MOUNT; + bl_msg->totallen = b->simple.len; + nfs4_encode_simple(msg->data + sizeof(*bl_msg), b); + + dprintk("%s CALLING USERSPACE DAEMON\n", __func__); + add_wait_queue(&nn->bl_wq, &wq); + rc = rpc_queue_upcall(nn->bl_device_pipe, msg); + if (rc < 0) { + remove_wait_queue(&nn->bl_wq, &wq); + goto out_free_data; + } + + set_current_state(TASK_UNINTERRUPTIBLE); + schedule(); + remove_wait_queue(&nn->bl_wq, &wq); + + if (reply->status != BL_DEVICE_REQUEST_PROC) { + printk(KERN_WARNING "%s failed to decode device: %d\n", + __func__, reply->status); + goto out_free_data; + } + + dev = MKDEV(reply->major, reply->minor); +out_free_data: + kfree(msg->data); +out_unlock: + mutex_unlock(&nn->bl_mutex); + return dev; +} + +static ssize_t bl_pipe_downcall(struct file *filp, const char __user *src, + size_t mlen) +{ + struct nfs_net *nn = net_generic(file_inode(filp)->i_sb->s_fs_info, + nfs_net_id); + + if (mlen != sizeof (struct bl_dev_msg)) + return -EINVAL; + + if (copy_from_user(&nn->bl_mount_reply, src, mlen) != 0) + return -EFAULT; + + wake_up(&nn->bl_wq); + + return mlen; +} + +static void bl_pipe_destroy_msg(struct rpc_pipe_msg *msg) +{ + struct bl_pipe_msg *bl_pipe_msg = + container_of(msg, struct bl_pipe_msg, msg); + + if (msg->errno >= 0) + return; + wake_up(bl_pipe_msg->bl_wq); +} + +static const struct rpc_pipe_ops bl_upcall_ops = { + .upcall = rpc_pipe_generic_upcall, + .downcall = bl_pipe_downcall, + .destroy_msg = bl_pipe_destroy_msg, +}; + +static int nfs4blocklayout_register_sb(struct super_block *sb, + struct rpc_pipe *pipe) +{ + struct dentry *dir; + int err; + + dir = rpc_d_lookup_sb(sb, NFS_PIPE_DIRNAME); + if (dir == NULL) + return -ENOENT; + err = rpc_mkpipe_dentry(dir, "blocklayout", NULL, pipe); + dput(dir); + return err; +} + +static int rpc_pipefs_event(struct notifier_block *nb, unsigned long event, + void *ptr) +{ + struct super_block *sb = ptr; + struct net *net = sb->s_fs_info; + struct nfs_net *nn = net_generic(net, nfs_net_id); + int ret = 0; + + if (!try_module_get(THIS_MODULE)) + return 0; + + if (nn->bl_device_pipe == NULL) { + module_put(THIS_MODULE); + return 0; + } + + switch (event) { + case RPC_PIPEFS_MOUNT: + ret = nfs4blocklayout_register_sb(sb, nn->bl_device_pipe); + break; + case RPC_PIPEFS_UMOUNT: + rpc_unlink(nn->bl_device_pipe); + break; + default: + ret = -ENOTSUPP; + break; + } + module_put(THIS_MODULE); + return ret; +} + +static struct notifier_block nfs4blocklayout_block = { + .notifier_call = rpc_pipefs_event, +}; + +static int nfs4blocklayout_register_net(struct net *net, struct rpc_pipe *pipe) +{ + struct super_block *pipefs_sb; + int ret; + + pipefs_sb = rpc_get_sb_net(net); + if (!pipefs_sb) + return 0; + ret = nfs4blocklayout_register_sb(pipefs_sb, pipe); + rpc_put_sb_net(net); + return ret; +} + +static void nfs4blocklayout_unregister_net(struct net *net, + struct rpc_pipe *pipe) +{ + struct super_block *pipefs_sb; + + pipefs_sb = rpc_get_sb_net(net); + if (pipefs_sb) { + rpc_unlink(pipe); + rpc_put_sb_net(net); + } +} + +static int nfs4blocklayout_net_init(struct net *net) +{ + struct nfs_net *nn = net_generic(net, nfs_net_id); + int err; + + mutex_init(&nn->bl_mutex); + init_waitqueue_head(&nn->bl_wq); + nn->bl_device_pipe = rpc_mkpipe_data(&bl_upcall_ops, 0); + if (IS_ERR(nn->bl_device_pipe)) + return PTR_ERR(nn->bl_device_pipe); + err = nfs4blocklayout_register_net(net, nn->bl_device_pipe); + if (unlikely(err)) + rpc_destroy_pipe_data(nn->bl_device_pipe); + return err; +} + +static void nfs4blocklayout_net_exit(struct net *net) +{ + struct nfs_net *nn = net_generic(net, nfs_net_id); + + nfs4blocklayout_unregister_net(net, nn->bl_device_pipe); + rpc_destroy_pipe_data(nn->bl_device_pipe); + nn->bl_device_pipe = NULL; +} + +static struct pernet_operations nfs4blocklayout_net_ops = { + .init = nfs4blocklayout_net_init, + .exit = nfs4blocklayout_net_exit, +}; + +int __init bl_init_pipefs(void) +{ + int ret; + + ret = rpc_pipefs_notifier_register(&nfs4blocklayout_block); + if (ret) + goto out; + ret = register_pernet_subsys(&nfs4blocklayout_net_ops); + if (ret) + goto out_unregister_notifier; + return 0; + +out_unregister_notifier: + rpc_pipefs_notifier_unregister(&nfs4blocklayout_block); +out: + return ret; +} + +void bl_cleanup_pipefs(void) +{ + rpc_pipefs_notifier_unregister(&nfs4blocklayout_block); + unregister_pernet_subsys(&nfs4blocklayout_net_ops); +} diff --git a/fs/nfs/cache_lib.c b/fs/nfs/cache_lib.c new file mode 100644 index 000000000..9738a1ae9 --- /dev/null +++ b/fs/nfs/cache_lib.c @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/nfs/cache_lib.c + * + * Helper routines for the NFS client caches + * + * Copyright (c) 2009 Trond Myklebust + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cache_lib.h" + +#define NFS_CACHE_UPCALL_PATHLEN 256 +#define NFS_CACHE_UPCALL_TIMEOUT 15 + +static char nfs_cache_getent_prog[NFS_CACHE_UPCALL_PATHLEN] = + "/sbin/nfs_cache_getent"; +static unsigned long nfs_cache_getent_timeout = NFS_CACHE_UPCALL_TIMEOUT; + +module_param_string(cache_getent, nfs_cache_getent_prog, + sizeof(nfs_cache_getent_prog), 0600); +MODULE_PARM_DESC(cache_getent, "Path to the client cache upcall program"); +module_param_named(cache_getent_timeout, nfs_cache_getent_timeout, ulong, 0600); +MODULE_PARM_DESC(cache_getent_timeout, "Timeout (in seconds) after which " + "the cache upcall is assumed to have failed"); + +int nfs_cache_upcall(struct cache_detail *cd, char *entry_name) +{ + static char *envp[] = { "HOME=/", + "TERM=linux", + "PATH=/sbin:/usr/sbin:/bin:/usr/bin", + NULL + }; + char *argv[] = { + nfs_cache_getent_prog, + cd->name, + entry_name, + NULL + }; + int ret = -EACCES; + + if (nfs_cache_getent_prog[0] == '\0') + goto out; + ret = call_usermodehelper(argv[0], argv, envp, UMH_WAIT_EXEC); + /* + * Disable the upcall mechanism if we're getting an ENOENT or + * EACCES error. The admin can re-enable it on the fly by using + * sysfs to set the 'cache_getent' parameter once the problem + * has been fixed. + */ + if (ret == -ENOENT || ret == -EACCES) + nfs_cache_getent_prog[0] = '\0'; +out: + return ret > 0 ? 0 : ret; +} + +/* + * Deferred request handling + */ +void nfs_cache_defer_req_put(struct nfs_cache_defer_req *dreq) +{ + if (refcount_dec_and_test(&dreq->count)) + kfree(dreq); +} + +static void nfs_dns_cache_revisit(struct cache_deferred_req *d, int toomany) +{ + struct nfs_cache_defer_req *dreq; + + dreq = container_of(d, struct nfs_cache_defer_req, deferred_req); + + complete(&dreq->completion); + nfs_cache_defer_req_put(dreq); +} + +static struct cache_deferred_req *nfs_dns_cache_defer(struct cache_req *req) +{ + struct nfs_cache_defer_req *dreq; + + dreq = container_of(req, struct nfs_cache_defer_req, req); + dreq->deferred_req.revisit = nfs_dns_cache_revisit; + refcount_inc(&dreq->count); + + return &dreq->deferred_req; +} + +struct nfs_cache_defer_req *nfs_cache_defer_req_alloc(void) +{ + struct nfs_cache_defer_req *dreq; + + dreq = kzalloc_obj(*dreq); + if (dreq) { + init_completion(&dreq->completion); + refcount_set(&dreq->count, 1); + dreq->req.defer = nfs_dns_cache_defer; + } + return dreq; +} + +int nfs_cache_wait_for_upcall(struct nfs_cache_defer_req *dreq) +{ + if (wait_for_completion_timeout(&dreq->completion, + nfs_cache_getent_timeout * HZ) == 0) + return -ETIMEDOUT; + return 0; +} + +int nfs_cache_register_sb(struct super_block *sb, struct cache_detail *cd) +{ + int ret; + struct dentry *dir; + + dir = rpc_d_lookup_sb(sb, "cache"); + ret = sunrpc_cache_register_pipefs(dir, cd->name, 0600, cd); + dput(dir); + return ret; +} + +int nfs_cache_register_net(struct net *net, struct cache_detail *cd) +{ + struct super_block *pipefs_sb; + int ret = 0; + + sunrpc_init_cache_detail(cd); + pipefs_sb = rpc_get_sb_net(net); + if (pipefs_sb) { + ret = nfs_cache_register_sb(pipefs_sb, cd); + rpc_put_sb_net(net); + if (ret) + sunrpc_destroy_cache_detail(cd); + } + return ret; +} + +void nfs_cache_unregister_sb(struct super_block *sb, struct cache_detail *cd) +{ + sunrpc_cache_unregister_pipefs(cd); +} + +void nfs_cache_unregister_net(struct net *net, struct cache_detail *cd) +{ + struct super_block *pipefs_sb; + + pipefs_sb = rpc_get_sb_net(net); + if (pipefs_sb) { + nfs_cache_unregister_sb(pipefs_sb, cd); + rpc_put_sb_net(net); + } + sunrpc_destroy_cache_detail(cd); +} diff --git a/fs/nfs/cache_lib.h b/fs/nfs/cache_lib.h new file mode 100644 index 000000000..220ee409a --- /dev/null +++ b/fs/nfs/cache_lib.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Helper routines for the NFS client caches + * + * Copyright (c) 2009 Trond Myklebust + */ + +#include +#include +#include + +/* + * Deferred request handling + */ +struct nfs_cache_defer_req { + struct cache_req req; + struct cache_deferred_req deferred_req; + struct completion completion; + refcount_t count; +}; + +extern int nfs_cache_upcall(struct cache_detail *cd, char *entry_name); +extern struct nfs_cache_defer_req *nfs_cache_defer_req_alloc(void); +extern void nfs_cache_defer_req_put(struct nfs_cache_defer_req *dreq); +extern int nfs_cache_wait_for_upcall(struct nfs_cache_defer_req *dreq); + +extern int nfs_cache_register_net(struct net *net, struct cache_detail *cd); +extern void nfs_cache_unregister_net(struct net *net, struct cache_detail *cd); +extern int nfs_cache_register_sb(struct super_block *sb, + struct cache_detail *cd); +extern void nfs_cache_unregister_sb(struct super_block *sb, + struct cache_detail *cd); diff --git a/fs/nfs/callback.c b/fs/nfs/callback.c new file mode 100644 index 000000000..bc282b744 --- /dev/null +++ b/fs/nfs/callback.c @@ -0,0 +1,355 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/nfs/callback.c + * + * Copyright (C) 2004 Trond Myklebust + * + * NFSv4 callback handling + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "nfs4_fs.h" +#include "callback.h" +#include "internal.h" +#include "netns.h" + +#define NFSDBG_FACILITY NFSDBG_CALLBACK + +struct nfs_callback_data { + unsigned int users; + struct svc_serv *serv; +}; + +static struct nfs_callback_data nfs_callback_info[NFS4_MAX_MINOR_VERSION + 1]; +static DEFINE_MUTEX(nfs_callback_mutex); +static struct svc_program nfs4_callback_program; + +static int nfs4_callback_up_net(struct svc_serv *serv, struct net *net) +{ + const struct cred *cred = current_cred(); + int ret; + struct nfs_net *nn = net_generic(net, nfs_net_id); + + ret = svc_xprt_create(serv, "tcp", net, PF_INET, + nfs_callback_set_tcpport, SVC_SOCK_ANONYMOUS, + cred); + if (ret <= 0) + goto out_err; + nn->nfs_callback_tcpport = ret; + dprintk("NFS: Callback listener port = %u (af %u, net %x)\n", + nn->nfs_callback_tcpport, PF_INET, net->ns.inum); + + ret = svc_xprt_create(serv, "tcp", net, PF_INET6, + nfs_callback_set_tcpport, SVC_SOCK_ANONYMOUS, + cred); + if (ret > 0) { + nn->nfs_callback_tcpport6 = ret; + dprintk("NFS: Callback listener port = %u (af %u, net %x)\n", + nn->nfs_callback_tcpport6, PF_INET6, net->ns.inum); + } else if (ret != -EAFNOSUPPORT) + goto out_err; + return 0; + +out_err: + return (ret) ? ret : -ENOMEM; +} + +/* + * This is the NFSv4 callback kernel thread. + */ +static int +nfs4_callback_svc(void *vrqstp) +{ + struct svc_rqst *rqstp = vrqstp; + + svc_thread_init_status(rqstp, 0); + + set_freezable(); + + while (!svc_thread_should_stop(rqstp)) + svc_recv(rqstp, 0); + + svc_exit_thread(rqstp); + return 0; +} + +static inline void nfs_callback_bc_serv(u32 minorversion, struct rpc_xprt *xprt, + struct svc_serv *serv) +{ + if (minorversion) + /* + * Save the svc_serv in the transport so that it can + * be referenced when the session backchannel is initialized + */ + xprt->bc_serv = serv; +} + +static int nfs_callback_start_svc(int minorversion, struct rpc_xprt *xprt, + struct svc_serv *serv) +{ + int nrservs = nfs_callback_nr_threads; + int ret; + + nfs_callback_bc_serv(minorversion, xprt, serv); + + if (nrservs < NFS4_MIN_NR_CALLBACK_THREADS) + nrservs = NFS4_MIN_NR_CALLBACK_THREADS; + + if (serv->sv_nrthreads == nrservs) + return 0; + + ret = svc_set_num_threads(serv, 0, nrservs); + if (ret) { + svc_set_num_threads(serv, 0, 0); + return ret; + } + dprintk("nfs_callback_up: service started\n"); + return 0; +} + +static void nfs_callback_down_net(u32 minorversion, struct svc_serv *serv, struct net *net) +{ + struct nfs_net *nn = net_generic(net, nfs_net_id); + + if (--nn->cb_users[minorversion]) + return; + + dprintk("NFS: destroy per-net callback data; net=%x\n", net->ns.inum); + svc_xprt_destroy_all(serv, net, false); +} + +static int nfs_callback_up_net(int minorversion, struct svc_serv *serv, + struct net *net, struct rpc_xprt *xprt) +{ + struct nfs_net *nn = net_generic(net, nfs_net_id); + int ret; + + if (nn->cb_users[minorversion]++) + return 0; + + dprintk("NFS: create per-net callback data; net=%x\n", net->ns.inum); + + ret = svc_bind(serv, net); + if (ret < 0) { + printk(KERN_WARNING "NFS: bind callback service failed\n"); + goto err; + } + + ret = 0; + if (minorversion == 0) + ret = nfs4_callback_up_net(serv, net); + else if (xprt->ops->bc_setup) + set_bc_enabled(serv); + else + ret = -EPROTONOSUPPORT; + + if (ret < 0) { + printk(KERN_ERR "NFS: callback service start failed\n"); + goto err; + } + return 0; + +err: + nn->cb_users[minorversion]--; + dprintk("NFS: Couldn't create callback socket: err = %d; " + "net = %x\n", ret, net->ns.inum); + return ret; +} + +static struct svc_serv *nfs_callback_create_svc(int minorversion) +{ + struct nfs_callback_data *cb_info = &nfs_callback_info[minorversion]; + int (*threadfn)(void *data); + struct svc_serv *serv; + + /* + * Check whether we're already up and running. + */ + if (cb_info->serv) + return cb_info->serv; + + /* + * Sanity check: if there's no task, + * we should be the first user ... + */ + if (cb_info->users) + printk(KERN_WARNING "nfs_callback_create_svc: no kthread, %d users??\n", + cb_info->users); + + threadfn = nfs4_callback_svc; + serv = svc_create(&nfs4_callback_program, NFS4_CALLBACK_BUFSIZE, + threadfn); + if (!serv) { + printk(KERN_ERR "nfs_callback_create_svc: create service failed\n"); + return ERR_PTR(-ENOMEM); + } + cb_info->serv = serv; + dprintk("nfs_callback_create_svc: service created\n"); + return serv; +} + +/* + * Bring up the callback thread if it is not already up. + */ +int nfs_callback_up(u32 minorversion, struct rpc_xprt *xprt) +{ + struct svc_serv *serv; + struct nfs_callback_data *cb_info = &nfs_callback_info[minorversion]; + int ret; + struct net *net = xprt->xprt_net; + + mutex_lock(&nfs_callback_mutex); + + serv = nfs_callback_create_svc(minorversion); + if (IS_ERR(serv)) { + ret = PTR_ERR(serv); + goto err_create; + } + + ret = nfs_callback_up_net(minorversion, serv, net, xprt); + if (ret < 0) + goto err_net; + + ret = nfs_callback_start_svc(minorversion, xprt, serv); + if (ret < 0) + goto err_start; + + cb_info->users++; +err_net: + if (!cb_info->users) { + xprt_svc_shutdown_bc(xprt); + svc_set_num_threads(cb_info->serv, 0, 0); + xprt_svc_destroy_nullify_bc(xprt, &cb_info->serv); + } +err_create: + mutex_unlock(&nfs_callback_mutex); + return ret; + +err_start: + nfs_callback_down_net(minorversion, serv, net); + dprintk("NFS: Couldn't create server thread; err = %d\n", ret); + goto err_net; +} + +/* + * Kill the callback thread if it's no longer being used. + */ +void nfs_callback_down(int minorversion, struct net *net, struct rpc_xprt *xprt) +{ + struct nfs_callback_data *cb_info = &nfs_callback_info[minorversion]; + struct svc_serv *serv; + + mutex_lock(&nfs_callback_mutex); + serv = cb_info->serv; + xprt_svc_shutdown_bc(xprt); + nfs_callback_down_net(minorversion, serv, net); + cb_info->users--; + if (cb_info->users == 0) { + svc_set_num_threads(serv, 0, 0); + dprintk("nfs_callback_down: service destroyed\n"); + xprt_svc_destroy_nullify_bc(xprt, &cb_info->serv); + } + mutex_unlock(&nfs_callback_mutex); +} + +/* Boolean check of RPC_AUTH_GSS principal */ +int +check_gss_callback_principal(struct nfs_client *clp, struct svc_rqst *rqstp) +{ + char *p = rqstp->rq_cred.cr_principal; + + if (rqstp->rq_authop->flavour != RPC_AUTH_GSS) + return 1; + + /* No RPC_AUTH_GSS on NFSv4.1 back channel yet */ + if (clp->cl_minorversion != 0) + return 0; + /* + * It might just be a normal user principal, in which case + * userspace won't bother to tell us the name at all. + */ + if (p == NULL) + return 0; + + /* + * Did we get the acceptor from userland during the SETCLIENID + * negotiation? + */ + if (clp->cl_acceptor) + return !strcmp(p, clp->cl_acceptor); + + /* + * Otherwise try to verify it using the cl_hostname. Note that this + * doesn't work if a non-canonical hostname was used in the devname. + */ + + /* Expect a GSS_C_NT_HOSTBASED_NAME like "nfs@serverhostname" */ + + if (memcmp(p, "nfs@", 4) != 0) + return 0; + p += 4; + if (strcmp(p, clp->cl_hostname) != 0) + return 0; + return 1; +} + +/* + * pg_authenticate method for nfsv4 callback threads. + * + * The authflavor has been negotiated, so an incorrect flavor is a server + * bug. Deny packets with incorrect authflavor. + * + * All other checking done after NFS decoding where the nfs_client can be + * found in nfs4_callback_compound + */ +static enum svc_auth_status nfs_callback_authenticate(struct svc_rqst *rqstp) +{ + rqstp->rq_auth_stat = rpc_autherr_badcred; + + switch (rqstp->rq_authop->flavour) { + case RPC_AUTH_NULL: + if (rqstp->rq_proc != CB_NULL) + return SVC_DENIED; + break; + case RPC_AUTH_GSS: + /* No RPC_AUTH_GSS support yet in NFSv4.1 */ + if (svc_is_backchannel(rqstp)) + return SVC_DENIED; + } + + rqstp->rq_auth_stat = rpc_auth_ok; + return SVC_OK; +} + +/* + * Define NFS4 callback program + */ +static const struct svc_version *nfs4_callback_version[] = { + [1] = &nfs4_callback_version1, + [4] = &nfs4_callback_version4, +}; + +static struct svc_program nfs4_callback_program = { + .pg_prog = NFS4_CALLBACK, /* RPC service number */ + .pg_nvers = ARRAY_SIZE(nfs4_callback_version), /* Number of entries */ + .pg_vers = nfs4_callback_version, /* version table */ + .pg_name = "NFSv4 callback", /* service name */ + .pg_class = "nfs", /* authentication class */ + .pg_authenticate = nfs_callback_authenticate, + .pg_init_request = svc_generic_init_request, + .pg_rpcbind_set = svc_generic_rpcbind_set, +}; diff --git a/fs/nfs/callback.h b/fs/nfs/callback.h new file mode 100644 index 000000000..2a721c422 --- /dev/null +++ b/fs/nfs/callback.h @@ -0,0 +1,204 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * linux/fs/nfs/callback.h + * + * Copyright (C) 2004 Trond Myklebust + * + * NFSv4 callback definitions + */ +#ifndef __LINUX_FS_NFS_CALLBACK_H +#define __LINUX_FS_NFS_CALLBACK_H +#include + +#define NFS4_CALLBACK 0x40000000 +#define NFS4_CALLBACK_XDRSIZE 2048 +#define NFS4_CALLBACK_BUFSIZE (1024 + NFS4_CALLBACK_XDRSIZE) + +enum nfs4_callback_procnum { + CB_NULL = 0, + CB_COMPOUND = 1, +}; + +struct nfs4_slot; +struct cb_process_state { + struct nfs_client *clp; + struct nfs4_slot *slot; + struct net *net; + u32 minorversion; + __be32 drc_status; + unsigned int referring_calls; +}; + +struct cb_compound_hdr_arg { + unsigned int taglen; + const char *tag; + unsigned int minorversion; + unsigned int cb_ident; /* v4.0 callback identifier */ + unsigned nops; +}; + +struct cb_compound_hdr_res { + __be32 *status; + unsigned int taglen; + const char *tag; + __be32 *nops; +}; + +struct cb_getattrargs { + struct nfs_fh fh; + uint32_t bitmap[3]; +}; + +struct cb_getattrres { + __be32 status; + uint32_t bitmap[3]; + uint64_t size; + uint64_t change_attr; + struct timespec64 atime; + struct timespec64 ctime; + struct timespec64 mtime; +}; + +struct cb_recallargs { + struct nfs_fh fh; + nfs4_stateid stateid; + uint32_t truncate; +}; + +struct referring_call { + uint32_t rc_sequenceid; + uint32_t rc_slotid; +}; + +struct referring_call_list { + struct nfs4_sessionid rcl_sessionid; + uint32_t rcl_nrefcalls; + struct referring_call *rcl_refcalls; +}; + +struct cb_sequenceargs { + struct sockaddr *csa_addr; + struct nfs4_sessionid csa_sessionid; + uint32_t csa_sequenceid; + uint32_t csa_slotid; + uint32_t csa_highestslotid; + uint32_t csa_cachethis; + uint32_t csa_nrclists; + struct referring_call_list *csa_rclists; +}; + +struct cb_sequenceres { + __be32 csr_status; + struct nfs4_sessionid csr_sessionid; + uint32_t csr_sequenceid; + uint32_t csr_slotid; + uint32_t csr_highestslotid; + uint32_t csr_target_highestslotid; +}; + +extern __be32 nfs4_callback_sequence(void *argp, void *resp, + struct cb_process_state *cps); + +#define RCA4_TYPE_MASK_RDATA_DLG 0 +#define RCA4_TYPE_MASK_WDATA_DLG 1 +#define RCA4_TYPE_MASK_DIR_DLG 2 +#define RCA4_TYPE_MASK_FILE_LAYOUT 3 +#define RCA4_TYPE_MASK_BLK_LAYOUT 4 +#define RCA4_TYPE_MASK_OBJ_LAYOUT_MIN 8 +#define RCA4_TYPE_MASK_OBJ_LAYOUT_MAX 9 +#define RCA4_TYPE_MASK_OTHER_LAYOUT_MIN 12 +#define RCA4_TYPE_MASK_OTHER_LAYOUT_MAX 15 +#define PNFS_FF_RCA4_TYPE_MASK_READ 16 +#define PNFS_FF_RCA4_TYPE_MASK_RW 17 +#define RCA4_TYPE_MASK_ALL 0x3f31f + +struct cb_recallanyargs { + uint32_t craa_objs_to_keep; + uint32_t craa_type_mask; +}; + +extern __be32 nfs4_callback_recallany(void *argp, void *resp, + struct cb_process_state *cps); + +struct cb_recallslotargs { + uint32_t crsa_target_highest_slotid; +}; +extern __be32 nfs4_callback_recallslot(void *argp, void *resp, + struct cb_process_state *cps); + +struct cb_layoutrecallargs { + uint32_t cbl_recall_type; + uint32_t cbl_layout_type; + uint32_t cbl_layoutchanged; + union { + struct { + struct nfs_fh cbl_fh; + struct pnfs_layout_range cbl_range; + nfs4_stateid cbl_stateid; + }; + struct nfs_fsid cbl_fsid; + }; +}; + +extern __be32 nfs4_callback_layoutrecall(void *argp, void *resp, + struct cb_process_state *cps); + +struct cb_devicenotifyitem { + uint32_t cbd_notify_type; + uint32_t cbd_layout_type; + struct nfs4_deviceid cbd_dev_id; + uint32_t cbd_immediate; +}; + +struct cb_devicenotifyargs { + uint32_t ndevs; + struct cb_devicenotifyitem *devs; +}; + +extern __be32 nfs4_callback_devicenotify(void *argp, void *resp, + struct cb_process_state *cps); + +struct cb_notify_lock_args { + struct nfs_fh cbnl_fh; + struct nfs_lowner cbnl_owner; + bool cbnl_valid; +}; + +extern __be32 nfs4_callback_notify_lock(void *argp, void *resp, + struct cb_process_state *cps); +#ifdef CONFIG_NFS_V4_2 +struct cb_offloadargs { + struct nfs_fh coa_fh; + nfs4_stateid coa_stateid; + uint32_t error; + uint64_t wr_count; + struct nfs_writeverf wr_writeverf; +}; + +extern __be32 nfs4_callback_offload(void *args, void *dummy, + struct cb_process_state *cps); +#endif /* CONFIG_NFS_V4_2 */ +extern int check_gss_callback_principal(struct nfs_client *, struct svc_rqst *); +extern __be32 nfs4_callback_getattr(void *argp, void *resp, + struct cb_process_state *cps); +extern __be32 nfs4_callback_recall(void *argp, void *resp, + struct cb_process_state *cps); +#if IS_ENABLED(CONFIG_NFS_V4) +extern int nfs_callback_up(u32 minorversion, struct rpc_xprt *xprt); +extern void nfs_callback_down(int minorversion, struct net *net, + struct rpc_xprt *xprt); +#endif /* CONFIG_NFS_V4 */ +/* + * nfs41: Callbacks are expected to not cause substantial latency, + * so we limit their concurrency to 1 by setting up the maximum number + * of slots for the backchannel. + */ +#define NFS41_BC_MIN_CALLBACKS 1 +#define NFS41_BC_MAX_CALLBACKS 1 + +#define NFS4_MIN_NR_CALLBACK_THREADS 1 + +extern unsigned int nfs_callback_set_tcpport; +extern unsigned short nfs_callback_nr_threads; + +#endif /* __LINUX_FS_NFS_CALLBACK_H */ diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c new file mode 100644 index 000000000..10f2354ba --- /dev/null +++ b/fs/nfs/callback_proc.c @@ -0,0 +1,756 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/nfs/callback_proc.c + * + * Copyright (C) 2004 Trond Myklebust + * + * NFSv4 callback procedures + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "nfs4_fs.h" +#include "callback.h" +#include "delegation.h" +#include "internal.h" +#include "pnfs.h" +#include "nfs4session.h" +#include "nfs4trace.h" + +#define NFSDBG_FACILITY NFSDBG_CALLBACK + +__be32 nfs4_callback_getattr(void *argp, void *resp, + struct cb_process_state *cps) +{ + struct cb_getattrargs *args = argp; + struct cb_getattrres *res = resp; + struct nfs_delegation *delegation; + struct inode *inode; + + res->status = htonl(NFS4ERR_OP_NOT_IN_SESSION); + if (!cps->clp) /* Always set for v4.0. Set in cb_sequence for v4.1 */ + goto out; + + memset(res->bitmap, 0, sizeof(res->bitmap)); + res->status = htonl(NFS4ERR_BADHANDLE); + + dprintk_rcu("NFS: GETATTR callback request from %s\n", + rpc_peeraddr2str(cps->clp->cl_rpcclient, RPC_DISPLAY_ADDR)); + + inode = nfs_delegation_find_inode(cps->clp, &args->fh); + if (IS_ERR(inode)) { + if (inode == ERR_PTR(-EAGAIN)) + res->status = htonl(NFS4ERR_DELAY); + trace_nfs4_cb_getattr(cps->clp, &args->fh, NULL, + -ntohl(res->status)); + goto out; + } + + delegation = nfs4_get_valid_delegation(inode); + if (!delegation) + goto out_iput; + if ((delegation->type & FMODE_WRITE) == 0) { + nfs_put_delegation(delegation); + goto out_iput; + } + res->change_attr = delegation->change_attr; + nfs_put_delegation(delegation); + + res->size = i_size_read(inode); + if (nfs_have_writebacks(inode)) + res->change_attr++; + res->atime = inode_get_atime(inode); + res->ctime = inode_get_ctime(inode); + res->mtime = inode_get_mtime(inode); + res->bitmap[0] = (FATTR4_WORD0_CHANGE | FATTR4_WORD0_SIZE) & + args->bitmap[0]; + res->bitmap[1] = (FATTR4_WORD1_TIME_ACCESS | + FATTR4_WORD1_TIME_METADATA | + FATTR4_WORD1_TIME_MODIFY) & args->bitmap[1]; + res->bitmap[2] = (FATTR4_WORD2_TIME_DELEG_ACCESS | + FATTR4_WORD2_TIME_DELEG_MODIFY) & args->bitmap[2]; + res->status = 0; +out_iput: + trace_nfs4_cb_getattr(cps->clp, &args->fh, inode, -ntohl(res->status)); + nfs_iput_and_deactive(inode); +out: + dprintk("%s: exit with status = %d\n", __func__, ntohl(res->status)); + return res->status; +} + +__be32 nfs4_callback_recall(void *argp, void *resp, + struct cb_process_state *cps) +{ + struct cb_recallargs *args = argp; + struct inode *inode; + __be32 res; + + res = htonl(NFS4ERR_OP_NOT_IN_SESSION); + if (!cps->clp) /* Always set for v4.0. Set in cb_sequence for v4.1 */ + goto out; + + dprintk_rcu("NFS: RECALL callback request from %s\n", + rpc_peeraddr2str(cps->clp->cl_rpcclient, RPC_DISPLAY_ADDR)); + + res = htonl(NFS4ERR_BADHANDLE); + inode = nfs_delegation_find_inode(cps->clp, &args->fh); + if (IS_ERR(inode)) { + if (inode == ERR_PTR(-EAGAIN)) + res = htonl(NFS4ERR_DELAY); + trace_nfs4_cb_recall(cps->clp, &args->fh, NULL, + &args->stateid, -ntohl(res)); + goto out; + } + /* Set up a helper thread to actually return the delegation */ + switch (nfs_async_inode_return_delegation(inode, &args->stateid)) { + case 0: + res = 0; + break; + case -ENOENT: + res = htonl(NFS4ERR_BAD_STATEID); + break; + default: + res = htonl(NFS4ERR_RESOURCE); + } + trace_nfs4_cb_recall(cps->clp, &args->fh, inode, + &args->stateid, -ntohl(res)); + nfs_iput_and_deactive(inode); +out: + dprintk("%s: exit with status = %d\n", __func__, ntohl(res)); + return res; +} + +/* + * Lookup a layout inode by stateid + * + * Note: returns a refcount on the inode and superblock + */ +static struct inode *nfs_layout_find_inode_by_stateid(struct nfs_client *clp, + const nfs4_stateid *stateid) + __must_hold(RCU) +{ + struct nfs_server *server; + struct inode *inode; + struct pnfs_layout_hdr *lo; + + rcu_read_lock(); + list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { + list_for_each_entry_rcu(lo, &server->layouts, plh_layouts) { + if (!pnfs_layout_is_valid(lo)) + continue; + if (!nfs4_stateid_match_other(stateid, &lo->plh_stateid)) + continue; + if (nfs_sb_active(server->super)) + inode = igrab(lo->plh_inode); + else + inode = ERR_PTR(-EAGAIN); + rcu_read_unlock(); + if (inode) + return inode; + nfs_sb_deactive(server->super); + return ERR_PTR(-EAGAIN); + } + } + rcu_read_unlock(); + return ERR_PTR(-ENOENT); +} + +/* + * Lookup a layout inode by filehandle. + * + * Note: returns a refcount on the inode and superblock + * + */ +static struct inode *nfs_layout_find_inode_by_fh(struct nfs_client *clp, + const struct nfs_fh *fh) +{ + struct nfs_server *server; + struct nfs_inode *nfsi; + struct inode *inode; + struct pnfs_layout_hdr *lo; + + rcu_read_lock(); + list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { + list_for_each_entry_rcu(lo, &server->layouts, plh_layouts) { + nfsi = NFS_I(lo->plh_inode); + if (nfs_compare_fh(fh, &nfsi->fh)) + continue; + if (nfsi->layout != lo) + continue; + if (nfs_sb_active(server->super)) + inode = igrab(lo->plh_inode); + else + inode = ERR_PTR(-EAGAIN); + rcu_read_unlock(); + if (inode) + return inode; + nfs_sb_deactive(server->super); + return ERR_PTR(-EAGAIN); + } + } + rcu_read_unlock(); + return ERR_PTR(-ENOENT); +} + +static struct inode *nfs_layout_find_inode(struct nfs_client *clp, + const struct nfs_fh *fh, + const nfs4_stateid *stateid) +{ + struct inode *inode; + + inode = nfs_layout_find_inode_by_stateid(clp, stateid); + if (inode == ERR_PTR(-ENOENT)) + inode = nfs_layout_find_inode_by_fh(clp, fh); + return inode; +} + +/* + * Enforce RFC5661 section 12.5.5.2.1. (Layout Recall and Return Sequencing) + */ +static u32 pnfs_check_callback_stateid(struct pnfs_layout_hdr *lo, + const nfs4_stateid *new, + struct cb_process_state *cps) +{ + u32 oldseq, newseq; + + /* Is the stateid not initialised? */ + if (!pnfs_layout_is_valid(lo)) + return NFS4ERR_NOMATCHING_LAYOUT; + + /* Mismatched stateid? */ + if (!nfs4_stateid_match_other(&lo->plh_stateid, new)) + return NFS4ERR_BAD_STATEID; + + newseq = be32_to_cpu(new->seqid); + /* Are we already in a layout recall situation? */ + if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) + return NFS4ERR_DELAY; + + /* + * Check that the stateid matches what we think it should be. + * Note that if the server sent us a list of referring calls, + * and we know that those have completed, then we trust the + * stateid argument is correct. + */ + oldseq = be32_to_cpu(lo->plh_stateid.seqid); + if (newseq > oldseq + 1 && !cps->referring_calls) + return NFS4ERR_DELAY; + + /* Crazy server! */ + if (newseq <= oldseq) + return NFS4ERR_OLD_STATEID; + + return NFS_OK; +} + +static u32 initiate_file_draining(struct nfs_client *clp, + struct cb_layoutrecallargs *args, + struct cb_process_state *cps) +{ + struct inode *ino; + struct pnfs_layout_hdr *lo; + u32 rv = NFS4ERR_NOMATCHING_LAYOUT; + LIST_HEAD(free_me_list); + bool return_range = false; + + ino = nfs_layout_find_inode(clp, &args->cbl_fh, &args->cbl_stateid); + if (IS_ERR(ino)) { + if (ino == ERR_PTR(-EAGAIN)) + rv = NFS4ERR_DELAY; + goto out_noput; + } + + pnfs_layoutcommit_inode(ino, false); + + + spin_lock(&ino->i_lock); + lo = NFS_I(ino)->layout; + if (!lo) { + spin_unlock(&ino->i_lock); + goto out; + } + pnfs_get_layout_hdr(lo); + rv = pnfs_check_callback_stateid(lo, &args->cbl_stateid, cps); + if (rv != NFS_OK) + goto unlock; + + /* + * Enforce RFC5661 Section 12.5.5.2.1.5 (Bulk Recall and Return) + */ + if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) { + rv = NFS4ERR_DELAY; + goto unlock; + } + + pnfs_set_layout_stateid(lo, &args->cbl_stateid, NULL, true); + switch (pnfs_mark_matching_lsegs_return(lo, &free_me_list, + &args->cbl_range, + be32_to_cpu(args->cbl_stateid.seqid))) { + case 0: + case -EBUSY: + /* There are layout segments that need to be returned */ + rv = NFS4_OK; + break; + case -ENOENT: + set_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags); + /* Embrace your forgetfulness! */ + rv = NFS4ERR_NOMATCHING_LAYOUT; + + return_range = true; + } +unlock: + spin_unlock(&ino->i_lock); + if (return_range && NFS_SERVER(ino)->pnfs_curr_ld->return_range) + NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, + &args->cbl_range); + pnfs_free_lseg_list(&free_me_list); + /* Free all lsegs that are attached to commit buckets */ + nfs_commit_inode(ino, 0); + pnfs_put_layout_hdr(lo); +out: + nfs_iput_and_deactive(ino); +out_noput: + trace_nfs4_cb_layoutrecall_file(clp, &args->cbl_fh, ino, + &args->cbl_stateid, -rv); + return rv; +} + +static u32 initiate_bulk_draining(struct nfs_client *clp, + struct cb_layoutrecallargs *args) +{ + int stat; + + if (args->cbl_recall_type == RETURN_FSID) + stat = pnfs_layout_destroy_byfsid(clp, &args->cbl_fsid, + PNFS_LAYOUT_BULK_RETURN); + else + stat = pnfs_layout_destroy_byclid(clp, PNFS_LAYOUT_BULK_RETURN); + if (stat != 0) + return NFS4ERR_DELAY; + return NFS4ERR_NOMATCHING_LAYOUT; +} + +static u32 do_callback_layoutrecall(struct nfs_client *clp, + struct cb_layoutrecallargs *args, + struct cb_process_state *cps) +{ + if (args->cbl_recall_type == RETURN_FILE) + return initiate_file_draining(clp, args, cps); + return initiate_bulk_draining(clp, args); +} + +__be32 nfs4_callback_layoutrecall(void *argp, void *resp, + struct cb_process_state *cps) +{ + struct cb_layoutrecallargs *args = argp; + u32 res = NFS4ERR_OP_NOT_IN_SESSION; + + if (cps->clp) + res = do_callback_layoutrecall(cps->clp, args, cps); + return cpu_to_be32(res); +} + +static void pnfs_recall_all_layouts(struct nfs_client *clp, + struct cb_process_state *cps) +{ + struct cb_layoutrecallargs args; + + /* Pretend we got a CB_LAYOUTRECALL(ALL) */ + memset(&args, 0, sizeof(args)); + args.cbl_recall_type = RETURN_ALL; + /* FIXME we ignore errors, what should we do? */ + do_callback_layoutrecall(clp, &args, cps); +} + +__be32 nfs4_callback_devicenotify(void *argp, void *resp, + struct cb_process_state *cps) +{ + struct cb_devicenotifyargs *args = argp; + const struct pnfs_layoutdriver_type *ld = NULL; + uint32_t i; + __be32 res = 0; + + if (!cps->clp) { + res = cpu_to_be32(NFS4ERR_OP_NOT_IN_SESSION); + goto out; + } + + for (i = 0; i < args->ndevs; i++) { + struct cb_devicenotifyitem *dev = &args->devs[i]; + + if (!ld || ld->id != dev->cbd_layout_type) { + pnfs_put_layoutdriver(ld); + ld = pnfs_find_layoutdriver(dev->cbd_layout_type); + if (!ld) + continue; + } + nfs4_delete_deviceid(ld, cps->clp, &dev->cbd_dev_id); + } + pnfs_put_layoutdriver(ld); +out: + kfree(args->devs); + return res; +} + +/* + * Validate the sequenceID sent by the server. + * Return success if the sequenceID is one more than what we last saw on + * this slot, accounting for wraparound. Increments the slot's sequence. + * + * We don't yet implement a duplicate request cache, instead we set the + * back channel ca_maxresponsesize_cached to zero. This is OK for now + * since we only currently implement idempotent callbacks anyway. + * + * We have a single slot backchannel at this time, so we don't bother + * checking the used_slots bit array on the table. The lower layer guarantees + * a single outstanding callback request at a time. + */ +static __be32 +validate_seqid(const struct nfs4_slot_table *tbl, const struct nfs4_slot *slot, + const struct cb_sequenceargs * args) +{ + __be32 ret; + + ret = cpu_to_be32(NFS4ERR_BADSLOT); + if (args->csa_slotid > tbl->server_highest_slotid) + goto out_err; + + /* Replay */ + if (args->csa_sequenceid == slot->seq_nr) { + ret = cpu_to_be32(NFS4ERR_DELAY); + if (nfs4_test_locked_slot(tbl, slot->slot_nr)) + goto out_err; + + /* Signal process_op to set this error on next op */ + ret = cpu_to_be32(NFS4ERR_RETRY_UNCACHED_REP); + if (args->csa_cachethis == 0) + goto out_err; + + /* Liar! We never allowed you to set csa_cachethis != 0 */ + ret = cpu_to_be32(NFS4ERR_SEQ_FALSE_RETRY); + goto out_err; + } + + /* Note: wraparound relies on seq_nr being of type u32 */ + /* Misordered request */ + ret = cpu_to_be32(NFS4ERR_SEQ_MISORDERED); + if (args->csa_sequenceid != slot->seq_nr + 1) + goto out_err; + + return cpu_to_be32(NFS4_OK); + +out_err: + trace_nfs4_cb_seqid_err(args, ret); + return ret; +} + +/* + * For each referring call triple, check the session's slot table for + * a match. If the slot is in use and the sequence numbers match, the + * client is still waiting for a response to the original request. + */ +static int referring_call_exists(struct nfs_client *clp, + uint32_t nrclists, + struct referring_call_list *rclists, + spinlock_t *lock) + __releases(lock) + __acquires(lock) +{ + int status = 0; + int found = 0; + int i, j; + struct nfs4_session *session; + struct nfs4_slot_table *tbl; + struct referring_call_list *rclist; + struct referring_call *ref; + + /* + * XXX When client trunking is implemented, this becomes + * a session lookup from within the loop + */ + session = clp->cl_session; + tbl = &session->fc_slot_table; + + for (i = 0; i < nrclists; i++) { + rclist = &rclists[i]; + if (memcmp(session->sess_id.data, + rclist->rcl_sessionid.data, + NFS4_MAX_SESSIONID_LEN) != 0) + continue; + + for (j = 0; j < rclist->rcl_nrefcalls; j++) { + ref = &rclist->rcl_refcalls[j]; + spin_unlock(lock); + status = nfs4_slot_wait_on_seqid(tbl, ref->rc_slotid, + ref->rc_sequenceid, HZ >> 1) < 0; + spin_lock(lock); + if (status) + goto out; + found++; + } + } + +out: + return status < 0 ? status : found; +} + +__be32 nfs4_callback_sequence(void *argp, void *resp, + struct cb_process_state *cps) +{ + struct cb_sequenceargs *args = argp; + struct cb_sequenceres *res = resp; + struct nfs4_slot_table *tbl; + struct nfs4_slot *slot; + struct nfs_client *clp; + int ret; + int i; + __be32 status = htonl(NFS4ERR_BADSESSION); + + clp = nfs4_find_client_sessionid(cps->net, args->csa_addr, + &args->csa_sessionid, cps->minorversion); + if (clp == NULL) + goto out; + + if (!(clp->cl_session->flags & SESSION4_BACK_CHAN)) + goto out; + + tbl = &clp->cl_session->bc_slot_table; + + /* Set up res before grabbing the spinlock */ + memcpy(&res->csr_sessionid, &args->csa_sessionid, + sizeof(res->csr_sessionid)); + res->csr_sequenceid = args->csa_sequenceid; + res->csr_slotid = args->csa_slotid; + + spin_lock(&tbl->slot_tbl_lock); + /* state manager is resetting the session */ + if (test_bit(NFS4_SLOT_TBL_DRAINING, &tbl->slot_tbl_state)) { + status = htonl(NFS4ERR_DELAY); + /* Return NFS4ERR_BADSESSION if we're draining the session + * in order to reset it. + */ + if (test_bit(NFS4CLNT_SESSION_RESET, &clp->cl_state)) + status = htonl(NFS4ERR_BADSESSION); + goto out_unlock; + } + + status = htonl(NFS4ERR_BADSLOT); + slot = nfs4_lookup_slot(tbl, args->csa_slotid); + if (IS_ERR(slot)) + goto out_unlock; + + res->csr_highestslotid = tbl->server_highest_slotid; + res->csr_target_highestslotid = tbl->target_highest_slotid; + + status = validate_seqid(tbl, slot, args); + if (status) + goto out_unlock; + if (!nfs4_try_to_lock_slot(tbl, slot)) { + status = htonl(NFS4ERR_DELAY); + goto out_unlock; + } + cps->slot = slot; + + /* The ca_maxresponsesize_cached is 0 with no DRC */ + if (args->csa_cachethis != 0) { + status = htonl(NFS4ERR_REP_TOO_BIG_TO_CACHE); + goto out_unlock; + } + + /* + * Check for pending referring calls. If a match is found, a + * related callback was received before the response to the original + * call. + */ + ret = referring_call_exists(clp, args->csa_nrclists, args->csa_rclists, + &tbl->slot_tbl_lock); + if (ret < 0) { + status = htonl(NFS4ERR_DELAY); + goto out_unlock; + } + cps->referring_calls = ret; + + /* + * RFC5661 20.9.3 + * If CB_SEQUENCE returns an error, then the state of the slot + * (sequence ID, cached reply) MUST NOT change. + */ + slot->seq_nr = args->csa_sequenceid; +out_unlock: + spin_unlock(&tbl->slot_tbl_lock); + +out: + cps->clp = clp; /* put in nfs4_callback_compound */ + for (i = 0; i < args->csa_nrclists; i++) + kfree(args->csa_rclists[i].rcl_refcalls); + kfree(args->csa_rclists); + + if (status == htonl(NFS4ERR_RETRY_UNCACHED_REP)) { + cps->drc_status = status; + status = 0; + } else + res->csr_status = status; + + trace_nfs4_cb_sequence(args, res, status); + return status; +} + +static bool +validate_bitmap_values(unsigned int mask) +{ + return (mask & ~RCA4_TYPE_MASK_ALL) == 0; +} + +__be32 nfs4_callback_recallany(void *argp, void *resp, + struct cb_process_state *cps) +{ + struct cb_recallanyargs *args = argp; + __be32 status; + fmode_t flags = 0; + bool schedule_manager = false; + + status = cpu_to_be32(NFS4ERR_OP_NOT_IN_SESSION); + if (!cps->clp) /* set in cb_sequence */ + goto out; + + dprintk_rcu("NFS: RECALL_ANY callback request from %s\n", + rpc_peeraddr2str(cps->clp->cl_rpcclient, RPC_DISPLAY_ADDR)); + + status = cpu_to_be32(NFS4ERR_INVAL); + if (!validate_bitmap_values(args->craa_type_mask)) + goto out; + + status = cpu_to_be32(NFS4_OK); + if (args->craa_type_mask & BIT(RCA4_TYPE_MASK_RDATA_DLG)) + flags = FMODE_READ; + if (args->craa_type_mask & BIT(RCA4_TYPE_MASK_WDATA_DLG)) + flags |= FMODE_WRITE; + if (flags) + nfs_expire_unused_delegation_types(cps->clp, flags); + + if (args->craa_type_mask & BIT(RCA4_TYPE_MASK_FILE_LAYOUT)) + pnfs_recall_all_layouts(cps->clp, cps); + + if (args->craa_type_mask & BIT(PNFS_FF_RCA4_TYPE_MASK_READ)) { + set_bit(NFS4CLNT_RECALL_ANY_LAYOUT_READ, &cps->clp->cl_state); + schedule_manager = true; + } + if (args->craa_type_mask & BIT(PNFS_FF_RCA4_TYPE_MASK_RW)) { + set_bit(NFS4CLNT_RECALL_ANY_LAYOUT_RW, &cps->clp->cl_state); + schedule_manager = true; + } + if (schedule_manager) + nfs4_schedule_state_manager(cps->clp); + +out: + dprintk("%s: exit with status = %d\n", __func__, ntohl(status)); + return status; +} + +/* Reduce the fore channel's max_slots to the target value */ +__be32 nfs4_callback_recallslot(void *argp, void *resp, + struct cb_process_state *cps) +{ + struct cb_recallslotargs *args = argp; + struct nfs4_slot_table *fc_tbl; + __be32 status; + + status = htonl(NFS4ERR_OP_NOT_IN_SESSION); + if (!cps->clp) /* set in cb_sequence */ + goto out; + + dprintk_rcu("NFS: CB_RECALL_SLOT request from %s target highest slotid %u\n", + rpc_peeraddr2str(cps->clp->cl_rpcclient, RPC_DISPLAY_ADDR), + args->crsa_target_highest_slotid); + + fc_tbl = &cps->clp->cl_session->fc_slot_table; + + status = htonl(NFS4_OK); + + nfs41_set_target_slotid(fc_tbl, args->crsa_target_highest_slotid); + nfs41_notify_server(cps->clp); +out: + dprintk("%s: exit with status = %d\n", __func__, ntohl(status)); + return status; +} + +__be32 nfs4_callback_notify_lock(void *argp, void *resp, + struct cb_process_state *cps) +{ + struct cb_notify_lock_args *args = argp; + + if (!cps->clp) /* set in cb_sequence */ + return htonl(NFS4ERR_OP_NOT_IN_SESSION); + + dprintk_rcu("NFS: CB_NOTIFY_LOCK request from %s\n", + rpc_peeraddr2str(cps->clp->cl_rpcclient, RPC_DISPLAY_ADDR)); + + /* Don't wake anybody if the string looked bogus */ + if (args->cbnl_valid) + __wake_up(&cps->clp->cl_lock_waitq, TASK_NORMAL, 0, args); + + return htonl(NFS4_OK); +} +#ifdef CONFIG_NFS_V4_2 +static void nfs4_copy_cb_args(struct nfs4_copy_state *cp_state, + struct cb_offloadargs *args) +{ + cp_state->count = args->wr_count; + cp_state->error = args->error; + if (!args->error) { + cp_state->verf.committed = args->wr_writeverf.committed; + memcpy(&cp_state->verf.verifier.data[0], + &args->wr_writeverf.verifier.data[0], + NFS4_VERIFIER_SIZE); + } +} + +__be32 nfs4_callback_offload(void *data, void *dummy, + struct cb_process_state *cps) +{ + struct cb_offloadargs *args = data; + struct nfs_server *server; + struct nfs4_copy_state *copy, *tmp_copy; + bool found = false; + + copy = kzalloc_obj(struct nfs4_copy_state); + if (!copy) + return cpu_to_be32(NFS4ERR_DELAY); + + spin_lock(&cps->clp->cl_lock); + rcu_read_lock(); + list_for_each_entry_rcu(server, &cps->clp->cl_superblocks, + client_link) { + list_for_each_entry(tmp_copy, &server->ss_copies, copies) { + if (memcmp(args->coa_stateid.other, + tmp_copy->stateid.other, + sizeof(args->coa_stateid.other))) + continue; + nfs4_copy_cb_args(tmp_copy, args); + complete(&tmp_copy->completion); + found = true; + goto out; + } + } +out: + rcu_read_unlock(); + if (!found) { + memcpy(©->stateid, &args->coa_stateid, NFS4_STATEID_SIZE); + nfs4_copy_cb_args(copy, args); + list_add_tail(©->copies, &cps->clp->pending_cb_stateids); + } else + kfree(copy); + spin_unlock(&cps->clp->cl_lock); + + trace_nfs4_cb_offload(&args->coa_fh, &args->coa_stateid, + args->wr_count, args->error, + args->wr_writeverf.committed); + return 0; +} +#endif /* CONFIG_NFS_V4_2 */ diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c new file mode 100644 index 000000000..eec604055 --- /dev/null +++ b/fs/nfs/callback_xdr.c @@ -0,0 +1,1111 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/nfs/callback_xdr.c + * + * Copyright (C) 2004 Trond Myklebust + * + * NFSv4 callback encode/decode procedures + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "nfs4_fs.h" +#include "callback.h" +#include "internal.h" +#include "nfs4session.h" +#include "nfs4trace.h" + +#define CB_OP_TAGLEN_MAXSZ (512) +#define CB_OP_HDR_RES_MAXSZ (2 * 4) // opcode, status +#define CB_OP_GETATTR_BITMAP_MAXSZ (4 * 4) // bitmap length, 3 bitmaps +#define CB_OP_GETATTR_RES_MAXSZ (CB_OP_HDR_RES_MAXSZ + \ + CB_OP_GETATTR_BITMAP_MAXSZ + \ + /* change, size, atime, ctime, + * mtime, deleg_atime, deleg_mtime */\ + (2 + 2 + 3 + 3 + 3 + 3 + 3) * 4) +#define CB_OP_RECALL_RES_MAXSZ (CB_OP_HDR_RES_MAXSZ) + +#define CB_OP_LAYOUTRECALL_RES_MAXSZ (CB_OP_HDR_RES_MAXSZ) +#define CB_OP_DEVICENOTIFY_RES_MAXSZ (CB_OP_HDR_RES_MAXSZ) +#define CB_OP_SEQUENCE_RES_MAXSZ (CB_OP_HDR_RES_MAXSZ + \ + NFS4_MAX_SESSIONID_LEN + \ + (1 + 3) * 4) // seqid, 3 slotids +#define CB_OP_RECALLANY_RES_MAXSZ (CB_OP_HDR_RES_MAXSZ) +#define CB_OP_RECALLSLOT_RES_MAXSZ (CB_OP_HDR_RES_MAXSZ) +#define CB_OP_NOTIFY_LOCK_RES_MAXSZ (CB_OP_HDR_RES_MAXSZ) +#ifdef CONFIG_NFS_V4_2 +#define CB_OP_OFFLOAD_RES_MAXSZ (CB_OP_HDR_RES_MAXSZ) +#endif /* CONFIG_NFS_V4_2 */ + +#define NFSDBG_FACILITY NFSDBG_CALLBACK + +/* Internal error code */ +#define NFS4ERR_RESOURCE_HDR 11050 + +struct callback_op { + __be32 (*process_op)(void *, void *, struct cb_process_state *); + __be32 (*decode_args)(struct svc_rqst *, struct xdr_stream *, void *); + __be32 (*encode_res)(struct svc_rqst *, struct xdr_stream *, + const void *); + long res_maxsize; +}; + +static struct callback_op callback_ops[]; + +static __be32 nfs4_callback_null(struct svc_rqst *rqstp) +{ + return htonl(NFS4_OK); +} + +/* + * svc_process_common() looks for an XDR encoder to know when + * not to drop a Reply. + */ +static bool nfs4_encode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + return true; +} + +static __be32 decode_string(struct xdr_stream *xdr, unsigned int *len, + const char **str, size_t maxlen) +{ + ssize_t err; + + err = xdr_stream_decode_opaque_inline(xdr, (void **)str, maxlen); + if (err < 0) + return cpu_to_be32(NFS4ERR_RESOURCE); + *len = err; + return 0; +} + +static __be32 decode_fh(struct xdr_stream *xdr, struct nfs_fh *fh) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(p == NULL)) + return htonl(NFS4ERR_RESOURCE); + fh->size = ntohl(*p); + if (fh->size > NFS4_FHSIZE) + return htonl(NFS4ERR_BADHANDLE); + p = xdr_inline_decode(xdr, fh->size); + if (unlikely(p == NULL)) + return htonl(NFS4ERR_RESOURCE); + memcpy_and_pad(fh->data, sizeof(fh->data), p, fh->size, 0); + return 0; +} + +static __be32 decode_bitmap(struct xdr_stream *xdr, uint32_t *bitmap) +{ + __be32 *p; + unsigned int attrlen; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(p == NULL)) + return htonl(NFS4ERR_RESOURCE); + attrlen = ntohl(*p); + p = xdr_inline_decode(xdr, attrlen << 2); + if (unlikely(p == NULL)) + return htonl(NFS4ERR_RESOURCE); + if (likely(attrlen > 0)) + bitmap[0] = ntohl(*p++); + if (attrlen > 1) + bitmap[1] = ntohl(*p++); + if (attrlen > 2) + bitmap[2] = ntohl(*p); + return 0; +} + +static __be32 decode_stateid(struct xdr_stream *xdr, nfs4_stateid *stateid) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, NFS4_STATEID_SIZE); + if (unlikely(p == NULL)) + return htonl(NFS4ERR_RESOURCE); + memcpy(stateid->data, p, NFS4_STATEID_SIZE); + return 0; +} + +static __be32 decode_delegation_stateid(struct xdr_stream *xdr, nfs4_stateid *stateid) +{ + stateid->type = NFS4_DELEGATION_STATEID_TYPE; + return decode_stateid(xdr, stateid); +} + +static __be32 decode_compound_hdr_arg(struct xdr_stream *xdr, struct cb_compound_hdr_arg *hdr) +{ + __be32 *p; + __be32 status; + + status = decode_string(xdr, &hdr->taglen, &hdr->tag, CB_OP_TAGLEN_MAXSZ); + if (unlikely(status != 0)) + return status; + p = xdr_inline_decode(xdr, 12); + if (unlikely(p == NULL)) + return htonl(NFS4ERR_RESOURCE); + hdr->minorversion = ntohl(*p++); + /* Check for minor version support */ + if (hdr->minorversion <= NFS4_MAX_MINOR_VERSION) { + hdr->cb_ident = ntohl(*p++); /* ignored by v4.1 and v4.2 */ + } else { + pr_warn_ratelimited("NFS: %s: NFSv4 server callback with " + "illegal minor version %u!\n", + __func__, hdr->minorversion); + return htonl(NFS4ERR_MINOR_VERS_MISMATCH); + } + hdr->nops = ntohl(*p); + return 0; +} + +static __be32 decode_op_hdr(struct xdr_stream *xdr, unsigned int *op) +{ + __be32 *p; + p = xdr_inline_decode(xdr, 4); + if (unlikely(p == NULL)) + return htonl(NFS4ERR_RESOURCE_HDR); + *op = ntohl(*p); + return 0; +} + +static __be32 decode_getattr_args(struct svc_rqst *rqstp, + struct xdr_stream *xdr, void *argp) +{ + struct cb_getattrargs *args = argp; + __be32 status; + + status = decode_fh(xdr, &args->fh); + if (unlikely(status != 0)) + return status; + return decode_bitmap(xdr, args->bitmap); +} + +static __be32 decode_recall_args(struct svc_rqst *rqstp, + struct xdr_stream *xdr, void *argp) +{ + struct cb_recallargs *args = argp; + __be32 *p; + __be32 status; + + status = decode_delegation_stateid(xdr, &args->stateid); + if (unlikely(status != 0)) + return status; + p = xdr_inline_decode(xdr, 4); + if (unlikely(p == NULL)) + return htonl(NFS4ERR_RESOURCE); + args->truncate = ntohl(*p); + return decode_fh(xdr, &args->fh); +} + +static __be32 decode_layout_stateid(struct xdr_stream *xdr, nfs4_stateid *stateid) +{ + stateid->type = NFS4_LAYOUT_STATEID_TYPE; + return decode_stateid(xdr, stateid); +} + +static __be32 decode_layoutrecall_args(struct svc_rqst *rqstp, + struct xdr_stream *xdr, void *argp) +{ + struct cb_layoutrecallargs *args = argp; + __be32 *p; + __be32 status = 0; + uint32_t iomode; + + p = xdr_inline_decode(xdr, 4 * sizeof(uint32_t)); + if (unlikely(p == NULL)) + return htonl(NFS4ERR_BADXDR); + + args->cbl_layout_type = ntohl(*p++); + /* Depite the spec's xdr, iomode really belongs in the FILE switch, + * as it is unusable and ignored with the other types. + */ + iomode = ntohl(*p++); + args->cbl_layoutchanged = ntohl(*p++); + args->cbl_recall_type = ntohl(*p++); + + if (args->cbl_recall_type == RETURN_FILE) { + args->cbl_range.iomode = iomode; + status = decode_fh(xdr, &args->cbl_fh); + if (unlikely(status != 0)) + return status; + + p = xdr_inline_decode(xdr, 2 * sizeof(uint64_t)); + if (unlikely(p == NULL)) + return htonl(NFS4ERR_BADXDR); + p = xdr_decode_hyper(p, &args->cbl_range.offset); + p = xdr_decode_hyper(p, &args->cbl_range.length); + return decode_layout_stateid(xdr, &args->cbl_stateid); + } else if (args->cbl_recall_type == RETURN_FSID) { + p = xdr_inline_decode(xdr, 2 * sizeof(uint64_t)); + if (unlikely(p == NULL)) + return htonl(NFS4ERR_BADXDR); + p = xdr_decode_hyper(p, &args->cbl_fsid.major); + p = xdr_decode_hyper(p, &args->cbl_fsid.minor); + } else if (args->cbl_recall_type != RETURN_ALL) + return htonl(NFS4ERR_BADXDR); + return 0; +} + +static +__be32 decode_devicenotify_args(struct svc_rqst *rqstp, + struct xdr_stream *xdr, + void *argp) +{ + struct cb_devicenotifyargs *args = argp; + uint32_t tmp, n, i; + __be32 *p; + __be32 status = 0; + + /* Num of device notifications */ + p = xdr_inline_decode(xdr, sizeof(uint32_t)); + if (unlikely(p == NULL)) { + status = htonl(NFS4ERR_BADXDR); + goto out; + } + n = ntohl(*p++); + if (n == 0) + goto out; + + args->devs = kmalloc_objs(*args->devs, n); + if (!args->devs) { + status = htonl(NFS4ERR_DELAY); + goto out; + } + + /* Decode each dev notification */ + for (i = 0; i < n; i++) { + struct cb_devicenotifyitem *dev = &args->devs[i]; + + p = xdr_inline_decode(xdr, (4 * sizeof(uint32_t)) + + NFS4_DEVICEID4_SIZE); + if (unlikely(p == NULL)) { + status = htonl(NFS4ERR_BADXDR); + goto err; + } + + tmp = ntohl(*p++); /* bitmap size */ + if (tmp != 1) { + status = htonl(NFS4ERR_INVAL); + goto err; + } + dev->cbd_notify_type = ntohl(*p++); + if (dev->cbd_notify_type != NOTIFY_DEVICEID4_CHANGE && + dev->cbd_notify_type != NOTIFY_DEVICEID4_DELETE) { + status = htonl(NFS4ERR_INVAL); + goto err; + } + + tmp = ntohl(*p++); /* opaque size */ + if (((dev->cbd_notify_type == NOTIFY_DEVICEID4_CHANGE) && + (tmp != NFS4_DEVICEID4_SIZE + 8)) || + ((dev->cbd_notify_type == NOTIFY_DEVICEID4_DELETE) && + (tmp != NFS4_DEVICEID4_SIZE + 4))) { + status = htonl(NFS4ERR_INVAL); + goto err; + } + dev->cbd_layout_type = ntohl(*p++); + memcpy(dev->cbd_dev_id.data, p, NFS4_DEVICEID4_SIZE); + p += XDR_QUADLEN(NFS4_DEVICEID4_SIZE); + + if (dev->cbd_layout_type == NOTIFY_DEVICEID4_CHANGE) { + p = xdr_inline_decode(xdr, sizeof(uint32_t)); + if (unlikely(p == NULL)) { + status = htonl(NFS4ERR_BADXDR); + goto err; + } + dev->cbd_immediate = ntohl(*p++); + } else { + dev->cbd_immediate = 0; + } + + dprintk("%s: type %d layout 0x%x immediate %d\n", + __func__, dev->cbd_notify_type, dev->cbd_layout_type, + dev->cbd_immediate); + } + args->ndevs = n; + dprintk("%s: ndevs %d\n", __func__, args->ndevs); + return 0; +err: + kfree(args->devs); +out: + args->devs = NULL; + args->ndevs = 0; + dprintk("%s: status %d ndevs %d\n", + __func__, ntohl(status), args->ndevs); + return status; +} + +static __be32 decode_sessionid(struct xdr_stream *xdr, + struct nfs4_sessionid *sid) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, NFS4_MAX_SESSIONID_LEN); + if (unlikely(p == NULL)) + return htonl(NFS4ERR_RESOURCE); + + memcpy(sid->data, p, NFS4_MAX_SESSIONID_LEN); + return 0; +} + +static __be32 decode_rc_list(struct xdr_stream *xdr, + struct referring_call_list *rc_list) +{ + __be32 *p; + int i; + __be32 status; + + status = decode_sessionid(xdr, &rc_list->rcl_sessionid); + if (status) + goto out; + + status = htonl(NFS4ERR_RESOURCE); + p = xdr_inline_decode(xdr, sizeof(uint32_t)); + if (unlikely(p == NULL)) + goto out; + + rc_list->rcl_nrefcalls = ntohl(*p++); + if (rc_list->rcl_nrefcalls) { + if (unlikely(rc_list->rcl_nrefcalls > xdr->buf->len)) + goto out; + p = xdr_inline_decode(xdr, + rc_list->rcl_nrefcalls * 2 * sizeof(uint32_t)); + if (unlikely(p == NULL)) + goto out; + rc_list->rcl_refcalls = kmalloc_objs(*rc_list->rcl_refcalls, + rc_list->rcl_nrefcalls); + if (unlikely(rc_list->rcl_refcalls == NULL)) + goto out; + for (i = 0; i < rc_list->rcl_nrefcalls; i++) { + rc_list->rcl_refcalls[i].rc_sequenceid = ntohl(*p++); + rc_list->rcl_refcalls[i].rc_slotid = ntohl(*p++); + } + } + status = 0; + +out: + return status; +} + +static __be32 decode_cb_sequence_args(struct svc_rqst *rqstp, + struct xdr_stream *xdr, + void *argp) +{ + struct cb_sequenceargs *args = argp; + __be32 *p; + int i; + __be32 status; + + status = decode_sessionid(xdr, &args->csa_sessionid); + if (status) + return status; + + p = xdr_inline_decode(xdr, 5 * sizeof(uint32_t)); + if (unlikely(p == NULL)) + return htonl(NFS4ERR_RESOURCE); + + args->csa_addr = svc_addr(rqstp); + args->csa_sequenceid = ntohl(*p++); + args->csa_slotid = ntohl(*p++); + args->csa_highestslotid = ntohl(*p++); + args->csa_cachethis = ntohl(*p++); + args->csa_nrclists = ntohl(*p++); + args->csa_rclists = NULL; + if (args->csa_nrclists) { + args->csa_rclists = kmalloc_objs(*args->csa_rclists, + args->csa_nrclists); + if (unlikely(args->csa_rclists == NULL)) + return htonl(NFS4ERR_RESOURCE); + + for (i = 0; i < args->csa_nrclists; i++) { + status = decode_rc_list(xdr, &args->csa_rclists[i]); + if (status) { + args->csa_nrclists = i; + goto out_free; + } + } + } + return 0; + +out_free: + for (i = 0; i < args->csa_nrclists; i++) + kfree(args->csa_rclists[i].rcl_refcalls); + kfree(args->csa_rclists); + return status; +} + +static __be32 decode_recallany_args(struct svc_rqst *rqstp, + struct xdr_stream *xdr, + void *argp) +{ + struct cb_recallanyargs *args = argp; + uint32_t bitmap[3]; + __be32 *p, status; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(p == NULL)) + return htonl(NFS4ERR_BADXDR); + args->craa_objs_to_keep = ntohl(*p++); + status = decode_bitmap(xdr, bitmap); + if (unlikely(status)) + return status; + args->craa_type_mask = bitmap[0]; + + return 0; +} + +static __be32 decode_recallslot_args(struct svc_rqst *rqstp, + struct xdr_stream *xdr, + void *argp) +{ + struct cb_recallslotargs *args = argp; + __be32 *p; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(p == NULL)) + return htonl(NFS4ERR_BADXDR); + args->crsa_target_highest_slotid = ntohl(*p++); + return 0; +} + +static __be32 decode_lockowner(struct xdr_stream *xdr, struct cb_notify_lock_args *args) +{ + __be32 *p; + unsigned int len; + + p = xdr_inline_decode(xdr, 12); + if (unlikely(p == NULL)) + return htonl(NFS4ERR_BADXDR); + + p = xdr_decode_hyper(p, &args->cbnl_owner.clientid); + len = be32_to_cpu(*p); + + p = xdr_inline_decode(xdr, len); + if (unlikely(p == NULL)) + return htonl(NFS4ERR_BADXDR); + + /* Only try to decode if the length is right */ + if (len == 20) { + p += 2; /* skip "lock id:" */ + args->cbnl_owner.s_dev = be32_to_cpu(*p++); + xdr_decode_hyper(p, &args->cbnl_owner.id); + args->cbnl_valid = true; + } else { + args->cbnl_owner.s_dev = 0; + args->cbnl_owner.id = 0; + args->cbnl_valid = false; + } + return 0; +} + +static __be32 decode_notify_lock_args(struct svc_rqst *rqstp, + struct xdr_stream *xdr, void *argp) +{ + struct cb_notify_lock_args *args = argp; + __be32 status; + + status = decode_fh(xdr, &args->cbnl_fh); + if (unlikely(status != 0)) + return status; + return decode_lockowner(xdr, args); +} + +#ifdef CONFIG_NFS_V4_2 +static __be32 decode_write_response(struct xdr_stream *xdr, + struct cb_offloadargs *args) +{ + __be32 *p; + + /* skip the always zero field */ + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + goto out; + p++; + + /* decode count, stable_how, verifier */ + p = xdr_inline_decode(xdr, 8 + 4); + if (unlikely(!p)) + goto out; + p = xdr_decode_hyper(p, &args->wr_count); + args->wr_writeverf.committed = be32_to_cpup(p); + p = xdr_inline_decode(xdr, NFS4_VERIFIER_SIZE); + if (likely(p)) { + memcpy(&args->wr_writeverf.verifier.data[0], p, + NFS4_VERIFIER_SIZE); + return 0; + } +out: + return htonl(NFS4ERR_RESOURCE); +} + +static __be32 decode_offload_args(struct svc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct cb_offloadargs *args = data; + __be32 *p; + __be32 status; + + /* decode fh */ + status = decode_fh(xdr, &args->coa_fh); + if (unlikely(status != 0)) + return status; + + /* decode stateid */ + status = decode_stateid(xdr, &args->coa_stateid); + if (unlikely(status != 0)) + return status; + + /* decode status */ + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + goto out; + args->error = ntohl(*p++); + if (!args->error) { + status = decode_write_response(xdr, args); + if (unlikely(status != 0)) + return status; + } else { + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + goto out; + p = xdr_decode_hyper(p, &args->wr_count); + } + return 0; +out: + return htonl(NFS4ERR_RESOURCE); +} +#endif /* CONFIG_NFS_V4_2 */ +static __be32 encode_string(struct xdr_stream *xdr, unsigned int len, const char *str) +{ + if (unlikely(xdr_stream_encode_opaque(xdr, str, len) < 0)) + return cpu_to_be32(NFS4ERR_RESOURCE); + return 0; +} + +static __be32 encode_attr_bitmap(struct xdr_stream *xdr, const uint32_t *bitmap, size_t sz) +{ + if (xdr_stream_encode_uint32_array(xdr, bitmap, sz) < 0) + return cpu_to_be32(NFS4ERR_RESOURCE); + return 0; +} + +static __be32 encode_attr_change(struct xdr_stream *xdr, const uint32_t *bitmap, uint64_t change) +{ + __be32 *p; + + if (!(bitmap[0] & FATTR4_WORD0_CHANGE)) + return 0; + p = xdr_reserve_space(xdr, 8); + if (unlikely(!p)) + return htonl(NFS4ERR_RESOURCE); + p = xdr_encode_hyper(p, change); + return 0; +} + +static __be32 encode_attr_size(struct xdr_stream *xdr, const uint32_t *bitmap, uint64_t size) +{ + __be32 *p; + + if (!(bitmap[0] & FATTR4_WORD0_SIZE)) + return 0; + p = xdr_reserve_space(xdr, 8); + if (unlikely(!p)) + return htonl(NFS4ERR_RESOURCE); + p = xdr_encode_hyper(p, size); + return 0; +} + +static __be32 encode_attr_time(struct xdr_stream *xdr, const struct timespec64 *time) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, 12); + if (unlikely(!p)) + return htonl(NFS4ERR_RESOURCE); + p = xdr_encode_hyper(p, time->tv_sec); + *p = htonl(time->tv_nsec); + return 0; +} + +static __be32 encode_attr_atime(struct xdr_stream *xdr, const uint32_t *bitmap, const struct timespec64 *time) +{ + if (!(bitmap[1] & FATTR4_WORD1_TIME_ACCESS)) + return 0; + return encode_attr_time(xdr,time); +} + +static __be32 encode_attr_ctime(struct xdr_stream *xdr, const uint32_t *bitmap, const struct timespec64 *time) +{ + if (!(bitmap[1] & FATTR4_WORD1_TIME_METADATA)) + return 0; + return encode_attr_time(xdr,time); +} + +static __be32 encode_attr_mtime(struct xdr_stream *xdr, const uint32_t *bitmap, const struct timespec64 *time) +{ + if (!(bitmap[1] & FATTR4_WORD1_TIME_MODIFY)) + return 0; + return encode_attr_time(xdr,time); +} + +static __be32 encode_attr_delegatime(struct xdr_stream *xdr, + const uint32_t *bitmap, + const struct timespec64 *time) +{ + if (!(bitmap[2] & FATTR4_WORD2_TIME_DELEG_ACCESS)) + return 0; + return encode_attr_time(xdr,time); +} + +static __be32 encode_attr_delegmtime(struct xdr_stream *xdr, + const uint32_t *bitmap, + const struct timespec64 *time) +{ + if (!(bitmap[2] & FATTR4_WORD2_TIME_DELEG_MODIFY)) + return 0; + return encode_attr_time(xdr,time); +} + +static __be32 encode_compound_hdr_res(struct xdr_stream *xdr, struct cb_compound_hdr_res *hdr) +{ + __be32 status; + + hdr->status = xdr_reserve_space(xdr, 4); + if (unlikely(hdr->status == NULL)) + return htonl(NFS4ERR_RESOURCE); + status = encode_string(xdr, hdr->taglen, hdr->tag); + if (unlikely(status != 0)) + return status; + hdr->nops = xdr_reserve_space(xdr, 4); + if (unlikely(hdr->nops == NULL)) + return htonl(NFS4ERR_RESOURCE); + return 0; +} + +static __be32 encode_op_hdr(struct xdr_stream *xdr, uint32_t op, __be32 res) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, 8); + if (unlikely(p == NULL)) + return htonl(NFS4ERR_RESOURCE_HDR); + *p++ = htonl(op); + *p = res; + return 0; +} + +static __be32 encode_getattr_res(struct svc_rqst *rqstp, struct xdr_stream *xdr, + const void *resp) +{ + const struct cb_getattrres *res = resp; + __be32 *savep = NULL; + __be32 status = res->status; + + if (unlikely(status != 0)) + goto out; + status = encode_attr_bitmap(xdr, res->bitmap, ARRAY_SIZE(res->bitmap)); + if (unlikely(status != 0)) + goto out; + status = cpu_to_be32(NFS4ERR_RESOURCE); + savep = xdr_reserve_space(xdr, sizeof(*savep)); + if (unlikely(!savep)) + goto out; + status = encode_attr_change(xdr, res->bitmap, res->change_attr); + if (unlikely(status != 0)) + goto out; + status = encode_attr_size(xdr, res->bitmap, res->size); + if (unlikely(status != 0)) + goto out; + status = encode_attr_atime(xdr, res->bitmap, &res->atime); + if (unlikely(status != 0)) + goto out; + status = encode_attr_ctime(xdr, res->bitmap, &res->ctime); + if (unlikely(status != 0)) + goto out; + status = encode_attr_mtime(xdr, res->bitmap, &res->mtime); + if (unlikely(status != 0)) + goto out; + status = encode_attr_delegatime(xdr, res->bitmap, &res->atime); + if (unlikely(status != 0)) + goto out; + status = encode_attr_delegmtime(xdr, res->bitmap, &res->mtime); + *savep = htonl((unsigned int)((char *)xdr->p - (char *)(savep+1))); +out: + return status; +} + +static __be32 encode_sessionid(struct xdr_stream *xdr, + const struct nfs4_sessionid *sid) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, NFS4_MAX_SESSIONID_LEN); + if (unlikely(p == NULL)) + return htonl(NFS4ERR_RESOURCE); + + memcpy(p, sid, NFS4_MAX_SESSIONID_LEN); + return 0; +} + +static __be32 encode_cb_sequence_res(struct svc_rqst *rqstp, + struct xdr_stream *xdr, + const void *resp) +{ + const struct cb_sequenceres *res = resp; + __be32 *p; + __be32 status = res->csr_status; + + if (unlikely(status != 0)) + return status; + + status = encode_sessionid(xdr, &res->csr_sessionid); + if (status) + return status; + + p = xdr_reserve_space(xdr, 4 * sizeof(uint32_t)); + if (unlikely(p == NULL)) + return htonl(NFS4ERR_RESOURCE); + + *p++ = htonl(res->csr_sequenceid); + *p++ = htonl(res->csr_slotid); + *p++ = htonl(res->csr_highestslotid); + *p++ = htonl(res->csr_target_highestslotid); + return 0; +} + +static __be32 +preprocess_nfs41_op(int nop, unsigned int op_nr, struct callback_op **op) +{ + if (op_nr == OP_CB_SEQUENCE) { + if (nop != 0) + return htonl(NFS4ERR_SEQUENCE_POS); + } else { + if (nop == 0) + return htonl(NFS4ERR_OP_NOT_IN_SESSION); + } + + switch (op_nr) { + case OP_CB_GETATTR: + case OP_CB_RECALL: + case OP_CB_SEQUENCE: + case OP_CB_RECALL_ANY: + case OP_CB_RECALL_SLOT: + case OP_CB_LAYOUTRECALL: + case OP_CB_NOTIFY_DEVICEID: + case OP_CB_NOTIFY_LOCK: + *op = &callback_ops[op_nr]; + break; + + case OP_CB_NOTIFY: + case OP_CB_PUSH_DELEG: + case OP_CB_RECALLABLE_OBJ_AVAIL: + case OP_CB_WANTS_CANCELLED: + return htonl(NFS4ERR_NOTSUPP); + + default: + return htonl(NFS4ERR_OP_ILLEGAL); + } + + return htonl(NFS_OK); +} + +static void nfs4_callback_free_slot(struct nfs4_session *session, + struct nfs4_slot *slot) +{ + struct nfs4_slot_table *tbl = &session->bc_slot_table; + + spin_lock(&tbl->slot_tbl_lock); + /* + * Let the state manager know callback processing done. + * A single slot, so highest used slotid is either 0 or -1 + */ + nfs4_free_slot(tbl, slot); + spin_unlock(&tbl->slot_tbl_lock); +} + +static void nfs4_cb_free_slot(struct cb_process_state *cps) +{ + if (cps->slot) { + nfs4_callback_free_slot(cps->clp->cl_session, cps->slot); + cps->slot = NULL; + } +} + +#ifdef CONFIG_NFS_V4_2 +static __be32 +preprocess_nfs42_op(int nop, unsigned int op_nr, struct callback_op **op) +{ + __be32 status = preprocess_nfs41_op(nop, op_nr, op); + if (status != htonl(NFS4ERR_OP_ILLEGAL)) + return status; + + if (op_nr == OP_CB_OFFLOAD) { + *op = &callback_ops[op_nr]; + return htonl(NFS_OK); + } else + return htonl(NFS4ERR_NOTSUPP); + return htonl(NFS4ERR_OP_ILLEGAL); +} +#else /* CONFIG_NFS_V4_2 */ +static __be32 +preprocess_nfs42_op(int nop, unsigned int op_nr, struct callback_op **op) +{ + return htonl(NFS4ERR_MINOR_VERS_MISMATCH); +} +#endif /* CONFIG_NFS_V4_2 */ + +static __be32 +preprocess_nfs4_op(unsigned int op_nr, struct callback_op **op) +{ + switch (op_nr) { + case OP_CB_GETATTR: + case OP_CB_RECALL: + *op = &callback_ops[op_nr]; + break; + default: + return htonl(NFS4ERR_OP_ILLEGAL); + } + + return htonl(NFS_OK); +} + +static __be32 process_op(int nop, struct svc_rqst *rqstp, + struct cb_process_state *cps) +{ + struct xdr_stream *xdr_out = &rqstp->rq_res_stream; + struct callback_op *op = &callback_ops[0]; + unsigned int op_nr; + __be32 status; + long maxlen; + __be32 res; + + status = decode_op_hdr(&rqstp->rq_arg_stream, &op_nr); + if (unlikely(status)) + return status; + + switch (cps->minorversion) { + case 0: + status = preprocess_nfs4_op(op_nr, &op); + break; + case 1: + status = preprocess_nfs41_op(nop, op_nr, &op); + break; + case 2: + status = preprocess_nfs42_op(nop, op_nr, &op); + break; + default: + status = htonl(NFS4ERR_MINOR_VERS_MISMATCH); + } + + if (status == htonl(NFS4ERR_OP_ILLEGAL)) + op_nr = OP_CB_ILLEGAL; + if (status) + goto encode_hdr; + + if (cps->drc_status) { + status = cps->drc_status; + goto encode_hdr; + } + + maxlen = xdr_out->end - xdr_out->p; + if (maxlen > 0 && maxlen < PAGE_SIZE) { + status = op->decode_args(rqstp, &rqstp->rq_arg_stream, + rqstp->rq_argp); + if (likely(status == 0)) + status = op->process_op(rqstp->rq_argp, rqstp->rq_resp, + cps); + } else + status = htonl(NFS4ERR_RESOURCE); + +encode_hdr: + res = encode_op_hdr(xdr_out, op_nr, status); + if (unlikely(res)) + return res; + if (op->encode_res != NULL && status == 0) + status = op->encode_res(rqstp, xdr_out, rqstp->rq_resp); + return status; +} + +/* + * Decode, process and encode a COMPOUND + */ +static __be32 nfs4_callback_compound(struct svc_rqst *rqstp) +{ + struct cb_compound_hdr_arg hdr_arg = { 0 }; + struct cb_compound_hdr_res hdr_res = { NULL }; + struct cb_process_state cps = { + .drc_status = 0, + .clp = NULL, + .net = SVC_NET(rqstp), + }; + unsigned int nops = 0; + __be32 status; + + status = decode_compound_hdr_arg(&rqstp->rq_arg_stream, &hdr_arg); + if (status == htonl(NFS4ERR_RESOURCE)) + return rpc_garbage_args; + + if (hdr_arg.minorversion == 0) { + cps.clp = nfs4_find_client_ident(SVC_NET(rqstp), hdr_arg.cb_ident); + if (!cps.clp) { + trace_nfs_cb_no_clp(rqstp->rq_xid, hdr_arg.cb_ident); + goto out_invalidcred; + } + if (!check_gss_callback_principal(cps.clp, rqstp)) { + trace_nfs_cb_badprinc(rqstp->rq_xid, hdr_arg.cb_ident); + nfs_put_client(cps.clp); + goto out_invalidcred; + } + svc_xprt_set_valid(rqstp->rq_xprt); + } + + cps.minorversion = hdr_arg.minorversion; + hdr_res.taglen = hdr_arg.taglen; + hdr_res.tag = hdr_arg.tag; + if (encode_compound_hdr_res(&rqstp->rq_res_stream, &hdr_res) != 0) { + if (cps.clp) + nfs_put_client(cps.clp); + return rpc_system_err; + } + while (status == 0 && nops != hdr_arg.nops) { + status = process_op(nops, rqstp, &cps); + nops++; + } + + /* Buffer overflow in decode_ops_hdr or encode_ops_hdr. Return + * resource error in cb_compound status without returning op */ + if (unlikely(status == htonl(NFS4ERR_RESOURCE_HDR))) { + status = htonl(NFS4ERR_RESOURCE); + nops--; + } + + if (svc_is_backchannel(rqstp) && cps.clp) { + rqstp->bc_to_initval = cps.clp->cl_rpcclient->cl_timeout->to_initval; + rqstp->bc_to_retries = cps.clp->cl_rpcclient->cl_timeout->to_retries; + } + + *hdr_res.status = status; + *hdr_res.nops = htonl(nops); + nfs4_cb_free_slot(&cps); + nfs_put_client(cps.clp); + return rpc_success; + +out_invalidcred: + pr_warn_ratelimited("NFS: NFSv4 callback contains invalid cred\n"); + rqstp->rq_auth_stat = rpc_autherr_badcred; + return rpc_success; +} + +static int +nfs_callback_dispatch(struct svc_rqst *rqstp) +{ + const struct svc_procedure *procp = rqstp->rq_procinfo; + + *rqstp->rq_accept_statp = procp->pc_func(rqstp); + return 1; +} + +/* + * Define NFS4 callback COMPOUND ops. + */ +static struct callback_op callback_ops[] = { + [0] = { + .res_maxsize = CB_OP_HDR_RES_MAXSZ, + }, + [OP_CB_GETATTR] = { + .process_op = nfs4_callback_getattr, + .decode_args = decode_getattr_args, + .encode_res = encode_getattr_res, + .res_maxsize = CB_OP_GETATTR_RES_MAXSZ, + }, + [OP_CB_RECALL] = { + .process_op = nfs4_callback_recall, + .decode_args = decode_recall_args, + .res_maxsize = CB_OP_RECALL_RES_MAXSZ, + }, + [OP_CB_LAYOUTRECALL] = { + .process_op = nfs4_callback_layoutrecall, + .decode_args = decode_layoutrecall_args, + .res_maxsize = CB_OP_LAYOUTRECALL_RES_MAXSZ, + }, + [OP_CB_NOTIFY_DEVICEID] = { + .process_op = nfs4_callback_devicenotify, + .decode_args = decode_devicenotify_args, + .res_maxsize = CB_OP_DEVICENOTIFY_RES_MAXSZ, + }, + [OP_CB_SEQUENCE] = { + .process_op = nfs4_callback_sequence, + .decode_args = decode_cb_sequence_args, + .encode_res = encode_cb_sequence_res, + .res_maxsize = CB_OP_SEQUENCE_RES_MAXSZ, + }, + [OP_CB_RECALL_ANY] = { + .process_op = nfs4_callback_recallany, + .decode_args = decode_recallany_args, + .res_maxsize = CB_OP_RECALLANY_RES_MAXSZ, + }, + [OP_CB_RECALL_SLOT] = { + .process_op = nfs4_callback_recallslot, + .decode_args = decode_recallslot_args, + .res_maxsize = CB_OP_RECALLSLOT_RES_MAXSZ, + }, + [OP_CB_NOTIFY_LOCK] = { + .process_op = nfs4_callback_notify_lock, + .decode_args = decode_notify_lock_args, + .res_maxsize = CB_OP_NOTIFY_LOCK_RES_MAXSZ, + }, +#ifdef CONFIG_NFS_V4_2 + [OP_CB_OFFLOAD] = { + .process_op = nfs4_callback_offload, + .decode_args = decode_offload_args, + .res_maxsize = CB_OP_OFFLOAD_RES_MAXSZ, + }, +#endif /* CONFIG_NFS_V4_2 */ +}; + +/* + * Define NFS4 callback procedures + */ +static const struct svc_procedure nfs4_callback_procedures1[] = { + [CB_NULL] = { + .pc_func = nfs4_callback_null, + .pc_encode = nfs4_encode_void, + .pc_xdrressize = 1, + .pc_name = "NULL", + }, + [CB_COMPOUND] = { + .pc_func = nfs4_callback_compound, + .pc_encode = nfs4_encode_void, + .pc_argsize = 256, + .pc_argzero = 256, + .pc_ressize = 256, + .pc_xdrressize = NFS4_CALLBACK_BUFSIZE, + .pc_name = "COMPOUND", + } +}; + +const struct svc_version nfs4_callback_version1 = { + .vs_vers = 1, + .vs_nproc = ARRAY_SIZE(nfs4_callback_procedures1), + .vs_proc = nfs4_callback_procedures1, + .vs_xdrsize = NFS4_CALLBACK_XDRSIZE, + .vs_dispatch = nfs_callback_dispatch, + .vs_hidden = true, + .vs_need_cong_ctrl = true, +}; + +const struct svc_version nfs4_callback_version4 = { + .vs_vers = 4, + .vs_nproc = ARRAY_SIZE(nfs4_callback_procedures1), + .vs_proc = nfs4_callback_procedures1, + .vs_xdrsize = NFS4_CALLBACK_XDRSIZE, + .vs_dispatch = nfs_callback_dispatch, + .vs_hidden = true, + .vs_need_cong_ctrl = true, +}; diff --git a/fs/nfs/client.c b/fs/nfs/client.c new file mode 100644 index 000000000..4dcb91ab3 --- /dev/null +++ b/fs/nfs/client.c @@ -0,0 +1,1542 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* client.c: NFS client sharing and management code + * + * Copyright (C) 2006 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nfs4_fs.h" +#include "callback.h" +#include "delegation.h" +#include "iostat.h" +#include "internal.h" +#include "fscache.h" +#include "pnfs.h" +#include "nfs.h" +#include "netns.h" +#include "sysfs.h" +#include "nfs42.h" + +#define NFSDBG_FACILITY NFSDBG_CLIENT + +static DECLARE_WAIT_QUEUE_HEAD(nfs_client_active_wq); +static DEFINE_RWLOCK(nfs_version_lock); + +static struct nfs_subversion *nfs_version_mods[5] = { + [2] = NULL, + [3] = NULL, + [4] = NULL, +}; + +/* + * RPC cruft for NFS + */ +static const struct rpc_version *nfs_version[5] = { + [2] = NULL, + [3] = NULL, + [4] = NULL, +}; + +const struct rpc_program nfs_program = { + .name = "nfs", + .number = NFS_PROGRAM, + .nrvers = ARRAY_SIZE(nfs_version), + .version = nfs_version, + .pipe_dir_name = NFS_PIPE_DIRNAME, +}; + +static struct nfs_subversion *__find_nfs_version(unsigned int version) +{ + struct nfs_subversion *nfs; + + read_lock(&nfs_version_lock); + nfs = nfs_version_mods[version]; + read_unlock(&nfs_version_lock); + return nfs; +} + +struct nfs_subversion *find_nfs_version(unsigned int version) +{ + struct nfs_subversion *nfs = __find_nfs_version(version); + + if (!nfs && request_module("nfsv%d", version) == 0) + nfs = __find_nfs_version(version); + + if (!nfs) + return ERR_PTR(-EPROTONOSUPPORT); + + if (!get_nfs_version(nfs)) + return ERR_PTR(-EAGAIN); + + return nfs; +} + +int get_nfs_version(struct nfs_subversion *nfs) +{ + return try_module_get(nfs->owner); +} +EXPORT_SYMBOL_GPL(get_nfs_version); + +void put_nfs_version(struct nfs_subversion *nfs) +{ + module_put(nfs->owner); +} + +void register_nfs_version(struct nfs_subversion *nfs) +{ + write_lock(&nfs_version_lock); + + nfs_version_mods[nfs->rpc_ops->version] = nfs; + nfs_version[nfs->rpc_ops->version] = nfs->rpc_vers; + + write_unlock(&nfs_version_lock); +} +EXPORT_SYMBOL_GPL(register_nfs_version); + +void unregister_nfs_version(struct nfs_subversion *nfs) +{ + write_lock(&nfs_version_lock); + + nfs_version[nfs->rpc_ops->version] = NULL; + nfs_version_mods[nfs->rpc_ops->version] = NULL; + + write_unlock(&nfs_version_lock); +} +EXPORT_SYMBOL_GPL(unregister_nfs_version); + +/* + * Allocate a shared client record + * + * Since these are allocated/deallocated very rarely, we don't + * bother putting them in a slab cache... + */ +struct nfs_client *nfs_alloc_client(const struct nfs_client_initdata *cl_init) +{ + struct nfs_client *clp; + int err = -ENOMEM; + + if ((clp = kzalloc_obj(*clp)) == NULL) + goto error_0; + + clp->cl_minorversion = cl_init->minorversion; + clp->cl_nfs_mod = cl_init->nfs_mod; + if (!get_nfs_version(clp->cl_nfs_mod)) + goto error_dealloc; + + clp->rpc_ops = clp->cl_nfs_mod->rpc_ops; + + refcount_set(&clp->cl_count, 1); + clp->cl_cons_state = NFS_CS_INITING; + + memcpy(&clp->cl_addr, cl_init->addr, cl_init->addrlen); + clp->cl_addrlen = cl_init->addrlen; + + if (cl_init->hostname) { + err = -ENOMEM; + clp->cl_hostname = kstrdup(cl_init->hostname, GFP_KERNEL); + if (!clp->cl_hostname) + goto error_cleanup; + } + + INIT_LIST_HEAD(&clp->cl_superblocks); + clp->cl_rpcclient = ERR_PTR(-EINVAL); + + clp->cl_flags = cl_init->init_flags; + clp->cl_proto = cl_init->proto; + clp->cl_nconnect = cl_init->nconnect; + clp->cl_max_connect = cl_init->max_connect ? cl_init->max_connect : 1; + clp->cl_net = get_net_track(cl_init->net, &clp->cl_ns_tracker, GFP_KERNEL); + +#if IS_ENABLED(CONFIG_NFS_LOCALIO) + seqlock_init(&clp->cl_boot_lock); + ktime_get_real_ts64(&clp->cl_nfssvc_boot); + nfs_uuid_init(&clp->cl_uuid); + INIT_WORK(&clp->cl_local_probe_work, nfs_local_probe_async_work); +#endif /* CONFIG_NFS_LOCALIO */ + + clp->cl_principal = "*"; + clp->cl_xprtsec = cl_init->xprtsec; + return clp; + +error_cleanup: + put_nfs_version(clp->cl_nfs_mod); +error_dealloc: + kfree(clp); +error_0: + return ERR_PTR(err); +} +EXPORT_SYMBOL_GPL(nfs_alloc_client); + +#if IS_ENABLED(CONFIG_NFS_V4) +static void nfs_cleanup_cb_ident_idr(struct net *net) +{ + struct nfs_net *nn = net_generic(net, nfs_net_id); + + idr_destroy(&nn->cb_ident_idr); +} + +/* nfs_client_lock held */ +static void nfs_cb_idr_remove_locked(struct nfs_client *clp) +{ + struct nfs_net *nn = net_generic(clp->cl_net, nfs_net_id); + + if (clp->cl_cb_ident) + idr_remove(&nn->cb_ident_idr, clp->cl_cb_ident); +} + +static void pnfs_init_server(struct nfs_server *server) +{ + rpc_init_wait_queue(&server->roc_rpcwaitq, "pNFS ROC"); +} + +#else +static void nfs_cleanup_cb_ident_idr(struct net *net) +{ +} + +static void nfs_cb_idr_remove_locked(struct nfs_client *clp) +{ +} + +static void pnfs_init_server(struct nfs_server *server) +{ +} + +#endif /* CONFIG_NFS_V4 */ + +/* + * Destroy a shared client record + */ +void nfs_free_client(struct nfs_client *clp) +{ + nfs_localio_disable_client(clp); + + /* -EIO all pending I/O */ + if (!IS_ERR(clp->cl_rpcclient)) + rpc_shutdown_client(clp->cl_rpcclient); + + put_net_track(clp->cl_net, &clp->cl_ns_tracker); + put_nfs_version(clp->cl_nfs_mod); + kfree(clp->cl_hostname); + kfree(clp->cl_acceptor); + kfree_rcu(clp, rcu); +} +EXPORT_SYMBOL_GPL(nfs_free_client); + +/* + * Release a reference to a shared client record + */ +void nfs_put_client(struct nfs_client *clp) +{ + struct nfs_net *nn; + + if (!clp) + return; + + nn = net_generic(clp->cl_net, nfs_net_id); + + if (refcount_dec_and_lock(&clp->cl_count, &nn->nfs_client_lock)) { + list_del(&clp->cl_share_link); + nfs_cb_idr_remove_locked(clp); + spin_unlock(&nn->nfs_client_lock); + + WARN_ON_ONCE(!list_empty(&clp->cl_superblocks)); + + clp->rpc_ops->free_client(clp); + } +} +EXPORT_SYMBOL_GPL(nfs_put_client); + +/* + * Find an nfs_client on the list that matches the initialisation data + * that is supplied. + */ +static struct nfs_client *nfs_match_client(const struct nfs_client_initdata *data) +{ + struct nfs_client *clp; + const struct sockaddr *sap = (struct sockaddr *)data->addr; + struct nfs_net *nn = net_generic(data->net, nfs_net_id); + int error; + +again: + list_for_each_entry(clp, &nn->nfs_client_list, cl_share_link) { + const struct sockaddr *clap = (struct sockaddr *)&clp->cl_addr; + /* Don't match clients that failed to initialise properly */ + if (clp->cl_cons_state < 0) + continue; + + /* If a client is still initializing then we need to wait */ + if (clp->cl_cons_state > NFS_CS_READY) { + refcount_inc(&clp->cl_count); + spin_unlock(&nn->nfs_client_lock); + error = nfs_wait_client_init_complete(clp); + nfs_put_client(clp); + spin_lock(&nn->nfs_client_lock); + if (error < 0) + return ERR_PTR(error); + goto again; + } + + /* Different NFS versions cannot share the same nfs_client */ + if (clp->rpc_ops != data->nfs_mod->rpc_ops) + continue; + + if (clp->cl_proto != data->proto) + continue; + /* Match nfsv4 minorversion */ + if (clp->cl_minorversion != data->minorversion) + continue; + + /* Match request for a dedicated DS */ + if (test_bit(NFS_CS_DS, &data->init_flags) != + test_bit(NFS_CS_DS, &clp->cl_flags)) + continue; + + /* Match the full socket address */ + if (!rpc_cmp_addr_port(sap, clap)) + /* Match all xprt_switch full socket addresses */ + if (IS_ERR(clp->cl_rpcclient) || + !rpc_clnt_xprt_switch_has_addr(clp->cl_rpcclient, + sap)) + continue; + + /* Match the xprt security policy */ + if (clp->cl_xprtsec.policy != data->xprtsec.policy) + continue; + if (clp->cl_xprtsec.policy == RPC_XPRTSEC_TLS_X509) { + if (clp->cl_xprtsec.cert_serial != + data->xprtsec.cert_serial) + continue; + if (clp->cl_xprtsec.privkey_serial != + data->xprtsec.privkey_serial) + continue; + } + + refcount_inc(&clp->cl_count); + return clp; + } + return NULL; +} + +/* + * Return true if @clp is done initializing, false if still working on it. + * + * Use nfs_client_init_status to check if it was successful. + */ +bool nfs_client_init_is_complete(const struct nfs_client *clp) +{ + return clp->cl_cons_state <= NFS_CS_READY; +} +EXPORT_SYMBOL_GPL(nfs_client_init_is_complete); + +/* + * Return 0 if @clp was successfully initialized, -errno otherwise. + * + * This must be called *after* nfs_client_init_is_complete() returns true, + * otherwise it will pop WARN_ON_ONCE and return -EINVAL + */ +int nfs_client_init_status(const struct nfs_client *clp) +{ + /* called without checking nfs_client_init_is_complete */ + if (clp->cl_cons_state > NFS_CS_READY) { + WARN_ON_ONCE(1); + return -EINVAL; + } + return clp->cl_cons_state; +} +EXPORT_SYMBOL_GPL(nfs_client_init_status); + +int nfs_wait_client_init_complete(const struct nfs_client *clp) +{ + return wait_event_killable(nfs_client_active_wq, + nfs_client_init_is_complete(clp)); +} +EXPORT_SYMBOL_GPL(nfs_wait_client_init_complete); + +/* + * Found an existing client. Make sure it's ready before returning. + */ +static struct nfs_client * +nfs_found_client(const struct nfs_client_initdata *cl_init, + struct nfs_client *clp) +{ + int error; + + error = nfs_wait_client_init_complete(clp); + if (error < 0) { + nfs_put_client(clp); + return ERR_PTR(-ERESTARTSYS); + } + + if (clp->cl_cons_state < NFS_CS_READY) { + error = clp->cl_cons_state; + nfs_put_client(clp); + return ERR_PTR(error); + } + + smp_rmb(); + return clp; +} + +/* + * Look up a client by IP address and protocol version + * - creates a new record if one doesn't yet exist + */ +struct nfs_client *nfs_get_client(const struct nfs_client_initdata *cl_init) +{ + struct nfs_client *clp, *new = NULL; + struct nfs_net *nn = net_generic(cl_init->net, nfs_net_id); + const struct nfs_rpc_ops *rpc_ops = cl_init->nfs_mod->rpc_ops; + + if (cl_init->hostname == NULL) { + WARN_ON(1); + return ERR_PTR(-EINVAL); + } + + /* see if the client already exists */ + do { + spin_lock(&nn->nfs_client_lock); + + clp = nfs_match_client(cl_init); + if (clp) { + spin_unlock(&nn->nfs_client_lock); + if (new) + new->rpc_ops->free_client(new); + if (IS_ERR(clp)) + return clp; + return nfs_found_client(cl_init, clp); + } + if (new) { + list_add_tail(&new->cl_share_link, + &nn->nfs_client_list); + spin_unlock(&nn->nfs_client_lock); + new = rpc_ops->init_client(new, cl_init); + if (!IS_ERR(new)) + nfs_local_probe_async(new); + return new; + } + + spin_unlock(&nn->nfs_client_lock); + + new = rpc_ops->alloc_client(cl_init); + } while (!IS_ERR(new)); + + return new; +} +EXPORT_SYMBOL_GPL(nfs_get_client); + +/* + * Mark a server as ready or failed + */ +void nfs_mark_client_ready(struct nfs_client *clp, int state) +{ + smp_wmb(); + clp->cl_cons_state = state; + wake_up_all(&nfs_client_active_wq); +} +EXPORT_SYMBOL_GPL(nfs_mark_client_ready); + +/* + * Initialise the timeout values for a connection + */ +void nfs_init_timeout_values(struct rpc_timeout *to, int proto, + int timeo, int retrans) +{ + to->to_initval = timeo * HZ / 10; + to->to_retries = retrans; + + switch (proto) { + case XPRT_TRANSPORT_TCP: + case XPRT_TRANSPORT_TCP_TLS: + case XPRT_TRANSPORT_RDMA: + if (retrans == NFS_UNSPEC_RETRANS) + to->to_retries = NFS_DEF_TCP_RETRANS; + if (timeo == NFS_UNSPEC_TIMEO || to->to_initval == 0) + to->to_initval = NFS_DEF_TCP_TIMEO * HZ / 10; + if (to->to_initval > NFS_MAX_TCP_TIMEOUT) + to->to_initval = NFS_MAX_TCP_TIMEOUT; + to->to_increment = to->to_initval; + to->to_maxval = to->to_initval + (to->to_increment * to->to_retries); + if (to->to_maxval > NFS_MAX_TCP_TIMEOUT) + to->to_maxval = NFS_MAX_TCP_TIMEOUT; + if (to->to_maxval < to->to_initval) + to->to_maxval = to->to_initval; + to->to_exponential = 0; + break; + case XPRT_TRANSPORT_UDP: + if (retrans == NFS_UNSPEC_RETRANS) + to->to_retries = NFS_DEF_UDP_RETRANS; + if (timeo == NFS_UNSPEC_TIMEO || to->to_initval == 0) + to->to_initval = NFS_DEF_UDP_TIMEO * HZ / 10; + if (to->to_initval > NFS_MAX_UDP_TIMEOUT) + to->to_initval = NFS_MAX_UDP_TIMEOUT; + to->to_maxval = NFS_MAX_UDP_TIMEOUT; + to->to_exponential = 1; + break; + default: + BUG(); + } +} +EXPORT_SYMBOL_GPL(nfs_init_timeout_values); + +/* + * Create an RPC client handle + */ +int nfs_create_rpc_client(struct nfs_client *clp, + const struct nfs_client_initdata *cl_init, + rpc_authflavor_t flavor) +{ + struct nfs_net *nn = net_generic(clp->cl_net, nfs_net_id); + struct rpc_clnt *clnt = NULL; + struct rpc_create_args args = { + .net = clp->cl_net, + .protocol = clp->cl_proto, + .nconnect = clp->cl_nconnect, + .address = (struct sockaddr *)&clp->cl_addr, + .addrsize = clp->cl_addrlen, + .timeout = cl_init->timeparms, + .servername = clp->cl_hostname, + .nodename = cl_init->nodename, + .program = &nfs_program, + .stats = &nn->rpcstats, + .version = clp->rpc_ops->version, + .authflavor = flavor, + .cred = cl_init->cred, + .xprtsec = cl_init->xprtsec, + .connect_timeout = cl_init->connect_timeout, + .reconnect_timeout = cl_init->reconnect_timeout, + }; + + if (test_bit(NFS_CS_DISCRTRY, &clp->cl_flags)) + args.flags |= RPC_CLNT_CREATE_DISCRTRY; + if (test_bit(NFS_CS_NO_RETRANS_TIMEOUT, &clp->cl_flags)) + args.flags |= RPC_CLNT_CREATE_NO_RETRANS_TIMEOUT; + if (test_bit(NFS_CS_NORESVPORT, &clp->cl_flags)) + args.flags |= RPC_CLNT_CREATE_NONPRIVPORT; + if (test_bit(NFS_CS_INFINITE_SLOTS, &clp->cl_flags)) + args.flags |= RPC_CLNT_CREATE_INFINITE_SLOTS; + if (test_bit(NFS_CS_NOPING, &clp->cl_flags)) + args.flags |= RPC_CLNT_CREATE_NOPING; + if (test_bit(NFS_CS_REUSEPORT, &clp->cl_flags)) + args.flags |= RPC_CLNT_CREATE_REUSEPORT; + if (test_bit(NFS_CS_NETUNREACH_FATAL, &clp->cl_flags)) + args.flags |= RPC_CLNT_CREATE_NETUNREACH_FATAL; + + if (!IS_ERR(clp->cl_rpcclient)) + return 0; + + clnt = rpc_create(&args); + if (IS_ERR(clnt)) { + dprintk("%s: cannot create RPC client. Error = %ld\n", + __func__, PTR_ERR(clnt)); + return PTR_ERR(clnt); + } + + clnt->cl_principal = clp->cl_principal; + clp->cl_rpcclient = clnt; + clnt->cl_max_connect = clp->cl_max_connect; + return 0; +} +EXPORT_SYMBOL_GPL(nfs_create_rpc_client); + +/* + * Version 2 or 3 client destruction + */ +static void nfs_destroy_server(struct nfs_server *server) +{ + if (server->nlm_host) + nlmclnt_done(server->nlm_host); +} + +/* + * Version 2 or 3 lockd setup + */ +static int nfs_start_lockd(struct nfs_server *server) +{ + struct nlm_host *host; + struct nfs_client *clp = server->nfs_client; + struct nlmclnt_initdata nlm_init = { + .hostname = clp->cl_hostname, + .address = (struct sockaddr *)&clp->cl_addr, + .addrlen = clp->cl_addrlen, + .nfs_version = clp->rpc_ops->version, + .noresvport = server->flags & NFS_MOUNT_NORESVPORT ? + 1 : 0, + .net = clp->cl_net, + .nlmclnt_ops = clp->cl_nfs_mod->rpc_ops->nlmclnt_ops, + .cred = server->cred, + }; + + if (nlm_init.nfs_version > 3) + return 0; + if ((server->flags & NFS_MOUNT_LOCAL_FLOCK) && + (server->flags & NFS_MOUNT_LOCAL_FCNTL)) + return 0; + + switch (clp->cl_proto) { + default: + nlm_init.protocol = IPPROTO_TCP; + break; +#ifndef CONFIG_NFS_DISABLE_UDP_SUPPORT + case XPRT_TRANSPORT_UDP: + nlm_init.protocol = IPPROTO_UDP; +#endif + } + + host = nlmclnt_init(&nlm_init); + if (IS_ERR(host)) + return PTR_ERR(host); + + server->nlm_host = host; + server->destroy = nfs_destroy_server; + nfs_sysfs_link_rpc_client(server, nlmclnt_rpc_clnt(host), NULL); + return 0; +} + +/* + * Create a general RPC client + */ +int nfs_init_server_rpcclient(struct nfs_server *server, + const struct rpc_timeout *timeo, + rpc_authflavor_t pseudoflavour) +{ + struct nfs_client *clp = server->nfs_client; + + server->client = rpc_clone_client_set_auth(clp->cl_rpcclient, + pseudoflavour); + if (IS_ERR(server->client)) { + dprintk("%s: couldn't create rpc_client!\n", __func__); + return PTR_ERR(server->client); + } + + memcpy(&server->client->cl_timeout_default, + timeo, + sizeof(server->client->cl_timeout_default)); + server->client->cl_timeout = &server->client->cl_timeout_default; + server->client->cl_softrtry = 0; + if (server->flags & NFS_MOUNT_SOFTERR) + server->client->cl_softerr = 1; + if (server->flags & NFS_MOUNT_SOFT) + server->client->cl_softrtry = 1; + + nfs_sysfs_link_rpc_client(server, server->client, NULL); + return 0; +} +EXPORT_SYMBOL_GPL(nfs_init_server_rpcclient); + +/** + * nfs_init_client - Initialise an NFS2 or NFS3 client + * + * @clp: nfs_client to initialise + * @cl_init: Initialisation parameters + * + * Returns pointer to an NFS client, or an ERR_PTR value. + */ +struct nfs_client *nfs_init_client(struct nfs_client *clp, + const struct nfs_client_initdata *cl_init) +{ + int error; + + /* the client is already initialised */ + if (clp->cl_cons_state == NFS_CS_READY) + return clp; + + /* + * Create a client RPC handle for doing FSSTAT with UNIX auth only + * - RFC 2623, sec 2.3.2 + */ + error = nfs_create_rpc_client(clp, cl_init, RPC_AUTH_UNIX); + nfs_mark_client_ready(clp, error == 0 ? NFS_CS_READY : error); + if (error < 0) { + nfs_put_client(clp); + clp = ERR_PTR(error); + } + return clp; +} +EXPORT_SYMBOL_GPL(nfs_init_client); + +static void nfs4_server_set_init_caps(struct nfs_server *server) +{ +#if IS_ENABLED(CONFIG_NFS_V4) + /* Set the basic capabilities */ + server->caps = server->nfs_client->cl_mvops->init_caps; + if (server->flags & NFS_MOUNT_NORDIRPLUS) + server->caps &= ~NFS_CAP_READDIRPLUS; + if (server->nfs_client->cl_proto == XPRT_TRANSPORT_RDMA) + server->caps &= ~NFS_CAP_READ_PLUS; + + /* + * Don't use NFS uid/gid mapping if we're using AUTH_SYS or lower + * authentication. + */ + if (nfs4_disable_idmapping && + server->client->cl_auth->au_flavor == RPC_AUTH_UNIX) + server->caps |= NFS_CAP_UIDGID_NOMAP; +#endif +} + +void nfs_server_set_init_caps(struct nfs_server *server) +{ + switch (server->nfs_client->rpc_ops->version) { + case 2: + server->caps = NFS_CAP_HARDLINKS | NFS_CAP_SYMLINKS; + break; + case 3: + server->caps = NFS_CAP_HARDLINKS | NFS_CAP_SYMLINKS; + if (!(server->flags & NFS_MOUNT_NORDIRPLUS)) + server->caps |= NFS_CAP_READDIRPLUS; + break; + default: + nfs4_server_set_init_caps(server); + break; + } +} +EXPORT_SYMBOL_GPL(nfs_server_set_init_caps); + +/* + * Create a version 2 or 3 client + */ +static int nfs_init_server(struct nfs_server *server, + const struct fs_context *fc) +{ + const struct nfs_fs_context *ctx = nfs_fc2context(fc); + struct rpc_timeout timeparms; + struct nfs_client_initdata cl_init = { + .hostname = ctx->nfs_server.hostname, + .addr = &ctx->nfs_server._address, + .addrlen = ctx->nfs_server.addrlen, + .nfs_mod = ctx->nfs_mod, + .proto = ctx->nfs_server.protocol, + .net = fc->net_ns, + .timeparms = &timeparms, + .cred = server->cred, + .nconnect = ctx->nfs_server.nconnect, + .init_flags = (1UL << NFS_CS_REUSEPORT), + .xprtsec = ctx->xprtsec, + }; + struct nfs_client *clp; + int error; + + nfs_init_timeout_values(&timeparms, ctx->nfs_server.protocol, + ctx->timeo, ctx->retrans); + if (ctx->flags & NFS_MOUNT_NORESVPORT) + set_bit(NFS_CS_NORESVPORT, &cl_init.init_flags); + + if (ctx->flags & NFS_MOUNT_NETUNREACH_FATAL) + __set_bit(NFS_CS_NETUNREACH_FATAL, &cl_init.init_flags); + + /* Allocate or find a client reference we can use */ + clp = nfs_get_client(&cl_init); + if (IS_ERR(clp)) + return PTR_ERR(clp); + + server->nfs_client = clp; + nfs_sysfs_add_server(server); + nfs_sysfs_link_rpc_client(server, clp->cl_rpcclient, "_state"); + + /* Initialise the client representation from the mount data */ + server->flags = ctx->flags; + server->options = ctx->options; + + switch (clp->rpc_ops->version) { + case 2: + server->fattr_valid = NFS_ATTR_FATTR_V2; + break; + case 3: + server->fattr_valid = NFS_ATTR_FATTR_V3; + break; + default: + server->fattr_valid = NFS_ATTR_FATTR_V4; + } + + if (ctx->bsize) { + server->bsize = ctx->bsize; + server->automount_inherit |= NFS_AUTOMOUNT_INHERIT_BSIZE; + } + if (ctx->rsize) { + server->rsize = nfs_io_size(ctx->rsize, clp->cl_proto); + server->automount_inherit |= NFS_AUTOMOUNT_INHERIT_RSIZE; + } + if (ctx->wsize) { + server->wsize = nfs_io_size(ctx->wsize, clp->cl_proto); + server->automount_inherit |= NFS_AUTOMOUNT_INHERIT_WSIZE; + } + + server->acregmin = ctx->acregmin * HZ; + server->acregmax = ctx->acregmax * HZ; + server->acdirmin = ctx->acdirmin * HZ; + server->acdirmax = ctx->acdirmax * HZ; + + /* Start lockd here, before we might error out */ + error = nfs_start_lockd(server); + if (error < 0) + goto error; + + server->port = ctx->nfs_server.port; + server->auth_info = ctx->auth_info; + + error = nfs_init_server_rpcclient(server, &timeparms, + ctx->selected_flavor); + if (error < 0) + goto error; + + nfs_server_set_init_caps(server); + + /* Preserve the values of mount_server-related mount options */ + if (ctx->mount_server.addrlen) { + memcpy(&server->mountd_address, &ctx->mount_server.address, + ctx->mount_server.addrlen); + server->mountd_addrlen = ctx->mount_server.addrlen; + } + server->mountd_version = ctx->mount_server.version; + server->mountd_port = ctx->mount_server.port; + server->mountd_protocol = ctx->mount_server.protocol; + + server->namelen = ctx->namlen; + return 0; + +error: + server->nfs_client = NULL; + nfs_put_client(clp); + return error; +} + +/* + * Load up the server record from information gained in an fsinfo record + */ +static void nfs_server_set_fsinfo(struct nfs_server *server, + struct nfs_fsinfo *fsinfo) +{ + struct nfs_client *clp = server->nfs_client; + unsigned long max_rpc_payload, raw_max_rpc_payload; + + /* Work out a lot of parameters */ + if (server->rsize == 0) + server->rsize = nfs_io_size(fsinfo->rtpref, clp->cl_proto); + if (server->wsize == 0) + server->wsize = nfs_io_size(fsinfo->wtpref, clp->cl_proto); + + if (fsinfo->rtmax >= 512 && server->rsize > fsinfo->rtmax) + server->rsize = nfs_io_size(fsinfo->rtmax, clp->cl_proto); + if (fsinfo->wtmax >= 512 && server->wsize > fsinfo->wtmax) + server->wsize = nfs_io_size(fsinfo->wtmax, clp->cl_proto); + + raw_max_rpc_payload = rpc_max_payload(server->client); + max_rpc_payload = nfs_block_size(raw_max_rpc_payload, NULL); + + if (server->rsize > max_rpc_payload) + server->rsize = max_rpc_payload; + if (server->rsize > NFS_MAX_FILE_IO_SIZE) + server->rsize = NFS_MAX_FILE_IO_SIZE; + server->rpages = (server->rsize + PAGE_SIZE - 1) >> PAGE_SHIFT; + + if (server->wsize > max_rpc_payload) + server->wsize = max_rpc_payload; + if (server->wsize > NFS_MAX_FILE_IO_SIZE) + server->wsize = NFS_MAX_FILE_IO_SIZE; + + server->wtmult = nfs_block_bits(fsinfo->wtmult, NULL); + + server->dtsize = nfs_block_size(fsinfo->dtpref, NULL); + if (server->dtsize > NFS_MAX_FILE_IO_SIZE) + server->dtsize = NFS_MAX_FILE_IO_SIZE; + if (server->dtsize > server->rsize) + server->dtsize = server->rsize; + + if (server->flags & NFS_MOUNT_NOAC) { + server->acregmin = server->acregmax = 0; + server->acdirmin = server->acdirmax = 0; + } + + server->maxfilesize = fsinfo->maxfilesize; + + server->change_attr_type = fsinfo->change_attr_type; + + server->clone_blksize = fsinfo->clone_blksize; + /* We're airborne Set socket buffersize */ + rpc_setbufsize(server->client, server->wsize + 100, server->rsize + 100); + +#ifdef CONFIG_NFS_V4_2 + /* + * Defaults until limited by the session parameters. + */ + server->gxasize = min_t(unsigned int, raw_max_rpc_payload, + XATTR_SIZE_MAX); + server->sxasize = min_t(unsigned int, raw_max_rpc_payload, + XATTR_SIZE_MAX); + server->lxasize = min_t(unsigned int, raw_max_rpc_payload, + nfs42_listxattr_xdrsize(XATTR_LIST_MAX)); + + if (fsinfo->xattr_support) + server->caps |= NFS_CAP_XATTR; + else + server->caps &= ~NFS_CAP_XATTR; +#endif +} + +/* + * Probe filesystem information, including the FSID on v2/v3 + */ +static int nfs_probe_fsinfo(struct nfs_server *server, struct nfs_fh *mntfh, struct nfs_fattr *fattr) +{ + struct nfs_pathconf pathinfo = { }; + struct nfs_fsinfo fsinfo; + struct nfs_client *clp = server->nfs_client; + int error; + + if (clp->rpc_ops->set_capabilities != NULL) { + error = clp->rpc_ops->set_capabilities(server, mntfh); + if (error < 0) + return error; + } + + fsinfo.fattr = fattr; + fsinfo.nlayouttypes = 0; + memset(fsinfo.layouttype, 0, sizeof(fsinfo.layouttype)); + error = clp->rpc_ops->fsinfo(server, mntfh, &fsinfo); + if (error < 0) + return error; + + nfs_server_set_fsinfo(server, &fsinfo); + + pathinfo.fattr = fattr; + nfs_fattr_init(fattr); + + if (clp->rpc_ops->version < 4 || server->namelen == 0) { + if (clp->rpc_ops->pathconf(server, mntfh, &pathinfo) >= 0) { + if (server->namelen == 0) + server->namelen = pathinfo.max_namelen; + if (clp->rpc_ops->version < 4) { + unsigned int caps = server->caps; + + caps &= ~(NFS_CAP_CASE_INSENSITIVE | + NFS_CAP_CASE_NONPRESERVING); + if (pathinfo.case_insensitive) + caps |= NFS_CAP_CASE_INSENSITIVE; + if (!pathinfo.case_preserving) + caps |= NFS_CAP_CASE_NONPRESERVING; + server->caps = caps; + } + } else if (clp->rpc_ops->version < 4) { + server->caps &= ~(NFS_CAP_CASE_INSENSITIVE | + NFS_CAP_CASE_NONPRESERVING); + } + } + + if (clp->rpc_ops->discover_trunking != NULL && + (server->caps & NFS_CAP_FS_LOCATIONS && + (server->flags & NFS_MOUNT_TRUNK_DISCOVERY))) { + error = clp->rpc_ops->discover_trunking(server, mntfh); + if (error < 0) + return error; + } + + return 0; +} + +/* + * Grab the destination's particulars, including lease expiry time. + * + * Returns zero if probe succeeded and retrieved FSID matches the FSID + * we have cached. + */ +int nfs_probe_server(struct nfs_server *server, struct nfs_fh *mntfh) +{ + struct nfs_fattr *fattr; + int error; + + fattr = nfs_alloc_fattr(); + if (fattr == NULL) + return -ENOMEM; + + /* Sanity: the probe won't work if the destination server + * does not recognize the migrated FH. */ + error = nfs_probe_fsinfo(server, mntfh, fattr); + + nfs_free_fattr(fattr); + return error; +} +EXPORT_SYMBOL_GPL(nfs_probe_server); + +/* + * Copy useful information when duplicating a server record + */ +void nfs_server_copy_userdata(struct nfs_server *target, struct nfs_server *source) +{ + target->flags = source->flags; + target->automount_inherit = source->automount_inherit; + if (source->automount_inherit & NFS_AUTOMOUNT_INHERIT_BSIZE) + target->bsize = source->bsize; + if (source->automount_inherit & NFS_AUTOMOUNT_INHERIT_RSIZE) + target->rsize = source->rsize; + if (source->automount_inherit & NFS_AUTOMOUNT_INHERIT_WSIZE) + target->wsize = source->wsize; + target->acregmin = source->acregmin; + target->acregmax = source->acregmax; + target->acdirmin = source->acdirmin; + target->acdirmax = source->acdirmax; + target->options = source->options; + target->auth_info = source->auth_info; + target->port = source->port; +} +EXPORT_SYMBOL_GPL(nfs_server_copy_userdata); + +void nfs_server_insert_lists(struct nfs_server *server) +{ + struct nfs_client *clp = server->nfs_client; + struct nfs_net *nn = net_generic(clp->cl_net, nfs_net_id); + + spin_lock(&nn->nfs_client_lock); + list_add_tail_rcu(&server->client_link, &clp->cl_superblocks); + list_add_tail(&server->master_link, &nn->nfs_volume_list); + clear_bit(NFS_CS_STOP_RENEW, &clp->cl_res_state); + spin_unlock(&nn->nfs_client_lock); + +} +EXPORT_SYMBOL_GPL(nfs_server_insert_lists); + +void nfs_server_remove_lists(struct nfs_server *server) +{ + struct nfs_client *clp = server->nfs_client; + struct nfs_net *nn; + + if (clp == NULL) + return; + nn = net_generic(clp->cl_net, nfs_net_id); + spin_lock(&nn->nfs_client_lock); + list_del_rcu(&server->client_link); + if (list_empty(&clp->cl_superblocks)) + set_bit(NFS_CS_STOP_RENEW, &clp->cl_res_state); + list_del(&server->master_link); + spin_unlock(&nn->nfs_client_lock); + + synchronize_rcu(); +} +EXPORT_SYMBOL_GPL(nfs_server_remove_lists); + +static DEFINE_IDA(s_sysfs_ids); + +/* + * Allocate and initialise a server record + */ +struct nfs_server *nfs_alloc_server(void) +{ + struct nfs_server *server; + + server = kzalloc_obj(struct nfs_server); + if (!server) + return NULL; + + server->s_sysfs_id = ida_alloc(&s_sysfs_ids, GFP_KERNEL); + if (server->s_sysfs_id < 0) + goto free_server; + + server->client = server->client_acl = ERR_PTR(-EINVAL); + + /* Zero out the NFS state stuff */ + INIT_LIST_HEAD(&server->client_link); + INIT_LIST_HEAD(&server->master_link); + INIT_LIST_HEAD(&server->delegations); + spin_lock_init(&server->delegations_lock); + INIT_LIST_HEAD(&server->delegations_return); + INIT_LIST_HEAD(&server->delegations_lru); + INIT_LIST_HEAD(&server->delegations_delayed); + INIT_LIST_HEAD(&server->layouts); + INIT_LIST_HEAD(&server->state_owners_lru); + INIT_LIST_HEAD(&server->ss_copies); + INIT_LIST_HEAD(&server->ss_src_copies); + + atomic_set(&server->active, 0); + atomic_long_set(&server->nr_active_delegations, 0); + + server->io_stats = nfs_alloc_iostats(); + if (!server->io_stats) { + ida_free(&s_sysfs_ids, server->s_sysfs_id); + goto free_server; + } + + server->change_attr_type = NFS4_CHANGE_TYPE_IS_UNDEFINED; + + init_waitqueue_head(&server->write_congestion_wait); + atomic_long_set(&server->writeback, 0); + + atomic64_set(&server->owner_ctr, 0); + + pnfs_init_server(server); + rpc_init_wait_queue(&server->uoc_rpcwaitq, "NFS UOC"); + + return server; + +free_server: + kfree(server); + return NULL; +} +EXPORT_SYMBOL_GPL(nfs_alloc_server); + +static void delayed_free(struct rcu_head *p) +{ + struct nfs_server *server = container_of(p, struct nfs_server, rcu); + + nfs_free_iostats(server->io_stats); + kfree(server); +} + +/* + * Free up a server record + */ +void nfs_free_server(struct nfs_server *server) +{ + nfs_server_remove_lists(server); + + if (server->destroy != NULL) + server->destroy(server); + + if (!IS_ERR(server->client_acl)) + rpc_shutdown_client(server->client_acl); + if (!IS_ERR(server->client)) + rpc_shutdown_client(server->client); + + nfs_put_client(server->nfs_client); + + if (server->kobj.state_initialized) { + nfs_sysfs_remove_server(server); + kobject_put(&server->kobj); + } + ida_free(&s_sysfs_ids, server->s_sysfs_id); + + put_cred(server->cred); + nfs_release_automount_timer(); + call_rcu(&server->rcu, delayed_free); +} +EXPORT_SYMBOL_GPL(nfs_free_server); + +/* + * Create a version 2 or 3 volume record + * - keyed on server and FSID + */ +struct nfs_server *nfs_create_server(struct fs_context *fc) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + struct nfs_server *server; + struct nfs_fattr *fattr; + int error; + + server = nfs_alloc_server(); + if (!server) + return ERR_PTR(-ENOMEM); + + server->cred = get_cred(fc->cred); + + error = -ENOMEM; + fattr = nfs_alloc_fattr(); + if (fattr == NULL) + goto error; + + /* Get a client representation */ + error = nfs_init_server(server, fc); + if (error < 0) + goto error; + + /* Probe the root fh to retrieve its FSID */ + error = nfs_probe_fsinfo(server, ctx->mntfh, fattr); + if (error < 0) + goto error; + if (server->nfs_client->rpc_ops->version == 3) { + if (server->namelen == 0 || server->namelen > NFS3_MAXNAMLEN) + server->namelen = NFS3_MAXNAMLEN; + if (!(ctx->flags & NFS_MOUNT_NORDIRPLUS)) + server->caps |= NFS_CAP_READDIRPLUS; + } else { + if (server->namelen == 0 || server->namelen > NFS2_MAXNAMLEN) + server->namelen = NFS2_MAXNAMLEN; + } + /* Linux 'subtree_check' borkenness mandates this setting */ + server->fh_expire_type = NFS_FH_VOL_RENAME; + + if (!(fattr->valid & NFS_ATTR_FATTR)) { + error = ctx->nfs_mod->rpc_ops->getattr(server, ctx->mntfh, + fattr, NULL); + if (error < 0) { + dprintk("nfs_create_server: getattr error = %d\n", -error); + goto error; + } + } + memcpy(&server->fsid, &fattr->fsid, sizeof(server->fsid)); + + dprintk("Server FSID: %llx:%llx\n", + (unsigned long long) server->fsid.major, + (unsigned long long) server->fsid.minor); + + nfs_server_insert_lists(server); + server->mount_time = jiffies; + nfs_free_fattr(fattr); + return server; + +error: + nfs_free_fattr(fattr); + nfs_free_server(server); + return ERR_PTR(error); +} +EXPORT_SYMBOL_GPL(nfs_create_server); + +/* + * Clone an NFS2, NFS3 or NFS4 server record + */ +struct nfs_server *nfs_clone_server(struct nfs_server *source, + struct nfs_fh *fh, + struct nfs_fattr *fattr, + rpc_authflavor_t flavor) +{ + struct nfs_server *server; + int error; + + server = nfs_alloc_server(); + if (!server) + return ERR_PTR(-ENOMEM); + + server->cred = get_cred(source->cred); + + /* Copy data from the source */ + server->nfs_client = source->nfs_client; + server->destroy = source->destroy; + refcount_inc(&server->nfs_client->cl_count); + nfs_server_copy_userdata(server, source); + + server->fsid = fattr->fsid; + + nfs_sysfs_add_server(server); + + nfs_sysfs_link_rpc_client(server, + server->nfs_client->cl_rpcclient, "_state"); + + error = nfs_init_server_rpcclient(server, + source->client->cl_timeout, + flavor); + if (error < 0) + goto out_free_server; + + nfs_server_set_init_caps(server); + + /* probe the filesystem info for this server filesystem */ + error = nfs_probe_server(server, fh); + if (error < 0) + goto out_free_server; + + if (server->namelen == 0 || server->namelen > NFS4_MAXNAMLEN) + server->namelen = NFS4_MAXNAMLEN; + + error = nfs_start_lockd(server); + if (error < 0) + goto out_free_server; + + nfs_server_insert_lists(server); + server->mount_time = jiffies; + + return server; + +out_free_server: + nfs_free_server(server); + return ERR_PTR(error); +} +EXPORT_SYMBOL_GPL(nfs_clone_server); + +void nfs_clients_init(struct net *net) +{ + struct nfs_net *nn = net_generic(net, nfs_net_id); + + INIT_LIST_HEAD(&nn->nfs_client_list); + INIT_LIST_HEAD(&nn->nfs_volume_list); +#if IS_ENABLED(CONFIG_NFS_V4) + idr_init(&nn->cb_ident_idr); + INIT_LIST_HEAD(&nn->nfs4_data_server_cache); + spin_lock_init(&nn->nfs4_data_server_lock); +#endif /* CONFIG_NFS_V4 */ + spin_lock_init(&nn->nfs_client_lock); + nn->boot_time = ktime_get_real(); + memset(&nn->rpcstats, 0, sizeof(nn->rpcstats)); + nn->rpcstats.program = &nfs_program; + + nfs_netns_sysfs_setup(nn, net); +} + +void nfs_clients_exit(struct net *net) +{ + struct nfs_net *nn = net_generic(net, nfs_net_id); + + nfs_netns_sysfs_destroy(nn); + nfs_cleanup_cb_ident_idr(net); + WARN_ON_ONCE(!list_empty(&nn->nfs_client_list)); + WARN_ON_ONCE(!list_empty(&nn->nfs_volume_list)); +#if IS_ENABLED(CONFIG_NFS_V4) + WARN_ON_ONCE(!list_empty(&nn->nfs4_data_server_cache)); +#endif /* CONFIG_NFS_V4 */ +} + +#ifdef CONFIG_PROC_FS +static void *nfs_server_list_start(struct seq_file *p, loff_t *pos); +static void *nfs_server_list_next(struct seq_file *p, void *v, loff_t *pos); +static void nfs_server_list_stop(struct seq_file *p, void *v); +static int nfs_server_list_show(struct seq_file *m, void *v); + +static const struct seq_operations nfs_server_list_ops = { + .start = nfs_server_list_start, + .next = nfs_server_list_next, + .stop = nfs_server_list_stop, + .show = nfs_server_list_show, +}; + +static void *nfs_volume_list_start(struct seq_file *p, loff_t *pos); +static void *nfs_volume_list_next(struct seq_file *p, void *v, loff_t *pos); +static void nfs_volume_list_stop(struct seq_file *p, void *v); +static int nfs_volume_list_show(struct seq_file *m, void *v); + +static const struct seq_operations nfs_volume_list_ops = { + .start = nfs_volume_list_start, + .next = nfs_volume_list_next, + .stop = nfs_volume_list_stop, + .show = nfs_volume_list_show, +}; + +/* + * set up the iterator to start reading from the server list and return the first item + */ +static void *nfs_server_list_start(struct seq_file *m, loff_t *_pos) + __acquires(&nn->nfs_client_lock) +{ + struct nfs_net *nn = net_generic(seq_file_net(m), nfs_net_id); + + /* lock the list against modification */ + spin_lock(&nn->nfs_client_lock); + return seq_list_start_head(&nn->nfs_client_list, *_pos); +} + +/* + * move to next server + */ +static void *nfs_server_list_next(struct seq_file *p, void *v, loff_t *pos) +{ + struct nfs_net *nn = net_generic(seq_file_net(p), nfs_net_id); + + return seq_list_next(v, &nn->nfs_client_list, pos); +} + +/* + * clean up after reading from the transports list + */ +static void nfs_server_list_stop(struct seq_file *p, void *v) + __releases(&nn->nfs_client_lock) +{ + struct nfs_net *nn = net_generic(seq_file_net(p), nfs_net_id); + + spin_unlock(&nn->nfs_client_lock); +} + +/* + * display a header line followed by a load of call lines + */ +static int nfs_server_list_show(struct seq_file *m, void *v) +{ + struct nfs_client *clp; + struct nfs_net *nn = net_generic(seq_file_net(m), nfs_net_id); + + /* display header on line 1 */ + if (v == &nn->nfs_client_list) { + seq_puts(m, "NV SERVER PORT USE HOSTNAME\n"); + return 0; + } + + /* display one transport per line on subsequent lines */ + clp = list_entry(v, struct nfs_client, cl_share_link); + + /* Check if the client is initialized */ + if (clp->cl_cons_state != NFS_CS_READY) + return 0; + + rcu_read_lock(); + seq_printf(m, "v%u %s %s %3d %s\n", + clp->rpc_ops->version, + rpc_peeraddr2str(clp->cl_rpcclient, RPC_DISPLAY_HEX_ADDR), + rpc_peeraddr2str(clp->cl_rpcclient, RPC_DISPLAY_HEX_PORT), + refcount_read(&clp->cl_count), + clp->cl_hostname); + rcu_read_unlock(); + + return 0; +} + +/* + * set up the iterator to start reading from the volume list and return the first item + */ +static void *nfs_volume_list_start(struct seq_file *m, loff_t *_pos) + __acquires(&nn->nfs_client_lock) +{ + struct nfs_net *nn = net_generic(seq_file_net(m), nfs_net_id); + + /* lock the list against modification */ + spin_lock(&nn->nfs_client_lock); + return seq_list_start_head(&nn->nfs_volume_list, *_pos); +} + +/* + * move to next volume + */ +static void *nfs_volume_list_next(struct seq_file *p, void *v, loff_t *pos) +{ + struct nfs_net *nn = net_generic(seq_file_net(p), nfs_net_id); + + return seq_list_next(v, &nn->nfs_volume_list, pos); +} + +/* + * clean up after reading from the transports list + */ +static void nfs_volume_list_stop(struct seq_file *p, void *v) + __releases(&nn->nfs_client_lock) +{ + struct nfs_net *nn = net_generic(seq_file_net(p), nfs_net_id); + + spin_unlock(&nn->nfs_client_lock); +} + +/* + * display a header line followed by a load of call lines + */ +static int nfs_volume_list_show(struct seq_file *m, void *v) +{ + struct nfs_server *server; + struct nfs_client *clp; + char dev[13]; // 8 for 2^24, 1 for ':', 3 for 2^8, 1 for '\0' + char fsid[34]; // 2 * 16 for %llx, 1 for ':', 1 for '\0' + struct nfs_net *nn = net_generic(seq_file_net(m), nfs_net_id); + + /* display header on line 1 */ + if (v == &nn->nfs_volume_list) { + seq_puts(m, "NV SERVER PORT DEV FSID" + " FSC\n"); + return 0; + } + /* display one transport per line on subsequent lines */ + server = list_entry(v, struct nfs_server, master_link); + clp = server->nfs_client; + + snprintf(dev, sizeof(dev), "%u:%u", + MAJOR(server->s_dev), MINOR(server->s_dev)); + + snprintf(fsid, sizeof(fsid), "%llx:%llx", + (unsigned long long) server->fsid.major, + (unsigned long long) server->fsid.minor); + + rcu_read_lock(); + seq_printf(m, "v%u %s %s %-12s %-33s %s\n", + clp->rpc_ops->version, + rpc_peeraddr2str(clp->cl_rpcclient, RPC_DISPLAY_HEX_ADDR), + rpc_peeraddr2str(clp->cl_rpcclient, RPC_DISPLAY_HEX_PORT), + dev, + fsid, + nfs_server_fscache_state(server)); + rcu_read_unlock(); + + return 0; +} + +int nfs_fs_proc_net_init(struct net *net) +{ + struct nfs_net *nn = net_generic(net, nfs_net_id); + struct proc_dir_entry *p; + + nn->proc_nfsfs = proc_net_mkdir(net, "nfsfs", net->proc_net); + if (!nn->proc_nfsfs) + goto error_0; + + /* a file of servers with which we're dealing */ + p = proc_create_net("servers", S_IFREG|S_IRUGO, nn->proc_nfsfs, + &nfs_server_list_ops, sizeof(struct seq_net_private)); + if (!p) + goto error_1; + + /* a file of volumes that we have mounted */ + p = proc_create_net("volumes", S_IFREG|S_IRUGO, nn->proc_nfsfs, + &nfs_volume_list_ops, sizeof(struct seq_net_private)); + if (!p) + goto error_1; + return 0; + +error_1: + remove_proc_subtree("nfsfs", net->proc_net); +error_0: + return -ENOMEM; +} + +void nfs_fs_proc_net_exit(struct net *net) +{ + remove_proc_subtree("nfsfs", net->proc_net); +} + +/* + * initialise the /proc/fs/nfsfs/ directory + */ +int __init nfs_fs_proc_init(void) +{ + if (!proc_mkdir("fs/nfsfs", NULL)) + goto error_0; + + /* a file of servers with which we're dealing */ + if (!proc_symlink("fs/nfsfs/servers", NULL, "../../net/nfsfs/servers")) + goto error_1; + + /* a file of volumes that we have mounted */ + if (!proc_symlink("fs/nfsfs/volumes", NULL, "../../net/nfsfs/volumes")) + goto error_1; + + return 0; +error_1: + remove_proc_subtree("fs/nfsfs", NULL); +error_0: + return -ENOMEM; +} + +/* + * clean up the /proc/fs/nfsfs/ directory + */ +void nfs_fs_proc_exit(void) +{ + remove_proc_subtree("fs/nfsfs", NULL); + ida_destroy(&s_sysfs_ids); +} + +#endif /* CONFIG_PROC_FS */ diff --git a/fs/nfs/delegation.c b/fs/nfs/delegation.c new file mode 100644 index 000000000..9546d2195 --- /dev/null +++ b/fs/nfs/delegation.c @@ -0,0 +1,1619 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/nfs/delegation.c + * + * Copyright (C) 2004 Trond Myklebust + * + * NFS file delegation management + * + */ +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "nfs4_fs.h" +#include "nfs4session.h" +#include "delegation.h" +#include "internal.h" +#include "nfs4trace.h" + +#define NFS_DEFAULT_DELEGATION_WATERMARK (5000U) + +static unsigned nfs_delegation_watermark = NFS_DEFAULT_DELEGATION_WATERMARK; +module_param_named(delegation_watermark, nfs_delegation_watermark, uint, 0644); + +bool directory_delegations = true; +module_param(directory_delegations, bool, 0644); +MODULE_PARM_DESC(directory_delegations, + "Enable the use of directory delegations, defaults to on."); + +static struct hlist_head *nfs_delegation_hash(struct nfs_server *server, + const struct nfs_fh *fhandle) +{ + return server->delegation_hash_table + + (nfs_fhandle_hash(fhandle) & server->delegation_hash_mask); +} + +static void __nfs_free_delegation(struct nfs_delegation *delegation) +{ + put_cred(delegation->cred); + delegation->cred = NULL; + kfree_rcu(delegation, rcu); +} + +static void nfs_mark_delegation_revoked(struct nfs_server *server, + struct nfs_delegation *delegation) +{ + bool put_ref = false; + + if (test_and_set_bit(NFS_DELEGATION_REVOKED, &delegation->flags)) + return; + + delegation->stateid.type = NFS4_INVALID_STATEID_TYPE; + atomic_long_dec(&server->nr_active_delegations); + if (!test_bit(NFS_DELEGATION_RETURNING, &delegation->flags)) + nfs_clear_verifier_delegated(delegation->inode); + + spin_lock(&server->delegations_lock); + if (!list_empty(&delegation->entry)) { + list_del_init(&delegation->entry); + put_ref = true; + } + spin_unlock(&server->delegations_lock); + + if (put_ref) + nfs_put_delegation(delegation); +} + +void nfs_put_delegation(struct nfs_delegation *delegation) +{ + if (refcount_dec_and_test(&delegation->refcount)) + __nfs_free_delegation(delegation); +} + +/** + * nfs_mark_delegation_referenced - set delegation's REFERENCED flag + * @delegation: delegation to process + * + */ +void nfs_mark_delegation_referenced(struct nfs_delegation *delegation) +{ + set_bit(NFS_DELEGATION_REFERENCED, &delegation->flags); +} + +static void nfs_mark_return_delegation(struct nfs_server *server, + struct nfs_delegation *delegation) +{ + spin_lock(&server->delegations_lock); + if (list_empty(&delegation->entry)) + refcount_inc(&delegation->refcount); + list_move_tail(&delegation->entry, &server->delegations_return); + spin_unlock(&server->delegations_lock); + + set_bit(NFS4CLNT_DELEGRETURN, &server->nfs_client->cl_state); +} + +static bool nfs4_is_valid_delegation(const struct nfs_delegation *delegation, + fmode_t type) +{ + if (delegation != NULL && (delegation->type & type) == type && + !test_bit(NFS_DELEGATION_REVOKED, &delegation->flags) && + !test_bit(NFS_DELEGATION_RETURNING, &delegation->flags)) + return true; + return false; +} + +struct nfs_delegation *nfs4_get_valid_delegation(const struct inode *inode) +{ + struct nfs_delegation *delegation; + + rcu_read_lock(); + delegation = rcu_dereference(NFS_I(inode)->delegation); + if (!nfs4_is_valid_delegation(delegation, 0) || + !refcount_inc_not_zero(&delegation->refcount)) + delegation = NULL; + rcu_read_unlock(); + + return delegation; +} + +static int nfs4_do_check_delegation(struct inode *inode, fmode_t type, + int flags, bool mark) +{ + struct nfs_delegation *delegation; + int ret = 0; + + type &= FMODE_READ|FMODE_WRITE; + rcu_read_lock(); + delegation = rcu_dereference(NFS_I(inode)->delegation); + if (nfs4_is_valid_delegation(delegation, type)) { + if (mark) + nfs_mark_delegation_referenced(delegation); + ret = 1; + if ((flags & NFS_DELEGATION_FLAG_TIME) && + !test_bit(NFS_DELEGATION_DELEGTIME, &delegation->flags)) + ret = 0; + } + rcu_read_unlock(); + return ret; +} +/** + * nfs4_have_delegation - check if inode has a delegation, mark it + * NFS_DELEGATION_REFERENCED if there is one. + * @inode: inode to check + * @type: delegation types to check for + * @flags: various modifiers + * + * Returns one if inode has the indicated delegation, otherwise zero. + */ +int nfs4_have_delegation(struct inode *inode, fmode_t type, int flags) +{ + if (S_ISDIR(inode->i_mode) && !directory_delegations) + nfs4_inode_set_return_delegation_on_close(inode); + return nfs4_do_check_delegation(inode, type, flags, true); +} + +/* + * nfs4_check_delegation - check if inode has a delegation, do not mark + * NFS_DELEGATION_REFERENCED if it has one. + */ +int nfs4_check_delegation(struct inode *inode, fmode_t type) +{ + return nfs4_do_check_delegation(inode, type, 0, false); +} + +static int nfs_delegation_claim_locks(struct nfs4_state *state, const nfs4_stateid *stateid) +{ + struct inode *inode = state->inode; + struct nfs_inode *nfsi = NFS_I(inode); + struct file_lock *fl; + struct file_lock_context *flctx = locks_inode_context(inode); + struct list_head *list; + int status = 0; + + if (flctx == NULL) + goto out; + + list = &flctx->flc_posix; + + /* Guard against reclaim and new lock/unlock calls */ + down_write(&nfsi->rwsem); + spin_lock(&flctx->flc_lock); +restart: + for_each_file_lock(fl, list) { + if (nfs_file_open_context(fl->c.flc_file)->state != state) + continue; + spin_unlock(&flctx->flc_lock); + status = nfs4_lock_delegation_recall(fl, state, stateid); + if (status < 0) { + up_write(&nfsi->rwsem); + goto out; + } + spin_lock(&flctx->flc_lock); + } + if (list == &flctx->flc_posix) { + list = &flctx->flc_flock; + goto restart; + } + spin_unlock(&flctx->flc_lock); + up_write(&nfsi->rwsem); +out: + return status; +} + +static int nfs_delegation_claim_opens(struct inode *inode, + const nfs4_stateid *stateid, fmode_t type) +{ + struct nfs_inode *nfsi = NFS_I(inode); + struct nfs_open_context *ctx; + struct nfs4_state_owner *sp; + struct nfs4_state *state; + int err; + +again: + rcu_read_lock(); + list_for_each_entry_rcu(ctx, &nfsi->open_files, list) { + state = ctx->state; + if (state == NULL) + continue; + if (!test_bit(NFS_DELEGATED_STATE, &state->flags)) + continue; + if (!nfs4_valid_open_stateid(state)) + continue; + if (!nfs4_stateid_match(&state->stateid, stateid)) + continue; + if (!get_nfs_open_context(ctx)) + continue; + rcu_read_unlock(); + sp = state->owner; + /* Block nfs4_proc_unlck */ + mutex_lock(&sp->so_delegreturn_mutex); + err = nfs4_open_delegation_recall(ctx, state, stateid); + if (!err) + err = nfs_delegation_claim_locks(state, stateid); + mutex_unlock(&sp->so_delegreturn_mutex); + put_nfs_open_context(ctx); + if (err != 0) + return err; + goto again; + } + rcu_read_unlock(); + return 0; +} + +/** + * nfs_inode_reclaim_delegation - process a delegation reclaim request + * @inode: inode to process + * @cred: credential to use for request + * @type: delegation type + * @stateid: delegation stateid + * @pagemod_limit: write delegation "space_limit" + * @deleg_type: raw delegation type + * + */ +void nfs_inode_reclaim_delegation(struct inode *inode, const struct cred *cred, + fmode_t type, const nfs4_stateid *stateid, + unsigned long pagemod_limit, u32 deleg_type) +{ + struct nfs_delegation *delegation; + const struct cred *oldcred = NULL; + + rcu_read_lock(); + delegation = rcu_dereference(NFS_I(inode)->delegation); + if (!delegation) { + rcu_read_unlock(); + nfs_inode_set_delegation(inode, cred, type, stateid, + pagemod_limit, deleg_type); + return; + } + + spin_lock(&delegation->lock); + nfs4_stateid_copy(&delegation->stateid, stateid); + delegation->type = type; + delegation->pagemod_limit = pagemod_limit; + oldcred = delegation->cred; + delegation->cred = get_cred(cred); + switch (deleg_type) { + case NFS4_OPEN_DELEGATE_READ_ATTRS_DELEG: + case NFS4_OPEN_DELEGATE_WRITE_ATTRS_DELEG: + set_bit(NFS_DELEGATION_DELEGTIME, &delegation->flags); + break; + default: + clear_bit(NFS_DELEGATION_DELEGTIME, &delegation->flags); + } + clear_bit(NFS_DELEGATION_NEED_RECLAIM, &delegation->flags); + if (test_and_clear_bit(NFS_DELEGATION_REVOKED, &delegation->flags)) + atomic_long_inc(&NFS_SERVER(inode)->nr_active_delegations); + spin_unlock(&delegation->lock); + rcu_read_unlock(); + put_cred(oldcred); + trace_nfs4_reclaim_delegation(inode, type); +} + +static int nfs_do_return_delegation(struct inode *inode, + struct nfs_delegation *delegation, + int issync) +{ + const struct cred *cred; + int res = 0; + + if (!test_bit(NFS_DELEGATION_REVOKED, &delegation->flags)) { + spin_lock(&delegation->lock); + cred = get_cred(delegation->cred); + spin_unlock(&delegation->lock); + res = nfs4_proc_delegreturn(inode, cred, &delegation->stateid, + delegation, issync); + put_cred(cred); + } + return res; +} + +static struct inode *nfs_delegation_grab_inode(struct nfs_delegation *delegation) +{ + struct inode *inode = NULL; + + spin_lock(&delegation->lock); + if (delegation->inode != NULL) + inode = igrab(delegation->inode); + spin_unlock(&delegation->lock); + return inode; +} + +static struct nfs_delegation * +nfs_start_delegation_return(struct nfs_inode *nfsi) +{ + struct nfs_delegation *delegation; + bool return_now = false; + + rcu_read_lock(); + delegation = rcu_dereference(nfsi->delegation); + if (!delegation || !refcount_inc_not_zero(&delegation->refcount)) { + rcu_read_unlock(); + return NULL; + } + rcu_read_unlock(); + + spin_lock(&delegation->lock); + if (delegation->inode && + !test_and_set_bit(NFS_DELEGATION_RETURNING, &delegation->flags)) + return_now = true; + spin_unlock(&delegation->lock); + + if (!return_now) { + nfs_put_delegation(delegation); + return NULL; + } + nfs_clear_verifier_delegated(&nfsi->vfs_inode); + return delegation; +} + +static bool +nfs_detach_delegations_locked(struct nfs_inode *nfsi, + struct nfs_delegation *delegation, + struct nfs_client *clp) +{ + lockdep_assert_held(&clp->cl_lock); + + trace_nfs4_detach_delegation(&nfsi->vfs_inode, delegation->type); + + spin_lock(&delegation->lock); + if (!delegation->inode) { + spin_unlock(&delegation->lock); + return false; + } + hlist_del_init_rcu(&delegation->hash); + list_del_rcu(&delegation->super_list); + delegation->inode = NULL; + rcu_assign_pointer(nfsi->delegation, NULL); + spin_unlock(&delegation->lock); + clear_bit(NFS_INO_REQ_DIR_DELEG, &nfsi->flags); + return true; +} + +static bool nfs_detach_delegation(struct nfs_inode *nfsi, + struct nfs_delegation *delegation, + struct nfs_server *server) +{ + struct nfs_client *clp = server->nfs_client; + struct nfs_delegation *deleg_cur; + bool ret = false; + + spin_lock(&clp->cl_lock); + deleg_cur = rcu_dereference_protected(nfsi->delegation, + lockdep_is_held(&clp->cl_lock)); + if (delegation == deleg_cur) + ret = nfs_detach_delegations_locked(nfsi, delegation, clp); + spin_unlock(&clp->cl_lock); + return ret; +} + +static void +nfs_update_delegation_cred(struct nfs_delegation *delegation, + const struct cred *cred) +{ + const struct cred *old; + + if (cred_fscmp(delegation->cred, cred) != 0) { + old = xchg(&delegation->cred, get_cred(cred)); + put_cred(old); + } +} + +static void +nfs_update_inplace_delegation(struct nfs_server *server, + struct nfs_delegation *delegation, + const struct nfs_delegation *update) +{ + if (nfs4_stateid_is_newer(&update->stateid, &delegation->stateid)) { + delegation->stateid.seqid = update->stateid.seqid; + smp_wmb(); + delegation->type = update->type; + delegation->pagemod_limit = update->pagemod_limit; + if (test_bit(NFS_DELEGATION_REVOKED, &delegation->flags)) { + delegation->change_attr = update->change_attr; + nfs_update_delegation_cred(delegation, update->cred); + /* smp_mb__before_atomic() is implicit due to xchg() */ + clear_bit(NFS_DELEGATION_REVOKED, &delegation->flags); + atomic_long_inc(&server->nr_active_delegations); + } + } +} + +/** + * nfs_inode_set_delegation - set up a delegation on an inode + * @inode: inode to which delegation applies + * @cred: cred to use for subsequent delegation processing + * @type: delegation type + * @stateid: delegation stateid + * @pagemod_limit: write delegation "space_limit" + * @deleg_type: raw delegation type + * + * Returns zero on success, or a negative errno value. + */ +int nfs_inode_set_delegation(struct inode *inode, const struct cred *cred, + fmode_t type, const nfs4_stateid *stateid, + unsigned long pagemod_limit, u32 deleg_type) +{ + struct nfs_server *server = NFS_SERVER(inode); + struct nfs_client *clp = server->nfs_client; + struct nfs_inode *nfsi = NFS_I(inode); + struct nfs_delegation *delegation, *old_delegation; + struct nfs_delegation *freeme = NULL; + int status = 0; + + delegation = kmalloc_obj(*delegation, GFP_KERNEL_ACCOUNT); + if (delegation == NULL) + return -ENOMEM; + nfs4_stateid_copy(&delegation->stateid, stateid); + refcount_set(&delegation->refcount, 1); + delegation->type = type; + delegation->pagemod_limit = pagemod_limit; + delegation->change_attr = inode_peek_iversion_raw(inode); + delegation->cred = get_cred(cred); + delegation->inode = inode; + delegation->flags = 1<entry); + switch (deleg_type) { + case NFS4_OPEN_DELEGATE_READ_ATTRS_DELEG: + case NFS4_OPEN_DELEGATE_WRITE_ATTRS_DELEG: + delegation->flags |= BIT(NFS_DELEGATION_DELEGTIME); + } + delegation->test_gen = 0; + spin_lock_init(&delegation->lock); + + spin_lock(&clp->cl_lock); + old_delegation = rcu_dereference_protected(nfsi->delegation, + lockdep_is_held(&clp->cl_lock)); + if (old_delegation == NULL) + goto add_new; + /* Is this an update of the existing delegation? */ + if (nfs4_stateid_match_other(&old_delegation->stateid, + &delegation->stateid)) { + spin_lock(&old_delegation->lock); + nfs_update_inplace_delegation(server, old_delegation, + delegation); + spin_unlock(&old_delegation->lock); + goto out; + } + if (!test_bit(NFS_DELEGATION_REVOKED, &old_delegation->flags)) { + /* + * Deal with broken servers that hand out two + * delegations for the same file. + * Allow for upgrades to a WRITE delegation, but + * nothing else. + */ + dfprintk(FILE, "%s: server %s handed out " + "a duplicate delegation!\n", + __func__, clp->cl_hostname); + if (delegation->type == old_delegation->type || + !(delegation->type & FMODE_WRITE)) { + freeme = delegation; + delegation = NULL; + goto out; + } + if (test_and_set_bit(NFS_DELEGATION_RETURNING, + &old_delegation->flags)) + goto out; + } + if (!nfs_detach_delegations_locked(nfsi, old_delegation, clp)) + goto out; + freeme = old_delegation; +add_new: + /* + * If we didn't revalidate the change attribute before setting + * the delegation, then pre-emptively ask for a full attribute + * cache revalidation. + */ + spin_lock(&inode->i_lock); + if (NFS_I(inode)->cache_validity & NFS_INO_INVALID_CHANGE) + nfs_set_cache_invalid(inode, + NFS_INO_INVALID_ATIME | NFS_INO_INVALID_CTIME | + NFS_INO_INVALID_MTIME | NFS_INO_INVALID_SIZE | + NFS_INO_INVALID_BLOCKS | NFS_INO_INVALID_NLINK | + NFS_INO_INVALID_OTHER | NFS_INO_INVALID_DATA | + NFS_INO_INVALID_ACCESS | NFS_INO_INVALID_ACL | + NFS_INO_INVALID_XATTR); + spin_unlock(&inode->i_lock); + + list_add_tail_rcu(&delegation->super_list, &server->delegations); + hlist_add_head_rcu(&delegation->hash, + nfs_delegation_hash(server, &NFS_I(inode)->fh)); + rcu_assign_pointer(nfsi->delegation, delegation); + delegation = NULL; + + atomic_long_inc(&server->nr_active_delegations); + + trace_nfs4_set_delegation(inode, type); + + /* If we hold writebacks and have delegated mtime then update */ + if (deleg_type == NFS4_OPEN_DELEGATE_WRITE_ATTRS_DELEG && + nfs_have_writebacks(inode)) + nfs_update_delegated_mtime(inode); +out: + spin_unlock(&clp->cl_lock); + if (delegation != NULL) + __nfs_free_delegation(delegation); + if (freeme != NULL) { + nfs_do_return_delegation(inode, freeme, 0); + nfs_mark_delegation_revoked(server, freeme); + nfs_put_delegation(freeme); + } + return status; +} + +/* + * Basic procedure for returning a delegation to the server. + * If @issync is set, wait until state recovery has finished. Otherwise + * return -EAGAIN to the caller if we need more time. + */ +static int nfs_end_delegation_return(struct inode *inode, + struct nfs_delegation *delegation, bool issync) +{ + struct nfs_server *server = NFS_SERVER(inode); + unsigned int mode = O_WRONLY | O_RDWR; + int err = 0; + + /* Directory delegations don't require any state recovery */ + if (!S_ISREG(inode->i_mode)) + goto out_return; + + if (!issync) + mode |= O_NONBLOCK; + /* Recall of any remaining application leases */ + err = break_lease(inode, mode); + + while (err == 0) { + if (test_bit(NFS_DELEGATION_REVOKED, &delegation->flags)) + break; + err = nfs_delegation_claim_opens(inode, &delegation->stateid, + delegation->type); + if (!err) + break; + if (err != -EAGAIN) + goto abort; + if (!issync) + goto delay; + + /* + * Guard against state recovery + */ + err = nfs4_wait_clnt_recover(server->nfs_client); + } + +out_return: + return nfs_do_return_delegation(inode, delegation, issync); +delay: + spin_lock(&server->delegations_lock); + if (list_empty(&delegation->entry)) + refcount_inc(&delegation->refcount); + list_move_tail(&delegation->entry, &server->delegations_return); + spin_unlock(&server->delegations_lock); + set_bit(NFS4CLNT_DELEGRETURN_DELAYED, &server->nfs_client->cl_state); +abort: + clear_bit(NFS_DELEGATION_RETURNING, &delegation->flags); + return err; +} + +static int nfs_return_one_delegation(struct nfs_server *server) +{ + struct nfs_delegation *delegation; + struct inode *inode; + int err = 0; + + spin_lock(&server->delegations_lock); + delegation = list_first_entry_or_null(&server->delegations_return, + struct nfs_delegation, entry); + if (!delegation) { + spin_unlock(&server->delegations_lock); + return 0; /* no more delegations */ + } + list_del_init(&delegation->entry); + spin_unlock(&server->delegations_lock); + + spin_lock(&delegation->lock); + inode = delegation->inode; + if (!inode || !igrab(inode)) { + spin_unlock(&delegation->lock); + goto out_put_delegation; + } + if (test_bit(NFS_DELEGATION_REVOKED, &delegation->flags) || + test_and_set_bit(NFS_DELEGATION_RETURNING, &delegation->flags)) { + spin_unlock(&delegation->lock); + goto out_put_inode; + } + spin_unlock(&delegation->lock); + + nfs_clear_verifier_delegated(inode); + + err = nfs_end_delegation_return(inode, delegation, false); + +out_put_inode: + iput(inode); +out_put_delegation: + nfs_put_delegation(delegation); + if (err) + return err; + return 1; /* keep going */ +} + +static int nfs_server_return_marked_delegations(struct nfs_server *server, + void __always_unused *data) +{ + int err; + + while ((err = nfs_return_one_delegation(server)) > 0) + cond_resched(); + return err; +} + +static inline bool nfs_delegations_over_limit(struct nfs_server *server) +{ + return !list_empty_careful(&server->delegations_lru) && + atomic_long_read(&server->nr_active_delegations) > + nfs_delegation_watermark; +} + +static void nfs_delegations_return_from_lru(struct nfs_server *server) +{ + struct nfs_delegation *d, *n; + unsigned int pass = 0; + bool moved = false; + +retry: + spin_lock(&server->delegations_lock); + list_for_each_entry_safe(d, n, &server->delegations_lru, entry) { + if (!nfs_delegations_over_limit(server)) + break; + if (pass == 0 && test_bit(NFS_DELEGATION_REFERENCED, &d->flags)) + continue; + list_move_tail(&d->entry, &server->delegations_return); + moved = true; + } + spin_unlock(&server->delegations_lock); + + /* + * If we are still over the limit, try to reclaim referenced delegations + * as well. + */ + if (pass == 0 && nfs_delegations_over_limit(server)) { + pass++; + goto retry; + } + + if (moved) { + set_bit(NFS4CLNT_DELEGRETURN, &server->nfs_client->cl_state); + nfs4_schedule_state_manager(server->nfs_client); + } +} + +static void nfs_delegation_add_lru(struct nfs_server *server, + struct nfs_delegation *delegation) +{ + spin_lock(&server->delegations_lock); + if (list_empty(&delegation->entry)) { + list_add_tail(&delegation->entry, &server->delegations_lru); + refcount_inc(&delegation->refcount); + } + spin_unlock(&server->delegations_lock); + + if (nfs_delegations_over_limit(server)) + nfs_delegations_return_from_lru(server); +} + +static bool nfs_server_clear_delayed_delegations(struct nfs_server *server) +{ + bool ret = false; + + if (list_empty_careful(&server->delegations_delayed)) + return false; + + spin_lock(&server->delegations_lock); + if (!list_empty(&server->delegations_delayed)) { + list_splice_tail_init(&server->delegations_delayed, + &server->delegations_return); + ret = true; + } + spin_unlock(&server->delegations_lock); + + return ret; +} + +static bool nfs_client_clear_delayed_delegations(struct nfs_client *clp) +{ + struct nfs_server *server; + bool ret = false; + + if (!test_and_clear_bit(NFS4CLNT_DELEGRETURN_DELAYED, &clp->cl_state)) + return false; + + rcu_read_lock(); + list_for_each_entry_rcu (server, &clp->cl_superblocks, client_link) { + if (nfs_server_clear_delayed_delegations(server)) + ret = true; + } + rcu_read_unlock(); + + if (ret) + set_bit(NFS4CLNT_DELEGRETURN, &clp->cl_state); + return ret; +} + +/** + * nfs_client_return_marked_delegations - return previously marked delegations + * @clp: nfs_client to process + * + * Note that this function is designed to be called by the state + * manager thread. For this reason, it cannot flush the dirty data, + * since that could deadlock in case of a state recovery error. + * + * Returns zero on success, or a negative errno value. + */ +int nfs_client_return_marked_delegations(struct nfs_client *clp) +{ + int err = nfs_client_for_each_server( + clp, nfs_server_return_marked_delegations, NULL); + if (err) + return err; + /* If a return was delayed, sleep to prevent hard looping */ + if (nfs_client_clear_delayed_delegations(clp)) + ssleep(1); + return 0; +} + +/** + * nfs_inode_evict_delegation - return delegation, don't reclaim opens + * @inode: inode to process + * + * Does not protect against delegation reclaims, therefore really only safe + * to be called from nfs4_clear_inode(). Guaranteed to always free + * the delegation structure. + */ +void nfs_inode_evict_delegation(struct inode *inode) +{ + struct nfs_inode *nfsi = NFS_I(inode); + struct nfs_server *server = NFS_SERVER(inode); + struct nfs_delegation *delegation; + + rcu_read_lock(); + delegation = rcu_dereference(nfsi->delegation); + if (delegation && !nfs_detach_delegation(nfsi, delegation, server)) + delegation = NULL; + rcu_read_unlock(); + + if (!delegation) + return; + + set_bit(NFS_DELEGATION_RETURNING, &delegation->flags); + nfs_do_return_delegation(inode, delegation, 1); + nfs_mark_delegation_revoked(server, delegation); + nfs_put_delegation(delegation); +} + +/** + * nfs4_inode_return_delegation - synchronously return a delegation + * @inode: inode to process + * + * This routine will always flush any dirty data to disk on the + * assumption that if we need to return the delegation, then + * we should stop caching. + * + * Returns zero on success, or a negative errno value. + */ +void nfs4_inode_return_delegation(struct inode *inode) +{ + struct nfs_inode *nfsi = NFS_I(inode); + struct nfs_delegation *delegation; + + delegation = nfs_start_delegation_return(nfsi); + if (!delegation) + return; + + /* Synchronous recall of any application leases */ + break_lease(inode, O_WRONLY | O_RDWR); + if (S_ISREG(inode->i_mode)) + nfs_wb_all(inode); + nfs_end_delegation_return(inode, delegation, true); + nfs_put_delegation(delegation); +} + +/** + * nfs4_inode_set_return_delegation_on_close - asynchronously return a delegation + * @inode: inode to process + * + * This routine is called to request that the delegation be returned as soon + * as the file is closed. If the file is already closed, the delegation is + * immediately returned. + */ +void nfs4_inode_set_return_delegation_on_close(struct inode *inode) +{ + struct nfs_delegation *delegation; + bool return_now = false; + + if (!inode) + return; + + delegation = nfs4_get_valid_delegation(inode); + if (!delegation) + return; + + spin_lock(&delegation->lock); + if (!delegation->inode) + goto out_unlock; + if (list_empty(&NFS_I(inode)->open_files) && + !test_and_set_bit(NFS_DELEGATION_RETURNING, &delegation->flags)) + return_now = true; + else + set_bit(NFS_DELEGATION_RETURN_IF_CLOSED, &delegation->flags); +out_unlock: + spin_unlock(&delegation->lock); + if (return_now) { + nfs_clear_verifier_delegated(inode); + nfs_end_delegation_return(inode, delegation, false); + } + nfs_put_delegation(delegation); +} + +/** + * nfs4_inode_return_delegation_on_close - asynchronously return a delegation + * @inode: inode to process + * + * This routine is called on file close in order to determine if the + * inode delegation needs to be returned immediately. + */ +void nfs4_inode_return_delegation_on_close(struct inode *inode) +{ + struct nfs_server *server = NFS_SERVER(inode); + struct nfs_delegation *delegation; + bool return_now = false; + + delegation = nfs4_get_valid_delegation(inode); + if (!delegation) + return; + + if (test_bit(NFS_DELEGATION_RETURN_IF_CLOSED, &delegation->flags)) { + spin_lock(&delegation->lock); + if (delegation->inode && + list_empty(&NFS_I(inode)->open_files) && + !test_and_set_bit(NFS_DELEGATION_RETURNING, &delegation->flags)) { + clear_bit(NFS_DELEGATION_RETURN_IF_CLOSED, &delegation->flags); + return_now = true; + } + spin_unlock(&delegation->lock); + } + + if (return_now) { + nfs_clear_verifier_delegated(inode); + nfs_end_delegation_return(inode, delegation, false); + } else { + nfs_delegation_add_lru(server, delegation); + } + nfs_put_delegation(delegation); +} + +/** + * nfs4_inode_make_writeable + * @inode: pointer to inode + * + * Make the inode writeable by returning the delegation if necessary + */ +void nfs4_inode_make_writeable(struct inode *inode) +{ + struct nfs_delegation *delegation; + + delegation = nfs4_get_valid_delegation(inode); + if (!delegation) + return; + + if (!nfs4_has_session(NFS_SERVER(inode)->nfs_client) || + !(delegation->type & FMODE_WRITE)) + nfs4_inode_return_delegation(inode); + nfs_put_delegation(delegation); +} + +static void +nfs_mark_return_if_closed_delegation(struct nfs_server *server, + struct nfs_delegation *delegation) +{ + struct inode *inode; + + if (!list_empty_careful(&server->delegations_return) || + test_bit(NFS_DELEGATION_RETURN_IF_CLOSED, &delegation->flags)) + return; + spin_lock(&delegation->lock); + inode = delegation->inode; + if (!inode) + goto out; + if (list_empty(&NFS_I(inode)->open_files)) + nfs_mark_return_delegation(server, delegation); + else + set_bit(NFS_DELEGATION_RETURN_IF_CLOSED, &delegation->flags); +out: + spin_unlock(&delegation->lock); +} + +static bool nfs_server_mark_return_all_delegations(struct nfs_server *server) +{ + struct nfs_delegation *delegation; + bool ret = false; + + list_for_each_entry_rcu(delegation, &server->delegations, super_list) { + nfs_mark_return_delegation(server, delegation); + ret = true; + } + return ret; +} + +static void nfs_delegation_run_state_manager(struct nfs_client *clp) +{ + if (test_bit(NFS4CLNT_DELEGRETURN, &clp->cl_state)) + nfs4_schedule_state_manager(clp); +} + +/** + * nfs_expire_all_delegations + * @clp: client to process + * + */ +void nfs_expire_all_delegations(struct nfs_client *clp) +{ + struct nfs_server *server; + + rcu_read_lock(); + list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) + nfs_server_mark_return_all_delegations(server); + rcu_read_unlock(); + + nfs_delegation_run_state_manager(clp); +} + +/** + * nfs_server_return_all_delegations - return delegations for one superblock + * @server: pointer to nfs_server to process + * + */ +void nfs_server_return_all_delegations(struct nfs_server *server) +{ + struct nfs_client *clp = server->nfs_client; + bool need_wait; + + if (clp == NULL) + return; + + rcu_read_lock(); + need_wait = nfs_server_mark_return_all_delegations(server); + rcu_read_unlock(); + + if (need_wait) { + nfs4_schedule_state_manager(clp); + nfs4_wait_clnt_recover(clp); + } +} + +static void nfs_mark_return_unused_delegation_types(struct nfs_server *server, + fmode_t flags) +{ + struct nfs_delegation *delegation; + + list_for_each_entry_rcu(delegation, &server->delegations, super_list) { + if ((delegation->type == (FMODE_READ|FMODE_WRITE)) && !(flags & FMODE_WRITE)) + continue; + if (delegation->type & flags) + nfs_mark_return_if_closed_delegation(server, delegation); + } +} + +void nfs_expire_unused_delegation_types(struct nfs_client *clp, fmode_t flags) +{ + struct nfs_server *server; + + rcu_read_lock(); + list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) + nfs_mark_return_unused_delegation_types(server, flags); + rcu_read_unlock(); + + nfs_delegation_run_state_manager(clp); +} + +static void nfs_revoke_delegation(struct inode *inode, + const nfs4_stateid *stateid) +{ + struct nfs_delegation *delegation; + nfs4_stateid tmp; + bool ret = false; + + rcu_read_lock(); + delegation = rcu_dereference(NFS_I(inode)->delegation); + if (delegation == NULL) + goto out; + if (stateid == NULL) { + nfs4_stateid_copy(&tmp, &delegation->stateid); + stateid = &tmp; + } else { + if (!nfs4_stateid_match_other(stateid, &delegation->stateid)) + goto out; + spin_lock(&delegation->lock); + if (stateid->seqid) { + if (nfs4_stateid_is_newer(&delegation->stateid, stateid)) { + spin_unlock(&delegation->lock); + goto out; + } + delegation->stateid.seqid = stateid->seqid; + } + spin_unlock(&delegation->lock); + } + nfs_mark_delegation_revoked(NFS_SERVER(inode), delegation); + ret = true; +out: + rcu_read_unlock(); + if (ret) + nfs_inode_find_state_and_recover(inode, stateid); +} + +void nfs_delegation_mark_returned(struct inode *inode, + const nfs4_stateid *stateid) +{ + struct nfs_delegation *delegation; + + if (!inode) + return; + + rcu_read_lock(); + delegation = rcu_dereference(NFS_I(inode)->delegation); + if (!delegation) + goto out_rcu_unlock; + + spin_lock(&delegation->lock); + if (!nfs4_stateid_match_other(stateid, &delegation->stateid)) + goto out_spin_unlock; + if (stateid->seqid) { + /* If delegation->stateid is newer, dont mark as returned */ + if (nfs4_stateid_is_newer(&delegation->stateid, stateid)) + goto out_clear_returning; + if (delegation->stateid.seqid != stateid->seqid) + delegation->stateid.seqid = stateid->seqid; + } + + nfs_mark_delegation_revoked(NFS_SERVER(inode), delegation); + clear_bit(NFS_DELEGATION_RETURNING, &delegation->flags); + spin_unlock(&delegation->lock); + if (nfs_detach_delegation(NFS_I(inode), delegation, NFS_SERVER(inode))) + nfs_put_delegation(delegation); + goto out_rcu_unlock; + +out_clear_returning: + clear_bit(NFS_DELEGATION_RETURNING, &delegation->flags); +out_spin_unlock: + spin_unlock(&delegation->lock); +out_rcu_unlock: + rcu_read_unlock(); + + nfs_inode_find_state_and_recover(inode, stateid); +} + +/** + * nfs_remove_bad_delegation - handle delegations that are unusable + * @inode: inode to process + * @stateid: the delegation's stateid + * + * If the server ACK-ed our FREE_STATEID then clean + * up the delegation, else mark and keep the revoked state. + */ +void nfs_remove_bad_delegation(struct inode *inode, + const nfs4_stateid *stateid) +{ + if (stateid && stateid->type == NFS4_FREED_STATEID_TYPE) + nfs_delegation_mark_returned(inode, stateid); + else + nfs_revoke_delegation(inode, stateid); +} +EXPORT_SYMBOL_GPL(nfs_remove_bad_delegation); + +static bool nfs_mark_return_unreferenced_delegations(struct nfs_server *server) +{ + struct nfs_delegation *d, *n; + bool marked = false; + + spin_lock(&server->delegations_lock); + list_for_each_entry_safe(d, n, &server->delegations_lru, entry) { + if (test_and_clear_bit(NFS_DELEGATION_REFERENCED, &d->flags)) + continue; + list_move_tail(&d->entry, &server->delegations_return); + marked = true; + } + spin_unlock(&server->delegations_lock); + + return marked; +} + +/** + * nfs_expire_unreferenced_delegations - Eliminate unused delegations + * @clp: nfs_client to process + * + */ +void nfs_expire_unreferenced_delegations(struct nfs_client *clp) +{ + struct nfs_server *server; + bool marked = false; + + rcu_read_lock(); + list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) + marked |= nfs_mark_return_unreferenced_delegations(server); + rcu_read_unlock(); + + if (marked) { + set_bit(NFS4CLNT_DELEGRETURN, &clp->cl_state); + nfs4_schedule_state_manager(clp); + } +} + +/** + * nfs_async_inode_return_delegation - asynchronously return a delegation + * @inode: inode to process + * @stateid: state ID information + * + * Returns zero on success, or a negative errno value. + */ +int nfs_async_inode_return_delegation(struct inode *inode, + const nfs4_stateid *stateid) +{ + struct nfs_server *server = NFS_SERVER(inode); + struct nfs_client *clp = server->nfs_client; + struct nfs_delegation *delegation; + + delegation = nfs4_get_valid_delegation(inode); + if (!delegation) + return -ENOENT; + + if (stateid != NULL && + !clp->cl_mvops->match_stateid(&delegation->stateid, stateid)) { + nfs_put_delegation(delegation); + return -ENOENT; + } + + nfs_mark_return_delegation(server, delegation); + nfs_put_delegation(delegation); + + /* If there are any application leases or delegations, recall them */ + break_lease(inode, O_WRONLY | O_RDWR | O_NONBLOCK); + + nfs_delegation_run_state_manager(clp); + return 0; +} + +static struct inode * +nfs_delegation_find_inode_server(struct nfs_server *server, + const struct nfs_fh *fhandle) +{ + struct hlist_head *head = nfs_delegation_hash(server, fhandle); + struct nfs_delegation *delegation; + struct super_block *freeme = NULL; + struct inode *res = NULL; + + hlist_for_each_entry_rcu(delegation, head, hash) { + spin_lock(&delegation->lock); + if (delegation->inode != NULL && + !test_bit(NFS_DELEGATION_REVOKED, &delegation->flags) && + nfs_compare_fh(fhandle, &NFS_I(delegation->inode)->fh) == 0) { + if (nfs_sb_active(server->super)) { + freeme = server->super; + res = igrab(delegation->inode); + } + spin_unlock(&delegation->lock); + if (res != NULL) + return res; + if (freeme) { + rcu_read_unlock(); + nfs_sb_deactive(freeme); + rcu_read_lock(); + } + return ERR_PTR(-EAGAIN); + } + spin_unlock(&delegation->lock); + } + return ERR_PTR(-ENOENT); +} + +/** + * nfs_delegation_find_inode - retrieve the inode associated with a delegation + * @clp: client state handle + * @fhandle: filehandle from a delegation recall + * + * Returns pointer to inode matching "fhandle," or NULL if a matching inode + * cannot be found. + */ +struct inode *nfs_delegation_find_inode(struct nfs_client *clp, + const struct nfs_fh *fhandle) +{ + struct nfs_server *server; + struct inode *res; + + rcu_read_lock(); + list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { + res = nfs_delegation_find_inode_server(server, fhandle); + if (res != ERR_PTR(-ENOENT)) { + rcu_read_unlock(); + return res; + } + } + rcu_read_unlock(); + return ERR_PTR(-ENOENT); +} + +static void nfs_delegation_mark_reclaim_server(struct nfs_server *server) +{ + struct nfs_delegation *delegation; + + list_for_each_entry_rcu(delegation, &server->delegations, super_list) { + /* + * If the delegation may have been admin revoked, then we + * cannot reclaim it. + */ + if (test_bit(NFS_DELEGATION_TEST_EXPIRED, &delegation->flags)) + continue; + set_bit(NFS_DELEGATION_NEED_RECLAIM, &delegation->flags); + } +} + +/** + * nfs_delegation_mark_reclaim - mark all delegations as needing to be reclaimed + * @clp: nfs_client to process + * + */ +void nfs_delegation_mark_reclaim(struct nfs_client *clp) +{ + struct nfs_server *server; + + rcu_read_lock(); + list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) + nfs_delegation_mark_reclaim_server(server); + rcu_read_unlock(); +} + +static int nfs_server_reap_unclaimed_delegations(struct nfs_server *server, + void __always_unused *data) +{ + struct nfs_delegation *delegation; + struct inode *inode; +restart: + rcu_read_lock(); + list_for_each_entry_rcu(delegation, &server->delegations, super_list) { + if (test_bit(NFS_DELEGATION_RETURNING, + &delegation->flags) || + test_bit(NFS_DELEGATION_NEED_RECLAIM, + &delegation->flags) == 0) + continue; + inode = nfs_delegation_grab_inode(delegation); + if (inode == NULL) + continue; + delegation = nfs_start_delegation_return(NFS_I(inode)); + rcu_read_unlock(); + if (delegation != NULL) { + if (nfs_detach_delegation(NFS_I(inode), delegation, + server)) { + nfs_mark_delegation_revoked(server, delegation); + nfs_put_delegation(delegation); + } + /* Match nfs_start_delegation_return */ + nfs_put_delegation(delegation); + } + iput(inode); + cond_resched(); + goto restart; + } + rcu_read_unlock(); + return 0; +} + +/** + * nfs_delegation_reap_unclaimed - reap unclaimed delegations after reboot recovery is done + * @clp: nfs_client to process + * + */ +void nfs_delegation_reap_unclaimed(struct nfs_client *clp) +{ + nfs_client_for_each_server(clp, nfs_server_reap_unclaimed_delegations, + NULL); +} + +static inline bool nfs4_server_rebooted(const struct nfs_client *clp) +{ + return (clp->cl_state & (BIT(NFS4CLNT_CHECK_LEASE) | + BIT(NFS4CLNT_LEASE_EXPIRED) | + BIT(NFS4CLNT_SESSION_RESET))) != 0; +} + +static void nfs_mark_test_expired_delegation(struct nfs_server *server, + struct nfs_delegation *delegation) +{ + if (delegation->stateid.type == NFS4_INVALID_STATEID_TYPE) + return; + clear_bit(NFS_DELEGATION_NEED_RECLAIM, &delegation->flags); + set_bit(NFS_DELEGATION_TEST_EXPIRED, &delegation->flags); + set_bit(NFS4SERV_DELEGATION_EXPIRED, &server->delegation_flags); + set_bit(NFS4CLNT_DELEGATION_EXPIRED, &server->nfs_client->cl_state); +} + +static void nfs_inode_mark_test_expired_delegation(struct nfs_server *server, + struct inode *inode) +{ + struct nfs_delegation *delegation; + + rcu_read_lock(); + delegation = rcu_dereference(NFS_I(inode)->delegation); + if (delegation) + nfs_mark_test_expired_delegation(server, delegation); + rcu_read_unlock(); + +} + +static void nfs_delegation_mark_test_expired_server(struct nfs_server *server) +{ + struct nfs_delegation *delegation; + + list_for_each_entry_rcu(delegation, &server->delegations, super_list) + nfs_mark_test_expired_delegation(server, delegation); +} + +/** + * nfs_mark_test_expired_all_delegations - mark all delegations for testing + * @clp: nfs_client to process + * + * Iterates through all the delegations associated with this server and + * marks them as needing to be checked for validity. + */ +void nfs_mark_test_expired_all_delegations(struct nfs_client *clp) +{ + struct nfs_server *server; + + rcu_read_lock(); + list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) + nfs_delegation_mark_test_expired_server(server); + rcu_read_unlock(); +} + +/** + * nfs_test_expired_all_delegations - test all delegations for a client + * @clp: nfs_client to process + * + * Helper for handling "recallable state revoked" status from server. + */ +void nfs_test_expired_all_delegations(struct nfs_client *clp) +{ + nfs_mark_test_expired_all_delegations(clp); + nfs4_schedule_state_manager(clp); +} + +static void +nfs_delegation_test_free_expired(struct inode *inode, + nfs4_stateid *stateid, + const struct cred *cred) +{ + struct nfs_server *server = NFS_SERVER(inode); + const struct nfs4_minor_version_ops *ops = server->nfs_client->cl_mvops; + int status; + + if (!cred) + return; + status = ops->test_and_free_expired(server, stateid, cred); + if (status == -NFS4ERR_EXPIRED || status == -NFS4ERR_BAD_STATEID) + nfs_remove_bad_delegation(inode, stateid); +} + +static int nfs_server_reap_expired_delegations(struct nfs_server *server, + void __always_unused *data) +{ + struct nfs_delegation *delegation; + struct inode *inode; + const struct cred *cred; + nfs4_stateid stateid; + unsigned long gen = ++server->delegation_gen; + + if (!test_and_clear_bit(NFS4SERV_DELEGATION_EXPIRED, + &server->delegation_flags)) + return 0; +restart: + rcu_read_lock(); + list_for_each_entry_rcu(delegation, &server->delegations, super_list) { + if (test_bit(NFS_DELEGATION_RETURNING, + &delegation->flags) || + test_bit(NFS_DELEGATION_TEST_EXPIRED, + &delegation->flags) == 0 || + delegation->test_gen == gen) + continue; + inode = nfs_delegation_grab_inode(delegation); + if (inode == NULL) + continue; + spin_lock(&delegation->lock); + cred = get_cred_rcu(delegation->cred); + nfs4_stateid_copy(&stateid, &delegation->stateid); + spin_unlock(&delegation->lock); + delegation->test_gen = gen; + clear_bit(NFS_DELEGATION_TEST_EXPIRED, &delegation->flags); + rcu_read_unlock(); + nfs_delegation_test_free_expired(inode, &stateid, cred); + put_cred(cred); + if (!nfs4_server_rebooted(server->nfs_client)) { + iput(inode); + cond_resched(); + goto restart; + } + nfs_inode_mark_test_expired_delegation(server,inode); + set_bit(NFS4SERV_DELEGATION_EXPIRED, &server->delegation_flags); + set_bit(NFS4CLNT_DELEGATION_EXPIRED, + &server->nfs_client->cl_state); + iput(inode); + return -EAGAIN; + } + rcu_read_unlock(); + return 0; +} + +/** + * nfs_reap_expired_delegations - reap expired delegations + * @clp: nfs_client to process + * + * Iterates through all the delegations associated with this server and + * checks if they have may have been revoked. This function is usually + * expected to be called in cases where the server may have lost its + * lease. + */ +void nfs_reap_expired_delegations(struct nfs_client *clp) +{ + nfs_client_for_each_server(clp, nfs_server_reap_expired_delegations, + NULL); +} + +void nfs_inode_find_delegation_state_and_recover(struct inode *inode, + const nfs4_stateid *stateid) +{ + struct nfs_client *clp = NFS_SERVER(inode)->nfs_client; + struct nfs_delegation *delegation; + bool found = false; + + rcu_read_lock(); + delegation = rcu_dereference(NFS_I(inode)->delegation); + if (delegation && + nfs4_stateid_match_or_older(&delegation->stateid, stateid) && + !test_bit(NFS_DELEGATION_REVOKED, &delegation->flags)) { + nfs_mark_test_expired_delegation(NFS_SERVER(inode), delegation); + found = true; + } + rcu_read_unlock(); + if (found) + nfs4_schedule_state_manager(clp); +} + +/** + * nfs_delegations_present - check for existence of delegations + * @clp: client state handle + * + * Returns one if there are any nfs_delegation structures attached + * to this nfs_client. + */ +int nfs_delegations_present(struct nfs_client *clp) +{ + struct nfs_server *server; + int ret = 0; + + rcu_read_lock(); + list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) + if (atomic_long_read(&server->nr_active_delegations) > 0) { + ret = 1; + break; + } + rcu_read_unlock(); + return ret; +} + +/** + * nfs4_refresh_delegation_stateid - Update delegation stateid seqid + * @dst: stateid to refresh + * @inode: inode to check + * + * Returns "true" and updates "dst->seqid" * if inode had a delegation + * that matches our delegation stateid. Otherwise "false" is returned. + */ +bool nfs4_refresh_delegation_stateid(nfs4_stateid *dst, struct inode *inode) +{ + struct nfs_delegation *delegation; + bool ret = false; + if (!inode) + goto out; + + rcu_read_lock(); + delegation = rcu_dereference(NFS_I(inode)->delegation); + if (delegation != NULL && + nfs4_stateid_match_other(dst, &delegation->stateid) && + nfs4_stateid_is_newer(&delegation->stateid, dst) && + !test_bit(NFS_DELEGATION_REVOKED, &delegation->flags)) { + dst->seqid = delegation->stateid.seqid; + ret = true; + } + rcu_read_unlock(); +out: + return ret; +} + +/** + * nfs4_copy_delegation_stateid - Copy inode's state ID information + * @inode: inode to check + * @flags: delegation type requirement + * @dst: stateid data structure to fill in + * @cred: optional argument to retrieve credential + * + * Returns "true" and fills in "dst->data" * if inode had a delegation, + * otherwise "false" is returned. + */ +bool nfs4_copy_delegation_stateid(struct inode *inode, fmode_t flags, + nfs4_stateid *dst, const struct cred **cred) +{ + struct nfs_inode *nfsi = NFS_I(inode); + struct nfs_delegation *delegation; + bool ret = false; + + flags &= FMODE_READ|FMODE_WRITE; + rcu_read_lock(); + delegation = rcu_dereference(nfsi->delegation); + if (!delegation) + goto out; + spin_lock(&delegation->lock); + ret = nfs4_is_valid_delegation(delegation, flags); + if (ret) { + nfs4_stateid_copy(dst, &delegation->stateid); + nfs_mark_delegation_referenced(delegation); + if (cred) + *cred = get_cred(delegation->cred); + } + spin_unlock(&delegation->lock); +out: + rcu_read_unlock(); + return ret; +} + +/** + * nfs4_delegation_flush_on_close - Check if we must flush file on close + * @inode: inode to check + * + * This function checks the number of outstanding writes to the file + * against the delegation 'space_limit' field to see if + * the spec requires us to flush the file on close. + */ +bool nfs4_delegation_flush_on_close(const struct inode *inode) +{ + struct nfs_inode *nfsi = NFS_I(inode); + struct nfs_delegation *delegation; + bool ret = true; + + rcu_read_lock(); + delegation = rcu_dereference(nfsi->delegation); + if (delegation == NULL || !(delegation->type & FMODE_WRITE)) + goto out; + if (atomic_long_read(&nfsi->nrequests) < delegation->pagemod_limit) + ret = false; +out: + rcu_read_unlock(); + return ret; +} + +int nfs4_delegation_hash_alloc(struct nfs_server *server) +{ + int delegation_buckets, i; + + delegation_buckets = roundup_pow_of_two(nfs_delegation_watermark / 16); + server->delegation_hash_mask = delegation_buckets - 1; + server->delegation_hash_table = kmalloc_objs(*server->delegation_hash_table, + delegation_buckets); + if (!server->delegation_hash_table) + return -ENOMEM; + for (i = 0; i < delegation_buckets; i++) + INIT_HLIST_HEAD(&server->delegation_hash_table[i]); + return 0; +} diff --git a/fs/nfs/delegation.h b/fs/nfs/delegation.h new file mode 100644 index 000000000..fba469995 --- /dev/null +++ b/fs/nfs/delegation.h @@ -0,0 +1,141 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * linux/fs/nfs/delegation.h + * + * Copyright (c) Trond Myklebust + * + * Definitions pertaining to NFS delegated files + */ +#ifndef FS_NFS_DELEGATION_H +#define FS_NFS_DELEGATION_H + +#if IS_ENABLED(CONFIG_NFS_V4) +/* + * NFSv4 delegation + */ +struct nfs_delegation { + struct hlist_node hash; + struct list_head super_list; + const struct cred *cred; + struct inode *inode; + nfs4_stateid stateid; + fmode_t type; + unsigned long pagemod_limit; + __u64 change_attr; + unsigned long test_gen; + unsigned long flags; + refcount_t refcount; + spinlock_t lock; + struct list_head entry; + struct rcu_head rcu; +}; + +enum { + NFS_DELEGATION_NEED_RECLAIM = 0, + NFS_DELEGATION_RETURN_IF_CLOSED, + NFS_DELEGATION_REFERENCED, + NFS_DELEGATION_RETURNING, + NFS_DELEGATION_REVOKED, + NFS_DELEGATION_TEST_EXPIRED, + NFS_DELEGATION_DELEGTIME, +}; + +int nfs_inode_set_delegation(struct inode *inode, const struct cred *cred, + fmode_t type, const nfs4_stateid *stateid, + unsigned long pagemod_limit, u32 deleg_type); +void nfs_inode_reclaim_delegation(struct inode *inode, const struct cred *cred, + fmode_t type, const nfs4_stateid *stateid, + unsigned long pagemod_limit, u32 deleg_type); +void nfs4_inode_return_delegation(struct inode *inode); +void nfs4_inode_return_delegation_on_close(struct inode *inode); +void nfs4_inode_set_return_delegation_on_close(struct inode *inode); +int nfs_async_inode_return_delegation(struct inode *inode, const nfs4_stateid *stateid); +void nfs_inode_evict_delegation(struct inode *inode); + +struct inode *nfs_delegation_find_inode(struct nfs_client *clp, const struct nfs_fh *fhandle); +void nfs_server_return_all_delegations(struct nfs_server *); +void nfs_expire_all_delegations(struct nfs_client *clp); +void nfs_expire_unused_delegation_types(struct nfs_client *clp, fmode_t flags); +void nfs_expire_unreferenced_delegations(struct nfs_client *clp); +int nfs_client_return_marked_delegations(struct nfs_client *clp); +int nfs_delegations_present(struct nfs_client *clp); +void nfs_remove_bad_delegation(struct inode *inode, const nfs4_stateid *stateid); +void nfs_delegation_mark_returned(struct inode *inode, const nfs4_stateid *stateid); + +void nfs_delegation_mark_reclaim(struct nfs_client *clp); +void nfs_delegation_reap_unclaimed(struct nfs_client *clp); + +void nfs_mark_test_expired_all_delegations(struct nfs_client *clp); +void nfs_test_expired_all_delegations(struct nfs_client *clp); +void nfs_reap_expired_delegations(struct nfs_client *clp); + +/* NFSv4 delegation-related procedures */ +int nfs4_proc_delegreturn(struct inode *inode, const struct cred *cred, + const nfs4_stateid *stateid, + struct nfs_delegation *delegation, int issync); +int nfs4_open_delegation_recall(struct nfs_open_context *ctx, struct nfs4_state *state, const nfs4_stateid *stateid); +int nfs4_lock_delegation_recall(struct file_lock *fl, struct nfs4_state *state, const nfs4_stateid *stateid); +bool nfs4_copy_delegation_stateid(struct inode *inode, fmode_t flags, nfs4_stateid *dst, const struct cred **cred); +bool nfs4_refresh_delegation_stateid(nfs4_stateid *dst, struct inode *inode); + +struct nfs_delegation *nfs4_get_valid_delegation(const struct inode *inode); +void nfs_put_delegation(struct nfs_delegation *delegation); +void nfs_mark_delegation_referenced(struct nfs_delegation *delegation); +int nfs4_have_delegation(struct inode *inode, fmode_t type, int flags); +int nfs4_check_delegation(struct inode *inode, fmode_t type); +bool nfs4_delegation_flush_on_close(const struct inode *inode); +void nfs_inode_find_delegation_state_and_recover(struct inode *inode, + const nfs4_stateid *stateid); +void nfs4_inode_make_writeable(struct inode *inode); + +#endif + +#define NFS_DELEGATION_FLAG_TIME BIT(1) + +void nfs_update_delegated_atime(struct inode *inode); +void nfs_update_delegated_mtime(struct inode *inode); +void nfs_update_delegated_mtime_locked(struct inode *inode); + +static inline int nfs_have_read_or_write_delegation(struct inode *inode) +{ + return NFS_PROTO(inode)->have_delegation(inode, FMODE_READ, 0); +} + +static inline int nfs_have_write_delegation(struct inode *inode) +{ + return NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE, 0); +} + +static inline int nfs_have_delegated_attributes(struct inode *inode) +{ + return NFS_PROTO(inode)->have_delegation(inode, FMODE_READ, 0); +} + +static inline int nfs_have_delegated_atime(struct inode *inode) +{ + return NFS_PROTO(inode)->have_delegation(inode, FMODE_READ, + NFS_DELEGATION_FLAG_TIME); +} + +static inline int nfs_have_delegated_mtime(struct inode *inode) +{ + return NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE, + NFS_DELEGATION_FLAG_TIME); +} + +extern bool directory_delegations; + +static inline void nfs_request_directory_delegation(struct inode *inode) +{ + if (S_ISDIR(inode->i_mode)) + set_bit(NFS_INO_REQ_DIR_DELEG, &NFS_I(inode)->flags); +} + +static inline bool nfs_have_directory_delegation(struct inode *inode) +{ + return S_ISDIR(inode->i_mode) && nfs_have_delegated_attributes(inode); +} + +int nfs4_delegation_hash_alloc(struct nfs_server *server); + +#endif diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c new file mode 100644 index 000000000..36f2e8588 --- /dev/null +++ b/fs/nfs/dir.c @@ -0,0 +1,3433 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/nfs/dir.c + * + * Copyright (C) 1992 Rick Sladkey + * + * nfs directory handling functions + * + * 10 Apr 1996 Added silly rename for unlink --okir + * 28 Sep 1996 Improved directory cache --okir + * 23 Aug 1997 Claus Heine claus@momo.math.rwth-aachen.de + * Re-implemented silly rename for unlink, newly implemented + * silly rename for nfs_rename() following the suggestions + * of Olaf Kirch (okir) found in this file. + * Following Linus comments on my original hack, this version + * depends only on the dcache stuff and doesn't touch the inode + * layer (iput() and friends). + * 6 Jun 1999 Cache readdir lookups in the page cache. -DaveM + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "delegation.h" +#include "iostat.h" +#include "internal.h" +#include "fscache.h" + +#include "nfstrace.h" + +/* #define NFS_DEBUG_VERBOSE 1 */ + +static int nfs_opendir(struct inode *, struct file *); +static int nfs_closedir(struct inode *, struct file *); +static int nfs_readdir(struct file *, struct dir_context *); +static int nfs_fsync_dir(struct file *, loff_t, loff_t, int); +static loff_t nfs_llseek_dir(struct file *, loff_t, int); +static void nfs_readdir_clear_array(struct folio *); +static int nfs_do_create(struct inode *dir, struct dentry *dentry, + umode_t mode, int open_flags); + +const struct file_operations nfs_dir_operations = { + .llseek = nfs_llseek_dir, + .read = generic_read_dir, + .iterate_shared = nfs_readdir, + .open = nfs_opendir, + .release = nfs_closedir, + .fsync = nfs_fsync_dir, +}; + +const struct address_space_operations nfs_dir_aops = { + .free_folio = nfs_readdir_clear_array, +}; + +#define NFS_INIT_DTSIZE SZ_64K + +static struct nfs_open_dir_context * +alloc_nfs_open_dir_context(struct inode *dir) +{ + struct nfs_inode *nfsi = NFS_I(dir); + struct nfs_open_dir_context *ctx; + + ctx = kzalloc_obj(*ctx, GFP_KERNEL_ACCOUNT); + if (ctx != NULL) { + ctx->attr_gencount = nfsi->attr_gencount; + ctx->dtsize = min(NFS_SERVER(dir)->dtsize, NFS_INIT_DTSIZE); + spin_lock(&dir->i_lock); + if (list_empty(&nfsi->open_files) && + (nfsi->cache_validity & NFS_INO_DATA_INVAL_DEFER)) + nfs_set_cache_invalid(dir, + NFS_INO_INVALID_DATA | + NFS_INO_REVAL_FORCED); + list_add_tail_rcu(&ctx->list, &nfsi->open_files); + memcpy(ctx->verf, nfsi->cookieverf, sizeof(ctx->verf)); + spin_unlock(&dir->i_lock); + return ctx; + } + return ERR_PTR(-ENOMEM); +} + +static void put_nfs_open_dir_context(struct inode *dir, struct nfs_open_dir_context *ctx) +{ + spin_lock(&dir->i_lock); + list_del_rcu(&ctx->list); + spin_unlock(&dir->i_lock); + kfree_rcu(ctx, rcu_head); +} + +/* + * Open file + */ +static int +nfs_opendir(struct inode *inode, struct file *filp) +{ + int res = 0; + struct nfs_open_dir_context *ctx; + + dfprintk(FILE, "NFS: open dir(%pD2)\n", filp); + + nfs_inc_stats(inode, NFSIOS_VFSOPEN); + + ctx = alloc_nfs_open_dir_context(inode); + if (IS_ERR(ctx)) { + res = PTR_ERR(ctx); + goto out; + } + filp->private_data = ctx; +out: + return res; +} + +static int +nfs_closedir(struct inode *inode, struct file *filp) +{ + put_nfs_open_dir_context(file_inode(filp), filp->private_data); + return 0; +} + +struct nfs_cache_array_entry { + u64 cookie; + u64 ino; + const char *name; + unsigned int name_len; + unsigned char d_type; +}; + +struct nfs_cache_array { + u64 change_attr; + u64 last_cookie; + unsigned int size; + unsigned char folio_full : 1, + folio_is_eof : 1, + cookies_are_ordered : 1; + struct nfs_cache_array_entry array[] __counted_by(size); +}; + +struct nfs_readdir_descriptor { + struct file *file; + struct folio *folio; + struct dir_context *ctx; + pgoff_t folio_index; + pgoff_t folio_index_max; + u64 dir_cookie; + u64 last_cookie; + loff_t current_index; + + __be32 verf[NFS_DIR_VERIFIER_SIZE]; + unsigned long dir_verifier; + unsigned long timestamp; + unsigned long gencount; + unsigned long attr_gencount; + unsigned int cache_entry_index; + unsigned int buffer_fills; + unsigned int dtsize; + bool clear_cache; + bool plus; + bool eob; + bool eof; +}; + +static void nfs_set_dtsize(struct nfs_readdir_descriptor *desc, unsigned int sz) +{ + struct nfs_server *server = NFS_SERVER(file_inode(desc->file)); + unsigned int maxsize = server->dtsize; + + if (sz > maxsize) + sz = maxsize; + if (sz < NFS_MIN_FILE_IO_SIZE) + sz = NFS_MIN_FILE_IO_SIZE; + desc->dtsize = sz; +} + +static void nfs_shrink_dtsize(struct nfs_readdir_descriptor *desc) +{ + nfs_set_dtsize(desc, desc->dtsize >> 1); +} + +static void nfs_grow_dtsize(struct nfs_readdir_descriptor *desc) +{ + nfs_set_dtsize(desc, desc->dtsize << 1); +} + +static void nfs_readdir_folio_init_array(struct folio *folio, u64 last_cookie, + u64 change_attr) +{ + struct nfs_cache_array *array; + + array = kmap_local_folio(folio, 0); + array->change_attr = change_attr; + array->last_cookie = last_cookie; + array->size = 0; + array->folio_full = 0; + array->folio_is_eof = 0; + array->cookies_are_ordered = 1; + kunmap_local(array); +} + +/* + * we are freeing strings created by nfs_add_to_readdir_array() + */ +static void nfs_readdir_clear_array(struct folio *folio) +{ + struct nfs_cache_array *array; + unsigned int i; + + array = kmap_local_folio(folio, 0); + for (i = 0; i < array->size; i++) + kfree(array->array[i].name); + array->size = 0; + kunmap_local(array); +} + +static void nfs_readdir_folio_reinit_array(struct folio *folio, u64 last_cookie, + u64 change_attr) +{ + nfs_readdir_clear_array(folio); + nfs_readdir_folio_init_array(folio, last_cookie, change_attr); +} + +static struct folio * +nfs_readdir_folio_array_alloc(u64 last_cookie, gfp_t gfp_flags) +{ + struct folio *folio = folio_alloc(gfp_flags, 0); + if (folio) + nfs_readdir_folio_init_array(folio, last_cookie, 0); + return folio; +} + +static void nfs_readdir_folio_array_free(struct folio *folio) +{ + if (folio) { + nfs_readdir_clear_array(folio); + folio_put(folio); + } +} + +static u64 nfs_readdir_array_index_cookie(struct nfs_cache_array *array) +{ + return array->size == 0 ? array->last_cookie : array->array[0].cookie; +} + +static void nfs_readdir_array_set_eof(struct nfs_cache_array *array) +{ + array->folio_is_eof = 1; + array->folio_full = 1; +} + +static bool nfs_readdir_array_is_full(struct nfs_cache_array *array) +{ + return array->folio_full; +} + +/* + * the caller is responsible for freeing qstr.name + * when called by nfs_readdir_add_to_array, the strings will be freed in + * nfs_clear_readdir_array() + */ +static const char *nfs_readdir_copy_name(const char *name, unsigned int len) +{ + const char *ret = kmemdup_nul(name, len, GFP_KERNEL); + + /* + * Avoid a kmemleak false positive. The pointer to the name is stored + * in a page cache page which kmemleak does not scan. + */ + if (ret != NULL) + kmemleak_not_leak(ret); + return ret; +} + +static size_t nfs_readdir_array_maxentries(void) +{ + return (PAGE_SIZE - sizeof(struct nfs_cache_array)) / + sizeof(struct nfs_cache_array_entry); +} + +/* + * Check that the next array entry lies entirely within the page bounds + */ +static int nfs_readdir_array_can_expand(struct nfs_cache_array *array) +{ + if (array->folio_full) + return -ENOSPC; + if (array->size == nfs_readdir_array_maxentries()) { + array->folio_full = 1; + return -ENOSPC; + } + return 0; +} + +static int nfs_readdir_folio_array_append(struct folio *folio, + const struct nfs_entry *entry, + u64 *cookie) +{ + struct nfs_cache_array *array; + struct nfs_cache_array_entry *cache_entry; + const char *name; + int ret = -ENOMEM; + + name = nfs_readdir_copy_name(entry->name, entry->len); + + array = kmap_local_folio(folio, 0); + if (!name) + goto out; + ret = nfs_readdir_array_can_expand(array); + if (ret) { + kfree(name); + goto out; + } + + array->size++; + cache_entry = &array->array[array->size - 1]; + cache_entry->cookie = array->last_cookie; + cache_entry->ino = entry->ino; + cache_entry->d_type = entry->d_type; + cache_entry->name_len = entry->len; + cache_entry->name = name; + array->last_cookie = entry->cookie; + if (array->last_cookie <= cache_entry->cookie) + array->cookies_are_ordered = 0; + if (entry->eof != 0) + nfs_readdir_array_set_eof(array); +out: + *cookie = array->last_cookie; + kunmap_local(array); + return ret; +} + +#define NFS_READDIR_COOKIE_MASK (U32_MAX >> 14) +/* + * Hash algorithm allowing content addressible access to sequences + * of directory cookies. Content is addressed by the value of the + * cookie index of the first readdir entry in a page. + * + * We select only the first 18 bits to avoid issues with excessive + * memory use for the page cache XArray. 18 bits should allow the caching + * of 262144 pages of sequences of readdir entries. Since each page holds + * 127 readdir entries for a typical 64-bit system, that works out to a + * cache of ~ 33 million entries per directory. + */ +static pgoff_t nfs_readdir_folio_cookie_hash(u64 cookie) +{ + if (cookie == 0) + return 0; + return hash_64(cookie, 18); +} + +static bool nfs_readdir_folio_validate(struct folio *folio, u64 last_cookie, + u64 change_attr) +{ + struct nfs_cache_array *array = kmap_local_folio(folio, 0); + int ret = true; + + if (array->change_attr != change_attr) + ret = false; + if (nfs_readdir_array_index_cookie(array) != last_cookie) + ret = false; + kunmap_local(array); + return ret; +} + +static void nfs_readdir_folio_unlock_and_put(struct folio *folio) +{ + folio_unlock(folio); + folio_put(folio); +} + +static void nfs_readdir_folio_init_and_validate(struct folio *folio, u64 cookie, + u64 change_attr) +{ + if (folio_test_uptodate(folio)) { + if (nfs_readdir_folio_validate(folio, cookie, change_attr)) + return; + nfs_readdir_clear_array(folio); + } + nfs_readdir_folio_init_array(folio, cookie, change_attr); + folio_mark_uptodate(folio); +} + +static struct folio *nfs_readdir_folio_get_locked(struct address_space *mapping, + u64 cookie, u64 change_attr) +{ + pgoff_t index = nfs_readdir_folio_cookie_hash(cookie); + struct folio *folio; + + folio = filemap_grab_folio(mapping, index); + if (IS_ERR(folio)) + return NULL; + nfs_readdir_folio_init_and_validate(folio, cookie, change_attr); + return folio; +} + +static u64 nfs_readdir_folio_last_cookie(struct folio *folio) +{ + struct nfs_cache_array *array; + u64 ret; + + array = kmap_local_folio(folio, 0); + ret = array->last_cookie; + kunmap_local(array); + return ret; +} + +static bool nfs_readdir_folio_needs_filling(struct folio *folio) +{ + struct nfs_cache_array *array; + bool ret; + + array = kmap_local_folio(folio, 0); + ret = !nfs_readdir_array_is_full(array); + kunmap_local(array); + return ret; +} + +static void nfs_readdir_folio_set_eof(struct folio *folio) +{ + struct nfs_cache_array *array; + + array = kmap_local_folio(folio, 0); + nfs_readdir_array_set_eof(array); + kunmap_local(array); +} + +static struct folio *nfs_readdir_folio_get_next(struct address_space *mapping, + u64 cookie, u64 change_attr) +{ + pgoff_t index = nfs_readdir_folio_cookie_hash(cookie); + struct folio *folio; + + folio = __filemap_get_folio(mapping, index, + FGP_LOCK|FGP_CREAT|FGP_NOFS|FGP_NOWAIT, + mapping_gfp_mask(mapping)); + if (IS_ERR(folio)) + return NULL; + nfs_readdir_folio_init_and_validate(folio, cookie, change_attr); + if (nfs_readdir_folio_last_cookie(folio) != cookie) + nfs_readdir_folio_reinit_array(folio, cookie, change_attr); + return folio; +} + +static inline +int is_32bit_api(void) +{ +#ifdef CONFIG_COMPAT + return in_compat_syscall(); +#else + return (BITS_PER_LONG == 32); +#endif +} + +static +bool nfs_readdir_use_cookie(const struct file *filp) +{ + if ((filp->f_mode & FMODE_32BITHASH) || + (!(filp->f_mode & FMODE_64BITHASH) && is_32bit_api())) + return false; + return true; +} + +static void nfs_readdir_seek_next_array(struct nfs_cache_array *array, + struct nfs_readdir_descriptor *desc) +{ + if (array->folio_full) { + desc->last_cookie = array->last_cookie; + desc->current_index += array->size; + desc->cache_entry_index = 0; + desc->folio_index++; + } else + desc->last_cookie = nfs_readdir_array_index_cookie(array); +} + +static void nfs_readdir_rewind_search(struct nfs_readdir_descriptor *desc) +{ + desc->current_index = 0; + desc->last_cookie = 0; + desc->folio_index = 0; +} + +static int nfs_readdir_search_for_pos(struct nfs_cache_array *array, + struct nfs_readdir_descriptor *desc) +{ + loff_t diff = desc->ctx->pos - desc->current_index; + unsigned int index; + + if (diff < 0) + goto out_eof; + if (diff >= array->size) { + if (array->folio_is_eof) + goto out_eof; + nfs_readdir_seek_next_array(array, desc); + return -EAGAIN; + } + + index = (unsigned int)diff; + desc->dir_cookie = array->array[index].cookie; + desc->cache_entry_index = index; + return 0; +out_eof: + desc->eof = true; + return -EBADCOOKIE; +} + +static bool nfs_readdir_array_cookie_in_range(struct nfs_cache_array *array, + u64 cookie) +{ + if (!array->cookies_are_ordered) + return true; + /* Optimisation for monotonically increasing cookies */ + if (cookie >= array->last_cookie) + return false; + if (array->size && cookie < array->array[0].cookie) + return false; + return true; +} + +static int nfs_readdir_search_for_cookie(struct nfs_cache_array *array, + struct nfs_readdir_descriptor *desc) +{ + unsigned int i; + int status = -EAGAIN; + + if (!nfs_readdir_array_cookie_in_range(array, desc->dir_cookie)) + goto check_eof; + + for (i = 0; i < array->size; i++) { + if (array->array[i].cookie == desc->dir_cookie) { + if (nfs_readdir_use_cookie(desc->file)) + desc->ctx->pos = desc->dir_cookie; + else + desc->ctx->pos = desc->current_index + i; + desc->cache_entry_index = i; + return 0; + } + } +check_eof: + if (array->folio_is_eof) { + status = -EBADCOOKIE; + if (desc->dir_cookie == array->last_cookie) + desc->eof = true; + } else + nfs_readdir_seek_next_array(array, desc); + return status; +} + +static int nfs_readdir_search_array(struct nfs_readdir_descriptor *desc) +{ + struct nfs_cache_array *array; + int status; + + array = kmap_local_folio(desc->folio, 0); + + if (desc->dir_cookie == 0) + status = nfs_readdir_search_for_pos(array, desc); + else + status = nfs_readdir_search_for_cookie(array, desc); + + kunmap_local(array); + return status; +} + +/* Fill a page with xdr information before transferring to the cache page */ +static int nfs_readdir_xdr_filler(struct nfs_readdir_descriptor *desc, + __be32 *verf, u64 cookie, + struct page **pages, size_t bufsize, + __be32 *verf_res) +{ + struct inode *inode = file_inode(desc->file); + struct nfs_readdir_arg arg = { + .dentry = file_dentry(desc->file), + .cred = desc->file->f_cred, + .verf = verf, + .cookie = cookie, + .pages = pages, + .page_len = bufsize, + .plus = desc->plus, + }; + struct nfs_readdir_res res = { + .verf = verf_res, + }; + unsigned long timestamp, gencount; + int error; + + again: + timestamp = jiffies; + gencount = nfs_inc_attr_generation_counter(); + desc->dir_verifier = nfs_save_change_attribute(inode); + error = NFS_PROTO(inode)->readdir(&arg, &res); + if (error < 0) { + /* We requested READDIRPLUS, but the server doesn't grok it */ + if (error == -ENOTSUPP && desc->plus) { + NFS_SERVER(inode)->caps &= ~NFS_CAP_READDIRPLUS; + desc->plus = arg.plus = false; + goto again; + } + goto error; + } + desc->timestamp = timestamp; + desc->gencount = gencount; +error: + return error; +} + +static int xdr_decode(struct nfs_readdir_descriptor *desc, + struct nfs_entry *entry, struct xdr_stream *xdr) +{ + struct inode *inode = file_inode(desc->file); + int error; + + error = NFS_PROTO(inode)->decode_dirent(xdr, entry, desc->plus); + if (error) + return error; + entry->fattr->time_start = desc->timestamp; + entry->fattr->gencount = desc->gencount; + return 0; +} + +/* Match file and dirent using either filehandle or fileid + * Note: caller is responsible for checking the fsid + */ +static +int nfs_same_file(struct dentry *dentry, struct nfs_entry *entry) +{ + struct inode *inode; + struct nfs_inode *nfsi; + + if (d_really_is_negative(dentry)) + return 0; + + inode = d_inode(dentry); + if (is_bad_inode(inode) || NFS_STALE(inode)) + return 0; + + nfsi = NFS_I(inode); + if (entry->fattr->fileid != inode->i_ino) + return 0; + if (entry->fh->size && nfs_compare_fh(entry->fh, &nfsi->fh) != 0) + return 0; + return 1; +} + +#define NFS_READDIR_CACHE_USAGE_THRESHOLD (8UL) + +static bool nfs_use_readdirplus(struct inode *dir, struct dir_context *ctx, + unsigned int cache_hits, + unsigned int cache_misses) +{ + if (!nfs_server_capable(dir, NFS_CAP_READDIRPLUS)) + return false; + if (NFS_SERVER(dir)->flags & NFS_MOUNT_FORCE_RDIRPLUS) + return true; + if (ctx->pos == 0 || + cache_hits + cache_misses > NFS_READDIR_CACHE_USAGE_THRESHOLD) + return true; + return false; +} + +/* + * This function is called by the getattr code to request the + * use of readdirplus to accelerate any future lookups in the same + * directory. + */ +void nfs_readdir_record_entry_cache_hit(struct inode *dir) +{ + struct nfs_inode *nfsi = NFS_I(dir); + struct nfs_open_dir_context *ctx; + + if (nfs_server_capable(dir, NFS_CAP_READDIRPLUS) && + S_ISDIR(dir->i_mode)) { + rcu_read_lock(); + list_for_each_entry_rcu (ctx, &nfsi->open_files, list) + atomic_inc(&ctx->cache_hits); + rcu_read_unlock(); + } +} + +/* + * This function is mainly for use by nfs_getattr(). + * + * If this is an 'ls -l', we want to force use of readdirplus. + */ +void nfs_readdir_record_entry_cache_miss(struct inode *dir) +{ + struct nfs_inode *nfsi = NFS_I(dir); + struct nfs_open_dir_context *ctx; + + if (nfs_server_capable(dir, NFS_CAP_READDIRPLUS) && + S_ISDIR(dir->i_mode)) { + rcu_read_lock(); + list_for_each_entry_rcu (ctx, &nfsi->open_files, list) + atomic_inc(&ctx->cache_misses); + rcu_read_unlock(); + } +} + +static void nfs_lookup_advise_force_readdirplus(struct inode *dir, + unsigned int flags) +{ + if (nfs_server_capable(dir, NFS_CAP_CASE_INSENSITIVE)) + return; + if (flags & (LOOKUP_EXCL | LOOKUP_PARENT | LOOKUP_REVAL)) + return; + nfs_readdir_record_entry_cache_miss(dir); +} + +static +void nfs_prime_dcache(struct dentry *parent, struct nfs_entry *entry, + unsigned long dir_verifier) +{ + struct qstr filename = QSTR_INIT(entry->name, entry->len); + struct dentry *dentry; + struct dentry *alias; + struct inode *inode; + int status; + + if (!(entry->fattr->valid & NFS_ATTR_FATTR_FILEID)) + return; + if (!(entry->fattr->valid & NFS_ATTR_FATTR_FSID)) + return; + if (filename.len == 0) + return; + /* Validate that the name doesn't contain any illegal '\0' */ + if (strnlen(filename.name, filename.len) != filename.len) + return; + /* ...or '/' */ + if (strnchr(filename.name, filename.len, '/')) + return; + if (filename.name[0] == '.') { + if (filename.len == 1) + return; + if (filename.len == 2 && filename.name[1] == '.') + return; + } + filename.hash = full_name_hash(parent, filename.name, filename.len); + + dentry = d_lookup(parent, &filename); +again: + if (!dentry) { + dentry = d_alloc_parallel(parent, &filename); + if (IS_ERR(dentry)) + return; + } + if (!d_in_lookup(dentry)) { + /* Is there a mountpoint here? If so, just exit */ + if (!nfs_fsid_equal(&NFS_SB(dentry->d_sb)->fsid, + &entry->fattr->fsid)) + goto out; + if (nfs_same_file(dentry, entry)) { + if (!entry->fh->size) + goto out; + nfs_set_verifier(dentry, dir_verifier); + status = nfs_refresh_inode(d_inode(dentry), entry->fattr); + if (!status) + nfs_setsecurity(d_inode(dentry), entry->fattr); + trace_nfs_readdir_lookup_revalidate(d_inode(parent), + dentry, 0, status); + goto out; + } else { + trace_nfs_readdir_lookup_revalidate_failed( + d_inode(parent), dentry, 0); + d_invalidate(dentry); + dput(dentry); + dentry = NULL; + goto again; + } + } + if (!entry->fh->size) { + d_lookup_done(dentry); + goto out; + } + + nfs_set_verifier(dentry, dir_verifier); + inode = nfs_fhget(dentry->d_sb, entry->fh, entry->fattr); + alias = d_splice_alias(inode, dentry); + d_lookup_done(dentry); + if (alias) { + if (IS_ERR(alias)) + goto out; + nfs_set_verifier(alias, dir_verifier); + dput(dentry); + dentry = alias; + } + trace_nfs_readdir_lookup(d_inode(parent), dentry, 0); +out: + dput(dentry); +} + +static int nfs_readdir_entry_decode(struct nfs_readdir_descriptor *desc, + struct nfs_entry *entry, + struct xdr_stream *stream) +{ + int ret; + + if (entry->fattr->label) + entry->fattr->label->len = NFS4_MAXLABELLEN; + ret = xdr_decode(desc, entry, stream); + if (ret || !desc->plus) + return ret; + nfs_prime_dcache(file_dentry(desc->file), entry, desc->dir_verifier); + return 0; +} + +/* Perform conversion from xdr to cache array */ +static int nfs_readdir_folio_filler(struct nfs_readdir_descriptor *desc, + struct nfs_entry *entry, + struct page **xdr_pages, unsigned int buflen, + struct folio **arrays, size_t narrays, + u64 change_attr) +{ + struct address_space *mapping = desc->file->f_mapping; + struct folio *new, *folio = *arrays; + struct xdr_stream stream; + struct folio *scratch; + struct xdr_buf buf; + u64 cookie; + int status; + + scratch = folio_alloc(GFP_KERNEL, 0); + if (scratch == NULL) + return -ENOMEM; + + xdr_init_decode_pages(&stream, &buf, xdr_pages, buflen); + xdr_set_scratch_folio(&stream, scratch); + + do { + status = nfs_readdir_entry_decode(desc, entry, &stream); + if (status != 0) + break; + + status = nfs_readdir_folio_array_append(folio, entry, &cookie); + if (status != -ENOSPC) + continue; + + if (folio->mapping != mapping) { + if (!--narrays) + break; + new = nfs_readdir_folio_array_alloc(cookie, GFP_KERNEL); + if (!new) + break; + arrays++; + *arrays = folio = new; + } else { + new = nfs_readdir_folio_get_next(mapping, cookie, + change_attr); + if (!new) + break; + if (folio != *arrays) + nfs_readdir_folio_unlock_and_put(folio); + folio = new; + } + desc->folio_index_max++; + status = nfs_readdir_folio_array_append(folio, entry, &cookie); + } while (!status && !entry->eof); + + switch (status) { + case -EBADCOOKIE: + if (!entry->eof) + break; + nfs_readdir_folio_set_eof(folio); + fallthrough; + case -EAGAIN: + status = 0; + break; + case -ENOSPC: + status = 0; + if (!desc->plus) + break; + while (!nfs_readdir_entry_decode(desc, entry, &stream)) + ; + } + + if (folio != *arrays) + nfs_readdir_folio_unlock_and_put(folio); + + folio_put(scratch); + return status; +} + +static void nfs_readdir_free_pages(struct page **pages, size_t npages) +{ + while (npages--) + put_page(pages[npages]); + kfree(pages); +} + +/* + * nfs_readdir_alloc_pages() will allocate pages that must be freed with a call + * to nfs_readdir_free_pages() + */ +static struct page **nfs_readdir_alloc_pages(size_t npages) +{ + struct page **pages; + size_t i; + + pages = kmalloc_objs(*pages, npages); + if (!pages) + return NULL; + for (i = 0; i < npages; i++) { + struct page *page = alloc_page(GFP_KERNEL); + if (page == NULL) + goto out_freepages; + pages[i] = page; + } + return pages; + +out_freepages: + nfs_readdir_free_pages(pages, i); + return NULL; +} + +static int nfs_readdir_xdr_to_array(struct nfs_readdir_descriptor *desc, + __be32 *verf_arg, __be32 *verf_res, + struct folio **arrays, size_t narrays) +{ + u64 change_attr; + struct page **pages; + struct folio *folio = *arrays; + struct nfs_entry *entry; + size_t array_size; + struct inode *inode = file_inode(desc->file); + unsigned int dtsize = desc->dtsize; + unsigned int pglen; + int status = -ENOMEM; + + entry = kzalloc_obj(*entry); + if (!entry) + return -ENOMEM; + entry->cookie = nfs_readdir_folio_last_cookie(folio); + entry->fh = nfs_alloc_fhandle(); + entry->fattr = nfs_alloc_fattr_with_label(NFS_SERVER(inode)); + entry->server = NFS_SERVER(inode); + if (entry->fh == NULL || entry->fattr == NULL) + goto out; + + array_size = (dtsize + PAGE_SIZE - 1) >> PAGE_SHIFT; + pages = nfs_readdir_alloc_pages(array_size); + if (!pages) + goto out; + + change_attr = inode_peek_iversion_raw(inode); + status = nfs_readdir_xdr_filler(desc, verf_arg, entry->cookie, pages, + dtsize, verf_res); + if (status < 0) + goto free_pages; + + pglen = status; + if (pglen != 0) + status = nfs_readdir_folio_filler(desc, entry, pages, pglen, + arrays, narrays, change_attr); + else + nfs_readdir_folio_set_eof(folio); + desc->buffer_fills++; + +free_pages: + nfs_readdir_free_pages(pages, array_size); +out: + nfs_free_fattr(entry->fattr); + nfs_free_fhandle(entry->fh); + kfree(entry); + return status; +} + +static void nfs_readdir_folio_put(struct nfs_readdir_descriptor *desc) +{ + folio_put(desc->folio); + desc->folio = NULL; +} + +static void +nfs_readdir_folio_unlock_and_put_cached(struct nfs_readdir_descriptor *desc) +{ + folio_unlock(desc->folio); + nfs_readdir_folio_put(desc); +} + +static struct folio * +nfs_readdir_folio_get_cached(struct nfs_readdir_descriptor *desc) +{ + struct address_space *mapping = desc->file->f_mapping; + u64 change_attr = inode_peek_iversion_raw(mapping->host); + u64 cookie = desc->last_cookie; + struct folio *folio; + + folio = nfs_readdir_folio_get_locked(mapping, cookie, change_attr); + if (!folio) + return NULL; + if (desc->clear_cache && !nfs_readdir_folio_needs_filling(folio)) + nfs_readdir_folio_reinit_array(folio, cookie, change_attr); + return folio; +} + +/* + * Returns 0 if desc->dir_cookie was found on page desc->page_index + * and locks the page to prevent removal from the page cache. + */ +static int find_and_lock_cache_page(struct nfs_readdir_descriptor *desc) +{ + struct inode *inode = file_inode(desc->file); + struct nfs_inode *nfsi = NFS_I(inode); + __be32 verf[NFS_DIR_VERIFIER_SIZE]; + int res; + + desc->folio = nfs_readdir_folio_get_cached(desc); + if (!desc->folio) + return -ENOMEM; + if (nfs_readdir_folio_needs_filling(desc->folio)) { + /* Grow the dtsize if we had to go back for more pages */ + if (desc->folio_index == desc->folio_index_max) + nfs_grow_dtsize(desc); + desc->folio_index_max = desc->folio_index; + trace_nfs_readdir_cache_fill(desc->file, nfsi->cookieverf, + desc->last_cookie, + desc->folio->index, desc->dtsize); + res = nfs_readdir_xdr_to_array(desc, nfsi->cookieverf, verf, + &desc->folio, 1); + if (res < 0) { + nfs_readdir_folio_unlock_and_put_cached(desc); + trace_nfs_readdir_cache_fill_done(inode, res); + if (res == -EBADCOOKIE || res == -ENOTSYNC) { + invalidate_inode_pages2(desc->file->f_mapping); + nfs_readdir_rewind_search(desc); + trace_nfs_readdir_invalidate_cache_range( + inode, 0, MAX_LFS_FILESIZE); + return -EAGAIN; + } + return res; + } + /* + * Set the cookie verifier if the page cache was empty + */ + if (desc->last_cookie == 0 && + memcmp(nfsi->cookieverf, verf, sizeof(nfsi->cookieverf))) { + memcpy(nfsi->cookieverf, verf, + sizeof(nfsi->cookieverf)); + invalidate_inode_pages2_range(desc->file->f_mapping, 1, + -1); + trace_nfs_readdir_invalidate_cache_range( + inode, 1, MAX_LFS_FILESIZE); + } + desc->clear_cache = false; + } + res = nfs_readdir_search_array(desc); + if (res == 0) + return 0; + nfs_readdir_folio_unlock_and_put_cached(desc); + return res; +} + +/* Search for desc->dir_cookie from the beginning of the page cache */ +static int readdir_search_pagecache(struct nfs_readdir_descriptor *desc) +{ + int res; + + do { + res = find_and_lock_cache_page(desc); + } while (res == -EAGAIN); + return res; +} + +#define NFS_READDIR_CACHE_MISS_THRESHOLD (16UL) + +/* + * Once we've found the start of the dirent within a page: fill 'er up... + */ +static void nfs_do_filldir(struct nfs_readdir_descriptor *desc, + const __be32 *verf) +{ + struct file *file = desc->file; + struct nfs_cache_array *array; + unsigned int i; + bool first_emit = !desc->dir_cookie; + + array = kmap_local_folio(desc->folio, 0); + for (i = desc->cache_entry_index; i < array->size; i++) { + struct nfs_cache_array_entry *ent; + + /* + * nfs_readdir_handle_cache_misses return force clear at + * (cache_misses > NFS_READDIR_CACHE_MISS_THRESHOLD) for + * readdir heuristic, NFS_READDIR_CACHE_MISS_THRESHOLD + 1 + * entries need be emitted here. + */ + if (first_emit && i > NFS_READDIR_CACHE_MISS_THRESHOLD + 2) { + desc->eob = true; + break; + } + + ent = &array->array[i]; + if (!dir_emit(desc->ctx, ent->name, ent->name_len, + ent->ino, ent->d_type)) { + desc->eob = true; + break; + } + memcpy(desc->verf, verf, sizeof(desc->verf)); + if (i == array->size - 1) { + desc->dir_cookie = array->last_cookie; + nfs_readdir_seek_next_array(array, desc); + } else { + desc->dir_cookie = array->array[i + 1].cookie; + desc->last_cookie = array->array[0].cookie; + } + if (nfs_readdir_use_cookie(file)) + desc->ctx->pos = desc->dir_cookie; + else + desc->ctx->pos++; + } + if (array->folio_is_eof) + desc->eof = !desc->eob; + + kunmap_local(array); + dfprintk(DIRCACHE, "NFS: nfs_do_filldir() filling ended @ cookie %llu\n", + (unsigned long long)desc->dir_cookie); +} + +/* + * If we cannot find a cookie in our cache, we suspect that this is + * because it points to a deleted file, so we ask the server to return + * whatever it thinks is the next entry. We then feed this to filldir. + * If all goes well, we should then be able to find our way round the + * cache on the next call to readdir_search_pagecache(); + * + * NOTE: we cannot add the anonymous page to the pagecache because + * the data it contains might not be page aligned. Besides, + * we should already have a complete representation of the + * directory in the page cache by the time we get here. + */ +static int uncached_readdir(struct nfs_readdir_descriptor *desc) +{ + struct folio **arrays; + size_t i, sz = 512; + __be32 verf[NFS_DIR_VERIFIER_SIZE]; + int status = -ENOMEM; + + dfprintk(DIRCACHE, "NFS: uncached_readdir() searching for cookie %llu\n", + (unsigned long long)desc->dir_cookie); + + arrays = kzalloc_objs(*arrays, sz); + if (!arrays) + goto out; + arrays[0] = nfs_readdir_folio_array_alloc(desc->dir_cookie, GFP_KERNEL); + if (!arrays[0]) + goto out; + + desc->folio_index = 0; + desc->cache_entry_index = 0; + desc->last_cookie = desc->dir_cookie; + desc->folio_index_max = 0; + + trace_nfs_readdir_uncached(desc->file, desc->verf, desc->last_cookie, + -1, desc->dtsize); + + status = nfs_readdir_xdr_to_array(desc, desc->verf, verf, arrays, sz); + if (status < 0) { + trace_nfs_readdir_uncached_done(file_inode(desc->file), status); + goto out_free; + } + + for (i = 0; !desc->eob && i < sz && arrays[i]; i++) { + desc->folio = arrays[i]; + nfs_do_filldir(desc, verf); + } + desc->folio = NULL; + + /* + * Grow the dtsize if we have to go back for more pages, + * or shrink it if we're reading too many. + */ + if (!desc->eof) { + if (!desc->eob) + nfs_grow_dtsize(desc); + else if (desc->buffer_fills == 1 && + i < (desc->folio_index_max >> 1)) + nfs_shrink_dtsize(desc); + } +out_free: + for (i = 0; i < sz && arrays[i]; i++) + nfs_readdir_folio_array_free(arrays[i]); +out: + if (!nfs_readdir_use_cookie(desc->file)) + nfs_readdir_rewind_search(desc); + desc->folio_index_max = -1; + kfree(arrays); + dfprintk(DIRCACHE, "NFS: %s: returns %d\n", __func__, status); + return status; +} + +static bool nfs_readdir_handle_cache_misses(struct inode *inode, + struct nfs_readdir_descriptor *desc, + unsigned int cache_misses, + bool force_clear) +{ + if (desc->ctx->pos == 0 || !desc->plus) + return false; + if (cache_misses <= NFS_READDIR_CACHE_MISS_THRESHOLD && !force_clear) + return false; + trace_nfs_readdir_force_readdirplus(inode); + return true; +} + +/* The file offset position represents the dirent entry number. A + last cookie cache takes care of the common case of reading the + whole directory. + */ +static int nfs_readdir(struct file *file, struct dir_context *ctx) +{ + struct dentry *dentry = file_dentry(file); + struct inode *inode = d_inode(dentry); + struct nfs_inode *nfsi = NFS_I(inode); + struct nfs_open_dir_context *dir_ctx = file->private_data; + struct nfs_readdir_descriptor *desc; + unsigned int cache_hits, cache_misses; + bool force_clear; + int res; + + dfprintk(FILE, "NFS: readdir(%pD2) starting at cookie %llu\n", + file, (long long)ctx->pos); + nfs_inc_stats(inode, NFSIOS_VFSGETDENTS); + + /* + * ctx->pos points to the dirent entry number. + * *desc->dir_cookie has the cookie for the next entry. We have + * to either find the entry with the appropriate number or + * revalidate the cookie. + */ + nfs_revalidate_mapping(inode, file->f_mapping); + + res = -ENOMEM; + desc = kzalloc_obj(*desc); + if (!desc) + goto out; + desc->file = file; + desc->ctx = ctx; + desc->folio_index_max = -1; + + spin_lock(&file->f_lock); + desc->dir_cookie = dir_ctx->dir_cookie; + desc->folio_index = dir_ctx->page_index; + desc->last_cookie = dir_ctx->last_cookie; + desc->attr_gencount = dir_ctx->attr_gencount; + desc->eof = dir_ctx->eof; + nfs_set_dtsize(desc, dir_ctx->dtsize); + memcpy(desc->verf, dir_ctx->verf, sizeof(desc->verf)); + cache_hits = atomic_xchg(&dir_ctx->cache_hits, 0); + cache_misses = atomic_xchg(&dir_ctx->cache_misses, 0); + force_clear = dir_ctx->force_clear; + spin_unlock(&file->f_lock); + + if (desc->eof) { + res = 0; + goto out_free; + } + + desc->plus = nfs_use_readdirplus(inode, ctx, cache_hits, cache_misses); + force_clear = nfs_readdir_handle_cache_misses(inode, desc, cache_misses, + force_clear); + desc->clear_cache = force_clear; + + do { + res = readdir_search_pagecache(desc); + + if (res == -EBADCOOKIE) { + res = 0; + /* This means either end of directory */ + if (desc->dir_cookie && !desc->eof) { + /* Or that the server has 'lost' a cookie */ + res = uncached_readdir(desc); + if (res == 0) + continue; + if (res == -EBADCOOKIE || res == -ENOTSYNC) + res = 0; + } + break; + } + if (res == -ETOOSMALL && desc->plus) { + nfs_zap_caches(inode); + desc->plus = false; + desc->eof = false; + continue; + } + if (res < 0) + break; + + nfs_do_filldir(desc, nfsi->cookieverf); + nfs_readdir_folio_unlock_and_put_cached(desc); + if (desc->folio_index == desc->folio_index_max) + desc->clear_cache = force_clear; + } while (!desc->eob && !desc->eof); + + spin_lock(&file->f_lock); + dir_ctx->dir_cookie = desc->dir_cookie; + dir_ctx->last_cookie = desc->last_cookie; + dir_ctx->attr_gencount = desc->attr_gencount; + dir_ctx->page_index = desc->folio_index; + dir_ctx->force_clear = force_clear; + dir_ctx->eof = desc->eof; + dir_ctx->dtsize = desc->dtsize; + memcpy(dir_ctx->verf, desc->verf, sizeof(dir_ctx->verf)); + spin_unlock(&file->f_lock); +out_free: + kfree(desc); + +out: + dfprintk(FILE, "NFS: readdir(%pD2) returns %d\n", file, res); + return res; +} + +static loff_t nfs_llseek_dir(struct file *filp, loff_t offset, int whence) +{ + struct nfs_open_dir_context *dir_ctx = filp->private_data; + + dfprintk(FILE, "NFS: llseek dir(%pD2, %lld, %d)\n", + filp, offset, whence); + + switch (whence) { + default: + return -EINVAL; + case SEEK_SET: + if (offset < 0) + return -EINVAL; + spin_lock(&filp->f_lock); + break; + case SEEK_CUR: + if (offset == 0) + return filp->f_pos; + spin_lock(&filp->f_lock); + offset += filp->f_pos; + if (offset < 0) { + spin_unlock(&filp->f_lock); + return -EINVAL; + } + } + if (offset != filp->f_pos) { + filp->f_pos = offset; + dir_ctx->page_index = 0; + if (!nfs_readdir_use_cookie(filp)) { + dir_ctx->dir_cookie = 0; + dir_ctx->last_cookie = 0; + } else { + dir_ctx->dir_cookie = offset; + dir_ctx->last_cookie = offset; + } + dir_ctx->eof = false; + } + spin_unlock(&filp->f_lock); + return offset; +} + +/* + * All directory operations under NFS are synchronous, so fsync() + * is a dummy operation. + */ +static int nfs_fsync_dir(struct file *filp, loff_t start, loff_t end, + int datasync) +{ + dfprintk(FILE, "NFS: fsync dir(%pD2) datasync %d\n", filp, datasync); + + nfs_inc_stats(file_inode(filp), NFSIOS_VFSFSYNC); + return 0; +} + +/** + * nfs_force_lookup_revalidate - Mark the directory as having changed + * @dir: pointer to directory inode + * + * This forces the revalidation code in nfs_lookup_revalidate() to do a + * full lookup on all child dentries of 'dir' whenever a change occurs + * on the server that might have invalidated our dcache. + * + * Note that we reserve bit '0' as a tag to let us know when a dentry + * was revalidated while holding a delegation on its inode. + * + * The caller should be holding dir->i_lock + */ +void nfs_force_lookup_revalidate(struct inode *dir) +{ + NFS_I(dir)->cache_change_attribute += 2; +} +EXPORT_SYMBOL_GPL(nfs_force_lookup_revalidate); + +/** + * nfs_verify_change_attribute - Detects NFS remote directory changes + * @dir: pointer to parent directory inode + * @verf: previously saved change attribute + * + * Return "false" if the verifiers doesn't match the change attribute. + * This would usually indicate that the directory contents have changed on + * the server, and that any dentries need revalidating. + */ +static bool nfs_verify_change_attribute(struct inode *dir, unsigned long verf) +{ + return (verf & ~1UL) == nfs_save_change_attribute(dir); +} + +static void nfs_set_verifier_delegated(unsigned long *verf) +{ + *verf |= 1UL; +} + +#if IS_ENABLED(CONFIG_NFS_V4) +static void nfs_unset_verifier_delegated(unsigned long *verf) +{ + *verf &= ~1UL; +} +#endif /* IS_ENABLED(CONFIG_NFS_V4) */ + +static bool nfs_test_verifier_delegated(unsigned long verf) +{ + return verf & 1; +} + +static bool nfs_verifier_is_delegated(struct dentry *dentry) +{ + return nfs_test_verifier_delegated(dentry->d_time); +} + +static void nfs_set_verifier_locked(struct dentry *dentry, unsigned long verf) +{ + struct inode *inode = d_inode(dentry); + struct inode *dir = d_inode_rcu(dentry->d_parent); + + if (!dir || !nfs_verify_change_attribute(dir, verf)) + return; + if (NFS_PROTO(dir)->have_delegation(dir, FMODE_READ, 0) || + (inode && NFS_PROTO(inode)->have_delegation(inode, FMODE_READ, 0))) + nfs_set_verifier_delegated(&verf); + dentry->d_time = verf; +} + +/** + * nfs_set_verifier - save a parent directory verifier in the dentry + * @dentry: pointer to dentry + * @verf: verifier to save + * + * Saves the parent directory verifier in @dentry. If the inode has + * a delegation, we also tag the dentry as having been revalidated + * while holding a delegation so that we know we don't have to + * look it up again after a directory change. + */ +void nfs_set_verifier(struct dentry *dentry, unsigned long verf) +{ + + spin_lock(&dentry->d_lock); + nfs_set_verifier_locked(dentry, verf); + spin_unlock(&dentry->d_lock); +} +EXPORT_SYMBOL_GPL(nfs_set_verifier); + +#if IS_ENABLED(CONFIG_NFS_V4) +static void nfs_clear_verifier_file(struct inode *inode) +{ + struct dentry *alias; + struct inode *dir; + + for_each_alias(alias, inode) { + spin_lock(&alias->d_lock); + dir = d_inode_rcu(alias->d_parent); + if (!dir || + !NFS_PROTO(dir)->have_delegation(dir, FMODE_READ, 0)) + nfs_unset_verifier_delegated(&alias->d_time); + spin_unlock(&alias->d_lock); + } +} + +static void nfs_clear_verifier_directory(struct inode *dir) +{ + struct dentry *this_parent; + struct dentry *dentry; + struct inode *inode; + + if (hlist_empty(&dir->i_dentry)) + return; + this_parent = + hlist_entry(dir->i_dentry.first, struct dentry, d_alias); + + spin_lock(&this_parent->d_lock); + nfs_unset_verifier_delegated(&this_parent->d_time); + dentry = d_first_child(this_parent); + hlist_for_each_entry_from(dentry, d_sib) { + if (unlikely(dentry->d_flags & DCACHE_DENTRY_CURSOR)) + continue; + inode = d_inode_rcu(dentry); + if (inode && + NFS_PROTO(inode)->have_delegation(inode, FMODE_READ, 0)) + continue; + spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); + nfs_unset_verifier_delegated(&dentry->d_time); + spin_unlock(&dentry->d_lock); + } + spin_unlock(&this_parent->d_lock); +} + +/** + * nfs_clear_verifier_delegated - clear the dir verifier delegation tag + * @inode: pointer to inode + * + * Iterates through the dentries in the inode alias list and clears + * the tag used to indicate that the dentry has been revalidated + * while holding a delegation. + * This function is intended for use when the delegation is being + * returned or revoked. + */ +void nfs_clear_verifier_delegated(struct inode *inode) +{ + if (!inode) + return; + spin_lock(&inode->i_lock); + if (S_ISREG(inode->i_mode)) + nfs_clear_verifier_file(inode); + else if (S_ISDIR(inode->i_mode)) + nfs_clear_verifier_directory(inode); + spin_unlock(&inode->i_lock); +} +EXPORT_SYMBOL_GPL(nfs_clear_verifier_delegated); +#endif /* IS_ENABLED(CONFIG_NFS_V4) */ + +static int nfs_dentry_verify_change(struct inode *dir, struct dentry *dentry) +{ + if (nfs_server_capable(dir, NFS_CAP_CASE_INSENSITIVE) && + d_really_is_negative(dentry)) + return dentry->d_time == inode_peek_iversion_raw(dir); + return nfs_verify_change_attribute(dir, dentry->d_time); +} + +/* + * A check for whether or not the parent directory has changed. + * In the case it has, we assume that the dentries are untrustworthy + * and may need to be looked up again. + * If rcu_walk prevents us from performing a full check, return 0. + */ +static int nfs_check_verifier(struct inode *dir, struct dentry *dentry, + int rcu_walk) +{ + if (IS_ROOT(dentry)) + return 1; + if (NFS_SERVER(dir)->flags & NFS_MOUNT_LOOKUP_CACHE_NONE) + return 0; + if (!nfs_dentry_verify_change(dir, dentry)) + return 0; + + /* Revalidate nfsi->cache_change_attribute before we declare a match */ + if (nfs_mapping_need_revalidate_inode(dir)) { + if (rcu_walk) + return 0; + if (__nfs_revalidate_inode(NFS_SERVER(dir), dir) < 0) + return 0; + } + if (!nfs_dentry_verify_change(dir, dentry)) + return 0; + return 1; +} + +/* + * Use intent information to check whether or not we're going to do + * an O_EXCL create using this path component. + */ +static int nfs_is_exclusive_create(struct inode *dir, unsigned int flags) +{ + if (NFS_PROTO(dir)->version == 2) + return 0; + return (flags & (LOOKUP_CREATE | LOOKUP_EXCL)) == + (LOOKUP_CREATE | LOOKUP_EXCL); +} + +/* + * Inode and filehandle revalidation for lookups. + * + * We force revalidation in the cases where the VFS sets LOOKUP_REVAL, + * or if the intent information indicates that we're about to open this + * particular file and the "nocto" mount flag is not set. + * + */ +static +int nfs_lookup_verify_inode(struct inode *inode, unsigned int flags) +{ + struct nfs_server *server = NFS_SERVER(inode); + int ret; + + if (IS_AUTOMOUNT(inode)) + return 0; + + if (flags & LOOKUP_OPEN) { + switch (inode->i_mode & S_IFMT) { + case S_IFREG: + /* A NFSv4 OPEN will revalidate later */ + if (server->caps & NFS_CAP_ATOMIC_OPEN) + goto out; + fallthrough; + case S_IFDIR: + if (server->flags & NFS_MOUNT_NOCTO) + break; + /* NFS close-to-open cache consistency validation */ + goto out_force; + } + } + + /* VFS wants an on-the-wire revalidation */ + if (flags & LOOKUP_REVAL) + goto out_force; +out: + if (inode->i_nlink > 0 || + (inode->i_nlink == 0 && + test_bit(NFS_INO_PRESERVE_UNLINKED, &NFS_I(inode)->flags))) + return 0; + else + return -ESTALE; +out_force: + if (flags & LOOKUP_RCU) + return -ECHILD; + ret = __nfs_revalidate_inode(server, inode); + if (ret != 0) + return ret; + goto out; +} + +static void nfs_mark_dir_for_revalidate(struct inode *inode) +{ + spin_lock(&inode->i_lock); + nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE); + spin_unlock(&inode->i_lock); +} + +/* + * We judge how long we want to trust negative + * dentries by looking at the parent inode mtime. + * + * If parent mtime has changed, we revalidate, else we wait for a + * period corresponding to the parent's attribute cache timeout value. + * + * If LOOKUP_RCU prevents us from performing a full check, return 1 + * suggesting a reval is needed. + * + * Note that when creating a new file, or looking up a rename target, + * then it shouldn't be necessary to revalidate a negative dentry. + */ +static inline +int nfs_neg_need_reval(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + if (flags & (LOOKUP_CREATE | LOOKUP_RENAME_TARGET)) + return 0; + if (NFS_SERVER(dir)->flags & NFS_MOUNT_LOOKUP_CACHE_NONEG) + return 1; + /* Case insensitive server? Revalidate negative dentries */ + if (nfs_server_capable(dir, NFS_CAP_CASE_INSENSITIVE)) + return 1; + return !nfs_check_verifier(dir, dentry, flags & LOOKUP_RCU); +} + +static int +nfs_lookup_revalidate_done(struct inode *dir, struct dentry *dentry, + struct inode *inode, int error) +{ + switch (error) { + case 1: + break; + case -ETIMEDOUT: + if (inode && (IS_ROOT(dentry) || + NFS_SERVER(inode)->flags & NFS_MOUNT_SOFTREVAL)) + error = 1; + break; + case -ESTALE: + case -ENOENT: + error = 0; + fallthrough; + default: + /* + * We can't d_drop the root of a disconnected tree: + * its d_hash is on the s_anon list and d_drop() would hide + * it from shrink_dcache_for_unmount(), leading to busy + * inodes on unmount and further oopses. + */ + if (inode && IS_ROOT(dentry)) + error = 1; + break; + } + trace_nfs_lookup_revalidate_exit(dir, dentry, 0, error); + return error; +} + +static int +nfs_lookup_revalidate_negative(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + int ret = 1; + if (nfs_neg_need_reval(dir, dentry, flags)) { + if (flags & LOOKUP_RCU) + return -ECHILD; + ret = 0; + } + return nfs_lookup_revalidate_done(dir, dentry, NULL, ret); +} + +static int +nfs_lookup_revalidate_delegated(struct inode *dir, struct dentry *dentry, + struct inode *inode) +{ + nfs_set_verifier(dentry, nfs_save_change_attribute(dir)); + return nfs_lookup_revalidate_done(dir, dentry, inode, 1); +} + +static int nfs_lookup_revalidate_dentry(struct inode *dir, const struct qstr *name, + struct dentry *dentry, + struct inode *inode, unsigned int flags) +{ + struct nfs_fh *fhandle; + struct nfs_fattr *fattr; + unsigned long dir_verifier; + int ret; + + trace_nfs_lookup_revalidate_enter(dir, dentry, flags); + + ret = -ENOMEM; + fhandle = nfs_alloc_fhandle(); + fattr = nfs_alloc_fattr_with_label(NFS_SERVER(inode)); + if (fhandle == NULL || fattr == NULL) + goto out; + + dir_verifier = nfs_save_change_attribute(dir); + ret = NFS_PROTO(dir)->lookup(dir, dentry, name, fhandle, fattr); + if (ret < 0) + goto out; + + /* Request help from readdirplus */ + nfs_lookup_advise_force_readdirplus(dir, flags); + + ret = 0; + if (nfs_compare_fh(NFS_FH(inode), fhandle)) + goto out; + if (nfs_refresh_inode(inode, fattr) < 0) + goto out; + + nfs_setsecurity(inode, fattr); + nfs_set_verifier(dentry, dir_verifier); + + ret = 1; +out: + nfs_free_fattr(fattr); + nfs_free_fhandle(fhandle); + + /* + * If the lookup failed despite the dentry change attribute being + * a match, then we should revalidate the directory cache. + */ + if (!ret && nfs_dentry_verify_change(dir, dentry)) + nfs_mark_dir_for_revalidate(dir); + return nfs_lookup_revalidate_done(dir, dentry, inode, ret); +} + +/* + * This is called every time the dcache has a lookup hit, + * and we should check whether we can really trust that + * lookup. + * + * NOTE! The hit can be a negative hit too, don't assume + * we have an inode! + * + * If the parent directory is seen to have changed, we throw out the + * cached dentry and do a new lookup. + */ +static int +nfs_do_lookup_revalidate(struct inode *dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + struct inode *inode; + int error = 0; + + nfs_inc_stats(dir, NFSIOS_DENTRYREVALIDATE); + inode = d_inode(dentry); + + if (!inode) + return nfs_lookup_revalidate_negative(dir, dentry, flags); + + if (is_bad_inode(inode)) { + dfprintk(LOOKUPCACHE, "%s: %pd2 has dud inode\n", + __func__, dentry); + goto out_bad; + } + + if ((flags & LOOKUP_RENAME_TARGET) && d_count(dentry) < 2 && + nfs_server_capable(dir, NFS_CAP_CASE_INSENSITIVE)) + goto out_bad; + + if (nfs_verifier_is_delegated(dentry)) + return nfs_lookup_revalidate_delegated(dir, dentry, inode); + + /* Force a full look up iff the parent directory has changed */ + if (!(flags & (LOOKUP_EXCL | LOOKUP_REVAL)) && + nfs_check_verifier(dir, dentry, flags & LOOKUP_RCU)) { + error = nfs_lookup_verify_inode(inode, flags); + if (error) { + if (error == -ESTALE) + nfs_mark_dir_for_revalidate(dir); + goto out_bad; + } + goto out_valid; + } + + if (flags & LOOKUP_RCU) + return -ECHILD; + + if (NFS_STALE(inode)) + goto out_bad; + + return nfs_lookup_revalidate_dentry(dir, name, dentry, inode, flags); +out_valid: + return nfs_lookup_revalidate_done(dir, dentry, inode, 1); +out_bad: + if (flags & LOOKUP_RCU) + return -ECHILD; + return nfs_lookup_revalidate_done(dir, dentry, inode, error); +} + +static int +__nfs_lookup_revalidate(struct dentry *dentry, unsigned int flags) +{ + if (flags & LOOKUP_RCU) { + if (dentry->d_fsdata == NFS_FSDATA_BLOCKED) + return -ECHILD; + } else { + /* Wait for unlink to complete - see unblock_revalidate() */ + wait_var_event(&dentry->d_fsdata, + smp_load_acquire(&dentry->d_fsdata) + != NFS_FSDATA_BLOCKED); + } + return 0; +} + +static int nfs_lookup_revalidate(struct inode *dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + if (__nfs_lookup_revalidate(dentry, flags)) + return -ECHILD; + return nfs_do_lookup_revalidate(dir, name, dentry, flags); +} + +static void block_revalidate(struct dentry *dentry) +{ + /* old devname - just in case */ + kfree(dentry->d_fsdata); + + /* Any new reference that could lead to an open + * will take ->d_lock in lookup_open() -> d_lookup(). + * Holding this lock ensures we cannot race with + * __nfs_lookup_revalidate() and removes and need + * for further barriers. + */ + lockdep_assert_held(&dentry->d_lock); + + dentry->d_fsdata = NFS_FSDATA_BLOCKED; +} + +static void unblock_revalidate(struct dentry *dentry) +{ + store_release_wake_up(&dentry->d_fsdata, NULL); +} + +/* + * A weaker form of d_revalidate for revalidating just the d_inode(dentry) + * when we don't really care about the dentry name. This is called when a + * pathwalk ends on a dentry that was not found via a normal lookup in the + * parent dir (e.g.: ".", "..", procfs symlinks or mountpoint traversals). + * + * In this situation, we just want to verify that the inode itself is OK + * since the dentry might have changed on the server. + */ +static int nfs_weak_revalidate(struct dentry *dentry, unsigned int flags) +{ + struct inode *inode = d_inode(dentry); + int error = 0; + + /* + * I believe we can only get a negative dentry here in the case of a + * procfs-style symlink. Just assume it's correct for now, but we may + * eventually need to do something more here. + */ + if (!inode) { + dfprintk(LOOKUPCACHE, "%s: %pd2 has negative inode\n", + __func__, dentry); + return 1; + } + + if (is_bad_inode(inode)) { + dfprintk(LOOKUPCACHE, "%s: %pd2 has dud inode\n", + __func__, dentry); + return 0; + } + + error = nfs_lookup_verify_inode(inode, flags); + dfprintk(LOOKUPCACHE, "NFS: %s: inode %llu is %s\n", + __func__, inode->i_ino, error ? "invalid" : "valid"); + return !error; +} + +/* + * This is called from dput() when d_count is going to 0. + */ +static int nfs_dentry_delete(const struct dentry *dentry) +{ + dfprintk(VFS, "NFS: dentry_delete(%pd2, %x)\n", + dentry, dentry->d_flags); + + /* Unhash any dentry with a stale inode */ + if (d_really_is_positive(dentry) && NFS_STALE(d_inode(dentry))) + return 1; + + if (dentry->d_flags & DCACHE_NFSFS_RENAMED) { + /* Unhash it, so that ->d_iput() would be called */ + return 1; + } + if (!(dentry->d_sb->s_flags & SB_ACTIVE)) { + /* Unhash it, so that ancestors of killed async unlink + * files will be cleaned up during umount */ + return 1; + } + return 0; + +} + +/* Ensure that we revalidate inode->i_nlink */ +static void nfs_drop_nlink(struct inode *inode, unsigned long gencount) +{ + struct nfs_inode *nfsi = NFS_I(inode); + + spin_lock(&inode->i_lock); + /* drop the inode if we're reasonably sure this is the last link */ + if (inode->i_nlink > 0 && gencount == nfsi->attr_gencount) + drop_nlink(inode); + nfsi->attr_gencount = nfs_inc_attr_generation_counter(); + nfs_set_cache_invalid( + inode, NFS_INO_INVALID_CHANGE | NFS_INO_INVALID_CTIME | + NFS_INO_INVALID_NLINK); + spin_unlock(&inode->i_lock); +} + +/* + * Called when the dentry loses inode. + * We use it to clean up silly-renamed files. + */ +static void nfs_dentry_iput(struct dentry *dentry, struct inode *inode) +{ + if (dentry->d_flags & DCACHE_NFSFS_RENAMED) { + unsigned long gencount = READ_ONCE(NFS_I(inode)->attr_gencount); + nfs_complete_unlink(dentry, inode); + nfs_drop_nlink(inode, gencount); + } + iput(inode); +} + +static void nfs_d_release(struct dentry *dentry) +{ + /* free cached devname value, if it survived that far */ + if (unlikely(dentry->d_fsdata)) { + if (dentry->d_flags & DCACHE_NFSFS_RENAMED) + WARN_ON(1); + else + kfree(dentry->d_fsdata); + } +} + +const struct dentry_operations nfs_dentry_operations = { + .d_revalidate = nfs_lookup_revalidate, + .d_weak_revalidate = nfs_weak_revalidate, + .d_delete = nfs_dentry_delete, + .d_iput = nfs_dentry_iput, + .d_automount = nfs_d_automount, + .d_release = nfs_d_release, +}; +EXPORT_SYMBOL_GPL(nfs_dentry_operations); + +struct dentry *nfs_lookup(struct inode *dir, struct dentry * dentry, unsigned int flags) +{ + struct dentry *res; + struct inode *inode = NULL; + struct nfs_fh *fhandle = NULL; + struct nfs_fattr *fattr = NULL; + unsigned long dir_verifier; + int error; + + dfprintk(VFS, "NFS: lookup(%pd2)\n", dentry); + nfs_inc_stats(dir, NFSIOS_VFSLOOKUP); + + if (unlikely(dentry->d_name.len > NFS_SERVER(dir)->namelen)) + return ERR_PTR(-ENAMETOOLONG); + + /* + * If we're doing an exclusive create, optimize away the lookup + * but don't hash the dentry. + */ + if (nfs_is_exclusive_create(dir, flags) || flags & LOOKUP_RENAME_TARGET) + return NULL; + + res = ERR_PTR(-ENOMEM); + fhandle = nfs_alloc_fhandle(); + fattr = nfs_alloc_fattr_with_label(NFS_SERVER(dir)); + if (fhandle == NULL || fattr == NULL) + goto out; + + dir_verifier = nfs_save_change_attribute(dir); + trace_nfs_lookup_enter(dir, dentry, flags); + error = NFS_PROTO(dir)->lookup(dir, dentry, &dentry->d_name, + fhandle, fattr); + if (error == -ENOENT) { + if (nfs_server_capable(dir, NFS_CAP_CASE_INSENSITIVE)) + dir_verifier = inode_peek_iversion_raw(dir); + goto no_entry; + } + if (error < 0) { + res = ERR_PTR(error); + goto out; + } + inode = nfs_fhget(dentry->d_sb, fhandle, fattr); + res = ERR_CAST(inode); + if (IS_ERR(res)) + goto out; + + /* Notify readdir to use READDIRPLUS */ + nfs_lookup_advise_force_readdirplus(dir, flags); + +no_entry: + nfs_set_verifier(dentry, dir_verifier); + res = d_splice_alias(inode, dentry); + if (res != NULL) { + if (IS_ERR(res)) + goto out; + nfs_set_verifier(res, dir_verifier); + dentry = res; + } +out: + trace_nfs_lookup_exit(dir, dentry, flags, PTR_ERR_OR_ZERO(res)); + nfs_free_fattr(fattr); + nfs_free_fhandle(fhandle); + return res; +} +EXPORT_SYMBOL_GPL(nfs_lookup); + +void nfs_d_prune_case_insensitive_aliases(struct inode *inode) +{ + /* Case insensitive server? Revalidate dentries */ + if (inode && nfs_server_capable(inode, NFS_CAP_CASE_INSENSITIVE)) + d_prune_aliases(inode); +} +EXPORT_SYMBOL_GPL(nfs_d_prune_case_insensitive_aliases); + +#if IS_ENABLED(CONFIG_NFS_V4) +static int nfs4_lookup_revalidate(struct inode *, const struct qstr *, + struct dentry *, unsigned int); + +const struct dentry_operations nfs4_dentry_operations = { + .d_revalidate = nfs4_lookup_revalidate, + .d_weak_revalidate = nfs_weak_revalidate, + .d_delete = nfs_dentry_delete, + .d_iput = nfs_dentry_iput, + .d_automount = nfs_d_automount, + .d_release = nfs_d_release, +}; +EXPORT_SYMBOL_GPL(nfs4_dentry_operations); + +static struct nfs_open_context *create_nfs_open_context(struct dentry *dentry, int open_flags, struct file *filp) +{ + return alloc_nfs_open_context(dentry, flags_to_mode(open_flags), filp); +} + +static int do_open(struct inode *inode, struct file *filp) +{ + nfs_fscache_open_file(inode, filp); + return 0; +} + +static int nfs_finish_open(struct nfs_open_context *ctx, + struct dentry *dentry, + struct file *file, unsigned open_flags) +{ + int err; + + err = finish_open(file, dentry, do_open); + if (err) + goto out; + if (S_ISREG(file_inode(file)->i_mode)) + nfs_file_set_open_context(file, ctx); + else + err = -EOPENSTALE; +out: + return err; +} + +int nfs_atomic_open(struct inode *dir, struct dentry *dentry, + struct file *file, unsigned open_flags, + umode_t mode) +{ + struct nfs_open_context *ctx; + struct dentry *res; + struct iattr attr = { .ia_valid = ATTR_OPEN }; + struct inode *inode; + unsigned int lookup_flags = 0; + unsigned long dir_verifier; + bool switched = false; + int created = 0; + int err; + + /* Expect a negative dentry */ + BUG_ON(d_inode(dentry)); + + dfprintk(VFS, "NFS: atomic_open(%s/%llu), %pd\n", + dir->i_sb->s_id, dir->i_ino, dentry); + + err = nfs_check_flags(open_flags); + if (err) + return err; + + /* NFS only supports OPEN on regular files */ + if ((open_flags & O_DIRECTORY)) { + if (!d_in_lookup(dentry)) { + /* + * Hashed negative dentry with O_DIRECTORY: dentry was + * revalidated and is fine, no need to perform lookup + * again + */ + return -ENOENT; + } + lookup_flags = LOOKUP_OPEN|LOOKUP_DIRECTORY; + goto no_open; + } + + if (dentry->d_name.len > NFS_SERVER(dir)->namelen) + return -ENAMETOOLONG; + + if (open_flags & O_CREAT) { + struct nfs_server *server = NFS_SERVER(dir); + + if (!(server->attr_bitmask[2] & FATTR4_WORD2_MODE_UMASK)) + mode &= ~current_umask(); + + attr.ia_valid |= ATTR_MODE; + attr.ia_mode = mode; + } + if (open_flags & O_TRUNC) { + attr.ia_valid |= ATTR_SIZE; + attr.ia_size = 0; + } + + if (!(open_flags & O_CREAT) && !d_in_lookup(dentry)) { + d_drop(dentry); + switched = true; + dentry = d_alloc_parallel(dentry->d_parent, + &dentry->d_name); + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + if (unlikely(!d_in_lookup(dentry))) + return finish_no_open(file, dentry); + } + + ctx = create_nfs_open_context(dentry, open_flags, file); + err = PTR_ERR(ctx); + if (IS_ERR(ctx)) + goto out; + + trace_nfs_atomic_open_enter(dir, ctx, open_flags); + inode = NFS_PROTO(dir)->open_context(dir, ctx, open_flags, &attr, &created); + if (created) + file->f_mode |= FMODE_CREATED; + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + trace_nfs_atomic_open_exit(dir, ctx, open_flags, err); + put_nfs_open_context(ctx); + d_drop(dentry); + switch (err) { + case -ENOENT: + if (nfs_server_capable(dir, NFS_CAP_CASE_INSENSITIVE)) + dir_verifier = inode_peek_iversion_raw(dir); + else + dir_verifier = nfs_save_change_attribute(dir); + nfs_set_verifier(dentry, dir_verifier); + d_splice_alias(NULL, dentry); + break; + case -EISDIR: + case -ENOTDIR: + if (open_flags & __O_REGULAR) { + err = -EFTYPE; + break; + } + goto no_open; + case -ELOOP: + if (!(open_flags & O_NOFOLLOW)) + goto no_open; + break; + /* case -EINVAL: */ + default: + break; + } + goto out; + } + file->f_mode |= FMODE_CAN_ODIRECT; + + err = nfs_finish_open(ctx, ctx->dentry, file, open_flags); + trace_nfs_atomic_open_exit(dir, ctx, open_flags, err); + put_nfs_open_context(ctx); +out: + if (unlikely(switched)) { + d_lookup_done(dentry); + dput(dentry); + } + return err; + +no_open: + res = nfs_lookup(dir, dentry, lookup_flags); + if (!res) { + inode = d_inode(dentry); + if ((lookup_flags & LOOKUP_DIRECTORY) && inode && + !(S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode))) + res = ERR_PTR(-ENOTDIR); + else if (inode && S_ISREG(inode->i_mode)) + res = ERR_PTR(-EOPENSTALE); + } else if (!IS_ERR(res)) { + inode = d_inode(res); + if ((lookup_flags & LOOKUP_DIRECTORY) && inode && + !(S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode))) { + dput(res); + res = ERR_PTR(-ENOTDIR); + } else if (inode && S_ISREG(inode->i_mode)) { + dput(res); + res = ERR_PTR(-EOPENSTALE); + } + } + if (switched) { + d_lookup_done(dentry); + if (!res) + res = dentry; + else + dput(dentry); + } + return finish_no_open(file, res); +} +EXPORT_SYMBOL_GPL(nfs_atomic_open); + +static int +nfs4_lookup_revalidate(struct inode *dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + struct inode *inode; + + if (__nfs_lookup_revalidate(dentry, flags)) + return -ECHILD; + + trace_nfs_lookup_revalidate_enter(dir, dentry, flags); + + if (!(flags & LOOKUP_OPEN) || (flags & LOOKUP_DIRECTORY)) + goto full_reval; + if (d_mountpoint(dentry)) + goto full_reval; + + inode = d_inode(dentry); + + /* We can't create new files in nfs_open_revalidate(), so we + * optimize away revalidation of negative dentries. + */ + if (inode == NULL) + goto full_reval; + + if (nfs_verifier_is_delegated(dentry) || + nfs_have_directory_delegation(inode)) + return nfs_lookup_revalidate_delegated(dir, dentry, inode); + + /* NFS only supports OPEN on regular files */ + if (!S_ISREG(inode->i_mode)) + goto full_reval; + + /* We cannot do exclusive creation on a positive dentry */ + if (flags & (LOOKUP_EXCL | LOOKUP_REVAL)) + goto reval_dentry; + + /* Check if the directory changed */ + if (!nfs_check_verifier(dir, dentry, flags & LOOKUP_RCU)) + goto reval_dentry; + + /* Let f_op->open() actually open (and revalidate) the file */ + return 1; +reval_dentry: + if (flags & LOOKUP_RCU) + return -ECHILD; + return nfs_lookup_revalidate_dentry(dir, name, dentry, inode, flags); + +full_reval: + return nfs_do_lookup_revalidate(dir, name, dentry, flags); +} + +#endif /* CONFIG_NFS_V4 */ + +int nfs_atomic_open_v23(struct inode *dir, struct dentry *dentry, + struct file *file, unsigned int open_flags, + umode_t mode) +{ + struct dentry *res = NULL; + /* Same as look+open from lookup_open(), but with different O_TRUNC + * handling. + */ + int error = 0; + + if (dentry->d_name.len > NFS_SERVER(dir)->namelen) + return -ENAMETOOLONG; + + if (open_flags & O_CREAT) { + error = nfs_do_create(dir, dentry, mode, open_flags); + if (!error) { + file->f_mode |= FMODE_CREATED; + return finish_open(file, dentry, NULL); + } else if (error != -EEXIST || open_flags & O_EXCL) + return error; + } + if (d_in_lookup(dentry)) { + /* The only flags nfs_lookup considers are + * LOOKUP_EXCL and LOOKUP_RENAME_TARGET, and + * we want those to be zero so the lookup isn't skipped. + */ + res = nfs_lookup(dir, dentry, 0); + } + return finish_no_open(file, res); + +} +EXPORT_SYMBOL_GPL(nfs_atomic_open_v23); + +struct dentry * +nfs_add_or_obtain(struct dentry *dentry, struct nfs_fh *fhandle, + struct nfs_fattr *fattr) +{ + struct dentry *parent = dget_parent(dentry); + struct inode *dir = d_inode(parent); + struct inode *inode; + struct dentry *d; + int error; + + d_drop(dentry); + + if (fhandle->size == 0) { + error = NFS_PROTO(dir)->lookup(dir, dentry, &dentry->d_name, + fhandle, fattr); + if (error) + goto out_error; + } + nfs_set_verifier(dentry, nfs_save_change_attribute(dir)); + if (!(fattr->valid & NFS_ATTR_FATTR)) { + struct nfs_server *server = NFS_SB(dentry->d_sb); + error = server->nfs_client->rpc_ops->getattr(server, fhandle, + fattr, NULL); + if (error < 0) + goto out_error; + } + inode = nfs_fhget(dentry->d_sb, fhandle, fattr); + d = d_splice_alias(inode, dentry); +out: + dput(parent); + return d; +out_error: + d = ERR_PTR(error); + goto out; +} +EXPORT_SYMBOL_GPL(nfs_add_or_obtain); + +/* + * Code common to create, mkdir, and mknod. + */ +int nfs_instantiate(struct dentry *dentry, struct nfs_fh *fhandle, + struct nfs_fattr *fattr) +{ + struct dentry *d; + + d = nfs_add_or_obtain(dentry, fhandle, fattr); + if (IS_ERR(d)) + return PTR_ERR(d); + + /* Callers don't care */ + dput(d); + return 0; +} +EXPORT_SYMBOL_GPL(nfs_instantiate); + +/* + * Following a failed create operation, we drop the dentry rather + * than retain a negative dentry. This avoids a problem in the event + * that the operation succeeded on the server, but an error in the + * reply path made it appear to have failed. + */ +static int nfs_do_create(struct inode *dir, struct dentry *dentry, + umode_t mode, int open_flags) +{ + struct iattr attr; + int error; + + open_flags |= O_CREAT; + + dfprintk(VFS, "NFS: create(%s/%llu), %pd\n", + dir->i_sb->s_id, dir->i_ino, dentry); + + attr.ia_mode = mode; + attr.ia_valid = ATTR_MODE; + if (open_flags & O_TRUNC) { + attr.ia_size = 0; + attr.ia_valid |= ATTR_SIZE; + } + + trace_nfs_create_enter(dir, dentry, open_flags); + error = NFS_PROTO(dir)->create(dir, dentry, &attr, open_flags); + trace_nfs_create_exit(dir, dentry, open_flags, error); + if (error != 0) + goto out_err; + return 0; +out_err: + d_drop(dentry); + return error; +} + +int nfs_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + return nfs_do_create(dir, dentry, mode, O_EXCL); +} +EXPORT_SYMBOL_GPL(nfs_create); + +/* + * See comments for nfs_proc_create regarding failed operations. + */ +int +nfs_mknod(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) +{ + struct iattr attr; + int status; + + dfprintk(VFS, "NFS: mknod(%s/%llu), %pd\n", + dir->i_sb->s_id, dir->i_ino, dentry); + + attr.ia_mode = mode; + attr.ia_valid = ATTR_MODE; + + trace_nfs_mknod_enter(dir, dentry); + status = NFS_PROTO(dir)->mknod(dir, dentry, &attr, rdev); + trace_nfs_mknod_exit(dir, dentry, status); + if (status != 0) + goto out_err; + return 0; +out_err: + d_drop(dentry); + return status; +} +EXPORT_SYMBOL_GPL(nfs_mknod); + +/* + * See comments for nfs_proc_create regarding failed operations. + */ +struct dentry *nfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct iattr attr; + struct dentry *ret; + + dfprintk(VFS, "NFS: mkdir(%s/%llu), %pd\n", + dir->i_sb->s_id, dir->i_ino, dentry); + + attr.ia_valid = ATTR_MODE; + attr.ia_mode = mode; + + trace_nfs_mkdir_enter(dir, dentry); + ret = NFS_PROTO(dir)->mkdir(dir, dentry, &attr); + trace_nfs_mkdir_exit(dir, dentry, PTR_ERR_OR_ZERO(ret)); + return ret; +} +EXPORT_SYMBOL_GPL(nfs_mkdir); + +static void nfs_dentry_handle_enoent(struct dentry *dentry) +{ + if (simple_positive(dentry)) + d_delete(dentry); +} + +static void nfs_dentry_remove_handle_error(struct inode *dir, + struct dentry *dentry, int error) +{ + switch (error) { + case -ENOENT: + if (d_really_is_positive(dentry)) + d_delete(dentry); + nfs_set_verifier(dentry, nfs_save_change_attribute(dir)); + break; + case 0: + nfs_d_prune_case_insensitive_aliases(d_inode(dentry)); + nfs_set_verifier(dentry, nfs_save_change_attribute(dir)); + } +} + +int nfs_rmdir(struct inode *dir, struct dentry *dentry) +{ + int error; + + dfprintk(VFS, "NFS: rmdir(%s/%llu), %pd\n", + dir->i_sb->s_id, dir->i_ino, dentry); + + trace_nfs_rmdir_enter(dir, dentry); + if (d_really_is_positive(dentry)) { + down_write(&NFS_I(d_inode(dentry))->rmdir_sem); + error = NFS_PROTO(dir)->rmdir(dir, &dentry->d_name); + /* Ensure the VFS deletes this inode */ + switch (error) { + case 0: + clear_nlink(d_inode(dentry)); + break; + case -ENOENT: + nfs_dentry_handle_enoent(dentry); + } + up_write(&NFS_I(d_inode(dentry))->rmdir_sem); + } else + error = NFS_PROTO(dir)->rmdir(dir, &dentry->d_name); + nfs_dentry_remove_handle_error(dir, dentry, error); + trace_nfs_rmdir_exit(dir, dentry, error); + + return error; +} +EXPORT_SYMBOL_GPL(nfs_rmdir); + +/* + * Remove a file after making sure there are no pending writes, + * and after checking that the file has only one user. + * + * We invalidate the attribute cache and free the inode prior to the operation + * to avoid possible races if the server reuses the inode. + */ +static int nfs_safe_remove(struct dentry *dentry) +{ + struct inode *dir = d_inode(dentry->d_parent); + struct inode *inode = d_inode(dentry); + int error = -EBUSY; + + dfprintk(VFS, "NFS: safe_remove(%pd2)\n", dentry); + + /* If the dentry was sillyrenamed, we simply call d_delete() */ + if (dentry->d_flags & DCACHE_NFSFS_RENAMED) { + error = 0; + goto out; + } + + trace_nfs_remove_enter(dir, dentry); + if (inode != NULL) { + unsigned long gencount = READ_ONCE(NFS_I(inode)->attr_gencount); + + error = NFS_PROTO(dir)->remove(dir, dentry); + if (error == 0) + nfs_drop_nlink(inode, gencount); + } else + error = NFS_PROTO(dir)->remove(dir, dentry); + if (error == -ENOENT) + nfs_dentry_handle_enoent(dentry); + trace_nfs_remove_exit(dir, dentry, error); +out: + return error; +} + +/* We do silly rename. In case sillyrename() returns -EBUSY, the inode + * belongs to an active ".nfs..." file and we return -EBUSY. + * + * If sillyrename() returns 0, we do nothing, otherwise we unlink. + */ +int nfs_unlink(struct inode *dir, struct dentry *dentry) +{ + int error; + + dfprintk(VFS, "NFS: unlink(%s/%llu, %pd)\n", dir->i_sb->s_id, + dir->i_ino, dentry); + + trace_nfs_unlink_enter(dir, dentry); + spin_lock(&dentry->d_lock); + if (d_count(dentry) > 1 && !test_bit(NFS_INO_PRESERVE_UNLINKED, + &NFS_I(d_inode(dentry))->flags)) { + spin_unlock(&dentry->d_lock); + /* Start asynchronous writeout of the inode */ + write_inode_now(d_inode(dentry), 0); + error = nfs_sillyrename(dir, dentry); + goto out; + } + /* We must prevent any concurrent open until the unlink + * completes. ->d_revalidate will wait for ->d_fsdata + * to clear. We set it here to ensure no lookup succeeds until + * the unlink is complete on the server. + */ + error = -ETXTBSY; + if (WARN_ON(dentry->d_flags & DCACHE_NFSFS_RENAMED) || + WARN_ON(dentry->d_fsdata == NFS_FSDATA_BLOCKED)) { + spin_unlock(&dentry->d_lock); + goto out; + } + block_revalidate(dentry); + + spin_unlock(&dentry->d_lock); + error = nfs_safe_remove(dentry); + nfs_dentry_remove_handle_error(dir, dentry, error); + unblock_revalidate(dentry); +out: + trace_nfs_unlink_exit(dir, dentry, error); + return error; +} +EXPORT_SYMBOL_GPL(nfs_unlink); + +/* + * To create a symbolic link, most file systems instantiate a new inode, + * add a page to it containing the path, then write it out to the disk + * using prepare_write/commit_write. + * + * Unfortunately the NFS client can't create the in-core inode first + * because it needs a file handle to create an in-core inode (see + * fs/nfs/inode.c:nfs_fhget). We only have a file handle *after* the + * symlink request has completed on the server. + * + * So instead we allocate a raw page, copy the symname into it, then do + * the SYMLINK request with the page as the buffer. If it succeeds, we + * now have a new file handle and can instantiate an in-core NFS inode + * and move the raw page into its mapping. + */ +int nfs_symlink(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const char *symname) +{ + struct folio *folio; + char *kaddr; + struct iattr attr; + unsigned int pathlen = strlen(symname); + int error; + + dfprintk(VFS, "NFS: symlink(%s/%llu, %pd, %s)\n", dir->i_sb->s_id, + dir->i_ino, dentry, symname); + + if (pathlen > PAGE_SIZE) + return -ENAMETOOLONG; + + attr.ia_mode = S_IFLNK | S_IRWXUGO; + attr.ia_valid = ATTR_MODE; + + folio = folio_alloc(GFP_USER, 0); + if (!folio) + return -ENOMEM; + + kaddr = folio_address(folio); + memcpy(kaddr, symname, pathlen); + if (pathlen < PAGE_SIZE) + memset(kaddr + pathlen, 0, PAGE_SIZE - pathlen); + + trace_nfs_symlink_enter(dir, dentry); + error = NFS_PROTO(dir)->symlink(dir, dentry, folio, pathlen, &attr); + trace_nfs_symlink_exit(dir, dentry, error); + if (error != 0) { + dfprintk(VFS, "NFS: symlink(%s/%llu, %pd, %s) error %d\n", + dir->i_sb->s_id, dir->i_ino, + dentry, symname, error); + d_drop(dentry); + folio_put(folio); + return error; + } + + nfs_set_verifier(dentry, nfs_save_change_attribute(dir)); + + /* + * No big deal if we can't add this page to the page cache here. + * READLINK will get the missing page from the server if needed. + */ + if (filemap_add_folio(d_inode(dentry)->i_mapping, folio, 0, + GFP_KERNEL) == 0) { + folio_mark_uptodate(folio); + folio_unlock(folio); + } + + folio_put(folio); + return 0; +} +EXPORT_SYMBOL_GPL(nfs_symlink); + +int +nfs_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry) +{ + struct inode *inode = d_inode(old_dentry); + int error; + + dfprintk(VFS, "NFS: link(%pd2 -> %pd2)\n", + old_dentry, dentry); + + trace_nfs_link_enter(inode, dir, dentry); + d_drop(dentry); + if (S_ISREG(inode->i_mode)) + nfs_sync_inode(inode); + error = NFS_PROTO(dir)->link(inode, dir, &dentry->d_name); + if (error == 0) { + nfs_set_verifier(dentry, nfs_save_change_attribute(dir)); + ihold(inode); + d_add(dentry, inode); + } + trace_nfs_link_exit(inode, dir, dentry, error); + return error; +} +EXPORT_SYMBOL_GPL(nfs_link); + +static void +nfs_unblock_rename(struct rpc_task *task, struct nfs_renamedata *data) +{ + struct dentry *new_dentry = data->new_dentry; + + unblock_revalidate(new_dentry); +} + +static bool nfs_rename_is_unsafe_cross_dir(struct dentry *old_dentry, + struct dentry *new_dentry) +{ + struct nfs_server *server = NFS_SB(old_dentry->d_sb); + + if (old_dentry->d_parent != new_dentry->d_parent) + return false; + if (server->fh_expire_type & NFS_FH_RENAME_UNSAFE) + return !(server->fh_expire_type & NFS_FH_NOEXPIRE_WITH_OPEN); + return true; +} + +/* + * RENAME + * FIXME: Some nfsds, like the Linux user space nfsd, may generate a + * different file handle for the same inode after a rename (e.g. when + * moving to a different directory). A fail-safe method to do so would + * be to look up old_dir/old_name, create a link to new_dir/new_name and + * rename the old file using the sillyrename stuff. This way, the original + * file in old_dir will go away when the last process iput()s the inode. + * + * FIXED. + * + * It actually works quite well. One needs to have the possibility for + * at least one ".nfs..." file in each directory the file ever gets + * moved or linked to which happens automagically with the new + * implementation that only depends on the dcache stuff instead of + * using the inode layer + * + * Unfortunately, things are a little more complicated than indicated + * above. For a cross-directory move, we want to make sure we can get + * rid of the old inode after the operation. This means there must be + * no pending writes (if it's a file), and the use count must be 1. + * If these conditions are met, we can drop the dentries before doing + * the rename. + */ +int nfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) +{ + struct inode *old_inode = d_inode(old_dentry); + struct inode *new_inode = d_inode(new_dentry); + unsigned long new_gencount = 0; + struct dentry *dentry = NULL; + struct rpc_task *task; + bool must_unblock = false; + int error = -EBUSY; + + if (flags) + return -EINVAL; + + dfprintk(VFS, "NFS: rename(%pd2 -> %pd2, ct=%d)\n", + old_dentry, new_dentry, + d_count(new_dentry)); + + trace_nfs_rename_enter(old_dir, old_dentry, new_dir, new_dentry); + /* + * For non-directories, check whether the target is busy and if so, + * make a copy of the dentry and then do a silly-rename. If the + * silly-rename succeeds, the copied dentry is hashed and becomes + * the new target. + */ + if (new_inode && !S_ISDIR(new_inode->i_mode)) { + /* We must prevent any concurrent open until the unlink + * completes. ->d_revalidate will wait for ->d_fsdata + * to clear. We set it here to ensure no lookup succeeds until + * the unlink is complete on the server. + */ + error = -ETXTBSY; + if (WARN_ON(new_dentry->d_flags & DCACHE_NFSFS_RENAMED) || + WARN_ON(new_dentry->d_fsdata == NFS_FSDATA_BLOCKED)) + goto out; + + spin_lock(&new_dentry->d_lock); + if (d_count(new_dentry) > 2) { + int err; + + spin_unlock(&new_dentry->d_lock); + + /* copy the target dentry's name */ + dentry = d_alloc(new_dentry->d_parent, + &new_dentry->d_name); + if (!dentry) + goto out; + + /* silly-rename the existing target ... */ + err = nfs_sillyrename(new_dir, new_dentry); + if (err) + goto out; + + new_dentry = dentry; + new_inode = NULL; + } else { + block_revalidate(new_dentry); + must_unblock = true; + new_gencount = NFS_I(new_inode)->attr_gencount; + spin_unlock(&new_dentry->d_lock); + } + + } + + if (S_ISREG(old_inode->i_mode) && + nfs_rename_is_unsafe_cross_dir(old_dentry, new_dentry)) + nfs_sync_inode(old_inode); + task = nfs_async_rename(old_dir, new_dir, old_dentry, new_dentry, + must_unblock ? nfs_unblock_rename : NULL); + if (IS_ERR(task)) { + if (must_unblock) + unblock_revalidate(new_dentry); + error = PTR_ERR(task); + goto out; + } + + error = rpc_wait_for_completion_task(task); + if (error != 0) { + ((struct nfs_renamedata *)task->tk_calldata)->cancelled = 1; + /* Paired with the atomic_dec_and_test() barrier in rpc_do_put_task() */ + smp_wmb(); + } else + error = task->tk_status; + rpc_put_task(task); + /* Ensure the inode attributes are revalidated */ + if (error == 0) { + spin_lock(&old_inode->i_lock); + NFS_I(old_inode)->attr_gencount = nfs_inc_attr_generation_counter(); + nfs_set_cache_invalid(old_inode, NFS_INO_INVALID_CHANGE | + NFS_INO_INVALID_CTIME | + NFS_INO_REVAL_FORCED); + spin_unlock(&old_inode->i_lock); + } +out: + trace_nfs_rename_exit(old_dir, old_dentry, + new_dir, new_dentry, error); + if (!error) { + if (new_inode != NULL) + nfs_drop_nlink(new_inode, new_gencount); + /* + * The d_move() should be here instead of in an async RPC completion + * handler because we need the proper locks to move the dentry. If + * we're interrupted by a signal, the async RPC completion handler + * should mark the directories for revalidation. + */ + d_move(old_dentry, new_dentry); + nfs_set_verifier(old_dentry, + nfs_save_change_attribute(new_dir)); + } else if (error == -ENOENT) + nfs_dentry_handle_enoent(old_dentry); + + /* new dentry created? */ + if (dentry) + dput(dentry); + return error; +} +EXPORT_SYMBOL_GPL(nfs_rename); + +static DEFINE_SPINLOCK(nfs_access_lru_lock); +static LIST_HEAD(nfs_access_lru_list); +static atomic_long_t nfs_access_nr_entries; + +static unsigned long nfs_access_max_cachesize = 4*1024*1024; +module_param(nfs_access_max_cachesize, ulong, 0644); +MODULE_PARM_DESC(nfs_access_max_cachesize, "NFS access maximum total cache length"); + +static void nfs_access_free_entry(struct nfs_access_entry *entry) +{ + put_group_info(entry->group_info); + kfree_rcu(entry, rcu_head); + smp_mb__before_atomic(); + atomic_long_dec(&nfs_access_nr_entries); + smp_mb__after_atomic(); +} + +static void nfs_access_free_list(struct list_head *head) +{ + struct nfs_access_entry *cache; + + while (!list_empty(head)) { + cache = list_entry(head->next, struct nfs_access_entry, lru); + list_del(&cache->lru); + nfs_access_free_entry(cache); + } +} + +static unsigned long +nfs_do_access_cache_scan(unsigned int nr_to_scan) +{ + LIST_HEAD(head); + struct nfs_inode *nfsi, *next; + struct nfs_access_entry *cache; + long freed = 0; + + spin_lock(&nfs_access_lru_lock); + list_for_each_entry_safe(nfsi, next, &nfs_access_lru_list, access_cache_inode_lru) { + struct inode *inode; + + if (nr_to_scan-- == 0) + break; + inode = &nfsi->vfs_inode; + spin_lock(&inode->i_lock); + if (list_empty(&nfsi->access_cache_entry_lru)) + goto remove_lru_entry; + cache = list_entry(nfsi->access_cache_entry_lru.next, + struct nfs_access_entry, lru); + list_move(&cache->lru, &head); + rb_erase(&cache->rb_node, &nfsi->access_cache); + freed++; + if (!list_empty(&nfsi->access_cache_entry_lru)) + list_move_tail(&nfsi->access_cache_inode_lru, + &nfs_access_lru_list); + else { +remove_lru_entry: + list_del_init(&nfsi->access_cache_inode_lru); + smp_mb__before_atomic(); + clear_bit(NFS_INO_ACL_LRU_SET, &nfsi->flags); + smp_mb__after_atomic(); + } + spin_unlock(&inode->i_lock); + } + spin_unlock(&nfs_access_lru_lock); + nfs_access_free_list(&head); + return freed; +} + +unsigned long +nfs_access_cache_scan(struct shrinker *shrink, struct shrink_control *sc) +{ + int nr_to_scan = sc->nr_to_scan; + gfp_t gfp_mask = sc->gfp_mask; + + if ((gfp_mask & GFP_KERNEL) != GFP_KERNEL) + return SHRINK_STOP; + return nfs_do_access_cache_scan(nr_to_scan); +} + + +unsigned long +nfs_access_cache_count(struct shrinker *shrink, struct shrink_control *sc) +{ + return vfs_pressure_ratio(atomic_long_read(&nfs_access_nr_entries)); +} + +static void +nfs_access_cache_enforce_limit(void) +{ + long nr_entries = atomic_long_read(&nfs_access_nr_entries); + unsigned long diff; + unsigned int nr_to_scan; + + if (nr_entries < 0 || nr_entries <= nfs_access_max_cachesize) + return; + nr_to_scan = 100; + diff = nr_entries - nfs_access_max_cachesize; + if (diff < nr_to_scan) + nr_to_scan = diff; + nfs_do_access_cache_scan(nr_to_scan); +} + +static void __nfs_access_zap_cache(struct nfs_inode *nfsi, struct list_head *head) +{ + struct rb_root *root_node = &nfsi->access_cache; + struct rb_node *n; + struct nfs_access_entry *entry; + + /* Unhook entries from the cache */ + while ((n = rb_first(root_node)) != NULL) { + entry = rb_entry(n, struct nfs_access_entry, rb_node); + rb_erase(n, root_node); + list_move(&entry->lru, head); + } + nfsi->cache_validity &= ~NFS_INO_INVALID_ACCESS; +} + +void nfs_access_zap_cache(struct inode *inode) +{ + LIST_HEAD(head); + + if (test_bit(NFS_INO_ACL_LRU_SET, &NFS_I(inode)->flags) == 0) + return; + /* Remove from global LRU init */ + spin_lock(&nfs_access_lru_lock); + if (test_and_clear_bit(NFS_INO_ACL_LRU_SET, &NFS_I(inode)->flags)) + list_del_init(&NFS_I(inode)->access_cache_inode_lru); + + spin_lock(&inode->i_lock); + __nfs_access_zap_cache(NFS_I(inode), &head); + spin_unlock(&inode->i_lock); + spin_unlock(&nfs_access_lru_lock); + nfs_access_free_list(&head); +} +EXPORT_SYMBOL_GPL(nfs_access_zap_cache); + +static int access_cmp(const struct cred *a, const struct nfs_access_entry *b) +{ + struct group_info *ga, *gb; + int g; + + if (uid_lt(a->fsuid, b->fsuid)) + return -1; + if (uid_gt(a->fsuid, b->fsuid)) + return 1; + + if (gid_lt(a->fsgid, b->fsgid)) + return -1; + if (gid_gt(a->fsgid, b->fsgid)) + return 1; + + ga = a->group_info; + gb = b->group_info; + if (ga == gb) + return 0; + if (ga == NULL) + return -1; + if (gb == NULL) + return 1; + if (ga->ngroups < gb->ngroups) + return -1; + if (ga->ngroups > gb->ngroups) + return 1; + + for (g = 0; g < ga->ngroups; g++) { + if (gid_lt(ga->gid[g], gb->gid[g])) + return -1; + if (gid_gt(ga->gid[g], gb->gid[g])) + return 1; + } + return 0; +} + +static struct nfs_access_entry *nfs_access_search_rbtree(struct inode *inode, const struct cred *cred) +{ + struct rb_node *n = NFS_I(inode)->access_cache.rb_node; + + while (n != NULL) { + struct nfs_access_entry *entry = + rb_entry(n, struct nfs_access_entry, rb_node); + int cmp = access_cmp(cred, entry); + + if (cmp < 0) + n = n->rb_left; + else if (cmp > 0) + n = n->rb_right; + else + return entry; + } + return NULL; +} + +static u64 nfs_access_login_time(const struct task_struct *task, + const struct cred *cred) +{ + const struct task_struct *parent; + const struct cred *pcred; + u64 ret; + + rcu_read_lock(); + for (;;) { + parent = rcu_dereference(task->real_parent); + pcred = __task_cred(parent); + if (parent == task || cred_fscmp(pcred, cred) != 0) + break; + task = parent; + } + ret = task->start_time; + rcu_read_unlock(); + return ret; +} + +static int nfs_access_get_cached_locked(struct inode *inode, const struct cred *cred, u32 *mask, bool may_block) +{ + struct nfs_inode *nfsi = NFS_I(inode); + u64 login_time = nfs_access_login_time(current, cred); + struct nfs_access_entry *cache; + bool retry = true; + int err; + + spin_lock(&inode->i_lock); + for(;;) { + if (nfsi->cache_validity & NFS_INO_INVALID_ACCESS) + goto out_zap; + cache = nfs_access_search_rbtree(inode, cred); + err = -ENOENT; + if (cache == NULL) + goto out; + /* Found an entry, is our attribute cache valid? */ + if (!nfs_check_cache_invalid(inode, NFS_INO_INVALID_ACCESS)) + break; + if (!retry) + break; + err = -ECHILD; + if (!may_block) + goto out; + spin_unlock(&inode->i_lock); + err = __nfs_revalidate_inode(NFS_SERVER(inode), inode); + if (err) + return err; + spin_lock(&inode->i_lock); + retry = false; + } + err = -ENOENT; + if ((s64)(login_time - cache->timestamp) > 0) + goto out; + *mask = cache->mask; + list_move_tail(&cache->lru, &nfsi->access_cache_entry_lru); + err = 0; +out: + spin_unlock(&inode->i_lock); + return err; +out_zap: + spin_unlock(&inode->i_lock); + nfs_access_zap_cache(inode); + return -ENOENT; +} + +static int nfs_access_get_cached_rcu(struct inode *inode, const struct cred *cred, u32 *mask) +{ + /* Only check the most recently returned cache entry, + * but do it without locking. + */ + struct nfs_inode *nfsi = NFS_I(inode); + u64 login_time = nfs_access_login_time(current, cred); + struct nfs_access_entry *cache; + int err = -ECHILD; + struct list_head *lh; + + rcu_read_lock(); + if (nfsi->cache_validity & NFS_INO_INVALID_ACCESS) + goto out; + lh = rcu_dereference(list_tail_rcu(&nfsi->access_cache_entry_lru)); + cache = list_entry(lh, struct nfs_access_entry, lru); + if (lh == &nfsi->access_cache_entry_lru || + access_cmp(cred, cache) != 0) + cache = NULL; + if (cache == NULL) + goto out; + if ((s64)(login_time - cache->timestamp) > 0) + goto out; + if (nfs_check_cache_invalid(inode, NFS_INO_INVALID_ACCESS)) + goto out; + *mask = cache->mask; + err = 0; +out: + rcu_read_unlock(); + return err; +} + +int nfs_access_get_cached(struct inode *inode, const struct cred *cred, + u32 *mask, bool may_block) +{ + int status; + + status = nfs_access_get_cached_rcu(inode, cred, mask); + if (status != 0) + status = nfs_access_get_cached_locked(inode, cred, mask, + may_block); + + return status; +} +EXPORT_SYMBOL_GPL(nfs_access_get_cached); + +static void nfs_access_add_rbtree(struct inode *inode, + struct nfs_access_entry *set, + const struct cred *cred) +{ + struct nfs_inode *nfsi = NFS_I(inode); + struct rb_root *root_node = &nfsi->access_cache; + struct rb_node **p = &root_node->rb_node; + struct rb_node *parent = NULL; + struct nfs_access_entry *entry; + int cmp; + + spin_lock(&inode->i_lock); + while (*p != NULL) { + parent = *p; + entry = rb_entry(parent, struct nfs_access_entry, rb_node); + cmp = access_cmp(cred, entry); + + if (cmp < 0) + p = &parent->rb_left; + else if (cmp > 0) + p = &parent->rb_right; + else + goto found; + } + rb_link_node(&set->rb_node, parent, p); + rb_insert_color(&set->rb_node, root_node); + list_add_tail(&set->lru, &nfsi->access_cache_entry_lru); + spin_unlock(&inode->i_lock); + return; +found: + rb_replace_node(parent, &set->rb_node, root_node); + list_add_tail(&set->lru, &nfsi->access_cache_entry_lru); + list_del(&entry->lru); + spin_unlock(&inode->i_lock); + nfs_access_free_entry(entry); +} + +void nfs_access_add_cache(struct inode *inode, struct nfs_access_entry *set, + const struct cred *cred) +{ + struct nfs_access_entry *cache = kmalloc_obj(*cache); + if (cache == NULL) + return; + RB_CLEAR_NODE(&cache->rb_node); + cache->fsuid = cred->fsuid; + cache->fsgid = cred->fsgid; + cache->group_info = get_group_info(cred->group_info); + cache->mask = set->mask; + cache->timestamp = ktime_get_ns(); + + /* The above field assignments must be visible + * before this item appears on the lru. We cannot easily + * use rcu_assign_pointer, so just force the memory barrier. + */ + smp_wmb(); + nfs_access_add_rbtree(inode, cache, cred); + + /* Update accounting */ + smp_mb__before_atomic(); + atomic_long_inc(&nfs_access_nr_entries); + smp_mb__after_atomic(); + + /* Add inode to global LRU list */ + if (!test_bit(NFS_INO_ACL_LRU_SET, &NFS_I(inode)->flags)) { + spin_lock(&nfs_access_lru_lock); + if (!test_and_set_bit(NFS_INO_ACL_LRU_SET, &NFS_I(inode)->flags)) + list_add_tail(&NFS_I(inode)->access_cache_inode_lru, + &nfs_access_lru_list); + spin_unlock(&nfs_access_lru_lock); + } + nfs_access_cache_enforce_limit(); +} +EXPORT_SYMBOL_GPL(nfs_access_add_cache); + +#define NFS_MAY_READ (NFS_ACCESS_READ) +#define NFS_MAY_WRITE (NFS_ACCESS_MODIFY | \ + NFS_ACCESS_EXTEND | \ + NFS_ACCESS_DELETE) +#define NFS_FILE_MAY_WRITE (NFS_ACCESS_MODIFY | \ + NFS_ACCESS_EXTEND) +#define NFS_DIR_MAY_WRITE NFS_MAY_WRITE +#define NFS_MAY_LOOKUP (NFS_ACCESS_LOOKUP) +#define NFS_MAY_EXECUTE (NFS_ACCESS_EXECUTE) +static int +nfs_access_calc_mask(u32 access_result, umode_t umode) +{ + int mask = 0; + + if (access_result & NFS_MAY_READ) + mask |= MAY_READ; + if (S_ISDIR(umode)) { + if ((access_result & NFS_DIR_MAY_WRITE) == NFS_DIR_MAY_WRITE) + mask |= MAY_WRITE; + if ((access_result & NFS_MAY_LOOKUP) == NFS_MAY_LOOKUP) + mask |= MAY_EXEC; + } else if (S_ISREG(umode)) { + if ((access_result & NFS_FILE_MAY_WRITE) == NFS_FILE_MAY_WRITE) + mask |= MAY_WRITE; + if ((access_result & NFS_MAY_EXECUTE) == NFS_MAY_EXECUTE) + mask |= MAY_EXEC; + } else if (access_result & NFS_MAY_WRITE) + mask |= MAY_WRITE; + return mask; +} + +void nfs_access_set_mask(struct nfs_access_entry *entry, u32 access_result) +{ + entry->mask = access_result; +} +EXPORT_SYMBOL_GPL(nfs_access_set_mask); + +static int nfs_do_access(struct inode *inode, const struct cred *cred, int mask) +{ + struct nfs_access_entry cache; + bool may_block = (mask & MAY_NOT_BLOCK) == 0; + int cache_mask = -1; + int status; + + trace_nfs_access_enter(inode); + + status = nfs_access_get_cached(inode, cred, &cache.mask, may_block); + if (status == 0) + goto out_cached; + + status = -ECHILD; + if (!may_block) + goto out; + + /* + * Determine which access bits we want to ask for... + */ + cache.mask = NFS_ACCESS_READ | NFS_ACCESS_MODIFY | NFS_ACCESS_EXTEND | + nfs_access_xattr_mask(NFS_SERVER(inode)); + if (S_ISDIR(inode->i_mode)) + cache.mask |= NFS_ACCESS_DELETE | NFS_ACCESS_LOOKUP; + else + cache.mask |= NFS_ACCESS_EXECUTE; + status = NFS_PROTO(inode)->access(inode, &cache, cred); + if (status != 0) { + if (status == -ESTALE) { + if (!S_ISDIR(inode->i_mode)) + nfs_set_inode_stale(inode); + else + nfs_zap_caches(inode); + } + goto out; + } + nfs_access_add_cache(inode, &cache, cred); +out_cached: + cache_mask = nfs_access_calc_mask(cache.mask, inode->i_mode); + if ((mask & ~cache_mask & (MAY_READ | MAY_WRITE | MAY_EXEC)) != 0) + status = -EACCES; +out: + trace_nfs_access_exit(inode, mask, cache_mask, status); + return status; +} + +static int nfs_open_permission_mask(int openflags) +{ + int mask = 0; + + if (openflags & __FMODE_EXEC) { + /* ONLY check exec rights */ + mask = MAY_EXEC; + } else { + if ((openflags & O_ACCMODE) != O_WRONLY) + mask |= MAY_READ; + if ((openflags & O_ACCMODE) != O_RDONLY) + mask |= MAY_WRITE; + if (openflags & O_TRUNC) + mask |= MAY_WRITE; + } + + return mask; +} + +int nfs_may_open(struct inode *inode, const struct cred *cred, int openflags) +{ + return nfs_do_access(inode, cred, nfs_open_permission_mask(openflags)); +} +EXPORT_SYMBOL_GPL(nfs_may_open); + +static int nfs_execute_ok(struct inode *inode, int mask) +{ + struct nfs_server *server = NFS_SERVER(inode); + int ret = 0; + + if (S_ISDIR(inode->i_mode)) + return 0; + if (nfs_check_cache_invalid(inode, NFS_INO_INVALID_MODE)) { + if (mask & MAY_NOT_BLOCK) + return -ECHILD; + ret = __nfs_revalidate_inode(server, inode); + } + if (ret == 0 && !execute_ok(inode)) + ret = -EACCES; + return ret; +} + +int nfs_permission(struct mnt_idmap *idmap, + struct inode *inode, + int mask) +{ + const struct cred *cred = current_cred(); + int res = 0; + + nfs_inc_stats(inode, NFSIOS_VFSACCESS); + + if ((mask & (MAY_READ | MAY_WRITE | MAY_EXEC)) == 0) + goto out; + /* Is this sys_access() ? */ + if (mask & (MAY_ACCESS | MAY_CHDIR)) + goto force_lookup; + + switch (inode->i_mode & S_IFMT) { + case S_IFLNK: + goto out; + case S_IFREG: + if ((mask & MAY_OPEN) && + nfs_server_capable(inode, NFS_CAP_ATOMIC_OPEN)) + return 0; + break; + case S_IFDIR: + /* + * Optimize away all write operations, since the server + * will check permissions when we perform the op. + */ + if ((mask & MAY_WRITE) && !(mask & MAY_READ)) + goto out; + } + +force_lookup: + if (!NFS_PROTO(inode)->access) + goto out_notsup; + + res = nfs_do_access(inode, cred, mask); +out: + if (!res && (mask & MAY_EXEC)) + res = nfs_execute_ok(inode, mask); + + dfprintk(VFS, "NFS: permission(%s/%llu), mask=0x%x, res=%d\n", + inode->i_sb->s_id, inode->i_ino, mask, res); + return res; +out_notsup: + if (mask & MAY_NOT_BLOCK) + return -ECHILD; + + res = nfs_revalidate_inode(inode, NFS_INO_INVALID_MODE | + NFS_INO_INVALID_OTHER); + if (res == 0) + res = generic_permission(&nop_mnt_idmap, inode, mask); + goto out; +} +EXPORT_SYMBOL_GPL(nfs_permission); diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c new file mode 100644 index 000000000..e626c7249 --- /dev/null +++ b/fs/nfs/direct.c @@ -0,0 +1,1112 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/nfs/direct.c + * + * Copyright (C) 2003 by Chuck Lever + * + * High-performance uncached I/O for the Linux NFS client + * + * There are important applications whose performance or correctness + * depends on uncached access to file data. Database clusters + * (multiple copies of the same instance running on separate hosts) + * implement their own cache coherency protocol that subsumes file + * system cache protocols. Applications that process datasets + * considerably larger than the client's memory do not always benefit + * from a local cache. A streaming video server, for instance, has no + * need to cache the contents of a file. + * + * When an application requests uncached I/O, all read and write requests + * are made directly to the server; data stored or fetched via these + * requests is not cached in the Linux page cache. The client does not + * correct unaligned requests from applications. All requested bytes are + * held on permanent storage before a direct write system call returns to + * an application. + * + * Solaris implements an uncached I/O facility called directio() that + * is used for backups and sequential I/O to very large files. Solaris + * also supports uncaching whole NFS partitions with "-o forcedirectio," + * an undocumented mount option. + * + * Designed by Jeff Kimmel, Chuck Lever, and Trond Myklebust, with + * help from Andrew Morton. + * + * 18 Dec 2001 Initial implementation for 2.4 --cel + * 08 Jul 2002 Version for 2.4.19, with bug fixes --trondmy + * 08 Jun 2003 Port to 2.5 APIs --cel + * 31 Mar 2004 Handle direct I/O without VFS support --cel + * 15 Sep 2004 Parallel async reads --cel + * 04 May 2005 support O_DIRECT with aio --cel + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +#include "delegation.h" +#include "internal.h" +#include "iostat.h" +#include "pnfs.h" +#include "fscache.h" +#include "nfstrace.h" + +#define NFSDBG_FACILITY NFSDBG_VFS + +static struct kmem_cache *nfs_direct_cachep; + +static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops; +static const struct nfs_commit_completion_ops nfs_direct_commit_completion_ops; +static void nfs_direct_write_complete(struct nfs_direct_req *dreq); +static void nfs_direct_write_schedule_work(struct work_struct *work); + +static inline void get_dreq(struct nfs_direct_req *dreq) +{ + atomic_inc(&dreq->io_count); +} + +static inline int put_dreq(struct nfs_direct_req *dreq) +{ + return atomic_dec_and_test(&dreq->io_count); +} + +static void +nfs_direct_handle_truncated(struct nfs_direct_req *dreq, + const struct nfs_pgio_header *hdr, + ssize_t dreq_len) +{ + if (!(test_bit(NFS_IOHDR_ERROR, &hdr->flags) || + test_bit(NFS_IOHDR_EOF, &hdr->flags))) + return; + if (dreq->max_count >= dreq_len) { + dreq->max_count = dreq_len; + if (dreq->count > dreq_len) + dreq->count = dreq_len; + } + + if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) && !dreq->error) + dreq->error = hdr->error; +} + +static void +nfs_direct_count_bytes(struct nfs_direct_req *dreq, + const struct nfs_pgio_header *hdr) +{ + loff_t hdr_end = hdr->io_start + hdr->good_bytes; + ssize_t dreq_len = 0; + + if (hdr_end > dreq->io_start) + dreq_len = hdr_end - dreq->io_start; + + nfs_direct_handle_truncated(dreq, hdr, dreq_len); + + if (dreq_len > dreq->max_count) + dreq_len = dreq->max_count; + + if (dreq->count < dreq_len) + dreq->count = dreq_len; +} + +static void nfs_direct_truncate_request(struct nfs_direct_req *dreq, + struct nfs_page *req) +{ + loff_t offs = req_offset(req); + size_t req_start = (size_t)(offs - dreq->io_start); + + if (req_start < dreq->max_count) + dreq->max_count = req_start; + if (req_start < dreq->count) + dreq->count = req_start; +} + +static void nfs_direct_file_adjust_size_locked(struct inode *inode, + loff_t offset, size_t count) +{ + loff_t newsize = offset + (loff_t)count; + loff_t oldsize = i_size_read(inode); + + if (newsize > oldsize) { + i_size_write(inode, newsize); + NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_SIZE; + trace_nfs_size_grow(inode, newsize); + nfs_inc_stats(inode, NFSIOS_EXTENDWRITE); + } +} + +/** + * nfs_swap_rw - NFS address space operation for swap I/O + * @iocb: target I/O control block + * @iter: I/O buffer + * + * Perform IO to the swap-file. This is much like direct IO. + */ +int nfs_swap_rw(struct kiocb *iocb, struct iov_iter *iter) +{ + ssize_t ret; + + if (iov_iter_rw(iter) == READ) + ret = nfs_file_direct_read(iocb, iter, true); + else + ret = nfs_file_direct_write(iocb, iter, true); + if (ret < 0) + return ret; + return 0; +} + +static void nfs_direct_release_pages(struct page **pages, unsigned int npages) +{ + unsigned int i; + for (i = 0; i < npages; i++) + put_page(pages[i]); +} + +void nfs_init_cinfo_from_dreq(struct nfs_commit_info *cinfo, + struct nfs_direct_req *dreq) +{ + cinfo->inode = dreq->inode; + cinfo->mds = &dreq->mds_cinfo; + cinfo->ds = &dreq->ds_cinfo; + cinfo->dreq = dreq; + cinfo->completion_ops = &nfs_direct_commit_completion_ops; +} + +static inline struct nfs_direct_req *nfs_direct_req_alloc(void) +{ + struct nfs_direct_req *dreq; + + dreq = kmem_cache_zalloc(nfs_direct_cachep, GFP_KERNEL); + if (!dreq) + return NULL; + + kref_init(&dreq->kref); + kref_get(&dreq->kref); + init_completion(&dreq->completion); + INIT_LIST_HEAD(&dreq->mds_cinfo.list); + pnfs_init_ds_commit_info(&dreq->ds_cinfo); + INIT_WORK(&dreq->work, nfs_direct_write_schedule_work); + spin_lock_init(&dreq->lock); + + return dreq; +} + +static void nfs_direct_req_free(struct kref *kref) +{ + struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref); + + pnfs_release_ds_info(&dreq->ds_cinfo, dreq->inode); + if (dreq->l_ctx != NULL) + nfs_put_lock_context(dreq->l_ctx); + if (dreq->ctx != NULL) + put_nfs_open_context(dreq->ctx); + kmem_cache_free(nfs_direct_cachep, dreq); +} + +static void nfs_direct_req_release(struct nfs_direct_req *dreq) +{ + kref_put(&dreq->kref, nfs_direct_req_free); +} + +ssize_t nfs_dreq_bytes_left(struct nfs_direct_req *dreq, loff_t offset) +{ + loff_t start = offset - dreq->io_start; + return dreq->max_count - start; +} +EXPORT_SYMBOL_GPL(nfs_dreq_bytes_left); + +/* + * Collects and returns the final error value/byte-count. + */ +static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq) +{ + ssize_t result = -EIOCBQUEUED; + + /* Async requests don't wait here */ + if (dreq->iocb) + goto out; + + result = wait_for_completion_killable(&dreq->completion); + + if (!result) { + result = dreq->count; + WARN_ON_ONCE(dreq->count < 0); + } + if (!result) + result = dreq->error; + +out: + return (ssize_t) result; +} + +/* + * Synchronous I/O uses a stack-allocated iocb. Thus we can't trust + * the iocb is still valid here if this is a synchronous request. + */ +static void nfs_direct_complete(struct nfs_direct_req *dreq) +{ + struct inode *inode = dreq->inode; + + inode_dio_end(inode); + + if (dreq->iocb) { + long res = (long) dreq->error; + if (dreq->count != 0) { + res = (long) dreq->count; + WARN_ON_ONCE(dreq->count < 0); + } + dreq->iocb->ki_complete(dreq->iocb, res); + } + + complete(&dreq->completion); + + nfs_direct_req_release(dreq); +} + +static void nfs_direct_read_completion(struct nfs_pgio_header *hdr) +{ + unsigned long bytes = 0; + struct nfs_direct_req *dreq = hdr->dreq; + + spin_lock(&dreq->lock); + if (test_bit(NFS_IOHDR_REDO, &hdr->flags)) { + spin_unlock(&dreq->lock); + goto out_put; + } + + nfs_direct_count_bytes(dreq, hdr); + spin_unlock(&dreq->lock); + + nfs_update_delegated_atime(dreq->inode); + + while (!list_empty(&hdr->pages)) { + struct nfs_page *req = nfs_list_entry(hdr->pages.next); + struct page *page = req->wb_page; + + if (!PageCompound(page) && bytes < hdr->good_bytes && + (dreq->flags == NFS_ODIRECT_SHOULD_DIRTY)) + set_page_dirty(page); + bytes += req->wb_bytes; + nfs_list_remove_request(req); + nfs_release_request(req); + } +out_put: + if (put_dreq(dreq)) + nfs_direct_complete(dreq); + hdr->release(hdr); +} + +static void nfs_read_sync_pgio_error(struct list_head *head, int error) +{ + struct nfs_page *req; + + while (!list_empty(head)) { + req = nfs_list_entry(head->next); + nfs_list_remove_request(req); + nfs_release_request(req); + } +} + +static void nfs_direct_pgio_init(struct nfs_pgio_header *hdr) +{ + get_dreq(hdr->dreq); + set_bit(NFS_IOHDR_ODIRECT, &hdr->flags); +} + +static const struct nfs_pgio_completion_ops nfs_direct_read_completion_ops = { + .error_cleanup = nfs_read_sync_pgio_error, + .init_hdr = nfs_direct_pgio_init, + .completion = nfs_direct_read_completion, +}; + +/* + * For each rsize'd chunk of the user's buffer, dispatch an NFS READ + * operation. If nfs_readdata_alloc() or get_user_pages() fails, + * bail and stop sending more reads. Read length accounting is + * handled automatically by nfs_direct_read_result(). Otherwise, if + * no requests have been sent, just return an error. + */ + +static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq, + struct iov_iter *iter, + loff_t pos) +{ + struct nfs_pageio_descriptor desc; + struct inode *inode = dreq->inode; + ssize_t result = -EINVAL; + size_t requested_bytes = 0; + size_t rsize = max_t(size_t, NFS_SERVER(inode)->rsize, PAGE_SIZE); + + nfs_pageio_init_read(&desc, dreq->inode, false, + &nfs_direct_read_completion_ops); + get_dreq(dreq); + desc.pg_dreq = dreq; + inode_dio_begin(inode); + + while (iov_iter_count(iter)) { + struct page **pagevec; + size_t bytes; + size_t pgbase; + unsigned npages, i; + + result = iov_iter_get_pages_alloc2(iter, &pagevec, + rsize, &pgbase); + if (result < 0) + break; + + bytes = result; + npages = (result + pgbase + PAGE_SIZE - 1) / PAGE_SIZE; + for (i = 0; i < npages; i++) { + struct nfs_page *req; + unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase); + /* XXX do we need to do the eof zeroing found in async_filler? */ + req = nfs_page_create_from_page(dreq->ctx, pagevec[i], + pgbase, pos, req_len); + if (IS_ERR(req)) { + result = PTR_ERR(req); + break; + } + if (!nfs_pageio_add_request(&desc, req)) { + result = desc.pg_error; + nfs_release_request(req); + break; + } + pgbase = 0; + bytes -= req_len; + requested_bytes += req_len; + pos += req_len; + } + nfs_direct_release_pages(pagevec, npages); + kvfree(pagevec); + if (result < 0) + break; + } + + nfs_pageio_complete(&desc); + + /* + * If no bytes were started, return the error, and let the + * generic layer handle the completion. + */ + if (requested_bytes == 0) { + inode_dio_end(inode); + nfs_direct_req_release(dreq); + return result < 0 ? result : -EIO; + } + + if (put_dreq(dreq)) + nfs_direct_complete(dreq); + return requested_bytes; +} + +/** + * nfs_file_direct_read - file direct read operation for NFS files + * @iocb: target I/O control block + * @iter: vector of user buffers into which to read data + * @swap: flag indicating this is swap IO, not O_DIRECT IO + * + * We use this function for direct reads instead of calling + * generic_file_aio_read() in order to avoid gfar's check to see if + * the request starts before the end of the file. For that check + * to work, we must generate a GETATTR before each direct read, and + * even then there is a window between the GETATTR and the subsequent + * READ where the file size could change. Our preference is simply + * to do all reads the application wants, and the server will take + * care of managing the end of file boundary. + * + * This function also eliminates unnecessarily updating the file's + * atime locally, as the NFS server sets the file's atime, and this + * client must read the updated atime from the server back into its + * cache. + */ +ssize_t nfs_file_direct_read(struct kiocb *iocb, struct iov_iter *iter, + bool swap) +{ + struct file *file = iocb->ki_filp; + struct address_space *mapping = file->f_mapping; + struct inode *inode = mapping->host; + struct nfs_direct_req *dreq; + struct nfs_lock_context *l_ctx; + ssize_t result, requested; + size_t count = iov_iter_count(iter); + nfs_add_stats(mapping->host, NFSIOS_DIRECTREADBYTES, count); + + dfprintk(FILE, "NFS: direct read(%pD2, %zd@%Ld)\n", + file, count, (long long) iocb->ki_pos); + + result = 0; + if (!count) + goto out; + + task_io_account_read(count); + + result = -ENOMEM; + dreq = nfs_direct_req_alloc(); + if (dreq == NULL) + goto out; + + dreq->inode = inode; + dreq->max_count = count; + dreq->io_start = iocb->ki_pos; + dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp)); + l_ctx = nfs_get_lock_context(dreq->ctx); + if (IS_ERR(l_ctx)) { + result = PTR_ERR(l_ctx); + nfs_direct_req_release(dreq); + goto out_release; + } + dreq->l_ctx = l_ctx; + if (!is_sync_kiocb(iocb)) { + dreq->iocb = iocb; + } else if (iocb->ki_flags & IOCB_NOWAIT) { + result = -EAGAIN; + nfs_direct_req_release(dreq); + goto out_release; + } + + if (user_backed_iter(iter)) + dreq->flags = NFS_ODIRECT_SHOULD_DIRTY; + + if (!swap) { + if (iocb->ki_flags & IOCB_NOWAIT) + result = nfs_start_io_direct_nowait(inode); + else + result = nfs_start_io_direct(inode); + if (result) { + /* release the reference that would usually be + * consumed by nfs_direct_read_schedule_iovec() + */ + nfs_direct_req_release(dreq); + goto out_release; + } + } + + NFS_I(inode)->read_io += count; + requested = nfs_direct_read_schedule_iovec(dreq, iter, iocb->ki_pos); + + if (!swap) + nfs_end_io_direct(inode); + + if (requested > 0) { + result = nfs_direct_wait(dreq); + if (result > 0) { + requested -= result; + iocb->ki_pos += result; + } + iov_iter_revert(iter, requested); + } else { + result = requested; + } + +out_release: + nfs_direct_req_release(dreq); +out: + return result; +} + +static void nfs_direct_add_page_head(struct list_head *list, + struct nfs_page *req) +{ + struct nfs_page *head = req->wb_head; + + if (!list_empty(&head->wb_list) || !nfs_lock_request(head)) + return; + if (!list_empty(&head->wb_list)) { + nfs_unlock_request(head); + return; + } + list_add(&head->wb_list, list); + kref_get(&head->wb_kref); + kref_get(&head->wb_kref); +} + +static void nfs_direct_join_group(struct list_head *list, + struct nfs_commit_info *cinfo, + struct inode *inode) +{ + struct nfs_page *req, *subreq; + + list_for_each_entry(req, list, wb_list) { + if (req->wb_head != req) { + nfs_direct_add_page_head(&req->wb_list, req); + continue; + } + subreq = req->wb_this_page; + if (subreq == req) + continue; + do { + /* + * Remove subrequests from this list before freeing + * them in the call to nfs_join_page_group(). + */ + if (!list_empty(&subreq->wb_list)) { + nfs_list_remove_request(subreq); + nfs_release_request(subreq); + } + } while ((subreq = subreq->wb_this_page) != req); + nfs_join_page_group(req, cinfo, inode); + } +} + +static void +nfs_direct_write_scan_commit_list(struct inode *inode, + struct list_head *list, + struct nfs_commit_info *cinfo) +{ + mutex_lock(&NFS_I(cinfo->inode)->commit_mutex); + pnfs_recover_commit_reqs(list, cinfo); + nfs_scan_commit_list(&cinfo->mds->list, list, cinfo, 0); + mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); +} + +static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq) +{ + struct nfs_pageio_descriptor desc; + struct nfs_page *req; + LIST_HEAD(reqs); + struct nfs_commit_info cinfo; + + nfs_init_cinfo_from_dreq(&cinfo, dreq); + nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo); + + nfs_direct_join_group(&reqs, &cinfo, dreq->inode); + + nfs_clear_pnfs_ds_commit_verifiers(&dreq->ds_cinfo); + get_dreq(dreq); + + nfs_pageio_init_write(&desc, dreq->inode, FLUSH_STABLE, false, + &nfs_direct_write_completion_ops); + desc.pg_dreq = dreq; + + while (!list_empty(&reqs)) { + req = nfs_list_entry(reqs.next); + /* Bump the transmission count */ + req->wb_nio++; + if (!nfs_pageio_add_request(&desc, req)) { + spin_lock(&dreq->lock); + if (dreq->error < 0) { + desc.pg_error = dreq->error; + } else if (desc.pg_error != -EAGAIN) { + dreq->flags = 0; + if (!desc.pg_error) + desc.pg_error = -EIO; + dreq->error = desc.pg_error; + } else + dreq->flags = NFS_ODIRECT_RESCHED_WRITES; + spin_unlock(&dreq->lock); + break; + } + nfs_release_request(req); + } + nfs_pageio_complete(&desc); + + while (!list_empty(&reqs)) { + req = nfs_list_entry(reqs.next); + nfs_list_remove_request(req); + nfs_unlock_and_release_request(req); + if (desc.pg_error == -EAGAIN) { + nfs_mark_request_commit(req, NULL, &cinfo, 0); + } else { + spin_lock(&dreq->lock); + nfs_direct_truncate_request(dreq, req); + spin_unlock(&dreq->lock); + nfs_release_request(req); + } + } + + if (put_dreq(dreq)) + nfs_direct_write_complete(dreq); +} + +static void nfs_direct_commit_complete(struct nfs_commit_data *data) +{ + const struct nfs_writeverf *verf = data->res.verf; + struct nfs_direct_req *dreq = data->dreq; + struct nfs_commit_info cinfo; + struct nfs_page *req; + int status = data->task.tk_status; + + trace_nfs_direct_commit_complete(dreq); + + spin_lock(&dreq->lock); + if (status < 0) { + /* Errors in commit are fatal */ + dreq->error = status; + dreq->flags = NFS_ODIRECT_DONE; + } else { + status = dreq->error; + } + spin_unlock(&dreq->lock); + + nfs_init_cinfo_from_dreq(&cinfo, dreq); + + while (!list_empty(&data->pages)) { + req = nfs_list_entry(data->pages.next); + nfs_list_remove_request(req); + if (status < 0) { + spin_lock(&dreq->lock); + nfs_direct_truncate_request(dreq, req); + spin_unlock(&dreq->lock); + nfs_release_request(req); + } else if (!nfs_write_match_verf(verf, req)) { + spin_lock(&dreq->lock); + if (dreq->flags == 0) + dreq->flags = NFS_ODIRECT_RESCHED_WRITES; + spin_unlock(&dreq->lock); + /* + * Despite the reboot, the write was successful, + * so reset wb_nio. + */ + req->wb_nio = 0; + nfs_mark_request_commit(req, NULL, &cinfo, 0); + } else + nfs_release_request(req); + nfs_unlock_and_release_request(req); + } + + if (nfs_commit_end(cinfo.mds)) + nfs_direct_write_complete(dreq); +} + +static void nfs_direct_resched_write(struct nfs_commit_info *cinfo, + struct nfs_page *req) +{ + struct nfs_direct_req *dreq = cinfo->dreq; + + trace_nfs_direct_resched_write(dreq); + + spin_lock(&dreq->lock); + if (dreq->flags != NFS_ODIRECT_DONE) + dreq->flags = NFS_ODIRECT_RESCHED_WRITES; + spin_unlock(&dreq->lock); + nfs_mark_request_commit(req, NULL, cinfo, 0); +} + +static const struct nfs_commit_completion_ops nfs_direct_commit_completion_ops = { + .completion = nfs_direct_commit_complete, + .resched_write = nfs_direct_resched_write, +}; + +static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq) +{ + int res; + struct nfs_commit_info cinfo; + LIST_HEAD(mds_list); + + nfs_init_cinfo_from_dreq(&cinfo, dreq); + nfs_commit_begin(cinfo.mds); + nfs_scan_commit(dreq->inode, &mds_list, &cinfo); + res = nfs_generic_commit_list(dreq->inode, &mds_list, 0, &cinfo); + if (res < 0) { /* res == -ENOMEM */ + spin_lock(&dreq->lock); + if (dreq->flags == 0) + dreq->flags = NFS_ODIRECT_RESCHED_WRITES; + spin_unlock(&dreq->lock); + } + if (nfs_commit_end(cinfo.mds)) + nfs_direct_write_complete(dreq); +} + +static void nfs_direct_write_clear_reqs(struct nfs_direct_req *dreq) +{ + struct nfs_commit_info cinfo; + struct nfs_page *req; + LIST_HEAD(reqs); + + nfs_init_cinfo_from_dreq(&cinfo, dreq); + nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo); + + while (!list_empty(&reqs)) { + req = nfs_list_entry(reqs.next); + nfs_list_remove_request(req); + nfs_direct_truncate_request(dreq, req); + nfs_release_request(req); + nfs_unlock_and_release_request(req); + } +} + +static void nfs_direct_write_schedule_work(struct work_struct *work) +{ + struct nfs_direct_req *dreq = container_of(work, struct nfs_direct_req, work); + int flags = dreq->flags; + + dreq->flags = 0; + switch (flags) { + case NFS_ODIRECT_DO_COMMIT: + nfs_direct_commit_schedule(dreq); + break; + case NFS_ODIRECT_RESCHED_WRITES: + nfs_direct_write_reschedule(dreq); + break; + default: + nfs_direct_write_clear_reqs(dreq); + nfs_zap_mapping(dreq->inode, dreq->inode->i_mapping); + nfs_direct_complete(dreq); + } +} + +static void nfs_direct_write_complete(struct nfs_direct_req *dreq) +{ + trace_nfs_direct_write_complete(dreq); + queue_work(nfsiod_workqueue, &dreq->work); /* Calls nfs_direct_write_schedule_work */ +} + +static void nfs_direct_write_completion(struct nfs_pgio_header *hdr) +{ + struct nfs_direct_req *dreq = hdr->dreq; + struct nfs_commit_info cinfo; + struct inode *inode = dreq->inode; + int flags = NFS_ODIRECT_DONE; + + trace_nfs_direct_write_completion(dreq); + + nfs_init_cinfo_from_dreq(&cinfo, dreq); + + spin_lock(&dreq->lock); + if (test_bit(NFS_IOHDR_REDO, &hdr->flags)) { + spin_unlock(&dreq->lock); + goto out_put; + } + + nfs_direct_count_bytes(dreq, hdr); + if (test_bit(NFS_IOHDR_UNSTABLE_WRITES, &hdr->flags) && + !test_bit(NFS_IOHDR_ERROR, &hdr->flags)) { + if (!dreq->flags) + dreq->flags = NFS_ODIRECT_DO_COMMIT; + flags = dreq->flags; + } + spin_unlock(&dreq->lock); + + spin_lock(&inode->i_lock); + nfs_direct_file_adjust_size_locked(inode, dreq->io_start, dreq->count); + nfs_update_delegated_mtime_locked(dreq->inode); + spin_unlock(&inode->i_lock); + + while (!list_empty(&hdr->pages)) { + struct nfs_page *req; + + req = nfs_list_entry(hdr->pages.next); + nfs_list_remove_request(req); + if (flags == NFS_ODIRECT_DO_COMMIT) { + kref_get(&req->wb_kref); + memcpy(&req->wb_verf, &hdr->verf.verifier, + sizeof(req->wb_verf)); + nfs_mark_request_commit(req, hdr->lseg, &cinfo, + hdr->ds_commit_idx); + } else if (flags == NFS_ODIRECT_RESCHED_WRITES) { + kref_get(&req->wb_kref); + nfs_mark_request_commit(req, NULL, &cinfo, 0); + } + nfs_unlock_and_release_request(req); + } + +out_put: + if (put_dreq(dreq)) + nfs_direct_write_complete(dreq); + hdr->release(hdr); +} + +static void nfs_write_sync_pgio_error(struct list_head *head, int error) +{ + struct nfs_page *req; + + while (!list_empty(head)) { + req = nfs_list_entry(head->next); + nfs_list_remove_request(req); + nfs_unlock_and_release_request(req); + } +} + +static void nfs_direct_write_reschedule_io(struct nfs_pgio_header *hdr) +{ + struct nfs_direct_req *dreq = hdr->dreq; + struct nfs_page *req; + struct nfs_commit_info cinfo; + + trace_nfs_direct_write_reschedule_io(dreq); + + nfs_init_cinfo_from_dreq(&cinfo, dreq); + spin_lock(&dreq->lock); + if (dreq->error == 0) + dreq->flags = NFS_ODIRECT_RESCHED_WRITES; + set_bit(NFS_IOHDR_REDO, &hdr->flags); + spin_unlock(&dreq->lock); + while (!list_empty(&hdr->pages)) { + req = nfs_list_entry(hdr->pages.next); + nfs_list_remove_request(req); + nfs_unlock_request(req); + nfs_mark_request_commit(req, NULL, &cinfo, 0); + } +} + +static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops = { + .error_cleanup = nfs_write_sync_pgio_error, + .init_hdr = nfs_direct_pgio_init, + .completion = nfs_direct_write_completion, + .reschedule_io = nfs_direct_write_reschedule_io, +}; + + +/* + * NB: Return the value of the first error return code. Subsequent + * errors after the first one are ignored. + */ +/* + * For each wsize'd chunk of the user's buffer, dispatch an NFS WRITE + * operation. If nfs_writedata_alloc() or get_user_pages() fails, + * bail and stop sending more writes. Write length accounting is + * handled automatically by nfs_direct_write_result(). Otherwise, if + * no requests have been sent, just return an error. + */ +static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq, + struct iov_iter *iter, + loff_t pos, int ioflags) +{ + struct nfs_pageio_descriptor desc; + struct inode *inode = dreq->inode; + struct nfs_commit_info cinfo; + ssize_t result = 0; + size_t requested_bytes = 0; + size_t wsize = max_t(size_t, NFS_SERVER(inode)->wsize, PAGE_SIZE); + bool defer = false; + + trace_nfs_direct_write_schedule_iovec(dreq); + + nfs_pageio_init_write(&desc, inode, ioflags, false, + &nfs_direct_write_completion_ops); + desc.pg_dreq = dreq; + get_dreq(dreq); + inode_dio_begin(inode); + + NFS_I(inode)->write_io += iov_iter_count(iter); + while (iov_iter_count(iter)) { + struct page **pagevec; + size_t bytes; + size_t pgbase; + unsigned npages, i; + + result = iov_iter_get_pages_alloc2(iter, &pagevec, + wsize, &pgbase); + if (result < 0) + break; + + bytes = result; + npages = (result + pgbase + PAGE_SIZE - 1) / PAGE_SIZE; + for (i = 0; i < npages; i++) { + struct nfs_page *req; + unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase); + + req = nfs_page_create_from_page(dreq->ctx, pagevec[i], + pgbase, pos, req_len); + if (IS_ERR(req)) { + result = PTR_ERR(req); + break; + } + + if (desc.pg_error < 0) { + nfs_free_request(req); + result = desc.pg_error; + break; + } + + pgbase = 0; + bytes -= req_len; + requested_bytes += req_len; + pos += req_len; + + if (defer) { + nfs_mark_request_commit(req, NULL, &cinfo, 0); + continue; + } + + nfs_lock_request(req); + if (nfs_pageio_add_request(&desc, req)) + continue; + + /* Exit on hard errors */ + if (desc.pg_error < 0 && desc.pg_error != -EAGAIN) { + result = desc.pg_error; + nfs_unlock_and_release_request(req); + break; + } + + /* If the error is soft, defer remaining requests */ + nfs_init_cinfo_from_dreq(&cinfo, dreq); + spin_lock(&dreq->lock); + dreq->flags = NFS_ODIRECT_RESCHED_WRITES; + spin_unlock(&dreq->lock); + nfs_unlock_request(req); + nfs_mark_request_commit(req, NULL, &cinfo, 0); + desc.pg_error = 0; + defer = true; + } + nfs_direct_release_pages(pagevec, npages); + kvfree(pagevec); + if (result < 0) + break; + } + nfs_pageio_complete(&desc); + + /* + * If no bytes were started, return the error, and let the + * generic layer handle the completion. + */ + if (requested_bytes == 0) { + inode_dio_end(inode); + nfs_direct_req_release(dreq); + return result < 0 ? result : -EIO; + } + + if (put_dreq(dreq)) + nfs_direct_write_complete(dreq); + return requested_bytes; +} + +/** + * nfs_file_direct_write - file direct write operation for NFS files + * @iocb: target I/O control block + * @iter: vector of user buffers from which to write data + * @swap: flag indicating this is swap IO, not O_DIRECT IO + * + * We use this function for direct writes instead of calling + * generic_file_aio_write() in order to avoid taking the inode + * semaphore and updating the i_size. The NFS server will set + * the new i_size and this client must read the updated size + * back into its cache. We let the server do generic write + * parameter checking and report problems. + * + * We eliminate local atime updates, see direct read above. + * + * We avoid unnecessary page cache invalidations for normal cached + * readers of this file. + * + * Note that O_APPEND is not supported for NFS direct writes, as there + * is no atomic O_APPEND write facility in the NFS protocol. + */ +ssize_t nfs_file_direct_write(struct kiocb *iocb, struct iov_iter *iter, + bool swap) +{ + ssize_t result, requested; + size_t count; + struct file *file = iocb->ki_filp; + struct address_space *mapping = file->f_mapping; + struct inode *inode = mapping->host; + struct nfs_direct_req *dreq; + struct nfs_lock_context *l_ctx; + loff_t pos, end; + + dfprintk(FILE, "NFS: direct write(%pD2, %zd@%Ld)\n", + file, iov_iter_count(iter), (long long) iocb->ki_pos); + + if (swap) + /* bypass generic checks */ + result = iov_iter_count(iter); + else + result = generic_write_checks(iocb, iter); + if (result <= 0) + return result; + count = result; + nfs_add_stats(mapping->host, NFSIOS_DIRECTWRITTENBYTES, count); + + pos = iocb->ki_pos; + end = (pos + iov_iter_count(iter) - 1) >> PAGE_SHIFT; + + task_io_account_write(count); + + result = -ENOMEM; + dreq = nfs_direct_req_alloc(); + if (!dreq) + goto out; + + dreq->inode = inode; + dreq->max_count = count; + dreq->io_start = pos; + dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp)); + l_ctx = nfs_get_lock_context(dreq->ctx); + if (IS_ERR(l_ctx)) { + result = PTR_ERR(l_ctx); + nfs_direct_req_release(dreq); + goto out_release; + } + dreq->l_ctx = l_ctx; + if (!is_sync_kiocb(iocb)) + dreq->iocb = iocb; + pnfs_init_ds_commit_info_ops(&dreq->ds_cinfo, inode); + + if (swap) { + requested = nfs_direct_write_schedule_iovec(dreq, iter, pos, + FLUSH_STABLE); + } else { + result = nfs_start_io_direct(inode); + if (result) { + /* release the reference that would usually be + * consumed by nfs_direct_write_schedule_iovec() + */ + nfs_direct_req_release(dreq); + goto out_release; + } + + requested = nfs_direct_write_schedule_iovec(dreq, iter, pos, + FLUSH_COND_STABLE); + + if (mapping->nrpages) { + invalidate_inode_pages2_range(mapping, + pos >> PAGE_SHIFT, end); + } + + nfs_end_io_direct(inode); + } + + if (requested > 0) { + result = nfs_direct_wait(dreq); + if (result > 0) { + requested -= result; + iocb->ki_pos = pos + result; + /* XXX: should check the generic_write_sync retval */ + generic_write_sync(iocb, result); + } + iov_iter_revert(iter, requested); + } else { + result = requested; + } + nfs_fscache_invalidate(inode, FSCACHE_INVAL_DIO_WRITE); +out_release: + nfs_direct_req_release(dreq); +out: + return result; +} + +/** + * nfs_init_directcache - create a slab cache for nfs_direct_req structures + * + */ +int __init nfs_init_directcache(void) +{ + nfs_direct_cachep = kmem_cache_create("nfs_direct_cache", + sizeof(struct nfs_direct_req), + 0, SLAB_RECLAIM_ACCOUNT, + NULL); + if (nfs_direct_cachep == NULL) + return -ENOMEM; + + return 0; +} + +/** + * nfs_destroy_directcache - destroy the slab cache for nfs_direct_req structures + * + */ +void nfs_destroy_directcache(void) +{ + kmem_cache_destroy(nfs_direct_cachep); +} diff --git a/fs/nfs/dns_resolve.c b/fs/nfs/dns_resolve.c new file mode 100644 index 000000000..acd3511c0 --- /dev/null +++ b/fs/nfs/dns_resolve.c @@ -0,0 +1,480 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/nfs/dns_resolve.c + * + * Copyright (c) 2009 Trond Myklebust + * + * Resolves DNS hostnames into valid ip addresses + */ + +#include +#include +#include + +#include "dns_resolve.h" + +#ifdef CONFIG_NFS_USE_KERNEL_DNS + +#include + +ssize_t nfs_dns_resolve_name(struct net *net, char *name, size_t namelen, + struct sockaddr_storage *ss, size_t salen) +{ + struct sockaddr *sa = (struct sockaddr *)ss; + ssize_t ret; + char *ip_addr = NULL; + int ip_len; + + ip_len = dns_query(net, NULL, name, namelen, NULL, &ip_addr, NULL, + false); + if (ip_len > 0) + ret = rpc_pton(net, ip_addr, ip_len, sa, salen); + else + ret = -ESRCH; + kfree(ip_addr); + return ret; +} + +#else + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nfs4_fs.h" +#include "cache_lib.h" +#include "netns.h" + +#define NFS_DNS_HASHBITS 4 +#define NFS_DNS_HASHTBL_SIZE (1 << NFS_DNS_HASHBITS) + +struct nfs_dns_ent { + struct cache_head h; + + char *hostname; + size_t namelen; + + struct sockaddr_storage addr; + size_t addrlen; + struct rcu_head rcu_head; +}; + + +static void nfs_dns_ent_update(struct cache_head *cnew, + struct cache_head *ckey) +{ + struct nfs_dns_ent *new; + struct nfs_dns_ent *key; + + new = container_of(cnew, struct nfs_dns_ent, h); + key = container_of(ckey, struct nfs_dns_ent, h); + + memcpy(&new->addr, &key->addr, key->addrlen); + new->addrlen = key->addrlen; +} + +static void nfs_dns_ent_init(struct cache_head *cnew, + struct cache_head *ckey) +{ + struct nfs_dns_ent *new; + struct nfs_dns_ent *key; + + new = container_of(cnew, struct nfs_dns_ent, h); + key = container_of(ckey, struct nfs_dns_ent, h); + + kfree(new->hostname); + new->hostname = kmemdup_nul(key->hostname, key->namelen, GFP_KERNEL); + if (new->hostname) { + new->namelen = key->namelen; + nfs_dns_ent_update(cnew, ckey); + } else { + new->namelen = 0; + new->addrlen = 0; + } +} + +static void nfs_dns_ent_free_rcu(struct rcu_head *head) +{ + struct nfs_dns_ent *item; + + item = container_of(head, struct nfs_dns_ent, rcu_head); + kfree(item->hostname); + kfree(item); +} + +static void nfs_dns_ent_put(struct kref *ref) +{ + struct nfs_dns_ent *item; + + item = container_of(ref, struct nfs_dns_ent, h.ref); + call_rcu(&item->rcu_head, nfs_dns_ent_free_rcu); +} + +static struct cache_head *nfs_dns_ent_alloc(void) +{ + struct nfs_dns_ent *item = kmalloc_obj(*item); + + if (item != NULL) { + item->hostname = NULL; + item->namelen = 0; + item->addrlen = 0; + return &item->h; + } + return NULL; +}; + +static unsigned int nfs_dns_hash(const struct nfs_dns_ent *key) +{ + return hash_str(key->hostname, NFS_DNS_HASHBITS); +} + +static void nfs_dns_request(struct cache_detail *cd, + struct cache_head *ch, + char **bpp, int *blen) +{ + struct nfs_dns_ent *key = container_of(ch, struct nfs_dns_ent, h); + + qword_add(bpp, blen, key->hostname); + (*bpp)[-1] = '\n'; +} + +static int nfs_dns_upcall(struct cache_detail *cd, + struct cache_head *ch) +{ + struct nfs_dns_ent *key = container_of(ch, struct nfs_dns_ent, h); + + if (test_and_set_bit(CACHE_PENDING, &ch->flags)) + return 0; + if (!nfs_cache_upcall(cd, key->hostname)) + return 0; + clear_bit(CACHE_PENDING, &ch->flags); + return sunrpc_cache_upcall_warn(cd, ch); +} + +static int nfs_dns_match(struct cache_head *ca, + struct cache_head *cb) +{ + struct nfs_dns_ent *a; + struct nfs_dns_ent *b; + + a = container_of(ca, struct nfs_dns_ent, h); + b = container_of(cb, struct nfs_dns_ent, h); + + if (a->namelen == 0 || a->namelen != b->namelen) + return 0; + return memcmp(a->hostname, b->hostname, a->namelen) == 0; +} + +static int nfs_dns_show(struct seq_file *m, struct cache_detail *cd, + struct cache_head *h) +{ + struct nfs_dns_ent *item; + long ttl; + + if (h == NULL) { + seq_puts(m, "# ip address hostname ttl\n"); + return 0; + } + item = container_of(h, struct nfs_dns_ent, h); + ttl = item->h.expiry_time - seconds_since_boot(); + if (ttl < 0) + ttl = 0; + + if (!test_bit(CACHE_NEGATIVE, &h->flags)) { + char buf[INET6_ADDRSTRLEN+IPV6_SCOPE_ID_LEN+1]; + + rpc_ntop((struct sockaddr *)&item->addr, buf, sizeof(buf)); + seq_printf(m, "%15s ", buf); + } else + seq_puts(m, " "); + seq_printf(m, "%15s %ld\n", item->hostname, ttl); + return 0; +} + +static struct nfs_dns_ent *nfs_dns_lookup(struct cache_detail *cd, + struct nfs_dns_ent *key) +{ + struct cache_head *ch; + + ch = sunrpc_cache_lookup_rcu(cd, + &key->h, + nfs_dns_hash(key)); + if (!ch) + return NULL; + return container_of(ch, struct nfs_dns_ent, h); +} + +static struct nfs_dns_ent *nfs_dns_update(struct cache_detail *cd, + struct nfs_dns_ent *new, + struct nfs_dns_ent *key) +{ + struct cache_head *ch; + + ch = sunrpc_cache_update(cd, + &new->h, &key->h, + nfs_dns_hash(key)); + if (!ch) + return NULL; + return container_of(ch, struct nfs_dns_ent, h); +} + +static int nfs_dns_parse(struct cache_detail *cd, char *buf, int buflen) +{ + char buf1[NFS_DNS_HOSTNAME_MAXLEN+1]; + struct nfs_dns_ent key, *item; + unsigned int ttl; + ssize_t len; + int ret = -EINVAL; + + if (buf[buflen-1] != '\n') + goto out; + buf[buflen-1] = '\0'; + + len = qword_get(&buf, buf1, sizeof(buf1)); + if (len <= 0) + goto out; + key.addrlen = rpc_pton(cd->net, buf1, len, + (struct sockaddr *)&key.addr, + sizeof(key.addr)); + + len = qword_get(&buf, buf1, sizeof(buf1)); + if (len <= 0) + goto out; + + key.hostname = buf1; + key.namelen = len; + memset(&key.h, 0, sizeof(key.h)); + + if (get_uint(&buf, &ttl) < 0) + goto out; + if (ttl == 0) + goto out; + key.h.expiry_time = ttl + seconds_since_boot(); + + ret = -ENOMEM; + item = nfs_dns_lookup(cd, &key); + if (item == NULL) + goto out; + + if (key.addrlen == 0) + set_bit(CACHE_NEGATIVE, &key.h.flags); + + item = nfs_dns_update(cd, &key, item); + if (item == NULL) + goto out; + + ret = 0; + cache_put(&item->h, cd); +out: + return ret; +} + +static int do_cache_lookup(struct cache_detail *cd, + struct nfs_dns_ent *key, + struct nfs_dns_ent **item, + struct nfs_cache_defer_req *dreq) +{ + int ret = -ENOMEM; + + *item = nfs_dns_lookup(cd, key); + if (*item) { + ret = cache_check(cd, &(*item)->h, &dreq->req); + if (ret) + *item = NULL; + } + return ret; +} + +static int do_cache_lookup_nowait(struct cache_detail *cd, + struct nfs_dns_ent *key, + struct nfs_dns_ent **item) +{ + int ret = -ENOMEM; + + *item = nfs_dns_lookup(cd, key); + if (!*item) + goto out_err; + ret = -ETIMEDOUT; + if (!test_bit(CACHE_VALID, &(*item)->h.flags) + || (*item)->h.expiry_time < seconds_since_boot() + || cd->flush_time > (*item)->h.last_refresh) + goto out_put; + ret = -ENOENT; + if (test_bit(CACHE_NEGATIVE, &(*item)->h.flags)) + goto out_put; + return 0; +out_put: + cache_put(&(*item)->h, cd); +out_err: + *item = NULL; + return ret; +} + +static int do_cache_lookup_wait(struct cache_detail *cd, + struct nfs_dns_ent *key, + struct nfs_dns_ent **item) +{ + struct nfs_cache_defer_req *dreq; + int ret = -ENOMEM; + + dreq = nfs_cache_defer_req_alloc(); + if (!dreq) + goto out; + ret = do_cache_lookup(cd, key, item, dreq); + if (ret == -EAGAIN) { + ret = nfs_cache_wait_for_upcall(dreq); + if (!ret) + ret = do_cache_lookup_nowait(cd, key, item); + } + nfs_cache_defer_req_put(dreq); +out: + return ret; +} + +ssize_t nfs_dns_resolve_name(struct net *net, char *name, + size_t namelen, struct sockaddr_storage *ss, size_t salen) +{ + struct nfs_dns_ent key = { + .hostname = name, + .namelen = namelen, + }; + struct nfs_dns_ent *item = NULL; + ssize_t ret; + struct nfs_net *nn = net_generic(net, nfs_net_id); + + ret = do_cache_lookup_wait(nn->nfs_dns_resolve, &key, &item); + if (ret == 0) { + if (salen >= item->addrlen) { + memcpy(ss, &item->addr, item->addrlen); + ret = item->addrlen; + } else + ret = -EOVERFLOW; + cache_put(&item->h, nn->nfs_dns_resolve); + } else if (ret == -ENOENT) + ret = -ESRCH; + return ret; +} + +static struct cache_detail nfs_dns_resolve_template = { + .owner = THIS_MODULE, + .hash_size = NFS_DNS_HASHTBL_SIZE, + .name = "dns_resolve", + .cache_put = nfs_dns_ent_put, + .cache_upcall = nfs_dns_upcall, + .cache_request = nfs_dns_request, + .cache_parse = nfs_dns_parse, + .cache_show = nfs_dns_show, + .match = nfs_dns_match, + .init = nfs_dns_ent_init, + .update = nfs_dns_ent_update, + .alloc = nfs_dns_ent_alloc, +}; + + +int nfs_dns_resolver_cache_init(struct net *net) +{ + int err; + struct nfs_net *nn = net_generic(net, nfs_net_id); + + nn->nfs_dns_resolve = cache_create_net(&nfs_dns_resolve_template, net); + if (IS_ERR(nn->nfs_dns_resolve)) + return PTR_ERR(nn->nfs_dns_resolve); + + err = nfs_cache_register_net(net, nn->nfs_dns_resolve); + if (err) + goto err_reg; + return 0; + +err_reg: + cache_destroy_net(nn->nfs_dns_resolve, net); + return err; +} + +void nfs_dns_resolver_cache_destroy(struct net *net) +{ + struct nfs_net *nn = net_generic(net, nfs_net_id); + + nfs_cache_unregister_net(net, nn->nfs_dns_resolve); + cache_destroy_net(nn->nfs_dns_resolve, net); +} + +static int nfs4_dns_net_init(struct net *net) +{ + return nfs_dns_resolver_cache_init(net); +} + +static void nfs4_dns_net_exit(struct net *net) +{ + nfs_dns_resolver_cache_destroy(net); +} + +static struct pernet_operations nfs4_dns_resolver_ops = { + .init = nfs4_dns_net_init, + .exit = nfs4_dns_net_exit, +}; + +static int rpc_pipefs_event(struct notifier_block *nb, unsigned long event, + void *ptr) +{ + struct super_block *sb = ptr; + struct net *net = sb->s_fs_info; + struct nfs_net *nn = net_generic(net, nfs_net_id); + struct cache_detail *cd = nn->nfs_dns_resolve; + int ret = 0; + + if (cd == NULL) + return 0; + + if (!try_module_get(THIS_MODULE)) + return 0; + + switch (event) { + case RPC_PIPEFS_MOUNT: + ret = nfs_cache_register_sb(sb, cd); + break; + case RPC_PIPEFS_UMOUNT: + nfs_cache_unregister_sb(sb, cd); + break; + default: + ret = -ENOTSUPP; + break; + } + module_put(THIS_MODULE); + return ret; +} + +static struct notifier_block nfs_dns_resolver_block = { + .notifier_call = rpc_pipefs_event, +}; + +int nfs_dns_resolver_init(void) +{ + int err; + + err = register_pernet_subsys(&nfs4_dns_resolver_ops); + if (err < 0) + goto out; + err = rpc_pipefs_notifier_register(&nfs_dns_resolver_block); + if (err < 0) + goto out1; + return 0; +out1: + unregister_pernet_subsys(&nfs4_dns_resolver_ops); +out: + return err; +} + +void nfs_dns_resolver_destroy(void) +{ + rpc_pipefs_notifier_unregister(&nfs_dns_resolver_block); + unregister_pernet_subsys(&nfs4_dns_resolver_ops); +} +#endif diff --git a/fs/nfs/dns_resolve.h b/fs/nfs/dns_resolve.h new file mode 100644 index 000000000..fe3b172c4 --- /dev/null +++ b/fs/nfs/dns_resolve.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Resolve DNS hostnames into valid ip addresses + */ +#ifndef __LINUX_FS_NFS_DNS_RESOLVE_H +#define __LINUX_FS_NFS_DNS_RESOLVE_H + +#define NFS_DNS_HOSTNAME_MAXLEN (128) + + +#ifdef CONFIG_NFS_USE_KERNEL_DNS +static inline int nfs_dns_resolver_init(void) +{ + return 0; +} + +static inline void nfs_dns_resolver_destroy(void) +{} + +static inline int nfs_dns_resolver_cache_init(struct net *net) +{ + return 0; +} + +static inline void nfs_dns_resolver_cache_destroy(struct net *net) +{} +#else +extern int nfs_dns_resolver_init(void); +extern void nfs_dns_resolver_destroy(void); +extern int nfs_dns_resolver_cache_init(struct net *net); +extern void nfs_dns_resolver_cache_destroy(struct net *net); +#endif + +extern ssize_t nfs_dns_resolve_name(struct net *net, char *name, + size_t namelen, struct sockaddr_storage *sa, size_t salen); + +#endif diff --git a/fs/nfs/export.c b/fs/nfs/export.c new file mode 100644 index 000000000..8fb08bce0 --- /dev/null +++ b/fs/nfs/export.c @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2015, Primary Data, Inc. All rights reserved. + * + * Tao Peng + */ +#include +#include +#include +#include + +#include "internal.h" +#include "nfstrace.h" + +#define NFSDBG_FACILITY NFSDBG_VFS + +enum { + FILEID_HIGH_OFF = 0, /* inode fileid high */ + FILEID_LOW_OFF, /* inode fileid low */ + FILE_I_TYPE_OFF, /* inode type */ + EMBED_FH_OFF /* embeded server fh */ +}; + + +static struct nfs_fh *nfs_exp_embedfh(__u32 *p) +{ + return (struct nfs_fh *)(p + EMBED_FH_OFF); +} + +/* + * Let's break subtree checking for now... otherwise we'll have to embed parent fh + * but there might not be enough space. + */ +static int +nfs_encode_fh(struct inode *inode, __u32 *p, int *max_len, struct inode *parent) +{ + struct nfs_fh *server_fh = NFS_FH(inode); + struct nfs_fh *clnt_fh = nfs_exp_embedfh(p); + size_t fh_size = offsetof(struct nfs_fh, data) + server_fh->size; + int len = EMBED_FH_OFF + XDR_QUADLEN(fh_size); + + dprintk("%s: max fh len %d inode %p parent %p", + __func__, *max_len, inode, parent); + + if (*max_len < len) { + dprintk("%s: fh len %d too small, required %d\n", + __func__, *max_len, len); + *max_len = len; + return FILEID_INVALID; + } + + p[FILEID_HIGH_OFF] = inode->i_ino >> 32; + p[FILEID_LOW_OFF] = inode->i_ino; + p[FILE_I_TYPE_OFF] = inode->i_mode & S_IFMT; + p[len - 1] = 0; /* Padding */ + nfs_copy_fh(clnt_fh, server_fh); + *max_len = len; + dprintk("%s: result fh fileid %llu mode %u size %d\n", + __func__, inode->i_ino, inode->i_mode, *max_len); + return *max_len; +} + +static struct dentry * +nfs_fh_to_dentry(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + struct nfs_fattr *fattr = NULL; + struct nfs_fh *server_fh = nfs_exp_embedfh(fid->raw); + size_t fh_size = offsetof(struct nfs_fh, data); + const struct nfs_rpc_ops *rpc_ops; + struct dentry *dentry; + struct inode *inode; + int len = EMBED_FH_OFF; + u32 *p = fid->raw; + int ret; + + /* Initial check of bounds */ + if (fh_len < len + XDR_QUADLEN(fh_size) || + fh_len > XDR_QUADLEN(NFS_MAXFHSIZE)) + return NULL; + /* Calculate embedded filehandle size */ + fh_size += server_fh->size; + len += XDR_QUADLEN(fh_size); + /* NULL translates to ESTALE */ + if (fh_len < len || fh_type != len) + return NULL; + + fattr = nfs_alloc_fattr_with_label(NFS_SB(sb)); + if (fattr == NULL) { + dentry = ERR_PTR(-ENOMEM); + goto out; + } + + fattr->fileid = ((u64)p[FILEID_HIGH_OFF] << 32) + p[FILEID_LOW_OFF]; + fattr->mode = p[FILE_I_TYPE_OFF]; + fattr->valid |= NFS_ATTR_FATTR_FILEID | NFS_ATTR_FATTR_TYPE; + + dprintk("%s: fileid %llu mode %d\n", __func__, fattr->fileid, fattr->mode); + + inode = nfs_ilookup(sb, fattr, server_fh); + if (inode) + goto out_found; + + rpc_ops = NFS_SB(sb)->nfs_client->rpc_ops; + ret = rpc_ops->getattr(NFS_SB(sb), server_fh, fattr, NULL); + if (ret) { + dprintk("%s: getattr failed %d\n", __func__, ret); + trace_nfs_fh_to_dentry(sb, server_fh, fattr->fileid, ret); + dentry = ERR_PTR(ret); + goto out_free_fattr; + } + + inode = nfs_fhget(sb, server_fh, fattr); + +out_found: + dentry = d_obtain_alias(inode); +out_free_fattr: + nfs_free_fattr(fattr); +out: + return dentry; +} + +static struct dentry * +nfs_get_parent(struct dentry *dentry) +{ + int ret; + struct inode *inode = d_inode(dentry), *pinode; + struct super_block *sb = inode->i_sb; + struct nfs_server *server = NFS_SB(sb); + struct nfs_fattr *fattr = NULL; + struct dentry *parent; + struct nfs_rpc_ops const *ops = server->nfs_client->rpc_ops; + struct nfs_fh fh; + + if (!ops->lookupp) + return ERR_PTR(-EACCES); + + fattr = nfs_alloc_fattr_with_label(server); + if (fattr == NULL) + return ERR_PTR(-ENOMEM); + + ret = ops->lookupp(inode, &fh, fattr); + if (ret) { + parent = ERR_PTR(ret); + goto out; + } + + pinode = nfs_fhget(sb, &fh, fattr); + parent = d_obtain_alias(pinode); +out: + nfs_free_fattr(fattr); + return parent; +} + +const struct export_operations nfs_export_ops = { + .encode_fh = nfs_encode_fh, + .fh_to_dentry = nfs_fh_to_dentry, + .get_parent = nfs_get_parent, + .flags = EXPORT_OP_NOWCC | + EXPORT_OP_NOSUBTREECHK | + EXPORT_OP_CLOSE_BEFORE_UNLINK | + EXPORT_OP_REMOTE_FS | + EXPORT_OP_NOATOMIC_ATTR | + EXPORT_OP_FLUSH_ON_CLOSE | + EXPORT_OP_NOLOCKS, +}; diff --git a/fs/nfs/file.c b/fs/nfs/file.c new file mode 100644 index 000000000..a0d8f1c1c --- /dev/null +++ b/fs/nfs/file.c @@ -0,0 +1,982 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/nfs/file.c + * + * Copyright (C) 1992 Rick Sladkey + * + * Changes Copyright (C) 1994 by Florian La Roche + * - Do not copy data too often around in the kernel. + * - In nfs_file_read the return value of kmalloc wasn't checked. + * - Put in a better version of read look-ahead buffering. Original idea + * and implementation by Wai S Kok elekokws@ee.nus.sg. + * + * Expire cache on write to a file by Wai S Kok (Oct 1994). + * + * Total rewrite of read side for new NFS buffer cache.. Linus. + * + * nfs regular file handling functions + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "delegation.h" +#include "internal.h" +#include "iostat.h" +#include "fscache.h" +#include "pnfs.h" + +#include "nfstrace.h" + +#define NFSDBG_FACILITY NFSDBG_FILE + +static const struct vm_operations_struct nfs_file_vm_ops; + +int nfs_check_flags(int flags) +{ + if ((flags & (O_APPEND | O_DIRECT)) == (O_APPEND | O_DIRECT)) + return -EINVAL; + + return 0; +} +EXPORT_SYMBOL_GPL(nfs_check_flags); + +/* + * Open file + */ +static int +nfs_file_open(struct inode *inode, struct file *filp) +{ + int res; + + dprintk("NFS: open file(%pD2)\n", filp); + + nfs_inc_stats(inode, NFSIOS_VFSOPEN); + res = nfs_check_flags(filp->f_flags); + if (res) + return res; + + res = nfs_open(inode, filp); + if (res == 0) { + filp->f_mode |= FMODE_CAN_ODIRECT; + /* flag NOWAIT on read-only files only */ + if (!(filp->f_mode & FMODE_WRITE)) + filp->f_mode |= FMODE_NOWAIT; + } + return res; +} + +int +nfs_file_release(struct inode *inode, struct file *filp) +{ + dprintk("NFS: release(%pD2)\n", filp); + + nfs_inc_stats(inode, NFSIOS_VFSRELEASE); + nfs_file_clear_open_context(filp); + nfs_fscache_release_file(inode, filp); + return 0; +} +EXPORT_SYMBOL_GPL(nfs_file_release); + +/** + * nfs_revalidate_file_size - Revalidate the file size + * @inode: pointer to inode struct + * @filp: pointer to struct file + * + * Revalidates the file length. This is basically a wrapper around + * nfs_revalidate_inode() that takes into account the fact that we may + * have cached writes (in which case we don't care about the server's + * idea of what the file length is), or O_DIRECT (in which case we + * shouldn't trust the cache). + */ +static int nfs_revalidate_file_size(struct inode *inode, struct file *filp) +{ + struct nfs_server *server = NFS_SERVER(inode); + + if (filp->f_flags & O_DIRECT) + goto force_reval; + if (nfs_check_cache_invalid(inode, NFS_INO_INVALID_SIZE)) + goto force_reval; + return 0; +force_reval: + return __nfs_revalidate_inode(server, inode); +} + +loff_t nfs_file_llseek(struct file *filp, loff_t offset, int whence) +{ + dprintk("NFS: llseek file(%pD2, %lld, %d)\n", + filp, offset, whence); + + /* + * whence == SEEK_END || SEEK_DATA || SEEK_HOLE => we must revalidate + * the cached file length + */ + if (whence != SEEK_SET && whence != SEEK_CUR) { + struct inode *inode = filp->f_mapping->host; + + int retval = nfs_revalidate_file_size(inode, filp); + if (retval < 0) + return (loff_t)retval; + } + + return generic_file_llseek(filp, offset, whence); +} +EXPORT_SYMBOL_GPL(nfs_file_llseek); + +/* + * Flush all dirty pages, and check for write errors. + */ +static int +nfs_file_flush(struct file *file, fl_owner_t id) +{ + struct inode *inode = file_inode(file); + errseq_t since; + + dprintk("NFS: flush(%pD2)\n", file); + + nfs_inc_stats(inode, NFSIOS_VFSFLUSH); + if ((file->f_mode & FMODE_WRITE) == 0) + return 0; + + /* Flush writes to the server and return any errors */ + since = filemap_sample_wb_err(file->f_mapping); + nfs_wb_all(inode); + return filemap_check_wb_err(file->f_mapping, since); +} + +ssize_t +nfs_file_read(struct kiocb *iocb, struct iov_iter *to) +{ + struct inode *inode = file_inode(iocb->ki_filp); + ssize_t result; + + trace_nfs_file_read(iocb, to); + + if (iocb->ki_flags & IOCB_DIRECT) + return nfs_file_direct_read(iocb, to, false); + + /* NOWAIT only supported on direct reads */ + if (iocb->ki_flags & IOCB_NOWAIT) + return -EAGAIN; + + dprintk("NFS: read(%pD2, %zu@%lu)\n", + iocb->ki_filp, + iov_iter_count(to), (unsigned long) iocb->ki_pos); + + result = nfs_start_io_read(inode); + if (result) + return result; + + result = nfs_revalidate_mapping(inode, iocb->ki_filp->f_mapping); + if (!result) { + result = generic_file_read_iter(iocb, to); + if (result > 0) + nfs_add_stats(inode, NFSIOS_NORMALREADBYTES, result); + } + nfs_end_io_read(inode); + return result; +} +EXPORT_SYMBOL_GPL(nfs_file_read); + +ssize_t +nfs_file_splice_read(struct file *in, loff_t *ppos, struct pipe_inode_info *pipe, + size_t len, unsigned int flags) +{ + struct inode *inode = file_inode(in); + ssize_t result; + + dprintk("NFS: splice_read(%pD2, %zu@%llu)\n", in, len, *ppos); + + result = nfs_start_io_read(inode); + if (result) + return result; + + result = nfs_revalidate_mapping(inode, in->f_mapping); + if (!result) { + result = filemap_splice_read(in, ppos, pipe, len, flags); + if (result > 0) + nfs_add_stats(inode, NFSIOS_NORMALREADBYTES, result); + } + nfs_end_io_read(inode); + return result; +} +EXPORT_SYMBOL_GPL(nfs_file_splice_read); + +int +nfs_file_mmap_prepare(struct vm_area_desc *desc) +{ + struct file *file = desc->file; + struct inode *inode = file_inode(file); + int status; + + dprintk("NFS: mmap(%pD2)\n", file); + + /* Note: generic_file_mmap_prepare() returns ENOSYS on nommu systems + * so we call that before revalidating the mapping + */ + status = generic_file_mmap_prepare(desc); + if (!status) { + desc->vm_ops = &nfs_file_vm_ops; + status = nfs_revalidate_mapping(inode, file->f_mapping); + } + return status; +} +EXPORT_SYMBOL_GPL(nfs_file_mmap_prepare); + +/* + * Flush any dirty pages for this process, and check for write errors. + * The return status from this call provides a reliable indication of + * whether any write errors occurred for this process. + */ +static int +nfs_file_fsync_commit(struct file *file, int datasync) +{ + struct inode *inode = file_inode(file); + int ret, ret2; + + dprintk("NFS: fsync file(%pD2) datasync %d\n", file, datasync); + + nfs_inc_stats(inode, NFSIOS_VFSFSYNC); + ret = nfs_commit_inode(inode, FLUSH_SYNC); + ret2 = file_check_and_advance_wb_err(file); + if (ret2 < 0) + return ret2; + return ret; +} + +int +nfs_file_fsync(struct file *file, loff_t start, loff_t end, int datasync) +{ + struct inode *inode = file_inode(file); + struct nfs_inode *nfsi = NFS_I(inode); + long save_nredirtied = atomic_long_read(&nfsi->redirtied_pages); + long nredirtied; + int ret; + + trace_nfs_fsync_enter(inode); + + for (;;) { + ret = file_write_and_wait_range(file, start, end); + if (ret != 0) + break; + ret = nfs_file_fsync_commit(file, datasync); + if (ret != 0) + break; + ret = pnfs_sync_inode(inode, !!datasync); + if (ret != 0) + break; + nredirtied = atomic_long_read(&nfsi->redirtied_pages); + if (nredirtied == save_nredirtied) + break; + save_nredirtied = nredirtied; + } + + trace_nfs_fsync_exit(inode, ret); + return ret; +} +EXPORT_SYMBOL_GPL(nfs_file_fsync); + +void nfs_truncate_last_folio(struct address_space *mapping, loff_t from, + loff_t to) +{ + struct folio *folio; + + if (from >= to) + return; + + folio = filemap_lock_folio(mapping, from >> PAGE_SHIFT); + if (IS_ERR(folio)) + return; + + if (folio_mkclean(folio)) + folio_mark_dirty(folio); + + if (folio_test_uptodate(folio)) { + loff_t fpos = folio_pos(folio); + size_t offset = from - fpos; + size_t end = folio_size(folio); + + if (to - fpos < end) + end = to - fpos; + folio_zero_segment(folio, offset, end); + trace_nfs_size_truncate_folio(mapping->host, to); + } + + folio_unlock(folio); + folio_put(folio); +} +EXPORT_SYMBOL_GPL(nfs_truncate_last_folio); + +/* + * Decide whether a read/modify/write cycle may be more efficient + * then a modify/write/read cycle when writing to a page in the + * page cache. + * + * Some pNFS layout drivers can only read/write at a certain block + * granularity like all block devices and therefore we must perform + * read/modify/write whenever a page hasn't read yet and the data + * to be written there is not aligned to a block boundary and/or + * smaller than the block size. + * + * The modify/write/read cycle may occur if a page is read before + * being completely filled by the writer. In this situation, the + * page must be completely written to stable storage on the server + * before it can be refilled by reading in the page from the server. + * This can lead to expensive, small, FILE_SYNC mode writes being + * done. + * + * It may be more efficient to read the page first if the file is + * open for reading in addition to writing, the page is not marked + * as Uptodate, it is not dirty or waiting to be committed, + * indicating that it was previously allocated and then modified, + * that there were valid bytes of data in that range of the file, + * and that the new data won't completely replace the old data in + * that range of the file. + */ +static bool nfs_folio_is_full_write(struct folio *folio, loff_t pos, + unsigned int len) +{ + unsigned int pglen = nfs_folio_length(folio); + unsigned int offset = offset_in_folio(folio, pos); + unsigned int end = offset + len; + + return !pglen || (end >= pglen && !offset); +} + +static bool nfs_want_read_modify_write(struct file *file, struct folio *folio, + loff_t pos, unsigned int len) +{ + /* + * Up-to-date pages, those with ongoing or full-page write + * don't need read/modify/write + */ + if (folio_test_uptodate(folio) || folio_test_private(folio) || + nfs_folio_is_full_write(folio, pos, len)) + return false; + + if (pnfs_ld_read_whole_page(file_inode(file))) + return true; + if (folio_test_dropbehind(folio)) + return false; + /* Open for reading too? */ + if (file->f_mode & FMODE_READ) + return true; + return false; +} + +/* + * This does the "real" work of the write. We must allocate and lock the + * page to be sent back to the generic routine, which then copies the + * data from user space. + * + * If the writer ends up delaying the write, the writer needs to + * increment the page use counts until he is done with the page. + */ +static int nfs_write_begin(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, struct folio **foliop, + void **fsdata) +{ + struct folio *folio; + struct file *file = iocb->ki_filp; + int once_thru = 0; + int ret; + + trace_nfs_write_begin(file_inode(file), pos, len); + + dfprintk(PAGECACHE, "NFS: write_begin(%pD2(%llu), %u@%lld)\n", + file, mapping->host->i_ino, len, (long long) pos); + nfs_truncate_last_folio(mapping, i_size_read(mapping->host), pos); + +start: + folio = write_begin_get_folio(iocb, mapping, pos >> PAGE_SHIFT, len); + if (IS_ERR(folio)) { + ret = PTR_ERR(folio); + goto out; + } + *foliop = folio; + + ret = nfs_flush_incompatible(file, folio); + if (ret) { + folio_unlock(folio); + folio_put(folio); + } else if (!once_thru && + nfs_want_read_modify_write(file, folio, pos, len)) { + once_thru = 1; + folio_clear_dropbehind(folio); + ret = nfs_read_folio(file, folio); + folio_put(folio); + if (!ret) + goto start; + } +out: + trace_nfs_write_begin_done(file_inode(file), pos, len, ret); + return ret; +} + +static int nfs_write_end(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct folio *folio, void *fsdata) +{ + struct file *file = iocb->ki_filp; + struct nfs_open_context *ctx = nfs_file_open_context(file); + unsigned offset = offset_in_folio(folio, pos); + int status; + + trace_nfs_write_end(file_inode(file), pos, len); + dfprintk(PAGECACHE, "NFS: write_end(%pD2(%llu), %u@%lld)\n", + file, mapping->host->i_ino, len, (long long) pos); + + /* + * Zero any uninitialised parts of the page, and then mark the page + * as up to date if it turns out that we're extending the file. + */ + if (!folio_test_uptodate(folio)) { + size_t fsize = folio_size(folio); + unsigned pglen = nfs_folio_length(folio); + unsigned end = offset + copied; + + if (pglen == 0) { + folio_zero_segments(folio, 0, offset, end, fsize); + folio_mark_uptodate(folio); + } else if (end >= pglen) { + folio_zero_segment(folio, end, fsize); + if (offset == 0) + folio_mark_uptodate(folio); + } else + folio_zero_segment(folio, pglen, fsize); + } + + status = nfs_update_folio(file, folio, offset, copied); + + folio_unlock(folio); + folio_put(folio); + + if (status < 0) { + trace_nfs_write_end_done(file_inode(file), pos, len, status); + return status; + } + NFS_I(mapping->host)->write_io += copied; + + if (nfs_ctx_key_to_expire(ctx, mapping->host)) + nfs_wb_all(mapping->host); + + trace_nfs_write_end_done(file_inode(file), pos, len, copied); + return copied; +} + +/* + * Partially or wholly invalidate a page + * - Release the private state associated with a page if undergoing complete + * page invalidation + * - Called if either PG_private or PG_fscache is set on the page + * - Caller holds page lock + */ +static void nfs_invalidate_folio(struct folio *folio, size_t offset, + size_t length) +{ + struct inode *inode = folio->mapping->host; + dfprintk(PAGECACHE, "NFS: invalidate_folio(%lu, %zu, %zu)\n", + folio->index, offset, length); + + /* Cancel any unstarted writes on this page */ + if (offset != 0 || length < folio_size(folio)) + nfs_wb_folio(inode, folio); + else + nfs_wb_folio_cancel(inode, folio); + folio_wait_private_2(folio); /* [DEPRECATED] */ + trace_nfs_invalidate_folio(inode, folio_pos(folio) + offset, length); +} + +/* + * Attempt to release the private state associated with a folio + * - Called if either private or fscache flags are set on the folio + * - Caller holds folio lock + * - Return true (may release folio) or false (may not) + */ +static bool nfs_release_folio(struct folio *folio, gfp_t gfp) +{ + dfprintk(PAGECACHE, "NFS: release_folio(%p)\n", folio); + + /* If the private flag is set, then the folio is not freeable */ + if (folio_test_private(folio)) { + if ((current_gfp_context(gfp) & GFP_KERNEL) != GFP_KERNEL || + current_is_kswapd() || current_is_kcompactd()) + return false; + if (nfs_wb_folio_reclaim(folio->mapping->host, folio) < 0 || + folio_test_private(folio)) + return false; + } + return nfs_fscache_release_folio(folio, gfp); +} + +static void nfs_check_dirty_writeback(struct folio *folio, + bool *dirty, bool *writeback) +{ + struct nfs_inode *nfsi; + struct address_space *mapping = folio->mapping; + + /* + * Check if an unstable folio is currently being committed and + * if so, have the VM treat it as if the folio is under writeback + * so it will not block due to folios that will shortly be freeable. + */ + nfsi = NFS_I(mapping->host); + if (atomic_read(&nfsi->commit_info.rpcs_out)) { + *writeback = true; + return; + } + + /* + * If the private flag is set, then the folio is not freeable + * and as the inode is not being committed, it's not going to + * be cleaned in the near future so treat it as dirty + */ + if (folio_test_private(folio)) + *dirty = true; +} + +/* + * Attempt to clear the private state associated with a page when an error + * occurs that requires the cached contents of an inode to be written back or + * destroyed + * - Called if either PG_private or fscache is set on the page + * - Caller holds page lock + * - Return 0 if successful, -error otherwise + */ +static int nfs_launder_folio(struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + int ret; + + dfprintk(PAGECACHE, "NFS: launder_folio(%llu, %llu)\n", + inode->i_ino, folio_pos(folio)); + + folio_wait_private_2(folio); /* [DEPRECATED] */ + ret = nfs_wb_folio(inode, folio); + trace_nfs_launder_folio_done(inode, folio_pos(folio), + folio_size(folio), ret); + return ret; +} + +static int nfs_swap_activate(struct swap_info_struct *sis, struct file *file, + sector_t *span) +{ + unsigned long blocks; + long long isize; + int ret; + struct inode *inode = file_inode(file); + struct rpc_clnt *clnt = NFS_CLIENT(inode); + struct nfs_client *cl = NFS_SERVER(inode)->nfs_client; + + spin_lock(&inode->i_lock); + blocks = inode->i_blocks; + isize = inode->i_size; + spin_unlock(&inode->i_lock); + if (blocks*512 < isize) { + pr_warn("swap activate: swapfile has holes\n"); + return -EINVAL; + } + + ret = rpc_clnt_swap_activate(clnt); + if (ret) + return ret; + ret = add_swap_extent(sis, 0, sis->max, 0); + if (ret < 0) { + rpc_clnt_swap_deactivate(clnt); + return ret; + } + + *span = sis->pages; + + if (cl->rpc_ops->enable_swap) + cl->rpc_ops->enable_swap(inode); + + sis->flags |= SWP_FS_OPS; + return ret; +} + +static void nfs_swap_deactivate(struct file *file) +{ + struct inode *inode = file_inode(file); + struct rpc_clnt *clnt = NFS_CLIENT(inode); + struct nfs_client *cl = NFS_SERVER(inode)->nfs_client; + + rpc_clnt_swap_deactivate(clnt); + if (cl->rpc_ops->disable_swap) + cl->rpc_ops->disable_swap(file_inode(file)); +} + +const struct address_space_operations nfs_file_aops = { + .read_folio = nfs_read_folio, + .readahead = nfs_readahead, + .dirty_folio = filemap_dirty_folio, + .writepages = nfs_writepages, + .write_begin = nfs_write_begin, + .write_end = nfs_write_end, + .invalidate_folio = nfs_invalidate_folio, + .release_folio = nfs_release_folio, + .migrate_folio = nfs_migrate_folio, + .launder_folio = nfs_launder_folio, + .is_dirty_writeback = nfs_check_dirty_writeback, + .error_remove_folio = generic_error_remove_folio, + .swap_activate = nfs_swap_activate, + .swap_deactivate = nfs_swap_deactivate, + .swap_rw = nfs_swap_rw, +}; + +/* + * Notification that a PTE pointing to an NFS page is about to be made + * writable, implying that someone is about to modify the page through a + * shared-writable mapping + */ +static vm_fault_t nfs_vm_page_mkwrite(struct vm_fault *vmf) +{ + struct file *filp = vmf->vma->vm_file; + struct inode *inode = file_inode(filp); + unsigned pagelen; + vm_fault_t ret = VM_FAULT_NOPAGE; + struct address_space *mapping; + struct folio *folio = page_folio(vmf->page); + + dfprintk(PAGECACHE, "NFS: vm_page_mkwrite(%pD2(%llu), offset %lld)\n", + filp, filp->f_mapping->host->i_ino, + (long long)folio_pos(folio)); + + sb_start_pagefault(inode->i_sb); + + /* make sure the cache has finished storing the page */ + if (folio_test_private_2(folio) && /* [DEPRECATED] */ + folio_wait_private_2_killable(folio) < 0) { + ret = VM_FAULT_RETRY; + goto out; + } + + wait_on_bit_action(&NFS_I(inode)->flags, NFS_INO_INVALIDATING, + nfs_wait_bit_killable, + TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); + + folio_lock(folio); + mapping = folio->mapping; + if (mapping != inode->i_mapping) + goto out_unlock; + + folio_wait_writeback(folio); + + pagelen = nfs_folio_length(folio); + if (pagelen == 0) + goto out_unlock; + + ret = VM_FAULT_LOCKED; + if (nfs_flush_incompatible(filp, folio) == 0 && + nfs_update_folio(filp, folio, 0, pagelen) == 0) + goto out; + + ret = VM_FAULT_SIGBUS; +out_unlock: + folio_unlock(folio); +out: + sb_end_pagefault(inode->i_sb); + return ret; +} + +static const struct vm_operations_struct nfs_file_vm_ops = { + .fault = filemap_fault, + .map_pages = filemap_map_pages, + .page_mkwrite = nfs_vm_page_mkwrite, +}; + +ssize_t nfs_file_write(struct kiocb *iocb, struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + unsigned int mntflags = NFS_SERVER(inode)->flags; + ssize_t result, written; + errseq_t since; + int error; + + trace_nfs_file_write(iocb, from); + + /* + * FMODE_NOWAIT is not set for writable files + */ + if (WARN_ON_ONCE(iocb->ki_flags & IOCB_NOWAIT)) + return -EAGAIN; + + result = nfs_key_timeout_notify(file, inode); + if (result) + return result; + + if (iocb->ki_flags & IOCB_DIRECT) + return nfs_file_direct_write(iocb, from, false); + + dprintk("NFS: write(%pD2, %zu@%Ld)\n", + file, iov_iter_count(from), (long long) iocb->ki_pos); + + if (IS_SWAPFILE(inode)) + goto out_swapfile; + /* + * O_APPEND implies that we must revalidate the file length. + */ + if (iocb->ki_flags & IOCB_APPEND || iocb->ki_pos > i_size_read(inode)) { + result = nfs_revalidate_file_size(inode, file); + if (result) + return result; + } + + nfs_clear_invalid_mapping(file->f_mapping); + + since = filemap_sample_wb_err(file->f_mapping); + error = nfs_start_io_write(inode); + if (error) + return error; + result = generic_write_checks(iocb, from); + if (result > 0) + result = generic_perform_write(iocb, from); + nfs_end_io_write(inode); + if (result <= 0) + goto out; + + written = result; + nfs_add_stats(inode, NFSIOS_NORMALWRITTENBYTES, written); + + if (mntflags & NFS_MOUNT_WRITE_EAGER) { + result = filemap_fdatawrite_range(file->f_mapping, + iocb->ki_pos - written, + iocb->ki_pos - 1); + if (result < 0) + goto out; + } + if (mntflags & NFS_MOUNT_WRITE_WAIT) { + filemap_fdatawait_range(file->f_mapping, + iocb->ki_pos - written, + iocb->ki_pos - 1); + } + result = generic_write_sync(iocb, written); + if (result < 0) + return result; + +out: + /* Return error values */ + error = filemap_check_wb_err(file->f_mapping, since); + switch (error) { + default: + break; + case -EDQUOT: + case -EFBIG: + case -ENOSPC: + nfs_wb_all(inode); + error = file_check_and_advance_wb_err(file); + if (error < 0) + result = error; + } + return result; + +out_swapfile: + printk(KERN_INFO "NFS: attempt to write to active swap file!\n"); + return -ETXTBSY; +} +EXPORT_SYMBOL_GPL(nfs_file_write); + +static int +do_getlk(struct file *filp, int cmd, struct file_lock *fl, int is_local) +{ + struct inode *inode = filp->f_mapping->host; + int status = 0; + unsigned int saved_type = fl->c.flc_type; + + /* Try local locking first */ + posix_test_lock(filp, fl); + if (fl->c.flc_type != F_UNLCK) { + /* found a conflict */ + goto out; + } + fl->c.flc_type = saved_type; + + if (nfs_have_read_or_write_delegation(inode)) + goto out_noconflict; + + if (is_local) + goto out_noconflict; + + status = NFS_PROTO(inode)->lock(filp, cmd, fl); +out: + return status; +out_noconflict: + fl->c.flc_type = F_UNLCK; + goto out; +} + +static int +do_unlk(struct file *filp, int cmd, struct file_lock *fl, int is_local) +{ + struct inode *inode = filp->f_mapping->host; + struct nfs_lock_context *l_ctx; + int status; + + /* + * Flush all pending writes before doing anything + * with locks.. + */ + nfs_wb_all(inode); + + l_ctx = nfs_get_lock_context(nfs_file_open_context(filp)); + if (!IS_ERR(l_ctx)) { + status = nfs_iocounter_wait(l_ctx); + nfs_put_lock_context(l_ctx); + /* NOTE: special case + * If we're signalled while cleaning up locks on process exit, we + * still need to complete the unlock. + */ + if (status < 0 && !(fl->c.flc_flags & FL_CLOSE)) + return status; + } + + /* + * Use local locking if mounted with "-onolock" or with appropriate + * "-olocal_lock=" + */ + if (!is_local) + status = NFS_PROTO(inode)->lock(filp, cmd, fl); + else + status = locks_lock_file_wait(filp, fl); + return status; +} + +static int +do_setlk(struct file *filp, int cmd, struct file_lock *fl, int is_local) +{ + struct inode *inode = filp->f_mapping->host; + int status; + + /* + * Flush all pending writes before doing anything + * with locks.. + */ + status = nfs_sync_mapping(filp->f_mapping); + if (status != 0) + goto out; + + /* + * Use local locking if mounted with "-onolock" or with appropriate + * "-olocal_lock=" + */ + if (!is_local) + status = NFS_PROTO(inode)->lock(filp, cmd, fl); + else + status = locks_lock_file_wait(filp, fl); + if (status < 0) + goto out; + + /* + * Invalidate cache to prevent missing any changes. If + * the file is mapped, clear the page cache as well so + * those mappings will be loaded. + * + * This makes locking act as a cache coherency point. + */ + nfs_sync_mapping(filp->f_mapping); + if (!nfs_have_read_or_write_delegation(inode)) { + nfs_zap_caches(inode); + if (mapping_mapped(filp->f_mapping)) + nfs_revalidate_mapping(inode, filp->f_mapping); + } +out: + return status; +} + +/* + * Lock a (portion of) a file + */ +int nfs_lock(struct file *filp, int cmd, struct file_lock *fl) +{ + struct inode *inode = filp->f_mapping->host; + int ret = -ENOLCK; + int is_local = 0; + + dprintk("NFS: lock(%pD2, t=%x, fl=%x, r=%lld:%lld)\n", + filp, fl->c.flc_type, fl->c.flc_flags, + (long long)fl->fl_start, (long long)fl->fl_end); + + nfs_inc_stats(inode, NFSIOS_VFSLOCK); + + if (fl->c.flc_flags & FL_RECLAIM) + return -ENOGRACE; + + if (NFS_SERVER(inode)->flags & NFS_MOUNT_LOCAL_FCNTL) + is_local = 1; + + if (NFS_PROTO(inode)->lock_check_bounds != NULL) { + ret = NFS_PROTO(inode)->lock_check_bounds(fl); + if (ret < 0) + goto out_err; + } + + if (IS_GETLK(cmd)) + ret = do_getlk(filp, cmd, fl, is_local); + else if (lock_is_unlock(fl)) + ret = do_unlk(filp, cmd, fl, is_local); + else + ret = do_setlk(filp, cmd, fl, is_local); +out_err: + return ret; +} +EXPORT_SYMBOL_GPL(nfs_lock); + +/* + * Lock a (portion of) a file + */ +int nfs_flock(struct file *filp, int cmd, struct file_lock *fl) +{ + struct inode *inode = filp->f_mapping->host; + int is_local = 0; + + dprintk("NFS: flock(%pD2, t=%x, fl=%x)\n", + filp, fl->c.flc_type, fl->c.flc_flags); + + if (!(fl->c.flc_flags & FL_FLOCK)) + return -ENOLCK; + + if (NFS_SERVER(inode)->flags & NFS_MOUNT_LOCAL_FLOCK) + is_local = 1; + + /* We're simulating flock() locks using posix locks on the server */ + if (lock_is_unlock(fl)) + return do_unlk(filp, cmd, fl, is_local); + return do_setlk(filp, cmd, fl, is_local); +} +EXPORT_SYMBOL_GPL(nfs_flock); + +const struct file_operations nfs_file_operations = { + .llseek = nfs_file_llseek, + .read_iter = nfs_file_read, + .write_iter = nfs_file_write, + .mmap_prepare = nfs_file_mmap_prepare, + .open = nfs_file_open, + .flush = nfs_file_flush, + .release = nfs_file_release, + .fsync = nfs_file_fsync, + .lock = nfs_lock, + .flock = nfs_flock, + .splice_read = nfs_file_splice_read, + .splice_write = iter_file_splice_write, + .check_flags = nfs_check_flags, + .fop_flags = FOP_DONTCACHE, +}; +EXPORT_SYMBOL_GPL(nfs_file_operations); diff --git a/fs/nfs/filelayout/Makefile b/fs/nfs/filelayout/Makefile new file mode 100644 index 000000000..de056312d --- /dev/null +++ b/fs/nfs/filelayout/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the pNFS Files Layout Driver kernel module +# +obj-$(CONFIG_PNFS_FILE_LAYOUT) += nfs_layout_nfsv41_files.o +nfs_layout_nfsv41_files-y := filelayout.o filelayoutdev.o diff --git a/fs/nfs/filelayout/filelayout.c b/fs/nfs/filelayout/filelayout.c new file mode 100644 index 000000000..72e20b56f --- /dev/null +++ b/fs/nfs/filelayout/filelayout.c @@ -0,0 +1,1155 @@ +/* + * Module for the pnfs nfs4 file layout driver. + * Defines all I/O and Policy interface operations, plus code + * to register itself with the pNFS client. + * + * Copyright (c) 2002 + * The Regents of the University of Michigan + * All Rights Reserved + * + * Dean Hildebrand + * + * Permission is granted to use, copy, create derivative works, and + * redistribute this software and such derivative works for any purpose, + * so long as the name of the University of Michigan is not used in + * any advertising or publicity pertaining to the use or distribution + * of this software without specific, written prior authorization. If + * the above copyright notice or any other identification of the + * University of Michigan is included in any copy of any portion of + * this software, then the disclaimer below must also be included. + * + * This software is provided as is, without representation or warranty + * of any kind either express or implied, including without limitation + * the implied warranties of merchantability, fitness for a particular + * purpose, or noninfringement. The Regents of the University of + * Michigan shall not be liable for any damages, including special, + * indirect, incidental, or consequential damages, with respect to any + * claim arising out of or in connection with the use of the software, + * even if it has been or is hereafter advised of the possibility of + * such damages. + */ + +#include +#include +#include +#include + +#include + +#include "../nfs4session.h" +#include "../internal.h" +#include "../delegation.h" +#include "filelayout.h" +#include "../nfs4trace.h" + +#define NFSDBG_FACILITY NFSDBG_PNFS_LD + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Dean Hildebrand "); +MODULE_DESCRIPTION("The NFSv4 file layout driver"); + +#define FILELAYOUT_POLL_RETRY_MAX (15*HZ) +static const struct pnfs_commit_ops filelayout_commit_ops; + +static loff_t +filelayout_get_dense_offset(struct nfs4_filelayout_segment *flseg, + loff_t offset) +{ + u32 stripe_width = flseg->stripe_unit * flseg->dsaddr->stripe_count; + u64 stripe_no; + u32 rem; + + offset -= flseg->pattern_offset; + stripe_no = div_u64(offset, stripe_width); + div_u64_rem(offset, flseg->stripe_unit, &rem); + + return stripe_no * flseg->stripe_unit + rem; +} + +/* This function is used by the layout driver to calculate the + * offset of the file on the dserver based on whether the + * layout type is STRIPE_DENSE or STRIPE_SPARSE + */ +static loff_t +filelayout_get_dserver_offset(struct pnfs_layout_segment *lseg, loff_t offset) +{ + struct nfs4_filelayout_segment *flseg = FILELAYOUT_LSEG(lseg); + + switch (flseg->stripe_type) { + case STRIPE_SPARSE: + return offset; + + case STRIPE_DENSE: + return filelayout_get_dense_offset(flseg, offset); + } + + BUG(); +} + +static void filelayout_reset_write(struct nfs_pgio_header *hdr) +{ + struct rpc_task *task = &hdr->task; + + if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { + dprintk("%s Reset task %5u for i/o through MDS " + "(req %s/%llu, %u bytes @ offset %llu)\n", __func__, + hdr->task.tk_pid, + hdr->inode->i_sb->s_id, + (unsigned long long)hdr->inode->i_ino, + hdr->args.count, + (unsigned long long)hdr->args.offset); + + task->tk_status = pnfs_write_done_resend_to_mds(hdr); + } +} + +static void filelayout_reset_read(struct nfs_pgio_header *hdr) +{ + struct rpc_task *task = &hdr->task; + + if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { + dprintk("%s Reset task %5u for i/o through MDS " + "(req %s/%llu, %u bytes @ offset %llu)\n", __func__, + hdr->task.tk_pid, + hdr->inode->i_sb->s_id, + (unsigned long long)hdr->inode->i_ino, + hdr->args.count, + (unsigned long long)hdr->args.offset); + + task->tk_status = pnfs_read_done_resend_to_mds(hdr); + } +} + +static int filelayout_async_handle_error(struct rpc_task *task, + struct nfs4_state *state, + struct nfs_client *clp, + struct pnfs_layout_segment *lseg) +{ + struct pnfs_layout_hdr *lo = lseg->pls_layout; + struct inode *inode = lo->plh_inode; + struct nfs4_deviceid_node *devid = FILELAYOUT_DEVID_NODE(lseg); + struct nfs4_slot_table *tbl = &clp->cl_session->fc_slot_table; + + if (task->tk_status >= 0) + return 0; + + switch (task->tk_status) { + /* DS session errors */ + case -NFS4ERR_BADSESSION: + case -NFS4ERR_BADSLOT: + case -NFS4ERR_BAD_HIGH_SLOT: + case -NFS4ERR_DEADSESSION: + case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION: + case -NFS4ERR_SEQ_FALSE_RETRY: + case -NFS4ERR_SEQ_MISORDERED: + dprintk("%s ERROR %d, Reset session. Exchangeid " + "flags 0x%x\n", __func__, task->tk_status, + clp->cl_exchange_flags); + nfs4_schedule_session_recovery(clp->cl_session, task->tk_status); + break; + case -NFS4ERR_DELAY: + case -NFS4ERR_GRACE: + rpc_delay(task, FILELAYOUT_POLL_RETRY_MAX); + break; + case -NFS4ERR_RETRY_UNCACHED_REP: + break; + /* Invalidate Layout errors */ + case -NFS4ERR_ACCESS: + case -NFS4ERR_PNFS_NO_LAYOUT: + case -ESTALE: /* mapped NFS4ERR_STALE */ + case -EBADHANDLE: /* mapped NFS4ERR_BADHANDLE */ + case -EISDIR: /* mapped NFS4ERR_ISDIR */ + case -NFS4ERR_FHEXPIRED: + case -NFS4ERR_WRONG_TYPE: + dprintk("%s Invalid layout error %d\n", __func__, + task->tk_status); + /* + * Destroy layout so new i/o will get a new layout. + * Layout will not be destroyed until all current lseg + * references are put. Mark layout as invalid to resend failed + * i/o and all i/o waiting on the slot table to the MDS until + * layout is destroyed and a new valid layout is obtained. + */ + pnfs_destroy_layout(NFS_I(inode)); + rpc_wake_up(&tbl->slot_tbl_waitq); + goto reset; + /* RPC connection errors */ + case -ECONNREFUSED: + case -EHOSTDOWN: + case -EHOSTUNREACH: + case -ENETUNREACH: + case -EIO: + case -ETIMEDOUT: + case -EPIPE: + case -EPROTO: + case -ENODEV: + dprintk("%s DS connection error %d\n", __func__, + task->tk_status); + nfs4_mark_deviceid_unavailable(devid); + pnfs_error_mark_layout_for_return(inode, lseg); + pnfs_set_lo_fail(lseg); + rpc_wake_up(&tbl->slot_tbl_waitq); + fallthrough; + default: +reset: + dprintk("%s Retry through MDS. Error %d\n", __func__, + task->tk_status); + return -NFS4ERR_RESET_TO_MDS; + } + task->tk_status = 0; + return -EAGAIN; +} + +/* NFS_PROTO call done callback routines */ + +static int filelayout_read_done_cb(struct rpc_task *task, + struct nfs_pgio_header *hdr) +{ + int err; + + trace_nfs4_pnfs_read(hdr, task->tk_status); + err = filelayout_async_handle_error(task, hdr->args.context->state, + hdr->ds_clp, hdr->lseg); + + switch (err) { + case -NFS4ERR_RESET_TO_MDS: + filelayout_reset_read(hdr); + return task->tk_status; + case -EAGAIN: + rpc_restart_call_prepare(task); + return -EAGAIN; + } + + return 0; +} + +/* + * We reference the rpc_cred of the first WRITE that triggers the need for + * a LAYOUTCOMMIT, and use it to send the layoutcommit compound. + * rfc5661 is not clear about which credential should be used. + */ +static void +filelayout_set_layoutcommit(struct nfs_pgio_header *hdr) +{ + loff_t end_offs = 0; + + if (FILELAYOUT_LSEG(hdr->lseg)->commit_through_mds || + hdr->res.verf->committed == NFS_FILE_SYNC) + return; + if (hdr->res.verf->committed == NFS_DATA_SYNC) + end_offs = hdr->mds_offset + (loff_t)hdr->res.count; + + /* Note: if the write is unstable, don't set end_offs until commit */ + pnfs_set_layoutcommit(hdr->inode, hdr->lseg, end_offs); + dprintk("%s inode %llu pls_end_pos %lu\n", __func__, hdr->inode->i_ino, + (unsigned long) NFS_I(hdr->inode)->layout->plh_lwb); +} + +bool +filelayout_test_devid_unavailable(struct nfs4_deviceid_node *node) +{ + return filelayout_test_devid_invalid(node) || + nfs4_test_deviceid_unavailable(node); +} + +static bool +filelayout_reset_to_mds(struct pnfs_layout_segment *lseg) +{ + struct nfs4_deviceid_node *node = FILELAYOUT_DEVID_NODE(lseg); + + return filelayout_test_devid_unavailable(node); +} + +/* + * Call ops for the async read/write cases + * In the case of dense layouts, the offset needs to be reset to its + * original value. + */ +static void filelayout_read_prepare(struct rpc_task *task, void *data) +{ + struct nfs_pgio_header *hdr = data; + + if (unlikely(test_bit(NFS_CONTEXT_BAD, &hdr->args.context->flags))) { + rpc_exit(task, -EIO); + return; + } + if (filelayout_reset_to_mds(hdr->lseg)) { + dprintk("%s task %u reset io to MDS\n", __func__, task->tk_pid); + filelayout_reset_read(hdr); + rpc_exit(task, 0); + return; + } + hdr->pgio_done_cb = filelayout_read_done_cb; + + if (nfs4_setup_sequence(hdr->ds_clp, + &hdr->args.seq_args, + &hdr->res.seq_res, + task)) + return; + if (nfs4_set_rw_stateid(&hdr->args.stateid, hdr->args.context, + hdr->args.lock_context, FMODE_READ) == -EIO) + rpc_exit(task, -EIO); /* lost lock, terminate I/O */ +} + +static void filelayout_read_call_done(struct rpc_task *task, void *data) +{ + struct nfs_pgio_header *hdr = data; + + if (test_bit(NFS_IOHDR_REDO, &hdr->flags) && + task->tk_status == 0) { + nfs41_sequence_done(task, &hdr->res.seq_res); + return; + } + + /* Note this may cause RPC to be resent */ + hdr->mds_ops->rpc_call_done(task, data); +} + +static void filelayout_read_count_stats(struct rpc_task *task, void *data) +{ + struct nfs_pgio_header *hdr = data; + + rpc_count_iostats(task, NFS_SERVER(hdr->inode)->client->cl_metrics); +} + +static int filelayout_write_done_cb(struct rpc_task *task, + struct nfs_pgio_header *hdr) +{ + int err; + + trace_nfs4_pnfs_write(hdr, task->tk_status); + err = filelayout_async_handle_error(task, hdr->args.context->state, + hdr->ds_clp, hdr->lseg); + + switch (err) { + case -NFS4ERR_RESET_TO_MDS: + filelayout_reset_write(hdr); + return task->tk_status; + case -EAGAIN: + rpc_restart_call_prepare(task); + return -EAGAIN; + } + + filelayout_set_layoutcommit(hdr); + + /* zero out the fattr */ + hdr->fattr.valid = 0; + if (task->tk_status >= 0) + nfs_writeback_update_inode(hdr); + + return 0; +} + +static int filelayout_commit_done_cb(struct rpc_task *task, + struct nfs_commit_data *data) +{ + int err; + + trace_nfs4_pnfs_commit_ds(data, task->tk_status); + err = filelayout_async_handle_error(task, NULL, data->ds_clp, + data->lseg); + + switch (err) { + case -NFS4ERR_RESET_TO_MDS: + pnfs_generic_prepare_to_resend_writes(data); + return -EAGAIN; + case -EAGAIN: + rpc_restart_call_prepare(task); + return -EAGAIN; + } + + pnfs_set_layoutcommit(data->inode, data->lseg, data->lwb); + + return 0; +} + +static void filelayout_write_prepare(struct rpc_task *task, void *data) +{ + struct nfs_pgio_header *hdr = data; + + if (unlikely(test_bit(NFS_CONTEXT_BAD, &hdr->args.context->flags))) { + rpc_exit(task, -EIO); + return; + } + if (filelayout_reset_to_mds(hdr->lseg)) { + dprintk("%s task %u reset io to MDS\n", __func__, task->tk_pid); + filelayout_reset_write(hdr); + rpc_exit(task, 0); + return; + } + if (nfs4_setup_sequence(hdr->ds_clp, + &hdr->args.seq_args, + &hdr->res.seq_res, + task)) + return; + if (nfs4_set_rw_stateid(&hdr->args.stateid, hdr->args.context, + hdr->args.lock_context, FMODE_WRITE) == -EIO) + rpc_exit(task, -EIO); /* lost lock, terminate I/O */ +} + +static void filelayout_write_call_done(struct rpc_task *task, void *data) +{ + struct nfs_pgio_header *hdr = data; + + if (test_bit(NFS_IOHDR_REDO, &hdr->flags) && + task->tk_status == 0) { + nfs41_sequence_done(task, &hdr->res.seq_res); + return; + } + + /* Note this may cause RPC to be resent */ + hdr->mds_ops->rpc_call_done(task, data); +} + +static void filelayout_write_count_stats(struct rpc_task *task, void *data) +{ + struct nfs_pgio_header *hdr = data; + + rpc_count_iostats(task, NFS_SERVER(hdr->inode)->client->cl_metrics); +} + +static void filelayout_commit_prepare(struct rpc_task *task, void *data) +{ + struct nfs_commit_data *wdata = data; + + nfs4_setup_sequence(wdata->ds_clp, + &wdata->args.seq_args, + &wdata->res.seq_res, + task); +} + +static void filelayout_commit_count_stats(struct rpc_task *task, void *data) +{ + struct nfs_commit_data *cdata = data; + + rpc_count_iostats(task, NFS_SERVER(cdata->inode)->client->cl_metrics); +} + +static const struct rpc_call_ops filelayout_read_call_ops = { + .rpc_call_prepare = filelayout_read_prepare, + .rpc_call_done = filelayout_read_call_done, + .rpc_count_stats = filelayout_read_count_stats, + .rpc_release = pnfs_generic_rw_release, +}; + +static const struct rpc_call_ops filelayout_write_call_ops = { + .rpc_call_prepare = filelayout_write_prepare, + .rpc_call_done = filelayout_write_call_done, + .rpc_count_stats = filelayout_write_count_stats, + .rpc_release = pnfs_generic_rw_release, +}; + +static const struct rpc_call_ops filelayout_commit_call_ops = { + .rpc_call_prepare = filelayout_commit_prepare, + .rpc_call_done = pnfs_generic_write_commit_done, + .rpc_count_stats = filelayout_commit_count_stats, + .rpc_release = pnfs_generic_commit_release, +}; + +static enum pnfs_try_status +filelayout_read_pagelist(struct nfs_pgio_header *hdr) +{ + struct pnfs_layout_segment *lseg = hdr->lseg; + struct nfs4_pnfs_ds *ds; + struct rpc_clnt *ds_clnt; + loff_t offset = hdr->args.offset; + u32 j, idx; + struct nfs_fh *fh; + + dprintk("--> %s ino %llu pgbase %u req %zu@%llu\n", + __func__, hdr->inode->i_ino, + hdr->args.pgbase, (size_t)hdr->args.count, offset); + + /* Retrieve the correct rpc_client for the byte range */ + j = nfs4_fl_calc_j_index(lseg, offset); + idx = nfs4_fl_calc_ds_index(lseg, j); + ds = nfs4_fl_prepare_ds(lseg, idx); + if (!ds) + return PNFS_NOT_ATTEMPTED; + + ds_clnt = nfs4_find_or_create_ds_client(ds->ds_clp, hdr->inode); + if (IS_ERR(ds_clnt)) + return PNFS_NOT_ATTEMPTED; + + dprintk("%s USE DS: %s cl_count %d\n", __func__, + ds->ds_remotestr, refcount_read(&ds->ds_clp->cl_count)); + + /* No multipath support. Use first DS */ + refcount_inc(&ds->ds_clp->cl_count); + hdr->ds_clp = ds->ds_clp; + hdr->ds_commit_idx = idx; + fh = nfs4_fl_select_ds_fh(lseg, j); + if (fh) + hdr->args.fh = fh; + + hdr->args.offset = filelayout_get_dserver_offset(lseg, offset); + hdr->mds_offset = offset; + + /* Perform an asynchronous read to ds */ + nfs_initiate_pgio(ds_clnt, hdr, hdr->cred, + NFS_PROTO(hdr->inode), &filelayout_read_call_ops, + 0, RPC_TASK_SOFTCONN, NULL); + return PNFS_ATTEMPTED; +} + +/* Perform async writes. */ +static enum pnfs_try_status +filelayout_write_pagelist(struct nfs_pgio_header *hdr, int sync) +{ + struct pnfs_layout_segment *lseg = hdr->lseg; + struct nfs4_pnfs_ds *ds; + struct rpc_clnt *ds_clnt; + loff_t offset = hdr->args.offset; + u32 j, idx; + struct nfs_fh *fh; + + /* Retrieve the correct rpc_client for the byte range */ + j = nfs4_fl_calc_j_index(lseg, offset); + idx = nfs4_fl_calc_ds_index(lseg, j); + ds = nfs4_fl_prepare_ds(lseg, idx); + if (!ds) + return PNFS_NOT_ATTEMPTED; + + ds_clnt = nfs4_find_or_create_ds_client(ds->ds_clp, hdr->inode); + if (IS_ERR(ds_clnt)) + return PNFS_NOT_ATTEMPTED; + + dprintk("%s ino %llu sync %d req %zu@%llu DS: %s cl_count %d\n", + __func__, hdr->inode->i_ino, sync, (size_t) hdr->args.count, + offset, ds->ds_remotestr, refcount_read(&ds->ds_clp->cl_count)); + + hdr->pgio_done_cb = filelayout_write_done_cb; + refcount_inc(&ds->ds_clp->cl_count); + hdr->ds_clp = ds->ds_clp; + hdr->ds_commit_idx = idx; + fh = nfs4_fl_select_ds_fh(lseg, j); + if (fh) + hdr->args.fh = fh; + hdr->args.offset = filelayout_get_dserver_offset(lseg, offset); + + /* Perform an asynchronous write */ + nfs_initiate_pgio(ds_clnt, hdr, hdr->cred, + NFS_PROTO(hdr->inode), &filelayout_write_call_ops, + sync, RPC_TASK_SOFTCONN, NULL); + return PNFS_ATTEMPTED; +} + +static int +filelayout_check_deviceid(struct pnfs_layout_hdr *lo, + struct nfs4_filelayout_segment *fl, + gfp_t gfp_flags) +{ + struct nfs4_deviceid_node *d; + struct nfs4_file_layout_dsaddr *dsaddr; + int status = -EINVAL; + + /* Is the deviceid already set? If so, we're good. */ + if (fl->dsaddr != NULL) + return 0; + + /* find and reference the deviceid */ + d = nfs4_find_get_deviceid(NFS_SERVER(lo->plh_inode), &fl->deviceid, + lo->plh_lc_cred, gfp_flags); + if (d == NULL) + goto out; + + dsaddr = container_of(d, struct nfs4_file_layout_dsaddr, id_node); + /* Found deviceid is unavailable */ + if (filelayout_test_devid_unavailable(&dsaddr->id_node)) + goto out_put; + + if (fl->first_stripe_index >= dsaddr->stripe_count) { + dprintk("%s Bad first_stripe_index %u\n", + __func__, fl->first_stripe_index); + goto out_put; + } + + if ((fl->stripe_type == STRIPE_SPARSE && + fl->num_fh > 1 && fl->num_fh != dsaddr->ds_num) || + (fl->stripe_type == STRIPE_DENSE && + fl->num_fh != dsaddr->stripe_count)) { + dprintk("%s num_fh %u not valid for given packing\n", + __func__, fl->num_fh); + goto out_put; + } + status = 0; + + /* + * Atomic compare and xchange to ensure we don't scribble + * over a non-NULL pointer. + */ + if (cmpxchg(&fl->dsaddr, NULL, dsaddr) != NULL) + goto out_put; +out: + return status; +out_put: + nfs4_fl_put_deviceid(dsaddr); + goto out; +} + +/* + * filelayout_check_layout() + * + * Make sure layout segment parameters are sane WRT the device. + * At this point no generic layer initialization of the lseg has occurred, + * and nothing has been added to the layout_hdr cache. + * + */ +static int +filelayout_check_layout(struct pnfs_layout_hdr *lo, + struct nfs4_filelayout_segment *fl, + struct nfs4_layoutget_res *lgr, + gfp_t gfp_flags) +{ + int status = -EINVAL; + + dprintk("--> %s\n", __func__); + + if (fl->pattern_offset > lgr->range.offset) { + dprintk("%s pattern_offset %lld too large\n", + __func__, fl->pattern_offset); + goto out; + } + + if (!fl->stripe_unit) { + dprintk("%s Invalid stripe unit (%u)\n", + __func__, fl->stripe_unit); + goto out; + } + + status = 0; +out: + dprintk("--> %s returns %d\n", __func__, status); + return status; +} + +static void _filelayout_free_lseg(struct nfs4_filelayout_segment *fl) +{ + int i; + + if (fl->fh_array) { + for (i = 0; i < fl->num_fh; i++) { + if (!fl->fh_array[i]) + break; + kfree(fl->fh_array[i]); + } + kfree(fl->fh_array); + } + kfree(fl); +} + +static int +filelayout_decode_layout(struct pnfs_layout_hdr *flo, + struct nfs4_filelayout_segment *fl, + struct nfs4_layoutget_res *lgr, + gfp_t gfp_flags) +{ + struct xdr_stream stream; + struct xdr_buf buf; + struct folio *scratch; + __be32 *p; + uint32_t nfl_util; + int i; + + dprintk("%s: set_layout_map Begin\n", __func__); + + scratch = folio_alloc(gfp_flags, 0); + if (!scratch) + return -ENOMEM; + + xdr_init_decode_pages(&stream, &buf, lgr->layoutp->pages, lgr->layoutp->len); + xdr_set_scratch_folio(&stream, scratch); + + /* 20 = ufl_util (4), first_stripe_index (4), pattern_offset (8), + * num_fh (4) */ + p = xdr_inline_decode(&stream, NFS4_DEVICEID4_SIZE + 20); + if (unlikely(!p)) + goto out_err; + + memcpy(&fl->deviceid, p, sizeof(fl->deviceid)); + p += XDR_QUADLEN(NFS4_DEVICEID4_SIZE); + nfs4_print_deviceid(&fl->deviceid); + + nfl_util = be32_to_cpup(p++); + if (nfl_util & NFL4_UFLG_COMMIT_THRU_MDS) + fl->commit_through_mds = 1; + if (nfl_util & NFL4_UFLG_DENSE) + fl->stripe_type = STRIPE_DENSE; + else + fl->stripe_type = STRIPE_SPARSE; + fl->stripe_unit = nfl_util & ~NFL4_UFLG_MASK; + + fl->first_stripe_index = be32_to_cpup(p++); + p = xdr_decode_hyper(p, &fl->pattern_offset); + fl->num_fh = be32_to_cpup(p++); + + dprintk("%s: nfl_util 0x%X num_fh %u fsi %u po %llu\n", + __func__, nfl_util, fl->num_fh, fl->first_stripe_index, + fl->pattern_offset); + + /* Note that a zero value for num_fh is legal for STRIPE_SPARSE. + * Futher checking is done in filelayout_check_layout */ + if (fl->num_fh > + max(NFS4_PNFS_MAX_STRIPE_CNT, NFS4_PNFS_MAX_MULTI_CNT)) + goto out_err; + + if (fl->num_fh > 0) { + fl->fh_array = kzalloc_objs(fl->fh_array[0], fl->num_fh, + gfp_flags); + if (!fl->fh_array) + goto out_err; + } + + for (i = 0; i < fl->num_fh; i++) { + /* Do we want to use a mempool here? */ + fl->fh_array[i] = kmalloc_obj(struct nfs_fh, gfp_flags); + if (!fl->fh_array[i]) + goto out_err; + + p = xdr_inline_decode(&stream, 4); + if (unlikely(!p)) + goto out_err; + fl->fh_array[i]->size = be32_to_cpup(p++); + if (fl->fh_array[i]->size > NFS_MAXFHSIZE) { + printk(KERN_ERR "NFS: Too big fh %d received %d\n", + i, fl->fh_array[i]->size); + goto out_err; + } + + p = xdr_inline_decode(&stream, fl->fh_array[i]->size); + if (unlikely(!p)) + goto out_err; + memcpy(fl->fh_array[i]->data, p, fl->fh_array[i]->size); + dprintk("DEBUG: %s: fh len %d\n", __func__, + fl->fh_array[i]->size); + } + + folio_put(scratch); + return 0; + +out_err: + folio_put(scratch); + return -EIO; +} + +static void +filelayout_free_lseg(struct pnfs_layout_segment *lseg) +{ + struct nfs4_filelayout_segment *fl = FILELAYOUT_LSEG(lseg); + + dprintk("--> %s\n", __func__); + if (fl->dsaddr != NULL) + nfs4_fl_put_deviceid(fl->dsaddr); + /* This assumes a single RW lseg */ + if (lseg->pls_range.iomode == IOMODE_RW) { + struct nfs4_filelayout *flo; + struct inode *inode; + + flo = FILELAYOUT_FROM_HDR(lseg->pls_layout); + inode = flo->generic_hdr.plh_inode; + spin_lock(&inode->i_lock); + pnfs_generic_ds_cinfo_release_lseg(&flo->commit_info, lseg); + spin_unlock(&inode->i_lock); + } + _filelayout_free_lseg(fl); +} + +static struct pnfs_layout_segment * +filelayout_alloc_lseg(struct pnfs_layout_hdr *layoutid, + struct nfs4_layoutget_res *lgr, + gfp_t gfp_flags) +{ + struct nfs4_filelayout_segment *fl; + int rc; + + dprintk("--> %s\n", __func__); + fl = kzalloc_obj(*fl, gfp_flags); + if (!fl) + return NULL; + + rc = filelayout_decode_layout(layoutid, fl, lgr, gfp_flags); + if (rc != 0 || filelayout_check_layout(layoutid, fl, lgr, gfp_flags)) { + _filelayout_free_lseg(fl); + return NULL; + } + return &fl->generic_hdr; +} + +static bool +filelayout_lseg_is_striped(const struct nfs4_filelayout_segment *flseg) +{ + if (flseg->dsaddr) + return flseg->dsaddr->stripe_count > 1; + return flseg->num_fh > 1; +} + +/* + * filelayout_pg_test(). Called by nfs_can_coalesce_requests() + * + * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number + * of bytes (maximum @req->wb_bytes) that can be coalesced. + */ +static size_t +filelayout_pg_test(struct nfs_pageio_descriptor *pgio, struct nfs_page *prev, + struct nfs_page *req) +{ + unsigned int size; + u64 p_stripe, r_stripe; + u32 stripe_offset; + u64 segment_offset = pgio->pg_lseg->pls_range.offset; + u32 stripe_unit = FILELAYOUT_LSEG(pgio->pg_lseg)->stripe_unit; + + /* calls nfs_generic_pg_test */ + size = pnfs_generic_pg_test(pgio, prev, req); + if (!size) + return 0; + else if (!filelayout_lseg_is_striped(FILELAYOUT_LSEG(pgio->pg_lseg))) + return size; + + /* see if req and prev are in the same stripe */ + if (prev) { + p_stripe = (u64)req_offset(prev) - segment_offset; + r_stripe = (u64)req_offset(req) - segment_offset; + do_div(p_stripe, stripe_unit); + do_div(r_stripe, stripe_unit); + + if (p_stripe != r_stripe) + return 0; + } + + /* calculate remaining bytes in the current stripe */ + div_u64_rem((u64)req_offset(req) - segment_offset, + stripe_unit, + &stripe_offset); + WARN_ON_ONCE(stripe_offset > stripe_unit); + if (stripe_offset >= stripe_unit) + return 0; + return min(stripe_unit - (unsigned int)stripe_offset, size); +} + +static struct pnfs_layout_segment * +fl_pnfs_update_layout(struct inode *ino, + struct nfs_open_context *ctx, + loff_t pos, + u64 count, + enum pnfs_iomode iomode, + bool strict_iomode, + gfp_t gfp_flags) +{ + struct pnfs_layout_segment *lseg = NULL; + struct pnfs_layout_hdr *lo; + struct nfs4_filelayout_segment *fl; + int status; + + lseg = pnfs_update_layout(ino, ctx, pos, count, iomode, strict_iomode, + gfp_flags); + if (IS_ERR(lseg)) { + /* Fall back to MDS on recoverable errors */ + if (!nfs_error_is_fatal_on_server(PTR_ERR(lseg))) + lseg = NULL; + goto out; + } else if (!lseg) + goto out; + + lo = NFS_I(ino)->layout; + fl = FILELAYOUT_LSEG(lseg); + + status = filelayout_check_deviceid(lo, fl, gfp_flags); + if (status) { + pnfs_error_mark_layout_for_return(ino, lseg); + pnfs_set_lo_fail(lseg); + pnfs_put_lseg(lseg); + lseg = NULL; + } +out: + return lseg; +} + +static void +filelayout_pg_init_read(struct nfs_pageio_descriptor *pgio, + struct nfs_page *req) +{ + pnfs_generic_pg_check_layout(pgio, req); + if (!pgio->pg_lseg) { + pgio->pg_lseg = fl_pnfs_update_layout(pgio->pg_inode, + nfs_req_openctx(req), + req_offset(req), + req->wb_bytes, + IOMODE_READ, + false, + nfs_io_gfp_mask()); + if (IS_ERR(pgio->pg_lseg)) { + pgio->pg_error = PTR_ERR(pgio->pg_lseg); + pgio->pg_lseg = NULL; + return; + } + } + /* If no lseg, fall back to read through mds */ + if (pgio->pg_lseg == NULL) + nfs_pageio_reset_read_mds(pgio); +} + +static void +filelayout_pg_init_write(struct nfs_pageio_descriptor *pgio, + struct nfs_page *req) +{ + pnfs_generic_pg_check_layout(pgio, req); + if (!pgio->pg_lseg) { + pgio->pg_lseg = fl_pnfs_update_layout(pgio->pg_inode, + nfs_req_openctx(req), + req_offset(req), + req->wb_bytes, + IOMODE_RW, + false, + nfs_io_gfp_mask()); + if (IS_ERR(pgio->pg_lseg)) { + pgio->pg_error = PTR_ERR(pgio->pg_lseg); + pgio->pg_lseg = NULL; + return; + } + } + + /* If no lseg, fall back to write through mds */ + if (pgio->pg_lseg == NULL) + nfs_pageio_reset_write_mds(pgio); +} + +static const struct nfs_pageio_ops filelayout_pg_read_ops = { + .pg_init = filelayout_pg_init_read, + .pg_test = filelayout_pg_test, + .pg_doio = pnfs_generic_pg_readpages, + .pg_cleanup = pnfs_generic_pg_cleanup, +}; + +static const struct nfs_pageio_ops filelayout_pg_write_ops = { + .pg_init = filelayout_pg_init_write, + .pg_test = filelayout_pg_test, + .pg_doio = pnfs_generic_pg_writepages, + .pg_cleanup = pnfs_generic_pg_cleanup, +}; + +static u32 select_bucket_index(struct nfs4_filelayout_segment *fl, u32 j) +{ + if (fl->stripe_type == STRIPE_SPARSE) + return nfs4_fl_calc_ds_index(&fl->generic_hdr, j); + else + return j; +} + +static void +filelayout_mark_request_commit(struct nfs_page *req, + struct pnfs_layout_segment *lseg, + struct nfs_commit_info *cinfo, + u32 ds_commit_idx) + +{ + struct nfs4_filelayout_segment *fl = FILELAYOUT_LSEG(lseg); + u32 i, j; + + if (fl->commit_through_mds) { + nfs_request_add_commit_list(req, cinfo); + } else { + /* Note that we are calling nfs4_fl_calc_j_index on each page + * that ends up being committed to a data server. An attractive + * alternative is to add a field to nfs_write_data and nfs_page + * to store the value calculated in filelayout_write_pagelist + * and just use that here. + */ + j = nfs4_fl_calc_j_index(lseg, req_offset(req)); + i = select_bucket_index(fl, j); + pnfs_layout_mark_request_commit(req, lseg, cinfo, i); + } +} + +static u32 calc_ds_index_from_commit(struct pnfs_layout_segment *lseg, u32 i) +{ + struct nfs4_filelayout_segment *flseg = FILELAYOUT_LSEG(lseg); + + if (flseg->stripe_type == STRIPE_SPARSE) + return i; + else + return nfs4_fl_calc_ds_index(lseg, i); +} + +static struct nfs_fh * +select_ds_fh_from_commit(struct pnfs_layout_segment *lseg, u32 i) +{ + struct nfs4_filelayout_segment *flseg = FILELAYOUT_LSEG(lseg); + + if (flseg->stripe_type == STRIPE_SPARSE) { + if (flseg->num_fh == 1) + i = 0; + else if (flseg->num_fh == 0) + /* Use the MDS OPEN fh set in nfs_read_rpcsetup */ + return NULL; + } + return flseg->fh_array[i]; +} + +static int filelayout_initiate_commit(struct nfs_commit_data *data, int how) +{ + struct pnfs_layout_segment *lseg = data->lseg; + struct nfs4_pnfs_ds *ds; + struct rpc_clnt *ds_clnt; + u32 idx; + struct nfs_fh *fh; + + idx = calc_ds_index_from_commit(lseg, data->ds_commit_index); + ds = nfs4_fl_prepare_ds(lseg, idx); + if (!ds) + goto out_err; + + ds_clnt = nfs4_find_or_create_ds_client(ds->ds_clp, data->inode); + if (IS_ERR(ds_clnt)) + goto out_err; + + dprintk("%s ino %llu, how %d cl_count %d\n", __func__, + data->inode->i_ino, how, refcount_read(&ds->ds_clp->cl_count)); + data->commit_done_cb = filelayout_commit_done_cb; + refcount_inc(&ds->ds_clp->cl_count); + data->ds_clp = ds->ds_clp; + fh = select_ds_fh_from_commit(lseg, data->ds_commit_index); + if (fh) + data->args.fh = fh; + return nfs_initiate_commit(ds_clnt, data, NFS_PROTO(data->inode), + &filelayout_commit_call_ops, how, + RPC_TASK_SOFTCONN, NULL); +out_err: + pnfs_generic_prepare_to_resend_writes(data); + pnfs_generic_commit_release(data); + return -EAGAIN; +} + +static int +filelayout_commit_pagelist(struct inode *inode, struct list_head *mds_pages, + int how, struct nfs_commit_info *cinfo) +{ + return pnfs_generic_commit_pagelist(inode, mds_pages, how, cinfo, + filelayout_initiate_commit); +} + +static struct nfs4_deviceid_node * +filelayout_alloc_deviceid_node(struct nfs_server *server, + struct pnfs_device *pdev, gfp_t gfp_flags) +{ + struct nfs4_file_layout_dsaddr *dsaddr; + + dsaddr = nfs4_fl_alloc_deviceid_node(server, pdev, gfp_flags); + if (!dsaddr) + return NULL; + return &dsaddr->id_node; +} + +static void +filelayout_free_deviceid_node(struct nfs4_deviceid_node *d) +{ + nfs4_fl_free_deviceid(container_of(d, struct nfs4_file_layout_dsaddr, id_node)); +} + +static struct pnfs_layout_hdr * +filelayout_alloc_layout_hdr(struct inode *inode, gfp_t gfp_flags) +{ + struct nfs4_filelayout *flo; + + flo = kzalloc_obj(*flo, gfp_flags); + if (flo == NULL) + return NULL; + pnfs_init_ds_commit_info(&flo->commit_info); + flo->commit_info.ops = &filelayout_commit_ops; + return &flo->generic_hdr; +} + +static void +filelayout_free_layout_hdr(struct pnfs_layout_hdr *lo) +{ + kfree_rcu(FILELAYOUT_FROM_HDR(lo), generic_hdr.plh_rcu); +} + +static struct pnfs_ds_commit_info * +filelayout_get_ds_info(struct inode *inode) +{ + struct pnfs_layout_hdr *layout = NFS_I(inode)->layout; + + if (layout == NULL) + return NULL; + else + return &FILELAYOUT_FROM_HDR(layout)->commit_info; +} + +static void +filelayout_setup_ds_info(struct pnfs_ds_commit_info *fl_cinfo, + struct pnfs_layout_segment *lseg) +{ + struct nfs4_filelayout_segment *fl = FILELAYOUT_LSEG(lseg); + struct inode *inode = lseg->pls_layout->plh_inode; + struct pnfs_commit_array *array, *new; + unsigned int size = (fl->stripe_type == STRIPE_SPARSE) ? + fl->dsaddr->ds_num : fl->dsaddr->stripe_count; + + new = pnfs_alloc_commit_array(size, nfs_io_gfp_mask()); + if (new) { + spin_lock(&inode->i_lock); + array = pnfs_add_commit_array(fl_cinfo, new, lseg); + spin_unlock(&inode->i_lock); + if (array != new) + pnfs_free_commit_array(new); + } +} + +static void +filelayout_release_ds_info(struct pnfs_ds_commit_info *fl_cinfo, + struct inode *inode) +{ + spin_lock(&inode->i_lock); + pnfs_generic_ds_cinfo_destroy(fl_cinfo); + spin_unlock(&inode->i_lock); +} + +static const struct pnfs_commit_ops filelayout_commit_ops = { + .setup_ds_info = filelayout_setup_ds_info, + .release_ds_info = filelayout_release_ds_info, + .mark_request_commit = filelayout_mark_request_commit, + .clear_request_commit = pnfs_generic_clear_request_commit, + .scan_commit_lists = pnfs_generic_scan_commit_lists, + .recover_commit_reqs = pnfs_generic_recover_commit_reqs, + .commit_pagelist = filelayout_commit_pagelist, +}; + +static struct pnfs_layoutdriver_type filelayout_type = { + .id = LAYOUT_NFSV4_1_FILES, + .name = "LAYOUT_NFSV4_1_FILES", + .owner = THIS_MODULE, + .flags = PNFS_LAYOUTGET_ON_OPEN, + .max_layoutget_response = 4096, /* 1 page or so... */ + .alloc_layout_hdr = filelayout_alloc_layout_hdr, + .free_layout_hdr = filelayout_free_layout_hdr, + .alloc_lseg = filelayout_alloc_lseg, + .free_lseg = filelayout_free_lseg, + .pg_read_ops = &filelayout_pg_read_ops, + .pg_write_ops = &filelayout_pg_write_ops, + .get_ds_info = &filelayout_get_ds_info, + .read_pagelist = filelayout_read_pagelist, + .write_pagelist = filelayout_write_pagelist, + .alloc_deviceid_node = filelayout_alloc_deviceid_node, + .free_deviceid_node = filelayout_free_deviceid_node, + .sync = pnfs_nfs_generic_sync, +}; + +static int __init nfs4filelayout_init(void) +{ + printk(KERN_INFO "%s: NFSv4 File Layout Driver Registering...\n", + __func__); + return pnfs_register_layoutdriver(&filelayout_type); +} + +static void __exit nfs4filelayout_exit(void) +{ + printk(KERN_INFO "%s: NFSv4 File Layout Driver Unregistering...\n", + __func__); + pnfs_unregister_layoutdriver(&filelayout_type); +} + +MODULE_ALIAS("nfs-layouttype4-1"); + +module_init(nfs4filelayout_init); +module_exit(nfs4filelayout_exit); diff --git a/fs/nfs/filelayout/filelayout.h b/fs/nfs/filelayout/filelayout.h new file mode 100644 index 000000000..c7bb5da93 --- /dev/null +++ b/fs/nfs/filelayout/filelayout.h @@ -0,0 +1,118 @@ +/* + * NFSv4 file layout driver data structures. + * + * Copyright (c) 2002 + * The Regents of the University of Michigan + * All Rights Reserved + * + * Dean Hildebrand + * + * Permission is granted to use, copy, create derivative works, and + * redistribute this software and such derivative works for any purpose, + * so long as the name of the University of Michigan is not used in + * any advertising or publicity pertaining to the use or distribution + * of this software without specific, written prior authorization. If + * the above copyright notice or any other identification of the + * University of Michigan is included in any copy of any portion of + * this software, then the disclaimer below must also be included. + * + * This software is provided as is, without representation or warranty + * of any kind either express or implied, including without limitation + * the implied warranties of merchantability, fitness for a particular + * purpose, or noninfringement. The Regents of the University of + * Michigan shall not be liable for any damages, including special, + * indirect, incidental, or consequential damages, with respect to any + * claim arising out of or in connection with the use of the software, + * even if it has been or is hereafter advised of the possibility of + * such damages. + */ + +#ifndef FS_NFS_NFS4FILELAYOUT_H +#define FS_NFS_NFS4FILELAYOUT_H + +#include "../pnfs.h" + +/* + * Field testing shows we need to support up to 4096 stripe indices. + * We store each index as a u8 (u32 on the wire) to keep the memory footprint + * reasonable. This in turn means we support a maximum of 256 + * RFC 5661 multipath_list4 structures. + */ +#define NFS4_PNFS_MAX_STRIPE_CNT 4096 +#define NFS4_PNFS_MAX_MULTI_CNT 256 /* 256 fit into a u8 stripe_index */ + +enum stripetype4 { + STRIPE_SPARSE = 1, + STRIPE_DENSE = 2 +}; + +struct nfs4_file_layout_dsaddr { + struct nfs4_deviceid_node id_node; + u32 stripe_count; + u8 *stripe_indices; + u32 ds_num; + struct nfs4_pnfs_ds *ds_list[] __counted_by(ds_num); +}; + +struct nfs4_filelayout_segment { + struct pnfs_layout_segment generic_hdr; + u32 stripe_type; + u32 commit_through_mds; + u32 stripe_unit; + u32 first_stripe_index; + u64 pattern_offset; + struct nfs4_deviceid deviceid; + struct nfs4_file_layout_dsaddr *dsaddr; /* Point to GETDEVINFO data */ + unsigned int num_fh; + struct nfs_fh **fh_array; +}; + +struct nfs4_filelayout { + struct pnfs_layout_hdr generic_hdr; + struct pnfs_ds_commit_info commit_info; +}; + +static inline struct nfs4_filelayout * +FILELAYOUT_FROM_HDR(struct pnfs_layout_hdr *lo) +{ + return container_of(lo, struct nfs4_filelayout, generic_hdr); +} + +static inline struct nfs4_filelayout_segment * +FILELAYOUT_LSEG(struct pnfs_layout_segment *lseg) +{ + return container_of(lseg, + struct nfs4_filelayout_segment, + generic_hdr); +} + +static inline struct nfs4_deviceid_node * +FILELAYOUT_DEVID_NODE(struct pnfs_layout_segment *lseg) +{ + return &FILELAYOUT_LSEG(lseg)->dsaddr->id_node; +} + +static inline bool +filelayout_test_devid_invalid(struct nfs4_deviceid_node *node) +{ + return test_bit(NFS_DEVICEID_INVALID, &node->flags); +} + +extern bool +filelayout_test_devid_unavailable(struct nfs4_deviceid_node *node); + +extern struct nfs_fh * +nfs4_fl_select_ds_fh(struct pnfs_layout_segment *lseg, u32 j); + +u32 nfs4_fl_calc_j_index(struct pnfs_layout_segment *lseg, loff_t offset); +u32 nfs4_fl_calc_ds_index(struct pnfs_layout_segment *lseg, u32 j); +struct nfs4_pnfs_ds *nfs4_fl_prepare_ds(struct pnfs_layout_segment *lseg, + u32 ds_idx); + +extern struct nfs4_file_layout_dsaddr * +nfs4_fl_alloc_deviceid_node(struct nfs_server *server, + struct pnfs_device *pdev, gfp_t gfp_flags); +extern void nfs4_fl_put_deviceid(struct nfs4_file_layout_dsaddr *dsaddr); +extern void nfs4_fl_free_deviceid(struct nfs4_file_layout_dsaddr *dsaddr); + +#endif /* FS_NFS_NFS4FILELAYOUT_H */ diff --git a/fs/nfs/filelayout/filelayoutdev.c b/fs/nfs/filelayout/filelayoutdev.c new file mode 100644 index 000000000..7226989ee --- /dev/null +++ b/fs/nfs/filelayout/filelayoutdev.c @@ -0,0 +1,305 @@ +/* + * Device operations for the pnfs nfs4 file layout driver. + * + * Copyright (c) 2002 + * The Regents of the University of Michigan + * All Rights Reserved + * + * Dean Hildebrand + * Garth Goodson + * + * Permission is granted to use, copy, create derivative works, and + * redistribute this software and such derivative works for any purpose, + * so long as the name of the University of Michigan is not used in + * any advertising or publicity pertaining to the use or distribution + * of this software without specific, written prior authorization. If + * the above copyright notice or any other identification of the + * University of Michigan is included in any copy of any portion of + * this software, then the disclaimer below must also be included. + * + * This software is provided as is, without representation or warranty + * of any kind either express or implied, including without limitation + * the implied warranties of merchantability, fitness for a particular + * purpose, or noninfringement. The Regents of the University of + * Michigan shall not be liable for any damages, including special, + * indirect, incidental, or consequential damages, with respect to any + * claim arising out of or in connection with the use of the software, + * even if it has been or is hereafter advised of the possibility of + * such damages. + */ + +#include +#include +#include + +#include "../internal.h" +#include "../nfs4session.h" +#include "filelayout.h" +#include "../nfs4trace.h" + +#define NFSDBG_FACILITY NFSDBG_PNFS_LD + +static unsigned int dataserver_timeo = NFS4_DEF_DS_TIMEO; +static unsigned int dataserver_retrans = NFS4_DEF_DS_RETRANS; + +void +nfs4_fl_free_deviceid(struct nfs4_file_layout_dsaddr *dsaddr) +{ + struct nfs4_pnfs_ds *ds; + int i; + + nfs4_print_deviceid(&dsaddr->id_node.deviceid); + + for (i = 0; i < dsaddr->ds_num; i++) { + ds = dsaddr->ds_list[i]; + if (ds != NULL) + nfs4_pnfs_ds_put(ds); + } + kfree(dsaddr->stripe_indices); + kfree_rcu(dsaddr, id_node.rcu); +} + +/* Decode opaque device data and return the result */ +struct nfs4_file_layout_dsaddr * +nfs4_fl_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev, + gfp_t gfp_flags) +{ + int i; + u32 cnt, num; + u8 *indexp; + __be32 *p; + u8 *stripe_indices; + u8 max_stripe_index; + struct nfs4_file_layout_dsaddr *dsaddr = NULL; + struct xdr_stream stream; + struct xdr_buf buf; + struct folio *scratch; + struct list_head dsaddrs; + struct nfs4_pnfs_ds_addr *da; + struct net *net = server->nfs_client->cl_net; + + /* set up xdr stream */ + scratch = folio_alloc(gfp_flags, 0); + if (!scratch) + goto out_err; + + xdr_init_decode_pages(&stream, &buf, pdev->pages, pdev->pglen); + xdr_set_scratch_folio(&stream, scratch); + + /* Get the stripe count (number of stripe index) */ + p = xdr_inline_decode(&stream, 4); + if (unlikely(!p)) + goto out_err_free_scratch; + + cnt = be32_to_cpup(p); + dprintk("%s stripe count %d\n", __func__, cnt); + if (cnt > NFS4_PNFS_MAX_STRIPE_CNT) { + printk(KERN_WARNING "NFS: %s: stripe count %d greater than " + "supported maximum %d\n", __func__, + cnt, NFS4_PNFS_MAX_STRIPE_CNT); + goto out_err_free_scratch; + } + + /* read stripe indices */ + stripe_indices = kcalloc(cnt, sizeof(u8), gfp_flags); + if (!stripe_indices) + goto out_err_free_scratch; + + p = xdr_inline_decode(&stream, cnt << 2); + if (unlikely(!p)) + goto out_err_free_stripe_indices; + + indexp = &stripe_indices[0]; + max_stripe_index = 0; + for (i = 0; i < cnt; i++) { + *indexp = be32_to_cpup(p++); + max_stripe_index = max(max_stripe_index, *indexp); + indexp++; + } + + /* Check the multipath list count */ + p = xdr_inline_decode(&stream, 4); + if (unlikely(!p)) + goto out_err_free_stripe_indices; + + num = be32_to_cpup(p); + dprintk("%s ds_num %u\n", __func__, num); + if (num > NFS4_PNFS_MAX_MULTI_CNT) { + printk(KERN_WARNING "NFS: %s: multipath count %d greater than " + "supported maximum %d\n", __func__, + num, NFS4_PNFS_MAX_MULTI_CNT); + goto out_err_free_stripe_indices; + } + + /* validate stripe indices are all < num */ + if (max_stripe_index >= num) { + printk(KERN_WARNING "NFS: %s: stripe index %u >= num ds %u\n", + __func__, max_stripe_index, num); + goto out_err_free_stripe_indices; + } + + dsaddr = kzalloc_flex(*dsaddr, ds_list, num, gfp_flags); + if (!dsaddr) + goto out_err_free_stripe_indices; + + dsaddr->stripe_count = cnt; + dsaddr->stripe_indices = stripe_indices; + stripe_indices = NULL; + dsaddr->ds_num = num; + nfs4_init_deviceid_node(&dsaddr->id_node, server, &pdev->dev_id); + + INIT_LIST_HEAD(&dsaddrs); + + for (i = 0; i < dsaddr->ds_num; i++) { + int j; + u32 mp_count; + + p = xdr_inline_decode(&stream, 4); + if (unlikely(!p)) + goto out_err_free_deviceid; + + mp_count = be32_to_cpup(p); /* multipath count */ + for (j = 0; j < mp_count; j++) { + da = nfs4_decode_mp_ds_addr(net, &stream, gfp_flags); + if (da) + list_add_tail(&da->da_node, &dsaddrs); + } + if (list_empty(&dsaddrs)) { + dprintk("%s: no suitable DS addresses found\n", + __func__); + goto out_err_free_deviceid; + } + + dsaddr->ds_list[i] = nfs4_pnfs_ds_add(net, &dsaddrs, gfp_flags); + if (!dsaddr->ds_list[i]) + goto out_err_drain_dsaddrs; + trace_fl_getdevinfo(server, &pdev->dev_id, dsaddr->ds_list[i]->ds_remotestr); + + /* If DS was already in cache, free ds addrs */ + while (!list_empty(&dsaddrs)) { + da = list_first_entry(&dsaddrs, + struct nfs4_pnfs_ds_addr, + da_node); + list_del_init(&da->da_node); + kfree(da->da_remotestr); + kfree(da); + } + } + + folio_put(scratch); + return dsaddr; + +out_err_drain_dsaddrs: + while (!list_empty(&dsaddrs)) { + da = list_first_entry(&dsaddrs, struct nfs4_pnfs_ds_addr, + da_node); + list_del_init(&da->da_node); + kfree(da->da_remotestr); + kfree(da); + } +out_err_free_deviceid: + nfs4_fl_free_deviceid(dsaddr); + /* stripe_indicies was part of dsaddr */ + goto out_err_free_scratch; +out_err_free_stripe_indices: + kfree(stripe_indices); +out_err_free_scratch: + folio_put(scratch); +out_err: + dprintk("%s ERROR: returning NULL\n", __func__); + return NULL; +} + +void +nfs4_fl_put_deviceid(struct nfs4_file_layout_dsaddr *dsaddr) +{ + nfs4_put_deviceid_node(&dsaddr->id_node); +} + +/* + * Want res = (offset - layout->pattern_offset)/ layout->stripe_unit + * Then: ((res + fsi) % dsaddr->stripe_count) + */ +u32 +nfs4_fl_calc_j_index(struct pnfs_layout_segment *lseg, loff_t offset) +{ + struct nfs4_filelayout_segment *flseg = FILELAYOUT_LSEG(lseg); + u64 tmp; + + tmp = offset - flseg->pattern_offset; + do_div(tmp, flseg->stripe_unit); + tmp += flseg->first_stripe_index; + return do_div(tmp, flseg->dsaddr->stripe_count); +} + +u32 +nfs4_fl_calc_ds_index(struct pnfs_layout_segment *lseg, u32 j) +{ + return FILELAYOUT_LSEG(lseg)->dsaddr->stripe_indices[j]; +} + +struct nfs_fh * +nfs4_fl_select_ds_fh(struct pnfs_layout_segment *lseg, u32 j) +{ + struct nfs4_filelayout_segment *flseg = FILELAYOUT_LSEG(lseg); + u32 i; + + if (flseg->stripe_type == STRIPE_SPARSE) { + if (flseg->num_fh == 1) + i = 0; + else if (flseg->num_fh == 0) + /* Use the MDS OPEN fh set in nfs_read_rpcsetup */ + return NULL; + else + i = nfs4_fl_calc_ds_index(lseg, j); + } else + i = j; + return flseg->fh_array[i]; +} + +/* Upon return, either ds is connected, or ds is NULL */ +struct nfs4_pnfs_ds * +nfs4_fl_prepare_ds(struct pnfs_layout_segment *lseg, u32 ds_idx) +{ + struct nfs4_file_layout_dsaddr *dsaddr = FILELAYOUT_LSEG(lseg)->dsaddr; + struct nfs4_pnfs_ds *ds = dsaddr->ds_list[ds_idx]; + struct nfs4_deviceid_node *devid = FILELAYOUT_DEVID_NODE(lseg); + struct nfs4_pnfs_ds *ret = ds; + struct nfs_server *s = NFS_SERVER(lseg->pls_layout->plh_inode); + int status; + + if (ds == NULL) { + printk(KERN_ERR "NFS: %s: No data server for offset index %d\n", + __func__, ds_idx); + pnfs_generic_mark_devid_invalid(devid); + goto out; + } + smp_rmb(); + if (ds->ds_clp) + goto out_test_devid; + + status = nfs4_pnfs_ds_connect(s, ds, devid, dataserver_timeo, + dataserver_retrans, 4, + s->nfs_client->cl_minorversion); + if (status) { + nfs4_mark_deviceid_unavailable(devid); + ret = NULL; + goto out; + } + +out_test_devid: + if (ret->ds_clp == NULL || + filelayout_test_devid_unavailable(devid)) + ret = NULL; +out: + return ret; +} + +module_param(dataserver_retrans, uint, 0644); +MODULE_PARM_DESC(dataserver_retrans, "The number of times the NFSv4.1 client " + "retries a request before it attempts further " + " recovery action."); +module_param(dataserver_timeo, uint, 0644); +MODULE_PARM_DESC(dataserver_timeo, "The time (in tenths of a second) the " + "NFSv4.1 client waits for a response from a " + " data server before it retries an NFS request."); diff --git a/fs/nfs/flexfilelayout/Makefile b/fs/nfs/flexfilelayout/Makefile new file mode 100644 index 000000000..49f03422b --- /dev/null +++ b/fs/nfs/flexfilelayout/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the pNFS Flexfile Layout Driver kernel module +# +obj-$(CONFIG_PNFS_FLEXFILE_LAYOUT) += nfs_layout_flexfiles.o +nfs_layout_flexfiles-y := flexfilelayout.o flexfilelayoutdev.o diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c new file mode 100644 index 000000000..c4aa99502 --- /dev/null +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -0,0 +1,3069 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Module for pnfs flexfile layout driver. + * + * Copyright (c) 2014, Primary Data, Inc. All rights reserved. + * + * Tao Peng + */ + +#include +#include +#include +#include +#include +#include + +#include + +#include "flexfilelayout.h" +#include "../nfs4session.h" +#include "../nfs4idmap.h" +#include "../internal.h" +#include "../delegation.h" +#include "../nfs4trace.h" +#include "../iostat.h" +#include "../nfs.h" +#include "../nfs42.h" + +#define NFSDBG_FACILITY NFSDBG_PNFS_LD + +#define FF_LAYOUT_POLL_RETRY_MAX (15*HZ) +#define FF_LAYOUTRETURN_MAXERR 20 + +enum nfs4_ff_op_type { + NFS4_FF_OP_LAYOUTSTATS, + NFS4_FF_OP_LAYOUTRETURN, +}; + +static unsigned short io_maxretrans; + +static const struct pnfs_commit_ops ff_layout_commit_ops; +static void ff_layout_read_record_layoutstats_done(struct rpc_task *task, + struct nfs_pgio_header *hdr); +static int +ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo, + struct nfs42_layoutstat_devinfo *devinfo, + int dev_limit, enum nfs4_ff_op_type type); +static void ff_layout_encode_ff_layoutupdate(struct xdr_stream *xdr, + const struct nfs42_layoutstat_devinfo *devinfo, + struct nfs4_ff_layout_ds_stripe *dss_info); + +static struct pnfs_layout_hdr * +ff_layout_alloc_layout_hdr(struct inode *inode, gfp_t gfp_flags) +{ + struct nfs4_flexfile_layout *ffl; + + ffl = kzalloc_obj(*ffl, gfp_flags); + if (ffl) { + pnfs_init_ds_commit_info(&ffl->commit_info); + INIT_LIST_HEAD(&ffl->error_list); + INIT_LIST_HEAD(&ffl->mirrors); + ffl->last_report_time = ktime_get(); + ffl->commit_info.ops = &ff_layout_commit_ops; + return &ffl->generic_hdr; + } else + return NULL; +} + +static void +ff_layout_free_layout_hdr(struct pnfs_layout_hdr *lo) +{ + struct nfs4_flexfile_layout *ffl = FF_LAYOUT_FROM_HDR(lo); + struct nfs4_ff_layout_ds_err *err, *n; + + list_for_each_entry_safe(err, n, &ffl->error_list, list) { + list_del(&err->list); + kfree(err); + } + kfree_rcu(ffl, generic_hdr.plh_rcu); +} + +static int decode_pnfs_stateid(struct xdr_stream *xdr, nfs4_stateid *stateid) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, NFS4_STATEID_SIZE); + if (unlikely(p == NULL)) + return -ENOBUFS; + stateid->type = NFS4_PNFS_DS_STATEID_TYPE; + memcpy(stateid->data, p, NFS4_STATEID_SIZE); + dprintk("%s: stateid id= [%x%x%x%x]\n", __func__, + p[0], p[1], p[2], p[3]); + return 0; +} + +static int decode_deviceid(struct xdr_stream *xdr, struct nfs4_deviceid *devid) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, NFS4_DEVICEID4_SIZE); + if (unlikely(!p)) + return -ENOBUFS; + memcpy(devid, p, NFS4_DEVICEID4_SIZE); + nfs4_print_deviceid(devid); + return 0; +} + +static int decode_nfs_fh(struct xdr_stream *xdr, struct nfs_fh *fh) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -ENOBUFS; + fh->size = be32_to_cpup(p++); + if (fh->size > NFS_MAXFHSIZE) { + printk(KERN_ERR "NFS flexfiles: Too big fh received %d\n", + fh->size); + return -EOVERFLOW; + } + /* fh.data */ + p = xdr_inline_decode(xdr, fh->size); + if (unlikely(!p)) + return -ENOBUFS; + memcpy(&fh->data, p, fh->size); + dprintk("%s: fh len %d\n", __func__, fh->size); + + return 0; +} + +/* + * Currently only stringified uids and gids are accepted. + * I.e., kerberos is not supported to the DSes, so no pricipals. + * + * That means that one common function will suffice, but when + * principals are added, this should be split to accomodate + * calls to both nfs_map_name_to_uid() and nfs_map_group_to_gid(). + */ +static int +decode_name(struct xdr_stream *xdr, u32 *id) +{ + __be32 *p; + int len; + + /* opaque_length(4)*/ + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -ENOBUFS; + len = be32_to_cpup(p++); + if (len < 0) + return -EINVAL; + + dprintk("%s: len %u\n", __func__, len); + + /* opaque body */ + p = xdr_inline_decode(xdr, len); + if (unlikely(!p)) + return -ENOBUFS; + + if (!nfs_map_string_to_numeric((char *)p, len, id)) + return -EINVAL; + + return 0; +} + +static struct nfsd_file * +ff_local_open_fh(struct pnfs_layout_segment *lseg, u32 ds_idx, u32 dss_id, + struct nfs_client *clp, const struct cred *cred, + struct nfs_fh *fh, fmode_t mode) +{ +#if IS_ENABLED(CONFIG_NFS_LOCALIO) + struct nfs4_ff_layout_mirror *mirror = FF_LAYOUT_COMP(lseg, ds_idx); + + return nfs_local_open_fh(clp, cred, fh, &mirror->dss[dss_id].nfl, mode); +#else + return NULL; +#endif +} + +static bool ff_dss_match_fh(const struct nfs4_ff_layout_ds_stripe *dss1, + const struct nfs4_ff_layout_ds_stripe *dss2) +{ + int i, j; + + if (dss1->fh_versions_cnt != dss2->fh_versions_cnt) + return false; + + for (i = 0; i < dss1->fh_versions_cnt; i++) { + bool found_fh = false; + for (j = 0; j < dss2->fh_versions_cnt; j++) { + if (nfs_compare_fh(&dss1->fh_versions[i], + &dss2->fh_versions[j]) == 0) { + found_fh = true; + break; + } + } + if (!found_fh) + return false; + } + return true; +} + +static bool ff_mirror_match_fh(const struct nfs4_ff_layout_mirror *m1, + const struct nfs4_ff_layout_mirror *m2) +{ + u32 dss_id; + + if (m1->dss_count != m2->dss_count) + return false; + + for (dss_id = 0; dss_id < m1->dss_count; dss_id++) + if (!ff_dss_match_fh(&m1->dss[dss_id], &m2->dss[dss_id])) + return false; + + return true; +} + +static bool ff_mirror_match_devid(const struct nfs4_ff_layout_mirror *m1, + const struct nfs4_ff_layout_mirror *m2) +{ + u32 dss_id; + + if (m1->dss_count != m2->dss_count) + return false; + + for (dss_id = 0; dss_id < m1->dss_count; dss_id++) + if (memcmp(&m1->dss[dss_id].devid, + &m2->dss[dss_id].devid, + sizeof(m1->dss[dss_id].devid)) != 0) + return false; + + return true; +} + +static struct nfs4_ff_layout_mirror * +ff_layout_add_mirror(struct pnfs_layout_hdr *lo, + struct nfs4_ff_layout_mirror *mirror) +{ + struct nfs4_flexfile_layout *ff_layout = FF_LAYOUT_FROM_HDR(lo); + struct nfs4_ff_layout_mirror *pos; + struct inode *inode = lo->plh_inode; + + spin_lock(&inode->i_lock); + list_for_each_entry(pos, &ff_layout->mirrors, mirrors) { + if (!ff_mirror_match_devid(mirror, pos)) + continue; + if (!ff_mirror_match_fh(mirror, pos)) + continue; + if (refcount_inc_not_zero(&pos->ref)) { + spin_unlock(&inode->i_lock); + return pos; + } + } + list_add(&mirror->mirrors, &ff_layout->mirrors); + mirror->layout = lo; + spin_unlock(&inode->i_lock); + return mirror; +} + +static void +ff_layout_remove_mirror(struct nfs4_ff_layout_mirror *mirror) +{ + struct inode *inode; + if (mirror->layout == NULL) + return; + inode = mirror->layout->plh_inode; + spin_lock(&inode->i_lock); + list_del(&mirror->mirrors); + spin_unlock(&inode->i_lock); + mirror->layout = NULL; +} + +static struct nfs4_ff_layout_mirror *ff_layout_alloc_mirror(u32 dss_count, + gfp_t gfp_flags) +{ + struct nfs4_ff_layout_mirror *mirror; + + mirror = kzalloc_obj(*mirror, gfp_flags); + if (mirror == NULL) + return NULL; + + spin_lock_init(&mirror->lock); + refcount_set(&mirror->ref, 1); + INIT_LIST_HEAD(&mirror->mirrors); + + mirror->dss_count = dss_count; + mirror->dss = + kzalloc_objs(struct nfs4_ff_layout_ds_stripe, dss_count, + gfp_flags); + if (mirror->dss == NULL) { + kfree(mirror); + return NULL; + } + + for (u32 dss_id = 0; dss_id < mirror->dss_count; dss_id++) + nfs_localio_file_init(&mirror->dss[dss_id].nfl); + + return mirror; +} + +static void ff_layout_free_mirror(struct nfs4_ff_layout_mirror *mirror) +{ + const struct cred *cred; + u32 dss_id; + + ff_layout_remove_mirror(mirror); + + for (dss_id = 0; dss_id < mirror->dss_count; dss_id++) { + kfree(mirror->dss[dss_id].fh_versions); + cred = rcu_access_pointer(mirror->dss[dss_id].ro_cred); + put_cred(cred); + cred = rcu_access_pointer(mirror->dss[dss_id].rw_cred); + put_cred(cred); + nfs_close_local_fh(&mirror->dss[dss_id].nfl); + nfs4_ff_layout_put_deviceid(mirror->dss[dss_id].mirror_ds); + } + + kfree(mirror->dss); + kfree(mirror); +} + +static void ff_layout_put_mirror(struct nfs4_ff_layout_mirror *mirror) +{ + if (mirror != NULL && refcount_dec_and_test(&mirror->ref)) + ff_layout_free_mirror(mirror); +} + +static void ff_layout_free_mirror_array(struct nfs4_ff_layout_segment *fls) +{ + u32 i; + + for (i = 0; i < fls->mirror_array_cnt; i++) + ff_layout_put_mirror(fls->mirror_array[i]); +} + +static void _ff_layout_free_lseg(struct nfs4_ff_layout_segment *fls) +{ + if (fls) { + ff_layout_free_mirror_array(fls); + kfree(fls); + } +} + +static bool +ff_lseg_match_mirrors(struct pnfs_layout_segment *l1, + struct pnfs_layout_segment *l2) +{ + const struct nfs4_ff_layout_segment *fl1 = FF_LAYOUT_LSEG(l1); + const struct nfs4_ff_layout_segment *fl2 = FF_LAYOUT_LSEG(l2); + u32 i; + + if (fl1->mirror_array_cnt != fl2->mirror_array_cnt) + return false; + for (i = 0; i < fl1->mirror_array_cnt; i++) { + if (fl1->mirror_array[i] != fl2->mirror_array[i]) + return false; + } + return true; +} + +static bool +ff_lseg_range_is_after(const struct pnfs_layout_range *l1, + const struct pnfs_layout_range *l2) +{ + u64 end1, end2; + + if (l1->iomode != l2->iomode) + return l1->iomode != IOMODE_READ; + end1 = pnfs_calc_offset_end(l1->offset, l1->length); + end2 = pnfs_calc_offset_end(l2->offset, l2->length); + if (end1 < l2->offset) + return false; + if (end2 < l1->offset) + return true; + return l2->offset <= l1->offset; +} + +static bool +ff_lseg_merge(struct pnfs_layout_segment *new, + struct pnfs_layout_segment *old) +{ + u64 new_end, old_end; + + if (test_bit(NFS_LSEG_LAYOUTRETURN, &old->pls_flags)) + return false; + if (new->pls_range.iomode != old->pls_range.iomode) + return false; + old_end = pnfs_calc_offset_end(old->pls_range.offset, + old->pls_range.length); + if (old_end < new->pls_range.offset) + return false; + new_end = pnfs_calc_offset_end(new->pls_range.offset, + new->pls_range.length); + if (new_end < old->pls_range.offset) + return false; + if (!ff_lseg_match_mirrors(new, old)) + return false; + + /* Mergeable: copy info from 'old' to 'new' */ + if (new_end < old_end) + new_end = old_end; + if (new->pls_range.offset < old->pls_range.offset) + new->pls_range.offset = old->pls_range.offset; + new->pls_range.length = pnfs_calc_offset_length(new->pls_range.offset, + new_end); + if (test_bit(NFS_LSEG_ROC, &old->pls_flags)) + set_bit(NFS_LSEG_ROC, &new->pls_flags); + return true; +} + +static void +ff_layout_add_lseg(struct pnfs_layout_hdr *lo, + struct pnfs_layout_segment *lseg, + struct list_head *free_me) +{ + pnfs_generic_layout_insert_lseg(lo, lseg, + ff_lseg_range_is_after, + ff_lseg_merge, + free_me); +} + +static u32 ff_mirror_efficiency_sum(const struct nfs4_ff_layout_mirror *mirror) +{ + u32 dss_id, sum = 0; + + for (dss_id = 0; dss_id < mirror->dss_count; dss_id++) + sum += mirror->dss[dss_id].efficiency; + + return sum; +} + +static void ff_layout_sort_mirrors(struct nfs4_ff_layout_segment *fls) +{ + int i, j; + + for (i = 0; i < fls->mirror_array_cnt - 1; i++) { + for (j = i + 1; j < fls->mirror_array_cnt; j++) + if (ff_mirror_efficiency_sum(fls->mirror_array[i]) < + ff_mirror_efficiency_sum(fls->mirror_array[j])) + swap(fls->mirror_array[i], + fls->mirror_array[j]); + } +} + +static struct pnfs_layout_segment * +ff_layout_alloc_lseg(struct pnfs_layout_hdr *lh, + struct nfs4_layoutget_res *lgr, + gfp_t gfp_flags) +{ + struct pnfs_layout_segment *ret; + struct nfs4_ff_layout_segment *fls = NULL; + struct xdr_stream stream; + struct xdr_buf buf; + struct folio *scratch; + u64 stripe_unit; + u32 mirror_array_cnt; + __be32 *p; + int i, rc; + struct nfs4_ff_layout_ds_stripe *dss_info; + + dprintk("--> %s\n", __func__); + scratch = folio_alloc(gfp_flags, 0); + if (!scratch) + return ERR_PTR(-ENOMEM); + + xdr_init_decode_pages(&stream, &buf, lgr->layoutp->pages, + lgr->layoutp->len); + xdr_set_scratch_folio(&stream, scratch); + + /* stripe unit and mirror_array_cnt */ + rc = -EIO; + p = xdr_inline_decode(&stream, 8 + 4); + if (!p) + goto out_err_free; + + p = xdr_decode_hyper(p, &stripe_unit); + mirror_array_cnt = be32_to_cpup(p++); + dprintk("%s: stripe_unit=%llu mirror_array_cnt=%u\n", __func__, + stripe_unit, mirror_array_cnt); + + if (mirror_array_cnt > NFS4_FLEXFILE_LAYOUT_MAX_MIRROR_CNT || + mirror_array_cnt == 0) + goto out_err_free; + + rc = -ENOMEM; + fls = kzalloc_flex(*fls, mirror_array, mirror_array_cnt, gfp_flags); + if (!fls) + goto out_err_free; + + fls->mirror_array_cnt = mirror_array_cnt; + fls->stripe_unit = stripe_unit; + + u32 dss_count = 0; + for (i = 0; i < fls->mirror_array_cnt; i++) { + struct nfs4_ff_layout_mirror *mirror; + struct cred *kcred; + const struct cred __rcu *cred; + kuid_t uid; + kgid_t gid; + u32 fh_count, id; + int j, dss_id; + + rc = -EIO; + p = xdr_inline_decode(&stream, 4); + if (!p) + goto out_err_free; + + // Ensure all mirrors have same stripe count. + if (dss_count == 0) + dss_count = be32_to_cpup(p); + else if (dss_count != be32_to_cpup(p)) + goto out_err_free; + + if (dss_count > NFS4_FLEXFILE_LAYOUT_MAX_STRIPE_CNT || + dss_count == 0) + goto out_err_free; + + if (dss_count > 1 && stripe_unit == 0) + goto out_err_free; + + fls->mirror_array[i] = ff_layout_alloc_mirror(dss_count, gfp_flags); + if (fls->mirror_array[i] == NULL) { + rc = -ENOMEM; + goto out_err_free; + } + + for (dss_id = 0; dss_id < dss_count; dss_id++) { + dss_info = &fls->mirror_array[i]->dss[dss_id]; + dss_info->mirror = fls->mirror_array[i]; + + /* deviceid */ + rc = decode_deviceid(&stream, &dss_info->devid); + if (rc) + goto out_err_free; + + /* efficiency */ + rc = -EIO; + p = xdr_inline_decode(&stream, 4); + if (!p) + goto out_err_free; + dss_info->efficiency = be32_to_cpup(p); + + /* stateid */ + rc = decode_pnfs_stateid(&stream, &dss_info->stateid); + if (rc) + goto out_err_free; + + /* fh */ + rc = -EIO; + p = xdr_inline_decode(&stream, 4); + if (!p) + goto out_err_free; + fh_count = be32_to_cpup(p); + if (fh_count == 0) { + rc = -EINVAL; + goto out_err_free; + } + + dss_info->fh_versions = + kzalloc_objs(struct nfs_fh, fh_count, gfp_flags); + if (dss_info->fh_versions == NULL) { + rc = -ENOMEM; + goto out_err_free; + } + + for (j = 0; j < fh_count; j++) { + rc = decode_nfs_fh(&stream, + &dss_info->fh_versions[j]); + if (rc) + goto out_err_free; + } + + dss_info->fh_versions_cnt = fh_count; + + /* user */ + rc = decode_name(&stream, &id); + if (rc) + goto out_err_free; + + uid = make_kuid(&init_user_ns, id); + + /* group */ + rc = decode_name(&stream, &id); + if (rc) + goto out_err_free; + + gid = make_kgid(&init_user_ns, id); + + if (gfp_flags & __GFP_FS) + kcred = prepare_kernel_cred(&init_task); + else { + unsigned int nofs_flags = memalloc_nofs_save(); + + kcred = prepare_kernel_cred(&init_task); + memalloc_nofs_restore(nofs_flags); + } + rc = -ENOMEM; + if (!kcred) + goto out_err_free; + kcred->fsuid = uid; + kcred->fsgid = gid; + cred = RCU_INITIALIZER(kcred); + + if (lgr->range.iomode == IOMODE_READ) + rcu_assign_pointer(dss_info->ro_cred, cred); + else + rcu_assign_pointer(dss_info->rw_cred, cred); + } + + mirror = ff_layout_add_mirror(lh, fls->mirror_array[i]); + if (mirror != fls->mirror_array[i]) { + for (dss_id = 0; dss_id < dss_count; dss_id++) { + dss_info = &fls->mirror_array[i]->dss[dss_id]; + /* swap cred ptrs so free_mirror will clean up old */ + if (lgr->range.iomode == IOMODE_READ) { + cred = xchg(&mirror->dss[dss_id].ro_cred, + dss_info->ro_cred); + rcu_assign_pointer(dss_info->ro_cred, cred); + } else { + cred = xchg(&mirror->dss[dss_id].rw_cred, + dss_info->rw_cred); + rcu_assign_pointer(dss_info->rw_cred, cred); + } + } + ff_layout_free_mirror(fls->mirror_array[i]); + fls->mirror_array[i] = mirror; + } + + dprintk("%s: iomode %s uid %u gid %u\n", __func__, + lgr->range.iomode == IOMODE_READ ? "READ" : "RW", + from_kuid(&init_user_ns, uid), + from_kgid(&init_user_ns, gid)); + } + + p = xdr_inline_decode(&stream, 4); + if (!p) + goto out_sort_mirrors; + fls->flags = be32_to_cpup(p); + if (fls->flags & FF_FLAGS_NO_IO_THRU_MDS) + set_bit(NFS4_FF_HDR_NO_IO_THRU_MDS, + &FF_LAYOUT_FROM_HDR(lh)->flags); + + p = xdr_inline_decode(&stream, 4); + if (!p) + goto out_sort_mirrors; + for (i=0; i < fls->mirror_array_cnt; i++) + fls->mirror_array[i]->report_interval = be32_to_cpup(p); + +out_sort_mirrors: + ff_layout_sort_mirrors(fls); + ret = &fls->generic_hdr; + dprintk("<-- %s (success)\n", __func__); +out_free_page: + folio_put(scratch); + return ret; +out_err_free: + _ff_layout_free_lseg(fls); + ret = ERR_PTR(rc); + dprintk("<-- %s (%d)\n", __func__, rc); + goto out_free_page; +} + +static void +ff_layout_free_lseg(struct pnfs_layout_segment *lseg) +{ + struct nfs4_ff_layout_segment *fls = FF_LAYOUT_LSEG(lseg); + + dprintk("--> %s\n", __func__); + + if (lseg->pls_range.iomode == IOMODE_RW) { + struct nfs4_flexfile_layout *ffl; + struct inode *inode; + + ffl = FF_LAYOUT_FROM_HDR(lseg->pls_layout); + inode = ffl->generic_hdr.plh_inode; + spin_lock(&inode->i_lock); + pnfs_generic_ds_cinfo_release_lseg(&ffl->commit_info, lseg); + spin_unlock(&inode->i_lock); + } + _ff_layout_free_lseg(fls); +} + +static u32 calc_commit_idx(struct pnfs_layout_segment *lseg, + u32 mirror_idx, u32 dss_id) +{ + struct nfs4_ff_layout_segment *flseg = FF_LAYOUT_LSEG(lseg); + + return (mirror_idx * flseg->mirror_array[0]->dss_count) + dss_id; +} + +static u32 calc_mirror_idx_from_commit(struct pnfs_layout_segment *lseg, + u32 commit_index) +{ + return commit_index / FF_LAYOUT_LSEG(lseg)->mirror_array[0]->dss_count; +} + +static u32 calc_dss_id_from_commit(struct pnfs_layout_segment *lseg, + u32 commit_index) +{ + return commit_index % FF_LAYOUT_LSEG(lseg)->mirror_array[0]->dss_count; +} + +static void +nfs4_ff_start_busy_timer(struct nfs4_ff_busy_timer *timer, ktime_t now) +{ + /* first IO request? */ + if (atomic_inc_return(&timer->n_ops) == 1) { + timer->start_time = now; + } +} + +static ktime_t +nfs4_ff_end_busy_timer(struct nfs4_ff_busy_timer *timer, ktime_t now) +{ + ktime_t start; + + if (atomic_dec_return(&timer->n_ops) < 0) + WARN_ON_ONCE(1); + + start = timer->start_time; + timer->start_time = now; + return ktime_sub(now, start); +} + +static bool +nfs4_ff_layoutstat_start_io(struct nfs4_ff_layout_mirror *mirror, + u32 dss_id, + struct nfs4_ff_layoutstat *layoutstat, + ktime_t now) +{ + s64 report_interval = FF_LAYOUTSTATS_REPORT_INTERVAL; + struct nfs4_flexfile_layout *ffl = FF_LAYOUT_FROM_HDR(mirror->layout); + + nfs4_ff_start_busy_timer(&layoutstat->busy_timer, now); + if (!mirror->dss[dss_id].start_time) + mirror->dss[dss_id].start_time = now; + if (mirror->report_interval != 0) + report_interval = (s64)mirror->report_interval * 1000LL; + else if (layoutstats_timer != 0) + report_interval = (s64)layoutstats_timer * 1000LL; + if (ktime_to_ms(ktime_sub(now, ffl->last_report_time)) >= + report_interval) { + ffl->last_report_time = now; + return true; + } + + return false; +} + +static void +nfs4_ff_layout_stat_io_update_requested(struct nfs4_ff_layoutstat *layoutstat, + __u64 requested) +{ + struct nfs4_ff_io_stat *iostat = &layoutstat->io_stat; + + iostat->ops_requested++; + iostat->bytes_requested += requested; +} + +static void +nfs4_ff_layout_stat_io_update_completed(struct nfs4_ff_layoutstat *layoutstat, + __u64 requested, + __u64 completed, + ktime_t time_completed, + ktime_t time_started) +{ + struct nfs4_ff_io_stat *iostat = &layoutstat->io_stat; + ktime_t completion_time = ktime_sub(time_completed, time_started); + ktime_t timer; + + iostat->ops_completed++; + iostat->bytes_completed += completed; + iostat->bytes_not_delivered += requested - completed; + + timer = nfs4_ff_end_busy_timer(&layoutstat->busy_timer, time_completed); + iostat->total_busy_time = + ktime_add(iostat->total_busy_time, timer); + iostat->aggregate_completion_time = + ktime_add(iostat->aggregate_completion_time, + completion_time); +} + +static void +nfs4_ff_layout_stat_io_start_read(struct inode *inode, + struct nfs4_ff_layout_mirror *mirror, + u32 dss_id, + __u64 requested, ktime_t now) +{ + bool report; + + spin_lock(&mirror->lock); + report = nfs4_ff_layoutstat_start_io( + mirror, dss_id, &mirror->dss[dss_id].read_stat, now); + nfs4_ff_layout_stat_io_update_requested( + &mirror->dss[dss_id].read_stat, requested); + set_bit(NFS4_FF_MIRROR_STAT_AVAIL, &mirror->flags); + spin_unlock(&mirror->lock); + + if (report) + pnfs_report_layoutstat(inode, nfs_io_gfp_mask()); +} + +static void +nfs4_ff_layout_stat_io_end_read(struct rpc_task *task, + struct nfs4_ff_layout_mirror *mirror, + u32 dss_id, + __u64 requested, + __u64 completed) +{ + spin_lock(&mirror->lock); + nfs4_ff_layout_stat_io_update_completed(&mirror->dss[dss_id].read_stat, + requested, completed, + ktime_get(), task->tk_start); + set_bit(NFS4_FF_MIRROR_STAT_AVAIL, &mirror->flags); + spin_unlock(&mirror->lock); +} + +static void +nfs4_ff_layout_stat_io_start_write(struct inode *inode, + struct nfs4_ff_layout_mirror *mirror, + u32 dss_id, + __u64 requested, ktime_t now) +{ + bool report; + + spin_lock(&mirror->lock); + report = nfs4_ff_layoutstat_start_io( + mirror, + dss_id, + &mirror->dss[dss_id].write_stat, + now); + nfs4_ff_layout_stat_io_update_requested( + &mirror->dss[dss_id].write_stat, + requested); + set_bit(NFS4_FF_MIRROR_STAT_AVAIL, &mirror->flags); + spin_unlock(&mirror->lock); + + if (report) + pnfs_report_layoutstat(inode, nfs_io_gfp_mask()); +} + +static void +nfs4_ff_layout_stat_io_end_write(struct rpc_task *task, + struct nfs4_ff_layout_mirror *mirror, + u32 dss_id, + __u64 requested, + __u64 completed, + enum nfs3_stable_how committed) +{ + if (committed == NFS_UNSTABLE) + requested = completed = 0; + + spin_lock(&mirror->lock); + nfs4_ff_layout_stat_io_update_completed(&mirror->dss[dss_id].write_stat, + requested, completed, ktime_get(), task->tk_start); + set_bit(NFS4_FF_MIRROR_STAT_AVAIL, &mirror->flags); + spin_unlock(&mirror->lock); +} + +static void +ff_layout_mark_ds_unreachable(struct pnfs_layout_segment *lseg, u32 idx, u32 dss_id) +{ + struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx, dss_id); + + if (devid) + nfs4_mark_deviceid_unavailable(devid); +} + +static void +ff_layout_mark_ds_reachable(struct pnfs_layout_segment *lseg, u32 idx, u32 dss_id) +{ + struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx, dss_id); + + if (devid) + nfs4_mark_deviceid_available(devid); +} + +static struct nfs4_pnfs_ds * +ff_layout_choose_ds_for_read(struct pnfs_layout_segment *lseg, + u32 start_idx, u32 *best_idx, + u32 offset, u32 *dss_id, + bool check_device) +{ + struct nfs4_ff_layout_segment *fls = FF_LAYOUT_LSEG(lseg); + struct nfs4_ff_layout_mirror *mirror; + struct nfs4_pnfs_ds *ds = ERR_PTR(-EAGAIN); + u32 idx; + + /* mirrors are initially sorted by efficiency */ + for (idx = start_idx; idx < fls->mirror_array_cnt; idx++) { + mirror = FF_LAYOUT_COMP(lseg, idx); + *dss_id = nfs4_ff_layout_calc_dss_id( + fls->stripe_unit, + fls->mirror_array[idx]->dss_count, + offset); + ds = nfs4_ff_layout_prepare_ds(lseg, mirror, *dss_id, false); + if (IS_ERR(ds)) + continue; + + if (check_device && + nfs4_test_deviceid_unavailable(&mirror->dss[*dss_id].mirror_ds->id_node)) { + // reinitialize the error state in case if this is the last iteration + ds = ERR_PTR(-EINVAL); + continue; + } + + *best_idx = idx; + break; + } + + return ds; +} + +static struct nfs4_pnfs_ds * +ff_layout_choose_any_ds_for_read(struct pnfs_layout_segment *lseg, + u32 start_idx, u32 *best_idx, + u32 offset, u32 *dss_id) +{ + return ff_layout_choose_ds_for_read(lseg, start_idx, best_idx, + offset, dss_id, false); +} + +static struct nfs4_pnfs_ds * +ff_layout_choose_valid_ds_for_read(struct pnfs_layout_segment *lseg, + u32 start_idx, u32 *best_idx, + u32 offset, u32 *dss_id) +{ + return ff_layout_choose_ds_for_read(lseg, start_idx, best_idx, + offset, dss_id, true); +} + +static struct nfs4_pnfs_ds * +ff_layout_choose_best_ds_for_read(struct pnfs_layout_segment *lseg, + u32 start_idx, u32 *best_idx, + u32 offset, u32 *dss_id) +{ + struct nfs4_pnfs_ds *ds; + + ds = ff_layout_choose_valid_ds_for_read(lseg, start_idx, best_idx, + offset, dss_id); + if (!IS_ERR(ds)) + return ds; + return ff_layout_choose_any_ds_for_read(lseg, start_idx, best_idx, + offset, dss_id); +} + +static struct nfs4_pnfs_ds * +ff_layout_get_ds_for_read(struct nfs_pageio_descriptor *pgio, + u32 *best_idx, + u32 offset, + u32 *dss_id) +{ + struct pnfs_layout_segment *lseg = pgio->pg_lseg; + struct nfs4_pnfs_ds *ds; + + ds = ff_layout_choose_best_ds_for_read(lseg, pgio->pg_mirror_idx, + best_idx, offset, dss_id); + if (!IS_ERR(ds) || !pgio->pg_mirror_idx) + return ds; + return ff_layout_choose_best_ds_for_read(lseg, 0, best_idx, + offset, dss_id); +} + +static void +ff_layout_pg_get_read(struct nfs_pageio_descriptor *pgio, + struct nfs_page *req, + bool strict_iomode) +{ + pnfs_put_lseg(pgio->pg_lseg); + pgio->pg_lseg = + pnfs_update_layout(pgio->pg_inode, nfs_req_openctx(req), + req_offset(req), req->wb_bytes, IOMODE_READ, + strict_iomode, nfs_io_gfp_mask()); + if (IS_ERR(pgio->pg_lseg)) { + pgio->pg_error = PTR_ERR(pgio->pg_lseg); + pgio->pg_lseg = NULL; + } +} + +static bool +ff_layout_lseg_is_striped(const struct nfs4_ff_layout_segment *fls) +{ + return fls->mirror_array[0]->dss_count > 1; +} + +/* + * ff_layout_pg_test(). Called by nfs_can_coalesce_requests() + * + * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number + * of bytes (maximum @req->wb_bytes) that can be coalesced. + */ +static size_t +ff_layout_pg_test(struct nfs_pageio_descriptor *pgio, struct nfs_page *prev, + struct nfs_page *req) +{ + unsigned int size; + u64 p_stripe, r_stripe; + u32 stripe_offset; + u64 segment_offset = pgio->pg_lseg->pls_range.offset; + u32 stripe_unit = FF_LAYOUT_LSEG(pgio->pg_lseg)->stripe_unit; + + /* calls nfs_generic_pg_test */ + size = pnfs_generic_pg_test(pgio, prev, req); + if (!size) + return 0; + else if (!ff_layout_lseg_is_striped(FF_LAYOUT_LSEG(pgio->pg_lseg))) + return size; + + /* see if req and prev are in the same stripe */ + if (prev) { + p_stripe = (u64)req_offset(prev) - segment_offset; + r_stripe = (u64)req_offset(req) - segment_offset; + do_div(p_stripe, stripe_unit); + do_div(r_stripe, stripe_unit); + + if (p_stripe != r_stripe) + return 0; + } + + /* calculate remaining bytes in the current stripe */ + div_u64_rem((u64)req_offset(req) - segment_offset, + stripe_unit, + &stripe_offset); + WARN_ON_ONCE(stripe_offset > stripe_unit); + if (stripe_offset >= stripe_unit) + return 0; + return min(stripe_unit - (unsigned int)stripe_offset, size); +} + +static void +ff_layout_pg_init_read(struct nfs_pageio_descriptor *pgio, + struct nfs_page *req) +{ + struct nfs_pgio_mirror *pgm; + struct nfs4_ff_layout_mirror *mirror; + struct nfs4_pnfs_ds *ds; + u32 ds_idx, dss_id; + + if (NFS_SERVER(pgio->pg_inode)->flags & + (NFS_MOUNT_SOFT|NFS_MOUNT_SOFTERR)) + pgio->pg_maxretrans = io_maxretrans; +retry: + pnfs_generic_pg_check_layout(pgio, req); + /* Use full layout for now */ + if (!pgio->pg_lseg) { + ff_layout_pg_get_read(pgio, req, false); + if (!pgio->pg_lseg) + goto out_nolseg; + } + if (ff_layout_avoid_read_on_rw(pgio->pg_lseg)) { + ff_layout_pg_get_read(pgio, req, true); + if (!pgio->pg_lseg) + goto out_nolseg; + } + /* Reset wb_nio, since getting layout segment was successful */ + req->wb_nio = 0; + + ds = ff_layout_get_ds_for_read(pgio, &ds_idx, + req_offset(req), &dss_id); + if (IS_ERR(ds)) { + if (!ff_layout_no_fallback_to_mds(pgio->pg_lseg)) + goto out_mds; + pnfs_generic_pg_cleanup(pgio); + /* Sleep for 1 second before retrying */ + ssleep(1); + goto retry; + } + + mirror = FF_LAYOUT_COMP(pgio->pg_lseg, ds_idx); + pgm = &pgio->pg_mirrors[0]; + pgm->pg_bsize = mirror->dss[dss_id].mirror_ds->ds_versions[0].rsize; + + pgio->pg_mirror_idx = ds_idx; + return; +out_nolseg: + if (pgio->pg_error < 0) { + if (pgio->pg_error != -EAGAIN) + return; + /* Retry getting layout segment if lower layer returned -EAGAIN */ + if (pgio->pg_maxretrans && req->wb_nio++ > pgio->pg_maxretrans) { + if (NFS_SERVER(pgio->pg_inode)->flags & NFS_MOUNT_SOFTERR) + pgio->pg_error = -ETIMEDOUT; + else + pgio->pg_error = -EIO; + return; + } + pgio->pg_error = 0; + /* Sleep for 1 second before retrying */ + ssleep(1); + goto retry; + } +out_mds: + trace_pnfs_mds_fallback_pg_init_read(pgio->pg_inode, + 0, NFS4_MAX_UINT64, IOMODE_READ, + NFS_I(pgio->pg_inode)->layout, + pgio->pg_lseg); + pgio->pg_maxretrans = 0; + nfs_pageio_reset_read_mds(pgio); +} + +static void +ff_layout_pg_init_write(struct nfs_pageio_descriptor *pgio, + struct nfs_page *req) +{ + struct nfs4_ff_layout_mirror *mirror; + struct nfs_pgio_mirror *pgm; + struct nfs4_pnfs_ds *ds; + u32 i, dss_id; + +retry: + pnfs_generic_pg_check_layout(pgio, req); + if (!pgio->pg_lseg) { + pgio->pg_lseg = + pnfs_update_layout(pgio->pg_inode, nfs_req_openctx(req), + req_offset(req), req->wb_bytes, + IOMODE_RW, false, nfs_io_gfp_mask()); + if (IS_ERR(pgio->pg_lseg)) { + pgio->pg_error = PTR_ERR(pgio->pg_lseg); + pgio->pg_lseg = NULL; + return; + } + } + /* If no lseg, fall back to write through mds */ + if (pgio->pg_lseg == NULL) + goto out_mds; + + /* Use a direct mapping of ds_idx to pgio mirror_idx */ + if (pgio->pg_mirror_count != FF_LAYOUT_MIRROR_COUNT(pgio->pg_lseg)) + goto out_eagain; + + for (i = 0; i < pgio->pg_mirror_count; i++) { + mirror = FF_LAYOUT_COMP(pgio->pg_lseg, i); + dss_id = nfs4_ff_layout_calc_dss_id( + FF_LAYOUT_LSEG(pgio->pg_lseg)->stripe_unit, + mirror->dss_count, + req_offset(req)); + ds = nfs4_ff_layout_prepare_ds(pgio->pg_lseg, mirror, + dss_id, true); + if (IS_ERR(ds)) { + if (!ff_layout_no_fallback_to_mds(pgio->pg_lseg)) + goto out_mds; + pnfs_generic_pg_cleanup(pgio); + /* Sleep for 1 second before retrying */ + ssleep(1); + goto retry; + } + pgm = &pgio->pg_mirrors[i]; + pgm->pg_bsize = mirror->dss[dss_id].mirror_ds->ds_versions[0].wsize; + } + + if (NFS_SERVER(pgio->pg_inode)->flags & + (NFS_MOUNT_SOFT|NFS_MOUNT_SOFTERR)) + pgio->pg_maxretrans = io_maxretrans; + return; +out_eagain: + pnfs_generic_pg_cleanup(pgio); + pgio->pg_error = -EAGAIN; + return; +out_mds: + trace_pnfs_mds_fallback_pg_init_write(pgio->pg_inode, + 0, NFS4_MAX_UINT64, IOMODE_RW, + NFS_I(pgio->pg_inode)->layout, + pgio->pg_lseg); + pgio->pg_maxretrans = 0; + nfs_pageio_reset_write_mds(pgio); + pgio->pg_error = -EAGAIN; +} + +static unsigned int +ff_layout_pg_get_mirror_count_write(struct nfs_pageio_descriptor *pgio, + struct nfs_page *req) +{ + if (!pgio->pg_lseg) { + pgio->pg_lseg = + pnfs_update_layout(pgio->pg_inode, nfs_req_openctx(req), + req_offset(req), req->wb_bytes, + IOMODE_RW, false, nfs_io_gfp_mask()); + if (IS_ERR(pgio->pg_lseg)) { + pgio->pg_error = PTR_ERR(pgio->pg_lseg); + pgio->pg_lseg = NULL; + goto out; + } + } + if (pgio->pg_lseg) + return FF_LAYOUT_MIRROR_COUNT(pgio->pg_lseg); + + trace_pnfs_mds_fallback_pg_get_mirror_count(pgio->pg_inode, + 0, NFS4_MAX_UINT64, IOMODE_RW, + NFS_I(pgio->pg_inode)->layout, + pgio->pg_lseg); + if (NFS_I(pgio->pg_inode)->layout && + ff_layout_hdr_no_fallback_to_mds(NFS_I(pgio->pg_inode)->layout)) { + /* + * FF_FLAGS_NO_IO_THRU_MDS: no current lseg but the server's + * policy forbids MDS fallback. Surface -EAGAIN so writeback + * retries rather than silently issuing the WRITE via MDS. + */ + pgio->pg_error = -EAGAIN; + goto out; + } + /* no lseg means that pnfs is not in use, so no mirroring here */ + nfs_pageio_reset_write_mds(pgio); +out: + return 1; +} + +static u32 +ff_layout_pg_set_mirror_write(struct nfs_pageio_descriptor *desc, u32 idx) +{ + u32 old = desc->pg_mirror_idx; + + desc->pg_mirror_idx = idx; + return old; +} + +static struct nfs_pgio_mirror * +ff_layout_pg_get_mirror_write(struct nfs_pageio_descriptor *desc, u32 idx) +{ + return &desc->pg_mirrors[idx]; +} + +static const struct nfs_pageio_ops ff_layout_pg_read_ops = { + .pg_init = ff_layout_pg_init_read, + .pg_test = ff_layout_pg_test, + .pg_doio = pnfs_generic_pg_readpages, + .pg_cleanup = pnfs_generic_pg_cleanup, +}; + +static const struct nfs_pageio_ops ff_layout_pg_write_ops = { + .pg_init = ff_layout_pg_init_write, + .pg_test = ff_layout_pg_test, + .pg_doio = pnfs_generic_pg_writepages, + .pg_get_mirror_count = ff_layout_pg_get_mirror_count_write, + .pg_cleanup = pnfs_generic_pg_cleanup, + .pg_get_mirror = ff_layout_pg_get_mirror_write, + .pg_set_mirror = ff_layout_pg_set_mirror_write, +}; + +static void ff_layout_reset_write(struct nfs_pgio_header *hdr, bool retry_pnfs) +{ + struct rpc_task *task = &hdr->task; + + pnfs_layoutcommit_inode(hdr->inode, false); + + if (retry_pnfs) { + dprintk("%s Reset task %5u for i/o through pNFS " + "(req %s/%llu, %u bytes @ offset %llu)\n", __func__, + hdr->task.tk_pid, + hdr->inode->i_sb->s_id, + (unsigned long long)hdr->inode->i_ino, + hdr->args.count, + (unsigned long long)hdr->args.offset); + + hdr->completion_ops->reschedule_io(hdr); + return; + } + + if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { + dprintk("%s Reset task %5u for i/o through MDS " + "(req %s/%llu, %u bytes @ offset %llu)\n", __func__, + hdr->task.tk_pid, + hdr->inode->i_sb->s_id, + (unsigned long long)hdr->inode->i_ino, + hdr->args.count, + (unsigned long long)hdr->args.offset); + + trace_pnfs_mds_fallback_write_done(hdr->inode, + hdr->args.offset, hdr->args.count, + IOMODE_RW, NFS_I(hdr->inode)->layout, + hdr->lseg); + task->tk_status = pnfs_write_done_resend_to_mds(hdr); + } +} + +static void ff_layout_resend_pnfs_read(struct nfs_pgio_header *hdr) +{ + u32 idx = hdr->pgio_mirror_idx + 1; + u32 new_idx = 0; + u32 dss_id = 0; + struct nfs4_pnfs_ds *ds; + + ds = ff_layout_choose_any_ds_for_read(hdr->lseg, idx, &new_idx, + hdr->args.offset, &dss_id); + if (IS_ERR(ds)) + pnfs_error_mark_layout_for_return(hdr->inode, hdr->lseg); + else + ff_layout_send_layouterror(hdr->lseg); + pnfs_read_resend_pnfs(hdr, new_idx); +} + +static void ff_layout_reset_read(struct nfs_pgio_header *hdr) +{ + struct rpc_task *task = &hdr->task; + + pnfs_layoutcommit_inode(hdr->inode, false); + pnfs_error_mark_layout_for_return(hdr->inode, hdr->lseg); + + if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { + dprintk("%s Reset task %5u for i/o through MDS " + "(req %s/%llu, %u bytes @ offset %llu)\n", __func__, + hdr->task.tk_pid, + hdr->inode->i_sb->s_id, + (unsigned long long)hdr->inode->i_ino, + hdr->args.count, + (unsigned long long)hdr->args.offset); + + trace_pnfs_mds_fallback_read_done(hdr->inode, + hdr->args.offset, hdr->args.count, + IOMODE_READ, NFS_I(hdr->inode)->layout, + hdr->lseg); + task->tk_status = pnfs_read_done_resend_to_mds(hdr); + } +} + +static int ff_layout_async_handle_error_v4(struct rpc_task *task, + u32 op_status, + struct nfs4_state *state, + struct nfs_client *clp, + struct pnfs_layout_segment *lseg, + u32 idx, u32 dss_id) +{ + struct pnfs_layout_hdr *lo = lseg->pls_layout; + struct inode *inode = lo->plh_inode; + struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx, dss_id); + struct nfs4_slot_table *tbl = &clp->cl_session->fc_slot_table; + + switch (op_status) { + case NFS4_OK: + case NFS4ERR_NXIO: + break; + case NFSERR_PERM: + if (!task->tk_xprt) + break; + xprt_force_disconnect(task->tk_xprt); + goto out_retry; + case NFS4ERR_BADSESSION: + case NFS4ERR_BADSLOT: + case NFS4ERR_BAD_HIGH_SLOT: + case NFS4ERR_DEADSESSION: + case NFS4ERR_CONN_NOT_BOUND_TO_SESSION: + case NFS4ERR_SEQ_FALSE_RETRY: + case NFS4ERR_SEQ_MISORDERED: + dprintk("%s ERROR %d, Reset session. Exchangeid " + "flags 0x%x\n", __func__, task->tk_status, + clp->cl_exchange_flags); + nfs4_schedule_session_recovery(clp->cl_session, task->tk_status); + goto out_retry; + case NFS4ERR_DELAY: + nfs_inc_stats(lseg->pls_layout->plh_inode, NFSIOS_DELAY); + fallthrough; + case NFS4ERR_GRACE: + rpc_delay(task, FF_LAYOUT_POLL_RETRY_MAX); + goto out_retry; + case NFS4ERR_RETRY_UNCACHED_REP: + goto out_retry; + /* Invalidate Layout errors */ + case NFS4ERR_PNFS_NO_LAYOUT: + case NFS4ERR_STALE: + case NFS4ERR_BADHANDLE: + case NFS4ERR_ISDIR: + case NFS4ERR_FHEXPIRED: + case NFS4ERR_WRONG_TYPE: + dprintk("%s Invalid layout error %d\n", __func__, + task->tk_status); + /* + * Destroy layout so new i/o will get a new layout. + * Layout will not be destroyed until all current lseg + * references are put. Mark layout as invalid to resend failed + * i/o and all i/o waiting on the slot table to the MDS until + * layout is destroyed and a new valid layout is obtained. + */ + pnfs_destroy_layout(NFS_I(inode)); + rpc_wake_up(&tbl->slot_tbl_waitq); + goto reset; + default: + break; + } + + switch (task->tk_status) { + /* RPC connection errors */ + case -ENETDOWN: + case -ENETUNREACH: + if (test_bit(NFS_CS_NETUNREACH_FATAL, &clp->cl_flags)) + return -NFS4ERR_FATAL_IOERROR; + fallthrough; + case -ECONNREFUSED: + case -EHOSTDOWN: + case -EHOSTUNREACH: + case -EIO: + case -ETIMEDOUT: + case -EPIPE: + case -EPROTO: + case -ENODEV: + dprintk("%s DS connection error %d\n", __func__, + task->tk_status); + nfs4_delete_deviceid(devid->ld, devid->nfs_client, + &devid->deviceid); + rpc_wake_up(&tbl->slot_tbl_waitq); + break; + default: + break; + } + + if (ff_layout_avoid_mds_available_ds(lseg)) + return -NFS4ERR_RESET_TO_PNFS; +reset: + dprintk("%s Retry through MDS. Error %d\n", __func__, + task->tk_status); + return -NFS4ERR_RESET_TO_MDS; + +out_retry: + task->tk_status = 0; + return -EAGAIN; +} + +/* Retry all errors through either pNFS or MDS except for -EJUKEBOX */ +static int ff_layout_async_handle_error_v3(struct rpc_task *task, + u32 op_status, + struct nfs_client *clp, + struct pnfs_layout_segment *lseg, + u32 idx, u32 dss_id) +{ + struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx, dss_id); + + switch (op_status) { + case NFS_OK: + case NFSERR_NXIO: + break; + case NFSERR_PERM: + if (!task->tk_xprt) + break; + xprt_force_disconnect(task->tk_xprt); + goto out_retry; + case NFSERR_ACCES: + case NFSERR_BADHANDLE: + case NFSERR_FBIG: + case NFSERR_IO: + case NFSERR_NOSPC: + case NFSERR_ROFS: + case NFSERR_STALE: + goto out_reset_to_pnfs; + case NFSERR_JUKEBOX: + nfs_inc_stats(lseg->pls_layout->plh_inode, NFSIOS_DELAY); + goto out_retry; + default: + break; + } + + switch (task->tk_status) { + /* File access problems. Don't mark the device as unavailable */ + case -EACCES: + case -ESTALE: + case -EISDIR: + case -EBADHANDLE: + case -ELOOP: + case -ENOSPC: + break; + case -EJUKEBOX: + nfs_inc_stats(lseg->pls_layout->plh_inode, NFSIOS_DELAY); + goto out_retry; + case -ENETDOWN: + case -ENETUNREACH: + if (test_bit(NFS_CS_NETUNREACH_FATAL, &clp->cl_flags)) + return -NFS4ERR_FATAL_IOERROR; + fallthrough; + default: + dprintk("%s DS connection error %d\n", __func__, + task->tk_status); + nfs4_delete_deviceid(devid->ld, devid->nfs_client, + &devid->deviceid); + } +out_reset_to_pnfs: + /* FIXME: Need to prevent infinite looping here. */ + return -NFS4ERR_RESET_TO_PNFS; +out_retry: + task->tk_status = 0; + rpc_restart_call_prepare(task); + rpc_delay(task, NFS_JUKEBOX_RETRY_TIME); + return -EAGAIN; +} + +static int ff_layout_async_handle_error(struct rpc_task *task, + u32 op_status, + struct nfs4_state *state, + struct nfs_client *clp, + struct pnfs_layout_segment *lseg, + u32 idx, u32 dss_id) +{ + int vers = clp->cl_nfs_mod->rpc_vers->number; + + if (task->tk_status >= 0) { + ff_layout_mark_ds_reachable(lseg, idx, dss_id); + return 0; + } + + /* Handle the case of an invalid layout segment */ + if (!pnfs_is_valid_lseg(lseg)) + return -NFS4ERR_RESET_TO_PNFS; + + switch (vers) { + case 3: + return ff_layout_async_handle_error_v3(task, op_status, clp, + lseg, idx, dss_id); + case 4: + return ff_layout_async_handle_error_v4(task, op_status, state, + clp, lseg, idx, dss_id); + default: + /* should never happen */ + WARN_ON_ONCE(1); + return 0; + } +} + +static void ff_layout_io_track_ds_error(struct pnfs_layout_segment *lseg, + u32 idx, u32 dss_id, u64 offset, u64 length, + u32 *op_status, int opnum, int error) +{ + struct nfs4_ff_layout_mirror *mirror; + u32 status = *op_status; + int err; + + if (status == 0) { + switch (error) { + case -ETIMEDOUT: + case -EPFNOSUPPORT: + case -EPROTONOSUPPORT: + case -EOPNOTSUPP: + case -EINVAL: + case -ECONNREFUSED: + case -ECONNRESET: + case -EHOSTDOWN: + case -EHOSTUNREACH: + case -ENETDOWN: + case -ENETUNREACH: + case -EADDRINUSE: + case -ENOBUFS: + case -EPIPE: + case -EPERM: + case -EPROTO: + case -ENODEV: + *op_status = status = NFS4ERR_NXIO; + break; + case -EACCES: + *op_status = status = NFS4ERR_ACCESS; + break; + default: + return; + } + } + + mirror = FF_LAYOUT_COMP(lseg, idx); + err = ff_layout_track_ds_error(FF_LAYOUT_FROM_HDR(lseg->pls_layout), + mirror, dss_id, offset, length, status, opnum, + nfs_io_gfp_mask()); + + switch (status) { + case NFS4ERR_DELAY: + case NFS4ERR_GRACE: + case NFS4ERR_PERM: + break; + case NFS4ERR_NXIO: + ff_layout_mark_ds_unreachable(lseg, idx, dss_id); + /* + * Don't return the layout if this is a read and we still + * have layouts to try + */ + if (opnum == OP_READ) + break; + fallthrough; + default: + pnfs_error_mark_layout_for_return(lseg->pls_layout->plh_inode, + lseg); + } + + dprintk("%s: err %d op %d status %u\n", __func__, err, opnum, status); +} + +/* NFS_PROTO call done callback routines */ +static int ff_layout_read_done_cb(struct rpc_task *task, + struct nfs_pgio_header *hdr) +{ + struct nfs4_ff_layout_segment *flseg = FF_LAYOUT_LSEG(hdr->lseg); + u32 dss_id = nfs4_ff_layout_calc_dss_id( + flseg->stripe_unit, + flseg->mirror_array[hdr->pgio_mirror_idx]->dss_count, + hdr->args.offset); + int err; + + if (task->tk_status < 0) { + ff_layout_io_track_ds_error(hdr->lseg, + hdr->pgio_mirror_idx, dss_id, + hdr->args.offset, hdr->args.count, + &hdr->res.op_status, OP_READ, + task->tk_status); + trace_ff_layout_read_error(hdr, task->tk_status); + } + + err = ff_layout_async_handle_error(task, hdr->res.op_status, + hdr->args.context->state, + hdr->ds_clp, hdr->lseg, + hdr->pgio_mirror_idx, + dss_id); + + trace_nfs4_pnfs_read(hdr, err); + clear_bit(NFS_IOHDR_RESEND_PNFS, &hdr->flags); + clear_bit(NFS_IOHDR_RESEND_MDS, &hdr->flags); + switch (err) { + case -NFS4ERR_RESET_TO_PNFS: + set_bit(NFS_IOHDR_RESEND_PNFS, &hdr->flags); + return task->tk_status; + case -NFS4ERR_RESET_TO_MDS: + set_bit(NFS_IOHDR_RESEND_MDS, &hdr->flags); + return task->tk_status; + case -EAGAIN: + goto out_eagain; + case -NFS4ERR_FATAL_IOERROR: + task->tk_status = -EIO; + return 0; + } + + return 0; +out_eagain: + rpc_restart_call_prepare(task); + return -EAGAIN; +} + +static bool +ff_layout_need_layoutcommit(struct pnfs_layout_segment *lseg) +{ + return !(FF_LAYOUT_LSEG(lseg)->flags & FF_FLAGS_NO_LAYOUTCOMMIT); +} + +/* + * We reference the rpc_cred of the first WRITE that triggers the need for + * a LAYOUTCOMMIT, and use it to send the layoutcommit compound. + * rfc5661 is not clear about which credential should be used. + * + * Flexlayout client should treat DS replied FILE_SYNC as DATA_SYNC, so + * to follow http://www.rfc-editor.org/errata_search.php?rfc=5661&eid=2751 + * we always send layoutcommit after DS writes. + */ +static void +ff_layout_set_layoutcommit(struct inode *inode, + struct pnfs_layout_segment *lseg, + loff_t end_offset) +{ + if (!ff_layout_need_layoutcommit(lseg)) + return; + + pnfs_set_layoutcommit(inode, lseg, end_offset); + dprintk("%s inode %llu pls_end_pos %llu\n", __func__, inode->i_ino, + (unsigned long long) NFS_I(inode)->layout->plh_lwb); +} + +static void ff_layout_read_record_layoutstats_start(struct rpc_task *task, + struct nfs_pgio_header *hdr) +{ + struct nfs4_ff_layout_mirror *mirror; + u32 dss_id; + + if (test_and_set_bit(NFS_IOHDR_STAT, &hdr->flags)) + return; + + mirror = FF_LAYOUT_COMP(hdr->lseg, hdr->pgio_mirror_idx); + dss_id = nfs4_ff_layout_calc_dss_id( + FF_LAYOUT_LSEG(hdr->lseg)->stripe_unit, + mirror->dss_count, + hdr->args.offset); + + nfs4_ff_layout_stat_io_start_read( + hdr->inode, + mirror, + dss_id, + hdr->args.count, + task->tk_start); +} + +static void ff_layout_read_record_layoutstats_done(struct rpc_task *task, + struct nfs_pgio_header *hdr) +{ + struct nfs4_ff_layout_mirror *mirror; + u32 dss_id; + + if (!test_and_clear_bit(NFS_IOHDR_STAT, &hdr->flags)) + return; + + mirror = FF_LAYOUT_COMP(hdr->lseg, hdr->pgio_mirror_idx); + dss_id = nfs4_ff_layout_calc_dss_id( + FF_LAYOUT_LSEG(hdr->lseg)->stripe_unit, + mirror->dss_count, + hdr->args.offset); + + nfs4_ff_layout_stat_io_end_read( + task, + mirror, + dss_id, + hdr->args.count, + hdr->res.count); + set_bit(NFS_LSEG_LAYOUTRETURN, &hdr->lseg->pls_flags); +} + +static int ff_layout_read_prepare_common(struct rpc_task *task, + struct nfs_pgio_header *hdr) +{ + if (unlikely(test_bit(NFS_CONTEXT_BAD, &hdr->args.context->flags))) { + rpc_exit(task, -EIO); + return -EIO; + } + + if (!pnfs_is_valid_lseg(hdr->lseg)) { + rpc_exit(task, -EAGAIN); + return -EAGAIN; + } + + ff_layout_read_record_layoutstats_start(task, hdr); + return 0; +} + +/* + * Call ops for the async read/write cases + * In the case of dense layouts, the offset needs to be reset to its + * original value. + */ +static void ff_layout_read_prepare_v3(struct rpc_task *task, void *data) +{ + struct nfs_pgio_header *hdr = data; + + if (ff_layout_read_prepare_common(task, hdr)) + return; + + rpc_call_start(task); +} + +static void ff_layout_read_prepare_v4(struct rpc_task *task, void *data) +{ + struct nfs_pgio_header *hdr = data; + + if (nfs4_setup_sequence(hdr->ds_clp, + &hdr->args.seq_args, + &hdr->res.seq_res, + task)) + return; + + ff_layout_read_prepare_common(task, hdr); +} + +static void ff_layout_read_call_done(struct rpc_task *task, void *data) +{ + struct nfs_pgio_header *hdr = data; + + if (test_bit(NFS_IOHDR_REDO, &hdr->flags) && + task->tk_status == 0) { + nfs4_sequence_done(task, &hdr->res.seq_res); + return; + } + + /* Note this may cause RPC to be resent */ + hdr->mds_ops->rpc_call_done(task, hdr); +} + +static void ff_layout_read_count_stats(struct rpc_task *task, void *data) +{ + struct nfs_pgio_header *hdr = data; + + ff_layout_read_record_layoutstats_done(task, hdr); + rpc_count_iostats_metrics(task, + &NFS_CLIENT(hdr->inode)->cl_metrics[NFSPROC4_CLNT_READ]); +} + +static void ff_layout_read_release(void *data) +{ + struct nfs_pgio_header *hdr = data; + + ff_layout_read_record_layoutstats_done(&hdr->task, hdr); + if (test_bit(NFS_IOHDR_RESEND_PNFS, &hdr->flags)) + ff_layout_resend_pnfs_read(hdr); + else if (test_bit(NFS_IOHDR_RESEND_MDS, &hdr->flags)) + ff_layout_reset_read(hdr); + pnfs_generic_rw_release(data); +} + + +static int ff_layout_write_done_cb(struct rpc_task *task, + struct nfs_pgio_header *hdr) +{ + struct nfs4_ff_layout_segment *flseg = FF_LAYOUT_LSEG(hdr->lseg); + u32 dss_id = nfs4_ff_layout_calc_dss_id( + flseg->stripe_unit, + flseg->mirror_array[hdr->pgio_mirror_idx]->dss_count, + hdr->args.offset); + loff_t end_offs = 0; + int err; + + if (task->tk_status < 0) { + ff_layout_io_track_ds_error(hdr->lseg, + hdr->pgio_mirror_idx, dss_id, + hdr->args.offset, hdr->args.count, + &hdr->res.op_status, OP_WRITE, + task->tk_status); + trace_ff_layout_write_error(hdr, task->tk_status); + } + + err = ff_layout_async_handle_error(task, hdr->res.op_status, + hdr->args.context->state, + hdr->ds_clp, hdr->lseg, + hdr->pgio_mirror_idx, + dss_id); + + trace_nfs4_pnfs_write(hdr, err); + clear_bit(NFS_IOHDR_RESEND_PNFS, &hdr->flags); + clear_bit(NFS_IOHDR_RESEND_MDS, &hdr->flags); + switch (err) { + case -NFS4ERR_RESET_TO_PNFS: + set_bit(NFS_IOHDR_RESEND_PNFS, &hdr->flags); + return task->tk_status; + case -NFS4ERR_RESET_TO_MDS: + set_bit(NFS_IOHDR_RESEND_MDS, &hdr->flags); + return task->tk_status; + case -EAGAIN: + return -EAGAIN; + case -NFS4ERR_FATAL_IOERROR: + task->tk_status = -EIO; + return 0; + } + + if (hdr->res.verf->committed == NFS_FILE_SYNC || + hdr->res.verf->committed == NFS_DATA_SYNC) + end_offs = hdr->mds_offset + (loff_t)hdr->res.count; + + /* Note: if the write is unstable, don't set end_offs until commit */ + ff_layout_set_layoutcommit(hdr->inode, hdr->lseg, end_offs); + + /* zero out fattr since we don't care DS attr at all */ + hdr->fattr.valid = 0; + if (task->tk_status >= 0) + nfs_writeback_update_inode(hdr); + + return 0; +} + +static int ff_layout_commit_done_cb(struct rpc_task *task, + struct nfs_commit_data *data) +{ + int err; + u32 idx = calc_mirror_idx_from_commit(data->lseg, data->ds_commit_index); + u32 dss_id = calc_dss_id_from_commit(data->lseg, data->ds_commit_index); + + if (task->tk_status < 0) { + ff_layout_io_track_ds_error(data->lseg, idx, dss_id, + data->args.offset, data->args.count, + &data->res.op_status, OP_COMMIT, + task->tk_status); + trace_ff_layout_commit_error(data, task->tk_status); + } + + err = ff_layout_async_handle_error(task, data->res.op_status, + NULL, data->ds_clp, data->lseg, idx, + dss_id); + + trace_nfs4_pnfs_commit_ds(data, err); + switch (err) { + case -NFS4ERR_RESET_TO_PNFS: + pnfs_generic_prepare_to_resend_writes(data); + return -EAGAIN; + case -NFS4ERR_RESET_TO_MDS: + pnfs_generic_prepare_to_resend_writes(data); + return -EAGAIN; + case -EAGAIN: + rpc_restart_call_prepare(task); + return -EAGAIN; + case -NFS4ERR_FATAL_IOERROR: + task->tk_status = -EIO; + return 0; + } + + ff_layout_set_layoutcommit(data->inode, data->lseg, data->lwb); + return 0; +} + +static void ff_layout_write_record_layoutstats_start(struct rpc_task *task, + struct nfs_pgio_header *hdr) +{ + struct nfs4_ff_layout_mirror *mirror; + u32 dss_id; + + if (test_and_set_bit(NFS_IOHDR_STAT, &hdr->flags)) + return; + + mirror = FF_LAYOUT_COMP(hdr->lseg, hdr->pgio_mirror_idx); + dss_id = nfs4_ff_layout_calc_dss_id( + FF_LAYOUT_LSEG(hdr->lseg)->stripe_unit, + mirror->dss_count, + hdr->args.offset); + + nfs4_ff_layout_stat_io_start_write( + hdr->inode, + mirror, + dss_id, + hdr->args.count, + task->tk_start); +} + +static void ff_layout_write_record_layoutstats_done(struct rpc_task *task, + struct nfs_pgio_header *hdr) +{ + struct nfs4_ff_layout_mirror *mirror; + u32 dss_id; + + if (!test_and_clear_bit(NFS_IOHDR_STAT, &hdr->flags)) + return; + + mirror = FF_LAYOUT_COMP(hdr->lseg, hdr->pgio_mirror_idx); + dss_id = nfs4_ff_layout_calc_dss_id( + FF_LAYOUT_LSEG(hdr->lseg)->stripe_unit, + mirror->dss_count, + hdr->args.offset); + + nfs4_ff_layout_stat_io_end_write( + task, + mirror, + dss_id, + hdr->args.count, + hdr->res.count, + hdr->res.verf->committed); + set_bit(NFS_LSEG_LAYOUTRETURN, &hdr->lseg->pls_flags); +} + +static int ff_layout_write_prepare_common(struct rpc_task *task, + struct nfs_pgio_header *hdr) +{ + if (unlikely(test_bit(NFS_CONTEXT_BAD, &hdr->args.context->flags))) { + rpc_exit(task, -EIO); + return -EIO; + } + + if (!pnfs_is_valid_lseg(hdr->lseg)) { + rpc_exit(task, -EAGAIN); + return -EAGAIN; + } + + ff_layout_write_record_layoutstats_start(task, hdr); + return 0; +} + +static void ff_layout_write_prepare_v3(struct rpc_task *task, void *data) +{ + struct nfs_pgio_header *hdr = data; + + if (ff_layout_write_prepare_common(task, hdr)) + return; + + rpc_call_start(task); +} + +static void ff_layout_write_prepare_v4(struct rpc_task *task, void *data) +{ + struct nfs_pgio_header *hdr = data; + + if (nfs4_setup_sequence(hdr->ds_clp, + &hdr->args.seq_args, + &hdr->res.seq_res, + task)) + return; + + ff_layout_write_prepare_common(task, hdr); +} + +static void ff_layout_write_call_done(struct rpc_task *task, void *data) +{ + struct nfs_pgio_header *hdr = data; + + if (test_bit(NFS_IOHDR_REDO, &hdr->flags) && + task->tk_status == 0) { + nfs4_sequence_done(task, &hdr->res.seq_res); + return; + } + + /* Note this may cause RPC to be resent */ + hdr->mds_ops->rpc_call_done(task, hdr); +} + +static void ff_layout_write_count_stats(struct rpc_task *task, void *data) +{ + struct nfs_pgio_header *hdr = data; + + ff_layout_write_record_layoutstats_done(task, hdr); + rpc_count_iostats_metrics(task, + &NFS_CLIENT(hdr->inode)->cl_metrics[NFSPROC4_CLNT_WRITE]); +} + +static void ff_layout_write_release(void *data) +{ + struct nfs_pgio_header *hdr = data; + + ff_layout_write_record_layoutstats_done(&hdr->task, hdr); + if (test_bit(NFS_IOHDR_RESEND_PNFS, &hdr->flags)) { + ff_layout_send_layouterror(hdr->lseg); + ff_layout_reset_write(hdr, true); + } else if (test_bit(NFS_IOHDR_RESEND_MDS, &hdr->flags)) + ff_layout_reset_write(hdr, false); + pnfs_generic_rw_release(data); +} + +static void ff_layout_commit_record_layoutstats_start(struct rpc_task *task, + struct nfs_commit_data *cdata) +{ + u32 idx, dss_id; + + if (test_and_set_bit(NFS_IOHDR_STAT, &cdata->flags)) + return; + + idx = calc_mirror_idx_from_commit(cdata->lseg, cdata->ds_commit_index); + dss_id = calc_dss_id_from_commit(cdata->lseg, cdata->ds_commit_index); + nfs4_ff_layout_stat_io_start_write(cdata->inode, + FF_LAYOUT_COMP(cdata->lseg, idx), + dss_id, + 0, task->tk_start); +} + +static void ff_layout_commit_record_layoutstats_done(struct rpc_task *task, + struct nfs_commit_data *cdata) +{ + struct nfs_page *req; + __u64 count = 0; + u32 idx, dss_id; + + if (!test_and_clear_bit(NFS_IOHDR_STAT, &cdata->flags)) + return; + + if (task->tk_status == 0) { + list_for_each_entry(req, &cdata->pages, wb_list) + count += req->wb_bytes; + } + + idx = calc_mirror_idx_from_commit(cdata->lseg, cdata->ds_commit_index); + dss_id = calc_dss_id_from_commit(cdata->lseg, cdata->ds_commit_index); + nfs4_ff_layout_stat_io_end_write(task, + FF_LAYOUT_COMP(cdata->lseg, idx), + dss_id, + count, count, NFS_FILE_SYNC); + set_bit(NFS_LSEG_LAYOUTRETURN, &cdata->lseg->pls_flags); +} + +static int ff_layout_commit_prepare_common(struct rpc_task *task, + struct nfs_commit_data *cdata) +{ + if (!pnfs_is_valid_lseg(cdata->lseg)) { + rpc_exit(task, -EAGAIN); + return -EAGAIN; + } + + ff_layout_commit_record_layoutstats_start(task, cdata); + return 0; +} + +static void ff_layout_commit_prepare_v3(struct rpc_task *task, void *data) +{ + if (ff_layout_commit_prepare_common(task, data)) + return; + + rpc_call_start(task); +} + +static void ff_layout_commit_prepare_v4(struct rpc_task *task, void *data) +{ + struct nfs_commit_data *wdata = data; + + if (nfs4_setup_sequence(wdata->ds_clp, + &wdata->args.seq_args, + &wdata->res.seq_res, + task)) + return; + ff_layout_commit_prepare_common(task, data); +} + +static void ff_layout_commit_done(struct rpc_task *task, void *data) +{ + pnfs_generic_write_commit_done(task, data); +} + +static void ff_layout_commit_count_stats(struct rpc_task *task, void *data) +{ + struct nfs_commit_data *cdata = data; + + ff_layout_commit_record_layoutstats_done(task, cdata); + rpc_count_iostats_metrics(task, + &NFS_CLIENT(cdata->inode)->cl_metrics[NFSPROC4_CLNT_COMMIT]); +} + +static void ff_layout_commit_release(void *data) +{ + struct nfs_commit_data *cdata = data; + + ff_layout_commit_record_layoutstats_done(&cdata->task, cdata); + pnfs_generic_commit_release(data); +} + +static const struct rpc_call_ops ff_layout_read_call_ops_v3 = { + .rpc_call_prepare = ff_layout_read_prepare_v3, + .rpc_call_done = ff_layout_read_call_done, + .rpc_count_stats = ff_layout_read_count_stats, + .rpc_release = ff_layout_read_release, +}; + +static const struct rpc_call_ops ff_layout_read_call_ops_v4 = { + .rpc_call_prepare = ff_layout_read_prepare_v4, + .rpc_call_done = ff_layout_read_call_done, + .rpc_count_stats = ff_layout_read_count_stats, + .rpc_release = ff_layout_read_release, +}; + +static const struct rpc_call_ops ff_layout_write_call_ops_v3 = { + .rpc_call_prepare = ff_layout_write_prepare_v3, + .rpc_call_done = ff_layout_write_call_done, + .rpc_count_stats = ff_layout_write_count_stats, + .rpc_release = ff_layout_write_release, +}; + +static const struct rpc_call_ops ff_layout_write_call_ops_v4 = { + .rpc_call_prepare = ff_layout_write_prepare_v4, + .rpc_call_done = ff_layout_write_call_done, + .rpc_count_stats = ff_layout_write_count_stats, + .rpc_release = ff_layout_write_release, +}; + +static const struct rpc_call_ops ff_layout_commit_call_ops_v3 = { + .rpc_call_prepare = ff_layout_commit_prepare_v3, + .rpc_call_done = ff_layout_commit_done, + .rpc_count_stats = ff_layout_commit_count_stats, + .rpc_release = ff_layout_commit_release, +}; + +static const struct rpc_call_ops ff_layout_commit_call_ops_v4 = { + .rpc_call_prepare = ff_layout_commit_prepare_v4, + .rpc_call_done = ff_layout_commit_done, + .rpc_count_stats = ff_layout_commit_count_stats, + .rpc_release = ff_layout_commit_release, +}; + +static enum pnfs_try_status +ff_layout_read_pagelist(struct nfs_pgio_header *hdr) +{ + struct pnfs_layout_segment *lseg = hdr->lseg; + struct nfs4_pnfs_ds *ds; + struct rpc_clnt *ds_clnt; + struct nfsd_file *localio; + struct nfs4_ff_layout_mirror *mirror; + const struct cred *ds_cred; + loff_t offset = hdr->args.offset; + u32 idx = hdr->pgio_mirror_idx; + int vers; + struct nfs_fh *fh; + u32 dss_id; + bool ds_fatal_error = false; + + dprintk("--> %s ino %llu pgbase %u req %zu@%llu\n", + __func__, hdr->inode->i_ino, + hdr->args.pgbase, (size_t)hdr->args.count, offset); + + mirror = FF_LAYOUT_COMP(lseg, idx); + dss_id = nfs4_ff_layout_calc_dss_id( + FF_LAYOUT_LSEG(lseg)->stripe_unit, + mirror->dss_count, + offset); + ds = nfs4_ff_layout_prepare_ds(lseg, mirror, dss_id, false); + if (IS_ERR(ds)) { + ds_fatal_error = nfs_error_is_fatal(PTR_ERR(ds)); + goto out_failed; + } + + ds_clnt = nfs4_ff_find_or_create_ds_client(mirror, ds->ds_clp, + hdr->inode, dss_id); + if (IS_ERR(ds_clnt)) + goto out_failed; + + ds_cred = ff_layout_get_ds_cred(mirror, &lseg->pls_range, hdr->cred, dss_id); + if (!ds_cred) + goto out_failed; + + vers = nfs4_ff_layout_ds_version(mirror, dss_id); + + dprintk("%s USE DS: %s cl_count %d vers %d\n", __func__, + ds->ds_remotestr, refcount_read(&ds->ds_clp->cl_count), vers); + + hdr->pgio_done_cb = ff_layout_read_done_cb; + refcount_inc(&ds->ds_clp->cl_count); + hdr->ds_clp = ds->ds_clp; + fh = nfs4_ff_layout_select_ds_fh(mirror, dss_id); + if (fh) + hdr->args.fh = fh; + + nfs4_ff_layout_select_ds_stateid(mirror, dss_id, &hdr->args.stateid); + + /* + * Note that if we ever decide to split across DSes, + * then we may need to handle dense-like offsets. + */ + hdr->args.offset = offset; + hdr->mds_offset = offset; + + /* Start IO accounting for local read */ + localio = ff_local_open_fh(lseg, idx, dss_id, ds->ds_clp, ds_cred, fh, + FMODE_READ); + if (localio) { + hdr->task.tk_start = ktime_get(); + ff_layout_read_record_layoutstats_start(&hdr->task, hdr); + } + + /* Perform an asynchronous read to ds */ + nfs_initiate_pgio(ds_clnt, hdr, ds_cred, ds->ds_clp->rpc_ops, + vers == 3 ? &ff_layout_read_call_ops_v3 : + &ff_layout_read_call_ops_v4, + 0, RPC_TASK_SOFTCONN, localio); + put_cred(ds_cred); + return PNFS_ATTEMPTED; + +out_failed: + if (ff_layout_avoid_mds_available_ds(lseg) && !ds_fatal_error) + return PNFS_TRY_AGAIN; + if (ff_layout_no_fallback_to_mds(lseg)) { + /* + * FF_FLAGS_NO_IO_THRU_MDS: force fresh LAYOUTGET, + * never fall through to MDS I/O. + */ + pnfs_error_mark_layout_for_return(hdr->inode, lseg); + return PNFS_TRY_AGAIN; + } + trace_pnfs_mds_fallback_read_pagelist(hdr->inode, + hdr->args.offset, hdr->args.count, + IOMODE_READ, NFS_I(hdr->inode)->layout, lseg); + return PNFS_NOT_ATTEMPTED; +} + +/* Perform async writes. */ +static enum pnfs_try_status +ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync) +{ + struct pnfs_layout_segment *lseg = hdr->lseg; + struct nfs4_pnfs_ds *ds; + struct rpc_clnt *ds_clnt; + struct nfsd_file *localio; + struct nfs4_ff_layout_mirror *mirror; + const struct cred *ds_cred; + loff_t offset = hdr->args.offset; + int vers; + struct nfs_fh *fh; + u32 idx = hdr->pgio_mirror_idx; + u32 dss_id; + bool ds_fatal_error = false; + + mirror = FF_LAYOUT_COMP(lseg, idx); + dss_id = nfs4_ff_layout_calc_dss_id( + FF_LAYOUT_LSEG(lseg)->stripe_unit, + mirror->dss_count, + offset); + ds = nfs4_ff_layout_prepare_ds(lseg, mirror, dss_id, true); + if (IS_ERR(ds)) { + ds_fatal_error = nfs_error_is_fatal(PTR_ERR(ds)); + goto out_failed; + } + + ds_clnt = nfs4_ff_find_or_create_ds_client(mirror, ds->ds_clp, + hdr->inode, dss_id); + if (IS_ERR(ds_clnt)) + goto out_failed; + + ds_cred = ff_layout_get_ds_cred(mirror, &lseg->pls_range, hdr->cred, dss_id); + if (!ds_cred) + goto out_failed; + + vers = nfs4_ff_layout_ds_version(mirror, dss_id); + + dprintk("%s ino %llu sync %d req %zu@%llu DS: %s cl_count %d vers %d\n", + __func__, hdr->inode->i_ino, sync, (size_t) hdr->args.count, + offset, ds->ds_remotestr, refcount_read(&ds->ds_clp->cl_count), + vers); + + hdr->pgio_done_cb = ff_layout_write_done_cb; + refcount_inc(&ds->ds_clp->cl_count); + hdr->ds_clp = ds->ds_clp; + hdr->ds_commit_idx = calc_commit_idx(lseg, idx, dss_id); + fh = nfs4_ff_layout_select_ds_fh(mirror, dss_id); + if (fh) + hdr->args.fh = fh; + + nfs4_ff_layout_select_ds_stateid(mirror, dss_id, &hdr->args.stateid); + + /* + * Note that if we ever decide to split across DSes, + * then we may need to handle dense-like offsets. + */ + hdr->args.offset = offset; + + /* Start IO accounting for local write */ + localio = ff_local_open_fh(lseg, idx, dss_id, ds->ds_clp, ds_cred, fh, + FMODE_READ|FMODE_WRITE); + if (localio) { + hdr->task.tk_start = ktime_get(); + ff_layout_write_record_layoutstats_start(&hdr->task, hdr); + } + + /* Perform an asynchronous write */ + nfs_initiate_pgio(ds_clnt, hdr, ds_cred, ds->ds_clp->rpc_ops, + vers == 3 ? &ff_layout_write_call_ops_v3 : + &ff_layout_write_call_ops_v4, + sync, RPC_TASK_SOFTCONN, localio); + put_cred(ds_cred); + return PNFS_ATTEMPTED; + +out_failed: + if (ff_layout_avoid_mds_available_ds(lseg) && !ds_fatal_error) + return PNFS_TRY_AGAIN; + if (ff_layout_no_fallback_to_mds(lseg)) { + /* + * FF_FLAGS_NO_IO_THRU_MDS: force fresh LAYOUTGET, + * never fall through to MDS I/O. + */ + pnfs_error_mark_layout_for_return(hdr->inode, lseg); + return PNFS_TRY_AGAIN; + } + trace_pnfs_mds_fallback_write_pagelist(hdr->inode, + hdr->args.offset, hdr->args.count, + IOMODE_RW, NFS_I(hdr->inode)->layout, lseg); + return PNFS_NOT_ATTEMPTED; +} + +static struct nfs_fh * +select_ds_fh_from_commit(struct pnfs_layout_segment *lseg, u32 i, u32 dss_id) +{ + struct nfs4_ff_layout_segment *flseg = FF_LAYOUT_LSEG(lseg); + + /* FIXME: Assume that there is only one NFS version available + * for the DS. + */ + return &flseg->mirror_array[i]->dss[dss_id].fh_versions[0]; +} + +static int ff_layout_initiate_commit(struct nfs_commit_data *data, int how) +{ + struct pnfs_layout_segment *lseg = data->lseg; + struct nfs4_pnfs_ds *ds; + struct rpc_clnt *ds_clnt; + struct nfsd_file *localio; + struct nfs4_ff_layout_mirror *mirror; + const struct cred *ds_cred; + u32 idx, dss_id; + int vers, ret; + struct nfs_fh *fh; + + if (!lseg || !(pnfs_is_valid_lseg(lseg) || + test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags))) + goto out_err; + + idx = calc_mirror_idx_from_commit(lseg, data->ds_commit_index); + mirror = FF_LAYOUT_COMP(lseg, idx); + dss_id = calc_dss_id_from_commit(lseg, data->ds_commit_index); + ds = nfs4_ff_layout_prepare_ds(lseg, mirror, dss_id, true); + if (IS_ERR(ds)) + goto out_err; + + ds_clnt = nfs4_ff_find_or_create_ds_client(mirror, ds->ds_clp, + data->inode, dss_id); + if (IS_ERR(ds_clnt)) + goto out_err; + + ds_cred = ff_layout_get_ds_cred(mirror, &lseg->pls_range, data->cred, dss_id); + if (!ds_cred) + goto out_err; + + vers = nfs4_ff_layout_ds_version(mirror, dss_id); + + dprintk("%s ino %llu, how %d cl_count %d vers %d\n", __func__, + data->inode->i_ino, how, refcount_read(&ds->ds_clp->cl_count), + vers); + data->commit_done_cb = ff_layout_commit_done_cb; + data->cred = ds_cred; + refcount_inc(&ds->ds_clp->cl_count); + data->ds_clp = ds->ds_clp; + fh = select_ds_fh_from_commit(lseg, idx, dss_id); + if (fh) + data->args.fh = fh; + + /* Start IO accounting for local commit */ + localio = ff_local_open_fh(lseg, idx, dss_id, ds->ds_clp, ds_cred, fh, + FMODE_READ|FMODE_WRITE); + if (localio) { + data->task.tk_start = ktime_get(); + ff_layout_commit_record_layoutstats_start(&data->task, data); + } + + ret = nfs_initiate_commit(ds_clnt, data, ds->ds_clp->rpc_ops, + vers == 3 ? &ff_layout_commit_call_ops_v3 : + &ff_layout_commit_call_ops_v4, + how, RPC_TASK_SOFTCONN, localio); + put_cred(ds_cred); + return ret; +out_err: + pnfs_generic_prepare_to_resend_writes(data); + pnfs_generic_commit_release(data); + return -EAGAIN; +} + +static int +ff_layout_commit_pagelist(struct inode *inode, struct list_head *mds_pages, + int how, struct nfs_commit_info *cinfo) +{ + return pnfs_generic_commit_pagelist(inode, mds_pages, how, cinfo, + ff_layout_initiate_commit); +} + +static bool ff_layout_match_rw(const struct rpc_task *task, + const struct nfs_pgio_header *hdr, + const struct pnfs_layout_segment *lseg) +{ + return hdr->lseg == lseg; +} + +static bool ff_layout_match_commit(const struct rpc_task *task, + const struct nfs_commit_data *cdata, + const struct pnfs_layout_segment *lseg) +{ + return cdata->lseg == lseg; +} + +static bool ff_layout_match_io(const struct rpc_task *task, const void *data) +{ + const struct rpc_call_ops *ops = task->tk_ops; + + if (ops == &ff_layout_read_call_ops_v3 || + ops == &ff_layout_read_call_ops_v4 || + ops == &ff_layout_write_call_ops_v3 || + ops == &ff_layout_write_call_ops_v4) + return ff_layout_match_rw(task, task->tk_calldata, data); + if (ops == &ff_layout_commit_call_ops_v3 || + ops == &ff_layout_commit_call_ops_v4) + return ff_layout_match_commit(task, task->tk_calldata, data); + return false; +} + +static void ff_layout_cancel_io(struct pnfs_layout_segment *lseg) +{ + struct nfs4_ff_layout_segment *flseg = FF_LAYOUT_LSEG(lseg); + struct nfs4_ff_layout_mirror *mirror; + struct nfs4_ff_layout_ds *mirror_ds; + struct nfs4_pnfs_ds *ds; + struct nfs_client *ds_clp; + struct rpc_clnt *clnt; + u32 idx, dss_id; + + for (idx = 0; idx < flseg->mirror_array_cnt; idx++) { + mirror = flseg->mirror_array[idx]; + for (dss_id = 0; dss_id < mirror->dss_count; dss_id++) { + mirror_ds = mirror->dss[dss_id].mirror_ds; + if (IS_ERR_OR_NULL(mirror_ds)) + continue; + ds = mirror->dss[dss_id].mirror_ds->ds; + if (!ds) + continue; + ds_clp = ds->ds_clp; + if (!ds_clp) + continue; + clnt = ds_clp->cl_rpcclient; + if (!clnt) + continue; + if (!rpc_cancel_tasks(clnt, -EAGAIN, + ff_layout_match_io, lseg)) + continue; + rpc_clnt_disconnect(clnt); + } + } +} + +static struct pnfs_ds_commit_info * +ff_layout_get_ds_info(struct inode *inode) +{ + struct pnfs_layout_hdr *layout = NFS_I(inode)->layout; + + if (layout == NULL) + return NULL; + + return &FF_LAYOUT_FROM_HDR(layout)->commit_info; +} + +static void +ff_layout_setup_ds_info(struct pnfs_ds_commit_info *fl_cinfo, + struct pnfs_layout_segment *lseg) +{ + struct nfs4_ff_layout_segment *flseg = FF_LAYOUT_LSEG(lseg); + struct inode *inode = lseg->pls_layout->plh_inode; + struct pnfs_commit_array *array, *new; + u32 size = flseg->mirror_array_cnt * flseg->mirror_array[0]->dss_count; + + new = pnfs_alloc_commit_array(size, + nfs_io_gfp_mask()); + if (new) { + spin_lock(&inode->i_lock); + array = pnfs_add_commit_array(fl_cinfo, new, lseg); + spin_unlock(&inode->i_lock); + if (array != new) + pnfs_free_commit_array(new); + } +} + +static void +ff_layout_release_ds_info(struct pnfs_ds_commit_info *fl_cinfo, + struct inode *inode) +{ + spin_lock(&inode->i_lock); + pnfs_generic_ds_cinfo_destroy(fl_cinfo); + spin_unlock(&inode->i_lock); +} + +static void +ff_layout_free_deviceid_node(struct nfs4_deviceid_node *d) +{ + nfs4_ff_layout_free_deviceid(container_of(d, struct nfs4_ff_layout_ds, + id_node)); +} + +static int ff_layout_encode_ioerr(struct xdr_stream *xdr, + const struct nfs4_layoutreturn_args *args, + const struct nfs4_flexfile_layoutreturn_args *ff_args) +{ + __be32 *start; + + start = xdr_reserve_space(xdr, 4); + if (unlikely(!start)) + return -E2BIG; + + *start = cpu_to_be32(ff_args->num_errors); + /* This assume we always return _ALL_ layouts */ + return ff_layout_encode_ds_ioerr(xdr, &ff_args->errors); +} + +static void +ff_layout_encode_ff_iostat_head(struct xdr_stream *xdr, + const nfs4_stateid *stateid, + const struct nfs42_layoutstat_devinfo *devinfo) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, 8 + 8); + p = xdr_encode_hyper(p, devinfo->offset); + p = xdr_encode_hyper(p, devinfo->length); + encode_opaque_fixed(xdr, stateid->data, NFS4_STATEID_SIZE); + p = xdr_reserve_space(xdr, 4*8); + p = xdr_encode_hyper(p, devinfo->read_count); + p = xdr_encode_hyper(p, devinfo->read_bytes); + p = xdr_encode_hyper(p, devinfo->write_count); + p = xdr_encode_hyper(p, devinfo->write_bytes); + encode_opaque_fixed(xdr, devinfo->dev_id.data, NFS4_DEVICEID4_SIZE); +} + +static void +ff_layout_encode_ff_iostat(struct xdr_stream *xdr, + const nfs4_stateid *stateid, + const struct nfs42_layoutstat_devinfo *devinfo) +{ + ff_layout_encode_ff_iostat_head(xdr, stateid, devinfo); + ff_layout_encode_ff_layoutupdate(xdr, devinfo, + devinfo->ld_private.data); +} + +/* report nothing for now */ +static void ff_layout_encode_iostats_array(struct xdr_stream *xdr, + const struct nfs4_layoutreturn_args *args, + struct nfs4_flexfile_layoutreturn_args *ff_args) +{ + __be32 *p; + int i; + + p = xdr_reserve_space(xdr, 4); + *p = cpu_to_be32(ff_args->num_dev); + for (i = 0; i < ff_args->num_dev; i++) + ff_layout_encode_ff_iostat(xdr, + &args->layout->plh_stateid, + &ff_args->devinfo[i]); +} + +static void +ff_layout_free_iostats_array(struct nfs42_layoutstat_devinfo *devinfo, + unsigned int num_entries) +{ + unsigned int i; + + for (i = 0; i < num_entries; i++) { + if (!devinfo[i].ld_private.ops) + continue; + if (!devinfo[i].ld_private.ops->free) + continue; + devinfo[i].ld_private.ops->free(&devinfo[i].ld_private); + } +} + +static struct nfs4_deviceid_node * +ff_layout_alloc_deviceid_node(struct nfs_server *server, + struct pnfs_device *pdev, gfp_t gfp_flags) +{ + struct nfs4_ff_layout_ds *dsaddr; + + dsaddr = nfs4_ff_alloc_deviceid_node(server, pdev, gfp_flags); + if (!dsaddr) + return NULL; + return &dsaddr->id_node; +} + +static void +ff_layout_encode_layoutreturn(struct xdr_stream *xdr, + const void *voidargs, + const struct nfs4_xdr_opaque_data *ff_opaque) +{ + const struct nfs4_layoutreturn_args *args = voidargs; + struct nfs4_flexfile_layoutreturn_args *ff_args = ff_opaque->data; + struct xdr_buf tmp_buf = { + .head = { + [0] = { + .iov_base = page_address(ff_args->pages[0]), + }, + }, + .buflen = PAGE_SIZE, + }; + struct xdr_stream tmp_xdr; + __be32 *start; + + dprintk("%s: Begin\n", __func__); + + xdr_init_encode(&tmp_xdr, &tmp_buf, NULL, NULL); + + ff_layout_encode_ioerr(&tmp_xdr, args, ff_args); + ff_layout_encode_iostats_array(&tmp_xdr, args, ff_args); + + start = xdr_reserve_space(xdr, 4); + *start = cpu_to_be32(tmp_buf.len); + xdr_write_pages(xdr, ff_args->pages, 0, tmp_buf.len); + + dprintk("%s: Return\n", __func__); +} + +static void +ff_layout_free_layoutreturn(struct nfs4_xdr_opaque_data *args) +{ + struct nfs4_flexfile_layoutreturn_args *ff_args; + + if (!args->data) + return; + ff_args = args->data; + args->data = NULL; + + ff_layout_free_ds_ioerr(&ff_args->errors); + ff_layout_free_iostats_array(ff_args->devinfo, ff_args->num_dev); + + put_page(ff_args->pages[0]); + kfree(ff_args); +} + +static const struct nfs4_xdr_opaque_ops layoutreturn_ops = { + .encode = ff_layout_encode_layoutreturn, + .free = ff_layout_free_layoutreturn, +}; + +static int +ff_layout_prepare_layoutreturn(struct nfs4_layoutreturn_args *args) +{ + struct nfs4_flexfile_layoutreturn_args *ff_args; + struct nfs4_flexfile_layout *ff_layout = FF_LAYOUT_FROM_HDR(args->layout); + + ff_args = kmalloc_obj(*ff_args, nfs_io_gfp_mask()); + if (!ff_args) + goto out_nomem; + ff_args->pages[0] = alloc_page(nfs_io_gfp_mask()); + if (!ff_args->pages[0]) + goto out_nomem_free; + + INIT_LIST_HEAD(&ff_args->errors); + ff_args->num_errors = ff_layout_fetch_ds_ioerr(args->layout, + &args->range, &ff_args->errors, + FF_LAYOUTRETURN_MAXERR); + + spin_lock(&args->inode->i_lock); + ff_args->num_dev = ff_layout_mirror_prepare_stats( + &ff_layout->generic_hdr, &ff_args->devinfo[0], + ARRAY_SIZE(ff_args->devinfo), NFS4_FF_OP_LAYOUTRETURN); + spin_unlock(&args->inode->i_lock); + + args->ld_private->ops = &layoutreturn_ops; + args->ld_private->data = ff_args; + return 0; +out_nomem_free: + kfree(ff_args); +out_nomem: + return -ENOMEM; +} + +#ifdef CONFIG_NFS_V4_2 +void +ff_layout_send_layouterror(struct pnfs_layout_segment *lseg) +{ + struct pnfs_layout_hdr *lo = lseg->pls_layout; + struct nfs42_layout_error *errors; + LIST_HEAD(head); + + if (!nfs_server_capable(lo->plh_inode, NFS_CAP_LAYOUTERROR)) + return; + ff_layout_fetch_ds_ioerr(lo, &lseg->pls_range, &head, -1); + if (list_empty(&head)) + return; + + errors = kmalloc_objs(*errors, NFS42_LAYOUTERROR_MAX, nfs_io_gfp_mask()); + if (errors != NULL) { + const struct nfs4_ff_layout_ds_err *pos; + size_t n = 0; + + list_for_each_entry(pos, &head, list) { + errors[n].offset = pos->offset; + errors[n].length = pos->length; + nfs4_stateid_copy(&errors[n].stateid, &pos->stateid); + errors[n].errors[0].dev_id = pos->deviceid; + errors[n].errors[0].status = pos->status; + errors[n].errors[0].opnum = pos->opnum; + n++; + if (!list_is_last(&pos->list, &head) && + n < NFS42_LAYOUTERROR_MAX) + continue; + if (nfs42_proc_layouterror(lseg, errors, n) < 0) + break; + n = 0; + } + kfree(errors); + } + ff_layout_free_ds_ioerr(&head); +} +#else +void +ff_layout_send_layouterror(struct pnfs_layout_segment *lseg) +{ +} +#endif + +static int +ff_layout_ntop4(const struct sockaddr *sap, char *buf, const size_t buflen) +{ + const struct sockaddr_in *sin = (struct sockaddr_in *)sap; + + return snprintf(buf, buflen, "%pI4", &sin->sin_addr); +} + +static size_t +ff_layout_ntop6_noscopeid(const struct sockaddr *sap, char *buf, + const int buflen) +{ + const struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)sap; + const struct in6_addr *addr = &sin6->sin6_addr; + + /* + * RFC 4291, Section 2.2.2 + * + * Shorthanded ANY address + */ + if (ipv6_addr_any(addr)) + return snprintf(buf, buflen, "::"); + + /* + * RFC 4291, Section 2.2.2 + * + * Shorthanded loopback address + */ + if (ipv6_addr_loopback(addr)) + return snprintf(buf, buflen, "::1"); + + /* + * RFC 4291, Section 2.2.3 + * + * Special presentation address format for mapped v4 + * addresses. + */ + if (ipv6_addr_v4mapped(addr)) + return snprintf(buf, buflen, "::ffff:%pI4", + &addr->s6_addr32[3]); + + /* + * RFC 4291, Section 2.2.1 + */ + return snprintf(buf, buflen, "%pI6c", addr); +} + +/* Derived from rpc_sockaddr2uaddr */ +static void +ff_layout_encode_netaddr(struct xdr_stream *xdr, struct nfs4_pnfs_ds_addr *da) +{ + struct sockaddr *sap = (struct sockaddr *)&da->da_addr; + char portbuf[RPCBIND_MAXUADDRPLEN]; + char addrbuf[RPCBIND_MAXUADDRLEN]; + unsigned short port; + int len, netid_len; + __be32 *p; + + switch (sap->sa_family) { + case AF_INET: + if (ff_layout_ntop4(sap, addrbuf, sizeof(addrbuf)) == 0) + return; + port = ntohs(((struct sockaddr_in *)sap)->sin_port); + break; + case AF_INET6: + if (ff_layout_ntop6_noscopeid(sap, addrbuf, sizeof(addrbuf)) == 0) + return; + port = ntohs(((struct sockaddr_in6 *)sap)->sin6_port); + break; + default: + WARN_ON_ONCE(1); + return; + } + + snprintf(portbuf, sizeof(portbuf), ".%u.%u", port >> 8, port & 0xff); + len = strlcat(addrbuf, portbuf, sizeof(addrbuf)); + + netid_len = strlen(da->da_netid); + p = xdr_reserve_space(xdr, 4 + netid_len); + xdr_encode_opaque(p, da->da_netid, netid_len); + + p = xdr_reserve_space(xdr, 4 + len); + xdr_encode_opaque(p, addrbuf, len); +} + +static void +ff_layout_encode_nfstime(struct xdr_stream *xdr, + ktime_t t) +{ + struct timespec64 ts; + __be32 *p; + + p = xdr_reserve_space(xdr, 12); + ts = ktime_to_timespec64(t); + p = xdr_encode_hyper(p, ts.tv_sec); + *p++ = cpu_to_be32(ts.tv_nsec); +} + +static void +ff_layout_encode_io_latency(struct xdr_stream *xdr, + struct nfs4_ff_io_stat *stat) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, 5 * 8); + p = xdr_encode_hyper(p, stat->ops_requested); + p = xdr_encode_hyper(p, stat->bytes_requested); + p = xdr_encode_hyper(p, stat->ops_completed); + p = xdr_encode_hyper(p, stat->bytes_completed); + p = xdr_encode_hyper(p, stat->bytes_not_delivered); + ff_layout_encode_nfstime(xdr, stat->total_busy_time); + ff_layout_encode_nfstime(xdr, stat->aggregate_completion_time); +} + +static void +ff_layout_encode_ff_layoutupdate(struct xdr_stream *xdr, + const struct nfs42_layoutstat_devinfo *devinfo, + struct nfs4_ff_layout_ds_stripe *dss_info) +{ + struct nfs4_pnfs_ds_addr *da; + struct nfs4_pnfs_ds *ds = dss_info->mirror_ds->ds; + struct nfs_fh *fh = &dss_info->fh_versions[0]; + __be32 *p; + + da = list_first_entry(&ds->ds_addrs, struct nfs4_pnfs_ds_addr, da_node); + dprintk("%s: DS %s: encoding address %s\n", + __func__, ds->ds_remotestr, da->da_remotestr); + /* netaddr4 */ + ff_layout_encode_netaddr(xdr, da); + /* nfs_fh4 */ + p = xdr_reserve_space(xdr, 4 + fh->size); + xdr_encode_opaque(p, fh->data, fh->size); + /* ff_io_latency4 read */ + spin_lock(&dss_info->mirror->lock); + ff_layout_encode_io_latency(xdr, + &dss_info->read_stat.io_stat); + /* ff_io_latency4 write */ + ff_layout_encode_io_latency(xdr, + &dss_info->write_stat.io_stat); + spin_unlock(&dss_info->mirror->lock); + /* nfstime4 */ + ff_layout_encode_nfstime(xdr, + ktime_sub(ktime_get(), + dss_info->start_time)); + /* bool */ + p = xdr_reserve_space(xdr, 4); + *p = cpu_to_be32(false); +} + +static void +ff_layout_encode_layoutstats(struct xdr_stream *xdr, const void *args, + const struct nfs4_xdr_opaque_data *opaque) +{ + struct nfs42_layoutstat_devinfo *devinfo = container_of(opaque, + struct nfs42_layoutstat_devinfo, ld_private); + __be32 *start; + + /* layoutupdate length */ + start = xdr_reserve_space(xdr, 4); + ff_layout_encode_ff_layoutupdate(xdr, devinfo, opaque->data); + + *start = cpu_to_be32((xdr->p - start - 1) * 4); +} + +static void +ff_layout_free_layoutstats(struct nfs4_xdr_opaque_data *opaque) +{ + struct nfs4_ff_layout_ds_stripe *dss_info = opaque->data; + struct nfs4_ff_layout_mirror *mirror = dss_info->mirror; + + ff_layout_put_mirror(mirror); +} + +static const struct nfs4_xdr_opaque_ops layoutstat_ops = { + .encode = ff_layout_encode_layoutstats, + .free = ff_layout_free_layoutstats, +}; + +static int +ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo, + struct nfs42_layoutstat_devinfo *devinfo, + int dev_limit, enum nfs4_ff_op_type type) +{ + struct nfs4_flexfile_layout *ff_layout = FF_LAYOUT_FROM_HDR(lo); + struct nfs4_ff_layout_mirror *mirror; + struct nfs4_ff_layout_ds_stripe *dss_info; + struct nfs4_deviceid_node *dev; + int i = 0, dss_id; + + list_for_each_entry(mirror, &ff_layout->mirrors, mirrors) { + for (dss_id = 0; dss_id < mirror->dss_count; ++dss_id) { + dss_info = &mirror->dss[dss_id]; + if (i >= dev_limit) + break; + if (IS_ERR_OR_NULL(dss_info->mirror_ds)) + continue; + if (!test_and_clear_bit(NFS4_FF_MIRROR_STAT_AVAIL, + &mirror->flags) && + type != NFS4_FF_OP_LAYOUTRETURN) + continue; + /* mirror refcount put in cleanup_layoutstats */ + if (!refcount_inc_not_zero(&mirror->ref)) + continue; + dev = &dss_info->mirror_ds->id_node; + memcpy(&devinfo->dev_id, + &dev->deviceid, + NFS4_DEVICEID4_SIZE); + devinfo->offset = 0; + devinfo->length = NFS4_MAX_UINT64; + spin_lock(&mirror->lock); + devinfo->read_count = + dss_info->read_stat.io_stat.ops_completed; + devinfo->read_bytes = + dss_info->read_stat.io_stat.bytes_completed; + devinfo->write_count = + dss_info->write_stat.io_stat.ops_completed; + devinfo->write_bytes = + dss_info->write_stat.io_stat.bytes_completed; + spin_unlock(&mirror->lock); + devinfo->layout_type = LAYOUT_FLEX_FILES; + devinfo->ld_private.ops = &layoutstat_ops; + devinfo->ld_private.data = &mirror->dss[dss_id]; + + devinfo++; + i++; + } + } + return i; +} + +static int ff_layout_prepare_layoutstats(struct nfs42_layoutstat_args *args) +{ + struct pnfs_layout_hdr *lo; + struct nfs4_flexfile_layout *ff_layout; + const int dev_count = PNFS_LAYOUTSTATS_MAXDEV; + + /* For now, send at most PNFS_LAYOUTSTATS_MAXDEV statistics */ + args->devinfo = kmalloc_objs(*args->devinfo, dev_count, + nfs_io_gfp_mask()); + if (!args->devinfo) + return -ENOMEM; + + spin_lock(&args->inode->i_lock); + lo = NFS_I(args->inode)->layout; + if (lo && pnfs_layout_is_valid(lo)) { + ff_layout = FF_LAYOUT_FROM_HDR(lo); + args->num_dev = ff_layout_mirror_prepare_stats( + &ff_layout->generic_hdr, &args->devinfo[0], dev_count, + NFS4_FF_OP_LAYOUTSTATS); + } else + args->num_dev = 0; + spin_unlock(&args->inode->i_lock); + if (!args->num_dev) { + kfree(args->devinfo); + args->devinfo = NULL; + return -ENOENT; + } + + return 0; +} + +static int +ff_layout_set_layoutdriver(struct nfs_server *server, + const struct nfs_fh *dummy) +{ +#if IS_ENABLED(CONFIG_NFS_V4_2) + server->caps |= NFS_CAP_LAYOUTSTATS | NFS_CAP_REBOOT_LAYOUTRETURN; +#endif + return 0; +} + +static const struct pnfs_commit_ops ff_layout_commit_ops = { + .setup_ds_info = ff_layout_setup_ds_info, + .release_ds_info = ff_layout_release_ds_info, + .mark_request_commit = pnfs_layout_mark_request_commit, + .clear_request_commit = pnfs_generic_clear_request_commit, + .scan_commit_lists = pnfs_generic_scan_commit_lists, + .recover_commit_reqs = pnfs_generic_recover_commit_reqs, + .commit_pagelist = ff_layout_commit_pagelist, +}; + +static struct pnfs_layoutdriver_type flexfilelayout_type = { + .id = LAYOUT_FLEX_FILES, + .name = "LAYOUT_FLEX_FILES", + .owner = THIS_MODULE, + .flags = PNFS_LAYOUTGET_ON_OPEN, + .max_layoutget_response = 4096, /* 1 page or so... */ + .set_layoutdriver = ff_layout_set_layoutdriver, + .alloc_layout_hdr = ff_layout_alloc_layout_hdr, + .free_layout_hdr = ff_layout_free_layout_hdr, + .alloc_lseg = ff_layout_alloc_lseg, + .free_lseg = ff_layout_free_lseg, + .add_lseg = ff_layout_add_lseg, + .pg_read_ops = &ff_layout_pg_read_ops, + .pg_write_ops = &ff_layout_pg_write_ops, + .get_ds_info = ff_layout_get_ds_info, + .free_deviceid_node = ff_layout_free_deviceid_node, + .read_pagelist = ff_layout_read_pagelist, + .write_pagelist = ff_layout_write_pagelist, + .alloc_deviceid_node = ff_layout_alloc_deviceid_node, + .prepare_layoutreturn = ff_layout_prepare_layoutreturn, + .sync = pnfs_nfs_generic_sync, + .prepare_layoutstats = ff_layout_prepare_layoutstats, + .cancel_io = ff_layout_cancel_io, +}; + +static int __init nfs4flexfilelayout_init(void) +{ + printk(KERN_INFO "%s: NFSv4 Flexfile Layout Driver Registering...\n", + __func__); + return pnfs_register_layoutdriver(&flexfilelayout_type); +} + +static void __exit nfs4flexfilelayout_exit(void) +{ + printk(KERN_INFO "%s: NFSv4 Flexfile Layout Driver Unregistering...\n", + __func__); + pnfs_unregister_layoutdriver(&flexfilelayout_type); +} + +MODULE_ALIAS("nfs-layouttype4-4"); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("The NFSv4 flexfile layout driver"); + +module_init(nfs4flexfilelayout_init); +module_exit(nfs4flexfilelayout_exit); + +module_param(io_maxretrans, ushort, 0644); +MODULE_PARM_DESC(io_maxretrans, "The number of times the NFSv4.1 client " + "retries an I/O request before returning an error. "); diff --git a/fs/nfs/flexfilelayout/flexfilelayout.h b/fs/nfs/flexfilelayout/flexfilelayout.h new file mode 100644 index 000000000..a5bd00f69 --- /dev/null +++ b/fs/nfs/flexfilelayout/flexfilelayout.h @@ -0,0 +1,269 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * NFSv4 flexfile layout driver data structures. + * + * Copyright (c) 2014, Primary Data, Inc. All rights reserved. + * + * Tao Peng + */ + +#ifndef FS_NFS_NFS4FLEXFILELAYOUT_H +#define FS_NFS_NFS4FLEXFILELAYOUT_H + +#define FF_FLAGS_NO_LAYOUTCOMMIT 1 +#define FF_FLAGS_NO_IO_THRU_MDS 2 +#define FF_FLAGS_NO_READ_IO 4 + +#include +#include "../pnfs.h" + +/* XXX: Let's filter out insanely large mirror count for now to avoid oom + * due to network error etc. */ +#define NFS4_FLEXFILE_LAYOUT_MAX_MIRROR_CNT 4096 + +#define NFS4_FLEXFILE_LAYOUT_MAX_STRIPE_CNT 4096 + +/* LAYOUTSTATS report interval in ms */ +#define FF_LAYOUTSTATS_REPORT_INTERVAL (60000L) +#define FF_LAYOUTSTATS_MAXDEV 4 + +struct nfs4_ff_ds_version { + u32 version; + u32 minor_version; + u32 rsize; + u32 wsize; + bool tightly_coupled; +}; + +/* chained in global deviceid hlist */ +struct nfs4_ff_layout_ds { + struct nfs4_deviceid_node id_node; + u32 ds_versions_cnt; + struct nfs4_ff_ds_version *ds_versions; + struct nfs4_pnfs_ds *ds; +}; + +struct nfs4_ff_layout_ds_err { + struct list_head list; /* linked in mirror error_list */ + u64 offset; + u64 length; + int status; + enum nfs_opnum4 opnum; + nfs4_stateid stateid; + struct nfs4_deviceid deviceid; +}; + +struct nfs4_ff_io_stat { + __u64 ops_requested; + __u64 bytes_requested; + __u64 ops_completed; + __u64 bytes_completed; + __u64 bytes_not_delivered; + ktime_t total_busy_time; + ktime_t aggregate_completion_time; +}; + +struct nfs4_ff_busy_timer { + ktime_t start_time; + atomic_t n_ops; +}; + +struct nfs4_ff_layoutstat { + struct nfs4_ff_io_stat io_stat; + struct nfs4_ff_busy_timer busy_timer; +}; + +struct nfs4_ff_layout_mirror; + +struct nfs4_ff_layout_ds_stripe { + struct nfs4_ff_layout_mirror *mirror; + struct nfs4_deviceid devid; + u32 efficiency; + struct nfs4_ff_layout_ds *mirror_ds; + u32 fh_versions_cnt; + struct nfs_fh *fh_versions; + nfs4_stateid stateid; + const struct cred __rcu *ro_cred; + const struct cred __rcu *rw_cred; + struct nfs_file_localio nfl; + struct nfs4_ff_layoutstat read_stat; + struct nfs4_ff_layoutstat write_stat; + ktime_t start_time; +}; + +struct nfs4_ff_layout_mirror { + struct pnfs_layout_hdr *layout; + struct list_head mirrors; + u32 dss_count; + struct nfs4_ff_layout_ds_stripe *dss; + refcount_t ref; + spinlock_t lock; + unsigned long flags; + u32 report_interval; +}; + +#define NFS4_FF_MIRROR_STAT_AVAIL (0) + +struct nfs4_ff_layout_segment { + struct pnfs_layout_segment generic_hdr; + u64 stripe_unit; + u32 flags; + u32 mirror_array_cnt; + struct nfs4_ff_layout_mirror *mirror_array[] __counted_by(mirror_array_cnt); +}; + +/* nfs4_flexfile_layout::flags bit indices */ +#define NFS4_FF_HDR_NO_IO_THRU_MDS 0 /* any lseg has had FF_FLAGS_NO_IO_THRU_MDS */ + +struct nfs4_flexfile_layout { + struct pnfs_layout_hdr generic_hdr; + struct pnfs_ds_commit_info commit_info; + struct list_head mirrors; + struct list_head error_list; /* nfs4_ff_layout_ds_err */ + ktime_t last_report_time; /* Layoutstat report times */ + unsigned long flags; +}; + +struct nfs4_flexfile_layoutreturn_args { + struct list_head errors; + struct nfs42_layoutstat_devinfo devinfo[FF_LAYOUTSTATS_MAXDEV]; + unsigned int num_errors; + unsigned int num_dev; + struct page *pages[1]; +}; + +static inline struct nfs4_flexfile_layout * +FF_LAYOUT_FROM_HDR(struct pnfs_layout_hdr *lo) +{ + return container_of(lo, struct nfs4_flexfile_layout, generic_hdr); +} + +static inline struct nfs4_ff_layout_segment * +FF_LAYOUT_LSEG(struct pnfs_layout_segment *lseg) +{ + return container_of(lseg, + struct nfs4_ff_layout_segment, + generic_hdr); +} + +static inline struct nfs4_ff_layout_ds * +FF_LAYOUT_MIRROR_DS(struct nfs4_deviceid_node *node) +{ + return container_of(node, struct nfs4_ff_layout_ds, id_node); +} + +static inline struct nfs4_ff_layout_mirror * +FF_LAYOUT_COMP(struct pnfs_layout_segment *lseg, u32 idx) +{ + struct nfs4_ff_layout_segment *fls = FF_LAYOUT_LSEG(lseg); + + if (idx < fls->mirror_array_cnt) + return fls->mirror_array[idx]; + return NULL; +} + +static inline struct nfs4_deviceid_node * +FF_LAYOUT_DEVID_NODE(struct pnfs_layout_segment *lseg, u32 idx, u32 dss_id) +{ + struct nfs4_ff_layout_mirror *mirror = FF_LAYOUT_COMP(lseg, idx); + + if (mirror != NULL) { + struct nfs4_ff_layout_ds *mirror_ds = mirror->dss[dss_id].mirror_ds; + + if (!IS_ERR_OR_NULL(mirror_ds)) + return &mirror_ds->id_node; + } + return NULL; +} + +static inline u32 +FF_LAYOUT_MIRROR_COUNT(struct pnfs_layout_segment *lseg) +{ + return FF_LAYOUT_LSEG(lseg)->mirror_array_cnt; +} + +static inline bool +ff_layout_no_fallback_to_mds(struct pnfs_layout_segment *lseg) +{ + return FF_LAYOUT_LSEG(lseg)->flags & FF_FLAGS_NO_IO_THRU_MDS; +} + +/* + * Sticky hdr-level mirror of FF_FLAGS_NO_IO_THRU_MDS so callers that have + * no current lseg (e.g. between LAYOUTRETURN and the next LAYOUTGET) can + * still honor the no-MDS-fallback policy. + */ +static inline bool +ff_layout_hdr_no_fallback_to_mds(struct pnfs_layout_hdr *lo) +{ + return test_bit(NFS4_FF_HDR_NO_IO_THRU_MDS, + &FF_LAYOUT_FROM_HDR(lo)->flags); +} + +static inline bool +ff_layout_no_read_on_rw(struct pnfs_layout_segment *lseg) +{ + return FF_LAYOUT_LSEG(lseg)->flags & FF_FLAGS_NO_READ_IO; +} + +static inline int +nfs4_ff_layout_ds_version(const struct nfs4_ff_layout_mirror *mirror, u32 dss_id) +{ + return mirror->dss[dss_id].mirror_ds->ds_versions[0].version; +} + +static inline u32 +nfs4_ff_layout_calc_dss_id(const u64 stripe_unit, const u32 dss_count, const loff_t offset) +{ + u64 tmp = offset; + + if (dss_count == 1 || stripe_unit == 0) + return 0; + + do_div(tmp, stripe_unit); + + return do_div(tmp, dss_count); +} + +struct nfs4_ff_layout_ds * +nfs4_ff_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev, + gfp_t gfp_flags); +void nfs4_ff_layout_put_deviceid(struct nfs4_ff_layout_ds *mirror_ds); +void nfs4_ff_layout_free_deviceid(struct nfs4_ff_layout_ds *mirror_ds); +int ff_layout_track_ds_error(struct nfs4_flexfile_layout *flo, + struct nfs4_ff_layout_mirror *mirror, + u32 dss_id, u64 offset, u64 length, int status, + enum nfs_opnum4 opnum, gfp_t gfp_flags); +void ff_layout_send_layouterror(struct pnfs_layout_segment *lseg); +int ff_layout_encode_ds_ioerr(struct xdr_stream *xdr, const struct list_head *head); +void ff_layout_free_ds_ioerr(struct list_head *head); +unsigned int ff_layout_fetch_ds_ioerr(struct pnfs_layout_hdr *lo, + const struct pnfs_layout_range *range, + struct list_head *head, + unsigned int maxnum); +struct nfs_fh * +nfs4_ff_layout_select_ds_fh(struct nfs4_ff_layout_mirror *mirror, u32 dss_id); +void +nfs4_ff_layout_select_ds_stateid(const struct nfs4_ff_layout_mirror *mirror, + u32 dss_id, + nfs4_stateid *stateid); + +struct nfs4_pnfs_ds * +nfs4_ff_layout_prepare_ds(struct pnfs_layout_segment *lseg, + struct nfs4_ff_layout_mirror *mirror, + u32 dss_id, + bool fail_return); + +struct rpc_clnt * +nfs4_ff_find_or_create_ds_client(struct nfs4_ff_layout_mirror *mirror, + struct nfs_client *ds_clp, + struct inode *inode, + u32 dss_id); +const struct cred *ff_layout_get_ds_cred(struct nfs4_ff_layout_mirror *mirror, + const struct pnfs_layout_range *range, + const struct cred *mdscred, + u32 dss_id); +bool ff_layout_avoid_mds_available_ds(struct pnfs_layout_segment *lseg); +bool ff_layout_avoid_read_on_rw(struct pnfs_layout_segment *lseg); + +#endif /* FS_NFS_NFS4FLEXFILELAYOUT_H */ diff --git a/fs/nfs/flexfilelayout/flexfilelayoutdev.c b/fs/nfs/flexfilelayout/flexfilelayoutdev.c new file mode 100644 index 000000000..1109462a9 --- /dev/null +++ b/fs/nfs/flexfilelayout/flexfilelayoutdev.c @@ -0,0 +1,644 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Device operations for the pnfs nfs4 file layout driver. + * + * Copyright (c) 2014, Primary Data, Inc. All rights reserved. + * + * Tao Peng + */ + +#include +#include +#include +#include + +#include "../internal.h" +#include "../nfs4session.h" +#include "flexfilelayout.h" + +#define NFSDBG_FACILITY NFSDBG_PNFS_LD + +static unsigned int dataserver_timeo = NFS_DEF_TCP_TIMEO; +static unsigned int dataserver_retrans; + +static bool ff_layout_has_available_ds(struct pnfs_layout_segment *lseg); + +void nfs4_ff_layout_put_deviceid(struct nfs4_ff_layout_ds *mirror_ds) +{ + if (!IS_ERR_OR_NULL(mirror_ds)) + nfs4_put_deviceid_node(&mirror_ds->id_node); +} + +void nfs4_ff_layout_free_deviceid(struct nfs4_ff_layout_ds *mirror_ds) +{ + nfs4_print_deviceid(&mirror_ds->id_node.deviceid); + nfs4_pnfs_ds_put(mirror_ds->ds); + kfree(mirror_ds->ds_versions); + kfree_rcu(mirror_ds, id_node.rcu); +} + +/* Decode opaque device data and construct new_ds using it */ +struct nfs4_ff_layout_ds * +nfs4_ff_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev, + gfp_t gfp_flags) +{ + struct xdr_stream stream; + struct xdr_buf buf; + struct folio *scratch; + struct list_head dsaddrs; + struct nfs4_pnfs_ds_addr *da; + struct nfs4_ff_layout_ds *new_ds = NULL; + struct nfs4_ff_ds_version *ds_versions = NULL; + struct net *net = server->nfs_client->cl_net; + u32 mp_count; + u32 version_count; + __be32 *p; + int i, ret = -ENOMEM; + + /* set up xdr stream */ + scratch = folio_alloc(gfp_flags, 0); + if (!scratch) + goto out_err; + + new_ds = kzalloc_obj(struct nfs4_ff_layout_ds, gfp_flags); + if (!new_ds) + goto out_scratch; + + nfs4_init_deviceid_node(&new_ds->id_node, + server, + &pdev->dev_id); + INIT_LIST_HEAD(&dsaddrs); + + xdr_init_decode_pages(&stream, &buf, pdev->pages, pdev->pglen); + xdr_set_scratch_folio(&stream, scratch); + + /* multipath count */ + p = xdr_inline_decode(&stream, 4); + if (unlikely(!p)) + goto out_err_drain_dsaddrs; + mp_count = be32_to_cpup(p); + dprintk("%s: multipath ds count %d\n", __func__, mp_count); + + for (i = 0; i < mp_count; i++) { + /* multipath ds */ + da = nfs4_decode_mp_ds_addr(net, &stream, gfp_flags); + if (da) + list_add_tail(&da->da_node, &dsaddrs); + } + if (list_empty(&dsaddrs)) { + dprintk("%s: no suitable DS addresses found\n", + __func__); + ret = -ENOMEDIUM; + goto out_err_drain_dsaddrs; + } + + /* version count */ + p = xdr_inline_decode(&stream, 4); + if (unlikely(!p)) + goto out_err_drain_dsaddrs; + version_count = be32_to_cpup(p); + + if (version_count == 0) { + ret = -EINVAL; + goto out_err_drain_dsaddrs; + } + dprintk("%s: version count %d\n", __func__, version_count); + + ds_versions = kzalloc_objs(struct nfs4_ff_ds_version, version_count, + gfp_flags); + if (!ds_versions) + goto out_err_drain_dsaddrs; + + for (i = 0; i < version_count; i++) { + /* 20 = version(4) + minor_version(4) + rsize(4) + wsize(4) + + * tightly_coupled(4) */ + p = xdr_inline_decode(&stream, 20); + if (unlikely(!p)) + goto out_err_drain_dsaddrs; + ds_versions[i].version = be32_to_cpup(p++); + ds_versions[i].minor_version = be32_to_cpup(p++); + ds_versions[i].rsize = nfs_io_size(be32_to_cpup(p++), + server->nfs_client->cl_proto); + ds_versions[i].wsize = nfs_io_size(be32_to_cpup(p++), + server->nfs_client->cl_proto); + ds_versions[i].tightly_coupled = be32_to_cpup(p); + + if (ds_versions[i].rsize > NFS_MAX_FILE_IO_SIZE) + ds_versions[i].rsize = NFS_MAX_FILE_IO_SIZE; + if (ds_versions[i].wsize > NFS_MAX_FILE_IO_SIZE) + ds_versions[i].wsize = NFS_MAX_FILE_IO_SIZE; + + /* + * check for valid major/minor combination. + * currently we support dataserver which talk: + * v3, v4.0, v4.1, v4.2 + */ + if (!((ds_versions[i].version == 3 && ds_versions[i].minor_version == 0) || + (ds_versions[i].version == 4 && ds_versions[i].minor_version < 3))) { + dprintk("%s: [%d] unsupported ds version %d-%d\n", __func__, + i, ds_versions[i].version, + ds_versions[i].minor_version); + ret = -EPROTONOSUPPORT; + goto out_err_drain_dsaddrs; + } + + dprintk("%s: [%d] vers %u minor_ver %u rsize %u wsize %u coupled %d\n", + __func__, i, ds_versions[i].version, + ds_versions[i].minor_version, + ds_versions[i].rsize, + ds_versions[i].wsize, + ds_versions[i].tightly_coupled); + } + + new_ds->ds_versions = ds_versions; + new_ds->ds_versions_cnt = version_count; + + new_ds->ds = nfs4_pnfs_ds_add(net, &dsaddrs, gfp_flags); + if (!new_ds->ds) + goto out_err_drain_dsaddrs; + + /* If DS was already in cache, free ds addrs */ + while (!list_empty(&dsaddrs)) { + da = list_first_entry(&dsaddrs, + struct nfs4_pnfs_ds_addr, + da_node); + list_del_init(&da->da_node); + kfree(da->da_remotestr); + kfree(da); + } + + folio_put(scratch); + return new_ds; + +out_err_drain_dsaddrs: + while (!list_empty(&dsaddrs)) { + da = list_first_entry(&dsaddrs, struct nfs4_pnfs_ds_addr, + da_node); + list_del_init(&da->da_node); + kfree(da->da_remotestr); + kfree(da); + } + + kfree(ds_versions); +out_scratch: + folio_put(scratch); +out_err: + kfree(new_ds); + + dprintk("%s ERROR: returning %d\n", __func__, ret); + return NULL; +} + +static void extend_ds_error(struct nfs4_ff_layout_ds_err *err, + u64 offset, u64 length) +{ + u64 end; + + end = max_t(u64, pnfs_end_offset(err->offset, err->length), + pnfs_end_offset(offset, length)); + err->offset = min_t(u64, err->offset, offset); + err->length = end - err->offset; +} + +static int +ff_ds_error_match(const struct nfs4_ff_layout_ds_err *e1, + const struct nfs4_ff_layout_ds_err *e2) +{ + int ret; + + if (e1->opnum != e2->opnum) + return e1->opnum < e2->opnum ? -1 : 1; + if (e1->status != e2->status) + return e1->status < e2->status ? -1 : 1; + ret = memcmp(e1->stateid.data, e2->stateid.data, + sizeof(e1->stateid.data)); + if (ret != 0) + return ret; + ret = memcmp(&e1->deviceid, &e2->deviceid, sizeof(e1->deviceid)); + if (ret != 0) + return ret; + if (pnfs_end_offset(e1->offset, e1->length) < e2->offset) + return -1; + if (e1->offset > pnfs_end_offset(e2->offset, e2->length)) + return 1; + /* If ranges overlap or are contiguous, they are the same */ + return 0; +} + +static void +ff_layout_add_ds_error_locked(struct nfs4_flexfile_layout *flo, + struct nfs4_ff_layout_ds_err *dserr) +{ + struct nfs4_ff_layout_ds_err *err, *tmp; + struct list_head *head = &flo->error_list; + int match; + + /* Do insertion sort w/ merges */ + list_for_each_entry_safe(err, tmp, &flo->error_list, list) { + match = ff_ds_error_match(err, dserr); + if (match < 0) + continue; + if (match > 0) { + /* Add entry "dserr" _before_ entry "err" */ + head = &err->list; + break; + } + /* Entries match, so merge "err" into "dserr" */ + extend_ds_error(dserr, err->offset, err->length); + list_replace(&err->list, &dserr->list); + kfree(err); + return; + } + + list_add_tail(&dserr->list, head); +} + +int ff_layout_track_ds_error(struct nfs4_flexfile_layout *flo, + struct nfs4_ff_layout_mirror *mirror, + u32 dss_id, u64 offset, u64 length, int status, + enum nfs_opnum4 opnum, gfp_t gfp_flags) +{ + struct nfs4_ff_layout_ds_err *dserr; + + if (status == 0) + return 0; + + if (IS_ERR_OR_NULL(mirror->dss[dss_id].mirror_ds)) + return -EINVAL; + + dserr = kmalloc_obj(*dserr, gfp_flags); + if (!dserr) + return -ENOMEM; + + INIT_LIST_HEAD(&dserr->list); + dserr->offset = offset; + dserr->length = length; + dserr->status = status; + dserr->opnum = opnum; + nfs4_stateid_copy(&dserr->stateid, &mirror->dss[dss_id].stateid); + memcpy(&dserr->deviceid, &mirror->dss[dss_id].mirror_ds->id_node.deviceid, + NFS4_DEVICEID4_SIZE); + + spin_lock(&flo->generic_hdr.plh_inode->i_lock); + ff_layout_add_ds_error_locked(flo, dserr); + spin_unlock(&flo->generic_hdr.plh_inode->i_lock); + return 0; +} + +static const struct cred * +ff_layout_get_mirror_cred(struct nfs4_ff_layout_mirror *mirror, u32 iomode, u32 dss_id) +{ + const struct cred *cred, __rcu **pcred; + + if (iomode == IOMODE_READ) + pcred = &mirror->dss[dss_id].ro_cred; + else + pcred = &mirror->dss[dss_id].rw_cred; + + rcu_read_lock(); + do { + cred = rcu_dereference(*pcred); + if (!cred) + break; + + cred = get_cred_rcu(cred); + } while(!cred); + rcu_read_unlock(); + return cred; +} + +struct nfs_fh * +nfs4_ff_layout_select_ds_fh(struct nfs4_ff_layout_mirror *mirror, u32 dss_id) +{ + /* FIXME: For now assume there is only 1 version available for the DS */ + return &mirror->dss[dss_id].fh_versions[0]; +} + +void +nfs4_ff_layout_select_ds_stateid(const struct nfs4_ff_layout_mirror *mirror, + u32 dss_id, + nfs4_stateid *stateid) +{ + if (nfs4_ff_layout_ds_version(mirror, dss_id) == 4) + nfs4_stateid_copy(stateid, &mirror->dss[dss_id].stateid); +} + +static bool +ff_layout_init_mirror_ds(struct pnfs_layout_hdr *lo, + struct nfs4_ff_layout_mirror *mirror, + u32 dss_id) +{ + if (mirror == NULL) + goto outerr; + if (mirror->dss[dss_id].mirror_ds == NULL) { + struct nfs4_deviceid_node *node; + struct nfs4_ff_layout_ds *mirror_ds = ERR_PTR(-ENODEV); + + node = nfs4_find_get_deviceid(NFS_SERVER(lo->plh_inode), + &mirror->dss[dss_id].devid, lo->plh_lc_cred, + GFP_KERNEL); + if (node) + mirror_ds = FF_LAYOUT_MIRROR_DS(node); + + /* check for race with another call to this function */ + if (cmpxchg(&mirror->dss[dss_id].mirror_ds, NULL, mirror_ds) && + mirror_ds != ERR_PTR(-ENODEV)) + nfs4_put_deviceid_node(node); + } + + if (IS_ERR(mirror->dss[dss_id].mirror_ds)) + goto outerr; + + return true; +outerr: + return false; +} + +/** + * nfs4_ff_layout_prepare_ds - prepare a DS connection for an RPC call + * @lseg: the layout segment we're operating on + * @mirror: layout mirror describing the DS to use + * @dss_id: DS stripe id to select stripe to use + * @fail_return: return layout on connect failure? + * + * Try to prepare a DS connection to accept an RPC call. This involves + * selecting a mirror to use and connecting the client to it if it's not + * already connected. + * + * Since we only need a single functioning mirror to satisfy a read, we don't + * want to return the layout if there is one. For writes though, any down + * mirror should result in a LAYOUTRETURN. @fail_return is how we distinguish + * between the two cases. + * + * Returns a pointer to a connected DS object on success or NULL on failure. + */ +struct nfs4_pnfs_ds * +nfs4_ff_layout_prepare_ds(struct pnfs_layout_segment *lseg, + struct nfs4_ff_layout_mirror *mirror, + u32 dss_id, + bool fail_return) +{ + struct nfs4_pnfs_ds *ds; + struct inode *ino = lseg->pls_layout->plh_inode; + struct nfs_server *s = NFS_SERVER(ino); + unsigned int max_payload; + int status = -EAGAIN; + + if (!ff_layout_init_mirror_ds(lseg->pls_layout, mirror, dss_id)) + goto noconnect; + + ds = mirror->dss[dss_id].mirror_ds->ds; + if (READ_ONCE(ds->ds_clp)) + goto out; + /* matching smp_wmb() in _nfs4_pnfs_v3/4_ds_connect */ + smp_rmb(); + + /* FIXME: For now we assume the server sent only one version of NFS + * to use for the DS. + */ + status = nfs4_pnfs_ds_connect(s, ds, &mirror->dss[dss_id].mirror_ds->id_node, + dataserver_timeo, dataserver_retrans, + mirror->dss[dss_id].mirror_ds->ds_versions[0].version, + mirror->dss[dss_id].mirror_ds->ds_versions[0].minor_version); + + /* connect success, check rsize/wsize limit */ + if (!status) { + /* + * ds_clp is put in destroy_ds(). + * keep ds_clp even if DS is local, so that if local IO cannot + * proceed somehow, we can fall back to NFS whenever we want. + */ + nfs_local_probe_async(ds->ds_clp); + max_payload = + nfs_block_size(rpc_max_payload(ds->ds_clp->cl_rpcclient), + NULL); + if (mirror->dss[dss_id].mirror_ds->ds_versions[0].rsize > max_payload) + mirror->dss[dss_id].mirror_ds->ds_versions[0].rsize = max_payload; + if (mirror->dss[dss_id].mirror_ds->ds_versions[0].wsize > max_payload) + mirror->dss[dss_id].mirror_ds->ds_versions[0].wsize = max_payload; + goto out; + } +noconnect: + ff_layout_track_ds_error(FF_LAYOUT_FROM_HDR(lseg->pls_layout), + mirror, dss_id, lseg->pls_range.offset, + lseg->pls_range.length, NFS4ERR_NXIO, + OP_ILLEGAL, GFP_NOIO); + ff_layout_send_layouterror(lseg); + if (fail_return || !ff_layout_has_available_ds(lseg)) + pnfs_error_mark_layout_for_return(ino, lseg); + ds = ERR_PTR(status); +out: + return ds; +} + +const struct cred * +ff_layout_get_ds_cred(struct nfs4_ff_layout_mirror *mirror, + const struct pnfs_layout_range *range, + const struct cred *mdscred, + u32 dss_id) +{ + const struct cred *cred; + + if (mirror && !mirror->dss[dss_id].mirror_ds->ds_versions[0].tightly_coupled) { + cred = ff_layout_get_mirror_cred(mirror, range->iomode, dss_id); + if (!cred) + cred = get_cred(mdscred); + } else { + cred = get_cred(mdscred); + } + return cred; +} + +/** + * nfs4_ff_find_or_create_ds_client - Find or create a DS rpc client + * @mirror: pointer to the mirror + * @ds_clp: nfs_client for the DS + * @inode: pointer to inode + * @dss_id: DS stripe id + * + * Find or create a DS rpc client with th MDS server rpc client auth flavor + * in the nfs_client cl_ds_clients list. + */ +struct rpc_clnt * +nfs4_ff_find_or_create_ds_client(struct nfs4_ff_layout_mirror *mirror, + struct nfs_client *ds_clp, struct inode *inode, + u32 dss_id) +{ + switch (mirror->dss[dss_id].mirror_ds->ds_versions[0].version) { + case 3: + /* For NFSv3 DS, flavor is set when creating DS connections */ + return ds_clp->cl_rpcclient; + case 4: + return nfs4_find_or_create_ds_client(ds_clp, inode); + default: + BUG(); + } +} + +void ff_layout_free_ds_ioerr(struct list_head *head) +{ + struct nfs4_ff_layout_ds_err *err; + + while (!list_empty(head)) { + err = list_first_entry(head, + struct nfs4_ff_layout_ds_err, + list); + list_del(&err->list); + kfree(err); + } +} + +/* called with inode i_lock held */ +int ff_layout_encode_ds_ioerr(struct xdr_stream *xdr, const struct list_head *head) +{ + struct nfs4_ff_layout_ds_err *err; + __be32 *p; + + list_for_each_entry(err, head, list) { + /* offset(8) + length(8) + stateid(NFS4_STATEID_SIZE) + * + array length + deviceid(NFS4_DEVICEID4_SIZE) + * + status(4) + opnum(4) + */ + p = xdr_reserve_space(xdr, + 28 + NFS4_STATEID_SIZE + NFS4_DEVICEID4_SIZE); + if (unlikely(!p)) + return -ENOBUFS; + p = xdr_encode_hyper(p, err->offset); + p = xdr_encode_hyper(p, err->length); + p = xdr_encode_opaque_fixed(p, &err->stateid, + NFS4_STATEID_SIZE); + /* Encode 1 error */ + *p++ = cpu_to_be32(1); + p = xdr_encode_opaque_fixed(p, &err->deviceid, + NFS4_DEVICEID4_SIZE); + *p++ = cpu_to_be32(err->status); + *p++ = cpu_to_be32(err->opnum); + dprintk("%s: offset %llu length %llu status %d op %d\n", + __func__, err->offset, err->length, err->status, + err->opnum); + } + + return 0; +} + +static +unsigned int do_layout_fetch_ds_ioerr(struct pnfs_layout_hdr *lo, + const struct pnfs_layout_range *range, + struct list_head *head, + unsigned int maxnum) +{ + struct nfs4_flexfile_layout *flo = FF_LAYOUT_FROM_HDR(lo); + struct inode *inode = lo->plh_inode; + struct nfs4_ff_layout_ds_err *err, *n; + unsigned int ret = 0; + + spin_lock(&inode->i_lock); + list_for_each_entry_safe(err, n, &flo->error_list, list) { + if (!pnfs_is_range_intersecting(err->offset, + pnfs_end_offset(err->offset, err->length), + range->offset, + pnfs_end_offset(range->offset, range->length))) + continue; + if (!maxnum) + break; + list_move(&err->list, head); + maxnum--; + ret++; + } + spin_unlock(&inode->i_lock); + return ret; +} + +unsigned int ff_layout_fetch_ds_ioerr(struct pnfs_layout_hdr *lo, + const struct pnfs_layout_range *range, + struct list_head *head, + unsigned int maxnum) +{ + unsigned int ret; + + ret = do_layout_fetch_ds_ioerr(lo, range, head, maxnum); + /* If we're over the max, discard all remaining entries */ + if (ret == maxnum) { + LIST_HEAD(discard); + do_layout_fetch_ds_ioerr(lo, range, &discard, -1); + ff_layout_free_ds_ioerr(&discard); + } + return ret; +} + +static bool ff_read_layout_has_available_ds(struct pnfs_layout_segment *lseg) +{ + struct nfs4_ff_layout_mirror *mirror; + struct nfs4_deviceid_node *devid; + u32 idx, dss_id; + + for (idx = 0; idx < FF_LAYOUT_MIRROR_COUNT(lseg); idx++) { + mirror = FF_LAYOUT_COMP(lseg, idx); + if (!mirror) + continue; + for (dss_id = 0; dss_id < mirror->dss_count; dss_id++) { + if (!mirror->dss[dss_id].mirror_ds) + return true; + if (IS_ERR(mirror->dss[dss_id].mirror_ds)) + continue; + devid = &mirror->dss[dss_id].mirror_ds->id_node; + if (!nfs4_test_deviceid_unavailable(devid)) + return true; + } + } + + return false; +} + +static bool ff_rw_layout_has_available_ds(struct pnfs_layout_segment *lseg) +{ + struct nfs4_ff_layout_mirror *mirror; + struct nfs4_deviceid_node *devid; + u32 idx, dss_id; + + for (idx = 0; idx < FF_LAYOUT_MIRROR_COUNT(lseg); idx++) { + mirror = FF_LAYOUT_COMP(lseg, idx); + if (!mirror) + return false; + for (dss_id = 0; dss_id < mirror->dss_count; dss_id++) { + if (IS_ERR(mirror->dss[dss_id].mirror_ds)) + return false; + if (!mirror->dss[dss_id].mirror_ds) + continue; + devid = &mirror->dss[dss_id].mirror_ds->id_node; + if (nfs4_test_deviceid_unavailable(devid)) + return false; + } + } + + return FF_LAYOUT_MIRROR_COUNT(lseg) != 0; +} + +static bool ff_layout_has_available_ds(struct pnfs_layout_segment *lseg) +{ + if (lseg->pls_range.iomode == IOMODE_READ) + return ff_read_layout_has_available_ds(lseg); + /* Note: RW layout needs all mirrors available */ + return ff_rw_layout_has_available_ds(lseg); +} + +bool ff_layout_avoid_mds_available_ds(struct pnfs_layout_segment *lseg) +{ + return ff_layout_no_fallback_to_mds(lseg) || + ff_layout_has_available_ds(lseg); +} + +bool ff_layout_avoid_read_on_rw(struct pnfs_layout_segment *lseg) +{ + return lseg->pls_range.iomode == IOMODE_RW && + ff_layout_no_read_on_rw(lseg); +} + +module_param(dataserver_retrans, uint, 0644); +MODULE_PARM_DESC(dataserver_retrans, "The number of times the NFSv4.1 client " + "retries a request before it attempts further " + " recovery action."); +module_param(dataserver_timeo, uint, 0644); +MODULE_PARM_DESC(dataserver_timeo, "The time (in tenths of a second) the " + "NFSv4.1 client waits for a response from a " + " data server before it retries an NFS request."); diff --git a/fs/nfs/fs_context.c b/fs/nfs/fs_context.c new file mode 100644 index 000000000..1967de7d1 --- /dev/null +++ b/fs/nfs/fs_context.c @@ -0,0 +1,1793 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/nfs/fs_context.c + * + * Copyright (C) 1992 Rick Sladkey + * Conversion to new mount api Copyright (C) David Howells + * + * NFS mount handling. + * + * Split from fs/nfs/super.c by David Howells + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "nfs.h" +#include "internal.h" + +#include "nfstrace.h" + +#define NFSDBG_FACILITY NFSDBG_MOUNT + +#if IS_ENABLED(CONFIG_NFS_V3) +#define NFS_DEFAULT_VERSION 3 +#else +#define NFS_DEFAULT_VERSION 2 +#endif + +#define NFS_MAX_CONNECTIONS 16 + +enum nfs_param { + Opt_ac, + Opt_acdirmax, + Opt_acdirmin, + Opt_acl, + Opt_acregmax, + Opt_acregmin, + Opt_actimeo, + Opt_addr, + Opt_bg, + Opt_bsize, + Opt_clientaddr, + Opt_cto, + Opt_alignwrite, + Opt_fatal_neterrors, + Opt_fg, + Opt_fscache, + Opt_fscache_flag, + Opt_hard, + Opt_intr, + Opt_local_lock, + Opt_lock, + Opt_lookupcache, + Opt_migration, + Opt_minorversion, + Opt_mountaddr, + Opt_mounthost, + Opt_mountport, + Opt_mountproto, + Opt_mountvers, + Opt_namelen, + Opt_nconnect, + Opt_max_connect, + Opt_port, + Opt_posix, + Opt_proto, + Opt_rdirplus, + Opt_rdirplus_none, + Opt_rdirplus_force, + Opt_rdma, + Opt_resvport, + Opt_retrans, + Opt_retry, + Opt_rsize, + Opt_sec, + Opt_sharecache, + Opt_sloppy, + Opt_soft, + Opt_softerr, + Opt_softreval, + Opt_source, + Opt_tcp, + Opt_timeo, + Opt_trunkdiscovery, + Opt_udp, + Opt_v, + Opt_vers, + Opt_wsize, + Opt_write, + Opt_xprtsec, + Opt_cert_serial, + Opt_privkey_serial, +}; + +enum { + Opt_fatal_neterrors_default, + Opt_fatal_neterrors_enetunreach, + Opt_fatal_neterrors_none, +}; + +static const struct constant_table nfs_param_enums_fatal_neterrors[] = { + { "default", Opt_fatal_neterrors_default }, + { "ENETDOWN:ENETUNREACH", Opt_fatal_neterrors_enetunreach }, + { "ENETUNREACH:ENETDOWN", Opt_fatal_neterrors_enetunreach }, + { "none", Opt_fatal_neterrors_none }, + {} +}; + +enum { + Opt_local_lock_all, + Opt_local_lock_flock, + Opt_local_lock_none, + Opt_local_lock_posix, +}; + +static const struct constant_table nfs_param_enums_local_lock[] = { + { "all", Opt_local_lock_all }, + { "flock", Opt_local_lock_flock }, + { "posix", Opt_local_lock_posix }, + { "none", Opt_local_lock_none }, + {} +}; + +enum { + Opt_lookupcache_all, + Opt_lookupcache_none, + Opt_lookupcache_positive, +}; + +static const struct constant_table nfs_param_enums_lookupcache[] = { + { "all", Opt_lookupcache_all }, + { "none", Opt_lookupcache_none }, + { "pos", Opt_lookupcache_positive }, + { "positive", Opt_lookupcache_positive }, + {} +}; + +enum { + Opt_write_lazy, + Opt_write_eager, + Opt_write_wait, +}; + +static const struct constant_table nfs_param_enums_write[] = { + { "lazy", Opt_write_lazy }, + { "eager", Opt_write_eager }, + { "wait", Opt_write_wait }, + {} +}; + +static const struct fs_parameter_spec nfs_fs_parameters[] = { + fsparam_flag_no("ac", Opt_ac), + fsparam_u32 ("acdirmax", Opt_acdirmax), + fsparam_u32 ("acdirmin", Opt_acdirmin), + fsparam_flag_no("acl", Opt_acl), + fsparam_u32 ("acregmax", Opt_acregmax), + fsparam_u32 ("acregmin", Opt_acregmin), + fsparam_u32 ("actimeo", Opt_actimeo), + fsparam_string("addr", Opt_addr), + fsparam_flag ("bg", Opt_bg), + fsparam_u32 ("bsize", Opt_bsize), + fsparam_string("clientaddr", Opt_clientaddr), + fsparam_flag_no("cto", Opt_cto), + fsparam_flag_no("alignwrite", Opt_alignwrite), + fsparam_enum("fatal_neterrors", Opt_fatal_neterrors, + nfs_param_enums_fatal_neterrors), + fsparam_flag ("fg", Opt_fg), + fsparam_flag_no("fsc", Opt_fscache_flag), + fsparam_string("fsc", Opt_fscache), + fsparam_flag ("hard", Opt_hard), + __fsparam(NULL, "intr", Opt_intr, + fs_param_neg_with_no|fs_param_deprecated, NULL), + fsparam_enum ("local_lock", Opt_local_lock, nfs_param_enums_local_lock), + fsparam_flag_no("lock", Opt_lock), + fsparam_enum ("lookupcache", Opt_lookupcache, nfs_param_enums_lookupcache), + fsparam_flag_no("migration", Opt_migration), + fsparam_u32 ("minorversion", Opt_minorversion), + fsparam_string("mountaddr", Opt_mountaddr), + fsparam_string("mounthost", Opt_mounthost), + fsparam_u32 ("mountport", Opt_mountport), + fsparam_string("mountproto", Opt_mountproto), + fsparam_u32 ("mountvers", Opt_mountvers), + fsparam_u32 ("namlen", Opt_namelen), + fsparam_u32 ("nconnect", Opt_nconnect), + fsparam_u32 ("max_connect", Opt_max_connect), + fsparam_string("nfsvers", Opt_vers), + fsparam_u32 ("port", Opt_port), + fsparam_flag_no("posix", Opt_posix), + fsparam_string("proto", Opt_proto), + fsparam_flag_no("rdirplus", Opt_rdirplus), // rdirplus|nordirplus + fsparam_string("rdirplus", Opt_rdirplus), // rdirplus=... + fsparam_flag ("rdma", Opt_rdma), + fsparam_flag_no("resvport", Opt_resvport), + fsparam_u32 ("retrans", Opt_retrans), + fsparam_string("retry", Opt_retry), + fsparam_u32 ("rsize", Opt_rsize), + fsparam_string("sec", Opt_sec), + fsparam_flag_no("sharecache", Opt_sharecache), + fsparam_flag ("sloppy", Opt_sloppy), + fsparam_flag ("soft", Opt_soft), + fsparam_flag ("softerr", Opt_softerr), + fsparam_flag ("softreval", Opt_softreval), + fsparam_string("source", Opt_source), + fsparam_flag ("tcp", Opt_tcp), + fsparam_u32 ("timeo", Opt_timeo), + fsparam_flag_no("trunkdiscovery", Opt_trunkdiscovery), + fsparam_flag ("udp", Opt_udp), + fsparam_flag ("v2", Opt_v), + fsparam_flag ("v3", Opt_v), + fsparam_flag ("v4", Opt_v), + fsparam_flag ("v4.0", Opt_v), + fsparam_flag ("v4.1", Opt_v), + fsparam_flag ("v4.2", Opt_v), + fsparam_string("vers", Opt_vers), + fsparam_enum ("write", Opt_write, nfs_param_enums_write), + fsparam_u32 ("wsize", Opt_wsize), + fsparam_string("xprtsec", Opt_xprtsec), + fsparam_s32("cert_serial", Opt_cert_serial), + fsparam_s32("privkey_serial", Opt_privkey_serial), + {} +}; + +enum { + Opt_vers_2, + Opt_vers_3, + Opt_vers_4, + Opt_vers_4_0, + Opt_vers_4_1, + Opt_vers_4_2, +}; + +static const struct constant_table nfs_vers_tokens[] = { + { "2", Opt_vers_2 }, + { "3", Opt_vers_3 }, + { "4", Opt_vers_4 }, + { "4.0", Opt_vers_4_0 }, + { "4.1", Opt_vers_4_1 }, + { "4.2", Opt_vers_4_2 }, + {} +}; + +enum { + Opt_xprt_rdma, + Opt_xprt_rdma6, + Opt_xprt_tcp, + Opt_xprt_tcp6, + Opt_xprt_udp, + Opt_xprt_udp6, + nr__Opt_xprt +}; + +static const struct constant_table nfs_xprt_protocol_tokens[] = { + { "rdma", Opt_xprt_rdma }, + { "rdma6", Opt_xprt_rdma6 }, + { "tcp", Opt_xprt_tcp }, + { "tcp6", Opt_xprt_tcp6 }, + { "udp", Opt_xprt_udp }, + { "udp6", Opt_xprt_udp6 }, + {} +}; + +enum { + Opt_sec_krb5, + Opt_sec_krb5i, + Opt_sec_krb5p, + Opt_sec_lkey, + Opt_sec_lkeyi, + Opt_sec_lkeyp, + Opt_sec_none, + Opt_sec_spkm, + Opt_sec_spkmi, + Opt_sec_spkmp, + Opt_sec_sys, + nr__Opt_sec +}; + +static const struct constant_table nfs_secflavor_tokens[] = { + { "krb5", Opt_sec_krb5 }, + { "krb5i", Opt_sec_krb5i }, + { "krb5p", Opt_sec_krb5p }, + { "lkey", Opt_sec_lkey }, + { "lkeyi", Opt_sec_lkeyi }, + { "lkeyp", Opt_sec_lkeyp }, + { "none", Opt_sec_none }, + { "null", Opt_sec_none }, + { "spkm3", Opt_sec_spkm }, + { "spkm3i", Opt_sec_spkmi }, + { "spkm3p", Opt_sec_spkmp }, + { "sys", Opt_sec_sys }, + {} +}; + +enum { + Opt_xprtsec_none, + Opt_xprtsec_tls, + Opt_xprtsec_mtls, + nr__Opt_xprtsec +}; + +static const struct constant_table nfs_xprtsec_policies[] = { + { "none", Opt_xprtsec_none }, + { "tls", Opt_xprtsec_tls }, + { "mtls", Opt_xprtsec_mtls }, + {} +}; + +static const struct constant_table nfs_rdirplus_tokens[] = { + { "none", Opt_rdirplus_none }, + { "force", Opt_rdirplus_force }, + {} +}; + +/* + * Sanity-check a server address provided by the mount command. + * + * Address family must be initialized, and address must not be + * the ANY address for that family. + */ +static int nfs_verify_server_address(struct sockaddr_storage *addr) +{ + switch (addr->ss_family) { + case AF_INET: { + struct sockaddr_in *sa = (struct sockaddr_in *)addr; + return sa->sin_addr.s_addr != htonl(INADDR_ANY); + } + case AF_INET6: { + struct in6_addr *sa = &((struct sockaddr_in6 *)addr)->sin6_addr; + return !ipv6_addr_any(sa); + } + } + + return 0; +} + +#ifdef CONFIG_NFS_DISABLE_UDP_SUPPORT +static bool nfs_server_transport_udp_invalid(const struct nfs_fs_context *ctx) +{ + return true; +} +#else +static bool nfs_server_transport_udp_invalid(const struct nfs_fs_context *ctx) +{ + if (ctx->version == 4) + return true; + return false; +} +#endif + +/* + * Sanity check the NFS transport protocol. + */ +static int nfs_validate_transport_protocol(struct fs_context *fc, + struct nfs_fs_context *ctx) +{ + switch (ctx->nfs_server.protocol) { + case XPRT_TRANSPORT_UDP: + if (nfs_server_transport_udp_invalid(ctx)) + goto out_invalid_transport_udp; + break; + case XPRT_TRANSPORT_TCP: + case XPRT_TRANSPORT_RDMA: + break; + default: + ctx->nfs_server.protocol = XPRT_TRANSPORT_TCP; + } + + if (ctx->xprtsec.policy != RPC_XPRTSEC_NONE) + switch (ctx->nfs_server.protocol) { + case XPRT_TRANSPORT_TCP: + ctx->nfs_server.protocol = XPRT_TRANSPORT_TCP_TLS; + break; + default: + goto out_invalid_xprtsec_policy; + } + + return 0; +out_invalid_transport_udp: + return nfs_invalf(fc, "NFS: Unsupported transport protocol udp"); +out_invalid_xprtsec_policy: + return nfs_invalf(fc, "NFS: Transport does not support xprtsec"); +} + +/* + * For text based NFSv2/v3 mounts, the mount protocol transport default + * settings should depend upon the specified NFS transport. + */ +static void nfs_set_mount_transport_protocol(struct nfs_fs_context *ctx) +{ + if (ctx->mount_server.protocol == XPRT_TRANSPORT_UDP || + ctx->mount_server.protocol == XPRT_TRANSPORT_TCP) + return; + switch (ctx->nfs_server.protocol) { + case XPRT_TRANSPORT_UDP: + ctx->mount_server.protocol = XPRT_TRANSPORT_UDP; + break; + case XPRT_TRANSPORT_TCP: + case XPRT_TRANSPORT_RDMA: + ctx->mount_server.protocol = XPRT_TRANSPORT_TCP; + } +} + +/* + * Add 'flavor' to 'auth_info' if not already present. + * Returns true if 'flavor' ends up in the list, false otherwise + */ +static int nfs_auth_info_add(struct fs_context *fc, + struct nfs_auth_info *auth_info, + rpc_authflavor_t flavor) +{ + unsigned int i; + unsigned int max_flavor_len = ARRAY_SIZE(auth_info->flavors); + + /* make sure this flavor isn't already in the list */ + for (i = 0; i < auth_info->flavor_len; i++) { + if (flavor == auth_info->flavors[i]) + return 0; + } + + if (auth_info->flavor_len + 1 >= max_flavor_len) + return nfs_invalf(fc, "NFS: too many sec= flavors"); + + auth_info->flavors[auth_info->flavor_len++] = flavor; + return 0; +} + +/* + * Parse the value of the 'sec=' option. + */ +static int nfs_parse_security_flavors(struct fs_context *fc, + struct fs_parameter *param) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + rpc_authflavor_t pseudoflavor; + char *string = param->string, *p; + int ret; + + trace_nfs_mount_assign(param->key, string); + + while ((p = strsep(&string, ":")) != NULL) { + if (!*p) + continue; + switch (lookup_constant(nfs_secflavor_tokens, p, -1)) { + case Opt_sec_none: + pseudoflavor = RPC_AUTH_NULL; + break; + case Opt_sec_sys: + pseudoflavor = RPC_AUTH_UNIX; + break; + case Opt_sec_krb5: + pseudoflavor = RPC_AUTH_GSS_KRB5; + break; + case Opt_sec_krb5i: + pseudoflavor = RPC_AUTH_GSS_KRB5I; + break; + case Opt_sec_krb5p: + pseudoflavor = RPC_AUTH_GSS_KRB5P; + break; + case Opt_sec_lkey: + pseudoflavor = RPC_AUTH_GSS_LKEY; + break; + case Opt_sec_lkeyi: + pseudoflavor = RPC_AUTH_GSS_LKEYI; + break; + case Opt_sec_lkeyp: + pseudoflavor = RPC_AUTH_GSS_LKEYP; + break; + case Opt_sec_spkm: + pseudoflavor = RPC_AUTH_GSS_SPKM; + break; + case Opt_sec_spkmi: + pseudoflavor = RPC_AUTH_GSS_SPKMI; + break; + case Opt_sec_spkmp: + pseudoflavor = RPC_AUTH_GSS_SPKMP; + break; + default: + return nfs_invalf(fc, "NFS: sec=%s option not recognized", p); + } + + ret = nfs_auth_info_add(fc, &ctx->auth_info, pseudoflavor); + if (ret < 0) + return ret; + } + + return 0; +} + +static int nfs_parse_xprtsec_policy(struct fs_context *fc, + struct fs_parameter *param) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + + trace_nfs_mount_assign(param->key, param->string); + + switch (lookup_constant(nfs_xprtsec_policies, param->string, -1)) { + case Opt_xprtsec_none: + ctx->xprtsec.policy = RPC_XPRTSEC_NONE; + break; + case Opt_xprtsec_tls: + ctx->xprtsec.policy = RPC_XPRTSEC_TLS_ANON; + break; + case Opt_xprtsec_mtls: + ctx->xprtsec.policy = RPC_XPRTSEC_TLS_X509; + break; + default: + return nfs_invalf(fc, "NFS: Unrecognized transport security policy"); + } + return 0; +} + +static int nfs_parse_version_string(struct fs_context *fc, + const char *string) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + + ctx->flags &= ~NFS_MOUNT_VER3; + switch (lookup_constant(nfs_vers_tokens, string, -1)) { + case Opt_vers_2: + ctx->version = 2; + break; + case Opt_vers_3: + ctx->flags |= NFS_MOUNT_VER3; + ctx->version = 3; + break; + case Opt_vers_4: + /* Backward compatibility option. In future, + * the mount program should always supply + * a NFSv4 minor version number. + */ + ctx->version = 4; + break; + case Opt_vers_4_0: + ctx->version = 4; + ctx->minorversion = 0; + break; + case Opt_vers_4_1: + ctx->version = 4; + ctx->minorversion = 1; + break; + case Opt_vers_4_2: + ctx->version = 4; + ctx->minorversion = 2; + break; + default: + return nfs_invalf(fc, "NFS: Unsupported NFS version"); + } + return 0; +} + +#ifdef CONFIG_KEYS +static int nfs_tls_key_verify(key_serial_t key_id) +{ + struct key *key = key_lookup(key_id); + int error = 0; + + if (IS_ERR(key)) { + pr_err("key id %08x not found\n", key_id); + return PTR_ERR(key); + } + if (test_bit(KEY_FLAG_REVOKED, &key->flags) || + test_bit(KEY_FLAG_INVALIDATED, &key->flags)) { + pr_err("key id %08x revoked\n", key_id); + error = -EKEYREVOKED; + } + + key_put(key); + return error; +} +#else +static inline int nfs_tls_key_verify(key_serial_t key_id) +{ + return -ENOENT; +} +#endif /* CONFIG_KEYS */ + +/* + * Parse a single mount parameter. + */ +static int nfs_fs_context_parse_param(struct fs_context *fc, + struct fs_parameter *param) +{ + struct fs_parse_result result; + struct nfs_fs_context *ctx = nfs_fc2context(fc); + unsigned short protofamily, mountfamily; + unsigned int len; + int ret, opt; + + trace_nfs_mount_option(param); + + opt = fs_parse(fc, nfs_fs_parameters, param, &result); + if (opt < 0) + return (opt == -ENOPARAM && ctx->sloppy) ? 1 : opt; + + if (fc->security) + ctx->has_sec_mnt_opts = 1; + + switch (opt) { + case Opt_source: + if (fc->source) + return nfs_invalf(fc, "NFS: Multiple sources not supported"); + fc->source = param->string; + param->string = NULL; + break; + + /* + * boolean options: foo/nofoo + */ + case Opt_soft: + ctx->flags |= NFS_MOUNT_SOFT; + ctx->flags &= ~NFS_MOUNT_SOFTERR; + break; + case Opt_softerr: + ctx->flags |= NFS_MOUNT_SOFTERR | NFS_MOUNT_SOFTREVAL; + ctx->flags &= ~NFS_MOUNT_SOFT; + break; + case Opt_hard: + ctx->flags &= ~(NFS_MOUNT_SOFT | + NFS_MOUNT_SOFTERR | + NFS_MOUNT_SOFTREVAL); + break; + case Opt_softreval: + if (result.negated) + ctx->flags &= ~NFS_MOUNT_SOFTREVAL; + else + ctx->flags |= NFS_MOUNT_SOFTREVAL; + break; + case Opt_posix: + if (result.negated) + ctx->flags &= ~NFS_MOUNT_POSIX; + else + ctx->flags |= NFS_MOUNT_POSIX; + break; + case Opt_cto: + if (result.negated) + ctx->flags |= NFS_MOUNT_NOCTO; + else + ctx->flags &= ~NFS_MOUNT_NOCTO; + break; + case Opt_trunkdiscovery: + if (result.negated) + ctx->flags &= ~NFS_MOUNT_TRUNK_DISCOVERY; + else + ctx->flags |= NFS_MOUNT_TRUNK_DISCOVERY; + break; + case Opt_alignwrite: + if (result.negated) + ctx->flags |= NFS_MOUNT_NO_ALIGNWRITE; + else + ctx->flags &= ~NFS_MOUNT_NO_ALIGNWRITE; + break; + case Opt_ac: + if (result.negated) + ctx->flags |= NFS_MOUNT_NOAC; + else + ctx->flags &= ~NFS_MOUNT_NOAC; + break; + case Opt_lock: + if (result.negated) { + ctx->lock_status = NFS_LOCK_NOLOCK; + ctx->flags |= NFS_MOUNT_NONLM; + ctx->flags |= (NFS_MOUNT_LOCAL_FLOCK | NFS_MOUNT_LOCAL_FCNTL); + } else { + ctx->lock_status = NFS_LOCK_LOCK; + ctx->flags &= ~NFS_MOUNT_NONLM; + ctx->flags &= ~(NFS_MOUNT_LOCAL_FLOCK | NFS_MOUNT_LOCAL_FCNTL); + } + break; + case Opt_udp: + ctx->flags &= ~NFS_MOUNT_TCP; + ctx->nfs_server.protocol = XPRT_TRANSPORT_UDP; + break; + case Opt_tcp: + case Opt_rdma: + ctx->flags |= NFS_MOUNT_TCP; /* for side protocols */ + ret = xprt_find_transport_ident(param->key); + if (ret < 0) + goto out_bad_transport; + ctx->nfs_server.protocol = ret; + break; + case Opt_acl: + if (result.negated) + ctx->flags |= NFS_MOUNT_NOACL; + else + ctx->flags &= ~NFS_MOUNT_NOACL; + break; + case Opt_rdirplus: + if (result.negated) { + ctx->flags &= ~NFS_MOUNT_FORCE_RDIRPLUS; + ctx->flags |= NFS_MOUNT_NORDIRPLUS; + } else if (!param->string) { + ctx->flags &= ~(NFS_MOUNT_NORDIRPLUS | NFS_MOUNT_FORCE_RDIRPLUS); + } else { + switch (lookup_constant(nfs_rdirplus_tokens, param->string, -1)) { + case Opt_rdirplus_none: + ctx->flags &= ~NFS_MOUNT_FORCE_RDIRPLUS; + ctx->flags |= NFS_MOUNT_NORDIRPLUS; + break; + case Opt_rdirplus_force: + ctx->flags &= ~NFS_MOUNT_NORDIRPLUS; + ctx->flags |= NFS_MOUNT_FORCE_RDIRPLUS; + break; + default: + goto out_invalid_value; + } + } + break; + case Opt_sharecache: + if (result.negated) + ctx->flags |= NFS_MOUNT_UNSHARED; + else + ctx->flags &= ~NFS_MOUNT_UNSHARED; + break; + case Opt_resvport: + if (result.negated) + ctx->flags |= NFS_MOUNT_NORESVPORT; + else + ctx->flags &= ~NFS_MOUNT_NORESVPORT; + break; + case Opt_fscache_flag: + if (result.negated) + ctx->options &= ~NFS_OPTION_FSCACHE; + else + ctx->options |= NFS_OPTION_FSCACHE; + kfree(ctx->fscache_uniq); + ctx->fscache_uniq = NULL; + break; + case Opt_fscache: + trace_nfs_mount_assign(param->key, param->string); + ctx->options |= NFS_OPTION_FSCACHE; + kfree(ctx->fscache_uniq); + ctx->fscache_uniq = param->string; + param->string = NULL; + break; + case Opt_migration: + if (result.negated) + ctx->options &= ~NFS_OPTION_MIGRATION; + else + ctx->options |= NFS_OPTION_MIGRATION; + break; + + /* + * options that take numeric values + */ + case Opt_port: + if (result.uint_32 > USHRT_MAX) + goto out_of_bounds; + ctx->nfs_server.port = result.uint_32; + break; + case Opt_rsize: + ctx->rsize = result.uint_32; + break; + case Opt_wsize: + ctx->wsize = result.uint_32; + break; + case Opt_bsize: + ctx->bsize = result.uint_32; + break; + case Opt_timeo: + if (result.uint_32 < 1 || result.uint_32 > INT_MAX) + goto out_of_bounds; + ctx->timeo = result.uint_32; + break; + case Opt_retrans: + if (result.uint_32 > INT_MAX) + goto out_of_bounds; + ctx->retrans = result.uint_32; + break; + case Opt_acregmin: + ctx->acregmin = result.uint_32; + break; + case Opt_acregmax: + ctx->acregmax = result.uint_32; + break; + case Opt_acdirmin: + ctx->acdirmin = result.uint_32; + break; + case Opt_acdirmax: + ctx->acdirmax = result.uint_32; + break; + case Opt_actimeo: + ctx->acregmin = result.uint_32; + ctx->acregmax = result.uint_32; + ctx->acdirmin = result.uint_32; + ctx->acdirmax = result.uint_32; + break; + case Opt_namelen: + ctx->namlen = result.uint_32; + break; + case Opt_mountport: + if (result.uint_32 > USHRT_MAX) + goto out_of_bounds; + ctx->mount_server.port = result.uint_32; + break; + case Opt_mountvers: + if (result.uint_32 < NFS_MNT_VERSION || + result.uint_32 > NFS_MNT3_VERSION) + goto out_of_bounds; + ctx->mount_server.version = result.uint_32; + break; + case Opt_minorversion: + if (result.uint_32 < NFS4_MIN_MINOR_VERSION || + result.uint_32 > NFS4_MAX_MINOR_VERSION) + goto out_of_bounds; + ctx->minorversion = result.uint_32; + break; + + /* + * options that take text values + */ + case Opt_v: + ret = nfs_parse_version_string(fc, param->key + 1); + if (ret < 0) + return ret; + break; + case Opt_vers: + if (!param->string) + goto out_invalid_value; + trace_nfs_mount_assign(param->key, param->string); + ret = nfs_parse_version_string(fc, param->string); + if (ret < 0) + return ret; + break; + case Opt_sec: + ret = nfs_parse_security_flavors(fc, param); + if (ret < 0) + return ret; + break; + case Opt_xprtsec: + ret = nfs_parse_xprtsec_policy(fc, param); + if (ret < 0) + return ret; + break; + case Opt_cert_serial: + ret = nfs_tls_key_verify(result.int_32); + if (ret < 0) + return ret; + ctx->xprtsec.cert_serial = result.int_32; + break; + case Opt_privkey_serial: + ret = nfs_tls_key_verify(result.int_32); + if (ret < 0) + return ret; + ctx->xprtsec.privkey_serial = result.int_32; + break; + + case Opt_proto: + if (!param->string) + goto out_invalid_value; + trace_nfs_mount_assign(param->key, param->string); + protofamily = AF_INET; + switch (lookup_constant(nfs_xprt_protocol_tokens, param->string, -1)) { + case Opt_xprt_udp6: + protofamily = AF_INET6; + fallthrough; + case Opt_xprt_udp: + ctx->flags &= ~NFS_MOUNT_TCP; + ctx->nfs_server.protocol = XPRT_TRANSPORT_UDP; + break; + case Opt_xprt_tcp6: + protofamily = AF_INET6; + fallthrough; + case Opt_xprt_tcp: + ctx->flags |= NFS_MOUNT_TCP; + ctx->nfs_server.protocol = XPRT_TRANSPORT_TCP; + break; + case Opt_xprt_rdma6: + protofamily = AF_INET6; + fallthrough; + case Opt_xprt_rdma: + /* vector side protocols to TCP */ + ctx->flags |= NFS_MOUNT_TCP; + ret = xprt_find_transport_ident(param->string); + if (ret < 0) + goto out_bad_transport; + ctx->nfs_server.protocol = ret; + break; + default: + goto out_bad_transport; + } + + ctx->protofamily = protofamily; + break; + + case Opt_mountproto: + if (!param->string) + goto out_invalid_value; + trace_nfs_mount_assign(param->key, param->string); + mountfamily = AF_INET; + switch (lookup_constant(nfs_xprt_protocol_tokens, param->string, -1)) { + case Opt_xprt_udp6: + mountfamily = AF_INET6; + fallthrough; + case Opt_xprt_udp: + ctx->mount_server.protocol = XPRT_TRANSPORT_UDP; + break; + case Opt_xprt_tcp6: + mountfamily = AF_INET6; + fallthrough; + case Opt_xprt_tcp: + ctx->mount_server.protocol = XPRT_TRANSPORT_TCP; + break; + case Opt_xprt_rdma: /* not used for side protocols */ + default: + goto out_bad_transport; + } + ctx->mountfamily = mountfamily; + break; + + case Opt_addr: + trace_nfs_mount_assign(param->key, param->string); + len = rpc_pton(fc->net_ns, param->string, param->size, + &ctx->nfs_server.address, + sizeof(ctx->nfs_server._address)); + if (len == 0) + goto out_invalid_address; + ctx->nfs_server.addrlen = len; + break; + case Opt_clientaddr: + trace_nfs_mount_assign(param->key, param->string); + kfree(ctx->client_address); + ctx->client_address = param->string; + param->string = NULL; + break; + case Opt_mounthost: + trace_nfs_mount_assign(param->key, param->string); + kfree(ctx->mount_server.hostname); + ctx->mount_server.hostname = param->string; + param->string = NULL; + break; + case Opt_mountaddr: + trace_nfs_mount_assign(param->key, param->string); + len = rpc_pton(fc->net_ns, param->string, param->size, + &ctx->mount_server.address, + sizeof(ctx->mount_server._address)); + if (len == 0) + goto out_invalid_address; + ctx->mount_server.addrlen = len; + break; + case Opt_nconnect: + trace_nfs_mount_assign(param->key, param->string); + if (result.uint_32 < 1 || result.uint_32 > NFS_MAX_CONNECTIONS) + goto out_of_bounds; + ctx->nfs_server.nconnect = result.uint_32; + break; + case Opt_max_connect: + trace_nfs_mount_assign(param->key, param->string); + if (result.uint_32 < 1 || result.uint_32 > NFS_MAX_TRANSPORTS) + goto out_of_bounds; + ctx->nfs_server.max_connect = result.uint_32; + break; + case Opt_fatal_neterrors: + trace_nfs_mount_assign(param->key, param->string); + switch (result.uint_32) { + case Opt_fatal_neterrors_default: + if (fc->net_ns != &init_net) + ctx->flags |= NFS_MOUNT_NETUNREACH_FATAL; + else + ctx->flags &= ~NFS_MOUNT_NETUNREACH_FATAL; + break; + case Opt_fatal_neterrors_enetunreach: + ctx->flags |= NFS_MOUNT_NETUNREACH_FATAL; + break; + case Opt_fatal_neterrors_none: + ctx->flags &= ~NFS_MOUNT_NETUNREACH_FATAL; + break; + default: + goto out_invalid_value; + } + break; + case Opt_lookupcache: + trace_nfs_mount_assign(param->key, param->string); + switch (result.uint_32) { + case Opt_lookupcache_all: + ctx->flags &= ~(NFS_MOUNT_LOOKUP_CACHE_NONEG|NFS_MOUNT_LOOKUP_CACHE_NONE); + break; + case Opt_lookupcache_positive: + ctx->flags &= ~NFS_MOUNT_LOOKUP_CACHE_NONE; + ctx->flags |= NFS_MOUNT_LOOKUP_CACHE_NONEG; + break; + case Opt_lookupcache_none: + ctx->flags |= NFS_MOUNT_LOOKUP_CACHE_NONEG|NFS_MOUNT_LOOKUP_CACHE_NONE; + break; + default: + goto out_invalid_value; + } + break; + case Opt_local_lock: + trace_nfs_mount_assign(param->key, param->string); + switch (result.uint_32) { + case Opt_local_lock_all: + ctx->flags |= (NFS_MOUNT_LOCAL_FLOCK | + NFS_MOUNT_LOCAL_FCNTL); + break; + case Opt_local_lock_flock: + ctx->flags |= NFS_MOUNT_LOCAL_FLOCK; + break; + case Opt_local_lock_posix: + ctx->flags |= NFS_MOUNT_LOCAL_FCNTL; + break; + case Opt_local_lock_none: + ctx->flags &= ~(NFS_MOUNT_LOCAL_FLOCK | + NFS_MOUNT_LOCAL_FCNTL); + break; + default: + goto out_invalid_value; + } + break; + case Opt_write: + trace_nfs_mount_assign(param->key, param->string); + switch (result.uint_32) { + case Opt_write_lazy: + ctx->flags &= + ~(NFS_MOUNT_WRITE_EAGER | NFS_MOUNT_WRITE_WAIT); + break; + case Opt_write_eager: + ctx->flags |= NFS_MOUNT_WRITE_EAGER; + ctx->flags &= ~NFS_MOUNT_WRITE_WAIT; + break; + case Opt_write_wait: + ctx->flags |= + NFS_MOUNT_WRITE_EAGER | NFS_MOUNT_WRITE_WAIT; + break; + default: + goto out_invalid_value; + } + break; + + /* + * Special options + */ + case Opt_sloppy: + ctx->sloppy = true; + break; + } + + return 0; + +out_invalid_value: + return nfs_invalf(fc, "NFS: Bad mount option value specified"); +out_invalid_address: + return nfs_invalf(fc, "NFS: Bad IP address specified"); +out_of_bounds: + return nfs_invalf(fc, "NFS: Value for '%s' out of range", param->key); +out_bad_transport: + return nfs_invalf(fc, "NFS: Unrecognized transport protocol"); +} + +/* + * Split fc->source into "hostname:export_path". + * + * The leftmost colon demarks the split between the server's hostname + * and the export path. If the hostname starts with a left square + * bracket, then it may contain colons. + * + * Note: caller frees hostname and export path, even on error. + */ +static int nfs_parse_source(struct fs_context *fc, + size_t maxnamlen, size_t maxpathlen) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + const char *dev_name = fc->source; + size_t len; + const char *end; + + if (unlikely(!dev_name || !*dev_name)) + return -EINVAL; + + /* Is the host name protected with square brakcets? */ + if (*dev_name == '[') { + end = strchr(++dev_name, ']'); + if (end == NULL || end[1] != ':') + goto out_bad_devname; + + len = end - dev_name; + end++; + } else { + const char *comma; + + end = strchr(dev_name, ':'); + if (end == NULL) + goto out_bad_devname; + len = end - dev_name; + + /* kill possible hostname list: not supported */ + comma = memchr(dev_name, ',', len); + if (comma) + len = comma - dev_name; + } + + if (len > maxnamlen) + goto out_hostname; + + kfree(ctx->nfs_server.hostname); + + /* N.B. caller will free nfs_server.hostname in all cases */ + ctx->nfs_server.hostname = kmemdup_nul(dev_name, len, GFP_KERNEL); + if (!ctx->nfs_server.hostname) + goto out_nomem; + len = strlen(++end); + if (len > maxpathlen) + goto out_path; + ctx->nfs_server.export_path = kmemdup_nul(end, len, GFP_KERNEL); + if (!ctx->nfs_server.export_path) + goto out_nomem; + + trace_nfs_mount_path(ctx->nfs_server.export_path); + return 0; + +out_bad_devname: + return nfs_invalf(fc, "NFS: device name not in host:path format"); +out_nomem: + nfs_errorf(fc, "NFS: not enough memory to parse device name"); + return -ENOMEM; +out_hostname: + nfs_errorf(fc, "NFS: server hostname too long"); + return -ENAMETOOLONG; +out_path: + nfs_errorf(fc, "NFS: export pathname too long"); + return -ENAMETOOLONG; +} + +static inline bool is_remount_fc(struct fs_context *fc) +{ + return fc->root != NULL; +} + +/* + * Parse monolithic NFS2/NFS3 mount data + * - fills in the mount root filehandle + * + * For option strings, user space handles the following behaviors: + * + * + DNS: mapping server host name to IP address ("addr=" option) + * + * + failure mode: how to behave if a mount request can't be handled + * immediately ("fg/bg" option) + * + * + retry: how often to retry a mount request ("retry=" option) + * + * + breaking back: trying proto=udp after proto=tcp, v2 after v3, + * mountproto=tcp after mountproto=udp, and so on + */ +static int nfs23_parse_monolithic(struct fs_context *fc, + struct nfs_mount_data *data) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + struct nfs_fh *mntfh = ctx->mntfh; + struct sockaddr_storage *sap = &ctx->nfs_server._address; + int extra_flags = NFS_MOUNT_LEGACY_INTERFACE; + int ret; + + if (data == NULL) + goto out_no_data; + + ctx->version = NFS_DEFAULT_VERSION; + switch (data->version) { + case 1: + data->namlen = 0; + fallthrough; + case 2: + data->bsize = 0; + fallthrough; + case 3: + if (data->flags & NFS_MOUNT_VER3) + goto out_no_v3; + data->root.size = NFS2_FHSIZE; + memcpy(data->root.data, data->old_root.data, NFS2_FHSIZE); + /* Turn off security negotiation */ + extra_flags |= NFS_MOUNT_SECFLAVOUR; + fallthrough; + case 4: + if (data->flags & NFS_MOUNT_SECFLAVOUR) + goto out_no_sec; + fallthrough; + case 5: + memset(data->context, 0, sizeof(data->context)); + fallthrough; + case 6: + if (data->flags & NFS_MOUNT_VER3) { + if (data->root.size > NFS3_FHSIZE || data->root.size == 0) + goto out_invalid_fh; + mntfh->size = data->root.size; + ctx->version = 3; + } else { + mntfh->size = NFS2_FHSIZE; + ctx->version = 2; + } + + + memcpy(mntfh->data, data->root.data, mntfh->size); + if (mntfh->size < sizeof(mntfh->data)) + memset(mntfh->data + mntfh->size, 0, + sizeof(mntfh->data) - mntfh->size); + + /* + * for proto == XPRT_TRANSPORT_UDP, which is what uses + * to_exponential, implying shift: limit the shift value + * to BITS_PER_LONG (majortimeo is unsigned long) + */ + if (!(data->flags & NFS_MOUNT_TCP)) /* this will be UDP */ + if (data->retrans >= 64) /* shift value is too large */ + goto out_invalid_data; + + /* + * Translate to nfs_fs_context, which nfs_fill_super + * can deal with. + */ + ctx->flags = data->flags & NFS_MOUNT_FLAGMASK; + ctx->flags |= extra_flags; + ctx->rsize = data->rsize; + ctx->wsize = data->wsize; + ctx->timeo = data->timeo; + ctx->retrans = data->retrans; + ctx->acregmin = data->acregmin; + ctx->acregmax = data->acregmax; + ctx->acdirmin = data->acdirmin; + ctx->acdirmax = data->acdirmax; + ctx->need_mount = false; + + if (!is_remount_fc(fc)) { + memcpy(sap, &data->addr, sizeof(data->addr)); + ctx->nfs_server.addrlen = sizeof(data->addr); + ctx->nfs_server.port = ntohs(data->addr.sin_port); + } + + if (sap->ss_family != AF_INET || + !nfs_verify_server_address(sap)) + goto out_no_address; + + if (!(data->flags & NFS_MOUNT_TCP)) + ctx->nfs_server.protocol = XPRT_TRANSPORT_UDP; + /* N.B. caller will free nfs_server.hostname in all cases */ + ctx->nfs_server.hostname = kstrdup(data->hostname, GFP_KERNEL); + if (!ctx->nfs_server.hostname) + goto out_nomem; + + ctx->namlen = data->namlen; + ctx->bsize = data->bsize; + + if (data->flags & NFS_MOUNT_SECFLAVOUR) + ctx->selected_flavor = data->pseudoflavor; + else + ctx->selected_flavor = RPC_AUTH_UNIX; + + if (!(data->flags & NFS_MOUNT_NONLM)) + ctx->flags &= ~(NFS_MOUNT_LOCAL_FLOCK| + NFS_MOUNT_LOCAL_FCNTL); + else + ctx->flags |= (NFS_MOUNT_LOCAL_FLOCK| + NFS_MOUNT_LOCAL_FCNTL); + + /* + * The legacy version 6 binary mount data from userspace has a + * field used only to transport selinux information into the + * kernel. To continue to support that functionality we + * have a touch of selinux knowledge here in the NFS code. The + * userspace code converted context=blah to just blah so we are + * converting back to the full string selinux understands. + */ + if (data->context[0]){ +#ifdef CONFIG_SECURITY_SELINUX + int ret; + + data->context[NFS_MAX_CONTEXT_LEN] = '\0'; + ret = vfs_parse_fs_string(fc, "context", data->context); + if (ret < 0) + return ret; +#else + return -EINVAL; +#endif + } + + break; + default: + goto generic; + } + + ret = nfs_validate_transport_protocol(fc, ctx); + if (ret) + return ret; + + ctx->skip_reconfig_option_check = true; + return 0; + +generic: + return generic_parse_monolithic(fc, data); + +out_no_data: + if (is_remount_fc(fc)) { + ctx->skip_reconfig_option_check = true; + return 0; + } + return nfs_invalf(fc, "NFS: mount program didn't pass any mount data"); + +out_no_v3: + return nfs_invalf(fc, "NFS: nfs_mount_data version does not support v3"); + +out_no_sec: + return nfs_invalf(fc, "NFS: nfs_mount_data version supports only AUTH_SYS"); + +out_nomem: + return -ENOMEM; + +out_no_address: + return nfs_invalf(fc, "NFS: mount program didn't pass remote address"); + +out_invalid_fh: + return nfs_invalf(fc, "NFS: invalid root filehandle"); + +out_invalid_data: + return nfs_invalf(fc, "NFS: invalid binary mount data"); +} + +#if IS_ENABLED(CONFIG_NFS_V4) +struct compat_nfs_string { + compat_uint_t len; + compat_uptr_t data; +}; + +static inline void compat_nfs_string(struct nfs_string *dst, + struct compat_nfs_string *src) +{ + dst->data = compat_ptr(src->data); + dst->len = src->len; +} + +struct compat_nfs4_mount_data_v1 { + compat_int_t version; + compat_int_t flags; + compat_int_t rsize; + compat_int_t wsize; + compat_int_t timeo; + compat_int_t retrans; + compat_int_t acregmin; + compat_int_t acregmax; + compat_int_t acdirmin; + compat_int_t acdirmax; + struct compat_nfs_string client_addr; + struct compat_nfs_string mnt_path; + struct compat_nfs_string hostname; + compat_uint_t host_addrlen; + compat_uptr_t host_addr; + compat_int_t proto; + compat_int_t auth_flavourlen; + compat_uptr_t auth_flavours; +}; + +static void nfs4_compat_mount_data_conv(struct nfs4_mount_data *data) +{ + struct compat_nfs4_mount_data_v1 *compat = + (struct compat_nfs4_mount_data_v1 *)data; + + /* copy the fields backwards */ + data->auth_flavours = compat_ptr(compat->auth_flavours); + data->auth_flavourlen = compat->auth_flavourlen; + data->proto = compat->proto; + data->host_addr = compat_ptr(compat->host_addr); + data->host_addrlen = compat->host_addrlen; + compat_nfs_string(&data->hostname, &compat->hostname); + compat_nfs_string(&data->mnt_path, &compat->mnt_path); + compat_nfs_string(&data->client_addr, &compat->client_addr); + data->acdirmax = compat->acdirmax; + data->acdirmin = compat->acdirmin; + data->acregmax = compat->acregmax; + data->acregmin = compat->acregmin; + data->retrans = compat->retrans; + data->timeo = compat->timeo; + data->wsize = compat->wsize; + data->rsize = compat->rsize; + data->flags = compat->flags; + data->version = compat->version; +} + +/* + * Validate NFSv4 mount options + */ +static int nfs4_parse_monolithic(struct fs_context *fc, + struct nfs4_mount_data *data) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + struct sockaddr_storage *sap = &ctx->nfs_server._address; + int ret; + char *c; + + if (!data) { + if (is_remount_fc(fc)) + goto done; + return nfs_invalf(fc, + "NFS4: mount program didn't pass any mount data"); + } + + ctx->version = 4; + + if (data->version != 1) + return generic_parse_monolithic(fc, data); + + if (in_compat_syscall()) + nfs4_compat_mount_data_conv(data); + + if (data->host_addrlen > sizeof(ctx->nfs_server.address)) + goto out_no_address; + if (data->host_addrlen == 0) + goto out_no_address; + ctx->nfs_server.addrlen = data->host_addrlen; + if (copy_from_user(sap, data->host_addr, data->host_addrlen)) + return -EFAULT; + if (!nfs_verify_server_address(sap)) + goto out_no_address; + ctx->nfs_server.port = ntohs(((struct sockaddr_in *)sap)->sin_port); + + if (data->auth_flavourlen) { + rpc_authflavor_t pseudoflavor; + + if (data->auth_flavourlen > 1) + goto out_inval_auth; + if (copy_from_user(&pseudoflavor, data->auth_flavours, + sizeof(pseudoflavor))) + return -EFAULT; + ctx->selected_flavor = pseudoflavor; + } else { + ctx->selected_flavor = RPC_AUTH_UNIX; + } + + c = strndup_user(data->hostname.data, NFS4_MAXNAMLEN); + if (IS_ERR(c)) + return PTR_ERR(c); + ctx->nfs_server.hostname = c; + + c = strndup_user(data->mnt_path.data, NFS4_MAXPATHLEN); + if (IS_ERR(c)) + return PTR_ERR(c); + ctx->nfs_server.export_path = c; + trace_nfs_mount_path(c); + + c = strndup_user(data->client_addr.data, 16); + if (IS_ERR(c)) + return PTR_ERR(c); + ctx->client_address = c; + + /* + * Translate to nfs_fs_context, which nfs_fill_super + * can deal with. + */ + + ctx->flags = data->flags & NFS4_MOUNT_FLAGMASK; + ctx->rsize = data->rsize; + ctx->wsize = data->wsize; + ctx->timeo = data->timeo; + ctx->retrans = data->retrans; + ctx->acregmin = data->acregmin; + ctx->acregmax = data->acregmax; + ctx->acdirmin = data->acdirmin; + ctx->acdirmax = data->acdirmax; + ctx->nfs_server.protocol = data->proto; + ret = nfs_validate_transport_protocol(fc, ctx); + if (ret) + return ret; +done: + ctx->skip_reconfig_option_check = true; + return 0; + +out_inval_auth: + return nfs_invalf(fc, "NFS4: Invalid number of RPC auth flavours %d", + data->auth_flavourlen); + +out_no_address: + return nfs_invalf(fc, "NFS4: mount program didn't pass remote address"); +} +#endif + +/* + * Parse a monolithic block of data from sys_mount(). + */ +static int nfs_fs_context_parse_monolithic(struct fs_context *fc, + void *data) +{ + if (fc->fs_type == &nfs_fs_type) + return nfs23_parse_monolithic(fc, data); + +#if IS_ENABLED(CONFIG_NFS_V4) + if (fc->fs_type == &nfs4_fs_type) + return nfs4_parse_monolithic(fc, data); +#endif + + return nfs_invalf(fc, "NFS: Unsupported monolithic data version"); +} + +/* + * Validate the preparsed information in the config. + */ +static int nfs_fs_context_validate(struct fs_context *fc) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + struct nfs_subversion *nfs_mod; + struct sockaddr_storage *sap = &ctx->nfs_server._address; + int max_namelen = PAGE_SIZE; + int max_pathlen = NFS_MAXPATHLEN; + int port = 0; + int ret; + + if (!fc->source) + goto out_no_device_name; + + /* Check for sanity first. */ + if (ctx->minorversion && ctx->version != 4) + goto out_minorversion_mismatch; + + if (ctx->options & NFS_OPTION_MIGRATION && + (ctx->version != 4 || ctx->minorversion != 0)) + goto out_migration_misuse; + + /* Verify that any proto=/mountproto= options match the address + * families in the addr=/mountaddr= options. + */ + if (ctx->protofamily != AF_UNSPEC && + ctx->protofamily != ctx->nfs_server.address.sa_family) + goto out_proto_mismatch; + + if (ctx->mountfamily != AF_UNSPEC) { + if (ctx->mount_server.addrlen) { + if (ctx->mountfamily != ctx->mount_server.address.sa_family) + goto out_mountproto_mismatch; + } else { + if (ctx->mountfamily != ctx->nfs_server.address.sa_family) + goto out_mountproto_mismatch; + } + } + + if (!nfs_verify_server_address(sap)) + goto out_no_address; + + ret = nfs_validate_transport_protocol(fc, ctx); + if (ret) + return ret; + + if (ctx->version == 4) { + if (IS_ENABLED(CONFIG_NFS_V4)) { + if (ctx->nfs_server.protocol == XPRT_TRANSPORT_RDMA) + port = NFS_RDMA_PORT; + else + port = NFS_PORT; + max_namelen = NFS4_MAXNAMLEN; + max_pathlen = NFS4_MAXPATHLEN; + ctx->flags &= ~(NFS_MOUNT_NONLM | NFS_MOUNT_NOACL | + NFS_MOUNT_VER3 | NFS_MOUNT_LOCAL_FLOCK | + NFS_MOUNT_LOCAL_FCNTL); + } else { + goto out_v4_not_compiled; + } + } else { + nfs_set_mount_transport_protocol(ctx); + if (ctx->nfs_server.protocol == XPRT_TRANSPORT_RDMA) + port = NFS_RDMA_PORT; + } + + nfs_set_port(sap, &ctx->nfs_server.port, port); + + ret = nfs_parse_source(fc, max_namelen, max_pathlen); + if (ret < 0) + return ret; + + /* Load the NFS protocol module if we haven't done so yet */ + if (!ctx->nfs_mod) { + nfs_mod = find_nfs_version(ctx->version); + if (IS_ERR(nfs_mod)) { + ret = PTR_ERR(nfs_mod); + goto out_version_unavailable; + } + ctx->nfs_mod = nfs_mod; + } + + /* Ensure the filesystem context has the correct fs_type */ + if (fc->fs_type != ctx->nfs_mod->nfs_fs) { + module_put(fc->fs_type->owner); + __module_get(ctx->nfs_mod->nfs_fs->owner); + fc->fs_type = ctx->nfs_mod->nfs_fs; + } + return 0; + +out_no_device_name: + return nfs_invalf(fc, "NFS: Device name not specified"); +out_v4_not_compiled: + nfs_errorf(fc, "NFS: NFSv4 is not compiled into kernel"); + return -EPROTONOSUPPORT; +out_no_address: + return nfs_invalf(fc, "NFS: mount program didn't pass remote address"); +out_mountproto_mismatch: + return nfs_invalf(fc, "NFS: Mount server address does not match mountproto= option"); +out_proto_mismatch: + return nfs_invalf(fc, "NFS: Server address does not match proto= option"); +out_minorversion_mismatch: + return nfs_invalf(fc, "NFS: Mount option vers=%u does not support minorversion=%u", + ctx->version, ctx->minorversion); +out_migration_misuse: + return nfs_invalf(fc, "NFS: 'Migration' not supported for this NFS version"); +out_version_unavailable: + nfs_errorf(fc, "NFS: Version unavailable"); + return ret; +} + +/* + * Create an NFS superblock by the appropriate method. + */ +static int nfs_get_tree(struct fs_context *fc) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + int err = nfs_fs_context_validate(fc); + + if (err) + return err; + if (!ctx->internal) + return ctx->nfs_mod->rpc_ops->try_get_tree(fc); + else + return nfs_get_tree_common(fc); +} + +/* + * Handle duplication of a configuration. The caller copied *src into *sc, but + * it can't deal with resource pointers in the filesystem context, so we have + * to do that. We need to clear pointers, copy data or get extra refs as + * appropriate. + */ +static int nfs_fs_context_dup(struct fs_context *fc, struct fs_context *src_fc) +{ + struct nfs_fs_context *src = nfs_fc2context(src_fc), *ctx; + + ctx = kmemdup(src, sizeof(struct nfs_fs_context), GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + ctx->mntfh = nfs_alloc_fhandle(); + if (!ctx->mntfh) { + kfree(ctx); + return -ENOMEM; + } + nfs_copy_fh(ctx->mntfh, src->mntfh); + + get_nfs_version(ctx->nfs_mod); + ctx->client_address = NULL; + ctx->mount_server.hostname = NULL; + ctx->nfs_server.export_path = NULL; + ctx->nfs_server.hostname = NULL; + ctx->fscache_uniq = NULL; + ctx->clone_data.fattr = NULL; + fc->fs_private = ctx; + return 0; +} + +static void nfs_fs_context_free(struct fs_context *fc) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + + if (ctx) { + if (ctx->server) + nfs_free_server(ctx->server); + if (ctx->nfs_mod) + put_nfs_version(ctx->nfs_mod); + kfree(ctx->client_address); + kfree(ctx->mount_server.hostname); + kfree(ctx->nfs_server.export_path); + kfree(ctx->nfs_server.hostname); + kfree(ctx->fscache_uniq); + nfs_free_fhandle(ctx->mntfh); + nfs_free_fattr(ctx->clone_data.fattr); + kfree(ctx); + } +} + +static const struct fs_context_operations nfs_fs_context_ops = { + .free = nfs_fs_context_free, + .dup = nfs_fs_context_dup, + .parse_param = nfs_fs_context_parse_param, + .parse_monolithic = nfs_fs_context_parse_monolithic, + .get_tree = nfs_get_tree, + .reconfigure = nfs_reconfigure, +}; + +/* + * Prepare superblock configuration. We use the namespaces attached to the + * context. This may be the current process's namespaces, or it may be a + * container's namespaces. + */ +static int nfs_init_fs_context(struct fs_context *fc) +{ + struct nfs_fs_context *ctx; + + ctx = kzalloc_obj(struct nfs_fs_context); + if (unlikely(!ctx)) + return -ENOMEM; + + ctx->mntfh = nfs_alloc_fhandle(); + if (unlikely(!ctx->mntfh)) { + kfree(ctx); + return -ENOMEM; + } + + ctx->protofamily = AF_UNSPEC; + ctx->mountfamily = AF_UNSPEC; + ctx->mount_server.port = NFS_UNSPEC_PORT; + + if (fc->root) { + /* reconfigure, start with the current config */ + struct nfs_server *nfss = fc->root->d_sb->s_fs_info; + struct net *net = nfss->nfs_client->cl_net; + + ctx->flags = nfss->flags; + ctx->rsize = nfss->rsize; + ctx->wsize = nfss->wsize; + ctx->retrans = nfss->client->cl_timeout->to_retries; + ctx->selected_flavor = nfss->client->cl_auth->au_flavor; + ctx->acregmin = nfss->acregmin / HZ; + ctx->acregmax = nfss->acregmax / HZ; + ctx->acdirmin = nfss->acdirmin / HZ; + ctx->acdirmax = nfss->acdirmax / HZ; + ctx->timeo = 10U * nfss->client->cl_timeout->to_initval / HZ; + ctx->nfs_server.port = nfss->port; + ctx->nfs_server.addrlen = nfss->nfs_client->cl_addrlen; + ctx->version = nfss->nfs_client->rpc_ops->version; + ctx->minorversion = nfss->nfs_client->cl_minorversion; + + memcpy(&ctx->nfs_server._address, &nfss->nfs_client->cl_addr, + ctx->nfs_server.addrlen); + + if (fc->net_ns != net) { + put_net(fc->net_ns); + fc->net_ns = get_net(net); + } + + ctx->nfs_mod = nfss->nfs_client->cl_nfs_mod; + get_nfs_version(ctx->nfs_mod); + } else { + /* defaults */ + ctx->timeo = NFS_UNSPEC_TIMEO; + ctx->retrans = NFS_UNSPEC_RETRANS; + ctx->acregmin = NFS_DEF_ACREGMIN; + ctx->acregmax = NFS_DEF_ACREGMAX; + ctx->acdirmin = NFS_DEF_ACDIRMIN; + ctx->acdirmax = NFS_DEF_ACDIRMAX; + ctx->nfs_server.port = NFS_UNSPEC_PORT; + ctx->nfs_server.protocol = XPRT_TRANSPORT_TCP; + ctx->selected_flavor = RPC_AUTH_MAXFLAVOR; + ctx->minorversion = 0; + ctx->need_mount = true; + ctx->xprtsec.policy = RPC_XPRTSEC_NONE; + ctx->xprtsec.cert_serial = TLS_NO_CERT; + ctx->xprtsec.privkey_serial = TLS_NO_PRIVKEY; + + if (fc->net_ns != &init_net) + ctx->flags |= NFS_MOUNT_NETUNREACH_FATAL; + + fc->s_iflags |= SB_I_STABLE_WRITES; + } + fc->fs_private = ctx; + fc->ops = &nfs_fs_context_ops; + return 0; +} + +struct file_system_type nfs_fs_type = { + .owner = THIS_MODULE, + .name = "nfs", + .init_fs_context = nfs_init_fs_context, + .parameters = nfs_fs_parameters, + .kill_sb = nfs_kill_super, + .fs_flags = FS_RENAME_DOES_D_MOVE | + FS_BINARY_MOUNTDATA | + FS_USERNS_DELEGATABLE, +}; +MODULE_ALIAS_FS("nfs"); +EXPORT_SYMBOL_GPL(nfs_fs_type); + +#if IS_ENABLED(CONFIG_NFS_V4) +struct file_system_type nfs4_fs_type = { + .owner = THIS_MODULE, + .name = "nfs4", + .init_fs_context = nfs_init_fs_context, + .parameters = nfs_fs_parameters, + .kill_sb = nfs_kill_super, + .fs_flags = FS_RENAME_DOES_D_MOVE | + FS_BINARY_MOUNTDATA | + FS_USERNS_DELEGATABLE, +}; +MODULE_ALIAS_FS("nfs4"); +MODULE_ALIAS("nfs4"); +EXPORT_SYMBOL_GPL(nfs4_fs_type); +#endif /* CONFIG_NFS_V4 */ diff --git a/fs/nfs/fscache.c b/fs/nfs/fscache.c new file mode 100644 index 000000000..9b7fdad4a --- /dev/null +++ b/fs/nfs/fscache.c @@ -0,0 +1,387 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* NFS filesystem cache interface + * + * Copyright (C) 2008 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" +#include "iostat.h" +#include "fscache.h" +#include "nfstrace.h" + +#define NFS_MAX_KEY_LEN 1000 + +static bool nfs_append_int(char *key, int *_len, unsigned long long x) +{ + if (*_len > NFS_MAX_KEY_LEN) + return false; + if (x == 0) + key[(*_len)++] = ','; + else + *_len += sprintf(key + *_len, ",%llx", x); + return true; +} + +/* + * Get the per-client index cookie for an NFS client if the appropriate mount + * flag was set + * - We always try and get an index cookie for the client, but get filehandle + * cookies on a per-superblock basis, depending on the mount flags + */ +static bool nfs_fscache_get_client_key(struct nfs_client *clp, + char *key, int *_len) +{ + const struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) &clp->cl_addr; + const struct sockaddr_in *sin = (struct sockaddr_in *) &clp->cl_addr; + + *_len += snprintf(key + *_len, NFS_MAX_KEY_LEN - *_len, + ",%u.%u,%x", + clp->rpc_ops->version, + clp->cl_minorversion, + clp->cl_addr.ss_family); + + switch (clp->cl_addr.ss_family) { + case AF_INET: + if (!nfs_append_int(key, _len, sin->sin_port) || + !nfs_append_int(key, _len, sin->sin_addr.s_addr)) + return false; + return true; + + case AF_INET6: + if (!nfs_append_int(key, _len, sin6->sin6_port) || + !nfs_append_int(key, _len, sin6->sin6_addr.s6_addr32[0]) || + !nfs_append_int(key, _len, sin6->sin6_addr.s6_addr32[1]) || + !nfs_append_int(key, _len, sin6->sin6_addr.s6_addr32[2]) || + !nfs_append_int(key, _len, sin6->sin6_addr.s6_addr32[3])) + return false; + return true; + + default: + printk(KERN_WARNING "NFS: Unknown network family '%d'\n", + clp->cl_addr.ss_family); + return false; + } +} + +/* + * Get the cache cookie for an NFS superblock. + * + * The default uniquifier is just an empty string, but it may be overridden + * either by the 'fsc=xxx' option to mount, or by inheriting it from the parent + * superblock across an automount point of some nature. + */ +int nfs_fscache_get_super_cookie(struct super_block *sb, const char *uniq, int ulen) +{ + struct fscache_volume *vcookie; + struct nfs_server *nfss = NFS_SB(sb); + unsigned int len = 3; + char *key; + + if (uniq) { + nfss->fscache_uniq = kmemdup_nul(uniq, ulen, GFP_KERNEL); + if (!nfss->fscache_uniq) + return -ENOMEM; + } + + key = kmalloc(NFS_MAX_KEY_LEN + 24, GFP_KERNEL); + if (!key) + return -ENOMEM; + + memcpy(key, "nfs", 3); + if (!nfs_fscache_get_client_key(nfss->nfs_client, key, &len) || + !nfs_append_int(key, &len, nfss->fsid.major) || + !nfs_append_int(key, &len, nfss->fsid.minor) || + !nfs_append_int(key, &len, sb->s_flags & NFS_SB_MASK) || + !nfs_append_int(key, &len, nfss->flags) || + !nfs_append_int(key, &len, nfss->rsize) || + !nfs_append_int(key, &len, nfss->wsize) || + !nfs_append_int(key, &len, nfss->acregmin) || + !nfs_append_int(key, &len, nfss->acregmax) || + !nfs_append_int(key, &len, nfss->acdirmin) || + !nfs_append_int(key, &len, nfss->acdirmax) || + !nfs_append_int(key, &len, nfss->client->cl_auth->au_flavor)) + goto out; + + if (ulen > 0) { + if (ulen > NFS_MAX_KEY_LEN - len) + goto out; + key[len++] = ','; + memcpy(key + len, uniq, ulen); + len += ulen; + } + key[len] = 0; + + /* create a cache index for looking up filehandles */ + vcookie = fscache_acquire_volume(key, + NULL, /* preferred_cache */ + NULL, 0 /* coherency_data */); + if (IS_ERR(vcookie)) { + if (vcookie != ERR_PTR(-EBUSY)) { + kfree(key); + return PTR_ERR(vcookie); + } + pr_err("NFS: Cache volume key already in use (%s)\n", key); + vcookie = NULL; + } + nfss->fscache = vcookie; + +out: + kfree(key); + return 0; +} + +/* + * release a per-superblock cookie + */ +void nfs_fscache_release_super_cookie(struct super_block *sb) +{ + struct nfs_server *nfss = NFS_SB(sb); + + fscache_relinquish_volume(nfss->fscache, NULL, false); + nfss->fscache = NULL; + kfree(nfss->fscache_uniq); +} + +/* + * Initialise the per-inode cache cookie pointer for an NFS inode. + */ +void nfs_fscache_init_inode(struct inode *inode) +{ + struct nfs_fscache_inode_auxdata auxdata; + struct nfs_server *nfss = NFS_SERVER(inode); + struct nfs_inode *nfsi = NFS_I(inode); + + netfs_inode(inode)->cache = NULL; + if (!(nfss->fscache && S_ISREG(inode->i_mode))) + return; + + nfs_fscache_update_auxdata(&auxdata, inode); + + netfs_inode(inode)->cache = fscache_acquire_cookie( + nfss->fscache, + 0, + nfsi->fh.data, /* index_key */ + nfsi->fh.size, + &auxdata, /* aux_data */ + sizeof(auxdata), + i_size_read(inode)); + + if (netfs_inode(inode)->cache) + mapping_set_release_always(inode->i_mapping); +} + +/* + * Release a per-inode cookie. + */ +void nfs_fscache_clear_inode(struct inode *inode) +{ + fscache_relinquish_cookie(netfs_i_cookie(netfs_inode(inode)), false); + netfs_inode(inode)->cache = NULL; +} + +/* + * Enable or disable caching for a file that is being opened as appropriate. + * The cookie is allocated when the inode is initialised, but is not enabled at + * that time. Enablement is deferred to file-open time to avoid stat() and + * access() thrashing the cache. + * + * For now, with NFS, only regular files that are open read-only will be able + * to use the cache. + * + * We enable the cache for an inode if we open it read-only and it isn't + * currently open for writing. We disable the cache if the inode is open + * write-only. + * + * The caller uses the file struct to pin i_writecount on the inode before + * calling us when a file is opened for writing, so we can make use of that. + * + * Note that this may be invoked multiple times in parallel by parallel + * nfs_open() functions. + */ +void nfs_fscache_open_file(struct inode *inode, struct file *filp) +{ + struct nfs_fscache_inode_auxdata auxdata; + struct fscache_cookie *cookie = netfs_i_cookie(netfs_inode(inode)); + bool open_for_write = inode_is_open_for_write(inode); + + if (!fscache_cookie_valid(cookie)) + return; + + fscache_use_cookie(cookie, open_for_write); + if (open_for_write) { + nfs_fscache_update_auxdata(&auxdata, inode); + fscache_invalidate(cookie, &auxdata, i_size_read(inode), + FSCACHE_INVAL_DIO_WRITE); + } +} +EXPORT_SYMBOL_GPL(nfs_fscache_open_file); + +void nfs_fscache_release_file(struct inode *inode, struct file *filp) +{ + struct nfs_fscache_inode_auxdata auxdata; + struct fscache_cookie *cookie = netfs_i_cookie(netfs_inode(inode)); + loff_t i_size = i_size_read(inode); + + nfs_fscache_update_auxdata(&auxdata, inode); + fscache_unuse_cookie(cookie, &auxdata, &i_size); +} + +int nfs_netfs_read_folio(struct file *file, struct folio *folio) +{ + if (!netfs_inode(folio_inode(folio))->cache) + return -ENOBUFS; + + return netfs_read_folio(file, folio); +} + +int nfs_netfs_readahead(struct readahead_control *ractl) +{ + struct inode *inode = ractl->mapping->host; + + if (!netfs_inode(inode)->cache) + return -ENOBUFS; + + netfs_readahead(ractl); + return 0; +} + +static atomic_t nfs_netfs_debug_id; +static int nfs_netfs_init_request(struct netfs_io_request *rreq, struct file *file) +{ + if (!file) { + if (WARN_ON_ONCE(rreq->origin != NETFS_PGPRIV2_COPY_TO_CACHE)) + return -EIO; + return 0; + } + + rreq->netfs_priv = get_nfs_open_context(nfs_file_open_context(file)); + rreq->debug_id = atomic_inc_return(&nfs_netfs_debug_id); + /* [DEPRECATED] Use PG_private_2 to mark folio being written to the cache. */ + __set_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags); + rreq->io_streams[0].sreq_max_len = NFS_SB(rreq->inode->i_sb)->rsize; + + return 0; +} + +static void nfs_netfs_free_request(struct netfs_io_request *rreq) +{ + if (rreq->netfs_priv) + put_nfs_open_context(rreq->netfs_priv); +} + +static struct nfs_netfs_io_data *nfs_netfs_alloc(struct netfs_io_subrequest *sreq) +{ + struct nfs_netfs_io_data *netfs; + + netfs = kzalloc_obj(*netfs, GFP_KERNEL_ACCOUNT); + if (!netfs) + return NULL; + netfs->sreq = sreq; + refcount_set(&netfs->refcount, 1); + return netfs; +} + +static void nfs_netfs_issue_read(struct netfs_io_subrequest *sreq) +{ + struct nfs_netfs_io_data *netfs; + struct nfs_pageio_descriptor pgio; + struct inode *inode = sreq->rreq->inode; + struct nfs_open_context *ctx = sreq->rreq->netfs_priv; + struct page *page; + unsigned long idx; + pgoff_t start, last; + int err; + + start = (sreq->start + sreq->transferred) >> PAGE_SHIFT; + last = ((sreq->start + sreq->len - sreq->transferred - 1) >> PAGE_SHIFT); + + nfs_pageio_init_read(&pgio, inode, false, + &nfs_async_read_completion_ops); + + netfs = nfs_netfs_alloc(sreq); + if (!netfs) { + sreq->error = -ENOMEM; + return netfs_read_subreq_terminated(sreq); + } + + pgio.pg_netfs = netfs; /* used in completion */ + + xa_for_each_range(&sreq->rreq->mapping->i_pages, idx, page, start, last) { + /* nfs_read_add_folio() may schedule() due to pNFS layout and other RPCs */ + err = nfs_read_add_folio(&pgio, ctx, page_folio(page)); + if (err < 0) { + netfs->error = err; + goto out; + } + } +out: + nfs_pageio_complete_read(&pgio); + nfs_netfs_put(netfs); +} + +void nfs_netfs_initiate_read(struct nfs_pgio_header *hdr) +{ + struct nfs_netfs_io_data *netfs = hdr->netfs; + + if (!netfs) + return; + + nfs_netfs_get(netfs); +} + +int nfs_netfs_folio_unlock(struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + + /* + * If fscache is enabled, netfs will unlock pages. + */ + if (netfs_inode(inode)->cache) + return 0; + + return 1; +} + +void nfs_netfs_read_completion(struct nfs_pgio_header *hdr) +{ + struct nfs_netfs_io_data *netfs = hdr->netfs; + struct netfs_io_subrequest *sreq; + + if (!netfs) + return; + + sreq = netfs->sreq; + if (test_bit(NFS_IOHDR_EOF, &hdr->flags) && + sreq->rreq->origin != NETFS_UNBUFFERED_READ && + sreq->rreq->origin != NETFS_DIO_READ) + __set_bit(NETFS_SREQ_CLEAR_TAIL, &sreq->flags); + + if (hdr->error) + netfs->error = hdr->error; + else + atomic64_add(hdr->res.count, &netfs->transferred); + + nfs_netfs_put(netfs); + hdr->netfs = NULL; +} + +const struct netfs_request_ops nfs_netfs_ops = { + .init_request = nfs_netfs_init_request, + .free_request = nfs_netfs_free_request, + .issue_read = nfs_netfs_issue_read, +}; diff --git a/fs/nfs/fscache.h b/fs/nfs/fscache.h new file mode 100644 index 000000000..9d86868f4 --- /dev/null +++ b/fs/nfs/fscache.h @@ -0,0 +1,202 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* NFS filesystem cache interface definitions + * + * Copyright (C) 2008 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#ifndef _NFS_FSCACHE_H +#define _NFS_FSCACHE_H + +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_NFS_FSCACHE + +/* + * Definition of the auxiliary data attached to NFS inode storage objects + * within the cache. + * + * The contents of this struct are recorded in the on-disk local cache in the + * auxiliary data attached to the data storage object backing an inode. This + * permits coherency to be managed when a new inode binds to an already extant + * cache object. + */ +struct nfs_fscache_inode_auxdata { + s64 mtime_sec; + s64 mtime_nsec; + s64 ctime_sec; + s64 ctime_nsec; + u64 change_attr; +}; + +struct nfs_netfs_io_data { + /* + * NFS may split a netfs_io_subrequest into multiple RPCs, each + * with their own read completion. In netfs, we can only call + * netfs_subreq_terminated() once for each subrequest. Use the + * refcount here to double as a marker of the last RPC completion, + * and only call netfs via netfs_subreq_terminated() once. + */ + refcount_t refcount; + struct netfs_io_subrequest *sreq; + + /* + * Final disposition of the netfs_io_subrequest, sent in + * netfs_subreq_terminated() + */ + atomic64_t transferred; + int error; +}; + +static inline void nfs_netfs_get(struct nfs_netfs_io_data *netfs) +{ + refcount_inc(&netfs->refcount); +} + +static inline void nfs_netfs_put(struct nfs_netfs_io_data *netfs) +{ + /* Only the last RPC completion should call netfs_subreq_terminated() */ + if (!refcount_dec_and_test(&netfs->refcount)) + return; + + /* + * The NFS pageio interface may read a complete page, even when netfs + * only asked for a partial page. Specifically, this may be seen when + * one thread is truncating a file while another one is reading the last + * page of the file. + * Correct the final length here to be no larger than the netfs subrequest + * length, and thus avoid netfs's "Subreq overread" warning message. + */ + netfs->sreq->transferred = min_t(s64, netfs->sreq->len, + atomic64_read(&netfs->transferred)); + netfs->sreq->error = netfs->error; + netfs_read_subreq_terminated(netfs->sreq); + kfree(netfs); +} +static inline void nfs_netfs_inode_init(struct nfs_inode *nfsi) +{ + netfs_inode_init(&nfsi->netfs, &nfs_netfs_ops, false); +} +extern void nfs_netfs_initiate_read(struct nfs_pgio_header *hdr); +extern void nfs_netfs_read_completion(struct nfs_pgio_header *hdr); +extern int nfs_netfs_folio_unlock(struct folio *folio); + +/* + * fscache.c + */ +extern int nfs_fscache_get_super_cookie(struct super_block *, const char *, int); +extern void nfs_fscache_release_super_cookie(struct super_block *); + +extern void nfs_fscache_init_inode(struct inode *); +extern void nfs_fscache_clear_inode(struct inode *); +extern void nfs_fscache_open_file(struct inode *, struct file *); +extern void nfs_fscache_release_file(struct inode *, struct file *); +extern int nfs_netfs_readahead(struct readahead_control *ractl); +extern int nfs_netfs_read_folio(struct file *file, struct folio *folio); + +static inline bool nfs_fscache_release_folio(struct folio *folio, gfp_t gfp) +{ + if (folio_test_private_2(folio)) { /* [DEPRECATED] */ + if (current_is_kswapd() || !(gfp & __GFP_FS)) + return false; + folio_wait_private_2(folio); + } + fscache_note_page_release(netfs_i_cookie(netfs_inode(folio->mapping->host))); + return true; +} + +static inline void nfs_fscache_update_auxdata(struct nfs_fscache_inode_auxdata *auxdata, + struct inode *inode) +{ + memset(auxdata, 0, sizeof(*auxdata)); + auxdata->mtime_sec = inode_get_mtime(inode).tv_sec; + auxdata->mtime_nsec = inode_get_mtime(inode).tv_nsec; + auxdata->ctime_sec = inode_get_ctime(inode).tv_sec; + auxdata->ctime_nsec = inode_get_ctime(inode).tv_nsec; + + if (NFS_SERVER(inode)->nfs_client->rpc_ops->version == 4) + auxdata->change_attr = inode_peek_iversion_raw(inode); +} + +/* + * Invalidate the contents of fscache for this inode. This will not sleep. + */ +static inline void nfs_fscache_invalidate(struct inode *inode, int flags) +{ + struct nfs_fscache_inode_auxdata auxdata; + struct fscache_cookie *cookie = netfs_i_cookie(&NFS_I(inode)->netfs); + + nfs_fscache_update_auxdata(&auxdata, inode); + fscache_invalidate(cookie, &auxdata, i_size_read(inode), flags); +} + +/* + * indicate the client caching state as readable text + */ +static inline const char *nfs_server_fscache_state(struct nfs_server *server) +{ + if (server->fscache) + return "yes"; + return "no "; +} + +static inline void nfs_netfs_set_pgio_header(struct nfs_pgio_header *hdr, + struct nfs_pageio_descriptor *desc) +{ + hdr->netfs = desc->pg_netfs; +} +static inline void nfs_netfs_set_pageio_descriptor(struct nfs_pageio_descriptor *desc, + struct nfs_pgio_header *hdr) +{ + desc->pg_netfs = hdr->netfs; +} +static inline void nfs_netfs_reset_pageio_descriptor(struct nfs_pageio_descriptor *desc) +{ + desc->pg_netfs = NULL; +} +#else /* CONFIG_NFS_FSCACHE */ +static inline void nfs_netfs_inode_init(struct nfs_inode *nfsi) {} +static inline void nfs_netfs_initiate_read(struct nfs_pgio_header *hdr) {} +static inline void nfs_netfs_read_completion(struct nfs_pgio_header *hdr) {} +static inline int nfs_netfs_folio_unlock(struct folio *folio) +{ + return 1; +} +static inline void nfs_fscache_release_super_cookie(struct super_block *sb) {} + +static inline void nfs_fscache_init_inode(struct inode *inode) {} +static inline void nfs_fscache_clear_inode(struct inode *inode) {} +static inline void nfs_fscache_open_file(struct inode *inode, + struct file *filp) {} +static inline void nfs_fscache_release_file(struct inode *inode, struct file *file) {} +static inline int nfs_netfs_readahead(struct readahead_control *ractl) +{ + return -ENOBUFS; +} +static inline int nfs_netfs_read_folio(struct file *file, struct folio *folio) +{ + return -ENOBUFS; +} + +static inline bool nfs_fscache_release_folio(struct folio *folio, gfp_t gfp) +{ + return true; /* may release folio */ +} +static inline void nfs_fscache_invalidate(struct inode *inode, int flags) {} + +static inline const char *nfs_server_fscache_state(struct nfs_server *server) +{ + return "no "; +} +static inline void nfs_netfs_set_pgio_header(struct nfs_pgio_header *hdr, + struct nfs_pageio_descriptor *desc) {} +static inline void nfs_netfs_set_pageio_descriptor(struct nfs_pageio_descriptor *desc, + struct nfs_pgio_header *hdr) {} +static inline void nfs_netfs_reset_pageio_descriptor(struct nfs_pageio_descriptor *desc) {} +#endif /* CONFIG_NFS_FSCACHE */ +#endif /* _NFS_FSCACHE_H */ diff --git a/fs/nfs/getroot.c b/fs/nfs/getroot.c new file mode 100644 index 000000000..ff7424bc4 --- /dev/null +++ b/fs/nfs/getroot.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* getroot.c: get the root dentry for an NFS mount + * + * Copyright (C) 2006 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "internal.h" + +#define NFSDBG_FACILITY NFSDBG_CLIENT + +/* + * get a root dentry from the root filehandle + */ +int nfs_get_root(struct super_block *s, struct fs_context *fc) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + struct nfs_server *server = NFS_SB(s), *clone_server; + struct nfs_fsinfo fsinfo; + struct dentry *root; + struct inode *inode; + char *name; + int error = -ENOMEM; + unsigned long kflags = 0, kflags_out = 0; + + name = kstrdup(fc->source, GFP_KERNEL); + if (!name) + goto out; + + /* get the actual root for this mount */ + fsinfo.fattr = nfs_alloc_fattr_with_label(server); + if (fsinfo.fattr == NULL) + goto out_name; + + error = server->nfs_client->rpc_ops->getroot(server, ctx->mntfh, &fsinfo); + if (error < 0) { + dprintk("nfs_get_root: getattr error = %d\n", -error); + nfs_errorf(fc, "NFS: Couldn't getattr on root"); + goto out_fattr; + } + + inode = nfs_fhget(s, ctx->mntfh, fsinfo.fattr); + if (IS_ERR(inode)) { + dprintk("nfs_get_root: get root inode failed\n"); + error = PTR_ERR(inode); + nfs_errorf(fc, "NFS: Couldn't get root inode"); + goto out_fattr; + } + + /* root dentries normally start off anonymous and get spliced in later + * if the dentry tree reaches them; however if the dentry already + * exists, we'll pick it up at this point and use it as the root + */ + root = d_obtain_root(inode); + if (IS_ERR(root)) { + dprintk("nfs_get_root: get root dentry failed\n"); + error = PTR_ERR(root); + nfs_errorf(fc, "NFS: Couldn't get root dentry"); + goto out_fattr; + } + + security_d_instantiate(root, inode); + spin_lock(&root->d_lock); + if (IS_ROOT(root) && !root->d_fsdata && + !(root->d_flags & DCACHE_NFSFS_RENAMED)) { + root->d_fsdata = name; + name = NULL; + } + spin_unlock(&root->d_lock); + if (!s->s_root) + s->s_root = dget(root); + fc->root = root; + if (server->caps & NFS_CAP_SECURITY_LABEL) + kflags |= SECURITY_LSM_NATIVE_LABELS; + if (ctx->clone_data.sb) { + if (d_inode(fc->root)->i_fop != &nfs_dir_operations) { + error = -ESTALE; + goto error_splat_root; + } + /* clone lsm security options from the parent to the new sb */ + error = security_sb_clone_mnt_opts(ctx->clone_data.sb, + s, kflags, &kflags_out); + if (error) + goto error_splat_root; + clone_server = NFS_SB(ctx->clone_data.sb); + server->has_sec_mnt_opts = clone_server->has_sec_mnt_opts; + } else { + error = security_sb_set_mnt_opts(s, fc->security, + kflags, &kflags_out); + } + if (error) + goto error_splat_root; + if (server->caps & NFS_CAP_SECURITY_LABEL && + !(kflags_out & SECURITY_LSM_NATIVE_LABELS)) + server->caps &= ~NFS_CAP_SECURITY_LABEL; + + nfs_setsecurity(inode, fsinfo.fattr); + error = 0; + +out_fattr: + nfs_free_fattr(fsinfo.fattr); +out_name: + kfree(name); +out: + return error; +error_splat_root: + dput(fc->root); + fc->root = NULL; + goto out_fattr; +} diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c new file mode 100644 index 000000000..5bcd4027d --- /dev/null +++ b/fs/nfs/inode.c @@ -0,0 +1,2793 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/nfs/inode.c + * + * Copyright (C) 1992 Rick Sladkey + * + * nfs inode and superblock handling functions + * + * Modularised by Alan Cox , while hacking some + * experimental NFS changes. Modularisation taken straight from SYS5 fs. + * + * Change to nfs_read_super() to permit NFS mounts to multi-homed hosts. + * J.S.Peatfield@damtp.cam.ac.uk + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nfs4_fs.h" +#include "callback.h" +#include "delegation.h" +#include "iostat.h" +#include "internal.h" +#include "fscache.h" +#include "pnfs.h" +#include "nfs.h" +#include "netns.h" +#include "sysfs.h" + +#include "nfstrace.h" + +#define NFSDBG_FACILITY NFSDBG_VFS + +static bool enable_ino64; + +static int param_set_enable_ino64(const char *val, const struct kernel_param *kp) +{ + pr_notice("enable_ino64 is deprecated and has no effect\n"); + return 0; +} + +static const struct kernel_param_ops param_ops_enable_ino64 = { + .set = param_set_enable_ino64, + .get = param_get_bool, +}; + +static int nfs_update_inode(struct inode *, struct nfs_fattr *); + +static struct kmem_cache * nfs_inode_cachep; + +int nfs_wait_bit_killable(struct wait_bit_key *key, int mode) +{ + if (unlikely(nfs_current_task_exiting())) + return -EINTR; + schedule(); + if (signal_pending_state(mode, current)) + return -ERESTARTSYS; + return 0; +} +EXPORT_SYMBOL_GPL(nfs_wait_bit_killable); + +int nfs_drop_inode(struct inode *inode) +{ + return NFS_STALE(inode) || inode_generic_drop(inode); +} +EXPORT_SYMBOL_GPL(nfs_drop_inode); + +void nfs_clear_inode(struct inode *inode) +{ + /* + * The following should never happen... + */ + WARN_ON_ONCE(nfs_have_writebacks(inode)); + WARN_ON_ONCE(!list_empty(&NFS_I(inode)->open_files)); + nfs_zap_acl_cache(inode); + nfs_access_zap_cache(inode); + nfs_fscache_clear_inode(inode); +} +EXPORT_SYMBOL_GPL(nfs_clear_inode); + +void nfs_evict_inode(struct inode *inode) +{ + truncate_inode_pages_final(&inode->i_data); + clear_inode(inode); + nfs_clear_inode(inode); +} + +int nfs_sync_inode(struct inode *inode) +{ + inode_dio_wait(inode); + return nfs_wb_all(inode); +} +EXPORT_SYMBOL_GPL(nfs_sync_inode); + +/** + * nfs_sync_mapping - helper to flush all mmapped dirty data to disk + * @mapping: pointer to struct address_space + */ +int nfs_sync_mapping(struct address_space *mapping) +{ + int ret = 0; + + if (mapping->nrpages != 0) { + unmap_mapping_range(mapping, 0, 0, 0); + ret = nfs_wb_all(mapping->host); + } + return ret; +} + +static int nfs_attribute_timeout(struct inode *inode) +{ + struct nfs_inode *nfsi = NFS_I(inode); + + return !time_in_range_open(jiffies, nfsi->read_cache_jiffies, nfsi->read_cache_jiffies + nfsi->attrtimeo); +} + +static bool nfs_check_cache_flags_invalid(struct inode *inode, + unsigned long flags) +{ + unsigned long cache_validity = READ_ONCE(NFS_I(inode)->cache_validity); + + return (cache_validity & flags) != 0; +} + +bool nfs_check_cache_invalid(struct inode *inode, unsigned long flags) +{ + if (nfs_check_cache_flags_invalid(inode, flags)) + return true; + return nfs_attribute_cache_expired(inode); +} +EXPORT_SYMBOL_GPL(nfs_check_cache_invalid); + +#ifdef CONFIG_NFS_V4_2 +static bool nfs_has_xattr_cache(const struct nfs_inode *nfsi) +{ + return nfsi->xattr_cache != NULL; +} +#else +static bool nfs_has_xattr_cache(const struct nfs_inode *nfsi) +{ + return false; +} +#endif + +void nfs_set_cache_invalid(struct inode *inode, unsigned long flags) +{ + struct nfs_inode *nfsi = NFS_I(inode); + + if (nfs_have_delegated_attributes(inode)) { + if (!(flags & NFS_INO_REVAL_FORCED)) + flags &= ~(NFS_INO_INVALID_MODE | + NFS_INO_INVALID_OTHER | + NFS_INO_INVALID_BTIME | + NFS_INO_INVALID_XATTR); + flags &= ~(NFS_INO_INVALID_CHANGE | NFS_INO_INVALID_SIZE); + } + + if (!nfs_has_xattr_cache(nfsi)) + flags &= ~NFS_INO_INVALID_XATTR; + if (flags & NFS_INO_INVALID_DATA) + nfs_fscache_invalidate(inode, 0); + flags &= ~NFS_INO_REVAL_FORCED; + + flags |= nfsi->cache_validity; + if (inode->i_mapping->nrpages == 0) + flags &= ~NFS_INO_INVALID_DATA; + + /* pairs with nfs_clear_invalid_mapping()'s smp_load_acquire() */ + smp_store_release(&nfsi->cache_validity, flags); + + if (inode->i_mapping->nrpages == 0 || + nfsi->cache_validity & NFS_INO_INVALID_DATA) { + nfs_ooo_clear(nfsi); + } + trace_nfs_set_cache_invalid(inode, 0); +} +EXPORT_SYMBOL_GPL(nfs_set_cache_invalid); + +/* + * Invalidate the local caches + */ +static void nfs_zap_caches_locked(struct inode *inode) +{ + struct nfs_inode *nfsi = NFS_I(inode); + int mode = inode->i_mode; + + nfs_inc_stats(inode, NFSIOS_ATTRINVALIDATE); + + nfsi->attrtimeo = NFS_MINATTRTIMEO(inode); + nfsi->attrtimeo_timestamp = jiffies; + + if (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode)) + nfs_set_cache_invalid(inode, NFS_INO_INVALID_ATTR | + NFS_INO_INVALID_DATA | + NFS_INO_INVALID_ACCESS | + NFS_INO_INVALID_ACL | + NFS_INO_INVALID_XATTR); + else + nfs_set_cache_invalid(inode, NFS_INO_INVALID_ATTR | + NFS_INO_INVALID_ACCESS | + NFS_INO_INVALID_ACL | + NFS_INO_INVALID_XATTR); + nfs_zap_label_cache_locked(nfsi); +} + +void nfs_zap_caches(struct inode *inode) +{ + spin_lock(&inode->i_lock); + nfs_zap_caches_locked(inode); + spin_unlock(&inode->i_lock); +} + +void nfs_zap_mapping(struct inode *inode, struct address_space *mapping) +{ + if (mapping->nrpages != 0) { + spin_lock(&inode->i_lock); + nfs_set_cache_invalid(inode, NFS_INO_INVALID_DATA); + spin_unlock(&inode->i_lock); + } +} + +void nfs_zap_acl_cache(struct inode *inode) +{ + void (*clear_acl_cache)(struct inode *); + + clear_acl_cache = NFS_PROTO(inode)->clear_acl_cache; + if (clear_acl_cache != NULL) + clear_acl_cache(inode); + spin_lock(&inode->i_lock); + NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_ACL; + spin_unlock(&inode->i_lock); +} +EXPORT_SYMBOL_GPL(nfs_zap_acl_cache); + +void nfs_invalidate_atime(struct inode *inode) +{ + if (nfs_have_delegated_atime(inode)) + return; + spin_lock(&inode->i_lock); + nfs_set_cache_invalid(inode, NFS_INO_INVALID_ATIME); + spin_unlock(&inode->i_lock); +} +EXPORT_SYMBOL_GPL(nfs_invalidate_atime); + +/* + * Invalidate, but do not unhash, the inode. + * NB: must be called with inode->i_lock held! + */ +static void nfs_set_inode_stale_locked(struct inode *inode) +{ + set_bit(NFS_INO_STALE, &NFS_I(inode)->flags); + nfs_zap_caches_locked(inode); + trace_nfs_set_inode_stale(inode); +} + +void nfs_set_inode_stale(struct inode *inode) +{ + spin_lock(&inode->i_lock); + nfs_set_inode_stale_locked(inode); + spin_unlock(&inode->i_lock); +} + +struct nfs_find_desc { + struct nfs_fh *fh; + struct nfs_fattr *fattr; +}; + +/* + * For re-exported directories (also seen in NFSv2) + * we are forced to allow 2 different inodes to have the same + * i_ino. + */ +static int +nfs_find_actor(struct inode *inode, void *opaque) +{ + struct nfs_find_desc *desc = opaque; + struct nfs_fh *fh = desc->fh; + struct nfs_fattr *fattr = desc->fattr; + + if (inode->i_ino != fattr->fileid) + return 0; + if (inode_wrong_type(inode, fattr->mode)) + return 0; + if (nfs_compare_fh(NFS_FH(inode), fh)) + return 0; + if (is_bad_inode(inode) || NFS_STALE(inode)) + return 0; + return 1; +} + +static int +nfs_init_locked(struct inode *inode, void *opaque) +{ + struct nfs_find_desc *desc = opaque; + struct nfs_fattr *fattr = desc->fattr; + + inode->i_ino = fattr->fileid; + inode->i_mode = fattr->mode; + nfs_copy_fh(NFS_FH(inode), desc->fh); + return 0; +} + +#ifdef CONFIG_NFS_V4_SECURITY_LABEL +static void nfs_clear_label_invalid(struct inode *inode) +{ + spin_lock(&inode->i_lock); + NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_LABEL; + spin_unlock(&inode->i_lock); +} + +void nfs_setsecurity(struct inode *inode, struct nfs_fattr *fattr) +{ + int error; + + if (fattr->label == NULL) + return; + + if ((fattr->valid & NFS_ATTR_FATTR_V4_SECURITY_LABEL) && inode->i_security) { + error = security_inode_notifysecctx(inode, fattr->label->label, + fattr->label->len); + if (error) + printk(KERN_ERR "%s() %s %d " + "security_inode_notifysecctx() %d\n", + __func__, + (char *)fattr->label->label, + fattr->label->len, error); + nfs_clear_label_invalid(inode); + } +} + +struct nfs4_label *nfs4_label_alloc(struct nfs_server *server, gfp_t flags) +{ + struct nfs4_label *label; + + if (!(server->caps & NFS_CAP_SECURITY_LABEL)) + return NULL; + + label = kzalloc_obj(struct nfs4_label, flags); + if (label == NULL) + return ERR_PTR(-ENOMEM); + + label->label = kzalloc(NFS4_MAXLABELLEN, flags); + if (label->label == NULL) { + kfree(label); + return ERR_PTR(-ENOMEM); + } + label->len = NFS4_MAXLABELLEN; + + return label; +} +EXPORT_SYMBOL_GPL(nfs4_label_alloc); +#else +void nfs_setsecurity(struct inode *inode, struct nfs_fattr *fattr) +{ +} +#endif +EXPORT_SYMBOL_GPL(nfs_setsecurity); + +/* Search for inode identified by fh, fileid and i_mode in inode cache. */ +struct inode * +nfs_ilookup(struct super_block *sb, struct nfs_fattr *fattr, struct nfs_fh *fh) +{ + struct nfs_find_desc desc = { + .fh = fh, + .fattr = fattr, + }; + struct inode *inode; + u64 hash; + + if (!(fattr->valid & NFS_ATTR_FATTR_FILEID) || + !(fattr->valid & NFS_ATTR_FATTR_TYPE)) + return NULL; + + hash = fattr->fileid; + inode = ilookup5(sb, hash, nfs_find_actor, &desc); + + dprintk("%s: returning %p\n", __func__, inode); + return inode; +} + +static void nfs_inode_init_regular(struct nfs_inode *nfsi) +{ + atomic_long_set(&nfsi->nrequests, 0); + atomic_long_set(&nfsi->redirtied_pages, 0); + INIT_LIST_HEAD(&nfsi->commit_info.list); + atomic_long_set(&nfsi->commit_info.ncommit, 0); + atomic_set(&nfsi->commit_info.rpcs_out, 0); + mutex_init(&nfsi->commit_mutex); +} + +static void nfs_inode_init_dir(struct nfs_inode *nfsi) +{ + nfsi->cache_change_attribute = 0; + memset(nfsi->cookieverf, 0, sizeof(nfsi->cookieverf)); + init_rwsem(&nfsi->rmdir_sem); +} + +/* + * This is our front-end to iget that looks up inodes by file handle + * instead of inode number. + */ +struct inode * +nfs_fhget(struct super_block *sb, struct nfs_fh *fh, struct nfs_fattr *fattr) +{ + struct nfs_find_desc desc = { + .fh = fh, + .fattr = fattr + }; + struct inode *inode = ERR_PTR(-ENOENT); + u64 fattr_supported = NFS_SB(sb)->fattr_valid; + u64 hash; + + nfs_attr_check_mountpoint(sb, fattr); + + if (nfs_attr_use_mounted_on_fileid(fattr)) + fattr->fileid = fattr->mounted_on_fileid; + else if ((fattr->valid & NFS_ATTR_FATTR_FILEID) == 0) + goto out_no_inode; + if ((fattr->valid & NFS_ATTR_FATTR_TYPE) == 0) + goto out_no_inode; + + hash = fattr->fileid; + + inode = iget5_locked(sb, hash, nfs_find_actor, nfs_init_locked, &desc); + if (inode == NULL) { + inode = ERR_PTR(-ENOMEM); + goto out_no_inode; + } + + if (inode_state_read_once(inode) & I_NEW) { + struct nfs_inode *nfsi = NFS_I(inode); + unsigned long now = jiffies; + + /* We can't support update_atime(), since the server will reset it */ + inode->i_flags |= S_NOATIME|S_NOCMTIME; + inode->i_mode = fattr->mode; + nfsi->cache_validity = 0; + if ((fattr->valid & NFS_ATTR_FATTR_MODE) == 0 + && (fattr_supported & NFS_ATTR_FATTR_MODE)) + nfs_set_cache_invalid(inode, NFS_INO_INVALID_MODE); + /* Why so? Because we want revalidate for devices/FIFOs, and + * that's precisely what we have in nfs_file_inode_operations. + */ + inode->i_op = NFS_SB(sb)->nfs_client->rpc_ops->file_inode_ops; + if (S_ISREG(inode->i_mode)) { + inode->i_fop = NFS_SB(sb)->nfs_client->rpc_ops->file_ops; + inode->i_data.a_ops = &nfs_file_aops; + nfs_inode_init_regular(nfsi); + mapping_set_large_folios(inode->i_mapping); + } else if (S_ISDIR(inode->i_mode)) { + inode->i_op = NFS_SB(sb)->nfs_client->rpc_ops->dir_inode_ops; + inode->i_fop = &nfs_dir_operations; + inode->i_data.a_ops = &nfs_dir_aops; + nfs_inode_init_dir(nfsi); + /* Deal with crossing mountpoints */ + if (fattr->valid & NFS_ATTR_FATTR_MOUNTPOINT || + fattr->valid & NFS_ATTR_FATTR_V4_REFERRAL) { + if (fattr->valid & NFS_ATTR_FATTR_V4_REFERRAL) + inode->i_op = &nfs_referral_inode_operations; + else + inode->i_op = &nfs_mountpoint_inode_operations; + inode->i_fop = NULL; + inode->i_flags |= S_AUTOMOUNT; + } + } else if (S_ISLNK(inode->i_mode)) { + inode->i_op = &nfs_symlink_inode_operations; + inode_nohighmem(inode); + } else + init_special_inode(inode, inode->i_mode, fattr->rdev); + + inode_set_atime(inode, 0, 0); + inode_set_mtime(inode, 0, 0); + inode_set_ctime(inode, 0, 0); + memset(&nfsi->btime, 0, sizeof(nfsi->btime)); + inode_set_iversion_raw(inode, 0); + inode->i_size = 0; + clear_nlink(inode); + inode->i_uid = make_kuid(&init_user_ns, -2); + inode->i_gid = make_kgid(&init_user_ns, -2); + inode->i_blocks = 0; + nfsi->write_io = 0; + nfsi->read_io = 0; + + nfsi->read_cache_jiffies = fattr->time_start; + nfsi->attr_gencount = fattr->gencount; + if (fattr->valid & NFS_ATTR_FATTR_ATIME) + inode_set_atime_to_ts(inode, fattr->atime); + else if (fattr_supported & NFS_ATTR_FATTR_ATIME) + nfs_set_cache_invalid(inode, NFS_INO_INVALID_ATIME); + if (fattr->valid & NFS_ATTR_FATTR_MTIME) + inode_set_mtime_to_ts(inode, fattr->mtime); + else if (fattr_supported & NFS_ATTR_FATTR_MTIME) + nfs_set_cache_invalid(inode, NFS_INO_INVALID_MTIME); + if (fattr->valid & NFS_ATTR_FATTR_CTIME) + inode_set_ctime_to_ts(inode, fattr->ctime); + else if (fattr_supported & NFS_ATTR_FATTR_CTIME) + nfs_set_cache_invalid(inode, NFS_INO_INVALID_CTIME); + if (fattr->valid & NFS_ATTR_FATTR_BTIME) + nfsi->btime = fattr->btime; + else if (fattr_supported & NFS_ATTR_FATTR_BTIME) + nfs_set_cache_invalid(inode, NFS_INO_INVALID_BTIME); + if (fattr->valid & NFS_ATTR_FATTR_CHANGE) + inode_set_iversion_raw(inode, fattr->change_attr); + else + nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE); + if (fattr->valid & NFS_ATTR_FATTR_SIZE) + inode->i_size = nfs_size_to_loff_t(fattr->size); + else + nfs_set_cache_invalid(inode, NFS_INO_INVALID_SIZE); + if (fattr->valid & NFS_ATTR_FATTR_NLINK) + set_nlink(inode, fattr->nlink); + else if (fattr_supported & NFS_ATTR_FATTR_NLINK) + nfs_set_cache_invalid(inode, NFS_INO_INVALID_NLINK); + else + set_nlink(inode, 1); + if (fattr->valid & NFS_ATTR_FATTR_OWNER) + inode->i_uid = fattr->uid; + else if (fattr_supported & NFS_ATTR_FATTR_OWNER) + nfs_set_cache_invalid(inode, NFS_INO_INVALID_OTHER); + if (fattr->valid & NFS_ATTR_FATTR_GROUP) + inode->i_gid = fattr->gid; + else if (fattr_supported & NFS_ATTR_FATTR_GROUP) + nfs_set_cache_invalid(inode, NFS_INO_INVALID_OTHER); + if (fattr->valid & NFS_ATTR_FATTR_BLOCKS_USED) + inode->i_blocks = fattr->du.nfs2.blocks; + else if (fattr_supported & NFS_ATTR_FATTR_BLOCKS_USED && + fattr->size != 0) + nfs_set_cache_invalid(inode, NFS_INO_INVALID_BLOCKS); + if (fattr->valid & NFS_ATTR_FATTR_SPACE_USED) { + /* + * report the blocks in 512byte units + */ + inode->i_blocks = nfs_calc_block_size(fattr->du.nfs3.used); + } else if (fattr_supported & NFS_ATTR_FATTR_SPACE_USED && + fattr->size != 0) + nfs_set_cache_invalid(inode, NFS_INO_INVALID_BLOCKS); + + nfs_setsecurity(inode, fattr); + + nfsi->attrtimeo = NFS_MINATTRTIMEO(inode); + nfsi->attrtimeo_timestamp = now; + nfsi->access_cache = RB_ROOT; + + nfs_fscache_init_inode(inode); + + unlock_new_inode(inode); + } else { + int err = nfs_refresh_inode(inode, fattr); + if (err < 0) { + iput(inode); + inode = ERR_PTR(err); + goto out_no_inode; + } + } + dprintk("NFS: nfs_fhget(%s/%Lu fh_crc=0x%08x ct=%d)\n", + inode->i_sb->s_id, + (unsigned long long)inode->i_ino, + nfs_display_fhandle_hash(fh), + icount_read_once(inode)); + +out: + return inode; + +out_no_inode: + dprintk("nfs_fhget: iget failed with error %ld\n", PTR_ERR(inode)); + goto out; +} +EXPORT_SYMBOL_GPL(nfs_fhget); + +static void +nfs_fattr_fixup_delegated(struct inode *inode, struct nfs_fattr *fattr) +{ + unsigned long cache_validity = NFS_I(inode)->cache_validity; + + if (nfs_have_delegated_mtime(inode)) { + if (!(cache_validity & NFS_INO_INVALID_CTIME)) + fattr->valid &= ~(NFS_ATTR_FATTR_PRECTIME | + NFS_ATTR_FATTR_CTIME); + + if (!(cache_validity & NFS_INO_INVALID_MTIME)) + fattr->valid &= ~(NFS_ATTR_FATTR_PREMTIME | + NFS_ATTR_FATTR_MTIME); + + if (!(cache_validity & NFS_INO_INVALID_ATIME)) + fattr->valid &= ~NFS_ATTR_FATTR_ATIME; + } else if (nfs_have_delegated_atime(inode)) { + if (!(cache_validity & NFS_INO_INVALID_ATIME)) + fattr->valid &= ~NFS_ATTR_FATTR_ATIME; + } +} + +static void nfs_set_timestamps_to_ts(struct inode *inode, struct iattr *attr) +{ + unsigned int cache_flags = 0; + + if (attr->ia_valid & ATTR_MTIME_SET) { + struct timespec64 ctime = inode_get_ctime(inode); + struct timespec64 mtime = inode_get_mtime(inode); + struct timespec64 now; + bool updated = false; + + now = inode_set_ctime_current(inode); + if (!timespec64_equal(&now, &ctime)) + updated = true; + + inode_set_mtime_to_ts(inode, attr->ia_mtime); + if (!timespec64_equal(&now, &mtime)) + updated = true; + + inode_maybe_inc_iversion(inode, updated); + cache_flags |= NFS_INO_INVALID_CTIME | NFS_INO_INVALID_MTIME; + } + if (attr->ia_valid & ATTR_ATIME_SET) { + inode_set_atime_to_ts(inode, attr->ia_atime); + cache_flags |= NFS_INO_INVALID_ATIME; + } + NFS_I(inode)->cache_validity &= ~cache_flags; +} + +static void nfs_update_atime(struct inode *inode) +{ + inode_update_time(inode, FS_UPD_ATIME, 0); + NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_ATIME; +} + +static void nfs_update_mtime(struct inode *inode) +{ + inode_update_time(inode, FS_UPD_CMTIME, 0); + NFS_I(inode)->cache_validity &= + ~(NFS_INO_INVALID_CTIME | NFS_INO_INVALID_MTIME); +} + +void nfs_update_delegated_atime(struct inode *inode) +{ + spin_lock(&inode->i_lock); + if (nfs_have_delegated_atime(inode)) + nfs_update_atime(inode); + spin_unlock(&inode->i_lock); +} + +void nfs_update_delegated_mtime_locked(struct inode *inode) +{ + if (nfs_have_delegated_mtime(inode) || + nfs_have_directory_delegation(inode)) + nfs_update_mtime(inode); +} + +void nfs_update_delegated_mtime(struct inode *inode) +{ + spin_lock(&inode->i_lock); + nfs_update_delegated_mtime_locked(inode); + spin_unlock(&inode->i_lock); +} +EXPORT_SYMBOL_GPL(nfs_update_delegated_mtime); + +#define NFS_VALID_ATTRS (ATTR_MODE|ATTR_UID|ATTR_GID|ATTR_SIZE|ATTR_ATIME|ATTR_ATIME_SET|ATTR_MTIME|ATTR_MTIME_SET|ATTR_FILE|ATTR_OPEN) + +int +nfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +{ + struct inode *inode = d_inode(dentry); + struct nfs_fattr *fattr; + loff_t oldsize; + int error = 0; + kuid_t task_uid = current_fsuid(); + kuid_t owner_uid = inode->i_uid; + + nfs_inc_stats(inode, NFSIOS_VFSSETATTR); + + /* skip mode change if it's just for clearing setuid/setgid */ + if (attr->ia_valid & (ATTR_KILL_SUID | ATTR_KILL_SGID)) + attr->ia_valid &= ~ATTR_MODE; + + if (S_ISREG(inode->i_mode)) + nfs_file_block_o_direct(NFS_I(inode)); + + oldsize = i_size_read(inode); + if (attr->ia_valid & ATTR_SIZE) { + BUG_ON(!S_ISREG(inode->i_mode)); + + error = inode_newsize_ok(inode, attr->ia_size); + if (error) + return error; + + if (attr->ia_size == oldsize) + attr->ia_valid &= ~ATTR_SIZE; + } + + if (nfs_have_delegated_mtime(inode) && attr->ia_valid & ATTR_MTIME) { + spin_lock(&inode->i_lock); + if (attr->ia_valid & ATTR_MTIME_SET) { + if (uid_eq(task_uid, owner_uid)) { + nfs_set_timestamps_to_ts(inode, attr); + attr->ia_valid &= ~(ATTR_MTIME|ATTR_MTIME_SET| + ATTR_ATIME|ATTR_ATIME_SET); + } + } else { + if (attr->ia_valid & ATTR_MTIME) + nfs_update_mtime(inode); + if (attr->ia_valid & ATTR_ATIME) + nfs_update_atime(inode); + attr->ia_valid &= ~(ATTR_MTIME|ATTR_ATIME); + } + spin_unlock(&inode->i_lock); + } else if (nfs_have_delegated_atime(inode) && + attr->ia_valid & ATTR_ATIME && + !(attr->ia_valid & ATTR_MTIME)) { + if (!(attr->ia_valid & ATTR_ATIME_SET)) { + nfs_update_delegated_atime(inode); + attr->ia_valid &= ~ATTR_ATIME; + } + } + + /* Optimization: if the end result is no change, don't RPC */ + if (((attr->ia_valid & NFS_VALID_ATTRS) & ~(ATTR_FILE|ATTR_OPEN)) == 0) + return 0; + + trace_nfs_setattr_enter(inode); + + /* Write all dirty data */ + if (S_ISREG(inode->i_mode)) + nfs_sync_inode(inode); + + fattr = nfs_alloc_fattr_with_label(NFS_SERVER(inode)); + if (fattr == NULL) { + error = -ENOMEM; + goto out; + } + + error = NFS_PROTO(inode)->setattr(dentry, fattr, attr); + if (error == 0) { + if (attr->ia_valid & ATTR_SIZE) + nfs_truncate_last_folio(inode->i_mapping, oldsize, + attr->ia_size); + error = nfs_refresh_inode(inode, fattr); + } + nfs_free_fattr(fattr); +out: + trace_nfs_setattr_exit(inode, error); + return error; +} +EXPORT_SYMBOL_GPL(nfs_setattr); + +/** + * nfs_vmtruncate - unmap mappings "freed" by truncate() syscall + * @inode: inode of the file used + * @offset: file offset to start truncating + * + * This is a copy of the common vmtruncate, but with the locking + * corrected to take into account the fact that NFS requires + * inode->i_size to be updated under the inode->i_lock. + * Note: must be called with inode->i_lock held! + */ +static int nfs_vmtruncate(struct inode * inode, loff_t offset) +{ + int err; + + err = inode_newsize_ok(inode, offset); + if (err) + goto out; + + trace_nfs_size_truncate(inode, offset); + i_size_write(inode, offset); + /* Optimisation */ + if (offset == 0) { + NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_DATA; + nfs_ooo_clear(NFS_I(inode)); + } + NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_SIZE; + + spin_unlock(&inode->i_lock); + truncate_pagecache(inode, offset); + nfs_update_delegated_mtime_locked(inode); + spin_lock(&inode->i_lock); +out: + return err; +} + +/** + * nfs_setattr_update_inode - Update inode metadata after a setattr call. + * @inode: pointer to struct inode + * @attr: pointer to struct iattr + * @fattr: pointer to struct nfs_fattr + * + * Note: we do this in the *proc.c in order to ensure that + * it works for things like exclusive creates too. + */ +void nfs_setattr_update_inode(struct inode *inode, struct iattr *attr, + struct nfs_fattr *fattr) +{ + /* Barrier: bump the attribute generation count. */ + nfs_fattr_set_barrier(fattr); + + spin_lock(&inode->i_lock); + NFS_I(inode)->attr_gencount = fattr->gencount; + if ((attr->ia_valid & ATTR_SIZE) != 0) { + if (!nfs_have_delegated_mtime(inode)) + nfs_set_cache_invalid(inode, NFS_INO_INVALID_MTIME); + nfs_set_cache_invalid(inode, NFS_INO_INVALID_BLOCKS); + nfs_inc_stats(inode, NFSIOS_SETATTRTRUNC); + nfs_vmtruncate(inode, attr->ia_size); + } + if ((attr->ia_valid & (ATTR_MODE|ATTR_UID|ATTR_GID)) != 0) { + NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_CTIME; + if ((attr->ia_valid & ATTR_KILL_SUID) != 0 && + inode->i_mode & S_ISUID) + inode->i_mode &= ~S_ISUID; + if (setattr_should_drop_sgid(&nop_mnt_idmap, inode)) + inode->i_mode &= ~S_ISGID; + if ((attr->ia_valid & ATTR_MODE) != 0) { + int mode = attr->ia_mode & S_IALLUGO; + mode |= inode->i_mode & ~S_IALLUGO; + inode->i_mode = mode; + } + if ((attr->ia_valid & ATTR_UID) != 0) + inode->i_uid = attr->ia_uid; + if ((attr->ia_valid & ATTR_GID) != 0) + inode->i_gid = attr->ia_gid; + if (fattr->valid & NFS_ATTR_FATTR_CTIME) + inode_set_ctime_to_ts(inode, fattr->ctime); + else + nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE + | NFS_INO_INVALID_CTIME); + nfs_set_cache_invalid(inode, NFS_INO_INVALID_ACCESS + | NFS_INO_INVALID_ACL); + } + if (attr->ia_valid & (ATTR_ATIME_SET|ATTR_ATIME)) { + NFS_I(inode)->cache_validity &= ~(NFS_INO_INVALID_ATIME + | NFS_INO_INVALID_CTIME); + if (fattr->valid & NFS_ATTR_FATTR_ATIME) + inode_set_atime_to_ts(inode, fattr->atime); + else if (attr->ia_valid & ATTR_ATIME_SET) + inode_set_atime_to_ts(inode, attr->ia_atime); + else + nfs_set_cache_invalid(inode, NFS_INO_INVALID_ATIME); + + if (fattr->valid & NFS_ATTR_FATTR_CTIME) + inode_set_ctime_to_ts(inode, fattr->ctime); + else + nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE + | NFS_INO_INVALID_CTIME); + } + if (attr->ia_valid & (ATTR_MTIME_SET|ATTR_MTIME)) { + NFS_I(inode)->cache_validity &= ~(NFS_INO_INVALID_MTIME + | NFS_INO_INVALID_CTIME); + if (fattr->valid & NFS_ATTR_FATTR_MTIME) + inode_set_mtime_to_ts(inode, fattr->mtime); + else if (attr->ia_valid & ATTR_MTIME_SET) + inode_set_mtime_to_ts(inode, attr->ia_mtime); + else + nfs_set_cache_invalid(inode, NFS_INO_INVALID_MTIME); + + if (fattr->valid & NFS_ATTR_FATTR_CTIME) + inode_set_ctime_to_ts(inode, fattr->ctime); + else + nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE + | NFS_INO_INVALID_CTIME); + } + if (fattr->valid) + nfs_update_inode(inode, fattr); + spin_unlock(&inode->i_lock); +} +EXPORT_SYMBOL_GPL(nfs_setattr_update_inode); + +/* + * Don't request help from readdirplus if the file is being written to, + * or if attribute caching is turned off + */ +static bool nfs_getattr_readdirplus_enable(const struct inode *inode) +{ + return nfs_server_capable(inode, NFS_CAP_READDIRPLUS) && + !nfs_have_writebacks(inode) && NFS_MAXATTRTIMEO(inode) > 5 * HZ; +} + +static void nfs_readdirplus_parent_cache_miss(struct dentry *dentry) +{ + if (!IS_ROOT(dentry)) { + struct dentry *parent = dget_parent(dentry); + nfs_readdir_record_entry_cache_miss(d_inode(parent)); + dput(parent); + } +} + +static void nfs_readdirplus_parent_cache_hit(struct dentry *dentry) +{ + if (!IS_ROOT(dentry)) { + struct dentry *parent = dget_parent(dentry); + nfs_readdir_record_entry_cache_hit(d_inode(parent)); + dput(parent); + } +} + +static u32 nfs_get_valid_attrmask(struct inode *inode) +{ + u64 fattr_valid = NFS_SERVER(inode)->fattr_valid; + unsigned long cache_validity = READ_ONCE(NFS_I(inode)->cache_validity); + u32 reply_mask = STATX_INO | STATX_TYPE; + + if (!(cache_validity & NFS_INO_INVALID_ATIME)) + reply_mask |= STATX_ATIME; + if (!(cache_validity & NFS_INO_INVALID_CTIME)) + reply_mask |= STATX_CTIME; + if (!(cache_validity & NFS_INO_INVALID_MTIME)) + reply_mask |= STATX_MTIME; + if (!(cache_validity & NFS_INO_INVALID_SIZE)) + reply_mask |= STATX_SIZE; + if (!(cache_validity & NFS_INO_INVALID_NLINK)) + reply_mask |= STATX_NLINK; + if (!(cache_validity & NFS_INO_INVALID_MODE)) + reply_mask |= STATX_MODE; + if (!(cache_validity & NFS_INO_INVALID_OTHER)) + reply_mask |= STATX_UID | STATX_GID; + if (!(cache_validity & NFS_INO_INVALID_BLOCKS)) + reply_mask |= STATX_BLOCKS; + if (!(cache_validity & NFS_INO_INVALID_BTIME) && + (fattr_valid & NFS_ATTR_FATTR_BTIME)) + reply_mask |= STATX_BTIME; + if (!(cache_validity & NFS_INO_INVALID_CHANGE)) + reply_mask |= STATX_CHANGE_COOKIE; + return reply_mask; +} + +int nfs_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int query_flags) +{ + struct inode *inode = d_inode(path->dentry); + struct nfs_server *server = NFS_SERVER(inode); + u64 fattr_valid = server->fattr_valid; + unsigned long cache_validity; + int err = 0; + bool force_sync = query_flags & AT_STATX_FORCE_SYNC; + bool do_update = false; + bool readdirplus_enabled = nfs_getattr_readdirplus_enable(inode); + + trace_nfs_getattr_enter(inode); + + request_mask &= STATX_TYPE | STATX_MODE | STATX_NLINK | STATX_UID | + STATX_GID | STATX_ATIME | STATX_MTIME | STATX_CTIME | + STATX_INO | STATX_SIZE | STATX_BLOCKS | STATX_BTIME | + STATX_CHANGE_COOKIE; + + if (!(fattr_valid & NFS_ATTR_FATTR_BTIME)) + request_mask &= ~STATX_BTIME; + + if ((query_flags & AT_STATX_DONT_SYNC) && !force_sync) { + if (readdirplus_enabled) + nfs_readdirplus_parent_cache_hit(path->dentry); + goto out_no_revalidate; + } + + /* Flush out writes to the server in order to update c/mtime/version. */ + if ((request_mask & (STATX_CTIME | STATX_MTIME | STATX_CHANGE_COOKIE)) && + S_ISREG(inode->i_mode)) { + if (nfs_have_delegated_mtime(inode)) + filemap_fdatawrite(inode->i_mapping); + else + filemap_write_and_wait(inode->i_mapping); + } + + /* + * We may force a getattr if the user cares about atime. + * + * Note that we only have to check the vfsmount flags here: + * - NFS always sets S_NOATIME by so checking it would give a + * bogus result + * - NFS never sets SB_NOATIME or SB_NODIRATIME so there is + * no point in checking those. + */ + if ((path->mnt->mnt_flags & MNT_NOATIME) || + ((path->mnt->mnt_flags & MNT_NODIRATIME) && S_ISDIR(inode->i_mode))) + request_mask &= ~STATX_ATIME; + + /* Is the user requesting attributes that might need revalidation? */ + if (!(request_mask & (STATX_MODE|STATX_NLINK|STATX_ATIME|STATX_CTIME| + STATX_MTIME|STATX_UID|STATX_GID| + STATX_SIZE|STATX_BLOCKS|STATX_BTIME| + STATX_CHANGE_COOKIE))) + goto out_no_revalidate; + + /* Check whether the cached attributes are stale */ + do_update |= force_sync || nfs_attribute_cache_expired(inode); + cache_validity = READ_ONCE(NFS_I(inode)->cache_validity); + do_update |= cache_validity & NFS_INO_INVALID_CHANGE; + if (request_mask & STATX_ATIME) + do_update |= cache_validity & NFS_INO_INVALID_ATIME; + if (request_mask & STATX_CTIME) + do_update |= cache_validity & NFS_INO_INVALID_CTIME; + if (request_mask & STATX_MTIME) + do_update |= cache_validity & NFS_INO_INVALID_MTIME; + if (request_mask & STATX_SIZE) + do_update |= cache_validity & NFS_INO_INVALID_SIZE; + if (request_mask & STATX_NLINK) + do_update |= cache_validity & NFS_INO_INVALID_NLINK; + if (request_mask & STATX_MODE) + do_update |= cache_validity & NFS_INO_INVALID_MODE; + if (request_mask & (STATX_UID | STATX_GID)) + do_update |= cache_validity & NFS_INO_INVALID_OTHER; + if (request_mask & STATX_BLOCKS) + do_update |= cache_validity & NFS_INO_INVALID_BLOCKS; + if (request_mask & STATX_BTIME) + do_update |= cache_validity & NFS_INO_INVALID_BTIME; + + if (do_update) { + if (readdirplus_enabled) + nfs_readdirplus_parent_cache_miss(path->dentry); + err = __nfs_revalidate_inode(server, inode); + if (err) + goto out; + } else if (readdirplus_enabled) + nfs_readdirplus_parent_cache_hit(path->dentry); +out_no_revalidate: + /* Only return attributes that were revalidated. */ + stat->result_mask = nfs_get_valid_attrmask(inode) | request_mask; + + generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat); + stat->change_cookie = inode_peek_iversion_raw(inode); + stat->attributes_mask |= STATX_ATTR_CHANGE_MONOTONIC; + if (server->change_attr_type != NFS4_CHANGE_TYPE_IS_UNDEFINED) + stat->attributes |= STATX_ATTR_CHANGE_MONOTONIC; + if (S_ISDIR(inode->i_mode)) + stat->blksize = NFS_SERVER(inode)->dtsize; + stat->btime = NFS_I(inode)->btime; + + /* Special handling for STATX_DIOALIGN and STATX_DIO_READ_ALIGN + * - NFS doesn't have DIO alignment constraints, avoid getting + * these DIO attrs from remote and just respond with most + * accommodating limits (so client will issue supported DIO). + * - this is unintuitive, but the most coarse-grained + * dio_offset_align is the most accommodating. + */ + if ((request_mask & (STATX_DIOALIGN | STATX_DIO_READ_ALIGN)) && + S_ISREG(inode->i_mode)) { + stat->result_mask |= STATX_DIOALIGN | STATX_DIO_READ_ALIGN; + stat->dio_mem_align = 4; /* 4-byte alignment */ + stat->dio_offset_align = PAGE_SIZE; + stat->dio_read_offset_align = stat->dio_offset_align; + } +out: + trace_nfs_getattr_exit(inode, err); + return err; +} +EXPORT_SYMBOL_GPL(nfs_getattr); + +int nfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + + if (nfs_server_capable(inode, NFS_CAP_CASE_INSENSITIVE)) { + fa->fsx_xflags |= FS_XFLAG_CASEFOLD; + fa->flags |= FS_CASEFOLD_FL; + } + if (nfs_server_capable(inode, NFS_CAP_CASE_NONPRESERVING)) + fa->fsx_xflags |= FS_XFLAG_CASENONPRESERVING; + return 0; +} +EXPORT_SYMBOL_GPL(nfs_fileattr_get); + +static void nfs_init_lock_context(struct nfs_lock_context *l_ctx) +{ + refcount_set(&l_ctx->count, 1); + l_ctx->lockowner = current->files; + INIT_LIST_HEAD(&l_ctx->list); + atomic_set(&l_ctx->io_count, 0); +} + +static struct nfs_lock_context *__nfs_find_lock_context(struct nfs_open_context *ctx) +{ + struct nfs_lock_context *pos; + + list_for_each_entry_rcu(pos, &ctx->lock_context.list, list) { + if (pos->lockowner != current->files) + continue; + if (refcount_inc_not_zero(&pos->count)) + return pos; + } + return NULL; +} + +struct nfs_lock_context *nfs_get_lock_context(struct nfs_open_context *ctx) +{ + struct nfs_lock_context *res, *new = NULL; + struct inode *inode = d_inode(ctx->dentry); + + rcu_read_lock(); + res = __nfs_find_lock_context(ctx); + rcu_read_unlock(); + if (res == NULL) { + new = kmalloc_obj(*new, GFP_KERNEL_ACCOUNT); + if (new == NULL) + return ERR_PTR(-ENOMEM); + nfs_init_lock_context(new); + spin_lock(&inode->i_lock); + res = __nfs_find_lock_context(ctx); + if (res == NULL) { + new->open_context = get_nfs_open_context(ctx); + if (new->open_context) { + list_add_tail_rcu(&new->list, + &ctx->lock_context.list); + res = new; + new = NULL; + } else + res = ERR_PTR(-EBADF); + } + spin_unlock(&inode->i_lock); + kfree(new); + } + return res; +} +EXPORT_SYMBOL_GPL(nfs_get_lock_context); + +void nfs_put_lock_context(struct nfs_lock_context *l_ctx) +{ + struct nfs_open_context *ctx = l_ctx->open_context; + struct inode *inode = d_inode(ctx->dentry); + + if (!refcount_dec_and_lock(&l_ctx->count, &inode->i_lock)) + return; + list_del_rcu(&l_ctx->list); + spin_unlock(&inode->i_lock); + put_nfs_open_context(ctx); + kfree_rcu(l_ctx, rcu_head); +} +EXPORT_SYMBOL_GPL(nfs_put_lock_context); + +/** + * nfs_close_context - Common close_context() routine NFSv2/v3 + * @ctx: pointer to context + * @is_sync: is this a synchronous close + * + * Ensure that the attributes are up to date if we're mounted + * with close-to-open semantics and we have cached data that will + * need to be revalidated on open. + */ +void nfs_close_context(struct nfs_open_context *ctx, int is_sync) +{ + struct nfs_inode *nfsi; + struct inode *inode; + + if (!(ctx->mode & FMODE_WRITE)) + return; + if (!is_sync) + return; + inode = d_inode(ctx->dentry); + if (nfs_have_read_or_write_delegation(inode)) + return; + nfsi = NFS_I(inode); + if (inode->i_mapping->nrpages == 0) + return; + if (nfsi->cache_validity & NFS_INO_INVALID_DATA) + return; + if (!list_empty(&nfsi->open_files)) + return; + if (NFS_SERVER(inode)->flags & NFS_MOUNT_NOCTO) + return; + nfs_revalidate_inode(inode, + NFS_INO_INVALID_CHANGE | NFS_INO_INVALID_SIZE); +} +EXPORT_SYMBOL_GPL(nfs_close_context); + +struct nfs_open_context *alloc_nfs_open_context(struct dentry *dentry, + fmode_t f_mode, + struct file *filp) +{ + struct nfs_open_context *ctx; + + ctx = kmalloc_obj(*ctx, GFP_KERNEL_ACCOUNT); + if (!ctx) + return ERR_PTR(-ENOMEM); + nfs_sb_active(dentry->d_sb); + ctx->dentry = dget(dentry); + if (filp) + ctx->cred = get_cred(filp->f_cred); + else + ctx->cred = get_current_cred(); + rcu_assign_pointer(ctx->ll_cred, NULL); + ctx->state = NULL; + ctx->mode = f_mode; + ctx->flags = 0; + ctx->error = 0; + ctx->flock_owner = (fl_owner_t)filp; + nfs_init_lock_context(&ctx->lock_context); + ctx->lock_context.open_context = ctx; + INIT_LIST_HEAD(&ctx->list); + ctx->mdsthreshold = NULL; + nfs_localio_file_init(&ctx->nfl); + + return ctx; +} +EXPORT_SYMBOL_GPL(alloc_nfs_open_context); + +struct nfs_open_context *get_nfs_open_context(struct nfs_open_context *ctx) +{ + if (ctx != NULL && refcount_inc_not_zero(&ctx->lock_context.count)) + return ctx; + return NULL; +} +EXPORT_SYMBOL_GPL(get_nfs_open_context); + +static void __put_nfs_open_context(struct nfs_open_context *ctx, int is_sync) +{ + struct inode *inode = d_inode(ctx->dentry); + struct super_block *sb = ctx->dentry->d_sb; + + if (!refcount_dec_and_test(&ctx->lock_context.count)) + return; + if (!list_empty(&ctx->list)) { + spin_lock(&inode->i_lock); + list_del_rcu(&ctx->list); + spin_unlock(&inode->i_lock); + } + if (inode != NULL) + NFS_PROTO(inode)->close_context(ctx, is_sync); + put_cred(ctx->cred); + dput(ctx->dentry); + nfs_sb_deactive(sb); + put_rpccred(rcu_dereference_protected(ctx->ll_cred, 1)); + kfree(ctx->mdsthreshold); + nfs_close_local_fh(&ctx->nfl); + kfree_rcu(ctx, rcu_head); +} + +void put_nfs_open_context(struct nfs_open_context *ctx) +{ + __put_nfs_open_context(ctx, 0); +} +EXPORT_SYMBOL_GPL(put_nfs_open_context); + +static void put_nfs_open_context_sync(struct nfs_open_context *ctx) +{ + __put_nfs_open_context(ctx, 1); +} + +/* + * Ensure that mmap has a recent RPC credential for use when writing out + * shared pages + */ +void nfs_inode_attach_open_context(struct nfs_open_context *ctx) +{ + struct inode *inode = d_inode(ctx->dentry); + struct nfs_inode *nfsi = NFS_I(inode); + + spin_lock(&inode->i_lock); + if (list_empty(&nfsi->open_files) && + nfs_ooo_test(nfsi)) + nfs_set_cache_invalid(inode, NFS_INO_INVALID_DATA | + NFS_INO_REVAL_FORCED); + list_add_tail_rcu(&ctx->list, &nfsi->open_files); + spin_unlock(&inode->i_lock); +} +EXPORT_SYMBOL_GPL(nfs_inode_attach_open_context); + +void nfs_file_set_open_context(struct file *filp, struct nfs_open_context *ctx) +{ + filp->private_data = get_nfs_open_context(ctx); + set_bit(NFS_CONTEXT_FILE_OPEN, &ctx->flags); + if (list_empty(&ctx->list)) + nfs_inode_attach_open_context(ctx); +} +EXPORT_SYMBOL_GPL(nfs_file_set_open_context); + +/* + * Given an inode, search for an open context with the desired characteristics + */ +struct nfs_open_context *nfs_find_open_context(struct inode *inode, const struct cred *cred, fmode_t mode) +{ + struct nfs_inode *nfsi = NFS_I(inode); + struct nfs_open_context *pos, *ctx = NULL; + + rcu_read_lock(); + list_for_each_entry_rcu(pos, &nfsi->open_files, list) { + if (cred != NULL && cred_fscmp(pos->cred, cred) != 0) + continue; + if ((pos->mode & (FMODE_READ|FMODE_WRITE)) != mode) + continue; + if (!test_bit(NFS_CONTEXT_FILE_OPEN, &pos->flags)) + continue; + ctx = get_nfs_open_context(pos); + if (ctx) + break; + } + rcu_read_unlock(); + return ctx; +} + +void nfs_file_clear_open_context(struct file *filp) +{ + struct nfs_open_context *ctx = nfs_file_open_context(filp); + + if (ctx) { + struct inode *inode = d_inode(ctx->dentry); + + clear_bit(NFS_CONTEXT_FILE_OPEN, &ctx->flags); + /* + * We fatal error on write before. Try to writeback + * every page again. + */ + if (ctx->error < 0) + invalidate_inode_pages2(inode->i_mapping); + filp->private_data = NULL; + put_nfs_open_context_sync(ctx); + } +} + +/* + * These allocate and release file read/write context information. + */ +int nfs_open(struct inode *inode, struct file *filp) +{ + struct nfs_open_context *ctx; + + ctx = alloc_nfs_open_context(file_dentry(filp), + flags_to_mode(filp->f_flags), filp); + if (IS_ERR(ctx)) + return PTR_ERR(ctx); + nfs_file_set_open_context(filp, ctx); + put_nfs_open_context(ctx); + nfs_fscache_open_file(inode, filp); + return 0; +} + +/* + * This function is called whenever some part of NFS notices that + * the cached attributes have to be refreshed. + */ +int +__nfs_revalidate_inode(struct nfs_server *server, struct inode *inode) +{ + int status = -ESTALE; + struct nfs_fattr *fattr = NULL; + struct nfs_inode *nfsi = NFS_I(inode); + + dfprintk(PAGECACHE, "NFS: revalidating (%s/%Lu)\n", + inode->i_sb->s_id, (unsigned long long)inode->i_ino); + + trace_nfs_revalidate_inode_enter(inode); + + if (is_bad_inode(inode)) + goto out; + if (NFS_STALE(inode)) + goto out; + + /* pNFS: Attributes aren't updated until we layoutcommit */ + if (S_ISREG(inode->i_mode)) { + status = pnfs_sync_inode(inode, false); + if (status) + goto out; + } else if (nfs_have_directory_delegation(inode) && + !(NFS_I(inode)->cache_validity & NFS_INO_INVALID_ATTR)) { + status = 0; + goto out; + } + + status = -ENOMEM; + fattr = nfs_alloc_fattr_with_label(NFS_SERVER(inode)); + if (fattr == NULL) + goto out; + + nfs_inc_stats(inode, NFSIOS_INODEREVALIDATE); + + status = NFS_PROTO(inode)->getattr(server, NFS_FH(inode), fattr, inode); + if (status != 0) { + dfprintk(PAGECACHE, "nfs_revalidate_inode: (%s/%Lu) getattr failed, error=%d\n", + inode->i_sb->s_id, + (unsigned long long)inode->i_ino, status); + switch (status) { + case -ETIMEDOUT: + /* A soft timeout occurred. Use cached information? */ + if (server->flags & NFS_MOUNT_SOFTREVAL) + status = 0; + break; + case -ESTALE: + if (!S_ISDIR(inode->i_mode)) + nfs_set_inode_stale(inode); + else + nfs_zap_caches(inode); + } + goto out; + } + + status = nfs_refresh_inode(inode, fattr); + if (status) { + dfprintk(PAGECACHE, "nfs_revalidate_inode: (%s/%Lu) refresh failed, error=%d\n", + inode->i_sb->s_id, + (unsigned long long)inode->i_ino, status); + goto out; + } + + if (nfsi->cache_validity & NFS_INO_INVALID_ACL) + nfs_zap_acl_cache(inode); + + nfs_setsecurity(inode, fattr); + + dfprintk(PAGECACHE, "NFS: (%s/%Lu) revalidation complete\n", + inode->i_sb->s_id, + (unsigned long long)inode->i_ino); + +out: + nfs_free_fattr(fattr); + trace_nfs_revalidate_inode_exit(inode, status); + return status; +} + +int nfs_attribute_cache_expired(struct inode *inode) +{ + if (nfs_have_delegated_attributes(inode)) + return 0; + return nfs_attribute_timeout(inode); +} + +/** + * nfs_revalidate_inode - Revalidate the inode attributes + * @inode: pointer to inode struct + * @flags: cache flags to check + * + * Updates inode attribute information by retrieving the data from the server. + */ +int nfs_revalidate_inode(struct inode *inode, unsigned long flags) +{ + if (!nfs_check_cache_invalid(inode, flags)) + return NFS_STALE(inode) ? -ESTALE : 0; + return __nfs_revalidate_inode(NFS_SERVER(inode), inode); +} +EXPORT_SYMBOL_GPL(nfs_revalidate_inode); + +static int nfs_invalidate_mapping(struct inode *inode, struct address_space *mapping) +{ + int ret; + + nfs_fscache_invalidate(inode, 0); + if (mapping->nrpages != 0) { + if (S_ISREG(inode->i_mode)) { + ret = nfs_sync_mapping(mapping); + if (ret < 0) + return ret; + } + ret = invalidate_inode_pages2(mapping); + if (ret < 0) + return ret; + } + nfs_inc_stats(inode, NFSIOS_DATAINVALIDATE); + + dfprintk(PAGECACHE, "NFS: (%s/%Lu) data cache invalidated\n", + inode->i_sb->s_id, + (unsigned long long)inode->i_ino); + return 0; +} + +/** + * nfs_clear_invalid_mapping - Conditionally clear a mapping + * @mapping: pointer to mapping + * + * If the NFS_INO_INVALID_DATA inode flag is set, clear the mapping. + */ +int nfs_clear_invalid_mapping(struct address_space *mapping) +{ + struct inode *inode = mapping->host; + struct nfs_inode *nfsi = NFS_I(inode); + unsigned long *bitlock = &nfsi->flags; + int ret = 0; + + /* + * We must clear NFS_INO_INVALID_DATA first to ensure that + * invalidations that come in while we're shooting down the mappings + * are respected. But, that leaves a race window where one revalidator + * can clear the flag, and then another checks it before the mapping + * gets invalidated. Fix that by serializing access to this part of + * the function. + * + * At the same time, we need to allow other tasks to see whether we + * might be in the middle of invalidating the pages, so we only set + * the bit lock here if it looks like we're going to be doing that. + */ + for (;;) { + ret = wait_on_bit_action(bitlock, NFS_INO_INVALIDATING, + nfs_wait_bit_killable, + TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); + if (ret) + goto out; + smp_rmb(); /* pairs with smp_wmb() below */ + if (test_bit(NFS_INO_INVALIDATING, bitlock)) + continue; + /* pairs with nfs_set_cache_invalid()'s smp_store_release() */ + if (!(smp_load_acquire(&nfsi->cache_validity) & NFS_INO_INVALID_DATA)) + goto out; + /* Slow-path that double-checks with spinlock held */ + spin_lock(&inode->i_lock); + if (test_bit(NFS_INO_INVALIDATING, bitlock)) { + spin_unlock(&inode->i_lock); + continue; + } + if (nfsi->cache_validity & NFS_INO_INVALID_DATA) + break; + spin_unlock(&inode->i_lock); + goto out; + } + + set_bit(NFS_INO_INVALIDATING, bitlock); + smp_wmb(); + nfsi->cache_validity &= ~NFS_INO_INVALID_DATA; + nfs_ooo_clear(nfsi); + spin_unlock(&inode->i_lock); + trace_nfs_invalidate_mapping_enter(inode); + ret = nfs_invalidate_mapping(inode, mapping); + trace_nfs_invalidate_mapping_exit(inode, ret); + + clear_bit_unlock(NFS_INO_INVALIDATING, bitlock); + smp_mb__after_atomic(); + wake_up_bit(bitlock, NFS_INO_INVALIDATING); +out: + return ret; +} + +bool nfs_mapping_need_revalidate_inode(struct inode *inode) +{ + return nfs_check_cache_invalid(inode, NFS_INO_INVALID_CHANGE) || + NFS_STALE(inode); +} + +int nfs_revalidate_mapping_rcu(struct inode *inode) +{ + struct nfs_inode *nfsi = NFS_I(inode); + unsigned long *bitlock = &nfsi->flags; + int ret = 0; + + if (IS_SWAPFILE(inode)) + goto out; + if (nfs_mapping_need_revalidate_inode(inode)) { + ret = -ECHILD; + goto out; + } + spin_lock(&inode->i_lock); + if (test_bit(NFS_INO_INVALIDATING, bitlock) || + (nfsi->cache_validity & NFS_INO_INVALID_DATA)) + ret = -ECHILD; + spin_unlock(&inode->i_lock); +out: + return ret; +} + +/** + * nfs_revalidate_mapping - Revalidate the pagecache + * @inode: pointer to host inode + * @mapping: pointer to mapping + */ +int nfs_revalidate_mapping(struct inode *inode, struct address_space *mapping) +{ + /* swapfiles are not supposed to be shared. */ + if (IS_SWAPFILE(inode)) + return 0; + + if (nfs_mapping_need_revalidate_inode(inode)) { + int ret = __nfs_revalidate_inode(NFS_SERVER(inode), inode); + if (ret < 0) + return ret; + } + + return nfs_clear_invalid_mapping(mapping); +} + +static bool nfs_file_has_writers(struct nfs_inode *nfsi) +{ + struct inode *inode = &nfsi->vfs_inode; + + if (!S_ISREG(inode->i_mode)) + return false; + if (list_empty(&nfsi->open_files)) + return false; + return inode_is_open_for_write(inode); +} + +static bool nfs_file_has_buffered_writers(struct nfs_inode *nfsi) +{ + return nfs_file_has_writers(nfsi) && nfs_file_io_is_buffered(nfsi); +} + +static void nfs_wcc_update_inode(struct inode *inode, struct nfs_fattr *fattr) +{ + struct timespec64 ts; + + if ((fattr->valid & NFS_ATTR_FATTR_PRECHANGE) + && (fattr->valid & NFS_ATTR_FATTR_CHANGE) + && inode_eq_iversion_raw(inode, fattr->pre_change_attr)) { + inode_set_iversion_raw(inode, fattr->change_attr); + if (S_ISDIR(inode->i_mode)) + nfs_set_cache_invalid(inode, NFS_INO_INVALID_DATA); + else if (nfs_server_capable(inode, NFS_CAP_XATTR)) + nfs_set_cache_invalid(inode, NFS_INO_INVALID_XATTR); + } + /* If we have atomic WCC data, we may update some attributes */ + ts = inode_get_ctime(inode); + if ((fattr->valid & NFS_ATTR_FATTR_PRECTIME) + && (fattr->valid & NFS_ATTR_FATTR_CTIME) + && timespec64_equal(&ts, &fattr->pre_ctime)) { + inode_set_ctime_to_ts(inode, fattr->ctime); + } + + ts = inode_get_mtime(inode); + if ((fattr->valid & NFS_ATTR_FATTR_PREMTIME) + && (fattr->valid & NFS_ATTR_FATTR_MTIME) + && timespec64_equal(&ts, &fattr->pre_mtime)) { + inode_set_mtime_to_ts(inode, fattr->mtime); + } + if ((fattr->valid & NFS_ATTR_FATTR_PRESIZE) + && (fattr->valid & NFS_ATTR_FATTR_SIZE) + && i_size_read(inode) == nfs_size_to_loff_t(fattr->pre_size) + && !nfs_have_writebacks(inode)) { + trace_nfs_size_wcc(inode, fattr->size); + i_size_write(inode, nfs_size_to_loff_t(fattr->size)); + } +} + +/** + * nfs_check_inode_attributes - verify consistency of the inode attribute cache + * @inode: pointer to inode + * @fattr: updated attributes + * + * Verifies the attribute cache. If we have just changed the attributes, + * so that fattr carries weak cache consistency data, then it may + * also update the ctime/mtime/change_attribute. + */ +static int nfs_check_inode_attributes(struct inode *inode, struct nfs_fattr *fattr) +{ + struct nfs_inode *nfsi = NFS_I(inode); + loff_t cur_size, new_isize; + unsigned long invalid = 0; + struct timespec64 ts; + + if (nfs_have_delegated_attributes(inode)) + return 0; + + if (!(fattr->valid & NFS_ATTR_FATTR_FILEID)) { + /* Only a mounted-on-fileid? Just exit */ + if (fattr->valid & NFS_ATTR_FATTR_MOUNTED_ON_FILEID) + return 0; + /* Has the inode gone and changed behind our back? */ + } else if (inode->i_ino != fattr->fileid) { + /* Is this perhaps the mounted-on fileid? */ + if ((fattr->valid & NFS_ATTR_FATTR_MOUNTED_ON_FILEID) && + inode->i_ino == fattr->mounted_on_fileid) + return 0; + return -ESTALE; + } + if ((fattr->valid & NFS_ATTR_FATTR_TYPE) && inode_wrong_type(inode, fattr->mode)) + return -ESTALE; + + + if (!nfs_file_has_buffered_writers(nfsi)) { + /* Verify a few of the more important attributes */ + if ((fattr->valid & NFS_ATTR_FATTR_CHANGE) != 0 && !inode_eq_iversion_raw(inode, fattr->change_attr)) + invalid |= NFS_INO_INVALID_CHANGE; + + ts = inode_get_mtime(inode); + if ((fattr->valid & NFS_ATTR_FATTR_MTIME) && !timespec64_equal(&ts, &fattr->mtime)) + invalid |= NFS_INO_INVALID_MTIME; + + ts = inode_get_ctime(inode); + if ((fattr->valid & NFS_ATTR_FATTR_CTIME) && !timespec64_equal(&ts, &fattr->ctime)) + invalid |= NFS_INO_INVALID_CTIME; + + if (fattr->valid & NFS_ATTR_FATTR_SIZE) { + cur_size = i_size_read(inode); + new_isize = nfs_size_to_loff_t(fattr->size); + if (cur_size != new_isize) + invalid |= NFS_INO_INVALID_SIZE; + } + } + + /* Have any file permissions changed? */ + if ((fattr->valid & NFS_ATTR_FATTR_MODE) && (inode->i_mode & S_IALLUGO) != (fattr->mode & S_IALLUGO)) + invalid |= NFS_INO_INVALID_MODE; + if ((fattr->valid & NFS_ATTR_FATTR_OWNER) && !uid_eq(inode->i_uid, fattr->uid)) + invalid |= NFS_INO_INVALID_OTHER; + if ((fattr->valid & NFS_ATTR_FATTR_GROUP) && !gid_eq(inode->i_gid, fattr->gid)) + invalid |= NFS_INO_INVALID_OTHER; + + /* Has the link count changed? */ + if ((fattr->valid & NFS_ATTR_FATTR_NLINK) && inode->i_nlink != fattr->nlink) + invalid |= NFS_INO_INVALID_NLINK; + + ts = inode_get_atime(inode); + if ((fattr->valid & NFS_ATTR_FATTR_ATIME) && !timespec64_equal(&ts, &fattr->atime)) + invalid |= NFS_INO_INVALID_ATIME; + + if (invalid != 0) + nfs_set_cache_invalid(inode, invalid); + + nfsi->read_cache_jiffies = fattr->time_start; + return 0; +} + +static atomic_long_t nfs_attr_generation_counter; + +static unsigned long nfs_read_attr_generation_counter(void) +{ + return atomic_long_read(&nfs_attr_generation_counter); +} + +unsigned long nfs_inc_attr_generation_counter(void) +{ + return atomic_long_inc_return(&nfs_attr_generation_counter); +} +EXPORT_SYMBOL_GPL(nfs_inc_attr_generation_counter); + +void nfs_fattr_init(struct nfs_fattr *fattr) +{ + fattr->valid = 0; + fattr->time_start = jiffies; + fattr->gencount = nfs_inc_attr_generation_counter(); + fattr->owner_name = NULL; + fattr->group_name = NULL; + fattr->mdsthreshold = NULL; +} +EXPORT_SYMBOL_GPL(nfs_fattr_init); + +/** + * nfs_fattr_set_barrier + * @fattr: attributes + * + * Used to set a barrier after an attribute was updated. This + * barrier ensures that older attributes from RPC calls that may + * have raced with our update cannot clobber these new values. + * Note that you are still responsible for ensuring that other + * operations which change the attribute on the server do not + * collide. + */ +void nfs_fattr_set_barrier(struct nfs_fattr *fattr) +{ + fattr->gencount = nfs_inc_attr_generation_counter(); +} + +struct nfs_fattr *nfs_alloc_fattr(void) +{ + struct nfs_fattr *fattr; + + fattr = kmalloc_obj(*fattr); + if (fattr != NULL) { + nfs_fattr_init(fattr); + fattr->label = NULL; + } + return fattr; +} +EXPORT_SYMBOL_GPL(nfs_alloc_fattr); + +struct nfs_fattr *nfs_alloc_fattr_with_label(struct nfs_server *server) +{ + struct nfs_fattr *fattr = nfs_alloc_fattr(); + + if (!fattr) + return NULL; + + fattr->label = nfs4_label_alloc(server, GFP_KERNEL); + if (IS_ERR(fattr->label)) { + kfree(fattr); + return NULL; + } + + return fattr; +} +EXPORT_SYMBOL_GPL(nfs_alloc_fattr_with_label); + +struct nfs_fh *nfs_alloc_fhandle(void) +{ + struct nfs_fh *fh; + + fh = kmalloc_obj(struct nfs_fh); + if (fh != NULL) + fh->size = 0; + return fh; +} +EXPORT_SYMBOL_GPL(nfs_alloc_fhandle); + +#ifdef NFS_DEBUG +/* + * _nfs_display_fhandle_hash - calculate the crc32 hash for the filehandle + * in the same way that wireshark does + * + * @fh: file handle + * + * For debugging only. + */ +u32 _nfs_display_fhandle_hash(const struct nfs_fh *fh) +{ + /* wireshark uses 32-bit AUTODIN crc and does a bitwise + * not on the result */ + return nfs_fhandle_hash(fh); +} +EXPORT_SYMBOL_GPL(_nfs_display_fhandle_hash); + +/* + * _nfs_display_fhandle - display an NFS file handle on the console + * + * @fh: file handle to display + * @caption: display caption + * + * For debugging only. + */ +void _nfs_display_fhandle(const struct nfs_fh *fh, const char *caption) +{ + unsigned short i; + + if (fh == NULL || fh->size == 0) { + printk(KERN_DEFAULT "%s at %p is empty\n", caption, fh); + return; + } + + printk(KERN_DEFAULT "%s at %p is %u bytes, crc: 0x%08x:\n", + caption, fh, fh->size, _nfs_display_fhandle_hash(fh)); + for (i = 0; i < fh->size; i += 16) { + __be32 *pos = (__be32 *)&fh->data[i]; + + switch ((fh->size - i - 1) >> 2) { + case 0: + printk(KERN_DEFAULT " %08x\n", + be32_to_cpup(pos)); + break; + case 1: + printk(KERN_DEFAULT " %08x %08x\n", + be32_to_cpup(pos), be32_to_cpup(pos + 1)); + break; + case 2: + printk(KERN_DEFAULT " %08x %08x %08x\n", + be32_to_cpup(pos), be32_to_cpup(pos + 1), + be32_to_cpup(pos + 2)); + break; + default: + printk(KERN_DEFAULT " %08x %08x %08x %08x\n", + be32_to_cpup(pos), be32_to_cpup(pos + 1), + be32_to_cpup(pos + 2), be32_to_cpup(pos + 3)); + } + } +} +EXPORT_SYMBOL_GPL(_nfs_display_fhandle); +#endif + +/** + * nfs_inode_attrs_cmp_generic - compare attributes + * @fattr: attributes + * @inode: pointer to inode + * + * Attempt to divine whether or not an RPC call reply carrying stale + * attributes got scheduled after another call carrying updated ones. + * Note also the check for wraparound of 'attr_gencount' + * + * The function returns '1' if it thinks the attributes in @fattr are + * more recent than the ones cached in @inode. Otherwise it returns + * the value '0'. + */ +static int nfs_inode_attrs_cmp_generic(const struct nfs_fattr *fattr, + const struct inode *inode) +{ + unsigned long attr_gencount = NFS_I(inode)->attr_gencount; + + return (long)(fattr->gencount - attr_gencount) > 0 || + (long)(attr_gencount - nfs_read_attr_generation_counter()) > 0; +} + +/** + * nfs_inode_attrs_cmp_monotonic - compare attributes + * @fattr: attributes + * @inode: pointer to inode + * + * Attempt to divine whether or not an RPC call reply carrying stale + * attributes got scheduled after another call carrying updated ones. + * + * We assume that the server observes monotonic semantics for + * the change attribute, so a larger value means that the attributes in + * @fattr are more recent, in which case the function returns the + * value '1'. + * A return value of '0' indicates no measurable change + * A return value of '-1' means that the attributes in @inode are + * more recent. + */ +static int nfs_inode_attrs_cmp_monotonic(const struct nfs_fattr *fattr, + const struct inode *inode) +{ + s64 diff = fattr->change_attr - inode_peek_iversion_raw(inode); + if (diff > 0) + return 1; + return diff == 0 ? 0 : -1; +} + +/** + * nfs_inode_attrs_cmp_strict_monotonic - compare attributes + * @fattr: attributes + * @inode: pointer to inode + * + * Attempt to divine whether or not an RPC call reply carrying stale + * attributes got scheduled after another call carrying updated ones. + * + * We assume that the server observes strictly monotonic semantics for + * the change attribute, so a larger value means that the attributes in + * @fattr are more recent, in which case the function returns the + * value '1'. + * A return value of '-1' means that the attributes in @inode are + * more recent or unchanged. + */ +static int nfs_inode_attrs_cmp_strict_monotonic(const struct nfs_fattr *fattr, + const struct inode *inode) +{ + return nfs_inode_attrs_cmp_monotonic(fattr, inode) > 0 ? 1 : -1; +} + +/** + * nfs_inode_attrs_cmp - compare attributes + * @fattr: attributes + * @inode: pointer to inode + * + * This function returns '1' if it thinks the attributes in @fattr are + * more recent than the ones cached in @inode. It returns '-1' if + * the attributes in @inode are more recent than the ones in @fattr, + * and it returns 0 if not sure. + */ +static int nfs_inode_attrs_cmp(const struct nfs_fattr *fattr, + const struct inode *inode) +{ + if (nfs_inode_attrs_cmp_generic(fattr, inode) > 0) + return 1; + switch (NFS_SERVER(inode)->change_attr_type) { + case NFS4_CHANGE_TYPE_IS_UNDEFINED: + break; + case NFS4_CHANGE_TYPE_IS_TIME_METADATA: + if (!(fattr->valid & NFS_ATTR_FATTR_CHANGE)) + break; + return nfs_inode_attrs_cmp_monotonic(fattr, inode); + default: + if (!(fattr->valid & NFS_ATTR_FATTR_CHANGE)) + break; + return nfs_inode_attrs_cmp_strict_monotonic(fattr, inode); + } + return 0; +} + +/** + * nfs_inode_finish_partial_attr_update - complete a previous inode update + * @fattr: attributes + * @inode: pointer to inode + * + * Returns '1' if the last attribute update left the inode cached + * attributes in a partially unrevalidated state, and @fattr + * matches the change attribute of that partial update. + * Otherwise returns '0'. + */ +static int nfs_inode_finish_partial_attr_update(const struct nfs_fattr *fattr, + const struct inode *inode) +{ + const unsigned long check_valid = + NFS_INO_INVALID_ATIME | NFS_INO_INVALID_CTIME | + NFS_INO_INVALID_MTIME | NFS_INO_INVALID_SIZE | + NFS_INO_INVALID_BLOCKS | NFS_INO_INVALID_OTHER | + NFS_INO_INVALID_NLINK | NFS_INO_INVALID_BTIME; + unsigned long cache_validity = NFS_I(inode)->cache_validity; + enum nfs4_change_attr_type ctype = NFS_SERVER(inode)->change_attr_type; + + if (ctype != NFS4_CHANGE_TYPE_IS_UNDEFINED && + !(cache_validity & NFS_INO_INVALID_CHANGE) && + (cache_validity & check_valid) != 0 && + (fattr->valid & NFS_ATTR_FATTR_CHANGE) != 0 && + nfs_inode_attrs_cmp_monotonic(fattr, inode) == 0) + return 1; + return 0; +} + +static void nfs_ooo_merge(struct nfs_inode *nfsi, + u64 start, u64 end) +{ + int i, cnt; + + if (nfsi->cache_validity & NFS_INO_DATA_INVAL_DEFER) + /* No point merging anything */ + return; + + if (!nfsi->ooo) { + nfsi->ooo = kmalloc_obj(*nfsi->ooo, GFP_ATOMIC); + if (!nfsi->ooo) { + nfsi->cache_validity |= NFS_INO_DATA_INVAL_DEFER; + return; + } + nfsi->ooo->cnt = 0; + } + + /* add this range, merging if possible */ + cnt = nfsi->ooo->cnt; + for (i = 0; i < cnt; i++) { + if (end == nfsi->ooo->gap[i].start) + end = nfsi->ooo->gap[i].end; + else if (start == nfsi->ooo->gap[i].end) + start = nfsi->ooo->gap[i].start; + else + continue; + /* Remove 'i' from table and loop to insert the new range */ + cnt -= 1; + nfsi->ooo->gap[i] = nfsi->ooo->gap[cnt]; + i = -1; + } + if (start != end) { + if (cnt >= ARRAY_SIZE(nfsi->ooo->gap)) { + nfsi->cache_validity |= NFS_INO_DATA_INVAL_DEFER; + kfree(nfsi->ooo); + nfsi->ooo = NULL; + return; + } + nfsi->ooo->gap[cnt].start = start; + nfsi->ooo->gap[cnt].end = end; + cnt += 1; + } + nfsi->ooo->cnt = cnt; +} + +static void nfs_ooo_record(struct nfs_inode *nfsi, + struct nfs_fattr *fattr) +{ + /* This reply was out-of-order, so record in the + * pre/post change id, possibly cancelling + * gaps created when iversion was jumpped forward. + */ + if ((fattr->valid & NFS_ATTR_FATTR_CHANGE) && + (fattr->valid & NFS_ATTR_FATTR_PRECHANGE)) + nfs_ooo_merge(nfsi, + fattr->change_attr, + fattr->pre_change_attr); +} + +static int nfs_refresh_inode_locked(struct inode *inode, + struct nfs_fattr *fattr) +{ + int attr_cmp = nfs_inode_attrs_cmp(fattr, inode); + int ret = 0; + + trace_nfs_refresh_inode_enter(inode); + + if (attr_cmp > 0 || nfs_inode_finish_partial_attr_update(fattr, inode)) + ret = nfs_update_inode(inode, fattr); + else { + nfs_ooo_record(NFS_I(inode), fattr); + + if (attr_cmp == 0) + ret = nfs_check_inode_attributes(inode, fattr); + } + + trace_nfs_refresh_inode_exit(inode, ret); + return ret; +} + +/** + * nfs_refresh_inode - try to update the inode attribute cache + * @inode: pointer to inode + * @fattr: updated attributes + * + * Check that an RPC call that returned attributes has not overlapped with + * other recent updates of the inode metadata, then decide whether it is + * safe to do a full update of the inode attributes, or whether just to + * call nfs_check_inode_attributes. + */ +int nfs_refresh_inode(struct inode *inode, struct nfs_fattr *fattr) +{ + int status; + + if ((fattr->valid & NFS_ATTR_FATTR) == 0) + return 0; + spin_lock(&inode->i_lock); + status = nfs_refresh_inode_locked(inode, fattr); + spin_unlock(&inode->i_lock); + + return status; +} +EXPORT_SYMBOL_GPL(nfs_refresh_inode); + +static int nfs_post_op_update_inode_locked(struct inode *inode, + struct nfs_fattr *fattr, unsigned int invalid) +{ + if (S_ISDIR(inode->i_mode)) + invalid |= NFS_INO_INVALID_DATA; + nfs_set_cache_invalid(inode, invalid); + if ((fattr->valid & NFS_ATTR_FATTR) == 0) + return 0; + return nfs_refresh_inode_locked(inode, fattr); +} + +/** + * nfs_post_op_update_inode - try to update the inode attribute cache + * @inode: pointer to inode + * @fattr: updated attributes + * + * After an operation that has changed the inode metadata, mark the + * attribute cache as being invalid, then try to update it. + * + * NB: if the server didn't return any post op attributes, this + * function will force the retrieval of attributes before the next + * NFS request. Thus it should be used only for operations that + * are expected to change one or more attributes, to avoid + * unnecessary NFS requests and trips through nfs_update_inode(). + */ +int nfs_post_op_update_inode(struct inode *inode, struct nfs_fattr *fattr) +{ + int status; + + spin_lock(&inode->i_lock); + nfs_fattr_set_barrier(fattr); + status = nfs_post_op_update_inode_locked(inode, fattr, + NFS_INO_INVALID_CHANGE + | NFS_INO_INVALID_CTIME + | NFS_INO_REVAL_FORCED); + spin_unlock(&inode->i_lock); + + return status; +} +EXPORT_SYMBOL_GPL(nfs_post_op_update_inode); + +/** + * nfs_post_op_update_inode_force_wcc_locked - update the inode attribute cache + * @inode: pointer to inode + * @fattr: updated attributes + * + * After an operation that has changed the inode metadata, mark the + * attribute cache as being invalid, then try to update it. Fake up + * weak cache consistency data, if none exist. + * + * This function is mainly designed to be used by the ->write_done() functions. + */ +int nfs_post_op_update_inode_force_wcc_locked(struct inode *inode, struct nfs_fattr *fattr) +{ + int attr_cmp = nfs_inode_attrs_cmp(fattr, inode); + int status; + + /* Don't do a WCC update if these attributes are already stale */ + if (attr_cmp < 0) + return 0; + if ((fattr->valid & NFS_ATTR_FATTR) == 0 || !attr_cmp) { + /* Record the pre/post change info before clearing PRECHANGE */ + nfs_ooo_record(NFS_I(inode), fattr); + fattr->valid &= ~(NFS_ATTR_FATTR_PRECHANGE + | NFS_ATTR_FATTR_PRESIZE + | NFS_ATTR_FATTR_PREMTIME + | NFS_ATTR_FATTR_PRECTIME); + goto out_noforce; + } + if ((fattr->valid & NFS_ATTR_FATTR_CHANGE) != 0 && + (fattr->valid & NFS_ATTR_FATTR_PRECHANGE) == 0) { + fattr->pre_change_attr = inode_peek_iversion_raw(inode); + fattr->valid |= NFS_ATTR_FATTR_PRECHANGE; + } + if ((fattr->valid & NFS_ATTR_FATTR_CTIME) != 0 && + (fattr->valid & NFS_ATTR_FATTR_PRECTIME) == 0) { + fattr->pre_ctime = inode_get_ctime(inode); + fattr->valid |= NFS_ATTR_FATTR_PRECTIME; + } + if ((fattr->valid & NFS_ATTR_FATTR_MTIME) != 0 && + (fattr->valid & NFS_ATTR_FATTR_PREMTIME) == 0) { + fattr->pre_mtime = inode_get_mtime(inode); + fattr->valid |= NFS_ATTR_FATTR_PREMTIME; + } + if ((fattr->valid & NFS_ATTR_FATTR_SIZE) != 0 && + (fattr->valid & NFS_ATTR_FATTR_PRESIZE) == 0) { + fattr->pre_size = i_size_read(inode); + fattr->valid |= NFS_ATTR_FATTR_PRESIZE; + } +out_noforce: + status = nfs_post_op_update_inode_locked(inode, fattr, + NFS_INO_INVALID_CHANGE + | NFS_INO_INVALID_CTIME + | NFS_INO_INVALID_MTIME + | NFS_INO_INVALID_BLOCKS); + return status; +} + +/** + * nfs_post_op_update_inode_force_wcc - try to update the inode attribute cache + * @inode: pointer to inode + * @fattr: updated attributes + * + * After an operation that has changed the inode metadata, mark the + * attribute cache as being invalid, then try to update it. Fake up + * weak cache consistency data, if none exist. + * + * This function is mainly designed to be used by the ->write_done() functions. + */ +int nfs_post_op_update_inode_force_wcc(struct inode *inode, struct nfs_fattr *fattr) +{ + int status; + + spin_lock(&inode->i_lock); + nfs_fattr_set_barrier(fattr); + status = nfs_post_op_update_inode_force_wcc_locked(inode, fattr); + spin_unlock(&inode->i_lock); + return status; +} +EXPORT_SYMBOL_GPL(nfs_post_op_update_inode_force_wcc); + + +/* + * Many nfs protocol calls return the new file attributes after + * an operation. Here we update the inode to reflect the state + * of the server's inode. + * + * This is a bit tricky because we have to make sure all dirty pages + * have been sent off to the server before calling invalidate_inode_pages. + * To make sure no other process adds more write requests while we try + * our best to flush them, we make them sleep during the attribute refresh. + * + * A very similar scenario holds for the dir cache. + */ +static int nfs_update_inode(struct inode *inode, struct nfs_fattr *fattr) +{ + struct nfs_server *server = NFS_SERVER(inode); + struct nfs_inode *nfsi = NFS_I(inode); + loff_t cur_isize, new_isize; + u64 fattr_supported = server->fattr_valid; + unsigned long invalid = 0; + unsigned long now = jiffies; + unsigned long save_cache_validity; + bool have_writers = nfs_file_has_buffered_writers(nfsi); + bool cache_revalidated = true; + bool attr_changed = false; + bool have_delegation; + + dfprintk(VFS, "NFS: %s(%s/%llu fh_crc=0x%08x ct=%d info=0x%llx)\n", + __func__, inode->i_sb->s_id, inode->i_ino, + nfs_display_fhandle_hash(NFS_FH(inode)), + icount_read_once(inode), fattr->valid); + + if (!(fattr->valid & NFS_ATTR_FATTR_FILEID)) { + /* Only a mounted-on-fileid? Just exit */ + if (fattr->valid & NFS_ATTR_FATTR_MOUNTED_ON_FILEID) + return 0; + /* Has the inode gone and changed behind our back? */ + } else if (inode->i_ino != fattr->fileid) { + /* Is this perhaps the mounted-on fileid? */ + if ((fattr->valid & NFS_ATTR_FATTR_MOUNTED_ON_FILEID) && + inode->i_ino == fattr->mounted_on_fileid) + return 0; + printk(KERN_ERR "NFS: server %s error: fileid changed\n" + "fsid %s: expected fileid 0x%Lx, got 0x%Lx\n", + NFS_SERVER(inode)->nfs_client->cl_hostname, + inode->i_sb->s_id, (long long)inode->i_ino, + (long long)fattr->fileid); + goto out_err; + } + + /* + * Make sure the inode's type hasn't changed. + */ + if ((fattr->valid & NFS_ATTR_FATTR_TYPE) && inode_wrong_type(inode, fattr->mode)) { + /* + * Big trouble! The inode has become a different object. + */ + printk(KERN_DEBUG "NFS: %s: inode %llu mode changed, %07o to %07o\n", + __func__, inode->i_ino, inode->i_mode, fattr->mode); + goto out_err; + } + + /* Update the fsid? */ + if (S_ISDIR(inode->i_mode) && (fattr->valid & NFS_ATTR_FATTR_FSID) && + !nfs_fsid_equal(&server->fsid, &fattr->fsid) && + !IS_AUTOMOUNT(inode)) + server->fsid = fattr->fsid; + + /* Save the delegation state before clearing cache_validity */ + have_delegation = nfs_have_delegated_attributes(inode); + + /* + * Update the read time so we don't revalidate too often. + */ + nfsi->read_cache_jiffies = fattr->time_start; + + /* Fix up any delegated attributes in the struct nfs_fattr */ + nfs_fattr_fixup_delegated(inode, fattr); + + save_cache_validity = nfsi->cache_validity; + nfsi->cache_validity &= ~(NFS_INO_INVALID_ATTR + | NFS_INO_INVALID_ATIME + | NFS_INO_REVAL_FORCED + | NFS_INO_INVALID_BLOCKS); + + /* Do atomic weak cache consistency updates */ + nfs_wcc_update_inode(inode, fattr); + + if (pnfs_layoutcommit_outstanding(inode)) { + nfsi->cache_validity |= + save_cache_validity & + (NFS_INO_INVALID_CHANGE | NFS_INO_INVALID_CTIME | + NFS_INO_INVALID_MTIME | NFS_INO_INVALID_SIZE | + NFS_INO_INVALID_BLOCKS); + cache_revalidated = false; + } + + /* More cache consistency checks */ + if (fattr->valid & NFS_ATTR_FATTR_CHANGE) { + if (!have_writers && nfsi->ooo && nfsi->ooo->cnt == 1 && + nfsi->ooo->gap[0].end == inode_peek_iversion_raw(inode)) { + /* There is one remaining gap that hasn't been + * merged into iversion - do that now. + */ + inode_set_iversion_raw(inode, nfsi->ooo->gap[0].start); + kfree(nfsi->ooo); + nfsi->ooo = NULL; + } + if (!inode_eq_iversion_raw(inode, fattr->change_attr)) { + /* Could it be a race with writeback? */ + if (!(have_writers || have_delegation)) { + invalid |= NFS_INO_INVALID_DATA + | NFS_INO_INVALID_ACCESS + | NFS_INO_INVALID_ACL + | NFS_INO_INVALID_XATTR; + /* Force revalidate of all attributes */ + save_cache_validity |= NFS_INO_INVALID_CTIME + | NFS_INO_INVALID_MTIME + | NFS_INO_INVALID_SIZE + | NFS_INO_INVALID_BLOCKS + | NFS_INO_INVALID_NLINK + | NFS_INO_INVALID_MODE + | NFS_INO_INVALID_OTHER + | NFS_INO_INVALID_BTIME; + if (S_ISDIR(inode->i_mode)) + nfs_force_lookup_revalidate(inode); + attr_changed = true; + dprintk("NFS: change_attr change on server for file %s/%llu\n", + inode->i_sb->s_id, + inode->i_ino); + } else if (!have_delegation) { + nfs_ooo_record(nfsi, fattr); + nfs_ooo_merge(nfsi, inode_peek_iversion_raw(inode), + fattr->change_attr); + } + inode_set_iversion_raw(inode, fattr->change_attr); + } + } else { + nfsi->cache_validity |= + save_cache_validity & NFS_INO_INVALID_CHANGE; + if (!have_delegation || + (nfsi->cache_validity & NFS_INO_INVALID_CHANGE) != 0) + cache_revalidated = false; + } + + if (fattr->valid & NFS_ATTR_FATTR_MTIME) + inode_set_mtime_to_ts(inode, fattr->mtime); + else if (fattr_supported & NFS_ATTR_FATTR_MTIME) + nfsi->cache_validity |= + save_cache_validity & NFS_INO_INVALID_MTIME; + + if (fattr->valid & NFS_ATTR_FATTR_CTIME) + inode_set_ctime_to_ts(inode, fattr->ctime); + else if (fattr_supported & NFS_ATTR_FATTR_CTIME) + nfsi->cache_validity |= + save_cache_validity & NFS_INO_INVALID_CTIME; + + if (fattr->valid & NFS_ATTR_FATTR_BTIME) + nfsi->btime = fattr->btime; + else if (fattr_supported & NFS_ATTR_FATTR_BTIME) + nfsi->cache_validity |= + save_cache_validity & NFS_INO_INVALID_BTIME; + + /* Check if our cached file size is stale */ + if (fattr->valid & NFS_ATTR_FATTR_SIZE) { + new_isize = nfs_size_to_loff_t(fattr->size); + cur_isize = i_size_read(inode); + if (new_isize != cur_isize && !have_delegation) { + /* Do we perhaps have any outstanding writes, or has + * the file grown beyond our last write? */ + if (!nfs_have_writebacks(inode) || new_isize > cur_isize) { + trace_nfs_size_update(inode, new_isize); + i_size_write(inode, new_isize); + if (!have_writers) + invalid |= NFS_INO_INVALID_DATA; + } + } + if (new_isize == 0 && + !(fattr->valid & (NFS_ATTR_FATTR_SPACE_USED | + NFS_ATTR_FATTR_BLOCKS_USED))) { + fattr->du.nfs3.used = 0; + fattr->valid |= NFS_ATTR_FATTR_SPACE_USED; + } + } else + nfsi->cache_validity |= + save_cache_validity & NFS_INO_INVALID_SIZE; + + if (fattr->valid & NFS_ATTR_FATTR_ATIME) + inode_set_atime_to_ts(inode, fattr->atime); + else if (fattr_supported & NFS_ATTR_FATTR_ATIME) + nfsi->cache_validity |= + save_cache_validity & NFS_INO_INVALID_ATIME; + + if (fattr->valid & NFS_ATTR_FATTR_MODE) { + if ((inode->i_mode & S_IALLUGO) != (fattr->mode & S_IALLUGO)) { + umode_t newmode = inode->i_mode & S_IFMT; + newmode |= fattr->mode & S_IALLUGO; + inode->i_mode = newmode; + invalid |= NFS_INO_INVALID_ACCESS + | NFS_INO_INVALID_ACL; + } + } else if (fattr_supported & NFS_ATTR_FATTR_MODE) + nfsi->cache_validity |= + save_cache_validity & NFS_INO_INVALID_MODE; + + if (fattr->valid & NFS_ATTR_FATTR_OWNER) { + if (!uid_eq(inode->i_uid, fattr->uid)) { + invalid |= NFS_INO_INVALID_ACCESS + | NFS_INO_INVALID_ACL; + inode->i_uid = fattr->uid; + } + } else if (fattr_supported & NFS_ATTR_FATTR_OWNER) + nfsi->cache_validity |= + save_cache_validity & NFS_INO_INVALID_OTHER; + + if (fattr->valid & NFS_ATTR_FATTR_GROUP) { + if (!gid_eq(inode->i_gid, fattr->gid)) { + invalid |= NFS_INO_INVALID_ACCESS + | NFS_INO_INVALID_ACL; + inode->i_gid = fattr->gid; + } + } else if (fattr_supported & NFS_ATTR_FATTR_GROUP) + nfsi->cache_validity |= + save_cache_validity & NFS_INO_INVALID_OTHER; + + if (fattr->valid & NFS_ATTR_FATTR_NLINK) { + if (inode->i_nlink != fattr->nlink) + set_nlink(inode, fattr->nlink); + } else if (fattr_supported & NFS_ATTR_FATTR_NLINK) + nfsi->cache_validity |= + save_cache_validity & NFS_INO_INVALID_NLINK; + + if (fattr->valid & NFS_ATTR_FATTR_SPACE_USED) { + /* + * report the blocks in 512byte units + */ + inode->i_blocks = nfs_calc_block_size(fattr->du.nfs3.used); + } else if (fattr_supported & NFS_ATTR_FATTR_SPACE_USED) + nfsi->cache_validity |= + save_cache_validity & NFS_INO_INVALID_BLOCKS; + + if (fattr->valid & NFS_ATTR_FATTR_BLOCKS_USED) + inode->i_blocks = fattr->du.nfs2.blocks; + else if (fattr_supported & NFS_ATTR_FATTR_BLOCKS_USED) + nfsi->cache_validity |= + save_cache_validity & NFS_INO_INVALID_BLOCKS; + + /* Update attrtimeo value if we're out of the unstable period */ + if (attr_changed) { + nfs_inc_stats(inode, NFSIOS_ATTRINVALIDATE); + nfsi->attrtimeo = NFS_MINATTRTIMEO(inode); + nfsi->attrtimeo_timestamp = now; + /* Set barrier to be more recent than all outstanding updates */ + nfsi->attr_gencount = nfs_inc_attr_generation_counter(); + } else { + if (cache_revalidated) { + if (!time_in_range_open(now, nfsi->attrtimeo_timestamp, + nfsi->attrtimeo_timestamp + nfsi->attrtimeo)) { + nfsi->attrtimeo <<= 1; + if (nfsi->attrtimeo > NFS_MAXATTRTIMEO(inode)) + nfsi->attrtimeo = NFS_MAXATTRTIMEO(inode); + } + nfsi->attrtimeo_timestamp = now; + } + /* Set the barrier to be more recent than this fattr */ + if ((long)(fattr->gencount - nfsi->attr_gencount) > 0) + nfsi->attr_gencount = fattr->gencount; + } + + /* Don't invalidate the data if we were to blame */ + if (!(S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) + || S_ISLNK(inode->i_mode))) + invalid &= ~NFS_INO_INVALID_DATA; + nfs_set_cache_invalid(inode, invalid); + + return 0; + out_err: + /* + * No need to worry about unhashing the dentry, as the + * lookup validation will know that the inode is bad. + * (But we fall through to invalidate the caches.) + */ + nfs_set_inode_stale_locked(inode); + return -ESTALE; +} + +struct inode *nfs_alloc_inode(struct super_block *sb) +{ + struct nfs_inode *nfsi; + nfsi = alloc_inode_sb(sb, nfs_inode_cachep, GFP_KERNEL); + if (!nfsi) + return NULL; + nfsi->flags = 0UL; + nfsi->cache_validity = 0UL; + nfsi->ooo = NULL; +#if IS_ENABLED(CONFIG_NFS_V4) + nfsi->nfs4_acl = NULL; +#endif /* CONFIG_NFS_V4 */ +#ifdef CONFIG_NFS_V4_2 + nfsi->xattr_cache = NULL; +#endif + nfs_netfs_inode_init(nfsi); + + return &nfsi->vfs_inode; +} +EXPORT_SYMBOL_GPL(nfs_alloc_inode); + +void nfs_free_inode(struct inode *inode) +{ + kfree(NFS_I(inode)->ooo); + kmem_cache_free(nfs_inode_cachep, NFS_I(inode)); +} +EXPORT_SYMBOL_GPL(nfs_free_inode); + +static inline void nfs4_init_once(struct nfs_inode *nfsi) +{ +#if IS_ENABLED(CONFIG_NFS_V4) + INIT_LIST_HEAD(&nfsi->open_states); + nfsi->delegation = NULL; + init_rwsem(&nfsi->rwsem); + nfsi->layout = NULL; +#endif +} + +static void init_once(void *foo) +{ + struct nfs_inode *nfsi = foo; + + inode_init_once(&nfsi->vfs_inode); + INIT_LIST_HEAD(&nfsi->open_files); + INIT_LIST_HEAD(&nfsi->access_cache_entry_lru); + INIT_LIST_HEAD(&nfsi->access_cache_inode_lru); + nfs4_init_once(nfsi); +} + +static int __init nfs_init_inodecache(void) +{ + nfs_inode_cachep = kmem_cache_create("nfs_inode_cache", + sizeof(struct nfs_inode), + 0, (SLAB_RECLAIM_ACCOUNT| + SLAB_ACCOUNT), + init_once); + if (nfs_inode_cachep == NULL) + return -ENOMEM; + + return 0; +} + +static void nfs_destroy_inodecache(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(nfs_inode_cachep); +} + +struct workqueue_struct *nfslocaliod_workqueue; +struct workqueue_struct *nfsiod_workqueue; +EXPORT_SYMBOL_GPL(nfsiod_workqueue); + +/* + * Destroy the nfsiod workqueues + */ +static void nfsiod_stop(void) +{ + struct workqueue_struct *wq; + + wq = nfsiod_workqueue; + if (wq != NULL) { + nfsiod_workqueue = NULL; + destroy_workqueue(wq); + } +#if IS_ENABLED(CONFIG_NFS_LOCALIO) + wq = nfslocaliod_workqueue; + if (wq != NULL) { + nfslocaliod_workqueue = NULL; + destroy_workqueue(wq); + } +#endif /* CONFIG_NFS_LOCALIO */ +} + +/* + * Start the nfsiod workqueues + */ +static int nfsiod_start(void) +{ + dprintk("RPC: creating workqueue nfsiod\n"); + nfsiod_workqueue = alloc_workqueue("nfsiod", WQ_MEM_RECLAIM | WQ_UNBOUND, 0); + if (nfsiod_workqueue == NULL) + return -ENOMEM; +#if IS_ENABLED(CONFIG_NFS_LOCALIO) + /* + * localio writes need to use a normal (non-memreclaim) workqueue. + * When we start getting low on space, XFS goes and calls flush_work() on + * a non-memreclaim work queue, which causes a priority inversion problem. + */ + dprintk("RPC: creating workqueue nfslocaliod\n"); + nfslocaliod_workqueue = alloc_workqueue("nfslocaliod", WQ_UNBOUND, 0); + if (unlikely(nfslocaliod_workqueue == NULL)) { + nfsiod_stop(); + return -ENOMEM; + } +#endif /* CONFIG_NFS_LOCALIO */ + return 0; +} + +unsigned int nfs_net_id; +EXPORT_SYMBOL_GPL(nfs_net_id); + +static int nfs_net_init(struct net *net) +{ + struct nfs_net *nn = net_generic(net, nfs_net_id); + int err; + + nfs_clients_init(net); + + if (!rpc_proc_register(net, &nn->rpcstats)) { + err = -ENOMEM; + goto err_proc_rpc; + } + + err = nfs_fs_proc_net_init(net); + if (err) + goto err_proc_nfs; + + return 0; + +err_proc_nfs: + rpc_proc_unregister(net, "nfs"); +err_proc_rpc: + nfs_clients_exit(net); + return err; +} + +static void nfs_net_exit(struct net *net) +{ + rpc_proc_unregister(net, "nfs"); + nfs_fs_proc_net_exit(net); + nfs_clients_exit(net); +} + +static struct pernet_operations nfs_net_ops = { + .init = nfs_net_init, + .exit = nfs_net_exit, + .id = &nfs_net_id, + .size = sizeof(struct nfs_net), +}; + +#ifdef CONFIG_KEYS +static struct key *nfs_keyring; + +static int __init nfs_init_keyring(void) +{ + nfs_keyring = keyring_alloc(".nfs", + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, + current_cred(), + (KEY_POS_ALL & ~KEY_POS_SETATTR) | + (KEY_USR_ALL & ~KEY_USR_SETATTR), + KEY_ALLOC_NOT_IN_QUOTA, NULL, NULL); + return PTR_ERR_OR_ZERO(nfs_keyring); +} + +static void nfs_exit_keyring(void) +{ + key_put(nfs_keyring); +} +#else +static inline int nfs_init_keyring(void) +{ + return 0; +} + +static inline void nfs_exit_keyring(void) +{ +} +#endif /* CONFIG_KEYS */ + +/* + * Initialize NFS + */ +static int __init init_nfs_fs(void) +{ + int err; + + err = nfs_init_keyring(); + if (err) + return err; + + err = nfs_sysfs_init(); + if (err < 0) + goto out10; + + err = register_pernet_subsys(&nfs_net_ops); + if (err < 0) + goto out9; + + err = nfsiod_start(); + if (err) + goto out7; + + err = nfs_fs_proc_init(); + if (err) + goto out6; + + err = nfs_init_nfspagecache(); + if (err) + goto out5; + + err = nfs_init_inodecache(); + if (err) + goto out4; + + err = nfs_init_readpagecache(); + if (err) + goto out3; + + err = nfs_init_writepagecache(); + if (err) + goto out2; + + err = nfs_init_directcache(); + if (err) + goto out1; + + err = register_nfs_fs(); + if (err) + goto out0; + + return 0; +out0: + nfs_destroy_directcache(); +out1: + nfs_destroy_writepagecache(); +out2: + nfs_destroy_readpagecache(); +out3: + nfs_destroy_inodecache(); +out4: + nfs_destroy_nfspagecache(); +out5: + nfs_fs_proc_exit(); +out6: + nfsiod_stop(); +out7: + unregister_pernet_subsys(&nfs_net_ops); +out9: + nfs_sysfs_exit(); +out10: + nfs_exit_keyring(); + return err; +} + +static void __exit exit_nfs_fs(void) +{ + nfs_destroy_directcache(); + nfs_destroy_writepagecache(); + nfs_destroy_readpagecache(); + nfs_destroy_inodecache(); + nfs_destroy_nfspagecache(); + unregister_pernet_subsys(&nfs_net_ops); + unregister_nfs_fs(); + nfs_fs_proc_exit(); + nfsiod_stop(); + nfs_sysfs_exit(); + nfs_exit_keyring(); +} + +/* Not quite true; I just maintain it */ +MODULE_AUTHOR("Olaf Kirch "); +MODULE_DESCRIPTION("NFS client support"); +MODULE_LICENSE("GPL"); +module_param_cb(enable_ino64, ¶m_ops_enable_ino64, &enable_ino64, 0644); + +module_init(init_nfs_fs) +module_exit(exit_nfs_fs) diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h new file mode 100644 index 000000000..7f96a258a --- /dev/null +++ b/fs/nfs/internal.h @@ -0,0 +1,1012 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * NFS internal definitions + */ + +#include "nfs4_fs.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#define NFS_SB_MASK (SB_RDONLY|SB_NOSUID|SB_NODEV|SB_NOEXEC|SB_SYNCHRONOUS) + +extern const struct export_operations nfs_export_ops; + +struct nfs_string; +struct nfs_pageio_descriptor; + +static inline void nfs_attr_check_mountpoint(struct super_block *parent, struct nfs_fattr *fattr) +{ + if (!nfs_fsid_equal(&NFS_SB(parent)->fsid, &fattr->fsid)) + fattr->valid |= NFS_ATTR_FATTR_MOUNTPOINT; +} + +static inline int nfs_attr_use_mounted_on_fileid(struct nfs_fattr *fattr) +{ + if (((fattr->valid & NFS_ATTR_FATTR_MOUNTED_ON_FILEID) == 0) || + (((fattr->valid & NFS_ATTR_FATTR_MOUNTPOINT) == 0) && + ((fattr->valid & NFS_ATTR_FATTR_V4_REFERRAL) == 0))) + return 0; + return 1; +} + +static inline bool nfs_lookup_is_soft_revalidate(const struct dentry *dentry) +{ + if (!(NFS_SB(dentry->d_sb)->flags & NFS_MOUNT_SOFTREVAL)) + return false; + if (!d_is_positive(dentry) || !NFS_FH(d_inode(dentry))->size) + return false; + return true; +} + +static inline fmode_t flags_to_mode(int flags) +{ + fmode_t res = (__force fmode_t)flags & FMODE_EXEC; + if ((flags & O_ACCMODE) != O_WRONLY) + res |= FMODE_READ; + if ((flags & O_ACCMODE) != O_RDONLY) + res |= FMODE_WRITE; + return res; +} + +/* + * Note: RFC 1813 doesn't limit the number of auth flavors that + * a server can return, so make something up. + */ +#define NFS_MAX_SECFLAVORS (12) + +/* + * Value used if the user did not specify a port value. + */ +#define NFS_UNSPEC_PORT (-1) + +#define NFS_UNSPEC_RETRANS (UINT_MAX) +#define NFS_UNSPEC_TIMEO (UINT_MAX) + +struct nfs_client_initdata { + unsigned long init_flags; + const char *hostname; /* Hostname of the server */ + const struct sockaddr_storage *addr; /* Address of the server */ + const char *nodename; /* Hostname of the client */ + const char *ip_addr; /* IP address of the client */ + size_t addrlen; + struct nfs_subversion *nfs_mod; + int proto; + u32 minorversion; + unsigned int nconnect; + unsigned int max_connect; + struct net *net; + const struct rpc_timeout *timeparms; + const struct cred *cred; + struct xprtsec_parms xprtsec; + unsigned long connect_timeout; + unsigned long reconnect_timeout; +}; + +/* + * In-kernel mount arguments + */ +struct nfs_fs_context { + bool internal; + bool skip_reconfig_option_check; + bool need_mount; + bool sloppy; + unsigned int flags; /* NFS{,4}_MOUNT_* flags */ + unsigned int rsize, wsize; + unsigned int timeo, retrans; + unsigned int acregmin, acregmax; + unsigned int acdirmin, acdirmax; + unsigned int namlen; + unsigned int options; + unsigned int bsize; + struct nfs_auth_info auth_info; + rpc_authflavor_t selected_flavor; + struct xprtsec_parms xprtsec; + char *client_address; + unsigned int version; + unsigned int minorversion; + char *fscache_uniq; + unsigned short protofamily; + unsigned short mountfamily; + bool has_sec_mnt_opts; + int lock_status; + + struct { + union { + struct sockaddr address; + struct sockaddr_storage _address; + }; + size_t addrlen; + char *hostname; + u32 version; + int port; + unsigned short protocol; + } mount_server; + + struct { + union { + struct sockaddr address; + struct sockaddr_storage _address; + }; + size_t addrlen; + char *hostname; + char *export_path; + int port; + unsigned short protocol; + unsigned short nconnect; + unsigned short max_connect; + unsigned short export_path_len; + } nfs_server; + + struct nfs_fh *mntfh; + struct nfs_server *server; + struct nfs_subversion *nfs_mod; + + /* Information for a cloned mount. */ + struct nfs_clone_mount { + struct super_block *sb; + struct dentry *dentry; + struct nfs_fattr *fattr; + } clone_data; +}; + +enum nfs_lock_status { + NFS_LOCK_NOT_SET = 0, + NFS_LOCK_LOCK = 1, + NFS_LOCK_NOLOCK = 2, +}; + +#define nfs_errorf(fc, fmt, ...) ((fc)->log.log ? \ + errorf(fc, fmt, ## __VA_ARGS__) : \ + ({ dprintk(fmt "\n", ## __VA_ARGS__); })) + +#define nfs_ferrorf(fc, fac, fmt, ...) ((fc)->log.log ? \ + errorf(fc, fmt, ## __VA_ARGS__) : \ + ({ dfprintk(fac, fmt "\n", ## __VA_ARGS__); })) + +#define nfs_invalf(fc, fmt, ...) ((fc)->log.log ? \ + invalf(fc, fmt, ## __VA_ARGS__) : \ + ({ dprintk(fmt "\n", ## __VA_ARGS__); -EINVAL; })) + +#define nfs_finvalf(fc, fac, fmt, ...) ((fc)->log.log ? \ + invalf(fc, fmt, ## __VA_ARGS__) : \ + ({ dfprintk(fac, fmt "\n", ## __VA_ARGS__); -EINVAL; })) + +#define nfs_warnf(fc, fmt, ...) ((fc)->log.log ? \ + warnf(fc, fmt, ## __VA_ARGS__) : \ + ({ dprintk(fmt "\n", ## __VA_ARGS__); })) + +#define nfs_fwarnf(fc, fac, fmt, ...) ((fc)->log.log ? \ + warnf(fc, fmt, ## __VA_ARGS__) : \ + ({ dfprintk(fac, fmt "\n", ## __VA_ARGS__); })) + +static inline struct nfs_fs_context *nfs_fc2context(const struct fs_context *fc) +{ + return fc->fs_private; +} + +/* mount_clnt.c */ +struct nfs_mount_request { + struct sockaddr_storage *sap; + size_t salen; + char *hostname; + char *dirpath; + u32 version; + unsigned short protocol; + struct nfs_fh *fh; + int noresvport; + unsigned int *auth_flav_len; + rpc_authflavor_t *auth_flavs; + struct net *net; +}; + +extern int nfs_mount(struct nfs_mount_request *info, int timeo, int retrans); + +/* client.c */ +extern const struct rpc_program nfs_program; +extern void nfs_clients_init(struct net *net); +extern void nfs_clients_exit(struct net *net); +extern struct nfs_client *nfs_alloc_client(const struct nfs_client_initdata *); +int nfs_create_rpc_client(struct nfs_client *, const struct nfs_client_initdata *, rpc_authflavor_t); +struct nfs_client *nfs_get_client(const struct nfs_client_initdata *); +int nfs_probe_server(struct nfs_server *, struct nfs_fh *); +void nfs_server_insert_lists(struct nfs_server *); +void nfs_server_remove_lists(struct nfs_server *); +void nfs_init_timeout_values(struct rpc_timeout *to, int proto, int timeo, int retrans); +int nfs_init_server_rpcclient(struct nfs_server *, const struct rpc_timeout *t, + rpc_authflavor_t); +struct nfs_server *nfs_alloc_server(void); +void nfs_server_copy_userdata(struct nfs_server *, struct nfs_server *); + +extern void nfs_put_client(struct nfs_client *); +extern void nfs_free_client(struct nfs_client *); +extern struct nfs_client *nfs4_find_client_ident(struct net *, int); +extern struct nfs_client * +nfs4_find_client_sessionid(struct net *, const struct sockaddr *, + struct nfs4_sessionid *, u32); +extern struct nfs_server *nfs_create_server(struct fs_context *); +extern void nfs_server_set_init_caps(struct nfs_server *); +extern struct nfs_server *nfs4_create_server(struct fs_context *); +extern struct nfs_server *nfs4_create_referral_server(struct fs_context *); +extern int nfs4_update_server(struct nfs_server *server, const char *hostname, + struct sockaddr_storage *sap, size_t salen, + struct net *net); +extern void nfs_free_server(struct nfs_server *server); +extern struct nfs_server *nfs_clone_server(struct nfs_server *, + struct nfs_fh *, + struct nfs_fattr *, + rpc_authflavor_t); +extern bool nfs_client_init_is_complete(const struct nfs_client *clp); +extern int nfs_client_init_status(const struct nfs_client *clp); +extern int nfs_wait_client_init_complete(const struct nfs_client *clp); +extern void nfs_mark_client_ready(struct nfs_client *clp, int state); +extern struct nfs_client *nfs4_set_ds_client(struct nfs_server *mds_srv, + const struct sockaddr_storage *ds_addr, + int ds_addrlen, int ds_proto, + unsigned int ds_timeo, + unsigned int ds_retrans, + u32 minor_version); +extern struct rpc_clnt *nfs4_find_or_create_ds_client(struct nfs_client *, + struct inode *); +extern void nfs4_session_limit_rwsize(struct nfs_server *server); +extern void nfs4_session_limit_xasize(struct nfs_server *server); +extern struct nfs_client *nfs3_set_ds_client(struct nfs_server *mds_srv, + const struct sockaddr_storage *ds_addr, int ds_addrlen, + int ds_proto, unsigned int ds_timeo, + unsigned int ds_retrans); +#ifdef CONFIG_PROC_FS +extern int __init nfs_fs_proc_init(void); +extern void nfs_fs_proc_exit(void); +extern int nfs_fs_proc_net_init(struct net *net); +extern void nfs_fs_proc_net_exit(struct net *net); +#else +static inline int nfs_fs_proc_net_init(struct net *net) +{ + return 0; +} +static inline void nfs_fs_proc_net_exit(struct net *net) +{ +} +static inline int nfs_fs_proc_init(void) +{ + return 0; +} +static inline void nfs_fs_proc_exit(void) +{ +} +#endif + +/* callback_xdr.c */ +extern const struct svc_version nfs4_callback_version1; +extern const struct svc_version nfs4_callback_version4; + +/* fs_context.c */ +extern struct file_system_type nfs_fs_type; + +/* pagelist.c */ +extern int __init nfs_init_nfspagecache(void); +extern void nfs_destroy_nfspagecache(void); +extern int __init nfs_init_readpagecache(void); +extern void nfs_destroy_readpagecache(void); +extern int __init nfs_init_writepagecache(void); +extern void nfs_destroy_writepagecache(void); + +extern int __init nfs_init_directcache(void); +extern void nfs_destroy_directcache(void); +extern void nfs_pgheader_init(struct nfs_pageio_descriptor *desc, + struct nfs_pgio_header *hdr, + void (*release)(struct nfs_pgio_header *hdr)); +void nfs_set_pgio_error(struct nfs_pgio_header *hdr, int error, loff_t pos); +int nfs_iocounter_wait(struct nfs_lock_context *l_ctx); + +extern const struct nfs_pageio_ops nfs_pgio_rw_ops; +struct nfs_pgio_header *nfs_pgio_header_alloc(const struct nfs_rw_ops *); +void nfs_pgio_header_free(struct nfs_pgio_header *); +int nfs_generic_pgio(struct nfs_pageio_descriptor *, struct nfs_pgio_header *); +int nfs_initiate_pgio(struct rpc_clnt *clnt, struct nfs_pgio_header *hdr, + const struct cred *cred, const struct nfs_rpc_ops *rpc_ops, + const struct rpc_call_ops *call_ops, int how, int flags, + struct nfsd_file *localio); +void nfs_free_request(struct nfs_page *req); +struct nfs_pgio_mirror * +nfs_pgio_current_mirror(struct nfs_pageio_descriptor *desc); + +static inline bool nfs_match_open_context(const struct nfs_open_context *ctx1, + const struct nfs_open_context *ctx2) +{ + return cred_fscmp(ctx1->cred, ctx2->cred) == 0 && ctx1->state == ctx2->state; +} + +/* nfs2xdr.c */ +extern const struct rpc_procinfo nfs_procedures[]; +extern int nfs2_decode_dirent(struct xdr_stream *, + struct nfs_entry *, bool); + +/* nfs3xdr.c */ +extern const struct rpc_procinfo nfs3_procedures[]; +extern int nfs3_decode_dirent(struct xdr_stream *, + struct nfs_entry *, bool); + +/* nfs4xdr.c */ +#if IS_ENABLED(CONFIG_NFS_V4) +extern int nfs4_decode_dirent(struct xdr_stream *, + struct nfs_entry *, bool); +extern const u32 nfs41_maxread_overhead; +extern const u32 nfs41_maxwrite_overhead; +extern const u32 nfs41_maxgetdevinfo_overhead; + +/* nfs4proc.c */ +extern const struct rpc_procinfo nfs4_procedures[]; +#endif /* CONFIG_NFS_V4 */ + +#ifdef CONFIG_NFS_V4_SECURITY_LABEL +extern struct nfs4_label *nfs4_label_alloc(struct nfs_server *server, gfp_t flags); +static inline struct nfs4_label * +nfs4_label_copy(struct nfs4_label *dst, struct nfs4_label *src) +{ + if (!dst || !src) + return NULL; + + if (src->len > NFS4_MAXLABELLEN) + return NULL; + + dst->lfs = src->lfs; + dst->pi = src->pi; + dst->len = src->len; + memcpy(dst->label, src->label, src->len); + + return dst; +} + +static inline void nfs_zap_label_cache_locked(struct nfs_inode *nfsi) +{ + if (nfs_server_capable(&nfsi->vfs_inode, NFS_CAP_SECURITY_LABEL)) + nfsi->cache_validity |= NFS_INO_INVALID_LABEL; +} +#else +static inline struct nfs4_label *nfs4_label_alloc(struct nfs_server *server, gfp_t flags) { return NULL; } +static inline void nfs_zap_label_cache_locked(struct nfs_inode *nfsi) +{ +} +static inline struct nfs4_label * +nfs4_label_copy(struct nfs4_label *dst, struct nfs4_label *src) +{ + return NULL; +} +#endif /* CONFIG_NFS_V4_SECURITY_LABEL */ + +/* proc.c */ +void nfs_close_context(struct nfs_open_context *ctx, int is_sync); +extern struct nfs_client *nfs_init_client(struct nfs_client *clp, + const struct nfs_client_initdata *); + +/* dir.c */ +extern void nfs_readdir_record_entry_cache_hit(struct inode *dir); +extern void nfs_readdir_record_entry_cache_miss(struct inode *dir); +extern unsigned long nfs_access_cache_count(struct shrinker *shrink, + struct shrink_control *sc); +extern unsigned long nfs_access_cache_scan(struct shrinker *shrink, + struct shrink_control *sc); +struct dentry *nfs_lookup(struct inode *, struct dentry *, unsigned int); +void nfs_d_prune_case_insensitive_aliases(struct inode *inode); +int nfs_create(struct mnt_idmap *, struct inode *, struct dentry *, + umode_t); +struct dentry *nfs_mkdir(struct mnt_idmap *, struct inode *, struct dentry *, + umode_t); +int nfs_rmdir(struct inode *, struct dentry *); +int nfs_unlink(struct inode *, struct dentry *); +int nfs_symlink(struct mnt_idmap *, struct inode *, struct dentry *, + const char *); +int nfs_link(struct dentry *, struct inode *, struct dentry *); +int nfs_mknod(struct mnt_idmap *, struct inode *, struct dentry *, umode_t, + dev_t); +int nfs_rename(struct mnt_idmap *, struct inode *, struct dentry *, + struct inode *, struct dentry *, unsigned int); + +#ifdef CONFIG_NFS_V4_2 +static inline __u32 nfs_access_xattr_mask(const struct nfs_server *server) +{ + if (!(server->caps & NFS_CAP_XATTR)) + return 0; + return NFS4_ACCESS_XAREAD | NFS4_ACCESS_XAWRITE | NFS4_ACCESS_XALIST; +} +#else +static inline __u32 nfs_access_xattr_mask(const struct nfs_server *server) +{ + return 0; +} +#endif + +/* file.c */ +int nfs_file_fsync(struct file *file, loff_t start, loff_t end, int datasync); +loff_t nfs_file_llseek(struct file *, loff_t, int); +ssize_t nfs_file_read(struct kiocb *, struct iov_iter *); +ssize_t nfs_file_splice_read(struct file *in, loff_t *ppos, struct pipe_inode_info *pipe, + size_t len, unsigned int flags); +int nfs_file_mmap_prepare(struct vm_area_desc *); +ssize_t nfs_file_write(struct kiocb *, struct iov_iter *); +int nfs_file_release(struct inode *, struct file *); +int nfs_lock(struct file *, int, struct file_lock *); +int nfs_flock(struct file *, int, struct file_lock *); +int nfs_check_flags(int); +void nfs_truncate_last_folio(struct address_space *mapping, loff_t from, + loff_t to); + +/* inode.c */ +extern struct workqueue_struct *nfsiod_workqueue; +extern struct workqueue_struct *nfslocaliod_workqueue; +extern struct inode *nfs_alloc_inode(struct super_block *sb); +extern void nfs_free_inode(struct inode *); +extern int nfs_write_inode(struct inode *, struct writeback_control *); +extern int nfs_drop_inode(struct inode *); +extern void nfs_clear_inode(struct inode *); +extern void nfs_evict_inode(struct inode *); +extern void nfs_zap_acl_cache(struct inode *inode); +extern void nfs_set_cache_invalid(struct inode *inode, unsigned long flags); +extern bool nfs_check_cache_invalid(struct inode *, unsigned long); +extern int nfs_wait_bit_killable(struct wait_bit_key *key, int mode); + +struct file_kattr; +int nfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa); + +#if IS_ENABLED(CONFIG_NFS_LOCALIO) +/* localio.c */ +struct nfs_local_dio { + u32 mem_align; + u32 offset_align; + loff_t middle_offset; + loff_t end_offset; + ssize_t start_len; /* Length for misaligned first extent */ + ssize_t middle_len; /* Length for DIO-aligned middle extent */ + ssize_t end_len; /* Length for misaligned last extent */ +}; + +extern void nfs_local_probe_async(struct nfs_client *); +extern void nfs_local_probe_async_work(struct work_struct *); +extern struct nfsd_file *nfs_local_open_fh(struct nfs_client *, + const struct cred *, + struct nfs_fh *, + struct nfs_file_localio *, + const fmode_t); +extern int nfs_local_doio(struct nfs_client *, + struct nfsd_file *, + struct nfs_pgio_header *, + const struct rpc_call_ops *); +extern int nfs_local_commit(struct nfsd_file *, + struct nfs_commit_data *, + const struct rpc_call_ops *, int); +extern bool nfs_server_is_local(const struct nfs_client *clp); + +#else /* CONFIG_NFS_LOCALIO */ +static inline void nfs_local_probe(struct nfs_client *clp) {} +static inline void nfs_local_probe_async(struct nfs_client *clp) {} +static inline struct nfsd_file * +nfs_local_open_fh(struct nfs_client *clp, const struct cred *cred, + struct nfs_fh *fh, struct nfs_file_localio *nfl, + const fmode_t mode) +{ + return NULL; +} +static inline int nfs_local_doio(struct nfs_client *clp, + struct nfsd_file *localio, + struct nfs_pgio_header *hdr, + const struct rpc_call_ops *call_ops) +{ + return -EINVAL; +} +static inline int nfs_local_commit(struct nfsd_file *localio, + struct nfs_commit_data *data, + const struct rpc_call_ops *call_ops, int how) +{ + return -EINVAL; +} +static inline bool nfs_server_is_local(const struct nfs_client *clp) +{ + return false; +} +#endif /* CONFIG_NFS_LOCALIO */ + +/* super.c */ +extern const struct super_operations nfs_sops; +bool nfs_auth_info_match(const struct nfs_auth_info *, rpc_authflavor_t); +int nfs_try_get_tree(struct fs_context *); +int nfs_get_tree_common(struct fs_context *); +void nfs_kill_super(struct super_block *); + +extern int __init register_nfs_fs(void); +extern void __exit unregister_nfs_fs(void); +extern bool nfs_sb_active(struct super_block *sb); +extern void nfs_sb_deactive(struct super_block *sb); +extern int nfs_client_for_each_server(struct nfs_client *clp, + int (*fn)(struct nfs_server *, void *), + void *data); +#ifdef CONFIG_NFS_FSCACHE +extern const struct netfs_request_ops nfs_netfs_ops; +#endif + +/* io.c */ +extern __must_check int nfs_start_io_read(struct inode *inode); +extern void nfs_end_io_read(struct inode *inode); +extern __must_check int nfs_start_io_write(struct inode *inode); +extern void nfs_end_io_write(struct inode *inode); +extern __must_check int nfs_start_io_direct(struct inode *inode); +extern __must_check int nfs_start_io_direct_nowait(struct inode *inode); +extern void nfs_end_io_direct(struct inode *inode); + +static inline bool nfs_file_io_is_buffered(struct nfs_inode *nfsi) +{ + return test_bit(NFS_INO_ODIRECT, &nfsi->flags) == 0; +} + +/* Must be called with exclusively locked inode->i_rwsem */ +static inline void nfs_file_block_o_direct(struct nfs_inode *nfsi) +{ + if (test_bit(NFS_INO_ODIRECT, &nfsi->flags)) { + clear_bit(NFS_INO_ODIRECT, &nfsi->flags); + inode_dio_wait(&nfsi->vfs_inode); + } +} + + +/* namespace.c */ +#define NFS_PATH_CANONICAL 1 +extern char *nfs_path(char **p, struct dentry *dentry, + char *buffer, ssize_t buflen, unsigned flags); +extern struct vfsmount *nfs_d_automount(struct path *path); +int nfs_submount(struct fs_context *, struct nfs_server *); +int nfs_do_submount(struct fs_context *); + +/* getroot.c */ +extern int nfs_get_root(struct super_block *s, struct fs_context *fc); +#if IS_ENABLED(CONFIG_NFS_V4) +extern int nfs4_get_rootfh(struct nfs_server *server, struct nfs_fh *mntfh, bool); +#endif + +struct nfs_pgio_completion_ops; +/* read.c */ +extern const struct nfs_pgio_completion_ops nfs_async_read_completion_ops; +extern void nfs_pageio_init_read(struct nfs_pageio_descriptor *pgio, + struct inode *inode, bool force_mds, + const struct nfs_pgio_completion_ops *compl_ops); +extern bool nfs_read_alloc_scratch(struct nfs_pgio_header *hdr, size_t size); +extern int nfs_read_add_folio(struct nfs_pageio_descriptor *pgio, + struct nfs_open_context *ctx, + struct folio *folio); +extern void nfs_pageio_complete_read(struct nfs_pageio_descriptor *pgio); +extern void nfs_pageio_reset_read_mds(struct nfs_pageio_descriptor *pgio); + +/* super.c */ +void nfs_umount_begin(struct super_block *); +int nfs_statfs(struct dentry *, struct kstatfs *); +int nfs_show_options(struct seq_file *, struct dentry *); +int nfs_show_devname(struct seq_file *, struct dentry *); +int nfs_show_path(struct seq_file *, struct dentry *); +int nfs_show_stats(struct seq_file *, struct dentry *); +int nfs_reconfigure(struct fs_context *); + +/* write.c */ +extern void nfs_pageio_init_write(struct nfs_pageio_descriptor *pgio, + struct inode *inode, int ioflags, bool force_mds, + const struct nfs_pgio_completion_ops *compl_ops); +extern void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio); +extern void nfs_commit_free(struct nfs_commit_data *p); +extern void nfs_commit_prepare(struct rpc_task *task, void *calldata); +extern int nfs_initiate_commit(struct rpc_clnt *clnt, + struct nfs_commit_data *data, + const struct nfs_rpc_ops *nfs_ops, + const struct rpc_call_ops *call_ops, + int how, int flags, + struct nfsd_file *localio); +extern void nfs_init_commit(struct nfs_commit_data *data, + struct list_head *head, + struct pnfs_layout_segment *lseg, + struct nfs_commit_info *cinfo); +int nfs_scan_commit_list(struct list_head *src, struct list_head *dst, + struct nfs_commit_info *cinfo, int max); +unsigned long nfs_reqs_to_commit(struct nfs_commit_info *); +int nfs_scan_commit(struct inode *inode, struct list_head *dst, + struct nfs_commit_info *cinfo); +void nfs_mark_request_commit(struct nfs_page *req, + struct pnfs_layout_segment *lseg, + struct nfs_commit_info *cinfo, + u32 ds_commit_idx); +int nfs_write_need_commit(struct nfs_pgio_header *); +void nfs_writeback_update_inode(struct nfs_pgio_header *hdr); +int nfs_generic_commit_list(struct inode *inode, struct list_head *head, + int how, struct nfs_commit_info *cinfo); +void nfs_retry_commit(struct list_head *page_list, + struct pnfs_layout_segment *lseg, + struct nfs_commit_info *cinfo, + u32 ds_commit_idx); +void nfs_commitdata_release(struct nfs_commit_data *data); +void nfs_request_add_commit_list(struct nfs_page *req, + struct nfs_commit_info *cinfo); +void nfs_request_add_commit_list_locked(struct nfs_page *req, + struct list_head *dst, + struct nfs_commit_info *cinfo); +void nfs_request_remove_commit_list(struct nfs_page *req, + struct nfs_commit_info *cinfo); +void nfs_init_cinfo(struct nfs_commit_info *cinfo, + struct inode *inode, + struct nfs_direct_req *dreq); +int nfs_key_timeout_notify(struct file *filp, struct inode *inode); +bool nfs_ctx_key_to_expire(struct nfs_open_context *ctx, struct inode *inode); +void nfs_pageio_stop_mirroring(struct nfs_pageio_descriptor *pgio); + +int nfs_filemap_write_and_wait_range(struct address_space *mapping, + loff_t lstart, loff_t lend); + +#ifdef CONFIG_NFS_V4 +static inline void +pnfs_bucket_clear_pnfs_ds_commit_verifiers(struct pnfs_commit_bucket *buckets, + unsigned int nbuckets) +{ + unsigned int i; + + for (i = 0; i < nbuckets; i++) + buckets[i].direct_verf.committed = NFS_INVALID_STABLE_HOW; +} +static inline +void nfs_clear_pnfs_ds_commit_verifiers(struct pnfs_ds_commit_info *cinfo) +{ + struct pnfs_commit_array *array; + + rcu_read_lock(); + list_for_each_entry_rcu(array, &cinfo->commits, cinfo_list) + pnfs_bucket_clear_pnfs_ds_commit_verifiers(array->buckets, + array->nbuckets); + rcu_read_unlock(); +} +#else /* CONFIG_NFS_V4 */ +static inline +void nfs_clear_pnfs_ds_commit_verifiers(struct pnfs_ds_commit_info *cinfo) +{ +} +#endif /* CONFIG_NFS_V4 */ + +#ifdef CONFIG_MIGRATION +int nfs_migrate_folio(struct address_space *, struct folio *dst, + struct folio *src, enum migrate_mode); +#else +#define nfs_migrate_folio NULL +#endif + +static inline int +nfs_write_verifier_cmp(const struct nfs_write_verifier *v1, + const struct nfs_write_verifier *v2) +{ + return memcmp(v1->data, v2->data, sizeof(v1->data)); +} + +static inline bool +nfs_write_match_verf(const struct nfs_writeverf *verf, + struct nfs_page *req) +{ + return verf->committed > NFS_UNSTABLE && + !nfs_write_verifier_cmp(&req->wb_verf, &verf->verifier); +} + +static inline gfp_t nfs_io_gfp_mask(void) +{ + gfp_t ret = current_gfp_context(GFP_KERNEL); + + /* For workers __GFP_NORETRY only with __GFP_IO or __GFP_FS */ + if ((current->flags & PF_WQ_WORKER) && ret == GFP_KERNEL) + ret |= __GFP_NORETRY | __GFP_NOWARN; + return ret; +} + +/* + * Special version of should_remove_suid() that ignores capabilities. + */ +static inline int nfs_should_remove_suid(const struct inode *inode) +{ + umode_t mode = inode->i_mode; + int kill = 0; + + /* suid always must be killed */ + if (unlikely(mode & S_ISUID)) + kill = ATTR_KILL_SUID; + + /* + * sgid without any exec bits is just a mandatory locking mark; leave + * it alone. If some exec bits are set, it's a real sgid; kill it. + */ + if (unlikely((mode & S_ISGID) && (mode & S_IXGRP))) + kill |= ATTR_KILL_SGID; + + if (unlikely(kill && S_ISREG(mode))) + return kill; + + return 0; +} + +/* unlink.c */ +extern struct rpc_task * +nfs_async_rename(struct inode *old_dir, struct inode *new_dir, + struct dentry *old_dentry, struct dentry *new_dentry, + void (*complete)(struct rpc_task *, struct nfs_renamedata *)); +extern int nfs_sillyrename(struct inode *dir, struct dentry *dentry); + +/* direct.c */ +void nfs_init_cinfo_from_dreq(struct nfs_commit_info *cinfo, + struct nfs_direct_req *dreq); +extern ssize_t nfs_dreq_bytes_left(struct nfs_direct_req *dreq, loff_t offset); + +/* nfs4proc.c */ +extern struct nfs_client *nfs4_init_client(struct nfs_client *clp, + const struct nfs_client_initdata *); +extern int nfs41_walk_client_list(struct nfs_client *clp, + struct nfs_client **result, + const struct cred *cred); +extern void nfs4_test_session_trunk(struct rpc_clnt *clnt, + struct rpc_xprt *xprt, + void *data); + +static inline struct inode *nfs_igrab_and_active(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + + if (sb && nfs_sb_active(sb)) { + if (igrab(inode)) + return inode; + nfs_sb_deactive(sb); + } + return NULL; +} + +static inline void nfs_iput_and_deactive(struct inode *inode) +{ + if (inode != NULL) { + struct super_block *sb = inode->i_sb; + + iput(inode); + nfs_sb_deactive(sb); + } +} + +/* + * Determine the device name as a string + */ +static inline char *nfs_devname(struct dentry *dentry, + char *buffer, ssize_t buflen) +{ + char *dummy; + return nfs_path(&dummy, dentry, buffer, buflen, NFS_PATH_CANONICAL); +} + +/* + * Determine the actual block size (and log2 thereof) + */ +static inline +unsigned long nfs_block_bits(unsigned long bsize, unsigned char *nrbitsp) +{ + /* make sure blocksize is a power of two */ + if ((bsize & (bsize - 1)) || nrbitsp) { + unsigned char nrbits; + + for (nrbits = 31; nrbits && !(bsize & (1UL << nrbits)); nrbits--) + ; + bsize = 1UL << nrbits; + if (nrbitsp) + *nrbitsp = nrbits; + } + + return bsize; +} + +/* + * Calculate the number of 512byte blocks used. + */ +static inline blkcnt_t nfs_calc_block_size(u64 tsize) +{ + blkcnt_t used = (tsize + 511) >> 9; + return (used > ULONG_MAX) ? ULONG_MAX : used; +} + +/* + * Compute and set NFS server blocksize + */ +static inline +unsigned long nfs_block_size(unsigned long bsize, unsigned char *nrbitsp) +{ + if (bsize < NFS_MIN_FILE_IO_SIZE) + bsize = NFS_DEF_FILE_IO_SIZE; + else if (bsize >= NFS_MAX_FILE_IO_SIZE) + bsize = NFS_MAX_FILE_IO_SIZE; + + return nfs_block_bits(bsize, nrbitsp); +} + +/* + * Compute and set NFS server rsize / wsize + */ +static inline +unsigned long nfs_io_size(unsigned long iosize, enum xprt_transports proto) +{ + if (iosize < NFS_MIN_FILE_IO_SIZE) + iosize = NFS_DEF_FILE_IO_SIZE; + else if (iosize >= NFS_MAX_FILE_IO_SIZE) + iosize = NFS_MAX_FILE_IO_SIZE; + + if (proto == XPRT_TRANSPORT_UDP || iosize < PAGE_SIZE) + return nfs_block_bits(iosize, NULL); + return iosize & PAGE_MASK; +} + +/* + * Determine the maximum file size for a superblock + */ +static inline +void nfs_super_set_maxbytes(struct super_block *sb, __u64 maxfilesize) +{ + sb->s_maxbytes = (loff_t)maxfilesize; + if (sb->s_maxbytes > MAX_LFS_FILESIZE || sb->s_maxbytes <= 0) + sb->s_maxbytes = MAX_LFS_FILESIZE; +} + +/* + * Record the request's range as unstable (an extra writeback period) and + * mark its inode as dirty. + */ +static inline void nfs_folio_mark_unstable(struct nfs_page *req, + struct nfs_commit_info *cinfo) +{ + struct folio *folio = nfs_page_to_folio(req); + + if (folio && !cinfo->dreq) { + struct inode *inode = folio->mapping->host; + long nr = DIV_ROUND_UP(req->wb_bytes, PAGE_SIZE); + + /* This range is really still in write-back - just that the + * writeback is happening on the server now. + */ + node_stat_mod_folio(folio, NR_WRITEBACK, nr); + bdi_wb_stat_mod(inode, WB_WRITEBACK, nr); + __mark_inode_dirty(inode, I_DIRTY_DATASYNC); + } +} + +/* + * Determine the number of bytes of data the page contains + */ +static inline size_t nfs_folio_length(struct folio *folio) +{ + loff_t i_size = i_size_read(folio->mapping->host); + + if (i_size > 0) { + pgoff_t index = folio->index >> folio_order(folio); + pgoff_t end_index = (i_size - 1) >> folio_shift(folio); + if (index < end_index) + return folio_size(folio); + if (index == end_index) + return offset_in_folio(folio, i_size - 1) + 1; + } + return 0; +} + +/* + * Convert a umode to a dirent->d_type + */ +static inline +unsigned char nfs_umode_to_dtype(umode_t mode) +{ + return (mode >> 12) & 15; +} + +/* + * Determine the number of pages in an array of length 'len' and + * with a base offset of 'base' + */ +static inline unsigned int nfs_page_array_len(unsigned int base, size_t len) +{ + return ((unsigned long)len + (unsigned long)base + PAGE_SIZE - 1) >> + PAGE_SHIFT; +} + +/* + * Convert a struct timespec64 into a 64-bit change attribute + * + * This does approximately the same thing as timespec64_to_ns(), + * but for calculation efficiency, we multiply the seconds by + * 1024*1024*1024. + */ +static inline +u64 nfs_timespec_to_change_attr(const struct timespec64 *ts) +{ + return ((u64)ts->tv_sec << 30) + ts->tv_nsec; +} + +static inline u32 nfs_stateid_hash(const nfs4_stateid *stateid) +{ + return ~crc32_le(0xFFFFFFFF, &stateid->other[0], + NFS4_STATEID_OTHER_SIZE); +} + +static inline bool nfs_current_task_exiting(void) +{ + return (current->flags & PF_EXITING) != 0; +} + +static inline bool nfs_error_is_fatal(int err) +{ + switch (err) { + case -ERESTARTSYS: + case -EINTR: + case -EACCES: + case -EDQUOT: + case -EFBIG: + case -EIO: + case -ENOSPC: + case -EROFS: + case -ESTALE: + case -E2BIG: + case -ENOMEM: + case -ETIMEDOUT: + return true; + default: + return false; + } +} + +static inline bool nfs_error_is_fatal_on_server(int err) +{ + switch (err) { + case 0: + case -ERESTARTSYS: + case -EINTR: + case -ENOMEM: + return false; + } + return nfs_error_is_fatal(err); +} + +/* + * Select between a default port value and a user-specified port value. + * If a zero value is set, then autobind will be used. + */ +static inline void nfs_set_port(struct sockaddr_storage *sap, int *port, + const unsigned short default_port) +{ + if (*port == NFS_UNSPEC_PORT) + *port = default_port; + + rpc_set_port((struct sockaddr *)sap, *port); +} + +struct nfs_direct_req { + struct kref kref; /* release manager */ + + /* I/O parameters */ + struct nfs_open_context *ctx; /* file open context info */ + struct nfs_lock_context *l_ctx; /* Lock context info */ + struct kiocb * iocb; /* controlling i/o request */ + struct inode * inode; /* target file of i/o */ + + /* completion state */ + atomic_t io_count; /* i/os we're waiting for */ + spinlock_t lock; /* protect completion state */ + + loff_t io_start; /* Start offset for I/O */ + ssize_t count, /* bytes actually processed */ + max_count, /* max expected count */ + error; /* any reported error */ + struct completion completion; /* wait for i/o completion */ + + /* commit state */ + struct nfs_mds_commit_info mds_cinfo; /* Storage for cinfo */ + struct pnfs_ds_commit_info ds_cinfo; /* Storage for cinfo */ + struct work_struct work; + int flags; + /* for write */ +#define NFS_ODIRECT_DO_COMMIT (1) /* an unstable reply was received */ +#define NFS_ODIRECT_RESCHED_WRITES (2) /* write verification failed */ + /* for read */ +#define NFS_ODIRECT_SHOULD_DIRTY (3) /* dirty user-space page after read */ +#define NFS_ODIRECT_DONE INT_MAX /* write verification failed */ +}; diff --git a/fs/nfs/io.c b/fs/nfs/io.c new file mode 100644 index 000000000..2faf2003f --- /dev/null +++ b/fs/nfs/io.c @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2016 Trond Myklebust + * + * I/O and data path helper functionality. + */ + +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +/** + * nfs_start_io_read - declare the file is being used for buffered reads + * @inode: file inode + * + * Declare that a buffered read operation is about to start, and ensure + * that we block all direct I/O. + * On exit, the function ensures that the NFS_INO_ODIRECT flag is unset, + * and holds a shared lock on inode->i_rwsem to ensure that the flag + * cannot be changed. + * In practice, this means that buffered read operations are allowed to + * execute in parallel, thanks to the shared lock, whereas direct I/O + * operations need to wait to grab an exclusive lock in order to set + * NFS_INO_ODIRECT. + * Note that buffered writes and truncates both take a write lock on + * inode->i_rwsem, meaning that those are serialised w.r.t. the reads. + */ +int +nfs_start_io_read(struct inode *inode) +{ + struct nfs_inode *nfsi = NFS_I(inode); + int err; + + /* Be an optimist! */ + err = down_read_killable(&inode->i_rwsem); + if (err) + return err; + if (test_bit(NFS_INO_ODIRECT, &nfsi->flags) == 0) + return 0; + up_read(&inode->i_rwsem); + + /* Slow path.... */ + err = down_write_killable(&inode->i_rwsem); + if (err) + return err; + nfs_file_block_o_direct(nfsi); + downgrade_write(&inode->i_rwsem); + + return 0; +} + +/** + * nfs_end_io_read - declare that the buffered read operation is done + * @inode: file inode + * + * Declare that a buffered read operation is done, and release the shared + * lock on inode->i_rwsem. + */ +void +nfs_end_io_read(struct inode *inode) +{ + up_read(&inode->i_rwsem); +} + +/** + * nfs_start_io_write - declare the file is being used for buffered writes + * @inode: file inode + * + * Declare that a buffered read operation is about to start, and ensure + * that we block all direct I/O. + */ +int +nfs_start_io_write(struct inode *inode) +{ + int err; + + err = down_write_killable(&inode->i_rwsem); + if (!err) + nfs_file_block_o_direct(NFS_I(inode)); + return err; +} +EXPORT_SYMBOL_GPL(nfs_start_io_write); + +/** + * nfs_end_io_write - declare that the buffered write operation is done + * @inode: file inode + * + * Declare that a buffered write operation is done, and release the + * lock on inode->i_rwsem. + */ +void +nfs_end_io_write(struct inode *inode) +{ + up_write(&inode->i_rwsem); +} +EXPORT_SYMBOL_GPL(nfs_end_io_write); + +/* Call with exclusively locked inode->i_rwsem */ +static void nfs_block_buffered(struct nfs_inode *nfsi, struct inode *inode) +{ + if (!test_bit(NFS_INO_ODIRECT, &nfsi->flags)) { + set_bit(NFS_INO_ODIRECT, &nfsi->flags); + nfs_sync_mapping(inode->i_mapping); + } +} + +static int nfs_block_buffered_nowait(struct nfs_inode *nfsi, struct inode *inode) +{ + if (!test_bit(NFS_INO_ODIRECT, &nfsi->flags)) { + if (inode->i_mapping->nrpages != 0) + return 1; + set_bit(NFS_INO_ODIRECT, &nfsi->flags); + } + return 0; +} + +/** + * nfs_start_io_direct - declare the file is being used for direct i/o + * @inode: file inode + * + * Declare that a direct I/O operation is about to start, and ensure + * that we block all buffered I/O. + * On exit, the function ensures that the NFS_INO_ODIRECT flag is set, + * and holds a shared lock on inode->i_rwsem to ensure that the flag + * cannot be changed. + * In practice, this means that direct I/O operations are allowed to + * execute in parallel, thanks to the shared lock, whereas buffered I/O + * operations need to wait to grab an exclusive lock in order to clear + * NFS_INO_ODIRECT. + * Note that buffered writes and truncates both take a write lock on + * inode->i_rwsem, meaning that those are serialised w.r.t. O_DIRECT. + */ +int +nfs_start_io_direct(struct inode *inode) +{ + struct nfs_inode *nfsi = NFS_I(inode); + int err; + + /* Be an optimist! */ + err = down_read_killable(&inode->i_rwsem); + if (err) + return err; + if (test_bit(NFS_INO_ODIRECT, &nfsi->flags) != 0) + return 0; + up_read(&inode->i_rwsem); + + /* Slow path.... */ + err = down_write_killable(&inode->i_rwsem); + if (err) + return err; + nfs_block_buffered(nfsi, inode); + downgrade_write(&inode->i_rwsem); + + return 0; +} + +/** + * nfs_start_io_direct_nowait - non-blocking variant of nfs_start_io_direct() + * @inode: file inode + * + * Try to declare that a direct I/O operation is about to start without + * blocking. + * Ensure all buffered I/O is blocked. + * If this could not be done without blocking then returns -EAGAIN. + */ +int +nfs_start_io_direct_nowait(struct inode *inode) +{ + struct nfs_inode *nfsi = NFS_I(inode); + + if (!down_read_trylock(&inode->i_rwsem)) + return -EAGAIN; + if (test_bit(NFS_INO_ODIRECT, &nfsi->flags)) + return 0; + up_read(&inode->i_rwsem); + + /* Slow path: try to flip NFS_INO_ODIRECT without blocking. */ + if (!down_write_trylock(&inode->i_rwsem)) + return -EAGAIN; + if (nfs_block_buffered_nowait(nfsi, inode)) { + up_write(&inode->i_rwsem); + return -EAGAIN; + } + downgrade_write(&inode->i_rwsem); + return 0; +} + +/** + * nfs_end_io_direct - declare that the direct i/o operation is done + * @inode: file inode + * + * Declare that a direct I/O operation is done, and release the shared + * lock on inode->i_rwsem. + */ +void +nfs_end_io_direct(struct inode *inode) +{ + up_read(&inode->i_rwsem); +} diff --git a/fs/nfs/iostat.h b/fs/nfs/iostat.h new file mode 100644 index 000000000..49862c95b --- /dev/null +++ b/fs/nfs/iostat.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * linux/fs/nfs/iostat.h + * + * Declarations for NFS client per-mount statistics + * + * Copyright (C) 2005, 2006 Chuck Lever + * + */ + +#ifndef _NFS_IOSTAT +#define _NFS_IOSTAT + +#include +#include +#include + +struct nfs_iostats { + unsigned long long bytes[__NFSIOS_BYTESMAX]; + unsigned long events[__NFSIOS_COUNTSMAX]; +} ____cacheline_aligned; + +static inline void nfs_inc_server_stats(const struct nfs_server *server, + enum nfs_stat_eventcounters stat) +{ + this_cpu_inc(server->io_stats->events[stat]); +} + +static inline void nfs_inc_stats(const struct inode *inode, + enum nfs_stat_eventcounters stat) +{ + nfs_inc_server_stats(NFS_SERVER(inode), stat); +} + +static inline void nfs_add_server_stats(const struct nfs_server *server, + enum nfs_stat_bytecounters stat, + long addend) +{ + this_cpu_add(server->io_stats->bytes[stat], addend); +} + +static inline void nfs_add_stats(const struct inode *inode, + enum nfs_stat_bytecounters stat, + long addend) +{ + nfs_add_server_stats(NFS_SERVER(inode), stat, addend); +} + +/* + * This specialized allocator has to be a macro for its allocations to be + * accounted separately (to have a separate alloc_tag). + */ +#define nfs_alloc_iostats() alloc_percpu(struct nfs_iostats) + +static inline void nfs_free_iostats(struct nfs_iostats __percpu *stats) +{ + if (stats != NULL) + free_percpu(stats); +} + +#endif /* _NFS_IOSTAT */ diff --git a/fs/nfs/localio.c b/fs/nfs/localio.c new file mode 100644 index 000000000..e55c5977f --- /dev/null +++ b/fs/nfs/localio.c @@ -0,0 +1,1120 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * NFS client support for local clients to bypass network stack + * + * Copyright (C) 2014 Weston Andros Adamson + * Copyright (C) 2019 Trond Myklebust + * Copyright (C) 2024 Mike Snitzer + * Copyright (C) 2024 NeilBrown + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "internal.h" +#include "pnfs.h" +#include "nfstrace.h" + +#define NFSDBG_FACILITY NFSDBG_VFS + +#define NFSLOCAL_MAX_IOS 3 + +struct nfs_local_kiocb { + struct kiocb kiocb; + struct bio_vec *bvec; + struct nfs_pgio_header *hdr; + struct work_struct work; + void (*aio_complete_work)(struct work_struct *); + struct nfsd_file *localio; + /* Begin mostly DIO-specific members */ + size_t end_len; + short int end_iter_index; + atomic_t n_iters; + struct iov_iter iters[NFSLOCAL_MAX_IOS]; + bool iter_is_dio_aligned[NFSLOCAL_MAX_IOS]; + /* End mostly DIO-specific members */ +}; + +struct nfs_local_fsync_ctx { + struct nfsd_file *localio; + struct nfs_commit_data *data; + struct work_struct work; + struct completion *done; +}; + +static bool localio_enabled __read_mostly = true; +module_param(localio_enabled, bool, 0644); + +static void nfs_local_do_read(struct nfs_local_kiocb *iocb, + const struct rpc_call_ops *call_ops); +static void nfs_local_do_write(struct nfs_local_kiocb *iocb, + const struct rpc_call_ops *call_ops); + +static inline bool nfs_client_is_local(const struct nfs_client *clp) +{ + return !!rcu_access_pointer(clp->cl_uuid.net); +} + +bool nfs_server_is_local(const struct nfs_client *clp) +{ + return nfs_client_is_local(clp) && localio_enabled; +} +EXPORT_SYMBOL_GPL(nfs_server_is_local); + +/* + * UUID_IS_LOCAL XDR functions + */ + +static void localio_xdr_enc_uuidargs(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const u8 *uuid = data; + + encode_opaque_fixed(xdr, uuid, UUID_SIZE); +} + +static int localio_xdr_dec_uuidres(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *result) +{ + /* void return */ + return 0; +} + +static const struct rpc_procinfo nfs_localio_procedures[] = { + [LOCALIOPROC_UUID_IS_LOCAL] = { + .p_proc = LOCALIOPROC_UUID_IS_LOCAL, + .p_encode = localio_xdr_enc_uuidargs, + .p_decode = localio_xdr_dec_uuidres, + .p_arglen = XDR_QUADLEN(UUID_SIZE), + .p_replen = 0, + .p_statidx = LOCALIOPROC_UUID_IS_LOCAL, + .p_name = "UUID_IS_LOCAL", + }, +}; + +static unsigned int nfs_localio_counts[ARRAY_SIZE(nfs_localio_procedures)]; +static const struct rpc_version nfslocalio_version1 = { + .number = 1, + .nrprocs = ARRAY_SIZE(nfs_localio_procedures), + .procs = nfs_localio_procedures, + .counts = nfs_localio_counts, +}; + +static const struct rpc_version *nfslocalio_version[] = { + [1] = &nfslocalio_version1, +}; + +extern const struct rpc_program nfslocalio_program; +static struct rpc_stat nfslocalio_rpcstat = { &nfslocalio_program }; + +const struct rpc_program nfslocalio_program = { + .name = "nfslocalio", + .number = NFS_LOCALIO_PROGRAM, + .nrvers = ARRAY_SIZE(nfslocalio_version), + .version = nfslocalio_version, + .stats = &nfslocalio_rpcstat, +}; + +/* + * nfs_init_localioclient - Initialise an NFS localio client connection + */ +static struct rpc_clnt *nfs_init_localioclient(struct nfs_client *clp) +{ + struct rpc_clnt *rpcclient_localio; + + rpcclient_localio = rpc_bind_new_program(clp->cl_rpcclient, + &nfslocalio_program, 1); + + dprintk_rcu("%s: server (%s) %s NFS LOCALIO.\n", + __func__, rpc_peeraddr2str(clp->cl_rpcclient, RPC_DISPLAY_ADDR), + (IS_ERR(rpcclient_localio) ? "does not support" : "supports")); + + return rpcclient_localio; +} + +static bool nfs_server_uuid_is_local(struct nfs_client *clp) +{ + u8 uuid[UUID_SIZE]; + struct rpc_message msg = { + .rpc_argp = &uuid, + }; + struct rpc_clnt *rpcclient_localio; + int status; + + rpcclient_localio = nfs_init_localioclient(clp); + if (IS_ERR(rpcclient_localio)) + return false; + + export_uuid(uuid, &clp->cl_uuid.uuid); + + msg.rpc_proc = &nfs_localio_procedures[LOCALIOPROC_UUID_IS_LOCAL]; + status = rpc_call_sync(rpcclient_localio, &msg, 0); + dprintk("%s: NFS reply UUID_IS_LOCAL: status=%d\n", + __func__, status); + rpc_shutdown_client(rpcclient_localio); + + /* Server is only local if it initialized required struct members */ + if (status || !rcu_access_pointer(clp->cl_uuid.net) || !clp->cl_uuid.dom) + return false; + + return true; +} + +/* + * nfs_local_probe - probe local i/o support for an nfs_server and nfs_client + * - called after alloc_client and init_client (so cl_rpcclient exists) + * - this function is idempotent, it can be called for old or new clients + */ +static void nfs_local_probe(struct nfs_client *clp) +{ + /* Disallow localio if disabled via sysfs or AUTH_SYS isn't used */ + if (!localio_enabled || + clp->cl_rpcclient->cl_auth->au_flavor != RPC_AUTH_UNIX) { + nfs_localio_disable_client(clp); + return; + } + + if (nfs_client_is_local(clp)) + return; + + if (!nfs_uuid_begin(&clp->cl_uuid)) + return; + if (nfs_server_uuid_is_local(clp)) + nfs_localio_enable_client(clp); + nfs_uuid_end(&clp->cl_uuid); +} + +void nfs_local_probe_async_work(struct work_struct *work) +{ + struct nfs_client *clp = + container_of(work, struct nfs_client, cl_local_probe_work); + + if (!refcount_inc_not_zero(&clp->cl_count)) + return; + nfs_local_probe(clp); + nfs_put_client(clp); +} + +void nfs_local_probe_async(struct nfs_client *clp) +{ + queue_work(nfsiod_workqueue, &clp->cl_local_probe_work); +} +EXPORT_SYMBOL_GPL(nfs_local_probe_async); + +static inline void nfs_local_file_put(struct nfsd_file *localio) +{ + /* nfs_to_nfsd_file_put_local() expects an __rcu pointer + * but we have a __kernel pointer. It is always safe + * to cast a __kernel pointer to an __rcu pointer + * because the cast only weakens what is known about the pointer. + */ + struct nfsd_file __rcu *nf = (struct nfsd_file __rcu*) localio; + + nfs_to_nfsd_file_put_local(&nf); +} + +/* + * __nfs_local_open_fh - open a local filehandle in terms of nfsd_file. + * + * Returns a pointer to a struct nfsd_file or ERR_PTR. + * Caller must release returned nfsd_file with nfs_to_nfsd_file_put_local(). + */ +static struct nfsd_file * +__nfs_local_open_fh(struct nfs_client *clp, const struct cred *cred, + struct nfs_fh *fh, struct nfs_file_localio *nfl, + struct nfsd_file __rcu **pnf, + const fmode_t mode) +{ + int status = 0; + struct nfsd_file *localio; + + localio = nfs_open_local_fh(&clp->cl_uuid, clp->cl_rpcclient, + cred, fh, nfl, pnf, mode); + if (IS_ERR(localio)) { + status = PTR_ERR(localio); + switch (status) { + case -ENOMEM: + case -ENXIO: + case -ENOENT: + /* Revalidate localio */ + nfs_localio_disable_client(clp); + nfs_local_probe(clp); + } + } + trace_nfs_local_open_fh(fh, mode, status); + return localio; +} + +/* + * nfs_local_open_fh - open a local filehandle in terms of nfsd_file. + * First checking if the open nfsd_file is already cached, otherwise + * must __nfs_local_open_fh and insert the nfsd_file in nfs_file_localio. + * + * Returns a pointer to a struct nfsd_file or NULL. + */ +struct nfsd_file * +nfs_local_open_fh(struct nfs_client *clp, const struct cred *cred, + struct nfs_fh *fh, struct nfs_file_localio *nfl, + const fmode_t mode) +{ + struct nfsd_file *nf, __rcu **pnf; + + if (!nfs_server_is_local(clp)) + return NULL; + if (mode & ~(FMODE_READ | FMODE_WRITE)) + return NULL; + + if (mode & FMODE_WRITE) + pnf = &nfl->rw_file; + else + pnf = &nfl->ro_file; + + nf = __nfs_local_open_fh(clp, cred, fh, nfl, pnf, mode); + if (IS_ERR(nf)) + return NULL; + return nf; +} +EXPORT_SYMBOL_GPL(nfs_local_open_fh); + +/* + * Ensure all page cache allocations are done from GFP_NOFS context to + * prevent direct reclaim recursion back into NFS via nfs_writepages. + */ +static void +nfs_local_mapping_set_gfp_nofs_context(struct address_space *m) +{ + gfp_t gfp_mask = mapping_gfp_mask(m); + + mapping_set_gfp_mask(m, (gfp_mask & ~(__GFP_FS))); +} + +static void +nfs_local_iocb_free(struct nfs_local_kiocb *iocb) +{ + kfree(iocb->bvec); + kfree(iocb); +} + +static struct nfs_local_kiocb * +nfs_local_iocb_alloc(struct nfs_pgio_header *hdr, + struct file *file, gfp_t flags) +{ + struct nfs_local_kiocb *iocb; + + iocb = kzalloc_obj(*iocb, flags); + if (iocb == NULL) + return NULL; + + iocb->bvec = kmalloc_objs(struct bio_vec, hdr->page_array.npages, flags); + if (iocb->bvec == NULL) { + kfree(iocb); + return NULL; + } + + nfs_local_mapping_set_gfp_nofs_context(file->f_mapping); + init_sync_kiocb(&iocb->kiocb, file); + + iocb->hdr = hdr; + iocb->kiocb.ki_pos = hdr->args.offset; + iocb->kiocb.ki_flags &= ~IOCB_APPEND; + iocb->kiocb.ki_complete = NULL; + iocb->aio_complete_work = NULL; + + iocb->end_iter_index = -1; + + return iocb; +} + +static bool +nfs_is_local_dio_possible(struct nfs_local_kiocb *iocb, int rw, + size_t len, struct nfs_local_dio *local_dio) +{ + struct nfs_pgio_header *hdr = iocb->hdr; + loff_t offset = hdr->args.offset; + u32 nf_dio_mem_align, nf_dio_offset_align, nf_dio_read_offset_align; + loff_t start_end, orig_end, middle_end; + + nfs_to->nfsd_file_dio_alignment(iocb->localio, &nf_dio_mem_align, + &nf_dio_offset_align, &nf_dio_read_offset_align); + if (rw == ITER_DEST) + nf_dio_offset_align = nf_dio_read_offset_align; + + if (unlikely(!nf_dio_mem_align || !nf_dio_offset_align)) + return false; + if (unlikely(len < nf_dio_offset_align)) + return false; + + local_dio->mem_align = nf_dio_mem_align; + local_dio->offset_align = nf_dio_offset_align; + + start_end = round_up(offset, nf_dio_offset_align); + orig_end = offset + len; + middle_end = round_down(orig_end, nf_dio_offset_align); + + local_dio->middle_offset = start_end; + local_dio->end_offset = middle_end; + + local_dio->start_len = start_end - offset; + local_dio->middle_len = middle_end - start_end; + local_dio->end_len = orig_end - middle_end; + + if (rw == ITER_DEST) + trace_nfs_local_dio_read(hdr->inode, offset, len, local_dio); + else + trace_nfs_local_dio_write(hdr->inode, offset, len, local_dio); + return true; +} + +static bool nfs_iov_iter_aligned_bvec(const struct iov_iter *i, + unsigned int addr_mask, unsigned int len_mask) +{ + const struct bio_vec *bvec = i->bvec; + size_t skip = i->iov_offset; + size_t size = i->count; + + if (size & len_mask) + return false; + do { + size_t len = bvec->bv_len; + + if (len > size) + len = size; + if ((unsigned long)(bvec->bv_offset + skip) & addr_mask) + return false; + bvec++; + size -= len; + skip = 0; + } while (size); + + return true; +} + +static void +nfs_local_iter_setup(struct iov_iter *iter, int rw, struct bio_vec *bvec, + unsigned int nvecs, unsigned long total, + size_t start, size_t len) +{ + iov_iter_bvec(iter, rw, bvec, nvecs, total); + if (start) + iov_iter_advance(iter, start); + iov_iter_truncate(iter, len); +} + +/* + * Setup as many as 3 iov_iter based on extents described by @local_dio. + * Returns the number of iov_iter that were setup. + */ +static int +nfs_local_iters_setup_dio(struct nfs_local_kiocb *iocb, int rw, + unsigned int nvecs, unsigned long total, + struct nfs_local_dio *local_dio) +{ + int n_iters = 0; + struct iov_iter *iters = iocb->iters; + + /* Setup misaligned start? */ + if (local_dio->start_len) { + nfs_local_iter_setup(&iters[n_iters], rw, iocb->bvec, + nvecs, total, 0, local_dio->start_len); + ++n_iters; + } + + /* + * Setup DIO-aligned middle, if there is no misaligned end (below) + * then AIO completion is used, see nfs_local_call_{read,write} + */ + nfs_local_iter_setup(&iters[n_iters], rw, iocb->bvec, nvecs, + total, local_dio->start_len, local_dio->middle_len); + + iocb->iter_is_dio_aligned[n_iters] = + nfs_iov_iter_aligned_bvec(&iters[n_iters], + local_dio->mem_align-1, local_dio->offset_align-1); + + if (unlikely(!iocb->iter_is_dio_aligned[n_iters])) { + trace_nfs_local_dio_misaligned(iocb->hdr->inode, + local_dio->start_len, local_dio->middle_len, local_dio); + return 0; /* no DIO-aligned IO possible */ + } + iocb->end_iter_index = n_iters; + ++n_iters; + + /* Setup misaligned end? */ + if (local_dio->end_len) { + nfs_local_iter_setup(&iters[n_iters], rw, iocb->bvec, + nvecs, total, local_dio->start_len + + local_dio->middle_len, local_dio->end_len); + iocb->end_iter_index = n_iters; + ++n_iters; + } + + atomic_set(&iocb->n_iters, n_iters); + return n_iters; +} + +static noinline_for_stack void +nfs_local_iters_init(struct nfs_local_kiocb *iocb, int rw) +{ + struct nfs_pgio_header *hdr = iocb->hdr; + struct page **pagevec = hdr->page_array.pagevec; + unsigned long v, total; + unsigned int base; + size_t len; + + v = 0; + total = hdr->args.count; + base = hdr->args.pgbase; + pagevec += base >> PAGE_SHIFT; + base &= ~PAGE_MASK; + while (total && v < hdr->page_array.npages) { + len = min_t(size_t, total, PAGE_SIZE - base); + bvec_set_page(&iocb->bvec[v], *pagevec, len, base); + total -= len; + ++pagevec; + ++v; + base = 0; + } + len = hdr->args.count - total; + + /* + * For each iocb, iocb->n_iters is always at least 1 and we always + * end io after first nfs_local_pgio_done call unless misaligned DIO. + */ + atomic_set(&iocb->n_iters, 1); + + if (test_bit(NFS_IOHDR_ODIRECT, &hdr->flags)) { + struct nfs_local_dio local_dio; + + if (nfs_is_local_dio_possible(iocb, rw, len, &local_dio) && + nfs_local_iters_setup_dio(iocb, rw, v, len, &local_dio) != 0) { + /* Ensure DIO WRITE's IO on stable storage upon completion */ + if (rw == ITER_SOURCE) + iocb->kiocb.ki_flags |= IOCB_DSYNC|IOCB_SYNC; + return; /* is DIO-aligned */ + } + } + + /* Use buffered IO */ + iov_iter_bvec(&iocb->iters[0], rw, iocb->bvec, v, len); +} + +static void +nfs_local_hdr_release(struct nfs_pgio_header *hdr, + const struct rpc_call_ops *call_ops) +{ + call_ops->rpc_call_done(&hdr->task, hdr); + call_ops->rpc_release(hdr); +} + +static void +nfs_local_pgio_init(struct nfs_pgio_header *hdr, + const struct rpc_call_ops *call_ops) +{ + hdr->task.tk_ops = call_ops; + if (!hdr->task.tk_start) + hdr->task.tk_start = ktime_get(); +} + +static bool nfs_local_pgio_done(struct nfs_local_kiocb *iocb, long status) +{ + struct nfs_pgio_header *hdr = iocb->hdr; + + /* Must handle partial completions */ + if (status >= 0) { + hdr->res.count += status; + /* @hdr was initialized to 0 (zeroed during allocation) */ + if (hdr->task.tk_status == 0) + hdr->res.op_status = NFS4_OK; + } else { + hdr->res.op_status = nfs_localio_errno_to_nfs4_stat(status); + hdr->task.tk_status = status; + } + + BUG_ON(atomic_read(&iocb->n_iters) <= 0); + return atomic_dec_and_test(&iocb->n_iters); +} + +static void +nfs_local_iocb_release(struct nfs_local_kiocb *iocb) +{ + nfs_local_file_put(iocb->localio); + nfs_local_iocb_free(iocb); +} + +static void nfs_local_pgio_restart(struct nfs_local_kiocb *iocb, + struct nfs_pgio_header *hdr) +{ + int status = 0; + + iocb->kiocb.ki_pos = hdr->args.offset; + iocb->kiocb.ki_flags &= ~(IOCB_DSYNC | IOCB_SYNC | IOCB_DIRECT); + iocb->kiocb.ki_complete = NULL; + iocb->aio_complete_work = NULL; + iocb->end_iter_index = -1; + + switch (hdr->rw_mode) { + case FMODE_READ: + nfs_local_iters_init(iocb, ITER_DEST); + nfs_local_do_read(iocb, hdr->task.tk_ops); + break; + case FMODE_WRITE: + nfs_local_iters_init(iocb, ITER_SOURCE); + nfs_local_do_write(iocb, hdr->task.tk_ops); + break; + default: + status = -EOPNOTSUPP; + } + + if (unlikely(status != 0)) { + nfs_local_iocb_release(iocb); + hdr->task.tk_status = status; + nfs_local_hdr_release(hdr, hdr->task.tk_ops); + } +} + +static void nfs_local_pgio_release(struct nfs_local_kiocb *iocb) +{ + struct nfs_pgio_header *hdr = iocb->hdr; + struct rpc_task *task = &hdr->task; + + task->tk_action = NULL; + task->tk_ops->rpc_call_done(task, hdr); + + if (task->tk_action == NULL) { + nfs_local_iocb_release(iocb); + task->tk_ops->rpc_release(hdr); + } else + nfs_local_pgio_restart(iocb, hdr); +} + +/* + * Complete the I/O from iocb->kiocb.ki_complete() + * + * Note that this function can be called from a bottom half context, + * hence we need to queue the rpc_call_done() etc to a workqueue + */ +static inline void nfs_local_pgio_aio_complete(struct nfs_local_kiocb *iocb) +{ + INIT_WORK(&iocb->work, iocb->aio_complete_work); + queue_work(nfsiod_workqueue, &iocb->work); +} + +static void nfs_local_read_done(struct nfs_local_kiocb *iocb) +{ + struct nfs_pgio_header *hdr = iocb->hdr; + struct file *filp = iocb->kiocb.ki_filp; + long status = hdr->task.tk_status; + + if ((iocb->kiocb.ki_flags & IOCB_DIRECT) && status == -EINVAL) { + /* Underlying FS will return -EINVAL if misaligned DIO is attempted. */ + pr_info_ratelimited("nfs: Unexpected direct I/O read alignment failure\n"); + } + + /* + * Must clear replen otherwise NFSv3 data corruption will occur + * if/when switching from LOCALIO back to using normal RPC. + */ + hdr->res.replen = 0; + + /* nfs_readpage_result() handles short read */ + + if (hdr->args.offset + hdr->res.count >= i_size_read(file_inode(filp))) + hdr->res.eof = true; + + dprintk("%s: read %ld bytes eof %d.\n", __func__, + status > 0 ? status : 0, hdr->res.eof); +} + +static inline void nfs_local_read_iocb_done(struct nfs_local_kiocb *iocb) +{ + nfs_local_read_done(iocb); + nfs_local_pgio_release(iocb); +} + +static void nfs_local_read_aio_complete_work(struct work_struct *work) +{ + struct nfs_local_kiocb *iocb = + container_of(work, struct nfs_local_kiocb, work); + + nfs_local_read_iocb_done(iocb); +} + +static void nfs_local_read_aio_complete(struct kiocb *kiocb, long ret) +{ + struct nfs_local_kiocb *iocb = + container_of(kiocb, struct nfs_local_kiocb, kiocb); + + /* AIO completion of DIO read should always be last to complete */ + if (unlikely(!nfs_local_pgio_done(iocb, ret))) + return; + + nfs_local_pgio_aio_complete(iocb); /* Calls nfs_local_read_aio_complete_work */ +} + +static void nfs_local_call_read(struct work_struct *work) +{ + struct nfs_local_kiocb *iocb = + container_of(work, struct nfs_local_kiocb, work); + struct file *filp = iocb->kiocb.ki_filp; + ssize_t status; + int n_iters; + + n_iters = atomic_read(&iocb->n_iters); + for (int i = 0; i < n_iters ; i++) { + if (iocb->iter_is_dio_aligned[i]) { + iocb->kiocb.ki_flags |= IOCB_DIRECT; + /* Only use AIO completion if DIO-aligned segment is last */ + if (i == iocb->end_iter_index) { + iocb->kiocb.ki_complete = nfs_local_read_aio_complete; + iocb->aio_complete_work = nfs_local_read_aio_complete_work; + } + } else + iocb->kiocb.ki_flags &= ~IOCB_DIRECT; + + scoped_with_creds(filp->f_cred) + status = filp->f_op->read_iter(&iocb->kiocb, &iocb->iters[i]); + + if (status == -EIOCBQUEUED) + continue; + /* Break on completion, errors, or short reads */ + if (nfs_local_pgio_done(iocb, status) || status < 0 || + (size_t)status < iov_iter_count(&iocb->iters[i])) { + nfs_local_read_iocb_done(iocb); + break; + } + } +} + +static void nfs_local_do_read(struct nfs_local_kiocb *iocb, + const struct rpc_call_ops *call_ops) +{ + struct nfs_pgio_header *hdr = iocb->hdr; + + dprintk("%s: vfs_read count=%u pos=%llu\n", + __func__, hdr->args.count, hdr->args.offset); + + nfs_local_pgio_init(hdr, call_ops); + hdr->res.eof = false; + + INIT_WORK(&iocb->work, nfs_local_call_read); + queue_work(nfslocaliod_workqueue, &iocb->work); +} + +static void +nfs_copy_boot_verifier(struct nfs_write_verifier *verifier, struct inode *inode) +{ + struct nfs_client *clp = NFS_SERVER(inode)->nfs_client; + u32 *verf = (u32 *)verifier->data; + unsigned int seq; + + do { + seq = read_seqbegin(&clp->cl_boot_lock); + verf[0] = (u32)clp->cl_nfssvc_boot.tv_sec; + verf[1] = (u32)clp->cl_nfssvc_boot.tv_nsec; + } while (read_seqretry(&clp->cl_boot_lock, seq)); +} + +static void +nfs_reset_boot_verifier(struct inode *inode) +{ + struct nfs_client *clp = NFS_SERVER(inode)->nfs_client; + + write_seqlock(&clp->cl_boot_lock); + ktime_get_real_ts64(&clp->cl_nfssvc_boot); + write_sequnlock(&clp->cl_boot_lock); +} + +static void +nfs_set_local_verifier(struct inode *inode, + struct nfs_writeverf *verf, + enum nfs3_stable_how how) +{ + nfs_copy_boot_verifier(&verf->verifier, inode); + verf->committed = how; +} + +/* Factored out from fs/nfsd/vfs.h:fh_getattr() */ +static int __vfs_getattr(const struct path *p, struct kstat *stat, int version) +{ + u32 request_mask = STATX_BASIC_STATS; + + if (version == 4) + request_mask |= (STATX_BTIME | STATX_CHANGE_COOKIE); + return vfs_getattr(p, stat, request_mask, AT_STATX_SYNC_AS_STAT); +} + +/* Copied from fs/nfsd/nfsfh.c:nfsd4_change_attribute() */ +static u64 __nfsd4_change_attribute(const struct kstat *stat, + const struct inode *inode) +{ + u64 chattr; + + if (stat->result_mask & STATX_CHANGE_COOKIE) { + chattr = stat->change_cookie; + if (S_ISREG(inode->i_mode) && + !(stat->attributes & STATX_ATTR_CHANGE_MONOTONIC)) { + chattr += (u64)stat->ctime.tv_sec << 30; + chattr += stat->ctime.tv_nsec; + } + } else { + chattr = time_to_chattr(&stat->ctime); + } + return chattr; +} + +static void nfs_local_vfs_getattr(struct nfs_local_kiocb *iocb) +{ + struct kstat stat; + struct file *filp = iocb->kiocb.ki_filp; + struct nfs_pgio_header *hdr = iocb->hdr; + struct nfs_fattr *fattr = hdr->res.fattr; + int version = NFS_PROTO(hdr->inode)->version; + + if (unlikely(!fattr) || __vfs_getattr(&filp->f_path, &stat, version)) + return; + + fattr->valid = (NFS_ATTR_FATTR_FILEID | + NFS_ATTR_FATTR_CHANGE | + NFS_ATTR_FATTR_SIZE | + NFS_ATTR_FATTR_ATIME | + NFS_ATTR_FATTR_MTIME | + NFS_ATTR_FATTR_CTIME | + NFS_ATTR_FATTR_SPACE_USED); + + fattr->fileid = stat.ino; + fattr->size = stat.size; + fattr->atime = stat.atime; + fattr->mtime = stat.mtime; + fattr->ctime = stat.ctime; + if (version == 4) { + fattr->change_attr = + __nfsd4_change_attribute(&stat, file_inode(filp)); + } else + fattr->change_attr = nfs_timespec_to_change_attr(&fattr->ctime); + fattr->du.nfs3.used = stat.blocks << 9; +} + +static void nfs_local_write_done(struct nfs_local_kiocb *iocb) +{ + struct nfs_pgio_header *hdr = iocb->hdr; + long status = hdr->task.tk_status; + + dprintk("%s: wrote %ld bytes.\n", __func__, status > 0 ? status : 0); + + if ((iocb->kiocb.ki_flags & IOCB_DIRECT) && status == -EINVAL) { + /* Underlying FS will return -EINVAL if misaligned DIO is attempted. */ + pr_info_ratelimited("nfs: Unexpected direct I/O write alignment failure\n"); + } + + if (status < 0) + nfs_reset_boot_verifier(hdr->inode); +} + +static inline void nfs_local_write_iocb_done(struct nfs_local_kiocb *iocb) +{ + nfs_local_write_done(iocb); + nfs_local_vfs_getattr(iocb); + nfs_local_pgio_release(iocb); +} + +static void nfs_local_write_aio_complete_work(struct work_struct *work) +{ + struct nfs_local_kiocb *iocb = + container_of(work, struct nfs_local_kiocb, work); + + nfs_local_write_iocb_done(iocb); +} + +static void nfs_local_write_aio_complete(struct kiocb *kiocb, long ret) +{ + struct nfs_local_kiocb *iocb = + container_of(kiocb, struct nfs_local_kiocb, kiocb); + + /* AIO completion of DIO write should always be last to complete */ + if (unlikely(!nfs_local_pgio_done(iocb, ret))) + return; + + nfs_local_pgio_aio_complete(iocb); /* Calls nfs_local_write_aio_complete_work */ +} + +static void nfs_local_call_write(struct work_struct *work) +{ + struct nfs_local_kiocb *iocb = + container_of(work, struct nfs_local_kiocb, work); + struct file *filp = iocb->kiocb.ki_filp; + unsigned long old_flags = current->flags; + ssize_t status; + int n_iters; + + current->flags |= PF_LOCAL_THROTTLE | PF_MEMALLOC_NOIO; + + file_start_write(filp); + n_iters = atomic_read(&iocb->n_iters); + for (int i = 0; i < n_iters ; i++) { + size_t icount; + + if (iocb->iter_is_dio_aligned[i]) { + iocb->kiocb.ki_flags |= IOCB_DIRECT; + /* Only use AIO completion if DIO-aligned segment is last */ + if (i == iocb->end_iter_index) { + iocb->kiocb.ki_complete = nfs_local_write_aio_complete; + iocb->aio_complete_work = nfs_local_write_aio_complete_work; + } + } else + iocb->kiocb.ki_flags &= ~IOCB_DIRECT; + + scoped_with_creds(filp->f_cred) + status = filp->f_op->write_iter(&iocb->kiocb, &iocb->iters[i]); + + if (status == -EIOCBQUEUED) + continue; + /* Break on completion, errors, or short writes */ + icount = iov_iter_count(&iocb->iters[i]); + if (nfs_local_pgio_done(iocb, status) || status < 0 || + (size_t)status < icount) { + if ((size_t)status < icount) { + struct nfs_lock_context *ctx = + iocb->hdr->req->wb_lock_context; + + set_bit(NFS_CONTEXT_WRITE_SYNC, + &ctx->open_context->flags); + } + nfs_local_write_iocb_done(iocb); + break; + } + } + file_end_write(filp); + + current->flags = old_flags; +} + +static void nfs_local_do_write(struct nfs_local_kiocb *iocb, + const struct rpc_call_ops *call_ops) +{ + struct nfs_pgio_header *hdr = iocb->hdr; + + dprintk("%s: vfs_write count=%u pos=%llu %s\n", + __func__, hdr->args.count, hdr->args.offset, + (hdr->args.stable == NFS_UNSTABLE) ? "unstable" : "stable"); + + if (test_bit(NFS_CONTEXT_WRITE_SYNC, + &hdr->req->wb_lock_context->open_context->flags)) + hdr->args.stable = NFS_FILE_SYNC; + switch (hdr->args.stable) { + default: + break; + case NFS_DATA_SYNC: + iocb->kiocb.ki_flags |= IOCB_DSYNC; + break; + case NFS_FILE_SYNC: + iocb->kiocb.ki_flags |= IOCB_DSYNC|IOCB_SYNC; + } + + nfs_local_pgio_init(hdr, call_ops); + + nfs_set_local_verifier(hdr->inode, hdr->res.verf, hdr->args.stable); + + INIT_WORK(&iocb->work, nfs_local_call_write); + queue_work(nfslocaliod_workqueue, &iocb->work); +} + +static struct nfs_local_kiocb * +nfs_local_iocb_init(struct nfs_pgio_header *hdr, struct nfsd_file *localio) +{ + struct file *file = nfs_to->nfsd_file_file(localio); + struct nfs_local_kiocb *iocb; + gfp_t gfp_mask; + int rw; + + if (hdr->rw_mode & FMODE_READ) { + if (!file->f_op->read_iter) + return ERR_PTR(-EOPNOTSUPP); + gfp_mask = GFP_KERNEL; + rw = ITER_DEST; + } else { + if (!file->f_op->write_iter) + return ERR_PTR(-EOPNOTSUPP); + gfp_mask = GFP_NOIO; + rw = ITER_SOURCE; + } + + iocb = nfs_local_iocb_alloc(hdr, file, gfp_mask); + if (iocb == NULL) + return ERR_PTR(-ENOMEM); + iocb->hdr = hdr; + iocb->localio = localio; + + nfs_local_iters_init(iocb, rw); + + return iocb; +} + +int nfs_local_doio(struct nfs_client *clp, struct nfsd_file *localio, + struct nfs_pgio_header *hdr, + const struct rpc_call_ops *call_ops) +{ + struct nfs_local_kiocb *iocb; + int status = 0; + + if (!hdr->args.count) + return 0; + + iocb = nfs_local_iocb_init(hdr, localio); + if (IS_ERR(iocb)) + return PTR_ERR(iocb); + + switch (hdr->rw_mode) { + case FMODE_READ: + nfs_local_do_read(iocb, call_ops); + break; + case FMODE_WRITE: + nfs_local_do_write(iocb, call_ops); + break; + default: + dprintk("%s: invalid mode: %d\n", __func__, + hdr->rw_mode); + status = -EOPNOTSUPP; + } + + if (unlikely(status != 0)) { + nfs_local_iocb_release(iocb); + hdr->task.tk_status = status; + nfs_local_hdr_release(hdr, call_ops); + } + return status; +} + +static void +nfs_local_init_commit(struct nfs_commit_data *data, + const struct rpc_call_ops *call_ops) +{ + data->task.tk_ops = call_ops; +} + +static int +nfs_local_run_commit(struct file *filp, struct nfs_commit_data *data) +{ + loff_t start = data->args.offset; + loff_t end = LLONG_MAX; + + if (data->args.count > 0) { + end = start + data->args.count - 1; + if (end < start) + end = LLONG_MAX; + } + + nfs_local_mapping_set_gfp_nofs_context(filp->f_mapping); + + dprintk("%s: commit %llu - %llu\n", __func__, start, end); + return vfs_fsync_range(filp, start, end, 0); +} + +static void +nfs_local_commit_done(struct nfs_commit_data *data, int status) +{ + if (status >= 0) { + nfs_set_local_verifier(data->inode, + data->res.verf, + NFS_FILE_SYNC); + data->res.op_status = NFS4_OK; + data->task.tk_status = 0; + } else { + nfs_reset_boot_verifier(data->inode); + data->res.op_status = nfs_localio_errno_to_nfs4_stat(status); + data->task.tk_status = status; + } +} + +static void +nfs_local_release_commit_data(struct nfsd_file *localio, + struct nfs_commit_data *data, + const struct rpc_call_ops *call_ops) +{ + nfs_local_file_put(localio); + call_ops->rpc_call_done(&data->task, data); + call_ops->rpc_release(data); +} + +static void +nfs_local_fsync_ctx_free(struct nfs_local_fsync_ctx *ctx) +{ + nfs_local_release_commit_data(ctx->localio, ctx->data, + ctx->data->task.tk_ops); + kfree(ctx); +} + +static void +nfs_local_fsync_work(struct work_struct *work) +{ + unsigned long old_flags = current->flags; + struct nfs_local_fsync_ctx *ctx; + int status; + + ctx = container_of(work, struct nfs_local_fsync_ctx, work); + + current->flags |= PF_LOCAL_THROTTLE | PF_MEMALLOC_NOIO; + + status = nfs_local_run_commit(nfs_to->nfsd_file_file(ctx->localio), + ctx->data); + nfs_local_commit_done(ctx->data, status); + if (ctx->done != NULL) + complete(ctx->done); + nfs_local_fsync_ctx_free(ctx); + + current->flags = old_flags; +} + +static struct nfs_local_fsync_ctx * +nfs_local_fsync_ctx_alloc(struct nfs_commit_data *data, + struct nfsd_file *localio, gfp_t flags) +{ + struct nfs_local_fsync_ctx *ctx = kmalloc_obj(*ctx, flags); + + if (ctx != NULL) { + ctx->localio = localio; + ctx->data = data; + INIT_WORK(&ctx->work, nfs_local_fsync_work); + ctx->done = NULL; + } + return ctx; +} + +int nfs_local_commit(struct nfsd_file *localio, + struct nfs_commit_data *data, + const struct rpc_call_ops *call_ops, int how) +{ + struct nfs_local_fsync_ctx *ctx; + + ctx = nfs_local_fsync_ctx_alloc(data, localio, GFP_NOIO); + if (!ctx) { + nfs_local_commit_done(data, -ENOMEM); + nfs_local_release_commit_data(localio, data, call_ops); + return -ENOMEM; + } + + nfs_local_init_commit(data, call_ops); + + if (how & FLUSH_SYNC) { + DECLARE_COMPLETION_ONSTACK(done); + ctx->done = &done; + queue_work(nfslocaliod_workqueue, &ctx->work); + wait_for_completion(&done); + } else + queue_work(nfslocaliod_workqueue, &ctx->work); + + return 0; +} diff --git a/fs/nfs/mount_clnt.c b/fs/nfs/mount_clnt.c new file mode 100644 index 000000000..db8dfb920 --- /dev/null +++ b/fs/nfs/mount_clnt.c @@ -0,0 +1,471 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * In-kernel MOUNT protocol client + * + * Copyright (C) 1997, Olaf Kirch + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +#define NFSDBG_FACILITY NFSDBG_MOUNT + +/* + * Defined by RFC 1094, section A.3; and RFC 1813, section 5.1.4 + */ +#define MNTPATHLEN (1024) + +/* + * XDR data type sizes + */ +#define encode_dirpath_sz (1 + XDR_QUADLEN(MNTPATHLEN)) +#define MNT_status_sz (1) +#define MNT_fhandle_sz XDR_QUADLEN(NFS2_FHSIZE) +#define MNT_fhandlev3_sz XDR_QUADLEN(NFS3_FHSIZE) +#define MNT_authflav3_sz (1 + NFS_MAX_SECFLAVORS) + +/* + * XDR argument and result sizes + */ +#define MNT_enc_dirpath_sz encode_dirpath_sz +#define MNT_dec_mountres_sz (MNT_status_sz + MNT_fhandle_sz) +#define MNT_dec_mountres3_sz (MNT_status_sz + MNT_fhandlev3_sz + \ + MNT_authflav3_sz) + +/* + * Defined by RFC 1094, section A.5 + */ +enum { + MOUNTPROC_NULL = 0, + MOUNTPROC_MNT = 1, + MOUNTPROC_DUMP = 2, + MOUNTPROC_UMNT = 3, + MOUNTPROC_UMNTALL = 4, + MOUNTPROC_EXPORT = 5, +}; + +/* + * Defined by RFC 1813, section 5.2 + */ +enum { + MOUNTPROC3_NULL = 0, + MOUNTPROC3_MNT = 1, + MOUNTPROC3_DUMP = 2, + MOUNTPROC3_UMNT = 3, + MOUNTPROC3_UMNTALL = 4, + MOUNTPROC3_EXPORT = 5, +}; + +static const struct rpc_program mnt_program; + +/* + * Defined by OpenGroup XNFS Version 3W, chapter 8 + */ +enum mountstat { + MNT_OK = 0, + MNT_EPERM = 1, + MNT_ENOENT = 2, + MNT_EACCES = 13, + MNT_EINVAL = 22, +}; + +static struct { + u32 status; + int errno; +} mnt_errtbl[] = { + { .status = MNT_OK, .errno = 0, }, + { .status = MNT_EPERM, .errno = -EPERM, }, + { .status = MNT_ENOENT, .errno = -ENOENT, }, + { .status = MNT_EACCES, .errno = -EACCES, }, + { .status = MNT_EINVAL, .errno = -EINVAL, }, +}; + +/* + * Defined by RFC 1813, section 5.1.5 + */ +enum mountstat3 { + MNT3_OK = 0, /* no error */ + MNT3ERR_PERM = 1, /* Not owner */ + MNT3ERR_NOENT = 2, /* No such file or directory */ + MNT3ERR_IO = 5, /* I/O error */ + MNT3ERR_ACCES = 13, /* Permission denied */ + MNT3ERR_NOTDIR = 20, /* Not a directory */ + MNT3ERR_INVAL = 22, /* Invalid argument */ + MNT3ERR_NAMETOOLONG = 63, /* Filename too long */ + MNT3ERR_NOTSUPP = 10004, /* Operation not supported */ + MNT3ERR_SERVERFAULT = 10006, /* A failure on the server */ +}; + +static struct { + u32 status; + int errno; +} mnt3_errtbl[] = { + { .status = MNT3_OK, .errno = 0, }, + { .status = MNT3ERR_PERM, .errno = -EPERM, }, + { .status = MNT3ERR_NOENT, .errno = -ENOENT, }, + { .status = MNT3ERR_IO, .errno = -EIO, }, + { .status = MNT3ERR_ACCES, .errno = -EACCES, }, + { .status = MNT3ERR_NOTDIR, .errno = -ENOTDIR, }, + { .status = MNT3ERR_INVAL, .errno = -EINVAL, }, + { .status = MNT3ERR_NAMETOOLONG, .errno = -ENAMETOOLONG, }, + { .status = MNT3ERR_NOTSUPP, .errno = -ENOTSUPP, }, + { .status = MNT3ERR_SERVERFAULT, .errno = -EREMOTEIO, }, +}; + +struct mountres { + int errno; + struct nfs_fh *fh; + unsigned int *auth_count; + rpc_authflavor_t *auth_flavors; +}; + +/** + * nfs_mount - Obtain an NFS file handle for the given host and path + * @info: pointer to mount request arguments + * @timeo: deciseconds the mount waits for a response before it retries + * @retrans: number of times the mount retries a request + * + * Uses timeout parameters specified by caller. On successful return, the + * auth_flavs list and auth_flav_len will be populated with the list from the + * server or a faked-up list if the server didn't provide one. + */ +int nfs_mount(struct nfs_mount_request *info, int timeo, int retrans) +{ + struct rpc_timeout mnt_timeout; + struct mountres result = { + .fh = info->fh, + .auth_count = info->auth_flav_len, + .auth_flavors = info->auth_flavs, + }; + struct rpc_message msg = { + .rpc_argp = info->dirpath, + .rpc_resp = &result, + }; + struct rpc_create_args args = { + .net = info->net, + .protocol = info->protocol, + .address = (struct sockaddr *)info->sap, + .addrsize = info->salen, + .timeout = &mnt_timeout, + .servername = info->hostname, + .program = &mnt_program, + .version = info->version, + .authflavor = RPC_AUTH_UNIX, + .cred = current_cred(), + }; + struct rpc_clnt *mnt_clnt; + int status; + + dprintk("NFS: sending MNT request for %s:%s\n", + (info->hostname ? info->hostname : "server"), + info->dirpath); + + if (strlen(info->dirpath) > MNTPATHLEN) + return -ENAMETOOLONG; + + if (info->noresvport) + args.flags |= RPC_CLNT_CREATE_NONPRIVPORT; + + nfs_init_timeout_values(&mnt_timeout, info->protocol, timeo, retrans); + mnt_clnt = rpc_create(&args); + if (IS_ERR(mnt_clnt)) + goto out_clnt_err; + + if (info->version == NFS_MNT3_VERSION) + msg.rpc_proc = &mnt_clnt->cl_procinfo[MOUNTPROC3_MNT]; + else + msg.rpc_proc = &mnt_clnt->cl_procinfo[MOUNTPROC_MNT]; + + status = rpc_call_sync(mnt_clnt, &msg, RPC_TASK_SOFT|RPC_TASK_TIMEOUT); + rpc_shutdown_client(mnt_clnt); + + if (status < 0) + goto out_call_err; + if (result.errno != 0) + goto out_mnt_err; + + dprintk("NFS: MNT request succeeded\n"); + status = 0; + + /* + * If the server didn't provide a flavor list, allow the + * client to try any flavor. + */ + if (info->version != NFS_MNT3_VERSION || *info->auth_flav_len == 0) { + dprintk("NFS: Faking up auth_flavs list\n"); + info->auth_flavs[0] = RPC_AUTH_NULL; + *info->auth_flav_len = 1; + } +out: + return status; + +out_clnt_err: + status = PTR_ERR(mnt_clnt); + dprintk("NFS: failed to create MNT RPC client, status=%d\n", status); + goto out; + +out_call_err: + dprintk("NFS: MNT request failed, status=%d\n", status); + goto out; + +out_mnt_err: + dprintk("NFS: MNT server returned result %d\n", result.errno); + status = result.errno; + goto out; +} + +/* + * XDR encode/decode functions for MOUNT + */ + +static void encode_mntdirpath(struct xdr_stream *xdr, const char *pathname) +{ + const u32 pathname_len = strlen(pathname); + __be32 *p; + + p = xdr_reserve_space(xdr, 4 + pathname_len); + xdr_encode_opaque(p, pathname, pathname_len); +} + +static void mnt_xdr_enc_dirpath(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *dirpath) +{ + encode_mntdirpath(xdr, dirpath); +} + +/* + * RFC 1094: "A non-zero status indicates some sort of error. In this + * case, the status is a UNIX error number." This can be problematic + * if the server and client use different errno values for the same + * error. + * + * However, the OpenGroup XNFS spec provides a simple mapping that is + * independent of local errno values on the server and the client. + */ +static int decode_status(struct xdr_stream *xdr, struct mountres *res) +{ + unsigned int i; + u32 status; + __be32 *p; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(p == NULL)) + return -EIO; + status = be32_to_cpup(p); + + for (i = 0; i < ARRAY_SIZE(mnt_errtbl); i++) { + if (mnt_errtbl[i].status == status) { + res->errno = mnt_errtbl[i].errno; + return 0; + } + } + + dprintk("NFS: unrecognized MNT status code: %u\n", status); + res->errno = -EACCES; + return 0; +} + +static int decode_fhandle(struct xdr_stream *xdr, struct mountres *res) +{ + struct nfs_fh *fh = res->fh; + __be32 *p; + + p = xdr_inline_decode(xdr, NFS2_FHSIZE); + if (unlikely(p == NULL)) + return -EIO; + + fh->size = NFS2_FHSIZE; + memcpy(fh->data, p, NFS2_FHSIZE); + return 0; +} + +static int mnt_xdr_dec_mountres(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *data) +{ + struct mountres *res = data; + int status; + + status = decode_status(xdr, res); + if (unlikely(status != 0 || res->errno != 0)) + return status; + return decode_fhandle(xdr, res); +} + +static int decode_fhs_status(struct xdr_stream *xdr, struct mountres *res) +{ + unsigned int i; + u32 status; + __be32 *p; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(p == NULL)) + return -EIO; + status = be32_to_cpup(p); + + for (i = 0; i < ARRAY_SIZE(mnt3_errtbl); i++) { + if (mnt3_errtbl[i].status == status) { + res->errno = mnt3_errtbl[i].errno; + return 0; + } + } + + dprintk("NFS: unrecognized MNT3 status code: %u\n", status); + res->errno = -EACCES; + return 0; +} + +static int decode_fhandle3(struct xdr_stream *xdr, struct mountres *res) +{ + struct nfs_fh *fh = res->fh; + u32 size; + __be32 *p; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(p == NULL)) + return -EIO; + + size = be32_to_cpup(p); + if (size > NFS3_FHSIZE || size == 0) + return -EIO; + + p = xdr_inline_decode(xdr, size); + if (unlikely(p == NULL)) + return -EIO; + + fh->size = size; + memcpy(fh->data, p, size); + return 0; +} + +static int decode_auth_flavors(struct xdr_stream *xdr, struct mountres *res) +{ + rpc_authflavor_t *flavors = res->auth_flavors; + unsigned int *count = res->auth_count; + u32 entries, i; + __be32 *p; + + if (*count == 0) + return 0; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(p == NULL)) + return -EIO; + entries = be32_to_cpup(p); + dprintk("NFS: received %u auth flavors\n", entries); + if (entries > NFS_MAX_SECFLAVORS) + entries = NFS_MAX_SECFLAVORS; + + p = xdr_inline_decode(xdr, 4 * entries); + if (unlikely(p == NULL)) + return -EIO; + + if (entries > *count) + entries = *count; + + for (i = 0; i < entries; i++) { + flavors[i] = be32_to_cpup(p++); + dprintk("NFS: auth flavor[%u]: %d\n", i, flavors[i]); + } + *count = i; + + return 0; +} + +static int mnt_xdr_dec_mountres3(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *data) +{ + struct mountres *res = data; + int status; + + status = decode_fhs_status(xdr, res); + if (unlikely(status != 0 || res->errno != 0)) + return status; + status = decode_fhandle3(xdr, res); + if (unlikely(status != 0)) { + res->errno = -EBADHANDLE; + return 0; + } + return decode_auth_flavors(xdr, res); +} + +static const struct rpc_procinfo mnt_procedures[] = { + [MOUNTPROC_MNT] = { + .p_proc = MOUNTPROC_MNT, + .p_encode = mnt_xdr_enc_dirpath, + .p_decode = mnt_xdr_dec_mountres, + .p_arglen = MNT_enc_dirpath_sz, + .p_replen = MNT_dec_mountres_sz, + .p_statidx = MOUNTPROC_MNT, + .p_name = "MOUNT", + }, + [MOUNTPROC_UMNT] = { + .p_proc = MOUNTPROC_UMNT, + .p_encode = mnt_xdr_enc_dirpath, + .p_arglen = MNT_enc_dirpath_sz, + .p_statidx = MOUNTPROC_UMNT, + .p_name = "UMOUNT", + }, +}; + +static const struct rpc_procinfo mnt3_procedures[] = { + [MOUNTPROC3_MNT] = { + .p_proc = MOUNTPROC3_MNT, + .p_encode = mnt_xdr_enc_dirpath, + .p_decode = mnt_xdr_dec_mountres3, + .p_arglen = MNT_enc_dirpath_sz, + .p_replen = MNT_dec_mountres3_sz, + .p_statidx = MOUNTPROC3_MNT, + .p_name = "MOUNT", + }, + [MOUNTPROC3_UMNT] = { + .p_proc = MOUNTPROC3_UMNT, + .p_encode = mnt_xdr_enc_dirpath, + .p_arglen = MNT_enc_dirpath_sz, + .p_statidx = MOUNTPROC3_UMNT, + .p_name = "UMOUNT", + }, +}; + +static unsigned int mnt_counts[ARRAY_SIZE(mnt_procedures)]; +static const struct rpc_version mnt_version1 = { + .number = 1, + .nrprocs = ARRAY_SIZE(mnt_procedures), + .procs = mnt_procedures, + .counts = mnt_counts, +}; + +static unsigned int mnt3_counts[ARRAY_SIZE(mnt3_procedures)]; +static const struct rpc_version mnt_version3 = { + .number = 3, + .nrprocs = ARRAY_SIZE(mnt3_procedures), + .procs = mnt3_procedures, + .counts = mnt3_counts, +}; + +static const struct rpc_version *mnt_version[] = { + NULL, + &mnt_version1, + NULL, + &mnt_version3, +}; + +static struct rpc_stat mnt_stats; + +static const struct rpc_program mnt_program = { + .name = "mount", + .number = NFS_MNT_PROGRAM, + .nrvers = ARRAY_SIZE(mnt_version), + .version = mnt_version, + .stats = &mnt_stats, +}; diff --git a/fs/nfs/namespace.c b/fs/nfs/namespace.c new file mode 100644 index 000000000..6d0073c24 --- /dev/null +++ b/fs/nfs/namespace.c @@ -0,0 +1,386 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/nfs/namespace.c + * + * Copyright (C) 2005 Trond Myklebust + * - Modified by David Howells + * + * NFS namespace + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" +#include "nfs.h" + +#define NFSDBG_FACILITY NFSDBG_VFS + +static void nfs_expire_automounts(struct work_struct *work); + +static LIST_HEAD(nfs_automount_list); +static DECLARE_DELAYED_WORK(nfs_automount_task, nfs_expire_automounts); +int nfs_mountpoint_expiry_timeout = 500 * HZ; + +/* + * nfs_path - reconstruct the path given an arbitrary dentry + * @base - used to return pointer to the end of devname part of path + * @dentry_in - pointer to dentry + * @buffer - result buffer + * @buflen_in - length of buffer + * @flags - options (see below) + * + * Helper function for constructing the server pathname + * by arbitrary hashed dentry. + * + * This is mainly for use in figuring out the path on the + * server side when automounting on top of an existing partition + * and in generating /proc/mounts and friends. + * + * Supported flags: + * NFS_PATH_CANONICAL: ensure there is exactly one slash after + * the original device (export) name + * (if unset, the original name is returned verbatim) + */ +char *nfs_path(char **p, struct dentry *dentry_in, char *buffer, + ssize_t buflen_in, unsigned flags) +{ + char *end; + int namelen; + unsigned seq; + const char *base; + struct dentry *dentry; + ssize_t buflen; + +rename_retry: + buflen = buflen_in; + dentry = dentry_in; + end = buffer+buflen; + *--end = '\0'; + buflen--; + + seq = read_seqbegin(&rename_lock); + rcu_read_lock(); + while (1) { + spin_lock(&dentry->d_lock); + if (IS_ROOT(dentry)) + break; + namelen = dentry->d_name.len; + buflen -= namelen + 1; + if (buflen < 0) + goto Elong_unlock; + end -= namelen; + memcpy(end, dentry->d_name.name, namelen); + *--end = '/'; + spin_unlock(&dentry->d_lock); + dentry = dentry->d_parent; + } + if (read_seqretry(&rename_lock, seq)) { + spin_unlock(&dentry->d_lock); + rcu_read_unlock(); + goto rename_retry; + } + if ((flags & NFS_PATH_CANONICAL) && *end != '/') { + if (--buflen < 0) { + spin_unlock(&dentry->d_lock); + rcu_read_unlock(); + goto Elong; + } + *--end = '/'; + } + *p = end; + base = dentry->d_fsdata; + if (!base) { + spin_unlock(&dentry->d_lock); + rcu_read_unlock(); + WARN_ON(1); + return end; + } + namelen = strlen(base); + if (*end == '/') { + /* Strip off excess slashes in base string */ + while (namelen > 0 && base[namelen - 1] == '/') + namelen--; + } + buflen -= namelen; + if (buflen < 0) { + spin_unlock(&dentry->d_lock); + rcu_read_unlock(); + goto Elong; + } + end -= namelen; + memcpy(end, base, namelen); + spin_unlock(&dentry->d_lock); + rcu_read_unlock(); + return end; +Elong_unlock: + spin_unlock(&dentry->d_lock); + rcu_read_unlock(); + if (read_seqretry(&rename_lock, seq)) + goto rename_retry; +Elong: + return ERR_PTR(-ENAMETOOLONG); +} +EXPORT_SYMBOL_GPL(nfs_path); + +/* + * nfs_d_automount - Handle crossing a mountpoint on the server + * @path - The mountpoint + * + * When we encounter a mountpoint on the server, we want to set up + * a mountpoint on the client too, to prevent inode numbers from + * colliding, and to allow "df" to work properly. + * On NFSv4, we also want to allow for the fact that different + * filesystems may be migrated to different servers in a failover + * situation, and that different filesystems may want to use + * different security flavours. + */ +struct vfsmount *nfs_d_automount(struct path *path) +{ + struct nfs_fs_context *ctx; + struct fs_context *fc; + struct vfsmount *mnt = ERR_PTR(-ENOMEM); + struct nfs_server *server = NFS_SB(path->dentry->d_sb); + struct nfs_client *client = server->nfs_client; + unsigned long s_flags = path->dentry->d_sb->s_flags; + int timeout = READ_ONCE(nfs_mountpoint_expiry_timeout); + int ret; + + if (IS_ROOT(path->dentry)) + return ERR_PTR(-ESTALE); + + /* Open a new filesystem context, transferring parameters from the + * parent superblock, including the network namespace. + */ + fc = fs_context_for_submount(path->mnt->mnt_sb->s_type, path->dentry); + if (IS_ERR(fc)) + return ERR_CAST(fc); + + ctx = nfs_fc2context(fc); + ctx->clone_data.dentry = path->dentry; + ctx->clone_data.sb = path->dentry->d_sb; + ctx->clone_data.fattr = nfs_alloc_fattr(); + if (!ctx->clone_data.fattr) + goto out_fc; + + if (fc->cred != server->cred) { + put_cred(fc->cred); + fc->cred = get_cred(server->cred); + } + + if (fc->net_ns != client->cl_net) { + put_net(fc->net_ns); + fc->net_ns = get_net(client->cl_net); + } + + /* Inherit the flags covered by NFS_SB_MASK */ + fc->sb_flags_mask |= NFS_SB_MASK; + fc->sb_flags &= ~NFS_SB_MASK; + fc->sb_flags |= s_flags & NFS_SB_MASK; + + /* for submounts we want the same server; referrals will reassign */ + memcpy(&ctx->nfs_server._address, &client->cl_addr, client->cl_addrlen); + ctx->nfs_server.addrlen = client->cl_addrlen; + ctx->nfs_server.port = server->port; + + ctx->version = client->rpc_ops->version; + ctx->minorversion = client->cl_minorversion; + ctx->nfs_mod = client->cl_nfs_mod; + get_nfs_version(ctx->nfs_mod); + + /* Inherit block sizes if they were specified as mount parameters */ + if (server->automount_inherit & NFS_AUTOMOUNT_INHERIT_BSIZE) + ctx->bsize = server->bsize; + + ret = client->rpc_ops->submount(fc, server); + if (ret < 0) { + mnt = ERR_PTR(ret); + goto out_fc; + } + + up_write(&fc->root->d_sb->s_umount); + mnt = vfs_create_mount(fc); + if (IS_ERR(mnt)) + goto out_fc; + + if (timeout <= 0) + goto out_fc; + + mnt_set_expiry(mnt, &nfs_automount_list); + schedule_delayed_work(&nfs_automount_task, timeout); + +out_fc: + put_fs_context(fc); + return mnt; +} + +static int +nfs_namespace_getattr(struct mnt_idmap *idmap, + const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int query_flags) +{ + if (NFS_FH(d_inode(path->dentry))->size != 0) + return nfs_getattr(idmap, path, stat, request_mask, + query_flags); + generic_fillattr(&nop_mnt_idmap, request_mask, d_inode(path->dentry), + stat); + return 0; +} + +static int +nfs_namespace_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +{ + if (NFS_FH(d_inode(dentry))->size != 0) + return nfs_setattr(idmap, dentry, attr); + return -EACCES; +} + +const struct inode_operations nfs_mountpoint_inode_operations = { + .getattr = nfs_getattr, + .setattr = nfs_setattr, + .fileattr_get = nfs_fileattr_get, +}; + +const struct inode_operations nfs_referral_inode_operations = { + .getattr = nfs_namespace_getattr, + .setattr = nfs_namespace_setattr, + .fileattr_get = nfs_fileattr_get, +}; + +static void nfs_expire_automounts(struct work_struct *work) +{ + struct list_head *list = &nfs_automount_list; + int timeout = READ_ONCE(nfs_mountpoint_expiry_timeout); + + mark_mounts_for_expiry(list); + if (!list_empty(list) && timeout > 0) + schedule_delayed_work(&nfs_automount_task, timeout); +} + +void nfs_release_automount_timer(void) +{ + if (list_empty(&nfs_automount_list)) + cancel_delayed_work(&nfs_automount_task); +} + +/** + * nfs_do_submount - set up mountpoint when crossing a filesystem boundary + * @fc: pointer to struct nfs_fs_context + * + */ +int nfs_do_submount(struct fs_context *fc) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + struct dentry *dentry = ctx->clone_data.dentry; + struct nfs_server *server; + char *buffer, *p; + int ret; + + /* create a new volume representation */ + server = ctx->nfs_mod->rpc_ops->clone_server(NFS_SB(ctx->clone_data.sb), + ctx->mntfh, + ctx->clone_data.fattr, + ctx->selected_flavor); + + if (IS_ERR(server)) + return PTR_ERR(server); + + ctx->server = server; + + buffer = kmalloc(4096, GFP_USER); + if (!buffer) + return -ENOMEM; + + ctx->internal = true; + + p = nfs_devname(dentry, buffer, 4096); + if (IS_ERR(p)) { + nfs_errorf(fc, "NFS: Couldn't determine submount pathname"); + ret = PTR_ERR(p); + } else { + ret = vfs_parse_fs_qstr(fc, "source", + &QSTR_LEN(p, buffer + 4096 - p)); + if (!ret) + ret = vfs_get_tree(fc); + } + kfree(buffer); + return ret; +} +EXPORT_SYMBOL_GPL(nfs_do_submount); + +int nfs_submount(struct fs_context *fc, struct nfs_server *server) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + struct dentry *dentry = ctx->clone_data.dentry; + struct dentry *parent = dget_parent(dentry); + int err; + + /* Look it up again to get its attributes */ + err = server->nfs_client->rpc_ops->lookup(d_inode(parent), dentry, &dentry->d_name, + ctx->mntfh, ctx->clone_data.fattr); + dput(parent); + if (err != 0) + return err; + + ctx->selected_flavor = server->client->cl_auth->au_flavor; + return nfs_do_submount(fc); +} +EXPORT_SYMBOL_GPL(nfs_submount); + +static int param_set_nfs_timeout(const char *val, const struct kernel_param *kp) +{ + long num; + int ret; + + if (!val) + return -EINVAL; + ret = kstrtol(val, 0, &num); + if (ret) + return -EINVAL; + if (num > 0) { + if (num >= INT_MAX / HZ) + num = INT_MAX; + else + num *= HZ; + *((int *)kp->arg) = num; + if (!list_empty(&nfs_automount_list)) + mod_delayed_work(system_percpu_wq, &nfs_automount_task, num); + } else { + *((int *)kp->arg) = -1*HZ; + cancel_delayed_work(&nfs_automount_task); + } + return 0; +} + +static int param_get_nfs_timeout(char *buffer, const struct kernel_param *kp) +{ + long num = *((int *)kp->arg); + + if (num > 0) { + if (num >= INT_MAX - (HZ - 1)) + num = INT_MAX / HZ; + else + num = (num + (HZ - 1)) / HZ; + } else + num = -1; + return sysfs_emit(buffer, "%li\n", num); +} + +static const struct kernel_param_ops param_ops_nfs_timeout = { + .set = param_set_nfs_timeout, + .get = param_get_nfs_timeout, +}; +#define param_check_nfs_timeout(name, p) __param_check(name, p, int) + +module_param(nfs_mountpoint_expiry_timeout, nfs_timeout, 0644); +MODULE_PARM_DESC(nfs_mountpoint_expiry_timeout, + "Set the NFS automounted mountpoint timeout value (seconds)." + "Values <= 0 turn expiration off."); diff --git a/fs/nfs/netns.h b/fs/nfs/netns.h new file mode 100644 index 000000000..366585791 --- /dev/null +++ b/fs/nfs/netns.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * NFS-private data for each "struct net". Accessed with net_generic(). + */ + +#ifndef __NFS_NETNS_H__ +#define __NFS_NETNS_H__ + +#include +#include +#include +#include + +struct bl_dev_msg { + int32_t status; + uint32_t major, minor; +}; + +struct nfs_netns_client; + +struct nfs_net { + struct cache_detail *nfs_dns_resolve; + struct rpc_pipe *bl_device_pipe; + struct bl_dev_msg bl_mount_reply; + wait_queue_head_t bl_wq; + struct mutex bl_mutex; + struct list_head nfs_client_list; + struct list_head nfs_volume_list; +#if IS_ENABLED(CONFIG_NFS_V4) + struct idr cb_ident_idr; /* Protected by nfs_client_lock */ + unsigned short nfs_callback_tcpport; + unsigned short nfs_callback_tcpport6; + int cb_users[NFS4_MAX_MINOR_VERSION + 1]; + struct list_head nfs4_data_server_cache; + spinlock_t nfs4_data_server_lock; +#endif /* CONFIG_NFS_V4 */ + struct nfs_netns_client *nfs_client; + spinlock_t nfs_client_lock; + ktime_t boot_time; + struct rpc_stat rpcstats; +#ifdef CONFIG_PROC_FS + struct proc_dir_entry *proc_nfsfs; +#endif +}; + +extern unsigned int nfs_net_id; + +#endif diff --git a/fs/nfs/nfs.h b/fs/nfs/nfs.h new file mode 100644 index 000000000..8a5f51be0 --- /dev/null +++ b/fs/nfs/nfs.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2012 Netapp, Inc. All rights reserved. + * + * Function and structures exported by the NFS module + * for use by NFS version-specific modules. + */ +#ifndef __LINUX_INTERNAL_NFS_H +#define __LINUX_INTERNAL_NFS_H + +#include +#include +#include + +struct nfs_subversion { + struct module *owner; /* THIS_MODULE pointer */ + struct file_system_type *nfs_fs; /* NFS filesystem type */ + const struct rpc_version *rpc_vers; /* NFS version information */ + const struct nfs_rpc_ops *rpc_ops; /* NFS operations */ + const struct super_operations *sops; /* NFS Super operations */ + const struct xattr_handler * const *xattr; /* NFS xattr handlers */ +}; + +struct nfs_subversion *find_nfs_version(unsigned int); +int get_nfs_version(struct nfs_subversion *); +void put_nfs_version(struct nfs_subversion *); +void register_nfs_version(struct nfs_subversion *); +void unregister_nfs_version(struct nfs_subversion *); + +#endif /* __LINUX_INTERNAL_NFS_H */ diff --git a/fs/nfs/nfs2super.c b/fs/nfs/nfs2super.c new file mode 100644 index 000000000..b1badc70b --- /dev/null +++ b/fs/nfs/nfs2super.c @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2012 Netapp, Inc. All rights reserved. + */ +#include +#include +#include "internal.h" +#include "nfs.h" + +static struct nfs_subversion nfs_v2 = { + .owner = THIS_MODULE, + .nfs_fs = &nfs_fs_type, + .rpc_vers = &nfs_version2, + .rpc_ops = &nfs_v2_clientops, + .sops = &nfs_sops, +}; + +static int __init init_nfs_v2(void) +{ + register_nfs_version(&nfs_v2); + return 0; +} + +static void __exit exit_nfs_v2(void) +{ + unregister_nfs_version(&nfs_v2); +} + +MODULE_DESCRIPTION("NFSv2 client support"); +MODULE_LICENSE("GPL"); + +module_init(init_nfs_v2); +module_exit(exit_nfs_v2); diff --git a/fs/nfs/nfs2xdr.c b/fs/nfs/nfs2xdr.c new file mode 100644 index 000000000..9eff09158 --- /dev/null +++ b/fs/nfs/nfs2xdr.c @@ -0,0 +1,1088 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/nfs/nfs2xdr.c + * + * XDR functions to encode/decode NFS RPC arguments and results. + * + * Copyright (C) 1992, 1993, 1994 Rick Sladkey + * Copyright (C) 1996 Olaf Kirch + * 04 Aug 1998 Ion Badulescu + * FIFO's need special handling in NFSv2 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" +#include "nfstrace.h" + +#define NFSDBG_FACILITY NFSDBG_XDR + +/* + * Declare the space requirements for NFS arguments and replies as + * number of 32bit-words + */ +#define NFS_pagepad_sz (1) /* Page padding */ +#define NFS_fhandle_sz (8) +#define NFS_sattr_sz (8) +#define NFS_filename_sz (1+(NFS2_MAXNAMLEN>>2)) +#define NFS_path_sz (1+(NFS2_MAXPATHLEN>>2)) +#define NFS_fattr_sz (17) +#define NFS_info_sz (5) +#define NFS_entry_sz (NFS_filename_sz+3) + +#define NFS_diropargs_sz (NFS_fhandle_sz+NFS_filename_sz) +#define NFS_removeargs_sz (NFS_fhandle_sz+NFS_filename_sz) +#define NFS_sattrargs_sz (NFS_fhandle_sz+NFS_sattr_sz) +#define NFS_readlinkargs_sz (NFS_fhandle_sz) +#define NFS_readargs_sz (NFS_fhandle_sz+3) +#define NFS_writeargs_sz (NFS_fhandle_sz+4) +#define NFS_createargs_sz (NFS_diropargs_sz+NFS_sattr_sz) +#define NFS_renameargs_sz (NFS_diropargs_sz+NFS_diropargs_sz) +#define NFS_linkargs_sz (NFS_fhandle_sz+NFS_diropargs_sz) +#define NFS_symlinkargs_sz (NFS_diropargs_sz+1+NFS_sattr_sz) +#define NFS_readdirargs_sz (NFS_fhandle_sz+2) + +#define NFS_attrstat_sz (1+NFS_fattr_sz) +#define NFS_diropres_sz (1+NFS_fhandle_sz+NFS_fattr_sz) +#define NFS_readlinkres_sz (2+NFS_pagepad_sz) +#define NFS_readres_sz (1+NFS_fattr_sz+1+NFS_pagepad_sz) +#define NFS_writeres_sz (NFS_attrstat_sz) +#define NFS_stat_sz (1) +#define NFS_readdirres_sz (1+NFS_pagepad_sz) +#define NFS_statfsres_sz (1+NFS_info_sz) + +/* + * Encode/decode NFSv2 basic data types + * + * Basic NFSv2 data types are defined in section 2.3 of RFC 1094: + * "NFS: Network File System Protocol Specification". + * + * Not all basic data types have their own encoding and decoding + * functions. For run-time efficiency, some data types are encoded + * or decoded inline. + */ + +static struct user_namespace *rpc_userns(const struct rpc_clnt *clnt) +{ + if (clnt && clnt->cl_cred) + return clnt->cl_cred->user_ns; + return &init_user_ns; +} + +static struct user_namespace *rpc_rqst_userns(const struct rpc_rqst *rqstp) +{ + if (rqstp->rq_task) + return rpc_userns(rqstp->rq_task->tk_client); + return &init_user_ns; +} + +/* + * typedef opaque nfsdata<>; + */ +static int decode_nfsdata(struct xdr_stream *xdr, struct nfs_pgio_res *result) +{ + u32 recvd, count; + __be32 *p; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + count = be32_to_cpup(p); + recvd = xdr_read_pages(xdr, count); + if (unlikely(count > recvd)) + goto out_cheating; +out: + result->eof = 0; /* NFSv2 does not pass EOF flag on the wire. */ + result->count = count; + return count; +out_cheating: + dprintk("NFS: server cheating in read result: " + "count %u > recvd %u\n", count, recvd); + count = recvd; + goto out; +} + +/* + * enum stat { + * NFS_OK = 0, + * NFSERR_PERM = 1, + * NFSERR_NOENT = 2, + * NFSERR_IO = 5, + * NFSERR_NXIO = 6, + * NFSERR_ACCES = 13, + * NFSERR_EXIST = 17, + * NFSERR_NODEV = 19, + * NFSERR_NOTDIR = 20, + * NFSERR_ISDIR = 21, + * NFSERR_FBIG = 27, + * NFSERR_NOSPC = 28, + * NFSERR_ROFS = 30, + * NFSERR_NAMETOOLONG = 63, + * NFSERR_NOTEMPTY = 66, + * NFSERR_DQUOT = 69, + * NFSERR_STALE = 70, + * NFSERR_WFLUSH = 99 + * }; + */ +static int decode_stat(struct xdr_stream *xdr, enum nfs_stat *status) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + if (unlikely(*p != cpu_to_be32(NFS_OK))) + goto out_status; + *status = 0; + return 0; +out_status: + *status = be32_to_cpup(p); + trace_nfs_xdr_status(xdr, (int)*status); + return 0; +} + +/* + * 2.3.2. ftype + * + * enum ftype { + * NFNON = 0, + * NFREG = 1, + * NFDIR = 2, + * NFBLK = 3, + * NFCHR = 4, + * NFLNK = 5 + * }; + * + */ +static __be32 *xdr_decode_ftype(__be32 *p, u32 *type) +{ + *type = be32_to_cpup(p++); + if (unlikely(*type > NF2FIFO)) + *type = NFBAD; + return p; +} + +/* + * 2.3.3. fhandle + * + * typedef opaque fhandle[FHSIZE]; + */ +static void encode_fhandle(struct xdr_stream *xdr, const struct nfs_fh *fh) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, NFS2_FHSIZE); + memcpy(p, fh->data, NFS2_FHSIZE); +} + +static int decode_fhandle(struct xdr_stream *xdr, struct nfs_fh *fh) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, NFS2_FHSIZE); + if (unlikely(!p)) + return -EIO; + fh->size = NFS2_FHSIZE; + memcpy(fh->data, p, NFS2_FHSIZE); + return 0; +} + +/* + * 2.3.4. timeval + * + * struct timeval { + * unsigned int seconds; + * unsigned int useconds; + * }; + */ +static __be32 *xdr_encode_time(__be32 *p, const struct timespec64 *timep) +{ + *p++ = cpu_to_be32((u32)timep->tv_sec); + if (timep->tv_nsec != 0) + *p++ = cpu_to_be32(timep->tv_nsec / NSEC_PER_USEC); + else + *p++ = cpu_to_be32(0); + return p; +} + +/* + * Passing the invalid value useconds=1000000 is a Sun convention for + * "set to current server time". It's needed to make permissions checks + * for the "touch" program across v2 mounts to Solaris and Irix servers + * work correctly. See description of sattr in section 6.1 of "NFS + * Illustrated" by Brent Callaghan, Addison-Wesley, ISBN 0-201-32750-5. + */ +static __be32 *xdr_encode_current_server_time(__be32 *p, + const struct timespec64 *timep) +{ + *p++ = cpu_to_be32(timep->tv_sec); + *p++ = cpu_to_be32(1000000); + return p; +} + +static __be32 *xdr_decode_time(__be32 *p, struct timespec64 *timep) +{ + timep->tv_sec = be32_to_cpup(p++); + timep->tv_nsec = be32_to_cpup(p++) * NSEC_PER_USEC; + return p; +} + +/* + * 2.3.5. fattr + * + * struct fattr { + * ftype type; + * unsigned int mode; + * unsigned int nlink; + * unsigned int uid; + * unsigned int gid; + * unsigned int size; + * unsigned int blocksize; + * unsigned int rdev; + * unsigned int blocks; + * unsigned int fsid; + * unsigned int fileid; + * timeval atime; + * timeval mtime; + * timeval ctime; + * }; + * + */ +static int decode_fattr(struct xdr_stream *xdr, struct nfs_fattr *fattr, + struct user_namespace *userns) +{ + u32 rdev, type; + __be32 *p; + + p = xdr_inline_decode(xdr, NFS_fattr_sz << 2); + if (unlikely(!p)) + return -EIO; + + fattr->valid |= NFS_ATTR_FATTR_V2; + + p = xdr_decode_ftype(p, &type); + + fattr->mode = be32_to_cpup(p++); + fattr->nlink = be32_to_cpup(p++); + fattr->uid = make_kuid(userns, be32_to_cpup(p++)); + if (!uid_valid(fattr->uid)) + goto out_uid; + fattr->gid = make_kgid(userns, be32_to_cpup(p++)); + if (!gid_valid(fattr->gid)) + goto out_gid; + + fattr->size = be32_to_cpup(p++); + fattr->du.nfs2.blocksize = be32_to_cpup(p++); + + rdev = be32_to_cpup(p++); + fattr->rdev = new_decode_dev(rdev); + if (type == (u32)NFCHR && rdev == (u32)NFS2_FIFO_DEV) { + fattr->mode = (fattr->mode & ~S_IFMT) | S_IFIFO; + fattr->rdev = 0; + } + + fattr->du.nfs2.blocks = be32_to_cpup(p++); + fattr->fsid.major = be32_to_cpup(p++); + fattr->fsid.minor = 0; + fattr->fileid = be32_to_cpup(p++); + + p = xdr_decode_time(p, &fattr->atime); + p = xdr_decode_time(p, &fattr->mtime); + xdr_decode_time(p, &fattr->ctime); + fattr->change_attr = nfs_timespec_to_change_attr(&fattr->ctime); + + return 0; +out_uid: + dprintk("NFS: returned invalid uid\n"); + return -EINVAL; +out_gid: + dprintk("NFS: returned invalid gid\n"); + return -EINVAL; +} + +/* + * 2.3.6. sattr + * + * struct sattr { + * unsigned int mode; + * unsigned int uid; + * unsigned int gid; + * unsigned int size; + * timeval atime; + * timeval mtime; + * }; + */ + +#define NFS2_SATTR_NOT_SET (0xffffffff) + +static __be32 *xdr_time_not_set(__be32 *p) +{ + *p++ = cpu_to_be32(NFS2_SATTR_NOT_SET); + *p++ = cpu_to_be32(NFS2_SATTR_NOT_SET); + return p; +} + +static void encode_sattr(struct xdr_stream *xdr, const struct iattr *attr, + struct user_namespace *userns) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, NFS_sattr_sz << 2); + + if (attr->ia_valid & ATTR_MODE) + *p++ = cpu_to_be32(attr->ia_mode); + else + *p++ = cpu_to_be32(NFS2_SATTR_NOT_SET); + if (attr->ia_valid & ATTR_UID) + *p++ = cpu_to_be32(from_kuid_munged(userns, attr->ia_uid)); + else + *p++ = cpu_to_be32(NFS2_SATTR_NOT_SET); + if (attr->ia_valid & ATTR_GID) + *p++ = cpu_to_be32(from_kgid_munged(userns, attr->ia_gid)); + else + *p++ = cpu_to_be32(NFS2_SATTR_NOT_SET); + if (attr->ia_valid & ATTR_SIZE) + *p++ = cpu_to_be32((u32)attr->ia_size); + else + *p++ = cpu_to_be32(NFS2_SATTR_NOT_SET); + + if (attr->ia_valid & ATTR_ATIME_SET) + p = xdr_encode_time(p, &attr->ia_atime); + else if (attr->ia_valid & ATTR_ATIME) + p = xdr_encode_current_server_time(p, &attr->ia_atime); + else + p = xdr_time_not_set(p); + if (attr->ia_valid & ATTR_MTIME_SET) + xdr_encode_time(p, &attr->ia_mtime); + else if (attr->ia_valid & ATTR_MTIME) + xdr_encode_current_server_time(p, &attr->ia_mtime); + else + xdr_time_not_set(p); +} + +/* + * 2.3.7. filename + * + * typedef string filename; + */ +static void encode_filename(struct xdr_stream *xdr, + const char *name, u32 length) +{ + __be32 *p; + + WARN_ON_ONCE(length > NFS2_MAXNAMLEN); + p = xdr_reserve_space(xdr, 4 + length); + xdr_encode_opaque(p, name, length); +} + +static int decode_filename_inline(struct xdr_stream *xdr, + const char **name, u32 *length) +{ + __be32 *p; + u32 count; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + count = be32_to_cpup(p); + if (count > NFS3_MAXNAMLEN) + goto out_nametoolong; + p = xdr_inline_decode(xdr, count); + if (unlikely(!p)) + return -EIO; + *name = (const char *)p; + *length = count; + return 0; +out_nametoolong: + dprintk("NFS: returned filename too long: %u\n", count); + return -ENAMETOOLONG; +} + +/* + * 2.3.8. path + * + * typedef string path; + */ +static void encode_path(struct xdr_stream *xdr, struct page **pages, u32 length) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, 4); + *p = cpu_to_be32(length); + xdr_write_pages(xdr, pages, 0, length); +} + +static int decode_path(struct xdr_stream *xdr) +{ + u32 length, recvd; + __be32 *p; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + length = be32_to_cpup(p); + if (unlikely(length >= xdr->buf->page_len || length > NFS_MAXPATHLEN)) + goto out_size; + recvd = xdr_read_pages(xdr, length); + if (unlikely(length > recvd)) + goto out_cheating; + xdr_terminate_string(xdr->buf, length); + return 0; +out_size: + dprintk("NFS: returned pathname too long: %u\n", length); + return -ENAMETOOLONG; +out_cheating: + dprintk("NFS: server cheating in pathname result: " + "length %u > received %u\n", length, recvd); + return -EIO; +} + +/* + * 2.3.9. attrstat + * + * union attrstat switch (stat status) { + * case NFS_OK: + * fattr attributes; + * default: + * void; + * }; + */ +static int decode_attrstat(struct xdr_stream *xdr, struct nfs_fattr *result, + __u32 *op_status, + struct user_namespace *userns) +{ + enum nfs_stat status; + int error; + + error = decode_stat(xdr, &status); + if (unlikely(error)) + goto out; + if (op_status) + *op_status = status; + if (status != NFS_OK) + goto out_default; + error = decode_fattr(xdr, result, userns); +out: + return error; +out_default: + return nfs_stat_to_errno(status); +} + +/* + * 2.3.10. diropargs + * + * struct diropargs { + * fhandle dir; + * filename name; + * }; + */ +static void encode_diropargs(struct xdr_stream *xdr, const struct nfs_fh *fh, + const char *name, u32 length) +{ + encode_fhandle(xdr, fh); + encode_filename(xdr, name, length); +} + +/* + * 2.3.11. diropres + * + * union diropres switch (stat status) { + * case NFS_OK: + * struct { + * fhandle file; + * fattr attributes; + * } diropok; + * default: + * void; + * }; + */ +static int decode_diropok(struct xdr_stream *xdr, struct nfs_diropok *result, + struct user_namespace *userns) +{ + int error; + + error = decode_fhandle(xdr, result->fh); + if (unlikely(error)) + goto out; + error = decode_fattr(xdr, result->fattr, userns); +out: + return error; +} + +static int decode_diropres(struct xdr_stream *xdr, struct nfs_diropok *result, + struct user_namespace *userns) +{ + enum nfs_stat status; + int error; + + error = decode_stat(xdr, &status); + if (unlikely(error)) + goto out; + if (status != NFS_OK) + goto out_default; + error = decode_diropok(xdr, result, userns); +out: + return error; +out_default: + return nfs_stat_to_errno(status); +} + + +/* + * NFSv2 XDR encode functions + * + * NFSv2 argument types are defined in section 2.2 of RFC 1094: + * "NFS: Network File System Protocol Specification". + */ + +static void nfs2_xdr_enc_fhandle(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_fh *fh = data; + + encode_fhandle(xdr, fh); +} + +/* + * 2.2.3. sattrargs + * + * struct sattrargs { + * fhandle file; + * sattr attributes; + * }; + */ +static void nfs2_xdr_enc_sattrargs(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_sattrargs *args = data; + + encode_fhandle(xdr, args->fh); + encode_sattr(xdr, args->sattr, rpc_rqst_userns(req)); +} + +static void nfs2_xdr_enc_diropargs(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_diropargs *args = data; + + encode_diropargs(xdr, args->fh, args->name, args->len); +} + +static void nfs2_xdr_enc_readlinkargs(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_readlinkargs *args = data; + + encode_fhandle(xdr, args->fh); + rpc_prepare_reply_pages(req, args->pages, args->pgbase, args->pglen, + NFS_readlinkres_sz - NFS_pagepad_sz); +} + +/* + * 2.2.7. readargs + * + * struct readargs { + * fhandle file; + * unsigned offset; + * unsigned count; + * unsigned totalcount; + * }; + */ +static void encode_readargs(struct xdr_stream *xdr, + const struct nfs_pgio_args *args) +{ + u32 offset = args->offset; + u32 count = args->count; + __be32 *p; + + encode_fhandle(xdr, args->fh); + + p = xdr_reserve_space(xdr, 4 + 4 + 4); + *p++ = cpu_to_be32(offset); + *p++ = cpu_to_be32(count); + *p = cpu_to_be32(count); +} + +static void nfs2_xdr_enc_readargs(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_pgio_args *args = data; + + encode_readargs(xdr, args); + rpc_prepare_reply_pages(req, args->pages, args->pgbase, args->count, + NFS_readres_sz - NFS_pagepad_sz); + req->rq_rcv_buf.flags |= XDRBUF_READ; +} + +/* + * 2.2.9. writeargs + * + * struct writeargs { + * fhandle file; + * unsigned beginoffset; + * unsigned offset; + * unsigned totalcount; + * nfsdata data; + * }; + */ +static void encode_writeargs(struct xdr_stream *xdr, + const struct nfs_pgio_args *args) +{ + u32 offset = args->offset; + u32 count = args->count; + __be32 *p; + + encode_fhandle(xdr, args->fh); + + p = xdr_reserve_space(xdr, 4 + 4 + 4 + 4); + *p++ = cpu_to_be32(offset); + *p++ = cpu_to_be32(offset); + *p++ = cpu_to_be32(count); + + /* nfsdata */ + *p = cpu_to_be32(count); + xdr_write_pages(xdr, args->pages, args->pgbase, count); +} + +static void nfs2_xdr_enc_writeargs(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_pgio_args *args = data; + + encode_writeargs(xdr, args); + xdr->buf->flags |= XDRBUF_WRITE; +} + +/* + * 2.2.10. createargs + * + * struct createargs { + * diropargs where; + * sattr attributes; + * }; + */ +static void nfs2_xdr_enc_createargs(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_createargs *args = data; + + encode_diropargs(xdr, args->fh, args->name, args->len); + encode_sattr(xdr, args->sattr, rpc_rqst_userns(req)); +} + +static void nfs2_xdr_enc_removeargs(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_removeargs *args = data; + + encode_diropargs(xdr, args->fh, args->name.name, args->name.len); +} + +/* + * 2.2.12. renameargs + * + * struct renameargs { + * diropargs from; + * diropargs to; + * }; + */ +static void nfs2_xdr_enc_renameargs(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_renameargs *args = data; + const struct qstr *old = args->old_name; + const struct qstr *new = args->new_name; + + encode_diropargs(xdr, args->old_dir, old->name, old->len); + encode_diropargs(xdr, args->new_dir, new->name, new->len); +} + +/* + * 2.2.13. linkargs + * + * struct linkargs { + * fhandle from; + * diropargs to; + * }; + */ +static void nfs2_xdr_enc_linkargs(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_linkargs *args = data; + + encode_fhandle(xdr, args->fromfh); + encode_diropargs(xdr, args->tofh, args->toname, args->tolen); +} + +/* + * 2.2.14. symlinkargs + * + * struct symlinkargs { + * diropargs from; + * path to; + * sattr attributes; + * }; + */ +static void nfs2_xdr_enc_symlinkargs(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_symlinkargs *args = data; + + encode_diropargs(xdr, args->fromfh, args->fromname, args->fromlen); + encode_path(xdr, args->pages, args->pathlen); + encode_sattr(xdr, args->sattr, rpc_rqst_userns(req)); +} + +/* + * 2.2.17. readdirargs + * + * struct readdirargs { + * fhandle dir; + * nfscookie cookie; + * unsigned count; + * }; + */ +static void encode_readdirargs(struct xdr_stream *xdr, + const struct nfs_readdirargs *args) +{ + __be32 *p; + + encode_fhandle(xdr, args->fh); + + p = xdr_reserve_space(xdr, 4 + 4); + *p++ = cpu_to_be32(args->cookie); + *p = cpu_to_be32(args->count); +} + +static void nfs2_xdr_enc_readdirargs(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_readdirargs *args = data; + + encode_readdirargs(xdr, args); + rpc_prepare_reply_pages(req, args->pages, 0, args->count, + NFS_readdirres_sz - NFS_pagepad_sz); +} + +/* + * NFSv2 XDR decode functions + * + * NFSv2 result types are defined in section 2.2 of RFC 1094: + * "NFS: Network File System Protocol Specification". + */ + +static int nfs2_xdr_dec_stat(struct rpc_rqst *req, struct xdr_stream *xdr, + void *__unused) +{ + enum nfs_stat status; + int error; + + error = decode_stat(xdr, &status); + if (unlikely(error)) + goto out; + if (status != NFS_OK) + goto out_default; +out: + return error; +out_default: + return nfs_stat_to_errno(status); +} + +static int nfs2_xdr_dec_attrstat(struct rpc_rqst *req, struct xdr_stream *xdr, + void *result) +{ + return decode_attrstat(xdr, result, NULL, rpc_rqst_userns(req)); +} + +static int nfs2_xdr_dec_diropres(struct rpc_rqst *req, struct xdr_stream *xdr, + void *result) +{ + return decode_diropres(xdr, result, rpc_rqst_userns(req)); +} + +/* + * 2.2.6. readlinkres + * + * union readlinkres switch (stat status) { + * case NFS_OK: + * path data; + * default: + * void; + * }; + */ +static int nfs2_xdr_dec_readlinkres(struct rpc_rqst *req, + struct xdr_stream *xdr, void *__unused) +{ + enum nfs_stat status; + int error; + + error = decode_stat(xdr, &status); + if (unlikely(error)) + goto out; + if (status != NFS_OK) + goto out_default; + error = decode_path(xdr); +out: + return error; +out_default: + return nfs_stat_to_errno(status); +} + +/* + * 2.2.7. readres + * + * union readres switch (stat status) { + * case NFS_OK: + * fattr attributes; + * nfsdata data; + * default: + * void; + * }; + */ +static int nfs2_xdr_dec_readres(struct rpc_rqst *req, struct xdr_stream *xdr, + void *data) +{ + struct nfs_pgio_res *result = data; + enum nfs_stat status; + int error; + + error = decode_stat(xdr, &status); + if (unlikely(error)) + goto out; + result->op_status = status; + if (status != NFS_OK) + goto out_default; + error = decode_fattr(xdr, result->fattr, rpc_rqst_userns(req)); + if (unlikely(error)) + goto out; + error = decode_nfsdata(xdr, result); +out: + return error; +out_default: + return nfs_stat_to_errno(status); +} + +static int nfs2_xdr_dec_writeres(struct rpc_rqst *req, struct xdr_stream *xdr, + void *data) +{ + struct nfs_pgio_res *result = data; + + /* All NFSv2 writes are "file sync" writes */ + result->verf->committed = NFS_FILE_SYNC; + return decode_attrstat(xdr, result->fattr, &result->op_status, + rpc_rqst_userns(req)); +} + +/** + * nfs2_decode_dirent - Decode a single NFSv2 directory entry stored in + * the local page cache. + * @xdr: XDR stream where entry resides + * @entry: buffer to fill in with entry data + * @plus: boolean indicating whether this should be a readdirplus entry + * + * Returns zero if successful, otherwise a negative errno value is + * returned. + * + * This function is not invoked during READDIR reply decoding, but + * rather whenever an application invokes the getdents(2) system call + * on a directory already in our cache. + * + * 2.2.17. entry + * + * struct entry { + * unsigned fileid; + * filename name; + * nfscookie cookie; + * entry *nextentry; + * }; + */ +int nfs2_decode_dirent(struct xdr_stream *xdr, struct nfs_entry *entry, + bool plus) +{ + __be32 *p; + int error; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EAGAIN; + if (*p++ == xdr_zero) { + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EAGAIN; + if (*p++ == xdr_zero) + return -EAGAIN; + entry->eof = 1; + return -EBADCOOKIE; + } + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EAGAIN; + entry->ino = be32_to_cpup(p); + + error = decode_filename_inline(xdr, &entry->name, &entry->len); + if (unlikely(error)) + return error == -ENAMETOOLONG ? -ENAMETOOLONG : -EAGAIN; + + /* + * The type (size and byte order) of nfscookie isn't defined in + * RFC 1094. This implementation assumes that it's an XDR uint32. + */ + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EAGAIN; + entry->cookie = be32_to_cpup(p); + + entry->d_type = DT_UNKNOWN; + + return 0; +} + +/* + * 2.2.17. readdirres + * + * union readdirres switch (stat status) { + * case NFS_OK: + * struct { + * entry *entries; + * bool eof; + * } readdirok; + * default: + * void; + * }; + * + * Read the directory contents into the page cache, but don't + * touch them. The actual decoding is done by nfs2_decode_dirent() + * during subsequent nfs_readdir() calls. + */ +static int decode_readdirok(struct xdr_stream *xdr) +{ + return xdr_read_pages(xdr, xdr->buf->page_len); +} + +static int nfs2_xdr_dec_readdirres(struct rpc_rqst *req, + struct xdr_stream *xdr, void *__unused) +{ + enum nfs_stat status; + int error; + + error = decode_stat(xdr, &status); + if (unlikely(error)) + goto out; + if (status != NFS_OK) + goto out_default; + error = decode_readdirok(xdr); +out: + return error; +out_default: + return nfs_stat_to_errno(status); +} + +/* + * 2.2.18. statfsres + * + * union statfsres (stat status) { + * case NFS_OK: + * struct { + * unsigned tsize; + * unsigned bsize; + * unsigned blocks; + * unsigned bfree; + * unsigned bavail; + * } info; + * default: + * void; + * }; + */ +static int decode_info(struct xdr_stream *xdr, struct nfs2_fsstat *result) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, NFS_info_sz << 2); + if (unlikely(!p)) + return -EIO; + result->tsize = be32_to_cpup(p++); + result->bsize = be32_to_cpup(p++); + result->blocks = be32_to_cpup(p++); + result->bfree = be32_to_cpup(p++); + result->bavail = be32_to_cpup(p); + return 0; +} + +static int nfs2_xdr_dec_statfsres(struct rpc_rqst *req, struct xdr_stream *xdr, + void *result) +{ + enum nfs_stat status; + int error; + + error = decode_stat(xdr, &status); + if (unlikely(error)) + goto out; + if (status != NFS_OK) + goto out_default; + error = decode_info(xdr, result); +out: + return error; +out_default: + return nfs_stat_to_errno(status); +} + +#define PROC(proc, argtype, restype, timer) \ +[NFSPROC_##proc] = { \ + .p_proc = NFSPROC_##proc, \ + .p_encode = nfs2_xdr_enc_##argtype, \ + .p_decode = nfs2_xdr_dec_##restype, \ + .p_arglen = NFS_##argtype##_sz, \ + .p_replen = NFS_##restype##_sz, \ + .p_timer = timer, \ + .p_statidx = NFSPROC_##proc, \ + .p_name = #proc, \ + } +const struct rpc_procinfo nfs_procedures[] = { + PROC(GETATTR, fhandle, attrstat, 1), + PROC(SETATTR, sattrargs, attrstat, 0), + PROC(LOOKUP, diropargs, diropres, 2), + PROC(READLINK, readlinkargs, readlinkres, 3), + PROC(READ, readargs, readres, 3), + PROC(WRITE, writeargs, writeres, 4), + PROC(CREATE, createargs, diropres, 0), + PROC(REMOVE, removeargs, stat, 0), + PROC(RENAME, renameargs, stat, 0), + PROC(LINK, linkargs, stat, 0), + PROC(SYMLINK, symlinkargs, stat, 0), + PROC(MKDIR, createargs, diropres, 0), + PROC(RMDIR, diropargs, stat, 0), + PROC(READDIR, readdirargs, readdirres, 3), + PROC(STATFS, fhandle, statfsres, 0), +}; + +static unsigned int nfs_version2_counts[ARRAY_SIZE(nfs_procedures)]; +const struct rpc_version nfs_version2 = { + .number = 2, + .nrprocs = ARRAY_SIZE(nfs_procedures), + .procs = nfs_procedures, + .counts = nfs_version2_counts, +}; diff --git a/fs/nfs/nfs3_fs.h b/fs/nfs/nfs3_fs.h new file mode 100644 index 000000000..b333ea119 --- /dev/null +++ b/fs/nfs/nfs3_fs.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2014 Anna Schumaker. + * + * NFSv3-specific filesystem definitions and declarations + */ +#ifndef __LINUX_FS_NFS_NFS3_FS_H +#define __LINUX_FS_NFS_NFS3_FS_H + +/* + * nfs3acl.c + */ +#ifdef CONFIG_NFS_V3_ACL +extern struct posix_acl *nfs3_get_acl(struct inode *inode, int type, bool rcu); +extern int nfs3_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type); +extern int nfs3_proc_setacls(struct inode *inode, struct posix_acl *acl, + struct posix_acl *dfacl); +extern ssize_t nfs3_listxattr(struct dentry *, char *, size_t); +#else +static inline int nfs3_proc_setacls(struct inode *inode, struct posix_acl *acl, + struct posix_acl *dfacl) +{ + return 0; +} +#define nfs3_listxattr NULL +#endif /* CONFIG_NFS_V3_ACL */ + +/* nfs3client.c */ +struct nfs_server *nfs3_create_server(struct fs_context *); +struct nfs_server *nfs3_clone_server(struct nfs_server *, struct nfs_fh *, + struct nfs_fattr *, rpc_authflavor_t); + +/* nfs3super.c */ +extern struct nfs_subversion nfs_v3; + +#endif /* __LINUX_FS_NFS_NFS3_FS_H */ diff --git a/fs/nfs/nfs3acl.c b/fs/nfs/nfs3acl.c new file mode 100644 index 000000000..a126eb31f --- /dev/null +++ b/fs/nfs/nfs3acl.c @@ -0,0 +1,343 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" +#include "nfs3_fs.h" + +#define NFSDBG_FACILITY NFSDBG_PROC + +/* + * nfs3_prepare_get_acl, nfs3_complete_get_acl, nfs3_abort_get_acl: Helpers for + * caching get_acl results in a race-free way. See fs/posix_acl.c:get_acl() + * for explanations. + */ +static void nfs3_prepare_get_acl(struct posix_acl **p) +{ + struct posix_acl *sentinel = uncached_acl_sentinel(current); + + /* If the ACL isn't being read yet, set our sentinel. */ + cmpxchg(p, ACL_NOT_CACHED, sentinel); +} + +static void nfs3_complete_get_acl(struct posix_acl **p, struct posix_acl *acl) +{ + struct posix_acl *sentinel = uncached_acl_sentinel(current); + + /* Only cache the ACL if our sentinel is still in place. */ + posix_acl_dup(acl); + if (cmpxchg(p, sentinel, acl) != sentinel) + posix_acl_release(acl); +} + +static void nfs3_abort_get_acl(struct posix_acl **p) +{ + struct posix_acl *sentinel = uncached_acl_sentinel(current); + + /* Remove our sentinel upon failure. */ + cmpxchg(p, sentinel, ACL_NOT_CACHED); +} + +struct posix_acl *nfs3_get_acl(struct inode *inode, int type, bool rcu) +{ + struct nfs_server *server = NFS_SERVER(inode); + struct page *pages[NFSACL_MAXPAGES] = { }; + struct nfs3_getaclargs args = { + .fh = NFS_FH(inode), + /* The xdr layer may allocate pages here. */ + .pages = pages, + }; + struct nfs3_getaclres res = { + NULL, + }; + struct rpc_message msg = { + .rpc_argp = &args, + .rpc_resp = &res, + }; + int status, count; + + if (rcu) + return ERR_PTR(-ECHILD); + + if (!nfs_server_capable(inode, NFS_CAP_ACLS)) + return ERR_PTR(-EOPNOTSUPP); + + status = nfs_revalidate_inode(inode, NFS_INO_INVALID_CHANGE); + if (status < 0) + return ERR_PTR(status); + + /* + * Only get the access acl when explicitly requested: We don't + * need it for access decisions, and only some applications use + * it. Applications which request the access acl first are not + * penalized from this optimization. + */ + if (type == ACL_TYPE_ACCESS) + args.mask |= NFS_ACLCNT|NFS_ACL; + if (S_ISDIR(inode->i_mode)) + args.mask |= NFS_DFACLCNT|NFS_DFACL; + if (args.mask == 0) + return NULL; + + dprintk("NFS call getacl\n"); + msg.rpc_proc = &server->client_acl->cl_procinfo[ACLPROC3_GETACL]; + res.fattr = nfs_alloc_fattr(); + if (res.fattr == NULL) + return ERR_PTR(-ENOMEM); + + if (args.mask & NFS_ACL) + nfs3_prepare_get_acl(&inode->i_acl); + if (args.mask & NFS_DFACL) + nfs3_prepare_get_acl(&inode->i_default_acl); + + status = rpc_call_sync(server->client_acl, &msg, 0); + dprintk("NFS reply getacl: %d\n", status); + + /* pages may have been allocated at the xdr layer. */ + for (count = 0; count < NFSACL_MAXPAGES && args.pages[count]; count++) + __free_page(args.pages[count]); + + switch (status) { + case 0: + nfs_refresh_inode(inode, res.fattr); + break; + case -EPFNOSUPPORT: + case -EPROTONOSUPPORT: + dprintk("NFS_V3_ACL extension not supported; disabling\n"); + server->caps &= ~NFS_CAP_ACLS; + fallthrough; + case -ENOTSUPP: + status = -EOPNOTSUPP; + goto getout; + default: + goto getout; + } + if ((args.mask & res.mask) != args.mask) { + status = -EIO; + goto getout; + } + + if (res.acl_access != NULL) { + if ((posix_acl_equiv_mode(res.acl_access, NULL) == 0) || + res.acl_access->a_count == 0) { + posix_acl_release(res.acl_access); + res.acl_access = NULL; + } + } + + if (res.mask & NFS_ACL) + nfs3_complete_get_acl(&inode->i_acl, res.acl_access); + else + forget_cached_acl(inode, ACL_TYPE_ACCESS); + + if (res.mask & NFS_DFACL) + nfs3_complete_get_acl(&inode->i_default_acl, res.acl_default); + else + forget_cached_acl(inode, ACL_TYPE_DEFAULT); + + nfs_free_fattr(res.fattr); + if (type == ACL_TYPE_ACCESS) { + posix_acl_release(res.acl_default); + return res.acl_access; + } else { + posix_acl_release(res.acl_access); + return res.acl_default; + } + +getout: + nfs3_abort_get_acl(&inode->i_acl); + nfs3_abort_get_acl(&inode->i_default_acl); + posix_acl_release(res.acl_access); + posix_acl_release(res.acl_default); + nfs_free_fattr(res.fattr); + return ERR_PTR(status); +} + +static int __nfs3_proc_setacls(struct inode *inode, struct posix_acl *acl, + struct posix_acl *dfacl) +{ + struct nfs_server *server = NFS_SERVER(inode); + struct nfs_fattr *fattr; + struct page *pages[NFSACL_MAXPAGES]; + struct nfs3_setaclargs args = { + .inode = inode, + .mask = NFS_ACL, + .acl_access = acl, + .pages = pages, + }; + struct rpc_message msg = { + .rpc_argp = &args, + .rpc_resp = &fattr, + }; + int status = 0; + + if (acl == NULL && (!S_ISDIR(inode->i_mode) || dfacl == NULL)) + goto out; + + status = -EOPNOTSUPP; + if (!nfs_server_capable(inode, NFS_CAP_ACLS)) + goto out; + + /* We are doing this here because XDR marshalling does not + * return any results, it BUGs. */ + status = -ENOSPC; + if (acl != NULL && acl->a_count > NFS_ACL_MAX_ENTRIES) + goto out; + if (dfacl != NULL && dfacl->a_count > NFS_ACL_MAX_ENTRIES) + goto out; + if (S_ISDIR(inode->i_mode)) { + args.mask |= NFS_DFACL; + args.acl_default = dfacl; + args.len = nfsacl_size(acl, dfacl); + } else + args.len = nfsacl_size(acl, NULL); + + if (args.len > NFS_ACL_INLINE_BUFSIZE) { + unsigned int npages = 1 + ((args.len - 1) >> PAGE_SHIFT); + + status = -ENOMEM; + do { + args.pages[args.npages] = alloc_page(GFP_KERNEL); + if (args.pages[args.npages] == NULL) + goto out_freepages; + args.npages++; + } while (args.npages < npages); + } + + dprintk("NFS call setacl\n"); + status = -ENOMEM; + fattr = nfs_alloc_fattr(); + if (fattr == NULL) + goto out_freepages; + + msg.rpc_proc = &server->client_acl->cl_procinfo[ACLPROC3_SETACL]; + msg.rpc_resp = fattr; + status = rpc_call_sync(server->client_acl, &msg, 0); + nfs_access_zap_cache(inode); + nfs_zap_acl_cache(inode); + dprintk("NFS reply setacl: %d\n", status); + + switch (status) { + case 0: + status = nfs_refresh_inode(inode, fattr); + break; + case -EPFNOSUPPORT: + case -EPROTONOSUPPORT: + dprintk("NFS_V3_ACL SETACL RPC not supported" + "(will not retry)\n"); + server->caps &= ~NFS_CAP_ACLS; + fallthrough; + case -ENOTSUPP: + status = -EOPNOTSUPP; + } + nfs_free_fattr(fattr); +out_freepages: + while (args.npages != 0) { + args.npages--; + __free_page(args.pages[args.npages]); + } +out: + return status; +} + +int nfs3_proc_setacls(struct inode *inode, struct posix_acl *acl, + struct posix_acl *dfacl) +{ + int ret; + ret = __nfs3_proc_setacls(inode, acl, dfacl); + return (ret == -EOPNOTSUPP) ? 0 : ret; + +} + +int nfs3_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type) +{ + struct posix_acl *orig = acl, *dfacl = NULL, *alloc; + struct inode *inode = d_inode(dentry); + int status; + + if (S_ISDIR(inode->i_mode)) { + switch(type) { + case ACL_TYPE_ACCESS: + alloc = get_inode_acl(inode, ACL_TYPE_DEFAULT); + if (IS_ERR(alloc)) + goto fail; + dfacl = alloc; + break; + + case ACL_TYPE_DEFAULT: + alloc = get_inode_acl(inode, ACL_TYPE_ACCESS); + if (IS_ERR(alloc)) + goto fail; + dfacl = acl; + acl = alloc; + break; + } + } + + if (acl == NULL) { + alloc = posix_acl_from_mode(inode->i_mode, GFP_KERNEL); + if (IS_ERR(alloc)) + goto fail; + acl = alloc; + } + status = __nfs3_proc_setacls(inode, acl, dfacl); +out: + if (acl != orig) + posix_acl_release(acl); + if (dfacl != orig) + posix_acl_release(dfacl); + return status; + +fail: + status = PTR_ERR(alloc); + goto out; +} + +static int +nfs3_list_one_acl(struct inode *inode, int type, const char *name, void *data, + size_t size, ssize_t *result) +{ + struct posix_acl *acl; + char *p = data + *result; + + acl = get_inode_acl(inode, type); + if (IS_ERR_OR_NULL(acl)) + return 0; + + posix_acl_release(acl); + + *result += strlen(name); + *result += 1; + if (!size) + return 0; + if (*result > size) + return -ERANGE; + + strcpy(p, name); + return 0; +} + +ssize_t +nfs3_listxattr(struct dentry *dentry, char *data, size_t size) +{ + struct inode *inode = d_inode(dentry); + ssize_t result = 0; + int error; + + error = nfs3_list_one_acl(inode, ACL_TYPE_ACCESS, + XATTR_NAME_POSIX_ACL_ACCESS, data, size, &result); + if (error) + return error; + + error = nfs3_list_one_acl(inode, ACL_TYPE_DEFAULT, + XATTR_NAME_POSIX_ACL_DEFAULT, data, size, &result); + if (error) + return error; + return result; +} diff --git a/fs/nfs/nfs3client.c b/fs/nfs/nfs3client.c new file mode 100644 index 000000000..5d97c1d38 --- /dev/null +++ b/fs/nfs/nfs3client.c @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include "internal.h" +#include "nfs3_fs.h" +#include "netns.h" +#include "sysfs.h" + +#ifdef CONFIG_NFS_V3_ACL +static struct rpc_stat nfsacl_rpcstat = { &nfsacl_program }; +static const struct rpc_version *nfsacl_version[] = { + [3] = &nfsacl_version3, +}; + +const struct rpc_program nfsacl_program = { + .name = "nfsacl", + .number = NFS_ACL_PROGRAM, + .nrvers = ARRAY_SIZE(nfsacl_version), + .version = nfsacl_version, + .stats = &nfsacl_rpcstat, +}; + +/* + * Initialise an NFSv3 ACL client connection + */ +static void nfs_init_server_aclclient(struct nfs_server *server) +{ + if (server->flags & NFS_MOUNT_NOACL) + goto out_noacl; + + server->client_acl = rpc_bind_new_program(server->client, &nfsacl_program, 3); + if (IS_ERR(server->client_acl)) + goto out_noacl; + + nfs_sysfs_link_rpc_client(server, server->client_acl, NULL); + + /* No errors! Assume that Sun nfsacls are supported */ + server->caps |= NFS_CAP_ACLS; + return; + +out_noacl: + server->caps &= ~NFS_CAP_ACLS; +} +#else +static inline void nfs_init_server_aclclient(struct nfs_server *server) +{ + server->flags &= ~NFS_MOUNT_NOACL; + server->caps &= ~NFS_CAP_ACLS; +} +#endif + +struct nfs_server *nfs3_create_server(struct fs_context *fc) +{ + struct nfs_server *server = nfs_create_server(fc); + + /* Create a client RPC handle for the NFS v3 ACL management interface */ + if (!IS_ERR(server)) + nfs_init_server_aclclient(server); + return server; +} + +struct nfs_server *nfs3_clone_server(struct nfs_server *source, + struct nfs_fh *fh, + struct nfs_fattr *fattr, + rpc_authflavor_t flavor) +{ + struct nfs_server *server = nfs_clone_server(source, fh, fattr, flavor); + if (!IS_ERR(server) && !IS_ERR(source->client_acl)) + nfs_init_server_aclclient(server); + return server; +} + +/* + * Set up a pNFS Data Server client over NFSv3. + * + * Return any existing nfs_client that matches server address,port,version + * and minorversion. + * + * For a new nfs_client, use a soft mount (default), a low retrans and a + * low timeout interval so that if a connection is lost, we retry through + * the MDS. + */ +struct nfs_client *nfs3_set_ds_client(struct nfs_server *mds_srv, + const struct sockaddr_storage *ds_addr, int ds_addrlen, + int ds_proto, unsigned int ds_timeo, unsigned int ds_retrans) +{ + struct rpc_timeout ds_timeout; + unsigned long connect_timeout = ds_timeo * (ds_retrans + 1) * HZ / 10; + struct nfs_client *mds_clp = mds_srv->nfs_client; + struct nfs_client_initdata cl_init = { + .addr = ds_addr, + .addrlen = ds_addrlen, + .nodename = mds_clp->cl_rpcclient->cl_nodename, + .ip_addr = mds_clp->cl_ipaddr, + .nfs_mod = &nfs_v3, + .proto = ds_proto, + .net = mds_clp->cl_net, + .timeparms = &ds_timeout, + .cred = mds_srv->cred, + .xprtsec = { + .policy = RPC_XPRTSEC_NONE, + .cert_serial = TLS_NO_CERT, + .privkey_serial = TLS_NO_PRIVKEY, + }, + .connect_timeout = connect_timeout, + .reconnect_timeout = connect_timeout, + }; + struct nfs_client *clp; + char buf[INET6_ADDRSTRLEN + 1]; + + /* fake a hostname because lockd wants it */ + if (rpc_ntop((struct sockaddr *)ds_addr, buf, sizeof(buf)) <= 0) + return ERR_PTR(-EINVAL); + cl_init.hostname = buf; + + switch (ds_proto) { + case XPRT_TRANSPORT_TCP_TLS: + if (mds_clp->cl_xprtsec.policy != RPC_XPRTSEC_NONE) + cl_init.xprtsec = mds_clp->cl_xprtsec; + else + ds_proto = XPRT_TRANSPORT_TCP; + fallthrough; + case XPRT_TRANSPORT_RDMA: + case XPRT_TRANSPORT_TCP: + if (mds_clp->cl_nconnect > 1) + cl_init.nconnect = mds_clp->cl_nconnect; + } + + if (mds_srv->flags & NFS_MOUNT_NORESVPORT) + __set_bit(NFS_CS_NORESVPORT, &cl_init.init_flags); + if (test_bit(NFS_CS_NETUNREACH_FATAL, &mds_clp->cl_flags)) + __set_bit(NFS_CS_NETUNREACH_FATAL, &cl_init.init_flags); + + __set_bit(NFS_CS_DS, &cl_init.init_flags); + + /* Use the MDS nfs_client cl_ipaddr. */ + nfs_init_timeout_values(&ds_timeout, ds_proto, ds_timeo, ds_retrans); + clp = nfs_get_client(&cl_init); + + return clp; +} +EXPORT_SYMBOL_GPL(nfs3_set_ds_client); diff --git a/fs/nfs/nfs3proc.c b/fs/nfs/nfs3proc.c new file mode 100644 index 000000000..b80d0c5ef --- /dev/null +++ b/fs/nfs/nfs3proc.c @@ -0,0 +1,1128 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/nfs/nfs3proc.c + * + * Client-side NFSv3 procedures stubs. + * + * Copyright (C) 1997, Olaf Kirch + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iostat.h" +#include "internal.h" +#include "nfs3_fs.h" + +#define NFSDBG_FACILITY NFSDBG_PROC + +/* A wrapper to handle the EJUKEBOX error messages */ +static int +nfs3_rpc_wrapper(struct rpc_clnt *clnt, struct rpc_message *msg, int flags) +{ + int res; + do { + res = rpc_call_sync(clnt, msg, flags); + if (res != -EJUKEBOX) + break; + __set_current_state(TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); + schedule_timeout(NFS_JUKEBOX_RETRY_TIME); + res = -ERESTARTSYS; + } while (!fatal_signal_pending(current) && !nfs_current_task_exiting()); + return res; +} + +#define rpc_call_sync(clnt, msg, flags) nfs3_rpc_wrapper(clnt, msg, flags) + +static int +nfs3_async_handle_jukebox(struct rpc_task *task, struct inode *inode) +{ + if (task->tk_status != -EJUKEBOX) + return 0; + nfs_inc_stats(inode, NFSIOS_DELAY); + task->tk_status = 0; + rpc_restart_call(task); + rpc_delay(task, NFS_JUKEBOX_RETRY_TIME); + return 1; +} + +static int +do_proc_get_root(struct rpc_clnt *client, struct nfs_fh *fhandle, + struct nfs_fsinfo *info) +{ + struct rpc_message msg = { + .rpc_proc = &nfs3_procedures[NFS3PROC_FSINFO], + .rpc_argp = fhandle, + .rpc_resp = info, + }; + int status; + + dprintk("%s: call fsinfo\n", __func__); + nfs_fattr_init(info->fattr); + status = rpc_call_sync(client, &msg, 0); + dprintk("%s: reply fsinfo: %d\n", __func__, status); + if (status == 0 && !(info->fattr->valid & NFS_ATTR_FATTR)) { + msg.rpc_proc = &nfs3_procedures[NFS3PROC_GETATTR]; + msg.rpc_resp = info->fattr; + status = rpc_call_sync(client, &msg, 0); + dprintk("%s: reply getattr: %d\n", __func__, status); + } + return status; +} + +/* + * Bare-bones access to getattr: this is for nfs_get_root/nfs_get_sb + */ +static int +nfs3_proc_get_root(struct nfs_server *server, struct nfs_fh *fhandle, + struct nfs_fsinfo *info) +{ + int status; + + status = do_proc_get_root(server->client, fhandle, info); + if (status && server->nfs_client->cl_rpcclient != server->client) + status = do_proc_get_root(server->nfs_client->cl_rpcclient, fhandle, info); + return status; +} + +/* + * One function for each procedure in the NFS protocol. + */ +static int +nfs3_proc_getattr(struct nfs_server *server, struct nfs_fh *fhandle, + struct nfs_fattr *fattr, struct inode *inode) +{ + struct rpc_message msg = { + .rpc_proc = &nfs3_procedures[NFS3PROC_GETATTR], + .rpc_argp = fhandle, + .rpc_resp = fattr, + }; + int status; + unsigned short task_flags = 0; + + /* Is this is an attribute revalidation, subject to softreval? */ + if (inode && (server->flags & NFS_MOUNT_SOFTREVAL)) + task_flags |= RPC_TASK_TIMEOUT; + + dprintk("NFS call getattr\n"); + nfs_fattr_init(fattr); + status = rpc_call_sync(server->client, &msg, task_flags); + dprintk("NFS reply getattr: %d\n", status); + return status; +} + +static int +nfs3_proc_setattr(struct dentry *dentry, struct nfs_fattr *fattr, + struct iattr *sattr) +{ + struct inode *inode = d_inode(dentry); + struct nfs3_sattrargs arg = { + .fh = NFS_FH(inode), + .sattr = sattr, + }; + struct rpc_message msg = { + .rpc_proc = &nfs3_procedures[NFS3PROC_SETATTR], + .rpc_argp = &arg, + .rpc_resp = fattr, + }; + int status; + + dprintk("NFS call setattr\n"); + if (sattr->ia_valid & ATTR_FILE) + msg.rpc_cred = nfs_file_cred(sattr->ia_file); + nfs_fattr_init(fattr); + status = rpc_call_sync(NFS_CLIENT(inode), &msg, 0); + if (status == 0) { + nfs_setattr_update_inode(inode, sattr, fattr); + if (NFS_I(inode)->cache_validity & NFS_INO_INVALID_ACL) + nfs_zap_acl_cache(inode); + } + dprintk("NFS reply setattr: %d\n", status); + return status; +} + +static int +__nfs3_proc_lookup(struct inode *dir, const char *name, size_t len, + struct nfs_fh *fhandle, struct nfs_fattr *fattr, + unsigned short task_flags) +{ + struct nfs3_diropargs arg = { + .fh = NFS_FH(dir), + .name = name, + .len = len + }; + struct nfs3_diropres res = { + .fh = fhandle, + .fattr = fattr + }; + struct rpc_message msg = { + .rpc_proc = &nfs3_procedures[NFS3PROC_LOOKUP], + .rpc_argp = &arg, + .rpc_resp = &res, + }; + int status; + + res.dir_attr = nfs_alloc_fattr(); + if (res.dir_attr == NULL) + return -ENOMEM; + + nfs_fattr_init(fattr); + status = rpc_call_sync(NFS_CLIENT(dir), &msg, task_flags); + nfs_refresh_inode(dir, res.dir_attr); + if (status >= 0 && !(fattr->valid & NFS_ATTR_FATTR)) { + msg.rpc_proc = &nfs3_procedures[NFS3PROC_GETATTR]; + msg.rpc_argp = fhandle; + msg.rpc_resp = fattr; + status = rpc_call_sync(NFS_CLIENT(dir), &msg, task_flags); + } + nfs_free_fattr(res.dir_attr); + dprintk("NFS reply lookup: %d\n", status); + return status; +} + +static int +nfs3_proc_lookup(struct inode *dir, struct dentry *dentry, const struct qstr *name, + struct nfs_fh *fhandle, struct nfs_fattr *fattr) +{ + unsigned short task_flags = 0; + + /* Is this is an attribute revalidation, subject to softreval? */ + if (nfs_lookup_is_soft_revalidate(dentry)) + task_flags |= RPC_TASK_TIMEOUT; + + dprintk("NFS call lookup %pd2\n", dentry); + return __nfs3_proc_lookup(dir, name->name, name->len, fhandle, fattr, + task_flags); +} + +static int nfs3_proc_lookupp(struct inode *inode, struct nfs_fh *fhandle, + struct nfs_fattr *fattr) +{ + const char dotdot[] = ".."; + const size_t len = strlen(dotdot); + unsigned short task_flags = 0; + + if (NFS_SERVER(inode)->flags & NFS_MOUNT_SOFTREVAL) + task_flags |= RPC_TASK_TIMEOUT; + + return __nfs3_proc_lookup(inode, dotdot, len, fhandle, fattr, + task_flags); +} + +static int nfs3_proc_access(struct inode *inode, struct nfs_access_entry *entry, + const struct cred *cred) +{ + struct nfs3_accessargs arg = { + .fh = NFS_FH(inode), + .access = entry->mask, + }; + struct nfs3_accessres res; + struct rpc_message msg = { + .rpc_proc = &nfs3_procedures[NFS3PROC_ACCESS], + .rpc_argp = &arg, + .rpc_resp = &res, + .rpc_cred = cred, + }; + int status = -ENOMEM; + + dprintk("NFS call access\n"); + res.fattr = nfs_alloc_fattr(); + if (res.fattr == NULL) + goto out; + + status = rpc_call_sync(NFS_CLIENT(inode), &msg, 0); + nfs_refresh_inode(inode, res.fattr); + if (status == 0) + nfs_access_set_mask(entry, res.access); + nfs_free_fattr(res.fattr); +out: + dprintk("NFS reply access: %d\n", status); + return status; +} + +static int nfs3_proc_readlink(struct inode *inode, struct page *page, + unsigned int pgbase, unsigned int pglen) +{ + struct nfs_fattr *fattr; + struct nfs3_readlinkargs args = { + .fh = NFS_FH(inode), + .pgbase = pgbase, + .pglen = pglen, + .pages = &page + }; + struct rpc_message msg = { + .rpc_proc = &nfs3_procedures[NFS3PROC_READLINK], + .rpc_argp = &args, + }; + int status = -ENOMEM; + + dprintk("NFS call readlink\n"); + fattr = nfs_alloc_fattr(); + if (fattr == NULL) + goto out; + msg.rpc_resp = fattr; + + status = rpc_call_sync(NFS_CLIENT(inode), &msg, 0); + nfs_refresh_inode(inode, fattr); + nfs_free_fattr(fattr); +out: + dprintk("NFS reply readlink: %d\n", status); + return status; +} + +struct nfs3_createdata { + struct rpc_message msg; + union { + struct nfs3_createargs create; + struct nfs3_mkdirargs mkdir; + struct nfs3_symlinkargs symlink; + struct nfs3_mknodargs mknod; + } arg; + struct nfs3_diropres res; + struct nfs_fh fh; + struct nfs_fattr fattr; + struct nfs_fattr dir_attr; +}; + +static struct nfs3_createdata *nfs3_alloc_createdata(void) +{ + struct nfs3_createdata *data; + + data = kzalloc_obj(*data); + if (data != NULL) { + data->msg.rpc_argp = &data->arg; + data->msg.rpc_resp = &data->res; + data->res.fh = &data->fh; + data->res.fattr = &data->fattr; + data->res.dir_attr = &data->dir_attr; + nfs_fattr_init(data->res.fattr); + nfs_fattr_init(data->res.dir_attr); + } + return data; +} + +static struct dentry * +nfs3_do_create(struct inode *dir, struct dentry *dentry, struct nfs3_createdata *data) +{ + int status; + + status = rpc_call_sync(NFS_CLIENT(dir), &data->msg, 0); + nfs_post_op_update_inode(dir, data->res.dir_attr); + if (status != 0) + return ERR_PTR(status); + + return nfs_add_or_obtain(dentry, data->res.fh, data->res.fattr); +} + +static void nfs3_free_createdata(struct nfs3_createdata *data) +{ + kfree(data); +} + +/* + * Create a regular file. + */ +static int +nfs3_proc_create(struct inode *dir, struct dentry *dentry, struct iattr *sattr, + int flags) +{ + struct posix_acl *default_acl, *acl; + struct nfs3_createdata *data; + struct dentry *d_alias; + int status = -ENOMEM; + + dprintk("NFS call create %pd\n", dentry); + + data = nfs3_alloc_createdata(); + if (data == NULL) + goto out; + + data->msg.rpc_proc = &nfs3_procedures[NFS3PROC_CREATE]; + data->arg.create.fh = NFS_FH(dir); + data->arg.create.name = dentry->d_name.name; + data->arg.create.len = dentry->d_name.len; + data->arg.create.sattr = sattr; + + data->arg.create.createmode = NFS3_CREATE_UNCHECKED; + if (flags & O_EXCL) { + data->arg.create.createmode = NFS3_CREATE_EXCLUSIVE; + data->arg.create.verifier[0] = cpu_to_be32(jiffies); + data->arg.create.verifier[1] = cpu_to_be32(current->pid); + } + + status = posix_acl_create(dir, &sattr->ia_mode, &default_acl, &acl); + if (status) + goto out; + + for (;;) { + d_alias = nfs3_do_create(dir, dentry, data); + status = PTR_ERR_OR_ZERO(d_alias); + + if (status != -ENOTSUPP) + break; + /* If the server doesn't support the exclusive creation + * semantics, try again with simple 'guarded' mode. */ + switch (data->arg.create.createmode) { + case NFS3_CREATE_EXCLUSIVE: + data->arg.create.createmode = NFS3_CREATE_GUARDED; + break; + + case NFS3_CREATE_GUARDED: + data->arg.create.createmode = NFS3_CREATE_UNCHECKED; + break; + + case NFS3_CREATE_UNCHECKED: + goto out_release_acls; + } + nfs_fattr_init(data->res.dir_attr); + nfs_fattr_init(data->res.fattr); + } + + if (status != 0) + goto out_release_acls; + + if (d_alias) { + if (d_is_dir(d_alias)) { + status = -EISDIR; + goto out_dput; + } + dentry = d_alias; + } + + /* When we created the file with exclusive semantics, make + * sure we set the attributes afterwards. */ + if (data->arg.create.createmode == NFS3_CREATE_EXCLUSIVE) { + dprintk("NFS call setattr (post-create)\n"); + + if (!(sattr->ia_valid & ATTR_ATIME_SET)) + sattr->ia_valid |= ATTR_ATIME; + if (!(sattr->ia_valid & ATTR_MTIME_SET)) + sattr->ia_valid |= ATTR_MTIME; + + /* Note: we could use a guarded setattr here, but I'm + * not sure this buys us anything (and I'd have + * to revamp the NFSv3 XDR code) */ + status = nfs3_proc_setattr(dentry, data->res.fattr, sattr); + nfs_post_op_update_inode(d_inode(dentry), data->res.fattr); + dprintk("NFS reply setattr (post-create): %d\n", status); + if (status != 0) + goto out_dput; + } + + status = nfs3_proc_setacls(d_inode(dentry), acl, default_acl); + +out_dput: + dput(d_alias); +out_release_acls: + posix_acl_release(acl); + posix_acl_release(default_acl); +out: + nfs3_free_createdata(data); + dprintk("NFS reply create: %d\n", status); + return status; +} + +static int +nfs3_proc_remove(struct inode *dir, struct dentry *dentry) +{ + struct nfs_removeargs arg = { + .fh = NFS_FH(dir), + .name = dentry->d_name, + }; + struct nfs_removeres res; + struct rpc_message msg = { + .rpc_proc = &nfs3_procedures[NFS3PROC_REMOVE], + .rpc_argp = &arg, + .rpc_resp = &res, + }; + int status = -ENOMEM; + + dprintk("NFS call remove %pd2\n", dentry); + res.dir_attr = nfs_alloc_fattr(); + if (res.dir_attr == NULL) + goto out; + + status = rpc_call_sync(NFS_CLIENT(dir), &msg, 0); + nfs_post_op_update_inode(dir, res.dir_attr); + nfs_free_fattr(res.dir_attr); +out: + dprintk("NFS reply remove: %d\n", status); + return status; +} + +static void +nfs3_proc_unlink_setup(struct rpc_message *msg, + struct dentry *dentry, + struct inode *inode) +{ + msg->rpc_proc = &nfs3_procedures[NFS3PROC_REMOVE]; +} + +static void nfs3_proc_unlink_rpc_prepare(struct rpc_task *task, struct nfs_unlinkdata *data) +{ + rpc_call_start(task); +} + +static int +nfs3_proc_unlink_done(struct rpc_task *task, struct inode *dir) +{ + struct nfs_removeres *res; + if (nfs3_async_handle_jukebox(task, dir)) + return 0; + res = task->tk_msg.rpc_resp; + nfs_post_op_update_inode(dir, res->dir_attr); + return 1; +} + +static void +nfs3_proc_rename_setup(struct rpc_message *msg, + struct dentry *old_dentry, + struct dentry *new_dentry, + struct inode *same_parent) +{ + msg->rpc_proc = &nfs3_procedures[NFS3PROC_RENAME]; +} + +static void nfs3_proc_rename_rpc_prepare(struct rpc_task *task, struct nfs_renamedata *data) +{ + rpc_call_start(task); +} + +static int +nfs3_proc_rename_done(struct rpc_task *task, struct inode *old_dir, + struct inode *new_dir) +{ + struct nfs_renameres *res; + + if (nfs3_async_handle_jukebox(task, old_dir)) + return 0; + res = task->tk_msg.rpc_resp; + + nfs_post_op_update_inode(old_dir, res->old_fattr); + nfs_post_op_update_inode(new_dir, res->new_fattr); + return 1; +} + +static int +nfs3_proc_link(struct inode *inode, struct inode *dir, const struct qstr *name) +{ + struct nfs3_linkargs arg = { + .fromfh = NFS_FH(inode), + .tofh = NFS_FH(dir), + .toname = name->name, + .tolen = name->len + }; + struct nfs3_linkres res; + struct rpc_message msg = { + .rpc_proc = &nfs3_procedures[NFS3PROC_LINK], + .rpc_argp = &arg, + .rpc_resp = &res, + }; + int status = -ENOMEM; + + dprintk("NFS call link %s\n", name->name); + res.fattr = nfs_alloc_fattr(); + res.dir_attr = nfs_alloc_fattr(); + if (res.fattr == NULL || res.dir_attr == NULL) + goto out; + + status = rpc_call_sync(NFS_CLIENT(inode), &msg, 0); + nfs_post_op_update_inode(dir, res.dir_attr); + nfs_post_op_update_inode(inode, res.fattr); +out: + nfs_free_fattr(res.dir_attr); + nfs_free_fattr(res.fattr); + dprintk("NFS reply link: %d\n", status); + return status; +} + +static int +nfs3_proc_symlink(struct inode *dir, struct dentry *dentry, struct folio *folio, + unsigned int len, struct iattr *sattr) +{ + struct page *page = &folio->page; + struct nfs3_createdata *data; + struct dentry *d_alias; + int status = -ENOMEM; + + if (len > NFS3_MAXPATHLEN) + return -ENAMETOOLONG; + + dprintk("NFS call symlink %pd\n", dentry); + + data = nfs3_alloc_createdata(); + if (data == NULL) + goto out; + data->msg.rpc_proc = &nfs3_procedures[NFS3PROC_SYMLINK]; + data->arg.symlink.fromfh = NFS_FH(dir); + data->arg.symlink.fromname = dentry->d_name.name; + data->arg.symlink.fromlen = dentry->d_name.len; + data->arg.symlink.pages = &page; + data->arg.symlink.pathlen = len; + data->arg.symlink.sattr = sattr; + + d_alias = nfs3_do_create(dir, dentry, data); + status = PTR_ERR_OR_ZERO(d_alias); + + if (status == 0) + dput(d_alias); + + nfs3_free_createdata(data); +out: + dprintk("NFS reply symlink: %d\n", status); + return status; +} + +static struct dentry * +nfs3_proc_mkdir(struct inode *dir, struct dentry *dentry, struct iattr *sattr) +{ + struct posix_acl *default_acl, *acl; + struct nfs3_createdata *data; + struct dentry *ret = ERR_PTR(-ENOMEM); + int status; + + dprintk("NFS call mkdir %pd\n", dentry); + + data = nfs3_alloc_createdata(); + if (data == NULL) + goto out; + + ret = ERR_PTR(posix_acl_create(dir, &sattr->ia_mode, + &default_acl, &acl)); + if (IS_ERR(ret)) + goto out; + + data->msg.rpc_proc = &nfs3_procedures[NFS3PROC_MKDIR]; + data->arg.mkdir.fh = NFS_FH(dir); + data->arg.mkdir.name = dentry->d_name.name; + data->arg.mkdir.len = dentry->d_name.len; + data->arg.mkdir.sattr = sattr; + + ret = nfs3_do_create(dir, dentry, data); + + if (IS_ERR(ret)) + goto out_release_acls; + + if (ret) + dentry = ret; + + status = nfs3_proc_setacls(d_inode(dentry), acl, default_acl); + if (status) { + dput(ret); + ret = ERR_PTR(status); + } + +out_release_acls: + posix_acl_release(acl); + posix_acl_release(default_acl); +out: + nfs3_free_createdata(data); + dprintk("NFS reply mkdir: %d\n", PTR_ERR_OR_ZERO(ret)); + return ret; +} + +static int +nfs3_proc_rmdir(struct inode *dir, const struct qstr *name) +{ + struct nfs_fattr *dir_attr; + struct nfs3_diropargs arg = { + .fh = NFS_FH(dir), + .name = name->name, + .len = name->len + }; + struct rpc_message msg = { + .rpc_proc = &nfs3_procedures[NFS3PROC_RMDIR], + .rpc_argp = &arg, + }; + int status = -ENOMEM; + + dprintk("NFS call rmdir %s\n", name->name); + dir_attr = nfs_alloc_fattr(); + if (dir_attr == NULL) + goto out; + + msg.rpc_resp = dir_attr; + status = rpc_call_sync(NFS_CLIENT(dir), &msg, 0); + nfs_post_op_update_inode(dir, dir_attr); + nfs_free_fattr(dir_attr); +out: + dprintk("NFS reply rmdir: %d\n", status); + return status; +} + +/* + * The READDIR implementation is somewhat hackish - we pass the user buffer + * to the encode function, which installs it in the receive iovec. + * The decode function itself doesn't perform any decoding, it just makes + * sure the reply is syntactically correct. + * + * Also note that this implementation handles both plain readdir and + * readdirplus. + */ +static int nfs3_proc_readdir(struct nfs_readdir_arg *nr_arg, + struct nfs_readdir_res *nr_res) +{ + struct inode *dir = d_inode(nr_arg->dentry); + struct nfs3_readdirargs arg = { + .fh = NFS_FH(dir), + .cookie = nr_arg->cookie, + .plus = nr_arg->plus, + .count = nr_arg->page_len, + .pages = nr_arg->pages + }; + struct nfs3_readdirres res = { + .verf = nr_res->verf, + .plus = nr_arg->plus, + }; + struct rpc_message msg = { + .rpc_proc = &nfs3_procedures[NFS3PROC_READDIR], + .rpc_argp = &arg, + .rpc_resp = &res, + .rpc_cred = nr_arg->cred, + }; + int status = -ENOMEM; + + if (nr_arg->plus) + msg.rpc_proc = &nfs3_procedures[NFS3PROC_READDIRPLUS]; + if (arg.cookie) + memcpy(arg.verf, nr_arg->verf, sizeof(arg.verf)); + + dprintk("NFS call readdir%s %llu\n", nr_arg->plus ? "plus" : "", + (unsigned long long)nr_arg->cookie); + + res.dir_attr = nfs_alloc_fattr(); + if (res.dir_attr == NULL) + goto out; + + status = rpc_call_sync(NFS_CLIENT(dir), &msg, 0); + + nfs_invalidate_atime(dir); + nfs_refresh_inode(dir, res.dir_attr); + + nfs_free_fattr(res.dir_attr); +out: + dprintk("NFS reply readdir%s: %d\n", nr_arg->plus ? "plus" : "", + status); + return status; +} + +static int +nfs3_proc_mknod(struct inode *dir, struct dentry *dentry, struct iattr *sattr, + dev_t rdev) +{ + struct posix_acl *default_acl, *acl; + struct nfs3_createdata *data; + struct dentry *d_alias; + int status = -ENOMEM; + + dprintk("NFS call mknod %pd %u:%u\n", dentry, + MAJOR(rdev), MINOR(rdev)); + + data = nfs3_alloc_createdata(); + if (data == NULL) + goto out; + + status = posix_acl_create(dir, &sattr->ia_mode, &default_acl, &acl); + if (status) + goto out; + + data->msg.rpc_proc = &nfs3_procedures[NFS3PROC_MKNOD]; + data->arg.mknod.fh = NFS_FH(dir); + data->arg.mknod.name = dentry->d_name.name; + data->arg.mknod.len = dentry->d_name.len; + data->arg.mknod.sattr = sattr; + data->arg.mknod.rdev = rdev; + + switch (sattr->ia_mode & S_IFMT) { + case S_IFBLK: + data->arg.mknod.type = NF3BLK; + break; + case S_IFCHR: + data->arg.mknod.type = NF3CHR; + break; + case S_IFIFO: + data->arg.mknod.type = NF3FIFO; + break; + case S_IFSOCK: + data->arg.mknod.type = NF3SOCK; + break; + default: + status = -EINVAL; + goto out_release_acls; + } + + d_alias = nfs3_do_create(dir, dentry, data); + status = PTR_ERR_OR_ZERO(d_alias); + if (status != 0) + goto out_release_acls; + + if (d_alias) + dentry = d_alias; + + status = nfs3_proc_setacls(d_inode(dentry), acl, default_acl); + + dput(d_alias); +out_release_acls: + posix_acl_release(acl); + posix_acl_release(default_acl); +out: + nfs3_free_createdata(data); + dprintk("NFS reply mknod: %d\n", status); + return status; +} + +static int +nfs3_proc_statfs(struct nfs_server *server, struct nfs_fh *fhandle, + struct nfs_fsstat *stat) +{ + struct rpc_message msg = { + .rpc_proc = &nfs3_procedures[NFS3PROC_FSSTAT], + .rpc_argp = fhandle, + .rpc_resp = stat, + }; + int status; + + dprintk("NFS call fsstat\n"); + nfs_fattr_init(stat->fattr); + status = rpc_call_sync(server->client, &msg, 0); + dprintk("NFS reply fsstat: %d\n", status); + return status; +} + +static int +do_proc_fsinfo(struct rpc_clnt *client, struct nfs_fh *fhandle, + struct nfs_fsinfo *info) +{ + struct rpc_message msg = { + .rpc_proc = &nfs3_procedures[NFS3PROC_FSINFO], + .rpc_argp = fhandle, + .rpc_resp = info, + }; + int status; + + dprintk("NFS call fsinfo\n"); + nfs_fattr_init(info->fattr); + status = rpc_call_sync(client, &msg, 0); + dprintk("NFS reply fsinfo: %d\n", status); + return status; +} + +/* + * Bare-bones access to fsinfo: this is for nfs_get_root/nfs_get_sb via + * nfs_create_server + */ +static int +nfs3_proc_fsinfo(struct nfs_server *server, struct nfs_fh *fhandle, + struct nfs_fsinfo *info) +{ + int status; + + status = do_proc_fsinfo(server->client, fhandle, info); + if (status && server->nfs_client->cl_rpcclient != server->client) + status = do_proc_fsinfo(server->nfs_client->cl_rpcclient, fhandle, info); + return status; +} + +static int +nfs3_proc_pathconf(struct nfs_server *server, struct nfs_fh *fhandle, + struct nfs_pathconf *info) +{ + struct rpc_message msg = { + .rpc_proc = &nfs3_procedures[NFS3PROC_PATHCONF], + .rpc_argp = fhandle, + .rpc_resp = info, + }; + int status; + + dprintk("NFS call pathconf\n"); + nfs_fattr_init(info->fattr); + status = rpc_call_sync(server->client, &msg, 0); + dprintk("NFS reply pathconf: %d\n", status); + return status; +} + +#if IS_ENABLED(CONFIG_NFS_LOCALIO) + +static unsigned nfs3_localio_probe_throttle __read_mostly = 0; +module_param(nfs3_localio_probe_throttle, uint, 0644); +MODULE_PARM_DESC(nfs3_localio_probe_throttle, + "Probe for NFSv3 LOCALIO every N IO requests. Must be power-of-2, defaults to 0 (probing disabled)."); + +static void nfs3_localio_probe(struct nfs_server *server) +{ + struct nfs_client *clp = server->nfs_client; + + /* Throttled to reduce nfs_local_probe_async() frequency */ + if (!nfs3_localio_probe_throttle || nfs_server_is_local(clp)) + return; + + /* + * Try (re)enabling LOCALIO if isn't enabled -- admin deems + * it worthwhile to periodically check if LOCALIO possible by + * setting the 'nfs3_localio_probe_throttle' module parameter. + * + * This is useful if LOCALIO was previously enabled, but was + * disabled due to server restart, and IO has successfully + * completed in terms of normal RPC. + */ + if ((clp->cl_uuid.nfs3_localio_probe_count++ & + (nfs3_localio_probe_throttle - 1)) == 0) { + if (!nfs_server_is_local(clp)) + nfs_local_probe_async(clp); + } +} + +#else +static void nfs3_localio_probe(struct nfs_server *server) {} +#endif + +static int nfs3_read_done(struct rpc_task *task, struct nfs_pgio_header *hdr) +{ + struct inode *inode = hdr->inode; + struct nfs_server *server = NFS_SERVER(inode); + + if (hdr->pgio_done_cb != NULL) + return hdr->pgio_done_cb(task, hdr); + + if (nfs3_async_handle_jukebox(task, inode)) + return -EAGAIN; + + if (task->tk_status >= 0) { + if (!server->read_hdrsize) + cmpxchg(&server->read_hdrsize, 0, hdr->res.replen); + nfs3_localio_probe(server); + } + + nfs_invalidate_atime(inode); + nfs_refresh_inode(inode, &hdr->fattr); + return 0; +} + +static void nfs3_proc_read_setup(struct nfs_pgio_header *hdr, + struct rpc_message *msg) +{ + msg->rpc_proc = &nfs3_procedures[NFS3PROC_READ]; + hdr->args.replen = NFS_SERVER(hdr->inode)->read_hdrsize; +} + +static int nfs3_proc_pgio_rpc_prepare(struct rpc_task *task, + struct nfs_pgio_header *hdr) +{ + rpc_call_start(task); + return 0; +} + +static int nfs3_write_done(struct rpc_task *task, struct nfs_pgio_header *hdr) +{ + struct inode *inode = hdr->inode; + + if (hdr->pgio_done_cb != NULL) + return hdr->pgio_done_cb(task, hdr); + + if (nfs3_async_handle_jukebox(task, inode)) + return -EAGAIN; + if (task->tk_status >= 0) { + nfs_writeback_update_inode(hdr); + nfs3_localio_probe(NFS_SERVER(inode)); + } + return 0; +} + +static void nfs3_proc_write_setup(struct nfs_pgio_header *hdr, + struct rpc_message *msg, + struct rpc_clnt **clnt) +{ + msg->rpc_proc = &nfs3_procedures[NFS3PROC_WRITE]; +} + +static void nfs3_proc_commit_rpc_prepare(struct rpc_task *task, struct nfs_commit_data *data) +{ + rpc_call_start(task); +} + +static int nfs3_commit_done(struct rpc_task *task, struct nfs_commit_data *data) +{ + if (data->commit_done_cb != NULL) + return data->commit_done_cb(task, data); + + if (nfs3_async_handle_jukebox(task, data->inode)) + return -EAGAIN; + nfs_refresh_inode(data->inode, data->res.fattr); + return 0; +} + +static void nfs3_proc_commit_setup(struct nfs_commit_data *data, struct rpc_message *msg, + struct rpc_clnt **clnt) +{ + msg->rpc_proc = &nfs3_procedures[NFS3PROC_COMMIT]; +} + +static void nfs3_nlm_alloc_call(void *data) +{ + struct nfs_lock_context *l_ctx = data; + if (l_ctx && test_bit(NFS_CONTEXT_UNLOCK, &l_ctx->open_context->flags)) { + get_nfs_open_context(l_ctx->open_context); + nfs_get_lock_context(l_ctx->open_context); + } +} + +static bool nfs3_nlm_unlock_prepare(struct rpc_task *task, void *data) +{ + struct nfs_lock_context *l_ctx = data; + if (l_ctx && test_bit(NFS_CONTEXT_UNLOCK, &l_ctx->open_context->flags)) + return nfs_async_iocounter_wait(task, l_ctx); + return false; + +} + +static void nfs3_nlm_release_call(void *data) +{ + struct nfs_lock_context *l_ctx = data; + struct nfs_open_context *ctx; + if (l_ctx && test_bit(NFS_CONTEXT_UNLOCK, &l_ctx->open_context->flags)) { + ctx = l_ctx->open_context; + nfs_put_lock_context(l_ctx); + put_nfs_open_context(ctx); + } +} + +static const struct nlmclnt_operations nlmclnt_fl_close_lock_ops = { + .nlmclnt_alloc_call = nfs3_nlm_alloc_call, + .nlmclnt_unlock_prepare = nfs3_nlm_unlock_prepare, + .nlmclnt_release_call = nfs3_nlm_release_call, +}; + +static int +nfs3_proc_lock(struct file *filp, int cmd, struct file_lock *fl) +{ + struct inode *inode = file_inode(filp); + struct nfs_lock_context *l_ctx = NULL; + struct nfs_open_context *ctx = nfs_file_open_context(filp); + int status; + + if (fl->c.flc_flags & FL_CLOSE) { + l_ctx = nfs_get_lock_context(ctx); + if (IS_ERR(l_ctx)) + l_ctx = NULL; + else + set_bit(NFS_CONTEXT_UNLOCK, &ctx->flags); + } + + status = nlmclnt_proc(NFS_SERVER(inode)->nlm_host, cmd, fl, l_ctx); + + if (l_ctx) + nfs_put_lock_context(l_ctx); + + return status; +} + +static int nfs3_have_delegation(struct inode *inode, fmode_t type, int flags) +{ + return 0; +} + +static void nfs3_return_delegation(struct inode *inode) +{ + if (S_ISREG(inode->i_mode)) + nfs_wb_all(inode); +} + +static const struct inode_operations nfs3_dir_inode_operations = { + .create = nfs_create, + .atomic_open = nfs_atomic_open_v23, + .lookup = nfs_lookup, + .link = nfs_link, + .unlink = nfs_unlink, + .symlink = nfs_symlink, + .mkdir = nfs_mkdir, + .rmdir = nfs_rmdir, + .mknod = nfs_mknod, + .rename = nfs_rename, + .permission = nfs_permission, + .getattr = nfs_getattr, + .setattr = nfs_setattr, + .fileattr_get = nfs_fileattr_get, +#ifdef CONFIG_NFS_V3_ACL + .listxattr = nfs3_listxattr, + .get_inode_acl = nfs3_get_acl, + .set_acl = nfs3_set_acl, +#endif +}; + +static const struct inode_operations nfs3_file_inode_operations = { + .permission = nfs_permission, + .getattr = nfs_getattr, + .setattr = nfs_setattr, + .fileattr_get = nfs_fileattr_get, +#ifdef CONFIG_NFS_V3_ACL + .listxattr = nfs3_listxattr, + .get_inode_acl = nfs3_get_acl, + .set_acl = nfs3_set_acl, +#endif +}; + +const struct nfs_rpc_ops nfs_v3_clientops = { + .version = 3, /* protocol version */ + .dentry_ops = &nfs_dentry_operations, + .dir_inode_ops = &nfs3_dir_inode_operations, + .file_inode_ops = &nfs3_file_inode_operations, + .file_ops = &nfs_file_operations, + .nlmclnt_ops = &nlmclnt_fl_close_lock_ops, + .getroot = nfs3_proc_get_root, + .submount = nfs_submount, + .try_get_tree = nfs_try_get_tree, + .getattr = nfs3_proc_getattr, + .setattr = nfs3_proc_setattr, + .lookup = nfs3_proc_lookup, + .lookupp = nfs3_proc_lookupp, + .access = nfs3_proc_access, + .readlink = nfs3_proc_readlink, + .create = nfs3_proc_create, + .remove = nfs3_proc_remove, + .unlink_setup = nfs3_proc_unlink_setup, + .unlink_rpc_prepare = nfs3_proc_unlink_rpc_prepare, + .unlink_done = nfs3_proc_unlink_done, + .rename_setup = nfs3_proc_rename_setup, + .rename_rpc_prepare = nfs3_proc_rename_rpc_prepare, + .rename_done = nfs3_proc_rename_done, + .link = nfs3_proc_link, + .symlink = nfs3_proc_symlink, + .mkdir = nfs3_proc_mkdir, + .rmdir = nfs3_proc_rmdir, + .readdir = nfs3_proc_readdir, + .mknod = nfs3_proc_mknod, + .statfs = nfs3_proc_statfs, + .fsinfo = nfs3_proc_fsinfo, + .pathconf = nfs3_proc_pathconf, + .decode_dirent = nfs3_decode_dirent, + .pgio_rpc_prepare = nfs3_proc_pgio_rpc_prepare, + .read_setup = nfs3_proc_read_setup, + .read_done = nfs3_read_done, + .write_setup = nfs3_proc_write_setup, + .write_done = nfs3_write_done, + .commit_setup = nfs3_proc_commit_setup, + .commit_rpc_prepare = nfs3_proc_commit_rpc_prepare, + .commit_done = nfs3_commit_done, + .lock = nfs3_proc_lock, + .clear_acl_cache = forget_all_cached_acls, + .close_context = nfs_close_context, + .have_delegation = nfs3_have_delegation, + .return_delegation = nfs3_return_delegation, + .alloc_client = nfs_alloc_client, + .init_client = nfs_init_client, + .free_client = nfs_free_client, + .create_server = nfs3_create_server, + .clone_server = nfs3_clone_server, +}; diff --git a/fs/nfs/nfs3super.c b/fs/nfs/nfs3super.c new file mode 100644 index 000000000..20a804784 --- /dev/null +++ b/fs/nfs/nfs3super.c @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2012 Netapp, Inc. All rights reserved. + */ +#include +#include +#include "internal.h" +#include "nfs3_fs.h" +#include "nfs.h" + +struct nfs_subversion nfs_v3 = { + .owner = THIS_MODULE, + .nfs_fs = &nfs_fs_type, + .rpc_vers = &nfs_version3, + .rpc_ops = &nfs_v3_clientops, + .sops = &nfs_sops, +}; + +static int __init init_nfs_v3(void) +{ + register_nfs_version(&nfs_v3); + return 0; +} + +static void __exit exit_nfs_v3(void) +{ + unregister_nfs_version(&nfs_v3); +} + +MODULE_DESCRIPTION("NFSv3 client support"); +MODULE_LICENSE("GPL"); + +module_init(init_nfs_v3); +module_exit(exit_nfs_v3); diff --git a/fs/nfs/nfs3xdr.c b/fs/nfs/nfs3xdr.c new file mode 100644 index 000000000..e745e78fa --- /dev/null +++ b/fs/nfs/nfs3xdr.c @@ -0,0 +1,2516 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/nfs/nfs3xdr.c + * + * XDR functions to encode/decode NFSv3 RPC arguments and results. + * + * Copyright (C) 1996, 1997 Olaf Kirch + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" +#include "nfstrace.h" + +#define NFSDBG_FACILITY NFSDBG_XDR + +/* + * Declare the space requirements for NFS arguments and replies as + * number of 32bit-words + */ +#define NFS3_pagepad_sz (1) /* Page padding */ +#define NFS3_fhandle_sz (1+16) +#define NFS3_fh_sz (NFS3_fhandle_sz) /* shorthand */ +#define NFS3_post_op_fh_sz (1+NFS3_fh_sz) +#define NFS3_sattr_sz (15) +#define NFS3_filename_sz (1+(NFS3_MAXNAMLEN>>2)) +#define NFS3_path_sz (1+(NFS3_MAXPATHLEN>>2)) +#define NFS3_fattr_sz (21) +#define NFS3_cookieverf_sz (NFS3_COOKIEVERFSIZE>>2) +#define NFS3_wcc_attr_sz (6) +#define NFS3_pre_op_attr_sz (1+NFS3_wcc_attr_sz) +#define NFS3_post_op_attr_sz (1+NFS3_fattr_sz) +#define NFS3_wcc_data_sz (NFS3_pre_op_attr_sz+NFS3_post_op_attr_sz) +#define NFS3_diropargs_sz (NFS3_fh_sz+NFS3_filename_sz) + +#define NFS3_getattrargs_sz (NFS3_fh_sz) +#define NFS3_setattrargs_sz (NFS3_fh_sz+NFS3_sattr_sz+3) +#define NFS3_lookupargs_sz (NFS3_fh_sz+NFS3_filename_sz) +#define NFS3_accessargs_sz (NFS3_fh_sz+1) +#define NFS3_readlinkargs_sz (NFS3_fh_sz) +#define NFS3_readargs_sz (NFS3_fh_sz+3) +#define NFS3_writeargs_sz (NFS3_fh_sz+5) +#define NFS3_createargs_sz (NFS3_diropargs_sz+NFS3_sattr_sz) +#define NFS3_mkdirargs_sz (NFS3_diropargs_sz+NFS3_sattr_sz) +#define NFS3_symlinkargs_sz (NFS3_diropargs_sz+1+NFS3_sattr_sz) +#define NFS3_mknodargs_sz (NFS3_diropargs_sz+2+NFS3_sattr_sz) +#define NFS3_removeargs_sz (NFS3_fh_sz+NFS3_filename_sz) +#define NFS3_renameargs_sz (NFS3_diropargs_sz+NFS3_diropargs_sz) +#define NFS3_linkargs_sz (NFS3_fh_sz+NFS3_diropargs_sz) +#define NFS3_readdirargs_sz (NFS3_fh_sz+NFS3_cookieverf_sz+3) +#define NFS3_readdirplusargs_sz (NFS3_fh_sz+NFS3_cookieverf_sz+4) +#define NFS3_commitargs_sz (NFS3_fh_sz+3) + +#define NFS3_getattrres_sz (1+NFS3_fattr_sz) +#define NFS3_setattrres_sz (1+NFS3_wcc_data_sz) +#define NFS3_removeres_sz (NFS3_setattrres_sz) +#define NFS3_lookupres_sz (1+NFS3_fh_sz+(2 * NFS3_post_op_attr_sz)) +#define NFS3_accessres_sz (1+NFS3_post_op_attr_sz+1) +#define NFS3_readlinkres_sz (1+NFS3_post_op_attr_sz+1+NFS3_pagepad_sz) +#define NFS3_readres_sz (1+NFS3_post_op_attr_sz+3+NFS3_pagepad_sz) +#define NFS3_writeres_sz (1+NFS3_wcc_data_sz+4) +#define NFS3_createres_sz (1+NFS3_post_op_fh_sz+NFS3_post_op_attr_sz+NFS3_wcc_data_sz) +#define NFS3_renameres_sz (1+(2 * NFS3_wcc_data_sz)) +#define NFS3_linkres_sz (1+NFS3_post_op_attr_sz+NFS3_wcc_data_sz) +#define NFS3_readdirres_sz (1+NFS3_post_op_attr_sz+2+NFS3_pagepad_sz) +#define NFS3_fsstatres_sz (1+NFS3_post_op_attr_sz+13) +#define NFS3_fsinfores_sz (1+NFS3_post_op_attr_sz+12) +#define NFS3_pathconfres_sz (1+NFS3_post_op_attr_sz+6) +#define NFS3_commitres_sz (1+NFS3_wcc_data_sz+2) + +#define ACL3_getaclargs_sz (NFS3_fh_sz+1) +#define ACL3_setaclargs_sz (NFS3_fh_sz+1+ \ + XDR_QUADLEN(NFS_ACL_INLINE_BUFSIZE)) +#define ACL3_getaclres_sz (1+NFS3_post_op_attr_sz+1+ \ + XDR_QUADLEN(NFS_ACL_INLINE_BUFSIZE)+\ + NFS3_pagepad_sz) +#define ACL3_setaclres_sz (1+NFS3_post_op_attr_sz) + +/* + * Map file type to S_IFMT bits + */ +static const umode_t nfs_type2fmt[] = { + [NF3BAD] = 0, + [NF3REG] = S_IFREG, + [NF3DIR] = S_IFDIR, + [NF3BLK] = S_IFBLK, + [NF3CHR] = S_IFCHR, + [NF3LNK] = S_IFLNK, + [NF3SOCK] = S_IFSOCK, + [NF3FIFO] = S_IFIFO, +}; + +static struct user_namespace *rpc_userns(const struct rpc_clnt *clnt) +{ + if (clnt && clnt->cl_cred) + return clnt->cl_cred->user_ns; + return &init_user_ns; +} + +static struct user_namespace *rpc_rqst_userns(const struct rpc_rqst *rqstp) +{ + if (rqstp->rq_task) + return rpc_userns(rqstp->rq_task->tk_client); + return &init_user_ns; +} + +/* + * Encode/decode NFSv3 basic data types + * + * Basic NFSv3 data types are defined in section 2.5 of RFC 1813: + * "NFS Version 3 Protocol Specification". + * + * Not all basic data types have their own encoding and decoding + * functions. For run-time efficiency, some data types are encoded + * or decoded inline. + */ + +static void encode_uint32(struct xdr_stream *xdr, u32 value) +{ + __be32 *p = xdr_reserve_space(xdr, 4); + *p = cpu_to_be32(value); +} + +static int decode_uint32(struct xdr_stream *xdr, u32 *value) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + *value = be32_to_cpup(p); + return 0; +} + +static int decode_uint64(struct xdr_stream *xdr, u64 *value) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + xdr_decode_hyper(p, value); + return 0; +} + +/* + * fileid3 + * + * typedef uint64 fileid3; + */ +static __be32 *xdr_decode_fileid3(__be32 *p, u64 *fileid) +{ + return xdr_decode_hyper(p, fileid); +} + +static int decode_fileid3(struct xdr_stream *xdr, u64 *fileid) +{ + return decode_uint64(xdr, fileid); +} + +/* + * filename3 + * + * typedef string filename3<>; + */ +static void encode_filename3(struct xdr_stream *xdr, + const char *name, u32 length) +{ + __be32 *p; + + WARN_ON_ONCE(length > NFS3_MAXNAMLEN); + p = xdr_reserve_space(xdr, 4 + length); + xdr_encode_opaque(p, name, length); +} + +static int decode_inline_filename3(struct xdr_stream *xdr, + const char **name, u32 *length) +{ + __be32 *p; + u32 count; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + count = be32_to_cpup(p); + if (count > NFS3_MAXNAMLEN) + goto out_nametoolong; + p = xdr_inline_decode(xdr, count); + if (unlikely(!p)) + return -EIO; + *name = (const char *)p; + *length = count; + return 0; + +out_nametoolong: + dprintk("NFS: returned filename too long: %u\n", count); + return -ENAMETOOLONG; +} + +/* + * nfspath3 + * + * typedef string nfspath3<>; + */ +static void encode_nfspath3(struct xdr_stream *xdr, struct page **pages, + const u32 length) +{ + encode_uint32(xdr, length); + xdr_write_pages(xdr, pages, 0, length); +} + +static int decode_nfspath3(struct xdr_stream *xdr) +{ + u32 recvd, count; + __be32 *p; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + count = be32_to_cpup(p); + if (unlikely(count >= xdr->buf->page_len || count > NFS3_MAXPATHLEN)) + goto out_nametoolong; + recvd = xdr_read_pages(xdr, count); + if (unlikely(count > recvd)) + goto out_cheating; + xdr_terminate_string(xdr->buf, count); + return 0; + +out_nametoolong: + dprintk("NFS: returned pathname too long: %u\n", count); + return -ENAMETOOLONG; +out_cheating: + dprintk("NFS: server cheating in pathname result: " + "count %u > recvd %u\n", count, recvd); + return -EIO; +} + +/* + * cookie3 + * + * typedef uint64 cookie3 + */ +static __be32 *xdr_encode_cookie3(__be32 *p, u64 cookie) +{ + return xdr_encode_hyper(p, cookie); +} + +static int decode_cookie3(struct xdr_stream *xdr, u64 *cookie) +{ + return decode_uint64(xdr, cookie); +} + +/* + * cookieverf3 + * + * typedef opaque cookieverf3[NFS3_COOKIEVERFSIZE]; + */ +static __be32 *xdr_encode_cookieverf3(__be32 *p, const __be32 *verifier) +{ + memcpy(p, verifier, NFS3_COOKIEVERFSIZE); + return p + XDR_QUADLEN(NFS3_COOKIEVERFSIZE); +} + +static int decode_cookieverf3(struct xdr_stream *xdr, __be32 *verifier) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, NFS3_COOKIEVERFSIZE); + if (unlikely(!p)) + return -EIO; + memcpy(verifier, p, NFS3_COOKIEVERFSIZE); + return 0; +} + +/* + * createverf3 + * + * typedef opaque createverf3[NFS3_CREATEVERFSIZE]; + */ +static void encode_createverf3(struct xdr_stream *xdr, const __be32 *verifier) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, NFS3_CREATEVERFSIZE); + memcpy(p, verifier, NFS3_CREATEVERFSIZE); +} + +static int decode_writeverf3(struct xdr_stream *xdr, struct nfs_write_verifier *verifier) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, NFS3_WRITEVERFSIZE); + if (unlikely(!p)) + return -EIO; + memcpy(verifier->data, p, NFS3_WRITEVERFSIZE); + return 0; +} + +/* + * size3 + * + * typedef uint64 size3; + */ +static __be32 *xdr_decode_size3(__be32 *p, u64 *size) +{ + return xdr_decode_hyper(p, size); +} + +/* + * nfsstat3 + * + * enum nfsstat3 { + * NFS3_OK = 0, + * ... + * } + */ +#define NFS3_OK NFS_OK + +static int decode_nfsstat3(struct xdr_stream *xdr, enum nfs_stat *status) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + if (unlikely(*p != cpu_to_be32(NFS3_OK))) + goto out_status; + *status = 0; + return 0; +out_status: + *status = be32_to_cpup(p); + trace_nfs_xdr_status(xdr, (int)*status); + return 0; +} + +/* + * ftype3 + * + * enum ftype3 { + * NF3REG = 1, + * NF3DIR = 2, + * NF3BLK = 3, + * NF3CHR = 4, + * NF3LNK = 5, + * NF3SOCK = 6, + * NF3FIFO = 7 + * }; + */ +static void encode_ftype3(struct xdr_stream *xdr, const u32 type) +{ + encode_uint32(xdr, type); +} + +static __be32 *xdr_decode_ftype3(__be32 *p, umode_t *mode) +{ + u32 type; + + type = be32_to_cpup(p++); + if (type > NF3FIFO) + type = NF3NON; + *mode = nfs_type2fmt[type]; + return p; +} + +/* + * specdata3 + * + * struct specdata3 { + * uint32 specdata1; + * uint32 specdata2; + * }; + */ +static void encode_specdata3(struct xdr_stream *xdr, const dev_t rdev) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, 8); + *p++ = cpu_to_be32(MAJOR(rdev)); + *p = cpu_to_be32(MINOR(rdev)); +} + +static __be32 *xdr_decode_specdata3(__be32 *p, dev_t *rdev) +{ + unsigned int major, minor; + + major = be32_to_cpup(p++); + minor = be32_to_cpup(p++); + *rdev = MKDEV(major, minor); + if (MAJOR(*rdev) != major || MINOR(*rdev) != minor) + *rdev = 0; + return p; +} + +/* + * nfs_fh3 + * + * struct nfs_fh3 { + * opaque data; + * }; + */ +static void encode_nfs_fh3(struct xdr_stream *xdr, const struct nfs_fh *fh) +{ + __be32 *p; + + WARN_ON_ONCE(fh->size > NFS3_FHSIZE); + p = xdr_reserve_space(xdr, 4 + fh->size); + xdr_encode_opaque(p, fh->data, fh->size); +} + +static int decode_nfs_fh3(struct xdr_stream *xdr, struct nfs_fh *fh) +{ + u32 length; + __be32 *p; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + length = be32_to_cpup(p++); + if (unlikely(length > NFS3_FHSIZE || length == 0)) + goto out_toobig; + p = xdr_inline_decode(xdr, length); + if (unlikely(!p)) + return -EIO; + fh->size = length; + memcpy(fh->data, p, length); + return 0; +out_toobig: + trace_nfs_xdr_bad_filehandle(xdr, NFSERR_BADHANDLE); + return -E2BIG; +} + +static void zero_nfs_fh3(struct nfs_fh *fh) +{ + memset(fh, 0, sizeof(*fh)); +} + +/* + * nfstime3 + * + * struct nfstime3 { + * uint32 seconds; + * uint32 nseconds; + * }; + */ +static __be32 *xdr_encode_nfstime3(__be32 *p, const struct timespec64 *timep) +{ + *p++ = cpu_to_be32((u32)timep->tv_sec); + *p++ = cpu_to_be32(timep->tv_nsec); + return p; +} + +static __be32 *xdr_decode_nfstime3(__be32 *p, struct timespec64 *timep) +{ + timep->tv_sec = be32_to_cpup(p++); + timep->tv_nsec = be32_to_cpup(p++); + return p; +} + +/* + * sattr3 + * + * enum time_how { + * DONT_CHANGE = 0, + * SET_TO_SERVER_TIME = 1, + * SET_TO_CLIENT_TIME = 2 + * }; + * + * union set_mode3 switch (bool set_it) { + * case TRUE: + * mode3 mode; + * default: + * void; + * }; + * + * union set_uid3 switch (bool set_it) { + * case TRUE: + * uid3 uid; + * default: + * void; + * }; + * + * union set_gid3 switch (bool set_it) { + * case TRUE: + * gid3 gid; + * default: + * void; + * }; + * + * union set_size3 switch (bool set_it) { + * case TRUE: + * size3 size; + * default: + * void; + * }; + * + * union set_atime switch (time_how set_it) { + * case SET_TO_CLIENT_TIME: + * nfstime3 atime; + * default: + * void; + * }; + * + * union set_mtime switch (time_how set_it) { + * case SET_TO_CLIENT_TIME: + * nfstime3 mtime; + * default: + * void; + * }; + * + * struct sattr3 { + * set_mode3 mode; + * set_uid3 uid; + * set_gid3 gid; + * set_size3 size; + * set_atime atime; + * set_mtime mtime; + * }; + */ +static void encode_sattr3(struct xdr_stream *xdr, const struct iattr *attr, + struct user_namespace *userns) +{ + u32 nbytes; + __be32 *p; + + /* + * In order to make only a single xdr_reserve_space() call, + * pre-compute the total number of bytes to be reserved. + * Six boolean values, one for each set_foo field, are always + * present in the encoded result, so start there. + */ + nbytes = 6 * 4; + if (attr->ia_valid & ATTR_MODE) + nbytes += 4; + if (attr->ia_valid & ATTR_UID) + nbytes += 4; + if (attr->ia_valid & ATTR_GID) + nbytes += 4; + if (attr->ia_valid & ATTR_SIZE) + nbytes += 8; + if (attr->ia_valid & ATTR_ATIME_SET) + nbytes += 8; + if (attr->ia_valid & ATTR_MTIME_SET) + nbytes += 8; + p = xdr_reserve_space(xdr, nbytes); + + if (attr->ia_valid & ATTR_MODE) { + *p++ = xdr_one; + *p++ = cpu_to_be32(attr->ia_mode & S_IALLUGO); + } else + *p++ = xdr_zero; + + if (attr->ia_valid & ATTR_UID) { + *p++ = xdr_one; + *p++ = cpu_to_be32(from_kuid_munged(userns, attr->ia_uid)); + } else + *p++ = xdr_zero; + + if (attr->ia_valid & ATTR_GID) { + *p++ = xdr_one; + *p++ = cpu_to_be32(from_kgid_munged(userns, attr->ia_gid)); + } else + *p++ = xdr_zero; + + if (attr->ia_valid & ATTR_SIZE) { + *p++ = xdr_one; + p = xdr_encode_hyper(p, (u64)attr->ia_size); + } else + *p++ = xdr_zero; + + if (attr->ia_valid & ATTR_ATIME_SET) { + *p++ = xdr_two; + p = xdr_encode_nfstime3(p, &attr->ia_atime); + } else if (attr->ia_valid & ATTR_ATIME) { + *p++ = xdr_one; + } else + *p++ = xdr_zero; + + if (attr->ia_valid & ATTR_MTIME_SET) { + *p++ = xdr_two; + xdr_encode_nfstime3(p, &attr->ia_mtime); + } else if (attr->ia_valid & ATTR_MTIME) { + *p = xdr_one; + } else + *p = xdr_zero; +} + +/* + * fattr3 + * + * struct fattr3 { + * ftype3 type; + * mode3 mode; + * uint32 nlink; + * uid3 uid; + * gid3 gid; + * size3 size; + * size3 used; + * specdata3 rdev; + * uint64 fsid; + * fileid3 fileid; + * nfstime3 atime; + * nfstime3 mtime; + * nfstime3 ctime; + * }; + */ +static int decode_fattr3(struct xdr_stream *xdr, struct nfs_fattr *fattr, + struct user_namespace *userns) +{ + umode_t fmode; + __be32 *p; + + p = xdr_inline_decode(xdr, NFS3_fattr_sz << 2); + if (unlikely(!p)) + return -EIO; + + p = xdr_decode_ftype3(p, &fmode); + + fattr->mode = (be32_to_cpup(p++) & ~S_IFMT) | fmode; + fattr->nlink = be32_to_cpup(p++); + fattr->uid = make_kuid(userns, be32_to_cpup(p++)); + if (!uid_valid(fattr->uid)) + goto out_uid; + fattr->gid = make_kgid(userns, be32_to_cpup(p++)); + if (!gid_valid(fattr->gid)) + goto out_gid; + + p = xdr_decode_size3(p, &fattr->size); + p = xdr_decode_size3(p, &fattr->du.nfs3.used); + p = xdr_decode_specdata3(p, &fattr->rdev); + + p = xdr_decode_hyper(p, &fattr->fsid.major); + fattr->fsid.minor = 0; + + p = xdr_decode_fileid3(p, &fattr->fileid); + p = xdr_decode_nfstime3(p, &fattr->atime); + p = xdr_decode_nfstime3(p, &fattr->mtime); + xdr_decode_nfstime3(p, &fattr->ctime); + fattr->change_attr = nfs_timespec_to_change_attr(&fattr->ctime); + + fattr->valid |= NFS_ATTR_FATTR_V3; + return 0; +out_uid: + dprintk("NFS: returned invalid uid\n"); + return -EINVAL; +out_gid: + dprintk("NFS: returned invalid gid\n"); + return -EINVAL; +} + +/* + * post_op_attr + * + * union post_op_attr switch (bool attributes_follow) { + * case TRUE: + * fattr3 attributes; + * case FALSE: + * void; + * }; + */ +static int decode_post_op_attr(struct xdr_stream *xdr, struct nfs_fattr *fattr, + struct user_namespace *userns) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + if (*p != xdr_zero) + return decode_fattr3(xdr, fattr, userns); + return 0; +} + +/* + * wcc_attr + * struct wcc_attr { + * size3 size; + * nfstime3 mtime; + * nfstime3 ctime; + * }; + */ +static int decode_wcc_attr(struct xdr_stream *xdr, struct nfs_fattr *fattr) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, NFS3_wcc_attr_sz << 2); + if (unlikely(!p)) + return -EIO; + + fattr->valid |= NFS_ATTR_FATTR_PRESIZE + | NFS_ATTR_FATTR_PRECHANGE + | NFS_ATTR_FATTR_PREMTIME + | NFS_ATTR_FATTR_PRECTIME; + + p = xdr_decode_size3(p, &fattr->pre_size); + p = xdr_decode_nfstime3(p, &fattr->pre_mtime); + xdr_decode_nfstime3(p, &fattr->pre_ctime); + fattr->pre_change_attr = nfs_timespec_to_change_attr(&fattr->pre_ctime); + + return 0; +} + +/* + * pre_op_attr + * union pre_op_attr switch (bool attributes_follow) { + * case TRUE: + * wcc_attr attributes; + * case FALSE: + * void; + * }; + * + * wcc_data + * + * struct wcc_data { + * pre_op_attr before; + * post_op_attr after; + * }; + */ +static int decode_pre_op_attr(struct xdr_stream *xdr, struct nfs_fattr *fattr) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + if (*p != xdr_zero) + return decode_wcc_attr(xdr, fattr); + return 0; +} + +static int decode_wcc_data(struct xdr_stream *xdr, struct nfs_fattr *fattr, + struct user_namespace *userns) +{ + int error; + + error = decode_pre_op_attr(xdr, fattr); + if (unlikely(error)) + goto out; + error = decode_post_op_attr(xdr, fattr, userns); +out: + return error; +} + +/* + * post_op_fh3 + * + * union post_op_fh3 switch (bool handle_follows) { + * case TRUE: + * nfs_fh3 handle; + * case FALSE: + * void; + * }; + */ +static int decode_post_op_fh3(struct xdr_stream *xdr, struct nfs_fh *fh) +{ + __be32 *p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + if (*p != xdr_zero) + return decode_nfs_fh3(xdr, fh); + zero_nfs_fh3(fh); + return 0; +} + +/* + * diropargs3 + * + * struct diropargs3 { + * nfs_fh3 dir; + * filename3 name; + * }; + */ +static void encode_diropargs3(struct xdr_stream *xdr, const struct nfs_fh *fh, + const char *name, u32 length) +{ + encode_nfs_fh3(xdr, fh); + encode_filename3(xdr, name, length); +} + + +/* + * NFSv3 XDR encode functions + * + * NFSv3 argument types are defined in section 3.3 of RFC 1813: + * "NFS Version 3 Protocol Specification". + */ + +/* + * 3.3.1 GETATTR3args + * + * struct GETATTR3args { + * nfs_fh3 object; + * }; + */ +static void nfs3_xdr_enc_getattr3args(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_fh *fh = data; + + encode_nfs_fh3(xdr, fh); +} + +/* + * 3.3.2 SETATTR3args + * + * union sattrguard3 switch (bool check) { + * case TRUE: + * nfstime3 obj_ctime; + * case FALSE: + * void; + * }; + * + * struct SETATTR3args { + * nfs_fh3 object; + * sattr3 new_attributes; + * sattrguard3 guard; + * }; + */ +static void encode_sattrguard3(struct xdr_stream *xdr, + const struct nfs3_sattrargs *args) +{ + __be32 *p; + + if (args->guard) { + p = xdr_reserve_space(xdr, 4 + 8); + *p++ = xdr_one; + xdr_encode_nfstime3(p, &args->guardtime); + } else { + p = xdr_reserve_space(xdr, 4); + *p = xdr_zero; + } +} + +static void nfs3_xdr_enc_setattr3args(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs3_sattrargs *args = data; + encode_nfs_fh3(xdr, args->fh); + encode_sattr3(xdr, args->sattr, rpc_rqst_userns(req)); + encode_sattrguard3(xdr, args); +} + +/* + * 3.3.3 LOOKUP3args + * + * struct LOOKUP3args { + * diropargs3 what; + * }; + */ +static void nfs3_xdr_enc_lookup3args(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs3_diropargs *args = data; + + encode_diropargs3(xdr, args->fh, args->name, args->len); +} + +/* + * 3.3.4 ACCESS3args + * + * struct ACCESS3args { + * nfs_fh3 object; + * uint32 access; + * }; + */ +static void encode_access3args(struct xdr_stream *xdr, + const struct nfs3_accessargs *args) +{ + encode_nfs_fh3(xdr, args->fh); + encode_uint32(xdr, args->access); +} + +static void nfs3_xdr_enc_access3args(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs3_accessargs *args = data; + + encode_access3args(xdr, args); +} + +/* + * 3.3.5 READLINK3args + * + * struct READLINK3args { + * nfs_fh3 symlink; + * }; + */ +static void nfs3_xdr_enc_readlink3args(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs3_readlinkargs *args = data; + + encode_nfs_fh3(xdr, args->fh); + rpc_prepare_reply_pages(req, args->pages, args->pgbase, args->pglen, + NFS3_readlinkres_sz - NFS3_pagepad_sz); +} + +/* + * 3.3.6 READ3args + * + * struct READ3args { + * nfs_fh3 file; + * offset3 offset; + * count3 count; + * }; + */ +static void encode_read3args(struct xdr_stream *xdr, + const struct nfs_pgio_args *args) +{ + __be32 *p; + + encode_nfs_fh3(xdr, args->fh); + + p = xdr_reserve_space(xdr, 8 + 4); + p = xdr_encode_hyper(p, args->offset); + *p = cpu_to_be32(args->count); +} + +static void nfs3_xdr_enc_read3args(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_pgio_args *args = data; + unsigned int replen = args->replen ? args->replen : + NFS3_readres_sz - NFS3_pagepad_sz; + + encode_read3args(xdr, args); + rpc_prepare_reply_pages(req, args->pages, args->pgbase, + args->count, replen); + req->rq_rcv_buf.flags |= XDRBUF_READ; +} + +/* + * 3.3.7 WRITE3args + * + * enum stable_how { + * UNSTABLE = 0, + * DATA_SYNC = 1, + * FILE_SYNC = 2 + * }; + * + * struct WRITE3args { + * nfs_fh3 file; + * offset3 offset; + * count3 count; + * stable_how stable; + * opaque data<>; + * }; + */ +static void encode_write3args(struct xdr_stream *xdr, + const struct nfs_pgio_args *args) +{ + __be32 *p; + + encode_nfs_fh3(xdr, args->fh); + + p = xdr_reserve_space(xdr, 8 + 4 + 4 + 4); + p = xdr_encode_hyper(p, args->offset); + *p++ = cpu_to_be32(args->count); + *p++ = cpu_to_be32(args->stable); + *p = cpu_to_be32(args->count); + xdr_write_pages(xdr, args->pages, args->pgbase, args->count); +} + +static void nfs3_xdr_enc_write3args(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_pgio_args *args = data; + + encode_write3args(xdr, args); + xdr->buf->flags |= XDRBUF_WRITE; +} + +/* + * 3.3.8 CREATE3args + * + * enum createmode3 { + * UNCHECKED = 0, + * GUARDED = 1, + * EXCLUSIVE = 2 + * }; + * + * union createhow3 switch (createmode3 mode) { + * case UNCHECKED: + * case GUARDED: + * sattr3 obj_attributes; + * case EXCLUSIVE: + * createverf3 verf; + * }; + * + * struct CREATE3args { + * diropargs3 where; + * createhow3 how; + * }; + */ +static void encode_createhow3(struct xdr_stream *xdr, + const struct nfs3_createargs *args, + struct user_namespace *userns) +{ + encode_uint32(xdr, args->createmode); + switch (args->createmode) { + case NFS3_CREATE_UNCHECKED: + case NFS3_CREATE_GUARDED: + encode_sattr3(xdr, args->sattr, userns); + break; + case NFS3_CREATE_EXCLUSIVE: + encode_createverf3(xdr, args->verifier); + break; + default: + BUG(); + } +} + +static void nfs3_xdr_enc_create3args(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs3_createargs *args = data; + + encode_diropargs3(xdr, args->fh, args->name, args->len); + encode_createhow3(xdr, args, rpc_rqst_userns(req)); +} + +/* + * 3.3.9 MKDIR3args + * + * struct MKDIR3args { + * diropargs3 where; + * sattr3 attributes; + * }; + */ +static void nfs3_xdr_enc_mkdir3args(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs3_mkdirargs *args = data; + + encode_diropargs3(xdr, args->fh, args->name, args->len); + encode_sattr3(xdr, args->sattr, rpc_rqst_userns(req)); +} + +/* + * 3.3.10 SYMLINK3args + * + * struct symlinkdata3 { + * sattr3 symlink_attributes; + * nfspath3 symlink_data; + * }; + * + * struct SYMLINK3args { + * diropargs3 where; + * symlinkdata3 symlink; + * }; + */ +static void encode_symlinkdata3(struct xdr_stream *xdr, + const void *data, + struct user_namespace *userns) +{ + const struct nfs3_symlinkargs *args = data; + + encode_sattr3(xdr, args->sattr, userns); + encode_nfspath3(xdr, args->pages, args->pathlen); +} + +static void nfs3_xdr_enc_symlink3args(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs3_symlinkargs *args = data; + + encode_diropargs3(xdr, args->fromfh, args->fromname, args->fromlen); + encode_symlinkdata3(xdr, args, rpc_rqst_userns(req)); + xdr->buf->flags |= XDRBUF_WRITE; +} + +/* + * 3.3.11 MKNOD3args + * + * struct devicedata3 { + * sattr3 dev_attributes; + * specdata3 spec; + * }; + * + * union mknoddata3 switch (ftype3 type) { + * case NF3CHR: + * case NF3BLK: + * devicedata3 device; + * case NF3SOCK: + * case NF3FIFO: + * sattr3 pipe_attributes; + * default: + * void; + * }; + * + * struct MKNOD3args { + * diropargs3 where; + * mknoddata3 what; + * }; + */ +static void encode_devicedata3(struct xdr_stream *xdr, + const struct nfs3_mknodargs *args, + struct user_namespace *userns) +{ + encode_sattr3(xdr, args->sattr, userns); + encode_specdata3(xdr, args->rdev); +} + +static void encode_mknoddata3(struct xdr_stream *xdr, + const struct nfs3_mknodargs *args, + struct user_namespace *userns) +{ + encode_ftype3(xdr, args->type); + switch (args->type) { + case NF3CHR: + case NF3BLK: + encode_devicedata3(xdr, args, userns); + break; + case NF3SOCK: + case NF3FIFO: + encode_sattr3(xdr, args->sattr, userns); + break; + case NF3REG: + case NF3DIR: + break; + default: + BUG(); + } +} + +static void nfs3_xdr_enc_mknod3args(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs3_mknodargs *args = data; + + encode_diropargs3(xdr, args->fh, args->name, args->len); + encode_mknoddata3(xdr, args, rpc_rqst_userns(req)); +} + +/* + * 3.3.12 REMOVE3args + * + * struct REMOVE3args { + * diropargs3 object; + * }; + */ +static void nfs3_xdr_enc_remove3args(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_removeargs *args = data; + + encode_diropargs3(xdr, args->fh, args->name.name, args->name.len); +} + +/* + * 3.3.14 RENAME3args + * + * struct RENAME3args { + * diropargs3 from; + * diropargs3 to; + * }; + */ +static void nfs3_xdr_enc_rename3args(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_renameargs *args = data; + const struct qstr *old = args->old_name; + const struct qstr *new = args->new_name; + + encode_diropargs3(xdr, args->old_dir, old->name, old->len); + encode_diropargs3(xdr, args->new_dir, new->name, new->len); +} + +/* + * 3.3.15 LINK3args + * + * struct LINK3args { + * nfs_fh3 file; + * diropargs3 link; + * }; + */ +static void nfs3_xdr_enc_link3args(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs3_linkargs *args = data; + + encode_nfs_fh3(xdr, args->fromfh); + encode_diropargs3(xdr, args->tofh, args->toname, args->tolen); +} + +/* + * 3.3.16 READDIR3args + * + * struct READDIR3args { + * nfs_fh3 dir; + * cookie3 cookie; + * cookieverf3 cookieverf; + * count3 count; + * }; + */ +static void encode_readdir3args(struct xdr_stream *xdr, + const struct nfs3_readdirargs *args) +{ + __be32 *p; + + encode_nfs_fh3(xdr, args->fh); + + p = xdr_reserve_space(xdr, 8 + NFS3_COOKIEVERFSIZE + 4); + p = xdr_encode_cookie3(p, args->cookie); + p = xdr_encode_cookieverf3(p, args->verf); + *p = cpu_to_be32(args->count); +} + +static void nfs3_xdr_enc_readdir3args(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs3_readdirargs *args = data; + + encode_readdir3args(xdr, args); + rpc_prepare_reply_pages(req, args->pages, 0, args->count, + NFS3_readdirres_sz - NFS3_pagepad_sz); +} + +/* + * 3.3.17 READDIRPLUS3args + * + * struct READDIRPLUS3args { + * nfs_fh3 dir; + * cookie3 cookie; + * cookieverf3 cookieverf; + * count3 dircount; + * count3 maxcount; + * }; + */ +static void encode_readdirplus3args(struct xdr_stream *xdr, + const struct nfs3_readdirargs *args) +{ + uint32_t dircount = args->count; + uint32_t maxcount = args->count; + __be32 *p; + + encode_nfs_fh3(xdr, args->fh); + + p = xdr_reserve_space(xdr, 8 + NFS3_COOKIEVERFSIZE + 4 + 4); + p = xdr_encode_cookie3(p, args->cookie); + p = xdr_encode_cookieverf3(p, args->verf); + + /* + * readdirplus: need dircount + buffer size. + * We just make sure we make dircount big enough + */ + *p++ = cpu_to_be32(dircount); + *p = cpu_to_be32(maxcount); +} + +static void nfs3_xdr_enc_readdirplus3args(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs3_readdirargs *args = data; + + encode_readdirplus3args(xdr, args); + rpc_prepare_reply_pages(req, args->pages, 0, args->count, + NFS3_readdirres_sz - NFS3_pagepad_sz); +} + +/* + * 3.3.21 COMMIT3args + * + * struct COMMIT3args { + * nfs_fh3 file; + * offset3 offset; + * count3 count; + * }; + */ +static void encode_commit3args(struct xdr_stream *xdr, + const struct nfs_commitargs *args) +{ + __be32 *p; + + encode_nfs_fh3(xdr, args->fh); + + p = xdr_reserve_space(xdr, 8 + 4); + p = xdr_encode_hyper(p, args->offset); + *p = cpu_to_be32(args->count); +} + +static void nfs3_xdr_enc_commit3args(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_commitargs *args = data; + + encode_commit3args(xdr, args); +} + +#ifdef CONFIG_NFS_V3_ACL + +static void nfs3_xdr_enc_getacl3args(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs3_getaclargs *args = data; + + encode_nfs_fh3(xdr, args->fh); + encode_uint32(xdr, args->mask); + if (args->mask & (NFS_ACL | NFS_DFACL)) { + rpc_prepare_reply_pages(req, args->pages, 0, + NFSACL_MAXPAGES << PAGE_SHIFT, + ACL3_getaclres_sz - NFS3_pagepad_sz); + req->rq_rcv_buf.flags |= XDRBUF_SPARSE_PAGES; + } +} + +static void nfs3_xdr_enc_setacl3args(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs3_setaclargs *args = data; + unsigned int base; + int error; + + encode_nfs_fh3(xdr, NFS_FH(args->inode)); + encode_uint32(xdr, args->mask); + + base = req->rq_slen; + if (args->npages != 0) + xdr_write_pages(xdr, args->pages, 0, args->len); + else + xdr_reserve_space(xdr, args->len); + + error = nfsacl_encode(xdr->buf, base, args->inode, + (args->mask & NFS_ACL) ? + args->acl_access : NULL, 1, 0); + /* FIXME: this is just broken */ + BUG_ON(error < 0); + error = nfsacl_encode(xdr->buf, base + error, args->inode, + (args->mask & NFS_DFACL) ? + args->acl_default : NULL, 1, + NFS_ACL_DEFAULT); + BUG_ON(error < 0); +} + +#endif /* CONFIG_NFS_V3_ACL */ + +/* + * NFSv3 XDR decode functions + * + * NFSv3 result types are defined in section 3.3 of RFC 1813: + * "NFS Version 3 Protocol Specification". + */ + +/* + * 3.3.1 GETATTR3res + * + * struct GETATTR3resok { + * fattr3 obj_attributes; + * }; + * + * union GETATTR3res switch (nfsstat3 status) { + * case NFS3_OK: + * GETATTR3resok resok; + * default: + * void; + * }; + */ +static int nfs3_xdr_dec_getattr3res(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *result) +{ + enum nfs_stat status; + int error; + + error = decode_nfsstat3(xdr, &status); + if (unlikely(error)) + goto out; + if (status != NFS3_OK) + goto out_default; + error = decode_fattr3(xdr, result, rpc_rqst_userns(req)); +out: + return error; +out_default: + return nfs_stat_to_errno(status); +} + +/* + * 3.3.2 SETATTR3res + * + * struct SETATTR3resok { + * wcc_data obj_wcc; + * }; + * + * struct SETATTR3resfail { + * wcc_data obj_wcc; + * }; + * + * union SETATTR3res switch (nfsstat3 status) { + * case NFS3_OK: + * SETATTR3resok resok; + * default: + * SETATTR3resfail resfail; + * }; + */ +static int nfs3_xdr_dec_setattr3res(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *result) +{ + enum nfs_stat status; + int error; + + error = decode_nfsstat3(xdr, &status); + if (unlikely(error)) + goto out; + error = decode_wcc_data(xdr, result, rpc_rqst_userns(req)); + if (unlikely(error)) + goto out; + if (status != NFS3_OK) + goto out_status; +out: + return error; +out_status: + return nfs_stat_to_errno(status); +} + +/* + * 3.3.3 LOOKUP3res + * + * struct LOOKUP3resok { + * nfs_fh3 object; + * post_op_attr obj_attributes; + * post_op_attr dir_attributes; + * }; + * + * struct LOOKUP3resfail { + * post_op_attr dir_attributes; + * }; + * + * union LOOKUP3res switch (nfsstat3 status) { + * case NFS3_OK: + * LOOKUP3resok resok; + * default: + * LOOKUP3resfail resfail; + * }; + */ +static int nfs3_xdr_dec_lookup3res(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *data) +{ + struct user_namespace *userns = rpc_rqst_userns(req); + struct nfs3_diropres *result = data; + enum nfs_stat status; + int error; + + error = decode_nfsstat3(xdr, &status); + if (unlikely(error)) + goto out; + if (status != NFS3_OK) + goto out_default; + error = decode_nfs_fh3(xdr, result->fh); + if (unlikely(error)) + goto out; + error = decode_post_op_attr(xdr, result->fattr, userns); + if (unlikely(error)) + goto out; + error = decode_post_op_attr(xdr, result->dir_attr, userns); +out: + return error; +out_default: + error = decode_post_op_attr(xdr, result->dir_attr, userns); + if (unlikely(error)) + goto out; + return nfs_stat_to_errno(status); +} + +/* + * 3.3.4 ACCESS3res + * + * struct ACCESS3resok { + * post_op_attr obj_attributes; + * uint32 access; + * }; + * + * struct ACCESS3resfail { + * post_op_attr obj_attributes; + * }; + * + * union ACCESS3res switch (nfsstat3 status) { + * case NFS3_OK: + * ACCESS3resok resok; + * default: + * ACCESS3resfail resfail; + * }; + */ +static int nfs3_xdr_dec_access3res(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *data) +{ + struct nfs3_accessres *result = data; + enum nfs_stat status; + int error; + + error = decode_nfsstat3(xdr, &status); + if (unlikely(error)) + goto out; + error = decode_post_op_attr(xdr, result->fattr, rpc_rqst_userns(req)); + if (unlikely(error)) + goto out; + if (status != NFS3_OK) + goto out_default; + error = decode_uint32(xdr, &result->access); +out: + return error; +out_default: + return nfs_stat_to_errno(status); +} + +/* + * 3.3.5 READLINK3res + * + * struct READLINK3resok { + * post_op_attr symlink_attributes; + * nfspath3 data; + * }; + * + * struct READLINK3resfail { + * post_op_attr symlink_attributes; + * }; + * + * union READLINK3res switch (nfsstat3 status) { + * case NFS3_OK: + * READLINK3resok resok; + * default: + * READLINK3resfail resfail; + * }; + */ +static int nfs3_xdr_dec_readlink3res(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *result) +{ + enum nfs_stat status; + int error; + + error = decode_nfsstat3(xdr, &status); + if (unlikely(error)) + goto out; + error = decode_post_op_attr(xdr, result, rpc_rqst_userns(req)); + if (unlikely(error)) + goto out; + if (status != NFS3_OK) + goto out_default; + error = decode_nfspath3(xdr); +out: + return error; +out_default: + return nfs_stat_to_errno(status); +} + +/* + * 3.3.6 READ3res + * + * struct READ3resok { + * post_op_attr file_attributes; + * count3 count; + * bool eof; + * opaque data<>; + * }; + * + * struct READ3resfail { + * post_op_attr file_attributes; + * }; + * + * union READ3res switch (nfsstat3 status) { + * case NFS3_OK: + * READ3resok resok; + * default: + * READ3resfail resfail; + * }; + */ +static int decode_read3resok(struct xdr_stream *xdr, + struct nfs_pgio_res *result) +{ + u32 eof, count, ocount, recvd; + __be32 *p; + + p = xdr_inline_decode(xdr, 4 + 4 + 4); + if (unlikely(!p)) + return -EIO; + count = be32_to_cpup(p++); + eof = be32_to_cpup(p++); + ocount = be32_to_cpup(p++); + if (unlikely(ocount != count)) + goto out_mismatch; + recvd = xdr_read_pages(xdr, count); + if (unlikely(count > recvd)) + goto out_cheating; +out: + result->eof = eof; + result->count = count; + return count; +out_mismatch: + dprintk("NFS: READ count doesn't match length of opaque: " + "count %u != ocount %u\n", count, ocount); + return -EIO; +out_cheating: + dprintk("NFS: server cheating in read result: " + "count %u > recvd %u\n", count, recvd); + count = recvd; + eof = 0; + goto out; +} + +static int nfs3_xdr_dec_read3res(struct rpc_rqst *req, struct xdr_stream *xdr, + void *data) +{ + struct nfs_pgio_res *result = data; + unsigned int pos; + enum nfs_stat status; + int error; + + pos = xdr_stream_pos(xdr); + error = decode_nfsstat3(xdr, &status); + if (unlikely(error)) + goto out; + error = decode_post_op_attr(xdr, result->fattr, rpc_rqst_userns(req)); + if (unlikely(error)) + goto out; + result->op_status = status; + if (status != NFS3_OK) + goto out_status; + result->replen = 3 + ((xdr_stream_pos(xdr) - pos) >> 2); + error = decode_read3resok(xdr, result); +out: + return error; +out_status: + return nfs_stat_to_errno(status); +} + +/* + * 3.3.7 WRITE3res + * + * enum stable_how { + * UNSTABLE = 0, + * DATA_SYNC = 1, + * FILE_SYNC = 2 + * }; + * + * struct WRITE3resok { + * wcc_data file_wcc; + * count3 count; + * stable_how committed; + * writeverf3 verf; + * }; + * + * struct WRITE3resfail { + * wcc_data file_wcc; + * }; + * + * union WRITE3res switch (nfsstat3 status) { + * case NFS3_OK: + * WRITE3resok resok; + * default: + * WRITE3resfail resfail; + * }; + */ +static int decode_write3resok(struct xdr_stream *xdr, + struct nfs_pgio_res *result) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, 4 + 4); + if (unlikely(!p)) + return -EIO; + result->count = be32_to_cpup(p++); + result->verf->committed = be32_to_cpup(p++); + if (unlikely(result->verf->committed > NFS_FILE_SYNC)) + goto out_badvalue; + if (decode_writeverf3(xdr, &result->verf->verifier)) + return -EIO; + return result->count; +out_badvalue: + dprintk("NFS: bad stable_how value: %u\n", result->verf->committed); + return -EIO; +} + +static int nfs3_xdr_dec_write3res(struct rpc_rqst *req, struct xdr_stream *xdr, + void *data) +{ + struct nfs_pgio_res *result = data; + enum nfs_stat status; + int error; + + error = decode_nfsstat3(xdr, &status); + if (unlikely(error)) + goto out; + error = decode_wcc_data(xdr, result->fattr, rpc_rqst_userns(req)); + if (unlikely(error)) + goto out; + result->op_status = status; + if (status != NFS3_OK) + goto out_status; + error = decode_write3resok(xdr, result); +out: + return error; +out_status: + return nfs_stat_to_errno(status); +} + +/* + * 3.3.8 CREATE3res + * + * struct CREATE3resok { + * post_op_fh3 obj; + * post_op_attr obj_attributes; + * wcc_data dir_wcc; + * }; + * + * struct CREATE3resfail { + * wcc_data dir_wcc; + * }; + * + * union CREATE3res switch (nfsstat3 status) { + * case NFS3_OK: + * CREATE3resok resok; + * default: + * CREATE3resfail resfail; + * }; + */ +static int decode_create3resok(struct xdr_stream *xdr, + struct nfs3_diropres *result, + struct user_namespace *userns) +{ + int error; + + error = decode_post_op_fh3(xdr, result->fh); + if (unlikely(error)) + goto out; + error = decode_post_op_attr(xdr, result->fattr, userns); + if (unlikely(error)) + goto out; + /* The server isn't required to return a file handle. + * If it didn't, force the client to perform a LOOKUP + * to determine the correct file handle and attribute + * values for the new object. */ + if (result->fh->size == 0) + result->fattr->valid = 0; + error = decode_wcc_data(xdr, result->dir_attr, userns); +out: + return error; +} + +static int nfs3_xdr_dec_create3res(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *data) +{ + struct user_namespace *userns = rpc_rqst_userns(req); + struct nfs3_diropres *result = data; + enum nfs_stat status; + int error; + + error = decode_nfsstat3(xdr, &status); + if (unlikely(error)) + goto out; + if (status != NFS3_OK) + goto out_default; + error = decode_create3resok(xdr, result, userns); +out: + return error; +out_default: + error = decode_wcc_data(xdr, result->dir_attr, userns); + if (unlikely(error)) + goto out; + return nfs_stat_to_errno(status); +} + +/* + * 3.3.12 REMOVE3res + * + * struct REMOVE3resok { + * wcc_data dir_wcc; + * }; + * + * struct REMOVE3resfail { + * wcc_data dir_wcc; + * }; + * + * union REMOVE3res switch (nfsstat3 status) { + * case NFS3_OK: + * REMOVE3resok resok; + * default: + * REMOVE3resfail resfail; + * }; + */ +static int nfs3_xdr_dec_remove3res(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *data) +{ + struct nfs_removeres *result = data; + enum nfs_stat status; + int error; + + error = decode_nfsstat3(xdr, &status); + if (unlikely(error)) + goto out; + error = decode_wcc_data(xdr, result->dir_attr, rpc_rqst_userns(req)); + if (unlikely(error)) + goto out; + if (status != NFS3_OK) + goto out_status; +out: + return error; +out_status: + return nfs_stat_to_errno(status); +} + +/* + * 3.3.14 RENAME3res + * + * struct RENAME3resok { + * wcc_data fromdir_wcc; + * wcc_data todir_wcc; + * }; + * + * struct RENAME3resfail { + * wcc_data fromdir_wcc; + * wcc_data todir_wcc; + * }; + * + * union RENAME3res switch (nfsstat3 status) { + * case NFS3_OK: + * RENAME3resok resok; + * default: + * RENAME3resfail resfail; + * }; + */ +static int nfs3_xdr_dec_rename3res(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *data) +{ + struct user_namespace *userns = rpc_rqst_userns(req); + struct nfs_renameres *result = data; + enum nfs_stat status; + int error; + + error = decode_nfsstat3(xdr, &status); + if (unlikely(error)) + goto out; + error = decode_wcc_data(xdr, result->old_fattr, userns); + if (unlikely(error)) + goto out; + error = decode_wcc_data(xdr, result->new_fattr, userns); + if (unlikely(error)) + goto out; + if (status != NFS3_OK) + goto out_status; +out: + return error; +out_status: + return nfs_stat_to_errno(status); +} + +/* + * 3.3.15 LINK3res + * + * struct LINK3resok { + * post_op_attr file_attributes; + * wcc_data linkdir_wcc; + * }; + * + * struct LINK3resfail { + * post_op_attr file_attributes; + * wcc_data linkdir_wcc; + * }; + * + * union LINK3res switch (nfsstat3 status) { + * case NFS3_OK: + * LINK3resok resok; + * default: + * LINK3resfail resfail; + * }; + */ +static int nfs3_xdr_dec_link3res(struct rpc_rqst *req, struct xdr_stream *xdr, + void *data) +{ + struct user_namespace *userns = rpc_rqst_userns(req); + struct nfs3_linkres *result = data; + enum nfs_stat status; + int error; + + error = decode_nfsstat3(xdr, &status); + if (unlikely(error)) + goto out; + error = decode_post_op_attr(xdr, result->fattr, userns); + if (unlikely(error)) + goto out; + error = decode_wcc_data(xdr, result->dir_attr, userns); + if (unlikely(error)) + goto out; + if (status != NFS3_OK) + goto out_status; +out: + return error; +out_status: + return nfs_stat_to_errno(status); +} + +/** + * nfs3_decode_dirent - Decode a single NFSv3 directory entry stored in + * the local page cache + * @xdr: XDR stream where entry resides + * @entry: buffer to fill in with entry data + * @plus: boolean indicating whether this should be a readdirplus entry + * + * Returns zero if successful, otherwise a negative errno value is + * returned. + * + * This function is not invoked during READDIR reply decoding, but + * rather whenever an application invokes the getdents(2) system call + * on a directory already in our cache. + * + * 3.3.16 entry3 + * + * struct entry3 { + * fileid3 fileid; + * filename3 name; + * cookie3 cookie; + * fhandle3 filehandle; + * post_op_attr3 attributes; + * entry3 *nextentry; + * }; + * + * 3.3.17 entryplus3 + * struct entryplus3 { + * fileid3 fileid; + * filename3 name; + * cookie3 cookie; + * post_op_attr name_attributes; + * post_op_fh3 name_handle; + * entryplus3 *nextentry; + * }; + */ +int nfs3_decode_dirent(struct xdr_stream *xdr, struct nfs_entry *entry, + bool plus) +{ + struct user_namespace *userns = rpc_userns(entry->server->client); + __be32 *p; + int error; + u64 new_cookie; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EAGAIN; + if (*p == xdr_zero) { + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EAGAIN; + if (*p == xdr_zero) + return -EAGAIN; + entry->eof = 1; + return -EBADCOOKIE; + } + + error = decode_fileid3(xdr, &entry->ino); + if (unlikely(error)) + return -EAGAIN; + + error = decode_inline_filename3(xdr, &entry->name, &entry->len); + if (unlikely(error)) + return error == -ENAMETOOLONG ? -ENAMETOOLONG : -EAGAIN; + + error = decode_cookie3(xdr, &new_cookie); + if (unlikely(error)) + return -EAGAIN; + + entry->d_type = DT_UNKNOWN; + + if (plus) { + entry->fattr->valid = 0; + error = decode_post_op_attr(xdr, entry->fattr, userns); + if (unlikely(error)) + return -EAGAIN; + if (entry->fattr->valid & NFS_ATTR_FATTR_V3) + entry->d_type = nfs_umode_to_dtype(entry->fattr->mode); + + if (entry->fattr->fileid != entry->ino) { + entry->fattr->mounted_on_fileid = entry->ino; + entry->fattr->valid |= NFS_ATTR_FATTR_MOUNTED_ON_FILEID; + } + + /* In fact, a post_op_fh3: */ + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EAGAIN; + if (*p != xdr_zero) { + error = decode_nfs_fh3(xdr, entry->fh); + if (unlikely(error)) + return -EAGAIN; + } else + zero_nfs_fh3(entry->fh); + } + + entry->cookie = new_cookie; + + return 0; +} + +/* + * 3.3.16 READDIR3res + * + * struct dirlist3 { + * entry3 *entries; + * bool eof; + * }; + * + * struct READDIR3resok { + * post_op_attr dir_attributes; + * cookieverf3 cookieverf; + * dirlist3 reply; + * }; + * + * struct READDIR3resfail { + * post_op_attr dir_attributes; + * }; + * + * union READDIR3res switch (nfsstat3 status) { + * case NFS3_OK: + * READDIR3resok resok; + * default: + * READDIR3resfail resfail; + * }; + * + * Read the directory contents into the page cache, but otherwise + * don't touch them. The actual decoding is done by nfs3_decode_entry() + * during subsequent nfs_readdir() calls. + */ +static int decode_dirlist3(struct xdr_stream *xdr) +{ + return xdr_read_pages(xdr, xdr->buf->page_len); +} + +static int decode_readdir3resok(struct xdr_stream *xdr, + struct nfs3_readdirres *result, + struct user_namespace *userns) +{ + int error; + + error = decode_post_op_attr(xdr, result->dir_attr, userns); + if (unlikely(error)) + goto out; + /* XXX: do we need to check if result->verf != NULL ? */ + error = decode_cookieverf3(xdr, result->verf); + if (unlikely(error)) + goto out; + error = decode_dirlist3(xdr); +out: + return error; +} + +static int nfs3_xdr_dec_readdir3res(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *data) +{ + struct nfs3_readdirres *result = data; + enum nfs_stat status; + int error; + + error = decode_nfsstat3(xdr, &status); + if (unlikely(error)) + goto out; + if (status != NFS3_OK) + goto out_default; + error = decode_readdir3resok(xdr, result, rpc_rqst_userns(req)); +out: + return error; +out_default: + error = decode_post_op_attr(xdr, result->dir_attr, rpc_rqst_userns(req)); + if (unlikely(error)) + goto out; + return nfs_stat_to_errno(status); +} + +/* + * 3.3.18 FSSTAT3res + * + * struct FSSTAT3resok { + * post_op_attr obj_attributes; + * size3 tbytes; + * size3 fbytes; + * size3 abytes; + * size3 tfiles; + * size3 ffiles; + * size3 afiles; + * uint32 invarsec; + * }; + * + * struct FSSTAT3resfail { + * post_op_attr obj_attributes; + * }; + * + * union FSSTAT3res switch (nfsstat3 status) { + * case NFS3_OK: + * FSSTAT3resok resok; + * default: + * FSSTAT3resfail resfail; + * }; + */ +static int decode_fsstat3resok(struct xdr_stream *xdr, + struct nfs_fsstat *result) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, 8 * 6 + 4); + if (unlikely(!p)) + return -EIO; + p = xdr_decode_size3(p, &result->tbytes); + p = xdr_decode_size3(p, &result->fbytes); + p = xdr_decode_size3(p, &result->abytes); + p = xdr_decode_size3(p, &result->tfiles); + p = xdr_decode_size3(p, &result->ffiles); + xdr_decode_size3(p, &result->afiles); + /* ignore invarsec */ + return 0; +} + +static int nfs3_xdr_dec_fsstat3res(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *data) +{ + struct nfs_fsstat *result = data; + enum nfs_stat status; + int error; + + error = decode_nfsstat3(xdr, &status); + if (unlikely(error)) + goto out; + error = decode_post_op_attr(xdr, result->fattr, rpc_rqst_userns(req)); + if (unlikely(error)) + goto out; + if (status != NFS3_OK) + goto out_status; + error = decode_fsstat3resok(xdr, result); +out: + return error; +out_status: + return nfs_stat_to_errno(status); +} + +/* + * 3.3.19 FSINFO3res + * + * struct FSINFO3resok { + * post_op_attr obj_attributes; + * uint32 rtmax; + * uint32 rtpref; + * uint32 rtmult; + * uint32 wtmax; + * uint32 wtpref; + * uint32 wtmult; + * uint32 dtpref; + * size3 maxfilesize; + * nfstime3 time_delta; + * uint32 properties; + * }; + * + * struct FSINFO3resfail { + * post_op_attr obj_attributes; + * }; + * + * union FSINFO3res switch (nfsstat3 status) { + * case NFS3_OK: + * FSINFO3resok resok; + * default: + * FSINFO3resfail resfail; + * }; + */ +static int decode_fsinfo3resok(struct xdr_stream *xdr, + struct nfs_fsinfo *result) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, 4 * 7 + 8 + 8 + 4); + if (unlikely(!p)) + return -EIO; + result->rtmax = be32_to_cpup(p++); + result->rtpref = be32_to_cpup(p++); + result->rtmult = be32_to_cpup(p++); + result->wtmax = be32_to_cpup(p++); + result->wtpref = be32_to_cpup(p++); + result->wtmult = be32_to_cpup(p++); + result->dtpref = be32_to_cpup(p++); + p = xdr_decode_size3(p, &result->maxfilesize); + xdr_decode_nfstime3(p, &result->time_delta); + + /* ignore properties */ + result->lease_time = 0; + result->change_attr_type = NFS4_CHANGE_TYPE_IS_UNDEFINED; + result->xattr_support = 0; + return 0; +} + +static int nfs3_xdr_dec_fsinfo3res(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *data) +{ + struct nfs_fsinfo *result = data; + enum nfs_stat status; + int error; + + error = decode_nfsstat3(xdr, &status); + if (unlikely(error)) + goto out; + error = decode_post_op_attr(xdr, result->fattr, rpc_rqst_userns(req)); + if (unlikely(error)) + goto out; + if (status != NFS3_OK) + goto out_status; + error = decode_fsinfo3resok(xdr, result); +out: + return error; +out_status: + return nfs_stat_to_errno(status); +} + +/* + * 3.3.20 PATHCONF3res + * + * struct PATHCONF3resok { + * post_op_attr obj_attributes; + * uint32 linkmax; + * uint32 name_max; + * bool no_trunc; + * bool chown_restricted; + * bool case_insensitive; + * bool case_preserving; + * }; + * + * struct PATHCONF3resfail { + * post_op_attr obj_attributes; + * }; + * + * union PATHCONF3res switch (nfsstat3 status) { + * case NFS3_OK: + * PATHCONF3resok resok; + * default: + * PATHCONF3resfail resfail; + * }; + */ +static int decode_pathconf3resok(struct xdr_stream *xdr, + struct nfs_pathconf *result) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, 4 * 6); + if (unlikely(!p)) + return -EIO; + result->max_link = be32_to_cpup(p++); + result->max_namelen = be32_to_cpup(p++); + p++; /* ignore no_trunc */ + p++; /* ignore chown_restricted */ + result->case_insensitive = be32_to_cpup(p++) != 0; + result->case_preserving = be32_to_cpup(p) != 0; + return 0; +} + +static int nfs3_xdr_dec_pathconf3res(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *data) +{ + struct nfs_pathconf *result = data; + enum nfs_stat status; + int error; + + error = decode_nfsstat3(xdr, &status); + if (unlikely(error)) + goto out; + error = decode_post_op_attr(xdr, result->fattr, rpc_rqst_userns(req)); + if (unlikely(error)) + goto out; + if (status != NFS3_OK) + goto out_status; + error = decode_pathconf3resok(xdr, result); +out: + return error; +out_status: + return nfs_stat_to_errno(status); +} + +/* + * 3.3.21 COMMIT3res + * + * struct COMMIT3resok { + * wcc_data file_wcc; + * writeverf3 verf; + * }; + * + * struct COMMIT3resfail { + * wcc_data file_wcc; + * }; + * + * union COMMIT3res switch (nfsstat3 status) { + * case NFS3_OK: + * COMMIT3resok resok; + * default: + * COMMIT3resfail resfail; + * }; + */ +static int nfs3_xdr_dec_commit3res(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *data) +{ + struct nfs_commitres *result = data; + struct nfs_writeverf *verf = result->verf; + enum nfs_stat status; + int error; + + error = decode_nfsstat3(xdr, &status); + if (unlikely(error)) + goto out; + error = decode_wcc_data(xdr, result->fattr, rpc_rqst_userns(req)); + if (unlikely(error)) + goto out; + result->op_status = status; + if (status != NFS3_OK) + goto out_status; + error = decode_writeverf3(xdr, &verf->verifier); + if (!error) + verf->committed = NFS_FILE_SYNC; +out: + return error; +out_status: + return nfs_stat_to_errno(status); +} + +#ifdef CONFIG_NFS_V3_ACL + +static inline int decode_getacl3resok(struct xdr_stream *xdr, + struct nfs3_getaclres *result, + struct user_namespace *userns) +{ + struct posix_acl **acl; + unsigned int *aclcnt; + size_t hdrlen; + int error; + + error = decode_post_op_attr(xdr, result->fattr, userns); + if (unlikely(error)) + goto out; + error = decode_uint32(xdr, &result->mask); + if (unlikely(error)) + goto out; + error = -EINVAL; + if (result->mask & ~(NFS_ACL|NFS_ACLCNT|NFS_DFACL|NFS_DFACLCNT)) + goto out; + + hdrlen = xdr_stream_pos(xdr); + + acl = NULL; + if (result->mask & NFS_ACL) + acl = &result->acl_access; + aclcnt = NULL; + if (result->mask & NFS_ACLCNT) + aclcnt = &result->acl_access_count; + error = nfsacl_decode(xdr->buf, hdrlen, aclcnt, acl); + if (unlikely(error <= 0)) + goto out; + + acl = NULL; + if (result->mask & NFS_DFACL) + acl = &result->acl_default; + aclcnt = NULL; + if (result->mask & NFS_DFACLCNT) + aclcnt = &result->acl_default_count; + error = nfsacl_decode(xdr->buf, hdrlen + error, aclcnt, acl); + if (unlikely(error <= 0)) + return error; + error = 0; +out: + return error; +} + +static int nfs3_xdr_dec_getacl3res(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *result) +{ + enum nfs_stat status; + int error; + + error = decode_nfsstat3(xdr, &status); + if (unlikely(error)) + goto out; + if (status != NFS3_OK) + goto out_default; + error = decode_getacl3resok(xdr, result, rpc_rqst_userns(req)); +out: + return error; +out_default: + return nfs_stat_to_errno(status); +} + +static int nfs3_xdr_dec_setacl3res(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *result) +{ + enum nfs_stat status; + int error; + + error = decode_nfsstat3(xdr, &status); + if (unlikely(error)) + goto out; + if (status != NFS3_OK) + goto out_default; + error = decode_post_op_attr(xdr, result, rpc_rqst_userns(req)); +out: + return error; +out_default: + return nfs_stat_to_errno(status); +} + +#endif /* CONFIG_NFS_V3_ACL */ + +#define PROC(proc, argtype, restype, timer) \ +[NFS3PROC_##proc] = { \ + .p_proc = NFS3PROC_##proc, \ + .p_encode = nfs3_xdr_enc_##argtype##3args, \ + .p_decode = nfs3_xdr_dec_##restype##3res, \ + .p_arglen = NFS3_##argtype##args_sz, \ + .p_replen = NFS3_##restype##res_sz, \ + .p_timer = timer, \ + .p_statidx = NFS3PROC_##proc, \ + .p_name = #proc, \ + } + +const struct rpc_procinfo nfs3_procedures[] = { + PROC(GETATTR, getattr, getattr, 1), + PROC(SETATTR, setattr, setattr, 0), + PROC(LOOKUP, lookup, lookup, 2), + PROC(ACCESS, access, access, 1), + PROC(READLINK, readlink, readlink, 3), + PROC(READ, read, read, 3), + PROC(WRITE, write, write, 4), + PROC(CREATE, create, create, 0), + PROC(MKDIR, mkdir, create, 0), + PROC(SYMLINK, symlink, create, 0), + PROC(MKNOD, mknod, create, 0), + PROC(REMOVE, remove, remove, 0), + PROC(RMDIR, lookup, setattr, 0), + PROC(RENAME, rename, rename, 0), + PROC(LINK, link, link, 0), + PROC(READDIR, readdir, readdir, 3), + PROC(READDIRPLUS, readdirplus, readdir, 3), + PROC(FSSTAT, getattr, fsstat, 0), + PROC(FSINFO, getattr, fsinfo, 0), + PROC(PATHCONF, getattr, pathconf, 0), + PROC(COMMIT, commit, commit, 5), +}; + +static unsigned int nfs_version3_counts[ARRAY_SIZE(nfs3_procedures)]; +const struct rpc_version nfs_version3 = { + .number = 3, + .nrprocs = ARRAY_SIZE(nfs3_procedures), + .procs = nfs3_procedures, + .counts = nfs_version3_counts, +}; + +#ifdef CONFIG_NFS_V3_ACL +static const struct rpc_procinfo nfs3_acl_procedures[] = { + [ACLPROC3_GETACL] = { + .p_proc = ACLPROC3_GETACL, + .p_encode = nfs3_xdr_enc_getacl3args, + .p_decode = nfs3_xdr_dec_getacl3res, + .p_arglen = ACL3_getaclargs_sz, + .p_replen = ACL3_getaclres_sz, + .p_timer = 1, + .p_name = "GETACL", + }, + [ACLPROC3_SETACL] = { + .p_proc = ACLPROC3_SETACL, + .p_encode = nfs3_xdr_enc_setacl3args, + .p_decode = nfs3_xdr_dec_setacl3res, + .p_arglen = ACL3_setaclargs_sz, + .p_replen = ACL3_setaclres_sz, + .p_timer = 0, + .p_name = "SETACL", + }, +}; + +static unsigned int nfs3_acl_counts[ARRAY_SIZE(nfs3_acl_procedures)]; +const struct rpc_version nfsacl_version3 = { + .number = 3, + .nrprocs = ARRAY_SIZE(nfs3_acl_procedures), + .procs = nfs3_acl_procedures, + .counts = nfs3_acl_counts, +}; +#endif /* CONFIG_NFS_V3_ACL */ diff --git a/fs/nfs/nfs40.h b/fs/nfs/nfs40.h new file mode 100644 index 000000000..ee09aac73 --- /dev/null +++ b/fs/nfs/nfs40.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_FS_NFS_NFS4_0_H +#define __LINUX_FS_NFS_NFS4_0_H + + +/* nfs40client.c */ +void nfs40_shutdown_client(struct nfs_client *); +int nfs40_init_client(struct nfs_client *); +void nfs40_handle_cb_pathdown(struct nfs_client *clp); + +/* nfs40proc.c */ +extern const struct nfs4_minor_version_ops nfs_v4_0_minor_ops; + +/* nfs40state.c */ +int nfs40_discover_server_trunking(struct nfs_client *clp, + struct nfs_client **result, + const struct cred *cred); + +#endif /* __LINUX_FS_NFS_NFS4_0_H */ diff --git a/fs/nfs/nfs40client.c b/fs/nfs/nfs40client.c new file mode 100644 index 000000000..f44d68fae --- /dev/null +++ b/fs/nfs/nfs40client.c @@ -0,0 +1,245 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include "nfs4_fs.h" +#include "nfs4session.h" +#include "callback.h" +#include "delegation.h" +#include "internal.h" +#include "netns.h" +#include "nfs40.h" + +#define NFSDBG_FACILITY NFSDBG_CLIENT + +/* + * SETCLIENTID just did a callback update with the callback ident in + * "drop," but server trunking discovery claims "drop" and "keep" are + * actually the same server. Swap the callback IDs so that "keep" + * will continue to use the callback ident the server now knows about, + * and so that "keep"'s original callback ident is destroyed when + * "drop" is freed. + */ +static void nfs4_swap_callback_idents(struct nfs_client *keep, + struct nfs_client *drop) +{ + struct nfs_net *nn = net_generic(keep->cl_net, nfs_net_id); + unsigned int save = keep->cl_cb_ident; + + if (keep->cl_cb_ident == drop->cl_cb_ident) + return; + + dprintk("%s: keeping callback ident %u and dropping ident %u\n", + __func__, keep->cl_cb_ident, drop->cl_cb_ident); + + spin_lock(&nn->nfs_client_lock); + + idr_replace(&nn->cb_ident_idr, keep, drop->cl_cb_ident); + keep->cl_cb_ident = drop->cl_cb_ident; + + idr_replace(&nn->cb_ident_idr, drop, save); + drop->cl_cb_ident = save; + + spin_unlock(&nn->nfs_client_lock); +} + +static bool nfs4_same_verifier(nfs4_verifier *v1, nfs4_verifier *v2) +{ + return memcmp(v1->data, v2->data, sizeof(v1->data)) == 0; +} + +void nfs40_shutdown_client(struct nfs_client *clp) +{ + if (clp->cl_slot_tbl) { + nfs4_shutdown_slot_table(clp->cl_slot_tbl); + kfree(clp->cl_slot_tbl); + } +} + +/** + * nfs40_init_client - nfs_client initialization tasks for NFSv4.0 + * @clp: nfs_client to initialize + * + * Returns zero on success, or a negative errno if some error occurred. + */ +int nfs40_init_client(struct nfs_client *clp) +{ + struct nfs4_slot_table *tbl; + int ret; + + tbl = kzalloc_obj(*tbl, GFP_NOFS); + if (tbl == NULL) + return -ENOMEM; + + ret = nfs4_setup_slot_table(tbl, NFS4_MAX_SLOT_TABLE, + "NFSv4.0 transport Slot table"); + if (ret) { + nfs4_shutdown_slot_table(tbl); + kfree(tbl); + return ret; + } + + clp->cl_slot_tbl = tbl; + return 0; +} + +/* + * nfs40_handle_cb_pathdown - return all delegations after NFS4ERR_CB_PATH_DOWN + * @clp: client to process + * + * Set the NFS4CLNT_LEASE_EXPIRED state in order to force a + * resend of the SETCLIENTID and hence re-establish the + * callback channel. Then return all existing delegations. + */ +void nfs40_handle_cb_pathdown(struct nfs_client *clp) +{ + set_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state); + nfs_expire_all_delegations(clp); + dprintk("%s: handling CB_PATHDOWN recovery for server %s\n", __func__, + clp->cl_hostname); +} + +void nfs4_schedule_path_down_recovery(struct nfs_client *clp) +{ + nfs40_handle_cb_pathdown(clp); + nfs4_schedule_state_manager(clp); +} + +/** + * nfs40_walk_client_list - Find server that recognizes a client ID + * + * @new: nfs_client with client ID to test + * @result: OUT: found nfs_client, or new + * @cred: credential to use for trunking test + * + * Returns zero, a negative errno, or a negative NFS4ERR status. + * If zero is returned, an nfs_client pointer is planted in "result." + * + * NB: nfs40_walk_client_list() relies on the new nfs_client being + * the last nfs_client on the list. + */ +static int nfs40_walk_client_list(struct nfs_client *new, + struct nfs_client **result, + const struct cred *cred) +{ + struct nfs_net *nn = net_generic(new->cl_net, nfs_net_id); + struct nfs_client *pos, *prev = NULL; + struct nfs4_setclientid_res clid = { + .clientid = new->cl_clientid, + .confirm = new->cl_confirm, + }; + int status = -NFS4ERR_STALE_CLIENTID; + + spin_lock(&nn->nfs_client_lock); + list_for_each_entry(pos, &nn->nfs_client_list, cl_share_link) { + + if (pos == new) + goto found; + + status = nfs4_match_client(pos, new, &prev, nn); + if (status < 0) + goto out_unlock; + if (status != 0) + continue; + /* + * We just sent a new SETCLIENTID, which should have + * caused the server to return a new cl_confirm. So if + * cl_confirm is the same, then this is a different + * server that just returned the same cl_confirm by + * coincidence: + */ + if ((new != pos) && nfs4_same_verifier(&pos->cl_confirm, + &new->cl_confirm)) + continue; + /* + * But if the cl_confirm's are different, then the only + * way that a SETCLIENTID_CONFIRM to pos can succeed is + * if new and pos point to the same server: + */ +found: + refcount_inc(&pos->cl_count); + spin_unlock(&nn->nfs_client_lock); + + nfs_put_client(prev); + prev = pos; + + status = nfs4_proc_setclientid_confirm(pos, &clid, cred); + switch (status) { + case -NFS4ERR_STALE_CLIENTID: + break; + case 0: + nfs4_swap_callback_idents(pos, new); + pos->cl_confirm = new->cl_confirm; + nfs_mark_client_ready(pos, NFS_CS_READY); + + prev = NULL; + *result = pos; + goto out; + case -ERESTARTSYS: + case -ETIMEDOUT: + /* The callback path may have been inadvertently + * changed. Schedule recovery! + */ + nfs4_schedule_path_down_recovery(pos); + goto out; + default: + goto out; + } + + spin_lock(&nn->nfs_client_lock); + } +out_unlock: + spin_unlock(&nn->nfs_client_lock); + + /* No match found. The server lost our clientid */ +out: + nfs_put_client(prev); + return status; +} + +/** + * nfs40_discover_server_trunking - Detect server IP address trunking (mv0) + * + * @clp: nfs_client under test + * @result: OUT: found nfs_client, or clp + * @cred: credential to use for trunking test + * + * Returns zero, a negative errno, or a negative NFS4ERR status. + * If zero is returned, an nfs_client pointer is planted in + * "result". + * + * Note: The returned client may not yet be marked ready. + */ +int nfs40_discover_server_trunking(struct nfs_client *clp, + struct nfs_client **result, + const struct cred *cred) +{ + struct nfs4_setclientid_res clid = { + .clientid = clp->cl_clientid, + .confirm = clp->cl_confirm, + }; + struct nfs_net *nn = net_generic(clp->cl_net, nfs_net_id); + unsigned short port; + int status; + + port = nn->nfs_callback_tcpport; + if (clp->cl_addr.ss_family == AF_INET6) + port = nn->nfs_callback_tcpport6; + + status = nfs4_proc_setclientid(clp, NFS4_CALLBACK, port, cred, &clid); + if (status != 0) + goto out; + clp->cl_clientid = clid.clientid; + clp->cl_confirm = clid.confirm; + + status = nfs40_walk_client_list(clp, result, cred); + if (status == 0) { + /* Sustain the lease, even if it's empty. If the clientid4 + * goes stale it's of no use for trunking discovery. */ + nfs4_schedule_state_renewal(*result); + + /* If the client state need to recover, do it. */ + if (clp->cl_state) + nfs4_schedule_state_manager(clp); + } +out: + return status; +} diff --git a/fs/nfs/nfs40proc.c b/fs/nfs/nfs40proc.c new file mode 100644 index 000000000..b07f2d6cc --- /dev/null +++ b/fs/nfs/nfs40proc.c @@ -0,0 +1,395 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include +#include +#include +#include "internal.h" +#include "nfs4_fs.h" +#include "nfs40.h" +#include "nfs4session.h" +#include "nfs4trace.h" + +static void nfs40_call_sync_prepare(struct rpc_task *task, void *calldata) +{ + struct nfs4_call_sync_data *data = calldata; + nfs4_setup_sequence(data->seq_server->nfs_client, + data->seq_args, data->seq_res, task); +} + +static void nfs40_call_sync_done(struct rpc_task *task, void *calldata) +{ + struct nfs4_call_sync_data *data = calldata; + nfs4_sequence_done(task, data->seq_res); +} + +static void nfs40_sequence_free_slot(struct nfs4_sequence_res *res) +{ + struct nfs4_slot *slot = res->sr_slot; + struct nfs4_slot_table *tbl; + + tbl = slot->table; + spin_lock(&tbl->slot_tbl_lock); + if (!nfs41_wake_and_assign_slot(tbl, slot)) + nfs4_free_slot(tbl, slot); + spin_unlock(&tbl->slot_tbl_lock); + + res->sr_slot = NULL; +} + +static int nfs40_sequence_done(struct rpc_task *task, + struct nfs4_sequence_res *res) +{ + if (res->sr_slot != NULL) + nfs40_sequence_free_slot(res); + return 1; +} + +static void nfs40_clear_delegation_stateid(struct nfs4_state *state) +{ + if (rcu_access_pointer(NFS_I(state->inode)->delegation) != NULL) + nfs_finish_clear_delegation_stateid(state, NULL); +} + +static int nfs40_open_expired(struct nfs4_state_owner *sp, struct nfs4_state *state) +{ + /* NFSv4.0 doesn't allow for delegation recovery on open expire */ + nfs40_clear_delegation_stateid(state); + nfs_state_clear_open_state_flags(state); + return nfs4_open_expired(sp, state); +} + +struct nfs4_renewdata { + struct nfs_client *client; + unsigned long timestamp; +}; + +/* + * nfs4_proc_async_renew(): This is not one of the nfs_rpc_ops; it is a special + * standalone procedure for queueing an asynchronous RENEW. + */ +static void nfs4_renew_release(void *calldata) +{ + struct nfs4_renewdata *data = calldata; + struct nfs_client *clp = data->client; + + if (refcount_read(&clp->cl_count) > 1) + nfs4_schedule_state_renewal(clp); + nfs_put_client(clp); + kfree(data); +} + +static void nfs4_renew_done(struct rpc_task *task, void *calldata) +{ + struct nfs4_renewdata *data = calldata; + struct nfs_client *clp = data->client; + unsigned long timestamp = data->timestamp; + + trace_nfs4_renew_async(clp, task->tk_status); + switch (task->tk_status) { + case 0: + break; + case -NFS4ERR_LEASE_MOVED: + nfs4_schedule_lease_moved_recovery(clp); + break; + default: + /* Unless we're shutting down, schedule state recovery! */ + if (test_bit(NFS_CS_RENEWD, &clp->cl_res_state) == 0) + return; + if (task->tk_status != NFS4ERR_CB_PATH_DOWN) { + nfs4_schedule_lease_recovery(clp); + return; + } + nfs4_schedule_path_down_recovery(clp); + } + do_renew_lease(clp, timestamp); +} + +static const struct rpc_call_ops nfs4_renew_ops = { + .rpc_call_done = nfs4_renew_done, + .rpc_release = nfs4_renew_release, +}; + +static int nfs4_proc_async_renew(struct nfs_client *clp, const struct cred *cred, unsigned renew_flags) +{ + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_RENEW], + .rpc_argp = clp, + .rpc_cred = cred, + }; + struct nfs4_renewdata *data; + + if (renew_flags == 0) + return 0; + if (!refcount_inc_not_zero(&clp->cl_count)) + return -EIO; + data = kmalloc_obj(*data, GFP_NOFS); + if (data == NULL) { + nfs_put_client(clp); + return -ENOMEM; + } + data->client = clp; + data->timestamp = jiffies; + return rpc_call_async(clp->cl_rpcclient, &msg, RPC_TASK_TIMEOUT, + &nfs4_renew_ops, data); +} + +static int nfs4_proc_renew(struct nfs_client *clp, const struct cred *cred) +{ + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_RENEW], + .rpc_argp = clp, + .rpc_cred = cred, + }; + unsigned long now = jiffies; + int status; + + status = rpc_call_sync(clp->cl_rpcclient, &msg, RPC_TASK_TIMEOUT); + if (status < 0) + return status; + do_renew_lease(clp, now); + return 0; +} + +static int nfs40_test_and_free_expired_stateid(struct nfs_server *server, + nfs4_stateid *stateid, + const struct cred *cred) +{ + return -NFS4ERR_BAD_STATEID; +} + +/* + * This operation also signals the server that this client is + * performing migration recovery. The server can stop returning + * NFS4ERR_LEASE_MOVED to this client. A RENEW operation is + * appended to this compound to identify the client ID which is + * performing recovery. + */ +static int _nfs40_proc_get_locations(struct nfs_server *server, + struct nfs_fh *fhandle, + struct nfs4_fs_locations *locations, + struct page *page, const struct cred *cred) +{ + struct rpc_clnt *clnt = server->client; + struct nfs_client *clp = server->nfs_client; + u32 bitmask[2] = { + [0] = FATTR4_WORD0_FSID | FATTR4_WORD0_FS_LOCATIONS, + }; + struct nfs4_fs_locations_arg args = { + .clientid = clp->cl_clientid, + .fh = fhandle, + .page = page, + .bitmask = bitmask, + .migration = 1, /* skip LOOKUP */ + .renew = 1, /* append RENEW */ + }; + struct nfs4_fs_locations_res res = { + .fs_locations = locations, + .migration = 1, + .renew = 1, + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_FS_LOCATIONS], + .rpc_argp = &args, + .rpc_resp = &res, + .rpc_cred = cred, + }; + unsigned long now = jiffies; + int status; + + nfs_fattr_init(locations->fattr); + locations->server = server; + locations->nlocations = 0; + + nfs4_init_sequence(clp, &args.seq_args, &res.seq_res, 0, 1); + status = nfs4_call_sync_sequence(clnt, server, &msg, + &args.seq_args, &res.seq_res); + if (status) + return status; + + renew_lease(server, now); + return 0; +} + +/* + * This operation also signals the server that this client is + * performing "lease moved" recovery. The server can stop + * returning NFS4ERR_LEASE_MOVED to this client. A RENEW operation + * is appended to this compound to identify the client ID which is + * performing recovery. + */ +static int _nfs40_proc_fsid_present(struct inode *inode, const struct cred *cred) +{ + struct nfs_server *server = NFS_SERVER(inode); + struct nfs_client *clp = NFS_SERVER(inode)->nfs_client; + struct rpc_clnt *clnt = server->client; + struct nfs4_fsid_present_arg args = { + .fh = NFS_FH(inode), + .clientid = clp->cl_clientid, + .renew = 1, /* append RENEW */ + }; + struct nfs4_fsid_present_res res = { + .renew = 1, + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_FSID_PRESENT], + .rpc_argp = &args, + .rpc_resp = &res, + .rpc_cred = cred, + }; + unsigned long now = jiffies; + int status; + + res.fh = nfs_alloc_fhandle(); + if (res.fh == NULL) + return -ENOMEM; + + nfs4_init_sequence(clp, &args.seq_args, &res.seq_res, 0, 1); + status = nfs4_call_sync_sequence(clnt, server, &msg, + &args.seq_args, &res.seq_res); + nfs_free_fhandle(res.fh); + if (status) + return status; + + do_renew_lease(clp, now); + return 0; +} + +struct nfs_release_lockowner_data { + struct nfs4_lock_state *lsp; + struct nfs_server *server; + struct nfs_release_lockowner_args args; + struct nfs_release_lockowner_res res; + unsigned long timestamp; +}; + +static void nfs4_release_lockowner_prepare(struct rpc_task *task, void *calldata) +{ + struct nfs_release_lockowner_data *data = calldata; + struct nfs_server *server = data->server; + nfs4_setup_sequence(server->nfs_client, &data->args.seq_args, + &data->res.seq_res, task); + data->args.lock_owner.clientid = server->nfs_client->cl_clientid; + data->timestamp = jiffies; +} + +static void nfs4_release_lockowner_done(struct rpc_task *task, void *calldata) +{ + struct nfs_release_lockowner_data *data = calldata; + struct nfs_server *server = data->server; + + nfs40_sequence_done(task, &data->res.seq_res); + + switch (task->tk_status) { + case 0: + renew_lease(server, data->timestamp); + break; + case -NFS4ERR_STALE_CLIENTID: + case -NFS4ERR_EXPIRED: + nfs4_schedule_lease_recovery(server->nfs_client); + break; + case -NFS4ERR_LEASE_MOVED: + case -NFS4ERR_DELAY: + if (nfs4_async_handle_error(task, server, + NULL, NULL) == -EAGAIN) + rpc_restart_call_prepare(task); + } +} + +static void nfs4_release_lockowner_release(void *calldata) +{ + struct nfs_release_lockowner_data *data = calldata; + nfs4_free_lock_state(data->server, data->lsp); + kfree(calldata); +} + +static const struct rpc_call_ops nfs4_release_lockowner_ops = { + .rpc_call_prepare = nfs4_release_lockowner_prepare, + .rpc_call_done = nfs4_release_lockowner_done, + .rpc_release = nfs4_release_lockowner_release, +}; + +static void +nfs4_release_lockowner(struct nfs_server *server, struct nfs4_lock_state *lsp) +{ + struct nfs_release_lockowner_data *data; + struct nfs_client *clp = server->nfs_client; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_RELEASE_LOCKOWNER], + }; + + if (clp->cl_mvops->minor_version != 0) + return; + + data = kmalloc_obj(*data); + if (!data) + return; + data->lsp = lsp; + data->server = server; + data->args.lock_owner.clientid = clp->cl_clientid; + data->args.lock_owner.id = lsp->ls_seqid.owner_id; + data->args.lock_owner.s_dev = server->s_dev; + + msg.rpc_argp = &data->args; + msg.rpc_resp = &data->res; + nfs4_init_sequence(clp, &data->args.seq_args, &data->res.seq_res, 0, 0); + rpc_call_async(server->client, &msg, 0, &nfs4_release_lockowner_ops, data); +} + +static const struct rpc_call_ops nfs40_call_sync_ops = { + .rpc_call_prepare = nfs40_call_sync_prepare, + .rpc_call_done = nfs40_call_sync_done, +}; + +static const struct nfs4_sequence_slot_ops nfs40_sequence_slot_ops = { + .process = nfs40_sequence_done, + .done = nfs40_sequence_done, + .free_slot = nfs40_sequence_free_slot, +}; + +static const struct nfs4_state_recovery_ops nfs40_reboot_recovery_ops = { + .owner_flag_bit = NFS_OWNER_RECLAIM_REBOOT, + .state_flag_bit = NFS_STATE_RECLAIM_REBOOT, + .recover_open = nfs4_open_reclaim, + .recover_lock = nfs4_lock_reclaim, + .establish_clid = nfs4_init_clientid, + .detect_trunking = nfs40_discover_server_trunking, +}; + +static const struct nfs4_state_recovery_ops nfs40_nograce_recovery_ops = { + .owner_flag_bit = NFS_OWNER_RECLAIM_NOGRACE, + .state_flag_bit = NFS_STATE_RECLAIM_NOGRACE, + .recover_open = nfs40_open_expired, + .recover_lock = nfs4_lock_expired, + .establish_clid = nfs4_init_clientid, +}; + +static const struct nfs4_state_maintenance_ops nfs40_state_renewal_ops = { + .sched_state_renewal = nfs4_proc_async_renew, + .get_state_renewal_cred = nfs4_get_renew_cred, + .renew_lease = nfs4_proc_renew, +}; + +static const struct nfs4_mig_recovery_ops nfs40_mig_recovery_ops = { + .get_locations = _nfs40_proc_get_locations, + .fsid_present = _nfs40_proc_fsid_present, +}; + +const struct nfs4_minor_version_ops nfs_v4_0_minor_ops = { + .minor_version = 0, + .init_caps = NFS_CAP_READDIRPLUS + | NFS_CAP_ATOMIC_OPEN + | NFS_CAP_POSIX_LOCK, + .init_client = nfs40_init_client, + .shutdown_client = nfs40_shutdown_client, + .match_stateid = nfs4_match_stateid, + .find_root_sec = nfs4_find_root_sec, + .free_lock_state = nfs4_release_lockowner, + .test_and_free_expired = nfs40_test_and_free_expired_stateid, + .alloc_seqid = nfs_alloc_seqid, + .call_sync_ops = &nfs40_call_sync_ops, + .sequence_slot_ops = &nfs40_sequence_slot_ops, + .reboot_recovery_ops = &nfs40_reboot_recovery_ops, + .nograce_recovery_ops = &nfs40_nograce_recovery_ops, + .state_renewal_ops = &nfs40_state_renewal_ops, + .mig_recovery_ops = &nfs40_mig_recovery_ops, +}; diff --git a/fs/nfs/nfs42.h b/fs/nfs/nfs42.h new file mode 100644 index 000000000..aafd15a4a --- /dev/null +++ b/fs/nfs/nfs42.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2014 Anna Schumaker + */ + +#ifndef __LINUX_FS_NFS_NFS4_2_H +#define __LINUX_FS_NFS_NFS4_2_H + +#include + +/* + * FIXME: four LAYOUTSTATS calls per compound at most! Do we need to support + * more? Need to consider not to pre-alloc too much for a compound. + */ +#define PNFS_LAYOUTSTATS_MAXDEV (4) +#define READ_PLUS_SCRATCH_SIZE (16) + +/* nfs4.2proc.c */ +#ifdef CONFIG_NFS_V4_2 +int nfs42_proc_allocate(struct file *, loff_t, loff_t); +ssize_t nfs42_proc_copy(struct file *, loff_t, struct file *, loff_t, size_t, + struct nl4_server *, nfs4_stateid *, bool); +int nfs42_proc_deallocate(struct file *, loff_t, loff_t); +int nfs42_proc_zero_range(struct file *, loff_t, loff_t); +loff_t nfs42_proc_llseek(struct file *, loff_t, int); +int nfs42_proc_layoutstats_generic(struct nfs_server *, + struct nfs42_layoutstat_data *); +int nfs42_proc_clone(struct file *, struct file *, loff_t, loff_t, loff_t); +int nfs42_proc_layouterror(struct pnfs_layout_segment *lseg, + const struct nfs42_layout_error *errors, + size_t n); +int nfs42_proc_copy_notify(struct file *, struct file *, + struct nfs42_copy_notify_res *); +static inline bool nfs42_files_from_same_server(struct file *in, + struct file *out) +{ + struct nfs_client *c_in = (NFS_SERVER(file_inode(in)))->nfs_client; + struct nfs_client *c_out = (NFS_SERVER(file_inode(out)))->nfs_client; + + return nfs4_check_serverowner_major_id(c_in->cl_serverowner, + c_out->cl_serverowner); +} + +ssize_t nfs42_proc_getxattr(struct inode *inode, const char *name, + void *buf, size_t buflen); +int nfs42_proc_setxattr(struct inode *inode, const char *name, + const void *buf, size_t buflen, int flags); +ssize_t nfs42_proc_listxattrs(struct inode *inode, void *buf, + size_t buflen, u64 *cookiep, bool *eofp); +int nfs42_proc_removexattr(struct inode *inode, const char *name); + +/* + * Maximum XDR buffer size needed for a listxattr buffer of buflen size. + * + * The upper boundary is a buffer with all 1-byte sized attribute names. + * They would be 7 bytes long in the eventual buffer ("user.x\0"), and + * 8 bytes long XDR-encoded. + * + * Include the trailing eof word as well and make the result a multiple + * of 4 bytes. + */ +static inline u32 nfs42_listxattr_xdrsize(u32 buflen) +{ + u32 size = 8 * buflen / (XATTR_USER_PREFIX_LEN + 2) + 4; + + return (size + 3) & ~3; +} +#endif /* CONFIG_NFS_V4_2 */ +#endif /* __LINUX_FS_NFS_NFS4_2_H */ diff --git a/fs/nfs/nfs42proc.c b/fs/nfs/nfs42proc.c new file mode 100644 index 000000000..ab86246fc --- /dev/null +++ b/fs/nfs/nfs42proc.c @@ -0,0 +1,1687 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2014 Anna Schumaker + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "nfs4_fs.h" +#include "nfs42.h" +#include "iostat.h" +#include "pnfs.h" +#include "nfs4session.h" +#include "internal.h" +#include "delegation.h" +#include "nfs4trace.h" + +#define NFSDBG_FACILITY NFSDBG_PROC +static int nfs42_do_offload_cancel_async(struct file *dst, nfs4_stateid *std); +static int nfs42_proc_offload_status(struct file *file, nfs4_stateid *stateid, + u64 *copied); + +static void nfs42_set_netaddr(struct file *filep, struct nfs42_netaddr *naddr) +{ + struct nfs_client *clp = (NFS_SERVER(file_inode(filep)))->nfs_client; + unsigned short port = 2049; + + rcu_read_lock(); + naddr->netid_len = scnprintf(naddr->netid, + sizeof(naddr->netid), "%s", + rpc_peeraddr2str(clp->cl_rpcclient, + RPC_DISPLAY_NETID)); + naddr->addr_len = scnprintf(naddr->addr, + sizeof(naddr->addr), + "%s.%u.%u", + rpc_peeraddr2str(clp->cl_rpcclient, + RPC_DISPLAY_ADDR), + port >> 8, port & 255); + rcu_read_unlock(); +} + +static int _nfs42_proc_fallocate(struct rpc_message *msg, struct file *filep, + struct nfs_lock_context *lock, loff_t offset, loff_t len) +{ + struct inode *inode = file_inode(filep); + struct nfs_server *server = NFS_SERVER(inode); + u32 bitmask[NFS_BITMASK_SZ]; + struct nfs42_falloc_args args = { + .falloc_fh = NFS_FH(inode), + .falloc_offset = offset, + .falloc_length = len, + .falloc_bitmask = bitmask, + }; + struct nfs42_falloc_res res = { + .falloc_server = server, + }; + int status; + + msg->rpc_argp = &args; + msg->rpc_resp = &res; + + status = nfs4_set_rw_stateid(&args.falloc_stateid, lock->open_context, + lock, FMODE_WRITE); + if (status) { + if (status == -EAGAIN) + status = -NFS4ERR_BAD_STATEID; + return status; + } + + nfs4_bitmask_set(bitmask, server->cache_consistency_bitmask, inode, + NFS_INO_INVALID_BLOCKS); + + res.falloc_fattr = nfs_alloc_fattr(); + if (!res.falloc_fattr) + return -ENOMEM; + + status = nfs4_call_sync(server->client, server, msg, + &args.seq_args, &res.seq_res, 0); + if (status == 0) { + loff_t newsize = offset + len; + + spin_lock(&inode->i_lock); + if (newsize > i_size_read(inode)) + i_size_write(inode, newsize); + nfs_set_cache_invalid(inode, NFS_INO_INVALID_BLOCKS); + if (nfs_should_remove_suid(inode)) + nfs_set_cache_invalid(inode, + NFS_INO_REVAL_FORCED | + NFS_INO_INVALID_MODE); + spin_unlock(&inode->i_lock); + status = nfs_post_op_update_inode_force_wcc(inode, + res.falloc_fattr); + } + if (msg->rpc_proc == &nfs4_procedures[NFSPROC4_CLNT_ALLOCATE]) + trace_nfs4_fallocate(inode, &args, status); + else + trace_nfs4_deallocate(inode, &args, status); + kfree(res.falloc_fattr); + return status; +} + +static int nfs42_proc_fallocate(struct rpc_message *msg, struct file *filep, + loff_t offset, loff_t len) +{ + struct inode *inode = file_inode(filep); + struct nfs_server *server = NFS_SERVER(inode); + struct nfs4_exception exception = { }; + struct nfs_lock_context *lock; + int err; + + lock = nfs_get_lock_context(nfs_file_open_context(filep)); + if (IS_ERR(lock)) + return PTR_ERR(lock); + + exception.inode = inode; + exception.state = lock->open_context->state; + + err = nfs_sync_inode(inode); + if (err) + goto out; + + do { + err = _nfs42_proc_fallocate(msg, filep, lock, offset, len); + if (err == -ENOTSUPP) { + err = -EOPNOTSUPP; + break; + } + err = nfs4_handle_exception(server, err, &exception); + } while (exception.retry); +out: + nfs_put_lock_context(lock); + return err; +} + +int nfs42_proc_allocate(struct file *filep, loff_t offset, loff_t len) +{ + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_ALLOCATE], + }; + struct inode *inode = file_inode(filep); + loff_t oldsize; + int err; + + if (!nfs_server_capable(inode, NFS_CAP_ALLOCATE)) + return -EOPNOTSUPP; + + err = nfs_start_io_write(inode); + if (err) + return err; + + oldsize = i_size_read(inode); + + err = nfs42_proc_fallocate(&msg, filep, offset, len); + + if (err == 0) + nfs_truncate_last_folio(inode->i_mapping, oldsize, + offset + len); + else if (err == -EOPNOTSUPP) + NFS_SERVER(inode)->caps &= ~(NFS_CAP_ALLOCATE | + NFS_CAP_ZERO_RANGE); + + nfs_end_io_write(inode); + return err; +} + +int nfs42_proc_deallocate(struct file *filep, loff_t offset, loff_t len) +{ + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_DEALLOCATE], + }; + struct inode *inode = file_inode(filep); + int err; + + if (!nfs_server_capable(inode, NFS_CAP_DEALLOCATE)) + return -EOPNOTSUPP; + + err = nfs_start_io_write(inode); + if (err) + return err; + + err = nfs42_proc_fallocate(&msg, filep, offset, len); + if (err == 0) + truncate_pagecache_range(inode, offset, (offset + len) -1); + if (err == -EOPNOTSUPP) + NFS_SERVER(inode)->caps &= ~(NFS_CAP_DEALLOCATE | + NFS_CAP_ZERO_RANGE); + + nfs_end_io_write(inode); + return err; +} + +int nfs42_proc_zero_range(struct file *filep, loff_t offset, loff_t len) +{ + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_ZERO_RANGE], + }; + struct inode *inode = file_inode(filep); + loff_t oldsize; + int err; + + if (!nfs_server_capable(inode, NFS_CAP_ZERO_RANGE)) + return -EOPNOTSUPP; + + err = nfs_start_io_write(inode); + if (err) + return err; + + oldsize = i_size_read(inode); + err = nfs42_proc_fallocate(&msg, filep, offset, len); + if (err == 0) { + nfs_truncate_last_folio(inode->i_mapping, oldsize, + offset + len); + truncate_pagecache_range(inode, offset, (offset + len) -1); + } else if (err == -EOPNOTSUPP) + NFS_SERVER(inode)->caps &= ~NFS_CAP_ZERO_RANGE; + + nfs_end_io_write(inode); + return err; +} + +static void nfs4_copy_dequeue_callback(struct nfs_server *dst_server, + struct nfs_server *src_server, + struct nfs4_copy_state *copy) +{ + spin_lock(&dst_server->nfs_client->cl_lock); + list_del_init(©->copies); + spin_unlock(&dst_server->nfs_client->cl_lock); + if (dst_server != src_server) { + spin_lock(&src_server->nfs_client->cl_lock); + list_del_init(©->src_copies); + spin_unlock(&src_server->nfs_client->cl_lock); + } +} + +static int handle_async_copy(struct nfs42_copy_res *res, + struct nfs_server *dst_server, + struct nfs_server *src_server, + struct file *src, + struct file *dst, + nfs4_stateid *src_stateid, + bool *restart) +{ + struct nfs4_copy_state *copy, *tmp_copy = NULL, *iter; + struct nfs_open_context *dst_ctx = nfs_file_open_context(dst); + struct nfs_open_context *src_ctx = nfs_file_open_context(src); + struct nfs_client *clp = dst_server->nfs_client; + unsigned long timeout = 3 * HZ; + int status = NFS4_OK; + u64 copied; + + copy = kzalloc_obj(struct nfs4_copy_state); + if (!copy) + return -ENOMEM; + + spin_lock(&dst_server->nfs_client->cl_lock); + list_for_each_entry(iter, + &dst_server->nfs_client->pending_cb_stateids, + copies) { + if (memcmp(&res->write_res.stateid, &iter->stateid, + NFS4_STATEID_SIZE)) + continue; + tmp_copy = iter; + list_del(&iter->copies); + break; + } + if (tmp_copy) { + spin_unlock(&dst_server->nfs_client->cl_lock); + kfree(copy); + copy = tmp_copy; + goto out; + } + + memcpy(©->stateid, &res->write_res.stateid, NFS4_STATEID_SIZE); + init_completion(©->completion); + copy->parent_dst_state = dst_ctx->state; + copy->parent_src_state = src_ctx->state; + + list_add_tail(©->copies, &dst_server->ss_copies); + spin_unlock(&dst_server->nfs_client->cl_lock); + + if (dst_server != src_server) { + spin_lock(&src_server->nfs_client->cl_lock); + list_add_tail(©->src_copies, &src_server->ss_src_copies); + spin_unlock(&src_server->nfs_client->cl_lock); + } + +wait: + status = wait_for_completion_interruptible_timeout(©->completion, + timeout); + if (!status) + goto timeout; + nfs4_copy_dequeue_callback(dst_server, src_server, copy); + if (status == -ERESTARTSYS) { + goto out_cancel; + } else if (copy->flags || copy->error == NFS4ERR_PARTNER_NO_AUTH) { + status = -EAGAIN; + *restart = true; + goto out_cancel; + } +out: + res->write_res.count = copy->count; + /* Copy out the updated write verifier provided by CB_OFFLOAD. */ + memcpy(&res->write_res.verifier, ©->verf, sizeof(copy->verf)); + status = -copy->error; + +out_free: + kfree(copy); + return status; +out_cancel: + nfs42_do_offload_cancel_async(dst, ©->stateid); + if (!nfs42_files_from_same_server(src, dst)) + nfs42_do_offload_cancel_async(src, src_stateid); + goto out_free; +timeout: + timeout <<= 1; + if (timeout > (clp->cl_lease_time >> 1)) + timeout = clp->cl_lease_time >> 1; + status = nfs42_proc_offload_status(dst, ©->stateid, &copied); + if (status == -EINPROGRESS) + goto wait; + nfs4_copy_dequeue_callback(dst_server, src_server, copy); + switch (status) { + case 0: + /* The server recognized the copy stateid, so it hasn't + * rebooted. Don't overwrite the verifier returned in the + * COPY result. */ + res->write_res.count = copied; + goto out_free; + case -EREMOTEIO: + /* COPY operation failed on the server. */ + status = -EOPNOTSUPP; + res->write_res.count = copied; + goto out_free; + case -EBADF: + /* Server did not recognize the copy stateid. It has + * probably restarted and lost the plot. */ + res->write_res.count = 0; + status = -EOPNOTSUPP; + break; + case -EOPNOTSUPP: + /* RFC 7862 REQUIREs server to support OFFLOAD_STATUS when + * it has signed up for an async COPY, so server is not + * spec-compliant. */ + res->write_res.count = 0; + } + goto out_free; +} + +static int process_copy_commit(struct file *dst, loff_t pos_dst, + struct nfs42_copy_res *res) +{ + struct nfs_commitres cres; + int status = -ENOMEM; + + cres.verf = kzalloc_obj(struct nfs_writeverf); + if (!cres.verf) + goto out; + + status = nfs4_proc_commit(dst, pos_dst, res->write_res.count, &cres); + if (status) + goto out_free; + if (nfs_write_verifier_cmp(&res->write_res.verifier.verifier, + &cres.verf->verifier)) { + dprintk("commit verf differs from copy verf\n"); + status = -EAGAIN; + } +out_free: + kfree(cres.verf); +out: + return status; +} + +/** + * nfs42_copy_dest_done - perform inode cache updates after clone/copy offload + * @file: pointer to destination file + * @pos: destination offset + * @len: copy length + * @oldsize: length of the file prior to clone/copy + * + * Punch a hole in the inode page cache, so that the NFS client will + * know to retrieve new data. + * Update the file size if necessary, and then mark the inode as having + * invalid cached values for change attribute, ctime, mtime and space used. + */ +static void nfs42_copy_dest_done(struct file *file, loff_t pos, loff_t len, + loff_t oldsize) +{ + struct inode *inode = file_inode(file); + struct address_space *mapping = file->f_mapping; + loff_t newsize = pos + len; + loff_t end = newsize - 1; + + nfs_truncate_last_folio(mapping, oldsize, pos); + WARN_ON_ONCE(invalidate_inode_pages2_range(mapping, pos >> PAGE_SHIFT, + end >> PAGE_SHIFT)); + + spin_lock(&inode->i_lock); + if (newsize > i_size_read(inode)) + i_size_write(inode, newsize); + nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE | + NFS_INO_INVALID_CTIME | + NFS_INO_INVALID_MTIME | + NFS_INO_INVALID_BLOCKS); + spin_unlock(&inode->i_lock); + nfs_update_delegated_mtime(inode); +} + +static ssize_t _nfs42_proc_copy(struct file *src, + struct nfs_lock_context *src_lock, + struct file *dst, + struct nfs_lock_context *dst_lock, + struct nfs42_copy_args *args, + struct nfs42_copy_res *res, + struct nl4_server *nss, + nfs4_stateid *cnr_stateid, + bool *restart) +{ + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_COPY], + .rpc_argp = args, + .rpc_resp = res, + }; + struct inode *dst_inode = file_inode(dst); + struct inode *src_inode = file_inode(src); + struct nfs_server *dst_server = NFS_SERVER(dst_inode); + struct nfs_server *src_server = NFS_SERVER(src_inode); + loff_t pos_src = args->src_pos; + loff_t pos_dst = args->dst_pos; + loff_t oldsize_dst; + size_t count = args->count; + ssize_t status; + + if (nss) { + args->cp_src = nss; + nfs4_stateid_copy(&args->src_stateid, cnr_stateid); + } else { + status = nfs4_set_rw_stateid(&args->src_stateid, + src_lock->open_context, src_lock, FMODE_READ); + if (status) { + if (status == -EAGAIN) + status = -NFS4ERR_BAD_STATEID; + return status; + } + } + status = nfs_filemap_write_and_wait_range(src->f_mapping, + pos_src, pos_src + (loff_t)count - 1); + if (status) + return status; + + status = nfs4_set_rw_stateid(&args->dst_stateid, dst_lock->open_context, + dst_lock, FMODE_WRITE); + if (status) { + if (status == -EAGAIN) + status = -NFS4ERR_BAD_STATEID; + return status; + } + + nfs_file_block_o_direct(NFS_I(dst_inode)); + status = nfs_sync_inode(dst_inode); + if (status) + return status; + + res->commit_res.verf = NULL; + if (args->sync) { + res->commit_res.verf = + kzalloc_obj(struct nfs_writeverf); + if (!res->commit_res.verf) + return -ENOMEM; + } + set_bit(NFS_CLNT_SRC_SSC_COPY_STATE, + &src_lock->open_context->state->flags); + set_bit(NFS_CLNT_DST_SSC_COPY_STATE, + &dst_lock->open_context->state->flags); + oldsize_dst = i_size_read(dst_inode); + + status = nfs4_call_sync(dst_server->client, dst_server, &msg, + &args->seq_args, &res->seq_res, 0); + trace_nfs4_copy(src_inode, dst_inode, args, res, nss, status); + if (status == -ENOTSUPP) + dst_server->caps &= ~NFS_CAP_COPY; + if (status) + goto out; + + if (args->sync && + nfs_write_verifier_cmp(&res->write_res.verifier.verifier, + &res->commit_res.verf->verifier)) { + status = -EAGAIN; + goto out; + } + + if (!res->synchronous) { + status = handle_async_copy(res, dst_server, src_server, src, + dst, &args->src_stateid, restart); + if (status) + goto out; + } + + if ((!res->synchronous || !args->sync) && + res->write_res.verifier.committed != NFS_FILE_SYNC) { + status = process_copy_commit(dst, pos_dst, res); + if (status) + goto out; + } + + nfs42_copy_dest_done(dst, pos_dst, res->write_res.count, oldsize_dst); + nfs_invalidate_atime(src_inode); + status = res->write_res.count; +out: + if (args->sync) + kfree(res->commit_res.verf); + return status; +} + +ssize_t nfs42_proc_copy(struct file *src, loff_t pos_src, + struct file *dst, loff_t pos_dst, size_t count, + struct nl4_server *nss, + nfs4_stateid *cnr_stateid, bool sync) +{ + struct nfs_server *server = NFS_SERVER(file_inode(dst)); + struct nfs_lock_context *src_lock; + struct nfs_lock_context *dst_lock; + struct nfs42_copy_args args = { + .src_fh = NFS_FH(file_inode(src)), + .src_pos = pos_src, + .dst_fh = NFS_FH(file_inode(dst)), + .dst_pos = pos_dst, + .count = count, + .sync = sync, + }; + struct nfs42_copy_res res; + struct nfs4_exception src_exception = { + .inode = file_inode(src), + .stateid = &args.src_stateid, + }; + struct nfs4_exception dst_exception = { + .inode = file_inode(dst), + .stateid = &args.dst_stateid, + }; + ssize_t err, err2; + bool restart = false; + + src_lock = nfs_get_lock_context(nfs_file_open_context(src)); + if (IS_ERR(src_lock)) + return PTR_ERR(src_lock); + + src_exception.state = src_lock->open_context->state; + + dst_lock = nfs_get_lock_context(nfs_file_open_context(dst)); + if (IS_ERR(dst_lock)) { + err = PTR_ERR(dst_lock); + goto out_put_src_lock; + } + + dst_exception.state = dst_lock->open_context->state; + + do { + inode_lock(file_inode(dst)); + err = _nfs42_proc_copy(src, src_lock, + dst, dst_lock, + &args, &res, + nss, cnr_stateid, &restart); + inode_unlock(file_inode(dst)); + + if (err >= 0) + break; + if ((err == -ENOTSUPP || + err == -NFS4ERR_OFFLOAD_DENIED) && + nfs42_files_from_same_server(src, dst)) { + err = -EOPNOTSUPP; + break; + } else if (err == -EAGAIN) { + if (!restart) { + dst_exception.retry = 1; + continue; + } + break; + } else if (err == -NFS4ERR_OFFLOAD_NO_REQS && + args.sync != res.synchronous) { + args.sync = res.synchronous; + dst_exception.retry = 1; + continue; + } else if ((err == -ESTALE || + err == -NFS4ERR_OFFLOAD_DENIED || + err == -ENOTSUPP) && + !nfs42_files_from_same_server(src, dst)) { + nfs42_do_offload_cancel_async(src, &args.src_stateid); + err = -EOPNOTSUPP; + break; + } + + err2 = nfs4_handle_exception(server, err, &src_exception); + err = nfs4_handle_exception(server, err, &dst_exception); + if (!err) + err = err2; + } while (src_exception.retry || dst_exception.retry); + + nfs_put_lock_context(dst_lock); +out_put_src_lock: + nfs_put_lock_context(src_lock); + return err; +} + +struct nfs42_offload_data { + struct nfs_server *seq_server; + struct nfs42_offload_status_args args; + struct nfs42_offload_status_res res; +}; + +static void nfs42_offload_prepare(struct rpc_task *task, void *calldata) +{ + struct nfs42_offload_data *data = calldata; + + nfs4_setup_sequence(data->seq_server->nfs_client, + &data->args.osa_seq_args, + &data->res.osr_seq_res, task); +} + +static void nfs42_offload_cancel_done(struct rpc_task *task, void *calldata) +{ + struct nfs42_offload_data *data = calldata; + + trace_nfs4_offload_cancel(&data->args, task->tk_status); + nfs41_sequence_done(task, &data->res.osr_seq_res); + if (task->tk_status && + nfs4_async_handle_error(task, data->seq_server, NULL, + NULL) == -EAGAIN) + rpc_restart_call_prepare(task); +} + +static void nfs42_offload_release(void *data) +{ + kfree(data); +} + +static const struct rpc_call_ops nfs42_offload_cancel_ops = { + .rpc_call_prepare = nfs42_offload_prepare, + .rpc_call_done = nfs42_offload_cancel_done, + .rpc_release = nfs42_offload_release, +}; + +static int nfs42_do_offload_cancel_async(struct file *dst, + nfs4_stateid *stateid) +{ + struct nfs_server *dst_server = NFS_SERVER(file_inode(dst)); + struct nfs42_offload_data *data = NULL; + struct nfs_open_context *ctx = nfs_file_open_context(dst); + struct rpc_task *task; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_OFFLOAD_CANCEL], + .rpc_cred = ctx->cred, + }; + struct rpc_task_setup task_setup_data = { + .rpc_client = dst_server->client, + .rpc_message = &msg, + .callback_ops = &nfs42_offload_cancel_ops, + .workqueue = nfsiod_workqueue, + .flags = RPC_TASK_ASYNC | RPC_TASK_MOVEABLE, + }; + int status; + + if (!(dst_server->caps & NFS_CAP_OFFLOAD_CANCEL)) + return -EOPNOTSUPP; + + data = kzalloc_obj(struct nfs42_offload_data); + if (data == NULL) + return -ENOMEM; + + data->seq_server = dst_server; + data->args.osa_src_fh = NFS_FH(file_inode(dst)); + memcpy(&data->args.osa_stateid, stateid, + sizeof(data->args.osa_stateid)); + msg.rpc_argp = &data->args; + msg.rpc_resp = &data->res; + task_setup_data.callback_data = data; + nfs4_init_sequence(dst_server->nfs_client, &data->args.osa_seq_args, + &data->res.osr_seq_res, 1, 0); + task = rpc_run_task(&task_setup_data); + if (IS_ERR(task)) + return PTR_ERR(task); + status = rpc_wait_for_completion_task(task); + if (status == -ENOTSUPP) + dst_server->caps &= ~NFS_CAP_OFFLOAD_CANCEL; + rpc_put_task(task); + return status; +} + +static int +_nfs42_proc_offload_status(struct nfs_server *server, struct file *file, + struct nfs42_offload_data *data) +{ + struct nfs_open_context *ctx = nfs_file_open_context(file); + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_OFFLOAD_STATUS], + .rpc_argp = &data->args, + .rpc_resp = &data->res, + .rpc_cred = ctx->cred, + }; + int status; + + status = nfs4_call_sync(server->client, server, &msg, + &data->args.osa_seq_args, + &data->res.osr_seq_res, 1); + trace_nfs4_offload_status(&data->args, status); + switch (status) { + case 0: + break; + + case -NFS4ERR_ADMIN_REVOKED: + case -NFS4ERR_BAD_STATEID: + case -NFS4ERR_OLD_STATEID: + /* + * Server does not recognize the COPY stateid. CB_OFFLOAD + * could have purged it, or server might have rebooted. + * Since COPY stateids don't have an associated inode, + * avoid triggering state recovery. + */ + status = -EBADF; + break; + case -NFS4ERR_NOTSUPP: + case -ENOTSUPP: + case -EOPNOTSUPP: + server->caps &= ~NFS_CAP_OFFLOAD_STATUS; + status = -EOPNOTSUPP; + break; + } + + return status; +} + +/** + * nfs42_proc_offload_status - Poll completion status of an async copy operation + * @dst: handle of file being copied into + * @stateid: copy stateid (from async COPY result) + * @copied: OUT: number of bytes copied so far + * + * Return values: + * %0: Server returned an NFS4_OK completion status + * %-EINPROGRESS: Server returned no completion status + * %-EREMOTEIO: Server returned an error completion status + * %-EBADF: Server did not recognize the copy stateid + * %-EOPNOTSUPP: Server does not support OFFLOAD_STATUS + * %-ERESTARTSYS: Wait interrupted by signal + * + * Other negative errnos indicate the client could not complete the + * request. + */ +static int +nfs42_proc_offload_status(struct file *dst, nfs4_stateid *stateid, u64 *copied) +{ + struct inode *inode = file_inode(dst); + struct nfs_server *server = NFS_SERVER(inode); + struct nfs4_exception exception = { + .inode = inode, + }; + struct nfs42_offload_data *data; + int status; + + if (!(server->caps & NFS_CAP_OFFLOAD_STATUS)) + return -EOPNOTSUPP; + + data = kzalloc_obj(*data); + if (!data) + return -ENOMEM; + data->seq_server = server; + data->args.osa_src_fh = NFS_FH(inode); + memcpy(&data->args.osa_stateid, stateid, + sizeof(data->args.osa_stateid)); + exception.stateid = &data->args.osa_stateid; + do { + status = _nfs42_proc_offload_status(server, dst, data); + if (status == -EOPNOTSUPP) + goto out; + status = nfs4_handle_exception(server, status, &exception); + } while (exception.retry); + if (status) + goto out; + + *copied = data->res.osr_count; + if (!data->res.complete_count) + status = -EINPROGRESS; + else if (data->res.osr_complete != NFS_OK) + status = -EREMOTEIO; + +out: + kfree(data); + return status; +} + +static int _nfs42_proc_copy_notify(struct file *src, struct file *dst, + struct nfs42_copy_notify_args *args, + struct nfs42_copy_notify_res *res) +{ + struct nfs_server *src_server = NFS_SERVER(file_inode(src)); + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_COPY_NOTIFY], + .rpc_argp = args, + .rpc_resp = res, + }; + int status; + struct nfs_open_context *ctx; + struct nfs_lock_context *l_ctx; + + ctx = get_nfs_open_context(nfs_file_open_context(src)); + l_ctx = nfs_get_lock_context(ctx); + if (IS_ERR(l_ctx)) { + status = PTR_ERR(l_ctx); + goto out; + } + + status = nfs4_set_rw_stateid(&args->cna_src_stateid, ctx, l_ctx, + FMODE_READ); + nfs_put_lock_context(l_ctx); + if (status) { + if (status == -EAGAIN) + status = -NFS4ERR_BAD_STATEID; + goto out; + } + + status = nfs4_call_sync(src_server->client, src_server, &msg, + &args->cna_seq_args, &res->cnr_seq_res, 0); + trace_nfs4_copy_notify(file_inode(src), args, res, status); + if (status == -ENOTSUPP) + src_server->caps &= ~NFS_CAP_COPY_NOTIFY; + +out: + put_nfs_open_context(nfs_file_open_context(src)); + return status; +} + +int nfs42_proc_copy_notify(struct file *src, struct file *dst, + struct nfs42_copy_notify_res *res) +{ + struct nfs_server *src_server = NFS_SERVER(file_inode(src)); + struct nfs42_copy_notify_args *args; + struct nfs4_exception exception = { + .inode = file_inode(src), + }; + int status; + + if (!(src_server->caps & NFS_CAP_COPY_NOTIFY)) + return -EOPNOTSUPP; + + args = kzalloc_obj(struct nfs42_copy_notify_args); + if (args == NULL) + return -ENOMEM; + + args->cna_src_fh = NFS_FH(file_inode(src)), + args->cna_dst.nl4_type = NL4_NETADDR; + nfs42_set_netaddr(dst, &args->cna_dst.u.nl4_addr); + exception.stateid = &args->cna_src_stateid; + + do { + status = _nfs42_proc_copy_notify(src, dst, args, res); + if (status == -ENOTSUPP) { + status = -EOPNOTSUPP; + goto out; + } + status = nfs4_handle_exception(src_server, status, &exception); + } while (exception.retry); + +out: + kfree(args); + return status; +} + +static loff_t _nfs42_proc_llseek(struct file *filep, + struct nfs_lock_context *lock, loff_t offset, int whence) +{ + struct inode *inode = file_inode(filep); + struct nfs42_seek_args args = { + .sa_fh = NFS_FH(inode), + .sa_offset = offset, + .sa_what = (whence == SEEK_HOLE) ? + NFS4_CONTENT_HOLE : NFS4_CONTENT_DATA, + }; + struct nfs42_seek_res res; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_SEEK], + .rpc_argp = &args, + .rpc_resp = &res, + }; + struct nfs_server *server = NFS_SERVER(inode); + int status; + + if (!nfs_server_capable(inode, NFS_CAP_SEEK)) + return -ENOTSUPP; + + status = nfs4_set_rw_stateid(&args.sa_stateid, lock->open_context, + lock, FMODE_READ); + if (status) { + if (status == -EAGAIN) + status = -NFS4ERR_BAD_STATEID; + return status; + } + + status = nfs_filemap_write_and_wait_range(inode->i_mapping, + offset, LLONG_MAX); + if (status) + return status; + + status = nfs4_call_sync(server->client, server, &msg, + &args.seq_args, &res.seq_res, 0); + trace_nfs4_llseek(inode, &args, &res, status); + if (status == -ENOTSUPP) + server->caps &= ~NFS_CAP_SEEK; + if (status) + return status; + + if (whence == SEEK_DATA && res.sr_eof) + return -NFS4ERR_NXIO; + else + return vfs_setpos(filep, res.sr_offset, inode->i_sb->s_maxbytes); +} + +loff_t nfs42_proc_llseek(struct file *filep, loff_t offset, int whence) +{ + struct nfs_server *server = NFS_SERVER(file_inode(filep)); + struct nfs4_exception exception = { }; + struct nfs_lock_context *lock; + loff_t err; + + lock = nfs_get_lock_context(nfs_file_open_context(filep)); + if (IS_ERR(lock)) + return PTR_ERR(lock); + + exception.inode = file_inode(filep); + exception.state = lock->open_context->state; + + do { + err = _nfs42_proc_llseek(filep, lock, offset, whence); + if (err >= 0) + break; + if (err == -ENOTSUPP) { + err = -EOPNOTSUPP; + break; + } + err = nfs4_handle_exception(server, err, &exception); + } while (exception.retry); + + nfs_put_lock_context(lock); + return err; +} + + +static void +nfs42_layoutstat_prepare(struct rpc_task *task, void *calldata) +{ + struct nfs42_layoutstat_data *data = calldata; + struct inode *inode = data->inode; + struct nfs_server *server = NFS_SERVER(inode); + struct pnfs_layout_hdr *lo; + + spin_lock(&inode->i_lock); + lo = NFS_I(inode)->layout; + if (!pnfs_layout_is_valid(lo)) { + spin_unlock(&inode->i_lock); + rpc_exit(task, 0); + return; + } + nfs4_stateid_copy(&data->args.stateid, &lo->plh_stateid); + spin_unlock(&inode->i_lock); + nfs4_setup_sequence(server->nfs_client, &data->args.seq_args, + &data->res.seq_res, task); +} + +static void +nfs42_layoutstat_done(struct rpc_task *task, void *calldata) +{ + struct nfs42_layoutstat_data *data = calldata; + struct inode *inode = data->inode; + struct pnfs_layout_hdr *lo; + + if (!nfs4_sequence_done(task, &data->res.seq_res)) + return; + + switch (task->tk_status) { + case 0: + return; + case -NFS4ERR_BADHANDLE: + case -ESTALE: + pnfs_destroy_layout(NFS_I(inode)); + break; + case -NFS4ERR_EXPIRED: + case -NFS4ERR_ADMIN_REVOKED: + case -NFS4ERR_DELEG_REVOKED: + case -NFS4ERR_STALE_STATEID: + case -NFS4ERR_BAD_STATEID: + spin_lock(&inode->i_lock); + lo = NFS_I(inode)->layout; + if (pnfs_layout_is_valid(lo) && + nfs4_stateid_match(&data->args.stateid, + &lo->plh_stateid)) { + LIST_HEAD(head); + + /* + * Mark the bad layout state as invalid, then retry + * with the current stateid. + */ + pnfs_mark_layout_stateid_invalid(lo, &head); + spin_unlock(&inode->i_lock); + pnfs_free_lseg_list(&head); + nfs_commit_inode(inode, 0); + } else + spin_unlock(&inode->i_lock); + break; + case -NFS4ERR_OLD_STATEID: + spin_lock(&inode->i_lock); + lo = NFS_I(inode)->layout; + if (pnfs_layout_is_valid(lo) && + nfs4_stateid_match_other(&data->args.stateid, + &lo->plh_stateid)) { + /* Do we need to delay before resending? */ + if (!nfs4_stateid_is_newer(&lo->plh_stateid, + &data->args.stateid)) + rpc_delay(task, HZ); + rpc_restart_call_prepare(task); + } + spin_unlock(&inode->i_lock); + break; + case -ENOTSUPP: + case -EOPNOTSUPP: + NFS_SERVER(inode)->caps &= ~NFS_CAP_LAYOUTSTATS; + } + + trace_nfs4_layoutstats(inode, &data->args.stateid, task->tk_status); +} + +static void +nfs42_layoutstat_release(void *calldata) +{ + struct nfs42_layoutstat_data *data = calldata; + struct nfs42_layoutstat_devinfo *devinfo = data->args.devinfo; + int i; + + for (i = 0; i < data->args.num_dev; i++) { + if (devinfo[i].ld_private.ops && devinfo[i].ld_private.ops->free) + devinfo[i].ld_private.ops->free(&devinfo[i].ld_private); + } + + pnfs_put_layout_hdr(NFS_I(data->args.inode)->layout); + smp_mb__before_atomic(); + clear_bit(NFS_INO_LAYOUTSTATS, &NFS_I(data->args.inode)->flags); + smp_mb__after_atomic(); + nfs_iput_and_deactive(data->inode); + kfree(data->args.devinfo); + kfree(data); +} + +static const struct rpc_call_ops nfs42_layoutstat_ops = { + .rpc_call_prepare = nfs42_layoutstat_prepare, + .rpc_call_done = nfs42_layoutstat_done, + .rpc_release = nfs42_layoutstat_release, +}; + +int nfs42_proc_layoutstats_generic(struct nfs_server *server, + struct nfs42_layoutstat_data *data) +{ + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_LAYOUTSTATS], + .rpc_argp = &data->args, + .rpc_resp = &data->res, + }; + struct rpc_task_setup task_setup = { + .rpc_client = server->client, + .rpc_message = &msg, + .callback_ops = &nfs42_layoutstat_ops, + .callback_data = data, + .flags = RPC_TASK_ASYNC | RPC_TASK_MOVEABLE, + }; + struct rpc_task *task; + + data->inode = nfs_igrab_and_active(data->args.inode); + if (!data->inode) { + nfs42_layoutstat_release(data); + return -EAGAIN; + } + nfs4_init_sequence(server->nfs_client, &data->args.seq_args, + &data->res.seq_res, 0, 0); + task = rpc_run_task(&task_setup); + if (IS_ERR(task)) + return PTR_ERR(task); + rpc_put_task(task); + return 0; +} + +static struct nfs42_layouterror_data * +nfs42_alloc_layouterror_data(struct pnfs_layout_segment *lseg, gfp_t gfp_flags) +{ + struct nfs42_layouterror_data *data; + struct inode *inode = lseg->pls_layout->plh_inode; + + data = kzalloc_obj(*data, gfp_flags); + if (data) { + data->args.inode = data->inode = nfs_igrab_and_active(inode); + if (data->inode) { + data->lseg = pnfs_get_lseg(lseg); + if (data->lseg) + return data; + nfs_iput_and_deactive(data->inode); + } + kfree(data); + } + return NULL; +} + +static void +nfs42_free_layouterror_data(struct nfs42_layouterror_data *data) +{ + pnfs_put_lseg(data->lseg); + nfs_iput_and_deactive(data->inode); + kfree(data); +} + +static void +nfs42_layouterror_prepare(struct rpc_task *task, void *calldata) +{ + struct nfs42_layouterror_data *data = calldata; + struct inode *inode = data->inode; + struct nfs_server *server = NFS_SERVER(inode); + struct pnfs_layout_hdr *lo = data->lseg->pls_layout; + unsigned i; + + spin_lock(&inode->i_lock); + if (!pnfs_layout_is_valid(lo)) { + spin_unlock(&inode->i_lock); + rpc_exit(task, 0); + return; + } + for (i = 0; i < data->args.num_errors; i++) + nfs4_stateid_copy(&data->args.errors[i].stateid, + &lo->plh_stateid); + spin_unlock(&inode->i_lock); + nfs4_setup_sequence(server->nfs_client, &data->args.seq_args, + &data->res.seq_res, task); +} + +static void +nfs42_layouterror_done(struct rpc_task *task, void *calldata) +{ + struct nfs42_layouterror_data *data = calldata; + struct inode *inode = data->inode; + struct pnfs_layout_hdr *lo = data->lseg->pls_layout; + + if (!nfs4_sequence_done(task, &data->res.seq_res)) + return; + + switch (task->tk_status) { + case 0: + return; + case -NFS4ERR_BADHANDLE: + case -ESTALE: + pnfs_destroy_layout(NFS_I(inode)); + break; + case -NFS4ERR_EXPIRED: + case -NFS4ERR_ADMIN_REVOKED: + case -NFS4ERR_DELEG_REVOKED: + case -NFS4ERR_STALE_STATEID: + case -NFS4ERR_BAD_STATEID: + spin_lock(&inode->i_lock); + if (pnfs_layout_is_valid(lo) && + nfs4_stateid_match(&data->args.errors[0].stateid, + &lo->plh_stateid)) { + LIST_HEAD(head); + + /* + * Mark the bad layout state as invalid, then retry + * with the current stateid. + */ + pnfs_mark_layout_stateid_invalid(lo, &head); + spin_unlock(&inode->i_lock); + pnfs_free_lseg_list(&head); + nfs_commit_inode(inode, 0); + } else + spin_unlock(&inode->i_lock); + break; + case -NFS4ERR_OLD_STATEID: + spin_lock(&inode->i_lock); + if (pnfs_layout_is_valid(lo) && + nfs4_stateid_match_other(&data->args.errors[0].stateid, + &lo->plh_stateid)) { + /* Do we need to delay before resending? */ + if (!nfs4_stateid_is_newer(&lo->plh_stateid, + &data->args.errors[0].stateid)) + rpc_delay(task, HZ); + rpc_restart_call_prepare(task); + } + spin_unlock(&inode->i_lock); + break; + case -ENOTSUPP: + case -EOPNOTSUPP: + NFS_SERVER(inode)->caps &= ~NFS_CAP_LAYOUTERROR; + } + + trace_nfs4_layouterror(inode, &data->args.errors[0].stateid, + task->tk_status); +} + +static void +nfs42_layouterror_release(void *calldata) +{ + struct nfs42_layouterror_data *data = calldata; + + nfs42_free_layouterror_data(data); +} + +static const struct rpc_call_ops nfs42_layouterror_ops = { + .rpc_call_prepare = nfs42_layouterror_prepare, + .rpc_call_done = nfs42_layouterror_done, + .rpc_release = nfs42_layouterror_release, +}; + +int nfs42_proc_layouterror(struct pnfs_layout_segment *lseg, + const struct nfs42_layout_error *errors, size_t n) +{ + struct inode *inode = lseg->pls_layout->plh_inode; + struct nfs_server *server = NFS_SERVER(inode); + struct nfs42_layouterror_data *data; + struct rpc_task *task; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_LAYOUTERROR], + }; + struct rpc_task_setup task_setup = { + .rpc_message = &msg, + .callback_ops = &nfs42_layouterror_ops, + .flags = RPC_TASK_ASYNC | RPC_TASK_MOVEABLE, + }; + unsigned int i; + + if (!nfs_server_capable(inode, NFS_CAP_LAYOUTERROR)) + return -EOPNOTSUPP; + if (n > NFS42_LAYOUTERROR_MAX) + return -EINVAL; + data = nfs42_alloc_layouterror_data(lseg, nfs_io_gfp_mask()); + if (!data) + return -ENOMEM; + for (i = 0; i < n; i++) { + data->args.errors[i] = errors[i]; + data->args.num_errors++; + data->res.num_errors++; + } + msg.rpc_argp = &data->args; + msg.rpc_resp = &data->res; + task_setup.callback_data = data; + task_setup.rpc_client = server->client; + nfs4_init_sequence(server->nfs_client, &data->args.seq_args, + &data->res.seq_res, 0, 0); + task = rpc_run_task(&task_setup); + if (IS_ERR(task)) + return PTR_ERR(task); + rpc_put_task(task); + return 0; +} +EXPORT_SYMBOL_GPL(nfs42_proc_layouterror); + +static int _nfs42_proc_clone(struct rpc_message *msg, struct file *src_f, + struct file *dst_f, struct nfs_lock_context *src_lock, + struct nfs_lock_context *dst_lock, loff_t src_offset, + loff_t dst_offset, loff_t count) +{ + struct inode *src_inode = file_inode(src_f); + struct inode *dst_inode = file_inode(dst_f); + struct nfs_server *server = NFS_SERVER(dst_inode); + __u32 dst_bitmask[NFS_BITMASK_SZ]; + struct nfs42_clone_args args = { + .src_fh = NFS_FH(src_inode), + .dst_fh = NFS_FH(dst_inode), + .src_offset = src_offset, + .dst_offset = dst_offset, + .count = count, + .dst_bitmask = dst_bitmask, + }; + struct nfs42_clone_res res = { + .server = server, + }; + loff_t oldsize_dst = i_size_read(dst_inode); + int status; + + msg->rpc_argp = &args; + msg->rpc_resp = &res; + + status = nfs4_set_rw_stateid(&args.src_stateid, src_lock->open_context, + src_lock, FMODE_READ); + if (status) { + if (status == -EAGAIN) + status = -NFS4ERR_BAD_STATEID; + return status; + } + status = nfs4_set_rw_stateid(&args.dst_stateid, dst_lock->open_context, + dst_lock, FMODE_WRITE); + if (status) { + if (status == -EAGAIN) + status = -NFS4ERR_BAD_STATEID; + return status; + } + + res.dst_fattr = nfs_alloc_fattr(); + if (!res.dst_fattr) + return -ENOMEM; + + nfs4_bitmask_set(dst_bitmask, server->cache_consistency_bitmask, + dst_inode, NFS_INO_INVALID_BLOCKS); + + status = nfs4_call_sync(server->client, server, msg, + &args.seq_args, &res.seq_res, 0); + trace_nfs4_clone(src_inode, dst_inode, &args, status); + if (status == 0) { + /* a zero-length count means clone to EOF in src */ + if (count == 0 && res.dst_fattr->valid & NFS_ATTR_FATTR_SIZE) + count = nfs_size_to_loff_t(res.dst_fattr->size) - dst_offset; + nfs42_copy_dest_done(dst_f, dst_offset, count, oldsize_dst); + status = nfs_post_op_update_inode(dst_inode, res.dst_fattr); + } + + kfree(res.dst_fattr); + return status; +} + +int nfs42_proc_clone(struct file *src_f, struct file *dst_f, + loff_t src_offset, loff_t dst_offset, loff_t count) +{ + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_CLONE], + }; + struct inode *inode = file_inode(src_f); + struct nfs_server *server = NFS_SERVER(file_inode(src_f)); + struct nfs_lock_context *src_lock; + struct nfs_lock_context *dst_lock; + struct nfs4_exception src_exception = { }; + struct nfs4_exception dst_exception = { }; + int err, err2; + + if (!nfs_server_capable(inode, NFS_CAP_CLONE)) + return -EOPNOTSUPP; + + src_lock = nfs_get_lock_context(nfs_file_open_context(src_f)); + if (IS_ERR(src_lock)) + return PTR_ERR(src_lock); + + src_exception.inode = file_inode(src_f); + src_exception.state = src_lock->open_context->state; + + dst_lock = nfs_get_lock_context(nfs_file_open_context(dst_f)); + if (IS_ERR(dst_lock)) { + err = PTR_ERR(dst_lock); + goto out_put_src_lock; + } + + dst_exception.inode = file_inode(dst_f); + dst_exception.state = dst_lock->open_context->state; + + do { + err = _nfs42_proc_clone(&msg, src_f, dst_f, src_lock, dst_lock, + src_offset, dst_offset, count); + if (err == -ENOTSUPP || err == -EOPNOTSUPP) { + NFS_SERVER(inode)->caps &= ~NFS_CAP_CLONE; + err = -EOPNOTSUPP; + break; + } + + err2 = nfs4_handle_exception(server, err, &src_exception); + err = nfs4_handle_exception(server, err, &dst_exception); + if (!err) + err = err2; + } while (src_exception.retry || dst_exception.retry); + + nfs_put_lock_context(dst_lock); +out_put_src_lock: + nfs_put_lock_context(src_lock); + return err; +} + +#define NFS4XATTR_MAXPAGES DIV_ROUND_UP(XATTR_SIZE_MAX, PAGE_SIZE) + +static int _nfs42_proc_removexattr(struct inode *inode, const char *name) +{ + struct nfs_server *server = NFS_SERVER(inode); + __u32 bitmask[NFS_BITMASK_SZ]; + struct nfs42_removexattrargs args = { + .fh = NFS_FH(inode), + .bitmask = bitmask, + .xattr_name = name, + }; + struct nfs42_removexattrres res = { + .server = server, + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_REMOVEXATTR], + .rpc_argp = &args, + .rpc_resp = &res, + }; + int ret; + unsigned long timestamp = jiffies; + + res.fattr = nfs_alloc_fattr(); + if (!res.fattr) + return -ENOMEM; + + nfs4_bitmask_set(bitmask, server->cache_consistency_bitmask, + inode, NFS_INO_INVALID_CHANGE); + + ret = nfs4_call_sync(server->client, server, &msg, &args.seq_args, + &res.seq_res, 1); + trace_nfs4_removexattr(inode, name, ret); + if (!ret) { + nfs4_update_changeattr(inode, &res.cinfo, timestamp, 0); + ret = nfs_post_op_update_inode(inode, res.fattr); + } + + kfree(res.fattr); + return ret; +} + +static int _nfs42_proc_setxattr(struct inode *inode, const char *name, + const void *buf, size_t buflen, int flags) +{ + struct nfs_server *server = NFS_SERVER(inode); + __u32 bitmask[NFS_BITMASK_SZ]; + struct page *pages[NFS4XATTR_MAXPAGES]; + struct nfs42_setxattrargs arg = { + .fh = NFS_FH(inode), + .bitmask = bitmask, + .xattr_pages = pages, + .xattr_len = buflen, + .xattr_name = name, + .xattr_flags = flags, + }; + struct nfs42_setxattrres res = { + .server = server, + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_SETXATTR], + .rpc_argp = &arg, + .rpc_resp = &res, + }; + int ret, np; + unsigned long timestamp = jiffies; + + if (buflen > server->sxasize) + return -ERANGE; + + res.fattr = nfs_alloc_fattr(); + if (!res.fattr) + return -ENOMEM; + + if (buflen > 0) { + np = nfs4_buf_to_pages_noslab(buf, buflen, arg.xattr_pages); + if (np < 0) { + ret = np; + goto out; + } + } else + np = 0; + + nfs4_bitmask_set(bitmask, server->cache_consistency_bitmask, + inode, NFS_INO_INVALID_CHANGE); + + ret = nfs4_call_sync(server->client, server, &msg, &arg.seq_args, + &res.seq_res, 1); + trace_nfs4_setxattr(inode, name, ret); + + for (; np > 0; np--) + put_page(pages[np - 1]); + + if (!ret) { + nfs4_update_changeattr(inode, &res.cinfo, timestamp, 0); + ret = nfs_post_op_update_inode(inode, res.fattr); + } + +out: + kfree(res.fattr); + return ret; +} + +static ssize_t _nfs42_proc_getxattr(struct inode *inode, const char *name, + void *buf, size_t buflen, struct page **pages, + size_t plen) +{ + struct nfs_server *server = NFS_SERVER(inode); + struct nfs42_getxattrargs arg = { + .fh = NFS_FH(inode), + .xattr_name = name, + }; + struct nfs42_getxattrres res; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_GETXATTR], + .rpc_argp = &arg, + .rpc_resp = &res, + }; + ssize_t ret; + + arg.xattr_len = plen; + arg.xattr_pages = pages; + + ret = nfs4_call_sync(server->client, server, &msg, &arg.seq_args, + &res.seq_res, 0); + trace_nfs4_getxattr(inode, name, ret); + if (ret < 0) + return ret; + + /* + * Normally, the caching is done one layer up, but for successful + * RPCS, always cache the result here, even if the caller was + * just querying the length, or if the reply was too big for + * the caller. This avoids a second RPC in the case of the + * common query-alloc-retrieve cycle for xattrs. + * + * Note that xattr_len is always capped to XATTR_SIZE_MAX. + */ + + nfs4_xattr_cache_add(inode, name, NULL, pages, res.xattr_len); + + if (buflen) { + if (res.xattr_len > buflen) + return -ERANGE; + _copy_from_pages(buf, pages, 0, res.xattr_len); + } + + return res.xattr_len; +} + +static ssize_t _nfs42_proc_listxattrs(struct inode *inode, void *buf, + size_t buflen, u64 *cookiep, bool *eofp) +{ + struct nfs_server *server = NFS_SERVER(inode); + struct page **pages; + struct nfs42_listxattrsargs arg = { + .fh = NFS_FH(inode), + .cookie = *cookiep, + }; + struct nfs42_listxattrsres res = { + .eof = false, + .xattr_buf = buf, + .xattr_len = buflen, + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_LISTXATTRS], + .rpc_argp = &arg, + .rpc_resp = &res, + }; + u32 xdrlen; + int ret, np, i; + + + ret = -ENOMEM; + res.scratch = folio_alloc(GFP_KERNEL, 0); + if (!res.scratch) + goto out; + + xdrlen = nfs42_listxattr_xdrsize(buflen); + if (xdrlen > server->lxasize) + xdrlen = server->lxasize; + np = xdrlen / PAGE_SIZE + 1; + + pages = kzalloc_objs(struct page *, np); + if (!pages) + goto out_free_scratch; + for (i = 0; i < np; i++) { + pages[i] = alloc_page(GFP_KERNEL); + if (!pages[i]) + goto out_free_pages; + } + + arg.xattr_pages = pages; + arg.count = xdrlen; + + ret = nfs4_call_sync(server->client, server, &msg, &arg.seq_args, + &res.seq_res, 0); + trace_nfs4_listxattr(inode, ret); + + if (ret >= 0) { + ret = res.copied; + *cookiep = res.cookie; + *eofp = res.eof; + } + +out_free_pages: + while (--np >= 0) { + if (pages[np]) + __free_page(pages[np]); + } + kfree(pages); +out_free_scratch: + folio_put(res.scratch); +out: + return ret; + +} + +ssize_t nfs42_proc_getxattr(struct inode *inode, const char *name, + void *buf, size_t buflen) +{ + struct nfs4_exception exception = { }; + ssize_t err, np, i; + struct page **pages; + + np = nfs_page_array_len(0, buflen ?: XATTR_SIZE_MAX); + pages = kmalloc_objs(*pages, np); + if (!pages) + return -ENOMEM; + + for (i = 0; i < np; i++) { + pages[i] = alloc_page(GFP_KERNEL); + if (!pages[i]) { + err = -ENOMEM; + goto out; + } + } + + /* + * The GETXATTR op has no length field in the call, and the + * xattr data is at the end of the reply. + * + * There is no downside in using the page-aligned length. It will + * allow receiving and caching xattrs that are too large for the + * caller but still fit in the page-rounded value. + */ + do { + err = _nfs42_proc_getxattr(inode, name, buf, buflen, + pages, np * PAGE_SIZE); + if (err >= 0) + break; + err = nfs4_handle_exception(NFS_SERVER(inode), err, + &exception); + } while (exception.retry); + +out: + while (--i >= 0) + __free_page(pages[i]); + kfree(pages); + + return err; +} + +int nfs42_proc_setxattr(struct inode *inode, const char *name, + const void *buf, size_t buflen, int flags) +{ + struct nfs4_exception exception = { }; + int err; + + do { + err = _nfs42_proc_setxattr(inode, name, buf, buflen, flags); + if (!err) + break; + err = nfs4_handle_exception(NFS_SERVER(inode), err, + &exception); + } while (exception.retry); + + return err; +} + +ssize_t nfs42_proc_listxattrs(struct inode *inode, void *buf, + size_t buflen, u64 *cookiep, bool *eofp) +{ + struct nfs4_exception exception = { }; + ssize_t err; + + do { + err = _nfs42_proc_listxattrs(inode, buf, buflen, + cookiep, eofp); + if (err >= 0) + break; + err = nfs4_handle_exception(NFS_SERVER(inode), err, + &exception); + } while (exception.retry); + + return err; +} + +int nfs42_proc_removexattr(struct inode *inode, const char *name) +{ + struct nfs4_exception exception = { }; + int err; + + do { + err = _nfs42_proc_removexattr(inode, name); + if (!err) + break; + err = nfs4_handle_exception(NFS_SERVER(inode), err, + &exception); + } while (exception.retry); + + return err; +} diff --git a/fs/nfs/nfs42xattr.c b/fs/nfs/nfs42xattr.c new file mode 100644 index 000000000..37d79400e --- /dev/null +++ b/fs/nfs/nfs42xattr.c @@ -0,0 +1,1067 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright 2019, 2020 Amazon.com, Inc. or its affiliates. All rights reserved. + * + * User extended attribute client side cache functions. + * + * Author: Frank van der Linden + */ +#include +#include +#include +#include +#include + +#include "nfs4_fs.h" +#include "internal.h" + +/* + * User extended attributes client side caching is implemented by having + * a cache structure attached to NFS inodes. This structure is allocated + * when needed, and freed when the cache is zapped. + * + * The cache structure contains as hash table of entries, and a pointer + * to a special-cased entry for the listxattr cache. + * + * Accessing and allocating / freeing the caches is done via reference + * counting. The cache entries use a similar refcounting scheme. + * + * This makes freeing a cache, both from the shrinker and from the + * zap cache path, easy. It also means that, in current use cases, + * the large majority of inodes will not waste any memory, as they + * will never have any user extended attributes assigned to them. + * + * Attribute entries are hashed in to a simple hash table. They are + * also part of an LRU. + * + * There are three shrinkers. + * + * Two shrinkers deal with the cache entries themselves: one for + * large entries (> PAGE_SIZE), and one for smaller entries. The + * shrinker for the larger entries works more aggressively than + * those for the smaller entries. + * + * The other shrinker frees the cache structures themselves. + */ + +/* + * 64 buckets is a good default. There is likely no reasonable + * workload that uses more than even 64 user extended attributes. + * You can certainly add a lot more - but you get what you ask for + * in those circumstances. + */ +#define NFS4_XATTR_HASH_SIZE 64 + +#define NFSDBG_FACILITY NFSDBG_XATTRCACHE + +struct nfs4_xattr_cache; +struct nfs4_xattr_entry; + +struct nfs4_xattr_bucket { + spinlock_t lock; + struct hlist_head hlist; + struct nfs4_xattr_cache *cache; + bool draining; +}; + +struct nfs4_xattr_cache { + struct kref ref; + struct nfs4_xattr_bucket buckets[NFS4_XATTR_HASH_SIZE]; + struct list_head lru; + struct list_head dispose; + atomic_long_t nent; + spinlock_t listxattr_lock; + struct inode *inode; + struct nfs4_xattr_entry *listxattr; +}; + +struct nfs4_xattr_entry { + struct kref ref; + struct hlist_node hnode; + struct list_head lru; + struct list_head dispose; + char *xattr_name; + void *xattr_value; + size_t xattr_size; + struct nfs4_xattr_bucket *bucket; + uint32_t flags; +}; + +#define NFS4_XATTR_ENTRY_EXTVAL 0x0001 + +/* + * LRU list of NFS inodes that have xattr caches. + */ +static struct list_lru nfs4_xattr_cache_lru; +static struct list_lru nfs4_xattr_entry_lru; +static struct list_lru nfs4_xattr_large_entry_lru; + +static struct kmem_cache *nfs4_xattr_cache_cachep; + +/* + * Hashing helper functions. + */ +static void +nfs4_xattr_hash_init(struct nfs4_xattr_cache *cache) +{ + unsigned int i; + + for (i = 0; i < NFS4_XATTR_HASH_SIZE; i++) { + INIT_HLIST_HEAD(&cache->buckets[i].hlist); + spin_lock_init(&cache->buckets[i].lock); + cache->buckets[i].cache = cache; + cache->buckets[i].draining = false; + } +} + +/* + * Locking order: + * 1. inode i_lock or bucket lock + * 2. list_lru lock (taken by list_lru_* functions) + */ + +/* + * Wrapper functions to add a cache entry to the right LRU. + */ +static bool +nfs4_xattr_entry_lru_add(struct nfs4_xattr_entry *entry) +{ + struct list_lru *lru; + + lru = (entry->flags & NFS4_XATTR_ENTRY_EXTVAL) ? + &nfs4_xattr_large_entry_lru : &nfs4_xattr_entry_lru; + + return list_lru_add_obj(lru, &entry->lru); +} + +static bool +nfs4_xattr_entry_lru_del(struct nfs4_xattr_entry *entry) +{ + struct list_lru *lru; + + lru = (entry->flags & NFS4_XATTR_ENTRY_EXTVAL) ? + &nfs4_xattr_large_entry_lru : &nfs4_xattr_entry_lru; + + return list_lru_del_obj(lru, &entry->lru); +} + +/* + * This function allocates cache entries. They are the normal + * extended attribute name/value pairs, but may also be a listxattr + * cache. Those allocations use the same entry so that they can be + * treated as one by the memory shrinker. + * + * xattr cache entries are allocated together with names. If the + * value fits in to one page with the entry structure and the name, + * it will also be part of the same allocation (kmalloc). This is + * expected to be the vast majority of cases. Larger allocations + * have a value pointer that is allocated separately by kvmalloc. + * + * Parameters: + * + * @name: Name of the extended attribute. NULL for listxattr cache + * entry. + * @value: Value of attribute, or listxattr cache. NULL if the + * value is to be copied from pages instead. + * @pages: Pages to copy the value from, if not NULL. Passed in to + * make it easier to copy the value after an RPC, even if + * the value will not be passed up to application (e.g. + * for a 'query' getxattr with NULL buffer). + * @len: Length of the value. Can be 0 for zero-length attributes. + * @value and @pages will be NULL if @len is 0. + */ +static struct nfs4_xattr_entry * +nfs4_xattr_alloc_entry(const char *name, const void *value, + struct page **pages, size_t len) +{ + struct nfs4_xattr_entry *entry; + void *valp; + char *namep; + size_t alloclen, slen; + char *buf; + uint32_t flags; + + BUILD_BUG_ON(sizeof(struct nfs4_xattr_entry) + + XATTR_NAME_MAX + 1 > PAGE_SIZE); + + alloclen = sizeof(struct nfs4_xattr_entry); + if (name != NULL) { + slen = strlen(name) + 1; + alloclen += slen; + } else + slen = 0; + + if (alloclen + len <= PAGE_SIZE) { + alloclen += len; + flags = 0; + } else { + flags = NFS4_XATTR_ENTRY_EXTVAL; + } + + buf = kmalloc(alloclen, GFP_KERNEL); + if (buf == NULL) + return NULL; + entry = (struct nfs4_xattr_entry *)buf; + + if (name != NULL) { + namep = buf + sizeof(struct nfs4_xattr_entry); + memcpy(namep, name, slen); + } else { + namep = NULL; + } + + + if (flags & NFS4_XATTR_ENTRY_EXTVAL) { + valp = kvmalloc(len, GFP_KERNEL); + if (valp == NULL) { + kfree(buf); + return NULL; + } + } else if (len != 0) { + valp = buf + sizeof(struct nfs4_xattr_entry) + slen; + } else + valp = NULL; + + if (valp != NULL) { + if (value != NULL) + memcpy(valp, value, len); + else + _copy_from_pages(valp, pages, 0, len); + } + + entry->flags = flags; + entry->xattr_value = valp; + kref_init(&entry->ref); + entry->xattr_name = namep; + entry->xattr_size = len; + entry->bucket = NULL; + INIT_LIST_HEAD(&entry->lru); + INIT_LIST_HEAD(&entry->dispose); + INIT_HLIST_NODE(&entry->hnode); + + return entry; +} + +static void +nfs4_xattr_free_entry(struct nfs4_xattr_entry *entry) +{ + if (entry->flags & NFS4_XATTR_ENTRY_EXTVAL) + kvfree(entry->xattr_value); + kfree(entry); +} + +static void +nfs4_xattr_free_entry_cb(struct kref *kref) +{ + struct nfs4_xattr_entry *entry; + + entry = container_of(kref, struct nfs4_xattr_entry, ref); + + if (WARN_ON(!list_empty(&entry->lru))) + return; + + nfs4_xattr_free_entry(entry); +} + +static void +nfs4_xattr_free_cache_cb(struct kref *kref) +{ + struct nfs4_xattr_cache *cache; + int i; + + cache = container_of(kref, struct nfs4_xattr_cache, ref); + + for (i = 0; i < NFS4_XATTR_HASH_SIZE; i++) { + if (WARN_ON(!hlist_empty(&cache->buckets[i].hlist))) + return; + cache->buckets[i].draining = false; + } + + cache->listxattr = NULL; + + kmem_cache_free(nfs4_xattr_cache_cachep, cache); + +} + +static struct nfs4_xattr_cache * +nfs4_xattr_alloc_cache(void) +{ + struct nfs4_xattr_cache *cache; + + cache = kmem_cache_alloc(nfs4_xattr_cache_cachep, GFP_KERNEL); + if (cache == NULL) + return NULL; + + kref_init(&cache->ref); + atomic_long_set(&cache->nent, 0); + + return cache; +} + +/* + * Set the listxattr cache, which is a special-cased cache entry. + * The special value ERR_PTR(-ESTALE) is used to indicate that + * the cache is being drained - this prevents a new listxattr + * cache from being added to what is now a stale cache. + */ +static int +nfs4_xattr_set_listcache(struct nfs4_xattr_cache *cache, + struct nfs4_xattr_entry *new) +{ + struct nfs4_xattr_entry *old; + int ret = 1; + + spin_lock(&cache->listxattr_lock); + + old = cache->listxattr; + + if (old == ERR_PTR(-ESTALE)) { + ret = 0; + goto out; + } + + cache->listxattr = new; + if (new != NULL && new != ERR_PTR(-ESTALE)) + nfs4_xattr_entry_lru_add(new); + + if (old != NULL) { + nfs4_xattr_entry_lru_del(old); + kref_put(&old->ref, nfs4_xattr_free_entry_cb); + } +out: + spin_unlock(&cache->listxattr_lock); + + return ret; +} + +/* + * Unlink a cache from its parent inode, clearing out an invalid + * cache. Must be called with i_lock held. + */ +static struct nfs4_xattr_cache * +nfs4_xattr_cache_unlink(struct inode *inode) +{ + struct nfs_inode *nfsi; + struct nfs4_xattr_cache *oldcache; + + nfsi = NFS_I(inode); + + oldcache = nfsi->xattr_cache; + if (oldcache != NULL) { + list_lru_del_obj(&nfs4_xattr_cache_lru, &oldcache->lru); + oldcache->inode = NULL; + } + nfsi->xattr_cache = NULL; + nfsi->cache_validity &= ~NFS_INO_INVALID_XATTR; + + return oldcache; + +} + +/* + * Discard a cache. Called by get_cache() if there was an old, + * invalid cache. Can also be called from a shrinker callback. + * + * The cache is dead, it has already been unlinked from its inode, + * and no longer appears on the cache LRU list. + * + * Mark all buckets as draining, so that no new entries are added. This + * could still happen in the unlikely, but possible case that another + * thread had grabbed a reference before it was unlinked from the inode, + * and is still holding it for an add operation. + * + * Remove all entries from the LRU lists, so that there is no longer + * any way to 'find' this cache. Then, remove the entries from the hash + * table. + * + * At that point, the cache will remain empty and can be freed when the final + * reference drops, which is very likely the kref_put at the end of + * this function, or the one called immediately afterwards in the + * shrinker callback. + */ +static void +nfs4_xattr_discard_cache(struct nfs4_xattr_cache *cache) +{ + unsigned int i; + struct nfs4_xattr_entry *entry; + struct nfs4_xattr_bucket *bucket; + struct hlist_node *n; + + nfs4_xattr_set_listcache(cache, ERR_PTR(-ESTALE)); + + for (i = 0; i < NFS4_XATTR_HASH_SIZE; i++) { + bucket = &cache->buckets[i]; + + spin_lock(&bucket->lock); + bucket->draining = true; + hlist_for_each_entry_safe(entry, n, &bucket->hlist, hnode) { + nfs4_xattr_entry_lru_del(entry); + hlist_del_init(&entry->hnode); + kref_put(&entry->ref, nfs4_xattr_free_entry_cb); + } + spin_unlock(&bucket->lock); + } + + atomic_long_set(&cache->nent, 0); + + kref_put(&cache->ref, nfs4_xattr_free_cache_cb); +} + +/* + * Get a referenced copy of the cache structure. Avoid doing allocs + * while holding i_lock. Which means that we do some optimistic allocation, + * and might have to free the result in rare cases. + * + * This function only checks the NFS_INO_INVALID_XATTR cache validity bit + * and acts accordingly, replacing the cache when needed. For the read case + * (!add), this means that the caller must make sure that the cache + * is valid before caling this function. getxattr and listxattr call + * revalidate_inode to do this. The attribute cache timeout (for the + * non-delegated case) is expected to be dealt with in the revalidate + * call. + */ + +static struct nfs4_xattr_cache * +nfs4_xattr_get_cache(struct inode *inode, int add) +{ + struct nfs_inode *nfsi; + struct nfs4_xattr_cache *cache, *oldcache, *newcache; + + nfsi = NFS_I(inode); + + cache = oldcache = NULL; + + spin_lock(&inode->i_lock); + + if (nfsi->cache_validity & NFS_INO_INVALID_XATTR) + oldcache = nfs4_xattr_cache_unlink(inode); + else + cache = nfsi->xattr_cache; + + if (cache != NULL) + kref_get(&cache->ref); + + spin_unlock(&inode->i_lock); + + if (add && cache == NULL) { + newcache = NULL; + + cache = nfs4_xattr_alloc_cache(); + if (cache == NULL) + goto out; + + spin_lock(&inode->i_lock); + if (nfsi->cache_validity & NFS_INO_INVALID_XATTR) { + /* + * The cache was invalidated again. Give up, + * since what we want to enter is now likely + * outdated anyway. + */ + spin_unlock(&inode->i_lock); + kref_put(&cache->ref, nfs4_xattr_free_cache_cb); + cache = NULL; + goto out; + } + + /* + * Check if someone beat us to it. + */ + if (nfsi->xattr_cache != NULL) { + newcache = nfsi->xattr_cache; + kref_get(&newcache->ref); + } else { + kref_get(&cache->ref); + nfsi->xattr_cache = cache; + cache->inode = inode; + list_lru_add_obj(&nfs4_xattr_cache_lru, &cache->lru); + } + + spin_unlock(&inode->i_lock); + + /* + * If there was a race, throw away the cache we just + * allocated, and use the new one allocated by someone + * else. + */ + if (newcache != NULL) { + kref_put(&cache->ref, nfs4_xattr_free_cache_cb); + cache = newcache; + } + } + +out: + /* + * Discard the now orphaned old cache. + */ + if (oldcache != NULL) + nfs4_xattr_discard_cache(oldcache); + + return cache; +} + +static inline struct nfs4_xattr_bucket * +nfs4_xattr_hash_bucket(struct nfs4_xattr_cache *cache, const char *name) +{ + return &cache->buckets[jhash(name, strlen(name), 0) & + (ARRAY_SIZE(cache->buckets) - 1)]; +} + +static struct nfs4_xattr_entry * +nfs4_xattr_get_entry(struct nfs4_xattr_bucket *bucket, const char *name) +{ + struct nfs4_xattr_entry *entry; + + entry = NULL; + + hlist_for_each_entry(entry, &bucket->hlist, hnode) { + if (!strcmp(entry->xattr_name, name)) + break; + } + + return entry; +} + +static int +nfs4_xattr_hash_add(struct nfs4_xattr_cache *cache, + struct nfs4_xattr_entry *entry) +{ + struct nfs4_xattr_bucket *bucket; + struct nfs4_xattr_entry *oldentry = NULL; + int ret = 1; + + bucket = nfs4_xattr_hash_bucket(cache, entry->xattr_name); + entry->bucket = bucket; + + spin_lock(&bucket->lock); + + if (bucket->draining) { + ret = 0; + goto out; + } + + oldentry = nfs4_xattr_get_entry(bucket, entry->xattr_name); + if (oldentry != NULL) { + hlist_del_init(&oldentry->hnode); + nfs4_xattr_entry_lru_del(oldentry); + } else { + atomic_long_inc(&cache->nent); + } + + hlist_add_head(&entry->hnode, &bucket->hlist); + nfs4_xattr_entry_lru_add(entry); + +out: + spin_unlock(&bucket->lock); + + if (oldentry != NULL) + kref_put(&oldentry->ref, nfs4_xattr_free_entry_cb); + + return ret; +} + +static void +nfs4_xattr_hash_remove(struct nfs4_xattr_cache *cache, const char *name) +{ + struct nfs4_xattr_bucket *bucket; + struct nfs4_xattr_entry *entry; + + bucket = nfs4_xattr_hash_bucket(cache, name); + + spin_lock(&bucket->lock); + + entry = nfs4_xattr_get_entry(bucket, name); + if (entry != NULL) { + hlist_del_init(&entry->hnode); + nfs4_xattr_entry_lru_del(entry); + atomic_long_dec(&cache->nent); + } + + spin_unlock(&bucket->lock); + + if (entry != NULL) + kref_put(&entry->ref, nfs4_xattr_free_entry_cb); +} + +static struct nfs4_xattr_entry * +nfs4_xattr_hash_find(struct nfs4_xattr_cache *cache, const char *name) +{ + struct nfs4_xattr_bucket *bucket; + struct nfs4_xattr_entry *entry; + + bucket = nfs4_xattr_hash_bucket(cache, name); + + spin_lock(&bucket->lock); + + entry = nfs4_xattr_get_entry(bucket, name); + if (entry != NULL) + kref_get(&entry->ref); + + spin_unlock(&bucket->lock); + + return entry; +} + +/* + * Entry point to retrieve an entry from the cache. + */ +ssize_t nfs4_xattr_cache_get(struct inode *inode, const char *name, char *buf, + ssize_t buflen) +{ + struct nfs4_xattr_cache *cache; + struct nfs4_xattr_entry *entry; + ssize_t ret; + + cache = nfs4_xattr_get_cache(inode, 0); + if (cache == NULL) + return -ENOENT; + + ret = 0; + entry = nfs4_xattr_hash_find(cache, name); + + if (entry != NULL) { + dprintk("%s: cache hit '%s', len %lu\n", __func__, + entry->xattr_name, (unsigned long)entry->xattr_size); + if (buflen == 0) { + /* Length probe only */ + ret = entry->xattr_size; + } else if (buflen < entry->xattr_size) + ret = -ERANGE; + else { + memcpy(buf, entry->xattr_value, entry->xattr_size); + ret = entry->xattr_size; + } + kref_put(&entry->ref, nfs4_xattr_free_entry_cb); + } else { + dprintk("%s: cache miss '%s'\n", __func__, name); + ret = -ENOENT; + } + + kref_put(&cache->ref, nfs4_xattr_free_cache_cb); + + return ret; +} + +/* + * Retrieve a cached list of xattrs from the cache. + */ +ssize_t nfs4_xattr_cache_list(struct inode *inode, char *buf, ssize_t buflen) +{ + struct nfs4_xattr_cache *cache; + struct nfs4_xattr_entry *entry; + ssize_t ret; + + cache = nfs4_xattr_get_cache(inode, 0); + if (cache == NULL) + return -ENOENT; + + spin_lock(&cache->listxattr_lock); + + entry = cache->listxattr; + + if (entry != NULL && entry != ERR_PTR(-ESTALE)) { + if (buflen == 0) { + /* Length probe only */ + ret = entry->xattr_size; + } else if (entry->xattr_size > buflen) + ret = -ERANGE; + else { + memcpy(buf, entry->xattr_value, entry->xattr_size); + ret = entry->xattr_size; + } + } else { + ret = -ENOENT; + } + + spin_unlock(&cache->listxattr_lock); + + kref_put(&cache->ref, nfs4_xattr_free_cache_cb); + + return ret; +} + +/* + * Add an xattr to the cache. + * + * This also invalidates the xattr list cache. + */ +void nfs4_xattr_cache_add(struct inode *inode, const char *name, + const char *buf, struct page **pages, ssize_t buflen) +{ + struct nfs4_xattr_cache *cache; + struct nfs4_xattr_entry *entry; + + dprintk("%s: add '%s' len %lu\n", __func__, + name, (unsigned long)buflen); + + cache = nfs4_xattr_get_cache(inode, 1); + if (cache == NULL) + return; + + entry = nfs4_xattr_alloc_entry(name, buf, pages, buflen); + if (entry == NULL) + goto out; + + (void)nfs4_xattr_set_listcache(cache, NULL); + + if (!nfs4_xattr_hash_add(cache, entry)) + kref_put(&entry->ref, nfs4_xattr_free_entry_cb); + +out: + kref_put(&cache->ref, nfs4_xattr_free_cache_cb); +} + + +/* + * Remove an xattr from the cache. + * + * This also invalidates the xattr list cache. + */ +void nfs4_xattr_cache_remove(struct inode *inode, const char *name) +{ + struct nfs4_xattr_cache *cache; + + dprintk("%s: remove '%s'\n", __func__, name); + + cache = nfs4_xattr_get_cache(inode, 0); + if (cache == NULL) + return; + + (void)nfs4_xattr_set_listcache(cache, NULL); + nfs4_xattr_hash_remove(cache, name); + + kref_put(&cache->ref, nfs4_xattr_free_cache_cb); +} + +/* + * Cache listxattr output, replacing any possible old one. + */ +void nfs4_xattr_cache_set_list(struct inode *inode, const char *buf, + ssize_t buflen) +{ + struct nfs4_xattr_cache *cache; + struct nfs4_xattr_entry *entry; + + cache = nfs4_xattr_get_cache(inode, 1); + if (cache == NULL) + return; + + entry = nfs4_xattr_alloc_entry(NULL, buf, NULL, buflen); + if (entry == NULL) + goto out; + + /* + * This is just there to be able to get to bucket->cache, + * which is obviously the same for all buckets, so just + * use bucket 0. + */ + entry->bucket = &cache->buckets[0]; + + if (!nfs4_xattr_set_listcache(cache, entry)) + kref_put(&entry->ref, nfs4_xattr_free_entry_cb); + +out: + kref_put(&cache->ref, nfs4_xattr_free_cache_cb); +} + +/* + * Zap the entire cache. Called when an inode is evicted. + */ +void nfs4_xattr_cache_zap(struct inode *inode) +{ + struct nfs4_xattr_cache *oldcache; + + spin_lock(&inode->i_lock); + oldcache = nfs4_xattr_cache_unlink(inode); + spin_unlock(&inode->i_lock); + + if (oldcache) + nfs4_xattr_discard_cache(oldcache); +} + +/* + * The entry LRU is shrunk more aggressively than the cache LRU, + * by settings @seeks to 1. + * + * Cache structures are freed only when they've become empty, after + * pruning all but one entry. + */ + +static unsigned long nfs4_xattr_cache_count(struct shrinker *shrink, + struct shrink_control *sc); +static unsigned long nfs4_xattr_entry_count(struct shrinker *shrink, + struct shrink_control *sc); +static unsigned long nfs4_xattr_cache_scan(struct shrinker *shrink, + struct shrink_control *sc); +static unsigned long nfs4_xattr_entry_scan(struct shrinker *shrink, + struct shrink_control *sc); + +static struct shrinker *nfs4_xattr_cache_shrinker; +static struct shrinker *nfs4_xattr_entry_shrinker; +static struct shrinker *nfs4_xattr_large_entry_shrinker; + +static enum lru_status +cache_lru_isolate(struct list_head *item, + struct list_lru_one *lru, void *arg) +{ + struct list_head *dispose = arg; + struct inode *inode; + struct nfs4_xattr_cache *cache = container_of(item, + struct nfs4_xattr_cache, lru); + + if (atomic_long_read(&cache->nent) > 1) + return LRU_SKIP; + + /* + * If a cache structure is on the LRU list, we know that + * its inode is valid. Try to lock it to break the link. + * Since we're inverting the lock order here, only try. + */ + inode = cache->inode; + + if (!spin_trylock(&inode->i_lock)) + return LRU_SKIP; + + kref_get(&cache->ref); + + cache->inode = NULL; + NFS_I(inode)->xattr_cache = NULL; + NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_XATTR; + list_lru_isolate(lru, &cache->lru); + + spin_unlock(&inode->i_lock); + + list_add_tail(&cache->dispose, dispose); + return LRU_REMOVED; +} + +static unsigned long +nfs4_xattr_cache_scan(struct shrinker *shrink, struct shrink_control *sc) +{ + LIST_HEAD(dispose); + unsigned long freed; + struct nfs4_xattr_cache *cache; + + freed = list_lru_shrink_walk(&nfs4_xattr_cache_lru, sc, + cache_lru_isolate, &dispose); + while (!list_empty(&dispose)) { + cache = list_first_entry(&dispose, struct nfs4_xattr_cache, + dispose); + list_del_init(&cache->dispose); + nfs4_xattr_discard_cache(cache); + kref_put(&cache->ref, nfs4_xattr_free_cache_cb); + } + + return freed; +} + + +static unsigned long +nfs4_xattr_cache_count(struct shrinker *shrink, struct shrink_control *sc) +{ + unsigned long count; + + count = list_lru_shrink_count(&nfs4_xattr_cache_lru, sc); + return vfs_pressure_ratio(count); +} + +static enum lru_status +entry_lru_isolate(struct list_head *item, + struct list_lru_one *lru, void *arg) +{ + struct list_head *dispose = arg; + struct nfs4_xattr_bucket *bucket; + struct nfs4_xattr_cache *cache; + struct nfs4_xattr_entry *entry = container_of(item, + struct nfs4_xattr_entry, lru); + + bucket = entry->bucket; + cache = bucket->cache; + + /* + * Unhook the entry from its parent (either a cache bucket + * or a cache structure if it's a listxattr buf), so that + * it's no longer found. Then add it to the isolate list, + * to be freed later. + * + * In both cases, we're reverting lock order, so use + * trylock and skip the entry if we can't get the lock. + */ + if (entry->xattr_name != NULL) { + /* Regular cache entry */ + if (!spin_trylock(&bucket->lock)) + return LRU_SKIP; + + kref_get(&entry->ref); + + hlist_del_init(&entry->hnode); + atomic_long_dec(&cache->nent); + list_lru_isolate(lru, &entry->lru); + + spin_unlock(&bucket->lock); + } else { + /* Listxattr cache entry */ + if (!spin_trylock(&cache->listxattr_lock)) + return LRU_SKIP; + + kref_get(&entry->ref); + + cache->listxattr = NULL; + list_lru_isolate(lru, &entry->lru); + + spin_unlock(&cache->listxattr_lock); + } + + list_add_tail(&entry->dispose, dispose); + return LRU_REMOVED; +} + +static unsigned long +nfs4_xattr_entry_scan(struct shrinker *shrink, struct shrink_control *sc) +{ + LIST_HEAD(dispose); + unsigned long freed; + struct nfs4_xattr_entry *entry; + struct list_lru *lru; + + lru = (shrink == nfs4_xattr_large_entry_shrinker) ? + &nfs4_xattr_large_entry_lru : &nfs4_xattr_entry_lru; + + freed = list_lru_shrink_walk(lru, sc, entry_lru_isolate, &dispose); + + while (!list_empty(&dispose)) { + entry = list_first_entry(&dispose, struct nfs4_xattr_entry, + dispose); + list_del_init(&entry->dispose); + + /* + * Drop two references: the one that we just grabbed + * in entry_lru_isolate, and the one that was set + * when the entry was first allocated. + */ + kref_put(&entry->ref, nfs4_xattr_free_entry_cb); + kref_put(&entry->ref, nfs4_xattr_free_entry_cb); + } + + return freed; +} + +static unsigned long +nfs4_xattr_entry_count(struct shrinker *shrink, struct shrink_control *sc) +{ + unsigned long count; + struct list_lru *lru; + + lru = (shrink == nfs4_xattr_large_entry_shrinker) ? + &nfs4_xattr_large_entry_lru : &nfs4_xattr_entry_lru; + + count = list_lru_shrink_count(lru, sc); + return vfs_pressure_ratio(count); +} + + +static void nfs4_xattr_cache_init_once(void *p) +{ + struct nfs4_xattr_cache *cache = p; + + spin_lock_init(&cache->listxattr_lock); + atomic_long_set(&cache->nent, 0); + nfs4_xattr_hash_init(cache); + cache->listxattr = NULL; + INIT_LIST_HEAD(&cache->lru); + INIT_LIST_HEAD(&cache->dispose); +} + +typedef unsigned long (*count_objects_cb)(struct shrinker *s, + struct shrink_control *sc); +typedef unsigned long (*scan_objects_cb)(struct shrinker *s, + struct shrink_control *sc); + +static int __init nfs4_xattr_shrinker_init(struct shrinker **shrinker, + struct list_lru *lru, const char *name, + count_objects_cb count, + scan_objects_cb scan, long batch, int seeks) +{ + int ret; + + *shrinker = shrinker_alloc(SHRINKER_MEMCG_AWARE, name); + if (!*shrinker) + return -ENOMEM; + + ret = list_lru_init_memcg(lru, *shrinker); + if (ret) { + shrinker_free(*shrinker); + return ret; + } + + (*shrinker)->count_objects = count; + (*shrinker)->scan_objects = scan; + (*shrinker)->batch = batch; + (*shrinker)->seeks = seeks; + + shrinker_register(*shrinker); + + return ret; +} + +static void nfs4_xattr_shrinker_destroy(struct shrinker *shrinker, + struct list_lru *lru) +{ + shrinker_free(shrinker); + list_lru_destroy(lru); +} + +int __init nfs4_xattr_cache_init(void) +{ + int ret = 0; + + nfs4_xattr_cache_cachep = kmem_cache_create("nfs4_xattr_cache_cache", + sizeof(struct nfs4_xattr_cache), 0, + (SLAB_RECLAIM_ACCOUNT), + nfs4_xattr_cache_init_once); + if (nfs4_xattr_cache_cachep == NULL) + return -ENOMEM; + + ret = nfs4_xattr_shrinker_init(&nfs4_xattr_cache_shrinker, + &nfs4_xattr_cache_lru, "nfs-xattr_cache", + nfs4_xattr_cache_count, + nfs4_xattr_cache_scan, 0, DEFAULT_SEEKS); + if (ret) + goto out1; + + ret = nfs4_xattr_shrinker_init(&nfs4_xattr_entry_shrinker, + &nfs4_xattr_entry_lru, "nfs-xattr_entry", + nfs4_xattr_entry_count, + nfs4_xattr_entry_scan, 512, DEFAULT_SEEKS); + if (ret) + goto out2; + + ret = nfs4_xattr_shrinker_init(&nfs4_xattr_large_entry_shrinker, + &nfs4_xattr_large_entry_lru, + "nfs-xattr_large_entry", + nfs4_xattr_entry_count, + nfs4_xattr_entry_scan, 512, 1); + if (!ret) + return 0; + + nfs4_xattr_shrinker_destroy(nfs4_xattr_entry_shrinker, + &nfs4_xattr_entry_lru); +out2: + nfs4_xattr_shrinker_destroy(nfs4_xattr_cache_shrinker, + &nfs4_xattr_cache_lru); +out1: + kmem_cache_destroy(nfs4_xattr_cache_cachep); + + return ret; +} + +void nfs4_xattr_cache_exit(void) +{ + nfs4_xattr_shrinker_destroy(nfs4_xattr_large_entry_shrinker, + &nfs4_xattr_large_entry_lru); + nfs4_xattr_shrinker_destroy(nfs4_xattr_entry_shrinker, + &nfs4_xattr_entry_lru); + nfs4_xattr_shrinker_destroy(nfs4_xattr_cache_shrinker, + &nfs4_xattr_cache_lru); + kmem_cache_destroy(nfs4_xattr_cache_cachep); +} diff --git a/fs/nfs/nfs42xdr.c b/fs/nfs/nfs42xdr.c new file mode 100644 index 000000000..ec105c62f --- /dev/null +++ b/fs/nfs/nfs42xdr.c @@ -0,0 +1,1830 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2014 Anna Schumaker + */ +#ifndef __LINUX_FS_NFS_NFS4_2XDR_H +#define __LINUX_FS_NFS_NFS4_2XDR_H + +#include "nfs42.h" + +/* Not limited by NFS itself, limited by the generic xattr code */ +#define nfs4_xattr_name_maxsz XDR_QUADLEN(XATTR_NAME_MAX) + +#define encode_fallocate_maxsz (encode_stateid_maxsz + \ + 2 /* offset */ + \ + 2 /* length */) +#define NFS42_WRITE_RES_SIZE (1 /* wr_callback_id size */ +\ + XDR_QUADLEN(NFS4_STATEID_SIZE) + \ + 2 /* wr_count */ + \ + 1 /* wr_committed */ + \ + XDR_QUADLEN(NFS4_VERIFIER_SIZE)) +#define encode_allocate_maxsz (op_encode_hdr_maxsz + \ + encode_fallocate_maxsz) +#define decode_allocate_maxsz (op_decode_hdr_maxsz) +#define encode_copy_maxsz (op_encode_hdr_maxsz + \ + XDR_QUADLEN(NFS4_STATEID_SIZE) + \ + XDR_QUADLEN(NFS4_STATEID_SIZE) + \ + 2 + 2 + 2 + 1 + 1 + 1 +\ + 1 + /* One cnr_source_server */\ + 1 + /* nl4_type */ \ + 1 + XDR_QUADLEN(NFS4_OPAQUE_LIMIT)) +#define decode_copy_maxsz (op_decode_hdr_maxsz + \ + NFS42_WRITE_RES_SIZE + \ + 1 /* cr_consecutive */ + \ + 1 /* cr_synchronous */) +#define encode_offload_cancel_maxsz (op_encode_hdr_maxsz + \ + XDR_QUADLEN(NFS4_STATEID_SIZE)) +#define decode_offload_cancel_maxsz (op_decode_hdr_maxsz) +#define encode_offload_status_maxsz (op_encode_hdr_maxsz + \ + XDR_QUADLEN(NFS4_STATEID_SIZE)) +#define decode_offload_status_maxsz (op_decode_hdr_maxsz + \ + 2 /* osr_count */ + \ + 2 /* osr_complete */) +#define encode_copy_notify_maxsz (op_encode_hdr_maxsz + \ + XDR_QUADLEN(NFS4_STATEID_SIZE) + \ + 1 + /* nl4_type */ \ + 1 + XDR_QUADLEN(NFS4_OPAQUE_LIMIT)) +#define decode_copy_notify_maxsz (op_decode_hdr_maxsz + \ + 3 + /* cnr_lease_time */\ + XDR_QUADLEN(NFS4_STATEID_SIZE) + \ + 1 + /* Support 1 cnr_source_server */\ + 1 + /* nl4_type */ \ + 1 + XDR_QUADLEN(NFS4_OPAQUE_LIMIT)) +#define encode_deallocate_maxsz (op_encode_hdr_maxsz + \ + encode_fallocate_maxsz) +#define decode_deallocate_maxsz (op_decode_hdr_maxsz) +#define encode_read_plus_maxsz (op_encode_hdr_maxsz + \ + encode_stateid_maxsz + 3) +#define NFS42_READ_PLUS_DATA_SEGMENT_SIZE \ + (1 /* data_content4 */ + \ + 2 /* data_info4.di_offset */ + \ + 1 /* data_info4.di_length */) +#define NFS42_READ_PLUS_HOLE_SEGMENT_SIZE \ + (1 /* data_content4 */ + \ + 2 /* data_info4.di_offset */ + \ + 2 /* data_info4.di_length */) +#define READ_PLUS_SEGMENT_SIZE_DIFF (NFS42_READ_PLUS_HOLE_SEGMENT_SIZE - \ + NFS42_READ_PLUS_DATA_SEGMENT_SIZE) +#define decode_read_plus_maxsz (op_decode_hdr_maxsz + \ + 1 /* rpr_eof */ + \ + 1 /* rpr_contents count */ + \ + NFS42_READ_PLUS_HOLE_SEGMENT_SIZE) +#define encode_seek_maxsz (op_encode_hdr_maxsz + \ + encode_stateid_maxsz + \ + 2 /* offset */ + \ + 1 /* whence */) +#define decode_seek_maxsz (op_decode_hdr_maxsz + \ + 1 /* eof */ + \ + 1 /* whence */ + \ + 2 /* offset */ + \ + 2 /* length */) +#define encode_io_info_maxsz 4 +#define encode_layoutstats_maxsz (op_decode_hdr_maxsz + \ + 2 /* offset */ + \ + 2 /* length */ + \ + encode_stateid_maxsz + \ + encode_io_info_maxsz + \ + encode_io_info_maxsz + \ + 1 /* opaque devaddr4 length */ + \ + XDR_QUADLEN(PNFS_LAYOUTSTATS_MAXSIZE)) +#define decode_layoutstats_maxsz (op_decode_hdr_maxsz) +#define encode_device_error_maxsz (XDR_QUADLEN(NFS4_DEVICEID4_SIZE) + \ + 1 /* status */ + 1 /* opnum */) +#define encode_layouterror_maxsz (op_decode_hdr_maxsz + \ + 2 /* offset */ + \ + 2 /* length */ + \ + encode_stateid_maxsz + \ + 1 /* Array size */ + \ + encode_device_error_maxsz) +#define decode_layouterror_maxsz (op_decode_hdr_maxsz) +#define encode_clone_maxsz (encode_stateid_maxsz + \ + encode_stateid_maxsz + \ + 2 /* src offset */ + \ + 2 /* dst offset */ + \ + 2 /* count */) +#define decode_clone_maxsz (op_decode_hdr_maxsz) +#define encode_getxattr_maxsz (op_encode_hdr_maxsz + 1 + \ + nfs4_xattr_name_maxsz) +#define decode_getxattr_maxsz (op_decode_hdr_maxsz + 1 + pagepad_maxsz) +#define encode_setxattr_maxsz (op_encode_hdr_maxsz + \ + 1 + nfs4_xattr_name_maxsz + 1) +#define decode_setxattr_maxsz (op_decode_hdr_maxsz + decode_change_info_maxsz) +#define encode_listxattrs_maxsz (op_encode_hdr_maxsz + 2 + 1) +#define decode_listxattrs_maxsz (op_decode_hdr_maxsz + 2 + 1 + 1 + 1) +#define encode_removexattr_maxsz (op_encode_hdr_maxsz + 1 + \ + nfs4_xattr_name_maxsz) +#define decode_removexattr_maxsz (op_decode_hdr_maxsz + \ + decode_change_info_maxsz) + +#define NFS4_enc_allocate_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_allocate_maxsz + \ + encode_getattr_maxsz) +#define NFS4_dec_allocate_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_allocate_maxsz + \ + decode_getattr_maxsz) +#define NFS4_enc_copy_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_savefh_maxsz + \ + encode_putfh_maxsz + \ + encode_copy_maxsz + \ + encode_commit_maxsz) +#define NFS4_dec_copy_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_savefh_maxsz + \ + decode_putfh_maxsz + \ + decode_copy_maxsz + \ + decode_commit_maxsz) +#define NFS4_enc_offload_cancel_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_offload_cancel_maxsz) +#define NFS4_dec_offload_cancel_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_offload_cancel_maxsz) +#define NFS4_enc_offload_status_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_offload_status_maxsz) +#define NFS4_dec_offload_status_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_offload_status_maxsz) +#define NFS4_enc_copy_notify_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_copy_notify_maxsz) +#define NFS4_dec_copy_notify_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_copy_notify_maxsz) +#define NFS4_enc_deallocate_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_deallocate_maxsz + \ + encode_getattr_maxsz) +#define NFS4_dec_deallocate_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_deallocate_maxsz + \ + decode_getattr_maxsz) +#define NFS4_enc_zero_range_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_deallocate_maxsz + \ + encode_allocate_maxsz + \ + encode_getattr_maxsz) +#define NFS4_dec_zero_range_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_deallocate_maxsz + \ + decode_allocate_maxsz + \ + decode_getattr_maxsz) +#define NFS4_enc_read_plus_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_read_plus_maxsz) +#define NFS4_dec_read_plus_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_read_plus_maxsz) +#define NFS4_enc_seek_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_seek_maxsz) +#define NFS4_dec_seek_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_seek_maxsz) +#define NFS4_enc_layoutstats_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + PNFS_LAYOUTSTATS_MAXDEV * encode_layoutstats_maxsz) +#define NFS4_dec_layoutstats_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + PNFS_LAYOUTSTATS_MAXDEV * decode_layoutstats_maxsz) +#define NFS4_enc_layouterror_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + NFS42_LAYOUTERROR_MAX * \ + encode_layouterror_maxsz) +#define NFS4_dec_layouterror_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + NFS42_LAYOUTERROR_MAX * \ + decode_layouterror_maxsz) +#define NFS4_enc_clone_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_savefh_maxsz + \ + encode_putfh_maxsz + \ + encode_clone_maxsz + \ + encode_getattr_maxsz) +#define NFS4_dec_clone_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_savefh_maxsz + \ + decode_putfh_maxsz + \ + decode_clone_maxsz + \ + decode_getattr_maxsz) +#define NFS4_enc_getxattr_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_getxattr_maxsz) +#define NFS4_dec_getxattr_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_getxattr_maxsz) +#define NFS4_enc_setxattr_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_setxattr_maxsz + \ + encode_getattr_maxsz) +#define NFS4_dec_setxattr_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_setxattr_maxsz + \ + decode_getattr_maxsz) +#define NFS4_enc_listxattrs_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_listxattrs_maxsz) +#define NFS4_dec_listxattrs_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_listxattrs_maxsz) +#define NFS4_enc_removexattr_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_removexattr_maxsz + \ + encode_getattr_maxsz) +#define NFS4_dec_removexattr_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_removexattr_maxsz + \ + decode_getattr_maxsz) + +/* + * These values specify the maximum amount of data that is not + * associated with the extended attribute name or extended + * attribute list in the SETXATTR, GETXATTR and LISTXATTR + * respectively. + */ +const u32 nfs42_maxsetxattr_overhead = ((RPC_MAX_HEADER_WITH_AUTH + + compound_encode_hdr_maxsz + + encode_sequence_maxsz + + encode_putfh_maxsz + 1 + + nfs4_xattr_name_maxsz) + * XDR_UNIT); + +const u32 nfs42_maxgetxattr_overhead = ((RPC_MAX_HEADER_WITH_AUTH + + compound_decode_hdr_maxsz + + decode_sequence_maxsz + + decode_putfh_maxsz + 1) * XDR_UNIT); + +const u32 nfs42_maxlistxattrs_overhead = ((RPC_MAX_HEADER_WITH_AUTH + + compound_decode_hdr_maxsz + + decode_sequence_maxsz + + decode_putfh_maxsz + 3) * XDR_UNIT); + +static void encode_fallocate(struct xdr_stream *xdr, + const struct nfs42_falloc_args *args) +{ + encode_nfs4_stateid(xdr, &args->falloc_stateid); + encode_uint64(xdr, args->falloc_offset); + encode_uint64(xdr, args->falloc_length); +} + +static void encode_allocate(struct xdr_stream *xdr, + const struct nfs42_falloc_args *args, + struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_ALLOCATE, decode_allocate_maxsz, hdr); + encode_fallocate(xdr, args); +} + +static void encode_nl4_server(struct xdr_stream *xdr, + const struct nl4_server *ns) +{ + encode_uint32(xdr, ns->nl4_type); + switch (ns->nl4_type) { + case NL4_NAME: + case NL4_URL: + encode_string(xdr, ns->u.nl4_str_sz, ns->u.nl4_str); + break; + case NL4_NETADDR: + encode_string(xdr, ns->u.nl4_addr.netid_len, + ns->u.nl4_addr.netid); + encode_string(xdr, ns->u.nl4_addr.addr_len, + ns->u.nl4_addr.addr); + break; + default: + WARN_ON_ONCE(1); + } +} + +static void encode_copy(struct xdr_stream *xdr, + const struct nfs42_copy_args *args, + struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_COPY, decode_copy_maxsz, hdr); + encode_nfs4_stateid(xdr, &args->src_stateid); + encode_nfs4_stateid(xdr, &args->dst_stateid); + + encode_uint64(xdr, args->src_pos); + encode_uint64(xdr, args->dst_pos); + encode_uint64(xdr, args->count); + + encode_uint32(xdr, 1); /* consecutive = true */ + encode_uint32(xdr, args->sync); + if (args->cp_src == NULL) { /* intra-ssc */ + encode_uint32(xdr, 0); /* no src server list */ + return; + } + encode_uint32(xdr, 1); /* supporting 1 server */ + encode_nl4_server(xdr, args->cp_src); +} + +static void encode_copy_commit(struct xdr_stream *xdr, + const struct nfs42_copy_args *args, + struct compound_hdr *hdr) +{ + __be32 *p; + + encode_op_hdr(xdr, OP_COMMIT, decode_commit_maxsz, hdr); + p = reserve_space(xdr, 12); + p = xdr_encode_hyper(p, args->dst_pos); + *p = cpu_to_be32(args->count); +} + +static void encode_offload_cancel(struct xdr_stream *xdr, + const struct nfs42_offload_status_args *args, + struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_OFFLOAD_CANCEL, decode_offload_cancel_maxsz, hdr); + encode_nfs4_stateid(xdr, &args->osa_stateid); +} + +static void encode_offload_status(struct xdr_stream *xdr, + const struct nfs42_offload_status_args *args, + struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_OFFLOAD_STATUS, decode_offload_status_maxsz, hdr); + encode_nfs4_stateid(xdr, &args->osa_stateid); +} + +static void encode_copy_notify(struct xdr_stream *xdr, + const struct nfs42_copy_notify_args *args, + struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_COPY_NOTIFY, decode_copy_notify_maxsz, hdr); + encode_nfs4_stateid(xdr, &args->cna_src_stateid); + encode_nl4_server(xdr, &args->cna_dst); +} + +static void encode_deallocate(struct xdr_stream *xdr, + const struct nfs42_falloc_args *args, + struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_DEALLOCATE, decode_deallocate_maxsz, hdr); + encode_fallocate(xdr, args); +} + +static void encode_read_plus(struct xdr_stream *xdr, + const struct nfs_pgio_args *args, + struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_READ_PLUS, decode_read_plus_maxsz, hdr); + encode_nfs4_stateid(xdr, &args->stateid); + encode_uint64(xdr, args->offset); + encode_uint32(xdr, args->count); +} + +static void encode_seek(struct xdr_stream *xdr, + const struct nfs42_seek_args *args, + struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_SEEK, decode_seek_maxsz, hdr); + encode_nfs4_stateid(xdr, &args->sa_stateid); + encode_uint64(xdr, args->sa_offset); + encode_uint32(xdr, args->sa_what); +} + +static void encode_layoutstats(struct xdr_stream *xdr, + const struct nfs42_layoutstat_args *args, + struct nfs42_layoutstat_devinfo *devinfo, + struct compound_hdr *hdr) +{ + __be32 *p; + + encode_op_hdr(xdr, OP_LAYOUTSTATS, decode_layoutstats_maxsz, hdr); + p = reserve_space(xdr, 8 + 8); + p = xdr_encode_hyper(p, devinfo->offset); + p = xdr_encode_hyper(p, devinfo->length); + encode_nfs4_stateid(xdr, &args->stateid); + p = reserve_space(xdr, 4*8 + NFS4_DEVICEID4_SIZE + 4); + p = xdr_encode_hyper(p, devinfo->read_count); + p = xdr_encode_hyper(p, devinfo->read_bytes); + p = xdr_encode_hyper(p, devinfo->write_count); + p = xdr_encode_hyper(p, devinfo->write_bytes); + p = xdr_encode_opaque_fixed(p, devinfo->dev_id.data, + NFS4_DEVICEID4_SIZE); + /* Encode layoutupdate4 */ + *p++ = cpu_to_be32(devinfo->layout_type); + if (devinfo->ld_private.ops) + devinfo->ld_private.ops->encode(xdr, args, + &devinfo->ld_private); + else + encode_uint32(xdr, 0); +} + +static void encode_clone(struct xdr_stream *xdr, + const struct nfs42_clone_args *args, + struct compound_hdr *hdr) +{ + __be32 *p; + + encode_op_hdr(xdr, OP_CLONE, decode_clone_maxsz, hdr); + encode_nfs4_stateid(xdr, &args->src_stateid); + encode_nfs4_stateid(xdr, &args->dst_stateid); + p = reserve_space(xdr, 3*8); + p = xdr_encode_hyper(p, args->src_offset); + p = xdr_encode_hyper(p, args->dst_offset); + xdr_encode_hyper(p, args->count); +} + +static void encode_device_error(struct xdr_stream *xdr, + const struct nfs42_device_error *error) +{ + __be32 *p; + + p = reserve_space(xdr, NFS4_DEVICEID4_SIZE + 2*4); + p = xdr_encode_opaque_fixed(p, error->dev_id.data, + NFS4_DEVICEID4_SIZE); + *p++ = cpu_to_be32(error->status); + *p = cpu_to_be32(error->opnum); +} + +static void encode_layouterror(struct xdr_stream *xdr, + const struct nfs42_layout_error *args, + struct compound_hdr *hdr) +{ + __be32 *p; + + encode_op_hdr(xdr, OP_LAYOUTERROR, decode_layouterror_maxsz, hdr); + p = reserve_space(xdr, 8 + 8); + p = xdr_encode_hyper(p, args->offset); + p = xdr_encode_hyper(p, args->length); + encode_nfs4_stateid(xdr, &args->stateid); + p = reserve_space(xdr, 4); + *p = cpu_to_be32(1); + encode_device_error(xdr, &args->errors[0]); +} + +static void encode_setxattr(struct xdr_stream *xdr, + const struct nfs42_setxattrargs *arg, + struct compound_hdr *hdr) +{ + __be32 *p; + + BUILD_BUG_ON(XATTR_CREATE != SETXATTR4_CREATE); + BUILD_BUG_ON(XATTR_REPLACE != SETXATTR4_REPLACE); + + encode_op_hdr(xdr, OP_SETXATTR, decode_setxattr_maxsz, hdr); + p = reserve_space(xdr, 4); + *p = cpu_to_be32(arg->xattr_flags); + encode_string(xdr, strlen(arg->xattr_name), arg->xattr_name); + p = reserve_space(xdr, 4); + *p = cpu_to_be32(arg->xattr_len); + if (arg->xattr_len) + xdr_write_pages(xdr, arg->xattr_pages, 0, arg->xattr_len); +} + +static void encode_getxattr(struct xdr_stream *xdr, const char *name, + struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_GETXATTR, decode_getxattr_maxsz, hdr); + encode_string(xdr, strlen(name), name); +} + +static void encode_removexattr(struct xdr_stream *xdr, const char *name, + struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_REMOVEXATTR, decode_removexattr_maxsz, hdr); + encode_string(xdr, strlen(name), name); +} + +static void encode_listxattrs(struct xdr_stream *xdr, + const struct nfs42_listxattrsargs *arg, + struct compound_hdr *hdr) +{ + __be32 *p; + + encode_op_hdr(xdr, OP_LISTXATTRS, decode_listxattrs_maxsz, hdr); + + p = reserve_space(xdr, 12); + if (unlikely(!p)) + return; + + p = xdr_encode_hyper(p, arg->cookie); + /* + * RFC 8276 says to specify the full max length of the LISTXATTRS + * XDR reply. Count is set to the XDR length of the names array + * plus the EOF marker. So, add the cookie and the names count. + */ + *p = cpu_to_be32(arg->count + 8 + 4); +} + +/* + * Encode ALLOCATE request + */ +static void nfs4_xdr_enc_allocate(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs42_falloc_args *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->falloc_fh, &hdr); + encode_allocate(xdr, args, &hdr); + encode_getfattr(xdr, args->falloc_bitmask, &hdr); + encode_nops(&hdr); +} + +/* + * Encode COPY request + */ +static void nfs4_xdr_enc_copy(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs42_copy_args *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->src_fh, &hdr); + encode_savefh(xdr, &hdr); + encode_putfh(xdr, args->dst_fh, &hdr); + encode_copy(xdr, args, &hdr); + if (args->sync) + encode_copy_commit(xdr, args, &hdr); + encode_nops(&hdr); +} + +/* + * Encode OFFLOAD_CANCEL request + */ +static void nfs4_xdr_enc_offload_cancel(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs42_offload_status_args *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->osa_seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->osa_seq_args, &hdr); + encode_putfh(xdr, args->osa_src_fh, &hdr); + encode_offload_cancel(xdr, args, &hdr); + encode_nops(&hdr); +} + +/* + * Encode OFFLOAD_STATUS request + */ +static void nfs4_xdr_enc_offload_status(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs42_offload_status_args *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->osa_seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->osa_seq_args, &hdr); + encode_putfh(xdr, args->osa_src_fh, &hdr); + encode_offload_status(xdr, args, &hdr); + encode_nops(&hdr); +} + +/* + * Encode COPY_NOTIFY request + */ +static void nfs4_xdr_enc_copy_notify(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs42_copy_notify_args *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->cna_seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->cna_seq_args, &hdr); + encode_putfh(xdr, args->cna_src_fh, &hdr); + encode_copy_notify(xdr, args, &hdr); + encode_nops(&hdr); +} + +/* + * Encode DEALLOCATE request + */ +static void nfs4_xdr_enc_deallocate(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs42_falloc_args *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->falloc_fh, &hdr); + encode_deallocate(xdr, args, &hdr); + encode_getfattr(xdr, args->falloc_bitmask, &hdr); + encode_nops(&hdr); +} + +/* + * Encode ZERO_RANGE request + */ +static void nfs4_xdr_enc_zero_range(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs42_falloc_args *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->falloc_fh, &hdr); + encode_deallocate(xdr, args, &hdr); + encode_allocate(xdr, args, &hdr); + encode_getfattr(xdr, args->falloc_bitmask, &hdr); + encode_nops(&hdr); +} + +/* + * Encode READ_PLUS request + */ +static void nfs4_xdr_enc_read_plus(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_pgio_args *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + encode_read_plus(xdr, args, &hdr); + + rpc_prepare_reply_pages(req, args->pages, args->pgbase, args->count, + hdr.replen - READ_PLUS_SEGMENT_SIZE_DIFF); + encode_nops(&hdr); +} + +/* + * Encode SEEK request + */ +static void nfs4_xdr_enc_seek(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs42_seek_args *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->sa_fh, &hdr); + encode_seek(xdr, args, &hdr); + encode_nops(&hdr); +} + +/* + * Encode LAYOUTSTATS request + */ +static void nfs4_xdr_enc_layoutstats(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs42_layoutstat_args *args = data; + int i; + + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + WARN_ON(args->num_dev > PNFS_LAYOUTSTATS_MAXDEV); + for (i = 0; i < args->num_dev; i++) + encode_layoutstats(xdr, args, &args->devinfo[i], &hdr); + encode_nops(&hdr); +} + +/* + * Encode CLONE request + */ +static void nfs4_xdr_enc_clone(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs42_clone_args *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->src_fh, &hdr); + encode_savefh(xdr, &hdr); + encode_putfh(xdr, args->dst_fh, &hdr); + encode_clone(xdr, args, &hdr); + encode_getfattr(xdr, args->dst_bitmask, &hdr); + encode_nops(&hdr); +} + +/* + * Encode LAYOUTERROR request + */ +static void nfs4_xdr_enc_layouterror(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs42_layouterror_args *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + int i; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, NFS_FH(args->inode), &hdr); + for (i = 0; i < args->num_errors; i++) + encode_layouterror(xdr, &args->errors[i], &hdr); + encode_nops(&hdr); +} + +/* + * Encode SETXATTR request + */ +static void nfs4_xdr_enc_setxattr(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs42_setxattrargs *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + encode_setxattr(xdr, args, &hdr); + encode_getfattr(xdr, args->bitmask, &hdr); + encode_nops(&hdr); +} + +/* + * Encode GETXATTR request + */ +static void nfs4_xdr_enc_getxattr(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs42_getxattrargs *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + uint32_t replen; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + replen = hdr.replen + op_decode_hdr_maxsz + 1; + encode_getxattr(xdr, args->xattr_name, &hdr); + + rpc_prepare_reply_pages(req, args->xattr_pages, 0, args->xattr_len, + replen); + + encode_nops(&hdr); +} + +/* + * Encode LISTXATTR request + */ +static void nfs4_xdr_enc_listxattrs(struct rpc_rqst *req, + struct xdr_stream *xdr, const void *data) +{ + const struct nfs42_listxattrsargs *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + uint32_t replen; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + replen = hdr.replen + op_decode_hdr_maxsz + 2 + 1; + encode_listxattrs(xdr, args, &hdr); + + rpc_prepare_reply_pages(req, args->xattr_pages, 0, args->count, replen); + + encode_nops(&hdr); +} + +/* + * Encode REMOVEXATTR request + */ +static void nfs4_xdr_enc_removexattr(struct rpc_rqst *req, + struct xdr_stream *xdr, const void *data) +{ + const struct nfs42_removexattrargs *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + encode_removexattr(xdr, args->xattr_name, &hdr); + encode_getfattr(xdr, args->bitmask, &hdr); + encode_nops(&hdr); +} + +static int decode_allocate(struct xdr_stream *xdr, struct nfs42_falloc_res *res) +{ + return decode_op_hdr(xdr, OP_ALLOCATE); +} + +static int decode_write_response(struct xdr_stream *xdr, + struct nfs42_write_res *res) +{ + __be32 *p; + int status, count; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + count = be32_to_cpup(p); + if (count > 1) + return -EREMOTEIO; + else if (count == 1) { + status = decode_opaque_fixed(xdr, &res->stateid, + NFS4_STATEID_SIZE); + if (unlikely(status)) + return -EIO; + } + p = xdr_inline_decode(xdr, 8 + 4); + if (unlikely(!p)) + return -EIO; + p = xdr_decode_hyper(p, &res->count); + res->verifier.committed = be32_to_cpup(p); + return decode_verifier(xdr, &res->verifier.verifier); +} + +static int decode_nl4_server(struct xdr_stream *xdr, struct nl4_server *ns) +{ + struct nfs42_netaddr *naddr; + uint32_t dummy; + char *dummy_str; + __be32 *p; + int status; + + /* nl_type */ + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + ns->nl4_type = be32_to_cpup(p); + switch (ns->nl4_type) { + case NL4_NAME: + case NL4_URL: + status = decode_opaque_inline(xdr, &dummy, &dummy_str); + if (unlikely(status)) + return status; + if (unlikely(dummy > NFS4_OPAQUE_LIMIT)) + return -EIO; + memcpy(&ns->u.nl4_str, dummy_str, dummy); + ns->u.nl4_str_sz = dummy; + break; + case NL4_NETADDR: + naddr = &ns->u.nl4_addr; + + /* netid string */ + status = decode_opaque_inline(xdr, &dummy, &dummy_str); + if (unlikely(status)) + return status; + if (unlikely(dummy > RPCBIND_MAXNETIDLEN)) + return -EIO; + naddr->netid_len = dummy; + memcpy(naddr->netid, dummy_str, naddr->netid_len); + + /* uaddr string */ + status = decode_opaque_inline(xdr, &dummy, &dummy_str); + if (unlikely(status)) + return status; + if (unlikely(dummy > RPCBIND_MAXUADDRLEN)) + return -EIO; + naddr->addr_len = dummy; + memcpy(naddr->addr, dummy_str, naddr->addr_len); + break; + default: + WARN_ON_ONCE(1); + return -EIO; + } + return 0; +} + +static int decode_copy_requirements(struct xdr_stream *xdr, + struct nfs42_copy_res *res) { + __be32 *p; + + p = xdr_inline_decode(xdr, 4 + 4); + if (unlikely(!p)) + return -EIO; + + res->consecutive = be32_to_cpup(p++); + res->synchronous = be32_to_cpup(p++); + return 0; +} + +static int decode_copy(struct xdr_stream *xdr, struct nfs42_copy_res *res) +{ + int status; + + status = decode_op_hdr(xdr, OP_COPY); + if (status == NFS4ERR_OFFLOAD_NO_REQS) { + status = decode_copy_requirements(xdr, res); + if (status) + return status; + return NFS4ERR_OFFLOAD_NO_REQS; + } else if (status) + return status; + + status = decode_write_response(xdr, &res->write_res); + if (status) + return status; + + return decode_copy_requirements(xdr, res); +} + +static int decode_offload_cancel(struct xdr_stream *xdr, + struct nfs42_offload_status_res *res) +{ + return decode_op_hdr(xdr, OP_OFFLOAD_CANCEL); +} + +static int decode_offload_status(struct xdr_stream *xdr, + struct nfs42_offload_status_res *res) +{ + ssize_t result; + int status; + + status = decode_op_hdr(xdr, OP_OFFLOAD_STATUS); + if (status) + return status; + /* osr_count */ + if (xdr_stream_decode_u64(xdr, &res->osr_count) < 0) + return -EIO; + /* osr_complete<1> */ + result = xdr_stream_decode_uint32_array(xdr, &res->osr_complete, 1); + if (result < 0) + return -EIO; + res->complete_count = result; + return 0; +} + +static int decode_copy_notify(struct xdr_stream *xdr, + struct nfs42_copy_notify_res *res) +{ + __be32 *p; + int status, count; + + status = decode_op_hdr(xdr, OP_COPY_NOTIFY); + if (status) + return status; + /* cnr_lease_time */ + p = xdr_inline_decode(xdr, 12); + if (unlikely(!p)) + return -EIO; + p = xdr_decode_hyper(p, &res->cnr_lease_time.seconds); + res->cnr_lease_time.nseconds = be32_to_cpup(p); + + status = decode_opaque_fixed(xdr, &res->cnr_stateid, NFS4_STATEID_SIZE); + if (unlikely(status)) + return -EIO; + + /* number of source addresses */ + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + + count = be32_to_cpup(p); + if (count > 1) + pr_warn("NFS: %s: nsvr %d > Supported. Use first servers\n", + __func__, count); + + status = decode_nl4_server(xdr, &res->cnr_src); + if (unlikely(status)) + return -EIO; + return 0; +} + +static int decode_deallocate(struct xdr_stream *xdr, struct nfs42_falloc_res *res) +{ + return decode_op_hdr(xdr, OP_DEALLOCATE); +} + +struct read_plus_segment { + enum data_content4 type; + uint64_t offset; + union { + struct { + uint64_t length; + } hole; + + struct { + uint32_t length; + unsigned int from; + } data; + }; +}; + +static inline uint64_t read_plus_segment_length(struct read_plus_segment *seg) +{ + return seg->type == NFS4_CONTENT_DATA ? seg->data.length : seg->hole.length; +} + +static int decode_read_plus_segment(struct xdr_stream *xdr, + struct read_plus_segment *seg) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, 4); + if (!p) + return -EIO; + seg->type = be32_to_cpup(p++); + + p = xdr_inline_decode(xdr, seg->type == NFS4_CONTENT_DATA ? 12 : 16); + if (!p) + return -EIO; + p = xdr_decode_hyper(p, &seg->offset); + + if (seg->type == NFS4_CONTENT_DATA) { + struct xdr_buf buf; + uint32_t len = be32_to_cpup(p); + + seg->data.length = len; + seg->data.from = xdr_stream_pos(xdr); + + if (!xdr_stream_subsegment(xdr, &buf, xdr_align_size(len))) + return -EIO; + } else if (seg->type == NFS4_CONTENT_HOLE) { + xdr_decode_hyper(p, &seg->hole.length); + } else + return -EINVAL; + return 0; +} + +static int process_read_plus_segment(struct xdr_stream *xdr, + struct nfs_pgio_args *args, + struct nfs_pgio_res *res, + struct read_plus_segment *seg) +{ + unsigned long offset = seg->offset; + unsigned long length = read_plus_segment_length(seg); + unsigned int bufpos; + + if (offset + length < args->offset) + return 0; + else if (offset > args->offset + args->count) { + res->eof = 0; + return 0; + } else if (offset < args->offset) { + length -= (args->offset - offset); + offset = args->offset; + } else if (offset + length > args->offset + args->count) { + length = (args->offset + args->count) - offset; + res->eof = 0; + } + + bufpos = xdr->buf->head[0].iov_len + (offset - args->offset); + if (seg->type == NFS4_CONTENT_HOLE) + return xdr_stream_zero(xdr, bufpos, length); + else + return xdr_stream_move_subsegment(xdr, seg->data.from, bufpos, length); +} + +static int decode_read_plus(struct xdr_stream *xdr, struct nfs_pgio_res *res) +{ + struct nfs_pgio_header *hdr = + container_of(res, struct nfs_pgio_header, res); + struct nfs_pgio_args *args = &hdr->args; + uint32_t segments; + struct read_plus_segment *segs; + int status, i; + __be32 *p; + + status = decode_op_hdr(xdr, OP_READ_PLUS); + if (status) + return status; + + p = xdr_inline_decode(xdr, 4 + 4); + if (unlikely(!p)) + return -EIO; + + res->count = 0; + res->eof = be32_to_cpup(p++); + segments = be32_to_cpup(p++); + if (segments == 0) + return 0; + + segs = kmalloc_objs(*segs, segments); + if (!segs) + return -ENOMEM; + + for (i = 0; i < segments; i++) { + status = decode_read_plus_segment(xdr, &segs[i]); + if (status < 0) + goto out; + } + + xdr_set_pagelen(xdr, xdr_align_size(args->count)); + for (i = segments; i > 0; i--) + res->count += process_read_plus_segment(xdr, args, res, &segs[i-1]); + status = 0; + +out: + kfree(segs); + return status; +} + +static int decode_seek(struct xdr_stream *xdr, struct nfs42_seek_res *res) +{ + int status; + __be32 *p; + + status = decode_op_hdr(xdr, OP_SEEK); + if (status) + return status; + + p = xdr_inline_decode(xdr, 4 + 8); + if (unlikely(!p)) + return -EIO; + + res->sr_eof = be32_to_cpup(p++); + p = xdr_decode_hyper(p, &res->sr_offset); + return 0; +} + +static int decode_layoutstats(struct xdr_stream *xdr) +{ + return decode_op_hdr(xdr, OP_LAYOUTSTATS); +} + +static int decode_clone(struct xdr_stream *xdr) +{ + return decode_op_hdr(xdr, OP_CLONE); +} + +static int decode_layouterror(struct xdr_stream *xdr) +{ + return decode_op_hdr(xdr, OP_LAYOUTERROR); +} + +static int decode_setxattr(struct xdr_stream *xdr, + struct nfs4_change_info *cinfo) +{ + int status; + + status = decode_op_hdr(xdr, OP_SETXATTR); + if (status) + goto out; + status = decode_change_info(xdr, cinfo); +out: + return status; +} + +static int decode_getxattr(struct xdr_stream *xdr, + struct nfs42_getxattrres *res, + struct rpc_rqst *req) +{ + int status; + __be32 *p; + u32 len, rdlen; + + status = decode_op_hdr(xdr, OP_GETXATTR); + if (status) + return status; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + + len = be32_to_cpup(p); + + /* + * Only check against the page length here. The actual + * requested length may be smaller, but that is only + * checked against after possibly caching a valid reply. + */ + if (len > req->rq_rcv_buf.page_len) + return -ERANGE; + + res->xattr_len = len; + + if (len > 0) { + rdlen = xdr_read_pages(xdr, len); + if (rdlen < len) + return -EIO; + } + + return 0; +} + +static int decode_removexattr(struct xdr_stream *xdr, + struct nfs4_change_info *cinfo) +{ + int status; + + status = decode_op_hdr(xdr, OP_REMOVEXATTR); + if (status) + goto out; + + status = decode_change_info(xdr, cinfo); +out: + return status; +} + +static int decode_listxattrs(struct xdr_stream *xdr, + struct nfs42_listxattrsres *res) +{ + int status; + __be32 *p; + u32 count, len, ulen; + size_t left, copied; + char *buf; + + status = decode_op_hdr(xdr, OP_LISTXATTRS); + if (status) { + /* + * Special case: for LISTXATTRS, NFS4ERR_TOOSMALL + * should be translated to ERANGE. + */ + if (status == -ETOOSMALL) + status = -ERANGE; + /* + * Special case: for LISTXATTRS, NFS4ERR_NOXATTR + * should be translated to success with zero-length reply. + */ + if (status == -ENODATA) { + res->eof = true; + status = 0; + } + goto out; + } + + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + + xdr_decode_hyper(p, &res->cookie); + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + + left = res->xattr_len; + buf = res->xattr_buf; + + count = be32_to_cpup(p); + copied = 0; + + /* + * We have asked for enough room to encode the maximum number + * of possible attribute names, so everything should fit. + * + * But, don't rely on that assumption. Just decode entries + * until they don't fit anymore, just in case the server did + * something odd. + */ + while (count--) { + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + + len = be32_to_cpup(p); + if (len > (XATTR_NAME_MAX - XATTR_USER_PREFIX_LEN)) { + status = -ERANGE; + goto out; + } + + p = xdr_inline_decode(xdr, len); + if (unlikely(!p)) + return -EIO; + + ulen = len + XATTR_USER_PREFIX_LEN + 1; + if (buf) { + if (ulen > left) { + status = -ERANGE; + goto out; + } + + memcpy(buf, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN); + memcpy(buf + XATTR_USER_PREFIX_LEN, p, len); + + buf[ulen - 1] = 0; + buf += ulen; + left -= ulen; + } + copied += ulen; + } + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + + res->eof = be32_to_cpup(p); + res->copied = copied; + +out: + if (status == -ERANGE && res->xattr_len == XATTR_LIST_MAX) + status = -E2BIG; + + return status; +} + +/* + * Decode ALLOCATE request + */ +static int nfs4_xdr_dec_allocate(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs42_falloc_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_allocate(xdr, res); + if (status) + goto out; + decode_getfattr(xdr, res->falloc_fattr, res->falloc_server); +out: + return status; +} + +/* + * Decode COPY response + */ +static int nfs4_xdr_dec_copy(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs42_copy_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_savefh(xdr); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_copy(xdr, res); + if (status) + goto out; + if (res->commit_res.verf) + status = decode_commit(xdr, &res->commit_res); +out: + return status; +} + +/* + * Decode OFFLOAD_CANCEL response + */ +static int nfs4_xdr_dec_offload_cancel(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs42_offload_status_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->osr_seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_offload_cancel(xdr, res); + +out: + return status; +} + +/* + * Decode OFFLOAD_STATUS response + */ +static int nfs4_xdr_dec_offload_status(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs42_offload_status_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->osr_seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_offload_status(xdr, res); + +out: + return status; +} + +/* + * Decode COPY_NOTIFY response + */ +static int nfs4_xdr_dec_copy_notify(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs42_copy_notify_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->cnr_seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_copy_notify(xdr, res); + +out: + return status; +} + +/* + * Decode DEALLOCATE request + */ +static int nfs4_xdr_dec_deallocate(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs42_falloc_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_deallocate(xdr, res); + if (status) + goto out; + decode_getfattr(xdr, res->falloc_fattr, res->falloc_server); +out: + return status; +} + +/* + * Decode ZERO_RANGE request + */ +static int nfs4_xdr_dec_zero_range(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs42_falloc_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_deallocate(xdr, res); + if (status) + goto out; + status = decode_allocate(xdr, res); + if (status) + goto out; + decode_getfattr(xdr, res->falloc_fattr, res->falloc_server); +out: + return status; +} + +/* + * Decode READ_PLUS request + */ +static int nfs4_xdr_dec_read_plus(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs_pgio_res *res = data; + struct compound_hdr hdr; + int status; + + xdr_set_scratch_buffer(xdr, res->scratch, READ_PLUS_SCRATCH_SIZE); + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_read_plus(xdr, res); + if (!status) + status = res->count; +out: + return status; +} + +/* + * Decode SEEK request + */ +static int nfs4_xdr_dec_seek(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs42_seek_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_seek(xdr, res); +out: + return status; +} + +/* + * Decode LAYOUTSTATS request + */ +static int nfs4_xdr_dec_layoutstats(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs42_layoutstat_res *res = data; + struct compound_hdr hdr; + int status, i; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + WARN_ON(res->num_dev > PNFS_LAYOUTSTATS_MAXDEV); + for (i = 0; i < res->num_dev; i++) { + status = decode_layoutstats(xdr); + if (status) + goto out; + } +out: + res->rpc_status = status; + return status; +} + +/* + * Decode CLONE request + */ +static int nfs4_xdr_dec_clone(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs42_clone_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_savefh(xdr); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_clone(xdr); + if (status) + goto out; + decode_getfattr(xdr, res->dst_fattr, res->server); +out: + res->rpc_status = status; + return status; +} + +/* + * Decode LAYOUTERROR request + */ +static int nfs4_xdr_dec_layouterror(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs42_layouterror_res *res = data; + struct compound_hdr hdr; + int status, i; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + + for (i = 0; i < res->num_errors && status == 0; i++) + status = decode_layouterror(xdr); +out: + res->rpc_status = status; + return status; +} + +/* + * Decode SETXATTR request + */ +static int nfs4_xdr_dec_setxattr(struct rpc_rqst *req, struct xdr_stream *xdr, + void *data) +{ + struct nfs42_setxattrres *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, req); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_setxattr(xdr, &res->cinfo); + if (status) + goto out; + status = decode_getfattr(xdr, res->fattr, res->server); +out: + return status; +} + +/* + * Decode GETXATTR request + */ +static int nfs4_xdr_dec_getxattr(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, void *data) +{ + struct nfs42_getxattrres *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_getxattr(xdr, res, rqstp); +out: + return status; +} + +/* + * Decode LISTXATTR request + */ +static int nfs4_xdr_dec_listxattrs(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, void *data) +{ + struct nfs42_listxattrsres *res = data; + struct compound_hdr hdr; + int status; + + xdr_set_scratch_folio(xdr, res->scratch); + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_listxattrs(xdr, res); +out: + return status; +} + +/* + * Decode REMOVEXATTR request + */ +static int nfs4_xdr_dec_removexattr(struct rpc_rqst *req, + struct xdr_stream *xdr, void *data) +{ + struct nfs42_removexattrres *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, req); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + + status = decode_removexattr(xdr, &res->cinfo); + if (status) + goto out; + status = decode_getfattr(xdr, res->fattr, res->server); +out: + return status; +} +#endif /* __LINUX_FS_NFS_NFS4_2XDR_H */ diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h new file mode 100644 index 000000000..b48e5b87c --- /dev/null +++ b/fs/nfs/nfs4_fs.h @@ -0,0 +1,682 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * linux/fs/nfs/nfs4_fs.h + * + * Copyright (C) 2005 Trond Myklebust + * + * NFSv4-specific filesystem definitions and declarations + */ + +#ifndef __LINUX_FS_NFS_NFS4_FS_H +#define __LINUX_FS_NFS_NFS4_FS_H + +#if defined(CONFIG_NFS_V4_0) +#define NFS4_MIN_MINOR_VERSION 0 +#else +#define NFS4_MIN_MINOR_VERSION 1 +#endif + +#if defined(CONFIG_NFS_V4_2) +#define NFS4_MAX_MINOR_VERSION 2 +#else +#define NFS4_MAX_MINOR_VERSION 1 +#endif + +#if IS_ENABLED(CONFIG_NFS_V4) + +#define NFS4_MAX_LOOP_ON_RECOVER (10) + +#include +#include + +struct idmap; + +enum nfs4_client_state { + NFS4CLNT_MANAGER_RUNNING = 0, + NFS4CLNT_CHECK_LEASE, + NFS4CLNT_LEASE_EXPIRED, + NFS4CLNT_RECLAIM_REBOOT, + NFS4CLNT_RECLAIM_NOGRACE, + NFS4CLNT_DELEGRETURN, + NFS4CLNT_SESSION_RESET, + NFS4CLNT_LEASE_CONFIRM, + NFS4CLNT_SERVER_SCOPE_MISMATCH, + NFS4CLNT_PURGE_STATE, + NFS4CLNT_BIND_CONN_TO_SESSION, + NFS4CLNT_MOVED, + NFS4CLNT_LEASE_MOVED, + NFS4CLNT_DELEGATION_EXPIRED, + NFS4CLNT_RUN_MANAGER, + NFS4CLNT_MANAGER_AVAILABLE, + NFS4CLNT_RECALL_RUNNING, + NFS4CLNT_RECALL_ANY_LAYOUT_READ, + NFS4CLNT_RECALL_ANY_LAYOUT_RW, + NFS4CLNT_DELEGRETURN_DELAYED, +}; + +#define NFS4_RENEW_TIMEOUT 0x01 +#define NFS4_RENEW_DELEGATION_CB 0x02 + +struct nfs_seqid_counter; +struct nfs4_minor_version_ops { + u32 minor_version; + unsigned init_caps; + + int (*init_client)(struct nfs_client *); + void (*shutdown_client)(struct nfs_client *); + bool (*match_stateid)(const nfs4_stateid *, + const nfs4_stateid *); + int (*find_root_sec)(struct nfs_server *, struct nfs_fh *, + struct nfs_fattr *); + void (*free_lock_state)(struct nfs_server *, + struct nfs4_lock_state *); + int (*test_and_free_expired)(struct nfs_server *, + nfs4_stateid *, const struct cred *); + struct nfs_seqid * + (*alloc_seqid)(struct nfs_seqid_counter *, gfp_t); + void (*session_trunk)(struct rpc_clnt *clnt, + struct rpc_xprt *xprt, void *data); + const struct rpc_call_ops *call_sync_ops; + const struct nfs4_sequence_slot_ops *sequence_slot_ops; + const struct nfs4_state_recovery_ops *reboot_recovery_ops; + const struct nfs4_state_recovery_ops *nograce_recovery_ops; + const struct nfs4_state_maintenance_ops *state_renewal_ops; + const struct nfs4_mig_recovery_ops *mig_recovery_ops; +}; + +#define NFS_SEQID_CONFIRMED 1 +struct nfs_seqid_counter { + ktime_t create_time; + u64 owner_id; + int flags; + u32 counter; + spinlock_t lock; /* Protects the list */ + struct list_head list; /* Defines sequence of RPC calls */ + struct rpc_wait_queue wait; /* RPC call delay queue */ +}; + +struct nfs_seqid { + struct nfs_seqid_counter *sequence; + struct list_head list; + struct rpc_task *task; +}; + +static inline void nfs_confirm_seqid(struct nfs_seqid_counter *seqid, int status) +{ + if (seqid_mutating_err(-status)) + seqid->flags |= NFS_SEQID_CONFIRMED; +} + +/* + * NFS4 state_owners and lock_owners are simply labels for ordered + * sequences of RPC calls. Their sole purpose is to provide once-only + * semantics by allowing the server to identify replayed requests. + */ +struct nfs4_state_owner { + struct nfs_server *so_server; + struct list_head so_lru; + unsigned long so_expires; + struct rb_node so_server_node; + + const struct cred *so_cred; /* Associated cred */ + + spinlock_t so_lock; + atomic_t so_count; + unsigned long so_flags; + struct list_head so_states; + struct nfs_seqid_counter so_seqid; + struct mutex so_delegreturn_mutex; +}; + +enum { + NFS_OWNER_RECLAIM_REBOOT, + NFS_OWNER_RECLAIM_NOGRACE +}; + +#define NFS_LOCK_NEW 0 +#define NFS_LOCK_RECLAIM 1 +#define NFS_LOCK_EXPIRED 2 + +/* + * struct nfs4_state maintains the client-side state for a given + * (state_owner,inode) tuple (OPEN) or state_owner (LOCK). + * + * OPEN: + * In order to know when to OPEN_DOWNGRADE or CLOSE the state on the server, + * we need to know how many files are open for reading or writing on a + * given inode. This information too is stored here. + * + * LOCK: one nfs4_state (LOCK) to hold the lock stateid nfs4_state(OPEN) + */ + +struct nfs4_lock_state { + struct list_head ls_locks; /* Other lock stateids */ + struct nfs4_state * ls_state; /* Pointer to open state */ +#define NFS_LOCK_INITIALIZED 0 +#define NFS_LOCK_LOST 1 +#define NFS_LOCK_UNLOCKING 2 + unsigned long ls_flags; + struct nfs_seqid_counter ls_seqid; + nfs4_stateid ls_stateid; + refcount_t ls_count; + fl_owner_t ls_owner; +}; + +/* bits for nfs4_state->flags */ +enum { + LK_STATE_IN_USE, + NFS_DELEGATED_STATE, /* Current stateid is delegation */ + NFS_OPEN_STATE, /* OPEN stateid is set */ + NFS_O_RDONLY_STATE, /* OPEN stateid has read-only state */ + NFS_O_WRONLY_STATE, /* OPEN stateid has write-only state */ + NFS_O_RDWR_STATE, /* OPEN stateid has read/write state */ + NFS_STATE_RECLAIM_REBOOT, /* OPEN stateid server rebooted */ + NFS_STATE_RECLAIM_NOGRACE, /* OPEN stateid needs to recover state */ + NFS_STATE_POSIX_LOCKS, /* Posix locks are supported */ + NFS_STATE_RECOVERY_FAILED, /* OPEN stateid state recovery failed */ + NFS_STATE_MAY_NOTIFY_LOCK, /* server may CB_NOTIFY_LOCK */ + NFS_STATE_CHANGE_WAIT, /* A state changing operation is outstanding */ + NFS_CLNT_DST_SSC_COPY_STATE, /* dst server open state on client*/ + NFS_CLNT_SRC_SSC_COPY_STATE, /* src server open state on client*/ + NFS_SRV_SSC_COPY_STATE, /* ssc state on the dst server */ +}; + +struct nfs4_state { + struct list_head open_states; /* List of states for the same state_owner */ + struct list_head inode_states; /* List of states for the same inode */ + struct list_head lock_states; /* List of subservient lock stateids */ + + struct nfs4_state_owner *owner; /* Pointer to the open owner */ + struct inode *inode; /* Pointer to the inode */ + + unsigned long flags; /* Do we hold any locks? */ + spinlock_t state_lock; /* Protects the lock_states list */ + + seqlock_t seqlock; /* Protects the stateid/open_stateid */ + nfs4_stateid stateid; /* Current stateid: may be delegation */ + nfs4_stateid open_stateid; /* OPEN stateid */ + + /* The following 3 fields are protected by owner->so_lock */ + unsigned int n_rdonly; /* Number of read-only references */ + unsigned int n_wronly; /* Number of write-only references */ + unsigned int n_rdwr; /* Number of read/write references */ + fmode_t state; /* State on the server (R,W, or RW) */ + refcount_t count; + + wait_queue_head_t waitq; + struct rcu_head rcu_head; +}; + + +struct nfs4_exception { + struct nfs4_state *state; + struct inode *inode; + nfs4_stateid *stateid; + long timeout; + unsigned short retrans; + unsigned char task_is_privileged : 1; + unsigned char delay : 1, + recovering : 1, + retry : 1; + bool interruptible; +}; + +struct nfs4_state_recovery_ops { + int owner_flag_bit; + int state_flag_bit; + int (*recover_open)(struct nfs4_state_owner *, struct nfs4_state *); + int (*recover_lock)(struct nfs4_state *, struct file_lock *); + int (*establish_clid)(struct nfs_client *, const struct cred *); + int (*reclaim_complete)(struct nfs_client *, const struct cred *); + int (*detect_trunking)(struct nfs_client *, struct nfs_client **, + const struct cred *); +}; + +struct nfs4_opendata { + struct kref kref; + struct nfs_openargs o_arg; + struct nfs_openres o_res; + struct nfs_open_confirmargs c_arg; + struct nfs_open_confirmres c_res; + struct nfs4_string owner_name; + struct nfs4_string group_name; + struct nfs4_label *a_label; + struct nfs_fattr f_attr; + struct dentry *dir; + struct dentry *dentry; + struct nfs4_state_owner *owner; + struct nfs4_state *state; + struct iattr attrs; + struct nfs4_layoutget *lgp; + unsigned long timestamp; + bool rpc_done; + bool file_created; + bool is_recover; + bool cancelled; + int rpc_status; +}; + +struct nfs4_add_xprt_data { + struct nfs_client *clp; + const struct cred *cred; +}; + +struct nfs4_sequence_slot_ops { + int (*process)(struct rpc_task *, struct nfs4_sequence_res *); + int (*done)(struct rpc_task *, struct nfs4_sequence_res *); + void (*free_slot)(struct nfs4_sequence_res *); +}; + +struct nfs4_state_maintenance_ops { + int (*sched_state_renewal)(struct nfs_client *, const struct cred *, unsigned); + const struct cred * (*get_state_renewal_cred)(struct nfs_client *); + int (*renew_lease)(struct nfs_client *, const struct cred *); +}; + +struct nfs4_mig_recovery_ops { + int (*get_locations)(struct nfs_server *, struct nfs_fh *, + struct nfs4_fs_locations *, struct page *, const struct cred *); + int (*fsid_present)(struct inode *, const struct cred *); +}; + +extern const struct dentry_operations nfs4_dentry_operations; + +/* dir.c */ +int nfs_atomic_open(struct inode *, struct dentry *, struct file *, + unsigned, umode_t); + +/* fs_context.c */ +extern struct file_system_type nfs4_fs_type; + +/* nfs4client.c */ +struct nfs_net; +int nfs4_match_client(struct nfs_client *pos, struct nfs_client *new, + struct nfs_client **prev, struct nfs_net *nn); + +/* nfs4namespace.c */ +struct rpc_clnt *nfs4_negotiate_security(struct rpc_clnt *, struct inode *, + const struct qstr *); +int nfs4_submount(struct fs_context *, struct nfs_server *); +int nfs4_replace_transport(struct nfs_server *server, + const struct nfs4_fs_locations *locations); +size_t nfs_parse_server_name(char *string, size_t len, struct sockaddr_storage *ss, + size_t salen, struct net *net, int port); +/* nfs4proc.c */ +struct nfs4_call_sync_data { + const struct nfs_server *seq_server; + struct nfs4_sequence_args *seq_args; + struct nfs4_sequence_res *seq_res; +}; + +extern int nfs4_handle_exception(struct nfs_server *, int, struct nfs4_exception *); +extern int nfs4_async_handle_error(struct rpc_task *task, + struct nfs_server *server, + struct nfs4_state *state, long *timeout); +extern int nfs4_call_sync(struct rpc_clnt *, struct nfs_server *, + struct rpc_message *, struct nfs4_sequence_args *, + struct nfs4_sequence_res *, int); +extern int nfs4_call_sync_sequence(struct rpc_clnt *clnt, + struct nfs_server *server, + struct rpc_message *msg, + struct nfs4_sequence_args *args, + struct nfs4_sequence_res *res); +extern void nfs4_init_sequence(struct nfs_client *clp, struct nfs4_sequence_args *, + struct nfs4_sequence_res *, int, int); +extern int nfs4_proc_setclientid(struct nfs_client *, u32, unsigned short, const struct cred *, struct nfs4_setclientid_res *); +extern int nfs4_proc_setclientid_confirm(struct nfs_client *, struct nfs4_setclientid_res *arg, const struct cred *); +extern void renew_lease(const struct nfs_server *server, unsigned long timestamp); +extern int nfs4_proc_get_rootfh(struct nfs_server *, struct nfs_fh *, + struct nfs_fattr *, bool); +extern int nfs4_proc_bind_conn_to_session(struct nfs_client *, const struct cred *cred); +extern int nfs4_proc_exchange_id(struct nfs_client *clp, const struct cred *cred); +extern int nfs4_destroy_clientid(struct nfs_client *clp); +extern int nfs4_init_clientid(struct nfs_client *, const struct cred *); +extern int nfs41_init_clientid(struct nfs_client *, const struct cred *); +extern int nfs4_do_close(struct nfs4_state *state, gfp_t gfp_mask, int wait); +extern int nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *fhandle); +extern int nfs4_proc_fs_locations(struct rpc_clnt *, struct inode *, const struct qstr *, + struct nfs4_fs_locations *, struct page *); +extern int nfs4_proc_get_locations(struct nfs_server *, struct nfs_fh *, + struct nfs4_fs_locations *, + struct page *page, const struct cred *); +extern int nfs4_proc_fsid_present(struct inode *, const struct cred *); +extern struct rpc_clnt *nfs4_proc_lookup_mountpoint(struct inode *, + struct dentry *, + struct nfs_fh *, + struct nfs_fattr *); +extern int nfs4_proc_secinfo(struct inode *, const struct qstr *, struct nfs4_secinfo_flavors *); +extern const struct xattr_handler * const nfs4_xattr_handlers[]; +extern int nfs4_set_rw_stateid(nfs4_stateid *stateid, + const struct nfs_open_context *ctx, + const struct nfs_lock_context *l_ctx, + fmode_t fmode); +extern void nfs4_bitmask_set(__u32 bitmask[], const __u32 src[], + struct inode *inode, unsigned long cache_validity); +extern int nfs4_proc_getattr(struct nfs_server *server, struct nfs_fh *fhandle, + struct nfs_fattr *fattr, struct inode *inode); +extern int update_open_stateid(struct nfs4_state *state, + const nfs4_stateid *open_stateid, + const nfs4_stateid *deleg_stateid, + fmode_t fmode); +extern int nfs4_proc_setlease(struct file *file, int arg, + struct file_lease **lease, void **priv); +extern int nfs4_proc_get_lease_time(struct nfs_client *clp, + struct nfs_fsinfo *fsinfo); +extern void nfs4_update_changeattr(struct inode *dir, + struct nfs4_change_info *cinfo, + unsigned long timestamp, + unsigned long cache_validity); +extern int nfs4_buf_to_pages_noslab(const void *buf, size_t buflen, + struct page **pages); +extern int nfs4_open_reclaim(struct nfs4_state_owner *, struct nfs4_state *); +extern int nfs4_open_expired(struct nfs4_state_owner *, struct nfs4_state *); +extern int nfs4_lock_reclaim(struct nfs4_state *state, struct file_lock *request); +extern int nfs4_lock_expired(struct nfs4_state *state, struct file_lock *request); +extern void nfs_state_clear_delegation(struct nfs4_state *state); +extern void nfs_finish_clear_delegation_stateid(struct nfs4_state *state, + const nfs4_stateid *stateid); +extern void nfs_state_clear_open_state_flags(struct nfs4_state *state); +extern void do_renew_lease(struct nfs_client *clp, unsigned long timestamp); +extern bool nfs4_match_stateid(const nfs4_stateid *s1, const nfs4_stateid *s2); +extern int nfs4_find_root_sec(struct nfs_server *server, struct nfs_fh *fhandle, + struct nfs_fattr *fattr); + +extern int nfs41_sequence_done(struct rpc_task *, struct nfs4_sequence_res *); +extern int nfs4_proc_create_session(struct nfs_client *, const struct cred *); +extern int nfs4_proc_destroy_session(struct nfs4_session *, const struct cred *); +extern int nfs4_proc_layoutcommit(struct nfs4_layoutcommit_data *data, + bool sync); +extern int nfs4_detect_session_trunking(struct nfs_client *clp, + struct nfs41_exchange_id_res *res, struct rpc_xprt *xprt); + +static inline bool +is_ds_only_client(struct nfs_client *clp) +{ + return (clp->cl_exchange_flags & EXCHGID4_FLAG_MASK_PNFS) == + EXCHGID4_FLAG_USE_PNFS_DS; +} + +static inline bool +is_ds_client(struct nfs_client *clp) +{ + return clp->cl_exchange_flags & EXCHGID4_FLAG_USE_PNFS_DS; +} + +static inline bool +_nfs4_state_protect(struct nfs_client *clp, unsigned long sp4_mode, + struct rpc_clnt **clntp, struct rpc_message *msg) +{ + rpc_authflavor_t flavor; + + if (sp4_mode == NFS_SP4_MACH_CRED_CLEANUP || + sp4_mode == NFS_SP4_MACH_CRED_PNFS_CLEANUP) { + /* Using machine creds for cleanup operations + * is only relevent if the client credentials + * might expire. So don't bother for + * RPC_AUTH_UNIX. If file was only exported to + * sec=sys, the PUTFH would fail anyway. + */ + if ((*clntp)->cl_auth->au_flavor == RPC_AUTH_UNIX) + return false; + } + if (test_bit(sp4_mode, &clp->cl_sp4_flags)) { + msg->rpc_cred = rpc_machine_cred(); + + flavor = clp->cl_rpcclient->cl_auth->au_flavor; + WARN_ON_ONCE(flavor != RPC_AUTH_GSS_KRB5I && + flavor != RPC_AUTH_GSS_KRB5P); + *clntp = clp->cl_rpcclient; + + return true; + } + return false; +} + +/* + * Function responsible for determining if an rpc_message should use the + * machine cred under SP4_MACH_CRED and if so switching the credential and + * authflavor (using the nfs_client's rpc_clnt which will be krb5i/p). + * Should be called before rpc_call_sync/rpc_call_async. + */ +static inline void +nfs4_state_protect(struct nfs_client *clp, unsigned long sp4_mode, + struct rpc_clnt **clntp, struct rpc_message *msg) +{ + _nfs4_state_protect(clp, sp4_mode, clntp, msg); +} + +/* + * Special wrapper to nfs4_state_protect for write. + * If WRITE can use machine cred but COMMIT cannot, make sure all writes + * that use machine cred use NFS_FILE_SYNC. + */ +static inline void +nfs4_state_protect_write(struct nfs_client *clp, struct rpc_clnt **clntp, + struct rpc_message *msg, struct nfs_pgio_header *hdr) +{ + if (_nfs4_state_protect(clp, NFS_SP4_MACH_CRED_WRITE, clntp, msg) && + !test_bit(NFS_SP4_MACH_CRED_COMMIT, &clp->cl_sp4_flags)) + hdr->args.stable = NFS_FILE_SYNC; +} + +extern const struct nfs4_minor_version_ops *nfs_v4_minor_ops[]; + +extern const u32 nfs4_fattr_bitmap[3]; +extern const u32 nfs4_statfs_bitmap[3]; +extern const u32 nfs4_pathconf_bitmap[3]; +extern const u32 nfs4_fsinfo_bitmap[3]; +extern const u32 nfs4_fs_locations_bitmap[3]; + +void nfs41_shutdown_client(struct nfs_client *); +int nfs41_init_client(struct nfs_client *); +void nfs4_free_client(struct nfs_client *); + +struct nfs_client *nfs4_alloc_client(const struct nfs_client_initdata *); + +/* nfs4renewd.c */ +extern void nfs4_schedule_state_renewal(struct nfs_client *); +extern void nfs4_kill_renewd(struct nfs_client *); +extern void nfs4_renew_state(struct work_struct *); +extern void nfs4_set_lease_period(struct nfs_client *clp, u32 period); + +/* nfs4state.c */ +extern const nfs4_stateid current_stateid; + +const struct cred *nfs4_get_clid_cred(struct nfs_client *clp); +const struct cred *nfs4_get_machine_cred(struct nfs_client *clp); +const struct cred *nfs4_get_renew_cred(struct nfs_client *clp); +int nfs4_discover_server_trunking(struct nfs_client *clp, + struct nfs_client **); +int nfs40_discover_server_trunking(struct nfs_client *clp, + struct nfs_client **, const struct cred *); +int nfs41_discover_server_trunking(struct nfs_client *clp, + struct nfs_client **, const struct cred *); +extern void nfs4_schedule_session_recovery(struct nfs4_session *, int); +extern void nfs41_notify_server(struct nfs_client *); +bool nfs4_check_serverowner_major_id(struct nfs41_server_owner *o1, + struct nfs41_server_owner *o2); + +extern struct nfs4_state_owner *nfs4_get_state_owner(struct nfs_server *, const struct cred *, gfp_t); +extern void nfs4_put_state_owner(struct nfs4_state_owner *); +extern void nfs4_purge_state_owners(struct nfs_server *, struct list_head *); +extern void nfs4_free_state_owners(struct list_head *head); +extern struct nfs4_state * nfs4_get_open_state(struct inode *, struct nfs4_state_owner *); +extern void nfs4_put_open_state(struct nfs4_state *); +extern void nfs4_close_state(struct nfs4_state *, fmode_t); +extern void nfs4_close_sync(struct nfs4_state *, fmode_t); +extern void nfs4_state_set_mode_locked(struct nfs4_state *, fmode_t); +extern void nfs_inode_find_state_and_recover(struct inode *inode, + const nfs4_stateid *stateid); +extern int nfs4_state_mark_reclaim_nograce(struct nfs_client *, struct nfs4_state *); +extern void nfs4_schedule_lease_recovery(struct nfs_client *); +extern int nfs4_wait_clnt_recover(struct nfs_client *clp); +extern int nfs4_client_recover_expired_lease(struct nfs_client *clp); +extern void nfs4_schedule_state_manager(struct nfs_client *); +extern void nfs4_schedule_path_down_recovery(struct nfs_client *clp); +extern int nfs4_schedule_stateid_recovery(const struct nfs_server *, struct nfs4_state *); +extern int nfs4_schedule_migration_recovery(const struct nfs_server *); +extern void nfs4_schedule_lease_moved_recovery(struct nfs_client *); +extern void nfs41_handle_sequence_flag_errors(struct nfs_client *clp, u32 flags, bool); +extern void nfs41_handle_server_scope(struct nfs_client *, + struct nfs41_server_scope **); +extern void nfs4_put_lock_state(struct nfs4_lock_state *lsp); +extern int nfs4_set_lock_state(struct nfs4_state *state, struct file_lock *fl); +extern int nfs4_select_rw_stateid(struct nfs4_state *, fmode_t, + const struct nfs_lock_context *, nfs4_stateid *, + const struct cred **); +extern bool nfs4_copy_open_stateid(nfs4_stateid *dst, + struct nfs4_state *state); + +extern struct nfs_seqid *nfs_alloc_seqid(struct nfs_seqid_counter *counter, gfp_t gfp_mask); +extern int nfs_wait_on_sequence(struct nfs_seqid *seqid, struct rpc_task *task); +extern void nfs_increment_open_seqid(int status, struct nfs_seqid *seqid); +extern void nfs_increment_lock_seqid(int status, struct nfs_seqid *seqid); +extern void nfs_release_seqid(struct nfs_seqid *seqid); +extern void nfs_free_seqid(struct nfs_seqid *seqid); +extern int nfs4_setup_sequence(struct nfs_client *client, + struct nfs4_sequence_args *args, + struct nfs4_sequence_res *res, + struct rpc_task *task); +extern int nfs4_sequence_done(struct rpc_task *task, + struct nfs4_sequence_res *res); + +extern void nfs4_free_lock_state(struct nfs_server *server, struct nfs4_lock_state *lsp); +extern int nfs4_proc_commit(struct file *dst, __u64 offset, __u32 count, struct nfs_commitres *res); +extern const nfs4_stateid zero_stateid; +extern const nfs4_stateid invalid_stateid; + +/* nfs4super.c */ +struct nfs_mount_info; +extern struct nfs_subversion nfs_v4; +extern bool nfs4_disable_idmapping; +extern unsigned short max_session_slots; +extern unsigned short max_session_cb_slots; +extern unsigned short send_implementation_id; +extern bool recover_lost_locks; +extern short nfs_delay_retrans; + +#define NFS4_CLIENT_ID_UNIQ_LEN (64) +extern char nfs4_client_id_uniquifier[NFS4_CLIENT_ID_UNIQ_LEN]; + +extern int nfs4_try_get_tree(struct fs_context *); +extern int nfs4_get_referral_tree(struct fs_context *); + +/* nfs4sysctl.c */ +#ifdef CONFIG_SYSCTL +int nfs4_register_sysctl(void); +void nfs4_unregister_sysctl(void); +#else +static inline int nfs4_register_sysctl(void) +{ + return 0; +} + +static inline void nfs4_unregister_sysctl(void) +{ +} +#endif + +/* nfs4xdr.c */ +extern const struct rpc_procinfo nfs4_procedures[]; + +#ifdef CONFIG_NFS_V4_2 +extern const u32 nfs42_maxsetxattr_overhead; +extern const u32 nfs42_maxgetxattr_overhead; +extern const u32 nfs42_maxlistxattrs_overhead; +#endif + +struct nfs4_mount_data; + +/* callback_xdr.c */ +extern const struct svc_version nfs4_callback_version1; +extern const struct svc_version nfs4_callback_version4; + +static inline void nfs4_stateid_copy(nfs4_stateid *dst, const nfs4_stateid *src) +{ + memcpy(dst->data, src->data, sizeof(dst->data)); + dst->type = src->type; +} + +static inline bool nfs4_stateid_match(const nfs4_stateid *dst, const nfs4_stateid *src) +{ + if (dst->type != src->type) + return false; + return memcmp(dst->data, src->data, sizeof(dst->data)) == 0; +} + +static inline bool nfs4_stateid_match_other(const nfs4_stateid *dst, const nfs4_stateid *src) +{ + return memcmp(dst->other, src->other, NFS4_STATEID_OTHER_SIZE) == 0; +} + +static inline bool nfs4_stateid_is_newer(const nfs4_stateid *s1, const nfs4_stateid *s2) +{ + return (s32)(be32_to_cpu(s1->seqid) - be32_to_cpu(s2->seqid)) > 0; +} + +static inline bool nfs4_stateid_is_next(const nfs4_stateid *s1, const nfs4_stateid *s2) +{ + u32 seq1 = be32_to_cpu(s1->seqid); + u32 seq2 = be32_to_cpu(s2->seqid); + + return seq2 == seq1 + 1U || (seq2 == 1U && seq1 == 0xffffffffU); +} + +static inline bool nfs4_stateid_match_or_older(const nfs4_stateid *dst, const nfs4_stateid *src) +{ + return nfs4_stateid_match_other(dst, src) && + !(src->seqid && nfs4_stateid_is_newer(dst, src)); +} + +static inline void nfs4_stateid_seqid_inc(nfs4_stateid *s1) +{ + u32 seqid = be32_to_cpu(s1->seqid); + + if (++seqid == 0) + ++seqid; + s1->seqid = cpu_to_be32(seqid); +} + +static inline bool nfs4_valid_open_stateid(const struct nfs4_state *state) +{ + return test_bit(NFS_STATE_RECOVERY_FAILED, &state->flags) == 0; +} + +static inline bool nfs4_state_match_open_stateid_other(const struct nfs4_state *state, + const nfs4_stateid *stateid) +{ + return test_bit(NFS_OPEN_STATE, &state->flags) && + nfs4_stateid_match_other(&state->open_stateid, stateid); +} + +/* nfs42xattr.c */ +#ifdef CONFIG_NFS_V4_2 +extern int __init nfs4_xattr_cache_init(void); +extern void nfs4_xattr_cache_exit(void); +extern void nfs4_xattr_cache_add(struct inode *inode, const char *name, + const char *buf, struct page **pages, + ssize_t buflen); +extern void nfs4_xattr_cache_remove(struct inode *inode, const char *name); +extern ssize_t nfs4_xattr_cache_get(struct inode *inode, const char *name, + char *buf, ssize_t buflen); +extern void nfs4_xattr_cache_set_list(struct inode *inode, const char *buf, + ssize_t buflen); +extern ssize_t nfs4_xattr_cache_list(struct inode *inode, char *buf, + ssize_t buflen); +extern void nfs4_xattr_cache_zap(struct inode *inode); +#else +static inline void nfs4_xattr_cache_zap(struct inode *inode) +{ +} +#endif /* CONFIG_NFS_V4_2 */ + +#else /* CONFIG_NFS_V4 */ + +#define nfs4_close_state(a, b) do { } while (0) +#define nfs4_close_sync(a, b) do { } while (0) +#define nfs4_state_protect(a, b, c, d) do { } while (0) +#define nfs4_state_protect_write(a, b, c, d) do { } while (0) + + +#endif /* CONFIG_NFS_V4 */ +#endif /* __LINUX_FS_NFS_NFS4_FS.H */ diff --git a/fs/nfs/nfs4client.c b/fs/nfs/nfs4client.c new file mode 100644 index 000000000..71c271a17 --- /dev/null +++ b/fs/nfs/nfs4client.c @@ -0,0 +1,1203 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2006 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" +#include "callback.h" +#include "delegation.h" +#include "nfs4session.h" +#include "nfs4idmap.h" +#include "pnfs.h" +#include "netns.h" +#include "sysfs.h" + +#define NFSDBG_FACILITY NFSDBG_CLIENT + +/* + * Get a unique NFSv4.0 callback identifier which will be used + * by the V4.0 callback service to lookup the nfs_client struct + */ +static int nfs_get_cb_ident_idr(struct nfs_client *clp, int minorversion) +{ + int ret = 0; + struct nfs_net *nn = net_generic(clp->cl_net, nfs_net_id); + + if (clp->rpc_ops->version != 4 || minorversion != 0) + return ret; + idr_preload(GFP_KERNEL); + spin_lock(&nn->nfs_client_lock); + ret = idr_alloc(&nn->cb_ident_idr, clp, 1, 0, GFP_NOWAIT); + if (ret >= 0) + clp->cl_cb_ident = ret; + spin_unlock(&nn->nfs_client_lock); + idr_preload_end(); + return ret < 0 ? ret : 0; +} + +/* + * Per auth flavor data server rpc clients + */ +struct nfs4_ds_server { + struct list_head list; /* ds_clp->cl_ds_clients */ + struct rpc_clnt *rpc_clnt; +}; + +/** + * nfs4_find_ds_client - Common lookup case for DS I/O + * @ds_clp: pointer to the DS's nfs_client + * @flavor: rpc auth flavour to match + */ +static struct nfs4_ds_server * +nfs4_find_ds_client(struct nfs_client *ds_clp, rpc_authflavor_t flavor) +{ + struct nfs4_ds_server *dss; + + rcu_read_lock(); + list_for_each_entry_rcu(dss, &ds_clp->cl_ds_clients, list) { + if (dss->rpc_clnt->cl_auth->au_flavor != flavor) + continue; + goto out; + } + dss = NULL; +out: + rcu_read_unlock(); + return dss; +} + +static struct nfs4_ds_server * +nfs4_add_ds_client(struct nfs_client *ds_clp, rpc_authflavor_t flavor, + struct nfs4_ds_server *new) +{ + struct nfs4_ds_server *dss; + + spin_lock(&ds_clp->cl_lock); + list_for_each_entry(dss, &ds_clp->cl_ds_clients, list) { + if (dss->rpc_clnt->cl_auth->au_flavor != flavor) + continue; + goto out; + } + if (new) + list_add_rcu(&new->list, &ds_clp->cl_ds_clients); + dss = new; +out: + spin_unlock(&ds_clp->cl_lock); /* need some lock to protect list */ + return dss; +} + +static struct nfs4_ds_server * +nfs4_alloc_ds_server(struct nfs_client *ds_clp, rpc_authflavor_t flavor) +{ + struct nfs4_ds_server *dss; + + dss = kmalloc_obj(*dss, GFP_NOFS); + if (dss == NULL) + return ERR_PTR(-ENOMEM); + + dss->rpc_clnt = rpc_clone_client_set_auth(ds_clp->cl_rpcclient, flavor); + if (IS_ERR(dss->rpc_clnt)) { + int err = PTR_ERR(dss->rpc_clnt); + kfree (dss); + return ERR_PTR(err); + } + INIT_LIST_HEAD(&dss->list); + + return dss; +} + +static void +nfs4_free_ds_server(struct nfs4_ds_server *dss) +{ + rpc_release_client(dss->rpc_clnt); + kfree(dss); +} + +/** + * nfs4_find_or_create_ds_client - Find or create a DS rpc client + * @ds_clp: pointer to the DS's nfs_client + * @inode: pointer to the inode + * + * Find or create a DS rpc client with th MDS server rpc client auth flavor + * in the nfs_client cl_ds_clients list. + */ +struct rpc_clnt * +nfs4_find_or_create_ds_client(struct nfs_client *ds_clp, struct inode *inode) +{ + struct nfs4_ds_server *dss, *new; + rpc_authflavor_t flavor = NFS_SERVER(inode)->client->cl_auth->au_flavor; + + dss = nfs4_find_ds_client(ds_clp, flavor); + if (dss != NULL) + goto out; + new = nfs4_alloc_ds_server(ds_clp, flavor); + if (IS_ERR(new)) + return ERR_CAST(new); + dss = nfs4_add_ds_client(ds_clp, flavor, new); + if (dss != new) + nfs4_free_ds_server(new); +out: + return dss->rpc_clnt; +} +EXPORT_SYMBOL_GPL(nfs4_find_or_create_ds_client); + +static void +nfs4_shutdown_ds_clients(struct nfs_client *clp) +{ + struct nfs4_ds_server *dss; + + while (!list_empty(&clp->cl_ds_clients)) { + dss = list_entry(clp->cl_ds_clients.next, + struct nfs4_ds_server, list); + list_del(&dss->list); + rpc_shutdown_client(dss->rpc_clnt); + kfree (dss); + } +} + +static void +nfs4_cleanup_callback(struct nfs_client *clp) +{ + struct nfs4_copy_state *cp_state; + + while (!list_empty(&clp->pending_cb_stateids)) { + cp_state = list_entry(clp->pending_cb_stateids.next, + struct nfs4_copy_state, copies); + list_del(&cp_state->copies); + kfree(cp_state); + } +} + +void nfs41_shutdown_client(struct nfs_client *clp) +{ + if (nfs4_has_session(clp)) { + nfs4_cleanup_callback(clp); + nfs4_shutdown_ds_clients(clp); + nfs4_destroy_session(clp->cl_session); + nfs4_destroy_clientid(clp); + } + +} + +struct nfs_client *nfs4_alloc_client(const struct nfs_client_initdata *cl_init) +{ + char buf[INET6_ADDRSTRLEN + 1]; + const char *ip_addr = cl_init->ip_addr; + struct nfs_client *clp = nfs_alloc_client(cl_init); + int err; + + if (IS_ERR(clp)) + return clp; + + err = nfs_get_cb_ident_idr(clp, cl_init->minorversion); + if (err) + goto error; + + if (cl_init->minorversion < NFS4_MIN_MINOR_VERSION || + cl_init->minorversion > NFS4_MAX_MINOR_VERSION) { + err = -EINVAL; + goto error; + } + + spin_lock_init(&clp->cl_lock); + INIT_DELAYED_WORK(&clp->cl_renewd, nfs4_renew_state); + INIT_LIST_HEAD(&clp->cl_ds_clients); + rpc_init_wait_queue(&clp->cl_rpcwaitq, "NFS client"); + clp->cl_state = 1 << NFS4CLNT_LEASE_EXPIRED; + clp->cl_mvops = nfs_v4_minor_ops[cl_init->minorversion]; + clp->cl_mig_gen = 1; + clp->cl_last_renewal = jiffies; + init_waitqueue_head(&clp->cl_lock_waitq); + INIT_LIST_HEAD(&clp->pending_cb_stateids); + + if (cl_init->minorversion != 0) + __set_bit(NFS_CS_INFINITE_SLOTS, &clp->cl_flags); + __set_bit(NFS_CS_DISCRTRY, &clp->cl_flags); + __set_bit(NFS_CS_NO_RETRANS_TIMEOUT, &clp->cl_flags); + if (test_bit(NFS_CS_PNFS, &cl_init->init_flags)) + __set_bit(NFS_CS_PNFS, &clp->cl_flags); + if (test_bit(NFS_CS_NETUNREACH_FATAL, &cl_init->init_flags)) + __set_bit(NFS_CS_NETUNREACH_FATAL, &clp->cl_flags); + /* + * Set up the connection to the server before we add add to the + * global list. + */ + err = nfs_create_rpc_client(clp, cl_init, RPC_AUTH_GSS_KRB5I); + if (err == -EINVAL) + err = nfs_create_rpc_client(clp, cl_init, RPC_AUTH_UNIX); + if (err < 0) + goto error; + + /* If no clientaddr= option was specified, find a usable cb address */ + if (ip_addr == NULL) { + struct sockaddr_storage cb_addr; + struct sockaddr *sap = (struct sockaddr *)&cb_addr; + + err = rpc_localaddr(clp->cl_rpcclient, sap, sizeof(cb_addr)); + if (err < 0) + goto error; + err = rpc_ntop(sap, buf, sizeof(buf)); + if (err < 0) + goto error; + ip_addr = (const char *)buf; + } + strscpy(clp->cl_ipaddr, ip_addr, sizeof(clp->cl_ipaddr)); + + err = nfs_idmap_new(clp); + if (err < 0) { + dprintk("%s: failed to create idmapper. Error = %d\n", + __func__, err); + goto error; + } + __set_bit(NFS_CS_IDMAP, &clp->cl_res_state); + return clp; + +error: + nfs_free_client(clp); + return ERR_PTR(err); +} + +/* + * Destroy the NFS4 callback service + */ +static void nfs4_destroy_callback(struct nfs_client *clp) +{ + if (__test_and_clear_bit(NFS_CS_CALLBACK, &clp->cl_res_state)) { + struct rpc_xprt *xprt; + + xprt = rcu_dereference_raw(clp->cl_rpcclient->cl_xprt); + nfs_callback_down(clp->cl_mvops->minor_version, clp->cl_net, + xprt); + } +} + +static void nfs4_shutdown_client(struct nfs_client *clp) +{ + if (__test_and_clear_bit(NFS_CS_RENEWD, &clp->cl_res_state)) + nfs4_kill_renewd(clp); + clp->cl_mvops->shutdown_client(clp); + nfs4_destroy_callback(clp); + if (__test_and_clear_bit(NFS_CS_IDMAP, &clp->cl_res_state)) + nfs_idmap_delete(clp); + + rpc_destroy_wait_queue(&clp->cl_rpcwaitq); + kfree(clp->cl_serverowner); + kfree(clp->cl_serverscope); + kfree(clp->cl_implid); + kfree(clp->cl_owner_id); +} + +void nfs4_free_client(struct nfs_client *clp) +{ + nfs4_shutdown_client(clp); + nfs_free_client(clp); +} + +/* + * Initialize the NFS4 callback service + */ +static int nfs4_init_callback(struct nfs_client *clp) +{ + struct rpc_xprt *xprt; + int error; + + xprt = rcu_dereference_raw(clp->cl_rpcclient->cl_xprt); + + if (nfs4_has_session(clp)) { + error = xprt_setup_backchannel(xprt, NFS41_BC_MIN_CALLBACKS); + if (error < 0) + return error; + } + + error = nfs_callback_up(clp->cl_mvops->minor_version, xprt); + if (error < 0) { + dprintk("%s: failed to start callback. Error = %d\n", + __func__, error); + return error; + } + __set_bit(NFS_CS_CALLBACK, &clp->cl_res_state); + + return 0; +} + +/** + * nfs41_init_client - nfs_client initialization tasks for NFSv4.1+ + * @clp: nfs_client to initialize + * + * Returns zero on success, or a negative errno if some error occurred. + */ +int nfs41_init_client(struct nfs_client *clp) +{ + struct nfs4_session *session = NULL; + + /* + * Create the session and mark it expired. + * When a SEQUENCE operation encounters the expired session + * it will do session recovery to initialize it. + */ + session = nfs4_alloc_session(clp); + if (!session) + return -ENOMEM; + + clp->cl_session = session; + + /* + * The create session reply races with the server back + * channel probe. Mark the client NFS_CS_SESSION_INITING + * so that the client back channel can find the + * nfs_client struct + */ + nfs_mark_client_ready(clp, NFS_CS_SESSION_INITING); + return 0; +} + +/* + * Initialize the minor version specific parts of an NFS4 client record + */ +static int nfs4_init_client_minor_version(struct nfs_client *clp) +{ + int ret; + + ret = clp->cl_mvops->init_client(clp); + if (ret) + return ret; + return nfs4_init_callback(clp); +} + +static void nfs4_add_trunk(struct nfs_client *clp, struct nfs_client *old) +{ + struct sockaddr_storage clp_addr, old_addr; + struct sockaddr *clp_sap = (struct sockaddr *)&clp_addr; + struct sockaddr *old_sap = (struct sockaddr *)&old_addr; + size_t clp_salen; + struct xprt_create xprt_args = { + .ident = old->cl_proto, + .net = old->cl_net, + .servername = old->cl_hostname, + }; + int max_connect = test_bit(NFS_CS_PNFS, &clp->cl_flags) ? + clp->cl_max_connect : old->cl_max_connect; + + if (clp->cl_proto != old->cl_proto) + return; + clp_salen = rpc_peeraddr(clp->cl_rpcclient, clp_sap, sizeof(clp_addr)); + rpc_peeraddr(old->cl_rpcclient, old_sap, sizeof(old_addr)); + + if (clp_addr.ss_family != old_addr.ss_family) + return; + + xprt_args.dstaddr = clp_sap; + xprt_args.addrlen = clp_salen; + + rpc_clnt_add_xprt(old->cl_rpcclient, &xprt_args, + rpc_clnt_test_and_add_xprt, &max_connect); +} + +/** + * nfs4_init_client - Initialise an NFS4 client record + * + * @clp: nfs_client to initialise + * @cl_init: pointer to nfs_client_initdata + * + * Returns pointer to an NFS client, or an ERR_PTR value. + */ +struct nfs_client *nfs4_init_client(struct nfs_client *clp, + const struct nfs_client_initdata *cl_init) +{ + struct nfs_client *old; + int error; + + if (clp->cl_cons_state == NFS_CS_READY) + /* the client is initialised already */ + return clp; + + error = nfs4_init_client_minor_version(clp); + if (error < 0) + goto error; + + error = nfs4_discover_server_trunking(clp, &old); + if (error < 0) + goto error; + + if (clp != old) { + clp->cl_preserve_clid = true; + /* + * Mark the client as having failed initialization so other + * processes walking the nfs_client_list in nfs_match_client() + * won't try to use it. + */ + nfs_mark_client_ready(clp, -EPERM); + if (old->cl_mvops->session_trunk) + nfs4_add_trunk(clp, old); + } + clear_bit(NFS_CS_TSM_POSSIBLE, &clp->cl_flags); + nfs_put_client(clp); + return old; + +error: + nfs_mark_client_ready(clp, error); + nfs_put_client(clp); + return ERR_PTR(error); +} + +static bool nfs4_match_client_owner_id(const struct nfs_client *clp1, + const struct nfs_client *clp2) +{ + if (clp1->cl_owner_id == NULL || clp2->cl_owner_id == NULL) + return true; + return strcmp(clp1->cl_owner_id, clp2->cl_owner_id) == 0; +} + +int nfs4_match_client(struct nfs_client *pos, struct nfs_client *new, + struct nfs_client **prev, struct nfs_net *nn) +{ + int status; + + if (pos->rpc_ops != new->rpc_ops) + return 1; + + if (pos->cl_minorversion != new->cl_minorversion) + return 1; + + /* If "pos" isn't marked ready, we can't trust the + * remaining fields in "pos", especially the client + * ID and serverowner fields. Wait for CREATE_SESSION + * to finish. */ + if (pos->cl_cons_state > NFS_CS_READY) { + refcount_inc(&pos->cl_count); + spin_unlock(&nn->nfs_client_lock); + + nfs_put_client(*prev); + *prev = pos; + + status = nfs_wait_client_init_complete(pos); + spin_lock(&nn->nfs_client_lock); + + if (status < 0) + return status; + } + + if (pos->cl_cons_state != NFS_CS_READY) + return 1; + + if (pos->cl_clientid != new->cl_clientid) + return 1; + + /* NFSv4.1 always uses the uniform string, however someone + * might switch the uniquifier string on us. + */ + if (!nfs4_match_client_owner_id(pos, new)) + return 1; + + return 0; +} + +/* + * Returns true if the server major ids match + */ +bool +nfs4_check_serverowner_major_id(struct nfs41_server_owner *o1, + struct nfs41_server_owner *o2) +{ + if (o1->major_id_sz != o2->major_id_sz) + return false; + return memcmp(o1->major_id, o2->major_id, o1->major_id_sz) == 0; +} + +/* + * Returns true if the server scopes match + */ +static bool +nfs4_check_server_scope(struct nfs41_server_scope *s1, + struct nfs41_server_scope *s2) +{ + if (s1->server_scope_sz != s2->server_scope_sz) + return false; + return memcmp(s1->server_scope, s2->server_scope, + s1->server_scope_sz) == 0; +} + +/** + * nfs4_detect_session_trunking - Checks for session trunking. + * @clp: original mount nfs_client + * @res: result structure from an exchange_id using the original mount + * nfs_client with a new multi_addr transport + * @xprt: pointer to the transport to add. + * + * Called after a successful EXCHANGE_ID on a multi-addr connection. + * Upon success, add the transport. + * + * Returns zero on success, otherwise -EINVAL + * + * Note: since the exchange_id for the new multi_addr transport uses the + * same nfs_client from the original mount, the cl_owner_id is reused, + * so eir_clientowner is the same. + */ +int nfs4_detect_session_trunking(struct nfs_client *clp, + struct nfs41_exchange_id_res *res, + struct rpc_xprt *xprt) +{ + /* Check eir_clientid */ + if (clp->cl_clientid != res->clientid) + goto out_err; + + /* Check eir_server_owner so_major_id */ + if (!nfs4_check_serverowner_major_id(clp->cl_serverowner, + res->server_owner)) + goto out_err; + + /* Check eir_server_owner so_minor_id */ + if (clp->cl_serverowner->minor_id != res->server_owner->minor_id) + goto out_err; + + /* Check eir_server_scope */ + if (!nfs4_check_server_scope(clp->cl_serverscope, res->server_scope)) + goto out_err; + + pr_info("NFS: %s: Session trunking succeeded for %s\n", + clp->cl_hostname, + xprt->address_strings[RPC_DISPLAY_ADDR]); + + return 0; +out_err: + pr_info("NFS: %s: Session trunking failed for %s\n", clp->cl_hostname, + xprt->address_strings[RPC_DISPLAY_ADDR]); + + return -EINVAL; +} + +/** + * nfs41_walk_client_list - Find nfs_client that matches a client/server owner + * + * @new: nfs_client with client ID to test + * @result: OUT: found nfs_client, or new + * @cred: credential to use for trunking test + * + * Returns zero, a negative errno, or a negative NFS4ERR status. + * If zero is returned, an nfs_client pointer is planted in "result." + * + * NB: nfs41_walk_client_list() relies on the new nfs_client being + * the last nfs_client on the list. + */ +int nfs41_walk_client_list(struct nfs_client *new, + struct nfs_client **result, + const struct cred *cred) +{ + struct nfs_net *nn = net_generic(new->cl_net, nfs_net_id); + struct nfs_client *pos, *prev = NULL; + int status = -NFS4ERR_STALE_CLIENTID; + + spin_lock(&nn->nfs_client_lock); + list_for_each_entry(pos, &nn->nfs_client_list, cl_share_link) { + + if (pos == new) + goto found; + + status = nfs4_match_client(pos, new, &prev, nn); + if (status < 0) + goto out; + if (status != 0) + continue; + + /* + * Note that session trunking is just a special subcase of + * client id trunking. In either case, we want to fall back + * to using the existing nfs_client. + */ + if (!nfs4_check_serverowner_major_id(pos->cl_serverowner, + new->cl_serverowner)) + continue; + +found: + refcount_inc(&pos->cl_count); + *result = pos; + status = 0; + break; + } + +out: + spin_unlock(&nn->nfs_client_lock); + nfs_put_client(prev); + return status; +} + +static void nfs4_destroy_server(struct nfs_server *server) +{ + LIST_HEAD(freeme); + + nfs_server_return_all_delegations(server); + unset_pnfs_layoutdriver(server); + nfs4_purge_state_owners(server, &freeme); + nfs4_free_state_owners(&freeme); + kfree(server->delegation_hash_table); +} + +/* + * NFSv4.0 callback thread helper + * + * Find a client by callback identifier + */ +struct nfs_client * +nfs4_find_client_ident(struct net *net, int cb_ident) +{ + struct nfs_client *clp; + struct nfs_net *nn = net_generic(net, nfs_net_id); + + spin_lock(&nn->nfs_client_lock); + clp = idr_find(&nn->cb_ident_idr, cb_ident); + if (clp) + refcount_inc(&clp->cl_count); + spin_unlock(&nn->nfs_client_lock); + return clp; +} + +/* Common match routine for v4.0 and v4.1 callback services */ +static bool nfs4_cb_match_client(const struct sockaddr *addr, + struct nfs_client *clp, u32 minorversion) +{ + struct sockaddr *clap = (struct sockaddr *)&clp->cl_addr; + + /* Don't match clients that failed to initialise */ + if (!(clp->cl_cons_state == NFS_CS_READY || + clp->cl_cons_state == NFS_CS_SESSION_INITING)) + return false; + + smp_rmb(); + + /* Match the version and minorversion */ + if (clp->rpc_ops->version != 4 || + clp->cl_minorversion != minorversion) + return false; + + /* Match only the IP address, not the port number */ + return rpc_cmp_addr(addr, clap); +} + +/* + * NFSv4.1 callback thread helper + * For CB_COMPOUND calls, find a client by IP address, protocol version, + * minorversion, and sessionID + * + * Returns NULL if no such client + */ +struct nfs_client * +nfs4_find_client_sessionid(struct net *net, const struct sockaddr *addr, + struct nfs4_sessionid *sid, u32 minorversion) +{ + struct nfs_client *clp; + struct nfs_net *nn = net_generic(net, nfs_net_id); + + spin_lock(&nn->nfs_client_lock); + list_for_each_entry(clp, &nn->nfs_client_list, cl_share_link) { + if (!nfs4_cb_match_client(addr, clp, minorversion)) + continue; + + if (!nfs4_has_session(clp)) + continue; + + /* Match sessionid*/ + if (memcmp(clp->cl_session->sess_id.data, + sid->data, NFS4_MAX_SESSIONID_LEN) != 0) + continue; + + refcount_inc(&clp->cl_count); + spin_unlock(&nn->nfs_client_lock); + return clp; + } + spin_unlock(&nn->nfs_client_lock); + return NULL; +} + +/* + * Set up an NFS4 client + */ +static int nfs4_set_client(struct nfs_server *server, + struct nfs_client_initdata *cl_init) +{ + struct nfs_client *clp; + + cl_init->nfs_mod = &nfs_v4; + cl_init->cred = server->cred; + + if (cl_init->minorversion == 0) { + __set_bit(NFS_CS_REUSEPORT, &cl_init->init_flags); + cl_init->max_connect = 0; + } + + switch (cl_init->proto) { + case XPRT_TRANSPORT_RDMA: + case XPRT_TRANSPORT_TCP: + case XPRT_TRANSPORT_TCP_TLS: + break; + default: + cl_init->nconnect = 0; + } + + if (server->flags & NFS_MOUNT_NORESVPORT) + __set_bit(NFS_CS_NORESVPORT, &cl_init->init_flags); + if (server->options & NFS_OPTION_MIGRATION) + __set_bit(NFS_CS_MIGRATION, &cl_init->init_flags); + if (test_bit(NFS_MIG_TSM_POSSIBLE, &server->mig_status)) + __set_bit(NFS_CS_TSM_POSSIBLE, &cl_init->init_flags); + server->port = rpc_get_port((struct sockaddr *)cl_init->addr); + + if (server->flags & NFS_MOUNT_NETUNREACH_FATAL) + __set_bit(NFS_CS_NETUNREACH_FATAL, &cl_init->init_flags); + + /* Allocate or find a client reference we can use */ + clp = nfs_get_client(cl_init); + if (IS_ERR(clp)) + return PTR_ERR(clp); + + if (server->nfs_client == clp) { + nfs_put_client(clp); + return -ELOOP; + } + + /* + * Query for the lease time on clientid setup or renewal + * + * Note that this will be set on nfs_clients that were created + * only for the DS role and did not set this bit, but now will + * serve a dual role. + */ + set_bit(NFS_CS_CHECK_LEASE_TIME, &clp->cl_res_state); + + server->nfs_client = clp; + nfs_sysfs_add_server(server); + nfs_sysfs_link_rpc_client(server, clp->cl_rpcclient, "_state"); + + return 0; +} + +/* + * Set up a pNFS Data Server client. + * + * Return any existing nfs_client that matches server address,port,version + * and minorversion. + * + * For a new nfs_client, use a soft mount (default), a low retrans and a + * low timeout interval so that if a connection is lost, we retry through + * the MDS. + */ +struct nfs_client *nfs4_set_ds_client(struct nfs_server *mds_srv, + const struct sockaddr_storage *ds_addr, int ds_addrlen, + int ds_proto, unsigned int ds_timeo, unsigned int ds_retrans, + u32 minor_version) +{ + struct rpc_timeout ds_timeout; + struct nfs_client *mds_clp = mds_srv->nfs_client; + struct nfs_client_initdata cl_init = { + .addr = ds_addr, + .addrlen = ds_addrlen, + .nodename = mds_clp->cl_rpcclient->cl_nodename, + .ip_addr = mds_clp->cl_ipaddr, + .nfs_mod = &nfs_v4, + .proto = ds_proto, + .minorversion = minor_version, + .net = mds_clp->cl_net, + .timeparms = &ds_timeout, + .cred = mds_srv->cred, + .xprtsec = { + .policy = RPC_XPRTSEC_NONE, + .cert_serial = TLS_NO_CERT, + .privkey_serial = TLS_NO_PRIVKEY, + }, + }; + char buf[INET6_ADDRSTRLEN + 1]; + + if (rpc_ntop((struct sockaddr *)ds_addr, buf, sizeof(buf)) <= 0) + return ERR_PTR(-EINVAL); + cl_init.hostname = buf; + + switch (ds_proto) { + case XPRT_TRANSPORT_TCP_TLS: + if (mds_srv->nfs_client->cl_xprtsec.policy != RPC_XPRTSEC_NONE) + cl_init.xprtsec = mds_srv->nfs_client->cl_xprtsec; + else + ds_proto = XPRT_TRANSPORT_TCP; + fallthrough; + case XPRT_TRANSPORT_RDMA: + case XPRT_TRANSPORT_TCP: + if (mds_clp->cl_nconnect > 1) { + cl_init.nconnect = mds_clp->cl_nconnect; + cl_init.max_connect = NFS_MAX_TRANSPORTS; + } + } + + if (mds_srv->flags & NFS_MOUNT_NORESVPORT) + __set_bit(NFS_CS_NORESVPORT, &cl_init.init_flags); + if (test_bit(NFS_CS_NETUNREACH_FATAL, &mds_clp->cl_flags)) + __set_bit(NFS_CS_NETUNREACH_FATAL, &cl_init.init_flags); + + __set_bit(NFS_CS_PNFS, &cl_init.init_flags); + cl_init.max_connect = NFS_MAX_TRANSPORTS; + /* + * Set an authflavor equual to the MDS value. Use the MDS nfs_client + * cl_ipaddr so as to use the same EXCHANGE_ID co_ownerid as the MDS + * (section 13.1 RFC 5661). + */ + nfs_init_timeout_values(&ds_timeout, ds_proto, ds_timeo, ds_retrans); + return nfs_get_client(&cl_init); +} +EXPORT_SYMBOL_GPL(nfs4_set_ds_client); + +/* + * Session has been established, and the client marked ready. + * Limit the mount rsize, wsize and dtsize using negotiated fore + * channel attributes. + */ +void nfs4_session_limit_rwsize(struct nfs_server *server) +{ + struct nfs4_session *sess; + u32 server_resp_sz; + u32 server_rqst_sz; + + if (!nfs4_has_session(server->nfs_client)) + return; + sess = server->nfs_client->cl_session; + server_resp_sz = sess->fc_attrs.max_resp_sz - nfs41_maxread_overhead; + server_rqst_sz = sess->fc_attrs.max_rqst_sz - nfs41_maxwrite_overhead; + + if (server->dtsize > server_resp_sz) + server->dtsize = server_resp_sz; + if (server->rsize > server_resp_sz) + server->rsize = server_resp_sz; + if (server->wsize > server_rqst_sz) + server->wsize = server_rqst_sz; +} + +/* + * Limit xattr sizes using the channel attributes. + */ +void nfs4_session_limit_xasize(struct nfs_server *server) +{ +#ifdef CONFIG_NFS_V4_2 + struct nfs4_session *sess; + u32 server_gxa_sz; + u32 server_sxa_sz; + u32 server_lxa_sz; + + if (!nfs4_has_session(server->nfs_client)) + return; + + sess = server->nfs_client->cl_session; + + server_gxa_sz = sess->fc_attrs.max_resp_sz - nfs42_maxgetxattr_overhead; + server_sxa_sz = sess->fc_attrs.max_rqst_sz - nfs42_maxsetxattr_overhead; + server_lxa_sz = sess->fc_attrs.max_resp_sz - + nfs42_maxlistxattrs_overhead; + + if (server->gxasize > server_gxa_sz) + server->gxasize = server_gxa_sz; + if (server->sxasize > server_sxa_sz) + server->sxasize = server_sxa_sz; + if (server->lxasize > server_lxa_sz) + server->lxasize = server_lxa_sz; +#endif +} + +static int nfs4_server_common_setup(struct nfs_server *server, + struct nfs_fh *mntfh, bool auth_probe) +{ + int error; + + error = nfs4_delegation_hash_alloc(server); + if (error) + return error; + + /* data servers support only a subset of NFSv4.1 */ + if (is_ds_only_client(server->nfs_client)) + return -EPROTONOSUPPORT; + + /* We must ensure the session is initialised first */ + error = nfs4_init_session(server->nfs_client); + if (error < 0) + return error; + + nfs_server_set_init_caps(server); + + /* Probe the root fh to retrieve its FSID and filehandle */ + error = nfs4_get_rootfh(server, mntfh, auth_probe); + if (error < 0) + return error; + + dprintk("Server FSID: %llx:%llx\n", + (unsigned long long) server->fsid.major, + (unsigned long long) server->fsid.minor); + nfs_display_fhandle(mntfh, "Pseudo-fs root FH"); + + error = nfs_probe_server(server, mntfh); + if (error < 0) + return error; + + nfs4_session_limit_rwsize(server); + nfs4_session_limit_xasize(server); + + if (server->namelen == 0 || server->namelen > NFS4_MAXNAMLEN) + server->namelen = NFS4_MAXNAMLEN; + + nfs_server_insert_lists(server); + server->mount_time = jiffies; + server->destroy = nfs4_destroy_server; + return 0; +} + +/* + * Create a version 4 volume record + */ +static int nfs4_init_server(struct nfs_server *server, struct fs_context *fc) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + struct rpc_timeout timeparms; + struct nfs_client_initdata cl_init = { + .hostname = ctx->nfs_server.hostname, + .addr = &ctx->nfs_server._address, + .addrlen = ctx->nfs_server.addrlen, + .ip_addr = ctx->client_address, + .proto = ctx->nfs_server.protocol, + .minorversion = ctx->minorversion, + .net = fc->net_ns, + .timeparms = &timeparms, + .xprtsec = ctx->xprtsec, + .nconnect = ctx->nfs_server.nconnect, + .max_connect = ctx->nfs_server.max_connect, + }; + int error; + + nfs_init_timeout_values(&timeparms, ctx->nfs_server.protocol, + ctx->timeo, ctx->retrans); + + /* Initialise the client representation from the mount data */ + server->flags = ctx->flags; + server->options = ctx->options; + server->auth_info = ctx->auth_info; + + /* Use the first specified auth flavor. If this flavor isn't + * allowed by the server, use the SECINFO path to try the + * other specified flavors */ + if (ctx->auth_info.flavor_len >= 1) + ctx->selected_flavor = ctx->auth_info.flavors[0]; + else + ctx->selected_flavor = RPC_AUTH_UNIX; + + /* Get a client record */ + error = nfs4_set_client(server, &cl_init); + if (error < 0) + return error; + + if (ctx->bsize) { + server->bsize = ctx->bsize; + server->automount_inherit |= NFS_AUTOMOUNT_INHERIT_BSIZE; + } + if (ctx->rsize) { + server->rsize = + nfs_io_size(ctx->rsize, server->nfs_client->cl_proto); + server->automount_inherit |= NFS_AUTOMOUNT_INHERIT_RSIZE; + } + if (ctx->wsize) { + server->wsize = + nfs_io_size(ctx->wsize, server->nfs_client->cl_proto); + server->automount_inherit |= NFS_AUTOMOUNT_INHERIT_WSIZE; + } + + server->acregmin = ctx->acregmin * HZ; + server->acregmax = ctx->acregmax * HZ; + server->acdirmin = ctx->acdirmin * HZ; + server->acdirmax = ctx->acdirmax * HZ; + server->port = ctx->nfs_server.port; + + return nfs_init_server_rpcclient(server, &timeparms, + ctx->selected_flavor); +} + +/* + * Create a version 4 volume record + * - keyed on server and FSID + */ +struct nfs_server *nfs4_create_server(struct fs_context *fc) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + struct nfs_server *server; + bool auth_probe; + int error; + + server = nfs_alloc_server(); + if (!server) + return ERR_PTR(-ENOMEM); + + server->cred = get_cred(fc->cred); + + auth_probe = ctx->auth_info.flavor_len < 1; + + /* set up the general RPC client */ + error = nfs4_init_server(server, fc); + if (error < 0) + goto error; + + error = nfs4_server_common_setup(server, ctx->mntfh, auth_probe); + if (error < 0) + goto error; + + return server; + +error: + nfs_free_server(server); + return ERR_PTR(error); +} + +/* + * Create an NFS4 referral server record + */ +struct nfs_server *nfs4_create_referral_server(struct fs_context *fc) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + struct nfs_server *parent_server = NFS_SB(ctx->clone_data.sb); + struct nfs_client *parent_client = parent_server->nfs_client; + struct nfs_client_initdata cl_init = { + .hostname = ctx->nfs_server.hostname, + .addr = &ctx->nfs_server._address, + .addrlen = ctx->nfs_server.addrlen, + .ip_addr = parent_client->cl_ipaddr, + .minorversion = parent_client->cl_mvops->minor_version, + .net = parent_client->cl_net, + .timeparms = parent_server->client->cl_timeout, + .xprtsec = parent_client->cl_xprtsec, + .nconnect = parent_client->cl_nconnect, + .max_connect = parent_client->cl_max_connect, + }; + struct nfs_server *server; + bool auth_probe; + int error; + + server = nfs_alloc_server(); + if (!server) + return ERR_PTR(-ENOMEM); + + server->cred = get_cred(parent_server->cred); + + /* Initialise the client representation from the parent server */ + nfs_server_copy_userdata(server, parent_server); + + /* Get a client representation */ +#if IS_ENABLED(CONFIG_SUNRPC_XPRT_RDMA) + rpc_set_port(&ctx->nfs_server.address, NFS_RDMA_PORT); + cl_init.proto = XPRT_TRANSPORT_RDMA; + error = nfs4_set_client(server, &cl_init); + if (!error) + goto init_server; +#endif /* IS_ENABLED(CONFIG_SUNRPC_XPRT_RDMA) */ + + cl_init.proto = XPRT_TRANSPORT_TCP; + if (parent_client->cl_xprtsec.policy != RPC_XPRTSEC_NONE) + cl_init.proto = XPRT_TRANSPORT_TCP_TLS; + rpc_set_port(&ctx->nfs_server.address, NFS_PORT); + error = nfs4_set_client(server, &cl_init); + if (error < 0) + goto error; + +#if IS_ENABLED(CONFIG_SUNRPC_XPRT_RDMA) +init_server: +#endif + error = nfs_init_server_rpcclient(server, parent_server->client->cl_timeout, + ctx->selected_flavor); + if (error < 0) + goto error; + + auth_probe = parent_server->auth_info.flavor_len < 1; + + error = nfs4_server_common_setup(server, ctx->mntfh, auth_probe); + if (error < 0) + goto error; + + return server; + +error: + nfs_free_server(server); + return ERR_PTR(error); +} + +/** + * nfs4_update_server - Move an nfs_server to a different nfs_client + * + * @server: represents FSID to be moved + * @hostname: new end-point's hostname + * @sap: new end-point's socket address + * @salen: size of "sap" + * @net: net namespace + * + * The nfs_server must be quiescent before this function is invoked. + * Either its session is drained (NFSv4.1+), or its transport is + * plugged and drained (NFSv4.0). + * + * Returns zero on success, or a negative errno value. + */ +int nfs4_update_server(struct nfs_server *server, const char *hostname, + struct sockaddr_storage *sap, size_t salen, struct net *net) +{ + struct nfs_client *clp = server->nfs_client; + struct rpc_clnt *clnt = server->client; + struct xprt_create xargs = { + .ident = clp->cl_proto, + .net = net, + .dstaddr = (struct sockaddr *)sap, + .addrlen = salen, + .servername = hostname, + /* cel: bleh. We might need to pass TLS parameters here */ + }; + char buf[INET6_ADDRSTRLEN + 1]; + struct sockaddr_storage address; + struct sockaddr *localaddr = (struct sockaddr *)&address; + struct nfs_client_initdata cl_init = { + .hostname = hostname, + .addr = sap, + .addrlen = salen, + .ip_addr = buf, + .proto = clp->cl_proto, + .minorversion = clp->cl_minorversion, + .net = net, + .timeparms = clnt->cl_timeout, + .xprtsec = clp->cl_xprtsec, + .nconnect = clp->cl_nconnect, + .max_connect = clp->cl_max_connect, + }; + int error; + + error = rpc_switch_client_transport(clnt, &xargs, clnt->cl_timeout); + if (error != 0) + return error; + + error = rpc_localaddr(clnt, localaddr, sizeof(address)); + if (error != 0) + return error; + + if (rpc_ntop(localaddr, buf, sizeof(buf)) == 0) + return -EAFNOSUPPORT; + + nfs_server_remove_lists(server); + set_bit(NFS_MIG_TSM_POSSIBLE, &server->mig_status); + error = nfs4_set_client(server, &cl_init); + clear_bit(NFS_MIG_TSM_POSSIBLE, &server->mig_status); + if (error != 0) { + nfs_server_insert_lists(server); + return error; + } + nfs_put_client(clp); + + if (server->nfs_client->cl_hostname == NULL) { + server->nfs_client->cl_hostname = kstrdup(hostname, GFP_KERNEL); + if (server->nfs_client->cl_hostname == NULL) + return -ENOMEM; + } + nfs_server_insert_lists(server); + + return nfs_probe_server(server, NFS_FH(d_inode(server->super->s_root))); +} diff --git a/fs/nfs/nfs4file.c b/fs/nfs/nfs4file.c new file mode 100644 index 000000000..be40e126c --- /dev/null +++ b/fs/nfs/nfs4file.c @@ -0,0 +1,459 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/nfs/file.c + * + * Copyright (C) 1992 Rick Sladkey + */ +#include +#include +#include +#include +#include +#include +#include +#include "delegation.h" +#include "internal.h" +#include "iostat.h" +#include "fscache.h" +#include "pnfs.h" + +#include "nfstrace.h" + +#ifdef CONFIG_NFS_V4_2 +#include "nfs42.h" +#endif + +#define NFSDBG_FACILITY NFSDBG_FILE + +static int +nfs4_file_open(struct inode *inode, struct file *filp) +{ + struct nfs_open_context *ctx; + struct dentry *dentry = file_dentry(filp); + struct dentry *parent = NULL; + struct inode *dir; + unsigned openflags = filp->f_flags; + struct iattr attr; + int err; + + /* + * If no cached dentry exists or if it's negative, NFSv4 handled the + * opens in ->lookup() or ->create(). + * + * We only get this far for a cached positive dentry. We skipped + * revalidation, so handle it here by dropping the dentry and returning + * -EOPENSTALE. The VFS will retry the lookup/create/open. + */ + + dprintk("NFS: open file(%pd2)\n", dentry); + + err = nfs_check_flags(openflags); + if (err) + return err; + + /* We can't create new files here */ + openflags &= ~(O_CREAT|O_EXCL); + + parent = dget_parent(dentry); + dir = d_inode(parent); + + ctx = alloc_nfs_open_context(file_dentry(filp), + flags_to_mode(openflags), filp); + err = PTR_ERR(ctx); + if (IS_ERR(ctx)) + goto out; + + attr.ia_valid = ATTR_OPEN; + if (openflags & O_TRUNC) { + attr.ia_valid |= ATTR_SIZE; + attr.ia_size = 0; + filemap_write_and_wait(inode->i_mapping); + } + + inode = NFS_PROTO(dir)->open_context(dir, ctx, openflags, &attr, NULL); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + switch (err) { + default: + goto out_put_ctx; + case -ENOENT: + case -ESTALE: + case -EISDIR: + case -ENOTDIR: + case -ELOOP: + goto out_drop; + } + } + if (inode != d_inode(dentry)) + goto out_drop; + + nfs_file_set_open_context(filp, ctx); + nfs_fscache_open_file(inode, filp); + err = 0; + filp->f_mode |= FMODE_CAN_ODIRECT; + +out_put_ctx: + put_nfs_open_context(ctx); +out: + dput(parent); + return err; + +out_drop: + d_drop(dentry); + err = -EOPENSTALE; + goto out_put_ctx; +} + +/* + * Flush all dirty pages, and check for write errors. + */ +static int +nfs4_file_flush(struct file *file, fl_owner_t id) +{ + struct inode *inode = file_inode(file); + errseq_t since; + + dprintk("NFS: flush(%pD2)\n", file); + + nfs_inc_stats(inode, NFSIOS_VFSFLUSH); + if ((file->f_mode & FMODE_WRITE) == 0) + return 0; + + /* + * If we're holding a write delegation, then check if we're required + * to flush the i/o on close. If not, then just start the i/o now. + */ + if (!nfs4_delegation_flush_on_close(inode)) + return filemap_fdatawrite(file->f_mapping); + + /* Flush writes to the server and return any errors */ + since = filemap_sample_wb_err(file->f_mapping); + nfs_wb_all(inode); + return filemap_check_wb_err(file->f_mapping, since); +} + +#ifdef CONFIG_NFS_V4_2 +static ssize_t __nfs4_copy_file_range(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + size_t count, unsigned int flags) +{ + struct nfs42_copy_notify_res *cn_resp = NULL; + struct nl4_server *nss = NULL; + nfs4_stateid *cnrs = NULL; + ssize_t ret; + bool sync = false; + + /* Only offload copy if superblock is the same */ + if (file_in->f_op != &nfs4_file_operations) + return -EXDEV; + if (!nfs_server_capable(file_inode(file_out), NFS_CAP_COPY) || + !nfs_server_capable(file_inode(file_in), NFS_CAP_COPY)) + return -EOPNOTSUPP; + if (file_inode(file_in) == file_inode(file_out)) + return -EOPNOTSUPP; + /* if the copy size if smaller than 2 RPC payloads, make it + * synchronous + */ + if (count <= 2 * NFS_SERVER(file_inode(file_in))->rsize) + sync = true; +retry: + if (!nfs42_files_from_same_server(file_in, file_out)) { + /* + * for inter copy, if copy size is too small + * then fallback to generic copy. + */ + if (sync) + return -EOPNOTSUPP; + cn_resp = kzalloc_obj(struct nfs42_copy_notify_res); + if (unlikely(cn_resp == NULL)) + return -ENOMEM; + + ret = nfs42_proc_copy_notify(file_in, file_out, cn_resp); + if (ret) { + ret = -EOPNOTSUPP; + goto out; + } + nss = &cn_resp->cnr_src; + cnrs = &cn_resp->cnr_stateid; + } + ret = nfs42_proc_copy(file_in, pos_in, file_out, pos_out, count, + nss, cnrs, sync); +out: + kfree(cn_resp); + + if (ret == -EAGAIN) + goto retry; + return ret; +} + +static ssize_t nfs4_copy_file_range(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + size_t count, unsigned int flags) +{ + ssize_t ret; + + ret = __nfs4_copy_file_range(file_in, pos_in, file_out, pos_out, count, + flags); + if (ret == -EOPNOTSUPP || ret == -EXDEV) + ret = splice_copy_file_range(file_in, pos_in, file_out, + pos_out, count); + return ret; +} + +static loff_t nfs4_file_llseek(struct file *filep, loff_t offset, int whence) +{ + loff_t ret; + + switch (whence) { + case SEEK_HOLE: + case SEEK_DATA: + ret = nfs42_proc_llseek(filep, offset, whence); + if (ret != -EOPNOTSUPP) + return ret; + fallthrough; + default: + return nfs_file_llseek(filep, offset, whence); + } +} + +static long nfs42_fallocate(struct file *filep, int mode, loff_t offset, loff_t len) +{ + struct inode *inode = file_inode(filep); + long ret; + + if (!S_ISREG(inode->i_mode)) + return -EOPNOTSUPP; + + switch (mode) { + case 0: + case FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE: + case FALLOC_FL_ZERO_RANGE: + break; + default: + return -EOPNOTSUPP; + } + + ret = inode_newsize_ok(inode, offset + len); + if (ret < 0) + return ret; + + if (mode & FALLOC_FL_PUNCH_HOLE) + return nfs42_proc_deallocate(filep, offset, len); + else if (mode & FALLOC_FL_ZERO_RANGE) + return nfs42_proc_zero_range(filep, offset ,len); + return nfs42_proc_allocate(filep, offset, len); +} + +static loff_t nfs42_remap_file_range(struct file *src_file, loff_t src_off, + struct file *dst_file, loff_t dst_off, loff_t count, + unsigned int remap_flags) +{ + struct inode *dst_inode = file_inode(dst_file); + struct nfs_server *server = NFS_SERVER(dst_inode); + struct inode *src_inode = file_inode(src_file); + unsigned int bs = server->clone_blksize; + int ret; + + /* NFS does not support deduplication. */ + if (remap_flags & REMAP_FILE_DEDUP) + return -EOPNOTSUPP; + + if (remap_flags & ~REMAP_FILE_ADVISORY) + return -EINVAL; + + if (IS_SWAPFILE(dst_inode) || IS_SWAPFILE(src_inode)) + return -ETXTBSY; + + /* check alignment w.r.t. clone_blksize */ + ret = -EINVAL; + if (bs) { + if (!IS_ALIGNED(src_off, bs) || !IS_ALIGNED(dst_off, bs)) + goto out; + if (!IS_ALIGNED(count, bs) && i_size_read(src_inode) != (src_off + count)) + goto out; + } + + /* XXX: do we lock at all? what if server needs CB_RECALL_LAYOUT? */ + lock_two_nondirectories(src_inode, dst_inode); + /* flush all pending writes on both src and dst so that server + * has the latest data */ + nfs_file_block_o_direct(NFS_I(src_inode)); + ret = nfs_sync_inode(src_inode); + if (ret) + goto out_unlock; + nfs_file_block_o_direct(NFS_I(dst_inode)); + ret = nfs_sync_inode(dst_inode); + if (ret) + goto out_unlock; + + ret = nfs42_proc_clone(src_file, dst_file, src_off, dst_off, count); + + /* truncate inode page cache of the dst range so that future reads can fetch + * new data from server */ + if (!ret) + truncate_inode_pages_range(&dst_inode->i_data, dst_off, dst_off + count - 1); + +out_unlock: + unlock_two_nondirectories(src_inode, dst_inode); +out: + return ret < 0 ? ret : count; +} + +static int read_name_gen = 1; +#define SSC_READ_NAME_BODY "ssc_read_%d" + +static struct file *__nfs42_ssc_open(struct vfsmount *ss_mnt, + struct nfs_fh *src_fh, nfs4_stateid *stateid) +{ + struct nfs_fattr *fattr = nfs_alloc_fattr(); + struct file *filep, *res; + struct nfs_server *server; + struct inode *r_ino = NULL; + struct nfs_open_context *ctx; + struct nfs4_state_owner *sp; + char *read_name = NULL; + int len, status = 0; + + server = NFS_SB(ss_mnt->mnt_sb); + + if (!fattr) + return ERR_PTR(-ENOMEM); + + status = nfs4_proc_getattr(server, src_fh, fattr, NULL); + if (status < 0) { + res = ERR_PTR(status); + goto out; + } + + if (!S_ISREG(fattr->mode)) { + res = ERR_PTR(-EBADF); + goto out; + } + + res = ERR_PTR(-ENOMEM); + len = strlen(SSC_READ_NAME_BODY) + 16; + read_name = kzalloc(len, GFP_KERNEL); + if (read_name == NULL) + goto out; + snprintf(read_name, len, SSC_READ_NAME_BODY, read_name_gen++); + + r_ino = nfs_fhget(ss_mnt->mnt_sb, src_fh, fattr); + if (IS_ERR(r_ino)) { + res = ERR_CAST(r_ino); + goto out_free_name; + } + + filep = alloc_file_pseudo(r_ino, ss_mnt, read_name, O_RDONLY, + r_ino->i_fop); + if (IS_ERR(filep)) { + res = ERR_CAST(filep); + iput(r_ino); + goto out_free_name; + } + + ctx = alloc_nfs_open_context(filep->f_path.dentry, + flags_to_mode(filep->f_flags), filep); + if (IS_ERR(ctx)) { + res = ERR_CAST(ctx); + goto out_filep; + } + + res = ERR_PTR(-EINVAL); + sp = nfs4_get_state_owner(server, ctx->cred, GFP_KERNEL); + if (sp == NULL) + goto out_ctx; + + ctx->state = nfs4_get_open_state(r_ino, sp); + if (ctx->state == NULL) + goto out_stateowner; + + set_bit(NFS_SRV_SSC_COPY_STATE, &ctx->state->flags); + memcpy(&ctx->state->open_stateid.other, &stateid->other, + NFS4_STATEID_OTHER_SIZE); + update_open_stateid(ctx->state, stateid, NULL, filep->f_mode); + set_bit(NFS_OPEN_STATE, &ctx->state->flags); + + nfs_file_set_open_context(filep, ctx); + put_nfs_open_context(ctx); + + file_ra_state_init(&filep->f_ra, filep->f_mapping->host->i_mapping); + res = filep; +out_free_name: + kfree(read_name); +out: + nfs_free_fattr(fattr); + return res; +out_stateowner: + nfs4_put_state_owner(sp); +out_ctx: + put_nfs_open_context(ctx); +out_filep: + fput(filep); + goto out_free_name; +} + +static void __nfs42_ssc_close(struct file *filep) +{ + struct nfs_open_context *ctx = nfs_file_open_context(filep); + + ctx->state->flags = 0; +} + +static const struct nfs4_ssc_client_ops nfs4_ssc_clnt_ops_tbl = { + .sco_open = __nfs42_ssc_open, + .sco_close = __nfs42_ssc_close, +}; + +/** + * nfs42_ssc_register_ops - Wrapper to register NFS_V4 ops in nfs_common + * + * Return values: + * None + */ +void nfs42_ssc_register_ops(void) +{ + nfs42_ssc_register(&nfs4_ssc_clnt_ops_tbl); +} + +/** + * nfs42_ssc_unregister_ops - wrapper to un-register NFS_V4 ops in nfs_common + * + * Return values: + * None. + */ +void nfs42_ssc_unregister_ops(void) +{ + nfs42_ssc_unregister(&nfs4_ssc_clnt_ops_tbl); +} +#endif /* CONFIG_NFS_V4_2 */ + +static int nfs4_setlease(struct file *file, int arg, struct file_lease **lease, + void **priv) +{ + return nfs4_proc_setlease(file, arg, lease, priv); +} + +const struct file_operations nfs4_file_operations = { + .read_iter = nfs_file_read, + .write_iter = nfs_file_write, + .mmap_prepare = nfs_file_mmap_prepare, + .open = nfs4_file_open, + .flush = nfs4_file_flush, + .release = nfs_file_release, + .fsync = nfs_file_fsync, + .lock = nfs_lock, + .flock = nfs_flock, + .splice_read = nfs_file_splice_read, + .splice_write = iter_file_splice_write, + .check_flags = nfs_check_flags, + .setlease = nfs4_setlease, +#ifdef CONFIG_NFS_V4_2 + .copy_file_range = nfs4_copy_file_range, + .llseek = nfs4_file_llseek, + .fallocate = nfs42_fallocate, + .remap_file_range = nfs42_remap_file_range, +#else + .llseek = nfs_file_llseek, +#endif + .fop_flags = FOP_DONTCACHE, +}; diff --git a/fs/nfs/nfs4getroot.c b/fs/nfs/nfs4getroot.c new file mode 100644 index 000000000..e67ea345d --- /dev/null +++ b/fs/nfs/nfs4getroot.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 +/* +* Copyright (C) 2006 Red Hat, Inc. All Rights Reserved. +* Written by David Howells (dhowells@redhat.com) +*/ + +#include +#include "nfs4_fs.h" +#include "internal.h" + +#define NFSDBG_FACILITY NFSDBG_CLIENT + +int nfs4_get_rootfh(struct nfs_server *server, struct nfs_fh *mntfh, bool auth_probe) +{ + struct nfs_fattr *fattr = nfs_alloc_fattr(); + int ret = -ENOMEM; + + if (fattr == NULL) + goto out; + + /* Start by getting the root filehandle from the server */ + ret = nfs4_proc_get_rootfh(server, mntfh, fattr, auth_probe); + if (ret < 0) { + dprintk("nfs4_get_rootfh: getroot error = %d\n", -ret); + goto out; + } + + if (!(fattr->valid & NFS_ATTR_FATTR_TYPE) || !S_ISDIR(fattr->mode)) { + printk(KERN_ERR "nfs4_get_rootfh:" + " getroot encountered non-directory\n"); + ret = -ENOTDIR; + goto out; + } + + memcpy(&server->fsid, &fattr->fsid, sizeof(server->fsid)); +out: + nfs_free_fattr(fattr); + return ret; +} diff --git a/fs/nfs/nfs4idmap.c b/fs/nfs/nfs4idmap.c new file mode 100644 index 000000000..bc397110d --- /dev/null +++ b/fs/nfs/nfs4idmap.c @@ -0,0 +1,793 @@ +/* + * fs/nfs/idmap.c + * + * UID and GID to name mapping for clients. + * + * Copyright (c) 2002 The Regents of the University of Michigan. + * All rights reserved. + * + * Marius Aamodt Eriksen + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" +#include "netns.h" +#include "nfs4idmap.h" +#include "nfs4trace.h" + +#define NFS_UINT_MAXLEN 11 + +static const struct cred *id_resolver_cache; +static struct key_type key_type_id_resolver_legacy; + +struct idmap_legacy_upcalldata { + struct rpc_pipe_msg pipe_msg; + struct idmap_msg idmap_msg; + struct key *authkey; + struct idmap *idmap; +}; + +struct idmap { + struct rpc_pipe_dir_object idmap_pdo; + struct rpc_pipe *idmap_pipe; + struct idmap_legacy_upcalldata *idmap_upcall_data; + struct mutex idmap_mutex; + struct user_namespace *user_ns; +}; + +static struct user_namespace *idmap_userns(const struct idmap *idmap) +{ + if (idmap && idmap->user_ns) + return idmap->user_ns; + return &init_user_ns; +} + +/** + * nfs_fattr_init_names - initialise the nfs_fattr owner_name/group_name fields + * @fattr: fully initialised struct nfs_fattr + * @owner_name: owner name string cache + * @group_name: group name string cache + */ +void nfs_fattr_init_names(struct nfs_fattr *fattr, + struct nfs4_string *owner_name, + struct nfs4_string *group_name) +{ + fattr->owner_name = owner_name; + fattr->group_name = group_name; +} + +static void nfs_fattr_free_owner_name(struct nfs_fattr *fattr) +{ + fattr->valid &= ~NFS_ATTR_FATTR_OWNER_NAME; + kfree(fattr->owner_name->data); +} + +static void nfs_fattr_free_group_name(struct nfs_fattr *fattr) +{ + fattr->valid &= ~NFS_ATTR_FATTR_GROUP_NAME; + kfree(fattr->group_name->data); +} + +static bool nfs_fattr_map_owner_name(struct nfs_server *server, struct nfs_fattr *fattr) +{ + struct nfs4_string *owner = fattr->owner_name; + kuid_t uid; + + if (!(fattr->valid & NFS_ATTR_FATTR_OWNER_NAME)) + return false; + if (nfs_map_name_to_uid(server, owner->data, owner->len, &uid) == 0) { + fattr->uid = uid; + fattr->valid |= NFS_ATTR_FATTR_OWNER; + } + return true; +} + +static bool nfs_fattr_map_group_name(struct nfs_server *server, struct nfs_fattr *fattr) +{ + struct nfs4_string *group = fattr->group_name; + kgid_t gid; + + if (!(fattr->valid & NFS_ATTR_FATTR_GROUP_NAME)) + return false; + if (nfs_map_group_to_gid(server, group->data, group->len, &gid) == 0) { + fattr->gid = gid; + fattr->valid |= NFS_ATTR_FATTR_GROUP; + } + return true; +} + +/** + * nfs_fattr_free_names - free up the NFSv4 owner and group strings + * @fattr: a fully initialised nfs_fattr structure + */ +void nfs_fattr_free_names(struct nfs_fattr *fattr) +{ + if (fattr->valid & NFS_ATTR_FATTR_OWNER_NAME) + nfs_fattr_free_owner_name(fattr); + if (fattr->valid & NFS_ATTR_FATTR_GROUP_NAME) + nfs_fattr_free_group_name(fattr); +} + +/** + * nfs_fattr_map_and_free_names - map owner/group strings into uid/gid and free + * @server: pointer to the filesystem nfs_server structure + * @fattr: a fully initialised nfs_fattr structure + * + * This helper maps the cached NFSv4 owner/group strings in fattr into + * their numeric uid/gid equivalents, and then frees the cached strings. + */ +void nfs_fattr_map_and_free_names(struct nfs_server *server, struct nfs_fattr *fattr) +{ + if (nfs_fattr_map_owner_name(server, fattr)) + nfs_fattr_free_owner_name(fattr); + if (nfs_fattr_map_group_name(server, fattr)) + nfs_fattr_free_group_name(fattr); +} + +int nfs_map_string_to_numeric(const char *name, size_t namelen, __u32 *res) +{ + unsigned long val; + char buf[16]; + + if (memchr(name, '@', namelen) != NULL || namelen >= sizeof(buf)) + return 0; + memcpy(buf, name, namelen); + buf[namelen] = '\0'; + if (kstrtoul(buf, 0, &val) != 0) + return 0; + *res = val; + return 1; +} +EXPORT_SYMBOL_GPL(nfs_map_string_to_numeric); + +static int nfs_map_numeric_to_string(__u32 id, char *buf, size_t buflen) +{ + return snprintf(buf, buflen, "%u", id); +} + +static struct key_type key_type_id_resolver = { + .name = "id_resolver", + .preparse = user_preparse, + .free_preparse = user_free_preparse, + .instantiate = generic_key_instantiate, + .revoke = user_revoke, + .destroy = user_destroy, + .describe = user_describe, + .read = user_read, +}; + +int nfs_idmap_init(void) +{ + struct cred *cred; + struct key *keyring; + int ret = 0; + + printk(KERN_NOTICE "NFS: Registering the %s key type\n", + key_type_id_resolver.name); + + cred = prepare_kernel_cred(&init_task); + if (!cred) + return -ENOMEM; + + keyring = keyring_alloc(".id_resolver", + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, cred, + (KEY_POS_ALL & ~KEY_POS_SETATTR) | + KEY_USR_VIEW | KEY_USR_READ, + KEY_ALLOC_NOT_IN_QUOTA, NULL, NULL); + if (IS_ERR(keyring)) { + ret = PTR_ERR(keyring); + goto failed_put_cred; + } + + ret = register_key_type(&key_type_id_resolver); + if (ret < 0) + goto failed_put_key; + + ret = register_key_type(&key_type_id_resolver_legacy); + if (ret < 0) + goto failed_reg_legacy; + + set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags); + cred->thread_keyring = keyring; + cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING; + id_resolver_cache = cred; + return 0; + +failed_reg_legacy: + unregister_key_type(&key_type_id_resolver); +failed_put_key: + key_put(keyring); +failed_put_cred: + put_cred(cred); + return ret; +} + +void nfs_idmap_quit(void) +{ + key_revoke(id_resolver_cache->thread_keyring); + unregister_key_type(&key_type_id_resolver); + unregister_key_type(&key_type_id_resolver_legacy); + put_cred(id_resolver_cache); +} + +/* + * Assemble the description to pass to request_key() + * This function will allocate a new string and update dest to point + * at it. The caller is responsible for freeing dest. + * + * On error 0 is returned. Otherwise, the length of dest is returned. + */ +static ssize_t nfs_idmap_get_desc(const char *name, size_t namelen, + const char *type, size_t typelen, char **desc) +{ + char *cp; + size_t desclen = typelen + namelen + 2; + + *desc = kmalloc(desclen, GFP_KERNEL); + if (!*desc) + return -ENOMEM; + + cp = *desc; + memcpy(cp, type, typelen); + cp += typelen; + *cp++ = ':'; + + memcpy(cp, name, namelen); + cp += namelen; + *cp = '\0'; + return desclen; +} + +static struct key *nfs_idmap_request_key(const char *name, size_t namelen, + const char *type, struct idmap *idmap) +{ + char *desc; + struct key *rkey = ERR_PTR(-EAGAIN); + ssize_t ret; + + ret = nfs_idmap_get_desc(name, namelen, type, strlen(type), &desc); + if (ret < 0) + return ERR_PTR(ret); + + if (!idmap->user_ns || idmap->user_ns == &init_user_ns) + rkey = request_key(&key_type_id_resolver, desc, ""); + if (IS_ERR(rkey)) { + mutex_lock(&idmap->idmap_mutex); + rkey = request_key_with_auxdata(&key_type_id_resolver_legacy, + desc, NULL, "", 0, idmap); + mutex_unlock(&idmap->idmap_mutex); + } + if (!IS_ERR(rkey)) + set_bit(KEY_FLAG_ROOT_CAN_INVAL, &rkey->flags); + + kfree(desc); + return rkey; +} + +static ssize_t nfs_idmap_get_key(const char *name, size_t namelen, + const char *type, void *data, + size_t data_size, struct idmap *idmap) +{ + struct key *rkey; + const struct user_key_payload *payload; + ssize_t ret; + + scoped_with_creds(id_resolver_cache) + rkey = nfs_idmap_request_key(name, namelen, type, idmap); + if (IS_ERR(rkey)) { + ret = PTR_ERR(rkey); + goto out; + } + + rcu_read_lock(); + rkey->perm |= KEY_USR_VIEW; + + ret = key_validate(rkey); + if (ret < 0) + goto out_up; + + payload = user_key_payload_rcu(rkey); + if (IS_ERR_OR_NULL(payload)) { + ret = PTR_ERR(payload); + goto out_up; + } + + ret = payload->datalen; + if (ret > 0 && ret <= data_size) + memcpy(data, payload->data, ret); + else + ret = -EINVAL; + +out_up: + rcu_read_unlock(); + key_put(rkey); +out: + return ret; +} + +/* ID -> Name */ +static ssize_t nfs_idmap_lookup_name(__u32 id, const char *type, char *buf, + size_t buflen, struct idmap *idmap) +{ + char id_str[NFS_UINT_MAXLEN]; + int id_len; + ssize_t ret; + + id_len = nfs_map_numeric_to_string(id, id_str, sizeof(id_str)); + ret = nfs_idmap_get_key(id_str, id_len, type, buf, buflen, idmap); + if (ret < 0) + return -EINVAL; + return ret; +} + +/* Name -> ID */ +static int nfs_idmap_lookup_id(const char *name, size_t namelen, const char *type, + __u32 *id, struct idmap *idmap) +{ + char id_str[NFS_UINT_MAXLEN]; + long id_long; + ssize_t data_size; + int ret = 0; + + data_size = nfs_idmap_get_key(name, namelen, type, id_str, NFS_UINT_MAXLEN, idmap); + if (data_size <= 0) { + ret = -EINVAL; + } else { + ret = kstrtol(id_str, 10, &id_long); + if (!ret) + *id = (__u32)id_long; + } + return ret; +} + +/* idmap classic begins here */ + +enum { + Opt_find_uid, Opt_find_gid, Opt_find_user, Opt_find_group, Opt_find_err +}; + +static const match_table_t nfs_idmap_tokens = { + { Opt_find_uid, "uid:%s" }, + { Opt_find_gid, "gid:%s" }, + { Opt_find_user, "user:%s" }, + { Opt_find_group, "group:%s" }, + { Opt_find_err, NULL } +}; + +static int nfs_idmap_legacy_upcall(struct key *, void *); +static ssize_t idmap_pipe_downcall(struct file *, const char __user *, + size_t); +static void idmap_release_pipe(struct inode *); +static void idmap_pipe_destroy_msg(struct rpc_pipe_msg *); + +static const struct rpc_pipe_ops idmap_upcall_ops = { + .upcall = rpc_pipe_generic_upcall, + .downcall = idmap_pipe_downcall, + .release_pipe = idmap_release_pipe, + .destroy_msg = idmap_pipe_destroy_msg, +}; + +static struct key_type key_type_id_resolver_legacy = { + .name = "id_legacy", + .preparse = user_preparse, + .free_preparse = user_free_preparse, + .instantiate = generic_key_instantiate, + .revoke = user_revoke, + .destroy = user_destroy, + .describe = user_describe, + .read = user_read, + .request_key = nfs_idmap_legacy_upcall, +}; + +static void nfs_idmap_pipe_destroy(struct dentry *dir, + struct rpc_pipe_dir_object *pdo) +{ + struct idmap *idmap = pdo->pdo_data; + + rpc_unlink(idmap->idmap_pipe); +} + +static int nfs_idmap_pipe_create(struct dentry *dir, + struct rpc_pipe_dir_object *pdo) +{ + struct idmap *idmap = pdo->pdo_data; + + return rpc_mkpipe_dentry(dir, "idmap", idmap, idmap->idmap_pipe); +} + +static const struct rpc_pipe_dir_object_ops nfs_idmap_pipe_dir_object_ops = { + .create = nfs_idmap_pipe_create, + .destroy = nfs_idmap_pipe_destroy, +}; + +int +nfs_idmap_new(struct nfs_client *clp) +{ + struct idmap *idmap; + struct rpc_pipe *pipe; + int error; + + idmap = kzalloc_obj(*idmap); + if (idmap == NULL) + return -ENOMEM; + + mutex_init(&idmap->idmap_mutex); + idmap->user_ns = get_user_ns(clp->cl_rpcclient->cl_cred->user_ns); + + rpc_init_pipe_dir_object(&idmap->idmap_pdo, + &nfs_idmap_pipe_dir_object_ops, + idmap); + + pipe = rpc_mkpipe_data(&idmap_upcall_ops, 0); + if (IS_ERR(pipe)) { + error = PTR_ERR(pipe); + goto err; + } + idmap->idmap_pipe = pipe; + + error = rpc_add_pipe_dir_object(clp->cl_net, + &clp->cl_rpcclient->cl_pipedir_objects, + &idmap->idmap_pdo); + if (error) + goto err_destroy_pipe; + + clp->cl_idmap = idmap; + return 0; +err_destroy_pipe: + rpc_destroy_pipe_data(idmap->idmap_pipe); +err: + put_user_ns(idmap->user_ns); + kfree(idmap); + return error; +} + +void +nfs_idmap_delete(struct nfs_client *clp) +{ + struct idmap *idmap = clp->cl_idmap; + + if (!idmap) + return; + clp->cl_idmap = NULL; + rpc_remove_pipe_dir_object(clp->cl_net, + &clp->cl_rpcclient->cl_pipedir_objects, + &idmap->idmap_pdo); + rpc_destroy_pipe_data(idmap->idmap_pipe); + put_user_ns(idmap->user_ns); + kfree(idmap); +} + +static int nfs_idmap_prepare_message(char *desc, struct idmap *idmap, + struct idmap_msg *im, + struct rpc_pipe_msg *msg) +{ + substring_t substr; + int token, ret; + + im->im_type = IDMAP_TYPE_GROUP; + token = match_token(desc, nfs_idmap_tokens, &substr); + + switch (token) { + case Opt_find_uid: + im->im_type = IDMAP_TYPE_USER; + fallthrough; + case Opt_find_gid: + im->im_conv = IDMAP_CONV_NAMETOID; + ret = match_strlcpy(im->im_name, &substr, IDMAP_NAMESZ); + break; + + case Opt_find_user: + im->im_type = IDMAP_TYPE_USER; + fallthrough; + case Opt_find_group: + im->im_conv = IDMAP_CONV_IDTONAME; + ret = match_int(&substr, &im->im_id); + if (ret) + goto out; + break; + + default: + ret = -EINVAL; + goto out; + } + + msg->data = im; + msg->len = sizeof(struct idmap_msg); + +out: + return ret; +} + +static bool +nfs_idmap_prepare_pipe_upcall(struct idmap *idmap, + struct idmap_legacy_upcalldata *data) +{ + if (idmap->idmap_upcall_data != NULL) { + WARN_ON_ONCE(1); + return false; + } + idmap->idmap_upcall_data = data; + return true; +} + +static void nfs_idmap_complete_pipe_upcall(struct idmap_legacy_upcalldata *data, + int ret) +{ + complete_request_key(data->authkey, ret); + key_put(data->authkey); + kfree(data); +} + +static void nfs_idmap_abort_pipe_upcall(struct idmap *idmap, + struct idmap_legacy_upcalldata *data, + int ret) +{ + if (cmpxchg(&idmap->idmap_upcall_data, data, NULL) == data) + nfs_idmap_complete_pipe_upcall(data, ret); +} + +static int nfs_idmap_legacy_upcall(struct key *authkey, void *aux) +{ + struct idmap_legacy_upcalldata *data; + struct request_key_auth *rka = get_request_key_auth(authkey); + struct rpc_pipe_msg *msg; + struct idmap_msg *im; + struct idmap *idmap = aux; + struct key *key = rka->target_key; + int ret = -ENOKEY; + + if (!aux) + goto out1; + + /* msg and im are freed in idmap_pipe_destroy_msg */ + ret = -ENOMEM; + data = kzalloc_obj(*data); + if (!data) + goto out1; + + msg = &data->pipe_msg; + im = &data->idmap_msg; + data->idmap = idmap; + data->authkey = key_get(authkey); + + ret = nfs_idmap_prepare_message(key->description, idmap, im, msg); + if (ret < 0) + goto out2; + + ret = -EAGAIN; + if (!nfs_idmap_prepare_pipe_upcall(idmap, data)) + goto out2; + + ret = rpc_queue_upcall(idmap->idmap_pipe, msg); + if (ret < 0) + nfs_idmap_abort_pipe_upcall(idmap, data, ret); + + return ret; +out2: + kfree(data); +out1: + complete_request_key(authkey, ret); + return ret; +} + +static int nfs_idmap_instantiate(struct key *key, struct key *authkey, char *data, size_t datalen) +{ + return key_instantiate_and_link(key, data, datalen, + id_resolver_cache->thread_keyring, + authkey); +} + +static int nfs_idmap_read_and_verify_message(struct idmap_msg *im, + struct idmap_msg *upcall, + struct key *key, struct key *authkey) +{ + char id_str[NFS_UINT_MAXLEN]; + size_t len; + int ret = -ENOKEY; + + /* ret = -ENOKEY */ + if (upcall->im_type != im->im_type || upcall->im_conv != im->im_conv) + goto out; + switch (im->im_conv) { + case IDMAP_CONV_NAMETOID: + if (strcmp(upcall->im_name, im->im_name) != 0) + break; + /* Note: here we store the NUL terminator too */ + len = 1 + nfs_map_numeric_to_string(im->im_id, id_str, + sizeof(id_str)); + ret = nfs_idmap_instantiate(key, authkey, id_str, len); + break; + case IDMAP_CONV_IDTONAME: + if (upcall->im_id != im->im_id) + break; + len = strlen(im->im_name); + ret = nfs_idmap_instantiate(key, authkey, im->im_name, len); + break; + default: + ret = -EINVAL; + } +out: + return ret; +} + +static ssize_t +idmap_pipe_downcall(struct file *filp, const char __user *src, size_t mlen) +{ + struct request_key_auth *rka; + struct rpc_inode *rpci = RPC_I(file_inode(filp)); + struct idmap *idmap = (struct idmap *)rpci->private; + struct idmap_legacy_upcalldata *data; + struct key *authkey; + struct idmap_msg im; + size_t namelen_in; + int ret = -ENOKEY; + + /* If instantiation is successful, anyone waiting for key construction + * will have been woken up and someone else may now have used + * idmap_key_cons - so after this point we may no longer touch it. + */ + data = xchg(&idmap->idmap_upcall_data, NULL); + if (data == NULL) + goto out_noupcall; + + authkey = data->authkey; + rka = get_request_key_auth(authkey); + + if (mlen != sizeof(im)) { + ret = -ENOSPC; + goto out; + } + + if (copy_from_user(&im, src, mlen) != 0) { + ret = -EFAULT; + goto out; + } + + if (!(im.im_status & IDMAP_STATUS_SUCCESS)) { + ret = -ENOKEY; + goto out; + } + + namelen_in = strnlen(im.im_name, IDMAP_NAMESZ); + if (namelen_in == 0 || namelen_in == IDMAP_NAMESZ) { + ret = -EINVAL; + goto out; + } + + ret = nfs_idmap_read_and_verify_message(&im, &data->idmap_msg, + rka->target_key, authkey); + if (ret >= 0) { + key_set_timeout(rka->target_key, nfs_idmap_cache_timeout); + ret = mlen; + } + +out: + nfs_idmap_complete_pipe_upcall(data, ret); +out_noupcall: + return ret; +} + +static void +idmap_pipe_destroy_msg(struct rpc_pipe_msg *msg) +{ + struct idmap_legacy_upcalldata *data = container_of(msg, + struct idmap_legacy_upcalldata, + pipe_msg); + struct idmap *idmap = data->idmap; + + if (msg->errno) + nfs_idmap_abort_pipe_upcall(idmap, data, msg->errno); +} + +static void +idmap_release_pipe(struct inode *inode) +{ + struct rpc_inode *rpci = RPC_I(inode); + struct idmap *idmap = (struct idmap *)rpci->private; + struct idmap_legacy_upcalldata *data; + + data = xchg(&idmap->idmap_upcall_data, NULL); + if (data) + nfs_idmap_complete_pipe_upcall(data, -EPIPE); +} + +int nfs_map_name_to_uid(const struct nfs_server *server, const char *name, size_t namelen, kuid_t *uid) +{ + struct idmap *idmap = server->nfs_client->cl_idmap; + __u32 id = -1; + int ret = 0; + + if (!nfs_map_string_to_numeric(name, namelen, &id)) + ret = nfs_idmap_lookup_id(name, namelen, "uid", &id, idmap); + if (ret == 0) { + *uid = make_kuid(idmap_userns(idmap), id); + if (!uid_valid(*uid)) + ret = -ERANGE; + } + trace_nfs4_map_name_to_uid(name, namelen, id, ret); + return ret; +} + +int nfs_map_group_to_gid(const struct nfs_server *server, const char *name, size_t namelen, kgid_t *gid) +{ + struct idmap *idmap = server->nfs_client->cl_idmap; + __u32 id = -1; + int ret = 0; + + if (!nfs_map_string_to_numeric(name, namelen, &id)) + ret = nfs_idmap_lookup_id(name, namelen, "gid", &id, idmap); + if (ret == 0) { + *gid = make_kgid(idmap_userns(idmap), id); + if (!gid_valid(*gid)) + ret = -ERANGE; + } + trace_nfs4_map_group_to_gid(name, namelen, id, ret); + return ret; +} + +int nfs_map_uid_to_name(const struct nfs_server *server, kuid_t uid, char *buf, size_t buflen) +{ + struct idmap *idmap = server->nfs_client->cl_idmap; + int ret = -EINVAL; + __u32 id; + + id = from_kuid_munged(idmap_userns(idmap), uid); + if (!(server->caps & NFS_CAP_UIDGID_NOMAP)) + ret = nfs_idmap_lookup_name(id, "user", buf, buflen, idmap); + if (ret < 0) + ret = nfs_map_numeric_to_string(id, buf, buflen); + trace_nfs4_map_uid_to_name(buf, ret, id, ret); + return ret; +} +int nfs_map_gid_to_group(const struct nfs_server *server, kgid_t gid, char *buf, size_t buflen) +{ + struct idmap *idmap = server->nfs_client->cl_idmap; + int ret = -EINVAL; + __u32 id; + + id = from_kgid_munged(idmap_userns(idmap), gid); + if (!(server->caps & NFS_CAP_UIDGID_NOMAP)) + ret = nfs_idmap_lookup_name(id, "group", buf, buflen, idmap); + if (ret < 0) + ret = nfs_map_numeric_to_string(id, buf, buflen); + trace_nfs4_map_gid_to_group(buf, ret, id, ret); + return ret; +} diff --git a/fs/nfs/nfs4idmap.h b/fs/nfs/nfs4idmap.h new file mode 100644 index 000000000..de44d7330 --- /dev/null +++ b/fs/nfs/nfs4idmap.h @@ -0,0 +1,68 @@ +/* + * fs/nfs/nfs4idmap.h + * + * UID and GID to name mapping for clients. + * + * Copyright (c) 2002 The Regents of the University of Michigan. + * All rights reserved. + * + * Marius Aamodt Eriksen + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef NFS_IDMAP_H +#define NFS_IDMAP_H + +#include +#include + + +/* Forward declaration to make this header independent of others */ +struct nfs_client; +struct nfs_server; +struct nfs_fattr; +struct nfs4_string; + +int nfs_idmap_init(void); +void nfs_idmap_quit(void); +int nfs_idmap_new(struct nfs_client *); +void nfs_idmap_delete(struct nfs_client *); + +void nfs_fattr_init_names(struct nfs_fattr *fattr, + struct nfs4_string *owner_name, + struct nfs4_string *group_name); +void nfs_fattr_free_names(struct nfs_fattr *); +void nfs_fattr_map_and_free_names(struct nfs_server *, struct nfs_fattr *); + +int nfs_map_name_to_uid(const struct nfs_server *, const char *, size_t, kuid_t *); +int nfs_map_group_to_gid(const struct nfs_server *, const char *, size_t, kgid_t *); +int nfs_map_uid_to_name(const struct nfs_server *, kuid_t, char *, size_t); +int nfs_map_gid_to_group(const struct nfs_server *, kgid_t, char *, size_t); + +int nfs_map_string_to_numeric(const char *name, size_t namelen, __u32 *res); + +extern unsigned int nfs_idmap_cache_timeout; +#endif /* NFS_IDMAP_H */ diff --git a/fs/nfs/nfs4namespace.c b/fs/nfs/nfs4namespace.c new file mode 100644 index 000000000..2a03f02bb --- /dev/null +++ b/fs/nfs/nfs4namespace.c @@ -0,0 +1,564 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/nfs/nfs4namespace.c + * + * Copyright (C) 2005 Trond Myklebust + * - Modified by David Howells + * + * NFSv4 namespace + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" +#include "nfs4_fs.h" +#include "nfs.h" +#include "dns_resolve.h" + +#define NFSDBG_FACILITY NFSDBG_VFS + +/* + * Work out the length that an NFSv4 path would render to as a standard posix + * path, with a leading slash but no terminating slash. + */ +static ssize_t nfs4_pathname_len(const struct nfs4_pathname *pathname) +{ + ssize_t len = 0; + int i; + + for (i = 0; i < pathname->ncomponents; i++) { + const struct nfs4_string *component = &pathname->components[i]; + + if (component->len > NAME_MAX) + goto too_long; + len += 1 + component->len; /* Adding "/foo" */ + if (len > PATH_MAX) + goto too_long; + } + return len; + +too_long: + return -ENAMETOOLONG; +} + +/* + * Convert the NFSv4 pathname components into a standard posix path. + */ +static char *nfs4_pathname_string(const struct nfs4_pathname *pathname, + unsigned short *_len) +{ + ssize_t len; + char *buf, *p; + int i; + + len = nfs4_pathname_len(pathname); + if (len < 0) + return ERR_PTR(len); + *_len = len; + + p = buf = kmalloc(len + 1, GFP_KERNEL); + if (!buf) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < pathname->ncomponents; i++) { + const struct nfs4_string *component = &pathname->components[i]; + + *p++ = '/'; + memcpy(p, component->data, component->len); + p += component->len; + } + + *p = 0; + return buf; +} + +/* + * return the path component of ":" + * nfspath - the ":" string + * end - one past the last char that could contain ":" + * returns NULL on failure + */ +static char *nfs_path_component(const char *nfspath, const char *end) +{ + char *p; + + if (*nfspath == '[') { + /* parse [] escaped IPv6 addrs */ + p = strchr(nfspath, ']'); + if (p != NULL && ++p < end && *p == ':') + return p + 1; + } else { + /* otherwise split on first colon */ + p = strchr(nfspath, ':'); + if (p != NULL && p < end) + return p + 1; + } + return NULL; +} + +/* + * Determine the mount path as a string + */ +static char *nfs4_path(struct dentry *dentry, char *buffer, ssize_t buflen) +{ + char *limit; + char *path = nfs_path(&limit, dentry, buffer, buflen, + NFS_PATH_CANONICAL); + if (!IS_ERR(path)) { + char *path_component = nfs_path_component(path, limit); + if (path_component) + return path_component; + } + return path; +} + +/* + * Check that fs_locations::fs_root [RFC3530 6.3] is a prefix for what we + * believe to be the server path to this dentry + */ +static int nfs4_validate_fspath(struct dentry *dentry, + const struct nfs4_fs_locations *locations, + struct nfs_fs_context *ctx) +{ + const char *path; + char *fs_path; + unsigned short len; + char *buf; + int n; + + buf = kmalloc(4096, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + path = nfs4_path(dentry, buf, 4096); + if (IS_ERR(path)) { + kfree(buf); + return PTR_ERR(path); + } + + fs_path = nfs4_pathname_string(&locations->fs_path, &len); + if (IS_ERR(fs_path)) { + kfree(buf); + return PTR_ERR(fs_path); + } + + n = strncmp(path, fs_path, len); + kfree(buf); + kfree(fs_path); + if (n != 0) { + dprintk("%s: path %s does not begin with fsroot %s\n", + __func__, path, ctx->nfs_server.export_path); + return -ENOENT; + } + + return 0; +} + +size_t nfs_parse_server_name(char *string, size_t len, struct sockaddr_storage *ss, + size_t salen, struct net *net, int port) +{ + struct sockaddr *sa = (struct sockaddr *)ss; + ssize_t ret; + + ret = rpc_pton(net, string, len, sa, salen); + if (ret == 0) { + ret = rpc_uaddr2sockaddr(net, string, len, sa, salen); + if (ret == 0) { + ret = nfs_dns_resolve_name(net, string, len, ss, salen); + if (ret < 0) + ret = 0; + } + } else if (port) { + rpc_set_port(sa, port); + } + return ret; +} + +/** + * nfs_find_best_sec - Find a security mechanism supported locally + * @clnt: pointer to rpc_clnt + * @server: NFS server struct + * @flavors: List of security tuples returned by SECINFO procedure + * + * Return an rpc client that uses the first security mechanism in + * "flavors" that is locally supported. The "flavors" array + * is searched in the order returned from the server, per RFC 3530 + * recommendation and each flavor is checked for membership in the + * sec= mount option list if it exists. + * + * Return -EPERM if no matching flavor is found in the array. + * + * Please call rpc_shutdown_client() when you are done with this rpc client. + * + */ +static struct rpc_clnt *nfs_find_best_sec(struct rpc_clnt *clnt, + struct nfs_server *server, + struct nfs4_secinfo_flavors *flavors) +{ + rpc_authflavor_t pflavor; + struct nfs4_secinfo4 *secinfo; + unsigned int i; + + for (i = 0; i < flavors->num_flavors; i++) { + secinfo = &flavors->flavors[i]; + + switch (secinfo->flavor) { + case RPC_AUTH_NULL: + case RPC_AUTH_UNIX: + case RPC_AUTH_GSS: + pflavor = rpcauth_get_pseudoflavor(secinfo->flavor, + &secinfo->flavor_info); + /* does the pseudoflavor match a sec= mount opt? */ + if (pflavor != RPC_AUTH_MAXFLAVOR && + nfs_auth_info_match(&server->auth_info, pflavor)) { + struct rpc_clnt *new; + struct rpc_cred *cred; + + /* Cloning creates an rpc_auth for the flavor */ + new = rpc_clone_client_set_auth(clnt, pflavor); + if (IS_ERR(new)) + continue; + /** + * Check that the user actually can use the + * flavor. This is mostly for RPC_AUTH_GSS + * where cr_init obtains a gss context + */ + cred = rpcauth_lookupcred(new->cl_auth, 0); + if (IS_ERR(cred)) { + rpc_shutdown_client(new); + continue; + } + put_rpccred(cred); + return new; + } + } + } + return ERR_PTR(-EPERM); +} + +/** + * nfs4_negotiate_security - in response to an NFS4ERR_WRONGSEC on lookup, + * return an rpc_clnt that uses the best available security flavor with + * respect to the secinfo flavor list and the sec= mount options. + * + * @clnt: RPC client to clone + * @inode: directory inode + * @name: lookup name + * + * Please call rpc_shutdown_client() when you are done with this rpc client. + */ +struct rpc_clnt * +nfs4_negotiate_security(struct rpc_clnt *clnt, struct inode *inode, + const struct qstr *name) +{ + struct page *page; + struct nfs4_secinfo_flavors *flavors; + struct rpc_clnt *new; + int err; + + page = alloc_page(GFP_KERNEL); + if (!page) + return ERR_PTR(-ENOMEM); + + flavors = page_address(page); + + err = nfs4_proc_secinfo(inode, name, flavors); + if (err < 0) { + new = ERR_PTR(err); + goto out; + } + + new = nfs_find_best_sec(clnt, NFS_SERVER(inode), flavors); + +out: + put_page(page); + return new; +} + +static int try_location(struct fs_context *fc, + const struct nfs4_fs_location *location) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + unsigned int len, s; + char *export_path, *source, *p; + int ret = -ENOENT; + + /* Allocate a buffer big enough to hold any of the hostnames plus a + * terminating char and also a buffer big enough to hold the hostname + * plus a colon plus the path. + */ + len = 0; + for (s = 0; s < location->nservers; s++) { + const struct nfs4_string *buf = &location->servers[s]; + if (buf->len > len) + len = buf->len; + } + + kfree(ctx->nfs_server.hostname); + ctx->nfs_server.hostname = kmalloc(len + 1, GFP_KERNEL); + if (!ctx->nfs_server.hostname) + return -ENOMEM; + + export_path = nfs4_pathname_string(&location->rootpath, + &ctx->nfs_server.export_path_len); + if (IS_ERR(export_path)) + return PTR_ERR(export_path); + + kfree(ctx->nfs_server.export_path); + ctx->nfs_server.export_path = export_path; + + source = kmalloc(len + 1 + ctx->nfs_server.export_path_len + 1, + GFP_KERNEL); + if (!source) + return -ENOMEM; + + kfree(fc->source); + fc->source = source; + for (s = 0; s < location->nservers; s++) { + const struct nfs4_string *buf = &location->servers[s]; + + if (memchr(buf->data, IPV6_SCOPE_DELIMITER, buf->len)) + continue; + + ctx->nfs_server.addrlen = + nfs_parse_server_name(buf->data, buf->len, + &ctx->nfs_server._address, + sizeof(ctx->nfs_server._address), + fc->net_ns, 0); + if (ctx->nfs_server.addrlen == 0) + continue; + + rpc_set_port(&ctx->nfs_server.address, NFS_PORT); + + memcpy(ctx->nfs_server.hostname, buf->data, buf->len); + ctx->nfs_server.hostname[buf->len] = '\0'; + + p = source; + memcpy(p, buf->data, buf->len); + p += buf->len; + *p++ = ':'; + memcpy(p, ctx->nfs_server.export_path, ctx->nfs_server.export_path_len); + p += ctx->nfs_server.export_path_len; + *p = 0; + + ret = nfs4_get_referral_tree(fc); + if (ret == 0) + return 0; + } + + return ret; +} + +/** + * nfs_follow_referral - set up mountpoint when hitting a referral on moved error + * @fc: pointer to struct nfs_fs_context + * @locations: array of NFSv4 server location information + * + */ +static int nfs_follow_referral(struct fs_context *fc, + const struct nfs4_fs_locations *locations) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + int loc, error; + + if (locations == NULL || locations->nlocations <= 0) + return -ENOENT; + + dprintk("%s: referral at %pd2\n", __func__, ctx->clone_data.dentry); + + /* Ensure fs path is a prefix of current dentry path */ + error = nfs4_validate_fspath(ctx->clone_data.dentry, locations, ctx); + if (error < 0) + return error; + + error = -ENOENT; + for (loc = 0; loc < locations->nlocations; loc++) { + const struct nfs4_fs_location *location = &locations->locations[loc]; + + if (location == NULL || location->nservers <= 0 || + location->rootpath.ncomponents == 0) + continue; + + error = try_location(fc, location); + if (error == 0) + return 0; + } + + return error; +} + +/* + * nfs_do_refmount - handle crossing a referral on server + * @dentry - dentry of referral + * + */ +static int nfs_do_refmount(struct fs_context *fc, struct rpc_clnt *client) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + struct dentry *dentry, *parent; + struct nfs4_fs_locations *fs_locations = NULL; + struct page *page; + int err = -ENOMEM; + + /* BUG_ON(IS_ROOT(dentry)); */ + page = alloc_page(GFP_KERNEL); + if (!page) + return -ENOMEM; + + fs_locations = kmalloc_obj(struct nfs4_fs_locations); + if (!fs_locations) + goto out_free; + fs_locations->fattr = nfs_alloc_fattr(); + if (!fs_locations->fattr) + goto out_free_2; + + /* Get locations */ + dentry = ctx->clone_data.dentry; + parent = dget_parent(dentry); + dprintk("%s: getting locations for %pd2\n", + __func__, dentry); + + err = nfs4_proc_fs_locations(client, d_inode(parent), &dentry->d_name, fs_locations, page); + dput(parent); + if (err != 0) + goto out_free_3; + + err = -ENOENT; + if (fs_locations->nlocations <= 0 || + fs_locations->fs_path.ncomponents <= 0) + goto out_free_3; + + err = nfs_follow_referral(fc, fs_locations); +out_free_3: + kfree(fs_locations->fattr); +out_free_2: + kfree(fs_locations); +out_free: + __free_page(page); + return err; +} + +int nfs4_submount(struct fs_context *fc, struct nfs_server *server) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + struct dentry *dentry = ctx->clone_data.dentry; + struct dentry *parent = dget_parent(dentry); + struct inode *dir = d_inode(parent); + struct rpc_clnt *client; + int ret; + + /* Look it up again to get its attributes and sec flavor */ + client = nfs4_proc_lookup_mountpoint(dir, dentry, ctx->mntfh, + ctx->clone_data.fattr); + dput(parent); + if (IS_ERR(client)) + return PTR_ERR(client); + + ctx->selected_flavor = client->cl_auth->au_flavor; + if (ctx->clone_data.fattr->valid & NFS_ATTR_FATTR_V4_REFERRAL) { + ret = nfs_do_refmount(fc, client); + } else { + ret = nfs_do_submount(fc); + } + + rpc_shutdown_client(client); + return ret; +} + +/* + * Try one location from the fs_locations array. + * + * Returns zero on success, or a negative errno value. + */ +static int nfs4_try_replacing_one_location(struct nfs_server *server, + const struct nfs4_fs_location *location) +{ + struct net *net = rpc_net_ns(server->client); + struct sockaddr_storage *sap; + unsigned int s; + size_t salen; + int error; + + sap = kmalloc_obj(*sap); + if (sap == NULL) + return -ENOMEM; + + error = -ENOENT; + for (s = 0; s < location->nservers; s++) { + const struct nfs4_string *buf = &location->servers[s]; + char *hostname; + + if (buf->len <= 0 || buf->len > PAGE_SIZE) + continue; + + if (memchr(buf->data, IPV6_SCOPE_DELIMITER, buf->len) != NULL) + continue; + + salen = nfs_parse_server_name(buf->data, buf->len, + sap, sizeof(*sap), net, 0); + if (salen == 0) + continue; + rpc_set_port((struct sockaddr *)sap, NFS_PORT); + + error = -ENOMEM; + hostname = kmemdup_nul(buf->data, buf->len, GFP_KERNEL); + if (hostname == NULL) + break; + + error = nfs4_update_server(server, hostname, sap, salen, net); + kfree(hostname); + if (error == 0) + break; + } + + kfree(sap); + return error; +} + +/** + * nfs4_replace_transport - set up transport to destination server + * + * @server: export being migrated + * @locations: fs_locations array + * + * Returns zero on success, or a negative errno value. + * + * The client tries all the entries in the "locations" array, in the + * order returned by the server, until one works or the end of the + * array is reached. + */ +int nfs4_replace_transport(struct nfs_server *server, + const struct nfs4_fs_locations *locations) +{ + int loc, error; + + error = -ENOENT; + if (locations == NULL || locations->nlocations <= 0) + goto out; + + for (loc = 0; loc < locations->nlocations; loc++) { + const struct nfs4_fs_location *location = + &locations->locations[loc]; + + if (location == NULL || location->nservers <= 0 || + location->rootpath.ncomponents == 0) + continue; + + error = nfs4_try_replacing_one_location(server, location); + if (error == 0) + break; + } + +out: + return error; +} diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c new file mode 100644 index 000000000..5709c6fea --- /dev/null +++ b/fs/nfs/nfs4proc.c @@ -0,0 +1,10797 @@ +/* + * fs/nfs/nfs4proc.c + * + * Client-side procedure declarations for NFSv4. + * + * Copyright (c) 2002 The Regents of the University of Michigan. + * All rights reserved. + * + * Kendrick Smith + * Andy Adamson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nfs4_fs.h" +#include "delegation.h" +#include "internal.h" +#include "iostat.h" +#include "callback.h" +#include "pnfs.h" +#include "netns.h" +#include "sysfs.h" +#include "nfs4idmap.h" +#include "nfs4session.h" +#include "fscache.h" +#include "nfs40.h" +#include "nfs42.h" + +#include "nfs4trace.h" + +#define NFSDBG_FACILITY NFSDBG_PROC + +#define NFS4_BITMASK_SZ 3 + +#define NFS4_POLL_RETRY_MIN (HZ/10) +#define NFS4_POLL_RETRY_MAX (15*HZ) + +/* file attributes which can be mapped to nfs attributes */ +#define NFS4_VALID_ATTRS (ATTR_MODE \ + | ATTR_UID \ + | ATTR_GID \ + | ATTR_SIZE \ + | ATTR_ATIME \ + | ATTR_MTIME \ + | ATTR_CTIME \ + | ATTR_ATIME_SET \ + | ATTR_MTIME_SET) + +struct nfs4_opendata; +static int _nfs4_recover_proc_open(struct nfs4_opendata *data); +static int nfs4_do_fsinfo(struct nfs_server *, struct nfs_fh *, struct nfs_fsinfo *); +static void nfs_fixup_referral_attributes(struct nfs_fattr *fattr); +static int _nfs4_proc_getattr(struct nfs_server *server, struct nfs_fh *fhandle, + struct nfs_fattr *fattr, struct inode *inode); +static int nfs4_do_setattr(struct inode *inode, const struct cred *cred, + struct nfs_fattr *fattr, struct iattr *sattr, + struct nfs_open_context *ctx, struct nfs4_label *ilabel); +static struct rpc_task *_nfs41_proc_sequence(struct nfs_client *clp, + const struct cred *cred, + struct nfs4_slot *slot, + bool is_privileged); +static int nfs41_test_stateid(struct nfs_server *, const nfs4_stateid *, + const struct cred *); +static int nfs41_free_stateid(struct nfs_server *, nfs4_stateid *, + const struct cred *, bool); + +#ifdef CONFIG_NFS_V4_SECURITY_LABEL +static inline struct nfs4_label * +nfs4_label_init_security(struct inode *dir, struct dentry *dentry, + struct iattr *sattr, struct nfs4_label *label) +{ + struct lsm_context shim; + int err; + + if (label == NULL) + return NULL; + + if (nfs_server_capable(dir, NFS_CAP_SECURITY_LABEL) == 0) + return NULL; + + label->lfs = 0; + label->pi = 0; + label->len = 0; + label->label = NULL; + + err = security_dentry_init_security(dentry, sattr->ia_mode, + &dentry->d_name, NULL, &shim); + if (err) + return NULL; + + label->lsmid = shim.id; + label->label = shim.context; + label->len = shim.len; + return label; +} +static inline void +nfs4_label_release_security(struct nfs4_label *label) +{ + struct lsm_context shim; + + if (label) { + shim.context = label->label; + shim.len = label->len; + shim.id = label->lsmid; + security_release_secctx(&shim); + } +} +static inline u32 *nfs4_bitmask(struct nfs_server *server, struct nfs4_label *label) +{ + if (label) + return server->attr_bitmask; + + return server->attr_bitmask_nl; +} +#else +static inline struct nfs4_label * +nfs4_label_init_security(struct inode *dir, struct dentry *dentry, + struct iattr *sattr, struct nfs4_label *l) +{ return NULL; } +static inline void +nfs4_label_release_security(struct nfs4_label *label) +{ return; } +static inline u32 * +nfs4_bitmask(struct nfs_server *server, struct nfs4_label *label) +{ return server->attr_bitmask; } +#endif + +/* Prevent leaks of NFSv4 errors into userland */ +static int nfs4_map_errors(int err) +{ + if (err >= -1000) + return err; + switch (err) { + case -NFS4ERR_RESOURCE: + case -NFS4ERR_LAYOUTTRYLATER: + case -NFS4ERR_RECALLCONFLICT: + case -NFS4ERR_RETURNCONFLICT: + return -EREMOTEIO; + case -NFS4ERR_WRONGSEC: + case -NFS4ERR_WRONG_CRED: + return -EPERM; + case -NFS4ERR_BADOWNER: + case -NFS4ERR_BADNAME: + return -EINVAL; + case -NFS4ERR_SHARE_DENIED: + return -EACCES; + case -NFS4ERR_MINOR_VERS_MISMATCH: + return -EPROTONOSUPPORT; + case -NFS4ERR_FILE_OPEN: + return -EBUSY; + case -NFS4ERR_NOT_SAME: + return -ENOTSYNC; + case -ENETDOWN: + case -ENETUNREACH: + break; + default: + dprintk("%s could not handle NFSv4 error %d\n", + __func__, -err); + break; + } + return -EIO; +} + +/* + * This is our standard bitmap for GETATTR requests. + */ +const u32 nfs4_fattr_bitmap[3] = { + FATTR4_WORD0_TYPE + | FATTR4_WORD0_CHANGE + | FATTR4_WORD0_SIZE + | FATTR4_WORD0_FSID + | FATTR4_WORD0_FILEID, + FATTR4_WORD1_MODE + | FATTR4_WORD1_NUMLINKS + | FATTR4_WORD1_OWNER + | FATTR4_WORD1_OWNER_GROUP + | FATTR4_WORD1_RAWDEV + | FATTR4_WORD1_SPACE_USED + | FATTR4_WORD1_TIME_ACCESS + | FATTR4_WORD1_TIME_CREATE + | FATTR4_WORD1_TIME_METADATA + | FATTR4_WORD1_TIME_MODIFY + | FATTR4_WORD1_MOUNTED_ON_FILEID, +#ifdef CONFIG_NFS_V4_SECURITY_LABEL + FATTR4_WORD2_SECURITY_LABEL +#endif +}; + +static const u32 nfs4_pnfs_open_bitmap[3] = { + FATTR4_WORD0_TYPE + | FATTR4_WORD0_CHANGE + | FATTR4_WORD0_SIZE + | FATTR4_WORD0_FSID + | FATTR4_WORD0_FILEID, + FATTR4_WORD1_MODE + | FATTR4_WORD1_NUMLINKS + | FATTR4_WORD1_OWNER + | FATTR4_WORD1_OWNER_GROUP + | FATTR4_WORD1_RAWDEV + | FATTR4_WORD1_SPACE_USED + | FATTR4_WORD1_TIME_ACCESS + | FATTR4_WORD1_TIME_CREATE + | FATTR4_WORD1_TIME_METADATA + | FATTR4_WORD1_TIME_MODIFY, + FATTR4_WORD2_MDSTHRESHOLD +#ifdef CONFIG_NFS_V4_SECURITY_LABEL + | FATTR4_WORD2_SECURITY_LABEL +#endif +}; + +static const u32 nfs4_open_noattr_bitmap[3] = { + FATTR4_WORD0_TYPE + | FATTR4_WORD0_FILEID, +}; + +const u32 nfs4_statfs_bitmap[3] = { + FATTR4_WORD0_FILES_AVAIL + | FATTR4_WORD0_FILES_FREE + | FATTR4_WORD0_FILES_TOTAL, + FATTR4_WORD1_SPACE_AVAIL + | FATTR4_WORD1_SPACE_FREE + | FATTR4_WORD1_SPACE_TOTAL +}; + +const u32 nfs4_pathconf_bitmap[3] = { + FATTR4_WORD0_MAXLINK + | FATTR4_WORD0_MAXNAME, + 0 +}; + +const u32 nfs4_fsinfo_bitmap[3] = { FATTR4_WORD0_MAXFILESIZE + | FATTR4_WORD0_MAXREAD + | FATTR4_WORD0_MAXWRITE + | FATTR4_WORD0_LEASE_TIME, + FATTR4_WORD1_TIME_DELTA + | FATTR4_WORD1_FS_LAYOUT_TYPES, + FATTR4_WORD2_LAYOUT_BLKSIZE + | FATTR4_WORD2_CLONE_BLKSIZE + | FATTR4_WORD2_CHANGE_ATTR_TYPE + | FATTR4_WORD2_XATTR_SUPPORT +}; + +const u32 nfs4_fs_locations_bitmap[3] = { + FATTR4_WORD0_CHANGE + | FATTR4_WORD0_SIZE + | FATTR4_WORD0_FSID + | FATTR4_WORD0_FILEID + | FATTR4_WORD0_FS_LOCATIONS, + FATTR4_WORD1_OWNER + | FATTR4_WORD1_OWNER_GROUP + | FATTR4_WORD1_RAWDEV + | FATTR4_WORD1_SPACE_USED + | FATTR4_WORD1_TIME_ACCESS + | FATTR4_WORD1_TIME_METADATA + | FATTR4_WORD1_TIME_MODIFY + | FATTR4_WORD1_MOUNTED_ON_FILEID, +}; + +static void nfs4_bitmap_copy_adjust(__u32 *dst, const __u32 *src, + struct inode *inode, unsigned long flags) +{ + unsigned long cache_validity; + + memcpy(dst, src, NFS4_BITMASK_SZ*sizeof(*dst)); + if (!inode || !nfs_have_read_or_write_delegation(inode)) + return; + + cache_validity = READ_ONCE(NFS_I(inode)->cache_validity) | flags; + + /* Remove the attributes over which we have full control */ + dst[1] &= ~FATTR4_WORD1_RAWDEV; + if (!(cache_validity & NFS_INO_INVALID_SIZE)) + dst[0] &= ~FATTR4_WORD0_SIZE; + + if (!(cache_validity & NFS_INO_INVALID_CHANGE)) + dst[0] &= ~FATTR4_WORD0_CHANGE; + + if (!(cache_validity & NFS_INO_INVALID_MODE)) + dst[1] &= ~FATTR4_WORD1_MODE; + if (!(cache_validity & NFS_INO_INVALID_OTHER)) + dst[1] &= ~(FATTR4_WORD1_OWNER | FATTR4_WORD1_OWNER_GROUP); + + if (!(cache_validity & NFS_INO_INVALID_BTIME)) + dst[1] &= ~FATTR4_WORD1_TIME_CREATE; + + if (nfs_have_delegated_mtime(inode)) { + if (!(cache_validity & NFS_INO_INVALID_ATIME)) + dst[1] &= ~(FATTR4_WORD1_TIME_ACCESS|FATTR4_WORD1_TIME_ACCESS_SET); + if (!(cache_validity & NFS_INO_INVALID_MTIME)) + dst[1] &= ~(FATTR4_WORD1_TIME_MODIFY|FATTR4_WORD1_TIME_MODIFY_SET); + if (!(cache_validity & NFS_INO_INVALID_CTIME)) + dst[1] &= ~(FATTR4_WORD1_TIME_METADATA|FATTR4_WORD1_TIME_MODIFY_SET); + } else if (nfs_have_delegated_atime(inode)) { + if (!(cache_validity & NFS_INO_INVALID_ATIME)) + dst[1] &= ~(FATTR4_WORD1_TIME_ACCESS|FATTR4_WORD1_TIME_ACCESS_SET); + } +} + +static void nfs4_setup_readdir(u64 cookie, __be32 *verifier, struct dentry *dentry, + struct nfs4_readdir_arg *readdir) +{ + unsigned int attrs = FATTR4_WORD0_FILEID | FATTR4_WORD0_TYPE; + __be32 *start, *p; + + if (cookie > 2) { + readdir->cookie = cookie; + memcpy(&readdir->verifier, verifier, sizeof(readdir->verifier)); + return; + } + + readdir->cookie = 0; + memset(&readdir->verifier, 0, sizeof(readdir->verifier)); + if (cookie == 2) + return; + + /* + * NFSv4 servers do not return entries for '.' and '..' + * Therefore, we fake these entries here. We let '.' + * have cookie 0 and '..' have cookie 1. Note that + * when talking to the server, we always send cookie 0 + * instead of 1 or 2. + */ + start = p = kmap_atomic(*readdir->pages); + + if (cookie == 0) { + *p++ = xdr_one; /* next */ + *p++ = xdr_zero; /* cookie, first word */ + *p++ = xdr_one; /* cookie, second word */ + *p++ = xdr_one; /* entry len */ + memcpy(p, ".\0\0\0", 4); /* entry */ + p++; + *p++ = xdr_one; /* bitmap length */ + *p++ = htonl(attrs); /* bitmap */ + *p++ = htonl(12); /* attribute buffer length */ + *p++ = htonl(NF4DIR); + p = xdr_encode_hyper(p, d_inode(dentry)->i_ino); + } + + *p++ = xdr_one; /* next */ + *p++ = xdr_zero; /* cookie, first word */ + *p++ = xdr_two; /* cookie, second word */ + *p++ = xdr_two; /* entry len */ + memcpy(p, "..\0\0", 4); /* entry */ + p++; + *p++ = xdr_one; /* bitmap length */ + *p++ = htonl(attrs); /* bitmap */ + *p++ = htonl(12); /* attribute buffer length */ + *p++ = htonl(NF4DIR); + spin_lock(&dentry->d_lock); + p = xdr_encode_hyper(p, d_inode(dentry->d_parent)->i_ino); + spin_unlock(&dentry->d_lock); + + readdir->pgbase = (char *)p - (char *)start; + readdir->count -= readdir->pgbase; + kunmap_atomic(start); +} + +static void nfs4_fattr_set_prechange(struct nfs_fattr *fattr, u64 version) +{ + if (!(fattr->valid & NFS_ATTR_FATTR_PRECHANGE)) { + fattr->pre_change_attr = version; + fattr->valid |= NFS_ATTR_FATTR_PRECHANGE; + } +} + +static void nfs4_test_and_free_stateid(struct nfs_server *server, + nfs4_stateid *stateid, + const struct cred *cred) +{ + const struct nfs4_minor_version_ops *ops = server->nfs_client->cl_mvops; + + ops->test_and_free_expired(server, stateid, cred); +} + +static void __nfs4_free_revoked_stateid(struct nfs_server *server, + nfs4_stateid *stateid, + const struct cred *cred) +{ + stateid->type = NFS4_REVOKED_STATEID_TYPE; + nfs4_test_and_free_stateid(server, stateid, cred); +} + +static void nfs4_free_revoked_stateid(struct nfs_server *server, + const nfs4_stateid *stateid, + const struct cred *cred) +{ + nfs4_stateid tmp; + + nfs4_stateid_copy(&tmp, stateid); + __nfs4_free_revoked_stateid(server, &tmp, cred); +} + +static long nfs4_update_delay(long *timeout) +{ + long ret; + if (!timeout) + return NFS4_POLL_RETRY_MAX; + if (*timeout <= 0) + *timeout = NFS4_POLL_RETRY_MIN; + if (*timeout > NFS4_POLL_RETRY_MAX) + *timeout = NFS4_POLL_RETRY_MAX; + ret = *timeout; + *timeout <<= 1; + return ret; +} + +static int nfs4_delay_killable(long *timeout) +{ + might_sleep(); + + if (unlikely(nfs_current_task_exiting())) + return -EINTR; + __set_current_state(TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); + schedule_timeout(nfs4_update_delay(timeout)); + if (!__fatal_signal_pending(current)) + return 0; + return -EINTR; +} + +static int nfs4_delay_interruptible(long *timeout) +{ + might_sleep(); + + if (unlikely(nfs_current_task_exiting())) + return -EINTR; + __set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE_UNSAFE); + schedule_timeout(nfs4_update_delay(timeout)); + if (!signal_pending(current)) + return 0; + return __fatal_signal_pending(current) ? -EINTR :-ERESTARTSYS; +} + +static int nfs4_delay(long *timeout, bool interruptible) +{ + if (interruptible) + return nfs4_delay_interruptible(timeout); + return nfs4_delay_killable(timeout); +} + +static const nfs4_stateid * +nfs4_recoverable_stateid(const nfs4_stateid *stateid) +{ + if (!stateid) + return NULL; + switch (stateid->type) { + case NFS4_OPEN_STATEID_TYPE: + case NFS4_LOCK_STATEID_TYPE: + case NFS4_DELEGATION_STATEID_TYPE: + return stateid; + default: + break; + } + return NULL; +} + +/* This is the error handling routine for processes that are allowed + * to sleep. + */ +static int nfs4_do_handle_exception(struct nfs_server *server, + int errorcode, struct nfs4_exception *exception) +{ + struct nfs_client *clp = server->nfs_client; + struct nfs4_state *state = exception->state; + const nfs4_stateid *stateid; + struct inode *inode = exception->inode; + int ret = errorcode; + + exception->delay = 0; + exception->recovering = 0; + exception->retry = 0; + + stateid = nfs4_recoverable_stateid(exception->stateid); + if (stateid == NULL && state != NULL) + stateid = nfs4_recoverable_stateid(&state->stateid); + + switch(errorcode) { + case 0: + return 0; + case -NFS4ERR_BADHANDLE: + case -ESTALE: + if (inode != NULL && S_ISREG(inode->i_mode)) + pnfs_destroy_layout(NFS_I(inode)); + break; + case -NFS4ERR_DELEG_REVOKED: + case -NFS4ERR_ADMIN_REVOKED: + case -NFS4ERR_EXPIRED: + case -NFS4ERR_BAD_STATEID: + case -NFS4ERR_PARTNER_NO_AUTH: + if (inode != NULL && stateid != NULL) { + nfs_inode_find_state_and_recover(inode, + stateid); + goto wait_on_recovery; + } + fallthrough; + case -NFS4ERR_OPENMODE: + if (inode) { + int err; + + err = nfs_async_inode_return_delegation(inode, + stateid); + if (err == 0) + goto wait_on_recovery; + if (stateid != NULL && stateid->type == NFS4_DELEGATION_STATEID_TYPE) { + exception->retry = 1; + break; + } + } + if (state == NULL) + break; + ret = nfs4_schedule_stateid_recovery(server, state); + if (ret < 0) + break; + goto wait_on_recovery; + case -NFS4ERR_STALE_STATEID: + case -NFS4ERR_STALE_CLIENTID: + nfs4_schedule_lease_recovery(clp); + goto wait_on_recovery; + case -NFS4ERR_MOVED: + ret = nfs4_schedule_migration_recovery(server); + if (ret < 0) + break; + goto wait_on_recovery; + case -NFS4ERR_LEASE_MOVED: + nfs4_schedule_lease_moved_recovery(clp); + goto wait_on_recovery; + case -NFS4ERR_BADSESSION: + case -NFS4ERR_BADSLOT: + case -NFS4ERR_BAD_HIGH_SLOT: + case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION: + case -NFS4ERR_DEADSESSION: + case -NFS4ERR_SEQ_FALSE_RETRY: + case -NFS4ERR_SEQ_MISORDERED: + /* Handled in nfs41_sequence_process() */ + goto wait_on_recovery; + case -NFS4ERR_FILE_OPEN: + if (exception->timeout > HZ) { + /* We have retried a decent amount, time to + * fail + */ + ret = -EBUSY; + break; + } + fallthrough; + case -NFS4ERR_DELAY: + nfs_inc_server_stats(server, NFSIOS_DELAY); + fallthrough; + case -NFS4ERR_GRACE: + case -NFS4ERR_LAYOUTTRYLATER: + case -NFS4ERR_RECALLCONFLICT: + case -NFS4ERR_RETURNCONFLICT: + exception->delay = 1; + return 0; + + case -NFS4ERR_RETRY_UNCACHED_REP: + case -NFS4ERR_OLD_STATEID: + exception->retry = 1; + break; + case -NFS4ERR_BADOWNER: + /* The following works around a Linux server bug! */ + case -NFS4ERR_BADNAME: + if (server->caps & NFS_CAP_UIDGID_NOMAP) { + server->caps &= ~NFS_CAP_UIDGID_NOMAP; + exception->retry = 1; + printk(KERN_WARNING "NFS: v4 server %s " + "does not accept raw " + "uid/gids. " + "Reenabling the idmapper.\n", + server->nfs_client->cl_hostname); + } + } + /* We failed to handle the error */ + return nfs4_map_errors(ret); +wait_on_recovery: + exception->recovering = 1; + return 0; +} + +/* + * Track the number of NFS4ERR_DELAY related retransmissions and return + * EAGAIN if the 'softerr' mount option is set, and we've exceeded the limit + * set by 'nfs_delay_retrans'. + */ +static int nfs4_exception_should_retrans(const struct nfs_server *server, + struct nfs4_exception *exception) +{ + if (server->flags & NFS_MOUNT_SOFTERR && nfs_delay_retrans >= 0) { + if (exception->retrans++ >= (unsigned short)nfs_delay_retrans) + return -EAGAIN; + } + return 0; +} + +/* This is the error handling routine for processes that are allowed + * to sleep. + */ +int nfs4_handle_exception(struct nfs_server *server, int errorcode, struct nfs4_exception *exception) +{ + struct nfs_client *clp = server->nfs_client; + int ret; + + ret = nfs4_do_handle_exception(server, errorcode, exception); + if (exception->delay) { + int ret2 = nfs4_exception_should_retrans(server, exception); + if (ret2 < 0) { + exception->retry = 0; + return ret2; + } + ret = nfs4_delay(&exception->timeout, + exception->interruptible); + goto out_retry; + } + if (exception->recovering) { + if (exception->task_is_privileged) + return -EDEADLOCK; + ret = nfs4_wait_clnt_recover(clp); + if (test_bit(NFS_MIG_FAILED, &server->mig_status)) + return -EIO; + goto out_retry; + } + return ret; +out_retry: + if (ret == 0) + exception->retry = 1; + return ret; +} + +static int +nfs4_async_handle_exception(struct rpc_task *task, struct nfs_server *server, + int errorcode, struct nfs4_exception *exception) +{ + struct nfs_client *clp = server->nfs_client; + int ret; + + if ((task->tk_rpc_status == -ENETDOWN || + task->tk_rpc_status == -ENETUNREACH) && + task->tk_flags & RPC_TASK_NETUNREACH_FATAL) { + exception->delay = 0; + exception->recovering = 0; + exception->retry = 0; + return -EIO; + } + + ret = nfs4_do_handle_exception(server, errorcode, exception); + if (exception->delay) { + int ret2 = nfs4_exception_should_retrans(server, exception); + if (ret2 < 0) { + exception->retry = 0; + return ret2; + } + rpc_delay(task, nfs4_update_delay(&exception->timeout)); + goto out_retry; + } + if (exception->recovering) { + if (exception->task_is_privileged) + return -EDEADLOCK; + rpc_sleep_on(&clp->cl_rpcwaitq, task, NULL); + if (test_bit(NFS4CLNT_MANAGER_RUNNING, &clp->cl_state) == 0) + rpc_wake_up_queued_task(&clp->cl_rpcwaitq, task); + goto out_retry; + } + if (test_bit(NFS_MIG_FAILED, &server->mig_status)) + ret = -EIO; + return ret; +out_retry: + if (ret == 0) { + exception->retry = 1; + /* + * For NFS4ERR_MOVED, the client transport will need to + * be recomputed after migration recovery has completed. + */ + if (errorcode == -NFS4ERR_MOVED) + rpc_task_release_transport(task); + } + return ret; +} + +int +nfs4_async_handle_error(struct rpc_task *task, struct nfs_server *server, + struct nfs4_state *state, long *timeout) +{ + struct nfs4_exception exception = { + .state = state, + }; + + if (task->tk_status >= 0) + return 0; + if (timeout) + exception.timeout = *timeout; + task->tk_status = nfs4_async_handle_exception(task, server, + task->tk_status, + &exception); + if (exception.delay && timeout) + *timeout = exception.timeout; + if (exception.retry) + return -EAGAIN; + return 0; +} + +/* + * Return 'true' if 'clp' is using an rpc_client that is integrity protected + * or 'false' otherwise. + */ +static bool _nfs4_is_integrity_protected(struct nfs_client *clp) +{ + rpc_authflavor_t flavor = clp->cl_rpcclient->cl_auth->au_flavor; + return (flavor == RPC_AUTH_GSS_KRB5I) || (flavor == RPC_AUTH_GSS_KRB5P); +} + +void do_renew_lease(struct nfs_client *clp, unsigned long timestamp) +{ + spin_lock(&clp->cl_lock); + if (time_before(clp->cl_last_renewal,timestamp)) + clp->cl_last_renewal = timestamp; + spin_unlock(&clp->cl_lock); +} + +void renew_lease(const struct nfs_server *server, unsigned long timestamp) +{ + struct nfs_client *clp = server->nfs_client; + + if (!nfs4_has_session(clp)) + do_renew_lease(clp, timestamp); +} + +void nfs4_init_sequence(struct nfs_client *clp, + struct nfs4_sequence_args *args, + struct nfs4_sequence_res *res, int cache_reply, + int privileged) +{ + args->sa_slot = NULL; + args->sa_cache_this = cache_reply; + args->sa_privileged = privileged; + + res->sr_slot = NULL; + res->sr_slot_ops = clp->cl_mvops->sequence_slot_ops; +} + +static void nfs41_release_slot(struct nfs4_slot *slot) +{ + struct nfs4_session *session; + struct nfs4_slot_table *tbl; + bool send_new_highest_used_slotid = false; + + if (!slot) + return; + tbl = slot->table; + session = tbl->session; + + /* Bump the slot sequence number */ + if (slot->seq_done) + slot->seq_nr++; + slot->seq_done = 0; + + spin_lock(&tbl->slot_tbl_lock); + /* Be nice to the server: try to ensure that the last transmitted + * value for highest_user_slotid <= target_highest_slotid + */ + if (tbl->highest_used_slotid > tbl->target_highest_slotid) + send_new_highest_used_slotid = true; + + if (nfs41_wake_and_assign_slot(tbl, slot)) { + send_new_highest_used_slotid = false; + goto out_unlock; + } + nfs4_free_slot(tbl, slot); + + if (tbl->highest_used_slotid != NFS4_NO_SLOT) + send_new_highest_used_slotid = false; +out_unlock: + spin_unlock(&tbl->slot_tbl_lock); + if (send_new_highest_used_slotid) + nfs41_notify_server(session->clp); + if (waitqueue_active(&tbl->slot_waitq)) + wake_up_all(&tbl->slot_waitq); +} + +static void nfs41_sequence_free_slot(struct nfs4_sequence_res *res) +{ + nfs41_release_slot(res->sr_slot); + res->sr_slot = NULL; +} + +static void nfs4_slot_sequence_record_sent(struct nfs4_slot *slot, + u32 seqnr) +{ + if ((s32)(seqnr - slot->seq_nr_highest_sent) > 0) + slot->seq_nr_highest_sent = seqnr; +} +static void nfs4_slot_sequence_acked(struct nfs4_slot *slot, u32 seqnr) +{ + nfs4_slot_sequence_record_sent(slot, seqnr); + slot->seq_nr_last_acked = seqnr; +} + +static void nfs4_probe_sequence(struct nfs_client *client, const struct cred *cred, + struct nfs4_slot *slot) +{ + struct rpc_task *task = _nfs41_proc_sequence(client, cred, slot, true); + if (!IS_ERR(task)) + rpc_put_task_async(task); +} + +static int nfs41_sequence_process(struct rpc_task *task, + struct nfs4_sequence_res *res) +{ + struct nfs4_session *session; + struct nfs4_slot *slot = res->sr_slot; + struct nfs_client *clp; + int status; + int ret = 1; + + if (slot == NULL) + goto out_noaction; + /* don't increment the sequence number if the task wasn't sent */ + if (!RPC_WAS_SENT(task) || slot->seq_done) + goto out; + + session = slot->table->session; + clp = session->clp; + + trace_nfs4_sequence_done(session, res); + + status = res->sr_status; + if (task->tk_status == -NFS4ERR_DEADSESSION) + status = -NFS4ERR_DEADSESSION; + + /* Check the SEQUENCE operation status */ + switch (status) { + case 0: + /* Mark this sequence number as having been acked */ + nfs4_slot_sequence_acked(slot, slot->seq_nr); + /* Update the slot's sequence and clientid lease timer */ + slot->seq_done = 1; + do_renew_lease(clp, res->sr_timestamp); + /* Check sequence flags */ + nfs41_handle_sequence_flag_errors(clp, res->sr_status_flags, + !!slot->privileged); + nfs41_update_target_slotid(slot->table, slot, res); + break; + case 1: + /* + * sr_status remains 1 if an RPC level error occurred. + * The server may or may not have processed the sequence + * operation.. + */ + nfs4_slot_sequence_record_sent(slot, slot->seq_nr); + slot->seq_done = 1; + goto out; + case -NFS4ERR_DELAY: + /* The server detected a resend of the RPC call and + * returned NFS4ERR_DELAY as per Section 2.10.6.2 + * of RFC5661. + */ + dprintk("%s: slot=%u seq=%u: Operation in progress\n", + __func__, + slot->slot_nr, + slot->seq_nr); + goto out_retry; + case -NFS4ERR_RETRY_UNCACHED_REP: + case -NFS4ERR_SEQ_FALSE_RETRY: + /* + * The server thinks we tried to replay a request. + * Retry the call after bumping the sequence ID. + */ + nfs4_slot_sequence_acked(slot, slot->seq_nr); + goto retry_new_seq; + case -NFS4ERR_BADSLOT: + /* + * The slot id we used was probably retired. Try again + * using a different slot id. + */ + if (slot->slot_nr < slot->table->target_highest_slotid) + goto session_recover; + goto retry_nowait; + case -NFS4ERR_SEQ_MISORDERED: + nfs4_slot_sequence_record_sent(slot, slot->seq_nr); + /* + * Were one or more calls using this slot interrupted? + * If the server never received the request, then our + * transmitted slot sequence number may be too high. However, + * if the server did receive the request then it might + * accidentally give us a reply with a mismatched operation. + * We can sort this out by sending a lone sequence operation + * to the server on the same slot. + */ + if ((s32)(slot->seq_nr - slot->seq_nr_last_acked) > 1) { + slot->seq_nr--; + if (task->tk_msg.rpc_proc != &nfs4_procedures[NFSPROC4_CLNT_SEQUENCE]) { + nfs4_probe_sequence(clp, task->tk_msg.rpc_cred, slot); + res->sr_slot = NULL; + } + goto retry_nowait; + } + /* + * RFC5661: + * A retry might be sent while the original request is + * still in progress on the replier. The replier SHOULD + * deal with the issue by returning NFS4ERR_DELAY as the + * reply to SEQUENCE or CB_SEQUENCE operation, but + * implementations MAY return NFS4ERR_SEQ_MISORDERED. + * + * Restart the search after a delay. + */ + slot->seq_nr = slot->seq_nr_highest_sent; + goto out_retry; + case -NFS4ERR_BADSESSION: + case -NFS4ERR_DEADSESSION: + case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION: + goto session_recover; + default: + /* Just update the slot sequence no. */ + slot->seq_done = 1; + } +out: + /* The session may be reset by one of the error handlers. */ + dprintk("%s: Error %d free the slot \n", __func__, res->sr_status); +out_noaction: + return ret; +session_recover: + set_bit(NFS4_SLOT_TBL_DRAINING, &session->fc_slot_table.slot_tbl_state); + nfs4_schedule_session_recovery(session, status); + dprintk("%s ERROR: %d Reset session\n", __func__, status); + nfs41_sequence_free_slot(res); + goto out; +retry_new_seq: + ++slot->seq_nr; +retry_nowait: + if (rpc_restart_call_prepare(task)) { + nfs41_sequence_free_slot(res); + task->tk_status = 0; + ret = 0; + } + goto out; +out_retry: + if (!rpc_restart_call(task)) + goto out; + rpc_delay(task, NFS4_POLL_RETRY_MAX); + return 0; +} + +int nfs41_sequence_done(struct rpc_task *task, struct nfs4_sequence_res *res) +{ + if (!nfs41_sequence_process(task, res)) + return 0; + if (res->sr_slot != NULL) + nfs41_sequence_free_slot(res); + return 1; + +} +EXPORT_SYMBOL_GPL(nfs41_sequence_done); + +static void nfs41_call_sync_prepare(struct rpc_task *task, void *calldata) +{ + struct nfs4_call_sync_data *data = calldata; + + dprintk("--> %s data->seq_server %p\n", __func__, data->seq_server); + + nfs4_setup_sequence(data->seq_server->nfs_client, + data->seq_args, data->seq_res, task); +} + +static void nfs41_call_sync_done(struct rpc_task *task, void *calldata) +{ + struct nfs4_call_sync_data *data = calldata; + + nfs41_sequence_done(task, data->seq_res); +} + +static const struct rpc_call_ops nfs41_call_sync_ops = { + .rpc_call_prepare = nfs41_call_sync_prepare, + .rpc_call_done = nfs41_call_sync_done, +}; + + +static void nfs41_sequence_res_init(struct nfs4_sequence_res *res) +{ + res->sr_timestamp = jiffies; + res->sr_status_flags = 0; + res->sr_status = 1; +} + +static +void nfs4_sequence_attach_slot(struct nfs4_sequence_args *args, + struct nfs4_sequence_res *res, + struct nfs4_slot *slot) +{ + if (!slot) + return; + slot->privileged = args->sa_privileged ? 1 : 0; + args->sa_slot = slot; + + res->sr_slot = slot; +} + +static void nfs4_sequence_free_slot(struct nfs4_sequence_res *res) +{ + if (res->sr_slot != NULL) + res->sr_slot_ops->free_slot(res); +} + +static int nfs4_sequence_process(struct rpc_task *task, struct nfs4_sequence_res *res) +{ + if (res->sr_slot == NULL) + return 1; + return res->sr_slot_ops->process(task, res); +} + +int nfs4_sequence_done(struct rpc_task *task, struct nfs4_sequence_res *res) +{ + if (res->sr_slot == NULL) + return 1; + return res->sr_slot_ops->done(task, res); +} +EXPORT_SYMBOL_GPL(nfs4_sequence_done); + + +int nfs4_setup_sequence(struct nfs_client *client, + struct nfs4_sequence_args *args, + struct nfs4_sequence_res *res, + struct rpc_task *task) +{ + struct nfs4_session *session = nfs4_get_session(client); + struct nfs4_slot_table *tbl = client->cl_slot_tbl; + struct nfs4_slot *slot; + + /* slot already allocated? */ + if (res->sr_slot != NULL) + goto out_start; + + if (session) + tbl = &session->fc_slot_table; + + spin_lock(&tbl->slot_tbl_lock); + /* The state manager will wait until the slot table is empty */ + if (nfs4_slot_tbl_draining(tbl) && !args->sa_privileged) + goto out_sleep; + + slot = nfs4_alloc_slot(tbl); + if (IS_ERR(slot)) { + if (slot == ERR_PTR(-ENOMEM)) + goto out_sleep_timeout; + goto out_sleep; + } + spin_unlock(&tbl->slot_tbl_lock); + + nfs4_sequence_attach_slot(args, res, slot); + + trace_nfs4_setup_sequence(session, args); +out_start: + nfs41_sequence_res_init(res); + rpc_call_start(task); + return 0; +out_sleep_timeout: + /* Try again in 1/4 second */ + if (args->sa_privileged) + rpc_sleep_on_priority_timeout(&tbl->slot_tbl_waitq, task, + jiffies + (HZ >> 2), RPC_PRIORITY_PRIVILEGED); + else + rpc_sleep_on_timeout(&tbl->slot_tbl_waitq, task, + NULL, jiffies + (HZ >> 2)); + spin_unlock(&tbl->slot_tbl_lock); + return -EAGAIN; +out_sleep: + if (args->sa_privileged) + rpc_sleep_on_priority(&tbl->slot_tbl_waitq, task, + RPC_PRIORITY_PRIVILEGED); + else + rpc_sleep_on(&tbl->slot_tbl_waitq, task, NULL); + spin_unlock(&tbl->slot_tbl_lock); + return -EAGAIN; +} +EXPORT_SYMBOL_GPL(nfs4_setup_sequence); + +static int nfs4_call_sync_custom(struct rpc_task_setup *task_setup) +{ + int ret; + struct rpc_task *task; + + task = rpc_run_task(task_setup); + if (IS_ERR(task)) + return PTR_ERR(task); + + ret = task->tk_status; + rpc_put_task(task); + return ret; +} + +static int nfs4_do_call_sync(struct rpc_clnt *clnt, + struct nfs_server *server, + struct rpc_message *msg, + struct nfs4_sequence_args *args, + struct nfs4_sequence_res *res, + unsigned short task_flags) +{ + struct nfs_client *clp = server->nfs_client; + struct nfs4_call_sync_data data = { + .seq_server = server, + .seq_args = args, + .seq_res = res, + }; + struct rpc_task_setup task_setup = { + .rpc_client = clnt, + .rpc_message = msg, + .callback_ops = clp->cl_mvops->call_sync_ops, + .callback_data = &data, + .flags = task_flags, + }; + + return nfs4_call_sync_custom(&task_setup); +} + +int nfs4_call_sync_sequence(struct rpc_clnt *clnt, + struct nfs_server *server, + struct rpc_message *msg, + struct nfs4_sequence_args *args, + struct nfs4_sequence_res *res) +{ + unsigned short task_flags = 0; + + if (server->caps & NFS_CAP_MOVEABLE) + task_flags = RPC_TASK_MOVEABLE; + return nfs4_do_call_sync(clnt, server, msg, args, res, task_flags); +} + + +int nfs4_call_sync(struct rpc_clnt *clnt, + struct nfs_server *server, + struct rpc_message *msg, + struct nfs4_sequence_args *args, + struct nfs4_sequence_res *res, + int cache_reply) +{ + nfs4_init_sequence(server->nfs_client, args, res, cache_reply, 0); + return nfs4_call_sync_sequence(clnt, server, msg, args, res); +} + +static void +nfs4_inc_nlink_locked(struct inode *inode) +{ + nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE | + NFS_INO_INVALID_CTIME | + NFS_INO_INVALID_NLINK); + inc_nlink(inode); +} + +static void +nfs4_inc_nlink(struct inode *inode) +{ + spin_lock(&inode->i_lock); + nfs4_inc_nlink_locked(inode); + spin_unlock(&inode->i_lock); +} + +static void +nfs4_dec_nlink_locked(struct inode *inode) +{ + nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE | + NFS_INO_INVALID_CTIME | + NFS_INO_INVALID_NLINK); + drop_nlink(inode); +} + +static void +nfs4_update_changeattr_locked(struct inode *inode, + struct nfs4_change_info *cinfo, + unsigned long timestamp, unsigned long cache_validity) +{ + struct nfs_inode *nfsi = NFS_I(inode); + u64 change_attr = inode_peek_iversion_raw(inode); + + if (!nfs_have_delegated_mtime(inode)) + cache_validity |= NFS_INO_INVALID_CTIME | NFS_INO_INVALID_MTIME; + if (S_ISDIR(inode->i_mode)) + cache_validity |= NFS_INO_INVALID_DATA; + + switch (NFS_SERVER(inode)->change_attr_type) { + case NFS4_CHANGE_TYPE_IS_UNDEFINED: + if (cinfo->after == change_attr) + goto out; + break; + default: + if ((s64)(change_attr - cinfo->after) >= 0) + goto out; + } + + inode_set_iversion_raw(inode, cinfo->after); + if (!cinfo->atomic || cinfo->before != change_attr) { + if (S_ISDIR(inode->i_mode)) + nfs_force_lookup_revalidate(inode); + + if (!nfs_have_delegated_attributes(inode)) + cache_validity |= + NFS_INO_INVALID_ACCESS | NFS_INO_INVALID_ACL | + NFS_INO_INVALID_SIZE | NFS_INO_INVALID_OTHER | + NFS_INO_INVALID_BLOCKS | NFS_INO_INVALID_NLINK | + NFS_INO_INVALID_MODE | NFS_INO_INVALID_BTIME | + NFS_INO_INVALID_XATTR; + nfsi->attrtimeo = NFS_MINATTRTIMEO(inode); + } + nfsi->attrtimeo_timestamp = jiffies; + nfsi->read_cache_jiffies = timestamp; + nfsi->attr_gencount = nfs_inc_attr_generation_counter(); + nfsi->cache_validity &= ~NFS_INO_INVALID_CHANGE; +out: + nfs_set_cache_invalid(inode, cache_validity); +} + +void +nfs4_update_changeattr(struct inode *dir, struct nfs4_change_info *cinfo, + unsigned long timestamp, unsigned long cache_validity) +{ + spin_lock(&dir->i_lock); + nfs4_update_changeattr_locked(dir, cinfo, timestamp, cache_validity); + spin_unlock(&dir->i_lock); +} + +struct nfs4_open_createattrs { + struct nfs4_label *label; + struct iattr *sattr; + const __u32 verf[2]; +}; + +static bool nfs4_clear_cap_atomic_open_v1(struct nfs_server *server, + int err, struct nfs4_exception *exception) +{ + if (err != -EINVAL) + return false; + if (!(server->caps & NFS_CAP_ATOMIC_OPEN_V1)) + return false; + server->caps &= ~NFS_CAP_ATOMIC_OPEN_V1; + exception->retry = 1; + return true; +} + +static fmode_t _nfs4_ctx_to_accessmode(const struct nfs_open_context *ctx) +{ + return ctx->mode & (FMODE_READ|FMODE_WRITE|FMODE_EXEC); +} + +static fmode_t _nfs4_ctx_to_openmode(const struct nfs_open_context *ctx) +{ + fmode_t ret = ctx->mode & (FMODE_READ|FMODE_WRITE); + + return (ctx->mode & FMODE_EXEC) ? FMODE_READ | ret : ret; +} + +static u32 +nfs4_fmode_to_share_access(fmode_t fmode) +{ + u32 res = 0; + + switch (fmode & (FMODE_READ | FMODE_WRITE)) { + case FMODE_READ: + res = NFS4_SHARE_ACCESS_READ; + break; + case FMODE_WRITE: + res = NFS4_SHARE_ACCESS_WRITE; + break; + case FMODE_READ|FMODE_WRITE: + res = NFS4_SHARE_ACCESS_BOTH; + } + return res; +} + +static u32 +nfs4_map_atomic_open_share(struct nfs_server *server, + fmode_t fmode, int openflags) +{ + u32 res = nfs4_fmode_to_share_access(fmode); + + if (!(server->caps & NFS_CAP_ATOMIC_OPEN_V1)) + goto out; + /* Want no delegation if we're using O_DIRECT */ + if (openflags & O_DIRECT) { + res |= NFS4_SHARE_WANT_NO_DELEG; + goto out; + } + /* res |= NFS4_SHARE_WANT_NO_PREFERENCE; */ + if (server->caps & NFS_CAP_DELEGTIME) + res |= NFS4_SHARE_WANT_DELEG_TIMESTAMPS; + if (server->caps & NFS_CAP_OPEN_XOR) + res |= NFS4_SHARE_WANT_OPEN_XOR_DELEGATION; +out: + return res; +} + +static enum open_claim_type4 +nfs4_map_atomic_open_claim(struct nfs_server *server, + enum open_claim_type4 claim) +{ + if (server->caps & NFS_CAP_ATOMIC_OPEN_V1) + return claim; + switch (claim) { + default: + return claim; + case NFS4_OPEN_CLAIM_FH: + return NFS4_OPEN_CLAIM_NULL; + case NFS4_OPEN_CLAIM_DELEG_CUR_FH: + return NFS4_OPEN_CLAIM_DELEGATE_CUR; + case NFS4_OPEN_CLAIM_DELEG_PREV_FH: + return NFS4_OPEN_CLAIM_DELEGATE_PREV; + } +} + +static void nfs4_init_opendata_res(struct nfs4_opendata *p) +{ + p->o_res.f_attr = &p->f_attr; + p->o_res.seqid = p->o_arg.seqid; + p->c_res.seqid = p->c_arg.seqid; + p->o_res.server = p->o_arg.server; + p->o_res.access_request = p->o_arg.access; + nfs_fattr_init(&p->f_attr); + nfs_fattr_init_names(&p->f_attr, &p->owner_name, &p->group_name); +} + +static struct nfs4_opendata *nfs4_opendata_alloc(struct dentry *dentry, + struct nfs4_state_owner *sp, fmode_t fmode, int flags, + const struct nfs4_open_createattrs *c, + enum open_claim_type4 claim, + gfp_t gfp_mask) +{ + struct dentry *parent = dget_parent(dentry); + struct inode *dir = d_inode(parent); + struct nfs_server *server = NFS_SERVER(dir); + struct nfs_seqid *(*alloc_seqid)(struct nfs_seqid_counter *, gfp_t); + struct nfs4_label *label = (c != NULL) ? c->label : NULL; + struct nfs4_opendata *p; + + p = kzalloc_obj(*p, gfp_mask); + if (p == NULL) + goto err; + + p->f_attr.label = nfs4_label_alloc(server, gfp_mask); + if (IS_ERR(p->f_attr.label)) + goto err_free_p; + + p->a_label = nfs4_label_alloc(server, gfp_mask); + if (IS_ERR(p->a_label)) + goto err_free_f; + + alloc_seqid = server->nfs_client->cl_mvops->alloc_seqid; + p->o_arg.seqid = alloc_seqid(&sp->so_seqid, gfp_mask); + if (IS_ERR(p->o_arg.seqid)) + goto err_free_label; + nfs_sb_active(dentry->d_sb); + p->dentry = dget(dentry); + p->dir = parent; + p->owner = sp; + atomic_inc(&sp->so_count); + p->o_arg.open_flags = flags; + p->o_arg.fmode = fmode & (FMODE_READ|FMODE_WRITE); + p->o_arg.claim = nfs4_map_atomic_open_claim(server, claim); + p->o_arg.share_access = nfs4_map_atomic_open_share(server, + fmode, flags); + if (flags & O_CREAT) { + p->o_arg.umask = current_umask(); + p->o_arg.label = nfs4_label_copy(p->a_label, label); + if (c->sattr != NULL && c->sattr->ia_valid != 0) { + p->o_arg.u.attrs = &p->attrs; + memcpy(&p->attrs, c->sattr, sizeof(p->attrs)); + + memcpy(p->o_arg.u.verifier.data, c->verf, + sizeof(p->o_arg.u.verifier.data)); + } + } + /* ask server to check for all possible rights as results + * are cached */ + switch (p->o_arg.claim) { + default: + break; + case NFS4_OPEN_CLAIM_NULL: + case NFS4_OPEN_CLAIM_FH: + p->o_arg.access = NFS4_ACCESS_READ | NFS4_ACCESS_MODIFY | + NFS4_ACCESS_EXTEND | NFS4_ACCESS_DELETE | + NFS4_ACCESS_EXECUTE | + nfs_access_xattr_mask(server); + } + p->o_arg.clientid = server->nfs_client->cl_clientid; + p->o_arg.id.create_time = ktime_to_ns(sp->so_seqid.create_time); + p->o_arg.id.uniquifier = sp->so_seqid.owner_id; + p->o_arg.name = &dentry->d_name; + p->o_arg.server = server; + p->o_arg.bitmask = nfs4_bitmask(server, label); + p->o_arg.open_bitmap = &nfs4_fattr_bitmap[0]; + switch (p->o_arg.claim) { + case NFS4_OPEN_CLAIM_NULL: + case NFS4_OPEN_CLAIM_DELEGATE_CUR: + case NFS4_OPEN_CLAIM_DELEGATE_PREV: + p->o_arg.fh = NFS_FH(dir); + break; + case NFS4_OPEN_CLAIM_PREVIOUS: + case NFS4_OPEN_CLAIM_FH: + case NFS4_OPEN_CLAIM_DELEG_CUR_FH: + case NFS4_OPEN_CLAIM_DELEG_PREV_FH: + p->o_arg.fh = NFS_FH(d_inode(dentry)); + } + p->c_arg.fh = &p->o_res.fh; + p->c_arg.stateid = &p->o_res.stateid; + p->c_arg.seqid = p->o_arg.seqid; + nfs4_init_opendata_res(p); + kref_init(&p->kref); + return p; + +err_free_label: + nfs4_label_free(p->a_label); +err_free_f: + nfs4_label_free(p->f_attr.label); +err_free_p: + kfree(p); +err: + dput(parent); + return NULL; +} + +static void nfs4_opendata_free(struct kref *kref) +{ + struct nfs4_opendata *p = container_of(kref, + struct nfs4_opendata, kref); + struct super_block *sb = p->dentry->d_sb; + + nfs4_lgopen_release(p->lgp); + nfs_free_seqid(p->o_arg.seqid); + nfs4_sequence_free_slot(&p->o_res.seq_res); + if (p->state != NULL) + nfs4_put_open_state(p->state); + nfs4_put_state_owner(p->owner); + + nfs4_label_free(p->a_label); + nfs4_label_free(p->f_attr.label); + + dput(p->dir); + dput(p->dentry); + nfs_sb_deactive(sb); + nfs_fattr_free_names(&p->f_attr); + kfree(p->f_attr.mdsthreshold); + kfree(p); +} + +static void nfs4_opendata_put(struct nfs4_opendata *p) +{ + if (p != NULL) + kref_put(&p->kref, nfs4_opendata_free); +} + +static bool nfs4_mode_match_open_stateid(struct nfs4_state *state, + fmode_t fmode) +{ + switch(fmode & (FMODE_READ|FMODE_WRITE)) { + case FMODE_READ|FMODE_WRITE: + return state->n_rdwr != 0; + case FMODE_WRITE: + return state->n_wronly != 0; + case FMODE_READ: + return state->n_rdonly != 0; + } + WARN_ON_ONCE(1); + return false; +} + +static int can_open_cached(struct nfs4_state *state, fmode_t mode, + int open_mode, enum open_claim_type4 claim) +{ + int ret = 0; + + if (open_mode & (O_EXCL|O_TRUNC)) + goto out; + switch (claim) { + case NFS4_OPEN_CLAIM_NULL: + case NFS4_OPEN_CLAIM_FH: + goto out; + default: + break; + } + switch (mode & (FMODE_READ|FMODE_WRITE)) { + case FMODE_READ: + ret |= test_bit(NFS_O_RDONLY_STATE, &state->flags) != 0 + && state->n_rdonly != 0; + break; + case FMODE_WRITE: + ret |= test_bit(NFS_O_WRONLY_STATE, &state->flags) != 0 + && state->n_wronly != 0; + break; + case FMODE_READ|FMODE_WRITE: + ret |= test_bit(NFS_O_RDWR_STATE, &state->flags) != 0 + && state->n_rdwr != 0; + } +out: + return ret; +} + +static bool can_open_delegated(const struct inode *inode, fmode_t fmode, + enum open_claim_type4 claim, nfs4_stateid *stateid) +{ + struct nfs_delegation *delegation; + bool ret = false; + + delegation = nfs4_get_valid_delegation(inode); + if (!delegation) + return false; + if ((delegation->type & fmode) != fmode) + goto out_put_delegation; + + switch (claim) { + case NFS4_OPEN_CLAIM_PREVIOUS: + if (test_bit(NFS_DELEGATION_NEED_RECLAIM, &delegation->flags)) + break; + fallthrough; + case NFS4_OPEN_CLAIM_NULL: + case NFS4_OPEN_CLAIM_FH: + nfs_mark_delegation_referenced(delegation); + /* Save the delegation stateid */ + if (stateid) + nfs4_stateid_copy(stateid, &delegation->stateid); + ret = true; + break; + default: + break; + } + +out_put_delegation: + nfs_put_delegation(delegation); + return ret; +} + +static void update_open_stateflags(struct nfs4_state *state, fmode_t fmode) +{ + switch (fmode) { + case FMODE_WRITE: + state->n_wronly++; + break; + case FMODE_READ: + state->n_rdonly++; + break; + case FMODE_READ|FMODE_WRITE: + state->n_rdwr++; + } + nfs4_state_set_mode_locked(state, state->state | fmode); +} + +static bool nfs_open_stateid_recover_openmode(struct nfs4_state *state) +{ + if (state->n_rdonly && !test_bit(NFS_O_RDONLY_STATE, &state->flags)) + return true; + if (state->n_wronly && !test_bit(NFS_O_WRONLY_STATE, &state->flags)) + return true; + if (state->n_rdwr && !test_bit(NFS_O_RDWR_STATE, &state->flags)) + return true; + return false; +} + +static void nfs_state_log_update_open_stateid(struct nfs4_state *state) +{ + if (test_and_clear_bit(NFS_STATE_CHANGE_WAIT, &state->flags)) + wake_up_all(&state->waitq); +} + +static void nfs_test_and_clear_all_open_stateid(struct nfs4_state *state) +{ + struct nfs_client *clp = state->owner->so_server->nfs_client; + bool need_recover = false; + + if (test_and_clear_bit(NFS_O_RDONLY_STATE, &state->flags) && state->n_rdonly) + need_recover = true; + if (test_and_clear_bit(NFS_O_WRONLY_STATE, &state->flags) && state->n_wronly) + need_recover = true; + if (test_and_clear_bit(NFS_O_RDWR_STATE, &state->flags) && state->n_rdwr) + need_recover = true; + if (need_recover) + nfs4_state_mark_reclaim_nograce(clp, state); +} + +/* + * Check for whether or not the caller may update the open stateid + * to the value passed in by stateid. + * + * Note: This function relies heavily on the server implementing + * RFC7530 Section 9.1.4.2, and RFC5661 Section 8.2.2 + * correctly. + * i.e. The stateid seqids have to be initialised to 1, and + * are then incremented on every state transition. + */ +static bool nfs_stateid_is_sequential(struct nfs4_state *state, + const nfs4_stateid *stateid) +{ + if (test_bit(NFS_OPEN_STATE, &state->flags)) { + /* The common case - we're updating to a new sequence number */ + if (nfs4_stateid_match_other(stateid, &state->open_stateid)) { + if (nfs4_stateid_is_next(&state->open_stateid, stateid)) + return true; + return false; + } + /* The server returned a new stateid */ + } + /* This is the first OPEN in this generation */ + if (stateid->seqid == cpu_to_be32(1)) + return true; + return false; +} + +static void nfs_resync_open_stateid_locked(struct nfs4_state *state) +{ + if (!(state->n_wronly || state->n_rdonly || state->n_rdwr)) + return; + if (state->n_wronly) + set_bit(NFS_O_WRONLY_STATE, &state->flags); + if (state->n_rdonly) + set_bit(NFS_O_RDONLY_STATE, &state->flags); + if (state->n_rdwr) + set_bit(NFS_O_RDWR_STATE, &state->flags); + set_bit(NFS_OPEN_STATE, &state->flags); +} + +static void nfs_clear_open_stateid_locked(struct nfs4_state *state, + nfs4_stateid *stateid, fmode_t fmode) +{ + clear_bit(NFS_O_RDWR_STATE, &state->flags); + switch (fmode & (FMODE_READ|FMODE_WRITE)) { + case FMODE_WRITE: + clear_bit(NFS_O_RDONLY_STATE, &state->flags); + break; + case FMODE_READ: + clear_bit(NFS_O_WRONLY_STATE, &state->flags); + break; + case 0: + clear_bit(NFS_O_RDONLY_STATE, &state->flags); + clear_bit(NFS_O_WRONLY_STATE, &state->flags); + clear_bit(NFS_OPEN_STATE, &state->flags); + } + if (stateid == NULL) + return; + /* Handle OPEN+OPEN_DOWNGRADE races */ + if (nfs4_stateid_match_other(stateid, &state->open_stateid) && + !nfs4_stateid_is_newer(stateid, &state->open_stateid)) { + nfs_resync_open_stateid_locked(state); + goto out; + } + if (test_bit(NFS_DELEGATED_STATE, &state->flags) == 0) + nfs4_stateid_copy(&state->stateid, stateid); + nfs4_stateid_copy(&state->open_stateid, stateid); + trace_nfs4_open_stateid_update(state->inode, stateid, 0); +out: + nfs_state_log_update_open_stateid(state); +} + +static void nfs_clear_open_stateid(struct nfs4_state *state, + nfs4_stateid *arg_stateid, + nfs4_stateid *stateid, fmode_t fmode) +{ + write_seqlock(&state->seqlock); + /* Ignore, if the CLOSE argment doesn't match the current stateid */ + if (nfs4_state_match_open_stateid_other(state, arg_stateid)) + nfs_clear_open_stateid_locked(state, stateid, fmode); + write_sequnlock(&state->seqlock); + if (test_bit(NFS_STATE_RECLAIM_NOGRACE, &state->flags)) + nfs4_schedule_state_manager(state->owner->so_server->nfs_client); +} + +static void nfs_set_open_stateid_locked(struct nfs4_state *state, + const nfs4_stateid *stateid, nfs4_stateid *freeme) + __must_hold(&state->owner->so_lock) + __must_hold(&state->seqlock) + __must_hold(RCU) + +{ + DEFINE_WAIT(wait); + int status = 0; + for (;;) { + + if (nfs_stateid_is_sequential(state, stateid)) + break; + + if (status) { + if (nfs4_stateid_match_other(stateid, &state->open_stateid) && + !nfs4_stateid_is_newer(stateid, &state->open_stateid)) { + trace_nfs4_open_stateid_update_skip(state->inode, + stateid, status); + return; + } else { + break; + } + } + + /* Rely on seqids for serialisation with NFSv4.0 */ + if (!nfs4_has_session(NFS_SERVER(state->inode)->nfs_client)) + break; + + set_bit(NFS_STATE_CHANGE_WAIT, &state->flags); + prepare_to_wait(&state->waitq, &wait, TASK_KILLABLE); + /* + * Ensure we process the state changes in the same order + * in which the server processed them by delaying the + * update of the stateid until we are in sequence. + */ + write_sequnlock(&state->seqlock); + spin_unlock(&state->owner->so_lock); + rcu_read_unlock(); + trace_nfs4_open_stateid_update_wait(state->inode, stateid, 0); + + if (!fatal_signal_pending(current) && + !nfs_current_task_exiting()) { + if (schedule_timeout(5*HZ) == 0) + status = -EAGAIN; + else + status = 0; + } else + status = -EINTR; + finish_wait(&state->waitq, &wait); + rcu_read_lock(); + spin_lock(&state->owner->so_lock); + write_seqlock(&state->seqlock); + } + + if (test_bit(NFS_OPEN_STATE, &state->flags) && + !nfs4_stateid_match_other(stateid, &state->open_stateid)) { + nfs4_stateid_copy(freeme, &state->open_stateid); + nfs_test_and_clear_all_open_stateid(state); + } + + if (test_bit(NFS_DELEGATED_STATE, &state->flags) == 0) + nfs4_stateid_copy(&state->stateid, stateid); + nfs4_stateid_copy(&state->open_stateid, stateid); + trace_nfs4_open_stateid_update(state->inode, stateid, status); + nfs_state_log_update_open_stateid(state); +} + +static void nfs_state_set_open_stateid(struct nfs4_state *state, + const nfs4_stateid *open_stateid, + fmode_t fmode, + nfs4_stateid *freeme) +{ + /* + * Protect the call to nfs4_state_set_mode_locked and + * serialise the stateid update + */ + write_seqlock(&state->seqlock); + nfs_set_open_stateid_locked(state, open_stateid, freeme); + switch (fmode) { + case FMODE_READ: + set_bit(NFS_O_RDONLY_STATE, &state->flags); + break; + case FMODE_WRITE: + set_bit(NFS_O_WRONLY_STATE, &state->flags); + break; + case FMODE_READ|FMODE_WRITE: + set_bit(NFS_O_RDWR_STATE, &state->flags); + } + set_bit(NFS_OPEN_STATE, &state->flags); + write_sequnlock(&state->seqlock); +} + +void nfs_state_clear_open_state_flags(struct nfs4_state *state) +{ + clear_bit(NFS_O_RDWR_STATE, &state->flags); + clear_bit(NFS_O_WRONLY_STATE, &state->flags); + clear_bit(NFS_O_RDONLY_STATE, &state->flags); + clear_bit(NFS_OPEN_STATE, &state->flags); +} + +static void nfs_state_set_delegation(struct nfs4_state *state, + const nfs4_stateid *deleg_stateid, + fmode_t fmode) +{ + /* + * Protect the call to nfs4_state_set_mode_locked and + * serialise the stateid update + */ + write_seqlock(&state->seqlock); + nfs4_stateid_copy(&state->stateid, deleg_stateid); + set_bit(NFS_DELEGATED_STATE, &state->flags); + write_sequnlock(&state->seqlock); +} + +void nfs_state_clear_delegation(struct nfs4_state *state) +{ + write_seqlock(&state->seqlock); + nfs4_stateid_copy(&state->stateid, &state->open_stateid); + clear_bit(NFS_DELEGATED_STATE, &state->flags); + write_sequnlock(&state->seqlock); +} + +int update_open_stateid(struct nfs4_state *state, + const nfs4_stateid *open_stateid, + const nfs4_stateid *delegation, + fmode_t fmode) +{ + struct nfs_server *server = NFS_SERVER(state->inode); + struct nfs_client *clp = server->nfs_client; + struct nfs_delegation *deleg_cur; + nfs4_stateid freeme = { }; + int ret = 0; + + fmode &= (FMODE_READ|FMODE_WRITE); + + spin_lock(&state->owner->so_lock); + if (open_stateid != NULL) { + rcu_read_lock(); + nfs_state_set_open_stateid(state, open_stateid, fmode, &freeme); + rcu_read_unlock(); + ret = 1; + } + + deleg_cur = nfs4_get_valid_delegation(state->inode); + if (deleg_cur == NULL) + goto no_delegation; + + spin_lock(&deleg_cur->lock); + if (!deleg_cur->inode || + test_bit(NFS_DELEGATION_RETURNING, &deleg_cur->flags) || + (deleg_cur->type & fmode) != fmode) + goto no_delegation_unlock; + + if (delegation == NULL) + delegation = &deleg_cur->stateid; + else if (!nfs4_stateid_match_other(&deleg_cur->stateid, delegation)) + goto no_delegation_unlock; + + nfs_mark_delegation_referenced(deleg_cur); + nfs_state_set_delegation(state, &deleg_cur->stateid, fmode); + ret = 1; +no_delegation_unlock: + spin_unlock(&deleg_cur->lock); + nfs_put_delegation(deleg_cur); +no_delegation: + if (ret) + update_open_stateflags(state, fmode); + spin_unlock(&state->owner->so_lock); + + if (test_bit(NFS_STATE_RECLAIM_NOGRACE, &state->flags)) + nfs4_schedule_state_manager(clp); + if (freeme.type != 0) + nfs4_test_and_free_stateid(server, &freeme, + state->owner->so_cred); + + return ret; +} + +static bool nfs4_update_lock_stateid(struct nfs4_lock_state *lsp, + const nfs4_stateid *stateid) +{ + struct nfs4_state *state = lsp->ls_state; + bool ret = false; + + spin_lock(&state->state_lock); + if (!nfs4_stateid_match_other(stateid, &lsp->ls_stateid)) + goto out_noupdate; + if (!nfs4_stateid_is_newer(stateid, &lsp->ls_stateid)) + goto out_noupdate; + nfs4_stateid_copy(&lsp->ls_stateid, stateid); + ret = true; +out_noupdate: + spin_unlock(&state->state_lock); + return ret; +} + +static void nfs4_return_incompatible_delegation(struct inode *inode, fmode_t fmode) +{ + struct nfs_delegation *delegation; + + fmode &= FMODE_READ|FMODE_WRITE; + delegation = nfs4_get_valid_delegation(inode); + if (!delegation) + return; + if ((delegation->type & fmode) != fmode) + nfs4_inode_return_delegation(inode); + nfs_put_delegation(delegation); +} + +static struct nfs4_state *nfs4_try_open_cached(struct nfs4_opendata *opendata) +{ + struct nfs4_state *state = opendata->state; + int open_mode = opendata->o_arg.open_flags; + fmode_t fmode = opendata->o_arg.fmode; + enum open_claim_type4 claim = opendata->o_arg.claim; + nfs4_stateid stateid; + int ret = -EAGAIN; + + for (;;) { + spin_lock(&state->owner->so_lock); + if (can_open_cached(state, fmode, open_mode, claim)) { + update_open_stateflags(state, fmode); + spin_unlock(&state->owner->so_lock); + goto out_return_state; + } + spin_unlock(&state->owner->so_lock); + + if (!can_open_delegated(state->inode, fmode, claim, &stateid)) + break; + + nfs_release_seqid(opendata->o_arg.seqid); + if (!opendata->is_recover) { + ret = nfs_may_open(state->inode, state->owner->so_cred, open_mode); + if (ret != 0) + goto out; + } + ret = -EAGAIN; + + /* Try to update the stateid using the delegation */ + if (update_open_stateid(state, NULL, &stateid, fmode)) + goto out_return_state; + } +out: + return ERR_PTR(ret); +out_return_state: + refcount_inc(&state->count); + return state; +} + +static void +nfs4_process_delegation(struct inode *inode, const struct cred *cred, + enum open_claim_type4 claim, + const struct nfs4_open_delegation *delegation) +{ + switch (delegation->open_delegation_type) { + case NFS4_OPEN_DELEGATE_READ: + case NFS4_OPEN_DELEGATE_WRITE: + case NFS4_OPEN_DELEGATE_READ_ATTRS_DELEG: + case NFS4_OPEN_DELEGATE_WRITE_ATTRS_DELEG: + break; + default: + return; + } + switch (claim) { + case NFS4_OPEN_CLAIM_DELEGATE_CUR: + case NFS4_OPEN_CLAIM_DELEG_CUR_FH: + pr_err_ratelimited("NFS: Broken NFSv4 server %s is " + "returning a delegation for " + "OPEN(CLAIM_DELEGATE_CUR)\n", + NFS_SERVER(inode)->nfs_client->cl_hostname); + break; + case NFS4_OPEN_CLAIM_PREVIOUS: + nfs_inode_reclaim_delegation(inode, cred, delegation->type, + &delegation->stateid, + delegation->pagemod_limit, + delegation->open_delegation_type); + break; + default: + nfs_inode_set_delegation(inode, cred, delegation->type, + &delegation->stateid, + delegation->pagemod_limit, + delegation->open_delegation_type); + } + if (delegation->do_recall) + nfs_async_inode_return_delegation(inode, &delegation->stateid); +} + +/* + * Check the inode attributes against the CLAIM_PREVIOUS returned attributes + * and update the nfs4_state. + */ +static struct nfs4_state * +_nfs4_opendata_reclaim_to_nfs4_state(struct nfs4_opendata *data) +{ + struct inode *inode = data->state->inode; + struct nfs4_state *state = data->state; + int ret; + + if (!data->rpc_done) { + if (data->rpc_status) + return ERR_PTR(data->rpc_status); + return nfs4_try_open_cached(data); + } + + ret = nfs_refresh_inode(inode, &data->f_attr); + if (ret) + return ERR_PTR(ret); + + nfs4_process_delegation(state->inode, + data->owner->so_cred, + data->o_arg.claim, + &data->o_res.delegation); + + if (!(data->o_res.rflags & NFS4_OPEN_RESULT_NO_OPEN_STATEID)) { + if (!update_open_stateid(state, &data->o_res.stateid, + NULL, data->o_arg.fmode)) + return ERR_PTR(-EAGAIN); + } else if (!update_open_stateid(state, NULL, NULL, data->o_arg.fmode)) + return ERR_PTR(-EAGAIN); + refcount_inc(&state->count); + + return state; +} + +static struct inode * +nfs4_opendata_get_inode(struct nfs4_opendata *data) +{ + struct inode *inode; + + switch (data->o_arg.claim) { + case NFS4_OPEN_CLAIM_NULL: + case NFS4_OPEN_CLAIM_DELEGATE_CUR: + case NFS4_OPEN_CLAIM_DELEGATE_PREV: + if (!(data->f_attr.valid & NFS_ATTR_FATTR)) + return ERR_PTR(-EAGAIN); + inode = nfs_fhget(data->dir->d_sb, &data->o_res.fh, + &data->f_attr); + break; + default: + inode = d_inode(data->dentry); + ihold(inode); + nfs_refresh_inode(inode, &data->f_attr); + } + return inode; +} + +static struct nfs4_state * +nfs4_opendata_find_nfs4_state(struct nfs4_opendata *data) +{ + struct nfs4_state *state; + struct inode *inode; + + inode = nfs4_opendata_get_inode(data); + if (IS_ERR(inode)) + return ERR_CAST(inode); + if (data->state != NULL && data->state->inode == inode) { + state = data->state; + refcount_inc(&state->count); + } else + state = nfs4_get_open_state(inode, data->owner); + iput(inode); + if (state == NULL) + state = ERR_PTR(-ENOMEM); + return state; +} + +static struct nfs4_state * +_nfs4_opendata_to_nfs4_state(struct nfs4_opendata *data) +{ + struct nfs4_state *state; + + if (!data->rpc_done) { + state = nfs4_try_open_cached(data); + trace_nfs4_cached_open(data->state); + goto out; + } + + state = nfs4_opendata_find_nfs4_state(data); + if (IS_ERR(state)) + goto out; + + nfs4_process_delegation(state->inode, + data->owner->so_cred, + data->o_arg.claim, + &data->o_res.delegation); + + if (!(data->o_res.rflags & NFS4_OPEN_RESULT_NO_OPEN_STATEID)) { + if (!update_open_stateid(state, &data->o_res.stateid, + NULL, data->o_arg.fmode)) { + nfs4_put_open_state(state); + state = ERR_PTR(-EAGAIN); + } + } else if (!update_open_stateid(state, NULL, NULL, data->o_arg.fmode)) { + nfs4_put_open_state(state); + state = ERR_PTR(-EAGAIN); + } +out: + nfs_release_seqid(data->o_arg.seqid); + return state; +} + +static struct nfs4_state * +nfs4_opendata_to_nfs4_state(struct nfs4_opendata *data) +{ + struct nfs4_state *ret; + + if (data->o_arg.claim == NFS4_OPEN_CLAIM_PREVIOUS) + ret =_nfs4_opendata_reclaim_to_nfs4_state(data); + else + ret = _nfs4_opendata_to_nfs4_state(data); + nfs4_sequence_free_slot(&data->o_res.seq_res); + return ret; +} + +static struct nfs_open_context * +nfs4_state_find_open_context_mode(struct nfs4_state *state, fmode_t mode) +{ + struct nfs_inode *nfsi = NFS_I(state->inode); + struct nfs_open_context *ctx; + + rcu_read_lock(); + list_for_each_entry_rcu(ctx, &nfsi->open_files, list) { + if (ctx->state != state) + continue; + if ((ctx->mode & mode) != mode) + continue; + if (!get_nfs_open_context(ctx)) + continue; + rcu_read_unlock(); + return ctx; + } + rcu_read_unlock(); + return ERR_PTR(-ENOENT); +} + +static struct nfs_open_context * +nfs4_state_find_open_context(struct nfs4_state *state) +{ + struct nfs_open_context *ctx; + + ctx = nfs4_state_find_open_context_mode(state, FMODE_READ|FMODE_WRITE); + if (!IS_ERR(ctx)) + return ctx; + ctx = nfs4_state_find_open_context_mode(state, FMODE_WRITE); + if (!IS_ERR(ctx)) + return ctx; + return nfs4_state_find_open_context_mode(state, FMODE_READ); +} + +static struct nfs4_opendata *nfs4_open_recoverdata_alloc(struct nfs_open_context *ctx, + struct nfs4_state *state, enum open_claim_type4 claim) +{ + struct nfs4_opendata *opendata; + + opendata = nfs4_opendata_alloc(ctx->dentry, state->owner, 0, 0, + NULL, claim, GFP_NOFS); + if (opendata == NULL) + return ERR_PTR(-ENOMEM); + opendata->state = state; + refcount_inc(&state->count); + return opendata; +} + +static int nfs4_open_recover_helper(struct nfs4_opendata *opendata, + fmode_t fmode) +{ + struct nfs4_state *newstate; + struct nfs_server *server = NFS_SB(opendata->dentry->d_sb); + int openflags = opendata->o_arg.open_flags; + int ret; + + if (!nfs4_mode_match_open_stateid(opendata->state, fmode)) + return 0; + opendata->o_arg.fmode = fmode; + opendata->o_arg.share_access = + nfs4_map_atomic_open_share(server, fmode, openflags); + memset(&opendata->o_res, 0, sizeof(opendata->o_res)); + memset(&opendata->c_res, 0, sizeof(opendata->c_res)); + nfs4_init_opendata_res(opendata); + ret = _nfs4_recover_proc_open(opendata); + if (ret != 0) + return ret; + newstate = nfs4_opendata_to_nfs4_state(opendata); + if (IS_ERR(newstate)) + return PTR_ERR(newstate); + if (newstate != opendata->state) + ret = -ESTALE; + nfs4_close_state(newstate, fmode); + return ret; +} + +static int nfs4_open_recover(struct nfs4_opendata *opendata, struct nfs4_state *state) +{ + int ret; + + /* memory barrier prior to reading state->n_* */ + smp_rmb(); + ret = nfs4_open_recover_helper(opendata, FMODE_READ|FMODE_WRITE); + if (ret != 0) + return ret; + ret = nfs4_open_recover_helper(opendata, FMODE_WRITE); + if (ret != 0) + return ret; + ret = nfs4_open_recover_helper(opendata, FMODE_READ); + if (ret != 0) + return ret; + /* + * We may have performed cached opens for all three recoveries. + * Check if we need to update the current stateid. + */ + if (test_bit(NFS_DELEGATED_STATE, &state->flags) == 0 && + !nfs4_stateid_match(&state->stateid, &state->open_stateid)) { + write_seqlock(&state->seqlock); + if (test_bit(NFS_DELEGATED_STATE, &state->flags) == 0) + nfs4_stateid_copy(&state->stateid, &state->open_stateid); + write_sequnlock(&state->seqlock); + } + return 0; +} + +/* + * OPEN_RECLAIM: + * reclaim state on the server after a reboot. + */ +static int _nfs4_do_open_reclaim(struct nfs_open_context *ctx, struct nfs4_state *state) +{ + struct nfs_delegation *delegation; + struct nfs4_opendata *opendata; + u32 delegation_type = NFS4_OPEN_DELEGATE_NONE; + int status; + + opendata = nfs4_open_recoverdata_alloc(ctx, state, + NFS4_OPEN_CLAIM_PREVIOUS); + if (IS_ERR(opendata)) + return PTR_ERR(opendata); + rcu_read_lock(); + delegation = rcu_dereference(NFS_I(state->inode)->delegation); + if (delegation != NULL && test_bit(NFS_DELEGATION_NEED_RECLAIM, &delegation->flags) != 0) { + switch(delegation->type) { + case FMODE_READ: + delegation_type = NFS4_OPEN_DELEGATE_READ; + if (test_bit(NFS_DELEGATION_DELEGTIME, &delegation->flags)) + delegation_type = NFS4_OPEN_DELEGATE_READ_ATTRS_DELEG; + break; + case FMODE_WRITE: + case FMODE_READ|FMODE_WRITE: + delegation_type = NFS4_OPEN_DELEGATE_WRITE; + if (test_bit(NFS_DELEGATION_DELEGTIME, &delegation->flags)) + delegation_type = NFS4_OPEN_DELEGATE_WRITE_ATTRS_DELEG; + } + } + rcu_read_unlock(); + opendata->o_arg.u.delegation_type = delegation_type; + status = nfs4_open_recover(opendata, state); + nfs4_opendata_put(opendata); + return status; +} + +static int nfs4_do_open_reclaim(struct nfs_open_context *ctx, struct nfs4_state *state) +{ + struct nfs_server *server = NFS_SERVER(state->inode); + struct nfs4_exception exception = { }; + int err; + do { + err = _nfs4_do_open_reclaim(ctx, state); + trace_nfs4_open_reclaim(ctx, 0, err); + if (nfs4_clear_cap_atomic_open_v1(server, err, &exception)) + continue; + if (err != -NFS4ERR_DELAY) + break; + nfs4_handle_exception(server, err, &exception); + } while (exception.retry); + return err; +} + +int nfs4_open_reclaim(struct nfs4_state_owner *sp, struct nfs4_state *state) +{ + struct nfs_open_context *ctx; + int ret; + + ctx = nfs4_state_find_open_context(state); + if (IS_ERR(ctx)) + return -EAGAIN; + clear_bit(NFS_DELEGATED_STATE, &state->flags); + nfs_state_clear_open_state_flags(state); + ret = nfs4_do_open_reclaim(ctx, state); + put_nfs_open_context(ctx); + return ret; +} + +static int nfs4_handle_delegation_recall_error(struct nfs_server *server, struct nfs4_state *state, const nfs4_stateid *stateid, struct file_lock *fl, int err) +{ + switch (err) { + default: + printk(KERN_ERR "NFS: %s: unhandled error " + "%d.\n", __func__, err); + fallthrough; + case 0: + case -ENOENT: + case -EAGAIN: + case -ESTALE: + case -ETIMEDOUT: + break; + case -NFS4ERR_BADSESSION: + case -NFS4ERR_BADSLOT: + case -NFS4ERR_BAD_HIGH_SLOT: + case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION: + case -NFS4ERR_DEADSESSION: + return -EAGAIN; + case -NFS4ERR_STALE_CLIENTID: + case -NFS4ERR_STALE_STATEID: + /* Don't recall a delegation if it was lost */ + nfs4_schedule_lease_recovery(server->nfs_client); + return -EAGAIN; + case -NFS4ERR_MOVED: + nfs4_schedule_migration_recovery(server); + return -EAGAIN; + case -NFS4ERR_LEASE_MOVED: + nfs4_schedule_lease_moved_recovery(server->nfs_client); + return -EAGAIN; + case -NFS4ERR_DELEG_REVOKED: + case -NFS4ERR_ADMIN_REVOKED: + case -NFS4ERR_EXPIRED: + case -NFS4ERR_BAD_STATEID: + case -NFS4ERR_OPENMODE: + nfs_inode_find_state_and_recover(state->inode, + stateid); + nfs4_schedule_stateid_recovery(server, state); + return -EAGAIN; + case -NFS4ERR_DELAY: + case -NFS4ERR_GRACE: + ssleep(1); + return -EAGAIN; + case -ENOMEM: + case -NFS4ERR_DENIED: + if (fl) { + struct nfs4_lock_state *lsp = fl->fl_u.nfs4_fl.owner; + if (lsp) + set_bit(NFS_LOCK_LOST, &lsp->ls_flags); + } + return 0; + } + return err; +} + +int nfs4_open_delegation_recall(struct nfs_open_context *ctx, + struct nfs4_state *state, const nfs4_stateid *stateid) +{ + struct nfs_server *server = NFS_SERVER(state->inode); + struct nfs4_opendata *opendata; + int err = 0; + + opendata = nfs4_open_recoverdata_alloc(ctx, state, + NFS4_OPEN_CLAIM_DELEG_CUR_FH); + if (IS_ERR(opendata)) + return PTR_ERR(opendata); + nfs4_stateid_copy(&opendata->o_arg.u.delegation, stateid); + if (!test_bit(NFS_O_RDWR_STATE, &state->flags)) { + err = nfs4_open_recover_helper(opendata, FMODE_READ|FMODE_WRITE); + if (err) + goto out; + } + if (!test_bit(NFS_O_WRONLY_STATE, &state->flags)) { + err = nfs4_open_recover_helper(opendata, FMODE_WRITE); + if (err) + goto out; + } + if (!test_bit(NFS_O_RDONLY_STATE, &state->flags)) { + err = nfs4_open_recover_helper(opendata, FMODE_READ); + if (err) + goto out; + } + nfs_state_clear_delegation(state); +out: + nfs4_opendata_put(opendata); + return nfs4_handle_delegation_recall_error(server, state, stateid, NULL, err); +} + +static void nfs4_open_confirm_prepare(struct rpc_task *task, void *calldata) +{ + struct nfs4_opendata *data = calldata; + + nfs4_setup_sequence(data->o_arg.server->nfs_client, + &data->c_arg.seq_args, &data->c_res.seq_res, task); +} + +static void nfs4_open_confirm_done(struct rpc_task *task, void *calldata) +{ + struct nfs4_opendata *data = calldata; + + data->c_res.seq_res.sr_slot_ops->done(task, &data->c_res.seq_res); + + data->rpc_status = task->tk_status; + if (data->rpc_status == 0) { + nfs4_stateid_copy(&data->o_res.stateid, &data->c_res.stateid); + nfs_confirm_seqid(&data->owner->so_seqid, 0); + renew_lease(data->o_res.server, data->timestamp); + data->rpc_done = true; + } +} + +static void nfs4_open_confirm_release(void *calldata) +{ + struct nfs4_opendata *data = calldata; + struct nfs4_state *state = NULL; + + /* If this request hasn't been cancelled, do nothing */ + if (!data->cancelled) + goto out_free; + /* In case of error, no cleanup! */ + if (!data->rpc_done) + goto out_free; + state = nfs4_opendata_to_nfs4_state(data); + if (!IS_ERR(state)) + nfs4_close_state(state, data->o_arg.fmode); +out_free: + nfs4_opendata_put(data); +} + +static const struct rpc_call_ops nfs4_open_confirm_ops = { + .rpc_call_prepare = nfs4_open_confirm_prepare, + .rpc_call_done = nfs4_open_confirm_done, + .rpc_release = nfs4_open_confirm_release, +}; + +/* + * Note: On error, nfs4_proc_open_confirm will free the struct nfs4_opendata + */ +static int _nfs4_proc_open_confirm(struct nfs4_opendata *data) +{ + struct nfs_server *server = NFS_SERVER(d_inode(data->dir)); + struct rpc_task *task; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_OPEN_CONFIRM], + .rpc_argp = &data->c_arg, + .rpc_resp = &data->c_res, + .rpc_cred = data->owner->so_cred, + }; + struct rpc_task_setup task_setup_data = { + .rpc_client = server->client, + .rpc_message = &msg, + .callback_ops = &nfs4_open_confirm_ops, + .callback_data = data, + .workqueue = nfsiod_workqueue, + .flags = RPC_TASK_ASYNC | RPC_TASK_CRED_NOREF, + }; + int status; + + nfs4_init_sequence(server->nfs_client, &data->c_arg.seq_args, + &data->c_res.seq_res, 1, data->is_recover); + kref_get(&data->kref); + data->rpc_done = false; + data->rpc_status = 0; + data->timestamp = jiffies; + task = rpc_run_task(&task_setup_data); + if (IS_ERR(task)) + return PTR_ERR(task); + status = rpc_wait_for_completion_task(task); + if (status != 0) { + data->cancelled = true; + smp_wmb(); + } else + status = data->rpc_status; + rpc_put_task(task); + return status; +} + +static void nfs4_open_prepare(struct rpc_task *task, void *calldata) +{ + struct nfs4_opendata *data = calldata; + struct nfs4_state_owner *sp = data->owner; + struct nfs_client *clp = sp->so_server->nfs_client; + enum open_claim_type4 claim = data->o_arg.claim; + + if (nfs_wait_on_sequence(data->o_arg.seqid, task) != 0) + goto out_wait; + /* + * Check if we still need to send an OPEN call, or if we can use + * a delegation instead. + */ + if (data->state != NULL) { + if (can_open_cached(data->state, data->o_arg.fmode, + data->o_arg.open_flags, claim)) + goto out_no_action; + if (can_open_delegated(data->state->inode, data->o_arg.fmode, + claim, NULL)) { + trace_nfs4_cached_open(data->state); + goto out_no_action; + } + } + /* Update client id. */ + data->o_arg.clientid = clp->cl_clientid; + switch (claim) { + default: + break; + case NFS4_OPEN_CLAIM_PREVIOUS: + case NFS4_OPEN_CLAIM_DELEG_CUR_FH: + case NFS4_OPEN_CLAIM_DELEG_PREV_FH: + data->o_arg.open_bitmap = &nfs4_open_noattr_bitmap[0]; + fallthrough; + case NFS4_OPEN_CLAIM_FH: + task->tk_msg.rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_OPEN_NOATTR]; + } + data->timestamp = jiffies; + if (nfs4_setup_sequence(data->o_arg.server->nfs_client, + &data->o_arg.seq_args, + &data->o_res.seq_res, + task) != 0) + nfs_release_seqid(data->o_arg.seqid); + + /* Set the create mode (note dependency on the session type) */ + data->o_arg.createmode = NFS4_CREATE_UNCHECKED; + if (data->o_arg.open_flags & O_EXCL) { + data->o_arg.createmode = NFS4_CREATE_EXCLUSIVE4_1; + if (clp->cl_mvops->minor_version == 0) { + data->o_arg.createmode = NFS4_CREATE_EXCLUSIVE; + /* don't put an ACCESS op in OPEN compound if O_EXCL, + * because ACCESS will return permission denied for + * all bits until close */ + data->o_res.access_request = data->o_arg.access = 0; + } else if (nfs4_has_persistent_session(clp)) + data->o_arg.createmode = NFS4_CREATE_GUARDED; + } + return; + +out_no_action: + task->tk_action = NULL; +out_wait: + nfs4_sequence_done(task, &data->o_res.seq_res); +} + +static void nfs4_open_done(struct rpc_task *task, void *calldata) +{ + struct nfs4_opendata *data = calldata; + + data->rpc_status = task->tk_status; + + if (!nfs4_sequence_process(task, &data->o_res.seq_res)) + return; + + if (task->tk_status == 0) { + if (data->o_res.f_attr->valid & NFS_ATTR_FATTR_TYPE) { + switch (data->o_res.f_attr->mode & S_IFMT) { + case S_IFREG: + break; + case S_IFLNK: + data->rpc_status = -ELOOP; + break; + case S_IFDIR: + data->rpc_status = -EISDIR; + break; + default: + data->rpc_status = -ENOTDIR; + } + } + renew_lease(data->o_res.server, data->timestamp); + if (!(data->o_res.rflags & NFS4_OPEN_RESULT_CONFIRM)) + nfs_confirm_seqid(&data->owner->so_seqid, 0); + } + data->rpc_done = true; +} + +static void nfs4_open_release(void *calldata) +{ + struct nfs4_opendata *data = calldata; + struct nfs4_state *state = NULL; + + /* In case of error, no cleanup! */ + if (data->rpc_status != 0 || !data->rpc_done) { + nfs_release_seqid(data->o_arg.seqid); + goto out_free; + } + /* If this request hasn't been cancelled, do nothing */ + if (!data->cancelled) + goto out_free; + /* In case we need an open_confirm, no cleanup! */ + if (data->o_res.rflags & NFS4_OPEN_RESULT_CONFIRM) + goto out_free; + state = nfs4_opendata_to_nfs4_state(data); + if (!IS_ERR(state)) + nfs4_close_state(state, data->o_arg.fmode); +out_free: + nfs4_opendata_put(data); +} + +static const struct rpc_call_ops nfs4_open_ops = { + .rpc_call_prepare = nfs4_open_prepare, + .rpc_call_done = nfs4_open_done, + .rpc_release = nfs4_open_release, +}; + +static int nfs4_run_open_task(struct nfs4_opendata *data, + struct nfs_open_context *ctx) +{ + struct inode *dir = d_inode(data->dir); + struct nfs_server *server = NFS_SERVER(dir); + struct nfs_client *clp = server->nfs_client; + struct nfs_openargs *o_arg = &data->o_arg; + struct nfs_openres *o_res = &data->o_res; + struct rpc_task *task; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_OPEN], + .rpc_argp = o_arg, + .rpc_resp = o_res, + .rpc_cred = data->owner->so_cred, + }; + struct rpc_task_setup task_setup_data = { + .rpc_client = server->client, + .rpc_message = &msg, + .callback_ops = &nfs4_open_ops, + .callback_data = data, + .workqueue = nfsiod_workqueue, + .flags = RPC_TASK_ASYNC | RPC_TASK_CRED_NOREF, + }; + int status; + + if (nfs_server_capable(dir, NFS_CAP_MOVEABLE)) + task_setup_data.flags |= RPC_TASK_MOVEABLE; + + kref_get(&data->kref); + data->rpc_done = false; + data->rpc_status = 0; + data->cancelled = false; + data->is_recover = false; + if (!ctx) { + nfs4_init_sequence(clp, &o_arg->seq_args, &o_res->seq_res, 1, 1); + data->is_recover = true; + task_setup_data.flags |= RPC_TASK_TIMEOUT; + } else { + nfs4_init_sequence(clp, &o_arg->seq_args, &o_res->seq_res, 1, 0); + pnfs_lgopen_prepare(data, ctx); + } + task = rpc_run_task(&task_setup_data); + if (IS_ERR(task)) + return PTR_ERR(task); + status = rpc_wait_for_completion_task(task); + if (status != 0) { + data->cancelled = true; + smp_wmb(); + } else + status = data->rpc_status; + rpc_put_task(task); + + return status; +} + +static int _nfs4_recover_proc_open(struct nfs4_opendata *data) +{ + struct inode *dir = d_inode(data->dir); + struct nfs_openres *o_res = &data->o_res; + int status; + + status = nfs4_run_open_task(data, NULL); + if (status != 0 || !data->rpc_done) + return status; + + nfs_fattr_map_and_free_names(NFS_SERVER(dir), &data->f_attr); + + if (o_res->rflags & NFS4_OPEN_RESULT_CONFIRM) + status = _nfs4_proc_open_confirm(data); + + return status; +} + +/* + * Additional permission checks in order to distinguish between an + * open for read, and an open for execute. This works around the + * fact that NFSv4 OPEN treats read and execute permissions as being + * the same. + * Note that in the non-execute case, we want to turn off permission + * checking if we just created a new file (POSIX open() semantics). + */ +static int nfs4_opendata_access(const struct cred *cred, + struct nfs4_opendata *opendata, + struct nfs4_state *state, fmode_t fmode) +{ + struct nfs_access_entry cache; + u32 mask, flags; + + /* access call failed or for some reason the server doesn't + * support any access modes -- defer access call until later */ + if (opendata->o_res.access_supported == 0) + return 0; + + mask = 0; + if (fmode & FMODE_EXEC) { + /* ONLY check for exec rights */ + if (S_ISDIR(state->inode->i_mode)) + mask = NFS4_ACCESS_LOOKUP; + else + mask = NFS4_ACCESS_EXECUTE; + } else if ((fmode & FMODE_READ) && !opendata->file_created) + mask = NFS4_ACCESS_READ; + + nfs_access_set_mask(&cache, opendata->o_res.access_result); + nfs_access_add_cache(state->inode, &cache, cred); + + flags = NFS4_ACCESS_READ | NFS4_ACCESS_EXECUTE | NFS4_ACCESS_LOOKUP; + if ((mask & ~cache.mask & flags) == 0) + return 0; + + return -EACCES; +} + +/* + * Note: On error, nfs4_proc_open will free the struct nfs4_opendata + */ +static int _nfs4_proc_open(struct nfs4_opendata *data, + struct nfs_open_context *ctx) +{ + struct inode *dir = d_inode(data->dir); + struct nfs_server *server = NFS_SERVER(dir); + struct nfs_openargs *o_arg = &data->o_arg; + struct nfs_openres *o_res = &data->o_res; + int status; + + status = nfs4_run_open_task(data, ctx); + if (!data->rpc_done) + return status; + if (status != 0) { + if (status == -NFS4ERR_BADNAME && + !(o_arg->open_flags & O_CREAT)) + return -ENOENT; + return status; + } + + nfs_fattr_map_and_free_names(server, &data->f_attr); + + if (o_arg->open_flags & O_CREAT) { + if (o_arg->open_flags & O_EXCL) + data->file_created = true; + else if (o_res->cinfo.before != o_res->cinfo.after) + data->file_created = true; + if (data->file_created || + inode_peek_iversion_raw(dir) != o_res->cinfo.after) + nfs4_update_changeattr(dir, &o_res->cinfo, + o_res->f_attr->time_start, + NFS_INO_INVALID_DATA); + } + if ((o_res->rflags & NFS4_OPEN_RESULT_LOCKTYPE_POSIX) == 0) + server->caps &= ~NFS_CAP_POSIX_LOCK; + if(o_res->rflags & NFS4_OPEN_RESULT_CONFIRM) { + status = _nfs4_proc_open_confirm(data); + if (status != 0) + return status; + } + if (!(o_res->f_attr->valid & NFS_ATTR_FATTR)) { + struct nfs_fh *fh = &o_res->fh; + + nfs4_sequence_free_slot(&o_res->seq_res); + if (o_arg->claim == NFS4_OPEN_CLAIM_FH) + fh = NFS_FH(d_inode(data->dentry)); + nfs4_proc_getattr(server, fh, o_res->f_attr, NULL); + } + return 0; +} + +/* + * OPEN_EXPIRED: + * reclaim state on the server after a network partition. + * Assumes caller holds the appropriate lock + */ +static int _nfs4_open_expired(struct nfs_open_context *ctx, struct nfs4_state *state) +{ + struct nfs4_opendata *opendata; + int ret; + + opendata = nfs4_open_recoverdata_alloc(ctx, state, NFS4_OPEN_CLAIM_FH); + if (IS_ERR(opendata)) + return PTR_ERR(opendata); + /* + * We're not recovering a delegation, so ask for no delegation. + * Otherwise the recovery thread could deadlock with an outstanding + * delegation return. + */ + opendata->o_arg.open_flags = O_DIRECT; + ret = nfs4_open_recover(opendata, state); + if (ret == -ESTALE) + d_drop(ctx->dentry); + nfs4_opendata_put(opendata); + return ret; +} + +static int nfs4_do_open_expired(struct nfs_open_context *ctx, struct nfs4_state *state) +{ + struct nfs_server *server = NFS_SERVER(state->inode); + struct nfs4_exception exception = { }; + int err; + + do { + err = _nfs4_open_expired(ctx, state); + trace_nfs4_open_expired(ctx, 0, err); + if (nfs4_clear_cap_atomic_open_v1(server, err, &exception)) + continue; + switch (err) { + default: + goto out; + case -NFS4ERR_GRACE: + case -NFS4ERR_DELAY: + nfs4_handle_exception(server, err, &exception); + err = 0; + } + } while (exception.retry); +out: + return err; +} + +int nfs4_open_expired(struct nfs4_state_owner *sp, struct nfs4_state *state) +{ + struct nfs_open_context *ctx; + int ret; + + ctx = nfs4_state_find_open_context(state); + if (IS_ERR(ctx)) + return -EAGAIN; + ret = nfs4_do_open_expired(ctx, state); + put_nfs_open_context(ctx); + return ret; +} + +void nfs_finish_clear_delegation_stateid(struct nfs4_state *state, + const nfs4_stateid *stateid) +{ + nfs_remove_bad_delegation(state->inode, stateid); + nfs_state_clear_delegation(state); +} + +static int nfs41_test_and_free_expired_stateid(struct nfs_server *server, + nfs4_stateid *stateid, const struct cred *cred) +{ + int status; + + switch (stateid->type) { + default: + break; + case NFS4_INVALID_STATEID_TYPE: + case NFS4_SPECIAL_STATEID_TYPE: + case NFS4_FREED_STATEID_TYPE: + return -NFS4ERR_BAD_STATEID; + case NFS4_REVOKED_STATEID_TYPE: + goto out_free; + } + + status = nfs41_test_stateid(server, stateid, cred); + switch (status) { + case -NFS4ERR_EXPIRED: + case -NFS4ERR_ADMIN_REVOKED: + case -NFS4ERR_DELEG_REVOKED: + break; + default: + return status; + } +out_free: + /* Ack the revoked state to the server */ + nfs41_free_stateid(server, stateid, cred, true); + return -NFS4ERR_EXPIRED; +} + +static int nfs41_check_delegation_stateid(struct nfs4_state *state) +{ + struct nfs_server *server = NFS_SERVER(state->inode); + nfs4_stateid stateid; + struct nfs_delegation *delegation; + const struct cred *cred = NULL; + int status, ret = NFS_OK; + + /* Get the delegation credential for use by test/free_stateid */ + rcu_read_lock(); + delegation = rcu_dereference(NFS_I(state->inode)->delegation); + if (delegation == NULL) { + rcu_read_unlock(); + nfs_state_clear_delegation(state); + return NFS_OK; + } + + spin_lock(&delegation->lock); + nfs4_stateid_copy(&stateid, &delegation->stateid); + + if (!test_and_clear_bit(NFS_DELEGATION_TEST_EXPIRED, + &delegation->flags)) { + spin_unlock(&delegation->lock); + rcu_read_unlock(); + return NFS_OK; + } + + if (delegation->cred) + cred = get_cred(delegation->cred); + spin_unlock(&delegation->lock); + rcu_read_unlock(); + status = nfs41_test_and_free_expired_stateid(server, &stateid, cred); + trace_nfs4_test_delegation_stateid(state, NULL, status); + if (status == -NFS4ERR_EXPIRED || status == -NFS4ERR_BAD_STATEID) + nfs_finish_clear_delegation_stateid(state, &stateid); + else + ret = status; + + put_cred(cred); + return ret; +} + +static void nfs41_delegation_recover_stateid(struct nfs4_state *state) +{ + nfs4_stateid tmp; + + if (test_bit(NFS_DELEGATED_STATE, &state->flags) && + nfs4_copy_delegation_stateid(state->inode, state->state, + &tmp, NULL) && + nfs4_stateid_match_other(&state->stateid, &tmp)) + nfs_state_set_delegation(state, &tmp, state->state); + else + nfs_state_clear_delegation(state); +} + +/** + * nfs41_check_expired_locks - possibly free a lock stateid + * + * @state: NFSv4 state for an inode + * + * Returns NFS_OK if recovery for this stateid is now finished. + * Otherwise a negative NFS4ERR value is returned. + */ +static int nfs41_check_expired_locks(struct nfs4_state *state) +{ + int status, ret = NFS_OK; + struct nfs4_lock_state *lsp, *prev = NULL; + struct nfs_server *server = NFS_SERVER(state->inode); + + if (!test_bit(LK_STATE_IN_USE, &state->flags)) + goto out; + + spin_lock(&state->state_lock); + list_for_each_entry(lsp, &state->lock_states, ls_locks) { + if (test_bit(NFS_LOCK_INITIALIZED, &lsp->ls_flags)) { + const struct cred *cred = lsp->ls_state->owner->so_cred; + + refcount_inc(&lsp->ls_count); + spin_unlock(&state->state_lock); + + nfs4_put_lock_state(prev); + prev = lsp; + + status = nfs41_test_and_free_expired_stateid(server, + &lsp->ls_stateid, + cred); + trace_nfs4_test_lock_stateid(state, lsp, status); + if (status == -NFS4ERR_EXPIRED || + status == -NFS4ERR_BAD_STATEID) { + clear_bit(NFS_LOCK_INITIALIZED, &lsp->ls_flags); + lsp->ls_stateid.type = NFS4_INVALID_STATEID_TYPE; + if (!recover_lost_locks) + set_bit(NFS_LOCK_LOST, &lsp->ls_flags); + } else if (status != NFS_OK) { + ret = status; + nfs4_put_lock_state(prev); + goto out; + } + spin_lock(&state->state_lock); + } + } + spin_unlock(&state->state_lock); + nfs4_put_lock_state(prev); +out: + return ret; +} + +/** + * nfs41_check_open_stateid - possibly free an open stateid + * + * @state: NFSv4 state for an inode + * + * Returns NFS_OK if recovery for this stateid is now finished. + * Otherwise a negative NFS4ERR value is returned. + */ +static int nfs41_check_open_stateid(struct nfs4_state *state) +{ + struct nfs_server *server = NFS_SERVER(state->inode); + nfs4_stateid *stateid = &state->open_stateid; + const struct cred *cred = state->owner->so_cred; + int status; + + if (test_bit(NFS_OPEN_STATE, &state->flags) == 0) + return -NFS4ERR_BAD_STATEID; + status = nfs41_test_and_free_expired_stateid(server, stateid, cred); + trace_nfs4_test_open_stateid(state, NULL, status); + if (status == -NFS4ERR_EXPIRED || status == -NFS4ERR_BAD_STATEID) { + nfs_state_clear_open_state_flags(state); + stateid->type = NFS4_INVALID_STATEID_TYPE; + return status; + } + if (nfs_open_stateid_recover_openmode(state)) + return -NFS4ERR_OPENMODE; + return NFS_OK; +} + +static int nfs41_open_expired(struct nfs4_state_owner *sp, struct nfs4_state *state) +{ + int status; + + status = nfs41_check_delegation_stateid(state); + if (status != NFS_OK) + return status; + nfs41_delegation_recover_stateid(state); + + status = nfs41_check_expired_locks(state); + if (status != NFS_OK) + return status; + status = nfs41_check_open_stateid(state); + if (status != NFS_OK) + status = nfs4_open_expired(sp, state); + return status; +} + +/* + * on an EXCLUSIVE create, the server should send back a bitmask with FATTR4-* + * fields corresponding to attributes that were used to store the verifier. + * Make sure we clobber those fields in the later setattr call + */ +static unsigned nfs4_exclusive_attrset(struct nfs4_opendata *opendata, + struct iattr *sattr, struct nfs4_label **label) +{ + const __u32 *bitmask = opendata->o_arg.server->exclcreat_bitmask; + __u32 attrset[3]; + unsigned ret; + unsigned i; + + for (i = 0; i < ARRAY_SIZE(attrset); i++) { + attrset[i] = opendata->o_res.attrset[i]; + if (opendata->o_arg.createmode == NFS4_CREATE_EXCLUSIVE4_1) + attrset[i] &= ~bitmask[i]; + } + + ret = (opendata->o_arg.createmode == NFS4_CREATE_EXCLUSIVE) ? + sattr->ia_valid : 0; + + if ((attrset[1] & (FATTR4_WORD1_TIME_ACCESS|FATTR4_WORD1_TIME_ACCESS_SET))) { + if (sattr->ia_valid & ATTR_ATIME_SET) + ret |= ATTR_ATIME_SET; + else + ret |= ATTR_ATIME; + } + + if ((attrset[1] & (FATTR4_WORD1_TIME_MODIFY|FATTR4_WORD1_TIME_MODIFY_SET))) { + if (sattr->ia_valid & ATTR_MTIME_SET) + ret |= ATTR_MTIME_SET; + else + ret |= ATTR_MTIME; + } + + if (!(attrset[2] & FATTR4_WORD2_SECURITY_LABEL)) + *label = NULL; + return ret; +} + +static int _nfs4_open_and_get_state(struct nfs4_opendata *opendata, + struct nfs_open_context *ctx) +{ + struct nfs4_state_owner *sp = opendata->owner; + struct nfs_server *server = sp->so_server; + struct dentry *dentry; + struct nfs4_state *state; + fmode_t acc_mode = _nfs4_ctx_to_accessmode(ctx); + struct inode *dir = d_inode(opendata->dir); + unsigned long dir_verifier; + int ret; + + dir_verifier = nfs_save_change_attribute(dir); + + ret = _nfs4_proc_open(opendata, ctx); + if (ret != 0) + goto out; + + state = _nfs4_opendata_to_nfs4_state(opendata); + ret = PTR_ERR(state); + if (IS_ERR(state)) + goto out; + ctx->state = state; + if (server->caps & NFS_CAP_POSIX_LOCK) + set_bit(NFS_STATE_POSIX_LOCKS, &state->flags); + if (opendata->o_res.rflags & NFS4_OPEN_RESULT_MAY_NOTIFY_LOCK) + set_bit(NFS_STATE_MAY_NOTIFY_LOCK, &state->flags); + if (opendata->o_res.rflags & NFS4_OPEN_RESULT_PRESERVE_UNLINKED) + set_bit(NFS_INO_PRESERVE_UNLINKED, &NFS_I(state->inode)->flags); + + switch(opendata->o_arg.claim) { + default: + break; + case NFS4_OPEN_CLAIM_NULL: + case NFS4_OPEN_CLAIM_DELEGATE_CUR: + case NFS4_OPEN_CLAIM_DELEGATE_PREV: + if (!opendata->rpc_done) + break; + if (opendata->o_res.delegation.type != 0) + dir_verifier = nfs_save_change_attribute(dir); + } + + dentry = opendata->dentry; + nfs_set_verifier(dentry, dir_verifier); + if (d_really_is_negative(dentry)) { + struct dentry *alias; + d_drop(dentry); + alias = d_splice_alias(igrab(state->inode), dentry); + /* d_splice_alias() can't fail here - it's a non-directory */ + if (alias) { + dput(ctx->dentry); + nfs_set_verifier(alias, dir_verifier); + ctx->dentry = dentry = alias; + } + } + + /* Parse layoutget results before we check for access */ + pnfs_parse_lgopen(state->inode, opendata->lgp, ctx); + + ret = nfs4_opendata_access(sp->so_cred, opendata, state, acc_mode); + if (ret != 0) + goto out; + + if (d_inode(dentry) == state->inode) + nfs_inode_attach_open_context(ctx); + +out: + if (!opendata->cancelled) { + if (opendata->lgp) { + nfs4_lgopen_release(opendata->lgp); + opendata->lgp = NULL; + } + nfs4_sequence_free_slot(&opendata->o_res.seq_res); + } + return ret; +} + +/* + * Returns a referenced nfs4_state + */ +static int _nfs4_do_open(struct inode *dir, + struct nfs_open_context *ctx, + int flags, + const struct nfs4_open_createattrs *c, + int *opened) +{ + struct nfs4_state_owner *sp; + struct nfs4_state *state = NULL; + struct nfs_server *server = NFS_SERVER(dir); + struct nfs4_opendata *opendata; + struct dentry *dentry = ctx->dentry; + const struct cred *cred = ctx->cred; + struct nfs4_threshold **ctx_th = &ctx->mdsthreshold; + fmode_t fmode = _nfs4_ctx_to_openmode(ctx); + enum open_claim_type4 claim = NFS4_OPEN_CLAIM_NULL; + struct iattr *sattr = c->sattr; + struct nfs4_label *label = c->label; + int status; + + /* Protect against reboot recovery conflicts */ + status = -ENOMEM; + sp = nfs4_get_state_owner(server, cred, GFP_KERNEL); + if (sp == NULL) { + dprintk("nfs4_do_open: nfs4_get_state_owner failed!\n"); + goto out_err; + } + status = nfs4_client_recover_expired_lease(server->nfs_client); + if (status != 0) + goto err_put_state_owner; + if (d_really_is_positive(dentry)) + nfs4_return_incompatible_delegation(d_inode(dentry), fmode); + status = -ENOMEM; + if (d_really_is_positive(dentry)) + claim = NFS4_OPEN_CLAIM_FH; + opendata = nfs4_opendata_alloc(dentry, sp, fmode, flags, + c, claim, GFP_KERNEL); + if (opendata == NULL) + goto err_put_state_owner; + + if (server->attr_bitmask[2] & FATTR4_WORD2_MDSTHRESHOLD) { + if (!opendata->f_attr.mdsthreshold) { + opendata->f_attr.mdsthreshold = pnfs_mdsthreshold_alloc(); + if (!opendata->f_attr.mdsthreshold) + goto err_opendata_put; + } + opendata->o_arg.open_bitmap = &nfs4_pnfs_open_bitmap[0]; + } + if (d_really_is_positive(dentry)) + opendata->state = nfs4_get_open_state(d_inode(dentry), sp); + + status = _nfs4_open_and_get_state(opendata, ctx); + if (status != 0) + goto err_opendata_put; + state = ctx->state; + + if ((opendata->o_arg.open_flags & (O_CREAT|O_EXCL)) == (O_CREAT|O_EXCL) && + (opendata->o_arg.createmode != NFS4_CREATE_GUARDED)) { + unsigned attrs = nfs4_exclusive_attrset(opendata, sattr, &label); + /* + * send create attributes which was not set by open + * with an extra setattr. + */ + if (attrs || label) { + unsigned ia_old = sattr->ia_valid; + + sattr->ia_valid = attrs; + nfs_fattr_init(opendata->o_res.f_attr); + status = nfs4_do_setattr(state->inode, cred, + opendata->o_res.f_attr, sattr, + ctx, label); + if (status == 0) { + nfs_setattr_update_inode(state->inode, sattr, + opendata->o_res.f_attr); + nfs_setsecurity(state->inode, opendata->o_res.f_attr); + } + sattr->ia_valid = ia_old; + } + } + if (opened && opendata->file_created) + *opened = 1; + + if (pnfs_use_threshold(ctx_th, opendata->f_attr.mdsthreshold, server)) { + *ctx_th = opendata->f_attr.mdsthreshold; + opendata->f_attr.mdsthreshold = NULL; + } + + nfs4_opendata_put(opendata); + nfs4_put_state_owner(sp); + return 0; +err_opendata_put: + nfs4_opendata_put(opendata); +err_put_state_owner: + nfs4_put_state_owner(sp); +out_err: + return status; +} + + +static struct nfs4_state *nfs4_do_open(struct inode *dir, + struct nfs_open_context *ctx, + int flags, + struct iattr *sattr, + struct nfs4_label *label, + int *opened) +{ + struct nfs_server *server = NFS_SERVER(dir); + struct nfs4_exception exception = { + .interruptible = true, + }; + struct nfs4_state *res; + struct nfs4_open_createattrs c = { + .label = label, + .sattr = sattr, + .verf = { + [0] = (__u32)jiffies, + [1] = (__u32)current->pid, + }, + }; + int status; + + do { + status = _nfs4_do_open(dir, ctx, flags, &c, opened); + res = ctx->state; + trace_nfs4_open_file(ctx, flags, status); + if (status == 0) + break; + /* NOTE: BAD_SEQID means the server and client disagree about the + * book-keeping w.r.t. state-changing operations + * (OPEN/CLOSE/LOCK/LOCKU...) + * It is actually a sign of a bug on the client or on the server. + * + * If we receive a BAD_SEQID error in the particular case of + * doing an OPEN, we assume that nfs_increment_open_seqid() will + * have unhashed the old state_owner for us, and that we can + * therefore safely retry using a new one. We should still warn + * the user though... + */ + if (status == -NFS4ERR_BAD_SEQID) { + pr_warn_ratelimited("NFS: v4 server %s " + " returned a bad sequence-id error!\n", + NFS_SERVER(dir)->nfs_client->cl_hostname); + exception.retry = 1; + continue; + } + /* + * BAD_STATEID on OPEN means that the server cancelled our + * state before it received the OPEN_CONFIRM. + * Recover by retrying the request as per the discussion + * on Page 181 of RFC3530. + */ + if (status == -NFS4ERR_BAD_STATEID) { + exception.retry = 1; + continue; + } + if (status == -NFS4ERR_EXPIRED) { + nfs4_schedule_lease_recovery(server->nfs_client); + exception.retry = 1; + continue; + } + if (status == -EAGAIN) { + /* We must have found a delegation */ + exception.retry = 1; + continue; + } + if (nfs4_clear_cap_atomic_open_v1(server, status, &exception)) + continue; + res = ERR_PTR(nfs4_handle_exception(server, + status, &exception)); + } while (exception.retry); + return res; +} + +static int _nfs4_do_setattr(struct inode *inode, + struct nfs_setattrargs *arg, + struct nfs_setattrres *res, + const struct cred *cred, + struct nfs_open_context *ctx) +{ + struct nfs_server *server = NFS_SERVER(inode); + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_SETATTR], + .rpc_argp = arg, + .rpc_resp = res, + .rpc_cred = cred, + }; + const struct cred *delegation_cred = NULL; + unsigned long timestamp = jiffies; + bool truncate; + int status; + + nfs_fattr_init(res->fattr); + + /* Servers should only apply open mode checks for file size changes */ + truncate = (arg->iap->ia_valid & ATTR_SIZE) ? true : false; + if (!truncate) { + nfs4_inode_make_writeable(inode); + goto zero_stateid; + } + + if (nfs4_copy_delegation_stateid(inode, FMODE_WRITE, &arg->stateid, &delegation_cred)) { + /* Use that stateid */ + } else if (ctx != NULL && ctx->state) { + struct nfs_lock_context *l_ctx; + if (!nfs4_valid_open_stateid(ctx->state)) + return -EBADF; + l_ctx = nfs_get_lock_context(ctx); + if (IS_ERR(l_ctx)) + return PTR_ERR(l_ctx); + status = nfs4_select_rw_stateid(ctx->state, FMODE_WRITE, l_ctx, + &arg->stateid, &delegation_cred); + nfs_put_lock_context(l_ctx); + if (status == -EIO) + return -EBADF; + else if (status == -EAGAIN) + goto zero_stateid; + } else { +zero_stateid: + nfs4_stateid_copy(&arg->stateid, &zero_stateid); + } + if (delegation_cred) + msg.rpc_cred = delegation_cred; + + status = nfs4_call_sync(server->client, server, &msg, &arg->seq_args, &res->seq_res, 1); + + put_cred(delegation_cred); + if (status == 0 && ctx != NULL) + renew_lease(server, timestamp); + trace_nfs4_setattr(inode, &arg->stateid, status); + return status; +} + +static int nfs4_do_setattr(struct inode *inode, const struct cred *cred, + struct nfs_fattr *fattr, struct iattr *sattr, + struct nfs_open_context *ctx, struct nfs4_label *ilabel) +{ + struct nfs_server *server = NFS_SERVER(inode); + __u32 bitmask[NFS4_BITMASK_SZ]; + struct nfs4_state *state = ctx ? ctx->state : NULL; + struct nfs_setattrargs arg = { + .fh = NFS_FH(inode), + .iap = sattr, + .server = server, + .bitmask = bitmask, + .label = ilabel, + }; + struct nfs_setattrres res = { + .fattr = fattr, + .server = server, + }; + struct nfs4_exception exception = { + .state = state, + .inode = inode, + .stateid = &arg.stateid, + }; + unsigned long adjust_flags = NFS_INO_INVALID_CHANGE | + NFS_INO_INVALID_CTIME; + int err; + + if (sattr->ia_valid & (ATTR_MODE | ATTR_KILL_SUID | ATTR_KILL_SGID)) + adjust_flags |= NFS_INO_INVALID_MODE; + if (sattr->ia_valid & (ATTR_UID | ATTR_GID)) + adjust_flags |= NFS_INO_INVALID_OTHER; + if (sattr->ia_valid & ATTR_ATIME) + adjust_flags |= NFS_INO_INVALID_ATIME; + if (sattr->ia_valid & ATTR_MTIME) + adjust_flags |= NFS_INO_INVALID_MTIME; + + do { + nfs4_bitmap_copy_adjust(bitmask, nfs4_bitmask(server, fattr->label), + inode, adjust_flags); + + err = _nfs4_do_setattr(inode, &arg, &res, cred, ctx); + switch (err) { + case -NFS4ERR_OPENMODE: + if (!(sattr->ia_valid & ATTR_SIZE)) { + pr_warn_once("NFSv4: server %s is incorrectly " + "applying open mode checks to " + "a SETATTR that is not " + "changing file size.\n", + server->nfs_client->cl_hostname); + } + if (state && !(state->state & FMODE_WRITE)) { + err = -EBADF; + if (sattr->ia_valid & ATTR_OPEN) + err = -EACCES; + goto out; + } + } + err = nfs4_handle_exception(server, err, &exception); + } while (exception.retry); +out: + return err; +} + +static bool +nfs4_wait_on_layoutreturn(struct inode *inode, struct rpc_task *task) +{ + if (inode == NULL || !nfs_have_layout(inode)) + return false; + + return pnfs_wait_on_layoutreturn(inode, task); +} + +/* + * Update the seqid of an open stateid + */ +static void nfs4_sync_open_stateid(nfs4_stateid *dst, + struct nfs4_state *state) +{ + __be32 seqid_open; + u32 dst_seqid; + int seq; + + for (;;) { + if (!nfs4_valid_open_stateid(state)) + break; + seq = read_seqbegin(&state->seqlock); + if (!nfs4_state_match_open_stateid_other(state, dst)) { + nfs4_stateid_copy(dst, &state->open_stateid); + if (read_seqretry(&state->seqlock, seq)) + continue; + break; + } + seqid_open = state->open_stateid.seqid; + if (read_seqretry(&state->seqlock, seq)) + continue; + + dst_seqid = be32_to_cpu(dst->seqid); + if ((s32)(dst_seqid - be32_to_cpu(seqid_open)) < 0) + dst->seqid = seqid_open; + break; + } +} + +/* + * Update the seqid of an open stateid after receiving + * NFS4ERR_OLD_STATEID + */ +static bool nfs4_refresh_open_old_stateid(nfs4_stateid *dst, + struct nfs4_state *state) +{ + __be32 seqid_open; + u32 dst_seqid; + bool ret; + int seq, status = -EAGAIN; + DEFINE_WAIT(wait); + + for (;;) { + ret = false; + if (!nfs4_valid_open_stateid(state)) + break; + seq = read_seqbegin(&state->seqlock); + if (!nfs4_state_match_open_stateid_other(state, dst)) { + if (read_seqretry(&state->seqlock, seq)) + continue; + break; + } + + write_seqlock(&state->seqlock); + seqid_open = state->open_stateid.seqid; + + dst_seqid = be32_to_cpu(dst->seqid); + + /* Did another OPEN bump the state's seqid? try again: */ + if ((s32)(be32_to_cpu(seqid_open) - dst_seqid) > 0) { + dst->seqid = seqid_open; + write_sequnlock(&state->seqlock); + ret = true; + break; + } + + /* server says we're behind but we haven't seen the update yet */ + set_bit(NFS_STATE_CHANGE_WAIT, &state->flags); + prepare_to_wait(&state->waitq, &wait, TASK_KILLABLE); + write_sequnlock(&state->seqlock); + trace_nfs4_close_stateid_update_wait(state->inode, dst, 0); + + if (fatal_signal_pending(current) || nfs_current_task_exiting()) + status = -EINTR; + else + if (schedule_timeout(5*HZ) != 0) + status = 0; + + finish_wait(&state->waitq, &wait); + + if (!status) + continue; + if (status == -EINTR) + break; + + /* we slept the whole 5 seconds, we must have lost a seqid */ + dst->seqid = cpu_to_be32(dst_seqid + 1); + ret = true; + break; + } + + return ret; +} + +struct nfs4_closedata { + struct inode *inode; + struct nfs4_state *state; + struct nfs_closeargs arg; + struct nfs_closeres res; + struct { + struct nfs4_layoutreturn_args arg; + struct nfs4_layoutreturn_res res; + struct nfs4_xdr_opaque_data ld_private; + u32 roc_barrier; + bool roc; + } lr; + struct nfs_fattr fattr; + unsigned long timestamp; + unsigned short retrans; +}; + +static void nfs4_free_closedata(void *data) +{ + struct nfs4_closedata *calldata = data; + struct nfs4_state_owner *sp = calldata->state->owner; + struct super_block *sb = calldata->state->inode->i_sb; + + if (calldata->lr.roc) + pnfs_roc_release(&calldata->lr.arg, &calldata->lr.res, + calldata->res.lr_ret); + nfs4_put_open_state(calldata->state); + nfs_free_seqid(calldata->arg.seqid); + nfs4_put_state_owner(sp); + nfs_sb_deactive(sb); + kfree(calldata); +} + +static void nfs4_close_done(struct rpc_task *task, void *data) +{ + struct nfs4_closedata *calldata = data; + struct nfs4_state *state = calldata->state; + struct nfs_server *server = NFS_SERVER(calldata->inode); + nfs4_stateid *res_stateid = NULL; + struct nfs4_exception exception = { + .state = state, + .inode = calldata->inode, + .stateid = &calldata->arg.stateid, + .retrans = calldata->retrans, + }; + + if (!nfs4_sequence_done(task, &calldata->res.seq_res)) + return; + trace_nfs4_close(state, &calldata->arg, &calldata->res, task->tk_status); + + /* Handle Layoutreturn errors */ + if (pnfs_roc_done(task, &calldata->arg.lr_args, &calldata->res.lr_res, + &calldata->res.lr_ret) == -EAGAIN) + goto out_restart; + + /* hmm. we are done with the inode, and in the process of freeing + * the state_owner. we keep this around to process errors + */ + switch (task->tk_status) { + case 0: + res_stateid = &calldata->res.stateid; + renew_lease(server, calldata->timestamp); + break; + case -NFS4ERR_ACCESS: + if (calldata->arg.bitmask != NULL) { + calldata->arg.bitmask = NULL; + calldata->res.fattr = NULL; + goto out_restart; + + } + break; + case -NFS4ERR_OLD_STATEID: + /* Did we race with OPEN? */ + if (nfs4_refresh_open_old_stateid(&calldata->arg.stateid, + state)) + goto out_restart; + goto out_release; + case -NFS4ERR_ADMIN_REVOKED: + case -NFS4ERR_STALE_STATEID: + case -NFS4ERR_EXPIRED: + nfs4_free_revoked_stateid(server, + &calldata->arg.stateid, + task->tk_msg.rpc_cred); + fallthrough; + case -NFS4ERR_BAD_STATEID: + if (calldata->arg.fmode == 0) + break; + fallthrough; + default: + task->tk_status = nfs4_async_handle_exception(task, + server, task->tk_status, &exception); + calldata->retrans = exception.retrans; + if (exception.retry) + goto out_restart; + } + nfs_clear_open_stateid(state, &calldata->arg.stateid, + res_stateid, calldata->arg.fmode); +out_release: + task->tk_status = 0; + nfs_release_seqid(calldata->arg.seqid); + nfs_refresh_inode(calldata->inode, &calldata->fattr); + dprintk("%s: ret = %d\n", __func__, task->tk_status); + return; +out_restart: + task->tk_status = 0; + rpc_restart_call_prepare(task); + goto out_release; +} + +static void nfs4_close_prepare(struct rpc_task *task, void *data) +{ + struct nfs4_closedata *calldata = data; + struct nfs4_state *state = calldata->state; + struct inode *inode = calldata->inode; + struct nfs_server *server = NFS_SERVER(inode); + struct pnfs_layout_hdr *lo; + bool is_rdonly, is_wronly, is_rdwr; + int call_close = 0; + + if (nfs_wait_on_sequence(calldata->arg.seqid, task) != 0) + goto out_wait; + + task->tk_msg.rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_OPEN_DOWNGRADE]; + spin_lock(&state->owner->so_lock); + is_rdwr = test_bit(NFS_O_RDWR_STATE, &state->flags); + is_rdonly = test_bit(NFS_O_RDONLY_STATE, &state->flags); + is_wronly = test_bit(NFS_O_WRONLY_STATE, &state->flags); + /* Calculate the change in open mode */ + calldata->arg.fmode = 0; + if (state->n_rdwr == 0) { + if (state->n_rdonly == 0) + call_close |= is_rdonly; + else if (is_rdonly) + calldata->arg.fmode |= FMODE_READ; + if (state->n_wronly == 0) + call_close |= is_wronly; + else if (is_wronly) + calldata->arg.fmode |= FMODE_WRITE; + if (calldata->arg.fmode != (FMODE_READ|FMODE_WRITE)) + call_close |= is_rdwr; + } else if (is_rdwr) + calldata->arg.fmode |= FMODE_READ|FMODE_WRITE; + + nfs4_sync_open_stateid(&calldata->arg.stateid, state); + if (!nfs4_valid_open_stateid(state)) + call_close = 0; + spin_unlock(&state->owner->so_lock); + + if (!call_close) { + /* Note: exit _without_ calling nfs4_close_done */ + goto out_no_action; + } + + if (!calldata->lr.roc && nfs4_wait_on_layoutreturn(inode, task)) { + nfs_release_seqid(calldata->arg.seqid); + goto out_wait; + } + + lo = calldata->arg.lr_args ? calldata->arg.lr_args->layout : NULL; + if (lo && !pnfs_layout_is_valid(lo)) { + calldata->arg.lr_args = NULL; + calldata->res.lr_res = NULL; + } + + if (calldata->arg.fmode == 0) + task->tk_msg.rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_CLOSE]; + + if (calldata->arg.fmode == 0 || calldata->arg.fmode == FMODE_READ) { + /* Close-to-open cache consistency revalidation */ + if (!nfs4_have_delegation(inode, FMODE_READ, 0)) { + nfs4_bitmask_set(calldata->arg.bitmask_store, + server->cache_consistency_bitmask, + inode, 0); + calldata->arg.bitmask = calldata->arg.bitmask_store; + } else + calldata->arg.bitmask = NULL; + } + + calldata->arg.share_access = + nfs4_fmode_to_share_access(calldata->arg.fmode); + + if (calldata->res.fattr == NULL) + calldata->arg.bitmask = NULL; + else if (calldata->arg.bitmask == NULL) + calldata->res.fattr = NULL; + calldata->timestamp = jiffies; + if (nfs4_setup_sequence(NFS_SERVER(inode)->nfs_client, + &calldata->arg.seq_args, + &calldata->res.seq_res, + task) != 0) + nfs_release_seqid(calldata->arg.seqid); + return; +out_no_action: + task->tk_action = NULL; +out_wait: + nfs4_sequence_done(task, &calldata->res.seq_res); +} + +static const struct rpc_call_ops nfs4_close_ops = { + .rpc_call_prepare = nfs4_close_prepare, + .rpc_call_done = nfs4_close_done, + .rpc_release = nfs4_free_closedata, +}; + +/* + * It is possible for data to be read/written from a mem-mapped file + * after the sys_close call (which hits the vfs layer as a flush). + * This means that we can't safely call nfsv4 close on a file until + * the inode is cleared. This in turn means that we are not good + * NFSv4 citizens - we do not indicate to the server to update the file's + * share state even when we are done with one of the three share + * stateid's in the inode. + * + * NOTE: Caller must be holding the sp->so_owner semaphore! + */ +int nfs4_do_close(struct nfs4_state *state, gfp_t gfp_mask, int wait) +{ + struct nfs_server *server = NFS_SERVER(state->inode); + struct nfs_client *clp = server->nfs_client; + struct nfs_seqid *(*alloc_seqid)(struct nfs_seqid_counter *, gfp_t); + struct nfs4_closedata *calldata; + struct nfs4_state_owner *sp = state->owner; + struct rpc_task *task; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_CLOSE], + .rpc_cred = state->owner->so_cred, + }; + struct rpc_task_setup task_setup_data = { + .rpc_client = server->client, + .rpc_message = &msg, + .callback_ops = &nfs4_close_ops, + .workqueue = nfsiod_workqueue, + .flags = RPC_TASK_ASYNC | RPC_TASK_CRED_NOREF, + }; + int status = -ENOMEM; + + if (nfs_server_capable(state->inode, NFS_CAP_MOVEABLE)) + task_setup_data.flags |= RPC_TASK_MOVEABLE; + + nfs4_state_protect(clp, NFS_SP4_MACH_CRED_CLEANUP, + &task_setup_data.rpc_client, &msg); + + calldata = kzalloc_obj(*calldata, gfp_mask); + if (calldata == NULL) + goto out; + nfs4_init_sequence(clp, &calldata->arg.seq_args, + &calldata->res.seq_res, 1, 0); + calldata->inode = state->inode; + calldata->state = state; + calldata->arg.fh = NFS_FH(state->inode); + if (!nfs4_copy_open_stateid(&calldata->arg.stateid, state)) + goto out_free_calldata; + /* Serialization for the sequence id */ + alloc_seqid = clp->cl_mvops->alloc_seqid; + calldata->arg.seqid = alloc_seqid(&state->owner->so_seqid, gfp_mask); + if (IS_ERR(calldata->arg.seqid)) + goto out_free_calldata; + nfs_fattr_init(&calldata->fattr); + calldata->arg.fmode = 0; + calldata->lr.arg.ld_private = &calldata->lr.ld_private; + calldata->res.fattr = &calldata->fattr; + calldata->res.seqid = calldata->arg.seqid; + calldata->res.server = server; + calldata->res.lr_ret = -NFS4ERR_NOMATCHING_LAYOUT; + calldata->lr.roc = pnfs_roc(state->inode, &calldata->lr.arg, + &calldata->lr.res, msg.rpc_cred, wait); + if (calldata->lr.roc) { + calldata->arg.lr_args = &calldata->lr.arg; + calldata->res.lr_res = &calldata->lr.res; + } + nfs_sb_active(calldata->inode->i_sb); + + msg.rpc_argp = &calldata->arg; + msg.rpc_resp = &calldata->res; + task_setup_data.callback_data = calldata; + task = rpc_run_task(&task_setup_data); + if (IS_ERR(task)) + return PTR_ERR(task); + status = 0; + if (wait) + status = rpc_wait_for_completion_task(task); + rpc_put_task(task); + return status; +out_free_calldata: + kfree(calldata); +out: + nfs4_put_open_state(state); + nfs4_put_state_owner(sp); + return status; +} + +static struct inode * +nfs4_atomic_open(struct inode *dir, struct nfs_open_context *ctx, + int open_flags, struct iattr *attr, int *opened) +{ + struct nfs4_state *state; + struct nfs4_label l, *label; + + label = nfs4_label_init_security(dir, ctx->dentry, attr, &l); + + /* Protect against concurrent sillydeletes */ + state = nfs4_do_open(dir, ctx, open_flags, attr, label, opened); + + nfs4_label_release_security(label); + + if (IS_ERR(state)) + return ERR_CAST(state); + return state->inode; +} + +static void nfs4_close_context(struct nfs_open_context *ctx, int is_sync) +{ + struct dentry *dentry = ctx->dentry; + if (ctx->state == NULL) + return; + if (dentry->d_flags & DCACHE_NFSFS_RENAMED) + nfs4_inode_set_return_delegation_on_close(d_inode(dentry)); + if (is_sync) + nfs4_close_sync(ctx->state, _nfs4_ctx_to_openmode(ctx)); + else + nfs4_close_state(ctx->state, _nfs4_ctx_to_openmode(ctx)); +} + +#define FATTR4_WORD1_NFS40_MASK (2*FATTR4_WORD1_MOUNTED_ON_FILEID - 1UL) +#define FATTR4_WORD2_NFS41_MASK (2*FATTR4_WORD2_SUPPATTR_EXCLCREAT - 1UL) +#define FATTR4_WORD2_NFS42_MASK (2*FATTR4_WORD2_OPEN_ARGUMENTS - 1UL) + +#define FATTR4_WORD2_NFS42_TIME_DELEG_MASK \ + (FATTR4_WORD2_TIME_DELEG_MODIFY|FATTR4_WORD2_TIME_DELEG_ACCESS) +static bool nfs4_server_delegtime_capable(struct nfs4_server_caps_res *res) +{ + u32 share_access_want = res->open_caps.oa_share_access_want[0]; + u32 attr_bitmask = res->attr_bitmask[2]; + + return (share_access_want & NFS4_SHARE_WANT_DELEG_TIMESTAMPS) && + ((attr_bitmask & FATTR4_WORD2_NFS42_TIME_DELEG_MASK) == + FATTR4_WORD2_NFS42_TIME_DELEG_MASK); +} + +static int _nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *fhandle) +{ + u32 minorversion = server->nfs_client->cl_minorversion; + u32 bitmask[3] = { + [0] = FATTR4_WORD0_SUPPORTED_ATTRS, + }; + struct nfs4_server_caps_arg args = { + .fhandle = fhandle, + .bitmask = bitmask, + }; + struct nfs4_server_caps_res res = {}; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_SERVER_CAPS], + .rpc_argp = &args, + .rpc_resp = &res, + }; + int status; + int i; + + bitmask[0] = FATTR4_WORD0_SUPPORTED_ATTRS | + FATTR4_WORD0_FH_EXPIRE_TYPE | + FATTR4_WORD0_LINK_SUPPORT | + FATTR4_WORD0_SYMLINK_SUPPORT | + FATTR4_WORD0_ACLSUPPORT | + FATTR4_WORD0_CASE_INSENSITIVE | + FATTR4_WORD0_CASE_PRESERVING; + if (minorversion) + bitmask[2] = FATTR4_WORD2_SUPPATTR_EXCLCREAT; + if (minorversion > 1) + bitmask[2] |= FATTR4_WORD2_OPEN_ARGUMENTS; + + status = nfs4_call_sync(server->client, server, &msg, &args.seq_args, &res.seq_res, 0); + if (status == 0) { + bitmask[0] = (FATTR4_WORD0_SUPPORTED_ATTRS | + FATTR4_WORD0_FH_EXPIRE_TYPE | + FATTR4_WORD0_LINK_SUPPORT | + FATTR4_WORD0_SYMLINK_SUPPORT | + FATTR4_WORD0_ACLSUPPORT | + FATTR4_WORD0_CASE_INSENSITIVE | + FATTR4_WORD0_CASE_PRESERVING) & + res.attr_bitmask[0]; + /* Sanity check the server answers */ + switch (minorversion) { + case 0: + res.attr_bitmask[1] &= FATTR4_WORD1_NFS40_MASK; + res.attr_bitmask[2] = 0; + break; + case 1: + res.attr_bitmask[2] &= FATTR4_WORD2_NFS41_MASK; + bitmask[2] = FATTR4_WORD2_SUPPATTR_EXCLCREAT & + res.attr_bitmask[2]; + break; + case 2: + res.attr_bitmask[2] &= FATTR4_WORD2_NFS42_MASK; + bitmask[2] = (FATTR4_WORD2_SUPPATTR_EXCLCREAT | + FATTR4_WORD2_OPEN_ARGUMENTS) & + res.attr_bitmask[2]; + } + memcpy(server->attr_bitmask, res.attr_bitmask, sizeof(server->attr_bitmask)); + server->caps &= + ~(NFS_CAP_ACLS | NFS_CAP_HARDLINKS | NFS_CAP_SYMLINKS | + NFS_CAP_SECURITY_LABEL | NFS_CAP_FS_LOCATIONS | + NFS_CAP_OPEN_XOR | NFS_CAP_DELEGTIME | + NFS_CAP_CASE_INSENSITIVE | NFS_CAP_CASE_NONPRESERVING); + server->fattr_valid = NFS_ATTR_FATTR_V4; + if (res.attr_bitmask[0] & FATTR4_WORD0_ACL && + res.acl_bitmask & ACL4_SUPPORT_ALLOW_ACL) + server->caps |= NFS_CAP_ACLS; + if (res.has_links != 0) + server->caps |= NFS_CAP_HARDLINKS; + if (res.has_symlinks != 0) + server->caps |= NFS_CAP_SYMLINKS; + if (res.case_insensitive) + server->caps |= NFS_CAP_CASE_INSENSITIVE; + if ((res.attr_bitmask[0] & FATTR4_WORD0_CASE_PRESERVING) && + !res.case_preserving) + server->caps |= NFS_CAP_CASE_NONPRESERVING; +#ifdef CONFIG_NFS_V4_SECURITY_LABEL + if (res.attr_bitmask[2] & FATTR4_WORD2_SECURITY_LABEL) + server->caps |= NFS_CAP_SECURITY_LABEL; +#endif + if (res.attr_bitmask[0] & FATTR4_WORD0_FS_LOCATIONS) + server->caps |= NFS_CAP_FS_LOCATIONS; + if (!(res.attr_bitmask[0] & FATTR4_WORD0_FILEID)) + server->fattr_valid &= ~NFS_ATTR_FATTR_FILEID; + if (!(res.attr_bitmask[1] & FATTR4_WORD1_MODE)) + server->fattr_valid &= ~NFS_ATTR_FATTR_MODE; + if (!(res.attr_bitmask[1] & FATTR4_WORD1_NUMLINKS)) + server->fattr_valid &= ~NFS_ATTR_FATTR_NLINK; + if (!(res.attr_bitmask[1] & FATTR4_WORD1_OWNER)) + server->fattr_valid &= ~(NFS_ATTR_FATTR_OWNER | + NFS_ATTR_FATTR_OWNER_NAME); + if (!(res.attr_bitmask[1] & FATTR4_WORD1_OWNER_GROUP)) + server->fattr_valid &= ~(NFS_ATTR_FATTR_GROUP | + NFS_ATTR_FATTR_GROUP_NAME); + if (!(res.attr_bitmask[1] & FATTR4_WORD1_SPACE_USED)) + server->fattr_valid &= ~NFS_ATTR_FATTR_SPACE_USED; + if (!(res.attr_bitmask[1] & FATTR4_WORD1_TIME_ACCESS)) + server->fattr_valid &= ~NFS_ATTR_FATTR_ATIME; + if (!(res.attr_bitmask[1] & FATTR4_WORD1_TIME_METADATA)) + server->fattr_valid &= ~NFS_ATTR_FATTR_CTIME; + if (!(res.attr_bitmask[1] & FATTR4_WORD1_TIME_MODIFY)) + server->fattr_valid &= ~NFS_ATTR_FATTR_MTIME; + if (!(res.attr_bitmask[1] & FATTR4_WORD1_TIME_MODIFY)) + server->fattr_valid &= ~NFS_ATTR_FATTR_MTIME; + if (!(res.attr_bitmask[1] & FATTR4_WORD1_TIME_CREATE)) + server->fattr_valid &= ~NFS_ATTR_FATTR_BTIME; + memcpy(server->attr_bitmask_nl, res.attr_bitmask, + sizeof(server->attr_bitmask)); + server->attr_bitmask_nl[2] &= ~FATTR4_WORD2_SECURITY_LABEL; + + if (res.open_caps.oa_share_access_want[0] & + NFS4_SHARE_WANT_OPEN_XOR_DELEGATION) + server->caps |= NFS_CAP_OPEN_XOR; + if (nfs4_server_delegtime_capable(&res)) + server->caps |= NFS_CAP_DELEGTIME; + + memcpy(server->cache_consistency_bitmask, res.attr_bitmask, sizeof(server->cache_consistency_bitmask)); + server->cache_consistency_bitmask[0] &= FATTR4_WORD0_CHANGE|FATTR4_WORD0_SIZE; + server->cache_consistency_bitmask[1] &= FATTR4_WORD1_TIME_METADATA|FATTR4_WORD1_TIME_MODIFY; + server->cache_consistency_bitmask[2] = 0; + + /* Avoid a regression due to buggy server */ + for (i = 0; i < ARRAY_SIZE(res.exclcreat_bitmask); i++) + res.exclcreat_bitmask[i] &= res.attr_bitmask[i]; + memcpy(server->exclcreat_bitmask, res.exclcreat_bitmask, + sizeof(server->exclcreat_bitmask)); + + server->acl_bitmask = res.acl_bitmask; + server->fh_expire_type = res.fh_expire_type; + } + + return status; +} + +int nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *fhandle) +{ + struct nfs4_exception exception = { + .interruptible = true, + }; + int err; + + do { + err = nfs4_handle_exception(server, + _nfs4_server_capabilities(server, fhandle), + &exception); + } while (exception.retry); + return err; +} + +static void test_fs_location_for_trunking(struct nfs4_fs_location *location, + struct nfs_client *clp, + struct nfs_server *server) +{ + int i; + + for (i = 0; i < location->nservers; i++) { + struct nfs4_string *srv_loc = &location->servers[i]; + struct sockaddr_storage addr; + size_t addrlen; + struct xprt_create xprt_args = { + .ident = 0, + .net = clp->cl_net, + }; + struct nfs4_add_xprt_data xprtdata = { + .clp = clp, + }; + struct rpc_add_xprt_test rpcdata = { + .add_xprt_test = clp->cl_mvops->session_trunk, + .data = &xprtdata, + }; + char *servername = NULL; + + if (!srv_loc->len) + continue; + + addrlen = nfs_parse_server_name(srv_loc->data, srv_loc->len, + &addr, sizeof(addr), + clp->cl_net, server->port); + if (!addrlen) + return; + xprt_args.dstaddr = (struct sockaddr *)&addr; + xprt_args.addrlen = addrlen; + servername = kmalloc(srv_loc->len + 1, GFP_KERNEL); + if (!servername) + return; + memcpy(servername, srv_loc->data, srv_loc->len); + servername[srv_loc->len] = '\0'; + xprt_args.servername = servername; + + xprtdata.cred = nfs4_get_clid_cred(clp); + rpc_clnt_add_xprt(clp->cl_rpcclient, &xprt_args, + rpc_clnt_setup_test_and_add_xprt, + &rpcdata); + if (xprtdata.cred) + put_cred(xprtdata.cred); + kfree(servername); + } +} + +static bool _is_same_nfs4_pathname(struct nfs4_pathname *path1, + struct nfs4_pathname *path2) +{ + int i; + + if (path1->ncomponents != path2->ncomponents) + return false; + for (i = 0; i < path1->ncomponents; i++) { + if (path1->components[i].len != path2->components[i].len) + return false; + if (memcmp(path1->components[i].data, path2->components[i].data, + path1->components[i].len)) + return false; + } + return true; +} + +static int _nfs4_discover_trunking(struct nfs_server *server, + struct nfs_fh *fhandle) +{ + struct nfs4_fs_locations *locations = NULL; + struct page *page; + const struct cred *cred; + struct nfs_client *clp = server->nfs_client; + const struct nfs4_state_maintenance_ops *ops = + clp->cl_mvops->state_renewal_ops; + int status = -ENOMEM, i; + + cred = ops->get_state_renewal_cred(clp); + if (cred == NULL) { + cred = nfs4_get_clid_cred(clp); + if (cred == NULL) + return -ENOKEY; + } + + page = alloc_page(GFP_KERNEL); + if (!page) + goto out_put_cred; + locations = kmalloc_obj(struct nfs4_fs_locations); + if (!locations) + goto out_free; + locations->fattr = nfs_alloc_fattr(); + if (!locations->fattr) + goto out_free_2; + + status = nfs4_proc_get_locations(server, fhandle, locations, page, + cred); + if (status) + goto out_free_3; + + for (i = 0; i < locations->nlocations; i++) { + if (!_is_same_nfs4_pathname(&locations->fs_path, + &locations->locations[i].rootpath)) + continue; + test_fs_location_for_trunking(&locations->locations[i], clp, + server); + } +out_free_3: + kfree(locations->fattr); +out_free_2: + kfree(locations); +out_free: + __free_page(page); +out_put_cred: + put_cred(cred); + return status; +} + +static int nfs4_discover_trunking(struct nfs_server *server, + struct nfs_fh *fhandle) +{ + struct nfs4_exception exception = { + .interruptible = true, + }; + struct nfs_client *clp = server->nfs_client; + int err = 0; + + if (!nfs4_has_session(clp)) + goto out; + do { + err = nfs4_handle_exception(server, + _nfs4_discover_trunking(server, fhandle), + &exception); + } while (exception.retry); +out: + return err; +} + +static int _nfs4_lookup_root(struct nfs_server *server, struct nfs_fh *fhandle, + struct nfs_fattr *fattr) +{ + u32 bitmask[3] = { + [0] = FATTR4_WORD0_TYPE | FATTR4_WORD0_CHANGE | + FATTR4_WORD0_SIZE | FATTR4_WORD0_FSID, + }; + struct nfs4_lookup_root_arg args = { + .bitmask = bitmask, + }; + struct nfs4_lookup_res res = { + .server = server, + .fattr = fattr, + .fh = fhandle, + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_LOOKUP_ROOT], + .rpc_argp = &args, + .rpc_resp = &res, + }; + + nfs_fattr_init(fattr); + return nfs4_call_sync(server->client, server, &msg, &args.seq_args, &res.seq_res, 0); +} + +static int nfs4_lookup_root(struct nfs_server *server, struct nfs_fh *fhandle, + struct nfs_fattr *fattr) +{ + struct nfs4_exception exception = { + .interruptible = true, + }; + int err; + do { + err = _nfs4_lookup_root(server, fhandle, fattr); + trace_nfs4_lookup_root(server, fhandle, fattr, err); + switch (err) { + case 0: + case -NFS4ERR_WRONGSEC: + goto out; + default: + err = nfs4_handle_exception(server, err, &exception); + } + } while (exception.retry); +out: + return err; +} + +static int nfs4_lookup_root_sec(struct nfs_server *server, + struct nfs_fh *fhandle, struct nfs_fattr *fattr, + rpc_authflavor_t flavor) +{ + struct rpc_auth_create_args auth_args = { + .pseudoflavor = flavor, + }; + struct rpc_auth *auth; + + auth = rpcauth_create(&auth_args, server->client); + if (IS_ERR(auth)) + return -EACCES; + return nfs4_lookup_root(server, fhandle, fattr); +} + +/* + * Retry pseudoroot lookup with various security flavors. We do this when: + * + * NFSv4.0: the PUTROOTFH operation returns NFS4ERR_WRONGSEC + * NFSv4.1: the server does not support the SECINFO_NO_NAME operation + * + * Returns zero on success, or a negative NFS4ERR value, or a + * negative errno value. + */ +int nfs4_find_root_sec(struct nfs_server *server, struct nfs_fh *fhandle, + struct nfs_fattr *fattr) +{ + /* Per 3530bis 15.33.5 */ + static const rpc_authflavor_t flav_array[] = { + RPC_AUTH_GSS_KRB5P, + RPC_AUTH_GSS_KRB5I, + RPC_AUTH_GSS_KRB5, + RPC_AUTH_UNIX, /* courtesy */ + RPC_AUTH_NULL, + }; + int status = -EPERM; + size_t i; + + if (server->auth_info.flavor_len > 0) { + /* try each flavor specified by user */ + for (i = 0; i < server->auth_info.flavor_len; i++) { + status = nfs4_lookup_root_sec( + server, fhandle, fattr, + server->auth_info.flavors[i]); + if (status == -NFS4ERR_WRONGSEC || status == -EACCES) + continue; + break; + } + } else { + /* no flavors specified by user, try default list */ + for (i = 0; i < ARRAY_SIZE(flav_array); i++) { + status = nfs4_lookup_root_sec(server, fhandle, fattr, + flav_array[i]); + if (status == -NFS4ERR_WRONGSEC || status == -EACCES) + continue; + break; + } + } + + /* + * -EACCES could mean that the user doesn't have correct permissions + * to access the mount. It could also mean that we tried to mount + * with a gss auth flavor, but rpc.gssd isn't running. Either way, + * existing mount programs don't handle -EACCES very well so it should + * be mapped to -EPERM instead. + */ + if (status == -EACCES) + status = -EPERM; + return status; +} + +/** + * nfs4_proc_get_rootfh - get file handle for server's pseudoroot + * @server: initialized nfs_server handle + * @fhandle: we fill in the pseudo-fs root file handle + * @fattr: we fill in a bare bones struct fattr + * @auth_probe: probe the auth flavours + * + * Returns zero on success, or a negative errno. + */ +int nfs4_proc_get_rootfh(struct nfs_server *server, struct nfs_fh *fhandle, + struct nfs_fattr *fattr, bool auth_probe) +{ + int status = 0; + + if (!auth_probe) + status = nfs4_lookup_root(server, fhandle, fattr); + + if (auth_probe || status == NFS4ERR_WRONGSEC) + status = server->nfs_client->cl_mvops->find_root_sec( + server, fhandle, fattr); + + return nfs4_map_errors(status); +} + +static int nfs4_proc_get_root(struct nfs_server *server, struct nfs_fh *mntfh, + struct nfs_fsinfo *info) +{ + int error; + struct nfs_fattr *fattr = info->fattr; + + error = nfs4_server_capabilities(server, mntfh); + if (error < 0) { + dprintk("nfs4_get_root: getcaps error = %d\n", -error); + return error; + } + + error = nfs4_proc_getattr(server, mntfh, fattr, NULL); + if (error < 0) { + dprintk("nfs4_get_root: getattr error = %d\n", -error); + goto out; + } + + if (fattr->valid & NFS_ATTR_FATTR_FSID && + !nfs_fsid_equal(&server->fsid, &fattr->fsid)) + memcpy(&server->fsid, &fattr->fsid, sizeof(server->fsid)); + +out: + return error; +} + +/* + * Get locations and (maybe) other attributes of a referral. + * Note that we'll actually follow the referral later when + * we detect fsid mismatch in inode revalidation + */ +static int nfs4_get_referral(struct rpc_clnt *client, struct inode *dir, + const struct qstr *name, struct nfs_fattr *fattr, + struct nfs_fh *fhandle) +{ + int status = -ENOMEM; + struct page *page = NULL; + struct nfs4_fs_locations *locations = NULL; + + page = alloc_page(GFP_KERNEL); + if (page == NULL) + goto out; + locations = kmalloc_obj(struct nfs4_fs_locations); + if (locations == NULL) + goto out; + + locations->fattr = fattr; + + status = nfs4_proc_fs_locations(client, dir, name, locations, page); + if (status != 0) + goto out; + + /* + * If the fsid didn't change, this is a migration event, not a + * referral. Cause us to drop into the exception handler, which + * will kick off migration recovery. + */ + if (nfs_fsid_equal(&NFS_SERVER(dir)->fsid, &fattr->fsid)) { + dprintk("%s: server did not return a different fsid for" + " a referral at %s\n", __func__, name->name); + status = -NFS4ERR_MOVED; + goto out; + } + /* Fixup attributes for the nfs_lookup() call to nfs_fhget() */ + nfs_fixup_referral_attributes(fattr); + memset(fhandle, 0, sizeof(struct nfs_fh)); +out: + if (page) + __free_page(page); + kfree(locations); + return status; +} + +static bool should_request_dir_deleg(struct inode *inode) +{ + if (!directory_delegations) + return false; + if (!inode) + return false; + if (!S_ISDIR(inode->i_mode)) + return false; + if (!nfs_server_capable(inode, NFS_CAP_DIR_DELEG)) + return false; + if (!test_and_clear_bit(NFS_INO_REQ_DIR_DELEG, &(NFS_I(inode)->flags))) + return false; + if (nfs4_have_delegation(inode, FMODE_READ, 0)) + return false; + return true; +} + +static void nfs4_call_getattr_prepare(struct rpc_task *task, void *calldata) +{ + struct nfs4_call_sync_data *data = calldata; + nfs4_setup_sequence(data->seq_server->nfs_client, data->seq_args, + data->seq_res, task); +} + +static void nfs4_call_getattr_done(struct rpc_task *task, void *calldata) +{ + struct nfs4_call_sync_data *data = calldata; + + nfs4_sequence_process(task, data->seq_res); +} + +static const struct rpc_call_ops nfs4_call_getattr_ops = { + .rpc_call_prepare = nfs4_call_getattr_prepare, + .rpc_call_done = nfs4_call_getattr_done, +}; + +static int _nfs4_proc_getattr(struct nfs_server *server, struct nfs_fh *fhandle, + struct nfs_fattr *fattr, struct inode *inode) +{ + __u32 bitmask[NFS4_BITMASK_SZ]; + struct nfs4_getattr_arg args = { + .fh = fhandle, + .bitmask = bitmask, + }; + struct nfs4_getattr_res res = { + .fattr = fattr, + .server = server, + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_GETATTR], + .rpc_argp = &args, + .rpc_resp = &res, + }; + struct nfs4_call_sync_data data = { + .seq_server = server, + .seq_args = &args.seq_args, + .seq_res = &res.seq_res, + }; + struct rpc_task_setup task_setup = { + .rpc_client = server->client, + .rpc_message = &msg, + .callback_ops = &nfs4_call_getattr_ops, + .callback_data = &data, + }; + struct nfs_client *clp = server->nfs_client; + struct nfs4_gdd_res gdd_res; + int status; + + if (nfs4_has_session(clp)) + task_setup.flags = RPC_TASK_MOVEABLE; + + /* Is this is an attribute revalidation, subject to softreval? */ + if (inode && (server->flags & NFS_MOUNT_SOFTREVAL)) + task_setup.flags |= RPC_TASK_TIMEOUT; + + args.get_dir_deleg = should_request_dir_deleg(inode); + if (args.get_dir_deleg) + res.gdd_res = &gdd_res; + + nfs4_bitmap_copy_adjust(bitmask, nfs4_bitmask(server, fattr->label), inode, 0); + nfs_fattr_init(fattr); + nfs4_init_sequence(clp, &args.seq_args, &res.seq_res, 0, 0); + + status = nfs4_call_sync_custom(&task_setup); + + if (args.get_dir_deleg) { + switch (status) { + case 0: + if (gdd_res.status != GDD4_OK) + break; + nfs_inode_set_delegation(inode, current_cred(), + FMODE_READ, &gdd_res.deleg, 0, + NFS4_OPEN_DELEGATE_READ); + break; + case -ENOTSUPP: + case -EOPNOTSUPP: + server->caps &= ~NFS_CAP_DIR_DELEG; + break; + case -NFS4ERR_INVAL: + case -NFS4ERR_IO: + case -NFS4ERR_DIRDELEG_UNAVAIL: + case -NFS4ERR_NOTDIR: + clear_bit(NFS_INO_REQ_DIR_DELEG, &(NFS_I(inode)->flags)); + status = -EAGAIN; + } + } + + nfs4_sequence_free_slot(&res.seq_res); + return status; +} + +int nfs4_proc_getattr(struct nfs_server *server, struct nfs_fh *fhandle, + struct nfs_fattr *fattr, struct inode *inode) +{ + struct nfs4_exception exception = { + .interruptible = true, + }; + int err; + do { + err = _nfs4_proc_getattr(server, fhandle, fattr, inode); + trace_nfs4_getattr(server, fhandle, fattr, err); + switch (err) { + default: + err = nfs4_handle_exception(server, err, &exception); + break; + case -EAGAIN: + case -ENOTSUPP: + case -EOPNOTSUPP: + exception.retry = true; + } + } while (exception.retry); + return err; +} + +/* + * The file is not closed if it is opened due to the a request to change + * the size of the file. The open call will not be needed once the + * VFS layer lookup-intents are implemented. + * + * Close is called when the inode is destroyed. + * If we haven't opened the file for O_WRONLY, we + * need to in the size_change case to obtain a stateid. + * + * Got race? + * Because OPEN is always done by name in nfsv4, it is + * possible that we opened a different file by the same + * name. We can recognize this race condition, but we + * can't do anything about it besides returning an error. + * + * This will be fixed with VFS changes (lookup-intent). + */ +static int +nfs4_proc_setattr(struct dentry *dentry, struct nfs_fattr *fattr, + struct iattr *sattr) +{ + struct inode *inode = d_inode(dentry); + const struct cred *cred = NULL; + struct nfs_open_context *ctx = NULL; + int status; + + if (pnfs_ld_layoutret_on_setattr(inode) && + sattr->ia_valid & ATTR_SIZE && + sattr->ia_size < i_size_read(inode)) + pnfs_commit_and_return_layout(inode); + + nfs_fattr_init(fattr); + + /* Deal with open(O_TRUNC) */ + if (sattr->ia_valid & ATTR_OPEN) + sattr->ia_valid &= ~(ATTR_MTIME|ATTR_CTIME); + + /* Optimization: if the end result is no change, don't RPC */ + if ((sattr->ia_valid & ~(ATTR_FILE|ATTR_OPEN)) == 0) + return 0; + + /* Search for an existing open(O_WRITE) file */ + if (sattr->ia_valid & ATTR_FILE) { + + ctx = nfs_file_open_context(sattr->ia_file); + if (ctx) + cred = ctx->cred; + } + + /* Return any delegations if we're going to change ACLs */ + if ((sattr->ia_valid & (ATTR_MODE|ATTR_UID|ATTR_GID)) != 0) + nfs4_inode_make_writeable(inode); + + status = nfs4_do_setattr(inode, cred, fattr, sattr, ctx, NULL); + if (status == 0) { + nfs_setattr_update_inode(inode, sattr, fattr); + nfs_setsecurity(inode, fattr); + } + return status; +} + +static int _nfs4_proc_lookup(struct rpc_clnt *clnt, struct inode *dir, + struct dentry *dentry, const struct qstr *name, + struct nfs_fh *fhandle, struct nfs_fattr *fattr) +{ + struct nfs_server *server = NFS_SERVER(dir); + int status; + struct nfs4_lookup_arg args = { + .bitmask = server->attr_bitmask, + .dir_fh = NFS_FH(dir), + .name = name, + }; + struct nfs4_lookup_res res = { + .server = server, + .fattr = fattr, + .fh = fhandle, + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_LOOKUP], + .rpc_argp = &args, + .rpc_resp = &res, + }; + unsigned short task_flags = 0; + + if (nfs_server_capable(dir, NFS_CAP_MOVEABLE)) + task_flags = RPC_TASK_MOVEABLE; + + /* Is this is an attribute revalidation, subject to softreval? */ + if (nfs_lookup_is_soft_revalidate(dentry)) + task_flags |= RPC_TASK_TIMEOUT; + + args.bitmask = nfs4_bitmask(server, fattr->label); + + nfs_fattr_init(fattr); + + dprintk("NFS call lookup %pd2\n", dentry); + nfs4_init_sequence(server->nfs_client, &args.seq_args, &res.seq_res, 0, 0); + status = nfs4_do_call_sync(clnt, server, &msg, + &args.seq_args, &res.seq_res, task_flags); + dprintk("NFS reply lookup: %d\n", status); + return status; +} + +static void nfs_fixup_secinfo_attributes(struct nfs_fattr *fattr) +{ + fattr->valid |= NFS_ATTR_FATTR_TYPE | NFS_ATTR_FATTR_MODE | + NFS_ATTR_FATTR_NLINK | NFS_ATTR_FATTR_MOUNTPOINT; + fattr->mode = S_IFDIR | S_IRUGO | S_IXUGO; + fattr->nlink = 2; +} + +static int nfs4_proc_lookup_common(struct rpc_clnt **clnt, struct inode *dir, + struct dentry *dentry, const struct qstr *name, + struct nfs_fh *fhandle, struct nfs_fattr *fattr) +{ + struct nfs4_exception exception = { + .interruptible = true, + }; + struct rpc_clnt *client = *clnt; + int err; + do { + err = _nfs4_proc_lookup(client, dir, dentry, name, fhandle, fattr); + trace_nfs4_lookup(dir, name, err); + switch (err) { + case -NFS4ERR_BADNAME: + err = -ENOENT; + goto out; + case -NFS4ERR_MOVED: + err = nfs4_get_referral(client, dir, name, fattr, fhandle); + if (err == -NFS4ERR_MOVED) + err = nfs4_handle_exception(NFS_SERVER(dir), err, &exception); + goto out; + case -NFS4ERR_WRONGSEC: + err = -EPERM; + if (client != *clnt) + goto out; + client = nfs4_negotiate_security(client, dir, name); + if (IS_ERR(client)) + return PTR_ERR(client); + + exception.retry = 1; + break; + default: + err = nfs4_handle_exception(NFS_SERVER(dir), err, &exception); + } + } while (exception.retry); + +out: + if (err == 0) + *clnt = client; + else if (client != *clnt) + rpc_shutdown_client(client); + + return err; +} + +static int nfs4_proc_lookup(struct inode *dir, struct dentry *dentry, const struct qstr *name, + struct nfs_fh *fhandle, struct nfs_fattr *fattr) +{ + int status; + struct rpc_clnt *client = NFS_CLIENT(dir); + + status = nfs4_proc_lookup_common(&client, dir, dentry, name, fhandle, fattr); + if (client != NFS_CLIENT(dir)) { + rpc_shutdown_client(client); + nfs_fixup_secinfo_attributes(fattr); + } + return status; +} + +struct rpc_clnt * +nfs4_proc_lookup_mountpoint(struct inode *dir, struct dentry *dentry, + struct nfs_fh *fhandle, struct nfs_fattr *fattr) +{ + struct rpc_clnt *client = NFS_CLIENT(dir); + int status; + + status = nfs4_proc_lookup_common(&client, dir, dentry, &dentry->d_name, + fhandle, fattr); + if (status < 0) + return ERR_PTR(status); + return (client == NFS_CLIENT(dir)) ? rpc_clone_client(client) : client; +} + +static int _nfs4_proc_lookupp(struct inode *inode, + struct nfs_fh *fhandle, struct nfs_fattr *fattr) +{ + struct rpc_clnt *clnt = NFS_CLIENT(inode); + struct nfs_server *server = NFS_SERVER(inode); + int status; + struct nfs4_lookupp_arg args = { + .bitmask = server->attr_bitmask, + .fh = NFS_FH(inode), + }; + struct nfs4_lookupp_res res = { + .server = server, + .fattr = fattr, + .fh = fhandle, + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_LOOKUPP], + .rpc_argp = &args, + .rpc_resp = &res, + }; + unsigned short task_flags = 0; + + if (server->flags & NFS_MOUNT_SOFTREVAL) + task_flags |= RPC_TASK_TIMEOUT; + if (server->caps & NFS_CAP_MOVEABLE) + task_flags |= RPC_TASK_MOVEABLE; + + args.bitmask = nfs4_bitmask(server, fattr->label); + + nfs_fattr_init(fattr); + nfs4_init_sequence(server->nfs_client, &args.seq_args, &res.seq_res, 0, 0); + + dprintk("NFS call lookupp ino=0x%llx\n", inode->i_ino); + status = nfs4_do_call_sync(clnt, server, &msg, &args.seq_args, + &res.seq_res, task_flags); + dprintk("NFS reply lookupp: %d\n", status); + return status; +} + +static int nfs4_proc_lookupp(struct inode *inode, struct nfs_fh *fhandle, + struct nfs_fattr *fattr) +{ + struct nfs4_exception exception = { + .interruptible = true, + }; + int err; + do { + err = _nfs4_proc_lookupp(inode, fhandle, fattr); + trace_nfs4_lookupp(inode, err); + err = nfs4_handle_exception(NFS_SERVER(inode), err, + &exception); + } while (exception.retry); + return err; +} + +static int _nfs4_proc_access(struct inode *inode, struct nfs_access_entry *entry, + const struct cred *cred) +{ + struct nfs_server *server = NFS_SERVER(inode); + struct nfs4_accessargs args = { + .fh = NFS_FH(inode), + .access = entry->mask, + }; + struct nfs4_accessres res = { + .server = server, + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_ACCESS], + .rpc_argp = &args, + .rpc_resp = &res, + .rpc_cred = cred, + }; + int status = 0; + + if (!nfs4_have_delegation(inode, FMODE_READ, 0)) { + nfs_request_directory_delegation(inode); + res.fattr = nfs_alloc_fattr(); + if (res.fattr == NULL) + return -ENOMEM; + args.bitmask = server->cache_consistency_bitmask; + } + status = nfs4_call_sync(server->client, server, &msg, &args.seq_args, &res.seq_res, 0); + if (!status) { + nfs_access_set_mask(entry, res.access); + if (res.fattr) + nfs_refresh_inode(inode, res.fattr); + } + nfs_free_fattr(res.fattr); + return status; +} + +static int nfs4_proc_access(struct inode *inode, struct nfs_access_entry *entry, + const struct cred *cred) +{ + struct nfs4_exception exception = { + .interruptible = true, + }; + int err; + do { + err = _nfs4_proc_access(inode, entry, cred); + trace_nfs4_access(inode, err); + err = nfs4_handle_exception(NFS_SERVER(inode), err, + &exception); + } while (exception.retry); + return err; +} + +/* + * TODO: For the time being, we don't try to get any attributes + * along with any of the zero-copy operations READ, READDIR, + * READLINK, WRITE. + * + * In the case of the first three, we want to put the GETATTR + * after the read-type operation -- this is because it is hard + * to predict the length of a GETATTR response in v4, and thus + * align the READ data correctly. This means that the GETATTR + * may end up partially falling into the page cache, and we should + * shift it into the 'tail' of the xdr_buf before processing. + * To do this efficiently, we need to know the total length + * of data received, which doesn't seem to be available outside + * of the RPC layer. + * + * In the case of WRITE, we also want to put the GETATTR after + * the operation -- in this case because we want to make sure + * we get the post-operation mtime and size. + * + * Both of these changes to the XDR layer would in fact be quite + * minor, but I decided to leave them for a subsequent patch. + */ +static int _nfs4_proc_readlink(struct inode *inode, struct page *page, + unsigned int pgbase, unsigned int pglen) +{ + struct nfs4_readlink args = { + .fh = NFS_FH(inode), + .pgbase = pgbase, + .pglen = pglen, + .pages = &page, + }; + struct nfs4_readlink_res res; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_READLINK], + .rpc_argp = &args, + .rpc_resp = &res, + }; + + return nfs4_call_sync(NFS_SERVER(inode)->client, NFS_SERVER(inode), &msg, &args.seq_args, &res.seq_res, 0); +} + +static int nfs4_proc_readlink(struct inode *inode, struct page *page, + unsigned int pgbase, unsigned int pglen) +{ + struct nfs4_exception exception = { + .interruptible = true, + }; + int err; + do { + err = _nfs4_proc_readlink(inode, page, pgbase, pglen); + trace_nfs4_readlink(inode, err); + err = nfs4_handle_exception(NFS_SERVER(inode), err, + &exception); + } while (exception.retry); + return err; +} + +/* + * This is just for mknod. open(O_CREAT) will always do ->open_context(). + */ +static int +nfs4_proc_create(struct inode *dir, struct dentry *dentry, struct iattr *sattr, + int flags) +{ + struct nfs_server *server = NFS_SERVER(dir); + struct nfs4_label l, *ilabel; + struct nfs_open_context *ctx; + struct nfs4_state *state; + int status = 0; + + ctx = alloc_nfs_open_context(dentry, FMODE_READ, NULL); + if (IS_ERR(ctx)) + return PTR_ERR(ctx); + + ilabel = nfs4_label_init_security(dir, dentry, sattr, &l); + + nfs_request_directory_delegation(dir); + + if (!(server->attr_bitmask[2] & FATTR4_WORD2_MODE_UMASK)) + sattr->ia_mode &= ~current_umask(); + state = nfs4_do_open(dir, ctx, flags, sattr, ilabel, NULL); + if (IS_ERR(state)) { + status = PTR_ERR(state); + goto out; + } +out: + nfs4_label_release_security(ilabel); + put_nfs_open_context(ctx); + return status; +} + +static int +_nfs4_proc_remove(struct inode *dir, const struct qstr *name, u32 ftype) +{ + struct nfs_server *server = NFS_SERVER(dir); + struct nfs_removeargs args = { + .fh = NFS_FH(dir), + .name = *name, + }; + struct nfs_removeres res = { + .server = server, + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_REMOVE], + .rpc_argp = &args, + .rpc_resp = &res, + }; + unsigned long timestamp = jiffies; + int status; + + status = nfs4_call_sync(server->client, server, &msg, &args.seq_args, &res.seq_res, 1); + if (status == 0) { + spin_lock(&dir->i_lock); + /* Removing a directory decrements nlink in the parent */ + if (ftype == NF4DIR && dir->i_nlink > 2) + nfs4_dec_nlink_locked(dir); + nfs4_update_changeattr_locked(dir, &res.cinfo, timestamp, + NFS_INO_INVALID_DATA); + spin_unlock(&dir->i_lock); + } + return status; +} + +static int nfs4_proc_remove(struct inode *dir, struct dentry *dentry) +{ + struct nfs4_exception exception = { + .interruptible = true, + }; + struct inode *inode = d_inode(dentry); + int err; + + if (inode) { + if (inode->i_nlink == 1) + nfs4_inode_return_delegation(inode); + else + nfs4_inode_make_writeable(inode); + } + do { + err = _nfs4_proc_remove(dir, &dentry->d_name, NF4REG); + trace_nfs4_remove(dir, &dentry->d_name, err); + err = nfs4_handle_exception(NFS_SERVER(dir), err, + &exception); + } while (exception.retry); + return err; +} + +static int nfs4_proc_rmdir(struct inode *dir, const struct qstr *name) +{ + struct nfs4_exception exception = { + .interruptible = true, + }; + int err; + + do { + err = _nfs4_proc_remove(dir, name, NF4DIR); + trace_nfs4_remove(dir, name, err); + err = nfs4_handle_exception(NFS_SERVER(dir), err, + &exception); + } while (exception.retry); + return err; +} + +static void nfs4_proc_unlink_setup(struct rpc_message *msg, + struct dentry *dentry, + struct inode *inode) +{ + struct nfs_removeargs *args = msg->rpc_argp; + struct nfs_removeres *res = msg->rpc_resp; + struct nfs_server *server = NFS_SB(dentry->d_sb); + + res->server = server; + msg->rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_REMOVE]; + nfs4_init_sequence(server->nfs_client, &args->seq_args, + &res->seq_res, 1, 0); + + nfs_fattr_init(res->dir_attr); + nfs_request_directory_delegation(d_inode(dentry->d_parent)); + + if (inode) { + nfs4_inode_return_delegation(inode); + nfs_d_prune_case_insensitive_aliases(inode); + } +} + +static void nfs4_proc_unlink_rpc_prepare(struct rpc_task *task, struct nfs_unlinkdata *data) +{ + nfs4_setup_sequence(NFS_SB(data->dentry->d_sb)->nfs_client, + &data->args.seq_args, + &data->res.seq_res, + task); +} + +static int nfs4_proc_unlink_done(struct rpc_task *task, struct inode *dir) +{ + struct nfs_unlinkdata *data = task->tk_calldata; + struct nfs_removeres *res = &data->res; + + if (!nfs4_sequence_done(task, &res->seq_res)) + return 0; + if (nfs4_async_handle_error(task, res->server, NULL, + &data->timeout) == -EAGAIN) + return 0; + if (task->tk_status == 0) + nfs4_update_changeattr(dir, &res->cinfo, + res->dir_attr->time_start, + NFS_INO_INVALID_DATA); + return 1; +} + +static void nfs4_proc_rename_setup(struct rpc_message *msg, + struct dentry *old_dentry, + struct dentry *new_dentry, + struct inode *same_parent) +{ + struct nfs_server *server = NFS_SB(old_dentry->d_sb); + struct nfs_renameargs *arg = msg->rpc_argp; + struct nfs_renameres *res = msg->rpc_resp; + struct inode *old_inode = d_inode(old_dentry); + struct inode *new_inode = d_inode(new_dentry); + + if (old_inode) + nfs4_inode_make_writeable(old_inode); + if (new_inode) + nfs4_inode_return_delegation(new_inode); + if (same_parent) + nfs_request_directory_delegation(same_parent); + msg->rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_RENAME]; + res->server = server; + nfs4_init_sequence(server->nfs_client, &arg->seq_args, + &res->seq_res, 1, 0); +} + +static void nfs4_proc_rename_rpc_prepare(struct rpc_task *task, struct nfs_renamedata *data) +{ + nfs4_setup_sequence(NFS_SERVER(data->old_dir)->nfs_client, + &data->args.seq_args, + &data->res.seq_res, + task); +} + +static int nfs4_proc_rename_done(struct rpc_task *task, struct inode *old_dir, + struct inode *new_dir) +{ + struct nfs_renamedata *data = task->tk_calldata; + struct nfs_renameres *res = &data->res; + + if (!nfs4_sequence_done(task, &res->seq_res)) + return 0; + if (nfs4_async_handle_error(task, res->server, NULL, &data->timeout) == -EAGAIN) + return 0; + + if (task->tk_status == 0) { + nfs_d_prune_case_insensitive_aliases(d_inode(data->old_dentry)); + if (new_dir != old_dir) { + /* Note: If we moved a directory, nlink will change */ + nfs4_update_changeattr(old_dir, &res->old_cinfo, + res->old_fattr->time_start, + NFS_INO_INVALID_NLINK | + NFS_INO_INVALID_DATA); + nfs4_update_changeattr(new_dir, &res->new_cinfo, + res->new_fattr->time_start, + NFS_INO_INVALID_NLINK | + NFS_INO_INVALID_DATA); + nfs_update_delegated_mtime(new_dir); + } else + nfs4_update_changeattr(old_dir, &res->old_cinfo, + res->old_fattr->time_start, + NFS_INO_INVALID_DATA); + } + return 1; +} + +static int _nfs4_proc_link(struct inode *inode, struct inode *dir, const struct qstr *name) +{ + struct nfs_server *server = NFS_SERVER(inode); + __u32 bitmask[NFS4_BITMASK_SZ]; + struct nfs4_link_arg arg = { + .fh = NFS_FH(inode), + .dir_fh = NFS_FH(dir), + .name = name, + .bitmask = bitmask, + }; + struct nfs4_link_res res = { + .server = server, + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_LINK], + .rpc_argp = &arg, + .rpc_resp = &res, + }; + int status = -ENOMEM; + + res.fattr = nfs_alloc_fattr_with_label(server); + if (res.fattr == NULL) + goto out; + + nfs4_inode_make_writeable(inode); + nfs4_bitmap_copy_adjust(bitmask, nfs4_bitmask(server, res.fattr->label), + inode, + NFS_INO_INVALID_CHANGE | NFS_INO_INVALID_CTIME); + status = nfs4_call_sync(server->client, server, &msg, &arg.seq_args, &res.seq_res, 1); + if (!status) { + nfs4_update_changeattr(dir, &res.cinfo, res.fattr->time_start, + NFS_INO_INVALID_DATA); + nfs4_inc_nlink(inode); + status = nfs_post_op_update_inode(inode, res.fattr); + if (!status) + nfs_setsecurity(inode, res.fattr); + } + +out: + nfs_free_fattr(res.fattr); + return status; +} + +static int nfs4_proc_link(struct inode *inode, struct inode *dir, const struct qstr *name) +{ + struct nfs4_exception exception = { + .interruptible = true, + }; + int err; + do { + err = nfs4_handle_exception(NFS_SERVER(inode), + _nfs4_proc_link(inode, dir, name), + &exception); + } while (exception.retry); + return err; +} + +struct nfs4_createdata { + struct rpc_message msg; + struct nfs4_create_arg arg; + struct nfs4_create_res res; + struct nfs_fh fh; + struct nfs_fattr fattr; +}; + +static struct nfs4_createdata *nfs4_alloc_createdata(struct inode *dir, + const struct qstr *name, struct iattr *sattr, u32 ftype) +{ + struct nfs4_createdata *data; + + data = kzalloc_obj(*data); + if (data != NULL) { + struct nfs_server *server = NFS_SERVER(dir); + + data->fattr.label = nfs4_label_alloc(server, GFP_KERNEL); + if (IS_ERR(data->fattr.label)) + goto out_free; + + data->msg.rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_CREATE]; + data->msg.rpc_argp = &data->arg; + data->msg.rpc_resp = &data->res; + data->arg.dir_fh = NFS_FH(dir); + data->arg.server = server; + data->arg.name = name; + data->arg.attrs = sattr; + data->arg.ftype = ftype; + data->arg.bitmask = nfs4_bitmask(server, data->fattr.label); + data->arg.umask = current_umask(); + data->res.server = server; + data->res.fh = &data->fh; + data->res.fattr = &data->fattr; + nfs_fattr_init(data->res.fattr); + } + return data; +out_free: + kfree(data); + return NULL; +} + +static int nfs4_do_create(struct inode *dir, struct dentry *dentry, struct nfs4_createdata *data) +{ + int status = nfs4_call_sync(NFS_SERVER(dir)->client, NFS_SERVER(dir), &data->msg, + &data->arg.seq_args, &data->res.seq_res, 1); + if (status == 0) { + spin_lock(&dir->i_lock); + nfs4_update_changeattr_locked(dir, &data->res.dir_cinfo, + data->res.fattr->time_start, + NFS_INO_INVALID_DATA); + spin_unlock(&dir->i_lock); + status = nfs_instantiate(dentry, data->res.fh, data->res.fattr); + } + return status; +} + +static struct dentry *nfs4_do_mkdir(struct inode *dir, struct dentry *dentry, + struct nfs4_createdata *data, int *statusp) +{ + struct dentry *ret; + + *statusp = nfs4_call_sync(NFS_SERVER(dir)->client, NFS_SERVER(dir), &data->msg, + &data->arg.seq_args, &data->res.seq_res, 1); + + if (*statusp) + return NULL; + + spin_lock(&dir->i_lock); + /* Creating a directory bumps nlink in the parent */ + nfs4_inc_nlink_locked(dir); + nfs4_update_changeattr_locked(dir, &data->res.dir_cinfo, + data->res.fattr->time_start, + NFS_INO_INVALID_DATA); + spin_unlock(&dir->i_lock); + ret = nfs_add_or_obtain(dentry, data->res.fh, data->res.fattr); + if (!IS_ERR(ret)) + return ret; + *statusp = PTR_ERR(ret); + return NULL; +} + +static void nfs4_free_createdata(struct nfs4_createdata *data) +{ + nfs4_label_free(data->fattr.label); + kfree(data); +} + +static int _nfs4_proc_symlink(struct inode *dir, struct dentry *dentry, + struct folio *folio, unsigned int len, struct iattr *sattr, + struct nfs4_label *label) +{ + struct page *page = &folio->page; + struct nfs4_createdata *data; + int status = -ENAMETOOLONG; + + if (len > NFS4_MAXPATHLEN) + goto out; + + status = -ENOMEM; + data = nfs4_alloc_createdata(dir, &dentry->d_name, sattr, NF4LNK); + if (data == NULL) + goto out; + + data->msg.rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_SYMLINK]; + data->arg.u.symlink.pages = &page; + data->arg.u.symlink.len = len; + data->arg.label = label; + + status = nfs4_do_create(dir, dentry, data); + + nfs4_free_createdata(data); +out: + return status; +} + +static int nfs4_proc_symlink(struct inode *dir, struct dentry *dentry, + struct folio *folio, unsigned int len, struct iattr *sattr) +{ + struct nfs4_exception exception = { + .interruptible = true, + }; + struct nfs4_label l, *label; + int err; + + label = nfs4_label_init_security(dir, dentry, sattr, &l); + + do { + err = _nfs4_proc_symlink(dir, dentry, folio, len, sattr, label); + trace_nfs4_symlink(dir, &dentry->d_name, err); + err = nfs4_handle_exception(NFS_SERVER(dir), err, + &exception); + } while (exception.retry); + + nfs4_label_release_security(label); + return err; +} + +static struct dentry *_nfs4_proc_mkdir(struct inode *dir, struct dentry *dentry, + struct iattr *sattr, + struct nfs4_label *label, int *statusp) +{ + struct nfs4_createdata *data; + struct dentry *ret = NULL; + + *statusp = -ENOMEM; + data = nfs4_alloc_createdata(dir, &dentry->d_name, sattr, NF4DIR); + if (data == NULL) + goto out; + + data->arg.label = label; + ret = nfs4_do_mkdir(dir, dentry, data, statusp); + + nfs4_free_createdata(data); +out: + return ret; +} + +static struct dentry *nfs4_proc_mkdir(struct inode *dir, struct dentry *dentry, + struct iattr *sattr) +{ + struct nfs_server *server = NFS_SERVER(dir); + struct nfs4_exception exception = { + .interruptible = true, + }; + struct nfs4_label l, *label; + struct dentry *alias; + int err; + + label = nfs4_label_init_security(dir, dentry, sattr, &l); + + if (!(server->attr_bitmask[2] & FATTR4_WORD2_MODE_UMASK)) + sattr->ia_mode &= ~current_umask(); + do { + alias = _nfs4_proc_mkdir(dir, dentry, sattr, label, &err); + trace_nfs4_mkdir(dir, &dentry->d_name, err); + err = nfs4_handle_exception(NFS_SERVER(dir), err, &exception); + if (err) + alias = ERR_PTR(err); + } while (exception.retry); + nfs4_label_release_security(label); + + return alias; +} + +static int _nfs4_proc_readdir(struct nfs_readdir_arg *nr_arg, + struct nfs_readdir_res *nr_res) +{ + struct inode *dir = d_inode(nr_arg->dentry); + struct nfs_server *server = NFS_SERVER(dir); + struct nfs4_readdir_arg args = { + .fh = NFS_FH(dir), + .pages = nr_arg->pages, + .pgbase = 0, + .count = nr_arg->page_len, + .plus = nr_arg->plus, + }; + struct nfs4_readdir_res res; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_READDIR], + .rpc_argp = &args, + .rpc_resp = &res, + .rpc_cred = nr_arg->cred, + }; + int status; + + dprintk("%s: dentry = %pd2, cookie = %llu\n", __func__, + nr_arg->dentry, (unsigned long long)nr_arg->cookie); + if (!(server->caps & NFS_CAP_SECURITY_LABEL)) + args.bitmask = server->attr_bitmask_nl; + else + args.bitmask = server->attr_bitmask; + + nfs4_setup_readdir(nr_arg->cookie, nr_arg->verf, nr_arg->dentry, &args); + res.pgbase = args.pgbase; + status = nfs4_call_sync(server->client, server, &msg, &args.seq_args, + &res.seq_res, 0); + if (status >= 0) { + memcpy(nr_res->verf, res.verifier.data, NFS4_VERIFIER_SIZE); + status += args.pgbase; + } + + nfs_invalidate_atime(dir); + + dprintk("%s: returns %d\n", __func__, status); + return status; +} + +static int nfs4_proc_readdir(struct nfs_readdir_arg *arg, + struct nfs_readdir_res *res) +{ + struct nfs4_exception exception = { + .interruptible = true, + }; + int err; + do { + err = _nfs4_proc_readdir(arg, res); + trace_nfs4_readdir(d_inode(arg->dentry), err); + err = nfs4_handle_exception(NFS_SERVER(d_inode(arg->dentry)), + err, &exception); + } while (exception.retry); + return err; +} + +static int _nfs4_proc_mknod(struct inode *dir, struct dentry *dentry, + struct iattr *sattr, struct nfs4_label *label, dev_t rdev) +{ + struct nfs4_createdata *data; + int mode = sattr->ia_mode; + int status = -ENOMEM; + + data = nfs4_alloc_createdata(dir, &dentry->d_name, sattr, NF4SOCK); + if (data == NULL) + goto out; + + if (S_ISFIFO(mode)) + data->arg.ftype = NF4FIFO; + else if (S_ISBLK(mode)) { + data->arg.ftype = NF4BLK; + data->arg.u.device.specdata1 = MAJOR(rdev); + data->arg.u.device.specdata2 = MINOR(rdev); + } + else if (S_ISCHR(mode)) { + data->arg.ftype = NF4CHR; + data->arg.u.device.specdata1 = MAJOR(rdev); + data->arg.u.device.specdata2 = MINOR(rdev); + } else if (!S_ISSOCK(mode)) { + status = -EINVAL; + goto out_free; + } + + data->arg.label = label; + status = nfs4_do_create(dir, dentry, data); +out_free: + nfs4_free_createdata(data); +out: + return status; +} + +static int nfs4_proc_mknod(struct inode *dir, struct dentry *dentry, + struct iattr *sattr, dev_t rdev) +{ + struct nfs_server *server = NFS_SERVER(dir); + struct nfs4_exception exception = { + .interruptible = true, + }; + struct nfs4_label l, *label; + int err; + + label = nfs4_label_init_security(dir, dentry, sattr, &l); + + if (!(server->attr_bitmask[2] & FATTR4_WORD2_MODE_UMASK)) + sattr->ia_mode &= ~current_umask(); + do { + err = _nfs4_proc_mknod(dir, dentry, sattr, label, rdev); + trace_nfs4_mknod(dir, &dentry->d_name, err); + err = nfs4_handle_exception(NFS_SERVER(dir), err, + &exception); + } while (exception.retry); + + nfs4_label_release_security(label); + + return err; +} + +static int _nfs4_proc_statfs(struct nfs_server *server, struct nfs_fh *fhandle, + struct nfs_fsstat *fsstat) +{ + struct nfs4_statfs_arg args = { + .fh = fhandle, + .bitmask = server->attr_bitmask, + }; + struct nfs4_statfs_res res = { + .fsstat = fsstat, + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_STATFS], + .rpc_argp = &args, + .rpc_resp = &res, + }; + + nfs_fattr_init(fsstat->fattr); + return nfs4_call_sync(server->client, server, &msg, &args.seq_args, &res.seq_res, 0); +} + +static int nfs4_proc_statfs(struct nfs_server *server, struct nfs_fh *fhandle, struct nfs_fsstat *fsstat) +{ + struct nfs4_exception exception = { + .interruptible = true, + }; + int err; + do { + err = nfs4_handle_exception(server, + _nfs4_proc_statfs(server, fhandle, fsstat), + &exception); + } while (exception.retry); + return err; +} + +static int _nfs4_do_fsinfo(struct nfs_server *server, struct nfs_fh *fhandle, + struct nfs_fsinfo *fsinfo) +{ + struct nfs4_fsinfo_arg args = { + .fh = fhandle, + .bitmask = server->attr_bitmask, + }; + struct nfs4_fsinfo_res res = { + .fsinfo = fsinfo, + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_FSINFO], + .rpc_argp = &args, + .rpc_resp = &res, + }; + + return nfs4_call_sync(server->client, server, &msg, &args.seq_args, &res.seq_res, 0); +} + +static int nfs4_do_fsinfo(struct nfs_server *server, struct nfs_fh *fhandle, struct nfs_fsinfo *fsinfo) +{ + struct nfs4_exception exception = { + .interruptible = true, + }; + int err; + + do { + err = _nfs4_do_fsinfo(server, fhandle, fsinfo); + trace_nfs4_fsinfo(server, fhandle, fsinfo->fattr, err); + if (err == 0) { + nfs4_set_lease_period(server->nfs_client, fsinfo->lease_time); + break; + } + err = nfs4_handle_exception(server, err, &exception); + } while (exception.retry); + return err; +} + +static int nfs4_proc_fsinfo(struct nfs_server *server, struct nfs_fh *fhandle, struct nfs_fsinfo *fsinfo) +{ + int error; + + nfs_fattr_init(fsinfo->fattr); + error = nfs4_do_fsinfo(server, fhandle, fsinfo); + if (error == 0) { + /* block layout checks this! */ + server->pnfs_blksize = fsinfo->blksize; + set_pnfs_layoutdriver(server, fhandle, fsinfo); + } + + return error; +} + +static int _nfs4_proc_pathconf(struct nfs_server *server, struct nfs_fh *fhandle, + struct nfs_pathconf *pathconf) +{ + struct nfs4_pathconf_arg args = { + .fh = fhandle, + .bitmask = server->attr_bitmask, + }; + struct nfs4_pathconf_res res = { + .pathconf = pathconf, + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_PATHCONF], + .rpc_argp = &args, + .rpc_resp = &res, + }; + + /* None of the pathconf attributes are mandatory to implement */ + if ((args.bitmask[0] & nfs4_pathconf_bitmap[0]) == 0) { + memset(pathconf, 0, sizeof(*pathconf)); + return 0; + } + + nfs_fattr_init(pathconf->fattr); + return nfs4_call_sync(server->client, server, &msg, &args.seq_args, &res.seq_res, 0); +} + +static int nfs4_proc_pathconf(struct nfs_server *server, struct nfs_fh *fhandle, + struct nfs_pathconf *pathconf) +{ + struct nfs4_exception exception = { + .interruptible = true, + }; + int err; + + do { + err = nfs4_handle_exception(server, + _nfs4_proc_pathconf(server, fhandle, pathconf), + &exception); + } while (exception.retry); + return err; +} + +int nfs4_set_rw_stateid(nfs4_stateid *stateid, + const struct nfs_open_context *ctx, + const struct nfs_lock_context *l_ctx, + fmode_t fmode) +{ + return nfs4_select_rw_stateid(ctx->state, fmode, l_ctx, stateid, NULL); +} +EXPORT_SYMBOL_GPL(nfs4_set_rw_stateid); + +static bool nfs4_stateid_is_current(nfs4_stateid *stateid, + const struct nfs_open_context *ctx, + const struct nfs_lock_context *l_ctx, + fmode_t fmode) +{ + nfs4_stateid _current_stateid; + + /* If the current stateid represents a lost lock, then exit */ + if (nfs4_set_rw_stateid(&_current_stateid, ctx, l_ctx, fmode) == -EIO) + return true; + return nfs4_stateid_match(stateid, &_current_stateid); +} + +static bool nfs4_error_stateid_expired(int err) +{ + switch (err) { + case -NFS4ERR_DELEG_REVOKED: + case -NFS4ERR_ADMIN_REVOKED: + case -NFS4ERR_BAD_STATEID: + case -NFS4ERR_STALE_STATEID: + case -NFS4ERR_OLD_STATEID: + case -NFS4ERR_OPENMODE: + case -NFS4ERR_EXPIRED: + return true; + } + return false; +} + +static int nfs4_read_done_cb(struct rpc_task *task, struct nfs_pgio_header *hdr) +{ + struct nfs_server *server = NFS_SERVER(hdr->inode); + + trace_nfs4_read(hdr, task->tk_status); + if (task->tk_status < 0) { + struct nfs4_exception exception = { + .inode = hdr->inode, + .state = hdr->args.context->state, + .stateid = &hdr->args.stateid, + .retrans = hdr->retrans, + }; + task->tk_status = nfs4_async_handle_exception(task, + server, task->tk_status, &exception); + hdr->retrans = exception.retrans; + if (exception.retry) { + rpc_restart_call_prepare(task); + return -EAGAIN; + } + } + + if (task->tk_status > 0) + renew_lease(server, hdr->timestamp); + return 0; +} + +static bool nfs4_read_stateid_changed(struct rpc_task *task, + struct nfs_pgio_args *args) +{ + + if (!nfs4_error_stateid_expired(task->tk_status) || + nfs4_stateid_is_current(&args->stateid, + args->context, + args->lock_context, + FMODE_READ)) + return false; + rpc_restart_call_prepare(task); + return true; +} + +static bool nfs4_read_plus_not_supported(struct rpc_task *task, + struct nfs_pgio_header *hdr) +{ + struct nfs_server *server = NFS_SERVER(hdr->inode); + struct rpc_message *msg = &task->tk_msg; + + if (msg->rpc_proc == &nfs4_procedures[NFSPROC4_CLNT_READ_PLUS] && + task->tk_status == -ENOTSUPP) { + server->caps &= ~NFS_CAP_READ_PLUS; + msg->rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_READ]; + rpc_restart_call_prepare(task); + return true; + } + return false; +} + +static int nfs4_read_done(struct rpc_task *task, struct nfs_pgio_header *hdr) +{ + if (!nfs4_sequence_done(task, &hdr->res.seq_res)) + return -EAGAIN; + if (nfs4_read_stateid_changed(task, &hdr->args)) + return -EAGAIN; + if (nfs4_read_plus_not_supported(task, hdr)) + return -EAGAIN; + if (task->tk_status > 0) + nfs_invalidate_atime(hdr->inode); + return hdr->pgio_done_cb ? hdr->pgio_done_cb(task, hdr) : + nfs4_read_done_cb(task, hdr); +} + +#if defined CONFIG_NFS_V4_2 && defined CONFIG_NFS_V4_2_READ_PLUS +static bool nfs42_read_plus_support(struct nfs_pgio_header *hdr, + struct rpc_message *msg) +{ + /* Note: We don't use READ_PLUS with pNFS yet */ + if (nfs_server_capable(hdr->inode, NFS_CAP_READ_PLUS) && !hdr->ds_clp) { + msg->rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_READ_PLUS]; + return nfs_read_alloc_scratch(hdr, READ_PLUS_SCRATCH_SIZE); + } + return false; +} +#else +static bool nfs42_read_plus_support(struct nfs_pgio_header *hdr, + struct rpc_message *msg) +{ + return false; +} +#endif /* CONFIG_NFS_V4_2 */ + +static void nfs4_proc_read_setup(struct nfs_pgio_header *hdr, + struct rpc_message *msg) +{ + hdr->timestamp = jiffies; + if (!hdr->pgio_done_cb) + hdr->pgio_done_cb = nfs4_read_done_cb; + if (!nfs42_read_plus_support(hdr, msg)) + msg->rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_READ]; + nfs4_init_sequence(NFS_SERVER(hdr->inode)->nfs_client, + &hdr->args.seq_args, &hdr->res.seq_res, 0, 0); +} + +static int nfs4_proc_pgio_rpc_prepare(struct rpc_task *task, + struct nfs_pgio_header *hdr) +{ + if (nfs4_setup_sequence(NFS_SERVER(hdr->inode)->nfs_client, + &hdr->args.seq_args, + &hdr->res.seq_res, + task)) + return 0; + if (nfs4_set_rw_stateid(&hdr->args.stateid, hdr->args.context, + hdr->args.lock_context, + hdr->rw_mode) == -EIO) + return -EIO; + if (unlikely(test_bit(NFS_CONTEXT_BAD, &hdr->args.context->flags))) + return -EIO; + return 0; +} + +static int nfs4_write_done_cb(struct rpc_task *task, + struct nfs_pgio_header *hdr) +{ + struct inode *inode = hdr->inode; + + trace_nfs4_write(hdr, task->tk_status); + if (task->tk_status < 0) { + struct nfs4_exception exception = { + .inode = hdr->inode, + .state = hdr->args.context->state, + .stateid = &hdr->args.stateid, + .retrans = hdr->retrans, + }; + task->tk_status = nfs4_async_handle_exception(task, + NFS_SERVER(inode), task->tk_status, + &exception); + hdr->retrans = exception.retrans; + if (exception.retry) { + rpc_restart_call_prepare(task); + return -EAGAIN; + } + } + if (task->tk_status >= 0) { + renew_lease(NFS_SERVER(inode), hdr->timestamp); + nfs_writeback_update_inode(hdr); + } + return 0; +} + +static bool nfs4_write_stateid_changed(struct rpc_task *task, + struct nfs_pgio_args *args) +{ + + if (!nfs4_error_stateid_expired(task->tk_status) || + nfs4_stateid_is_current(&args->stateid, + args->context, + args->lock_context, + FMODE_WRITE)) + return false; + rpc_restart_call_prepare(task); + return true; +} + +static int nfs4_write_done(struct rpc_task *task, struct nfs_pgio_header *hdr) +{ + if (!nfs4_sequence_done(task, &hdr->res.seq_res)) + return -EAGAIN; + if (nfs4_write_stateid_changed(task, &hdr->args)) + return -EAGAIN; + return hdr->pgio_done_cb ? hdr->pgio_done_cb(task, hdr) : + nfs4_write_done_cb(task, hdr); +} + +static +bool nfs4_write_need_cache_consistency_data(struct nfs_pgio_header *hdr) +{ + /* Don't request attributes for pNFS or O_DIRECT writes */ + if (hdr->ds_clp != NULL || hdr->dreq != NULL) + return false; + /* Otherwise, request attributes if and only if we don't hold + * a delegation + */ + return nfs4_have_delegation(hdr->inode, FMODE_READ, 0) == 0; +} + +void nfs4_bitmask_set(__u32 bitmask[], const __u32 src[], + struct inode *inode, unsigned long cache_validity) +{ + struct nfs_server *server = NFS_SERVER(inode); + unsigned int i; + + memcpy(bitmask, src, sizeof(*bitmask) * NFS4_BITMASK_SZ); + cache_validity |= READ_ONCE(NFS_I(inode)->cache_validity); + + if (cache_validity & NFS_INO_INVALID_CHANGE) + bitmask[0] |= FATTR4_WORD0_CHANGE; + if (cache_validity & NFS_INO_INVALID_ATIME) + bitmask[1] |= FATTR4_WORD1_TIME_ACCESS; + if (cache_validity & NFS_INO_INVALID_MODE) + bitmask[1] |= FATTR4_WORD1_MODE; + if (cache_validity & NFS_INO_INVALID_OTHER) + bitmask[1] |= FATTR4_WORD1_OWNER | FATTR4_WORD1_OWNER_GROUP; + if (cache_validity & NFS_INO_INVALID_NLINK) + bitmask[1] |= FATTR4_WORD1_NUMLINKS; + if (cache_validity & NFS_INO_INVALID_CTIME) + bitmask[1] |= FATTR4_WORD1_TIME_METADATA; + if (cache_validity & NFS_INO_INVALID_MTIME) + bitmask[1] |= FATTR4_WORD1_TIME_MODIFY; + if (cache_validity & NFS_INO_INVALID_BLOCKS) + bitmask[1] |= FATTR4_WORD1_SPACE_USED; + if (cache_validity & NFS_INO_INVALID_BTIME) + bitmask[1] |= FATTR4_WORD1_TIME_CREATE; + + if (cache_validity & NFS_INO_INVALID_SIZE) + bitmask[0] |= FATTR4_WORD0_SIZE; + + for (i = 0; i < NFS4_BITMASK_SZ; i++) + bitmask[i] &= server->attr_bitmask[i]; +} + +static void nfs4_proc_write_setup(struct nfs_pgio_header *hdr, + struct rpc_message *msg, + struct rpc_clnt **clnt) +{ + struct nfs_server *server = NFS_SERVER(hdr->inode); + + if (!nfs4_write_need_cache_consistency_data(hdr)) { + hdr->args.bitmask = NULL; + hdr->res.fattr = NULL; + } else { + nfs4_bitmask_set(hdr->args.bitmask_store, + server->cache_consistency_bitmask, + hdr->inode, NFS_INO_INVALID_BLOCKS); + hdr->args.bitmask = hdr->args.bitmask_store; + } + + if (!hdr->pgio_done_cb) + hdr->pgio_done_cb = nfs4_write_done_cb; + hdr->res.server = server; + hdr->timestamp = jiffies; + + msg->rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_WRITE]; + nfs4_init_sequence(server->nfs_client, &hdr->args.seq_args, + &hdr->res.seq_res, 0, 0); + nfs4_state_protect_write(hdr->ds_clp ? hdr->ds_clp : server->nfs_client, clnt, msg, hdr); +} + +static void nfs4_proc_commit_rpc_prepare(struct rpc_task *task, struct nfs_commit_data *data) +{ + nfs4_setup_sequence(NFS_SERVER(data->inode)->nfs_client, + &data->args.seq_args, + &data->res.seq_res, + task); +} + +static int nfs4_commit_done_cb(struct rpc_task *task, struct nfs_commit_data *data) +{ + struct inode *inode = data->inode; + + trace_nfs4_commit(data, task->tk_status); + if (nfs4_async_handle_error(task, NFS_SERVER(inode), + NULL, NULL) == -EAGAIN) { + rpc_restart_call_prepare(task); + return -EAGAIN; + } + return 0; +} + +static int nfs4_commit_done(struct rpc_task *task, struct nfs_commit_data *data) +{ + if (!nfs4_sequence_done(task, &data->res.seq_res)) + return -EAGAIN; + return data->commit_done_cb(task, data); +} + +static void nfs4_proc_commit_setup(struct nfs_commit_data *data, struct rpc_message *msg, + struct rpc_clnt **clnt) +{ + struct nfs_server *server = NFS_SERVER(data->inode); + + if (data->commit_done_cb == NULL) + data->commit_done_cb = nfs4_commit_done_cb; + data->res.server = server; + msg->rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_COMMIT]; + nfs4_init_sequence(server->nfs_client, &data->args.seq_args, + &data->res.seq_res, 1, 0); + nfs4_state_protect(data->ds_clp ? data->ds_clp : server->nfs_client, + NFS_SP4_MACH_CRED_COMMIT, clnt, msg); +} + +static int _nfs4_proc_commit(struct file *dst, struct nfs_commitargs *args, + struct nfs_commitres *res) +{ + struct inode *dst_inode = file_inode(dst); + struct nfs_server *server = NFS_SERVER(dst_inode); + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_COMMIT], + .rpc_argp = args, + .rpc_resp = res, + }; + + args->fh = NFS_FH(dst_inode); + return nfs4_call_sync(server->client, server, &msg, + &args->seq_args, &res->seq_res, 1); +} + +int nfs4_proc_commit(struct file *dst, __u64 offset, __u32 count, struct nfs_commitres *res) +{ + struct nfs_commitargs args = { + .offset = offset, + .count = count, + }; + struct nfs_server *dst_server = NFS_SERVER(file_inode(dst)); + struct nfs4_exception exception = { }; + int status; + + do { + status = _nfs4_proc_commit(dst, &args, res); + status = nfs4_handle_exception(dst_server, status, &exception); + } while (exception.retry); + + return status; +} + +static bool nfs4_server_supports_acls(const struct nfs_server *server, + enum nfs4_acl_type type) +{ + switch (type) { + default: + return server->attr_bitmask[0] & FATTR4_WORD0_ACL; + case NFS4ACL_DACL: + return server->attr_bitmask[1] & FATTR4_WORD1_DACL; + case NFS4ACL_SACL: + return server->attr_bitmask[1] & FATTR4_WORD1_SACL; + } +} + +/* Assuming that XATTR_SIZE_MAX is a multiple of PAGE_SIZE, and that + * it's OK to put sizeof(void) * (XATTR_SIZE_MAX/PAGE_SIZE) bytes on + * the stack. + */ +#define NFS4ACL_MAXPAGES DIV_ROUND_UP(XATTR_SIZE_MAX, PAGE_SIZE) + +int nfs4_buf_to_pages_noslab(const void *buf, size_t buflen, + struct page **pages) +{ + struct page *newpage, **spages; + int rc = 0; + size_t len; + spages = pages; + + do { + len = min_t(size_t, PAGE_SIZE, buflen); + newpage = alloc_page(GFP_KERNEL); + + if (newpage == NULL) + goto unwind; + memcpy(page_address(newpage), buf, len); + buf += len; + buflen -= len; + *pages++ = newpage; + rc++; + } while (buflen != 0); + + return rc; + +unwind: + for(; rc > 0; rc--) + __free_page(spages[rc-1]); + return -ENOMEM; +} + +struct nfs4_cached_acl { + enum nfs4_acl_type type; + int cached; + size_t len; + char data[]; +}; + +static void nfs4_set_cached_acl(struct inode *inode, struct nfs4_cached_acl *acl) +{ + struct nfs_inode *nfsi = NFS_I(inode); + + spin_lock(&inode->i_lock); + kfree(nfsi->nfs4_acl); + nfsi->nfs4_acl = acl; + spin_unlock(&inode->i_lock); +} + +static void nfs4_zap_acl_attr(struct inode *inode) +{ + nfs4_set_cached_acl(inode, NULL); +} + +static ssize_t nfs4_read_cached_acl(struct inode *inode, char *buf, + size_t buflen, enum nfs4_acl_type type) +{ + struct nfs_inode *nfsi = NFS_I(inode); + struct nfs4_cached_acl *acl; + int ret = -ENOENT; + + spin_lock(&inode->i_lock); + acl = nfsi->nfs4_acl; + if (acl == NULL) + goto out; + if (acl->type != type) + goto out; + if (buf == NULL) /* user is just asking for length */ + goto out_len; + if (acl->cached == 0) + goto out; + ret = -ERANGE; /* see getxattr(2) man page */ + if (acl->len > buflen) + goto out; + memcpy(buf, acl->data, acl->len); +out_len: + ret = acl->len; +out: + spin_unlock(&inode->i_lock); + return ret; +} + +static void nfs4_write_cached_acl(struct inode *inode, struct page **pages, + size_t pgbase, size_t acl_len, + enum nfs4_acl_type type) +{ + struct nfs4_cached_acl *acl; + size_t buflen = sizeof(*acl) + acl_len; + + if (buflen <= PAGE_SIZE) { + acl = kmalloc(buflen, GFP_KERNEL); + if (acl == NULL) + goto out; + acl->cached = 1; + _copy_from_pages(acl->data, pages, pgbase, acl_len); + } else { + acl = kmalloc_obj(*acl); + if (acl == NULL) + goto out; + acl->cached = 0; + } + acl->type = type; + acl->len = acl_len; +out: + nfs4_set_cached_acl(inode, acl); +} + +/* + * The getxattr API returns the required buffer length when called with a + * NULL buf. The NFSv4 acl tool then calls getxattr again after allocating + * the required buf. On a NULL buf, we send a page of data to the server + * guessing that the ACL request can be serviced by a page. If so, we cache + * up to the page of ACL data, and the 2nd call to getxattr is serviced by + * the cache. If not so, we throw away the page, and cache the required + * length. The next getxattr call will then produce another round trip to + * the server, this time with the input buf of the required size. + */ +static ssize_t __nfs4_get_acl_uncached(struct inode *inode, void *buf, + size_t buflen, enum nfs4_acl_type type) +{ + struct page **pages; + struct nfs_getaclargs args = { + .fh = NFS_FH(inode), + .acl_type = type, + .acl_len = buflen, + }; + struct nfs_getaclres res = { + .acl_type = type, + .acl_len = buflen, + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_GETACL], + .rpc_argp = &args, + .rpc_resp = &res, + }; + unsigned int npages; + int ret = -ENOMEM, i; + struct nfs_server *server = NFS_SERVER(inode); + + if (buflen == 0) + buflen = server->rsize; + + npages = DIV_ROUND_UP(buflen, PAGE_SIZE) + 1; + pages = kmalloc_objs(struct page *, npages); + if (!pages) + return -ENOMEM; + + args.acl_pages = pages; + + for (i = 0; i < npages; i++) { + pages[i] = alloc_page(GFP_KERNEL); + if (!pages[i]) + goto out_free; + } + + /* for decoding across pages */ + res.acl_scratch = folio_alloc(GFP_KERNEL, 0); + if (!res.acl_scratch) + goto out_free; + + args.acl_len = npages * PAGE_SIZE; + + dprintk("%s buf %p buflen %zu npages %d args.acl_len %zu\n", + __func__, buf, buflen, npages, args.acl_len); + ret = nfs4_call_sync(NFS_SERVER(inode)->client, NFS_SERVER(inode), + &msg, &args.seq_args, &res.seq_res, 0); + if (ret) + goto out_free; + + /* Handle the case where the passed-in buffer is too short */ + if (res.acl_flags & NFS4_ACL_TRUNC) { + /* Did the user only issue a request for the acl length? */ + if (buf == NULL) + goto out_ok; + ret = -ERANGE; + goto out_free; + } + nfs4_write_cached_acl(inode, pages, res.acl_data_offset, res.acl_len, + type); + if (buf) { + if (res.acl_len > buflen) { + ret = -ERANGE; + goto out_free; + } + _copy_from_pages(buf, pages, res.acl_data_offset, res.acl_len); + } +out_ok: + ret = res.acl_len; +out_free: + while (--i >= 0) + __free_page(pages[i]); + if (res.acl_scratch) + folio_put(res.acl_scratch); + kfree(pages); + return ret; +} + +static ssize_t nfs4_get_acl_uncached(struct inode *inode, void *buf, + size_t buflen, enum nfs4_acl_type type) +{ + struct nfs4_exception exception = { + .interruptible = true, + }; + ssize_t ret; + do { + ret = __nfs4_get_acl_uncached(inode, buf, buflen, type); + trace_nfs4_get_acl(inode, ret); + if (ret >= 0) + break; + ret = nfs4_handle_exception(NFS_SERVER(inode), ret, &exception); + } while (exception.retry); + return ret; +} + +static ssize_t nfs4_proc_get_acl(struct inode *inode, void *buf, size_t buflen, + enum nfs4_acl_type type) +{ + struct nfs_server *server = NFS_SERVER(inode); + int ret; + + if (unlikely(NFS_FH(inode)->size == 0)) + return -ENODATA; + if (!nfs4_server_supports_acls(server, type)) + return -EOPNOTSUPP; + ret = nfs_revalidate_inode(inode, NFS_INO_INVALID_CHANGE); + if (ret < 0) + return ret; + if (NFS_I(inode)->cache_validity & NFS_INO_INVALID_ACL) + nfs_zap_acl_cache(inode); + ret = nfs4_read_cached_acl(inode, buf, buflen, type); + if (ret != -ENOENT) + /* -ENOENT is returned if there is no ACL or if there is an ACL + * but no cached acl data, just the acl length */ + return ret; + return nfs4_get_acl_uncached(inode, buf, buflen, type); +} + +static int __nfs4_proc_set_acl(struct inode *inode, const void *buf, + size_t buflen, enum nfs4_acl_type type) +{ + struct nfs_server *server = NFS_SERVER(inode); + struct page *pages[NFS4ACL_MAXPAGES]; + struct nfs_setaclargs arg = { + .fh = NFS_FH(inode), + .acl_type = type, + .acl_len = buflen, + .acl_pages = pages, + }; + struct nfs_setaclres res; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_SETACL], + .rpc_argp = &arg, + .rpc_resp = &res, + }; + unsigned int npages = DIV_ROUND_UP(buflen, PAGE_SIZE); + int ret, i; + + /* You can't remove system.nfs4_acl: */ + if (buflen == 0) + return -EINVAL; + if (!nfs4_server_supports_acls(server, type)) + return -EOPNOTSUPP; + if (npages > ARRAY_SIZE(pages)) + return -ERANGE; + i = nfs4_buf_to_pages_noslab(buf, buflen, arg.acl_pages); + if (i < 0) + return i; + nfs4_inode_make_writeable(inode); + ret = nfs4_call_sync(server->client, server, &msg, &arg.seq_args, &res.seq_res, 1); + + /* + * Free each page after tx, so the only ref left is + * held by the network stack + */ + for (; i > 0; i--) + put_page(pages[i-1]); + + /* + * Acl update can result in inode attribute update. + * so mark the attribute cache invalid. + */ + spin_lock(&inode->i_lock); + nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE | + NFS_INO_INVALID_CTIME | + NFS_INO_REVAL_FORCED); + spin_unlock(&inode->i_lock); + nfs_access_zap_cache(inode); + nfs_zap_acl_cache(inode); + return ret; +} + +static int nfs4_proc_set_acl(struct inode *inode, const void *buf, + size_t buflen, enum nfs4_acl_type type) +{ + struct nfs4_exception exception = { }; + int err; + + if (unlikely(NFS_FH(inode)->size == 0)) + return -ENODATA; + do { + err = __nfs4_proc_set_acl(inode, buf, buflen, type); + trace_nfs4_set_acl(inode, err); + if (err == -NFS4ERR_BADOWNER || err == -NFS4ERR_BADNAME) { + /* + * no need to retry since the kernel + * isn't involved in encoding the ACEs. + */ + err = -EINVAL; + break; + } + err = nfs4_handle_exception(NFS_SERVER(inode), err, + &exception); + } while (exception.retry); + return err; +} + +#ifdef CONFIG_NFS_V4_SECURITY_LABEL +static int _nfs4_get_security_label(struct inode *inode, void *buf, + size_t buflen) +{ + struct nfs_server *server = NFS_SERVER(inode); + struct nfs4_label label = {0, 0, 0, buflen, buf}; + + u32 bitmask[3] = { 0, 0, FATTR4_WORD2_SECURITY_LABEL }; + struct nfs_fattr fattr = { + .label = &label, + }; + struct nfs4_getattr_arg arg = { + .fh = NFS_FH(inode), + .bitmask = bitmask, + }; + struct nfs4_getattr_res res = { + .fattr = &fattr, + .server = server, + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_GETATTR], + .rpc_argp = &arg, + .rpc_resp = &res, + }; + int ret; + + nfs_fattr_init(&fattr); + + ret = nfs4_call_sync(server->client, server, &msg, &arg.seq_args, &res.seq_res, 0); + if (ret) + return ret; + if (!(fattr.valid & NFS_ATTR_FATTR_V4_SECURITY_LABEL)) + return -ENOENT; + return label.len; +} + +static int nfs4_get_security_label(struct inode *inode, void *buf, + size_t buflen) +{ + struct nfs4_exception exception = { + .interruptible = true, + }; + int err; + + if (!nfs_server_capable(inode, NFS_CAP_SECURITY_LABEL)) + return -EOPNOTSUPP; + + do { + err = _nfs4_get_security_label(inode, buf, buflen); + trace_nfs4_get_security_label(inode, err); + err = nfs4_handle_exception(NFS_SERVER(inode), err, + &exception); + } while (exception.retry); + return err; +} + +static int _nfs4_do_set_security_label(struct inode *inode, + struct nfs4_label *ilabel, + struct nfs_fattr *fattr) +{ + + struct iattr sattr = {0}; + struct nfs_server *server = NFS_SERVER(inode); + const u32 bitmask[3] = { 0, 0, FATTR4_WORD2_SECURITY_LABEL }; + struct nfs_setattrargs arg = { + .fh = NFS_FH(inode), + .iap = &sattr, + .server = server, + .bitmask = bitmask, + .label = ilabel, + }; + struct nfs_setattrres res = { + .fattr = fattr, + .server = server, + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_SETATTR], + .rpc_argp = &arg, + .rpc_resp = &res, + }; + int status; + + nfs4_stateid_copy(&arg.stateid, &zero_stateid); + + status = nfs4_call_sync(server->client, server, &msg, &arg.seq_args, &res.seq_res, 1); + if (status) + dprintk("%s failed: %d\n", __func__, status); + + return status; +} + +static int nfs4_do_set_security_label(struct inode *inode, + struct nfs4_label *ilabel, + struct nfs_fattr *fattr) +{ + struct nfs4_exception exception = { }; + int err; + + do { + err = _nfs4_do_set_security_label(inode, ilabel, fattr); + trace_nfs4_set_security_label(inode, err); + err = nfs4_handle_exception(NFS_SERVER(inode), err, + &exception); + } while (exception.retry); + return err; +} + +static int +nfs4_set_security_label(struct inode *inode, const void *buf, size_t buflen) +{ + struct nfs4_label ilabel = {0, 0, 0, buflen, (char *)buf }; + struct nfs_fattr *fattr; + int status; + + if (!nfs_server_capable(inode, NFS_CAP_SECURITY_LABEL)) + return -EOPNOTSUPP; + + fattr = nfs_alloc_fattr_with_label(NFS_SERVER(inode)); + if (fattr == NULL) + return -ENOMEM; + + status = nfs4_do_set_security_label(inode, &ilabel, fattr); + if (status == 0) + nfs_setsecurity(inode, fattr); + + nfs_free_fattr(fattr); + return status; +} +#endif /* CONFIG_NFS_V4_SECURITY_LABEL */ + + +static void nfs4_init_boot_verifier(const struct nfs_client *clp, + nfs4_verifier *bootverf) +{ + __be32 verf[2]; + + if (test_bit(NFS4CLNT_PURGE_STATE, &clp->cl_state)) { + /* An impossible timestamp guarantees this value + * will never match a generated boot time. */ + verf[0] = cpu_to_be32(U32_MAX); + verf[1] = cpu_to_be32(U32_MAX); + } else { + struct nfs_net *nn = net_generic(clp->cl_net, nfs_net_id); + u64 ns = ktime_to_ns(nn->boot_time); + + verf[0] = cpu_to_be32(ns >> 32); + verf[1] = cpu_to_be32(ns); + } + memcpy(bootverf->data, verf, sizeof(bootverf->data)); +} + +static size_t +nfs4_get_uniquifier(struct nfs_client *clp, char *buf, size_t buflen) +{ + struct nfs_net *nn = net_generic(clp->cl_net, nfs_net_id); + struct nfs_netns_client *nn_clp = nn->nfs_client; + const char *id; + + buf[0] = '\0'; + + if (nn_clp) { + rcu_read_lock(); + id = rcu_dereference(nn_clp->identifier); + if (id) + strscpy(buf, id, buflen); + rcu_read_unlock(); + } + + if (nfs4_client_id_uniquifier[0] != '\0' && buf[0] == '\0') + strscpy(buf, nfs4_client_id_uniquifier, buflen); + + return strlen(buf); +} + +static int +nfs4_init_nonuniform_client_string(struct nfs_client *clp) +{ + char buf[NFS4_CLIENT_ID_UNIQ_LEN]; + size_t buflen; + size_t len; + char *str; + + if (clp->cl_owner_id != NULL) + return 0; + + rcu_read_lock(); + len = 14 + + strlen(clp->cl_rpcclient->cl_nodename) + + 1 + + strlen(rpc_peeraddr2str(clp->cl_rpcclient, RPC_DISPLAY_ADDR)) + + 1; + rcu_read_unlock(); + + buflen = nfs4_get_uniquifier(clp, buf, sizeof(buf)); + if (buflen) + len += buflen + 1; + + if (len > NFS4_OPAQUE_LIMIT + 1) + return -EINVAL; + + /* + * Since this string is allocated at mount time, and held until the + * nfs_client is destroyed, we can use GFP_KERNEL here w/o worrying + * about a memory-reclaim deadlock. + */ + str = kmalloc(len, GFP_KERNEL); + if (!str) + return -ENOMEM; + + rcu_read_lock(); + if (buflen) + scnprintf(str, len, "Linux NFSv4.0 %s/%s/%s", + clp->cl_rpcclient->cl_nodename, buf, + rpc_peeraddr2str(clp->cl_rpcclient, + RPC_DISPLAY_ADDR)); + else + scnprintf(str, len, "Linux NFSv4.0 %s/%s", + clp->cl_rpcclient->cl_nodename, + rpc_peeraddr2str(clp->cl_rpcclient, + RPC_DISPLAY_ADDR)); + rcu_read_unlock(); + + clp->cl_owner_id = str; + return 0; +} + +static int +nfs4_init_uniform_client_string(struct nfs_client *clp) +{ + char buf[NFS4_CLIENT_ID_UNIQ_LEN]; + size_t buflen; + size_t len; + char *str; + + if (clp->cl_owner_id != NULL) + return 0; + + len = 10 + 10 + 1 + 10 + 1 + + strlen(clp->cl_rpcclient->cl_nodename) + 1; + + buflen = nfs4_get_uniquifier(clp, buf, sizeof(buf)); + if (buflen) + len += buflen + 1; + + if (len > NFS4_OPAQUE_LIMIT + 1) + return -EINVAL; + + /* + * Since this string is allocated at mount time, and held until the + * nfs_client is destroyed, we can use GFP_KERNEL here w/o worrying + * about a memory-reclaim deadlock. + */ + str = kmalloc(len, GFP_KERNEL); + if (!str) + return -ENOMEM; + + if (buflen) + scnprintf(str, len, "Linux NFSv%u.%u %s/%s", + clp->rpc_ops->version, clp->cl_minorversion, + buf, clp->cl_rpcclient->cl_nodename); + else + scnprintf(str, len, "Linux NFSv%u.%u %s", + clp->rpc_ops->version, clp->cl_minorversion, + clp->cl_rpcclient->cl_nodename); + clp->cl_owner_id = str; + return 0; +} + +/* + * nfs4_callback_up_net() starts only "tcp" and "tcp6" callback + * services. Advertise one based on the address family of the + * clientaddr. + */ +static unsigned int +nfs4_init_callback_netid(const struct nfs_client *clp, char *buf, size_t len) +{ + if (strchr(clp->cl_ipaddr, ':') != NULL) + return scnprintf(buf, len, "tcp6"); + else + return scnprintf(buf, len, "tcp"); +} + +static void nfs4_setclientid_done(struct rpc_task *task, void *calldata) +{ + struct nfs4_setclientid *sc = calldata; + + if (task->tk_status == 0) + sc->sc_cred = get_rpccred(task->tk_rqstp->rq_cred); +} + +static const struct rpc_call_ops nfs4_setclientid_ops = { + .rpc_call_done = nfs4_setclientid_done, +}; + +/** + * nfs4_proc_setclientid - Negotiate client ID + * @clp: state data structure + * @program: RPC program for NFSv4 callback service + * @port: IP port number for NFS4 callback service + * @cred: credential to use for this call + * @res: where to place the result + * + * Returns zero, a negative errno, or a negative NFS4ERR status code. + */ +int nfs4_proc_setclientid(struct nfs_client *clp, u32 program, + unsigned short port, const struct cred *cred, + struct nfs4_setclientid_res *res) +{ + nfs4_verifier sc_verifier; + struct nfs4_setclientid setclientid = { + .sc_verifier = &sc_verifier, + .sc_prog = program, + .sc_clnt = clp, + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_SETCLIENTID], + .rpc_argp = &setclientid, + .rpc_resp = res, + .rpc_cred = cred, + }; + struct rpc_task_setup task_setup_data = { + .rpc_client = clp->cl_rpcclient, + .rpc_message = &msg, + .callback_ops = &nfs4_setclientid_ops, + .callback_data = &setclientid, + .flags = RPC_TASK_TIMEOUT | RPC_TASK_NO_ROUND_ROBIN, + }; + unsigned long now = jiffies; + int status; + + /* nfs_client_id4 */ + nfs4_init_boot_verifier(clp, &sc_verifier); + + if (test_bit(NFS_CS_MIGRATION, &clp->cl_flags)) + status = nfs4_init_uniform_client_string(clp); + else + status = nfs4_init_nonuniform_client_string(clp); + + if (status) + goto out; + + /* cb_client4 */ + setclientid.sc_netid_len = + nfs4_init_callback_netid(clp, + setclientid.sc_netid, + sizeof(setclientid.sc_netid)); + setclientid.sc_uaddr_len = scnprintf(setclientid.sc_uaddr, + sizeof(setclientid.sc_uaddr), "%s.%u.%u", + clp->cl_ipaddr, port >> 8, port & 255); + + dprintk("NFS call setclientid auth=%s, '%s'\n", + clp->cl_rpcclient->cl_auth->au_ops->au_name, + clp->cl_owner_id); + + status = nfs4_call_sync_custom(&task_setup_data); + if (setclientid.sc_cred) { + kfree(clp->cl_acceptor); + clp->cl_acceptor = rpcauth_stringify_acceptor(setclientid.sc_cred); + put_rpccred(setclientid.sc_cred); + } + + if (status == 0) + do_renew_lease(clp, now); +out: + trace_nfs4_setclientid(clp, status); + dprintk("NFS reply setclientid: %d\n", status); + return status; +} + +/** + * nfs4_proc_setclientid_confirm - Confirm client ID + * @clp: state data structure + * @arg: result of a previous SETCLIENTID + * @cred: credential to use for this call + * + * Returns zero, a negative errno, or a negative NFS4ERR status code. + */ +int nfs4_proc_setclientid_confirm(struct nfs_client *clp, + struct nfs4_setclientid_res *arg, + const struct cred *cred) +{ + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_SETCLIENTID_CONFIRM], + .rpc_argp = arg, + .rpc_cred = cred, + }; + int status; + + dprintk("NFS call setclientid_confirm auth=%s, (client ID %llx)\n", + clp->cl_rpcclient->cl_auth->au_ops->au_name, + clp->cl_clientid); + status = rpc_call_sync(clp->cl_rpcclient, &msg, + RPC_TASK_TIMEOUT | RPC_TASK_NO_ROUND_ROBIN); + trace_nfs4_setclientid_confirm(clp, status); + dprintk("NFS reply setclientid_confirm: %d\n", status); + return status; +} + +struct nfs4_delegreturndata { + struct nfs4_delegreturnargs args; + struct nfs4_delegreturnres res; + struct nfs_fh fh; + nfs4_stateid stateid; + unsigned long timestamp; + unsigned short retrans; + struct { + struct nfs4_layoutreturn_args arg; + struct nfs4_layoutreturn_res res; + struct nfs4_xdr_opaque_data ld_private; + u32 roc_barrier; + bool roc; + } lr; + struct nfs4_delegattr sattr; + struct nfs_fattr fattr; + int rpc_status; + struct inode *inode; +}; + +static void nfs4_delegreturn_done(struct rpc_task *task, void *calldata) +{ + struct nfs4_delegreturndata *data = calldata; + struct nfs4_exception exception = { + .inode = data->inode, + .stateid = &data->stateid, + .task_is_privileged = data->args.seq_args.sa_privileged, + .retrans = data->retrans, + }; + + if (!nfs4_sequence_done(task, &data->res.seq_res)) + return; + + trace_nfs4_delegreturn_exit(&data->args, &data->res, task->tk_status); + + /* Handle Layoutreturn errors */ + if (pnfs_roc_done(task, &data->args.lr_args, &data->res.lr_res, + &data->res.lr_ret) == -EAGAIN) + goto out_restart; + + if (data->args.sattr_args && task->tk_status != 0) { + switch(data->res.sattr_ret) { + case 0: + data->args.sattr_args = NULL; + data->res.sattr_res = false; + break; + case -NFS4ERR_ADMIN_REVOKED: + case -NFS4ERR_DELEG_REVOKED: + case -NFS4ERR_EXPIRED: + case -NFS4ERR_BAD_STATEID: + /* Let the main handler below do stateid recovery */ + break; + case -NFS4ERR_OLD_STATEID: + if (nfs4_refresh_delegation_stateid(&data->stateid, + data->inode)) + goto out_restart; + fallthrough; + default: + data->args.sattr_args = NULL; + data->res.sattr_res = false; + goto out_restart; + } + } + + switch (task->tk_status) { + case 0: + renew_lease(data->res.server, data->timestamp); + break; + case -NFS4ERR_ADMIN_REVOKED: + case -NFS4ERR_DELEG_REVOKED: + case -NFS4ERR_EXPIRED: + nfs4_free_revoked_stateid(data->res.server, + data->args.stateid, + task->tk_msg.rpc_cred); + fallthrough; + case -NFS4ERR_BAD_STATEID: + case -NFS4ERR_STALE_STATEID: + case -ETIMEDOUT: + task->tk_status = 0; + break; + case -NFS4ERR_OLD_STATEID: + if (!nfs4_refresh_delegation_stateid(&data->stateid, data->inode)) + nfs4_stateid_seqid_inc(&data->stateid); + if (data->args.bitmask) { + data->args.bitmask = NULL; + data->res.fattr = NULL; + } + goto out_restart; + case -NFS4ERR_ACCESS: + if (data->args.bitmask) { + data->args.bitmask = NULL; + data->res.fattr = NULL; + goto out_restart; + } + fallthrough; + default: + task->tk_status = nfs4_async_handle_exception(task, + data->res.server, task->tk_status, + &exception); + data->retrans = exception.retrans; + if (exception.retry) + goto out_restart; + } + nfs_delegation_mark_returned(data->inode, data->args.stateid); + data->rpc_status = task->tk_status; + return; +out_restart: + task->tk_status = 0; + rpc_restart_call_prepare(task); +} + +static void nfs4_delegreturn_release(void *calldata) +{ + struct nfs4_delegreturndata *data = calldata; + struct inode *inode = data->inode; + + if (data->lr.roc) + pnfs_roc_release(&data->lr.arg, &data->lr.res, + data->res.lr_ret); + if (inode) { + nfs4_fattr_set_prechange(&data->fattr, + inode_peek_iversion_raw(inode)); + nfs_refresh_inode(inode, &data->fattr); + nfs_iput_and_deactive(inode); + } + kfree(calldata); +} + +static void nfs4_delegreturn_prepare(struct rpc_task *task, void *data) +{ + struct nfs4_delegreturndata *d_data; + struct pnfs_layout_hdr *lo; + + d_data = data; + + if (!d_data->lr.roc && nfs4_wait_on_layoutreturn(d_data->inode, task)) { + nfs4_sequence_done(task, &d_data->res.seq_res); + return; + } + + lo = d_data->args.lr_args ? d_data->args.lr_args->layout : NULL; + if (lo && !pnfs_layout_is_valid(lo)) { + d_data->args.lr_args = NULL; + d_data->res.lr_res = NULL; + } + + nfs4_setup_sequence(d_data->res.server->nfs_client, + &d_data->args.seq_args, + &d_data->res.seq_res, + task); +} + +static const struct rpc_call_ops nfs4_delegreturn_ops = { + .rpc_call_prepare = nfs4_delegreturn_prepare, + .rpc_call_done = nfs4_delegreturn_done, + .rpc_release = nfs4_delegreturn_release, +}; + +static int _nfs4_proc_delegreturn(struct inode *inode, const struct cred *cred, + const nfs4_stateid *stateid, + struct nfs_delegation *delegation, + int issync) +{ + struct nfs4_delegreturndata *data; + struct nfs_server *server = NFS_SERVER(inode); + struct rpc_task *task; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_DELEGRETURN], + .rpc_cred = cred, + }; + struct rpc_task_setup task_setup_data = { + .rpc_client = server->client, + .rpc_message = &msg, + .callback_ops = &nfs4_delegreturn_ops, + .flags = RPC_TASK_ASYNC | RPC_TASK_TIMEOUT, + }; + int status = 0; + + if (nfs_server_capable(inode, NFS_CAP_MOVEABLE)) + task_setup_data.flags |= RPC_TASK_MOVEABLE; + + data = kzalloc_obj(*data); + if (data == NULL) + return -ENOMEM; + + nfs4_state_protect(server->nfs_client, + NFS_SP4_MACH_CRED_CLEANUP, + &task_setup_data.rpc_client, &msg); + + data->args.fhandle = &data->fh; + data->args.stateid = &data->stateid; + nfs4_bitmask_set(data->args.bitmask_store, + server->cache_consistency_bitmask, inode, 0); + data->args.bitmask = data->args.bitmask_store; + nfs_copy_fh(&data->fh, NFS_FH(inode)); + nfs4_stateid_copy(&data->stateid, stateid); + data->res.fattr = &data->fattr; + data->res.server = server; + data->res.lr_ret = -NFS4ERR_NOMATCHING_LAYOUT; + data->lr.arg.ld_private = &data->lr.ld_private; + nfs_fattr_init(data->res.fattr); + data->timestamp = jiffies; + data->rpc_status = 0; + data->inode = nfs_igrab_and_active(inode); + if (data->inode || issync) { + data->lr.roc = pnfs_roc(inode, &data->lr.arg, &data->lr.res, + cred, issync); + if (data->lr.roc) { + data->args.lr_args = &data->lr.arg; + data->res.lr_res = &data->lr.res; + } + } + + if (delegation && + test_bit(NFS_DELEGATION_DELEGTIME, &delegation->flags)) { + if (delegation->type & FMODE_READ) { + data->sattr.atime = inode_get_atime(inode); + data->sattr.atime_set = true; + } + if (delegation->type & FMODE_WRITE) { + data->sattr.mtime = inode_get_mtime(inode); + data->sattr.mtime_set = true; + } + data->args.sattr_args = &data->sattr; + data->res.sattr_res = true; + } + + nfs4_init_sequence(server->nfs_client, &data->args.seq_args, + &data->res.seq_res, 1, !data->inode ? 1 : 0); + + task_setup_data.callback_data = data; + msg.rpc_argp = &data->args; + msg.rpc_resp = &data->res; + task = rpc_run_task(&task_setup_data); + if (IS_ERR(task)) + return PTR_ERR(task); + if (!issync) + goto out; + status = rpc_wait_for_completion_task(task); + if (status != 0) + goto out; + status = data->rpc_status; +out: + rpc_put_task(task); + return status; +} + +int nfs4_proc_delegreturn(struct inode *inode, const struct cred *cred, + const nfs4_stateid *stateid, + struct nfs_delegation *delegation, int issync) +{ + struct nfs_server *server = NFS_SERVER(inode); + struct nfs4_exception exception = { }; + int err; + do { + err = _nfs4_proc_delegreturn(inode, cred, stateid, + delegation, issync); + trace_nfs4_delegreturn(inode, stateid, err); + switch (err) { + case -NFS4ERR_STALE_STATEID: + case -NFS4ERR_EXPIRED: + case 0: + return 0; + } + err = nfs4_handle_exception(server, err, &exception); + } while (exception.retry); + return err; +} + +static int _nfs4_proc_getlk(struct nfs4_state *state, int cmd, struct file_lock *request) +{ + struct inode *inode = state->inode; + struct nfs_server *server = NFS_SERVER(inode); + struct nfs_client *clp = server->nfs_client; + struct nfs_lockt_args arg = { + .fh = NFS_FH(inode), + .fl = request, + }; + struct nfs_lockt_res res = { + .denied = request, + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_LOCKT], + .rpc_argp = &arg, + .rpc_resp = &res, + .rpc_cred = state->owner->so_cred, + }; + struct nfs4_lock_state *lsp; + int status; + + arg.lock_owner.clientid = clp->cl_clientid; + status = nfs4_set_lock_state(state, request); + if (status != 0) + goto out; + lsp = request->fl_u.nfs4_fl.owner; + arg.lock_owner.id = lsp->ls_seqid.owner_id; + arg.lock_owner.s_dev = server->s_dev; + status = nfs4_call_sync(server->client, server, &msg, &arg.seq_args, &res.seq_res, 1); + switch (status) { + case 0: + request->c.flc_type = F_UNLCK; + break; + case -NFS4ERR_DENIED: + status = 0; + } + request->fl_ops->fl_release_private(request); + request->fl_ops = NULL; +out: + return status; +} + +static int nfs4_proc_getlk(struct nfs4_state *state, int cmd, struct file_lock *request) +{ + struct nfs4_exception exception = { + .interruptible = true, + }; + int err; + + do { + err = _nfs4_proc_getlk(state, cmd, request); + trace_nfs4_get_lock(request, state, cmd, err); + err = nfs4_handle_exception(NFS_SERVER(state->inode), err, + &exception); + } while (exception.retry); + return err; +} + +/* + * Update the seqid of a lock stateid after receiving + * NFS4ERR_OLD_STATEID + */ +static bool nfs4_refresh_lock_old_stateid(nfs4_stateid *dst, + struct nfs4_lock_state *lsp) +{ + struct nfs4_state *state = lsp->ls_state; + bool ret = false; + + spin_lock(&state->state_lock); + if (!nfs4_stateid_match_other(dst, &lsp->ls_stateid)) + goto out; + if (!nfs4_stateid_is_newer(&lsp->ls_stateid, dst)) + nfs4_stateid_seqid_inc(dst); + else + dst->seqid = lsp->ls_stateid.seqid; + ret = true; +out: + spin_unlock(&state->state_lock); + return ret; +} + +static bool nfs4_sync_lock_stateid(nfs4_stateid *dst, + struct nfs4_lock_state *lsp) +{ + struct nfs4_state *state = lsp->ls_state; + bool ret; + + spin_lock(&state->state_lock); + ret = !nfs4_stateid_match_other(dst, &lsp->ls_stateid); + nfs4_stateid_copy(dst, &lsp->ls_stateid); + spin_unlock(&state->state_lock); + return ret; +} + +struct nfs4_unlockdata { + struct nfs_locku_args arg; + struct nfs_locku_res res; + struct nfs4_lock_state *lsp; + struct nfs_open_context *ctx; + struct nfs_lock_context *l_ctx; + struct file_lock fl; + struct nfs_server *server; + unsigned long timestamp; + unsigned short retrans; +}; + +static struct nfs4_unlockdata *nfs4_alloc_unlockdata(struct file_lock *fl, + struct nfs_open_context *ctx, + struct nfs4_lock_state *lsp, + struct nfs_seqid *seqid) +{ + struct nfs4_unlockdata *p; + struct nfs4_state *state = lsp->ls_state; + struct inode *inode = state->inode; + struct nfs_lock_context *l_ctx; + + p = kzalloc_obj(*p); + if (p == NULL) + return NULL; + l_ctx = nfs_get_lock_context(ctx); + if (!IS_ERR(l_ctx)) { + p->l_ctx = l_ctx; + } else { + kfree(p); + return NULL; + } + p->arg.fh = NFS_FH(inode); + p->arg.fl = &p->fl; + p->arg.seqid = seqid; + p->res.seqid = seqid; + p->lsp = lsp; + /* Ensure we don't close file until we're done freeing locks! */ + p->ctx = get_nfs_open_context(ctx); + locks_init_lock(&p->fl); + locks_copy_lock(&p->fl, fl); + p->server = NFS_SERVER(inode); + spin_lock(&state->state_lock); + nfs4_stateid_copy(&p->arg.stateid, &lsp->ls_stateid); + spin_unlock(&state->state_lock); + return p; +} + +static void nfs4_locku_release_calldata(void *data) +{ + struct nfs4_unlockdata *calldata = data; + nfs_free_seqid(calldata->arg.seqid); + nfs4_put_lock_state(calldata->lsp); + nfs_put_lock_context(calldata->l_ctx); + put_nfs_open_context(calldata->ctx); + kfree(calldata); +} + +static void nfs4_locku_done(struct rpc_task *task, void *data) +{ + struct nfs4_unlockdata *calldata = data; + struct nfs4_exception exception = { + .inode = calldata->lsp->ls_state->inode, + .stateid = &calldata->arg.stateid, + .retrans = calldata->retrans, + }; + + if (!nfs4_sequence_done(task, &calldata->res.seq_res)) + return; + switch (task->tk_status) { + case 0: + renew_lease(calldata->server, calldata->timestamp); + if (nfs4_update_lock_stateid(calldata->lsp, + &calldata->res.stateid)) + break; + fallthrough; + case -NFS4ERR_ADMIN_REVOKED: + case -NFS4ERR_EXPIRED: + nfs4_free_revoked_stateid(calldata->server, + &calldata->arg.stateid, + task->tk_msg.rpc_cred); + fallthrough; + case -NFS4ERR_BAD_STATEID: + case -NFS4ERR_STALE_STATEID: + if (nfs4_sync_lock_stateid(&calldata->arg.stateid, + calldata->lsp)) + rpc_restart_call_prepare(task); + break; + case -NFS4ERR_OLD_STATEID: + if (nfs4_refresh_lock_old_stateid(&calldata->arg.stateid, + calldata->lsp)) + rpc_restart_call_prepare(task); + break; + default: + task->tk_status = nfs4_async_handle_exception(task, + calldata->server, task->tk_status, + &exception); + calldata->retrans = exception.retrans; + if (exception.retry) + rpc_restart_call_prepare(task); + } + nfs_release_seqid(calldata->arg.seqid); +} + +static void nfs4_locku_prepare(struct rpc_task *task, void *data) +{ + struct nfs4_unlockdata *calldata = data; + + if (test_bit(NFS_CONTEXT_UNLOCK, &calldata->l_ctx->open_context->flags) && + nfs_async_iocounter_wait(task, calldata->l_ctx)) + return; + + if (nfs_wait_on_sequence(calldata->arg.seqid, task) != 0) + goto out_wait; + if (test_bit(NFS_LOCK_INITIALIZED, &calldata->lsp->ls_flags) == 0) { + /* Note: exit _without_ running nfs4_locku_done */ + goto out_no_action; + } + calldata->timestamp = jiffies; + if (nfs4_setup_sequence(calldata->server->nfs_client, + &calldata->arg.seq_args, + &calldata->res.seq_res, + task) != 0) + nfs_release_seqid(calldata->arg.seqid); + return; +out_no_action: + task->tk_action = NULL; +out_wait: + nfs4_sequence_done(task, &calldata->res.seq_res); +} + +static const struct rpc_call_ops nfs4_locku_ops = { + .rpc_call_prepare = nfs4_locku_prepare, + .rpc_call_done = nfs4_locku_done, + .rpc_release = nfs4_locku_release_calldata, +}; + +static struct rpc_task *nfs4_do_unlck(struct file_lock *fl, + struct nfs_open_context *ctx, + struct nfs4_lock_state *lsp, + struct nfs_seqid *seqid) +{ + struct nfs4_unlockdata *data; + struct nfs_client *clp = NFS_SERVER(lsp->ls_state->inode)->nfs_client; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_LOCKU], + .rpc_cred = ctx->cred, + }; + struct rpc_task_setup task_setup_data = { + .rpc_client = NFS_CLIENT(lsp->ls_state->inode), + .rpc_message = &msg, + .callback_ops = &nfs4_locku_ops, + .workqueue = nfsiod_workqueue, + .flags = RPC_TASK_ASYNC, + }; + + if (nfs_server_capable(lsp->ls_state->inode, NFS_CAP_MOVEABLE)) + task_setup_data.flags |= RPC_TASK_MOVEABLE; + + nfs4_state_protect(clp, NFS_SP4_MACH_CRED_CLEANUP, + &task_setup_data.rpc_client, &msg); + + /* Ensure this is an unlock - when canceling a lock, the + * canceled lock is passed in, and it won't be an unlock. + */ + fl->c.flc_type = F_UNLCK; + if (fl->c.flc_flags & FL_CLOSE) + set_bit(NFS_CONTEXT_UNLOCK, &ctx->flags); + + data = nfs4_alloc_unlockdata(fl, ctx, lsp, seqid); + if (data == NULL) { + nfs_free_seqid(seqid); + return ERR_PTR(-ENOMEM); + } + + nfs4_init_sequence(clp, &data->arg.seq_args, &data->res.seq_res, 1, 0); + msg.rpc_argp = &data->arg; + msg.rpc_resp = &data->res; + task_setup_data.callback_data = data; + return rpc_run_task(&task_setup_data); +} + +static int nfs4_proc_unlck(struct nfs4_state *state, int cmd, struct file_lock *request) +{ + struct inode *inode = state->inode; + struct nfs4_state_owner *sp = state->owner; + struct nfs_inode *nfsi = NFS_I(inode); + struct nfs_seqid *seqid; + struct nfs4_lock_state *lsp; + struct rpc_task *task; + struct nfs_seqid *(*alloc_seqid)(struct nfs_seqid_counter *, gfp_t); + int status = 0; + unsigned char saved_flags = request->c.flc_flags; + + status = nfs4_set_lock_state(state, request); + /* Unlock _before_ we do the RPC call */ + request->c.flc_flags |= FL_EXISTS; + /* Exclude nfs_delegation_claim_locks() */ + mutex_lock(&sp->so_delegreturn_mutex); + /* Exclude nfs4_reclaim_open_stateid() - note nesting! */ + down_read(&nfsi->rwsem); + if (locks_lock_inode_wait(inode, request) == -ENOENT) { + up_read(&nfsi->rwsem); + mutex_unlock(&sp->so_delegreturn_mutex); + goto out; + } + lsp = request->fl_u.nfs4_fl.owner; + set_bit(NFS_LOCK_UNLOCKING, &lsp->ls_flags); + up_read(&nfsi->rwsem); + mutex_unlock(&sp->so_delegreturn_mutex); + if (status != 0) + goto out; + /* Is this a delegated lock? */ + if (test_bit(NFS_LOCK_INITIALIZED, &lsp->ls_flags) == 0) + goto out; + alloc_seqid = NFS_SERVER(inode)->nfs_client->cl_mvops->alloc_seqid; + seqid = alloc_seqid(&lsp->ls_seqid, GFP_KERNEL); + status = -ENOMEM; + if (IS_ERR(seqid)) + goto out; + task = nfs4_do_unlck(request, + nfs_file_open_context(request->c.flc_file), + lsp, seqid); + status = PTR_ERR(task); + if (IS_ERR(task)) + goto out; + status = rpc_wait_for_completion_task(task); + rpc_put_task(task); +out: + request->c.flc_flags = saved_flags; + trace_nfs4_unlock(request, state, F_SETLK, status); + return status; +} + +struct nfs4_lockdata { + struct nfs_lock_args arg; + struct nfs_lock_res res; + struct nfs4_lock_state *lsp; + struct nfs_open_context *ctx; + struct file_lock fl; + unsigned long timestamp; + int rpc_status; + int cancelled; + struct nfs_server *server; +}; + +static struct nfs4_lockdata *nfs4_alloc_lockdata(struct file_lock *fl, + struct nfs_open_context *ctx, struct nfs4_lock_state *lsp, + gfp_t gfp_mask) +{ + struct nfs4_lockdata *p; + struct inode *inode = lsp->ls_state->inode; + struct nfs_server *server = NFS_SERVER(inode); + struct nfs_seqid *(*alloc_seqid)(struct nfs_seqid_counter *, gfp_t); + + p = kzalloc_obj(*p, gfp_mask); + if (p == NULL) + return NULL; + + p->arg.fh = NFS_FH(inode); + p->arg.fl = &p->fl; + p->arg.open_seqid = nfs_alloc_seqid(&lsp->ls_state->owner->so_seqid, gfp_mask); + if (IS_ERR(p->arg.open_seqid)) + goto out_free; + alloc_seqid = server->nfs_client->cl_mvops->alloc_seqid; + p->arg.lock_seqid = alloc_seqid(&lsp->ls_seqid, gfp_mask); + if (IS_ERR(p->arg.lock_seqid)) + goto out_free_seqid; + p->arg.lock_owner.clientid = server->nfs_client->cl_clientid; + p->arg.lock_owner.id = lsp->ls_seqid.owner_id; + p->arg.lock_owner.s_dev = server->s_dev; + p->res.lock_seqid = p->arg.lock_seqid; + p->lsp = lsp; + p->server = server; + p->ctx = get_nfs_open_context(ctx); + locks_init_lock(&p->fl); + locks_copy_lock(&p->fl, fl); + return p; +out_free_seqid: + nfs_free_seqid(p->arg.open_seqid); +out_free: + kfree(p); + return NULL; +} + +static void nfs4_lock_prepare(struct rpc_task *task, void *calldata) +{ + struct nfs4_lockdata *data = calldata; + struct nfs4_state *state = data->lsp->ls_state; + + if (nfs_wait_on_sequence(data->arg.lock_seqid, task) != 0) + goto out_wait; + /* Do we need to do an open_to_lock_owner? */ + if (!test_bit(NFS_LOCK_INITIALIZED, &data->lsp->ls_flags)) { + if (nfs_wait_on_sequence(data->arg.open_seqid, task) != 0) { + goto out_release_lock_seqid; + } + nfs4_stateid_copy(&data->arg.open_stateid, + &state->open_stateid); + data->arg.new_lock_owner = 1; + data->res.open_seqid = data->arg.open_seqid; + } else { + data->arg.new_lock_owner = 0; + nfs4_stateid_copy(&data->arg.lock_stateid, + &data->lsp->ls_stateid); + } + if (!nfs4_valid_open_stateid(state)) { + data->rpc_status = -EBADF; + task->tk_action = NULL; + goto out_release_open_seqid; + } + data->timestamp = jiffies; + if (nfs4_setup_sequence(data->server->nfs_client, + &data->arg.seq_args, + &data->res.seq_res, + task) == 0) + return; +out_release_open_seqid: + nfs_release_seqid(data->arg.open_seqid); +out_release_lock_seqid: + nfs_release_seqid(data->arg.lock_seqid); +out_wait: + nfs4_sequence_done(task, &data->res.seq_res); + dprintk("%s: ret = %d\n", __func__, data->rpc_status); +} + +static void nfs4_lock_done(struct rpc_task *task, void *calldata) +{ + struct nfs4_lockdata *data = calldata; + struct nfs4_lock_state *lsp = data->lsp; + + if (!nfs4_sequence_done(task, &data->res.seq_res)) + return; + + data->rpc_status = task->tk_status; + switch (task->tk_status) { + case 0: + renew_lease(NFS_SERVER(d_inode(data->ctx->dentry)), + data->timestamp); + if (data->arg.new_lock_owner != 0) { + nfs_confirm_seqid(&lsp->ls_seqid, 0); + nfs4_stateid_copy(&lsp->ls_stateid, &data->res.stateid); + set_bit(NFS_LOCK_INITIALIZED, &lsp->ls_flags); + } else if (!nfs4_update_lock_stateid(lsp, &data->res.stateid)) + goto out_restart; + break; + case -NFS4ERR_OLD_STATEID: + if (data->arg.new_lock_owner != 0 && + nfs4_refresh_open_old_stateid(&data->arg.open_stateid, + lsp->ls_state)) + goto out_restart; + if (nfs4_refresh_lock_old_stateid(&data->arg.lock_stateid, lsp)) + goto out_restart; + fallthrough; + case -NFS4ERR_BAD_STATEID: + case -NFS4ERR_STALE_STATEID: + case -NFS4ERR_EXPIRED: + if (data->arg.new_lock_owner != 0) { + if (!nfs4_stateid_match(&data->arg.open_stateid, + &lsp->ls_state->open_stateid)) + goto out_restart; + } else if (!nfs4_stateid_match(&data->arg.lock_stateid, + &lsp->ls_stateid)) + goto out_restart; + } +out_done: + dprintk("%s: ret = %d!\n", __func__, data->rpc_status); + return; +out_restart: + if (!data->cancelled) + rpc_restart_call_prepare(task); + goto out_done; +} + +static void nfs4_lock_release(void *calldata) +{ + struct nfs4_lockdata *data = calldata; + + nfs_free_seqid(data->arg.open_seqid); + if (data->cancelled && data->rpc_status == 0) { + struct rpc_task *task; + task = nfs4_do_unlck(&data->fl, data->ctx, data->lsp, + data->arg.lock_seqid); + if (!IS_ERR(task)) + rpc_put_task_async(task); + dprintk("%s: cancelling lock!\n", __func__); + } else + nfs_free_seqid(data->arg.lock_seqid); + nfs4_put_lock_state(data->lsp); + put_nfs_open_context(data->ctx); + kfree(data); +} + +static const struct rpc_call_ops nfs4_lock_ops = { + .rpc_call_prepare = nfs4_lock_prepare, + .rpc_call_done = nfs4_lock_done, + .rpc_release = nfs4_lock_release, +}; + +static void nfs4_handle_setlk_error(struct nfs_server *server, struct nfs4_lock_state *lsp, int new_lock_owner, int error) +{ + switch (error) { + case -NFS4ERR_ADMIN_REVOKED: + case -NFS4ERR_EXPIRED: + case -NFS4ERR_BAD_STATEID: + lsp->ls_seqid.flags &= ~NFS_SEQID_CONFIRMED; + if (new_lock_owner != 0 || + test_bit(NFS_LOCK_INITIALIZED, &lsp->ls_flags) != 0) + nfs4_schedule_stateid_recovery(server, lsp->ls_state); + break; + case -NFS4ERR_STALE_STATEID: + lsp->ls_seqid.flags &= ~NFS_SEQID_CONFIRMED; + nfs4_schedule_lease_recovery(server->nfs_client); + } +} + +static int _nfs4_do_setlk(struct nfs4_state *state, int cmd, struct file_lock *fl, int recovery_type) +{ + struct nfs4_lockdata *data; + struct rpc_task *task; + struct nfs_client *clp = NFS_SERVER(state->inode)->nfs_client; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_LOCK], + .rpc_cred = state->owner->so_cred, + }; + struct rpc_task_setup task_setup_data = { + .rpc_client = NFS_CLIENT(state->inode), + .rpc_message = &msg, + .callback_ops = &nfs4_lock_ops, + .workqueue = nfsiod_workqueue, + .flags = RPC_TASK_ASYNC | RPC_TASK_CRED_NOREF, + }; + int ret; + + if (nfs_server_capable(state->inode, NFS_CAP_MOVEABLE)) + task_setup_data.flags |= RPC_TASK_MOVEABLE; + + data = nfs4_alloc_lockdata(fl, + nfs_file_open_context(fl->c.flc_file), + fl->fl_u.nfs4_fl.owner, GFP_KERNEL); + if (data == NULL) + return -ENOMEM; + if (IS_SETLKW(cmd)) + data->arg.block = 1; + nfs4_init_sequence(clp, &data->arg.seq_args, &data->res.seq_res, 1, + recovery_type > NFS_LOCK_NEW); + msg.rpc_argp = &data->arg; + msg.rpc_resp = &data->res; + task_setup_data.callback_data = data; + + if (recovery_type == NFS_LOCK_RECLAIM) + data->arg.reclaim = NFS_LOCK_RECLAIM; + + task = rpc_run_task(&task_setup_data); + if (IS_ERR(task)) + return PTR_ERR(task); + ret = rpc_wait_for_completion_task(task); + if (ret == 0) { + ret = data->rpc_status; + if (ret) + nfs4_handle_setlk_error(data->server, data->lsp, + data->arg.new_lock_owner, ret); + } else + data->cancelled = true; + trace_nfs4_set_lock(fl, state, &data->res.stateid, cmd, ret); + rpc_put_task(task); + dprintk("%s: ret = %d\n", __func__, ret); + return ret; +} + +int nfs4_lock_reclaim(struct nfs4_state *state, struct file_lock *request) +{ + struct nfs_server *server = NFS_SERVER(state->inode); + struct nfs4_exception exception = { + .inode = state->inode, + }; + int err; + + do { + /* Cache the lock if possible... */ + if (test_bit(NFS_DELEGATED_STATE, &state->flags) != 0) + return 0; + err = _nfs4_do_setlk(state, F_SETLK, request, NFS_LOCK_RECLAIM); + if (err != -NFS4ERR_DELAY) + break; + nfs4_handle_exception(server, err, &exception); + } while (exception.retry); + return err; +} + +int nfs4_lock_expired(struct nfs4_state *state, struct file_lock *request) +{ + struct nfs_server *server = NFS_SERVER(state->inode); + struct nfs4_exception exception = { + .inode = state->inode, + }; + int err; + + err = nfs4_set_lock_state(state, request); + if (err != 0) + return err; + if (!recover_lost_locks) { + set_bit(NFS_LOCK_LOST, &request->fl_u.nfs4_fl.owner->ls_flags); + return 0; + } + do { + if (test_bit(NFS_DELEGATED_STATE, &state->flags) != 0) + return 0; + err = _nfs4_do_setlk(state, F_SETLK, request, NFS_LOCK_EXPIRED); + switch (err) { + default: + goto out; + case -NFS4ERR_GRACE: + case -NFS4ERR_DELAY: + nfs4_handle_exception(server, err, &exception); + err = 0; + } + } while (exception.retry); +out: + return err; +} + +static int nfs41_lock_expired(struct nfs4_state *state, struct file_lock *request) +{ + struct nfs4_lock_state *lsp; + int status; + + status = nfs4_set_lock_state(state, request); + if (status != 0) + return status; + lsp = request->fl_u.nfs4_fl.owner; + if (test_bit(NFS_LOCK_INITIALIZED, &lsp->ls_flags) || + test_bit(NFS_LOCK_LOST, &lsp->ls_flags)) + return 0; + return nfs4_lock_expired(state, request); +} + +static int _nfs4_proc_setlk(struct nfs4_state *state, int cmd, struct file_lock *request) +{ + struct nfs_inode *nfsi = NFS_I(state->inode); + struct nfs4_state_owner *sp = state->owner; + unsigned char flags = request->c.flc_flags; + int status; + + request->c.flc_flags |= FL_ACCESS; + status = locks_lock_inode_wait(state->inode, request); + if (status < 0) + goto out; + mutex_lock(&sp->so_delegreturn_mutex); + down_read(&nfsi->rwsem); + if (test_bit(NFS_DELEGATED_STATE, &state->flags)) { + /* Yes: cache locks! */ + /* ...but avoid races with delegation recall... */ + request->c.flc_flags = flags & ~FL_SLEEP; + status = locks_lock_inode_wait(state->inode, request); + up_read(&nfsi->rwsem); + mutex_unlock(&sp->so_delegreturn_mutex); + goto out; + } + up_read(&nfsi->rwsem); + mutex_unlock(&sp->so_delegreturn_mutex); + status = _nfs4_do_setlk(state, cmd, request, NFS_LOCK_NEW); + if (status) + goto out; + + down_read(&nfsi->rwsem); + request->c.flc_flags &= ~(FL_SLEEP | FL_ACCESS); + status = locks_lock_inode_wait(state->inode, request); + up_read(&nfsi->rwsem); +out: + request->c.flc_flags = flags; + return status; +} + +static int nfs4_proc_setlk(struct nfs4_state *state, int cmd, struct file_lock *request) +{ + struct nfs4_exception exception = { + .state = state, + .inode = state->inode, + .interruptible = true, + }; + int err; + + do { + err = _nfs4_proc_setlk(state, cmd, request); + if (err == -NFS4ERR_DENIED) + err = -EAGAIN; + err = nfs4_handle_exception(NFS_SERVER(state->inode), + err, &exception); + } while (exception.retry); + return err; +} + +#define NFS4_LOCK_MINTIMEOUT (1 * HZ) +#define NFS4_LOCK_MAXTIMEOUT (30 * HZ) + +static int +nfs4_retry_setlk_simple(struct nfs4_state *state, int cmd, + struct file_lock *request) +{ + int status = -ERESTARTSYS; + unsigned long timeout = NFS4_LOCK_MINTIMEOUT; + + while(!signalled()) { + status = nfs4_proc_setlk(state, cmd, request); + if ((status != -EAGAIN) || IS_SETLK(cmd)) + break; + __set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE); + schedule_timeout(timeout); + timeout *= 2; + timeout = min_t(unsigned long, NFS4_LOCK_MAXTIMEOUT, timeout); + status = -ERESTARTSYS; + } + return status; +} + +struct nfs4_lock_waiter { + struct inode *inode; + struct nfs_lowner owner; + wait_queue_entry_t wait; +}; + +static int +nfs4_wake_lock_waiter(wait_queue_entry_t *wait, unsigned int mode, int flags, void *key) +{ + struct nfs4_lock_waiter *waiter = + container_of(wait, struct nfs4_lock_waiter, wait); + + /* NULL key means to wake up everyone */ + if (key) { + struct cb_notify_lock_args *cbnl = key; + struct nfs_lowner *lowner = &cbnl->cbnl_owner, + *wowner = &waiter->owner; + + /* Only wake if the callback was for the same owner. */ + if (lowner->id != wowner->id || lowner->s_dev != wowner->s_dev) + return 0; + + /* Make sure it's for the right inode */ + if (nfs_compare_fh(NFS_FH(waiter->inode), &cbnl->cbnl_fh)) + return 0; + } + + return woken_wake_function(wait, mode, flags, key); +} + +static int +nfs4_retry_setlk(struct nfs4_state *state, int cmd, struct file_lock *request) +{ + struct nfs4_lock_state *lsp = request->fl_u.nfs4_fl.owner; + struct nfs_server *server = NFS_SERVER(state->inode); + struct nfs_client *clp = server->nfs_client; + wait_queue_head_t *q = &clp->cl_lock_waitq; + struct nfs4_lock_waiter waiter = { + .inode = state->inode, + .owner = { .clientid = clp->cl_clientid, + .id = lsp->ls_seqid.owner_id, + .s_dev = server->s_dev }, + }; + int status; + + /* Don't bother with waitqueue if we don't expect a callback */ + if (!test_bit(NFS_STATE_MAY_NOTIFY_LOCK, &state->flags)) + return nfs4_retry_setlk_simple(state, cmd, request); + + init_wait(&waiter.wait); + waiter.wait.func = nfs4_wake_lock_waiter; + add_wait_queue(q, &waiter.wait); + + do { + status = nfs4_proc_setlk(state, cmd, request); + if (status != -EAGAIN || IS_SETLK(cmd)) + break; + + status = -ERESTARTSYS; + wait_woken(&waiter.wait, TASK_INTERRUPTIBLE|TASK_FREEZABLE, + NFS4_LOCK_MAXTIMEOUT); + } while (!signalled()); + + remove_wait_queue(q, &waiter.wait); + + return status; +} + +static int +nfs4_proc_lock(struct file *filp, int cmd, struct file_lock *request) +{ + struct nfs_open_context *ctx; + struct nfs4_state *state; + int status; + + /* verify open state */ + ctx = nfs_file_open_context(filp); + state = ctx->state; + + if (IS_GETLK(cmd)) { + if (state != NULL) + return nfs4_proc_getlk(state, F_GETLK, request); + return 0; + } + + if (!(IS_SETLK(cmd) || IS_SETLKW(cmd))) + return -EINVAL; + + if (lock_is_unlock(request)) { + if (state != NULL) + return nfs4_proc_unlck(state, cmd, request); + return 0; + } + + if (state == NULL) + return -ENOLCK; + + if ((request->c.flc_flags & FL_POSIX) && + !test_bit(NFS_STATE_POSIX_LOCKS, &state->flags)) + return -ENOLCK; + + /* + * Don't rely on the VFS having checked the file open mode, + * since it won't do this for flock() locks. + */ + switch (request->c.flc_type) { + case F_RDLCK: + if (!(filp->f_mode & FMODE_READ)) + return -EBADF; + break; + case F_WRLCK: + if (!(filp->f_mode & FMODE_WRITE)) + return -EBADF; + } + + status = nfs4_set_lock_state(state, request); + if (status != 0) + return status; + + return nfs4_retry_setlk(state, cmd, request); +} + +static int nfs4_delete_lease(struct file *file, void **priv) +{ + return generic_setlease(file, F_UNLCK, NULL, priv); +} + +static int nfs4_add_lease(struct file *file, int arg, struct file_lease **lease, + void **priv) +{ + struct inode *inode = file_inode(file); + fmode_t type = arg == F_RDLCK ? FMODE_READ : FMODE_WRITE; + int ret; + + /* No delegation, no lease */ + if (!nfs4_have_delegation(inode, type, 0)) + return -EAGAIN; + ret = generic_setlease(file, arg, lease, priv); + if (ret || nfs4_have_delegation(inode, type, 0)) + return ret; + /* We raced with a delegation return */ + nfs4_delete_lease(file, priv); + return -EAGAIN; +} + +int nfs4_proc_setlease(struct file *file, int arg, struct file_lease **lease, + void **priv) +{ + switch (arg) { + case F_RDLCK: + case F_WRLCK: + return nfs4_add_lease(file, arg, lease, priv); + case F_UNLCK: + return nfs4_delete_lease(file, priv); + default: + return -EINVAL; + } +} + +int nfs4_lock_delegation_recall(struct file_lock *fl, struct nfs4_state *state, const nfs4_stateid *stateid) +{ + struct nfs_server *server = NFS_SERVER(state->inode); + int err; + + err = nfs4_set_lock_state(state, fl); + if (err != 0) + return err; + do { + err = _nfs4_do_setlk(state, F_SETLK, fl, NFS_LOCK_NEW); + if (err != -NFS4ERR_DELAY && err != -NFS4ERR_GRACE) + break; + ssleep(1); + } while (err == -NFS4ERR_DELAY || err == -NFSERR_GRACE); + return nfs4_handle_delegation_recall_error(server, state, stateid, fl, err); +} + +#define XATTR_NAME_NFSV4_ACL "system.nfs4_acl" + +static int nfs4_xattr_set_nfs4_acl(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *key, const void *buf, + size_t buflen, int flags) +{ + return nfs4_proc_set_acl(inode, buf, buflen, NFS4ACL_ACL); +} + +static int nfs4_xattr_get_nfs4_acl(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *key, void *buf, size_t buflen) +{ + return nfs4_proc_get_acl(inode, buf, buflen, NFS4ACL_ACL); +} + +static bool nfs4_xattr_list_nfs4_acl(struct dentry *dentry) +{ + return nfs4_server_supports_acls(NFS_SB(dentry->d_sb), NFS4ACL_ACL); +} + +#define XATTR_NAME_NFSV4_DACL "system.nfs4_dacl" + +static int nfs4_xattr_set_nfs4_dacl(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *key, const void *buf, + size_t buflen, int flags) +{ + return nfs4_proc_set_acl(inode, buf, buflen, NFS4ACL_DACL); +} + +static int nfs4_xattr_get_nfs4_dacl(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *key, void *buf, size_t buflen) +{ + return nfs4_proc_get_acl(inode, buf, buflen, NFS4ACL_DACL); +} + +static bool nfs4_xattr_list_nfs4_dacl(struct dentry *dentry) +{ + return nfs4_server_supports_acls(NFS_SB(dentry->d_sb), NFS4ACL_DACL); +} + +#define XATTR_NAME_NFSV4_SACL "system.nfs4_sacl" + +static int nfs4_xattr_set_nfs4_sacl(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *key, const void *buf, + size_t buflen, int flags) +{ + return nfs4_proc_set_acl(inode, buf, buflen, NFS4ACL_SACL); +} + +static int nfs4_xattr_get_nfs4_sacl(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *key, void *buf, size_t buflen) +{ + return nfs4_proc_get_acl(inode, buf, buflen, NFS4ACL_SACL); +} + +static bool nfs4_xattr_list_nfs4_sacl(struct dentry *dentry) +{ + return nfs4_server_supports_acls(NFS_SB(dentry->d_sb), NFS4ACL_SACL); +} + +#ifdef CONFIG_NFS_V4_SECURITY_LABEL + +static int nfs4_xattr_set_nfs4_label(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *key, const void *buf, + size_t buflen, int flags) +{ + if (security_ismaclabel(key)) + return nfs4_set_security_label(inode, buf, buflen); + + return -EOPNOTSUPP; +} + +static int nfs4_xattr_get_nfs4_label(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *key, void *buf, size_t buflen) +{ + if (security_ismaclabel(key)) + return nfs4_get_security_label(inode, buf, buflen); + return -EOPNOTSUPP; +} + +static const struct xattr_handler nfs4_xattr_nfs4_label_handler = { + .prefix = XATTR_SECURITY_PREFIX, + .get = nfs4_xattr_get_nfs4_label, + .set = nfs4_xattr_set_nfs4_label, +}; + +#endif + +#ifdef CONFIG_NFS_V4_2 +static int nfs4_xattr_set_nfs4_user(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *key, const void *buf, + size_t buflen, int flags) +{ + u32 mask; + int ret; + + if (!nfs_server_capable(inode, NFS_CAP_XATTR)) + return -EOPNOTSUPP; + + /* + * There is no mapping from the MAY_* flags to the NFS_ACCESS_XA* + * flags right now. Handling of xattr operations use the normal + * file read/write permissions. + * + * Just in case the server has other ideas (which RFC 8276 allows), + * do a cached access check for the XA* flags to possibly avoid + * doing an RPC and getting EACCES back. + */ + if (!nfs_access_get_cached(inode, current_cred(), &mask, true)) { + if (!(mask & NFS_ACCESS_XAWRITE)) + return -EACCES; + } + + if (buf == NULL) { + ret = nfs42_proc_removexattr(inode, key); + if (!ret) + nfs4_xattr_cache_remove(inode, key); + } else { + ret = nfs42_proc_setxattr(inode, key, buf, buflen, flags); + if (!ret) + nfs4_xattr_cache_add(inode, key, buf, NULL, buflen); + } + + return ret; +} + +static int nfs4_xattr_get_nfs4_user(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *key, void *buf, size_t buflen) +{ + u32 mask; + ssize_t ret; + + if (!nfs_server_capable(inode, NFS_CAP_XATTR)) + return -EOPNOTSUPP; + + if (!nfs_access_get_cached(inode, current_cred(), &mask, true)) { + if (!(mask & NFS_ACCESS_XAREAD)) + return -EACCES; + } + + ret = nfs_revalidate_inode(inode, NFS_INO_INVALID_CHANGE); + if (ret) + return ret; + + ret = nfs4_xattr_cache_get(inode, key, buf, buflen); + if (ret >= 0 || (ret < 0 && ret != -ENOENT)) + return ret; + + ret = nfs42_proc_getxattr(inode, key, buf, buflen); + + return ret; +} + +static ssize_t +nfs4_listxattr_nfs4_user(struct inode *inode, char *list, size_t list_len) +{ + u64 cookie; + bool eof; + ssize_t ret, size; + char *buf; + size_t buflen; + u32 mask; + + if (!nfs_server_capable(inode, NFS_CAP_XATTR)) + return 0; + + if (!nfs_access_get_cached(inode, current_cred(), &mask, true)) { + if (!(mask & NFS_ACCESS_XALIST)) + return 0; + } + + ret = nfs_revalidate_inode(inode, NFS_INO_INVALID_CHANGE); + if (ret) + return ret; + + ret = nfs4_xattr_cache_list(inode, list, list_len); + if (ret >= 0 || (ret < 0 && ret != -ENOENT)) + return ret; + + cookie = 0; + eof = false; + buflen = list_len ? list_len : XATTR_LIST_MAX; + buf = list_len ? list : NULL; + size = 0; + + while (!eof) { + ret = nfs42_proc_listxattrs(inode, buf, buflen, + &cookie, &eof); + if (ret < 0) + return ret; + + if (list_len) { + buf += ret; + buflen -= ret; + } + size += ret; + } + + if (list_len) + nfs4_xattr_cache_set_list(inode, list, size); + + return size; +} + +#else + +static ssize_t +nfs4_listxattr_nfs4_user(struct inode *inode, char *list, size_t list_len) +{ + return 0; +} +#endif /* CONFIG_NFS_V4_2 */ + +/* + * nfs_fhget will use either the mounted_on_fileid or the fileid + */ +static void nfs_fixup_referral_attributes(struct nfs_fattr *fattr) +{ + if (!(((fattr->valid & NFS_ATTR_FATTR_MOUNTED_ON_FILEID) || + (fattr->valid & NFS_ATTR_FATTR_FILEID)) && + (fattr->valid & NFS_ATTR_FATTR_FSID) && + (fattr->valid & NFS_ATTR_FATTR_V4_LOCATIONS))) + return; + + fattr->valid |= NFS_ATTR_FATTR_TYPE | NFS_ATTR_FATTR_MODE | + NFS_ATTR_FATTR_NLINK | NFS_ATTR_FATTR_V4_REFERRAL; + fattr->mode = S_IFDIR | S_IRUGO | S_IXUGO; + fattr->nlink = 2; +} + +static int _nfs4_proc_fs_locations(struct rpc_clnt *client, struct inode *dir, + const struct qstr *name, + struct nfs4_fs_locations *fs_locations, + struct page *page) +{ + struct nfs_server *server = NFS_SERVER(dir); + u32 bitmask[3]; + struct nfs4_fs_locations_arg args = { + .dir_fh = NFS_FH(dir), + .name = name, + .page = page, + .bitmask = bitmask, + }; + struct nfs4_fs_locations_res res = { + .fs_locations = fs_locations, + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_FS_LOCATIONS], + .rpc_argp = &args, + .rpc_resp = &res, + }; + int status; + + dprintk("%s: start\n", __func__); + + bitmask[0] = nfs4_fattr_bitmap[0] | FATTR4_WORD0_FS_LOCATIONS; + bitmask[1] = nfs4_fattr_bitmap[1]; + + /* Ask for the fileid of the absent filesystem if mounted_on_fileid + * is not supported */ + if (NFS_SERVER(dir)->attr_bitmask[1] & FATTR4_WORD1_MOUNTED_ON_FILEID) + bitmask[0] &= ~FATTR4_WORD0_FILEID; + else + bitmask[1] &= ~FATTR4_WORD1_MOUNTED_ON_FILEID; + + nfs_fattr_init(fs_locations->fattr); + fs_locations->server = server; + fs_locations->nlocations = 0; + status = nfs4_call_sync(client, server, &msg, &args.seq_args, &res.seq_res, 0); + dprintk("%s: returned status = %d\n", __func__, status); + return status; +} + +int nfs4_proc_fs_locations(struct rpc_clnt *client, struct inode *dir, + const struct qstr *name, + struct nfs4_fs_locations *fs_locations, + struct page *page) +{ + struct nfs4_exception exception = { + .interruptible = true, + }; + int err; + do { + err = _nfs4_proc_fs_locations(client, dir, name, + fs_locations, page); + trace_nfs4_get_fs_locations(dir, name, err); + err = nfs4_handle_exception(NFS_SERVER(dir), err, + &exception); + } while (exception.retry); + return err; +} + +/* + * This operation also signals the server that this client is + * performing migration recovery. The server can stop asserting + * SEQ4_STATUS_LEASE_MOVED for this client. The client ID + * performing this operation is identified in the SEQUENCE + * operation in this compound. + * + * When the client supports GETATTR(fs_locations_info), it can + * be plumbed in here. + */ +static int _nfs41_proc_get_locations(struct nfs_server *server, + struct nfs_fh *fhandle, + struct nfs4_fs_locations *locations, + struct page *page, const struct cred *cred) +{ + struct rpc_clnt *clnt = server->client; + struct nfs_client *clp = server->nfs_client; + u32 bitmask[2] = { + [0] = FATTR4_WORD0_FSID | FATTR4_WORD0_FS_LOCATIONS, + }; + struct nfs4_fs_locations_arg args = { + .fh = fhandle, + .page = page, + .bitmask = bitmask, + .migration = 1, /* skip LOOKUP */ + }; + struct nfs4_fs_locations_res res = { + .fs_locations = locations, + .migration = 1, + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_FS_LOCATIONS], + .rpc_argp = &args, + .rpc_resp = &res, + .rpc_cred = cred, + }; + struct nfs4_call_sync_data data = { + .seq_server = server, + .seq_args = &args.seq_args, + .seq_res = &res.seq_res, + }; + struct rpc_task_setup task_setup_data = { + .rpc_client = clnt, + .rpc_message = &msg, + .callback_ops = clp->cl_mvops->call_sync_ops, + .callback_data = &data, + .flags = RPC_TASK_NO_ROUND_ROBIN, + }; + int status; + + nfs_fattr_init(locations->fattr); + locations->server = server; + locations->nlocations = 0; + + nfs4_init_sequence(clp, &args.seq_args, &res.seq_res, 0, 1); + status = nfs4_call_sync_custom(&task_setup_data); + if (status == NFS4_OK && + res.seq_res.sr_status_flags & SEQ4_STATUS_LEASE_MOVED) + status = -NFS4ERR_LEASE_MOVED; + return status; +} + +/** + * nfs4_proc_get_locations - discover locations for a migrated FSID + * @server: pointer to nfs_server to process + * @fhandle: pointer to the kernel NFS client file handle + * @locations: result of query + * @page: buffer + * @cred: credential to use for this operation + * + * Returns NFS4_OK on success, a negative NFS4ERR status code if the + * operation failed, or a negative errno if a local error occurred. + * + * On success, "locations" is filled in, but if the server has + * no locations information, NFS_ATTR_FATTR_V4_LOCATIONS is not + * asserted. + * + * -NFS4ERR_LEASE_MOVED is returned if the server still has leases + * from this client that require migration recovery. + */ +int nfs4_proc_get_locations(struct nfs_server *server, + struct nfs_fh *fhandle, + struct nfs4_fs_locations *locations, + struct page *page, const struct cred *cred) +{ + struct nfs_client *clp = server->nfs_client; + const struct nfs4_mig_recovery_ops *ops = + clp->cl_mvops->mig_recovery_ops; + struct nfs4_exception exception = { + .interruptible = true, + }; + int status; + + dprintk("%s: FSID %llx:%llx on \"%s\"\n", __func__, + (unsigned long long)server->fsid.major, + (unsigned long long)server->fsid.minor, + clp->cl_hostname); + nfs_display_fhandle(fhandle, __func__); + + do { + status = ops->get_locations(server, fhandle, locations, page, + cred); + if (status != -NFS4ERR_DELAY) + break; + nfs4_handle_exception(server, status, &exception); + } while (exception.retry); + return status; +} + +/* + * This operation also signals the server that this client is + * performing "lease moved" recovery. The server can stop asserting + * SEQ4_STATUS_LEASE_MOVED for this client. The client ID performing + * this operation is identified in the SEQUENCE operation in this + * compound. + */ +static int _nfs41_proc_fsid_present(struct inode *inode, const struct cred *cred) +{ + struct nfs_server *server = NFS_SERVER(inode); + struct rpc_clnt *clnt = server->client; + struct nfs4_fsid_present_arg args = { + .fh = NFS_FH(inode), + }; + struct nfs4_fsid_present_res res = { + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_FSID_PRESENT], + .rpc_argp = &args, + .rpc_resp = &res, + .rpc_cred = cred, + }; + int status; + + res.fh = nfs_alloc_fhandle(); + if (res.fh == NULL) + return -ENOMEM; + + nfs4_init_sequence(server->nfs_client, &args.seq_args, &res.seq_res, 0, 1); + status = nfs4_call_sync_sequence(clnt, server, &msg, + &args.seq_args, &res.seq_res); + nfs_free_fhandle(res.fh); + if (status == NFS4_OK && + res.seq_res.sr_status_flags & SEQ4_STATUS_LEASE_MOVED) + status = -NFS4ERR_LEASE_MOVED; + return status; +} + +/** + * nfs4_proc_fsid_present - Is this FSID present or absent on server? + * @inode: inode on FSID to check + * @cred: credential to use for this operation + * + * Server indicates whether the FSID is present, moved, or not + * recognized. This operation is necessary to clear a LEASE_MOVED + * condition for this client ID. + * + * Returns NFS4_OK if the FSID is present on this server, + * -NFS4ERR_MOVED if the FSID is no longer present, a negative + * NFS4ERR code if some error occurred on the server, or a + * negative errno if a local failure occurred. + */ +int nfs4_proc_fsid_present(struct inode *inode, const struct cred *cred) +{ + struct nfs_server *server = NFS_SERVER(inode); + struct nfs_client *clp = server->nfs_client; + const struct nfs4_mig_recovery_ops *ops = + clp->cl_mvops->mig_recovery_ops; + struct nfs4_exception exception = { + .interruptible = true, + }; + int status; + + dprintk("%s: FSID %llx:%llx on \"%s\"\n", __func__, + (unsigned long long)server->fsid.major, + (unsigned long long)server->fsid.minor, + clp->cl_hostname); + nfs_display_fhandle(NFS_FH(inode), __func__); + + do { + status = ops->fsid_present(inode, cred); + if (status != -NFS4ERR_DELAY) + break; + nfs4_handle_exception(server, status, &exception); + } while (exception.retry); + return status; +} + +/* + * If 'use_integrity' is true and the state managment nfs_client + * cl_rpcclient is using krb5i/p, use the integrity protected cl_rpcclient + * and the machine credential as per RFC3530bis and RFC5661 Security + * Considerations sections. Otherwise, just use the user cred with the + * filesystem's rpc_client. + */ +static int _nfs4_proc_secinfo(struct inode *dir, const struct qstr *name, struct nfs4_secinfo_flavors *flavors, bool use_integrity) +{ + int status; + struct rpc_clnt *clnt = NFS_SERVER(dir)->client; + struct nfs_client *clp = NFS_SERVER(dir)->nfs_client; + struct nfs4_secinfo_arg args = { + .dir_fh = NFS_FH(dir), + .name = name, + }; + struct nfs4_secinfo_res res = { + .flavors = flavors, + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_SECINFO], + .rpc_argp = &args, + .rpc_resp = &res, + }; + struct nfs4_call_sync_data data = { + .seq_server = NFS_SERVER(dir), + .seq_args = &args.seq_args, + .seq_res = &res.seq_res, + }; + struct rpc_task_setup task_setup = { + .rpc_client = clnt, + .rpc_message = &msg, + .callback_ops = clp->cl_mvops->call_sync_ops, + .callback_data = &data, + .flags = RPC_TASK_NO_ROUND_ROBIN, + }; + const struct cred *cred = NULL; + + if (use_integrity) { + clnt = clp->cl_rpcclient; + task_setup.rpc_client = clnt; + + cred = nfs4_get_clid_cred(clp); + msg.rpc_cred = cred; + } + + dprintk("NFS call secinfo %s\n", name->name); + + nfs4_state_protect(clp, NFS_SP4_MACH_CRED_SECINFO, &clnt, &msg); + nfs4_init_sequence(clp, &args.seq_args, &res.seq_res, 0, 0); + status = nfs4_call_sync_custom(&task_setup); + + dprintk("NFS reply secinfo: %d\n", status); + + put_cred(cred); + return status; +} + +int nfs4_proc_secinfo(struct inode *dir, const struct qstr *name, + struct nfs4_secinfo_flavors *flavors) +{ + struct nfs4_exception exception = { + .interruptible = true, + }; + int err; + do { + err = -NFS4ERR_WRONGSEC; + + /* try to use integrity protection with machine cred */ + if (_nfs4_is_integrity_protected(NFS_SERVER(dir)->nfs_client)) + err = _nfs4_proc_secinfo(dir, name, flavors, true); + + /* + * if unable to use integrity protection, or SECINFO with + * integrity protection returns NFS4ERR_WRONGSEC (which is + * disallowed by spec, but exists in deployed servers) use + * the current filesystem's rpc_client and the user cred. + */ + if (err == -NFS4ERR_WRONGSEC) + err = _nfs4_proc_secinfo(dir, name, flavors, false); + + trace_nfs4_secinfo(dir, name, err); + err = nfs4_handle_exception(NFS_SERVER(dir), err, + &exception); + } while (exception.retry); + return err; +} + +/* + * Check the exchange flags returned by the server for invalid flags, having + * both PNFS and NON_PNFS flags set, and not having one of NON_PNFS, PNFS, or + * DS flags set. + */ +static int nfs4_check_cl_exchange_flags(u32 flags, u32 version) +{ + if (version >= 2 && (flags & ~EXCHGID4_2_FLAG_MASK_R)) + goto out_inval; + else if (version < 2 && (flags & ~EXCHGID4_FLAG_MASK_R)) + goto out_inval; + if ((flags & EXCHGID4_FLAG_USE_PNFS_MDS) && + (flags & EXCHGID4_FLAG_USE_NON_PNFS)) + goto out_inval; + if (!(flags & (EXCHGID4_FLAG_MASK_PNFS))) + goto out_inval; + return NFS_OK; +out_inval: + return -NFS4ERR_INVAL; +} + +static bool +nfs41_same_server_scope(struct nfs41_server_scope *a, + struct nfs41_server_scope *b) +{ + if (a->server_scope_sz != b->server_scope_sz) + return false; + return memcmp(a->server_scope, b->server_scope, a->server_scope_sz) == 0; +} + +static void +nfs4_bind_one_conn_to_session_done(struct rpc_task *task, void *calldata) +{ + struct nfs41_bind_conn_to_session_args *args = task->tk_msg.rpc_argp; + struct nfs41_bind_conn_to_session_res *res = task->tk_msg.rpc_resp; + struct nfs_client *clp = args->client; + + switch (task->tk_status) { + case -NFS4ERR_BADSESSION: + case -NFS4ERR_DEADSESSION: + nfs4_schedule_session_recovery(clp->cl_session, + task->tk_status); + return; + } + if (args->dir == NFS4_CDFC4_FORE_OR_BOTH && + res->dir != NFS4_CDFS4_BOTH) { + rpc_task_close_connection(task); + if (args->retries++ < MAX_BIND_CONN_TO_SESSION_RETRIES) + rpc_restart_call(task); + } +} + +static const struct rpc_call_ops nfs4_bind_one_conn_to_session_ops = { + .rpc_call_done = nfs4_bind_one_conn_to_session_done, +}; + +/* + * nfs4_proc_bind_one_conn_to_session() + * + * The 4.1 client currently uses the same TCP connection for the + * fore and backchannel. + */ +static +int nfs4_proc_bind_one_conn_to_session(struct rpc_clnt *clnt, + struct rpc_xprt *xprt, + struct nfs_client *clp, + const struct cred *cred) +{ + int status; + struct nfs41_bind_conn_to_session_args args = { + .client = clp, + .dir = NFS4_CDFC4_FORE_OR_BOTH, + .retries = 0, + }; + struct nfs41_bind_conn_to_session_res res; + struct rpc_message msg = { + .rpc_proc = + &nfs4_procedures[NFSPROC4_CLNT_BIND_CONN_TO_SESSION], + .rpc_argp = &args, + .rpc_resp = &res, + .rpc_cred = cred, + }; + struct rpc_task_setup task_setup_data = { + .rpc_client = clnt, + .rpc_xprt = xprt, + .callback_ops = &nfs4_bind_one_conn_to_session_ops, + .rpc_message = &msg, + .flags = RPC_TASK_TIMEOUT, + }; + struct rpc_task *task; + + nfs4_copy_sessionid(&args.sessionid, &clp->cl_session->sess_id); + if (!(clp->cl_session->flags & SESSION4_BACK_CHAN)) + args.dir = NFS4_CDFC4_FORE; + + /* Do not set the backchannel flag unless this is clnt->cl_xprt */ + if (xprt != rcu_access_pointer(clnt->cl_xprt)) + args.dir = NFS4_CDFC4_FORE; + + task = rpc_run_task(&task_setup_data); + if (!IS_ERR(task)) { + status = task->tk_status; + rpc_put_task(task); + } else + status = PTR_ERR(task); + trace_nfs4_bind_conn_to_session(clp, status); + if (status == 0) { + if (memcmp(res.sessionid.data, + clp->cl_session->sess_id.data, NFS4_MAX_SESSIONID_LEN)) { + dprintk("NFS: %s: Session ID mismatch\n", __func__); + return -EIO; + } + if ((res.dir & args.dir) != res.dir || res.dir == 0) { + dprintk("NFS: %s: Unexpected direction from server\n", + __func__); + return -EIO; + } + if (res.use_conn_in_rdma_mode != args.use_conn_in_rdma_mode) { + dprintk("NFS: %s: Server returned RDMA mode = true\n", + __func__); + return -EIO; + } + } + + return status; +} + +struct rpc_bind_conn_calldata { + struct nfs_client *clp; + const struct cred *cred; +}; + +static int +nfs4_proc_bind_conn_to_session_callback(struct rpc_clnt *clnt, + struct rpc_xprt *xprt, + void *calldata) +{ + struct rpc_bind_conn_calldata *p = calldata; + + return nfs4_proc_bind_one_conn_to_session(clnt, xprt, p->clp, p->cred); +} + +int nfs4_proc_bind_conn_to_session(struct nfs_client *clp, const struct cred *cred) +{ + struct rpc_bind_conn_calldata data = { + .clp = clp, + .cred = cred, + }; + return rpc_clnt_iterate_for_each_xprt(clp->cl_rpcclient, + nfs4_proc_bind_conn_to_session_callback, &data); +} + +/* + * Minimum set of SP4_MACH_CRED operations from RFC 5661 in the enforce map + * and operations we'd like to see to enable certain features in the allow map + */ +static const struct nfs41_state_protection nfs4_sp4_mach_cred_request = { + .how = SP4_MACH_CRED, + .enforce.u.words = { + [1] = 1 << (OP_BIND_CONN_TO_SESSION - 32) | + 1 << (OP_EXCHANGE_ID - 32) | + 1 << (OP_CREATE_SESSION - 32) | + 1 << (OP_DESTROY_SESSION - 32) | + 1 << (OP_DESTROY_CLIENTID - 32) + }, + .allow.u.words = { + [0] = 1 << (OP_CLOSE) | + 1 << (OP_OPEN_DOWNGRADE) | + 1 << (OP_LOCKU) | + 1 << (OP_DELEGRETURN) | + 1 << (OP_COMMIT), + [1] = 1 << (OP_SECINFO - 32) | + 1 << (OP_SECINFO_NO_NAME - 32) | + 1 << (OP_LAYOUTRETURN - 32) | + 1 << (OP_TEST_STATEID - 32) | + 1 << (OP_FREE_STATEID - 32) | + 1 << (OP_WRITE - 32) + } +}; + +/* + * Select the state protection mode for client `clp' given the server results + * from exchange_id in `sp'. + * + * Returns 0 on success, negative errno otherwise. + */ +static int nfs4_sp4_select_mode(struct nfs_client *clp, + struct nfs41_state_protection *sp) +{ + static const u32 supported_enforce[NFS4_OP_MAP_NUM_WORDS] = { + [1] = 1 << (OP_BIND_CONN_TO_SESSION - 32) | + 1 << (OP_EXCHANGE_ID - 32) | + 1 << (OP_CREATE_SESSION - 32) | + 1 << (OP_DESTROY_SESSION - 32) | + 1 << (OP_DESTROY_CLIENTID - 32) + }; + unsigned long flags = 0; + unsigned int i; + int ret = 0; + + if (sp->how == SP4_MACH_CRED) { + /* Print state protect result */ + dfprintk(MOUNT, "Server SP4_MACH_CRED support:\n"); + for (i = 0; i <= LAST_NFS4_OP; i++) { + if (test_bit(i, sp->enforce.u.longs)) + dfprintk(MOUNT, " enforce op %d\n", i); + if (test_bit(i, sp->allow.u.longs)) + dfprintk(MOUNT, " allow op %d\n", i); + } + + /* make sure nothing is on enforce list that isn't supported */ + for (i = 0; i < NFS4_OP_MAP_NUM_WORDS; i++) { + if (sp->enforce.u.words[i] & ~supported_enforce[i]) { + dfprintk(MOUNT, "sp4_mach_cred: disabled\n"); + ret = -EINVAL; + goto out; + } + } + + /* + * Minimal mode - state operations are allowed to use machine + * credential. Note this already happens by default, so the + * client doesn't have to do anything more than the negotiation. + * + * NOTE: we don't care if EXCHANGE_ID is in the list - + * we're already using the machine cred for exchange_id + * and will never use a different cred. + */ + if (test_bit(OP_BIND_CONN_TO_SESSION, sp->enforce.u.longs) && + test_bit(OP_CREATE_SESSION, sp->enforce.u.longs) && + test_bit(OP_DESTROY_SESSION, sp->enforce.u.longs) && + test_bit(OP_DESTROY_CLIENTID, sp->enforce.u.longs)) { + dfprintk(MOUNT, "sp4_mach_cred:\n"); + dfprintk(MOUNT, " minimal mode enabled\n"); + __set_bit(NFS_SP4_MACH_CRED_MINIMAL, &flags); + } else { + dfprintk(MOUNT, "sp4_mach_cred: disabled\n"); + ret = -EINVAL; + goto out; + } + + if (test_bit(OP_CLOSE, sp->allow.u.longs) && + test_bit(OP_OPEN_DOWNGRADE, sp->allow.u.longs) && + test_bit(OP_DELEGRETURN, sp->allow.u.longs) && + test_bit(OP_LOCKU, sp->allow.u.longs)) { + dfprintk(MOUNT, " cleanup mode enabled\n"); + __set_bit(NFS_SP4_MACH_CRED_CLEANUP, &flags); + } + + if (test_bit(OP_LAYOUTRETURN, sp->allow.u.longs)) { + dfprintk(MOUNT, " pnfs cleanup mode enabled\n"); + __set_bit(NFS_SP4_MACH_CRED_PNFS_CLEANUP, &flags); + } + + if (test_bit(OP_SECINFO, sp->allow.u.longs) && + test_bit(OP_SECINFO_NO_NAME, sp->allow.u.longs)) { + dfprintk(MOUNT, " secinfo mode enabled\n"); + __set_bit(NFS_SP4_MACH_CRED_SECINFO, &flags); + } + + if (test_bit(OP_TEST_STATEID, sp->allow.u.longs) && + test_bit(OP_FREE_STATEID, sp->allow.u.longs)) { + dfprintk(MOUNT, " stateid mode enabled\n"); + __set_bit(NFS_SP4_MACH_CRED_STATEID, &flags); + } + + if (test_bit(OP_WRITE, sp->allow.u.longs)) { + dfprintk(MOUNT, " write mode enabled\n"); + __set_bit(NFS_SP4_MACH_CRED_WRITE, &flags); + } + + if (test_bit(OP_COMMIT, sp->allow.u.longs)) { + dfprintk(MOUNT, " commit mode enabled\n"); + __set_bit(NFS_SP4_MACH_CRED_COMMIT, &flags); + } + } +out: + clp->cl_sp4_flags = flags; + return ret; +} + +struct nfs41_exchange_id_data { + struct nfs41_exchange_id_res res; + struct nfs41_exchange_id_args args; +}; + +static void nfs4_exchange_id_release(void *data) +{ + struct nfs41_exchange_id_data *cdata = + (struct nfs41_exchange_id_data *)data; + + nfs_put_client(cdata->args.client); + kfree(cdata->res.impl_id); + kfree(cdata->res.server_scope); + kfree(cdata->res.server_owner); + kfree(cdata); +} + +static const struct rpc_call_ops nfs4_exchange_id_call_ops = { + .rpc_release = nfs4_exchange_id_release, +}; + +/* + * _nfs4_proc_exchange_id() + * + * Wrapper for EXCHANGE_ID operation. + */ +static struct rpc_task * +nfs4_run_exchange_id(struct nfs_client *clp, const struct cred *cred, + u32 sp4_how, struct rpc_xprt *xprt) +{ + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_EXCHANGE_ID], + .rpc_cred = cred, + }; + struct rpc_task_setup task_setup_data = { + .rpc_client = clp->cl_rpcclient, + .callback_ops = &nfs4_exchange_id_call_ops, + .rpc_message = &msg, + .flags = RPC_TASK_TIMEOUT | RPC_TASK_NO_ROUND_ROBIN, + }; + struct nfs41_exchange_id_data *calldata; + int status; + + if (!refcount_inc_not_zero(&clp->cl_count)) + return ERR_PTR(-EIO); + + status = -ENOMEM; + calldata = kzalloc_obj(*calldata, GFP_NOFS); + if (!calldata) + goto out; + + nfs4_init_boot_verifier(clp, &calldata->args.verifier); + + status = nfs4_init_uniform_client_string(clp); + if (status) + goto out_calldata; + + calldata->res.server_owner = kzalloc_obj(struct nfs41_server_owner, + GFP_NOFS); + status = -ENOMEM; + if (unlikely(calldata->res.server_owner == NULL)) + goto out_calldata; + + calldata->res.server_scope = kzalloc_obj(struct nfs41_server_scope, + GFP_NOFS); + if (unlikely(calldata->res.server_scope == NULL)) + goto out_server_owner; + + calldata->res.impl_id = kzalloc_obj(struct nfs41_impl_id, GFP_NOFS); + if (unlikely(calldata->res.impl_id == NULL)) + goto out_server_scope; + + switch (sp4_how) { + case SP4_NONE: + calldata->args.state_protect.how = SP4_NONE; + break; + + case SP4_MACH_CRED: + calldata->args.state_protect = nfs4_sp4_mach_cred_request; + break; + + default: + /* unsupported! */ + WARN_ON_ONCE(1); + status = -EINVAL; + goto out_impl_id; + } + if (xprt) { + task_setup_data.rpc_xprt = xprt; + task_setup_data.flags |= RPC_TASK_SOFTCONN; + memcpy(calldata->args.verifier.data, clp->cl_confirm.data, + sizeof(calldata->args.verifier.data)); + } + calldata->args.client = clp; + calldata->args.flags = EXCHGID4_FLAG_SUPP_MOVED_REFER | + EXCHGID4_FLAG_BIND_PRINC_STATEID; +#ifdef CONFIG_NFS_V4_1_MIGRATION + calldata->args.flags |= EXCHGID4_FLAG_SUPP_MOVED_MIGR; +#endif + if (test_bit(NFS_CS_PNFS, &clp->cl_flags)) + calldata->args.flags |= EXCHGID4_FLAG_USE_PNFS_DS; + msg.rpc_argp = &calldata->args; + msg.rpc_resp = &calldata->res; + task_setup_data.callback_data = calldata; + + return rpc_run_task(&task_setup_data); + +out_impl_id: + kfree(calldata->res.impl_id); +out_server_scope: + kfree(calldata->res.server_scope); +out_server_owner: + kfree(calldata->res.server_owner); +out_calldata: + kfree(calldata); +out: + nfs_put_client(clp); + return ERR_PTR(status); +} + +/* + * _nfs4_proc_exchange_id() + * + * Wrapper for EXCHANGE_ID operation. + */ +static int _nfs4_proc_exchange_id(struct nfs_client *clp, const struct cred *cred, + u32 sp4_how) +{ + struct rpc_task *task; + struct nfs41_exchange_id_args *argp; + struct nfs41_exchange_id_res *resp; + unsigned long now = jiffies; + int status; + + task = nfs4_run_exchange_id(clp, cred, sp4_how, NULL); + if (IS_ERR(task)) + return PTR_ERR(task); + + argp = task->tk_msg.rpc_argp; + resp = task->tk_msg.rpc_resp; + status = task->tk_status; + if (status != 0) + goto out; + + status = nfs4_check_cl_exchange_flags(resp->flags, + clp->cl_mvops->minor_version); + if (status != 0) + goto out; + + status = nfs4_sp4_select_mode(clp, &resp->state_protect); + if (status != 0) + goto out; + + do_renew_lease(clp, now); + + clp->cl_clientid = resp->clientid; + clp->cl_exchange_flags = resp->flags; + clp->cl_seqid = resp->seqid; + /* Client ID is not confirmed */ + if (!(resp->flags & EXCHGID4_FLAG_CONFIRMED_R)) + clear_bit(NFS4_SESSION_ESTABLISHED, + &clp->cl_session->session_state); + + if (clp->cl_serverscope != NULL && + !nfs41_same_server_scope(clp->cl_serverscope, + resp->server_scope)) { + dprintk("%s: server_scope mismatch detected\n", + __func__); + set_bit(NFS4CLNT_SERVER_SCOPE_MISMATCH, &clp->cl_state); + } + + swap(clp->cl_serverowner, resp->server_owner); + swap(clp->cl_serverscope, resp->server_scope); + swap(clp->cl_implid, resp->impl_id); + + /* Save the EXCHANGE_ID verifier session trunk tests */ + memcpy(clp->cl_confirm.data, argp->verifier.data, + sizeof(clp->cl_confirm.data)); +out: + trace_nfs4_exchange_id(clp, status); + rpc_put_task(task); + return status; +} + +/* + * nfs4_proc_exchange_id() + * + * Returns zero, a negative errno, or a negative NFS4ERR status code. + * + * Since the clientid has expired, all compounds using sessions + * associated with the stale clientid will be returning + * NFS4ERR_BADSESSION in the sequence operation, and will therefore + * be in some phase of session reset. + * + * Will attempt to negotiate SP4_MACH_CRED if krb5i / krb5p auth is used. + */ +int nfs4_proc_exchange_id(struct nfs_client *clp, const struct cred *cred) +{ + rpc_authflavor_t authflavor = clp->cl_rpcclient->cl_auth->au_flavor; + int status; + + /* try SP4_MACH_CRED if krb5i/p */ + if (authflavor == RPC_AUTH_GSS_KRB5I || + authflavor == RPC_AUTH_GSS_KRB5P) { + status = _nfs4_proc_exchange_id(clp, cred, SP4_MACH_CRED); + if (!status) + return 0; + } + + /* try SP4_NONE */ + return _nfs4_proc_exchange_id(clp, cred, SP4_NONE); +} + +/** + * nfs4_test_session_trunk + * + * This is an add_xprt_test() test function called from + * rpc_clnt_setup_test_and_add_xprt. + * + * The rpc_xprt_switch is referrenced by rpc_clnt_setup_test_and_add_xprt + * and is dereferrenced in nfs4_exchange_id_release + * + * Upon success, add the new transport to the rpc_clnt + * + * @clnt: struct rpc_clnt to get new transport + * @xprt: the rpc_xprt to test + * @data: call data for _nfs4_proc_exchange_id. + */ +void nfs4_test_session_trunk(struct rpc_clnt *clnt, struct rpc_xprt *xprt, + void *data) +{ + struct nfs4_add_xprt_data *adata = data; + struct rpc_task *task; + int status; + + u32 sp4_how; + + dprintk("--> %s try %s\n", __func__, + xprt->address_strings[RPC_DISPLAY_ADDR]); + + sp4_how = (adata->clp->cl_sp4_flags == 0 ? SP4_NONE : SP4_MACH_CRED); + +try_again: + /* Test connection for session trunking. Async exchange_id call */ + task = nfs4_run_exchange_id(adata->clp, adata->cred, sp4_how, xprt); + if (IS_ERR(task)) + return; + + status = task->tk_status; + if (status == 0) { + status = nfs4_detect_session_trunking(adata->clp, + task->tk_msg.rpc_resp, xprt); + trace_nfs4_trunked_exchange_id(adata->clp, + xprt->address_strings[RPC_DISPLAY_ADDR], status); + } + if (status == 0) + rpc_clnt_xprt_switch_add_xprt(clnt, xprt); + else if (status != -NFS4ERR_DELAY && rpc_clnt_xprt_switch_has_addr(clnt, + (struct sockaddr *)&xprt->addr)) + rpc_clnt_xprt_switch_remove_xprt(clnt, xprt); + + rpc_put_task(task); + if (status == -NFS4ERR_DELAY) { + ssleep(1); + goto try_again; + } +} +EXPORT_SYMBOL_GPL(nfs4_test_session_trunk); + +static int _nfs4_proc_destroy_clientid(struct nfs_client *clp, + const struct cred *cred) +{ + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_DESTROY_CLIENTID], + .rpc_argp = clp, + .rpc_cred = cred, + }; + int status; + + status = rpc_call_sync(clp->cl_rpcclient, &msg, + RPC_TASK_TIMEOUT | RPC_TASK_NO_ROUND_ROBIN); + trace_nfs4_destroy_clientid(clp, status); + if (status) + dprintk("NFS: Got error %d from the server %s on " + "DESTROY_CLIENTID.", status, clp->cl_hostname); + return status; +} + +static int nfs4_proc_destroy_clientid(struct nfs_client *clp, + const struct cred *cred) +{ + unsigned int loop; + int ret; + + for (loop = NFS4_MAX_LOOP_ON_RECOVER; loop != 0; loop--) { + ret = _nfs4_proc_destroy_clientid(clp, cred); + switch (ret) { + case -NFS4ERR_DELAY: + case -NFS4ERR_CLIENTID_BUSY: + ssleep(1); + break; + default: + return ret; + } + } + return 0; +} + +int nfs4_destroy_clientid(struct nfs_client *clp) +{ + const struct cred *cred; + int ret = 0; + + if (clp->cl_mvops->minor_version < 1) + goto out; + if (clp->cl_exchange_flags == 0) + goto out; + if (clp->cl_preserve_clid) + goto out; + cred = nfs4_get_clid_cred(clp); + ret = nfs4_proc_destroy_clientid(clp, cred); + put_cred(cred); + switch (ret) { + case 0: + case -NFS4ERR_STALE_CLIENTID: + clp->cl_exchange_flags = 0; + } +out: + return ret; +} + +struct nfs4_get_lease_time_data { + struct nfs4_get_lease_time_args *args; + struct nfs4_get_lease_time_res *res; + struct nfs_client *clp; +}; + +static void nfs4_get_lease_time_prepare(struct rpc_task *task, + void *calldata) +{ + struct nfs4_get_lease_time_data *data = + (struct nfs4_get_lease_time_data *)calldata; + + /* just setup sequence, do not trigger session recovery + since we're invoked within one */ + nfs4_setup_sequence(data->clp, + &data->args->la_seq_args, + &data->res->lr_seq_res, + task); +} + +/* + * Called from nfs4_state_manager thread for session setup, so don't recover + * from sequence operation or clientid errors. + */ +static void nfs4_get_lease_time_done(struct rpc_task *task, void *calldata) +{ + struct nfs4_get_lease_time_data *data = + (struct nfs4_get_lease_time_data *)calldata; + + if (!nfs4_sequence_done(task, &data->res->lr_seq_res)) + return; + switch (task->tk_status) { + case -NFS4ERR_DELAY: + case -NFS4ERR_GRACE: + rpc_delay(task, NFS4_POLL_RETRY_MIN); + task->tk_status = 0; + fallthrough; + case -NFS4ERR_RETRY_UNCACHED_REP: + rpc_restart_call_prepare(task); + return; + } +} + +static const struct rpc_call_ops nfs4_get_lease_time_ops = { + .rpc_call_prepare = nfs4_get_lease_time_prepare, + .rpc_call_done = nfs4_get_lease_time_done, +}; + +int nfs4_proc_get_lease_time(struct nfs_client *clp, struct nfs_fsinfo *fsinfo) +{ + struct nfs4_get_lease_time_args args; + struct nfs4_get_lease_time_res res = { + .lr_fsinfo = fsinfo, + }; + struct nfs4_get_lease_time_data data = { + .args = &args, + .res = &res, + .clp = clp, + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_GET_LEASE_TIME], + .rpc_argp = &args, + .rpc_resp = &res, + }; + struct rpc_task_setup task_setup = { + .rpc_client = clp->cl_rpcclient, + .rpc_message = &msg, + .callback_ops = &nfs4_get_lease_time_ops, + .callback_data = &data, + .flags = RPC_TASK_TIMEOUT, + }; + + nfs4_init_sequence(clp, &args.la_seq_args, &res.lr_seq_res, 0, 1); + return nfs4_call_sync_custom(&task_setup); +} + +/* + * Initialize the values to be used by the client in CREATE_SESSION + * If nfs4_init_session set the fore channel request and response sizes, + * use them. + * + * Set the back channel max_resp_sz_cached to zero to force the client to + * always set csa_cachethis to FALSE because the current implementation + * of the back channel DRC only supports caching the CB_SEQUENCE operation. + */ +static void nfs4_init_channel_attrs(struct nfs41_create_session_args *args, + struct rpc_clnt *clnt) +{ + unsigned int max_rqst_sz, max_resp_sz; + unsigned int max_bc_payload = rpc_max_bc_payload(clnt); + unsigned int max_bc_slots = rpc_num_bc_slots(clnt); + + max_rqst_sz = NFS_MAX_FILE_IO_SIZE + nfs41_maxwrite_overhead; + max_resp_sz = NFS_MAX_FILE_IO_SIZE + nfs41_maxread_overhead; + + /* Fore channel attributes */ + args->fc_attrs.max_rqst_sz = max_rqst_sz; + args->fc_attrs.max_resp_sz = max_resp_sz; + args->fc_attrs.max_ops = NFS4_MAX_OPS; + args->fc_attrs.max_reqs = max_session_slots; + + dprintk("%s: Fore Channel : max_rqst_sz=%u max_resp_sz=%u " + "max_ops=%u max_reqs=%u\n", + __func__, + args->fc_attrs.max_rqst_sz, args->fc_attrs.max_resp_sz, + args->fc_attrs.max_ops, args->fc_attrs.max_reqs); + + /* Back channel attributes */ + args->bc_attrs.max_rqst_sz = max_bc_payload; + args->bc_attrs.max_resp_sz = max_bc_payload; + args->bc_attrs.max_resp_sz_cached = 0; + args->bc_attrs.max_ops = NFS4_MAX_BACK_CHANNEL_OPS; + args->bc_attrs.max_reqs = max_t(unsigned short, max_session_cb_slots, 1); + if (args->bc_attrs.max_reqs > max_bc_slots) + args->bc_attrs.max_reqs = max_bc_slots; + + dprintk("%s: Back Channel : max_rqst_sz=%u max_resp_sz=%u " + "max_resp_sz_cached=%u max_ops=%u max_reqs=%u\n", + __func__, + args->bc_attrs.max_rqst_sz, args->bc_attrs.max_resp_sz, + args->bc_attrs.max_resp_sz_cached, args->bc_attrs.max_ops, + args->bc_attrs.max_reqs); +} + +static int nfs4_verify_fore_channel_attrs(struct nfs41_create_session_args *args, + struct nfs41_create_session_res *res) +{ + struct nfs4_channel_attrs *sent = &args->fc_attrs; + struct nfs4_channel_attrs *rcvd = &res->fc_attrs; + + if (rcvd->max_resp_sz > sent->max_resp_sz) + return -EINVAL; + /* + * Our requested max_ops is the minimum we need; we're not + * prepared to break up compounds into smaller pieces than that. + * So, no point even trying to continue if the server won't + * cooperate: + */ + if (rcvd->max_ops < sent->max_ops) + return -EINVAL; + if (rcvd->max_reqs == 0) + return -EINVAL; + if (rcvd->max_reqs > NFS4_MAX_SLOT_TABLE) + rcvd->max_reqs = NFS4_MAX_SLOT_TABLE; + return 0; +} + +static int nfs4_verify_back_channel_attrs(struct nfs41_create_session_args *args, + struct nfs41_create_session_res *res) +{ + struct nfs4_channel_attrs *sent = &args->bc_attrs; + struct nfs4_channel_attrs *rcvd = &res->bc_attrs; + + if (!(res->flags & SESSION4_BACK_CHAN)) + goto out; + if (rcvd->max_rqst_sz > sent->max_rqst_sz) + return -EINVAL; + if (rcvd->max_resp_sz > sent->max_resp_sz) + return -EINVAL; + if (rcvd->max_resp_sz_cached > sent->max_resp_sz_cached) + return -EINVAL; + if (rcvd->max_ops > sent->max_ops) + return -EINVAL; + if (rcvd->max_reqs > sent->max_reqs) + return -EINVAL; +out: + return 0; +} + +static int nfs4_verify_channel_attrs(struct nfs41_create_session_args *args, + struct nfs41_create_session_res *res) +{ + int ret; + + ret = nfs4_verify_fore_channel_attrs(args, res); + if (ret) + return ret; + return nfs4_verify_back_channel_attrs(args, res); +} + +static void nfs4_update_session(struct nfs4_session *session, + struct nfs41_create_session_res *res) +{ + nfs4_copy_sessionid(&session->sess_id, &res->sessionid); + /* Mark client id and session as being confirmed */ + session->clp->cl_exchange_flags |= EXCHGID4_FLAG_CONFIRMED_R; + set_bit(NFS4_SESSION_ESTABLISHED, &session->session_state); + session->flags = res->flags; + memcpy(&session->fc_attrs, &res->fc_attrs, sizeof(session->fc_attrs)); + if (res->flags & SESSION4_BACK_CHAN) + memcpy(&session->bc_attrs, &res->bc_attrs, + sizeof(session->bc_attrs)); +} + +static int _nfs4_proc_create_session(struct nfs_client *clp, + const struct cred *cred) +{ + struct nfs4_session *session = clp->cl_session; + struct nfs41_create_session_args args = { + .client = clp, + .clientid = clp->cl_clientid, + .seqid = clp->cl_seqid, + .cb_program = NFS4_CALLBACK, + }; + struct nfs41_create_session_res res; + + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_CREATE_SESSION], + .rpc_argp = &args, + .rpc_resp = &res, + .rpc_cred = cred, + }; + int status; + + nfs4_init_channel_attrs(&args, clp->cl_rpcclient); + args.flags = (SESSION4_PERSIST | SESSION4_BACK_CHAN); + + status = rpc_call_sync(session->clp->cl_rpcclient, &msg, + RPC_TASK_TIMEOUT | RPC_TASK_NO_ROUND_ROBIN); + trace_nfs4_create_session(clp, status); + + switch (status) { + case -NFS4ERR_STALE_CLIENTID: + case -NFS4ERR_DELAY: + case -ETIMEDOUT: + case -EACCES: + case -EAGAIN: + goto out; + } + + clp->cl_seqid++; + if (!status) { + /* Verify the session's negotiated channel_attrs values */ + status = nfs4_verify_channel_attrs(&args, &res); + /* Increment the clientid slot sequence id */ + if (status) + goto out; + nfs4_update_session(session, &res); + } +out: + return status; +} + +/* + * Issues a CREATE_SESSION operation to the server. + * It is the responsibility of the caller to verify the session is + * expired before calling this routine. + */ +int nfs4_proc_create_session(struct nfs_client *clp, const struct cred *cred) +{ + int status; + unsigned *ptr; + struct nfs4_session *session = clp->cl_session; + struct nfs4_add_xprt_data xprtdata = { + .clp = clp, + }; + struct rpc_add_xprt_test rpcdata = { + .add_xprt_test = clp->cl_mvops->session_trunk, + .data = &xprtdata, + }; + + dprintk("--> %s clp=%p session=%p\n", __func__, clp, session); + + status = _nfs4_proc_create_session(clp, cred); + if (status) + goto out; + + /* Init or reset the session slot tables */ + status = nfs4_setup_session_slot_tables(session); + dprintk("slot table setup returned %d\n", status); + if (status) + goto out; + + ptr = (unsigned *)&session->sess_id.data[0]; + dprintk("%s client>seqid %d sessionid %u:%u:%u:%u\n", __func__, + clp->cl_seqid, ptr[0], ptr[1], ptr[2], ptr[3]); + rpc_clnt_probe_trunked_xprts(clp->cl_rpcclient, &rpcdata); +out: + return status; +} + +/* + * Issue the over-the-wire RPC DESTROY_SESSION. + * The caller must serialize access to this routine. + */ +int nfs4_proc_destroy_session(struct nfs4_session *session, + const struct cred *cred) +{ + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_DESTROY_SESSION], + .rpc_argp = session, + .rpc_cred = cred, + }; + int status = 0; + + /* session is still being setup */ + if (!test_and_clear_bit(NFS4_SESSION_ESTABLISHED, &session->session_state)) + return 0; + + status = rpc_call_sync(session->clp->cl_rpcclient, &msg, + RPC_TASK_TIMEOUT | RPC_TASK_NO_ROUND_ROBIN); + trace_nfs4_destroy_session(session->clp, status); + + if (status) + dprintk("NFS: Got error %d from the server on DESTROY_SESSION. " + "Session has been destroyed regardless...\n", status); + rpc_clnt_manage_trunked_xprts(session->clp->cl_rpcclient); + return status; +} + +/* + * Renew the cl_session lease. + */ +struct nfs4_sequence_data { + struct nfs_client *clp; + struct nfs4_sequence_args args; + struct nfs4_sequence_res res; +}; + +static void nfs41_sequence_release(void *data) +{ + struct nfs4_sequence_data *calldata = data; + struct nfs_client *clp = calldata->clp; + + if (refcount_read(&clp->cl_count) > 1) + nfs4_schedule_state_renewal(clp); + nfs_put_client(clp); + kfree(calldata); +} + +static int nfs41_sequence_handle_errors(struct rpc_task *task, struct nfs_client *clp) +{ + switch(task->tk_status) { + case -NFS4ERR_DELAY: + rpc_delay(task, NFS4_POLL_RETRY_MAX); + return -EAGAIN; + default: + nfs4_schedule_lease_recovery(clp); + } + return 0; +} + +static void nfs41_sequence_call_done(struct rpc_task *task, void *data) +{ + struct nfs4_sequence_data *calldata = data; + struct nfs_client *clp = calldata->clp; + + if (!nfs41_sequence_done(task, task->tk_msg.rpc_resp)) + return; + + trace_nfs4_sequence(clp, task->tk_status); + if (task->tk_status < 0 && clp->cl_cons_state >= 0) { + dprintk("%s ERROR %d\n", __func__, task->tk_status); + if (refcount_read(&clp->cl_count) == 1) + return; + + if (nfs41_sequence_handle_errors(task, clp) == -EAGAIN) { + rpc_restart_call_prepare(task); + return; + } + } + dprintk("%s rpc_cred %p\n", __func__, task->tk_msg.rpc_cred); +} + +static void nfs41_sequence_prepare(struct rpc_task *task, void *data) +{ + struct nfs4_sequence_data *calldata = data; + struct nfs_client *clp = calldata->clp; + struct nfs4_sequence_args *args; + struct nfs4_sequence_res *res; + + args = task->tk_msg.rpc_argp; + res = task->tk_msg.rpc_resp; + + nfs4_setup_sequence(clp, args, res, task); +} + +static const struct rpc_call_ops nfs41_sequence_ops = { + .rpc_call_done = nfs41_sequence_call_done, + .rpc_call_prepare = nfs41_sequence_prepare, + .rpc_release = nfs41_sequence_release, +}; + +static struct rpc_task *_nfs41_proc_sequence(struct nfs_client *clp, + const struct cred *cred, + struct nfs4_slot *slot, + bool is_privileged) +{ + struct nfs4_sequence_data *calldata; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_SEQUENCE], + .rpc_cred = cred, + }; + struct rpc_task_setup task_setup_data = { + .rpc_client = clp->cl_rpcclient, + .rpc_message = &msg, + .callback_ops = &nfs41_sequence_ops, + .flags = RPC_TASK_ASYNC | RPC_TASK_TIMEOUT | RPC_TASK_MOVEABLE, + }; + struct rpc_task *ret; + + ret = ERR_PTR(-EIO); + if (!refcount_inc_not_zero(&clp->cl_count)) + goto out_err; + + ret = ERR_PTR(-ENOMEM); + calldata = kzalloc_obj(*calldata); + if (calldata == NULL) + goto out_put_clp; + nfs4_init_sequence(clp, &calldata->args, &calldata->res, 0, is_privileged); + nfs4_sequence_attach_slot(&calldata->args, &calldata->res, slot); + msg.rpc_argp = &calldata->args; + msg.rpc_resp = &calldata->res; + calldata->clp = clp; + task_setup_data.callback_data = calldata; + + ret = rpc_run_task(&task_setup_data); + if (IS_ERR(ret)) + goto out_err; + return ret; +out_put_clp: + nfs_put_client(clp); +out_err: + nfs41_release_slot(slot); + return ret; +} + +static int nfs41_proc_async_sequence(struct nfs_client *clp, const struct cred *cred, unsigned renew_flags) +{ + struct rpc_task *task; + int ret = 0; + + if ((renew_flags & NFS4_RENEW_TIMEOUT) == 0) + return -EAGAIN; + task = _nfs41_proc_sequence(clp, cred, NULL, false); + if (IS_ERR(task)) + ret = PTR_ERR(task); + else + rpc_put_task_async(task); + dprintk("<-- %s status=%d\n", __func__, ret); + return ret; +} + +static int nfs4_proc_sequence(struct nfs_client *clp, const struct cred *cred) +{ + struct rpc_task *task; + int ret; + + task = _nfs41_proc_sequence(clp, cred, NULL, true); + if (IS_ERR(task)) { + ret = PTR_ERR(task); + goto out; + } + ret = rpc_wait_for_completion_task(task); + if (!ret) + ret = task->tk_status; + rpc_put_task(task); +out: + dprintk("<-- %s status=%d\n", __func__, ret); + return ret; +} + +struct nfs4_reclaim_complete_data { + struct nfs_client *clp; + struct nfs41_reclaim_complete_args arg; + struct nfs41_reclaim_complete_res res; +}; + +static void nfs4_reclaim_complete_prepare(struct rpc_task *task, void *data) +{ + struct nfs4_reclaim_complete_data *calldata = data; + + nfs4_setup_sequence(calldata->clp, + &calldata->arg.seq_args, + &calldata->res.seq_res, + task); +} + +static int nfs41_reclaim_complete_handle_errors(struct rpc_task *task, struct nfs_client *clp) +{ + switch(task->tk_status) { + case 0: + wake_up_all(&clp->cl_lock_waitq); + fallthrough; + case -NFS4ERR_COMPLETE_ALREADY: + case -NFS4ERR_WRONG_CRED: /* What to do here? */ + break; + case -NFS4ERR_DELAY: + rpc_delay(task, NFS4_POLL_RETRY_MAX); + fallthrough; + case -NFS4ERR_RETRY_UNCACHED_REP: + case -EACCES: + dprintk("%s: failed to reclaim complete error %d for server %s, retrying\n", + __func__, task->tk_status, clp->cl_hostname); + return -EAGAIN; + case -NFS4ERR_BADSESSION: + case -NFS4ERR_DEADSESSION: + case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION: + break; + default: + nfs4_schedule_lease_recovery(clp); + } + return 0; +} + +static void nfs4_reclaim_complete_done(struct rpc_task *task, void *data) +{ + struct nfs4_reclaim_complete_data *calldata = data; + struct nfs_client *clp = calldata->clp; + struct nfs4_sequence_res *res = &calldata->res.seq_res; + + if (!nfs41_sequence_done(task, res)) + return; + + trace_nfs4_reclaim_complete(clp, task->tk_status); + if (nfs41_reclaim_complete_handle_errors(task, clp) == -EAGAIN) { + rpc_restart_call_prepare(task); + return; + } +} + +static void nfs4_free_reclaim_complete_data(void *data) +{ + struct nfs4_reclaim_complete_data *calldata = data; + + kfree(calldata); +} + +static const struct rpc_call_ops nfs4_reclaim_complete_call_ops = { + .rpc_call_prepare = nfs4_reclaim_complete_prepare, + .rpc_call_done = nfs4_reclaim_complete_done, + .rpc_release = nfs4_free_reclaim_complete_data, +}; + +/* + * Issue a global reclaim complete. + */ +static int nfs41_proc_reclaim_complete(struct nfs_client *clp, + const struct cred *cred) +{ + struct nfs4_reclaim_complete_data *calldata; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_RECLAIM_COMPLETE], + .rpc_cred = cred, + }; + struct rpc_task_setup task_setup_data = { + .rpc_client = clp->cl_rpcclient, + .rpc_message = &msg, + .callback_ops = &nfs4_reclaim_complete_call_ops, + .flags = RPC_TASK_NO_ROUND_ROBIN, + }; + int status = -ENOMEM; + + calldata = kzalloc_obj(*calldata, GFP_NOFS); + if (calldata == NULL) + goto out; + calldata->clp = clp; + calldata->arg.one_fs = 0; + + nfs4_init_sequence(clp, &calldata->arg.seq_args, &calldata->res.seq_res, 0, 1); + msg.rpc_argp = &calldata->arg; + msg.rpc_resp = &calldata->res; + task_setup_data.callback_data = calldata; + status = nfs4_call_sync_custom(&task_setup_data); +out: + dprintk("<-- %s status=%d\n", __func__, status); + return status; +} + +static void +nfs4_layoutget_prepare(struct rpc_task *task, void *calldata) +{ + struct nfs4_layoutget *lgp = calldata; + struct nfs_server *server = NFS_SERVER(lgp->args.inode); + + nfs4_setup_sequence(server->nfs_client, &lgp->args.seq_args, + &lgp->res.seq_res, task); +} + +static void nfs4_layoutget_done(struct rpc_task *task, void *calldata) +{ + struct nfs4_layoutget *lgp = calldata; + + nfs41_sequence_process(task, &lgp->res.seq_res); +} + +static int +nfs4_layoutget_handle_exception(struct rpc_task *task, + struct nfs4_layoutget *lgp, struct nfs4_exception *exception) +{ + struct inode *inode = lgp->args.inode; + struct nfs_server *server = NFS_SERVER(inode); + struct pnfs_layout_hdr *lo = lgp->lo; + int nfs4err = task->tk_status; + int err, status = 0; + LIST_HEAD(head); + + dprintk("--> %s tk_status => %d\n", __func__, -task->tk_status); + + nfs4_sequence_free_slot(&lgp->res.seq_res); + + exception->state = NULL; + exception->stateid = NULL; + + switch (nfs4err) { + case 0: + goto out; + + /* + * NFS4ERR_LAYOUTUNAVAILABLE means we are not supposed to use pnfs + * on the file. set tk_status to -ENODATA to tell upper layer to + * retry go inband. + */ + case -NFS4ERR_LAYOUTUNAVAILABLE: + status = -ENODATA; + goto out; + /* + * NFS4ERR_BADLAYOUT means the MDS cannot return a layout of + * length lgp->args.minlength != 0 (see RFC5661 section 18.43.3). + */ + case -NFS4ERR_BADLAYOUT: + status = -EOVERFLOW; + goto out; + /* + * NFS4ERR_LAYOUTTRYLATER is a conflict with another client + * (or clients) writing to the same RAID stripe except when + * the minlength argument is 0 (see RFC5661 section 18.43.3). + * + * Treat it like we would RECALLCONFLICT -- we retry for a little + * while, and then eventually give up. + */ + case -NFS4ERR_LAYOUTTRYLATER: + if (lgp->args.minlength == 0) { + status = -EOVERFLOW; + goto out; + } + status = -EBUSY; + break; + case -NFS4ERR_RECALLCONFLICT: + case -NFS4ERR_RETURNCONFLICT: + status = -ERECALLCONFLICT; + break; + case -NFS4ERR_DELEG_REVOKED: + case -NFS4ERR_ADMIN_REVOKED: + case -NFS4ERR_EXPIRED: + case -NFS4ERR_BAD_STATEID: + exception->timeout = 0; + spin_lock(&inode->i_lock); + /* If the open stateid was bad, then recover it. */ + if (!lo || test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags) || + !nfs4_stateid_match_other(&lgp->args.stateid, &lo->plh_stateid)) { + spin_unlock(&inode->i_lock); + exception->state = lgp->args.ctx->state; + exception->stateid = &lgp->args.stateid; + break; + } + + /* + * Mark the bad layout state as invalid, then retry + */ + pnfs_mark_layout_stateid_invalid(lo, &head); + spin_unlock(&inode->i_lock); + nfs_commit_inode(inode, 0); + pnfs_free_lseg_list(&head); + status = -EAGAIN; + goto out; + } + + err = nfs4_handle_exception(server, nfs4err, exception); + if (!status) { + if (exception->retry) + status = -EAGAIN; + else + status = err; + } +out: + return status; +} + +size_t max_response_pages(struct nfs_server *server) +{ + u32 max_resp_sz = server->nfs_client->cl_session->fc_attrs.max_resp_sz; + return nfs_page_array_len(0, max_resp_sz); +} + +static void nfs4_layoutget_release(void *calldata) +{ + struct nfs4_layoutget *lgp = calldata; + + nfs4_sequence_free_slot(&lgp->res.seq_res); + pnfs_layoutget_free(lgp); +} + +static const struct rpc_call_ops nfs4_layoutget_call_ops = { + .rpc_call_prepare = nfs4_layoutget_prepare, + .rpc_call_done = nfs4_layoutget_done, + .rpc_release = nfs4_layoutget_release, +}; + +struct pnfs_layout_segment * +nfs4_proc_layoutget(struct nfs4_layoutget *lgp, + struct nfs4_exception *exception) +{ + struct inode *inode = lgp->args.inode; + struct nfs_server *server = NFS_SERVER(inode); + struct rpc_task *task; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_LAYOUTGET], + .rpc_argp = &lgp->args, + .rpc_resp = &lgp->res, + .rpc_cred = lgp->cred, + }; + struct rpc_task_setup task_setup_data = { + .rpc_client = server->client, + .rpc_message = &msg, + .callback_ops = &nfs4_layoutget_call_ops, + .callback_data = lgp, + .flags = RPC_TASK_ASYNC | RPC_TASK_CRED_NOREF | + RPC_TASK_MOVEABLE, + }; + struct pnfs_layout_segment *lseg = NULL; + int status = 0; + + nfs4_init_sequence(server->nfs_client, &lgp->args.seq_args, + &lgp->res.seq_res, 0, 0); + exception->retry = 0; + + task = rpc_run_task(&task_setup_data); + if (IS_ERR(task)) + return ERR_CAST(task); + + status = rpc_wait_for_completion_task(task); + if (status != 0) + goto out; + + if (task->tk_status < 0) { + exception->retry = 1; + status = nfs4_layoutget_handle_exception(task, lgp, exception); + } else if (lgp->res.layoutp->len == 0) { + exception->retry = 1; + status = -EAGAIN; + nfs4_update_delay(&exception->timeout); + } else + lseg = pnfs_layout_process(lgp); +out: + trace_nfs4_layoutget(lgp->args.ctx, + &lgp->args.range, + &lgp->res.range, + &lgp->res.stateid, + status); + + rpc_put_task(task); + dprintk("<-- %s status=%d\n", __func__, status); + if (status) + return ERR_PTR(status); + return lseg; +} + +static void +nfs4_layoutreturn_prepare(struct rpc_task *task, void *calldata) +{ + struct nfs4_layoutreturn *lrp = calldata; + + nfs4_setup_sequence(lrp->clp, + &lrp->args.seq_args, + &lrp->res.seq_res, + task); + if (!pnfs_layout_is_valid(lrp->args.layout)) + rpc_exit(task, 0); +} + +static void nfs4_layoutreturn_done(struct rpc_task *task, void *calldata) +{ + struct nfs4_layoutreturn *lrp = calldata; + struct nfs_server *server; + + if (!nfs41_sequence_process(task, &lrp->res.seq_res)) + return; + + if (task->tk_rpc_status < 0) { + switch (task->tk_rpc_status) { + case -EACCES: + case -EIO: + case -EKEYEXPIRED: + case -ERESTARTSYS: + case -EINTR: + lrp->rpc_status = 0; + break; + case -ENETDOWN: + case -ENETUNREACH: + if (task->tk_flags & RPC_TASK_NETUNREACH_FATAL) + lrp->rpc_status = 0; + else + lrp->rpc_status = -EAGAIN; + break; + default: + lrp->rpc_status = -EAGAIN; + break; + } + lrp->res.lrs_present = 0; + return; + } + + server = NFS_SERVER(lrp->args.inode); + switch (task->tk_status) { + case -NFS4ERR_OLD_STATEID: + if (nfs4_layout_refresh_old_stateid(&lrp->args.stateid, + &lrp->args.range, + lrp->args.inode)) + goto out_restart; + fallthrough; + default: + task->tk_status = 0; + lrp->res.lrs_present = 0; + fallthrough; + case 0: + break; + case -NFS4ERR_BADSESSION: + case -NFS4ERR_DEADSESSION: + case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION: + nfs4_schedule_session_recovery(server->nfs_client->cl_session, + task->tk_status); + lrp->res.lrs_present = 0; + lrp->rpc_status = -EAGAIN; + task->tk_status = 0; + break; + case -NFS4ERR_DELAY: + if (nfs4_async_handle_error(task, server, NULL, NULL) == + -EAGAIN) + goto out_restart; + lrp->res.lrs_present = 0; + break; + } + return; +out_restart: + task->tk_status = 0; + nfs4_sequence_free_slot(&lrp->res.seq_res); + rpc_restart_call_prepare(task); +} + +static void nfs4_layoutreturn_release(void *calldata) +{ + struct nfs4_layoutreturn *lrp = calldata; + struct pnfs_layout_hdr *lo = lrp->args.layout; + + if (lrp->rpc_status == 0 || !lrp->inode) + pnfs_layoutreturn_free_lsegs( + lo, &lrp->args.stateid, &lrp->args.range, + lrp->res.lrs_present ? &lrp->res.stateid : NULL); + else + pnfs_layoutreturn_retry_later(lo, &lrp->args.stateid, + &lrp->args.range); + nfs4_sequence_free_slot(&lrp->res.seq_res); + if (lrp->ld_private.ops && lrp->ld_private.ops->free) + lrp->ld_private.ops->free(&lrp->ld_private); + pnfs_put_layout_hdr(lrp->args.layout); + nfs_iput_and_deactive(lrp->inode); + put_cred(lrp->cred); + kfree(calldata); +} + +static const struct rpc_call_ops nfs4_layoutreturn_call_ops = { + .rpc_call_prepare = nfs4_layoutreturn_prepare, + .rpc_call_done = nfs4_layoutreturn_done, + .rpc_release = nfs4_layoutreturn_release, +}; + +int nfs4_proc_layoutreturn(struct nfs4_layoutreturn *lrp, unsigned int flags) +{ + struct nfs_client *clp = NFS_SERVER(lrp->args.inode)->nfs_client; + struct rpc_task *task; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_LAYOUTRETURN], + .rpc_argp = &lrp->args, + .rpc_resp = &lrp->res, + .rpc_cred = lrp->cred, + }; + struct rpc_task_setup task_setup_data = { + .rpc_client = NFS_SERVER(lrp->args.inode)->client, + .rpc_message = &msg, + .callback_ops = &nfs4_layoutreturn_call_ops, + .callback_data = lrp, + .flags = RPC_TASK_MOVEABLE, + }; + int status = 0; + + nfs4_state_protect(clp, NFS_SP4_MACH_CRED_PNFS_CLEANUP, + &task_setup_data.rpc_client, &msg); + + lrp->inode = nfs_igrab_and_active(lrp->args.inode); + if (flags & PNFS_FL_LAYOUTRETURN_ASYNC) { + if (!lrp->inode) { + nfs4_layoutreturn_release(lrp); + return -EAGAIN; + } + task_setup_data.flags |= RPC_TASK_ASYNC; + } + if (!lrp->inode) + flags |= PNFS_FL_LAYOUTRETURN_PRIVILEGED; + + nfs4_init_sequence(clp, &lrp->args.seq_args, &lrp->res.seq_res, 1, + flags & PNFS_FL_LAYOUTRETURN_PRIVILEGED ? 1 : 0); + task = rpc_run_task(&task_setup_data); + if (IS_ERR(task)) + return PTR_ERR(task); + if (!(flags & PNFS_FL_LAYOUTRETURN_ASYNC)) + status = task->tk_status; + trace_nfs4_layoutreturn(lrp->args.inode, &lrp->args.stateid, status); + dprintk("<-- %s status=%d\n", __func__, status); + rpc_put_task(task); + return status; +} + +static int +_nfs4_proc_getdeviceinfo(struct nfs_server *server, + struct pnfs_device *pdev, + const struct cred *cred) +{ + struct nfs4_getdeviceinfo_args args = { + .pdev = pdev, + .notify_types = NOTIFY_DEVICEID4_CHANGE | + NOTIFY_DEVICEID4_DELETE, + }; + struct nfs4_getdeviceinfo_res res = { + .pdev = pdev, + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_GETDEVICEINFO], + .rpc_argp = &args, + .rpc_resp = &res, + .rpc_cred = cred, + }; + int status; + + status = nfs4_call_sync(server->client, server, &msg, &args.seq_args, &res.seq_res, 0); + if (res.notification & ~args.notify_types) + dprintk("%s: unsupported notification\n", __func__); + if (res.notification != args.notify_types) + pdev->nocache = 1; + + trace_nfs4_getdeviceinfo(server, &pdev->dev_id, status); + + dprintk("<-- %s status=%d\n", __func__, status); + + return status; +} + +int nfs4_proc_getdeviceinfo(struct nfs_server *server, + struct pnfs_device *pdev, + const struct cred *cred) +{ + struct nfs4_exception exception = { }; + int err; + + do { + err = nfs4_handle_exception(server, + _nfs4_proc_getdeviceinfo(server, pdev, cred), + &exception); + } while (exception.retry); + return err; +} +EXPORT_SYMBOL_GPL(nfs4_proc_getdeviceinfo); + +static void nfs4_layoutcommit_prepare(struct rpc_task *task, void *calldata) +{ + struct nfs4_layoutcommit_data *data = calldata; + struct nfs_server *server = NFS_SERVER(data->args.inode); + + nfs4_setup_sequence(server->nfs_client, + &data->args.seq_args, + &data->res.seq_res, + task); +} + +static void +nfs4_layoutcommit_done(struct rpc_task *task, void *calldata) +{ + struct nfs4_layoutcommit_data *data = calldata; + struct nfs_server *server = NFS_SERVER(data->args.inode); + + if (!nfs41_sequence_done(task, &data->res.seq_res)) + return; + + switch (task->tk_status) { /* Just ignore these failures */ + case -NFS4ERR_DELEG_REVOKED: /* layout was recalled */ + case -NFS4ERR_BADIOMODE: /* no IOMODE_RW layout for range */ + case -NFS4ERR_BADLAYOUT: /* no layout */ + case -NFS4ERR_GRACE: /* loca_recalim always false */ + task->tk_status = 0; + break; + case -NFS4ERR_OLD_STATEID: { + u32 old_seqid = be32_to_cpu(data->args.stateid.seqid); + struct pnfs_layout_range range = { + .iomode = IOMODE_ANY, + .offset = 0, + .length = NFS4_MAX_UINT64, + }; + + if (nfs4_layout_refresh_old_stateid(&data->args.stateid, + &range, + data->args.inode)) { + struct pnfs_layout_hdr *lo; + + spin_lock(&data->args.inode->i_lock); + lo = NFS_I(data->args.inode)->layout; + if (lo && pnfs_layout_is_valid(lo) && + nfs4_stateid_match_other(&data->args.stateid, + &lo->plh_stateid)) + pnfs_set_layout_stateid(lo, &data->args.stateid, + NULL, false); + spin_unlock(&data->args.inode->i_lock); + + dprintk("%s: refreshed OLD_STATEID inode %llu seq %u->%u\n", + __func__, data->args.inode->i_ino, + old_seqid, + be32_to_cpu(data->args.stateid.seqid)); + + rpc_restart_call_prepare(task); + return; + } + fallthrough; + } + case 0: + break; + default: + if (nfs4_async_handle_error(task, server, NULL, NULL) == -EAGAIN) { + rpc_restart_call_prepare(task); + return; + } + } +} + +static void nfs4_layoutcommit_release(void *calldata) +{ + struct nfs4_layoutcommit_data *data = calldata; + + pnfs_cleanup_layoutcommit(data); + nfs_post_op_update_inode_force_wcc(data->args.inode, + data->res.fattr); + put_cred(data->cred); + nfs_iput_and_deactive(data->inode); + kfree(data); +} + +static const struct rpc_call_ops nfs4_layoutcommit_ops = { + .rpc_call_prepare = nfs4_layoutcommit_prepare, + .rpc_call_done = nfs4_layoutcommit_done, + .rpc_release = nfs4_layoutcommit_release, +}; + +int +nfs4_proc_layoutcommit(struct nfs4_layoutcommit_data *data, bool sync) +{ + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_LAYOUTCOMMIT], + .rpc_argp = &data->args, + .rpc_resp = &data->res, + .rpc_cred = data->cred, + }; + struct rpc_task_setup task_setup_data = { + .task = &data->task, + .rpc_client = NFS_CLIENT(data->args.inode), + .rpc_message = &msg, + .callback_ops = &nfs4_layoutcommit_ops, + .callback_data = data, + .flags = RPC_TASK_MOVEABLE, + }; + struct rpc_task *task; + int status = 0; + + dprintk("NFS: initiating layoutcommit call. sync %d " + "lbw: %llu inode %llu\n", sync, + data->args.lastbytewritten, + data->args.inode->i_ino); + + if (!sync) { + data->inode = nfs_igrab_and_active(data->args.inode); + if (data->inode == NULL) { + nfs4_layoutcommit_release(data); + return -EAGAIN; + } + task_setup_data.flags = RPC_TASK_ASYNC; + } + nfs4_init_sequence(NFS_SERVER(data->args.inode)->nfs_client, + &data->args.seq_args, &data->res.seq_res, 1, 0); + task = rpc_run_task(&task_setup_data); + if (IS_ERR(task)) + return PTR_ERR(task); + if (sync) + status = task->tk_status; + trace_nfs4_layoutcommit(data->args.inode, &data->args.stateid, status); + dprintk("%s: status %d\n", __func__, status); + rpc_put_task(task); + return status; +} + +/* + * Use the state managment nfs_client cl_rpcclient, which uses krb5i (if + * possible) as per RFC3530bis and RFC5661 Security Considerations sections + */ +static int _nfs41_proc_secinfo_no_name(struct nfs_server *server, + struct nfs_fh *fhandle, + struct nfs4_secinfo_flavors *flavors, + bool use_integrity) +{ + struct nfs_client *clp = server->nfs_client; + struct nfs41_secinfo_no_name_args args = { + .style = SECINFO_STYLE_CURRENT_FH, + }; + struct nfs4_secinfo_res res = { + .flavors = flavors, + }; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_SECINFO_NO_NAME], + .rpc_argp = &args, + .rpc_resp = &res, + }; + struct nfs4_call_sync_data data = { + .seq_server = server, + .seq_args = &args.seq_args, + .seq_res = &res.seq_res, + }; + struct rpc_task_setup task_setup = { + .rpc_client = server->client, + .rpc_message = &msg, + .callback_ops = clp->cl_mvops->call_sync_ops, + .callback_data = &data, + .flags = RPC_TASK_NO_ROUND_ROBIN, + }; + const struct cred *cred = NULL; + int status; + + if (use_integrity) { + task_setup.rpc_client = clp->cl_rpcclient; + + cred = nfs4_get_clid_cred(clp); + msg.rpc_cred = cred; + } + + nfs4_init_sequence(clp, &args.seq_args, &res.seq_res, 0, 0); + status = nfs4_call_sync_custom(&task_setup); + dprintk("<-- %s status=%d\n", __func__, status); + + put_cred(cred); + + return status; +} + +static int nfs41_proc_secinfo_no_name(struct nfs_server *server, + struct nfs_fh *fhandle, + struct nfs4_secinfo_flavors *flavors) +{ + struct nfs4_exception exception = { + .interruptible = true, + }; + int err; + do { + /* first try using integrity protection */ + err = -NFS4ERR_WRONGSEC; + + /* try to use integrity protection with machine cred */ + if (_nfs4_is_integrity_protected(server->nfs_client)) + err = _nfs41_proc_secinfo_no_name(server, fhandle, + flavors, true); + + /* + * if unable to use integrity protection, or SECINFO with + * integrity protection returns NFS4ERR_WRONGSEC (which is + * disallowed by spec, but exists in deployed servers) use + * the current filesystem's rpc_client and the user cred. + */ + if (err == -NFS4ERR_WRONGSEC) + err = _nfs41_proc_secinfo_no_name(server, fhandle, + flavors, false); + + switch (err) { + case 0: + case -NFS4ERR_WRONGSEC: + case -ENOTSUPP: + goto out; + default: + err = nfs4_handle_exception(server, err, &exception); + } + } while (exception.retry); +out: + return err; +} + +static int nfs41_find_root_sec(struct nfs_server *server, + struct nfs_fh *fhandle, struct nfs_fattr *fattr) +{ + int err; + struct page *page; + rpc_authflavor_t flavor = RPC_AUTH_MAXFLAVOR; + struct nfs4_secinfo_flavors *flavors; + struct nfs4_secinfo4 *secinfo; + int i; + + page = alloc_page(GFP_KERNEL); + if (!page) { + err = -ENOMEM; + goto out; + } + + flavors = page_address(page); + err = nfs41_proc_secinfo_no_name(server, fhandle, flavors); + + /* + * Fall back on "guess and check" method if + * the server doesn't support SECINFO_NO_NAME + */ + if (err == -NFS4ERR_WRONGSEC || err == -ENOTSUPP) { + err = nfs4_find_root_sec(server, fhandle, fattr); + goto out_freepage; + } + if (err) + goto out_freepage; + + for (i = 0; i < flavors->num_flavors; i++) { + secinfo = &flavors->flavors[i]; + + switch (secinfo->flavor) { + case RPC_AUTH_NULL: + case RPC_AUTH_UNIX: + case RPC_AUTH_GSS: + flavor = rpcauth_get_pseudoflavor(secinfo->flavor, + &secinfo->flavor_info); + break; + default: + flavor = RPC_AUTH_MAXFLAVOR; + break; + } + + if (!nfs_auth_info_match(&server->auth_info, flavor)) + flavor = RPC_AUTH_MAXFLAVOR; + + if (flavor != RPC_AUTH_MAXFLAVOR) { + err = nfs4_lookup_root_sec(server, fhandle, fattr, + flavor); + if (!err) + break; + } + } + + if (flavor == RPC_AUTH_MAXFLAVOR) + err = -EPERM; + +out_freepage: + put_page(page); + if (err == -EACCES) + return -EPERM; +out: + return err; +} + +static int _nfs41_test_stateid(struct nfs_server *server, + const nfs4_stateid *stateid, + const struct cred *cred) +{ + int status; + struct nfs41_test_stateid_args args = { + .stateid = *stateid, + }; + struct nfs41_test_stateid_res res; + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_TEST_STATEID], + .rpc_argp = &args, + .rpc_resp = &res, + .rpc_cred = cred, + }; + struct rpc_clnt *rpc_client = server->client; + struct nfs_client *clp = server->nfs_client; + + nfs4_state_protect(clp, NFS_SP4_MACH_CRED_STATEID, &rpc_client, &msg); + + dprintk("NFS call test_stateid %p\n", stateid); + nfs4_init_sequence(clp, &args.seq_args, &res.seq_res, 0, 1); + status = nfs4_call_sync_sequence(rpc_client, server, &msg, + &args.seq_args, &res.seq_res); + if (status != NFS_OK) { + dprintk("NFS reply test_stateid: failed, %d\n", status); + return status; + } + dprintk("NFS reply test_stateid: succeeded, %d\n", -res.status); + return -res.status; +} + +static void nfs4_handle_delay_or_session_error(struct nfs_server *server, + int err, struct nfs4_exception *exception) +{ + exception->retry = 0; + switch(err) { + case -NFS4ERR_DELAY: + case -NFS4ERR_RETRY_UNCACHED_REP: + nfs4_handle_exception(server, err, exception); + break; + case -NFS4ERR_BADSESSION: + case -NFS4ERR_BADSLOT: + case -NFS4ERR_BAD_HIGH_SLOT: + case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION: + case -NFS4ERR_DEADSESSION: + nfs4_do_handle_exception(server, err, exception); + } +} + +/** + * nfs41_test_stateid - perform a TEST_STATEID operation + * + * @server: server / transport on which to perform the operation + * @stateid: state ID to test + * @cred: credential + * + * Returns NFS_OK if the server recognizes that "stateid" is valid. + * Otherwise a negative NFS4ERR value is returned if the operation + * failed or the state ID is not currently valid. + */ +static int nfs41_test_stateid(struct nfs_server *server, + const nfs4_stateid *stateid, + const struct cred *cred) +{ + struct nfs4_exception exception = { + .interruptible = true, + }; + int err; + do { + err = _nfs41_test_stateid(server, stateid, cred); + nfs4_handle_delay_or_session_error(server, err, &exception); + } while (exception.retry); + return err; +} + +struct nfs_free_stateid_data { + struct nfs_server *server; + struct nfs41_free_stateid_args args; + struct nfs41_free_stateid_res res; +}; + +static void nfs41_free_stateid_prepare(struct rpc_task *task, void *calldata) +{ + struct nfs_free_stateid_data *data = calldata; + nfs4_setup_sequence(data->server->nfs_client, + &data->args.seq_args, + &data->res.seq_res, + task); +} + +static void nfs41_free_stateid_done(struct rpc_task *task, void *calldata) +{ + struct nfs_free_stateid_data *data = calldata; + + nfs41_sequence_done(task, &data->res.seq_res); + + switch (task->tk_status) { + case -NFS4ERR_DELAY: + if (nfs4_async_handle_error(task, data->server, NULL, NULL) == -EAGAIN) + rpc_restart_call_prepare(task); + } +} + +static void nfs41_free_stateid_release(void *calldata) +{ + struct nfs_free_stateid_data *data = calldata; + struct nfs_client *clp = data->server->nfs_client; + + nfs_sb_deactive(data->server->super); + nfs_put_client(clp); + kfree(calldata); +} + +static const struct rpc_call_ops nfs41_free_stateid_ops = { + .rpc_call_prepare = nfs41_free_stateid_prepare, + .rpc_call_done = nfs41_free_stateid_done, + .rpc_release = nfs41_free_stateid_release, +}; + +/** + * nfs41_free_stateid - perform a FREE_STATEID operation + * + * @server: server / transport on which to perform the operation + * @stateid: state ID to release + * @cred: credential + * @privileged: set to true if this call needs to be privileged + * + * Note: this function is always asynchronous. + */ +static int nfs41_free_stateid(struct nfs_server *server, + nfs4_stateid *stateid, + const struct cred *cred, + bool privileged) +{ + struct rpc_message msg = { + .rpc_proc = &nfs4_procedures[NFSPROC4_CLNT_FREE_STATEID], + .rpc_cred = cred, + }; + struct rpc_task_setup task_setup = { + .rpc_client = server->client, + .rpc_message = &msg, + .callback_ops = &nfs41_free_stateid_ops, + .flags = RPC_TASK_ASYNC | RPC_TASK_MOVEABLE, + }; + struct nfs_free_stateid_data *data; + struct rpc_task *task; + struct nfs_client *clp = server->nfs_client; + int ret = -EIO; + + if (!refcount_inc_not_zero(&clp->cl_count)) + return ret; + if (!nfs_sb_active(server->super)) + goto out_put_clp; + + nfs4_state_protect(clp, NFS_SP4_MACH_CRED_STATEID, + &task_setup.rpc_client, &msg); + + dprintk("NFS call free_stateid %p\n", stateid); + data = kmalloc_obj(*data); + if (!data) { + ret = -ENOMEM; + goto out_put_server; + } + data->server = server; + nfs4_stateid_copy(&data->args.stateid, stateid); + + task_setup.callback_data = data; + + msg.rpc_argp = &data->args; + msg.rpc_resp = &data->res; + nfs4_init_sequence(clp, &data->args.seq_args, &data->res.seq_res, 1, + privileged); + task = rpc_run_task(&task_setup); + if (IS_ERR(task)) + return PTR_ERR(task); + rpc_put_task(task); + stateid->type = NFS4_FREED_STATEID_TYPE; + return 0; +out_put_server: + nfs_sb_deactive(server->super); +out_put_clp: + nfs_put_client(clp); + return ret; +} + +static void +nfs41_free_lock_state(struct nfs_server *server, struct nfs4_lock_state *lsp) +{ + const struct cred *cred = lsp->ls_state->owner->so_cred; + + nfs41_free_stateid(server, &lsp->ls_stateid, cred, false); + nfs4_free_lock_state(server, lsp); +} + +static bool nfs41_match_stateid(const nfs4_stateid *s1, + const nfs4_stateid *s2) +{ + trace_nfs41_match_stateid(s1, s2); + + if (s1->type != s2->type) + return false; + + if (memcmp(s1->other, s2->other, sizeof(s1->other)) != 0) + return false; + + if (s1->seqid == s2->seqid) + return true; + + return s1->seqid == 0 || s2->seqid == 0; +} + +bool nfs4_match_stateid(const nfs4_stateid *s1, + const nfs4_stateid *s2) +{ + trace_nfs4_match_stateid(s1, s2); + + return nfs4_stateid_match(s1, s2); +} + + +static const struct nfs4_sequence_slot_ops nfs41_sequence_slot_ops = { + .process = nfs41_sequence_process, + .done = nfs41_sequence_done, + .free_slot = nfs41_sequence_free_slot, +}; + +static const struct nfs4_state_recovery_ops nfs41_reboot_recovery_ops = { + .owner_flag_bit = NFS_OWNER_RECLAIM_REBOOT, + .state_flag_bit = NFS_STATE_RECLAIM_REBOOT, + .recover_open = nfs4_open_reclaim, + .recover_lock = nfs4_lock_reclaim, + .establish_clid = nfs41_init_clientid, + .reclaim_complete = nfs41_proc_reclaim_complete, + .detect_trunking = nfs41_discover_server_trunking, +}; + +static const struct nfs4_state_recovery_ops nfs41_nograce_recovery_ops = { + .owner_flag_bit = NFS_OWNER_RECLAIM_NOGRACE, + .state_flag_bit = NFS_STATE_RECLAIM_NOGRACE, + .recover_open = nfs41_open_expired, + .recover_lock = nfs41_lock_expired, + .establish_clid = nfs41_init_clientid, +}; + +static const struct nfs4_state_maintenance_ops nfs41_state_renewal_ops = { + .sched_state_renewal = nfs41_proc_async_sequence, + .get_state_renewal_cred = nfs4_get_machine_cred, + .renew_lease = nfs4_proc_sequence, +}; + +static const struct nfs4_mig_recovery_ops nfs41_mig_recovery_ops = { + .get_locations = _nfs41_proc_get_locations, + .fsid_present = _nfs41_proc_fsid_present, +}; + +static struct nfs_seqid * +nfs_alloc_no_seqid(struct nfs_seqid_counter *arg1, gfp_t arg2) +{ + return NULL; +} + +static const struct nfs4_minor_version_ops nfs_v4_1_minor_ops = { + .minor_version = 1, + .init_caps = NFS_CAP_READDIRPLUS + | NFS_CAP_ATOMIC_OPEN + | NFS_CAP_DIR_DELEG + | NFS_CAP_POSIX_LOCK + | NFS_CAP_STATEID_NFSV41 + | NFS_CAP_ATOMIC_OPEN_V1 + | NFS_CAP_LGOPEN + | NFS_CAP_MOVEABLE, + .init_client = nfs41_init_client, + .shutdown_client = nfs41_shutdown_client, + .match_stateid = nfs41_match_stateid, + .find_root_sec = nfs41_find_root_sec, + .free_lock_state = nfs41_free_lock_state, + .test_and_free_expired = nfs41_test_and_free_expired_stateid, + .alloc_seqid = nfs_alloc_no_seqid, + .session_trunk = nfs4_test_session_trunk, + .call_sync_ops = &nfs41_call_sync_ops, + .sequence_slot_ops = &nfs41_sequence_slot_ops, + .reboot_recovery_ops = &nfs41_reboot_recovery_ops, + .nograce_recovery_ops = &nfs41_nograce_recovery_ops, + .state_renewal_ops = &nfs41_state_renewal_ops, + .mig_recovery_ops = &nfs41_mig_recovery_ops, +}; + +#if defined(CONFIG_NFS_V4_2) +static const struct nfs4_minor_version_ops nfs_v4_2_minor_ops = { + .minor_version = 2, + .init_caps = NFS_CAP_READDIRPLUS + | NFS_CAP_ATOMIC_OPEN + | NFS_CAP_DIR_DELEG + | NFS_CAP_POSIX_LOCK + | NFS_CAP_STATEID_NFSV41 + | NFS_CAP_ATOMIC_OPEN_V1 + | NFS_CAP_LGOPEN + | NFS_CAP_ALLOCATE + | NFS_CAP_COPY + | NFS_CAP_OFFLOAD_CANCEL + | NFS_CAP_COPY_NOTIFY + | NFS_CAP_DEALLOCATE + | NFS_CAP_ZERO_RANGE + | NFS_CAP_SEEK + | NFS_CAP_LAYOUTSTATS + | NFS_CAP_CLONE + | NFS_CAP_LAYOUTERROR + | NFS_CAP_READ_PLUS + | NFS_CAP_MOVEABLE + | NFS_CAP_OFFLOAD_STATUS, + .init_client = nfs41_init_client, + .shutdown_client = nfs41_shutdown_client, + .match_stateid = nfs41_match_stateid, + .find_root_sec = nfs41_find_root_sec, + .free_lock_state = nfs41_free_lock_state, + .call_sync_ops = &nfs41_call_sync_ops, + .sequence_slot_ops = &nfs41_sequence_slot_ops, + .test_and_free_expired = nfs41_test_and_free_expired_stateid, + .alloc_seqid = nfs_alloc_no_seqid, + .session_trunk = nfs4_test_session_trunk, + .reboot_recovery_ops = &nfs41_reboot_recovery_ops, + .nograce_recovery_ops = &nfs41_nograce_recovery_ops, + .state_renewal_ops = &nfs41_state_renewal_ops, + .mig_recovery_ops = &nfs41_mig_recovery_ops, +}; +#endif + +const struct nfs4_minor_version_ops *nfs_v4_minor_ops[] = { +#if defined(CONFIG_NFS_V4_0) + [0] = &nfs_v4_0_minor_ops, +#endif /* CONFIG_NFS_V4_0 */ + [1] = &nfs_v4_1_minor_ops, +#if defined(CONFIG_NFS_V4_2) + [2] = &nfs_v4_2_minor_ops, +#endif +}; + +static ssize_t nfs4_listxattr(struct dentry *dentry, char *list, size_t size) +{ + ssize_t error, error2, error3; + ssize_t left = size; + ssize_t left2; + + error = generic_listxattr(dentry, list, left); + if (error < 0) + return error; + if (list) { + list += error; + left -= error; + } + + left2 = left; + error2 = security_inode_listsecurity(d_inode(dentry), &list, &left2); + if (error2 < 0) + return error2; + error2 = left - left2; + if (list) + left -= error2; + + error3 = nfs4_listxattr_nfs4_user(d_inode(dentry), list, left); + if (error3 < 0) + return error3; + + error += error2 + error3; + if (size && error > size) + return -ERANGE; + return error; +} + +static void nfs4_enable_swap(struct inode *inode) +{ + /* The state manager thread must always be running. + * It will notice the client is a swapper, and stay put. + */ + struct nfs_client *clp = NFS_SERVER(inode)->nfs_client; + + nfs4_schedule_state_manager(clp); +} + +static void nfs4_disable_swap(struct inode *inode) +{ + /* The state manager thread will now exit once it is + * woken. + */ + struct nfs_client *clp = NFS_SERVER(inode)->nfs_client; + + set_bit(NFS4CLNT_RUN_MANAGER, &clp->cl_state); + clear_bit(NFS4CLNT_MANAGER_AVAILABLE, &clp->cl_state); + wake_up_var(&clp->cl_state); +} + +static const struct inode_operations nfs4_dir_inode_operations = { + .create = nfs_create, + .lookup = nfs_lookup, + .atomic_open = nfs_atomic_open, + .link = nfs_link, + .unlink = nfs_unlink, + .symlink = nfs_symlink, + .mkdir = nfs_mkdir, + .rmdir = nfs_rmdir, + .mknod = nfs_mknod, + .rename = nfs_rename, + .permission = nfs_permission, + .getattr = nfs_getattr, + .setattr = nfs_setattr, + .listxattr = nfs4_listxattr, + .fileattr_get = nfs_fileattr_get, +}; + +static const struct inode_operations nfs4_file_inode_operations = { + .permission = nfs_permission, + .getattr = nfs_getattr, + .setattr = nfs_setattr, + .listxattr = nfs4_listxattr, + .fileattr_get = nfs_fileattr_get, +}; + +static struct nfs_server *nfs4_clone_server(struct nfs_server *source, + struct nfs_fh *fh, struct nfs_fattr *fattr, + rpc_authflavor_t flavor) +{ + struct nfs_server *server; + int error; + + server = nfs_clone_server(source, fh, fattr, flavor); + if (IS_ERR(server)) + return server; + + nfs4_session_limit_rwsize(server); + nfs4_session_limit_xasize(server); + + error = nfs4_delegation_hash_alloc(server); + if (error) { + nfs_free_server(server); + return ERR_PTR(error); + } + + return server; +} + +const struct nfs_rpc_ops nfs_v4_clientops = { + .version = 4, /* protocol version */ + .dentry_ops = &nfs4_dentry_operations, + .dir_inode_ops = &nfs4_dir_inode_operations, + .file_inode_ops = &nfs4_file_inode_operations, + .file_ops = &nfs4_file_operations, + .getroot = nfs4_proc_get_root, + .submount = nfs4_submount, + .try_get_tree = nfs4_try_get_tree, + .getattr = nfs4_proc_getattr, + .setattr = nfs4_proc_setattr, + .lookup = nfs4_proc_lookup, + .lookupp = nfs4_proc_lookupp, + .access = nfs4_proc_access, + .readlink = nfs4_proc_readlink, + .create = nfs4_proc_create, + .remove = nfs4_proc_remove, + .unlink_setup = nfs4_proc_unlink_setup, + .unlink_rpc_prepare = nfs4_proc_unlink_rpc_prepare, + .unlink_done = nfs4_proc_unlink_done, + .rename_setup = nfs4_proc_rename_setup, + .rename_rpc_prepare = nfs4_proc_rename_rpc_prepare, + .rename_done = nfs4_proc_rename_done, + .link = nfs4_proc_link, + .symlink = nfs4_proc_symlink, + .mkdir = nfs4_proc_mkdir, + .rmdir = nfs4_proc_rmdir, + .readdir = nfs4_proc_readdir, + .mknod = nfs4_proc_mknod, + .statfs = nfs4_proc_statfs, + .fsinfo = nfs4_proc_fsinfo, + .pathconf = nfs4_proc_pathconf, + .set_capabilities = nfs4_server_capabilities, + .decode_dirent = nfs4_decode_dirent, + .pgio_rpc_prepare = nfs4_proc_pgio_rpc_prepare, + .read_setup = nfs4_proc_read_setup, + .read_done = nfs4_read_done, + .write_setup = nfs4_proc_write_setup, + .write_done = nfs4_write_done, + .commit_setup = nfs4_proc_commit_setup, + .commit_rpc_prepare = nfs4_proc_commit_rpc_prepare, + .commit_done = nfs4_commit_done, + .lock = nfs4_proc_lock, + .clear_acl_cache = nfs4_zap_acl_attr, + .close_context = nfs4_close_context, + .open_context = nfs4_atomic_open, + .have_delegation = nfs4_have_delegation, + .return_delegation = nfs4_inode_return_delegation, + .alloc_client = nfs4_alloc_client, + .init_client = nfs4_init_client, + .free_client = nfs4_free_client, + .create_server = nfs4_create_server, + .clone_server = nfs4_clone_server, + .discover_trunking = nfs4_discover_trunking, + .enable_swap = nfs4_enable_swap, + .disable_swap = nfs4_disable_swap, +}; + +static const struct xattr_handler nfs4_xattr_nfs4_acl_handler = { + .name = XATTR_NAME_NFSV4_ACL, + .list = nfs4_xattr_list_nfs4_acl, + .get = nfs4_xattr_get_nfs4_acl, + .set = nfs4_xattr_set_nfs4_acl, +}; + +static const struct xattr_handler nfs4_xattr_nfs4_dacl_handler = { + .name = XATTR_NAME_NFSV4_DACL, + .list = nfs4_xattr_list_nfs4_dacl, + .get = nfs4_xattr_get_nfs4_dacl, + .set = nfs4_xattr_set_nfs4_dacl, +}; + +static const struct xattr_handler nfs4_xattr_nfs4_sacl_handler = { + .name = XATTR_NAME_NFSV4_SACL, + .list = nfs4_xattr_list_nfs4_sacl, + .get = nfs4_xattr_get_nfs4_sacl, + .set = nfs4_xattr_set_nfs4_sacl, +}; + +#ifdef CONFIG_NFS_V4_2 +static const struct xattr_handler nfs4_xattr_nfs4_user_handler = { + .prefix = XATTR_USER_PREFIX, + .get = nfs4_xattr_get_nfs4_user, + .set = nfs4_xattr_set_nfs4_user, +}; +#endif + +const struct xattr_handler * const nfs4_xattr_handlers[] = { + &nfs4_xattr_nfs4_acl_handler, + &nfs4_xattr_nfs4_dacl_handler, + &nfs4_xattr_nfs4_sacl_handler, +#ifdef CONFIG_NFS_V4_SECURITY_LABEL + &nfs4_xattr_nfs4_label_handler, +#endif +#ifdef CONFIG_NFS_V4_2 + &nfs4_xattr_nfs4_user_handler, +#endif + NULL +}; diff --git a/fs/nfs/nfs4renewd.c b/fs/nfs/nfs4renewd.c new file mode 100644 index 000000000..30065df14 --- /dev/null +++ b/fs/nfs/nfs4renewd.c @@ -0,0 +1,160 @@ +/* + * fs/nfs/nfs4renewd.c + * + * Copyright (c) 2002 The Regents of the University of Michigan. + * All rights reserved. + * + * Kendrick Smith + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Implementation of the NFSv4 "renew daemon", which wakes up periodically to + * send a RENEW, to keep state alive on the server. The daemon is implemented + * as an rpc_task, not a real kernel thread, so it always runs in rpciod's + * context. There is one renewd per nfs_server. + * + */ + +#include +#include +#include +#include + +#include +#include +#include +#include "nfs4_fs.h" +#include "delegation.h" + +#define NFSDBG_FACILITY NFSDBG_STATE + +void +nfs4_renew_state(struct work_struct *work) +{ + const struct nfs4_state_maintenance_ops *ops; + struct nfs_client *clp = + container_of(work, struct nfs_client, cl_renewd.work); + const struct cred *cred; + long lease; + unsigned long last, now; + unsigned renew_flags = 0; + + ops = clp->cl_mvops->state_renewal_ops; + dprintk("%s: start\n", __func__); + + if (test_bit(NFS_CS_STOP_RENEW, &clp->cl_res_state)) + goto out; + + lease = clp->cl_lease_time; + last = clp->cl_last_renewal; + now = jiffies; + /* Are we close to a lease timeout? */ + if (time_after(now, last + lease/3)) + renew_flags |= NFS4_RENEW_TIMEOUT; + if (nfs_delegations_present(clp)) + renew_flags |= NFS4_RENEW_DELEGATION_CB; + + if (renew_flags != 0) { + cred = ops->get_state_renewal_cred(clp); + if (cred == NULL) { + if (!(renew_flags & NFS4_RENEW_DELEGATION_CB)) { + set_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state); + goto out; + } + nfs_expire_all_delegations(clp); + } else { + int ret; + + /* Queue an asynchronous RENEW. */ + ret = ops->sched_state_renewal(clp, cred, renew_flags); + put_cred(cred); + switch (ret) { + default: + goto out_exp; + case -EAGAIN: + case -ENOMEM: + break; + } + } + } else { + dprintk("%s: failed to call renewd. Reason: lease not expired \n", + __func__); + } + nfs4_schedule_state_renewal(clp); +out_exp: + nfs_expire_unreferenced_delegations(clp); +out: + dprintk("%s: done\n", __func__); +} + +void +nfs4_schedule_state_renewal(struct nfs_client *clp) +{ + long timeout; + + spin_lock(&clp->cl_lock); + timeout = (2 * clp->cl_lease_time) / 3 + (long)clp->cl_last_renewal + - (long)jiffies; + if (timeout < 5 * HZ) + timeout = 5 * HZ; + dprintk("%s: requeueing work. Lease period = %ld\n", + __func__, (timeout + HZ - 1) / HZ); + mod_delayed_work(system_percpu_wq, &clp->cl_renewd, timeout); + set_bit(NFS_CS_RENEWD, &clp->cl_res_state); + spin_unlock(&clp->cl_lock); +} + +void +nfs4_kill_renewd(struct nfs_client *clp) +{ + cancel_delayed_work_sync(&clp->cl_renewd); +} + +#define MAX_LEASE_PERIOD (60 * 60) /* 1 hour */ + +/** + * nfs4_set_lease_period - Sets the lease period on a nfs_client + * + * @clp: pointer to nfs_client + * @period: new value for lease period (in seconds) + */ +void nfs4_set_lease_period(struct nfs_client *clp, u32 period) +{ + unsigned long lease; + + /* Limit the lease period */ + if (period < MAX_LEASE_PERIOD) + lease = period * HZ; + else + lease = MAX_LEASE_PERIOD * HZ; + + spin_lock(&clp->cl_lock); + clp->cl_lease_time = lease; + spin_unlock(&clp->cl_lock); + + /* Cap maximum reconnect timeout at 1/2 lease period */ + rpc_set_connect_timeout(clp->cl_rpcclient, lease, lease >> 1); +} diff --git a/fs/nfs/nfs4session.c b/fs/nfs/nfs4session.c new file mode 100644 index 000000000..5c128957a --- /dev/null +++ b/fs/nfs/nfs4session.c @@ -0,0 +1,653 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * fs/nfs/nfs4session.c + * + * Copyright (c) 2012 Trond Myklebust + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nfs4_fs.h" +#include "internal.h" +#include "nfs4session.h" +#include "callback.h" + +#define NFSDBG_FACILITY NFSDBG_STATE + +static void nfs4_init_slot_table(struct nfs4_slot_table *tbl, const char *queue) +{ + tbl->highest_used_slotid = NFS4_NO_SLOT; + spin_lock_init(&tbl->slot_tbl_lock); + rpc_init_priority_wait_queue(&tbl->slot_tbl_waitq, queue); + init_waitqueue_head(&tbl->slot_waitq); + init_completion(&tbl->complete); +} + +/* + * nfs4_shrink_slot_table - free retired slots from the slot table + */ +static void nfs4_shrink_slot_table(struct nfs4_slot_table *tbl, u32 newsize) +{ + struct nfs4_slot **p; + if (newsize >= tbl->max_slots) + return; + + p = &tbl->slots; + while (newsize--) + p = &(*p)->next; + while (*p) { + struct nfs4_slot *slot = *p; + + *p = slot->next; + kfree(slot); + tbl->max_slots--; + } +} + +/** + * nfs4_slot_tbl_drain_complete - wake waiters when drain is complete + * @tbl: controlling slot table + * + */ +void nfs4_slot_tbl_drain_complete(struct nfs4_slot_table *tbl) +{ + if (nfs4_slot_tbl_draining(tbl)) + complete(&tbl->complete); +} + +/* + * nfs4_free_slot - free a slot and efficiently update slot table. + * + * freeing a slot is trivially done by clearing its respective bit + * in the bitmap. + * If the freed slotid equals highest_used_slotid we want to update it + * so that the server would be able to size down the slot table if needed, + * otherwise we know that the highest_used_slotid is still in use. + * When updating highest_used_slotid there may be "holes" in the bitmap + * so we need to scan down from highest_used_slotid to 0 looking for the now + * highest slotid in use. + * If none found, highest_used_slotid is set to NFS4_NO_SLOT. + * + * Must be called while holding tbl->slot_tbl_lock + */ +void nfs4_free_slot(struct nfs4_slot_table *tbl, struct nfs4_slot *slot) +{ + u32 slotid = slot->slot_nr; + + /* clear used bit in bitmap */ + __clear_bit(slotid, tbl->used_slots); + + /* update highest_used_slotid when it is freed */ + if (slotid == tbl->highest_used_slotid) { + u32 new_max = find_last_bit(tbl->used_slots, slotid); + if (new_max < slotid) + tbl->highest_used_slotid = new_max; + else { + tbl->highest_used_slotid = NFS4_NO_SLOT; + nfs4_slot_tbl_drain_complete(tbl); + } + } + dprintk("%s: slotid %u highest_used_slotid %u\n", __func__, + slotid, tbl->highest_used_slotid); +} + +static struct nfs4_slot *nfs4_new_slot(struct nfs4_slot_table *tbl, + u32 slotid, u32 seq_init, gfp_t gfp_mask) +{ + struct nfs4_slot *slot; + + slot = kzalloc_obj(*slot, gfp_mask); + if (slot) { + slot->table = tbl; + slot->slot_nr = slotid; + slot->seq_nr = seq_init; + slot->seq_nr_highest_sent = seq_init; + slot->seq_nr_last_acked = seq_init - 1; + } + return slot; +} + +static struct nfs4_slot *nfs4_find_or_create_slot(struct nfs4_slot_table *tbl, + u32 slotid, u32 seq_init, gfp_t gfp_mask) +{ + struct nfs4_slot **p, *slot; + + p = &tbl->slots; + for (;;) { + if (*p == NULL) { + *p = nfs4_new_slot(tbl, tbl->max_slots, + seq_init, gfp_mask); + if (*p == NULL) + break; + tbl->max_slots++; + } + slot = *p; + if (slot->slot_nr == slotid) + return slot; + p = &slot->next; + } + return ERR_PTR(-ENOMEM); +} + +static void nfs4_lock_slot(struct nfs4_slot_table *tbl, + struct nfs4_slot *slot) +{ + u32 slotid = slot->slot_nr; + + __set_bit(slotid, tbl->used_slots); + if (slotid > tbl->highest_used_slotid || + tbl->highest_used_slotid == NFS4_NO_SLOT) + tbl->highest_used_slotid = slotid; + slot->generation = tbl->generation; +} + +/* + * nfs4_try_to_lock_slot - Given a slot try to allocate it + * + * Note: must be called with the slot_tbl_lock held. + */ +bool nfs4_try_to_lock_slot(struct nfs4_slot_table *tbl, struct nfs4_slot *slot) +{ + if (nfs4_test_locked_slot(tbl, slot->slot_nr)) + return false; + nfs4_lock_slot(tbl, slot); + return true; +} + +/* + * nfs4_lookup_slot - Find a slot but don't allocate it + * + * Note: must be called with the slot_tbl_lock held. + */ +struct nfs4_slot *nfs4_lookup_slot(struct nfs4_slot_table *tbl, u32 slotid) +{ + if (slotid <= tbl->max_slotid) + return nfs4_find_or_create_slot(tbl, slotid, 0, GFP_NOWAIT); + return ERR_PTR(-E2BIG); +} + +static int nfs4_slot_get_seqid(struct nfs4_slot_table *tbl, u32 slotid, + u32 *seq_nr) + __must_hold(&tbl->slot_tbl_lock) +{ + struct nfs4_slot *slot; + int ret; + + slot = nfs4_lookup_slot(tbl, slotid); + ret = PTR_ERR_OR_ZERO(slot); + if (!ret) + *seq_nr = slot->seq_nr; + + return ret; +} + +/* + * nfs4_slot_seqid_in_use - test if a slot sequence id is still in use + * + * Given a slot table, slot id and sequence number, determine if the + * RPC call in question is still in flight. This function is mainly + * intended for use by the callback channel. + */ +static bool nfs4_slot_seqid_in_use(struct nfs4_slot_table *tbl, + u32 slotid, u32 seq_nr) +{ + u32 cur_seq = 0; + bool ret = false; + + spin_lock(&tbl->slot_tbl_lock); + if (nfs4_slot_get_seqid(tbl, slotid, &cur_seq) == 0 && + cur_seq == seq_nr && test_bit(slotid, tbl->used_slots)) + ret = true; + spin_unlock(&tbl->slot_tbl_lock); + return ret; +} + +/* + * nfs4_slot_wait_on_seqid - wait until a slot sequence id is complete + * + * Given a slot table, slot id and sequence number, wait until the + * corresponding RPC call completes. This function is mainly + * intended for use by the callback channel. + */ +int nfs4_slot_wait_on_seqid(struct nfs4_slot_table *tbl, + u32 slotid, u32 seq_nr, + unsigned long timeout) +{ + if (wait_event_timeout(tbl->slot_waitq, + !nfs4_slot_seqid_in_use(tbl, slotid, seq_nr), + timeout) == 0) + return -ETIMEDOUT; + return 0; +} + +/* + * nfs4_alloc_slot - efficiently look for a free slot + * + * nfs4_alloc_slot looks for an unset bit in the used_slots bitmap. + * If found, we mark the slot as used, update the highest_used_slotid, + * and respectively set up the sequence operation args. + * + * Note: must be called with under the slot_tbl_lock. + */ +struct nfs4_slot *nfs4_alloc_slot(struct nfs4_slot_table *tbl) +{ + struct nfs4_slot *ret = ERR_PTR(-EBUSY); + u32 slotid; + + dprintk("--> %s used_slots=%04lx highest_used=%u max_slots=%u\n", + __func__, tbl->used_slots[0], tbl->highest_used_slotid, + tbl->max_slotid + 1); + slotid = find_first_zero_bit(tbl->used_slots, tbl->max_slotid + 1); + if (slotid <= tbl->max_slotid) { + ret = nfs4_find_or_create_slot(tbl, slotid, 1, GFP_NOWAIT); + if (!IS_ERR(ret)) + nfs4_lock_slot(tbl, ret); + } + dprintk("<-- %s used_slots=%04lx highest_used=%u slotid=%u\n", + __func__, tbl->used_slots[0], tbl->highest_used_slotid, + !IS_ERR(ret) ? ret->slot_nr : NFS4_NO_SLOT); + return ret; +} + +static int nfs4_grow_slot_table(struct nfs4_slot_table *tbl, + u32 max_reqs, u32 ivalue) +{ + if (max_reqs <= tbl->max_slots) + return 0; + if (!IS_ERR(nfs4_find_or_create_slot(tbl, max_reqs - 1, ivalue, GFP_NOFS))) + return 0; + return -ENOMEM; +} + +static void nfs4_reset_slot_table(struct nfs4_slot_table *tbl, + u32 server_highest_slotid, + u32 ivalue) +{ + struct nfs4_slot **p; + + nfs4_shrink_slot_table(tbl, server_highest_slotid + 1); + p = &tbl->slots; + while (*p) { + (*p)->seq_nr = ivalue; + (*p)->seq_nr_highest_sent = ivalue; + (*p)->seq_nr_last_acked = ivalue - 1; + p = &(*p)->next; + } + tbl->highest_used_slotid = NFS4_NO_SLOT; + tbl->target_highest_slotid = server_highest_slotid; + tbl->server_highest_slotid = server_highest_slotid; + tbl->d_target_highest_slotid = 0; + tbl->d2_target_highest_slotid = 0; + tbl->max_slotid = server_highest_slotid; +} + +/* + * (re)Initialise a slot table + */ +static int nfs4_realloc_slot_table(struct nfs4_slot_table *tbl, + u32 max_reqs, u32 ivalue) +{ + int ret; + + dprintk("--> %s: max_reqs=%u, tbl->max_slots %u\n", __func__, + max_reqs, tbl->max_slots); + + if (max_reqs > NFS4_MAX_SLOT_TABLE) + max_reqs = NFS4_MAX_SLOT_TABLE; + + ret = nfs4_grow_slot_table(tbl, max_reqs, ivalue); + if (ret) + goto out; + + spin_lock(&tbl->slot_tbl_lock); + nfs4_reset_slot_table(tbl, max_reqs - 1, ivalue); + spin_unlock(&tbl->slot_tbl_lock); + + dprintk("%s: tbl=%p slots=%p max_slots=%u\n", __func__, + tbl, tbl->slots, tbl->max_slots); +out: + dprintk("<-- %s: return %d\n", __func__, ret); + return ret; +} + +/* + * nfs4_release_slot_table - release all slot table entries + */ +static void nfs4_release_slot_table(struct nfs4_slot_table *tbl) +{ + nfs4_shrink_slot_table(tbl, 0); +} + +/** + * nfs4_shutdown_slot_table - release resources attached to a slot table + * @tbl: slot table to shut down + * + */ +void nfs4_shutdown_slot_table(struct nfs4_slot_table *tbl) +{ + nfs4_release_slot_table(tbl); + rpc_destroy_wait_queue(&tbl->slot_tbl_waitq); +} + +/** + * nfs4_setup_slot_table - prepare a stand-alone slot table for use + * @tbl: slot table to set up + * @max_reqs: maximum number of requests allowed + * @queue: name to give RPC wait queue + * + * Returns zero on success, or a negative errno. + */ +int nfs4_setup_slot_table(struct nfs4_slot_table *tbl, unsigned int max_reqs, + const char *queue) +{ + nfs4_init_slot_table(tbl, queue); + return nfs4_realloc_slot_table(tbl, max_reqs, 0); +} + +static bool nfs41_assign_slot(struct rpc_task *task, void *pslot) +{ + struct nfs4_sequence_args *args = task->tk_msg.rpc_argp; + struct nfs4_sequence_res *res = task->tk_msg.rpc_resp; + struct nfs4_slot *slot = pslot; + struct nfs4_slot_table *tbl = slot->table; + + if (nfs4_slot_tbl_draining(tbl) && !args->sa_privileged) + return false; + slot->generation = tbl->generation; + args->sa_slot = slot; + res->sr_timestamp = jiffies; + res->sr_slot = slot; + res->sr_status_flags = 0; + res->sr_status = 1; + return true; +} + +static bool __nfs41_wake_and_assign_slot(struct nfs4_slot_table *tbl, + struct nfs4_slot *slot) +{ + if (rpc_wake_up_first(&tbl->slot_tbl_waitq, nfs41_assign_slot, slot)) + return true; + return false; +} + +bool nfs41_wake_and_assign_slot(struct nfs4_slot_table *tbl, + struct nfs4_slot *slot) +{ + if (slot->slot_nr > tbl->max_slotid) + return false; + return __nfs41_wake_and_assign_slot(tbl, slot); +} + +static bool nfs41_try_wake_next_slot_table_entry(struct nfs4_slot_table *tbl) +{ + struct nfs4_slot *slot = nfs4_alloc_slot(tbl); + if (!IS_ERR(slot)) { + bool ret = __nfs41_wake_and_assign_slot(tbl, slot); + if (ret) + return ret; + nfs4_free_slot(tbl, slot); + } + return false; +} + +void nfs41_wake_slot_table(struct nfs4_slot_table *tbl) +{ + for (;;) { + if (!nfs41_try_wake_next_slot_table_entry(tbl)) + break; + } +} + +static void nfs41_set_max_slotid_locked(struct nfs4_slot_table *tbl, + u32 target_highest_slotid) +{ + u32 max_slotid; + + max_slotid = min(NFS4_MAX_SLOT_TABLE - 1, target_highest_slotid); + if (max_slotid > tbl->server_highest_slotid) + max_slotid = tbl->server_highest_slotid; + if (max_slotid > tbl->target_highest_slotid) + max_slotid = tbl->target_highest_slotid; + tbl->max_slotid = max_slotid; + nfs41_wake_slot_table(tbl); +} + +/* Update the client's idea of target_highest_slotid */ +static void nfs41_set_target_slotid_locked(struct nfs4_slot_table *tbl, + u32 target_highest_slotid) +{ + if (tbl->target_highest_slotid == target_highest_slotid) + return; + tbl->target_highest_slotid = target_highest_slotid; + tbl->generation++; +} + +void nfs41_set_target_slotid(struct nfs4_slot_table *tbl, + u32 target_highest_slotid) +{ + spin_lock(&tbl->slot_tbl_lock); + nfs41_set_target_slotid_locked(tbl, target_highest_slotid); + tbl->d_target_highest_slotid = 0; + tbl->d2_target_highest_slotid = 0; + nfs41_set_max_slotid_locked(tbl, target_highest_slotid); + spin_unlock(&tbl->slot_tbl_lock); +} + +static void nfs41_set_server_slotid_locked(struct nfs4_slot_table *tbl, + u32 highest_slotid) +{ + if (tbl->server_highest_slotid == highest_slotid) + return; + if (tbl->highest_used_slotid > highest_slotid) + return; + /* Deallocate slots */ + nfs4_shrink_slot_table(tbl, highest_slotid + 1); + tbl->server_highest_slotid = highest_slotid; +} + +static s32 nfs41_derivative_target_slotid(s32 s1, s32 s2) +{ + s1 -= s2; + if (s1 == 0) + return 0; + if (s1 < 0) + return (s1 - 1) >> 1; + return (s1 + 1) >> 1; +} + +static int nfs41_sign_s32(s32 s1) +{ + if (s1 > 0) + return 1; + if (s1 < 0) + return -1; + return 0; +} + +static bool nfs41_same_sign_or_zero_s32(s32 s1, s32 s2) +{ + if (!s1 || !s2) + return true; + return nfs41_sign_s32(s1) == nfs41_sign_s32(s2); +} + +/* Try to eliminate outliers by checking for sharp changes in the + * derivatives and second derivatives + */ +static bool nfs41_is_outlier_target_slotid(struct nfs4_slot_table *tbl, + u32 new_target) +{ + s32 d_target, d2_target; + bool ret = true; + + d_target = nfs41_derivative_target_slotid(new_target, + tbl->target_highest_slotid); + d2_target = nfs41_derivative_target_slotid(d_target, + tbl->d_target_highest_slotid); + /* Is first derivative same sign? */ + if (nfs41_same_sign_or_zero_s32(d_target, tbl->d_target_highest_slotid)) + ret = false; + /* Is second derivative same sign? */ + if (nfs41_same_sign_or_zero_s32(d2_target, tbl->d2_target_highest_slotid)) + ret = false; + tbl->d_target_highest_slotid = d_target; + tbl->d2_target_highest_slotid = d2_target; + return ret; +} + +void nfs41_update_target_slotid(struct nfs4_slot_table *tbl, + struct nfs4_slot *slot, + struct nfs4_sequence_res *res) +{ + u32 target_highest_slotid = min(res->sr_target_highest_slotid, + NFS4_MAX_SLOTID); + u32 highest_slotid = min(res->sr_highest_slotid, NFS4_MAX_SLOTID); + + spin_lock(&tbl->slot_tbl_lock); + if (!nfs41_is_outlier_target_slotid(tbl, target_highest_slotid)) + nfs41_set_target_slotid_locked(tbl, target_highest_slotid); + if (tbl->generation == slot->generation) + nfs41_set_server_slotid_locked(tbl, highest_slotid); + nfs41_set_max_slotid_locked(tbl, target_highest_slotid); + spin_unlock(&tbl->slot_tbl_lock); +} + +static void nfs4_release_session_slot_tables(struct nfs4_session *session) +{ + nfs4_release_slot_table(&session->fc_slot_table); + nfs4_release_slot_table(&session->bc_slot_table); +} + +/* + * Initialize or reset the forechannel and backchannel tables + */ +int nfs4_setup_session_slot_tables(struct nfs4_session *ses) +{ + struct nfs4_slot_table *tbl; + int status; + + dprintk("--> %s\n", __func__); + /* Fore channel */ + tbl = &ses->fc_slot_table; + tbl->session = ses; + status = nfs4_realloc_slot_table(tbl, ses->fc_attrs.max_reqs, 1); + if (status || !(ses->flags & SESSION4_BACK_CHAN)) /* -ENOMEM */ + return status; + /* Back channel */ + tbl = &ses->bc_slot_table; + tbl->session = ses; + status = nfs4_realloc_slot_table(tbl, ses->bc_attrs.max_reqs, 0); + if (status && tbl->slots == NULL) + /* Fore and back channel share a connection so get + * both slot tables or neither */ + nfs4_release_session_slot_tables(ses); + return status; +} + +struct nfs4_session *nfs4_alloc_session(struct nfs_client *clp) +{ + struct nfs4_session *session; + + session = kzalloc_obj(struct nfs4_session, GFP_NOFS); + if (!session) + return NULL; + + nfs4_init_slot_table(&session->fc_slot_table, "ForeChannel Slot table"); + nfs4_init_slot_table(&session->bc_slot_table, "BackChannel Slot table"); + session->session_state = 1<clp = clp; + return session; +} + +static void nfs4_destroy_session_slot_tables(struct nfs4_session *session) +{ + nfs4_shutdown_slot_table(&session->fc_slot_table); + nfs4_shutdown_slot_table(&session->bc_slot_table); +} + +void nfs4_destroy_session(struct nfs4_session *session) +{ + struct rpc_xprt *xprt; + const struct cred *cred; + + cred = nfs4_get_clid_cred(session->clp); + nfs4_proc_destroy_session(session, cred); + put_cred(cred); + + rcu_read_lock(); + xprt = rcu_dereference(session->clp->cl_rpcclient->cl_xprt); + rcu_read_unlock(); + dprintk("%s Destroy backchannel for xprt %p\n", + __func__, xprt); + xprt_destroy_backchannel(xprt, NFS41_BC_MIN_CALLBACKS); + nfs4_destroy_session_slot_tables(session); + kfree(session); +} + +/* + * With sessions, the client is not marked ready until after a + * successful EXCHANGE_ID and CREATE_SESSION. + * + * Map errors cl_cons_state errors to EPROTONOSUPPORT to indicate + * other versions of NFS can be tried. + */ +static int nfs41_check_session_ready(struct nfs_client *clp) +{ + int ret; + + if (clp->cl_cons_state == NFS_CS_SESSION_INITING) { + ret = nfs4_client_recover_expired_lease(clp); + if (ret) + return ret; + } + if (clp->cl_cons_state < NFS_CS_READY) + return -EPROTONOSUPPORT; + smp_rmb(); + return 0; +} + +int nfs4_init_session(struct nfs_client *clp) +{ + if (!nfs4_has_session(clp)) + return 0; + + clear_bit(NFS4_SESSION_INITING, &clp->cl_session->session_state); + return nfs41_check_session_ready(clp); +} + +int nfs4_init_ds_session(struct nfs_client *clp, unsigned long lease_time) +{ + struct nfs4_session *session = clp->cl_session; + int ret; + + spin_lock(&clp->cl_lock); + if (test_and_clear_bit(NFS4_SESSION_INITING, &session->session_state)) { + /* + * Do not set NFS_CS_CHECK_LEASE_TIME instead set the + * DS lease to be equal to the MDS lease. + */ + clp->cl_lease_time = lease_time; + clp->cl_last_renewal = jiffies; + } + spin_unlock(&clp->cl_lock); + + ret = nfs41_check_session_ready(clp); + if (ret) + return ret; + /* Test for the DS role */ + if (!is_ds_client(clp)) + return -ENODEV; + return 0; +} +EXPORT_SYMBOL_GPL(nfs4_init_ds_session); diff --git a/fs/nfs/nfs4session.h b/fs/nfs/nfs4session.h new file mode 100644 index 000000000..d2569f599 --- /dev/null +++ b/fs/nfs/nfs4session.h @@ -0,0 +1,158 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * fs/nfs/nfs4session.h + * + * Copyright (c) 2012 Trond Myklebust + * + */ +#ifndef __LINUX_FS_NFS_NFS4SESSION_H +#define __LINUX_FS_NFS_NFS4SESSION_H + +/* maximum number of slots to use */ +#define NFS4_DEF_SLOT_TABLE_SIZE (64U) +#define NFS4_DEF_CB_SLOT_TABLE_SIZE (16U) +#define NFS4_MAX_SLOT_TABLE (1024U) +#define NFS4_MAX_SLOTID (NFS4_MAX_SLOT_TABLE - 1U) +#define NFS4_NO_SLOT ((u32)-1) + +#if IS_ENABLED(CONFIG_NFS_V4) + +/* Sessions slot seqid */ +struct nfs4_slot { + struct nfs4_slot_table *table; + struct nfs4_slot *next; + unsigned long generation; + u32 slot_nr; + u32 seq_nr; + u32 seq_nr_last_acked; + u32 seq_nr_highest_sent; + unsigned int privileged : 1, + seq_done : 1; +}; + +/* Sessions */ +enum nfs4_slot_tbl_state { + NFS4_SLOT_TBL_DRAINING, +}; + +#define SLOT_TABLE_SZ DIV_ROUND_UP(NFS4_MAX_SLOT_TABLE, BITS_PER_LONG) +struct nfs4_slot_table { + struct nfs4_session *session; /* Parent session */ + struct nfs4_slot *slots; /* seqid per slot */ + unsigned long used_slots[SLOT_TABLE_SZ]; /* used/unused bitmap */ + spinlock_t slot_tbl_lock; + struct rpc_wait_queue slot_tbl_waitq; /* allocators may wait here */ + wait_queue_head_t slot_waitq; /* Completion wait on slot */ + u32 max_slots; /* # slots in table */ + u32 max_slotid; /* Max allowed slotid value */ + u32 highest_used_slotid; /* sent to server on each SEQ. + * op for dynamic resizing */ + u32 target_highest_slotid; /* Server max_slot target */ + u32 server_highest_slotid; /* Server highest slotid */ + s32 d_target_highest_slotid; /* Derivative */ + s32 d2_target_highest_slotid; /* 2nd derivative */ + unsigned long generation; /* Generation counter for + target_highest_slotid */ + struct completion complete; + unsigned long slot_tbl_state; +}; + +/* + * Session related parameters + */ +struct nfs4_session { + struct nfs4_sessionid sess_id; + u32 flags; + unsigned long session_state; + u32 hash_alg; + u32 ssv_len; + + /* The fore and back channel */ + struct nfs4_channel_attrs fc_attrs; + struct nfs4_slot_table fc_slot_table; + struct nfs4_channel_attrs bc_attrs; + struct nfs4_slot_table bc_slot_table; + struct nfs_client *clp; +}; + +enum nfs4_session_state { + NFS4_SESSION_INITING, + NFS4_SESSION_ESTABLISHED, +}; + +extern int nfs4_setup_slot_table(struct nfs4_slot_table *tbl, + unsigned int max_reqs, const char *queue); +extern void nfs4_shutdown_slot_table(struct nfs4_slot_table *tbl); +extern struct nfs4_slot *nfs4_alloc_slot(struct nfs4_slot_table *tbl); +extern struct nfs4_slot *nfs4_lookup_slot(struct nfs4_slot_table *tbl, u32 slotid); +extern int nfs4_slot_wait_on_seqid(struct nfs4_slot_table *tbl, + u32 slotid, u32 seq_nr, + unsigned long timeout); +extern bool nfs4_try_to_lock_slot(struct nfs4_slot_table *tbl, struct nfs4_slot *slot); +extern void nfs4_free_slot(struct nfs4_slot_table *tbl, struct nfs4_slot *slot); +extern void nfs4_slot_tbl_drain_complete(struct nfs4_slot_table *tbl); +bool nfs41_wake_and_assign_slot(struct nfs4_slot_table *tbl, + struct nfs4_slot *slot); +void nfs41_wake_slot_table(struct nfs4_slot_table *tbl); + +static inline bool nfs4_slot_tbl_draining(struct nfs4_slot_table *tbl) +{ + return !!test_bit(NFS4_SLOT_TBL_DRAINING, &tbl->slot_tbl_state); +} + +static inline bool nfs4_test_locked_slot(const struct nfs4_slot_table *tbl, + u32 slotid) +{ + return !!test_bit(slotid, tbl->used_slots); +} + +static inline struct nfs4_session *nfs4_get_session(const struct nfs_client *clp) +{ + return clp->cl_session; +} + +extern void nfs41_set_target_slotid(struct nfs4_slot_table *tbl, + u32 target_highest_slotid); +extern void nfs41_update_target_slotid(struct nfs4_slot_table *tbl, + struct nfs4_slot *slot, + struct nfs4_sequence_res *res); + +extern int nfs4_setup_session_slot_tables(struct nfs4_session *ses); + +extern struct nfs4_session *nfs4_alloc_session(struct nfs_client *clp); +extern void nfs4_destroy_session(struct nfs4_session *session); +extern int nfs4_init_session(struct nfs_client *clp); +extern int nfs4_init_ds_session(struct nfs_client *, unsigned long); + +/* + * Determine if sessions are in use. + */ +static inline int nfs4_has_session(const struct nfs_client *clp) +{ + if (clp->cl_session) + return 1; + return 0; +} + +static inline int nfs4_has_persistent_session(const struct nfs_client *clp) +{ + if (nfs4_has_session(clp)) + return (clp->cl_session->flags & SESSION4_PERSIST); + return 0; +} + +static inline void nfs4_copy_sessionid(struct nfs4_sessionid *dst, + const struct nfs4_sessionid *src) +{ + memcpy(dst->data, src->data, NFS4_MAX_SESSIONID_LEN); +} + +/* + * nfs_session_id_hash - calculate the crc32 hash for the session id + * @session - pointer to session + */ +#define nfs_session_id_hash(sess_id) \ + (~crc32_le(0xFFFFFFFF, &(sess_id)->data[0], sizeof((sess_id)->data))) + +#endif /* IS_ENABLED(CONFIG_NFS_V4) */ +#endif /* __LINUX_FS_NFS_NFS4SESSION_H */ diff --git a/fs/nfs/nfs4state.c b/fs/nfs/nfs4state.c new file mode 100644 index 000000000..305a772e5 --- /dev/null +++ b/fs/nfs/nfs4state.c @@ -0,0 +1,2713 @@ +/* + * fs/nfs/nfs4state.c + * + * Client-side XDR for NFSv4. + * + * Copyright (c) 2002 The Regents of the University of Michigan. + * All rights reserved. + * + * Kendrick Smith + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Implementation of the NFSv4 state model. For the time being, + * this is minimal, but will be made much more complex in a + * subsequent patch. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "nfs4_fs.h" +#include "nfs40.h" +#include "callback.h" +#include "delegation.h" +#include "internal.h" +#include "nfs4idmap.h" +#include "nfs4session.h" +#include "pnfs.h" +#include "netns.h" +#include "nfs4trace.h" + +#define NFSDBG_FACILITY NFSDBG_STATE + +#define OPENOWNER_POOL_SIZE 8 + +static void nfs4_state_start_reclaim_reboot(struct nfs_client *clp); + +const nfs4_stateid zero_stateid = { + { .data = { 0 } }, + .type = NFS4_SPECIAL_STATEID_TYPE, +}; +const nfs4_stateid invalid_stateid = { + { + /* Funky initialiser keeps older gcc versions happy */ + .data = { 0xff, 0xff, 0xff, 0xff, 0 }, + }, + .type = NFS4_INVALID_STATEID_TYPE, +}; + +const nfs4_stateid current_stateid = { + { + /* Funky initialiser keeps older gcc versions happy */ + .data = { 0x0, 0x0, 0x0, 0x1, 0 }, + }, + .type = NFS4_SPECIAL_STATEID_TYPE, +}; + +static DEFINE_MUTEX(nfs_clid_init_mutex); + +static int nfs4_setup_state_renewal(struct nfs_client *clp) +{ + int status; + struct nfs_fsinfo fsinfo; + + if (!test_bit(NFS_CS_CHECK_LEASE_TIME, &clp->cl_res_state)) { + nfs4_schedule_state_renewal(clp); + return 0; + } + + status = nfs4_proc_get_lease_time(clp, &fsinfo); + if (status == 0) { + nfs4_set_lease_period(clp, fsinfo.lease_time); + nfs4_schedule_state_renewal(clp); + } + + return status; +} + +int nfs4_init_clientid(struct nfs_client *clp, const struct cred *cred) +{ + struct nfs4_setclientid_res clid = { + .clientid = clp->cl_clientid, + .confirm = clp->cl_confirm, + }; + unsigned short port; + int status; + struct nfs_net *nn = net_generic(clp->cl_net, nfs_net_id); + + if (test_bit(NFS4CLNT_LEASE_CONFIRM, &clp->cl_state)) + goto do_confirm; + port = nn->nfs_callback_tcpport; + if (clp->cl_addr.ss_family == AF_INET6) + port = nn->nfs_callback_tcpport6; + + status = nfs4_proc_setclientid(clp, NFS4_CALLBACK, port, cred, &clid); + if (status != 0) + goto out; + clp->cl_clientid = clid.clientid; + clp->cl_confirm = clid.confirm; + set_bit(NFS4CLNT_LEASE_CONFIRM, &clp->cl_state); +do_confirm: + status = nfs4_proc_setclientid_confirm(clp, &clid, cred); + if (status != 0) + goto out; + clear_bit(NFS4CLNT_LEASE_CONFIRM, &clp->cl_state); + nfs4_setup_state_renewal(clp); +out: + return status; +} + +const struct cred *nfs4_get_machine_cred(struct nfs_client *clp) +{ + return get_cred(rpc_machine_cred()); +} + +static void nfs4_root_machine_cred(struct nfs_client *clp) +{ + + /* Force root creds instead of machine */ + clp->cl_principal = NULL; + clp->cl_rpcclient->cl_principal = NULL; +} + +static const struct cred * +nfs4_get_renew_cred_server_locked(struct nfs_server *server) +{ + const struct cred *cred = NULL; + struct nfs4_state_owner *sp; + struct rb_node *pos; + + for (pos = rb_first(&server->state_owners); + pos != NULL; + pos = rb_next(pos)) { + sp = rb_entry(pos, struct nfs4_state_owner, so_server_node); + if (list_empty(&sp->so_states)) + continue; + cred = get_cred(sp->so_cred); + break; + } + return cred; +} + +/** + * nfs4_get_renew_cred - Acquire credential for a renew operation + * @clp: client state handle + * + * Returns an rpc_cred with reference count bumped, or NULL. + * Caller must hold clp->cl_lock. + */ +const struct cred *nfs4_get_renew_cred(struct nfs_client *clp) +{ + const struct cred *cred = NULL; + struct nfs_server *server; + + /* Use machine credentials if available */ + cred = nfs4_get_machine_cred(clp); + if (cred != NULL) + goto out; + + spin_lock(&clp->cl_lock); + rcu_read_lock(); + list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { + cred = nfs4_get_renew_cred_server_locked(server); + if (cred != NULL) + break; + } + rcu_read_unlock(); + spin_unlock(&clp->cl_lock); + +out: + return cred; +} + +static void nfs4_end_drain_slot_table(struct nfs4_slot_table *tbl) +{ + if (test_and_clear_bit(NFS4_SLOT_TBL_DRAINING, &tbl->slot_tbl_state)) { + spin_lock(&tbl->slot_tbl_lock); + nfs41_wake_slot_table(tbl); + spin_unlock(&tbl->slot_tbl_lock); + } +} + +static void nfs4_end_drain_session(struct nfs_client *clp) +{ + struct nfs4_session *ses = clp->cl_session; + + if (clp->cl_slot_tbl) { + nfs4_end_drain_slot_table(clp->cl_slot_tbl); + return; + } + + if (ses != NULL) { + nfs4_end_drain_slot_table(&ses->bc_slot_table); + nfs4_end_drain_slot_table(&ses->fc_slot_table); + } +} + +static int nfs4_drain_slot_tbl(struct nfs4_slot_table *tbl) +{ + set_bit(NFS4_SLOT_TBL_DRAINING, &tbl->slot_tbl_state); + spin_lock(&tbl->slot_tbl_lock); + if (tbl->highest_used_slotid != NFS4_NO_SLOT) { + reinit_completion(&tbl->complete); + spin_unlock(&tbl->slot_tbl_lock); + return wait_for_completion_interruptible(&tbl->complete); + } + spin_unlock(&tbl->slot_tbl_lock); + return 0; +} + +static int nfs4_begin_drain_session(struct nfs_client *clp) +{ + struct nfs4_session *ses = clp->cl_session; + int ret; + + if (clp->cl_slot_tbl) + return nfs4_drain_slot_tbl(clp->cl_slot_tbl); + + /* back channel */ + ret = nfs4_drain_slot_tbl(&ses->bc_slot_table); + if (ret) + return ret; + /* fore channel */ + return nfs4_drain_slot_tbl(&ses->fc_slot_table); +} + +static void nfs41_finish_session_reset(struct nfs_client *clp) +{ + clear_bit(NFS4CLNT_LEASE_CONFIRM, &clp->cl_state); + clear_bit(NFS4CLNT_SESSION_RESET, &clp->cl_state); + /* create_session negotiated new slot table */ + clear_bit(NFS4CLNT_BIND_CONN_TO_SESSION, &clp->cl_state); + nfs4_setup_state_renewal(clp); +} + +int nfs41_init_clientid(struct nfs_client *clp, const struct cred *cred) +{ + int status; + + if (test_bit(NFS4CLNT_LEASE_CONFIRM, &clp->cl_state)) + goto do_confirm; + status = nfs4_proc_exchange_id(clp, cred); + if (status != 0) + goto out; + set_bit(NFS4CLNT_LEASE_CONFIRM, &clp->cl_state); +do_confirm: + status = nfs4_proc_create_session(clp, cred); + if (status != 0) + goto out; + if (!(clp->cl_exchange_flags & EXCHGID4_FLAG_CONFIRMED_R)) + nfs4_state_start_reclaim_reboot(clp); + nfs41_finish_session_reset(clp); + nfs_mark_client_ready(clp, NFS_CS_READY); +out: + return status; +} + +/** + * nfs41_discover_server_trunking - Detect server IP address trunking (mv1) + * + * @clp: nfs_client under test + * @result: OUT: found nfs_client, or clp + * @cred: credential to use for trunking test + * + * Returns NFS4_OK, a negative errno, or a negative NFS4ERR status. + * If NFS4_OK is returned, an nfs_client pointer is planted in + * "result". + * + * Note: The returned client may not yet be marked ready. + */ +int nfs41_discover_server_trunking(struct nfs_client *clp, + struct nfs_client **result, + const struct cred *cred) +{ + int status; + + status = nfs4_proc_exchange_id(clp, cred); + if (status != NFS4_OK) + return status; + + status = nfs41_walk_client_list(clp, result, cred); + if (status < 0) + return status; + if (clp != *result) + return 0; + + /* + * Purge state if the client id was established in a prior + * instance and the client id could not have arrived on the + * server via Transparent State Migration. + */ + if (clp->cl_exchange_flags & EXCHGID4_FLAG_CONFIRMED_R) { + if (!test_bit(NFS_CS_TSM_POSSIBLE, &clp->cl_flags)) + set_bit(NFS4CLNT_PURGE_STATE, &clp->cl_state); + else + set_bit(NFS4CLNT_LEASE_CONFIRM, &clp->cl_state); + } + nfs4_schedule_state_manager(clp); + status = nfs_wait_client_init_complete(clp); + if (status < 0) + nfs_put_client(clp); + return status; +} + +/** + * nfs4_get_clid_cred - Acquire credential for a setclientid operation + * @clp: client state handle + * + * Returns a cred with reference count bumped, or NULL. + */ +const struct cred *nfs4_get_clid_cred(struct nfs_client *clp) +{ + const struct cred *cred; + + cred = nfs4_get_machine_cred(clp); + return cred; +} + +static struct nfs4_state_owner * +nfs4_find_state_owner_locked(struct nfs_server *server, const struct cred *cred) +{ + struct rb_node **p = &server->state_owners.rb_node, + *parent = NULL; + struct nfs4_state_owner *sp; + int cmp; + + while (*p != NULL) { + parent = *p; + sp = rb_entry(parent, struct nfs4_state_owner, so_server_node); + cmp = cred_fscmp(cred, sp->so_cred); + + if (cmp < 0) + p = &parent->rb_left; + else if (cmp > 0) + p = &parent->rb_right; + else { + if (!list_empty(&sp->so_lru)) + list_del_init(&sp->so_lru); + atomic_inc(&sp->so_count); + return sp; + } + } + return NULL; +} + +static struct nfs4_state_owner * +nfs4_insert_state_owner_locked(struct nfs4_state_owner *new) +{ + struct nfs_server *server = new->so_server; + struct rb_node **p = &server->state_owners.rb_node, + *parent = NULL; + struct nfs4_state_owner *sp; + int cmp; + + while (*p != NULL) { + parent = *p; + sp = rb_entry(parent, struct nfs4_state_owner, so_server_node); + cmp = cred_fscmp(new->so_cred, sp->so_cred); + + if (cmp < 0) + p = &parent->rb_left; + else if (cmp > 0) + p = &parent->rb_right; + else { + if (!list_empty(&sp->so_lru)) + list_del_init(&sp->so_lru); + atomic_inc(&sp->so_count); + return sp; + } + } + rb_link_node(&new->so_server_node, parent, p); + rb_insert_color(&new->so_server_node, &server->state_owners); + return new; +} + +static void +nfs4_remove_state_owner_locked(struct nfs4_state_owner *sp) +{ + struct nfs_server *server = sp->so_server; + + if (!RB_EMPTY_NODE(&sp->so_server_node)) + rb_erase(&sp->so_server_node, &server->state_owners); +} + +static void +nfs4_init_seqid_counter(struct nfs_seqid_counter *sc) +{ + sc->create_time = ktime_get(); + sc->flags = 0; + sc->counter = 0; + spin_lock_init(&sc->lock); + INIT_LIST_HEAD(&sc->list); + rpc_init_wait_queue(&sc->wait, "Seqid_waitqueue"); +} + +static void +nfs4_destroy_seqid_counter(struct nfs_seqid_counter *sc) +{ + rpc_destroy_wait_queue(&sc->wait); +} + +/* + * nfs4_alloc_state_owner(): this is called on the OPEN or CREATE path to + * create a new state_owner. + * + */ +static struct nfs4_state_owner * +nfs4_alloc_state_owner(struct nfs_server *server, + const struct cred *cred, + gfp_t gfp_flags) +{ + struct nfs4_state_owner *sp; + + sp = kzalloc_obj(*sp, gfp_flags); + if (!sp) + return NULL; + sp->so_seqid.owner_id = atomic64_inc_return(&server->owner_ctr); + sp->so_server = server; + sp->so_cred = get_cred(cred); + spin_lock_init(&sp->so_lock); + INIT_LIST_HEAD(&sp->so_states); + nfs4_init_seqid_counter(&sp->so_seqid); + atomic_set(&sp->so_count, 1); + INIT_LIST_HEAD(&sp->so_lru); + mutex_init(&sp->so_delegreturn_mutex); + return sp; +} + +static void +nfs4_reset_state_owner(struct nfs4_state_owner *sp) +{ + /* This state_owner is no longer usable, but must + * remain in place so that state recovery can find it + * and the opens associated with it. + * It may also be used for new 'open' request to + * return a delegation to the server. + * So update the 'create_time' so that it looks like + * a new state_owner. This will cause the server to + * request an OPEN_CONFIRM to start a new sequence. + */ + sp->so_seqid.create_time = ktime_get(); +} + +static void nfs4_free_state_owner(struct nfs4_state_owner *sp) +{ + nfs4_destroy_seqid_counter(&sp->so_seqid); + put_cred(sp->so_cred); + kfree(sp); +} + +static void nfs4_gc_state_owners(struct nfs_server *server) +{ + struct nfs_client *clp = server->nfs_client; + struct nfs4_state_owner *sp, *tmp; + unsigned long time_min, time_max; + LIST_HEAD(doomed); + + spin_lock(&clp->cl_lock); + time_max = jiffies; + time_min = (long)time_max - (long)clp->cl_lease_time; + list_for_each_entry_safe(sp, tmp, &server->state_owners_lru, so_lru) { + /* NB: LRU is sorted so that oldest is at the head */ + if (time_in_range(sp->so_expires, time_min, time_max)) + break; + list_move(&sp->so_lru, &doomed); + nfs4_remove_state_owner_locked(sp); + } + spin_unlock(&clp->cl_lock); + + list_for_each_entry_safe(sp, tmp, &doomed, so_lru) { + list_del(&sp->so_lru); + nfs4_free_state_owner(sp); + } +} + +/** + * nfs4_get_state_owner - Look up a state owner given a credential + * @server: nfs_server to search + * @cred: RPC credential to match + * @gfp_flags: allocation mode + * + * Returns a pointer to an instantiated nfs4_state_owner struct, or NULL. + */ +struct nfs4_state_owner *nfs4_get_state_owner(struct nfs_server *server, + const struct cred *cred, + gfp_t gfp_flags) +{ + struct nfs_client *clp = server->nfs_client; + struct nfs4_state_owner *sp, *new; + + spin_lock(&clp->cl_lock); + sp = nfs4_find_state_owner_locked(server, cred); + spin_unlock(&clp->cl_lock); + if (sp != NULL) + goto out; + new = nfs4_alloc_state_owner(server, cred, gfp_flags); + if (new == NULL) + goto out; + spin_lock(&clp->cl_lock); + sp = nfs4_insert_state_owner_locked(new); + spin_unlock(&clp->cl_lock); + if (sp != new) + nfs4_free_state_owner(new); +out: + nfs4_gc_state_owners(server); + return sp; +} + +/** + * nfs4_put_state_owner - Release a nfs4_state_owner + * @sp: state owner data to release + * + * Note that we keep released state owners on an LRU + * list. + * This caches valid state owners so that they can be + * reused, to avoid the OPEN_CONFIRM on minor version 0. + * It also pins the uniquifier of dropped state owners for + * a while, to ensure that those state owner names are + * never reused. + */ +void nfs4_put_state_owner(struct nfs4_state_owner *sp) +{ + struct nfs_server *server = sp->so_server; + struct nfs_client *clp = server->nfs_client; + + if (!atomic_dec_and_lock(&sp->so_count, &clp->cl_lock)) + return; + + sp->so_expires = jiffies; + list_add_tail(&sp->so_lru, &server->state_owners_lru); + spin_unlock(&clp->cl_lock); +} + +/** + * nfs4_purge_state_owners - Release all cached state owners + * @server: nfs_server with cached state owners to release + * @head: resulting list of state owners + * + * Called at umount time. Remaining state owners will be on + * the LRU with ref count of zero. + * Note that the state owners are not freed, but are added + * to the list @head, which can later be used as an argument + * to nfs4_free_state_owners. + */ +void nfs4_purge_state_owners(struct nfs_server *server, struct list_head *head) +{ + struct nfs_client *clp = server->nfs_client; + struct nfs4_state_owner *sp, *tmp; + + spin_lock(&clp->cl_lock); + list_for_each_entry_safe(sp, tmp, &server->state_owners_lru, so_lru) { + list_move(&sp->so_lru, head); + nfs4_remove_state_owner_locked(sp); + } + spin_unlock(&clp->cl_lock); +} + +/** + * nfs4_free_state_owners - Release all cached state owners + * @head: resulting list of state owners + * + * Frees a list of state owners that was generated by + * nfs4_purge_state_owners + */ +void nfs4_free_state_owners(struct list_head *head) +{ + struct nfs4_state_owner *sp, *tmp; + + list_for_each_entry_safe(sp, tmp, head, so_lru) { + list_del(&sp->so_lru); + nfs4_free_state_owner(sp); + } +} + +static struct nfs4_state * +nfs4_alloc_open_state(void) +{ + struct nfs4_state *state; + + state = kzalloc_obj(*state, GFP_KERNEL_ACCOUNT); + if (!state) + return NULL; + refcount_set(&state->count, 1); + INIT_LIST_HEAD(&state->lock_states); + spin_lock_init(&state->state_lock); + seqlock_init(&state->seqlock); + init_waitqueue_head(&state->waitq); + return state; +} + +void +nfs4_state_set_mode_locked(struct nfs4_state *state, fmode_t fmode) +{ + if (state->state == fmode) + return; + /* NB! List reordering - see the reclaim code for why. */ + if ((fmode & FMODE_WRITE) != (state->state & FMODE_WRITE)) { + if (fmode & FMODE_WRITE) + list_move(&state->open_states, &state->owner->so_states); + else + list_move_tail(&state->open_states, &state->owner->so_states); + } + state->state = fmode; +} + +static struct nfs4_state * +__nfs4_find_state_byowner(struct inode *inode, struct nfs4_state_owner *owner) +{ + struct nfs_inode *nfsi = NFS_I(inode); + struct nfs4_state *state; + + list_for_each_entry_rcu(state, &nfsi->open_states, inode_states) { + if (state->owner != owner) + continue; + if (!nfs4_valid_open_stateid(state)) + continue; + if (refcount_inc_not_zero(&state->count)) + return state; + } + return NULL; +} + +static void +nfs4_free_open_state(struct nfs4_state *state) +{ + kfree_rcu(state, rcu_head); +} + +struct nfs4_state * +nfs4_get_open_state(struct inode *inode, struct nfs4_state_owner *owner) +{ + struct nfs4_state *state, *new; + struct nfs_inode *nfsi = NFS_I(inode); + + rcu_read_lock(); + state = __nfs4_find_state_byowner(inode, owner); + rcu_read_unlock(); + if (state) + goto out; + new = nfs4_alloc_open_state(); + spin_lock(&owner->so_lock); + spin_lock(&inode->i_lock); + state = __nfs4_find_state_byowner(inode, owner); + if (state == NULL && new != NULL) { + state = new; + state->owner = owner; + atomic_inc(&owner->so_count); + ihold(inode); + state->inode = inode; + list_add_rcu(&state->inode_states, &nfsi->open_states); + spin_unlock(&inode->i_lock); + /* Note: The reclaim code dictates that we add stateless + * and read-only stateids to the end of the list */ + list_add_tail(&state->open_states, &owner->so_states); + spin_unlock(&owner->so_lock); + } else { + spin_unlock(&inode->i_lock); + spin_unlock(&owner->so_lock); + if (new) + nfs4_free_open_state(new); + } +out: + return state; +} + +void nfs4_put_open_state(struct nfs4_state *state) +{ + struct inode *inode = state->inode; + struct nfs4_state_owner *owner = state->owner; + + if (!refcount_dec_and_lock(&state->count, &owner->so_lock)) + return; + spin_lock(&inode->i_lock); + list_del_rcu(&state->inode_states); + list_del(&state->open_states); + spin_unlock(&inode->i_lock); + spin_unlock(&owner->so_lock); + nfs4_inode_return_delegation_on_close(inode); + iput(inode); + nfs4_free_open_state(state); + nfs4_put_state_owner(owner); +} + +/* + * Close the current file. + */ +static void __nfs4_close(struct nfs4_state *state, + fmode_t fmode, gfp_t gfp_mask, int wait) +{ + struct nfs4_state_owner *owner = state->owner; + int call_close = 0; + fmode_t newstate; + + atomic_inc(&owner->so_count); + /* Protect against nfs4_find_state() */ + spin_lock(&owner->so_lock); + switch (fmode & (FMODE_READ | FMODE_WRITE)) { + case FMODE_READ: + state->n_rdonly--; + break; + case FMODE_WRITE: + state->n_wronly--; + break; + case FMODE_READ|FMODE_WRITE: + state->n_rdwr--; + } + newstate = FMODE_READ|FMODE_WRITE; + if (state->n_rdwr == 0) { + if (state->n_rdonly == 0) { + newstate &= ~FMODE_READ; + call_close |= test_bit(NFS_O_RDONLY_STATE, &state->flags); + call_close |= test_bit(NFS_O_RDWR_STATE, &state->flags); + } + if (state->n_wronly == 0) { + newstate &= ~FMODE_WRITE; + call_close |= test_bit(NFS_O_WRONLY_STATE, &state->flags); + call_close |= test_bit(NFS_O_RDWR_STATE, &state->flags); + } + if (newstate == 0) + clear_bit(NFS_DELEGATED_STATE, &state->flags); + } + nfs4_state_set_mode_locked(state, newstate); + spin_unlock(&owner->so_lock); + + if (!call_close) { + nfs4_put_open_state(state); + nfs4_put_state_owner(owner); + } else + nfs4_do_close(state, gfp_mask, wait); +} + +void nfs4_close_state(struct nfs4_state *state, fmode_t fmode) +{ + __nfs4_close(state, fmode, GFP_KERNEL, 0); +} + +void nfs4_close_sync(struct nfs4_state *state, fmode_t fmode) +{ + __nfs4_close(state, fmode, GFP_KERNEL, 1); +} + +/* + * Search the state->lock_states for an existing lock_owner + * that is compatible with either of the given owners. + * If the second is non-zero, then the first refers to a Posix-lock + * owner (current->files) and the second refers to a flock/OFD + * owner (struct file*). In that case, prefer a match for the first + * owner. + * If both sorts of locks are held on the one file we cannot know + * which stateid was intended to be used, so a "correct" choice cannot + * be made. Failing that, a "consistent" choice is preferable. The + * consistent choice we make is to prefer the first owner, that of a + * Posix lock. + */ +static struct nfs4_lock_state * +__nfs4_find_lock_state(struct nfs4_state *state, + fl_owner_t owner, fl_owner_t owner2) +{ + struct nfs4_lock_state *pos, *ret = NULL; + list_for_each_entry(pos, &state->lock_states, ls_locks) { + if (pos->ls_owner == owner) { + ret = pos; + break; + } + if (pos->ls_owner == owner2) + ret = pos; + } + if (ret) + refcount_inc(&ret->ls_count); + return ret; +} + +/* + * Return a compatible lock_state. If no initialized lock_state structure + * exists, return an uninitialized one. + * + */ +static struct nfs4_lock_state *nfs4_alloc_lock_state(struct nfs4_state *state, fl_owner_t owner) +{ + struct nfs4_lock_state *lsp; + struct nfs_server *server = state->owner->so_server; + + lsp = kzalloc_obj(*lsp, GFP_KERNEL_ACCOUNT); + if (lsp == NULL) + return NULL; + nfs4_init_seqid_counter(&lsp->ls_seqid); + refcount_set(&lsp->ls_count, 1); + lsp->ls_state = state; + lsp->ls_owner = owner; + lsp->ls_seqid.owner_id = atomic64_inc_return(&server->owner_ctr); + INIT_LIST_HEAD(&lsp->ls_locks); + return lsp; +} + +void nfs4_free_lock_state(struct nfs_server *server, struct nfs4_lock_state *lsp) +{ + nfs4_destroy_seqid_counter(&lsp->ls_seqid); + kfree(lsp); +} + +/* + * Return a compatible lock_state. If no initialized lock_state structure + * exists, return an uninitialized one. + * + */ +static struct nfs4_lock_state *nfs4_get_lock_state(struct nfs4_state *state, fl_owner_t owner) +{ + struct nfs4_lock_state *lsp, *new = NULL; + + for(;;) { + spin_lock(&state->state_lock); + lsp = __nfs4_find_lock_state(state, owner, NULL); + if (lsp != NULL) + break; + if (new != NULL) { + list_add(&new->ls_locks, &state->lock_states); + set_bit(LK_STATE_IN_USE, &state->flags); + lsp = new; + new = NULL; + break; + } + spin_unlock(&state->state_lock); + new = nfs4_alloc_lock_state(state, owner); + if (new == NULL) + return NULL; + } + spin_unlock(&state->state_lock); + if (new != NULL) + nfs4_free_lock_state(state->owner->so_server, new); + return lsp; +} + +/* + * Release reference to lock_state, and free it if we see that + * it is no longer in use + */ +void nfs4_put_lock_state(struct nfs4_lock_state *lsp) +{ + struct nfs_server *server; + struct nfs4_state *state; + + if (lsp == NULL) + return; + state = lsp->ls_state; + if (!refcount_dec_and_lock(&lsp->ls_count, &state->state_lock)) + return; + list_del(&lsp->ls_locks); + if (list_empty(&state->lock_states)) + clear_bit(LK_STATE_IN_USE, &state->flags); + spin_unlock(&state->state_lock); + server = state->owner->so_server; + if (test_bit(NFS_LOCK_INITIALIZED, &lsp->ls_flags)) { + struct nfs_client *clp = server->nfs_client; + + clp->cl_mvops->free_lock_state(server, lsp); + } else + nfs4_free_lock_state(server, lsp); +} + +static void nfs4_fl_copy_lock(struct file_lock *dst, struct file_lock *src) +{ + struct nfs4_lock_state *lsp = src->fl_u.nfs4_fl.owner; + + dst->fl_u.nfs4_fl.owner = lsp; + refcount_inc(&lsp->ls_count); +} + +static void nfs4_fl_release_lock(struct file_lock *fl) +{ + nfs4_put_lock_state(fl->fl_u.nfs4_fl.owner); +} + +static const struct file_lock_operations nfs4_fl_lock_ops = { + .fl_copy_lock = nfs4_fl_copy_lock, + .fl_release_private = nfs4_fl_release_lock, +}; + +int nfs4_set_lock_state(struct nfs4_state *state, struct file_lock *fl) +{ + struct nfs4_lock_state *lsp; + + if (fl->fl_ops != NULL) + return 0; + lsp = nfs4_get_lock_state(state, fl->c.flc_owner); + if (lsp == NULL) + return -ENOMEM; + fl->fl_u.nfs4_fl.owner = lsp; + fl->fl_ops = &nfs4_fl_lock_ops; + return 0; +} + +static int nfs4_copy_lock_stateid(nfs4_stateid *dst, + struct nfs4_state *state, + const struct nfs_lock_context *l_ctx) +{ + struct nfs4_lock_state *lsp; + fl_owner_t owner, fl_flock_owner; + int ret = -ENOENT; + + if (l_ctx == NULL) + goto out; + + if (test_bit(LK_STATE_IN_USE, &state->flags) == 0) + goto out; + + owner = l_ctx->lockowner; + fl_flock_owner = l_ctx->open_context->flock_owner; + + spin_lock(&state->state_lock); + lsp = __nfs4_find_lock_state(state, owner, fl_flock_owner); + if (lsp && test_bit(NFS_LOCK_LOST, &lsp->ls_flags)) + ret = -EIO; + else if (lsp != NULL && test_bit(NFS_LOCK_INITIALIZED, &lsp->ls_flags) != 0) { + nfs4_stateid_copy(dst, &lsp->ls_stateid); + ret = 0; + } + spin_unlock(&state->state_lock); + nfs4_put_lock_state(lsp); +out: + return ret; +} + +bool nfs4_copy_open_stateid(nfs4_stateid *dst, struct nfs4_state *state) +{ + bool ret; + const nfs4_stateid *src; + int seq; + + do { + ret = false; + src = &zero_stateid; + seq = read_seqbegin(&state->seqlock); + if (test_bit(NFS_OPEN_STATE, &state->flags)) { + src = &state->open_stateid; + ret = true; + } + nfs4_stateid_copy(dst, src); + } while (read_seqretry(&state->seqlock, seq)); + return ret; +} + +/* + * Byte-range lock aware utility to initialize the stateid of read/write + * requests. + */ +int nfs4_select_rw_stateid(struct nfs4_state *state, + fmode_t fmode, const struct nfs_lock_context *l_ctx, + nfs4_stateid *dst, const struct cred **cred) +{ + int ret; + + if (!nfs4_valid_open_stateid(state)) + return -EIO; + if (cred != NULL) + *cred = NULL; + ret = nfs4_copy_lock_stateid(dst, state, l_ctx); + if (ret == -EIO) + /* A lost lock - don't even consider delegations */ + goto out; + /* returns true if delegation stateid found and copied */ + if (nfs4_copy_delegation_stateid(state->inode, fmode, dst, cred)) { + ret = 0; + goto out; + } + if (ret != -ENOENT) + /* nfs4_copy_delegation_stateid() didn't over-write + * dst, so it still has the lock stateid which we now + * choose to use. + */ + goto out; + ret = nfs4_copy_open_stateid(dst, state) ? 0 : -EAGAIN; +out: + if (nfs_server_capable(state->inode, NFS_CAP_STATEID_NFSV41)) + dst->seqid = 0; + return ret; +} + +struct nfs_seqid *nfs_alloc_seqid(struct nfs_seqid_counter *counter, gfp_t gfp_mask) +{ + struct nfs_seqid *new; + + new = kmalloc_obj(*new, gfp_mask); + if (new == NULL) + return ERR_PTR(-ENOMEM); + new->sequence = counter; + INIT_LIST_HEAD(&new->list); + new->task = NULL; + return new; +} + +void nfs_release_seqid(struct nfs_seqid *seqid) +{ + struct nfs_seqid_counter *sequence; + + if (seqid == NULL || list_empty(&seqid->list)) + return; + sequence = seqid->sequence; + spin_lock(&sequence->lock); + if (list_is_first(&seqid->list, &sequence->list) && + !list_is_singular(&sequence->list)) { + struct nfs_seqid *next = list_next_entry(seqid, list); + rpc_wake_up_queued_task(&sequence->wait, next->task); + } + list_del_init(&seqid->list); + spin_unlock(&sequence->lock); +} + +void nfs_free_seqid(struct nfs_seqid *seqid) +{ + nfs_release_seqid(seqid); + kfree(seqid); +} + +/* + * Increment the seqid if the OPEN/OPEN_DOWNGRADE/CLOSE succeeded, or + * failed with a seqid incrementing error - + * see comments nfs4.h:seqid_mutating_error() + */ +static void nfs_increment_seqid(int status, struct nfs_seqid *seqid) +{ + switch (status) { + case 0: + break; + case -NFS4ERR_BAD_SEQID: + if (seqid->sequence->flags & NFS_SEQID_CONFIRMED) + return; + pr_warn_ratelimited("NFS: v4 server returned a bad" + " sequence-id error on an" + " unconfirmed sequence %p!\n", + seqid->sequence); + return; + case -NFS4ERR_STALE_CLIENTID: + case -NFS4ERR_STALE_STATEID: + case -NFS4ERR_BAD_STATEID: + case -NFS4ERR_BADXDR: + case -NFS4ERR_RESOURCE: + case -NFS4ERR_NOFILEHANDLE: + case -NFS4ERR_MOVED: + /* Non-seqid mutating errors */ + return; + } + /* + * Note: no locking needed as we are guaranteed to be first + * on the sequence list + */ + seqid->sequence->counter++; +} + +void nfs_increment_open_seqid(int status, struct nfs_seqid *seqid) +{ + struct nfs4_state_owner *sp; + + if (seqid == NULL) + return; + + sp = container_of(seqid->sequence, struct nfs4_state_owner, so_seqid); + if (status == -NFS4ERR_BAD_SEQID) + nfs4_reset_state_owner(sp); + if (!nfs4_has_session(sp->so_server->nfs_client)) + nfs_increment_seqid(status, seqid); +} + +/* + * Increment the seqid if the LOCK/LOCKU succeeded, or + * failed with a seqid incrementing error - + * see comments nfs4.h:seqid_mutating_error() + */ +void nfs_increment_lock_seqid(int status, struct nfs_seqid *seqid) +{ + if (seqid != NULL) + nfs_increment_seqid(status, seqid); +} + +int nfs_wait_on_sequence(struct nfs_seqid *seqid, struct rpc_task *task) +{ + struct nfs_seqid_counter *sequence; + int status = 0; + + if (seqid == NULL) + goto out; + sequence = seqid->sequence; + spin_lock(&sequence->lock); + seqid->task = task; + if (list_empty(&seqid->list)) + list_add_tail(&seqid->list, &sequence->list); + if (list_first_entry(&sequence->list, struct nfs_seqid, list) == seqid) + goto unlock; + rpc_sleep_on(&sequence->wait, task, NULL); + status = -EAGAIN; +unlock: + spin_unlock(&sequence->lock); +out: + return status; +} + +static int nfs4_run_state_manager(void *); + +static void nfs4_clear_state_manager_bit(struct nfs_client *clp) +{ + clear_and_wake_up_bit(NFS4CLNT_MANAGER_RUNNING, &clp->cl_state); + rpc_wake_up(&clp->cl_rpcwaitq); +} + +/* + * Schedule the nfs_client asynchronous state management routine + */ +void nfs4_schedule_state_manager(struct nfs_client *clp) +{ + struct task_struct *task; + char buf[INET6_ADDRSTRLEN + sizeof("-manager") + 1]; + struct rpc_clnt *clnt = clp->cl_rpcclient; + bool swapon = false; + + if (clp->cl_cons_state < 0) + return; + + set_bit(NFS4CLNT_RUN_MANAGER, &clp->cl_state); + + if (atomic_read(&clnt->cl_swapper)) { + swapon = !test_and_set_bit(NFS4CLNT_MANAGER_AVAILABLE, + &clp->cl_state); + if (!swapon) { + wake_up_var(&clp->cl_state); + return; + } + } + + if (test_and_set_bit(NFS4CLNT_MANAGER_RUNNING, &clp->cl_state) != 0) + return; + + __module_get(THIS_MODULE); + refcount_inc(&clp->cl_count); + + /* The rcu_read_lock() is not strictly necessary, as the state + * manager is the only thread that ever changes the rpc_xprt + * after it's initialized. At this point, we're single threaded. */ + rcu_read_lock(); + snprintf(buf, sizeof(buf), "%s-manager", + rpc_peeraddr2str(clp->cl_rpcclient, RPC_DISPLAY_ADDR)); + rcu_read_unlock(); + task = kthread_run(nfs4_run_state_manager, clp, "%s", buf); + if (IS_ERR(task)) { + printk(KERN_ERR "%s: kthread_run: %ld\n", + __func__, PTR_ERR(task)); + if (!nfs_client_init_is_complete(clp)) + nfs_mark_client_ready(clp, PTR_ERR(task)); + if (swapon) + clear_bit(NFS4CLNT_MANAGER_AVAILABLE, &clp->cl_state); + nfs4_clear_state_manager_bit(clp); + nfs_put_client(clp); + module_put(THIS_MODULE); + } +} + +/* + * Schedule a lease recovery attempt + */ +void nfs4_schedule_lease_recovery(struct nfs_client *clp) +{ + if (!clp) + return; + if (!test_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state)) + set_bit(NFS4CLNT_CHECK_LEASE, &clp->cl_state); + dprintk("%s: scheduling lease recovery for server %s\n", __func__, + clp->cl_hostname); + nfs4_schedule_state_manager(clp); +} +EXPORT_SYMBOL_GPL(nfs4_schedule_lease_recovery); + +/** + * nfs4_schedule_migration_recovery - trigger migration recovery + * + * @server: FSID that is migrating + * + * Returns zero if recovery has started, otherwise a negative NFS4ERR + * value is returned. + */ +int nfs4_schedule_migration_recovery(const struct nfs_server *server) +{ + struct nfs_client *clp = server->nfs_client; + + if (server->fh_expire_type != NFS4_FH_PERSISTENT) { + pr_err("NFS: volatile file handles not supported (server %s)\n", + clp->cl_hostname); + return -NFS4ERR_IO; + } + + if (test_bit(NFS_MIG_FAILED, &server->mig_status)) + return -NFS4ERR_IO; + + dprintk("%s: scheduling migration recovery for (%llx:%llx) on %s\n", + __func__, + (unsigned long long)server->fsid.major, + (unsigned long long)server->fsid.minor, + clp->cl_hostname); + + set_bit(NFS_MIG_IN_TRANSITION, + &((struct nfs_server *)server)->mig_status); + set_bit(NFS4CLNT_MOVED, &clp->cl_state); + + nfs4_schedule_state_manager(clp); + return 0; +} +EXPORT_SYMBOL_GPL(nfs4_schedule_migration_recovery); + +/** + * nfs4_schedule_lease_moved_recovery - start lease-moved recovery + * + * @clp: server to check for moved leases + * + */ +void nfs4_schedule_lease_moved_recovery(struct nfs_client *clp) +{ + dprintk("%s: scheduling lease-moved recovery for client ID %llx on %s\n", + __func__, clp->cl_clientid, clp->cl_hostname); + + set_bit(NFS4CLNT_LEASE_MOVED, &clp->cl_state); + nfs4_schedule_state_manager(clp); +} +EXPORT_SYMBOL_GPL(nfs4_schedule_lease_moved_recovery); + +int nfs4_wait_clnt_recover(struct nfs_client *clp) +{ + int res; + + might_sleep(); + + refcount_inc(&clp->cl_count); + res = wait_on_bit_action(&clp->cl_state, NFS4CLNT_MANAGER_RUNNING, + nfs_wait_bit_killable, + TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); + if (res) + goto out; + if (clp->cl_cons_state < 0) + res = clp->cl_cons_state; +out: + nfs_put_client(clp); + return res; +} + +int nfs4_client_recover_expired_lease(struct nfs_client *clp) +{ + unsigned int loop; + int ret; + + for (loop = NFS4_MAX_LOOP_ON_RECOVER; loop != 0; loop--) { + ret = nfs4_wait_clnt_recover(clp); + if (ret != 0) + break; + if (!test_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state) && + !test_bit(NFS4CLNT_CHECK_LEASE,&clp->cl_state)) + break; + nfs4_schedule_state_manager(clp); + ret = -EIO; + } + return ret; +} + +static int nfs4_state_mark_reclaim_reboot(struct nfs_client *clp, struct nfs4_state *state) +{ + + if (!nfs4_valid_open_stateid(state)) + return 0; + set_bit(NFS_STATE_RECLAIM_REBOOT, &state->flags); + /* Don't recover state that expired before the reboot */ + if (test_bit(NFS_STATE_RECLAIM_NOGRACE, &state->flags)) { + clear_bit(NFS_STATE_RECLAIM_REBOOT, &state->flags); + return 0; + } + set_bit(NFS_OWNER_RECLAIM_REBOOT, &state->owner->so_flags); + set_bit(NFS4CLNT_RECLAIM_REBOOT, &clp->cl_state); + return 1; +} + +int nfs4_state_mark_reclaim_nograce(struct nfs_client *clp, struct nfs4_state *state) +{ + if (!nfs4_valid_open_stateid(state)) + return 0; + set_bit(NFS_STATE_RECLAIM_NOGRACE, &state->flags); + clear_bit(NFS_STATE_RECLAIM_REBOOT, &state->flags); + set_bit(NFS_OWNER_RECLAIM_NOGRACE, &state->owner->so_flags); + set_bit(NFS4CLNT_RECLAIM_NOGRACE, &clp->cl_state); + return 1; +} + +int nfs4_schedule_stateid_recovery(const struct nfs_server *server, struct nfs4_state *state) +{ + struct nfs_client *clp = server->nfs_client; + + if (!nfs4_state_mark_reclaim_nograce(clp, state)) + return -EBADF; + nfs_inode_find_delegation_state_and_recover(state->inode, + &state->stateid); + dprintk("%s: scheduling stateid recovery for server %s\n", __func__, + clp->cl_hostname); + nfs4_schedule_state_manager(clp); + return clp->cl_cons_state < 0 ? clp->cl_cons_state : 0; +} +EXPORT_SYMBOL_GPL(nfs4_schedule_stateid_recovery); + +static struct nfs4_lock_state * +nfs_state_find_lock_state_by_stateid(struct nfs4_state *state, + const nfs4_stateid *stateid) +{ + struct nfs4_lock_state *pos; + + list_for_each_entry(pos, &state->lock_states, ls_locks) { + if (!test_bit(NFS_LOCK_INITIALIZED, &pos->ls_flags)) + continue; + if (nfs4_stateid_match_or_older(&pos->ls_stateid, stateid)) + return pos; + } + return NULL; +} + +static bool nfs_state_lock_state_matches_stateid(struct nfs4_state *state, + const nfs4_stateid *stateid) +{ + bool found = false; + + if (test_bit(LK_STATE_IN_USE, &state->flags)) { + spin_lock(&state->state_lock); + if (nfs_state_find_lock_state_by_stateid(state, stateid)) + found = true; + spin_unlock(&state->state_lock); + } + return found; +} + +void nfs_inode_find_state_and_recover(struct inode *inode, + const nfs4_stateid *stateid) +{ + struct nfs_client *clp = NFS_SERVER(inode)->nfs_client; + struct nfs_inode *nfsi = NFS_I(inode); + struct nfs_open_context *ctx; + struct nfs4_state *state; + bool found = false; + + if (!S_ISREG(inode->i_mode)) + goto out; + rcu_read_lock(); + list_for_each_entry_rcu(ctx, &nfsi->open_files, list) { + state = ctx->state; + if (state == NULL) + continue; + if (nfs4_stateid_match_or_older(&state->stateid, stateid) && + nfs4_state_mark_reclaim_nograce(clp, state)) { + found = true; + continue; + } + if (test_bit(NFS_OPEN_STATE, &state->flags) && + nfs4_stateid_match_or_older(&state->open_stateid, stateid) && + nfs4_state_mark_reclaim_nograce(clp, state)) { + found = true; + continue; + } + if (nfs_state_lock_state_matches_stateid(state, stateid) && + nfs4_state_mark_reclaim_nograce(clp, state)) + found = true; + } + rcu_read_unlock(); +out: + nfs_inode_find_delegation_state_and_recover(inode, stateid); + if (found) + nfs4_schedule_state_manager(clp); +} + +static void nfs4_state_mark_open_context_bad(struct nfs4_state *state, int err) +{ + struct inode *inode = state->inode; + struct nfs_inode *nfsi = NFS_I(inode); + struct nfs_open_context *ctx; + + if (!S_ISREG(inode->i_mode)) + return; + rcu_read_lock(); + list_for_each_entry_rcu(ctx, &nfsi->open_files, list) { + if (ctx->state != state) + continue; + set_bit(NFS_CONTEXT_BAD, &ctx->flags); + pr_warn("NFSv4: state recovery failed for open file %pd2, " + "error = %d\n", ctx->dentry, err); + } + rcu_read_unlock(); +} + +static void nfs4_state_mark_recovery_failed(struct nfs4_state *state, int error) +{ + set_bit(NFS_STATE_RECOVERY_FAILED, &state->flags); + nfs4_state_mark_open_context_bad(state, error); +} + + +static int nfs4_reclaim_locks(struct nfs4_state *state, const struct nfs4_state_recovery_ops *ops) +{ + struct inode *inode = state->inode; + struct nfs_inode *nfsi = NFS_I(inode); + struct file_lock *fl; + struct nfs4_lock_state *lsp; + int status = 0; + struct file_lock_context *flctx = locks_inode_context(inode); + struct list_head *list; + + if (flctx == NULL) + return 0; + + list = &flctx->flc_posix; + + /* Guard against delegation returns and new lock/unlock calls */ + down_write(&nfsi->rwsem); + spin_lock(&flctx->flc_lock); +restart: + for_each_file_lock(fl, list) { + if (nfs_file_open_context(fl->c.flc_file)->state != state) + continue; + spin_unlock(&flctx->flc_lock); + status = ops->recover_lock(state, fl); + switch (status) { + case 0: + break; + case -ETIMEDOUT: + case -ESTALE: + case -NFS4ERR_ADMIN_REVOKED: + case -NFS4ERR_STALE_STATEID: + case -NFS4ERR_BAD_STATEID: + case -NFS4ERR_EXPIRED: + case -NFS4ERR_NO_GRACE: + case -NFS4ERR_STALE_CLIENTID: + case -NFS4ERR_BADSESSION: + case -NFS4ERR_BADSLOT: + case -NFS4ERR_BAD_HIGH_SLOT: + case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION: + goto out; + default: + pr_err("NFS: %s: unhandled error %d\n", + __func__, status); + fallthrough; + case -ENOMEM: + case -NFS4ERR_DENIED: + case -NFS4ERR_RECLAIM_BAD: + case -NFS4ERR_RECLAIM_CONFLICT: + lsp = fl->fl_u.nfs4_fl.owner; + if (lsp) + set_bit(NFS_LOCK_LOST, &lsp->ls_flags); + status = 0; + } + spin_lock(&flctx->flc_lock); + } + if (list == &flctx->flc_posix) { + list = &flctx->flc_flock; + goto restart; + } + spin_unlock(&flctx->flc_lock); +out: + up_write(&nfsi->rwsem); + return status; +} + +#ifdef CONFIG_NFS_V4_2 +static void nfs42_complete_copies(struct nfs4_state_owner *sp, struct nfs4_state *state) +{ + struct nfs4_copy_state *copy; + + if (!test_bit(NFS_CLNT_DST_SSC_COPY_STATE, &state->flags) && + !test_bit(NFS_CLNT_SRC_SSC_COPY_STATE, &state->flags)) + return; + + spin_lock(&sp->so_server->nfs_client->cl_lock); + list_for_each_entry(copy, &sp->so_server->ss_copies, copies) { + if ((test_bit(NFS_CLNT_DST_SSC_COPY_STATE, &state->flags) && + !nfs4_stateid_match_other(&state->stateid, + ©->parent_dst_state->stateid))) + continue; + copy->flags = 1; + if (test_and_clear_bit(NFS_CLNT_DST_SSC_COPY_STATE, + &state->flags)) { + clear_bit(NFS_CLNT_SRC_SSC_COPY_STATE, &state->flags); + complete(©->completion); + } + } + list_for_each_entry(copy, &sp->so_server->ss_src_copies, src_copies) { + if ((test_bit(NFS_CLNT_SRC_SSC_COPY_STATE, &state->flags) && + !nfs4_stateid_match_other(&state->stateid, + ©->parent_src_state->stateid))) + continue; + copy->flags = 1; + if (test_and_clear_bit(NFS_CLNT_DST_SSC_COPY_STATE, + &state->flags)) + complete(©->completion); + } + spin_unlock(&sp->so_server->nfs_client->cl_lock); +} +#else /* !CONFIG_NFS_V4_2 */ +static inline void nfs42_complete_copies(struct nfs4_state_owner *sp, + struct nfs4_state *state) +{ +} +#endif /* CONFIG_NFS_V4_2 */ + +static int __nfs4_reclaim_open_state(struct nfs4_state_owner *sp, struct nfs4_state *state, + const struct nfs4_state_recovery_ops *ops, + int *lost_locks) +{ + struct nfs4_lock_state *lock; + int status; + + status = ops->recover_open(sp, state); + if (status < 0) + return status; + + status = nfs4_reclaim_locks(state, ops); + if (status < 0) + return status; + + if (!test_bit(NFS_DELEGATED_STATE, &state->flags)) { + spin_lock(&state->state_lock); + list_for_each_entry(lock, &state->lock_states, ls_locks) { + trace_nfs4_state_lock_reclaim(state, lock); + if (!test_bit(NFS_LOCK_INITIALIZED, &lock->ls_flags) && + !test_bit(NFS_LOCK_UNLOCKING, &lock->ls_flags)) + *lost_locks += 1; + } + spin_unlock(&state->state_lock); + } + + nfs42_complete_copies(sp, state); + clear_bit(NFS_STATE_RECLAIM_NOGRACE, &state->flags); + return status; +} + +static int nfs4_reclaim_open_state(struct nfs4_state_owner *sp, + const struct nfs4_state_recovery_ops *ops, + int *lost_locks) +{ + struct nfs4_state *state; + unsigned int loop = 0; + int status = 0; +#ifdef CONFIG_NFS_V4_2 + bool found_ssc_copy_state = false; +#endif /* CONFIG_NFS_V4_2 */ + + /* Note: we rely on the sp->so_states list being ordered + * so that we always reclaim open(O_RDWR) and/or open(O_WRITE) + * states first. + * This is needed to ensure that the server won't give us any + * read delegations that we have to return if, say, we are + * recovering after a network partition or a reboot from a + * server that doesn't support a grace period. + */ + spin_lock(&sp->so_lock); +restart: + list_for_each_entry(state, &sp->so_states, open_states) { + if (!test_and_clear_bit(ops->state_flag_bit, &state->flags)) + continue; + if (!nfs4_valid_open_stateid(state)) + continue; + if (state->state == 0) + continue; +#ifdef CONFIG_NFS_V4_2 + if (test_bit(NFS_SRV_SSC_COPY_STATE, &state->flags)) { + nfs4_state_mark_recovery_failed(state, -EIO); + found_ssc_copy_state = true; + continue; + } +#endif /* CONFIG_NFS_V4_2 */ + refcount_inc(&state->count); + spin_unlock(&sp->so_lock); + status = __nfs4_reclaim_open_state(sp, state, ops, lost_locks); + + switch (status) { + default: + if (status >= 0) { + loop = 0; + break; + } + printk(KERN_ERR "NFS: %s: unhandled error %d\n", __func__, status); + fallthrough; + case -ENOENT: + case -ENOMEM: + case -EACCES: + case -EROFS: + case -EIO: + case -ESTALE: + /* Open state on this file cannot be recovered */ + nfs4_state_mark_recovery_failed(state, status); + break; + case -EAGAIN: + ssleep(1); + if (loop++ < 10) { + set_bit(ops->state_flag_bit, &state->flags); + break; + } + fallthrough; + case -NFS4ERR_ADMIN_REVOKED: + case -NFS4ERR_STALE_STATEID: + case -NFS4ERR_OLD_STATEID: + case -NFS4ERR_BAD_STATEID: + case -NFS4ERR_RECLAIM_BAD: + case -NFS4ERR_RECLAIM_CONFLICT: + nfs4_state_mark_reclaim_nograce(sp->so_server->nfs_client, state); + break; + case -NFS4ERR_EXPIRED: + case -NFS4ERR_NO_GRACE: + nfs4_state_mark_reclaim_nograce(sp->so_server->nfs_client, state); + fallthrough; + case -NFS4ERR_STALE_CLIENTID: + case -NFS4ERR_BADSESSION: + case -NFS4ERR_BADSLOT: + case -NFS4ERR_BAD_HIGH_SLOT: + case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION: + case -ETIMEDOUT: + goto out_err; + } + nfs4_put_open_state(state); + spin_lock(&sp->so_lock); + goto restart; + } + spin_unlock(&sp->so_lock); +#ifdef CONFIG_NFS_V4_2 + if (found_ssc_copy_state) + return -EIO; +#endif /* CONFIG_NFS_V4_2 */ + return 0; +out_err: + nfs4_put_open_state(state); + spin_lock(&sp->so_lock); + spin_unlock(&sp->so_lock); + return status; +} + +static void nfs4_clear_open_state(struct nfs4_state *state) +{ + struct nfs4_lock_state *lock; + + clear_bit(NFS_DELEGATED_STATE, &state->flags); + clear_bit(NFS_O_RDONLY_STATE, &state->flags); + clear_bit(NFS_O_WRONLY_STATE, &state->flags); + clear_bit(NFS_O_RDWR_STATE, &state->flags); + spin_lock(&state->state_lock); + list_for_each_entry(lock, &state->lock_states, ls_locks) { + lock->ls_seqid.flags = 0; + clear_bit(NFS_LOCK_INITIALIZED, &lock->ls_flags); + } + spin_unlock(&state->state_lock); +} + +static void nfs4_reset_seqids(struct nfs_server *server, + int (*mark_reclaim)(struct nfs_client *clp, struct nfs4_state *state)) +{ + struct nfs_client *clp = server->nfs_client; + struct nfs4_state_owner *sp; + struct rb_node *pos; + struct nfs4_state *state; + + spin_lock(&clp->cl_lock); + for (pos = rb_first(&server->state_owners); + pos != NULL; + pos = rb_next(pos)) { + sp = rb_entry(pos, struct nfs4_state_owner, so_server_node); + sp->so_seqid.flags = 0; + spin_lock(&sp->so_lock); + list_for_each_entry(state, &sp->so_states, open_states) { + if (mark_reclaim(clp, state)) + nfs4_clear_open_state(state); + } + spin_unlock(&sp->so_lock); + } + spin_unlock(&clp->cl_lock); +} + +static void nfs4_state_mark_reclaim_helper(struct nfs_client *clp, + int (*mark_reclaim)(struct nfs_client *clp, struct nfs4_state *state)) +{ + struct nfs_server *server; + + rcu_read_lock(); + list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) + nfs4_reset_seqids(server, mark_reclaim); + rcu_read_unlock(); +} + +static void nfs4_state_start_reclaim_reboot(struct nfs_client *clp) +{ + set_bit(NFS4CLNT_RECLAIM_REBOOT, &clp->cl_state); + /* Mark all delegations for reclaim */ + nfs_delegation_mark_reclaim(clp); + nfs4_state_mark_reclaim_helper(clp, nfs4_state_mark_reclaim_reboot); +} + +static int nfs4_reclaim_complete(struct nfs_client *clp, + const struct nfs4_state_recovery_ops *ops, + const struct cred *cred) +{ + /* Notify the server we're done reclaiming our state */ + if (ops->reclaim_complete) + return ops->reclaim_complete(clp, cred); + return 0; +} + +static void nfs4_clear_reclaim_server(struct nfs_server *server) +{ + struct nfs_client *clp = server->nfs_client; + struct nfs4_state_owner *sp; + struct rb_node *pos; + struct nfs4_state *state; + + spin_lock(&clp->cl_lock); + for (pos = rb_first(&server->state_owners); + pos != NULL; + pos = rb_next(pos)) { + sp = rb_entry(pos, struct nfs4_state_owner, so_server_node); + spin_lock(&sp->so_lock); + list_for_each_entry(state, &sp->so_states, open_states) { + if (!test_and_clear_bit(NFS_STATE_RECLAIM_REBOOT, + &state->flags)) + continue; + nfs4_state_mark_reclaim_nograce(clp, state); + } + spin_unlock(&sp->so_lock); + } + spin_unlock(&clp->cl_lock); +} + +static int nfs4_state_clear_reclaim_reboot(struct nfs_client *clp) +{ + struct nfs_server *server; + + if (!test_and_clear_bit(NFS4CLNT_RECLAIM_REBOOT, &clp->cl_state)) + return 0; + + rcu_read_lock(); + list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) + nfs4_clear_reclaim_server(server); + rcu_read_unlock(); + + nfs_delegation_reap_unclaimed(clp); + return 1; +} + +static void nfs4_state_end_reclaim_reboot(struct nfs_client *clp) +{ + const struct nfs4_state_recovery_ops *ops; + const struct cred *cred; + int err; + + if (!nfs4_state_clear_reclaim_reboot(clp)) + return; + pnfs_destroy_all_layouts(clp); + ops = clp->cl_mvops->reboot_recovery_ops; + cred = nfs4_get_clid_cred(clp); + err = nfs4_reclaim_complete(clp, ops, cred); + put_cred(cred); + if (err == -NFS4ERR_CONN_NOT_BOUND_TO_SESSION) + set_bit(NFS4CLNT_RECLAIM_REBOOT, &clp->cl_state); +} + +static void nfs4_state_start_reclaim_nograce(struct nfs_client *clp) +{ + nfs_mark_test_expired_all_delegations(clp); + nfs4_state_mark_reclaim_helper(clp, nfs4_state_mark_reclaim_nograce); +} + +static int nfs4_recovery_handle_error(struct nfs_client *clp, int error) +{ + switch (error) { + case 0: + break; +#if IS_ENABLED(CONFIG_NFS_V4_0) + case -NFS4ERR_CB_PATH_DOWN: + nfs40_handle_cb_pathdown(clp); + break; +#endif /* CONFIG_NFS_V4_0 */ + case -NFS4ERR_NO_GRACE: + nfs4_state_end_reclaim_reboot(clp); + break; + case -NFS4ERR_STALE_CLIENTID: + set_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state); + nfs4_state_start_reclaim_reboot(clp); + break; + case -NFS4ERR_EXPIRED: + set_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state); + nfs4_state_start_reclaim_nograce(clp); + break; + case -NFS4ERR_BADSESSION: + case -NFS4ERR_BADSLOT: + case -NFS4ERR_BAD_HIGH_SLOT: + case -NFS4ERR_DEADSESSION: + case -NFS4ERR_SEQ_FALSE_RETRY: + case -NFS4ERR_SEQ_MISORDERED: + set_bit(NFS4CLNT_SESSION_RESET, &clp->cl_state); + /* Zero session reset errors */ + break; + case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION: + set_bit(NFS4CLNT_BIND_CONN_TO_SESSION, &clp->cl_state); + break; + default: + dprintk("%s: failed to handle error %d for server %s\n", + __func__, error, clp->cl_hostname); + return error; + } + dprintk("%s: handled error %d for server %s\n", __func__, error, + clp->cl_hostname); + return 0; +} + +static int nfs4_do_reclaim(struct nfs_client *clp, const struct nfs4_state_recovery_ops *ops) +{ + struct nfs4_state_owner *sp; + struct nfs_server *server; + struct rb_node *pos; + LIST_HEAD(freeme); + int lost_locks = 0; + int status; + + status = nfs4_begin_drain_session(clp); + if (status < 0) + return status; +restart: + rcu_read_lock(); + list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { + nfs4_purge_state_owners(server, &freeme); + spin_lock(&clp->cl_lock); + for (pos = rb_first(&server->state_owners); + pos != NULL; + pos = rb_next(pos)) { + sp = rb_entry(pos, + struct nfs4_state_owner, so_server_node); + if (!test_and_clear_bit(ops->owner_flag_bit, + &sp->so_flags)) + continue; + if (!atomic_inc_not_zero(&sp->so_count)) + continue; + spin_unlock(&clp->cl_lock); + rcu_read_unlock(); + + status = nfs4_reclaim_open_state(sp, ops, &lost_locks); + if (status < 0) { + if (lost_locks) + pr_warn("NFS: %s: lost %d locks\n", + clp->cl_hostname, lost_locks); + set_bit(ops->owner_flag_bit, &sp->so_flags); + nfs4_put_state_owner(sp); + status = nfs4_recovery_handle_error(clp, status); + nfs4_free_state_owners(&freeme); + return (status != 0) ? status : -EAGAIN; + } + + nfs4_put_state_owner(sp); + goto restart; + } + spin_unlock(&clp->cl_lock); + } + rcu_read_unlock(); + nfs4_free_state_owners(&freeme); + nfs_local_probe_async(clp); + if (lost_locks) + pr_warn("NFS: %s: lost %d locks\n", + clp->cl_hostname, lost_locks); + return 0; +} + +static int nfs4_check_lease(struct nfs_client *clp) +{ + const struct cred *cred; + const struct nfs4_state_maintenance_ops *ops = + clp->cl_mvops->state_renewal_ops; + int status; + + /* Is the client already known to have an expired lease? */ + if (test_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state)) + return 0; + cred = ops->get_state_renewal_cred(clp); + if (cred == NULL) { + cred = nfs4_get_clid_cred(clp); + status = -ENOKEY; + if (cred == NULL) + goto out; + } + status = ops->renew_lease(clp, cred); + put_cred(cred); + if (status == -ETIMEDOUT) { + set_bit(NFS4CLNT_CHECK_LEASE, &clp->cl_state); + return 0; + } +out: + return nfs4_recovery_handle_error(clp, status); +} + +/* Set NFS4CLNT_LEASE_EXPIRED and reclaim reboot state for all v4.0 errors + * and for recoverable errors on EXCHANGE_ID for v4.1 + */ +static int nfs4_handle_reclaim_lease_error(struct nfs_client *clp, int status) +{ + switch (status) { + case -NFS4ERR_SEQ_MISORDERED: + if (test_and_set_bit(NFS4CLNT_PURGE_STATE, &clp->cl_state)) + return -ESERVERFAULT; + /* Lease confirmation error: retry after purging the lease */ + ssleep(1); + clear_bit(NFS4CLNT_LEASE_CONFIRM, &clp->cl_state); + break; + case -NFS4ERR_STALE_CLIENTID: + clear_bit(NFS4CLNT_LEASE_CONFIRM, &clp->cl_state); + nfs4_state_start_reclaim_reboot(clp); + break; + case -NFS4ERR_CLID_INUSE: + pr_err("NFS: Server %s reports our clientid is in use\n", + clp->cl_hostname); + nfs_mark_client_ready(clp, -EPERM); + clear_bit(NFS4CLNT_LEASE_CONFIRM, &clp->cl_state); + return -EPERM; + case -ETIMEDOUT: + if (clp->cl_cons_state == NFS_CS_SESSION_INITING) { + nfs_mark_client_ready(clp, -EIO); + return -EIO; + } + fallthrough; + case -EACCES: + case -NFS4ERR_DELAY: + case -EAGAIN: + ssleep(1); + break; + + case -NFS4ERR_MINOR_VERS_MISMATCH: + if (clp->cl_cons_state == NFS_CS_SESSION_INITING) + nfs_mark_client_ready(clp, -EPROTONOSUPPORT); + dprintk("%s: exit with error %d for server %s\n", + __func__, -EPROTONOSUPPORT, clp->cl_hostname); + return -EPROTONOSUPPORT; + case -ENOSPC: + if (clp->cl_cons_state == NFS_CS_SESSION_INITING) + nfs_mark_client_ready(clp, -EIO); + return -EIO; + case -NFS4ERR_NOT_SAME: /* FixMe: implement recovery + * in nfs4_exchange_id */ + default: + dprintk("%s: exit with error %d for server %s\n", __func__, + status, clp->cl_hostname); + return status; + } + set_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state); + dprintk("%s: handled error %d for server %s\n", __func__, status, + clp->cl_hostname); + return 0; +} + +static int nfs4_establish_lease(struct nfs_client *clp) +{ + const struct cred *cred; + const struct nfs4_state_recovery_ops *ops = + clp->cl_mvops->reboot_recovery_ops; + int status; + + status = nfs4_begin_drain_session(clp); + if (status != 0) + return status; + cred = nfs4_get_clid_cred(clp); + if (cred == NULL) + return -ENOENT; + status = ops->establish_clid(clp, cred); + put_cred(cred); + if (status != 0) + return status; + return 0; +} + +/* + * Returns zero or a negative errno. NFS4ERR values are converted + * to local errno values. + */ +static int nfs4_reclaim_lease(struct nfs_client *clp) +{ + int status; + + status = nfs4_establish_lease(clp); + if (status < 0) + return nfs4_handle_reclaim_lease_error(clp, status); + if (test_and_clear_bit(NFS4CLNT_SERVER_SCOPE_MISMATCH, &clp->cl_state)) + nfs4_state_start_reclaim_nograce(clp); + if (!test_bit(NFS4CLNT_RECLAIM_NOGRACE, &clp->cl_state)) + set_bit(NFS4CLNT_RECLAIM_REBOOT, &clp->cl_state); + clear_bit(NFS4CLNT_CHECK_LEASE, &clp->cl_state); + clear_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state); + return 0; +} + +static int nfs4_purge_lease(struct nfs_client *clp) +{ + int status; + + status = nfs4_establish_lease(clp); + if (status < 0) + return nfs4_handle_reclaim_lease_error(clp, status); + clear_bit(NFS4CLNT_PURGE_STATE, &clp->cl_state); + set_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state); + nfs4_state_start_reclaim_nograce(clp); + return 0; +} + +/* + * Try remote migration of one FSID from a source server to a + * destination server. The source server provides a list of + * potential destinations. + * + * Returns zero or a negative NFS4ERR status code. + */ +static int nfs4_try_migration(struct nfs_server *server, const struct cred *cred) +{ + struct nfs_client *clp = server->nfs_client; + struct nfs4_fs_locations *locations = NULL; + struct nfs_fattr *fattr; + struct inode *inode; + struct page *page; + int status, result; + + dprintk("--> %s: FSID %llx:%llx on \"%s\"\n", __func__, + (unsigned long long)server->fsid.major, + (unsigned long long)server->fsid.minor, + clp->cl_hostname); + + page = alloc_page(GFP_KERNEL); + locations = kmalloc_obj(struct nfs4_fs_locations); + fattr = nfs_alloc_fattr(); + if (page == NULL || locations == NULL || fattr == NULL) { + dprintk("<-- %s: no memory\n", __func__); + result = 0; + goto out; + } + + locations->fattr = fattr; + inode = d_inode(server->super->s_root); + result = nfs4_proc_get_locations(server, NFS_FH(inode), locations, + page, cred); + if (result) { + dprintk("<-- %s: failed to retrieve fs_locations: %d\n", + __func__, result); + goto out; + } + + result = -NFS4ERR_NXIO; + if (!locations->nlocations) + goto out; + + if (!(locations->fattr->valid & NFS_ATTR_FATTR_V4_LOCATIONS)) { + dprintk("<-- %s: No fs_locations data, migration skipped\n", + __func__); + goto out; + } + + status = nfs4_begin_drain_session(clp); + if (status != 0) { + result = status; + goto out; + } + + status = nfs4_replace_transport(server, locations); + if (status != 0) { + dprintk("<-- %s: failed to replace transport: %d\n", + __func__, status); + goto out; + } + + result = 0; + dprintk("<-- %s: migration succeeded\n", __func__); + +out: + if (page != NULL) + __free_page(page); + if (locations != NULL) + kfree(locations->fattr); + kfree(locations); + if (result) { + pr_err("NFS: migration recovery failed (server %s)\n", + clp->cl_hostname); + set_bit(NFS_MIG_FAILED, &server->mig_status); + } + return result; +} + +/* + * Returns zero or a negative NFS4ERR status code. + */ +static int nfs4_handle_migration(struct nfs_client *clp) +{ + const struct nfs4_state_maintenance_ops *ops = + clp->cl_mvops->state_renewal_ops; + struct nfs_server *server; + const struct cred *cred; + + dprintk("%s: migration reported on \"%s\"\n", __func__, + clp->cl_hostname); + + cred = ops->get_state_renewal_cred(clp); + if (cred == NULL) + return -NFS4ERR_NOENT; + + clp->cl_mig_gen++; +restart: + rcu_read_lock(); + list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { + int status; + + if (server->mig_gen == clp->cl_mig_gen) + continue; + server->mig_gen = clp->cl_mig_gen; + + if (!test_and_clear_bit(NFS_MIG_IN_TRANSITION, + &server->mig_status)) + continue; + + rcu_read_unlock(); + status = nfs4_try_migration(server, cred); + if (status < 0) { + put_cred(cred); + return status; + } + goto restart; + } + rcu_read_unlock(); + put_cred(cred); + return 0; +} + +/* + * Test each nfs_server on the clp's cl_superblocks list to see + * if it's moved to another server. Stop when the server no longer + * returns NFS4ERR_LEASE_MOVED. + */ +static int nfs4_handle_lease_moved(struct nfs_client *clp) +{ + const struct nfs4_state_maintenance_ops *ops = + clp->cl_mvops->state_renewal_ops; + struct nfs_server *server; + const struct cred *cred; + + dprintk("%s: lease moved reported on \"%s\"\n", __func__, + clp->cl_hostname); + + cred = ops->get_state_renewal_cred(clp); + if (cred == NULL) + return -NFS4ERR_NOENT; + + clp->cl_mig_gen++; +restart: + rcu_read_lock(); + list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { + struct inode *inode; + int status; + + if (server->mig_gen == clp->cl_mig_gen) + continue; + server->mig_gen = clp->cl_mig_gen; + + rcu_read_unlock(); + + inode = d_inode(server->super->s_root); + status = nfs4_proc_fsid_present(inode, cred); + if (status != -NFS4ERR_MOVED) + goto restart; /* wasn't this one */ + if (nfs4_try_migration(server, cred) == -NFS4ERR_LEASE_MOVED) + goto restart; /* there are more */ + goto out; + } + rcu_read_unlock(); + +out: + put_cred(cred); + return 0; +} + +/** + * nfs4_discover_server_trunking - Detect server IP address trunking + * + * @clp: nfs_client under test + * @result: OUT: found nfs_client, or clp + * + * Returns zero or a negative errno. If zero is returned, + * an nfs_client pointer is planted in "result". + * + * Note: since we are invoked in process context, and + * not from inside the state manager, we cannot use + * nfs4_handle_reclaim_lease_error(). + */ +int nfs4_discover_server_trunking(struct nfs_client *clp, + struct nfs_client **result) +{ + const struct nfs4_state_recovery_ops *ops = + clp->cl_mvops->reboot_recovery_ops; + struct rpc_clnt *clnt; + const struct cred *cred; + int i, status; + + dprintk("NFS: %s: testing '%s'\n", __func__, clp->cl_hostname); + + clnt = clp->cl_rpcclient; + i = 0; + + mutex_lock(&nfs_clid_init_mutex); +again: + status = -ENOENT; + cred = nfs4_get_clid_cred(clp); + if (cred == NULL) + goto out_unlock; + + status = ops->detect_trunking(clp, result, cred); + put_cred(cred); + switch (status) { + case 0: + case -EINTR: + case -ERESTARTSYS: + break; + case -ETIMEDOUT: + if (clnt->cl_softrtry) + break; + fallthrough; + case -NFS4ERR_DELAY: + case -EAGAIN: + ssleep(1); + fallthrough; + case -NFS4ERR_STALE_CLIENTID: + dprintk("NFS: %s after status %d, retrying\n", + __func__, status); + goto again; + case -EACCES: + if (i++ == 0) { + nfs4_root_machine_cred(clp); + goto again; + } + if (clnt->cl_auth->au_flavor == RPC_AUTH_UNIX) + break; + fallthrough; + case -NFS4ERR_CLID_INUSE: + case -NFS4ERR_WRONGSEC: + /* No point in retrying if we already used RPC_AUTH_UNIX */ + if (clnt->cl_auth->au_flavor == RPC_AUTH_UNIX) { + status = -EPERM; + break; + } + clnt = rpc_clone_client_set_auth(clnt, RPC_AUTH_UNIX); + if (IS_ERR(clnt)) { + status = PTR_ERR(clnt); + break; + } + /* Note: this is safe because we haven't yet marked the + * client as ready, so we are the only user of + * clp->cl_rpcclient + */ + clnt = xchg(&clp->cl_rpcclient, clnt); + rpc_shutdown_client(clnt); + clnt = clp->cl_rpcclient; + goto again; + + case -NFS4ERR_MINOR_VERS_MISMATCH: + status = -EPROTONOSUPPORT; + break; + + case -EKEYEXPIRED: + case -NFS4ERR_NOT_SAME: /* FixMe: implement recovery + * in nfs4_exchange_id */ + status = -EKEYEXPIRED; + break; + default: + pr_warn("NFS: %s unhandled error %d. Exiting with error EIO\n", + __func__, status); + status = -EIO; + } + +out_unlock: + mutex_unlock(&nfs_clid_init_mutex); + dprintk("NFS: %s: status = %d\n", __func__, status); + return status; +} + +void nfs4_schedule_session_recovery(struct nfs4_session *session, int err) +{ + struct nfs_client *clp = session->clp; + + switch (err) { + default: + set_bit(NFS4CLNT_SESSION_RESET, &clp->cl_state); + break; + case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION: + set_bit(NFS4CLNT_BIND_CONN_TO_SESSION, &clp->cl_state); + } + nfs4_schedule_state_manager(clp); +} +EXPORT_SYMBOL_GPL(nfs4_schedule_session_recovery); + +void nfs41_notify_server(struct nfs_client *clp) +{ + /* Use CHECK_LEASE to ping the server with a SEQUENCE */ + set_bit(NFS4CLNT_CHECK_LEASE, &clp->cl_state); + nfs4_schedule_state_manager(clp); +} + +static void nfs4_reset_all_state(struct nfs_client *clp) +{ + if (test_and_set_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state) == 0) { + set_bit(NFS4CLNT_PURGE_STATE, &clp->cl_state); + clear_bit(NFS4CLNT_LEASE_CONFIRM, &clp->cl_state); + nfs4_state_start_reclaim_nograce(clp); + dprintk("%s: scheduling reset of all state for server %s!\n", + __func__, clp->cl_hostname); + nfs4_schedule_state_manager(clp); + } +} + +static void nfs41_handle_server_reboot(struct nfs_client *clp) +{ + if (test_and_set_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state) == 0) { + nfs4_state_start_reclaim_reboot(clp); + dprintk("%s: server %s rebooted!\n", __func__, + clp->cl_hostname); + nfs4_schedule_state_manager(clp); + } +} + +static void nfs41_handle_all_state_revoked(struct nfs_client *clp) +{ + nfs4_reset_all_state(clp); + dprintk("%s: state revoked on server %s\n", __func__, clp->cl_hostname); +} + +static void nfs41_handle_some_state_revoked(struct nfs_client *clp) +{ + nfs4_state_start_reclaim_nograce(clp); + nfs4_schedule_state_manager(clp); + + dprintk("%s: state revoked on server %s\n", __func__, clp->cl_hostname); +} + +static void nfs41_handle_recallable_state_revoked(struct nfs_client *clp) +{ + /* FIXME: For now, we destroy all layouts. */ + pnfs_destroy_all_layouts(clp); + nfs_test_expired_all_delegations(clp); + dprintk("%s: Recallable state revoked on server %s!\n", __func__, + clp->cl_hostname); +} + +static void nfs41_handle_backchannel_fault(struct nfs_client *clp) +{ + set_bit(NFS4CLNT_SESSION_RESET, &clp->cl_state); + nfs4_schedule_state_manager(clp); + + dprintk("%s: server %s declared a backchannel fault\n", __func__, + clp->cl_hostname); +} + +static void nfs41_handle_cb_path_down(struct nfs_client *clp) +{ + if (test_and_set_bit(NFS4CLNT_BIND_CONN_TO_SESSION, + &clp->cl_state) == 0) + nfs4_schedule_state_manager(clp); +} + +void nfs41_handle_sequence_flag_errors(struct nfs_client *clp, u32 flags, + bool recovery) +{ + if (!flags) + return; + + dprintk("%s: \"%s\" (client ID %llx) flags=0x%08x\n", + __func__, clp->cl_hostname, clp->cl_clientid, flags); + /* + * If we're called from the state manager thread, then assume we're + * already handling the RECLAIM_NEEDED and/or STATE_REVOKED. + * Those flags are expected to remain set until we're done + * recovering (see RFC5661, section 18.46.3). + */ + if (recovery) + goto out_recovery; + + if (flags & SEQ4_STATUS_RESTART_RECLAIM_NEEDED) + nfs41_handle_server_reboot(clp); + if (flags & (SEQ4_STATUS_EXPIRED_ALL_STATE_REVOKED)) + nfs41_handle_all_state_revoked(clp); + if (flags & (SEQ4_STATUS_EXPIRED_SOME_STATE_REVOKED | + SEQ4_STATUS_ADMIN_STATE_REVOKED)) + nfs41_handle_some_state_revoked(clp); + if (flags & SEQ4_STATUS_LEASE_MOVED) + nfs4_schedule_lease_moved_recovery(clp); + if (flags & SEQ4_STATUS_RECALLABLE_STATE_REVOKED) + nfs41_handle_recallable_state_revoked(clp); +out_recovery: + if (flags & SEQ4_STATUS_BACKCHANNEL_FAULT) + nfs41_handle_backchannel_fault(clp); + else if (flags & (SEQ4_STATUS_CB_PATH_DOWN | + SEQ4_STATUS_CB_PATH_DOWN_SESSION)) + nfs41_handle_cb_path_down(clp); +} + +static int nfs4_reset_session(struct nfs_client *clp) +{ + const struct cred *cred; + int status; + + if (!nfs4_has_session(clp)) + return 0; + status = nfs4_begin_drain_session(clp); + if (status != 0) + return status; + cred = nfs4_get_clid_cred(clp); + status = nfs4_proc_destroy_session(clp->cl_session, cred); + switch (status) { + case 0: + case -NFS4ERR_BADSESSION: + case -NFS4ERR_DEADSESSION: + break; + case -NFS4ERR_BACK_CHAN_BUSY: + case -NFS4ERR_DELAY: + set_bit(NFS4CLNT_SESSION_RESET, &clp->cl_state); + status = 0; + ssleep(1); + goto out; + default: + status = nfs4_recovery_handle_error(clp, status); + goto out; + } + + memset(clp->cl_session->sess_id.data, 0, NFS4_MAX_SESSIONID_LEN); + status = nfs4_proc_create_session(clp, cred); + if (status) { + dprintk("%s: session reset failed with status %d for server %s!\n", + __func__, status, clp->cl_hostname); + status = nfs4_handle_reclaim_lease_error(clp, status); + goto out; + } + nfs41_finish_session_reset(clp); + dprintk("%s: session reset was successful for server %s!\n", + __func__, clp->cl_hostname); +out: + put_cred(cred); + return status; +} + +static int nfs4_bind_conn_to_session(struct nfs_client *clp) +{ + const struct cred *cred; + int ret; + + if (!nfs4_has_session(clp)) + return 0; + ret = nfs4_begin_drain_session(clp); + if (ret != 0) + return ret; + cred = nfs4_get_clid_cred(clp); + ret = nfs4_proc_bind_conn_to_session(clp, cred); + put_cred(cred); + clear_bit(NFS4CLNT_BIND_CONN_TO_SESSION, &clp->cl_state); + switch (ret) { + case 0: + dprintk("%s: bind_conn_to_session was successful for server %s!\n", + __func__, clp->cl_hostname); + break; + case -NFS4ERR_DELAY: + ssleep(1); + set_bit(NFS4CLNT_BIND_CONN_TO_SESSION, &clp->cl_state); + break; + default: + return nfs4_recovery_handle_error(clp, ret); + } + return 0; +} + +static void nfs4_layoutreturn_any_run(struct nfs_client *clp) +{ + int iomode = 0; + + if (test_and_clear_bit(NFS4CLNT_RECALL_ANY_LAYOUT_READ, &clp->cl_state)) + iomode += IOMODE_READ; + if (test_and_clear_bit(NFS4CLNT_RECALL_ANY_LAYOUT_RW, &clp->cl_state)) + iomode += IOMODE_RW; + /* Note: IOMODE_READ + IOMODE_RW == IOMODE_ANY */ + if (iomode) { + pnfs_layout_return_unused_byclid(clp, iomode); + set_bit(NFS4CLNT_RUN_MANAGER, &clp->cl_state); + } +} + +static void nfs4_state_manager(struct nfs_client *clp) +{ + unsigned int memflags; + int status = 0; + const char *section = "", *section_sep = ""; + + /* + * State recovery can deadlock if the direct reclaim code tries + * start NFS writeback. So ensure memory allocations are all + * GFP_NOFS. + */ + memflags = memalloc_nofs_save(); + + /* Ensure exclusive access to NFSv4 state */ + do { + trace_nfs4_state_mgr(clp); + clear_bit(NFS4CLNT_RUN_MANAGER, &clp->cl_state); + if (test_bit(NFS4CLNT_PURGE_STATE, &clp->cl_state)) { + section = "purge state"; + status = nfs4_purge_lease(clp); + if (status < 0) + goto out_error; + continue; + } + + if (test_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state)) { + section = "lease expired"; + /* We're going to have to re-establish a clientid */ + status = nfs4_reclaim_lease(clp); + if (status < 0) + goto out_error; + continue; + } + + /* Initialize or reset the session */ + if (test_and_clear_bit(NFS4CLNT_SESSION_RESET, &clp->cl_state)) { + section = "reset session"; + status = nfs4_reset_session(clp); + if (test_bit(NFS4CLNT_LEASE_EXPIRED, &clp->cl_state)) + continue; + if (status < 0) + goto out_error; + } + + /* Send BIND_CONN_TO_SESSION */ + if (test_and_clear_bit(NFS4CLNT_BIND_CONN_TO_SESSION, + &clp->cl_state)) { + section = "bind conn to session"; + status = nfs4_bind_conn_to_session(clp); + if (status < 0) + goto out_error; + continue; + } + + if (test_and_clear_bit(NFS4CLNT_CHECK_LEASE, &clp->cl_state)) { + section = "check lease"; + status = nfs4_check_lease(clp); + if (status < 0) + goto out_error; + continue; + } + + if (test_and_clear_bit(NFS4CLNT_MOVED, &clp->cl_state)) { + section = "migration"; + status = nfs4_handle_migration(clp); + if (status < 0) + goto out_error; + } + + if (test_and_clear_bit(NFS4CLNT_LEASE_MOVED, &clp->cl_state)) { + section = "lease moved"; + status = nfs4_handle_lease_moved(clp); + if (status < 0) + goto out_error; + } + + /* First recover reboot state... */ + if (test_bit(NFS4CLNT_RECLAIM_REBOOT, &clp->cl_state)) { + section = "reclaim reboot"; + status = nfs4_do_reclaim(clp, + clp->cl_mvops->reboot_recovery_ops); + if (status == 0) + status = pnfs_layout_handle_reboot(clp); + if (status == -EAGAIN) + continue; + if (status < 0) + goto out_error; + nfs4_state_end_reclaim_reboot(clp); + continue; + } + + /* Detect expired delegations... */ + if (test_and_clear_bit(NFS4CLNT_DELEGATION_EXPIRED, &clp->cl_state)) { + section = "detect expired delegations"; + status = nfs4_begin_drain_session(clp); + if (status < 0) + goto out_error; + nfs_reap_expired_delegations(clp); + continue; + } + + /* Now recover expired state... */ + if (test_bit(NFS4CLNT_RECLAIM_NOGRACE, &clp->cl_state)) { + section = "reclaim nograce"; + status = nfs4_do_reclaim(clp, + clp->cl_mvops->nograce_recovery_ops); + if (status == -EAGAIN) + continue; + if (status < 0) + goto out_error; + clear_bit(NFS4CLNT_RECLAIM_NOGRACE, &clp->cl_state); + } + + memalloc_nofs_restore(memflags); + nfs4_end_drain_session(clp); + nfs4_clear_state_manager_bit(clp); + + if (test_bit(NFS4CLNT_RUN_MANAGER, &clp->cl_state) && + !test_and_set_bit(NFS4CLNT_MANAGER_RUNNING, + &clp->cl_state)) { + memflags = memalloc_nofs_save(); + continue; + } + + if (!test_and_set_bit(NFS4CLNT_RECALL_RUNNING, &clp->cl_state)) { + if (test_and_clear_bit(NFS4CLNT_DELEGRETURN, &clp->cl_state)) { + nfs_client_return_marked_delegations(clp); + set_bit(NFS4CLNT_RUN_MANAGER, &clp->cl_state); + } + nfs4_layoutreturn_any_run(clp); + clear_bit(NFS4CLNT_RECALL_RUNNING, &clp->cl_state); + } + + return; + + } while (refcount_read(&clp->cl_count) > 1 && !signalled()); + goto out_drain; + +out_error: + if (strlen(section)) + section_sep = ": "; + trace_nfs4_state_mgr_failed(clp, section, status); + pr_warn_ratelimited("NFS: state manager%s%s failed on NFSv4 server %s" + " with error %d\n", section_sep, section, + clp->cl_hostname, -status); + switch (status) { + case -ENETDOWN: + case -ENETUNREACH: + nfs_mark_client_ready(clp, -EIO); + break; + case -EINVAL: + nfs_mark_client_ready(clp, status); + break; + default: + ssleep(1); + break; + } +out_drain: + memalloc_nofs_restore(memflags); + nfs4_end_drain_session(clp); + nfs4_clear_state_manager_bit(clp); +} + +static int nfs4_run_state_manager(void *ptr) +{ + struct nfs_client *clp = ptr; + struct rpc_clnt *cl = clp->cl_rpcclient; + + while (cl != cl->cl_parent) + cl = cl->cl_parent; + + allow_signal(SIGKILL); +again: + nfs4_state_manager(clp); + + if (test_bit(NFS4CLNT_MANAGER_AVAILABLE, &clp->cl_state) && + !test_bit(NFS4CLNT_MANAGER_RUNNING, &clp->cl_state)) { + wait_var_event_interruptible(&clp->cl_state, + test_bit(NFS4CLNT_RUN_MANAGER, + &clp->cl_state)); + if (!atomic_read(&cl->cl_swapper)) + clear_bit(NFS4CLNT_MANAGER_AVAILABLE, &clp->cl_state); + if (refcount_read(&clp->cl_count) > 1 && !signalled() && + !test_and_set_bit(NFS4CLNT_MANAGER_RUNNING, &clp->cl_state)) + goto again; + /* Either no longer a swapper, or were signalled */ + clear_bit(NFS4CLNT_MANAGER_AVAILABLE, &clp->cl_state); + } + + if (refcount_read(&clp->cl_count) > 1 && !signalled() && + test_bit(NFS4CLNT_RUN_MANAGER, &clp->cl_state) && + !test_and_set_bit(NFS4CLNT_MANAGER_RUNNING, &clp->cl_state)) + goto again; + + nfs_put_client(clp); + module_put_and_kthread_exit(0); + return 0; +} diff --git a/fs/nfs/nfs4super.c b/fs/nfs/nfs4super.c new file mode 100644 index 000000000..9c5895b55 --- /dev/null +++ b/fs/nfs/nfs4super.c @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2012 Bryan Schumaker + */ +#include +#include +#include +#include +#include +#include +#include "delegation.h" +#include "internal.h" +#include "nfs4_fs.h" +#include "nfs4idmap.h" +#include "dns_resolve.h" +#include "pnfs.h" +#include "nfs.h" + +#define NFSDBG_FACILITY NFSDBG_VFS + +static int nfs4_write_inode(struct inode *inode, struct writeback_control *wbc); +static void nfs4_evict_inode(struct inode *inode); + +static const struct super_operations nfs4_sops = { + .alloc_inode = nfs_alloc_inode, + .free_inode = nfs_free_inode, + .write_inode = nfs4_write_inode, + .drop_inode = nfs_drop_inode, + .statfs = nfs_statfs, + .evict_inode = nfs4_evict_inode, + .umount_begin = nfs_umount_begin, + .show_options = nfs_show_options, + .show_devname = nfs_show_devname, + .show_path = nfs_show_path, + .show_stats = nfs_show_stats, +}; + +struct nfs_subversion nfs_v4 = { + .owner = THIS_MODULE, + .nfs_fs = &nfs4_fs_type, + .rpc_vers = &nfs_version4, + .rpc_ops = &nfs_v4_clientops, + .sops = &nfs4_sops, + .xattr = nfs4_xattr_handlers, +}; + +static int nfs4_write_inode(struct inode *inode, struct writeback_control *wbc) +{ + int ret = nfs_write_inode(inode, wbc); + + if (ret == 0) + ret = pnfs_layoutcommit_inode(inode, + wbc->sync_mode == WB_SYNC_ALL); + return ret; +} + +/* + * Clean out any remaining NFSv4 state that might be left over due + * to open() calls that passed nfs_atomic_lookup, but failed to call + * nfs_open(). + */ +static void nfs4_evict_inode(struct inode *inode) +{ + truncate_inode_pages_final(&inode->i_data); + clear_inode(inode); + /* If we are holding a delegation, return and free it */ + nfs_inode_evict_delegation(inode); + /* Note that above delegreturn would trigger pnfs return-on-close */ + pnfs_return_layout(inode); + pnfs_destroy_layout_final(NFS_I(inode)); + /* First call standard NFS clear_inode() code */ + nfs_clear_inode(inode); + nfs4_xattr_cache_zap(inode); +} + +struct nfs_referral_count { + struct list_head list; + const struct task_struct *task; + unsigned int referral_count; +}; + +static LIST_HEAD(nfs_referral_count_list); +static DEFINE_SPINLOCK(nfs_referral_count_list_lock); + +static struct nfs_referral_count *nfs_find_referral_count(void) +{ + struct nfs_referral_count *p; + + list_for_each_entry(p, &nfs_referral_count_list, list) { + if (p->task == current) + return p; + } + return NULL; +} + +#define NFS_MAX_NESTED_REFERRALS 2 + +static int nfs_referral_loop_protect(void) +{ + struct nfs_referral_count *p, *new; + int ret = -ENOMEM; + + new = kmalloc_obj(*new); + if (!new) + goto out; + new->task = current; + new->referral_count = 1; + + ret = 0; + spin_lock(&nfs_referral_count_list_lock); + p = nfs_find_referral_count(); + if (p != NULL) { + if (p->referral_count >= NFS_MAX_NESTED_REFERRALS) + ret = -ELOOP; + else + p->referral_count++; + } else { + list_add(&new->list, &nfs_referral_count_list); + new = NULL; + } + spin_unlock(&nfs_referral_count_list_lock); + kfree(new); +out: + return ret; +} + +static void nfs_referral_loop_unprotect(void) +{ + struct nfs_referral_count *p; + + spin_lock(&nfs_referral_count_list_lock); + p = nfs_find_referral_count(); + p->referral_count--; + if (p->referral_count == 0) + list_del(&p->list); + else + p = NULL; + spin_unlock(&nfs_referral_count_list_lock); + kfree(p); +} + +static int do_nfs4_mount(struct nfs_server *server, + struct fs_context *fc, + const char *hostname, + const char *export_path) +{ + struct nfs_fs_context *root_ctx; + struct nfs_fs_context *ctx; + struct fs_context *root_fc; + struct vfsmount *root_mnt; + struct dentry *dentry; + char *source; + int ret; + + if (IS_ERR(server)) + return PTR_ERR(server); + + root_fc = vfs_dup_fs_context(fc); + if (IS_ERR(root_fc)) { + nfs_free_server(server); + return PTR_ERR(root_fc); + } + kfree(root_fc->source); + root_fc->source = NULL; + + ctx = nfs_fc2context(fc); + root_ctx = nfs_fc2context(root_fc); + root_ctx->internal = true; + root_ctx->server = server; + + if (ctx->fscache_uniq) { + ret = vfs_parse_fs_string(root_fc, "fsc", ctx->fscache_uniq); + if (ret < 0) { + put_fs_context(root_fc); + return ret; + } + } + /* We leave export_path unset as it's not used to find the root. */ + + /* Does hostname needs to be enclosed in brackets? */ + if (strchr(hostname, ':')) + source = kasprintf(GFP_KERNEL, "[%s]:/", hostname); + else + source = kasprintf(GFP_KERNEL, "%s:/", hostname); + + if (!source) { + put_fs_context(root_fc); + return -ENOMEM; + } + ret = vfs_parse_fs_string(root_fc, "source", source); + kfree(source); + if (ret < 0) { + put_fs_context(root_fc); + return ret; + } + root_mnt = fc_mount(root_fc); + put_fs_context(root_fc); + + if (IS_ERR(root_mnt)) + return PTR_ERR(root_mnt); + + ret = nfs_referral_loop_protect(); + if (ret) { + mntput(root_mnt); + return ret; + } + + dentry = mount_subtree(root_mnt, export_path); + nfs_referral_loop_unprotect(); + + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + + fc->root = dentry; + return 0; +} + +int nfs4_try_get_tree(struct fs_context *fc) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + int err; + + dfprintk(MOUNT, "--> nfs4_try_get_tree()\n"); + + /* We create a mount for the server's root, walk to the requested + * location and then create another mount for that. + */ + err= do_nfs4_mount(nfs4_create_server(fc), + fc, ctx->nfs_server.hostname, + ctx->nfs_server.export_path); + if (err) { + nfs_ferrorf(fc, MOUNT, "NFS4: Couldn't follow remote path"); + dfprintk(MOUNT, "<-- nfs4_try_get_tree() = %d [error]\n", err); + } else { + dfprintk(MOUNT, "<-- nfs4_try_get_tree() = 0\n"); + } + return err; +} + +/* + * Create an NFS4 server record on referral traversal + */ +int nfs4_get_referral_tree(struct fs_context *fc) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + int err; + + dprintk("--> nfs4_referral_mount()\n"); + + /* create a new volume representation */ + err = do_nfs4_mount(nfs4_create_referral_server(fc), + fc, ctx->nfs_server.hostname, + ctx->nfs_server.export_path); + if (err) { + nfs_ferrorf(fc, MOUNT, "NFS4: Couldn't follow remote path"); + dfprintk(MOUNT, "<-- nfs4_get_referral_tree() = %d [error]\n", err); + } else { + dfprintk(MOUNT, "<-- nfs4_get_referral_tree() = 0\n"); + } + return err; +} + +static int __init init_nfs_v4(void) +{ + int err; + + err = nfs_dns_resolver_init(); + if (err) + goto out; + + err = nfs_idmap_init(); + if (err) + goto out1; + +#ifdef CONFIG_NFS_V4_2 + err = nfs4_xattr_cache_init(); + if (err) + goto out2; +#endif + + err = nfs4_register_sysctl(); + if (err) + goto out2; + +#ifdef CONFIG_NFS_V4_2 + nfs42_ssc_register_ops(); +#endif + register_nfs_version(&nfs_v4); + return 0; +out2: + nfs_idmap_quit(); +out1: + nfs_dns_resolver_destroy(); +out: + return err; +} + +static void __exit exit_nfs_v4(void) +{ + /* Not called in the _init(), conditionally loaded */ + nfs4_pnfs_v3_ds_connect_unload(); + + unregister_nfs_version(&nfs_v4); +#ifdef CONFIG_NFS_V4_2 + nfs4_xattr_cache_exit(); + nfs42_ssc_unregister_ops(); +#endif + nfs4_unregister_sysctl(); + nfs_idmap_quit(); + nfs_dns_resolver_destroy(); +} + +MODULE_DESCRIPTION("NFSv4 client support"); +MODULE_LICENSE("GPL"); + +module_init(init_nfs_v4); +module_exit(exit_nfs_v4); diff --git a/fs/nfs/nfs4sysctl.c b/fs/nfs/nfs4sysctl.c new file mode 100644 index 000000000..d1a92d8f8 --- /dev/null +++ b/fs/nfs/nfs4sysctl.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/nfs/nfs4sysctl.c + * + * Sysctl interface to NFS v4 parameters + * + * Copyright (c) 2006 Trond Myklebust + */ +#include +#include + +#include "nfs4_fs.h" +#include "nfs4idmap.h" +#include "callback.h" + +static const int nfs_set_port_min; +static const int nfs_set_port_max = 65535; +static struct ctl_table_header *nfs4_callback_sysctl_table; + +static const struct ctl_table nfs4_cb_sysctls[] = { + { + .procname = "nfs_callback_tcpport", + .data = &nfs_callback_set_tcpport, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = (int *)&nfs_set_port_min, + .extra2 = (int *)&nfs_set_port_max, + }, + { + .procname = "idmap_cache_timeout", + .data = &nfs_idmap_cache_timeout, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +}; + +int nfs4_register_sysctl(void) +{ + nfs4_callback_sysctl_table = register_sysctl("fs/nfs", + nfs4_cb_sysctls); + if (nfs4_callback_sysctl_table == NULL) + return -ENOMEM; + return 0; +} + +void nfs4_unregister_sysctl(void) +{ + unregister_sysctl_table(nfs4_callback_sysctl_table); + nfs4_callback_sysctl_table = NULL; +} diff --git a/fs/nfs/nfs4trace.c b/fs/nfs/nfs4trace.c new file mode 100644 index 000000000..3fdc013f5 --- /dev/null +++ b/fs/nfs/nfs4trace.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2013 Trond Myklebust + */ +#include +#include +#include +#include "nfs4_fs.h" +#include "internal.h" +#include "nfs4session.h" +#include "callback.h" +#include "pnfs.h" + +#define CREATE_TRACE_POINTS +#include "nfs4trace.h" + +EXPORT_TRACEPOINT_SYMBOL_GPL(nfs4_pnfs_read); +EXPORT_TRACEPOINT_SYMBOL_GPL(nfs4_pnfs_write); +EXPORT_TRACEPOINT_SYMBOL_GPL(nfs4_pnfs_commit_ds); + +EXPORT_TRACEPOINT_SYMBOL_GPL(pnfs_mds_fallback_pg_init_read); +EXPORT_TRACEPOINT_SYMBOL_GPL(pnfs_mds_fallback_pg_init_write); +EXPORT_TRACEPOINT_SYMBOL_GPL(pnfs_mds_fallback_pg_get_mirror_count); +EXPORT_TRACEPOINT_SYMBOL_GPL(pnfs_mds_fallback_read_done); +EXPORT_TRACEPOINT_SYMBOL_GPL(pnfs_mds_fallback_write_done); +EXPORT_TRACEPOINT_SYMBOL_GPL(pnfs_mds_fallback_read_pagelist); +EXPORT_TRACEPOINT_SYMBOL_GPL(pnfs_mds_fallback_write_pagelist); +EXPORT_TRACEPOINT_SYMBOL_GPL(pnfs_ds_connect); + +EXPORT_TRACEPOINT_SYMBOL_GPL(ff_layout_read_error); +EXPORT_TRACEPOINT_SYMBOL_GPL(ff_layout_write_error); +EXPORT_TRACEPOINT_SYMBOL_GPL(ff_layout_commit_error); + +EXPORT_TRACEPOINT_SYMBOL_GPL(bl_ext_tree_prepare_commit); +EXPORT_TRACEPOINT_SYMBOL_GPL(bl_pr_key_reg); +EXPORT_TRACEPOINT_SYMBOL_GPL(bl_pr_key_reg_err); +EXPORT_TRACEPOINT_SYMBOL_GPL(bl_pr_key_unreg); +EXPORT_TRACEPOINT_SYMBOL_GPL(bl_pr_key_unreg_err); + +EXPORT_TRACEPOINT_SYMBOL_GPL(fl_getdevinfo); diff --git a/fs/nfs/nfs4trace.h b/fs/nfs/nfs4trace.h new file mode 100644 index 000000000..1ed677810 --- /dev/null +++ b/fs/nfs/nfs4trace.h @@ -0,0 +1,2859 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2013 Trond Myklebust + */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM nfs4 + +#if !defined(_TRACE_NFS4_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_NFS4_H + +#include +#include + +#include +#include + +#include "delegation.h" + +#define show_nfs_fattr_flags(valid) \ + __print_flags((unsigned long)valid, "|", \ + { NFS_ATTR_FATTR_TYPE, "TYPE" }, \ + { NFS_ATTR_FATTR_MODE, "MODE" }, \ + { NFS_ATTR_FATTR_NLINK, "NLINK" }, \ + { NFS_ATTR_FATTR_OWNER, "OWNER" }, \ + { NFS_ATTR_FATTR_GROUP, "GROUP" }, \ + { NFS_ATTR_FATTR_RDEV, "RDEV" }, \ + { NFS_ATTR_FATTR_SIZE, "SIZE" }, \ + { NFS_ATTR_FATTR_FSID, "FSID" }, \ + { NFS_ATTR_FATTR_FILEID, "FILEID" }, \ + { NFS_ATTR_FATTR_ATIME, "ATIME" }, \ + { NFS_ATTR_FATTR_MTIME, "MTIME" }, \ + { NFS_ATTR_FATTR_CTIME, "CTIME" }, \ + { NFS_ATTR_FATTR_CHANGE, "CHANGE" }, \ + { NFS_ATTR_FATTR_OWNER_NAME, "OWNER_NAME" }, \ + { NFS_ATTR_FATTR_GROUP_NAME, "GROUP_NAME" }, \ + { NFS_ATTR_FATTR_BTIME, "BTIME" }) + +DECLARE_EVENT_CLASS(nfs4_clientid_event, + TP_PROTO( + const struct nfs_client *clp, + int error + ), + + TP_ARGS(clp, error), + + TP_STRUCT__entry( + __string(dstaddr, clp->cl_hostname) + __field(unsigned long, error) + ), + + TP_fast_assign( + __entry->error = error < 0 ? -error : 0; + __assign_str(dstaddr); + ), + + TP_printk( + "error=%ld (%s) dstaddr=%s", + -__entry->error, + show_nfs4_status(__entry->error), + __get_str(dstaddr) + ) +); +#define DEFINE_NFS4_CLIENTID_EVENT(name) \ + DEFINE_EVENT(nfs4_clientid_event, name, \ + TP_PROTO( \ + const struct nfs_client *clp, \ + int error \ + ), \ + TP_ARGS(clp, error)) +DEFINE_NFS4_CLIENTID_EVENT(nfs4_setclientid); +DEFINE_NFS4_CLIENTID_EVENT(nfs4_setclientid_confirm); +DEFINE_NFS4_CLIENTID_EVENT(nfs4_renew); +DEFINE_NFS4_CLIENTID_EVENT(nfs4_renew_async); +DEFINE_NFS4_CLIENTID_EVENT(nfs4_exchange_id); +DEFINE_NFS4_CLIENTID_EVENT(nfs4_create_session); +DEFINE_NFS4_CLIENTID_EVENT(nfs4_destroy_session); +DEFINE_NFS4_CLIENTID_EVENT(nfs4_destroy_clientid); +DEFINE_NFS4_CLIENTID_EVENT(nfs4_bind_conn_to_session); +DEFINE_NFS4_CLIENTID_EVENT(nfs4_sequence); +DEFINE_NFS4_CLIENTID_EVENT(nfs4_reclaim_complete); + +TRACE_EVENT(nfs4_trunked_exchange_id, + TP_PROTO( + const struct nfs_client *clp, + const char *addr, + int error + ), + + TP_ARGS(clp, addr, error), + + TP_STRUCT__entry( + __string(main_addr, clp->cl_hostname) + __string(trunk_addr, addr) + __field(unsigned long, error) + ), + + TP_fast_assign( + __entry->error = error < 0 ? -error : 0; + __assign_str(main_addr); + __assign_str(trunk_addr); + ), + + TP_printk( + "error=%ld (%s) main_addr=%s trunk_addr=%s", + -__entry->error, + show_nfs4_status(__entry->error), + __get_str(main_addr), + __get_str(trunk_addr) + ) +); + +TRACE_EVENT(nfs4_sequence_done, + TP_PROTO( + const struct nfs4_session *session, + const struct nfs4_sequence_res *res + ), + TP_ARGS(session, res), + + TP_STRUCT__entry( + __field(unsigned int, session) + __field(unsigned int, slot_nr) + __field(unsigned int, seq_nr) + __field(unsigned int, highest_slotid) + __field(unsigned int, target_highest_slotid) + __field(unsigned long, status_flags) + __field(unsigned long, error) + ), + + TP_fast_assign( + const struct nfs4_slot *sr_slot = res->sr_slot; + __entry->session = nfs_session_id_hash(&session->sess_id); + __entry->slot_nr = sr_slot->slot_nr; + __entry->seq_nr = sr_slot->seq_nr; + __entry->highest_slotid = res->sr_highest_slotid; + __entry->target_highest_slotid = + res->sr_target_highest_slotid; + __entry->status_flags = res->sr_status_flags; + __entry->error = res->sr_status < 0 ? + -res->sr_status : 0; + ), + TP_printk( + "error=%ld (%s) session=0x%08x slot_nr=%u seq_nr=%u " + "highest_slotid=%u target_highest_slotid=%u " + "status_flags=0x%lx (%s)", + -__entry->error, + show_nfs4_status(__entry->error), + __entry->session, + __entry->slot_nr, + __entry->seq_nr, + __entry->highest_slotid, + __entry->target_highest_slotid, + __entry->status_flags, + show_nfs4_seq4_status(__entry->status_flags) + ) +); + +struct cb_sequenceargs; +struct cb_sequenceres; + +TRACE_EVENT(nfs4_cb_sequence, + TP_PROTO( + const struct cb_sequenceargs *args, + const struct cb_sequenceres *res, + __be32 status + ), + TP_ARGS(args, res, status), + + TP_STRUCT__entry( + __field(unsigned int, session) + __field(unsigned int, slot_nr) + __field(unsigned int, seq_nr) + __field(unsigned int, highest_slotid) + __field(unsigned int, cachethis) + __field(unsigned long, error) + ), + + TP_fast_assign( + __entry->session = nfs_session_id_hash(&args->csa_sessionid); + __entry->slot_nr = args->csa_slotid; + __entry->seq_nr = args->csa_sequenceid; + __entry->highest_slotid = args->csa_highestslotid; + __entry->cachethis = args->csa_cachethis; + __entry->error = be32_to_cpu(status); + ), + + TP_printk( + "error=%ld (%s) session=0x%08x slot_nr=%u seq_nr=%u " + "highest_slotid=%u", + -__entry->error, + show_nfs4_status(__entry->error), + __entry->session, + __entry->slot_nr, + __entry->seq_nr, + __entry->highest_slotid + ) +); + +TRACE_EVENT(nfs4_cb_seqid_err, + TP_PROTO( + const struct cb_sequenceargs *args, + __be32 status + ), + TP_ARGS(args, status), + + TP_STRUCT__entry( + __field(unsigned int, session) + __field(unsigned int, slot_nr) + __field(unsigned int, seq_nr) + __field(unsigned int, highest_slotid) + __field(unsigned int, cachethis) + __field(unsigned long, error) + ), + + TP_fast_assign( + __entry->session = nfs_session_id_hash(&args->csa_sessionid); + __entry->slot_nr = args->csa_slotid; + __entry->seq_nr = args->csa_sequenceid; + __entry->highest_slotid = args->csa_highestslotid; + __entry->cachethis = args->csa_cachethis; + __entry->error = be32_to_cpu(status); + ), + + TP_printk( + "error=%ld (%s) session=0x%08x slot_nr=%u seq_nr=%u " + "highest_slotid=%u", + -__entry->error, + show_nfs4_status(__entry->error), + __entry->session, + __entry->slot_nr, + __entry->seq_nr, + __entry->highest_slotid + ) +); + +TRACE_EVENT(nfs4_cb_offload, + TP_PROTO( + const struct nfs_fh *cb_fh, + const nfs4_stateid *cb_stateid, + uint64_t cb_count, + int cb_error, + int cb_how_stable + ), + + TP_ARGS(cb_fh, cb_stateid, cb_count, cb_error, + cb_how_stable), + + TP_STRUCT__entry( + __field(unsigned long, error) + __field(u32, fhandle) + __field(loff_t, cb_count) + __field(int, cb_how) + __field(int, cb_stateid_seq) + __field(u32, cb_stateid_hash) + ), + + TP_fast_assign( + __entry->error = cb_error < 0 ? -cb_error : 0; + __entry->fhandle = nfs_fhandle_hash(cb_fh); + __entry->cb_stateid_seq = + be32_to_cpu(cb_stateid->seqid); + __entry->cb_stateid_hash = + nfs_stateid_hash(cb_stateid); + __entry->cb_count = cb_count; + __entry->cb_how = cb_how_stable; + ), + + TP_printk( + "error=%ld (%s) fhandle=0x%08x cb_stateid=%d:0x%08x " + "cb_count=%llu cb_how=%s", + -__entry->error, + show_nfs4_status(__entry->error), + __entry->fhandle, + __entry->cb_stateid_seq, __entry->cb_stateid_hash, + __entry->cb_count, + show_nfs_stable_how(__entry->cb_how) + ) +); + +TRACE_EVENT(pnfs_ds_connect, + TP_PROTO( + char *ds_remotestr, + int status + ), + + TP_ARGS(ds_remotestr, status), + + TP_STRUCT__entry( + __string(ds_ips, ds_remotestr) + __field(int, status) + ), + + TP_fast_assign( + __assign_str(ds_ips); + __entry->status = status; + ), + + TP_printk( + "ds_ips=%s, status=%d", + __get_str(ds_ips), + __entry->status + ) +); + +TRACE_EVENT(nfs4_setup_sequence, + TP_PROTO( + const struct nfs4_session *session, + const struct nfs4_sequence_args *args + ), + TP_ARGS(session, args), + + TP_STRUCT__entry( + __field(unsigned int, session) + __field(unsigned int, slot_nr) + __field(unsigned int, seq_nr) + __field(unsigned int, highest_used_slotid) + ), + + TP_fast_assign( + const struct nfs4_slot *sa_slot = args->sa_slot; + __entry->session = session ? nfs_session_id_hash(&session->sess_id) : 0; + __entry->slot_nr = sa_slot->slot_nr; + __entry->seq_nr = sa_slot->seq_nr; + __entry->highest_used_slotid = + sa_slot->table->highest_used_slotid; + ), + TP_printk( + "session=0x%08x slot_nr=%u seq_nr=%u " + "highest_used_slotid=%u", + __entry->session, + __entry->slot_nr, + __entry->seq_nr, + __entry->highest_used_slotid + ) +); + +TRACE_DEFINE_ENUM(NFS4CLNT_MANAGER_RUNNING); +TRACE_DEFINE_ENUM(NFS4CLNT_CHECK_LEASE); +TRACE_DEFINE_ENUM(NFS4CLNT_LEASE_EXPIRED); +TRACE_DEFINE_ENUM(NFS4CLNT_RECLAIM_REBOOT); +TRACE_DEFINE_ENUM(NFS4CLNT_RECLAIM_NOGRACE); +TRACE_DEFINE_ENUM(NFS4CLNT_DELEGRETURN); +TRACE_DEFINE_ENUM(NFS4CLNT_SESSION_RESET); +TRACE_DEFINE_ENUM(NFS4CLNT_LEASE_CONFIRM); +TRACE_DEFINE_ENUM(NFS4CLNT_SERVER_SCOPE_MISMATCH); +TRACE_DEFINE_ENUM(NFS4CLNT_PURGE_STATE); +TRACE_DEFINE_ENUM(NFS4CLNT_BIND_CONN_TO_SESSION); +TRACE_DEFINE_ENUM(NFS4CLNT_MOVED); +TRACE_DEFINE_ENUM(NFS4CLNT_LEASE_MOVED); +TRACE_DEFINE_ENUM(NFS4CLNT_DELEGATION_EXPIRED); +TRACE_DEFINE_ENUM(NFS4CLNT_RUN_MANAGER); +TRACE_DEFINE_ENUM(NFS4CLNT_MANAGER_AVAILABLE); +TRACE_DEFINE_ENUM(NFS4CLNT_RECALL_RUNNING); +TRACE_DEFINE_ENUM(NFS4CLNT_RECALL_ANY_LAYOUT_READ); +TRACE_DEFINE_ENUM(NFS4CLNT_RECALL_ANY_LAYOUT_RW); +TRACE_DEFINE_ENUM(NFS4CLNT_DELEGRETURN_DELAYED); + +#define show_nfs4_clp_state(state) \ + __print_flags(state, "|", \ + { BIT(NFS4CLNT_MANAGER_RUNNING), "MANAGER_RUNNING" }, \ + { BIT(NFS4CLNT_CHECK_LEASE), "CHECK_LEASE" }, \ + { BIT(NFS4CLNT_LEASE_EXPIRED), "LEASE_EXPIRED" }, \ + { BIT(NFS4CLNT_RECLAIM_REBOOT), "RECLAIM_REBOOT" }, \ + { BIT(NFS4CLNT_RECLAIM_NOGRACE), "RECLAIM_NOGRACE" }, \ + { BIT(NFS4CLNT_DELEGRETURN), "DELEGRETURN" }, \ + { BIT(NFS4CLNT_SESSION_RESET), "SESSION_RESET" }, \ + { BIT(NFS4CLNT_LEASE_CONFIRM), "LEASE_CONFIRM" }, \ + { BIT(NFS4CLNT_SERVER_SCOPE_MISMATCH), "SERVER_SCOPE_MISMATCH" }, \ + { BIT(NFS4CLNT_PURGE_STATE), "PURGE_STATE" }, \ + { BIT(NFS4CLNT_BIND_CONN_TO_SESSION), "BIND_CONN_TO_SESSION" }, \ + { BIT(NFS4CLNT_MOVED), "MOVED" }, \ + { BIT(NFS4CLNT_LEASE_MOVED), "LEASE_MOVED" }, \ + { BIT(NFS4CLNT_DELEGATION_EXPIRED), "DELEGATION_EXPIRED" }, \ + { BIT(NFS4CLNT_RUN_MANAGER), "RUN_MANAGER" }, \ + { BIT(NFS4CLNT_MANAGER_AVAILABLE), "MANAGER_AVAILABLE" }, \ + { BIT(NFS4CLNT_RECALL_RUNNING), "RECALL_RUNNING" }, \ + { BIT(NFS4CLNT_RECALL_ANY_LAYOUT_READ), "RECALL_ANY_LAYOUT_READ" }, \ + { BIT(NFS4CLNT_RECALL_ANY_LAYOUT_RW), "RECALL_ANY_LAYOUT_RW" }, \ + { BIT(NFS4CLNT_DELEGRETURN_DELAYED), "DELERETURN_DELAYED" }) + +TRACE_EVENT(nfs4_state_mgr, + TP_PROTO( + const struct nfs_client *clp + ), + + TP_ARGS(clp), + + TP_STRUCT__entry( + __field(unsigned long, state) + __string(hostname, clp->cl_hostname) + ), + + TP_fast_assign( + __entry->state = clp->cl_state; + __assign_str(hostname); + ), + + TP_printk( + "hostname=%s clp state=%s", __get_str(hostname), + show_nfs4_clp_state(__entry->state) + ) +) + +TRACE_EVENT(nfs4_state_mgr_failed, + TP_PROTO( + const struct nfs_client *clp, + const char *section, + int status + ), + + TP_ARGS(clp, section, status), + + TP_STRUCT__entry( + __field(unsigned long, error) + __field(unsigned long, state) + __string(hostname, clp->cl_hostname) + __string(section, section) + ), + + TP_fast_assign( + __entry->error = status < 0 ? -status : 0; + __entry->state = clp->cl_state; + __assign_str(hostname); + __assign_str(section); + ), + + TP_printk( + "hostname=%s clp state=%s error=%ld (%s) section=%s", + __get_str(hostname), + show_nfs4_clp_state(__entry->state), -__entry->error, + show_nfs4_status(__entry->error), __get_str(section) + + ) +) + +TRACE_EVENT(nfs4_xdr_bad_operation, + TP_PROTO( + const struct xdr_stream *xdr, + u32 op, + u32 expected + ), + + TP_ARGS(xdr, op, expected), + + TP_STRUCT__entry( + __field(unsigned int, task_id) + __field(unsigned int, client_id) + __field(u32, xid) + __field(u32, op) + __field(u32, expected) + ), + + TP_fast_assign( + const struct rpc_rqst *rqstp = xdr->rqst; + const struct rpc_task *task = rqstp->rq_task; + + __entry->task_id = task->tk_pid; + __entry->client_id = task->tk_client->cl_clid; + __entry->xid = be32_to_cpu(rqstp->rq_xid); + __entry->op = op; + __entry->expected = expected; + ), + + TP_printk(SUNRPC_TRACE_TASK_SPECIFIER + " xid=0x%08x operation=%u, expected=%u", + __entry->task_id, __entry->client_id, __entry->xid, + __entry->op, __entry->expected + ) +); + +DECLARE_EVENT_CLASS(nfs4_xdr_event, + TP_PROTO( + const struct xdr_stream *xdr, + u32 op, + u32 error + ), + + TP_ARGS(xdr, op, error), + + TP_STRUCT__entry( + __field(unsigned int, task_id) + __field(unsigned int, client_id) + __field(u32, xid) + __field(u32, op) + __field(unsigned long, error) + ), + + TP_fast_assign( + const struct rpc_rqst *rqstp = xdr->rqst; + const struct rpc_task *task = rqstp->rq_task; + + __entry->task_id = task->tk_pid; + __entry->client_id = task->tk_client->cl_clid; + __entry->xid = be32_to_cpu(rqstp->rq_xid); + __entry->op = op; + __entry->error = error; + ), + + TP_printk(SUNRPC_TRACE_TASK_SPECIFIER + " xid=0x%08x error=%ld (%s) operation=%u", + __entry->task_id, __entry->client_id, __entry->xid, + -__entry->error, show_nfs4_status(__entry->error), + __entry->op + ) +); +#define DEFINE_NFS4_XDR_EVENT(name) \ + DEFINE_EVENT(nfs4_xdr_event, name, \ + TP_PROTO( \ + const struct xdr_stream *xdr, \ + u32 op, \ + u32 error \ + ), \ + TP_ARGS(xdr, op, error)) +DEFINE_NFS4_XDR_EVENT(nfs4_xdr_status); +DEFINE_NFS4_XDR_EVENT(nfs4_xdr_bad_filehandle); + +DECLARE_EVENT_CLASS(nfs4_cb_error_class, + TP_PROTO( + __be32 xid, + u32 cb_ident + ), + + TP_ARGS(xid, cb_ident), + + TP_STRUCT__entry( + __field(u32, xid) + __field(u32, cbident) + ), + + TP_fast_assign( + __entry->xid = be32_to_cpu(xid); + __entry->cbident = cb_ident; + ), + + TP_printk( + "xid=0x%08x cb_ident=0x%08x", + __entry->xid, __entry->cbident + ) +); + +#define DEFINE_CB_ERROR_EVENT(name) \ + DEFINE_EVENT(nfs4_cb_error_class, nfs_cb_##name, \ + TP_PROTO( \ + __be32 xid, \ + u32 cb_ident \ + ), \ + TP_ARGS(xid, cb_ident)) + +DEFINE_CB_ERROR_EVENT(no_clp); +DEFINE_CB_ERROR_EVENT(badprinc); + +DECLARE_EVENT_CLASS(nfs4_open_event, + TP_PROTO( + const struct nfs_open_context *ctx, + int flags, + int error + ), + + TP_ARGS(ctx, flags, error), + + TP_STRUCT__entry( + __field(unsigned long, error) + __field(unsigned long, flags) + __field(unsigned long, fmode) + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(u64, dir) + __string(name, ctx->dentry->d_name.name) + __field(int, stateid_seq) + __field(u32, stateid_hash) + __field(int, openstateid_seq) + __field(u32, openstateid_hash) + ), + + TP_fast_assign( + const struct nfs4_state *state = ctx->state; + const struct inode *inode = NULL; + + __entry->error = -error; + __entry->flags = flags; + __entry->fmode = (__force unsigned long)ctx->mode; + __entry->dev = ctx->dentry->d_sb->s_dev; + if (!IS_ERR_OR_NULL(state)) { + inode = state->inode; + __entry->stateid_seq = + be32_to_cpu(state->stateid.seqid); + __entry->stateid_hash = + nfs_stateid_hash(&state->stateid); + __entry->openstateid_seq = + be32_to_cpu(state->open_stateid.seqid); + __entry->openstateid_hash = + nfs_stateid_hash(&state->open_stateid); + } else { + __entry->stateid_seq = 0; + __entry->stateid_hash = 0; + __entry->openstateid_seq = 0; + __entry->openstateid_hash = 0; + } + if (inode != NULL) { + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); + } else { + __entry->fileid = 0; + __entry->fhandle = 0; + } + __entry->dir = d_inode(ctx->dentry->d_parent)->i_ino; + __assign_str(name); + ), + + TP_printk( + "error=%ld (%s) flags=%lu (%s) fmode=%s " + "fileid=%02x:%02x:%llu fhandle=0x%08x " + "name=%02x:%02x:%llu/%s stateid=%d:0x%08x " + "openstateid=%d:0x%08x", + -__entry->error, + show_nfs4_status(__entry->error), + __entry->flags, + show_fs_fcntl_open_flags(__entry->flags), + show_fs_fmode_flags(__entry->fmode), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->dir, + __get_str(name), + __entry->stateid_seq, __entry->stateid_hash, + __entry->openstateid_seq, __entry->openstateid_hash + ) +); + +#define DEFINE_NFS4_OPEN_EVENT(name) \ + DEFINE_EVENT(nfs4_open_event, name, \ + TP_PROTO( \ + const struct nfs_open_context *ctx, \ + int flags, \ + int error \ + ), \ + TP_ARGS(ctx, flags, error)) +DEFINE_NFS4_OPEN_EVENT(nfs4_open_reclaim); +DEFINE_NFS4_OPEN_EVENT(nfs4_open_expired); +DEFINE_NFS4_OPEN_EVENT(nfs4_open_file); + +TRACE_EVENT(nfs4_cached_open, + TP_PROTO( + const struct nfs4_state *state + ), + TP_ARGS(state), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(unsigned int, fmode) + __field(int, stateid_seq) + __field(u32, stateid_hash) + ), + + TP_fast_assign( + const struct inode *inode = state->inode; + + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); + __entry->fmode = (__force unsigned int)state->state; + __entry->stateid_seq = + be32_to_cpu(state->stateid.seqid); + __entry->stateid_hash = + nfs_stateid_hash(&state->stateid); + ), + + TP_printk( + "fmode=%s fileid=%02x:%02x:%llu " + "fhandle=0x%08x stateid=%d:0x%08x", + __entry->fmode ? show_fs_fmode_flags(__entry->fmode) : + "closed", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + __entry->stateid_seq, __entry->stateid_hash + ) +); + +TRACE_EVENT(nfs4_close, + TP_PROTO( + const struct nfs4_state *state, + const struct nfs_closeargs *args, + const struct nfs_closeres *res, + int error + ), + + TP_ARGS(state, args, res, error), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(unsigned int, fmode) + __field(unsigned long, error) + __field(int, stateid_seq) + __field(u32, stateid_hash) + ), + + TP_fast_assign( + const struct inode *inode = state->inode; + + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); + __entry->fmode = (__force unsigned int)state->state; + __entry->error = error < 0 ? -error : 0; + __entry->stateid_seq = + be32_to_cpu(args->stateid.seqid); + __entry->stateid_hash = + nfs_stateid_hash(&args->stateid); + ), + + TP_printk( + "error=%ld (%s) fmode=%s fileid=%02x:%02x:%llu " + "fhandle=0x%08x openstateid=%d:0x%08x", + -__entry->error, + show_nfs4_status(__entry->error), + __entry->fmode ? show_fs_fmode_flags(__entry->fmode) : + "closed", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + __entry->stateid_seq, __entry->stateid_hash + ) +); + +DECLARE_EVENT_CLASS(nfs4_lock_event, + TP_PROTO( + const struct file_lock *request, + const struct nfs4_state *state, + int cmd, + int error + ), + + TP_ARGS(request, state, cmd, error), + + TP_STRUCT__entry( + __field(unsigned long, error) + __field(unsigned long, cmd) + __field(unsigned long, type) + __field(loff_t, start) + __field(loff_t, end) + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(int, stateid_seq) + __field(u32, stateid_hash) + ), + + TP_fast_assign( + const struct inode *inode = state->inode; + + __entry->error = error < 0 ? -error : 0; + __entry->cmd = cmd; + __entry->type = request->c.flc_type; + __entry->start = request->fl_start; + __entry->end = request->fl_end; + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); + __entry->stateid_seq = + be32_to_cpu(state->stateid.seqid); + __entry->stateid_hash = + nfs_stateid_hash(&state->stateid); + ), + + TP_printk( + "error=%ld (%s) cmd=%s:%s range=%lld:%lld " + "fileid=%02x:%02x:%llu fhandle=0x%08x " + "stateid=%d:0x%08x", + -__entry->error, + show_nfs4_status(__entry->error), + show_fs_fcntl_cmd(__entry->cmd), + show_fs_fcntl_lock_type(__entry->type), + (long long)__entry->start, + (long long)__entry->end, + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + __entry->stateid_seq, __entry->stateid_hash + ) +); + +#define DEFINE_NFS4_LOCK_EVENT(name) \ + DEFINE_EVENT(nfs4_lock_event, name, \ + TP_PROTO( \ + const struct file_lock *request, \ + const struct nfs4_state *state, \ + int cmd, \ + int error \ + ), \ + TP_ARGS(request, state, cmd, error)) +DEFINE_NFS4_LOCK_EVENT(nfs4_get_lock); +DEFINE_NFS4_LOCK_EVENT(nfs4_unlock); + +TRACE_EVENT(nfs4_set_lock, + TP_PROTO( + const struct file_lock *request, + const struct nfs4_state *state, + const nfs4_stateid *lockstateid, + int cmd, + int error + ), + + TP_ARGS(request, state, lockstateid, cmd, error), + + TP_STRUCT__entry( + __field(unsigned long, error) + __field(unsigned long, cmd) + __field(unsigned long, type) + __field(loff_t, start) + __field(loff_t, end) + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(int, stateid_seq) + __field(u32, stateid_hash) + __field(int, lockstateid_seq) + __field(u32, lockstateid_hash) + ), + + TP_fast_assign( + const struct inode *inode = state->inode; + + __entry->error = error < 0 ? -error : 0; + __entry->cmd = cmd; + __entry->type = request->c.flc_type; + __entry->start = request->fl_start; + __entry->end = request->fl_end; + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); + __entry->stateid_seq = + be32_to_cpu(state->stateid.seqid); + __entry->stateid_hash = + nfs_stateid_hash(&state->stateid); + __entry->lockstateid_seq = + be32_to_cpu(lockstateid->seqid); + __entry->lockstateid_hash = + nfs_stateid_hash(lockstateid); + ), + + TP_printk( + "error=%ld (%s) cmd=%s:%s range=%lld:%lld " + "fileid=%02x:%02x:%llu fhandle=0x%08x " + "stateid=%d:0x%08x lockstateid=%d:0x%08x", + -__entry->error, + show_nfs4_status(__entry->error), + show_fs_fcntl_cmd(__entry->cmd), + show_fs_fcntl_lock_type(__entry->type), + (long long)__entry->start, + (long long)__entry->end, + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + __entry->stateid_seq, __entry->stateid_hash, + __entry->lockstateid_seq, __entry->lockstateid_hash + ) +); + +TRACE_DEFINE_ENUM(LK_STATE_IN_USE); +TRACE_DEFINE_ENUM(NFS_DELEGATED_STATE); +TRACE_DEFINE_ENUM(NFS_OPEN_STATE); +TRACE_DEFINE_ENUM(NFS_O_RDONLY_STATE); +TRACE_DEFINE_ENUM(NFS_O_WRONLY_STATE); +TRACE_DEFINE_ENUM(NFS_O_RDWR_STATE); +TRACE_DEFINE_ENUM(NFS_STATE_RECLAIM_REBOOT); +TRACE_DEFINE_ENUM(NFS_STATE_RECLAIM_NOGRACE); +TRACE_DEFINE_ENUM(NFS_STATE_POSIX_LOCKS); +TRACE_DEFINE_ENUM(NFS_STATE_RECOVERY_FAILED); +TRACE_DEFINE_ENUM(NFS_STATE_MAY_NOTIFY_LOCK); +TRACE_DEFINE_ENUM(NFS_STATE_CHANGE_WAIT); +TRACE_DEFINE_ENUM(NFS_CLNT_DST_SSC_COPY_STATE); +TRACE_DEFINE_ENUM(NFS_CLNT_SRC_SSC_COPY_STATE); +TRACE_DEFINE_ENUM(NFS_SRV_SSC_COPY_STATE); + +#define show_nfs4_state_flags(flags) \ + __print_flags(flags, "|", \ + { LK_STATE_IN_USE, "IN_USE" }, \ + { NFS_DELEGATED_STATE, "DELEGATED" }, \ + { NFS_OPEN_STATE, "OPEN" }, \ + { NFS_O_RDONLY_STATE, "O_RDONLY" }, \ + { NFS_O_WRONLY_STATE, "O_WRONLY" }, \ + { NFS_O_RDWR_STATE, "O_RDWR" }, \ + { NFS_STATE_RECLAIM_REBOOT, "RECLAIM_REBOOT" }, \ + { NFS_STATE_RECLAIM_NOGRACE, "RECLAIM_NOGRACE" }, \ + { NFS_STATE_POSIX_LOCKS, "POSIX_LOCKS" }, \ + { NFS_STATE_RECOVERY_FAILED, "RECOVERY_FAILED" }, \ + { NFS_STATE_MAY_NOTIFY_LOCK, "MAY_NOTIFY_LOCK" }, \ + { NFS_STATE_CHANGE_WAIT, "CHANGE_WAIT" }, \ + { NFS_CLNT_DST_SSC_COPY_STATE, "CLNT_DST_SSC_COPY" }, \ + { NFS_CLNT_SRC_SSC_COPY_STATE, "CLNT_SRC_SSC_COPY" }, \ + { NFS_SRV_SSC_COPY_STATE, "SRV_SSC_COPY" }) + +#define show_nfs4_lock_flags(flags) \ + __print_flags(flags, "|", \ + { BIT(NFS_LOCK_INITIALIZED), "INITIALIZED" }, \ + { BIT(NFS_LOCK_LOST), "LOST" }) + +TRACE_EVENT(nfs4_state_lock_reclaim, + TP_PROTO( + const struct nfs4_state *state, + const struct nfs4_lock_state *lock + ), + + TP_ARGS(state, lock), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(unsigned long, state_flags) + __field(unsigned long, lock_flags) + __field(int, stateid_seq) + __field(u32, stateid_hash) + ), + + TP_fast_assign( + const struct inode *inode = state->inode; + + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); + __entry->state_flags = state->flags; + __entry->lock_flags = lock->ls_flags; + __entry->stateid_seq = + be32_to_cpu(state->stateid.seqid); + __entry->stateid_hash = + nfs_stateid_hash(&state->stateid); + ), + + TP_printk( + "fileid=%02x:%02x:%llu fhandle=0x%08x " + "stateid=%d:0x%08x state_flags=%s lock_flags=%s", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, __entry->fhandle, + __entry->stateid_seq, __entry->stateid_hash, + show_nfs4_state_flags(__entry->state_flags), + show_nfs4_lock_flags(__entry->lock_flags) + ) +) + +DECLARE_EVENT_CLASS(nfs4_set_delegation_event, + TP_PROTO( + const struct inode *inode, + fmode_t fmode + ), + + TP_ARGS(inode, fmode), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(unsigned int, fmode) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); + __entry->fmode = (__force unsigned int)fmode; + ), + + TP_printk( + "fmode=%s fileid=%02x:%02x:%llu fhandle=0x%08x", + show_fs_fmode_flags(__entry->fmode), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle + ) +); +#define DEFINE_NFS4_SET_DELEGATION_EVENT(name) \ + DEFINE_EVENT(nfs4_set_delegation_event, name, \ + TP_PROTO( \ + const struct inode *inode, \ + fmode_t fmode \ + ), \ + TP_ARGS(inode, fmode)) +DEFINE_NFS4_SET_DELEGATION_EVENT(nfs4_set_delegation); +DEFINE_NFS4_SET_DELEGATION_EVENT(nfs4_reclaim_delegation); +DEFINE_NFS4_SET_DELEGATION_EVENT(nfs4_detach_delegation); + +#define show_delegation_flags(flags) \ + __print_flags(flags, "|", \ + { BIT(NFS_DELEGATION_NEED_RECLAIM), "NEED_RECLAIM" }, \ + { BIT(NFS_DELEGATION_RETURN_IF_CLOSED), "RETURN_IF_CLOSED" }, \ + { BIT(NFS_DELEGATION_REFERENCED), "REFERENCED" }, \ + { BIT(NFS_DELEGATION_RETURNING), "RETURNING" }, \ + { BIT(NFS_DELEGATION_REVOKED), "REVOKED" }, \ + { BIT(NFS_DELEGATION_TEST_EXPIRED), "TEST_EXPIRED" }) + +DECLARE_EVENT_CLASS(nfs4_delegation_event, + TP_PROTO( + const struct nfs_delegation *delegation + ), + + TP_ARGS(delegation), + + TP_STRUCT__entry( + __field(u32, fhandle) + __field(unsigned int, fmode) + __field(unsigned long, flags) + ), + + TP_fast_assign( + __entry->fhandle = nfs_fhandle_hash(NFS_FH(delegation->inode)); + __entry->fmode = delegation->type; + __entry->flags = delegation->flags; + ), + + TP_printk( + "fhandle=0x%08x fmode=%s flags=%s", + __entry->fhandle, show_fs_fmode_flags(__entry->fmode), + show_delegation_flags(__entry->flags) + ) +); +#define DEFINE_NFS4_DELEGATION_EVENT(name) \ + DEFINE_EVENT(nfs4_delegation_event, name, \ + TP_PROTO( \ + const struct nfs_delegation *delegation \ + ), \ + TP_ARGS(delegation)) +DEFINE_NFS4_DELEGATION_EVENT(nfs_delegation_need_return); + +TRACE_EVENT(nfs4_delegreturn_exit, + TP_PROTO( + const struct nfs4_delegreturnargs *args, + const struct nfs4_delegreturnres *res, + int error + ), + + TP_ARGS(args, res, error), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(unsigned long, error) + __field(int, stateid_seq) + __field(u32, stateid_hash) + ), + + TP_fast_assign( + __entry->dev = res->server->s_dev; + __entry->fhandle = nfs_fhandle_hash(args->fhandle); + __entry->error = error < 0 ? -error : 0; + __entry->stateid_seq = + be32_to_cpu(args->stateid->seqid); + __entry->stateid_hash = + nfs_stateid_hash(args->stateid); + ), + + TP_printk( + "error=%ld (%s) dev=%02x:%02x fhandle=0x%08x " + "stateid=%d:0x%08x", + -__entry->error, + show_nfs4_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->fhandle, + __entry->stateid_seq, __entry->stateid_hash + ) +); + +DECLARE_EVENT_CLASS(nfs4_test_stateid_event, + TP_PROTO( + const struct nfs4_state *state, + const struct nfs4_lock_state *lsp, + int error + ), + + TP_ARGS(state, lsp, error), + + TP_STRUCT__entry( + __field(unsigned long, error) + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(int, stateid_seq) + __field(u32, stateid_hash) + ), + + TP_fast_assign( + const struct inode *inode = state->inode; + + __entry->error = error < 0 ? -error : 0; + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); + __entry->stateid_seq = + be32_to_cpu(state->stateid.seqid); + __entry->stateid_hash = + nfs_stateid_hash(&state->stateid); + ), + + TP_printk( + "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " + "stateid=%d:0x%08x", + -__entry->error, + show_nfs4_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + __entry->stateid_seq, __entry->stateid_hash + ) +); + +#define DEFINE_NFS4_TEST_STATEID_EVENT(name) \ + DEFINE_EVENT(nfs4_test_stateid_event, name, \ + TP_PROTO( \ + const struct nfs4_state *state, \ + const struct nfs4_lock_state *lsp, \ + int error \ + ), \ + TP_ARGS(state, lsp, error)) +DEFINE_NFS4_TEST_STATEID_EVENT(nfs4_test_delegation_stateid); +DEFINE_NFS4_TEST_STATEID_EVENT(nfs4_test_open_stateid); +DEFINE_NFS4_TEST_STATEID_EVENT(nfs4_test_lock_stateid); + +DECLARE_EVENT_CLASS(nfs4_lookup_event, + TP_PROTO( + const struct inode *dir, + const struct qstr *name, + int error + ), + + TP_ARGS(dir, name, error), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned long, error) + __field(u64, dir) + __string(name, name->name) + ), + + TP_fast_assign( + __entry->dev = dir->i_sb->s_dev; + __entry->dir = dir->i_ino; + __entry->error = -error; + __assign_str(name); + ), + + TP_printk( + "error=%ld (%s) name=%02x:%02x:%llu/%s", + -__entry->error, + show_nfs4_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->dir, + __get_str(name) + ) +); + +#define DEFINE_NFS4_LOOKUP_EVENT(name) \ + DEFINE_EVENT(nfs4_lookup_event, name, \ + TP_PROTO( \ + const struct inode *dir, \ + const struct qstr *name, \ + int error \ + ), \ + TP_ARGS(dir, name, error)) + +DEFINE_NFS4_LOOKUP_EVENT(nfs4_lookup); +DEFINE_NFS4_LOOKUP_EVENT(nfs4_symlink); +DEFINE_NFS4_LOOKUP_EVENT(nfs4_mkdir); +DEFINE_NFS4_LOOKUP_EVENT(nfs4_mknod); +DEFINE_NFS4_LOOKUP_EVENT(nfs4_remove); +DEFINE_NFS4_LOOKUP_EVENT(nfs4_get_fs_locations); +DEFINE_NFS4_LOOKUP_EVENT(nfs4_secinfo); + +TRACE_EVENT(nfs4_lookupp, + TP_PROTO( + const struct inode *inode, + int error + ), + + TP_ARGS(inode, error), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u64, ino) + __field(unsigned long, error) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->error = error < 0 ? -error : 0; + ), + + TP_printk( + "error=%ld (%s) inode=%02x:%02x:%llu", + -__entry->error, + show_nfs4_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->ino + ) +); + +TRACE_EVENT(nfs4_rename, + TP_PROTO( + const struct inode *olddir, + const struct qstr *oldname, + const struct inode *newdir, + const struct qstr *newname, + int error + ), + + TP_ARGS(olddir, oldname, newdir, newname, error), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned long, error) + __field(u64, olddir) + __string(oldname, oldname->name) + __field(u64, newdir) + __string(newname, newname->name) + ), + + TP_fast_assign( + __entry->dev = olddir->i_sb->s_dev; + __entry->olddir = olddir->i_ino; + __entry->newdir = newdir->i_ino; + __entry->error = error < 0 ? -error : 0; + __assign_str(oldname); + __assign_str(newname); + ), + + TP_printk( + "error=%ld (%s) oldname=%02x:%02x:%llu/%s " + "newname=%02x:%02x:%llu/%s", + -__entry->error, + show_nfs4_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->olddir, + __get_str(oldname), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->newdir, + __get_str(newname) + ) +); + +DECLARE_EVENT_CLASS(nfs4_inode_event, + TP_PROTO( + const struct inode *inode, + int error + ), + + TP_ARGS(inode, error), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(unsigned long, error) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); + __entry->error = error < 0 ? -error : 0; + ), + + TP_printk( + "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x", + -__entry->error, + show_nfs4_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle + ) +); + +#define DEFINE_NFS4_INODE_EVENT(name) \ + DEFINE_EVENT(nfs4_inode_event, name, \ + TP_PROTO( \ + const struct inode *inode, \ + int error \ + ), \ + TP_ARGS(inode, error)) + +DEFINE_NFS4_INODE_EVENT(nfs4_access); +DEFINE_NFS4_INODE_EVENT(nfs4_readlink); +DEFINE_NFS4_INODE_EVENT(nfs4_readdir); +DEFINE_NFS4_INODE_EVENT(nfs4_get_acl); +DEFINE_NFS4_INODE_EVENT(nfs4_set_acl); +#ifdef CONFIG_NFS_V4_SECURITY_LABEL +DEFINE_NFS4_INODE_EVENT(nfs4_get_security_label); +DEFINE_NFS4_INODE_EVENT(nfs4_set_security_label); +#endif /* CONFIG_NFS_V4_SECURITY_LABEL */ + +DECLARE_EVENT_CLASS(nfs4_inode_stateid_event, + TP_PROTO( + const struct inode *inode, + const nfs4_stateid *stateid, + int error + ), + + TP_ARGS(inode, stateid, error), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(unsigned long, error) + __field(int, stateid_seq) + __field(u32, stateid_hash) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); + __entry->error = error < 0 ? -error : 0; + __entry->stateid_seq = + be32_to_cpu(stateid->seqid); + __entry->stateid_hash = + nfs_stateid_hash(stateid); + ), + + TP_printk( + "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " + "stateid=%d:0x%08x", + -__entry->error, + show_nfs4_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + __entry->stateid_seq, __entry->stateid_hash + ) +); + +#define DEFINE_NFS4_INODE_STATEID_EVENT(name) \ + DEFINE_EVENT(nfs4_inode_stateid_event, name, \ + TP_PROTO( \ + const struct inode *inode, \ + const nfs4_stateid *stateid, \ + int error \ + ), \ + TP_ARGS(inode, stateid, error)) + +DEFINE_NFS4_INODE_STATEID_EVENT(nfs4_setattr); +DEFINE_NFS4_INODE_STATEID_EVENT(nfs4_delegreturn); +DEFINE_NFS4_INODE_STATEID_EVENT(nfs4_open_stateid_update); +DEFINE_NFS4_INODE_STATEID_EVENT(nfs4_open_stateid_update_wait); +DEFINE_NFS4_INODE_STATEID_EVENT(nfs4_open_stateid_update_skip); +DEFINE_NFS4_INODE_STATEID_EVENT(nfs4_close_stateid_update_wait); + +DECLARE_EVENT_CLASS(nfs4_getattr_event, + TP_PROTO( + const struct nfs_server *server, + const struct nfs_fh *fhandle, + const struct nfs_fattr *fattr, + int error + ), + + TP_ARGS(server, fhandle, fattr, error), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(unsigned int, valid) + __field(unsigned long, error) + ), + + TP_fast_assign( + __entry->dev = server->s_dev; + __entry->valid = fattr->valid; + __entry->fhandle = nfs_fhandle_hash(fhandle); + __entry->fileid = (fattr->valid & NFS_ATTR_FATTR_FILEID) ? fattr->fileid : 0; + __entry->error = error < 0 ? -error : 0; + ), + + TP_printk( + "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " + "valid=%s", + -__entry->error, + show_nfs4_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + show_nfs_fattr_flags(__entry->valid) + ) +); + +#define DEFINE_NFS4_GETATTR_EVENT(name) \ + DEFINE_EVENT(nfs4_getattr_event, name, \ + TP_PROTO( \ + const struct nfs_server *server, \ + const struct nfs_fh *fhandle, \ + const struct nfs_fattr *fattr, \ + int error \ + ), \ + TP_ARGS(server, fhandle, fattr, error)) +DEFINE_NFS4_GETATTR_EVENT(nfs4_getattr); +DEFINE_NFS4_GETATTR_EVENT(nfs4_lookup_root); +DEFINE_NFS4_GETATTR_EVENT(nfs4_fsinfo); + +DECLARE_EVENT_CLASS(nfs4_inode_callback_event, + TP_PROTO( + const struct nfs_client *clp, + const struct nfs_fh *fhandle, + const struct inode *inode, + int error + ), + + TP_ARGS(clp, fhandle, inode, error), + + TP_STRUCT__entry( + __field(unsigned long, error) + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __string(dstaddr, clp ? clp->cl_hostname : "unknown") + ), + + TP_fast_assign( + __entry->error = error < 0 ? -error : 0; + __entry->fhandle = nfs_fhandle_hash(fhandle); + if (!IS_ERR_OR_NULL(inode)) { + __entry->fileid = inode->i_ino; + __entry->dev = inode->i_sb->s_dev; + } else { + __entry->fileid = 0; + __entry->dev = 0; + } + __assign_str(dstaddr); + ), + + TP_printk( + "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " + "dstaddr=%s", + -__entry->error, + show_nfs4_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + __get_str(dstaddr) + ) +); + +#define DEFINE_NFS4_INODE_CALLBACK_EVENT(name) \ + DEFINE_EVENT(nfs4_inode_callback_event, name, \ + TP_PROTO( \ + const struct nfs_client *clp, \ + const struct nfs_fh *fhandle, \ + const struct inode *inode, \ + int error \ + ), \ + TP_ARGS(clp, fhandle, inode, error)) +DEFINE_NFS4_INODE_CALLBACK_EVENT(nfs4_cb_getattr); + +DECLARE_EVENT_CLASS(nfs4_inode_stateid_callback_event, + TP_PROTO( + const struct nfs_client *clp, + const struct nfs_fh *fhandle, + const struct inode *inode, + const nfs4_stateid *stateid, + int error + ), + + TP_ARGS(clp, fhandle, inode, stateid, error), + + TP_STRUCT__entry( + __field(unsigned long, error) + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __string(dstaddr, clp ? clp->cl_hostname : "unknown") + __field(int, stateid_seq) + __field(u32, stateid_hash) + ), + + TP_fast_assign( + __entry->error = error < 0 ? -error : 0; + __entry->fhandle = nfs_fhandle_hash(fhandle); + if (!IS_ERR_OR_NULL(inode)) { + __entry->fileid = inode->i_ino; + __entry->dev = inode->i_sb->s_dev; + } else { + __entry->fileid = 0; + __entry->dev = 0; + } + __assign_str(dstaddr); + __entry->stateid_seq = + be32_to_cpu(stateid->seqid); + __entry->stateid_hash = + nfs_stateid_hash(stateid); + ), + + TP_printk( + "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " + "stateid=%d:0x%08x dstaddr=%s", + -__entry->error, + show_nfs4_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + __entry->stateid_seq, __entry->stateid_hash, + __get_str(dstaddr) + ) +); + +#define DEFINE_NFS4_INODE_STATEID_CALLBACK_EVENT(name) \ + DEFINE_EVENT(nfs4_inode_stateid_callback_event, name, \ + TP_PROTO( \ + const struct nfs_client *clp, \ + const struct nfs_fh *fhandle, \ + const struct inode *inode, \ + const nfs4_stateid *stateid, \ + int error \ + ), \ + TP_ARGS(clp, fhandle, inode, stateid, error)) +DEFINE_NFS4_INODE_STATEID_CALLBACK_EVENT(nfs4_cb_recall); +DEFINE_NFS4_INODE_STATEID_CALLBACK_EVENT(nfs4_cb_layoutrecall_file); + +#define show_stateid_type(type) \ + __print_symbolic(type, \ + { NFS4_INVALID_STATEID_TYPE, "INVALID" }, \ + { NFS4_SPECIAL_STATEID_TYPE, "SPECIAL" }, \ + { NFS4_OPEN_STATEID_TYPE, "OPEN" }, \ + { NFS4_LOCK_STATEID_TYPE, "LOCK" }, \ + { NFS4_DELEGATION_STATEID_TYPE, "DELEGATION" }, \ + { NFS4_LAYOUT_STATEID_TYPE, "LAYOUT" }, \ + { NFS4_PNFS_DS_STATEID_TYPE, "PNFS_DS" }, \ + { NFS4_REVOKED_STATEID_TYPE, "REVOKED" }, \ + { NFS4_FREED_STATEID_TYPE, "FREED" }) + +DECLARE_EVENT_CLASS(nfs4_match_stateid_event, + TP_PROTO( + const nfs4_stateid *s1, + const nfs4_stateid *s2 + ), + + TP_ARGS(s1, s2), + + TP_STRUCT__entry( + __field(int, s1_seq) + __field(int, s2_seq) + __field(u32, s1_hash) + __field(u32, s2_hash) + __field(int, s1_type) + __field(int, s2_type) + ), + + TP_fast_assign( + __entry->s1_seq = s1->seqid; + __entry->s1_hash = nfs_stateid_hash(s1); + __entry->s1_type = s1->type; + __entry->s2_seq = s2->seqid; + __entry->s2_hash = nfs_stateid_hash(s2); + __entry->s2_type = s2->type; + ), + + TP_printk( + "s1=%s:%x:%u s2=%s:%x:%u", + show_stateid_type(__entry->s1_type), + __entry->s1_hash, __entry->s1_seq, + show_stateid_type(__entry->s2_type), + __entry->s2_hash, __entry->s2_seq + ) +); + +#define DEFINE_NFS4_MATCH_STATEID_EVENT(name) \ + DEFINE_EVENT(nfs4_match_stateid_event, name, \ + TP_PROTO( \ + const nfs4_stateid *s1, \ + const nfs4_stateid *s2 \ + ), \ + TP_ARGS(s1, s2)) +DEFINE_NFS4_MATCH_STATEID_EVENT(nfs41_match_stateid); +DEFINE_NFS4_MATCH_STATEID_EVENT(nfs4_match_stateid); + +DECLARE_EVENT_CLASS(nfs4_idmap_event, + TP_PROTO( + const char *name, + int len, + u32 id, + int error + ), + + TP_ARGS(name, len, id, error), + + TP_STRUCT__entry( + __field(unsigned long, error) + __field(u32, id) + __dynamic_array(char, name, len > 0 ? len + 1 : 1) + ), + + TP_fast_assign( + if (len < 0) + len = 0; + __entry->error = error < 0 ? error : 0; + __entry->id = id; + memcpy(__get_str(name), name, len); + __get_str(name)[len] = 0; + ), + + TP_printk( + "error=%ld (%s) id=%u name=%s", + -__entry->error, show_nfs4_status(__entry->error), + __entry->id, + __get_str(name) + ) +); +#define DEFINE_NFS4_IDMAP_EVENT(name) \ + DEFINE_EVENT(nfs4_idmap_event, name, \ + TP_PROTO( \ + const char *name, \ + int len, \ + u32 id, \ + int error \ + ), \ + TP_ARGS(name, len, id, error)) +DEFINE_NFS4_IDMAP_EVENT(nfs4_map_name_to_uid); +DEFINE_NFS4_IDMAP_EVENT(nfs4_map_group_to_gid); +DEFINE_NFS4_IDMAP_EVENT(nfs4_map_uid_to_name); +DEFINE_NFS4_IDMAP_EVENT(nfs4_map_gid_to_group); + +#define NFS4_LSEG_LAYOUT_STATEID_HASH(lseg) \ + (lseg ? nfs_stateid_hash(&lseg->pls_layout->plh_stateid) : 0) + +DECLARE_EVENT_CLASS(nfs4_read_event, + TP_PROTO( + const struct nfs_pgio_header *hdr, + int error + ), + + TP_ARGS(hdr, error), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(loff_t, offset) + __field(u32, arg_count) + __field(u32, res_count) + __field(unsigned long, error) + __field(int, stateid_seq) + __field(u32, stateid_hash) + __field(int, layoutstateid_seq) + __field(u32, layoutstateid_hash) + ), + + TP_fast_assign( + const struct inode *inode = hdr->inode; + const struct nfs_inode *nfsi = NFS_I(inode); + const struct nfs_fh *fh = hdr->args.fh ? + hdr->args.fh : &nfsi->fh; + const struct nfs4_state *state = + hdr->args.context->state; + const struct pnfs_layout_segment *lseg = hdr->lseg; + + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(fh); + __entry->offset = hdr->args.offset; + __entry->arg_count = hdr->args.count; + __entry->res_count = hdr->res.count; + __entry->error = error < 0 ? -error : 0; + __entry->stateid_seq = + be32_to_cpu(state->stateid.seqid); + __entry->stateid_hash = + nfs_stateid_hash(&state->stateid); + __entry->layoutstateid_seq = lseg ? lseg->pls_seq : 0; + __entry->layoutstateid_hash = + NFS4_LSEG_LAYOUT_STATEID_HASH(lseg); + ), + + TP_printk( + "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " + "offset=%lld count=%u res=%u stateid=%d:0x%08x " + "layoutstateid=%d:0x%08x", + -__entry->error, + show_nfs4_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + (long long)__entry->offset, + __entry->arg_count, __entry->res_count, + __entry->stateid_seq, __entry->stateid_hash, + __entry->layoutstateid_seq, __entry->layoutstateid_hash + ) +); +#define DEFINE_NFS4_READ_EVENT(name) \ + DEFINE_EVENT(nfs4_read_event, name, \ + TP_PROTO( \ + const struct nfs_pgio_header *hdr, \ + int error \ + ), \ + TP_ARGS(hdr, error)) +DEFINE_NFS4_READ_EVENT(nfs4_read); +DEFINE_NFS4_READ_EVENT(nfs4_pnfs_read); + +DECLARE_EVENT_CLASS(nfs4_write_event, + TP_PROTO( + const struct nfs_pgio_header *hdr, + int error + ), + + TP_ARGS(hdr, error), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(loff_t, offset) + __field(u32, arg_count) + __field(u32, res_count) + __field(unsigned long, error) + __field(int, stateid_seq) + __field(u32, stateid_hash) + __field(int, layoutstateid_seq) + __field(u32, layoutstateid_hash) + ), + + TP_fast_assign( + const struct inode *inode = hdr->inode; + const struct nfs_inode *nfsi = NFS_I(inode); + const struct nfs_fh *fh = hdr->args.fh ? + hdr->args.fh : &nfsi->fh; + const struct nfs4_state *state = + hdr->args.context->state; + const struct pnfs_layout_segment *lseg = hdr->lseg; + + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(fh); + __entry->offset = hdr->args.offset; + __entry->arg_count = hdr->args.count; + __entry->res_count = hdr->res.count; + __entry->error = error < 0 ? -error : 0; + __entry->stateid_seq = + be32_to_cpu(state->stateid.seqid); + __entry->stateid_hash = + nfs_stateid_hash(&state->stateid); + __entry->layoutstateid_seq = lseg ? lseg->pls_seq : 0; + __entry->layoutstateid_hash = + NFS4_LSEG_LAYOUT_STATEID_HASH(lseg); + ), + + TP_printk( + "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " + "offset=%lld count=%u res=%u stateid=%d:0x%08x " + "layoutstateid=%d:0x%08x", + -__entry->error, + show_nfs4_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + (long long)__entry->offset, + __entry->arg_count, __entry->res_count, + __entry->stateid_seq, __entry->stateid_hash, + __entry->layoutstateid_seq, __entry->layoutstateid_hash + ) +); + +#define DEFINE_NFS4_WRITE_EVENT(name) \ + DEFINE_EVENT(nfs4_write_event, name, \ + TP_PROTO( \ + const struct nfs_pgio_header *hdr, \ + int error \ + ), \ + TP_ARGS(hdr, error)) +DEFINE_NFS4_WRITE_EVENT(nfs4_write); +DEFINE_NFS4_WRITE_EVENT(nfs4_pnfs_write); + +DECLARE_EVENT_CLASS(nfs4_commit_event, + TP_PROTO( + const struct nfs_commit_data *data, + int error + ), + + TP_ARGS(data, error), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(unsigned long, error) + __field(loff_t, offset) + __field(u32, count) + __field(int, layoutstateid_seq) + __field(u32, layoutstateid_hash) + ), + + TP_fast_assign( + const struct inode *inode = data->inode; + const struct nfs_inode *nfsi = NFS_I(inode); + const struct nfs_fh *fh = data->args.fh ? + data->args.fh : &nfsi->fh; + const struct pnfs_layout_segment *lseg = data->lseg; + + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(fh); + __entry->offset = data->args.offset; + __entry->count = data->args.count; + __entry->error = error < 0 ? -error : 0; + __entry->layoutstateid_seq = lseg ? lseg->pls_seq : 0; + __entry->layoutstateid_hash = + NFS4_LSEG_LAYOUT_STATEID_HASH(lseg); + ), + + TP_printk( + "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " + "offset=%lld count=%u layoutstateid=%d:0x%08x", + -__entry->error, + show_nfs4_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + (long long)__entry->offset, + __entry->count, + __entry->layoutstateid_seq, __entry->layoutstateid_hash + ) +); +#define DEFINE_NFS4_COMMIT_EVENT(name) \ + DEFINE_EVENT(nfs4_commit_event, name, \ + TP_PROTO( \ + const struct nfs_commit_data *data, \ + int error \ + ), \ + TP_ARGS(data, error)) +DEFINE_NFS4_COMMIT_EVENT(nfs4_commit); +DEFINE_NFS4_COMMIT_EVENT(nfs4_pnfs_commit_ds); + +TRACE_EVENT(nfs4_layoutget, + TP_PROTO( + const struct nfs_open_context *ctx, + const struct pnfs_layout_range *args, + const struct pnfs_layout_range *res, + const nfs4_stateid *layout_stateid, + int error + ), + + TP_ARGS(ctx, args, res, layout_stateid, error), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(u32, iomode) + __field(u64, offset) + __field(u64, count) + __field(unsigned long, error) + __field(int, stateid_seq) + __field(u32, stateid_hash) + __field(int, layoutstateid_seq) + __field(u32, layoutstateid_hash) + ), + + TP_fast_assign( + const struct inode *inode = d_inode(ctx->dentry); + const struct nfs4_state *state = ctx->state; + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); + __entry->iomode = args->iomode; + __entry->offset = args->offset; + __entry->count = args->length; + __entry->error = error < 0 ? -error : 0; + __entry->stateid_seq = + be32_to_cpu(state->stateid.seqid); + __entry->stateid_hash = + nfs_stateid_hash(&state->stateid); + if (!error) { + __entry->layoutstateid_seq = + be32_to_cpu(layout_stateid->seqid); + __entry->layoutstateid_hash = + nfs_stateid_hash(layout_stateid); + } else { + __entry->layoutstateid_seq = 0; + __entry->layoutstateid_hash = 0; + } + ), + + TP_printk( + "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " + "iomode=%s offset=%llu count=%llu stateid=%d:0x%08x " + "layoutstateid=%d:0x%08x", + -__entry->error, + show_nfs4_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + show_pnfs_layout_iomode(__entry->iomode), + (unsigned long long)__entry->offset, + (unsigned long long)__entry->count, + __entry->stateid_seq, __entry->stateid_hash, + __entry->layoutstateid_seq, __entry->layoutstateid_hash + ) +); + +DEFINE_NFS4_INODE_STATEID_EVENT(nfs4_layoutcommit); +DEFINE_NFS4_INODE_STATEID_EVENT(nfs4_layoutreturn); +DEFINE_NFS4_INODE_STATEID_EVENT(nfs4_layoutreturn_on_close); +DEFINE_NFS4_INODE_STATEID_EVENT(nfs4_layouterror); +DEFINE_NFS4_INODE_STATEID_EVENT(nfs4_layoutstats); + +TRACE_DEFINE_ENUM(PNFS_UPDATE_LAYOUT_UNKNOWN); +TRACE_DEFINE_ENUM(PNFS_UPDATE_LAYOUT_NO_PNFS); +TRACE_DEFINE_ENUM(PNFS_UPDATE_LAYOUT_RD_ZEROLEN); +TRACE_DEFINE_ENUM(PNFS_UPDATE_LAYOUT_MDSTHRESH); +TRACE_DEFINE_ENUM(PNFS_UPDATE_LAYOUT_NOMEM); +TRACE_DEFINE_ENUM(PNFS_UPDATE_LAYOUT_BULK_RECALL); +TRACE_DEFINE_ENUM(PNFS_UPDATE_LAYOUT_IO_TEST_FAIL); +TRACE_DEFINE_ENUM(PNFS_UPDATE_LAYOUT_FOUND_CACHED); +TRACE_DEFINE_ENUM(PNFS_UPDATE_LAYOUT_RETURN); +TRACE_DEFINE_ENUM(PNFS_UPDATE_LAYOUT_BLOCKED); +TRACE_DEFINE_ENUM(PNFS_UPDATE_LAYOUT_INVALID_OPEN); +TRACE_DEFINE_ENUM(PNFS_UPDATE_LAYOUT_RETRY); +TRACE_DEFINE_ENUM(PNFS_UPDATE_LAYOUT_SEND_LAYOUTGET); +TRACE_DEFINE_ENUM(PNFS_UPDATE_LAYOUT_EXIT); + +#define show_pnfs_update_layout_reason(reason) \ + __print_symbolic(reason, \ + { PNFS_UPDATE_LAYOUT_UNKNOWN, "unknown" }, \ + { PNFS_UPDATE_LAYOUT_NO_PNFS, "no pnfs" }, \ + { PNFS_UPDATE_LAYOUT_RD_ZEROLEN, "read+zerolen" }, \ + { PNFS_UPDATE_LAYOUT_MDSTHRESH, "mdsthresh" }, \ + { PNFS_UPDATE_LAYOUT_NOMEM, "nomem" }, \ + { PNFS_UPDATE_LAYOUT_BULK_RECALL, "bulk recall" }, \ + { PNFS_UPDATE_LAYOUT_IO_TEST_FAIL, "io test fail" }, \ + { PNFS_UPDATE_LAYOUT_FOUND_CACHED, "found cached" }, \ + { PNFS_UPDATE_LAYOUT_RETURN, "layoutreturn" }, \ + { PNFS_UPDATE_LAYOUT_BLOCKED, "layouts blocked" }, \ + { PNFS_UPDATE_LAYOUT_INVALID_OPEN, "invalid open" }, \ + { PNFS_UPDATE_LAYOUT_RETRY, "retrying" }, \ + { PNFS_UPDATE_LAYOUT_SEND_LAYOUTGET, "sent layoutget" }, \ + { PNFS_UPDATE_LAYOUT_EXIT, "exit" }) + +TRACE_EVENT(pnfs_update_layout, + TP_PROTO(struct inode *inode, + loff_t pos, + u64 count, + enum pnfs_iomode iomode, + struct pnfs_layout_hdr *lo, + struct pnfs_layout_segment *lseg, + enum pnfs_update_layout_reason reason + ), + TP_ARGS(inode, pos, count, iomode, lo, lseg, reason), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u64, fileid) + __field(u32, fhandle) + __field(loff_t, pos) + __field(u64, count) + __field(enum pnfs_iomode, iomode) + __field(int, layoutstateid_seq) + __field(u32, layoutstateid_hash) + __field(long, lseg) + __field(enum pnfs_update_layout_reason, reason) + ), + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); + __entry->pos = pos; + __entry->count = count; + __entry->iomode = iomode; + __entry->reason = reason; + if (lo != NULL && pnfs_layout_is_valid(lo)) { + __entry->layoutstateid_seq = + be32_to_cpu(lo->plh_stateid.seqid); + __entry->layoutstateid_hash = + nfs_stateid_hash(&lo->plh_stateid); + } else { + __entry->layoutstateid_seq = 0; + __entry->layoutstateid_hash = 0; + } + __entry->lseg = (long)lseg; + ), + TP_printk( + "fileid=%02x:%02x:%llu fhandle=0x%08x " + "iomode=%s pos=%llu count=%llu " + "layoutstateid=%d:0x%08x lseg=0x%lx (%s)", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + show_pnfs_layout_iomode(__entry->iomode), + (unsigned long long)__entry->pos, + (unsigned long long)__entry->count, + __entry->layoutstateid_seq, __entry->layoutstateid_hash, + __entry->lseg, + show_pnfs_update_layout_reason(__entry->reason) + ) +); + +DECLARE_EVENT_CLASS(pnfs_layout_event, + TP_PROTO(struct inode *inode, + loff_t pos, + u64 count, + enum pnfs_iomode iomode, + struct pnfs_layout_hdr *lo, + struct pnfs_layout_segment *lseg + ), + TP_ARGS(inode, pos, count, iomode, lo, lseg), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u64, fileid) + __field(u32, fhandle) + __field(loff_t, pos) + __field(u64, count) + __field(enum pnfs_iomode, iomode) + __field(int, layoutstateid_seq) + __field(u32, layoutstateid_hash) + __field(long, lseg) + ), + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); + __entry->pos = pos; + __entry->count = count; + __entry->iomode = iomode; + if (lo != NULL && pnfs_layout_is_valid(lo)) { + __entry->layoutstateid_seq = + be32_to_cpu(lo->plh_stateid.seqid); + __entry->layoutstateid_hash = + nfs_stateid_hash(&lo->plh_stateid); + } else { + __entry->layoutstateid_seq = 0; + __entry->layoutstateid_hash = 0; + } + __entry->lseg = (long)lseg; + ), + TP_printk( + "fileid=%02x:%02x:%llu fhandle=0x%08x " + "iomode=%s pos=%llu count=%llu " + "layoutstateid=%d:0x%08x lseg=0x%lx", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + show_pnfs_layout_iomode(__entry->iomode), + (unsigned long long)__entry->pos, + (unsigned long long)__entry->count, + __entry->layoutstateid_seq, __entry->layoutstateid_hash, + __entry->lseg + ) +); + +#define DEFINE_PNFS_LAYOUT_EVENT(name) \ + DEFINE_EVENT(pnfs_layout_event, name, \ + TP_PROTO(struct inode *inode, \ + loff_t pos, \ + u64 count, \ + enum pnfs_iomode iomode, \ + struct pnfs_layout_hdr *lo, \ + struct pnfs_layout_segment *lseg \ + ), \ + TP_ARGS(inode, pos, count, iomode, lo, lseg)) + +DEFINE_PNFS_LAYOUT_EVENT(pnfs_mds_fallback_pg_init_read); +DEFINE_PNFS_LAYOUT_EVENT(pnfs_mds_fallback_pg_init_write); +DEFINE_PNFS_LAYOUT_EVENT(pnfs_mds_fallback_pg_get_mirror_count); +DEFINE_PNFS_LAYOUT_EVENT(pnfs_mds_fallback_read_done); +DEFINE_PNFS_LAYOUT_EVENT(pnfs_mds_fallback_write_done); +DEFINE_PNFS_LAYOUT_EVENT(pnfs_mds_fallback_read_pagelist); +DEFINE_PNFS_LAYOUT_EVENT(pnfs_mds_fallback_write_pagelist); + +DECLARE_EVENT_CLASS(nfs4_deviceid_event, + TP_PROTO( + const struct nfs_client *clp, + const struct nfs4_deviceid *deviceid + ), + + TP_ARGS(clp, deviceid), + + TP_STRUCT__entry( + __string(dstaddr, clp->cl_hostname) + __array(unsigned char, deviceid, NFS4_DEVICEID4_SIZE) + ), + + TP_fast_assign( + __assign_str(dstaddr); + memcpy(__entry->deviceid, deviceid->data, + NFS4_DEVICEID4_SIZE); + ), + + TP_printk( + "deviceid=%s, dstaddr=%s", + __print_hex(__entry->deviceid, NFS4_DEVICEID4_SIZE), + __get_str(dstaddr) + ) +); +#define DEFINE_PNFS_DEVICEID_EVENT(name) \ + DEFINE_EVENT(nfs4_deviceid_event, name, \ + TP_PROTO(const struct nfs_client *clp, \ + const struct nfs4_deviceid *deviceid \ + ), \ + TP_ARGS(clp, deviceid)) +DEFINE_PNFS_DEVICEID_EVENT(nfs4_deviceid_free); + +DECLARE_EVENT_CLASS(nfs4_deviceid_status, + TP_PROTO( + const struct nfs_server *server, + const struct nfs4_deviceid *deviceid, + int status + ), + + TP_ARGS(server, deviceid, status), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(int, status) + __string(dstaddr, server->nfs_client->cl_hostname) + __array(unsigned char, deviceid, NFS4_DEVICEID4_SIZE) + ), + + TP_fast_assign( + __entry->dev = server->s_dev; + __entry->status = status; + __assign_str(dstaddr); + memcpy(__entry->deviceid, deviceid->data, + NFS4_DEVICEID4_SIZE); + ), + + TP_printk( + "dev=%02x:%02x: deviceid=%s, dstaddr=%s, status=%d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_hex(__entry->deviceid, NFS4_DEVICEID4_SIZE), + __get_str(dstaddr), + __entry->status + ) +); +#define DEFINE_PNFS_DEVICEID_STATUS(name) \ + DEFINE_EVENT(nfs4_deviceid_status, name, \ + TP_PROTO(const struct nfs_server *server, \ + const struct nfs4_deviceid *deviceid, \ + int status \ + ), \ + TP_ARGS(server, deviceid, status)) +DEFINE_PNFS_DEVICEID_STATUS(nfs4_getdeviceinfo); +DEFINE_PNFS_DEVICEID_STATUS(nfs4_find_deviceid); + +TRACE_EVENT(fl_getdevinfo, + TP_PROTO( + const struct nfs_server *server, + const struct nfs4_deviceid *deviceid, + char *ds_remotestr + ), + TP_ARGS(server, deviceid, ds_remotestr), + + TP_STRUCT__entry( + __string(mds_addr, server->nfs_client->cl_hostname) + __array(unsigned char, deviceid, NFS4_DEVICEID4_SIZE) + __string(ds_ips, ds_remotestr) + ), + + TP_fast_assign( + __assign_str(mds_addr); + __assign_str(ds_ips); + memcpy(__entry->deviceid, deviceid->data, + NFS4_DEVICEID4_SIZE); + ), + TP_printk( + "deviceid=%s, mds_addr=%s, ds_ips=%s", + __print_hex(__entry->deviceid, NFS4_DEVICEID4_SIZE), + __get_str(mds_addr), + __get_str(ds_ips) + ) +); + +DECLARE_EVENT_CLASS(nfs4_flexfiles_io_event, + TP_PROTO( + const struct nfs_pgio_header *hdr, + int error + ), + + TP_ARGS(hdr, error), + + TP_STRUCT__entry( + __field(unsigned long, error) + __field(unsigned long, nfs_error) + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(loff_t, offset) + __field(u32, count) + __field(int, stateid_seq) + __field(u32, stateid_hash) + __string(dstaddr, hdr->ds_clp ? + rpc_peeraddr2str(hdr->ds_clp->cl_rpcclient, + RPC_DISPLAY_ADDR) : "unknown") + ), + + TP_fast_assign( + const struct inode *inode = hdr->inode; + + __entry->error = -error; + __entry->nfs_error = hdr->res.op_status; + __entry->fhandle = nfs_fhandle_hash(hdr->args.fh); + __entry->fileid = inode->i_ino; + __entry->dev = inode->i_sb->s_dev; + __entry->offset = hdr->args.offset; + __entry->count = hdr->args.count; + __entry->stateid_seq = + be32_to_cpu(hdr->args.stateid.seqid); + __entry->stateid_hash = + nfs_stateid_hash(&hdr->args.stateid); + __assign_str(dstaddr); + ), + + TP_printk( + "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " + "offset=%llu count=%u stateid=%d:0x%08x dstaddr=%s " + "nfs_error=%lu (%s)", + -__entry->error, + show_nfs4_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + __entry->offset, __entry->count, + __entry->stateid_seq, __entry->stateid_hash, + __get_str(dstaddr), __entry->nfs_error, + show_nfs4_status(__entry->nfs_error) + ) +); + +#define DEFINE_NFS4_FLEXFILES_IO_EVENT(name) \ + DEFINE_EVENT(nfs4_flexfiles_io_event, name, \ + TP_PROTO( \ + const struct nfs_pgio_header *hdr, \ + int error \ + ), \ + TP_ARGS(hdr, error)) +DEFINE_NFS4_FLEXFILES_IO_EVENT(ff_layout_read_error); +DEFINE_NFS4_FLEXFILES_IO_EVENT(ff_layout_write_error); + +TRACE_EVENT(ff_layout_commit_error, + TP_PROTO( + const struct nfs_commit_data *data, + int error + ), + + TP_ARGS(data, error), + + TP_STRUCT__entry( + __field(unsigned long, error) + __field(unsigned long, nfs_error) + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(loff_t, offset) + __field(u32, count) + __string(dstaddr, data->ds_clp ? + rpc_peeraddr2str(data->ds_clp->cl_rpcclient, + RPC_DISPLAY_ADDR) : "unknown") + ), + + TP_fast_assign( + const struct inode *inode = data->inode; + + __entry->error = -error; + __entry->nfs_error = data->res.op_status; + __entry->fhandle = nfs_fhandle_hash(data->args.fh); + __entry->fileid = inode->i_ino; + __entry->dev = inode->i_sb->s_dev; + __entry->offset = data->args.offset; + __entry->count = data->args.count; + __assign_str(dstaddr); + ), + + TP_printk( + "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " + "offset=%llu count=%u dstaddr=%s nfs_error=%lu (%s)", + -__entry->error, + show_nfs4_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + __entry->offset, __entry->count, + __get_str(dstaddr), __entry->nfs_error, + show_nfs4_status(__entry->nfs_error) + ) +); + +TRACE_EVENT(bl_ext_tree_prepare_commit, + TP_PROTO( + int ret, + size_t count, + u64 lwb, + bool not_all_ranges + ), + + TP_ARGS(ret, count, lwb, not_all_ranges), + + TP_STRUCT__entry( + __field(int, ret) + __field(size_t, count) + __field(u64, lwb) + __field(bool, not_all_ranges) + ), + + TP_fast_assign( + __entry->ret = ret; + __entry->count = count; + __entry->lwb = lwb; + __entry->not_all_ranges = not_all_ranges; + ), + + TP_printk( + "ret=%d, found %zu ranges, lwb=%llu%s", + __entry->ret, + __entry->count, + __entry->lwb, + __entry->not_all_ranges ? ", not all ranges encoded" : + "" + ) +); + +DECLARE_EVENT_CLASS(pnfs_bl_pr_key_class, + TP_PROTO( + const struct block_device *bdev, + u64 key + ), + TP_ARGS(bdev, key), + TP_STRUCT__entry( + __field(u64, key) + __field(dev_t, dev) + __string(device, bdev->bd_disk->disk_name) + ), + TP_fast_assign( + __entry->key = key; + __entry->dev = bdev->bd_dev; + __assign_str(device); + ), + TP_printk("dev=%d,%d (%s) key=0x%016llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __get_str(device), __entry->key + ) +); + +#define DEFINE_NFS4_BLOCK_PRKEY_EVENT(name) \ + DEFINE_EVENT(pnfs_bl_pr_key_class, name, \ + TP_PROTO( \ + const struct block_device *bdev, \ + u64 key \ + ), \ + TP_ARGS(bdev, key)) +DEFINE_NFS4_BLOCK_PRKEY_EVENT(bl_pr_key_reg); +DEFINE_NFS4_BLOCK_PRKEY_EVENT(bl_pr_key_unreg); + +/* + * From uapi/linux/pr.h + */ +TRACE_DEFINE_ENUM(PR_STS_SUCCESS); +TRACE_DEFINE_ENUM(PR_STS_IOERR); +TRACE_DEFINE_ENUM(PR_STS_RESERVATION_CONFLICT); +TRACE_DEFINE_ENUM(PR_STS_RETRY_PATH_FAILURE); +TRACE_DEFINE_ENUM(PR_STS_PATH_FAST_FAILED); +TRACE_DEFINE_ENUM(PR_STS_PATH_FAILED); + +#define show_pr_status(x) \ + __print_symbolic(x, \ + { PR_STS_SUCCESS, "SUCCESS" }, \ + { PR_STS_IOERR, "IOERR" }, \ + { PR_STS_RESERVATION_CONFLICT, "RESERVATION_CONFLICT" }, \ + { PR_STS_RETRY_PATH_FAILURE, "RETRY_PATH_FAILURE" }, \ + { PR_STS_PATH_FAST_FAILED, "PATH_FAST_FAILED" }, \ + { PR_STS_PATH_FAILED, "PATH_FAILED" }) + +DECLARE_EVENT_CLASS(pnfs_bl_pr_key_err_class, + TP_PROTO( + const struct block_device *bdev, + u64 key, + int status + ), + TP_ARGS(bdev, key, status), + TP_STRUCT__entry( + __field(u64, key) + __field(dev_t, dev) + __field(unsigned long, status) + __string(device, bdev->bd_disk->disk_name) + ), + TP_fast_assign( + __entry->key = key; + __entry->dev = bdev->bd_dev; + __entry->status = status; + __assign_str(device); + ), + TP_printk("dev=%d,%d (%s) key=0x%016llx status=%s", + MAJOR(__entry->dev), MINOR(__entry->dev), + __get_str(device), __entry->key, + show_pr_status(__entry->status) + ) +); + +#define DEFINE_NFS4_BLOCK_PRKEY_ERR_EVENT(name) \ + DEFINE_EVENT(pnfs_bl_pr_key_err_class, name, \ + TP_PROTO( \ + const struct block_device *bdev, \ + u64 key, \ + int status \ + ), \ + TP_ARGS(bdev, key, status)) +DEFINE_NFS4_BLOCK_PRKEY_ERR_EVENT(bl_pr_key_reg_err); +DEFINE_NFS4_BLOCK_PRKEY_ERR_EVENT(bl_pr_key_unreg_err); + +#ifdef CONFIG_NFS_V4_2 +TRACE_DEFINE_ENUM(NFS4_CONTENT_DATA); +TRACE_DEFINE_ENUM(NFS4_CONTENT_HOLE); + +#define show_llseek_mode(what) \ + __print_symbolic(what, \ + { NFS4_CONTENT_DATA, "DATA" }, \ + { NFS4_CONTENT_HOLE, "HOLE" }) + +TRACE_EVENT(nfs4_llseek, + TP_PROTO( + const struct inode *inode, + const struct nfs42_seek_args *args, + const struct nfs42_seek_res *res, + int error + ), + + TP_ARGS(inode, args, res, error), + + TP_STRUCT__entry( + __field(unsigned long, error) + __field(u32, fhandle) + __field(u64, fileid) + __field(dev_t, dev) + __field(int, stateid_seq) + __field(u32, stateid_hash) + __field(loff_t, offset_s) + __field(u32, what) + __field(loff_t, offset_r) + __field(u32, eof) + ), + + TP_fast_assign( + const struct nfs_fh *fh = args->sa_fh; + + __entry->fileid = inode->i_ino; + __entry->dev = inode->i_sb->s_dev; + __entry->fhandle = nfs_fhandle_hash(fh); + __entry->offset_s = args->sa_offset; + __entry->stateid_seq = + be32_to_cpu(args->sa_stateid.seqid); + __entry->stateid_hash = + nfs_stateid_hash(&args->sa_stateid); + __entry->what = args->sa_what; + if (error) { + __entry->error = -error; + __entry->offset_r = 0; + __entry->eof = 0; + } else { + __entry->error = 0; + __entry->offset_r = res->sr_offset; + __entry->eof = res->sr_eof; + } + ), + + TP_printk( + "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " + "stateid=%d:0x%08x offset_s=%llu what=%s " + "offset_r=%llu eof=%u", + -__entry->error, + show_nfs4_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + __entry->stateid_seq, __entry->stateid_hash, + __entry->offset_s, + show_llseek_mode(__entry->what), + __entry->offset_r, + __entry->eof + ) +); + +DECLARE_EVENT_CLASS(nfs4_sparse_event, + TP_PROTO( + const struct inode *inode, + const struct nfs42_falloc_args *args, + int error + ), + + TP_ARGS(inode, args, error), + + TP_STRUCT__entry( + __field(unsigned long, error) + __field(loff_t, offset) + __field(loff_t, len) + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(int, stateid_seq) + __field(u32, stateid_hash) + ), + + TP_fast_assign( + __entry->error = error < 0 ? -error : 0; + __entry->offset = args->falloc_offset; + __entry->len = args->falloc_length; + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); + __entry->stateid_seq = + be32_to_cpu(args->falloc_stateid.seqid); + __entry->stateid_hash = + nfs_stateid_hash(&args->falloc_stateid); + ), + + TP_printk( + "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " + "stateid=%d:0x%08x offset=%llu len=%llu", + -__entry->error, + show_nfs4_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + __entry->stateid_seq, __entry->stateid_hash, + (long long)__entry->offset, + (long long)__entry->len + ) +); +#define DEFINE_NFS4_SPARSE_EVENT(name) \ + DEFINE_EVENT(nfs4_sparse_event, name, \ + TP_PROTO( \ + const struct inode *inode, \ + const struct nfs42_falloc_args *args, \ + int error \ + ), \ + TP_ARGS(inode, args, error)) +DEFINE_NFS4_SPARSE_EVENT(nfs4_fallocate); +DEFINE_NFS4_SPARSE_EVENT(nfs4_deallocate); + +TRACE_EVENT(nfs4_copy, + TP_PROTO( + const struct inode *src_inode, + const struct inode *dst_inode, + const struct nfs42_copy_args *args, + const struct nfs42_copy_res *res, + const struct nl4_server *nss, + int error + ), + + TP_ARGS(src_inode, dst_inode, args, res, nss, error), + + TP_STRUCT__entry( + __field(unsigned long, error) + __field(u32, src_fhandle) + __field(u32, src_fileid) + __field(u32, dst_fhandle) + __field(u32, dst_fileid) + __field(dev_t, src_dev) + __field(dev_t, dst_dev) + __field(int, src_stateid_seq) + __field(u32, src_stateid_hash) + __field(int, dst_stateid_seq) + __field(u32, dst_stateid_hash) + __field(loff_t, src_offset) + __field(loff_t, dst_offset) + __field(bool, sync) + __field(loff_t, len) + __field(int, res_stateid_seq) + __field(u32, res_stateid_hash) + __field(loff_t, res_count) + __field(bool, res_sync) + __field(bool, res_cons) + __field(bool, intra) + ), + + TP_fast_assign( + __entry->src_fileid = src_inode->i_ino; + __entry->src_dev = src_inode->i_sb->s_dev; + __entry->src_fhandle = nfs_fhandle_hash(args->src_fh); + __entry->src_offset = args->src_pos; + __entry->dst_fileid = dst_inode->i_ino; + __entry->dst_dev = dst_inode->i_sb->s_dev; + __entry->dst_fhandle = nfs_fhandle_hash(args->dst_fh); + __entry->dst_offset = args->dst_pos; + __entry->len = args->count; + __entry->sync = args->sync; + __entry->src_stateid_seq = + be32_to_cpu(args->src_stateid.seqid); + __entry->src_stateid_hash = + nfs_stateid_hash(&args->src_stateid); + __entry->dst_stateid_seq = + be32_to_cpu(args->dst_stateid.seqid); + __entry->dst_stateid_hash = + nfs_stateid_hash(&args->dst_stateid); + __entry->intra = nss ? 0 : 1; + if (error) { + __entry->error = -error; + __entry->res_stateid_seq = 0; + __entry->res_stateid_hash = 0; + __entry->res_count = 0; + __entry->res_sync = 0; + __entry->res_cons = 0; + } else { + __entry->error = 0; + __entry->res_stateid_seq = + be32_to_cpu(res->write_res.stateid.seqid); + __entry->res_stateid_hash = + nfs_stateid_hash(&res->write_res.stateid); + __entry->res_count = res->write_res.count; + __entry->res_sync = res->synchronous; + __entry->res_cons = res->consecutive; + } + ), + + TP_printk( + "error=%ld (%s) intra=%d src_fileid=%02x:%02x:%llu " + "src_fhandle=0x%08x dst_fileid=%02x:%02x:%llu " + "dst_fhandle=0x%08x src_stateid=%d:0x%08x " + "dst_stateid=%d:0x%08x src_offset=%llu dst_offset=%llu " + "len=%llu sync=%d cb_stateid=%d:0x%08x res_sync=%d " + "res_cons=%d res_count=%llu", + -__entry->error, + show_nfs4_status(__entry->error), + __entry->intra, + MAJOR(__entry->src_dev), MINOR(__entry->src_dev), + (unsigned long long)__entry->src_fileid, + __entry->src_fhandle, + MAJOR(__entry->dst_dev), MINOR(__entry->dst_dev), + (unsigned long long)__entry->dst_fileid, + __entry->dst_fhandle, + __entry->src_stateid_seq, __entry->src_stateid_hash, + __entry->dst_stateid_seq, __entry->dst_stateid_hash, + __entry->src_offset, + __entry->dst_offset, + __entry->len, + __entry->sync, + __entry->res_stateid_seq, __entry->res_stateid_hash, + __entry->res_sync, + __entry->res_cons, + __entry->res_count + ) +); + +TRACE_EVENT(nfs4_clone, + TP_PROTO( + const struct inode *src_inode, + const struct inode *dst_inode, + const struct nfs42_clone_args *args, + int error + ), + + TP_ARGS(src_inode, dst_inode, args, error), + + TP_STRUCT__entry( + __field(unsigned long, error) + __field(u32, src_fhandle) + __field(u32, src_fileid) + __field(u32, dst_fhandle) + __field(u32, dst_fileid) + __field(dev_t, src_dev) + __field(dev_t, dst_dev) + __field(loff_t, src_offset) + __field(loff_t, dst_offset) + __field(int, src_stateid_seq) + __field(u32, src_stateid_hash) + __field(int, dst_stateid_seq) + __field(u32, dst_stateid_hash) + __field(loff_t, len) + ), + + TP_fast_assign( + __entry->src_fileid = src_inode->i_ino; + __entry->src_dev = src_inode->i_sb->s_dev; + __entry->src_fhandle = nfs_fhandle_hash(args->src_fh); + __entry->src_offset = args->src_offset; + __entry->dst_fileid = dst_inode->i_ino; + __entry->dst_dev = dst_inode->i_sb->s_dev; + __entry->dst_fhandle = nfs_fhandle_hash(args->dst_fh); + __entry->dst_offset = args->dst_offset; + __entry->len = args->count; + __entry->error = error < 0 ? -error : 0; + __entry->src_stateid_seq = + be32_to_cpu(args->src_stateid.seqid); + __entry->src_stateid_hash = + nfs_stateid_hash(&args->src_stateid); + __entry->dst_stateid_seq = + be32_to_cpu(args->dst_stateid.seqid); + __entry->dst_stateid_hash = + nfs_stateid_hash(&args->dst_stateid); + ), + + TP_printk( + "error=%ld (%s) src_fileid=%02x:%02x:%llu " + "src_fhandle=0x%08x dst_fileid=%02x:%02x:%llu " + "dst_fhandle=0x%08x src_stateid=%d:0x%08x " + "dst_stateid=%d:0x%08x src_offset=%llu " + "dst_offset=%llu len=%llu", + -__entry->error, + show_nfs4_status(__entry->error), + MAJOR(__entry->src_dev), MINOR(__entry->src_dev), + (unsigned long long)__entry->src_fileid, + __entry->src_fhandle, + MAJOR(__entry->dst_dev), MINOR(__entry->dst_dev), + (unsigned long long)__entry->dst_fileid, + __entry->dst_fhandle, + __entry->src_stateid_seq, __entry->src_stateid_hash, + __entry->dst_stateid_seq, __entry->dst_stateid_hash, + __entry->src_offset, + __entry->dst_offset, + __entry->len + ) +); + +TRACE_EVENT(nfs4_copy_notify, + TP_PROTO( + const struct inode *inode, + const struct nfs42_copy_notify_args *args, + const struct nfs42_copy_notify_res *res, + int error + ), + + TP_ARGS(inode, args, res, error), + + TP_STRUCT__entry( + __field(unsigned long, error) + __field(u32, fhandle) + __field(u64, fileid) + __field(dev_t, dev) + __field(int, stateid_seq) + __field(u32, stateid_hash) + __field(int, res_stateid_seq) + __field(u32, res_stateid_hash) + ), + + TP_fast_assign( + __entry->fileid = inode->i_ino; + __entry->dev = inode->i_sb->s_dev; + __entry->fhandle = nfs_fhandle_hash(args->cna_src_fh); + __entry->stateid_seq = + be32_to_cpu(args->cna_src_stateid.seqid); + __entry->stateid_hash = + nfs_stateid_hash(&args->cna_src_stateid); + if (error) { + __entry->error = -error; + __entry->res_stateid_seq = 0; + __entry->res_stateid_hash = 0; + } else { + __entry->error = 0; + __entry->res_stateid_seq = + be32_to_cpu(res->cnr_stateid.seqid); + __entry->res_stateid_hash = + nfs_stateid_hash(&res->cnr_stateid); + } + ), + + TP_printk( + "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " + "stateid=%d:0x%08x res_stateid=%d:0x%08x", + -__entry->error, + show_nfs4_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + __entry->stateid_seq, __entry->stateid_hash, + __entry->res_stateid_seq, __entry->res_stateid_hash + ) +); + +DECLARE_EVENT_CLASS(nfs4_offload_class, + TP_PROTO( + const struct nfs42_offload_status_args *args, + int error + ), + + TP_ARGS(args, error), + + TP_STRUCT__entry( + __field(unsigned long, error) + __field(u32, fhandle) + __field(int, stateid_seq) + __field(u32, stateid_hash) + ), + + TP_fast_assign( + __entry->fhandle = nfs_fhandle_hash(args->osa_src_fh); + __entry->error = error < 0 ? -error : 0; + __entry->stateid_seq = + be32_to_cpu(args->osa_stateid.seqid); + __entry->stateid_hash = + nfs_stateid_hash(&args->osa_stateid); + ), + + TP_printk( + "error=%ld (%s) fhandle=0x%08x stateid=%d:0x%08x", + -__entry->error, + show_nfs4_status(__entry->error), + __entry->fhandle, + __entry->stateid_seq, __entry->stateid_hash + ) +); +#define DEFINE_NFS4_OFFLOAD_EVENT(name) \ + DEFINE_EVENT(nfs4_offload_class, name, \ + TP_PROTO( \ + const struct nfs42_offload_status_args *args, \ + int error \ + ), \ + TP_ARGS(args, error)) +DEFINE_NFS4_OFFLOAD_EVENT(nfs4_offload_cancel); +DEFINE_NFS4_OFFLOAD_EVENT(nfs4_offload_status); + +DECLARE_EVENT_CLASS(nfs4_xattr_event, + TP_PROTO( + const struct inode *inode, + const char *name, + int error + ), + + TP_ARGS(inode, name, error), + + TP_STRUCT__entry( + __field(unsigned long, error) + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __string(name, name) + ), + + TP_fast_assign( + __entry->error = error < 0 ? -error : 0; + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(NFS_FH(inode)); + __assign_str(name); + ), + + TP_printk( + "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " + "name=%s", + -__entry->error, show_nfs4_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, __get_str(name) + ) +); +#define DEFINE_NFS4_XATTR_EVENT(name) \ + DEFINE_EVENT(nfs4_xattr_event, name, \ + TP_PROTO( \ + const struct inode *inode, \ + const char *name, \ + int error \ + ), \ + TP_ARGS(inode, name, error)) +DEFINE_NFS4_XATTR_EVENT(nfs4_getxattr); +DEFINE_NFS4_XATTR_EVENT(nfs4_setxattr); +DEFINE_NFS4_XATTR_EVENT(nfs4_removexattr); + +DEFINE_NFS4_INODE_EVENT(nfs4_listxattr); +#endif /* CONFIG_NFS_V4_2 */ + +#endif /* _TRACE_NFS4_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE nfs4trace +/* This part must be outside protection */ +#include diff --git a/fs/nfs/nfs4xdr.c b/fs/nfs/nfs4xdr.c new file mode 100644 index 000000000..c23c2eee1 --- /dev/null +++ b/fs/nfs/nfs4xdr.c @@ -0,0 +1,7786 @@ +/* + * fs/nfs/nfs4xdr.c + * + * Client-side XDR for NFSv4. + * + * Copyright (c) 2002 The Regents of the University of Michigan. + * All rights reserved. + * + * Kendrick Smith + * Andy Adamson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nfs4_fs.h" +#include "nfs4trace.h" +#include "internal.h" +#include "nfs4idmap.h" +#include "nfs4session.h" +#include "pnfs.h" +#include "netns.h" + +#define NFSDBG_FACILITY NFSDBG_XDR + +struct compound_hdr; +static void encode_layoutget(struct xdr_stream *xdr, + const struct nfs4_layoutget_args *args, + struct compound_hdr *hdr); +static int decode_layoutget(struct xdr_stream *xdr, struct rpc_rqst *req, + struct nfs4_layoutget_res *res); + +/* NFSv4 COMPOUND tags are only wanted for debugging purposes */ +#ifdef DEBUG +#define NFS4_MAXTAGLEN 20 +#else +#define NFS4_MAXTAGLEN 0 +#endif + +/* lock,open owner id: + * we currently use size 2 (u64) out of (NFS4_OPAQUE_LIMIT >> 2) + */ +#define pagepad_maxsz (1) +#define open_owner_id_maxsz (2 + 1 + 2 + 2) +#define lock_owner_id_maxsz (2 + 1 + 2) +#define compound_encode_hdr_maxsz (3 + (NFS4_MAXTAGLEN >> 2)) +#define compound_decode_hdr_maxsz (3 + (NFS4_MAXTAGLEN >> 2)) +#define op_encode_hdr_maxsz (1) +#define op_decode_hdr_maxsz (2) +#define encode_stateid_maxsz (XDR_QUADLEN(NFS4_STATEID_SIZE)) +#define decode_stateid_maxsz (XDR_QUADLEN(NFS4_STATEID_SIZE)) +#define encode_verifier_maxsz (XDR_QUADLEN(NFS4_VERIFIER_SIZE)) +#define decode_verifier_maxsz (XDR_QUADLEN(NFS4_VERIFIER_SIZE)) +#define encode_putfh_maxsz (op_encode_hdr_maxsz + 1 + \ + (NFS4_FHSIZE >> 2)) +#define decode_putfh_maxsz (op_decode_hdr_maxsz) +#define encode_putrootfh_maxsz (op_encode_hdr_maxsz) +#define decode_putrootfh_maxsz (op_decode_hdr_maxsz) +#define encode_getfh_maxsz (op_encode_hdr_maxsz) +#define decode_getfh_maxsz (op_decode_hdr_maxsz + 1 + \ + ((3+NFS4_FHSIZE) >> 2)) +#define nfs4_fattr_bitmap_maxsz 4 +#define encode_getattr_maxsz (op_encode_hdr_maxsz + nfs4_fattr_bitmap_maxsz) +#define nfstime4_maxsz (3) +#define nfs4_name_maxsz (1 + ((3 + NFS4_MAXNAMLEN) >> 2)) +#define nfs4_path_maxsz (1 + ((3 + NFS4_MAXPATHLEN) >> 2)) +#define nfs4_owner_maxsz (1 + XDR_QUADLEN(IDMAP_NAMESZ)) +#define nfs4_group_maxsz (1 + XDR_QUADLEN(IDMAP_NAMESZ)) +#ifdef CONFIG_NFS_V4_SECURITY_LABEL +/* PI(4 bytes) + LFS(4 bytes) + 1(for null terminator?) + MAXLABELLEN */ +#define nfs4_label_maxsz (4 + 4 + 1 + XDR_QUADLEN(NFS4_MAXLABELLEN)) +#else +#define nfs4_label_maxsz 0 +#endif +/* We support only one layout type per file system */ +#define decode_mdsthreshold_maxsz (1 + 1 + nfs4_fattr_bitmap_maxsz + 1 + 8) +/* This is based on getfattr, which uses the most attributes: */ +#define nfs4_fattr_value_maxsz (1 + (1 + 2 + 2 + 4 + 2 + 1 + 1 + 2 + 2 + \ + 3*nfstime4_maxsz + \ + nfs4_owner_maxsz + \ + nfs4_group_maxsz + nfs4_label_maxsz + \ + decode_mdsthreshold_maxsz)) +#define nfs4_fattr_maxsz (nfs4_fattr_bitmap_maxsz + \ + nfs4_fattr_value_maxsz) +#define decode_getattr_maxsz (op_decode_hdr_maxsz + nfs4_fattr_maxsz) +#define encode_attrs_maxsz (nfs4_fattr_bitmap_maxsz + \ + 1 + 2 + 1 + \ + nfs4_owner_maxsz + \ + nfs4_group_maxsz + \ + nfs4_label_maxsz + \ + 1 + nfstime4_maxsz + \ + 1 + nfstime4_maxsz) +#define encode_savefh_maxsz (op_encode_hdr_maxsz) +#define decode_savefh_maxsz (op_decode_hdr_maxsz) +#define encode_restorefh_maxsz (op_encode_hdr_maxsz) +#define decode_restorefh_maxsz (op_decode_hdr_maxsz) +#define encode_fsinfo_maxsz (encode_getattr_maxsz) +/* The 5 accounts for the PNFS attributes, and assumes that at most three + * layout types will be returned. + */ +#define decode_fsinfo_maxsz (op_decode_hdr_maxsz + \ + nfs4_fattr_bitmap_maxsz + 1 + \ + 1 /* lease time */ + \ + 2 /* max filesize */ + \ + 2 /* max read */ + \ + 2 /* max write */ + \ + nfstime4_maxsz /* time delta */ + \ + 5 /* fs layout types */ + \ + 1 /* layout blksize */ + \ + 1 /* clone blksize */ + \ + 1 /* change attr type */ + \ + 1 /* xattr support */) +#define encode_renew_maxsz (op_encode_hdr_maxsz + 3) +#define decode_renew_maxsz (op_decode_hdr_maxsz) +#define encode_setclientid_maxsz \ + (op_encode_hdr_maxsz + \ + XDR_QUADLEN(NFS4_VERIFIER_SIZE) + \ + /* client name */ \ + 1 + XDR_QUADLEN(NFS4_OPAQUE_LIMIT) + \ + 1 /* sc_prog */ + \ + 1 + XDR_QUADLEN(RPCBIND_MAXNETIDLEN) + \ + 1 + XDR_QUADLEN(RPCBIND_MAXUADDRLEN) + \ + 1) /* sc_cb_ident */ +#define decode_setclientid_maxsz \ + (op_decode_hdr_maxsz + \ + 2 /* clientid */ + \ + XDR_QUADLEN(NFS4_VERIFIER_SIZE) + \ + 1 + XDR_QUADLEN(RPCBIND_MAXNETIDLEN) + \ + 1 + XDR_QUADLEN(RPCBIND_MAXUADDRLEN)) +#define encode_setclientid_confirm_maxsz \ + (op_encode_hdr_maxsz + \ + 3 + (NFS4_VERIFIER_SIZE >> 2)) +#define decode_setclientid_confirm_maxsz \ + (op_decode_hdr_maxsz) +#define encode_lookup_maxsz (op_encode_hdr_maxsz + nfs4_name_maxsz) +#define decode_lookup_maxsz (op_decode_hdr_maxsz) +#define encode_lookupp_maxsz (op_encode_hdr_maxsz) +#define decode_lookupp_maxsz (op_decode_hdr_maxsz) +#define encode_share_access_maxsz \ + (2) +#define encode_createmode_maxsz (1 + encode_attrs_maxsz + encode_verifier_maxsz) +#define encode_opentype_maxsz (1 + encode_createmode_maxsz) +#define encode_claim_null_maxsz (1 + nfs4_name_maxsz) +#define encode_open_maxsz (op_encode_hdr_maxsz + \ + 2 + encode_share_access_maxsz + 2 + \ + 1 + open_owner_id_maxsz + \ + encode_opentype_maxsz + \ + encode_claim_null_maxsz) +#define decode_space_limit_maxsz (3) +#define decode_ace_maxsz (3 + nfs4_owner_maxsz) +#define decode_delegation_maxsz (1 + decode_stateid_maxsz + 1 + \ + decode_space_limit_maxsz + \ + decode_ace_maxsz) +#define decode_change_info_maxsz (5) +#define decode_open_maxsz (op_decode_hdr_maxsz + \ + decode_stateid_maxsz + \ + decode_change_info_maxsz + 1 + \ + nfs4_fattr_bitmap_maxsz + \ + decode_delegation_maxsz) +#define encode_open_confirm_maxsz \ + (op_encode_hdr_maxsz + \ + encode_stateid_maxsz + 1) +#define decode_open_confirm_maxsz \ + (op_decode_hdr_maxsz + \ + decode_stateid_maxsz) +#define encode_open_downgrade_maxsz \ + (op_encode_hdr_maxsz + \ + encode_stateid_maxsz + 1 + \ + encode_share_access_maxsz) +#define decode_open_downgrade_maxsz \ + (op_decode_hdr_maxsz + \ + decode_stateid_maxsz) +#define encode_close_maxsz (op_encode_hdr_maxsz + \ + 1 + encode_stateid_maxsz) +#define decode_close_maxsz (op_decode_hdr_maxsz + \ + decode_stateid_maxsz) +#define encode_setattr_maxsz (op_encode_hdr_maxsz + \ + encode_stateid_maxsz + \ + encode_attrs_maxsz) +#define decode_setattr_maxsz (op_decode_hdr_maxsz + \ + nfs4_fattr_bitmap_maxsz) +#define encode_delegattr_maxsz (op_encode_hdr_maxsz + \ + encode_stateid_maxsz + \ + nfs4_fattr_bitmap_maxsz + \ + 2*nfstime4_maxsz) +#define decode_delegattr_maxsz (decode_setattr_maxsz) +#define encode_read_maxsz (op_encode_hdr_maxsz + \ + encode_stateid_maxsz + 3) +#define decode_read_maxsz (op_decode_hdr_maxsz + 2 + pagepad_maxsz) +#define encode_readdir_maxsz (op_encode_hdr_maxsz + \ + 2 + encode_verifier_maxsz + 5 + \ + nfs4_label_maxsz) +#define decode_readdir_maxsz (op_decode_hdr_maxsz + \ + decode_verifier_maxsz + pagepad_maxsz) +#define encode_readlink_maxsz (op_encode_hdr_maxsz) +#define decode_readlink_maxsz (op_decode_hdr_maxsz + 1 + pagepad_maxsz) +#define encode_write_maxsz (op_encode_hdr_maxsz + \ + encode_stateid_maxsz + 4) +#define decode_write_maxsz (op_decode_hdr_maxsz + \ + 2 + decode_verifier_maxsz) +#define encode_commit_maxsz (op_encode_hdr_maxsz + 3) +#define decode_commit_maxsz (op_decode_hdr_maxsz + \ + decode_verifier_maxsz) +#define encode_remove_maxsz (op_encode_hdr_maxsz + \ + nfs4_name_maxsz) +#define decode_remove_maxsz (op_decode_hdr_maxsz + \ + decode_change_info_maxsz) +#define encode_rename_maxsz (op_encode_hdr_maxsz + \ + 2 * nfs4_name_maxsz) +#define decode_rename_maxsz (op_decode_hdr_maxsz + \ + decode_change_info_maxsz + \ + decode_change_info_maxsz) +#define encode_link_maxsz (op_encode_hdr_maxsz + \ + nfs4_name_maxsz) +#define decode_link_maxsz (op_decode_hdr_maxsz + decode_change_info_maxsz) +#define encode_lockowner_maxsz (2 + 1 + lock_owner_id_maxsz) + +#define encode_lock_maxsz (op_encode_hdr_maxsz + \ + 7 + \ + 1 + encode_stateid_maxsz + 1 + \ + encode_lockowner_maxsz) +#define decode_lock_denied_maxsz \ + (2 + 2 + 1 + 2 + 1 + lock_owner_id_maxsz) +#define decode_lock_maxsz (op_decode_hdr_maxsz + \ + decode_lock_denied_maxsz) +#define encode_lockt_maxsz (op_encode_hdr_maxsz + 5 + \ + encode_lockowner_maxsz) +#define decode_lockt_maxsz (op_decode_hdr_maxsz + \ + decode_lock_denied_maxsz) +#define encode_locku_maxsz (op_encode_hdr_maxsz + 3 + \ + encode_stateid_maxsz + \ + 4) +#define decode_locku_maxsz (op_decode_hdr_maxsz + \ + decode_stateid_maxsz) +#define encode_release_lockowner_maxsz \ + (op_encode_hdr_maxsz + \ + encode_lockowner_maxsz) +#define decode_release_lockowner_maxsz \ + (op_decode_hdr_maxsz) +#define encode_access_maxsz (op_encode_hdr_maxsz + 1) +#define decode_access_maxsz (op_decode_hdr_maxsz + 2) +#define encode_symlink_maxsz (op_encode_hdr_maxsz + \ + 1 + nfs4_name_maxsz + \ + 1 + \ + nfs4_fattr_maxsz) +#define decode_symlink_maxsz (op_decode_hdr_maxsz + 8) +#define encode_create_maxsz (op_encode_hdr_maxsz + \ + 1 + 2 + nfs4_name_maxsz + \ + encode_attrs_maxsz) +#define decode_create_maxsz (op_decode_hdr_maxsz + \ + decode_change_info_maxsz + \ + nfs4_fattr_bitmap_maxsz) +#define encode_statfs_maxsz (encode_getattr_maxsz) +#define decode_statfs_maxsz (decode_getattr_maxsz) +#define encode_delegreturn_maxsz (op_encode_hdr_maxsz + 4) +#define decode_delegreturn_maxsz (op_decode_hdr_maxsz) +#define encode_getacl_maxsz (encode_getattr_maxsz) +#define decode_getacl_maxsz (op_decode_hdr_maxsz + \ + nfs4_fattr_bitmap_maxsz + 1 + pagepad_maxsz) +#define encode_setacl_maxsz (op_encode_hdr_maxsz + \ + encode_stateid_maxsz + 3) +#define decode_setacl_maxsz (decode_setattr_maxsz) +#define encode_fs_locations_maxsz \ + (encode_getattr_maxsz) +#define decode_fs_locations_maxsz \ + (pagepad_maxsz) +#define encode_secinfo_maxsz (op_encode_hdr_maxsz + nfs4_name_maxsz) +#define decode_secinfo_maxsz (op_decode_hdr_maxsz + 1 + ((NFS_MAX_SECFLAVORS * (16 + GSS_OID_MAX_LEN)) / 4)) + +#define NFS4_MAX_MACHINE_NAME_LEN (64) +#define IMPL_NAME_LIMIT (sizeof(utsname()->sysname) + sizeof(utsname()->release) + \ + sizeof(utsname()->version) + sizeof(utsname()->machine) + 8) + +#define encode_exchange_id_maxsz (op_encode_hdr_maxsz + \ + encode_verifier_maxsz + \ + 1 /* co_ownerid.len */ + \ + /* eia_clientowner */ \ + 1 + XDR_QUADLEN(NFS4_OPAQUE_LIMIT) + \ + 1 /* flags */ + \ + 1 /* spa_how */ + \ + /* max is SP4_MACH_CRED (for now) */ + \ + 1 + NFS4_OP_MAP_NUM_WORDS + \ + 1 + NFS4_OP_MAP_NUM_WORDS + \ + 1 /* implementation id array of size 1 */ + \ + 1 /* nii_domain */ + \ + XDR_QUADLEN(NFS4_OPAQUE_LIMIT) + \ + 1 /* nii_name */ + \ + XDR_QUADLEN(IMPL_NAME_LIMIT) + \ + 3 /* nii_date */) +#define decode_exchange_id_maxsz (op_decode_hdr_maxsz + \ + 2 /* eir_clientid */ + \ + 1 /* eir_sequenceid */ + \ + 1 /* eir_flags */ + \ + 1 /* spr_how */ + \ + /* max is SP4_MACH_CRED (for now) */ + \ + 1 + NFS4_OP_MAP_NUM_WORDS + \ + 1 + NFS4_OP_MAP_NUM_WORDS + \ + 2 /* eir_server_owner.so_minor_id */ + \ + /* eir_server_owner.so_major_id<> */ \ + XDR_QUADLEN(NFS4_OPAQUE_LIMIT) + 1 + \ + /* eir_server_scope<> */ \ + XDR_QUADLEN(NFS4_OPAQUE_LIMIT) + 1 + \ + 1 /* eir_server_impl_id array length */ + \ + 1 /* nii_domain */ + \ + XDR_QUADLEN(NFS4_OPAQUE_LIMIT) + \ + 1 /* nii_name */ + \ + XDR_QUADLEN(NFS4_OPAQUE_LIMIT) + \ + 3 /* nii_date */) +#define encode_channel_attrs_maxsz (6 + 1 /* ca_rdma_ird.len (0) */) +#define decode_channel_attrs_maxsz (6 + \ + 1 /* ca_rdma_ird.len */ + \ + 1 /* ca_rdma_ird */) +#define encode_create_session_maxsz (op_encode_hdr_maxsz + \ + 2 /* csa_clientid */ + \ + 1 /* csa_sequence */ + \ + 1 /* csa_flags */ + \ + encode_channel_attrs_maxsz + \ + encode_channel_attrs_maxsz + \ + 1 /* csa_cb_program */ + \ + 1 /* csa_sec_parms.len (1) */ + \ + 1 /* cb_secflavor (AUTH_SYS) */ + \ + 1 /* stamp */ + \ + 1 /* machinename.len */ + \ + XDR_QUADLEN(NFS4_MAX_MACHINE_NAME_LEN) + \ + 1 /* uid */ + \ + 1 /* gid */ + \ + 1 /* gids.len (0) */) +#define decode_create_session_maxsz (op_decode_hdr_maxsz + \ + XDR_QUADLEN(NFS4_MAX_SESSIONID_LEN) + \ + 1 /* csr_sequence */ + \ + 1 /* csr_flags */ + \ + decode_channel_attrs_maxsz + \ + decode_channel_attrs_maxsz) +#define encode_bind_conn_to_session_maxsz (op_encode_hdr_maxsz + \ + /* bctsa_sessid */ \ + XDR_QUADLEN(NFS4_MAX_SESSIONID_LEN) + \ + 1 /* bctsa_dir */ + \ + 1 /* bctsa_use_conn_in_rdma_mode */) +#define decode_bind_conn_to_session_maxsz (op_decode_hdr_maxsz + \ + /* bctsr_sessid */ \ + XDR_QUADLEN(NFS4_MAX_SESSIONID_LEN) + \ + 1 /* bctsr_dir */ + \ + 1 /* bctsr_use_conn_in_rdma_mode */) +#define encode_destroy_session_maxsz (op_encode_hdr_maxsz + 4) +#define decode_destroy_session_maxsz (op_decode_hdr_maxsz) +#define encode_destroy_clientid_maxsz (op_encode_hdr_maxsz + 2) +#define decode_destroy_clientid_maxsz (op_decode_hdr_maxsz) +#define encode_sequence_maxsz (op_encode_hdr_maxsz + \ + XDR_QUADLEN(NFS4_MAX_SESSIONID_LEN) + 4) +#define decode_sequence_maxsz (op_decode_hdr_maxsz + \ + XDR_QUADLEN(NFS4_MAX_SESSIONID_LEN) + 5) +#define encode_reclaim_complete_maxsz (op_encode_hdr_maxsz + 4) +#define decode_reclaim_complete_maxsz (op_decode_hdr_maxsz + 4) +#define encode_get_dir_deleg_maxsz (op_encode_hdr_maxsz + \ + 4 /* gdda_signal_deleg_avail */ + \ + 8 /* gdda_notification_types */ + \ + nfstime4_maxsz /* gdda_child_attr_delay */ + \ + nfstime4_maxsz /* gdda_dir_attr_delay */ + \ + nfs4_fattr_bitmap_maxsz /* gdda_child_attributes */ + \ + nfs4_fattr_bitmap_maxsz /* gdda_dir_attributes */) +#define decode_get_dir_deleg_maxsz (op_decode_hdr_maxsz + \ + 4 /* gddrnf_status */ + \ + encode_verifier_maxsz /* gddr_cookieverf */ + \ + encode_stateid_maxsz /* gddr_stateid */ + \ + 8 /* gddr_notification */ + \ + nfs4_fattr_maxsz /* gddr_child_attributes */ + \ + nfs4_fattr_maxsz /* gddr_dir_attributes */) +#define encode_getdeviceinfo_maxsz (op_encode_hdr_maxsz + \ + XDR_QUADLEN(NFS4_DEVICEID4_SIZE) + \ + 1 /* layout type */ + \ + 1 /* maxcount */ + \ + 1 /* bitmap size */ + \ + 1 /* notification bitmap length */ + \ + 1 /* notification bitmap, word 0 */) +#define decode_getdeviceinfo_maxsz (op_decode_hdr_maxsz + \ + 1 /* layout type */ + \ + 1 /* opaque devaddr4 length */ + \ + /* devaddr4 payload is read into page */ \ + 1 /* notification bitmap length */ + \ + 1 /* notification bitmap, word 0 */ + \ + pagepad_maxsz /* possible XDR padding */) +#define encode_layoutget_maxsz (op_encode_hdr_maxsz + 10 + \ + encode_stateid_maxsz) +#define decode_layoutget_maxsz (op_decode_hdr_maxsz + 8 + \ + decode_stateid_maxsz + \ + XDR_QUADLEN(PNFS_LAYOUT_MAXSIZE) + \ + pagepad_maxsz) +#define encode_layoutcommit_maxsz (op_encode_hdr_maxsz + \ + 2 /* offset */ + \ + 2 /* length */ + \ + 1 /* reclaim */ + \ + encode_stateid_maxsz + \ + 1 /* new offset (true) */ + \ + 2 /* last byte written */ + \ + 1 /* nt_timechanged (false) */ + \ + 1 /* layoutupdate4 layout type */ + \ + 1 /* layoutupdate4 opaqueue len */) + /* the actual content of layoutupdate4 should + be allocated by drivers and spliced in + using xdr_write_pages */ +#define decode_layoutcommit_maxsz (op_decode_hdr_maxsz + 3) +#define encode_layoutreturn_maxsz (8 + op_encode_hdr_maxsz + \ + encode_stateid_maxsz + \ + 1 + \ + XDR_QUADLEN(NFS4_OPAQUE_LIMIT)) +#define decode_layoutreturn_maxsz (op_decode_hdr_maxsz + \ + 1 + decode_stateid_maxsz) +#define encode_secinfo_no_name_maxsz (op_encode_hdr_maxsz + 1) +#define decode_secinfo_no_name_maxsz decode_secinfo_maxsz +#define encode_test_stateid_maxsz (op_encode_hdr_maxsz + 2 + \ + XDR_QUADLEN(NFS4_STATEID_SIZE)) +#define decode_test_stateid_maxsz (op_decode_hdr_maxsz + 2 + 1) +#define encode_free_stateid_maxsz (op_encode_hdr_maxsz + 1 + \ + XDR_QUADLEN(NFS4_STATEID_SIZE)) +#define decode_free_stateid_maxsz (op_decode_hdr_maxsz) + +#define NFS4_enc_compound_sz (1024) /* XXX: large enough? */ +#define NFS4_dec_compound_sz (1024) /* XXX: large enough? */ +#define NFS4_enc_read_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_read_maxsz) +#define NFS4_dec_read_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_read_maxsz) +#define NFS4_enc_readlink_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_readlink_maxsz) +#define NFS4_dec_readlink_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_readlink_maxsz) +#define NFS4_enc_readdir_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_readdir_maxsz) +#define NFS4_dec_readdir_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_readdir_maxsz) +#define NFS4_enc_write_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_write_maxsz + \ + encode_getattr_maxsz) +#define NFS4_dec_write_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_write_maxsz + \ + decode_getattr_maxsz) +#define NFS4_enc_commit_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_commit_maxsz) +#define NFS4_dec_commit_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_commit_maxsz) +#define NFS4_enc_open_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_open_maxsz + \ + encode_access_maxsz + \ + encode_getfh_maxsz + \ + encode_getattr_maxsz + \ + encode_layoutget_maxsz) +#define NFS4_dec_open_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_open_maxsz + \ + decode_access_maxsz + \ + decode_getfh_maxsz + \ + decode_getattr_maxsz + \ + decode_layoutget_maxsz) +#define NFS4_enc_open_confirm_sz \ + (compound_encode_hdr_maxsz + \ + encode_putfh_maxsz + \ + encode_open_confirm_maxsz) +#define NFS4_dec_open_confirm_sz \ + (compound_decode_hdr_maxsz + \ + decode_putfh_maxsz + \ + decode_open_confirm_maxsz) +#define NFS4_enc_open_noattr_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_open_maxsz + \ + encode_access_maxsz + \ + encode_getattr_maxsz + \ + encode_layoutget_maxsz) +#define NFS4_dec_open_noattr_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_open_maxsz + \ + decode_access_maxsz + \ + decode_getattr_maxsz + \ + decode_layoutget_maxsz) +#define NFS4_enc_open_downgrade_sz \ + (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_layoutreturn_maxsz + \ + encode_open_downgrade_maxsz) +#define NFS4_dec_open_downgrade_sz \ + (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_layoutreturn_maxsz + \ + decode_open_downgrade_maxsz) +#define NFS4_enc_close_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_layoutreturn_maxsz + \ + encode_close_maxsz + \ + encode_getattr_maxsz) +#define NFS4_dec_close_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_layoutreturn_maxsz + \ + decode_close_maxsz + \ + decode_getattr_maxsz) +#define NFS4_enc_setattr_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_setattr_maxsz + \ + encode_getattr_maxsz) +#define NFS4_dec_setattr_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_setattr_maxsz + \ + decode_getattr_maxsz) +#define NFS4_enc_fsinfo_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_fsinfo_maxsz) +#define NFS4_dec_fsinfo_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_fsinfo_maxsz) +#define NFS4_enc_renew_sz (compound_encode_hdr_maxsz + \ + encode_renew_maxsz) +#define NFS4_dec_renew_sz (compound_decode_hdr_maxsz + \ + decode_renew_maxsz) +#define NFS4_enc_setclientid_sz (compound_encode_hdr_maxsz + \ + encode_setclientid_maxsz) +#define NFS4_dec_setclientid_sz (compound_decode_hdr_maxsz + \ + decode_setclientid_maxsz) +#define NFS4_enc_setclientid_confirm_sz \ + (compound_encode_hdr_maxsz + \ + encode_setclientid_confirm_maxsz) +#define NFS4_dec_setclientid_confirm_sz \ + (compound_decode_hdr_maxsz + \ + decode_setclientid_confirm_maxsz) +#define NFS4_enc_lock_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_lock_maxsz) +#define NFS4_dec_lock_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_lock_maxsz) +#define NFS4_enc_lockt_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_lockt_maxsz) +#define NFS4_dec_lockt_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_lockt_maxsz) +#define NFS4_enc_locku_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_locku_maxsz) +#define NFS4_dec_locku_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_locku_maxsz) +#define NFS4_enc_release_lockowner_sz \ + (compound_encode_hdr_maxsz + \ + encode_lockowner_maxsz) +#define NFS4_dec_release_lockowner_sz \ + (compound_decode_hdr_maxsz + \ + decode_release_lockowner_maxsz) +#define NFS4_enc_access_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_access_maxsz + \ + encode_getattr_maxsz) +#define NFS4_dec_access_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_access_maxsz + \ + decode_getattr_maxsz) +#define NFS4_enc_getattr_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_get_dir_deleg_maxsz + \ + encode_getattr_maxsz + \ + encode_renew_maxsz) +#define NFS4_dec_getattr_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_get_dir_deleg_maxsz + \ + decode_getattr_maxsz + \ + decode_renew_maxsz) +#define NFS4_enc_lookup_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_lookup_maxsz + \ + encode_getattr_maxsz + \ + encode_getfh_maxsz) +#define NFS4_dec_lookup_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_lookup_maxsz + \ + decode_getattr_maxsz + \ + decode_getfh_maxsz) +#define NFS4_enc_lookupp_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_lookupp_maxsz + \ + encode_getattr_maxsz + \ + encode_getfh_maxsz) +#define NFS4_dec_lookupp_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_lookupp_maxsz + \ + decode_getattr_maxsz + \ + decode_getfh_maxsz) +#define NFS4_enc_lookup_root_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putrootfh_maxsz + \ + encode_getattr_maxsz + \ + encode_getfh_maxsz) +#define NFS4_dec_lookup_root_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putrootfh_maxsz + \ + decode_getattr_maxsz + \ + decode_getfh_maxsz) +#define NFS4_enc_remove_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_remove_maxsz) +#define NFS4_dec_remove_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_remove_maxsz) +#define NFS4_enc_rename_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_savefh_maxsz + \ + encode_putfh_maxsz + \ + encode_rename_maxsz) +#define NFS4_dec_rename_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_savefh_maxsz + \ + decode_putfh_maxsz + \ + decode_rename_maxsz) +#define NFS4_enc_link_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_savefh_maxsz + \ + encode_putfh_maxsz + \ + encode_link_maxsz + \ + encode_restorefh_maxsz + \ + encode_getattr_maxsz) +#define NFS4_dec_link_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_savefh_maxsz + \ + decode_putfh_maxsz + \ + decode_link_maxsz + \ + decode_restorefh_maxsz + \ + decode_getattr_maxsz) +#define NFS4_enc_symlink_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_symlink_maxsz + \ + encode_getattr_maxsz + \ + encode_getfh_maxsz) +#define NFS4_dec_symlink_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_symlink_maxsz + \ + decode_getattr_maxsz + \ + decode_getfh_maxsz) +#define NFS4_enc_create_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_create_maxsz + \ + encode_getfh_maxsz + \ + encode_getattr_maxsz) +#define NFS4_dec_create_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_create_maxsz + \ + decode_getfh_maxsz + \ + decode_getattr_maxsz) +#define NFS4_enc_pathconf_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_getattr_maxsz) +#define NFS4_dec_pathconf_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_getattr_maxsz) +#define NFS4_enc_statfs_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_statfs_maxsz) +#define NFS4_dec_statfs_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_statfs_maxsz) +#define NFS4_enc_server_caps_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_getattr_maxsz) +#define NFS4_dec_server_caps_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_getattr_maxsz) +#define NFS4_enc_delegreturn_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_layoutreturn_maxsz + \ + encode_delegattr_maxsz + \ + encode_delegreturn_maxsz + \ + encode_getattr_maxsz) +#define NFS4_dec_delegreturn_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_layoutreturn_maxsz + \ + decode_delegattr_maxsz + \ + decode_delegreturn_maxsz + \ + decode_getattr_maxsz) +#define NFS4_enc_getacl_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_getacl_maxsz) +#define NFS4_dec_getacl_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_getacl_maxsz) +#define NFS4_enc_setacl_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_setacl_maxsz) +#define NFS4_dec_setacl_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_setacl_maxsz) +#define NFS4_enc_fs_locations_sz \ + (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_lookup_maxsz + \ + encode_fs_locations_maxsz + \ + encode_renew_maxsz) +#define NFS4_dec_fs_locations_sz \ + (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_lookup_maxsz + \ + decode_fs_locations_maxsz + \ + decode_renew_maxsz) +#define NFS4_enc_secinfo_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_secinfo_maxsz) +#define NFS4_dec_secinfo_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_secinfo_maxsz) +#define NFS4_enc_fsid_present_sz \ + (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_getfh_maxsz + \ + encode_renew_maxsz) +#define NFS4_dec_fsid_present_sz \ + (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_getfh_maxsz + \ + decode_renew_maxsz) +#define NFS4_enc_bind_conn_to_session_sz \ + (compound_encode_hdr_maxsz + \ + encode_bind_conn_to_session_maxsz) +#define NFS4_dec_bind_conn_to_session_sz \ + (compound_decode_hdr_maxsz + \ + decode_bind_conn_to_session_maxsz) +#define NFS4_enc_exchange_id_sz \ + (compound_encode_hdr_maxsz + \ + encode_exchange_id_maxsz) +#define NFS4_dec_exchange_id_sz \ + (compound_decode_hdr_maxsz + \ + decode_exchange_id_maxsz) +#define NFS4_enc_create_session_sz \ + (compound_encode_hdr_maxsz + \ + encode_create_session_maxsz) +#define NFS4_dec_create_session_sz \ + (compound_decode_hdr_maxsz + \ + decode_create_session_maxsz) +#define NFS4_enc_destroy_session_sz (compound_encode_hdr_maxsz + \ + encode_destroy_session_maxsz) +#define NFS4_dec_destroy_session_sz (compound_decode_hdr_maxsz + \ + decode_destroy_session_maxsz) +#define NFS4_enc_destroy_clientid_sz (compound_encode_hdr_maxsz + \ + encode_destroy_clientid_maxsz) +#define NFS4_dec_destroy_clientid_sz (compound_decode_hdr_maxsz + \ + decode_destroy_clientid_maxsz) +#define NFS4_enc_sequence_sz \ + (compound_decode_hdr_maxsz + \ + encode_sequence_maxsz) +#define NFS4_dec_sequence_sz \ + (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz) +#define NFS4_enc_get_lease_time_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putrootfh_maxsz + \ + encode_fsinfo_maxsz) +#define NFS4_dec_get_lease_time_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putrootfh_maxsz + \ + decode_fsinfo_maxsz) +#define NFS4_enc_reclaim_complete_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_reclaim_complete_maxsz) +#define NFS4_dec_reclaim_complete_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_reclaim_complete_maxsz) +#define NFS4_enc_getdeviceinfo_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz +\ + encode_getdeviceinfo_maxsz) +#define NFS4_dec_getdeviceinfo_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_getdeviceinfo_maxsz) +#define NFS4_enc_layoutget_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_layoutget_maxsz) +#define NFS4_dec_layoutget_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_layoutget_maxsz) +#define NFS4_enc_layoutcommit_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz +\ + encode_putfh_maxsz + \ + encode_layoutcommit_maxsz + \ + encode_getattr_maxsz) +#define NFS4_dec_layoutcommit_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_layoutcommit_maxsz + \ + decode_getattr_maxsz) +#define NFS4_enc_layoutreturn_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putfh_maxsz + \ + encode_layoutreturn_maxsz) +#define NFS4_dec_layoutreturn_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putfh_maxsz + \ + decode_layoutreturn_maxsz) +#define NFS4_enc_secinfo_no_name_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_putrootfh_maxsz +\ + encode_secinfo_no_name_maxsz) +#define NFS4_dec_secinfo_no_name_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_putrootfh_maxsz + \ + decode_secinfo_no_name_maxsz) +#define NFS4_enc_test_stateid_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_test_stateid_maxsz) +#define NFS4_dec_test_stateid_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_test_stateid_maxsz) +#define NFS4_enc_free_stateid_sz (compound_encode_hdr_maxsz + \ + encode_sequence_maxsz + \ + encode_free_stateid_maxsz) +#define NFS4_dec_free_stateid_sz (compound_decode_hdr_maxsz + \ + decode_sequence_maxsz + \ + decode_free_stateid_maxsz) + +const u32 nfs41_maxwrite_overhead = ((RPC_MAX_HEADER_WITH_AUTH + + compound_encode_hdr_maxsz + + encode_sequence_maxsz + + encode_putfh_maxsz + + encode_getattr_maxsz) * + XDR_UNIT); + +const u32 nfs41_maxread_overhead = ((RPC_MAX_HEADER_WITH_AUTH + + compound_decode_hdr_maxsz + + decode_sequence_maxsz + + decode_putfh_maxsz) * + XDR_UNIT); + +const u32 nfs41_maxgetdevinfo_overhead = ((RPC_MAX_REPHEADER_WITH_AUTH + + compound_decode_hdr_maxsz + + decode_sequence_maxsz) * + XDR_UNIT); +EXPORT_SYMBOL_GPL(nfs41_maxgetdevinfo_overhead); + +static const umode_t nfs_type2fmt[] = { + [NF4BAD] = 0, + [NF4REG] = S_IFREG, + [NF4DIR] = S_IFDIR, + [NF4BLK] = S_IFBLK, + [NF4CHR] = S_IFCHR, + [NF4LNK] = S_IFLNK, + [NF4SOCK] = S_IFSOCK, + [NF4FIFO] = S_IFIFO, + [NF4ATTRDIR] = 0, + [NF4NAMEDATTR] = 0, +}; + +struct compound_hdr { + int32_t status; + uint32_t nops; + __be32 * nops_p; + uint32_t taglen; + char * tag; + uint32_t replen; /* expected reply words */ + u32 minorversion; +}; + +static __be32 *reserve_space(struct xdr_stream *xdr, size_t nbytes) +{ + __be32 *p = xdr_reserve_space(xdr, nbytes); + BUG_ON(!p); + return p; +} + +static void encode_string(struct xdr_stream *xdr, unsigned int len, const char *str) +{ + WARN_ON_ONCE(xdr_stream_encode_opaque(xdr, str, len) < 0); +} + +static void encode_uint32(struct xdr_stream *xdr, u32 n) +{ + WARN_ON_ONCE(xdr_stream_encode_u32(xdr, n) < 0); +} + +static void encode_uint64(struct xdr_stream *xdr, u64 n) +{ + WARN_ON_ONCE(xdr_stream_encode_u64(xdr, n) < 0); +} + +static ssize_t xdr_encode_bitmap4(struct xdr_stream *xdr, + const __u32 *bitmap, size_t len) +{ + ssize_t ret; + + /* Trim empty words */ + while (len > 0 && bitmap[len-1] == 0) + len--; + ret = xdr_stream_encode_uint32_array(xdr, bitmap, len); + if (WARN_ON_ONCE(ret < 0)) + return ret; + return len; +} + +static size_t mask_bitmap4(const __u32 *bitmap, const __u32 *mask, + __u32 *res, size_t len) +{ + size_t i; + __u32 tmp; + + while (len > 0 && (bitmap[len-1] == 0 || mask[len-1] == 0)) + len--; + for (i = len; i-- > 0;) { + tmp = bitmap[i] & mask[i]; + res[i] = tmp; + } + return len; +} + +static void encode_nfs4_seqid(struct xdr_stream *xdr, + const struct nfs_seqid *seqid) +{ + if (seqid != NULL) + encode_uint32(xdr, seqid->sequence->counter); + else + encode_uint32(xdr, 0); +} + +static void encode_compound_hdr(struct xdr_stream *xdr, + struct rpc_rqst *req, + struct compound_hdr *hdr) +{ + __be32 *p; + + /* initialize running count of expected bytes in reply. + * NOTE: the replied tag SHOULD be the same is the one sent, + * but this is not required as a MUST for the server to do so. */ + hdr->replen = 3 + hdr->taglen; + + WARN_ON_ONCE(hdr->taglen > NFS4_MAXTAGLEN); + encode_string(xdr, hdr->taglen, hdr->tag); + p = reserve_space(xdr, 8); + *p++ = cpu_to_be32(hdr->minorversion); + hdr->nops_p = p; + *p = cpu_to_be32(hdr->nops); +} + +static void encode_op_hdr(struct xdr_stream *xdr, enum nfs_opnum4 op, + uint32_t replen, + struct compound_hdr *hdr) +{ + encode_uint32(xdr, op); + hdr->nops++; + hdr->replen += replen; +} + +static void encode_nops(struct compound_hdr *hdr) +{ + WARN_ON_ONCE(hdr->nops > NFS4_MAX_OPS); + *hdr->nops_p = htonl(hdr->nops); +} + +static void encode_nfs4_stateid(struct xdr_stream *xdr, + const nfs4_stateid *stateid) +{ + encode_opaque_fixed(xdr, stateid->data, NFS4_STATEID_SIZE); +} + +static void encode_nfs4_verifier(struct xdr_stream *xdr, const nfs4_verifier *verf) +{ + encode_opaque_fixed(xdr, verf->data, NFS4_VERIFIER_SIZE); +} + +static __be32 * +xdr_encode_nfstime4(__be32 *p, const struct timespec64 *t) +{ + p = xdr_encode_hyper(p, t->tv_sec); + *p++ = cpu_to_be32(t->tv_nsec); + return p; +} + +static void encode_attrs(struct xdr_stream *xdr, const struct iattr *iap, + const struct nfs4_label *label, + const umode_t *umask, + const struct nfs_server *server, + const uint32_t attrmask[]) +{ + char owner_name[IDMAP_NAMESZ]; + char owner_group[IDMAP_NAMESZ]; + int owner_namelen = 0; + int owner_grouplen = 0; + __be32 *p; + uint32_t len = 0; + uint32_t bmval[3] = { 0 }; + + /* + * We reserve enough space to write the entire attribute buffer at once. + */ + if ((iap->ia_valid & ATTR_SIZE) && (attrmask[0] & FATTR4_WORD0_SIZE)) { + bmval[0] |= FATTR4_WORD0_SIZE; + len += 8; + } + if (iap->ia_valid & ATTR_MODE) { + if (umask && (attrmask[2] & FATTR4_WORD2_MODE_UMASK)) { + bmval[2] |= FATTR4_WORD2_MODE_UMASK; + len += 8; + } else if (attrmask[1] & FATTR4_WORD1_MODE) { + bmval[1] |= FATTR4_WORD1_MODE; + len += 4; + } + } + if ((iap->ia_valid & ATTR_UID) && (attrmask[1] & FATTR4_WORD1_OWNER)) { + owner_namelen = nfs_map_uid_to_name(server, iap->ia_uid, owner_name, IDMAP_NAMESZ); + if (owner_namelen < 0) { + dprintk("nfs: couldn't resolve uid %d to string\n", + from_kuid(&init_user_ns, iap->ia_uid)); + /* XXX */ + strcpy(owner_name, "nobody"); + owner_namelen = sizeof("nobody") - 1; + /* goto out; */ + } + bmval[1] |= FATTR4_WORD1_OWNER; + len += 4 + (XDR_QUADLEN(owner_namelen) << 2); + } + if ((iap->ia_valid & ATTR_GID) && + (attrmask[1] & FATTR4_WORD1_OWNER_GROUP)) { + owner_grouplen = nfs_map_gid_to_group(server, iap->ia_gid, owner_group, IDMAP_NAMESZ); + if (owner_grouplen < 0) { + dprintk("nfs: couldn't resolve gid %d to string\n", + from_kgid(&init_user_ns, iap->ia_gid)); + strcpy(owner_group, "nobody"); + owner_grouplen = sizeof("nobody") - 1; + /* goto out; */ + } + bmval[1] |= FATTR4_WORD1_OWNER_GROUP; + len += 4 + (XDR_QUADLEN(owner_grouplen) << 2); + } + if (attrmask[1] & FATTR4_WORD1_TIME_ACCESS_SET) { + if (iap->ia_valid & ATTR_ATIME_SET) { + bmval[1] |= FATTR4_WORD1_TIME_ACCESS_SET; + len += 4 + (nfstime4_maxsz << 2); + } else if (iap->ia_valid & ATTR_ATIME) { + bmval[1] |= FATTR4_WORD1_TIME_ACCESS_SET; + len += 4; + } + } + if (attrmask[1] & FATTR4_WORD1_TIME_MODIFY_SET) { + if (iap->ia_valid & ATTR_MTIME_SET) { + bmval[1] |= FATTR4_WORD1_TIME_MODIFY_SET; + len += 4 + (nfstime4_maxsz << 2); + } else if (iap->ia_valid & ATTR_MTIME) { + bmval[1] |= FATTR4_WORD1_TIME_MODIFY_SET; + len += 4; + } + } + + if (label && (attrmask[2] & FATTR4_WORD2_SECURITY_LABEL)) { + len += 4 + 4 + 4 + (XDR_QUADLEN(label->len) << 2); + bmval[2] |= FATTR4_WORD2_SECURITY_LABEL; + } + + xdr_encode_bitmap4(xdr, bmval, ARRAY_SIZE(bmval)); + xdr_stream_encode_opaque_inline(xdr, (void **)&p, len); + + if (bmval[0] & FATTR4_WORD0_SIZE) + p = xdr_encode_hyper(p, iap->ia_size); + if (bmval[1] & FATTR4_WORD1_MODE) + *p++ = cpu_to_be32(iap->ia_mode & S_IALLUGO); + if (bmval[1] & FATTR4_WORD1_OWNER) + p = xdr_encode_opaque(p, owner_name, owner_namelen); + if (bmval[1] & FATTR4_WORD1_OWNER_GROUP) + p = xdr_encode_opaque(p, owner_group, owner_grouplen); + if (bmval[1] & FATTR4_WORD1_TIME_ACCESS_SET) { + if (iap->ia_valid & ATTR_ATIME_SET) { + *p++ = cpu_to_be32(NFS4_SET_TO_CLIENT_TIME); + p = xdr_encode_nfstime4(p, &iap->ia_atime); + } else + *p++ = cpu_to_be32(NFS4_SET_TO_SERVER_TIME); + } + if (bmval[1] & FATTR4_WORD1_TIME_MODIFY_SET) { + if (iap->ia_valid & ATTR_MTIME_SET) { + *p++ = cpu_to_be32(NFS4_SET_TO_CLIENT_TIME); + p = xdr_encode_nfstime4(p, &iap->ia_mtime); + } else + *p++ = cpu_to_be32(NFS4_SET_TO_SERVER_TIME); + } + if (label && (bmval[2] & FATTR4_WORD2_SECURITY_LABEL)) { + *p++ = cpu_to_be32(label->lfs); + *p++ = cpu_to_be32(label->pi); + *p++ = cpu_to_be32(label->len); + p = xdr_encode_opaque_fixed(p, label->label, label->len); + } + if (bmval[2] & FATTR4_WORD2_MODE_UMASK) { + *p++ = cpu_to_be32(iap->ia_mode & S_IALLUGO); + *p++ = cpu_to_be32(*umask); + } + +/* out: */ +} + +static void encode_access(struct xdr_stream *xdr, u32 access, struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_ACCESS, decode_access_maxsz, hdr); + encode_uint32(xdr, access); +} + +static void encode_close(struct xdr_stream *xdr, const struct nfs_closeargs *arg, struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_CLOSE, decode_close_maxsz, hdr); + encode_nfs4_seqid(xdr, arg->seqid); + encode_nfs4_stateid(xdr, &arg->stateid); +} + +static void encode_commit(struct xdr_stream *xdr, const struct nfs_commitargs *args, struct compound_hdr *hdr) +{ + __be32 *p; + + encode_op_hdr(xdr, OP_COMMIT, decode_commit_maxsz, hdr); + p = reserve_space(xdr, 12); + p = xdr_encode_hyper(p, args->offset); + *p = cpu_to_be32(args->count); +} + +static void encode_create(struct xdr_stream *xdr, const struct nfs4_create_arg *create, struct compound_hdr *hdr) +{ + __be32 *p; + + encode_op_hdr(xdr, OP_CREATE, decode_create_maxsz, hdr); + encode_uint32(xdr, create->ftype); + + switch (create->ftype) { + case NF4LNK: + p = reserve_space(xdr, 4); + *p = cpu_to_be32(create->u.symlink.len); + xdr_write_pages(xdr, create->u.symlink.pages, 0, + create->u.symlink.len); + xdr->buf->flags |= XDRBUF_WRITE; + break; + + case NF4BLK: case NF4CHR: + p = reserve_space(xdr, 8); + *p++ = cpu_to_be32(create->u.device.specdata1); + *p = cpu_to_be32(create->u.device.specdata2); + break; + + default: + break; + } + + encode_string(xdr, create->name->len, create->name->name); + encode_attrs(xdr, create->attrs, create->label, &create->umask, + create->server, create->server->attr_bitmask); +} + +static void encode_getattr(struct xdr_stream *xdr, + const __u32 *bitmap, const __u32 *mask, size_t len, + struct compound_hdr *hdr) +{ + __u32 masked_bitmap[nfs4_fattr_bitmap_maxsz]; + + encode_op_hdr(xdr, OP_GETATTR, decode_getattr_maxsz, hdr); + if (mask) { + if (WARN_ON_ONCE(len > ARRAY_SIZE(masked_bitmap))) + len = ARRAY_SIZE(masked_bitmap); + len = mask_bitmap4(bitmap, mask, masked_bitmap, len); + bitmap = masked_bitmap; + } + xdr_encode_bitmap4(xdr, bitmap, len); +} + +static void encode_getfattr(struct xdr_stream *xdr, const u32* bitmask, struct compound_hdr *hdr) +{ + encode_getattr(xdr, nfs4_fattr_bitmap, bitmask, + ARRAY_SIZE(nfs4_fattr_bitmap), hdr); +} + +static void encode_getfattr_open(struct xdr_stream *xdr, const u32 *bitmask, + const u32 *open_bitmap, + struct compound_hdr *hdr) +{ + encode_getattr(xdr, open_bitmap, bitmask, 3, hdr); +} + +static void encode_fsinfo(struct xdr_stream *xdr, const u32* bitmask, struct compound_hdr *hdr) +{ + encode_getattr(xdr, nfs4_fsinfo_bitmap, bitmask, + ARRAY_SIZE(nfs4_fsinfo_bitmap), hdr); +} + +static void encode_fs_locations(struct xdr_stream *xdr, const u32* bitmask, struct compound_hdr *hdr) +{ + encode_getattr(xdr, nfs4_fs_locations_bitmap, bitmask, + ARRAY_SIZE(nfs4_fs_locations_bitmap), hdr); +} + +static void encode_getfh(struct xdr_stream *xdr, struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_GETFH, decode_getfh_maxsz, hdr); +} + +static void encode_link(struct xdr_stream *xdr, const struct qstr *name, struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_LINK, decode_link_maxsz, hdr); + encode_string(xdr, name->len, name->name); +} + +static inline int nfs4_lock_type(struct file_lock *fl, int block) +{ + if (lock_is_read(fl)) + return block ? NFS4_READW_LT : NFS4_READ_LT; + return block ? NFS4_WRITEW_LT : NFS4_WRITE_LT; +} + +static inline uint64_t nfs4_lock_length(struct file_lock *fl) +{ + if (fl->fl_end == OFFSET_MAX) + return ~(uint64_t)0; + return fl->fl_end - fl->fl_start + 1; +} + +static void encode_lockowner(struct xdr_stream *xdr, const struct nfs_lowner *lowner) +{ + __be32 *p; + + p = reserve_space(xdr, 32); + p = xdr_encode_hyper(p, lowner->clientid); + *p++ = cpu_to_be32(20); + p = xdr_encode_opaque_fixed(p, "lock id:", 8); + *p++ = cpu_to_be32(lowner->s_dev); + xdr_encode_hyper(p, lowner->id); +} + +/* + * opcode,type,reclaim,offset,length,new_lock_owner = 32 + * open_seqid,open_stateid,lock_seqid,lock_owner.clientid, lock_owner.id = 40 + */ +static void encode_lock(struct xdr_stream *xdr, const struct nfs_lock_args *args, struct compound_hdr *hdr) +{ + __be32 *p; + + encode_op_hdr(xdr, OP_LOCK, decode_lock_maxsz, hdr); + p = reserve_space(xdr, 28); + *p++ = cpu_to_be32(nfs4_lock_type(args->fl, args->block)); + *p++ = cpu_to_be32(args->reclaim); + p = xdr_encode_hyper(p, args->fl->fl_start); + p = xdr_encode_hyper(p, nfs4_lock_length(args->fl)); + *p = cpu_to_be32(args->new_lock_owner); + if (args->new_lock_owner){ + encode_nfs4_seqid(xdr, args->open_seqid); + encode_nfs4_stateid(xdr, &args->open_stateid); + encode_nfs4_seqid(xdr, args->lock_seqid); + encode_lockowner(xdr, &args->lock_owner); + } + else { + encode_nfs4_stateid(xdr, &args->lock_stateid); + encode_nfs4_seqid(xdr, args->lock_seqid); + } +} + +static void encode_lockt(struct xdr_stream *xdr, const struct nfs_lockt_args *args, struct compound_hdr *hdr) +{ + __be32 *p; + + encode_op_hdr(xdr, OP_LOCKT, decode_lockt_maxsz, hdr); + p = reserve_space(xdr, 20); + *p++ = cpu_to_be32(nfs4_lock_type(args->fl, 0)); + p = xdr_encode_hyper(p, args->fl->fl_start); + p = xdr_encode_hyper(p, nfs4_lock_length(args->fl)); + encode_lockowner(xdr, &args->lock_owner); +} + +static void encode_locku(struct xdr_stream *xdr, const struct nfs_locku_args *args, struct compound_hdr *hdr) +{ + __be32 *p; + + encode_op_hdr(xdr, OP_LOCKU, decode_locku_maxsz, hdr); + encode_uint32(xdr, nfs4_lock_type(args->fl, 0)); + encode_nfs4_seqid(xdr, args->seqid); + encode_nfs4_stateid(xdr, &args->stateid); + p = reserve_space(xdr, 16); + p = xdr_encode_hyper(p, args->fl->fl_start); + xdr_encode_hyper(p, nfs4_lock_length(args->fl)); +} + +#if defined(CONFIG_NFS_V4_0) +static void encode_release_lockowner(struct xdr_stream *xdr, const struct nfs_lowner *lowner, struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_RELEASE_LOCKOWNER, decode_release_lockowner_maxsz, hdr); + encode_lockowner(xdr, lowner); +} +#endif /* CONFIG_NFS_V4_0 */ + +static void encode_lookup(struct xdr_stream *xdr, const struct qstr *name, struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_LOOKUP, decode_lookup_maxsz, hdr); + encode_string(xdr, name->len, name->name); +} + +static void encode_lookupp(struct xdr_stream *xdr, struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_LOOKUPP, decode_lookupp_maxsz, hdr); +} + +static void encode_share_access(struct xdr_stream *xdr, u32 share_access) +{ + __be32 *p; + + p = reserve_space(xdr, 8); + *p++ = cpu_to_be32(share_access); + *p = cpu_to_be32(0); /* for linux, share_deny = 0 always */ +} + +static inline void encode_openhdr(struct xdr_stream *xdr, const struct nfs_openargs *arg) +{ + __be32 *p; + /* + * opcode 4, seqid 4, share_access 4, share_deny 4, clientid 8, ownerlen 4, + * owner 28 + */ + encode_nfs4_seqid(xdr, arg->seqid); + encode_share_access(xdr, arg->share_access); + p = reserve_space(xdr, 40); + p = xdr_encode_hyper(p, arg->clientid); + *p++ = cpu_to_be32(28); + p = xdr_encode_opaque_fixed(p, "open id:", 8); + *p++ = cpu_to_be32(arg->server->s_dev); + p = xdr_encode_hyper(p, arg->id.uniquifier); + xdr_encode_hyper(p, arg->id.create_time); +} + +static inline void encode_createmode(struct xdr_stream *xdr, const struct nfs_openargs *arg) +{ + __be32 *p; + + p = reserve_space(xdr, 4); + switch(arg->createmode) { + case NFS4_CREATE_UNCHECKED: + *p = cpu_to_be32(NFS4_CREATE_UNCHECKED); + encode_attrs(xdr, arg->u.attrs, arg->label, &arg->umask, + arg->server, arg->server->attr_bitmask); + break; + case NFS4_CREATE_GUARDED: + *p = cpu_to_be32(NFS4_CREATE_GUARDED); + encode_attrs(xdr, arg->u.attrs, arg->label, &arg->umask, + arg->server, arg->server->attr_bitmask); + break; + case NFS4_CREATE_EXCLUSIVE: + *p = cpu_to_be32(NFS4_CREATE_EXCLUSIVE); + encode_nfs4_verifier(xdr, &arg->u.verifier); + break; + case NFS4_CREATE_EXCLUSIVE4_1: + *p = cpu_to_be32(NFS4_CREATE_EXCLUSIVE4_1); + encode_nfs4_verifier(xdr, &arg->u.verifier); + encode_attrs(xdr, arg->u.attrs, arg->label, &arg->umask, + arg->server, arg->server->exclcreat_bitmask); + } +} + +static void encode_opentype(struct xdr_stream *xdr, const struct nfs_openargs *arg) +{ + __be32 *p; + + p = reserve_space(xdr, 4); + switch (arg->open_flags & O_CREAT) { + case 0: + *p = cpu_to_be32(NFS4_OPEN_NOCREATE); + break; + default: + *p = cpu_to_be32(NFS4_OPEN_CREATE); + encode_createmode(xdr, arg); + } +} + +static inline void encode_delegation_type(struct xdr_stream *xdr, u32 delegation_type) +{ + __be32 *p; + + p = reserve_space(xdr, 4); + switch (delegation_type) { + case NFS4_OPEN_DELEGATE_NONE: + case NFS4_OPEN_DELEGATE_READ: + case NFS4_OPEN_DELEGATE_WRITE: + case NFS4_OPEN_DELEGATE_READ_ATTRS_DELEG: + case NFS4_OPEN_DELEGATE_WRITE_ATTRS_DELEG: + *p = cpu_to_be32(delegation_type); + break; + default: + BUG(); + } +} + +static inline void encode_claim_null(struct xdr_stream *xdr, const struct qstr *name) +{ + __be32 *p; + + p = reserve_space(xdr, 4); + *p = cpu_to_be32(NFS4_OPEN_CLAIM_NULL); + encode_string(xdr, name->len, name->name); +} + +static inline void encode_claim_previous(struct xdr_stream *xdr, u32 type) +{ + __be32 *p; + + p = reserve_space(xdr, 4); + *p = cpu_to_be32(NFS4_OPEN_CLAIM_PREVIOUS); + encode_delegation_type(xdr, type); +} + +static inline void encode_claim_delegate_cur(struct xdr_stream *xdr, const struct qstr *name, const nfs4_stateid *stateid) +{ + __be32 *p; + + p = reserve_space(xdr, 4); + *p = cpu_to_be32(NFS4_OPEN_CLAIM_DELEGATE_CUR); + encode_nfs4_stateid(xdr, stateid); + encode_string(xdr, name->len, name->name); +} + +static inline void encode_claim_fh(struct xdr_stream *xdr) +{ + __be32 *p; + + p = reserve_space(xdr, 4); + *p = cpu_to_be32(NFS4_OPEN_CLAIM_FH); +} + +static inline void encode_claim_delegate_cur_fh(struct xdr_stream *xdr, const nfs4_stateid *stateid) +{ + __be32 *p; + + p = reserve_space(xdr, 4); + *p = cpu_to_be32(NFS4_OPEN_CLAIM_DELEG_CUR_FH); + encode_nfs4_stateid(xdr, stateid); +} + +static void encode_open(struct xdr_stream *xdr, const struct nfs_openargs *arg, struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_OPEN, decode_open_maxsz, hdr); + encode_openhdr(xdr, arg); + encode_opentype(xdr, arg); + switch (arg->claim) { + case NFS4_OPEN_CLAIM_NULL: + encode_claim_null(xdr, arg->name); + break; + case NFS4_OPEN_CLAIM_PREVIOUS: + encode_claim_previous(xdr, arg->u.delegation_type); + break; + case NFS4_OPEN_CLAIM_DELEGATE_CUR: + encode_claim_delegate_cur(xdr, arg->name, &arg->u.delegation); + break; + case NFS4_OPEN_CLAIM_FH: + encode_claim_fh(xdr); + break; + case NFS4_OPEN_CLAIM_DELEG_CUR_FH: + encode_claim_delegate_cur_fh(xdr, &arg->u.delegation); + break; + default: + BUG(); + } +} + +static void encode_open_confirm(struct xdr_stream *xdr, const struct nfs_open_confirmargs *arg, struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_OPEN_CONFIRM, decode_open_confirm_maxsz, hdr); + encode_nfs4_stateid(xdr, arg->stateid); + encode_nfs4_seqid(xdr, arg->seqid); +} + +static void encode_open_downgrade(struct xdr_stream *xdr, const struct nfs_closeargs *arg, struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_OPEN_DOWNGRADE, decode_open_downgrade_maxsz, hdr); + encode_nfs4_stateid(xdr, &arg->stateid); + encode_nfs4_seqid(xdr, arg->seqid); + encode_share_access(xdr, arg->share_access); +} + +static void +encode_putfh(struct xdr_stream *xdr, const struct nfs_fh *fh, struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_PUTFH, decode_putfh_maxsz, hdr); + encode_string(xdr, fh->size, fh->data); +} + +static void encode_putrootfh(struct xdr_stream *xdr, struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_PUTROOTFH, decode_putrootfh_maxsz, hdr); +} + +static void encode_read(struct xdr_stream *xdr, const struct nfs_pgio_args *args, + struct compound_hdr *hdr) +{ + __be32 *p; + + encode_op_hdr(xdr, OP_READ, decode_read_maxsz, hdr); + encode_nfs4_stateid(xdr, &args->stateid); + + p = reserve_space(xdr, 12); + p = xdr_encode_hyper(p, args->offset); + *p = cpu_to_be32(args->count); +} + +static void encode_readdir(struct xdr_stream *xdr, const struct nfs4_readdir_arg *readdir, struct rpc_rqst *req, struct compound_hdr *hdr) +{ + uint32_t attrs[3] = { + FATTR4_WORD0_TYPE + | FATTR4_WORD0_RDATTR_ERROR, + FATTR4_WORD1_MOUNTED_ON_FILEID, + }; + uint32_t dircount = readdir->count; + uint32_t maxcount = readdir->count; + __be32 *p, verf[2]; + uint32_t attrlen = 0; + unsigned int i; + + if (readdir->plus) { + attrs[0] |= FATTR4_WORD0_CHANGE + | FATTR4_WORD0_SIZE + | FATTR4_WORD0_FSID + | FATTR4_WORD0_FILEHANDLE + | FATTR4_WORD0_FILEID; + attrs[1] |= FATTR4_WORD1_MODE + | FATTR4_WORD1_NUMLINKS + | FATTR4_WORD1_OWNER + | FATTR4_WORD1_OWNER_GROUP + | FATTR4_WORD1_RAWDEV + | FATTR4_WORD1_SPACE_USED + | FATTR4_WORD1_TIME_ACCESS + | FATTR4_WORD1_TIME_CREATE + | FATTR4_WORD1_TIME_METADATA + | FATTR4_WORD1_TIME_MODIFY; + attrs[2] |= FATTR4_WORD2_SECURITY_LABEL; + } + /* Use mounted_on_fileid only if the server supports it */ + if (!(readdir->bitmask[1] & FATTR4_WORD1_MOUNTED_ON_FILEID)) + attrs[0] |= FATTR4_WORD0_FILEID; + for (i = 0; i < ARRAY_SIZE(attrs); i++) { + attrs[i] &= readdir->bitmask[i]; + if (attrs[i] != 0) + attrlen = i+1; + } + + encode_op_hdr(xdr, OP_READDIR, decode_readdir_maxsz, hdr); + encode_uint64(xdr, readdir->cookie); + encode_nfs4_verifier(xdr, &readdir->verifier); + p = reserve_space(xdr, 12 + (attrlen << 2)); + *p++ = cpu_to_be32(dircount); + *p++ = cpu_to_be32(maxcount); + *p++ = cpu_to_be32(attrlen); + for (i = 0; i < attrlen; i++) + *p++ = cpu_to_be32(attrs[i]); + memcpy(verf, readdir->verifier.data, sizeof(verf)); + + dprintk("%s: cookie = %llu, verifier = %08x:%08x, bitmap = %08x:%08x:%08x\n", + __func__, + (unsigned long long)readdir->cookie, + verf[0], verf[1], + attrs[0] & readdir->bitmask[0], + attrs[1] & readdir->bitmask[1], + attrs[2] & readdir->bitmask[2]); +} + +static void encode_readlink(struct xdr_stream *xdr, const struct nfs4_readlink *readlink, struct rpc_rqst *req, struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_READLINK, decode_readlink_maxsz, hdr); +} + +static void encode_remove(struct xdr_stream *xdr, const struct qstr *name, struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_REMOVE, decode_remove_maxsz, hdr); + encode_string(xdr, name->len, name->name); +} + +static void encode_rename(struct xdr_stream *xdr, const struct qstr *oldname, const struct qstr *newname, struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_RENAME, decode_rename_maxsz, hdr); + encode_string(xdr, oldname->len, oldname->name); + encode_string(xdr, newname->len, newname->name); +} + +static void encode_renew(struct xdr_stream *xdr, clientid4 clid, + struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_RENEW, decode_renew_maxsz, hdr); + encode_uint64(xdr, clid); +} + +static void +encode_restorefh(struct xdr_stream *xdr, struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_RESTOREFH, decode_restorefh_maxsz, hdr); +} + +static void nfs4_acltype_to_bitmap(enum nfs4_acl_type type, __u32 bitmap[2]) +{ + switch (type) { + default: + bitmap[0] = FATTR4_WORD0_ACL; + bitmap[1] = 0; + break; + case NFS4ACL_DACL: + bitmap[0] = 0; + bitmap[1] = FATTR4_WORD1_DACL; + break; + case NFS4ACL_SACL: + bitmap[0] = 0; + bitmap[1] = FATTR4_WORD1_SACL; + } +} + +static void encode_setacl(struct xdr_stream *xdr, + const struct nfs_setaclargs *arg, + struct compound_hdr *hdr) +{ + __u32 bitmap[2]; + + nfs4_acltype_to_bitmap(arg->acl_type, bitmap); + + encode_op_hdr(xdr, OP_SETATTR, decode_setacl_maxsz, hdr); + encode_nfs4_stateid(xdr, &zero_stateid); + xdr_encode_bitmap4(xdr, bitmap, ARRAY_SIZE(bitmap)); + encode_uint32(xdr, arg->acl_len); + xdr_write_pages(xdr, arg->acl_pages, 0, arg->acl_len); +} + +static void +encode_savefh(struct xdr_stream *xdr, struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_SAVEFH, decode_savefh_maxsz, hdr); +} + +static void encode_setattr(struct xdr_stream *xdr, const struct nfs_setattrargs *arg, const struct nfs_server *server, struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_SETATTR, decode_setattr_maxsz, hdr); + encode_nfs4_stateid(xdr, &arg->stateid); + encode_attrs(xdr, arg->iap, arg->label, NULL, server, + server->attr_bitmask); +} + +static void encode_delegattr(struct xdr_stream *xdr, + const nfs4_stateid *stateid, + const struct nfs4_delegattr *attr, + struct compound_hdr *hdr) +{ + uint32_t bitmap[3] = { 0 }; + uint32_t len = 0; + __be32 *p; + + encode_op_hdr(xdr, OP_SETATTR, encode_delegattr_maxsz, hdr); + encode_nfs4_stateid(xdr, stateid); + if (attr->atime_set) { + bitmap[2] |= FATTR4_WORD2_TIME_DELEG_ACCESS; + len += (nfstime4_maxsz << 2); + } + if (attr->mtime_set) { + bitmap[2] |= FATTR4_WORD2_TIME_DELEG_MODIFY; + len += (nfstime4_maxsz << 2); + } + xdr_encode_bitmap4(xdr, bitmap, ARRAY_SIZE(bitmap)); + xdr_stream_encode_opaque_inline(xdr, (void **)&p, len); + if (bitmap[2] & FATTR4_WORD2_TIME_DELEG_ACCESS) + p = xdr_encode_nfstime4(p, &attr->atime); + if (bitmap[2] & FATTR4_WORD2_TIME_DELEG_MODIFY) + p = xdr_encode_nfstime4(p, &attr->mtime); +} + +static void encode_setclientid(struct xdr_stream *xdr, const struct nfs4_setclientid *setclientid, struct compound_hdr *hdr) +{ + __be32 *p; + + encode_op_hdr(xdr, OP_SETCLIENTID, decode_setclientid_maxsz, hdr); + encode_nfs4_verifier(xdr, setclientid->sc_verifier); + + encode_string(xdr, strlen(setclientid->sc_clnt->cl_owner_id), + setclientid->sc_clnt->cl_owner_id); + p = reserve_space(xdr, 4); + *p = cpu_to_be32(setclientid->sc_prog); + encode_string(xdr, setclientid->sc_netid_len, setclientid->sc_netid); + encode_string(xdr, setclientid->sc_uaddr_len, setclientid->sc_uaddr); + p = reserve_space(xdr, 4); + *p = cpu_to_be32(setclientid->sc_clnt->cl_cb_ident); +} + +static void encode_setclientid_confirm(struct xdr_stream *xdr, const struct nfs4_setclientid_res *arg, struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_SETCLIENTID_CONFIRM, + decode_setclientid_confirm_maxsz, hdr); + encode_uint64(xdr, arg->clientid); + encode_nfs4_verifier(xdr, &arg->confirm); +} + +static void encode_write(struct xdr_stream *xdr, const struct nfs_pgio_args *args, + struct compound_hdr *hdr) +{ + __be32 *p; + + encode_op_hdr(xdr, OP_WRITE, decode_write_maxsz, hdr); + encode_nfs4_stateid(xdr, &args->stateid); + + p = reserve_space(xdr, 16); + p = xdr_encode_hyper(p, args->offset); + *p++ = cpu_to_be32(args->stable); + *p = cpu_to_be32(args->count); + + xdr_write_pages(xdr, args->pages, args->pgbase, args->count); +} + +static void encode_delegreturn(struct xdr_stream *xdr, const nfs4_stateid *stateid, struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_DELEGRETURN, decode_delegreturn_maxsz, hdr); + encode_nfs4_stateid(xdr, stateid); +} + +static void encode_secinfo(struct xdr_stream *xdr, const struct qstr *name, struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_SECINFO, decode_secinfo_maxsz, hdr); + encode_string(xdr, name->len, name->name); +} + +/* NFSv4.1 operations */ +static void encode_bind_conn_to_session(struct xdr_stream *xdr, + const struct nfs41_bind_conn_to_session_args *args, + struct compound_hdr *hdr) +{ + __be32 *p; + + encode_op_hdr(xdr, OP_BIND_CONN_TO_SESSION, + decode_bind_conn_to_session_maxsz, hdr); + encode_opaque_fixed(xdr, args->sessionid.data, NFS4_MAX_SESSIONID_LEN); + p = xdr_reserve_space(xdr, 8); + *p++ = cpu_to_be32(args->dir); + *p = (args->use_conn_in_rdma_mode) ? cpu_to_be32(1) : cpu_to_be32(0); +} + +static void encode_op_map(struct xdr_stream *xdr, const struct nfs4_op_map *op_map) +{ + unsigned int i; + encode_uint32(xdr, NFS4_OP_MAP_NUM_WORDS); + for (i = 0; i < NFS4_OP_MAP_NUM_WORDS; i++) + encode_uint32(xdr, op_map->u.words[i]); +} + +static void encode_exchange_id(struct xdr_stream *xdr, + const struct nfs41_exchange_id_args *args, + struct compound_hdr *hdr) +{ + __be32 *p; + char impl_name[IMPL_NAME_LIMIT]; + int len = 0; + + encode_op_hdr(xdr, OP_EXCHANGE_ID, decode_exchange_id_maxsz, hdr); + encode_nfs4_verifier(xdr, &args->verifier); + + encode_string(xdr, strlen(args->client->cl_owner_id), + args->client->cl_owner_id); + + encode_uint32(xdr, args->flags); + encode_uint32(xdr, args->state_protect.how); + + switch (args->state_protect.how) { + case SP4_NONE: + break; + case SP4_MACH_CRED: + encode_op_map(xdr, &args->state_protect.enforce); + encode_op_map(xdr, &args->state_protect.allow); + break; + default: + WARN_ON_ONCE(1); + break; + } + + if (send_implementation_id && + sizeof(CONFIG_NFS_V4_1_IMPLEMENTATION_ID_DOMAIN) > 1 && + sizeof(CONFIG_NFS_V4_1_IMPLEMENTATION_ID_DOMAIN) + <= sizeof(impl_name) + 1) + len = snprintf(impl_name, sizeof(impl_name), "%s %s %s %s", + utsname()->sysname, utsname()->release, + utsname()->version, utsname()->machine); + + if (len > 0) { + encode_uint32(xdr, 1); /* implementation id array length=1 */ + + encode_string(xdr, + sizeof(CONFIG_NFS_V4_1_IMPLEMENTATION_ID_DOMAIN) - 1, + CONFIG_NFS_V4_1_IMPLEMENTATION_ID_DOMAIN); + encode_string(xdr, len, impl_name); + /* just send zeros for nii_date - the date is in nii_name */ + p = reserve_space(xdr, 12); + p = xdr_encode_hyper(p, 0); + *p = cpu_to_be32(0); + } else + encode_uint32(xdr, 0); /* implementation id array length=0 */ +} + +static void encode_create_session(struct xdr_stream *xdr, + const struct nfs41_create_session_args *args, + struct compound_hdr *hdr) +{ + __be32 *p; + struct nfs_client *clp = args->client; + struct rpc_clnt *clnt = clp->cl_rpcclient; + struct nfs_net *nn = net_generic(clp->cl_net, nfs_net_id); + u32 max_resp_sz_cached; + + /* + * Assumes OPEN is the biggest non-idempotent compound. + * 2 is the verifier. + */ + max_resp_sz_cached = (NFS4_dec_open_sz + RPC_REPHDRSIZE + 2) + * XDR_UNIT + RPC_MAX_AUTH_SIZE; + + encode_op_hdr(xdr, OP_CREATE_SESSION, decode_create_session_maxsz, hdr); + p = reserve_space(xdr, 16 + 2*28 + 20 + clnt->cl_nodelen + 12); + p = xdr_encode_hyper(p, args->clientid); + *p++ = cpu_to_be32(args->seqid); /*Sequence id */ + *p++ = cpu_to_be32(args->flags); /*flags */ + + /* Fore Channel */ + *p++ = cpu_to_be32(0); /* header padding size */ + *p++ = cpu_to_be32(args->fc_attrs.max_rqst_sz); /* max req size */ + *p++ = cpu_to_be32(args->fc_attrs.max_resp_sz); /* max resp size */ + *p++ = cpu_to_be32(max_resp_sz_cached); /* Max resp sz cached */ + *p++ = cpu_to_be32(args->fc_attrs.max_ops); /* max operations */ + *p++ = cpu_to_be32(args->fc_attrs.max_reqs); /* max requests */ + *p++ = cpu_to_be32(0); /* rdmachannel_attrs */ + + /* Back Channel */ + *p++ = cpu_to_be32(0); /* header padding size */ + *p++ = cpu_to_be32(args->bc_attrs.max_rqst_sz); /* max req size */ + *p++ = cpu_to_be32(args->bc_attrs.max_resp_sz); /* max resp size */ + *p++ = cpu_to_be32(args->bc_attrs.max_resp_sz_cached); /* Max resp sz cached */ + *p++ = cpu_to_be32(args->bc_attrs.max_ops); /* max operations */ + *p++ = cpu_to_be32(args->bc_attrs.max_reqs); /* max requests */ + *p++ = cpu_to_be32(0); /* rdmachannel_attrs */ + + *p++ = cpu_to_be32(args->cb_program); /* cb_program */ + *p++ = cpu_to_be32(1); + *p++ = cpu_to_be32(RPC_AUTH_UNIX); /* auth_sys */ + + /* authsys_parms rfc1831 */ + *p++ = cpu_to_be32(ktime_to_ns(nn->boot_time)); /* stamp */ + p = xdr_encode_array(p, clnt->cl_nodename, clnt->cl_nodelen); + *p++ = cpu_to_be32(0); /* UID */ + *p++ = cpu_to_be32(0); /* GID */ + *p = cpu_to_be32(0); /* No more gids */ +} + +static void encode_destroy_session(struct xdr_stream *xdr, + const struct nfs4_session *session, + struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_DESTROY_SESSION, decode_destroy_session_maxsz, hdr); + encode_opaque_fixed(xdr, session->sess_id.data, NFS4_MAX_SESSIONID_LEN); +} + +static void encode_destroy_clientid(struct xdr_stream *xdr, + uint64_t clientid, + struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_DESTROY_CLIENTID, decode_destroy_clientid_maxsz, hdr); + encode_uint64(xdr, clientid); +} + +static void encode_reclaim_complete(struct xdr_stream *xdr, + const struct nfs41_reclaim_complete_args *args, + struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_RECLAIM_COMPLETE, decode_reclaim_complete_maxsz, hdr); + encode_uint32(xdr, args->one_fs); +} + +static void encode_sequence(struct xdr_stream *xdr, + const struct nfs4_sequence_args *args, + struct compound_hdr *hdr) +{ + struct nfs4_session *session; + struct nfs4_slot_table *tp; + struct nfs4_slot *slot = args->sa_slot; + __be32 *p; + + tp = slot->table; + session = tp->session; + if (!session) + return; + + encode_op_hdr(xdr, OP_SEQUENCE, decode_sequence_maxsz, hdr); + + /* + * Sessionid + seqid + slotid + max slotid + cache_this + */ + dprintk("%s: sessionid=%u:%u:%u:%u seqid=%d slotid=%d " + "max_slotid=%d cache_this=%d\n", + __func__, + ((u32 *)session->sess_id.data)[0], + ((u32 *)session->sess_id.data)[1], + ((u32 *)session->sess_id.data)[2], + ((u32 *)session->sess_id.data)[3], + slot->seq_nr, slot->slot_nr, + tp->highest_used_slotid, args->sa_cache_this); + p = reserve_space(xdr, NFS4_MAX_SESSIONID_LEN + 16); + p = xdr_encode_opaque_fixed(p, session->sess_id.data, NFS4_MAX_SESSIONID_LEN); + *p++ = cpu_to_be32(slot->seq_nr); + *p++ = cpu_to_be32(slot->slot_nr); + *p++ = cpu_to_be32(tp->highest_used_slotid); + *p = cpu_to_be32(args->sa_cache_this); +} + +static void +encode_get_dir_delegation(struct xdr_stream *xdr, struct compound_hdr *hdr) +{ + struct timespec64 ts = { 0, 0 }; + u32 notifications[1] = { 0 }; + u32 attributes[1] = { 0 }; + __be32 *p; + + encode_op_hdr(xdr, OP_GET_DIR_DELEGATION, decode_get_dir_deleg_maxsz, hdr); + + /* We don't handle CB_RECALLABLE_OBJ_AVAIL yet. */ + xdr_stream_encode_bool(xdr, false); + + xdr_encode_bitmap4(xdr, notifications, ARRAY_SIZE(notifications)); + + /* Request no delay on attribute updates */ + p = reserve_space(xdr, 12 + 12); + p = xdr_encode_nfstime4(p, &ts); + xdr_encode_nfstime4(p, &ts); + + /* Requested child attributes */ + xdr_encode_bitmap4(xdr, attributes, ARRAY_SIZE(attributes)); + + /* Requested dir attributes */ + xdr_encode_bitmap4(xdr, attributes, ARRAY_SIZE(attributes)); +} + +static void +encode_getdeviceinfo(struct xdr_stream *xdr, + const struct nfs4_getdeviceinfo_args *args, + struct compound_hdr *hdr) +{ + __be32 *p; + + encode_op_hdr(xdr, OP_GETDEVICEINFO, decode_getdeviceinfo_maxsz, hdr); + p = reserve_space(xdr, NFS4_DEVICEID4_SIZE + 4 + 4); + p = xdr_encode_opaque_fixed(p, args->pdev->dev_id.data, + NFS4_DEVICEID4_SIZE); + *p++ = cpu_to_be32(args->pdev->layout_type); + *p++ = cpu_to_be32(args->pdev->maxcount); /* gdia_maxcount */ + + p = reserve_space(xdr, 4 + 4); + *p++ = cpu_to_be32(1); /* bitmap length */ + *p++ = cpu_to_be32(args->notify_types); +} + +static void +encode_layoutget(struct xdr_stream *xdr, + const struct nfs4_layoutget_args *args, + struct compound_hdr *hdr) +{ + __be32 *p; + + encode_op_hdr(xdr, OP_LAYOUTGET, decode_layoutget_maxsz, hdr); + p = reserve_space(xdr, 36); + *p++ = cpu_to_be32(0); /* Signal layout available */ + *p++ = cpu_to_be32(args->type); + *p++ = cpu_to_be32(args->range.iomode); + p = xdr_encode_hyper(p, args->range.offset); + p = xdr_encode_hyper(p, args->range.length); + p = xdr_encode_hyper(p, args->minlength); + encode_nfs4_stateid(xdr, &args->stateid); + encode_uint32(xdr, args->maxcount); + + dprintk("%s: 1st type:0x%x iomode:%d off:%lu len:%lu mc:%d\n", + __func__, + args->type, + args->range.iomode, + (unsigned long)args->range.offset, + (unsigned long)args->range.length, + args->maxcount); +} + +static int +encode_layoutcommit(struct xdr_stream *xdr, + struct inode *inode, + const struct nfs4_layoutcommit_args *args, + struct compound_hdr *hdr) +{ + __be32 *p; + + dprintk("%s: lbw: %llu type: %d\n", __func__, args->lastbytewritten, + NFS_SERVER(args->inode)->pnfs_curr_ld->id); + + encode_op_hdr(xdr, OP_LAYOUTCOMMIT, decode_layoutcommit_maxsz, hdr); + p = reserve_space(xdr, 20); + /* Only whole file layouts */ + p = xdr_encode_hyper(p, 0); /* offset */ + p = xdr_encode_hyper(p, args->lastbytewritten + 1); /* length */ + *p = cpu_to_be32(0); /* reclaim */ + encode_nfs4_stateid(xdr, &args->stateid); + if (args->lastbytewritten != U64_MAX) { + p = reserve_space(xdr, 20); + *p++ = cpu_to_be32(1); /* newoffset = TRUE */ + p = xdr_encode_hyper(p, args->lastbytewritten); + } else { + p = reserve_space(xdr, 12); + *p++ = cpu_to_be32(0); /* newoffset = FALSE */ + } + *p++ = cpu_to_be32(0); /* Never send time_modify_changed */ + *p++ = cpu_to_be32(NFS_SERVER(args->inode)->pnfs_curr_ld->id);/* type */ + + encode_uint32(xdr, args->layoutupdate_len); + if (args->layoutupdate_pages) + xdr_write_pages(xdr, args->layoutupdate_pages, 0, + args->layoutupdate_len); + + return 0; +} + +static void +encode_layoutreturn(struct xdr_stream *xdr, + const struct nfs4_layoutreturn_args *args, + struct compound_hdr *hdr) +{ + __be32 *p; + + encode_op_hdr(xdr, OP_LAYOUTRETURN, decode_layoutreturn_maxsz, hdr); + p = reserve_space(xdr, 16); + *p++ = cpu_to_be32(0); /* reclaim. always 0 for now */ + *p++ = cpu_to_be32(args->layout_type); + *p++ = cpu_to_be32(args->range.iomode); + *p = cpu_to_be32(RETURN_FILE); + p = reserve_space(xdr, 16); + p = xdr_encode_hyper(p, args->range.offset); + p = xdr_encode_hyper(p, args->range.length); + spin_lock(&args->inode->i_lock); + encode_nfs4_stateid(xdr, &args->stateid); + spin_unlock(&args->inode->i_lock); + if (args->ld_private->ops && args->ld_private->ops->encode) + args->ld_private->ops->encode(xdr, args, args->ld_private); + else + encode_uint32(xdr, 0); +} + +static int +encode_secinfo_no_name(struct xdr_stream *xdr, + const struct nfs41_secinfo_no_name_args *args, + struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_SECINFO_NO_NAME, decode_secinfo_no_name_maxsz, hdr); + encode_uint32(xdr, args->style); + return 0; +} + +static void encode_test_stateid(struct xdr_stream *xdr, + const struct nfs41_test_stateid_args *args, + struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_TEST_STATEID, decode_test_stateid_maxsz, hdr); + encode_uint32(xdr, 1); + encode_nfs4_stateid(xdr, &args->stateid); +} + +static void encode_free_stateid(struct xdr_stream *xdr, + const struct nfs41_free_stateid_args *args, + struct compound_hdr *hdr) +{ + encode_op_hdr(xdr, OP_FREE_STATEID, decode_free_stateid_maxsz, hdr); + encode_nfs4_stateid(xdr, &args->stateid); +} + +/* + * END OF "GENERIC" ENCODE ROUTINES. + */ + +static u32 nfs4_xdr_minorversion(const struct nfs4_sequence_args *args) +{ + struct nfs4_session *session = args->sa_slot->table->session; + if (session) + return session->clp->cl_mvops->minor_version; + return 0; +} + +/* + * Encode an ACCESS request + */ +static void nfs4_xdr_enc_access(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs4_accessargs *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + encode_access(xdr, args->access, &hdr); + if (args->bitmask) + encode_getfattr(xdr, args->bitmask, &hdr); + encode_nops(&hdr); +} + +/* + * Encode LOOKUP request + */ +static void nfs4_xdr_enc_lookup(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs4_lookup_arg *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->dir_fh, &hdr); + encode_lookup(xdr, args->name, &hdr); + encode_getfh(xdr, &hdr); + encode_getfattr(xdr, args->bitmask, &hdr); + encode_nops(&hdr); +} + +/* + * Encode LOOKUPP request + */ +static void nfs4_xdr_enc_lookupp(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs4_lookupp_arg *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + encode_lookupp(xdr, &hdr); + encode_getfh(xdr, &hdr); + encode_getfattr(xdr, args->bitmask, &hdr); + encode_nops(&hdr); +} + +/* + * Encode LOOKUP_ROOT request + */ +static void nfs4_xdr_enc_lookup_root(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs4_lookup_root_arg *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putrootfh(xdr, &hdr); + encode_getfh(xdr, &hdr); + encode_getfattr(xdr, args->bitmask, &hdr); + encode_nops(&hdr); +} + +/* + * Encode REMOVE request + */ +static void nfs4_xdr_enc_remove(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_removeargs *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + encode_remove(xdr, &args->name, &hdr); + encode_nops(&hdr); +} + +/* + * Encode RENAME request + */ +static void nfs4_xdr_enc_rename(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_renameargs *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->old_dir, &hdr); + encode_savefh(xdr, &hdr); + encode_putfh(xdr, args->new_dir, &hdr); + encode_rename(xdr, args->old_name, args->new_name, &hdr); + encode_nops(&hdr); +} + +/* + * Encode LINK request + */ +static void nfs4_xdr_enc_link(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs4_link_arg *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + encode_savefh(xdr, &hdr); + encode_putfh(xdr, args->dir_fh, &hdr); + encode_link(xdr, args->name, &hdr); + encode_restorefh(xdr, &hdr); + encode_getfattr(xdr, args->bitmask, &hdr); + encode_nops(&hdr); +} + +/* + * Encode CREATE request + */ +static void nfs4_xdr_enc_create(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs4_create_arg *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->dir_fh, &hdr); + encode_create(xdr, args, &hdr); + encode_getfh(xdr, &hdr); + encode_getfattr(xdr, args->bitmask, &hdr); + encode_nops(&hdr); +} + +/* + * Encode SYMLINK request + */ +static void nfs4_xdr_enc_symlink(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs4_create_arg *args = data; + + nfs4_xdr_enc_create(req, xdr, args); +} + +/* + * Encode GETATTR request + */ +static void nfs4_xdr_enc_getattr(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs4_getattr_arg *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + if (args->get_dir_deleg) + encode_get_dir_delegation(xdr, &hdr); + encode_getfattr(xdr, args->bitmask, &hdr); + encode_nops(&hdr); +} + +/* + * Encode a CLOSE request + */ +static void nfs4_xdr_enc_close(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_closeargs *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + if (args->lr_args) + encode_layoutreturn(xdr, args->lr_args, &hdr); + if (args->bitmask != NULL) + encode_getfattr(xdr, args->bitmask, &hdr); + encode_close(xdr, args, &hdr); + encode_nops(&hdr); +} + +/* + * Encode an OPEN request + */ +static void nfs4_xdr_enc_open(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_openargs *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + encode_open(xdr, args, &hdr); + encode_getfh(xdr, &hdr); + if (args->access) + encode_access(xdr, args->access, &hdr); + encode_getfattr_open(xdr, args->bitmask, args->open_bitmap, &hdr); + if (args->lg_args) { + encode_layoutget(xdr, args->lg_args, &hdr); + rpc_prepare_reply_pages(req, args->lg_args->layout.pages, 0, + args->lg_args->layout.pglen, + hdr.replen - pagepad_maxsz); + } + encode_nops(&hdr); +} + +/* + * Encode an OPEN_CONFIRM request + */ +static void nfs4_xdr_enc_open_confirm(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_open_confirmargs *args = data; + struct compound_hdr hdr = { + .nops = 0, + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_putfh(xdr, args->fh, &hdr); + encode_open_confirm(xdr, args, &hdr); + encode_nops(&hdr); +} + +/* + * Encode an OPEN request with no attributes. + */ +static void nfs4_xdr_enc_open_noattr(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_openargs *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + encode_open(xdr, args, &hdr); + if (args->access) + encode_access(xdr, args->access, &hdr); + encode_getfattr_open(xdr, args->bitmask, args->open_bitmap, &hdr); + if (args->lg_args) { + encode_layoutget(xdr, args->lg_args, &hdr); + rpc_prepare_reply_pages(req, args->lg_args->layout.pages, 0, + args->lg_args->layout.pglen, + hdr.replen - pagepad_maxsz); + } + encode_nops(&hdr); +} + +/* + * Encode an OPEN_DOWNGRADE request + */ +static void nfs4_xdr_enc_open_downgrade(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_closeargs *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + if (args->lr_args) + encode_layoutreturn(xdr, args->lr_args, &hdr); + encode_open_downgrade(xdr, args, &hdr); + encode_nops(&hdr); +} + +/* + * Encode a LOCK request + */ +static void nfs4_xdr_enc_lock(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_lock_args *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + encode_lock(xdr, args, &hdr); + encode_nops(&hdr); +} + +/* + * Encode a LOCKT request + */ +static void nfs4_xdr_enc_lockt(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_lockt_args *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + encode_lockt(xdr, args, &hdr); + encode_nops(&hdr); +} + +/* + * Encode a LOCKU request + */ +static void nfs4_xdr_enc_locku(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_locku_args *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + encode_locku(xdr, args, &hdr); + encode_nops(&hdr); +} + +#if defined(CONFIG_NFS_V4_0) +static void nfs4_xdr_enc_release_lockowner(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_release_lockowner_args *args = data; + struct compound_hdr hdr = { + .minorversion = 0, + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_release_lockowner(xdr, &args->lock_owner, &hdr); + encode_nops(&hdr); +} +#endif /* CONFIG_NFS_V4_0 */ + +/* + * Encode a READLINK request + */ +static void nfs4_xdr_enc_readlink(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs4_readlink *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + encode_readlink(xdr, args, req, &hdr); + + rpc_prepare_reply_pages(req, args->pages, args->pgbase, + args->pglen, hdr.replen - pagepad_maxsz); + encode_nops(&hdr); +} + +/* + * Encode a READDIR request + */ +static void nfs4_xdr_enc_readdir(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs4_readdir_arg *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + encode_readdir(xdr, args, req, &hdr); + + rpc_prepare_reply_pages(req, args->pages, args->pgbase, + args->count, hdr.replen - pagepad_maxsz); + encode_nops(&hdr); +} + +/* + * Encode a READ request + */ +static void nfs4_xdr_enc_read(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_pgio_args *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + encode_read(xdr, args, &hdr); + + rpc_prepare_reply_pages(req, args->pages, args->pgbase, + args->count, hdr.replen - pagepad_maxsz); + req->rq_rcv_buf.flags |= XDRBUF_READ; + encode_nops(&hdr); +} + +/* + * Encode an SETATTR request + */ +static void nfs4_xdr_enc_setattr(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_setattrargs *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + encode_setattr(xdr, args, args->server, &hdr); + encode_getfattr(xdr, args->bitmask, &hdr); + encode_nops(&hdr); +} + +/* + * Encode a GETACL request + */ +static void nfs4_xdr_enc_getacl(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_getaclargs *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + __u32 nfs4_acl_bitmap[2]; + uint32_t replen; + + nfs4_acltype_to_bitmap(args->acl_type, nfs4_acl_bitmap); + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + replen = hdr.replen + op_decode_hdr_maxsz; + encode_getattr(xdr, nfs4_acl_bitmap, NULL, + ARRAY_SIZE(nfs4_acl_bitmap), &hdr); + + rpc_prepare_reply_pages(req, args->acl_pages, 0, + args->acl_len, replen); + encode_nops(&hdr); +} + +/* + * Encode a WRITE request + */ +static void nfs4_xdr_enc_write(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_pgio_args *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + encode_write(xdr, args, &hdr); + req->rq_snd_buf.flags |= XDRBUF_WRITE; + if (args->bitmask) + encode_getfattr(xdr, args->bitmask, &hdr); + encode_nops(&hdr); +} + +/* + * a COMMIT request + */ +static void nfs4_xdr_enc_commit(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_commitargs *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + encode_commit(xdr, args, &hdr); + encode_nops(&hdr); +} + +/* + * FSINFO request + */ +static void nfs4_xdr_enc_fsinfo(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs4_fsinfo_arg *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + encode_fsinfo(xdr, args->bitmask, &hdr); + encode_nops(&hdr); +} + +/* + * a PATHCONF request + */ +static void nfs4_xdr_enc_pathconf(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs4_pathconf_arg *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + encode_getattr(xdr, nfs4_pathconf_bitmap, args->bitmask, + ARRAY_SIZE(nfs4_pathconf_bitmap), &hdr); + encode_nops(&hdr); +} + +/* + * a STATFS request + */ +static void nfs4_xdr_enc_statfs(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs4_statfs_arg *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + encode_getattr(xdr, nfs4_statfs_bitmap, args->bitmask, + ARRAY_SIZE(nfs4_statfs_bitmap), &hdr); + encode_nops(&hdr); +} + +/* + * GETATTR_BITMAP request + */ +static void nfs4_xdr_enc_server_caps(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs4_server_caps_arg *args = data; + const u32 *bitmask = args->bitmask; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fhandle, &hdr); + encode_getattr(xdr, bitmask, NULL, 3, &hdr); + encode_nops(&hdr); +} + +/* + * a RENEW request + */ +#if defined(CONFIG_NFS_V4_0) +static void nfs4_xdr_enc_renew(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) + +{ + const struct nfs_client *clp = data; + struct compound_hdr hdr = { + .nops = 0, + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_renew(xdr, clp->cl_clientid, &hdr); + encode_nops(&hdr); +} +#endif /* CONFIG_NFS_V4_0 */ + +/* + * a SETCLIENTID request + */ +static void nfs4_xdr_enc_setclientid(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs4_setclientid *sc = data; + struct compound_hdr hdr = { + .nops = 0, + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_setclientid(xdr, sc, &hdr); + encode_nops(&hdr); +} + +/* + * a SETCLIENTID_CONFIRM request + */ +static void nfs4_xdr_enc_setclientid_confirm(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs4_setclientid_res *arg = data; + struct compound_hdr hdr = { + .nops = 0, + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_setclientid_confirm(xdr, arg, &hdr); + encode_nops(&hdr); +} + +/* + * DELEGRETURN request + */ +static void nfs4_xdr_enc_delegreturn(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs4_delegreturnargs *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fhandle, &hdr); + if (args->lr_args) + encode_layoutreturn(xdr, args->lr_args, &hdr); + if (args->sattr_args) + encode_delegattr(xdr, args->stateid, args->sattr_args, &hdr); + if (args->bitmask) + encode_getfattr(xdr, args->bitmask, &hdr); + encode_delegreturn(xdr, args->stateid, &hdr); + encode_nops(&hdr); +} + +/* + * Encode FS_LOCATIONS request + */ +static void nfs4_xdr_enc_fs_locations(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs4_fs_locations_arg *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + uint32_t replen; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + if (args->migration) { + encode_putfh(xdr, args->fh, &hdr); + replen = hdr.replen; + encode_fs_locations(xdr, args->bitmask, &hdr); + if (args->renew) + encode_renew(xdr, args->clientid, &hdr); + } else { + encode_putfh(xdr, args->dir_fh, &hdr); + encode_lookup(xdr, args->name, &hdr); + replen = hdr.replen; + encode_fs_locations(xdr, args->bitmask, &hdr); + } + + rpc_prepare_reply_pages(req, (struct page **)&args->page, 0, + PAGE_SIZE, replen); + encode_nops(&hdr); +} + +/* + * Encode SECINFO request + */ +static void nfs4_xdr_enc_secinfo(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs4_secinfo_arg *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->dir_fh, &hdr); + encode_secinfo(xdr, args->name, &hdr); + encode_nops(&hdr); +} + +/* + * Encode FSID_PRESENT request + */ +static void nfs4_xdr_enc_fsid_present(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs4_fsid_present_arg *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + encode_getfh(xdr, &hdr); + if (args->renew) + encode_renew(xdr, args->clientid, &hdr); + encode_nops(&hdr); +} + +/* + * BIND_CONN_TO_SESSION request + */ +static void nfs4_xdr_enc_bind_conn_to_session(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs41_bind_conn_to_session_args *args = data; + struct compound_hdr hdr = { + .minorversion = args->client->cl_mvops->minor_version, + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_bind_conn_to_session(xdr, args, &hdr); + encode_nops(&hdr); +} + +/* + * EXCHANGE_ID request + */ +static void nfs4_xdr_enc_exchange_id(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs41_exchange_id_args *args = data; + struct compound_hdr hdr = { + .minorversion = args->client->cl_mvops->minor_version, + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_exchange_id(xdr, args, &hdr); + encode_nops(&hdr); +} + +/* + * a CREATE_SESSION request + */ +static void nfs4_xdr_enc_create_session(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs41_create_session_args *args = data; + struct compound_hdr hdr = { + .minorversion = args->client->cl_mvops->minor_version, + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_create_session(xdr, args, &hdr); + encode_nops(&hdr); +} + +/* + * a DESTROY_SESSION request + */ +static void nfs4_xdr_enc_destroy_session(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs4_session *session = data; + struct compound_hdr hdr = { + .minorversion = session->clp->cl_mvops->minor_version, + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_destroy_session(xdr, session, &hdr); + encode_nops(&hdr); +} + +/* + * a DESTROY_CLIENTID request + */ +static void nfs4_xdr_enc_destroy_clientid(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_client *clp = data; + struct compound_hdr hdr = { + .minorversion = clp->cl_mvops->minor_version, + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_destroy_clientid(xdr, clp->cl_clientid, &hdr); + encode_nops(&hdr); +} + +/* + * a SEQUENCE request + */ +static void nfs4_xdr_enc_sequence(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs4_sequence_args *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, args, &hdr); + encode_nops(&hdr); +} + +/* + * a GET_LEASE_TIME request + */ +static void nfs4_xdr_enc_get_lease_time(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs4_get_lease_time_args *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->la_seq_args), + }; + const u32 lease_bitmap[3] = { FATTR4_WORD0_LEASE_TIME }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->la_seq_args, &hdr); + encode_putrootfh(xdr, &hdr); + encode_fsinfo(xdr, lease_bitmap, &hdr); + encode_nops(&hdr); +} + +/* + * a RECLAIM_COMPLETE request + */ +static void nfs4_xdr_enc_reclaim_complete(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs41_reclaim_complete_args *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args) + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_reclaim_complete(xdr, args, &hdr); + encode_nops(&hdr); +} + +/* + * Encode GETDEVICEINFO request + */ +static void nfs4_xdr_enc_getdeviceinfo(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs4_getdeviceinfo_args *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + uint32_t replen; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + + replen = hdr.replen + op_decode_hdr_maxsz + 2; + + encode_getdeviceinfo(xdr, args, &hdr); + + /* set up reply kvec. device_addr4 opaque data is read into the + * pages */ + rpc_prepare_reply_pages(req, args->pdev->pages, args->pdev->pgbase, + args->pdev->pglen, replen); + encode_nops(&hdr); +} + +/* + * Encode LAYOUTGET request + */ +static void nfs4_xdr_enc_layoutget(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs4_layoutget_args *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, NFS_FH(args->inode), &hdr); + encode_layoutget(xdr, args, &hdr); + + rpc_prepare_reply_pages(req, args->layout.pages, 0, + args->layout.pglen, hdr.replen - pagepad_maxsz); + encode_nops(&hdr); +} + +/* + * Encode LAYOUTCOMMIT request + */ +static void nfs4_xdr_enc_layoutcommit(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *priv) +{ + const struct nfs4_layoutcommit_args *args = priv; + struct nfs4_layoutcommit_data *data = + container_of(args, struct nfs4_layoutcommit_data, args); + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, NFS_FH(args->inode), &hdr); + encode_layoutcommit(xdr, data->args.inode, args, &hdr); + encode_getfattr(xdr, args->bitmask, &hdr); + encode_nops(&hdr); +} + +/* + * Encode LAYOUTRETURN request + */ +static void nfs4_xdr_enc_layoutreturn(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs4_layoutreturn_args *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, NFS_FH(args->inode), &hdr); + encode_layoutreturn(xdr, args, &hdr); + encode_nops(&hdr); +} + +/* + * Encode SECINFO_NO_NAME request + */ +static void nfs4_xdr_enc_secinfo_no_name(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs41_secinfo_no_name_args *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putrootfh(xdr, &hdr); + encode_secinfo_no_name(xdr, args, &hdr); + encode_nops(&hdr); +} + +/* + * Encode TEST_STATEID request + */ +static void nfs4_xdr_enc_test_stateid(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs41_test_stateid_args *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_test_stateid(xdr, args, &hdr); + encode_nops(&hdr); +} + +/* + * Encode FREE_STATEID request + */ +static void nfs4_xdr_enc_free_stateid(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfs41_free_stateid_args *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_free_stateid(xdr, args, &hdr); + encode_nops(&hdr); +} + +static int decode_opaque_inline(struct xdr_stream *xdr, unsigned int *len, char **string) +{ + ssize_t ret = xdr_stream_decode_opaque_inline(xdr, (void **)string, + NFS4_OPAQUE_LIMIT); + if (unlikely(ret < 0)) + return -EIO; + *len = ret; + return 0; +} + +static int decode_compound_hdr(struct xdr_stream *xdr, struct compound_hdr *hdr) +{ + ssize_t ret; + void *ptr; + u32 tmp; + + if (xdr_stream_decode_u32(xdr, &tmp) < 0) + return -EIO; + hdr->status = tmp; + + ret = xdr_stream_decode_opaque_inline(xdr, &ptr, NFS4_OPAQUE_LIMIT); + if (ret < 0) + return -EIO; + hdr->taglen = ret; + hdr->tag = ptr; + + if (xdr_stream_decode_u32(xdr, &tmp) < 0) + return -EIO; + hdr->nops = tmp; + if (unlikely(hdr->nops < 1)) + return nfs4_stat_to_errno(hdr->status); + return 0; +} + +static bool __decode_op_hdr(struct xdr_stream *xdr, enum nfs_opnum4 expected, + int *nfs_retval) +{ + __be32 *p; + uint32_t opnum; + int32_t nfserr; + + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + goto out_overflow; + opnum = be32_to_cpup(p++); + if (unlikely(opnum != expected)) + goto out_bad_operation; + if (unlikely(*p != cpu_to_be32(NFS_OK))) + goto out_status; + *nfs_retval = 0; + return true; +out_status: + nfserr = be32_to_cpup(p); + trace_nfs4_xdr_status(xdr, opnum, nfserr); + *nfs_retval = nfs4_stat_to_errno(nfserr); + return true; +out_bad_operation: + trace_nfs4_xdr_bad_operation(xdr, opnum, expected); + *nfs_retval = -EREMOTEIO; + return false; +out_overflow: + *nfs_retval = -EIO; + return false; +} + +static int decode_op_hdr(struct xdr_stream *xdr, enum nfs_opnum4 expected) +{ + int retval; + + __decode_op_hdr(xdr, expected, &retval); + return retval; +} + +/* Dummy routine */ +static int decode_ace(struct xdr_stream *xdr, void *ace) +{ + __be32 *p; + unsigned int strlen; + char *str; + + p = xdr_inline_decode(xdr, 12); + if (unlikely(!p)) + return -EIO; + return decode_opaque_inline(xdr, &strlen, &str); +} + +static ssize_t +decode_bitmap4(struct xdr_stream *xdr, uint32_t *bitmap, size_t sz) +{ + ssize_t ret; + + ret = xdr_stream_decode_uint32_array(xdr, bitmap, sz); + if (likely(ret >= 0)) + return ret; + if (ret != -EMSGSIZE) + return -EIO; + return sz; +} + +static int decode_attr_bitmap(struct xdr_stream *xdr, uint32_t *bitmap) +{ + ssize_t ret; + ret = decode_bitmap4(xdr, bitmap, 3); + return ret < 0 ? ret : 0; +} + +static int decode_attr_length(struct xdr_stream *xdr, uint32_t *attrlen, unsigned int *savep) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + *attrlen = be32_to_cpup(p); + *savep = xdr_stream_pos(xdr); + return 0; +} + +static int decode_attr_supported(struct xdr_stream *xdr, uint32_t *bitmap, uint32_t *bitmask) +{ + if (likely(bitmap[0] & FATTR4_WORD0_SUPPORTED_ATTRS)) { + int ret; + ret = decode_attr_bitmap(xdr, bitmask); + if (unlikely(ret < 0)) + return ret; + bitmap[0] &= ~FATTR4_WORD0_SUPPORTED_ATTRS; + } else + bitmask[0] = bitmask[1] = bitmask[2] = 0; + dprintk("%s: bitmask=%08x:%08x:%08x\n", __func__, + bitmask[0], bitmask[1], bitmask[2]); + return 0; +} + +static int decode_attr_type(struct xdr_stream *xdr, uint32_t *bitmap, uint32_t *type) +{ + __be32 *p; + int ret = 0; + + *type = 0; + if (unlikely(bitmap[0] & (FATTR4_WORD0_TYPE - 1U))) + return -EIO; + if (likely(bitmap[0] & FATTR4_WORD0_TYPE)) { + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + *type = be32_to_cpup(p); + if (*type < NF4REG || *type > NF4NAMEDATTR) { + dprintk("%s: bad type %d\n", __func__, *type); + return -EIO; + } + bitmap[0] &= ~FATTR4_WORD0_TYPE; + ret = NFS_ATTR_FATTR_TYPE; + } + dprintk("%s: type=0%o\n", __func__, nfs_type2fmt[*type]); + return ret; +} + +static int decode_attr_fh_expire_type(struct xdr_stream *xdr, + uint32_t *bitmap, uint32_t *type) +{ + __be32 *p; + + *type = 0; + if (unlikely(bitmap[0] & (FATTR4_WORD0_FH_EXPIRE_TYPE - 1U))) + return -EIO; + if (likely(bitmap[0] & FATTR4_WORD0_FH_EXPIRE_TYPE)) { + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + *type = be32_to_cpup(p); + bitmap[0] &= ~FATTR4_WORD0_FH_EXPIRE_TYPE; + } + dprintk("%s: expire type=0x%x\n", __func__, *type); + return 0; +} + +static int decode_attr_change(struct xdr_stream *xdr, uint32_t *bitmap, uint64_t *change) +{ + __be32 *p; + int ret = 0; + + *change = 0; + if (unlikely(bitmap[0] & (FATTR4_WORD0_CHANGE - 1U))) + return -EIO; + if (likely(bitmap[0] & FATTR4_WORD0_CHANGE)) { + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + xdr_decode_hyper(p, change); + bitmap[0] &= ~FATTR4_WORD0_CHANGE; + ret = NFS_ATTR_FATTR_CHANGE; + } + dprintk("%s: change attribute=%Lu\n", __func__, + (unsigned long long)*change); + return ret; +} + +static int decode_attr_size(struct xdr_stream *xdr, uint32_t *bitmap, uint64_t *size) +{ + __be32 *p; + int ret = 0; + + *size = 0; + if (unlikely(bitmap[0] & (FATTR4_WORD0_SIZE - 1U))) + return -EIO; + if (likely(bitmap[0] & FATTR4_WORD0_SIZE)) { + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + xdr_decode_hyper(p, size); + bitmap[0] &= ~FATTR4_WORD0_SIZE; + ret = NFS_ATTR_FATTR_SIZE; + } + dprintk("%s: file size=%Lu\n", __func__, (unsigned long long)*size); + return ret; +} + +static int decode_attr_link_support(struct xdr_stream *xdr, uint32_t *bitmap, uint32_t *res) +{ + __be32 *p; + + *res = 0; + if (unlikely(bitmap[0] & (FATTR4_WORD0_LINK_SUPPORT - 1U))) + return -EIO; + if (likely(bitmap[0] & FATTR4_WORD0_LINK_SUPPORT)) { + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + *res = be32_to_cpup(p); + bitmap[0] &= ~FATTR4_WORD0_LINK_SUPPORT; + } + dprintk("%s: link support=%s\n", __func__, str_false_true(*res == 0)); + return 0; +} + +static int decode_attr_symlink_support(struct xdr_stream *xdr, uint32_t *bitmap, uint32_t *res) +{ + __be32 *p; + + *res = 0; + if (unlikely(bitmap[0] & (FATTR4_WORD0_SYMLINK_SUPPORT - 1U))) + return -EIO; + if (likely(bitmap[0] & FATTR4_WORD0_SYMLINK_SUPPORT)) { + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + *res = be32_to_cpup(p); + bitmap[0] &= ~FATTR4_WORD0_SYMLINK_SUPPORT; + } + dprintk("%s: symlink support=%s\n", __func__, str_false_true(*res == 0)); + return 0; +} + +static int decode_attr_fsid(struct xdr_stream *xdr, uint32_t *bitmap, struct nfs_fsid *fsid) +{ + __be32 *p; + int ret = 0; + + fsid->major = 0; + fsid->minor = 0; + if (unlikely(bitmap[0] & (FATTR4_WORD0_FSID - 1U))) + return -EIO; + if (likely(bitmap[0] & FATTR4_WORD0_FSID)) { + p = xdr_inline_decode(xdr, 16); + if (unlikely(!p)) + return -EIO; + p = xdr_decode_hyper(p, &fsid->major); + xdr_decode_hyper(p, &fsid->minor); + bitmap[0] &= ~FATTR4_WORD0_FSID; + ret = NFS_ATTR_FATTR_FSID; + } + dprintk("%s: fsid=(0x%Lx/0x%Lx)\n", __func__, + (unsigned long long)fsid->major, + (unsigned long long)fsid->minor); + return ret; +} + +static int decode_attr_lease_time(struct xdr_stream *xdr, uint32_t *bitmap, uint32_t *res) +{ + __be32 *p; + + *res = 60; + if (unlikely(bitmap[0] & (FATTR4_WORD0_LEASE_TIME - 1U))) + return -EIO; + if (likely(bitmap[0] & FATTR4_WORD0_LEASE_TIME)) { + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + *res = be32_to_cpup(p); + bitmap[0] &= ~FATTR4_WORD0_LEASE_TIME; + } + dprintk("%s: lease time=%u\n", __func__, (unsigned int)*res); + return 0; +} + +static int decode_attr_error(struct xdr_stream *xdr, uint32_t *bitmap, int32_t *res) +{ + __be32 *p; + + if (unlikely(bitmap[0] & (FATTR4_WORD0_RDATTR_ERROR - 1U))) + return -EIO; + if (likely(bitmap[0] & FATTR4_WORD0_RDATTR_ERROR)) { + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + bitmap[0] &= ~FATTR4_WORD0_RDATTR_ERROR; + *res = -be32_to_cpup(p); + } + return 0; +} + +static int decode_attr_exclcreat_supported(struct xdr_stream *xdr, + uint32_t *bitmap, uint32_t *bitmask) +{ + if (likely(bitmap[2] & FATTR4_WORD2_SUPPATTR_EXCLCREAT)) { + int ret; + ret = decode_attr_bitmap(xdr, bitmask); + if (unlikely(ret < 0)) + return ret; + bitmap[2] &= ~FATTR4_WORD2_SUPPATTR_EXCLCREAT; + } else + bitmask[0] = bitmask[1] = bitmask[2] = 0; + dprintk("%s: bitmask=%08x:%08x:%08x\n", __func__, + bitmask[0], bitmask[1], bitmask[2]); + return 0; +} + +static int decode_attr_filehandle(struct xdr_stream *xdr, uint32_t *bitmap, struct nfs_fh *fh) +{ + __be32 *p; + u32 len; + + if (fh != NULL) + memset(fh, 0, sizeof(*fh)); + + if (unlikely(bitmap[0] & (FATTR4_WORD0_FILEHANDLE - 1U))) + return -EIO; + if (likely(bitmap[0] & FATTR4_WORD0_FILEHANDLE)) { + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + len = be32_to_cpup(p); + if (len > NFS4_FHSIZE || len == 0) { + trace_nfs4_xdr_bad_filehandle(xdr, OP_READDIR, + NFS4ERR_BADHANDLE); + return -EREMOTEIO; + } + p = xdr_inline_decode(xdr, len); + if (unlikely(!p)) + return -EIO; + if (fh != NULL) { + memcpy(fh->data, p, len); + fh->size = len; + } + bitmap[0] &= ~FATTR4_WORD0_FILEHANDLE; + } + return 0; +} + +static int decode_attr_aclsupport(struct xdr_stream *xdr, uint32_t *bitmap, uint32_t *res) +{ + __be32 *p; + + *res = 0; + if (unlikely(bitmap[0] & (FATTR4_WORD0_ACLSUPPORT - 1U))) + return -EIO; + if (likely(bitmap[0] & FATTR4_WORD0_ACLSUPPORT)) { + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + *res = be32_to_cpup(p); + bitmap[0] &= ~FATTR4_WORD0_ACLSUPPORT; + } + dprintk("%s: ACLs supported=%u\n", __func__, (unsigned int)*res); + return 0; +} + +static int decode_attr_case_insensitive(struct xdr_stream *xdr, uint32_t *bitmap, uint32_t *res) +{ + __be32 *p; + + *res = 0; + if (unlikely(bitmap[0] & (FATTR4_WORD0_CASE_INSENSITIVE - 1U))) + return -EIO; + if (likely(bitmap[0] & FATTR4_WORD0_CASE_INSENSITIVE)) { + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + *res = be32_to_cpup(p); + bitmap[0] &= ~FATTR4_WORD0_CASE_INSENSITIVE; + } + dprintk("%s: case_insensitive=%s\n", __func__, str_false_true(*res == 0)); + return 0; +} + +static int decode_attr_case_preserving(struct xdr_stream *xdr, uint32_t *bitmap, uint32_t *res) +{ + __be32 *p; + + *res = 0; + if (unlikely(bitmap[0] & (FATTR4_WORD0_CASE_PRESERVING - 1U))) + return -EIO; + if (likely(bitmap[0] & FATTR4_WORD0_CASE_PRESERVING)) { + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + *res = be32_to_cpup(p); + bitmap[0] &= ~FATTR4_WORD0_CASE_PRESERVING; + } + dprintk("%s: case_preserving=%s\n", __func__, str_false_true(*res == 0)); + return 0; +} + +static int decode_attr_fileid(struct xdr_stream *xdr, uint32_t *bitmap, uint64_t *fileid) +{ + __be32 *p; + int ret = 0; + + *fileid = 0; + if (unlikely(bitmap[0] & (FATTR4_WORD0_FILEID - 1U))) + return -EIO; + if (likely(bitmap[0] & FATTR4_WORD0_FILEID)) { + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + xdr_decode_hyper(p, fileid); + bitmap[0] &= ~FATTR4_WORD0_FILEID; + ret = NFS_ATTR_FATTR_FILEID; + } + dprintk("%s: fileid=%Lu\n", __func__, (unsigned long long)*fileid); + return ret; +} + +static int decode_attr_mounted_on_fileid(struct xdr_stream *xdr, uint32_t *bitmap, uint64_t *fileid) +{ + __be32 *p; + int ret = 0; + + *fileid = 0; + if (unlikely(bitmap[1] & (FATTR4_WORD1_MOUNTED_ON_FILEID - 1U))) + return -EIO; + if (likely(bitmap[1] & FATTR4_WORD1_MOUNTED_ON_FILEID)) { + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + xdr_decode_hyper(p, fileid); + bitmap[1] &= ~FATTR4_WORD1_MOUNTED_ON_FILEID; + ret = NFS_ATTR_FATTR_MOUNTED_ON_FILEID; + } + dprintk("%s: fileid=%Lu\n", __func__, (unsigned long long)*fileid); + return ret; +} + +static int decode_attr_files_avail(struct xdr_stream *xdr, uint32_t *bitmap, uint64_t *res) +{ + __be32 *p; + int status = 0; + + *res = 0; + if (unlikely(bitmap[0] & (FATTR4_WORD0_FILES_AVAIL - 1U))) + return -EIO; + if (likely(bitmap[0] & FATTR4_WORD0_FILES_AVAIL)) { + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + xdr_decode_hyper(p, res); + bitmap[0] &= ~FATTR4_WORD0_FILES_AVAIL; + } + dprintk("%s: files avail=%Lu\n", __func__, (unsigned long long)*res); + return status; +} + +static int decode_attr_files_free(struct xdr_stream *xdr, uint32_t *bitmap, uint64_t *res) +{ + __be32 *p; + int status = 0; + + *res = 0; + if (unlikely(bitmap[0] & (FATTR4_WORD0_FILES_FREE - 1U))) + return -EIO; + if (likely(bitmap[0] & FATTR4_WORD0_FILES_FREE)) { + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + xdr_decode_hyper(p, res); + bitmap[0] &= ~FATTR4_WORD0_FILES_FREE; + } + dprintk("%s: files free=%Lu\n", __func__, (unsigned long long)*res); + return status; +} + +static int decode_attr_files_total(struct xdr_stream *xdr, uint32_t *bitmap, uint64_t *res) +{ + __be32 *p; + int status = 0; + + *res = 0; + if (unlikely(bitmap[0] & (FATTR4_WORD0_FILES_TOTAL - 1U))) + return -EIO; + if (likely(bitmap[0] & FATTR4_WORD0_FILES_TOTAL)) { + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + xdr_decode_hyper(p, res); + bitmap[0] &= ~FATTR4_WORD0_FILES_TOTAL; + } + dprintk("%s: files total=%Lu\n", __func__, (unsigned long long)*res); + return status; +} + +static int decode_pathname(struct xdr_stream *xdr, struct nfs4_pathname *path) +{ + u32 n; + __be32 *p; + int status = 0; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + n = be32_to_cpup(p); + if (n == 0) + goto root_path; + dprintk("pathname4: "); + if (n > NFS4_PATHNAME_MAXCOMPONENTS) { + dprintk("cannot parse %d components in path\n", n); + goto out_eio; + } + for (path->ncomponents = 0; path->ncomponents < n; path->ncomponents++) { + struct nfs4_string *component = &path->components[path->ncomponents]; + status = decode_opaque_inline(xdr, &component->len, &component->data); + if (unlikely(status != 0)) + goto out_eio; + ifdebug (XDR) + pr_cont("%s%.*s ", + (path->ncomponents != n ? "/ " : ""), + component->len, component->data); + } +out: + return status; +root_path: +/* a root pathname is sent as a zero component4 */ + path->ncomponents = 1; + path->components[0].len=0; + path->components[0].data=NULL; + dprintk("pathname4: /\n"); + goto out; +out_eio: + dprintk(" status %d", status); + status = -EIO; + goto out; +} + +static int decode_attr_fs_locations(struct xdr_stream *xdr, uint32_t *bitmap, struct nfs4_fs_locations *res) +{ + int n; + __be32 *p; + int status = -EIO; + + if (unlikely(bitmap[0] & (FATTR4_WORD0_FS_LOCATIONS -1U))) + goto out; + status = 0; + if (unlikely(!(bitmap[0] & FATTR4_WORD0_FS_LOCATIONS))) + goto out; + bitmap[0] &= ~FATTR4_WORD0_FS_LOCATIONS; + status = -EIO; + /* Ignore borken servers that return unrequested attrs */ + if (unlikely(res == NULL)) + goto out; + dprintk("%s: fsroot:\n", __func__); + status = decode_pathname(xdr, &res->fs_path); + if (unlikely(status != 0)) + goto out; + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + goto out_eio; + n = be32_to_cpup(p); + for (res->nlocations = 0; res->nlocations < n; res->nlocations++) { + u32 m; + struct nfs4_fs_location *loc; + + if (res->nlocations == NFS4_FS_LOCATIONS_MAXENTRIES) + break; + loc = &res->locations[res->nlocations]; + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + goto out_eio; + m = be32_to_cpup(p); + + dprintk("%s: servers:\n", __func__); + for (loc->nservers = 0; loc->nservers < m; loc->nservers++) { + struct nfs4_string *server; + + if (loc->nservers == NFS4_FS_LOCATION_MAXSERVERS) { + unsigned int i; + dprintk("%s: using first %u of %u servers " + "returned for location %u\n", + __func__, + NFS4_FS_LOCATION_MAXSERVERS, + m, res->nlocations); + for (i = loc->nservers; i < m; i++) { + unsigned int len; + char *data; + status = decode_opaque_inline(xdr, &len, &data); + if (unlikely(status != 0)) + goto out_eio; + } + break; + } + server = &loc->servers[loc->nservers]; + status = decode_opaque_inline(xdr, &server->len, &server->data); + if (unlikely(status != 0)) + goto out_eio; + dprintk("%s ", server->data); + } + status = decode_pathname(xdr, &loc->rootpath); + if (unlikely(status != 0)) + goto out_eio; + } + if (res->nlocations != 0) + status = NFS_ATTR_FATTR_V4_LOCATIONS; +out: + dprintk("%s: fs_locations done, error = %d\n", __func__, status); + return status; +out_eio: + status = -EIO; + goto out; +} + +static int decode_attr_maxfilesize(struct xdr_stream *xdr, uint32_t *bitmap, uint64_t *res) +{ + __be32 *p; + int status = 0; + + *res = 0; + if (unlikely(bitmap[0] & (FATTR4_WORD0_MAXFILESIZE - 1U))) + return -EIO; + if (likely(bitmap[0] & FATTR4_WORD0_MAXFILESIZE)) { + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + xdr_decode_hyper(p, res); + bitmap[0] &= ~FATTR4_WORD0_MAXFILESIZE; + } + dprintk("%s: maxfilesize=%Lu\n", __func__, (unsigned long long)*res); + return status; +} + +static int decode_attr_maxlink(struct xdr_stream *xdr, uint32_t *bitmap, uint32_t *maxlink) +{ + __be32 *p; + int status = 0; + + *maxlink = 1; + if (unlikely(bitmap[0] & (FATTR4_WORD0_MAXLINK - 1U))) + return -EIO; + if (likely(bitmap[0] & FATTR4_WORD0_MAXLINK)) { + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + *maxlink = be32_to_cpup(p); + bitmap[0] &= ~FATTR4_WORD0_MAXLINK; + } + dprintk("%s: maxlink=%u\n", __func__, *maxlink); + return status; +} + +static int decode_attr_maxname(struct xdr_stream *xdr, uint32_t *bitmap, uint32_t *maxname) +{ + __be32 *p; + int status = 0; + + *maxname = 1024; + if (unlikely(bitmap[0] & (FATTR4_WORD0_MAXNAME - 1U))) + return -EIO; + if (likely(bitmap[0] & FATTR4_WORD0_MAXNAME)) { + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + *maxname = be32_to_cpup(p); + bitmap[0] &= ~FATTR4_WORD0_MAXNAME; + } + dprintk("%s: maxname=%u\n", __func__, *maxname); + return status; +} + +static int decode_attr_maxread(struct xdr_stream *xdr, uint32_t *bitmap, uint32_t *res) +{ + __be32 *p; + int status = 0; + + *res = 1024; + if (unlikely(bitmap[0] & (FATTR4_WORD0_MAXREAD - 1U))) + return -EIO; + if (likely(bitmap[0] & FATTR4_WORD0_MAXREAD)) { + uint64_t maxread; + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + xdr_decode_hyper(p, &maxread); + if (maxread > 0x7FFFFFFF) + maxread = 0x7FFFFFFF; + *res = (uint32_t)maxread; + bitmap[0] &= ~FATTR4_WORD0_MAXREAD; + } + dprintk("%s: maxread=%lu\n", __func__, (unsigned long)*res); + return status; +} + +static int decode_attr_maxwrite(struct xdr_stream *xdr, uint32_t *bitmap, uint32_t *res) +{ + __be32 *p; + int status = 0; + + *res = 1024; + if (unlikely(bitmap[0] & (FATTR4_WORD0_MAXWRITE - 1U))) + return -EIO; + if (likely(bitmap[0] & FATTR4_WORD0_MAXWRITE)) { + uint64_t maxwrite; + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + xdr_decode_hyper(p, &maxwrite); + if (maxwrite > 0x7FFFFFFF) + maxwrite = 0x7FFFFFFF; + *res = (uint32_t)maxwrite; + bitmap[0] &= ~FATTR4_WORD0_MAXWRITE; + } + dprintk("%s: maxwrite=%lu\n", __func__, (unsigned long)*res); + return status; +} + +static int decode_attr_mode(struct xdr_stream *xdr, uint32_t *bitmap, umode_t *mode) +{ + uint32_t tmp; + __be32 *p; + int ret = 0; + + *mode = 0; + if (unlikely(bitmap[1] & (FATTR4_WORD1_MODE - 1U))) + return -EIO; + if (likely(bitmap[1] & FATTR4_WORD1_MODE)) { + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + tmp = be32_to_cpup(p); + *mode = tmp & ~S_IFMT; + bitmap[1] &= ~FATTR4_WORD1_MODE; + ret = NFS_ATTR_FATTR_MODE; + } + dprintk("%s: file mode=0%o\n", __func__, (unsigned int)*mode); + return ret; +} + +static int decode_attr_nlink(struct xdr_stream *xdr, uint32_t *bitmap, uint32_t *nlink) +{ + __be32 *p; + int ret = 0; + + *nlink = 1; + if (unlikely(bitmap[1] & (FATTR4_WORD1_NUMLINKS - 1U))) + return -EIO; + if (likely(bitmap[1] & FATTR4_WORD1_NUMLINKS)) { + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + *nlink = be32_to_cpup(p); + bitmap[1] &= ~FATTR4_WORD1_NUMLINKS; + ret = NFS_ATTR_FATTR_NLINK; + } + dprintk("%s: nlink=%u\n", __func__, (unsigned int)*nlink); + return ret; +} + +static ssize_t decode_nfs4_string(struct xdr_stream *xdr, + struct nfs4_string *name, gfp_t gfp_flags) +{ + ssize_t ret; + + ret = xdr_stream_decode_string_dup(xdr, &name->data, + XDR_MAX_NETOBJ, gfp_flags); + name->len = 0; + if (ret > 0) + name->len = ret; + return ret; +} + +static int decode_attr_owner(struct xdr_stream *xdr, uint32_t *bitmap, + const struct nfs_server *server, kuid_t *uid, + struct nfs4_string *owner_name) +{ + ssize_t len; + char *p; + + *uid = make_kuid(&init_user_ns, -2); + if (unlikely(bitmap[1] & (FATTR4_WORD1_OWNER - 1U))) + return -EIO; + if (!(bitmap[1] & FATTR4_WORD1_OWNER)) + return 0; + bitmap[1] &= ~FATTR4_WORD1_OWNER; + + if (owner_name != NULL) { + len = decode_nfs4_string(xdr, owner_name, GFP_NOIO); + if (len <= 0) + goto out; + dprintk("%s: name=%s\n", __func__, owner_name->data); + return NFS_ATTR_FATTR_OWNER_NAME; + } else { + len = xdr_stream_decode_opaque_inline(xdr, (void **)&p, + XDR_MAX_NETOBJ); + if (len <= 0 || nfs_map_name_to_uid(server, p, len, uid) != 0) + goto out; + dprintk("%s: uid=%d\n", __func__, (int)from_kuid(&init_user_ns, *uid)); + return NFS_ATTR_FATTR_OWNER; + } +out: + if (len == -EBADMSG) + return -EIO; + return 0; +} + +static int decode_attr_group(struct xdr_stream *xdr, uint32_t *bitmap, + const struct nfs_server *server, kgid_t *gid, + struct nfs4_string *group_name) +{ + ssize_t len; + char *p; + + *gid = make_kgid(&init_user_ns, -2); + if (unlikely(bitmap[1] & (FATTR4_WORD1_OWNER_GROUP - 1U))) + return -EIO; + if (!(bitmap[1] & FATTR4_WORD1_OWNER_GROUP)) + return 0; + bitmap[1] &= ~FATTR4_WORD1_OWNER_GROUP; + + if (group_name != NULL) { + len = decode_nfs4_string(xdr, group_name, GFP_NOIO); + if (len <= 0) + goto out; + dprintk("%s: name=%s\n", __func__, group_name->data); + return NFS_ATTR_FATTR_GROUP_NAME; + } else { + len = xdr_stream_decode_opaque_inline(xdr, (void **)&p, + XDR_MAX_NETOBJ); + if (len <= 0 || nfs_map_group_to_gid(server, p, len, gid) != 0) + goto out; + dprintk("%s: gid=%d\n", __func__, (int)from_kgid(&init_user_ns, *gid)); + return NFS_ATTR_FATTR_GROUP; + } +out: + if (len == -EBADMSG) + return -EIO; + return 0; +} + +static int decode_attr_rdev(struct xdr_stream *xdr, uint32_t *bitmap, dev_t *rdev) +{ + uint32_t major = 0, minor = 0; + __be32 *p; + int ret = 0; + + *rdev = MKDEV(0,0); + if (unlikely(bitmap[1] & (FATTR4_WORD1_RAWDEV - 1U))) + return -EIO; + if (likely(bitmap[1] & FATTR4_WORD1_RAWDEV)) { + dev_t tmp; + + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + major = be32_to_cpup(p++); + minor = be32_to_cpup(p); + tmp = MKDEV(major, minor); + if (MAJOR(tmp) == major && MINOR(tmp) == minor) + *rdev = tmp; + bitmap[1] &= ~ FATTR4_WORD1_RAWDEV; + ret = NFS_ATTR_FATTR_RDEV; + } + dprintk("%s: rdev=(0x%x:0x%x)\n", __func__, major, minor); + return ret; +} + +static int decode_attr_space_avail(struct xdr_stream *xdr, uint32_t *bitmap, uint64_t *res) +{ + __be32 *p; + int status = 0; + + *res = 0; + if (unlikely(bitmap[1] & (FATTR4_WORD1_SPACE_AVAIL - 1U))) + return -EIO; + if (likely(bitmap[1] & FATTR4_WORD1_SPACE_AVAIL)) { + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + xdr_decode_hyper(p, res); + bitmap[1] &= ~FATTR4_WORD1_SPACE_AVAIL; + } + dprintk("%s: space avail=%Lu\n", __func__, (unsigned long long)*res); + return status; +} + +static int decode_attr_space_free(struct xdr_stream *xdr, uint32_t *bitmap, uint64_t *res) +{ + __be32 *p; + int status = 0; + + *res = 0; + if (unlikely(bitmap[1] & (FATTR4_WORD1_SPACE_FREE - 1U))) + return -EIO; + if (likely(bitmap[1] & FATTR4_WORD1_SPACE_FREE)) { + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + xdr_decode_hyper(p, res); + bitmap[1] &= ~FATTR4_WORD1_SPACE_FREE; + } + dprintk("%s: space free=%Lu\n", __func__, (unsigned long long)*res); + return status; +} + +static int decode_attr_space_total(struct xdr_stream *xdr, uint32_t *bitmap, uint64_t *res) +{ + __be32 *p; + int status = 0; + + *res = 0; + if (unlikely(bitmap[1] & (FATTR4_WORD1_SPACE_TOTAL - 1U))) + return -EIO; + if (likely(bitmap[1] & FATTR4_WORD1_SPACE_TOTAL)) { + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + xdr_decode_hyper(p, res); + bitmap[1] &= ~FATTR4_WORD1_SPACE_TOTAL; + } + dprintk("%s: space total=%Lu\n", __func__, (unsigned long long)*res); + return status; +} + +static int decode_attr_space_used(struct xdr_stream *xdr, uint32_t *bitmap, uint64_t *used) +{ + __be32 *p; + int ret = 0; + + *used = 0; + if (unlikely(bitmap[1] & (FATTR4_WORD1_SPACE_USED - 1U))) + return -EIO; + if (likely(bitmap[1] & FATTR4_WORD1_SPACE_USED)) { + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + xdr_decode_hyper(p, used); + bitmap[1] &= ~FATTR4_WORD1_SPACE_USED; + ret = NFS_ATTR_FATTR_SPACE_USED; + } + dprintk("%s: space used=%Lu\n", __func__, + (unsigned long long)*used); + return ret; +} + +static __be32 * +xdr_decode_nfstime4(__be32 *p, struct timespec64 *t) +{ + __u64 sec; + + p = xdr_decode_hyper(p, &sec); + t-> tv_sec = sec; + t->tv_nsec = be32_to_cpup(p++); + return p; +} + +static int decode_attr_time(struct xdr_stream *xdr, struct timespec64 *time) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, nfstime4_maxsz << 2); + if (unlikely(!p)) + return -EIO; + xdr_decode_nfstime4(p, time); + return 0; +} + +static int decode_attr_time_access(struct xdr_stream *xdr, uint32_t *bitmap, struct timespec64 *time) +{ + int status = 0; + + time->tv_sec = 0; + time->tv_nsec = 0; + if (unlikely(bitmap[1] & (FATTR4_WORD1_TIME_ACCESS - 1U))) + return -EIO; + if (likely(bitmap[1] & FATTR4_WORD1_TIME_ACCESS)) { + status = decode_attr_time(xdr, time); + if (status == 0) + status = NFS_ATTR_FATTR_ATIME; + bitmap[1] &= ~FATTR4_WORD1_TIME_ACCESS; + } + dprintk("%s: atime=%lld\n", __func__, time->tv_sec); + return status; +} + +static int decode_attr_time_create(struct xdr_stream *xdr, uint32_t *bitmap, struct timespec64 *time) +{ + int status = 0; + + time->tv_sec = 0; + time->tv_nsec = 0; + if (unlikely(bitmap[1] & (FATTR4_WORD1_TIME_CREATE - 1U))) + return -EIO; + if (likely(bitmap[1] & FATTR4_WORD1_TIME_CREATE)) { + status = decode_attr_time(xdr, time); + if (status == 0) + status = NFS_ATTR_FATTR_BTIME; + bitmap[1] &= ~FATTR4_WORD1_TIME_CREATE; + } + dprintk("%s: btime=%lld\n", __func__, time->tv_sec); + return status; +} + +static int decode_attr_time_metadata(struct xdr_stream *xdr, uint32_t *bitmap, struct timespec64 *time) +{ + int status = 0; + + time->tv_sec = 0; + time->tv_nsec = 0; + if (unlikely(bitmap[1] & (FATTR4_WORD1_TIME_METADATA - 1U))) + return -EIO; + if (likely(bitmap[1] & FATTR4_WORD1_TIME_METADATA)) { + status = decode_attr_time(xdr, time); + if (status == 0) + status = NFS_ATTR_FATTR_CTIME; + bitmap[1] &= ~FATTR4_WORD1_TIME_METADATA; + } + dprintk("%s: ctime=%lld\n", __func__, time->tv_sec); + return status; +} + +static int decode_attr_time_delta(struct xdr_stream *xdr, uint32_t *bitmap, + struct timespec64 *time) +{ + int status = 0; + + time->tv_sec = 0; + time->tv_nsec = 0; + if (unlikely(bitmap[1] & (FATTR4_WORD1_TIME_DELTA - 1U))) + return -EIO; + if (likely(bitmap[1] & FATTR4_WORD1_TIME_DELTA)) { + status = decode_attr_time(xdr, time); + bitmap[1] &= ~FATTR4_WORD1_TIME_DELTA; + } + dprintk("%s: time_delta=%lld %ld\n", __func__, time->tv_sec, + time->tv_nsec); + return status; +} + +static int decode_attr_security_label(struct xdr_stream *xdr, uint32_t *bitmap, + struct nfs4_label *label) +{ + uint32_t pi = 0; + uint32_t lfs = 0; + __u32 len; + __be32 *p; + int status = 0; + + if (unlikely(bitmap[2] & (FATTR4_WORD2_SECURITY_LABEL - 1U))) + return -EIO; + if (likely(bitmap[2] & FATTR4_WORD2_SECURITY_LABEL)) { + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + lfs = be32_to_cpup(p++); + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + pi = be32_to_cpup(p++); + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + len = be32_to_cpup(p++); + p = xdr_inline_decode(xdr, len); + if (unlikely(!p)) + return -EIO; + bitmap[2] &= ~FATTR4_WORD2_SECURITY_LABEL; + if (len < NFS4_MAXLABELLEN) { + if (label && label->len) { + if (label->len < len) + return -ERANGE; + memcpy(label->label, p, len); + label->len = len; + label->pi = pi; + label->lfs = lfs; + status = NFS_ATTR_FATTR_V4_SECURITY_LABEL; + } + } else + printk(KERN_WARNING "%s: label too long (%u)!\n", + __func__, len); + if (label && label->label) + dprintk("%s: label=%.*s, len=%d, PI=%d, LFS=%d\n", + __func__, label->len, (char *)label->label, + label->len, label->pi, label->lfs); + } + return status; +} + +static int decode_attr_time_modify(struct xdr_stream *xdr, uint32_t *bitmap, struct timespec64 *time) +{ + int status = 0; + + time->tv_sec = 0; + time->tv_nsec = 0; + if (unlikely(bitmap[1] & (FATTR4_WORD1_TIME_MODIFY - 1U))) + return -EIO; + if (likely(bitmap[1] & FATTR4_WORD1_TIME_MODIFY)) { + status = decode_attr_time(xdr, time); + if (status == 0) + status = NFS_ATTR_FATTR_MTIME; + bitmap[1] &= ~FATTR4_WORD1_TIME_MODIFY; + } + dprintk("%s: mtime=%lld\n", __func__, time->tv_sec); + return status; +} + +static int decode_attr_xattrsupport(struct xdr_stream *xdr, uint32_t *bitmap, + uint32_t *res) +{ + __be32 *p; + + *res = 0; + if (unlikely(bitmap[2] & (FATTR4_WORD2_XATTR_SUPPORT - 1U))) + return -EIO; + if (likely(bitmap[2] & FATTR4_WORD2_XATTR_SUPPORT)) { + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + *res = be32_to_cpup(p); + bitmap[2] &= ~FATTR4_WORD2_XATTR_SUPPORT; + } + dprintk("%s: XATTR support=%s\n", __func__, str_false_true(*res == 0)); + return 0; +} + +static int decode_attr_open_arguments(struct xdr_stream *xdr, uint32_t *bitmap, + struct nfs4_open_caps *res) +{ + memset(res, 0, sizeof(*res)); + if (unlikely(bitmap[2] & (FATTR4_WORD2_OPEN_ARGUMENTS - 1U))) + return -EIO; + if (likely(bitmap[2] & FATTR4_WORD2_OPEN_ARGUMENTS)) { + if (decode_bitmap4(xdr, res->oa_share_access, ARRAY_SIZE(res->oa_share_access)) < 0) + return -EIO; + if (decode_bitmap4(xdr, res->oa_share_deny, ARRAY_SIZE(res->oa_share_deny)) < 0) + return -EIO; + if (decode_bitmap4(xdr, res->oa_share_access_want, ARRAY_SIZE(res->oa_share_access_want)) < 0) + return -EIO; + if (decode_bitmap4(xdr, res->oa_open_claim, ARRAY_SIZE(res->oa_open_claim)) < 0) + return -EIO; + if (decode_bitmap4(xdr, res->oa_createmode, ARRAY_SIZE(res->oa_createmode)) < 0) + return -EIO; + bitmap[2] &= ~FATTR4_WORD2_OPEN_ARGUMENTS; + } + return 0; +} + +static int verify_attr_len(struct xdr_stream *xdr, unsigned int savep, uint32_t attrlen) +{ + unsigned int attrwords = XDR_QUADLEN(attrlen); + unsigned int nwords = (xdr_stream_pos(xdr) - savep) >> 2; + + if (unlikely(attrwords != nwords)) { + dprintk("%s: server returned incorrect attribute length: " + "%u %c %u\n", + __func__, + attrwords << 2, + (attrwords < nwords) ? '<' : '>', + nwords << 2); + return -EIO; + } + return 0; +} + +static int decode_change_info(struct xdr_stream *xdr, struct nfs4_change_info *cinfo) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, 20); + if (unlikely(!p)) + return -EIO; + cinfo->atomic = be32_to_cpup(p++); + p = xdr_decode_hyper(p, &cinfo->before); + xdr_decode_hyper(p, &cinfo->after); + return 0; +} + +static int decode_access(struct xdr_stream *xdr, u32 *supported, u32 *access) +{ + __be32 *p; + uint32_t supp, acc; + int status; + + status = decode_op_hdr(xdr, OP_ACCESS); + if (status) + return status; + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + supp = be32_to_cpup(p++); + acc = be32_to_cpup(p); + *supported = supp; + *access = acc; + return 0; +} + +static int decode_stateid(struct xdr_stream *xdr, nfs4_stateid *stateid) +{ + return decode_opaque_fixed(xdr, stateid, NFS4_STATEID_SIZE); +} + +static int decode_open_stateid(struct xdr_stream *xdr, nfs4_stateid *stateid) +{ + stateid->type = NFS4_OPEN_STATEID_TYPE; + return decode_stateid(xdr, stateid); +} + +static int decode_lock_stateid(struct xdr_stream *xdr, nfs4_stateid *stateid) +{ + stateid->type = NFS4_LOCK_STATEID_TYPE; + return decode_stateid(xdr, stateid); +} + +static int decode_delegation_stateid(struct xdr_stream *xdr, nfs4_stateid *stateid) +{ + stateid->type = NFS4_DELEGATION_STATEID_TYPE; + return decode_stateid(xdr, stateid); +} + +static int decode_invalid_stateid(struct xdr_stream *xdr, nfs4_stateid *stateid) +{ + nfs4_stateid dummy; + + nfs4_stateid_copy(stateid, &invalid_stateid); + return decode_stateid(xdr, &dummy); +} + +static int decode_close(struct xdr_stream *xdr, struct nfs_closeres *res) +{ + int status; + + status = decode_op_hdr(xdr, OP_CLOSE); + if (status != -EIO) + nfs_increment_open_seqid(status, res->seqid); + if (!status) + status = decode_invalid_stateid(xdr, &res->stateid); + return status; +} + +static int decode_verifier(struct xdr_stream *xdr, void *verifier) +{ + return decode_opaque_fixed(xdr, verifier, NFS4_VERIFIER_SIZE); +} + +static int decode_write_verifier(struct xdr_stream *xdr, struct nfs_write_verifier *verifier) +{ + return decode_opaque_fixed(xdr, verifier->data, NFS4_VERIFIER_SIZE); +} + +static int decode_commit(struct xdr_stream *xdr, struct nfs_commitres *res) +{ + struct nfs_writeverf *verf = res->verf; + int status; + + status = decode_op_hdr(xdr, OP_COMMIT); + if (!status) + status = decode_write_verifier(xdr, &verf->verifier); + if (!status) + verf->committed = NFS_FILE_SYNC; + return status; +} + +static int decode_create(struct xdr_stream *xdr, struct nfs4_change_info *cinfo) +{ + __be32 *p; + uint32_t bmlen; + int status; + + status = decode_op_hdr(xdr, OP_CREATE); + if (status) + return status; + if ((status = decode_change_info(xdr, cinfo))) + return status; + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + bmlen = be32_to_cpup(p); + p = xdr_inline_decode(xdr, bmlen << 2); + if (likely(p)) + return 0; + return -EIO; +} + +static int decode_server_caps(struct xdr_stream *xdr, struct nfs4_server_caps_res *res) +{ + unsigned int savep; + uint32_t attrlen, bitmap[3] = {0}; + int status; + + if ((status = decode_op_hdr(xdr, OP_GETATTR)) != 0) + goto xdr_error; + if ((status = decode_attr_bitmap(xdr, bitmap)) != 0) + goto xdr_error; + if ((status = decode_attr_length(xdr, &attrlen, &savep)) != 0) + goto xdr_error; + if ((status = decode_attr_supported(xdr, bitmap, res->attr_bitmask)) != 0) + goto xdr_error; + if ((status = decode_attr_fh_expire_type(xdr, bitmap, + &res->fh_expire_type)) != 0) + goto xdr_error; + if ((status = decode_attr_link_support(xdr, bitmap, &res->has_links)) != 0) + goto xdr_error; + if ((status = decode_attr_symlink_support(xdr, bitmap, &res->has_symlinks)) != 0) + goto xdr_error; + if ((status = decode_attr_aclsupport(xdr, bitmap, &res->acl_bitmask)) != 0) + goto xdr_error; + if ((status = decode_attr_case_insensitive(xdr, bitmap, &res->case_insensitive)) != 0) + goto xdr_error; + if ((status = decode_attr_case_preserving(xdr, bitmap, &res->case_preserving)) != 0) + goto xdr_error; + if ((status = decode_attr_exclcreat_supported(xdr, bitmap, + res->exclcreat_bitmask)) != 0) + goto xdr_error; + if ((status = decode_attr_open_arguments(xdr, bitmap, &res->open_caps)) != 0) + goto xdr_error; + status = verify_attr_len(xdr, savep, attrlen); +xdr_error: + dprintk("%s: xdr returned %d!\n", __func__, -status); + return status; +} + +static int decode_statfs(struct xdr_stream *xdr, struct nfs_fsstat *fsstat) +{ + unsigned int savep; + uint32_t attrlen, bitmap[3] = {0}; + int status; + + if ((status = decode_op_hdr(xdr, OP_GETATTR)) != 0) + goto xdr_error; + if ((status = decode_attr_bitmap(xdr, bitmap)) != 0) + goto xdr_error; + if ((status = decode_attr_length(xdr, &attrlen, &savep)) != 0) + goto xdr_error; + + if ((status = decode_attr_files_avail(xdr, bitmap, &fsstat->afiles)) != 0) + goto xdr_error; + if ((status = decode_attr_files_free(xdr, bitmap, &fsstat->ffiles)) != 0) + goto xdr_error; + if ((status = decode_attr_files_total(xdr, bitmap, &fsstat->tfiles)) != 0) + goto xdr_error; + + status = -EIO; + if (unlikely(bitmap[0])) + goto xdr_error; + + if ((status = decode_attr_space_avail(xdr, bitmap, &fsstat->abytes)) != 0) + goto xdr_error; + if ((status = decode_attr_space_free(xdr, bitmap, &fsstat->fbytes)) != 0) + goto xdr_error; + if ((status = decode_attr_space_total(xdr, bitmap, &fsstat->tbytes)) != 0) + goto xdr_error; + + status = verify_attr_len(xdr, savep, attrlen); +xdr_error: + dprintk("%s: xdr returned %d!\n", __func__, -status); + return status; +} + +static int decode_pathconf(struct xdr_stream *xdr, struct nfs_pathconf *pathconf) +{ + unsigned int savep; + uint32_t attrlen, bitmap[3] = {0}; + int status; + + if ((status = decode_op_hdr(xdr, OP_GETATTR)) != 0) + goto xdr_error; + if ((status = decode_attr_bitmap(xdr, bitmap)) != 0) + goto xdr_error; + if ((status = decode_attr_length(xdr, &attrlen, &savep)) != 0) + goto xdr_error; + + if ((status = decode_attr_maxlink(xdr, bitmap, &pathconf->max_link)) != 0) + goto xdr_error; + if ((status = decode_attr_maxname(xdr, bitmap, &pathconf->max_namelen)) != 0) + goto xdr_error; + + status = verify_attr_len(xdr, savep, attrlen); +xdr_error: + dprintk("%s: xdr returned %d!\n", __func__, -status); + return status; +} + +static int decode_threshold_hint(struct xdr_stream *xdr, + uint32_t *bitmap, + uint64_t *res, + uint32_t hint_bit) +{ + __be32 *p; + + *res = 0; + if (likely(bitmap[0] & hint_bit)) { + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + xdr_decode_hyper(p, res); + } + return 0; +} + +static int decode_first_threshold_item4(struct xdr_stream *xdr, + struct nfs4_threshold *res) +{ + __be32 *p; + unsigned int savep; + uint32_t bitmap[3] = {0,}, attrlen; + int status; + + /* layout type */ + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + res->l_type = be32_to_cpup(p); + + /* thi_hintset bitmap */ + status = decode_attr_bitmap(xdr, bitmap); + if (status < 0) + goto xdr_error; + + /* thi_hintlist length */ + status = decode_attr_length(xdr, &attrlen, &savep); + if (status < 0) + goto xdr_error; + /* thi_hintlist */ + status = decode_threshold_hint(xdr, bitmap, &res->rd_sz, THRESHOLD_RD); + if (status < 0) + goto xdr_error; + status = decode_threshold_hint(xdr, bitmap, &res->wr_sz, THRESHOLD_WR); + if (status < 0) + goto xdr_error; + status = decode_threshold_hint(xdr, bitmap, &res->rd_io_sz, + THRESHOLD_RD_IO); + if (status < 0) + goto xdr_error; + status = decode_threshold_hint(xdr, bitmap, &res->wr_io_sz, + THRESHOLD_WR_IO); + if (status < 0) + goto xdr_error; + + status = verify_attr_len(xdr, savep, attrlen); + res->bm = bitmap[0]; + + dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n", + __func__, res->bm, res->rd_sz, res->wr_sz, res->rd_io_sz, + res->wr_io_sz); +xdr_error: + dprintk("%s ret=%d!\n", __func__, status); + return status; +} + +/* + * Thresholds on pNFS direct I/O vrs MDS I/O + */ +static int decode_attr_mdsthreshold(struct xdr_stream *xdr, + uint32_t *bitmap, + struct nfs4_threshold *res) +{ + __be32 *p; + int status = 0; + uint32_t num; + + if (unlikely(bitmap[2] & (FATTR4_WORD2_MDSTHRESHOLD - 1U))) + return -EIO; + if (bitmap[2] & FATTR4_WORD2_MDSTHRESHOLD) { + /* Did the server return an unrequested attribute? */ + if (unlikely(res == NULL)) + return -EREMOTEIO; + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + num = be32_to_cpup(p); + if (num == 0) + return 0; + if (num > 1) + printk(KERN_INFO "%s: Warning: Multiple pNFS layout " + "drivers per filesystem not supported\n", + __func__); + + status = decode_first_threshold_item4(xdr, res); + bitmap[2] &= ~FATTR4_WORD2_MDSTHRESHOLD; + } + return status; +} + +static int decode_getfattr_attrs(struct xdr_stream *xdr, uint32_t *bitmap, + struct nfs_fattr *fattr, struct nfs_fh *fh, + struct nfs4_fs_locations *fs_loc, const struct nfs_server *server) +{ + int status; + umode_t fmode = 0; + uint32_t type; + int32_t err; + + status = decode_attr_type(xdr, bitmap, &type); + if (status < 0) + goto xdr_error; + fattr->mode = 0; + if (status != 0) { + fattr->mode |= nfs_type2fmt[type]; + fattr->valid |= status; + } + + status = decode_attr_change(xdr, bitmap, &fattr->change_attr); + if (status < 0) + goto xdr_error; + fattr->valid |= status; + + status = decode_attr_size(xdr, bitmap, &fattr->size); + if (status < 0) + goto xdr_error; + fattr->valid |= status; + + status = decode_attr_fsid(xdr, bitmap, &fattr->fsid); + if (status < 0) + goto xdr_error; + fattr->valid |= status; + + err = 0; + status = decode_attr_error(xdr, bitmap, &err); + if (status < 0) + goto xdr_error; + + status = decode_attr_filehandle(xdr, bitmap, fh); + if (status < 0) + goto xdr_error; + + status = decode_attr_fileid(xdr, bitmap, &fattr->fileid); + if (status < 0) + goto xdr_error; + fattr->valid |= status; + + status = decode_attr_fs_locations(xdr, bitmap, fs_loc); + if (status < 0) + goto xdr_error; + fattr->valid |= status; + + status = -EIO; + if (unlikely(bitmap[0])) + goto xdr_error; + + status = decode_attr_mode(xdr, bitmap, &fmode); + if (status < 0) + goto xdr_error; + if (status != 0) { + fattr->mode |= fmode; + fattr->valid |= status; + } + + status = decode_attr_nlink(xdr, bitmap, &fattr->nlink); + if (status < 0) + goto xdr_error; + fattr->valid |= status; + + status = decode_attr_owner(xdr, bitmap, server, &fattr->uid, fattr->owner_name); + if (status < 0) + goto xdr_error; + fattr->valid |= status; + + status = decode_attr_group(xdr, bitmap, server, &fattr->gid, fattr->group_name); + if (status < 0) + goto xdr_error; + fattr->valid |= status; + + status = decode_attr_rdev(xdr, bitmap, &fattr->rdev); + if (status < 0) + goto xdr_error; + fattr->valid |= status; + + status = decode_attr_space_used(xdr, bitmap, &fattr->du.nfs3.used); + if (status < 0) + goto xdr_error; + fattr->valid |= status; + + status = decode_attr_time_access(xdr, bitmap, &fattr->atime); + if (status < 0) + goto xdr_error; + fattr->valid |= status; + + status = decode_attr_time_create(xdr, bitmap, &fattr->btime); + if (status < 0) + goto xdr_error; + fattr->valid |= status; + + status = decode_attr_time_metadata(xdr, bitmap, &fattr->ctime); + if (status < 0) + goto xdr_error; + fattr->valid |= status; + + status = decode_attr_time_modify(xdr, bitmap, &fattr->mtime); + if (status < 0) + goto xdr_error; + fattr->valid |= status; + + status = decode_attr_mounted_on_fileid(xdr, bitmap, &fattr->mounted_on_fileid); + if (status < 0) + goto xdr_error; + fattr->valid |= status; + + status = -EIO; + if (unlikely(bitmap[1])) + goto xdr_error; + + status = decode_attr_mdsthreshold(xdr, bitmap, fattr->mdsthreshold); + if (status < 0) + goto xdr_error; + + status = decode_attr_security_label(xdr, bitmap, fattr->label); + if (status < 0) + goto xdr_error; + fattr->valid |= status; + +xdr_error: + dprintk("%s: xdr returned %d\n", __func__, -status); + return status; +} + +static int decode_getfattr_generic(struct xdr_stream *xdr, struct nfs_fattr *fattr, + struct nfs_fh *fh, struct nfs4_fs_locations *fs_loc, + const struct nfs_server *server) +{ + unsigned int savep; + uint32_t attrlen, + bitmap[3] = {0}; + int status; + + status = decode_op_hdr(xdr, OP_GETATTR); + if (status < 0) + goto xdr_error; + + status = decode_attr_bitmap(xdr, bitmap); + if (status < 0) + goto xdr_error; + + status = decode_attr_length(xdr, &attrlen, &savep); + if (status < 0) + goto xdr_error; + + status = decode_getfattr_attrs(xdr, bitmap, fattr, fh, fs_loc, server); + if (status < 0) + goto xdr_error; + + status = verify_attr_len(xdr, savep, attrlen); +xdr_error: + dprintk("%s: xdr returned %d\n", __func__, -status); + return status; +} + +static int decode_getfattr(struct xdr_stream *xdr, struct nfs_fattr *fattr, + const struct nfs_server *server) +{ + return decode_getfattr_generic(xdr, fattr, NULL, NULL, server); +} + +/* + * Decode potentially multiple layout types. + */ +static int decode_pnfs_layout_types(struct xdr_stream *xdr, + struct nfs_fsinfo *fsinfo) +{ + __be32 *p; + uint32_t i; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + fsinfo->nlayouttypes = be32_to_cpup(p); + + /* pNFS is not supported by the underlying file system */ + if (fsinfo->nlayouttypes == 0) + return 0; + + /* Decode and set first layout type, move xdr->p past unused types */ + p = xdr_inline_decode(xdr, fsinfo->nlayouttypes * 4); + if (unlikely(!p)) + return -EIO; + + /* If we get too many, then just cap it at the max */ + if (fsinfo->nlayouttypes > NFS_MAX_LAYOUT_TYPES) { + printk(KERN_INFO "NFS: %s: Warning: Too many (%u) pNFS layout types\n", + __func__, fsinfo->nlayouttypes); + fsinfo->nlayouttypes = NFS_MAX_LAYOUT_TYPES; + } + + for(i = 0; i < fsinfo->nlayouttypes; ++i) + fsinfo->layouttype[i] = be32_to_cpup(p++); + return 0; +} + +/* + * The type of file system exported. + * Note we must ensure that layouttype is set in any non-error case. + */ +static int decode_attr_pnfstype(struct xdr_stream *xdr, uint32_t *bitmap, + struct nfs_fsinfo *fsinfo) +{ + int status = 0; + + dprintk("%s: bitmap is %x\n", __func__, bitmap[1]); + if (unlikely(bitmap[1] & (FATTR4_WORD1_FS_LAYOUT_TYPES - 1U))) + return -EIO; + if (bitmap[1] & FATTR4_WORD1_FS_LAYOUT_TYPES) { + status = decode_pnfs_layout_types(xdr, fsinfo); + bitmap[1] &= ~FATTR4_WORD1_FS_LAYOUT_TYPES; + } + return status; +} + +/* + * The preferred block size for layout directed io + */ +static int decode_attr_layout_blksize(struct xdr_stream *xdr, uint32_t *bitmap, + uint32_t *res) +{ + __be32 *p; + + dprintk("%s: bitmap is %x\n", __func__, bitmap[2]); + *res = 0; + if (bitmap[2] & FATTR4_WORD2_LAYOUT_BLKSIZE) { + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + *res = be32_to_cpup(p); + bitmap[2] &= ~FATTR4_WORD2_LAYOUT_BLKSIZE; + } + return 0; +} + +/* + * The granularity of a CLONE operation. + */ +static int decode_attr_clone_blksize(struct xdr_stream *xdr, uint32_t *bitmap, + uint32_t *res) +{ + __be32 *p; + + dprintk("%s: bitmap is %x\n", __func__, bitmap[2]); + *res = 0; + if (bitmap[2] & FATTR4_WORD2_CLONE_BLKSIZE) { + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + *res = be32_to_cpup(p); + bitmap[2] &= ~FATTR4_WORD2_CLONE_BLKSIZE; + } + return 0; +} + +static int decode_attr_change_attr_type(struct xdr_stream *xdr, + uint32_t *bitmap, + enum nfs4_change_attr_type *res) +{ + u32 tmp = NFS4_CHANGE_TYPE_IS_UNDEFINED; + + dprintk("%s: bitmap is %x\n", __func__, bitmap[2]); + if (bitmap[2] & FATTR4_WORD2_CHANGE_ATTR_TYPE) { + if (xdr_stream_decode_u32(xdr, &tmp)) + return -EIO; + bitmap[2] &= ~FATTR4_WORD2_CHANGE_ATTR_TYPE; + } + + switch(tmp) { + case NFS4_CHANGE_TYPE_IS_MONOTONIC_INCR: + case NFS4_CHANGE_TYPE_IS_VERSION_COUNTER: + case NFS4_CHANGE_TYPE_IS_VERSION_COUNTER_NOPNFS: + case NFS4_CHANGE_TYPE_IS_TIME_METADATA: + *res = tmp; + break; + default: + *res = NFS4_CHANGE_TYPE_IS_UNDEFINED; + } + return 0; +} + +static int decode_fsinfo(struct xdr_stream *xdr, struct nfs_fsinfo *fsinfo) +{ + unsigned int savep; + uint32_t attrlen, bitmap[3]; + int status; + + if ((status = decode_op_hdr(xdr, OP_GETATTR)) != 0) + goto xdr_error; + if ((status = decode_attr_bitmap(xdr, bitmap)) != 0) + goto xdr_error; + if ((status = decode_attr_length(xdr, &attrlen, &savep)) != 0) + goto xdr_error; + + fsinfo->rtmult = fsinfo->wtmult = 512; /* ??? */ + + if ((status = decode_attr_lease_time(xdr, bitmap, &fsinfo->lease_time)) != 0) + goto xdr_error; + if ((status = decode_attr_maxfilesize(xdr, bitmap, &fsinfo->maxfilesize)) != 0) + goto xdr_error; + if ((status = decode_attr_maxread(xdr, bitmap, &fsinfo->rtmax)) != 0) + goto xdr_error; + fsinfo->rtpref = fsinfo->dtpref = fsinfo->rtmax; + if ((status = decode_attr_maxwrite(xdr, bitmap, &fsinfo->wtmax)) != 0) + goto xdr_error; + fsinfo->wtpref = fsinfo->wtmax; + + status = -EIO; + if (unlikely(bitmap[0])) + goto xdr_error; + + status = decode_attr_time_delta(xdr, bitmap, &fsinfo->time_delta); + if (status != 0) + goto xdr_error; + status = decode_attr_pnfstype(xdr, bitmap, fsinfo); + if (status != 0) + goto xdr_error; + + status = -EIO; + if (unlikely(bitmap[1])) + goto xdr_error; + + status = decode_attr_layout_blksize(xdr, bitmap, &fsinfo->blksize); + if (status) + goto xdr_error; + status = decode_attr_clone_blksize(xdr, bitmap, &fsinfo->clone_blksize); + if (status) + goto xdr_error; + + status = decode_attr_change_attr_type(xdr, bitmap, + &fsinfo->change_attr_type); + if (status) + goto xdr_error; + + status = decode_attr_xattrsupport(xdr, bitmap, + &fsinfo->xattr_support); + if (status) + goto xdr_error; + + status = verify_attr_len(xdr, savep, attrlen); +xdr_error: + dprintk("%s: xdr returned %d!\n", __func__, -status); + return status; +} + +static int decode_getfh(struct xdr_stream *xdr, struct nfs_fh *fh) +{ + __be32 *p; + uint32_t len; + int status; + + /* Zero handle first to allow comparisons */ + memset(fh, 0, sizeof(*fh)); + + status = decode_op_hdr(xdr, OP_GETFH); + if (status) + return status; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + len = be32_to_cpup(p); + if (len > NFS4_FHSIZE || len == 0) { + trace_nfs4_xdr_bad_filehandle(xdr, OP_GETFH, NFS4ERR_BADHANDLE); + return -EREMOTEIO; + } + fh->size = len; + p = xdr_inline_decode(xdr, len); + if (unlikely(!p)) + return -EIO; + memcpy(fh->data, p, len); + return 0; +} + +static int decode_link(struct xdr_stream *xdr, struct nfs4_change_info *cinfo) +{ + int status; + + status = decode_op_hdr(xdr, OP_LINK); + if (status) + return status; + return decode_change_info(xdr, cinfo); +} + +/* + * We create the owner, so we know a proper owner.id length is 4. + */ +static int decode_lock_denied(struct xdr_stream *xdr, struct file_lock *fl) +{ + uint64_t offset, length, clientid; + __be32 *p; + uint32_t namelen, type; + + p = xdr_inline_decode(xdr, 32); /* read 32 bytes */ + if (unlikely(!p)) + return -EIO; + p = xdr_decode_hyper(p, &offset); /* read 2 8-byte long words */ + p = xdr_decode_hyper(p, &length); + type = be32_to_cpup(p++); /* 4 byte read */ + if (fl != NULL) { /* manipulate file lock */ + fl->fl_start = (loff_t)offset; + fl->fl_end = fl->fl_start + (loff_t)length - 1; + if (length == ~(uint64_t)0) + fl->fl_end = OFFSET_MAX; + fl->c.flc_type = F_WRLCK; + if (type & 1) + fl->c.flc_type = F_RDLCK; + fl->c.flc_pid = 0; + } + p = xdr_decode_hyper(p, &clientid); /* read 8 bytes */ + namelen = be32_to_cpup(p); /* read 4 bytes */ /* have read all 32 bytes now */ + p = xdr_inline_decode(xdr, namelen); /* variable size field */ + if (likely(!p)) + return -EIO; + return -NFS4ERR_DENIED; +} + +static int decode_lock(struct xdr_stream *xdr, struct nfs_lock_res *res) +{ + int status; + + status = decode_op_hdr(xdr, OP_LOCK); + if (status == -EIO) + goto out; + if (status == 0) { + status = decode_lock_stateid(xdr, &res->stateid); + if (unlikely(status)) + goto out; + } else if (status == -NFS4ERR_DENIED) + status = decode_lock_denied(xdr, NULL); + if (res->open_seqid != NULL) + nfs_increment_open_seqid(status, res->open_seqid); + nfs_increment_lock_seqid(status, res->lock_seqid); +out: + return status; +} + +static int decode_lockt(struct xdr_stream *xdr, struct nfs_lockt_res *res) +{ + int status; + status = decode_op_hdr(xdr, OP_LOCKT); + if (status == -NFS4ERR_DENIED) + return decode_lock_denied(xdr, res->denied); + return status; +} + +static int decode_locku(struct xdr_stream *xdr, struct nfs_locku_res *res) +{ + int status; + + status = decode_op_hdr(xdr, OP_LOCKU); + if (status != -EIO) + nfs_increment_lock_seqid(status, res->seqid); + if (status == 0) + status = decode_lock_stateid(xdr, &res->stateid); + return status; +} + +#if defined(CONFIG_NFS_V4_0) +static int decode_release_lockowner(struct xdr_stream *xdr) +{ + return decode_op_hdr(xdr, OP_RELEASE_LOCKOWNER); +} +#endif /* CONFIG_NFS_V4_0 */ + +static int decode_lookup(struct xdr_stream *xdr) +{ + return decode_op_hdr(xdr, OP_LOOKUP); +} + +static int decode_lookupp(struct xdr_stream *xdr) +{ + return decode_op_hdr(xdr, OP_LOOKUPP); +} + +/* This is too sick! */ +static int decode_space_limit(struct xdr_stream *xdr, + unsigned long *pagemod_limit) +{ + __be32 *p; + uint32_t limit_type, nblocks, blocksize; + u64 maxsize = 0; + + p = xdr_inline_decode(xdr, 12); + if (unlikely(!p)) + return -EIO; + limit_type = be32_to_cpup(p++); + switch (limit_type) { + case NFS4_LIMIT_SIZE: + xdr_decode_hyper(p, &maxsize); + break; + case NFS4_LIMIT_BLOCKS: + nblocks = be32_to_cpup(p++); + blocksize = be32_to_cpup(p); + maxsize = (uint64_t)nblocks * (uint64_t)blocksize; + } + maxsize >>= PAGE_SHIFT; + *pagemod_limit = min_t(u64, maxsize, ULONG_MAX); + return 0; +} + +static int decode_rw_delegation(struct xdr_stream *xdr, + struct nfs4_open_delegation *res) +{ + __be32 *p; + int status; + + status = decode_delegation_stateid(xdr, &res->stateid); + if (unlikely(status)) + return status; + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + res->do_recall = be32_to_cpup(p); + + switch (res->open_delegation_type) { + case NFS4_OPEN_DELEGATE_READ: + case NFS4_OPEN_DELEGATE_READ_ATTRS_DELEG: + res->type = FMODE_READ; + break; + case NFS4_OPEN_DELEGATE_WRITE: + case NFS4_OPEN_DELEGATE_WRITE_ATTRS_DELEG: + res->type = FMODE_WRITE|FMODE_READ; + if (decode_space_limit(xdr, &res->pagemod_limit) < 0) + return -EIO; + } + return decode_ace(xdr, NULL); +} + +static int decode_no_delegation(struct xdr_stream *xdr, + struct nfs4_open_delegation *res) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + res->why_no_delegation = be32_to_cpup(p); + switch (res->why_no_delegation) { + case WND4_CONTENTION: + case WND4_RESOURCE: + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + res->will_notify = be32_to_cpup(p); + } + return 0; +} + +static int decode_delegation(struct xdr_stream *xdr, + struct nfs4_open_delegation *res) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + res->open_delegation_type = be32_to_cpup(p); + switch (res->open_delegation_type) { + case NFS4_OPEN_DELEGATE_NONE: + return 0; + case NFS4_OPEN_DELEGATE_READ: + case NFS4_OPEN_DELEGATE_WRITE: + case NFS4_OPEN_DELEGATE_READ_ATTRS_DELEG: + case NFS4_OPEN_DELEGATE_WRITE_ATTRS_DELEG: + return decode_rw_delegation(xdr, res); + case NFS4_OPEN_DELEGATE_NONE_EXT: + return decode_no_delegation(xdr, res); + } + return -EIO; +} + +static int decode_open(struct xdr_stream *xdr, struct nfs_openres *res) +{ + __be32 *p; + uint32_t savewords, bmlen, i; + int status; + + if (!__decode_op_hdr(xdr, OP_OPEN, &status)) + return status; + nfs_increment_open_seqid(status, res->seqid); + if (status) + return status; + status = decode_open_stateid(xdr, &res->stateid); + if (unlikely(status)) + return status; + + decode_change_info(xdr, &res->cinfo); + + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + res->rflags = be32_to_cpup(p++); + bmlen = be32_to_cpup(p); + if (bmlen > 10) + goto xdr_error; + + p = xdr_inline_decode(xdr, bmlen << 2); + if (unlikely(!p)) + return -EIO; + savewords = min_t(uint32_t, bmlen, NFS4_BITMAP_SIZE); + for (i = 0; i < savewords; ++i) + res->attrset[i] = be32_to_cpup(p++); + for (; i < NFS4_BITMAP_SIZE; i++) + res->attrset[i] = 0; + + return decode_delegation(xdr, &res->delegation); +xdr_error: + dprintk("%s: Bitmap too large! Length = %u\n", __func__, bmlen); + return -EIO; +} + +static int decode_open_confirm(struct xdr_stream *xdr, struct nfs_open_confirmres *res) +{ + int status; + + status = decode_op_hdr(xdr, OP_OPEN_CONFIRM); + if (status != -EIO) + nfs_increment_open_seqid(status, res->seqid); + if (!status) + status = decode_open_stateid(xdr, &res->stateid); + return status; +} + +static int decode_open_downgrade(struct xdr_stream *xdr, struct nfs_closeres *res) +{ + int status; + + status = decode_op_hdr(xdr, OP_OPEN_DOWNGRADE); + if (status != -EIO) + nfs_increment_open_seqid(status, res->seqid); + if (!status) + status = decode_open_stateid(xdr, &res->stateid); + return status; +} + +static int decode_putfh(struct xdr_stream *xdr) +{ + return decode_op_hdr(xdr, OP_PUTFH); +} + +static int decode_putrootfh(struct xdr_stream *xdr) +{ + return decode_op_hdr(xdr, OP_PUTROOTFH); +} + +static int decode_read(struct xdr_stream *xdr, struct rpc_rqst *req, + struct nfs_pgio_res *res) +{ + __be32 *p; + uint32_t count, eof, recvd; + int status; + + status = decode_op_hdr(xdr, OP_READ); + if (status) + return status; + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + eof = be32_to_cpup(p++); + count = be32_to_cpup(p); + recvd = xdr_read_pages(xdr, count); + if (count > recvd) { + dprintk("NFS: server cheating in read reply: " + "count %u > recvd %u\n", count, recvd); + count = recvd; + eof = 0; + } + res->eof = eof; + res->count = count; + return 0; +} + +static int decode_readdir(struct xdr_stream *xdr, struct rpc_rqst *req, struct nfs4_readdir_res *readdir) +{ + int status; + __be32 verf[2]; + + status = decode_op_hdr(xdr, OP_READDIR); + if (!status) + status = decode_verifier(xdr, readdir->verifier.data); + if (unlikely(status)) + return status; + memcpy(verf, readdir->verifier.data, sizeof(verf)); + dprintk("%s: verifier = %08x:%08x\n", + __func__, verf[0], verf[1]); + return xdr_read_pages(xdr, xdr->buf->page_len); +} + +static int decode_readlink(struct xdr_stream *xdr, struct rpc_rqst *req) +{ + struct xdr_buf *rcvbuf = &req->rq_rcv_buf; + u32 len, recvd; + __be32 *p; + int status; + + status = decode_op_hdr(xdr, OP_READLINK); + if (status) + return status; + + /* Convert length of symlink */ + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + len = be32_to_cpup(p); + if (len >= rcvbuf->page_len || len <= 0) { + dprintk("nfs: server returned giant symlink!\n"); + return -ENAMETOOLONG; + } + recvd = xdr_read_pages(xdr, len); + if (recvd < len) { + dprintk("NFS: server cheating in readlink reply: " + "count %u > recvd %u\n", len, recvd); + return -EIO; + } + /* + * The XDR encode routine has set things up so that + * the link text will be copied directly into the + * buffer. We just have to do overflow-checking, + * and null-terminate the text (the VFS expects + * null-termination). + */ + xdr_terminate_string(rcvbuf, len); + return 0; +} + +static int decode_remove(struct xdr_stream *xdr, struct nfs4_change_info *cinfo) +{ + int status; + + status = decode_op_hdr(xdr, OP_REMOVE); + if (status) + goto out; + status = decode_change_info(xdr, cinfo); +out: + return status; +} + +static int decode_rename(struct xdr_stream *xdr, struct nfs4_change_info *old_cinfo, + struct nfs4_change_info *new_cinfo) +{ + int status; + + status = decode_op_hdr(xdr, OP_RENAME); + if (status) + goto out; + if ((status = decode_change_info(xdr, old_cinfo))) + goto out; + status = decode_change_info(xdr, new_cinfo); +out: + return status; +} + +static int decode_renew(struct xdr_stream *xdr) +{ + return decode_op_hdr(xdr, OP_RENEW); +} + +static int +decode_restorefh(struct xdr_stream *xdr) +{ + return decode_op_hdr(xdr, OP_RESTOREFH); +} + +static int decode_getacl(struct xdr_stream *xdr, struct rpc_rqst *req, + struct nfs_getaclres *res, enum nfs4_acl_type type) +{ + unsigned int savep; + uint32_t attrlen, + bitmap[3] = {0}; + int status; + + res->acl_len = 0; + if ((status = decode_op_hdr(xdr, OP_GETATTR)) != 0) + goto out; + + xdr_enter_page(xdr, xdr->buf->page_len); + + if ((status = decode_attr_bitmap(xdr, bitmap)) != 0) + goto out; + if ((status = decode_attr_length(xdr, &attrlen, &savep)) != 0) + goto out; + + switch (type) { + default: + if (unlikely(bitmap[0] & (FATTR4_WORD0_ACL - 1U))) + return -EIO; + if (!(bitmap[0] & FATTR4_WORD0_ACL)) + return -EOPNOTSUPP; + break; + case NFS4ACL_DACL: + if (unlikely(bitmap[0] || bitmap[1] & (FATTR4_WORD1_DACL - 1U))) + return -EIO; + if (!(bitmap[1] & FATTR4_WORD1_DACL)) + return -EOPNOTSUPP; + break; + case NFS4ACL_SACL: + if (unlikely(bitmap[0] || bitmap[1] & (FATTR4_WORD1_SACL - 1U))) + return -EIO; + if (!(bitmap[1] & FATTR4_WORD1_SACL)) + return -EOPNOTSUPP; + } + + /* The bitmap (xdr len + bitmaps) and the attr xdr len words + * are stored with the acl data to handle the problem of + * variable length bitmaps.*/ + res->acl_data_offset = xdr_page_pos(xdr); + res->acl_len = attrlen; + + /* Check for receive buffer overflow */ + if (res->acl_len > xdr_stream_remaining(xdr) || + res->acl_len + res->acl_data_offset > xdr->buf->page_len) { + res->acl_flags |= NFS4_ACL_TRUNC; + dprintk("NFS: acl reply: attrlen %u > page_len %zu\n", + attrlen, xdr_stream_remaining(xdr)); + } +out: + return status; +} + +static int +decode_savefh(struct xdr_stream *xdr) +{ + return decode_op_hdr(xdr, OP_SAVEFH); +} + +static int decode_setattr(struct xdr_stream *xdr) +{ + int status; + + status = decode_op_hdr(xdr, OP_SETATTR); + if (status) + return status; + if (decode_bitmap4(xdr, NULL, 0) >= 0) + return 0; + return -EIO; +} + +static int decode_delegattr(struct xdr_stream *xdr) +{ + return decode_setattr(xdr); +} + +static int decode_setclientid(struct xdr_stream *xdr, struct nfs4_setclientid_res *res) +{ + __be32 *p; + uint32_t opnum; + int32_t nfserr; + + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + opnum = be32_to_cpup(p++); + if (opnum != OP_SETCLIENTID) { + dprintk("nfs: decode_setclientid: Server returned operation" + " %d\n", opnum); + return -EIO; + } + nfserr = be32_to_cpup(p); + if (nfserr == NFS_OK) { + p = xdr_inline_decode(xdr, 8 + NFS4_VERIFIER_SIZE); + if (unlikely(!p)) + return -EIO; + p = xdr_decode_hyper(p, &res->clientid); + memcpy(res->confirm.data, p, NFS4_VERIFIER_SIZE); + } else if (nfserr == NFSERR_CLID_INUSE) { + uint32_t len; + + /* skip netid string */ + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + len = be32_to_cpup(p); + p = xdr_inline_decode(xdr, len); + if (unlikely(!p)) + return -EIO; + + /* skip uaddr string */ + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + len = be32_to_cpup(p); + p = xdr_inline_decode(xdr, len); + if (unlikely(!p)) + return -EIO; + return -NFSERR_CLID_INUSE; + } else + return nfs4_stat_to_errno(nfserr); + + return 0; +} + +static int decode_setclientid_confirm(struct xdr_stream *xdr) +{ + return decode_op_hdr(xdr, OP_SETCLIENTID_CONFIRM); +} + +static int decode_write(struct xdr_stream *xdr, struct nfs_pgio_res *res) +{ + __be32 *p; + int status; + + status = decode_op_hdr(xdr, OP_WRITE); + if (status) + return status; + + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + res->count = be32_to_cpup(p++); + res->verf->committed = be32_to_cpup(p++); + return decode_write_verifier(xdr, &res->verf->verifier); +} + +static int decode_delegreturn(struct xdr_stream *xdr) +{ + return decode_op_hdr(xdr, OP_DELEGRETURN); +} + +static int decode_secinfo_gss(struct xdr_stream *xdr, + struct nfs4_secinfo4 *flavor) +{ + u32 oid_len; + __be32 *p; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + oid_len = be32_to_cpup(p); + if (oid_len > GSS_OID_MAX_LEN) + return -EINVAL; + + p = xdr_inline_decode(xdr, oid_len); + if (unlikely(!p)) + return -EIO; + memcpy(flavor->flavor_info.oid.data, p, oid_len); + flavor->flavor_info.oid.len = oid_len; + + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + flavor->flavor_info.qop = be32_to_cpup(p++); + flavor->flavor_info.service = be32_to_cpup(p); + + return 0; +} + +static int decode_secinfo_common(struct xdr_stream *xdr, struct nfs4_secinfo_res *res) +{ + struct nfs4_secinfo4 *sec_flavor; + unsigned int i, num_flavors; + int status; + __be32 *p; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + + res->flavors->num_flavors = 0; + num_flavors = be32_to_cpup(p); + + for (i = 0; i < num_flavors; i++) { + sec_flavor = &res->flavors->flavors[i]; + if ((char *)&sec_flavor[1] - (char *)res->flavors > PAGE_SIZE) + break; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + sec_flavor->flavor = be32_to_cpup(p); + + if (sec_flavor->flavor == RPC_AUTH_GSS) { + status = decode_secinfo_gss(xdr, sec_flavor); + if (status) + goto out; + } + res->flavors->num_flavors++; + } + + status = 0; +out: + return status; +} + +static int decode_secinfo(struct xdr_stream *xdr, struct nfs4_secinfo_res *res) +{ + int status = decode_op_hdr(xdr, OP_SECINFO); + if (status) + return status; + return decode_secinfo_common(xdr, res); +} + +static int decode_secinfo_no_name(struct xdr_stream *xdr, struct nfs4_secinfo_res *res) +{ + int status = decode_op_hdr(xdr, OP_SECINFO_NO_NAME); + if (status) + return status; + return decode_secinfo_common(xdr, res); +} + +static int decode_op_map(struct xdr_stream *xdr, struct nfs4_op_map *op_map) +{ + if (xdr_stream_decode_uint32_array(xdr, op_map->u.words, + ARRAY_SIZE(op_map->u.words)) < 0) + return -EIO; + return 0; +} + +static int decode_exchange_id(struct xdr_stream *xdr, + struct nfs41_exchange_id_res *res) +{ + __be32 *p; + uint32_t dummy; + char *dummy_str; + int status; + uint32_t impl_id_count; + + status = decode_op_hdr(xdr, OP_EXCHANGE_ID); + if (status) + return status; + + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + xdr_decode_hyper(p, &res->clientid); + p = xdr_inline_decode(xdr, 12); + if (unlikely(!p)) + return -EIO; + res->seqid = be32_to_cpup(p++); + res->flags = be32_to_cpup(p++); + + res->state_protect.how = be32_to_cpup(p); + switch (res->state_protect.how) { + case SP4_NONE: + break; + case SP4_MACH_CRED: + status = decode_op_map(xdr, &res->state_protect.enforce); + if (status) + return status; + status = decode_op_map(xdr, &res->state_protect.allow); + if (status) + return status; + break; + default: + WARN_ON_ONCE(1); + return -EIO; + } + + /* server_owner4.so_minor_id */ + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + p = xdr_decode_hyper(p, &res->server_owner->minor_id); + + /* server_owner4.so_major_id */ + status = decode_opaque_inline(xdr, &dummy, &dummy_str); + if (unlikely(status)) + return status; + memcpy(res->server_owner->major_id, dummy_str, dummy); + res->server_owner->major_id_sz = dummy; + + /* server_scope4 */ + status = decode_opaque_inline(xdr, &dummy, &dummy_str); + if (unlikely(status)) + return status; + memcpy(res->server_scope->server_scope, dummy_str, dummy); + res->server_scope->server_scope_sz = dummy; + + /* Implementation Id */ + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + impl_id_count = be32_to_cpup(p++); + + if (impl_id_count) { + /* nii_domain */ + status = decode_opaque_inline(xdr, &dummy, &dummy_str); + if (unlikely(status)) + return status; + memcpy(res->impl_id->domain, dummy_str, dummy); + + /* nii_name */ + status = decode_opaque_inline(xdr, &dummy, &dummy_str); + if (unlikely(status)) + return status; + memcpy(res->impl_id->name, dummy_str, dummy); + + /* nii_date */ + p = xdr_inline_decode(xdr, 12); + if (unlikely(!p)) + return -EIO; + p = xdr_decode_hyper(p, &res->impl_id->date.seconds); + res->impl_id->date.nseconds = be32_to_cpup(p); + + /* if there's more than one entry, ignore the rest */ + } + return 0; +} + +static int decode_chan_attrs(struct xdr_stream *xdr, + struct nfs4_channel_attrs *attrs) +{ + __be32 *p; + u32 nr_attrs, val; + + p = xdr_inline_decode(xdr, 28); + if (unlikely(!p)) + return -EIO; + val = be32_to_cpup(p++); /* headerpadsz */ + if (val) + return -EINVAL; /* no support for header padding yet */ + attrs->max_rqst_sz = be32_to_cpup(p++); + attrs->max_resp_sz = be32_to_cpup(p++); + attrs->max_resp_sz_cached = be32_to_cpup(p++); + attrs->max_ops = be32_to_cpup(p++); + attrs->max_reqs = be32_to_cpup(p++); + nr_attrs = be32_to_cpup(p); + if (unlikely(nr_attrs > 1)) { + printk(KERN_WARNING "NFS: %s: Invalid rdma channel attrs " + "count %u\n", __func__, nr_attrs); + return -EINVAL; + } + if (nr_attrs == 1) { + p = xdr_inline_decode(xdr, 4); /* skip rdma_attrs */ + if (unlikely(!p)) + return -EIO; + } + return 0; +} + +static int decode_sessionid(struct xdr_stream *xdr, struct nfs4_sessionid *sid) +{ + return decode_opaque_fixed(xdr, sid->data, NFS4_MAX_SESSIONID_LEN); +} + +static int decode_bind_conn_to_session(struct xdr_stream *xdr, + struct nfs41_bind_conn_to_session_res *res) +{ + __be32 *p; + int status; + + status = decode_op_hdr(xdr, OP_BIND_CONN_TO_SESSION); + if (!status) + status = decode_sessionid(xdr, &res->sessionid); + if (unlikely(status)) + return status; + + /* dir flags, rdma mode bool */ + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + + res->dir = be32_to_cpup(p++); + if (res->dir == 0 || res->dir > NFS4_CDFS4_BOTH) + return -EIO; + if (be32_to_cpup(p) == 0) + res->use_conn_in_rdma_mode = false; + else + res->use_conn_in_rdma_mode = true; + + return 0; +} + +static int decode_create_session(struct xdr_stream *xdr, + struct nfs41_create_session_res *res) +{ + __be32 *p; + int status; + + status = decode_op_hdr(xdr, OP_CREATE_SESSION); + if (!status) + status = decode_sessionid(xdr, &res->sessionid); + if (unlikely(status)) + return status; + + /* seqid, flags */ + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + res->seqid = be32_to_cpup(p++); + res->flags = be32_to_cpup(p); + + /* Channel attributes */ + status = decode_chan_attrs(xdr, &res->fc_attrs); + if (!status) + status = decode_chan_attrs(xdr, &res->bc_attrs); + return status; +} + +static int decode_destroy_session(struct xdr_stream *xdr, void *dummy) +{ + return decode_op_hdr(xdr, OP_DESTROY_SESSION); +} + +static int decode_destroy_clientid(struct xdr_stream *xdr, void *dummy) +{ + return decode_op_hdr(xdr, OP_DESTROY_CLIENTID); +} + +static int decode_reclaim_complete(struct xdr_stream *xdr, void *dummy) +{ + return decode_op_hdr(xdr, OP_RECLAIM_COMPLETE); +} + +static int decode_sequence(struct xdr_stream *xdr, + struct nfs4_sequence_res *res, + struct rpc_rqst *rqstp) +{ + struct nfs4_session *session; + struct nfs4_sessionid id; + u32 dummy; + int status; + __be32 *p; + + if (res->sr_slot == NULL) + return 0; + if (!res->sr_slot->table->session) + return 0; + + status = decode_op_hdr(xdr, OP_SEQUENCE); + if (!status) + status = decode_sessionid(xdr, &id); + if (unlikely(status)) + goto out_err; + + /* + * If the server returns different values for sessionID, slotID or + * sequence number, the server is looney tunes. + */ + status = -EREMOTEIO; + session = res->sr_slot->table->session; + + if (memcmp(id.data, session->sess_id.data, + NFS4_MAX_SESSIONID_LEN)) { + dprintk("%s Invalid session id\n", __func__); + goto out_err; + } + + p = xdr_inline_decode(xdr, 20); + if (unlikely(!p)) + goto out_overflow; + + /* seqid */ + dummy = be32_to_cpup(p++); + if (dummy != res->sr_slot->seq_nr) { + dprintk("%s Invalid sequence number\n", __func__); + goto out_err; + } + /* slot id */ + dummy = be32_to_cpup(p++); + if (dummy != res->sr_slot->slot_nr) { + dprintk("%s Invalid slot id\n", __func__); + goto out_err; + } + /* highest slot id */ + res->sr_highest_slotid = be32_to_cpup(p++); + /* target highest slot id */ + res->sr_target_highest_slotid = be32_to_cpup(p++); + /* result flags */ + res->sr_status_flags = be32_to_cpup(p); + status = 0; +out_err: + res->sr_status = status; + return status; +out_overflow: + status = -EIO; + goto out_err; +} + +static int decode_layout_stateid(struct xdr_stream *xdr, nfs4_stateid *stateid) +{ + stateid->type = NFS4_LAYOUT_STATEID_TYPE; + return decode_stateid(xdr, stateid); +} + +static int decode_get_dir_delegation(struct xdr_stream *xdr, + struct nfs4_getattr_res *res) +{ + struct nfs4_gdd_res *gdd_res = res->gdd_res; + nfs4_verifier cookieverf; + u32 bitmap[1]; + int status; + + status = decode_op_hdr(xdr, OP_GET_DIR_DELEGATION); + if (status) + return status; + + if (xdr_stream_decode_u32(xdr, &gdd_res->status)) + return -EIO; + + if (gdd_res->status == GDD4_UNAVAIL) + return xdr_inline_decode(xdr, 4) ? 0 : -EIO; + + status = decode_verifier(xdr, &cookieverf); + if (status) + return status; + + status = decode_delegation_stateid(xdr, &gdd_res->deleg); + if (status) + return status; + + /* Decode supported notification types. */ + status = decode_bitmap4(xdr, bitmap, ARRAY_SIZE(bitmap)); + if (status < 0) + return status; + + /* Decode supported child attributes. */ + status = decode_bitmap4(xdr, bitmap, ARRAY_SIZE(bitmap)); + if (status < 0) + return status; + + /* Decode supported attributes. */ + status = decode_bitmap4(xdr, bitmap, ARRAY_SIZE(bitmap)); + if (status < 0) + return status; + return 0; +} + +static int decode_getdeviceinfo(struct xdr_stream *xdr, + struct nfs4_getdeviceinfo_res *res) +{ + struct pnfs_device *pdev = res->pdev; + __be32 *p; + uint32_t len, type; + int status; + + status = decode_op_hdr(xdr, OP_GETDEVICEINFO); + if (status) { + if (status == -ETOOSMALL) { + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + pdev->mincount = be32_to_cpup(p); + dprintk("%s: Min count too small. mincnt = %u\n", + __func__, pdev->mincount); + } + return status; + } + + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + type = be32_to_cpup(p++); + if (type != pdev->layout_type) { + dprintk("%s: layout mismatch req: %u pdev: %u\n", + __func__, pdev->layout_type, type); + return -EINVAL; + } + /* + * Get the length of the opaque device_addr4. xdr_read_pages places + * the opaque device_addr4 in the xdr_buf->pages (pnfs_device->pages) + * and places the remaining xdr data in xdr_buf->tail + */ + pdev->mincount = be32_to_cpup(p); + if (xdr_read_pages(xdr, pdev->mincount) != pdev->mincount) + return -EIO; + + /* Parse notification bitmap, verifying that it is zero. */ + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + len = be32_to_cpup(p); + if (len) { + uint32_t i; + + p = xdr_inline_decode(xdr, 4 * len); + if (unlikely(!p)) + return -EIO; + + res->notification = be32_to_cpup(p++); + for (i = 1; i < len; i++) { + if (be32_to_cpup(p++)) { + dprintk("%s: unsupported notification\n", + __func__); + return -EIO; + } + } + } + return 0; +} + +static int decode_layoutget(struct xdr_stream *xdr, struct rpc_rqst *req, + struct nfs4_layoutget_res *res) +{ + __be32 *p; + int status; + u32 layout_count; + u32 recvd; + + status = decode_op_hdr(xdr, OP_LAYOUTGET); + if (status) + goto out; + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + goto out_overflow; + res->return_on_close = be32_to_cpup(p); + decode_layout_stateid(xdr, &res->stateid); + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + goto out_overflow; + layout_count = be32_to_cpup(p); + if (!layout_count) { + dprintk("%s: server responded with empty layout array\n", + __func__); + status = -EINVAL; + goto out; + } + + p = xdr_inline_decode(xdr, 28); + if (unlikely(!p)) + goto out_overflow; + p = xdr_decode_hyper(p, &res->range.offset); + p = xdr_decode_hyper(p, &res->range.length); + res->range.iomode = be32_to_cpup(p++); + res->type = be32_to_cpup(p++); + res->layoutp->len = be32_to_cpup(p); + + dprintk("%s roff:%lu rlen:%lu riomode:%d, lo_type:0x%x, lo.len:%d\n", + __func__, + (unsigned long)res->range.offset, + (unsigned long)res->range.length, + res->range.iomode, + res->type, + res->layoutp->len); + + recvd = xdr_read_pages(xdr, res->layoutp->len); + if (res->layoutp->len > recvd) { + dprintk("NFS: server cheating in layoutget reply: " + "layout len %u > recvd %u\n", + res->layoutp->len, recvd); + status = -EINVAL; + goto out; + } + + if (layout_count > 1) { + /* We only handle a length one array at the moment. Any + * further entries are just ignored. Note that this means + * the client may see a response that is less than the + * minimum it requested. + */ + dprintk("%s: server responded with %d layouts, dropping tail\n", + __func__, layout_count); + } + +out: + res->status = status; + return status; +out_overflow: + status = -EIO; + goto out; +} + +static int decode_layoutreturn(struct xdr_stream *xdr, + struct nfs4_layoutreturn_res *res) +{ + __be32 *p; + int status; + + status = decode_op_hdr(xdr, OP_LAYOUTRETURN); + if (status) + return status; + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + res->lrs_present = be32_to_cpup(p); + if (res->lrs_present) + status = decode_layout_stateid(xdr, &res->stateid); + else + nfs4_stateid_copy(&res->stateid, &invalid_stateid); + return status; +} + +static int decode_layoutcommit(struct xdr_stream *xdr, + struct rpc_rqst *req, + struct nfs4_layoutcommit_res *res) +{ + __be32 *p; + __u32 sizechanged; + int status; + + status = decode_op_hdr(xdr, OP_LAYOUTCOMMIT); + res->status = status; + if (status) + return status; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + sizechanged = be32_to_cpup(p); + + if (sizechanged) { + /* throw away new size */ + p = xdr_inline_decode(xdr, 8); + if (unlikely(!p)) + return -EIO; + } + return 0; +} + +static int decode_test_stateid(struct xdr_stream *xdr, + struct nfs41_test_stateid_res *res) +{ + __be32 *p; + int status; + int num_res; + + status = decode_op_hdr(xdr, OP_TEST_STATEID); + if (status) + return status; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + num_res = be32_to_cpup(p++); + if (num_res != 1) + return -EIO; + + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EIO; + res->status = be32_to_cpup(p++); + + return status; +} + +static int decode_free_stateid(struct xdr_stream *xdr, + struct nfs41_free_stateid_res *res) +{ + res->status = decode_op_hdr(xdr, OP_FREE_STATEID); + return res->status; +} + +/* + * END OF "GENERIC" DECODE ROUTINES. + */ + +/* + * Decode OPEN_DOWNGRADE response + */ +static int nfs4_xdr_dec_open_downgrade(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs_closeres *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + if (res->lr_res) { + status = decode_layoutreturn(xdr, res->lr_res); + res->lr_ret = status; + if (status) + goto out; + } + status = decode_open_downgrade(xdr, res); +out: + return status; +} + +/* + * Decode ACCESS response + */ +static int nfs4_xdr_dec_access(struct rpc_rqst *rqstp, struct xdr_stream *xdr, + void *data) +{ + struct nfs4_accessres *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status != 0) + goto out; + status = decode_access(xdr, &res->supported, &res->access); + if (status != 0) + goto out; + if (res->fattr) + decode_getfattr(xdr, res->fattr, res->server); +out: + return status; +} + +/* + * Decode LOOKUP response + */ +static int nfs4_xdr_dec_lookup(struct rpc_rqst *rqstp, struct xdr_stream *xdr, + void *data) +{ + struct nfs4_lookup_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_lookup(xdr); + if (status) + goto out; + status = decode_getfh(xdr, res->fh); + if (status) + goto out; + status = decode_getfattr(xdr, res->fattr, res->server); +out: + return status; +} + +/* + * Decode LOOKUPP response + */ +static int nfs4_xdr_dec_lookupp(struct rpc_rqst *rqstp, struct xdr_stream *xdr, + void *data) +{ + struct nfs4_lookupp_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_lookupp(xdr); + if (status) + goto out; + status = decode_getfh(xdr, res->fh); + if (status) + goto out; + status = decode_getfattr(xdr, res->fattr, res->server); +out: + return status; +} + +/* + * Decode LOOKUP_ROOT response + */ +static int nfs4_xdr_dec_lookup_root(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs4_lookup_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putrootfh(xdr); + if (status) + goto out; + status = decode_getfh(xdr, res->fh); + if (status == 0) + status = decode_getfattr(xdr, res->fattr, res->server); +out: + return status; +} + +/* + * Decode REMOVE response + */ +static int nfs4_xdr_dec_remove(struct rpc_rqst *rqstp, struct xdr_stream *xdr, + void *data) +{ + struct nfs_removeres *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_remove(xdr, &res->cinfo); +out: + return status; +} + +/* + * Decode RENAME response + */ +static int nfs4_xdr_dec_rename(struct rpc_rqst *rqstp, struct xdr_stream *xdr, + void *data) +{ + struct nfs_renameres *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_savefh(xdr); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_rename(xdr, &res->old_cinfo, &res->new_cinfo); +out: + return status; +} + +/* + * Decode LINK response + */ +static int nfs4_xdr_dec_link(struct rpc_rqst *rqstp, struct xdr_stream *xdr, + void *data) +{ + struct nfs4_link_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_savefh(xdr); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_link(xdr, &res->cinfo); + if (status) + goto out; + /* + * Note order: OP_LINK leaves the directory as the current + * filehandle. + */ + status = decode_restorefh(xdr); + if (status) + goto out; + decode_getfattr(xdr, res->fattr, res->server); +out: + return status; +} + +/* + * Decode CREATE response + */ +static int nfs4_xdr_dec_create(struct rpc_rqst *rqstp, struct xdr_stream *xdr, + void *data) +{ + struct nfs4_create_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_create(xdr, &res->dir_cinfo); + if (status) + goto out; + status = decode_getfh(xdr, res->fh); + if (status) + goto out; + decode_getfattr(xdr, res->fattr, res->server); +out: + return status; +} + +/* + * Decode SYMLINK response + */ +static int nfs4_xdr_dec_symlink(struct rpc_rqst *rqstp, struct xdr_stream *xdr, + void *res) +{ + return nfs4_xdr_dec_create(rqstp, xdr, res); +} + +/* + * Decode GETATTR response + */ +static int nfs4_xdr_dec_getattr(struct rpc_rqst *rqstp, struct xdr_stream *xdr, + void *data) +{ + struct nfs4_getattr_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + if (res->gdd_res) { + status = decode_get_dir_delegation(xdr, res); + if (status) + goto out; + } + status = decode_getfattr(xdr, res->fattr, res->server); +out: + return status; +} + +/* + * Encode an SETACL request + */ +static void nfs4_xdr_enc_setacl(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfs_setaclargs *args = data; + struct compound_hdr hdr = { + .minorversion = nfs4_xdr_minorversion(&args->seq_args), + }; + + encode_compound_hdr(xdr, req, &hdr); + encode_sequence(xdr, &args->seq_args, &hdr); + encode_putfh(xdr, args->fh, &hdr); + encode_setacl(xdr, args, &hdr); + encode_nops(&hdr); +} + +/* + * Decode SETACL response + */ +static int +nfs4_xdr_dec_setacl(struct rpc_rqst *rqstp, struct xdr_stream *xdr, + void *data) +{ + struct nfs_setaclres *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_setattr(xdr); +out: + return status; +} + +/* + * Decode GETACL response + */ +static int +nfs4_xdr_dec_getacl(struct rpc_rqst *rqstp, struct xdr_stream *xdr, + void *data) +{ + struct nfs_getaclres *res = data; + struct compound_hdr hdr; + int status; + + if (res->acl_scratch != NULL) + xdr_set_scratch_folio(xdr, res->acl_scratch); + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_getacl(xdr, rqstp, res, res->acl_type); + +out: + return status; +} + +/* + * Decode CLOSE response + */ +static int nfs4_xdr_dec_close(struct rpc_rqst *rqstp, struct xdr_stream *xdr, + void *data) +{ + struct nfs_closeres *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + if (res->lr_res) { + status = decode_layoutreturn(xdr, res->lr_res); + res->lr_ret = status; + if (status) + goto out; + } + if (res->fattr != NULL) { + status = decode_getfattr(xdr, res->fattr, res->server); + if (status != 0) + goto out; + } + status = decode_close(xdr, res); +out: + return status; +} + +/* + * Decode OPEN response + */ +static int nfs4_xdr_dec_open(struct rpc_rqst *rqstp, struct xdr_stream *xdr, + void *data) +{ + struct nfs_openres *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_open(xdr, res); + if (status) + goto out; + status = decode_getfh(xdr, &res->fh); + if (status) + goto out; + if (res->access_request) + decode_access(xdr, &res->access_supported, &res->access_result); + decode_getfattr(xdr, res->f_attr, res->server); + if (res->lg_res) + decode_layoutget(xdr, rqstp, res->lg_res); +out: + return status; +} + +/* + * Decode OPEN_CONFIRM response + */ +static int nfs4_xdr_dec_open_confirm(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs_open_confirmres *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_open_confirm(xdr, res); +out: + return status; +} + +/* + * Decode OPEN response + */ +static int nfs4_xdr_dec_open_noattr(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs_openres *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_open(xdr, res); + if (status) + goto out; + if (res->access_request) + decode_access(xdr, &res->access_supported, &res->access_result); + decode_getfattr(xdr, res->f_attr, res->server); + if (res->lg_res) + decode_layoutget(xdr, rqstp, res->lg_res); +out: + return status; +} + +/* + * Decode SETATTR response + */ +static int nfs4_xdr_dec_setattr(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs_setattrres *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_setattr(xdr); + if (status) + goto out; + decode_getfattr(xdr, res->fattr, res->server); +out: + return status; +} + +/* + * Decode LOCK response + */ +static int nfs4_xdr_dec_lock(struct rpc_rqst *rqstp, struct xdr_stream *xdr, + void *data) +{ + struct nfs_lock_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_lock(xdr, res); +out: + return status; +} + +/* + * Decode LOCKT response + */ +static int nfs4_xdr_dec_lockt(struct rpc_rqst *rqstp, struct xdr_stream *xdr, + void *data) +{ + struct nfs_lockt_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_lockt(xdr, res); +out: + return status; +} + +/* + * Decode LOCKU response + */ +static int nfs4_xdr_dec_locku(struct rpc_rqst *rqstp, struct xdr_stream *xdr, + void *data) +{ + struct nfs_locku_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_locku(xdr, res); +out: + return status; +} + +#if defined(CONFIG_NFS_V4_0) +static int nfs4_xdr_dec_release_lockowner(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, void *dummy) +{ + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (!status) + status = decode_release_lockowner(xdr); + return status; +} +#endif /* CONFIG_NFS_V4_0 */ + +/* + * Decode READLINK response + */ +static int nfs4_xdr_dec_readlink(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs4_readlink_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_readlink(xdr, rqstp); +out: + return status; +} + +/* + * Decode READDIR response + */ +static int nfs4_xdr_dec_readdir(struct rpc_rqst *rqstp, struct xdr_stream *xdr, + void *data) +{ + struct nfs4_readdir_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_readdir(xdr, rqstp, res); +out: + return status; +} + +/* + * Decode Read response + */ +static int nfs4_xdr_dec_read(struct rpc_rqst *rqstp, struct xdr_stream *xdr, + void *data) +{ + struct nfs_pgio_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + res->op_status = hdr.status; + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_read(xdr, rqstp, res); + if (!status) + status = res->count; +out: + return status; +} + +/* + * Decode WRITE response + */ +static int nfs4_xdr_dec_write(struct rpc_rqst *rqstp, struct xdr_stream *xdr, + void *data) +{ + struct nfs_pgio_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + res->op_status = hdr.status; + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_write(xdr, res); + if (status) + goto out; + if (res->fattr) + decode_getfattr(xdr, res->fattr, res->server); + if (!status) + status = res->count; +out: + return status; +} + +/* + * Decode COMMIT response + */ +static int nfs4_xdr_dec_commit(struct rpc_rqst *rqstp, struct xdr_stream *xdr, + void *data) +{ + struct nfs_commitres *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + res->op_status = hdr.status; + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_commit(xdr, res); +out: + return status; +} + +/* + * Decode FSINFO response + */ +static int nfs4_xdr_dec_fsinfo(struct rpc_rqst *req, struct xdr_stream *xdr, + void *data) +{ + struct nfs4_fsinfo_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (!status) + status = decode_sequence(xdr, &res->seq_res, req); + if (!status) + status = decode_putfh(xdr); + if (!status) + status = decode_fsinfo(xdr, res->fsinfo); + return status; +} + +/* + * Decode PATHCONF response + */ +static int nfs4_xdr_dec_pathconf(struct rpc_rqst *req, struct xdr_stream *xdr, + void *data) +{ + struct nfs4_pathconf_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (!status) + status = decode_sequence(xdr, &res->seq_res, req); + if (!status) + status = decode_putfh(xdr); + if (!status) + status = decode_pathconf(xdr, res->pathconf); + return status; +} + +/* + * Decode STATFS response + */ +static int nfs4_xdr_dec_statfs(struct rpc_rqst *req, struct xdr_stream *xdr, + void *data) +{ + struct nfs4_statfs_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (!status) + status = decode_sequence(xdr, &res->seq_res, req); + if (!status) + status = decode_putfh(xdr); + if (!status) + status = decode_statfs(xdr, res->fsstat); + return status; +} + +/* + * Decode GETATTR_BITMAP response + */ +static int nfs4_xdr_dec_server_caps(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *data) +{ + struct nfs4_server_caps_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, req); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_server_caps(xdr, res); +out: + return status; +} + +/* + * Decode RENEW response + */ +#if defined(CONFIG_NFS_V4_0) +static int nfs4_xdr_dec_renew(struct rpc_rqst *rqstp, struct xdr_stream *xdr, + void *__unused) +{ + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (!status) + status = decode_renew(xdr); + return status; +} +#endif /* CONFIG_NFS_V4_0 */ + +/* + * Decode SETCLIENTID response + */ +static int nfs4_xdr_dec_setclientid(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *data) +{ + struct nfs4_setclientid_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (!status) + status = decode_setclientid(xdr, res); + return status; +} + +/* + * Decode SETCLIENTID_CONFIRM response + */ +static int nfs4_xdr_dec_setclientid_confirm(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *data) +{ + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (!status) + status = decode_setclientid_confirm(xdr); + return status; +} + +/* + * Decode DELEGRETURN response + */ +static int nfs4_xdr_dec_delegreturn(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs4_delegreturnres *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status != 0) + goto out; + if (res->lr_res) { + status = decode_layoutreturn(xdr, res->lr_res); + res->lr_ret = status; + if (status) + goto out; + } + if (res->sattr_res) { + status = decode_delegattr(xdr); + res->sattr_ret = status; + if (status) + goto out; + } + if (res->fattr) { + status = decode_getfattr(xdr, res->fattr, res->server); + if (status != 0) + goto out; + } + status = decode_delegreturn(xdr); +out: + return status; +} + +/* + * Decode FS_LOCATIONS response + */ +static int nfs4_xdr_dec_fs_locations(struct rpc_rqst *req, + struct xdr_stream *xdr, + void *data) +{ + struct nfs4_fs_locations_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, req); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + if (res->migration) { + xdr_enter_page(xdr, PAGE_SIZE); + status = decode_getfattr_generic(xdr, + res->fs_locations->fattr, + NULL, res->fs_locations, + res->fs_locations->server); + if (status) + goto out; + if (res->renew) + status = decode_renew(xdr); + } else { + status = decode_lookup(xdr); + if (status) + goto out; + xdr_enter_page(xdr, PAGE_SIZE); + status = decode_getfattr_generic(xdr, + res->fs_locations->fattr, + NULL, res->fs_locations, + res->fs_locations->server); + } +out: + return status; +} + +/* + * Decode SECINFO response + */ +static int nfs4_xdr_dec_secinfo(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs4_secinfo_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_secinfo(xdr, res); +out: + return status; +} + +/* + * Decode FSID_PRESENT response + */ +static int nfs4_xdr_dec_fsid_present(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs4_fsid_present_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_getfh(xdr, res->fh); + if (status) + goto out; + if (res->renew) + status = decode_renew(xdr); +out: + return status; +} + +/* + * Decode BIND_CONN_TO_SESSION response + */ +static int nfs4_xdr_dec_bind_conn_to_session(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *res) +{ + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (!status) + status = decode_bind_conn_to_session(xdr, res); + return status; +} + +/* + * Decode EXCHANGE_ID response + */ +static int nfs4_xdr_dec_exchange_id(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *res) +{ + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (!status) + status = decode_exchange_id(xdr, res); + return status; +} + +/* + * Decode CREATE_SESSION response + */ +static int nfs4_xdr_dec_create_session(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *res) +{ + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (!status) + status = decode_create_session(xdr, res); + return status; +} + +/* + * Decode DESTROY_SESSION response + */ +static int nfs4_xdr_dec_destroy_session(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *res) +{ + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (!status) + status = decode_destroy_session(xdr, res); + return status; +} + +/* + * Decode DESTROY_CLIENTID response + */ +static int nfs4_xdr_dec_destroy_clientid(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *res) +{ + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (!status) + status = decode_destroy_clientid(xdr, res); + return status; +} + +/* + * Decode SEQUENCE response + */ +static int nfs4_xdr_dec_sequence(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *res) +{ + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (!status) + status = decode_sequence(xdr, res, rqstp); + return status; +} + +/* + * Decode GET_LEASE_TIME response + */ +static int nfs4_xdr_dec_get_lease_time(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs4_get_lease_time_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (!status) + status = decode_sequence(xdr, &res->lr_seq_res, rqstp); + if (!status) + status = decode_putrootfh(xdr); + if (!status) + status = decode_fsinfo(xdr, res->lr_fsinfo); + return status; +} + +/* + * Decode RECLAIM_COMPLETE response + */ +static int nfs4_xdr_dec_reclaim_complete(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs41_reclaim_complete_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (!status) + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (!status) + status = decode_reclaim_complete(xdr, NULL); + return status; +} + +/* + * Decode GETDEVINFO response + */ +static int nfs4_xdr_dec_getdeviceinfo(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs4_getdeviceinfo_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status != 0) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status != 0) + goto out; + status = decode_getdeviceinfo(xdr, res); +out: + return status; +} + +/* + * Decode LAYOUTGET response + */ +static int nfs4_xdr_dec_layoutget(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs4_layoutget_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_layoutget(xdr, rqstp, res); +out: + return status; +} + +/* + * Decode LAYOUTRETURN response + */ +static int nfs4_xdr_dec_layoutreturn(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs4_layoutreturn_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_layoutreturn(xdr, res); +out: + return status; +} + +/* + * Decode LAYOUTCOMMIT response + */ +static int nfs4_xdr_dec_layoutcommit(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs4_layoutcommit_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putfh(xdr); + if (status) + goto out; + status = decode_layoutcommit(xdr, rqstp, res); + if (status) + goto out; + decode_getfattr(xdr, res->fattr, res->server); +out: + return status; +} + +/* + * Decode SECINFO_NO_NAME response + */ +static int nfs4_xdr_dec_secinfo_no_name(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs4_secinfo_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_putrootfh(xdr); + if (status) + goto out; + status = decode_secinfo_no_name(xdr, res); +out: + return status; +} + +/* + * Decode TEST_STATEID response + */ +static int nfs4_xdr_dec_test_stateid(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs41_test_stateid_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_test_stateid(xdr, res); +out: + return status; +} + +/* + * Decode FREE_STATEID response + */ +static int nfs4_xdr_dec_free_stateid(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfs41_free_stateid_res *res = data; + struct compound_hdr hdr; + int status; + + status = decode_compound_hdr(xdr, &hdr); + if (status) + goto out; + status = decode_sequence(xdr, &res->seq_res, rqstp); + if (status) + goto out; + status = decode_free_stateid(xdr, res); +out: + return status; +} + +/** + * nfs4_decode_dirent - Decode a single NFSv4 directory entry stored in + * the local page cache. + * @xdr: XDR stream where entry resides + * @entry: buffer to fill in with entry data + * @plus: boolean indicating whether this should be a readdirplus entry + * + * Returns zero if successful, otherwise a negative errno value is + * returned. + * + * This function is not invoked during READDIR reply decoding, but + * rather whenever an application invokes the getdents(2) system call + * on a directory already in our cache. + */ +int nfs4_decode_dirent(struct xdr_stream *xdr, struct nfs_entry *entry, + bool plus) +{ + unsigned int savep; + uint32_t bitmap[3] = {0}; + uint32_t len; + uint64_t new_cookie; + __be32 *p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EAGAIN; + if (*p == xdr_zero) { + p = xdr_inline_decode(xdr, 4); + if (unlikely(!p)) + return -EAGAIN; + if (*p == xdr_zero) + return -EAGAIN; + entry->eof = 1; + return -EBADCOOKIE; + } + + p = xdr_inline_decode(xdr, 12); + if (unlikely(!p)) + return -EAGAIN; + p = xdr_decode_hyper(p, &new_cookie); + entry->len = be32_to_cpup(p); + + p = xdr_inline_decode(xdr, entry->len); + if (unlikely(!p)) + return -EAGAIN; + entry->name = (const char *) p; + + /* + * In case the server doesn't return an inode number, + * we fake one here. (We don't use inode number 0, + * since glibc seems to choke on it...) + */ + entry->ino = 1; + entry->fattr->valid = 0; + + if (decode_attr_bitmap(xdr, bitmap) < 0) + return -EAGAIN; + + if (decode_attr_length(xdr, &len, &savep) < 0) + return -EAGAIN; + + if (decode_getfattr_attrs(xdr, bitmap, entry->fattr, entry->fh, + NULL, entry->server) < 0) + return -EAGAIN; + if (entry->fattr->valid & NFS_ATTR_FATTR_MOUNTED_ON_FILEID) + entry->ino = entry->fattr->mounted_on_fileid; + else if (entry->fattr->valid & NFS_ATTR_FATTR_FILEID) + entry->ino = entry->fattr->fileid; + + entry->d_type = DT_UNKNOWN; + if (entry->fattr->valid & NFS_ATTR_FATTR_TYPE) + entry->d_type = nfs_umode_to_dtype(entry->fattr->mode); + + entry->cookie = new_cookie; + + return 0; +} + +#ifdef CONFIG_NFS_V4_2 +#include "nfs42xdr.c" +#endif /* CONFIG_NFS_V4_2 */ + +#define PROC(proc, argtype, restype) \ +[NFSPROC4_CLNT_##proc] = { \ + .p_proc = NFSPROC4_COMPOUND, \ + .p_encode = nfs4_xdr_##argtype, \ + .p_decode = nfs4_xdr_##restype, \ + .p_arglen = NFS4_##argtype##_sz, \ + .p_replen = NFS4_##restype##_sz, \ + .p_statidx = NFSPROC4_CLNT_##proc, \ + .p_name = #proc, \ +} + +#define STUB(proc) \ +[NFSPROC4_CLNT_##proc] = { \ + .p_name = #proc, \ +} + +#if defined(CONFIG_NFS_V4_0) +#define PROC40(proc, argtype, restype) \ + PROC(proc, argtype, restype) +#else +#define PROC40(proc, argtype, restype) \ + STUB(proc) +#endif /* CONFIG_NFS_V4_0 */ + +#define PROC41(proc, argtype, restype) \ + PROC(proc, argtype, restype) + +#if defined(CONFIG_NFS_V4_2) +#define PROC42(proc, argtype, restype) \ + PROC(proc, argtype, restype) +#else +#define PROC42(proc, argtype, restype) \ + STUB(proc) +#endif + +const struct rpc_procinfo nfs4_procedures[] = { + PROC(READ, enc_read, dec_read), + PROC(WRITE, enc_write, dec_write), + PROC(COMMIT, enc_commit, dec_commit), + PROC(OPEN, enc_open, dec_open), + PROC(OPEN_CONFIRM, enc_open_confirm, dec_open_confirm), + PROC(OPEN_NOATTR, enc_open_noattr, dec_open_noattr), + PROC(OPEN_DOWNGRADE, enc_open_downgrade, dec_open_downgrade), + PROC(CLOSE, enc_close, dec_close), + PROC(SETATTR, enc_setattr, dec_setattr), + PROC(FSINFO, enc_fsinfo, dec_fsinfo), + PROC40(RENEW, enc_renew, dec_renew), + PROC(SETCLIENTID, enc_setclientid, dec_setclientid), + PROC(SETCLIENTID_CONFIRM, enc_setclientid_confirm, dec_setclientid_confirm), + PROC(LOCK, enc_lock, dec_lock), + PROC(LOCKT, enc_lockt, dec_lockt), + PROC(LOCKU, enc_locku, dec_locku), + PROC(ACCESS, enc_access, dec_access), + PROC(GETATTR, enc_getattr, dec_getattr), + PROC(LOOKUP, enc_lookup, dec_lookup), + PROC(LOOKUP_ROOT, enc_lookup_root, dec_lookup_root), + PROC(REMOVE, enc_remove, dec_remove), + PROC(RENAME, enc_rename, dec_rename), + PROC(LINK, enc_link, dec_link), + PROC(SYMLINK, enc_symlink, dec_symlink), + PROC(CREATE, enc_create, dec_create), + PROC(PATHCONF, enc_pathconf, dec_pathconf), + PROC(STATFS, enc_statfs, dec_statfs), + PROC(READLINK, enc_readlink, dec_readlink), + PROC(READDIR, enc_readdir, dec_readdir), + PROC(SERVER_CAPS, enc_server_caps, dec_server_caps), + PROC(DELEGRETURN, enc_delegreturn, dec_delegreturn), + PROC(GETACL, enc_getacl, dec_getacl), + PROC(SETACL, enc_setacl, dec_setacl), + PROC(FS_LOCATIONS, enc_fs_locations, dec_fs_locations), + PROC40(RELEASE_LOCKOWNER, enc_release_lockowner, dec_release_lockowner), + PROC(SECINFO, enc_secinfo, dec_secinfo), + PROC(FSID_PRESENT, enc_fsid_present, dec_fsid_present), + PROC41(EXCHANGE_ID, enc_exchange_id, dec_exchange_id), + PROC41(CREATE_SESSION, enc_create_session, dec_create_session), + PROC41(DESTROY_SESSION, enc_destroy_session, dec_destroy_session), + PROC41(SEQUENCE, enc_sequence, dec_sequence), + PROC(GET_LEASE_TIME, enc_get_lease_time, dec_get_lease_time), + PROC41(RECLAIM_COMPLETE,enc_reclaim_complete, dec_reclaim_complete), + PROC41(GETDEVICEINFO, enc_getdeviceinfo, dec_getdeviceinfo), + PROC41(LAYOUTGET, enc_layoutget, dec_layoutget), + PROC41(LAYOUTCOMMIT, enc_layoutcommit, dec_layoutcommit), + PROC41(LAYOUTRETURN, enc_layoutreturn, dec_layoutreturn), + PROC41(SECINFO_NO_NAME, enc_secinfo_no_name, dec_secinfo_no_name), + PROC41(TEST_STATEID, enc_test_stateid, dec_test_stateid), + PROC41(FREE_STATEID, enc_free_stateid, dec_free_stateid), + STUB(GETDEVICELIST), + PROC41(BIND_CONN_TO_SESSION, + enc_bind_conn_to_session, dec_bind_conn_to_session), + PROC41(DESTROY_CLIENTID,enc_destroy_clientid, dec_destroy_clientid), + PROC42(SEEK, enc_seek, dec_seek), + PROC42(ALLOCATE, enc_allocate, dec_allocate), + PROC42(DEALLOCATE, enc_deallocate, dec_deallocate), + PROC42(LAYOUTSTATS, enc_layoutstats, dec_layoutstats), + PROC42(CLONE, enc_clone, dec_clone), + PROC42(COPY, enc_copy, dec_copy), + PROC42(OFFLOAD_CANCEL, enc_offload_cancel, dec_offload_cancel), + PROC42(OFFLOAD_STATUS, enc_offload_status, dec_offload_status), + PROC42(COPY_NOTIFY, enc_copy_notify, dec_copy_notify), + PROC(LOOKUPP, enc_lookupp, dec_lookupp), + PROC42(LAYOUTERROR, enc_layouterror, dec_layouterror), + PROC42(GETXATTR, enc_getxattr, dec_getxattr), + PROC42(SETXATTR, enc_setxattr, dec_setxattr), + PROC42(LISTXATTRS, enc_listxattrs, dec_listxattrs), + PROC42(REMOVEXATTR, enc_removexattr, dec_removexattr), + PROC42(READ_PLUS, enc_read_plus, dec_read_plus), + PROC42(ZERO_RANGE, enc_zero_range, dec_zero_range), +}; + +static unsigned int nfs_version4_counts[ARRAY_SIZE(nfs4_procedures)]; +const struct rpc_version nfs_version4 = { + .number = 4, + .nrprocs = ARRAY_SIZE(nfs4_procedures), + .procs = nfs4_procedures, + .counts = nfs_version4_counts, +}; diff --git a/fs/nfs/nfsroot.c b/fs/nfs/nfsroot.c new file mode 100644 index 000000000..432612d22 --- /dev/null +++ b/fs/nfs/nfsroot.c @@ -0,0 +1,316 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 1995, 1996 Gero Kuhlmann + * + * Allow an NFS filesystem to be mounted as root. The way this works is: + * (1) Use the IP autoconfig mechanism to set local IP addresses and routes. + * (2) Construct the device string and the options string using DHCP + * option 17 and/or kernel command line options. + * (3) When mount_root() sets up the root file system, pass these strings + * to the NFS client's regular mount interface via sys_mount(). + * + * + * Changes: + * + * Alan Cox : Removed get_address name clash with FPU. + * Alan Cox : Reformatted a bit. + * Gero Kuhlmann : Code cleanup + * Michael Rausch : Fixed recognition of an incoming RARP answer. + * Martin Mares : (2.0) Auto-configuration via BOOTP supported. + * Martin Mares : Manual selection of interface & BOOTP/RARP. + * Martin Mares : Using network routes instead of host routes, + * allowing the default configuration to be used + * for normal operation of the host. + * Martin Mares : Randomized timer with exponential backoff + * installed to minimize network congestion. + * Martin Mares : Code cleanup. + * Martin Mares : (2.1) BOOTP and RARP made configuration options. + * Martin Mares : Server hostname generation fixed. + * Gerd Knorr : Fixed wired inode handling + * Martin Mares : (2.2) "0.0.0.0" addresses from command line ignored. + * Martin Mares : RARP replies not tested for server address. + * Gero Kuhlmann : (2.3) Some bug fixes and code cleanup again (please + * send me your new patches _before_ bothering + * Linus so that I don' always have to cleanup + * _afterwards_ - thanks) + * Gero Kuhlmann : Last changes of Martin Mares undone. + * Gero Kuhlmann : RARP replies are tested for specified server + * again. However, it's now possible to have + * different RARP and NFS servers. + * Gero Kuhlmann : "0.0.0.0" addresses from command line are + * now mapped to INADDR_NONE. + * Gero Kuhlmann : Fixed a bug which prevented BOOTP path name + * from being used (thanks to Leo Spiekman) + * Andy Walker : Allow to specify the NFS server in nfs_root + * without giving a path name + * Swen Thümmler : Allow to specify the NFS options in nfs_root + * without giving a path name. Fix BOOTP request + * for domainname (domainname is NIS domain, not + * DNS domain!). Skip dummy devices for BOOTP. + * Jacek Zapala : Fixed a bug which prevented server-ip address + * from nfsroot parameter from being used. + * Olaf Kirch : Adapted to new NFS code. + * Jakub Jelinek : Free used code segment. + * Marko Kohtala : Fixed some bugs. + * Martin Mares : Debug message cleanup + * Martin Mares : Changed to use the new generic IP layer autoconfig + * code. BOOTP and RARP moved there. + * Martin Mares : Default path now contains host name instead of + * host IP address (but host name defaults to IP + * address anyway). + * Martin Mares : Use root_server_addr appropriately during setup. + * Martin Mares : Rewrote parameter parsing, now hopefully giving + * correct overriding. + * Trond Myklebust : Add in preliminary support for NFSv3 and TCP. + * Fix bug in root_nfs_addr(). nfs_data.namlen + * is NOT for the length of the hostname. + * Hua Qin : Support for mounting root file system via + * NFS over TCP. + * Fabian Frederick: Option parser rebuilt (using parser lib) + * Chuck Lever : Use super.c's text-based mount option parsing + * Chuck Lever : Add "nfsrootdebug". + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +#define NFSDBG_FACILITY NFSDBG_ROOT + +/* Default path we try to mount. "%s" gets replaced by our IP address */ +#define NFS_ROOT "/tftpboot/%s" + +/* Default NFSROOT mount options. */ +#if defined(CONFIG_NFS_V2) +#define NFS_DEF_OPTIONS "vers=2,tcp,rsize=4096,wsize=4096" +#elif defined(CONFIG_NFS_V3) +#define NFS_DEF_OPTIONS "vers=3,tcp,rsize=4096,wsize=4096" +#else +#define NFS_DEF_OPTIONS "vers=4,tcp,rsize=4096,wsize=4096" +#endif + +/* Parameters passed from the kernel command line */ +static char nfs_root_parms[NFS_MAXPATHLEN + 1] __initdata = ""; + +/* Text-based mount options passed to super.c */ +static char nfs_root_options[256] __initdata = NFS_DEF_OPTIONS; + +/* Address of NFS server */ +static __be32 servaddr __initdata = htonl(INADDR_NONE); + +/* Name of directory to mount */ +static char nfs_export_path[NFS_MAXPATHLEN + 1] __initdata = ""; + +/* server:export path string passed to super.c */ +static char nfs_root_device[NFS_MAXPATHLEN + 1] __initdata = ""; + +#ifdef NFS_DEBUG +/* + * When the "nfsrootdebug" kernel command line option is specified, + * enable debugging messages for NFSROOT. + */ +static int __init nfs_root_debug(char *__unused) +{ + nfs_debug |= NFSDBG_ROOT | NFSDBG_MOUNT; + return 1; +} + +__setup("nfsrootdebug", nfs_root_debug); +#endif + +/* + * Parse NFS server and directory information passed on the kernel + * command line. + * + * nfsroot=[:][,] + * + * If there is a "%s" token in the string, it is replaced + * by the ASCII-representation of the client's IP address. + */ +static int __init nfs_root_setup(char *line) +{ + ROOT_DEV = Root_NFS; + + if (line[0] == '/' || line[0] == ',' || (line[0] >= '0' && line[0] <= '9')) { + strscpy(nfs_root_parms, line, sizeof(nfs_root_parms)); + } else { + size_t n = strlen(line) + sizeof(NFS_ROOT) - 1; + if (n >= sizeof(nfs_root_parms)) + line[sizeof(nfs_root_parms) - sizeof(NFS_ROOT) - 2] = '\0'; + sprintf(nfs_root_parms, NFS_ROOT, line); + } + + /* + * Extract the IP address of the NFS server containing our + * root file system, if one was specified. + * + * Note: root_nfs_parse_addr() removes the server-ip from + * nfs_root_parms, if it exists. + */ + root_server_addr = root_nfs_parse_addr(nfs_root_parms); + + return 1; +} + +__setup("nfsroot=", nfs_root_setup); + +static int __init root_nfs_copy(char *dest, const char *src, + const size_t destlen) +{ + if (strscpy(dest, src, destlen) == -E2BIG) + return -1; + return 0; +} + +static int __init root_nfs_cat(char *dest, const char *src, + const size_t destlen) +{ + size_t len = strlen(dest); + + if (len && dest[len - 1] != ',') + if (strlcat(dest, ",", destlen) >= destlen) + return -1; + + if (strlcat(dest, src, destlen) >= destlen) + return -1; + return 0; +} + +/* + * Parse out root export path and mount options from + * passed-in string @incoming. + * + * Copy the export path into @exppath. + */ +static int __init root_nfs_parse_options(char *incoming, char *exppath, + const size_t exppathlen) +{ + char *p; + + /* + * Set the NFS remote path + */ + p = strsep(&incoming, ","); + if (*p != '\0' && strcmp(p, "default") != 0) + if (root_nfs_copy(exppath, p, exppathlen)) + return -1; + + /* + * @incoming now points to the rest of the string; if it + * contains something, append it to our root options buffer + */ + if (incoming != NULL && *incoming != '\0') + if (root_nfs_cat(nfs_root_options, incoming, + sizeof(nfs_root_options))) + return -1; + return 0; +} + +/* + * Decode the export directory path name and NFS options from + * the kernel command line. This has to be done late in order to + * use a dynamically acquired client IP address for the remote + * root directory path. + * + * Returns zero if successful; otherwise -1 is returned. + */ +static int __init root_nfs_data(char *cmdline) +{ + char mand_options[sizeof("nolock,addr=") + INET_ADDRSTRLEN + 1]; + int len, retval = -1; + char *tmp = NULL; + const size_t tmplen = sizeof(nfs_export_path); + + tmp = kzalloc(tmplen, GFP_KERNEL); + if (tmp == NULL) + goto out_nomem; + strcpy(tmp, NFS_ROOT); + + if (root_server_path[0] != '\0') { + dprintk("Root-NFS: DHCPv4 option 17: %s\n", + root_server_path); + if (root_nfs_parse_options(root_server_path, tmp, tmplen)) + goto out_optionstoolong; + } + + if (cmdline[0] != '\0') { + dprintk("Root-NFS: nfsroot=%s\n", cmdline); + if (root_nfs_parse_options(cmdline, tmp, tmplen)) + goto out_optionstoolong; + } + + /* + * Append mandatory options for nfsroot so they override + * what has come before + */ + snprintf(mand_options, sizeof(mand_options), "nolock,addr=%pI4", + &servaddr); + if (root_nfs_cat(nfs_root_options, mand_options, + sizeof(nfs_root_options))) + goto out_optionstoolong; + + /* + * Set up nfs_root_device. For NFS mounts, this looks like + * + * server:/path + * + * At this point, utsname()->nodename contains our local + * IP address or hostname, set by ipconfig. If "%s" exists + * in tmp, substitute the nodename, then shovel the whole + * mess into nfs_root_device. + */ + len = snprintf(nfs_export_path, sizeof(nfs_export_path), + tmp, utsname()->nodename); + if (len >= (int)sizeof(nfs_export_path)) + goto out_devnametoolong; + len = snprintf(nfs_root_device, sizeof(nfs_root_device), + "%pI4:%s", &servaddr, nfs_export_path); + if (len >= (int)sizeof(nfs_root_device)) + goto out_devnametoolong; + + retval = 0; + +out: + kfree(tmp); + return retval; +out_nomem: + printk(KERN_ERR "Root-NFS: could not allocate memory\n"); + goto out; +out_optionstoolong: + printk(KERN_ERR "Root-NFS: mount options string too long\n"); + goto out; +out_devnametoolong: + printk(KERN_ERR "Root-NFS: root device name too long.\n"); + goto out; +} + +/** + * nfs_root_data - Return prepared 'data' for NFSROOT mount + * @root_device: OUT: address of string containing NFSROOT device + * @root_data: OUT: address of string containing NFSROOT mount options + * + * Returns zero and sets @root_device and @root_data if successful, + * otherwise -1 is returned. + */ +int __init nfs_root_data(char **root_device, char **root_data) +{ + servaddr = root_server_addr; + if (servaddr == htonl(INADDR_NONE)) { + printk(KERN_ERR "Root-NFS: no NFS server address\n"); + return -1; + } + + if (root_nfs_data(nfs_root_parms) < 0) + return -1; + + *root_device = nfs_root_device; + *root_data = nfs_root_options; + return 0; +} diff --git a/fs/nfs/nfstrace.c b/fs/nfs/nfstrace.c new file mode 100644 index 000000000..5d1bfccbb --- /dev/null +++ b/fs/nfs/nfstrace.c @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2013 Trond Myklebust + */ +#include +#include +#include "internal.h" + +#define CREATE_TRACE_POINTS +#include "nfstrace.h" + +EXPORT_TRACEPOINT_SYMBOL_GPL(nfs_fsync_enter); +EXPORT_TRACEPOINT_SYMBOL_GPL(nfs_fsync_exit); +EXPORT_TRACEPOINT_SYMBOL_GPL(nfs_xdr_status); +EXPORT_TRACEPOINT_SYMBOL_GPL(nfs_xdr_bad_filehandle); diff --git a/fs/nfs/nfstrace.h b/fs/nfs/nfstrace.h new file mode 100644 index 000000000..4ada21f4e --- /dev/null +++ b/fs/nfs/nfstrace.h @@ -0,0 +1,1986 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2013 Trond Myklebust + */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM nfs + +#if !defined(_TRACE_NFS_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_NFS_H + +#include +#include + +#include +#include +#include + +#define nfs_show_cache_validity(v) \ + __print_flags(v, "|", \ + { NFS_INO_INVALID_DATA, "INVALID_DATA" }, \ + { NFS_INO_INVALID_ATIME, "INVALID_ATIME" }, \ + { NFS_INO_INVALID_ACCESS, "INVALID_ACCESS" }, \ + { NFS_INO_INVALID_ACL, "INVALID_ACL" }, \ + { NFS_INO_REVAL_FORCED, "REVAL_FORCED" }, \ + { NFS_INO_INVALID_LABEL, "INVALID_LABEL" }, \ + { NFS_INO_INVALID_CHANGE, "INVALID_CHANGE" }, \ + { NFS_INO_INVALID_CTIME, "INVALID_CTIME" }, \ + { NFS_INO_INVALID_MTIME, "INVALID_MTIME" }, \ + { NFS_INO_INVALID_SIZE, "INVALID_SIZE" }, \ + { NFS_INO_INVALID_OTHER, "INVALID_OTHER" }, \ + { NFS_INO_DATA_INVAL_DEFER, "DATA_INVAL_DEFER" }, \ + { NFS_INO_INVALID_BLOCKS, "INVALID_BLOCKS" }, \ + { NFS_INO_INVALID_XATTR, "INVALID_XATTR" }, \ + { NFS_INO_INVALID_NLINK, "INVALID_NLINK" }, \ + { NFS_INO_INVALID_MODE, "INVALID_MODE" }, \ + { NFS_INO_INVALID_BTIME, "INVALID_BTIME" }) + +#define nfs_show_nfsi_flags(v) \ + __print_flags(v, "|", \ + { BIT(NFS_INO_STALE), "STALE" }, \ + { BIT(NFS_INO_ACL_LRU_SET), "ACL_LRU_SET" }, \ + { BIT(NFS_INO_INVALIDATING), "INVALIDATING" }, \ + { BIT(NFS_INO_LAYOUTCOMMIT), "NEED_LAYOUTCOMMIT" }, \ + { BIT(NFS_INO_LAYOUTCOMMITTING), "LAYOUTCOMMIT" }, \ + { BIT(NFS_INO_LAYOUTSTATS), "LAYOUTSTATS" }, \ + { BIT(NFS_INO_ODIRECT), "ODIRECT" }) + +#define nfs_show_wb_flags(v) \ + __print_flags(v, "|", \ + { BIT(PG_BUSY), "BUSY" }, \ + { BIT(PG_MAPPED), "MAPPED" }, \ + { BIT(PG_FOLIO), "FOLIO" }, \ + { BIT(PG_CLEAN), "CLEAN" }, \ + { BIT(PG_COMMIT_TO_DS), "COMMIT_TO_DS" }, \ + { BIT(PG_INODE_REF), "INODE_REF" }, \ + { BIT(PG_HEADLOCK), "HEADLOCK" }, \ + { BIT(PG_TEARDOWN), "TEARDOWN" }, \ + { BIT(PG_UNLOCKPAGE), "UNLOCKPAGE" }, \ + { BIT(PG_UPTODATE), "UPTODATE" }, \ + { BIT(PG_WB_END), "WB_END" }, \ + { BIT(PG_REMOVE), "REMOVE" }, \ + { BIT(PG_CONTENDED1), "CONTENDED1" }, \ + { BIT(PG_CONTENDED2), "CONTENDED2" }) + +DECLARE_EVENT_CLASS(nfs_inode_event, + TP_PROTO( + const struct inode *inode + ), + + TP_ARGS(inode), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(u64, version) + __field(unsigned long, cache_validity) + ), + + TP_fast_assign( + const struct nfs_inode *nfsi = NFS_I(inode); + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); + __entry->version = inode_peek_iversion_raw(inode); + __entry->cache_validity = nfsi->cache_validity; + ), + + TP_printk( + "fileid=%02x:%02x:%llu fhandle=0x%08x version=%llu cache_validity=0x%lx (%s)", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + (unsigned long long)__entry->version, + __entry->cache_validity, + nfs_show_cache_validity(__entry->cache_validity) + ) +); + +DECLARE_EVENT_CLASS(nfs_inode_event_done, + TP_PROTO( + const struct inode *inode, + int error + ), + + TP_ARGS(inode, error), + + TP_STRUCT__entry( + __field(unsigned long, error) + __field(dev_t, dev) + __field(u32, fhandle) + __field(unsigned char, type) + __field(u64, fileid) + __field(u64, version) + __field(loff_t, size) + __field(unsigned long, nfsi_flags) + __field(unsigned long, cache_validity) + ), + + TP_fast_assign( + const struct nfs_inode *nfsi = NFS_I(inode); + __entry->error = error < 0 ? -error : 0; + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); + __entry->type = nfs_umode_to_dtype(inode->i_mode); + __entry->version = inode_peek_iversion_raw(inode); + __entry->size = i_size_read(inode); + __entry->nfsi_flags = nfsi->flags; + __entry->cache_validity = nfsi->cache_validity; + ), + + TP_printk( + "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " + "type=%u (%s) version=%llu size=%lld " + "cache_validity=0x%lx (%s) nfs_flags=0x%lx (%s)", + -__entry->error, show_nfs_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + __entry->type, + show_fs_dirent_type(__entry->type), + (unsigned long long)__entry->version, + (long long)__entry->size, + __entry->cache_validity, + nfs_show_cache_validity(__entry->cache_validity), + __entry->nfsi_flags, + nfs_show_nfsi_flags(__entry->nfsi_flags) + ) +); + +#define DEFINE_NFS_INODE_EVENT(name) \ + DEFINE_EVENT(nfs_inode_event, name, \ + TP_PROTO( \ + const struct inode *inode \ + ), \ + TP_ARGS(inode)) +#define DEFINE_NFS_INODE_EVENT_DONE(name) \ + DEFINE_EVENT(nfs_inode_event_done, name, \ + TP_PROTO( \ + const struct inode *inode, \ + int error \ + ), \ + TP_ARGS(inode, error)) +DEFINE_NFS_INODE_EVENT(nfs_set_inode_stale); +DEFINE_NFS_INODE_EVENT(nfs_refresh_inode_enter); +DEFINE_NFS_INODE_EVENT_DONE(nfs_refresh_inode_exit); +DEFINE_NFS_INODE_EVENT(nfs_revalidate_inode_enter); +DEFINE_NFS_INODE_EVENT_DONE(nfs_revalidate_inode_exit); +DEFINE_NFS_INODE_EVENT(nfs_invalidate_mapping_enter); +DEFINE_NFS_INODE_EVENT_DONE(nfs_invalidate_mapping_exit); +DEFINE_NFS_INODE_EVENT(nfs_getattr_enter); +DEFINE_NFS_INODE_EVENT_DONE(nfs_getattr_exit); +DEFINE_NFS_INODE_EVENT(nfs_setattr_enter); +DEFINE_NFS_INODE_EVENT_DONE(nfs_setattr_exit); +DEFINE_NFS_INODE_EVENT(nfs_writeback_inode_enter); +DEFINE_NFS_INODE_EVENT_DONE(nfs_writeback_inode_exit); +DEFINE_NFS_INODE_EVENT(nfs_fsync_enter); +DEFINE_NFS_INODE_EVENT_DONE(nfs_fsync_exit); +DEFINE_NFS_INODE_EVENT(nfs_access_enter); +DEFINE_NFS_INODE_EVENT_DONE(nfs_set_cache_invalid); +DEFINE_NFS_INODE_EVENT(nfs_readdir_force_readdirplus); +DEFINE_NFS_INODE_EVENT_DONE(nfs_readdir_cache_fill_done); +DEFINE_NFS_INODE_EVENT_DONE(nfs_readdir_uncached_done); + +TRACE_EVENT(nfs_access_exit, + TP_PROTO( + const struct inode *inode, + unsigned int mask, + unsigned int permitted, + int error + ), + + TP_ARGS(inode, mask, permitted, error), + + TP_STRUCT__entry( + __field(unsigned long, error) + __field(dev_t, dev) + __field(u32, fhandle) + __field(unsigned char, type) + __field(u64, fileid) + __field(u64, version) + __field(loff_t, size) + __field(unsigned long, nfsi_flags) + __field(unsigned long, cache_validity) + __field(unsigned int, mask) + __field(unsigned int, permitted) + ), + + TP_fast_assign( + const struct nfs_inode *nfsi = NFS_I(inode); + __entry->error = error < 0 ? -error : 0; + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); + __entry->type = nfs_umode_to_dtype(inode->i_mode); + __entry->version = inode_peek_iversion_raw(inode); + __entry->size = i_size_read(inode); + __entry->nfsi_flags = nfsi->flags; + __entry->cache_validity = nfsi->cache_validity; + __entry->mask = mask; + __entry->permitted = permitted; + ), + + TP_printk( + "error=%ld (%s) fileid=%02x:%02x:%llu fhandle=0x%08x " + "type=%u (%s) version=%llu size=%lld " + "cache_validity=0x%lx (%s) nfs_flags=0x%lx (%s) " + "mask=0x%x permitted=0x%x", + -__entry->error, show_nfs_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + __entry->type, + show_fs_dirent_type(__entry->type), + (unsigned long long)__entry->version, + (long long)__entry->size, + __entry->cache_validity, + nfs_show_cache_validity(__entry->cache_validity), + __entry->nfsi_flags, + nfs_show_nfsi_flags(__entry->nfsi_flags), + __entry->mask, __entry->permitted + ) +); + +DECLARE_EVENT_CLASS(nfs_update_size_class, + TP_PROTO( + const struct inode *inode, + loff_t new_size + ), + + TP_ARGS(inode, new_size), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(u64, version) + __field(loff_t, cur_size) + __field(loff_t, new_size) + ), + + TP_fast_assign( + const struct nfs_inode *nfsi = NFS_I(inode); + + __entry->dev = inode->i_sb->s_dev; + __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); + __entry->fileid = inode->i_ino; + __entry->version = inode_peek_iversion_raw(inode); + __entry->cur_size = i_size_read(inode); + __entry->new_size = new_size; + ), + + TP_printk( + "fileid=%02x:%02x:%llu fhandle=0x%08x version=%llu cursize=%lld newsize=%lld", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, __entry->version, + __entry->cur_size, __entry->new_size + ) +); + +#define DEFINE_NFS_UPDATE_SIZE_EVENT(name) \ + DEFINE_EVENT(nfs_update_size_class, nfs_size_##name, \ + TP_PROTO( \ + const struct inode *inode, \ + loff_t new_size \ + ), \ + TP_ARGS(inode, new_size)) + +DEFINE_NFS_UPDATE_SIZE_EVENT(truncate); +DEFINE_NFS_UPDATE_SIZE_EVENT(truncate_folio); +DEFINE_NFS_UPDATE_SIZE_EVENT(wcc); +DEFINE_NFS_UPDATE_SIZE_EVENT(update); +DEFINE_NFS_UPDATE_SIZE_EVENT(grow); + +DECLARE_EVENT_CLASS(nfs_inode_range_event, + TP_PROTO( + const struct inode *inode, + loff_t range_start, + loff_t range_end + ), + + TP_ARGS(inode, range_start, range_end), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(u64, version) + __field(loff_t, range_start) + __field(loff_t, range_end) + ), + + TP_fast_assign( + const struct nfs_inode *nfsi = NFS_I(inode); + + __entry->dev = inode->i_sb->s_dev; + __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); + __entry->fileid = inode->i_ino; + __entry->version = inode_peek_iversion_raw(inode); + __entry->range_start = range_start; + __entry->range_end = range_end; + ), + + TP_printk( + "fileid=%02x:%02x:%llu fhandle=0x%08x version=%llu " + "range=[%lld, %lld]", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, __entry->version, + __entry->range_start, __entry->range_end + ) +); + +#define DEFINE_NFS_INODE_RANGE_EVENT(name) \ + DEFINE_EVENT(nfs_inode_range_event, name, \ + TP_PROTO( \ + const struct inode *inode, \ + loff_t range_start, \ + loff_t range_end \ + ), \ + TP_ARGS(inode, range_start, range_end)) + +DEFINE_NFS_INODE_RANGE_EVENT(nfs_readdir_invalidate_cache_range); + +DECLARE_EVENT_CLASS(nfs_readdir_event, + TP_PROTO( + const struct file *file, + const __be32 *verifier, + u64 cookie, + pgoff_t page_index, + unsigned int dtsize + ), + + TP_ARGS(file, verifier, cookie, page_index, dtsize), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(u64, version) + __array(char, verifier, NFS4_VERIFIER_SIZE) + __field(u64, cookie) + __field(pgoff_t, index) + __field(unsigned int, dtsize) + ), + + TP_fast_assign( + const struct inode *dir = file_inode(file); + const struct nfs_inode *nfsi = NFS_I(dir); + + __entry->dev = dir->i_sb->s_dev; + __entry->fileid = dir->i_ino; + __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); + __entry->version = inode_peek_iversion_raw(dir); + if (cookie != 0) + memcpy(__entry->verifier, verifier, + NFS4_VERIFIER_SIZE); + else + memset(__entry->verifier, 0, + NFS4_VERIFIER_SIZE); + __entry->cookie = cookie; + __entry->index = page_index; + __entry->dtsize = dtsize; + ), + + TP_printk( + "fileid=%02x:%02x:%llu fhandle=0x%08x version=%llu " + "cookie=%s:0x%llx cache_index=%lu dtsize=%u", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, __entry->fhandle, + __entry->version, show_nfs4_verifier(__entry->verifier), + (unsigned long long)__entry->cookie, __entry->index, + __entry->dtsize + ) +); + +#define DEFINE_NFS_READDIR_EVENT(name) \ + DEFINE_EVENT(nfs_readdir_event, name, \ + TP_PROTO( \ + const struct file *file, \ + const __be32 *verifier, \ + u64 cookie, \ + pgoff_t page_index, \ + unsigned int dtsize \ + ), \ + TP_ARGS(file, verifier, cookie, page_index, dtsize)) + +DEFINE_NFS_READDIR_EVENT(nfs_readdir_cache_fill); +DEFINE_NFS_READDIR_EVENT(nfs_readdir_uncached); + +DECLARE_EVENT_CLASS(nfs_lookup_event, + TP_PROTO( + const struct inode *dir, + const struct dentry *dentry, + unsigned int flags + ), + + TP_ARGS(dir, dentry, flags), + + TP_STRUCT__entry( + __field(unsigned long, flags) + __field(dev_t, dev) + __field(u64, dir) + __field(u64, fileid) + __string(name, dentry->d_name.name) + ), + + TP_fast_assign( + __entry->dev = dir->i_sb->s_dev; + __entry->dir = dir->i_ino; + __entry->flags = flags; + __entry->fileid = d_is_negative(dentry) ? 0 : d_inode(dentry)->i_ino; + __assign_str(name); + ), + + TP_printk( + "flags=0x%lx (%s) name=%02x:%02x:%llu/%s fileid=%llu", + __entry->flags, + show_fs_lookup_flags(__entry->flags), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->dir, + __get_str(name), + __entry->fileid + ) +); + +#define DEFINE_NFS_LOOKUP_EVENT(name) \ + DEFINE_EVENT(nfs_lookup_event, name, \ + TP_PROTO( \ + const struct inode *dir, \ + const struct dentry *dentry, \ + unsigned int flags \ + ), \ + TP_ARGS(dir, dentry, flags)) + +DECLARE_EVENT_CLASS(nfs_lookup_event_done, + TP_PROTO( + const struct inode *dir, + const struct dentry *dentry, + unsigned int flags, + int error + ), + + TP_ARGS(dir, dentry, flags, error), + + TP_STRUCT__entry( + __field(unsigned long, error) + __field(unsigned long, flags) + __field(dev_t, dev) + __field(u64, dir) + __field(u64, fileid) + __string(name, dentry->d_name.name) + ), + + TP_fast_assign( + __entry->dev = dir->i_sb->s_dev; + __entry->dir = dir->i_ino; + __entry->error = error < 0 ? -error : 0; + __entry->flags = flags; + __entry->fileid = d_is_negative(dentry) ? 0 : d_inode(dentry)->i_ino; + __assign_str(name); + ), + + TP_printk( + "error=%ld (%s) flags=0x%lx (%s) name=%02x:%02x:%llu/%s fileid=%llu", + -__entry->error, show_nfs_status(__entry->error), + __entry->flags, + show_fs_lookup_flags(__entry->flags), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->dir, + __get_str(name), + __entry->fileid + ) +); + +#define DEFINE_NFS_LOOKUP_EVENT_DONE(name) \ + DEFINE_EVENT(nfs_lookup_event_done, name, \ + TP_PROTO( \ + const struct inode *dir, \ + const struct dentry *dentry, \ + unsigned int flags, \ + int error \ + ), \ + TP_ARGS(dir, dentry, flags, error)) + +DEFINE_NFS_LOOKUP_EVENT(nfs_lookup_enter); +DEFINE_NFS_LOOKUP_EVENT_DONE(nfs_lookup_exit); +DEFINE_NFS_LOOKUP_EVENT(nfs_lookup_revalidate_enter); +DEFINE_NFS_LOOKUP_EVENT_DONE(nfs_lookup_revalidate_exit); +DEFINE_NFS_LOOKUP_EVENT(nfs_readdir_lookup); +DEFINE_NFS_LOOKUP_EVENT(nfs_readdir_lookup_revalidate_failed); +DEFINE_NFS_LOOKUP_EVENT_DONE(nfs_readdir_lookup_revalidate); + +TRACE_EVENT(nfs_atomic_open_enter, + TP_PROTO( + const struct inode *dir, + const struct nfs_open_context *ctx, + unsigned int flags + ), + + TP_ARGS(dir, ctx, flags), + + TP_STRUCT__entry( + __field(unsigned long, flags) + __field(unsigned long, fmode) + __field(dev_t, dev) + __field(u64, dir) + __string(name, ctx->dentry->d_name.name) + ), + + TP_fast_assign( + __entry->dev = dir->i_sb->s_dev; + __entry->dir = dir->i_ino; + __entry->flags = flags; + __entry->fmode = (__force unsigned long)ctx->mode; + __assign_str(name); + ), + + TP_printk( + "flags=0x%lx (%s) fmode=%s name=%02x:%02x:%llu/%s", + __entry->flags, + show_fs_fcntl_open_flags(__entry->flags), + show_fs_fmode_flags(__entry->fmode), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->dir, + __get_str(name) + ) +); + +TRACE_EVENT(nfs_atomic_open_exit, + TP_PROTO( + const struct inode *dir, + const struct nfs_open_context *ctx, + unsigned int flags, + int error + ), + + TP_ARGS(dir, ctx, flags, error), + + TP_STRUCT__entry( + __field(unsigned long, error) + __field(unsigned long, flags) + __field(unsigned long, fmode) + __field(dev_t, dev) + __field(u64, dir) + __string(name, ctx->dentry->d_name.name) + ), + + TP_fast_assign( + __entry->error = -error; + __entry->dev = dir->i_sb->s_dev; + __entry->dir = dir->i_ino; + __entry->flags = flags; + __entry->fmode = (__force unsigned long)ctx->mode; + __assign_str(name); + ), + + TP_printk( + "error=%ld (%s) flags=0x%lx (%s) fmode=%s " + "name=%02x:%02x:%llu/%s", + -__entry->error, show_nfs_status(__entry->error), + __entry->flags, + show_fs_fcntl_open_flags(__entry->flags), + show_fs_fmode_flags(__entry->fmode), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->dir, + __get_str(name) + ) +); + +TRACE_EVENT(nfs_create_enter, + TP_PROTO( + const struct inode *dir, + const struct dentry *dentry, + unsigned int flags + ), + + TP_ARGS(dir, dentry, flags), + + TP_STRUCT__entry( + __field(unsigned long, flags) + __field(dev_t, dev) + __field(u64, dir) + __string(name, dentry->d_name.name) + ), + + TP_fast_assign( + __entry->dev = dir->i_sb->s_dev; + __entry->dir = dir->i_ino; + __entry->flags = flags; + __assign_str(name); + ), + + TP_printk( + "flags=0x%lx (%s) name=%02x:%02x:%llu/%s", + __entry->flags, + show_fs_fcntl_open_flags(__entry->flags), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->dir, + __get_str(name) + ) +); + +TRACE_EVENT(nfs_create_exit, + TP_PROTO( + const struct inode *dir, + const struct dentry *dentry, + unsigned int flags, + int error + ), + + TP_ARGS(dir, dentry, flags, error), + + TP_STRUCT__entry( + __field(unsigned long, error) + __field(unsigned long, flags) + __field(dev_t, dev) + __field(u64, dir) + __string(name, dentry->d_name.name) + ), + + TP_fast_assign( + __entry->error = -error; + __entry->dev = dir->i_sb->s_dev; + __entry->dir = dir->i_ino; + __entry->flags = flags; + __assign_str(name); + ), + + TP_printk( + "error=%ld (%s) flags=0x%lx (%s) name=%02x:%02x:%llu/%s", + -__entry->error, show_nfs_status(__entry->error), + __entry->flags, + show_fs_fcntl_open_flags(__entry->flags), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->dir, + __get_str(name) + ) +); + +DECLARE_EVENT_CLASS(nfs_directory_event, + TP_PROTO( + const struct inode *dir, + const struct dentry *dentry + ), + + TP_ARGS(dir, dentry), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u64, dir) + __string(name, dentry->d_name.name) + ), + + TP_fast_assign( + __entry->dev = dir->i_sb->s_dev; + __entry->dir = dir->i_ino; + __assign_str(name); + ), + + TP_printk( + "name=%02x:%02x:%llu/%s", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->dir, + __get_str(name) + ) +); + +#define DEFINE_NFS_DIRECTORY_EVENT(name) \ + DEFINE_EVENT(nfs_directory_event, name, \ + TP_PROTO( \ + const struct inode *dir, \ + const struct dentry *dentry \ + ), \ + TP_ARGS(dir, dentry)) + +DECLARE_EVENT_CLASS(nfs_directory_event_done, + TP_PROTO( + const struct inode *dir, + const struct dentry *dentry, + int error + ), + + TP_ARGS(dir, dentry, error), + + TP_STRUCT__entry( + __field(unsigned long, error) + __field(dev_t, dev) + __field(u64, dir) + __string(name, dentry->d_name.name) + ), + + TP_fast_assign( + __entry->dev = dir->i_sb->s_dev; + __entry->dir = dir->i_ino; + __entry->error = error < 0 ? -error : 0; + __assign_str(name); + ), + + TP_printk( + "error=%ld (%s) name=%02x:%02x:%llu/%s", + -__entry->error, show_nfs_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->dir, + __get_str(name) + ) +); + +#define DEFINE_NFS_DIRECTORY_EVENT_DONE(name) \ + DEFINE_EVENT(nfs_directory_event_done, name, \ + TP_PROTO( \ + const struct inode *dir, \ + const struct dentry *dentry, \ + int error \ + ), \ + TP_ARGS(dir, dentry, error)) + +DEFINE_NFS_DIRECTORY_EVENT(nfs_mknod_enter); +DEFINE_NFS_DIRECTORY_EVENT_DONE(nfs_mknod_exit); +DEFINE_NFS_DIRECTORY_EVENT(nfs_mkdir_enter); +DEFINE_NFS_DIRECTORY_EVENT_DONE(nfs_mkdir_exit); +DEFINE_NFS_DIRECTORY_EVENT(nfs_rmdir_enter); +DEFINE_NFS_DIRECTORY_EVENT_DONE(nfs_rmdir_exit); +DEFINE_NFS_DIRECTORY_EVENT(nfs_remove_enter); +DEFINE_NFS_DIRECTORY_EVENT_DONE(nfs_remove_exit); +DEFINE_NFS_DIRECTORY_EVENT(nfs_unlink_enter); +DEFINE_NFS_DIRECTORY_EVENT_DONE(nfs_unlink_exit); +DEFINE_NFS_DIRECTORY_EVENT(nfs_symlink_enter); +DEFINE_NFS_DIRECTORY_EVENT_DONE(nfs_symlink_exit); + +TRACE_EVENT(nfs_link_enter, + TP_PROTO( + const struct inode *inode, + const struct inode *dir, + const struct dentry *dentry + ), + + TP_ARGS(inode, dir, dentry), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u64, fileid) + __field(u64, dir) + __string(name, dentry->d_name.name) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->dir = dir->i_ino; + __assign_str(name); + ), + + TP_printk( + "fileid=%02x:%02x:%llu name=%02x:%02x:%llu/%s", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->fileid, + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->dir, + __get_str(name) + ) +); + +TRACE_EVENT(nfs_link_exit, + TP_PROTO( + const struct inode *inode, + const struct inode *dir, + const struct dentry *dentry, + int error + ), + + TP_ARGS(inode, dir, dentry, error), + + TP_STRUCT__entry( + __field(unsigned long, error) + __field(dev_t, dev) + __field(u64, fileid) + __field(u64, dir) + __string(name, dentry->d_name.name) + ), + + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->dir = dir->i_ino; + __entry->error = error < 0 ? -error : 0; + __assign_str(name); + ), + + TP_printk( + "error=%ld (%s) fileid=%02x:%02x:%llu name=%02x:%02x:%llu/%s", + -__entry->error, show_nfs_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->fileid, + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->dir, + __get_str(name) + ) +); + +DECLARE_EVENT_CLASS(nfs_rename_event, + TP_PROTO( + const struct inode *old_dir, + const struct dentry *old_dentry, + const struct inode *new_dir, + const struct dentry *new_dentry + ), + + TP_ARGS(old_dir, old_dentry, new_dir, new_dentry), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u64, old_dir) + __field(u64, new_dir) + __string(old_name, old_dentry->d_name.name) + __string(new_name, new_dentry->d_name.name) + ), + + TP_fast_assign( + __entry->dev = old_dir->i_sb->s_dev; + __entry->old_dir = old_dir->i_ino; + __entry->new_dir = new_dir->i_ino; + __assign_str(old_name); + __assign_str(new_name); + ), + + TP_printk( + "old_name=%02x:%02x:%llu/%s new_name=%02x:%02x:%llu/%s", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->old_dir, + __get_str(old_name), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->new_dir, + __get_str(new_name) + ) +); +#define DEFINE_NFS_RENAME_EVENT(name) \ + DEFINE_EVENT(nfs_rename_event, name, \ + TP_PROTO( \ + const struct inode *old_dir, \ + const struct dentry *old_dentry, \ + const struct inode *new_dir, \ + const struct dentry *new_dentry \ + ), \ + TP_ARGS(old_dir, old_dentry, new_dir, new_dentry)) + +DECLARE_EVENT_CLASS(nfs_rename_event_done, + TP_PROTO( + const struct inode *old_dir, + const struct dentry *old_dentry, + const struct inode *new_dir, + const struct dentry *new_dentry, + int error + ), + + TP_ARGS(old_dir, old_dentry, new_dir, new_dentry, error), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned long, error) + __field(u64, old_dir) + __string(old_name, old_dentry->d_name.name) + __field(u64, new_dir) + __string(new_name, new_dentry->d_name.name) + ), + + TP_fast_assign( + __entry->dev = old_dir->i_sb->s_dev; + __entry->error = -error; + __entry->old_dir = old_dir->i_ino; + __entry->new_dir = new_dir->i_ino; + __assign_str(old_name); + __assign_str(new_name); + ), + + TP_printk( + "error=%ld (%s) old_name=%02x:%02x:%llu/%s " + "new_name=%02x:%02x:%llu/%s", + -__entry->error, show_nfs_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->old_dir, + __get_str(old_name), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->new_dir, + __get_str(new_name) + ) +); +#define DEFINE_NFS_RENAME_EVENT_DONE(name) \ + DEFINE_EVENT(nfs_rename_event_done, name, \ + TP_PROTO( \ + const struct inode *old_dir, \ + const struct dentry *old_dentry, \ + const struct inode *new_dir, \ + const struct dentry *new_dentry, \ + int error \ + ), \ + TP_ARGS(old_dir, old_dentry, new_dir, \ + new_dentry, error)) + +DEFINE_NFS_RENAME_EVENT(nfs_rename_enter); +DEFINE_NFS_RENAME_EVENT_DONE(nfs_rename_exit); + +DEFINE_NFS_RENAME_EVENT_DONE(nfs_async_rename_done); + +TRACE_EVENT(nfs_sillyrename_unlink, + TP_PROTO( + const struct nfs_unlinkdata *data, + int error + ), + + TP_ARGS(data, error), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned long, error) + __field(u64, dir) + __dynamic_array(char, name, data->args.name.len + 1) + ), + + TP_fast_assign( + struct inode *dir = d_inode(data->dentry->d_parent); + size_t len = data->args.name.len; + __entry->dev = dir->i_sb->s_dev; + __entry->dir = dir->i_ino; + __entry->error = -error; + memcpy(__get_str(name), + data->args.name.name, len); + __get_str(name)[len] = 0; + ), + + TP_printk( + "error=%ld (%s) name=%02x:%02x:%llu/%s", + -__entry->error, show_nfs_status(__entry->error), + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->dir, + __get_str(name) + ) +); + +DECLARE_EVENT_CLASS(nfs_folio_event, + TP_PROTO( + const struct inode *inode, + loff_t offset, + size_t count + ), + + TP_ARGS(inode, offset, count), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(u64, version) + __field(loff_t, offset) + __field(size_t, count) + ), + + TP_fast_assign( + const struct nfs_inode *nfsi = NFS_I(inode); + + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); + __entry->version = inode_peek_iversion_raw(inode); + __entry->offset = offset; + __entry->count = count; + ), + + TP_printk( + "fileid=%02x:%02x:%llu fhandle=0x%08x version=%llu " + "offset=%lld count=%zu", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, __entry->version, + __entry->offset, __entry->count + ) +); + +#define DEFINE_NFS_FOLIO_EVENT(name) \ + DEFINE_EVENT(nfs_folio_event, name, \ + TP_PROTO( \ + const struct inode *inode, \ + loff_t offset, \ + size_t count \ + ), \ + TP_ARGS(inode, offset, count)) + +DECLARE_EVENT_CLASS(nfs_folio_event_done, + TP_PROTO( + const struct inode *inode, + loff_t offset, + size_t count, + int ret + ), + + TP_ARGS(inode, offset, count, ret), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(int, ret) + __field(u64, fileid) + __field(u64, version) + __field(loff_t, offset) + __field(size_t, count) + ), + + TP_fast_assign( + const struct nfs_inode *nfsi = NFS_I(inode); + + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); + __entry->version = inode_peek_iversion_raw(inode); + __entry->offset = offset; + __entry->count = count; + __entry->ret = ret; + ), + + TP_printk( + "fileid=%02x:%02x:%llu fhandle=0x%08x version=%llu " + "offset=%lld count=%zu ret=%d", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, __entry->version, + __entry->offset, __entry->count, __entry->ret + ) +); + +#define DEFINE_NFS_FOLIO_EVENT_DONE(name) \ + DEFINE_EVENT(nfs_folio_event_done, name, \ + TP_PROTO( \ + const struct inode *inode, \ + loff_t offset, \ + size_t count, \ + int ret \ + ), \ + TP_ARGS(inode, offset, count, ret)) + +DEFINE_NFS_FOLIO_EVENT(nfs_aop_readpage); +DEFINE_NFS_FOLIO_EVENT_DONE(nfs_aop_readpage_done); + +DEFINE_NFS_FOLIO_EVENT(nfs_writeback_folio_reclaim); +DEFINE_NFS_FOLIO_EVENT_DONE(nfs_writeback_folio_reclaim_done); + +DEFINE_NFS_FOLIO_EVENT(nfs_writeback_folio); +DEFINE_NFS_FOLIO_EVENT_DONE(nfs_writeback_folio_done); + +DEFINE_NFS_FOLIO_EVENT(nfs_invalidate_folio); +DEFINE_NFS_FOLIO_EVENT_DONE(nfs_launder_folio_done); + +DEFINE_NFS_FOLIO_EVENT(nfs_try_to_update_request); +DEFINE_NFS_FOLIO_EVENT_DONE(nfs_try_to_update_request_done); + +DEFINE_NFS_FOLIO_EVENT(nfs_update_folio); +DEFINE_NFS_FOLIO_EVENT_DONE(nfs_update_folio_done); + +DEFINE_NFS_FOLIO_EVENT(nfs_write_begin); +DEFINE_NFS_FOLIO_EVENT_DONE(nfs_write_begin_done); + +DEFINE_NFS_FOLIO_EVENT(nfs_write_end); +DEFINE_NFS_FOLIO_EVENT_DONE(nfs_write_end_done); + +DEFINE_NFS_FOLIO_EVENT(nfs_writepages); +DEFINE_NFS_FOLIO_EVENT_DONE(nfs_writepages_done); + +DECLARE_EVENT_CLASS(nfs_kiocb_event, + TP_PROTO( + const struct kiocb *iocb, + const struct iov_iter *iter + ), + + TP_ARGS(iocb, iter), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(u64, version) + __field(loff_t, offset) + __field(size_t, count) + __field(int, flags) + ), + + TP_fast_assign( + const struct inode *inode = file_inode(iocb->ki_filp); + const struct nfs_inode *nfsi = NFS_I(inode); + + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); + __entry->version = inode_peek_iversion_raw(inode); + __entry->offset = iocb->ki_pos; + __entry->count = iov_iter_count(iter); + __entry->flags = iocb->ki_flags; + ), + + TP_printk( + "fileid=%02x:%02x:%llu fhandle=0x%08x version=%llu offset=%lld count=%zu ki_flags=%s", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, __entry->version, + __entry->offset, __entry->count, + __print_flags(__entry->flags, "|", TRACE_IOCB_STRINGS) + ) +); + +#define DEFINE_NFS_KIOCB_EVENT(name) \ + DEFINE_EVENT(nfs_kiocb_event, name, \ + TP_PROTO( \ + const struct kiocb *iocb, \ + const struct iov_iter *iter \ + ), \ + TP_ARGS(iocb, iter)) + +DEFINE_NFS_KIOCB_EVENT(nfs_file_read); +DEFINE_NFS_KIOCB_EVENT(nfs_file_write); + +TRACE_EVENT(nfs_aop_readahead, + TP_PROTO( + const struct inode *inode, + loff_t pos, + unsigned int nr_pages + ), + + TP_ARGS(inode, pos, nr_pages), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(u64, version) + __field(loff_t, offset) + __field(unsigned int, nr_pages) + ), + + TP_fast_assign( + const struct nfs_inode *nfsi = NFS_I(inode); + + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); + __entry->version = inode_peek_iversion_raw(inode); + __entry->offset = pos; + __entry->nr_pages = nr_pages; + ), + + TP_printk( + "fileid=%02x:%02x:%llu fhandle=0x%08x version=%llu offset=%lld nr_pages=%u", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, __entry->version, + __entry->offset, __entry->nr_pages + ) +); + +TRACE_EVENT(nfs_aop_readahead_done, + TP_PROTO( + const struct inode *inode, + unsigned int nr_pages, + int ret + ), + + TP_ARGS(inode, nr_pages, ret), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(int, ret) + __field(u64, fileid) + __field(u64, version) + __field(loff_t, offset) + __field(unsigned int, nr_pages) + ), + + TP_fast_assign( + const struct nfs_inode *nfsi = NFS_I(inode); + + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); + __entry->version = inode_peek_iversion_raw(inode); + __entry->nr_pages = nr_pages; + __entry->ret = ret; + ), + + TP_printk( + "fileid=%02x:%02x:%llu fhandle=0x%08x version=%llu nr_pages=%u ret=%d", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, __entry->version, + __entry->nr_pages, __entry->ret + ) +); + +TRACE_EVENT(nfs_initiate_read, + TP_PROTO( + const struct nfs_pgio_header *hdr + ), + + TP_ARGS(hdr), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(loff_t, offset) + __field(u32, count) + ), + + TP_fast_assign( + const struct inode *inode = hdr->inode; + const struct nfs_inode *nfsi = NFS_I(inode); + const struct nfs_fh *fh = hdr->args.fh ? + hdr->args.fh : &nfsi->fh; + + __entry->offset = hdr->args.offset; + __entry->count = hdr->args.count; + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(fh); + ), + + TP_printk( + "fileid=%02x:%02x:%llu fhandle=0x%08x " + "offset=%lld count=%u", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + (long long)__entry->offset, __entry->count + ) +); + +TRACE_EVENT(nfs_readpage_done, + TP_PROTO( + const struct rpc_task *task, + const struct nfs_pgio_header *hdr + ), + + TP_ARGS(task, hdr), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(loff_t, offset) + __field(u32, arg_count) + __field(u32, res_count) + __field(bool, eof) + __field(int, error) + ), + + TP_fast_assign( + const struct inode *inode = hdr->inode; + const struct nfs_inode *nfsi = NFS_I(inode); + const struct nfs_fh *fh = hdr->args.fh ? + hdr->args.fh : &nfsi->fh; + + __entry->error = task->tk_status; + __entry->offset = hdr->args.offset; + __entry->arg_count = hdr->args.count; + __entry->res_count = hdr->res.count; + __entry->eof = hdr->res.eof; + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(fh); + ), + + TP_printk( + "error=%d fileid=%02x:%02x:%llu fhandle=0x%08x " + "offset=%lld count=%u res=%u%s", __entry->error, + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + (long long)__entry->offset, __entry->arg_count, + __entry->res_count, __entry->eof ? " eof" : "" + ) +); + +TRACE_EVENT(nfs_readpage_short, + TP_PROTO( + const struct rpc_task *task, + const struct nfs_pgio_header *hdr + ), + + TP_ARGS(task, hdr), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(loff_t, offset) + __field(u32, arg_count) + __field(u32, res_count) + __field(bool, eof) + __field(int, error) + ), + + TP_fast_assign( + const struct inode *inode = hdr->inode; + const struct nfs_inode *nfsi = NFS_I(inode); + const struct nfs_fh *fh = hdr->args.fh ? + hdr->args.fh : &nfsi->fh; + + __entry->error = task->tk_status; + __entry->offset = hdr->args.offset; + __entry->arg_count = hdr->args.count; + __entry->res_count = hdr->res.count; + __entry->eof = hdr->res.eof; + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(fh); + ), + + TP_printk( + "error=%d fileid=%02x:%02x:%llu fhandle=0x%08x " + "offset=%lld count=%u res=%u%s", __entry->error, + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + (long long)__entry->offset, __entry->arg_count, + __entry->res_count, __entry->eof ? " eof" : "" + ) +); + + +TRACE_EVENT(nfs_pgio_error, + TP_PROTO( + const struct nfs_pgio_header *hdr, + int error, + loff_t pos + ), + + TP_ARGS(hdr, error, pos), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(loff_t, offset) + __field(u32, arg_count) + __field(u32, res_count) + __field(loff_t, pos) + __field(int, error) + ), + + TP_fast_assign( + const struct inode *inode = hdr->inode; + const struct nfs_inode *nfsi = NFS_I(inode); + const struct nfs_fh *fh = hdr->args.fh ? + hdr->args.fh : &nfsi->fh; + + __entry->error = error; + __entry->offset = hdr->args.offset; + __entry->arg_count = hdr->args.count; + __entry->res_count = hdr->res.count; + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(fh); + ), + + TP_printk("error=%d fileid=%02x:%02x:%llu fhandle=0x%08x " + "offset=%lld count=%u res=%u pos=%llu", __entry->error, + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, __entry->fhandle, + (long long)__entry->offset, __entry->arg_count, __entry->res_count, + __entry->pos + ) +); + +TRACE_EVENT(nfs_initiate_write, + TP_PROTO( + const struct nfs_pgio_header *hdr + ), + + TP_ARGS(hdr), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(loff_t, offset) + __field(u32, count) + __field(unsigned long, stable) + ), + + TP_fast_assign( + const struct inode *inode = hdr->inode; + const struct nfs_inode *nfsi = NFS_I(inode); + const struct nfs_fh *fh = hdr->args.fh ? + hdr->args.fh : &nfsi->fh; + + __entry->offset = hdr->args.offset; + __entry->count = hdr->args.count; + __entry->stable = hdr->args.stable; + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(fh); + ), + + TP_printk( + "fileid=%02x:%02x:%llu fhandle=0x%08x " + "offset=%lld count=%u stable=%s", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + (long long)__entry->offset, __entry->count, + show_nfs_stable_how(__entry->stable) + ) +); + +TRACE_EVENT(nfs_writeback_done, + TP_PROTO( + const struct rpc_task *task, + const struct nfs_pgio_header *hdr + ), + + TP_ARGS(task, hdr), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(loff_t, offset) + __field(u32, arg_count) + __field(u32, res_count) + __field(int, error) + __field(unsigned long, stable) + __array(char, verifier, NFS4_VERIFIER_SIZE) + ), + + TP_fast_assign( + const struct inode *inode = hdr->inode; + const struct nfs_inode *nfsi = NFS_I(inode); + const struct nfs_fh *fh = hdr->args.fh ? + hdr->args.fh : &nfsi->fh; + const struct nfs_writeverf *verf = hdr->res.verf; + + __entry->error = task->tk_status; + __entry->offset = hdr->args.offset; + __entry->arg_count = hdr->args.count; + __entry->res_count = hdr->res.count; + __entry->stable = verf->committed; + memcpy(__entry->verifier, + &verf->verifier, + NFS4_VERIFIER_SIZE); + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(fh); + ), + + TP_printk( + "error=%d fileid=%02x:%02x:%llu fhandle=0x%08x " + "offset=%lld count=%u res=%u stable=%s " + "verifier=%s", __entry->error, + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + (long long)__entry->offset, __entry->arg_count, + __entry->res_count, + show_nfs_stable_how(__entry->stable), + show_nfs4_verifier(__entry->verifier) + ) +); + +DECLARE_EVENT_CLASS(nfs_page_class, + TP_PROTO( + const struct nfs_page *req + ), + + TP_ARGS(req), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(const struct nfs_page *, req) + __field(loff_t, offset) + __field(unsigned int, count) + __field(unsigned long, flags) + ), + + TP_fast_assign( + const struct inode *inode = folio_inode(req->wb_folio); + const struct nfs_inode *nfsi = NFS_I(inode); + + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); + __entry->req = req; + __entry->offset = req_offset(req); + __entry->count = req->wb_bytes; + __entry->flags = req->wb_flags; + ), + + TP_printk( + "fileid=%02x:%02x:%llu fhandle=0x%08x req=%p offset=%lld count=%u flags=%s", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, __entry->fhandle, + __entry->req, __entry->offset, __entry->count, + nfs_show_wb_flags(__entry->flags) + ) +); + +#define DEFINE_NFS_PAGE_EVENT(name) \ + DEFINE_EVENT(nfs_page_class, name, \ + TP_PROTO( \ + const struct nfs_page *req \ + ), \ + TP_ARGS(req)) + +DEFINE_NFS_PAGE_EVENT(nfs_writepage_setup); +DEFINE_NFS_PAGE_EVENT(nfs_do_writepage); + +DECLARE_EVENT_CLASS(nfs_page_error_class, + TP_PROTO( + const struct inode *inode, + const struct nfs_page *req, + int error + ), + + TP_ARGS(inode, req, error), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(loff_t, offset) + __field(unsigned int, count) + __field(int, error) + ), + + TP_fast_assign( + const struct nfs_inode *nfsi = NFS_I(inode); + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); + __entry->offset = req_offset(req); + __entry->count = req->wb_bytes; + __entry->error = error; + ), + + TP_printk( + "error=%d fileid=%02x:%02x:%llu fhandle=0x%08x " + "offset=%lld count=%u", __entry->error, + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, __entry->offset, + __entry->count + ) +); + +#define DEFINE_NFS_PAGEERR_EVENT(name) \ + DEFINE_EVENT(nfs_page_error_class, name, \ + TP_PROTO( \ + const struct inode *inode, \ + const struct nfs_page *req, \ + int error \ + ), \ + TP_ARGS(inode, req, error)) + +DEFINE_NFS_PAGEERR_EVENT(nfs_write_error); +DEFINE_NFS_PAGEERR_EVENT(nfs_comp_error); +DEFINE_NFS_PAGEERR_EVENT(nfs_commit_error); + +TRACE_EVENT(nfs_initiate_commit, + TP_PROTO( + const struct nfs_commit_data *data + ), + + TP_ARGS(data), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(loff_t, offset) + __field(u32, count) + ), + + TP_fast_assign( + const struct inode *inode = data->inode; + const struct nfs_inode *nfsi = NFS_I(inode); + const struct nfs_fh *fh = data->args.fh ? + data->args.fh : &nfsi->fh; + + __entry->offset = data->args.offset; + __entry->count = data->args.count; + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(fh); + ), + + TP_printk( + "fileid=%02x:%02x:%llu fhandle=0x%08x " + "offset=%lld count=%u", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + (long long)__entry->offset, __entry->count + ) +); + +TRACE_EVENT(nfs_commit_done, + TP_PROTO( + const struct rpc_task *task, + const struct nfs_commit_data *data + ), + + TP_ARGS(task, data), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(loff_t, offset) + __field(int, error) + __field(unsigned long, stable) + __array(char, verifier, NFS4_VERIFIER_SIZE) + ), + + TP_fast_assign( + const struct inode *inode = data->inode; + const struct nfs_inode *nfsi = NFS_I(inode); + const struct nfs_fh *fh = data->args.fh ? + data->args.fh : &nfsi->fh; + const struct nfs_writeverf *verf = data->res.verf; + + __entry->error = task->tk_status; + __entry->offset = data->args.offset; + __entry->stable = verf->committed; + memcpy(__entry->verifier, + &verf->verifier, + NFS4_VERIFIER_SIZE); + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(fh); + ), + + TP_printk( + "error=%d fileid=%02x:%02x:%llu fhandle=0x%08x " + "offset=%lld stable=%s verifier=%s", __entry->error, + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + (long long)__entry->offset, + show_nfs_stable_how(__entry->stable), + show_nfs4_verifier(__entry->verifier) + ) +); + +#define nfs_show_direct_req_flags(v) \ + __print_flags(v, "|", \ + { NFS_ODIRECT_DO_COMMIT, "DO_COMMIT" }, \ + { NFS_ODIRECT_RESCHED_WRITES, "RESCHED_WRITES" }, \ + { NFS_ODIRECT_SHOULD_DIRTY, "SHOULD DIRTY" }, \ + { NFS_ODIRECT_DONE, "DONE" } ) + +DECLARE_EVENT_CLASS(nfs_direct_req_class, + TP_PROTO( + const struct nfs_direct_req *dreq + ), + + TP_ARGS(dreq), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u64, fileid) + __field(u32, fhandle) + __field(loff_t, offset) + __field(ssize_t, count) + __field(ssize_t, error) + __field(int, flags) + ), + + TP_fast_assign( + const struct inode *inode = dreq->inode; + const struct nfs_inode *nfsi = NFS_I(inode); + const struct nfs_fh *fh = &nfsi->fh; + + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(fh); + __entry->offset = dreq->io_start; + __entry->count = dreq->count; + __entry->error = dreq->error; + __entry->flags = dreq->flags; + ), + + TP_printk( + "error=%zd fileid=%02x:%02x:%llu fhandle=0x%08x " + "offset=%lld count=%zd flags=%s", + __entry->error, MAJOR(__entry->dev), + MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, __entry->offset, + __entry->count, + nfs_show_direct_req_flags(__entry->flags) + ) +); + +#define DEFINE_NFS_DIRECT_REQ_EVENT(name) \ + DEFINE_EVENT(nfs_direct_req_class, name, \ + TP_PROTO( \ + const struct nfs_direct_req *dreq \ + ), \ + TP_ARGS(dreq)) + +DEFINE_NFS_DIRECT_REQ_EVENT(nfs_direct_commit_complete); +DEFINE_NFS_DIRECT_REQ_EVENT(nfs_direct_resched_write); +DEFINE_NFS_DIRECT_REQ_EVENT(nfs_direct_write_complete); +DEFINE_NFS_DIRECT_REQ_EVENT(nfs_direct_write_completion); +DEFINE_NFS_DIRECT_REQ_EVENT(nfs_direct_write_schedule_iovec); +DEFINE_NFS_DIRECT_REQ_EVENT(nfs_direct_write_reschedule_io); + +#if IS_ENABLED(CONFIG_NFS_LOCALIO) + +DECLARE_EVENT_CLASS(nfs_local_dio_class, + TP_PROTO( + const struct inode *inode, + loff_t offset, + ssize_t count, + const struct nfs_local_dio *local_dio + ), + TP_ARGS(inode, offset, count, local_dio), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u64, fileid) + __field(u32, fhandle) + __field(loff_t, offset) + __field(ssize_t, count) + __field(u32, mem_align) + __field(u32, offset_align) + __field(loff_t, start) + __field(ssize_t, start_len) + __field(loff_t, middle) + __field(ssize_t, middle_len) + __field(loff_t, end) + __field(ssize_t, end_len) + ), + TP_fast_assign( + const struct nfs_inode *nfsi = NFS_I(inode); + const struct nfs_fh *fh = &nfsi->fh; + + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = inode->i_ino; + __entry->fhandle = nfs_fhandle_hash(fh); + __entry->offset = offset; + __entry->count = count; + __entry->mem_align = local_dio->mem_align; + __entry->offset_align = local_dio->offset_align; + __entry->start = offset; + __entry->start_len = local_dio->start_len; + __entry->middle = local_dio->middle_offset; + __entry->middle_len = local_dio->middle_len; + __entry->end = local_dio->end_offset; + __entry->end_len = local_dio->end_len; + ), + TP_printk("fileid=%02x:%02x:%llu fhandle=0x%08x " + "offset=%lld count=%zd " + "mem_align=%u offset_align=%u " + "start=%llu+%zd middle=%llu+%zd end=%llu+%zd", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, __entry->offset, __entry->count, + __entry->mem_align, __entry->offset_align, + __entry->start, __entry->start_len, + __entry->middle, __entry->middle_len, + __entry->end, __entry->end_len) +) + +#define DEFINE_NFS_LOCAL_DIO_EVENT(name) \ +DEFINE_EVENT(nfs_local_dio_class, nfs_local_dio_##name, \ + TP_PROTO(const struct inode *inode, \ + loff_t offset, \ + ssize_t count, \ + const struct nfs_local_dio *local_dio),\ + TP_ARGS(inode, offset, count, local_dio)) + +DEFINE_NFS_LOCAL_DIO_EVENT(read); +DEFINE_NFS_LOCAL_DIO_EVENT(write); +DEFINE_NFS_LOCAL_DIO_EVENT(misaligned); + +#endif /* CONFIG_NFS_LOCALIO */ + +TRACE_EVENT(nfs_fh_to_dentry, + TP_PROTO( + const struct super_block *sb, + const struct nfs_fh *fh, + u64 fileid, + int error + ), + + TP_ARGS(sb, fh, fileid, error), + + TP_STRUCT__entry( + __field(int, error) + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + ), + + TP_fast_assign( + __entry->error = error; + __entry->dev = sb->s_dev; + __entry->fileid = fileid; + __entry->fhandle = nfs_fhandle_hash(fh); + ), + + TP_printk( + "error=%d fileid=%02x:%02x:%llu fhandle=0x%08x ", + __entry->error, + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle + ) +); + +TRACE_EVENT(nfs_mount_assign, + TP_PROTO( + const char *option, + const char *value + ), + + TP_ARGS(option, value), + + TP_STRUCT__entry( + __string(option, option) + __string(value, value) + ), + + TP_fast_assign( + __assign_str(option); + __assign_str(value); + ), + + TP_printk("option %s=%s", + __get_str(option), __get_str(value) + ) +); + +TRACE_EVENT(nfs_mount_option, + TP_PROTO( + const struct fs_parameter *param + ), + + TP_ARGS(param), + + TP_STRUCT__entry( + __string(option, param->key) + ), + + TP_fast_assign( + __assign_str(option); + ), + + TP_printk("option %s", __get_str(option)) +); + +TRACE_EVENT(nfs_mount_path, + TP_PROTO( + const char *path + ), + + TP_ARGS(path), + + TP_STRUCT__entry( + __string(path, path) + ), + + TP_fast_assign( + __assign_str(path); + ), + + TP_printk("path='%s'", __get_str(path)) +); + +TRACE_EVENT(nfs_local_open_fh, + TP_PROTO( + const struct nfs_fh *fh, + fmode_t fmode, + int error + ), + + TP_ARGS(fh, fmode, error), + + TP_STRUCT__entry( + __field(int, error) + __field(u32, fhandle) + __field(unsigned int, fmode) + ), + + TP_fast_assign( + __entry->error = error; + __entry->fhandle = nfs_fhandle_hash(fh); + __entry->fmode = (__force unsigned int)fmode; + ), + + TP_printk( + "fhandle=0x%08x mode=%s result=%d", + __entry->fhandle, + show_fs_fmode_flags(__entry->fmode), + __entry->error + ) +); + +DECLARE_EVENT_CLASS(nfs_xdr_event, + TP_PROTO( + const struct xdr_stream *xdr, + int error + ), + + TP_ARGS(xdr, error), + + TP_STRUCT__entry( + __field(unsigned int, task_id) + __field(unsigned int, client_id) + __field(u32, xid) + __field(int, version) + __field(unsigned long, error) + __string(program, + xdr->rqst->rq_task->tk_client->cl_program->name) + __string(procedure, + xdr->rqst->rq_task->tk_msg.rpc_proc->p_name) + ), + + TP_fast_assign( + const struct rpc_rqst *rqstp = xdr->rqst; + const struct rpc_task *task = rqstp->rq_task; + + __entry->task_id = task->tk_pid; + __entry->client_id = task->tk_client->cl_clid; + __entry->xid = be32_to_cpu(rqstp->rq_xid); + __entry->version = task->tk_client->cl_vers; + __entry->error = error; + __assign_str(program); + __assign_str(procedure); + ), + + TP_printk(SUNRPC_TRACE_TASK_SPECIFIER + " xid=0x%08x %sv%d %s error=%ld (%s)", + __entry->task_id, __entry->client_id, __entry->xid, + __get_str(program), __entry->version, + __get_str(procedure), -__entry->error, + show_nfs_status(__entry->error) + ) +); +#define DEFINE_NFS_XDR_EVENT(name) \ + DEFINE_EVENT(nfs_xdr_event, name, \ + TP_PROTO( \ + const struct xdr_stream *xdr, \ + int error \ + ), \ + TP_ARGS(xdr, error)) +DEFINE_NFS_XDR_EVENT(nfs_xdr_status); +DEFINE_NFS_XDR_EVENT(nfs_xdr_bad_filehandle); + +#endif /* _TRACE_NFS_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE nfstrace +/* This part must be outside protection */ +#include diff --git a/fs/nfs/pagelist.c b/fs/nfs/pagelist.c new file mode 100644 index 000000000..7dd478ffc --- /dev/null +++ b/fs/nfs/pagelist.c @@ -0,0 +1,1496 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/nfs/pagelist.c + * + * A set of helper functions for managing NFS read and write requests. + * The main purpose of these routines is to provide support for the + * coalescing of several requests into a single RPC call. + * + * Copyright 2000, 2001 (c) Trond Myklebust + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" +#include "pnfs.h" +#include "nfstrace.h" +#include "fscache.h" + +#define NFSDBG_FACILITY NFSDBG_PAGECACHE + +static struct kmem_cache *nfs_page_cachep; +static const struct rpc_call_ops nfs_pgio_common_ops; + +struct nfs_page_iter_page { + const struct nfs_page *req; + size_t count; +}; + +static void nfs_page_iter_page_init(struct nfs_page_iter_page *i, + const struct nfs_page *req) +{ + i->req = req; + i->count = 0; +} + +static void nfs_page_iter_page_advance(struct nfs_page_iter_page *i, size_t sz) +{ + const struct nfs_page *req = i->req; + size_t tmp = i->count + sz; + + i->count = (tmp < req->wb_bytes) ? tmp : req->wb_bytes; +} + +static struct page *nfs_page_iter_page_get(struct nfs_page_iter_page *i) +{ + const struct nfs_page *req = i->req; + struct page *page; + + if (i->count != req->wb_bytes) { + size_t base = i->count + req->wb_pgbase; + size_t len = PAGE_SIZE - offset_in_page(base); + + page = nfs_page_to_page(req, base); + nfs_page_iter_page_advance(i, len); + return page; + } + return NULL; +} + +static struct nfs_pgio_mirror * +nfs_pgio_get_mirror(struct nfs_pageio_descriptor *desc, u32 idx) +{ + if (desc->pg_ops->pg_get_mirror) + return desc->pg_ops->pg_get_mirror(desc, idx); + return &desc->pg_mirrors[0]; +} + +struct nfs_pgio_mirror * +nfs_pgio_current_mirror(struct nfs_pageio_descriptor *desc) +{ + return nfs_pgio_get_mirror(desc, desc->pg_mirror_idx); +} +EXPORT_SYMBOL_GPL(nfs_pgio_current_mirror); + +static u32 +nfs_pgio_set_current_mirror(struct nfs_pageio_descriptor *desc, u32 idx) +{ + if (desc->pg_ops->pg_set_mirror) + return desc->pg_ops->pg_set_mirror(desc, idx); + return desc->pg_mirror_idx; +} + +void nfs_pgheader_init(struct nfs_pageio_descriptor *desc, + struct nfs_pgio_header *hdr, + void (*release)(struct nfs_pgio_header *hdr)) +{ + struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); + + + hdr->req = nfs_list_entry(mirror->pg_list.next); + hdr->inode = desc->pg_inode; + hdr->cred = nfs_req_openctx(hdr->req)->cred; + hdr->io_start = req_offset(hdr->req); + hdr->good_bytes = mirror->pg_count; + hdr->io_completion = desc->pg_io_completion; + hdr->dreq = desc->pg_dreq; + nfs_netfs_set_pgio_header(hdr, desc); + hdr->release = release; + hdr->completion_ops = desc->pg_completion_ops; + if (hdr->completion_ops->init_hdr) + hdr->completion_ops->init_hdr(hdr); + + hdr->pgio_mirror_idx = desc->pg_mirror_idx; +} +EXPORT_SYMBOL_GPL(nfs_pgheader_init); + +void nfs_set_pgio_error(struct nfs_pgio_header *hdr, int error, loff_t pos) +{ + unsigned int new = pos - hdr->io_start; + + trace_nfs_pgio_error(hdr, error, pos); + if (hdr->good_bytes > new) { + hdr->good_bytes = new; + clear_bit(NFS_IOHDR_EOF, &hdr->flags); + if (!test_and_set_bit(NFS_IOHDR_ERROR, &hdr->flags)) + hdr->error = error; + } +} + +static inline struct nfs_page *nfs_page_alloc(void) +{ + struct nfs_page *p = + kmem_cache_zalloc(nfs_page_cachep, nfs_io_gfp_mask()); + if (p) + INIT_LIST_HEAD(&p->wb_list); + return p; +} + +static inline void +nfs_page_free(struct nfs_page *p) +{ + kmem_cache_free(nfs_page_cachep, p); +} + +/** + * nfs_iocounter_wait - wait for i/o to complete + * @l_ctx: nfs_lock_context with io_counter to use + * + * returns -ERESTARTSYS if interrupted by a fatal signal. + * Otherwise returns 0 once the io_count hits 0. + */ +int +nfs_iocounter_wait(struct nfs_lock_context *l_ctx) +{ + return wait_var_event_killable(&l_ctx->io_count, + !atomic_read(&l_ctx->io_count)); +} + +/** + * nfs_async_iocounter_wait - wait on a rpc_waitqueue for I/O + * to complete + * @task: the rpc_task that should wait + * @l_ctx: nfs_lock_context with io_counter to check + * + * Returns true if there is outstanding I/O to wait on and the + * task has been put to sleep. + */ +bool +nfs_async_iocounter_wait(struct rpc_task *task, struct nfs_lock_context *l_ctx) +{ + struct inode *inode = d_inode(l_ctx->open_context->dentry); + bool ret = false; + + if (atomic_read(&l_ctx->io_count) > 0) { + rpc_sleep_on(&NFS_SERVER(inode)->uoc_rpcwaitq, task, NULL); + ret = true; + } + + if (atomic_read(&l_ctx->io_count) == 0) { + rpc_wake_up_queued_task(&NFS_SERVER(inode)->uoc_rpcwaitq, task); + ret = false; + } + + return ret; +} +EXPORT_SYMBOL_GPL(nfs_async_iocounter_wait); + +/* + * nfs_page_set_headlock - set the request PG_HEADLOCK + * @req: request that is to be locked + * + * this lock must be held when modifying req->wb_head + * + * return 0 on success, < 0 on error + */ +int +nfs_page_set_headlock(struct nfs_page *req) +{ + if (!test_and_set_bit(PG_HEADLOCK, &req->wb_flags)) + return 0; + + set_bit(PG_CONTENDED1, &req->wb_flags); + smp_mb__after_atomic(); + return wait_on_bit_lock(&req->wb_flags, PG_HEADLOCK, + TASK_UNINTERRUPTIBLE); +} + +/* + * nfs_page_clear_headlock - clear the request PG_HEADLOCK + * @req: request that is to be locked + */ +void +nfs_page_clear_headlock(struct nfs_page *req) +{ + clear_bit_unlock(PG_HEADLOCK, &req->wb_flags); + smp_mb__after_atomic(); + if (!test_bit(PG_CONTENDED1, &req->wb_flags)) + return; + wake_up_bit(&req->wb_flags, PG_HEADLOCK); +} + +/* + * nfs_page_group_lock - lock the head of the page group + * @req: request in group that is to be locked + * + * this lock must be held when traversing or modifying the page + * group list + * + * return 0 on success, < 0 on error + */ +int +nfs_page_group_lock(struct nfs_page *req) +{ + int ret; + + ret = nfs_page_set_headlock(req); + if (ret || req->wb_head == req) + return ret; + return nfs_page_set_headlock(req->wb_head); +} + +/* + * nfs_page_group_unlock - unlock the head of the page group + * @req: request in group that is to be unlocked + */ +void +nfs_page_group_unlock(struct nfs_page *req) +{ + if (req != req->wb_head) + nfs_page_clear_headlock(req->wb_head); + nfs_page_clear_headlock(req); +} + +/** + * nfs_page_group_sync_on_bit_locked - Test if all requests have @bit set + * @req: request in page group + * @bit: PG_* bit that is used to sync page group + * + * must be called with page group lock held + */ +bool nfs_page_group_sync_on_bit_locked(struct nfs_page *req, unsigned int bit) +{ + struct nfs_page *head = req->wb_head; + struct nfs_page *tmp; + + WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &head->wb_flags)); + WARN_ON_ONCE(test_and_set_bit(bit, &req->wb_flags)); + + tmp = req->wb_this_page; + while (tmp != req) { + if (!test_bit(bit, &tmp->wb_flags)) + return false; + tmp = tmp->wb_this_page; + } + + /* true! reset all bits */ + tmp = req; + do { + clear_bit(bit, &tmp->wb_flags); + tmp = tmp->wb_this_page; + } while (tmp != req); + + return true; +} + +/* + * nfs_page_group_sync_on_bit - set bit on current request, but only + * return true if the bit is set for all requests in page group + * @req - request in page group + * @bit - PG_* bit that is used to sync page group + */ +bool nfs_page_group_sync_on_bit(struct nfs_page *req, unsigned int bit) +{ + bool ret; + + nfs_page_group_lock(req); + ret = nfs_page_group_sync_on_bit_locked(req, bit); + nfs_page_group_unlock(req); + + return ret; +} + +/* + * nfs_page_group_init - Initialize the page group linkage for @req + * @req - a new nfs request + * @prev - the previous request in page group, or NULL if @req is the first + * or only request in the group (the head). + */ +static inline void +nfs_page_group_init(struct nfs_page *req, struct nfs_page *prev) +{ + struct inode *inode; + WARN_ON_ONCE(prev == req); + + if (!prev) { + /* a head request */ + req->wb_head = req; + req->wb_this_page = req; + } else { + /* a subrequest */ + WARN_ON_ONCE(prev->wb_this_page != prev->wb_head); + WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &prev->wb_head->wb_flags)); + req->wb_head = prev->wb_head; + req->wb_this_page = prev->wb_this_page; + prev->wb_this_page = req; + + /* All subrequests take a ref on the head request until + * nfs_page_group_destroy is called */ + kref_get(&req->wb_head->wb_kref); + + /* grab extra ref and bump the request count if head request + * has extra ref from the write/commit path to handle handoff + * between write and commit lists. */ + if (test_bit(PG_INODE_REF, &prev->wb_head->wb_flags)) { + inode = nfs_page_to_inode(req); + set_bit(PG_INODE_REF, &req->wb_flags); + kref_get(&req->wb_kref); + atomic_long_inc(&NFS_I(inode)->nrequests); + } + } +} + +/* + * nfs_page_group_destroy - sync the destruction of page groups + * @req - request that no longer needs the page group + * + * releases the page group reference from each member once all + * members have called this function. + */ +static void +nfs_page_group_destroy(struct kref *kref) +{ + struct nfs_page *req = container_of(kref, struct nfs_page, wb_kref); + struct nfs_page *head = req->wb_head; + struct nfs_page *tmp, *next; + + if (!nfs_page_group_sync_on_bit(req, PG_TEARDOWN)) + goto out; + + tmp = req; + do { + next = tmp->wb_this_page; + /* unlink and free */ + tmp->wb_this_page = tmp; + tmp->wb_head = tmp; + nfs_free_request(tmp); + tmp = next; + } while (tmp != req); +out: + /* subrequests must release the ref on the head request */ + if (head != req) + nfs_release_request(head); +} + +static struct nfs_page *nfs_page_create(struct nfs_lock_context *l_ctx, + unsigned int pgbase, pgoff_t index, + unsigned int offset, unsigned int count) +{ + struct nfs_page *req; + struct nfs_open_context *ctx = l_ctx->open_context; + + if (test_bit(NFS_CONTEXT_BAD, &ctx->flags)) + return ERR_PTR(-EBADF); + /* try to allocate the request struct */ + req = nfs_page_alloc(); + if (req == NULL) + return ERR_PTR(-ENOMEM); + + req->wb_lock_context = l_ctx; + refcount_inc(&l_ctx->count); + atomic_inc(&l_ctx->io_count); + + /* Initialize the request struct. Initially, we assume a + * long write-back delay. This will be adjusted in + * update_nfs_request below if the region is not locked. */ + req->wb_pgbase = pgbase; + req->wb_index = index; + req->wb_offset = offset; + req->wb_bytes = count; + kref_init(&req->wb_kref); + req->wb_nio = 0; + return req; +} + +static void nfs_page_assign_folio(struct nfs_page *req, struct folio *folio) +{ + if (folio != NULL) { + req->wb_folio = folio; + folio_get(folio); + set_bit(PG_FOLIO, &req->wb_flags); + } +} + +static void nfs_page_assign_page(struct nfs_page *req, struct page *page) +{ + if (page != NULL) { + req->wb_page = page; + get_page(page); + } +} + +/** + * nfs_page_create_from_page - Create an NFS read/write request. + * @ctx: open context to use + * @page: page to write + * @pgbase: starting offset within the page for the write + * @offset: file offset for the write + * @count: number of bytes to read/write + * + * The page must be locked by the caller. This makes sure we never + * create two different requests for the same page. + * User should ensure it is safe to sleep in this function. + */ +struct nfs_page *nfs_page_create_from_page(struct nfs_open_context *ctx, + struct page *page, + unsigned int pgbase, loff_t offset, + unsigned int count) +{ + struct nfs_lock_context *l_ctx = nfs_get_lock_context(ctx); + struct nfs_page *ret; + + if (IS_ERR(l_ctx)) + return ERR_CAST(l_ctx); + ret = nfs_page_create(l_ctx, pgbase, offset >> PAGE_SHIFT, + offset_in_page(offset), count); + if (!IS_ERR(ret)) { + nfs_page_assign_page(ret, page); + nfs_page_group_init(ret, NULL); + } + nfs_put_lock_context(l_ctx); + return ret; +} + +/** + * nfs_page_create_from_folio - Create an NFS read/write request. + * @ctx: open context to use + * @folio: folio to write + * @offset: starting offset within the folio for the write + * @count: number of bytes to read/write + * + * The page must be locked by the caller. This makes sure we never + * create two different requests for the same page. + * User should ensure it is safe to sleep in this function. + */ +struct nfs_page *nfs_page_create_from_folio(struct nfs_open_context *ctx, + struct folio *folio, + unsigned int offset, + unsigned int count) +{ + struct nfs_lock_context *l_ctx = nfs_get_lock_context(ctx); + struct nfs_page *ret; + + if (IS_ERR(l_ctx)) + return ERR_CAST(l_ctx); + ret = nfs_page_create(l_ctx, offset, folio->index, offset, count); + if (!IS_ERR(ret)) { + nfs_page_assign_folio(ret, folio); + nfs_page_group_init(ret, NULL); + } + nfs_put_lock_context(l_ctx); + return ret; +} + +static struct nfs_page * +nfs_create_subreq(struct nfs_page *req, + unsigned int pgbase, + unsigned int offset, + unsigned int count) +{ + struct nfs_page *last; + struct nfs_page *ret; + struct folio *folio = nfs_page_to_folio(req); + struct page *page = nfs_page_to_page(req, pgbase); + + ret = nfs_page_create(req->wb_lock_context, pgbase, req->wb_index, + offset, count); + if (!IS_ERR(ret)) { + if (folio) + nfs_page_assign_folio(ret, folio); + else + nfs_page_assign_page(ret, page); + /* find the last request */ + for (last = req->wb_head; + last->wb_this_page != req->wb_head; + last = last->wb_this_page) + ; + + nfs_lock_request(ret); + nfs_page_group_init(ret, last); + ret->wb_nio = req->wb_nio; + } + return ret; +} + +/** + * nfs_unlock_request - Unlock request and wake up sleepers. + * @req: pointer to request + */ +void nfs_unlock_request(struct nfs_page *req) +{ + clear_bit_unlock(PG_BUSY, &req->wb_flags); + smp_mb__after_atomic(); + if (!test_bit(PG_CONTENDED2, &req->wb_flags)) + return; + wake_up_bit(&req->wb_flags, PG_BUSY); +} + +/** + * nfs_unlock_and_release_request - Unlock request and release the nfs_page + * @req: pointer to request + */ +void nfs_unlock_and_release_request(struct nfs_page *req) +{ + nfs_unlock_request(req); + nfs_release_request(req); +} + +/* + * nfs_clear_request - Free up all resources allocated to the request + * @req: + * + * Release page and open context resources associated with a read/write + * request after it has completed. + */ +static void nfs_clear_request(struct nfs_page *req) +{ + struct folio *folio = nfs_page_to_folio(req); + struct page *page = req->wb_page; + struct nfs_lock_context *l_ctx = req->wb_lock_context; + struct nfs_open_context *ctx; + + if (folio != NULL) { + folio_put(folio); + req->wb_folio = NULL; + clear_bit(PG_FOLIO, &req->wb_flags); + } else if (page != NULL) { + put_page(page); + req->wb_page = NULL; + } + if (l_ctx != NULL) { + if (atomic_dec_and_test(&l_ctx->io_count)) { + wake_up_var(&l_ctx->io_count); + ctx = l_ctx->open_context; + if (test_bit(NFS_CONTEXT_UNLOCK, &ctx->flags)) + rpc_wake_up(&NFS_SERVER(d_inode(ctx->dentry))->uoc_rpcwaitq); + } + nfs_put_lock_context(l_ctx); + req->wb_lock_context = NULL; + } +} + +/** + * nfs_free_request - Release the count on an NFS read/write request + * @req: request to release + * + * Note: Should never be called with the spinlock held! + */ +void nfs_free_request(struct nfs_page *req) +{ + WARN_ON_ONCE(req->wb_this_page != req); + + /* extra debug: make sure no sync bits are still set */ + WARN_ON_ONCE(test_bit(PG_TEARDOWN, &req->wb_flags)); + WARN_ON_ONCE(test_bit(PG_UNLOCKPAGE, &req->wb_flags)); + WARN_ON_ONCE(test_bit(PG_UPTODATE, &req->wb_flags)); + WARN_ON_ONCE(test_bit(PG_WB_END, &req->wb_flags)); + WARN_ON_ONCE(test_bit(PG_REMOVE, &req->wb_flags)); + + /* Release struct file and open context */ + nfs_clear_request(req); + nfs_page_free(req); +} + +void nfs_release_request(struct nfs_page *req) +{ + kref_put(&req->wb_kref, nfs_page_group_destroy); +} +EXPORT_SYMBOL_GPL(nfs_release_request); + +/* + * nfs_generic_pg_test - determine if requests can be coalesced + * @desc: pointer to descriptor + * @prev: previous request in desc, or NULL + * @req: this request + * + * Returns zero if @req cannot be coalesced into @desc, otherwise it returns + * the size of the request. + */ +size_t nfs_generic_pg_test(struct nfs_pageio_descriptor *desc, + struct nfs_page *prev, struct nfs_page *req) +{ + struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); + + + if (mirror->pg_count > mirror->pg_bsize) { + /* should never happen */ + WARN_ON_ONCE(1); + return 0; + } + + /* + * Limit the request size so that we can still allocate a page array + * for it without upsetting the slab allocator. + */ + if (((mirror->pg_count + req->wb_bytes) >> PAGE_SHIFT) * + sizeof(struct page *) > PAGE_SIZE) + return 0; + + return min(mirror->pg_bsize - mirror->pg_count, (size_t)req->wb_bytes); +} +EXPORT_SYMBOL_GPL(nfs_generic_pg_test); + +struct nfs_pgio_header *nfs_pgio_header_alloc(const struct nfs_rw_ops *ops) +{ + struct nfs_pgio_header *hdr = ops->rw_alloc_header(); + + if (hdr) { + INIT_LIST_HEAD(&hdr->pages); + hdr->rw_ops = ops; + } + return hdr; +} +EXPORT_SYMBOL_GPL(nfs_pgio_header_alloc); + +/** + * nfs_pgio_data_destroy - make @hdr suitable for reuse + * + * Frees memory and releases refs from nfs_generic_pgio, so that it may + * be called again. + * + * @hdr: A header that has had nfs_generic_pgio called + */ +static void nfs_pgio_data_destroy(struct nfs_pgio_header *hdr) +{ + if (hdr->args.context) + put_nfs_open_context(hdr->args.context); + if (hdr->page_array.pagevec != hdr->page_array.page_array) + kfree(hdr->page_array.pagevec); +} + +/* + * nfs_pgio_header_free - Free a read or write header + * @hdr: The header to free + */ +void nfs_pgio_header_free(struct nfs_pgio_header *hdr) +{ + nfs_pgio_data_destroy(hdr); + hdr->rw_ops->rw_free_header(hdr); +} +EXPORT_SYMBOL_GPL(nfs_pgio_header_free); + +/** + * nfs_pgio_rpcsetup - Set up arguments for a pageio call + * @hdr: The pageio hdr + * @pgbase: base + * @count: Number of bytes to read + * @how: How to commit data (writes only) + * @cinfo: Commit information for the call (writes only) + */ +static void nfs_pgio_rpcsetup(struct nfs_pgio_header *hdr, unsigned int pgbase, + unsigned int count, int how, + struct nfs_commit_info *cinfo) +{ + struct nfs_page *req = hdr->req; + + /* Set up the RPC argument and reply structs + * NB: take care not to mess about with hdr->commit et al. */ + + hdr->args.fh = NFS_FH(hdr->inode); + hdr->args.offset = req_offset(req); + /* pnfs_set_layoutcommit needs this */ + hdr->mds_offset = hdr->args.offset; + hdr->args.pgbase = pgbase; + hdr->args.pages = hdr->page_array.pagevec; + hdr->args.count = count; + hdr->args.context = get_nfs_open_context(nfs_req_openctx(req)); + hdr->args.lock_context = req->wb_lock_context; + hdr->args.stable = NFS_UNSTABLE; + switch (how & (FLUSH_STABLE | FLUSH_COND_STABLE)) { + case 0: + break; + case FLUSH_COND_STABLE: + if (nfs_reqs_to_commit(cinfo)) + break; + fallthrough; + default: + hdr->args.stable = NFS_FILE_SYNC; + } + + hdr->res.fattr = &hdr->fattr; + hdr->res.count = 0; + hdr->res.eof = 0; + hdr->res.verf = &hdr->verf; + nfs_fattr_init(&hdr->fattr); +} + +/** + * nfs_pgio_prepare - Prepare pageio hdr to go over the wire + * @task: The current task + * @calldata: pageio header to prepare + */ +static void nfs_pgio_prepare(struct rpc_task *task, void *calldata) +{ + struct nfs_pgio_header *hdr = calldata; + int err; + err = NFS_PROTO(hdr->inode)->pgio_rpc_prepare(task, hdr); + if (err) + rpc_exit(task, err); +} + +int nfs_initiate_pgio(struct rpc_clnt *clnt, struct nfs_pgio_header *hdr, + const struct cred *cred, const struct nfs_rpc_ops *rpc_ops, + const struct rpc_call_ops *call_ops, int how, int flags, + struct nfsd_file *localio) +{ + struct rpc_task *task; + struct rpc_message msg = { + .rpc_argp = &hdr->args, + .rpc_resp = &hdr->res, + .rpc_cred = cred, + }; + struct rpc_task_setup task_setup_data = { + .rpc_client = clnt, + .task = &hdr->task, + .rpc_message = &msg, + .callback_ops = call_ops, + .callback_data = hdr, + .workqueue = nfsiod_workqueue, + .flags = RPC_TASK_ASYNC | flags, + }; + + if (nfs_server_capable(hdr->inode, NFS_CAP_MOVEABLE)) + task_setup_data.flags |= RPC_TASK_MOVEABLE; + + hdr->rw_ops->rw_initiate(hdr, &msg, rpc_ops, &task_setup_data, how); + + dprintk("NFS: initiated pgio call " + "(req %s/%llu, %u bytes @ offset %llu)\n", + hdr->inode->i_sb->s_id, + (unsigned long long)hdr->inode->i_ino, + hdr->args.count, + (unsigned long long)hdr->args.offset); + + if (localio) + return nfs_local_doio(NFS_SERVER(hdr->inode)->nfs_client, + localio, hdr, call_ops); + + task = rpc_run_task(&task_setup_data); + if (IS_ERR(task)) + return PTR_ERR(task); + rpc_put_task(task); + return 0; +} +EXPORT_SYMBOL_GPL(nfs_initiate_pgio); + +/** + * nfs_pgio_error - Clean up from a pageio error + * @hdr: pageio header + */ +static void nfs_pgio_error(struct nfs_pgio_header *hdr) +{ + set_bit(NFS_IOHDR_REDO, &hdr->flags); + hdr->completion_ops->completion(hdr); +} + +/** + * nfs_pgio_release - Release pageio data + * @calldata: The pageio header to release + */ +static void nfs_pgio_release(void *calldata) +{ + struct nfs_pgio_header *hdr = calldata; + hdr->completion_ops->completion(hdr); +} + +static void nfs_pageio_mirror_init(struct nfs_pgio_mirror *mirror, + unsigned int bsize) +{ + INIT_LIST_HEAD(&mirror->pg_list); + mirror->pg_bytes_written = 0; + mirror->pg_count = 0; + mirror->pg_bsize = bsize; + mirror->pg_base = 0; + mirror->pg_recoalesce = 0; +} + +/** + * nfs_pageio_init - initialise a page io descriptor + * @desc: pointer to descriptor + * @inode: pointer to inode + * @pg_ops: pointer to pageio operations + * @compl_ops: pointer to pageio completion operations + * @rw_ops: pointer to nfs read/write operations + * @bsize: io block size + * @io_flags: extra parameters for the io function + */ +void nfs_pageio_init(struct nfs_pageio_descriptor *desc, + struct inode *inode, + const struct nfs_pageio_ops *pg_ops, + const struct nfs_pgio_completion_ops *compl_ops, + const struct nfs_rw_ops *rw_ops, + size_t bsize, + int io_flags) +{ + desc->pg_moreio = 0; + desc->pg_inode = inode; + desc->pg_ops = pg_ops; + desc->pg_completion_ops = compl_ops; + desc->pg_rw_ops = rw_ops; + desc->pg_ioflags = io_flags; + desc->pg_error = 0; + desc->pg_lseg = NULL; + desc->pg_io_completion = NULL; + desc->pg_dreq = NULL; + nfs_netfs_reset_pageio_descriptor(desc); + desc->pg_bsize = bsize; + + desc->pg_mirror_count = 1; + desc->pg_mirror_idx = 0; + + desc->pg_mirrors_dynamic = NULL; + desc->pg_mirrors = desc->pg_mirrors_static; + nfs_pageio_mirror_init(&desc->pg_mirrors[0], bsize); + desc->pg_maxretrans = 0; +} + +/** + * nfs_pgio_result - Basic pageio error handling + * @task: The task that ran + * @calldata: Pageio header to check + */ +static void nfs_pgio_result(struct rpc_task *task, void *calldata) +{ + struct nfs_pgio_header *hdr = calldata; + struct inode *inode = hdr->inode; + + if (hdr->rw_ops->rw_done(task, hdr, inode) != 0) + return; + if (task->tk_status < 0) + nfs_set_pgio_error(hdr, task->tk_status, hdr->args.offset); + else + hdr->rw_ops->rw_result(task, hdr); +} + +/* + * Create an RPC task for the given read or write request and kick it. + * The page must have been locked by the caller. + * + * It may happen that the page we're passed is not marked dirty. + * This is the case if nfs_updatepage detects a conflicting request + * that has been written but not committed. + */ +int nfs_generic_pgio(struct nfs_pageio_descriptor *desc, + struct nfs_pgio_header *hdr) +{ + struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); + + struct nfs_page *req; + struct page **pages, + *last_page; + struct list_head *head = &mirror->pg_list; + struct nfs_commit_info cinfo; + struct nfs_page_array *pg_array = &hdr->page_array; + unsigned int pagecount, pageused; + unsigned int pg_base = offset_in_page(mirror->pg_base); + gfp_t gfp_flags = nfs_io_gfp_mask(); + + pagecount = nfs_page_array_len(pg_base, mirror->pg_count); + pg_array->npages = pagecount; + + if (pagecount <= ARRAY_SIZE(pg_array->page_array)) + pg_array->pagevec = pg_array->page_array; + else { + pg_array->pagevec = kzalloc_objs(struct page *, pagecount, + gfp_flags); + if (!pg_array->pagevec) { + pg_array->npages = 0; + nfs_pgio_error(hdr); + desc->pg_error = -ENOMEM; + return desc->pg_error; + } + } + + nfs_init_cinfo(&cinfo, desc->pg_inode, desc->pg_dreq); + pages = hdr->page_array.pagevec; + last_page = NULL; + pageused = 0; + while (!list_empty(head)) { + struct nfs_page_iter_page i; + struct page *page; + + req = nfs_list_entry(head->next); + nfs_list_move_request(req, &hdr->pages); + + if (req->wb_pgbase == 0) + last_page = NULL; + + nfs_page_iter_page_init(&i, req); + while ((page = nfs_page_iter_page_get(&i)) != NULL) { + if (last_page != page) { + pageused++; + if (pageused > pagecount) + goto full; + *pages++ = last_page = page; + } + } + } +full: + if (WARN_ON_ONCE(pageused != pagecount)) { + nfs_pgio_error(hdr); + desc->pg_error = -EINVAL; + return desc->pg_error; + } + + if ((desc->pg_ioflags & FLUSH_COND_STABLE) && + (desc->pg_moreio || nfs_reqs_to_commit(&cinfo))) + desc->pg_ioflags &= ~FLUSH_COND_STABLE; + + /* Set up the argument struct */ + nfs_pgio_rpcsetup(hdr, pg_base, mirror->pg_count, desc->pg_ioflags, + &cinfo); + desc->pg_rpc_callops = &nfs_pgio_common_ops; + return 0; +} +EXPORT_SYMBOL_GPL(nfs_generic_pgio); + +static int nfs_generic_pg_pgios(struct nfs_pageio_descriptor *desc) +{ + struct nfs_pgio_header *hdr; + int ret; + unsigned short task_flags = 0; + + hdr = nfs_pgio_header_alloc(desc->pg_rw_ops); + if (!hdr) { + desc->pg_error = -ENOMEM; + return desc->pg_error; + } + nfs_pgheader_init(desc, hdr, nfs_pgio_header_free); + ret = nfs_generic_pgio(desc, hdr); + if (ret == 0) { + struct nfs_client *clp = NFS_SERVER(hdr->inode)->nfs_client; + + struct nfsd_file *localio = + nfs_local_open_fh(clp, hdr->cred, hdr->args.fh, + &hdr->args.context->nfl, + hdr->args.context->mode); + + if (NFS_SERVER(hdr->inode)->nfs_client->cl_minorversion) + task_flags = RPC_TASK_MOVEABLE; + ret = nfs_initiate_pgio(NFS_CLIENT(hdr->inode), + hdr, + hdr->cred, + NFS_PROTO(hdr->inode), + desc->pg_rpc_callops, + desc->pg_ioflags, + RPC_TASK_CRED_NOREF | task_flags, + localio); + } + return ret; +} + +static struct nfs_pgio_mirror * +nfs_pageio_alloc_mirrors(struct nfs_pageio_descriptor *desc, + unsigned int mirror_count) +{ + struct nfs_pgio_mirror *ret; + unsigned int i; + + kfree(desc->pg_mirrors_dynamic); + desc->pg_mirrors_dynamic = NULL; + if (mirror_count == 1) + return desc->pg_mirrors_static; + ret = kmalloc_objs(*ret, mirror_count, nfs_io_gfp_mask()); + if (ret != NULL) { + for (i = 0; i < mirror_count; i++) + nfs_pageio_mirror_init(&ret[i], desc->pg_bsize); + desc->pg_mirrors_dynamic = ret; + } + return ret; +} + +/* + * nfs_pageio_setup_mirroring - determine if mirroring is to be used + * by calling the pg_get_mirror_count op + */ +static void nfs_pageio_setup_mirroring(struct nfs_pageio_descriptor *pgio, + struct nfs_page *req) +{ + unsigned int mirror_count = 1; + + if (pgio->pg_ops->pg_get_mirror_count) + mirror_count = pgio->pg_ops->pg_get_mirror_count(pgio, req); + if (mirror_count == pgio->pg_mirror_count || pgio->pg_error < 0) + return; + + if (!mirror_count || mirror_count > NFS_PAGEIO_DESCRIPTOR_MIRROR_MAX) { + pgio->pg_error = -EINVAL; + return; + } + + pgio->pg_mirrors = nfs_pageio_alloc_mirrors(pgio, mirror_count); + if (pgio->pg_mirrors == NULL) { + pgio->pg_error = -ENOMEM; + pgio->pg_mirrors = pgio->pg_mirrors_static; + mirror_count = 1; + } + pgio->pg_mirror_count = mirror_count; +} + +static void nfs_pageio_cleanup_mirroring(struct nfs_pageio_descriptor *pgio) +{ + pgio->pg_mirror_count = 1; + pgio->pg_mirror_idx = 0; + pgio->pg_mirrors = pgio->pg_mirrors_static; + kfree(pgio->pg_mirrors_dynamic); + pgio->pg_mirrors_dynamic = NULL; +} + +static bool nfs_match_lock_context(const struct nfs_lock_context *l1, + const struct nfs_lock_context *l2) +{ + return l1->lockowner == l2->lockowner; +} + +static bool nfs_page_is_contiguous(const struct nfs_page *prev, + const struct nfs_page *req) +{ + size_t prev_end = prev->wb_pgbase + prev->wb_bytes; + + if (req_offset(req) != req_offset(prev) + prev->wb_bytes) + return false; + if (req->wb_pgbase == 0) + return prev_end == nfs_page_max_length(prev); + if (req->wb_pgbase == prev_end) { + struct folio *folio = nfs_page_to_folio(req); + if (folio) + return folio == nfs_page_to_folio(prev); + return req->wb_page == prev->wb_page; + } + return false; +} + +/** + * nfs_coalesce_size - test two requests for compatibility + * @prev: pointer to nfs_page + * @req: pointer to nfs_page + * @pgio: pointer to nfs_pagio_descriptor + * + * The nfs_page structures 'prev' and 'req' are compared to ensure that the + * page data area they describe is contiguous, and that their RPC + * credentials, NFSv4 open state, and lockowners are the same. + * + * Returns size of the request that can be coalesced + */ +static unsigned int nfs_coalesce_size(struct nfs_page *prev, + struct nfs_page *req, + struct nfs_pageio_descriptor *pgio) +{ + struct file_lock_context *flctx; + + if (prev) { + if (!nfs_match_open_context(nfs_req_openctx(req), nfs_req_openctx(prev))) + return 0; + flctx = locks_inode_context(d_inode(nfs_req_openctx(req)->dentry)); + if (flctx != NULL && + !(list_empty_careful(&flctx->flc_posix) && + list_empty_careful(&flctx->flc_flock)) && + !nfs_match_lock_context(req->wb_lock_context, + prev->wb_lock_context)) + return 0; + if (!nfs_page_is_contiguous(prev, req)) + return 0; + } + return pgio->pg_ops->pg_test(pgio, prev, req); +} + +/** + * nfs_pageio_do_add_request - Attempt to coalesce a request into a page list. + * @desc: destination io descriptor + * @req: request + * + * If the request 'req' was successfully coalesced into the existing list + * of pages 'desc', it returns the size of req. + */ +static unsigned int +nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc, + struct nfs_page *req) +{ + struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); + struct nfs_page *prev = NULL; + unsigned int size; + + if (list_empty(&mirror->pg_list)) { + if (desc->pg_ops->pg_init) + desc->pg_ops->pg_init(desc, req); + if (desc->pg_error < 0) + return 0; + mirror->pg_base = req->wb_pgbase; + mirror->pg_count = 0; + mirror->pg_recoalesce = 0; + } else + prev = nfs_list_entry(mirror->pg_list.prev); + + if (desc->pg_maxretrans && req->wb_nio > desc->pg_maxretrans) { + if (NFS_SERVER(desc->pg_inode)->flags & NFS_MOUNT_SOFTERR) + desc->pg_error = -ETIMEDOUT; + else + desc->pg_error = -EIO; + return 0; + } + + size = nfs_coalesce_size(prev, req, desc); + if (size < req->wb_bytes) + return size; + nfs_list_move_request(req, &mirror->pg_list); + mirror->pg_count += req->wb_bytes; + return req->wb_bytes; +} + +/* + * Helper for nfs_pageio_add_request and nfs_pageio_complete + */ +static void nfs_pageio_doio(struct nfs_pageio_descriptor *desc) +{ + struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); + + if (!list_empty(&mirror->pg_list)) { + int error = desc->pg_ops->pg_doio(desc); + if (error < 0) + desc->pg_error = error; + if (list_empty(&mirror->pg_list)) + mirror->pg_bytes_written += mirror->pg_count; + } +} + +static void +nfs_pageio_cleanup_request(struct nfs_pageio_descriptor *desc, + struct nfs_page *req) +{ + LIST_HEAD(head); + + nfs_list_move_request(req, &head); + desc->pg_completion_ops->error_cleanup(&head, desc->pg_error); +} + +/** + * __nfs_pageio_add_request - Attempt to coalesce a request into a page list. + * @desc: destination io descriptor + * @req: request + * + * This may split a request into subrequests which are all part of the + * same page group. If so, it will submit @req as the last one, to ensure + * the pointer to @req is still valid in case of failure. + * + * Returns true if the request 'req' was successfully coalesced into the + * existing list of pages 'desc'. + */ +static int __nfs_pageio_add_request(struct nfs_pageio_descriptor *desc, + struct nfs_page *req) +{ + struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); + struct nfs_page *subreq; + unsigned int size, subreq_size; + + nfs_page_group_lock(req); + + if (test_bit(NFS_CONTEXT_WRITE_SYNC, + &req->wb_lock_context->open_context->flags)) + desc->pg_ioflags |= FLUSH_STABLE; + subreq = req; + subreq_size = subreq->wb_bytes; + for(;;) { + size = nfs_pageio_do_add_request(desc, subreq); + if (size == subreq_size) { + /* We successfully submitted a request */ + if (subreq == req) + break; + req->wb_pgbase += size; + req->wb_bytes -= size; + req->wb_offset += size; + subreq_size = req->wb_bytes; + subreq = req; + continue; + } + if (WARN_ON_ONCE(subreq != req)) { + nfs_page_group_unlock(req); + nfs_pageio_cleanup_request(desc, subreq); + subreq = req; + subreq_size = req->wb_bytes; + nfs_page_group_lock(req); + } + if (!size) { + /* Can't coalesce any more, so do I/O */ + nfs_page_group_unlock(req); + desc->pg_moreio = 1; + nfs_pageio_doio(desc); + if (desc->pg_error < 0 || mirror->pg_recoalesce) + return 0; + /* retry add_request for this subreq */ + nfs_page_group_lock(req); + continue; + } + subreq = nfs_create_subreq(req, req->wb_pgbase, + req->wb_offset, size); + if (IS_ERR(subreq)) + goto err_ptr; + subreq_size = size; + } + + nfs_page_group_unlock(req); + return 1; +err_ptr: + desc->pg_error = PTR_ERR(subreq); + nfs_page_group_unlock(req); + return 0; +} + +static int nfs_do_recoalesce(struct nfs_pageio_descriptor *desc) +{ + struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); + LIST_HEAD(head); + + do { + list_splice_init(&mirror->pg_list, &head); + mirror->pg_recoalesce = 0; + + while (!list_empty(&head)) { + struct nfs_page *req; + + req = list_first_entry(&head, struct nfs_page, wb_list); + if (__nfs_pageio_add_request(desc, req)) + continue; + if (desc->pg_error < 0) { + list_splice_tail(&head, &mirror->pg_list); + mirror->pg_recoalesce = 1; + return 0; + } + break; + } + } while (mirror->pg_recoalesce); + return 1; +} + +static int nfs_pageio_add_request_mirror(struct nfs_pageio_descriptor *desc, + struct nfs_page *req) +{ + int ret; + + do { + ret = __nfs_pageio_add_request(desc, req); + if (ret) + break; + if (desc->pg_error < 0) + break; + ret = nfs_do_recoalesce(desc); + } while (ret); + + return ret; +} + +static void nfs_pageio_error_cleanup(struct nfs_pageio_descriptor *desc) +{ + u32 midx; + struct nfs_pgio_mirror *mirror; + + if (!desc->pg_error) + return; + + for (midx = 0; midx < desc->pg_mirror_count; midx++) { + mirror = nfs_pgio_get_mirror(desc, midx); + desc->pg_completion_ops->error_cleanup(&mirror->pg_list, + desc->pg_error); + } +} + +int nfs_pageio_add_request(struct nfs_pageio_descriptor *desc, + struct nfs_page *req) +{ + u32 midx; + unsigned int pgbase, offset, bytes; + struct nfs_page *dupreq; + + pgbase = req->wb_pgbase; + offset = req->wb_offset; + bytes = req->wb_bytes; + + nfs_pageio_setup_mirroring(desc, req); + if (desc->pg_error < 0) + goto out_failed; + + /* Create the mirror instances first, and fire them off */ + for (midx = 1; midx < desc->pg_mirror_count; midx++) { + nfs_page_group_lock(req); + + dupreq = nfs_create_subreq(req, + pgbase, offset, bytes); + + nfs_page_group_unlock(req); + if (IS_ERR(dupreq)) { + desc->pg_error = PTR_ERR(dupreq); + goto out_failed; + } + + nfs_pgio_set_current_mirror(desc, midx); + if (!nfs_pageio_add_request_mirror(desc, dupreq)) + goto out_cleanup_subreq; + } + + nfs_pgio_set_current_mirror(desc, 0); + if (!nfs_pageio_add_request_mirror(desc, req)) + goto out_failed; + + return 1; + +out_cleanup_subreq: + nfs_pageio_cleanup_request(desc, dupreq); +out_failed: + nfs_pageio_error_cleanup(desc); + return 0; +} + +/* + * nfs_pageio_complete_mirror - Complete I/O on the current mirror of an + * nfs_pageio_descriptor + * @desc: pointer to io descriptor + * @mirror_idx: pointer to mirror index + */ +static void nfs_pageio_complete_mirror(struct nfs_pageio_descriptor *desc, + u32 mirror_idx) +{ + struct nfs_pgio_mirror *mirror; + u32 restore_idx; + + restore_idx = nfs_pgio_set_current_mirror(desc, mirror_idx); + mirror = nfs_pgio_current_mirror(desc); + + for (;;) { + nfs_pageio_doio(desc); + if (desc->pg_error < 0 || !mirror->pg_recoalesce) + break; + if (!nfs_do_recoalesce(desc)) + break; + } + nfs_pgio_set_current_mirror(desc, restore_idx); +} + +/* + * nfs_pageio_resend - Transfer requests to new descriptor and resend + * @hdr - the pgio header to move request from + * @desc - the pageio descriptor to add requests to + * + * Try to move each request (nfs_page) from @hdr to @desc then attempt + * to send them. + * + * Returns 0 on success and < 0 on error. + */ +int nfs_pageio_resend(struct nfs_pageio_descriptor *desc, + struct nfs_pgio_header *hdr) +{ + LIST_HEAD(pages); + + desc->pg_io_completion = hdr->io_completion; + desc->pg_dreq = hdr->dreq; + nfs_netfs_set_pageio_descriptor(desc, hdr); + list_splice_init(&hdr->pages, &pages); + while (!list_empty(&pages)) { + struct nfs_page *req = nfs_list_entry(pages.next); + + if (!nfs_pageio_add_request(desc, req)) + break; + } + nfs_pageio_complete(desc); + if (!list_empty(&pages)) { + int err = desc->pg_error < 0 ? desc->pg_error : -EIO; + hdr->completion_ops->error_cleanup(&pages, err); + nfs_set_pgio_error(hdr, err, hdr->io_start); + return err; + } + return 0; +} +EXPORT_SYMBOL_GPL(nfs_pageio_resend); + +/** + * nfs_pageio_complete - Complete I/O then cleanup an nfs_pageio_descriptor + * @desc: pointer to io descriptor + */ +void nfs_pageio_complete(struct nfs_pageio_descriptor *desc) +{ + u32 midx; + + for (midx = 0; midx < desc->pg_mirror_count; midx++) + nfs_pageio_complete_mirror(desc, midx); + + if (desc->pg_error < 0) + nfs_pageio_error_cleanup(desc); + if (desc->pg_ops->pg_cleanup) + desc->pg_ops->pg_cleanup(desc); + nfs_pageio_cleanup_mirroring(desc); +} + +/** + * nfs_pageio_cond_complete - Conditional I/O completion + * @desc: pointer to io descriptor + * @index: page index + * + * It is important to ensure that processes don't try to take locks + * on non-contiguous ranges of pages as that might deadlock. This + * function should be called before attempting to wait on a locked + * nfs_page. It will complete the I/O if the page index 'index' + * is not contiguous with the existing list of pages in 'desc'. + */ +void nfs_pageio_cond_complete(struct nfs_pageio_descriptor *desc, pgoff_t index) +{ + struct nfs_pgio_mirror *mirror; + struct nfs_page *prev; + struct folio *folio; + u32 midx; + + for (midx = 0; midx < desc->pg_mirror_count; midx++) { + mirror = nfs_pgio_get_mirror(desc, midx); + if (!list_empty(&mirror->pg_list)) { + prev = nfs_list_entry(mirror->pg_list.prev); + folio = nfs_page_to_folio(prev); + if (folio) { + if (index == folio_next_index(folio)) + continue; + } else if (index == prev->wb_index + 1) + continue; + /* + * We will submit more requests after these. Indicate + * this to the underlying layers. + */ + desc->pg_moreio = 1; + nfs_pageio_complete(desc); + break; + } + } +} + +/* + * nfs_pageio_stop_mirroring - stop using mirroring (set mirror count to 1) + */ +void nfs_pageio_stop_mirroring(struct nfs_pageio_descriptor *pgio) +{ + nfs_pageio_complete(pgio); +} + +int __init nfs_init_nfspagecache(void) +{ + nfs_page_cachep = kmem_cache_create("nfs_page", + sizeof(struct nfs_page), + 0, SLAB_HWCACHE_ALIGN, + NULL); + if (nfs_page_cachep == NULL) + return -ENOMEM; + + return 0; +} + +void nfs_destroy_nfspagecache(void) +{ + kmem_cache_destroy(nfs_page_cachep); +} + +static const struct rpc_call_ops nfs_pgio_common_ops = { + .rpc_call_prepare = nfs_pgio_prepare, + .rpc_call_done = nfs_pgio_result, + .rpc_release = nfs_pgio_release, +}; + +const struct nfs_pageio_ops nfs_pgio_rw_ops = { + .pg_test = nfs_generic_pg_test, + .pg_doio = nfs_generic_pg_pgios, +}; diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c new file mode 100644 index 000000000..7715e2bd5 --- /dev/null +++ b/fs/nfs/pnfs.c @@ -0,0 +1,3535 @@ +/* + * pNFS functions to call and manage layout drivers. + * + * Copyright (c) 2002 [year of first publication] + * The Regents of the University of Michigan + * All Rights Reserved + * + * Dean Hildebrand + * + * Permission is granted to use, copy, create derivative works, and + * redistribute this software and such derivative works for any purpose, + * so long as the name of the University of Michigan is not used in + * any advertising or publicity pertaining to the use or distribution + * of this software without specific, written prior authorization. If + * the above copyright notice or any other identification of the + * University of Michigan is included in any copy of any portion of + * this software, then the disclaimer below must also be included. + * + * This software is provided as is, without representation or warranty + * of any kind either express or implied, including without limitation + * the implied warranties of merchantability, fitness for a particular + * purpose, or noninfringement. The Regents of the University of + * Michigan shall not be liable for any damages, including special, + * indirect, incidental, or consequential damages, with respect to any + * claim arising out of or in connection with the use of the software, + * even if it has been or is hereafter advised of the possibility of + * such damages. + */ + +#include +#include +#include +#include +#include "internal.h" +#include "pnfs.h" +#include "iostat.h" +#include "nfs4trace.h" +#include "delegation.h" +#include "nfs42.h" +#include "nfs4_fs.h" + +#define NFSDBG_FACILITY NFSDBG_PNFS +#define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ) + +/* Locking: + * + * pnfs_spinlock: + * protects pnfs_modules_tbl. + */ +static DEFINE_SPINLOCK(pnfs_spinlock); + +/* + * pnfs_modules_tbl holds all pnfs modules + */ +static LIST_HEAD(pnfs_modules_tbl); + +static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo); +static void pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo, + struct list_head *free_me, + const struct pnfs_layout_range *range, + u32 seq); +static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg, + struct list_head *tmp_list); +static int pnfs_layout_return_on_reboot(struct pnfs_layout_hdr *lo); + +/* Return the registered pnfs layout driver module matching given id */ +static struct pnfs_layoutdriver_type * +find_pnfs_driver_locked(u32 id) +{ + struct pnfs_layoutdriver_type *local; + + list_for_each_entry(local, &pnfs_modules_tbl, pnfs_tblid) + if (local->id == id) + goto out; + local = NULL; +out: + dprintk("%s: Searching for id %u, found %p\n", __func__, id, local); + return local; +} + +static struct pnfs_layoutdriver_type * +find_pnfs_driver(u32 id) +{ + struct pnfs_layoutdriver_type *local; + + spin_lock(&pnfs_spinlock); + local = find_pnfs_driver_locked(id); + if (local != NULL && !try_module_get(local->owner)) { + dprintk("%s: Could not grab reference on module\n", __func__); + local = NULL; + } + spin_unlock(&pnfs_spinlock); + return local; +} + +const struct pnfs_layoutdriver_type *pnfs_find_layoutdriver(u32 id) +{ + return find_pnfs_driver(id); +} + +void pnfs_put_layoutdriver(const struct pnfs_layoutdriver_type *ld) +{ + if (ld) + module_put(ld->owner); +} + +void +unset_pnfs_layoutdriver(struct nfs_server *nfss) +{ + if (nfss->pnfs_curr_ld) { + if (nfss->pnfs_curr_ld->clear_layoutdriver) + nfss->pnfs_curr_ld->clear_layoutdriver(nfss); + /* Decrement the MDS count. Purge the deviceid cache if zero */ + if (atomic_dec_and_test(&nfss->nfs_client->cl_mds_count)) + nfs4_deviceid_purge_client(nfss->nfs_client); + module_put(nfss->pnfs_curr_ld->owner); + } + nfss->pnfs_curr_ld = NULL; +} + +/* + * When the server sends a list of layout types, we choose one in the order + * given in the list below. + * + * FIXME: should this list be configurable in some fashion? module param? + * mount option? something else? + */ +static const u32 ld_prefs[] = { + LAYOUT_SCSI, + LAYOUT_BLOCK_VOLUME, + LAYOUT_OSD2_OBJECTS, + LAYOUT_FLEX_FILES, + LAYOUT_NFSV4_1_FILES, + 0 +}; + +static int +ld_cmp(const void *e1, const void *e2) +{ + u32 ld1 = *((u32 *)e1); + u32 ld2 = *((u32 *)e2); + int i; + + for (i = 0; ld_prefs[i] != 0; i++) { + if (ld1 == ld_prefs[i]) + return -1; + + if (ld2 == ld_prefs[i]) + return 1; + } + return 0; +} + +/* + * Try to set the server's pnfs module to the pnfs layout type specified by id. + * Currently only one pNFS layout driver per filesystem is supported. + * + * @ids array of layout types supported by MDS. + */ +void +set_pnfs_layoutdriver(struct nfs_server *server, const struct nfs_fh *mntfh, + struct nfs_fsinfo *fsinfo) +{ + struct pnfs_layoutdriver_type *ld_type = NULL; + u32 id; + int i; + + if (fsinfo->nlayouttypes == 0) + goto out_no_driver; + if (!(server->nfs_client->cl_exchange_flags & + (EXCHGID4_FLAG_USE_NON_PNFS | EXCHGID4_FLAG_USE_PNFS_MDS))) { + printk(KERN_ERR "NFS: %s: cl_exchange_flags 0x%x\n", + __func__, server->nfs_client->cl_exchange_flags); + goto out_no_driver; + } + + sort(fsinfo->layouttype, fsinfo->nlayouttypes, + sizeof(*fsinfo->layouttype), ld_cmp, NULL); + + for (i = 0; i < fsinfo->nlayouttypes; i++) { + id = fsinfo->layouttype[i]; + ld_type = find_pnfs_driver(id); + if (!ld_type) { + request_module("%s-%u", LAYOUT_NFSV4_1_MODULE_PREFIX, + id); + ld_type = find_pnfs_driver(id); + } + if (ld_type) + break; + } + + if (!ld_type) { + dprintk("%s: No pNFS module found!\n", __func__); + goto out_no_driver; + } + + server->pnfs_curr_ld = ld_type; + if (ld_type->set_layoutdriver + && ld_type->set_layoutdriver(server, mntfh)) { + printk(KERN_ERR "NFS: %s: Error initializing pNFS layout " + "driver %u.\n", __func__, id); + module_put(ld_type->owner); + goto out_no_driver; + } + /* Bump the MDS count */ + atomic_inc(&server->nfs_client->cl_mds_count); + + dprintk("%s: pNFS module for %u set\n", __func__, id); + return; + +out_no_driver: + dprintk("%s: Using NFSv4 I/O\n", __func__); + server->pnfs_curr_ld = NULL; +} + +int +pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *ld_type) +{ + int status = -EINVAL; + struct pnfs_layoutdriver_type *tmp; + + if (ld_type->id == 0) { + printk(KERN_ERR "NFS: %s id 0 is reserved\n", __func__); + return status; + } + if (!ld_type->alloc_lseg || !ld_type->free_lseg) { + printk(KERN_ERR "NFS: %s Layout driver must provide " + "alloc_lseg and free_lseg.\n", __func__); + return status; + } + + spin_lock(&pnfs_spinlock); + tmp = find_pnfs_driver_locked(ld_type->id); + if (!tmp) { + list_add(&ld_type->pnfs_tblid, &pnfs_modules_tbl); + status = 0; + dprintk("%s Registering id:%u name:%s\n", __func__, ld_type->id, + ld_type->name); + } else { + printk(KERN_ERR "NFS: %s Module with id %d already loaded!\n", + __func__, ld_type->id); + } + spin_unlock(&pnfs_spinlock); + + return status; +} +EXPORT_SYMBOL_GPL(pnfs_register_layoutdriver); + +void +pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *ld_type) +{ + dprintk("%s Deregistering id:%u\n", __func__, ld_type->id); + spin_lock(&pnfs_spinlock); + list_del(&ld_type->pnfs_tblid); + spin_unlock(&pnfs_spinlock); +} +EXPORT_SYMBOL_GPL(pnfs_unregister_layoutdriver); + +/* + * pNFS client layout cache + */ + +/* Need to hold i_lock if caller does not already hold reference */ +void +pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo) +{ + refcount_inc(&lo->plh_refcount); +} + +static struct pnfs_layout_hdr * +pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags) +{ + struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; + return ld->alloc_layout_hdr(ino, gfp_flags); +} + +static void +pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo) +{ + struct nfs_server *server = NFS_SERVER(lo->plh_inode); + struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld; + + if (test_and_clear_bit(NFS_LAYOUT_HASHED, &lo->plh_flags)) { + struct nfs_client *clp = server->nfs_client; + + spin_lock(&clp->cl_lock); + list_del_rcu(&lo->plh_layouts); + spin_unlock(&clp->cl_lock); + } + put_cred(lo->plh_lc_cred); + return ld->free_layout_hdr(lo); +} + +static void +pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo) +{ + struct nfs_inode *nfsi = NFS_I(lo->plh_inode); + dprintk("%s: freeing layout cache %p\n", __func__, lo); + nfsi->layout = NULL; + /* Reset MDS Threshold I/O counters */ + nfsi->write_io = 0; + nfsi->read_io = 0; +} + +void +pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo) +{ + struct inode *inode; + + if (!lo) + return; + inode = lo->plh_inode; + pnfs_layoutreturn_before_put_layout_hdr(lo); + + if (refcount_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) { + if (!list_empty(&lo->plh_segs)) + WARN_ONCE(1, "NFS: BUG unfreed layout segments.\n"); + pnfs_detach_layout_hdr(lo); + /* Notify pnfs_destroy_layout_final() that we're done */ + if (inode_state_read(inode) & (I_FREEING | I_CLEAR)) + wake_up_var_locked(lo, &inode->i_lock); + spin_unlock(&inode->i_lock); + pnfs_free_layout_hdr(lo); + } +} + +static struct inode * +pnfs_grab_inode_layout_hdr(struct pnfs_layout_hdr *lo) +{ + struct inode *inode = igrab(lo->plh_inode); + if (inode) + return inode; + set_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags); + return NULL; +} + +/* + * Compare 2 layout stateid sequence ids, to see which is newer, + * taking into account wraparound issues. + */ +static bool pnfs_seqid_is_newer(u32 s1, u32 s2) +{ + return (s32)(s1 - s2) > 0; +} + +static void pnfs_barrier_update(struct pnfs_layout_hdr *lo, u32 newseq) +{ + if (pnfs_seqid_is_newer(newseq, lo->plh_barrier) || !lo->plh_barrier) + lo->plh_barrier = newseq; +} + +static void +pnfs_set_plh_return_info(struct pnfs_layout_hdr *lo, enum pnfs_iomode iomode, + u32 seq) +{ + if (lo->plh_return_iomode != 0 && lo->plh_return_iomode != iomode) + iomode = IOMODE_ANY; + lo->plh_return_iomode = iomode; + set_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags); + /* + * We must set lo->plh_return_seq to avoid livelocks with + * pnfs_layout_need_return() + */ + if (seq == 0) + seq = be32_to_cpu(lo->plh_stateid.seqid); + if (!lo->plh_return_seq || pnfs_seqid_is_newer(seq, lo->plh_return_seq)) + lo->plh_return_seq = seq; + pnfs_barrier_update(lo, seq); +} + +static void +pnfs_clear_layoutreturn_info(struct pnfs_layout_hdr *lo) +{ + struct pnfs_layout_segment *lseg; + lo->plh_return_iomode = 0; + lo->plh_return_seq = 0; + clear_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags); + list_for_each_entry(lseg, &lo->plh_segs, pls_list) { + if (!test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) + continue; + pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); + } +} + +static void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo) +{ + clear_bit_unlock(NFS_LAYOUT_RETURN, &lo->plh_flags); + clear_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags); + smp_mb__after_atomic(); + wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN); + rpc_wake_up(&NFS_SERVER(lo->plh_inode)->roc_rpcwaitq); +} + +static void +pnfs_clear_lseg_state(struct pnfs_layout_segment *lseg, + struct list_head *free_me) +{ + clear_bit(NFS_LSEG_ROC, &lseg->pls_flags); + clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); + if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) + pnfs_lseg_dec_and_remove_zero(lseg, free_me); + if (test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) + pnfs_lseg_dec_and_remove_zero(lseg, free_me); +} + +/* + * Update the seqid of a layout stateid after receiving + * NFS4ERR_OLD_STATEID + */ +bool nfs4_layout_refresh_old_stateid(nfs4_stateid *dst, + struct pnfs_layout_range *dst_range, + struct inode *inode) +{ + struct pnfs_layout_hdr *lo; + struct pnfs_layout_range range = { + .iomode = IOMODE_ANY, + .offset = 0, + .length = NFS4_MAX_UINT64, + }; + bool ret = false; + LIST_HEAD(head); + int err; + + spin_lock(&inode->i_lock); + lo = NFS_I(inode)->layout; + if (lo && pnfs_layout_is_valid(lo) && + nfs4_stateid_match_other(dst, &lo->plh_stateid)) { + /* Is our call using the most recent seqid? If so, bump it */ + if (!nfs4_stateid_is_newer(&lo->plh_stateid, dst)) { + nfs4_stateid_seqid_inc(dst); + ret = true; + goto out; + } + /* Try to update the seqid to the most recent */ + err = pnfs_mark_matching_lsegs_return(lo, &head, &range, 0); + if (err != -EBUSY) { + dst->seqid = lo->plh_stateid.seqid; + *dst_range = range; + ret = true; + } + } +out: + spin_unlock(&inode->i_lock); + pnfs_free_lseg_list(&head); + return ret; +} + +/* + * Mark a pnfs_layout_hdr and all associated layout segments as invalid + * + * In order to continue using the pnfs_layout_hdr, a full recovery + * is required. + * Note that caller must hold inode->i_lock. + */ +int +pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo, + struct list_head *lseg_list) +{ + struct pnfs_layout_range range = { + .iomode = IOMODE_ANY, + .offset = 0, + .length = NFS4_MAX_UINT64, + }; + struct pnfs_layout_segment *lseg, *next; + + if (test_and_set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) + return !list_empty(&lo->plh_segs); + clear_bit(NFS_INO_LAYOUTCOMMIT, &NFS_I(lo->plh_inode)->flags); + list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) + pnfs_clear_lseg_state(lseg, lseg_list); + pnfs_clear_layoutreturn_info(lo); + pnfs_free_returned_lsegs(lo, lseg_list, &range, 0); + set_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags); + if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags) && + !test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) + pnfs_clear_layoutreturn_waitbit(lo); + return !list_empty(&lo->plh_segs); +} + +static int pnfs_mark_layout_stateid_return(struct pnfs_layout_hdr *lo, + struct list_head *lseg_list, + enum pnfs_iomode iomode, u32 seq) +{ + struct pnfs_layout_range range = { + .iomode = iomode, + .length = NFS4_MAX_UINT64, + }; + + return pnfs_mark_matching_lsegs_return(lo, lseg_list, &range, seq); +} + +static int +pnfs_iomode_to_fail_bit(u32 iomode) +{ + return iomode == IOMODE_RW ? + NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED; +} + +static void +pnfs_layout_set_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit) +{ + lo->plh_retry_timestamp = jiffies; + if (!test_and_set_bit(fail_bit, &lo->plh_flags)) + refcount_inc(&lo->plh_refcount); +} + +static void +pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit) +{ + if (test_and_clear_bit(fail_bit, &lo->plh_flags)) + refcount_dec(&lo->plh_refcount); +} + +static void +pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode) +{ + struct inode *inode = lo->plh_inode; + struct pnfs_layout_range range = { + .iomode = iomode, + .offset = 0, + .length = NFS4_MAX_UINT64, + }; + LIST_HEAD(head); + + spin_lock(&inode->i_lock); + pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); + pnfs_mark_matching_lsegs_return(lo, &head, &range, 0); + spin_unlock(&inode->i_lock); + pnfs_free_lseg_list(&head); + dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__, + iomode == IOMODE_RW ? "RW" : "READ"); +} + +static bool +pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode) +{ + unsigned long start, end; + int fail_bit = pnfs_iomode_to_fail_bit(iomode); + + if (test_bit(fail_bit, &lo->plh_flags) == 0) + return false; + end = jiffies; + start = end - PNFS_LAYOUTGET_RETRY_TIMEOUT; + if (!time_in_range(lo->plh_retry_timestamp, start, end)) { + /* It is time to retry the failed layoutgets */ + pnfs_layout_clear_fail_bit(lo, fail_bit); + return false; + } + return true; +} + +static void +pnfs_init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg, + const struct pnfs_layout_range *range, + const nfs4_stateid *stateid) +{ + INIT_LIST_HEAD(&lseg->pls_list); + INIT_LIST_HEAD(&lseg->pls_lc_list); + INIT_LIST_HEAD(&lseg->pls_commits); + refcount_set(&lseg->pls_refcount, 1); + set_bit(NFS_LSEG_VALID, &lseg->pls_flags); + lseg->pls_layout = lo; + lseg->pls_range = *range; + lseg->pls_seq = be32_to_cpu(stateid->seqid); +} + +static void pnfs_free_lseg(struct pnfs_layout_segment *lseg) +{ + if (lseg != NULL) { + struct inode *inode = lseg->pls_layout->plh_inode; + NFS_SERVER(inode)->pnfs_curr_ld->free_lseg(lseg); + } +} + +static void +pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo, + struct pnfs_layout_segment *lseg) +{ + WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags)); + list_del_init(&lseg->pls_list); + /* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */ + refcount_dec(&lo->plh_refcount); + if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) + return; + if (list_empty(&lo->plh_segs) && + !test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) && + !test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { + if (atomic_read(&lo->plh_outstanding) == 0) + set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); + clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); + } +} + +static bool +pnfs_cache_lseg_for_layoutreturn(struct pnfs_layout_hdr *lo, + struct pnfs_layout_segment *lseg) +{ + if (test_and_clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) && + pnfs_layout_is_valid(lo)) { + pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); + list_move_tail(&lseg->pls_list, &lo->plh_return_segs); + return true; + } + return false; +} + +void +pnfs_put_lseg(struct pnfs_layout_segment *lseg) +{ + struct pnfs_layout_hdr *lo; + struct inode *inode; + + if (!lseg) + return; + + dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg, + refcount_read(&lseg->pls_refcount), + test_bit(NFS_LSEG_VALID, &lseg->pls_flags)); + + lo = lseg->pls_layout; + inode = lo->plh_inode; + + if (refcount_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) { + pnfs_get_layout_hdr(lo); + pnfs_layout_remove_lseg(lo, lseg); + if (pnfs_cache_lseg_for_layoutreturn(lo, lseg)) + lseg = NULL; + spin_unlock(&inode->i_lock); + pnfs_free_lseg(lseg); + pnfs_put_layout_hdr(lo); + } +} +EXPORT_SYMBOL_GPL(pnfs_put_lseg); + +/* + * is l2 fully contained in l1? + * start1 end1 + * [----------------------------------) + * start2 end2 + * [----------------) + */ +static bool +pnfs_lseg_range_contained(const struct pnfs_layout_range *l1, + const struct pnfs_layout_range *l2) +{ + u64 start1 = l1->offset; + u64 end1 = pnfs_end_offset(start1, l1->length); + u64 start2 = l2->offset; + u64 end2 = pnfs_end_offset(start2, l2->length); + + return (start1 <= start2) && (end1 >= end2); +} + +static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg, + struct list_head *tmp_list) +{ + if (!refcount_dec_and_test(&lseg->pls_refcount)) + return false; + pnfs_layout_remove_lseg(lseg->pls_layout, lseg); + list_add(&lseg->pls_list, tmp_list); + return true; +} + +/* Returns 1 if lseg is removed from list, 0 otherwise */ +static int mark_lseg_invalid(struct pnfs_layout_segment *lseg, + struct list_head *tmp_list) +{ + int rv = 0; + + if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) { + /* Remove the reference keeping the lseg in the + * list. It will now be removed when all + * outstanding io is finished. + */ + dprintk("%s: lseg %p ref %d\n", __func__, lseg, + refcount_read(&lseg->pls_refcount)); + if (pnfs_lseg_dec_and_remove_zero(lseg, tmp_list)) + rv = 1; + } + return rv; +} + +static bool +pnfs_should_free_range(const struct pnfs_layout_range *lseg_range, + const struct pnfs_layout_range *recall_range) +{ + return (recall_range->iomode == IOMODE_ANY || + lseg_range->iomode == recall_range->iomode) && + pnfs_lseg_range_intersecting(lseg_range, recall_range); +} + +static bool +pnfs_match_lseg_recall(const struct pnfs_layout_segment *lseg, + const struct pnfs_layout_range *recall_range, + u32 seq) +{ + if (seq != 0 && pnfs_seqid_is_newer(lseg->pls_seq, seq)) + return false; + if (recall_range == NULL) + return true; + return pnfs_should_free_range(&lseg->pls_range, recall_range); +} + +/** + * pnfs_mark_matching_lsegs_invalid - tear down lsegs or mark them for later + * @lo: layout header containing the lsegs + * @tmp_list: list head where doomed lsegs should go + * @recall_range: optional recall range argument to match (may be NULL) + * @seq: only invalidate lsegs obtained prior to this sequence (may be 0) + * + * Walk the list of lsegs in the layout header, and tear down any that should + * be destroyed. If "recall_range" is specified then the segment must match + * that range. If "seq" is non-zero, then only match segments that were handed + * out at or before that sequence. + * + * Returns number of matching invalid lsegs remaining in list after scanning + * it and purging them. + */ +int +pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo, + struct list_head *tmp_list, + const struct pnfs_layout_range *recall_range, + u32 seq) +{ + struct pnfs_layout_segment *lseg, *next; + struct nfs_server *server = NFS_SERVER(lo->plh_inode); + int remaining = 0; + + dprintk("%s:Begin lo %p\n", __func__, lo); + + if (list_empty(&lo->plh_segs)) + return 0; + list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) + if (pnfs_match_lseg_recall(lseg, recall_range, seq)) { + dprintk("%s: freeing lseg %p iomode %d seq %u " + "offset %llu length %llu\n", __func__, + lseg, lseg->pls_range.iomode, lseg->pls_seq, + lseg->pls_range.offset, lseg->pls_range.length); + if (mark_lseg_invalid(lseg, tmp_list)) + continue; + remaining++; + pnfs_lseg_cancel_io(server, lseg); + } + dprintk("%s:Return %i\n", __func__, remaining); + return remaining; +} + +static void pnfs_reset_return_info(struct pnfs_layout_hdr *lo) +{ + struct pnfs_layout_segment *lseg; + + list_for_each_entry(lseg, &lo->plh_return_segs, pls_list) + pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); +} + +static void +pnfs_free_returned_lsegs(struct pnfs_layout_hdr *lo, + struct list_head *free_me, + const struct pnfs_layout_range *range, + u32 seq) +{ + struct pnfs_layout_segment *lseg, *next; + + list_for_each_entry_safe(lseg, next, &lo->plh_return_segs, pls_list) { + if (pnfs_match_lseg_recall(lseg, range, seq)) + list_move_tail(&lseg->pls_list, free_me); + } +} + +/* note free_me must contain lsegs from a single layout_hdr */ +void +pnfs_free_lseg_list(struct list_head *free_me) +{ + struct pnfs_layout_segment *lseg, *tmp; + + if (list_empty(free_me)) + return; + + list_for_each_entry_safe(lseg, tmp, free_me, pls_list) { + list_del(&lseg->pls_list); + pnfs_free_lseg(lseg); + } +} + +static struct pnfs_layout_hdr *__pnfs_destroy_layout(struct nfs_inode *nfsi) +{ + struct pnfs_layout_hdr *lo; + LIST_HEAD(tmp_list); + + spin_lock(&nfsi->vfs_inode.i_lock); + lo = nfsi->layout; + if (lo) { + pnfs_get_layout_hdr(lo); + pnfs_mark_layout_stateid_invalid(lo, &tmp_list); + pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED); + pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED); + spin_unlock(&nfsi->vfs_inode.i_lock); + pnfs_free_lseg_list(&tmp_list); + nfs_commit_inode(&nfsi->vfs_inode, 0); + pnfs_put_layout_hdr(lo); + } else + spin_unlock(&nfsi->vfs_inode.i_lock); + return lo; +} + +void pnfs_destroy_layout(struct nfs_inode *nfsi) +{ + __pnfs_destroy_layout(nfsi); +} +EXPORT_SYMBOL_GPL(pnfs_destroy_layout); + +void pnfs_destroy_layout_final(struct nfs_inode *nfsi) +{ + struct pnfs_layout_hdr *lo = __pnfs_destroy_layout(nfsi); + struct inode *inode = &nfsi->vfs_inode; + + if (lo) { + spin_lock(&inode->i_lock); + wait_var_event_spinlock(lo, nfsi->layout != lo, + &inode->i_lock); + spin_unlock(&inode->i_lock); + } +} + +static bool +pnfs_layout_add_bulk_destroy_list(struct inode *inode, + struct list_head *layout_list) +{ + struct pnfs_layout_hdr *lo; + bool ret = false; + + spin_lock(&inode->i_lock); + lo = NFS_I(inode)->layout; + if (lo != NULL && list_empty(&lo->plh_bulk_destroy)) { + pnfs_get_layout_hdr(lo); + list_add(&lo->plh_bulk_destroy, layout_list); + ret = true; + } + spin_unlock(&inode->i_lock); + return ret; +} + +/* Caller must hold rcu_read_lock and clp->cl_lock */ +static int +pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp, + struct nfs_server *server, + struct list_head *layout_list) + __must_hold(&clp->cl_lock) + __must_hold(RCU) +{ + struct pnfs_layout_hdr *lo, *next; + struct inode *inode; + + list_for_each_entry_safe(lo, next, &server->layouts, plh_layouts) { + if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags) || + test_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags) || + !list_empty(&lo->plh_bulk_destroy)) + continue; + /* If the sb is being destroyed, just bail */ + if (!nfs_sb_active(server->super)) + break; + inode = pnfs_grab_inode_layout_hdr(lo); + if (inode != NULL) { + if (pnfs_layout_add_bulk_destroy_list(inode, + layout_list)) + continue; + rcu_read_unlock(); + spin_unlock(&clp->cl_lock); + iput(inode); + } else { + rcu_read_unlock(); + spin_unlock(&clp->cl_lock); + } + nfs_sb_deactive(server->super); + spin_lock(&clp->cl_lock); + rcu_read_lock(); + return -EAGAIN; + } + return 0; +} + +static int +pnfs_layout_free_bulk_destroy_list(struct list_head *layout_list, + enum pnfs_layout_destroy_mode mode) +{ + struct pnfs_layout_hdr *lo; + struct inode *inode; + LIST_HEAD(lseg_list); + int ret = 0; + + while (!list_empty(layout_list)) { + lo = list_entry(layout_list->next, struct pnfs_layout_hdr, + plh_bulk_destroy); + dprintk("%s freeing layout for inode %llu\n", __func__, + lo->plh_inode->i_ino); + inode = lo->plh_inode; + + pnfs_layoutcommit_inode(inode, false); + + spin_lock(&inode->i_lock); + list_del_init(&lo->plh_bulk_destroy); + if (mode == PNFS_LAYOUT_FILE_BULK_RETURN) { + pnfs_mark_layout_stateid_return(lo, &lseg_list, + IOMODE_ANY, 0); + } else if (pnfs_mark_layout_stateid_invalid(lo, &lseg_list)) { + if (mode == PNFS_LAYOUT_BULK_RETURN) + set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); + ret = -EAGAIN; + } + spin_unlock(&inode->i_lock); + pnfs_free_lseg_list(&lseg_list); + /* Free all lsegs that are attached to commit buckets */ + nfs_commit_inode(inode, 0); + pnfs_put_layout_hdr(lo); + nfs_iput_and_deactive(inode); + } + return ret; +} + +int pnfs_layout_destroy_byfsid(struct nfs_client *clp, struct nfs_fsid *fsid, + enum pnfs_layout_destroy_mode mode) +{ + struct nfs_server *server; + LIST_HEAD(layout_list); + + spin_lock(&clp->cl_lock); + rcu_read_lock(); +restart: + list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { + if (memcmp(&server->fsid, fsid, sizeof(*fsid)) != 0) + continue; + if (pnfs_layout_bulk_destroy_byserver_locked(clp, + server, + &layout_list) != 0) + goto restart; + } + rcu_read_unlock(); + spin_unlock(&clp->cl_lock); + + return pnfs_layout_free_bulk_destroy_list(&layout_list, mode); +} + +static void pnfs_layout_build_destroy_list_byclient(struct nfs_client *clp, + struct list_head *list) +{ + struct nfs_server *server; + + spin_lock(&clp->cl_lock); + rcu_read_lock(); +restart: + list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { + if (pnfs_layout_bulk_destroy_byserver_locked(clp, server, + list) != 0) + goto restart; + } + rcu_read_unlock(); + spin_unlock(&clp->cl_lock); +} + +static int pnfs_layout_do_destroy_byclid(struct nfs_client *clp, + struct list_head *list, + enum pnfs_layout_destroy_mode mode) +{ + pnfs_layout_build_destroy_list_byclient(clp, list); + return pnfs_layout_free_bulk_destroy_list(list, mode); +} + +int pnfs_layout_destroy_byclid(struct nfs_client *clp, + enum pnfs_layout_destroy_mode mode) +{ + LIST_HEAD(layout_list); + + return pnfs_layout_do_destroy_byclid(clp, &layout_list, mode); +} + +/* + * Called by the state manager to remove all layouts established under an + * expired lease. + */ +void +pnfs_destroy_all_layouts(struct nfs_client *clp) +{ + nfs4_deviceid_mark_client_invalid(clp); + nfs4_deviceid_purge_client(clp); + + pnfs_layout_destroy_byclid(clp, PNFS_LAYOUT_INVALIDATE); +} + +static void pnfs_layout_build_recover_list_byclient(struct nfs_client *clp, + struct list_head *list) +{ + struct nfs_server *server; + + spin_lock(&clp->cl_lock); + rcu_read_lock(); +restart: + list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { + if (!(server->caps & NFS_CAP_REBOOT_LAYOUTRETURN)) + continue; + if (pnfs_layout_bulk_destroy_byserver_locked(clp, server, + list) != 0) + goto restart; + } + rcu_read_unlock(); + spin_unlock(&clp->cl_lock); +} + +static int pnfs_layout_bulk_list_reboot(struct list_head *list) +{ + struct pnfs_layout_hdr *lo; + struct nfs_server *server; + int ret; + + list_for_each_entry(lo, list, plh_bulk_destroy) { + server = NFS_SERVER(lo->plh_inode); + ret = pnfs_layout_return_on_reboot(lo); + switch (ret) { + case 0: + continue; + case -NFS4ERR_BAD_STATEID: + server->caps &= ~NFS_CAP_REBOOT_LAYOUTRETURN; + break; + case -NFS4ERR_NO_GRACE: + break; + default: + goto err; + } + break; + } + return 0; +err: + return ret; +} + +int pnfs_layout_handle_reboot(struct nfs_client *clp) +{ + LIST_HEAD(list); + int ret = 0, ret2; + + pnfs_layout_build_recover_list_byclient(clp, &list); + if (!list_empty(&list)) + ret = pnfs_layout_bulk_list_reboot(&list); + ret2 = pnfs_layout_do_destroy_byclid(clp, &list, + PNFS_LAYOUT_INVALIDATE); + if (!ret) + ret = ret2; + return (ret == 0) ? 0 : -EAGAIN; +} + +static void +pnfs_set_layout_cred(struct pnfs_layout_hdr *lo, const struct cred *cred) +{ + const struct cred *old; + + if (cred && cred_fscmp(lo->plh_lc_cred, cred) != 0) { + old = xchg(&lo->plh_lc_cred, get_cred(cred)); + put_cred(old); + } +} + +/* update lo->plh_stateid with new if is more recent */ +void +pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new, + const struct cred *cred, bool update_barrier) +{ + u32 oldseq = be32_to_cpu(lo->plh_stateid.seqid); + u32 newseq = be32_to_cpu(new->seqid); + + if (!pnfs_layout_is_valid(lo)) { + pnfs_set_layout_cred(lo, cred); + nfs4_stateid_copy(&lo->plh_stateid, new); + lo->plh_barrier = newseq; + pnfs_clear_layoutreturn_info(lo); + clear_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags); + return; + } + + if (pnfs_seqid_is_newer(newseq, oldseq)) + nfs4_stateid_copy(&lo->plh_stateid, new); + + if (update_barrier) { + pnfs_barrier_update(lo, newseq); + return; + } + /* + * Because of wraparound, we want to keep the barrier + * "close" to the current seqids. We really only want to + * get here from a layoutget call. + */ + if (atomic_read(&lo->plh_outstanding) == 1) + pnfs_barrier_update(lo, be32_to_cpu(lo->plh_stateid.seqid)); +} + +static bool +pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo, + const nfs4_stateid *stateid) +{ + u32 seqid = be32_to_cpu(stateid->seqid); + + return lo->plh_barrier && pnfs_seqid_is_newer(lo->plh_barrier, seqid); +} + +/* lget is set to 1 if called from inside send_layoutget call chain */ +static bool +pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo) +{ + return lo->plh_block_lgets || + test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags); +} + +static struct nfs_server * +pnfs_find_server(struct inode *inode, struct nfs_open_context *ctx) +{ + struct nfs_server *server; + + if (inode) { + server = NFS_SERVER(inode); + } else { + struct dentry *parent_dir = dget_parent(ctx->dentry); + server = NFS_SERVER(parent_dir->d_inode); + dput(parent_dir); + } + return server; +} + +static void nfs4_free_pages(struct page **pages, size_t size) +{ + int i; + + if (!pages) + return; + + for (i = 0; i < size; i++) { + if (!pages[i]) + break; + __free_page(pages[i]); + } + kfree(pages); +} + +static struct page **nfs4_alloc_pages(size_t size, gfp_t gfp_flags) +{ + struct page **pages; + int i; + + pages = kmalloc_objs(struct page *, size, gfp_flags); + if (!pages) { + dprintk("%s: can't alloc array of %zu pages\n", __func__, size); + return NULL; + } + + for (i = 0; i < size; i++) { + pages[i] = alloc_page(gfp_flags); + if (!pages[i]) { + dprintk("%s: failed to allocate page\n", __func__); + nfs4_free_pages(pages, i); + return NULL; + } + } + + return pages; +} + +static struct nfs4_layoutget * +pnfs_alloc_init_layoutget_args(struct inode *ino, + struct nfs_open_context *ctx, + const nfs4_stateid *stateid, + const struct pnfs_layout_range *range, + gfp_t gfp_flags) +{ + struct nfs_server *server = pnfs_find_server(ino, ctx); + size_t max_reply_sz = server->pnfs_curr_ld->max_layoutget_response; + size_t max_pages = max_response_pages(server); + struct nfs4_layoutget *lgp; + + dprintk("--> %s\n", __func__); + + lgp = kzalloc_obj(*lgp, gfp_flags); + if (lgp == NULL) + return NULL; + + if (max_reply_sz) { + size_t npages = (max_reply_sz + PAGE_SIZE - 1) >> PAGE_SHIFT; + if (npages < max_pages) + max_pages = npages; + } + + lgp->args.layout.pages = nfs4_alloc_pages(max_pages, gfp_flags); + if (!lgp->args.layout.pages) { + kfree(lgp); + return NULL; + } + lgp->args.layout.pglen = max_pages * PAGE_SIZE; + lgp->res.layoutp = &lgp->args.layout; + + /* Don't confuse uninitialised result and success */ + lgp->res.status = -NFS4ERR_DELAY; + + lgp->args.minlength = PAGE_SIZE; + if (lgp->args.minlength > range->length) + lgp->args.minlength = range->length; + if (ino) { + loff_t i_size = i_size_read(ino); + + if (range->iomode == IOMODE_READ) { + if (range->offset >= i_size) + lgp->args.minlength = 0; + else if (i_size - range->offset < lgp->args.minlength) + lgp->args.minlength = i_size - range->offset; + } + } + lgp->args.maxcount = PNFS_LAYOUT_MAXSIZE; + pnfs_copy_range(&lgp->args.range, range); + lgp->args.type = server->pnfs_curr_ld->id; + lgp->args.inode = ino; + lgp->args.ctx = get_nfs_open_context(ctx); + nfs4_stateid_copy(&lgp->args.stateid, stateid); + lgp->gfp_flags = gfp_flags; + lgp->cred = ctx->cred; + return lgp; +} + +void pnfs_layoutget_free(struct nfs4_layoutget *lgp) +{ + size_t max_pages = lgp->args.layout.pglen / PAGE_SIZE; + + nfs4_free_pages(lgp->args.layout.pages, max_pages); + pnfs_put_layout_hdr(lgp->lo); + put_nfs_open_context(lgp->args.ctx); + kfree(lgp); +} + +static void pnfs_clear_layoutcommit(struct inode *inode, + struct list_head *head) +{ + struct nfs_inode *nfsi = NFS_I(inode); + struct pnfs_layout_segment *lseg, *tmp; + + if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) + return; + list_for_each_entry_safe(lseg, tmp, &nfsi->layout->plh_segs, pls_list) { + if (!test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) + continue; + pnfs_lseg_dec_and_remove_zero(lseg, head); + } +} + +static void +pnfs_layoutreturn_retry_later_locked(struct pnfs_layout_hdr *lo, + const nfs4_stateid *arg_stateid, + const struct pnfs_layout_range *range, + struct list_head *freeme) +{ + if (pnfs_layout_is_valid(lo) && + nfs4_stateid_match_other(&lo->plh_stateid, arg_stateid)) + pnfs_reset_return_info(lo); + else + pnfs_mark_layout_stateid_invalid(lo, freeme); + pnfs_clear_layoutreturn_waitbit(lo); +} + +void pnfs_layoutreturn_retry_later(struct pnfs_layout_hdr *lo, + const nfs4_stateid *arg_stateid, + const struct pnfs_layout_range *range) +{ + struct inode *inode = lo->plh_inode; + LIST_HEAD(freeme); + + spin_lock(&inode->i_lock); + pnfs_layoutreturn_retry_later_locked(lo, arg_stateid, range, &freeme); + spin_unlock(&inode->i_lock); + pnfs_free_lseg_list(&freeme); +} + +void pnfs_layoutreturn_free_lsegs(struct pnfs_layout_hdr *lo, + const nfs4_stateid *arg_stateid, + const struct pnfs_layout_range *range, + const nfs4_stateid *stateid) +{ + struct inode *inode = lo->plh_inode; + LIST_HEAD(freeme); + + spin_lock(&inode->i_lock); + if (!nfs4_stateid_match_other(&lo->plh_stateid, arg_stateid)) + goto out_unlock; + if (stateid && pnfs_layout_is_valid(lo)) { + u32 seq = be32_to_cpu(arg_stateid->seqid); + + pnfs_mark_matching_lsegs_invalid(lo, &freeme, range, seq); + pnfs_free_returned_lsegs(lo, &freeme, range, seq); + pnfs_set_layout_stateid(lo, stateid, NULL, true); + pnfs_reset_return_info(lo); + } else + pnfs_mark_layout_stateid_invalid(lo, &freeme); +out_unlock: + pnfs_clear_layoutreturn_waitbit(lo); + spin_unlock(&inode->i_lock); + pnfs_free_lseg_list(&freeme); + +} + +static bool +pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo, + nfs4_stateid *stateid, + const struct cred **cred, + enum pnfs_iomode *iomode) +{ + /* Serialise LAYOUTGET/LAYOUTRETURN */ + if (atomic_read(&lo->plh_outstanding) != 0 && lo->plh_return_seq == 0) + return false; + if (test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) + return false; + set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags); + pnfs_get_layout_hdr(lo); + nfs4_stateid_copy(stateid, &lo->plh_stateid); + *cred = get_cred(lo->plh_lc_cred); + if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) { + if (lo->plh_return_seq != 0) + stateid->seqid = cpu_to_be32(lo->plh_return_seq); + if (iomode != NULL) + *iomode = lo->plh_return_iomode; + pnfs_clear_layoutreturn_info(lo); + } else if (iomode != NULL) + *iomode = IOMODE_ANY; + pnfs_barrier_update(lo, be32_to_cpu(stateid->seqid)); + return true; +} + +static void +pnfs_init_layoutreturn_args(struct nfs4_layoutreturn_args *args, + struct pnfs_layout_hdr *lo, + const nfs4_stateid *stateid, + enum pnfs_iomode iomode) +{ + struct inode *inode = lo->plh_inode; + + args->layout_type = NFS_SERVER(inode)->pnfs_curr_ld->id; + args->inode = inode; + args->range.iomode = iomode; + args->range.offset = 0; + args->range.length = NFS4_MAX_UINT64; + args->layout = lo; + nfs4_stateid_copy(&args->stateid, stateid); +} + +static int +pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, + const nfs4_stateid *stateid, + const struct cred **pcred, + enum pnfs_iomode iomode, + unsigned int flags) +{ + struct inode *ino = lo->plh_inode; + struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; + struct nfs4_layoutreturn *lrp; + const struct cred *cred = *pcred; + int status = 0; + + *pcred = NULL; + lrp = kzalloc_obj(*lrp, nfs_io_gfp_mask()); + if (unlikely(lrp == NULL)) { + status = -ENOMEM; + spin_lock(&ino->i_lock); + pnfs_clear_layoutreturn_waitbit(lo); + spin_unlock(&ino->i_lock); + put_cred(cred); + pnfs_put_layout_hdr(lo); + goto out; + } + + pnfs_init_layoutreturn_args(&lrp->args, lo, stateid, iomode); + lrp->args.ld_private = &lrp->ld_private; + lrp->clp = NFS_SERVER(ino)->nfs_client; + lrp->cred = cred; + if (ld->prepare_layoutreturn) + ld->prepare_layoutreturn(&lrp->args); + + status = nfs4_proc_layoutreturn(lrp, flags); +out: + dprintk("<-- %s status: %d\n", __func__, status); + return status; +} + +/* Return true if layoutreturn is needed */ +static bool +pnfs_layout_need_return(struct pnfs_layout_hdr *lo) +{ + if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) + return false; + return pnfs_mark_layout_stateid_return(lo, &lo->plh_return_segs, + lo->plh_return_iomode, + lo->plh_return_seq) != EBUSY; +} + +static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo) +{ + struct inode *inode= lo->plh_inode; + + if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) + return; + spin_lock(&inode->i_lock); + if (pnfs_layout_need_return(lo)) { + const struct cred *cred; + nfs4_stateid stateid; + enum pnfs_iomode iomode; + bool send; + + send = pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode); + spin_unlock(&inode->i_lock); + if (send) { + /* Send an async layoutreturn so we dont deadlock */ + pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, + PNFS_FL_LAYOUTRETURN_ASYNC); + } + } else + spin_unlock(&inode->i_lock); +} + +/* + * Initiates a LAYOUTRETURN(FILE), and removes the pnfs_layout_hdr + * when the layout segment list is empty. + * + * Note that a pnfs_layout_hdr can exist with an empty layout segment + * list when LAYOUTGET has failed, or when LAYOUTGET succeeded, but the + * deviceid is marked invalid. + */ +int +_pnfs_return_layout(struct inode *ino) +{ + struct pnfs_layout_hdr *lo = NULL; + struct nfs_inode *nfsi = NFS_I(ino); + struct pnfs_layout_range range = { + .iomode = IOMODE_ANY, + .offset = 0, + .length = NFS4_MAX_UINT64, + }; + LIST_HEAD(tmp_list); + const struct cred *cred; + nfs4_stateid stateid; + int status = 0; + bool send, valid_layout; + + dprintk("NFS: %s for inode %llu\n", __func__, ino->i_ino); + + spin_lock(&ino->i_lock); + lo = nfsi->layout; + if (!lo) { + spin_unlock(&ino->i_lock); + dprintk("NFS: %s no layout to return\n", __func__); + goto out; + } + /* Reference matched in nfs4_layoutreturn_release */ + pnfs_get_layout_hdr(lo); + /* Is there an outstanding layoutreturn ? */ + if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) { + spin_unlock(&ino->i_lock); + if (wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, + TASK_UNINTERRUPTIBLE)) + goto out_put_layout_hdr; + spin_lock(&ino->i_lock); + } + valid_layout = pnfs_layout_is_valid(lo); + pnfs_clear_layoutcommit(ino, &tmp_list); + pnfs_mark_matching_lsegs_return(lo, &tmp_list, &range, 0); + + + /* Don't send a LAYOUTRETURN if list was initially empty */ + if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) || + !valid_layout) { + spin_unlock(&ino->i_lock); + dprintk("NFS: %s no layout segments to return\n", __func__); + goto out_wait_layoutreturn; + } + + send = pnfs_prepare_layoutreturn(lo, &stateid, &cred, NULL); + spin_unlock(&ino->i_lock); + if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) + NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range); + if (send) + status = pnfs_send_layoutreturn(lo, &stateid, &cred, IOMODE_ANY, + 0); +out_wait_layoutreturn: + wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, TASK_UNINTERRUPTIBLE); +out_put_layout_hdr: + pnfs_free_lseg_list(&tmp_list); + pnfs_put_layout_hdr(lo); +out: + dprintk("<-- %s status: %d\n", __func__, status); + return status; +} + +int +pnfs_commit_and_return_layout(struct inode *inode) +{ + struct pnfs_layout_hdr *lo; + int ret; + + spin_lock(&inode->i_lock); + lo = NFS_I(inode)->layout; + if (lo == NULL) { + spin_unlock(&inode->i_lock); + return 0; + } + pnfs_get_layout_hdr(lo); + /* Block new layoutgets and read/write to ds */ + lo->plh_block_lgets++; + spin_unlock(&inode->i_lock); + filemap_fdatawait(inode->i_mapping); + ret = pnfs_layoutcommit_inode(inode, true); + if (ret == 0) + ret = _pnfs_return_layout(inode); + spin_lock(&inode->i_lock); + lo->plh_block_lgets--; + spin_unlock(&inode->i_lock); + pnfs_put_layout_hdr(lo); + return ret; +} + +static int pnfs_layout_return_on_reboot(struct pnfs_layout_hdr *lo) +{ + struct inode *inode = lo->plh_inode; + const struct cred *cred; + + spin_lock(&inode->i_lock); + if (!pnfs_layout_is_valid(lo)) { + spin_unlock(&inode->i_lock); + return 0; + } + cred = get_cred(lo->plh_lc_cred); + pnfs_get_layout_hdr(lo); + spin_unlock(&inode->i_lock); + + return pnfs_send_layoutreturn(lo, &zero_stateid, &cred, IOMODE_ANY, + PNFS_FL_LAYOUTRETURN_PRIVILEGED); +} + +bool pnfs_roc(struct inode *ino, struct nfs4_layoutreturn_args *args, + struct nfs4_layoutreturn_res *res, const struct cred *cred, + bool sync) +{ + struct nfs_inode *nfsi = NFS_I(ino); + struct nfs_open_context *ctx; + struct nfs4_state *state; + struct pnfs_layout_hdr *lo; + struct pnfs_layout_segment *lseg, *next; + const struct cred *lc_cred; + nfs4_stateid stateid; + enum pnfs_iomode iomode = 0; + bool layoutreturn = false, roc = false; + bool skip_read; + + if (!nfs_have_layout(ino)) + return false; +retry: + rcu_read_lock(); + spin_lock(&ino->i_lock); + lo = nfsi->layout; + if (!lo || !pnfs_layout_is_valid(lo) || + test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) { + lo = NULL; + goto out_noroc; + } + + /* no roc if we hold a delegation */ + skip_read = false; + if (nfs4_check_delegation(ino, FMODE_READ)) { + if (nfs4_check_delegation(ino, FMODE_WRITE)) { + lo = NULL; + goto out_noroc; + } + skip_read = true; + } + + list_for_each_entry_rcu(ctx, &nfsi->open_files, list) { + state = ctx->state; + if (state == NULL) + continue; + /* Don't return layout if there is open file state */ + if (state->state & FMODE_WRITE) { + lo = NULL; + goto out_noroc; + } + if (state->state & FMODE_READ) + skip_read = true; + } + + if (skip_read) { + bool writes = false; + + list_for_each_entry(lseg, &lo->plh_segs, pls_list) { + if (lseg->pls_range.iomode != IOMODE_READ) { + writes = true; + break; + } + } + if (!writes) { + lo = NULL; + goto out_noroc; + } + } + + pnfs_get_layout_hdr(lo); + if (test_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags)) { + if (!sync) { + pnfs_set_plh_return_info( + lo, skip_read ? IOMODE_RW : IOMODE_ANY, 0); + goto out_noroc; + } + spin_unlock(&ino->i_lock); + rcu_read_unlock(); + wait_on_bit(&lo->plh_flags, NFS_LAYOUT_RETURN, + TASK_UNINTERRUPTIBLE); + pnfs_put_layout_hdr(lo); + goto retry; + } + + list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) { + if (skip_read && lseg->pls_range.iomode == IOMODE_READ) + continue; + /* If we are sending layoutreturn, invalidate all valid lsegs */ + if (!test_and_clear_bit(NFS_LSEG_ROC, &lseg->pls_flags)) + continue; + /* + * Note: mark lseg for return so pnfs_layout_remove_lseg + * doesn't invalidate the layout for us. + */ + set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); + if (!mark_lseg_invalid(lseg, &lo->plh_return_segs)) + continue; + pnfs_set_plh_return_info(lo, lseg->pls_range.iomode, 0); + } + + if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) + goto out_noroc; + + /* ROC in two conditions: + * 1. there are ROC lsegs + * 2. we don't send layoutreturn + */ + /* lo ref dropped in pnfs_roc_release() */ + layoutreturn = pnfs_prepare_layoutreturn(lo, &stateid, &lc_cred, &iomode); + /* If the creds don't match, we can't compound the layoutreturn */ + if (!layoutreturn || cred_fscmp(cred, lc_cred) != 0) + goto out_noroc; + + roc = layoutreturn; + pnfs_init_layoutreturn_args(args, lo, &stateid, iomode); + res->lrs_present = 0; + layoutreturn = false; + put_cred(lc_cred); + +out_noroc: + spin_unlock(&ino->i_lock); + rcu_read_unlock(); + pnfs_layoutcommit_inode(ino, sync); + if (roc) { + struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld; + if (ld->prepare_layoutreturn) + ld->prepare_layoutreturn(args); + pnfs_put_layout_hdr(lo); + return true; + } + if (layoutreturn) + pnfs_send_layoutreturn(lo, &stateid, &lc_cred, iomode, 0); + pnfs_put_layout_hdr(lo); + return false; +} + +int pnfs_roc_done(struct rpc_task *task, struct nfs4_layoutreturn_args **argpp, + struct nfs4_layoutreturn_res **respp, int *ret) +{ + struct nfs4_layoutreturn_args *arg = *argpp; + int retval = -EAGAIN; + + if (!arg) + return 0; + /* Handle Layoutreturn errors */ + switch (*ret) { + case 0: + retval = 0; + break; + case -NFS4ERR_NOMATCHING_LAYOUT: + /* Was there an RPC level error? If not, retry */ + if (task->tk_rpc_status == 0) + break; + /* + * Is there a fatal network level error? + * If so release the layout, but flag the error. + */ + if ((task->tk_rpc_status == -ENETDOWN || + task->tk_rpc_status == -ENETUNREACH) && + task->tk_flags & RPC_TASK_NETUNREACH_FATAL) { + *ret = 0; + (*respp)->lrs_present = 0; + retval = -EIO; + break; + } + /* If the call was not sent, let caller handle it */ + if (!RPC_WAS_SENT(task)) + return 0; + switch (task->tk_rpc_status) { + default: + /* + * Defer the layoutreturn if it was due + * to the server being down. + */ + *ret = -NFS4ERR_NOMATCHING_LAYOUT; + break; + case -EACCES: + case -EIO: + case -EKEYEXPIRED: + case -ERESTARTSYS: + case -EINTR: + /* Don't retry */ + *ret = 0; + break; + } + (*respp)->lrs_present = 0; + retval = 0; + break; + case -NFS4ERR_DELAY: + /* Let the caller handle the retry */ + *ret = -NFS4ERR_NOMATCHING_LAYOUT; + return 0; + case -NFS4ERR_OLD_STATEID: + if (!nfs4_layout_refresh_old_stateid(&arg->stateid, + &arg->range, arg->inode)) + break; + *ret = -NFS4ERR_NOMATCHING_LAYOUT; + return -EAGAIN; + } + *argpp = NULL; + *respp = NULL; + return retval; +} + +void pnfs_roc_release(struct nfs4_layoutreturn_args *args, + struct nfs4_layoutreturn_res *res, int ret) +{ + struct pnfs_layout_hdr *lo = args->layout; + struct inode *inode = args->inode; + const nfs4_stateid *res_stateid = NULL; + struct nfs4_xdr_opaque_data *ld_private = args->ld_private; + LIST_HEAD(freeme); + + switch (ret) { + case -NFS4ERR_BADSESSION: + case -NFS4ERR_DEADSESSION: + case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION: + case -NFS4ERR_NOMATCHING_LAYOUT: + spin_lock(&inode->i_lock); + pnfs_layoutreturn_retry_later_locked(lo, &args->stateid, + &args->range, &freeme); + spin_unlock(&inode->i_lock); + pnfs_free_lseg_list(&freeme); + break; + case 0: + if (res->lrs_present) + res_stateid = &res->stateid; + fallthrough; + default: + pnfs_layoutreturn_free_lsegs(lo, &args->stateid, &args->range, + res_stateid); + } + trace_nfs4_layoutreturn_on_close(args->inode, &args->stateid, ret); + if (ld_private && ld_private->ops && ld_private->ops->free) + ld_private->ops->free(ld_private); + pnfs_put_layout_hdr(lo); +} + +bool pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task) +{ + struct nfs_inode *nfsi = NFS_I(ino); + struct pnfs_layout_hdr *lo; + bool sleep = false; + + /* we might not have grabbed lo reference. so need to check under + * i_lock */ + spin_lock(&ino->i_lock); + lo = nfsi->layout; + if (lo && test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { + rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL); + sleep = true; + } + spin_unlock(&ino->i_lock); + return sleep; +} + +/* + * Compare two layout segments for sorting into layout cache. + * We want to preferentially return RW over RO layouts, so ensure those + * are seen first. + */ +static s64 +pnfs_lseg_range_cmp(const struct pnfs_layout_range *l1, + const struct pnfs_layout_range *l2) +{ + s64 d; + + /* high offset > low offset */ + d = l1->offset - l2->offset; + if (d) + return d; + + /* short length > long length */ + d = l2->length - l1->length; + if (d) + return d; + + /* read > read/write */ + return (int)(l1->iomode == IOMODE_READ) - (int)(l2->iomode == IOMODE_READ); +} + +static bool +pnfs_lseg_range_is_after(const struct pnfs_layout_range *l1, + const struct pnfs_layout_range *l2) +{ + return pnfs_lseg_range_cmp(l1, l2) > 0; +} + +static bool +pnfs_lseg_no_merge(struct pnfs_layout_segment *lseg, + struct pnfs_layout_segment *old) +{ + return false; +} + +void +pnfs_generic_layout_insert_lseg(struct pnfs_layout_hdr *lo, + struct pnfs_layout_segment *lseg, + bool (*is_after)(const struct pnfs_layout_range *, + const struct pnfs_layout_range *), + bool (*do_merge)(struct pnfs_layout_segment *, + struct pnfs_layout_segment *), + struct list_head *free_me) +{ + struct pnfs_layout_segment *lp, *tmp; + + dprintk("%s:Begin\n", __func__); + + list_for_each_entry_safe(lp, tmp, &lo->plh_segs, pls_list) { + if (test_bit(NFS_LSEG_VALID, &lp->pls_flags) == 0) + continue; + if (do_merge(lseg, lp)) { + mark_lseg_invalid(lp, free_me); + continue; + } + if (is_after(&lseg->pls_range, &lp->pls_range)) + continue; + list_add_tail(&lseg->pls_list, &lp->pls_list); + dprintk("%s: inserted lseg %p " + "iomode %d offset %llu length %llu before " + "lp %p iomode %d offset %llu length %llu\n", + __func__, lseg, lseg->pls_range.iomode, + lseg->pls_range.offset, lseg->pls_range.length, + lp, lp->pls_range.iomode, lp->pls_range.offset, + lp->pls_range.length); + goto out; + } + list_add_tail(&lseg->pls_list, &lo->plh_segs); + dprintk("%s: inserted lseg %p " + "iomode %d offset %llu length %llu at tail\n", + __func__, lseg, lseg->pls_range.iomode, + lseg->pls_range.offset, lseg->pls_range.length); +out: + pnfs_get_layout_hdr(lo); + + dprintk("%s:Return\n", __func__); +} +EXPORT_SYMBOL_GPL(pnfs_generic_layout_insert_lseg); + +static void +pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo, + struct pnfs_layout_segment *lseg, + struct list_head *free_me) +{ + struct inode *inode = lo->plh_inode; + struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; + + if (ld->add_lseg != NULL) + ld->add_lseg(lo, lseg, free_me); + else + pnfs_generic_layout_insert_lseg(lo, lseg, + pnfs_lseg_range_is_after, + pnfs_lseg_no_merge, + free_me); +} + +static struct pnfs_layout_hdr * +alloc_init_layout_hdr(struct inode *ino, + struct nfs_open_context *ctx, + gfp_t gfp_flags) +{ + struct pnfs_layout_hdr *lo; + + lo = pnfs_alloc_layout_hdr(ino, gfp_flags); + if (!lo) + return NULL; + refcount_set(&lo->plh_refcount, 1); + INIT_LIST_HEAD(&lo->plh_layouts); + INIT_LIST_HEAD(&lo->plh_segs); + INIT_LIST_HEAD(&lo->plh_return_segs); + INIT_LIST_HEAD(&lo->plh_bulk_destroy); + lo->plh_inode = ino; + lo->plh_lc_cred = get_cred(ctx->cred); + lo->plh_flags |= 1 << NFS_LAYOUT_INVALID_STID; + return lo; +} + +static struct pnfs_layout_hdr * +pnfs_find_alloc_layout(struct inode *ino, + struct nfs_open_context *ctx, + gfp_t gfp_flags) + __releases(&ino->i_lock) + __acquires(&ino->i_lock) +{ + struct nfs_inode *nfsi = NFS_I(ino); + struct pnfs_layout_hdr *new = NULL; + + dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout); + + if (nfsi->layout != NULL) + goto out_existing; + spin_unlock(&ino->i_lock); + new = alloc_init_layout_hdr(ino, ctx, gfp_flags); + spin_lock(&ino->i_lock); + + if (likely(nfsi->layout == NULL)) { /* Won the race? */ + nfsi->layout = new; + return new; + } else if (new != NULL) + pnfs_free_layout_hdr(new); +out_existing: + pnfs_get_layout_hdr(nfsi->layout); + return nfsi->layout; +} + +/* + * iomode matching rules: + * iomode lseg strict match + * iomode + * ----- ----- ------ ----- + * ANY READ N/A true + * ANY RW N/A true + * RW READ N/A false + * RW RW N/A true + * READ READ N/A true + * READ RW true false + * READ RW false true + */ +static bool +pnfs_lseg_range_match(const struct pnfs_layout_range *ls_range, + const struct pnfs_layout_range *range, + bool strict_iomode) +{ + struct pnfs_layout_range range1; + + if ((range->iomode == IOMODE_RW && + ls_range->iomode != IOMODE_RW) || + (range->iomode != ls_range->iomode && + strict_iomode) || + !pnfs_lseg_range_intersecting(ls_range, range)) + return false; + + /* range1 covers only the first byte in the range */ + range1 = *range; + range1.length = 1; + return pnfs_lseg_range_contained(ls_range, &range1); +} + +/* + * lookup range in layout + */ +static struct pnfs_layout_segment * +pnfs_find_lseg(struct pnfs_layout_hdr *lo, + struct pnfs_layout_range *range, + bool strict_iomode) +{ + struct pnfs_layout_segment *lseg, *ret = NULL; + + dprintk("%s:Begin\n", __func__); + + list_for_each_entry(lseg, &lo->plh_segs, pls_list) { + if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) && + pnfs_lseg_range_match(&lseg->pls_range, range, + strict_iomode)) { + ret = pnfs_get_lseg(lseg); + break; + } + } + + dprintk("%s:Return lseg %p ref %d\n", + __func__, ret, ret ? refcount_read(&ret->pls_refcount) : 0); + return ret; +} + +/* + * Use mdsthreshold hints set at each OPEN to determine if I/O should go + * to the MDS or over pNFS + * + * The nfs_inode read_io and write_io fields are cumulative counters reset + * when there are no layout segments. Note that in pnfs_update_layout iomode + * is set to IOMODE_READ for a READ request, and set to IOMODE_RW for a + * WRITE request. + * + * A return of true means use MDS I/O. + * + * From rfc 5661: + * If a file's size is smaller than the file size threshold, data accesses + * SHOULD be sent to the metadata server. If an I/O request has a length that + * is below the I/O size threshold, the I/O SHOULD be sent to the metadata + * server. If both file size and I/O size are provided, the client SHOULD + * reach or exceed both thresholds before sending its read or write + * requests to the data server. + */ +static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx, + struct inode *ino, int iomode) +{ + struct nfs4_threshold *t = ctx->mdsthreshold; + struct nfs_inode *nfsi = NFS_I(ino); + loff_t fsize = i_size_read(ino); + bool size = false, size_set = false, io = false, io_set = false, ret = false; + + if (t == NULL) + return ret; + + dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n", + __func__, t->bm, t->rd_sz, t->wr_sz, t->rd_io_sz, t->wr_io_sz); + + switch (iomode) { + case IOMODE_READ: + if (t->bm & THRESHOLD_RD) { + dprintk("%s fsize %llu\n", __func__, fsize); + size_set = true; + if (fsize < t->rd_sz) + size = true; + } + if (t->bm & THRESHOLD_RD_IO) { + dprintk("%s nfsi->read_io %llu\n", __func__, + nfsi->read_io); + io_set = true; + if (nfsi->read_io < t->rd_io_sz) + io = true; + } + break; + case IOMODE_RW: + if (t->bm & THRESHOLD_WR) { + dprintk("%s fsize %llu\n", __func__, fsize); + size_set = true; + if (fsize < t->wr_sz) + size = true; + } + if (t->bm & THRESHOLD_WR_IO) { + dprintk("%s nfsi->write_io %llu\n", __func__, + nfsi->write_io); + io_set = true; + if (nfsi->write_io < t->wr_io_sz) + io = true; + } + break; + } + if (size_set && io_set) { + if (size && io) + ret = true; + } else if (size || io) + ret = true; + + dprintk("<-- %s size %d io %d ret %d\n", __func__, size, io, ret); + return ret; +} + +static int pnfs_prepare_to_retry_layoutget(struct pnfs_layout_hdr *lo) +{ + /* + * send layoutcommit as it can hold up layoutreturn due to lseg + * reference + */ + pnfs_layoutcommit_inode(lo->plh_inode, false); + return wait_on_bit_action(&lo->plh_flags, NFS_LAYOUT_RETURN, + nfs_wait_bit_killable, + TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); +} + +static void nfs_layoutget_begin(struct pnfs_layout_hdr *lo) +{ + atomic_inc(&lo->plh_outstanding); +} + +static void nfs_layoutget_end(struct pnfs_layout_hdr *lo) +{ + if (atomic_dec_and_test(&lo->plh_outstanding) && + test_and_clear_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags)) { + smp_mb__after_atomic(); + wake_up_bit(&lo->plh_flags, NFS_LAYOUT_DRAIN); + } +} + +static bool pnfs_is_first_layoutget(struct pnfs_layout_hdr *lo) +{ + return test_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags); +} + +static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo) +{ + unsigned long *bitlock = &lo->plh_flags; + + clear_bit_unlock(NFS_LAYOUT_FIRST_LAYOUTGET, bitlock); + smp_mb__after_atomic(); + wake_up_bit(bitlock, NFS_LAYOUT_FIRST_LAYOUTGET); +} + +static void _add_to_server_list(struct pnfs_layout_hdr *lo, + struct nfs_server *server) +{ + if (!test_and_set_bit(NFS_LAYOUT_HASHED, &lo->plh_flags)) { + struct nfs_client *clp = server->nfs_client; + + /* The lo must be on the clp list if there is any + * chance of a CB_LAYOUTRECALL(FILE) coming in. + */ + spin_lock(&clp->cl_lock); + list_add_tail_rcu(&lo->plh_layouts, &server->layouts); + spin_unlock(&clp->cl_lock); + } +} + +/* + * Layout segment is retreived from the server if not cached. + * The appropriate layout segment is referenced and returned to the caller. + */ +struct pnfs_layout_segment * +pnfs_update_layout(struct inode *ino, + struct nfs_open_context *ctx, + loff_t pos, + u64 count, + enum pnfs_iomode iomode, + bool strict_iomode, + gfp_t gfp_flags) +{ + struct pnfs_layout_range arg = { + .iomode = iomode, + .offset = pos, + .length = count, + }; + unsigned pg_offset; + struct nfs_server *server = NFS_SERVER(ino); + struct nfs_client *clp = server->nfs_client; + struct pnfs_layout_hdr *lo = NULL; + struct pnfs_layout_segment *lseg = NULL; + struct nfs4_layoutget *lgp; + nfs4_stateid stateid; + struct nfs4_exception exception = { + .inode = ino, + }; + unsigned long giveup = jiffies + (clp->cl_lease_time << 1); + bool first; + + if (!pnfs_enabled_sb(NFS_SERVER(ino))) { + trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, + PNFS_UPDATE_LAYOUT_NO_PNFS); + goto out; + } + + if (pnfs_within_mdsthreshold(ctx, ino, iomode)) { + trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, + PNFS_UPDATE_LAYOUT_MDSTHRESH); + goto out; + } + +lookup_again: + if (!nfs4_valid_open_stateid(ctx->state)) { + trace_pnfs_update_layout(ino, pos, count, + iomode, lo, lseg, + PNFS_UPDATE_LAYOUT_INVALID_OPEN); + lseg = ERR_PTR(-EIO); + goto out; + } + + lseg = ERR_PTR(nfs4_client_recover_expired_lease(clp)); + if (IS_ERR(lseg)) + goto out; + first = false; + spin_lock(&ino->i_lock); + lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags); + if (lo == NULL) { + spin_unlock(&ino->i_lock); + lseg = ERR_PTR(-ENOMEM); + trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, + PNFS_UPDATE_LAYOUT_NOMEM); + goto out; + } + + /* Do we even need to bother with this? */ + if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) { + trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, + PNFS_UPDATE_LAYOUT_BULK_RECALL); + dprintk("%s matches recall, use MDS\n", __func__); + goto out_unlock; + } + + /* if LAYOUTGET already failed once we don't try again */ + if (pnfs_layout_io_test_failed(lo, iomode)) { + trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, + PNFS_UPDATE_LAYOUT_IO_TEST_FAIL); + goto out_unlock; + } + + /* + * If the layout segment list is empty, but there are outstanding + * layoutget calls, then they might be subject to a layoutrecall. + */ + if (test_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags) && + atomic_read(&lo->plh_outstanding) != 0) { + spin_unlock(&ino->i_lock); + lseg = ERR_PTR(wait_on_bit(&lo->plh_flags, NFS_LAYOUT_DRAIN, + TASK_KILLABLE)); + if (IS_ERR(lseg)) + goto out_put_layout_hdr; + pnfs_put_layout_hdr(lo); + goto lookup_again; + } + + /* + * Because we free lsegs when sending LAYOUTRETURN, we need to wait + * for LAYOUTRETURN. + */ + if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) { + spin_unlock(&ino->i_lock); + dprintk("%s wait for layoutreturn\n", __func__); + lseg = ERR_PTR(pnfs_prepare_to_retry_layoutget(lo)); + if (!IS_ERR(lseg)) { + dprintk("%s retrying\n", __func__); + trace_pnfs_update_layout(ino, pos, count, iomode, lo, + lseg, + PNFS_UPDATE_LAYOUT_RETRY); + pnfs_put_layout_hdr(lo); + goto lookup_again; + } + trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, + PNFS_UPDATE_LAYOUT_RETURN); + goto out_put_layout_hdr; + } + + lseg = pnfs_find_lseg(lo, &arg, strict_iomode); + if (lseg) { + trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, + PNFS_UPDATE_LAYOUT_FOUND_CACHED); + goto out_unlock; + } + + /* + * Choose a stateid for the LAYOUTGET. If we don't have a layout + * stateid, or it has been invalidated, then we must use the open + * stateid. + */ + if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) { + int status; + + /* + * The first layoutget for the file. Need to serialize per + * RFC 5661 Errata 3208. + */ + if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET, + &lo->plh_flags)) { + spin_unlock(&ino->i_lock); + lseg = ERR_PTR(wait_on_bit(&lo->plh_flags, + NFS_LAYOUT_FIRST_LAYOUTGET, + TASK_KILLABLE)); + if (IS_ERR(lseg)) + goto out_put_layout_hdr; + pnfs_put_layout_hdr(lo); + dprintk("%s retrying\n", __func__); + goto lookup_again; + } + + spin_unlock(&ino->i_lock); + first = true; + status = nfs4_select_rw_stateid(ctx->state, + iomode == IOMODE_RW ? FMODE_WRITE : FMODE_READ, + NULL, &stateid, NULL); + if (status != 0) { + lseg = ERR_PTR(status); + trace_pnfs_update_layout(ino, pos, count, + iomode, lo, lseg, + PNFS_UPDATE_LAYOUT_INVALID_OPEN); + nfs4_schedule_stateid_recovery(server, ctx->state); + pnfs_clear_first_layoutget(lo); + pnfs_put_layout_hdr(lo); + goto lookup_again; + } + spin_lock(&ino->i_lock); + } else { + nfs4_stateid_copy(&stateid, &lo->plh_stateid); + } + + if (pnfs_layoutgets_blocked(lo)) { + trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, + PNFS_UPDATE_LAYOUT_BLOCKED); + goto out_unlock; + } + nfs_layoutget_begin(lo); + spin_unlock(&ino->i_lock); + + _add_to_server_list(lo, server); + + pg_offset = arg.offset & ~PAGE_MASK; + if (pg_offset) { + arg.offset -= pg_offset; + arg.length += pg_offset; + } + if (arg.length != NFS4_MAX_UINT64) + arg.length = PAGE_ALIGN(arg.length); + + lgp = pnfs_alloc_init_layoutget_args(ino, ctx, &stateid, &arg, gfp_flags); + if (!lgp) { + lseg = ERR_PTR(-ENOMEM); + trace_pnfs_update_layout(ino, pos, count, iomode, lo, NULL, + PNFS_UPDATE_LAYOUT_NOMEM); + nfs_layoutget_end(lo); + goto out_put_layout_hdr; + } + + lgp->lo = lo; + pnfs_get_layout_hdr(lo); + + lseg = nfs4_proc_layoutget(lgp, &exception); + trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, + PNFS_UPDATE_LAYOUT_SEND_LAYOUTGET); + nfs_layoutget_end(lo); + if (IS_ERR(lseg)) { + switch(PTR_ERR(lseg)) { + case -EBUSY: + if (time_after(jiffies, giveup)) + lseg = NULL; + break; + case -ERECALLCONFLICT: + case -EAGAIN: + break; + case -ENODATA: + /* The server returned NFS4ERR_LAYOUTUNAVAILABLE */ + pnfs_layout_set_fail_bit( + lo, pnfs_iomode_to_fail_bit(iomode)); + lseg = NULL; + goto out_put_layout_hdr; + default: + if (!nfs_error_is_fatal(PTR_ERR(lseg))) { + pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); + lseg = NULL; + } + goto out_put_layout_hdr; + } + if (lseg) { + if (!exception.retry) + goto out_put_layout_hdr; + if (first) + pnfs_clear_first_layoutget(lo); + trace_pnfs_update_layout(ino, pos, count, + iomode, lo, lseg, PNFS_UPDATE_LAYOUT_RETRY); + pnfs_put_layout_hdr(lo); + goto lookup_again; + } + } else { + pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); + } + +out_put_layout_hdr: + if (first) + pnfs_clear_first_layoutget(lo); + trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, + PNFS_UPDATE_LAYOUT_EXIT); + pnfs_put_layout_hdr(lo); +out: + dprintk("%s: inode %s/%llu pNFS layout segment %s for " + "(%s, offset: %llu, length: %llu)\n", + __func__, ino->i_sb->s_id, + (unsigned long long)ino->i_ino, + IS_ERR_OR_NULL(lseg) ? "not found" : "found", + iomode==IOMODE_RW ? "read/write" : "read-only", + (unsigned long long)pos, + (unsigned long long)count); + return lseg; +out_unlock: + spin_unlock(&ino->i_lock); + goto out_put_layout_hdr; +} +EXPORT_SYMBOL_GPL(pnfs_update_layout); + +static bool +pnfs_sanity_check_layout_range(struct pnfs_layout_range *range) +{ + switch (range->iomode) { + case IOMODE_READ: + case IOMODE_RW: + break; + default: + return false; + } + if (range->offset == NFS4_MAX_UINT64) + return false; + if (range->length == 0) + return false; + if (range->length != NFS4_MAX_UINT64 && + range->length > NFS4_MAX_UINT64 - range->offset) + return false; + return true; +} + +static struct pnfs_layout_hdr * +_pnfs_grab_empty_layout(struct inode *ino, struct nfs_open_context *ctx) +{ + struct pnfs_layout_hdr *lo; + + spin_lock(&ino->i_lock); + lo = pnfs_find_alloc_layout(ino, ctx, nfs_io_gfp_mask()); + if (!lo) + goto out_unlock; + if (!test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) + goto out_unlock; + if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) + goto out_unlock; + if (pnfs_layoutgets_blocked(lo)) + goto out_unlock; + if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags)) + goto out_unlock; + nfs_layoutget_begin(lo); + spin_unlock(&ino->i_lock); + _add_to_server_list(lo, NFS_SERVER(ino)); + return lo; + +out_unlock: + spin_unlock(&ino->i_lock); + pnfs_put_layout_hdr(lo); + return NULL; +} + +static void _lgopen_prepare_attached(struct nfs4_opendata *data, + struct nfs_open_context *ctx) +{ + struct inode *ino = data->dentry->d_inode; + struct pnfs_layout_range rng = { + .iomode = (data->o_arg.fmode & FMODE_WRITE) ? + IOMODE_RW: IOMODE_READ, + .offset = 0, + .length = NFS4_MAX_UINT64, + }; + struct nfs4_layoutget *lgp; + struct pnfs_layout_hdr *lo; + + /* Heuristic: don't send layoutget if we have cached data */ + if (rng.iomode == IOMODE_READ && + (i_size_read(ino) == 0 || ino->i_mapping->nrpages != 0)) + return; + + lo = _pnfs_grab_empty_layout(ino, ctx); + if (!lo) + return; + lgp = pnfs_alloc_init_layoutget_args(ino, ctx, ¤t_stateid, &rng, + nfs_io_gfp_mask()); + if (!lgp) { + pnfs_clear_first_layoutget(lo); + nfs_layoutget_end(lo); + pnfs_put_layout_hdr(lo); + return; + } + lgp->lo = lo; + data->lgp = lgp; + data->o_arg.lg_args = &lgp->args; + data->o_res.lg_res = &lgp->res; +} + +static void _lgopen_prepare_floating(struct nfs4_opendata *data, + struct nfs_open_context *ctx) +{ + struct inode *ino = data->dentry->d_inode; + struct pnfs_layout_range rng = { + .iomode = (data->o_arg.fmode & FMODE_WRITE) ? + IOMODE_RW: IOMODE_READ, + .offset = 0, + .length = NFS4_MAX_UINT64, + }; + struct nfs4_layoutget *lgp; + + lgp = pnfs_alloc_init_layoutget_args(ino, ctx, ¤t_stateid, &rng, + nfs_io_gfp_mask()); + if (!lgp) + return; + data->lgp = lgp; + data->o_arg.lg_args = &lgp->args; + data->o_res.lg_res = &lgp->res; +} + +void pnfs_lgopen_prepare(struct nfs4_opendata *data, + struct nfs_open_context *ctx) +{ + struct nfs_server *server = NFS_SERVER(data->dir->d_inode); + + if (!(pnfs_enabled_sb(server) && + server->pnfs_curr_ld->flags & PNFS_LAYOUTGET_ON_OPEN)) + return; + /* Could check on max_ops, but currently hardcoded high enough */ + if (!nfs_server_capable(data->dir->d_inode, NFS_CAP_LGOPEN)) + return; + if (data->lgp) + return; + if (data->state) + _lgopen_prepare_attached(data, ctx); + else + _lgopen_prepare_floating(data, ctx); +} + +void pnfs_parse_lgopen(struct inode *ino, struct nfs4_layoutget *lgp, + struct nfs_open_context *ctx) +{ + struct pnfs_layout_hdr *lo; + struct pnfs_layout_segment *lseg; + struct nfs_server *srv = NFS_SERVER(ino); + u32 iomode; + + if (!lgp) + return; + dprintk("%s: entered with status %i\n", __func__, lgp->res.status); + if (lgp->res.status) { + switch (lgp->res.status) { + default: + break; + /* + * Halt lgopen attempts if the server doesn't recognise + * the "current stateid" value, the layout type, or the + * layoutget operation as being valid. + * Also if it complains about too many ops in the compound + * or of the request/reply being too big. + */ + case -NFS4ERR_BAD_STATEID: + case -NFS4ERR_NOTSUPP: + case -NFS4ERR_REP_TOO_BIG: + case -NFS4ERR_REP_TOO_BIG_TO_CACHE: + case -NFS4ERR_REQ_TOO_BIG: + case -NFS4ERR_TOO_MANY_OPS: + case -NFS4ERR_UNKNOWN_LAYOUTTYPE: + srv->caps &= ~NFS_CAP_LGOPEN; + } + return; + } + if (!lgp->lo) { + lo = _pnfs_grab_empty_layout(ino, ctx); + if (!lo) + return; + lgp->lo = lo; + } else + lo = lgp->lo; + + lseg = pnfs_layout_process(lgp); + if (!IS_ERR(lseg)) { + iomode = lgp->args.range.iomode; + pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode)); + pnfs_put_lseg(lseg); + } +} + +void nfs4_lgopen_release(struct nfs4_layoutget *lgp) +{ + if (lgp != NULL) { + if (lgp->lo) { + pnfs_clear_first_layoutget(lgp->lo); + nfs_layoutget_end(lgp->lo); + } + pnfs_layoutget_free(lgp); + } +} + +struct pnfs_layout_segment * +pnfs_layout_process(struct nfs4_layoutget *lgp) +{ + struct pnfs_layout_hdr *lo = lgp->lo; + struct nfs4_layoutget_res *res = &lgp->res; + struct pnfs_layout_segment *lseg; + struct inode *ino = lo->plh_inode; + LIST_HEAD(free_me); + + if (!pnfs_sanity_check_layout_range(&res->range)) + return ERR_PTR(-EINVAL); + + /* Inject layout blob into I/O device driver */ + lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags); + if (IS_ERR_OR_NULL(lseg)) { + if (!lseg) + lseg = ERR_PTR(-ENOMEM); + + dprintk("%s: Could not allocate layout: error %ld\n", + __func__, PTR_ERR(lseg)); + return lseg; + } + + pnfs_init_lseg(lo, lseg, &res->range, &res->stateid); + + spin_lock(&ino->i_lock); + if (pnfs_layoutgets_blocked(lo)) { + dprintk("%s forget reply due to state\n", __func__); + goto out_forget; + } + + if (test_bit(NFS_LAYOUT_DRAIN, &lo->plh_flags) && + !pnfs_is_first_layoutget(lo)) + goto out_forget; + + if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) { + /* existing state ID, make sure the sequence number matches. */ + if (pnfs_layout_stateid_blocked(lo, &res->stateid)) { + if (!pnfs_layout_is_valid(lo)) + lo->plh_barrier = 0; + dprintk("%s forget reply due to sequence\n", __func__); + goto out_forget; + } + pnfs_set_layout_stateid(lo, &res->stateid, lgp->cred, false); + } else if (pnfs_layout_is_valid(lo)) { + /* + * We got an entirely new state ID. Mark all segments for the + * inode invalid, and retry the layoutget + */ + struct pnfs_layout_range range = { + .iomode = IOMODE_ANY, + .length = NFS4_MAX_UINT64, + }; + pnfs_mark_matching_lsegs_return(lo, &free_me, &range, 0); + goto out_forget; + } else { + /* We have a completely new layout */ + pnfs_set_layout_stateid(lo, &res->stateid, lgp->cred, true); + } + + pnfs_get_lseg(lseg); + pnfs_layout_insert_lseg(lo, lseg, &free_me); + + + if (res->return_on_close) + set_bit(NFS_LSEG_ROC, &lseg->pls_flags); + + spin_unlock(&ino->i_lock); + pnfs_free_lseg_list(&free_me); + return lseg; + +out_forget: + spin_unlock(&ino->i_lock); + lseg->pls_layout = lo; + NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg); + return ERR_PTR(-EAGAIN); +} + +/** + * pnfs_mark_matching_lsegs_return - Free or return matching layout segments + * @lo: pointer to layout header + * @tmp_list: list header to be used with pnfs_free_lseg_list() + * @return_range: describe layout segment ranges to be returned + * @seq: stateid seqid to match + * + * This function is mainly intended for use by layoutrecall. It attempts + * to free the layout segment immediately, or else to mark it for return + * as soon as its reference count drops to zero. + * + * Returns + * - 0: a layoutreturn needs to be scheduled. + * - EBUSY: there are layout segment that are still in use. + * - ENOENT: there are no layout segments that need to be returned. + */ +int +pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo, + struct list_head *tmp_list, + const struct pnfs_layout_range *return_range, + u32 seq) +{ + struct pnfs_layout_segment *lseg, *next; + struct nfs_server *server = NFS_SERVER(lo->plh_inode); + int remaining = 0; + + dprintk("%s:Begin lo %p\n", __func__, lo); + + assert_spin_locked(&lo->plh_inode->i_lock); + + if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) + tmp_list = &lo->plh_return_segs; + + list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list) + if (pnfs_match_lseg_recall(lseg, return_range, seq)) { + dprintk("%s: marking lseg %p iomode %d " + "offset %llu length %llu\n", __func__, + lseg, lseg->pls_range.iomode, + lseg->pls_range.offset, + lseg->pls_range.length); + if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) + tmp_list = &lo->plh_return_segs; + if (mark_lseg_invalid(lseg, tmp_list)) + continue; + remaining++; + set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags); + pnfs_lseg_cancel_io(server, lseg); + } + + if (remaining) { + pnfs_set_plh_return_info(lo, return_range->iomode, seq); + return -EBUSY; + } + + if (!list_empty(&lo->plh_return_segs)) { + pnfs_set_plh_return_info(lo, return_range->iomode, seq); + return 0; + } + + return -ENOENT; +} + +static void +pnfs_mark_layout_for_return(struct inode *inode, + const struct pnfs_layout_range *range) +{ + struct pnfs_layout_hdr *lo; + bool return_now = false; + + spin_lock(&inode->i_lock); + lo = NFS_I(inode)->layout; + if (!pnfs_layout_is_valid(lo)) { + spin_unlock(&inode->i_lock); + return; + } + pnfs_set_plh_return_info(lo, range->iomode, 0); + /* + * mark all matching lsegs so that we are sure to have no live + * segments at hand when sending layoutreturn. See pnfs_put_lseg() + * for how it works. + */ + if (pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, range, 0) != -EBUSY) { + const struct cred *cred; + nfs4_stateid stateid; + enum pnfs_iomode iomode; + + return_now = pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode); + spin_unlock(&inode->i_lock); + if (return_now) + pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, + PNFS_FL_LAYOUTRETURN_ASYNC); + } else { + spin_unlock(&inode->i_lock); + nfs_commit_inode(inode, 0); + } +} + +void pnfs_error_mark_layout_for_return(struct inode *inode, + struct pnfs_layout_segment *lseg) +{ + struct pnfs_layout_range range = { + .iomode = lseg->pls_range.iomode, + .offset = 0, + .length = NFS4_MAX_UINT64, + }; + + pnfs_mark_layout_for_return(inode, &range); +} +EXPORT_SYMBOL_GPL(pnfs_error_mark_layout_for_return); + +static bool +pnfs_layout_can_be_returned(struct pnfs_layout_hdr *lo) +{ + return pnfs_layout_is_valid(lo) && + !test_bit(NFS_LAYOUT_INODE_FREEING, &lo->plh_flags) && + !test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags); +} + +static struct pnfs_layout_segment * +pnfs_find_first_lseg(struct pnfs_layout_hdr *lo, + const struct pnfs_layout_range *range, + enum pnfs_iomode iomode) +{ + struct pnfs_layout_segment *lseg; + + list_for_each_entry(lseg, &lo->plh_segs, pls_list) { + if (!test_bit(NFS_LSEG_VALID, &lseg->pls_flags)) + continue; + if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)) + continue; + if (lseg->pls_range.iomode != iomode && iomode != IOMODE_ANY) + continue; + if (pnfs_lseg_range_intersecting(&lseg->pls_range, range)) + return lseg; + } + return NULL; +} + +/* Find open file states whose mode matches that of the range */ +static bool +pnfs_should_return_unused_layout(struct pnfs_layout_hdr *lo, + const struct pnfs_layout_range *range) +{ + struct list_head *head; + struct nfs_open_context *ctx; + fmode_t mode = 0; + + if (!pnfs_layout_can_be_returned(lo) || + !pnfs_find_first_lseg(lo, range, range->iomode)) + return false; + + head = &NFS_I(lo->plh_inode)->open_files; + list_for_each_entry_rcu(ctx, head, list) { + if (ctx->state) + mode |= ctx->state->state & (FMODE_READ|FMODE_WRITE); + } + + switch (range->iomode) { + default: + break; + case IOMODE_READ: + mode &= ~FMODE_WRITE; + break; + case IOMODE_RW: + if (pnfs_find_first_lseg(lo, range, IOMODE_READ)) + mode &= ~FMODE_READ; + } + return mode == 0; +} + +static int pnfs_layout_return_unused_byserver(struct nfs_server *server, + void *data) +{ + const struct pnfs_layout_range *range = data; + const struct cred *cred; + struct pnfs_layout_hdr *lo; + struct inode *inode; + nfs4_stateid stateid; + enum pnfs_iomode iomode; + +restart: + rcu_read_lock(); + list_for_each_entry_rcu(lo, &server->layouts, plh_layouts) { + inode = lo->plh_inode; + if (!inode || !pnfs_layout_can_be_returned(lo) || + test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) + continue; + spin_lock(&inode->i_lock); + if (!lo->plh_inode || + !pnfs_should_return_unused_layout(lo, range)) { + spin_unlock(&inode->i_lock); + continue; + } + pnfs_get_layout_hdr(lo); + pnfs_set_plh_return_info(lo, range->iomode, 0); + if (pnfs_mark_matching_lsegs_return(lo, &lo->plh_return_segs, + range, 0) != 0 || + !pnfs_prepare_layoutreturn(lo, &stateid, &cred, &iomode)) { + spin_unlock(&inode->i_lock); + rcu_read_unlock(); + pnfs_put_layout_hdr(lo); + cond_resched(); + goto restart; + } + spin_unlock(&inode->i_lock); + rcu_read_unlock(); + pnfs_send_layoutreturn(lo, &stateid, &cred, iomode, + PNFS_FL_LAYOUTRETURN_ASYNC); + pnfs_put_layout_hdr(lo); + cond_resched(); + goto restart; + } + rcu_read_unlock(); + return 0; +} + +void +pnfs_layout_return_unused_byclid(struct nfs_client *clp, + enum pnfs_iomode iomode) +{ + struct pnfs_layout_range range = { + .iomode = iomode, + .offset = 0, + .length = NFS4_MAX_UINT64, + }; + + nfs_client_for_each_server(clp, pnfs_layout_return_unused_byserver, + &range); +} + +/* Check if we have we have a valid layout but if there isn't an intersection + * between the request and the pgio->pg_lseg, put this pgio->pg_lseg away. + */ +void +pnfs_generic_pg_check_layout(struct nfs_pageio_descriptor *pgio, + struct nfs_page *req) +{ + if (pgio->pg_lseg == NULL || + (test_bit(NFS_LSEG_VALID, &pgio->pg_lseg->pls_flags) && + pnfs_lseg_request_intersecting(pgio->pg_lseg, req))) + return; + pnfs_put_lseg(pgio->pg_lseg); + pgio->pg_lseg = NULL; +} +EXPORT_SYMBOL_GPL(pnfs_generic_pg_check_layout); + +void +pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req) +{ + u64 rd_size; + + pnfs_generic_pg_check_layout(pgio, req); + if (pgio->pg_lseg == NULL) { + if (pgio->pg_dreq == NULL) + rd_size = i_size_read(pgio->pg_inode) - req_offset(req); + else + rd_size = nfs_dreq_bytes_left(pgio->pg_dreq, + req_offset(req)); + + pgio->pg_lseg = + pnfs_update_layout(pgio->pg_inode, nfs_req_openctx(req), + req_offset(req), rd_size, + IOMODE_READ, false, + nfs_io_gfp_mask()); + if (IS_ERR(pgio->pg_lseg)) { + pgio->pg_error = PTR_ERR(pgio->pg_lseg); + pgio->pg_lseg = NULL; + return; + } + } + /* If no lseg, fall back to read through mds */ + if (pgio->pg_lseg == NULL) + nfs_pageio_reset_read_mds(pgio); + +} +EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_read); + +void +pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio, + struct nfs_page *req, u64 wb_size) +{ + pnfs_generic_pg_check_layout(pgio, req); + if (pgio->pg_lseg == NULL) { + pgio->pg_lseg = + pnfs_update_layout(pgio->pg_inode, nfs_req_openctx(req), + req_offset(req), wb_size, IOMODE_RW, + false, nfs_io_gfp_mask()); + if (IS_ERR(pgio->pg_lseg)) { + pgio->pg_error = PTR_ERR(pgio->pg_lseg); + pgio->pg_lseg = NULL; + return; + } + } + /* If no lseg, fall back to write through mds */ + if (pgio->pg_lseg == NULL) + nfs_pageio_reset_write_mds(pgio); +} +EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_write); + +void +pnfs_generic_pg_cleanup(struct nfs_pageio_descriptor *desc) +{ + if (desc->pg_lseg) { + pnfs_put_lseg(desc->pg_lseg); + desc->pg_lseg = NULL; + } +} +EXPORT_SYMBOL_GPL(pnfs_generic_pg_cleanup); + +/* + * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number + * of bytes (maximum @req->wb_bytes) that can be coalesced. + */ +size_t +pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio, + struct nfs_page *prev, struct nfs_page *req) +{ + unsigned int size; + u64 seg_end, req_start, seg_left; + + size = nfs_generic_pg_test(pgio, prev, req); + if (!size) + return 0; + + /* + * 'size' contains the number of bytes left in the current page (up + * to the original size asked for in @req->wb_bytes). + * + * Calculate how many bytes are left in the layout segment + * and if there are less bytes than 'size', return that instead. + * + * Please also note that 'end_offset' is actually the offset of the + * first byte that lies outside the pnfs_layout_range. FIXME? + * + */ + if (pgio->pg_lseg) { + seg_end = pnfs_end_offset(pgio->pg_lseg->pls_range.offset, + pgio->pg_lseg->pls_range.length); + req_start = req_offset(req); + + /* start of request is past the last byte of this segment */ + if (req_start >= seg_end) + return 0; + + /* adjust 'size' iff there are fewer bytes left in the + * segment than what nfs_generic_pg_test returned */ + seg_left = seg_end - req_start; + if (seg_left < size) + size = (unsigned int)seg_left; + } + + return size; +} +EXPORT_SYMBOL_GPL(pnfs_generic_pg_test); + +int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *hdr) +{ + struct nfs_pageio_descriptor pgio; + + /* Resend all requests through the MDS */ + nfs_pageio_init_write(&pgio, hdr->inode, FLUSH_STABLE, true, + hdr->completion_ops); + return nfs_pageio_resend(&pgio, hdr); +} +EXPORT_SYMBOL_GPL(pnfs_write_done_resend_to_mds); + +static void pnfs_ld_handle_write_error(struct nfs_pgio_header *hdr) +{ + + dprintk("pnfs write error = %d\n", hdr->pnfs_error); + if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags & + PNFS_LAYOUTRET_ON_ERROR) { + pnfs_return_layout(hdr->inode); + } + if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) + hdr->task.tk_status = pnfs_write_done_resend_to_mds(hdr); +} + +/* + * Called by non rpc-based layout drivers + */ +void pnfs_ld_write_done(struct nfs_pgio_header *hdr) +{ + if (likely(!hdr->pnfs_error)) { + pnfs_set_layoutcommit(hdr->inode, hdr->lseg, + hdr->mds_offset + hdr->res.count); + hdr->mds_ops->rpc_call_done(&hdr->task, hdr); + } + trace_nfs4_pnfs_write(hdr, hdr->pnfs_error); + if (unlikely(hdr->pnfs_error)) + pnfs_ld_handle_write_error(hdr); + hdr->mds_ops->rpc_release(hdr); +} +EXPORT_SYMBOL_GPL(pnfs_ld_write_done); + +static void +pnfs_write_through_mds(struct nfs_pageio_descriptor *desc, + struct nfs_pgio_header *hdr) +{ + struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); + + if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { + list_splice_tail_init(&hdr->pages, &mirror->pg_list); + nfs_pageio_reset_write_mds(desc); + mirror->pg_recoalesce = 1; + } + hdr->completion_ops->completion(hdr); +} + +static enum pnfs_try_status +pnfs_try_to_write_data(struct nfs_pgio_header *hdr, + const struct rpc_call_ops *call_ops, + struct pnfs_layout_segment *lseg, + int how) +{ + struct inode *inode = hdr->inode; + enum pnfs_try_status trypnfs; + struct nfs_server *nfss = NFS_SERVER(inode); + + hdr->mds_ops = call_ops; + + dprintk("%s: Writing ino:%llu %u@%llu (how %d)\n", __func__, + inode->i_ino, hdr->args.count, hdr->args.offset, how); + trypnfs = nfss->pnfs_curr_ld->write_pagelist(hdr, how); + if (trypnfs != PNFS_NOT_ATTEMPTED) + nfs_inc_stats(inode, NFSIOS_PNFS_WRITE); + dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs); + return trypnfs; +} + +static void +pnfs_do_write(struct nfs_pageio_descriptor *desc, + struct nfs_pgio_header *hdr, int how) +{ + const struct rpc_call_ops *call_ops = desc->pg_rpc_callops; + struct pnfs_layout_segment *lseg = desc->pg_lseg; + enum pnfs_try_status trypnfs; + + trypnfs = pnfs_try_to_write_data(hdr, call_ops, lseg, how); + switch (trypnfs) { + case PNFS_NOT_ATTEMPTED: + pnfs_write_through_mds(desc, hdr); + break; + case PNFS_ATTEMPTED: + break; + case PNFS_TRY_AGAIN: + /* cleanup hdr and prepare to redo pnfs */ + if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { + struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); + list_splice_init(&hdr->pages, &mirror->pg_list); + mirror->pg_recoalesce = 1; + } + hdr->mds_ops->rpc_release(hdr); + } +} + +static void pnfs_writehdr_free(struct nfs_pgio_header *hdr) +{ + pnfs_put_lseg(hdr->lseg); + nfs_pgio_header_free(hdr); +} + +int +pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc) +{ + struct nfs_pgio_header *hdr; + int ret; + + hdr = nfs_pgio_header_alloc(desc->pg_rw_ops); + if (!hdr) { + desc->pg_error = -ENOMEM; + return desc->pg_error; + } + nfs_pgheader_init(desc, hdr, pnfs_writehdr_free); + + hdr->lseg = pnfs_get_lseg(desc->pg_lseg); + ret = nfs_generic_pgio(desc, hdr); + if (!ret) + pnfs_do_write(desc, hdr, desc->pg_ioflags); + + return ret; +} +EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages); + +int pnfs_read_done_resend_to_mds(struct nfs_pgio_header *hdr) +{ + struct nfs_pageio_descriptor pgio; + + /* Resend all requests through the MDS */ + nfs_pageio_init_read(&pgio, hdr->inode, true, hdr->completion_ops); + return nfs_pageio_resend(&pgio, hdr); +} +EXPORT_SYMBOL_GPL(pnfs_read_done_resend_to_mds); + +static void pnfs_ld_handle_read_error(struct nfs_pgio_header *hdr) +{ + dprintk("pnfs read error = %d\n", hdr->pnfs_error); + if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags & + PNFS_LAYOUTRET_ON_ERROR) { + pnfs_return_layout(hdr->inode); + } + if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) + hdr->task.tk_status = pnfs_read_done_resend_to_mds(hdr); +} + +/* + * Called by non rpc-based layout drivers + */ +void pnfs_ld_read_done(struct nfs_pgio_header *hdr) +{ + if (likely(!hdr->pnfs_error)) + hdr->mds_ops->rpc_call_done(&hdr->task, hdr); + trace_nfs4_pnfs_read(hdr, hdr->pnfs_error); + if (unlikely(hdr->pnfs_error)) + pnfs_ld_handle_read_error(hdr); + hdr->mds_ops->rpc_release(hdr); +} +EXPORT_SYMBOL_GPL(pnfs_ld_read_done); + +static void +pnfs_read_through_mds(struct nfs_pageio_descriptor *desc, + struct nfs_pgio_header *hdr) +{ + struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); + + if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { + list_splice_tail_init(&hdr->pages, &mirror->pg_list); + nfs_pageio_reset_read_mds(desc); + mirror->pg_recoalesce = 1; + } + hdr->completion_ops->completion(hdr); +} + +/* + * Call the appropriate parallel I/O subsystem read function. + */ +static enum pnfs_try_status +pnfs_try_to_read_data(struct nfs_pgio_header *hdr, + const struct rpc_call_ops *call_ops, + struct pnfs_layout_segment *lseg) +{ + struct inode *inode = hdr->inode; + struct nfs_server *nfss = NFS_SERVER(inode); + enum pnfs_try_status trypnfs; + + hdr->mds_ops = call_ops; + + dprintk("%s: Reading ino:%llu %u@%llu\n", + __func__, inode->i_ino, hdr->args.count, hdr->args.offset); + + trypnfs = nfss->pnfs_curr_ld->read_pagelist(hdr); + if (trypnfs != PNFS_NOT_ATTEMPTED) + nfs_inc_stats(inode, NFSIOS_PNFS_READ); + dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs); + return trypnfs; +} + +/* Resend all requests through pnfs. */ +void pnfs_read_resend_pnfs(struct nfs_pgio_header *hdr, + unsigned int mirror_idx) +{ + struct nfs_pageio_descriptor pgio; + + if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { + /* Prevent deadlocks with layoutreturn! */ + pnfs_put_lseg(hdr->lseg); + hdr->lseg = NULL; + + nfs_pageio_init_read(&pgio, hdr->inode, false, + hdr->completion_ops); + pgio.pg_mirror_idx = mirror_idx; + hdr->task.tk_status = nfs_pageio_resend(&pgio, hdr); + } +} +EXPORT_SYMBOL_GPL(pnfs_read_resend_pnfs); + +static void +pnfs_do_read(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr) +{ + const struct rpc_call_ops *call_ops = desc->pg_rpc_callops; + struct pnfs_layout_segment *lseg = desc->pg_lseg; + enum pnfs_try_status trypnfs; + + trypnfs = pnfs_try_to_read_data(hdr, call_ops, lseg); + switch (trypnfs) { + case PNFS_NOT_ATTEMPTED: + pnfs_read_through_mds(desc, hdr); + break; + case PNFS_ATTEMPTED: + break; + case PNFS_TRY_AGAIN: + /* cleanup hdr and prepare to redo pnfs */ + if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) { + struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); + list_splice_init(&hdr->pages, &mirror->pg_list); + mirror->pg_recoalesce = 1; + } + hdr->mds_ops->rpc_release(hdr); + } +} + +static void pnfs_readhdr_free(struct nfs_pgio_header *hdr) +{ + pnfs_put_lseg(hdr->lseg); + nfs_pgio_header_free(hdr); +} + +int +pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc) +{ + struct nfs_pgio_header *hdr; + int ret; + + hdr = nfs_pgio_header_alloc(desc->pg_rw_ops); + if (!hdr) { + desc->pg_error = -ENOMEM; + return desc->pg_error; + } + nfs_pgheader_init(desc, hdr, pnfs_readhdr_free); + hdr->lseg = pnfs_get_lseg(desc->pg_lseg); + ret = nfs_generic_pgio(desc, hdr); + if (!ret) + pnfs_do_read(desc, hdr); + return ret; +} +EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages); + +static void pnfs_clear_layoutcommitting(struct inode *inode) +{ + unsigned long *bitlock = &NFS_I(inode)->flags; + + clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock); + smp_mb__after_atomic(); + wake_up_bit(bitlock, NFS_INO_LAYOUTCOMMITTING); +} + +/* + * There can be multiple RW segments. + */ +static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp) +{ + struct pnfs_layout_segment *lseg; + + list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) { + if (lseg->pls_range.iomode == IOMODE_RW && + test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) + list_add(&lseg->pls_lc_list, listp); + } +} + +static void pnfs_list_write_lseg_done(struct inode *inode, struct list_head *listp) +{ + struct pnfs_layout_segment *lseg, *tmp; + + /* Matched by references in pnfs_set_layoutcommit */ + list_for_each_entry_safe(lseg, tmp, listp, pls_lc_list) { + list_del_init(&lseg->pls_lc_list); + pnfs_put_lseg(lseg); + } + + pnfs_clear_layoutcommitting(inode); +} + +void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg) +{ + pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode); +} +EXPORT_SYMBOL_GPL(pnfs_set_lo_fail); + +void +pnfs_set_layoutcommit(struct inode *inode, struct pnfs_layout_segment *lseg, + loff_t end_pos) +{ + struct nfs_inode *nfsi = NFS_I(inode); + bool mark_as_dirty = false; + + spin_lock(&inode->i_lock); + if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) { + nfsi->layout->plh_lwb = end_pos; + mark_as_dirty = true; + dprintk("%s: Set layoutcommit for inode %llu ", + __func__, inode->i_ino); + } else if (end_pos > nfsi->layout->plh_lwb) + nfsi->layout->plh_lwb = end_pos; + if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) { + /* references matched in nfs4_layoutcommit_release */ + pnfs_get_lseg(lseg); + } + spin_unlock(&inode->i_lock); + dprintk("%s: lseg %p end_pos %llu\n", + __func__, lseg, nfsi->layout->plh_lwb); + + /* if pnfs_layoutcommit_inode() runs between inode locks, the next one + * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */ + if (mark_as_dirty) + mark_inode_dirty_sync(inode); +} +EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit); + +void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data) +{ + struct nfs_server *nfss = NFS_SERVER(data->args.inode); + + if (nfss->pnfs_curr_ld->cleanup_layoutcommit) + nfss->pnfs_curr_ld->cleanup_layoutcommit(data); + pnfs_list_write_lseg_done(data->args.inode, &data->lseg_list); +} + +/* + * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and + * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough + * data to disk to allow the server to recover the data if it crashes. + * LAYOUTCOMMIT is only needed when the NFL4_UFLG_COMMIT_THRU_MDS flag + * is off, and a COMMIT is sent to a data server, or + * if WRITEs to a data server return NFS_DATA_SYNC. + */ +int +pnfs_layoutcommit_inode(struct inode *inode, bool sync) +{ + struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; + struct nfs4_layoutcommit_data *data; + struct nfs_inode *nfsi = NFS_I(inode); + loff_t end_pos; + int status; + bool mark_as_dirty = false; + + if (!pnfs_layoutcommit_outstanding(inode)) + return 0; + + dprintk("--> %s inode %llu\n", __func__, inode->i_ino); + + status = -EAGAIN; + if (test_and_set_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags)) { + if (!sync) + goto out; + status = wait_on_bit_lock_action(&nfsi->flags, + NFS_INO_LAYOUTCOMMITTING, + nfs_wait_bit_killable, + TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); + if (status) + goto out; + } + + status = -ENOMEM; + /* Note kzalloc ensures data->res.seq_res.sr_slot == NULL */ + data = kzalloc_obj(*data, nfs_io_gfp_mask()); + if (!data) + goto clear_layoutcommitting; + + status = 0; + spin_lock(&inode->i_lock); + if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) + goto out_unlock; + + INIT_LIST_HEAD(&data->lseg_list); + pnfs_list_write_lseg(inode, &data->lseg_list); + + end_pos = nfsi->layout->plh_lwb; + + nfs4_stateid_copy(&data->args.stateid, &nfsi->layout->plh_stateid); + data->cred = get_cred(nfsi->layout->plh_lc_cred); + spin_unlock(&inode->i_lock); + + data->args.inode = inode; + nfs_fattr_init(&data->fattr); + data->args.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask; + data->res.fattr = &data->fattr; + if (end_pos != 0) + data->args.lastbytewritten = end_pos - 1; + else + data->args.lastbytewritten = U64_MAX; + data->res.server = NFS_SERVER(inode); + + if (ld->prepare_layoutcommit) { + status = ld->prepare_layoutcommit(&data->args); + if (status) { + if (status != -ENOSPC) + put_cred(data->cred); + spin_lock(&inode->i_lock); + set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags); + if (end_pos > nfsi->layout->plh_lwb) + nfsi->layout->plh_lwb = end_pos; + if (status != -ENOSPC) + goto out_unlock; + spin_unlock(&inode->i_lock); + mark_as_dirty = true; + } + } + + + status = nfs4_proc_layoutcommit(data, sync); +out: + if (status || mark_as_dirty) + mark_inode_dirty_sync(inode); + dprintk("<-- %s status %d\n", __func__, status); + return status; +out_unlock: + spin_unlock(&inode->i_lock); + kfree(data); +clear_layoutcommitting: + pnfs_clear_layoutcommitting(inode); + goto out; +} +EXPORT_SYMBOL_GPL(pnfs_layoutcommit_inode); + +int +pnfs_generic_sync(struct inode *inode, bool datasync) +{ + return pnfs_layoutcommit_inode(inode, true); +} +EXPORT_SYMBOL_GPL(pnfs_generic_sync); + +struct nfs4_threshold *pnfs_mdsthreshold_alloc(void) +{ + struct nfs4_threshold *thp; + + thp = kzalloc_obj(*thp, nfs_io_gfp_mask()); + if (!thp) { + dprintk("%s mdsthreshold allocation failed\n", __func__); + return NULL; + } + return thp; +} + +#if IS_ENABLED(CONFIG_NFS_V4_2) +int +pnfs_report_layoutstat(struct inode *inode, gfp_t gfp_flags) +{ + struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; + struct nfs_server *server = NFS_SERVER(inode); + struct nfs_inode *nfsi = NFS_I(inode); + struct nfs42_layoutstat_data *data; + struct pnfs_layout_hdr *hdr; + int status = 0; + + if (!pnfs_enabled_sb(server) || !ld->prepare_layoutstats) + goto out; + + if (!nfs_server_capable(inode, NFS_CAP_LAYOUTSTATS)) + goto out; + + if (test_and_set_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags)) + goto out; + + spin_lock(&inode->i_lock); + if (!NFS_I(inode)->layout) { + spin_unlock(&inode->i_lock); + goto out_clear_layoutstats; + } + hdr = NFS_I(inode)->layout; + pnfs_get_layout_hdr(hdr); + spin_unlock(&inode->i_lock); + + data = kzalloc_obj(*data, gfp_flags); + if (!data) { + status = -ENOMEM; + goto out_put; + } + + data->args.fh = NFS_FH(inode); + data->args.inode = inode; + status = ld->prepare_layoutstats(&data->args); + if (status) + goto out_free; + + status = nfs42_proc_layoutstats_generic(NFS_SERVER(inode), data); + +out: + dprintk("%s returns %d\n", __func__, status); + return status; + +out_free: + kfree(data); +out_put: + pnfs_put_layout_hdr(hdr); +out_clear_layoutstats: + smp_mb__before_atomic(); + clear_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags); + smp_mb__after_atomic(); + goto out; +} +EXPORT_SYMBOL_GPL(pnfs_report_layoutstat); +#endif + +unsigned int layoutstats_timer; +module_param(layoutstats_timer, uint, 0644); +EXPORT_SYMBOL_GPL(layoutstats_timer); diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h new file mode 100644 index 000000000..eb39859c2 --- /dev/null +++ b/fs/nfs/pnfs.h @@ -0,0 +1,928 @@ +/* + * pNFS client data structures. + * + * Copyright (c) 2002 + * The Regents of the University of Michigan + * All Rights Reserved + * + * Dean Hildebrand + * + * Permission is granted to use, copy, create derivative works, and + * redistribute this software and such derivative works for any purpose, + * so long as the name of the University of Michigan is not used in + * any advertising or publicity pertaining to the use or distribution + * of this software without specific, written prior authorization. If + * the above copyright notice or any other identification of the + * University of Michigan is included in any copy of any portion of + * this software, then the disclaimer below must also be included. + * + * This software is provided as is, without representation or warranty + * of any kind either express or implied, including without limitation + * the implied warranties of merchantability, fitness for a particular + * purpose, or noninfringement. The Regents of the University of + * Michigan shall not be liable for any damages, including special, + * indirect, incidental, or consequential damages, with respect to any + * claim arising out of or in connection with the use of the software, + * even if it has been or is hereafter advised of the possibility of + * such damages. + */ + +#ifndef FS_NFS_PNFS_H +#define FS_NFS_PNFS_H + +#include +#include +#include +#include + +struct nfs4_exception; +struct nfs4_opendata; + +enum { + NFS_LSEG_VALID = 0, /* cleared when lseg is recalled/returned */ + NFS_LSEG_ROC, /* roc bit received from server */ + NFS_LSEG_LAYOUTCOMMIT, /* layoutcommit bit set for layoutcommit */ + NFS_LSEG_LAYOUTRETURN, /* layoutreturn bit set for layoutreturn */ + NFS_LSEG_UNAVAILABLE, /* unavailable bit set for temporary problem */ +}; + +/* Individual ip address */ +struct nfs4_pnfs_ds_addr { + struct sockaddr_storage da_addr; + size_t da_addrlen; + struct list_head da_node; /* nfs4_pnfs_dev_hlist dev_dslist */ + char *da_remotestr; /* human readable addr+port */ + const char *da_netid; + int da_transport; +}; + +struct nfs4_pnfs_ds { + struct list_head ds_node; /* nfs4_pnfs_dev_hlist dev_dslist */ + char *ds_remotestr; /* comma sep list of addrs */ + struct list_head ds_addrs; + const struct net *ds_net; + struct nfs_client *ds_clp; + refcount_t ds_count; + unsigned long ds_state; +#define NFS4DS_CONNECTING 0 /* ds is establishing connection */ +}; + +struct pnfs_layout_segment { + struct list_head pls_list; + struct list_head pls_lc_list; + struct list_head pls_commits; + struct pnfs_layout_range pls_range; + refcount_t pls_refcount; + u32 pls_seq; + unsigned long pls_flags; + struct pnfs_layout_hdr *pls_layout; +}; + +enum pnfs_try_status { + PNFS_ATTEMPTED = 0, + PNFS_NOT_ATTEMPTED = 1, + PNFS_TRY_AGAIN = 2, +}; + +#if IS_ENABLED(CONFIG_NFS_V4) + +#define LAYOUT_NFSV4_1_MODULE_PREFIX "nfs-layouttype4" + +/* + * Default data server connection timeout and retrans vaules. + * Set by module parameters dataserver_timeo and dataserver_retrans. + */ +#define NFS4_DEF_DS_TIMEO 600 /* in tenths of a second */ +#define NFS4_DEF_DS_RETRANS 5 +#define PNFS_DEVICE_RETRY_TIMEOUT (120*HZ) + +enum { + NFS_LAYOUT_RO_FAILED = 0, /* get ro layout failed stop trying */ + NFS_LAYOUT_RW_FAILED, /* get rw layout failed stop trying */ + NFS_LAYOUT_BULK_RECALL, /* bulk recall affecting layout */ + NFS_LAYOUT_RETURN, /* layoutreturn in progress */ + NFS_LAYOUT_RETURN_LOCK, /* Serialise layoutreturn */ + NFS_LAYOUT_RETURN_REQUESTED, /* Return this layout ASAP */ + NFS_LAYOUT_INVALID_STID, /* layout stateid id is invalid */ + NFS_LAYOUT_FIRST_LAYOUTGET, /* Serialize first layoutget */ + NFS_LAYOUT_INODE_FREEING, /* The inode is being freed */ + NFS_LAYOUT_HASHED, /* The layout visible */ + NFS_LAYOUT_DRAIN, +}; + +enum layoutdriver_policy_flags { + /* Should the pNFS client commit and return the layout upon truncate to + * a smaller size */ + PNFS_LAYOUTRET_ON_SETATTR = 1 << 0, + PNFS_LAYOUTRET_ON_ERROR = 1 << 1, + PNFS_READ_WHOLE_PAGE = 1 << 2, + PNFS_LAYOUTGET_ON_OPEN = 1 << 3, +}; + +enum pnfs_layout_destroy_mode { + PNFS_LAYOUT_INVALIDATE = 0, + PNFS_LAYOUT_BULK_RETURN, + PNFS_LAYOUT_FILE_BULK_RETURN, +}; + +struct nfs4_deviceid_node; + +/* Per-layout driver specific registration structure */ +struct pnfs_layoutdriver_type { + struct list_head pnfs_tblid; + const u32 id; + const char *name; + struct module *owner; + unsigned flags; + unsigned max_layoutget_response; + + int (*set_layoutdriver) (struct nfs_server *, const struct nfs_fh *); + int (*clear_layoutdriver) (struct nfs_server *); + + struct pnfs_layout_hdr * (*alloc_layout_hdr) (struct inode *inode, gfp_t gfp_flags); + void (*free_layout_hdr) (struct pnfs_layout_hdr *); + + struct pnfs_layout_segment * (*alloc_lseg) (struct pnfs_layout_hdr *layoutid, struct nfs4_layoutget_res *lgr, gfp_t gfp_flags); + void (*free_lseg) (struct pnfs_layout_segment *lseg); + void (*add_lseg) (struct pnfs_layout_hdr *layoutid, + struct pnfs_layout_segment *lseg, + struct list_head *free_me); + + void (*return_range) (struct pnfs_layout_hdr *lo, + struct pnfs_layout_range *range); + + /* test for nfs page cache coalescing */ + const struct nfs_pageio_ops *pg_read_ops; + const struct nfs_pageio_ops *pg_write_ops; + + struct pnfs_ds_commit_info *(*get_ds_info) (struct inode *inode); + + int (*sync)(struct inode *inode, bool datasync); + + /* + * Return PNFS_ATTEMPTED to indicate the layout code has attempted + * I/O, else return PNFS_NOT_ATTEMPTED to fall back to normal NFS + */ + enum pnfs_try_status (*read_pagelist)(struct nfs_pgio_header *); + enum pnfs_try_status (*write_pagelist)(struct nfs_pgio_header *, int); + + void (*free_deviceid_node) (struct nfs4_deviceid_node *); + struct nfs4_deviceid_node * (*alloc_deviceid_node) + (struct nfs_server *server, struct pnfs_device *pdev, + gfp_t gfp_flags); + + int (*prepare_layoutreturn) (struct nfs4_layoutreturn_args *); + + void (*cleanup_layoutcommit) (struct nfs4_layoutcommit_data *data); + int (*prepare_layoutcommit) (struct nfs4_layoutcommit_args *args); + int (*prepare_layoutstats) (struct nfs42_layoutstat_args *args); + + void (*cancel_io)(struct pnfs_layout_segment *lseg); +}; + +struct pnfs_commit_ops { + void (*setup_ds_info)(struct pnfs_ds_commit_info *, + struct pnfs_layout_segment *); + void (*release_ds_info)(struct pnfs_ds_commit_info *, + struct inode *inode); + int (*commit_pagelist)(struct inode *inode, + struct list_head *mds_pages, + int how, + struct nfs_commit_info *cinfo); + void (*mark_request_commit) (struct nfs_page *req, + struct pnfs_layout_segment *lseg, + struct nfs_commit_info *cinfo, + u32 ds_commit_idx); + void (*clear_request_commit) (struct nfs_page *req, + struct nfs_commit_info *cinfo); + int (*scan_commit_lists) (struct nfs_commit_info *cinfo, + int max); + void (*recover_commit_reqs) (struct list_head *list, + struct nfs_commit_info *cinfo); +}; + +struct pnfs_layout_hdr { + refcount_t plh_refcount; + atomic_t plh_outstanding; /* number of RPCs out */ + struct list_head plh_layouts; /* other client layouts */ + struct list_head plh_bulk_destroy; + struct list_head plh_segs; /* layout segments list */ + struct list_head plh_return_segs; /* invalid layout segments */ + unsigned long plh_block_lgets; /* block LAYOUTGET if >0 */ + unsigned long plh_retry_timestamp; + unsigned long plh_flags; + nfs4_stateid plh_stateid; + u32 plh_barrier; /* ignore lower seqids */ + u32 plh_return_seq; + enum pnfs_iomode plh_return_iomode; + loff_t plh_lwb; /* last write byte for layoutcommit */ + const struct cred *plh_lc_cred; /* layoutcommit cred */ + struct inode *plh_inode; + struct rcu_head plh_rcu; +}; + +struct pnfs_device { + struct nfs4_deviceid dev_id; + unsigned int layout_type; + unsigned int mincount; + unsigned int maxcount; /* gdia_maxcount */ + struct page **pages; + unsigned int pgbase; + unsigned int pglen; /* reply buffer length */ + unsigned char nocache : 1;/* May not be cached */ +}; + +#define NFS4_PNFS_GETDEVLIST_MAXNUM 16 + +struct pnfs_devicelist { + unsigned int eof; + unsigned int num_devs; + struct nfs4_deviceid dev_id[NFS4_PNFS_GETDEVLIST_MAXNUM]; +}; + +extern int pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *); +extern void pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *); +extern const struct pnfs_layoutdriver_type *pnfs_find_layoutdriver(u32 id); +extern void pnfs_put_layoutdriver(const struct pnfs_layoutdriver_type *ld); + +/* nfs4proc.c */ +#define PNFS_FL_LAYOUTRETURN_ASYNC (1U << 0) +#define PNFS_FL_LAYOUTRETURN_PRIVILEGED (1U << 1) + +extern size_t max_response_pages(struct nfs_server *server); +extern int nfs4_proc_getdeviceinfo(struct nfs_server *server, + struct pnfs_device *dev, + const struct cred *cred); +extern struct pnfs_layout_segment * +nfs4_proc_layoutget(struct nfs4_layoutget *lgp, + struct nfs4_exception *exception); +extern int nfs4_proc_layoutreturn(struct nfs4_layoutreturn *lrp, + unsigned int flags); + +/* pnfs.c */ +void pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo); +void pnfs_put_lseg(struct pnfs_layout_segment *lseg); + +void set_pnfs_layoutdriver(struct nfs_server *, const struct nfs_fh *, struct nfs_fsinfo *); +void unset_pnfs_layoutdriver(struct nfs_server *); +void pnfs_generic_pg_check_layout(struct nfs_pageio_descriptor *pgio, struct nfs_page *req); +void pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *, struct nfs_page *); +int pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc); +void pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio, + struct nfs_page *req, u64 wb_size); +void pnfs_generic_pg_cleanup(struct nfs_pageio_descriptor *); +int pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc); +size_t pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio, + struct nfs_page *prev, struct nfs_page *req); +void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg); +struct pnfs_layout_segment *pnfs_layout_process(struct nfs4_layoutget *lgp); +void pnfs_layoutget_free(struct nfs4_layoutget *lgp); +void pnfs_free_lseg_list(struct list_head *tmp_list); +void pnfs_destroy_layout(struct nfs_inode *); +void pnfs_destroy_layout_final(struct nfs_inode *); +void pnfs_destroy_all_layouts(struct nfs_client *); +int pnfs_layout_destroy_byfsid(struct nfs_client *clp, struct nfs_fsid *fsid, + enum pnfs_layout_destroy_mode mode); +int pnfs_layout_destroy_byclid(struct nfs_client *clp, + enum pnfs_layout_destroy_mode mode); +bool nfs4_layout_refresh_old_stateid(nfs4_stateid *dst, + struct pnfs_layout_range *dst_range, + struct inode *inode); +void pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo); +void pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, + const nfs4_stateid *new, + const struct cred *cred, + bool update_barrier); +int pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo, + struct list_head *tmp_list, + const struct pnfs_layout_range *recall_range, + u32 seq); +int pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo, + struct list_head *tmp_list, + const struct pnfs_layout_range *recall_range, + u32 seq); +int pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo, + struct list_head *lseg_list); +bool pnfs_roc(struct inode *ino, struct nfs4_layoutreturn_args *args, + struct nfs4_layoutreturn_res *res, const struct cred *cred, + bool sync); +int pnfs_roc_done(struct rpc_task *task, struct nfs4_layoutreturn_args **argpp, + struct nfs4_layoutreturn_res **respp, int *ret); +void pnfs_roc_release(struct nfs4_layoutreturn_args *args, + struct nfs4_layoutreturn_res *res, + int ret); +bool pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task); +void pnfs_set_layoutcommit(struct inode *, struct pnfs_layout_segment *, loff_t); +void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data); +int pnfs_layoutcommit_inode(struct inode *inode, bool sync); +int pnfs_generic_sync(struct inode *inode, bool datasync); +int pnfs_nfs_generic_sync(struct inode *inode, bool datasync); +int _pnfs_return_layout(struct inode *); +int pnfs_commit_and_return_layout(struct inode *); +void pnfs_ld_write_done(struct nfs_pgio_header *); +void pnfs_ld_read_done(struct nfs_pgio_header *); +void pnfs_read_resend_pnfs(struct nfs_pgio_header *, unsigned int mirror_idx); +struct pnfs_layout_segment *pnfs_update_layout(struct inode *ino, + struct nfs_open_context *ctx, + loff_t pos, + u64 count, + enum pnfs_iomode iomode, + bool strict_iomode, + gfp_t gfp_flags); +void pnfs_layoutreturn_retry_later(struct pnfs_layout_hdr *lo, + const nfs4_stateid *arg_stateid, + const struct pnfs_layout_range *range); +void pnfs_layoutreturn_free_lsegs(struct pnfs_layout_hdr *lo, + const nfs4_stateid *arg_stateid, + const struct pnfs_layout_range *range, + const nfs4_stateid *stateid); + +void pnfs_generic_layout_insert_lseg(struct pnfs_layout_hdr *lo, + struct pnfs_layout_segment *lseg, + bool (*is_after)(const struct pnfs_layout_range *lseg_range, + const struct pnfs_layout_range *old), + bool (*do_merge)(struct pnfs_layout_segment *lseg, + struct pnfs_layout_segment *old), + struct list_head *free_me); + +void nfs4_deviceid_mark_client_invalid(struct nfs_client *clp); +int pnfs_read_done_resend_to_mds(struct nfs_pgio_header *); +int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *); +struct nfs4_threshold *pnfs_mdsthreshold_alloc(void); +void pnfs_error_mark_layout_for_return(struct inode *inode, + struct pnfs_layout_segment *lseg); +void pnfs_layout_return_unused_byclid(struct nfs_client *clp, + enum pnfs_iomode iomode); +int pnfs_layout_handle_reboot(struct nfs_client *clp); + +/* nfs4_deviceid_flags */ +enum { + NFS_DEVICEID_INVALID = 0, /* set when MDS clientid recalled */ + NFS_DEVICEID_UNAVAILABLE, /* device temporarily unavailable */ + NFS_DEVICEID_NOCACHE, /* device may not be cached */ +}; + +/* pnfs_dev.c */ +struct nfs4_deviceid_node { + struct hlist_node node; + struct hlist_node tmpnode; + const struct pnfs_layoutdriver_type *ld; + const struct nfs_client *nfs_client; + unsigned long flags; + unsigned long timestamp_unavailable; + struct nfs4_deviceid deviceid; + struct rcu_head rcu; + atomic_t ref; +}; + +struct nfs4_deviceid_node * +nfs4_find_get_deviceid(struct nfs_server *server, + const struct nfs4_deviceid *id, const struct cred *cred, + gfp_t gfp_mask); +void nfs4_delete_deviceid(const struct pnfs_layoutdriver_type *, const struct nfs_client *, const struct nfs4_deviceid *); +void nfs4_init_deviceid_node(struct nfs4_deviceid_node *, struct nfs_server *, + const struct nfs4_deviceid *); +bool nfs4_put_deviceid_node(struct nfs4_deviceid_node *); +void nfs4_mark_deviceid_available(struct nfs4_deviceid_node *node); +void nfs4_mark_deviceid_unavailable(struct nfs4_deviceid_node *node); +bool nfs4_test_deviceid_unavailable(struct nfs4_deviceid_node *node); +void nfs4_deviceid_purge_client(const struct nfs_client *); + +/* pnfs_nfs.c */ +struct pnfs_commit_array *pnfs_alloc_commit_array(size_t n, gfp_t gfp_flags); +void pnfs_free_commit_array(struct pnfs_commit_array *p); +struct pnfs_commit_array *pnfs_add_commit_array(struct pnfs_ds_commit_info *, + struct pnfs_commit_array *, + struct pnfs_layout_segment *); + +void pnfs_generic_ds_cinfo_release_lseg(struct pnfs_ds_commit_info *fl_cinfo, + struct pnfs_layout_segment *lseg); +void pnfs_generic_ds_cinfo_destroy(struct pnfs_ds_commit_info *fl_cinfo); + +void pnfs_generic_clear_request_commit(struct nfs_page *req, + struct nfs_commit_info *cinfo); +void pnfs_generic_commit_release(void *calldata); +void pnfs_generic_prepare_to_resend_writes(struct nfs_commit_data *data); +void pnfs_generic_rw_release(void *data); +void pnfs_generic_recover_commit_reqs(struct list_head *dst, + struct nfs_commit_info *cinfo); +int pnfs_generic_commit_pagelist(struct inode *inode, + struct list_head *mds_pages, + int how, + struct nfs_commit_info *cinfo, + int (*initiate_commit)(struct nfs_commit_data *data, + int how)); +int pnfs_generic_scan_commit_lists(struct nfs_commit_info *cinfo, int max); +void pnfs_generic_write_commit_done(struct rpc_task *task, void *data); +void nfs4_pnfs_ds_put(struct nfs4_pnfs_ds *ds); +struct nfs4_pnfs_ds *nfs4_pnfs_ds_add(const struct net *net, + struct list_head *dsaddrs, + gfp_t gfp_flags); +void nfs4_pnfs_v3_ds_connect_unload(void); +int nfs4_pnfs_ds_connect(struct nfs_server *mds_srv, struct nfs4_pnfs_ds *ds, + struct nfs4_deviceid_node *devid, unsigned int timeo, + unsigned int retrans, u32 version, u32 minor_version); +struct nfs4_pnfs_ds_addr *nfs4_decode_mp_ds_addr(struct net *net, + struct xdr_stream *xdr, + gfp_t gfp_flags); +void pnfs_layout_mark_request_commit(struct nfs_page *req, + struct pnfs_layout_segment *lseg, + struct nfs_commit_info *cinfo, + u32 ds_commit_idx); +void pnfs_lgopen_prepare(struct nfs4_opendata *data, + struct nfs_open_context *ctx); +void pnfs_parse_lgopen(struct inode *ino, struct nfs4_layoutget *lgp, + struct nfs_open_context *ctx); +void nfs4_lgopen_release(struct nfs4_layoutget *lgp); + +static inline bool nfs_have_layout(struct inode *inode) +{ + return NFS_I(inode)->layout != NULL; +} + +static inline bool pnfs_layout_is_valid(const struct pnfs_layout_hdr *lo) +{ + return test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags) == 0; +} + +static inline struct nfs4_deviceid_node * +nfs4_get_deviceid(struct nfs4_deviceid_node *d) +{ + atomic_inc(&d->ref); + return d; +} + +static inline struct pnfs_layout_segment * +pnfs_get_lseg(struct pnfs_layout_segment *lseg) +{ + if (lseg) { + refcount_inc(&lseg->pls_refcount); + smp_mb__after_atomic(); + } + return lseg; +} + +static inline bool +pnfs_is_valid_lseg(struct pnfs_layout_segment *lseg) +{ + return test_bit(NFS_LSEG_VALID, &lseg->pls_flags) != 0; +} + +/* Return true if a layout driver is being used for this mountpoint */ +static inline int pnfs_enabled_sb(struct nfs_server *nfss) +{ + return nfss->pnfs_curr_ld != NULL; +} + +static inline int +pnfs_commit_list(struct inode *inode, struct list_head *mds_pages, int how, + struct nfs_commit_info *cinfo) +{ + struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds; + + if (fl_cinfo == NULL || fl_cinfo->ncommitting == 0) + return PNFS_NOT_ATTEMPTED; + return fl_cinfo->ops->commit_pagelist(inode, mds_pages, how, cinfo); +} + +static inline struct pnfs_ds_commit_info * +pnfs_get_ds_info(struct inode *inode) +{ + struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld; + + if (ld == NULL || ld->get_ds_info == NULL) + return NULL; + return ld->get_ds_info(inode); +} + +static inline void +pnfs_init_ds_commit_info_ops(struct pnfs_ds_commit_info *fl_cinfo, struct inode *inode) +{ + struct pnfs_ds_commit_info *inode_cinfo = pnfs_get_ds_info(inode); + if (inode_cinfo != NULL) + fl_cinfo->ops = inode_cinfo->ops; +} + +static inline void +pnfs_init_ds_commit_info(struct pnfs_ds_commit_info *fl_cinfo) +{ + INIT_LIST_HEAD(&fl_cinfo->commits); + fl_cinfo->ops = NULL; +} + +static inline void +pnfs_release_ds_info(struct pnfs_ds_commit_info *fl_cinfo, struct inode *inode) +{ + if (fl_cinfo->ops != NULL && fl_cinfo->ops->release_ds_info != NULL) + fl_cinfo->ops->release_ds_info(fl_cinfo, inode); +} + +static inline void +pnfs_generic_mark_devid_invalid(struct nfs4_deviceid_node *node) +{ + set_bit(NFS_DEVICEID_INVALID, &node->flags); +} + +static inline bool +pnfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg, + struct nfs_commit_info *cinfo, u32 ds_commit_idx) +{ + struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds; + + if (!lseg || !fl_cinfo->ops || !fl_cinfo->ops->mark_request_commit) + return false; + fl_cinfo->ops->mark_request_commit(req, lseg, cinfo, ds_commit_idx); + return true; +} + +static inline bool +pnfs_clear_request_commit(struct nfs_page *req, struct nfs_commit_info *cinfo) +{ + struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds; + + if (!fl_cinfo || !fl_cinfo->ops || !fl_cinfo->ops->clear_request_commit) + return false; + fl_cinfo->ops->clear_request_commit(req, cinfo); + return true; +} + +static inline int +pnfs_scan_commit_lists(struct inode *inode, struct nfs_commit_info *cinfo, + int max) +{ + struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds; + + if (!fl_cinfo || fl_cinfo->nwritten == 0) + return 0; + return fl_cinfo->ops->scan_commit_lists(cinfo, max); +} + +static inline void +pnfs_recover_commit_reqs(struct list_head *head, struct nfs_commit_info *cinfo) +{ + struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds; + + if (fl_cinfo && fl_cinfo->nwritten != 0) + fl_cinfo->ops->recover_commit_reqs(head, cinfo); +} + +/* Should the pNFS client commit and return the layout upon a setattr */ +static inline bool +pnfs_ld_layoutret_on_setattr(struct inode *inode) +{ + if (!pnfs_enabled_sb(NFS_SERVER(inode))) + return false; + return NFS_SERVER(inode)->pnfs_curr_ld->flags & + PNFS_LAYOUTRET_ON_SETATTR; +} + +static inline bool +pnfs_ld_read_whole_page(struct inode *inode) +{ + if (!pnfs_enabled_sb(NFS_SERVER(inode))) + return false; + return NFS_SERVER(inode)->pnfs_curr_ld->flags & PNFS_READ_WHOLE_PAGE; +} + +static inline int +pnfs_sync_inode(struct inode *inode, bool datasync) +{ + if (!pnfs_enabled_sb(NFS_SERVER(inode))) + return 0; + return NFS_SERVER(inode)->pnfs_curr_ld->sync(inode, datasync); +} + +static inline bool +pnfs_layoutcommit_outstanding(struct inode *inode) +{ + struct nfs_inode *nfsi = NFS_I(inode); + + return test_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags) != 0 || + test_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags) != 0; +} + +static inline int pnfs_return_layout(struct inode *ino) +{ + struct nfs_inode *nfsi = NFS_I(ino); + struct nfs_server *nfss = NFS_SERVER(ino); + + if (pnfs_enabled_sb(nfss) && nfsi->layout) { + set_bit(NFS_LAYOUT_RETURN_REQUESTED, &nfsi->layout->plh_flags); + return _pnfs_return_layout(ino); + } + + return 0; +} + +static inline bool +pnfs_use_threshold(struct nfs4_threshold **dst, struct nfs4_threshold *src, + struct nfs_server *nfss) +{ + return (dst && src && src->bm != 0 && nfss->pnfs_curr_ld && + nfss->pnfs_curr_ld->id == src->l_type); +} + +static inline u64 +pnfs_calc_offset_end(u64 offset, u64 len) +{ + if (len == NFS4_MAX_UINT64 || len >= NFS4_MAX_UINT64 - offset) + return NFS4_MAX_UINT64; + return offset + len - 1; +} + +static inline u64 +pnfs_calc_offset_length(u64 offset, u64 end) +{ + if (end == NFS4_MAX_UINT64 || end <= offset) + return NFS4_MAX_UINT64; + return 1 + end - offset; +} + +static inline void +pnfs_copy_range(struct pnfs_layout_range *dst, + const struct pnfs_layout_range *src) +{ + memcpy(dst, src, sizeof(*dst)); +} + +static inline u64 +pnfs_end_offset(u64 start, u64 len) +{ + if (NFS4_MAX_UINT64 - start <= len) + return NFS4_MAX_UINT64; + return start + len; +} + +/* + * Are 2 ranges intersecting? + * start1 end1 + * [----------------------------------) + * start2 end2 + * [----------------) + */ +static inline bool +pnfs_is_range_intersecting(u64 start1, u64 end1, u64 start2, u64 end2) +{ + return (end1 == NFS4_MAX_UINT64 || start2 < end1) && + (end2 == NFS4_MAX_UINT64 || start1 < end2); +} + +static inline bool +pnfs_lseg_range_intersecting(const struct pnfs_layout_range *l1, + const struct pnfs_layout_range *l2) +{ + u64 end1 = pnfs_end_offset(l1->offset, l1->length); + u64 end2 = pnfs_end_offset(l2->offset, l2->length); + + return pnfs_is_range_intersecting(l1->offset, end1, l2->offset, end2); +} + +static inline bool +pnfs_lseg_request_intersecting(struct pnfs_layout_segment *lseg, struct nfs_page *req) +{ + u64 seg_last = pnfs_end_offset(lseg->pls_range.offset, lseg->pls_range.length); + u64 req_last = req_offset(req) + req->wb_bytes; + + return pnfs_is_range_intersecting(lseg->pls_range.offset, seg_last, + req_offset(req), req_last); +} + +static inline void pnfs_lseg_cancel_io(struct nfs_server *server, + struct pnfs_layout_segment *lseg) +{ + if (server->pnfs_curr_ld->cancel_io) + server->pnfs_curr_ld->cancel_io(lseg); +} + +extern unsigned int layoutstats_timer; + +#ifdef NFS_DEBUG +void nfs4_print_deviceid(const struct nfs4_deviceid *dev_id); +#else +static inline void nfs4_print_deviceid(const struct nfs4_deviceid *dev_id) +{ +} + +#endif /* NFS_DEBUG */ +#else /* CONFIG_NFS_V4 */ + +static inline bool nfs_have_layout(struct inode *inode) +{ + return false; +} + +static inline void pnfs_destroy_all_layouts(struct nfs_client *clp) +{ +} + +static inline void pnfs_destroy_layout(struct nfs_inode *nfsi) +{ +} + +static inline void pnfs_destroy_layout_final(struct nfs_inode *nfsi) +{ +} + +static inline int pnfs_layout_handle_reboot(struct nfs_client *clp) +{ + return 0; +} + +static inline struct pnfs_layout_segment * +pnfs_get_lseg(struct pnfs_layout_segment *lseg) +{ + return NULL; +} + +static inline void pnfs_put_lseg(struct pnfs_layout_segment *lseg) +{ +} + +static inline int pnfs_return_layout(struct inode *ino) +{ + return 0; +} + +static inline int pnfs_commit_and_return_layout(struct inode *inode) +{ + return 0; +} + +static inline bool +pnfs_ld_layoutret_on_setattr(struct inode *inode) +{ + return false; +} + +static inline bool +pnfs_ld_read_whole_page(struct inode *inode) +{ + return false; +} + +static inline int +pnfs_sync_inode(struct inode *inode, bool datasync) +{ + return 0; +} + +static inline bool +pnfs_layoutcommit_outstanding(struct inode *inode) +{ + return false; +} + +static inline bool pnfs_roc(struct inode *ino, + struct nfs4_layoutreturn_args *args, + struct nfs4_layoutreturn_res *res, + const struct cred *cred, bool sync) +{ + return false; +} + +static inline int +pnfs_roc_done(struct rpc_task *task, + struct nfs4_layoutreturn_args **argpp, + struct nfs4_layoutreturn_res **respp, + int *ret) +{ + return 0; +} + +static inline void +pnfs_roc_release(struct nfs4_layoutreturn_args *args, + struct nfs4_layoutreturn_res *res, + int ret) +{ +} + +static inline bool +pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task) +{ + return false; +} + +static inline void set_pnfs_layoutdriver(struct nfs_server *s, + const struct nfs_fh *mntfh, + struct nfs_fsinfo *fsinfo) +{ +} + +static inline void unset_pnfs_layoutdriver(struct nfs_server *s) +{ +} + +static inline int +pnfs_commit_list(struct inode *inode, struct list_head *mds_pages, int how, + struct nfs_commit_info *cinfo) +{ + return PNFS_NOT_ATTEMPTED; +} + +static inline struct pnfs_ds_commit_info * +pnfs_get_ds_info(struct inode *inode) +{ + return NULL; +} + +static inline void +pnfs_init_ds_commit_info_ops(struct pnfs_ds_commit_info *fl_cinfo, struct inode *inode) +{ +} + +static inline void +pnfs_init_ds_commit_info(struct pnfs_ds_commit_info *fl_cinfo) +{ +} + +static inline void +pnfs_release_ds_info(struct pnfs_ds_commit_info *fl_cinfo, struct inode *inode) +{ +} + +static inline bool +pnfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg, + struct nfs_commit_info *cinfo, u32 ds_commit_idx) +{ + return false; +} + +static inline bool +pnfs_clear_request_commit(struct nfs_page *req, struct nfs_commit_info *cinfo) +{ + return false; +} + +static inline int +pnfs_scan_commit_lists(struct inode *inode, struct nfs_commit_info *cinfo, + int max) +{ + return 0; +} + +static inline void +pnfs_recover_commit_reqs(struct list_head *head, struct nfs_commit_info *cinfo) +{ +} + +static inline int pnfs_layoutcommit_inode(struct inode *inode, bool sync) +{ + return 0; +} + +static inline bool +pnfs_use_threshold(struct nfs4_threshold **dst, struct nfs4_threshold *src, + struct nfs_server *nfss) +{ + return false; +} + +static inline struct nfs4_threshold *pnfs_mdsthreshold_alloc(void) +{ + return NULL; +} + +static inline void nfs4_pnfs_v3_ds_connect_unload(void) +{ +} + +static inline bool nfs4_layout_refresh_old_stateid(nfs4_stateid *dst, + struct pnfs_layout_range *dst_range, + struct inode *inode) +{ + return false; +} + +static inline void pnfs_lgopen_prepare(struct nfs4_opendata *data, + struct nfs_open_context *ctx) +{ +} + +static inline void pnfs_parse_lgopen(struct inode *ino, + struct nfs4_layoutget *lgp, + struct nfs_open_context *ctx) +{ +} + +static inline void nfs4_lgopen_release(struct nfs4_layoutget *lgp) +{ +} + +static inline bool pnfs_layout_is_valid(const struct pnfs_layout_hdr *lo) +{ + return false; +} + +#endif /* CONFIG_NFS_V4 */ + +#if IS_ENABLED(CONFIG_NFS_V4_2) +int pnfs_report_layoutstat(struct inode *inode, gfp_t gfp_flags); +#else +static inline int +pnfs_report_layoutstat(struct inode *inode, gfp_t gfp_flags) +{ + return 0; +} +#endif + +#endif /* FS_NFS_PNFS_H */ diff --git a/fs/nfs/pnfs_dev.c b/fs/nfs/pnfs_dev.c new file mode 100644 index 000000000..274abdd6d --- /dev/null +++ b/fs/nfs/pnfs_dev.c @@ -0,0 +1,381 @@ +/* + * Device operations for the pnfs client. + * + * Copyright (c) 2002 + * The Regents of the University of Michigan + * All Rights Reserved + * + * Dean Hildebrand + * Garth Goodson + * + * Permission is granted to use, copy, create derivative works, and + * redistribute this software and such derivative works for any purpose, + * so long as the name of the University of Michigan is not used in + * any advertising or publicity pertaining to the use or distribution + * of this software without specific, written prior authorization. If + * the above copyright notice or any other identification of the + * University of Michigan is included in any copy of any portion of + * this software, then the disclaimer below must also be included. + * + * This software is provided as is, without representation or warranty + * of any kind either express or implied, including without limitation + * the implied warranties of merchantability, fitness for a particular + * purpose, or noninfringement. The Regents of the University of + * Michigan shall not be liable for any damages, including special, + * indirect, incidental, or consequential damages, with respect to any + * claim arising out of or in connection with the use of the software, + * even if it has been or is hereafter advised of the possibility of + * such damages. + */ + +#include +#include +#include "nfs4session.h" +#include "internal.h" +#include "pnfs.h" + +#include "nfs4trace.h" + +#define NFSDBG_FACILITY NFSDBG_PNFS + +/* + * Device ID RCU cache. A device ID is unique per server and layout type. + */ +#define NFS4_DEVICE_ID_HASH_BITS 5 +#define NFS4_DEVICE_ID_HASH_SIZE (1 << NFS4_DEVICE_ID_HASH_BITS) +#define NFS4_DEVICE_ID_HASH_MASK (NFS4_DEVICE_ID_HASH_SIZE - 1) + + +static struct hlist_head nfs4_deviceid_cache[NFS4_DEVICE_ID_HASH_SIZE]; +static DEFINE_SPINLOCK(nfs4_deviceid_lock); + +#ifdef NFS_DEBUG +void +nfs4_print_deviceid(const struct nfs4_deviceid *id) +{ + u32 *p = (u32 *)id; + + dprintk("%s: device id= [%x%x%x%x]\n", __func__, + p[0], p[1], p[2], p[3]); +} +EXPORT_SYMBOL_GPL(nfs4_print_deviceid); +#endif + +static inline u32 +nfs4_deviceid_hash(const struct nfs4_deviceid *id) +{ + unsigned char *cptr = (unsigned char *)id->data; + unsigned int nbytes = NFS4_DEVICEID4_SIZE; + u32 x = 0; + + while (nbytes--) { + x *= 37; + x += *cptr++; + } + return x & NFS4_DEVICE_ID_HASH_MASK; +} + +static struct nfs4_deviceid_node * +_lookup_deviceid(const struct pnfs_layoutdriver_type *ld, + const struct nfs_client *clp, const struct nfs4_deviceid *id, + long hash) +{ + struct nfs4_deviceid_node *d; + + hlist_for_each_entry_rcu(d, &nfs4_deviceid_cache[hash], node) + if (d->ld == ld && d->nfs_client == clp && + !memcmp(&d->deviceid, id, sizeof(*id))) { + if (atomic_read(&d->ref)) + return d; + else + continue; + } + return NULL; +} + +static struct nfs4_deviceid_node * +nfs4_get_device_info(struct nfs_server *server, + const struct nfs4_deviceid *dev_id, + const struct cred *cred, gfp_t gfp_flags) +{ + struct nfs4_deviceid_node *d = NULL; + struct pnfs_device *pdev = NULL; + struct page **pages = NULL; + u32 max_resp_sz; + int max_pages; + int rc, i; + + /* + * Use the session max response size as the basis for setting + * GETDEVICEINFO's maxcount + */ + max_resp_sz = server->nfs_client->cl_session->fc_attrs.max_resp_sz; + max_pages = nfs_page_array_len(0, max_resp_sz); + dprintk("%s: server %p max_resp_sz %u max_pages %d\n", + __func__, server, max_resp_sz, max_pages); + + pdev = kzalloc_obj(*pdev, gfp_flags); + if (!pdev) + return NULL; + + pages = kzalloc_objs(struct page *, max_pages, gfp_flags); + if (!pages) + goto out_free_pdev; + + for (i = 0; i < max_pages; i++) { + pages[i] = alloc_page(gfp_flags); + if (!pages[i]) + goto out_free_pages; + } + + memcpy(&pdev->dev_id, dev_id, sizeof(*dev_id)); + pdev->layout_type = server->pnfs_curr_ld->id; + pdev->pages = pages; + pdev->pgbase = 0; + pdev->pglen = max_resp_sz; + pdev->mincount = 0; + pdev->maxcount = max_resp_sz - nfs41_maxgetdevinfo_overhead; + + rc = nfs4_proc_getdeviceinfo(server, pdev, cred); + dprintk("%s getdevice info returns %d\n", __func__, rc); + if (rc) + goto out_free_pages; + + /* + * Found new device, need to decode it and then add it to the + * list of known devices for this mountpoint. + */ + d = server->pnfs_curr_ld->alloc_deviceid_node(server, pdev, + gfp_flags); + if (d && pdev->nocache) + set_bit(NFS_DEVICEID_NOCACHE, &d->flags); + +out_free_pages: + while (--i >= 0) + __free_page(pages[i]); + kfree(pages); +out_free_pdev: + kfree(pdev); + dprintk("<-- %s d %p\n", __func__, d); + return d; +} + +/* + * Lookup a deviceid in cache and get a reference count on it if found + * + * @clp nfs_client associated with deviceid + * @id deviceid to look up + */ +static struct nfs4_deviceid_node * +__nfs4_find_get_deviceid(struct nfs_server *server, + const struct nfs4_deviceid *id, long hash) +{ + struct nfs4_deviceid_node *d; + + rcu_read_lock(); + d = _lookup_deviceid(server->pnfs_curr_ld, server->nfs_client, id, + hash); + if (d != NULL && !atomic_inc_not_zero(&d->ref)) + d = NULL; + rcu_read_unlock(); + return d; +} + +struct nfs4_deviceid_node * +nfs4_find_get_deviceid(struct nfs_server *server, + const struct nfs4_deviceid *id, const struct cred *cred, + gfp_t gfp_mask) +{ + long hash = nfs4_deviceid_hash(id); + struct nfs4_deviceid_node *d, *new; + + d = __nfs4_find_get_deviceid(server, id, hash); + if (d) + goto found; + + new = nfs4_get_device_info(server, id, cred, gfp_mask); + if (!new) { + trace_nfs4_find_deviceid(server, id, -ENOENT); + return new; + } + + spin_lock(&nfs4_deviceid_lock); + d = __nfs4_find_get_deviceid(server, id, hash); + if (d) { + spin_unlock(&nfs4_deviceid_lock); + server->pnfs_curr_ld->free_deviceid_node(new); + } else { + atomic_inc(&new->ref); + hlist_add_head_rcu(&new->node, &nfs4_deviceid_cache[hash]); + spin_unlock(&nfs4_deviceid_lock); + d = new; + } +found: + trace_nfs4_find_deviceid(server, id, 0); + return d; +} +EXPORT_SYMBOL_GPL(nfs4_find_get_deviceid); + +/* + * Remove a deviceid from cache + * + * @clp nfs_client associated with deviceid + * @id the deviceid to unhash + * + * @ret the unhashed node, if found and dereferenced to zero, NULL otherwise. + */ +void +nfs4_delete_deviceid(const struct pnfs_layoutdriver_type *ld, + const struct nfs_client *clp, const struct nfs4_deviceid *id) +{ + struct nfs4_deviceid_node *d; + + spin_lock(&nfs4_deviceid_lock); + rcu_read_lock(); + d = _lookup_deviceid(ld, clp, id, nfs4_deviceid_hash(id)); + rcu_read_unlock(); + if (!d) { + spin_unlock(&nfs4_deviceid_lock); + return; + } + hlist_del_init_rcu(&d->node); + clear_bit(NFS_DEVICEID_NOCACHE, &d->flags); + spin_unlock(&nfs4_deviceid_lock); + + /* balance the initial ref set in pnfs_insert_deviceid */ + nfs4_put_deviceid_node(d); +} +EXPORT_SYMBOL_GPL(nfs4_delete_deviceid); + +void +nfs4_init_deviceid_node(struct nfs4_deviceid_node *d, struct nfs_server *server, + const struct nfs4_deviceid *id) +{ + INIT_HLIST_NODE(&d->node); + INIT_HLIST_NODE(&d->tmpnode); + d->ld = server->pnfs_curr_ld; + d->nfs_client = server->nfs_client; + d->flags = 0; + d->deviceid = *id; + atomic_set(&d->ref, 1); +} +EXPORT_SYMBOL_GPL(nfs4_init_deviceid_node); + +/* + * Dereference a deviceid node and delete it when its reference count drops + * to zero. + * + * @d deviceid node to put + * + * return true iff the node was deleted + * Note that since the test for d->ref == 0 is sufficient to establish + * that the node is no longer hashed in the global device id cache. + */ +bool +nfs4_put_deviceid_node(struct nfs4_deviceid_node *d) +{ + if (test_bit(NFS_DEVICEID_NOCACHE, &d->flags)) { + if (atomic_add_unless(&d->ref, -1, 2)) + return false; + nfs4_delete_deviceid(d->ld, d->nfs_client, &d->deviceid); + } + if (!atomic_dec_and_test(&d->ref)) + return false; + trace_nfs4_deviceid_free(d->nfs_client, &d->deviceid); + d->ld->free_deviceid_node(d); + return true; +} +EXPORT_SYMBOL_GPL(nfs4_put_deviceid_node); + +void +nfs4_mark_deviceid_available(struct nfs4_deviceid_node *node) +{ + if (test_bit(NFS_DEVICEID_UNAVAILABLE, &node->flags)) { + clear_bit(NFS_DEVICEID_UNAVAILABLE, &node->flags); + smp_mb__after_atomic(); + } +} +EXPORT_SYMBOL_GPL(nfs4_mark_deviceid_available); + +void +nfs4_mark_deviceid_unavailable(struct nfs4_deviceid_node *node) +{ + node->timestamp_unavailable = jiffies; + smp_mb__before_atomic(); + set_bit(NFS_DEVICEID_UNAVAILABLE, &node->flags); + smp_mb__after_atomic(); +} +EXPORT_SYMBOL_GPL(nfs4_mark_deviceid_unavailable); + +bool +nfs4_test_deviceid_unavailable(struct nfs4_deviceid_node *node) +{ + if (test_bit(NFS_DEVICEID_UNAVAILABLE, &node->flags)) { + unsigned long start, end; + + end = jiffies; + start = end - PNFS_DEVICE_RETRY_TIMEOUT; + if (time_in_range(node->timestamp_unavailable, start, end)) + return true; + clear_bit(NFS_DEVICEID_UNAVAILABLE, &node->flags); + smp_mb__after_atomic(); + } + return false; +} +EXPORT_SYMBOL_GPL(nfs4_test_deviceid_unavailable); + +static void +_deviceid_purge_client(const struct nfs_client *clp, long hash) +{ + struct nfs4_deviceid_node *d; + HLIST_HEAD(tmp); + + spin_lock(&nfs4_deviceid_lock); + rcu_read_lock(); + hlist_for_each_entry_rcu(d, &nfs4_deviceid_cache[hash], node) + if (d->nfs_client == clp && atomic_read(&d->ref)) { + hlist_del_init_rcu(&d->node); + hlist_add_head(&d->tmpnode, &tmp); + clear_bit(NFS_DEVICEID_NOCACHE, &d->flags); + } + rcu_read_unlock(); + spin_unlock(&nfs4_deviceid_lock); + + if (hlist_empty(&tmp)) + return; + + while (!hlist_empty(&tmp)) { + d = hlist_entry(tmp.first, struct nfs4_deviceid_node, tmpnode); + hlist_del(&d->tmpnode); + nfs4_put_deviceid_node(d); + } +} + +void +nfs4_deviceid_purge_client(const struct nfs_client *clp) +{ + long h; + + if (!(clp->cl_exchange_flags & EXCHGID4_FLAG_USE_PNFS_MDS)) + return; + for (h = 0; h < NFS4_DEVICE_ID_HASH_SIZE; h++) + _deviceid_purge_client(clp, h); +} + +/* + * Stop use of all deviceids associated with an nfs_client + */ +void +nfs4_deviceid_mark_client_invalid(struct nfs_client *clp) +{ + struct nfs4_deviceid_node *d; + int i; + + rcu_read_lock(); + for (i = 0; i < NFS4_DEVICE_ID_HASH_SIZE; i ++){ + hlist_for_each_entry_rcu(d, &nfs4_deviceid_cache[i], node) + if (d->nfs_client == clp) + set_bit(NFS_DEVICEID_INVALID, &d->flags); + } + rcu_read_unlock(); +} diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c new file mode 100644 index 000000000..648c95b78 --- /dev/null +++ b/fs/nfs/pnfs_nfs.c @@ -0,0 +1,1225 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Common NFS I/O operations for the pnfs file based + * layout drivers. + * + * Copyright (c) 2014, Primary Data, Inc. All rights reserved. + * + * Tom Haynes + */ + +#include +#include +#include +#include + +#include "nfs4session.h" +#include "internal.h" +#include "pnfs.h" +#include "netns.h" +#include "nfs4trace.h" + +#define NFSDBG_FACILITY NFSDBG_PNFS + +void pnfs_generic_rw_release(void *data) +{ + struct nfs_pgio_header *hdr = data; + + nfs_put_client(hdr->ds_clp); + hdr->mds_ops->rpc_release(data); +} +EXPORT_SYMBOL_GPL(pnfs_generic_rw_release); + +/* Fake up some data that will cause nfs_commit_release to retry the writes. */ +void pnfs_generic_prepare_to_resend_writes(struct nfs_commit_data *data) +{ + struct nfs_writeverf *verf = data->res.verf; + + data->task.tk_status = 0; + memset(&verf->verifier, 0, sizeof(verf->verifier)); + verf->committed = NFS_UNSTABLE; +} +EXPORT_SYMBOL_GPL(pnfs_generic_prepare_to_resend_writes); + +void pnfs_generic_write_commit_done(struct rpc_task *task, void *data) +{ + struct nfs_commit_data *wdata = data; + + /* Note this may cause RPC to be resent */ + wdata->mds_ops->rpc_call_done(task, data); +} +EXPORT_SYMBOL_GPL(pnfs_generic_write_commit_done); + +void pnfs_generic_commit_release(void *calldata) +{ + struct nfs_commit_data *data = calldata; + + data->completion_ops->completion(data); + pnfs_put_lseg(data->lseg); + nfs_put_client(data->ds_clp); + nfs_commitdata_release(data); +} +EXPORT_SYMBOL_GPL(pnfs_generic_commit_release); + +static struct pnfs_layout_segment * +pnfs_free_bucket_lseg(struct pnfs_commit_bucket *bucket) +{ + if (list_empty(&bucket->committing) && list_empty(&bucket->written)) { + struct pnfs_layout_segment *freeme = bucket->lseg; + bucket->lseg = NULL; + return freeme; + } + return NULL; +} + +/* The generic layer is about to remove the req from the commit list. + * If this will make the bucket empty, it will need to put the lseg reference. + * Note this must be called holding nfsi->commit_mutex + */ +void +pnfs_generic_clear_request_commit(struct nfs_page *req, + struct nfs_commit_info *cinfo) +{ + struct pnfs_commit_bucket *bucket = NULL; + + if (!test_and_clear_bit(PG_COMMIT_TO_DS, &req->wb_flags)) + goto out; + cinfo->ds->nwritten--; + if (list_is_singular(&req->wb_list)) + bucket = list_first_entry(&req->wb_list, + struct pnfs_commit_bucket, written); +out: + nfs_request_remove_commit_list(req, cinfo); + if (bucket) + pnfs_put_lseg(pnfs_free_bucket_lseg(bucket)); +} +EXPORT_SYMBOL_GPL(pnfs_generic_clear_request_commit); + +struct pnfs_commit_array * +pnfs_alloc_commit_array(size_t n, gfp_t gfp_flags) +{ + struct pnfs_commit_array *p; + struct pnfs_commit_bucket *b; + + p = kmalloc_flex(*p, buckets, n, gfp_flags); + if (!p) + return NULL; + p->nbuckets = n; + INIT_LIST_HEAD(&p->cinfo_list); + INIT_LIST_HEAD(&p->lseg_list); + p->lseg = NULL; + for (b = &p->buckets[0]; n != 0; b++, n--) { + INIT_LIST_HEAD(&b->written); + INIT_LIST_HEAD(&b->committing); + b->lseg = NULL; + b->direct_verf.committed = NFS_INVALID_STABLE_HOW; + } + return p; +} +EXPORT_SYMBOL_GPL(pnfs_alloc_commit_array); + +void +pnfs_free_commit_array(struct pnfs_commit_array *p) +{ + kfree_rcu(p, rcu); +} +EXPORT_SYMBOL_GPL(pnfs_free_commit_array); + +static struct pnfs_commit_array * +pnfs_find_commit_array_by_lseg(struct pnfs_ds_commit_info *fl_cinfo, + struct pnfs_layout_segment *lseg) +{ + struct pnfs_commit_array *array; + + list_for_each_entry_rcu(array, &fl_cinfo->commits, cinfo_list) { + if (array->lseg == lseg) + return array; + } + return NULL; +} + +struct pnfs_commit_array * +pnfs_add_commit_array(struct pnfs_ds_commit_info *fl_cinfo, + struct pnfs_commit_array *new, + struct pnfs_layout_segment *lseg) +{ + struct pnfs_commit_array *array; + + array = pnfs_find_commit_array_by_lseg(fl_cinfo, lseg); + if (array) + return array; + new->lseg = lseg; + refcount_set(&new->refcount, 1); + list_add_rcu(&new->cinfo_list, &fl_cinfo->commits); + list_add(&new->lseg_list, &lseg->pls_commits); + return new; +} +EXPORT_SYMBOL_GPL(pnfs_add_commit_array); + +static struct pnfs_commit_array * +pnfs_lookup_commit_array(struct pnfs_ds_commit_info *fl_cinfo, + struct pnfs_layout_segment *lseg) +{ + struct pnfs_commit_array *array; + + rcu_read_lock(); + array = pnfs_find_commit_array_by_lseg(fl_cinfo, lseg); + if (!array) { + rcu_read_unlock(); + fl_cinfo->ops->setup_ds_info(fl_cinfo, lseg); + rcu_read_lock(); + array = pnfs_find_commit_array_by_lseg(fl_cinfo, lseg); + } + rcu_read_unlock(); + return array; +} + +static void +pnfs_release_commit_array_locked(struct pnfs_commit_array *array) +{ + list_del_rcu(&array->cinfo_list); + list_del(&array->lseg_list); + pnfs_free_commit_array(array); +} + +static void +pnfs_put_commit_array_locked(struct pnfs_commit_array *array) +{ + if (refcount_dec_and_test(&array->refcount)) + pnfs_release_commit_array_locked(array); +} + +static void +pnfs_put_commit_array(struct pnfs_commit_array *array, struct inode *inode) +{ + if (refcount_dec_and_lock(&array->refcount, &inode->i_lock)) { + pnfs_release_commit_array_locked(array); + spin_unlock(&inode->i_lock); + } +} + +static struct pnfs_commit_array * +pnfs_get_commit_array(struct pnfs_commit_array *array) +{ + if (refcount_inc_not_zero(&array->refcount)) + return array; + return NULL; +} + +static void +pnfs_remove_and_free_commit_array(struct pnfs_commit_array *array) +{ + array->lseg = NULL; + list_del_init(&array->lseg_list); + pnfs_put_commit_array_locked(array); +} + +void +pnfs_generic_ds_cinfo_release_lseg(struct pnfs_ds_commit_info *fl_cinfo, + struct pnfs_layout_segment *lseg) +{ + struct pnfs_commit_array *array, *tmp; + + list_for_each_entry_safe(array, tmp, &lseg->pls_commits, lseg_list) + pnfs_remove_and_free_commit_array(array); +} +EXPORT_SYMBOL_GPL(pnfs_generic_ds_cinfo_release_lseg); + +void +pnfs_generic_ds_cinfo_destroy(struct pnfs_ds_commit_info *fl_cinfo) +{ + struct pnfs_commit_array *array, *tmp; + + list_for_each_entry_safe(array, tmp, &fl_cinfo->commits, cinfo_list) + pnfs_remove_and_free_commit_array(array); +} +EXPORT_SYMBOL_GPL(pnfs_generic_ds_cinfo_destroy); + +/* + * Locks the nfs_page requests for commit and moves them to + * @bucket->committing. + */ +static int +pnfs_bucket_scan_ds_commit_list(struct pnfs_commit_bucket *bucket, + struct nfs_commit_info *cinfo, + int max) +{ + struct list_head *src = &bucket->written; + struct list_head *dst = &bucket->committing; + int ret; + + lockdep_assert_held(&NFS_I(cinfo->inode)->commit_mutex); + ret = nfs_scan_commit_list(src, dst, cinfo, max); + if (ret) { + cinfo->ds->nwritten -= ret; + cinfo->ds->ncommitting += ret; + } + return ret; +} + +static int pnfs_bucket_scan_array(struct nfs_commit_info *cinfo, + struct pnfs_commit_bucket *buckets, + unsigned int nbuckets, + int max) +{ + unsigned int i; + int rv = 0, cnt; + + for (i = 0; i < nbuckets && max != 0; i++) { + cnt = pnfs_bucket_scan_ds_commit_list(&buckets[i], cinfo, max); + rv += cnt; + max -= cnt; + } + return rv; +} + +/* Move reqs from written to committing lists, returning count + * of number moved. + */ +int pnfs_generic_scan_commit_lists(struct nfs_commit_info *cinfo, int max) +{ + struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds; + struct pnfs_commit_array *array; + int rv = 0, cnt; + + rcu_read_lock(); + list_for_each_entry_rcu(array, &fl_cinfo->commits, cinfo_list) { + if (!array->lseg || !pnfs_get_commit_array(array)) + continue; + rcu_read_unlock(); + cnt = pnfs_bucket_scan_array(cinfo, array->buckets, + array->nbuckets, max); + rcu_read_lock(); + pnfs_put_commit_array(array, cinfo->inode); + rv += cnt; + max -= cnt; + if (!max) + break; + } + rcu_read_unlock(); + return rv; +} +EXPORT_SYMBOL_GPL(pnfs_generic_scan_commit_lists); + +static unsigned int +pnfs_bucket_recover_commit_reqs(struct list_head *dst, + struct pnfs_commit_bucket *buckets, + unsigned int nbuckets, + struct nfs_commit_info *cinfo) +{ + struct pnfs_commit_bucket *b; + struct pnfs_layout_segment *freeme; + unsigned int nwritten, ret = 0; + unsigned int i; + +restart: + for (i = 0, b = buckets; i < nbuckets; i++, b++) { + nwritten = nfs_scan_commit_list(&b->written, dst, cinfo, 0); + if (!nwritten) + continue; + ret += nwritten; + freeme = pnfs_free_bucket_lseg(b); + if (freeme) { + pnfs_put_lseg(freeme); + goto restart; + } + } + return ret; +} + +/* Pull everything off the committing lists and dump into @dst. */ +void pnfs_generic_recover_commit_reqs(struct list_head *dst, + struct nfs_commit_info *cinfo) +{ + struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds; + struct pnfs_commit_array *array; + unsigned int nwritten; + + lockdep_assert_held(&NFS_I(cinfo->inode)->commit_mutex); + rcu_read_lock(); + list_for_each_entry_rcu(array, &fl_cinfo->commits, cinfo_list) { + if (!array->lseg || !pnfs_get_commit_array(array)) + continue; + rcu_read_unlock(); + nwritten = pnfs_bucket_recover_commit_reqs(dst, + array->buckets, + array->nbuckets, + cinfo); + rcu_read_lock(); + pnfs_put_commit_array(array, cinfo->inode); + fl_cinfo->nwritten -= nwritten; + } + rcu_read_unlock(); +} +EXPORT_SYMBOL_GPL(pnfs_generic_recover_commit_reqs); + +static struct pnfs_layout_segment * +pnfs_bucket_get_committing(struct list_head *head, + struct pnfs_commit_bucket *bucket, + struct nfs_commit_info *cinfo) +{ + struct pnfs_layout_segment *lseg; + struct list_head *pos; + + list_for_each(pos, &bucket->committing) + cinfo->ds->ncommitting--; + list_splice_init(&bucket->committing, head); + lseg = pnfs_free_bucket_lseg(bucket); + if (!lseg) + lseg = pnfs_get_lseg(bucket->lseg); + return lseg; +} + +static struct nfs_commit_data * +pnfs_bucket_fetch_commitdata(struct pnfs_commit_bucket *bucket, + struct nfs_commit_info *cinfo) +{ + struct nfs_commit_data *data = nfs_commitdata_alloc(); + + if (!data) + return NULL; + data->lseg = pnfs_bucket_get_committing(&data->pages, bucket, cinfo); + return data; +} + +static void pnfs_generic_retry_commit(struct pnfs_commit_bucket *buckets, + unsigned int nbuckets, + struct nfs_commit_info *cinfo, + unsigned int idx) +{ + struct pnfs_commit_bucket *bucket; + struct pnfs_layout_segment *freeme; + LIST_HEAD(pages); + + for (bucket = buckets; idx < nbuckets; bucket++, idx++) { + if (list_empty(&bucket->committing)) + continue; + mutex_lock(&NFS_I(cinfo->inode)->commit_mutex); + freeme = pnfs_bucket_get_committing(&pages, bucket, cinfo); + mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); + nfs_retry_commit(&pages, freeme, cinfo, idx); + pnfs_put_lseg(freeme); + } +} + +static unsigned int +pnfs_bucket_alloc_ds_commits(struct list_head *list, + struct pnfs_commit_bucket *buckets, + unsigned int nbuckets, + struct nfs_commit_info *cinfo) +{ + struct pnfs_commit_bucket *bucket; + struct nfs_commit_data *data; + unsigned int i; + unsigned int nreq = 0; + + for (i = 0, bucket = buckets; i < nbuckets; i++, bucket++) { + if (list_empty(&bucket->committing)) + continue; + mutex_lock(&NFS_I(cinfo->inode)->commit_mutex); + if (!list_empty(&bucket->committing)) { + data = pnfs_bucket_fetch_commitdata(bucket, cinfo); + if (!data) + goto out_error; + data->ds_commit_index = i; + list_add_tail(&data->list, list); + nreq++; + } + mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); + } + return nreq; +out_error: + mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); + /* Clean up on error */ + pnfs_generic_retry_commit(buckets, nbuckets, cinfo, i); + return nreq; +} + +static unsigned int +pnfs_alloc_ds_commits_list(struct list_head *list, + struct pnfs_ds_commit_info *fl_cinfo, + struct nfs_commit_info *cinfo) +{ + struct pnfs_commit_array *array; + unsigned int ret = 0; + + rcu_read_lock(); + list_for_each_entry_rcu(array, &fl_cinfo->commits, cinfo_list) { + if (!array->lseg || !pnfs_get_commit_array(array)) + continue; + rcu_read_unlock(); + ret += pnfs_bucket_alloc_ds_commits(list, array->buckets, + array->nbuckets, cinfo); + rcu_read_lock(); + pnfs_put_commit_array(array, cinfo->inode); + } + rcu_read_unlock(); + return ret; +} + +/* This follows nfs_commit_list pretty closely */ +int +pnfs_generic_commit_pagelist(struct inode *inode, struct list_head *mds_pages, + int how, struct nfs_commit_info *cinfo, + int (*initiate_commit)(struct nfs_commit_data *data, + int how)) +{ + struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds; + struct nfs_commit_data *data, *tmp; + LIST_HEAD(list); + unsigned int nreq = 0; + + if (!list_empty(mds_pages)) { + data = nfs_commitdata_alloc(); + if (!data) { + nfs_retry_commit(mds_pages, NULL, cinfo, -1); + return -ENOMEM; + } + data->ds_commit_index = -1; + list_splice_init(mds_pages, &data->pages); + list_add_tail(&data->list, &list); + nreq++; + } + + nreq += pnfs_alloc_ds_commits_list(&list, fl_cinfo, cinfo); + if (nreq == 0) + goto out; + + list_for_each_entry_safe(data, tmp, &list, list) { + list_del(&data->list); + if (data->ds_commit_index < 0) { + nfs_init_commit(data, NULL, NULL, cinfo); + nfs_initiate_commit(NFS_CLIENT(inode), data, + NFS_PROTO(data->inode), + data->mds_ops, how, + RPC_TASK_CRED_NOREF, NULL); + } else { + nfs_init_commit(data, NULL, data->lseg, cinfo); + initiate_commit(data, how); + } + } +out: + return PNFS_ATTEMPTED; +} +EXPORT_SYMBOL_GPL(pnfs_generic_commit_pagelist); + +/* + * Data server cache + * + * Data servers can be mapped to different device ids, but should + * never be shared between net namespaces. + * + * nfs4_pnfs_ds reference counting: + * - set to 1 on allocation + * - incremented when a device id maps a data server already in the cache. + * - decremented when deviceid is removed from the cache. + */ + +/* Debug routines */ +static void +print_ds(struct nfs4_pnfs_ds *ds) +{ + if (ds == NULL) { + printk(KERN_WARNING "%s NULL device\n", __func__); + return; + } + printk(KERN_WARNING " ds %s\n" + " ref count %d\n" + " client %p\n" + " cl_exchange_flags %x\n", + ds->ds_remotestr, + refcount_read(&ds->ds_count), ds->ds_clp, + ds->ds_clp ? ds->ds_clp->cl_exchange_flags : 0); +} + +static bool +same_sockaddr(struct sockaddr *addr1, struct sockaddr *addr2) +{ + struct sockaddr_in *a, *b; + struct sockaddr_in6 *a6, *b6; + + if (addr1->sa_family != addr2->sa_family) + return false; + + switch (addr1->sa_family) { + case AF_INET: + a = (struct sockaddr_in *)addr1; + b = (struct sockaddr_in *)addr2; + + if (a->sin_addr.s_addr == b->sin_addr.s_addr && + a->sin_port == b->sin_port) + return true; + break; + + case AF_INET6: + a6 = (struct sockaddr_in6 *)addr1; + b6 = (struct sockaddr_in6 *)addr2; + + /* LINKLOCAL addresses must have matching scope_id */ + if (ipv6_addr_src_scope(&a6->sin6_addr) == + IPV6_ADDR_SCOPE_LINKLOCAL && + a6->sin6_scope_id != b6->sin6_scope_id) + return false; + + if (ipv6_addr_equal(&a6->sin6_addr, &b6->sin6_addr) && + a6->sin6_port == b6->sin6_port) + return true; + break; + + default: + dprintk("%s: unhandled address family: %u\n", + __func__, addr1->sa_family); + return false; + } + + return false; +} + +/* + * Checks if 'dsaddrs1' contains a subset of 'dsaddrs2'. If it does, + * declare a match. + */ +static bool +_same_data_server_addrs_locked(const struct list_head *dsaddrs1, + const struct list_head *dsaddrs2) +{ + struct nfs4_pnfs_ds_addr *da1, *da2; + struct sockaddr *sa1, *sa2; + bool match = false; + + list_for_each_entry(da1, dsaddrs1, da_node) { + sa1 = (struct sockaddr *)&da1->da_addr; + match = false; + list_for_each_entry(da2, dsaddrs2, da_node) { + sa2 = (struct sockaddr *)&da2->da_addr; + match = same_sockaddr(sa1, sa2); + if (match) + break; + } + if (!match) + break; + } + return match; +} + +/* + * Lookup DS by addresses. nfs4_ds_cache_lock is held + */ +static struct nfs4_pnfs_ds * +_data_server_lookup_locked(const struct nfs_net *nn, const struct list_head *dsaddrs) +{ + struct nfs4_pnfs_ds *ds; + + list_for_each_entry(ds, &nn->nfs4_data_server_cache, ds_node) + if (_same_data_server_addrs_locked(&ds->ds_addrs, dsaddrs)) + return ds; + return NULL; +} + +static struct nfs4_pnfs_ds_addr *nfs4_pnfs_ds_addr_alloc(gfp_t gfp_flags) +{ + struct nfs4_pnfs_ds_addr *da = kzalloc_obj(*da, gfp_flags); + if (da) + INIT_LIST_HEAD(&da->da_node); + return da; +} + +static void nfs4_pnfs_ds_addr_free(struct nfs4_pnfs_ds_addr *da) +{ + kfree(da->da_remotestr); + kfree(da->da_netid); + kfree(da); +} + +static void destroy_ds(struct nfs4_pnfs_ds *ds) +{ + struct nfs4_pnfs_ds_addr *da; + + dprintk("--> %s\n", __func__); + ifdebug(FACILITY) + print_ds(ds); + + nfs_put_client(ds->ds_clp); + + while (!list_empty(&ds->ds_addrs)) { + da = list_first_entry(&ds->ds_addrs, + struct nfs4_pnfs_ds_addr, + da_node); + list_del_init(&da->da_node); + nfs4_pnfs_ds_addr_free(da); + } + + kfree(ds->ds_remotestr); + kfree(ds); +} + +void nfs4_pnfs_ds_put(struct nfs4_pnfs_ds *ds) +{ + struct nfs_net *nn = net_generic(ds->ds_net, nfs_net_id); + + if (refcount_dec_and_lock(&ds->ds_count, &nn->nfs4_data_server_lock)) { + list_del_init(&ds->ds_node); + spin_unlock(&nn->nfs4_data_server_lock); + destroy_ds(ds); + } +} +EXPORT_SYMBOL_GPL(nfs4_pnfs_ds_put); + +/* + * Create a string with a human readable address and port to avoid + * complicated setup around many dprinks. + */ +static char * +nfs4_pnfs_remotestr(struct list_head *dsaddrs, gfp_t gfp_flags) +{ + struct nfs4_pnfs_ds_addr *da; + char *remotestr; + size_t len; + char *p; + + len = 3; /* '{', '}' and eol */ + list_for_each_entry(da, dsaddrs, da_node) { + len += strlen(da->da_remotestr) + 1; /* string plus comma */ + } + + remotestr = kzalloc(len, gfp_flags); + if (!remotestr) + return NULL; + + p = remotestr; + *(p++) = '{'; + len--; + list_for_each_entry(da, dsaddrs, da_node) { + size_t ll = strlen(da->da_remotestr); + + if (ll > len) + goto out_err; + + memcpy(p, da->da_remotestr, ll); + p += ll; + len -= ll; + + if (len < 1) + goto out_err; + (*p++) = ','; + len--; + } + if (len < 2) + goto out_err; + *(p++) = '}'; + *p = '\0'; + return remotestr; +out_err: + kfree(remotestr); + return NULL; +} + +/* + * Given a list of multipath struct nfs4_pnfs_ds_addr, add it to ds cache if + * uncached and return cached struct nfs4_pnfs_ds. + */ +struct nfs4_pnfs_ds * +nfs4_pnfs_ds_add(const struct net *net, struct list_head *dsaddrs, gfp_t gfp_flags) +{ + struct nfs_net *nn = net_generic(net, nfs_net_id); + struct nfs4_pnfs_ds *tmp_ds, *ds = NULL; + char *remotestr; + + if (list_empty(dsaddrs)) { + dprintk("%s: no addresses defined\n", __func__); + goto out; + } + + ds = kzalloc_obj(*ds, gfp_flags); + if (!ds) + goto out; + + /* this is only used for debugging, so it's ok if its NULL */ + remotestr = nfs4_pnfs_remotestr(dsaddrs, gfp_flags); + + spin_lock(&nn->nfs4_data_server_lock); + tmp_ds = _data_server_lookup_locked(nn, dsaddrs); + if (tmp_ds == NULL) { + INIT_LIST_HEAD(&ds->ds_addrs); + list_splice_init(dsaddrs, &ds->ds_addrs); + ds->ds_remotestr = remotestr; + refcount_set(&ds->ds_count, 1); + INIT_LIST_HEAD(&ds->ds_node); + ds->ds_net = net; + ds->ds_clp = NULL; + list_add(&ds->ds_node, &nn->nfs4_data_server_cache); + dprintk("%s add new data server %s\n", __func__, + ds->ds_remotestr); + } else { + kfree(remotestr); + kfree(ds); + refcount_inc(&tmp_ds->ds_count); + dprintk("%s data server %s found, inc'ed ds_count to %d\n", + __func__, tmp_ds->ds_remotestr, + refcount_read(&tmp_ds->ds_count)); + ds = tmp_ds; + } + spin_unlock(&nn->nfs4_data_server_lock); +out: + return ds; +} +EXPORT_SYMBOL_GPL(nfs4_pnfs_ds_add); + +static int nfs4_wait_ds_connect(struct nfs4_pnfs_ds *ds) +{ + might_sleep(); + return wait_on_bit(&ds->ds_state, NFS4DS_CONNECTING, TASK_KILLABLE); +} + +static void nfs4_clear_ds_conn_bit(struct nfs4_pnfs_ds *ds) +{ + smp_mb__before_atomic(); + clear_and_wake_up_bit(NFS4DS_CONNECTING, &ds->ds_state); +} + +static struct nfs_client *(*get_v3_ds_connect)( + struct nfs_server *mds_srv, + const struct sockaddr_storage *ds_addr, + int ds_addrlen, + int ds_proto, + unsigned int ds_timeo, + unsigned int ds_retrans); + +static bool load_v3_ds_connect(void) +{ + if (!get_v3_ds_connect) { + get_v3_ds_connect = symbol_request(nfs3_set_ds_client); + WARN_ON_ONCE(!get_v3_ds_connect); + } + + return(get_v3_ds_connect != NULL); +} + +void nfs4_pnfs_v3_ds_connect_unload(void) +{ + if (get_v3_ds_connect) { + symbol_put(nfs3_set_ds_client); + get_v3_ds_connect = NULL; + } +} + +static int _nfs4_pnfs_v3_ds_connect(struct nfs_server *mds_srv, + struct nfs4_pnfs_ds *ds, + unsigned int timeo, + unsigned int retrans) +{ + struct nfs_client *clp = ERR_PTR(-EIO); + struct nfs_client *mds_clp = mds_srv->nfs_client; + enum xprtsec_policies xprtsec_policy = mds_clp->cl_xprtsec.policy; + struct nfs4_pnfs_ds_addr *da; + unsigned long connect_timeout = timeo * (retrans + 1) * HZ / 10; + int ds_proto; + int status = 0; + + dprintk("--> %s DS %s\n", __func__, ds->ds_remotestr); + + if (!load_v3_ds_connect()) + return -EPROTONOSUPPORT; + + list_for_each_entry(da, &ds->ds_addrs, da_node) { + dprintk("%s: DS %s: trying address %s\n", + __func__, ds->ds_remotestr, da->da_remotestr); + + if (!IS_ERR(clp)) { + struct xprt_create xprt_args = { + .ident = da->da_transport, + .net = clp->cl_net, + .dstaddr = (struct sockaddr *)&da->da_addr, + .addrlen = da->da_addrlen, + .servername = clp->cl_hostname, + .connect_timeout = connect_timeout, + .reconnect_timeout = connect_timeout, + .xprtsec = clp->cl_xprtsec, + }; + + if (xprt_args.ident == XPRT_TRANSPORT_TCP && + clp->cl_proto == XPRT_TRANSPORT_TCP_TLS) + xprt_args.ident = XPRT_TRANSPORT_TCP_TLS; + + if (xprt_args.ident != clp->cl_proto) + continue; + if (xprt_args.dstaddr->sa_family != + clp->cl_addr.ss_family) + continue; + /* Add this address as an alias */ + rpc_clnt_add_xprt(clp->cl_rpcclient, &xprt_args, + rpc_clnt_test_and_add_xprt, NULL); + continue; + } + + ds_proto = da->da_transport; + if (ds_proto == XPRT_TRANSPORT_TCP && + xprtsec_policy != RPC_XPRTSEC_NONE) + ds_proto = XPRT_TRANSPORT_TCP_TLS; + + clp = get_v3_ds_connect(mds_srv, &da->da_addr, da->da_addrlen, + ds_proto, timeo, retrans); + if (IS_ERR(clp)) + continue; + clp->cl_rpcclient->cl_softerr = 0; + clp->cl_rpcclient->cl_softrtry = 0; + } + + if (IS_ERR(clp)) { + status = PTR_ERR(clp); + goto out; + } + + smp_wmb(); + WRITE_ONCE(ds->ds_clp, clp); + dprintk("%s [new] addr: %s\n", __func__, ds->ds_remotestr); +out: + return status; +} + +static int _nfs4_pnfs_v4_ds_connect(struct nfs_server *mds_srv, + struct nfs4_pnfs_ds *ds, + unsigned int timeo, + unsigned int retrans, + u32 minor_version) +{ + struct nfs_client *clp = ERR_PTR(-EIO); + struct nfs_client *mds_clp = mds_srv->nfs_client; + enum xprtsec_policies xprtsec_policy = mds_clp->cl_xprtsec.policy; + struct nfs4_pnfs_ds_addr *da; + int ds_proto; + int status = 0; + + dprintk("--> %s DS %s\n", __func__, ds->ds_remotestr); + + list_for_each_entry(da, &ds->ds_addrs, da_node) { + char servername[48]; + + dprintk("%s: DS %s: trying address %s\n", + __func__, ds->ds_remotestr, da->da_remotestr); + + if (!IS_ERR(clp) && clp->cl_mvops->session_trunk) { + struct xprt_create xprt_args = { + .ident = da->da_transport, + .net = clp->cl_net, + .dstaddr = (struct sockaddr *)&da->da_addr, + .addrlen = da->da_addrlen, + .servername = clp->cl_hostname, + .xprtsec = clp->cl_xprtsec, + }; + struct nfs4_add_xprt_data xprtdata = { + .clp = clp, + }; + struct rpc_add_xprt_test rpcdata = { + .add_xprt_test = clp->cl_mvops->session_trunk, + .data = &xprtdata, + }; + + if (xprt_args.ident == XPRT_TRANSPORT_TCP && + clp->cl_proto == XPRT_TRANSPORT_TCP_TLS) { + struct sockaddr *addr = + (struct sockaddr *)&da->da_addr; + struct sockaddr_in *sin = + (struct sockaddr_in *)&da->da_addr; + struct sockaddr_in6 *sin6 = + (struct sockaddr_in6 *)&da->da_addr; + + /* for NFS with TLS we need to supply a correct + * servername of the trunked transport, not the + * servername of the main transport stored in + * clp->cl_hostname. And set the protocol to + * indicate to use TLS + */ + servername[0] = '\0'; + switch(addr->sa_family) { + case AF_INET: + snprintf(servername, sizeof(servername), + "%pI4", &sin->sin_addr.s_addr); + break; + case AF_INET6: + snprintf(servername, sizeof(servername), + "%pI6", &sin6->sin6_addr); + break; + default: + /* do not consider this address */ + continue; + } + xprt_args.ident = XPRT_TRANSPORT_TCP_TLS; + xprt_args.servername = servername; + } + if (xprt_args.ident != clp->cl_proto) + continue; + if (xprt_args.dstaddr->sa_family != + clp->cl_addr.ss_family) + continue; + + /** + * Test this address for session trunking and + * add as an alias + */ + xprtdata.cred = nfs4_get_clid_cred(clp); + rpc_clnt_add_xprt(clp->cl_rpcclient, &xprt_args, + rpc_clnt_setup_test_and_add_xprt, + &rpcdata); + if (xprtdata.cred) + put_cred(xprtdata.cred); + } else { + ds_proto = da->da_transport; + if (ds_proto == XPRT_TRANSPORT_TCP && + xprtsec_policy != RPC_XPRTSEC_NONE) + ds_proto = XPRT_TRANSPORT_TCP_TLS; + + clp = nfs4_set_ds_client(mds_srv, &da->da_addr, + da->da_addrlen, ds_proto, + timeo, retrans, minor_version); + if (IS_ERR(clp)) + continue; + + status = nfs4_init_ds_session(clp, + mds_srv->nfs_client->cl_lease_time); + if (status) { + nfs_put_client(clp); + clp = ERR_PTR(-EIO); + continue; + } + } + } + + if (IS_ERR(clp)) { + status = PTR_ERR(clp); + goto out; + } + + smp_wmb(); + WRITE_ONCE(ds->ds_clp, clp); + dprintk("%s [new] addr: %s\n", __func__, ds->ds_remotestr); +out: + return status; +} + +/* + * Create an rpc connection to the nfs4_pnfs_ds data server. + * Currently only supports IPv4 and IPv6 addresses. + * If connection fails, make devid unavailable and return a -errno. + */ +int nfs4_pnfs_ds_connect(struct nfs_server *mds_srv, struct nfs4_pnfs_ds *ds, + struct nfs4_deviceid_node *devid, unsigned int timeo, + unsigned int retrans, u32 version, u32 minor_version) +{ + int err; + + do { + err = nfs4_wait_ds_connect(ds); + if (err || ds->ds_clp) + goto out; + if (nfs4_test_deviceid_unavailable(devid)) { + err = -ENODEV; + goto out; + } + } while (test_and_set_bit(NFS4DS_CONNECTING, &ds->ds_state) != 0); + + if (ds->ds_clp) + goto connect_done; + + switch (version) { + case 3: + err = _nfs4_pnfs_v3_ds_connect(mds_srv, ds, timeo, retrans); + break; + case 4: + err = _nfs4_pnfs_v4_ds_connect(mds_srv, ds, timeo, retrans, + minor_version); + break; + default: + dprintk("%s: unsupported DS version %d\n", __func__, version); + err = -EPROTONOSUPPORT; + } + +connect_done: + nfs4_clear_ds_conn_bit(ds); +out: + /* + * At this point the ds->ds_clp should be ready, but it might have + * hit an error. + */ + if (!err) { + if (!ds->ds_clp || !nfs_client_init_is_complete(ds->ds_clp)) { + WARN_ON_ONCE(ds->ds_clp || + !nfs4_test_deviceid_unavailable(devid)); + err = -EINVAL; + } else + err = nfs_client_init_status(ds->ds_clp); + } + + trace_pnfs_ds_connect(ds->ds_remotestr, err); + return err; +} +EXPORT_SYMBOL_GPL(nfs4_pnfs_ds_connect); + +/* + * Currently only supports ipv4, ipv6 and one multi-path address. + */ +struct nfs4_pnfs_ds_addr * +nfs4_decode_mp_ds_addr(struct net *net, struct xdr_stream *xdr, gfp_t gfp_flags) +{ + struct nfs4_pnfs_ds_addr *da = NULL; + char *buf, *portstr; + __be16 port; + ssize_t nlen, rlen; + int tmp[2]; + char *netid; + size_t len; + char *startsep = ""; + char *endsep = ""; + + + /* r_netid */ + nlen = xdr_stream_decode_string_dup(xdr, &netid, XDR_MAX_NETOBJ, + gfp_flags); + if (unlikely(nlen <= 0)) + goto out_err; + + /* r_addr: ip/ip6addr with port in dec octets - see RFC 5665 */ + /* port is ".ABC.DEF", 8 chars max */ + rlen = xdr_stream_decode_string_dup(xdr, &buf, INET6_ADDRSTRLEN + + IPV6_SCOPE_ID_LEN + 8, gfp_flags); + if (unlikely(rlen <= 0)) + goto out_free_netid; + + /* replace port '.' with '-' */ + portstr = strrchr(buf, '.'); + if (!portstr) { + dprintk("%s: Failed finding expected dot in port\n", + __func__); + goto out_free_buf; + } + *portstr = '-'; + + /* find '.' between address and port */ + portstr = strrchr(buf, '.'); + if (!portstr) { + dprintk("%s: Failed finding expected dot between address and " + "port\n", __func__); + goto out_free_buf; + } + *portstr = '\0'; + + da = nfs4_pnfs_ds_addr_alloc(gfp_flags); + if (unlikely(!da)) + goto out_free_buf; + + if (!rpc_pton(net, buf, portstr-buf, (struct sockaddr *)&da->da_addr, + sizeof(da->da_addr))) { + dprintk("%s: error parsing address %s\n", __func__, buf); + goto out_free_da; + } + + portstr++; + sscanf(portstr, "%d-%d", &tmp[0], &tmp[1]); + port = htons((tmp[0] << 8) | (tmp[1])); + + switch (da->da_addr.ss_family) { + case AF_INET: + ((struct sockaddr_in *)&da->da_addr)->sin_port = port; + da->da_addrlen = sizeof(struct sockaddr_in); + break; + + case AF_INET6: + ((struct sockaddr_in6 *)&da->da_addr)->sin6_port = port; + da->da_addrlen = sizeof(struct sockaddr_in6); + startsep = "["; + endsep = "]"; + break; + + default: + dprintk("%s: unsupported address family: %u\n", + __func__, da->da_addr.ss_family); + goto out_free_da; + } + + da->da_transport = xprt_find_transport_ident(netid); + if (da->da_transport < 0) { + dprintk("%s: ERROR: unknown r_netid \"%s\"\n", + __func__, netid); + goto out_free_da; + } + + da->da_netid = netid; + + /* save human readable address */ + len = strlen(startsep) + strlen(buf) + strlen(endsep) + 7; + da->da_remotestr = kzalloc(len, gfp_flags); + + /* NULL is ok, only used for dprintk */ + if (da->da_remotestr) + snprintf(da->da_remotestr, len, "%s%s%s:%u", startsep, + buf, endsep, ntohs(port)); + + dprintk("%s: Parsed DS addr %s\n", __func__, da->da_remotestr); + kfree(buf); + return da; + +out_free_da: + kfree(da); +out_free_buf: + dprintk("%s: Error parsing DS addr: %s\n", __func__, buf); + kfree(buf); +out_free_netid: + kfree(netid); +out_err: + return NULL; +} +EXPORT_SYMBOL_GPL(nfs4_decode_mp_ds_addr); + +void +pnfs_layout_mark_request_commit(struct nfs_page *req, + struct pnfs_layout_segment *lseg, + struct nfs_commit_info *cinfo, + u32 ds_commit_idx) +{ + struct list_head *list; + struct pnfs_commit_array *array; + struct pnfs_commit_bucket *bucket; + + mutex_lock(&NFS_I(cinfo->inode)->commit_mutex); + array = pnfs_lookup_commit_array(cinfo->ds, lseg); + if (!array || !pnfs_is_valid_lseg(lseg)) + goto out_resched; + bucket = &array->buckets[ds_commit_idx]; + list = &bucket->written; + /* Non-empty buckets hold a reference on the lseg. That ref + * is normally transferred to the COMMIT call and released + * there. It could also be released if the last req is pulled + * off due to a rewrite, in which case it will be done in + * pnfs_common_clear_request_commit + */ + if (!bucket->lseg) + bucket->lseg = pnfs_get_lseg(lseg); + set_bit(PG_COMMIT_TO_DS, &req->wb_flags); + cinfo->ds->nwritten++; + + nfs_request_add_commit_list_locked(req, list, cinfo); + mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); + nfs_folio_mark_unstable(req, cinfo); + return; +out_resched: + mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); + cinfo->completion_ops->resched_write(cinfo, req); +} +EXPORT_SYMBOL_GPL(pnfs_layout_mark_request_commit); + +int +pnfs_nfs_generic_sync(struct inode *inode, bool datasync) +{ + int ret; + + if (!pnfs_layoutcommit_outstanding(inode)) + return 0; + ret = nfs_commit_inode(inode, FLUSH_SYNC); + if (ret < 0) + return ret; + if (datasync) + return 0; + return pnfs_layoutcommit_inode(inode, true); +} +EXPORT_SYMBOL_GPL(pnfs_nfs_generic_sync); + diff --git a/fs/nfs/proc.c b/fs/nfs/proc.c new file mode 100644 index 000000000..03c2c1f31 --- /dev/null +++ b/fs/nfs/proc.c @@ -0,0 +1,781 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/nfs/proc.c + * + * Copyright (C) 1992, 1993, 1994 Rick Sladkey + * + * OS-independent nfs remote procedure call functions + * + * Tuned by Alan Cox for >3K buffers + * so at last we can have decent(ish) throughput off a + * Sun server. + * + * Coding optimized and cleaned up by Florian La Roche. + * Note: Error returns are optimized for NFS_OK, which isn't translated via + * nfs_stat_to_errno(), but happens to be already the right return code. + * + * Also, the code currently doesn't check the size of the packet, when + * it decodes the packet. + * + * Feel free to fix it and mail me the diffs if it worries you. + * + * Completely rewritten to support the new RPC call interface; + * rewrote and moved the entire XDR stuff to xdr.c + * --Olaf Kirch June 1996 + * + * The code below initializes all auto variables explicitly, otherwise + * it will fail to work as a module (gcc generates a memset call for an + * incomplete struct). + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +#define NFSDBG_FACILITY NFSDBG_PROC + +/* + * Bare-bones access to getattr: this is for nfs_read_super. + */ +static int +nfs_proc_get_root(struct nfs_server *server, struct nfs_fh *fhandle, + struct nfs_fsinfo *info) +{ + struct nfs_fattr *fattr = info->fattr; + struct nfs2_fsstat fsinfo; + struct rpc_message msg = { + .rpc_proc = &nfs_procedures[NFSPROC_GETATTR], + .rpc_argp = fhandle, + .rpc_resp = fattr, + }; + int status; + + dprintk("%s: call getattr\n", __func__); + nfs_fattr_init(fattr); + status = rpc_call_sync(server->client, &msg, 0); + /* Retry with default authentication if different */ + if (status && server->nfs_client->cl_rpcclient != server->client) + status = rpc_call_sync(server->nfs_client->cl_rpcclient, &msg, 0); + dprintk("%s: reply getattr: %d\n", __func__, status); + if (status) + return status; + dprintk("%s: call statfs\n", __func__); + msg.rpc_proc = &nfs_procedures[NFSPROC_STATFS]; + msg.rpc_resp = &fsinfo; + status = rpc_call_sync(server->client, &msg, 0); + /* Retry with default authentication if different */ + if (status && server->nfs_client->cl_rpcclient != server->client) + status = rpc_call_sync(server->nfs_client->cl_rpcclient, &msg, 0); + dprintk("%s: reply statfs: %d\n", __func__, status); + if (status) + return status; + info->rtmax = NFS_MAXDATA; + info->rtpref = fsinfo.tsize; + info->rtmult = fsinfo.bsize; + info->wtmax = NFS_MAXDATA; + info->wtpref = fsinfo.tsize; + info->wtmult = fsinfo.bsize; + info->dtpref = fsinfo.tsize; + info->maxfilesize = 0x7FFFFFFF; + info->lease_time = 0; + info->change_attr_type = NFS4_CHANGE_TYPE_IS_UNDEFINED; + info->xattr_support = 0; + return 0; +} + +/* + * One function for each procedure in the NFS protocol. + */ +static int +nfs_proc_getattr(struct nfs_server *server, struct nfs_fh *fhandle, + struct nfs_fattr *fattr, struct inode *inode) +{ + struct rpc_message msg = { + .rpc_proc = &nfs_procedures[NFSPROC_GETATTR], + .rpc_argp = fhandle, + .rpc_resp = fattr, + }; + int status; + unsigned short task_flags = 0; + + /* Is this is an attribute revalidation, subject to softreval? */ + if (inode && (server->flags & NFS_MOUNT_SOFTREVAL)) + task_flags |= RPC_TASK_TIMEOUT; + + dprintk("NFS call getattr\n"); + nfs_fattr_init(fattr); + status = rpc_call_sync(server->client, &msg, task_flags); + dprintk("NFS reply getattr: %d\n", status); + return status; +} + +static int +nfs_proc_setattr(struct dentry *dentry, struct nfs_fattr *fattr, + struct iattr *sattr) +{ + struct inode *inode = d_inode(dentry); + struct nfs_sattrargs arg = { + .fh = NFS_FH(inode), + .sattr = sattr + }; + struct rpc_message msg = { + .rpc_proc = &nfs_procedures[NFSPROC_SETATTR], + .rpc_argp = &arg, + .rpc_resp = fattr, + }; + int status; + + /* Mask out the non-modebit related stuff from attr->ia_mode */ + sattr->ia_mode &= S_IALLUGO; + + dprintk("NFS call setattr\n"); + if (sattr->ia_valid & ATTR_FILE) + msg.rpc_cred = nfs_file_cred(sattr->ia_file); + nfs_fattr_init(fattr); + status = rpc_call_sync(NFS_CLIENT(inode), &msg, 0); + if (status == 0) + nfs_setattr_update_inode(inode, sattr, fattr); + dprintk("NFS reply setattr: %d\n", status); + return status; +} + +static int +nfs_proc_lookup(struct inode *dir, struct dentry *dentry, const struct qstr *name, + struct nfs_fh *fhandle, struct nfs_fattr *fattr) +{ + struct nfs_diropargs arg = { + .fh = NFS_FH(dir), + .name = name->name, + .len = name->len + }; + struct nfs_diropok res = { + .fh = fhandle, + .fattr = fattr + }; + struct rpc_message msg = { + .rpc_proc = &nfs_procedures[NFSPROC_LOOKUP], + .rpc_argp = &arg, + .rpc_resp = &res, + }; + int status; + unsigned short task_flags = 0; + + /* Is this is an attribute revalidation, subject to softreval? */ + if (nfs_lookup_is_soft_revalidate(dentry)) + task_flags |= RPC_TASK_TIMEOUT; + + dprintk("NFS call lookup %pd2\n", dentry); + nfs_fattr_init(fattr); + status = rpc_call_sync(NFS_CLIENT(dir), &msg, task_flags); + dprintk("NFS reply lookup: %d\n", status); + return status; +} + +static int nfs_proc_readlink(struct inode *inode, struct page *page, + unsigned int pgbase, unsigned int pglen) +{ + struct nfs_readlinkargs args = { + .fh = NFS_FH(inode), + .pgbase = pgbase, + .pglen = pglen, + .pages = &page + }; + struct rpc_message msg = { + .rpc_proc = &nfs_procedures[NFSPROC_READLINK], + .rpc_argp = &args, + }; + int status; + + dprintk("NFS call readlink\n"); + status = rpc_call_sync(NFS_CLIENT(inode), &msg, 0); + dprintk("NFS reply readlink: %d\n", status); + return status; +} + +struct nfs_createdata { + struct nfs_createargs arg; + struct nfs_diropok res; + struct nfs_fh fhandle; + struct nfs_fattr fattr; +}; + +static struct nfs_createdata *nfs_alloc_createdata(struct inode *dir, + struct dentry *dentry, struct iattr *sattr) +{ + struct nfs_createdata *data; + + data = kmalloc_obj(*data); + + if (data != NULL) { + data->arg.fh = NFS_FH(dir); + data->arg.name = dentry->d_name.name; + data->arg.len = dentry->d_name.len; + data->arg.sattr = sattr; + nfs_fattr_init(&data->fattr); + data->fhandle.size = 0; + data->res.fh = &data->fhandle; + data->res.fattr = &data->fattr; + } + return data; +}; + +static void nfs_free_createdata(const struct nfs_createdata *data) +{ + kfree(data); +} + +static int +nfs_proc_create(struct inode *dir, struct dentry *dentry, struct iattr *sattr, + int flags) +{ + struct nfs_createdata *data; + struct rpc_message msg = { + .rpc_proc = &nfs_procedures[NFSPROC_CREATE], + }; + int status = -ENOMEM; + + dprintk("NFS call create %pd\n", dentry); + data = nfs_alloc_createdata(dir, dentry, sattr); + if (data == NULL) + goto out; + msg.rpc_argp = &data->arg; + msg.rpc_resp = &data->res; + status = rpc_call_sync(NFS_CLIENT(dir), &msg, 0); + nfs_mark_for_revalidate(dir); + if (status == 0) + status = nfs_instantiate(dentry, data->res.fh, data->res.fattr); + nfs_free_createdata(data); +out: + dprintk("NFS reply create: %d\n", status); + return status; +} + +/* + * In NFSv2, mknod is grafted onto the create call. + */ +static int +nfs_proc_mknod(struct inode *dir, struct dentry *dentry, struct iattr *sattr, + dev_t rdev) +{ + struct nfs_createdata *data; + struct rpc_message msg = { + .rpc_proc = &nfs_procedures[NFSPROC_CREATE], + }; + umode_t mode; + int status = -ENOMEM; + + dprintk("NFS call mknod %pd\n", dentry); + + mode = sattr->ia_mode; + if (S_ISFIFO(mode)) { + sattr->ia_mode = (mode & ~S_IFMT) | S_IFCHR; + sattr->ia_valid &= ~ATTR_SIZE; + } else if (S_ISCHR(mode) || S_ISBLK(mode)) { + sattr->ia_valid |= ATTR_SIZE; + sattr->ia_size = new_encode_dev(rdev);/* get out your barf bag */ + } + + data = nfs_alloc_createdata(dir, dentry, sattr); + if (data == NULL) + goto out; + msg.rpc_argp = &data->arg; + msg.rpc_resp = &data->res; + + status = rpc_call_sync(NFS_CLIENT(dir), &msg, 0); + nfs_mark_for_revalidate(dir); + + if (status == -EINVAL && S_ISFIFO(mode)) { + sattr->ia_mode = mode; + nfs_fattr_init(data->res.fattr); + status = rpc_call_sync(NFS_CLIENT(dir), &msg, 0); + } + if (status == 0) + status = nfs_instantiate(dentry, data->res.fh, data->res.fattr); + nfs_free_createdata(data); +out: + dprintk("NFS reply mknod: %d\n", status); + return status; +} + +static int +nfs_proc_remove(struct inode *dir, struct dentry *dentry) +{ + struct nfs_removeargs arg = { + .fh = NFS_FH(dir), + .name = dentry->d_name, + }; + struct rpc_message msg = { + .rpc_proc = &nfs_procedures[NFSPROC_REMOVE], + .rpc_argp = &arg, + }; + int status; + + dprintk("NFS call remove %pd2\n",dentry); + status = rpc_call_sync(NFS_CLIENT(dir), &msg, 0); + nfs_mark_for_revalidate(dir); + + dprintk("NFS reply remove: %d\n", status); + return status; +} + +static void +nfs_proc_unlink_setup(struct rpc_message *msg, + struct dentry *dentry, + struct inode *inode) +{ + msg->rpc_proc = &nfs_procedures[NFSPROC_REMOVE]; +} + +static void nfs_proc_unlink_rpc_prepare(struct rpc_task *task, struct nfs_unlinkdata *data) +{ + rpc_call_start(task); +} + +static int nfs_proc_unlink_done(struct rpc_task *task, struct inode *dir) +{ + nfs_mark_for_revalidate(dir); + return 1; +} + +static void +nfs_proc_rename_setup(struct rpc_message *msg, + struct dentry *old_dentry, + struct dentry *new_dentry, + struct inode *same_parent) +{ + msg->rpc_proc = &nfs_procedures[NFSPROC_RENAME]; +} + +static void nfs_proc_rename_rpc_prepare(struct rpc_task *task, struct nfs_renamedata *data) +{ + rpc_call_start(task); +} + +static int +nfs_proc_rename_done(struct rpc_task *task, struct inode *old_dir, + struct inode *new_dir) +{ + nfs_mark_for_revalidate(old_dir); + nfs_mark_for_revalidate(new_dir); + return 1; +} + +static int +nfs_proc_link(struct inode *inode, struct inode *dir, const struct qstr *name) +{ + struct nfs_linkargs arg = { + .fromfh = NFS_FH(inode), + .tofh = NFS_FH(dir), + .toname = name->name, + .tolen = name->len + }; + struct rpc_message msg = { + .rpc_proc = &nfs_procedures[NFSPROC_LINK], + .rpc_argp = &arg, + }; + int status; + + dprintk("NFS call link %s\n", name->name); + status = rpc_call_sync(NFS_CLIENT(inode), &msg, 0); + nfs_mark_for_revalidate(inode); + nfs_mark_for_revalidate(dir); + dprintk("NFS reply link: %d\n", status); + return status; +} + +static int +nfs_proc_symlink(struct inode *dir, struct dentry *dentry, struct folio *folio, + unsigned int len, struct iattr *sattr) +{ + struct page *page = &folio->page; + struct nfs_fh *fh; + struct nfs_fattr *fattr; + struct nfs_symlinkargs arg = { + .fromfh = NFS_FH(dir), + .fromname = dentry->d_name.name, + .fromlen = dentry->d_name.len, + .pages = &page, + .pathlen = len, + .sattr = sattr + }; + struct rpc_message msg = { + .rpc_proc = &nfs_procedures[NFSPROC_SYMLINK], + .rpc_argp = &arg, + }; + int status = -ENAMETOOLONG; + + dprintk("NFS call symlink %pd\n", dentry); + + if (len > NFS2_MAXPATHLEN) + goto out; + + fh = nfs_alloc_fhandle(); + fattr = nfs_alloc_fattr(); + status = -ENOMEM; + if (fh == NULL || fattr == NULL) + goto out_free; + + status = rpc_call_sync(NFS_CLIENT(dir), &msg, 0); + nfs_mark_for_revalidate(dir); + + /* + * V2 SYMLINK requests don't return any attributes. Setting the + * filehandle size to zero indicates to nfs_instantiate that it + * should fill in the data with a LOOKUP call on the wire. + */ + if (status == 0) + status = nfs_instantiate(dentry, fh, fattr); + +out_free: + nfs_free_fattr(fattr); + nfs_free_fhandle(fh); +out: + dprintk("NFS reply symlink: %d\n", status); + return status; +} + +static struct dentry * +nfs_proc_mkdir(struct inode *dir, struct dentry *dentry, struct iattr *sattr) +{ + struct nfs_createdata *data; + struct rpc_message msg = { + .rpc_proc = &nfs_procedures[NFSPROC_MKDIR], + }; + struct dentry *alias = NULL; + int status = -ENOMEM; + + dprintk("NFS call mkdir %pd\n", dentry); + data = nfs_alloc_createdata(dir, dentry, sattr); + if (data == NULL) + goto out; + msg.rpc_argp = &data->arg; + msg.rpc_resp = &data->res; + + status = rpc_call_sync(NFS_CLIENT(dir), &msg, 0); + nfs_mark_for_revalidate(dir); + if (status == 0) { + alias = nfs_add_or_obtain(dentry, data->res.fh, data->res.fattr); + status = PTR_ERR_OR_ZERO(alias); + } else + alias = ERR_PTR(status); + nfs_free_createdata(data); +out: + dprintk("NFS reply mkdir: %d\n", status); + return alias; +} + +static int +nfs_proc_rmdir(struct inode *dir, const struct qstr *name) +{ + struct nfs_diropargs arg = { + .fh = NFS_FH(dir), + .name = name->name, + .len = name->len + }; + struct rpc_message msg = { + .rpc_proc = &nfs_procedures[NFSPROC_RMDIR], + .rpc_argp = &arg, + }; + int status; + + dprintk("NFS call rmdir %s\n", name->name); + status = rpc_call_sync(NFS_CLIENT(dir), &msg, 0); + nfs_mark_for_revalidate(dir); + dprintk("NFS reply rmdir: %d\n", status); + return status; +} + +/* + * The READDIR implementation is somewhat hackish - we pass a temporary + * buffer to the encode function, which installs it in the receive + * the receive iovec. The decode function just parses the reply to make + * sure it is syntactically correct; the entries itself are decoded + * from nfs_readdir by calling the decode_entry function directly. + */ +static int nfs_proc_readdir(struct nfs_readdir_arg *nr_arg, + struct nfs_readdir_res *nr_res) +{ + struct inode *dir = d_inode(nr_arg->dentry); + struct nfs_readdirargs arg = { + .fh = NFS_FH(dir), + .cookie = nr_arg->cookie, + .count = nr_arg->page_len, + .pages = nr_arg->pages, + }; + struct rpc_message msg = { + .rpc_proc = &nfs_procedures[NFSPROC_READDIR], + .rpc_argp = &arg, + .rpc_cred = nr_arg->cred, + }; + int status; + + dprintk("NFS call readdir %llu\n", (unsigned long long)nr_arg->cookie); + status = rpc_call_sync(NFS_CLIENT(dir), &msg, 0); + nr_res->verf[0] = nr_res->verf[1] = 0; + + nfs_invalidate_atime(dir); + + dprintk("NFS reply readdir: %d\n", status); + return status; +} + +static int +nfs_proc_statfs(struct nfs_server *server, struct nfs_fh *fhandle, + struct nfs_fsstat *stat) +{ + struct nfs2_fsstat fsinfo; + struct rpc_message msg = { + .rpc_proc = &nfs_procedures[NFSPROC_STATFS], + .rpc_argp = fhandle, + .rpc_resp = &fsinfo, + }; + int status; + + dprintk("NFS call statfs\n"); + nfs_fattr_init(stat->fattr); + status = rpc_call_sync(server->client, &msg, 0); + dprintk("NFS reply statfs: %d\n", status); + if (status) + goto out; + stat->tbytes = (u64)fsinfo.blocks * fsinfo.bsize; + stat->fbytes = (u64)fsinfo.bfree * fsinfo.bsize; + stat->abytes = (u64)fsinfo.bavail * fsinfo.bsize; + stat->tfiles = 0; + stat->ffiles = 0; + stat->afiles = 0; +out: + return status; +} + +static int +nfs_proc_fsinfo(struct nfs_server *server, struct nfs_fh *fhandle, + struct nfs_fsinfo *info) +{ + struct nfs2_fsstat fsinfo; + struct rpc_message msg = { + .rpc_proc = &nfs_procedures[NFSPROC_STATFS], + .rpc_argp = fhandle, + .rpc_resp = &fsinfo, + }; + int status; + + dprintk("NFS call fsinfo\n"); + nfs_fattr_init(info->fattr); + status = rpc_call_sync(server->client, &msg, 0); + dprintk("NFS reply fsinfo: %d\n", status); + if (status) + goto out; + info->rtmax = NFS_MAXDATA; + info->rtpref = fsinfo.tsize; + info->rtmult = fsinfo.bsize; + info->wtmax = NFS_MAXDATA; + info->wtpref = fsinfo.tsize; + info->wtmult = fsinfo.bsize; + info->dtpref = fsinfo.tsize; + info->maxfilesize = 0x7FFFFFFF; + info->lease_time = 0; +out: + return status; +} + +static int +nfs_proc_pathconf(struct nfs_server *server, struct nfs_fh *fhandle, + struct nfs_pathconf *info) +{ + info->max_link = 0; + info->max_namelen = NFS2_MAXNAMLEN; + info->case_preserving = true; + return 0; +} + +static int nfs_read_done(struct rpc_task *task, struct nfs_pgio_header *hdr) +{ + struct inode *inode = hdr->inode; + + nfs_invalidate_atime(inode); + if (task->tk_status >= 0) { + nfs_refresh_inode(inode, hdr->res.fattr); + /* Emulate the eof flag, which isn't normally needed in NFSv2 + * as it is guaranteed to always return the file attributes + */ + if ((hdr->res.count == 0 && hdr->args.count > 0) || + hdr->args.offset + hdr->res.count >= hdr->res.fattr->size) + hdr->res.eof = 1; + } + return 0; +} + +static void nfs_proc_read_setup(struct nfs_pgio_header *hdr, + struct rpc_message *msg) +{ + msg->rpc_proc = &nfs_procedures[NFSPROC_READ]; +} + +static int nfs_proc_pgio_rpc_prepare(struct rpc_task *task, + struct nfs_pgio_header *hdr) +{ + rpc_call_start(task); + return 0; +} + +static int nfs_write_done(struct rpc_task *task, struct nfs_pgio_header *hdr) +{ + if (task->tk_status >= 0) { + hdr->res.count = hdr->args.count; + nfs_writeback_update_inode(hdr); + } + return 0; +} + +static void nfs_proc_write_setup(struct nfs_pgio_header *hdr, + struct rpc_message *msg, + struct rpc_clnt **clnt) +{ + /* Note: NFSv2 ignores @stable and always uses NFS_FILE_SYNC */ + hdr->args.stable = NFS_FILE_SYNC; + msg->rpc_proc = &nfs_procedures[NFSPROC_WRITE]; +} + +static void nfs_proc_commit_rpc_prepare(struct rpc_task *task, struct nfs_commit_data *data) +{ + BUG(); +} + +static void +nfs_proc_commit_setup(struct nfs_commit_data *data, struct rpc_message *msg, + struct rpc_clnt **clnt) +{ + BUG(); +} + +static int +nfs_proc_lock(struct file *filp, int cmd, struct file_lock *fl) +{ + struct inode *inode = file_inode(filp); + + return nlmclnt_proc(NFS_SERVER(inode)->nlm_host, cmd, fl, NULL); +} + +/* Helper functions for NFS lock bounds checking */ +#define NFS_LOCK32_OFFSET_MAX ((__s32)0x7fffffffUL) +static int nfs_lock_check_bounds(const struct file_lock *fl) +{ + __s32 start, end; + + start = (__s32)fl->fl_start; + if ((loff_t)start != fl->fl_start) + goto out_einval; + + if (fl->fl_end != OFFSET_MAX) { + end = (__s32)fl->fl_end; + if ((loff_t)end != fl->fl_end) + goto out_einval; + } else + end = NFS_LOCK32_OFFSET_MAX; + + if (start < 0 || start > end) + goto out_einval; + return 0; +out_einval: + return -EINVAL; +} + +static int nfs_have_delegation(struct inode *inode, fmode_t type, int flags) +{ + return 0; +} + +static void nfs_return_delegation(struct inode *inode) +{ + if (S_ISREG(inode->i_mode)) + nfs_wb_all(inode); +} + +static const struct inode_operations nfs_dir_inode_operations = { + .create = nfs_create, + .lookup = nfs_lookup, + .atomic_open = nfs_atomic_open_v23, + .link = nfs_link, + .unlink = nfs_unlink, + .symlink = nfs_symlink, + .mkdir = nfs_mkdir, + .rmdir = nfs_rmdir, + .mknod = nfs_mknod, + .rename = nfs_rename, + .permission = nfs_permission, + .getattr = nfs_getattr, + .setattr = nfs_setattr, + .fileattr_get = nfs_fileattr_get, +}; + +static const struct inode_operations nfs_file_inode_operations = { + .permission = nfs_permission, + .getattr = nfs_getattr, + .setattr = nfs_setattr, + .fileattr_get = nfs_fileattr_get, +}; + +const struct nfs_rpc_ops nfs_v2_clientops = { + .version = 2, /* protocol version */ + .dentry_ops = &nfs_dentry_operations, + .dir_inode_ops = &nfs_dir_inode_operations, + .file_inode_ops = &nfs_file_inode_operations, + .file_ops = &nfs_file_operations, + .getroot = nfs_proc_get_root, + .submount = nfs_submount, + .try_get_tree = nfs_try_get_tree, + .getattr = nfs_proc_getattr, + .setattr = nfs_proc_setattr, + .lookup = nfs_proc_lookup, + .access = NULL, /* access */ + .readlink = nfs_proc_readlink, + .create = nfs_proc_create, + .remove = nfs_proc_remove, + .unlink_setup = nfs_proc_unlink_setup, + .unlink_rpc_prepare = nfs_proc_unlink_rpc_prepare, + .unlink_done = nfs_proc_unlink_done, + .rename_setup = nfs_proc_rename_setup, + .rename_rpc_prepare = nfs_proc_rename_rpc_prepare, + .rename_done = nfs_proc_rename_done, + .link = nfs_proc_link, + .symlink = nfs_proc_symlink, + .mkdir = nfs_proc_mkdir, + .rmdir = nfs_proc_rmdir, + .readdir = nfs_proc_readdir, + .mknod = nfs_proc_mknod, + .statfs = nfs_proc_statfs, + .fsinfo = nfs_proc_fsinfo, + .pathconf = nfs_proc_pathconf, + .decode_dirent = nfs2_decode_dirent, + .pgio_rpc_prepare = nfs_proc_pgio_rpc_prepare, + .read_setup = nfs_proc_read_setup, + .read_done = nfs_read_done, + .write_setup = nfs_proc_write_setup, + .write_done = nfs_write_done, + .commit_setup = nfs_proc_commit_setup, + .commit_rpc_prepare = nfs_proc_commit_rpc_prepare, + .lock = nfs_proc_lock, + .lock_check_bounds = nfs_lock_check_bounds, + .close_context = nfs_close_context, + .have_delegation = nfs_have_delegation, + .return_delegation = nfs_return_delegation, + .alloc_client = nfs_alloc_client, + .init_client = nfs_init_client, + .free_client = nfs_free_client, + .create_server = nfs_create_server, + .clone_server = nfs_clone_server, +}; diff --git a/fs/nfs/read.c b/fs/nfs/read.c new file mode 100644 index 000000000..2b70bd2b9 --- /dev/null +++ b/fs/nfs/read.c @@ -0,0 +1,499 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/nfs/read.c + * + * Block I/O for NFS + * + * Partial copy of Linus' read cache modifications to fs/nfs/file.c + * modified for async RPC by okir@monad.swb.de + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nfs4_fs.h" +#include "internal.h" +#include "iostat.h" +#include "fscache.h" +#include "pnfs.h" +#include "nfstrace.h" +#include "delegation.h" + +#define NFSDBG_FACILITY NFSDBG_PAGECACHE + +const struct nfs_pgio_completion_ops nfs_async_read_completion_ops; +static const struct nfs_rw_ops nfs_rw_read_ops; + +static struct kmem_cache *nfs_rdata_cachep; + +static struct nfs_pgio_header *nfs_readhdr_alloc(void) +{ + struct nfs_pgio_header *p = kmem_cache_zalloc(nfs_rdata_cachep, GFP_KERNEL); + + if (p) + p->rw_mode = FMODE_READ; + return p; +} + +static void nfs_readhdr_free(struct nfs_pgio_header *rhdr) +{ + kfree(rhdr->res.scratch); + kmem_cache_free(nfs_rdata_cachep, rhdr); +} + +static int nfs_return_empty_folio(struct folio *folio) +{ + folio_zero_segment(folio, 0, folio_size(folio)); + folio_mark_uptodate(folio); + if (nfs_netfs_folio_unlock(folio)) + folio_unlock(folio); + return 0; +} + +void nfs_pageio_init_read(struct nfs_pageio_descriptor *pgio, + struct inode *inode, bool force_mds, + const struct nfs_pgio_completion_ops *compl_ops) +{ + struct nfs_server *server = NFS_SERVER(inode); + const struct nfs_pageio_ops *pg_ops = &nfs_pgio_rw_ops; + +#if IS_ENABLED(CONFIG_NFS_V4) + if (server->pnfs_curr_ld && !force_mds) + pg_ops = server->pnfs_curr_ld->pg_read_ops; +#endif /* CONFIG_NFS_V4 */ + nfs_pageio_init(pgio, inode, pg_ops, compl_ops, &nfs_rw_read_ops, + server->rsize, 0); +} +EXPORT_SYMBOL_GPL(nfs_pageio_init_read); + +void nfs_pageio_complete_read(struct nfs_pageio_descriptor *pgio) +{ + struct nfs_pgio_mirror *pgm; + unsigned long npages; + + nfs_pageio_complete(pgio); + + /* It doesn't make sense to do mirrored reads! */ + WARN_ON_ONCE(pgio->pg_mirror_count != 1); + + pgm = &pgio->pg_mirrors[0]; + NFS_I(pgio->pg_inode)->read_io += pgm->pg_bytes_written; + npages = (pgm->pg_bytes_written + PAGE_SIZE - 1) >> PAGE_SHIFT; + nfs_add_stats(pgio->pg_inode, NFSIOS_READPAGES, npages); +} + + +void nfs_pageio_reset_read_mds(struct nfs_pageio_descriptor *pgio) +{ + struct nfs_pgio_mirror *mirror; + + if (pgio->pg_ops && pgio->pg_ops->pg_cleanup) + pgio->pg_ops->pg_cleanup(pgio); + + pgio->pg_ops = &nfs_pgio_rw_ops; + + /* read path should never have more than one mirror */ + WARN_ON_ONCE(pgio->pg_mirror_count != 1); + + mirror = &pgio->pg_mirrors[0]; + mirror->pg_bsize = NFS_SERVER(pgio->pg_inode)->rsize; +} +EXPORT_SYMBOL_GPL(nfs_pageio_reset_read_mds); + +bool nfs_read_alloc_scratch(struct nfs_pgio_header *hdr, size_t size) +{ + WARN_ON(hdr->res.scratch != NULL); + hdr->res.scratch = kmalloc(size, GFP_KERNEL); + return hdr->res.scratch != NULL; +} +EXPORT_SYMBOL_GPL(nfs_read_alloc_scratch); + +static void nfs_readpage_release(struct nfs_page *req, int error) +{ + struct folio *folio = nfs_page_to_folio(req); + + if (nfs_page_group_sync_on_bit(req, PG_UNLOCKPAGE)) + if (nfs_netfs_folio_unlock(folio)) + folio_unlock(folio); + + nfs_release_request(req); +} + +static void nfs_page_group_set_uptodate(struct nfs_page *req) +{ + bool uptodate = false; + + nfs_page_group_lock(req); + if (!test_bit(PG_READ_FAILED, &req->wb_head->wb_flags) && + nfs_page_group_sync_on_bit_locked(req, PG_UPTODATE)) + uptodate = true; + nfs_page_group_unlock(req); + + if (uptodate) + folio_mark_uptodate(nfs_page_to_folio(req)); +} + +static void nfs_page_group_mark_read_failed(struct nfs_page *req) +{ + struct nfs_page *tmp; + + nfs_page_group_lock(req); + set_bit(PG_READ_FAILED, &req->wb_head->wb_flags); + tmp = req; + do { + clear_bit(PG_UPTODATE, &tmp->wb_flags); + tmp = tmp->wb_this_page; + } while (tmp != req); + nfs_page_group_unlock(req); +} + +static void nfs_read_completion(struct nfs_pgio_header *hdr) +{ + unsigned long bytes = 0; + int error; + + if (test_bit(NFS_IOHDR_REDO, &hdr->flags)) + goto out; + while (!list_empty(&hdr->pages)) { + struct nfs_page *req = nfs_list_entry(hdr->pages.next); + struct folio *folio = nfs_page_to_folio(req); + unsigned long start = req->wb_pgbase; + unsigned long end = req->wb_pgbase + req->wb_bytes; + + if (test_bit(NFS_IOHDR_EOF, &hdr->flags)) { + /* note: regions of the page not covered by a + * request are zeroed in nfs_read_add_folio + */ + if (bytes > hdr->good_bytes) { + /* nothing in this request was good, so zero + * the full extent of the request */ + folio_zero_segment(folio, start, end); + + } else if (hdr->good_bytes - bytes < req->wb_bytes) { + /* part of this request has good bytes, but + * not all. zero the bad bytes */ + start += hdr->good_bytes - bytes; + WARN_ON(start < req->wb_pgbase); + folio_zero_segment(folio, start, end); + } + } + error = 0; + bytes += req->wb_bytes; + if (test_bit(NFS_IOHDR_ERROR, &hdr->flags)) { + if (bytes <= hdr->good_bytes) + nfs_page_group_set_uptodate(req); + else { + nfs_page_group_mark_read_failed(req); + error = hdr->error; + xchg(&nfs_req_openctx(req)->error, error); + } + } else + nfs_page_group_set_uptodate(req); + nfs_list_remove_request(req); + nfs_readpage_release(req, error); + } + nfs_netfs_read_completion(hdr); + +out: + hdr->release(hdr); +} + +static void nfs_initiate_read(struct nfs_pgio_header *hdr, + struct rpc_message *msg, + const struct nfs_rpc_ops *rpc_ops, + struct rpc_task_setup *task_setup_data, int how) +{ + rpc_ops->read_setup(hdr, msg); + nfs_netfs_initiate_read(hdr); + trace_nfs_initiate_read(hdr); +} + +static void +nfs_async_read_error(struct list_head *head, int error) +{ + struct nfs_page *req; + + while (!list_empty(head)) { + req = nfs_list_entry(head->next); + nfs_list_remove_request(req); + nfs_readpage_release(req, error); + } +} + +const struct nfs_pgio_completion_ops nfs_async_read_completion_ops = { + .error_cleanup = nfs_async_read_error, + .completion = nfs_read_completion, +}; + +/* + * This is the callback from RPC telling us whether a reply was + * received or some error occurred (timeout or socket shutdown). + */ +static int nfs_readpage_done(struct rpc_task *task, + struct nfs_pgio_header *hdr, + struct inode *inode) +{ + int status = NFS_PROTO(inode)->read_done(task, hdr); + if (status != 0) + return status; + + nfs_add_stats(inode, NFSIOS_SERVERREADBYTES, hdr->res.count); + trace_nfs_readpage_done(task, hdr); + + if (task->tk_status == -ESTALE) { + nfs_set_inode_stale(inode); + nfs_mark_for_revalidate(inode); + } + return 0; +} + +static void nfs_readpage_retry(struct rpc_task *task, + struct nfs_pgio_header *hdr) +{ + struct nfs_pgio_args *argp = &hdr->args; + struct nfs_pgio_res *resp = &hdr->res; + + /* This is a short read! */ + nfs_inc_stats(hdr->inode, NFSIOS_SHORTREAD); + trace_nfs_readpage_short(task, hdr); + + /* Has the server at least made some progress? */ + if (resp->count == 0) { + nfs_set_pgio_error(hdr, -EIO, argp->offset); + return; + } + + /* For non rpc-based layout drivers, retry-through-MDS */ + if (!task->tk_ops) { + hdr->pnfs_error = -EAGAIN; + return; + } + + /* Yes, so retry the read at the end of the hdr */ + hdr->mds_offset += resp->count; + argp->offset += resp->count; + argp->pgbase += resp->count; + argp->count -= resp->count; + resp->count = 0; + resp->eof = 0; + rpc_restart_call_prepare(task); +} + +static void nfs_readpage_result(struct rpc_task *task, + struct nfs_pgio_header *hdr) +{ + if (hdr->res.eof) { + loff_t pos = hdr->args.offset + hdr->res.count; + unsigned int new = pos - hdr->io_start; + + if (hdr->good_bytes > new) { + hdr->good_bytes = new; + set_bit(NFS_IOHDR_EOF, &hdr->flags); + clear_bit(NFS_IOHDR_ERROR, &hdr->flags); + } + } else if (hdr->res.count < hdr->args.count) + nfs_readpage_retry(task, hdr); +} + +int nfs_read_add_folio(struct nfs_pageio_descriptor *pgio, + struct nfs_open_context *ctx, + struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + struct nfs_server *server = NFS_SERVER(inode); + size_t fsize = folio_size(folio); + unsigned int rsize = server->rsize; + struct nfs_page *new; + unsigned int len, aligned_len; + int error; + + len = nfs_folio_length(folio); + if (len == 0) + return nfs_return_empty_folio(folio); + + aligned_len = min_t(unsigned int, ALIGN(len, rsize), fsize); + + new = nfs_page_create_from_folio(ctx, folio, 0, aligned_len); + if (IS_ERR(new)) { + error = PTR_ERR(new); + if (nfs_netfs_folio_unlock(folio)) + folio_unlock(folio); + goto out; + } + + if (len < fsize) + folio_zero_segment(folio, len, fsize); + if (!nfs_pageio_add_request(pgio, new)) { + nfs_list_remove_request(new); + error = pgio->pg_error; + nfs_readpage_release(new, error); + goto out; + } + return 0; +out: + return error; +} + +/* + * Actually read a folio over the wire. + */ +static int nfs_do_read_folio(struct file *file, struct folio *folio) +{ + struct inode *inode = file_inode(file); + struct nfs_pageio_descriptor pgio; + struct nfs_open_context *ctx; + int ret; + + ctx = get_nfs_open_context(nfs_file_open_context(file)); + + xchg(&ctx->error, 0); + nfs_pageio_init_read(&pgio, inode, false, + &nfs_async_read_completion_ops); + + ret = nfs_read_add_folio(&pgio, ctx, folio); + if (ret) + goto out_put; + + nfs_pageio_complete_read(&pgio); + nfs_update_delegated_atime(inode); + if (pgio.pg_error < 0) { + ret = pgio.pg_error; + goto out_put; + } + + ret = folio_wait_locked_killable(folio); + if (!folio_test_uptodate(folio) && !ret) + ret = xchg(&ctx->error, 0); + +out_put: + put_nfs_open_context(ctx); + return ret; +} + +/* + * Synchronously read a folio. + * + * This is not heavily used as most users to try an asynchronous + * large read through ->readahead first. + */ +int nfs_read_folio(struct file *file, struct folio *folio) +{ + struct inode *inode = file_inode(file); + loff_t pos = folio_pos(folio); + size_t len = folio_size(folio); + int ret; + + trace_nfs_aop_readpage(inode, pos, len); + nfs_inc_stats(inode, NFSIOS_VFSREADPAGE); + task_io_account_read(len); + + /* + * Try to flush any pending writes to the file.. + * + * NOTE! Because we own the folio lock, there cannot + * be any new pending writes generated at this point + * for this folio (other folios can be written to). + */ + ret = nfs_wb_folio(inode, folio); + if (ret) + goto out_unlock; + if (folio_test_uptodate(folio)) + goto out_unlock; + + ret = -ESTALE; + if (NFS_STALE(inode)) + goto out_unlock; + + ret = nfs_netfs_read_folio(file, folio); + if (ret) + ret = nfs_do_read_folio(file, folio); +out: + trace_nfs_aop_readpage_done(inode, pos, len, ret); + return ret; +out_unlock: + folio_unlock(folio); + goto out; +} + +void nfs_readahead(struct readahead_control *ractl) +{ + struct nfs_pageio_descriptor pgio; + struct nfs_open_context *ctx; + unsigned int nr_pages = readahead_count(ractl); + struct file *file = ractl->file; + struct inode *inode = ractl->mapping->host; + struct folio *folio; + int ret; + + trace_nfs_aop_readahead(inode, readahead_pos(ractl), nr_pages); + nfs_inc_stats(inode, NFSIOS_VFSREADPAGES); + task_io_account_read(readahead_length(ractl)); + + ret = -ESTALE; + if (NFS_STALE(inode)) + goto out; + + ret = nfs_netfs_readahead(ractl); + if (!ret) + goto out; + + if (file == NULL) { + ret = -EBADF; + ctx = nfs_find_open_context(inode, NULL, FMODE_READ); + if (ctx == NULL) + goto out; + } else + ctx = get_nfs_open_context(nfs_file_open_context(file)); + + nfs_pageio_init_read(&pgio, inode, false, + &nfs_async_read_completion_ops); + + while ((folio = readahead_folio(ractl)) != NULL) { + ret = nfs_read_add_folio(&pgio, ctx, folio); + if (ret) + break; + } + + nfs_pageio_complete_read(&pgio); + nfs_update_delegated_atime(inode); + + put_nfs_open_context(ctx); +out: + trace_nfs_aop_readahead_done(inode, nr_pages, ret); +} + +int __init nfs_init_readpagecache(void) +{ + nfs_rdata_cachep = kmem_cache_create("nfs_read_data", + sizeof(struct nfs_pgio_header), + 0, SLAB_HWCACHE_ALIGN, + NULL); + if (nfs_rdata_cachep == NULL) + return -ENOMEM; + + return 0; +} + +void nfs_destroy_readpagecache(void) +{ + kmem_cache_destroy(nfs_rdata_cachep); +} + +static const struct nfs_rw_ops nfs_rw_read_ops = { + .rw_alloc_header = nfs_readhdr_alloc, + .rw_free_header = nfs_readhdr_free, + .rw_done = nfs_readpage_done, + .rw_result = nfs_readpage_result, + .rw_initiate = nfs_initiate_read, +}; diff --git a/fs/nfs/super.c b/fs/nfs/super.c new file mode 100644 index 000000000..cb19f1540 --- /dev/null +++ b/fs/nfs/super.c @@ -0,0 +1,1463 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/nfs/super.c + * + * Copyright (C) 1992 Rick Sladkey + * + * nfs superblock handling functions + * + * Modularised by Alan Cox , while hacking some + * experimental NFS changes. Modularisation taken straight from SYS5 fs. + * + * Change to nfs_read_super() to permit NFS mounts to multi-homed hosts. + * J.S.Peatfield@damtp.cam.ac.uk + * + * Split from inode.c by David Howells + * + * - superblocks are indexed on server only - all inodes, dentries, etc. associated with a + * particular server are held in the same superblock + * - NFS superblocks can have several effective roots to the dentry tree + * - directory type roots are spliced into the tree when a path from one root reaches the root + * of another (see nfs_lookup()) + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include "nfs4_fs.h" +#include "callback.h" +#include "delegation.h" +#include "iostat.h" +#include "internal.h" +#include "fscache.h" +#include "nfs4session.h" +#include "pnfs.h" +#include "nfs.h" +#include "netns.h" +#include "sysfs.h" +#include "nfs4idmap.h" + +#define NFSDBG_FACILITY NFSDBG_VFS + +const struct super_operations nfs_sops = { + .alloc_inode = nfs_alloc_inode, + .free_inode = nfs_free_inode, + .write_inode = nfs_write_inode, + .drop_inode = nfs_drop_inode, + .statfs = nfs_statfs, + .evict_inode = nfs_evict_inode, + .umount_begin = nfs_umount_begin, + .show_options = nfs_show_options, + .show_devname = nfs_show_devname, + .show_path = nfs_show_path, + .show_stats = nfs_show_stats, +}; +EXPORT_SYMBOL_GPL(nfs_sops); + +#ifdef CONFIG_NFS_V4_2 +static const struct nfs_ssc_client_ops nfs_ssc_clnt_ops_tbl = { + .sco_sb_deactive = nfs_sb_deactive, +}; +#endif + +#if IS_ENABLED(CONFIG_NFS_V4) +static int __init register_nfs4_fs(void) +{ + return register_filesystem(&nfs4_fs_type); +} + +static void unregister_nfs4_fs(void) +{ + unregister_filesystem(&nfs4_fs_type); +} +#else +static int __init register_nfs4_fs(void) +{ + return 0; +} + +static void unregister_nfs4_fs(void) +{ +} +#endif + +#ifdef CONFIG_NFS_V4_2 +static void nfs_ssc_register_ops(void) +{ + nfs_ssc_register(&nfs_ssc_clnt_ops_tbl); +} + +static void nfs_ssc_unregister_ops(void) +{ + nfs_ssc_unregister(&nfs_ssc_clnt_ops_tbl); +} +#endif /* CONFIG_NFS_V4_2 */ + +static struct shrinker *acl_shrinker; + +/* + * Register the NFS filesystems + */ +int __init register_nfs_fs(void) +{ + int ret; + + ret = register_filesystem(&nfs_fs_type); + if (ret < 0) + goto error_0; + + ret = register_nfs4_fs(); + if (ret < 0) + goto error_1; + + ret = nfs_register_sysctl(); + if (ret < 0) + goto error_2; + + acl_shrinker = shrinker_alloc(0, "nfs-acl"); + if (!acl_shrinker) { + ret = -ENOMEM; + goto error_3; + } + + acl_shrinker->count_objects = nfs_access_cache_count; + acl_shrinker->scan_objects = nfs_access_cache_scan; + + shrinker_register(acl_shrinker); + +#ifdef CONFIG_NFS_V4_2 + nfs_ssc_register_ops(); +#endif + return 0; +error_3: + nfs_unregister_sysctl(); +error_2: + unregister_nfs4_fs(); +error_1: + unregister_filesystem(&nfs_fs_type); +error_0: + return ret; +} + +/* + * Unregister the NFS filesystems + */ +void __exit unregister_nfs_fs(void) +{ + shrinker_free(acl_shrinker); + nfs_unregister_sysctl(); + unregister_nfs4_fs(); +#ifdef CONFIG_NFS_V4_2 + nfs_ssc_unregister_ops(); +#endif + unregister_filesystem(&nfs_fs_type); +} + +bool nfs_sb_active(struct super_block *sb) +{ + struct nfs_server *server = NFS_SB(sb); + + if (!atomic_inc_not_zero(&sb->s_active)) + return false; + if (atomic_inc_return(&server->active) != 1) + atomic_dec(&sb->s_active); + return true; +} +EXPORT_SYMBOL_GPL(nfs_sb_active); + +void nfs_sb_deactive(struct super_block *sb) +{ + struct nfs_server *server = NFS_SB(sb); + + if (atomic_dec_and_test(&server->active)) + deactivate_super(sb); +} +EXPORT_SYMBOL_GPL(nfs_sb_deactive); + +int nfs_client_for_each_server(struct nfs_client *clp, + int (*fn)(struct nfs_server *server, void *data), void *data) +{ + struct nfs_server *server, *last = NULL; + int ret = 0; + + rcu_read_lock(); + list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { + if (!(server->super && nfs_sb_active(server->super))) + continue; + rcu_read_unlock(); + if (last) + nfs_sb_deactive(last->super); + last = server; + ret = fn(server, data); + if (ret) + goto out; + cond_resched(); + rcu_read_lock(); + } + rcu_read_unlock(); +out: + if (last) + nfs_sb_deactive(last->super); + return ret; +} +EXPORT_SYMBOL_GPL(nfs_client_for_each_server); + +/* + * Deliver file system statistics to userspace + */ +int nfs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct nfs_server *server = NFS_SB(dentry->d_sb); + unsigned char blockbits; + unsigned long blockres; + struct nfs_fh *fh = NFS_FH(d_inode(dentry)); + struct nfs_fsstat res; + int error = -ENOMEM; + + res.fattr = nfs_alloc_fattr(); + if (res.fattr == NULL) + goto out_err; + + error = server->nfs_client->rpc_ops->statfs(server, fh, &res); + if (unlikely(error == -ESTALE)) { + struct dentry *pd_dentry; + + pd_dentry = dget_parent(dentry); + nfs_zap_caches(d_inode(pd_dentry)); + dput(pd_dentry); + } + nfs_free_fattr(res.fattr); + if (error < 0) + goto out_err; + + buf->f_type = NFS_SUPER_MAGIC; + + /* + * Current versions of glibc do not correctly handle the + * case where f_frsize != f_bsize. Eventually we want to + * report the value of wtmult in this field. + */ + buf->f_frsize = dentry->d_sb->s_blocksize; + + /* + * On most *nix systems, f_blocks, f_bfree, and f_bavail + * are reported in units of f_frsize. Linux hasn't had + * an f_frsize field in its statfs struct until recently, + * thus historically Linux's sys_statfs reports these + * fields in units of f_bsize. + */ + buf->f_bsize = dentry->d_sb->s_blocksize; + blockbits = dentry->d_sb->s_blocksize_bits; + blockres = (1 << blockbits) - 1; + buf->f_blocks = (res.tbytes + blockres) >> blockbits; + buf->f_bfree = (res.fbytes + blockres) >> blockbits; + buf->f_bavail = (res.abytes + blockres) >> blockbits; + + buf->f_files = res.tfiles; + buf->f_ffree = res.afiles; + + buf->f_namelen = server->namelen; + + return 0; + + out_err: + dprintk("%s: statfs error = %d\n", __func__, -error); + return error; +} +EXPORT_SYMBOL_GPL(nfs_statfs); + +/* + * Map the security flavour number to a name + */ +static const char *nfs_pseudoflavour_to_name(rpc_authflavor_t flavour) +{ + static const struct { + rpc_authflavor_t flavour; + const char *str; + } sec_flavours[NFS_AUTH_INFO_MAX_FLAVORS] = { + /* update NFS_AUTH_INFO_MAX_FLAVORS when this list changes! */ + { RPC_AUTH_NULL, "null" }, + { RPC_AUTH_UNIX, "sys" }, + { RPC_AUTH_GSS_KRB5, "krb5" }, + { RPC_AUTH_GSS_KRB5I, "krb5i" }, + { RPC_AUTH_GSS_KRB5P, "krb5p" }, + { RPC_AUTH_GSS_LKEY, "lkey" }, + { RPC_AUTH_GSS_LKEYI, "lkeyi" }, + { RPC_AUTH_GSS_LKEYP, "lkeyp" }, + { RPC_AUTH_GSS_SPKM, "spkm" }, + { RPC_AUTH_GSS_SPKMI, "spkmi" }, + { RPC_AUTH_GSS_SPKMP, "spkmp" }, + { UINT_MAX, "unknown" } + }; + int i; + + for (i = 0; sec_flavours[i].flavour != UINT_MAX; i++) { + if (sec_flavours[i].flavour == flavour) + break; + } + return sec_flavours[i].str; +} + +static void nfs_show_mountd_netid(struct seq_file *m, struct nfs_server *nfss, + int showdefaults) +{ + struct sockaddr *sap = (struct sockaddr *) &nfss->mountd_address; + char *proto = NULL; + + switch (sap->sa_family) { + case AF_INET: + switch (nfss->mountd_protocol) { + case IPPROTO_UDP: + proto = RPCBIND_NETID_UDP; + break; + case IPPROTO_TCP: + proto = RPCBIND_NETID_TCP; + break; + } + break; + case AF_INET6: + switch (nfss->mountd_protocol) { + case IPPROTO_UDP: + proto = RPCBIND_NETID_UDP6; + break; + case IPPROTO_TCP: + proto = RPCBIND_NETID_TCP6; + break; + } + break; + } + if (proto || showdefaults) + seq_printf(m, ",mountproto=%s", proto ?: "auto"); +} + +static void nfs_show_mountd_options(struct seq_file *m, struct nfs_server *nfss, + int showdefaults) +{ + struct sockaddr *sap = (struct sockaddr *)&nfss->mountd_address; + + if (nfss->flags & NFS_MOUNT_LEGACY_INTERFACE) + return; + + switch (sap->sa_family) { + case AF_INET: { + struct sockaddr_in *sin = (struct sockaddr_in *)sap; + seq_printf(m, ",mountaddr=%pI4", &sin->sin_addr.s_addr); + break; + } + case AF_INET6: { + struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)sap; + seq_printf(m, ",mountaddr=%pI6c", &sin6->sin6_addr); + break; + } + default: + if (showdefaults) + seq_puts(m, ",mountaddr=unspecified"); + } + + if (nfss->mountd_version || showdefaults) + seq_printf(m, ",mountvers=%u", nfss->mountd_version); + if ((nfss->mountd_port && + nfss->mountd_port != (unsigned short)NFS_UNSPEC_PORT) || + showdefaults) + seq_printf(m, ",mountport=%u", nfss->mountd_port); + + nfs_show_mountd_netid(m, nfss, showdefaults); +} + +#if IS_ENABLED(CONFIG_NFS_V4) +static void nfs_show_nfsv4_options(struct seq_file *m, struct nfs_server *nfss, + int showdefaults) +{ + struct nfs_client *clp = nfss->nfs_client; + + seq_printf(m, ",clientaddr=%s", clp->cl_ipaddr); +} +#else +static void nfs_show_nfsv4_options(struct seq_file *m, struct nfs_server *nfss, + int showdefaults) +{ +} +#endif + +static void nfs_show_nfs_version(struct seq_file *m, + unsigned int version, + unsigned int minorversion) +{ + seq_printf(m, ",vers=%u", version); + if (version == 4) + seq_printf(m, ".%u", minorversion); +} + +/* + * Describe the mount options in force on this server representation + */ +static void nfs_show_mount_options(struct seq_file *m, struct nfs_server *nfss, + int showdefaults) +{ + static const struct proc_nfs_info { + int flag; + const char *str; + const char *nostr; + } nfs_info[] = { + { NFS_MOUNT_SOFT, ",soft", "" }, + { NFS_MOUNT_SOFTERR, ",softerr", "" }, + { NFS_MOUNT_SOFTREVAL, ",softreval", "" }, + { NFS_MOUNT_POSIX, ",posix", "" }, + { NFS_MOUNT_NOCTO, ",nocto", "" }, + { NFS_MOUNT_NOAC, ",noac", "" }, + { NFS_MOUNT_NONLM, ",nolock", "" }, + { NFS_MOUNT_NOACL, ",noacl", "" }, + { NFS_MOUNT_NORDIRPLUS, ",nordirplus", "" }, + { NFS_MOUNT_FORCE_RDIRPLUS, ",rdirplus=force", "" }, + { NFS_MOUNT_UNSHARED, ",nosharecache", "" }, + { NFS_MOUNT_NORESVPORT, ",noresvport", "" }, + { NFS_MOUNT_NETUNREACH_FATAL, + ",fatal_neterrors=ENETDOWN:ENETUNREACH", + ",fatal_neterrors=none" }, + { 0, NULL, NULL } + }; + const struct proc_nfs_info *nfs_infop; + struct nfs_client *clp = nfss->nfs_client; + u32 version = clp->rpc_ops->version; + int local_flock, local_fcntl; + + nfs_show_nfs_version(m, version, clp->cl_minorversion); + seq_printf(m, ",rsize=%u", nfss->rsize); + seq_printf(m, ",wsize=%u", nfss->wsize); + if (nfss->bsize != 0) + seq_printf(m, ",bsize=%u", nfss->bsize); + seq_printf(m, ",namlen=%u", nfss->namelen); + if (nfss->acregmin != NFS_DEF_ACREGMIN*HZ || showdefaults) + seq_printf(m, ",acregmin=%u", nfss->acregmin/HZ); + if (nfss->acregmax != NFS_DEF_ACREGMAX*HZ || showdefaults) + seq_printf(m, ",acregmax=%u", nfss->acregmax/HZ); + if (nfss->acdirmin != NFS_DEF_ACDIRMIN*HZ || showdefaults) + seq_printf(m, ",acdirmin=%u", nfss->acdirmin/HZ); + if (nfss->acdirmax != NFS_DEF_ACDIRMAX*HZ || showdefaults) + seq_printf(m, ",acdirmax=%u", nfss->acdirmax/HZ); + if (!(nfss->flags & (NFS_MOUNT_SOFT|NFS_MOUNT_SOFTERR))) + seq_puts(m, ",hard"); + for (nfs_infop = nfs_info; nfs_infop->flag; nfs_infop++) { + if (nfss->flags & nfs_infop->flag) + seq_puts(m, nfs_infop->str); + else + seq_puts(m, nfs_infop->nostr); + } + rcu_read_lock(); + seq_printf(m, ",proto=%s", + rpc_peeraddr2str(nfss->client, RPC_DISPLAY_NETID)); + rcu_read_unlock(); + if (clp->cl_nconnect > 0) + seq_printf(m, ",nconnect=%u", clp->cl_nconnect); + if (version == 4) { + if (clp->cl_max_connect > 1) + seq_printf(m, ",max_connect=%u", clp->cl_max_connect); + if (nfss->port != NFS_PORT) + seq_printf(m, ",port=%u", nfss->port); + } else + if (nfss->port) + seq_printf(m, ",port=%u", nfss->port); + + seq_printf(m, ",timeo=%lu", 10U * nfss->client->cl_timeout->to_initval / HZ); + seq_printf(m, ",retrans=%u", nfss->client->cl_timeout->to_retries); + seq_printf(m, ",sec=%s", nfs_pseudoflavour_to_name(nfss->client->cl_auth->au_flavor)); + switch (clp->cl_xprtsec.policy) { + case RPC_XPRTSEC_TLS_ANON: + seq_puts(m, ",xprtsec=tls"); + break; + case RPC_XPRTSEC_TLS_X509: + seq_puts(m, ",xprtsec=mtls"); + break; + default: + break; + } + if (clp->cl_xprtsec.cert_serial) + seq_puts(m, ",cert_serial="); + if (clp->cl_xprtsec.privkey_serial) + seq_puts(m, ",privkey_serial="); + + if (version != 4) + nfs_show_mountd_options(m, nfss, showdefaults); + else + nfs_show_nfsv4_options(m, nfss, showdefaults); + + if (nfss->options & NFS_OPTION_FSCACHE) { +#ifdef CONFIG_NFS_FSCACHE + if (nfss->fscache_uniq) + seq_printf(m, ",fsc=%s", nfss->fscache_uniq); + else + seq_puts(m, ",fsc"); +#else + seq_puts(m, ",fsc"); +#endif + } + + if (nfss->options & NFS_OPTION_MIGRATION) + seq_puts(m, ",migration"); + + if (nfss->flags & NFS_MOUNT_LOOKUP_CACHE_NONEG) { + if (nfss->flags & NFS_MOUNT_LOOKUP_CACHE_NONE) + seq_puts(m, ",lookupcache=none"); + else + seq_puts(m, ",lookupcache=pos"); + } + + local_flock = nfss->flags & NFS_MOUNT_LOCAL_FLOCK; + local_fcntl = nfss->flags & NFS_MOUNT_LOCAL_FCNTL; + + if (!local_flock && !local_fcntl) + seq_puts(m, ",local_lock=none"); + else if (local_flock && local_fcntl) + seq_puts(m, ",local_lock=all"); + else if (local_flock) + seq_puts(m, ",local_lock=flock"); + else + seq_puts(m, ",local_lock=posix"); + + if (nfss->flags & NFS_MOUNT_NO_ALIGNWRITE) + seq_puts(m, ",noalignwrite"); + + if (nfss->flags & NFS_MOUNT_WRITE_EAGER) { + if (nfss->flags & NFS_MOUNT_WRITE_WAIT) + seq_puts(m, ",write=wait"); + else + seq_puts(m, ",write=eager"); + } +} + +/* + * Describe the mount options on this VFS mountpoint + */ +int nfs_show_options(struct seq_file *m, struct dentry *root) +{ + struct nfs_server *nfss = NFS_SB(root->d_sb); + + nfs_show_mount_options(m, nfss, 0); + + rcu_read_lock(); + seq_printf(m, ",addr=%s", + rpc_peeraddr2str(nfss->nfs_client->cl_rpcclient, + RPC_DISPLAY_ADDR)); + rcu_read_unlock(); + + return 0; +} +EXPORT_SYMBOL_GPL(nfs_show_options); + +#if IS_ENABLED(CONFIG_NFS_V4) +static void show_lease(struct seq_file *m, struct nfs_server *server) +{ + struct nfs_client *clp = server->nfs_client; + unsigned long expire; + + seq_printf(m, ",lease_time=%ld", clp->cl_lease_time / HZ); + expire = clp->cl_last_renewal + clp->cl_lease_time; + seq_printf(m, ",lease_expired=%ld", + time_after(expire, jiffies) ? 0 : (jiffies - expire) / HZ); +} + +static void show_sessions(struct seq_file *m, struct nfs_server *server) +{ + if (nfs4_has_session(server->nfs_client)) + seq_puts(m, ",sessions"); +} + +static void show_pnfs(struct seq_file *m, struct nfs_server *server) +{ + seq_printf(m, ",pnfs="); + if (server->pnfs_curr_ld) + seq_printf(m, "%s", server->pnfs_curr_ld->name); + else + seq_printf(m, "not configured"); +} + +static void show_implementation_id(struct seq_file *m, struct nfs_server *nfss) +{ + if (nfss->nfs_client && nfss->nfs_client->cl_implid) { + struct nfs41_impl_id *impl_id = nfss->nfs_client->cl_implid; + seq_printf(m, "\n\timpl_id:\tname='%s',domain='%s'," + "date='%llu,%u'", + impl_id->name, impl_id->domain, + impl_id->date.seconds, impl_id->date.nseconds); + } +} +#else /* CONFIG_NFS_V4 */ +static void show_implementation_id(struct seq_file *m, struct nfs_server *nfss) +{ +} +#endif /* CONFIG_NFS_V4 */ + +int nfs_show_devname(struct seq_file *m, struct dentry *root) +{ + char *page = kmalloc(PAGE_SIZE, GFP_KERNEL); + char *devname, *dummy; + int err = 0; + if (!page) + return -ENOMEM; + devname = nfs_path(&dummy, root, page, PAGE_SIZE, 0); + if (IS_ERR(devname)) + err = PTR_ERR(devname); + else + seq_escape(m, devname, " \t\n\\"); + kfree(page); + return err; +} +EXPORT_SYMBOL_GPL(nfs_show_devname); + +int nfs_show_path(struct seq_file *m, struct dentry *dentry) +{ + seq_puts(m, "/"); + return 0; +} +EXPORT_SYMBOL_GPL(nfs_show_path); + +/* + * Present statistical information for this VFS mountpoint + */ +int nfs_show_stats(struct seq_file *m, struct dentry *root) +{ + int i, cpu; + struct nfs_server *nfss = NFS_SB(root->d_sb); + struct rpc_auth *auth = nfss->client->cl_auth; + struct nfs_iostats totals = { }; + + seq_printf(m, "statvers=%s", NFS_IOSTAT_VERS); + + /* + * Display all mount option settings + */ + seq_puts(m, "\n\topts:\t"); + seq_puts(m, sb_rdonly(root->d_sb) ? "ro" : "rw"); + seq_puts(m, root->d_sb->s_flags & SB_SYNCHRONOUS ? ",sync" : ""); + seq_puts(m, root->d_sb->s_flags & SB_NOATIME ? ",noatime" : ""); + seq_puts(m, root->d_sb->s_flags & SB_NODIRATIME ? ",nodiratime" : ""); + nfs_show_mount_options(m, nfss, 1); + + seq_printf(m, "\n\tage:\t%lu", (jiffies - nfss->mount_time) / HZ); + + show_implementation_id(m, nfss); + + seq_puts(m, "\n\tcaps:\t"); + seq_printf(m, "caps=0x%x", nfss->caps); + seq_printf(m, ",wtmult=%u", nfss->wtmult); + seq_printf(m, ",dtsize=%u", nfss->dtsize); + seq_printf(m, ",bsize=%u", nfss->bsize); + seq_printf(m, ",namlen=%u", nfss->namelen); + +#if IS_ENABLED(CONFIG_NFS_V4) + if (nfss->nfs_client->rpc_ops->version == 4) { + seq_puts(m, "\n\tnfsv4:\t"); + seq_printf(m, "bm0=0x%x", nfss->attr_bitmask[0]); + seq_printf(m, ",bm1=0x%x", nfss->attr_bitmask[1]); + seq_printf(m, ",bm2=0x%x", nfss->attr_bitmask[2]); + seq_printf(m, ",acl=0x%x", nfss->acl_bitmask); + show_sessions(m, nfss); + show_pnfs(m, nfss); + show_lease(m, nfss); + } +#endif + + /* + * Display security flavor in effect for this mount + */ + seq_printf(m, "\n\tsec:\tflavor=%u", auth->au_ops->au_flavor); + if (auth->au_flavor) + seq_printf(m, ",pseudoflavor=%u", auth->au_flavor); + + /* + * Display superblock I/O counters + */ + for_each_possible_cpu(cpu) { + struct nfs_iostats *stats; + + preempt_disable(); + stats = per_cpu_ptr(nfss->io_stats, cpu); + + for (i = 0; i < __NFSIOS_COUNTSMAX; i++) + totals.events[i] += stats->events[i]; + for (i = 0; i < __NFSIOS_BYTESMAX; i++) + totals.bytes[i] += stats->bytes[i]; + + preempt_enable(); + } + + seq_puts(m, "\n\tevents:\t"); + for (i = 0; i < __NFSIOS_COUNTSMAX; i++) + seq_printf(m, "%lu ", totals.events[i]); + seq_puts(m, "\n\tbytes:\t"); + for (i = 0; i < __NFSIOS_BYTESMAX; i++) + seq_printf(m, "%Lu ", totals.bytes[i]); + seq_putc(m, '\n'); + + rpc_clnt_show_stats(m, nfss->client); + + return 0; +} +EXPORT_SYMBOL_GPL(nfs_show_stats); + +/* + * Begin unmount by attempting to remove all automounted mountpoints we added + * in response to xdev traversals and referrals + */ +void nfs_umount_begin(struct super_block *sb) +{ + struct nfs_server *server; + struct rpc_clnt *rpc; + + server = NFS_SB(sb); + /* -EIO all pending I/O */ + rpc = server->client_acl; + if (!IS_ERR(rpc)) + rpc_killall_tasks(rpc); + rpc = server->client; + if (!IS_ERR(rpc)) + rpc_killall_tasks(rpc); +} +EXPORT_SYMBOL_GPL(nfs_umount_begin); + +/* + * Return true if 'match' is in auth_info or auth_info is empty. + * Return false otherwise. + */ +bool nfs_auth_info_match(const struct nfs_auth_info *auth_info, + rpc_authflavor_t match) +{ + int i; + + if (!auth_info->flavor_len) + return true; + + for (i = 0; i < auth_info->flavor_len; i++) { + if (auth_info->flavors[i] == match) + return true; + } + return false; +} +EXPORT_SYMBOL_GPL(nfs_auth_info_match); + +/* + * Ensure that a specified authtype in ctx->auth_info is supported by + * the server. Returns 0 and sets ctx->selected_flavor if it's ok, and + * -EACCES if not. + */ +static int nfs_verify_authflavors(struct nfs_fs_context *ctx, + rpc_authflavor_t *server_authlist, + unsigned int count) +{ + rpc_authflavor_t flavor = RPC_AUTH_MAXFLAVOR; + bool found_auth_null = false; + unsigned int i; + + /* + * If the sec= mount option is used, the specified flavor or AUTH_NULL + * must be in the list returned by the server. + * + * AUTH_NULL has a special meaning when it's in the server list - it + * means that the server will ignore the rpc creds, so any flavor + * can be used but still use the sec= that was specified. + * + * Note also that the MNT procedure in MNTv1 does not return a list + * of supported security flavors. In this case, nfs_mount() fabricates + * a security flavor list containing just AUTH_NULL. + */ + for (i = 0; i < count; i++) { + flavor = server_authlist[i]; + + if (nfs_auth_info_match(&ctx->auth_info, flavor)) + goto out; + + if (flavor == RPC_AUTH_NULL) + found_auth_null = true; + } + + if (found_auth_null) { + flavor = ctx->auth_info.flavors[0]; + goto out; + } + + dfprintk(MOUNT, + "NFS: specified auth flavors not supported by server\n"); + return -EACCES; + +out: + ctx->selected_flavor = flavor; + dfprintk(MOUNT, "NFS: using auth flavor %u\n", ctx->selected_flavor); + return 0; +} + +/* + * Use the remote server's MOUNT service to request the NFS file handle + * corresponding to the provided path. + */ +static int nfs_request_mount(struct fs_context *fc, + struct nfs_fh *root_fh, + rpc_authflavor_t *server_authlist, + unsigned int *server_authlist_len) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + struct nfs_mount_request request = { + .sap = &ctx->mount_server._address, + .dirpath = ctx->nfs_server.export_path, + .protocol = ctx->mount_server.protocol, + .fh = root_fh, + .noresvport = ctx->flags & NFS_MOUNT_NORESVPORT, + .auth_flav_len = server_authlist_len, + .auth_flavs = server_authlist, + .net = fc->net_ns, + }; + int status; + + if (ctx->mount_server.version == 0) { + switch (ctx->version) { + default: + ctx->mount_server.version = NFS_MNT3_VERSION; + break; + case 2: + ctx->mount_server.version = NFS_MNT_VERSION; + } + } + request.version = ctx->mount_server.version; + + if (ctx->mount_server.hostname) + request.hostname = ctx->mount_server.hostname; + else + request.hostname = ctx->nfs_server.hostname; + + /* + * Construct the mount server's address. + */ + if (ctx->mount_server.address.sa_family == AF_UNSPEC) { + memcpy(request.sap, &ctx->nfs_server._address, + ctx->nfs_server.addrlen); + ctx->mount_server.addrlen = ctx->nfs_server.addrlen; + } + request.salen = ctx->mount_server.addrlen; + nfs_set_port(request.sap, &ctx->mount_server.port, 0); + + /* + * Now ask the mount server to map our export path + * to a file handle. + */ + if ((request.protocol == XPRT_TRANSPORT_UDP) == + !(ctx->flags & NFS_MOUNT_TCP)) + /* + * NFS protocol and mount protocol are both UDP or neither UDP + * so timeouts are compatible. Use NFS timeouts for MOUNT + */ + status = nfs_mount(&request, ctx->timeo, ctx->retrans); + else + status = nfs_mount(&request, NFS_UNSPEC_TIMEO, NFS_UNSPEC_RETRANS); + if (status != 0) { + dfprintk(MOUNT, "NFS: unable to mount server %s, error %d\n", + request.hostname, status); + return status; + } + + return 0; +} + +static struct nfs_server *nfs_try_mount_request(struct fs_context *fc) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + int status; + unsigned int i; + bool tried_auth_unix = false; + bool auth_null_in_list = false; + struct nfs_server *server = ERR_PTR(-EACCES); + rpc_authflavor_t authlist[NFS_MAX_SECFLAVORS]; + unsigned int authlist_len = ARRAY_SIZE(authlist); + + /* make sure 'nolock'/'lock' override the 'local_lock' mount option */ + if (ctx->lock_status) { + if (ctx->lock_status == NFS_LOCK_NOLOCK) { + ctx->flags |= NFS_MOUNT_NONLM; + ctx->flags |= (NFS_MOUNT_LOCAL_FLOCK | NFS_MOUNT_LOCAL_FCNTL); + } else { + ctx->flags &= ~NFS_MOUNT_NONLM; + ctx->flags &= ~(NFS_MOUNT_LOCAL_FLOCK | NFS_MOUNT_LOCAL_FCNTL); + } + } + status = nfs_request_mount(fc, ctx->mntfh, authlist, &authlist_len); + if (status) + return ERR_PTR(status); + + /* + * Was a sec= authflavor specified in the options? First, verify + * whether the server supports it, and then just try to use it if so. + */ + if (ctx->auth_info.flavor_len > 0) { + status = nfs_verify_authflavors(ctx, authlist, authlist_len); + dfprintk(MOUNT, "NFS: using auth flavor %u\n", + ctx->selected_flavor); + if (status) + return ERR_PTR(status); + return ctx->nfs_mod->rpc_ops->create_server(fc); + } + + /* + * No sec= option was provided. RFC 2623, section 2.7 suggests we + * SHOULD prefer the flavor listed first. However, some servers list + * AUTH_NULL first. Avoid ever choosing AUTH_NULL. + */ + for (i = 0; i < authlist_len; ++i) { + rpc_authflavor_t flavor; + struct rpcsec_gss_info info; + + flavor = authlist[i]; + switch (flavor) { + case RPC_AUTH_UNIX: + tried_auth_unix = true; + break; + case RPC_AUTH_NULL: + auth_null_in_list = true; + continue; + default: + if (rpcauth_get_gssinfo(flavor, &info) != 0) + continue; + break; + } + dfprintk(MOUNT, "NFS: attempting to use auth flavor %u\n", flavor); + ctx->selected_flavor = flavor; + server = ctx->nfs_mod->rpc_ops->create_server(fc); + if (!IS_ERR(server)) + return server; + } + + /* + * Nothing we tried so far worked. At this point, give up if we've + * already tried AUTH_UNIX or if the server's list doesn't contain + * AUTH_NULL + */ + if (tried_auth_unix || !auth_null_in_list) + return server; + + /* Last chance! Try AUTH_UNIX */ + dfprintk(MOUNT, "NFS: attempting to use auth flavor %u\n", RPC_AUTH_UNIX); + ctx->selected_flavor = RPC_AUTH_UNIX; + return ctx->nfs_mod->rpc_ops->create_server(fc); +} + +int nfs_try_get_tree(struct fs_context *fc) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + + if (ctx->need_mount) + ctx->server = nfs_try_mount_request(fc); + else + ctx->server = ctx->nfs_mod->rpc_ops->create_server(fc); + + return nfs_get_tree_common(fc); +} +EXPORT_SYMBOL_GPL(nfs_try_get_tree); + + +#define NFS_REMOUNT_CMP_FLAGMASK ~(NFS_MOUNT_INTR \ + | NFS_MOUNT_SECURE \ + | NFS_MOUNT_TCP \ + | NFS_MOUNT_VER3 \ + | NFS_MOUNT_KERBEROS \ + | NFS_MOUNT_NONLM \ + | NFS_MOUNT_BROKEN_SUID \ + | NFS_MOUNT_STRICTLOCK \ + | NFS_MOUNT_LEGACY_INTERFACE) + +#define NFS_MOUNT_CMP_FLAGMASK (NFS_REMOUNT_CMP_FLAGMASK & \ + ~(NFS_MOUNT_UNSHARED | NFS_MOUNT_NORESVPORT)) + +static int +nfs_compare_remount_data(struct nfs_server *nfss, + struct nfs_fs_context *ctx) +{ + if ((ctx->flags ^ nfss->flags) & NFS_REMOUNT_CMP_FLAGMASK || + ctx->rsize != nfss->rsize || + ctx->wsize != nfss->wsize || + ctx->version != nfss->nfs_client->rpc_ops->version || + ctx->minorversion != nfss->nfs_client->cl_minorversion || + ctx->retrans != nfss->client->cl_timeout->to_retries || + !nfs_auth_info_match(&ctx->auth_info, nfss->client->cl_auth->au_flavor) || + ctx->acregmin != nfss->acregmin / HZ || + ctx->acregmax != nfss->acregmax / HZ || + ctx->acdirmin != nfss->acdirmin / HZ || + ctx->acdirmax != nfss->acdirmax / HZ || + ctx->timeo != (10U * nfss->client->cl_timeout->to_initval / HZ) || + (ctx->options & NFS_OPTION_FSCACHE) != (nfss->options & NFS_OPTION_FSCACHE) || + ctx->nfs_server.port != nfss->port || + ctx->nfs_server.addrlen != nfss->nfs_client->cl_addrlen || + !rpc_cmp_addr((struct sockaddr *)&ctx->nfs_server.address, + (struct sockaddr *)&nfss->nfs_client->cl_addr)) + return -EINVAL; + + return 0; +} + +int nfs_reconfigure(struct fs_context *fc) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + struct super_block *sb = fc->root->d_sb; + struct nfs_server *nfss = sb->s_fs_info; + int ret; + + sync_filesystem(sb); + + /* + * Userspace mount programs that send binary options generally send + * them populated with default values. We have no way to know which + * ones were explicitly specified. Fall back to legacy behavior and + * just return success. + */ + if (ctx->skip_reconfig_option_check) + return 0; + + /* + * noac is a special case. It implies -o sync, but that's not + * necessarily reflected in the mtab options. reconfigure_super + * will clear SB_SYNCHRONOUS if -o sync wasn't specified in the + * remount options, so we have to explicitly reset it. + */ + if (ctx->flags & NFS_MOUNT_NOAC) { + fc->sb_flags |= SB_SYNCHRONOUS; + fc->sb_flags_mask |= SB_SYNCHRONOUS; + } + + /* compare new mount options with old ones */ + ret = nfs_compare_remount_data(nfss, ctx); + if (ret) + return ret; + + return nfs_probe_server(nfss, NFS_FH(d_inode(fc->root))); +} +EXPORT_SYMBOL_GPL(nfs_reconfigure); + +/* + * Finish setting up an NFS superblock + */ +static void nfs_fill_super(struct super_block *sb, struct nfs_fs_context *ctx) +{ + struct nfs_server *server = NFS_SB(sb); + + sb->s_blocksize_bits = 0; + sb->s_blocksize = 0; + sb->s_xattr = server->nfs_client->cl_nfs_mod->xattr; + sb->s_op = server->nfs_client->cl_nfs_mod->sops; + if (server->bsize) + sb->s_blocksize = + nfs_block_size(server->bsize, &sb->s_blocksize_bits); + + switch (server->nfs_client->rpc_ops->version) { + case 2: + sb->s_time_gran = 1000; + sb->s_time_min = 0; + sb->s_time_max = U32_MAX; + break; + case 3: + /* + * The VFS shouldn't apply the umask to mode bits. + * We will do so ourselves when necessary. + */ + sb->s_flags |= SB_POSIXACL; + sb->s_time_gran = 1; + sb->s_time_min = 0; + sb->s_time_max = U32_MAX; + sb->s_export_op = &nfs_export_ops; + break; + case 4: + sb->s_iflags |= SB_I_NOUMASK; + sb->s_time_gran = 1; + sb->s_time_min = S64_MIN; + sb->s_time_max = S64_MAX; + if (server->caps & NFS_CAP_ATOMIC_OPEN_V1) + sb->s_export_op = &nfs_export_ops; + break; + } + + sb->s_magic = NFS_SUPER_MAGIC; + + /* We probably want something more informative here */ + snprintf(sb->s_id, sizeof(sb->s_id), + "%u:%u", MAJOR(sb->s_dev), MINOR(sb->s_dev)); + + if (sb->s_blocksize == 0) + sb->s_blocksize = nfs_block_bits(server->wsize, + &sb->s_blocksize_bits); + + nfs_super_set_maxbytes(sb, server->maxfilesize); + nfs_sysfs_move_server_to_sb(sb); + server->has_sec_mnt_opts = ctx->has_sec_mnt_opts; +} + +static int nfs_compare_mount_options(const struct super_block *s, const struct nfs_server *b, + const struct fs_context *fc) +{ + const struct nfs_server *a = s->s_fs_info; + const struct rpc_clnt *clnt_a = a->client; + const struct rpc_clnt *clnt_b = b->client; + + if ((s->s_flags & NFS_SB_MASK) != (fc->sb_flags & NFS_SB_MASK)) + goto Ebusy; + if (a->nfs_client != b->nfs_client) + goto Ebusy; + if ((a->flags ^ b->flags) & NFS_MOUNT_CMP_FLAGMASK) + goto Ebusy; + if (a->wsize != b->wsize) + goto Ebusy; + if (a->rsize != b->rsize) + goto Ebusy; + if (a->acregmin != b->acregmin) + goto Ebusy; + if (a->acregmax != b->acregmax) + goto Ebusy; + if (a->acdirmin != b->acdirmin) + goto Ebusy; + if (a->acdirmax != b->acdirmax) + goto Ebusy; + if (clnt_a->cl_auth->au_flavor != clnt_b->cl_auth->au_flavor) + goto Ebusy; + return 1; +Ebusy: + return 0; +} + +static int nfs_set_super(struct super_block *s, struct fs_context *fc) +{ + struct nfs_server *server = fc->s_fs_info; + int ret; + + set_default_d_op(s, server->nfs_client->rpc_ops->dentry_ops); + ret = set_anon_super(s, server); + if (ret == 0) + server->s_dev = s->s_dev; + return ret; +} + +static int nfs_compare_super_address(struct nfs_server *server1, + struct nfs_server *server2) +{ + struct rpc_xprt *xprt1, *xprt2; + struct sockaddr *sap1, *sap2; + + rcu_read_lock(); + + xprt1 = rcu_dereference(server1->client->cl_xprt); + xprt2 = rcu_dereference(server2->client->cl_xprt); + + if (!net_eq(xprt1->xprt_net, xprt2->xprt_net)) + goto out_unlock; + + rcu_read_unlock(); + + sap1 = (struct sockaddr *)&server1->nfs_client->cl_addr; + sap2 = (struct sockaddr *)&server2->nfs_client->cl_addr; + + if (sap1->sa_family != sap2->sa_family) + return 0; + + switch (sap1->sa_family) { + case AF_INET: { + struct sockaddr_in *sin1 = (struct sockaddr_in *)sap1; + struct sockaddr_in *sin2 = (struct sockaddr_in *)sap2; + if (sin1->sin_addr.s_addr != sin2->sin_addr.s_addr) + return 0; + if (sin1->sin_port != sin2->sin_port) + return 0; + break; + } + case AF_INET6: { + struct sockaddr_in6 *sin1 = (struct sockaddr_in6 *)sap1; + struct sockaddr_in6 *sin2 = (struct sockaddr_in6 *)sap2; + if (!ipv6_addr_equal(&sin1->sin6_addr, &sin2->sin6_addr)) + return 0; + if (sin1->sin6_port != sin2->sin6_port) + return 0; + break; + } + default: + return 0; + } + + return 1; + +out_unlock: + rcu_read_unlock(); + return 0; +} + +static int nfs_compare_userns(const struct nfs_server *old, + const struct nfs_server *new) +{ + const struct user_namespace *oldns = &init_user_ns; + const struct user_namespace *newns = &init_user_ns; + + if (old->client && old->client->cl_cred) + oldns = old->client->cl_cred->user_ns; + if (new->client && new->client->cl_cred) + newns = new->client->cl_cred->user_ns; + if (oldns != newns) + return 0; + return 1; +} + +static int nfs_compare_super(struct super_block *sb, struct fs_context *fc) +{ + struct nfs_server *server = fc->s_fs_info, *old = NFS_SB(sb); + + if (!nfs_compare_super_address(old, server)) + return 0; + /* Note: NFS_MOUNT_UNSHARED == NFS4_MOUNT_UNSHARED */ + if (old->flags & NFS_MOUNT_UNSHARED) + return 0; + if (memcmp(&old->fsid, &server->fsid, sizeof(old->fsid)) != 0) + return 0; + if (!nfs_compare_userns(old, server)) + return 0; + if ((old->has_sec_mnt_opts || fc->security) && + security_sb_mnt_opts_compat(sb, fc->security)) + return 0; + return nfs_compare_mount_options(sb, server, fc); +} + +#ifdef CONFIG_NFS_FSCACHE +static int nfs_get_cache_cookie(struct super_block *sb, + struct nfs_fs_context *ctx) +{ + struct nfs_server *nfss = NFS_SB(sb); + char *uniq = NULL; + int ulen = 0; + + nfss->fscache = NULL; + + if (!ctx) + return 0; + + if (ctx->clone_data.sb) { + struct nfs_server *mnt_s = NFS_SB(ctx->clone_data.sb); + if (!(mnt_s->options & NFS_OPTION_FSCACHE)) + return 0; + if (mnt_s->fscache_uniq) { + uniq = mnt_s->fscache_uniq; + ulen = strlen(uniq); + } + } else { + if (!(ctx->options & NFS_OPTION_FSCACHE)) + return 0; + if (ctx->fscache_uniq) { + uniq = ctx->fscache_uniq; + ulen = strlen(ctx->fscache_uniq); + } + } + + return nfs_fscache_get_super_cookie(sb, uniq, ulen); +} +#else +static int nfs_get_cache_cookie(struct super_block *sb, + struct nfs_fs_context *ctx) +{ + return 0; +} +#endif + +int nfs_get_tree_common(struct fs_context *fc) +{ + struct nfs_fs_context *ctx = nfs_fc2context(fc); + struct super_block *s; + int (*compare_super)(struct super_block *, struct fs_context *) = nfs_compare_super; + struct nfs_server *server = ctx->server; + int error; + + ctx->server = NULL; + if (IS_ERR(server)) + return PTR_ERR(server); + + if (server->flags & NFS_MOUNT_UNSHARED) + compare_super = NULL; + + /* -o noac implies -o sync */ + if (server->flags & NFS_MOUNT_NOAC) + fc->sb_flags |= SB_SYNCHRONOUS; + + /* Get a superblock - note that we may end up sharing one that already exists */ + fc->s_fs_info = server; + s = sget_fc(fc, compare_super, nfs_set_super); + fc->s_fs_info = NULL; + if (IS_ERR(s)) { + error = PTR_ERR(s); + nfs_errorf(fc, "NFS: Couldn't get superblock"); + goto out_err_nosb; + } + + if (s->s_fs_info != server) { + nfs_free_server(server); + server = NULL; + } else { + error = super_setup_bdi_name(s, "%u:%u", MAJOR(server->s_dev), + MINOR(server->s_dev)); + if (error) + goto error_splat_super; + s->s_bdi->io_pages = server->rpages; + server->super = s; + } + + if (!s->s_root) { + /* initial superblock/root creation */ + nfs_fill_super(s, ctx); + error = nfs_get_cache_cookie(s, ctx); + if (error < 0) + goto error_splat_super; + } + + error = nfs_get_root(s, fc); + if (error < 0) { + nfs_errorf(fc, "NFS: Couldn't get root dentry"); + goto error_splat_super; + } + + s->s_flags |= SB_ACTIVE; + error = 0; + +out: + return error; + +out_err_nosb: + nfs_free_server(server); + goto out; +error_splat_super: + deactivate_locked_super(s); + goto out; +} + +/* + * Destroy an NFS superblock + */ +void nfs_kill_super(struct super_block *s) +{ + struct nfs_server *server = NFS_SB(s); + + nfs_sysfs_move_sb_to_server(server); + kill_anon_super(s); + + nfs_fscache_release_super_cookie(s); + + nfs_free_server(server); +} +EXPORT_SYMBOL_GPL(nfs_kill_super); + +#if IS_ENABLED(CONFIG_NFS_V4) + +/* + * NFS v4 module parameters need to stay in the + * NFS client for backwards compatibility + */ +unsigned int nfs_callback_set_tcpport; +unsigned short nfs_callback_nr_threads; +/* Default cache timeout is 10 minutes */ +unsigned int nfs_idmap_cache_timeout = 600; +/* Turn off NFSv4 uid/gid mapping when using AUTH_SYS */ +bool nfs4_disable_idmapping = true; +unsigned short max_session_slots = NFS4_DEF_SLOT_TABLE_SIZE; +unsigned short max_session_cb_slots = NFS4_DEF_CB_SLOT_TABLE_SIZE; +unsigned short send_implementation_id = 1; +char nfs4_client_id_uniquifier[NFS4_CLIENT_ID_UNIQ_LEN] = ""; +bool recover_lost_locks = false; +short nfs_delay_retrans = -1; + +EXPORT_SYMBOL_GPL(nfs_callback_nr_threads); +EXPORT_SYMBOL_GPL(nfs_callback_set_tcpport); +EXPORT_SYMBOL_GPL(nfs_idmap_cache_timeout); +EXPORT_SYMBOL_GPL(nfs4_disable_idmapping); +EXPORT_SYMBOL_GPL(max_session_slots); +EXPORT_SYMBOL_GPL(max_session_cb_slots); +EXPORT_SYMBOL_GPL(send_implementation_id); +EXPORT_SYMBOL_GPL(nfs4_client_id_uniquifier); +EXPORT_SYMBOL_GPL(recover_lost_locks); +EXPORT_SYMBOL_GPL(nfs_delay_retrans); + +#define NFS_CALLBACK_MAXPORTNR (65535U) + +static int param_set_portnr(const char *val, const struct kernel_param *kp) +{ + unsigned long num; + int ret; + + if (!val) + return -EINVAL; + ret = kstrtoul(val, 0, &num); + if (ret || num > NFS_CALLBACK_MAXPORTNR) + return -EINVAL; + *((unsigned int *)kp->arg) = num; + return 0; +} +static const struct kernel_param_ops param_ops_portnr = { + .set = param_set_portnr, + .get = param_get_uint, +}; +#define param_check_portnr(name, p) __param_check(name, p, unsigned int) + +module_param_named(callback_tcpport, nfs_callback_set_tcpport, portnr, 0644); +module_param_named(callback_nr_threads, nfs_callback_nr_threads, ushort, 0644); +MODULE_PARM_DESC(callback_nr_threads, "Number of threads that will be " + "assigned to the NFSv4 callback channels."); +module_param(nfs_idmap_cache_timeout, int, 0644); +module_param(nfs4_disable_idmapping, bool, 0644); +module_param_string(nfs4_unique_id, nfs4_client_id_uniquifier, + NFS4_CLIENT_ID_UNIQ_LEN, 0600); +MODULE_PARM_DESC(nfs4_disable_idmapping, + "Turn off NFSv4 idmapping when using 'sec=sys'"); +module_param(max_session_slots, ushort, 0644); +MODULE_PARM_DESC(max_session_slots, "Maximum number of outstanding NFSv4.1 " + "requests the client will negotiate"); +module_param(max_session_cb_slots, ushort, 0644); +MODULE_PARM_DESC(max_session_cb_slots, "Maximum number of parallel NFSv4.1 " + "callbacks the client will process for a given server"); +module_param(send_implementation_id, ushort, 0644); +MODULE_PARM_DESC(send_implementation_id, + "Send implementation ID with NFSv4.1 exchange_id"); +MODULE_PARM_DESC(nfs4_unique_id, "nfs_client_id4 uniquifier string"); + +module_param(recover_lost_locks, bool, 0644); +MODULE_PARM_DESC(recover_lost_locks, + "If the server reports that a lock might be lost, " + "try to recover it risking data corruption."); + +module_param_named(delay_retrans, nfs_delay_retrans, short, 0644); +MODULE_PARM_DESC(delay_retrans, + "Unless negative, specifies the number of times the NFSv4 " + "client retries a request before returning an EAGAIN error, " + "after a reply of NFS4ERR_DELAY from the server."); +#endif /* CONFIG_NFS_V4 */ diff --git a/fs/nfs/symlink.c b/fs/nfs/symlink.c new file mode 100644 index 000000000..74a072896 --- /dev/null +++ b/fs/nfs/symlink.c @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/nfs/symlink.c + * + * Copyright (C) 1992 Rick Sladkey + * + * Optimization changes Copyright (C) 1994 Florian La Roche + * + * Jun 7 1999, cache symlink lookups in the page cache. -DaveM + * + * nfs symlink handling code + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +/* Symlink caching in the page cache is even more simplistic + * and straight-forward than readdir caching. + */ + +static int nfs_symlink_filler(struct file *file, struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + int error; + + error = NFS_PROTO(inode)->readlink(inode, &folio->page, 0, PAGE_SIZE); + folio_end_read(folio, error == 0); + return error; +} + +static const char *nfs_get_link(struct dentry *dentry, + struct inode *inode, + struct delayed_call *done) +{ + struct folio *folio; + void *err; + + if (!dentry) { + err = ERR_PTR(nfs_revalidate_mapping_rcu(inode)); + if (err) + return err; + folio = filemap_get_folio(inode->i_mapping, 0); + if (IS_ERR(folio)) + return ERR_PTR(-ECHILD); + if (!folio_test_uptodate(folio)) { + folio_put(folio); + return ERR_PTR(-ECHILD); + } + } else { + err = ERR_PTR(nfs_revalidate_mapping(inode, inode->i_mapping)); + if (err) + return err; + folio = read_cache_folio(&inode->i_data, 0, nfs_symlink_filler, + NULL); + if (IS_ERR(folio)) + return ERR_CAST(folio); + } + set_delayed_call(done, page_put_link, folio); + return folio_address(folio); +} + +/* + * symlinks can't do much... + */ +const struct inode_operations nfs_symlink_inode_operations = { + .get_link = nfs_get_link, + .getattr = nfs_getattr, + .setattr = nfs_setattr, + .fileattr_get = nfs_fileattr_get, +}; diff --git a/fs/nfs/sysctl.c b/fs/nfs/sysctl.c new file mode 100644 index 000000000..f579df0e8 --- /dev/null +++ b/fs/nfs/sysctl.c @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/nfs/sysctl.c + * + * Sysctl interface to NFS parameters + */ +#include +#include +#include +#include +#include +#include +#include + +static struct ctl_table_header *nfs_callback_sysctl_table; + +static const struct ctl_table nfs_cb_sysctls[] = { + { + .procname = "nfs_mountpoint_timeout", + .data = &nfs_mountpoint_expiry_timeout, + .maxlen = sizeof(nfs_mountpoint_expiry_timeout), + .mode = 0644, + .proc_handler = proc_dointvec_jiffies, + }, + { + .procname = "nfs_congestion_kb", + .data = &nfs_congestion_kb, + .maxlen = sizeof(nfs_congestion_kb), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +}; + +int nfs_register_sysctl(void) +{ + nfs_callback_sysctl_table = register_sysctl("fs/nfs", nfs_cb_sysctls); + if (nfs_callback_sysctl_table == NULL) + return -ENOMEM; + return 0; +} + +void nfs_unregister_sysctl(void) +{ + unregister_sysctl_table(nfs_callback_sysctl_table); + nfs_callback_sysctl_table = NULL; +} diff --git a/fs/nfs/sysfs.c b/fs/nfs/sysfs.c new file mode 100644 index 000000000..3a197252a --- /dev/null +++ b/fs/nfs/sysfs.c @@ -0,0 +1,474 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2019 Hammerspace Inc + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" +#include "nfs4_fs.h" +#include "netns.h" +#include "sysfs.h" + +static struct kset *nfs_kset; + +static void nfs_kset_release(struct kobject *kobj) +{ + struct kset *kset = container_of(kobj, struct kset, kobj); + kfree(kset); +} + +static const struct kobj_ns_type_operations *nfs_netns_object_child_ns_type( + const struct kobject *kobj) +{ + return &net_ns_type_operations; +} + +static struct kobj_type nfs_kset_type = { + .release = nfs_kset_release, + .sysfs_ops = &kobj_sysfs_ops, + .child_ns_type = nfs_netns_object_child_ns_type, +}; + +int nfs_sysfs_init(void) +{ + int ret; + + nfs_kset = kzalloc_obj(*nfs_kset); + if (!nfs_kset) + return -ENOMEM; + + ret = kobject_set_name(&nfs_kset->kobj, "nfs"); + if (ret) { + kfree(nfs_kset); + return ret; + } + + nfs_kset->kobj.parent = fs_kobj; + nfs_kset->kobj.ktype = &nfs_kset_type; + nfs_kset->kobj.kset = NULL; + + ret = kset_register(nfs_kset); + if (ret) { + kfree(nfs_kset); + return ret; + } + + return 0; +} + +void nfs_sysfs_exit(void) +{ + kset_unregister(nfs_kset); +} + +static ssize_t nfs_netns_identifier_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + struct nfs_netns_client *c = container_of(kobj, + struct nfs_netns_client, + kobject); + ssize_t ret; + + rcu_read_lock(); + ret = sysfs_emit(buf, "%s\n", rcu_dereference(c->identifier)); + rcu_read_unlock(); + return ret; +} + +/* Strip trailing '\n' */ +static size_t nfs_string_strip(const char *c, size_t len) +{ + while (len > 0 && c[len-1] == '\n') + --len; + return len; +} + +static ssize_t nfs_netns_identifier_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct nfs_netns_client *c = container_of(kobj, + struct nfs_netns_client, + kobject); + const char *old; + char *p; + size_t len; + + len = nfs_string_strip(buf, min_t(size_t, count, CONTAINER_ID_MAXLEN)); + if (!len) + return 0; + p = kmemdup_nul(buf, len, GFP_KERNEL); + if (!p) + return -ENOMEM; + old = rcu_dereference_protected(xchg(&c->identifier, (char __rcu *)p), 1); + if (old) { + synchronize_rcu(); + kfree(old); + } + return count; +} + +static void nfs_netns_client_release(struct kobject *kobj) +{ + struct nfs_netns_client *c = container_of(kobj, + struct nfs_netns_client, + kobject); + + kfree(rcu_dereference_raw(c->identifier)); +} + +static const struct ns_common *nfs_netns_client_namespace(const struct kobject *kobj) +{ + return to_ns_common(container_of(kobj, struct nfs_netns_client, + kobject)->net); +} + +static struct kobj_attribute nfs_netns_client_id = __ATTR(identifier, + 0644, nfs_netns_identifier_show, nfs_netns_identifier_store); + +static struct attribute *nfs_netns_client_attrs[] = { + &nfs_netns_client_id.attr, + NULL, +}; +ATTRIBUTE_GROUPS(nfs_netns_client); + +static struct kobj_type nfs_netns_client_type = { + .release = nfs_netns_client_release, + .default_groups = nfs_netns_client_groups, + .sysfs_ops = &kobj_sysfs_ops, + .namespace = nfs_netns_client_namespace, +}; + +static void nfs_netns_object_release(struct kobject *kobj) +{ + struct nfs_netns_client *c = container_of(kobj, + struct nfs_netns_client, + nfs_net_kobj); + kfree(c); +} + +static const struct ns_common *nfs_netns_namespace(const struct kobject *kobj) +{ + return to_ns_common(container_of(kobj, struct nfs_netns_client, + nfs_net_kobj)->net); +} + +static struct kobj_type nfs_netns_object_type = { + .release = nfs_netns_object_release, + .sysfs_ops = &kobj_sysfs_ops, + .namespace = nfs_netns_namespace, +}; + +static struct nfs_netns_client *nfs_netns_client_alloc(struct kobject *parent, + struct net *net) +{ + struct nfs_netns_client *p; + + p = kzalloc_obj(*p); + if (p) { + p->net = net; + p->kobject.kset = nfs_kset; + p->nfs_net_kobj.kset = nfs_kset; + + if (kobject_init_and_add(&p->nfs_net_kobj, &nfs_netns_object_type, + parent, "net") != 0) { + kobject_put(&p->nfs_net_kobj); + return NULL; + } + + if (kobject_init_and_add(&p->kobject, &nfs_netns_client_type, + &p->nfs_net_kobj, "nfs_client") == 0) + return p; + + kobject_put(&p->kobject); + kobject_put(&p->nfs_net_kobj); + } + return NULL; +} + +void nfs_netns_sysfs_setup(struct nfs_net *netns, struct net *net) +{ + struct nfs_netns_client *clp; + + clp = nfs_netns_client_alloc(&nfs_kset->kobj, net); + if (clp) { + netns->nfs_client = clp; + kobject_uevent(&clp->kobject, KOBJ_ADD); + } +} + +void nfs_netns_sysfs_destroy(struct nfs_net *netns) +{ + struct nfs_netns_client *clp = netns->nfs_client; + + if (clp) { + kobject_uevent(&clp->kobject, KOBJ_REMOVE); + kobject_del(&clp->kobject); + kobject_put(&clp->kobject); + kobject_del(&clp->nfs_net_kobj); + kobject_put(&clp->nfs_net_kobj); + netns->nfs_client = NULL; + } +} + +static bool shutdown_match_client(const struct rpc_task *task, const void *data) +{ + return true; +} + +static void shutdown_client(struct rpc_clnt *clnt) +{ + clnt->cl_shutdown = 1; + rpc_cancel_tasks(clnt, -EIO, shutdown_match_client, NULL); +} + +/* + * Shut down the nfs_client only once all the superblocks + * have been shut down. + */ +static void shutdown_nfs_client(struct nfs_client *clp) +{ + struct nfs_server *server; + rcu_read_lock(); + list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) { + if (!(server->flags & NFS_MOUNT_SHUTDOWN)) { + rcu_read_unlock(); + return; + } + } + rcu_read_unlock(); + nfs_mark_client_ready(clp, -EIO); + shutdown_client(clp->cl_rpcclient); +} + +static ssize_t +shutdown_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + struct nfs_server *server = container_of(kobj, struct nfs_server, kobj); + bool shutdown = server->flags & NFS_MOUNT_SHUTDOWN; + return sysfs_emit(buf, "%d\n", shutdown); +} + +static ssize_t +shutdown_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct nfs_server *server; + int ret, val; + + server = container_of(kobj, struct nfs_server, kobj); + + ret = kstrtoint(buf, 0, &val); + if (ret) + return ret; + + if (val != 1) + return -EINVAL; + + /* already shut down? */ + if (server->flags & NFS_MOUNT_SHUTDOWN) + goto out; + + server->flags |= NFS_MOUNT_SHUTDOWN; + shutdown_client(server->client); + + if (!IS_ERR(server->client_acl)) + shutdown_client(server->client_acl); + + if (server->nlm_host) + nlmclnt_shutdown_rpc_clnt(server->nlm_host); +out: + shutdown_nfs_client(server->nfs_client); + return count; +} + +static struct kobj_attribute nfs_sysfs_attr_shutdown = __ATTR_RW(shutdown); + +#if IS_ENABLED(CONFIG_NFS_V4) +static ssize_t +implid_domain_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + struct nfs_server *server = container_of(kobj, struct nfs_server, kobj); + struct nfs41_impl_id *impl_id = server->nfs_client->cl_implid; + + if (!impl_id || strlen(impl_id->domain) == 0) + return 0; //sysfs_emit(buf, ""); + return sysfs_emit(buf, "%s\n", impl_id->domain); +} + +static struct kobj_attribute nfs_sysfs_attr_implid_domain = __ATTR_RO(implid_domain); + + +static ssize_t +implid_name_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + struct nfs_server *server = container_of(kobj, struct nfs_server, kobj); + struct nfs41_impl_id *impl_id = server->nfs_client->cl_implid; + + if (!impl_id || strlen(impl_id->name) == 0) + return 0; //sysfs_emit(buf, ""); + return sysfs_emit(buf, "%s\n", impl_id->name); +} + +static struct kobj_attribute nfs_sysfs_attr_implid_name = __ATTR_RO(implid_name); + +#endif /* IS_ENABLED(CONFIG_NFS_V4) */ + +#define RPC_CLIENT_NAME_SIZE 64 + +void nfs_sysfs_link_rpc_client(struct nfs_server *server, + struct rpc_clnt *clnt, const char *uniq) +{ + char name[RPC_CLIENT_NAME_SIZE]; + int ret; + + strscpy(name, clnt->cl_program->name, sizeof(name)); + strncat(name, uniq ? uniq : "", sizeof(name) - strlen(name) - 1); + strncat(name, "_client", sizeof(name) - strlen(name) - 1); + + ret = sysfs_create_link_nowarn(&server->kobj, + &clnt->cl_sysfs->kobject, name); + if (ret < 0) + pr_warn("NFS: can't create link to %s in sysfs (%d)\n", + name, ret); +} +EXPORT_SYMBOL_GPL(nfs_sysfs_link_rpc_client); + +static void nfs_sysfs_sb_release(struct kobject *kobj) +{ + /* no-op: why? see lib/kobject.c kobject_cleanup() */ +} + +static const struct ns_common *nfs_netns_server_namespace(const struct kobject *kobj) +{ + return to_ns_common(container_of(kobj, struct nfs_server, + kobj)->nfs_client->cl_net); +} + +static struct kobj_type nfs_sb_ktype = { + .release = nfs_sysfs_sb_release, + .sysfs_ops = &kobj_sysfs_ops, + .namespace = nfs_netns_server_namespace, + .child_ns_type = nfs_netns_object_child_ns_type, +}; + +#if IS_ENABLED(CONFIG_NFS_V4) +static void nfs_sysfs_add_nfsv41_server(struct nfs_server *server) +{ + int ret; + + if (!server->nfs_client->cl_implid) + return; + + ret = sysfs_create_file_ns(&server->kobj, &nfs_sysfs_attr_implid_domain.attr, + nfs_netns_server_namespace(&server->kobj)); + if (ret < 0) + pr_warn("NFS: sysfs_create_file_ns for server-%d failed (%d)\n", + server->s_sysfs_id, ret); + + ret = sysfs_create_file_ns(&server->kobj, &nfs_sysfs_attr_implid_name.attr, + nfs_netns_server_namespace(&server->kobj)); + if (ret < 0) + pr_warn("NFS: sysfs_create_file_ns for server-%d failed (%d)\n", + server->s_sysfs_id, ret); +} +#else /* CONFIG_NFS_V4 */ +static inline void nfs_sysfs_add_nfsv41_server(struct nfs_server *server) +{ +} +#endif /* CONFIG_NFS_V4 */ + +#if IS_ENABLED(CONFIG_NFS_LOCALIO) + +static ssize_t +localio_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + struct nfs_server *server = container_of(kobj, struct nfs_server, kobj); + bool localio = nfs_server_is_local(server->nfs_client); + return sysfs_emit(buf, "%d\n", localio); +} + +static struct kobj_attribute nfs_sysfs_attr_localio = __ATTR_RO(localio); + +static void nfs_sysfs_add_nfs_localio_server(struct nfs_server *server) +{ + int ret = sysfs_create_file_ns(&server->kobj, &nfs_sysfs_attr_localio.attr, + nfs_netns_server_namespace(&server->kobj)); + if (ret < 0) + pr_warn("NFS: sysfs_create_file_ns for server-%d failed (%d)\n", + server->s_sysfs_id, ret); +} +#else +static inline void nfs_sysfs_add_nfs_localio_server(struct nfs_server *server) +{ +} +#endif /* IS_ENABLED(CONFIG_NFS_LOCALIO) */ + +void nfs_sysfs_add_server(struct nfs_server *server) +{ + int ret; + + ret = kobject_init_and_add(&server->kobj, &nfs_sb_ktype, + &nfs_kset->kobj, "server-%d", server->s_sysfs_id); + if (ret < 0) { + pr_warn("NFS: nfs sysfs add server-%d failed (%d)\n", + server->s_sysfs_id, ret); + return; + } + ret = sysfs_create_file_ns(&server->kobj, &nfs_sysfs_attr_shutdown.attr, + nfs_netns_server_namespace(&server->kobj)); + if (ret < 0) + pr_warn("NFS: sysfs_create_file_ns for server-%d failed (%d)\n", + server->s_sysfs_id, ret); + + nfs_sysfs_add_nfsv41_server(server); + nfs_sysfs_add_nfs_localio_server(server); +} +EXPORT_SYMBOL_GPL(nfs_sysfs_add_server); + +void nfs_sysfs_move_server_to_sb(struct super_block *s) +{ + struct nfs_server *server = s->s_fs_info; + int ret; + + ret = kobject_rename(&server->kobj, s->s_id); + if (ret < 0) + pr_warn("NFS: rename sysfs %s failed (%d)\n", + server->kobj.name, ret); +} + +void nfs_sysfs_move_sb_to_server(struct nfs_server *server) +{ + const char *s; + int ret = -ENOMEM; + + s = kasprintf(GFP_KERNEL, "server-%d", server->s_sysfs_id); + if (s) { + ret = kobject_rename(&server->kobj, s); + kfree(s); + } + if (ret < 0) + pr_warn("NFS: rename sysfs %s failed (%d)\n", + server->kobj.name, ret); +} + +/* unlink, not dec-ref */ +void nfs_sysfs_remove_server(struct nfs_server *server) +{ + kobject_del(&server->kobj); +} diff --git a/fs/nfs/sysfs.h b/fs/nfs/sysfs.h new file mode 100644 index 000000000..c5d1990ca --- /dev/null +++ b/fs/nfs/sysfs.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2019 Hammerspace Inc + */ + +#ifndef __NFS_SYSFS_H +#define __NFS_SYSFS_H + +#define CONTAINER_ID_MAXLEN (64) + +struct nfs_netns_client { + struct kobject kobject; + struct kobject nfs_net_kobj; + struct net *net; + const char __rcu *identifier; +}; + +extern struct kobject *nfs_net_kobj; + +extern int nfs_sysfs_init(void); +extern void nfs_sysfs_exit(void); + +void nfs_netns_sysfs_setup(struct nfs_net *netns, struct net *net); +void nfs_netns_sysfs_destroy(struct nfs_net *netns); + +void nfs_sysfs_link_rpc_client(struct nfs_server *server, + struct rpc_clnt *clnt, const char *sysfs_prefix); +void nfs_sysfs_add_server(struct nfs_server *s); +void nfs_sysfs_move_server_to_sb(struct super_block *s); +void nfs_sysfs_move_sb_to_server(struct nfs_server *s); +void nfs_sysfs_remove_server(struct nfs_server *s); + +#endif diff --git a/fs/nfs/unlink.c b/fs/nfs/unlink.c new file mode 100644 index 000000000..b57cfaa4d --- /dev/null +++ b/fs/nfs/unlink.c @@ -0,0 +1,532 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/nfs/unlink.c + * + * nfs sillydelete handling + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" +#include "nfs4_fs.h" +#include "iostat.h" +#include "delegation.h" + +#include "nfstrace.h" + +/** + * nfs_free_unlinkdata - release data from a sillydelete operation. + * @data: pointer to unlink structure. + */ +static void +nfs_free_unlinkdata(struct nfs_unlinkdata *data) +{ + put_cred(data->cred); + kfree(data->args.name.name); + kfree(data); +} + +/** + * nfs_async_unlink_done - Sillydelete post-processing + * @task: rpc_task of the sillydelete + * @calldata: pointer to nfs_unlinkdata + * + * Do the directory attribute update. + */ +static void nfs_async_unlink_done(struct rpc_task *task, void *calldata) +{ + struct nfs_unlinkdata *data = calldata; + struct inode *dir = d_inode(data->dentry->d_parent); + + trace_nfs_sillyrename_unlink(data, task->tk_status); + if (!NFS_PROTO(dir)->unlink_done(task, dir)) + rpc_restart_call_prepare(task); +} + +/** + * nfs_async_unlink_release - Release the sillydelete data. + * @calldata: struct nfs_unlinkdata to release + * + * We need to call nfs_put_unlinkdata as a 'tk_release' task since the + * rpc_task would be freed too. + */ +static void nfs_async_unlink_release(void *calldata) +{ + struct nfs_unlinkdata *data = calldata; + struct dentry *dentry = data->dentry; + struct super_block *sb = dentry->d_sb; + + up_read_non_owner(&NFS_I(d_inode(dentry->d_parent))->rmdir_sem); + d_lookup_done(dentry); + nfs_free_unlinkdata(data); + dput(dentry); + nfs_sb_deactive(sb); +} + +static void nfs_unlink_prepare(struct rpc_task *task, void *calldata) +{ + struct nfs_unlinkdata *data = calldata; + struct inode *dir = d_inode(data->dentry->d_parent); + NFS_PROTO(dir)->unlink_rpc_prepare(task, data); +} + +static const struct rpc_call_ops nfs_unlink_ops = { + .rpc_call_done = nfs_async_unlink_done, + .rpc_release = nfs_async_unlink_release, + .rpc_call_prepare = nfs_unlink_prepare, +}; + +static void nfs_do_call_unlink(struct inode *inode, struct nfs_unlinkdata *data) +{ + struct rpc_message msg = { + .rpc_argp = &data->args, + .rpc_resp = &data->res, + .rpc_cred = data->cred, + }; + struct rpc_task_setup task_setup_data = { + .rpc_message = &msg, + .callback_ops = &nfs_unlink_ops, + .callback_data = data, + .workqueue = nfsiod_workqueue, + .flags = RPC_TASK_ASYNC | RPC_TASK_CRED_NOREF, + }; + struct rpc_task *task; + struct inode *dir = d_inode(data->dentry->d_parent); + + if (nfs_server_capable(inode, NFS_CAP_MOVEABLE)) + task_setup_data.flags |= RPC_TASK_MOVEABLE; + + nfs_sb_active(dir->i_sb); + data->args.fh = NFS_FH(dir); + nfs_fattr_init(data->res.dir_attr); + + NFS_PROTO(dir)->unlink_setup(&msg, data->dentry, inode); + + task_setup_data.rpc_client = NFS_CLIENT(dir); + task = rpc_run_task(&task_setup_data); + if (!IS_ERR(task)) + rpc_put_task_async(task); +} + +static int nfs_call_unlink(struct dentry *dentry, struct inode *inode, struct nfs_unlinkdata *data) +{ + struct inode *dir = d_inode(dentry->d_parent); + struct dentry *alias; + + down_read_non_owner(&NFS_I(dir)->rmdir_sem); + alias = d_alloc_parallel(dentry->d_parent, &data->args.name); + if (IS_ERR(alias)) { + up_read_non_owner(&NFS_I(dir)->rmdir_sem); + return 0; + } + if (!d_in_lookup(alias)) { + int ret; + void *devname_garbage = NULL; + + /* + * Hey, we raced with lookup... See if we need to transfer + * the sillyrename information to the aliased dentry. + */ + spin_lock(&alias->d_lock); + if (d_really_is_positive(alias) && + !nfs_compare_fh(NFS_FH(inode), NFS_FH(d_inode(alias))) && + !(alias->d_flags & DCACHE_NFSFS_RENAMED)) { + devname_garbage = alias->d_fsdata; + alias->d_fsdata = data; + alias->d_flags |= DCACHE_NFSFS_RENAMED; + ret = 1; + } else + ret = 0; + spin_unlock(&alias->d_lock); + dput(alias); + up_read_non_owner(&NFS_I(dir)->rmdir_sem); + /* + * If we'd displaced old cached devname, free it. At that + * point dentry is definitely not a root, so we won't need + * that anymore. + */ + kfree(devname_garbage); + return ret; + } + data->dentry = alias; + nfs_do_call_unlink(inode, data); + return 1; +} + +/** + * nfs_async_unlink - asynchronous unlinking of a file + * @dentry: parent directory of dentry + * @name: name of dentry to unlink + */ +static int +nfs_async_unlink(struct dentry *dentry, const struct qstr *name) +{ + struct nfs_unlinkdata *data; + int status = -ENOMEM; + void *devname_garbage = NULL; + + data = kzalloc_obj(*data); + if (data == NULL) + goto out; + data->args.name.name = kstrdup(name->name, GFP_KERNEL); + if (!data->args.name.name) + goto out_free; + data->args.name.len = name->len; + + data->cred = get_current_cred(); + data->res.dir_attr = &data->dir_attr; + + status = -EBUSY; + spin_lock(&dentry->d_lock); + if (dentry->d_flags & DCACHE_NFSFS_RENAMED) + goto out_unlock; + dentry->d_flags |= DCACHE_NFSFS_RENAMED; + devname_garbage = dentry->d_fsdata; + dentry->d_fsdata = data; + spin_unlock(&dentry->d_lock); + /* + * If we'd displaced old cached devname, free it. At that + * point dentry is definitely not a root, so we won't need + * that anymore. + */ + kfree(devname_garbage); + return 0; +out_unlock: + spin_unlock(&dentry->d_lock); + put_cred(data->cred); + kfree(data->args.name.name); +out_free: + kfree(data); +out: + return status; +} + +/** + * nfs_complete_unlink - Initialize completion of the sillydelete + * @dentry: dentry to delete + * @inode: inode + * + * Since we're most likely to be called by dentry_iput(), we + * only use the dentry to find the sillydelete. We then copy the name + * into the qstr. + */ +void +nfs_complete_unlink(struct dentry *dentry, struct inode *inode) +{ + struct nfs_unlinkdata *data; + + spin_lock(&dentry->d_lock); + dentry->d_flags &= ~DCACHE_NFSFS_RENAMED; + data = dentry->d_fsdata; + dentry->d_fsdata = NULL; + spin_unlock(&dentry->d_lock); + + NFS_PROTO(inode)->return_delegation(inode); + + if (NFS_STALE(inode) || !nfs_call_unlink(dentry, inode, data)) + nfs_free_unlinkdata(data); +} + +/* Cancel a queued async unlink. Called when a sillyrename run fails. */ +static void +nfs_cancel_async_unlink(struct dentry *dentry) +{ + spin_lock(&dentry->d_lock); + if (dentry->d_flags & DCACHE_NFSFS_RENAMED) { + struct nfs_unlinkdata *data = dentry->d_fsdata; + + dentry->d_flags &= ~DCACHE_NFSFS_RENAMED; + dentry->d_fsdata = NULL; + spin_unlock(&dentry->d_lock); + nfs_free_unlinkdata(data); + return; + } + spin_unlock(&dentry->d_lock); +} + +/** + * nfs_async_rename_done - Sillyrename post-processing + * @task: rpc_task of the sillyrename + * @calldata: nfs_renamedata for the sillyrename + * + * Do the directory attribute updates and the d_move + */ +static void nfs_async_rename_done(struct rpc_task *task, void *calldata) +{ + struct nfs_renamedata *data = calldata; + struct inode *old_dir = data->old_dir; + struct inode *new_dir = data->new_dir; + struct dentry *old_dentry = data->old_dentry; + + trace_nfs_async_rename_done(old_dir, old_dentry, + new_dir, data->new_dentry, task->tk_status); + if (!NFS_PROTO(old_dir)->rename_done(task, old_dir, new_dir)) { + rpc_restart_call_prepare(task); + return; + } + + if (data->complete) + data->complete(task, data); +} + +/** + * nfs_async_rename_release - Release the sillyrename data. + * @calldata: the struct nfs_renamedata to be released + */ +static void nfs_async_rename_release(void *calldata) +{ + struct nfs_renamedata *data = calldata; + struct super_block *sb = data->old_dir->i_sb; + + if (d_really_is_positive(data->old_dentry)) + nfs_mark_for_revalidate(d_inode(data->old_dentry)); + + /* The result of the rename is unknown. Play it safe by + * forcing a new lookup */ + if (data->cancelled) { + spin_lock(&data->old_dir->i_lock); + nfs_force_lookup_revalidate(data->old_dir); + spin_unlock(&data->old_dir->i_lock); + if (data->new_dir != data->old_dir) { + spin_lock(&data->new_dir->i_lock); + nfs_force_lookup_revalidate(data->new_dir); + spin_unlock(&data->new_dir->i_lock); + } + } + + dput(data->old_dentry); + dput(data->new_dentry); + iput(data->old_dir); + iput(data->new_dir); + nfs_sb_deactive(sb); + put_cred(data->cred); + kfree(data); +} + +static void nfs_rename_prepare(struct rpc_task *task, void *calldata) +{ + struct nfs_renamedata *data = calldata; + NFS_PROTO(data->old_dir)->rename_rpc_prepare(task, data); +} + +static const struct rpc_call_ops nfs_rename_ops = { + .rpc_call_done = nfs_async_rename_done, + .rpc_release = nfs_async_rename_release, + .rpc_call_prepare = nfs_rename_prepare, +}; + +/** + * nfs_async_rename - perform an asynchronous rename operation + * @old_dir: directory that currently holds the dentry to be renamed + * @new_dir: target directory for the rename + * @old_dentry: original dentry to be renamed + * @new_dentry: dentry to which the old_dentry should be renamed + * @complete: Function to run on successful completion + * + * It's expected that valid references to the dentries and inodes are held + */ +struct rpc_task * +nfs_async_rename(struct inode *old_dir, struct inode *new_dir, + struct dentry *old_dentry, struct dentry *new_dentry, + void (*complete)(struct rpc_task *, struct nfs_renamedata *)) +{ + struct nfs_renamedata *data; + struct rpc_message msg = { }; + struct rpc_task_setup task_setup_data = { + .rpc_message = &msg, + .callback_ops = &nfs_rename_ops, + .workqueue = nfsiod_workqueue, + .rpc_client = NFS_CLIENT(old_dir), + .flags = RPC_TASK_ASYNC | RPC_TASK_CRED_NOREF, + }; + + if (nfs_server_capable(old_dir, NFS_CAP_MOVEABLE) && + nfs_server_capable(new_dir, NFS_CAP_MOVEABLE)) + task_setup_data.flags |= RPC_TASK_MOVEABLE; + + data = kzalloc_obj(*data); + if (data == NULL) + return ERR_PTR(-ENOMEM); + task_setup_data.task = &data->task; + task_setup_data.callback_data = data; + + data->cred = get_current_cred(); + + msg.rpc_argp = &data->args; + msg.rpc_resp = &data->res; + msg.rpc_cred = data->cred; + + /* set up nfs_renamedata */ + data->old_dir = old_dir; + ihold(old_dir); + data->new_dir = new_dir; + ihold(new_dir); + data->old_dentry = dget(old_dentry); + data->new_dentry = dget(new_dentry); + nfs_fattr_init(&data->old_fattr); + nfs_fattr_init(&data->new_fattr); + data->complete = complete; + + /* set up nfs_renameargs */ + data->args.old_dir = NFS_FH(old_dir); + data->args.old_name = &old_dentry->d_name; + data->args.new_dir = NFS_FH(new_dir); + data->args.new_name = &new_dentry->d_name; + + /* set up nfs_renameres */ + data->res.old_fattr = &data->old_fattr; + data->res.new_fattr = &data->new_fattr; + + nfs_sb_active(old_dir->i_sb); + + NFS_PROTO(data->old_dir)->rename_setup(&msg, old_dentry, new_dentry, + old_dir == new_dir ? old_dir : NULL); + + return rpc_run_task(&task_setup_data); +} + +/* + * Perform tasks needed when a sillyrename is done such as cancelling the + * queued async unlink if it failed. + */ +static void +nfs_complete_sillyrename(struct rpc_task *task, struct nfs_renamedata *data) +{ + struct dentry *dentry = data->old_dentry; + + if (task->tk_status != 0) { + nfs_cancel_async_unlink(dentry); + return; + } +} + +#define SILLYNAME_PREFIX ".nfs" +#define SILLYNAME_PREFIX_LEN ((unsigned)sizeof(SILLYNAME_PREFIX) - 1) +#define SILLYNAME_FILEID_LEN ((unsigned)sizeof(u64) << 1) +#define SILLYNAME_COUNTER_LEN ((unsigned)sizeof(unsigned int) << 1) +#define SILLYNAME_LEN (SILLYNAME_PREFIX_LEN + \ + SILLYNAME_FILEID_LEN + \ + SILLYNAME_COUNTER_LEN) + +/** + * nfs_sillyrename - Perform a silly-rename of a dentry + * @dir: inode of directory that contains dentry + * @dentry: dentry to be sillyrenamed + * + * NFSv2/3 is stateless and the server doesn't know when the client is + * holding a file open. To prevent application problems when a file is + * unlinked while it's still open, the client performs a "silly-rename". + * That is, it renames the file to a hidden file in the same directory, + * and only performs the unlink once the last reference to it is put. + * + * The final cleanup is done during dentry_iput. + * + * (Note: NFSv4 is stateful, and has opens, so in theory an NFSv4 server + * could take responsibility for keeping open files referenced. The server + * would also need to ensure that opened-but-deleted files were kept over + * reboots. However, we may not assume a server does so. (RFC 5661 + * does provide an OPEN4_RESULT_PRESERVE_UNLINKED flag that a server can + * use to advertise that it does this; some day we may take advantage of + * it.)) + */ +int +nfs_sillyrename(struct inode *dir, struct dentry *dentry) +{ + static unsigned int sillycounter; + unsigned char silly[SILLYNAME_LEN + 1]; + unsigned long long fileid; + struct dentry *sdentry; + struct inode *inode = d_inode(dentry); + struct rpc_task *task; + int error = -EBUSY; + + dfprintk(VFS, "NFS: silly-rename(%pd2, ct=%d)\n", + dentry, d_count(dentry)); + nfs_inc_stats(dir, NFSIOS_SILLYRENAME); + + /* + * We don't allow a dentry to be silly-renamed twice. + */ + if (dentry->d_flags & DCACHE_NFSFS_RENAMED) + goto out; + + fileid = d_inode(dentry)->i_ino; + + sdentry = NULL; + do { + dput(sdentry); + sillycounter++; + scnprintf(silly, sizeof(silly), + SILLYNAME_PREFIX "%0*llx%0*x", + SILLYNAME_FILEID_LEN, fileid, + SILLYNAME_COUNTER_LEN, sillycounter); + + dfprintk(VFS, "NFS: trying to rename %pd to %s\n", + dentry, silly); + + sdentry = lookup_noperm(&QSTR(silly), dentry->d_parent); + /* + * N.B. Better to return EBUSY here ... it could be + * dangerous to delete the file while it's in use. + */ + if (IS_ERR(sdentry)) + goto out; + } while (d_inode(sdentry) != NULL); /* need negative lookup */ + + ihold(inode); + + /* queue unlink first. Can't do this from rpc_release as it + * has to allocate memory + */ + error = nfs_async_unlink(dentry, &sdentry->d_name); + if (error) + goto out_dput; + + /* run the rename task, undo unlink if it fails */ + task = nfs_async_rename(dir, dir, dentry, sdentry, + nfs_complete_sillyrename); + if (IS_ERR(task)) { + error = -EBUSY; + nfs_cancel_async_unlink(dentry); + goto out_dput; + } + + /* wait for the RPC task to complete, unless a SIGKILL intervenes */ + error = rpc_wait_for_completion_task(task); + if (error == 0) + error = task->tk_status; + switch (error) { + case 0: + /* The rename succeeded */ + nfs_set_verifier(dentry, nfs_save_change_attribute(dir)); + spin_lock(&inode->i_lock); + NFS_I(inode)->attr_gencount = nfs_inc_attr_generation_counter(); + nfs_set_cache_invalid(inode, NFS_INO_INVALID_CHANGE | + NFS_INO_INVALID_CTIME | + NFS_INO_REVAL_FORCED); + spin_unlock(&inode->i_lock); + d_move(dentry, sdentry); + break; + case -ERESTARTSYS: + /* The result of the rename is unknown. Play it safe by + * forcing a new lookup */ + d_drop(dentry); + d_drop(sdentry); + } + rpc_put_task(task); +out_dput: + iput(inode); + dput(sdentry); +out: + return error; +} diff --git a/fs/nfs/write.c b/fs/nfs/write.c new file mode 100644 index 000000000..d2b03ceae --- /dev/null +++ b/fs/nfs/write.c @@ -0,0 +1,2212 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/nfs/write.c + * + * Write file data over NFS. + * + * Copyright (C) 1996, 1997, Olaf Kirch + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "delegation.h" +#include "internal.h" +#include "iostat.h" +#include "nfs4_fs.h" +#include "fscache.h" +#include "pnfs.h" + +#include "nfstrace.h" + +#define NFSDBG_FACILITY NFSDBG_PAGECACHE + +#define MIN_POOL_WRITE (32) +#define MIN_POOL_COMMIT (4) + +struct nfs_io_completion { + void (*complete)(void *data); + void *data; + struct kref refcount; +}; + +/* + * Local function declarations + */ +static void nfs_redirty_request(struct nfs_page *req); +static const struct rpc_call_ops nfs_commit_ops; +static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops; +static const struct nfs_commit_completion_ops nfs_commit_completion_ops; +static const struct nfs_rw_ops nfs_rw_write_ops; +static void nfs_inode_remove_request(struct nfs_page *req); +static void nfs_clear_request_commit(struct nfs_commit_info *cinfo, + struct nfs_page *req); +static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo, + struct inode *inode); + +static struct kmem_cache *nfs_wdata_cachep; +static mempool_t *nfs_wdata_mempool; +static struct kmem_cache *nfs_cdata_cachep; +static mempool_t *nfs_commit_mempool; + +struct nfs_commit_data *nfs_commitdata_alloc(void) +{ + struct nfs_commit_data *p; + + p = kmem_cache_zalloc(nfs_cdata_cachep, nfs_io_gfp_mask()); + if (!p) { + p = mempool_alloc(nfs_commit_mempool, GFP_NOWAIT); + if (!p) + return NULL; + memset(p, 0, sizeof(*p)); + } + INIT_LIST_HEAD(&p->pages); + return p; +} +EXPORT_SYMBOL_GPL(nfs_commitdata_alloc); + +void nfs_commit_free(struct nfs_commit_data *p) +{ + mempool_free(p, nfs_commit_mempool); +} +EXPORT_SYMBOL_GPL(nfs_commit_free); + +static struct nfs_pgio_header *nfs_writehdr_alloc(void) +{ + struct nfs_pgio_header *p; + + p = kmem_cache_zalloc(nfs_wdata_cachep, nfs_io_gfp_mask()); + if (!p) { + p = mempool_alloc(nfs_wdata_mempool, GFP_NOWAIT); + if (!p) + return NULL; + memset(p, 0, sizeof(*p)); + } + p->rw_mode = FMODE_WRITE; + return p; +} + +static void nfs_writehdr_free(struct nfs_pgio_header *hdr) +{ + mempool_free(hdr, nfs_wdata_mempool); +} + +static struct nfs_io_completion *nfs_io_completion_alloc(gfp_t gfp_flags) +{ + return kmalloc_obj(struct nfs_io_completion, gfp_flags); +} + +static void nfs_io_completion_init(struct nfs_io_completion *ioc, + void (*complete)(void *), void *data) +{ + ioc->complete = complete; + ioc->data = data; + kref_init(&ioc->refcount); +} + +static void nfs_io_completion_release(struct kref *kref) +{ + struct nfs_io_completion *ioc = container_of(kref, + struct nfs_io_completion, refcount); + ioc->complete(ioc->data); + kfree(ioc); +} + +static void nfs_io_completion_get(struct nfs_io_completion *ioc) +{ + if (ioc != NULL) + kref_get(&ioc->refcount); +} + +static void nfs_io_completion_put(struct nfs_io_completion *ioc) +{ + if (ioc != NULL) + kref_put(&ioc->refcount, nfs_io_completion_release); +} + +static void +nfs_page_set_inode_ref(struct nfs_page *req, struct inode *inode) +{ + if (!test_and_set_bit(PG_INODE_REF, &req->wb_flags)) { + kref_get(&req->wb_kref); + atomic_long_inc(&NFS_I(inode)->nrequests); + } +} + +static void nfs_cancel_remove_inode(struct nfs_page *req, struct inode *inode) +{ + if (test_and_clear_bit(PG_REMOVE, &req->wb_flags)) + nfs_page_set_inode_ref(req, inode); +} + +/** + * nfs_folio_find_head_request - find head request associated with a folio + * @folio: pointer to folio + * + * must be called while holding the inode lock. + * + * returns matching head request with reference held, or NULL if not found. + */ +static struct nfs_page *nfs_folio_find_head_request(struct folio *folio) +{ + struct address_space *mapping = folio->mapping; + struct nfs_page *req; + + if (!folio_test_private(folio)) + return NULL; + spin_lock(&mapping->i_private_lock); + req = folio->private; + if (req) { + WARN_ON_ONCE(req->wb_head != req); + kref_get(&req->wb_kref); + } + spin_unlock(&mapping->i_private_lock); + return req; +} + +/* Adjust the file length if we're writing beyond the end */ +static void nfs_grow_file(struct folio *folio, unsigned int offset, + unsigned int count) +{ + struct inode *inode = folio->mapping->host; + loff_t end, i_size; + pgoff_t end_index; + + spin_lock(&inode->i_lock); + i_size = i_size_read(inode); + end_index = ((i_size - 1) >> folio_shift(folio)) << folio_order(folio); + if (i_size > 0 && folio->index < end_index) + goto out; + end = folio_pos(folio) + (loff_t)offset + (loff_t)count; + if (i_size >= end) + goto out; + trace_nfs_size_grow(inode, end); + i_size_write(inode, end); + NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_SIZE; + nfs_inc_stats(inode, NFSIOS_EXTENDWRITE); +out: + /* Atomically update timestamps if they are delegated to us. */ + nfs_update_delegated_mtime_locked(inode); + spin_unlock(&inode->i_lock); + nfs_fscache_invalidate(inode, 0); +} + +/* A writeback failed: mark the page as bad, and invalidate the page cache */ +static void nfs_set_pageerror(struct address_space *mapping) +{ + struct inode *inode = mapping->host; + + nfs_zap_mapping(mapping->host, mapping); + /* Force file size revalidation */ + spin_lock(&inode->i_lock); + nfs_set_cache_invalid(inode, NFS_INO_REVAL_FORCED | + NFS_INO_INVALID_CHANGE | + NFS_INO_INVALID_SIZE); + spin_unlock(&inode->i_lock); +} + +static void nfs_mapping_set_error(struct folio *folio, int error) +{ + struct address_space *mapping = folio->mapping; + + filemap_set_wb_err(mapping, error); + if (mapping->host) + errseq_set(&mapping->host->i_sb->s_wb_err, + error == -ENOSPC ? -ENOSPC : -EIO); + nfs_set_pageerror(mapping); +} + +/* + * nfs_page_covers_folio + * @req: struct nfs_page + * + * Return true if the request covers the whole folio. + * Note that the caller should ensure all subrequests have been joined + */ +static bool nfs_page_group_covers_page(struct nfs_page *req) +{ + unsigned int len = nfs_folio_length(nfs_page_to_folio(req)); + + return req->wb_pgbase == 0 && req->wb_bytes == len; +} + +/* We can set the PG_uptodate flag if we see that a write request + * covers the full page. + */ +static void nfs_mark_uptodate(struct nfs_page *req) +{ + struct folio *folio = nfs_page_to_folio(req); + + if (folio_test_uptodate(folio)) + return; + if (!nfs_page_group_covers_page(req)) + return; + folio_mark_uptodate(folio); +} + +static int wb_priority(struct writeback_control *wbc) +{ + int ret = 0; + + if (wbc->sync_mode == WB_SYNC_ALL) + ret = FLUSH_COND_STABLE; + return ret; +} + +/* + * NFS congestion control + */ + +int nfs_congestion_kb; + +#define NFS_CONGESTION_ON_THRESH (nfs_congestion_kb >> (PAGE_SHIFT-10)) +#define NFS_CONGESTION_OFF_THRESH \ + (NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2)) + +static void nfs_folio_set_writeback(struct folio *folio) +{ + struct nfs_server *nfss = NFS_SERVER(folio->mapping->host); + + folio_start_writeback(folio); + if (atomic_long_inc_return(&nfss->writeback) > NFS_CONGESTION_ON_THRESH) + nfss->write_congested = 1; +} + +static void nfs_folio_end_writeback(struct folio *folio) +{ + struct nfs_server *nfss = NFS_SERVER(folio->mapping->host); + + folio_end_writeback_no_dropbehind(folio); + if (atomic_long_dec_return(&nfss->writeback) < + NFS_CONGESTION_OFF_THRESH) { + nfss->write_congested = 0; + wake_up_all(&nfss->write_congestion_wait); + } +} + +static void nfs_page_end_writeback(struct nfs_page *req) +{ + if (nfs_page_group_sync_on_bit(req, PG_WB_END)) { + nfs_unlock_request(req); + nfs_folio_end_writeback(nfs_page_to_folio(req)); + } else + nfs_unlock_request(req); +} + +/* + * nfs_destroy_unlinked_subrequests - destroy recently unlinked subrequests + * + * @destroy_list - request list (using wb_this_page) terminated by @old_head + * @old_head - the old head of the list + * + * All subrequests must be locked and removed from all lists, so at this point + * they are only "active" in this function, and possibly in nfs_wait_on_request + * with a reference held by some other context. + */ +static void +nfs_destroy_unlinked_subrequests(struct nfs_page *destroy_list, + struct nfs_page *old_head, + struct inode *inode) +{ + while (destroy_list) { + struct nfs_page *subreq = destroy_list; + + destroy_list = (subreq->wb_this_page == old_head) ? + NULL : subreq->wb_this_page; + + /* Note: lock subreq in order to change subreq->wb_head */ + nfs_page_set_headlock(subreq); + WARN_ON_ONCE(old_head != subreq->wb_head); + + /* make sure old group is not used */ + subreq->wb_this_page = subreq; + subreq->wb_head = subreq; + + clear_bit(PG_REMOVE, &subreq->wb_flags); + + /* Note: races with nfs_page_group_destroy() */ + if (!kref_read(&subreq->wb_kref)) { + /* Check if we raced with nfs_page_group_destroy() */ + if (test_and_clear_bit(PG_TEARDOWN, &subreq->wb_flags)) { + nfs_page_clear_headlock(subreq); + nfs_free_request(subreq); + } else + nfs_page_clear_headlock(subreq); + continue; + } + nfs_page_clear_headlock(subreq); + + nfs_release_request(old_head); + + if (test_and_clear_bit(PG_INODE_REF, &subreq->wb_flags)) { + nfs_release_request(subreq); + atomic_long_dec(&NFS_I(inode)->nrequests); + } + + /* subreq is now totally disconnected from page group or any + * write / commit lists. last chance to wake any waiters */ + nfs_unlock_and_release_request(subreq); + } +} + +/* + * nfs_join_page_group - destroy subrequests of the head req + * @head: the page used to lookup the "page group" of nfs_page structures + * @inode: Inode to which the request belongs. + * + * This function joins all sub requests to the head request by first + * locking all requests in the group, cancelling any pending operations + * and finally updating the head request to cover the whole range covered by + * the (former) group. All subrequests are removed from any write or commit + * lists, unlinked from the group and destroyed. + */ +void nfs_join_page_group(struct nfs_page *head, struct nfs_commit_info *cinfo, + struct inode *inode) +{ + struct nfs_page *subreq; + struct nfs_page *destroy_list = NULL; + unsigned int pgbase, off, bytes; + + pgbase = head->wb_pgbase; + bytes = head->wb_bytes; + off = head->wb_offset; + for (subreq = head->wb_this_page; subreq != head; + subreq = subreq->wb_this_page) { + /* Subrequests should always form a contiguous range */ + if (pgbase > subreq->wb_pgbase) { + off -= pgbase - subreq->wb_pgbase; + bytes += pgbase - subreq->wb_pgbase; + pgbase = subreq->wb_pgbase; + } + bytes = max(subreq->wb_pgbase + subreq->wb_bytes + - pgbase, bytes); + } + + /* Set the head request's range to cover the former page group */ + head->wb_pgbase = pgbase; + head->wb_bytes = bytes; + head->wb_offset = off; + + /* Now that all requests are locked, make sure they aren't on any list. + * Commit list removal accounting is done after locks are dropped */ + subreq = head; + do { + nfs_clear_request_commit(cinfo, subreq); + subreq = subreq->wb_this_page; + } while (subreq != head); + + /* unlink subrequests from head, destroy them later */ + if (head->wb_this_page != head) { + /* destroy list will be terminated by head */ + destroy_list = head->wb_this_page; + head->wb_this_page = head; + } + + nfs_destroy_unlinked_subrequests(destroy_list, head, inode); +} + +/** + * nfs_wait_on_request - Wait for a request to complete. + * @req: request to wait upon. + * + * Interruptible by fatal signals only. + * The user is responsible for holding a count on the request. + */ +static int nfs_wait_on_request(struct nfs_page *req) +{ + if (!test_bit(PG_BUSY, &req->wb_flags)) + return 0; + set_bit(PG_CONTENDED2, &req->wb_flags); + smp_mb__after_atomic(); + return wait_on_bit_io(&req->wb_flags, PG_BUSY, + TASK_UNINTERRUPTIBLE); +} + +/* + * nfs_unroll_locks - unlock all newly locked reqs and wait on @req + * @head: head request of page group, must be holding head lock + * @req: request that couldn't lock and needs to wait on the req bit lock + * + * This is a helper function for nfs_lock_and_join_requests + * returns 0 on success, < 0 on error. + */ +static void +nfs_unroll_locks(struct nfs_page *head, struct nfs_page *req) +{ + struct nfs_page *tmp; + + /* relinquish all the locks successfully grabbed this run */ + for (tmp = head->wb_this_page ; tmp != req; tmp = tmp->wb_this_page) { + if (!kref_read(&tmp->wb_kref)) + continue; + nfs_unlock_and_release_request(tmp); + } +} + +/* + * nfs_page_group_lock_subreq - try to lock a subrequest + * @head: head request of page group + * @subreq: request to lock + * + * This is a helper function for nfs_lock_and_join_requests which + * must be called with the head request and page group both locked. + * On error, it returns with the page group unlocked. + */ +static int +nfs_page_group_lock_subreq(struct nfs_page *head, struct nfs_page *subreq) +{ + int ret; + + if (!kref_get_unless_zero(&subreq->wb_kref)) + return 0; + while (!nfs_lock_request(subreq)) { + nfs_page_group_unlock(head); + ret = nfs_wait_on_request(subreq); + if (!ret) + ret = nfs_page_group_lock(head); + if (ret < 0) { + nfs_unroll_locks(head, subreq); + nfs_release_request(subreq); + return ret; + } + } + return 0; +} + +/* + * nfs_lock_and_join_requests - join all subreqs to the head req + * @folio: the folio used to lookup the "page group" of nfs_page structures + * + * This function joins all sub requests to the head request by first + * locking all requests in the group, cancelling any pending operations + * and finally updating the head request to cover the whole range covered by + * the (former) group. All subrequests are removed from any write or commit + * lists, unlinked from the group and destroyed. + * + * Returns a locked, referenced pointer to the head request - which after + * this call is guaranteed to be the only request associated with the page. + * Returns NULL if no requests are found for @folio, or a ERR_PTR if an + * error was encountered. + */ +static struct nfs_page *nfs_lock_and_join_requests(struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + struct nfs_page *head, *subreq; + struct nfs_commit_info cinfo; + int ret; + + /* + * A reference is taken only on the head request which acts as a + * reference to the whole page group - the group will not be destroyed + * until the head reference is released. + */ +retry: + head = nfs_folio_find_head_request(folio); + if (!head) + return NULL; + + while (!nfs_lock_request(head)) { + ret = nfs_wait_on_request(head); + if (ret < 0) { + nfs_release_request(head); + return ERR_PTR(ret); + } + } + + ret = nfs_page_group_lock(head); + if (ret < 0) + goto out_unlock; + + /* Ensure that nobody removed the request before we locked it */ + if (head != folio->private) { + nfs_page_group_unlock(head); + nfs_unlock_and_release_request(head); + goto retry; + } + + nfs_cancel_remove_inode(head, inode); + + /* lock each request in the page group */ + for (subreq = head->wb_this_page; + subreq != head; + subreq = subreq->wb_this_page) { + ret = nfs_page_group_lock_subreq(head, subreq); + if (ret < 0) + goto out_unlock; + } + + nfs_page_group_unlock(head); + + nfs_init_cinfo_from_inode(&cinfo, inode); + nfs_join_page_group(head, &cinfo, inode); + return head; + +out_unlock: + nfs_unlock_and_release_request(head); + return ERR_PTR(ret); +} + +static void nfs_write_error(struct nfs_page *req, int error) +{ + trace_nfs_write_error(nfs_page_to_inode(req), req, error); + nfs_mapping_set_error(nfs_page_to_folio(req), error); + nfs_inode_remove_request(req); + nfs_page_end_writeback(req); + nfs_release_request(req); +} + +/* + * Find an associated nfs write request, and prepare to flush it out + * May return an error if the user signalled nfs_wait_on_request(). + */ +static int nfs_do_writepage(struct folio *folio, struct writeback_control *wbc, + struct nfs_pageio_descriptor *pgio) +{ + struct nfs_page *req; + int ret; + + nfs_pageio_cond_complete(pgio, folio->index); + + req = nfs_lock_and_join_requests(folio); + if (!req) + return 0; + if (IS_ERR(req)) + return PTR_ERR(req); + + trace_nfs_do_writepage(req); + nfs_folio_set_writeback(folio); + WARN_ON_ONCE(test_bit(PG_CLEAN, &req->wb_flags)); + + /* If there is a fatal error that covers this write, just exit */ + ret = pgio->pg_error; + if (nfs_error_is_fatal_on_server(ret)) + goto out_launder; + + if (!nfs_pageio_add_request(pgio, req)) { + ret = pgio->pg_error; + /* + * Remove the problematic req upon fatal errors on the server + */ + if (nfs_error_is_fatal_on_server(ret)) + goto out_launder; + folio_redirty_for_writepage(wbc, folio); + nfs_redirty_request(req); + pgio->pg_error = 0; + return ret; + } + + nfs_add_stats(folio->mapping->host, NFSIOS_WRITEPAGES, 1); + return 0; + +out_launder: + nfs_write_error(req, ret); + return 0; +} + +/* + * Write an mmapped page to the server. + */ +static int nfs_writepage_locked(struct folio *folio, + struct writeback_control *wbc) +{ + struct nfs_pageio_descriptor pgio; + struct inode *inode = folio->mapping->host; + int err; + + nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE); + nfs_pageio_init_write(&pgio, inode, 0, false, + &nfs_async_write_completion_ops); + err = nfs_do_writepage(folio, wbc, &pgio); + pgio.pg_error = 0; + nfs_pageio_complete(&pgio); + return err; +} + +static void nfs_io_completion_commit(void *inode) +{ + nfs_commit_inode(inode, 0); +} + +int nfs_writepages(struct address_space *mapping, struct writeback_control *wbc) +{ + struct inode *inode = mapping->host; + struct nfs_pageio_descriptor pgio; + struct nfs_io_completion *ioc = NULL; + unsigned int mntflags = NFS_SERVER(inode)->flags; + struct nfs_server *nfss = NFS_SERVER(inode); + int priority = 0; + int err; + + trace_nfs_writepages(inode, wbc->range_start, wbc->range_end - wbc->range_start); + + /* Wait with writeback until write congestion eases */ + if (wbc->sync_mode == WB_SYNC_NONE && nfss->write_congested) { + err = wait_event_killable(nfss->write_congestion_wait, + nfss->write_congested == 0); + if (err) + goto out_err; + } + + nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGES); + + if (!(mntflags & NFS_MOUNT_WRITE_EAGER) || wbc->for_kupdate || + wbc->for_background || wbc->for_sync) { + ioc = nfs_io_completion_alloc(GFP_KERNEL); + if (ioc) + nfs_io_completion_init(ioc, nfs_io_completion_commit, + inode); + priority = wb_priority(wbc); + } + + do { + struct folio *folio = NULL; + + nfs_pageio_init_write(&pgio, inode, priority, false, + &nfs_async_write_completion_ops); + pgio.pg_io_completion = ioc; + while ((folio = writeback_iter(mapping, wbc, folio, &err))) { + err = nfs_do_writepage(folio, wbc, &pgio); + folio_unlock(folio); + } + pgio.pg_error = 0; + nfs_pageio_complete(&pgio); + if (err == -EAGAIN && mntflags & NFS_MOUNT_SOFTERR) + break; + } while (err < 0 && !nfs_error_is_fatal(err)); + nfs_io_completion_put(ioc); + + if (err > 0) + err = 0; +out_err: + trace_nfs_writepages_done(inode, wbc->range_start, wbc->range_end - wbc->range_start, err); + return err; +} + +/* + * Insert a write request into an inode + */ +static void nfs_inode_add_request(struct nfs_page *req) +{ + struct folio *folio = nfs_page_to_folio(req); + struct address_space *mapping = folio->mapping; + struct nfs_inode *nfsi = NFS_I(mapping->host); + + WARN_ON_ONCE(req->wb_this_page != req); + + /* Lock the request! */ + nfs_lock_request(req); + spin_lock(&mapping->i_private_lock); + set_bit(PG_MAPPED, &req->wb_flags); + folio_set_private(folio); + folio->private = req; + spin_unlock(&mapping->i_private_lock); + atomic_long_inc(&nfsi->nrequests); + /* this a head request for a page group - mark it as having an + * extra reference so sub groups can follow suit. + * This flag also informs pgio layer when to bump nrequests when + * adding subrequests. */ + WARN_ON(test_and_set_bit(PG_INODE_REF, &req->wb_flags)); + kref_get(&req->wb_kref); +} + +/* + * Remove a write request from an inode + */ +static void nfs_inode_remove_request(struct nfs_page *req) +{ + struct nfs_inode *nfsi = NFS_I(nfs_page_to_inode(req)); + + nfs_page_group_lock(req); + if (nfs_page_group_sync_on_bit_locked(req, PG_REMOVE)) { + struct folio *folio = nfs_page_to_folio(req->wb_head); + struct address_space *mapping = folio->mapping; + + spin_lock(&mapping->i_private_lock); + if (likely(folio)) { + folio->private = NULL; + folio_clear_private(folio); + clear_bit(PG_MAPPED, &req->wb_head->wb_flags); + } + spin_unlock(&mapping->i_private_lock); + + folio_end_dropbehind(folio); + } + nfs_page_group_unlock(req); + + if (test_and_clear_bit(PG_INODE_REF, &req->wb_flags)) { + atomic_long_dec(&nfsi->nrequests); + nfs_release_request(req); + } +} + +static void nfs_mark_request_dirty(struct nfs_page *req) +{ + struct folio *folio = nfs_page_to_folio(req); + if (folio) + filemap_dirty_folio(folio_mapping(folio), folio); +} + +/** + * nfs_request_add_commit_list_locked - add request to a commit list + * @req: pointer to a struct nfs_page + * @dst: commit list head + * @cinfo: holds list lock and accounting info + * + * This sets the PG_CLEAN bit, updates the cinfo count of + * number of outstanding requests requiring a commit as well as + * the MM page stats. + * + * The caller must hold NFS_I(cinfo->inode)->commit_mutex, and the + * nfs_page lock. + */ +void +nfs_request_add_commit_list_locked(struct nfs_page *req, struct list_head *dst, + struct nfs_commit_info *cinfo) +{ + set_bit(PG_CLEAN, &req->wb_flags); + nfs_list_add_request(req, dst); + atomic_long_inc(&cinfo->mds->ncommit); +} +EXPORT_SYMBOL_GPL(nfs_request_add_commit_list_locked); + +/** + * nfs_request_add_commit_list - add request to a commit list + * @req: pointer to a struct nfs_page + * @cinfo: holds list lock and accounting info + * + * This sets the PG_CLEAN bit, updates the cinfo count of + * number of outstanding requests requiring a commit as well as + * the MM page stats. + * + * The caller must _not_ hold the cinfo->lock, but must be + * holding the nfs_page lock. + */ +void +nfs_request_add_commit_list(struct nfs_page *req, struct nfs_commit_info *cinfo) +{ + mutex_lock(&NFS_I(cinfo->inode)->commit_mutex); + nfs_request_add_commit_list_locked(req, &cinfo->mds->list, cinfo); + mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); + nfs_folio_mark_unstable(req, cinfo); +} +EXPORT_SYMBOL_GPL(nfs_request_add_commit_list); + +/** + * nfs_request_remove_commit_list - Remove request from a commit list + * @req: pointer to a nfs_page + * @cinfo: holds list lock and accounting info + * + * This clears the PG_CLEAN bit, and updates the cinfo's count of + * number of outstanding requests requiring a commit + * It does not update the MM page stats. + * + * The caller _must_ hold the cinfo->lock and the nfs_page lock. + */ +void +nfs_request_remove_commit_list(struct nfs_page *req, + struct nfs_commit_info *cinfo) +{ + if (!test_and_clear_bit(PG_CLEAN, &(req)->wb_flags)) + return; + nfs_list_remove_request(req); + atomic_long_dec(&cinfo->mds->ncommit); +} +EXPORT_SYMBOL_GPL(nfs_request_remove_commit_list); + +static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo, + struct inode *inode) +{ + cinfo->inode = inode; + cinfo->mds = &NFS_I(inode)->commit_info; + cinfo->ds = pnfs_get_ds_info(inode); + cinfo->dreq = NULL; + cinfo->completion_ops = &nfs_commit_completion_ops; +} + +void nfs_init_cinfo(struct nfs_commit_info *cinfo, + struct inode *inode, + struct nfs_direct_req *dreq) +{ + if (dreq) + nfs_init_cinfo_from_dreq(cinfo, dreq); + else + nfs_init_cinfo_from_inode(cinfo, inode); +} +EXPORT_SYMBOL_GPL(nfs_init_cinfo); + +/* + * Add a request to the inode's commit list. + */ +void +nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg, + struct nfs_commit_info *cinfo, u32 ds_commit_idx) +{ + if (pnfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx)) + return; + nfs_request_add_commit_list(req, cinfo); +} + +static void nfs_folio_clear_commit(struct nfs_page *req) +{ + struct folio *folio = nfs_page_to_folio(req); + + if (folio) { + long nr = DIV_ROUND_UP(req->wb_bytes, PAGE_SIZE); + + node_stat_mod_folio(folio, NR_WRITEBACK, -nr); + bdi_wb_stat_mod(folio->mapping->host, WB_WRITEBACK, -nr); + } +} + +/* Called holding the request lock on @req */ +static void nfs_clear_request_commit(struct nfs_commit_info *cinfo, + struct nfs_page *req) +{ + if (test_bit(PG_CLEAN, &req->wb_flags)) { + struct nfs_open_context *ctx = nfs_req_openctx(req); + struct inode *inode = d_inode(ctx->dentry); + + mutex_lock(&NFS_I(inode)->commit_mutex); + if (!pnfs_clear_request_commit(req, cinfo)) { + nfs_request_remove_commit_list(req, cinfo); + } + mutex_unlock(&NFS_I(inode)->commit_mutex); + nfs_folio_clear_commit(req); + } +} + +int nfs_write_need_commit(struct nfs_pgio_header *hdr) +{ + if (hdr->verf.committed == NFS_DATA_SYNC) + return hdr->lseg == NULL; + return hdr->verf.committed != NFS_FILE_SYNC; +} + +static void nfs_async_write_init(struct nfs_pgio_header *hdr) +{ + nfs_io_completion_get(hdr->io_completion); +} + +static void nfs_write_completion(struct nfs_pgio_header *hdr) +{ + struct nfs_commit_info cinfo; + unsigned long bytes = 0; + + if (test_bit(NFS_IOHDR_REDO, &hdr->flags)) + goto out; + nfs_init_cinfo_from_inode(&cinfo, hdr->inode); + while (!list_empty(&hdr->pages)) { + struct nfs_page *req = nfs_list_entry(hdr->pages.next); + + bytes += req->wb_bytes; + nfs_list_remove_request(req); + if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) && + (hdr->good_bytes < bytes)) { + trace_nfs_comp_error(hdr->inode, req, hdr->error); + nfs_mapping_set_error(nfs_page_to_folio(req), + hdr->error); + goto remove_req; + } + if (nfs_write_need_commit(hdr)) { + struct nfs_open_context *ctx = + req->wb_lock_context->open_context; + + /* Reset wb_nio, since the write was successful. */ + req->wb_nio = 0; + memcpy(&req->wb_verf, &hdr->verf.verifier, sizeof(req->wb_verf)); + clear_bit(NFS_CONTEXT_WRITE_SYNC, &ctx->flags); + nfs_mark_request_commit(req, hdr->lseg, &cinfo, + hdr->ds_commit_idx); + goto next; + } +remove_req: + nfs_inode_remove_request(req); +next: + nfs_page_end_writeback(req); + nfs_release_request(req); + } +out: + nfs_io_completion_put(hdr->io_completion); + hdr->release(hdr); +} + +unsigned long +nfs_reqs_to_commit(struct nfs_commit_info *cinfo) +{ + return atomic_long_read(&cinfo->mds->ncommit); +} + +/* NFS_I(cinfo->inode)->commit_mutex held by caller */ +int +nfs_scan_commit_list(struct list_head *src, struct list_head *dst, + struct nfs_commit_info *cinfo, int max) +{ + struct nfs_page *req, *tmp; + int ret = 0; + + list_for_each_entry_safe(req, tmp, src, wb_list) { + kref_get(&req->wb_kref); + if (!nfs_lock_request(req)) { + nfs_release_request(req); + continue; + } + nfs_request_remove_commit_list(req, cinfo); + clear_bit(PG_COMMIT_TO_DS, &req->wb_flags); + nfs_list_add_request(req, dst); + ret++; + if ((ret == max) && !cinfo->dreq) + break; + cond_resched(); + } + return ret; +} +EXPORT_SYMBOL_GPL(nfs_scan_commit_list); + +/* + * nfs_scan_commit - Scan an inode for commit requests + * @inode: NFS inode to scan + * @dst: mds destination list + * @cinfo: mds and ds lists of reqs ready to commit + * + * Moves requests from the inode's 'commit' request list. + * The requests are *not* checked to ensure that they form a contiguous set. + */ +int +nfs_scan_commit(struct inode *inode, struct list_head *dst, + struct nfs_commit_info *cinfo) +{ + int ret = 0; + + if (!atomic_long_read(&cinfo->mds->ncommit)) + return 0; + mutex_lock(&NFS_I(cinfo->inode)->commit_mutex); + if (atomic_long_read(&cinfo->mds->ncommit) > 0) { + const int max = INT_MAX; + + ret = nfs_scan_commit_list(&cinfo->mds->list, dst, + cinfo, max); + ret += pnfs_scan_commit_lists(inode, cinfo, max - ret); + } + mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex); + return ret; +} + +/* + * Search for an existing write request, and attempt to update + * it to reflect a new dirty region on a given page. + * + * If the attempt fails, then the existing request is flushed out + * to disk. + */ +static struct nfs_page *nfs_try_to_update_request(struct folio *folio, + unsigned int offset, + unsigned int bytes) +{ + struct nfs_page *req; + unsigned int rqend; + unsigned int end; + int error; + + trace_nfs_try_to_update_request(folio_inode(folio), offset, bytes); + end = offset + bytes; + + req = nfs_lock_and_join_requests(folio); + if (IS_ERR_OR_NULL(req)) + goto out; + + rqend = req->wb_offset + req->wb_bytes; + /* + * Tell the caller to flush out the request if + * the offsets are non-contiguous. + * Note: nfs_flush_incompatible() will already + * have flushed out requests having wrong owners. + */ + if (offset > rqend || end < req->wb_offset) + goto out_flushme; + + /* Okay, the request matches. Update the region */ + if (offset < req->wb_offset) { + req->wb_offset = offset; + req->wb_pgbase = offset; + } + if (end > rqend) + req->wb_bytes = end - req->wb_offset; + else + req->wb_bytes = rqend - req->wb_offset; + req->wb_nio = 0; +out: + trace_nfs_try_to_update_request_done(folio_inode(folio), offset, bytes, + PTR_ERR_OR_ZERO(req)); + return req; +out_flushme: + /* + * Note: we mark the request dirty here because + * nfs_lock_and_join_requests() cannot preserve + * commit flags, so we have to replay the write. + */ + nfs_mark_request_dirty(req); + nfs_unlock_and_release_request(req); + error = nfs_wb_folio(folio->mapping->host, folio); + trace_nfs_try_to_update_request_done(folio_inode(folio), offset, bytes, error); + return (error < 0) ? ERR_PTR(error) : NULL; +} + +/* + * Try to update an existing write request, or create one if there is none. + * + * Note: Should always be called with the Page Lock held to prevent races + * if we have to add a new request. Also assumes that the caller has + * already called nfs_flush_incompatible() if necessary. + */ +static struct nfs_page *nfs_setup_write_request(struct nfs_open_context *ctx, + struct folio *folio, + unsigned int offset, + unsigned int bytes) +{ + struct nfs_page *req; + + req = nfs_try_to_update_request(folio, offset, bytes); + if (req != NULL) + goto out; + req = nfs_page_create_from_folio(ctx, folio, offset, bytes); + if (IS_ERR(req)) + goto out; + nfs_inode_add_request(req); +out: + return req; +} + +static int nfs_writepage_setup(struct nfs_open_context *ctx, + struct folio *folio, unsigned int offset, + unsigned int count) +{ + struct nfs_page *req; + + req = nfs_setup_write_request(ctx, folio, offset, count); + if (IS_ERR(req)) + return PTR_ERR(req); + trace_nfs_writepage_setup(req); + /* Update file length */ + nfs_grow_file(folio, offset, count); + nfs_mark_uptodate(req); + nfs_mark_request_dirty(req); + nfs_unlock_and_release_request(req); + return 0; +} + +int nfs_flush_incompatible(struct file *file, struct folio *folio) +{ + struct nfs_open_context *ctx = nfs_file_open_context(file); + struct nfs_lock_context *l_ctx; + struct file_lock_context *flctx = locks_inode_context(file_inode(file)); + struct nfs_page *req; + int do_flush, status; + /* + * Look for a request corresponding to this page. If there + * is one, and it belongs to another file, we flush it out + * before we try to copy anything into the page. Do this + * due to the lack of an ACCESS-type call in NFSv2. + * Also do the same if we find a request from an existing + * dropped page. + */ + do { + req = nfs_folio_find_head_request(folio); + if (req == NULL) + return 0; + l_ctx = req->wb_lock_context; + do_flush = nfs_page_to_folio(req) != folio || + !nfs_match_open_context(nfs_req_openctx(req), ctx); + if (l_ctx && flctx && + !(list_empty_careful(&flctx->flc_posix) && + list_empty_careful(&flctx->flc_flock))) { + do_flush |= l_ctx->lockowner != current->files; + } + nfs_release_request(req); + if (!do_flush) + return 0; + status = nfs_wb_folio(folio->mapping->host, folio); + } while (status == 0); + return status; +} + +/* + * Avoid buffered writes when a open context credential's key would + * expire soon. + * + * Returns -EACCES if the key will expire within RPC_KEY_EXPIRE_FAIL. + * + * Return 0 and set a credential flag which triggers the inode to flush + * and performs NFS_FILE_SYNC writes if the key will expired within + * RPC_KEY_EXPIRE_TIMEO. + */ +int +nfs_key_timeout_notify(struct file *filp, struct inode *inode) +{ + struct nfs_open_context *ctx = nfs_file_open_context(filp); + + if (nfs_ctx_key_to_expire(ctx, inode) && + !rcu_access_pointer(ctx->ll_cred)) + /* Already expired! */ + return -EACCES; + return 0; +} + +/* + * Test if the open context credential key is marked to expire soon. + */ +bool nfs_ctx_key_to_expire(struct nfs_open_context *ctx, struct inode *inode) +{ + struct rpc_auth *auth = NFS_SERVER(inode)->client->cl_auth; + struct rpc_cred *cred, *new, *old = NULL; + struct auth_cred acred = { + .cred = ctx->cred, + }; + bool ret = false; + + rcu_read_lock(); + cred = rcu_dereference(ctx->ll_cred); + if (cred && !(cred->cr_ops->crkey_timeout && + cred->cr_ops->crkey_timeout(cred))) + goto out; + rcu_read_unlock(); + + new = auth->au_ops->lookup_cred(auth, &acred, 0); + if (new == cred) { + put_rpccred(new); + return true; + } + if (IS_ERR_OR_NULL(new)) { + new = NULL; + ret = true; + } else if (new->cr_ops->crkey_timeout && + new->cr_ops->crkey_timeout(new)) + ret = true; + + rcu_read_lock(); + old = rcu_dereference_protected(xchg(&ctx->ll_cred, + RCU_INITIALIZER(new)), 1); +out: + rcu_read_unlock(); + put_rpccred(old); + return ret; +} + +/* + * If the page cache is marked as unsafe or invalid, then we can't rely on + * the PageUptodate() flag. In this case, we will need to turn off + * write optimisations that depend on the page contents being correct. + */ +static bool nfs_folio_write_uptodate(struct folio *folio, unsigned int pagelen) +{ + struct inode *inode = folio->mapping->host; + struct nfs_inode *nfsi = NFS_I(inode); + + if (nfs_have_delegated_attributes(inode)) + goto out; + if (nfsi->cache_validity & + (NFS_INO_INVALID_CHANGE | NFS_INO_INVALID_SIZE)) + return false; + smp_rmb(); + if (test_bit(NFS_INO_INVALIDATING, &nfsi->flags) && pagelen != 0) + return false; +out: + if (nfsi->cache_validity & NFS_INO_INVALID_DATA && pagelen != 0) + return false; + return folio_test_uptodate(folio) != 0; +} + +static bool +is_whole_file_wrlock(struct file_lock *fl) +{ + return fl->fl_start == 0 && fl->fl_end == OFFSET_MAX && + lock_is_write(fl); +} + +/* If we know the page is up to date, and we're not using byte range locks (or + * if we have the whole file locked for writing), it may be more efficient to + * extend the write to cover the entire page in order to avoid fragmentation + * inefficiencies. + * + * If the file is opened for synchronous writes then we can just skip the rest + * of the checks. + */ +static int nfs_can_extend_write(struct file *file, struct folio *folio, + unsigned int pagelen) +{ + struct inode *inode = file_inode(file); + struct file_lock_context *flctx = locks_inode_context(inode); + struct file_lock *fl; + int ret; + unsigned int mntflags = NFS_SERVER(inode)->flags; + + if (mntflags & NFS_MOUNT_NO_ALIGNWRITE) + return 0; + if (file->f_flags & O_DSYNC) + return 0; + if (!nfs_folio_write_uptodate(folio, pagelen)) + return 0; + if (nfs_have_write_delegation(inode)) + return 1; + if (!flctx || (list_empty_careful(&flctx->flc_flock) && + list_empty_careful(&flctx->flc_posix))) + return 1; + + /* Check to see if there are whole file write locks */ + ret = 0; + spin_lock(&flctx->flc_lock); + if (!list_empty(&flctx->flc_posix)) { + fl = list_first_entry(&flctx->flc_posix, struct file_lock, + c.flc_list); + if (is_whole_file_wrlock(fl)) + ret = 1; + } else if (!list_empty(&flctx->flc_flock)) { + fl = list_first_entry(&flctx->flc_flock, struct file_lock, + c.flc_list); + if (lock_is_write(fl)) + ret = 1; + } + spin_unlock(&flctx->flc_lock); + return ret; +} + +/* + * Update and possibly write a cached page of an NFS file. + * + * XXX: Keep an eye on generic_file_read to make sure it doesn't do bad + * things with a page scheduled for an RPC call (e.g. invalidate it). + */ +int nfs_update_folio(struct file *file, struct folio *folio, + unsigned int offset, unsigned int count) +{ + struct nfs_open_context *ctx = nfs_file_open_context(file); + struct address_space *mapping = folio->mapping; + struct inode *inode = mapping->host; + unsigned int pagelen = nfs_folio_length(folio); + int status = 0; + + nfs_inc_stats(inode, NFSIOS_VFSUPDATEPAGE); + + trace_nfs_update_folio(inode, offset, count); + + dprintk("NFS: nfs_update_folio(%pD2 %d@%lld)\n", file, count, + (long long)(folio_pos(folio) + offset)); + + if (!count) + goto out; + + if (nfs_can_extend_write(file, folio, pagelen)) { + unsigned int end = count + offset; + + offset = round_down(offset, PAGE_SIZE); + if (end < pagelen) + end = min(round_up(end, PAGE_SIZE), pagelen); + count = end - offset; + } + + status = nfs_writepage_setup(ctx, folio, offset, count); + if (status < 0) + nfs_set_pageerror(mapping); +out: + trace_nfs_update_folio_done(inode, offset, count, status); + dprintk("NFS: nfs_update_folio returns %d (isize %lld)\n", + status, (long long)i_size_read(inode)); + return status; +} + +static int flush_task_priority(int how) +{ + switch (how & (FLUSH_HIGHPRI|FLUSH_LOWPRI)) { + case FLUSH_HIGHPRI: + return RPC_PRIORITY_HIGH; + case FLUSH_LOWPRI: + return RPC_PRIORITY_LOW; + } + return RPC_PRIORITY_NORMAL; +} + +static void nfs_initiate_write(struct nfs_pgio_header *hdr, + struct rpc_message *msg, + const struct nfs_rpc_ops *rpc_ops, + struct rpc_task_setup *task_setup_data, int how) +{ + int priority = flush_task_priority(how); + + if (IS_SWAPFILE(hdr->inode)) + task_setup_data->flags |= RPC_TASK_SWAPPER; + task_setup_data->priority = priority; + rpc_ops->write_setup(hdr, msg, &task_setup_data->rpc_client); + trace_nfs_initiate_write(hdr); +} + +/* If a nfs_flush_* function fails, it should remove reqs from @head and + * call this on each, which will prepare them to be retried on next + * writeback using standard nfs. + */ +static void nfs_redirty_request(struct nfs_page *req) +{ + struct nfs_inode *nfsi = NFS_I(nfs_page_to_inode(req)); + + /* Bump the transmission count */ + req->wb_nio++; + nfs_mark_request_dirty(req); + atomic_long_inc(&nfsi->redirtied_pages); + nfs_page_end_writeback(req); + nfs_release_request(req); +} + +static void nfs_async_write_error(struct list_head *head, int error) +{ + struct nfs_page *req; + + while (!list_empty(head)) { + req = nfs_list_entry(head->next); + nfs_list_remove_request(req); + if (nfs_error_is_fatal_on_server(error)) + nfs_write_error(req, error); + else + nfs_redirty_request(req); + } +} + +static void nfs_async_write_reschedule_io(struct nfs_pgio_header *hdr) +{ + nfs_async_write_error(&hdr->pages, 0); +} + +static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops = { + .init_hdr = nfs_async_write_init, + .error_cleanup = nfs_async_write_error, + .completion = nfs_write_completion, + .reschedule_io = nfs_async_write_reschedule_io, +}; + +void nfs_pageio_init_write(struct nfs_pageio_descriptor *pgio, + struct inode *inode, int ioflags, bool force_mds, + const struct nfs_pgio_completion_ops *compl_ops) +{ + struct nfs_server *server = NFS_SERVER(inode); + const struct nfs_pageio_ops *pg_ops = &nfs_pgio_rw_ops; + +#if IS_ENABLED(CONFIG_NFS_V4) + if (server->pnfs_curr_ld && !force_mds) + pg_ops = server->pnfs_curr_ld->pg_write_ops; +#endif + nfs_pageio_init(pgio, inode, pg_ops, compl_ops, &nfs_rw_write_ops, + server->wsize, ioflags); +} +EXPORT_SYMBOL_GPL(nfs_pageio_init_write); + +void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio) +{ + struct nfs_pgio_mirror *mirror; + + if (pgio->pg_ops && pgio->pg_ops->pg_cleanup) + pgio->pg_ops->pg_cleanup(pgio); + + pgio->pg_ops = &nfs_pgio_rw_ops; + + nfs_pageio_stop_mirroring(pgio); + + mirror = &pgio->pg_mirrors[0]; + mirror->pg_bsize = NFS_SERVER(pgio->pg_inode)->wsize; +} +EXPORT_SYMBOL_GPL(nfs_pageio_reset_write_mds); + + +void nfs_commit_prepare(struct rpc_task *task, void *calldata) +{ + struct nfs_commit_data *data = calldata; + + NFS_PROTO(data->inode)->commit_rpc_prepare(task, data); +} + +static void nfs_writeback_check_extend(struct nfs_pgio_header *hdr, + struct nfs_fattr *fattr) +{ + struct nfs_pgio_args *argp = &hdr->args; + struct nfs_pgio_res *resp = &hdr->res; + u64 size = argp->offset + resp->count; + + if (!(fattr->valid & NFS_ATTR_FATTR_SIZE)) + fattr->size = size; + if (nfs_size_to_loff_t(fattr->size) < i_size_read(hdr->inode)) { + fattr->valid &= ~NFS_ATTR_FATTR_SIZE; + return; + } + if (size != fattr->size) + return; + /* Set attribute barrier */ + nfs_fattr_set_barrier(fattr); + /* ...and update size */ + fattr->valid |= NFS_ATTR_FATTR_SIZE; +} + +void nfs_writeback_update_inode(struct nfs_pgio_header *hdr) +{ + struct nfs_fattr *fattr = &hdr->fattr; + struct inode *inode = hdr->inode; + + if (nfs_have_delegated_mtime(inode)) { + spin_lock(&inode->i_lock); + nfs_set_cache_invalid(inode, NFS_INO_INVALID_BLOCKS); + spin_unlock(&inode->i_lock); + return; + } + + spin_lock(&inode->i_lock); + nfs_writeback_check_extend(hdr, fattr); + nfs_post_op_update_inode_force_wcc_locked(inode, fattr); + spin_unlock(&inode->i_lock); +} +EXPORT_SYMBOL_GPL(nfs_writeback_update_inode); + +/* + * This function is called when the WRITE call is complete. + */ +static int nfs_writeback_done(struct rpc_task *task, + struct nfs_pgio_header *hdr, + struct inode *inode) +{ + int status; + + /* + * ->write_done will attempt to use post-op attributes to detect + * conflicting writes by other clients. A strict interpretation + * of close-to-open would allow us to continue caching even if + * another writer had changed the file, but some applications + * depend on tighter cache coherency when writing. + */ + status = NFS_PROTO(inode)->write_done(task, hdr); + if (status != 0) + return status; + + nfs_add_stats(inode, NFSIOS_SERVERWRITTENBYTES, hdr->res.count); + trace_nfs_writeback_done(task, hdr); + + if (task->tk_status >= 0) { + enum nfs3_stable_how committed = hdr->res.verf->committed; + + if (committed == NFS_UNSTABLE) { + /* + * We have some uncommitted data on the server at + * this point, so ensure that we keep track of that + * fact irrespective of what later writes do. + */ + set_bit(NFS_IOHDR_UNSTABLE_WRITES, &hdr->flags); + } + + if (committed < hdr->args.stable) { + /* We tried a write call, but the server did not + * commit data to stable storage even though we + * requested it. + * Note: There is a known bug in Tru64 < 5.0 in which + * the server reports NFS_DATA_SYNC, but performs + * NFS_FILE_SYNC. We therefore implement this checking + * as a dprintk() in order to avoid filling syslog. + */ + static unsigned long complain; + + /* Note this will print the MDS for a DS write */ + if (time_before(complain, jiffies)) { + dprintk("NFS: faulty NFS server %s:" + " (committed = %d) != (stable = %d)\n", + NFS_SERVER(inode)->nfs_client->cl_hostname, + committed, hdr->args.stable); + complain = jiffies + 300 * HZ; + } + } + } + + /* Deal with the suid/sgid bit corner case */ + if (nfs_should_remove_suid(inode)) { + spin_lock(&inode->i_lock); + nfs_set_cache_invalid(inode, NFS_INO_INVALID_MODE + | NFS_INO_REVAL_FORCED); + spin_unlock(&inode->i_lock); + } + return 0; +} + +/* + * This function is called when the WRITE call is complete. + */ +static void nfs_writeback_result(struct rpc_task *task, + struct nfs_pgio_header *hdr) +{ + struct nfs_pgio_args *argp = &hdr->args; + struct nfs_pgio_res *resp = &hdr->res; + + if (resp->count < argp->count && !list_empty(&hdr->pages)) { + static unsigned long complain; + struct nfs_open_context *ctx = + hdr->req->wb_lock_context->open_context; + + set_bit(NFS_CONTEXT_WRITE_SYNC, &ctx->flags); + /* This a short write! */ + nfs_inc_stats(hdr->inode, NFSIOS_SHORTWRITE); + + /* Has the server at least made some progress? */ + if (resp->count == 0) { + if (time_before(complain, jiffies)) { + printk(KERN_WARNING + "NFS: Server wrote zero bytes, expected %u.\n", + argp->count); + complain = jiffies + 300 * HZ; + } + nfs_set_pgio_error(hdr, -EIO, argp->offset); + task->tk_status = -EIO; + return; + } + + /* For non rpc-based layout drivers, retry-through-MDS */ + if (!task->tk_ops) { + hdr->pnfs_error = -EAGAIN; + return; + } + + /* Was this an NFSv2 write or an NFSv3 stable write? */ + if (resp->verf->committed != NFS_UNSTABLE) { + /* Resend from where the server left off */ + hdr->mds_offset += resp->count; + argp->offset += resp->count; + argp->pgbase += resp->count; + argp->count -= resp->count; + } else { + /* Resend as a stable write in order to avoid + * headaches in the case of a server crash. + */ + argp->stable = NFS_FILE_SYNC; + } + resp->count = 0; + resp->verf->committed = 0; + rpc_restart_call_prepare(task); + } +} + +static int wait_on_commit(struct nfs_mds_commit_info *cinfo) +{ + return wait_var_event_killable(&cinfo->rpcs_out, + !atomic_read(&cinfo->rpcs_out)); +} + +void nfs_commit_begin(struct nfs_mds_commit_info *cinfo) +{ + atomic_inc(&cinfo->rpcs_out); +} + +bool nfs_commit_end(struct nfs_mds_commit_info *cinfo) +{ + if (atomic_dec_and_test(&cinfo->rpcs_out)) { + wake_up_var(&cinfo->rpcs_out); + return true; + } + return false; +} + +void nfs_commitdata_release(struct nfs_commit_data *data) +{ + put_nfs_open_context(data->context); + nfs_commit_free(data); +} +EXPORT_SYMBOL_GPL(nfs_commitdata_release); + +int nfs_initiate_commit(struct rpc_clnt *clnt, struct nfs_commit_data *data, + const struct nfs_rpc_ops *nfs_ops, + const struct rpc_call_ops *call_ops, + int how, int flags, + struct nfsd_file *localio) +{ + struct rpc_task *task; + int priority = flush_task_priority(how); + struct rpc_message msg = { + .rpc_argp = &data->args, + .rpc_resp = &data->res, + .rpc_cred = data->cred, + }; + struct rpc_task_setup task_setup_data = { + .task = &data->task, + .rpc_client = clnt, + .rpc_message = &msg, + .callback_ops = call_ops, + .callback_data = data, + .workqueue = nfsiod_workqueue, + .flags = RPC_TASK_ASYNC | flags, + .priority = priority, + }; + + if (nfs_server_capable(data->inode, NFS_CAP_MOVEABLE)) + task_setup_data.flags |= RPC_TASK_MOVEABLE; + + /* Set up the initial task struct. */ + nfs_ops->commit_setup(data, &msg, &task_setup_data.rpc_client); + trace_nfs_initiate_commit(data); + + dprintk("NFS: initiated commit call\n"); + + if (localio) + return nfs_local_commit(localio, data, call_ops, how); + + task = rpc_run_task(&task_setup_data); + if (IS_ERR(task)) + return PTR_ERR(task); + if (how & FLUSH_SYNC) + rpc_wait_for_completion_task(task); + rpc_put_task(task); + return 0; +} +EXPORT_SYMBOL_GPL(nfs_initiate_commit); + +static loff_t nfs_get_lwb(struct list_head *head) +{ + loff_t lwb = 0; + struct nfs_page *req; + + list_for_each_entry(req, head, wb_list) + if (lwb < (req_offset(req) + req->wb_bytes)) + lwb = req_offset(req) + req->wb_bytes; + + return lwb; +} + +/* + * Set up the argument/result storage required for the RPC call. + */ +void nfs_init_commit(struct nfs_commit_data *data, + struct list_head *head, + struct pnfs_layout_segment *lseg, + struct nfs_commit_info *cinfo) +{ + struct nfs_page *first; + struct nfs_open_context *ctx; + struct inode *inode; + + /* Set up the RPC argument and reply structs + * NB: take care not to mess about with data->commit et al. */ + + if (head) + list_splice_init(head, &data->pages); + + first = nfs_list_entry(data->pages.next); + ctx = nfs_req_openctx(first); + inode = d_inode(ctx->dentry); + + data->inode = inode; + data->cred = ctx->cred; + data->lseg = lseg; /* reference transferred */ + /* only set lwb for pnfs commit */ + if (lseg) + data->lwb = nfs_get_lwb(&data->pages); + data->mds_ops = &nfs_commit_ops; + data->completion_ops = cinfo->completion_ops; + data->dreq = cinfo->dreq; + + data->args.fh = NFS_FH(data->inode); + /* Note: we always request a commit of the entire inode */ + data->args.offset = 0; + data->args.count = 0; + data->context = get_nfs_open_context(ctx); + data->res.fattr = &data->fattr; + data->res.verf = &data->verf; + nfs_fattr_init(&data->fattr); + nfs_commit_begin(cinfo->mds); +} +EXPORT_SYMBOL_GPL(nfs_init_commit); + +void nfs_retry_commit(struct list_head *page_list, + struct pnfs_layout_segment *lseg, + struct nfs_commit_info *cinfo, + u32 ds_commit_idx) +{ + struct nfs_page *req; + + while (!list_empty(page_list)) { + req = nfs_list_entry(page_list->next); + nfs_list_remove_request(req); + nfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx); + nfs_folio_clear_commit(req); + nfs_unlock_and_release_request(req); + } +} +EXPORT_SYMBOL_GPL(nfs_retry_commit); + +static void nfs_commit_resched_write(struct nfs_commit_info *cinfo, + struct nfs_page *req) +{ + struct folio *folio = nfs_page_to_folio(req); + + filemap_dirty_folio(folio_mapping(folio), folio); +} + +/* + * Commit dirty pages + */ +static int +nfs_commit_list(struct inode *inode, struct list_head *head, int how, + struct nfs_commit_info *cinfo) +{ + struct nfs_commit_data *data; + struct nfsd_file *localio; + unsigned short task_flags = 0; + + /* another commit raced with us */ + if (list_empty(head)) + return 0; + + data = nfs_commitdata_alloc(); + if (!data) { + nfs_retry_commit(head, NULL, cinfo, -1); + return -ENOMEM; + } + + /* Set up the argument struct */ + nfs_init_commit(data, head, NULL, cinfo); + if (NFS_SERVER(inode)->nfs_client->cl_minorversion) + task_flags = RPC_TASK_MOVEABLE; + + localio = nfs_local_open_fh(NFS_SERVER(inode)->nfs_client, data->cred, + data->args.fh, &data->context->nfl, + data->context->mode); + return nfs_initiate_commit(NFS_CLIENT(inode), data, NFS_PROTO(inode), + data->mds_ops, how, + RPC_TASK_CRED_NOREF | task_flags, localio); +} + +/* + * COMMIT call returned + */ +static void nfs_commit_done(struct rpc_task *task, void *calldata) +{ + struct nfs_commit_data *data = calldata; + + /* Call the NFS version-specific code */ + NFS_PROTO(data->inode)->commit_done(task, data); + trace_nfs_commit_done(task, data); +} + +static void nfs_commit_release_pages(struct nfs_commit_data *data) +{ + const struct nfs_writeverf *verf = data->res.verf; + struct nfs_page *req; + int status = data->task.tk_status; + struct nfs_commit_info cinfo; + struct folio *folio; + + while (!list_empty(&data->pages)) { + req = nfs_list_entry(data->pages.next); + nfs_list_remove_request(req); + folio = nfs_page_to_folio(req); + nfs_folio_clear_commit(req); + + dprintk("NFS: commit (%s/%llu %d@%lld)", + nfs_req_openctx(req)->dentry->d_sb->s_id, + (unsigned long long)d_inode(nfs_req_openctx(req)->dentry)->i_ino, + req->wb_bytes, + (long long)req_offset(req)); + if (status < 0) { + if (folio) { + trace_nfs_commit_error(data->inode, req, + status); + nfs_mapping_set_error(folio, status); + nfs_inode_remove_request(req); + } + dprintk(", error = %d\n", status); + goto next; + } + + /* Okay, COMMIT succeeded, apparently. Check the verifier + * returned by the server against all stored verfs. */ + if (nfs_write_match_verf(verf, req)) { + /* We have a match */ + if (folio) + nfs_inode_remove_request(req); + dprintk(" OK\n"); + goto next; + } + /* We have a mismatch. Write the page again */ + dprintk(" mismatch\n"); + nfs_mark_request_dirty(req); + set_bit(NFS_CONTEXT_WRITE_SYNC, + &req->wb_lock_context->open_context->flags); + atomic_long_inc(&NFS_I(data->inode)->redirtied_pages); + next: + nfs_unlock_and_release_request(req); + /* Latency breaker */ + cond_resched(); + } + + nfs_init_cinfo(&cinfo, data->inode, data->dreq); + nfs_commit_end(cinfo.mds); +} + +static void nfs_commit_release(void *calldata) +{ + struct nfs_commit_data *data = calldata; + + data->completion_ops->completion(data); + nfs_commitdata_release(calldata); +} + +static const struct rpc_call_ops nfs_commit_ops = { + .rpc_call_prepare = nfs_commit_prepare, + .rpc_call_done = nfs_commit_done, + .rpc_release = nfs_commit_release, +}; + +static const struct nfs_commit_completion_ops nfs_commit_completion_ops = { + .completion = nfs_commit_release_pages, + .resched_write = nfs_commit_resched_write, +}; + +int nfs_generic_commit_list(struct inode *inode, struct list_head *head, + int how, struct nfs_commit_info *cinfo) +{ + int status; + + status = pnfs_commit_list(inode, head, how, cinfo); + if (status == PNFS_NOT_ATTEMPTED) + status = nfs_commit_list(inode, head, how, cinfo); + return status; +} + +static int __nfs_commit_inode(struct inode *inode, int how, + struct writeback_control *wbc) +{ + LIST_HEAD(head); + struct nfs_commit_info cinfo; + int may_wait = how & FLUSH_SYNC; + int ret, nscan; + + how &= ~FLUSH_SYNC; + nfs_init_cinfo_from_inode(&cinfo, inode); + nfs_commit_begin(cinfo.mds); + for (;;) { + ret = nscan = nfs_scan_commit(inode, &head, &cinfo); + if (ret <= 0) + break; + ret = nfs_generic_commit_list(inode, &head, how, &cinfo); + if (ret < 0) + break; + ret = 0; + if (wbc && wbc->sync_mode == WB_SYNC_NONE) { + if (nscan < wbc->nr_to_write) + wbc->nr_to_write -= nscan; + else + wbc->nr_to_write = 0; + } + if (nscan < INT_MAX) + break; + cond_resched(); + } + nfs_commit_end(cinfo.mds); + if (ret || !may_wait) + return ret; + return wait_on_commit(cinfo.mds); +} + +int nfs_commit_inode(struct inode *inode, int how) +{ + return __nfs_commit_inode(inode, how, NULL); +} +EXPORT_SYMBOL_GPL(nfs_commit_inode); + +int nfs_write_inode(struct inode *inode, struct writeback_control *wbc) +{ + struct nfs_inode *nfsi = NFS_I(inode); + int flags = FLUSH_SYNC; + int ret = 0; + + if (wbc->sync_mode == WB_SYNC_NONE) { + /* no commits means nothing needs to be done */ + if (!atomic_long_read(&nfsi->commit_info.ncommit)) + goto check_requests_outstanding; + + /* Don't commit yet if this is a non-blocking flush and there + * are a lot of outstanding writes for this mapping. + */ + if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK)) + goto out_mark_dirty; + + /* don't wait for the COMMIT response */ + flags = 0; + } + + ret = __nfs_commit_inode(inode, flags, wbc); + if (!ret) { + if (flags & FLUSH_SYNC) + return 0; + } else if (atomic_long_read(&nfsi->commit_info.ncommit)) + goto out_mark_dirty; + +check_requests_outstanding: + if (!atomic_read(&nfsi->commit_info.rpcs_out)) + return ret; +out_mark_dirty: + __mark_inode_dirty(inode, I_DIRTY_DATASYNC); + return ret; +} +EXPORT_SYMBOL_GPL(nfs_write_inode); + +/* + * Wrapper for filemap_write_and_wait_range() + * + * Needed for pNFS in order to ensure data becomes visible to the + * client. + */ +int nfs_filemap_write_and_wait_range(struct address_space *mapping, + loff_t lstart, loff_t lend) +{ + int ret; + + ret = filemap_write_and_wait_range(mapping, lstart, lend); + if (ret == 0) + ret = pnfs_sync_inode(mapping->host, true); + return ret; +} +EXPORT_SYMBOL_GPL(nfs_filemap_write_and_wait_range); + +/* + * flush the inode to disk. + */ +int nfs_wb_all(struct inode *inode) +{ + int ret; + + trace_nfs_writeback_inode_enter(inode); + + ret = filemap_write_and_wait(inode->i_mapping); + if (ret) + goto out; + ret = nfs_commit_inode(inode, FLUSH_SYNC); + if (ret < 0) + goto out; + pnfs_sync_inode(inode, true); + ret = 0; + +out: + trace_nfs_writeback_inode_exit(inode, ret); + return ret; +} +EXPORT_SYMBOL_GPL(nfs_wb_all); + +int nfs_wb_folio_cancel(struct inode *inode, struct folio *folio) +{ + struct nfs_page *req; + int ret = 0; + + folio_wait_writeback(folio); + + /* blocking call to cancel all requests and join to a single (head) + * request */ + req = nfs_lock_and_join_requests(folio); + + if (IS_ERR(req)) { + ret = PTR_ERR(req); + } else if (req) { + /* all requests from this folio have been cancelled by + * nfs_lock_and_join_requests, so just remove the head + * request from the inode / page_private pointer and + * release it */ + nfs_inode_remove_request(req); + nfs_unlock_and_release_request(req); + folio_cancel_dirty(folio); + } + + return ret; +} + +/** + * nfs_wb_folio_reclaim - Write back all requests on one page + * @inode: pointer to page + * @folio: pointer to folio + * + * Assumes that the folio has been locked by the caller + */ +int nfs_wb_folio_reclaim(struct inode *inode, struct folio *folio) +{ + loff_t range_start = folio_pos(folio); + size_t len = folio_size(folio); + struct writeback_control wbc = { + .sync_mode = WB_SYNC_ALL, + .nr_to_write = 0, + .range_start = range_start, + .range_end = range_start + len - 1, + .for_sync = 1, + }; + int ret; + + if (folio_test_writeback(folio)) + return -EBUSY; + if (folio_clear_dirty_for_io(folio)) { + trace_nfs_writeback_folio_reclaim(inode, range_start, len); + ret = nfs_writepage_locked(folio, &wbc); + trace_nfs_writeback_folio_reclaim_done(inode, range_start, len, + ret); + return ret; + } + nfs_commit_inode(inode, 0); + return 0; +} + +/** + * nfs_wb_folio - Write back all requests on one page + * @inode: pointer to page + * @folio: pointer to folio + * + * Assumes that the folio has been locked by the caller, and will + * not unlock it. + */ +int nfs_wb_folio(struct inode *inode, struct folio *folio) +{ + loff_t range_start = folio_pos(folio); + size_t len = folio_size(folio); + struct writeback_control wbc = { + .sync_mode = WB_SYNC_ALL, + .nr_to_write = 0, + .range_start = range_start, + .range_end = range_start + len - 1, + }; + int ret; + + trace_nfs_writeback_folio(inode, range_start, len); + + for (;;) { + folio_wait_writeback(folio); + if (folio_clear_dirty_for_io(folio)) { + ret = nfs_writepage_locked(folio, &wbc); + if (ret < 0) + goto out_error; + continue; + } + ret = 0; + if (!folio_test_private(folio)) + break; + ret = nfs_commit_inode(inode, FLUSH_SYNC); + if (ret < 0) + goto out_error; + } +out_error: + trace_nfs_writeback_folio_done(inode, range_start, len, ret); + return ret; +} + +#ifdef CONFIG_MIGRATION +int nfs_migrate_folio(struct address_space *mapping, struct folio *dst, + struct folio *src, enum migrate_mode mode) +{ + /* + * If the private flag is set, the folio is currently associated with + * an in-progress read or write request. Don't try to migrate it. + * + * FIXME: we could do this in principle, but we'll need a way to ensure + * that we can safely release the inode reference while holding + * the folio lock. + */ + if (folio_test_private(src)) { + if (mode == MIGRATE_SYNC) + nfs_wb_folio(src->mapping->host, src); + if (folio_test_private(src)) + return -EBUSY; + } + + if (folio_test_private_2(src)) { /* [DEPRECATED] */ + if (mode == MIGRATE_ASYNC) + return -EBUSY; + folio_wait_private_2(src); + } + + return migrate_folio(mapping, dst, src, mode); +} +#endif + +int __init nfs_init_writepagecache(void) +{ + nfs_wdata_cachep = kmem_cache_create("nfs_write_data", + sizeof(struct nfs_pgio_header), + 0, SLAB_HWCACHE_ALIGN, + NULL); + if (nfs_wdata_cachep == NULL) + return -ENOMEM; + + nfs_wdata_mempool = mempool_create_slab_pool(MIN_POOL_WRITE, + nfs_wdata_cachep); + if (nfs_wdata_mempool == NULL) + goto out_destroy_write_cache; + + nfs_cdata_cachep = kmem_cache_create("nfs_commit_data", + sizeof(struct nfs_commit_data), + 0, SLAB_HWCACHE_ALIGN, + NULL); + if (nfs_cdata_cachep == NULL) + goto out_destroy_write_mempool; + + nfs_commit_mempool = mempool_create_slab_pool(MIN_POOL_COMMIT, + nfs_cdata_cachep); + if (nfs_commit_mempool == NULL) + goto out_destroy_commit_cache; + + /* + * NFS congestion size, scale with available memory. + * + * 64MB: 8192k + * 128MB: 11585k + * 256MB: 16384k + * 512MB: 23170k + * 1GB: 32768k + * 2GB: 46340k + * 4GB: 65536k + * 8GB: 92681k + * 16GB: 131072k + * + * This allows larger machines to have larger/more transfers. + * Limit the default to 256M + */ + nfs_congestion_kb = (16*int_sqrt(totalram_pages())) << (PAGE_SHIFT-10); + if (nfs_congestion_kb > 256*1024) + nfs_congestion_kb = 256*1024; + + return 0; + +out_destroy_commit_cache: + kmem_cache_destroy(nfs_cdata_cachep); +out_destroy_write_mempool: + mempool_destroy(nfs_wdata_mempool); +out_destroy_write_cache: + kmem_cache_destroy(nfs_wdata_cachep); + return -ENOMEM; +} + +void nfs_destroy_writepagecache(void) +{ + mempool_destroy(nfs_commit_mempool); + kmem_cache_destroy(nfs_cdata_cachep); + mempool_destroy(nfs_wdata_mempool); + kmem_cache_destroy(nfs_wdata_cachep); +} + +static const struct nfs_rw_ops nfs_rw_write_ops = { + .rw_alloc_header = nfs_writehdr_alloc, + .rw_free_header = nfs_writehdr_free, + .rw_done = nfs_writeback_done, + .rw_result = nfs_writeback_result, + .rw_initiate = nfs_initiate_write, +}; diff --git a/fs/nfs_common/Makefile b/fs/nfs_common/Makefile new file mode 100644 index 000000000..c10ead273 --- /dev/null +++ b/fs/nfs_common/Makefile @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for Linux filesystem routines that are shared by client and server. +# + +obj-$(CONFIG_NFS_ACL_SUPPORT) += nfs_acl.o +nfs_acl-objs := nfsacl.o + +CFLAGS_localio_trace.o += -I$(src) +obj-$(CONFIG_NFS_COMMON_LOCALIO_SUPPORT) += nfs_localio.o +nfs_localio-objs := nfslocalio.o localio_trace.o + +obj-$(CONFIG_GRACE_PERIOD) += grace.o +obj-$(CONFIG_NFS_V4_2_SSC_HELPER) += nfs_ssc.o + +obj-$(CONFIG_NFS_COMMON) += common.o diff --git a/fs/nfs_common/common.c b/fs/nfs_common/common.c new file mode 100644 index 000000000..0778743ae --- /dev/null +++ b/fs/nfs_common/common.c @@ -0,0 +1,200 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include + +/* + * We need to translate between nfs status return values and + * the local errno values which may not be the same. + */ +static const struct { + int stat; + int errno; +} nfs_errtbl[] = { + { NFS_OK, 0 }, + { NFSERR_PERM, -EPERM }, + { NFSERR_NOENT, -ENOENT }, + { NFSERR_IO, -EIO }, + { NFSERR_NXIO, -ENXIO }, + { NFSERR_ACCES, -EACCES }, + { NFSERR_EXIST, -EEXIST }, + { NFSERR_XDEV, -EXDEV }, + { NFSERR_NODEV, -ENODEV }, + { NFSERR_NOTDIR, -ENOTDIR }, + { NFSERR_ISDIR, -EISDIR }, + { NFSERR_INVAL, -EINVAL }, + { NFSERR_FBIG, -EFBIG }, + { NFSERR_NOSPC, -ENOSPC }, + { NFSERR_ROFS, -EROFS }, + { NFSERR_MLINK, -EMLINK }, + { NFSERR_NAMETOOLONG, -ENAMETOOLONG }, + { NFSERR_NOTEMPTY, -ENOTEMPTY }, + { NFSERR_DQUOT, -EDQUOT }, + { NFSERR_STALE, -ESTALE }, + { NFSERR_REMOTE, -EREMOTE }, +#ifdef EWFLUSH + { NFSERR_WFLUSH, -EWFLUSH }, +#endif + { NFSERR_BADHANDLE, -EBADHANDLE }, + { NFSERR_NOT_SYNC, -ENOTSYNC }, + { NFSERR_BAD_COOKIE, -EBADCOOKIE }, + { NFSERR_NOTSUPP, -ENOTSUPP }, + { NFSERR_TOOSMALL, -ETOOSMALL }, + { NFSERR_SERVERFAULT, -EREMOTEIO }, + { NFSERR_BADTYPE, -EBADTYPE }, + { NFSERR_JUKEBOX, -EJUKEBOX }, +}; + +/** + * nfs_stat_to_errno - convert an NFS status code to a local errno + * @status: NFS status code to convert + * + * Returns a local errno value, or -EIO if the NFS status code is + * not recognized. This function is used jointly by NFSv2 and NFSv3. + */ +int nfs_stat_to_errno(enum nfs_stat status) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(nfs_errtbl); i++) { + if (nfs_errtbl[i].stat == (int)status) + return nfs_errtbl[i].errno; + } + return -EIO; +} +EXPORT_SYMBOL_GPL(nfs_stat_to_errno); + +/* + * We need to translate between nfs v4 status return values and + * the local errno values which may not be the same. + * + * nfs4_errtbl_common[] is used before more specialized mappings + * available in nfs4_errtbl[] or nfs4_errtbl_localio[]. + */ +static const struct { + int stat; + int errno; +} nfs4_errtbl_common[] = { + { NFS4_OK, 0 }, + { NFS4ERR_PERM, -EPERM }, + { NFS4ERR_NOENT, -ENOENT }, + { NFS4ERR_IO, -EIO }, + { NFS4ERR_NXIO, -ENXIO }, + { NFS4ERR_ACCESS, -EACCES }, + { NFS4ERR_EXIST, -EEXIST }, + { NFS4ERR_XDEV, -EXDEV }, + { NFS4ERR_NOTDIR, -ENOTDIR }, + { NFS4ERR_ISDIR, -EISDIR }, + { NFS4ERR_INVAL, -EINVAL }, + { NFS4ERR_FBIG, -EFBIG }, + { NFS4ERR_NOSPC, -ENOSPC }, + { NFS4ERR_ROFS, -EROFS }, + { NFS4ERR_MLINK, -EMLINK }, + { NFS4ERR_NAMETOOLONG, -ENAMETOOLONG }, + { NFS4ERR_NOTEMPTY, -ENOTEMPTY }, + { NFS4ERR_DQUOT, -EDQUOT }, + { NFS4ERR_STALE, -ESTALE }, + { NFS4ERR_BADHANDLE, -EBADHANDLE }, + { NFS4ERR_BAD_COOKIE, -EBADCOOKIE }, + { NFS4ERR_NOTSUPP, -ENOTSUPP }, + { NFS4ERR_TOOSMALL, -ETOOSMALL }, + { NFS4ERR_BADTYPE, -EBADTYPE }, + { NFS4ERR_SYMLINK, -ELOOP }, + { NFS4ERR_DEADLOCK, -EDEADLK }, +}; + +static const struct { + int stat; + int errno; +} nfs4_errtbl[] = { + { NFS4ERR_SERVERFAULT, -EREMOTEIO }, + { NFS4ERR_LOCKED, -EAGAIN }, + { NFS4ERR_OP_ILLEGAL, -EOPNOTSUPP }, + { NFS4ERR_NOXATTR, -ENODATA }, + { NFS4ERR_XATTR2BIG, -E2BIG }, +}; + +/* + * Convert an NFS error code to a local one. + * This one is used by NFSv4. + */ +int nfs4_stat_to_errno(int stat) +{ + int i; + + /* First check nfs4_errtbl_common */ + for (i = 0; i < ARRAY_SIZE(nfs4_errtbl_common); i++) { + if (nfs4_errtbl_common[i].stat == stat) + return nfs4_errtbl_common[i].errno; + } + /* Then check nfs4_errtbl */ + for (i = 0; i < ARRAY_SIZE(nfs4_errtbl); i++) { + if (nfs4_errtbl[i].stat == stat) + return nfs4_errtbl[i].errno; + } + if (stat <= 10000 || stat > 10100) { + /* The server is looney tunes. */ + return -EREMOTEIO; + } + /* If we cannot translate the error, the recovery routines should + * handle it. + * Note: remaining NFSv4 error codes have values > 10000, so should + * not conflict with native Linux error codes. + */ + return -stat; +} +EXPORT_SYMBOL_GPL(nfs4_stat_to_errno); + +/* + * This table is useful for conversion from local errno to NFS error. + * It provides more logically correct mappings for use with LOCALIO + * (which is focused on converting from errno to NFS status). + */ +static const struct { + int stat; + int errno; +} nfs4_errtbl_localio[] = { + /* Map errors differently than nfs4_errtbl */ + { NFS4ERR_IO, -EREMOTEIO }, + { NFS4ERR_DELAY, -EAGAIN }, + { NFS4ERR_FBIG, -E2BIG }, + /* Map errors not handled by nfs4_errtbl */ + { NFS4ERR_STALE, -EBADF }, + { NFS4ERR_STALE, -EOPENSTALE }, + { NFS4ERR_DELAY, -ETIMEDOUT }, + { NFS4ERR_DELAY, -ERESTARTSYS }, + { NFS4ERR_DELAY, -ENOMEM }, + { NFS4ERR_IO, -ETXTBSY }, + { NFS4ERR_IO, -EBUSY }, + { NFS4ERR_SERVERFAULT, -ESERVERFAULT }, + { NFS4ERR_SERVERFAULT, -ENFILE }, + { NFS4ERR_IO, -EUCLEAN }, + { NFS4ERR_PERM, -ENOKEY }, +}; + +/* + * Convert an errno to an NFS error code for LOCALIO. + */ +__u32 nfs_localio_errno_to_nfs4_stat(int errno) +{ + int i; + + /* First check nfs4_errtbl_common */ + for (i = 0; i < ARRAY_SIZE(nfs4_errtbl_common); i++) { + if (nfs4_errtbl_common[i].errno == errno) + return nfs4_errtbl_common[i].stat; + } + /* Then check nfs4_errtbl_localio */ + for (i = 0; i < ARRAY_SIZE(nfs4_errtbl_localio); i++) { + if (nfs4_errtbl_localio[i].errno == errno) + return nfs4_errtbl_localio[i].stat; + } + /* If we cannot translate the error, the recovery routines should + * handle it. + * Note: remaining NFSv4 error codes have values > 10000, so should + * not conflict with native Linux error codes. + */ + return NFS4ERR_SERVERFAULT; +} +EXPORT_SYMBOL_GPL(nfs_localio_errno_to_nfs4_stat); diff --git a/fs/nfs_common/grace.c b/fs/nfs_common/grace.c new file mode 100644 index 000000000..27cd0d131 --- /dev/null +++ b/fs/nfs_common/grace.c @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Common code for control of lockd and nfsv4 grace periods. + * + * Transplanted from lockd code + */ + +#include +#include +#include +#include +#include + +static unsigned int grace_net_id; +static DEFINE_SPINLOCK(grace_lock); + +/** + * locks_start_grace + * @net: net namespace that this lock manager belongs to + * @lm: who this grace period is for + * + * A grace period is a period during which locks should not be given + * out. Currently grace periods are only enforced by the two lock + * managers (lockd and nfsd), using the locks_in_grace() function to + * check when they are in a grace period. + * + * This function is called to start a grace period. + */ +void +locks_start_grace(struct net *net, struct lock_manager *lm) +{ + struct list_head *grace_list = net_generic(net, grace_net_id); + + spin_lock(&grace_lock); + if (list_empty(&lm->list)) + list_add(&lm->list, grace_list); + else + WARN(1, "double list_add attempt detected in net %x %s\n", + net->ns.inum, (net == &init_net) ? "(init_net)" : ""); + spin_unlock(&grace_lock); +} +EXPORT_SYMBOL_GPL(locks_start_grace); + +/** + * locks_end_grace + * @lm: who this grace period is for + * + * Call this function to state that the given lock manager is ready to + * resume regular locking. The grace period will not end until all lock + * managers that called locks_start_grace() also call locks_end_grace(). + * Note that callers count on it being safe to call this more than once, + * and the second call should be a no-op. + */ +void +locks_end_grace(struct lock_manager *lm) +{ + spin_lock(&grace_lock); + list_del_init(&lm->list); + spin_unlock(&grace_lock); +} +EXPORT_SYMBOL_GPL(locks_end_grace); + +static bool +__state_in_grace(struct net *net, bool open) +{ + struct list_head *grace_list = net_generic(net, grace_net_id); + struct lock_manager *lm; + + if (!open) + return !list_empty(grace_list); + + spin_lock(&grace_lock); + list_for_each_entry(lm, grace_list, list) { + if (lm->block_opens) { + spin_unlock(&grace_lock); + return true; + } + } + spin_unlock(&grace_lock); + return false; +} + +/** + * locks_in_grace + * @net: network namespace + * + * Lock managers call this function to determine when it is OK for them + * to answer ordinary lock requests, and when they should accept only + * lock reclaims. + */ +bool locks_in_grace(struct net *net) +{ + return __state_in_grace(net, false); +} +EXPORT_SYMBOL_GPL(locks_in_grace); + +bool opens_in_grace(struct net *net) +{ + return __state_in_grace(net, true); +} +EXPORT_SYMBOL_GPL(opens_in_grace); + +static int __net_init +grace_init_net(struct net *net) +{ + struct list_head *grace_list = net_generic(net, grace_net_id); + + INIT_LIST_HEAD(grace_list); + return 0; +} + +static void __net_exit +grace_exit_net(struct net *net) +{ + struct list_head *grace_list = net_generic(net, grace_net_id); + + WARN_ONCE(!list_empty(grace_list), + "net %x %s: grace_list is not empty\n", + net->ns.inum, __func__); +} + +static struct pernet_operations grace_net_ops = { + .init = grace_init_net, + .exit = grace_exit_net, + .id = &grace_net_id, + .size = sizeof(struct list_head), +}; + +static int __init +init_grace(void) +{ + return register_pernet_subsys(&grace_net_ops); +} + +static void __exit +exit_grace(void) +{ + unregister_pernet_subsys(&grace_net_ops); +} + +MODULE_AUTHOR("Jeff Layton "); +MODULE_DESCRIPTION("NFS client and server infrastructure"); +MODULE_LICENSE("GPL"); +module_init(init_grace) +module_exit(exit_grace) diff --git a/fs/nfs_common/localio_trace.c b/fs/nfs_common/localio_trace.c new file mode 100644 index 000000000..7decfe57a --- /dev/null +++ b/fs/nfs_common/localio_trace.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2024 Trond Myklebust + * Copyright (C) 2024 Mike Snitzer + */ +#include +#include + +#define CREATE_TRACE_POINTS +#include "localio_trace.h" diff --git a/fs/nfs_common/localio_trace.h b/fs/nfs_common/localio_trace.h new file mode 100644 index 000000000..4055aec9f --- /dev/null +++ b/fs/nfs_common/localio_trace.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2024 Trond Myklebust + * Copyright (C) 2024 Mike Snitzer + */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM nfs_localio + +#if !defined(_TRACE_NFS_COMMON_LOCALIO_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_NFS_COMMON_LOCALIO_H + +#include + +#include +#include +#include + +DECLARE_EVENT_CLASS(nfs_local_client_event, + TP_PROTO( + const struct nfs_client *clp + ), + + TP_ARGS(clp), + + TP_STRUCT__entry( + __field(unsigned int, protocol) + __string(server, clp->cl_hostname) + ), + + TP_fast_assign( + __entry->protocol = clp->rpc_ops->version; + __assign_str(server); + ), + + TP_printk( + "server=%s NFSv%u", __get_str(server), __entry->protocol + ) +); + +#define DEFINE_NFS_LOCAL_CLIENT_EVENT(name) \ + DEFINE_EVENT(nfs_local_client_event, name, \ + TP_PROTO( \ + const struct nfs_client *clp \ + ), \ + TP_ARGS(clp)) + +DEFINE_NFS_LOCAL_CLIENT_EVENT(nfs_localio_enable_client); +DEFINE_NFS_LOCAL_CLIENT_EVENT(nfs_localio_disable_client); + +#endif /* _TRACE_NFS_COMMON_LOCALIO_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE localio_trace +/* This part must be outside protection */ +#include diff --git a/fs/nfs_common/nfs_ssc.c b/fs/nfs_common/nfs_ssc.c new file mode 100644 index 000000000..832246b22 --- /dev/null +++ b/fs/nfs_common/nfs_ssc.c @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Helper for knfsd's SSC to access ops in NFS client modules + * + * Author: Dai Ngo + * + * Copyright (c) 2020, Oracle and/or its affiliates. + */ + +#include +#include +#include +#include "../nfs/nfs4_fs.h" + + +struct nfs_ssc_client_ops_tbl nfs_ssc_client_tbl; +EXPORT_SYMBOL_GPL(nfs_ssc_client_tbl); + +#ifdef CONFIG_NFS_V4_2 +/** + * nfs42_ssc_register - install the NFS_V4 client ops in the nfs_ssc_client_tbl + * @ops: NFS_V4 ops to be installed + * + * Return values: + * None + */ +void nfs42_ssc_register(const struct nfs4_ssc_client_ops *ops) +{ + nfs_ssc_client_tbl.ssc_nfs4_ops = ops; +} +EXPORT_SYMBOL_GPL(nfs42_ssc_register); + +/** + * nfs42_ssc_unregister - uninstall the NFS_V4 client ops from + * the nfs_ssc_client_tbl + * @ops: ops to be uninstalled + * + * Return values: + * None + */ +void nfs42_ssc_unregister(const struct nfs4_ssc_client_ops *ops) +{ + if (nfs_ssc_client_tbl.ssc_nfs4_ops != ops) + return; + + nfs_ssc_client_tbl.ssc_nfs4_ops = NULL; +} +EXPORT_SYMBOL_GPL(nfs42_ssc_unregister); +#endif /* CONFIG_NFS_V4_2 */ + +#ifdef CONFIG_NFS_V4_2 +/** + * nfs_ssc_register - install the NFS_FS client ops in the nfs_ssc_client_tbl + * @ops: NFS_FS ops to be installed + * + * Return values: + * None + */ +void nfs_ssc_register(const struct nfs_ssc_client_ops *ops) +{ + nfs_ssc_client_tbl.ssc_nfs_ops = ops; +} +EXPORT_SYMBOL_GPL(nfs_ssc_register); + +/** + * nfs_ssc_unregister - uninstall the NFS_FS client ops from + * the nfs_ssc_client_tbl + * @ops: ops to be uninstalled + * + * Return values: + * None + */ +void nfs_ssc_unregister(const struct nfs_ssc_client_ops *ops) +{ + if (nfs_ssc_client_tbl.ssc_nfs_ops != ops) + return; + nfs_ssc_client_tbl.ssc_nfs_ops = NULL; +} +EXPORT_SYMBOL_GPL(nfs_ssc_unregister); + +#else +void nfs_ssc_register(const struct nfs_ssc_client_ops *ops) +{ +} +EXPORT_SYMBOL_GPL(nfs_ssc_register); + +void nfs_ssc_unregister(const struct nfs_ssc_client_ops *ops) +{ +} +EXPORT_SYMBOL_GPL(nfs_ssc_unregister); +#endif /* CONFIG_NFS_V4_2 */ diff --git a/fs/nfs_common/nfsacl.c b/fs/nfs_common/nfsacl.c new file mode 100644 index 000000000..e2eaac14f --- /dev/null +++ b/fs/nfs_common/nfsacl.c @@ -0,0 +1,423 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * fs/nfs_common/nfsacl.c + * + * Copyright (C) 2002-2003 Andreas Gruenbacher + */ + +/* + * The Solaris nfsacl protocol represents some ACLs slightly differently + * than POSIX 1003.1e draft 17 does (and we do): + * + * - Minimal ACLs always have an ACL_MASK entry, so they have + * four instead of three entries. + * - The ACL_MASK entry in such minimal ACLs always has the same + * permissions as the ACL_GROUP_OBJ entry. (In extended ACLs + * the ACL_MASK and ACL_GROUP_OBJ entries may differ.) + * - The identifier fields of the ACL_USER_OBJ and ACL_GROUP_OBJ + * entries contain the identifiers of the owner and owning group. + * (In POSIX ACLs we always set them to ACL_UNDEFINED_ID). + * - ACL entries in the kernel are kept sorted in ascending order + * of (e_tag, e_id). Solaris ACLs are unsorted. + */ + +#include +#include +#include +#include +#include +#include +#include + +MODULE_DESCRIPTION("NFS ACL support"); +MODULE_LICENSE("GPL"); + +struct nfsacl_encode_desc { + struct xdr_array2_desc desc; + unsigned int count; + struct posix_acl *acl; + int typeflag; + kuid_t uid; + kgid_t gid; +}; + +struct nfsacl_simple_acl { + struct posix_acl_hdr acl; + struct posix_acl_entry ace[4]; +}; + +static int +xdr_nfsace_encode(struct xdr_array2_desc *desc, void *elem) +{ + struct nfsacl_encode_desc *nfsacl_desc = + (struct nfsacl_encode_desc *) desc; + __be32 *p = elem; + + struct posix_acl_entry *entry = + &nfsacl_desc->acl->a_entries[nfsacl_desc->count++]; + + *p++ = htonl(entry->e_tag | nfsacl_desc->typeflag); + switch(entry->e_tag) { + case ACL_USER_OBJ: + *p++ = htonl(from_kuid(&init_user_ns, nfsacl_desc->uid)); + break; + case ACL_GROUP_OBJ: + *p++ = htonl(from_kgid(&init_user_ns, nfsacl_desc->gid)); + break; + case ACL_USER: + *p++ = htonl(from_kuid(&init_user_ns, entry->e_uid)); + break; + case ACL_GROUP: + *p++ = htonl(from_kgid(&init_user_ns, entry->e_gid)); + break; + default: /* Solaris depends on that! */ + *p++ = 0; + break; + } + *p++ = htonl(entry->e_perm & S_IRWXO); + return 0; +} + +/** + * nfsacl_encode - Encode an NFSv3 ACL + * + * @buf: destination xdr_buf to contain XDR encoded ACL + * @base: byte offset in xdr_buf where XDR'd ACL begins + * @inode: inode of file whose ACL this is + * @acl: posix_acl to encode + * @encode_entries: whether to encode ACEs as well + * @typeflag: ACL type: NFS_ACL_DEFAULT or zero + * + * Returns size of encoded ACL in bytes or a negative errno value. + */ +int nfsacl_encode(struct xdr_buf *buf, unsigned int base, struct inode *inode, + struct posix_acl *acl, int encode_entries, int typeflag) +{ + int entries = (acl && acl->a_count) ? max_t(int, acl->a_count, 4) : 0; + struct nfsacl_encode_desc nfsacl_desc = { + .desc = { + .elem_size = 12, + .array_len = encode_entries ? entries : 0, + .xcode = xdr_nfsace_encode, + }, + .acl = acl, + .typeflag = typeflag, + .uid = inode->i_uid, + .gid = inode->i_gid, + }; + struct nfsacl_simple_acl aclbuf; + int err; + + if (entries > NFS_ACL_MAX_ENTRIES || + xdr_encode_word(buf, base, entries)) + return -EINVAL; + if (encode_entries && acl && acl->a_count == 3) { + struct posix_acl *acl2 = + container_of(&aclbuf.acl, struct posix_acl, hdr); + + /* Avoid the use of posix_acl_alloc(). nfsacl_encode() is + * invoked in contexts where a memory allocation failure is + * fatal. Fortunately this fake ACL is small enough to + * construct on the stack. */ + posix_acl_init(acl2, 4); + + /* Insert entries in canonical order: other orders seem + to confuse Solaris VxFS. */ + acl2->a_entries[0] = acl->a_entries[0]; /* ACL_USER_OBJ */ + acl2->a_entries[1] = acl->a_entries[1]; /* ACL_GROUP_OBJ */ + acl2->a_entries[2] = acl->a_entries[1]; /* ACL_MASK */ + acl2->a_entries[2].e_tag = ACL_MASK; + acl2->a_entries[3] = acl->a_entries[2]; /* ACL_OTHER */ + nfsacl_desc.acl = acl2; + } + err = xdr_encode_array2(buf, base + 4, &nfsacl_desc.desc); + if (!err) + err = 8 + nfsacl_desc.desc.elem_size * + nfsacl_desc.desc.array_len; + return err; +} +EXPORT_SYMBOL_GPL(nfsacl_encode); + +/** + * nfs_stream_encode_acl - Encode an NFSv3 ACL + * + * @xdr: an xdr_stream positioned to receive an encoded ACL + * @inode: inode of file whose ACL this is + * @acl: posix_acl to encode + * @encode_entries: whether to encode ACEs as well + * @typeflag: ACL type: NFS_ACL_DEFAULT or zero + * + * Return values: + * %false: The ACL could not be encoded + * %true: @xdr is advanced to the next available position + */ +bool nfs_stream_encode_acl(struct xdr_stream *xdr, struct inode *inode, + struct posix_acl *acl, int encode_entries, + int typeflag) +{ + const size_t elem_size = XDR_UNIT * 3; + u32 entries = (acl && acl->a_count) ? max_t(int, acl->a_count, 4) : 0; + struct nfsacl_encode_desc nfsacl_desc = { + .desc = { + .elem_size = elem_size, + .array_len = encode_entries ? entries : 0, + .xcode = xdr_nfsace_encode, + }, + .acl = acl, + .typeflag = typeflag, + .uid = inode->i_uid, + .gid = inode->i_gid, + }; + struct nfsacl_simple_acl aclbuf; + unsigned int base; + int err; + + if (entries > NFS_ACL_MAX_ENTRIES) + return false; + if (xdr_stream_encode_u32(xdr, entries) < 0) + return false; + + if (encode_entries && acl && acl->a_count == 3) { + struct posix_acl *acl2 = + container_of(&aclbuf.acl, struct posix_acl, hdr); + + /* Avoid the use of posix_acl_alloc(). nfsacl_encode() is + * invoked in contexts where a memory allocation failure is + * fatal. Fortunately this fake ACL is small enough to + * construct on the stack. */ + posix_acl_init(acl2, 4); + + /* Insert entries in canonical order: other orders seem + to confuse Solaris VxFS. */ + acl2->a_entries[0] = acl->a_entries[0]; /* ACL_USER_OBJ */ + acl2->a_entries[1] = acl->a_entries[1]; /* ACL_GROUP_OBJ */ + acl2->a_entries[2] = acl->a_entries[1]; /* ACL_MASK */ + acl2->a_entries[2].e_tag = ACL_MASK; + acl2->a_entries[3] = acl->a_entries[2]; /* ACL_OTHER */ + nfsacl_desc.acl = acl2; + } + + base = xdr_stream_pos(xdr); + if (!xdr_reserve_space(xdr, XDR_UNIT + + elem_size * nfsacl_desc.desc.array_len)) + return false; + err = xdr_encode_array2(xdr->buf, base, &nfsacl_desc.desc); + if (err) + return false; + + return true; +} +EXPORT_SYMBOL_GPL(nfs_stream_encode_acl); + + +struct nfsacl_decode_desc { + struct xdr_array2_desc desc; + unsigned int count; + struct posix_acl *acl; +}; + +static int +xdr_nfsace_decode(struct xdr_array2_desc *desc, void *elem) +{ + struct nfsacl_decode_desc *nfsacl_desc = + (struct nfsacl_decode_desc *) desc; + __be32 *p = elem; + struct posix_acl_entry *entry; + unsigned int id; + + if (!nfsacl_desc->acl) { + if (desc->array_len > NFS_ACL_MAX_ENTRIES) + return -EINVAL; + nfsacl_desc->acl = posix_acl_alloc(desc->array_len, GFP_KERNEL); + if (!nfsacl_desc->acl) + return -ENOMEM; + nfsacl_desc->count = 0; + } + + entry = &nfsacl_desc->acl->a_entries[nfsacl_desc->count++]; + entry->e_tag = ntohl(*p++) & ~NFS_ACL_DEFAULT; + id = ntohl(*p++); + entry->e_perm = ntohl(*p++); + + switch(entry->e_tag) { + case ACL_USER: + entry->e_uid = make_kuid(&init_user_ns, id); + if (!uid_valid(entry->e_uid)) + return -EINVAL; + break; + case ACL_GROUP: + entry->e_gid = make_kgid(&init_user_ns, id); + if (!gid_valid(entry->e_gid)) + return -EINVAL; + break; + case ACL_USER_OBJ: + case ACL_GROUP_OBJ: + case ACL_OTHER: + if (entry->e_perm & ~S_IRWXO) + return -EINVAL; + break; + case ACL_MASK: + /* Solaris sometimes sets additional bits in the mask */ + entry->e_perm &= S_IRWXO; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int +cmp_acl_entry(const void *x, const void *y) +{ + const struct posix_acl_entry *a = x, *b = y; + + if (a->e_tag != b->e_tag) + return a->e_tag - b->e_tag; + else if ((a->e_tag == ACL_USER) && uid_gt(a->e_uid, b->e_uid)) + return 1; + else if ((a->e_tag == ACL_USER) && uid_lt(a->e_uid, b->e_uid)) + return -1; + else if ((a->e_tag == ACL_GROUP) && gid_gt(a->e_gid, b->e_gid)) + return 1; + else if ((a->e_tag == ACL_GROUP) && gid_lt(a->e_gid, b->e_gid)) + return -1; + else + return 0; +} + +/* + * Convert from a Solaris ACL to a POSIX 1003.1e draft 17 ACL. + */ +static int +posix_acl_from_nfsacl(struct posix_acl *acl) +{ + struct posix_acl_entry *pa, *pe, + *group_obj = NULL, *mask = NULL; + + if (!acl) + return 0; + + sort(acl->a_entries, acl->a_count, sizeof(struct posix_acl_entry), + cmp_acl_entry, NULL); + + /* Find the ACL_GROUP_OBJ and ACL_MASK entries. */ + FOREACH_ACL_ENTRY(pa, acl, pe) { + switch(pa->e_tag) { + case ACL_USER_OBJ: + break; + case ACL_GROUP_OBJ: + group_obj = pa; + break; + case ACL_MASK: + mask = pa; + fallthrough; + case ACL_OTHER: + break; + } + } + if (acl->a_count == 4 && group_obj && mask && + mask->e_perm == group_obj->e_perm) { + /* remove bogus ACL_MASK entry */ + memmove(mask, mask+1, (3 - (mask - acl->a_entries)) * + sizeof(struct posix_acl_entry)); + acl->a_count = 3; + } + return 0; +} + +/** + * nfsacl_decode - Decode an NFSv3 ACL + * + * @buf: xdr_buf containing XDR'd ACL data to decode + * @base: byte offset in xdr_buf where XDR'd ACL begins + * @aclcnt: count of ACEs in decoded posix_acl + * @pacl: buffer in which to place decoded posix_acl + * + * Returns the length of the decoded ACL in bytes, or a negative errno value. + */ +int nfsacl_decode(struct xdr_buf *buf, unsigned int base, unsigned int *aclcnt, + struct posix_acl **pacl) +{ + struct nfsacl_decode_desc nfsacl_desc = { + .desc = { + .elem_size = 12, + .xcode = pacl ? xdr_nfsace_decode : NULL, + }, + }; + u32 entries; + int err; + + if (xdr_decode_word(buf, base, &entries) || + entries > NFS_ACL_MAX_ENTRIES) + return -EINVAL; + nfsacl_desc.desc.array_maxlen = entries; + err = xdr_decode_array2(buf, base + 4, &nfsacl_desc.desc); + if (err) + return err; + if (pacl) { + if (entries != nfsacl_desc.desc.array_len || + posix_acl_from_nfsacl(nfsacl_desc.acl) != 0) { + posix_acl_release(nfsacl_desc.acl); + return -EINVAL; + } + *pacl = nfsacl_desc.acl; + } + if (aclcnt) + *aclcnt = entries; + return 8 + nfsacl_desc.desc.elem_size * + nfsacl_desc.desc.array_len; +} +EXPORT_SYMBOL_GPL(nfsacl_decode); + +/** + * nfs_stream_decode_acl - Decode an NFSv3 ACL + * + * @xdr: an xdr_stream positioned at an encoded ACL + * @aclcnt: OUT: count of ACEs in decoded posix_acl + * @pacl: OUT: a dynamically-allocated buffer containing the decoded posix_acl + * + * Return values: + * %false: The encoded ACL is not valid + * %true: @pacl contains a decoded ACL, and @xdr is advanced + * + * On a successful return, caller must release *pacl using posix_acl_release(). + */ +bool nfs_stream_decode_acl(struct xdr_stream *xdr, unsigned int *aclcnt, + struct posix_acl **pacl) +{ + const size_t elem_size = XDR_UNIT * 3; + struct nfsacl_decode_desc nfsacl_desc = { + .desc = { + .elem_size = elem_size, + .xcode = pacl ? xdr_nfsace_decode : NULL, + }, + }; + unsigned int base; + u32 entries; + + if (xdr_stream_decode_u32(xdr, &entries) < 0) + return false; + if (entries > NFS_ACL_MAX_ENTRIES) + return false; + + base = xdr_stream_pos(xdr); + if (!xdr_inline_decode(xdr, XDR_UNIT + elem_size * entries)) + return false; + nfsacl_desc.desc.array_maxlen = entries; + if (xdr_decode_array2(xdr->buf, base, &nfsacl_desc.desc)) + return false; + + if (pacl) { + if (entries != nfsacl_desc.desc.array_len || + posix_acl_from_nfsacl(nfsacl_desc.acl) != 0) { + posix_acl_release(nfsacl_desc.acl); + return false; + } + *pacl = nfsacl_desc.acl; + } + if (aclcnt) + *aclcnt = entries; + return true; +} +EXPORT_SYMBOL_GPL(nfs_stream_decode_acl); diff --git a/fs/nfs_common/nfslocalio.c b/fs/nfs_common/nfslocalio.c new file mode 100644 index 000000000..85aa03a7b --- /dev/null +++ b/fs/nfs_common/nfslocalio.c @@ -0,0 +1,387 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2024 Mike Snitzer + * Copyright (C) 2024 NeilBrown + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "localio_trace.h" + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("NFS localio protocol bypass support"); + +static DEFINE_SPINLOCK(nfs_uuids_lock); + +/* + * Global list of nfs_uuid_t instances + * that is protected by nfs_uuids_lock. + */ +static LIST_HEAD(nfs_uuids); + +/* + * Lock ordering: + * 1: nfs_uuid->lock + * 2: nfs_uuids_lock + * 3: nfs_uuid->list_lock (aka nn->local_clients_lock) + * + * May skip locks in select cases, but never hold multiple + * locks out of order. + */ + +void nfs_uuid_init(nfs_uuid_t *nfs_uuid) +{ + RCU_INIT_POINTER(nfs_uuid->net, NULL); + nfs_uuid->dom = NULL; + nfs_uuid->list_lock = NULL; + INIT_LIST_HEAD(&nfs_uuid->list); + INIT_LIST_HEAD(&nfs_uuid->files); + spin_lock_init(&nfs_uuid->lock); + nfs_uuid->nfs3_localio_probe_count = 0; +} +EXPORT_SYMBOL_GPL(nfs_uuid_init); + +bool nfs_uuid_begin(nfs_uuid_t *nfs_uuid) +{ + spin_lock(&nfs_uuid->lock); + if (rcu_access_pointer(nfs_uuid->net)) { + /* This nfs_uuid is already in use */ + spin_unlock(&nfs_uuid->lock); + return false; + } + + spin_lock(&nfs_uuids_lock); + if (!list_empty(&nfs_uuid->list)) { + /* This nfs_uuid is already in use */ + spin_unlock(&nfs_uuids_lock); + spin_unlock(&nfs_uuid->lock); + return false; + } + list_add_tail(&nfs_uuid->list, &nfs_uuids); + spin_unlock(&nfs_uuids_lock); + + uuid_gen(&nfs_uuid->uuid); + spin_unlock(&nfs_uuid->lock); + + return true; +} +EXPORT_SYMBOL_GPL(nfs_uuid_begin); + +void nfs_uuid_end(nfs_uuid_t *nfs_uuid) +{ + if (!rcu_access_pointer(nfs_uuid->net)) { + spin_lock(&nfs_uuid->lock); + if (!rcu_access_pointer(nfs_uuid->net)) { + /* Not local, remove from nfs_uuids */ + spin_lock(&nfs_uuids_lock); + list_del_init(&nfs_uuid->list); + spin_unlock(&nfs_uuids_lock); + } + spin_unlock(&nfs_uuid->lock); + } +} +EXPORT_SYMBOL_GPL(nfs_uuid_end); + +static nfs_uuid_t * nfs_uuid_lookup_locked(const uuid_t *uuid) +{ + nfs_uuid_t *nfs_uuid; + + list_for_each_entry(nfs_uuid, &nfs_uuids, list) + if (uuid_equal(&nfs_uuid->uuid, uuid)) + return nfs_uuid; + + return NULL; +} + +static struct module *nfsd_mod; + +void nfs_uuid_is_local(const uuid_t *uuid, struct list_head *list, + spinlock_t *list_lock, struct net *net, + struct auth_domain *dom, struct module *mod) +{ + nfs_uuid_t *nfs_uuid; + + spin_lock(&nfs_uuids_lock); + nfs_uuid = nfs_uuid_lookup_locked(uuid); + if (!nfs_uuid) { + spin_unlock(&nfs_uuids_lock); + return; + } + + /* + * We don't hold a ref on the net, but instead put + * ourselves on @list (nn->local_clients) so the net + * pointer can be invalidated. + */ + spin_lock(list_lock); /* list_lock is nn->local_clients_lock */ + list_move(&nfs_uuid->list, list); + spin_unlock(list_lock); + + spin_unlock(&nfs_uuids_lock); + /* Once nfs_uuid is parented to @list, avoid global nfs_uuids_lock */ + spin_lock(&nfs_uuid->lock); + + __module_get(mod); + nfsd_mod = mod; + + nfs_uuid->list_lock = list_lock; + kref_get(&dom->ref); + nfs_uuid->dom = dom; + rcu_assign_pointer(nfs_uuid->net, net); + spin_unlock(&nfs_uuid->lock); +} +EXPORT_SYMBOL_GPL(nfs_uuid_is_local); + +void nfs_localio_enable_client(struct nfs_client *clp) +{ + /* nfs_uuid_is_local() does the actual enablement */ + trace_nfs_localio_enable_client(clp); +} +EXPORT_SYMBOL_GPL(nfs_localio_enable_client); + +/* + * Cleanup the nfs_uuid_t embedded in an nfs_client. + * This is the long-form of nfs_uuid_init(). + */ +static bool nfs_uuid_put(nfs_uuid_t *nfs_uuid) +{ + struct nfs_file_localio *nfl; + + spin_lock(&nfs_uuid->lock); + if (unlikely(!rcu_access_pointer(nfs_uuid->net))) { + spin_unlock(&nfs_uuid->lock); + return false; + } + RCU_INIT_POINTER(nfs_uuid->net, NULL); + + if (nfs_uuid->dom) { + auth_domain_put(nfs_uuid->dom); + nfs_uuid->dom = NULL; + } + + /* Walk list of files and ensure their last references dropped */ + + while ((nfl = list_first_entry_or_null(&nfs_uuid->files, + struct nfs_file_localio, + list)) != NULL) { + /* If nfs_uuid is already NULL, nfs_close_local_fh is + * closing and we must wait, else we unlink and close. + */ + if (rcu_access_pointer(nfl->nfs_uuid) == NULL) { + /* nfs_close_local_fh() is doing the + * close and we must wait. until it unlinks + */ + wait_var_event_spinlock(nfs_uuid, + list_first_entry_or_null( + &nfs_uuid->files, + struct nfs_file_localio, + list) != nfl, + &nfs_uuid->lock); + continue; + } + + /* Remove nfl from nfs_uuid->files list */ + list_del_init(&nfl->list); + spin_unlock(&nfs_uuid->lock); + + nfs_to_nfsd_file_put_local(&nfl->ro_file); + nfs_to_nfsd_file_put_local(&nfl->rw_file); + cond_resched(); + + spin_lock(&nfs_uuid->lock); + /* Now we can allow racing nfs_close_local_fh() to + * skip the locking. + */ + store_release_wake_up(&nfl->nfs_uuid, RCU_INITIALIZER(NULL)); + } + + /* Remove client from nn->local_clients */ + if (nfs_uuid->list_lock) { + spin_lock(nfs_uuid->list_lock); + BUG_ON(list_empty(&nfs_uuid->list)); + list_del_init(&nfs_uuid->list); + spin_unlock(nfs_uuid->list_lock); + nfs_uuid->list_lock = NULL; + } + + module_put(nfsd_mod); + spin_unlock(&nfs_uuid->lock); + + return true; +} + +void nfs_localio_disable_client(struct nfs_client *clp) +{ + if (nfs_uuid_put(&clp->cl_uuid)) + trace_nfs_localio_disable_client(clp); +} +EXPORT_SYMBOL_GPL(nfs_localio_disable_client); + +void nfs_localio_invalidate_clients(struct list_head *nn_local_clients, + spinlock_t *nn_local_clients_lock) +{ + LIST_HEAD(local_clients); + nfs_uuid_t *nfs_uuid, *tmp; + struct nfs_client *clp; + + spin_lock(nn_local_clients_lock); + list_splice_init(nn_local_clients, &local_clients); + spin_unlock(nn_local_clients_lock); + list_for_each_entry_safe(nfs_uuid, tmp, &local_clients, list) { + if (WARN_ON(nfs_uuid->list_lock != nn_local_clients_lock)) + break; + clp = container_of(nfs_uuid, struct nfs_client, cl_uuid); + nfs_localio_disable_client(clp); + } +} +EXPORT_SYMBOL_GPL(nfs_localio_invalidate_clients); + +static int nfs_uuid_add_file(nfs_uuid_t *nfs_uuid, struct nfs_file_localio *nfl) +{ + int ret = 0; + + /* Add nfl to nfs_uuid->files if it isn't already */ + spin_lock(&nfs_uuid->lock); + if (rcu_access_pointer(nfs_uuid->net) == NULL) { + ret = -ENXIO; + } else if (list_empty(&nfl->list)) { + rcu_assign_pointer(nfl->nfs_uuid, nfs_uuid); + list_add_tail(&nfl->list, &nfs_uuid->files); + } + spin_unlock(&nfs_uuid->lock); + return ret; +} + +/* + * Caller is responsible for calling nfsd_net_put and + * nfsd_file_put (via nfs_to_nfsd_file_put_local). + */ +struct nfsd_file *nfs_open_local_fh(nfs_uuid_t *uuid, + struct rpc_clnt *rpc_clnt, const struct cred *cred, + const struct nfs_fh *nfs_fh, struct nfs_file_localio *nfl, + struct nfsd_file __rcu **pnf, + const fmode_t fmode) +{ + struct net *net; + struct nfsd_file *localio; + + /* + * Not running in nfsd context, so must safely get reference on nfsd_serv. + * But the server may already be shutting down, if so disallow new localio. + * uuid->net is NOT a counted reference, but rcu_read_lock() ensures that + * if uuid->net is not NULL, then calling nfsd_net_try_get() is safe + * and if it succeeds we will have an implied reference to the net. + * + * Otherwise NFS may not have ref on NFSD and therefore cannot safely + * make 'nfs_to' calls. + */ + rcu_read_lock(); + net = rcu_dereference(uuid->net); + if (!net || !nfs_to->nfsd_net_try_get(net)) { + rcu_read_unlock(); + return ERR_PTR(-ENXIO); + } + rcu_read_unlock(); + /* We have an implied reference to net thanks to nfsd_net_try_get */ + localio = nfs_to->nfsd_open_local_fh(net, uuid->dom, rpc_clnt, cred, + nfs_fh, pnf, fmode); + if (!IS_ERR(localio) && nfs_uuid_add_file(uuid, nfl) < 0) { + /* + * Delete the cached file when racing with nfs_uuid_put(). + * Since nfl->nfs_uuid was never published via + * rcu_assign_pointer(), nfs_close_local_fh() will early-return + * and cannot clean up after us. Drop the slot's file ref and + * its paired net ref, then drop the caller-owned nfsd_file ref + * (+1) and the entry-time nfsd_net ref carried via nf->nf_net, + * and return -ENXIO so the caller never dereferences the + * now-cleared localio. + */ + struct nfsd_file __rcu *tmp = + (struct nfsd_file __force __rcu *)localio; + + nfs_to_nfsd_file_put_local(pnf); + nfs_to_nfsd_file_put_local(&tmp); + localio = ERR_PTR(-ENXIO); + } + nfs_to_nfsd_net_put(net); + + return localio; +} +EXPORT_SYMBOL_GPL(nfs_open_local_fh); + +void nfs_close_local_fh(struct nfs_file_localio *nfl) +{ + nfs_uuid_t *nfs_uuid; + + rcu_read_lock(); + nfs_uuid = rcu_dereference(nfl->nfs_uuid); + if (!nfs_uuid) { + /* regular (non-LOCALIO) NFS will hammer this */ + rcu_read_unlock(); + return; + } + + spin_lock(&nfs_uuid->lock); + if (!rcu_access_pointer(nfl->nfs_uuid)) { + /* nfs_uuid_put has finished here */ + spin_unlock(&nfs_uuid->lock); + rcu_read_unlock(); + return; + } + if (list_empty(&nfl->list)) { + /* nfs_uuid_put() has started closing files, wait for it + * to finished + */ + spin_unlock(&nfs_uuid->lock); + rcu_read_unlock(); + wait_var_event(&nfl->nfs_uuid, + rcu_access_pointer(nfl->nfs_uuid) == NULL); + return; + } + /* tell nfs_uuid_put() to wait for us */ + RCU_INIT_POINTER(nfl->nfs_uuid, NULL); + spin_unlock(&nfs_uuid->lock); + rcu_read_unlock(); + + nfs_to_nfsd_file_put_local(&nfl->ro_file); + nfs_to_nfsd_file_put_local(&nfl->rw_file); + + /* Remove nfl from nfs_uuid->files list and signal nfs_uuid_put() + * that we are done. The moment we drop the spinlock the + * nfs_uuid could be freed. + */ + spin_lock(&nfs_uuid->lock); + list_del_init(&nfl->list); + wake_up_var_locked(nfs_uuid, &nfs_uuid->lock); + spin_unlock(&nfs_uuid->lock); +} +EXPORT_SYMBOL_GPL(nfs_close_local_fh); + +/* + * The NFS LOCALIO code needs to call into NFSD using various symbols, + * but cannot be statically linked, because that will make the NFS + * module always depend on the NFSD module. + * + * 'nfs_to' provides NFS access to NFSD functions needed for LOCALIO, + * its lifetime is tightly coupled to the NFSD module and will always + * be available to NFS LOCALIO because any successful client<->server + * LOCALIO handshake results in a reference on the NFSD module (above), + * so NFS implicitly holds a reference to the NFSD module and its + * functions in the 'nfs_to' nfsd_localio_operations cannot disappear. + * + * If the last NFS client using LOCALIO disconnects (and its reference + * on NFSD dropped) then NFSD could be unloaded, resulting in 'nfs_to' + * functions being invalid pointers. But if NFSD isn't loaded then NFS + * will not be able to handshake with NFSD and will have no cause to + * try to call 'nfs_to' function pointers. If/when NFSD is reloaded it + * will reinitialize the 'nfs_to' function pointers and make LOCALIO + * possible. + */ +const struct nfsd_localio_operations *nfs_to; +EXPORT_SYMBOL_GPL(nfs_to); diff --git a/fs/nfsd/Kconfig b/fs/nfsd/Kconfig new file mode 100644 index 000000000..ffb76761d --- /dev/null +++ b/fs/nfsd/Kconfig @@ -0,0 +1,197 @@ +# SPDX-License-Identifier: GPL-2.0-only +config NFSD + tristate "NFS server support" + depends on INET + depends on FILE_LOCKING + depends on FSNOTIFY + select CRC32 + select CRYPTO_LIB_MD5 if NFSD_LEGACY_CLIENT_TRACKING + select CRYPTO_LIB_SHA256 if NFSD_V4 + select CRYPTO # required by RPCSEC_GSS_KRB5 and signed filehandles + select LOCKD + select SUNRPC + select EXPORTFS + select NFS_COMMON + select NFS_ACL_SUPPORT if NFSD_V2_ACL + select NFS_ACL_SUPPORT if NFSD_V3_ACL + depends on MULTIUSER + help + Choose Y here if you want to allow other computers to access + files residing on this system using Sun's Network File System + protocol. To compile the NFS server support as a module, + choose M here: the module will be called nfsd. + + You may choose to use a user-space NFS server instead, in which + case you can choose N here. + + To export local file systems using NFS, you also need to install + user space programs which can be found in the Linux nfs-utils + package, available from http://linux-nfs.org/. More detail about + the Linux NFS server implementation is available via the + exports(5) man page. + + Below you can choose which versions of the NFS protocol are + available to clients mounting the NFS server on this system. + Support for NFS version 3 (RFC 1813) is always available when + CONFIG_NFSD is selected. + + If unsure, say N. + +config NFSD_V2 + bool "NFS server support for NFS version 2 (DEPRECATED)" + depends on NFSD + default n + help + NFSv2 (RFC 1094) was the first publicly-released version of NFS. + Unless you are hosting ancient (1990's era) NFS clients, you don't + need this. + + If unsure, say N. + +config NFSD_V2_ACL + bool "NFS server support for the NFSv2 ACL protocol extension" + depends on NFSD_V2 + +config NFSD_V3_ACL + bool "NFS server support for the NFSv3 ACL protocol extension" + depends on NFSD + help + Solaris NFS servers support an auxiliary NFSv3 ACL protocol that + never became an official part of the NFS version 3 protocol. + This protocol extension allows applications on NFS clients to + manipulate POSIX Access Control Lists on files residing on NFS + servers. NFS servers enforce POSIX ACLs on local files whether + this protocol is available or not. + + This option enables support in your system's NFS server for the + NFSv3 ACL protocol extension allowing NFS clients to manipulate + POSIX ACLs on files exported by your system's NFS server. NFS + clients which support the Solaris NFSv3 ACL protocol can then + access and modify ACLs on your NFS server. + + To store ACLs on your NFS server, you also need to enable ACL- + related CONFIG options for your local file systems of choice. + + If unsure, say N. + +config NFSD_V4 + bool "NFS server support for NFS version 4" + depends on NFSD && PROC_FS + select FS_POSIX_ACL + select RPCSEC_GSS_KRB5 + select GRACE_PERIOD + select NFS_V4_2_SSC_HELPER if NFS_V4_2 + help + This option enables support in your system's NFS server for + version 4 of the NFS protocol (RFC 3530). + + To export files using NFSv4, you need to install additional user + space programs which can be found in the Linux nfs-utils package, + available from http://linux-nfs.org/. + + If unsure, say N. + +config NFSD_PNFS + bool + +config NFSD_BLOCKLAYOUT + bool "NFSv4.1 server support for pNFS block layouts" + depends on NFSD_V4 && BLOCK + select NFSD_PNFS + select EXPORTFS_BLOCK_OPS + help + This option enables support for the exporting pNFS block layouts + in the kernel's NFS server. The pNFS block layout enables NFS + clients to directly perform I/O to block devices accessible to both + the server and the clients. See RFC 5663 for more details. + + If unsure, say N. + +config NFSD_SCSILAYOUT + bool "NFSv4.1 server support for pNFS SCSI layouts" + depends on NFSD_V4 && BLOCK + select NFSD_PNFS + select EXPORTFS_BLOCK_OPS + help + This option enables support for the exporting pNFS SCSI layouts + in the kernel's NFS server. The pNFS SCSI layout enables NFS + clients to directly perform I/O to SCSI devices accessible to both + the server and the clients. See draft-ietf-nfsv4-scsi-layout for + more details. + + If unsure, say N. + +config NFSD_FLEXFILELAYOUT + bool "NFSv4.1 server support for pNFS Flex File layouts" + depends on NFSD_V4 + select NFSD_PNFS + help + This option enables support for the exporting pNFS Flex File + layouts in the kernel's NFS server. The pNFS Flex File layout + enables NFS clients to directly perform I/O to NFSv3 devices + accessible to both the server and the clients. See + draft-ietf-nfsv4-flex-files for more details. + + Warning, this server implements the bare minimum functionality + to be a flex file server - it is for testing the client, + not for use in production. + + If unsure, say N. + +config NFSD_V4_2_INTER_SSC + bool "NFSv4.2 inter server to server COPY" + depends on NFSD_V4 && NFS_V4_2 + help + This option enables support for NFSv4.2 inter server to + server copy where the destination server calls the NFSv4.2 + client to read the data to copy from the source server. + + If unsure, say N. + +config NFSD_V4_SECURITY_LABEL + bool "Provide Security Label support for NFSv4 server" + depends on NFSD_V4 && SECURITY + help + + Say Y here if you want enable fine-grained security label attribute + support for NFS version 4. Security labels allow security modules like + SELinux and Smack to label files to facilitate enforcement of their policies. + Without this an NFSv4 mount will have the same label on each file. + + If you do not wish to enable fine-grained security labels SELinux or + Smack policies on NFSv4 files, say N. + +config NFSD_LEGACY_CLIENT_TRACKING + bool "Support legacy NFSv4 client tracking methods (DEPRECATED)" + depends on NFSD_V4 + default n + help + The NFSv4 server needs to store a small amount of information on + stable storage in order to handle state recovery after reboot. Most + modern deployments upcall to a userland daemon for this (nfsdcld), + but older NFS servers may store information directly in a + recoverydir, or spawn a process directly using a usermodehelper + upcall. + + These legacy client tracking methods have proven to be problematic + and will be removed in the future. Say Y here if you need support + for them in the interim. + +config NFSD_V4_POSIX_ACLS + bool "Support NFSv4 POSIX draft ACLs" + depends on NFSD_V4 + default n + help + Include experimental support for POSIX Access Control Lists + (ACLs) in NFSv4 as specified in the IETF draft + draft-ietf-nfsv4-posix-acls. This protocol extension enables + NFSv4 clients to retrieve and modify POSIX ACLs on exported + filesystems that support them. + + This feature is based on an unratified IETF draft + specification that may change in ways that impact + interoperability with existing clients. Enable only for + testing environments or when interoperability with specific + clients that implement this draft is required. + + If unsure, say N. diff --git a/fs/nfsd/Makefile b/fs/nfsd/Makefile new file mode 100644 index 000000000..f0da4d69d --- /dev/null +++ b/fs/nfsd/Makefile @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the Linux nfs server +# + +ccflags-y += -I$(src) # needed for trace events + +obj-$(CONFIG_NFSD) += nfsd.o + +# this one should be compiled first, as the tracing macros can easily blow up +nfsd-y += trace.o + +nfsd-y += nfssvc.o nfsctl.o nfsfh.o vfs.o \ + export.o auth.o lockd.o nfscache.o \ + stats.o filecache.o nfs3proc.o nfs3xdr.o \ + netlink.o +nfsd-$(CONFIG_NFSD_V2) += nfsproc.o nfsxdr.o +nfsd-$(CONFIG_NFSD_V2_ACL) += nfs2acl.o +nfsd-$(CONFIG_NFSD_V3_ACL) += nfs3acl.o +nfsd-$(CONFIG_NFSD_V4) += nfs4proc.o nfs4xdr.o nfs4state.o nfs4idmap.o \ + nfs4acl.o nfs4callback.o nfs4recover.o nfs4xdr_gen.o +nfsd-$(CONFIG_NFSD_PNFS) += nfs4layouts.o +nfsd-$(CONFIG_NFSD_BLOCKLAYOUT) += blocklayout.o blocklayoutxdr.o +nfsd-$(CONFIG_NFSD_SCSILAYOUT) += blocklayout.o blocklayoutxdr.o +nfsd-$(CONFIG_NFSD_FLEXFILELAYOUT) += flexfilelayout.o flexfilelayoutxdr.o +nfsd-$(CONFIG_NFS_LOCALIO) += localio.o +nfsd-$(CONFIG_DEBUG_FS) += debugfs.o + +# +# XDR code generation (requires Python and additional packages) +# +# The generated *xdr_gen.{h,c} files are checked into git. Normal kernel +# builds do not require the xdrgen tool or its Python dependencies. +# +# Developers modifying .x files in Documentation/sunrpc/xdr/ should run +# "make xdrgen" to regenerate the affected files. +# +.PHONY: xdrgen + +xdrgen: ../../include/linux/sunrpc/xdrgen/nfs4_1.h nfs4xdr_gen.h nfs4xdr_gen.c + +../../include/linux/sunrpc/xdrgen/nfs4_1.h: ../../Documentation/sunrpc/xdr/nfs4_1.x + ../../tools/net/sunrpc/xdrgen/xdrgen definitions $< > $@ + +nfs4xdr_gen.h: ../../Documentation/sunrpc/xdr/nfs4_1.x + ../../tools/net/sunrpc/xdrgen/xdrgen declarations $< > $@ + +nfs4xdr_gen.c: ../../Documentation/sunrpc/xdr/nfs4_1.x + ../../tools/net/sunrpc/xdrgen/xdrgen source $< > $@ diff --git a/fs/nfsd/acl.h b/fs/nfsd/acl.h new file mode 100644 index 000000000..2003523d0 --- /dev/null +++ b/fs/nfsd/acl.h @@ -0,0 +1,54 @@ +/* + * Common NFSv4 ACL handling definitions. + * + * Copyright (c) 2002 The Regents of the University of Michigan. + * All rights reserved. + * + * Marius Aamodt Eriksen + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef LINUX_NFS4_ACL_H +#define LINUX_NFS4_ACL_H + +struct nfs4_acl; +struct svc_fh; +struct svc_rqst; +struct nfsd_attrs; +enum nfs_ftype4; + +int nfs4_acl_bytes(int entries); +int nfs4_acl_get_whotype(char *, u32); +__be32 nfs4_acl_write_who(struct xdr_stream *xdr, int who); + +int nfsd4_get_nfs4_acl(struct svc_rqst *rqstp, struct dentry *dentry, + struct nfs4_acl **acl); +__be32 nfsd4_acl_to_attr(enum nfs_ftype4 type, struct nfs4_acl *acl, + struct nfsd_attrs *attr); +void sort_pacl_range(struct posix_acl *pacl, int start, int end); + +#endif /* LINUX_NFS4_ACL_H */ diff --git a/fs/nfsd/attr4.h b/fs/nfsd/attr4.h new file mode 100644 index 000000000..f0b51f805 --- /dev/null +++ b/fs/nfsd/attr4.h @@ -0,0 +1,162 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * NFSv4 file attributes supported by this implementation + */ + +#ifndef _LINUX_NFSD_ATTR4_H +#define _LINUX_NFSD_ATTR4_H + +#include +#include + +/* + * The following attributes are not implemented by NFSD: + * ARCHIVE (deprecated anyway) + * HIDDEN (unlikely to be supported any time soon) + * MIMETYPE (unlikely to be supported any time soon) + * QUOTA_* (unlikely to be supported any time soon) + * SYSTEM (unlikely to be supported any time soon) + * TIME_BACKUP (unlikely to be supported any time soon) + */ +#define NFSD4_SUPPORTED_ATTRS_WORD0 \ +(FATTR4_WORD0_SUPPORTED_ATTRS | FATTR4_WORD0_TYPE | FATTR4_WORD0_FH_EXPIRE_TYPE \ + | FATTR4_WORD0_CHANGE | FATTR4_WORD0_SIZE | FATTR4_WORD0_LINK_SUPPORT \ + | FATTR4_WORD0_SYMLINK_SUPPORT | FATTR4_WORD0_NAMED_ATTR | FATTR4_WORD0_FSID \ + | FATTR4_WORD0_UNIQUE_HANDLES | FATTR4_WORD0_LEASE_TIME | FATTR4_WORD0_RDATTR_ERROR \ + | FATTR4_WORD0_ACLSUPPORT | FATTR4_WORD0_CANSETTIME | FATTR4_WORD0_CASE_INSENSITIVE \ + | FATTR4_WORD0_CASE_PRESERVING | FATTR4_WORD0_CHOWN_RESTRICTED \ + | FATTR4_WORD0_FILEHANDLE | FATTR4_WORD0_FILEID | FATTR4_WORD0_FILES_AVAIL \ + | FATTR4_WORD0_FILES_FREE | FATTR4_WORD0_FILES_TOTAL | FATTR4_WORD0_FS_LOCATIONS | FATTR4_WORD0_HOMOGENEOUS \ + | FATTR4_WORD0_MAXFILESIZE | FATTR4_WORD0_MAXLINK | FATTR4_WORD0_MAXNAME \ + | FATTR4_WORD0_MAXREAD | FATTR4_WORD0_MAXWRITE | FATTR4_WORD0_ACL) + +#define NFSD4_SUPPORTED_ATTRS_WORD1 \ +(FATTR4_WORD1_MODE | FATTR4_WORD1_NO_TRUNC | FATTR4_WORD1_NUMLINKS \ + | FATTR4_WORD1_OWNER | FATTR4_WORD1_OWNER_GROUP | FATTR4_WORD1_RAWDEV \ + | FATTR4_WORD1_SPACE_AVAIL | FATTR4_WORD1_SPACE_FREE | FATTR4_WORD1_SPACE_TOTAL \ + | FATTR4_WORD1_SPACE_USED | FATTR4_WORD1_TIME_ACCESS | FATTR4_WORD1_TIME_ACCESS_SET \ + | FATTR4_WORD1_TIME_DELTA | FATTR4_WORD1_TIME_METADATA | FATTR4_WORD1_TIME_CREATE \ + | FATTR4_WORD1_TIME_MODIFY | FATTR4_WORD1_TIME_MODIFY_SET | FATTR4_WORD1_MOUNTED_ON_FILEID) + +#define NFSD4_SUPPORTED_ATTRS_WORD2 0 + +/* 4.1 */ +#ifdef CONFIG_NFSD_PNFS +#define PNFSD_SUPPORTED_ATTRS_WORD1 FATTR4_WORD1_FS_LAYOUT_TYPES +#define PNFSD_SUPPORTED_ATTRS_WORD2 \ +(FATTR4_WORD2_LAYOUT_BLKSIZE | FATTR4_WORD2_LAYOUT_TYPES) +#else +#define PNFSD_SUPPORTED_ATTRS_WORD1 0 +#define PNFSD_SUPPORTED_ATTRS_WORD2 0 +#endif /* CONFIG_NFSD_PNFS */ + +#define NFSD4_1_SUPPORTED_ATTRS_WORD0 \ + NFSD4_SUPPORTED_ATTRS_WORD0 + +#define NFSD4_1_SUPPORTED_ATTRS_WORD1 \ + (NFSD4_SUPPORTED_ATTRS_WORD1 | PNFSD_SUPPORTED_ATTRS_WORD1) + +#define NFSD4_1_SUPPORTED_ATTRS_WORD2 \ + (NFSD4_SUPPORTED_ATTRS_WORD2 | PNFSD_SUPPORTED_ATTRS_WORD2 | \ + FATTR4_WORD2_SUPPATTR_EXCLCREAT) + +/* 4.2 */ +#ifdef CONFIG_NFSD_V4_SECURITY_LABEL +#define NFSD4_2_SECURITY_ATTRS FATTR4_WORD2_SECURITY_LABEL +#else +#define NFSD4_2_SECURITY_ATTRS 0 +#endif + +#ifdef CONFIG_NFSD_V4_POSIX_ACLS +#define NFSD4_2_POSIX_ACL_ATTRS \ + (FATTR4_WORD2_ACL_TRUEFORM | \ + FATTR4_WORD2_ACL_TRUEFORM_SCOPE | \ + FATTR4_WORD2_POSIX_DEFAULT_ACL | \ + FATTR4_WORD2_POSIX_ACCESS_ACL) +#else +#define NFSD4_2_POSIX_ACL_ATTRS 0 +#endif + +#define NFSD4_2_SUPPORTED_ATTRS_WORD2 \ + (NFSD4_1_SUPPORTED_ATTRS_WORD2 | \ + FATTR4_WORD2_MODE_UMASK | \ + FATTR4_WORD2_CLONE_BLKSIZE | \ + NFSD4_2_SECURITY_ATTRS | \ + FATTR4_WORD2_XATTR_SUPPORT | \ + FATTR4_WORD2_TIME_DELEG_ACCESS | \ + FATTR4_WORD2_TIME_DELEG_MODIFY | \ + FATTR4_WORD2_OPEN_ARGUMENTS | \ + NFSD4_2_POSIX_ACL_ATTRS) + +/* These will return ERR_INVAL if specified in GETATTR or READDIR. */ +#define NFSD_WRITEONLY_ATTRS_WORD1 \ + (FATTR4_WORD1_TIME_ACCESS_SET | FATTR4_WORD1_TIME_MODIFY_SET) + +/* + * These are the only attrs allowed in CREATE/OPEN/SETATTR. Don't add + * a writeable attribute here without also adding code to parse it to + * nfsd4_decode_fattr4(). + */ +#define NFSD_WRITEABLE_ATTRS_WORD0 \ + (FATTR4_WORD0_SIZE | FATTR4_WORD0_ACL) +#define NFSD_WRITEABLE_ATTRS_WORD1 \ + (FATTR4_WORD1_MODE | FATTR4_WORD1_OWNER | FATTR4_WORD1_OWNER_GROUP \ + | FATTR4_WORD1_TIME_ACCESS_SET | FATTR4_WORD1_TIME_CREATE \ + | FATTR4_WORD1_TIME_MODIFY_SET) +#ifdef CONFIG_NFSD_V4_SECURITY_LABEL +#define MAYBE_FATTR4_WORD2_SECURITY_LABEL \ + FATTR4_WORD2_SECURITY_LABEL +#else +#define MAYBE_FATTR4_WORD2_SECURITY_LABEL 0 +#endif +#ifdef CONFIG_NFSD_V4_POSIX_ACLS +#define MAYBE_FATTR4_WORD2_POSIX_ACL_ATTRS \ + FATTR4_WORD2_POSIX_DEFAULT_ACL | FATTR4_WORD2_POSIX_ACCESS_ACL +#else +#define MAYBE_FATTR4_WORD2_POSIX_ACL_ATTRS 0 +#endif +#define NFSD_WRITEABLE_ATTRS_WORD2 \ + (FATTR4_WORD2_MODE_UMASK \ + | MAYBE_FATTR4_WORD2_SECURITY_LABEL \ + | FATTR4_WORD2_TIME_DELEG_ACCESS \ + | FATTR4_WORD2_TIME_DELEG_MODIFY \ + | MAYBE_FATTR4_WORD2_POSIX_ACL_ATTRS \ + ) + +#define NFSD_SUPPATTR_EXCLCREAT_WORD0 \ + NFSD_WRITEABLE_ATTRS_WORD0 +/* + * we currently store the exclusive create verifier in the v_{a,m}time + * attributes so the client can't set these at create time using EXCLUSIVE4_1 + */ +#define NFSD_SUPPATTR_EXCLCREAT_WORD1 \ + (NFSD_WRITEABLE_ATTRS_WORD1 & \ + ~(FATTR4_WORD1_TIME_ACCESS_SET | FATTR4_WORD1_TIME_MODIFY_SET)) +/* + * The FATTR4_WORD2_TIME_DELEG attributes are not to be allowed for + * OPEN(create) with EXCLUSIVE4_1. It doesn't make sense to set a + * delegated timestamp on a new file. + * + * This mask includes NFSv4.2-only attributes (e.g., POSIX ACLs). + * Version filtering occurs via nfsd_suppattrs[] before this mask + * is applied, so pre-4.2 clients never see unsupported attributes. + */ +#define NFSD_SUPPATTR_EXCLCREAT_WORD2 \ + (NFSD_WRITEABLE_ATTRS_WORD2 & \ + ~(FATTR4_WORD2_TIME_DELEG_ACCESS | FATTR4_WORD2_TIME_DELEG_MODIFY)) + +extern const u32 nfsd_suppattrs[3][3]; + +static inline bool bmval_is_subset(const u32 *bm1, const u32 *bm2) +{ + return !((bm1[0] & ~bm2[0]) || + (bm1[1] & ~bm2[1]) || + (bm1[2] & ~bm2[2])); +} + +static inline bool nfsd_attrs_supported(u32 minorversion, const u32 *bmval) +{ + return bmval_is_subset(bmval, nfsd_suppattrs[minorversion]); +} + +#endif /* _LINUX_NFSD_ATTR4_H */ diff --git a/fs/nfsd/auth.c b/fs/nfsd/auth.c new file mode 100644 index 000000000..5ef9a0466 --- /dev/null +++ b/fs/nfsd/auth.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 1995, 1996 Olaf Kirch */ + +#include +#include "nfsd.h" +#include "export.h" +#include "auth.h" + +int nfsexp_flags(struct svc_cred *cred, struct svc_export *exp) +{ + struct exp_flavor_info *f; + struct exp_flavor_info *end = exp->ex_flavors + exp->ex_nflavors; + + for (f = exp->ex_flavors; f < end; f++) { + if (f->pseudoflavor == cred->cr_flavor) + return f->flags; + } + return exp->ex_flags; + +} + +int nfsd_setuser(struct svc_cred *cred, struct svc_export *exp) +{ + struct group_info *rqgi; + struct group_info *gi; + struct cred *new; + int i; + int flags = nfsexp_flags(cred, exp); + + /* discard any old override before preparing the new set */ + put_cred(revert_creds(get_cred(current_real_cred()))); + new = prepare_creds(); + if (!new) + return -ENOMEM; + + new->fsuid = cred->cr_uid; + new->fsgid = cred->cr_gid; + + rqgi = cred->cr_group_info; + + if (flags & NFSEXP_ALLSQUASH) { + new->fsuid = exp->ex_anon_uid; + new->fsgid = exp->ex_anon_gid; + gi = groups_alloc(0); + if (!gi) + goto oom; + } else if (flags & NFSEXP_ROOTSQUASH) { + if (uid_eq(new->fsuid, GLOBAL_ROOT_UID)) + new->fsuid = exp->ex_anon_uid; + if (gid_eq(new->fsgid, GLOBAL_ROOT_GID)) + new->fsgid = exp->ex_anon_gid; + + gi = groups_alloc(rqgi->ngroups); + if (!gi) + goto oom; + + for (i = 0; i < rqgi->ngroups; i++) { + if (gid_eq(GLOBAL_ROOT_GID, rqgi->gid[i])) + gi->gid[i] = exp->ex_anon_gid; + else + gi->gid[i] = rqgi->gid[i]; + } + + /* Each thread allocates its own gi, no race */ + groups_sort(gi); + } else { + gi = get_group_info(rqgi); + } + + if (uid_eq(new->fsuid, INVALID_UID)) + new->fsuid = exp->ex_anon_uid; + if (gid_eq(new->fsgid, INVALID_GID)) + new->fsgid = exp->ex_anon_gid; + + set_groups(new, gi); + put_group_info(gi); + + if (!uid_eq(new->fsuid, GLOBAL_ROOT_UID)) + new->cap_effective = cap_drop_nfsd_set(new->cap_effective); + else + new->cap_effective = cap_raise_nfsd_set(new->cap_effective, + new->cap_permitted); + put_cred(override_creds(new)); + return 0; + +oom: + abort_creds(new); + return -ENOMEM; +} + +/** + * nfsd_user_namespace - Get user_namespace in effect for an RPC request + * @rqstp: RPC execution context + * + * xpt_cred is set once at transport creation and never modified. The + * transport itself is reference-counted during request processing, so + * no explicit reference on the namespace is necessary. + * + * Return: the user_namespace from the transport credential, or + * init_user_ns if no credential was set. The returned namespace pointer + * is valid for the duration of the RPC request. + */ +struct user_namespace *nfsd_user_namespace(const struct svc_rqst *rqstp) +{ + const struct cred *cred = rqstp->rq_xprt->xpt_cred; + + return cred ? cred->user_ns : &init_user_ns; +} diff --git a/fs/nfsd/auth.h b/fs/nfsd/auth.h new file mode 100644 index 000000000..832bc9579 --- /dev/null +++ b/fs/nfsd/auth.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * nfsd-specific authentication stuff. + * + * Copyright (C) 1995, 1996 Olaf Kirch + */ + +#ifndef LINUX_NFSD_AUTH_H +#define LINUX_NFSD_AUTH_H + +struct user_namespace; +struct svc_export; +struct svc_rqst; + +/* + * Set the current process's fsuid/fsgid etc to those of the NFS + * client user + */ +int nfsd_setuser(struct svc_cred *cred, struct svc_export *exp); + +struct user_namespace *nfsd_user_namespace(const struct svc_rqst *rqstp); + +#endif /* LINUX_NFSD_AUTH_H */ diff --git a/fs/nfsd/blocklayout.c b/fs/nfsd/blocklayout.c new file mode 100644 index 000000000..5be7721c2 --- /dev/null +++ b/fs/nfsd/blocklayout.c @@ -0,0 +1,526 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2014-2016 Christoph Hellwig. + */ +#include +#include +#include +#include + +#include + +#include "blocklayoutxdr.h" +#include "pnfs.h" +#include "filecache.h" +#include "vfs.h" +#include "trace.h" + +#define NFSDDBG_FACILITY NFSDDBG_PNFS + + +/* + * Get an extent from the file system that starts at offset or below + * and may be shorter than the requested length. + */ +static __be32 +nfsd4_block_map_extent(struct inode *inode, const struct svc_fh *fhp, + u64 offset, u64 length, u32 iomode, u64 minlength, + struct pnfs_block_extent *bex) +{ + struct super_block *sb = inode->i_sb; + struct iomap iomap; + u32 device_generation = 0; + int error; + + error = sb->s_export_op->block_ops->map_blocks(inode, offset, length, + &iomap, iomode != IOMODE_READ, &device_generation); + if (error) { + if (error == -ENXIO) + return nfserr_layoutunavailable; + return nfserrno(error); + } + + switch (iomap.type) { + case IOMAP_MAPPED: + if (iomode == IOMODE_READ) + bex->es = PNFS_BLOCK_READ_DATA; + else + bex->es = PNFS_BLOCK_READWRITE_DATA; + bex->soff = iomap.addr; + break; + case IOMAP_UNWRITTEN: + if (iomode & IOMODE_RW) { + /* + * Crack monkey special case from section 2.3.1. + */ + if (minlength == 0) { + dprintk("pnfsd: no soup for you!\n"); + return nfserr_layoutunavailable; + } + + bex->es = PNFS_BLOCK_INVALID_DATA; + bex->soff = iomap.addr; + break; + } + fallthrough; + case IOMAP_HOLE: + if (iomode == IOMODE_READ) { + bex->es = PNFS_BLOCK_NONE_DATA; + break; + } + fallthrough; + case IOMAP_DELALLOC: + default: + WARN(1, "pnfsd: filesystem returned %d extent\n", iomap.type); + return nfserr_layoutunavailable; + } + + error = nfsd4_set_deviceid(&bex->vol_id, fhp, device_generation); + if (error) + return nfserrno(error); + + bex->foff = iomap.offset; + bex->len = iomap.length; + return nfs_ok; +} + +static __be32 +nfsd4_block_proc_layoutget(struct svc_rqst *rqstp, struct inode *inode, + const struct svc_fh *fhp, struct nfsd4_layoutget *args) +{ + struct nfsd4_layout_seg *seg = &args->lg_seg; + struct pnfs_block_layout *bl; + struct pnfs_block_extent *first_bex, *last_bex; + u64 offset = seg->offset, length = seg->length; + u32 i, nr_extents_max, block_size = i_blocksize(inode); + __be32 nfserr; + + if (locks_in_grace(SVC_NET(rqstp))) + return nfserr_grace; + + nfserr = nfserr_layoutunavailable; + if (seg->offset & (block_size - 1)) { + dprintk("pnfsd: I/O misaligned\n"); + goto out_error; + } + + /* + * RFC 8881, section 3.3.17: + * The layout4 data type defines a layout for a file. + * + * RFC 8881, section 18.43.3: + * The loga_maxcount field specifies the maximum layout size + * (in bytes) that the client can handle. If the size of the + * layout structure exceeds the size specified by maxcount, + * the metadata server will return the NFS4ERR_TOOSMALL error. + */ + nfserr = nfserr_toosmall; + if (args->lg_maxcount < PNFS_BLOCK_LAYOUT4_SIZE + + PNFS_BLOCK_EXTENT_SIZE) + goto out_error; + + /* + * Limit the maximum layout size to avoid allocating + * a large buffer on the server for each layout request. + */ + nr_extents_max = (min(args->lg_maxcount, PAGE_SIZE) - + PNFS_BLOCK_LAYOUT4_SIZE) / PNFS_BLOCK_EXTENT_SIZE; + + /* + * Some clients barf on non-zero block numbers for NONE or INVALID + * layouts, so make sure to zero the whole structure. + */ + nfserr = nfserrno(-ENOMEM); + bl = kzalloc_flex(*bl, extents, nr_extents_max); + if (!bl) + goto out_error; + bl->nr_extents = nr_extents_max; + args->lg_content = bl; + + for (i = 0; i < bl->nr_extents; i++) { + struct pnfs_block_extent *bex = bl->extents + i; + u64 bex_length; + + nfserr = nfsd4_block_map_extent(inode, fhp, offset, length, + seg->iomode, args->lg_minlength, bex); + if (nfserr != nfs_ok) + goto out_error; + + bex_length = bex->len - (offset - bex->foff); + if (bex_length >= length) { + bl->nr_extents = i + 1; + break; + } + + offset = bex->foff + bex->len; + length -= bex_length; + } + + first_bex = bl->extents; + last_bex = bl->extents + bl->nr_extents - 1; + + nfserr = nfserr_layoutunavailable; + length = last_bex->foff + last_bex->len - seg->offset; + if (length < args->lg_minlength) { + dprintk("pnfsd: extent smaller than minlength\n"); + goto out_error; + } + + seg->offset = first_bex->foff; + seg->length = last_bex->foff - first_bex->foff + last_bex->len; + return nfs_ok; + +out_error: + seg->length = 0; + return nfserr; +} + +static __be32 +nfsd4_block_commit_blocks(struct inode *inode, struct nfsd4_layoutcommit *lcp, + struct iomap *iomaps, int nr_iomaps) +{ + int error; + + /* + * This ignores the client provided mtime in loca_time_modify, as a + * fully client specified mtime doesn't really fit into the Linux + * multi-grain timestamp architecture. + * + * RFC 8881 Section 18.42 makes it clear that the client provided + * timestamp is a "may" condition, and clients that want to force a + * specific timestamp should send a separate SETATTR in the compound. + */ + error = inode->i_sb->s_export_op->block_ops->commit_blocks(inode, + iomaps, nr_iomaps, + lcp->lc_size_chg ? lcp->lc_newsize : 0); + kfree(iomaps); + return nfserrno(error); +} + +#ifdef CONFIG_NFSD_BLOCKLAYOUT +static int +nfsd4_block_get_device_info_simple(struct super_block *sb, + struct nfsd4_getdeviceinfo *gdp) +{ + struct pnfs_block_deviceaddr *dev; + struct pnfs_block_volume *b; + + dev = kzalloc_flex(*dev, volumes, 1); + if (!dev) + return -ENOMEM; + gdp->gd_device = dev; + + dev->nr_volumes = 1; + b = &dev->volumes[0]; + + b->type = PNFS_BLOCK_VOLUME_SIMPLE; + b->simple.sig_len = PNFS_BLOCK_UUID_LEN; + return sb->s_export_op->block_ops->get_uuid(sb, b->simple.sig, + &b->simple.sig_len, &b->simple.offset); +} + +static __be32 +nfsd4_block_proc_getdeviceinfo(struct super_block *sb, + struct svc_rqst *rqstp, + struct nfs4_client *clp, + struct nfsd4_getdeviceinfo *gdp) +{ + if (bdev_is_partition(sb->s_bdev)) + return nfserr_inval; + return nfserrno(nfsd4_block_get_device_info_simple(sb, gdp)); +} + +static __be32 +nfsd4_block_proc_layoutcommit(struct inode *inode, struct svc_rqst *rqstp, + struct nfsd4_layoutcommit *lcp) +{ + struct iomap *iomaps; + int nr_iomaps; + __be32 nfserr; + + rqstp->rq_arg = lcp->lc_up_layout; + svcxdr_init_decode(rqstp); + + nfserr = nfsd4_block_decode_layoutupdate(&rqstp->rq_arg_stream, + &iomaps, &nr_iomaps, i_blocksize(inode)); + if (nfserr != nfs_ok) + return nfserr; + + return nfsd4_block_commit_blocks(inode, lcp, iomaps, nr_iomaps); +} + +const struct nfsd4_layout_ops bl_layout_ops = { + /* + * Pretend that we send notification to the client. This is a blatant + * lie to force recent Linux clients to cache our device IDs. + * We rarely ever change the device ID, so the harm of leaking deviceids + * for a while isn't too bad. Unfortunately RFC5661 is a complete mess + * in this regard, but I filed errata 4119 for this a while ago, and + * hopefully the Linux client will eventually start caching deviceids + * without this again. + */ + .notify_types = + NOTIFY_DEVICEID4_DELETE | NOTIFY_DEVICEID4_CHANGE, + .proc_getdeviceinfo = nfsd4_block_proc_getdeviceinfo, + .encode_getdeviceinfo = nfsd4_block_encode_getdeviceinfo, + .proc_layoutget = nfsd4_block_proc_layoutget, + .encode_layoutget = nfsd4_block_encode_layoutget, + .proc_layoutcommit = nfsd4_block_proc_layoutcommit, +}; +#endif /* CONFIG_NFSD_BLOCKLAYOUT */ + +#ifdef CONFIG_NFSD_SCSILAYOUT + +#define NFSD_MDS_PR_FENCED XA_MARK_0 + +/* + * Clear the fence flag if the device already has an entry. This occurs + * when a client re-registers after a previous fence, allowing new + * layouts for this device. + * + * Insert only on first registration. This bounds cl_dev_fences to the + * count of devices this client has accessed, preventing unbounded growth. + */ +static inline int nfsd4_scsi_fence_insert(struct nfs4_client *clp, + dev_t device) +{ + struct xarray *xa = &clp->cl_dev_fences; + int ret; + + xa_lock(xa); + ret = __xa_insert(xa, device, XA_ZERO_ENTRY, GFP_KERNEL); + if (ret == -EBUSY) { + __xa_clear_mark(xa, device, NFSD_MDS_PR_FENCED); + ret = 0; + } + xa_unlock(xa); + clp->cl_fence_retry_warn = false; + return ret; +} + +static inline bool nfsd4_scsi_fence_set(struct nfs4_client *clp, dev_t device) +{ + struct xarray *xa = &clp->cl_dev_fences; + bool skip; + + xa_lock(xa); + skip = xa_get_mark(xa, device, NFSD_MDS_PR_FENCED); + if (!skip) + __xa_set_mark(xa, device, NFSD_MDS_PR_FENCED); + xa_unlock(xa); + return skip; +} + +static inline void nfsd4_scsi_fence_clear(struct nfs4_client *clp, dev_t device) +{ + xa_clear_mark(&clp->cl_dev_fences, device, NFSD_MDS_PR_FENCED); +} + +#define NFSD_MDS_PR_KEY 0x0100000000000000ULL + +/* + * We use the client ID as a unique key for the reservations. + * This allows us to easily fence a client when recalls fail. + */ +static u64 nfsd4_scsi_pr_key(struct nfs4_client *clp) +{ + return ((u64)clp->cl_clientid.cl_boot << 32) | clp->cl_clientid.cl_id; +} + +static const u8 designator_types[] = { + PS_DESIGNATOR_EUI64, + PS_DESIGNATOR_NAA, +}; + +static int +nfsd4_block_get_unique_id(struct gendisk *disk, struct pnfs_block_volume *b) +{ + int ret, i; + + for (i = 0; i < ARRAY_SIZE(designator_types); i++) { + u8 type = designator_types[i]; + + ret = disk->fops->get_unique_id(disk, b->scsi.designator, type); + if (ret > 0) { + b->scsi.code_set = PS_CODE_SET_BINARY; + b->scsi.designator_type = type; + b->scsi.designator_len = ret; + return 0; + } + } + + return -EINVAL; +} + +static int +nfsd4_block_get_device_info_scsi(struct super_block *sb, + struct nfs4_client *clp, + struct nfsd4_getdeviceinfo *gdp) +{ + struct pnfs_block_deviceaddr *dev; + struct pnfs_block_volume *b; + const struct pr_ops *ops; + int ret; + + dev = kzalloc_flex(*dev, volumes, 1); + if (!dev) + return -ENOMEM; + gdp->gd_device = dev; + + dev->nr_volumes = 1; + b = &dev->volumes[0]; + + b->type = PNFS_BLOCK_VOLUME_SCSI; + b->scsi.pr_key = nfsd4_scsi_pr_key(clp); + + ret = nfsd4_block_get_unique_id(sb->s_bdev->bd_disk, b); + if (ret < 0) + goto out_free_dev; + + ret = -EINVAL; + ops = sb->s_bdev->bd_disk->fops->pr_ops; + if (!ops) { + pr_err("pNFS: device %s does not support PRs.\n", + sb->s_id); + goto out_free_dev; + } + + ret = nfsd4_scsi_fence_insert(clp, sb->s_bdev->bd_dev); + if (ret < 0) + goto out_free_dev; + + ret = ops->pr_register(sb->s_bdev, 0, NFSD_MDS_PR_KEY, true); + if (ret) { + pr_err("pNFS: failed to register key for device %s.\n", + sb->s_id); + goto out_free_dev; + } + + ret = ops->pr_reserve(sb->s_bdev, NFSD_MDS_PR_KEY, + PR_EXCLUSIVE_ACCESS_REG_ONLY, 0); + if (ret) { + pr_err("pNFS: failed to reserve device %s.\n", + sb->s_id); + goto out_free_dev; + } + + return 0; + +out_free_dev: + kfree(dev); + gdp->gd_device = NULL; + return ret; +} + +static __be32 +nfsd4_scsi_proc_getdeviceinfo(struct super_block *sb, + struct svc_rqst *rqstp, + struct nfs4_client *clp, + struct nfsd4_getdeviceinfo *gdp) +{ + if (bdev_is_partition(sb->s_bdev)) + return nfserr_inval; + return nfserrno(nfsd4_block_get_device_info_scsi(sb, clp, gdp)); +} +static __be32 +nfsd4_scsi_proc_layoutcommit(struct inode *inode, struct svc_rqst *rqstp, + struct nfsd4_layoutcommit *lcp) +{ + struct iomap *iomaps; + int nr_iomaps; + __be32 nfserr; + + rqstp->rq_arg = lcp->lc_up_layout; + svcxdr_init_decode(rqstp); + + nfserr = nfsd4_scsi_decode_layoutupdate(&rqstp->rq_arg_stream, + &iomaps, &nr_iomaps, i_blocksize(inode)); + if (nfserr != nfs_ok) + return nfserr; + + return nfsd4_block_commit_blocks(inode, lcp, iomaps, nr_iomaps); +} + +/* + * Perform the fence operation to prevent the client from accessing the + * block device. If a fence operation is already in progress, wait for + * it to complete before checking the NFSD_MDS_PR_FENCED flag. Once the + * operation is complete, check the flag. If NFSD_MDS_PR_FENCED is set, + * update the layout stateid by setting the ls_fenced flag to indicate + * that the client has been fenced. + * + * The cl_fence_mutex ensures that the fence operation has been fully + * completed, rather than just in progress, when returning from this + * function. + * + * Return true if client was fenced otherwise return false. + */ +static bool +nfsd4_scsi_fence_client(struct nfs4_layout_stateid *ls, struct nfsd_file *file) +{ + struct nfs4_client *clp = ls->ls_stid.sc_client; + struct block_device *bdev = file->nf_file->f_path.mnt->mnt_sb->s_bdev; + int status; + bool ret; + + mutex_lock(&clp->cl_fence_mutex); + if (nfsd4_scsi_fence_set(clp, bdev->bd_dev)) { + mutex_unlock(&clp->cl_fence_mutex); + return true; + } + + status = bdev->bd_disk->fops->pr_ops->pr_preempt(bdev, NFSD_MDS_PR_KEY, + nfsd4_scsi_pr_key(clp), + PR_EXCLUSIVE_ACCESS_REG_ONLY, true); + /* + * Reset to allow retry only when the command could not have + * reached the device. Negative status means a local error + * (e.g., -ENOMEM) prevented the command from being sent. + * PR_STS_PATH_FAILED, PR_STS_PATH_FAST_FAILED, and + * PR_STS_RETRY_PATH_FAILURE indicate transport path failures + * before device delivery. + * + * For all other errors, the command may have reached the device + * and the preempt may have succeeded. Avoid resetting, since + * retrying a successful preempt returns PR_STS_IOERR or + * PR_STS_RESERVATION_CONFLICT, which would cause an infinite + * retry loop. + */ + switch (status) { + case 0: + case PR_STS_IOERR: + case PR_STS_RESERVATION_CONFLICT: + ret = true; + break; + default: + /* retry-able and other errors */ + ret = false; + nfsd4_scsi_fence_clear(clp, bdev->bd_dev); + break; + } + mutex_unlock(&clp->cl_fence_mutex); + + trace_nfsd_pnfs_fence(clp, bdev->bd_disk->disk_name, status); + return ret; +} + +const struct nfsd4_layout_ops scsi_layout_ops = { + /* + * Pretend that we send notification to the client. This is a blatant + * lie to force recent Linux clients to cache our device IDs. + * We rarely ever change the device ID, so the harm of leaking deviceids + * for a while isn't too bad. Unfortunately RFC5661 is a complete mess + * in this regard, but I filed errata 4119 for this a while ago, and + * hopefully the Linux client will eventually start caching deviceids + * without this again. + */ + .notify_types = + NOTIFY_DEVICEID4_DELETE | NOTIFY_DEVICEID4_CHANGE, + .proc_getdeviceinfo = nfsd4_scsi_proc_getdeviceinfo, + .encode_getdeviceinfo = nfsd4_block_encode_getdeviceinfo, + .proc_layoutget = nfsd4_block_proc_layoutget, + .encode_layoutget = nfsd4_block_encode_layoutget, + .proc_layoutcommit = nfsd4_scsi_proc_layoutcommit, + .fence_client = nfsd4_scsi_fence_client, +}; +#endif /* CONFIG_NFSD_SCSILAYOUT */ diff --git a/fs/nfsd/blocklayoutxdr.c b/fs/nfsd/blocklayoutxdr.c new file mode 100644 index 000000000..f80dbc41f --- /dev/null +++ b/fs/nfsd/blocklayoutxdr.c @@ -0,0 +1,295 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2014-2016 Christoph Hellwig. + */ +#include +#include +#include +#include + +#include "nfsd.h" +#include "blocklayoutxdr.h" +#include "vfs.h" + +#define NFSDDBG_FACILITY NFSDDBG_PNFS + + +/** + * nfsd4_block_encode_layoutget - encode block/scsi layout extent array + * @xdr: stream for data encoding + * @lgp: layoutget content, actually an array of extents to encode + * + * Encode the opaque loc_body field in the layoutget response. Since the + * pnfs_block_layout4 and pnfs_scsi_layout4 structures on the wire are + * the same, this function is used by both layout drivers. + * + * Return values: + * %nfs_ok: Success, all extents encoded into @xdr + * %nfserr_toosmall: Not enough space in @xdr to encode all the data + */ +__be32 +nfsd4_block_encode_layoutget(struct xdr_stream *xdr, + const struct nfsd4_layoutget *lgp) +{ + const struct pnfs_block_layout *bl = lgp->lg_content; + u32 i, len = sizeof(__be32) + bl->nr_extents * PNFS_BLOCK_EXTENT_SIZE; + __be32 *p; + + p = xdr_reserve_space(xdr, sizeof(__be32) + len); + if (!p) + return nfserr_toosmall; + + *p++ = cpu_to_be32(len); + *p++ = cpu_to_be32(bl->nr_extents); + + for (i = 0; i < bl->nr_extents; i++) { + const struct pnfs_block_extent *bex = bl->extents + i; + + p = svcxdr_encode_deviceid4(p, &bex->vol_id); + p = xdr_encode_hyper(p, bex->foff); + p = xdr_encode_hyper(p, bex->len); + p = xdr_encode_hyper(p, bex->soff); + *p++ = cpu_to_be32(bex->es); + } + + return nfs_ok; +} + +static int +nfsd4_block_encode_volume(struct xdr_stream *xdr, struct pnfs_block_volume *b) +{ + __be32 *p; + int len; + + switch (b->type) { + case PNFS_BLOCK_VOLUME_SIMPLE: + len = 4 + 4 + 8 + 4 + (XDR_QUADLEN(b->simple.sig_len) << 2); + p = xdr_reserve_space(xdr, len); + if (!p) + return -ETOOSMALL; + + *p++ = cpu_to_be32(b->type); + *p++ = cpu_to_be32(1); /* single signature */ + p = xdr_encode_hyper(p, b->simple.offset); + p = xdr_encode_opaque(p, b->simple.sig, b->simple.sig_len); + break; + case PNFS_BLOCK_VOLUME_SCSI: + len = 4 + 4 + 4 + 4 + (XDR_QUADLEN(b->scsi.designator_len) << 2) + 8; + p = xdr_reserve_space(xdr, len); + if (!p) + return -ETOOSMALL; + + *p++ = cpu_to_be32(b->type); + *p++ = cpu_to_be32(b->scsi.code_set); + *p++ = cpu_to_be32(b->scsi.designator_type); + p = xdr_encode_opaque(p, b->scsi.designator, b->scsi.designator_len); + p = xdr_encode_hyper(p, b->scsi.pr_key); + break; + default: + return -ENOTSUPP; + } + + return len; +} + +__be32 +nfsd4_block_encode_getdeviceinfo(struct xdr_stream *xdr, + const struct nfsd4_getdeviceinfo *gdp) +{ + struct pnfs_block_deviceaddr *dev = gdp->gd_device; + int len = sizeof(__be32), ret, i; + __be32 *p; + + /* + * See paragraph 5 of RFC 8881 S18.40.3. + */ + if (!gdp->gd_maxcount) { + if (xdr_stream_encode_u32(xdr, 0) != XDR_UNIT) + return nfserr_resource; + return nfs_ok; + } + + p = xdr_reserve_space(xdr, len + sizeof(__be32)); + if (!p) + return nfserr_resource; + + for (i = 0; i < dev->nr_volumes; i++) { + ret = nfsd4_block_encode_volume(xdr, &dev->volumes[i]); + if (ret < 0) + return nfserrno(ret); + len += ret; + } + + /* + * Fill in the overall length and number of volumes at the beginning + * of the layout. + */ + *p++ = cpu_to_be32(len); + *p++ = cpu_to_be32(dev->nr_volumes); + return 0; +} + +/** + * nfsd4_block_decode_layoutupdate - decode the block layout extent array + * @xdr: subbuf set to the encoded array + * @iomapp: pointer to store the decoded extent array + * @nr_iomapsp: pointer to store the number of extents + * @block_size: alignment of extent offset and length + * + * This function decodes the opaque field of the layoutupdate4 structure + * in a layoutcommit request for the block layout driver. The field is + * actually an array of extents sent by the client. It also checks that + * the file offset, storage offset and length of each extent are aligned + * by @block_size. + * + * Return values: + * %nfs_ok: Successful decoding, @iomapp and @nr_iomapsp are valid + * %nfserr_bad_xdr: The encoded array in @xdr is invalid + * %nfserr_inval: An unaligned extent found + * %nfserr_delay: Failed to allocate memory for @iomapp + */ +__be32 +nfsd4_block_decode_layoutupdate(struct xdr_stream *xdr, struct iomap **iomapp, + int *nr_iomapsp, u32 block_size) +{ + struct iomap *iomaps; + u32 nr_iomaps, expected, len, i; + __be32 nfserr; + + if (xdr_stream_decode_u32(xdr, &nr_iomaps)) + return nfserr_bad_xdr; + + len = sizeof(__be32) + xdr_stream_remaining(xdr); + expected = sizeof(__be32) + nr_iomaps * PNFS_BLOCK_EXTENT_SIZE; + if (len != expected) + return nfserr_bad_xdr; + + iomaps = kzalloc_objs(*iomaps, nr_iomaps); + if (!iomaps) + return nfserr_delay; + + for (i = 0; i < nr_iomaps; i++) { + struct pnfs_block_extent bex; + + if (nfsd4_decode_deviceid4(xdr, &bex.vol_id)) { + nfserr = nfserr_bad_xdr; + goto fail; + } + + if (xdr_stream_decode_u64(xdr, &bex.foff)) { + nfserr = nfserr_bad_xdr; + goto fail; + } + if (bex.foff & (block_size - 1)) { + nfserr = nfserr_inval; + goto fail; + } + + if (xdr_stream_decode_u64(xdr, &bex.len)) { + nfserr = nfserr_bad_xdr; + goto fail; + } + if (bex.len & (block_size - 1)) { + nfserr = nfserr_inval; + goto fail; + } + + if (xdr_stream_decode_u64(xdr, &bex.soff)) { + nfserr = nfserr_bad_xdr; + goto fail; + } + if (bex.soff & (block_size - 1)) { + nfserr = nfserr_inval; + goto fail; + } + + if (xdr_stream_decode_u32(xdr, &bex.es)) { + nfserr = nfserr_bad_xdr; + goto fail; + } + if (bex.es != PNFS_BLOCK_READWRITE_DATA) { + nfserr = nfserr_inval; + goto fail; + } + + iomaps[i].offset = bex.foff; + iomaps[i].length = bex.len; + } + + *iomapp = iomaps; + *nr_iomapsp = nr_iomaps; + return nfs_ok; +fail: + kfree(iomaps); + return nfserr; +} + +/** + * nfsd4_scsi_decode_layoutupdate - decode the scsi layout extent array + * @xdr: subbuf set to the encoded array + * @iomapp: pointer to store the decoded extent array + * @nr_iomapsp: pointer to store the number of extents + * @block_size: alignment of extent offset and length + * + * This function decodes the opaque field of the layoutupdate4 structure + * in a layoutcommit request for the scsi layout driver. The field is + * actually an array of extents sent by the client. It also checks that + * the offset and length of each extent are aligned by @block_size. + * + * Return values: + * %nfs_ok: Successful decoding, @iomapp and @nr_iomapsp are valid + * %nfserr_bad_xdr: The encoded array in @xdr is invalid + * %nfserr_inval: An unaligned extent found + * %nfserr_delay: Failed to allocate memory for @iomapp + */ +__be32 +nfsd4_scsi_decode_layoutupdate(struct xdr_stream *xdr, struct iomap **iomapp, + int *nr_iomapsp, u32 block_size) +{ + struct iomap *iomaps; + u32 nr_iomaps, expected, len, i; + __be32 nfserr; + + if (xdr_stream_decode_u32(xdr, &nr_iomaps)) + return nfserr_bad_xdr; + + len = sizeof(__be32) + xdr_stream_remaining(xdr); + expected = sizeof(__be32) + nr_iomaps * PNFS_SCSI_RANGE_SIZE; + if (len != expected) + return nfserr_bad_xdr; + + iomaps = kzalloc_objs(*iomaps, nr_iomaps); + if (!iomaps) + return nfserr_delay; + + for (i = 0; i < nr_iomaps; i++) { + u64 val; + + if (xdr_stream_decode_u64(xdr, &val)) { + nfserr = nfserr_bad_xdr; + goto fail; + } + if (val & (block_size - 1)) { + nfserr = nfserr_inval; + goto fail; + } + iomaps[i].offset = val; + + if (xdr_stream_decode_u64(xdr, &val)) { + nfserr = nfserr_bad_xdr; + goto fail; + } + if (val & (block_size - 1)) { + nfserr = nfserr_inval; + goto fail; + } + iomaps[i].length = val; + } + + *iomapp = iomaps; + *nr_iomapsp = nr_iomaps; + return nfs_ok; +fail: + kfree(iomaps); + return nfserr; +} diff --git a/fs/nfsd/blocklayoutxdr.h b/fs/nfsd/blocklayoutxdr.h new file mode 100644 index 000000000..2e0c6c7d2 --- /dev/null +++ b/fs/nfsd/blocklayoutxdr.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _NFSD_BLOCKLAYOUTXDR_H +#define _NFSD_BLOCKLAYOUTXDR_H 1 + +#include +#include "xdr4.h" + +struct iomap; +struct xdr_stream; + +/* On the wire size of the layout4 struct with zero number of extents */ +#define PNFS_BLOCK_LAYOUT4_SIZE \ + (sizeof(__be32) * 2 + /* offset4 */ \ + sizeof(__be32) * 2 + /* length4 */ \ + sizeof(__be32) + /* layoutiomode4 */ \ + sizeof(__be32) + /* layouttype4 */ \ + sizeof(__be32) + /* number of bytes */ \ + sizeof(__be32)) /* number of extents */ + +struct pnfs_block_extent { + struct nfsd4_deviceid vol_id; + u64 foff; + u64 len; + u64 soff; + enum pnfs_block_extent_state es; +}; + +struct pnfs_block_range { + u64 foff; + u64 len; +}; + +struct pnfs_block_layout { + u32 nr_extents; + struct pnfs_block_extent extents[] __counted_by(nr_extents); +}; + +/* + * Random upper cap for the uuid length to avoid unbounded allocation. + * Not actually limited by the protocol. + */ +#define PNFS_BLOCK_UUID_LEN 128 + +struct pnfs_block_volume { + enum pnfs_block_volume_type type; + union { + struct { + u64 offset; + u32 sig_len; + u8 sig[PNFS_BLOCK_UUID_LEN]; + } simple; + struct { + enum scsi_code_set code_set; + enum scsi_designator_type designator_type; + int designator_len; + u8 designator[256]; + u64 pr_key; + } scsi; + }; +}; + +struct pnfs_block_deviceaddr { + u32 nr_volumes; + struct pnfs_block_volume volumes[] __counted_by(nr_volumes); +}; + +__be32 nfsd4_block_encode_getdeviceinfo(struct xdr_stream *xdr, + const struct nfsd4_getdeviceinfo *gdp); +__be32 nfsd4_block_encode_layoutget(struct xdr_stream *xdr, + const struct nfsd4_layoutget *lgp); +__be32 nfsd4_block_decode_layoutupdate(struct xdr_stream *xdr, + struct iomap **iomapp, int *nr_iomapsp, u32 block_size); +__be32 nfsd4_scsi_decode_layoutupdate(struct xdr_stream *xdr, + struct iomap **iomapp, int *nr_iomapsp, u32 block_size); + +#endif /* _NFSD_BLOCKLAYOUTXDR_H */ diff --git a/fs/nfsd/cache.h b/fs/nfsd/cache.h new file mode 100644 index 000000000..3bc4856e3 --- /dev/null +++ b/fs/nfsd/cache.h @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Request reply cache. This was heavily inspired by the + * implementation in 4.3BSD/4.4BSD. + * + * Copyright (C) 1995, 1996 Olaf Kirch + */ + +#ifndef NFSCACHE_H +#define NFSCACHE_H + +#include + +struct nfsd_net; + +/* + * Representation of a reply cache entry. + * + * Note that we use a sockaddr_in6 to hold the address instead of the more + * typical sockaddr_storage. This is for space reasons, since sockaddr_storage + * is much larger than a sockaddr_in6. + */ +struct nfsd_cacherep { + struct { + /* Keep often-read xid, csum in the same cache line: */ + __be32 k_xid; + __wsum k_csum; + u32 k_proc; + u32 k_prot; + u32 k_vers; + unsigned int k_len; + struct sockaddr_in6 k_addr; + } c_key; + + struct rb_node c_node; + struct list_head c_lru; + unsigned char c_state, /* unused, inprog, done */ + c_type, /* status, buffer */ + c_secure : 1; /* req came from port < 1024 */ + unsigned long c_timestamp; + union { + struct kvec u_vec; + __be32 u_status; + } c_u; +}; + +#define c_replvec c_u.u_vec +#define c_replstat c_u.u_status + +/* cache entry states */ +enum { + RC_UNUSED, + RC_INPROG, + RC_DONE +}; + +/* return values */ +enum { + RC_DROPIT, + RC_REPLY, + RC_DOIT +}; + +/* + * Cache types. + * We may want to add more types one day, e.g. for diropres and + * attrstat replies. Using cache entries with fixed length instead + * of buffer pointers may be more efficient. + */ +enum { + RC_NOCACHE, + RC_REPLSTAT, + RC_REPLBUFF, +}; + +/* Cache entries expire after this time period */ +#define RC_EXPIRE (120 * HZ) + +/* Checksum this amount of the request */ +#define RC_CSUMLEN (256U) + +int nfsd_drc_slab_create(void); +void nfsd_drc_slab_free(void); +int nfsd_reply_cache_init(struct nfsd_net *); +void nfsd_reply_cache_shutdown(struct nfsd_net *); +int nfsd_cache_lookup(struct svc_rqst *rqstp, unsigned int start, + unsigned int len, struct nfsd_cacherep **cacherep); +void nfsd_cache_update(struct svc_rqst *rqstp, struct nfsd_cacherep *rp, + int cachetype, __be32 *statp); +int nfsd_reply_cache_stats_show(struct seq_file *m, void *v); + +#endif /* NFSCACHE_H */ diff --git a/fs/nfsd/current_stateid.h b/fs/nfsd/current_stateid.h new file mode 100644 index 000000000..c28540d86 --- /dev/null +++ b/fs/nfsd/current_stateid.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _NFSD4_CURRENT_STATE_H +#define _NFSD4_CURRENT_STATE_H + +#include "state.h" +#include "xdr4.h" + +extern void clear_current_stateid(struct nfsd4_compound_state *cstate); +/* + * functions to set current state id + */ +extern void nfsd4_set_opendowngradestateid(struct nfsd4_compound_state *, + union nfsd4_op_u *); +extern void nfsd4_set_openstateid(struct nfsd4_compound_state *, + union nfsd4_op_u *); +extern void nfsd4_set_lockstateid(struct nfsd4_compound_state *, + union nfsd4_op_u *); +extern void nfsd4_set_closestateid(struct nfsd4_compound_state *, + union nfsd4_op_u *); + +/* + * functions to consume current state id + */ +extern void nfsd4_get_opendowngradestateid(struct nfsd4_compound_state *, + union nfsd4_op_u *); +extern void nfsd4_get_delegreturnstateid(struct nfsd4_compound_state *, + union nfsd4_op_u *); +extern void nfsd4_get_freestateid(struct nfsd4_compound_state *, + union nfsd4_op_u *); +extern void nfsd4_get_setattrstateid(struct nfsd4_compound_state *, + union nfsd4_op_u *); +extern void nfsd4_get_closestateid(struct nfsd4_compound_state *, + union nfsd4_op_u *); +extern void nfsd4_get_lockustateid(struct nfsd4_compound_state *, + union nfsd4_op_u *); +extern void nfsd4_get_readstateid(struct nfsd4_compound_state *, + union nfsd4_op_u *); +extern void nfsd4_get_writestateid(struct nfsd4_compound_state *, + union nfsd4_op_u *); + +#endif /* _NFSD4_CURRENT_STATE_H */ diff --git a/fs/nfsd/debugfs.c b/fs/nfsd/debugfs.c new file mode 100644 index 000000000..386fd1c54 --- /dev/null +++ b/fs/nfsd/debugfs.c @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +#include "nfsd.h" + +static struct dentry *nfsd_top_dir __read_mostly; + +/* + * /sys/kernel/debug/nfsd/disable-splice-read + * + * Contents: + * %0: NFS READ is allowed to use page splicing + * %1: NFS READ uses only iov iter read + * + * The default value of this setting is zero (page splicing is + * allowed). This setting takes immediate effect for all NFS + * versions, all exports, and in all NFSD net namespaces. + */ + +static int nfsd_dsr_get(void *data, u64 *val) +{ + *val = nfsd_disable_splice_read ? 1 : 0; + return 0; +} + +static int nfsd_dsr_set(void *data, u64 val) +{ + nfsd_disable_splice_read = (val > 0); + if (!nfsd_disable_splice_read) { + /* + * Must use buffered I/O if splice_read is enabled. + */ + nfsd_io_cache_read = NFSD_IO_BUFFERED; + } + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(nfsd_dsr_fops, nfsd_dsr_get, nfsd_dsr_set, "%llu\n"); + +/* + * /sys/kernel/debug/nfsd/io_cache_read + * + * Contents: + * %0: NFS READ will use buffered IO + * %1: NFS READ will use dontcache (buffered IO w/ dropbehind) + * %2: NFS READ will use direct IO + * + * This setting takes immediate effect for all NFS versions, + * all exports, and in all NFSD net namespaces. + */ + +static int nfsd_io_cache_read_get(void *data, u64 *val) +{ + *val = nfsd_io_cache_read; + return 0; +} + +static int nfsd_io_cache_read_set(void *data, u64 val) +{ + int ret = 0; + + switch (val) { + case NFSD_IO_BUFFERED: + nfsd_io_cache_read = NFSD_IO_BUFFERED; + break; + case NFSD_IO_DONTCACHE: + case NFSD_IO_DIRECT: + /* + * Must disable splice_read when enabling + * NFSD_IO_DONTCACHE. + */ + nfsd_disable_splice_read = true; + nfsd_io_cache_read = val; + break; + default: + ret = -EINVAL; + break; + } + + return ret; +} + +DEFINE_DEBUGFS_ATTRIBUTE(nfsd_io_cache_read_fops, nfsd_io_cache_read_get, + nfsd_io_cache_read_set, "%llu\n"); + +/* + * /sys/kernel/debug/nfsd/io_cache_write + * + * Contents: + * %0: NFS WRITE will use buffered IO + * %1: NFS WRITE will use dontcache (buffered IO w/ dropbehind) + * + * This setting takes immediate effect for all NFS versions, + * all exports, and in all NFSD net namespaces. + */ + +static int nfsd_io_cache_write_get(void *data, u64 *val) +{ + *val = nfsd_io_cache_write; + return 0; +} + +static int nfsd_io_cache_write_set(void *data, u64 val) +{ + int ret = 0; + + switch (val) { + case NFSD_IO_BUFFERED: + case NFSD_IO_DONTCACHE: + case NFSD_IO_DIRECT: + nfsd_io_cache_write = val; + break; + default: + ret = -EINVAL; + break; + } + + return ret; +} + +DEFINE_DEBUGFS_ATTRIBUTE(nfsd_io_cache_write_fops, nfsd_io_cache_write_get, + nfsd_io_cache_write_set, "%llu\n"); + +void nfsd_debugfs_exit(void) +{ + debugfs_remove_recursive(nfsd_top_dir); + nfsd_top_dir = NULL; +} + +void nfsd_debugfs_init(void) +{ + nfsd_top_dir = debugfs_create_dir("nfsd", NULL); + + debugfs_create_file("disable-splice-read", S_IWUSR | S_IRUGO, + nfsd_top_dir, NULL, &nfsd_dsr_fops); + + debugfs_create_file("io_cache_read", 0644, nfsd_top_dir, NULL, + &nfsd_io_cache_read_fops); + + debugfs_create_file("io_cache_write", 0644, nfsd_top_dir, NULL, + &nfsd_io_cache_write_fops); +#ifdef CONFIG_NFSD_V4 + debugfs_create_bool("delegated_timestamps", 0644, nfsd_top_dir, + &nfsd_delegts_enabled); +#endif +} diff --git a/fs/nfsd/export.c b/fs/nfsd/export.c new file mode 100644 index 000000000..a47c90f40 --- /dev/null +++ b/fs/nfsd/export.c @@ -0,0 +1,2252 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * NFS exporting and validation. + * + * We maintain a list of clients, each of which has a list of + * exports. To export an fs to a given client, you first have + * to create the client entry with NFSCTL_ADDCLIENT, which + * creates a client control block and adds it to the hash + * table. Then, you call NFSCTL_EXPORT for each fs. + * + * + * Copyright (C) 1995, 1996 Olaf Kirch, + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "nfsd.h" +#include "nfsfh.h" +#include "netns.h" +#include "pnfs.h" +#include "filecache.h" +#include "trace.h" +#include "netlink.h" + +#define NFSDDBG_FACILITY NFSDDBG_EXPORT + +/* + * We have two caches. + * One maps client+vfsmnt+dentry to export options - the export map + * The other maps client+filehandle-fragment to export options. - the expkey map + * + * The export options are actually stored in the first map, and the + * second map contains a reference to the entry in the first map. + */ + +#define EXPKEY_HASHBITS 8 +#define EXPKEY_HASHMAX (1 << EXPKEY_HASHBITS) +#define EXPKEY_HASHMASK (EXPKEY_HASHMAX -1) + +static void expkey_put(struct kref *ref) +{ + struct svc_expkey *key = container_of(ref, struct svc_expkey, h.ref); + + if (test_bit(CACHE_VALID, &key->h.flags) && + !test_bit(CACHE_NEGATIVE, &key->h.flags)) + path_put(&key->ek_path); + auth_domain_put(key->ek_client); + kfree_rcu(key, ek_rcu); +} + +static int expkey_upcall(struct cache_detail *cd, struct cache_head *h) +{ + return sunrpc_cache_upcall(cd, h); +} + +static void expkey_request(struct cache_detail *cd, + struct cache_head *h, + char **bpp, int *blen) +{ + /* client fsidtype \xfsid */ + struct svc_expkey *ek = container_of(h, struct svc_expkey, h); + char type[5]; + + qword_add(bpp, blen, ek->ek_client->name); + snprintf(type, 5, "%d", ek->ek_fsidtype); + qword_add(bpp, blen, type); + qword_addhex(bpp, blen, (char*)ek->ek_fsid, key_len(ek->ek_fsidtype)); + (*bpp)[-1] = '\n'; +} + +static struct svc_expkey *svc_expkey_update(struct cache_detail *cd, struct svc_expkey *new, + struct svc_expkey *old); +static struct svc_expkey *svc_expkey_lookup(struct cache_detail *cd, struct svc_expkey *); + +static int expkey_parse(struct cache_detail *cd, char *mesg, int mlen) +{ + /* client fsidtype fsid expiry [path] */ + char *buf; + int len; + struct auth_domain *dom = NULL; + int err; + u8 fsidtype; + struct svc_expkey key; + struct svc_expkey *ek = NULL; + + if (mesg[mlen - 1] != '\n') + return -EINVAL; + mesg[mlen-1] = 0; + + buf = kmalloc(PAGE_SIZE, GFP_KERNEL); + err = -ENOMEM; + if (!buf) + goto out; + + err = -EINVAL; + if (qword_get(&mesg, buf, PAGE_SIZE) <= 0) + goto out; + + err = -ENOENT; + dom = auth_domain_find(buf); + if (!dom) + goto out; + dprintk("found domain %s\n", buf); + + err = -EINVAL; + if (qword_get(&mesg, buf, PAGE_SIZE) <= 0) + goto out; + if (kstrtou8(buf, 10, &fsidtype)) + goto out; + dprintk("found fsidtype %u\n", fsidtype); + if (key_len(fsidtype)==0) /* invalid type */ + goto out; + if ((len=qword_get(&mesg, buf, PAGE_SIZE)) <= 0) + goto out; + dprintk("found fsid length %d\n", len); + if (len != key_len(fsidtype)) + goto out; + + /* OK, we seem to have a valid key */ + key.h.flags = 0; + err = get_expiry(&mesg, &key.h.expiry_time); + if (err) + goto out; + + key.ek_client = dom; + key.ek_fsidtype = fsidtype; + memcpy(key.ek_fsid, buf, len); + + ek = svc_expkey_lookup(cd, &key); + err = -ENOMEM; + if (!ek) + goto out; + + /* now we want a pathname, or empty meaning NEGATIVE */ + err = -EINVAL; + len = qword_get(&mesg, buf, PAGE_SIZE); + if (len < 0) + goto out; + dprintk("Path seems to be <%s>\n", buf); + err = 0; + if (len == 0) { + set_bit(CACHE_NEGATIVE, &key.h.flags); + ek = svc_expkey_update(cd, &key, ek); + if (ek) + trace_nfsd_expkey_update(ek, NULL); + else + err = -ENOMEM; + } else { + err = kern_path(buf, 0, &key.ek_path); + if (err) + goto out; + + dprintk("Found the path %s\n", buf); + + ek = svc_expkey_update(cd, &key, ek); + if (ek) + trace_nfsd_expkey_update(ek, buf); + else + err = -ENOMEM; + path_put(&key.ek_path); + } + cache_flush(); + out: + if (ek) + cache_put(&ek->h, cd); + if (dom) + auth_domain_put(dom); + kfree(buf); + return err; +} + +static int expkey_show(struct seq_file *m, + struct cache_detail *cd, + struct cache_head *h) +{ + struct svc_expkey *ek ; + int i; + + if (h ==NULL) { + seq_puts(m, "#domain fsidtype fsid [path]\n"); + return 0; + } + ek = container_of(h, struct svc_expkey, h); + seq_printf(m, "%s %d 0x", ek->ek_client->name, + ek->ek_fsidtype); + for (i=0; i < key_len(ek->ek_fsidtype)/4; i++) + seq_printf(m, "%08x", ek->ek_fsid[i]); + if (test_bit(CACHE_VALID, &h->flags) && + !test_bit(CACHE_NEGATIVE, &h->flags)) { + seq_printf(m, " "); + seq_path(m, &ek->ek_path, "\\ \t\n"); + } + seq_printf(m, "\n"); + return 0; +} + +static inline int expkey_match (struct cache_head *a, struct cache_head *b) +{ + struct svc_expkey *orig = container_of(a, struct svc_expkey, h); + struct svc_expkey *new = container_of(b, struct svc_expkey, h); + + if (orig->ek_fsidtype != new->ek_fsidtype || + orig->ek_client != new->ek_client || + memcmp(orig->ek_fsid, new->ek_fsid, key_len(orig->ek_fsidtype)) != 0) + return 0; + return 1; +} + +static inline void expkey_init(struct cache_head *cnew, + struct cache_head *citem) +{ + struct svc_expkey *new = container_of(cnew, struct svc_expkey, h); + struct svc_expkey *item = container_of(citem, struct svc_expkey, h); + + kref_get(&item->ek_client->ref); + new->ek_client = item->ek_client; + new->ek_fsidtype = item->ek_fsidtype; + + memcpy(new->ek_fsid, item->ek_fsid, sizeof(new->ek_fsid)); +} + +static inline void expkey_update(struct cache_head *cnew, + struct cache_head *citem) +{ + struct svc_expkey *new = container_of(cnew, struct svc_expkey, h); + struct svc_expkey *item = container_of(citem, struct svc_expkey, h); + + new->ek_path = item->ek_path; + path_get(&item->ek_path); +} + +static struct cache_head *expkey_alloc(void) +{ + struct svc_expkey *i = kmalloc_obj(*i); + if (i) + return &i->h; + else + return NULL; +} + +static void expkey_flush(void) +{ + /* + * Take the nfsd_mutex here to ensure that the file cache is not + * destroyed while we're in the middle of flushing. + */ + mutex_lock(&nfsd_mutex); + nfsd_file_cache_purge(current->nsproxy->net_ns); + mutex_unlock(&nfsd_mutex); +} + +static int expkey_notify(struct cache_detail *cd, struct cache_head *h) +{ + return nfsd_cache_notify(cd, h, NFSD_CACHE_TYPE_EXPKEY); +} + +static const struct cache_detail svc_expkey_cache_template = { + .owner = THIS_MODULE, + .hash_size = EXPKEY_HASHMAX, + .name = "nfsd.fh", + .cache_put = expkey_put, + .cache_upcall = expkey_upcall, + .cache_notify = expkey_notify, + .cache_request = expkey_request, + .cache_parse = expkey_parse, + .cache_show = expkey_show, + .match = expkey_match, + .init = expkey_init, + .update = expkey_update, + .alloc = expkey_alloc, + .flush = expkey_flush, +}; + +static int +svc_expkey_hash(struct svc_expkey *item) +{ + int hash = item->ek_fsidtype; + char * cp = (char*)item->ek_fsid; + int len = key_len(item->ek_fsidtype); + + hash ^= hash_mem(cp, len, EXPKEY_HASHBITS); + hash ^= hash_ptr(item->ek_client, EXPKEY_HASHBITS); + hash &= EXPKEY_HASHMASK; + return hash; +} + +static struct svc_expkey * +svc_expkey_lookup(struct cache_detail *cd, struct svc_expkey *item) +{ + struct cache_head *ch; + int hash = svc_expkey_hash(item); + + ch = sunrpc_cache_lookup_rcu(cd, &item->h, hash); + if (ch) + return container_of(ch, struct svc_expkey, h); + else + return NULL; +} + +static struct svc_expkey * +svc_expkey_update(struct cache_detail *cd, struct svc_expkey *new, + struct svc_expkey *old) +{ + struct cache_head *ch; + int hash = svc_expkey_hash(new); + + ch = sunrpc_cache_update(cd, &new->h, &old->h, hash); + if (ch) + return container_of(ch, struct svc_expkey, h); + else + return NULL; +} + +/** + * nfsd_nl_expkey_get_reqs_dumpit - dump pending expkey requests + * @skb: reply buffer + * @cb: netlink metadata and command arguments + * + * Walk the expkey cache's pending request list and create a netlink + * message with a nested entry for each cache_request, containing the + * seqno, client string, fsidtype and fsid. + * + * Uses cb->args[0] as a seqno cursor for dump continuation across + * multiple netlink messages. + * + * Returns the size of the reply or a negative errno. + */ +int nfsd_nl_expkey_get_reqs_dumpit(struct sk_buff *skb, + struct netlink_callback *cb) +{ + struct nfsd_net *nn; + struct cache_detail *cd; + struct cache_head **items; + u64 *seqnos; + int cnt, i, emitted; + void *hdr; + int ret; + + nn = net_generic(sock_net(skb->sk), nfsd_net_id); + + mutex_lock(&nfsd_mutex); + + cd = nn->svc_expkey_cache; + if (!cd) { + ret = -ENODEV; + goto out_unlock; + } + + cnt = sunrpc_cache_requests_count(cd); + if (!cnt) { + ret = 0; + goto out_unlock; + } + + items = kcalloc(cnt, sizeof(*items), GFP_KERNEL); + seqnos = kcalloc(cnt, sizeof(*seqnos), GFP_KERNEL); + if (!items || !seqnos) { + ret = -ENOMEM; + goto out_alloc; + } + + cnt = sunrpc_cache_requests_snapshot(cd, items, seqnos, cnt, + cb->args[0]); + if (!cnt) { + ret = 0; + goto out_alloc; + } + + hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, + cb->nlh->nlmsg_seq, &nfsd_nl_family, + NLM_F_MULTI, NFSD_CMD_EXPKEY_GET_REQS); + if (!hdr) { + ret = -ENOBUFS; + goto out_put; + } + + emitted = 0; + for (i = 0; i < cnt; i++) { + struct svc_expkey *ek; + struct nlattr *nest; + + ek = container_of(items[i], struct svc_expkey, h); + + nest = nla_nest_start(skb, NFSD_A_EXPKEY_REQS_REQUESTS); + if (!nest) + break; + + if (nla_put_u64_64bit(skb, NFSD_A_EXPKEY_SEQNO, + seqnos[i], 0) || + nla_put_string(skb, NFSD_A_EXPKEY_CLIENT, + ek->ek_client->name) || + nla_put_u8(skb, NFSD_A_EXPKEY_FSIDTYPE, + ek->ek_fsidtype) || + nla_put(skb, NFSD_A_EXPKEY_FSID, + key_len(ek->ek_fsidtype), ek->ek_fsid)) { + nla_nest_cancel(skb, nest); + break; + } + + nla_nest_end(skb, nest); + cb->args[0] = seqnos[i]; + emitted++; + } + + if (!emitted) { + genlmsg_cancel(skb, hdr); + ret = -EMSGSIZE; + goto out_put; + } + + genlmsg_end(skb, hdr); + ret = skb->len; +out_put: + for (i = 0; i < cnt; i++) + cache_put(items[i], cd); +out_alloc: + kfree(seqnos); + kfree(items); +out_unlock: + mutex_unlock(&nfsd_mutex); + return ret; +} + +/** + * nfsd_nl_parse_one_expkey - parse one expkey entry from netlink + * @cd: cache_detail for the expkey cache + * @attr: nested attribute containing expkey fields + * + * Parses one expkey entry from a netlink message and updates the + * cache. Mirrors the logic in expkey_parse(). + * + * Returns 0 on success or a negative errno. + */ +static int nfsd_nl_parse_one_expkey(struct cache_detail *cd, + struct nlattr *attr) +{ + struct nlattr *tb[NFSD_A_EXPKEY_PATH + 1]; + struct auth_domain *dom = NULL; + struct svc_expkey key; + struct svc_expkey *ek = NULL; + struct timespec64 boot; + int err; + u8 fsidtype; + int fsid_len; + + err = nla_parse_nested(tb, NFSD_A_EXPKEY_PATH, attr, + nfsd_expkey_nl_policy, NULL); + if (err) + return err; + + /* client (required) */ + if (!tb[NFSD_A_EXPKEY_CLIENT]) + return -EINVAL; + + dom = auth_domain_find(nla_data(tb[NFSD_A_EXPKEY_CLIENT])); + if (!dom) + return -ENOENT; + + /* fsidtype (required) */ + if (!tb[NFSD_A_EXPKEY_FSIDTYPE]) { + err = -EINVAL; + goto out_dom; + } + fsidtype = nla_get_u8(tb[NFSD_A_EXPKEY_FSIDTYPE]); + if (key_len(fsidtype) == 0) { + err = -EINVAL; + goto out_dom; + } + + /* fsid (required) */ + if (!tb[NFSD_A_EXPKEY_FSID]) { + err = -EINVAL; + goto out_dom; + } + fsid_len = nla_len(tb[NFSD_A_EXPKEY_FSID]); + if (fsid_len != key_len(fsidtype)) { + err = -EINVAL; + goto out_dom; + } + + /* expiry (required, wallclock seconds) */ + if (!tb[NFSD_A_EXPKEY_EXPIRY]) { + err = -EINVAL; + goto out_dom; + } + + key.h.flags = 0; + getboottime64(&boot); + key.h.expiry_time = nla_get_u64(tb[NFSD_A_EXPKEY_EXPIRY]) - + boot.tv_sec; + key.ek_client = dom; + key.ek_fsidtype = fsidtype; + memcpy(key.ek_fsid, nla_data(tb[NFSD_A_EXPKEY_FSID]), fsid_len); + + ek = svc_expkey_lookup(cd, &key); + if (!ek) { + err = -ENOMEM; + goto out_dom; + } + + if (tb[NFSD_A_EXPKEY_NEGATIVE]) { + set_bit(CACHE_NEGATIVE, &key.h.flags); + ek = svc_expkey_update(cd, &key, ek); + if (ek) + trace_nfsd_expkey_update(ek, NULL); + else + err = -ENOMEM; + } else if (tb[NFSD_A_EXPKEY_PATH]) { + err = kern_path(nla_data(tb[NFSD_A_EXPKEY_PATH]), 0, + &key.ek_path); + if (err) + goto out_ek; + ek = svc_expkey_update(cd, &key, ek); + if (ek) + trace_nfsd_expkey_update(ek, + nla_data(tb[NFSD_A_EXPKEY_PATH])); + else + err = -ENOMEM; + path_put(&key.ek_path); + } else { + err = -EINVAL; + goto out_ek; + } + + cache_flush(); + +out_ek: + if (ek) + cache_put(&ek->h, cd); +out_dom: + auth_domain_put(dom); + return err; +} + +/** + * nfsd_nl_expkey_set_reqs_doit - respond to expkey requests + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Parse one or more expkey cache responses from userspace and + * update the expkey cache accordingly. + * + * Returns 0 on success or a negative errno. + */ +int nfsd_nl_expkey_set_reqs_doit(struct sk_buff *skb, + struct genl_info *info) +{ + struct nfsd_net *nn; + struct cache_detail *cd; + const struct nlattr *attr; + int rem, ret = 0; + + nn = net_generic(genl_info_net(info), nfsd_net_id); + + mutex_lock(&nfsd_mutex); + + cd = nn->svc_expkey_cache; + if (!cd) { + ret = -ENODEV; + goto out_unlock; + } + + nlmsg_for_each_attr_type(attr, NFSD_A_EXPKEY_REQS_REQUESTS, + info->nlhdr, GENL_HDRLEN, rem) { + ret = nfsd_nl_parse_one_expkey(cd, (struct nlattr *)attr); + if (ret) + break; + } + +out_unlock: + mutex_unlock(&nfsd_mutex); + return ret; +} + +#define EXPORT_HASHBITS 8 +#define EXPORT_HASHMAX (1<< EXPORT_HASHBITS) + +static void nfsd4_fslocs_free(struct nfsd4_fs_locations *fsloc) +{ + struct nfsd4_fs_location *locations = fsloc->locations; + int i; + + if (!locations) + return; + + for (i = 0; i < fsloc->locations_count; i++) { + kfree(locations[i].path); + kfree(locations[i].hosts); + } + + kfree(locations); + fsloc->locations = NULL; +} + +static int export_stats_init(struct export_stats *stats) +{ + stats->start_time = ktime_get_seconds(); + return percpu_counter_init_many(stats->counter, 0, GFP_KERNEL, + EXP_STATS_COUNTERS_NUM); +} + +static void export_stats_reset(struct export_stats *stats) +{ + if (stats) { + int i; + + for (i = 0; i < EXP_STATS_COUNTERS_NUM; i++) + percpu_counter_set(&stats->counter[i], 0); + } +} + +static void export_stats_destroy(struct export_stats *stats) +{ + if (stats) + percpu_counter_destroy_many(stats->counter, + EXP_STATS_COUNTERS_NUM); +} + +static void svc_export_release(struct rcu_head *rcu_head) +{ + struct svc_export *exp = container_of(rcu_head, struct svc_export, + ex_rcu); + + nfsd4_fslocs_free(&exp->ex_fslocs); + export_stats_destroy(exp->ex_stats); + kfree(exp->ex_stats); + kfree(exp->ex_uuid); + kfree(exp); +} + +static void svc_export_put(struct kref *ref) +{ + struct svc_export *exp = container_of(ref, struct svc_export, h.ref); + + path_put(&exp->ex_path); + auth_domain_put(exp->ex_client); + call_rcu(&exp->ex_rcu, svc_export_release); +} + +/** + * nfsd_nl_svc_export_get_reqs_dumpit - dump pending svc_export requests + * @skb: reply buffer + * @cb: netlink metadata and command arguments + * + * Walk the svc_export cache's pending request list and create a netlink + * message with a nested entry for each cache_request, containing the + * seqno, client string, and path. + * + * Uses cb->args[0] as a seqno cursor for dump continuation across + * multiple netlink messages. + * + * Returns the size of the reply or a negative errno. + */ +int nfsd_nl_svc_export_get_reqs_dumpit(struct sk_buff *skb, + struct netlink_callback *cb) +{ + struct nfsd_net *nn; + struct cache_detail *cd; + struct cache_head **items; + u64 *seqnos; + int cnt, i, emitted; + char *pathbuf; + void *hdr; + int ret; + + nn = net_generic(sock_net(skb->sk), nfsd_net_id); + + mutex_lock(&nfsd_mutex); + + cd = nn->svc_export_cache; + if (!cd) { + ret = -ENODEV; + goto out_unlock; + } + + cnt = sunrpc_cache_requests_count(cd); + if (!cnt) { + ret = 0; + goto out_unlock; + } + + items = kcalloc(cnt, sizeof(*items), GFP_KERNEL); + seqnos = kcalloc(cnt, sizeof(*seqnos), GFP_KERNEL); + pathbuf = kmalloc(PATH_MAX, GFP_KERNEL); + if (!items || !seqnos || !pathbuf) { + ret = -ENOMEM; + goto out_alloc; + } + + cnt = sunrpc_cache_requests_snapshot(cd, items, seqnos, cnt, + cb->args[0]); + if (!cnt) { + ret = 0; + goto out_alloc; + } + + hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, + cb->nlh->nlmsg_seq, &nfsd_nl_family, + NLM_F_MULTI, NFSD_CMD_SVC_EXPORT_GET_REQS); + if (!hdr) { + ret = -ENOBUFS; + goto out_put; + } + + emitted = 0; + for (i = 0; i < cnt; i++) { + struct svc_export *exp; + struct nlattr *nest; + char *pth; + + exp = container_of(items[i], struct svc_export, h); + + pth = d_path(&exp->ex_path, pathbuf, PATH_MAX); + if (IS_ERR(pth)) + continue; + + nest = nla_nest_start(skb, + NFSD_A_SVC_EXPORT_REQS_REQUESTS); + if (!nest) + break; + + if (nla_put_u64_64bit(skb, NFSD_A_SVC_EXPORT_SEQNO, + seqnos[i], 0) || + nla_put_string(skb, NFSD_A_SVC_EXPORT_CLIENT, + exp->ex_client->name) || + nla_put_string(skb, NFSD_A_SVC_EXPORT_PATH, pth)) { + nla_nest_cancel(skb, nest); + break; + } + + nla_nest_end(skb, nest); + cb->args[0] = seqnos[i]; + emitted++; + } + + if (!emitted) { + genlmsg_cancel(skb, hdr); + ret = -EMSGSIZE; + goto out_put; + } + + genlmsg_end(skb, hdr); + ret = skb->len; +out_put: + for (i = 0; i < cnt; i++) + cache_put(items[i], cd); +out_alloc: + kfree(pathbuf); + kfree(seqnos); + kfree(items); +out_unlock: + mutex_unlock(&nfsd_mutex); + return ret; +} + +/** + * nfsd_nl_parse_fslocations - parse fslocations from netlink + * @attr: NFSD_A_SVC_EXPORT_FSLOCATIONS nested attribute + * @fsloc: fslocations struct to fill in + * + * Returns 0 on success or a negative errno. + */ +static int nfsd_nl_parse_fslocations(struct nlattr *attr, + struct nfsd4_fs_locations *fsloc) +{ + struct nlattr *loc_attr; + int rem, count = 0; + int err; + + if (fsloc->locations) + return -EINVAL; + + /* Count locations first */ + nla_for_each_nested_type(loc_attr, NFSD_A_FSLOCATIONS_LOCATION, + attr, rem) + count++; + + if (count > MAX_FS_LOCATIONS) + return -EINVAL; + if (!count) + return 0; + + fsloc->locations = kcalloc(count, sizeof(struct nfsd4_fs_location), + GFP_KERNEL); + if (!fsloc->locations) + return -ENOMEM; + + nla_for_each_nested_type(loc_attr, NFSD_A_FSLOCATIONS_LOCATION, + attr, rem) { + struct nlattr *tb[NFSD_A_FSLOCATION_PATH + 1]; + struct nfsd4_fs_location *loc; + + err = nla_parse_nested(tb, NFSD_A_FSLOCATION_PATH, loc_attr, + nfsd_fslocation_nl_policy, NULL); + if (err) + goto out_free; + + if (!tb[NFSD_A_FSLOCATION_HOST] || + !tb[NFSD_A_FSLOCATION_PATH]) { + err = -EINVAL; + goto out_free; + } + + loc = &fsloc->locations[fsloc->locations_count++]; + loc->hosts = kstrdup(nla_data(tb[NFSD_A_FSLOCATION_HOST]), + GFP_KERNEL); + loc->path = kstrdup(nla_data(tb[NFSD_A_FSLOCATION_PATH]), + GFP_KERNEL); + if (!loc->hosts || !loc->path) { + err = -ENOMEM; + goto out_free; + } + } + + return 0; +out_free: + nfsd4_fslocs_free(fsloc); + return err; +} + +static struct svc_export *svc_export_update(struct svc_export *new, + struct svc_export *old); +static struct svc_export *svc_export_lookup(struct svc_export *); +static int check_export(const struct path *path, int *flags, + unsigned char *uuid); + +/** + * nfsd_nl_parse_one_export - parse one svc_export entry from a netlink message + * @cd: cache_detail for the svc_export cache + * @attr: nested attribute containing svc-export fields + * + * Parses one svc-export entry from a netlink message and updates the + * cache. Mirrors the logic in svc_export_parse(). + * + * Returns 0 on success or a negative errno. + */ +static int nfsd_nl_parse_one_export(struct cache_detail *cd, + struct nlattr *attr) +{ + struct nlattr *tb[NFSD_A_SVC_EXPORT_FSID + 1]; + struct auth_domain *dom = NULL; + struct svc_export exp = {}, *expp; + struct nlattr *secinfo_attr; + struct timespec64 boot; + int err, rem; + + err = nla_parse_nested(tb, NFSD_A_SVC_EXPORT_FSID, attr, + nfsd_svc_export_nl_policy, NULL); + if (err) + return err; + + /* client (required) */ + if (!tb[NFSD_A_SVC_EXPORT_CLIENT]) + return -EINVAL; + + dom = auth_domain_find(nla_data(tb[NFSD_A_SVC_EXPORT_CLIENT])); + if (!dom) + return -ENOENT; + + /* path (required) */ + if (!tb[NFSD_A_SVC_EXPORT_PATH]) { + err = -EINVAL; + goto out_dom; + } + + err = kern_path(nla_data(tb[NFSD_A_SVC_EXPORT_PATH]), 0, + &exp.ex_path); + if (err) + goto out_dom; + + exp.ex_client = dom; + exp.cd = cd; + exp.ex_devid_map = NULL; + exp.ex_xprtsec_modes = NFSEXP_XPRTSEC_ALL; + + /* expiry (required, wallclock seconds) */ + if (!tb[NFSD_A_SVC_EXPORT_EXPIRY]) { + err = -EINVAL; + goto out_path; + } + getboottime64(&boot); + exp.h.expiry_time = nla_get_u64(tb[NFSD_A_SVC_EXPORT_EXPIRY]) - + boot.tv_sec; + + if (tb[NFSD_A_SVC_EXPORT_NEGATIVE]) { + set_bit(CACHE_NEGATIVE, &exp.h.flags); + } else { + /* flags */ + if (tb[NFSD_A_SVC_EXPORT_FLAGS]) + exp.ex_flags = nla_get_u32(tb[NFSD_A_SVC_EXPORT_FLAGS]); + + /* anon uid */ + if (tb[NFSD_A_SVC_EXPORT_ANON_UID]) { + u32 uid = nla_get_u32(tb[NFSD_A_SVC_EXPORT_ANON_UID]); + + exp.ex_anon_uid = make_kuid(current_user_ns(), uid); + } + + /* anon gid */ + if (tb[NFSD_A_SVC_EXPORT_ANON_GID]) { + u32 gid = nla_get_u32(tb[NFSD_A_SVC_EXPORT_ANON_GID]); + + exp.ex_anon_gid = make_kgid(current_user_ns(), gid); + } + + /* fsid */ + if (tb[NFSD_A_SVC_EXPORT_FSID]) + exp.ex_fsid = nla_get_s32(tb[NFSD_A_SVC_EXPORT_FSID]); + + /* fslocations */ + if (tb[NFSD_A_SVC_EXPORT_FSLOCATIONS]) { + struct nlattr *fsl = tb[NFSD_A_SVC_EXPORT_FSLOCATIONS]; + + err = nfsd_nl_parse_fslocations(fsl, + &exp.ex_fslocs); + if (err) + goto out_path; + } + + /* uuid */ + if (tb[NFSD_A_SVC_EXPORT_UUID]) { + if (nla_len(tb[NFSD_A_SVC_EXPORT_UUID]) != + EX_UUID_LEN) { + err = -EINVAL; + goto out_fslocs; + } + exp.ex_uuid = kmemdup(nla_data(tb[NFSD_A_SVC_EXPORT_UUID]), + EX_UUID_LEN, GFP_KERNEL); + if (!exp.ex_uuid) { + err = -ENOMEM; + goto out_fslocs; + } + } + + /* secinfo (multi-attr) */ + nla_for_each_nested_type(secinfo_attr, + NFSD_A_SVC_EXPORT_SECINFO, + attr, rem) { + struct nlattr *ftb[NFSD_A_AUTH_FLAVOR_FLAGS + 1]; + struct exp_flavor_info *f; + + if (exp.ex_nflavors >= MAX_SECINFO_LIST) { + err = -EINVAL; + goto out_uuid; + } + + err = nla_parse_nested(ftb, + NFSD_A_AUTH_FLAVOR_FLAGS, + secinfo_attr, + nfsd_auth_flavor_nl_policy, + NULL); + if (err) + goto out_uuid; + + f = &exp.ex_flavors[exp.ex_nflavors++]; + + if (ftb[NFSD_A_AUTH_FLAVOR_PSEUDOFLAVOR]) + f->pseudoflavor = nla_get_u32(ftb[NFSD_A_AUTH_FLAVOR_PSEUDOFLAVOR]); + + if (ftb[NFSD_A_AUTH_FLAVOR_FLAGS]) + f->flags = nla_get_u32(ftb[NFSD_A_AUTH_FLAVOR_FLAGS]); + + /* Only some flags are allowed to differ between flavors: */ + if (~NFSEXP_SECINFO_FLAGS & (f->flags ^ exp.ex_flags)) { + err = -EINVAL; + goto out_uuid; + } + } + + /* xprtsec (multi-attr u32) */ + if (tb[NFSD_A_SVC_EXPORT_XPRTSEC]) { + struct nlattr *xp_attr; + + exp.ex_xprtsec_modes = 0; + nla_for_each_nested_type(xp_attr, + NFSD_A_SVC_EXPORT_XPRTSEC, + attr, rem) { + u32 mode = nla_get_u32(xp_attr); + + if (mode > NFSEXP_XPRTSEC_MTLS) { + err = -EINVAL; + goto out_uuid; + } + exp.ex_xprtsec_modes |= mode; + } + } + + err = check_export(&exp.ex_path, &exp.ex_flags, + exp.ex_uuid); + if (err) + goto out_uuid; + + if (exp.h.expiry_time < seconds_since_boot()) + goto out_uuid; + + err = -EINVAL; + if (!uid_valid(exp.ex_anon_uid)) + goto out_uuid; + if (!gid_valid(exp.ex_anon_gid)) + goto out_uuid; + err = 0; + + nfsd4_setup_layout_type(&exp); + } + + expp = svc_export_lookup(&exp); + if (!expp) { + err = -ENOMEM; + goto out_uuid; + } + expp = svc_export_update(&exp, expp); + if (expp) { + trace_nfsd_export_update(expp); + cache_flush(); + exp_put(expp); + } else { + err = -ENOMEM; + } + +out_uuid: + kfree(exp.ex_uuid); +out_fslocs: + nfsd4_fslocs_free(&exp.ex_fslocs); +out_path: + path_put(&exp.ex_path); +out_dom: + auth_domain_put(dom); + return err; +} + +/** + * nfsd_nl_svc_export_set_reqs_doit - respond to svc_export requests + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Parse one or more svc_export cache responses from userspace and + * update the export cache accordingly. + * + * Returns 0 on success or a negative errno. + */ +int nfsd_nl_svc_export_set_reqs_doit(struct sk_buff *skb, + struct genl_info *info) +{ + struct nfsd_net *nn; + struct cache_detail *cd; + const struct nlattr *attr; + int rem, ret = 0; + + nn = net_generic(genl_info_net(info), nfsd_net_id); + + mutex_lock(&nfsd_mutex); + + cd = nn->svc_export_cache; + if (!cd) { + ret = -ENODEV; + goto out_unlock; + } + + nlmsg_for_each_attr_type(attr, NFSD_A_SVC_EXPORT_REQS_REQUESTS, + info->nlhdr, GENL_HDRLEN, rem) { + ret = nfsd_nl_parse_one_export(cd, (struct nlattr *)attr); + if (ret) + break; + } + +out_unlock: + mutex_unlock(&nfsd_mutex); + return ret; +} + +static int svc_export_upcall(struct cache_detail *cd, struct cache_head *h) +{ + return sunrpc_cache_upcall(cd, h); +} + +static int svc_export_notify(struct cache_detail *cd, struct cache_head *h) +{ + return nfsd_cache_notify(cd, h, NFSD_CACHE_TYPE_SVC_EXPORT); +} + +static void svc_export_request(struct cache_detail *cd, + struct cache_head *h, + char **bpp, int *blen) +{ + /* client path */ + struct svc_export *exp = container_of(h, struct svc_export, h); + char *pth; + + qword_add(bpp, blen, exp->ex_client->name); + pth = d_path(&exp->ex_path, *bpp, *blen); + if (IS_ERR(pth)) { + /* is this correct? */ + (*bpp)[0] = '\n'; + return; + } + qword_add(bpp, blen, pth); + (*bpp)[-1] = '\n'; +} + +static int check_export(const struct path *path, int *flags, unsigned char *uuid) +{ + struct inode *inode = d_inode(path->dentry); + + /* + * We currently export only dirs, regular files, and (for v4 + * pseudoroot) symlinks. + */ + if (!S_ISDIR(inode->i_mode) && + !S_ISLNK(inode->i_mode) && + !S_ISREG(inode->i_mode)) + return -ENOTDIR; + + /* + * Mountd should never pass down a writeable V4ROOT export, but, + * just to make sure: + */ + if (*flags & NFSEXP_V4ROOT) + *flags |= NFSEXP_READONLY; + + /* There are two requirements on a filesystem to be exportable. + * 1: We must be able to identify the filesystem from a number. + * either a device number (so FS_REQUIRES_DEV needed) + * or an FSID number (so NFSEXP_FSID or ->uuid is needed). + * 2: We must be able to find an inode from a filehandle. + * This means that s_export_op must be set and comply with + * the requirements for remote filesystem export. + * 3: We must not currently be on an idmapped mount. + */ + if (!(inode->i_sb->s_type->fs_flags & FS_REQUIRES_DEV) && + !(*flags & NFSEXP_FSID) && + uuid == NULL) { + dprintk("exp_export: export of non-dev fs without fsid\n"); + return -EINVAL; + } + + if (!exportfs_may_export(inode->i_sb->s_export_op)) { + dprintk("exp_export: export of invalid fs type (%s).\n", + inode->i_sb->s_type->name); + return -EINVAL; + } + + if (is_idmapped_mnt(path->mnt)) { + dprintk("exp_export: export of idmapped mounts not yet supported.\n"); + return -EINVAL; + } + + if (inode->i_sb->s_export_op->flags & EXPORT_OP_NOSUBTREECHK && + !(*flags & NFSEXP_NOSUBTREECHECK)) { + dprintk("%s: %s does not support subtree checking!\n", + __func__, inode->i_sb->s_type->name); + return -EINVAL; + } + return 0; +} + +#ifdef CONFIG_NFSD_V4 + +static int +fsloc_parse(char **mesg, char *buf, struct nfsd4_fs_locations *fsloc) +{ + int len; + int migrated, i, err; + + /* more than one fsloc */ + if (fsloc->locations) + return -EINVAL; + + /* listsize */ + err = get_uint(mesg, &fsloc->locations_count); + if (err) + return err; + if (fsloc->locations_count > MAX_FS_LOCATIONS) + return -EINVAL; + if (fsloc->locations_count == 0) + return 0; + + fsloc->locations = kzalloc_objs(struct nfsd4_fs_location, + fsloc->locations_count); + if (!fsloc->locations) + return -ENOMEM; + for (i=0; i < fsloc->locations_count; i++) { + /* colon separated host list */ + err = -EINVAL; + len = qword_get(mesg, buf, PAGE_SIZE); + if (len <= 0) + goto out_free_all; + err = -ENOMEM; + fsloc->locations[i].hosts = kstrdup(buf, GFP_KERNEL); + if (!fsloc->locations[i].hosts) + goto out_free_all; + err = -EINVAL; + /* slash separated path component list */ + len = qword_get(mesg, buf, PAGE_SIZE); + if (len <= 0) + goto out_free_all; + err = -ENOMEM; + fsloc->locations[i].path = kstrdup(buf, GFP_KERNEL); + if (!fsloc->locations[i].path) + goto out_free_all; + } + /* migrated */ + err = get_int(mesg, &migrated); + if (err) + goto out_free_all; + err = -EINVAL; + if (migrated < 0 || migrated > 1) + goto out_free_all; + fsloc->migrated = migrated; + return 0; +out_free_all: + nfsd4_fslocs_free(fsloc); + return err; +} + +static int secinfo_parse(char **mesg, char *buf, struct svc_export *exp) +{ + struct exp_flavor_info *f; + u32 listsize; + int err; + + /* more than one secinfo */ + if (exp->ex_nflavors) + return -EINVAL; + + err = get_uint(mesg, &listsize); + if (err) + return err; + if (listsize > MAX_SECINFO_LIST) + return -EINVAL; + + for (f = exp->ex_flavors; f < exp->ex_flavors + listsize; f++) { + err = get_uint(mesg, &f->pseudoflavor); + if (err) + return err; + /* + * XXX: It would be nice to also check whether this + * pseudoflavor is supported, so we can discover the + * problem at export time instead of when a client fails + * to authenticate. + */ + err = get_uint(mesg, &f->flags); + if (err) + return err; + /* Only some flags are allowed to differ between flavors: */ + if (~NFSEXP_SECINFO_FLAGS & (f->flags ^ exp->ex_flags)) + return -EINVAL; + } + exp->ex_nflavors = listsize; + return 0; +} + +#else /* CONFIG_NFSD_V4 */ +static inline int +fsloc_parse(char **mesg, char *buf, struct nfsd4_fs_locations *fsloc){return 0;} +static inline int +secinfo_parse(char **mesg, char *buf, struct svc_export *exp) { return 0; } +#endif + +static int xprtsec_parse(char **mesg, char *buf, struct svc_export *exp) +{ + unsigned int i, mode, listsize; + int err; + + err = get_uint(mesg, &listsize); + if (err) + return err; + if (listsize > NFSEXP_XPRTSEC_NUM) + return -EINVAL; + + exp->ex_xprtsec_modes = 0; + for (i = 0; i < listsize; i++) { + err = get_uint(mesg, &mode); + if (err) + return err; + if (mode > NFSEXP_XPRTSEC_MTLS) + return -EINVAL; + exp->ex_xprtsec_modes |= mode; + } + return 0; +} + +static inline int +nfsd_uuid_parse(char **mesg, char *buf, unsigned char **puuid) +{ + int len; + + /* more than one uuid */ + if (*puuid) + return -EINVAL; + + /* expect a 16 byte uuid encoded as \xXXXX... */ + len = qword_get(mesg, buf, PAGE_SIZE); + if (len != EX_UUID_LEN) + return -EINVAL; + + *puuid = kmemdup(buf, EX_UUID_LEN, GFP_KERNEL); + if (*puuid == NULL) + return -ENOMEM; + + return 0; +} + +static int svc_export_parse(struct cache_detail *cd, char *mesg, int mlen) +{ + /* client path expiry [flags anonuid anongid fsid] */ + char *buf; + int err; + struct auth_domain *dom = NULL; + struct svc_export exp = {}, *expp; + int an_int; + + if (mesg[mlen-1] != '\n') + return -EINVAL; + mesg[mlen-1] = 0; + + buf = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + /* client */ + err = -EINVAL; + if (qword_get(&mesg, buf, PAGE_SIZE) <= 0) + goto out; + + err = -ENOENT; + dom = auth_domain_find(buf); + if (!dom) + goto out; + + /* path */ + err = -EINVAL; + if (qword_get(&mesg, buf, PAGE_SIZE) <= 0) + goto out1; + + err = kern_path(buf, 0, &exp.ex_path); + if (err) + goto out1; + + exp.ex_client = dom; + exp.cd = cd; + exp.ex_devid_map = NULL; + exp.ex_xprtsec_modes = NFSEXP_XPRTSEC_ALL; + + /* expiry */ + err = get_expiry(&mesg, &exp.h.expiry_time); + if (err) + goto out3; + + /* flags */ + err = get_int(&mesg, &an_int); + if (err == -ENOENT) { + err = 0; + set_bit(CACHE_NEGATIVE, &exp.h.flags); + } else { + if (err || an_int < 0) + goto out3; + exp.ex_flags= an_int; + + /* anon uid */ + err = get_int(&mesg, &an_int); + if (err) + goto out3; + exp.ex_anon_uid= make_kuid(current_user_ns(), an_int); + + /* anon gid */ + err = get_int(&mesg, &an_int); + if (err) + goto out3; + exp.ex_anon_gid= make_kgid(current_user_ns(), an_int); + + /* fsid */ + err = get_int(&mesg, &an_int); + if (err) + goto out3; + exp.ex_fsid = an_int; + + while (qword_get(&mesg, buf, PAGE_SIZE) > 0) { + if (strcmp(buf, "fsloc") == 0) + err = fsloc_parse(&mesg, buf, &exp.ex_fslocs); + else if (strcmp(buf, "uuid") == 0) + err = nfsd_uuid_parse(&mesg, buf, &exp.ex_uuid); + else if (strcmp(buf, "secinfo") == 0) + err = secinfo_parse(&mesg, buf, &exp); + else if (strcmp(buf, "xprtsec") == 0) + err = xprtsec_parse(&mesg, buf, &exp); + else + /* quietly ignore unknown words and anything + * following. Newer user-space can try to set + * new values, then see what the result was. + */ + break; + if (err) + goto out4; + } + + err = check_export(&exp.ex_path, &exp.ex_flags, exp.ex_uuid); + if (err) + goto out4; + + /* + * No point caching this if it would immediately expire. + * Also, this protects exportfs's dummy export from the + * anon_uid/anon_gid checks: + */ + if (exp.h.expiry_time < seconds_since_boot()) + goto out4; + /* + * For some reason exportfs has been passing down an + * invalid (-1) uid & gid on the "dummy" export which it + * uses to test export support. To make sure exportfs + * sees errors from check_export we therefore need to + * delay these checks till after check_export: + */ + err = -EINVAL; + if (!uid_valid(exp.ex_anon_uid)) + goto out4; + if (!gid_valid(exp.ex_anon_gid)) + goto out4; + err = 0; + + if (exp.ex_flags & NFSEXP_PNFS) + nfsd4_setup_layout_type(&exp); + } + + expp = svc_export_lookup(&exp); + if (!expp) { + err = -ENOMEM; + goto out4; + } + expp = svc_export_update(&exp, expp); + if (expp) { + trace_nfsd_export_update(expp); + cache_flush(); + exp_put(expp); + } else + err = -ENOMEM; +out4: + nfsd4_fslocs_free(&exp.ex_fslocs); + kfree(exp.ex_uuid); +out3: + path_put(&exp.ex_path); +out1: + auth_domain_put(dom); +out: + kfree(buf); + return err; +} + +static void exp_flags(struct seq_file *m, int flag, int fsid, + kuid_t anonu, kgid_t anong, struct nfsd4_fs_locations *fslocs); +static void show_secinfo(struct seq_file *m, struct svc_export *exp); + +static int is_export_stats_file(struct seq_file *m) +{ + /* + * The export_stats file uses the same ops as the exports file. + * We use the file's name to determine the reported info per export. + * There is no rename in nsfdfs, so d_name.name is stable. + */ + return !strcmp(m->file->f_path.dentry->d_name.name, "export_stats"); +} + +static int svc_export_show(struct seq_file *m, + struct cache_detail *cd, + struct cache_head *h) +{ + struct svc_export *exp; + bool export_stats = is_export_stats_file(m); + + if (h == NULL) { + if (export_stats) + seq_puts(m, "#path domain start-time\n#\tstats\n"); + else + seq_puts(m, "#path domain(flags)\n"); + return 0; + } + exp = container_of(h, struct svc_export, h); + seq_path(m, &exp->ex_path, " \t\n\\"); + seq_putc(m, '\t'); + seq_escape(m, exp->ex_client->name, " \t\n\\"); + if (export_stats) { + struct percpu_counter *counter = exp->ex_stats->counter; + + seq_printf(m, "\t%lld\n", exp->ex_stats->start_time); + seq_printf(m, "\tfh_stale: %lld\n", + percpu_counter_sum_positive(&counter[EXP_STATS_FH_STALE])); + seq_printf(m, "\tio_read: %lld\n", + percpu_counter_sum_positive(&counter[EXP_STATS_IO_READ])); + seq_printf(m, "\tio_write: %lld\n", + percpu_counter_sum_positive(&counter[EXP_STATS_IO_WRITE])); + seq_putc(m, '\n'); + return 0; + } + seq_putc(m, '('); + if (test_bit(CACHE_VALID, &h->flags) && + !test_bit(CACHE_NEGATIVE, &h->flags)) { + exp_flags(m, exp->ex_flags, exp->ex_fsid, + exp->ex_anon_uid, exp->ex_anon_gid, &exp->ex_fslocs); + if (exp->ex_uuid) { + int i; + seq_puts(m, ",uuid="); + for (i = 0; i < EX_UUID_LEN; i++) { + if ((i&3) == 0 && i) + seq_putc(m, ':'); + seq_printf(m, "%02x", exp->ex_uuid[i]); + } + } + show_secinfo(m, exp); + } + seq_puts(m, ")\n"); + return 0; +} +static int svc_export_match(struct cache_head *a, struct cache_head *b) +{ + struct svc_export *orig = container_of(a, struct svc_export, h); + struct svc_export *new = container_of(b, struct svc_export, h); + return orig->ex_client == new->ex_client && + path_equal(&orig->ex_path, &new->ex_path); +} + +static void svc_export_init(struct cache_head *cnew, struct cache_head *citem) +{ + struct svc_export *new = container_of(cnew, struct svc_export, h); + struct svc_export *item = container_of(citem, struct svc_export, h); + + kref_get(&item->ex_client->ref); + new->ex_client = item->ex_client; + new->ex_path = item->ex_path; + path_get(&item->ex_path); + new->ex_fslocs.locations = NULL; + new->ex_fslocs.locations_count = 0; + new->ex_fslocs.migrated = 0; + new->ex_layout_types = 0; + new->ex_uuid = NULL; + new->cd = item->cd; + export_stats_reset(new->ex_stats); +} + +static void export_update(struct cache_head *cnew, struct cache_head *citem) +{ + struct svc_export *new = container_of(cnew, struct svc_export, h); + struct svc_export *item = container_of(citem, struct svc_export, h); + int i; + + new->ex_flags = item->ex_flags; + new->ex_anon_uid = item->ex_anon_uid; + new->ex_anon_gid = item->ex_anon_gid; + new->ex_fsid = item->ex_fsid; + new->ex_devid_map = item->ex_devid_map; + item->ex_devid_map = NULL; + new->ex_uuid = item->ex_uuid; + item->ex_uuid = NULL; + new->ex_fslocs.locations = item->ex_fslocs.locations; + item->ex_fslocs.locations = NULL; + new->ex_fslocs.locations_count = item->ex_fslocs.locations_count; + item->ex_fslocs.locations_count = 0; + new->ex_fslocs.migrated = item->ex_fslocs.migrated; + item->ex_fslocs.migrated = 0; + new->ex_layout_types = item->ex_layout_types; + new->ex_nflavors = item->ex_nflavors; + for (i = 0; i < MAX_SECINFO_LIST; i++) { + new->ex_flavors[i] = item->ex_flavors[i]; + } + new->ex_xprtsec_modes = item->ex_xprtsec_modes; +} + +static struct cache_head *svc_export_alloc(void) +{ + struct svc_export *i = kmalloc_obj(*i); + if (!i) + return NULL; + + i->ex_stats = kmalloc_obj(*(i->ex_stats)); + if (!i->ex_stats) { + kfree(i); + return NULL; + } + + if (export_stats_init(i->ex_stats)) { + kfree(i->ex_stats); + kfree(i); + return NULL; + } + + return &i->h; +} + +static const struct cache_detail svc_export_cache_template = { + .owner = THIS_MODULE, + .hash_size = EXPORT_HASHMAX, + .name = "nfsd.export", + .cache_put = svc_export_put, + .cache_upcall = svc_export_upcall, + .cache_notify = svc_export_notify, + .cache_request = svc_export_request, + .cache_parse = svc_export_parse, + .cache_show = svc_export_show, + .match = svc_export_match, + .init = svc_export_init, + .update = export_update, + .alloc = svc_export_alloc, +}; + +static int +svc_export_hash(struct svc_export *exp) +{ + int hash; + + hash = hash_ptr(exp->ex_client, EXPORT_HASHBITS); + hash ^= hash_ptr(exp->ex_path.dentry, EXPORT_HASHBITS); + hash ^= hash_ptr(exp->ex_path.mnt, EXPORT_HASHBITS); + return hash; +} + +static struct svc_export * +svc_export_lookup(struct svc_export *exp) +{ + struct cache_head *ch; + int hash = svc_export_hash(exp); + + ch = sunrpc_cache_lookup_rcu(exp->cd, &exp->h, hash); + if (ch) + return container_of(ch, struct svc_export, h); + else + return NULL; +} + +static struct svc_export * +svc_export_update(struct svc_export *new, struct svc_export *old) +{ + struct cache_head *ch; + int hash = svc_export_hash(old); + + ch = sunrpc_cache_update(old->cd, &new->h, &old->h, hash); + if (ch) + return container_of(ch, struct svc_export, h); + else + return NULL; +} + + +static struct svc_expkey * +exp_find_key(struct cache_detail *cd, struct auth_domain *clp, int fsid_type, + u32 *fsidv, struct cache_req *reqp) +{ + struct svc_expkey key, *ek; + int err; + + if (!clp) + return ERR_PTR(-ENOENT); + + key.ek_client = clp; + key.ek_fsidtype = fsid_type; + memcpy(key.ek_fsid, fsidv, key_len(fsid_type)); + + ek = svc_expkey_lookup(cd, &key); + if (ek == NULL) + return ERR_PTR(-ENOMEM); + err = cache_check(cd, &ek->h, reqp); + if (err) { + trace_nfsd_exp_find_key(&key, err); + return ERR_PTR(err); + } + return ek; +} + +static struct svc_export * +exp_get_by_name(struct cache_detail *cd, struct auth_domain *clp, + const struct path *path, struct cache_req *reqp) +{ + struct svc_export *exp, key; + int err; + + if (!clp) + return ERR_PTR(-ENOENT); + + key.ex_client = clp; + key.ex_path = *path; + key.cd = cd; + + exp = svc_export_lookup(&key); + if (exp == NULL) + return ERR_PTR(-ENOMEM); + err = cache_check(cd, &exp->h, reqp); + if (err) { + trace_nfsd_exp_get_by_name(&key, err); + return ERR_PTR(err); + } + return exp; +} + +/* + * Find the export entry for a given dentry. + */ +static struct svc_export * +exp_parent(struct cache_detail *cd, struct auth_domain *clp, struct path *path) +{ + struct dentry *saved = dget(path->dentry); + struct svc_export *exp = exp_get_by_name(cd, clp, path, NULL); + + while (PTR_ERR(exp) == -ENOENT && !IS_ROOT(path->dentry)) { + struct dentry *parent = dget_parent(path->dentry); + dput(path->dentry); + path->dentry = parent; + exp = exp_get_by_name(cd, clp, path, NULL); + } + dput(path->dentry); + path->dentry = saved; + return exp; +} + + + +/* + * Obtain the root fh on behalf of a client. + * This could be done in user space, but I feel that it adds some safety + * since its harder to fool a kernel module than a user space program. + */ +int +exp_rootfh(struct net *net, struct auth_domain *clp, char *name, + struct knfsd_fh *f, int maxsize) +{ + struct svc_export *exp; + struct path path; + struct inode *inode __maybe_unused; + struct svc_fh fh; + int err; + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct cache_detail *cd = nn->svc_export_cache; + + err = -EPERM; + /* NB: we probably ought to check that it's NUL-terminated */ + if (kern_path(name, 0, &path)) { + printk("nfsd: exp_rootfh path not found %s", name); + return err; + } + inode = d_inode(path.dentry); + + dprintk("nfsd: exp_rootfh(%s [%p] %s:%s/%llu)\n", + name, path.dentry, clp->name, + inode->i_sb->s_id, inode->i_ino); + exp = exp_parent(cd, clp, &path); + if (IS_ERR(exp)) { + err = PTR_ERR(exp); + goto out; + } + + /* + * fh must be initialized before calling fh_compose + */ + fh_init(&fh, maxsize); + if (fh_compose(&fh, exp, path.dentry, NULL)) + err = -EINVAL; + else + err = 0; + memcpy(f, &fh.fh_handle, sizeof(struct knfsd_fh)); + fh_put(&fh); + exp_put(exp); +out: + path_put(&path); + return err; +} + +static struct svc_export *exp_find(struct cache_detail *cd, + struct auth_domain *clp, int fsid_type, + u32 *fsidv, struct cache_req *reqp) +{ + struct svc_export *exp; + struct nfsd_net *nn = net_generic(cd->net, nfsd_net_id); + struct svc_expkey *ek = exp_find_key(nn->svc_expkey_cache, clp, fsid_type, fsidv, reqp); + if (IS_ERR(ek)) + return ERR_CAST(ek); + + exp = exp_get_by_name(cd, clp, &ek->ek_path, reqp); + cache_put(&ek->h, nn->svc_expkey_cache); + + if (IS_ERR(exp)) + return ERR_CAST(exp); + return exp; +} + +/** + * check_xprtsec_policy - check if access to export is allowed by the + * xprtsec policy + * @exp: svc_export that is being accessed. + * @rqstp: svc_rqst attempting to access @exp. + * + * Helper function for check_nfsd_access(). Note that callers should be + * using check_nfsd_access() instead of calling this function directly. The + * one exception is __fh_verify() since it has logic that may result in one + * or both of the helpers being skipped. + * + * Return values: + * %nfs_ok if access is granted, or + * %nfserr_wrongsec if access is denied + */ +__be32 check_xprtsec_policy(struct svc_export *exp, struct svc_rqst *rqstp) +{ + struct svc_xprt *xprt = rqstp->rq_xprt; + + if (exp->ex_xprtsec_modes & NFSEXP_XPRTSEC_NONE) { + if (!test_bit(XPT_TLS_SESSION, &xprt->xpt_flags)) + return nfs_ok; + } + if (exp->ex_xprtsec_modes & NFSEXP_XPRTSEC_TLS) { + if (test_bit(XPT_TLS_SESSION, &xprt->xpt_flags) && + !test_bit(XPT_PEER_AUTH, &xprt->xpt_flags)) + return nfs_ok; + } + if (exp->ex_xprtsec_modes & NFSEXP_XPRTSEC_MTLS) { + if (test_bit(XPT_TLS_SESSION, &xprt->xpt_flags) && + test_bit(XPT_PEER_AUTH, &xprt->xpt_flags)) + return nfs_ok; + } + return nfserr_wrongsec; +} + +/** + * check_security_flavor - check if access to export is allowed by the + * security flavor + * @exp: svc_export that is being accessed. + * @rqstp: svc_rqst attempting to access @exp. + * @may_bypass_gss: reduce strictness of authorization check + * + * Helper function for check_nfsd_access(). Note that callers should be + * using check_nfsd_access() instead of calling this function directly. The + * one exception is __fh_verify() since it has logic that may result in one + * or both of the helpers being skipped. + * + * Return values: + * %nfs_ok if access is granted, or + * %nfserr_wrongsec if access is denied + */ +__be32 check_security_flavor(struct svc_export *exp, struct svc_rqst *rqstp, + bool may_bypass_gss) +{ + struct exp_flavor_info *f, *end = exp->ex_flavors + exp->ex_nflavors; + + /* legacy gss-only clients are always OK: */ + if (exp->ex_client == rqstp->rq_gssclient) + return nfs_ok; + /* ip-address based client; check sec= export option: */ + for (f = exp->ex_flavors; f < end; f++) { + if (f->pseudoflavor == rqstp->rq_cred.cr_flavor) + return nfs_ok; + } + /* defaults in absence of sec= options: */ + if (exp->ex_nflavors == 0) { + if (rqstp->rq_cred.cr_flavor == RPC_AUTH_NULL || + rqstp->rq_cred.cr_flavor == RPC_AUTH_UNIX) + return nfs_ok; + } + + /* If the compound op contains a spo_must_allowed op, + * it will be sent with integrity/protection which + * will have to be expressly allowed on mounts that + * don't support it + */ + + if (nfsd4_spo_must_allow(rqstp)) + return nfs_ok; + + /* Some calls may be processed without authentication + * on GSS exports. For example NFS2/3 calls on root + * directory, see section 2.3.2 of rfc 2623. + * For "may_bypass_gss" check that export has really + * enabled some flavor with authentication (GSS or any + * other) and also check that the used auth flavor is + * without authentication (none or sys). + */ + if (may_bypass_gss && ( + rqstp->rq_cred.cr_flavor == RPC_AUTH_NULL || + rqstp->rq_cred.cr_flavor == RPC_AUTH_UNIX)) { + for (f = exp->ex_flavors; f < end; f++) { + if (f->pseudoflavor >= RPC_AUTH_DES) + return 0; + } + } + + return nfserr_wrongsec; +} + +/** + * check_nfsd_access - check if access to export is allowed. + * @exp: svc_export that is being accessed. + * @rqstp: svc_rqst attempting to access @exp. + * @may_bypass_gss: reduce strictness of authorization check + * + * Return values: + * %nfs_ok if access is granted, or + * %nfserr_wrongsec if access is denied + */ +__be32 check_nfsd_access(struct svc_export *exp, struct svc_rqst *rqstp, + bool may_bypass_gss) +{ + __be32 status; + + status = check_xprtsec_policy(exp, rqstp); + if (status != nfs_ok) + return status; + return check_security_flavor(exp, rqstp, may_bypass_gss); +} + +/* + * Uses rq_client and rq_gssclient to find an export; uses rq_client (an + * auth_unix client) if it's available and has secinfo information; + * otherwise, will try to use rq_gssclient. + * + * Called from functions that handle requests; functions that do work on + * behalf of mountd are passed a single client name to use, and should + * use exp_get_by_name() or exp_find(). + */ +struct svc_export * +rqst_exp_get_by_name(struct svc_rqst *rqstp, const struct path *path) +{ + struct svc_export *gssexp, *exp = ERR_PTR(-ENOENT); + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + struct cache_detail *cd = nn->svc_export_cache; + + if (rqstp->rq_client == NULL) + goto gss; + + /* First try the auth_unix client: */ + exp = exp_get_by_name(cd, rqstp->rq_client, path, &rqstp->rq_chandle); + if (PTR_ERR(exp) == -ENOENT) + goto gss; + if (IS_ERR(exp)) + return exp; + /* If it has secinfo, assume there are no gss/... clients */ + if (exp->ex_nflavors > 0) + return exp; +gss: + /* Otherwise, try falling back on gss client */ + if (rqstp->rq_gssclient == NULL) + return exp; + gssexp = exp_get_by_name(cd, rqstp->rq_gssclient, path, &rqstp->rq_chandle); + if (PTR_ERR(gssexp) == -ENOENT) + return exp; + if (!IS_ERR(exp)) + exp_put(exp); + return gssexp; +} + +/** + * rqst_exp_find - Find an svc_export in the context of a rqst or similar + * @reqp: The handle to be used to suspend the request if a cache-upcall is needed + * If NULL, missing in-cache information will result in failure. + * @net: The network namespace in which the request exists + * @cl: default auth_domain to use for looking up the export + * @gsscl: an alternate auth_domain defined using deprecated gss/krb5 format. + * @fsid_type: The type of fsid to look for + * @fsidv: The actual fsid to look up in the context of either client. + * + * Perform a lookup for @cl/@fsidv in the given @net for an export. If + * none found and @gsscl specified, repeat the lookup. + * + * Returns an export, or an error pointer. + */ +struct svc_export * +rqst_exp_find(struct cache_req *reqp, struct net *net, + struct auth_domain *cl, struct auth_domain *gsscl, + int fsid_type, u32 *fsidv) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct svc_export *gssexp, *exp = ERR_PTR(-ENOENT); + struct cache_detail *cd = nn->svc_export_cache; + + if (!cl) + goto gss; + + /* First try the auth_unix client: */ + exp = exp_find(cd, cl, fsid_type, fsidv, reqp); + if (PTR_ERR(exp) == -ENOENT) + goto gss; + if (IS_ERR(exp)) + return exp; + /* If it has secinfo, assume there are no gss/... clients */ + if (exp->ex_nflavors > 0) + return exp; +gss: + /* Otherwise, try falling back on gss client */ + if (!gsscl) + return exp; + gssexp = exp_find(cd, gsscl, fsid_type, fsidv, reqp); + if (PTR_ERR(gssexp) == -ENOENT) + return exp; + if (!IS_ERR(exp)) + exp_put(exp); + return gssexp; +} + +struct svc_export * +rqst_exp_parent(struct svc_rqst *rqstp, struct path *path) +{ + struct dentry *saved = dget(path->dentry); + struct svc_export *exp = rqst_exp_get_by_name(rqstp, path); + + while (PTR_ERR(exp) == -ENOENT && !IS_ROOT(path->dentry)) { + struct dentry *parent = dget_parent(path->dentry); + dput(path->dentry); + path->dentry = parent; + exp = rqst_exp_get_by_name(rqstp, path); + } + dput(path->dentry); + path->dentry = saved; + return exp; +} + +struct svc_export *rqst_find_fsidzero_export(struct svc_rqst *rqstp) +{ + u32 fsidv[2]; + + mk_fsid(FSID_NUM, fsidv, 0, 0, 0, NULL); + + return rqst_exp_find(&rqstp->rq_chandle, SVC_NET(rqstp), + rqstp->rq_client, rqstp->rq_gssclient, + FSID_NUM, fsidv); +} + +/* + * Called when we need the filehandle for the root of the pseudofs, + * for a given NFSv4 client. The root is defined to be the + * export point with fsid==0 + */ +__be32 +exp_pseudoroot(struct svc_rqst *rqstp, struct svc_fh *fhp) +{ + struct svc_export *exp; + __be32 rv; + + exp = rqst_find_fsidzero_export(rqstp); + if (IS_ERR(exp)) + return nfserrno(PTR_ERR(exp)); + rv = fh_compose(fhp, exp, exp->ex_path.dentry, NULL); + exp_put(exp); + return rv; +} + +static struct flags { + int flag; + char *name[2]; +} expflags[] = { + { NFSEXP_READONLY, {"ro", "rw"}}, + { NFSEXP_INSECURE_PORT, {"insecure", ""}}, + { NFSEXP_ROOTSQUASH, {"root_squash", "no_root_squash"}}, + { NFSEXP_ALLSQUASH, {"all_squash", ""}}, + { NFSEXP_ASYNC, {"async", "sync"}}, + { NFSEXP_GATHERED_WRITES, {"wdelay", "no_wdelay"}}, + { NFSEXP_NOREADDIRPLUS, {"nordirplus", ""}}, + { NFSEXP_SECURITY_LABEL, {"security_label", ""}}, + { NFSEXP_SIGN_FH, {"sign_fh", ""}}, + { NFSEXP_NOHIDE, {"nohide", ""}}, + { NFSEXP_NOSUBTREECHECK, {"no_subtree_check", ""}}, + { NFSEXP_NOAUTHNLM, {"insecure_locks", ""}}, + { NFSEXP_CROSSMOUNT, {"crossmnt", ""}}, + { NFSEXP_V4ROOT, {"v4root", ""}}, + { NFSEXP_PNFS, {"pnfs", ""}}, + { 0, {"", ""}} +}; + +static void show_expflags(struct seq_file *m, int flags, int mask) +{ + struct flags *flg; + int state, first = 0; + + for (flg = expflags; flg->flag; flg++) { + if (flg->flag & ~mask) + continue; + state = (flg->flag & flags) ? 0 : 1; + if (*flg->name[state]) + seq_printf(m, "%s%s", first++?",":"", flg->name[state]); + } +} + +static void show_secinfo_flags(struct seq_file *m, int flags) +{ + seq_printf(m, ","); + show_expflags(m, flags, NFSEXP_SECINFO_FLAGS); +} + +static bool secinfo_flags_equal(int f, int g) +{ + f &= NFSEXP_SECINFO_FLAGS; + g &= NFSEXP_SECINFO_FLAGS; + return f == g; +} + +static int show_secinfo_run(struct seq_file *m, struct exp_flavor_info **fp, struct exp_flavor_info *end) +{ + int flags; + + flags = (*fp)->flags; + seq_printf(m, ",sec=%d", (*fp)->pseudoflavor); + (*fp)++; + while (*fp != end && secinfo_flags_equal(flags, (*fp)->flags)) { + seq_printf(m, ":%d", (*fp)->pseudoflavor); + (*fp)++; + } + return flags; +} + +static void show_secinfo(struct seq_file *m, struct svc_export *exp) +{ + struct exp_flavor_info *f; + struct exp_flavor_info *end = exp->ex_flavors + exp->ex_nflavors; + int flags; + + if (exp->ex_nflavors == 0) + return; + f = exp->ex_flavors; + flags = show_secinfo_run(m, &f, end); + if (!secinfo_flags_equal(flags, exp->ex_flags)) + show_secinfo_flags(m, flags); + while (f != end) { + flags = show_secinfo_run(m, &f, end); + show_secinfo_flags(m, flags); + } +} + +static void exp_flags(struct seq_file *m, int flag, int fsid, + kuid_t anonu, kgid_t anong, struct nfsd4_fs_locations *fsloc) +{ + struct user_namespace *userns = m->file->f_cred->user_ns; + + show_expflags(m, flag, NFSEXP_ALLFLAGS); + if (flag & NFSEXP_FSID) + seq_printf(m, ",fsid=%d", fsid); + if (!uid_eq(anonu, make_kuid(userns, (uid_t)-2)) && + !uid_eq(anonu, make_kuid(userns, 0x10000-2))) + seq_printf(m, ",anonuid=%u", from_kuid_munged(userns, anonu)); + if (!gid_eq(anong, make_kgid(userns, (gid_t)-2)) && + !gid_eq(anong, make_kgid(userns, 0x10000-2))) + seq_printf(m, ",anongid=%u", from_kgid_munged(userns, anong)); + if (fsloc && fsloc->locations_count > 0) { + char *loctype = (fsloc->migrated) ? "refer" : "replicas"; + int i; + + seq_printf(m, ",%s=", loctype); + seq_escape(m, fsloc->locations[0].path, ",;@ \t\n\\"); + seq_putc(m, '@'); + seq_escape(m, fsloc->locations[0].hosts, ",;@ \t\n\\"); + for (i = 1; i < fsloc->locations_count; i++) { + seq_putc(m, ';'); + seq_escape(m, fsloc->locations[i].path, ",;@ \t\n\\"); + seq_putc(m, '@'); + seq_escape(m, fsloc->locations[i].hosts, ",;@ \t\n\\"); + } + } +} + +static int e_show(struct seq_file *m, void *p) +{ + struct cache_head *cp = p; + struct svc_export *exp = container_of(cp, struct svc_export, h); + struct cache_detail *cd = m->private; + bool export_stats = is_export_stats_file(m); + + if (p == SEQ_START_TOKEN) { + seq_puts(m, "# Version 1.1\n"); + if (export_stats) + seq_puts(m, "# Path Client Start-time\n#\tStats\n"); + else + seq_puts(m, "# Path Client(Flags) # IPs\n"); + return 0; + } + + if (cache_check_rcu(cd, &exp->h, NULL)) + return 0; + + return svc_export_show(m, cd, cp); +} + +const struct seq_operations nfs_exports_op = { + .start = cache_seq_start_rcu, + .next = cache_seq_next_rcu, + .stop = cache_seq_stop_rcu, + .show = e_show, +}; + +/* + * Initialize the exports module. + */ +int +nfsd_export_init(struct net *net) +{ + int rv; + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + dprintk("nfsd: initializing export module (net: %x).\n", net->ns.inum); + + nn->svc_export_cache = cache_create_net(&svc_export_cache_template, net); + if (IS_ERR(nn->svc_export_cache)) + return PTR_ERR(nn->svc_export_cache); + rv = cache_register_net(nn->svc_export_cache, net); + if (rv) + goto destroy_export_cache; + + nn->svc_expkey_cache = cache_create_net(&svc_expkey_cache_template, net); + if (IS_ERR(nn->svc_expkey_cache)) { + rv = PTR_ERR(nn->svc_expkey_cache); + goto unregister_export_cache; + } + rv = cache_register_net(nn->svc_expkey_cache, net); + if (rv) + goto destroy_expkey_cache; + return 0; + +destroy_expkey_cache: + cache_destroy_net(nn->svc_expkey_cache, net); +unregister_export_cache: + cache_unregister_net(nn->svc_export_cache, net); +destroy_export_cache: + cache_destroy_net(nn->svc_export_cache, net); + return rv; +} + +/* + * Flush exports table - called when last nfsd thread is killed + */ +void +nfsd_export_flush(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + cache_purge(nn->svc_expkey_cache); + cache_purge(nn->svc_export_cache); +} + +/* + * Shutdown the exports module. + */ +void +nfsd_export_shutdown(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + dprintk("nfsd: shutting down export module (net: %x).\n", net->ns.inum); + + cache_unregister_net(nn->svc_expkey_cache, net); + cache_unregister_net(nn->svc_export_cache, net); + cache_destroy_net(nn->svc_expkey_cache, net); + cache_destroy_net(nn->svc_export_cache, net); + svcauth_unix_purge(net); + + dprintk("nfsd: export shutdown complete (net: %x).\n", net->ns.inum); +} diff --git a/fs/nfsd/export.h b/fs/nfsd/export.h new file mode 100644 index 000000000..d2b09cd76 --- /dev/null +++ b/fs/nfsd/export.h @@ -0,0 +1,139 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 1995-1997 Olaf Kirch + */ +#ifndef NFSD_EXPORT_H +#define NFSD_EXPORT_H + +#include +#include +#include +#include + +struct knfsd_fh; +struct svc_fh; +struct svc_rqst; + +/* + * FS Locations + */ + +#define MAX_FS_LOCATIONS 128 + +struct nfsd4_fs_location { + char *hosts; /* colon separated list of hosts */ + char *path; /* slash separated list of path components */ +}; + +struct nfsd4_fs_locations { + uint32_t locations_count; + struct nfsd4_fs_location *locations; +/* If we're not actually serving this data ourselves (only providing a + * list of replicas that do serve it) then we set "migrated": */ + int migrated; +}; + +/* + * We keep an array of pseudoflavors with the export, in order from most + * to least preferred. For the foreseeable future, we don't expect more + * than the eight pseudoflavors null, unix, krb5, krb5i, krb5p, skpm3, + * spkm3i, and spkm3p (and using all 8 at once should be rare). + */ +#define MAX_SECINFO_LIST 8 +#define EX_UUID_LEN 16 + +struct exp_flavor_info { + u32 pseudoflavor; + u32 flags; +}; + +/* Per-export stats */ +enum { + EXP_STATS_FH_STALE, + EXP_STATS_IO_READ, + EXP_STATS_IO_WRITE, + EXP_STATS_COUNTERS_NUM +}; + +struct export_stats { + time64_t start_time; + struct percpu_counter counter[EXP_STATS_COUNTERS_NUM]; +}; + +struct svc_export { + struct cache_head h; + struct auth_domain * ex_client; + int ex_flags; + int ex_fsid; + struct path ex_path; + kuid_t ex_anon_uid; + kgid_t ex_anon_gid; + unsigned char * ex_uuid; /* 16 byte fsid */ + struct nfsd4_fs_locations ex_fslocs; + uint32_t ex_nflavors; + struct exp_flavor_info ex_flavors[MAX_SECINFO_LIST]; + u32 ex_layout_types; + struct nfsd4_deviceid_map *ex_devid_map; + struct cache_detail *cd; + struct rcu_head ex_rcu; + unsigned long ex_xprtsec_modes; + struct export_stats *ex_stats; +}; + +/* an "export key" (expkey) maps a filehandlefragement to an + * svc_export for a given client. There can be several per export, + * for the different fsid types. + */ +struct svc_expkey { + struct cache_head h; + + struct auth_domain * ek_client; + u8 ek_fsidtype; + u32 ek_fsid[6]; + + struct path ek_path; + struct rcu_head ek_rcu; +}; + +#define EX_ISSYNC(exp) (!((exp)->ex_flags & NFSEXP_ASYNC)) +#define EX_NOHIDE(exp) ((exp)->ex_flags & NFSEXP_NOHIDE) +#define EX_WGATHER(exp) ((exp)->ex_flags & NFSEXP_GATHERED_WRITES) + +struct svc_cred; +int nfsexp_flags(struct svc_cred *cred, struct svc_export *exp); +__be32 check_xprtsec_policy(struct svc_export *exp, struct svc_rqst *rqstp); +__be32 check_security_flavor(struct svc_export *exp, struct svc_rqst *rqstp, + bool may_bypass_gss); +__be32 check_nfsd_access(struct svc_export *exp, struct svc_rqst *rqstp, + bool may_bypass_gss); + +/* + * Function declarations + */ +int nfsd_export_init(struct net *); +void nfsd_export_shutdown(struct net *); +void nfsd_export_flush(struct net *); +struct svc_export * rqst_exp_get_by_name(struct svc_rqst *, + const struct path *); +struct svc_export * rqst_exp_parent(struct svc_rqst *, + struct path *); +struct svc_export * rqst_find_fsidzero_export(struct svc_rqst *); +int exp_rootfh(struct net *, struct auth_domain *, + char *path, struct knfsd_fh *, int maxsize); +__be32 exp_pseudoroot(struct svc_rqst *, struct svc_fh *); + +static inline void exp_put(struct svc_export *exp) +{ + cache_put(&exp->h, exp->cd); +} + +static inline struct svc_export *exp_get(struct svc_export *exp) +{ + cache_get(&exp->h); + return exp; +} +struct svc_export *rqst_exp_find(struct cache_req *reqp, struct net *net, + struct auth_domain *cl, struct auth_domain *gsscl, + int fsid_type, u32 *fsidv); + +#endif /* NFSD_EXPORT_H */ diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c new file mode 100644 index 000000000..b9548eb17 --- /dev/null +++ b/fs/nfsd/filecache.c @@ -0,0 +1,1565 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * The NFSD open file cache. + * + * (c) 2015 - Jeff Layton + * + * An nfsd_file object is a per-file collection of open state that binds + * together: + * - a struct file * + * - a user credential + * - a network namespace + * - a read-ahead context + * - monitoring for writeback errors + * + * nfsd_file objects are reference-counted. Consumers acquire a new + * object via the nfsd_file_acquire API. They manage their interest in + * the acquired object, and hence the object's reference count, via + * nfsd_file_get and nfsd_file_put. There are two varieties of nfsd_file + * object: + * + * * non-garbage-collected: When a consumer wants to precisely control + * the lifetime of a file's open state, it acquires a non-garbage- + * collected nfsd_file. The final nfsd_file_put releases the open + * state immediately. + * + * * garbage-collected: When a consumer does not control the lifetime + * of open state, it acquires a garbage-collected nfsd_file. The + * final nfsd_file_put allows the open state to linger for a period + * during which it may be re-used. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "vfs.h" +#include "nfsd.h" +#include "nfsfh.h" +#include "netns.h" +#include "filecache.h" +#include "trace.h" + +#define NFSD_LAUNDRETTE_DELAY (2 * HZ) + +#define NFSD_FILE_CACHE_UP (0) + +/* We only care about NFSD_MAY_READ/WRITE for this cache */ +#define NFSD_FILE_MAY_MASK (NFSD_MAY_READ|NFSD_MAY_WRITE|NFSD_MAY_LOCALIO) + +/* If the shrinker runs between calls to list_lru_walk_node() in + * nfsd_file_gc(), the "remaining" count will be wrong. This could + * result in premature freeing of some files. This may not matter much + * but is easy to fix with this spinlock which temporarily disables + * the shrinker. + * + * It also serializes callers of nfsd_file_dispose_list_delayed() + * against per-net shutdown. + */ +static DEFINE_SPINLOCK(nfsd_gc_lock); + +static DEFINE_PER_CPU(unsigned long, nfsd_file_cache_hits); +static DEFINE_PER_CPU(unsigned long, nfsd_file_acquisitions); +static DEFINE_PER_CPU(unsigned long, nfsd_file_allocations); +static DEFINE_PER_CPU(unsigned long, nfsd_file_releases); +static DEFINE_PER_CPU(unsigned long, nfsd_file_total_age); +static DEFINE_PER_CPU(unsigned long, nfsd_file_evictions); + +static struct kmem_cache *nfsd_file_slab; +static struct kmem_cache *nfsd_file_mark_slab; +static struct list_lru nfsd_file_lru; +static unsigned long nfsd_file_flags; +static struct fsnotify_group *nfsd_file_fsnotify_group; +static struct fsnotify_group *nfsd_dir_fsnotify_group; +static struct delayed_work nfsd_filecache_laundrette; +static struct rhltable nfsd_file_rhltable + ____cacheline_aligned_in_smp; + +static bool +nfsd_match_cred(const struct cred *c1, const struct cred *c2) +{ + int i; + + if (!uid_eq(c1->fsuid, c2->fsuid)) + return false; + if (!gid_eq(c1->fsgid, c2->fsgid)) + return false; + if (c1->group_info == NULL || c2->group_info == NULL) + return c1->group_info == c2->group_info; + if (c1->group_info->ngroups != c2->group_info->ngroups) + return false; + for (i = 0; i < c1->group_info->ngroups; i++) { + if (!gid_eq(c1->group_info->gid[i], c2->group_info->gid[i])) + return false; + } + return true; +} + +static const struct rhashtable_params nfsd_file_rhash_params = { + .key_len = sizeof_field(struct nfsd_file, nf_inode), + .key_offset = offsetof(struct nfsd_file, nf_inode), + .head_offset = offsetof(struct nfsd_file, nf_rlist), + + /* + * Start with a single page hash table to reduce resizing churn + * on light workloads. + */ + .min_size = 256, + .automatic_shrinking = true, +}; + +static void +nfsd_file_schedule_laundrette(void) +{ + if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags)) + queue_delayed_work(system_dfl_wq, &nfsd_filecache_laundrette, + NFSD_LAUNDRETTE_DELAY); +} + +static void +nfsd_file_slab_free(struct rcu_head *rcu) +{ + struct nfsd_file *nf = container_of(rcu, struct nfsd_file, nf_rcu); + + put_cred(nf->nf_cred); + kmem_cache_free(nfsd_file_slab, nf); +} + +static void +nfsd_file_mark_free(struct fsnotify_mark *mark) +{ + struct nfsd_file_mark *nfm = container_of(mark, struct nfsd_file_mark, + nfm_mark); + + kmem_cache_free(nfsd_file_mark_slab, nfm); +} + +static struct nfsd_file_mark * +nfsd_file_mark_get(struct nfsd_file_mark *nfm) +{ + if (!refcount_inc_not_zero(&nfm->nfm_ref)) + return NULL; + return nfm; +} + +static void +nfsd_file_mark_put(struct nfsd_file_mark *nfm) +{ + if (refcount_dec_and_test(&nfm->nfm_ref)) { + fsnotify_destroy_mark(&nfm->nfm_mark, nfm->nfm_mark.group); + fsnotify_put_mark(&nfm->nfm_mark); + } +} + +static struct nfsd_file_mark * +nfsd_file_mark_find_or_create(struct inode *inode) +{ + struct nfsd_file_mark *nfm = NULL, *new; + struct fsnotify_group *group; + struct fsnotify_mark *mark; + int err; + + group = S_ISDIR(inode->i_mode) ? nfsd_dir_fsnotify_group : nfsd_file_fsnotify_group; + + do { + fsnotify_group_lock(group); + mark = fsnotify_find_inode_mark(inode, group); + if (mark) { + nfm = nfsd_file_mark_get(container_of(mark, + struct nfsd_file_mark, + nfm_mark)); + fsnotify_group_unlock(group); + if (nfm) { + fsnotify_put_mark(mark); + break; + } + /* Avoid soft lockup race with nfsd_file_mark_put() */ + fsnotify_destroy_mark(mark, group); + fsnotify_put_mark(mark); + } else { + fsnotify_group_unlock(group); + } + + /* allocate a new nfm */ + new = kmem_cache_alloc(nfsd_file_mark_slab, GFP_KERNEL); + if (!new) + return NULL; + fsnotify_init_mark(&new->nfm_mark, group); + new->nfm_mark.mask = FS_ATTRIB|FS_DELETE_SELF; + refcount_set(&new->nfm_ref, 1); + mutex_init(&new->nfm_recalc_mutex); + + err = fsnotify_add_inode_mark(&new->nfm_mark, inode, 0); + + /* + * If the add was successful, then return the object. + * Otherwise, we need to put the reference we hold on the + * nfm_mark. The fsnotify code will take a reference and put + * it on failure, so we can't just free it directly. It's also + * not safe to call fsnotify_destroy_mark on it as the + * mark->group will be NULL. Thus, we can't let the nfm_ref + * counter drive the destruction at this point. + */ + if (likely(!err)) + nfm = new; + else + fsnotify_put_mark(&new->nfm_mark); + } while (unlikely(err == -EEXIST)); + + return nfm; +} + +static struct nfsd_file * +nfsd_file_alloc(struct net *net, struct inode *inode, unsigned char need, + bool want_gc) +{ + struct nfsd_file *nf; + + nf = kmem_cache_alloc(nfsd_file_slab, GFP_KERNEL); + if (unlikely(!nf)) + return NULL; + + this_cpu_inc(nfsd_file_allocations); + INIT_LIST_HEAD(&nf->nf_lru); + INIT_LIST_HEAD(&nf->nf_gc); + nf->nf_birthtime = ktime_get(); + nf->nf_file = NULL; + nf->nf_cred = get_current_cred(); + nf->nf_net = net; + nf->nf_flags = want_gc ? + BIT(NFSD_FILE_HASHED) | BIT(NFSD_FILE_PENDING) | BIT(NFSD_FILE_GC) : + BIT(NFSD_FILE_HASHED) | BIT(NFSD_FILE_PENDING); + nf->nf_inode = inode; + refcount_set(&nf->nf_ref, 1); + nf->nf_may = need; + nf->nf_mark = NULL; + nf->nf_dio_mem_align = 0; + nf->nf_dio_offset_align = 0; + nf->nf_dio_read_offset_align = 0; + return nf; +} + +/** + * nfsd_file_check_write_error - check for writeback errors on a file + * @nf: nfsd_file to check for writeback errors + * + * Check whether a nfsd_file has an unseen error. Reset the write + * verifier if so. + */ +static void +nfsd_file_check_write_error(struct nfsd_file *nf) +{ + struct file *file = nf->nf_file; + + if ((file->f_mode & FMODE_WRITE) && + filemap_check_wb_err(file->f_mapping, READ_ONCE(file->f_wb_err))) + nfsd_reset_write_verifier(net_generic(nf->nf_net, nfsd_net_id)); +} + +static void +nfsd_file_hash_remove(struct nfsd_file *nf) +{ + trace_nfsd_file_unhash(nf); + rhltable_remove(&nfsd_file_rhltable, &nf->nf_rlist, + nfsd_file_rhash_params); +} + +static bool +nfsd_file_unhash(struct nfsd_file *nf) +{ + if (test_and_clear_bit(NFSD_FILE_HASHED, &nf->nf_flags)) { + nfsd_file_hash_remove(nf); + return true; + } + return false; +} + +static void +nfsd_file_free(struct nfsd_file *nf) +{ + s64 age = ktime_to_ms(ktime_sub(ktime_get(), nf->nf_birthtime)); + + trace_nfsd_file_free(nf); + + this_cpu_inc(nfsd_file_releases); + this_cpu_add(nfsd_file_total_age, age); + + nfsd_file_unhash(nf); + if (nf->nf_mark) + nfsd_file_mark_put(nf->nf_mark); + if (nf->nf_file) { + nfsd_file_check_write_error(nf); + nfsd_filp_close(nf->nf_file); + } + + /* + * If this item is still linked via nf_lru, that's a bug. + * WARN and leak it to preserve system stability. + */ + if (WARN_ON_ONCE(!list_empty(&nf->nf_lru))) + return; + + call_rcu(&nf->nf_rcu, nfsd_file_slab_free); +} + +static bool +nfsd_file_check_writeback(struct nfsd_file *nf) +{ + struct file *file = nf->nf_file; + struct address_space *mapping; + + /* File not open for write? */ + if (!(file->f_mode & FMODE_WRITE)) + return false; + + /* + * Some filesystems (e.g. NFS) flush all dirty data on close. + * On others, there is no need to wait for writeback. + */ + if (!(file_inode(file)->i_sb->s_export_op->flags & EXPORT_OP_FLUSH_ON_CLOSE)) + return false; + + mapping = file->f_mapping; + return mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) || + mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK); +} + +static void nfsd_file_lru_add(struct nfsd_file *nf) +{ + refcount_inc(&nf->nf_ref); + if (list_lru_add_obj(&nfsd_file_lru, &nf->nf_lru)) + trace_nfsd_file_lru_add(nf); + else { + refcount_dec(&nf->nf_ref); + WARN_ON_ONCE(1); + return; + } + nfsd_file_schedule_laundrette(); +} + +static bool nfsd_file_lru_remove(struct nfsd_file *nf) +{ + if (list_lru_del_obj(&nfsd_file_lru, &nf->nf_lru)) { + trace_nfsd_file_lru_del(nf); + return true; + } + return false; +} + +struct nfsd_file * +nfsd_file_get(struct nfsd_file *nf) +{ + if (nf && refcount_inc_not_zero(&nf->nf_ref)) + return nf; + return NULL; +} + +/** + * nfsd_file_put - put the reference to a nfsd_file + * @nf: nfsd_file of which to put the reference + * + * Put a reference to a nfsd_file. In the non-GC case, we just put the + * reference immediately. In the GC case, if the reference would be + * the last one, the put it on the LRU instead to be cleaned up later. + */ +void +nfsd_file_put(struct nfsd_file *nf) +{ + might_sleep(); + trace_nfsd_file_put(nf); + + if (test_bit(NFSD_FILE_GC, &nf->nf_flags) && + test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) { + set_bit(NFSD_FILE_REFERENCED, &nf->nf_flags); + set_bit(NFSD_FILE_RECENT, &nf->nf_flags); + } + + if (refcount_dec_and_test(&nf->nf_ref)) + nfsd_file_free(nf); +} + +/** + * nfsd_file_put_local - put nfsd_file reference and arm nfsd_net_put in caller + * @pnf: nfsd_file of which to put the reference + * + * First save the associated net to return to caller, then put + * the reference of the nfsd_file. + */ +struct net * +nfsd_file_put_local(struct nfsd_file __rcu **pnf) +{ + struct nfsd_file *nf; + struct net *net = NULL; + + nf = unrcu_pointer(xchg(pnf, NULL)); + if (nf) { + net = nf->nf_net; + nfsd_file_put(nf); + } + return net; +} + +/** + * nfsd_file_file - get the backing file of an nfsd_file + * @nf: nfsd_file of which to access the backing file. + * + * Return backing file for @nf. + */ +struct file * +nfsd_file_file(struct nfsd_file *nf) +{ + return nf->nf_file; +} + +static void +nfsd_file_dispose_list(struct list_head *dispose) +{ + struct nfsd_file *nf; + + while (!list_empty(dispose)) { + nf = list_first_entry(dispose, struct nfsd_file, nf_gc); + list_del_init(&nf->nf_gc); + nfsd_file_free(nf); + } +} + +/** + * nfsd_file_dispose_list_delayed - queue dead files for nfsd thread disposal + * @dispose: list of nfsd_files to be disposed + * + * Transfers each file to the dispose list in its nfsd_net and wakes an nfsd + * thread to do the actual close. This keeps the cost of fput() in the nfsd + * threads rather than in the shrinker or GC worker. + * + * All callers must hold nfsd_gc_lock, so that nfsd_file_cache_shutdown_net() + * can synchronize against them before draining the per-net dispose list. + * This guarantees nf_net is still live when we call net_generic(). + */ +static void +nfsd_file_dispose_list_delayed(struct list_head *dispose) +{ + lockdep_assert_held(&nfsd_gc_lock); + + while (!list_empty(dispose)) { + struct nfsd_file *nf = list_first_entry(dispose, + struct nfsd_file, nf_gc); + struct nfsd_net *nn = net_generic(nf->nf_net, nfsd_net_id); + struct svc_serv *serv; + + spin_lock(&nn->fcache_dispose_lock); + list_move_tail(&nf->nf_gc, &nn->fcache_dispose_list); + spin_unlock(&nn->fcache_dispose_lock); + + /* + * The filecache laundrette is shut down after the + * nn->nfsd_serv pointer is cleared, but before the + * svc_serv is freed. + */ + serv = nn->nfsd_serv; + if (serv) + svc_wake_up(serv); + } +} + +/** + * nfsd_file_net_dispose - deal with nfsd_files waiting to be disposed. + * @nn: nfsd_net in which to find files to be disposed. + * + * When files held open for nfsv3 are removed from the filecache, whether + * due to memory pressure or garbage collection, they are queued to + * a per-net-ns queue. This function completes the disposal, either + * directly or by waking another nfsd thread to help with the work. + */ +void nfsd_file_net_dispose(struct nfsd_net *nn) +{ + if (!list_empty(&nn->fcache_dispose_list)) { + LIST_HEAD(dispose); + int i; + + spin_lock(&nn->fcache_dispose_lock); + for (i = 0; i < 8 && !list_empty(&nn->fcache_dispose_list); i++) + list_move(nn->fcache_dispose_list.next, &dispose); + spin_unlock(&nn->fcache_dispose_lock); + if (!list_empty(&nn->fcache_dispose_list)) { + /* + * Wake up another thread to share the work + * *before* doing any actual disposing. + * + * The filecache laundrette is shut down after + * the nn->nfsd_serv pointer is cleared, but + * before the svc_serv is freed. + */ + struct svc_serv *serv = nn->nfsd_serv; + + if (serv) + svc_wake_up(serv); + } + nfsd_file_dispose_list(&dispose); + } +} + +/** + * nfsd_file_lru_cb - Examine an entry on the LRU list + * @item: LRU entry to examine + * @lru: controlling LRU + * @arg: dispose list + * + * Return values: + * %LRU_REMOVED: @item was removed from the LRU + * %LRU_ROTATE: @item is to be moved to the LRU tail + * %LRU_SKIP: @item cannot be evicted + */ +static enum lru_status +nfsd_file_lru_cb(struct list_head *item, struct list_lru_one *lru, + void *arg) +{ + struct list_head *head = arg; + struct nfsd_file *nf = list_entry(item, struct nfsd_file, nf_lru); + + /* We should only be dealing with GC entries here */ + WARN_ON_ONCE(!test_bit(NFSD_FILE_GC, &nf->nf_flags)); + + /* + * Don't throw out files that are still undergoing I/O or + * that have uncleared errors pending. + */ + if (nfsd_file_check_writeback(nf)) { + trace_nfsd_file_gc_writeback(nf); + return LRU_SKIP; + } + + /* If it was recently added to the list, skip it */ + if (test_and_clear_bit(NFSD_FILE_REFERENCED, &nf->nf_flags)) { + trace_nfsd_file_gc_referenced(nf); + return LRU_ROTATE; + } + + /* + * Put the reference held on behalf of the LRU if it is the last + * reference, else rotate. + */ + if (!refcount_dec_if_one(&nf->nf_ref)) { + trace_nfsd_file_gc_in_use(nf); + return LRU_ROTATE; + } + + /* Refcount went to zero. Unhash it and queue it to the dispose list */ + nfsd_file_unhash(nf); + list_lru_isolate(lru, &nf->nf_lru); + list_add(&nf->nf_gc, head); + this_cpu_inc(nfsd_file_evictions); + trace_nfsd_file_gc_disposed(nf); + return LRU_REMOVED; +} + +static enum lru_status +nfsd_file_gc_cb(struct list_head *item, struct list_lru_one *lru, + void *arg) +{ + struct nfsd_file *nf = list_entry(item, struct nfsd_file, nf_lru); + + if (test_and_clear_bit(NFSD_FILE_RECENT, &nf->nf_flags)) { + /* + * "REFERENCED" really means "should be at the end of the + * LRU. As we are putting it there we can clear the flag. + */ + clear_bit(NFSD_FILE_REFERENCED, &nf->nf_flags); + trace_nfsd_file_gc_aged(nf); + return LRU_ROTATE; + } + return nfsd_file_lru_cb(item, lru, arg); +} + +static void +nfsd_file_gc(void) +{ + unsigned long ret = 0; + LIST_HEAD(dispose); + int nid; + + spin_lock(&nfsd_gc_lock); + for_each_node_state(nid, N_NORMAL_MEMORY) { + unsigned long remaining = list_lru_count_node(&nfsd_file_lru, nid); + + while (remaining > 0) { + unsigned long nr = min(remaining, NFSD_FILE_GC_BATCH); + + remaining -= nr; + ret += list_lru_walk_node(&nfsd_file_lru, nid, nfsd_file_gc_cb, + &dispose, &nr); + if (nr) + /* walk aborted early */ + remaining = 0; + } + } + nfsd_file_dispose_list_delayed(&dispose); + spin_unlock(&nfsd_gc_lock); + trace_nfsd_file_gc_removed(ret, list_lru_count(&nfsd_file_lru)); +} + +static void +nfsd_file_gc_worker(struct work_struct *work) +{ + if (list_lru_count(&nfsd_file_lru)) + nfsd_file_gc(); + nfsd_file_schedule_laundrette(); +} + +static unsigned long +nfsd_file_lru_count(struct shrinker *s, struct shrink_control *sc) +{ + return list_lru_count(&nfsd_file_lru); +} + +static unsigned long +nfsd_file_lru_scan(struct shrinker *s, struct shrink_control *sc) +{ + LIST_HEAD(dispose); + unsigned long ret; + + if (!spin_trylock(&nfsd_gc_lock)) + return SHRINK_STOP; + + ret = list_lru_shrink_walk(&nfsd_file_lru, sc, + nfsd_file_lru_cb, &dispose); + nfsd_file_dispose_list_delayed(&dispose); + spin_unlock(&nfsd_gc_lock); + trace_nfsd_file_shrinker_removed(ret, list_lru_count(&nfsd_file_lru)); + return ret; +} + +static struct shrinker *nfsd_file_shrinker; + +/** + * nfsd_file_cond_queue - conditionally unhash and queue a nfsd_file + * @nf: nfsd_file to attempt to queue + * @dispose: private list to queue successfully-put objects + * + * Unhash an nfsd_file, try to get a reference to it, and then put that + * reference. If it's the last reference, queue it to the dispose list. + */ +static void +nfsd_file_cond_queue(struct nfsd_file *nf, struct list_head *dispose) + __must_hold(RCU) +{ + int decrement = 1; + + /* If we raced with someone else unhashing, ignore it */ + if (!nfsd_file_unhash(nf)) + return; + + /* If we can't get a reference, ignore it */ + if (!nfsd_file_get(nf)) + return; + + /* Extra decrement if we remove from the LRU */ + if (nfsd_file_lru_remove(nf)) + ++decrement; + + /* If refcount goes to 0, then put on the dispose list */ + if (refcount_sub_and_test(decrement, &nf->nf_ref)) { + list_add(&nf->nf_gc, dispose); + trace_nfsd_file_closing(nf); + } +} + +/** + * nfsd_file_queue_for_close: try to close out any open nfsd_files for an inode + * @inode: inode on which to close out nfsd_files + * @dispose: list on which to gather nfsd_files to close out + * + * An nfsd_file represents a struct file being held open on behalf of nfsd. + * An open file however can block other activity (such as leases), or cause + * undesirable behavior (e.g. spurious silly-renames when reexporting NFS). + * + * This function is intended to find open nfsd_files when this sort of + * conflicting access occurs and then attempt to close those files out. + * + * Populates the dispose list with entries that have already had their + * refcounts go to zero. The actual free of an nfsd_file can be expensive, + * so we leave it up to the caller whether it wants to wait or not. + */ +static void +nfsd_file_queue_for_close(struct inode *inode, struct list_head *dispose) +{ + struct rhlist_head *tmp, *list; + struct nfsd_file *nf; + + rcu_read_lock(); + list = rhltable_lookup(&nfsd_file_rhltable, &inode, + nfsd_file_rhash_params); + rhl_for_each_entry_rcu(nf, tmp, list, nf_rlist) { + if (!test_bit(NFSD_FILE_GC, &nf->nf_flags)) + continue; + nfsd_file_cond_queue(nf, dispose); + } + rcu_read_unlock(); +} + +/** + * nfsd_file_close_inode - attempt a deferred close of a nfsd_file + * @inode: inode of the file to attempt to remove + * + * Close out any open nfsd_files that can be reaped for @inode. The + * actual freeing is deferred to the nfsd service threads. + * + * This is used by the fsnotify callbacks and setlease notifier. + */ +static void +nfsd_file_close_inode(struct inode *inode) +{ + LIST_HEAD(dispose); + + spin_lock(&nfsd_gc_lock); + nfsd_file_queue_for_close(inode, &dispose); + nfsd_file_dispose_list_delayed(&dispose); + spin_unlock(&nfsd_gc_lock); +} + +/** + * nfsd_file_close_inode_sync - attempt to forcibly close a nfsd_file + * @inode: inode of the file to attempt to remove + * + * Close out any open nfsd_files that can be reaped for @inode. The + * nfsd_files are closed out synchronously. + * + * This is called from nfsd_rename and nfsd_unlink to avoid silly-renames + * when reexporting NFS. + */ +void +nfsd_file_close_inode_sync(struct inode *inode) +{ + LIST_HEAD(dispose); + + trace_nfsd_file_close(inode); + + nfsd_file_queue_for_close(inode, &dispose); + nfsd_file_dispose_list(&dispose); +} + +/** + * nfsd_file_close_export - close cached file handles for an export + * @net: net namespace in which to operate + * @path: export path whose cached files should be closed + * + * Close out GC-managed nfsd_file entries whose underlying file is on + * the same filesystem as, and a descendant of, @path. nfsd_file + * entries do not carry an export reference, so the check uses the + * file's dentry ancestry. False positives (closing a cached handle + * that did not originate from the target export) are harmless -- the + * handle is simply reopened on the next access. + * + * Called from the NFSD_CMD_UNLOCK_EXPORT handler before revoking + * NFSv4 state, to ensure that cached file handles do not hold the + * filesystem busy. + */ +void nfsd_file_close_export(struct net *net, const struct path *path) +{ + struct rhashtable_iter iter; + struct nfsd_file *nf; + LIST_HEAD(dispose); + + rhltable_walk_enter(&nfsd_file_rhltable, &iter); + do { + rhashtable_walk_start(&iter); + + nf = rhashtable_walk_next(&iter); + while (!IS_ERR_OR_NULL(nf)) { + if (nf->nf_net == net && + test_bit(NFSD_FILE_GC, &nf->nf_flags) && + nf->nf_file && + file_inode(nf->nf_file)->i_sb == + path->dentry->d_sb && + is_subdir(nf->nf_file->f_path.dentry, + path->dentry)) + nfsd_file_cond_queue(nf, &dispose); + nf = rhashtable_walk_next(&iter); + } + + rhashtable_walk_stop(&iter); + } while (nf == ERR_PTR(-EAGAIN)); + rhashtable_walk_exit(&iter); + + nfsd_file_dispose_list(&dispose); +} + +static int +nfsd_file_lease_notifier_call(struct notifier_block *nb, unsigned long arg, + void *data) +{ + struct file_lease *fl = data; + + /* Only close files for F_SETLEASE leases */ + if (fl->c.flc_flags & FL_LEASE) + nfsd_file_close_inode(file_inode(fl->c.flc_file)); + return 0; +} + +static struct notifier_block nfsd_file_lease_notifier = { + .notifier_call = nfsd_file_lease_notifier_call, +}; + +static int +nfsd_file_fsnotify_handle_event(struct fsnotify_mark *mark, u32 mask, + struct inode *inode, struct inode *dir, + const struct qstr *name, u32 cookie) +{ + if (WARN_ON_ONCE(!inode)) + return 0; + + trace_nfsd_file_fsnotify_handle_event(inode, mask); + + /* Should be no marks on non-regular files */ + if (!S_ISREG(inode->i_mode)) { + WARN_ON_ONCE(1); + return 0; + } + + /* don't close files if this was not the last link */ + if (mask & FS_ATTRIB) { + if (inode->i_nlink) + return 0; + } + + nfsd_file_close_inode(inode); + return 0; +} + +#ifdef CONFIG_NFSD_V4 +static int +nfsd_dir_fsnotify_handle_event(struct fsnotify_group *group, u32 mask, + const void *data, int data_type, struct inode *dir, + const struct qstr *name, u32 cookie, + struct fsnotify_iter_info *iter_info) +{ + return nfsd_handle_dir_event(mask, dir, data, data_type, name); +} +#else +static int +nfsd_dir_fsnotify_handle_event(struct fsnotify_group *group, u32 mask, + const void *data, int data_type, struct inode *dir, + const struct qstr *name, u32 cookie, + struct fsnotify_iter_info *iter_info) +{ + return 0; +} +#endif + +static const struct fsnotify_ops nfsd_file_fsnotify_ops = { + .handle_inode_event = nfsd_file_fsnotify_handle_event, + .free_mark = nfsd_file_mark_free, +}; + +static const struct fsnotify_ops nfsd_dir_fsnotify_ops = { + .handle_event = nfsd_dir_fsnotify_handle_event, + .free_mark = nfsd_file_mark_free, +}; + +int +nfsd_file_cache_init(void) +{ + int ret; + + lockdep_assert_held(&nfsd_mutex); + if (test_and_set_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1) + return 0; + + ret = rhltable_init(&nfsd_file_rhltable, &nfsd_file_rhash_params); + if (ret) + goto out; + + ret = -ENOMEM; + nfsd_file_slab = KMEM_CACHE(nfsd_file, 0); + if (!nfsd_file_slab) { + pr_err("nfsd: unable to create nfsd_file_slab\n"); + goto out_err; + } + + nfsd_file_mark_slab = KMEM_CACHE(nfsd_file_mark, 0); + if (!nfsd_file_mark_slab) { + pr_err("nfsd: unable to create nfsd_file_mark_slab\n"); + goto out_err; + } + + ret = list_lru_init(&nfsd_file_lru); + if (ret) { + pr_err("nfsd: failed to init nfsd_file_lru: %d\n", ret); + goto out_err; + } + + nfsd_file_shrinker = shrinker_alloc(0, "nfsd-filecache"); + if (!nfsd_file_shrinker) { + ret = -ENOMEM; + pr_err("nfsd: failed to allocate nfsd_file_shrinker\n"); + goto out_lru; + } + + nfsd_file_shrinker->count_objects = nfsd_file_lru_count; + nfsd_file_shrinker->scan_objects = nfsd_file_lru_scan; + nfsd_file_shrinker->seeks = 1; + + shrinker_register(nfsd_file_shrinker); + + ret = lease_register_notifier(&nfsd_file_lease_notifier); + if (ret) { + pr_err("nfsd: unable to register lease notifier: %d\n", ret); + goto out_shrinker; + } + + nfsd_file_fsnotify_group = fsnotify_alloc_group(&nfsd_file_fsnotify_ops, 0); + if (IS_ERR(nfsd_file_fsnotify_group)) { + pr_err("nfsd: unable to create fsnotify group: %ld\n", + PTR_ERR(nfsd_file_fsnotify_group)); + ret = PTR_ERR(nfsd_file_fsnotify_group); + nfsd_file_fsnotify_group = NULL; + goto out_notifier; + } + + nfsd_dir_fsnotify_group = fsnotify_alloc_group(&nfsd_dir_fsnotify_ops, 0); + if (IS_ERR(nfsd_dir_fsnotify_group)) { + pr_err("nfsd: unable to create fsnotify group: %ld\n", + PTR_ERR(nfsd_dir_fsnotify_group)); + ret = PTR_ERR(nfsd_dir_fsnotify_group); + nfsd_dir_fsnotify_group = NULL; + goto out_notify_group; + } + + INIT_DELAYED_WORK(&nfsd_filecache_laundrette, nfsd_file_gc_worker); +out: + if (ret) + clear_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags); + return ret; +out_notify_group: + fsnotify_put_group(nfsd_file_fsnotify_group); + nfsd_file_fsnotify_group = NULL; +out_notifier: + lease_unregister_notifier(&nfsd_file_lease_notifier); +out_shrinker: + shrinker_free(nfsd_file_shrinker); +out_lru: + list_lru_destroy(&nfsd_file_lru); +out_err: + kmem_cache_destroy(nfsd_file_slab); + nfsd_file_slab = NULL; + kmem_cache_destroy(nfsd_file_mark_slab); + nfsd_file_mark_slab = NULL; + rhltable_destroy(&nfsd_file_rhltable); + goto out; +} + +/** + * __nfsd_file_cache_purge: clean out the cache for shutdown + * @net: net-namespace to shut down the cache (may be NULL) + * + * Walk the nfsd_file cache and close out any that match @net. If @net is NULL, + * then close out everything. Called when an nfsd instance is being shut down, + * and when the exports table is flushed. + */ +static void +__nfsd_file_cache_purge(struct net *net) +{ + struct rhashtable_iter iter; + struct nfsd_file *nf; + LIST_HEAD(dispose); + +#if IS_ENABLED(CONFIG_NFS_LOCALIO) + if (net) { + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + nfs_localio_invalidate_clients(&nn->local_clients, + &nn->local_clients_lock); + } +#endif + + rhltable_walk_enter(&nfsd_file_rhltable, &iter); + do { + rhashtable_walk_start(&iter); + + nf = rhashtable_walk_next(&iter); + while (!IS_ERR_OR_NULL(nf)) { + if (!net || nf->nf_net == net) + nfsd_file_cond_queue(nf, &dispose); + nf = rhashtable_walk_next(&iter); + } + + rhashtable_walk_stop(&iter); + } while (nf == ERR_PTR(-EAGAIN)); + rhashtable_walk_exit(&iter); + + nfsd_file_dispose_list(&dispose); +} + +int +nfsd_file_cache_start_net(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + spin_lock_init(&nn->fcache_dispose_lock); + INIT_LIST_HEAD(&nn->fcache_dispose_list); + return 0; +} + +/** + * nfsd_file_cache_purge - Remove all cache items associated with @net + * @net: target net namespace + * + */ +void +nfsd_file_cache_purge(struct net *net) +{ + lockdep_assert_held(&nfsd_mutex); + if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1) + __nfsd_file_cache_purge(net); +} + +void +nfsd_file_cache_shutdown_net(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + nfsd_file_cache_purge(net); + /* + * Ensure any in-progress shrinker, GC, or fsnotify/lease callback + * (all of which hold nfsd_gc_lock while calling + * nfsd_file_dispose_list_delayed()) has fully completed before + * draining the per-net dispose list. + */ + spin_lock(&nfsd_gc_lock); + spin_unlock(&nfsd_gc_lock); + nfsd_file_dispose_list(&nn->fcache_dispose_list); +} + +void +nfsd_file_cache_shutdown(void) +{ + int i; + + lockdep_assert_held(&nfsd_mutex); + if (test_and_clear_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 0) + return; + + lease_unregister_notifier(&nfsd_file_lease_notifier); + shrinker_free(nfsd_file_shrinker); + /* + * make sure all callers of nfsd_file_lru_cb are done before + * calling nfsd_file_cache_purge + */ + cancel_delayed_work_sync(&nfsd_filecache_laundrette); + __nfsd_file_cache_purge(NULL); + list_lru_destroy(&nfsd_file_lru); + rcu_barrier(); + fsnotify_put_group(nfsd_file_fsnotify_group); + nfsd_file_fsnotify_group = NULL; + fsnotify_put_group(nfsd_dir_fsnotify_group); + nfsd_dir_fsnotify_group = NULL; + kmem_cache_destroy(nfsd_file_slab); + nfsd_file_slab = NULL; + fsnotify_wait_marks_destroyed(); + kmem_cache_destroy(nfsd_file_mark_slab); + nfsd_file_mark_slab = NULL; + rhltable_destroy(&nfsd_file_rhltable); + + for_each_possible_cpu(i) { + per_cpu(nfsd_file_cache_hits, i) = 0; + per_cpu(nfsd_file_acquisitions, i) = 0; + per_cpu(nfsd_file_allocations, i) = 0; + per_cpu(nfsd_file_releases, i) = 0; + per_cpu(nfsd_file_total_age, i) = 0; + per_cpu(nfsd_file_evictions, i) = 0; + } +} + +static struct nfsd_file * +nfsd_file_lookup_locked(const struct net *net, const struct cred *cred, + struct inode *inode, unsigned char need, + bool want_gc) +{ + struct rhlist_head *tmp, *list; + struct nfsd_file *nf; + + list = rhltable_lookup(&nfsd_file_rhltable, &inode, + nfsd_file_rhash_params); + rhl_for_each_entry_rcu(nf, tmp, list, nf_rlist) { + if (nf->nf_may != need) + continue; + if (nf->nf_net != net) + continue; + if (!nfsd_match_cred(nf->nf_cred, cred)) + continue; + if (test_bit(NFSD_FILE_GC, &nf->nf_flags) != want_gc) + continue; + if (test_bit(NFSD_FILE_HASHED, &nf->nf_flags) == 0) + continue; + + if (!nfsd_file_get(nf)) + continue; + return nf; + } + return NULL; +} + +/** + * nfsd_file_is_cached - are there any cached open files for this inode? + * @inode: inode to check + * + * The lookup matches inodes in all net namespaces and is atomic wrt + * nfsd_file_acquire(). + * + * Return values: + * %true: filecache contains at least one file matching this inode + * %false: filecache contains no files matching this inode + */ +bool +nfsd_file_is_cached(struct inode *inode) +{ + struct rhlist_head *tmp, *list; + struct nfsd_file *nf; + bool ret = false; + + rcu_read_lock(); + list = rhltable_lookup(&nfsd_file_rhltable, &inode, + nfsd_file_rhash_params); + rhl_for_each_entry_rcu(nf, tmp, list, nf_rlist) + if (test_bit(NFSD_FILE_GC, &nf->nf_flags)) { + ret = true; + break; + } + rcu_read_unlock(); + + trace_nfsd_file_is_cached(inode, (int)ret); + return ret; +} + +static __be32 +nfsd_file_get_dio_attrs(const struct svc_fh *fhp, struct nfsd_file *nf) +{ + struct inode *inode = file_inode(nf->nf_file); + struct kstat stat; + __be32 status; + + /* Currently only need to get DIO alignment info for regular files */ + if (!S_ISREG(inode->i_mode)) + return nfs_ok; + + status = fh_getattr(fhp, &stat); + if (status != nfs_ok) + return status; + + trace_nfsd_file_get_dio_attrs(inode, &stat); + + if (stat.result_mask & STATX_DIOALIGN) { + nf->nf_dio_mem_align = stat.dio_mem_align; + nf->nf_dio_offset_align = stat.dio_offset_align; + } + if (stat.result_mask & STATX_DIO_READ_ALIGN) + nf->nf_dio_read_offset_align = stat.dio_read_offset_align; + else + nf->nf_dio_read_offset_align = nf->nf_dio_offset_align; + + return nfs_ok; +} + +static __be32 +nfsd_file_do_acquire(struct svc_rqst *rqstp, struct net *net, + struct svc_cred *cred, + struct auth_domain *client, + struct svc_fh *fhp, + unsigned int may_flags, struct file *file, + umode_t type, bool want_gc, struct nfsd_file **pnf) +{ + unsigned char need = may_flags & NFSD_FILE_MAY_MASK; + struct nfsd_file *new, *nf; + bool stale_retry = true; + bool open_retry = true; + struct inode *inode; + __be32 status; + int ret; + +retry: + if (rqstp) + status = fh_verify(rqstp, fhp, type, + may_flags|NFSD_MAY_OWNER_OVERRIDE); + else + status = fh_verify_local(net, cred, client, fhp, type, + may_flags|NFSD_MAY_OWNER_OVERRIDE); + + if (status != nfs_ok) + return status; + inode = d_inode(fhp->fh_dentry); + + rcu_read_lock(); + nf = nfsd_file_lookup_locked(net, current_cred(), inode, need, want_gc); + rcu_read_unlock(); + + if (nf) + goto wait_for_construction; + + new = nfsd_file_alloc(net, inode, need, want_gc); + if (!new) { + status = nfserr_jukebox; + goto out; + } + + rcu_read_lock(); + spin_lock(&inode->i_lock); + nf = nfsd_file_lookup_locked(net, current_cred(), inode, need, want_gc); + if (unlikely(nf)) { + spin_unlock(&inode->i_lock); + rcu_read_unlock(); + nfsd_file_free(new); + goto wait_for_construction; + } + nf = new; + ret = rhltable_insert(&nfsd_file_rhltable, &nf->nf_rlist, + nfsd_file_rhash_params); + spin_unlock(&inode->i_lock); + rcu_read_unlock(); + if (likely(ret == 0)) + goto open_file; + + trace_nfsd_file_insert_err(rqstp, inode, may_flags, ret); + status = nfserr_jukebox; + goto construction_err; + +wait_for_construction: + wait_on_bit(&nf->nf_flags, NFSD_FILE_PENDING, TASK_UNINTERRUPTIBLE); + + /* Did construction of this file fail? */ + if (!test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) { + trace_nfsd_file_cons_err(rqstp, inode, may_flags, nf); + if (!open_retry) { + status = nfserr_jukebox; + goto construction_err; + } + nfsd_file_put(nf); + open_retry = false; + fh_put(fhp); + goto retry; + } + this_cpu_inc(nfsd_file_cache_hits); + + status = nfserrno(nfsd_open_break_lease(file_inode(nf->nf_file), may_flags)); + if (status != nfs_ok) { + nfsd_file_put(nf); + nf = NULL; + } + +out: + if (status == nfs_ok) { + this_cpu_inc(nfsd_file_acquisitions); + nfsd_file_check_write_error(nf); + *pnf = nf; + } + trace_nfsd_file_acquire(rqstp, inode, may_flags, nf, status); + return status; + +open_file: + trace_nfsd_file_alloc(nf); + + nf->nf_mark = nfsd_file_mark_find_or_create(inode); + if (nf->nf_mark) { + if (file && (file->f_mode & FMODE_OPENED)) { + get_file(file); + nf->nf_file = file; + status = nfs_ok; + trace_nfsd_file_opened(nf, status); + } else { + ret = nfsd_open_verified(fhp, type, may_flags, &nf->nf_file); + if (ret == -EOPENSTALE && stale_retry) { + stale_retry = false; + nfsd_file_unhash(nf); + clear_and_wake_up_bit(NFSD_FILE_PENDING, + &nf->nf_flags); + if (refcount_dec_and_test(&nf->nf_ref)) + nfsd_file_free(nf); + nf = NULL; + fh_put(fhp); + goto retry; + } + status = nfserrno(ret); + trace_nfsd_file_open(nf, status); + if (status == nfs_ok) + status = nfsd_file_get_dio_attrs(fhp, nf); + } + } else + status = nfserr_jukebox; + /* + * If construction failed, or we raced with a call to unlink() + * then unhash. + */ + if (status != nfs_ok || inode->i_nlink == 0) + nfsd_file_unhash(nf); + else if (want_gc) + nfsd_file_lru_add(nf); + + clear_and_wake_up_bit(NFSD_FILE_PENDING, &nf->nf_flags); + if (status == nfs_ok) + goto out; + +construction_err: + if (refcount_dec_and_test(&nf->nf_ref)) + nfsd_file_free(nf); + nf = NULL; + goto out; +} + +/** + * nfsd_file_acquire_gc - Get a struct nfsd_file with an open file + * @rqstp: the RPC transaction being executed + * @fhp: the NFS filehandle of the file to be opened + * @may_flags: NFSD_MAY_ settings for the file + * @pnf: OUT: new or found "struct nfsd_file" object + * + * The nfsd_file object returned by this API is reference-counted + * and garbage-collected. The object is retained for a few + * seconds after the final nfsd_file_put() in case the caller + * wants to re-use it. + * + * Return values: + * %nfs_ok - @pnf points to an nfsd_file with its reference + * count boosted. + * + * On error, an nfsstat value in network byte order is returned. + */ +__be32 +nfsd_file_acquire_gc(struct svc_rqst *rqstp, struct svc_fh *fhp, + unsigned int may_flags, struct nfsd_file **pnf) +{ + return nfsd_file_do_acquire(rqstp, SVC_NET(rqstp), NULL, NULL, + fhp, may_flags, NULL, S_IFREG, true, pnf); +} + +/** + * nfsd_file_acquire - Get a struct nfsd_file with an open file + * @rqstp: the RPC transaction being executed + * @fhp: the NFS filehandle of the file to be opened + * @may_flags: NFSD_MAY_ settings for the file + * @pnf: OUT: new or found "struct nfsd_file" object + * + * The nfsd_file_object returned by this API is reference-counted + * but not garbage-collected. The object is unhashed after the + * final nfsd_file_put(). + * + * Return values: + * %nfs_ok - @pnf points to an nfsd_file with its reference + * count boosted. + * + * On error, an nfsstat value in network byte order is returned. + */ +__be32 +nfsd_file_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp, + unsigned int may_flags, struct nfsd_file **pnf) +{ + return nfsd_file_do_acquire(rqstp, SVC_NET(rqstp), NULL, NULL, + fhp, may_flags, NULL, S_IFREG, false, pnf); +} + +/** + * nfsd_file_acquire_local - Get a struct nfsd_file with an open file for localio + * @net: The network namespace in which to perform a lookup + * @cred: the user credential with which to validate access + * @client: the auth_domain for LOCALIO lookup + * @fhp: the NFS filehandle of the file to be opened + * @may_flags: NFSD_MAY_ settings for the file + * @pnf: OUT: new or found "struct nfsd_file" object + * + * This file lookup interface provide access to a file given the + * filehandle and credential. No connection-based authorisation + * is performed and in that way it is quite different to other + * file access mediated by nfsd. It allows a kernel module such as the NFS + * client to reach across network and filesystem namespaces to access + * a file. The security implications of this should be carefully + * considered before use. + * + * The nfsd_file_object returned by this API is reference-counted + * but not garbage-collected. The object is unhashed after the + * final nfsd_file_put(). + * + * Return values: + * %nfs_ok - @pnf points to an nfsd_file with its reference + * count boosted. + * + * On error, an nfsstat value in network byte order is returned. + */ +__be32 +nfsd_file_acquire_local(struct net *net, struct svc_cred *cred, + struct auth_domain *client, struct svc_fh *fhp, + unsigned int may_flags, struct nfsd_file **pnf) +{ + /* + * Save creds before calling nfsd_file_do_acquire() (which calls + * nfsd_setuser). Important because caller (LOCALIO) is from + * client context. + */ + const struct cred *save_cred = get_current_cred(); + __be32 beres; + + beres = nfsd_file_do_acquire(NULL, net, cred, client, fhp, may_flags, + NULL, S_IFREG, false, pnf); + put_cred(revert_creds(save_cred)); + return beres; +} + +/** + * nfsd_file_acquire_opened - Get a struct nfsd_file using existing open file + * @rqstp: the RPC transaction being executed + * @fhp: the NFS filehandle of the file just created + * @may_flags: NFSD_MAY_ settings for the file + * @file: cached, already-open file (may be NULL or not yet opened) + * @pnf: OUT: new or found "struct nfsd_file" object + * + * Acquire a nfsd_file object that is not GC'ed. If one doesn't already exist, + * and @file has FMODE_OPENED set, use it to instantiate a new nfsd_file + * instead of opening a new one. + * + * Return values: + * %nfs_ok - @pnf points to an nfsd_file with its reference + * count boosted. + * + * On error, an nfsstat value in network byte order is returned. + */ +__be32 +nfsd_file_acquire_opened(struct svc_rqst *rqstp, struct svc_fh *fhp, + unsigned int may_flags, struct file *file, + struct nfsd_file **pnf) +{ + return nfsd_file_do_acquire(rqstp, SVC_NET(rqstp), NULL, NULL, + fhp, may_flags, file, S_IFREG, false, pnf); +} + +/** + * nfsd_file_acquire_dir - Get a struct nfsd_file with an open directory + * @rqstp: the RPC transaction being executed + * @fhp: the NFS filehandle of the file to be opened + * @pnf: OUT: new or found "struct nfsd_file" object + * + * The nfsd_file_object returned by this API is reference-counted + * but not garbage-collected. The object is unhashed after the + * final nfsd_file_put(). This opens directories only, and only + * in O_RDONLY mode. + * + * Return values: + * %nfs_ok - @pnf points to an nfsd_file with its reference + * count boosted. + * + * On error, an nfsstat value in network byte order is returned. + */ +__be32 +nfsd_file_acquire_dir(struct svc_rqst *rqstp, struct svc_fh *fhp, + struct nfsd_file **pnf) +{ + return nfsd_file_do_acquire(rqstp, SVC_NET(rqstp), NULL, NULL, fhp, + NFSD_MAY_READ|NFSD_MAY_64BIT_COOKIE, + NULL, S_IFDIR, false, pnf); +} + +/* + * Note that fields may be added, removed or reordered in the future. Programs + * scraping this file for info should test the labels to ensure they're + * getting the correct field. + */ +int nfsd_file_cache_stats_show(struct seq_file *m, void *v) +{ + unsigned long allocations = 0, releases = 0, evictions = 0; + unsigned long hits = 0, acquisitions = 0; + unsigned int i, count = 0, buckets = 0; + unsigned long lru = 0, total_age = 0; + + /* Serialize with server shutdown */ + mutex_lock(&nfsd_mutex); + if (test_bit(NFSD_FILE_CACHE_UP, &nfsd_file_flags) == 1) { + struct bucket_table *tbl; + struct rhashtable *ht; + + lru = list_lru_count(&nfsd_file_lru); + + rcu_read_lock(); + ht = &nfsd_file_rhltable.ht; + count = atomic_read(&ht->nelems); + tbl = rht_dereference_rcu(ht->tbl, ht); + buckets = tbl->size; + rcu_read_unlock(); + } + mutex_unlock(&nfsd_mutex); + + for_each_possible_cpu(i) { + hits += per_cpu(nfsd_file_cache_hits, i); + acquisitions += per_cpu(nfsd_file_acquisitions, i); + allocations += per_cpu(nfsd_file_allocations, i); + releases += per_cpu(nfsd_file_releases, i); + total_age += per_cpu(nfsd_file_total_age, i); + evictions += per_cpu(nfsd_file_evictions, i); + } + + seq_printf(m, "total inodes: %u\n", count); + seq_printf(m, "hash buckets: %u\n", buckets); + seq_printf(m, "lru entries: %lu\n", lru); + seq_printf(m, "cache hits: %lu\n", hits); + seq_printf(m, "acquisitions: %lu\n", acquisitions); + seq_printf(m, "allocations: %lu\n", allocations); + seq_printf(m, "releases: %lu\n", releases); + seq_printf(m, "evictions: %lu\n", evictions); + if (releases) + seq_printf(m, "mean age (ms): %ld\n", total_age / releases); + else + seq_printf(m, "mean age (ms): -\n"); + return 0; +} + +/** + * nfsd_fsnotify_recalc_mask - recalculate the fsnotify mask for a nfsd_file + * @nf: nfsd_file to recalculate the mask on + * + * When a directory nfsd_file has a delegation added or removed, that may + * change the events that nfsd requires from the VFS layer. This function + * recalculates the fsnotify mask based on the leases present. + */ +void nfsd_fsnotify_recalc_mask(struct nfsd_file *nf) +{ + struct inode *inode = file_inode(nf->nf_file); + u32 lease_mask, set = 0, clear = 0; + struct fsnotify_mark *mark; + + /* This is only needed when adding or removing dir delegs */ + if (!S_ISDIR(inode->i_mode) || !nf->nf_mark) + return; + + mark = &nf->nf_mark->nfm_mark; + + /* + * The mark is shared by every nfsd_file on this inode, so concurrent + * delegation add/remove on the same directory can recalc it in + * parallel. Serialize the read of the lease state and the update of + * the mark so that a recalc working from a stale snapshot of the + * lease list can't clobber a concurrent recalc's update. + */ + mutex_lock(&nf->nf_mark->nfm_recalc_mutex); + + /* Set up notifications for any ignored delegation events */ + lease_mask = inode_lease_ignore_mask(inode); + + if (lease_mask & FL_IGN_DIR_CREATE) + set |= FS_CREATE | FS_MOVED_TO; + else + clear |= FS_CREATE | FS_MOVED_TO; + + if (lease_mask & FL_IGN_DIR_DELETE) + set |= FS_DELETE | FS_MOVED_FROM; + else + clear |= FS_DELETE | FS_MOVED_FROM; + + if (lease_mask & FL_IGN_DIR_RENAME) + set |= FS_RENAME; + else + clear |= FS_RENAME; + + fsnotify_modify_mark_mask(mark, set, clear); + mutex_unlock(&nf->nf_mark->nfm_recalc_mutex); +} diff --git a/fs/nfsd/filecache.h b/fs/nfsd/filecache.h new file mode 100644 index 000000000..b224902b4 --- /dev/null +++ b/fs/nfsd/filecache.h @@ -0,0 +1,92 @@ +#ifndef _FS_NFSD_FILECACHE_H +#define _FS_NFSD_FILECACHE_H + +#include + +/* + * Limit the time that the list_lru_one lock is held during + * an LRU scan. + */ +#define NFSD_FILE_GC_BATCH (16UL) + +/* + * This is the fsnotify_mark container that nfsd attaches to the files that it + * is holding open. Note that we have a separate refcount here aside from the + * one in the fsnotify_mark. We only want a single fsnotify_mark attached to + * the inode, and for each nfsd_file to hold a reference to it. + * + * The fsnotify_mark is itself refcounted, but that's not sufficient to tell us + * how to put that reference. If there are still outstanding nfsd_files that + * reference the mark, then we would want to call fsnotify_put_mark on it. + * If there were not, then we'd need to call fsnotify_destroy_mark. Since we + * can't really tell the difference, we use the nfm_mark to keep track of how + * many nfsd_files hold references to the mark. When that counter goes to zero + * then we know to call fsnotify_destroy_mark on it. + */ +struct nfsd_file_mark { + struct fsnotify_mark nfm_mark; + refcount_t nfm_ref; + /* serializes nfsd_fsnotify_recalc_mask() against itself */ + struct mutex nfm_recalc_mutex; +}; + +/* + * A representation of a file that has been opened by knfsd. These are hashed + * in the hashtable by inode pointer value. Note that this object doesn't + * hold a reference to the inode by itself, so the nf_inode pointer should + * never be dereferenced, only used for comparison. + */ +struct nfsd_file { + struct rhlist_head nf_rlist; + void *nf_inode; + struct file *nf_file; + const struct cred *nf_cred; + struct net *nf_net; +#define NFSD_FILE_HASHED (0) +#define NFSD_FILE_PENDING (1) +#define NFSD_FILE_REFERENCED (2) +#define NFSD_FILE_GC (3) +#define NFSD_FILE_RECENT (4) + unsigned long nf_flags; + refcount_t nf_ref; + unsigned char nf_may; + + struct nfsd_file_mark *nf_mark; + struct list_head nf_lru; + struct list_head nf_gc; + struct rcu_head nf_rcu; + ktime_t nf_birthtime; + + u32 nf_dio_mem_align; + u32 nf_dio_offset_align; + u32 nf_dio_read_offset_align; +}; + +int nfsd_file_cache_init(void); +void nfsd_file_cache_purge(struct net *); +void nfsd_file_cache_shutdown(void); +int nfsd_file_cache_start_net(struct net *net); +void nfsd_file_cache_shutdown_net(struct net *net); +void nfsd_file_put(struct nfsd_file *nf); +struct net *nfsd_file_put_local(struct nfsd_file __rcu **nf); +struct nfsd_file *nfsd_file_get(struct nfsd_file *nf); +struct file *nfsd_file_file(struct nfsd_file *nf); +void nfsd_file_close_inode_sync(struct inode *inode); +void nfsd_file_close_export(struct net *net, const struct path *path); +void nfsd_file_net_dispose(struct nfsd_net *nn); +bool nfsd_file_is_cached(struct inode *inode); +__be32 nfsd_file_acquire_gc(struct svc_rqst *rqstp, struct svc_fh *fhp, + unsigned int may_flags, struct nfsd_file **nfp); +__be32 nfsd_file_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp, + unsigned int may_flags, struct nfsd_file **nfp); +__be32 nfsd_file_acquire_opened(struct svc_rqst *rqstp, struct svc_fh *fhp, + unsigned int may_flags, struct file *file, + struct nfsd_file **nfp); +__be32 nfsd_file_acquire_local(struct net *net, struct svc_cred *cred, + struct auth_domain *client, struct svc_fh *fhp, + unsigned int may_flags, struct nfsd_file **pnf); +__be32 nfsd_file_acquire_dir(struct svc_rqst *rqstp, struct svc_fh *fhp, + struct nfsd_file **pnf); +int nfsd_file_cache_stats_show(struct seq_file *m, void *v); +void nfsd_fsnotify_recalc_mask(struct nfsd_file *nf); +#endif /* _FS_NFSD_FILECACHE_H */ diff --git a/fs/nfsd/flexfilelayout.c b/fs/nfsd/flexfilelayout.c new file mode 100644 index 000000000..6d531285a --- /dev/null +++ b/fs/nfsd/flexfilelayout.c @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2016 Tom Haynes + * + * The following implements a super-simple flex-file server + * where the NFSv4.1 mds is also the ds. And the storage is + * the same. I.e., writing to the mds via a NFSv4.1 WRITE + * goes to the same location as the NFSv3 WRITE. + */ +#include + +#include + +#include + +#include "flexfilelayoutxdr.h" +#include "pnfs.h" +#include "vfs.h" + +#define NFSDDBG_FACILITY NFSDDBG_PNFS + +static __be32 +nfsd4_ff_proc_layoutget(struct svc_rqst *rqstp, struct inode *inode, + const struct svc_fh *fhp, struct nfsd4_layoutget *args) +{ + struct nfsd4_layout_seg *seg = &args->lg_seg; + u32 device_generation = 0; + int error; + uid_t u; + + struct pnfs_ff_layout *fl; + + /* + * The super simple flex file server has 1 mirror, 1 data server, + * and 1 file handle. So instead of 4 allocs, do 1 for now. + * Zero it out for the stateid - don't want junk in there! + */ + error = -ENOMEM; + fl = kzalloc_obj(*fl); + if (!fl) + goto out_error; + args->lg_content = fl; + + /* + * Avoid layout commit, try to force the I/O to the DS, + * and for fun, cause all IOMODE_RW layout segments to + * effectively be WRITE only. + */ + fl->flags = FF_FLAGS_NO_LAYOUTCOMMIT | FF_FLAGS_NO_IO_THRU_MDS | + FF_FLAGS_NO_READ_IO; + + /* Do not allow a IOMODE_READ segment to have write pemissions */ + if (seg->iomode == IOMODE_READ) { + u = from_kuid(&init_user_ns, inode->i_uid) + 1; + fl->uid = make_kuid(&init_user_ns, u); + } else + fl->uid = inode->i_uid; + fl->gid = inode->i_gid; + + error = nfsd4_set_deviceid(&fl->deviceid, fhp, device_generation); + if (error) + goto out_error; + + fl->fh.size = fhp->fh_handle.fh_size; + memcpy(fl->fh.data, &fhp->fh_handle.fh_raw, fl->fh.size); + + /* Give whole file layout segments */ + seg->offset = 0; + seg->length = NFS4_MAX_UINT64; + + dprintk("GET: 0x%llx:0x%llx %d\n", seg->offset, seg->length, + seg->iomode); + return 0; + +out_error: + seg->length = 0; + return nfserrno(error); +} + +static __be32 +nfsd4_ff_proc_getdeviceinfo(struct super_block *sb, struct svc_rqst *rqstp, + struct nfs4_client *clp, struct nfsd4_getdeviceinfo *gdp) +{ + struct pnfs_ff_device_addr *da; + + u16 port; + char addr[INET6_ADDRSTRLEN]; + + da = kzalloc_obj(struct pnfs_ff_device_addr); + if (!da) + return nfserrno(-ENOMEM); + + gdp->gd_device = da; + + da->version = 3; + da->minor_version = 0; + + da->rsize = svc_max_payload(rqstp); + da->wsize = da->rsize; + + rpc_ntop((struct sockaddr *)&rqstp->rq_daddr, + addr, INET6_ADDRSTRLEN); + if (rqstp->rq_daddr.ss_family == AF_INET) { + struct sockaddr_in *sin; + + sin = (struct sockaddr_in *)&rqstp->rq_daddr; + port = ntohs(sin->sin_port); + snprintf(da->netaddr.netid, FF_NETID_LEN + 1, "tcp"); + da->netaddr.netid_len = 3; + } else { + struct sockaddr_in6 *sin6; + + sin6 = (struct sockaddr_in6 *)&rqstp->rq_daddr; + port = ntohs(sin6->sin6_port); + snprintf(da->netaddr.netid, FF_NETID_LEN + 1, "tcp6"); + da->netaddr.netid_len = 4; + } + + da->netaddr.addr_len = + snprintf(da->netaddr.addr, FF_ADDR_LEN + 1, + "%s.%d.%d", addr, port >> 8, port & 0xff); + + da->tightly_coupled = false; + + return 0; +} + +static __be32 +nfsd4_ff_proc_layoutcommit(struct inode *inode, struct svc_rqst *rqstp, + struct nfsd4_layoutcommit *lcp) +{ + return nfs_ok; +} + +const struct nfsd4_layout_ops ff_layout_ops = { + .notify_types = + NOTIFY_DEVICEID4_DELETE | NOTIFY_DEVICEID4_CHANGE, + .disable_recalls = true, + .proc_getdeviceinfo = nfsd4_ff_proc_getdeviceinfo, + .encode_getdeviceinfo = nfsd4_ff_encode_getdeviceinfo, + .proc_layoutget = nfsd4_ff_proc_layoutget, + .encode_layoutget = nfsd4_ff_encode_layoutget, + .proc_layoutcommit = nfsd4_ff_proc_layoutcommit, +}; diff --git a/fs/nfsd/flexfilelayoutxdr.c b/fs/nfsd/flexfilelayoutxdr.c new file mode 100644 index 000000000..374e52d30 --- /dev/null +++ b/fs/nfsd/flexfilelayoutxdr.c @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2016 Tom Haynes + */ +#include +#include + +#include "nfsd.h" +#include "flexfilelayoutxdr.h" + +#define NFSDDBG_FACILITY NFSDDBG_PNFS + +struct ff_idmap { + char buf[11]; + int len; +}; + +__be32 +nfsd4_ff_encode_layoutget(struct xdr_stream *xdr, + const struct nfsd4_layoutget *lgp) +{ + const struct pnfs_ff_layout *fl = lgp->lg_content; + int len, mirror_len, ds_len, fh_len; + __be32 *p; + + /* + * Unlike nfsd4_encode_user, we know these will + * always be stringified. + */ + struct ff_idmap uid; + struct ff_idmap gid; + + fh_len = 4 + xdr_align_size(fl->fh.size); + + uid.len = sprintf(uid.buf, "%u", from_kuid(&init_user_ns, fl->uid)); + gid.len = sprintf(gid.buf, "%u", from_kgid(&init_user_ns, fl->gid)); + + /* data server entry: deviceid + efficiency + stateid + fh list + + * user + group + flags + stats_collect_hint + */ + ds_len = 16 + 4 + 4 + sizeof(stateid_opaque_t) + 4 + fh_len + + 4 + xdr_align_size(uid.len) + + 4 + xdr_align_size(gid.len) + + 4 + 4; + + /* mirror: ds_count + ds */ + mirror_len = 4 + ds_len; + + /* stripe_unit + mirror_count + mirror */ + len = 12 + mirror_len; + + p = xdr_reserve_space(xdr, sizeof(__be32) + len); + if (!p) + return nfserr_toosmall; + + *p++ = cpu_to_be32(len); + p = xdr_encode_hyper(p, 0); /* stripe unit of 1 */ + + *p++ = cpu_to_be32(1); /* single mirror */ + *p++ = cpu_to_be32(1); /* single data server */ + + p = svcxdr_encode_deviceid4(p, &fl->deviceid); + + *p++ = cpu_to_be32(1); /* efficiency */ + + *p++ = cpu_to_be32(fl->stateid.si_generation); + p = xdr_encode_opaque_fixed(p, &fl->stateid.si_opaque, + sizeof(stateid_opaque_t)); + + *p++ = cpu_to_be32(1); /* single file handle */ + p = xdr_encode_opaque(p, fl->fh.data, fl->fh.size); + + p = xdr_encode_opaque(p, uid.buf, uid.len); + p = xdr_encode_opaque(p, gid.buf, gid.len); + + *p++ = cpu_to_be32(fl->flags); + *p++ = cpu_to_be32(0); /* No stats collect hint */ + + return 0; +} + +__be32 +nfsd4_ff_encode_getdeviceinfo(struct xdr_stream *xdr, + const struct nfsd4_getdeviceinfo *gdp) +{ + struct pnfs_ff_device_addr *da = gdp->gd_device; + int len; + int ver_len; + int addr_len; + __be32 *p; + + /* + * See paragraph 5 of RFC 8881 S18.40.3. + */ + if (!gdp->gd_maxcount) { + if (xdr_stream_encode_u32(xdr, 0) != XDR_UNIT) + return nfserr_resource; + return nfs_ok; + } + + /* len + padding for two strings */ + addr_len = 8 + xdr_align_size(da->netaddr.netid_len) + + xdr_align_size(da->netaddr.addr_len); + ver_len = 20; + + len = 4 + ver_len + 4 + addr_len; + + p = xdr_reserve_space(xdr, len + sizeof(__be32)); + if (!p) + return nfserr_resource; + + /* + * Fill in the overall length and number of volumes at the beginning + * of the layout. + */ + *p++ = cpu_to_be32(len); + *p++ = cpu_to_be32(1); /* 1 netaddr */ + p = xdr_encode_opaque(p, da->netaddr.netid, da->netaddr.netid_len); + p = xdr_encode_opaque(p, da->netaddr.addr, da->netaddr.addr_len); + + *p++ = cpu_to_be32(1); /* 1 versions */ + + *p++ = cpu_to_be32(da->version); + *p++ = cpu_to_be32(da->minor_version); + *p++ = cpu_to_be32(da->rsize); + *p++ = cpu_to_be32(da->wsize); + *p++ = cpu_to_be32(da->tightly_coupled); + + return 0; +} diff --git a/fs/nfsd/flexfilelayoutxdr.h b/fs/nfsd/flexfilelayoutxdr.h new file mode 100644 index 000000000..6d5a1066a --- /dev/null +++ b/fs/nfsd/flexfilelayoutxdr.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2016 Tom Haynes + */ +#ifndef _NFSD_FLEXFILELAYOUTXDR_H +#define _NFSD_FLEXFILELAYOUTXDR_H 1 + +#include +#include "xdr4.h" + +#define FF_FLAGS_NO_LAYOUTCOMMIT 1 +#define FF_FLAGS_NO_IO_THRU_MDS 2 +#define FF_FLAGS_NO_READ_IO 4 + +struct xdr_stream; + +#define FF_NETID_LEN (4) +#define FF_ADDR_LEN (INET6_ADDRSTRLEN + 8) +struct pnfs_ff_netaddr { + char netid[FF_NETID_LEN + 1]; + char addr[FF_ADDR_LEN + 1]; + u32 netid_len; + u32 addr_len; +}; + +struct pnfs_ff_device_addr { + struct pnfs_ff_netaddr netaddr; + u32 version; + u32 minor_version; + u32 rsize; + u32 wsize; + bool tightly_coupled; +}; + +struct pnfs_ff_layout { + u32 flags; + u32 stats_collect_hint; + kuid_t uid; + kgid_t gid; + struct nfsd4_deviceid deviceid; + stateid_t stateid; + struct nfs_fh fh; +}; + +__be32 nfsd4_ff_encode_getdeviceinfo(struct xdr_stream *xdr, + const struct nfsd4_getdeviceinfo *gdp); +__be32 nfsd4_ff_encode_layoutget(struct xdr_stream *xdr, + const struct nfsd4_layoutget *lgp); + +#endif /* _NFSD_FLEXFILELAYOUTXDR_H */ diff --git a/fs/nfsd/idmap.h b/fs/nfsd/idmap.h new file mode 100644 index 000000000..23cc85d1e --- /dev/null +++ b/fs/nfsd/idmap.h @@ -0,0 +1,60 @@ +/* + * Mapping of UID to name and vice versa. + * + * Copyright (c) 2002, 2003 The Regents of the University of + * Michigan. All rights reserved. +> * + * Marius Aamodt Eriksen + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef LINUX_NFSD_IDMAP_H +#define LINUX_NFSD_IDMAP_H + +#include +#include +#include + +#ifdef CONFIG_NFSD_V4 +int nfsd_idmap_init(struct net *); +void nfsd_idmap_shutdown(struct net *); +#else +static inline int nfsd_idmap_init(struct net *net) +{ + return 0; +} +static inline void nfsd_idmap_shutdown(struct net *net) +{ +} +#endif + +__be32 nfsd_map_name_to_uid(struct svc_rqst *, const char *, size_t, kuid_t *); +__be32 nfsd_map_name_to_gid(struct svc_rqst *, const char *, size_t, kgid_t *); +__be32 nfsd4_encode_user(struct xdr_stream *, struct svc_rqst *, kuid_t); +__be32 nfsd4_encode_group(struct xdr_stream *, struct svc_rqst *, kgid_t); + +#endif /* LINUX_NFSD_IDMAP_H */ diff --git a/fs/nfsd/localio.c b/fs/nfsd/localio.c new file mode 100644 index 000000000..c458c01e9 --- /dev/null +++ b/fs/nfsd/localio.c @@ -0,0 +1,220 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * NFS server support for local clients to bypass network stack + * + * Copyright (C) 2014 Weston Andros Adamson + * Copyright (C) 2019 Trond Myklebust + * Copyright (C) 2024 Mike Snitzer + * Copyright (C) 2024 NeilBrown + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nfsd.h" +#include "vfs.h" +#include "netns.h" +#include "filecache.h" +#include "cache.h" + +/** + * nfsd_open_local_fh - lookup a local filehandle @nfs_fh and map to nfsd_file + * + * @net: 'struct net' to get the proper nfsd_net required for LOCALIO access + * @dom: 'struct auth_domain' required for LOCALIO access + * @rpc_clnt: rpc_clnt that the client established + * @cred: cred that the client established + * @nfs_fh: filehandle to lookup + * @pnf: place to find the nfsd_file, or store it if it was non-NULL + * @fmode: fmode_t to use for open + * + * This function maps a local fh to a path on a local filesystem. + * This is useful when the nfs client has the local server mounted - it can + * avoid all the NFS overhead with reads, writes and commits. + * + * On successful return, returned nfsd_file will have its nf_net member + * set. Caller (NFS client) is responsible for calling nfsd_net_put and + * nfsd_file_put (via nfs_to_nfsd_file_put_local). + */ +static struct nfsd_file * +nfsd_open_local_fh(struct net *net, struct auth_domain *dom, + struct rpc_clnt *rpc_clnt, const struct cred *cred, + const struct nfs_fh *nfs_fh, struct nfsd_file __rcu **pnf, + const fmode_t fmode) +{ + int mayflags = NFSD_MAY_LOCALIO; + struct svc_cred rq_cred; + struct svc_fh fh; + struct nfsd_file *localio; + __be32 beres; + + if (nfs_fh->size > NFS4_FHSIZE) + return ERR_PTR(-EINVAL); + + if (!nfsd_net_try_get(net)) + return ERR_PTR(-ENXIO); + + rcu_read_lock(); + localio = nfsd_file_get(rcu_dereference(*pnf)); + rcu_read_unlock(); + if (localio) + return localio; + + /* nfs_fh -> svc_fh */ + fh_init(&fh, NFS4_FHSIZE); + fh.fh_handle.fh_size = nfs_fh->size; + memcpy(fh.fh_handle.fh_raw, nfs_fh->data, nfs_fh->size); + + if (fmode & FMODE_READ) + mayflags |= NFSD_MAY_READ; + if (fmode & FMODE_WRITE) + mayflags |= NFSD_MAY_WRITE; + + svcauth_map_clnt_to_svc_cred_local(rpc_clnt, cred, &rq_cred); + + beres = nfsd_file_acquire_local(net, &rq_cred, dom, + &fh, mayflags, &localio); + if (beres) + localio = ERR_PTR(nfs_stat_to_errno(be32_to_cpu(beres))); + + fh_put(&fh); + if (rq_cred.cr_group_info) + put_group_info(rq_cred.cr_group_info); + + if (!IS_ERR(localio)) { + struct nfsd_file *new; + if (!nfsd_net_try_get(net)) { + nfsd_file_put(localio); + nfsd_net_put(net); + return ERR_PTR(-ENXIO); + } + nfsd_file_get(localio); + again: + rcu_read_lock(); + new = unrcu_pointer(cmpxchg(pnf, NULL, RCU_INITIALIZER(localio))); + if (new) { + /* Some other thread installed an nfsd_file */ + if (nfsd_file_get(new) == NULL) { + rcu_read_unlock(); + goto again; + } + rcu_read_unlock(); + /* + * Drop the ref we were going to install (both file and + * net) and the one we were going to return (only file). + */ + nfsd_file_put(localio); + nfsd_net_put(net); + nfsd_file_put(localio); + localio = new; + } else { + rcu_read_unlock(); + } + } else + nfsd_net_put(net); + + return localio; +} + +static void nfsd_file_dio_alignment(struct nfsd_file *nf, + u32 *nf_dio_mem_align, + u32 *nf_dio_offset_align, + u32 *nf_dio_read_offset_align) +{ + *nf_dio_mem_align = nf->nf_dio_mem_align; + *nf_dio_offset_align = nf->nf_dio_offset_align; + *nf_dio_read_offset_align = nf->nf_dio_read_offset_align; +} + +static const struct nfsd_localio_operations nfsd_localio_ops = { + .nfsd_net_try_get = nfsd_net_try_get, + .nfsd_net_put = nfsd_net_put, + .nfsd_open_local_fh = nfsd_open_local_fh, + .nfsd_file_put_local = nfsd_file_put_local, + .nfsd_file_file = nfsd_file_file, + .nfsd_file_dio_alignment = nfsd_file_dio_alignment, +}; + +void nfsd_localio_ops_init(void) +{ + nfs_to = &nfsd_localio_ops; +} + +/* + * UUID_IS_LOCAL XDR functions + */ + +static __be32 localio_proc_null(struct svc_rqst *rqstp) +{ + return rpc_success; +} + +struct localio_uuidarg { + uuid_t uuid; +}; + +static __be32 localio_proc_uuid_is_local(struct svc_rqst *rqstp) +{ + struct localio_uuidarg *argp = rqstp->rq_argp; + struct net *net = SVC_NET(rqstp); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + nfs_uuid_is_local(&argp->uuid, &nn->local_clients, + &nn->local_clients_lock, + net, rqstp->rq_client, THIS_MODULE); + + return rpc_success; +} + +static bool localio_decode_uuidarg(struct svc_rqst *rqstp, + struct xdr_stream *xdr) +{ + struct localio_uuidarg *argp = rqstp->rq_argp; + u8 uuid[UUID_SIZE]; + + if (decode_opaque_fixed(xdr, uuid, UUID_SIZE)) + return false; + import_uuid(&argp->uuid, uuid); + + return true; +} + +static const struct svc_procedure localio_procedures1[] = { + [LOCALIOPROC_NULL] = { + .pc_func = localio_proc_null, + .pc_decode = nfssvc_decode_voidarg, + .pc_encode = nfssvc_encode_voidres, + .pc_argsize = sizeof(struct nfsd_voidargs), + .pc_ressize = sizeof(struct nfsd_voidres), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = 0, + .pc_name = "NULL", + }, + [LOCALIOPROC_UUID_IS_LOCAL] = { + .pc_func = localio_proc_uuid_is_local, + .pc_decode = localio_decode_uuidarg, + .pc_encode = nfssvc_encode_voidres, + .pc_argsize = sizeof(struct localio_uuidarg), + .pc_argzero = sizeof(struct localio_uuidarg), + .pc_ressize = sizeof(struct nfsd_voidres), + .pc_cachetype = RC_NOCACHE, + .pc_name = "UUID_IS_LOCAL", + }, +}; + +#define LOCALIO_NR_PROCEDURES ARRAY_SIZE(localio_procedures1) +const struct svc_version localio_version1 = { + .vs_vers = 1, + .vs_nproc = LOCALIO_NR_PROCEDURES, + .vs_proc = localio_procedures1, + .vs_dispatch = nfsd_dispatch, + .vs_xdrsize = XDR_QUADLEN(UUID_SIZE), + .vs_hidden = true, +}; diff --git a/fs/nfsd/lockd.c b/fs/nfsd/lockd.c new file mode 100644 index 000000000..72a5b4998 --- /dev/null +++ b/fs/nfsd/lockd.c @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * This file contains all the stubs needed when communicating with lockd. + * This level of indirection is necessary so we can run nfsd+lockd without + * requiring the nfs client to be compiled in/loaded, and vice versa. + * + * Copyright (C) 1996, Olaf Kirch + */ + +#include +#include +#include "nfsd.h" +#include "vfs.h" + +#define NFSDDBG_FACILITY NFSDDBG_LOCKD + +/** + * nlm_fopen - Open an NFSD file + * @rqstp: NLM RPC procedure execution context + * @f: NFS file handle to be opened + * @filp: OUT: an opened struct file + * @flags: the POSIX open flags to use + * + * nlm_fopen() holds the dentry reference until nlm_fclose() releases it. + * + * Returns zero on success or a negative errno value if the file + * cannot be opened. + */ +static int nlm_fopen(struct svc_rqst *rqstp, struct nfs_fh *f, + struct file **filp, int flags) +{ + __be32 nfserr; + int access; + struct svc_fh fh; + + /* must initialize before using! but maxsize doesn't matter */ + fh_init(&fh,0); + fh.fh_handle.fh_size = f->size; + memcpy(&fh.fh_handle.fh_raw, f->data, f->size); + fh.fh_export = NULL; + + /* + * Allow BYPASS_GSS as some client implementations use AUTH_SYS + * for NLM even when GSS is used for NFS. + * Allow OWNER_OVERRIDE as permission might have been changed + * after the file was opened. + * Pass MAY_NLM so that authentication can be completely bypassed + * if NFSEXP_NOAUTHNLM is set. Some older clients use AUTH_NULL + * for NLM requests. + */ + access = (flags == O_WRONLY) ? NFSD_MAY_WRITE : NFSD_MAY_READ; + access |= NFSD_MAY_NLM | NFSD_MAY_OWNER_OVERRIDE | NFSD_MAY_BYPASS_GSS; + nfserr = nfsd_open(rqstp, &fh, S_IFREG, access, filp); + fh_put(&fh); + + switch (nfserr) { + case nfs_ok: + break; + case nfserr_jukebox: + /* + * This error can indicate a presence of a conflicting + * delegation to an NLM lock request. Options are: + * (1) For now, drop this request and make the client + * retry. When delegation is returned, client's lock retry + * will complete. + * (2) NLM4_DENIED as per "spec" signals to the client + * that the lock is unavailable now but client can retry. + * Linux client implementation does not. It treats + * NLM4_DENIED same as NLM4_FAILED and fails the request. + * (3) For the future, treat this as blocked lock and try + * to callback when the delegation is returned but might + * not have a proper lock request to block on. + */ + return -EWOULDBLOCK; + case nfserr_stale: + return -ESTALE; + default: + return -ENOLCK; + } + + return 0; +} + +/** + * nlm_fclose - Close an NFSD file + * @filp: a struct file that was opened by nlm_fopen() + */ +static void +nlm_fclose(struct file *filp) +{ + fput(filp); +} + +static const struct nlmsvc_binding nfsd_nlm_ops = { + .owner = THIS_MODULE, + .fopen = nlm_fopen, /* open file for locking */ + .fclose = nlm_fclose, /* close file */ +}; + +void +nfsd_lockd_init(void) +{ + dprintk("nfsd: initializing lockd\n"); + rcu_assign_pointer(nlmsvc_ops, &nfsd_nlm_ops); +} + +void +nfsd_lockd_shutdown(void) +{ + RCU_INIT_POINTER(nlmsvc_ops, NULL); + synchronize_rcu(); +} diff --git a/fs/nfsd/netlink.c b/fs/nfsd/netlink.c new file mode 100644 index 000000000..eba8b353f --- /dev/null +++ b/fs/nfsd/netlink.c @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) +/* Do not edit directly, auto-generated from: */ +/* Documentation/netlink/specs/nfsd.yaml */ +/* YNL-GEN kernel source */ +/* To regenerate run: tools/net/ynl/ynl-regen.sh */ + +#include +#include + +#include "netlink.h" + +#include + +/* Common nested types */ +const struct nla_policy nfsd_auth_flavor_nl_policy[NFSD_A_AUTH_FLAVOR_FLAGS + 1] = { + [NFSD_A_AUTH_FLAVOR_PSEUDOFLAVOR] = { .type = NLA_U32, }, + [NFSD_A_AUTH_FLAVOR_FLAGS] = NLA_POLICY_MASK(NLA_U32, 0x3ffff), +}; + +const struct nla_policy nfsd_expkey_nl_policy[NFSD_A_EXPKEY_PATH + 1] = { + [NFSD_A_EXPKEY_SEQNO] = { .type = NLA_U64, }, + [NFSD_A_EXPKEY_CLIENT] = { .type = NLA_NUL_STRING, }, + [NFSD_A_EXPKEY_FSIDTYPE] = { .type = NLA_U8, }, + [NFSD_A_EXPKEY_FSID] = { .type = NLA_BINARY, }, + [NFSD_A_EXPKEY_NEGATIVE] = { .type = NLA_FLAG, }, + [NFSD_A_EXPKEY_EXPIRY] = { .type = NLA_U64, }, + [NFSD_A_EXPKEY_PATH] = { .type = NLA_NUL_STRING, }, +}; + +const struct nla_policy nfsd_fslocation_nl_policy[NFSD_A_FSLOCATION_PATH + 1] = { + [NFSD_A_FSLOCATION_HOST] = { .type = NLA_NUL_STRING, }, + [NFSD_A_FSLOCATION_PATH] = { .type = NLA_NUL_STRING, }, +}; + +const struct nla_policy nfsd_fslocations_nl_policy[NFSD_A_FSLOCATIONS_LOCATION + 1] = { + [NFSD_A_FSLOCATIONS_LOCATION] = NLA_POLICY_NESTED(nfsd_fslocation_nl_policy), +}; + +const struct nla_policy nfsd_sock_nl_policy[NFSD_A_SOCK_TRANSPORT_NAME + 1] = { + [NFSD_A_SOCK_ADDR] = { .type = NLA_BINARY, }, + [NFSD_A_SOCK_TRANSPORT_NAME] = { .type = NLA_NUL_STRING, }, +}; + +const struct nla_policy nfsd_svc_export_nl_policy[NFSD_A_SVC_EXPORT_FSID + 1] = { + [NFSD_A_SVC_EXPORT_SEQNO] = { .type = NLA_U64, }, + [NFSD_A_SVC_EXPORT_CLIENT] = { .type = NLA_NUL_STRING, }, + [NFSD_A_SVC_EXPORT_PATH] = { .type = NLA_NUL_STRING, }, + [NFSD_A_SVC_EXPORT_NEGATIVE] = { .type = NLA_FLAG, }, + [NFSD_A_SVC_EXPORT_EXPIRY] = { .type = NLA_U64, }, + [NFSD_A_SVC_EXPORT_ANON_UID] = { .type = NLA_U32, }, + [NFSD_A_SVC_EXPORT_ANON_GID] = { .type = NLA_U32, }, + [NFSD_A_SVC_EXPORT_FSLOCATIONS] = NLA_POLICY_NESTED(nfsd_fslocations_nl_policy), + [NFSD_A_SVC_EXPORT_UUID] = { .type = NLA_BINARY, }, + [NFSD_A_SVC_EXPORT_SECINFO] = NLA_POLICY_NESTED(nfsd_auth_flavor_nl_policy), + [NFSD_A_SVC_EXPORT_XPRTSEC] = NLA_POLICY_MASK(NLA_U32, 0x7), + [NFSD_A_SVC_EXPORT_FLAGS] = NLA_POLICY_MASK(NLA_U32, 0x3ffff), + [NFSD_A_SVC_EXPORT_FSID] = { .type = NLA_S32, }, +}; + +const struct nla_policy nfsd_version_nl_policy[NFSD_A_VERSION_ENABLED + 1] = { + [NFSD_A_VERSION_MAJOR] = { .type = NLA_U32, }, + [NFSD_A_VERSION_MINOR] = { .type = NLA_U32, }, + [NFSD_A_VERSION_ENABLED] = { .type = NLA_FLAG, }, +}; + +/* NFSD_CMD_THREADS_SET - do */ +static const struct nla_policy nfsd_threads_set_nl_policy[NFSD_A_SERVER_FH_KEY + 1] = { + [NFSD_A_SERVER_THREADS] = { .type = NLA_U32, }, + [NFSD_A_SERVER_GRACETIME] = { .type = NLA_U32, }, + [NFSD_A_SERVER_LEASETIME] = { .type = NLA_U32, }, + [NFSD_A_SERVER_SCOPE] = { .type = NLA_NUL_STRING, }, + [NFSD_A_SERVER_MIN_THREADS] = { .type = NLA_U32, }, + [NFSD_A_SERVER_FH_KEY] = NLA_POLICY_EXACT_LEN(16), +}; + +/* NFSD_CMD_VERSION_SET - do */ +static const struct nla_policy nfsd_version_set_nl_policy[NFSD_A_SERVER_PROTO_VERSION + 1] = { + [NFSD_A_SERVER_PROTO_VERSION] = NLA_POLICY_NESTED(nfsd_version_nl_policy), +}; + +/* NFSD_CMD_LISTENER_SET - do */ +static const struct nla_policy nfsd_listener_set_nl_policy[NFSD_A_SERVER_SOCK_ADDR + 1] = { + [NFSD_A_SERVER_SOCK_ADDR] = NLA_POLICY_NESTED(nfsd_sock_nl_policy), +}; + +/* NFSD_CMD_POOL_MODE_SET - do */ +static const struct nla_policy nfsd_pool_mode_set_nl_policy[NFSD_A_POOL_MODE_MODE + 1] = { + [NFSD_A_POOL_MODE_MODE] = { .type = NLA_NUL_STRING, }, +}; + +/* NFSD_CMD_SVC_EXPORT_SET_REQS - do */ +static const struct nla_policy nfsd_svc_export_set_reqs_nl_policy[NFSD_A_SVC_EXPORT_REQS_REQUESTS + 1] = { + [NFSD_A_SVC_EXPORT_REQS_REQUESTS] = NLA_POLICY_NESTED(nfsd_svc_export_nl_policy), +}; + +/* NFSD_CMD_EXPKEY_SET_REQS - do */ +static const struct nla_policy nfsd_expkey_set_reqs_nl_policy[NFSD_A_EXPKEY_REQS_REQUESTS + 1] = { + [NFSD_A_EXPKEY_REQS_REQUESTS] = NLA_POLICY_NESTED(nfsd_expkey_nl_policy), +}; + +/* NFSD_CMD_CACHE_FLUSH - do */ +static const struct nla_policy nfsd_cache_flush_nl_policy[NFSD_A_CACHE_FLUSH_MASK + 1] = { + [NFSD_A_CACHE_FLUSH_MASK] = NLA_POLICY_MASK(NLA_U32, 0x3), +}; + +/* NFSD_CMD_UNLOCK_IP - do */ +static const struct nla_policy nfsd_unlock_ip_nl_policy[NFSD_A_UNLOCK_IP_ADDRESS + 1] = { + [NFSD_A_UNLOCK_IP_ADDRESS] = NLA_POLICY_MIN_LEN(16), +}; + +/* NFSD_CMD_UNLOCK_FILESYSTEM - do */ +static const struct nla_policy nfsd_unlock_filesystem_nl_policy[NFSD_A_UNLOCK_FILESYSTEM_PATH + 1] = { + [NFSD_A_UNLOCK_FILESYSTEM_PATH] = { .type = NLA_NUL_STRING, }, +}; + +/* NFSD_CMD_UNLOCK_EXPORT - do */ +static const struct nla_policy nfsd_unlock_export_nl_policy[NFSD_A_UNLOCK_EXPORT_PATH + 1] = { + [NFSD_A_UNLOCK_EXPORT_PATH] = { .type = NLA_NUL_STRING, }, +}; + +/* Ops table for nfsd */ +static const struct genl_split_ops nfsd_nl_ops[] = { + { + .cmd = NFSD_CMD_RPC_STATUS_GET, + .dumpit = nfsd_nl_rpc_status_get_dumpit, + .flags = GENL_CMD_CAP_DUMP, + }, + { + .cmd = NFSD_CMD_THREADS_SET, + .doit = nfsd_nl_threads_set_doit, + .policy = nfsd_threads_set_nl_policy, + .maxattr = NFSD_A_SERVER_FH_KEY, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = NFSD_CMD_THREADS_GET, + .doit = nfsd_nl_threads_get_doit, + .flags = GENL_CMD_CAP_DO, + }, + { + .cmd = NFSD_CMD_VERSION_SET, + .doit = nfsd_nl_version_set_doit, + .policy = nfsd_version_set_nl_policy, + .maxattr = NFSD_A_SERVER_PROTO_VERSION, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = NFSD_CMD_VERSION_GET, + .doit = nfsd_nl_version_get_doit, + .flags = GENL_CMD_CAP_DO, + }, + { + .cmd = NFSD_CMD_LISTENER_SET, + .doit = nfsd_nl_listener_set_doit, + .policy = nfsd_listener_set_nl_policy, + .maxattr = NFSD_A_SERVER_SOCK_ADDR, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = NFSD_CMD_LISTENER_GET, + .doit = nfsd_nl_listener_get_doit, + .flags = GENL_CMD_CAP_DO, + }, + { + .cmd = NFSD_CMD_POOL_MODE_SET, + .doit = nfsd_nl_pool_mode_set_doit, + .policy = nfsd_pool_mode_set_nl_policy, + .maxattr = NFSD_A_POOL_MODE_MODE, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = NFSD_CMD_POOL_MODE_GET, + .doit = nfsd_nl_pool_mode_get_doit, + .flags = GENL_CMD_CAP_DO, + }, + { + .cmd = NFSD_CMD_SVC_EXPORT_GET_REQS, + .dumpit = nfsd_nl_svc_export_get_reqs_dumpit, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DUMP, + }, + { + .cmd = NFSD_CMD_SVC_EXPORT_SET_REQS, + .doit = nfsd_nl_svc_export_set_reqs_doit, + .policy = nfsd_svc_export_set_reqs_nl_policy, + .maxattr = NFSD_A_SVC_EXPORT_REQS_REQUESTS, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = NFSD_CMD_EXPKEY_GET_REQS, + .dumpit = nfsd_nl_expkey_get_reqs_dumpit, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DUMP, + }, + { + .cmd = NFSD_CMD_EXPKEY_SET_REQS, + .doit = nfsd_nl_expkey_set_reqs_doit, + .policy = nfsd_expkey_set_reqs_nl_policy, + .maxattr = NFSD_A_EXPKEY_REQS_REQUESTS, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = NFSD_CMD_CACHE_FLUSH, + .doit = nfsd_nl_cache_flush_doit, + .policy = nfsd_cache_flush_nl_policy, + .maxattr = NFSD_A_CACHE_FLUSH_MASK, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = NFSD_CMD_UNLOCK_IP, + .doit = nfsd_nl_unlock_ip_doit, + .policy = nfsd_unlock_ip_nl_policy, + .maxattr = NFSD_A_UNLOCK_IP_ADDRESS, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = NFSD_CMD_UNLOCK_FILESYSTEM, + .doit = nfsd_nl_unlock_filesystem_doit, + .policy = nfsd_unlock_filesystem_nl_policy, + .maxattr = NFSD_A_UNLOCK_FILESYSTEM_PATH, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = NFSD_CMD_UNLOCK_EXPORT, + .doit = nfsd_nl_unlock_export_doit, + .policy = nfsd_unlock_export_nl_policy, + .maxattr = NFSD_A_UNLOCK_EXPORT_PATH, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = NFSD_CMD_SERVER_STATS_GET, + .dumpit = nfsd_nl_server_stats_get_dumpit, + .flags = GENL_CMD_CAP_DUMP, + }, +}; + +static const struct genl_multicast_group nfsd_nl_mcgrps[] = { + [NFSD_NLGRP_NONE] = { "none", }, + [NFSD_NLGRP_EXPORTD] = { "exportd", }, +}; + +struct genl_family nfsd_nl_family __ro_after_init = { + .name = NFSD_FAMILY_NAME, + .version = NFSD_FAMILY_VERSION, + .netnsok = true, + .parallel_ops = true, + .module = THIS_MODULE, + .split_ops = nfsd_nl_ops, + .n_split_ops = ARRAY_SIZE(nfsd_nl_ops), + .mcgrps = nfsd_nl_mcgrps, + .n_mcgrps = ARRAY_SIZE(nfsd_nl_mcgrps), +}; diff --git a/fs/nfsd/netlink.h b/fs/nfsd/netlink.h new file mode 100644 index 000000000..027e2953d --- /dev/null +++ b/fs/nfsd/netlink.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */ +/* Do not edit directly, auto-generated from: */ +/* Documentation/netlink/specs/nfsd.yaml */ +/* YNL-GEN kernel header */ +/* To regenerate run: tools/net/ynl/ynl-regen.sh */ + +#ifndef _LINUX_NFSD_GEN_H +#define _LINUX_NFSD_GEN_H + +#include +#include + +#include + +/* Common nested types */ +extern const struct nla_policy nfsd_auth_flavor_nl_policy[NFSD_A_AUTH_FLAVOR_FLAGS + 1]; +extern const struct nla_policy nfsd_expkey_nl_policy[NFSD_A_EXPKEY_PATH + 1]; +extern const struct nla_policy nfsd_fslocation_nl_policy[NFSD_A_FSLOCATION_PATH + 1]; +extern const struct nla_policy nfsd_fslocations_nl_policy[NFSD_A_FSLOCATIONS_LOCATION + 1]; +extern const struct nla_policy nfsd_sock_nl_policy[NFSD_A_SOCK_TRANSPORT_NAME + 1]; +extern const struct nla_policy nfsd_svc_export_nl_policy[NFSD_A_SVC_EXPORT_FSID + 1]; +extern const struct nla_policy nfsd_version_nl_policy[NFSD_A_VERSION_ENABLED + 1]; + +int nfsd_nl_rpc_status_get_dumpit(struct sk_buff *skb, + struct netlink_callback *cb); +int nfsd_nl_threads_set_doit(struct sk_buff *skb, struct genl_info *info); +int nfsd_nl_threads_get_doit(struct sk_buff *skb, struct genl_info *info); +int nfsd_nl_version_set_doit(struct sk_buff *skb, struct genl_info *info); +int nfsd_nl_version_get_doit(struct sk_buff *skb, struct genl_info *info); +int nfsd_nl_listener_set_doit(struct sk_buff *skb, struct genl_info *info); +int nfsd_nl_listener_get_doit(struct sk_buff *skb, struct genl_info *info); +int nfsd_nl_pool_mode_set_doit(struct sk_buff *skb, struct genl_info *info); +int nfsd_nl_pool_mode_get_doit(struct sk_buff *skb, struct genl_info *info); +int nfsd_nl_svc_export_get_reqs_dumpit(struct sk_buff *skb, + struct netlink_callback *cb); +int nfsd_nl_svc_export_set_reqs_doit(struct sk_buff *skb, + struct genl_info *info); +int nfsd_nl_expkey_get_reqs_dumpit(struct sk_buff *skb, + struct netlink_callback *cb); +int nfsd_nl_expkey_set_reqs_doit(struct sk_buff *skb, struct genl_info *info); +int nfsd_nl_cache_flush_doit(struct sk_buff *skb, struct genl_info *info); +int nfsd_nl_unlock_ip_doit(struct sk_buff *skb, struct genl_info *info); +int nfsd_nl_unlock_filesystem_doit(struct sk_buff *skb, struct genl_info *info); +int nfsd_nl_unlock_export_doit(struct sk_buff *skb, struct genl_info *info); +int nfsd_nl_server_stats_get_dumpit(struct sk_buff *skb, + struct netlink_callback *cb); + +enum { + NFSD_NLGRP_NONE, + NFSD_NLGRP_EXPORTD, +}; + +extern struct genl_family nfsd_nl_family; + +#endif /* _LINUX_NFSD_GEN_H */ diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h new file mode 100644 index 000000000..71eebfea0 --- /dev/null +++ b/fs/nfsd/netns.h @@ -0,0 +1,268 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * per net namespace data structures for nfsd + * + * Copyright (C) 2012, Jeff Layton + */ + +#ifndef __NFSD_NETNS_H__ +#define __NFSD_NETNS_H__ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* Hash tables for nfs4_clientid state */ +#define CLIENT_HASH_BITS 4 +#define CLIENT_HASH_SIZE (1 << CLIENT_HASH_BITS) +#define CLIENT_HASH_MASK (CLIENT_HASH_SIZE - 1) + +#define SESSION_HASH_SIZE 512 + +struct cld_net; +struct nfsd_net_cb; +struct nfsd4_client_tracking_ops; + +enum nfsd_net_flag { + NFSD_NET_GRACE_ENDED, + NFSD_NET_GRACE_END_FORCED, + NFSD_NET_IN_GRACE, + NFSD_NET_SOMEBODY_RECLAIMED, + NFSD_NET_TRACK_RECLAIM_COMPLETES, + NFSD_NET_UP, + NFSD_NET_LOCKD_UP, +}; + +enum { + /* cache misses due only to checksum comparison failures */ + NFSD_STATS_PAYLOAD_MISSES, + /* amount of memory (in bytes) currently consumed by the DRC */ + NFSD_STATS_DRC_MEM_USAGE, + NFSD_STATS_RC_HITS, /* repcache hits */ + NFSD_STATS_RC_MISSES, /* repcache misses */ + NFSD_STATS_RC_NOCACHE, /* uncached reqs */ + NFSD_STATS_FH_STALE, /* FH stale error */ + NFSD_STATS_IO_READ, /* bytes returned to read requests */ + NFSD_STATS_IO_WRITE, /* bytes passed in write requests */ +#ifdef CONFIG_NFSD_V4 + NFSD_STATS_FIRST_NFS4_OP, /* count of individual nfsv4 operations */ + NFSD_STATS_LAST_NFS4_OP = NFSD_STATS_FIRST_NFS4_OP + LAST_NFS4_OP, +#define NFSD_STATS_NFS4_OP(op) (NFSD_STATS_FIRST_NFS4_OP + (op)) +#endif + NFSD_STATS_COUNTERS_NUM +}; + +/* + * Per-netns NFSv4 callback (backchannel) per-operation counters, indexed + * directly by RFC 8881 callback opcode (OP_CB_GETATTR..OP_CB_OFFLOAD). + */ +#define NFSD_STATS_CB_OPS_NUM (OP_CB_OFFLOAD + 1) + +/* + * Represents a nfsd "container". With respect to nfsv4 state tracking, the + * fields of interest are the *_id_hashtbls and the *_name_tree. These track + * the nfs4_client objects by either short or long form clientid. + * + * Each nfsd_net runs a nfs4_laundromat workqueue job when necessary to clean + * up expired clients and delegations within the container. + */ +struct nfsd_net { + struct cld_net *cld_net; + + struct cache_detail *svc_expkey_cache; + struct cache_detail *svc_export_cache; + + struct cache_detail *idtoname_cache; + struct cache_detail *nametoid_cache; + + struct lock_manager nfsd4_manager; + unsigned long flags; + time64_t boot_time; + time64_t boot_time_bt; /* same instant in CLOCK_BOOTTIME */ + + struct dentry *nfsd_client_dir; + + /* + * reclaim_str_hashtbl[] holds known client info from previous reset/reboot + * used in reboot/reset lease grace period processing + * + * conf_id_hashtbl[], and conf_name_tree hold confirmed + * setclientid_confirmed info. + * + * unconf_str_hastbl[] and unconf_name_tree hold unconfirmed + * setclientid info. + */ + struct list_head *reclaim_str_hashtbl; + int reclaim_str_hashtbl_size; + struct rw_semaphore reclaim_str_hashtbl_lock; + struct list_head *conf_id_hashtbl; + struct rb_root conf_name_tree; + struct list_head *unconf_id_hashtbl; + struct rb_root unconf_name_tree; + struct list_head *sessionid_hashtbl; + /* + * client_lru holds client queue ordered by nfs4_client.cl_time + * for lease renewal. + * + * close_lru holds (open) stateowner queue ordered by nfs4_stateowner.so_time + * for last close replay. + * + * reclaim_str_hashtbl[], reclaim_str_hashtbl_size are protected by + * reclaim_str_hashtbl_lock. + * + * All of the remaining fields are protected by the client_lock. + */ + struct list_head client_lru; + struct list_head close_lru; + + /* protects del_recall_lru and delegation hash/unhash; + * nests outside client_lock */ + spinlock_t deleg_lock ____cacheline_aligned; + struct list_head del_recall_lru; + + /* protected by blocked_locks_lock */ + struct list_head blocked_locks_lru; + + struct delayed_work laundromat_work; + + /* client_lock protects the client lru list and session hash + * table; nests inside deleg_lock */ + spinlock_t client_lock; + + /* protects blocked_locks_lru */ + spinlock_t blocked_locks_lock; + + struct file *rec_file; + const struct nfsd4_client_tracking_ops *client_tracking_ops; + + time64_t nfsd4_lease; + time64_t nfsd4_grace; + + atomic_t nr_reclaim_complete; + + seqlock_t writeverf_lock; + unsigned char writeverf[8]; + + /* + * Minimum number of threads to run per pool. If 0 then the + * min == max requested number of threads. + */ + unsigned int min_threads; + + u32 clientid_base; + u32 clientid_counter; + u32 clverifier_counter; + + struct svc_info nfsd_info; +#define nfsd_serv nfsd_info.serv + + struct percpu_ref nfsd_net_ref; + struct completion nfsd_net_confirm_done; + struct completion nfsd_net_free_done; + + /* + * clientid and stateid data for construction of net unique COPY + * stateids. + */ + u32 s2s_cp_cl_id; + struct idr s2s_cp_stateids; + spinlock_t s2s_cp_lock; + atomic_t pending_async_copies; + + /* + * Version information + */ + bool nfsd_versions[NFSD_MAXVERS + 1]; + bool nfsd4_minorversions[NFSD_SUPPORTED_MINOR_VERSION + 1]; + + /* + * Duplicate reply cache + */ + struct nfsd_drc_bucket *drc_hashtbl; + + /* max number of entries allowed in the cache */ + unsigned int max_drc_entries; + + /* number of significant bits in the hash value */ + unsigned int maskbits; + unsigned int drc_hashsize; + + /* + * Stats and other tracking of on the duplicate reply cache. + * The longest_chain* fields are modified with only the per-bucket + * cache lock, which isn't really safe and should be fixed if we want + * these statistics to be completely accurate. + */ + + /* total number of entries */ + atomic_t num_drc_entries; + + /* Per-netns stats counters */ + struct percpu_counter counter[NFSD_STATS_COUNTERS_NUM]; + +#ifdef CONFIG_NFSD_V4 + /* Per-netns NFSv4 callback (backchannel) per-operation counters */ + struct percpu_counter cb_counter[NFSD_STATS_CB_OPS_NUM]; +#endif + + /* sunrpc svc stats */ + struct svc_stat nfsd_svcstats; + + /* longest hash chain seen */ + unsigned int longest_chain; + + /* size of cache when we saw the longest hash chain */ + unsigned int longest_chain_cachesize; + + struct shrinker *nfsd_reply_cache_shrinker; + + /* tracking server-to-server copy mounts */ + spinlock_t nfsd_ssc_lock; + struct list_head nfsd_ssc_mount_list; + wait_queue_head_t nfsd_ssc_waitq; + + /* utsname taken from the process that starts the server */ + char nfsd_name[UNX_MAXNODENAME+1]; + + spinlock_t fcache_dispose_lock; + struct list_head fcache_dispose_list; + + siphash_key_t siphash_key; + + atomic_t nfs4_client_count; + int nfs4_max_clients; + + atomic_t nfsd_courtesy_clients; + struct shrinker *nfsd_client_shrinker; + struct work_struct nfsd_shrinker_work; + + /* last time an admin-revoke happened for NFSv4.0 */ + time64_t nfs40_last_revoke; + +#if IS_ENABLED(CONFIG_NFS_LOCALIO) + /* Local clients to be invalidated when net is shut down */ + spinlock_t local_clients_lock; + struct list_head local_clients; +#endif + siphash_key_t *fh_key; + + struct nfsd_net_cb *nfsd_cb; +}; + +/* Simple check to find out if a given net was properly initialized */ +#define nfsd_netns_ready(nn) ((nn)->sessionid_hashtbl) + +extern bool nfsd_support_version(int vers); +extern unsigned int nfsd_net_id; + +bool nfsd_net_try_get(struct net *net); +void nfsd_net_put(struct net *net); + +void nfsd_copy_write_verifier(__be32 verf[2], struct nfsd_net *nn); +void nfsd_reset_write_verifier(struct nfsd_net *nn); +#endif /* __NFSD_NETNS_H__ */ diff --git a/fs/nfsd/nfs2acl.c b/fs/nfsd/nfs2acl.c new file mode 100644 index 000000000..190f5a001 --- /dev/null +++ b/fs/nfsd/nfs2acl.c @@ -0,0 +1,397 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Process version 2 NFSACL requests. + * + * Copyright (C) 2002-2003 Andreas Gruenbacher + */ + +#include "nfsd.h" +/* FIXME: nfsacl.h is a broken header */ +#include +#include +#include "cache.h" +#include "xdr3.h" +#include "vfs.h" + +#define NFSDDBG_FACILITY NFSDDBG_PROC + +/* + * NULL call. + */ +static __be32 +nfsacld_proc_null(struct svc_rqst *rqstp) +{ + return rpc_success; +} + +/* + * Get the Access and/or Default ACL of a file. + */ +static __be32 nfsacld_proc_getacl(struct svc_rqst *rqstp) +{ + struct nfsd3_getaclargs *argp = rqstp->rq_argp; + struct nfsd3_getaclres *resp = rqstp->rq_resp; + struct posix_acl *acl; + struct inode *inode; + svc_fh *fh; + + dprintk("nfsd: GETACL(2acl) %s\n", SVCFH_fmt(&argp->fh)); + + fh = fh_copy(&resp->fh, &argp->fh); + resp->status = fh_verify(rqstp, &resp->fh, 0, NFSD_MAY_NOP); + if (resp->status != nfs_ok) + goto out; + + inode = d_inode(fh->fh_dentry); + + if (argp->mask & ~NFS_ACL_MASK) { + resp->status = nfserr_io; + goto out; + } + resp->mask = argp->mask; + + resp->status = fh_getattr(fh, &resp->stat); + if (resp->status != nfs_ok) + goto out; + + if (resp->mask & (NFS_ACL|NFS_ACLCNT)) { + acl = get_inode_acl(inode, ACL_TYPE_ACCESS); + if (acl == NULL) { + /* Solaris returns the inode's minimum ACL. */ + acl = posix_acl_from_mode(inode->i_mode, GFP_KERNEL); + } + if (IS_ERR(acl)) { + resp->status = nfserrno(PTR_ERR(acl)); + goto fail; + } + resp->acl_access = acl; + } + if (resp->mask & (NFS_DFACL|NFS_DFACLCNT)) { + /* Check how Solaris handles requests for the Default ACL + of a non-directory! */ + acl = get_inode_acl(inode, ACL_TYPE_DEFAULT); + if (IS_ERR(acl)) { + resp->status = nfserrno(PTR_ERR(acl)); + goto fail; + } + resp->acl_default = acl; + } + + /* resp->acl_{access,default} are released in nfssvc_release_getacl. */ +out: + return rpc_success; + +fail: + posix_acl_release(resp->acl_access); + posix_acl_release(resp->acl_default); + resp->acl_access = NULL; + resp->acl_default = NULL; + goto out; +} + +/* + * Set the Access and/or Default ACL of a file. + */ +static __be32 nfsacld_proc_setacl(struct svc_rqst *rqstp) +{ + struct nfsd3_setaclargs *argp = rqstp->rq_argp; + struct nfsd_attrstat *resp = rqstp->rq_resp; + struct inode *inode; + svc_fh *fh; + int error; + + dprintk("nfsd: SETACL(2acl) %s\n", SVCFH_fmt(&argp->fh)); + + fh = fh_copy(&resp->fh, &argp->fh); + resp->status = fh_verify(rqstp, &resp->fh, 0, NFSD_MAY_SATTR); + if (resp->status != nfs_ok) + goto out; + + inode = d_inode(fh->fh_dentry); + + error = fh_want_write(fh); + if (error) + goto out_errno; + + inode_lock(inode); + + error = 0; + if (argp->mask & NFS_ACL) { + error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, + ACL_TYPE_ACCESS, argp->acl_access); + if (error) + goto out_drop_lock; + } + if (argp->mask & NFS_DFACL) { + error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, + ACL_TYPE_DEFAULT, argp->acl_default); + if (error) + goto out_drop_lock; + } + + inode_unlock(inode); + + fh_drop_write(fh); + + resp->status = fh_getattr(fh, &resp->stat); + +out: + /* argp->acl_{access,default} are released in nfsaclsvc_release_setacl. */ + return rpc_success; + +out_drop_lock: + inode_unlock(inode); + fh_drop_write(fh); +out_errno: + resp->status = nfserrno(error); + goto out; +} + +/* + * Check file attributes + */ +static __be32 nfsacld_proc_getattr(struct svc_rqst *rqstp) +{ + struct nfsd_fhandle *argp = rqstp->rq_argp; + struct nfsd_attrstat *resp = rqstp->rq_resp; + + dprintk("nfsd: GETATTR %s\n", SVCFH_fmt(&argp->fh)); + + fh_copy(&resp->fh, &argp->fh); + resp->status = fh_verify(rqstp, &resp->fh, 0, NFSD_MAY_NOP); + if (resp->status != nfs_ok) + goto out; + resp->status = fh_getattr(&resp->fh, &resp->stat); +out: + return rpc_success; +} + +/* + * Check file access + */ +static __be32 nfsacld_proc_access(struct svc_rqst *rqstp) +{ + struct nfsd3_accessargs *argp = rqstp->rq_argp; + struct nfsd3_accessres *resp = rqstp->rq_resp; + + dprintk("nfsd: ACCESS(2acl) %s 0x%x\n", + SVCFH_fmt(&argp->fh), + argp->access); + + fh_copy(&resp->fh, &argp->fh); + resp->access = argp->access; + resp->status = nfsd_access(rqstp, &resp->fh, &resp->access, NULL); + if (resp->status != nfs_ok) + goto out; + resp->status = fh_getattr(&resp->fh, &resp->stat); +out: + return rpc_success; +} + +/* + * XDR decode functions + */ + +static bool +nfsaclsvc_decode_getaclargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_getaclargs *argp = rqstp->rq_argp; + + if (!svcxdr_decode_fhandle(xdr, &argp->fh)) + return false; + if (xdr_stream_decode_u32(xdr, &argp->mask) < 0) + return false; + + return true; +} + +static bool +nfsaclsvc_decode_setaclargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_setaclargs *argp = rqstp->rq_argp; + + if (!svcxdr_decode_fhandle(xdr, &argp->fh)) + return false; + if (xdr_stream_decode_u32(xdr, &argp->mask) < 0) + return false; + if (argp->mask & ~NFS_ACL_MASK) + return false; + if (!nfs_stream_decode_acl(xdr, NULL, (argp->mask & NFS_ACL) ? + &argp->acl_access : NULL)) + return false; + if (!nfs_stream_decode_acl(xdr, NULL, (argp->mask & NFS_DFACL) ? + &argp->acl_default : NULL)) + return false; + + return true; +} + +static bool +nfsaclsvc_decode_accessargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_accessargs *args = rqstp->rq_argp; + + if (!svcxdr_decode_fhandle(xdr, &args->fh)) + return false; + if (xdr_stream_decode_u32(xdr, &args->access) < 0) + return false; + + return true; +} + +/* + * XDR encode functions + */ + +/* GETACL */ +static bool +nfsaclsvc_encode_getaclres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_getaclres *resp = rqstp->rq_resp; + struct dentry *dentry = resp->fh.fh_dentry; + struct inode *inode; + + if (!svcxdr_encode_stat(xdr, resp->status)) + return false; + switch (resp->status) { + case nfs_ok: + inode = d_inode(dentry); + if (!svcxdr_encode_fattr(rqstp, xdr, &resp->fh, &resp->stat)) + return false; + if (xdr_stream_encode_u32(xdr, resp->mask) < 0) + return false; + if (!nfs_stream_encode_acl(xdr, inode, resp->acl_access, + resp->mask & NFS_ACL, 0)) + return false; + if (!nfs_stream_encode_acl(xdr, inode, resp->acl_default, + resp->mask & NFS_DFACL, NFS_ACL_DEFAULT)) + return false; + break; + } + + return true; +} + +/* ACCESS */ +static bool +nfsaclsvc_encode_accessres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_accessres *resp = rqstp->rq_resp; + + if (!svcxdr_encode_stat(xdr, resp->status)) + return false; + switch (resp->status) { + case nfs_ok: + if (!svcxdr_encode_fattr(rqstp, xdr, &resp->fh, &resp->stat)) + return false; + if (xdr_stream_encode_u32(xdr, resp->access) < 0) + return false; + break; + } + + return true; +} + +/* + * XDR release functions + */ +static void nfsaclsvc_release_getacl(struct svc_rqst *rqstp) +{ + struct nfsd3_getaclres *resp = rqstp->rq_resp; + + fh_put(&resp->fh); + posix_acl_release(resp->acl_access); + posix_acl_release(resp->acl_default); +} + +static void nfsaclsvc_release_access(struct svc_rqst *rqstp) +{ + struct nfsd3_accessres *resp = rqstp->rq_resp; + + fh_put(&resp->fh); +} + +static void nfsaclsvc_release_setacl(struct svc_rqst *rqstp) +{ + struct nfsd3_setaclargs *argp = rqstp->rq_argp; + struct nfsd_attrstat *resp = rqstp->rq_resp; + + fh_put(&resp->fh); + posix_acl_release(argp->acl_access); + posix_acl_release(argp->acl_default); +} + +#define ST 1 /* status*/ +#define AT 21 /* attributes */ +#define pAT (1+AT) /* post attributes - conditional */ +#define ACL (1+NFS_ACL_MAX_ENTRIES*3) /* Access Control List */ + +static const struct svc_procedure nfsd_acl_procedures2[5] = { + [ACLPROC2_NULL] = { + .pc_func = nfsacld_proc_null, + .pc_decode = nfssvc_decode_voidarg, + .pc_encode = nfssvc_encode_voidres, + .pc_argsize = sizeof(struct nfsd_voidargs), + .pc_argzero = sizeof(struct nfsd_voidargs), + .pc_ressize = sizeof(struct nfsd_voidres), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = ST, + .pc_name = "NULL", + }, + [ACLPROC2_GETACL] = { + .pc_func = nfsacld_proc_getacl, + .pc_decode = nfsaclsvc_decode_getaclargs, + .pc_encode = nfsaclsvc_encode_getaclres, + .pc_release = nfsaclsvc_release_getacl, + .pc_argsize = sizeof(struct nfsd3_getaclargs), + .pc_argzero = sizeof(struct nfsd3_getaclargs), + .pc_ressize = sizeof(struct nfsd3_getaclres), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = ST+1+2*(1+ACL), + .pc_name = "GETACL", + }, + [ACLPROC2_SETACL] = { + .pc_func = nfsacld_proc_setacl, + .pc_decode = nfsaclsvc_decode_setaclargs, + .pc_encode = nfssvc_encode_attrstatres, + .pc_release = nfsaclsvc_release_setacl, + .pc_argsize = sizeof(struct nfsd3_setaclargs), + .pc_argzero = sizeof(struct nfsd3_setaclargs), + .pc_ressize = sizeof(struct nfsd_attrstat), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = ST+AT, + .pc_name = "SETACL", + }, + [ACLPROC2_GETATTR] = { + .pc_func = nfsacld_proc_getattr, + .pc_decode = nfssvc_decode_fhandleargs, + .pc_encode = nfssvc_encode_attrstatres, + .pc_release = nfssvc_release_attrstat, + .pc_argsize = sizeof(struct nfsd_fhandle), + .pc_argzero = sizeof(struct nfsd_fhandle), + .pc_ressize = sizeof(struct nfsd_attrstat), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = ST+AT, + .pc_name = "GETATTR", + }, + [ACLPROC2_ACCESS] = { + .pc_func = nfsacld_proc_access, + .pc_decode = nfsaclsvc_decode_accessargs, + .pc_encode = nfsaclsvc_encode_accessres, + .pc_release = nfsaclsvc_release_access, + .pc_argsize = sizeof(struct nfsd3_accessargs), + .pc_argzero = sizeof(struct nfsd3_accessargs), + .pc_ressize = sizeof(struct nfsd3_accessres), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = ST+AT+1, + .pc_name = "SETATTR", + }, +}; + +const struct svc_version nfsd_acl_version2 = { + .vs_vers = 2, + .vs_nproc = ARRAY_SIZE(nfsd_acl_procedures2), + .vs_proc = nfsd_acl_procedures2, + .vs_dispatch = nfsd_dispatch, + .vs_xdrsize = NFS3_SVC_XDRSIZE, +}; diff --git a/fs/nfsd/nfs3acl.c b/fs/nfsd/nfs3acl.c new file mode 100644 index 000000000..6b6b289db --- /dev/null +++ b/fs/nfsd/nfs3acl.c @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Process version 3 NFSACL requests. + * + * Copyright (C) 2002-2003 Andreas Gruenbacher + */ + +#include "nfsd.h" +/* FIXME: nfsacl.h is a broken header */ +#include +#include +#include "cache.h" +#include "xdr3.h" +#include "vfs.h" + +/* + * NULL call. + */ +static __be32 +nfsd3_proc_null(struct svc_rqst *rqstp) +{ + return rpc_success; +} + +/* + * Get the Access and/or Default ACL of a file. + */ +static __be32 nfsd3_proc_getacl(struct svc_rqst *rqstp) +{ + struct nfsd3_getaclargs *argp = rqstp->rq_argp; + struct nfsd3_getaclres *resp = rqstp->rq_resp; + struct posix_acl *acl; + struct inode *inode; + svc_fh *fh; + + fh = fh_copy(&resp->fh, &argp->fh); + resp->status = fh_verify(rqstp, &resp->fh, 0, NFSD_MAY_NOP); + if (resp->status != nfs_ok) + goto out; + + inode = d_inode(fh->fh_dentry); + + if (argp->mask & ~NFS_ACL_MASK) { + resp->status = nfserr_inval; + goto out; + } + resp->mask = argp->mask; + + if (resp->mask & (NFS_ACL|NFS_ACLCNT)) { + acl = get_inode_acl(inode, ACL_TYPE_ACCESS); + if (acl == NULL) { + /* Solaris returns the inode's minimum ACL. */ + acl = posix_acl_from_mode(inode->i_mode, GFP_KERNEL); + } + if (IS_ERR(acl)) { + resp->status = nfserrno(PTR_ERR(acl)); + goto fail; + } + resp->acl_access = acl; + } + if (resp->mask & (NFS_DFACL|NFS_DFACLCNT)) { + /* Check how Solaris handles requests for the Default ACL + of a non-directory! */ + acl = get_inode_acl(inode, ACL_TYPE_DEFAULT); + if (IS_ERR(acl)) { + resp->status = nfserrno(PTR_ERR(acl)); + goto fail; + } + resp->acl_default = acl; + } + + /* resp->acl_{access,default} are released in nfs3svc_release_getacl. */ +out: + return rpc_success; + +fail: + posix_acl_release(resp->acl_access); + posix_acl_release(resp->acl_default); + resp->acl_access = NULL; + resp->acl_default = NULL; + goto out; +} + +/* + * Set the Access and/or Default ACL of a file. + */ +static __be32 nfsd3_proc_setacl(struct svc_rqst *rqstp) +{ + struct nfsd3_setaclargs *argp = rqstp->rq_argp; + struct nfsd3_attrstat *resp = rqstp->rq_resp; + struct inode *inode; + svc_fh *fh; + int error; + + fh = fh_copy(&resp->fh, &argp->fh); + resp->status = fh_verify(rqstp, &resp->fh, 0, NFSD_MAY_SATTR); + if (resp->status != nfs_ok) + goto out; + + inode = d_inode(fh->fh_dentry); + + error = fh_want_write(fh); + if (error) + goto out_errno; + + inode_lock(inode); + + error = 0; + if (argp->mask & NFS_ACL) { + error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, + ACL_TYPE_ACCESS, argp->acl_access); + if (error) + goto out_drop_lock; + } + if (argp->mask & NFS_DFACL) { + error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, + ACL_TYPE_DEFAULT, argp->acl_default); + } + +out_drop_lock: + inode_unlock(inode); + fh_drop_write(fh); +out_errno: + resp->status = nfserrno(error); +out: + /* argp->acl_{access,default} are released in nfs3svc_release_setacl. */ + return rpc_success; +} + +/* + * XDR decode functions + */ + +static bool +nfs3svc_decode_getaclargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_getaclargs *args = rqstp->rq_argp; + + if (!svcxdr_decode_nfs_fh3(xdr, &args->fh)) + return false; + if (xdr_stream_decode_u32(xdr, &args->mask) < 0) + return false; + + return true; +} + +static bool +nfs3svc_decode_setaclargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_setaclargs *argp = rqstp->rq_argp; + + if (!svcxdr_decode_nfs_fh3(xdr, &argp->fh)) + return false; + if (xdr_stream_decode_u32(xdr, &argp->mask) < 0) + return false; + if (argp->mask & ~NFS_ACL_MASK) + return false; + if (!nfs_stream_decode_acl(xdr, NULL, (argp->mask & NFS_ACL) ? + &argp->acl_access : NULL)) + return false; + if (!nfs_stream_decode_acl(xdr, NULL, (argp->mask & NFS_DFACL) ? + &argp->acl_default : NULL)) + return false; + + return true; +} + +/* + * XDR encode functions + */ + +/* GETACL */ +static bool +nfs3svc_encode_getaclres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_getaclres *resp = rqstp->rq_resp; + struct dentry *dentry = resp->fh.fh_dentry; + struct inode *inode; + + if (!svcxdr_encode_nfsstat3(xdr, resp->status)) + return false; + switch (resp->status) { + case nfs_ok: + inode = d_inode(dentry); + if (!svcxdr_encode_post_op_attr(rqstp, xdr, &resp->fh)) + return false; + if (xdr_stream_encode_u32(xdr, resp->mask) < 0) + return false; + + if (!nfs_stream_encode_acl(xdr, inode, resp->acl_access, + resp->mask & NFS_ACL, 0)) + return false; + if (!nfs_stream_encode_acl(xdr, inode, resp->acl_default, + resp->mask & NFS_DFACL, + NFS_ACL_DEFAULT)) + return false; + break; + default: + if (!svcxdr_encode_post_op_attr(rqstp, xdr, &resp->fh)) + return false; + } + + return true; +} + +/* SETACL */ +static bool +nfs3svc_encode_setaclres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_attrstat *resp = rqstp->rq_resp; + + return svcxdr_encode_nfsstat3(xdr, resp->status) && + svcxdr_encode_post_op_attr(rqstp, xdr, &resp->fh); +} + +/* + * XDR release functions + */ +static void nfs3svc_release_getacl(struct svc_rqst *rqstp) +{ + struct nfsd3_getaclres *resp = rqstp->rq_resp; + + fh_put(&resp->fh); + posix_acl_release(resp->acl_access); + posix_acl_release(resp->acl_default); +} + +static void nfs3svc_release_setacl(struct svc_rqst *rqstp) +{ + struct nfsd3_setaclargs *argp = rqstp->rq_argp; + struct nfsd3_attrstat *resp = rqstp->rq_resp; + + fh_put(&resp->fh); + posix_acl_release(argp->acl_access); + posix_acl_release(argp->acl_default); +} + +#define ST 1 /* status*/ +#define AT 21 /* attributes */ +#define pAT (1+AT) /* post attributes - conditional */ +#define ACL (1+NFS_ACL_MAX_ENTRIES*3) /* Access Control List */ + +static const struct svc_procedure nfsd_acl_procedures3[3] = { + [ACLPROC3_NULL] = { + .pc_func = nfsd3_proc_null, + .pc_decode = nfssvc_decode_voidarg, + .pc_encode = nfssvc_encode_voidres, + .pc_argsize = sizeof(struct nfsd_voidargs), + .pc_argzero = sizeof(struct nfsd_voidargs), + .pc_ressize = sizeof(struct nfsd_voidres), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = ST, + .pc_name = "NULL", + }, + [ACLPROC3_GETACL] = { + .pc_func = nfsd3_proc_getacl, + .pc_decode = nfs3svc_decode_getaclargs, + .pc_encode = nfs3svc_encode_getaclres, + .pc_release = nfs3svc_release_getacl, + .pc_argsize = sizeof(struct nfsd3_getaclargs), + .pc_argzero = sizeof(struct nfsd3_getaclargs), + .pc_ressize = sizeof(struct nfsd3_getaclres), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = ST+1+2*(1+ACL), + .pc_name = "GETACL", + }, + [ACLPROC3_SETACL] = { + .pc_func = nfsd3_proc_setacl, + .pc_decode = nfs3svc_decode_setaclargs, + .pc_encode = nfs3svc_encode_setaclres, + .pc_release = nfs3svc_release_setacl, + .pc_argsize = sizeof(struct nfsd3_setaclargs), + .pc_argzero = sizeof(struct nfsd3_setaclargs), + .pc_ressize = sizeof(struct nfsd3_attrstat), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = ST+pAT, + .pc_name = "SETACL", + }, +}; + +const struct svc_version nfsd_acl_version3 = { + .vs_vers = 3, + .vs_nproc = ARRAY_SIZE(nfsd_acl_procedures3), + .vs_proc = nfsd_acl_procedures3, + .vs_dispatch = nfsd_dispatch, + .vs_xdrsize = NFS3_SVC_XDRSIZE, +}; + diff --git a/fs/nfsd/nfs3proc.c b/fs/nfsd/nfs3proc.c new file mode 100644 index 000000000..0904d953d --- /dev/null +++ b/fs/nfsd/nfs3proc.c @@ -0,0 +1,1117 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Process version 3 NFS requests. + * + * Copyright (C) 1996, 1997, 1998 Olaf Kirch + */ + +#include +#include +#include +#include + +#include "cache.h" +#include "xdr3.h" +#include "vfs.h" +#include "filecache.h" +#include "trace.h" + +#define NFSDDBG_FACILITY NFSDDBG_PROC + +static int nfs3_ftypes[] = { + 0, /* NF3NON */ + S_IFREG, /* NF3REG */ + S_IFDIR, /* NF3DIR */ + S_IFBLK, /* NF3BLK */ + S_IFCHR, /* NF3CHR */ + S_IFLNK, /* NF3LNK */ + S_IFSOCK, /* NF3SOCK */ + S_IFIFO, /* NF3FIFO */ +}; + +/* + * Reject a client-supplied atime or mtime whose nanoseconds field is out + * of range. Such a value is well-formed on the wire but is not a valid + * timespec64, and storing it verbatim can corrupt on-disk timestamps. + * tv_nsec is a long, so it is cast to unsigned long (the same width) to + * catch both an over-large value and one that became negative when an + * out-of-range u32 wire nseconds was assigned to a 32-bit long. + */ +static bool nfsd3_time_in_range(const struct iattr *iap) +{ + if ((iap->ia_valid & ATTR_ATIME_SET) && + (unsigned long)iap->ia_atime.tv_nsec >= NSEC_PER_SEC) + return false; + if ((iap->ia_valid & ATTR_MTIME_SET) && + (unsigned long)iap->ia_mtime.tv_nsec >= NSEC_PER_SEC) + return false; + return true; +} + +static __be32 nfsd3_map_status(__be32 status) +{ + switch (status) { + case nfs_ok: + break; + case nfserr_nofilehandle: + status = nfserr_badhandle; + break; + case nfserr_wrongsec: + case nfserr_file_open: + status = nfserr_acces; + break; + case nfserr_symlink_not_dir: + status = nfserr_notdir; + break; + case nfserr_symlink: + case nfserr_wrong_type: + status = nfserr_inval; + break; + } + return status; +} + +/* + * NULL call. + */ +static __be32 +nfsd3_proc_null(struct svc_rqst *rqstp) +{ + return rpc_success; +} + +/* + * Get a file's attributes + */ +static __be32 +nfsd3_proc_getattr(struct svc_rqst *rqstp) +{ + struct nfsd_fhandle *argp = rqstp->rq_argp; + struct nfsd3_attrstat *resp = rqstp->rq_resp; + + trace_nfsd_vfs_getattr(rqstp, &argp->fh); + + fh_copy(&resp->fh, &argp->fh); + resp->status = fh_verify(rqstp, &resp->fh, 0, + NFSD_MAY_NOP | NFSD_MAY_BYPASS_GSS_ON_ROOT); + if (resp->status != nfs_ok) + goto out; + + resp->status = fh_getattr(&resp->fh, &resp->stat); +out: + resp->status = nfsd3_map_status(resp->status); + return rpc_success; +} + +/* + * Set a file's attributes + */ +static __be32 +nfsd3_proc_setattr(struct svc_rqst *rqstp) +{ + struct nfsd3_sattrargs *argp = rqstp->rq_argp; + struct nfsd3_attrstat *resp = rqstp->rq_resp; + struct nfsd_attrs attrs = { + .na_iattr = &argp->attrs, + }; + const struct timespec64 *guardtime = NULL; + + dprintk("nfsd: SETATTR(3) %s\n", + SVCFH_fmt(&argp->fh)); + + fh_copy(&resp->fh, &argp->fh); + if (!nfsd3_time_in_range(&argp->attrs)) { + resp->status = nfserr_inval; + goto out; + } + if (argp->check_guard) + guardtime = &argp->guardtime; + resp->status = nfsd_setattr(rqstp, &resp->fh, &attrs, guardtime); +out: + resp->status = nfsd3_map_status(resp->status); + return rpc_success; +} + +/* + * Look up a path name component + */ +static __be32 +nfsd3_proc_lookup(struct svc_rqst *rqstp) +{ + struct nfsd3_diropargs *argp = rqstp->rq_argp; + struct nfsd3_diropres *resp = rqstp->rq_resp; + + dprintk("nfsd: LOOKUP(3) %s %.*s\n", + SVCFH_fmt(&argp->fh), + argp->len, + argp->name); + + fh_copy(&resp->dirfh, &argp->fh); + fh_init(&resp->fh, NFS3_FHSIZE); + + resp->status = nfsd_lookup(rqstp, &resp->dirfh, + argp->name, argp->len, + &resp->fh); + resp->status = nfsd3_map_status(resp->status); + return rpc_success; +} + +/* + * Check file access + */ +static __be32 +nfsd3_proc_access(struct svc_rqst *rqstp) +{ + struct nfsd3_accessargs *argp = rqstp->rq_argp; + struct nfsd3_accessres *resp = rqstp->rq_resp; + + dprintk("nfsd: ACCESS(3) %s 0x%x\n", + SVCFH_fmt(&argp->fh), + argp->access); + + fh_copy(&resp->fh, &argp->fh); + resp->access = argp->access; + resp->status = nfsd_access(rqstp, &resp->fh, &resp->access, NULL); + resp->status = nfsd3_map_status(resp->status); + return rpc_success; +} + +/* + * Read a symlink. + */ +static __be32 +nfsd3_proc_readlink(struct svc_rqst *rqstp) +{ + struct nfsd_fhandle *argp = rqstp->rq_argp; + struct nfsd3_readlinkres *resp = rqstp->rq_resp; + + dprintk("nfsd: READLINK(3) %s\n", SVCFH_fmt(&argp->fh)); + + /* Read the symlink. */ + fh_copy(&resp->fh, &argp->fh); + resp->len = NFS3_MAXPATHLEN; + resp->pages = rqstp->rq_next_page++; + resp->status = nfsd_readlink(rqstp, &resp->fh, + page_address(*resp->pages), &resp->len); + resp->status = nfsd3_map_status(resp->status); + return rpc_success; +} + +/* + * Read a portion of a file. + */ +static __be32 +nfsd3_proc_read(struct svc_rqst *rqstp) +{ + struct nfsd3_readargs *argp = rqstp->rq_argp; + struct nfsd3_readres *resp = rqstp->rq_resp; + + dprintk("nfsd: READ(3) %s %lu bytes at %Lu\n", + SVCFH_fmt(&argp->fh), + (unsigned long) argp->count, + (unsigned long long) argp->offset); + + argp->count = min_t(u32, argp->count, svc_max_payload(rqstp)); + argp->count = min_t(u32, argp->count, rqstp->rq_res.buflen); + if (argp->offset > (u64)OFFSET_MAX) + argp->offset = (u64)OFFSET_MAX; + if (argp->offset + argp->count > (u64)OFFSET_MAX) + argp->count = (u64)OFFSET_MAX - argp->offset; + + resp->pages = rqstp->rq_next_page; + + /* Obtain buffer pointer for payload. + * 1 (status) + 22 (post_op_attr) + 1 (count) + 1 (eof) + * + 1 (xdr opaque byte count) = 26 + */ + resp->count = argp->count; + svc_reserve_auth(rqstp, ((1 + NFS3_POST_OP_ATTR_WORDS + 3) << 2) + + resp->count + 4); + + fh_copy(&resp->fh, &argp->fh); + resp->status = nfsd_read(rqstp, &resp->fh, argp->offset, + &resp->count, &resp->eof); + resp->status = nfsd3_map_status(resp->status); + return rpc_success; +} + +/* + * Write data to a file + */ +static __be32 +nfsd3_proc_write(struct svc_rqst *rqstp) +{ + struct nfsd3_writeargs *argp = rqstp->rq_argp; + struct nfsd3_writeres *resp = rqstp->rq_resp; + unsigned long cnt = argp->len; + + dprintk("nfsd: WRITE(3) %s %d bytes at %Lu%s\n", + SVCFH_fmt(&argp->fh), + argp->len, + (unsigned long long) argp->offset, + argp->stable ? " stable" : ""); + + resp->status = nfserr_fbig; + if (argp->offset > (u64)OFFSET_MAX || + argp->offset + argp->len > (u64)OFFSET_MAX) + return rpc_success; + + fh_copy(&resp->fh, &argp->fh); + resp->committed = argp->stable; + resp->status = nfsd_write(rqstp, &resp->fh, argp->offset, + &argp->payload, &cnt, + resp->committed, resp->verf); + resp->count = cnt; + resp->status = nfsd3_map_status(resp->status); + return rpc_success; +} + +/* + * Implement NFSv3's unchecked, guarded, and exclusive CREATE + * semantics for regular files. Except for the created file, + * this operation is stateless on the server. + * + * Upon return, caller must release @fhp and @resfhp. + */ +static __be32 +nfsd3_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp, + struct svc_fh *resfhp, struct nfsd3_createargs *argp) +{ + struct iattr *iap = &argp->attrs; + struct dentry *parent, *child; + struct nfsd_attrs attrs = { + .na_iattr = iap, + }; + __u32 v_mtime, v_atime; + struct inode *inode; + __be32 status; + int host_err; + + trace_nfsd_vfs_create(rqstp, fhp, S_IFREG, argp->name, argp->len); + + if (!nfsd3_time_in_range(iap)) + return nfserr_inval; + if (name_is_dot_dotdot(argp->name, argp->len)) + return nfserr_exist; + if (!(iap->ia_valid & ATTR_MODE)) + iap->ia_mode = 0; + + status = fh_verify(rqstp, fhp, S_IFDIR, NFSD_MAY_EXEC); + if (status != nfs_ok) + return status; + + parent = fhp->fh_dentry; + inode = d_inode(parent); + + host_err = fh_want_write(fhp); + if (host_err) + return nfserrno(host_err); + + child = start_creating(&nop_mnt_idmap, parent, + &QSTR_LEN(argp->name, argp->len)); + if (IS_ERR(child)) { + status = nfserrno(PTR_ERR(child)); + goto out_write; + } + + if (d_really_is_negative(child)) { + status = fh_verify(rqstp, fhp, S_IFDIR, NFSD_MAY_CREATE); + if (status != nfs_ok) + goto out; + } + + status = fh_compose(resfhp, fhp->fh_export, child, fhp); + if (status != nfs_ok) + goto out; + + v_mtime = 0; + v_atime = 0; + if (argp->createmode == NFS3_CREATE_EXCLUSIVE) { + u32 *verifier = (u32 *)argp->verf; + + /* + * Solaris 7 gets confused (bugid 4218508) if these have + * the high bit set, as do xfs filesystems without the + * "bigtime" feature. So just clear the high bits. + */ + v_mtime = verifier[0] & 0x7fffffff; + v_atime = verifier[1] & 0x7fffffff; + } + + if (d_really_is_positive(child)) { + status = nfs_ok; + + switch (argp->createmode) { + case NFS3_CREATE_UNCHECKED: + if (!d_is_reg(child)) + break; + iap->ia_valid &= ATTR_SIZE; + goto set_attr; + case NFS3_CREATE_GUARDED: + status = nfserr_exist; + break; + case NFS3_CREATE_EXCLUSIVE: + if (inode_get_mtime_sec(d_inode(child)) == v_mtime && + inode_get_atime_sec(d_inode(child)) == v_atime && + d_inode(child)->i_size == 0) { + break; + } + status = nfserr_exist; + } + goto out; + } + + if (!IS_POSIXACL(inode)) + iap->ia_mode &= ~current_umask(); + + status = fh_fill_pre_attrs(fhp); + if (status != nfs_ok) + goto out; + host_err = vfs_create(&nop_mnt_idmap, child, iap->ia_mode, NULL); + if (host_err < 0) { + status = nfserrno(host_err); + goto out; + } + fh_fill_post_attrs(fhp); + + /* A newly created file already has a file size of zero. */ + if ((iap->ia_valid & ATTR_SIZE) && (iap->ia_size == 0)) + iap->ia_valid &= ~ATTR_SIZE; + if (argp->createmode == NFS3_CREATE_EXCLUSIVE) { + iap->ia_valid = ATTR_MTIME | ATTR_ATIME | + ATTR_MTIME_SET | ATTR_ATIME_SET; + iap->ia_mtime.tv_sec = v_mtime; + iap->ia_atime.tv_sec = v_atime; + iap->ia_mtime.tv_nsec = 0; + iap->ia_atime.tv_nsec = 0; + } + +set_attr: + status = nfsd_create_setattr(rqstp, fhp, resfhp, &attrs); + +out: + end_creating(child); +out_write: + fh_drop_write(fhp); + return status; +} + +static __be32 +nfsd3_proc_create(struct svc_rqst *rqstp) +{ + struct nfsd3_createargs *argp = rqstp->rq_argp; + struct nfsd3_diropres *resp = rqstp->rq_resp; + svc_fh *dirfhp, *newfhp; + + dirfhp = fh_copy(&resp->dirfh, &argp->fh); + newfhp = fh_init(&resp->fh, NFS3_FHSIZE); + + resp->status = nfsd3_create_file(rqstp, dirfhp, newfhp, argp); + resp->status = nfsd3_map_status(resp->status); + return rpc_success; +} + +/* + * Make directory. This operation is not idempotent. + */ +static __be32 +nfsd3_proc_mkdir(struct svc_rqst *rqstp) +{ + struct nfsd3_createargs *argp = rqstp->rq_argp; + struct nfsd3_diropres *resp = rqstp->rq_resp; + struct nfsd_attrs attrs = { + .na_iattr = &argp->attrs, + }; + + argp->attrs.ia_valid &= ~ATTR_SIZE; + fh_copy(&resp->dirfh, &argp->fh); + fh_init(&resp->fh, NFS3_FHSIZE); + if (!nfsd3_time_in_range(&argp->attrs)) { + resp->status = nfserr_inval; + goto out; + } + resp->status = nfsd_create(rqstp, &resp->dirfh, argp->name, argp->len, + &attrs, S_IFDIR, 0, &resp->fh); +out: + resp->status = nfsd3_map_status(resp->status); + return rpc_success; +} + +static __be32 +nfsd3_proc_symlink(struct svc_rqst *rqstp) +{ + struct nfsd3_symlinkargs *argp = rqstp->rq_argp; + struct nfsd3_diropres *resp = rqstp->rq_resp; + struct nfsd_attrs attrs = { + .na_iattr = &argp->attrs, + }; + + if (!nfsd3_time_in_range(&argp->attrs)) { + resp->status = nfserr_inval; + goto out; + } + if (argp->tlen == 0) { + resp->status = nfserr_inval; + goto out; + } + if (argp->tlen > NFS3_MAXPATHLEN) { + resp->status = nfserr_nametoolong; + goto out; + } + + argp->tname = svc_fill_symlink_pathname(rqstp, &argp->first, + page_address(rqstp->rq_arg.pages[0]), + argp->tlen); + if (IS_ERR(argp->tname)) { + resp->status = nfserrno(PTR_ERR(argp->tname)); + goto out; + } + + fh_copy(&resp->dirfh, &argp->ffh); + fh_init(&resp->fh, NFS3_FHSIZE); + resp->status = nfsd_symlink(rqstp, &resp->dirfh, argp->fname, + argp->flen, argp->tname, &attrs, &resp->fh); + kfree(argp->tname); +out: + resp->status = nfsd3_map_status(resp->status); + return rpc_success; +} + +/* + * Make socket/fifo/device. + */ +static __be32 +nfsd3_proc_mknod(struct svc_rqst *rqstp) +{ + struct nfsd3_mknodargs *argp = rqstp->rq_argp; + struct nfsd3_diropres *resp = rqstp->rq_resp; + struct nfsd_attrs attrs = { + .na_iattr = &argp->attrs, + }; + int type; + dev_t rdev = 0; + + fh_copy(&resp->dirfh, &argp->fh); + fh_init(&resp->fh, NFS3_FHSIZE); + + if (argp->ftype == NF3CHR || argp->ftype == NF3BLK) { + rdev = MKDEV(argp->major, argp->minor); + if (MAJOR(rdev) != argp->major || + MINOR(rdev) != argp->minor) { + resp->status = nfserr_inval; + goto out; + } + } else if (argp->ftype != NF3SOCK && argp->ftype != NF3FIFO) { + resp->status = nfserr_badtype; + goto out; + } + + if (!nfsd3_time_in_range(&argp->attrs)) { + resp->status = nfserr_inval; + goto out; + } + + type = nfs3_ftypes[argp->ftype]; + resp->status = nfsd_create(rqstp, &resp->dirfh, argp->name, argp->len, + &attrs, type, rdev, &resp->fh); +out: + resp->status = nfsd3_map_status(resp->status); + return rpc_success; +} + +/* + * Remove file/fifo/socket etc. + */ +static __be32 +nfsd3_proc_remove(struct svc_rqst *rqstp) +{ + struct nfsd3_diropargs *argp = rqstp->rq_argp; + struct nfsd3_attrstat *resp = rqstp->rq_resp; + + /* Unlink. -S_IFDIR means file must not be a directory */ + fh_copy(&resp->fh, &argp->fh); + resp->status = nfsd_unlink(rqstp, &resp->fh, -S_IFDIR, + argp->name, argp->len); + resp->status = nfsd3_map_status(resp->status); + return rpc_success; +} + +/* + * Remove a directory + */ +static __be32 +nfsd3_proc_rmdir(struct svc_rqst *rqstp) +{ + struct nfsd3_diropargs *argp = rqstp->rq_argp; + struct nfsd3_attrstat *resp = rqstp->rq_resp; + + fh_copy(&resp->fh, &argp->fh); + resp->status = nfsd_unlink(rqstp, &resp->fh, S_IFDIR, + argp->name, argp->len); + resp->status = nfsd3_map_status(resp->status); + return rpc_success; +} + +static __be32 +nfsd3_proc_rename(struct svc_rqst *rqstp) +{ + struct nfsd3_renameargs *argp = rqstp->rq_argp; + struct nfsd3_renameres *resp = rqstp->rq_resp; + + fh_copy(&resp->ffh, &argp->ffh); + fh_copy(&resp->tfh, &argp->tfh); + resp->status = nfsd_rename(rqstp, &resp->ffh, argp->fname, argp->flen, + &resp->tfh, argp->tname, argp->tlen); + resp->status = nfsd3_map_status(resp->status); + return rpc_success; +} + +static __be32 +nfsd3_proc_link(struct svc_rqst *rqstp) +{ + struct nfsd3_linkargs *argp = rqstp->rq_argp; + struct nfsd3_linkres *resp = rqstp->rq_resp; + + fh_copy(&resp->fh, &argp->ffh); + fh_copy(&resp->tfh, &argp->tfh); + resp->status = nfsd_link(rqstp, &resp->tfh, argp->tname, argp->tlen, + &resp->fh); + resp->status = nfsd3_map_status(resp->status); + return rpc_success; +} + +static void nfsd3_init_dirlist_pages(struct svc_rqst *rqstp, + struct nfsd3_readdirres *resp, + u32 count) +{ + struct xdr_buf *buf = &resp->dirlist; + struct xdr_stream *xdr = &resp->xdr; + unsigned int sendbuf = min_t(unsigned int, rqstp->rq_res.buflen, + svc_max_payload(rqstp)); + + memset(buf, 0, sizeof(*buf)); + + /* Reserve room for the NULL ptr & eof flag (-2 words) */ + buf->buflen = clamp(count, (u32)(XDR_UNIT * 2), sendbuf); + buf->buflen -= XDR_UNIT * 2; + buf->pages = rqstp->rq_next_page; + rqstp->rq_next_page += (buf->buflen + PAGE_SIZE - 1) >> PAGE_SHIFT; + + xdr_init_encode_pages(xdr, buf); +} + +/* + * Read a portion of a directory. + */ +static __be32 +nfsd3_proc_readdir(struct svc_rqst *rqstp) +{ + struct nfsd3_readdirargs *argp = rqstp->rq_argp; + struct nfsd3_readdirres *resp = rqstp->rq_resp; + loff_t offset; + + trace_nfsd_vfs_readdir(rqstp, &argp->fh, argp->count, argp->cookie); + + nfsd3_init_dirlist_pages(rqstp, resp, argp->count); + + fh_copy(&resp->fh, &argp->fh); + resp->common.err = nfs_ok; + resp->cookie_offset = 0; + resp->rqstp = rqstp; + offset = argp->cookie; + resp->status = nfsd_readdir(rqstp, &resp->fh, &offset, + &resp->common, nfs3svc_encode_entry3); + memcpy(resp->verf, argp->verf, 8); + nfs3svc_encode_cookie3(resp, offset); + + /* Recycle only pages that were part of the reply */ + rqstp->rq_next_page = resp->xdr.page_ptr + 1; + + resp->status = nfsd3_map_status(resp->status); + return rpc_success; +} + +/* + * Read a portion of a directory, including file handles and attrs. + * For now, we choose to ignore the dircount parameter. + */ +static __be32 +nfsd3_proc_readdirplus(struct svc_rqst *rqstp) +{ + struct nfsd3_readdirargs *argp = rqstp->rq_argp; + struct nfsd3_readdirres *resp = rqstp->rq_resp; + loff_t offset; + + trace_nfsd_vfs_readdir(rqstp, &argp->fh, argp->count, argp->cookie); + + nfsd3_init_dirlist_pages(rqstp, resp, argp->count); + + fh_copy(&resp->fh, &argp->fh); + resp->common.err = nfs_ok; + resp->cookie_offset = 0; + resp->rqstp = rqstp; + offset = argp->cookie; + + resp->status = fh_verify(rqstp, &resp->fh, S_IFDIR, NFSD_MAY_NOP); + if (resp->status != nfs_ok) + goto out; + + if (resp->fh.fh_export->ex_flags & NFSEXP_NOREADDIRPLUS) { + resp->status = nfserr_notsupp; + goto out; + } + + resp->status = nfsd_readdir(rqstp, &resp->fh, &offset, + &resp->common, nfs3svc_encode_entryplus3); + memcpy(resp->verf, argp->verf, 8); + nfs3svc_encode_cookie3(resp, offset); + + /* Recycle only pages that were part of the reply */ + rqstp->rq_next_page = resp->xdr.page_ptr + 1; + +out: + resp->status = nfsd3_map_status(resp->status); + return rpc_success; +} + +/* + * Get file system stats + */ +static __be32 +nfsd3_proc_fsstat(struct svc_rqst *rqstp) +{ + struct nfsd_fhandle *argp = rqstp->rq_argp; + struct nfsd3_fsstatres *resp = rqstp->rq_resp; + + resp->status = nfsd_statfs(rqstp, &argp->fh, &resp->stats, 0); + fh_put(&argp->fh); + resp->status = nfsd3_map_status(resp->status); + return rpc_success; +} + +/* + * Get file system info + */ +static __be32 +nfsd3_proc_fsinfo(struct svc_rqst *rqstp) +{ + struct nfsd_fhandle *argp = rqstp->rq_argp; + struct nfsd3_fsinfores *resp = rqstp->rq_resp; + u32 max_blocksize = svc_max_payload(rqstp); + + dprintk("nfsd: FSINFO(3) %s\n", + SVCFH_fmt(&argp->fh)); + + resp->f_rtmax = max_blocksize; + resp->f_rtpref = max_blocksize; + resp->f_rtmult = PAGE_SIZE; + resp->f_wtmax = max_blocksize; + resp->f_wtpref = max_blocksize; + resp->f_wtmult = PAGE_SIZE; + resp->f_dtpref = max_blocksize; + resp->f_maxfilesize = ~(u32) 0; + resp->f_properties = NFS3_FSF_DEFAULT; + + resp->status = fh_verify(rqstp, &argp->fh, 0, + NFSD_MAY_NOP | NFSD_MAY_BYPASS_GSS_ON_ROOT); + + /* Check special features of the file system. May request + * different read/write sizes for file systems known to have + * problems with large blocks */ + if (resp->status == nfs_ok) { + struct super_block *sb = argp->fh.fh_dentry->d_sb; + + /* Note that we don't care for remote fs's here */ + if (sb->s_magic == MSDOS_SUPER_MAGIC) { + resp->f_properties = NFS3_FSF_BILLYBOY; + } + resp->f_maxfilesize = sb->s_maxbytes; + } + + fh_put(&argp->fh); + resp->status = nfsd3_map_status(resp->status); + return rpc_success; +} + +/* + * Get pathconf info for the specified file + */ +static __be32 +nfsd3_proc_pathconf(struct svc_rqst *rqstp) +{ + struct nfsd_fhandle *argp = rqstp->rq_argp; + struct nfsd3_pathconfres *resp = rqstp->rq_resp; + + dprintk("nfsd: PATHCONF(3) %s\n", + SVCFH_fmt(&argp->fh)); + + /* Set default pathconf */ + resp->p_link_max = 255; /* at least */ + resp->p_name_max = 255; /* at least */ + resp->p_no_trunc = 0; + resp->p_chown_restricted = 1; + resp->p_case_insensitive = false; + resp->p_case_preserving = true; + + resp->status = fh_verify(rqstp, &argp->fh, 0, NFSD_MAY_NOP); + + if (resp->status == nfs_ok) { + struct super_block *sb = argp->fh.fh_dentry->d_sb; + int err; + + if (sb->s_magic == EXT2_SUPER_MAGIC) { + resp->p_link_max = EXT2_LINK_MAX; + resp->p_name_max = EXT2_NAME_LEN; + } + + err = nfsd_get_case_info(argp->fh.fh_dentry, + &resp->p_case_insensitive, + &resp->p_case_preserving); + /* + * RFC 1813 lists NFS3ERR_STALE, NFS3ERR_BADHANDLE, and + * NFS3ERR_SERVERFAULT as the only PATHCONF errors. + */ + switch (err) { + case 0: + case -EOPNOTSUPP: + /* Both arms leave the output booleans valid. */ + break; + case -EACCES: + case -EPERM: + /* + * Policy denied the query. Report STALE so the + * handle is unusable without implying a server + * malfunction. + */ + resp->status = nfserr_stale; + break; + case -ESTALE: + resp->status = nfserr_stale; + break; + default: + resp->status = nfserr_serverfault; + break; + } + } + + fh_put(&argp->fh); + resp->status = nfsd3_map_status(resp->status); + return rpc_success; +} + +/* + * Commit a file (range) to stable storage. + */ +static __be32 +nfsd3_proc_commit(struct svc_rqst *rqstp) +{ + struct nfsd3_commitargs *argp = rqstp->rq_argp; + struct nfsd3_commitres *resp = rqstp->rq_resp; + struct nfsd_file *nf; + + dprintk("nfsd: COMMIT(3) %s %u@%Lu\n", + SVCFH_fmt(&argp->fh), + argp->count, + (unsigned long long) argp->offset); + + fh_copy(&resp->fh, &argp->fh); + resp->status = nfsd_file_acquire_gc(rqstp, &resp->fh, NFSD_MAY_WRITE | + NFSD_MAY_NOT_BREAK_LEASE, &nf); + if (resp->status) + goto out; + resp->status = nfsd_commit(rqstp, &resp->fh, nf, argp->offset, + argp->count, resp->verf); + nfsd_file_put(nf); +out: + resp->status = nfsd3_map_status(resp->status); + return rpc_success; +} + + +/* + * NFSv3 Server procedures. + * Only the results of non-idempotent operations are cached. + */ +#define nfs3svc_encode_attrstatres nfs3svc_encode_attrstat +#define nfs3svc_encode_wccstatres nfs3svc_encode_wccstat +#define nfsd3_mkdirargs nfsd3_createargs +#define nfsd3_readdirplusargs nfsd3_readdirargs +#define nfsd3_fhandleargs nfsd_fhandle +#define nfsd3_attrstatres nfsd3_attrstat +#define nfsd3_wccstatres nfsd3_attrstat +#define nfsd3_createres nfsd3_diropres + +#define ST 1 /* status*/ +#define FH 17 /* filehandle with length */ +#define AT 21 /* attributes */ +#define pAT (1+AT) /* post attributes - conditional */ +#define WC (7+pAT) /* WCC attributes */ + +static const struct svc_procedure nfsd_procedures3[22] = { + [NFS3PROC_NULL] = { + .pc_func = nfsd3_proc_null, + .pc_decode = nfssvc_decode_voidarg, + .pc_encode = nfssvc_encode_voidres, + .pc_argsize = sizeof(struct nfsd_voidargs), + .pc_argzero = sizeof(struct nfsd_voidargs), + .pc_ressize = sizeof(struct nfsd_voidres), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = ST, + .pc_name = "NULL", + }, + [NFS3PROC_GETATTR] = { + .pc_func = nfsd3_proc_getattr, + .pc_decode = nfs3svc_decode_fhandleargs, + .pc_encode = nfs3svc_encode_getattrres, + .pc_release = nfs3svc_release_fhandle, + .pc_argsize = sizeof(struct nfsd_fhandle), + .pc_argzero = sizeof(struct nfsd_fhandle), + .pc_ressize = sizeof(struct nfsd3_attrstatres), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = ST+AT, + .pc_name = "GETATTR", + }, + [NFS3PROC_SETATTR] = { + .pc_func = nfsd3_proc_setattr, + .pc_decode = nfs3svc_decode_sattrargs, + .pc_encode = nfs3svc_encode_wccstatres, + .pc_release = nfs3svc_release_fhandle, + .pc_argsize = sizeof(struct nfsd3_sattrargs), + .pc_argzero = sizeof(struct nfsd3_sattrargs), + .pc_ressize = sizeof(struct nfsd3_wccstatres), + .pc_cachetype = RC_REPLBUFF, + .pc_xdrressize = ST+WC, + .pc_name = "SETATTR", + }, + [NFS3PROC_LOOKUP] = { + .pc_func = nfsd3_proc_lookup, + .pc_decode = nfs3svc_decode_diropargs, + .pc_encode = nfs3svc_encode_lookupres, + .pc_release = nfs3svc_release_fhandle2, + .pc_argsize = sizeof(struct nfsd3_diropargs), + .pc_argzero = sizeof(struct nfsd3_diropargs), + .pc_ressize = sizeof(struct nfsd3_diropres), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = ST+FH+pAT+pAT, + .pc_name = "LOOKUP", + }, + [NFS3PROC_ACCESS] = { + .pc_func = nfsd3_proc_access, + .pc_decode = nfs3svc_decode_accessargs, + .pc_encode = nfs3svc_encode_accessres, + .pc_release = nfs3svc_release_fhandle, + .pc_argsize = sizeof(struct nfsd3_accessargs), + .pc_argzero = sizeof(struct nfsd3_accessargs), + .pc_ressize = sizeof(struct nfsd3_accessres), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = ST+pAT+1, + .pc_name = "ACCESS", + }, + [NFS3PROC_READLINK] = { + .pc_func = nfsd3_proc_readlink, + .pc_decode = nfs3svc_decode_fhandleargs, + .pc_encode = nfs3svc_encode_readlinkres, + .pc_release = nfs3svc_release_fhandle, + .pc_argsize = sizeof(struct nfsd_fhandle), + .pc_argzero = sizeof(struct nfsd_fhandle), + .pc_ressize = sizeof(struct nfsd3_readlinkres), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = ST+pAT+1+NFS3_MAXPATHLEN/4, + .pc_name = "READLINK", + }, + [NFS3PROC_READ] = { + .pc_func = nfsd3_proc_read, + .pc_decode = nfs3svc_decode_readargs, + .pc_encode = nfs3svc_encode_readres, + .pc_release = nfs3svc_release_fhandle, + .pc_argsize = sizeof(struct nfsd3_readargs), + .pc_argzero = sizeof(struct nfsd3_readargs), + .pc_ressize = sizeof(struct nfsd3_readres), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = ST+pAT+4+NFSSVC_MAXBLKSIZE/4, + .pc_name = "READ", + }, + [NFS3PROC_WRITE] = { + .pc_func = nfsd3_proc_write, + .pc_decode = nfs3svc_decode_writeargs, + .pc_encode = nfs3svc_encode_writeres, + .pc_release = nfs3svc_release_fhandle, + .pc_argsize = sizeof(struct nfsd3_writeargs), + .pc_argzero = sizeof(struct nfsd3_writeargs), + .pc_ressize = sizeof(struct nfsd3_writeres), + .pc_cachetype = RC_REPLBUFF, + .pc_xdrressize = ST+WC+4, + .pc_name = "WRITE", + }, + [NFS3PROC_CREATE] = { + .pc_func = nfsd3_proc_create, + .pc_decode = nfs3svc_decode_createargs, + .pc_encode = nfs3svc_encode_createres, + .pc_release = nfs3svc_release_fhandle2, + .pc_argsize = sizeof(struct nfsd3_createargs), + .pc_argzero = sizeof(struct nfsd3_createargs), + .pc_ressize = sizeof(struct nfsd3_createres), + .pc_cachetype = RC_REPLBUFF, + .pc_xdrressize = ST+(1+FH+pAT)+WC, + .pc_name = "CREATE", + }, + [NFS3PROC_MKDIR] = { + .pc_func = nfsd3_proc_mkdir, + .pc_decode = nfs3svc_decode_mkdirargs, + .pc_encode = nfs3svc_encode_createres, + .pc_release = nfs3svc_release_fhandle2, + .pc_argsize = sizeof(struct nfsd3_mkdirargs), + .pc_argzero = sizeof(struct nfsd3_mkdirargs), + .pc_ressize = sizeof(struct nfsd3_createres), + .pc_cachetype = RC_REPLBUFF, + .pc_xdrressize = ST+(1+FH+pAT)+WC, + .pc_name = "MKDIR", + }, + [NFS3PROC_SYMLINK] = { + .pc_func = nfsd3_proc_symlink, + .pc_decode = nfs3svc_decode_symlinkargs, + .pc_encode = nfs3svc_encode_createres, + .pc_release = nfs3svc_release_fhandle2, + .pc_argsize = sizeof(struct nfsd3_symlinkargs), + .pc_argzero = sizeof(struct nfsd3_symlinkargs), + .pc_ressize = sizeof(struct nfsd3_createres), + .pc_cachetype = RC_REPLBUFF, + .pc_xdrressize = ST+(1+FH+pAT)+WC, + .pc_name = "SYMLINK", + }, + [NFS3PROC_MKNOD] = { + .pc_func = nfsd3_proc_mknod, + .pc_decode = nfs3svc_decode_mknodargs, + .pc_encode = nfs3svc_encode_createres, + .pc_release = nfs3svc_release_fhandle2, + .pc_argsize = sizeof(struct nfsd3_mknodargs), + .pc_argzero = sizeof(struct nfsd3_mknodargs), + .pc_ressize = sizeof(struct nfsd3_createres), + .pc_cachetype = RC_REPLBUFF, + .pc_xdrressize = ST+(1+FH+pAT)+WC, + .pc_name = "MKNOD", + }, + [NFS3PROC_REMOVE] = { + .pc_func = nfsd3_proc_remove, + .pc_decode = nfs3svc_decode_diropargs, + .pc_encode = nfs3svc_encode_wccstatres, + .pc_release = nfs3svc_release_fhandle, + .pc_argsize = sizeof(struct nfsd3_diropargs), + .pc_argzero = sizeof(struct nfsd3_diropargs), + .pc_ressize = sizeof(struct nfsd3_wccstatres), + .pc_cachetype = RC_REPLBUFF, + .pc_xdrressize = ST+WC, + .pc_name = "REMOVE", + }, + [NFS3PROC_RMDIR] = { + .pc_func = nfsd3_proc_rmdir, + .pc_decode = nfs3svc_decode_diropargs, + .pc_encode = nfs3svc_encode_wccstatres, + .pc_release = nfs3svc_release_fhandle, + .pc_argsize = sizeof(struct nfsd3_diropargs), + .pc_argzero = sizeof(struct nfsd3_diropargs), + .pc_ressize = sizeof(struct nfsd3_wccstatres), + .pc_cachetype = RC_REPLBUFF, + .pc_xdrressize = ST+WC, + .pc_name = "RMDIR", + }, + [NFS3PROC_RENAME] = { + .pc_func = nfsd3_proc_rename, + .pc_decode = nfs3svc_decode_renameargs, + .pc_encode = nfs3svc_encode_renameres, + .pc_release = nfs3svc_release_fhandle2, + .pc_argsize = sizeof(struct nfsd3_renameargs), + .pc_argzero = sizeof(struct nfsd3_renameargs), + .pc_ressize = sizeof(struct nfsd3_renameres), + .pc_cachetype = RC_REPLBUFF, + .pc_xdrressize = ST+WC+WC, + .pc_name = "RENAME", + }, + [NFS3PROC_LINK] = { + .pc_func = nfsd3_proc_link, + .pc_decode = nfs3svc_decode_linkargs, + .pc_encode = nfs3svc_encode_linkres, + .pc_release = nfs3svc_release_fhandle2, + .pc_argsize = sizeof(struct nfsd3_linkargs), + .pc_argzero = sizeof(struct nfsd3_linkargs), + .pc_ressize = sizeof(struct nfsd3_linkres), + .pc_cachetype = RC_REPLBUFF, + .pc_xdrressize = ST+pAT+WC, + .pc_name = "LINK", + }, + [NFS3PROC_READDIR] = { + .pc_func = nfsd3_proc_readdir, + .pc_decode = nfs3svc_decode_readdirargs, + .pc_encode = nfs3svc_encode_readdirres, + .pc_release = nfs3svc_release_fhandle, + .pc_argsize = sizeof(struct nfsd3_readdirargs), + .pc_argzero = sizeof(struct nfsd3_readdirargs), + .pc_ressize = sizeof(struct nfsd3_readdirres), + .pc_cachetype = RC_NOCACHE, + .pc_name = "READDIR", + }, + [NFS3PROC_READDIRPLUS] = { + .pc_func = nfsd3_proc_readdirplus, + .pc_decode = nfs3svc_decode_readdirplusargs, + .pc_encode = nfs3svc_encode_readdirres, + .pc_release = nfs3svc_release_fhandle, + .pc_argsize = sizeof(struct nfsd3_readdirplusargs), + .pc_argzero = sizeof(struct nfsd3_readdirplusargs), + .pc_ressize = sizeof(struct nfsd3_readdirres), + .pc_cachetype = RC_NOCACHE, + .pc_name = "READDIRPLUS", + }, + [NFS3PROC_FSSTAT] = { + .pc_func = nfsd3_proc_fsstat, + .pc_decode = nfs3svc_decode_fhandleargs, + .pc_encode = nfs3svc_encode_fsstatres, + .pc_argsize = sizeof(struct nfsd3_fhandleargs), + .pc_argzero = sizeof(struct nfsd3_fhandleargs), + .pc_ressize = sizeof(struct nfsd3_fsstatres), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = ST+pAT+2*6+1, + .pc_name = "FSSTAT", + }, + [NFS3PROC_FSINFO] = { + .pc_func = nfsd3_proc_fsinfo, + .pc_decode = nfs3svc_decode_fhandleargs, + .pc_encode = nfs3svc_encode_fsinfores, + .pc_argsize = sizeof(struct nfsd3_fhandleargs), + .pc_argzero = sizeof(struct nfsd3_fhandleargs), + .pc_ressize = sizeof(struct nfsd3_fsinfores), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = ST+pAT+12, + .pc_name = "FSINFO", + }, + [NFS3PROC_PATHCONF] = { + .pc_func = nfsd3_proc_pathconf, + .pc_decode = nfs3svc_decode_fhandleargs, + .pc_encode = nfs3svc_encode_pathconfres, + .pc_argsize = sizeof(struct nfsd3_fhandleargs), + .pc_argzero = sizeof(struct nfsd3_fhandleargs), + .pc_ressize = sizeof(struct nfsd3_pathconfres), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = ST+pAT+6, + .pc_name = "PATHCONF", + }, + [NFS3PROC_COMMIT] = { + .pc_func = nfsd3_proc_commit, + .pc_decode = nfs3svc_decode_commitargs, + .pc_encode = nfs3svc_encode_commitres, + .pc_release = nfs3svc_release_fhandle, + .pc_argsize = sizeof(struct nfsd3_commitargs), + .pc_argzero = sizeof(struct nfsd3_commitargs), + .pc_ressize = sizeof(struct nfsd3_commitres), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = ST+WC+2, + .pc_name = "COMMIT", + }, +}; + +const struct svc_version nfsd_version3 = { + .vs_vers = 3, + .vs_nproc = ARRAY_SIZE(nfsd_procedures3), + .vs_proc = nfsd_procedures3, + .vs_dispatch = nfsd_dispatch, + .vs_xdrsize = NFS3_SVC_XDRSIZE, +}; diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c new file mode 100644 index 000000000..e481804bb --- /dev/null +++ b/fs/nfsd/nfs3xdr.c @@ -0,0 +1,1356 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * XDR support for nfsd/protocol version 3. + * + * Copyright (C) 1995, 1996, 1997 Olaf Kirch + * + * 2003-08-09 Jamie Lokier: Use htonl() for nanoseconds, not htons()! + */ + +#include +#include +#include "xdr3.h" +#include "auth.h" +#include "netns.h" +#include "vfs.h" + +/* + * Force construction of an empty post-op attr + */ +static const struct svc_fh nfs3svc_null_fh = { + .fh_no_wcc = true, +}; + +/* + * time_delta. {1, 0} means the server is accurate only + * to the nearest second. + */ +static const struct timespec64 nfs3svc_time_delta = { + .tv_sec = 1, + .tv_nsec = 0, +}; + +/* + * Mapping of S_IF* types to NFS file types + */ +static const u32 nfs3_ftypes[] = { + NF3NON, NF3FIFO, NF3CHR, NF3BAD, + NF3DIR, NF3BAD, NF3BLK, NF3BAD, + NF3REG, NF3BAD, NF3LNK, NF3BAD, + NF3SOCK, NF3BAD, NF3LNK, NF3BAD, +}; + + +/* + * Basic NFSv3 data types (RFC 1813 Sections 2.5 and 2.6) + */ + +static __be32 * +encode_nfstime3(__be32 *p, const struct timespec64 *time) +{ + *p++ = cpu_to_be32((u32)time->tv_sec); + *p++ = cpu_to_be32(time->tv_nsec); + + return p; +} + +static bool +svcxdr_decode_nfstime3(struct xdr_stream *xdr, struct timespec64 *timep) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, XDR_UNIT * 2); + if (!p) + return false; + timep->tv_sec = be32_to_cpup(p++); + timep->tv_nsec = be32_to_cpup(p); + + return true; +} + +/** + * svcxdr_decode_nfs_fh3 - Decode an NFSv3 file handle + * @xdr: XDR stream positioned at an undecoded NFSv3 FH + * @fhp: OUT: filled-in server file handle + * + * Return values: + * %false: The encoded file handle was not valid + * %true: @fhp has been initialized + */ +bool +svcxdr_decode_nfs_fh3(struct xdr_stream *xdr, struct svc_fh *fhp) +{ + __be32 *p; + u32 size; + + if (xdr_stream_decode_u32(xdr, &size) < 0) + return false; + if (size == 0 || size > NFS3_FHSIZE) + return false; + p = xdr_inline_decode(xdr, size); + if (!p) + return false; + fh_init(fhp, NFS3_FHSIZE); + fhp->fh_handle.fh_size = size; + memcpy(&fhp->fh_handle.fh_raw, p, size); + + return true; +} + +/** + * svcxdr_encode_nfsstat3 - Encode an NFSv3 status code + * @xdr: XDR stream + * @status: status value to encode + * + * Return values: + * %false: Send buffer space was exhausted + * %true: Success + */ +bool +svcxdr_encode_nfsstat3(struct xdr_stream *xdr, __be32 status) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, sizeof(status)); + if (!p) + return false; + *p = status; + + return true; +} + +static bool +svcxdr_encode_nfs_fh3(struct xdr_stream *xdr, const struct svc_fh *fhp) +{ + u32 size = fhp->fh_handle.fh_size; + __be32 *p; + + p = xdr_reserve_space(xdr, XDR_UNIT + size); + if (!p) + return false; + *p++ = cpu_to_be32(size); + if (size) + p[XDR_QUADLEN(size) - 1] = 0; + memcpy(p, &fhp->fh_handle.fh_raw, size); + + return true; +} + +static bool +svcxdr_encode_post_op_fh3(struct xdr_stream *xdr, const struct svc_fh *fhp) +{ + if (xdr_stream_encode_item_present(xdr) < 0) + return false; + if (!svcxdr_encode_nfs_fh3(xdr, fhp)) + return false; + + return true; +} + +static bool +svcxdr_encode_cookieverf3(struct xdr_stream *xdr, const __be32 *verf) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, NFS3_COOKIEVERFSIZE); + if (!p) + return false; + memcpy(p, verf, NFS3_COOKIEVERFSIZE); + + return true; +} + +static bool +svcxdr_encode_writeverf3(struct xdr_stream *xdr, const __be32 *verf) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, NFS3_WRITEVERFSIZE); + if (!p) + return false; + memcpy(p, verf, NFS3_WRITEVERFSIZE); + + return true; +} + +static bool +svcxdr_decode_filename3(struct xdr_stream *xdr, char **name, unsigned int *len) +{ + u32 size, i; + __be32 *p; + char *c; + + if (xdr_stream_decode_u32(xdr, &size) < 0) + return false; + if (size == 0 || size > NFS3_MAXNAMLEN) + return false; + p = xdr_inline_decode(xdr, size); + if (!p) + return false; + + *len = size; + *name = (char *)p; + for (i = 0, c = *name; i < size; i++, c++) { + if (*c == '\0' || *c == '/') + return false; + } + + return true; +} + +static bool +svcxdr_decode_diropargs3(struct xdr_stream *xdr, struct svc_fh *fhp, + char **name, unsigned int *len) +{ + return svcxdr_decode_nfs_fh3(xdr, fhp) && + svcxdr_decode_filename3(xdr, name, len); +} + +static bool +svcxdr_decode_sattr3(struct svc_rqst *rqstp, struct xdr_stream *xdr, + struct iattr *iap) +{ + u32 set_it; + + iap->ia_valid = 0; + + if (xdr_stream_decode_bool(xdr, &set_it) < 0) + return false; + if (set_it) { + u32 mode; + + if (xdr_stream_decode_u32(xdr, &mode) < 0) + return false; + iap->ia_valid |= ATTR_MODE; + iap->ia_mode = mode; + } + if (xdr_stream_decode_bool(xdr, &set_it) < 0) + return false; + if (set_it) { + u32 uid; + + if (xdr_stream_decode_u32(xdr, &uid) < 0) + return false; + iap->ia_uid = make_kuid(nfsd_user_namespace(rqstp), uid); + if (uid_valid(iap->ia_uid)) + iap->ia_valid |= ATTR_UID; + } + if (xdr_stream_decode_bool(xdr, &set_it) < 0) + return false; + if (set_it) { + u32 gid; + + if (xdr_stream_decode_u32(xdr, &gid) < 0) + return false; + iap->ia_gid = make_kgid(nfsd_user_namespace(rqstp), gid); + if (gid_valid(iap->ia_gid)) + iap->ia_valid |= ATTR_GID; + } + if (xdr_stream_decode_bool(xdr, &set_it) < 0) + return false; + if (set_it) { + u64 newsize; + + if (xdr_stream_decode_u64(xdr, &newsize) < 0) + return false; + iap->ia_valid |= ATTR_SIZE; + iap->ia_size = newsize; + } + if (xdr_stream_decode_u32(xdr, &set_it) < 0) + return false; + switch (set_it) { + case DONT_CHANGE: + break; + case SET_TO_SERVER_TIME: + iap->ia_valid |= ATTR_ATIME; + break; + case SET_TO_CLIENT_TIME: + if (!svcxdr_decode_nfstime3(xdr, &iap->ia_atime)) + return false; + iap->ia_valid |= ATTR_ATIME | ATTR_ATIME_SET; + break; + default: + return false; + } + if (xdr_stream_decode_u32(xdr, &set_it) < 0) + return false; + switch (set_it) { + case DONT_CHANGE: + break; + case SET_TO_SERVER_TIME: + iap->ia_valid |= ATTR_MTIME; + break; + case SET_TO_CLIENT_TIME: + if (!svcxdr_decode_nfstime3(xdr, &iap->ia_mtime)) + return false; + iap->ia_valid |= ATTR_MTIME | ATTR_MTIME_SET; + break; + default: + return false; + } + + return true; +} + +static bool +svcxdr_decode_sattrguard3(struct xdr_stream *xdr, struct nfsd3_sattrargs *args) +{ + u32 check; + + if (xdr_stream_decode_bool(xdr, &check) < 0) + return false; + if (check) { + if (!svcxdr_decode_nfstime3(xdr, &args->guardtime)) + return false; + args->check_guard = 1; + } else + args->check_guard = 0; + + return true; +} + +static bool +svcxdr_decode_specdata3(struct xdr_stream *xdr, struct nfsd3_mknodargs *args) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, XDR_UNIT * 2); + if (!p) + return false; + args->major = be32_to_cpup(p++); + args->minor = be32_to_cpup(p); + + return true; +} + +static bool +svcxdr_decode_devicedata3(struct svc_rqst *rqstp, struct xdr_stream *xdr, + struct nfsd3_mknodargs *args) +{ + return svcxdr_decode_sattr3(rqstp, xdr, &args->attrs) && + svcxdr_decode_specdata3(xdr, args); +} + +static bool +svcxdr_encode_fattr3(struct svc_rqst *rqstp, struct xdr_stream *xdr, + const struct svc_fh *fhp, const struct kstat *stat) +{ + struct user_namespace *userns = nfsd_user_namespace(rqstp); + __be32 *p; + u64 fsid; + + p = xdr_reserve_space(xdr, XDR_UNIT * 21); + if (!p) + return false; + + *p++ = cpu_to_be32(nfs3_ftypes[(stat->mode & S_IFMT) >> 12]); + *p++ = cpu_to_be32((u32)(stat->mode & S_IALLUGO)); + *p++ = cpu_to_be32((u32)stat->nlink); + *p++ = cpu_to_be32((u32)from_kuid_munged(userns, stat->uid)); + *p++ = cpu_to_be32((u32)from_kgid_munged(userns, stat->gid)); + if (S_ISLNK(stat->mode) && stat->size > NFS3_MAXPATHLEN) + p = xdr_encode_hyper(p, (u64)NFS3_MAXPATHLEN); + else + p = xdr_encode_hyper(p, (u64)stat->size); + + /* used */ + p = xdr_encode_hyper(p, ((u64)stat->blocks) << 9); + + /* rdev */ + *p++ = cpu_to_be32((u32)MAJOR(stat->rdev)); + *p++ = cpu_to_be32((u32)MINOR(stat->rdev)); + + switch(fsid_source(fhp)) { + case FSIDSOURCE_FSID: + fsid = (u64)fhp->fh_export->ex_fsid; + break; + case FSIDSOURCE_UUID: + fsid = ((u64 *)fhp->fh_export->ex_uuid)[0]; + fsid ^= ((u64 *)fhp->fh_export->ex_uuid)[1]; + break; + default: + fsid = (u64)huge_encode_dev(fhp->fh_dentry->d_sb->s_dev); + } + p = xdr_encode_hyper(p, fsid); + + /* fileid */ + p = xdr_encode_hyper(p, stat->ino); + + p = encode_nfstime3(p, &stat->atime); + p = encode_nfstime3(p, &stat->mtime); + encode_nfstime3(p, &stat->ctime); + + return true; +} + +static bool +svcxdr_encode_wcc_attr(struct xdr_stream *xdr, const struct svc_fh *fhp) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, XDR_UNIT * 6); + if (!p) + return false; + p = xdr_encode_hyper(p, (u64)fhp->fh_pre_size); + p = encode_nfstime3(p, &fhp->fh_pre_mtime); + encode_nfstime3(p, &fhp->fh_pre_ctime); + + return true; +} + +static bool +svcxdr_encode_pre_op_attr(struct xdr_stream *xdr, const struct svc_fh *fhp) +{ + if (!fhp->fh_pre_saved) { + if (xdr_stream_encode_item_absent(xdr) < 0) + return false; + return true; + } + + if (xdr_stream_encode_item_present(xdr) < 0) + return false; + return svcxdr_encode_wcc_attr(xdr, fhp); +} + +/** + * svcxdr_encode_post_op_attr - Encode NFSv3 post-op attributes + * @rqstp: Context of a completed RPC transaction + * @xdr: XDR stream + * @fhp: File handle to encode + * + * Return values: + * %false: Send buffer space was exhausted + * %true: Success + */ +bool +svcxdr_encode_post_op_attr(struct svc_rqst *rqstp, struct xdr_stream *xdr, + const struct svc_fh *fhp) +{ + struct dentry *dentry = fhp->fh_dentry; + struct kstat stat; + + /* + * The inode may be NULL if the call failed because of a + * stale file handle. In this case, no attributes are + * returned. + */ + if (fhp->fh_no_wcc || !dentry || !d_really_is_positive(dentry)) + goto no_post_op_attrs; + if (fh_getattr(fhp, &stat) != nfs_ok) + goto no_post_op_attrs; + + if (xdr_stream_encode_item_present(xdr) < 0) + return false; + lease_get_mtime(d_inode(dentry), &stat.mtime); + if (!svcxdr_encode_fattr3(rqstp, xdr, fhp, &stat)) + return false; + + return true; + +no_post_op_attrs: + return xdr_stream_encode_item_absent(xdr) > 0; +} + +/* + * Encode weak cache consistency data + */ +static bool +svcxdr_encode_wcc_data(struct svc_rqst *rqstp, struct xdr_stream *xdr, + const struct svc_fh *fhp) +{ + struct dentry *dentry = fhp->fh_dentry; + + if (!dentry || !d_really_is_positive(dentry) || !fhp->fh_post_saved) + goto neither; + + /* before */ + if (!svcxdr_encode_pre_op_attr(xdr, fhp)) + return false; + + /* after */ + if (xdr_stream_encode_item_present(xdr) < 0) + return false; + if (!svcxdr_encode_fattr3(rqstp, xdr, fhp, &fhp->fh_post_attr)) + return false; + + return true; + +neither: + if (xdr_stream_encode_item_absent(xdr) < 0) + return false; + if (!svcxdr_encode_post_op_attr(rqstp, xdr, fhp)) + return false; + + return true; +} + +/* + * XDR decode functions + */ + +bool +nfs3svc_decode_fhandleargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd_fhandle *args = rqstp->rq_argp; + + return svcxdr_decode_nfs_fh3(xdr, &args->fh); +} + +bool +nfs3svc_decode_sattrargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_sattrargs *args = rqstp->rq_argp; + + return svcxdr_decode_nfs_fh3(xdr, &args->fh) && + svcxdr_decode_sattr3(rqstp, xdr, &args->attrs) && + svcxdr_decode_sattrguard3(xdr, args); +} + +bool +nfs3svc_decode_diropargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_diropargs *args = rqstp->rq_argp; + + return svcxdr_decode_diropargs3(xdr, &args->fh, &args->name, &args->len); +} + +bool +nfs3svc_decode_accessargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_accessargs *args = rqstp->rq_argp; + + if (!svcxdr_decode_nfs_fh3(xdr, &args->fh)) + return false; + if (xdr_stream_decode_u32(xdr, &args->access) < 0) + return false; + + return true; +} + +bool +nfs3svc_decode_readargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_readargs *args = rqstp->rq_argp; + + if (!svcxdr_decode_nfs_fh3(xdr, &args->fh)) + return false; + if (xdr_stream_decode_u64(xdr, &args->offset) < 0) + return false; + if (xdr_stream_decode_u32(xdr, &args->count) < 0) + return false; + + return true; +} + +bool +nfs3svc_decode_writeargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_writeargs *args = rqstp->rq_argp; + u32 max_blocksize = svc_max_payload(rqstp); + + if (!svcxdr_decode_nfs_fh3(xdr, &args->fh)) + return false; + if (xdr_stream_decode_u64(xdr, &args->offset) < 0) + return false; + if (xdr_stream_decode_u32(xdr, &args->count) < 0) + return false; + if (xdr_stream_decode_u32(xdr, &args->stable) < 0) + return false; + + /* opaque data */ + if (xdr_stream_decode_u32(xdr, &args->len) < 0) + return false; + + /* request sanity */ + if (args->count != args->len) + return false; + if (args->count > max_blocksize) { + args->count = max_blocksize; + args->len = max_blocksize; + } + + return xdr_stream_subsegment(xdr, &args->payload, args->count); +} + +bool +nfs3svc_decode_createargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_createargs *args = rqstp->rq_argp; + + if (!svcxdr_decode_diropargs3(xdr, &args->fh, &args->name, &args->len)) + return false; + if (xdr_stream_decode_u32(xdr, &args->createmode) < 0) + return false; + switch (args->createmode) { + case NFS3_CREATE_UNCHECKED: + case NFS3_CREATE_GUARDED: + return svcxdr_decode_sattr3(rqstp, xdr, &args->attrs); + case NFS3_CREATE_EXCLUSIVE: + args->verf = xdr_inline_decode(xdr, NFS3_CREATEVERFSIZE); + if (!args->verf) + return false; + break; + default: + return false; + } + return true; +} + +bool +nfs3svc_decode_mkdirargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_createargs *args = rqstp->rq_argp; + + return svcxdr_decode_diropargs3(xdr, &args->fh, + &args->name, &args->len) && + svcxdr_decode_sattr3(rqstp, xdr, &args->attrs); +} + +bool +nfs3svc_decode_symlinkargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_symlinkargs *args = rqstp->rq_argp; + struct kvec *head = rqstp->rq_arg.head; + + if (!svcxdr_decode_diropargs3(xdr, &args->ffh, &args->fname, &args->flen)) + return false; + if (!svcxdr_decode_sattr3(rqstp, xdr, &args->attrs)) + return false; + if (xdr_stream_decode_u32(xdr, &args->tlen) < 0) + return false; + + /* symlink_data */ + args->first.iov_len = head->iov_len - xdr_stream_pos(xdr); + args->first.iov_base = xdr_inline_decode(xdr, args->tlen); + return args->first.iov_base != NULL; +} + +bool +nfs3svc_decode_mknodargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_mknodargs *args = rqstp->rq_argp; + + if (!svcxdr_decode_diropargs3(xdr, &args->fh, &args->name, &args->len)) + return false; + if (xdr_stream_decode_u32(xdr, &args->ftype) < 0) + return false; + switch (args->ftype) { + case NF3CHR: + case NF3BLK: + return svcxdr_decode_devicedata3(rqstp, xdr, args); + case NF3SOCK: + case NF3FIFO: + return svcxdr_decode_sattr3(rqstp, xdr, &args->attrs); + case NF3REG: + case NF3DIR: + case NF3LNK: + /* Valid XDR but illegal file types */ + break; + default: + return false; + } + + return true; +} + +bool +nfs3svc_decode_renameargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_renameargs *args = rqstp->rq_argp; + + return svcxdr_decode_diropargs3(xdr, &args->ffh, + &args->fname, &args->flen) && + svcxdr_decode_diropargs3(xdr, &args->tfh, + &args->tname, &args->tlen); +} + +bool +nfs3svc_decode_linkargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_linkargs *args = rqstp->rq_argp; + + return svcxdr_decode_nfs_fh3(xdr, &args->ffh) && + svcxdr_decode_diropargs3(xdr, &args->tfh, + &args->tname, &args->tlen); +} + +bool +nfs3svc_decode_readdirargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_readdirargs *args = rqstp->rq_argp; + + if (!svcxdr_decode_nfs_fh3(xdr, &args->fh)) + return false; + if (xdr_stream_decode_u64(xdr, &args->cookie) < 0) + return false; + args->verf = xdr_inline_decode(xdr, NFS3_COOKIEVERFSIZE); + if (!args->verf) + return false; + if (xdr_stream_decode_u32(xdr, &args->count) < 0) + return false; + + return true; +} + +bool +nfs3svc_decode_readdirplusargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_readdirargs *args = rqstp->rq_argp; + u32 dircount; + + if (!svcxdr_decode_nfs_fh3(xdr, &args->fh)) + return false; + if (xdr_stream_decode_u64(xdr, &args->cookie) < 0) + return false; + args->verf = xdr_inline_decode(xdr, NFS3_COOKIEVERFSIZE); + if (!args->verf) + return false; + /* dircount is ignored */ + if (xdr_stream_decode_u32(xdr, &dircount) < 0) + return false; + if (xdr_stream_decode_u32(xdr, &args->count) < 0) + return false; + + return true; +} + +bool +nfs3svc_decode_commitargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_commitargs *args = rqstp->rq_argp; + + if (!svcxdr_decode_nfs_fh3(xdr, &args->fh)) + return false; + if (xdr_stream_decode_u64(xdr, &args->offset) < 0) + return false; + if (xdr_stream_decode_u32(xdr, &args->count) < 0) + return false; + + return true; +} + +/* + * XDR encode functions + */ + +/* GETATTR */ +bool +nfs3svc_encode_getattrres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_attrstat *resp = rqstp->rq_resp; + + if (!svcxdr_encode_nfsstat3(xdr, resp->status)) + return false; + switch (resp->status) { + case nfs_ok: + lease_get_mtime(d_inode(resp->fh.fh_dentry), &resp->stat.mtime); + if (!svcxdr_encode_fattr3(rqstp, xdr, &resp->fh, &resp->stat)) + return false; + break; + } + + return true; +} + +/* SETATTR, REMOVE, RMDIR */ +bool +nfs3svc_encode_wccstat(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_attrstat *resp = rqstp->rq_resp; + + return svcxdr_encode_nfsstat3(xdr, resp->status) && + svcxdr_encode_wcc_data(rqstp, xdr, &resp->fh); +} + +/* LOOKUP */ +bool +nfs3svc_encode_lookupres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_diropres *resp = rqstp->rq_resp; + + if (!svcxdr_encode_nfsstat3(xdr, resp->status)) + return false; + switch (resp->status) { + case nfs_ok: + if (!svcxdr_encode_nfs_fh3(xdr, &resp->fh)) + return false; + if (!svcxdr_encode_post_op_attr(rqstp, xdr, &resp->fh)) + return false; + if (!svcxdr_encode_post_op_attr(rqstp, xdr, &resp->dirfh)) + return false; + break; + default: + if (!svcxdr_encode_post_op_attr(rqstp, xdr, &resp->dirfh)) + return false; + } + + return true; +} + +/* ACCESS */ +bool +nfs3svc_encode_accessres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_accessres *resp = rqstp->rq_resp; + + if (!svcxdr_encode_nfsstat3(xdr, resp->status)) + return false; + switch (resp->status) { + case nfs_ok: + if (!svcxdr_encode_post_op_attr(rqstp, xdr, &resp->fh)) + return false; + if (xdr_stream_encode_u32(xdr, resp->access) < 0) + return false; + break; + default: + if (!svcxdr_encode_post_op_attr(rqstp, xdr, &resp->fh)) + return false; + } + + return true; +} + +/* READLINK */ +bool +nfs3svc_encode_readlinkres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_readlinkres *resp = rqstp->rq_resp; + struct kvec *head = rqstp->rq_res.head; + + if (!svcxdr_encode_nfsstat3(xdr, resp->status)) + return false; + switch (resp->status) { + case nfs_ok: + if (!svcxdr_encode_post_op_attr(rqstp, xdr, &resp->fh)) + return false; + if (xdr_stream_encode_u32(xdr, resp->len) < 0) + return false; + svcxdr_encode_opaque_pages(rqstp, xdr, resp->pages, 0, + resp->len); + if (svc_encode_result_payload(rqstp, head->iov_len, resp->len) < 0) + return false; + break; + default: + if (!svcxdr_encode_post_op_attr(rqstp, xdr, &resp->fh)) + return false; + } + + return true; +} + +/* READ */ +bool +nfs3svc_encode_readres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_readres *resp = rqstp->rq_resp; + struct kvec *head = rqstp->rq_res.head; + + if (!svcxdr_encode_nfsstat3(xdr, resp->status)) + return false; + switch (resp->status) { + case nfs_ok: + if (!svcxdr_encode_post_op_attr(rqstp, xdr, &resp->fh)) + return false; + if (xdr_stream_encode_u32(xdr, resp->count) < 0) + return false; + if (xdr_stream_encode_bool(xdr, resp->eof) < 0) + return false; + if (xdr_stream_encode_u32(xdr, resp->count) < 0) + return false; + svcxdr_encode_opaque_pages(rqstp, xdr, resp->pages, + rqstp->rq_res.page_base, + resp->count); + if (svc_encode_result_payload(rqstp, head->iov_len, resp->count) < 0) + return false; + break; + default: + if (!svcxdr_encode_post_op_attr(rqstp, xdr, &resp->fh)) + return false; + } + + return true; +} + +/* WRITE */ +bool +nfs3svc_encode_writeres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_writeres *resp = rqstp->rq_resp; + + if (!svcxdr_encode_nfsstat3(xdr, resp->status)) + return false; + switch (resp->status) { + case nfs_ok: + if (!svcxdr_encode_wcc_data(rqstp, xdr, &resp->fh)) + return false; + if (xdr_stream_encode_u32(xdr, resp->count) < 0) + return false; + if (xdr_stream_encode_u32(xdr, resp->committed) < 0) + return false; + if (!svcxdr_encode_writeverf3(xdr, resp->verf)) + return false; + break; + default: + if (!svcxdr_encode_wcc_data(rqstp, xdr, &resp->fh)) + return false; + } + + return true; +} + +/* CREATE, MKDIR, SYMLINK, MKNOD */ +bool +nfs3svc_encode_createres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_diropres *resp = rqstp->rq_resp; + + if (!svcxdr_encode_nfsstat3(xdr, resp->status)) + return false; + switch (resp->status) { + case nfs_ok: + if (!svcxdr_encode_post_op_fh3(xdr, &resp->fh)) + return false; + if (!svcxdr_encode_post_op_attr(rqstp, xdr, &resp->fh)) + return false; + if (!svcxdr_encode_wcc_data(rqstp, xdr, &resp->dirfh)) + return false; + break; + default: + if (!svcxdr_encode_wcc_data(rqstp, xdr, &resp->dirfh)) + return false; + } + + return true; +} + +/* RENAME */ +bool +nfs3svc_encode_renameres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_renameres *resp = rqstp->rq_resp; + + return svcxdr_encode_nfsstat3(xdr, resp->status) && + svcxdr_encode_wcc_data(rqstp, xdr, &resp->ffh) && + svcxdr_encode_wcc_data(rqstp, xdr, &resp->tfh); +} + +/* LINK */ +bool +nfs3svc_encode_linkres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_linkres *resp = rqstp->rq_resp; + + return svcxdr_encode_nfsstat3(xdr, resp->status) && + svcxdr_encode_post_op_attr(rqstp, xdr, &resp->fh) && + svcxdr_encode_wcc_data(rqstp, xdr, &resp->tfh); +} + +/* READDIR */ +bool +nfs3svc_encode_readdirres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_readdirres *resp = rqstp->rq_resp; + struct xdr_buf *dirlist = &resp->dirlist; + + if (!svcxdr_encode_nfsstat3(xdr, resp->status)) + return false; + switch (resp->status) { + case nfs_ok: + if (!svcxdr_encode_post_op_attr(rqstp, xdr, &resp->fh)) + return false; + if (!svcxdr_encode_cookieverf3(xdr, resp->verf)) + return false; + svcxdr_encode_opaque_pages(rqstp, xdr, dirlist->pages, 0, + dirlist->len); + /* no more entries */ + if (xdr_stream_encode_item_absent(xdr) < 0) + return false; + if (xdr_stream_encode_bool(xdr, resp->common.err == nfserr_eof) < 0) + return false; + break; + default: + if (!svcxdr_encode_post_op_attr(rqstp, xdr, &resp->fh)) + return false; + } + + return true; +} + +static __be32 +compose_entry_fh(struct nfsd3_readdirres *cd, struct svc_fh *fhp, + const char *name, int namlen, u64 ino) +{ + struct svc_export *exp; + struct dentry *dparent, *dchild; + __be32 rv = nfserr_noent; + + dparent = cd->fh.fh_dentry; + exp = cd->fh.fh_export; + + if (name_is_dot_dotdot(name, namlen)) { + if (namlen == 2) { + dchild = dget_parent(dparent); + /* + * Don't return filehandle for ".." if we're at + * the filesystem or export root: + */ + if (dchild == dparent) + goto out; + if (dparent == exp->ex_path.dentry) + goto out; + } else + dchild = dget(dparent); + } else + dchild = lookup_one_positive_unlocked(&nop_mnt_idmap, + &QSTR_LEN(name, namlen), + dparent); + if (IS_ERR(dchild)) + return rv; + if (d_mountpoint(dchild)) + goto out; + if (dchild->d_inode->i_ino != ino) + goto out; + rv = fh_compose(fhp, exp, dchild, &cd->fh); +out: + dput(dchild); + return rv; +} + +/** + * nfs3svc_encode_cookie3 - Encode a directory offset cookie + * @resp: readdir result context + * @offset: offset cookie to encode + * + * The buffer space for the offset cookie has already been reserved + * by svcxdr_encode_entry3_common(). + */ +void nfs3svc_encode_cookie3(struct nfsd3_readdirres *resp, u64 offset) +{ + __be64 cookie = cpu_to_be64(offset); + + if (!resp->cookie_offset) + return; + write_bytes_to_xdr_buf(&resp->dirlist, resp->cookie_offset, &cookie, + sizeof(cookie)); + resp->cookie_offset = 0; +} + +static bool +svcxdr_encode_entry3_common(struct nfsd3_readdirres *resp, const char *name, + int namlen, loff_t offset, u64 ino) +{ + struct xdr_buf *dirlist = &resp->dirlist; + struct xdr_stream *xdr = &resp->xdr; + + if (xdr_stream_encode_item_present(xdr) < 0) + return false; + /* fileid */ + if (xdr_stream_encode_u64(xdr, ino) < 0) + return false; + /* name */ + if (xdr_stream_encode_opaque(xdr, name, min(namlen, NFS3_MAXNAMLEN)) < 0) + return false; + /* cookie */ + resp->cookie_offset = dirlist->len; + if (xdr_stream_encode_u64(xdr, OFFSET_MAX) < 0) + return false; + + return true; +} + +/** + * nfs3svc_encode_entry3 - encode one NFSv3 READDIR entry + * @data: directory context + * @name: name of the object to be encoded + * @namlen: length of that name, in bytes + * @offset: the offset of the previous entry + * @ino: the fileid of this entry + * @d_type: unused + * + * Return values: + * %0: Entry was successfully encoded. + * %-EINVAL: An encoding problem occurred, secondary status code in resp->common.err + * + * On exit, the following fields are updated: + * - resp->xdr + * - resp->common.err + * - resp->cookie_offset + */ +int nfs3svc_encode_entry3(void *data, const char *name, int namlen, + loff_t offset, u64 ino, unsigned int d_type) +{ + struct readdir_cd *ccd = data; + struct nfsd3_readdirres *resp = container_of(ccd, + struct nfsd3_readdirres, + common); + unsigned int starting_length = resp->dirlist.len; + + /* The offset cookie for the previous entry */ + nfs3svc_encode_cookie3(resp, offset); + + if (!svcxdr_encode_entry3_common(resp, name, namlen, offset, ino)) + goto out_toosmall; + + xdr_commit_encode(&resp->xdr); + resp->common.err = nfs_ok; + return 0; + +out_toosmall: + resp->cookie_offset = 0; + resp->common.err = nfserr_toosmall; + resp->dirlist.len = starting_length; + return -EINVAL; +} + +static bool +svcxdr_encode_entry3_plus(struct nfsd3_readdirres *resp, const char *name, + int namlen, u64 ino) +{ + struct xdr_stream *xdr = &resp->xdr; + struct svc_fh *fhp = &resp->scratch; + bool result; + + result = false; + fh_init(fhp, NFS3_FHSIZE); + if (compose_entry_fh(resp, fhp, name, namlen, ino) != nfs_ok) + goto out_noattrs; + + if (!svcxdr_encode_post_op_attr(resp->rqstp, xdr, fhp)) + goto out; + if (!svcxdr_encode_post_op_fh3(xdr, fhp)) + goto out; + result = true; + +out: + fh_put(fhp); + return result; + +out_noattrs: + if (xdr_stream_encode_item_absent(xdr) < 0) + return false; + if (xdr_stream_encode_item_absent(xdr) < 0) + return false; + return true; +} + +/** + * nfs3svc_encode_entryplus3 - encode one NFSv3 READDIRPLUS entry + * @data: directory context + * @name: name of the object to be encoded + * @namlen: length of that name, in bytes + * @offset: the offset of the previous entry + * @ino: the fileid of this entry + * @d_type: unused + * + * Return values: + * %0: Entry was successfully encoded. + * %-EINVAL: An encoding problem occurred, secondary status code in resp->common.err + * + * On exit, the following fields are updated: + * - resp->xdr + * - resp->common.err + * - resp->cookie_offset + */ +int nfs3svc_encode_entryplus3(void *data, const char *name, int namlen, + loff_t offset, u64 ino, unsigned int d_type) +{ + struct readdir_cd *ccd = data; + struct nfsd3_readdirres *resp = container_of(ccd, + struct nfsd3_readdirres, + common); + unsigned int starting_length = resp->dirlist.len; + + /* The offset cookie for the previous entry */ + nfs3svc_encode_cookie3(resp, offset); + + if (!svcxdr_encode_entry3_common(resp, name, namlen, offset, ino)) + goto out_toosmall; + if (!svcxdr_encode_entry3_plus(resp, name, namlen, ino)) + goto out_toosmall; + + xdr_commit_encode(&resp->xdr); + resp->common.err = nfs_ok; + return 0; + +out_toosmall: + resp->cookie_offset = 0; + resp->common.err = nfserr_toosmall; + resp->dirlist.len = starting_length; + return -EINVAL; +} + +static bool +svcxdr_encode_fsstat3resok(struct xdr_stream *xdr, + const struct nfsd3_fsstatres *resp) +{ + const struct kstatfs *s = &resp->stats; + u64 bs = s->f_bsize; + __be32 *p; + + p = xdr_reserve_space(xdr, XDR_UNIT * 13); + if (!p) + return false; + p = xdr_encode_hyper(p, bs * s->f_blocks); /* total bytes */ + p = xdr_encode_hyper(p, bs * s->f_bfree); /* free bytes */ + p = xdr_encode_hyper(p, bs * s->f_bavail); /* user available bytes */ + p = xdr_encode_hyper(p, s->f_files); /* total inodes */ + p = xdr_encode_hyper(p, s->f_ffree); /* free inodes */ + p = xdr_encode_hyper(p, s->f_ffree); /* user available inodes */ + *p = cpu_to_be32(resp->invarsec); /* mean unchanged time */ + + return true; +} + +/* FSSTAT */ +bool +nfs3svc_encode_fsstatres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_fsstatres *resp = rqstp->rq_resp; + + if (!svcxdr_encode_nfsstat3(xdr, resp->status)) + return false; + switch (resp->status) { + case nfs_ok: + if (!svcxdr_encode_post_op_attr(rqstp, xdr, &nfs3svc_null_fh)) + return false; + if (!svcxdr_encode_fsstat3resok(xdr, resp)) + return false; + break; + default: + if (!svcxdr_encode_post_op_attr(rqstp, xdr, &nfs3svc_null_fh)) + return false; + } + + return true; +} + +static bool +svcxdr_encode_fsinfo3resok(struct xdr_stream *xdr, + const struct nfsd3_fsinfores *resp) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, XDR_UNIT * 12); + if (!p) + return false; + *p++ = cpu_to_be32(resp->f_rtmax); + *p++ = cpu_to_be32(resp->f_rtpref); + *p++ = cpu_to_be32(resp->f_rtmult); + *p++ = cpu_to_be32(resp->f_wtmax); + *p++ = cpu_to_be32(resp->f_wtpref); + *p++ = cpu_to_be32(resp->f_wtmult); + *p++ = cpu_to_be32(resp->f_dtpref); + p = xdr_encode_hyper(p, resp->f_maxfilesize); + p = encode_nfstime3(p, &nfs3svc_time_delta); + *p = cpu_to_be32(resp->f_properties); + + return true; +} + +/* FSINFO */ +bool +nfs3svc_encode_fsinfores(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_fsinfores *resp = rqstp->rq_resp; + + if (!svcxdr_encode_nfsstat3(xdr, resp->status)) + return false; + switch (resp->status) { + case nfs_ok: + if (!svcxdr_encode_post_op_attr(rqstp, xdr, &nfs3svc_null_fh)) + return false; + if (!svcxdr_encode_fsinfo3resok(xdr, resp)) + return false; + break; + default: + if (!svcxdr_encode_post_op_attr(rqstp, xdr, &nfs3svc_null_fh)) + return false; + } + + return true; +} + +static bool +svcxdr_encode_pathconf3resok(struct xdr_stream *xdr, + const struct nfsd3_pathconfres *resp) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, XDR_UNIT * 6); + if (!p) + return false; + *p++ = cpu_to_be32(resp->p_link_max); + *p++ = cpu_to_be32(resp->p_name_max); + p = xdr_encode_bool(p, resp->p_no_trunc); + p = xdr_encode_bool(p, resp->p_chown_restricted); + p = xdr_encode_bool(p, resp->p_case_insensitive); + xdr_encode_bool(p, resp->p_case_preserving); + + return true; +} + +/* PATHCONF */ +bool +nfs3svc_encode_pathconfres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_pathconfres *resp = rqstp->rq_resp; + + if (!svcxdr_encode_nfsstat3(xdr, resp->status)) + return false; + switch (resp->status) { + case nfs_ok: + if (!svcxdr_encode_post_op_attr(rqstp, xdr, &nfs3svc_null_fh)) + return false; + if (!svcxdr_encode_pathconf3resok(xdr, resp)) + return false; + break; + default: + if (!svcxdr_encode_post_op_attr(rqstp, xdr, &nfs3svc_null_fh)) + return false; + } + + return true; +} + +/* COMMIT */ +bool +nfs3svc_encode_commitres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd3_commitres *resp = rqstp->rq_resp; + + if (!svcxdr_encode_nfsstat3(xdr, resp->status)) + return false; + switch (resp->status) { + case nfs_ok: + if (!svcxdr_encode_wcc_data(rqstp, xdr, &resp->fh)) + return false; + if (!svcxdr_encode_writeverf3(xdr, resp->verf)) + return false; + break; + default: + if (!svcxdr_encode_wcc_data(rqstp, xdr, &resp->fh)) + return false; + } + + return true; +} + +/* + * XDR release functions + */ +void +nfs3svc_release_fhandle(struct svc_rqst *rqstp) +{ + struct nfsd3_attrstat *resp = rqstp->rq_resp; + + fh_put(&resp->fh); +} + +void +nfs3svc_release_fhandle2(struct svc_rqst *rqstp) +{ + struct nfsd3_fhandle_pair *resp = rqstp->rq_resp; + + fh_put(&resp->fh1); + fh_put(&resp->fh2); +} diff --git a/fs/nfsd/nfs4acl.c b/fs/nfsd/nfs4acl.c new file mode 100644 index 000000000..2c2f2fd89 --- /dev/null +++ b/fs/nfsd/nfs4acl.c @@ -0,0 +1,884 @@ +/* + * Common NFSv4 ACL handling code. + * + * Copyright (c) 2002, 2003 The Regents of the University of Michigan. + * All rights reserved. + * + * Marius Aamodt Eriksen + * Jeff Sedlak + * J. Bruce Fields + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +#include "nfsfh.h" +#include "nfsd.h" +#include "acl.h" +#include "vfs.h" + +#define NFS4_ACL_TYPE_DEFAULT 0x01 +#define NFS4_ACL_DIR 0x02 +#define NFS4_ACL_OWNER 0x04 + +/* mode bit translations: */ +#define NFS4_READ_MODE (NFS4_ACE_READ_DATA) +#define NFS4_WRITE_MODE (NFS4_ACE_WRITE_DATA | NFS4_ACE_APPEND_DATA) +#define NFS4_EXECUTE_MODE NFS4_ACE_EXECUTE +#define NFS4_ANYONE_MODE (NFS4_ACE_READ_ATTRIBUTES | NFS4_ACE_READ_ACL | NFS4_ACE_SYNCHRONIZE) +#define NFS4_OWNER_MODE (NFS4_ACE_WRITE_ATTRIBUTES | NFS4_ACE_WRITE_ACL) + +/* flags used to simulate posix default ACLs */ +#define NFS4_INHERITANCE_FLAGS (NFS4_ACE_FILE_INHERIT_ACE \ + | NFS4_ACE_DIRECTORY_INHERIT_ACE) + +#define NFS4_SUPPORTED_FLAGS (NFS4_INHERITANCE_FLAGS \ + | NFS4_ACE_INHERIT_ONLY_ACE \ + | NFS4_ACE_IDENTIFIER_GROUP) + +static u32 +mask_from_posix(unsigned short perm, unsigned int flags) +{ + int mask = NFS4_ANYONE_MODE; + + if (flags & NFS4_ACL_OWNER) + mask |= NFS4_OWNER_MODE; + if (perm & ACL_READ) + mask |= NFS4_READ_MODE; + if (perm & ACL_WRITE) + mask |= NFS4_WRITE_MODE; + if ((perm & ACL_WRITE) && (flags & NFS4_ACL_DIR)) + mask |= NFS4_ACE_DELETE_CHILD; + if (perm & ACL_EXECUTE) + mask |= NFS4_EXECUTE_MODE; + return mask; +} + +static u32 +deny_mask_from_posix(unsigned short perm, u32 flags) +{ + u32 mask = 0; + + if (perm & ACL_READ) + mask |= NFS4_READ_MODE; + if (perm & ACL_WRITE) + mask |= NFS4_WRITE_MODE; + if ((perm & ACL_WRITE) && (flags & NFS4_ACL_DIR)) + mask |= NFS4_ACE_DELETE_CHILD; + if (perm & ACL_EXECUTE) + mask |= NFS4_EXECUTE_MODE; + return mask; +} + +/* XXX: modify functions to return NFS errors; they're only ever + * used by nfs code, after all.... */ + +/* We only map from NFSv4 to POSIX ACLs when setting ACLs, when we err on the + * side of being more restrictive, so the mode bit mapping below is + * pessimistic. An optimistic version would be needed to handle DENY's, + * but we expect to coalesce all ALLOWs and DENYs before mapping to mode + * bits. */ + +static void +low_mode_from_nfs4(u32 perm, unsigned short *mode, unsigned int flags) +{ + u32 write_mode = NFS4_WRITE_MODE; + + if (flags & NFS4_ACL_DIR) + write_mode |= NFS4_ACE_DELETE_CHILD; + *mode = 0; + if ((perm & NFS4_READ_MODE) == NFS4_READ_MODE) + *mode |= ACL_READ; + if ((perm & write_mode) == write_mode) + *mode |= ACL_WRITE; + if ((perm & NFS4_EXECUTE_MODE) == NFS4_EXECUTE_MODE) + *mode |= ACL_EXECUTE; +} + +static short ace2type(struct nfs4_ace *); +static void _posix_to_nfsv4_one(struct posix_acl *, struct nfs4_acl *, + unsigned int); + +int +nfsd4_get_nfs4_acl(struct svc_rqst *rqstp, struct dentry *dentry, + struct nfs4_acl **acl) +{ + struct inode *inode = d_inode(dentry); + int error = 0; + struct posix_acl *pacl = NULL, *dpacl = NULL; + unsigned int flags = 0; + int size = 0; + + pacl = get_inode_acl(inode, ACL_TYPE_ACCESS); + if (!pacl) + pacl = posix_acl_from_mode(inode->i_mode, GFP_KERNEL); + + if (IS_ERR(pacl)) + return PTR_ERR(pacl); + + /* allocate for worst case: one (deny, allow) pair each: */ + size += 2 * pacl->a_count; + + if (S_ISDIR(inode->i_mode)) { + flags = NFS4_ACL_DIR; + dpacl = get_inode_acl(inode, ACL_TYPE_DEFAULT); + if (IS_ERR(dpacl)) { + error = PTR_ERR(dpacl); + goto rel_pacl; + } + + if (dpacl) + size += 2 * dpacl->a_count; + } + + *acl = kmalloc(nfs4_acl_bytes(size), GFP_KERNEL); + if (*acl == NULL) { + error = -ENOMEM; + goto out; + } + (*acl)->naces = 0; + + _posix_to_nfsv4_one(pacl, *acl, flags & ~NFS4_ACL_TYPE_DEFAULT); + + if (dpacl) + _posix_to_nfsv4_one(dpacl, *acl, flags | NFS4_ACL_TYPE_DEFAULT); + +out: + posix_acl_release(dpacl); +rel_pacl: + posix_acl_release(pacl); + return error; +} + +struct posix_acl_summary { + unsigned short owner; + unsigned short users; + unsigned short group; + unsigned short groups; + unsigned short other; + unsigned short mask; +}; + +static void +summarize_posix_acl(struct posix_acl *acl, struct posix_acl_summary *pas) +{ + struct posix_acl_entry *pa, *pe; + + /* + * Only pas.users and pas.groups need initialization; previous + * posix_acl_valid() calls ensure that the other fields will be + * initialized in the following loop. But, just to placate gcc: + */ + memset(pas, 0, sizeof(*pas)); + pas->mask = 07; + + FOREACH_ACL_ENTRY(pa, acl, pe) { + switch (pa->e_tag) { + case ACL_USER_OBJ: + pas->owner = pa->e_perm; + break; + case ACL_GROUP_OBJ: + pas->group = pa->e_perm; + break; + case ACL_USER: + pas->users |= pa->e_perm; + break; + case ACL_GROUP: + pas->groups |= pa->e_perm; + break; + case ACL_OTHER: + pas->other = pa->e_perm; + break; + case ACL_MASK: + pas->mask = pa->e_perm; + break; + } + } + /* We'll only care about effective permissions: */ + pas->users &= pas->mask; + pas->group &= pas->mask; + pas->groups &= pas->mask; +} + +/* We assume the acl has been verified with posix_acl_valid. */ +static void +_posix_to_nfsv4_one(struct posix_acl *pacl, struct nfs4_acl *acl, + unsigned int flags) +{ + struct posix_acl_entry *pa, *group_owner_entry; + struct nfs4_ace *ace; + struct posix_acl_summary pas; + unsigned short deny; + int eflag = ((flags & NFS4_ACL_TYPE_DEFAULT) ? + NFS4_INHERITANCE_FLAGS | NFS4_ACE_INHERIT_ONLY_ACE : 0); + + BUG_ON(pacl->a_count < 3); + summarize_posix_acl(pacl, &pas); + + pa = pacl->a_entries; + ace = acl->aces + acl->naces; + + /* We could deny everything not granted by the owner: */ + deny = ~pas.owner; + /* + * but it is equivalent (and simpler) to deny only what is not + * granted by later entries: + */ + deny &= pas.users | pas.group | pas.groups | pas.other; + if (deny) { + ace->type = NFS4_ACE_ACCESS_DENIED_ACE_TYPE; + ace->flag = eflag; + ace->access_mask = deny_mask_from_posix(deny, flags); + ace->whotype = NFS4_ACL_WHO_OWNER; + ace++; + acl->naces++; + } + + ace->type = NFS4_ACE_ACCESS_ALLOWED_ACE_TYPE; + ace->flag = eflag; + ace->access_mask = mask_from_posix(pa->e_perm, flags | NFS4_ACL_OWNER); + ace->whotype = NFS4_ACL_WHO_OWNER; + ace++; + acl->naces++; + pa++; + + while (pa->e_tag == ACL_USER) { + deny = ~(pa->e_perm & pas.mask); + deny &= pas.groups | pas.group | pas.other; + if (deny) { + ace->type = NFS4_ACE_ACCESS_DENIED_ACE_TYPE; + ace->flag = eflag; + ace->access_mask = deny_mask_from_posix(deny, flags); + ace->whotype = NFS4_ACL_WHO_NAMED; + ace->who_uid = pa->e_uid; + ace++; + acl->naces++; + } + ace->type = NFS4_ACE_ACCESS_ALLOWED_ACE_TYPE; + ace->flag = eflag; + ace->access_mask = mask_from_posix(pa->e_perm & pas.mask, + flags); + ace->whotype = NFS4_ACL_WHO_NAMED; + ace->who_uid = pa->e_uid; + ace++; + acl->naces++; + pa++; + } + + /* In the case of groups, we apply allow ACEs first, then deny ACEs, + * since a user can be in more than one group. */ + + /* allow ACEs */ + + group_owner_entry = pa; + + ace->type = NFS4_ACE_ACCESS_ALLOWED_ACE_TYPE; + ace->flag = eflag; + ace->access_mask = mask_from_posix(pas.group, flags); + ace->whotype = NFS4_ACL_WHO_GROUP; + ace++; + acl->naces++; + pa++; + + while (pa->e_tag == ACL_GROUP) { + ace->type = NFS4_ACE_ACCESS_ALLOWED_ACE_TYPE; + ace->flag = eflag | NFS4_ACE_IDENTIFIER_GROUP; + ace->access_mask = mask_from_posix(pa->e_perm & pas.mask, + flags); + ace->whotype = NFS4_ACL_WHO_NAMED; + ace->who_gid = pa->e_gid; + ace++; + acl->naces++; + pa++; + } + + /* deny ACEs */ + + pa = group_owner_entry; + + deny = ~pas.group & pas.other; + if (deny) { + ace->type = NFS4_ACE_ACCESS_DENIED_ACE_TYPE; + ace->flag = eflag; + ace->access_mask = deny_mask_from_posix(deny, flags); + ace->whotype = NFS4_ACL_WHO_GROUP; + ace++; + acl->naces++; + } + pa++; + + while (pa->e_tag == ACL_GROUP) { + deny = ~(pa->e_perm & pas.mask); + deny &= pas.other; + if (deny) { + ace->type = NFS4_ACE_ACCESS_DENIED_ACE_TYPE; + ace->flag = eflag | NFS4_ACE_IDENTIFIER_GROUP; + ace->access_mask = deny_mask_from_posix(deny, flags); + ace->whotype = NFS4_ACL_WHO_NAMED; + ace->who_gid = pa->e_gid; + ace++; + acl->naces++; + } + pa++; + } + + if (pa->e_tag == ACL_MASK) + pa++; + ace->type = NFS4_ACE_ACCESS_ALLOWED_ACE_TYPE; + ace->flag = eflag; + ace->access_mask = mask_from_posix(pa->e_perm, flags); + ace->whotype = NFS4_ACL_WHO_EVERYONE; + acl->naces++; +} + +static bool +pace_gt(struct posix_acl_entry *pace1, struct posix_acl_entry *pace2) +{ + if (pace1->e_tag != pace2->e_tag) + return pace1->e_tag > pace2->e_tag; + if (pace1->e_tag == ACL_USER) + return uid_gt(pace1->e_uid, pace2->e_uid); + if (pace1->e_tag == ACL_GROUP) + return gid_gt(pace1->e_gid, pace2->e_gid); + return false; +} + +/** + * sort_pacl_range - sort a range of POSIX ACL entries by tag and id + * @pacl: POSIX ACL containing entries to sort + * @start: starting index of range to sort + * @end: ending index of range to sort (inclusive) + * + * Sorts ACL entries in place so that USER entries are ordered by UID + * and GROUP entries are ordered by GID. Required before calling + * posix_acl_valid(). + */ +void sort_pacl_range(struct posix_acl *pacl, int start, int end) +{ + int sorted = 0, i; + + /* Bubble sort: acceptable here because ACLs are typically short. */ + while (!sorted) { + sorted = 1; + for (i = start; i < end; i++) { + if (pace_gt(&pacl->a_entries[i], + &pacl->a_entries[i+1])) { + sorted = 0; + swap(pacl->a_entries[i], + pacl->a_entries[i + 1]); + } + } + } +} + +static void +sort_pacl(struct posix_acl *pacl) +{ + /* posix_acl_valid requires that users and groups be in order + * by uid/gid. */ + int i, j; + + /* no users or groups */ + if (!pacl || pacl->a_count <= 4) + return; + + i = 1; + while (pacl->a_entries[i].e_tag == ACL_USER) + i++; + sort_pacl_range(pacl, 1, i-1); + + BUG_ON(pacl->a_entries[i].e_tag != ACL_GROUP_OBJ); + j = ++i; + while (pacl->a_entries[j].e_tag == ACL_GROUP) + j++; + sort_pacl_range(pacl, i, j-1); + return; +} + +/* + * While processing the NFSv4 ACE, this maintains bitmasks representing + * which permission bits have been allowed and which denied to a given + * entity: */ +struct posix_ace_state { + u32 allow; + u32 deny; +}; + +struct posix_user_ace_state { + union { + kuid_t uid; + kgid_t gid; + }; + struct posix_ace_state perms; +}; + +struct posix_ace_state_array { + int n; + struct posix_user_ace_state aces[]; +}; + +/* + * While processing the NFSv4 ACE, this maintains the partial permissions + * calculated so far: */ + +struct posix_acl_state { + unsigned char valid; + struct posix_ace_state owner; + struct posix_ace_state group; + struct posix_ace_state other; + struct posix_ace_state everyone; + struct posix_ace_state mask; /* Deny unused in this case */ + struct posix_ace_state_array *users; + struct posix_ace_state_array *groups; +}; + +static int +init_state(struct posix_acl_state *state, int cnt) +{ + int alloc; + + memset(state, 0, sizeof(struct posix_acl_state)); + /* + * In the worst case, each individual acl could be for a distinct + * named user or group, but we don't know which, so we allocate + * enough space for either: + */ + alloc = sizeof(struct posix_ace_state_array) + + cnt*sizeof(struct posix_user_ace_state); + state->users = kzalloc(alloc, GFP_KERNEL); + if (!state->users) + return -ENOMEM; + state->groups = kzalloc(alloc, GFP_KERNEL); + if (!state->groups) { + kfree(state->users); + return -ENOMEM; + } + return 0; +} + +static void +free_state(struct posix_acl_state *state) { + kfree(state->users); + kfree(state->groups); +} + +static inline void add_to_mask(struct posix_acl_state *state, struct posix_ace_state *astate) +{ + state->mask.allow |= astate->allow; +} + +static struct posix_acl * +posix_state_to_acl(struct posix_acl_state *state, unsigned int flags) +{ + struct posix_acl_entry *pace; + struct posix_acl *pacl; + int nace; + int i; + + /* + * ACLs with no ACEs are treated differently in the inheritable + * and effective cases: when there are no inheritable ACEs, + * calls ->set_acl with a NULL ACL structure. + */ + if (!state->valid && (flags & NFS4_ACL_TYPE_DEFAULT)) + return NULL; + + /* + * When there are no effective ACEs, the following will end + * up setting a 3-element effective posix ACL with all + * permissions zero. + */ + if (!state->users->n && !state->groups->n) + nace = 3; + else /* Note we also include a MASK ACE in this case: */ + nace = 4 + state->users->n + state->groups->n; + pacl = posix_acl_alloc(nace, GFP_KERNEL); + if (!pacl) + return ERR_PTR(-ENOMEM); + + pace = pacl->a_entries; + pace->e_tag = ACL_USER_OBJ; + low_mode_from_nfs4(state->owner.allow, &pace->e_perm, flags); + + for (i=0; i < state->users->n; i++) { + pace++; + pace->e_tag = ACL_USER; + low_mode_from_nfs4(state->users->aces[i].perms.allow, + &pace->e_perm, flags); + pace->e_uid = state->users->aces[i].uid; + add_to_mask(state, &state->users->aces[i].perms); + } + + pace++; + pace->e_tag = ACL_GROUP_OBJ; + low_mode_from_nfs4(state->group.allow, &pace->e_perm, flags); + add_to_mask(state, &state->group); + + for (i=0; i < state->groups->n; i++) { + pace++; + pace->e_tag = ACL_GROUP; + low_mode_from_nfs4(state->groups->aces[i].perms.allow, + &pace->e_perm, flags); + pace->e_gid = state->groups->aces[i].gid; + add_to_mask(state, &state->groups->aces[i].perms); + } + + if (state->users->n || state->groups->n) { + pace++; + pace->e_tag = ACL_MASK; + low_mode_from_nfs4(state->mask.allow, &pace->e_perm, flags); + } + + pace++; + pace->e_tag = ACL_OTHER; + low_mode_from_nfs4(state->other.allow, &pace->e_perm, flags); + + return pacl; +} + +static inline void allow_bits(struct posix_ace_state *astate, u32 mask) +{ + /* Allow all bits in the mask not already denied: */ + astate->allow |= mask & ~astate->deny; +} + +static inline void deny_bits(struct posix_ace_state *astate, u32 mask) +{ + /* Deny all bits in the mask not already allowed: */ + astate->deny |= mask & ~astate->allow; +} + +static int find_uid(struct posix_acl_state *state, kuid_t uid) +{ + struct posix_ace_state_array *a = state->users; + int i; + + for (i = 0; i < a->n; i++) + if (uid_eq(a->aces[i].uid, uid)) + return i; + /* Not found: */ + a->n++; + a->aces[i].uid = uid; + a->aces[i].perms.allow = state->everyone.allow; + a->aces[i].perms.deny = state->everyone.deny; + + return i; +} + +static int find_gid(struct posix_acl_state *state, kgid_t gid) +{ + struct posix_ace_state_array *a = state->groups; + int i; + + for (i = 0; i < a->n; i++) + if (gid_eq(a->aces[i].gid, gid)) + return i; + /* Not found: */ + a->n++; + a->aces[i].gid = gid; + a->aces[i].perms.allow = state->everyone.allow; + a->aces[i].perms.deny = state->everyone.deny; + + return i; +} + +static void deny_bits_array(struct posix_ace_state_array *a, u32 mask) +{ + int i; + + for (i=0; i < a->n; i++) + deny_bits(&a->aces[i].perms, mask); +} + +static void allow_bits_array(struct posix_ace_state_array *a, u32 mask) +{ + int i; + + for (i=0; i < a->n; i++) + allow_bits(&a->aces[i].perms, mask); +} + +static void process_one_v4_ace(struct posix_acl_state *state, + struct nfs4_ace *ace) +{ + u32 mask = ace->access_mask; + short type = ace2type(ace); + int i; + + state->valid |= type; + + switch (type) { + case ACL_USER_OBJ: + if (ace->type == NFS4_ACE_ACCESS_ALLOWED_ACE_TYPE) { + allow_bits(&state->owner, mask); + } else { + deny_bits(&state->owner, mask); + } + break; + case ACL_USER: + i = find_uid(state, ace->who_uid); + if (ace->type == NFS4_ACE_ACCESS_ALLOWED_ACE_TYPE) { + allow_bits(&state->users->aces[i].perms, mask); + } else { + deny_bits(&state->users->aces[i].perms, mask); + mask = state->users->aces[i].perms.deny; + deny_bits(&state->owner, mask); + } + break; + case ACL_GROUP_OBJ: + if (ace->type == NFS4_ACE_ACCESS_ALLOWED_ACE_TYPE) { + allow_bits(&state->group, mask); + } else { + deny_bits(&state->group, mask); + mask = state->group.deny; + deny_bits(&state->owner, mask); + deny_bits(&state->everyone, mask); + deny_bits_array(state->users, mask); + deny_bits_array(state->groups, mask); + } + break; + case ACL_GROUP: + i = find_gid(state, ace->who_gid); + if (ace->type == NFS4_ACE_ACCESS_ALLOWED_ACE_TYPE) { + allow_bits(&state->groups->aces[i].perms, mask); + } else { + deny_bits(&state->groups->aces[i].perms, mask); + mask = state->groups->aces[i].perms.deny; + deny_bits(&state->owner, mask); + deny_bits(&state->group, mask); + deny_bits(&state->everyone, mask); + deny_bits_array(state->users, mask); + deny_bits_array(state->groups, mask); + } + break; + case ACL_OTHER: + if (ace->type == NFS4_ACE_ACCESS_ALLOWED_ACE_TYPE) { + allow_bits(&state->owner, mask); + allow_bits(&state->group, mask); + allow_bits(&state->other, mask); + allow_bits(&state->everyone, mask); + allow_bits_array(state->users, mask); + allow_bits_array(state->groups, mask); + } else { + deny_bits(&state->owner, mask); + deny_bits(&state->group, mask); + deny_bits(&state->other, mask); + deny_bits(&state->everyone, mask); + deny_bits_array(state->users, mask); + deny_bits_array(state->groups, mask); + } + } +} + +static int nfs4_acl_nfsv4_to_posix(struct nfs4_acl *acl, + struct posix_acl **pacl, struct posix_acl **dpacl, + unsigned int flags) +{ + struct posix_acl_state effective_acl_state, default_acl_state; + struct nfs4_ace *ace; + int ret; + + ret = init_state(&effective_acl_state, acl->naces); + if (ret) + return ret; + ret = init_state(&default_acl_state, acl->naces); + if (ret) + goto out_estate; + ret = -EINVAL; + for (ace = acl->aces; ace < acl->aces + acl->naces; ace++) { + if (ace->type != NFS4_ACE_ACCESS_ALLOWED_ACE_TYPE && + ace->type != NFS4_ACE_ACCESS_DENIED_ACE_TYPE) + goto out_dstate; + if (ace->flag & ~NFS4_SUPPORTED_FLAGS) + goto out_dstate; + if ((ace->flag & NFS4_INHERITANCE_FLAGS) == 0) { + process_one_v4_ace(&effective_acl_state, ace); + continue; + } + if (!(flags & NFS4_ACL_DIR)) + goto out_dstate; + /* + * Note that when only one of FILE_INHERIT or DIRECTORY_INHERIT + * is set, we're effectively turning on the other. That's OK, + * according to rfc 3530. + */ + process_one_v4_ace(&default_acl_state, ace); + + if (!(ace->flag & NFS4_ACE_INHERIT_ONLY_ACE)) + process_one_v4_ace(&effective_acl_state, ace); + } + + /* + * At this point, the default ACL may have zeroed-out entries for owner, + * group and other. That usually results in a non-sensical resulting ACL + * that denies all access except to any ACE that was explicitly added. + * + * The setfacl command solves a similar problem with this logic: + * + * "If a Default ACL entry is created, and the Default ACL contains + * no owner, owning group, or others entry, a copy of the ACL + * owner, owning group, or others entry is added to the Default ACL." + * + * Copy any missing ACEs from the effective set, if any ACEs were + * explicitly set. + */ + if (default_acl_state.valid) { + if (!(default_acl_state.valid & ACL_USER_OBJ)) + default_acl_state.owner = effective_acl_state.owner; + if (!(default_acl_state.valid & ACL_GROUP_OBJ)) + default_acl_state.group = effective_acl_state.group; + if (!(default_acl_state.valid & ACL_OTHER)) + default_acl_state.other = effective_acl_state.other; + } + + *pacl = posix_state_to_acl(&effective_acl_state, flags); + if (IS_ERR(*pacl)) { + ret = PTR_ERR(*pacl); + *pacl = NULL; + goto out_dstate; + } + *dpacl = posix_state_to_acl(&default_acl_state, + flags | NFS4_ACL_TYPE_DEFAULT); + if (IS_ERR(*dpacl)) { + ret = PTR_ERR(*dpacl); + *dpacl = NULL; + posix_acl_release(*pacl); + *pacl = NULL; + goto out_dstate; + } + sort_pacl(*pacl); + sort_pacl(*dpacl); + ret = 0; +out_dstate: + free_state(&default_acl_state); +out_estate: + free_state(&effective_acl_state); + return ret; +} + +__be32 nfsd4_acl_to_attr(enum nfs_ftype4 type, struct nfs4_acl *acl, + struct nfsd_attrs *attr) +{ + int host_error; + unsigned int flags = 0; + + if (!acl) + return nfs_ok; + + if (type == NF4DIR) + flags = NFS4_ACL_DIR; + + host_error = nfs4_acl_nfsv4_to_posix(acl, &attr->na_pacl, + &attr->na_dpacl, flags); + if (host_error == -EINVAL) + return nfserr_attrnotsupp; + else + return nfserrno(host_error); +} + +static short +ace2type(struct nfs4_ace *ace) +{ + switch (ace->whotype) { + case NFS4_ACL_WHO_NAMED: + return (ace->flag & NFS4_ACE_IDENTIFIER_GROUP ? + ACL_GROUP : ACL_USER); + case NFS4_ACL_WHO_OWNER: + return ACL_USER_OBJ; + case NFS4_ACL_WHO_GROUP: + return ACL_GROUP_OBJ; + case NFS4_ACL_WHO_EVERYONE: + return ACL_OTHER; + } + BUG(); + return -1; +} + +/* + * return the size of the struct nfs4_acl required to represent an acl + * with @entries entries. + */ +int nfs4_acl_bytes(int entries) +{ + return sizeof(struct nfs4_acl) + entries * sizeof(struct nfs4_ace); +} + +static struct { + char *string; + int stringlen; + int type; +} s2t_map[] = { + { + .string = "OWNER@", + .stringlen = sizeof("OWNER@") - 1, + .type = NFS4_ACL_WHO_OWNER, + }, + { + .string = "GROUP@", + .stringlen = sizeof("GROUP@") - 1, + .type = NFS4_ACL_WHO_GROUP, + }, + { + .string = "EVERYONE@", + .stringlen = sizeof("EVERYONE@") - 1, + .type = NFS4_ACL_WHO_EVERYONE, + }, +}; + +int +nfs4_acl_get_whotype(char *p, u32 len) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(s2t_map); i++) { + if (s2t_map[i].stringlen == len && + 0 == memcmp(s2t_map[i].string, p, len)) + return s2t_map[i].type; + } + return NFS4_ACL_WHO_NAMED; +} + +__be32 nfs4_acl_write_who(struct xdr_stream *xdr, int who) +{ + __be32 *p; + int i; + + for (i = 0; i < ARRAY_SIZE(s2t_map); i++) { + if (s2t_map[i].type != who) + continue; + p = xdr_reserve_space(xdr, s2t_map[i].stringlen + 4); + if (!p) + return nfserr_resource; + p = xdr_encode_opaque(p, s2t_map[i].string, + s2t_map[i].stringlen); + return 0; + } + WARN_ON_ONCE(1); + return nfserr_serverfault; +} diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c new file mode 100644 index 000000000..a901bbe67 --- /dev/null +++ b/fs/nfsd/nfs4callback.c @@ -0,0 +1,2018 @@ +/* + * Copyright (c) 2001 The Regents of the University of Michigan. + * All rights reserved. + * + * Kendrick Smith + * Andy Adamson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include "nfsd.h" +#include "state.h" +#include "netns.h" +#include "stats.h" +#include "trace.h" +#include "xdr4cb.h" +#include "xdr4.h" +#include "nfs4xdr_gen.h" + +#define NFSDDBG_FACILITY NFSDDBG_PROC + +#define NFSPROC4_CB_NULL 0 +#define NFSPROC4_CB_COMPOUND 1 + +/* Index of predefined Linux callback client operations */ + +struct nfs4_cb_compound_hdr { + /* args */ + u32 ident; /* minorversion 0 only */ + u32 nops; + __be32 *nops_p; + u32 minorversion; + /* res */ + int status; +}; + +static __be32 *xdr_encode_empty_array(__be32 *p) +{ + *p++ = xdr_zero; + return p; +} + +/* + * Encode/decode NFSv4 CB basic data types + * + * Basic NFSv4 callback data types are defined in section 15 of RFC + * 3530: "Network File System (NFS) version 4 Protocol" and section + * 20 of RFC 5661: "Network File System (NFS) Version 4 Minor Version + * 1 Protocol" + */ + +static void encode_uint32(struct xdr_stream *xdr, u32 n) +{ + WARN_ON_ONCE(xdr_stream_encode_u32(xdr, n) < 0); +} + +static void encode_bitmap4(struct xdr_stream *xdr, const __u32 *bitmap, + size_t len) +{ + xdr_stream_encode_uint32_array(xdr, bitmap, len); +} + +static int decode_cb_fattr4(struct xdr_stream *xdr, uint32_t *bitmap, + struct nfs4_cb_fattr *fattr) +{ + fattr->ncf_cb_change = 0; + fattr->ncf_cb_fsize = 0; + fattr->ncf_cb_atime.tv_sec = 0; + fattr->ncf_cb_atime.tv_nsec = 0; + fattr->ncf_cb_mtime.tv_sec = 0; + fattr->ncf_cb_mtime.tv_nsec = 0; + + if (bitmap[0] & FATTR4_WORD0_CHANGE) + if (xdr_stream_decode_u64(xdr, &fattr->ncf_cb_change) < 0) + return -EIO; + if (bitmap[0] & FATTR4_WORD0_SIZE) + if (xdr_stream_decode_u64(xdr, &fattr->ncf_cb_fsize) < 0) + return -EIO; + if (bitmap[2] & FATTR4_WORD2_TIME_DELEG_ACCESS) { + fattr4_time_deleg_access access; + + if (!xdrgen_decode_fattr4_time_deleg_access(xdr, &access)) + return -EIO; + if (access.nseconds >= NSEC_PER_SEC) + return -EIO; + fattr->ncf_cb_atime.tv_sec = access.seconds; + fattr->ncf_cb_atime.tv_nsec = access.nseconds; + + } + if (bitmap[2] & FATTR4_WORD2_TIME_DELEG_MODIFY) { + fattr4_time_deleg_modify modify; + + if (!xdrgen_decode_fattr4_time_deleg_modify(xdr, &modify)) + return -EIO; + if (modify.nseconds >= NSEC_PER_SEC) + return -EIO; + fattr->ncf_cb_mtime.tv_sec = modify.seconds; + fattr->ncf_cb_mtime.tv_nsec = modify.nseconds; + + } + return 0; +} + +static void encode_nfs_cb_opnum4(struct xdr_stream *xdr, enum nfs_cb_opnum4 op) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, 4); + *p = cpu_to_be32(op); +} + +/* + * nfs_fh4 + * + * typedef opaque nfs_fh4; + */ +static void encode_nfs_fh4(struct xdr_stream *xdr, const struct knfsd_fh *fh) +{ + u32 length = fh->fh_size; + __be32 *p; + + BUG_ON(length > NFS4_FHSIZE); + p = xdr_reserve_space(xdr, 4 + length); + xdr_encode_opaque(p, &fh->fh_raw, length); +} + +/* + * stateid4 + * + * struct stateid4 { + * uint32_t seqid; + * opaque other[12]; + * }; + */ +static void encode_stateid4(struct xdr_stream *xdr, const stateid_t *sid) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, NFS4_STATEID_SIZE); + *p++ = cpu_to_be32(sid->si_generation); + xdr_encode_opaque_fixed(p, &sid->si_opaque, NFS4_STATEID_OTHER_SIZE); +} + +/* + * sessionid4 + * + * typedef opaque sessionid4[NFS4_SESSIONID_SIZE]; + */ +static void encode_sessionid4(struct xdr_stream *xdr, + const struct nfsd4_session *session) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, NFS4_MAX_SESSIONID_LEN); + xdr_encode_opaque_fixed(p, session->se_sessionid.data, + NFS4_MAX_SESSIONID_LEN); +} + +/* + * nfsstat4 + */ +static const struct { + int stat; + int errno; +} nfs_cb_errtbl[] = { + { NFS4_OK, 0 }, + { NFS4ERR_PERM, -EPERM }, + { NFS4ERR_NOENT, -ENOENT }, + { NFS4ERR_IO, -EIO }, + { NFS4ERR_NXIO, -ENXIO }, + { NFS4ERR_ACCESS, -EACCES }, + { NFS4ERR_EXIST, -EEXIST }, + { NFS4ERR_XDEV, -EXDEV }, + { NFS4ERR_NOTDIR, -ENOTDIR }, + { NFS4ERR_ISDIR, -EISDIR }, + { NFS4ERR_INVAL, -EINVAL }, + { NFS4ERR_FBIG, -EFBIG }, + { NFS4ERR_NOSPC, -ENOSPC }, + { NFS4ERR_ROFS, -EROFS }, + { NFS4ERR_MLINK, -EMLINK }, + { NFS4ERR_NAMETOOLONG, -ENAMETOOLONG }, + { NFS4ERR_NOTEMPTY, -ENOTEMPTY }, + { NFS4ERR_DQUOT, -EDQUOT }, + { NFS4ERR_STALE, -ESTALE }, + { NFS4ERR_BADHANDLE, -EBADHANDLE }, + { NFS4ERR_BAD_COOKIE, -EBADCOOKIE }, + { NFS4ERR_NOTSUPP, -ENOTSUPP }, + { NFS4ERR_TOOSMALL, -ETOOSMALL }, + { NFS4ERR_SERVERFAULT, -ESERVERFAULT }, + { NFS4ERR_BADTYPE, -EBADTYPE }, + { NFS4ERR_LOCKED, -EAGAIN }, + { NFS4ERR_RESOURCE, -EREMOTEIO }, + { NFS4ERR_SYMLINK, -ELOOP }, + { NFS4ERR_OP_ILLEGAL, -EOPNOTSUPP }, + { NFS4ERR_DEADLOCK, -EDEADLK }, + { -1, -EIO } +}; + +/* + * If we cannot translate the error, the recovery routines should + * handle it. + * + * Note: remaining NFSv4 error codes have values > 10000, so should + * not conflict with native Linux error codes. + */ +static int nfs_cb_stat_to_errno(int status) +{ + int i; + + for (i = 0; nfs_cb_errtbl[i].stat != -1; i++) { + if (nfs_cb_errtbl[i].stat == status) + return nfs_cb_errtbl[i].errno; + } + + dprintk("NFSD: Unrecognized NFS CB status value: %u\n", status); + return -status; +} + +static int decode_cb_op_status(struct xdr_stream *xdr, + enum nfs_cb_opnum4 expected, int *status) +{ + __be32 *p; + u32 op; + + p = xdr_inline_decode(xdr, 4 + 4); + if (unlikely(p == NULL)) + goto out_overflow; + op = be32_to_cpup(p++); + if (unlikely(op != expected)) + goto out_unexpected; + *status = nfs_cb_stat_to_errno(be32_to_cpup(p)); + return 0; +out_overflow: + return -EIO; +out_unexpected: + dprintk("NFSD: Callback server returned operation %d but " + "we issued a request for %d\n", op, expected); + return -EIO; +} + +/* + * CB_COMPOUND4args + * + * struct CB_COMPOUND4args { + * utf8str_cs tag; + * uint32_t minorversion; + * uint32_t callback_ident; + * nfs_cb_argop4 argarray<>; + * }; +*/ +static void encode_cb_compound4args(struct xdr_stream *xdr, + struct nfs4_cb_compound_hdr *hdr) +{ + __be32 * p; + + p = xdr_reserve_space(xdr, 4 + 4 + 4 + 4); + p = xdr_encode_empty_array(p); /* empty tag */ + *p++ = cpu_to_be32(hdr->minorversion); + *p++ = cpu_to_be32(hdr->ident); + + hdr->nops_p = p; + *p = cpu_to_be32(hdr->nops); /* argarray element count */ +} + +/* + * Update argarray element count + */ +static void encode_cb_nops(struct nfs4_cb_compound_hdr *hdr) +{ + BUG_ON(hdr->nops > NFS4_MAX_BACK_CHANNEL_OPS); + *hdr->nops_p = cpu_to_be32(hdr->nops); +} + +/* + * CB_COMPOUND4res + * + * struct CB_COMPOUND4res { + * nfsstat4 status; + * utf8str_cs tag; + * nfs_cb_resop4 resarray<>; + * }; + */ +static int decode_cb_compound4res(struct xdr_stream *xdr, + struct nfs4_cb_compound_hdr *hdr) +{ + u32 length; + __be32 *p; + + p = xdr_inline_decode(xdr, XDR_UNIT); + if (unlikely(p == NULL)) + goto out_overflow; + hdr->status = be32_to_cpup(p); + /* Ignore the tag */ + if (xdr_stream_decode_u32(xdr, &length) < 0) + goto out_overflow; + if (xdr_inline_decode(xdr, length) == NULL) + goto out_overflow; + if (xdr_stream_decode_u32(xdr, &hdr->nops) < 0) + goto out_overflow; + return 0; +out_overflow: + return -EIO; +} + +/* + * CB_RECALL4args + * + * struct CB_RECALL4args { + * stateid4 stateid; + * bool truncate; + * nfs_fh4 fh; + * }; + */ +static void encode_cb_recall4args(struct xdr_stream *xdr, + const struct nfs4_delegation *dp, + struct nfs4_cb_compound_hdr *hdr) +{ + __be32 *p; + + encode_nfs_cb_opnum4(xdr, OP_CB_RECALL); + encode_stateid4(xdr, &dp->dl_stid.sc_stateid); + + p = xdr_reserve_space(xdr, 4); + *p++ = xdr_zero; /* truncate */ + + encode_nfs_fh4(xdr, &dp->dl_stid.sc_file->fi_fhandle); + + hdr->nops++; +} + +/* + * CB_RECALLANY4args + * + * struct CB_RECALLANY4args { + * uint32_t craa_objects_to_keep; + * bitmap4 craa_type_mask; + * }; + */ +static void +encode_cb_recallany4args(struct xdr_stream *xdr, + struct nfs4_cb_compound_hdr *hdr, struct nfsd4_cb_recall_any *ra) +{ + encode_nfs_cb_opnum4(xdr, OP_CB_RECALL_ANY); + encode_uint32(xdr, ra->ra_keep); + encode_bitmap4(xdr, ra->ra_bmval, ARRAY_SIZE(ra->ra_bmval)); + hdr->nops++; +} + +/* + * CB_GETATTR4args + * struct CB_GETATTR4args { + * nfs_fh4 fh; + * bitmap4 attr_request; + * }; + * + * The size and change attributes are the only one + * guaranteed to be serviced by the client. + */ +static void +encode_cb_getattr4args(struct xdr_stream *xdr, struct nfs4_cb_compound_hdr *hdr, + struct nfs4_cb_fattr *fattr) +{ + struct nfs4_delegation *dp = container_of(fattr, struct nfs4_delegation, dl_cb_fattr); + struct knfsd_fh *fh = &dp->dl_stid.sc_file->fi_fhandle; + struct nfs4_cb_fattr *ncf = &dp->dl_cb_fattr; + u32 bmap_size = 1; + u32 bmap[3]; + + bmap[0] = FATTR4_WORD0_SIZE; + if (!ncf->ncf_file_modified) + bmap[0] |= FATTR4_WORD0_CHANGE; + + if (deleg_attrs_deleg(dp->dl_type)) { + bmap[1] = 0; + bmap[2] = FATTR4_WORD2_TIME_DELEG_ACCESS | FATTR4_WORD2_TIME_DELEG_MODIFY; + bmap_size = 3; + } + encode_nfs_cb_opnum4(xdr, OP_CB_GETATTR); + encode_nfs_fh4(xdr, fh); + encode_bitmap4(xdr, bmap, bmap_size); + hdr->nops++; +} + +static u32 highest_slotid(struct nfsd4_session *ses) +{ + u32 idx; + + spin_lock(&ses->se_lock); + idx = fls(~ses->se_cb_slot_avail); + if (idx > 0) + --idx; + idx = max(idx, ses->se_cb_highest_slot); + spin_unlock(&ses->se_lock); + return idx; +} + +static void +encode_referring_call4(struct xdr_stream *xdr, + const struct nfsd4_referring_call *rc) +{ + encode_uint32(xdr, rc->rc_sequenceid); + encode_uint32(xdr, rc->rc_slotid); +} + +static void +encode_referring_call_list4(struct xdr_stream *xdr, + const struct nfsd4_referring_call_list *rcl) +{ + struct nfsd4_referring_call *rc; + __be32 *p; + + p = xdr_reserve_space(xdr, NFS4_MAX_SESSIONID_LEN); + xdr_encode_opaque_fixed(p, rcl->rcl_sessionid.data, + NFS4_MAX_SESSIONID_LEN); + encode_uint32(xdr, rcl->__nr_referring_calls); + list_for_each_entry(rc, &rcl->rcl_referring_calls, __list) + encode_referring_call4(xdr, rc); +} + +/* + * CB_SEQUENCE4args + * + * struct CB_SEQUENCE4args { + * sessionid4 csa_sessionid; + * sequenceid4 csa_sequenceid; + * slotid4 csa_slotid; + * slotid4 csa_highest_slotid; + * bool csa_cachethis; + * referring_call_list4 csa_referring_call_lists<>; + * }; + */ +static void encode_cb_sequence4args(struct xdr_stream *xdr, + const struct nfsd4_callback *cb, + struct nfs4_cb_compound_hdr *hdr) +{ + struct nfsd4_session *session; + struct nfsd4_referring_call_list *rcl; + __be32 *p; + + if (hdr->minorversion == 0) + return; + + rcu_read_lock(); + session = rcu_dereference(cb->cb_clp->cl_cb_session); + if (!session) { + rcu_read_unlock(); + return; + } + + encode_nfs_cb_opnum4(xdr, OP_CB_SEQUENCE); + encode_sessionid4(xdr, session); + + p = xdr_reserve_space(xdr, XDR_UNIT * 4); + *p++ = cpu_to_be32(session->se_cb_seq_nr[cb->cb_held_slot]); /* csa_sequenceid */ + *p++ = cpu_to_be32(cb->cb_held_slot); /* csa_slotid */ + *p++ = cpu_to_be32(highest_slotid(session)); /* csa_highest_slotid */ + *p++ = xdr_zero; /* csa_cachethis */ + + /* csa_referring_call_lists */ + encode_uint32(xdr, cb->cb_nr_referring_call_list); + list_for_each_entry(rcl, &cb->cb_referring_call_list, __list) + encode_referring_call_list4(xdr, rcl); + + hdr->nops++; + rcu_read_unlock(); +} + +static void update_cb_slot_table(struct nfsd4_session *ses, u32 target) +{ + /* No need to do anything if nothing changed */ + if (likely(target == READ_ONCE(ses->se_cb_highest_slot))) + return; + + spin_lock(&ses->se_lock); + if (target > ses->se_cb_highest_slot) { + int i; + + target = min(target, NFSD_BC_SLOT_TABLE_SIZE - 1); + + /* + * Growing the slot table. Reset any new sequences to 1. + * + * NB: There is some debate about whether the RFC requires this, + * but the Linux client expects it. + */ + for (i = ses->se_cb_highest_slot + 1; i <= target; ++i) + ses->se_cb_seq_nr[i] = 1; + } + ses->se_cb_highest_slot = target; + spin_unlock(&ses->se_lock); +} + +/* + * CB_SEQUENCE4resok + * + * struct CB_SEQUENCE4resok { + * sessionid4 csr_sessionid; + * sequenceid4 csr_sequenceid; + * slotid4 csr_slotid; + * slotid4 csr_highest_slotid; + * slotid4 csr_target_highest_slotid; + * }; + * + * union CB_SEQUENCE4res switch (nfsstat4 csr_status) { + * case NFS4_OK: + * CB_SEQUENCE4resok csr_resok4; + * default: + * void; + * }; + * + * Our current back channel implmentation supports a single backchannel + * with a single slot. + */ +static int decode_cb_sequence4resok(struct xdr_stream *xdr, + struct nfsd4_callback *cb) +{ + struct nfsd4_session *session; + int status = -ESERVERFAULT; + __be32 *p; + u32 seqid, slotid, target; + + rcu_read_lock(); + session = rcu_dereference(cb->cb_clp->cl_cb_session); + if (!session) { + rcu_read_unlock(); + cb->cb_seq_status = -NFS4ERR_BADSESSION; + return -NFS4ERR_BADSESSION; + } + + /* + * If the server returns different values for sessionID, slotID or + * sequence number, the server is looney tunes. + */ + p = xdr_inline_decode(xdr, NFS4_MAX_SESSIONID_LEN + 4 + 4 + 4 + 4); + if (unlikely(p == NULL)) { + rcu_read_unlock(); + goto out_overflow; + } + + if (memcmp(p, session->se_sessionid.data, NFS4_MAX_SESSIONID_LEN)) { + dprintk("NFS: %s Invalid session id\n", __func__); + rcu_read_unlock(); + goto out; + } + p += XDR_QUADLEN(NFS4_MAX_SESSIONID_LEN); + + seqid = be32_to_cpup(p++); + if (seqid != session->se_cb_seq_nr[cb->cb_held_slot]) { + dprintk("NFS: %s Invalid sequence number\n", __func__); + rcu_read_unlock(); + goto out; + } + + slotid = be32_to_cpup(p++); + if (slotid != cb->cb_held_slot) { + dprintk("NFS: %s Invalid slotid\n", __func__); + rcu_read_unlock(); + goto out; + } + + p++; // ignore current highest slot value + + target = be32_to_cpup(p++); + update_cb_slot_table(session, target); + rcu_read_unlock(); + status = 0; +out: + cb->cb_seq_status = status; + return status; +out_overflow: + status = -EIO; + goto out; +} + +static int decode_cb_sequence4res(struct xdr_stream *xdr, + struct nfsd4_callback *cb) +{ + int status; + + if (cb->cb_clp->cl_minorversion == 0) + return 0; + + status = decode_cb_op_status(xdr, OP_CB_SEQUENCE, &cb->cb_seq_status); + if (unlikely(status || cb->cb_seq_status)) + return status; + + return decode_cb_sequence4resok(xdr, cb); +} + +/* + * NFSv4.0 and NFSv4.1 XDR encode functions + * + * NFSv4.0 callback argument types are defined in section 15 of RFC + * 3530: "Network File System (NFS) version 4 Protocol" and section 20 + * of RFC 5661: "Network File System (NFS) Version 4 Minor Version 1 + * Protocol". + */ + +/* + * NB: Without this zero space reservation, callbacks over krb5p fail + */ +static void nfs4_xdr_enc_cb_null(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *__unused) +{ + xdr_reserve_space(xdr, 0); +} + +/* + * 20.1. Operation 3: CB_GETATTR - Get Attributes + */ +static void nfs4_xdr_enc_cb_getattr(struct rpc_rqst *req, + struct xdr_stream *xdr, const void *data) +{ + const struct nfsd4_callback *cb = data; + struct nfs4_cb_fattr *ncf = + container_of(cb, struct nfs4_cb_fattr, ncf_getattr); + struct nfs4_cb_compound_hdr hdr = { + .ident = cb->cb_clp->cl_cb_ident, + .minorversion = cb->cb_clp->cl_minorversion, + }; + + encode_cb_compound4args(xdr, &hdr); + encode_cb_sequence4args(xdr, cb, &hdr); + encode_cb_getattr4args(xdr, &hdr, ncf); + encode_cb_nops(&hdr); +} + +/* + * 20.2. Operation 4: CB_RECALL - Recall a Delegation + */ +static void nfs4_xdr_enc_cb_recall(struct rpc_rqst *req, struct xdr_stream *xdr, + const void *data) +{ + const struct nfsd4_callback *cb = data; + const struct nfs4_delegation *dp = cb_to_delegation(cb); + struct nfs4_cb_compound_hdr hdr = { + .ident = cb->cb_clp->cl_cb_ident, + .minorversion = cb->cb_clp->cl_minorversion, + }; + + encode_cb_compound4args(xdr, &hdr); + encode_cb_sequence4args(xdr, cb, &hdr); + encode_cb_recall4args(xdr, dp, &hdr); + encode_cb_nops(&hdr); +} + +/* + * 20.6. Operation 8: CB_RECALL_ANY - Keep Any N Recallable Objects + */ +static void +nfs4_xdr_enc_cb_recall_any(struct rpc_rqst *req, + struct xdr_stream *xdr, const void *data) +{ + const struct nfsd4_callback *cb = data; + struct nfsd4_cb_recall_any *ra; + struct nfs4_cb_compound_hdr hdr = { + .ident = cb->cb_clp->cl_cb_ident, + .minorversion = cb->cb_clp->cl_minorversion, + }; + + ra = container_of(cb, struct nfsd4_cb_recall_any, ra_cb); + encode_cb_compound4args(xdr, &hdr); + encode_cb_sequence4args(xdr, cb, &hdr); + encode_cb_recallany4args(xdr, &hdr, ra); + encode_cb_nops(&hdr); +} + +/* + * NFSv4.0 and NFSv4.1 XDR decode functions + * + * NFSv4.0 callback result types are defined in section 15 of RFC + * 3530: "Network File System (NFS) version 4 Protocol" and section 20 + * of RFC 5661: "Network File System (NFS) Version 4 Minor Version 1 + * Protocol". + */ + +static int nfs4_xdr_dec_cb_null(struct rpc_rqst *req, struct xdr_stream *xdr, + void *__unused) +{ + return 0; +} + +/* + * 20.1. Operation 3: CB_GETATTR - Get Attributes + */ +static int nfs4_xdr_dec_cb_getattr(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfsd4_callback *cb = data; + struct nfs4_cb_compound_hdr hdr; + int status; + u32 bitmap[3] = {0}; + u32 attrlen, maxlen; + struct nfs4_cb_fattr *ncf = + container_of(cb, struct nfs4_cb_fattr, ncf_getattr); + + status = decode_cb_compound4res(xdr, &hdr); + if (unlikely(status)) + return status; + + status = decode_cb_sequence4res(xdr, cb); + if (unlikely(status || cb->cb_seq_status)) + return status; + + status = decode_cb_op_status(xdr, OP_CB_GETATTR, &cb->cb_status); + if (unlikely(status || cb->cb_status)) + return status; + if (xdr_stream_decode_uint32_array(xdr, bitmap, 3) < 0) + return -EIO; + if (xdr_stream_decode_u32(xdr, &attrlen) < 0) + return -EIO; + maxlen = sizeof(ncf->ncf_cb_change) + sizeof(ncf->ncf_cb_fsize); + if (bitmap[2] != 0) + maxlen += (sizeof(ncf->ncf_cb_mtime.tv_sec) + + sizeof(ncf->ncf_cb_mtime.tv_nsec)) * 2; + if (attrlen > maxlen) + return -EIO; + status = decode_cb_fattr4(xdr, bitmap, ncf); + return status; +} + +/* + * 20.2. Operation 4: CB_RECALL - Recall a Delegation + */ +static int nfs4_xdr_dec_cb_recall(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfsd4_callback *cb = data; + struct nfs4_cb_compound_hdr hdr; + int status; + + status = decode_cb_compound4res(xdr, &hdr); + if (unlikely(status)) + return status; + + status = decode_cb_sequence4res(xdr, cb); + if (unlikely(status || cb->cb_seq_status)) + return status; + + return decode_cb_op_status(xdr, OP_CB_RECALL, &cb->cb_status); +} + +/* + * 20.6. Operation 8: CB_RECALL_ANY - Keep Any N Recallable Objects + */ +static int +nfs4_xdr_dec_cb_recall_any(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfsd4_callback *cb = data; + struct nfs4_cb_compound_hdr hdr; + int status; + + status = decode_cb_compound4res(xdr, &hdr); + if (unlikely(status)) + return status; + status = decode_cb_sequence4res(xdr, cb); + if (unlikely(status || cb->cb_seq_status)) + return status; + status = decode_cb_op_status(xdr, OP_CB_RECALL_ANY, &cb->cb_status); + return status; +} + +#ifdef CONFIG_NFSD_PNFS +/* + * CB_LAYOUTRECALL4args + * + * struct layoutrecall_file4 { + * nfs_fh4 lor_fh; + * offset4 lor_offset; + * length4 lor_length; + * stateid4 lor_stateid; + * }; + * + * union layoutrecall4 switch(layoutrecall_type4 lor_recalltype) { + * case LAYOUTRECALL4_FILE: + * layoutrecall_file4 lor_layout; + * case LAYOUTRECALL4_FSID: + * fsid4 lor_fsid; + * case LAYOUTRECALL4_ALL: + * void; + * }; + * + * struct CB_LAYOUTRECALL4args { + * layouttype4 clora_type; + * layoutiomode4 clora_iomode; + * bool clora_changed; + * layoutrecall4 clora_recall; + * }; + */ +static void encode_cb_layout4args(struct xdr_stream *xdr, + const struct nfs4_layout_stateid *ls, + struct nfs4_cb_compound_hdr *hdr) +{ + __be32 *p; + + BUG_ON(hdr->minorversion == 0); + + p = xdr_reserve_space(xdr, 5 * 4); + *p++ = cpu_to_be32(OP_CB_LAYOUTRECALL); + *p++ = cpu_to_be32(ls->ls_layout_type); + *p++ = cpu_to_be32(IOMODE_ANY); + *p++ = cpu_to_be32(1); + *p = cpu_to_be32(RETURN_FILE); + + encode_nfs_fh4(xdr, &ls->ls_stid.sc_file->fi_fhandle); + + p = xdr_reserve_space(xdr, 2 * 8); + p = xdr_encode_hyper(p, 0); + xdr_encode_hyper(p, NFS4_MAX_UINT64); + + encode_stateid4(xdr, &ls->ls_recall_sid); + + hdr->nops++; +} + +static void nfs4_xdr_enc_cb_layout(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfsd4_callback *cb = data; + const struct nfs4_layout_stateid *ls = + container_of(cb, struct nfs4_layout_stateid, ls_recall); + struct nfs4_cb_compound_hdr hdr = { + .ident = 0, + .minorversion = cb->cb_clp->cl_minorversion, + }; + + encode_cb_compound4args(xdr, &hdr); + encode_cb_sequence4args(xdr, cb, &hdr); + encode_cb_layout4args(xdr, ls, &hdr); + encode_cb_nops(&hdr); +} + +static int nfs4_xdr_dec_cb_layout(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfsd4_callback *cb = data; + struct nfs4_cb_compound_hdr hdr; + int status; + + status = decode_cb_compound4res(xdr, &hdr); + if (unlikely(status)) + return status; + + status = decode_cb_sequence4res(xdr, cb); + if (unlikely(status || cb->cb_seq_status)) + return status; + + return decode_cb_op_status(xdr, OP_CB_LAYOUTRECALL, &cb->cb_status); +} +#endif /* CONFIG_NFSD_PNFS */ + +static void encode_stateowner(struct xdr_stream *xdr, struct nfs4_stateowner *so) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, 8 + 4 + so->so_owner.len); + p = xdr_encode_opaque_fixed(p, &so->so_client->cl_clientid, 8); + xdr_encode_opaque(p, so->so_owner.data, so->so_owner.len); +} + +static void nfs4_xdr_enc_cb_notify(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfsd4_callback *cb = data; + struct nfsd4_cb_notify *ncn = container_of(cb, struct nfsd4_cb_notify, ncn_cb); + struct nfs4_delegation *dp = container_of(ncn, struct nfs4_delegation, dl_cb_notify); + struct nfs4_cb_compound_hdr hdr = { + .ident = 0, + .minorversion = cb->cb_clp->cl_minorversion, + }; + struct CB_NOTIFY4args args; + unsigned int start; + + WARN_ON_ONCE(hdr.minorversion == 0); + + encode_cb_compound4args(xdr, &hdr); + encode_cb_sequence4args(xdr, cb, &hdr); + + /* + * nfsd4_cb_notify_prepare() sized the payload against a single page, + * but did not account for the compound, sequence, stateid, and + * filehandle encoded here. If the variable-length encode overflows the + * backchannel send buffer, roll back to before the operation so that a + * truncated CB_NOTIFY is never placed on the wire. + */ + start = xdr_stream_pos(xdr); + + if (xdr_stream_encode_u32(xdr, OP_CB_NOTIFY) < 0) + goto out_err; + + args.cna_stateid.seqid = dp->dl_stid.sc_stateid.si_generation; + memcpy(&args.cna_stateid.other, &dp->dl_stid.sc_stateid.si_opaque, + ARRAY_SIZE(args.cna_stateid.other)); + args.cna_fh.len = dp->dl_stid.sc_file->fi_fhandle.fh_size; + args.cna_fh.data = dp->dl_stid.sc_file->fi_fhandle.fh_raw; + args.cna_changes.count = ncn->ncn_nf_cnt; + args.cna_changes.element = ncn->ncn_nf; + if (!xdrgen_encode_CB_NOTIFY4args(xdr, &args)) + goto out_err; + + hdr.nops++; + encode_cb_nops(&hdr); + return; + +out_err: + /* + * Drop the CB_NOTIFY op and emit a valid CB_SEQUENCE-only compound so + * the client still advances its slot. Flag the failure so the done + * handler recalls the delegation and the missed notification is not + * silently lost. The flag is written here in the transmit path and read + * in the done handler; the two are serialized phases of the same + * rpc_task, so no additional barrier is needed. + */ + ncn->ncn_encode_err = true; + xdr_truncate_encode(xdr, start); + encode_cb_nops(&hdr); +} + +static int nfs4_xdr_dec_cb_notify(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfsd4_callback *cb = data; + struct nfs4_cb_compound_hdr hdr; + int status; + + status = decode_cb_compound4res(xdr, &hdr); + if (unlikely(status)) + return status; + + status = decode_cb_sequence4res(xdr, cb); + if (unlikely(status || cb->cb_seq_status)) + return status; + + return decode_cb_op_status(xdr, OP_CB_NOTIFY, &cb->cb_status); +} + +static void nfs4_xdr_enc_cb_notify_lock(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfsd4_callback *cb = data; + const struct nfsd4_blocked_lock *nbl = + container_of(cb, struct nfsd4_blocked_lock, nbl_cb); + struct nfs4_lockowner *lo = (struct nfs4_lockowner *)nbl->nbl_lock.c.flc_owner; + struct nfs4_cb_compound_hdr hdr = { + .ident = 0, + .minorversion = cb->cb_clp->cl_minorversion, + }; + + __be32 *p; + + BUG_ON(hdr.minorversion == 0); + + encode_cb_compound4args(xdr, &hdr); + encode_cb_sequence4args(xdr, cb, &hdr); + + p = xdr_reserve_space(xdr, 4); + *p = cpu_to_be32(OP_CB_NOTIFY_LOCK); + encode_nfs_fh4(xdr, &nbl->nbl_fh); + encode_stateowner(xdr, &lo->lo_owner); + hdr.nops++; + + encode_cb_nops(&hdr); +} + +static int nfs4_xdr_dec_cb_notify_lock(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfsd4_callback *cb = data; + struct nfs4_cb_compound_hdr hdr; + int status; + + status = decode_cb_compound4res(xdr, &hdr); + if (unlikely(status)) + return status; + + status = decode_cb_sequence4res(xdr, cb); + if (unlikely(status || cb->cb_seq_status)) + return status; + + return decode_cb_op_status(xdr, OP_CB_NOTIFY_LOCK, &cb->cb_status); +} + +/* + * struct write_response4 { + * stateid4 wr_callback_id<1>; + * length4 wr_count; + * stable_how4 wr_committed; + * verifier4 wr_writeverf; + * }; + * union offload_info4 switch (nfsstat4 coa_status) { + * case NFS4_OK: + * write_response4 coa_resok4; + * default: + * length4 coa_bytes_copied; + * }; + * struct CB_OFFLOAD4args { + * nfs_fh4 coa_fh; + * stateid4 coa_stateid; + * offload_info4 coa_offload_info; + * }; + */ +static void encode_offload_info4(struct xdr_stream *xdr, + const struct nfsd4_cb_offload *cbo) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, 4); + *p = cbo->co_nfserr; + switch (cbo->co_nfserr) { + case nfs_ok: + p = xdr_reserve_space(xdr, 4 + 8 + 4 + NFS4_VERIFIER_SIZE); + p = xdr_encode_empty_array(p); + p = xdr_encode_hyper(p, cbo->co_res.wr_bytes_written); + *p++ = cpu_to_be32(cbo->co_res.wr_stable_how); + p = xdr_encode_opaque_fixed(p, cbo->co_res.wr_verifier.data, + NFS4_VERIFIER_SIZE); + break; + default: + p = xdr_reserve_space(xdr, 8); + /* We always return success if bytes were written */ + p = xdr_encode_hyper(p, 0); + } +} + +static void encode_cb_offload4args(struct xdr_stream *xdr, + const struct nfsd4_cb_offload *cbo, + struct nfs4_cb_compound_hdr *hdr) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, 4); + *p = cpu_to_be32(OP_CB_OFFLOAD); + encode_nfs_fh4(xdr, &cbo->co_fh); + encode_stateid4(xdr, &cbo->co_res.cb_stateid); + encode_offload_info4(xdr, cbo); + + hdr->nops++; +} + +static void nfs4_xdr_enc_cb_offload(struct rpc_rqst *req, + struct xdr_stream *xdr, + const void *data) +{ + const struct nfsd4_callback *cb = data; + const struct nfsd4_cb_offload *cbo = + container_of(cb, struct nfsd4_cb_offload, co_cb); + struct nfs4_cb_compound_hdr hdr = { + .ident = 0, + .minorversion = cb->cb_clp->cl_minorversion, + }; + + encode_cb_compound4args(xdr, &hdr); + encode_cb_sequence4args(xdr, cb, &hdr); + encode_cb_offload4args(xdr, cbo, &hdr); + encode_cb_nops(&hdr); +} + +static int nfs4_xdr_dec_cb_offload(struct rpc_rqst *rqstp, + struct xdr_stream *xdr, + void *data) +{ + struct nfsd4_callback *cb = data; + struct nfs4_cb_compound_hdr hdr; + int status; + + status = decode_cb_compound4res(xdr, &hdr); + if (unlikely(status)) + return status; + + status = decode_cb_sequence4res(xdr, cb); + if (unlikely(status || cb->cb_seq_status)) + return status; + + return decode_cb_op_status(xdr, OP_CB_OFFLOAD, &cb->cb_status); +} +/* + * RPC procedure tables + */ +#define PROC(proc, call, argtype, restype) \ +[NFSPROC4_CLNT_##proc] = { \ + .p_proc = NFSPROC4_CB_##call, \ + .p_encode = nfs4_xdr_enc_##argtype, \ + .p_decode = nfs4_xdr_dec_##restype, \ + .p_arglen = NFS4_enc_##argtype##_sz, \ + .p_replen = NFS4_dec_##restype##_sz, \ + .p_statidx = NFSPROC4_CLNT_##proc, \ + .p_name = #proc, \ +} + +static const struct rpc_procinfo nfs4_cb_procedures[] = { + PROC(CB_NULL, NULL, cb_null, cb_null), + PROC(CB_RECALL, COMPOUND, cb_recall, cb_recall), +#ifdef CONFIG_NFSD_PNFS + PROC(CB_LAYOUT, COMPOUND, cb_layout, cb_layout), +#endif + PROC(CB_NOTIFY, COMPOUND, cb_notify, cb_notify), + PROC(CB_NOTIFY_LOCK, COMPOUND, cb_notify_lock, cb_notify_lock), + PROC(CB_OFFLOAD, COMPOUND, cb_offload, cb_offload), + PROC(CB_RECALL_ANY, COMPOUND, cb_recall_any, cb_recall_any), + PROC(CB_GETATTR, COMPOUND, cb_getattr, cb_getattr), +}; + +#define NFS4_CB_PROGRAM 0x40000000 +#define NFS4_CB_VERSION 1 + +struct nfsd_net_cb { + struct rpc_version version4; + const struct rpc_version *versions[NFS4_CB_VERSION + 1]; + struct rpc_program program; + struct rpc_stat stat; +}; + +static int max_cb_time(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + /* + * nfsd4_lease is set to at most one hour in __nfsd4_write_time, + * so we can use 32-bit math on it. Warn if that assumption + * ever stops being true. + */ + if (WARN_ON_ONCE(nn->nfsd4_lease > 3600)) + return 360 * HZ; + + return max(((u32)nn->nfsd4_lease)/10, 1u) * HZ; +} + +static bool nfsd4_queue_cb(struct nfsd4_callback *cb) +{ + struct nfs4_client *clp = cb->cb_clp; + + trace_nfsd_cb_queue(clp, cb); + return queue_work(clp->cl_callback_wq, &cb->cb_work); +} + +static void nfsd4_requeue_cb(struct rpc_task *task, struct nfsd4_callback *cb) +{ + struct nfs4_client *clp = cb->cb_clp; + + if (!test_bit(NFSD4_CLIENT_CB_KILL, &clp->cl_flags)) { + trace_nfsd_cb_restart(clp, cb); + task->tk_status = 0; + set_bit(NFSD4_CALLBACK_REQUEUE, &cb->cb_flags); + } +} + +static void nfsd41_cb_inflight_begin(struct nfs4_client *clp) +{ + atomic_inc(&clp->cl_cb_inflight); +} + +static void nfsd41_cb_inflight_end(struct nfs4_client *clp) +{ + + atomic_dec_and_wake_up(&clp->cl_cb_inflight); +} + +static void nfsd41_cb_inflight_wait_complete(struct nfs4_client *clp) +{ + wait_var_event(&clp->cl_cb_inflight, + !atomic_read(&clp->cl_cb_inflight)); +} + +static const struct cred *get_backchannel_cred(struct nfs4_client *clp, struct rpc_clnt *client, struct nfsd4_session *ses) +{ + if (clp->cl_minorversion == 0) { + client->cl_principal = clp->cl_cred.cr_targ_princ ? + clp->cl_cred.cr_targ_princ : "nfs"; + + return get_cred(rpc_machine_cred()); + } else { + struct cred *kcred; + + kcred = prepare_kernel_cred(&init_task); + if (!kcred) + return NULL; + + kcred->fsuid = ses->se_cb_sec.uid; + kcred->fsgid = ses->se_cb_sec.gid; + return kcred; + } +} + +static int setup_callback_client(struct nfs4_client *clp, struct nfs4_cb_conn *conn, struct nfsd4_session *ses) +{ + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + int maxtime = max_cb_time(clp->net); + struct rpc_timeout timeparms = { + .to_initval = maxtime, + .to_retries = 0, + .to_maxval = maxtime, + }; + struct rpc_create_args args = { + .net = clp->net, + .address = (struct sockaddr *) &conn->cb_addr, + .addrsize = conn->cb_addrlen, + .saddress = (struct sockaddr *) &conn->cb_saddr, + .timeout = &timeparms, + .version = NFS4_CB_VERSION, + .flags = (RPC_CLNT_CREATE_NOPING | RPC_CLNT_CREATE_QUIET), + .cred = current_cred(), + }; + struct rpc_clnt *client; + const struct cred *cred; + + args.program = &nn->nfsd_cb->program; + if (clp->cl_minorversion == 0) { + if (!clp->cl_cred.cr_principal && + (clp->cl_cred.cr_flavor >= RPC_AUTH_GSS_KRB5)) { + trace_nfsd_cb_setup_err(clp, -EINVAL); + return -EINVAL; + } + args.client_name = clp->cl_cred.cr_principal; + args.prognumber = conn->cb_prog; + args.protocol = XPRT_TRANSPORT_TCP; + args.authflavor = clp->cl_cred.cr_flavor; + clp->cl_cb_ident = conn->cb_ident; + } else { + if (!conn->cb_xprt || !ses) + return -EINVAL; + args.bc_xprt = conn->cb_xprt; + args.prognumber = ses->se_cb_prog; + args.protocol = conn->cb_xprt->xpt_class->xcl_ident | + XPRT_TRANSPORT_BC; + args.authflavor = ses->se_cb_sec.flavor; + } + /* Create RPC client */ + client = rpc_create(&args); + if (IS_ERR(client)) { + trace_nfsd_cb_setup_err(clp, PTR_ERR(client)); + return PTR_ERR(client); + } + cred = get_backchannel_cred(clp, client, ses); + if (!cred) { + trace_nfsd_cb_setup_err(clp, -ENOMEM); + rpc_shutdown_client(client); + return -ENOMEM; + } + + if (clp->cl_minorversion != 0) { + clp->cl_cb_conn.cb_xprt = conn->cb_xprt; + rcu_assign_pointer(clp->cl_cb_session, ses); + } + clp->cl_cb_client = client; + clp->cl_cb_cred = cred; + rcu_read_lock(); + trace_nfsd_cb_setup(clp, rpc_peeraddr2str(client, RPC_DISPLAY_NETID), + args.authflavor); + rcu_read_unlock(); + return 0; +} + +static void nfsd4_mark_cb_state(struct nfs4_client *clp, int newstate) +{ + if (clp->cl_cb_state != newstate) { + clp->cl_cb_state = newstate; + trace_nfsd_cb_new_state(clp); + } +} + +static void nfsd4_mark_cb_down(struct nfs4_client *clp) +{ + if (test_bit(NFSD4_CLIENT_CB_UPDATE, &clp->cl_flags)) + return; + nfsd4_mark_cb_state(clp, NFSD4_CB_DOWN); +} + +static void nfsd4_mark_cb_fault(struct nfs4_client *clp) +{ + if (test_bit(NFSD4_CLIENT_CB_UPDATE, &clp->cl_flags)) + return; + nfsd4_mark_cb_state(clp, NFSD4_CB_FAULT); +} + +static void nfsd4_cb_probe_done(struct rpc_task *task, void *calldata) +{ + struct nfs4_client *clp = container_of(calldata, struct nfs4_client, cl_cb_null); + + if (task->tk_status) + nfsd4_mark_cb_down(clp); + else + nfsd4_mark_cb_state(clp, NFSD4_CB_UP); +} + +static void nfsd4_cb_probe_release(void *calldata) +{ + struct nfs4_client *clp = container_of(calldata, struct nfs4_client, cl_cb_null); + + nfsd41_cb_inflight_end(clp); + +} + +static const struct rpc_call_ops nfsd4_cb_probe_ops = { + /* XXX: release method to ensure we set the cb channel down if + * necessary on early failure? */ + .rpc_call_done = nfsd4_cb_probe_done, + .rpc_release = nfsd4_cb_probe_release, +}; + +/* + * Poke the callback thread to process any updates to the callback + * parameters, and send a null probe. + */ +void nfsd4_probe_callback(struct nfs4_client *clp) +{ + trace_nfsd_cb_probe(clp); + nfsd4_mark_cb_state(clp, NFSD4_CB_UNKNOWN); + set_bit(NFSD4_CLIENT_CB_UPDATE, &clp->cl_flags); + nfsd4_run_cb(&clp->cl_cb_null); +} + +void nfsd4_probe_callback_sync(struct nfs4_client *clp) +{ + nfsd4_probe_callback(clp); + flush_workqueue(clp->cl_callback_wq); +} + +void nfsd4_change_callback(struct nfs4_client *clp, struct nfs4_cb_conn *conn) +{ + nfsd4_mark_cb_state(clp, NFSD4_CB_UNKNOWN); + spin_lock(&clp->cl_lock); + memcpy(&clp->cl_cb_conn, conn, sizeof(struct nfs4_cb_conn)); + spin_unlock(&clp->cl_lock); +} + +static int grab_slot(struct nfsd4_session *ses) +{ + int idx; + + spin_lock(&ses->se_lock); + idx = ffs(ses->se_cb_slot_avail) - 1; + if (idx < 0 || idx > ses->se_cb_highest_slot) { + spin_unlock(&ses->se_lock); + return -1; + } + /* clear the bit for the slot */ + ses->se_cb_slot_avail &= ~BIT(idx); + spin_unlock(&ses->se_lock); + return idx; +} + +/* + * There's currently a single callback channel slot. + * If the slot is available, then mark it busy. Otherwise, set the + * thread for sleeping on the callback RPC wait queue. + */ +static bool nfsd41_cb_get_slot(struct nfsd4_callback *cb, struct rpc_task *task) +{ + struct nfs4_client *clp = cb->cb_clp; + struct nfsd4_session *ses; + + if (cb->cb_held_slot >= 0) + return true; + + rcu_read_lock(); + ses = rcu_dereference(clp->cl_cb_session); + if (!ses) { + rcu_read_unlock(); + rpc_sleep_on(&clp->cl_cb_waitq, task, NULL); + return false; + } + cb->cb_held_slot = grab_slot(ses); + if (cb->cb_held_slot < 0) { + rcu_read_unlock(); + rpc_sleep_on(&clp->cl_cb_waitq, task, NULL); + /* Race breaker */ + rcu_read_lock(); + ses = rcu_dereference(clp->cl_cb_session); + if (ses) + cb->cb_held_slot = grab_slot(ses); + rcu_read_unlock(); + if (cb->cb_held_slot < 0) + return false; + rpc_wake_up_queued_task(&clp->cl_cb_waitq, task); + } else { + rcu_read_unlock(); + } + return true; +} + +static void nfsd41_cb_release_slot(struct nfsd4_callback *cb) +{ + struct nfs4_client *clp = cb->cb_clp; + struct nfsd4_session *ses; + + if (cb->cb_held_slot >= 0) { + rcu_read_lock(); + ses = rcu_dereference(clp->cl_cb_session); + if (ses) { + spin_lock(&ses->se_lock); + ses->se_cb_slot_avail |= BIT(cb->cb_held_slot); + spin_unlock(&ses->se_lock); + } + rcu_read_unlock(); + cb->cb_held_slot = -1; + rpc_wake_up_next(&clp->cl_cb_waitq); + } +} + +static void nfsd41_destroy_cb(struct nfsd4_callback *cb) +{ + struct nfs4_client *clp = cb->cb_clp; + + trace_nfsd_cb_destroy(clp, cb); + nfsd41_cb_release_slot(cb); + if (test_bit(NFSD4_CALLBACK_WAKE, &cb->cb_flags)) + clear_and_wake_up_bit(NFSD4_CALLBACK_RUNNING, &cb->cb_flags); + else + clear_bit(NFSD4_CALLBACK_RUNNING, &cb->cb_flags); + + /* + * Order the clear of NFSD4_CALLBACK_RUNNING above before the ->release() + * callback below. A release op may re-check producer-side state to decide + * whether to requeue itself (see nfsd4_cb_notify_release()), and that + * check must not be reordered ahead of the clear. The plain clear_bit() + * path carries no ordering; clear_and_wake_up_bit() already issues this + * barrier internally, so the extra one is harmless there. + */ + smp_mb__after_atomic(); + + if (cb->cb_ops && cb->cb_ops->release) + cb->cb_ops->release(cb); + nfsd41_cb_inflight_end(clp); +} + +/** + * nfsd41_cb_referring_call - add a referring call to a callback operation + * @cb: context of callback to add the rc to + * @sessionid: referring call's session ID + * @slotid: referring call's session slot index + * @seqno: referring call's slot sequence number + * + * Caller serializes access to @cb. + * + * NB: If memory allocation fails, the referring call is not added. + */ +void nfsd41_cb_referring_call(struct nfsd4_callback *cb, + struct nfs4_sessionid *sessionid, + u32 slotid, u32 seqno) +{ + struct nfsd4_referring_call_list *rcl; + struct nfsd4_referring_call *rc; + bool found; + + might_sleep(); + + found = false; + list_for_each_entry(rcl, &cb->cb_referring_call_list, __list) { + if (!memcmp(rcl->rcl_sessionid.data, sessionid->data, + NFS4_MAX_SESSIONID_LEN)) { + found = true; + break; + } + } + if (!found) { + rcl = kmalloc_obj(*rcl); + if (!rcl) + return; + memcpy(rcl->rcl_sessionid.data, sessionid->data, + NFS4_MAX_SESSIONID_LEN); + rcl->__nr_referring_calls = 0; + INIT_LIST_HEAD(&rcl->rcl_referring_calls); + list_add(&rcl->__list, &cb->cb_referring_call_list); + cb->cb_nr_referring_call_list++; + } + + found = false; + list_for_each_entry(rc, &rcl->rcl_referring_calls, __list) { + if (rc->rc_sequenceid == seqno && rc->rc_slotid == slotid) { + found = true; + break; + } + } + if (!found) { + rc = kmalloc_obj(*rc); + if (!rc) + goto out; + rc->rc_sequenceid = seqno; + rc->rc_slotid = slotid; + rcl->__nr_referring_calls++; + list_add(&rc->__list, &rcl->rcl_referring_calls); + } + +out: + if (!rcl->__nr_referring_calls) { + cb->cb_nr_referring_call_list--; + list_del(&rcl->__list); + kfree(rcl); + } +} + +/** + * nfsd41_cb_destroy_referring_call_list - release referring call info + * @cb: context of a callback that has completed + * + * Callers who allocate referring calls using nfsd41_cb_referring_call() must + * release those resources by calling nfsd41_cb_destroy_referring_call_list. + * + * Caller serializes access to @cb. + */ +void nfsd41_cb_destroy_referring_call_list(struct nfsd4_callback *cb) +{ + struct nfsd4_referring_call_list *rcl; + struct nfsd4_referring_call *rc; + + while (!list_empty(&cb->cb_referring_call_list)) { + rcl = list_first_entry(&cb->cb_referring_call_list, + struct nfsd4_referring_call_list, + __list); + + while (!list_empty(&rcl->rcl_referring_calls)) { + rc = list_first_entry(&rcl->rcl_referring_calls, + struct nfsd4_referring_call, + __list); + list_del(&rc->__list); + kfree(rc); + } + list_del(&rcl->__list); + kfree(rcl); + } +} + +static void nfsd4_cb_prepare(struct rpc_task *task, void *calldata) +{ + struct nfsd4_callback *cb = calldata; + struct nfs4_client *clp = cb->cb_clp; + u32 minorversion = clp->cl_minorversion; + + /* + * cb_seq_status is only set in decode_cb_sequence4res, + * and so will remain 1 if an rpc level failure occurs. + */ + trace_nfsd_cb_rpc_prepare(clp); + cb->cb_seq_status = 1; + cb->cb_status = 0; + if (minorversion) { + if (!rcu_access_pointer(clp->cl_cb_session)) { + rpc_exit(task, -EIO); + return; + } + if (!nfsd41_cb_get_slot(cb, task)) + return; + } + rpc_call_start(task); +} + +/* Returns true if CB_COMPOUND processing should continue */ +static bool nfsd4_cb_sequence_done(struct rpc_task *task, struct nfsd4_callback *cb) +{ + struct nfsd4_session *session; + bool ret = false; + + if (cb->cb_held_slot < 0) + goto requeue; + + rcu_read_lock(); + session = rcu_dereference(cb->cb_clp->cl_cb_session); + if (!session) { + rcu_read_unlock(); + goto requeue; + } + + /* This is the operation status code for CB_SEQUENCE */ + trace_nfsd_cb_seq_status(task, cb, session); + switch (cb->cb_seq_status) { + case 0: + /* + * No need for lock, access serialized in nfsd4_cb_prepare + * + * RFC5661 20.9.3 + * If CB_SEQUENCE returns an error, then the state of the slot + * (sequence ID, cached reply) MUST NOT change. + */ + ++session->se_cb_seq_nr[cb->cb_held_slot]; + ret = true; + break; + case -ESERVERFAULT: + /* + * Call succeeded, but the session, slot index, or slot + * sequence number in the response do not match the same + * in the server's call. The sequence information is thus + * untrustworthy. + */ + nfsd4_mark_cb_fault(cb->cb_clp); + break; + case 1: + /* + * cb_seq_status remains 1 if an RPC Reply was never + * received. NFSD can't know if the client processed + * the CB_SEQUENCE operation. Ask the client to send a + * DESTROY_SESSION to recover. + */ + fallthrough; + case -NFS4ERR_BADSESSION: + nfsd4_mark_cb_fault(cb->cb_clp); + rcu_read_unlock(); + goto requeue; + case -NFS4ERR_DELAY: + cb->cb_seq_status = 1; + if (RPC_SIGNALLED(task) || !rpc_restart_call(task)) { + rcu_read_unlock(); + goto requeue; + } + rpc_delay(task, 2 * HZ); + rcu_read_unlock(); + return false; + case -NFS4ERR_SEQ_MISORDERED: + case -NFS4ERR_BADSLOT: + /* + * A SEQ_MISORDERED or BADSLOT error means that the client and + * server are out of sync as to the backchannel parameters. Mark + * the backchannel faulty and restart the RPC, but leak the slot + * so that it's no longer used. + */ + nfsd4_mark_cb_fault(cb->cb_clp); + cb->cb_held_slot = -1; + rcu_read_unlock(); + goto retry_nowait; + default: + nfsd4_mark_cb_fault(cb->cb_clp); + } + trace_nfsd_cb_free_slot(task, cb, session); + rcu_read_unlock(); + nfsd41_cb_release_slot(cb); + return ret; +retry_nowait: + /* + * RPC_SIGNALLED() means that the rpc_client is being torn down and + * (possibly) recreated. Requeue the call in that case. + */ + if (!RPC_SIGNALLED(task)) { + if (rpc_restart_call_prepare(task)) + return false; + } +requeue: + nfsd41_cb_release_slot(cb); + nfsd4_requeue_cb(task, cb); + return false; +} + +static void nfsd4_cb_done(struct rpc_task *task, void *calldata) +{ + struct nfsd4_callback *cb = calldata; + struct nfs4_client *clp = cb->cb_clp; + + trace_nfsd_cb_rpc_done(clp); + + if (!clp->cl_minorversion) { + /* + * If the backchannel connection was shut down while this + * task was queued, we need to resubmit it after setting up + * a new backchannel connection. + * + * Note that if we lost our callback connection permanently + * the submission code will error out, so we don't need to + * handle that case here. + */ + if (RPC_SIGNALLED(task)) + nfsd4_requeue_cb(task, cb); + } else if (!nfsd4_cb_sequence_done(task, cb)) { + return; + } + + if (cb->cb_status) { + WARN_ONCE(task->tk_status, + "cb_status=%d tk_status=%d cb_opcode=%d", + cb->cb_status, task->tk_status, cb->cb_ops->opcode); + task->tk_status = cb->cb_status; + } + + switch (cb->cb_ops->done(cb, task)) { + case 0: + task->tk_status = 0; + rpc_restart_call_prepare(task); + return; + case 1: + switch (task->tk_status) { + case -EIO: + case -ETIMEDOUT: + case -EACCES: + nfsd4_mark_cb_down(clp); + } + break; + default: + BUG(); + } +} + +static void nfsd4_cb_release(void *calldata) +{ + struct nfsd4_callback *cb = calldata; + + trace_nfsd_cb_rpc_release(cb->cb_clp); + + if (test_bit(NFSD4_CALLBACK_REQUEUE, &cb->cb_flags)) + nfsd4_queue_cb(cb); + else + nfsd41_destroy_cb(cb); + +} + +static const struct rpc_call_ops nfsd4_cb_ops = { + .rpc_call_prepare = nfsd4_cb_prepare, + .rpc_call_done = nfsd4_cb_done, + .rpc_release = nfsd4_cb_release, +}; + +/* must be called under the state lock */ +void nfsd4_shutdown_callback(struct nfs4_client *clp) +{ + if (clp->cl_cb_state != NFSD4_CB_UNKNOWN) + trace_nfsd_cb_shutdown(clp); + + set_bit(NFSD4_CLIENT_CB_KILL, &clp->cl_flags); + /* + * Note this won't actually result in a null callback; + * instead, nfsd4_run_cb_null() will detect the killed + * client, destroy the rpc client, and stop: + */ + nfsd4_run_cb(&clp->cl_cb_null); + flush_workqueue(clp->cl_callback_wq); + nfsd41_cb_inflight_wait_complete(clp); +} + +static struct nfsd4_conn * __nfsd4_find_backchannel(struct nfs4_client *clp) +{ + struct nfsd4_session *s; + struct nfsd4_conn *c; + + lockdep_assert_held(&clp->cl_lock); + + list_for_each_entry(s, &clp->cl_sessions, se_perclnt) { + list_for_each_entry(c, &s->se_conns, cn_persession) { + if (c->cn_flags & NFS4_CDFC4_BACK) + return c; + } + } + return NULL; +} + +/* + * Note there isn't a lot of locking in this code; instead we depend on + * the fact that it is run from clp->cl_callback_wq, which won't run two + * work items at once. So, for example, clp->cl_callback_wq handles all + * access of cl_cb_client, and all calls to rpc_create or + * rpc_shutdown_client. + * + * cl_cb_session is written only from cl_callback_wq (via + * rcu_assign_pointer) and read from rpciod under rcu_read_lock (via + * rcu_dereference) by encode_cb_sequence4args(), decode_cb_sequence4resok(), + * nfsd4_cb_sequence_done(), and the cb-slot helpers. Sessions are freed + * with kfree_rcu() so that rpciod readers in an RCU read-side critical + * section never dereference a freed session. + */ +static void nfsd4_process_cb_update(struct nfsd4_callback *cb) +{ + struct nfs4_cb_conn conn; + struct nfs4_client *clp = cb->cb_clp; + struct nfsd4_session *ses = NULL; + struct nfsd4_conn *c; + int err; + + trace_nfsd_cb_bc_update(clp, cb); + + /* + * This is either an update, or the client dying; in either case, + * kill the old client: + */ + if (clp->cl_cb_client) { + trace_nfsd_cb_bc_shutdown(clp, cb); + rpc_shutdown_client(clp->cl_cb_client); + clp->cl_cb_client = NULL; + put_cred(clp->cl_cb_cred); + clp->cl_cb_cred = NULL; + } + if (clp->cl_cb_conn.cb_xprt) { + svc_xprt_put(clp->cl_cb_conn.cb_xprt); + clp->cl_cb_conn.cb_xprt = NULL; + } + if (test_bit(NFSD4_CLIENT_CB_KILL, &clp->cl_flags)) + return; + + spin_lock(&clp->cl_lock); + /* + * Only serialized callback code is allowed to clear these + * flags; main nfsd code can only set them: + */ + WARN_ON(!(clp->cl_flags & NFSD4_CLIENT_CB_FLAG_MASK)); + clear_bit(NFSD4_CLIENT_CB_UPDATE, &clp->cl_flags); + + memcpy(&conn, &cb->cb_clp->cl_cb_conn, sizeof(struct nfs4_cb_conn)); + c = __nfsd4_find_backchannel(clp); + if (c) { + svc_xprt_get(c->cn_xprt); + conn.cb_xprt = c->cn_xprt; + ses = c->cn_session; + } + spin_unlock(&clp->cl_lock); + + err = setup_callback_client(clp, &conn, ses); + if (err) { + nfsd4_mark_cb_down(clp); + if (c) + svc_xprt_put(c->cn_xprt); + rcu_assign_pointer(clp->cl_cb_session, ses); + return; + } +} + +static void +nfsd4_run_cb_work(struct work_struct *work) +{ + struct nfsd4_callback *cb = + container_of(work, struct nfsd4_callback, cb_work); + struct nfs4_client *clp = cb->cb_clp; + struct rpc_clnt *clnt; + int flags, ret; + + trace_nfsd_cb_start(clp); + + if (clp->cl_flags & NFSD4_CLIENT_CB_FLAG_MASK) + nfsd4_process_cb_update(cb); + + clnt = clp->cl_cb_client; + if (!clnt || clp->cl_state == NFSD4_COURTESY) { + /* + * Callback channel broken, client killed or + * nfs4_client in courtesy state; give up. + */ + nfsd41_destroy_cb(cb); + return; + } + + /* + * Don't send probe messages for 4.1 or later. + */ + if (!cb->cb_ops && clp->cl_minorversion) { + nfsd4_mark_cb_state(clp, NFSD4_CB_UP); + nfsd41_destroy_cb(cb); + return; + } + + if (!test_and_clear_bit(NFSD4_CALLBACK_REQUEUE, &cb->cb_flags)) { + if (cb->cb_ops && cb->cb_ops->prepare) + if (!cb->cb_ops->prepare(cb)) { + nfsd41_destroy_cb(cb); + return; + } + } + + cb->cb_msg.rpc_cred = clp->cl_cb_cred; + flags = clp->cl_minorversion ? RPC_TASK_NOCONNECT : RPC_TASK_SOFTCONN; + ret = rpc_call_async(clnt, &cb->cb_msg, RPC_TASK_SOFT | flags, + cb->cb_ops ? &nfsd4_cb_ops : &nfsd4_cb_probe_ops, cb); + if (ret != 0) { + set_bit(NFSD4_CALLBACK_REQUEUE, &cb->cb_flags); + nfsd4_queue_cb(cb); + } +} + +void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp, + const struct nfsd4_callback_ops *ops, enum nfsd4_cb_op op) +{ + cb->cb_clp = clp; + cb->cb_msg.rpc_proc = &nfs4_cb_procedures[op]; + cb->cb_msg.rpc_argp = cb; + cb->cb_msg.rpc_resp = cb; + cb->cb_flags = 0; + cb->cb_ops = ops; + INIT_WORK(&cb->cb_work, nfsd4_run_cb_work); + cb->cb_status = 0; + cb->cb_held_slot = -1; + cb->cb_nr_referring_call_list = 0; + INIT_LIST_HEAD(&cb->cb_referring_call_list); +} + +/** + * nfsd4_run_cb - queue up a callback job to run + * @cb: callback to queue + * + * Kick off a callback to do its thing. Returns false if it was already + * on a queue, true otherwise. + */ +bool nfsd4_run_cb(struct nfsd4_callback *cb) +{ + struct nfs4_client *clp = cb->cb_clp; + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + const struct nfsd4_callback_ops *ops = cb->cb_ops; + u32 minorversion = clp->cl_minorversion; + bool queued; + + nfsd41_cb_inflight_begin(clp); + queued = nfsd4_queue_cb(cb); + if (queued) { + if (ops) { + nfsd_stats_cb_op_inc(nn, ops->opcode); + /* + * Minorversion > 0 callbacks prepend a CB_SEQUENCE op + * (see encode_cb_sequence4args()); count it like the + * forechannel counts SEQUENCE, so it isn't perpetually + * reported as zero. CB_NULL probes (ops == NULL) carry + * no CB_SEQUENCE -- and on 4.1+ they are dropped without + * sending any RPC (see nfsd4_run_cb_work()) -- so they + * must not be counted here. + */ + if (minorversion > 0) + nfsd_stats_cb_op_inc(nn, OP_CB_SEQUENCE); + } + } else { + nfsd41_cb_inflight_end(clp); + } + return queued; +} + +/** + * nfsd_net_cb_shutdown - release per-netns callback RPC program resources + * @nn: NFS server network namespace + * + * Frees resources allocated by nfsd_net_cb_init(). + */ +void nfsd_net_cb_shutdown(struct nfsd_net *nn) +{ + struct nfsd_net_cb *cb = nn->nfsd_cb; + + if (cb) { + kfree(cb->version4.counts); + kfree(cb); + nn->nfsd_cb = NULL; + } +} + +/** + * nfsd_net_cb_init - initialize per-netns callback RPC program + * @nn: NFS server network namespace + * + * Sets up the callback RPC program, version table, procedure + * counts, and statistics structure for @nn. Caller must release + * these resources using nfsd_net_cb_shutdown(). + * + * Return: 0 on success, or -ENOMEM if allocation fails. + */ +int nfsd_net_cb_init(struct nfsd_net *nn) +{ + struct nfsd_net_cb *cb; + + cb = kzalloc(sizeof(*cb), GFP_KERNEL); + if (!cb) + return -ENOMEM; + + cb->version4.counts = kzalloc_objs(unsigned int, + ARRAY_SIZE(nfs4_cb_procedures), GFP_KERNEL); + if (!cb->version4.counts) { + kfree(cb); + return -ENOMEM; + } + /* + * Note on the callback rpc program version number: despite language + * in rfc 5661 section 18.36.3 requiring servers to use 4 in this + * field, the official xdr descriptions for both 4.0 and 4.1 specify + * version 1, and in practice that appears to be what implementations + * use. The section 18.36.3 language is expected to be fixed in an + * erratum. + */ + cb->version4.number = NFS4_CB_VERSION; + cb->version4.nrprocs = ARRAY_SIZE(nfs4_cb_procedures); + cb->version4.procs = nfs4_cb_procedures; + cb->versions[NFS4_CB_VERSION] = &cb->version4; + + cb->program.name = "nfs4_cb"; + cb->program.number = NFS4_CB_PROGRAM; + cb->program.nrvers = ARRAY_SIZE(cb->versions); + cb->program.version = &cb->versions[0]; + cb->program.pipe_dir_name = "nfsd4_cb"; + cb->program.stats = &cb->stat; + cb->stat.program = &cb->program; + + nn->nfsd_cb = cb; + + return 0; +} diff --git a/fs/nfsd/nfs4idmap.c b/fs/nfsd/nfs4idmap.c new file mode 100644 index 000000000..e9faf8b78 --- /dev/null +++ b/fs/nfsd/nfs4idmap.c @@ -0,0 +1,733 @@ +/* + * Mapping of UID/GIDs to name and vice versa. + * + * Copyright (c) 2002, 2003 The Regents of the University of + * Michigan. All rights reserved. + * + * Marius Aamodt Eriksen + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include "auth.h" +#include "idmap.h" +#include "nfsd.h" +#include "netns.h" +#include "vfs.h" + +/* + * Turn off idmapping when using AUTH_SYS. + */ +static bool nfs4_disable_idmapping = true; +module_param(nfs4_disable_idmapping, bool, 0644); +MODULE_PARM_DESC(nfs4_disable_idmapping, + "Turn off server's NFSv4 idmapping when using 'sec=sys'"); + +/* + * Cache entry + */ + +/* + * XXX we know that IDMAP_NAMESZ < PAGE_SIZE, but it's ugly to rely on + * that. + */ + +struct ent { + struct cache_head h; + int type; /* User / Group */ + u32 id; + char name[IDMAP_NAMESZ]; + char authname[IDMAP_NAMESZ]; + struct rcu_head rcu_head; +}; + +/* Common entry handling */ + +#define ENT_HASHBITS 8 +#define ENT_HASHMAX (1 << ENT_HASHBITS) + +static void +ent_init(struct cache_head *cnew, struct cache_head *citm) +{ + struct ent *new = container_of(cnew, struct ent, h); + struct ent *itm = container_of(citm, struct ent, h); + + new->id = itm->id; + new->type = itm->type; + + strscpy(new->name, itm->name, sizeof(new->name)); + strscpy(new->authname, itm->authname, sizeof(new->authname)); +} + +static void +ent_put(struct kref *ref) +{ + struct ent *map = container_of(ref, struct ent, h.ref); + kfree_rcu(map, rcu_head); +} + +static struct cache_head * +ent_alloc(void) +{ + struct ent *e = kmalloc_obj(*e); + if (e) + return &e->h; + else + return NULL; +} + +/* + * ID -> Name cache + */ + +static uint32_t +idtoname_hash(struct ent *ent) +{ + uint32_t hash; + + hash = hash_str(ent->authname, ENT_HASHBITS); + hash = hash_long(hash ^ ent->id, ENT_HASHBITS); + + /* Flip LSB for user/group */ + if (ent->type == IDMAP_TYPE_GROUP) + hash ^= 1; + + return hash; +} + +static int +idtoname_upcall(struct cache_detail *cd, struct cache_head *h) +{ + return sunrpc_cache_upcall_warn(cd, h); +} + +static void +idtoname_request(struct cache_detail *cd, struct cache_head *ch, char **bpp, + int *blen) +{ + struct ent *ent = container_of(ch, struct ent, h); + char idstr[11]; + + qword_add(bpp, blen, ent->authname); + snprintf(idstr, sizeof(idstr), "%u", ent->id); + qword_add(bpp, blen, ent->type == IDMAP_TYPE_GROUP ? "group" : "user"); + qword_add(bpp, blen, idstr); + + (*bpp)[-1] = '\n'; +} + +static int +idtoname_match(struct cache_head *ca, struct cache_head *cb) +{ + struct ent *a = container_of(ca, struct ent, h); + struct ent *b = container_of(cb, struct ent, h); + + return (a->id == b->id && a->type == b->type && + strcmp(a->authname, b->authname) == 0); +} + +static int +idtoname_show(struct seq_file *m, struct cache_detail *cd, struct cache_head *h) +{ + struct ent *ent; + + if (h == NULL) { + seq_puts(m, "#domain type id [name]\n"); + return 0; + } + ent = container_of(h, struct ent, h); + seq_printf(m, "%s %s %u", ent->authname, + ent->type == IDMAP_TYPE_GROUP ? "group" : "user", + ent->id); + if (test_bit(CACHE_VALID, &h->flags)) + seq_printf(m, " %s", ent->name); + seq_putc(m, '\n'); + return 0; +} + +static void +warn_no_idmapd(struct cache_detail *detail, int has_died) +{ + printk("nfsd: nfsv4 idmapping failing: has idmapd %s?\n", + has_died ? "died" : "not been started"); +} + + +static int idtoname_parse(struct cache_detail *, char *, int); +static struct ent *idtoname_lookup(struct cache_detail *, struct ent *); +static struct ent *idtoname_update(struct cache_detail *, struct ent *, + struct ent *); + +static const struct cache_detail idtoname_cache_template = { + .owner = THIS_MODULE, + .hash_size = ENT_HASHMAX, + .name = "nfs4.idtoname", + .cache_put = ent_put, + .cache_upcall = idtoname_upcall, + .cache_request = idtoname_request, + .cache_parse = idtoname_parse, + .cache_show = idtoname_show, + .warn_no_listener = warn_no_idmapd, + .match = idtoname_match, + .init = ent_init, + .update = ent_init, + .alloc = ent_alloc, +}; + +static int +idtoname_parse(struct cache_detail *cd, char *buf, int buflen) +{ + struct ent ent, *res; + char *buf1, *bp; + int len; + int error = -EINVAL; + + if (buf[buflen - 1] != '\n') + return (-EINVAL); + buf[buflen - 1]= '\0'; + + buf1 = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (buf1 == NULL) + return (-ENOMEM); + + memset(&ent, 0, sizeof(ent)); + + /* Authentication name */ + len = qword_get(&buf, buf1, PAGE_SIZE); + if (len <= 0 || len >= IDMAP_NAMESZ) + goto out; + memcpy(ent.authname, buf1, sizeof(ent.authname)); + + /* Type */ + if (qword_get(&buf, buf1, PAGE_SIZE) <= 0) + goto out; + ent.type = strcmp(buf1, "user") == 0 ? + IDMAP_TYPE_USER : IDMAP_TYPE_GROUP; + + /* ID */ + if (qword_get(&buf, buf1, PAGE_SIZE) <= 0) + goto out; + ent.id = simple_strtoul(buf1, &bp, 10); + if (bp == buf1) + goto out; + + /* expiry */ + error = get_expiry(&buf, &ent.h.expiry_time); + if (error) + goto out; + + error = -ENOMEM; + res = idtoname_lookup(cd, &ent); + if (!res) + goto out; + + /* Name */ + error = -EINVAL; + len = qword_get(&buf, buf1, PAGE_SIZE); + if (len < 0 || len >= IDMAP_NAMESZ) + goto out; + if (len == 0) + set_bit(CACHE_NEGATIVE, &ent.h.flags); + else + memcpy(ent.name, buf1, sizeof(ent.name)); + error = -ENOMEM; + res = idtoname_update(cd, &ent, res); + if (res == NULL) + goto out; + + cache_put(&res->h, cd); + error = 0; +out: + kfree(buf1); + return error; +} + +static struct ent * +idtoname_lookup(struct cache_detail *cd, struct ent *item) +{ + struct cache_head *ch = sunrpc_cache_lookup_rcu(cd, &item->h, + idtoname_hash(item)); + if (ch) + return container_of(ch, struct ent, h); + else + return NULL; +} + +static struct ent * +idtoname_update(struct cache_detail *cd, struct ent *new, struct ent *old) +{ + struct cache_head *ch = sunrpc_cache_update(cd, &new->h, &old->h, + idtoname_hash(new)); + if (ch) + return container_of(ch, struct ent, h); + else + return NULL; +} + + +/* + * Name -> ID cache + */ + +static inline int +nametoid_hash(struct ent *ent) +{ + return hash_str(ent->name, ENT_HASHBITS); +} + +static int +nametoid_upcall(struct cache_detail *cd, struct cache_head *h) +{ + return sunrpc_cache_upcall_warn(cd, h); +} + +static void +nametoid_request(struct cache_detail *cd, struct cache_head *ch, char **bpp, + int *blen) +{ + struct ent *ent = container_of(ch, struct ent, h); + + qword_add(bpp, blen, ent->authname); + qword_add(bpp, blen, ent->type == IDMAP_TYPE_GROUP ? "group" : "user"); + qword_add(bpp, blen, ent->name); + + (*bpp)[-1] = '\n'; +} + +static int +nametoid_match(struct cache_head *ca, struct cache_head *cb) +{ + struct ent *a = container_of(ca, struct ent, h); + struct ent *b = container_of(cb, struct ent, h); + + return (a->type == b->type && strcmp(a->name, b->name) == 0 && + strcmp(a->authname, b->authname) == 0); +} + +static int +nametoid_show(struct seq_file *m, struct cache_detail *cd, struct cache_head *h) +{ + struct ent *ent; + + if (h == NULL) { + seq_puts(m, "#domain type name [id]\n"); + return 0; + } + ent = container_of(h, struct ent, h); + seq_printf(m, "%s %s %s", ent->authname, + ent->type == IDMAP_TYPE_GROUP ? "group" : "user", + ent->name); + if (test_bit(CACHE_VALID, &h->flags)) + seq_printf(m, " %u", ent->id); + seq_putc(m, '\n'); + return 0; +} + +static struct ent *nametoid_lookup(struct cache_detail *, struct ent *); +static struct ent *nametoid_update(struct cache_detail *, struct ent *, + struct ent *); +static int nametoid_parse(struct cache_detail *, char *, int); + +static const struct cache_detail nametoid_cache_template = { + .owner = THIS_MODULE, + .hash_size = ENT_HASHMAX, + .name = "nfs4.nametoid", + .cache_put = ent_put, + .cache_upcall = nametoid_upcall, + .cache_request = nametoid_request, + .cache_parse = nametoid_parse, + .cache_show = nametoid_show, + .warn_no_listener = warn_no_idmapd, + .match = nametoid_match, + .init = ent_init, + .update = ent_init, + .alloc = ent_alloc, +}; + +static int +nametoid_parse(struct cache_detail *cd, char *buf, int buflen) +{ + struct ent ent, *res; + char *buf1; + int len, error = -EINVAL; + + if (buf[buflen - 1] != '\n') + return (-EINVAL); + buf[buflen - 1]= '\0'; + + buf1 = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (buf1 == NULL) + return (-ENOMEM); + + memset(&ent, 0, sizeof(ent)); + + /* Authentication name */ + len = qword_get(&buf, buf1, PAGE_SIZE); + if (len <= 0 || len >= IDMAP_NAMESZ) + goto out; + memcpy(ent.authname, buf1, sizeof(ent.authname)); + + /* Type */ + if (qword_get(&buf, buf1, PAGE_SIZE) <= 0) + goto out; + ent.type = strcmp(buf1, "user") == 0 ? + IDMAP_TYPE_USER : IDMAP_TYPE_GROUP; + + /* Name */ + len = qword_get(&buf, buf1, PAGE_SIZE); + if (len <= 0 || len >= IDMAP_NAMESZ) + goto out; + memcpy(ent.name, buf1, sizeof(ent.name)); + + /* expiry */ + error = get_expiry(&buf, &ent.h.expiry_time); + if (error) + goto out; + + /* ID */ + error = get_int(&buf, &ent.id); + if (error == -EINVAL) + goto out; + if (error == -ENOENT) + set_bit(CACHE_NEGATIVE, &ent.h.flags); + + error = -ENOMEM; + res = nametoid_lookup(cd, &ent); + if (res == NULL) + goto out; + res = nametoid_update(cd, &ent, res); + if (res == NULL) + goto out; + + cache_put(&res->h, cd); + error = 0; +out: + kfree(buf1); + return (error); +} + + +static struct ent * +nametoid_lookup(struct cache_detail *cd, struct ent *item) +{ + struct cache_head *ch = sunrpc_cache_lookup_rcu(cd, &item->h, + nametoid_hash(item)); + if (ch) + return container_of(ch, struct ent, h); + else + return NULL; +} + +static struct ent * +nametoid_update(struct cache_detail *cd, struct ent *new, struct ent *old) +{ + struct cache_head *ch = sunrpc_cache_update(cd, &new->h, &old->h, + nametoid_hash(new)); + if (ch) + return container_of(ch, struct ent, h); + else + return NULL; +} + +/* + * Exported API + */ + +int +nfsd_idmap_init(struct net *net) +{ + int rv; + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + nn->idtoname_cache = cache_create_net(&idtoname_cache_template, net); + if (IS_ERR(nn->idtoname_cache)) + return PTR_ERR(nn->idtoname_cache); + rv = cache_register_net(nn->idtoname_cache, net); + if (rv) + goto destroy_idtoname_cache; + nn->nametoid_cache = cache_create_net(&nametoid_cache_template, net); + if (IS_ERR(nn->nametoid_cache)) { + rv = PTR_ERR(nn->nametoid_cache); + goto unregister_idtoname_cache; + } + rv = cache_register_net(nn->nametoid_cache, net); + if (rv) + goto destroy_nametoid_cache; + return 0; + +destroy_nametoid_cache: + cache_destroy_net(nn->nametoid_cache, net); +unregister_idtoname_cache: + cache_unregister_net(nn->idtoname_cache, net); +destroy_idtoname_cache: + cache_destroy_net(nn->idtoname_cache, net); + return rv; +} + +void +nfsd_idmap_shutdown(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + cache_unregister_net(nn->idtoname_cache, net); + cache_unregister_net(nn->nametoid_cache, net); + cache_destroy_net(nn->idtoname_cache, net); + cache_destroy_net(nn->nametoid_cache, net); +} + +static int +idmap_lookup(struct svc_rqst *rqstp, + struct ent *(*lookup_fn)(struct cache_detail *, struct ent *), + struct ent *key, struct cache_detail *detail, struct ent **item) +{ + int ret; + + *item = lookup_fn(detail, key); + if (!*item) + return -ENOMEM; + retry: + ret = cache_check(detail, &(*item)->h, &rqstp->rq_chandle); + + if (ret == -ETIMEDOUT) { + struct ent *prev_item = *item; + *item = lookup_fn(detail, key); + if (*item != prev_item) + goto retry; + cache_put(&(*item)->h, detail); + } + return ret; +} + +static char * +rqst_authname(struct svc_rqst *rqstp) +{ + struct auth_domain *clp; + + clp = rqstp->rq_gssclient ? rqstp->rq_gssclient : rqstp->rq_client; + return clp->name; +} + +static __be32 +idmap_name_to_id(struct svc_rqst *rqstp, int type, const char *name, u32 namelen, + u32 *id) +{ + struct ent *item, key = { + .type = type, + }; + int ret; + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + + if (namelen + 1 > sizeof(key.name)) + return nfserr_badowner; + memcpy(key.name, name, namelen); + key.name[namelen] = '\0'; + strscpy(key.authname, rqst_authname(rqstp), sizeof(key.authname)); + ret = idmap_lookup(rqstp, nametoid_lookup, &key, nn->nametoid_cache, &item); + if (ret == -ENOENT) + return nfserr_badowner; + if (ret) + return nfserrno(ret); + *id = item->id; + cache_put(&item->h, nn->nametoid_cache); + return 0; +} + +static __be32 encode_ascii_id(struct xdr_stream *xdr, u32 id) +{ + char buf[11]; + int len; + __be32 *p; + + len = sprintf(buf, "%u", id); + p = xdr_reserve_space(xdr, len + 4); + if (!p) + return nfserr_resource; + p = xdr_encode_opaque(p, buf, len); + return 0; +} + +static __be32 idmap_id_to_name(struct xdr_stream *xdr, + struct svc_rqst *rqstp, int type, u32 id) +{ + struct ent *item, key = { + .id = id, + .type = type, + }; + __be32 status = nfs_ok; + __be32 *p; + int ret; + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + + strscpy(key.authname, rqst_authname(rqstp), sizeof(key.authname)); + ret = idmap_lookup(rqstp, idtoname_lookup, &key, nn->idtoname_cache, &item); + if (ret == -ENOENT) + return encode_ascii_id(xdr, id); + if (ret) + return nfserrno(ret); + ret = strlen(item->name); + WARN_ON_ONCE(ret > IDMAP_NAMESZ); + + p = xdr_reserve_space(xdr, ret + 4); + if (unlikely(!p)) { + status = nfserr_resource; + goto out_put; + } + xdr_encode_opaque(p, item->name, ret); +out_put: + cache_put(&item->h, nn->idtoname_cache); + return status; +} + +static bool +numeric_name_to_id(struct svc_rqst *rqstp, int type, const char *name, u32 namelen, u32 *id) +{ + int ret; + char buf[11]; + + if (namelen + 1 > sizeof(buf)) + /* too long to represent a 32-bit id: */ + return false; + /* Just to make sure it's null-terminated: */ + memcpy(buf, name, namelen); + buf[namelen] = '\0'; + ret = kstrtouint(buf, 10, id); + return ret == 0; +} + +static __be32 +do_name_to_id(struct svc_rqst *rqstp, int type, const char *name, u32 namelen, u32 *id) +{ + if (nfs4_disable_idmapping && rqstp->rq_cred.cr_flavor < RPC_AUTH_GSS) + if (numeric_name_to_id(rqstp, type, name, namelen, id)) + return 0; + /* + * otherwise, fall through and try idmapping, for + * backwards compatibility with clients sending names: + */ + return idmap_name_to_id(rqstp, type, name, namelen, id); +} + +static __be32 encode_name_from_id(struct xdr_stream *xdr, + struct svc_rqst *rqstp, int type, u32 id) +{ + if (nfs4_disable_idmapping && rqstp->rq_cred.cr_flavor < RPC_AUTH_GSS) + return encode_ascii_id(xdr, id); + return idmap_id_to_name(xdr, rqstp, type, id); +} + +/** + * nfsd_map_name_to_uid - Map user@domain to local UID + * @rqstp: RPC execution context + * @name: user@domain name to be mapped + * @namelen: length of name, in bytes + * @uid: OUT: mapped local UID value + * + * Returns nfs_ok on success or an NFSv4 status code on failure. + */ +__be32 nfsd_map_name_to_uid(struct svc_rqst *rqstp, const char *name, + size_t namelen, kuid_t *uid) +{ + __be32 status; + u32 id = -1; + + /* + * The idmap lookup below triggers an upcall that invokes + * cache_check(). RQ_USEDEFERRAL must be clear to prevent + * cache_check() from setting RQ_DROPME via svc_defer(). + * NFSv4 servers are not permitted to drop requests. Also + * RQ_DROPME will force NFSv4.1 session slot processing to + * be skipped. + */ + WARN_ON_ONCE(test_bit(RQ_USEDEFERRAL, &rqstp->rq_flags)); + + if (name == NULL || namelen == 0) + return nfserr_inval; + + status = do_name_to_id(rqstp, IDMAP_TYPE_USER, name, namelen, &id); + if (status) + return status; + *uid = make_kuid(nfsd_user_namespace(rqstp), id); + if (!uid_valid(*uid)) + status = nfserr_badowner; + return status; +} + +/** + * nfsd_map_name_to_gid - Map user@domain to local GID + * @rqstp: RPC execution context + * @name: user@domain name to be mapped + * @namelen: length of name, in bytes + * @gid: OUT: mapped local GID value + * + * Returns nfs_ok on success or an NFSv4 status code on failure. + */ +__be32 nfsd_map_name_to_gid(struct svc_rqst *rqstp, const char *name, + size_t namelen, kgid_t *gid) +{ + __be32 status; + u32 id = -1; + + /* + * The idmap lookup below triggers an upcall that invokes + * cache_check(). RQ_USEDEFERRAL must be clear to prevent + * cache_check() from setting RQ_DROPME via svc_defer(). + * NFSv4 servers are not permitted to drop requests. Also + * RQ_DROPME will force NFSv4.1 session slot processing to + * be skipped. + */ + WARN_ON_ONCE(test_bit(RQ_USEDEFERRAL, &rqstp->rq_flags)); + + if (name == NULL || namelen == 0) + return nfserr_inval; + + status = do_name_to_id(rqstp, IDMAP_TYPE_GROUP, name, namelen, &id); + if (status) + return status; + *gid = make_kgid(nfsd_user_namespace(rqstp), id); + if (!gid_valid(*gid)) + status = nfserr_badowner; + return status; +} + +__be32 nfsd4_encode_user(struct xdr_stream *xdr, struct svc_rqst *rqstp, + kuid_t uid) +{ + u32 id = from_kuid_munged(nfsd_user_namespace(rqstp), uid); + return encode_name_from_id(xdr, rqstp, IDMAP_TYPE_USER, id); +} + +__be32 nfsd4_encode_group(struct xdr_stream *xdr, struct svc_rqst *rqstp, + kgid_t gid) +{ + u32 id = from_kgid_munged(nfsd_user_namespace(rqstp), gid); + return encode_name_from_id(xdr, rqstp, IDMAP_TYPE_GROUP, id); +} diff --git a/fs/nfsd/nfs4layouts.c b/fs/nfsd/nfs4layouts.c new file mode 100644 index 000000000..22bcb6d09 --- /dev/null +++ b/fs/nfsd/nfs4layouts.c @@ -0,0 +1,967 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2014 Christoph Hellwig. + */ +#include +#include +#include +#include +#include +#include + +#include "pnfs.h" +#include "netns.h" +#include "trace.h" + +#define NFSDDBG_FACILITY NFSDDBG_PNFS + +struct nfs4_layout { + struct list_head lo_perstate; + struct nfs4_layout_stateid *lo_state; + struct nfsd4_layout_seg lo_seg; +}; + +static struct kmem_cache *nfs4_layout_cache; +static struct kmem_cache *nfs4_layout_stateid_cache; + +static const struct nfsd4_callback_ops nfsd4_cb_layout_ops; +static const struct lease_manager_operations nfsd4_layouts_lm_ops; + +static void nfsd4_layout_fence_worker(struct work_struct *work); + +const struct nfsd4_layout_ops *nfsd4_layout_ops[LAYOUT_TYPE_MAX] = { +#ifdef CONFIG_NFSD_FLEXFILELAYOUT + [LAYOUT_FLEX_FILES] = &ff_layout_ops, +#endif +#ifdef CONFIG_NFSD_BLOCKLAYOUT + [LAYOUT_BLOCK_VOLUME] = &bl_layout_ops, +#endif +#ifdef CONFIG_NFSD_SCSILAYOUT + [LAYOUT_SCSI] = &scsi_layout_ops, +#endif +}; + +/* pNFS device ID to export fsid mapping */ +#define DEVID_HASH_BITS 8 +#define DEVID_HASH_SIZE (1 << DEVID_HASH_BITS) +#define DEVID_HASH_MASK (DEVID_HASH_SIZE - 1) +static u64 nfsd_devid_seq = 1; +static struct list_head nfsd_devid_hash[DEVID_HASH_SIZE]; +static DEFINE_SPINLOCK(nfsd_devid_lock); + +static inline u32 devid_hashfn(u64 idx) +{ + return jhash_2words(idx, idx >> 32, 0) & DEVID_HASH_MASK; +} + +static void +nfsd4_alloc_devid_map(const struct svc_fh *fhp) +{ + const struct knfsd_fh *fh = &fhp->fh_handle; + size_t fsid_len = key_len(fh->fh_fsid_type); + struct nfsd4_deviceid_map *map, *old; + int i; + + map = kzalloc(sizeof(*map) + fsid_len, GFP_KERNEL); + if (!map) + return; + + map->fsid_type = fh->fh_fsid_type; + memcpy(&map->fsid, fh_fsid(fh), fsid_len); + + spin_lock(&nfsd_devid_lock); + if (fhp->fh_export->ex_devid_map) + goto out_unlock; + + for (i = 0; i < DEVID_HASH_SIZE; i++) { + list_for_each_entry(old, &nfsd_devid_hash[i], hash) { + if (old->fsid_type != fh->fh_fsid_type) + continue; + if (memcmp(old->fsid, fh_fsid(fh), + key_len(old->fsid_type))) + continue; + + fhp->fh_export->ex_devid_map = old; + goto out_unlock; + } + } + + map->idx = nfsd_devid_seq++; + list_add_tail_rcu(&map->hash, &nfsd_devid_hash[devid_hashfn(map->idx)]); + fhp->fh_export->ex_devid_map = map; + map = NULL; + +out_unlock: + spin_unlock(&nfsd_devid_lock); + kfree(map); +} + +struct nfsd4_deviceid_map * +nfsd4_find_devid_map(int idx) +{ + struct nfsd4_deviceid_map *map, *ret = NULL; + + rcu_read_lock(); + list_for_each_entry_rcu(map, &nfsd_devid_hash[devid_hashfn(idx)], hash) + if (map->idx == idx) + ret = map; + rcu_read_unlock(); + + return ret; +} + +int +nfsd4_set_deviceid(struct nfsd4_deviceid *id, const struct svc_fh *fhp, + u32 device_generation) +{ + if (!fhp->fh_export->ex_devid_map) { + nfsd4_alloc_devid_map(fhp); + if (!fhp->fh_export->ex_devid_map) + return -ENOMEM; + } + + id->fsid_idx = fhp->fh_export->ex_devid_map->idx; + id->generation = device_generation; + return 0; +} + +void nfsd4_setup_layout_type(struct svc_export *exp) +{ + struct super_block *sb = exp->ex_path.mnt->mnt_sb; + expfs_block_layouts_t block_supported = exportfs_layouts_supported(sb); + + if (IS_ENABLED(CONFIG_NFSD_FLEXFILELAYOUT)) + exp->ex_layout_types |= 1 << LAYOUT_FLEX_FILES; + if (IS_ENABLED(CONFIG_NFSD_BLOCKLAYOUT) && + (block_supported & EXPFS_BLOCK_IN_BAND_ID)) + exp->ex_layout_types |= 1 << LAYOUT_BLOCK_VOLUME; + if (IS_ENABLED(CONFIG_NFSD_SCSILAYOUT) && + (block_supported & EXPFS_BLOCK_OUT_OF_BAND_ID)) + exp->ex_layout_types |= 1 << LAYOUT_SCSI; +} + +void nfsd4_close_layout(struct nfs4_layout_stateid *ls) +{ + struct nfsd_file *fl; + + spin_lock(&ls->ls_stid.sc_file->fi_lock); + fl = ls->ls_file; + ls->ls_file = NULL; + spin_unlock(&ls->ls_stid.sc_file->fi_lock); + + if (fl) { + if (!nfsd4_layout_ops[ls->ls_layout_type]->disable_recalls) + kernel_setlease(fl->nf_file, F_UNLCK, NULL, + (void **)&ls); + nfsd_file_put(fl); + } +} + +static void +nfsd4_free_layout_stateid(struct nfs4_stid *stid) +{ + struct nfs4_layout_stateid *ls = layoutstateid(stid); + struct nfs4_client *clp = ls->ls_stid.sc_client; + struct nfs4_file *fp = ls->ls_stid.sc_file; + + trace_nfsd_layoutstate_free(&ls->ls_stid.sc_stateid); + + spin_lock(&ls->ls_lock); + if (delayed_work_pending(&ls->ls_fence_work)) { + spin_unlock(&ls->ls_lock); + cancel_delayed_work_sync(&ls->ls_fence_work); + } else + spin_unlock(&ls->ls_lock); + + spin_lock(&clp->cl_lock); + list_del_init(&ls->ls_perclnt); + spin_unlock(&clp->cl_lock); + + spin_lock(&fp->fi_lock); + list_del_init(&ls->ls_perfile); + spin_unlock(&fp->fi_lock); + + nfsd4_close_layout(ls); + + if (ls->ls_recalled) + atomic_dec(&ls->ls_stid.sc_file->fi_lo_recalls); + + kmem_cache_free(nfs4_layout_stateid_cache, ls); +} + +static int +nfsd4_layout_setlease(struct nfs4_layout_stateid *ls) +{ + struct file_lease *fl; + int status; + + if (nfsd4_layout_ops[ls->ls_layout_type]->disable_recalls) + return 0; + + fl = locks_alloc_lease(); + if (!fl) + return -ENOMEM; + locks_init_lease(fl); + fl->fl_lmops = &nfsd4_layouts_lm_ops; + fl->c.flc_flags = FL_LAYOUT; + fl->c.flc_type = F_RDLCK; + fl->c.flc_owner = ls; + fl->c.flc_pid = current->tgid; + fl->c.flc_file = ls->ls_file->nf_file; + + status = kernel_setlease(fl->c.flc_file, fl->c.flc_type, &fl, NULL); + if (status) { + locks_free_lease(fl); + return status; + } + BUG_ON(fl != NULL); + return 0; +} + +static struct nfs4_layout_stateid * +nfsd4_alloc_layout_stateid(struct nfsd4_compound_state *cstate, + struct nfs4_stid *parent, u32 layout_type) +{ + struct nfs4_client *clp = cstate->clp; + struct nfs4_file *fp = parent->sc_file; + struct nfs4_layout_stateid *ls; + struct nfs4_stid *stp; + + stp = nfs4_alloc_stid(cstate->clp, nfs4_layout_stateid_cache, + nfsd4_free_layout_stateid); + if (!stp) + return NULL; + + get_nfs4_file(fp); + stp->sc_file = fp; + if (parent->sc_export) + stp->sc_export = exp_get(parent->sc_export); + + ls = layoutstateid(stp); + INIT_LIST_HEAD(&ls->ls_perclnt); + INIT_LIST_HEAD(&ls->ls_perfile); + spin_lock_init(&ls->ls_lock); + INIT_LIST_HEAD(&ls->ls_layouts); + mutex_init(&ls->ls_mutex); + ls->ls_layout_type = layout_type; + nfsd4_init_cb(&ls->ls_recall, clp, &nfsd4_cb_layout_ops, + NFSPROC4_CLNT_CB_LAYOUT); + + if (parent->sc_type == SC_TYPE_DELEG) { + rcu_read_lock(); + ls->ls_file = nfsd_file_get(rcu_dereference(fp->fi_deleg_file)); + rcu_read_unlock(); + } else { + ls->ls_file = find_any_file(fp); + } + + if (!ls->ls_file) { + nfs4_put_stid(stp); + return NULL; + } + + ls->ls_fenced = false; + ls->ls_fence_inflight = false; + ls->ls_fence_delay = 0; + INIT_DELAYED_WORK(&ls->ls_fence_work, nfsd4_layout_fence_worker); + + if (nfsd4_layout_setlease(ls)) { + nfs4_put_stid(stp); + return NULL; + } + + spin_lock(&clp->cl_lock); + stp->sc_type = SC_TYPE_LAYOUT; + list_add(&ls->ls_perclnt, &clp->cl_lo_states); + spin_unlock(&clp->cl_lock); + + spin_lock(&fp->fi_lock); + list_add(&ls->ls_perfile, &fp->fi_lo_states); + spin_unlock(&fp->fi_lock); + + trace_nfsd_layoutstate_alloc(&ls->ls_stid.sc_stateid); + return ls; +} + +__be32 +nfsd4_preprocess_layout_stateid(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, stateid_t *stateid, + bool create, u32 layout_type, struct nfs4_layout_stateid **lsp) +{ + struct nfs4_layout_stateid *ls; + struct nfs4_stid *stid; + unsigned short typemask = SC_TYPE_LAYOUT; + __be32 status; + + if (create) + typemask |= (SC_TYPE_OPEN | SC_TYPE_LOCK | SC_TYPE_DELEG); + + status = nfsd4_lookup_stateid(cstate, stateid, typemask, 0, &stid, + net_generic(SVC_NET(rqstp), nfsd_net_id)); + if (status) + goto out; + + if (!fh_match(&cstate->current_fh.fh_handle, + &stid->sc_file->fi_fhandle)) { + status = nfserr_bad_stateid; + goto out_put_stid; + } + + if (stid->sc_type != SC_TYPE_LAYOUT) { + ls = nfsd4_alloc_layout_stateid(cstate, stid, layout_type); + nfs4_put_stid(stid); + + status = nfserr_jukebox; + if (!ls) + goto out; + mutex_lock(&ls->ls_mutex); + } else { + ls = container_of(stid, struct nfs4_layout_stateid, ls_stid); + + status = nfserr_bad_stateid; + mutex_lock(&ls->ls_mutex); + if (nfsd4_stateid_generation_after(stateid, &stid->sc_stateid)) + goto out_unlock_stid; + if (layout_type != ls->ls_layout_type) + goto out_unlock_stid; + } + + *lsp = ls; + return 0; + +out_unlock_stid: + mutex_unlock(&ls->ls_mutex); +out_put_stid: + nfs4_put_stid(stid); +out: + return status; +} + +static void +nfsd4_recall_file_layout(struct nfs4_layout_stateid *ls) +{ + spin_lock(&ls->ls_lock); + if (ls->ls_recalled) + goto out_unlock; + + if (list_empty(&ls->ls_layouts)) + goto out_unlock; + + ls->ls_recalled = true; + atomic_inc(&ls->ls_stid.sc_file->fi_lo_recalls); + trace_nfsd_layout_recall(&ls->ls_stid.sc_stateid); + + if (!test_and_set_bit(NFSD4_CALLBACK_RUNNING, &ls->ls_recall.cb_flags)) { + refcount_inc(&ls->ls_stid.sc_count); + nfsd4_run_cb(&ls->ls_recall); + } +out_unlock: + spin_unlock(&ls->ls_lock); +} + +static inline u64 +layout_end(struct nfsd4_layout_seg *seg) +{ + u64 end = seg->offset + seg->length; + return end >= seg->offset ? end : NFS4_MAX_UINT64; +} + +static void +layout_update_len(struct nfsd4_layout_seg *lo, u64 end) +{ + if (end == NFS4_MAX_UINT64) + lo->length = NFS4_MAX_UINT64; + else + lo->length = end - lo->offset; +} + +static bool +layouts_overlapping(struct nfs4_layout *lo, struct nfsd4_layout_seg *s) +{ + if (s->iomode != IOMODE_ANY && s->iomode != lo->lo_seg.iomode) + return false; + if (layout_end(&lo->lo_seg) <= s->offset) + return false; + if (layout_end(s) <= lo->lo_seg.offset) + return false; + return true; +} + +static bool +layouts_try_merge(struct nfsd4_layout_seg *lo, struct nfsd4_layout_seg *new) +{ + if (lo->iomode != new->iomode) + return false; + if (layout_end(new) < lo->offset) + return false; + if (layout_end(lo) < new->offset) + return false; + + lo->offset = min(lo->offset, new->offset); + layout_update_len(lo, max(layout_end(lo), layout_end(new))); + return true; +} + +static __be32 +nfsd4_recall_conflict(struct nfs4_layout_stateid *ls) +{ + struct nfs4_file *fp = ls->ls_stid.sc_file; + struct nfs4_layout_stateid *l, *n; + __be32 nfserr = nfs_ok; + + assert_spin_locked(&fp->fi_lock); + + list_for_each_entry_safe(l, n, &fp->fi_lo_states, ls_perfile) { + if (l != ls) { + nfsd4_recall_file_layout(l); + nfserr = nfserr_recallconflict; + } + } + + return nfserr; +} + +__be32 +nfsd4_insert_layout(struct nfsd4_layoutget *lgp, struct nfs4_layout_stateid *ls) +{ + struct nfsd4_layout_seg *seg = &lgp->lg_seg; + struct nfs4_file *fp = ls->ls_stid.sc_file; + struct nfs4_layout *lp, *new = NULL; + __be32 nfserr; + + spin_lock(&fp->fi_lock); + nfserr = nfsd4_recall_conflict(ls); + if (nfserr) + goto out; + spin_lock(&ls->ls_lock); + list_for_each_entry(lp, &ls->ls_layouts, lo_perstate) { + if (layouts_try_merge(&lp->lo_seg, seg)) + goto done; + } + spin_unlock(&ls->ls_lock); + spin_unlock(&fp->fi_lock); + + new = kmem_cache_alloc(nfs4_layout_cache, GFP_KERNEL); + if (!new) + return nfserr_jukebox; + memcpy(&new->lo_seg, seg, sizeof(new->lo_seg)); + new->lo_state = ls; + + spin_lock(&fp->fi_lock); + nfserr = nfsd4_recall_conflict(ls); + if (nfserr) + goto out; + spin_lock(&ls->ls_lock); + list_for_each_entry(lp, &ls->ls_layouts, lo_perstate) { + if (layouts_try_merge(&lp->lo_seg, seg)) + goto done; + } + + refcount_inc(&ls->ls_stid.sc_count); + list_add_tail(&new->lo_perstate, &ls->ls_layouts); + new = NULL; +done: + nfs4_inc_and_copy_stateid(&lgp->lg_sid, &ls->ls_stid); + spin_unlock(&ls->ls_lock); +out: + spin_unlock(&fp->fi_lock); + if (new) + kmem_cache_free(nfs4_layout_cache, new); + return nfserr; +} + +static void +nfsd4_free_layouts(struct list_head *reaplist) +{ + while (!list_empty(reaplist)) { + struct nfs4_layout *lp = list_first_entry(reaplist, + struct nfs4_layout, lo_perstate); + + list_del(&lp->lo_perstate); + nfs4_put_stid(&lp->lo_state->ls_stid); + kmem_cache_free(nfs4_layout_cache, lp); + } +} + +static void +nfsd4_return_file_layout(struct nfs4_layout *lp, struct nfsd4_layout_seg *seg, + struct list_head *reaplist) +{ + struct nfsd4_layout_seg *lo = &lp->lo_seg; + u64 end = layout_end(lo); + + if (seg->offset <= lo->offset) { + if (layout_end(seg) >= end) { + list_move_tail(&lp->lo_perstate, reaplist); + return; + } + lo->offset = layout_end(seg); + } else { + /* retain the whole layout segment on a split. */ + if (layout_end(seg) < end) { + dprintk("%s: split not supported\n", __func__); + return; + } + end = seg->offset; + } + + layout_update_len(lo, end); +} + +__be32 +nfsd4_return_file_layouts(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, + struct nfsd4_layoutreturn *lrp) +{ + struct nfs4_layout_stateid *ls; + struct nfs4_layout *lp, *n; + LIST_HEAD(reaplist); + __be32 nfserr; + int found = 0; + + nfserr = nfsd4_preprocess_layout_stateid(rqstp, cstate, &lrp->lr_sid, + false, lrp->lr_layout_type, + &ls); + if (nfserr) { + trace_nfsd_layout_return_lookup_fail(&lrp->lr_sid); + return nfserr; + } + + spin_lock(&ls->ls_lock); + list_for_each_entry_safe(lp, n, &ls->ls_layouts, lo_perstate) { + if (layouts_overlapping(lp, &lrp->lr_seg)) { + nfsd4_return_file_layout(lp, &lrp->lr_seg, &reaplist); + found++; + } + } + if (!list_empty(&ls->ls_layouts)) { + if (found) + nfs4_inc_and_copy_stateid(&lrp->lr_sid, &ls->ls_stid); + lrp->lrs_present = true; + } else { + trace_nfsd_layoutstate_unhash(&ls->ls_stid.sc_stateid); + ls->ls_stid.sc_status |= SC_STATUS_CLOSED; + lrp->lrs_present = false; + } + spin_unlock(&ls->ls_lock); + + mutex_unlock(&ls->ls_mutex); + nfs4_put_stid(&ls->ls_stid); + nfsd4_free_layouts(&reaplist); + return nfs_ok; +} + +__be32 +nfsd4_return_client_layouts(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, + struct nfsd4_layoutreturn *lrp) +{ + struct nfs4_layout_stateid *ls, *n; + struct nfs4_client *clp = cstate->clp; + struct nfs4_layout *lp, *t; + LIST_HEAD(reaplist); + + lrp->lrs_present = false; + + spin_lock(&clp->cl_lock); + list_for_each_entry_safe(ls, n, &clp->cl_lo_states, ls_perclnt) { + if (ls->ls_layout_type != lrp->lr_layout_type) + continue; + + if (lrp->lr_return_type == RETURN_FSID && + !fh_fsid_match(&ls->ls_stid.sc_file->fi_fhandle, + &cstate->current_fh.fh_handle)) + continue; + + spin_lock(&ls->ls_lock); + list_for_each_entry_safe(lp, t, &ls->ls_layouts, lo_perstate) { + if (lrp->lr_seg.iomode == IOMODE_ANY || + lrp->lr_seg.iomode == lp->lo_seg.iomode) + list_move_tail(&lp->lo_perstate, &reaplist); + } + spin_unlock(&ls->ls_lock); + } + spin_unlock(&clp->cl_lock); + + nfsd4_free_layouts(&reaplist); + return 0; +} + +static void +nfsd4_return_all_layouts(struct nfs4_layout_stateid *ls, + struct list_head *reaplist) +{ + spin_lock(&ls->ls_lock); + list_splice_init(&ls->ls_layouts, reaplist); + spin_unlock(&ls->ls_lock); +} + +void +nfsd4_return_all_client_layouts(struct nfs4_client *clp) +{ + struct nfs4_layout_stateid *ls, *n; + LIST_HEAD(reaplist); + + spin_lock(&clp->cl_lock); + list_for_each_entry_safe(ls, n, &clp->cl_lo_states, ls_perclnt) + nfsd4_return_all_layouts(ls, &reaplist); + spin_unlock(&clp->cl_lock); + + nfsd4_free_layouts(&reaplist); +} + +void +nfsd4_return_all_file_layouts(struct nfs4_client *clp, struct nfs4_file *fp) +{ + struct nfs4_layout_stateid *ls, *n; + LIST_HEAD(reaplist); + + spin_lock(&fp->fi_lock); + list_for_each_entry_safe(ls, n, &fp->fi_lo_states, ls_perfile) { + if (ls->ls_stid.sc_client == clp) + nfsd4_return_all_layouts(ls, &reaplist); + } + spin_unlock(&fp->fi_lock); + + nfsd4_free_layouts(&reaplist); +} + +static void +nfsd4_cb_layout_fail(struct nfs4_layout_stateid *ls, struct nfsd_file *file) +{ + struct nfs4_client *clp = ls->ls_stid.sc_client; + char addr_str[INET6_ADDRSTRLEN]; + static char const nfsd_recall_failed[] = "/sbin/nfsd-recall-failed"; + static char *envp[] = { + "HOME=/", + "TERM=linux", + "PATH=/sbin:/usr/sbin:/bin:/usr/bin", + NULL + }; + char *argv[8]; + int error; + + rpc_ntop((struct sockaddr *)&clp->cl_addr, addr_str, sizeof(addr_str)); + + printk(KERN_WARNING + "nfsd: client %s failed to respond to layout recall. " + " Fencing..\n", addr_str); + + argv[0] = (char *)nfsd_recall_failed; + argv[1] = addr_str; + argv[2] = file->nf_file->f_path.mnt->mnt_sb->s_id; + argv[3] = NULL; + + error = call_usermodehelper(nfsd_recall_failed, argv, envp, + UMH_WAIT_PROC); + if (error) { + printk(KERN_ERR "nfsd: fence failed for client %s: %d!\n", + addr_str, error); + } +} + +static bool +nfsd4_cb_layout_prepare(struct nfsd4_callback *cb) +{ + struct nfs4_layout_stateid *ls = + container_of(cb, struct nfs4_layout_stateid, ls_recall); + + mutex_lock(&ls->ls_mutex); + nfs4_inc_and_copy_stateid(&ls->ls_recall_sid, &ls->ls_stid); + mutex_unlock(&ls->ls_mutex); + return true; +} + +static int +nfsd4_cb_layout_done(struct nfsd4_callback *cb, struct rpc_task *task) +{ + struct nfs4_layout_stateid *ls = + container_of(cb, struct nfs4_layout_stateid, ls_recall); + struct nfsd_net *nn; + ktime_t now, cutoff; + const struct nfsd4_layout_ops *ops; + struct nfsd_file *fl; + + trace_nfsd_cb_layout_done(&ls->ls_stid.sc_stateid, task); + switch (task->tk_status) { + case 0: + case -NFS4ERR_DELAY: + /* + * Anything left? If not, then call it done. Note that we don't + * take the spinlock since this is an optimization and nothing + * should get added until the cb counter goes to zero. + */ + if (list_empty(&ls->ls_layouts)) + return 1; + + /* Poll the client until it's done with the layout */ + now = ktime_get(); + nn = net_generic(ls->ls_stid.sc_client->net, nfsd_net_id); + + /* Client gets 2 lease periods to return it */ + cutoff = ktime_add_ns(task->tk_start, + (u64)nn->nfsd4_lease * NSEC_PER_SEC * 2); + + if (ktime_before(now, cutoff)) { + rpc_delay(task, HZ/100); /* 10 mili-seconds */ + return 0; + } + fallthrough; + default: + /* + * Unknown error or non-responding client, we'll need to fence. + */ + trace_nfsd_layout_recall_fail(&ls->ls_stid.sc_stateid); + rcu_read_lock(); + fl = nfsd_file_get(ls->ls_file); + rcu_read_unlock(); + if (fl) { + ops = nfsd4_layout_ops[ls->ls_layout_type]; + if (ops->fence_client) + ops->fence_client(ls, fl); + else + nfsd4_cb_layout_fail(ls, fl); + nfsd_file_put(fl); + } + return 1; + case -NFS4ERR_NOMATCHING_LAYOUT: + trace_nfsd_layout_recall_done(&ls->ls_stid.sc_stateid); + task->tk_status = 0; + return 1; + } +} + +static void +nfsd4_cb_layout_release(struct nfsd4_callback *cb) +{ + struct nfs4_layout_stateid *ls = + container_of(cb, struct nfs4_layout_stateid, ls_recall); + LIST_HEAD(reaplist); + + trace_nfsd_layout_recall_release(&ls->ls_stid.sc_stateid); + + nfsd4_return_all_layouts(ls, &reaplist); + nfsd4_free_layouts(&reaplist); + nfs4_put_stid(&ls->ls_stid); +} + +static const struct nfsd4_callback_ops nfsd4_cb_layout_ops = { + .prepare = nfsd4_cb_layout_prepare, + .done = nfsd4_cb_layout_done, + .release = nfsd4_cb_layout_release, + .opcode = OP_CB_LAYOUTRECALL, +}; + +static bool +nfsd4_layout_lm_break(struct file_lease *fl) +{ + /* + * Enforce break lease timeout to prevent NFSD + * thread from hanging in __break_lease. + */ + nfsd4_recall_file_layout(fl->c.flc_owner); + return false; +} + +static int +nfsd4_layout_lm_change(struct file_lease *onlist, int arg, + struct list_head *dispose) +{ + BUG_ON(!(arg & F_UNLCK)); + return lease_modify(onlist, arg, dispose); +} + +/** + * nfsd4_layout_lm_open_conflict - see if the given file points to an inode that has + * an existing open that would conflict with the + * desired lease. + * @filp: file to check + * @arg: type of lease that we're trying to acquire + * + * The kernel will call into this operation to determine whether there + * are conflicting opens that may prevent the layout from being granted. + * For nfsd, that check is done at a higher level, so this trivially + * returns 0. + */ +static int +nfsd4_layout_lm_open_conflict(struct file *filp, int arg) +{ + return 0; +} + +static void +nfsd4_layout_fence_worker(struct work_struct *work) +{ + struct delayed_work *dwork = to_delayed_work(work); + struct nfs4_layout_stateid *ls = container_of(dwork, + struct nfs4_layout_stateid, ls_fence_work); + struct nfsd_file *nf; + struct block_device *bdev; + struct nfs4_client *clp; + struct nfsd_net *nn; + + spin_lock(&ls->ls_lock); + if (list_empty(&ls->ls_layouts)) { + spin_unlock(&ls->ls_lock); +dispose: + cancel_delayed_work(&ls->ls_fence_work); + /* unlock the lease so that tasks waiting on it can proceed */ + nfsd4_close_layout(ls); + + ls->ls_fenced = true; + spin_lock(&ls->ls_lock); + ls->ls_fence_inflight = false; + spin_unlock(&ls->ls_lock); + nfs4_put_stid(&ls->ls_stid); + return; + } + spin_unlock(&ls->ls_lock); + + rcu_read_lock(); + nf = nfsd_file_get(ls->ls_file); + rcu_read_unlock(); + if (!nf) + goto dispose; + + clp = ls->ls_stid.sc_client; + nn = net_generic(clp->net, nfsd_net_id); + bdev = nf->nf_file->f_path.mnt->mnt_sb->s_bdev; + if (nfsd4_layout_ops[ls->ls_layout_type]->fence_client(ls, nf)) { + /* fenced ok */ + nfsd_file_put(nf); + pr_warn("%s: FENCED client[%pISpc] clid[%d] to device[%s]\n", + __func__, (struct sockaddr *)&clp->cl_addr, + clp->cl_clientid.cl_id - nn->clientid_base, + bdev->bd_disk->disk_name); + goto dispose; + } + /* fence failed */ + nfsd_file_put(nf); + + if (!clp->cl_fence_retry_warn) { + pr_warn("%s: FENCE failed client[%pISpc] clid[%d] device[%s]\n", + __func__, (struct sockaddr *)&clp->cl_addr, + clp->cl_clientid.cl_id - nn->clientid_base, + bdev->bd_disk->disk_name); + clp->cl_fence_retry_warn = true; + } + /* + * The fence worker retries the fencing operation indefinitely to + * prevent data corruption. The admin needs to take the following + * actions to restore access to the file for other clients: + * + * . shutdown or power off the client being fenced. + * . manually expire the client to release all its state on the server; + * echo 'expire' > /proc/fs/nfsd/clients/clid/ctl'. + * + * Where: + * + * clid: is the unique client identifier displayed in + * the warning message above. + */ + if (!ls->ls_fence_delay) + ls->ls_fence_delay = HZ; + else + ls->ls_fence_delay = min(ls->ls_fence_delay << 1, + MAX_FENCE_DELAY); + mod_delayed_work(system_dfl_wq, &ls->ls_fence_work, ls->ls_fence_delay); +} + +/** + * nfsd4_layout_lm_breaker_timedout - The layout recall has timed out. + * @fl: file to check + * + * If the layout type supports a fence operation, schedule a worker to + * fence the client from accessing the block device. + * + * This function runs under the protection of the spin_lock flc_lock. + * At this time, the file_lease associated with the layout stateid is + * on the flc_list. A reference count is incremented on the layout + * stateid to prevent it from being freed while the fence worker is + * executing. Once the fence worker finishes its operation, it releases + * this reference. + * + * The fence worker continues to run until either the client has been + * fenced or the layout becomes invalid. The layout can become invalid + * as a result of a LAYOUTRETURN or when the CB_LAYOUT recall callback + * has completed. + * + * Return true if the file_lease should be disposed of by the caller; + * otherwise, return false. + */ +static bool +nfsd4_layout_lm_breaker_timedout(struct file_lease *fl) +{ + struct nfs4_layout_stateid *ls = fl->c.flc_owner; + + if ((!nfsd4_layout_ops[ls->ls_layout_type]->fence_client) || + ls->ls_fenced) + return true; + /* + * Make sure layout has not been returned yet before + * taking a reference count on the layout stateid. The + * ls_fence_inflight flag is set together with the sc_count + * increment under ls_lock so that a fence worker invocation + * already in progress (which has cleared WORK_STRUCT_PENDING + * but not yet reached dispose:) cannot be coalesced with a + * fresh schedule that takes an extra unmatched reference. + */ + spin_lock(&ls->ls_lock); + if (ls->ls_fence_inflight) { + spin_unlock(&ls->ls_lock); + return false; + } + if (list_empty(&ls->ls_layouts) || + !refcount_inc_not_zero(&ls->ls_stid.sc_count)) { + spin_unlock(&ls->ls_lock); + return true; + } + ls->ls_fence_inflight = true; + spin_unlock(&ls->ls_lock); + + mod_delayed_work(system_dfl_wq, &ls->ls_fence_work, 0); + return false; +} + +static const struct lease_manager_operations nfsd4_layouts_lm_ops = { + .lm_break = nfsd4_layout_lm_break, + .lm_change = nfsd4_layout_lm_change, + .lm_open_conflict = nfsd4_layout_lm_open_conflict, + .lm_breaker_timedout = nfsd4_layout_lm_breaker_timedout, +}; + +int +nfsd4_init_pnfs(void) +{ + int i; + + for (i = 0; i < DEVID_HASH_SIZE; i++) + INIT_LIST_HEAD(&nfsd_devid_hash[i]); + + nfs4_layout_cache = KMEM_CACHE(nfs4_layout, 0); + if (!nfs4_layout_cache) + return -ENOMEM; + + nfs4_layout_stateid_cache = KMEM_CACHE(nfs4_layout_stateid, 0); + if (!nfs4_layout_stateid_cache) { + kmem_cache_destroy(nfs4_layout_cache); + return -ENOMEM; + } + return 0; +} + +void +nfsd4_exit_pnfs(void) +{ + int i; + + kmem_cache_destroy(nfs4_layout_cache); + kmem_cache_destroy(nfs4_layout_stateid_cache); + + for (i = 0; i < DEVID_HASH_SIZE; i++) { + struct nfsd4_deviceid_map *map, *n; + + list_for_each_entry_safe(map, n, &nfsd_devid_hash[i], hash) + kfree(map); + } +} diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c new file mode 100644 index 000000000..50c07561e --- /dev/null +++ b/fs/nfsd/nfs4proc.c @@ -0,0 +1,4273 @@ +/* + * Server-side procedures for NFSv4. + * + * Copyright (c) 2002 The Regents of the University of Michigan. + * All rights reserved. + * + * Kendrick Smith + * Andy Adamson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "attr4.h" +#include "idmap.h" +#include "cache.h" +#include "xdr4.h" +#include "vfs.h" +#include "current_stateid.h" +#include "netns.h" +#include "acl.h" +#include "pnfs.h" +#include "trace.h" + +static bool inter_copy_offload_enable; +module_param(inter_copy_offload_enable, bool, 0644); +MODULE_PARM_DESC(inter_copy_offload_enable, + "Enable inter server to server copy offload. Default: false"); + +static void cleanup_async_copy(struct nfsd4_async_copy *copy); + +#ifdef CONFIG_NFSD_V4_2_INTER_SSC +static int nfsd4_ssc_umount_timeout = 900000; /* default to 15 mins */ +module_param(nfsd4_ssc_umount_timeout, int, 0644); +MODULE_PARM_DESC(nfsd4_ssc_umount_timeout, + "idle msecs before unmount export from source server"); +#endif + +#define NFSDDBG_FACILITY NFSDDBG_PROC + +static u32 nfsd_attrmask[] = { + NFSD_WRITEABLE_ATTRS_WORD0, + NFSD_WRITEABLE_ATTRS_WORD1, + NFSD_WRITEABLE_ATTRS_WORD2 +}; + +static u32 nfsd41_ex_attrmask[] = { + NFSD_SUPPATTR_EXCLCREAT_WORD0, + NFSD_SUPPATTR_EXCLCREAT_WORD1, + NFSD_SUPPATTR_EXCLCREAT_WORD2 +}; + +static __be32 +check_attr_support(struct nfsd4_compound_state *cstate, u32 *bmval, + u32 *writable) +{ + struct dentry *dentry = cstate->current_fh.fh_dentry; + struct svc_export *exp = cstate->current_fh.fh_export; + + if (!nfsd_attrs_supported(cstate->minorversion, bmval)) + return nfserr_attrnotsupp; + if ((bmval[0] & FATTR4_WORD0_ACL) && !IS_POSIXACL(d_inode(dentry))) + return nfserr_attrnotsupp; + if ((bmval[2] & (FATTR4_WORD2_POSIX_DEFAULT_ACL | + FATTR4_WORD2_POSIX_ACCESS_ACL)) && + !IS_POSIXACL(d_inode(dentry))) + return nfserr_attrnotsupp; + if ((bmval[2] & FATTR4_WORD2_SECURITY_LABEL) && + !(exp->ex_flags & NFSEXP_SECURITY_LABEL)) + return nfserr_attrnotsupp; + if (writable && !bmval_is_subset(bmval, writable)) + return nfserr_inval; + if (writable && (bmval[2] & FATTR4_WORD2_MODE_UMASK) && + (bmval[1] & FATTR4_WORD1_MODE)) + return nfserr_inval; + return nfs_ok; +} + +static __be32 +nfsd4_check_open_attributes(struct nfsd4_compound_state *cstate, + struct nfsd4_open *open) +{ + __be32 status = nfs_ok; + + if (open->op_create != NFS4_OPEN_CREATE) + return status; + + switch (open->op_createmode) { + case NFS4_CREATE_UNCHECKED: + case NFS4_CREATE_GUARDED: + status = check_attr_support(cstate, open->op_bmval, + nfsd_attrmask); + break; + case NFS4_CREATE_EXCLUSIVE4_1: + status = check_attr_support(cstate, open->op_bmval, + nfsd41_ex_attrmask); + break; + } + return status; +} + +static int +is_create_with_attrs(struct nfsd4_open *open) +{ + return open->op_create == NFS4_OPEN_CREATE + && (open->op_createmode == NFS4_CREATE_UNCHECKED + || open->op_createmode == NFS4_CREATE_GUARDED + || open->op_createmode == NFS4_CREATE_EXCLUSIVE4_1); +} + +static inline void +fh_dup2(struct svc_fh *dst, struct svc_fh *src) +{ + fh_put(dst); + dget(src->fh_dentry); + if (src->fh_export) + exp_get(src->fh_export); + *dst = *src; +} + +static __be32 +do_open_permission(struct svc_rqst *rqstp, struct svc_fh *current_fh, struct nfsd4_open *open, int accmode) +{ + + if (open->op_truncate && + !(open->op_share_access & NFS4_SHARE_ACCESS_WRITE)) + return nfserr_inval; + + accmode |= NFSD_MAY_READ_IF_EXEC; + + if (open->op_share_access & NFS4_SHARE_ACCESS_READ) + accmode |= NFSD_MAY_READ; + if (open->op_share_access & NFS4_SHARE_ACCESS_WRITE) + accmode |= (NFSD_MAY_WRITE | NFSD_MAY_TRUNC); + if (open->op_share_deny & NFS4_SHARE_DENY_READ) + accmode |= NFSD_MAY_WRITE; + + return fh_verify(rqstp, current_fh, S_IFREG, accmode); +} + +static __be32 nfsd_check_obj_isreg(struct svc_fh *fh, u32 minor_version) +{ + umode_t mode = d_inode(fh->fh_dentry)->i_mode; + + if (S_ISREG(mode)) + return nfs_ok; + if (S_ISDIR(mode)) + return nfserr_isdir; + if (S_ISLNK(mode)) + return nfserr_symlink; + + /* RFC 7530 - 16.16.6 */ + if (minor_version == 0) + return nfserr_symlink; + else + return nfserr_wrong_type; + +} + +static void nfsd4_set_open_owner_reply_cache(struct nfsd4_compound_state *cstate, struct nfsd4_open *open, struct svc_fh *resfh) +{ + if (nfsd4_has_session(cstate)) + return; + fh_copy_shallow(&open->op_openowner->oo_owner.so_replay.rp_openfh, + &resfh->fh_handle); +} + +static inline bool nfsd4_create_is_exclusive(int createmode) +{ + return createmode == NFS4_CREATE_EXCLUSIVE || + createmode == NFS4_CREATE_EXCLUSIVE4_1; +} + +static __be32 +nfsd4_vfs_create(struct svc_fh *fhp, struct dentry **child, + struct nfsd4_open *open) +{ + struct file *filp; + struct path path; + int oflags; + + oflags = O_CREAT | O_LARGEFILE; + if (nfsd4_create_is_exclusive(open->op_createmode)) + oflags |= O_EXCL; + + switch (open->op_share_access & NFS4_SHARE_ACCESS_BOTH) { + case NFS4_SHARE_ACCESS_WRITE: + oflags |= O_WRONLY; + break; + case NFS4_SHARE_ACCESS_BOTH: + oflags |= O_RDWR; + break; + default: + oflags |= O_RDONLY; + } + + path.mnt = fhp->fh_export->ex_path.mnt; + path.dentry = *child; + filp = dentry_create(&path, oflags, open->op_iattr.ia_mode, + current_cred()); + *child = path.dentry; + + if (IS_ERR(filp)) + return nfserrno(PTR_ERR(filp)); + + open->op_filp = filp; + return nfs_ok; +} + +/* + * Implement NFSv4's unchecked, guarded, and exclusive create + * semantics for regular files. Open state for this new file is + * subsequently fabricated in nfsd4_process_open2(). + * + * Upon return, caller must release @fhp and @resfhp. + */ +static __be32 +nfsd4_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp, + struct svc_fh *resfhp, struct nfsd4_open *open) +{ + struct iattr *iap = &open->op_iattr; + struct nfsd_attrs attrs = { + .na_iattr = iap, + .na_seclabel = &open->op_label, + }; + struct dentry *parent, *child = ERR_PTR(-EINVAL); + __u32 v_mtime, v_atime; + struct inode *inode; + __be32 status; + int host_err; + + if (name_is_dot_dotdot(open->op_fname, open->op_fnamelen)) + return nfserr_exist; + if (!(iap->ia_valid & ATTR_MODE)) + iap->ia_mode = 0; + + status = fh_verify(rqstp, fhp, S_IFDIR, NFSD_MAY_EXEC); + if (status != nfs_ok) + return status; + parent = fhp->fh_dentry; + inode = d_inode(parent); + + host_err = fh_want_write(fhp); + if (host_err) + return nfserrno(host_err); + + if (open->op_acl) { + if (open->op_dpacl || open->op_pacl) { + status = nfserr_inval; + goto out; + } + if (is_create_with_attrs(open)) { + status = nfsd4_acl_to_attr(NF4REG, open->op_acl, + &attrs); + if (status) + goto out; + } + } else if (is_create_with_attrs(open)) { + /* The dpacl and pacl will get released by nfsd_attrs_free(). */ + attrs.na_dpacl = open->op_dpacl; + attrs.na_pacl = open->op_pacl; + open->op_dpacl = NULL; + open->op_pacl = NULL; + } + + child = start_creating(&nop_mnt_idmap, parent, + &QSTR_LEN(open->op_fname, open->op_fnamelen)); + if (IS_ERR(child)) { + status = nfserrno(PTR_ERR(child)); + goto out; + } + + if (d_really_is_negative(child)) { + status = fh_verify(rqstp, fhp, S_IFDIR, NFSD_MAY_CREATE); + if (status != nfs_ok) + goto out; + } + + v_mtime = 0; + v_atime = 0; + if (nfsd4_create_is_exclusive(open->op_createmode)) { + u32 *verifier = (u32 *)open->op_verf.data; + + /* + * Solaris 7 gets confused (bugid 4218508) if these have + * the high bit set, as do xfs filesystems without the + * "bigtime" feature. So just clear the high bits. If this + * is ever changed to use different attrs for storing the + * verifier, then do_open_lookup() will also need to be + * fixed accordingly. + */ + v_mtime = verifier[0] & 0x7fffffff; + v_atime = verifier[1] & 0x7fffffff; + } + + if (d_really_is_positive(child)) { + /* NFSv4 protocol requires change attributes even though + * no change happened. + */ + status = fh_fill_both_attrs(fhp); + if (status != nfs_ok) + goto out; + + status = fh_compose(resfhp, fhp->fh_export, child, fhp); + if (status != nfs_ok) + goto out; + + switch (open->op_createmode) { + case NFS4_CREATE_UNCHECKED: + if (!d_is_reg(child)) + break; + + /* + * In NFSv4, we don't want to truncate the file + * now. This would be wrong if the OPEN fails for + * some other reason. Furthermore, if the size is + * nonzero, we should ignore it according to spec! + */ + open->op_truncate = (iap->ia_valid & ATTR_SIZE) && + !iap->ia_size; + break; + case NFS4_CREATE_GUARDED: + status = nfserr_exist; + break; + case NFS4_CREATE_EXCLUSIVE: + if (inode_get_mtime_sec(d_inode(child)) == v_mtime && + inode_get_atime_sec(d_inode(child)) == v_atime && + d_inode(child)->i_size == 0) { + open->op_created = true; + break; /* subtle */ + } + status = nfserr_exist; + break; + case NFS4_CREATE_EXCLUSIVE4_1: + if (inode_get_mtime_sec(d_inode(child)) == v_mtime && + inode_get_atime_sec(d_inode(child)) == v_atime && + d_inode(child)->i_size == 0) { + open->op_created = true; + goto set_attr; /* subtle */ + } + status = nfserr_exist; + } + goto out; + } + + if (!IS_POSIXACL(inode)) + iap->ia_mode &= ~current_umask(); + + status = fh_fill_pre_attrs(fhp); + if (status != nfs_ok) + goto out; + status = nfsd4_vfs_create(fhp, &child, open); + if (status != nfs_ok) + goto out; + open->op_created = true; + fh_fill_post_attrs(fhp); + + status = fh_compose(resfhp, fhp->fh_export, child, fhp); + if (status != nfs_ok) + goto out; + + /* A newly created file already has a file size of zero. */ + if ((iap->ia_valid & ATTR_SIZE) && (iap->ia_size == 0)) + iap->ia_valid &= ~ATTR_SIZE; + if (nfsd4_create_is_exclusive(open->op_createmode)) { + iap->ia_valid = ATTR_MTIME | ATTR_ATIME | + ATTR_MTIME_SET|ATTR_ATIME_SET; + iap->ia_mtime.tv_sec = v_mtime; + iap->ia_atime.tv_sec = v_atime; + iap->ia_mtime.tv_nsec = 0; + iap->ia_atime.tv_nsec = 0; + } + +set_attr: + status = nfsd_create_setattr(rqstp, fhp, resfhp, &attrs); + + if (attrs.na_labelerr) + open->op_bmval[2] &= ~FATTR4_WORD2_SECURITY_LABEL; + if (attrs.na_paclerr || attrs.na_dpaclerr) + open->op_bmval[0] &= ~FATTR4_WORD0_ACL; + if (attrs.na_dpaclerr) + open->op_bmval[2] &= ~FATTR4_WORD2_POSIX_DEFAULT_ACL; + if (attrs.na_paclerr) + open->op_bmval[2] &= ~FATTR4_WORD2_POSIX_ACCESS_ACL; +out: + end_creating(child); + nfsd_attrs_free(&attrs); + fh_drop_write(fhp); + return status; +} + +/** + * set_change_info - set up the change_info4 for a reply + * @cinfo: pointer to nfsd4_change_info to be populated + * @fhp: pointer to svc_fh to use as source + * + * Many operations in NFSv4 require change_info4 in the reply. This function + * populates that from the info that we (should!) have already collected. In + * the event that we didn't get any pre-attrs, just zero out both. + */ +static void +set_change_info(struct nfsd4_change_info *cinfo, struct svc_fh *fhp) +{ + cinfo->atomic = (u32)(fhp->fh_pre_saved && fhp->fh_post_saved && !fhp->fh_no_atomic_attr); + cinfo->before_change = fhp->fh_pre_change; + cinfo->after_change = fhp->fh_post_change; + + /* + * If fetching the pre-change attributes failed, then we should + * have already failed the whole operation. We could have still + * failed to fetch post-change attributes however. + * + * If we didn't get post-op attrs, just zero-out the after + * field since we don't know what it should be. If the pre_saved + * field isn't set for some reason, throw warning and just copy + * whatever is in the after field. + */ + if (WARN_ON_ONCE(!fhp->fh_pre_saved)) + cinfo->before_change = 0; + if (!fhp->fh_post_saved) + cinfo->after_change = cinfo->before_change + 1; +} + +static __be32 +do_open_lookup(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, struct nfsd4_open *open, struct svc_fh **resfh) +{ + struct svc_fh *current_fh = &cstate->current_fh; + int accmode; + __be32 status; + + *resfh = kmalloc_obj(struct svc_fh); + if (!*resfh) + return nfserr_jukebox; + fh_init(*resfh, NFS4_FHSIZE); + open->op_truncate = false; + + if (open->op_create) { + /* FIXME: check session persistence and pnfs flags. + * The nfsv4.1 spec requires the following semantics: + * + * Persistent | pNFS | Server REQUIRED | Client Allowed + * Reply Cache | server | | + * -------------+--------+-----------------+-------------------- + * no | no | EXCLUSIVE4_1 | EXCLUSIVE4_1 + * | | | (SHOULD) + * | | and EXCLUSIVE4 | or EXCLUSIVE4 + * | | | (SHOULD NOT) + * no | yes | EXCLUSIVE4_1 | EXCLUSIVE4_1 + * yes | no | GUARDED4 | GUARDED4 + * yes | yes | GUARDED4 | GUARDED4 + */ + + current->fs->umask = open->op_umask; + status = nfsd4_create_file(rqstp, current_fh, *resfh, open); + current->fs->umask = 0; + + /* + * Following rfc 3530 14.2.16, and rfc 5661 18.16.4 + * use the returned bitmask to indicate which attributes + * we used to store the verifier: + */ + if (nfsd4_create_is_exclusive(open->op_createmode) && status == 0) + open->op_bmval[1] |= (FATTR4_WORD1_TIME_ACCESS | + FATTR4_WORD1_TIME_MODIFY); + } else { + status = nfsd_lookup(rqstp, current_fh, + open->op_fname, open->op_fnamelen, *resfh); + if (status == nfs_ok) + /* NFSv4 protocol requires change attributes even though + * no change happened. + */ + status = fh_fill_both_attrs(current_fh); + } + if (status) + goto out; + status = nfsd_check_obj_isreg(*resfh, cstate->minorversion); + if (status) + goto out; + + nfsd4_set_open_owner_reply_cache(cstate, open, *resfh); + accmode = NFSD_MAY_NOP; + if (open->op_created || + open->op_claim_type == NFS4_OPEN_CLAIM_DELEGATE_CUR) + accmode |= NFSD_MAY_OWNER_OVERRIDE; + status = do_open_permission(rqstp, *resfh, open, accmode); + set_change_info(&open->op_cinfo, current_fh); +out: + return status; +} + +static __be32 +do_open_fhandle(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, struct nfsd4_open *open) +{ + struct svc_fh *current_fh = &cstate->current_fh; + int accmode = 0; + + /* We don't know the target directory, and therefore can not + * set the change info + */ + + memset(&open->op_cinfo, 0, sizeof(struct nfsd4_change_info)); + + nfsd4_set_open_owner_reply_cache(cstate, open, current_fh); + + open->op_truncate = (open->op_iattr.ia_valid & ATTR_SIZE) && + (open->op_iattr.ia_size == 0); + /* + * In the delegation case, the client is telling us about an + * open that it *already* performed locally, some time ago. We + * should let it succeed now if possible. + * + * In the case of a CLAIM_FH open, on the other hand, the client + * may be counting on us to enforce permissions (the Linux 4.1 + * client uses this for normal opens, for example). + */ + if (open->op_claim_type == NFS4_OPEN_CLAIM_DELEG_CUR_FH) + accmode = NFSD_MAY_OWNER_OVERRIDE; + + return do_open_permission(rqstp, current_fh, open, accmode); +} + +static void +copy_clientid(clientid_t *clid, struct nfsd4_session *session) +{ + struct nfsd4_sessionid *sid = + (struct nfsd4_sessionid *)session->se_sessionid.data; + + clid->cl_boot = sid->clientid.cl_boot; + clid->cl_id = sid->clientid.cl_id; +} + +static __be32 +nfsd4_open(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_open *open = &u->open; + __be32 status; + struct svc_fh *resfh = NULL; + struct net *net = SVC_NET(rqstp); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + bool reclaim = false; + + dprintk("NFSD: nfsd4_open filename %.*s op_openowner %p\n", + (int)open->op_fnamelen, open->op_fname, + open->op_openowner); + + open->op_filp = NULL; + open->op_rqstp = rqstp; + + /* This check required by spec. */ + if (open->op_create && open->op_claim_type != NFS4_OPEN_CLAIM_NULL) { + status = nfserr_inval; + goto out_err; + } + + open->op_created = false; + /* + * RFC5661 18.51.3 + * Before RECLAIM_COMPLETE done, server should deny new lock + */ + if (nfsd4_has_session(cstate) && + !test_bit(NFSD4_CLIENT_RECLAIM_COMPLETE, &cstate->clp->cl_flags) && + open->op_claim_type != NFS4_OPEN_CLAIM_PREVIOUS) { + status = nfserr_grace; + goto out_err; + } + + if (nfsd4_has_session(cstate)) + copy_clientid(&open->op_clientid, cstate->session); + + /* check seqid for replay. set nfs4_owner */ + status = nfsd4_process_open1(cstate, open, nn); + if (status == nfserr_replay_me) { + struct nfs4_replay *rp = &open->op_openowner->oo_owner.so_replay; + fh_put(&cstate->current_fh); + fh_copy_shallow(&cstate->current_fh.fh_handle, + &rp->rp_openfh); + status = fh_verify(rqstp, &cstate->current_fh, 0, NFSD_MAY_NOP); + if (status) + dprintk("nfsd4_open: replay failed" + " restoring previous filehandle\n"); + else + status = nfserr_replay_me; + } + if (status) + goto out; + if (open->op_xdr_error) { + status = open->op_xdr_error; + goto out; + } + + status = nfsd4_check_open_attributes(cstate, open); + if (status) + goto out; + + /* Openowner is now set, so sequence id will get bumped. Now we need + * these checks before we do any creates: */ + status = nfserr_grace; + if (opens_in_grace(net) && open->op_claim_type != NFS4_OPEN_CLAIM_PREVIOUS) + goto out; + status = nfserr_no_grace; + if (!opens_in_grace(net) && open->op_claim_type == NFS4_OPEN_CLAIM_PREVIOUS) + goto out; + + switch (open->op_claim_type) { + case NFS4_OPEN_CLAIM_DELEGATE_CUR: + case NFS4_OPEN_CLAIM_NULL: + status = do_open_lookup(rqstp, cstate, open, &resfh); + if (status) + goto out; + break; + case NFS4_OPEN_CLAIM_PREVIOUS: + status = nfs4_check_open_reclaim(cstate->clp); + if (status) + goto out; + open->op_openowner->oo_flags |= NFS4_OO_CONFIRMED; + reclaim = true; + fallthrough; + case NFS4_OPEN_CLAIM_FH: + case NFS4_OPEN_CLAIM_DELEG_CUR_FH: + status = do_open_fhandle(rqstp, cstate, open); + if (status) + goto out; + resfh = &cstate->current_fh; + break; + case NFS4_OPEN_CLAIM_DELEG_PREV_FH: + case NFS4_OPEN_CLAIM_DELEGATE_PREV: + status = nfserr_notsupp; + goto out; + default: + status = nfserr_inval; + goto out; + } + + status = nfsd4_process_open2(rqstp, resfh, open); + if (status && open->op_created) + pr_warn("nfsd4_process_open2 failed to open newly-created file: status=%u\n", + be32_to_cpu(status)); + if (reclaim && !status) + set_bit(NFSD_NET_SOMEBODY_RECLAIMED, &nn->flags); +out: + if (open->op_filp) { + fput(open->op_filp); + open->op_filp = NULL; + } + if (resfh && resfh != &cstate->current_fh) { + fh_dup2(&cstate->current_fh, resfh); + fh_put(resfh); + kfree(resfh); + } + nfsd4_cleanup_open_state(cstate, open); + nfsd4_bump_seqid(cstate, status); +out_err: + return status; +} + +/* + * OPEN is the only seqid-mutating operation whose decoding can fail + * with a seqid-mutating error (specifically, decoding of user names in + * the attributes). Therefore we have to do some processing to look up + * the stateowner so that we can bump the seqid. + */ +static __be32 nfsd4_open_omfg(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, struct nfsd4_op *op) +{ + struct nfsd4_open *open = &op->u.open; + + if (!seqid_mutating_err(ntohl(op->status))) + return op->status; + if (nfsd4_has_session(cstate)) + return op->status; + open->op_xdr_error = op->status; + return nfsd4_open(rqstp, cstate, &op->u); +} + +static void +nfsd4_open_release(union nfsd4_op_u *u) +{ + posix_acl_release(u->open.op_dpacl); + posix_acl_release(u->open.op_pacl); +} + +/* + * filehandle-manipulating ops. + */ +static __be32 +nfsd4_getfh(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + u->getfh = &cstate->current_fh; + return nfs_ok; +} + +static __be32 +nfsd4_putfh(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_putfh *putfh = &u->putfh; + __be32 ret; + + fh_put(&cstate->current_fh); + cstate->current_fh.fh_handle.fh_size = putfh->pf_fhlen; + memcpy(&cstate->current_fh.fh_handle.fh_raw, putfh->pf_fhval, + putfh->pf_fhlen); + ret = fh_verify(rqstp, &cstate->current_fh, 0, NFSD_MAY_BYPASS_GSS); +#ifdef CONFIG_NFSD_V4_2_INTER_SSC + if (ret == nfserr_stale && putfh->no_verify) { + SET_FH_FLAG(&cstate->current_fh, NFSD4_FH_FOREIGN); + ret = 0; + } +#endif + return ret; +} + +static __be32 +nfsd4_putrootfh(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + fh_put(&cstate->current_fh); + + return exp_pseudoroot(rqstp, &cstate->current_fh); +} + +static __be32 +nfsd4_restorefh(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + if (!cstate->save_fh.fh_dentry) + return nfserr_restorefh; + + fh_dup2(&cstate->current_fh, &cstate->save_fh); + if (HAS_CSTATE_FLAG(cstate, SAVED_STATE_ID_FLAG)) { + memcpy(&cstate->current_stateid, &cstate->save_stateid, sizeof(stateid_t)); + SET_CSTATE_FLAG(cstate, CURRENT_STATE_ID_FLAG); + } + return nfs_ok; +} + +static __be32 +nfsd4_savefh(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + fh_dup2(&cstate->save_fh, &cstate->current_fh); + if (HAS_CSTATE_FLAG(cstate, CURRENT_STATE_ID_FLAG)) { + memcpy(&cstate->save_stateid, &cstate->current_stateid, sizeof(stateid_t)); + SET_CSTATE_FLAG(cstate, SAVED_STATE_ID_FLAG); + } + return nfs_ok; +} + +/* + * misc nfsv4 ops + */ +static __be32 +nfsd4_access(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_access *access = &u->access; + u32 access_full; + + access_full = NFS3_ACCESS_FULL; + if (cstate->minorversion >= 2) + access_full |= NFS4_ACCESS_XALIST | NFS4_ACCESS_XAREAD | + NFS4_ACCESS_XAWRITE; + + if (access->ac_req_access & ~access_full) + return nfserr_inval; + + access->ac_resp_access = access->ac_req_access; + return nfsd_access(rqstp, &cstate->current_fh, &access->ac_resp_access, + &access->ac_supported); +} + +static __be32 +nfsd4_commit(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_commit *commit = &u->commit; + struct nfsd_file *nf; + __be32 status; + + status = nfsd_file_acquire(rqstp, &cstate->current_fh, NFSD_MAY_WRITE | + NFSD_MAY_NOT_BREAK_LEASE, &nf); + if (status != nfs_ok) + return status; + + status = nfsd_commit(rqstp, &cstate->current_fh, nf, commit->co_offset, + commit->co_count, + (__be32 *)commit->co_verf.data); + nfsd_file_put(nf); + return status; +} + +static __be32 +nfsd4_create(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_create *create = &u->create; + struct nfsd_attrs attrs = { + .na_iattr = &create->cr_iattr, + .na_seclabel = &create->cr_label, + .na_dpacl = create->cr_dpacl, + .na_pacl = create->cr_pacl, + }; + struct svc_fh resfh; + __be32 status; + dev_t rdev; + + create->cr_dpacl = NULL; + create->cr_pacl = NULL; + + fh_init(&resfh, NFS4_FHSIZE); + + status = fh_verify(rqstp, &cstate->current_fh, S_IFDIR, NFSD_MAY_NOP); + if (status) + goto out_aftermask; + + status = check_attr_support(cstate, create->cr_bmval, nfsd_attrmask); + if (status) + goto out_aftermask; + + /* Sanitize cr_type to avoid returning ATTRNOTSUPP. */ + switch (create->cr_type) { + case NF4LNK: + case NF4BLK: + case NF4CHR: + case NF4SOCK: + case NF4FIFO: + case NF4DIR: + break; + default: + status = nfserr_badtype; + goto out_aftermask; + } + + if (create->cr_acl) { + if (attrs.na_dpacl || attrs.na_pacl) { + status = nfserr_inval; + goto out_aftermask; + } + status = nfsd4_acl_to_attr(create->cr_type, create->cr_acl, + &attrs); + if (status != nfs_ok) + goto out_aftermask; + } + current->fs->umask = create->cr_umask; + switch (create->cr_type) { + case NF4LNK: + status = nfsd_symlink(rqstp, &cstate->current_fh, + create->cr_name, create->cr_namelen, + create->cr_data, &attrs, &resfh); + break; + + case NF4BLK: + status = nfserr_inval; + rdev = MKDEV(create->cr_specdata1, create->cr_specdata2); + if (MAJOR(rdev) != create->cr_specdata1 || + MINOR(rdev) != create->cr_specdata2) + goto out_umask; + status = nfsd_create(rqstp, &cstate->current_fh, + create->cr_name, create->cr_namelen, + &attrs, S_IFBLK, rdev, &resfh); + break; + + case NF4CHR: + status = nfserr_inval; + rdev = MKDEV(create->cr_specdata1, create->cr_specdata2); + if (MAJOR(rdev) != create->cr_specdata1 || + MINOR(rdev) != create->cr_specdata2) + goto out_umask; + status = nfsd_create(rqstp, &cstate->current_fh, + create->cr_name, create->cr_namelen, + &attrs, S_IFCHR, rdev, &resfh); + break; + + case NF4SOCK: + status = nfsd_create(rqstp, &cstate->current_fh, + create->cr_name, create->cr_namelen, + &attrs, S_IFSOCK, 0, &resfh); + break; + + case NF4FIFO: + status = nfsd_create(rqstp, &cstate->current_fh, + create->cr_name, create->cr_namelen, + &attrs, S_IFIFO, 0, &resfh); + break; + + case NF4DIR: + create->cr_iattr.ia_valid &= ~ATTR_SIZE; + status = nfsd_create(rqstp, &cstate->current_fh, + create->cr_name, create->cr_namelen, + &attrs, S_IFDIR, 0, &resfh); + break; + + default: + status = nfserr_badtype; + } + + if (status) + goto out; + + if (attrs.na_labelerr) + create->cr_bmval[2] &= ~FATTR4_WORD2_SECURITY_LABEL; + if (attrs.na_paclerr || attrs.na_dpaclerr) + create->cr_bmval[0] &= ~FATTR4_WORD0_ACL; + if (attrs.na_dpaclerr) + create->cr_bmval[2] &= ~FATTR4_WORD2_POSIX_DEFAULT_ACL; + if (attrs.na_paclerr) + create->cr_bmval[2] &= ~FATTR4_WORD2_POSIX_ACCESS_ACL; + set_change_info(&create->cr_cinfo, &cstate->current_fh); + fh_dup2(&cstate->current_fh, &resfh); +out: + fh_put(&resfh); +out_umask: + current->fs->umask = 0; +out_aftermask: + nfsd_attrs_free(&attrs); + return status; +} + +static __be32 +nfsd4_getattr(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_getattr *getattr = &u->getattr; + __be32 status; + + trace_nfsd_vfs_getattr(rqstp, &cstate->current_fh); + + status = fh_verify(rqstp, &cstate->current_fh, 0, NFSD_MAY_NOP); + if (status) + return status; + + if (getattr->ga_bmval[1] & NFSD_WRITEONLY_ATTRS_WORD1) + return nfserr_inval; + + getattr->ga_bmval[0] &= nfsd_suppattrs[cstate->minorversion][0]; + getattr->ga_bmval[1] &= nfsd_suppattrs[cstate->minorversion][1]; + getattr->ga_bmval[2] &= nfsd_suppattrs[cstate->minorversion][2]; + + getattr->ga_fhp = &cstate->current_fh; + return nfs_ok; +} + +static __be32 +nfsd4_link(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_link *link = &u->link; + __be32 status; + + status = nfsd_link(rqstp, &cstate->current_fh, + link->li_name, link->li_namelen, &cstate->save_fh); + if (!status) + set_change_info(&link->li_cinfo, &cstate->current_fh); + return status; +} + +static __be32 nfsd4_do_lookupp(struct svc_rqst *rqstp, struct svc_fh *fh) +{ + struct svc_fh tmp_fh; + __be32 ret; + + fh_init(&tmp_fh, NFS4_FHSIZE); + ret = exp_pseudoroot(rqstp, &tmp_fh); + if (ret) + return ret; + if (tmp_fh.fh_dentry == fh->fh_dentry) { + fh_put(&tmp_fh); + return nfserr_noent; + } + fh_put(&tmp_fh); + return nfsd_lookup(rqstp, fh, "..", 2, fh); +} + +static __be32 +nfsd4_lookupp(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + return nfsd4_do_lookupp(rqstp, &cstate->current_fh); +} + +static __be32 +nfsd4_lookup(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + return nfsd_lookup(rqstp, &cstate->current_fh, + u->lookup.lo_name, u->lookup.lo_len, + &cstate->current_fh); +} + +static __be32 +nfsd4_read(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_read *read = &u->read; + __be32 status; + + read->rd_nf = NULL; + + trace_nfsd_read_start(rqstp, &cstate->current_fh, + read->rd_offset, read->rd_length); + + read->rd_length = min_t(u32, read->rd_length, svc_max_payload(rqstp)); + if (read->rd_offset > (u64)OFFSET_MAX) + read->rd_offset = (u64)OFFSET_MAX; + if (read->rd_offset + read->rd_length > (u64)OFFSET_MAX) + read->rd_length = (u64)OFFSET_MAX - read->rd_offset; + + /* + * If we do a zero copy read, then a client will see read data + * that reflects the state of the file *after* performing the + * following compound. + * + * To ensure proper ordering, we therefore turn off zero copy if + * the client wants us to do more in this compound: + */ + if (!nfsd4_last_compound_op(rqstp)) { + struct nfsd4_compoundargs *argp = rqstp->rq_argp; + + argp->splice_ok = false; + } + + /* check stateid */ + status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh, + &read->rd_stateid, RD_STATE, + &read->rd_nf, NULL); + + read->rd_rqstp = rqstp; + read->rd_fhp = &cstate->current_fh; + return status; +} + + +static void +nfsd4_read_release(union nfsd4_op_u *u) +{ + if (u->read.rd_nf) { + trace_nfsd_read_done(u->read.rd_rqstp, u->read.rd_fhp, + u->read.rd_offset, u->read.rd_length); + nfsd_file_put(u->read.rd_nf); + } +} + +static __be32 +nfsd4_readdir(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_readdir *readdir = &u->readdir; + u64 cookie = readdir->rd_cookie; + static const nfs4_verifier zeroverf; + + trace_nfsd_vfs_readdir(rqstp, &cstate->current_fh, + readdir->rd_maxcount, readdir->rd_cookie); + + /* no need to check permission - this will be done in nfsd_readdir() */ + + if (readdir->rd_bmval[1] & NFSD_WRITEONLY_ATTRS_WORD1) + return nfserr_inval; + + readdir->rd_bmval[0] &= nfsd_suppattrs[cstate->minorversion][0]; + readdir->rd_bmval[1] &= nfsd_suppattrs[cstate->minorversion][1]; + readdir->rd_bmval[2] &= nfsd_suppattrs[cstate->minorversion][2]; + + if ((cookie == 1) || (cookie == 2) || + (cookie == 0 && memcmp(readdir->rd_verf.data, zeroverf.data, NFS4_VERIFIER_SIZE))) + return nfserr_bad_cookie; + + readdir->rd_rqstp = rqstp; + readdir->rd_fhp = &cstate->current_fh; + return nfs_ok; +} + +static __be32 +nfsd4_readlink(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + u->readlink.rl_rqstp = rqstp; + u->readlink.rl_fhp = &cstate->current_fh; + return nfs_ok; +} + +static __be32 +nfsd4_remove(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_remove *remove = &u->remove; + __be32 status; + + if (opens_in_grace(SVC_NET(rqstp))) + return nfserr_grace; + status = nfsd_unlink(rqstp, &cstate->current_fh, 0, + remove->rm_name, remove->rm_namelen); + if (!status) + set_change_info(&remove->rm_cinfo, &cstate->current_fh); + return status; +} + +static __be32 +nfsd4_rename(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_rename *rename = &u->rename; + __be32 status; + + if (opens_in_grace(SVC_NET(rqstp))) + return nfserr_grace; + status = nfsd_rename(rqstp, &cstate->save_fh, rename->rn_sname, + rename->rn_snamelen, &cstate->current_fh, + rename->rn_tname, rename->rn_tnamelen); + if (status) + return status; + set_change_info(&rename->rn_sinfo, &cstate->save_fh); + set_change_info(&rename->rn_tinfo, &cstate->current_fh); + return nfs_ok; +} + +static __be32 +nfsd4_secinfo(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_secinfo *secinfo = &u->secinfo; + struct svc_export *exp; + struct dentry *dentry; + __be32 err; + + err = fh_verify(rqstp, &cstate->current_fh, S_IFDIR, NFSD_MAY_EXEC); + if (err) + return err; + err = nfsd_lookup_dentry(rqstp, &cstate->current_fh, + secinfo->si_name, secinfo->si_namelen, + &exp, &dentry); + if (err) + return err; + if (d_really_is_negative(dentry)) { + exp_put(exp); + err = nfserr_noent; + } else + secinfo->si_exp = exp; + dput(dentry); + if (cstate->minorversion) + /* See rfc 5661 section 2.6.3.1.1.8 */ + fh_put(&cstate->current_fh); + return err; +} + +static __be32 +nfsd4_secinfo_no_name(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + __be32 err; + + switch (u->secinfo_no_name.sin_style) { + case NFS4_SECINFO_STYLE4_CURRENT_FH: + break; + case NFS4_SECINFO_STYLE4_PARENT: + err = nfsd4_do_lookupp(rqstp, &cstate->current_fh); + if (err) + return err; + break; + default: + return nfserr_inval; + } + + u->secinfo_no_name.sin_exp = exp_get(cstate->current_fh.fh_export); + fh_put(&cstate->current_fh); + return nfs_ok; +} + +static void +nfsd4_secinfo_release(union nfsd4_op_u *u) +{ + if (u->secinfo.si_exp) + exp_put(u->secinfo.si_exp); +} + +static void +nfsd4_secinfo_no_name_release(union nfsd4_op_u *u) +{ + if (u->secinfo_no_name.sin_exp) + exp_put(u->secinfo_no_name.sin_exp); +} + +/* + * Validate that the requested timestamps are within the acceptable range. If + * timestamp appears to be in the future, then it will be clamped to + * current_time(). + */ +static void +vet_deleg_attrs(struct nfsd4_setattr *setattr, struct nfs4_delegation *dp) +{ + struct timespec64 now = current_time(dp->dl_stid.sc_file->fi_inode); + struct iattr *iattr = &setattr->sa_iattr; + + if ((setattr->sa_bmval[2] & FATTR4_WORD2_TIME_DELEG_ACCESS) && + !nfsd4_vet_deleg_time(&iattr->ia_atime, &dp->dl_atime, &now)) + iattr->ia_valid &= ~(ATTR_ATIME | ATTR_ATIME_SET); + + if (setattr->sa_bmval[2] & FATTR4_WORD2_TIME_DELEG_MODIFY) { + if (nfsd4_vet_deleg_time(&iattr->ia_mtime, &dp->dl_mtime, &now)) { + iattr->ia_ctime = iattr->ia_mtime; + if (nfsd4_vet_deleg_time(&iattr->ia_ctime, &dp->dl_ctime, &now)) + dp->dl_setattr = true; + else + iattr->ia_valid &= ~(ATTR_CTIME | ATTR_CTIME_SET); + } else { + iattr->ia_valid &= ~(ATTR_CTIME | ATTR_CTIME_SET | + ATTR_MTIME | ATTR_MTIME_SET); + } + } +} + +static __be32 +nfsd4_setattr(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_setattr *setattr = &u->setattr; + struct nfsd_attrs attrs = { + .na_iattr = &setattr->sa_iattr, + .na_seclabel = &setattr->sa_label, + .na_pacl = setattr->sa_pacl, + .na_dpacl = setattr->sa_dpacl, + }; + bool save_no_wcc, deleg_attrs; + struct nfs4_stid *st = NULL; + struct inode *inode; + __be32 status = nfs_ok; + int err; + + /* Transfer ownership to attrs for cleanup via nfsd_attrs_free() */ + setattr->sa_pacl = NULL; + setattr->sa_dpacl = NULL; + + deleg_attrs = setattr->sa_bmval[2] & (FATTR4_WORD2_TIME_DELEG_ACCESS | + FATTR4_WORD2_TIME_DELEG_MODIFY); + + if (deleg_attrs || (setattr->sa_iattr.ia_valid & ATTR_SIZE)) { + int flags = WR_STATE; + + if (setattr->sa_bmval[2] & FATTR4_WORD2_TIME_DELEG_ACCESS) + flags |= RD_STATE; + + status = nfs4_preprocess_stateid_op(rqstp, cstate, + &cstate->current_fh, &setattr->sa_stateid, + flags, NULL, &st); + if (status) + goto out_err; + } + + if (deleg_attrs) { + status = nfserr_bad_stateid; + if (st && (st->sc_type & SC_TYPE_DELEG)) { + struct nfs4_delegation *dp = delegstateid(st); + + /* Only for *_ATTRS_DELEG flavors */ + if (deleg_attrs_deleg(dp->dl_type)) { + vet_deleg_attrs(setattr, dp); + status = nfs_ok; + } + } + } + if (st) + nfs4_put_stid(st); + if (status) + goto out_err; + + err = fh_want_write(&cstate->current_fh); + if (err) { + status = nfserrno(err); + goto out_err; + } + status = nfs_ok; + + status = check_attr_support(cstate, setattr->sa_bmval, nfsd_attrmask); + if (status) + goto out; + + if (setattr->sa_acl && (attrs.na_dpacl || attrs.na_pacl)) { + status = nfserr_inval; + goto out; + } + + inode = cstate->current_fh.fh_dentry->d_inode; + status = nfsd4_acl_to_attr(S_ISDIR(inode->i_mode) ? NF4DIR : NF4REG, + setattr->sa_acl, &attrs); + + if (status) + goto out; + save_no_wcc = cstate->current_fh.fh_no_wcc; + cstate->current_fh.fh_no_wcc = true; + status = nfsd_setattr(rqstp, &cstate->current_fh, &attrs, NULL); + cstate->current_fh.fh_no_wcc = save_no_wcc; + if (!status) + status = nfserrno(attrs.na_labelerr); + if (!status) + status = nfserrno(attrs.na_dpaclerr); + if (!status) + status = nfserrno(attrs.na_paclerr); +out: + fh_drop_write(&cstate->current_fh); +out_err: + nfsd_attrs_free(&attrs); + return status; +} + +static void nfsd4_file_mark_deleg_written(struct nfs4_file *fi) +{ + spin_lock(&fi->fi_lock); + if (!list_empty(&fi->fi_delegations)) { + struct nfs4_delegation *dp = list_first_entry(&fi->fi_delegations, + struct nfs4_delegation, dl_perfile); + + if (dp->dl_type == OPEN_DELEGATE_WRITE_ATTRS_DELEG) + dp->dl_written = true; + } + spin_unlock(&fi->fi_lock); +} + +static __be32 +nfsd4_write(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_write *write = &u->write; + stateid_t *stateid = &write->wr_stateid; + struct nfs4_stid *stid = NULL; + struct nfsd_file *nf = NULL; + __be32 status = nfs_ok; + unsigned long cnt; + + if (write->wr_offset > (u64)OFFSET_MAX || + write->wr_offset + write->wr_buflen > (u64)OFFSET_MAX) + return nfserr_fbig; + + cnt = write->wr_buflen; + trace_nfsd_write_start(rqstp, &cstate->current_fh, + write->wr_offset, cnt); + status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh, + stateid, WR_STATE, &nf, &stid); + if (status) + return status; + + if (stid) { + nfsd4_file_mark_deleg_written(stid->sc_file); + nfs4_put_stid(stid); + } + + write->wr_how_written = write->wr_stable_how; + status = nfsd_vfs_write(rqstp, &cstate->current_fh, nf, + write->wr_offset, &write->wr_payload, + &cnt, write->wr_how_written, + (__be32 *)write->wr_verifier.data); + nfsd_file_put(nf); + + write->wr_bytes_written = cnt; + trace_nfsd_write_done(rqstp, &cstate->current_fh, + write->wr_offset, cnt); + return status; +} + +static __be32 +nfsd4_verify_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + stateid_t *src_stateid, struct nfsd_file **src, + stateid_t *dst_stateid, struct nfsd_file **dst) +{ + __be32 status; + + if (!cstate->save_fh.fh_dentry) + return nfserr_nofilehandle; + + status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->save_fh, + src_stateid, RD_STATE, src, NULL); + if (status) + goto out; + + status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh, + dst_stateid, WR_STATE, dst, NULL); + if (status) + goto out_put_src; + + /* fix up for NFS-specific error code */ + if (!S_ISREG(file_inode((*src)->nf_file)->i_mode) || + !S_ISREG(file_inode((*dst)->nf_file)->i_mode)) { + status = nfserr_wrong_type; + goto out_put_dst; + } + +out: + return status; +out_put_dst: + nfsd_file_put(*dst); + *dst = NULL; +out_put_src: + nfsd_file_put(*src); + *src = NULL; + goto out; +} + +static __be32 +nfsd4_clone(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_clone *clone = &u->clone; + struct nfsd_file *src, *dst; + __be32 status; + + status = nfsd4_verify_copy(rqstp, cstate, &clone->cl_src_stateid, &src, + &clone->cl_dst_stateid, &dst); + if (status) + goto out; + + status = nfsd4_clone_file_range(rqstp, src, clone->cl_src_pos, + dst, clone->cl_dst_pos, clone->cl_count, + EX_ISSYNC(cstate->current_fh.fh_export)); + + if (!status && (READ_ONCE(dst->nf_file->f_mode) & FMODE_NOCMTIME) != 0) + nfsd_update_cmtime_attr(dst->nf_file, 0); + + nfsd_file_put(dst); + nfsd_file_put(src); +out: + return status; +} + +/** + * nfsd4_has_active_async_copies - Check for ongoing copy operations + * @clp: Client to be checked + * + * NFSD maintains state for async COPY operations after they complete, + * and this state remains in the nfs4_client's async_copies list. + * Ongoing copies should block the destruction of the nfs4_client, but + * completed copies should not. + * + * Return values: + * %true: At least one active async COPY is ongoing + * %false: No active async COPY operations were found + */ +bool nfsd4_has_active_async_copies(struct nfs4_client *clp) +{ + struct nfsd4_async_copy *copy; + bool result = false; + + spin_lock(&clp->async_lock); + list_for_each_entry(copy, &clp->async_copies, copies) { + if (!test_bit(NFSD4_COPY_F_COMPLETED, ©->cp_copy.cp_flags) && + !test_bit(NFSD4_COPY_F_STOPPED, ©->cp_copy.cp_flags)) { + result = true; + break; + } + } + spin_unlock(&clp->async_lock); + return result; +} + +/** + * nfsd4_async_copy_reaper - Purge completed copies + * @nn: Network namespace with possible active copy information + */ +void nfsd4_async_copy_reaper(struct nfsd_net *nn) +{ + struct nfs4_client *clp; + struct nfsd4_async_copy *copy; + LIST_HEAD(reaplist); + + spin_lock(&nn->client_lock); + list_for_each_entry(clp, &nn->client_lru, cl_lru) { + struct list_head *pos, *next; + + spin_lock(&clp->async_lock); + list_for_each_safe(pos, next, &clp->async_copies) { + copy = list_entry(pos, struct nfsd4_async_copy, copies); + if (test_bit(NFSD4_COPY_F_OFFLOAD_DONE, + ©->cp_copy.cp_flags)) { + if (!--copy->cp_ttl) { + list_del_init(©->copies); + list_add(©->copies, &reaplist); + } + } + } + spin_unlock(&clp->async_lock); + } + spin_unlock(&nn->client_lock); + + while (!list_empty(&reaplist)) { + copy = list_first_entry(&reaplist, struct nfsd4_async_copy, + copies); + list_del_init(©->copies); + cleanup_async_copy(copy); + } +} + +static void nfs4_put_copy(struct nfsd4_async_copy *copy) +{ + if (!refcount_dec_and_test(©->refcount)) + return; + /* + * Drop the copy offload stateid's sole reference: removes it from + * cl_stateids and frees the async_copy via nfsd4_free_async_copy_stid(). + */ + nfs4_put_stid(©->cp_stid); +} + +static void release_copy_files(struct nfsd4_copy *copy); + +static void nfsd4_stop_copy(struct nfsd4_async_copy *copy) +{ + trace_nfsd_copy_async_cancel(©->cp_copy); + /* + * Join the kthread before releasing its resources. The task_struct is + * pinned in nfsd4_copy(), so kthread_stop() is safe even after the + * one-shot kthread has exited. The caller already unlinked the copy, + * so this runs once per copy. + */ + set_bit(NFSD4_COPY_F_STOPPED, ©->cp_copy.cp_flags); + kthread_stop(copy->copy_task); + if (!test_bit(NFSD4_COPY_F_CB_ERROR, ©->cp_copy.cp_flags)) + copy->cp_copy.nfserr = nfs_ok; + set_bit(NFSD4_COPY_F_COMPLETED, ©->cp_copy.cp_flags); + + release_copy_files(©->cp_copy); + nfs4_put_copy(copy); +} + +static struct nfsd4_async_copy *nfsd4_unhash_copy(struct nfs4_client *clp) +{ + struct nfsd4_async_copy *copy = NULL; + + spin_lock(&clp->async_lock); + if (!list_empty(&clp->async_copies)) { + copy = list_first_entry(&clp->async_copies, + struct nfsd4_async_copy, copies); + refcount_inc(©->refcount); + /* Pairs with smp_load_acquire() in nfsd4_send_cb_offload(). */ + smp_store_release(©->cp_copy.cp_clp, NULL); + if (!list_empty(©->copies)) + list_del_init(©->copies); + } + spin_unlock(&clp->async_lock); + return copy; +} + +void nfsd4_shutdown_copy(struct nfs4_client *clp) +{ + struct nfsd4_async_copy *copy; + + while ((copy = nfsd4_unhash_copy(clp)) != NULL) { + nfsd4_stop_copy(copy); + /* Reaper can't reach the unhashed copy; drop its membership ref. */ + nfs4_put_copy(copy); + } +} + +static bool nfsd4_copy_on_sb(const struct nfsd4_copy *copy, + const struct super_block *sb) +{ + if (copy->nf_src && + file_inode(copy->nf_src->nf_file)->i_sb == sb) + return true; + if (copy->nf_dst && + file_inode(copy->nf_dst->nf_file)->i_sb == sb) + return true; + return false; +} + +/** + * nfsd4_cancel_copy_by_sb - cancel async copy operations on @sb + * @net: net namespace containing the copy operations + * @sb: targeted superblock + * + * Context: Caller must hold nfsd_mutex with NFSD_NET_UP set. Outside + * that window nn->conf_id_hashtbl is unallocated or freed, + * so the walk would dereference a NULL or dangling pointer. + */ +void nfsd4_cancel_copy_by_sb(struct net *net, struct super_block *sb) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct nfsd4_async_copy *copy, *tmp; + struct nfs4_client *clp; + unsigned int idhashval; + LIST_HEAD(to_cancel); + + lockdep_assert_held(&nfsd_mutex); + spin_lock(&nn->client_lock); + for (idhashval = 0; idhashval < CLIENT_HASH_SIZE; idhashval++) { + struct list_head *head = &nn->conf_id_hashtbl[idhashval]; + + list_for_each_entry(clp, head, cl_idhash) { + spin_lock(&clp->async_lock); + list_for_each_entry_safe(copy, tmp, + &clp->async_copies, copies) { + if (nfsd4_copy_on_sb(©->cp_copy, sb)) { + refcount_inc(©->refcount); + /* + * Hold a reference on the client while + * nfsd4_stop_copy() runs. Unlike + * nfsd4_unhash_copy(), cp_clp is not + * NULLed here because nfsd4_send_cb_offload() + * needs a valid client to send CB_OFFLOAD. + * That function takes its own reference to + * survive callback flight. + */ + kref_get(&clp->cl_nfsdfs.cl_ref); + copy->cp_copy.nfserr = nfserr_admin_revoked; + set_bit(NFSD4_COPY_F_CB_ERROR, + ©->cp_copy.cp_flags); + list_move(©->copies, &to_cancel); + } + } + spin_unlock(&clp->async_lock); + } + } + spin_unlock(&nn->client_lock); + + list_for_each_entry_safe(copy, tmp, &to_cancel, copies) { + struct nfs4_client *clp = copy->cp_copy.cp_clp; + + list_del_init(©->copies); + nfsd4_stop_copy(copy); + /* Reaper can't reach the unlinked copy; drop the membership ref here. */ + nfs4_put_copy(copy); + nfsd4_put_client(clp); + } +} + +#ifdef CONFIG_NFSD_V4_2_INTER_SSC + +extern struct file *nfs42_ssc_open(struct vfsmount *ss_mnt, + struct nfs_fh *src_fh, + nfs4_stateid *stateid); +extern void nfs42_ssc_close(struct file *filep); + +extern void nfs_sb_deactive(struct super_block *sb); + +#define NFSD42_INTERSSC_MOUNTOPS "vers=4.2,addr=%s,sec=sys" + +/* + * setup a work entry in the ssc delayed unmount list. + */ +static __be32 nfsd4_ssc_setup_dul(struct nfsd_net *nn, char *ipaddr, + struct nfsd4_ssc_umount_item **nsui, + struct svc_rqst *rqstp) +{ + struct nfsd4_ssc_umount_item *ni = NULL; + struct nfsd4_ssc_umount_item *work = NULL; + struct nfsd4_ssc_umount_item *tmp; + DEFINE_WAIT(wait); + __be32 status = 0; + + *nsui = NULL; + work = kzalloc_obj(*work); +try_again: + spin_lock(&nn->nfsd_ssc_lock); + list_for_each_entry_safe(ni, tmp, &nn->nfsd_ssc_mount_list, nsui_list) { + if (strncmp(ni->nsui_ipaddr, ipaddr, sizeof(ni->nsui_ipaddr))) + continue; + /* found a match */ + if (ni->nsui_busy) { + /* wait - and try again */ + prepare_to_wait(&nn->nfsd_ssc_waitq, &wait, TASK_IDLE); + spin_unlock(&nn->nfsd_ssc_lock); + + /* allow 20secs for mount/unmount for now - revisit */ + if (svc_thread_should_stop(rqstp) || + (schedule_timeout(20*HZ) == 0)) { + finish_wait(&nn->nfsd_ssc_waitq, &wait); + kfree(work); + return nfserr_jukebox; + } + finish_wait(&nn->nfsd_ssc_waitq, &wait); + goto try_again; + } + *nsui = ni; + refcount_inc(&ni->nsui_refcnt); + spin_unlock(&nn->nfsd_ssc_lock); + kfree(work); + + /* return vfsmount in (*nsui)->nsui_vfsmount */ + return 0; + } + if (work) { + strscpy(work->nsui_ipaddr, ipaddr, sizeof(work->nsui_ipaddr)); + refcount_set(&work->nsui_refcnt, 2); + work->nsui_busy = true; + list_add_tail(&work->nsui_list, &nn->nfsd_ssc_mount_list); + *nsui = work; + } else + status = nfserr_resource; + spin_unlock(&nn->nfsd_ssc_lock); + return status; +} + +static void nfsd4_ssc_update_dul(struct nfsd_net *nn, + struct nfsd4_ssc_umount_item *nsui, + struct vfsmount *ss_mnt) +{ + spin_lock(&nn->nfsd_ssc_lock); + nsui->nsui_vfsmount = ss_mnt; + nsui->nsui_busy = false; + wake_up_all(&nn->nfsd_ssc_waitq); + spin_unlock(&nn->nfsd_ssc_lock); +} + +static void nfsd4_ssc_cancel_dul(struct nfsd_net *nn, + struct nfsd4_ssc_umount_item *nsui) +{ + spin_lock(&nn->nfsd_ssc_lock); + list_del(&nsui->nsui_list); + wake_up_all(&nn->nfsd_ssc_waitq); + spin_unlock(&nn->nfsd_ssc_lock); + kfree(nsui); +} + +/* + * Support one copy source server for now. + */ +static __be32 +nfsd4_interssc_connect(struct nl4_server *nss, struct svc_rqst *rqstp, + struct nfsd4_ssc_umount_item **nsui) +{ + struct file_system_type *type; + struct vfsmount *ss_mnt; + struct nfs42_netaddr *naddr; + struct sockaddr_storage tmp_addr; + size_t tmp_addrlen, match_netid_len = 3; + char *startsep = "", *endsep = "", *match_netid = "tcp"; + char *ipaddr, *dev_name, *raw_data; + int len, raw_len; + __be32 status = nfserr_inval; + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + + naddr = &nss->u.nl4_addr; + tmp_addrlen = rpc_uaddr2sockaddr(SVC_NET(rqstp), naddr->addr, + naddr->addr_len, + (struct sockaddr *)&tmp_addr, + sizeof(tmp_addr)); + *nsui = NULL; + if (tmp_addrlen == 0) + goto out_err; + + if (tmp_addr.ss_family == AF_INET6) { + startsep = "["; + endsep = "]"; + match_netid = "tcp6"; + match_netid_len = 4; + } + + if (naddr->netid_len != match_netid_len || + strncmp(naddr->netid, match_netid, naddr->netid_len)) + goto out_err; + + /* Construct the raw data for the vfs_kern_mount call */ + len = RPC_MAX_ADDRBUFLEN + 1; + ipaddr = kzalloc(len, GFP_KERNEL); + if (!ipaddr) + goto out_err; + + rpc_ntop((struct sockaddr *)&tmp_addr, ipaddr, len); + + /* 2 for ipv6 endsep and startsep. 3 for ":/" and trailing '/0'*/ + + raw_len = strlen(NFSD42_INTERSSC_MOUNTOPS) + strlen(ipaddr); + raw_data = kzalloc(raw_len, GFP_KERNEL); + if (!raw_data) + goto out_free_ipaddr; + + snprintf(raw_data, raw_len, NFSD42_INTERSSC_MOUNTOPS, ipaddr); + + status = nfserr_nodev; + type = get_fs_type("nfs"); + if (!type) + goto out_free_rawdata; + + /* Set the server: for the vfs_kern_mount call */ + dev_name = kzalloc(len + 5, GFP_KERNEL); + if (!dev_name) + goto out_free_rawdata; + snprintf(dev_name, len + 5, "%s%s%s:/", startsep, ipaddr, endsep); + + status = nfsd4_ssc_setup_dul(nn, ipaddr, nsui, rqstp); + if (status) + goto out_free_devname; + if ((*nsui)->nsui_vfsmount) + goto out_done; + + /* Use an 'internal' mount: SB_KERNMOUNT -> MNT_INTERNAL */ + ss_mnt = vfs_kern_mount(type, SB_KERNMOUNT, dev_name, raw_data); + module_put(type->owner); + if (IS_ERR(ss_mnt)) { + status = nfserr_nodev; + nfsd4_ssc_cancel_dul(nn, *nsui); + goto out_free_devname; + } + nfsd4_ssc_update_dul(nn, *nsui, ss_mnt); +out_done: + status = 0; + +out_free_devname: + kfree(dev_name); +out_free_rawdata: + kfree(raw_data); +out_free_ipaddr: + kfree(ipaddr); +out_err: + return status; +} + +/* + * Verify COPY destination stateid. + * + * Connect to the source server with NFSv4.1. + * Create the source struct file for nfsd_copy_range. + * Called with COPY cstate: + * SAVED_FH: source filehandle + * CURRENT_FH: destination filehandle + */ +static __be32 +nfsd4_setup_inter_ssc(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, + struct nfsd4_copy *copy) +{ + struct svc_fh *s_fh = NULL; + stateid_t *s_stid = ©->cp_src_stateid; + __be32 status = nfserr_inval; + + /* Verify the destination stateid and set dst struct file*/ + status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh, + ©->cp_dst_stateid, + WR_STATE, ©->nf_dst, NULL); + if (status) + goto out; + + status = nfsd4_interssc_connect(copy->cp_src, rqstp, ©->ss_nsui); + if (status) + goto out; + + s_fh = &cstate->save_fh; + + copy->c_fh.size = s_fh->fh_handle.fh_size; + memcpy(copy->c_fh.data, &s_fh->fh_handle.fh_raw, copy->c_fh.size); + copy->stateid.seqid = cpu_to_be32(s_stid->si_generation); + memcpy(copy->stateid.other, (void *)&s_stid->si_opaque, + sizeof(stateid_opaque_t)); + + status = 0; +out: + return status; +} + +static void +nfsd4_cleanup_inter_ssc(struct nfsd4_ssc_umount_item *nsui, struct file *filp, + struct nfsd_file *dst) +{ + struct nfsd_net *nn = net_generic(dst->nf_net, nfsd_net_id); + long timeout = msecs_to_jiffies(nfsd4_ssc_umount_timeout); + + nfs42_ssc_close(filp); + fput(filp); + + spin_lock(&nn->nfsd_ssc_lock); + list_del(&nsui->nsui_list); + /* + * vfsmount can be shared by multiple exports, + * decrement refcnt. If the count drops to 1 it + * will be unmounted when nsui_expire expires. + */ + refcount_dec(&nsui->nsui_refcnt); + nsui->nsui_expire = jiffies + timeout; + list_add_tail(&nsui->nsui_list, &nn->nfsd_ssc_mount_list); + spin_unlock(&nn->nfsd_ssc_lock); +} + +#else /* CONFIG_NFSD_V4_2_INTER_SSC */ + +static __be32 +nfsd4_setup_inter_ssc(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, + struct nfsd4_copy *copy) +{ + return nfserr_inval; +} + +static void +nfsd4_cleanup_inter_ssc(struct nfsd4_ssc_umount_item *nsui, struct file *filp, + struct nfsd_file *dst) +{ +} + +static struct file *nfs42_ssc_open(struct vfsmount *ss_mnt, + struct nfs_fh *src_fh, + nfs4_stateid *stateid) +{ + return NULL; +} +#endif /* CONFIG_NFSD_V4_2_INTER_SSC */ + +static __be32 +nfsd4_setup_intra_ssc(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, + struct nfsd4_copy *copy) +{ + return nfsd4_verify_copy(rqstp, cstate, ©->cp_src_stateid, + ©->nf_src, ©->cp_dst_stateid, + ©->nf_dst); +} + +static void nfsd4_cb_offload_release(struct nfsd4_callback *cb) +{ + struct nfsd4_cb_offload *cbo = + container_of(cb, struct nfsd4_cb_offload, co_cb); + struct nfsd4_async_copy *copy = + container_of(cbo, struct nfsd4_async_copy, cp_cb_offload); + + set_bit(NFSD4_COPY_F_OFFLOAD_DONE, ©->cp_copy.cp_flags); + /* nfs4_put_copy needs clp; cl_cb_inflight defers free_client. */ + nfsd4_put_client(cb->cb_clp); + /* Drop the copy reference taken in nfsd4_send_cb_offload(). */ + nfs4_put_copy(copy); +} + +static int nfsd4_cb_offload_done(struct nfsd4_callback *cb, + struct rpc_task *task) +{ + struct nfsd4_cb_offload *cbo = + container_of(cb, struct nfsd4_cb_offload, co_cb); + + trace_nfsd_cb_offload_done(&cbo->co_res.cb_stateid, task); + switch (task->tk_status) { + case -NFS4ERR_DELAY: + if (cbo->co_retries--) { + rpc_delay(task, HZ / 5); + return 0; + } + } + nfsd41_cb_destroy_referring_call_list(cb); + return 1; +} + +static const struct nfsd4_callback_ops nfsd4_cb_offload_ops = { + .release = nfsd4_cb_offload_release, + .done = nfsd4_cb_offload_done, + .opcode = OP_CB_OFFLOAD, +}; + +static void nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync) +{ + copy->cp_res.wr_stable_how = + test_bit(NFSD4_COPY_F_COMMITTED, ©->cp_flags) ? + NFS_FILE_SYNC : NFS_UNSTABLE; + nfsd4_copy_set_sync(copy, sync); +} + +static ssize_t _nfsd_copy_file_range(struct nfsd4_copy *copy, + struct file *dst, + struct file *src) +{ + errseq_t since; + ssize_t bytes_copied = 0; + u64 bytes_total = copy->cp_count; + u64 src_pos = copy->cp_src_pos; + u64 dst_pos = copy->cp_dst_pos; + int status; + loff_t end; + + /* See RFC 7862 p.67: */ + if (bytes_total == 0) + bytes_total = ULLONG_MAX; + since = READ_ONCE(dst->f_wb_err); + do { + /* Only async copies can be stopped here */ + if (kthread_should_stop()) + break; + bytes_copied = nfsd_copy_file_range(src, src_pos, dst, dst_pos, + bytes_total); + if (bytes_copied <= 0) + break; + bytes_total -= bytes_copied; + copy->cp_res.wr_bytes_written += bytes_copied; + src_pos += bytes_copied; + dst_pos += bytes_copied; + } while (bytes_total > 0 && nfsd4_copy_is_async(copy)); + /* for a non-zero asynchronous copy do a commit of data */ + if (nfsd4_copy_is_async(copy) && copy->cp_res.wr_bytes_written > 0) { + end = copy->cp_dst_pos + copy->cp_res.wr_bytes_written - 1; + status = vfs_fsync_range(dst, copy->cp_dst_pos, end, 0); + if (!status) + status = filemap_check_wb_err(dst->f_mapping, since); + if (!status) + set_bit(NFSD4_COPY_F_COMMITTED, ©->cp_flags); + else if (status != -EAGAIN && status != -ESTALE) + nfsd_reset_write_verifier(copy->cp_nn); + } + return bytes_copied; +} + +static __be32 nfsd4_do_copy(struct nfsd4_copy *copy, + struct file *src, struct file *dst, + bool sync) +{ + __be32 status; + ssize_t bytes; + + bytes = _nfsd_copy_file_range(copy, dst, src); + + /* for async copy, we ignore the error, client can always retry + * to get the error + */ + if (bytes < 0 && !copy->cp_res.wr_bytes_written) + status = nfserrno(bytes); + else { + nfsd4_init_copy_res(copy, sync); + status = nfs_ok; + } + return status; +} + +static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst) +{ + dst->cp_src_pos = src->cp_src_pos; + dst->cp_dst_pos = src->cp_dst_pos; + dst->cp_count = src->cp_count; + dst->cp_flags = src->cp_flags; + memcpy(&dst->cp_res, &src->cp_res, sizeof(src->cp_res)); + memcpy(&dst->fh, &src->fh, sizeof(src->fh)); + dst->cp_clp = src->cp_clp; + dst->nf_dst = nfsd_file_get(src->nf_dst); + /* for inter, nf_src doesn't exist yet */ + if (!nfsd4_ssc_is_inter(src)) + dst->nf_src = nfsd_file_get(src->nf_src); + + memcpy(dst->cp_src, src->cp_src, sizeof(struct nl4_server)); + memcpy(&dst->stateid, &src->stateid, sizeof(src->stateid)); + memcpy(&dst->c_fh, &src->c_fh, sizeof(src->c_fh)); + dst->ss_nsui = src->ss_nsui; +} + +static void release_copy_files(struct nfsd4_copy *copy) +{ + if (copy->nf_src) { + nfsd_file_put(copy->nf_src); + copy->nf_src = NULL; + } + if (copy->nf_dst) { + nfsd_file_put(copy->nf_dst); + copy->nf_dst = NULL; + } +} + +/* + * Called from the reaper and from nfsd4_copy()'s error path; in both + * cases the copy is already unreachable from clp->async_copies. + */ +static void cleanup_async_copy(struct nfsd4_async_copy *copy) +{ + release_copy_files(©->cp_copy); + nfs4_put_copy(copy); +} + +static void nfsd4_send_cb_offload(struct nfsd4_async_copy *copy) +{ + struct nfsd4_cb_offload *cbo = ©->cp_cb_offload; + struct nfs4_client *clp; + + /* + * Pairs with smp_store_release(&cp_clp) in find_async_copy() and + * nfsd4_unhash_copy(); the set_bit/clear_bit writers are unordered. + * cp_clp is NULL once the copy was canceled; skip the callback, the + * canceling path owns the notification. + */ + clp = smp_load_acquire(©->cp_copy.cp_clp); + if (!clp) { + set_bit(NFSD4_COPY_F_OFFLOAD_DONE, ©->cp_copy.cp_flags); + return; + } + + memcpy(&cbo->co_res, ©->cp_copy.cp_res, sizeof(copy->cp_copy.cp_res)); + memcpy(&cbo->co_fh, ©->cp_copy.fh, sizeof(copy->cp_copy.fh)); + cbo->co_nfserr = copy->cp_copy.nfserr; + cbo->co_retries = 5; + + /* + * Hold the client and the copy across the in-flight callback; co_cb is + * embedded in the copy, so it must outlive the callback. Both are + * dropped in nfsd4_cb_offload_release(). + */ + kref_get(&clp->cl_nfsdfs.cl_ref); + refcount_inc(©->refcount); + + nfsd4_init_cb(&cbo->co_cb, clp, &nfsd4_cb_offload_ops, + NFSPROC4_CLNT_CB_OFFLOAD); + nfsd41_cb_referring_call(&cbo->co_cb, &cbo->co_referring_sessionid, + cbo->co_referring_slotid, + cbo->co_referring_seqno); + trace_nfsd_cb_offload(clp, &cbo->co_res.cb_stateid, + &cbo->co_fh, copy->cp_copy.cp_count, + copy->cp_copy.nfserr); + nfsd4_try_run_cb(&cbo->co_cb); +} + +/** + * nfsd4_do_async_copy - kthread function for background server-side COPY + * @data: arguments for COPY operation + * + * Return values: + * %0: Copy operation is done. + */ +static int nfsd4_do_async_copy(void *data) +{ + struct nfsd4_async_copy *async = data; + struct nfsd4_copy *copy = &async->cp_copy; + __be32 nfserr = nfs_ok; + + trace_nfsd_copy_async(copy); + if (nfsd4_ssc_is_inter(copy)) { + struct file *filp; + + filp = nfs42_ssc_open(copy->ss_nsui->nsui_vfsmount, + ©->c_fh, ©->stateid); + if (IS_ERR(filp)) { + switch (PTR_ERR(filp)) { + case -EBADF: + nfserr = nfserr_wrong_type; + break; + default: + nfserr = nfserr_offload_denied; + } + /* ss_mnt will be unmounted by the laundromat */ + goto do_callback; + } + nfserr = nfsd4_do_copy(copy, filp, copy->nf_dst->nf_file, + false); + nfsd4_cleanup_inter_ssc(copy->ss_nsui, filp, copy->nf_dst); + } else { + nfserr = nfsd4_do_copy(copy, copy->nf_src->nf_file, + copy->nf_dst->nf_file, false); + } + +do_callback: + if (!test_bit(NFSD4_COPY_F_CB_ERROR, ©->cp_flags)) + copy->nfserr = nfserr; + /* + * Don't set NFSD4_COPY_F_STOPPED here: it tells a teardown caller it + * may skip kthread_stop(), which would then release nf_dst and the + * client while still in use. Only nfsd4_stop_copy() sets it, after + * joining. + */ + set_bit(NFSD4_COPY_F_COMPLETED, ©->cp_flags); + trace_nfsd_copy_async_done(copy); + atomic_dec(©->cp_nn->pending_async_copies); + if (copy->cp_res.wr_bytes_written > 0 && copy->attr_update) + nfsd_update_cmtime_attr(copy->nf_dst->nf_file, 0); + nfsd4_send_cb_offload(async); + /* Drop the kthread's reference (taken in nfsd4_copy()); copy may be freed after this. */ + nfs4_put_copy(async); + return 0; +} + +static __be32 +nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + struct nfsd4_async_copy *async_copy = NULL; + struct nfsd4_copy *copy = &u->copy; + struct nfsd42_write_res *result; + __be32 status; + + result = ©->cp_res; + nfsd_copy_write_verifier((__be32 *)&result->wr_verifier.data, nn); + + copy->cp_clp = cstate->clp; + if (nfsd4_ssc_is_inter(copy)) { + trace_nfsd_copy_inter(copy); + if (!inter_copy_offload_enable || nfsd4_copy_is_sync(copy)) { + status = nfserr_notsupp; + goto out; + } + status = nfsd4_setup_inter_ssc(rqstp, cstate, copy); + if (status) { + status = nfserr_offload_denied; + goto out; + } + } else { + trace_nfsd_copy_intra(copy); + status = nfsd4_setup_intra_ssc(rqstp, cstate, copy); + if (status) + goto out; + } + + memcpy(©->fh, &cstate->current_fh.fh_handle, + sizeof(struct knfsd_fh)); + if (nfsd4_copy_is_async(copy)) { + struct task_struct *task; + + /* + * Allocate the durable async copy. Its offload stateid is a + * first-class nfs4_stid in clp->cl_stateids, returned to the + * client and freed only when the background copy is torn down. + */ + async_copy = nfs4_alloc_copy_stid(cstate->clp); + if (!async_copy) + goto out_err; + async_copy->cp_copy.cp_nn = nn; + INIT_LIST_HEAD(&async_copy->copies); + refcount_set(&async_copy->refcount, 1); + async_copy->cp_ttl = NFSD_COPY_INITIAL_TTL; + /* Arbitrary cap on number of pending async copy operations */ + if (atomic_inc_return(&nn->pending_async_copies) > + (int)rqstp->rq_pool->sp_nrthreads) + goto out_dec_async_copy_err; + async_copy->cp_copy.cp_src = kmalloc_obj(*async_copy->cp_copy.cp_src); + if (!async_copy->cp_copy.cp_src) + goto out_dec_async_copy_err; + memcpy(&result->cb_stateid, &async_copy->cp_stid.sc_stateid, + sizeof(result->cb_stateid)); + /* + * dup after writing cb_stateid; duplicating first would leave + * the callback stateid zeroed. + */ + dup_copy_fields(copy, &async_copy->cp_copy); + if ((READ_ONCE(copy->nf_dst->nf_file->f_mode) & + FMODE_NOCMTIME) != 0) + async_copy->cp_copy.attr_update = true; + memcpy(async_copy->cp_cb_offload.co_referring_sessionid.data, + cstate->session->se_sessionid.data, + NFS4_MAX_SESSIONID_LEN); + async_copy->cp_cb_offload.co_referring_slotid = cstate->slot->sl_index; + async_copy->cp_cb_offload.co_referring_seqno = cstate->slot->sl_seqid; + task = kthread_create(nfsd4_do_async_copy, async_copy, + "%s", "copy thread"); + if (IS_ERR(task)) + goto out_dec_async_copy_err; + /* + * Pin the task_struct so kthread_stop() is safe after this + * one-shot kthread exits. Released by nfs4_put_copy(). + */ + get_task_struct(task); + async_copy->copy_task = task; + /* + * Take the kthread's ref and wake it before publishing, so the + * publisher touches async_copy no further and teardown can + * drain it. + */ + refcount_inc(&async_copy->refcount); + wake_up_process(async_copy->copy_task); + spin_lock(&async_copy->cp_copy.cp_clp->async_lock); + list_add(&async_copy->copies, + &async_copy->cp_copy.cp_clp->async_copies); + spin_unlock(&async_copy->cp_copy.cp_clp->async_lock); + status = nfs_ok; + } else { + status = nfsd4_do_copy(copy, copy->nf_src->nf_file, + copy->nf_dst->nf_file, true); + if ((READ_ONCE(copy->nf_dst->nf_file->f_mode) & + FMODE_NOCMTIME) != 0 && + copy->cp_res.wr_bytes_written > 0) + nfsd_update_cmtime_attr(copy->nf_dst->nf_file, 0); + } +out: + trace_nfsd_copy_done(copy, status); + release_copy_files(copy); + return status; +out_dec_async_copy_err: + if (async_copy) + atomic_dec(&nn->pending_async_copies); +out_err: + if (nfsd4_ssc_is_inter(copy)) { + /* + * Source's vfsmount of inter-copy will be unmounted + * by the laundromat. Use copy instead of async_copy + * since async_copy->ss_nsui might not be set yet. + */ + refcount_dec(©->ss_nsui->nsui_refcnt); + } + if (async_copy) + cleanup_async_copy(async_copy); + status = nfserr_jukebox; + goto out; +} + +static struct nfsd4_async_copy * +find_async_copy_locked(struct nfs4_client *clp, stateid_t *stateid) +{ + struct nfsd4_async_copy *copy; + + lockdep_assert_held(&clp->async_lock); + + list_for_each_entry(copy, &clp->async_copies, copies) { + if (memcmp(©->cp_stid.sc_stateid, stateid, NFS4_STATEID_SIZE)) + continue; + return copy; + } + return NULL; +} + +static struct nfsd4_async_copy * +find_async_copy(struct nfs4_client *clp, stateid_t *stateid) +{ + struct nfsd4_async_copy *copy; + + spin_lock(&clp->async_lock); + copy = find_async_copy_locked(clp, stateid); + if (copy) { + refcount_inc(©->refcount); + /* + * Mirror nfsd4_unhash_copy(): unlink and clear cp_clp under + * async_lock so the reaper can't reach it. Caller drops the + * membership ref after nfsd4_stop_copy(). + */ + smp_store_release(©->cp_copy.cp_clp, NULL); + if (!list_empty(©->copies)) + list_del_init(©->copies); + } + spin_unlock(&clp->async_lock); + return copy; +} + +static __be32 +nfsd4_offload_cancel(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_offload_status *os = &u->offload_status; + struct nfsd4_async_copy *copy; + struct nfs4_client *clp = cstate->clp; + + copy = find_async_copy(clp, &os->stateid); + if (!copy) { + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + + return manage_cpntf_state(nn, &os->stateid, clp, NULL); + } else { + nfsd4_stop_copy(copy); + /* find_async_copy() unlinked it from the reaper; drop the membership ref. */ + nfs4_put_copy(copy); + } + + return nfs_ok; +} + +static __be32 +nfsd4_set_netaddr(struct sockaddr *addr, struct nfs42_netaddr *netaddr) +{ + struct sockaddr_in *sin = (struct sockaddr_in *)addr; + struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr; + unsigned int port; + size_t ret_addr, ret_port; + + switch (addr->sa_family) { + case AF_INET: + port = ntohs(sin->sin_port); + sprintf(netaddr->netid, "tcp"); + netaddr->netid_len = 3; + break; + case AF_INET6: + port = ntohs(sin6->sin6_port); + sprintf(netaddr->netid, "tcp6"); + netaddr->netid_len = 4; + break; + default: + return nfserr_inval; + } + ret_addr = rpc_ntop(addr, netaddr->addr, sizeof(netaddr->addr)); + ret_port = snprintf(netaddr->addr + ret_addr, + RPCBIND_MAXUADDRLEN + 1 - ret_addr, + ".%u.%u", port >> 8, port & 0xff); + WARN_ON(ret_port >= RPCBIND_MAXUADDRLEN + 1 - ret_addr); + netaddr->addr_len = ret_addr + ret_port; + return 0; +} + +static __be32 +nfsd4_copy_notify(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_copy_notify *cn = &u->copy_notify; + __be32 status; + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + struct nfs4_stid *stid = NULL; + struct nfs4_cpntf_state *cps; + + status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh, + &cn->cpn_src_stateid, RD_STATE, NULL, + &stid); + if (status) + return status; + if (!stid) + return nfserr_bad_stateid; + + cn->cpn_lease_time.tv_sec = nn->nfsd4_lease; + cn->cpn_lease_time.tv_nsec = 0; + + status = nfserrno(-ENOMEM); + /* + * The returned cps is published and fully initialized, and carries an + * extra reference for us; drop it once we are done with it. + */ + cps = nfs4_alloc_init_cpntf_state(nn, stid); + if (!cps) + goto out; + memcpy(&cn->cpn_cnr_stateid, &cps->cp_stateid.cs_stid, sizeof(stateid_t)); + + /* For now, only return one server address in cpn_src, the + * address used by the client to connect to this server. + */ + cn->cpn_src->nl4_type = NL4_NETADDR; + status = nfsd4_set_netaddr((struct sockaddr *)&rqstp->rq_daddr, + &cn->cpn_src->u.nl4_addr); + WARN_ON_ONCE(status); + /* + * Drop our extra reference. The membership reference keeps the entry + * alive for a later inter-server READ, or until the laundromat reaps it. + */ + nfs4_put_cpntf_state(nn, cps); +out: + nfs4_put_stid(stid); + return status; +} + +static __be32 +nfsd4_fallocate(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + struct nfsd4_fallocate *fallocate, int flags) +{ + __be32 status; + struct nfsd_file *nf; + + status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh, + &fallocate->falloc_stateid, + WR_STATE, &nf, NULL); + if (status != nfs_ok) + return status; + + status = nfsd4_vfs_fallocate(rqstp, &cstate->current_fh, nf->nf_file, + fallocate->falloc_offset, + fallocate->falloc_length, + flags); + nfsd_file_put(nf); + return status; +} + +static __be32 +nfsd4_offload_status(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_offload_status *os = &u->offload_status; + __be32 status = nfs_ok; + struct nfsd4_async_copy *copy; + struct nfs4_client *clp = cstate->clp; + + os->completed = false; + spin_lock(&clp->async_lock); + copy = find_async_copy_locked(clp, &os->stateid); + if (copy) { + os->count = copy->cp_copy.cp_res.wr_bytes_written; + if (test_bit(NFSD4_COPY_F_COMPLETED, ©->cp_copy.cp_flags)) { + os->completed = true; + os->status = copy->cp_copy.nfserr; + } + } else + status = nfserr_bad_stateid; + spin_unlock(&clp->async_lock); + + return status; +} + +static __be32 +nfsd4_allocate(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + return nfsd4_fallocate(rqstp, cstate, &u->allocate, 0); +} + +static __be32 +nfsd4_deallocate(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + return nfsd4_fallocate(rqstp, cstate, &u->deallocate, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE); +} + +static __be32 +nfsd4_seek(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_seek *seek = &u->seek; + int whence; + __be32 status; + struct nfsd_file *nf; + + status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh, + &seek->seek_stateid, + RD_STATE, &nf, NULL); + if (status) + return status; + + switch (seek->seek_whence) { + case NFS4_CONTENT_DATA: + whence = SEEK_DATA; + break; + case NFS4_CONTENT_HOLE: + whence = SEEK_HOLE; + break; + default: + status = nfserr_union_notsupp; + goto out; + } + + /* + * Note: This call does change file->f_pos, but nothing in NFSD + * should ever file->f_pos. + */ + seek->seek_pos = vfs_llseek(nf->nf_file, seek->seek_offset, whence); + if (seek->seek_pos < 0) + status = nfserrno(seek->seek_pos); + else if (seek->seek_pos >= i_size_read(file_inode(nf->nf_file))) + seek->seek_eof = true; + +out: + nfsd_file_put(nf); + return status; +} + +/* This routine never returns NFS_OK! If there are no other errors, it + * will return NFSERR_SAME or NFSERR_NOT_SAME depending on whether the + * attributes matched. VERIFY is implemented by mapping NFSERR_SAME + * to NFS_OK after the call; NVERIFY by mapping NFSERR_NOT_SAME to NFS_OK. + */ +static __be32 +_nfsd4_verify(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + struct nfsd4_verify *verify) +{ + __be32 *buf, *p; + int count; + __be32 status; + + status = fh_verify(rqstp, &cstate->current_fh, 0, NFSD_MAY_NOP); + if (status) + return status; + + status = check_attr_support(cstate, verify->ve_bmval, NULL); + if (status) + return status; + + if ((verify->ve_bmval[0] & FATTR4_WORD0_RDATTR_ERROR) + || (verify->ve_bmval[1] & NFSD_WRITEONLY_ATTRS_WORD1)) + return nfserr_inval; + if (verify->ve_attrlen & 3) + return nfserr_inval; + + /* The POSIX draft ACLs cannot be tested via (N)VERIFY. */ + if (verify->ve_bmval[2] & (FATTR4_WORD2_POSIX_DEFAULT_ACL | + FATTR4_WORD2_POSIX_ACCESS_ACL)) + return nfserr_inval; + + /* count in words: + * bitmap_len(1) + bitmap(2) + attr_len(1) = 4 + */ + count = 4 + (verify->ve_attrlen >> 2); + buf = kmalloc(count << 2, GFP_KERNEL); + if (!buf) + return nfserr_jukebox; + + p = buf; + status = nfsd4_encode_fattr_to_buf(&p, count, &cstate->current_fh, + cstate->current_fh.fh_export, + cstate->current_fh.fh_dentry, + verify->ve_bmval, + rqstp, 0); + /* + * If nfsd4_encode_fattr() ran out of space, assume that's because + * the attributes are longer (hence different) than those given: + */ + if (status == nfserr_resource) + status = nfserr_not_same; + if (status) + goto out_kfree; + + /* skip bitmap */ + p = buf + 1 + ntohl(buf[0]); + status = nfserr_not_same; + if (ntohl(*p++) != verify->ve_attrlen) + goto out_kfree; + if (!memcmp(p, verify->ve_attrval, verify->ve_attrlen)) + status = nfserr_same; + +out_kfree: + kfree(buf); + return status; +} + +static __be32 +nfsd4_nverify(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + __be32 status; + + status = _nfsd4_verify(rqstp, cstate, &u->verify); + return status == nfserr_not_same ? nfs_ok : status; +} + +static __be32 +nfsd4_verify(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + __be32 status; + + status = _nfsd4_verify(rqstp, cstate, &u->nverify); + return status == nfserr_same ? nfs_ok : status; +} + +#define SUPPORTED_NOTIFY_MASK (BIT(NOTIFY4_CHANGE_DIR_ATTRS) | \ + BIT(NOTIFY4_REMOVE_ENTRY) | \ + BIT(NOTIFY4_ADD_ENTRY) | \ + BIT(NOTIFY4_RENAME_ENTRY) | \ + BIT(NOTIFY4_GFLAG_EXTEND)) + +static __be32 +nfsd4_get_dir_delegation(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_get_dir_delegation *gdd = &u->get_dir_delegation; + u32 requested = gdd->gdda_notification_types[0]; + struct nfs4_delegation *dd; + struct nfsd_file *nf; + __be32 status; + + status = nfsd_file_acquire_dir(rqstp, &cstate->current_fh, &nf); + if (status != nfs_ok) + return status; + + /* + * Offer no notifications to an order-aware client. RFC8881bis section + * 16.2.13 defines order-aware as NOTIFY4_CFLAG_ORDER being set or + * NOTIFY4_GFLAG_EXTEND being reset. Such a client expects cookie and + * previous-entry information with its notifications (e.g. 27.4.5), and + * nfsd does not track or emit directory offset information. Per + * 16.2.11.3 the alternative would be to recall the delegation, so it's + * simpler to just decline the notifications here. + */ + if (!(requested & BIT(NOTIFY4_GFLAG_EXTEND)) || + (requested & BIT(NOTIFY4_CFLAG_ORDER))) + requested = 0; + + gdd->gddr_notification[0] = requested & SUPPORTED_NOTIFY_MASK; + + /* + * RFC 8881, section 18.39.3 says: + * + * "The server may refuse to grant the delegation. In that case, the + * server will return NFS4ERR_DIRDELEG_UNAVAIL." + * + * This is sub-optimal, since it means that the server would need to + * abort compound processing just because the delegation wasn't + * available. RFC8881bis should change this to allow the server to + * return NFS4_OK with a non-fatal status of GDD4_UNAVAIL in this + * situation. + */ + dd = nfsd_get_dir_deleg(cstate, gdd, nf); + nfsd_file_put(nf); + if (IS_ERR(dd)) { + gdd->gddrnf_status = GDD4_UNAVAIL; + return nfs_ok; + } + + gdd->gddrnf_status = GDD4_OK; + memcpy(&gdd->gddr_stateid, &dd->dl_stid.sc_stateid, sizeof(gdd->gddr_stateid)); + gdd->gddr_child_attributes[0] = dd->dl_child_attrs[0]; + gdd->gddr_child_attributes[1] = dd->dl_child_attrs[1]; + gdd->gddr_dir_attributes[0] = dd->dl_dir_attrs[0]; + gdd->gddr_dir_attributes[1] = dd->dl_dir_attrs[1]; + nfs4_put_stid(&dd->dl_stid); + return nfs_ok; +} + +#ifdef CONFIG_NFSD_PNFS +static const struct nfsd4_layout_ops * +nfsd4_layout_verify(struct svc_export *exp, unsigned int layout_type) +{ + if (!exp->ex_layout_types) { + dprintk("%s: export does not support pNFS\n", __func__); + return NULL; + } + + if (layout_type >= LAYOUT_TYPE_MAX || + !(exp->ex_layout_types & (1 << layout_type))) { + dprintk("%s: layout type %d not supported\n", + __func__, layout_type); + return NULL; + } + + return nfsd4_layout_ops[layout_type]; +} + +static __be32 +nfsd4_getdeviceinfo(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, union nfsd4_op_u *u) +{ + struct nfsd4_getdeviceinfo *gdp = &u->getdeviceinfo; + const struct nfsd4_layout_ops *ops; + struct nfsd4_deviceid_map *map; + struct svc_export *exp; + __be32 nfserr; + + dprintk("%s: layout_type %u dev_id [0x%llx:0x%x] maxcnt %u\n", + __func__, + gdp->gd_layout_type, + gdp->gd_devid.fsid_idx, gdp->gd_devid.generation, + gdp->gd_maxcount); + + map = nfsd4_find_devid_map(gdp->gd_devid.fsid_idx); + if (!map) { + dprintk("%s: couldn't find device ID to export mapping!\n", + __func__); + return nfserr_noent; + } + + exp = rqst_exp_find(&rqstp->rq_chandle, SVC_NET(rqstp), + rqstp->rq_client, rqstp->rq_gssclient, + map->fsid_type, map->fsid); + if (IS_ERR(exp)) { + dprintk("%s: could not find device id\n", __func__); + return nfserr_noent; + } + + nfserr = nfserr_layoutunavailable; + ops = nfsd4_layout_verify(exp, gdp->gd_layout_type); + if (!ops) + goto out; + + nfserr = nfs_ok; + if (gdp->gd_maxcount != 0) { + nfserr = ops->proc_getdeviceinfo(exp->ex_path.mnt->mnt_sb, + rqstp, cstate->clp, gdp); + } + + gdp->gd_notify_types &= ops->notify_types; +out: + exp_put(exp); + return nfserr; +} + +static void +nfsd4_getdeviceinfo_release(union nfsd4_op_u *u) +{ + kfree(u->getdeviceinfo.gd_device); +} + +static __be32 +nfsd4_layoutget(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, union nfsd4_op_u *u) +{ + struct nfsd4_layoutget *lgp = &u->layoutget; + struct svc_fh *current_fh = &cstate->current_fh; + const struct nfsd4_layout_ops *ops; + struct nfs4_layout_stateid *ls; + __be32 nfserr; + int accmode = NFSD_MAY_READ_IF_EXEC | NFSD_MAY_OWNER_OVERRIDE; + + switch (lgp->lg_seg.iomode) { + case IOMODE_READ: + accmode |= NFSD_MAY_READ; + break; + case IOMODE_RW: + accmode |= NFSD_MAY_READ | NFSD_MAY_WRITE; + break; + default: + dprintk("%s: invalid iomode %d\n", + __func__, lgp->lg_seg.iomode); + nfserr = nfserr_badiomode; + goto out; + } + + nfserr = fh_verify(rqstp, current_fh, 0, accmode); + if (nfserr) + goto out; + + nfserr = nfserr_layoutunavailable; + ops = nfsd4_layout_verify(current_fh->fh_export, lgp->lg_layout_type); + if (!ops) + goto out; + + /* + * Verify minlength and range as per RFC5661: + * o If loga_length is less than loga_minlength, + * the metadata server MUST return NFS4ERR_INVAL. + * o If the sum of loga_offset and loga_minlength exceeds + * NFS4_UINT64_MAX, and loga_minlength is not + * NFS4_UINT64_MAX, the error NFS4ERR_INVAL MUST result. + * o If the sum of loga_offset and loga_length exceeds + * NFS4_UINT64_MAX, and loga_length is not NFS4_UINT64_MAX, + * the error NFS4ERR_INVAL MUST result. + */ + nfserr = nfserr_inval; + if (lgp->lg_seg.length < lgp->lg_minlength || + (lgp->lg_minlength != NFS4_MAX_UINT64 && + lgp->lg_minlength > NFS4_MAX_UINT64 - lgp->lg_seg.offset) || + (lgp->lg_seg.length != NFS4_MAX_UINT64 && + lgp->lg_seg.length > NFS4_MAX_UINT64 - lgp->lg_seg.offset)) + goto out; + if (lgp->lg_seg.length == 0) + goto out; + + nfserr = nfsd4_preprocess_layout_stateid(rqstp, cstate, &lgp->lg_sid, + true, lgp->lg_layout_type, &ls); + if (nfserr) { + trace_nfsd_layout_get_lookup_fail(&lgp->lg_sid); + goto out; + } + + nfserr = nfserr_recallconflict; + if (atomic_read(&ls->ls_stid.sc_file->fi_lo_recalls)) + goto out_put_stid; + + nfserr = ops->proc_layoutget(rqstp, d_inode(current_fh->fh_dentry), + current_fh, lgp); + if (nfserr) + goto out_put_stid; + + nfserr = nfsd4_insert_layout(lgp, ls); + +out_put_stid: + mutex_unlock(&ls->ls_mutex); + nfs4_put_stid(&ls->ls_stid); +out: + return nfserr; +} + +static void +nfsd4_layoutget_release(union nfsd4_op_u *u) +{ + kfree(u->layoutget.lg_content); +} + +static __be32 +nfsd4_layoutcommit(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, union nfsd4_op_u *u) +{ + struct net *net = SVC_NET(rqstp); + struct nfsd4_layoutcommit *lcp = &u->layoutcommit; + const struct nfsd4_layout_seg *seg = &lcp->lc_seg; + struct svc_fh *current_fh = &cstate->current_fh; + const struct nfsd4_layout_ops *ops; + struct inode *inode; + struct nfs4_layout_stateid *ls; + __be32 nfserr; + + nfserr = fh_verify(rqstp, current_fh, 0, + NFSD_MAY_WRITE | NFSD_MAY_OWNER_OVERRIDE); + if (nfserr) + goto out; + + nfserr = nfserr_layoutunavailable; + ops = nfsd4_layout_verify(current_fh->fh_export, lcp->lc_layout_type); + if (!ops) + goto out; + inode = d_inode(current_fh->fh_dentry); + + lcp->lc_size_chg = false; + if (lcp->lc_newoffset) { + loff_t new_size = lcp->lc_last_wr + 1; + + nfserr = nfserr_inval; + if (new_size <= seg->offset) + goto out; + if (new_size > seg->offset + seg->length) + goto out; + + if (new_size > i_size_read(inode)) { + lcp->lc_size_chg = true; + lcp->lc_newsize = new_size; + } + } + + nfserr = nfserr_grace; + if (locks_in_grace(net) && !lcp->lc_reclaim) + goto out; + nfserr = nfserr_no_grace; + if (!locks_in_grace(net) && lcp->lc_reclaim) + goto out; + + if (!lcp->lc_reclaim) { + nfserr = nfsd4_preprocess_layout_stateid(rqstp, cstate, + &lcp->lc_sid, false, lcp->lc_layout_type, &ls); + if (nfserr) { + trace_nfsd_layout_commit_lookup_fail(&lcp->lc_sid); + /* fixup error code as per RFC5661 */ + if (nfserr == nfserr_bad_stateid) + nfserr = nfserr_badlayout; + goto out; + } + + /* LAYOUTCOMMIT does not require any serialization */ + mutex_unlock(&ls->ls_mutex); + } + + nfserr = ops->proc_layoutcommit(inode, rqstp, lcp); + + if (!lcp->lc_reclaim) { + nfsd4_file_mark_deleg_written(ls->ls_stid.sc_file); + nfs4_put_stid(&ls->ls_stid); + } +out: + return nfserr; +} + +static __be32 +nfsd4_layoutreturn(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, union nfsd4_op_u *u) +{ + struct nfsd4_layoutreturn *lrp = &u->layoutreturn; + struct svc_fh *current_fh = &cstate->current_fh; + __be32 nfserr; + + nfserr = fh_verify(rqstp, current_fh, 0, NFSD_MAY_NOP); + if (nfserr) + goto out; + + nfserr = nfserr_layoutunavailable; + if (!nfsd4_layout_verify(current_fh->fh_export, lrp->lr_layout_type)) + goto out; + + switch (lrp->lr_seg.iomode) { + case IOMODE_READ: + case IOMODE_RW: + case IOMODE_ANY: + break; + default: + dprintk("%s: invalid iomode %d\n", __func__, + lrp->lr_seg.iomode); + nfserr = nfserr_inval; + goto out; + } + + switch (lrp->lr_return_type) { + case RETURN_FILE: + nfserr = nfsd4_return_file_layouts(rqstp, cstate, lrp); + break; + case RETURN_FSID: + case RETURN_ALL: + nfserr = nfsd4_return_client_layouts(rqstp, cstate, lrp); + break; + default: + dprintk("%s: invalid return_type %d\n", __func__, + lrp->lr_return_type); + nfserr = nfserr_inval; + break; + } +out: + return nfserr; +} +#endif /* CONFIG_NFSD_PNFS */ + +static __be32 +nfsd4_getxattr(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_getxattr *getxattr = &u->getxattr; + + return nfsd_getxattr(rqstp, &cstate->current_fh, + getxattr->getxa_name, &getxattr->getxa_buf, + &getxattr->getxa_len); +} + +static __be32 +nfsd4_setxattr(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_setxattr *setxattr = &u->setxattr; + __be32 ret; + + if (opens_in_grace(SVC_NET(rqstp))) + return nfserr_grace; + + ret = nfsd_setxattr(rqstp, &cstate->current_fh, setxattr->setxa_name, + setxattr->setxa_buf, setxattr->setxa_len, + setxattr->setxa_flags); + + if (!ret) + set_change_info(&setxattr->setxa_cinfo, &cstate->current_fh); + + return ret; +} + +static __be32 +nfsd4_listxattrs(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + /* + * Get the entire list, then copy out only the user attributes + * in the encode function. + */ + return nfsd_listxattr(rqstp, &cstate->current_fh, + &u->listxattrs.lsxa_buf, &u->listxattrs.lsxa_len); +} + +static __be32 +nfsd4_removexattr(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_removexattr *removexattr = &u->removexattr; + __be32 ret; + + if (opens_in_grace(SVC_NET(rqstp))) + return nfserr_grace; + + ret = nfsd_removexattr(rqstp, &cstate->current_fh, + removexattr->rmxa_name); + + if (!ret) + set_change_info(&removexattr->rmxa_cinfo, &cstate->current_fh); + + return ret; +} + +/* + * NULL call. + */ +static __be32 +nfsd4_proc_null(struct svc_rqst *rqstp) +{ + return rpc_success; +} + +static inline void nfsd4_increment_op_stats(struct nfsd_net *nn, u32 opnum) +{ + if (opnum >= FIRST_NFS4_OP && opnum <= LAST_NFS4_OP) + percpu_counter_inc(&nn->counter[NFSD_STATS_NFS4_OP(opnum)]); +} + +static const struct nfsd4_operation nfsd4_ops[]; + +static const char *nfsd4_op_name(unsigned opnum); + +/* + * Enforce NFSv4.1 COMPOUND ordering rules: + * + * Also note, enforced elsewhere: + * - SEQUENCE other than as first op results in + * NFS4ERR_SEQUENCE_POS. (Enforced in nfsd4_sequence().) + * - BIND_CONN_TO_SESSION must be the only op in its compound. + * (Enforced in nfsd4_bind_conn_to_session().) + * - DESTROY_SESSION must be the final operation in a compound, if + * sessionid's in SEQUENCE and DESTROY_SESSION are the same. + * (Enforced in nfsd4_destroy_session().) + */ +static __be32 nfs41_check_op_ordering(struct nfsd4_compoundargs *args) +{ + struct nfsd4_op *first_op = &args->ops[0]; + + /* These ordering requirements don't apply to NFSv4.0: */ + if (args->minorversion == 0) + return nfs_ok; + /* This is weird, but OK, not our problem: */ + if (args->opcnt == 0) + return nfs_ok; + if (first_op->status == nfserr_op_illegal) + return nfs_ok; + if (!(nfsd4_ops[first_op->opnum].op_flags & ALLOWED_AS_FIRST_OP)) + return nfserr_op_not_in_session; + if (first_op->opnum == OP_SEQUENCE) + return nfs_ok; + /* + * So first_op is something allowed outside a session, like + * EXCHANGE_ID; but then it has to be the only op in the + * compound: + */ + if (args->opcnt != 1) + return nfserr_not_only_op; + return nfs_ok; +} + +const struct nfsd4_operation *OPDESC(struct nfsd4_op *op) +{ + return &nfsd4_ops[op->opnum]; +} + +bool nfsd4_cache_this_op(struct nfsd4_op *op) +{ + if (op->opnum == OP_ILLEGAL) + return false; + return OPDESC(op)->op_flags & OP_CACHEME; +} + +static bool need_wrongsec_check(struct svc_rqst *rqstp) +{ + struct nfsd4_compoundres *resp = rqstp->rq_resp; + struct nfsd4_compoundargs *argp = rqstp->rq_argp; + struct nfsd4_op *this = &argp->ops[resp->opcnt - 1]; + struct nfsd4_op *next = &argp->ops[resp->opcnt]; + const struct nfsd4_operation *thisd = OPDESC(this); + const struct nfsd4_operation *nextd; + + /* + * Most ops check wronsec on our own; only the putfh-like ops + * have special rules. + */ + if (!(thisd->op_flags & OP_IS_PUTFH_LIKE)) + return false; + /* + * rfc 5661 2.6.3.1.1.6: don't bother erroring out a + * put-filehandle operation if we're not going to use the + * result: + */ + if (argp->opcnt == resp->opcnt) + return false; + if (next->opnum == OP_ILLEGAL) + return false; + nextd = OPDESC(next); + /* + * Rest of 2.6.3.1.1: certain operations will return WRONGSEC + * errors themselves as necessary; others should check for them + * now: + */ + return !(nextd->op_flags & OP_HANDLES_WRONGSEC); +} + +#ifdef CONFIG_NFSD_V4_2_INTER_SSC +static void +check_if_stalefh_allowed(struct nfsd4_compoundargs *args) +{ + struct nfsd4_op *op, *current_op = NULL, *saved_op = NULL; + struct nfsd4_copy *copy; + struct nfsd4_putfh *putfh; + int i; + + /* traverse all operation and if it's a COPY compound, mark the + * source filehandle to skip verification + */ + for (i = 0; i < args->opcnt; i++) { + op = &args->ops[i]; + if (op->opnum == OP_PUTFH) + current_op = op; + else if (op->opnum == OP_SAVEFH) + saved_op = current_op; + else if (op->opnum == OP_RESTOREFH) + current_op = saved_op; + else if (op->opnum == OP_COPY) { + copy = (struct nfsd4_copy *)&op->u; + if (!saved_op) { + op->status = nfserr_nofilehandle; + return; + } + putfh = (struct nfsd4_putfh *)&saved_op->u; + if (nfsd4_ssc_is_inter(copy)) + putfh->no_verify = true; + } + } +} +#else +static void +check_if_stalefh_allowed(struct nfsd4_compoundargs *args) +{ +} +#endif + +/* + * COMPOUND call. + */ +static __be32 +nfsd4_proc_compound(struct svc_rqst *rqstp) +{ + struct nfsd4_compoundargs *args = rqstp->rq_argp; + struct nfsd4_compoundres *resp = rqstp->rq_resp; + struct nfsd4_op *op; + struct nfsd4_compound_state *cstate = &resp->cstate; + struct svc_fh *current_fh = &cstate->current_fh; + struct svc_fh *save_fh = &cstate->save_fh; + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + struct nfsd_thread_local_info *ntli = rqstp->rq_private; + __be32 status; + + resp->xdr = &rqstp->rq_res_stream; + resp->statusp = resp->xdr->p; + + /* reserve space for: NFS status code */ + xdr_reserve_space(resp->xdr, XDR_UNIT); + + /* reserve space for: taglen, tag, and opcnt */ + xdr_reserve_space(resp->xdr, XDR_UNIT * 2 + args->taglen); + resp->taglen = args->taglen; + resp->tag = args->tag; + resp->rqstp = rqstp; + cstate->minorversion = args->minorversion; + fh_init(current_fh, NFS4_FHSIZE); + fh_init(save_fh, NFS4_FHSIZE); + /* + * Don't use the deferral mechanism for NFSv4; compounds make it + * too hard to avoid non-idempotency problems. + */ + clear_bit(RQ_USEDEFERRAL, &rqstp->rq_flags); + + /* + * According to RFC3010, this takes precedence over all other errors. + */ + status = nfserr_minor_vers_mismatch; + if (nfsd_minorversion(nn, args->minorversion, NFSD_TEST) <= 0) + goto out; + + status = nfs41_check_op_ordering(args); + if (status) { + op = &args->ops[0]; + op->status = status; + resp->opcnt = 1; + goto encode_op; + } + check_if_stalefh_allowed(args); + + ntli->ntli_lease_breaker = &cstate->clp; + + trace_nfsd_compound(rqstp, args->tag, args->taglen, args->client_opcnt); + while (!status && resp->opcnt < args->opcnt) { + op = &args->ops[resp->opcnt++]; + + if (unlikely(resp->opcnt == NFSD_MAX_OPS_PER_COMPOUND)) { + /* If there are still more operations to process, + * stop here and report NFS4ERR_RESOURCE. */ + if (cstate->minorversion == 0 && + args->client_opcnt > resp->opcnt) { + op->status = nfserr_resource; + goto encode_op; + } + } + + /* + * The XDR decode routines may have pre-set op->status; + * for example, if there is a miscellaneous XDR error + * it will be set to nfserr_bad_xdr. + */ + if (op->status) { + if (op->opnum == OP_OPEN) + op->status = nfsd4_open_omfg(rqstp, cstate, op); + goto encode_op; + } + if (!current_fh->fh_dentry) { + if (HAS_FH_FLAG(current_fh, NFSD4_FH_FOREIGN)) { + /* + * FOREIGN fh from inter-SSC PUTFH: only + * SAVEFH may proceed with a NULL fh_dentry. + * Per RFC 7862 S15.2.3, validation of a + * foreign fh is deferred to the operation + * that consumes it, and NFS4ERR_STALE is + * returned at that point. + */ + if (op->opnum != OP_SAVEFH && + !(op->opdesc->op_flags & ALLOWED_WITHOUT_FH)) { + op->status = nfserr_stale; + goto encode_op; + } + } else if (!(op->opdesc->op_flags & ALLOWED_WITHOUT_FH)) { + op->status = nfserr_nofilehandle; + goto encode_op; + } + } else if (current_fh->fh_export && + current_fh->fh_export->ex_fslocs.migrated && + !(op->opdesc->op_flags & ALLOWED_ON_ABSENT_FS)) { + op->status = nfserr_moved; + goto encode_op; + } + + fh_clear_pre_post_attrs(current_fh); + + /* If op is non-idempotent */ + if (op->opdesc->op_flags & OP_MODIFIES_SOMETHING) { + /* + * Don't execute this op if we couldn't encode a + * successful reply: + */ + u32 plen = op->opdesc->op_rsize_bop(rqstp, op); + /* + * Plus if there's another operation, make sure + * we'll have space to at least encode an error: + */ + if (resp->opcnt < args->opcnt) + plen += COMPOUND_ERR_SLACK_SPACE; + op->status = nfsd4_check_resp_size(resp, plen); + } + + if (op->status) + goto encode_op; + + if (op->opdesc->op_get_currentstateid) + op->opdesc->op_get_currentstateid(cstate, &op->u); + op->status = op->opdesc->op_func(rqstp, cstate, &op->u); + trace_nfsd_compound_op_err(rqstp, op->opnum, op->status); + + /* Only from SEQUENCE */ + if (cstate->status == nfserr_replay_cache) { + dprintk("%s NFS4.1 replay from cache\n", __func__); + status = op->status; + goto out; + } + if (!op->status) { + if (op->opdesc->op_set_currentstateid) + op->opdesc->op_set_currentstateid(cstate, &op->u); + + if (op->opdesc->op_flags & OP_CLEAR_STATEID) + clear_current_stateid(cstate); + + if (current_fh->fh_export && + need_wrongsec_check(rqstp)) + op->status = check_nfsd_access(current_fh->fh_export, rqstp, false); + } +encode_op: + if (op->status == nfserr_replay_me) { + op->replay = &cstate->replay_owner->so_replay; + nfsd4_encode_replay(resp->xdr, op); + status = op->status = op->replay->rp_status; + } else { + nfsd4_encode_operation(resp, op); + status = op->status; + } + + if (op->opdesc && op->opdesc->op_release) + op->opdesc->op_release(&op->u); + + trace_nfsd_compound_status(args->client_opcnt, resp->opcnt, + status, nfsd4_op_name(op->opnum)); + + nfsd4_cstate_clear_replay(cstate); + nfsd4_increment_op_stats(nn, op->opnum); + } + + fh_put(current_fh); + fh_put(save_fh); + BUG_ON(cstate->replay_owner); +out: + cstate->status = status; + return rpc_success; +} + +#define op_encode_hdr_size (2) +#define op_encode_stateid_maxsz (XDR_QUADLEN(NFS4_STATEID_SIZE)) +#define op_encode_verifier_maxsz (XDR_QUADLEN(NFS4_VERIFIER_SIZE)) +#define op_encode_change_info_maxsz (5) +#define nfs4_fattr_bitmap_maxsz (4) + +/* We'll fall back on returning no lockowner if run out of space: */ +#define op_encode_lockowner_maxsz (0) +#define op_encode_lock_denied_maxsz (8 + op_encode_lockowner_maxsz) + +#define nfs4_owner_maxsz (1 + XDR_QUADLEN(IDMAP_NAMESZ)) + +#define op_encode_ace_maxsz (3 + nfs4_owner_maxsz) +#define op_encode_delegation_maxsz (1 + op_encode_stateid_maxsz + 1 + \ + op_encode_ace_maxsz) + +#define op_encode_channel_attrs_maxsz (6 + 1 + 1) + +/* + * The _rsize() helpers are invoked by the NFSv4 COMPOUND decoder, which + * is called before sunrpc sets rq_res.buflen. Thus we have to compute + * the maximum payload size here, based on transport limits and the size + * of the remaining space in the rq_pages array. + */ +static u32 nfsd4_max_payload(const struct svc_rqst *rqstp) +{ + u32 buflen; + + buflen = (rqstp->rq_page_end - rqstp->rq_next_page) * PAGE_SIZE; + buflen -= rqstp->rq_auth_slack; + buflen -= rqstp->rq_res.head[0].iov_len; + return min_t(u32, buflen, svc_max_payload(rqstp)); +} + +static u32 nfsd4_only_status_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size) * sizeof(__be32); +} + +static u32 nfsd4_status_stateid_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + op_encode_stateid_maxsz)* sizeof(__be32); +} + +static u32 nfsd4_access_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + /* ac_supported, ac_resp_access */ + return (op_encode_hdr_size + 2)* sizeof(__be32); +} + +static u32 nfsd4_commit_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + op_encode_verifier_maxsz) * sizeof(__be32); +} + +static u32 nfsd4_create_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + op_encode_change_info_maxsz + + nfs4_fattr_bitmap_maxsz) * sizeof(__be32); +} + +/* + * Note since this is an idempotent operation we won't insist on failing + * the op prematurely if the estimate is too large. We may turn off splice + * reads unnecessarily. + */ +static u32 nfsd4_getattr_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + const u32 *bmap = op->u.getattr.ga_bmval; + u32 bmap0 = bmap[0], bmap1 = bmap[1], bmap2 = bmap[2]; + u32 ret = 0; + + if (bmap0 & FATTR4_WORD0_ACL) + return nfsd4_max_payload(rqstp); + if (bmap0 & FATTR4_WORD0_FS_LOCATIONS) + return nfsd4_max_payload(rqstp); + + if (bmap1 & FATTR4_WORD1_OWNER) { + ret += IDMAP_NAMESZ + 4; + bmap1 &= ~FATTR4_WORD1_OWNER; + } + if (bmap1 & FATTR4_WORD1_OWNER_GROUP) { + ret += IDMAP_NAMESZ + 4; + bmap1 &= ~FATTR4_WORD1_OWNER_GROUP; + } + if (bmap0 & FATTR4_WORD0_FILEHANDLE) { + ret += NFS4_FHSIZE + 4; + bmap0 &= ~FATTR4_WORD0_FILEHANDLE; + } + if (bmap2 & FATTR4_WORD2_SECURITY_LABEL) { + ret += NFS4_MAXLABELLEN + 12; + bmap2 &= ~FATTR4_WORD2_SECURITY_LABEL; + } + /* + * Largest of remaining attributes are 16 bytes (e.g., + * supported_attributes) + */ + ret += 16 * (hweight32(bmap0) + hweight32(bmap1) + hweight32(bmap2)); + /* bitmask, length */ + ret += 20; + return ret; +} + +static u32 nfsd4_getfh_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + 1) * sizeof(__be32) + NFS4_FHSIZE; +} + +static u32 nfsd4_link_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + op_encode_change_info_maxsz) + * sizeof(__be32); +} + +static u32 nfsd4_lock_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + op_encode_lock_denied_maxsz) + * sizeof(__be32); +} + +static u32 nfsd4_open_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + op_encode_stateid_maxsz + + op_encode_change_info_maxsz + 1 + + nfs4_fattr_bitmap_maxsz + + op_encode_delegation_maxsz) * sizeof(__be32); +} + +static u32 nfsd4_read_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + u32 rlen = min(op->u.read.rd_length, nfsd4_max_payload(rqstp)); + + return (op_encode_hdr_size + 2 + XDR_QUADLEN(rlen)) * sizeof(__be32); +} + +static u32 nfsd4_read_plus_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + u32 rlen = min(op->u.read.rd_length, nfsd4_max_payload(rqstp)); + /* + * If we detect that the file changed during hole encoding, then we + * recover by encoding the remaining reply as data. This means we need + * to set aside enough room to encode two data segments. + */ + u32 seg_len = 2 * (1 + 2 + 1); + + return (op_encode_hdr_size + 2 + seg_len + XDR_QUADLEN(rlen)) * sizeof(__be32); +} + +static u32 nfsd4_readdir_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + u32 rlen = min(op->u.readdir.rd_maxcount, nfsd4_max_payload(rqstp)); + + return (op_encode_hdr_size + op_encode_verifier_maxsz + + XDR_QUADLEN(rlen)) * sizeof(__be32); +} + +static u32 nfsd4_readlink_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + 1) * sizeof(__be32) + PAGE_SIZE; +} + +static u32 nfsd4_remove_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + op_encode_change_info_maxsz) + * sizeof(__be32); +} + +static u32 nfsd4_rename_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + op_encode_change_info_maxsz + + op_encode_change_info_maxsz) * sizeof(__be32); +} + +static u32 nfsd4_sequence_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + + XDR_QUADLEN(NFS4_MAX_SESSIONID_LEN) + 5) * sizeof(__be32); +} + +static u32 nfsd4_test_stateid_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + 1 + op->u.test_stateid.ts_num_ids) + * sizeof(__be32); +} + +static u32 nfsd4_setattr_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + nfs4_fattr_bitmap_maxsz) * sizeof(__be32); +} + +static u32 nfsd4_secinfo_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + RPC_AUTH_MAXFLAVOR * + (4 + XDR_QUADLEN(GSS_OID_MAX_LEN))) * sizeof(__be32); +} + +static u32 nfsd4_setclientid_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + 2 + XDR_QUADLEN(NFS4_VERIFIER_SIZE)) * + sizeof(__be32); +} + +static u32 nfsd4_write_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + 2 + op_encode_verifier_maxsz) * sizeof(__be32); +} + +static u32 nfsd4_exchange_id_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + 2 + 1 + /* eir_clientid, eir_sequenceid */\ + 1 + 1 + /* eir_flags, spr_how */\ + 4 + /* spo_must_enforce & _allow with bitmap */\ + 2 + /*eir_server_owner.so_minor_id */\ + /* eir_server_owner.so_major_id<> */\ + XDR_QUADLEN(NFS4_OPAQUE_LIMIT) + 1 +\ + /* eir_server_scope<> */\ + XDR_QUADLEN(NFS4_OPAQUE_LIMIT) + 1 +\ + 1 + /* eir_server_impl_id array length */\ + 0 /* ignored eir_server_impl_id contents */) * sizeof(__be32); +} + +static u32 nfsd4_bind_conn_to_session_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + \ + XDR_QUADLEN(NFS4_MAX_SESSIONID_LEN) + /* bctsr_sessid */\ + 2 /* bctsr_dir, use_conn_in_rdma_mode */) * sizeof(__be32); +} + +static u32 nfsd4_create_session_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + \ + XDR_QUADLEN(NFS4_MAX_SESSIONID_LEN) + /* sessionid */\ + 2 + /* csr_sequence, csr_flags */\ + op_encode_channel_attrs_maxsz + \ + op_encode_channel_attrs_maxsz) * sizeof(__be32); +} + +static u32 nfsd4_copy_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + + 1 /* wr_callback */ + + op_encode_stateid_maxsz /* wr_callback */ + + 2 /* wr_count */ + + 1 /* wr_committed */ + + op_encode_verifier_maxsz + + 1 /* cr_consecutive */ + + 1 /* cr_synchronous */) * sizeof(__be32); +} + +static u32 nfsd4_offload_status_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + + 2 /* osr_count */ + + 1 /* osr_complete<1> optional 0 for now */) * sizeof(__be32); +} + +static u32 nfsd4_copy_notify_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + + 3 /* cnr_lease_time */ + + 1 /* We support one cnr_source_server */ + + 1 /* cnr_stateid seq */ + + op_encode_stateid_maxsz /* cnr_stateid */ + + 1 /* num cnr_source_server*/ + + 1 /* nl4_type */ + + 1 /* nl4 size */ + + XDR_QUADLEN(NFS4_OPAQUE_LIMIT) /*nl4_loc + nl4_loc_sz */) + * sizeof(__be32); +} + +static u32 nfsd4_get_dir_delegation_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + + 1 /* gddr_status */ + + op_encode_verifier_maxsz + + op_encode_stateid_maxsz + + 2 /* gddr_notification */ + + 3 /* gddr_child_attributes */ + + 3 /* gddr_dir_attributes */) * sizeof(__be32); +} + +#ifdef CONFIG_NFSD_PNFS +static u32 nfsd4_getdeviceinfo_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + u32 rlen = min(op->u.getdeviceinfo.gd_maxcount, nfsd4_max_payload(rqstp)); + + return (op_encode_hdr_size + + 1 /* gd_layout_type*/ + + XDR_QUADLEN(rlen) + + 2 /* gd_notify_types */) * sizeof(__be32); +} + +/* + * At this stage we don't really know what layout driver will handle the request, + * so we need to define an arbitrary upper bound here. + */ +#define MAX_LAYOUT_SIZE 128 +static u32 nfsd4_layoutget_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + + 1 /* logr_return_on_close */ + + op_encode_stateid_maxsz + + 1 /* nr of layouts */ + + MAX_LAYOUT_SIZE) * sizeof(__be32); +} + +static u32 nfsd4_layoutcommit_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + + 1 /* locr_newsize */ + + 2 /* ns_size */) * sizeof(__be32); +} + +static u32 nfsd4_layoutreturn_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + + 1 /* lrs_stateid */ + + op_encode_stateid_maxsz) * sizeof(__be32); +} +#endif /* CONFIG_NFSD_PNFS */ + + +static u32 nfsd4_seek_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + 3) * sizeof(__be32); +} + +static u32 nfsd4_getxattr_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + u32 rlen = min_t(u32, XATTR_SIZE_MAX, nfsd4_max_payload(rqstp)); + + return (op_encode_hdr_size + 1 + XDR_QUADLEN(rlen)) * sizeof(__be32); +} + +static u32 nfsd4_setxattr_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + op_encode_change_info_maxsz) + * sizeof(__be32); +} +static u32 nfsd4_listxattrs_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + u32 rlen = min(op->u.listxattrs.lsxa_maxcount, nfsd4_max_payload(rqstp)); + + return (op_encode_hdr_size + 4 + XDR_QUADLEN(rlen)) * sizeof(__be32); +} + +static u32 nfsd4_removexattr_rsize(const struct svc_rqst *rqstp, + const struct nfsd4_op *op) +{ + return (op_encode_hdr_size + op_encode_change_info_maxsz) + * sizeof(__be32); +} + + +static const struct nfsd4_operation nfsd4_ops[] = { + [OP_ACCESS] = { + .op_func = nfsd4_access, + .op_name = "OP_ACCESS", + .op_rsize_bop = nfsd4_access_rsize, + }, + [OP_CLOSE] = { + .op_func = nfsd4_close, + .op_flags = OP_MODIFIES_SOMETHING, + .op_name = "OP_CLOSE", + .op_rsize_bop = nfsd4_status_stateid_rsize, + .op_get_currentstateid = nfsd4_get_closestateid, + .op_set_currentstateid = nfsd4_set_closestateid, + }, + [OP_COMMIT] = { + .op_func = nfsd4_commit, + .op_flags = OP_MODIFIES_SOMETHING, + .op_name = "OP_COMMIT", + .op_rsize_bop = nfsd4_commit_rsize, + }, + [OP_CREATE] = { + .op_func = nfsd4_create, + .op_flags = OP_MODIFIES_SOMETHING | OP_CACHEME | OP_CLEAR_STATEID, + .op_name = "OP_CREATE", + .op_rsize_bop = nfsd4_create_rsize, + }, + [OP_DELEGRETURN] = { + .op_func = nfsd4_delegreturn, + .op_flags = OP_MODIFIES_SOMETHING, + .op_name = "OP_DELEGRETURN", + .op_rsize_bop = nfsd4_only_status_rsize, + .op_get_currentstateid = nfsd4_get_delegreturnstateid, + }, + [OP_GETATTR] = { + .op_func = nfsd4_getattr, + .op_flags = ALLOWED_ON_ABSENT_FS, + .op_rsize_bop = nfsd4_getattr_rsize, + .op_name = "OP_GETATTR", + }, + [OP_GETFH] = { + .op_func = nfsd4_getfh, + .op_name = "OP_GETFH", + .op_rsize_bop = nfsd4_getfh_rsize, + }, + [OP_LINK] = { + .op_func = nfsd4_link, + .op_flags = ALLOWED_ON_ABSENT_FS | OP_MODIFIES_SOMETHING + | OP_CACHEME, + .op_name = "OP_LINK", + .op_rsize_bop = nfsd4_link_rsize, + }, + [OP_LOCK] = { + .op_func = nfsd4_lock, + .op_release = nfsd4_lock_release, + .op_flags = OP_MODIFIES_SOMETHING | + OP_NONTRIVIAL_ERROR_ENCODE, + .op_name = "OP_LOCK", + .op_rsize_bop = nfsd4_lock_rsize, + .op_set_currentstateid = nfsd4_set_lockstateid, + }, + [OP_LOCKT] = { + .op_func = nfsd4_lockt, + .op_release = nfsd4_lockt_release, + .op_flags = OP_NONTRIVIAL_ERROR_ENCODE, + .op_name = "OP_LOCKT", + .op_rsize_bop = nfsd4_lock_rsize, + }, + [OP_LOCKU] = { + .op_func = nfsd4_locku, + .op_flags = OP_MODIFIES_SOMETHING, + .op_name = "OP_LOCKU", + .op_rsize_bop = nfsd4_status_stateid_rsize, + .op_get_currentstateid = nfsd4_get_lockustateid, + }, + [OP_LOOKUP] = { + .op_func = nfsd4_lookup, + .op_flags = OP_HANDLES_WRONGSEC | OP_CLEAR_STATEID, + .op_name = "OP_LOOKUP", + .op_rsize_bop = nfsd4_only_status_rsize, + }, + [OP_LOOKUPP] = { + .op_func = nfsd4_lookupp, + .op_flags = OP_HANDLES_WRONGSEC | OP_CLEAR_STATEID, + .op_name = "OP_LOOKUPP", + .op_rsize_bop = nfsd4_only_status_rsize, + }, + [OP_NVERIFY] = { + .op_func = nfsd4_nverify, + .op_name = "OP_NVERIFY", + .op_rsize_bop = nfsd4_only_status_rsize, + }, + [OP_OPEN] = { + .op_func = nfsd4_open, + .op_release = nfsd4_open_release, + .op_flags = OP_HANDLES_WRONGSEC | OP_MODIFIES_SOMETHING, + .op_name = "OP_OPEN", + .op_rsize_bop = nfsd4_open_rsize, + .op_set_currentstateid = nfsd4_set_openstateid, + }, + [OP_OPEN_CONFIRM] = { + .op_func = nfsd4_open_confirm, + .op_flags = OP_MODIFIES_SOMETHING, + .op_name = "OP_OPEN_CONFIRM", + .op_rsize_bop = nfsd4_status_stateid_rsize, + }, + [OP_OPEN_DOWNGRADE] = { + .op_func = nfsd4_open_downgrade, + .op_flags = OP_MODIFIES_SOMETHING, + .op_name = "OP_OPEN_DOWNGRADE", + .op_rsize_bop = nfsd4_status_stateid_rsize, + .op_get_currentstateid = nfsd4_get_opendowngradestateid, + .op_set_currentstateid = nfsd4_set_opendowngradestateid, + }, + [OP_PUTFH] = { + .op_func = nfsd4_putfh, + .op_flags = ALLOWED_WITHOUT_FH | ALLOWED_ON_ABSENT_FS + | OP_IS_PUTFH_LIKE | OP_CLEAR_STATEID, + .op_name = "OP_PUTFH", + .op_rsize_bop = nfsd4_only_status_rsize, + }, + [OP_PUTPUBFH] = { + .op_func = nfsd4_putrootfh, + .op_flags = ALLOWED_WITHOUT_FH | ALLOWED_ON_ABSENT_FS + | OP_IS_PUTFH_LIKE | OP_CLEAR_STATEID, + .op_name = "OP_PUTPUBFH", + .op_rsize_bop = nfsd4_only_status_rsize, + }, + [OP_PUTROOTFH] = { + .op_func = nfsd4_putrootfh, + .op_flags = ALLOWED_WITHOUT_FH | ALLOWED_ON_ABSENT_FS + | OP_IS_PUTFH_LIKE | OP_CLEAR_STATEID, + .op_name = "OP_PUTROOTFH", + .op_rsize_bop = nfsd4_only_status_rsize, + }, + [OP_READ] = { + .op_func = nfsd4_read, + .op_release = nfsd4_read_release, + .op_name = "OP_READ", + .op_rsize_bop = nfsd4_read_rsize, + .op_get_currentstateid = nfsd4_get_readstateid, + }, + [OP_READDIR] = { + .op_func = nfsd4_readdir, + .op_name = "OP_READDIR", + .op_rsize_bop = nfsd4_readdir_rsize, + }, + [OP_READLINK] = { + .op_func = nfsd4_readlink, + .op_name = "OP_READLINK", + .op_rsize_bop = nfsd4_readlink_rsize, + }, + [OP_REMOVE] = { + .op_func = nfsd4_remove, + .op_flags = OP_MODIFIES_SOMETHING | OP_CACHEME, + .op_name = "OP_REMOVE", + .op_rsize_bop = nfsd4_remove_rsize, + }, + [OP_RENAME] = { + .op_func = nfsd4_rename, + .op_flags = OP_MODIFIES_SOMETHING | OP_CACHEME, + .op_name = "OP_RENAME", + .op_rsize_bop = nfsd4_rename_rsize, + }, + [OP_RENEW] = { + .op_func = nfsd4_renew, + .op_flags = ALLOWED_WITHOUT_FH | ALLOWED_ON_ABSENT_FS + | OP_MODIFIES_SOMETHING, + .op_name = "OP_RENEW", + .op_rsize_bop = nfsd4_only_status_rsize, + + }, + [OP_RESTOREFH] = { + .op_func = nfsd4_restorefh, + .op_flags = ALLOWED_WITHOUT_FH | ALLOWED_ON_ABSENT_FS + | OP_IS_PUTFH_LIKE | OP_MODIFIES_SOMETHING, + .op_name = "OP_RESTOREFH", + .op_rsize_bop = nfsd4_only_status_rsize, + }, + [OP_SAVEFH] = { + .op_func = nfsd4_savefh, + .op_flags = OP_HANDLES_WRONGSEC | OP_MODIFIES_SOMETHING, + .op_name = "OP_SAVEFH", + .op_rsize_bop = nfsd4_only_status_rsize, + }, + [OP_SECINFO] = { + .op_func = nfsd4_secinfo, + .op_release = nfsd4_secinfo_release, + .op_flags = OP_HANDLES_WRONGSEC, + .op_name = "OP_SECINFO", + .op_rsize_bop = nfsd4_secinfo_rsize, + }, + [OP_SETATTR] = { + .op_func = nfsd4_setattr, + .op_name = "OP_SETATTR", + .op_flags = OP_MODIFIES_SOMETHING | OP_CACHEME + | OP_NONTRIVIAL_ERROR_ENCODE, + .op_rsize_bop = nfsd4_setattr_rsize, + .op_get_currentstateid = nfsd4_get_setattrstateid, + }, + [OP_SETCLIENTID] = { + .op_func = nfsd4_setclientid, + .op_flags = ALLOWED_WITHOUT_FH | ALLOWED_ON_ABSENT_FS + | OP_MODIFIES_SOMETHING | OP_CACHEME + | OP_NONTRIVIAL_ERROR_ENCODE, + .op_name = "OP_SETCLIENTID", + .op_rsize_bop = nfsd4_setclientid_rsize, + }, + [OP_SETCLIENTID_CONFIRM] = { + .op_func = nfsd4_setclientid_confirm, + .op_flags = ALLOWED_WITHOUT_FH | ALLOWED_ON_ABSENT_FS + | OP_MODIFIES_SOMETHING | OP_CACHEME, + .op_name = "OP_SETCLIENTID_CONFIRM", + .op_rsize_bop = nfsd4_only_status_rsize, + }, + [OP_VERIFY] = { + .op_func = nfsd4_verify, + .op_name = "OP_VERIFY", + .op_rsize_bop = nfsd4_only_status_rsize, + }, + [OP_WRITE] = { + .op_func = nfsd4_write, + .op_flags = OP_MODIFIES_SOMETHING | OP_CACHEME, + .op_name = "OP_WRITE", + .op_rsize_bop = nfsd4_write_rsize, + .op_get_currentstateid = nfsd4_get_writestateid, + }, + [OP_RELEASE_LOCKOWNER] = { + .op_func = nfsd4_release_lockowner, + .op_flags = ALLOWED_WITHOUT_FH | ALLOWED_ON_ABSENT_FS + | OP_MODIFIES_SOMETHING, + .op_name = "OP_RELEASE_LOCKOWNER", + .op_rsize_bop = nfsd4_only_status_rsize, + }, + + /* NFSv4.1 operations */ + [OP_EXCHANGE_ID] = { + .op_func = nfsd4_exchange_id, + .op_release = nfsd4_exchange_id_release, + .op_flags = ALLOWED_WITHOUT_FH | ALLOWED_AS_FIRST_OP + | OP_MODIFIES_SOMETHING, + .op_name = "OP_EXCHANGE_ID", + .op_rsize_bop = nfsd4_exchange_id_rsize, + }, + [OP_BACKCHANNEL_CTL] = { + .op_func = nfsd4_backchannel_ctl, + .op_flags = ALLOWED_WITHOUT_FH | OP_MODIFIES_SOMETHING, + .op_name = "OP_BACKCHANNEL_CTL", + .op_rsize_bop = nfsd4_only_status_rsize, + }, + [OP_BIND_CONN_TO_SESSION] = { + .op_func = nfsd4_bind_conn_to_session, + .op_flags = ALLOWED_WITHOUT_FH | ALLOWED_AS_FIRST_OP + | OP_MODIFIES_SOMETHING, + .op_name = "OP_BIND_CONN_TO_SESSION", + .op_rsize_bop = nfsd4_bind_conn_to_session_rsize, + }, + [OP_CREATE_SESSION] = { + .op_func = nfsd4_create_session, + .op_flags = ALLOWED_WITHOUT_FH | ALLOWED_AS_FIRST_OP + | OP_MODIFIES_SOMETHING, + .op_name = "OP_CREATE_SESSION", + .op_rsize_bop = nfsd4_create_session_rsize, + }, + [OP_DESTROY_SESSION] = { + .op_func = nfsd4_destroy_session, + .op_flags = ALLOWED_WITHOUT_FH | ALLOWED_AS_FIRST_OP + | OP_MODIFIES_SOMETHING, + .op_name = "OP_DESTROY_SESSION", + .op_rsize_bop = nfsd4_only_status_rsize, + }, + [OP_SEQUENCE] = { + .op_func = nfsd4_sequence, + .op_flags = ALLOWED_WITHOUT_FH | ALLOWED_AS_FIRST_OP, + .op_name = "OP_SEQUENCE", + .op_rsize_bop = nfsd4_sequence_rsize, + }, + [OP_DESTROY_CLIENTID] = { + .op_func = nfsd4_destroy_clientid, + .op_flags = ALLOWED_WITHOUT_FH | ALLOWED_AS_FIRST_OP + | OP_MODIFIES_SOMETHING, + .op_name = "OP_DESTROY_CLIENTID", + .op_rsize_bop = nfsd4_only_status_rsize, + }, + [OP_RECLAIM_COMPLETE] = { + .op_func = nfsd4_reclaim_complete, + .op_flags = ALLOWED_WITHOUT_FH | OP_MODIFIES_SOMETHING, + .op_name = "OP_RECLAIM_COMPLETE", + .op_rsize_bop = nfsd4_only_status_rsize, + }, + [OP_SECINFO_NO_NAME] = { + .op_func = nfsd4_secinfo_no_name, + .op_release = nfsd4_secinfo_no_name_release, + .op_flags = OP_HANDLES_WRONGSEC, + .op_name = "OP_SECINFO_NO_NAME", + .op_rsize_bop = nfsd4_secinfo_rsize, + }, + [OP_TEST_STATEID] = { + .op_func = nfsd4_test_stateid, + .op_flags = ALLOWED_WITHOUT_FH, + .op_name = "OP_TEST_STATEID", + .op_rsize_bop = nfsd4_test_stateid_rsize, + }, + [OP_FREE_STATEID] = { + .op_func = nfsd4_free_stateid, + .op_flags = ALLOWED_WITHOUT_FH | OP_MODIFIES_SOMETHING, + .op_name = "OP_FREE_STATEID", + .op_get_currentstateid = nfsd4_get_freestateid, + .op_rsize_bop = nfsd4_only_status_rsize, + }, + [OP_GET_DIR_DELEGATION] = { + .op_func = nfsd4_get_dir_delegation, + .op_flags = OP_MODIFIES_SOMETHING, + .op_name = "OP_GET_DIR_DELEGATION", + .op_rsize_bop = nfsd4_get_dir_delegation_rsize, + }, +#ifdef CONFIG_NFSD_PNFS + [OP_GETDEVICEINFO] = { + .op_func = nfsd4_getdeviceinfo, + .op_release = nfsd4_getdeviceinfo_release, + .op_flags = ALLOWED_WITHOUT_FH, + .op_name = "OP_GETDEVICEINFO", + .op_rsize_bop = nfsd4_getdeviceinfo_rsize, + }, + [OP_LAYOUTGET] = { + .op_func = nfsd4_layoutget, + .op_release = nfsd4_layoutget_release, + .op_flags = OP_MODIFIES_SOMETHING, + .op_name = "OP_LAYOUTGET", + .op_rsize_bop = nfsd4_layoutget_rsize, + }, + [OP_LAYOUTCOMMIT] = { + .op_func = nfsd4_layoutcommit, + .op_flags = OP_MODIFIES_SOMETHING, + .op_name = "OP_LAYOUTCOMMIT", + .op_rsize_bop = nfsd4_layoutcommit_rsize, + }, + [OP_LAYOUTRETURN] = { + .op_func = nfsd4_layoutreturn, + .op_flags = OP_MODIFIES_SOMETHING, + .op_name = "OP_LAYOUTRETURN", + .op_rsize_bop = nfsd4_layoutreturn_rsize, + }, +#endif /* CONFIG_NFSD_PNFS */ + + /* NFSv4.2 operations */ + [OP_ALLOCATE] = { + .op_func = nfsd4_allocate, + .op_flags = OP_MODIFIES_SOMETHING, + .op_name = "OP_ALLOCATE", + .op_rsize_bop = nfsd4_only_status_rsize, + }, + [OP_DEALLOCATE] = { + .op_func = nfsd4_deallocate, + .op_flags = OP_MODIFIES_SOMETHING, + .op_name = "OP_DEALLOCATE", + .op_rsize_bop = nfsd4_only_status_rsize, + }, + [OP_CLONE] = { + .op_func = nfsd4_clone, + .op_flags = OP_MODIFIES_SOMETHING, + .op_name = "OP_CLONE", + .op_rsize_bop = nfsd4_only_status_rsize, + }, + [OP_COPY] = { + .op_func = nfsd4_copy, + .op_flags = OP_MODIFIES_SOMETHING, + .op_name = "OP_COPY", + .op_rsize_bop = nfsd4_copy_rsize, + }, + [OP_READ_PLUS] = { + .op_func = nfsd4_read, + .op_release = nfsd4_read_release, + .op_name = "OP_READ_PLUS", + .op_rsize_bop = nfsd4_read_plus_rsize, + .op_get_currentstateid = nfsd4_get_readstateid, + }, + [OP_SEEK] = { + .op_func = nfsd4_seek, + .op_name = "OP_SEEK", + .op_rsize_bop = nfsd4_seek_rsize, + }, + [OP_OFFLOAD_STATUS] = { + .op_func = nfsd4_offload_status, + .op_name = "OP_OFFLOAD_STATUS", + .op_rsize_bop = nfsd4_offload_status_rsize, + }, + [OP_OFFLOAD_CANCEL] = { + .op_func = nfsd4_offload_cancel, + .op_flags = OP_MODIFIES_SOMETHING, + .op_name = "OP_OFFLOAD_CANCEL", + .op_rsize_bop = nfsd4_only_status_rsize, + }, + [OP_COPY_NOTIFY] = { + .op_func = nfsd4_copy_notify, + .op_flags = OP_MODIFIES_SOMETHING, + .op_name = "OP_COPY_NOTIFY", + .op_rsize_bop = nfsd4_copy_notify_rsize, + }, + [OP_GETXATTR] = { + .op_func = nfsd4_getxattr, + .op_name = "OP_GETXATTR", + .op_rsize_bop = nfsd4_getxattr_rsize, + }, + [OP_SETXATTR] = { + .op_func = nfsd4_setxattr, + .op_flags = OP_MODIFIES_SOMETHING | OP_CACHEME, + .op_name = "OP_SETXATTR", + .op_rsize_bop = nfsd4_setxattr_rsize, + }, + [OP_LISTXATTRS] = { + .op_func = nfsd4_listxattrs, + .op_name = "OP_LISTXATTRS", + .op_rsize_bop = nfsd4_listxattrs_rsize, + }, + [OP_REMOVEXATTR] = { + .op_func = nfsd4_removexattr, + .op_flags = OP_MODIFIES_SOMETHING | OP_CACHEME, + .op_name = "OP_REMOVEXATTR", + .op_rsize_bop = nfsd4_removexattr_rsize, + }, +}; + +/** + * nfsd4_spo_must_allow - Determine if the compound op contains an + * operation that is allowed to be sent with machine credentials + * + * @rqstp: a pointer to the struct svc_rqst + * + * Checks to see if the compound contains a spo_must_allow op + * and confirms that it was sent with the proper machine creds. + */ + +bool nfsd4_spo_must_allow(struct svc_rqst *rqstp) +{ + struct nfsd4_compoundres *resp = rqstp->rq_resp; + struct nfsd4_compoundargs *argp = rqstp->rq_argp; + struct nfsd4_op *this; + struct nfsd4_compound_state *cstate = &resp->cstate; + struct nfs4_op_map *allow = &cstate->clp->cl_spo_must_allow; + u32 opiter; + + if (rqstp->rq_procinfo != &nfsd_version4.vs_proc[NFSPROC4_COMPOUND] || + cstate->minorversion == 0) + return false; + + if (cstate->spo_must_allowed) + return true; + + opiter = resp->opcnt; + while (opiter < argp->opcnt) { + this = &argp->ops[opiter++]; + if (test_bit(this->opnum, allow->u.longs) && + cstate->clp->cl_mach_cred && + nfsd4_mach_creds_match(cstate->clp, rqstp)) { + cstate->spo_must_allowed = true; + return true; + } + } + cstate->spo_must_allowed = false; + return false; +} + +int nfsd4_max_reply(struct svc_rqst *rqstp, struct nfsd4_op *op) +{ + if (op->opnum == OP_ILLEGAL || op->status == nfserr_notsupp) + return op_encode_hdr_size * sizeof(__be32); + + BUG_ON(OPDESC(op)->op_rsize_bop == NULL); + return OPDESC(op)->op_rsize_bop(rqstp, op); +} + +void warn_on_nonidempotent_op(struct nfsd4_op *op) +{ + if (OPDESC(op)->op_flags & OP_MODIFIES_SOMETHING) { + pr_err("unable to encode reply to nonidempotent op %u (%s)\n", + op->opnum, nfsd4_op_name(op->opnum)); + WARN_ON_ONCE(1); + } +} + +static const char *nfsd4_op_name(unsigned opnum) +{ + if (opnum < ARRAY_SIZE(nfsd4_ops)) + return nfsd4_ops[opnum].op_name; + return "unknown_operation"; +} + +static const struct svc_procedure nfsd_procedures4[2] = { + [NFSPROC4_NULL] = { + .pc_func = nfsd4_proc_null, + .pc_decode = nfssvc_decode_voidarg, + .pc_encode = nfssvc_encode_voidres, + .pc_argsize = sizeof(struct nfsd_voidargs), + .pc_argzero = sizeof(struct nfsd_voidargs), + .pc_ressize = sizeof(struct nfsd_voidres), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = 1, + .pc_name = "NULL", + }, + [NFSPROC4_COMPOUND] = { + .pc_func = nfsd4_proc_compound, + .pc_decode = nfs4svc_decode_compoundargs, + .pc_encode = nfs4svc_encode_compoundres, + .pc_argsize = sizeof(struct nfsd4_compoundargs), + .pc_argzero = offsetof(struct nfsd4_compoundargs, iops), + .pc_ressize = sizeof(struct nfsd4_compoundres), + .pc_release = nfsd4_release_compoundargs, + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = 3+NFSSVC_MAXBLKSIZE/4, + .pc_name = "COMPOUND", + }, +}; + +const struct svc_version nfsd_version4 = { + .vs_vers = 4, + .vs_nproc = ARRAY_SIZE(nfsd_procedures4), + .vs_proc = nfsd_procedures4, + .vs_dispatch = nfsd_dispatch, + .vs_xdrsize = NFS4_SVC_XDRSIZE, + .vs_rpcb_optnl = true, + .vs_need_cong_ctrl = true, +}; diff --git a/fs/nfsd/nfs4recover.c b/fs/nfsd/nfs4recover.c new file mode 100644 index 000000000..d513971fb --- /dev/null +++ b/fs/nfsd/nfs4recover.c @@ -0,0 +1,2060 @@ +/* +* Copyright (c) 2004 The Regents of the University of Michigan. +* Copyright (c) 2012 Jeff Layton +* All rights reserved. +* +* Andy Adamson +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions +* are met: +* +* 1. Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* 2. Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in the +* documentation and/or other materials provided with the distribution. +* 3. Neither the name of the University nor the names of its +* contributors may be used to endorse or promote products derived +* from this software without specific prior written permission. +* +* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED +* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE +* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +* +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nfsd.h" +#include "state.h" +#include "vfs.h" +#include "netns.h" + +#define NFSDDBG_FACILITY NFSDDBG_PROC + +/* Declarations */ +struct nfsd4_client_tracking_ops { + int (*init)(struct net *); + void (*exit)(struct net *); + void (*create)(struct nfs4_client *); + void (*remove)(struct nfs4_client *); + int (*check)(struct nfs4_client *); + void (*grace_done)(struct nfsd_net *); + uint8_t version; + size_t msglen; +}; + +static const struct nfsd4_client_tracking_ops nfsd4_cld_tracking_ops; +static const struct nfsd4_client_tracking_ops nfsd4_cld_tracking_ops_v2; + +#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING +/* Globals */ +static char user_recovery_dirname[PATH_MAX] = "/var/lib/nfs/v4recovery"; + +static int +nfs4_save_creds(const struct cred **original_creds) +{ + struct cred *new; + + new = prepare_creds(); + if (!new) + return -ENOMEM; + + new->fsuid = GLOBAL_ROOT_UID; + new->fsgid = GLOBAL_ROOT_GID; + *original_creds = override_creds(new); + return 0; +} + +static void +nfs4_reset_creds(const struct cred *original) +{ + put_cred(revert_creds(original)); +} + +static void +nfs4_make_rec_clidname(char dname[HEXDIR_LEN], const struct xdr_netobj *clname) +{ + u8 digest[MD5_DIGEST_SIZE]; + + dprintk("NFSD: nfs4_make_rec_clidname for %.*s\n", + clname->len, clname->data); + + md5(clname->data, clname->len, digest); + + static_assert(HEXDIR_LEN == 2 * MD5_DIGEST_SIZE + 1); + sprintf(dname, "%*phN", MD5_DIGEST_SIZE, digest); +} + +static void +__nfsd4_create_reclaim_record_grace(struct nfs4_client *clp, + char *dname, struct nfsd_net *nn) +{ + struct xdr_netobj name = { .len = strlen(dname), .data = dname }; + struct xdr_netobj princhash = { .len = 0, .data = NULL }; + struct nfs4_client_reclaim *crp; + + crp = nfs4_client_to_reclaim(name, princhash, nn); + crp->cr_clp = clp; +} + +static void +nfsd4_create_clid_dir(struct nfs4_client *clp) +{ + const struct cred *original_cred; + char dname[HEXDIR_LEN]; + struct dentry *dir, *dentry; + int status; + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + if (test_and_set_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags)) + return; + if (!nn->rec_file) + return; + + nfs4_make_rec_clidname(dname, &clp->cl_name); + + status = nfs4_save_creds(&original_cred); + if (status < 0) + return; + + status = mnt_want_write_file(nn->rec_file); + if (status) + goto out_creds; + + dir = nn->rec_file->f_path.dentry; + + dentry = start_creating(&nop_mnt_idmap, dir, &QSTR(dname)); + if (IS_ERR(dentry)) { + status = PTR_ERR(dentry); + goto out; + } + if (d_really_is_positive(dentry)) + /* + * In the 4.1 case, where we're called from + * reclaim_complete(), records from the previous reboot + * may still be left, so this is OK. + * + * In the 4.0 case, we should never get here; but we may + * as well be forgiving and just succeed silently. + */ + goto out_end; + dentry = vfs_mkdir(&nop_mnt_idmap, d_inode(dir), dentry, 0700, NULL); + if (IS_ERR(dentry)) + status = PTR_ERR(dentry); +out_end: + end_creating(dentry); +out: + if (status == 0) { + if (test_bit(NFSD_NET_IN_GRACE, &nn->flags)) + __nfsd4_create_reclaim_record_grace(clp, dname, nn); + vfs_fsync(nn->rec_file, 0); + } else { + printk(KERN_ERR "NFSD: failed to write recovery record" + " (err %d); please check that %s exists" + " and is writeable", status, + user_recovery_dirname); + } + mnt_drop_write_file(nn->rec_file); +out_creds: + nfs4_reset_creds(original_cred); +} + +typedef int (recdir_func)(struct dentry *, char *, struct nfsd_net *); + +struct name_list { + char name[HEXDIR_LEN]; + struct list_head list; +}; + +struct nfs4_dir_ctx { + struct dir_context ctx; + struct list_head names; +}; + +static bool +nfsd4_build_namelist(struct dir_context *__ctx, const char *name, int namlen, + loff_t offset, u64 ino, unsigned int d_type) +{ + struct nfs4_dir_ctx *ctx = + container_of(__ctx, struct nfs4_dir_ctx, ctx); + struct name_list *entry; + + if (namlen != HEXDIR_LEN - 1) + return true; + entry = kmalloc_obj(struct name_list); + if (entry == NULL) + return false; + memcpy(entry->name, name, HEXDIR_LEN - 1); + entry->name[HEXDIR_LEN - 1] = '\0'; + list_add(&entry->list, &ctx->names); + return true; +} + +static int +nfsd4_list_rec_dir(recdir_func *f, struct nfsd_net *nn) +{ + const struct cred *original_cred; + struct dentry *dir = nn->rec_file->f_path.dentry; + struct nfs4_dir_ctx ctx = { + .ctx.actor = nfsd4_build_namelist, + .names = LIST_HEAD_INIT(ctx.names) + }; + struct name_list *entry, *tmp; + int status; + + status = nfs4_save_creds(&original_cred); + if (status < 0) + return status; + + status = vfs_llseek(nn->rec_file, 0, SEEK_SET); + if (status < 0) { + nfs4_reset_creds(original_cred); + return status; + } + + status = iterate_dir(nn->rec_file, &ctx.ctx); + + list_for_each_entry_safe(entry, tmp, &ctx.names, list) { + if (!status) + status = f(dir, entry->name, nn); + + list_del(&entry->list); + kfree(entry); + } + nfs4_reset_creds(original_cred); + + list_for_each_entry_safe(entry, tmp, &ctx.names, list) { + dprintk("NFSD: %s. Left entry %s\n", __func__, entry->name); + list_del(&entry->list); + kfree(entry); + } + return status; +} + +static int +nfsd4_unlink_clid_dir(char *name, struct nfsd_net *nn) +{ + struct dentry *dir, *dentry; + int status; + + dprintk("NFSD: nfsd4_unlink_clid_dir. name %s\n", name); + + dir = nn->rec_file->f_path.dentry; + dentry = start_removing(&nop_mnt_idmap, dir, &QSTR(name)); + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + + status = vfs_rmdir(&nop_mnt_idmap, d_inode(dir), dentry, NULL); + end_removing(dentry); + return status; +} + +static void +__nfsd4_remove_reclaim_record_grace(const char *dname, int len, + struct nfsd_net *nn) +{ + struct xdr_netobj name; + struct nfs4_client_reclaim *crp; + + name.data = kmemdup(dname, len, GFP_KERNEL); + if (!name.data) { + dprintk("%s: failed to allocate memory for name.data!\n", + __func__); + return; + } + name.len = len; + down_write(&nn->reclaim_str_hashtbl_lock); + crp = nfsd4_find_reclaim_client(name, nn); + if (crp) + nfs4_remove_reclaim_record(crp, nn); + up_write(&nn->reclaim_str_hashtbl_lock); + kfree(name.data); +} + +static void +nfsd4_remove_clid_dir(struct nfs4_client *clp) +{ + const struct cred *original_cred; + char dname[HEXDIR_LEN]; + int status; + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + if (!nn->rec_file || !test_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags)) + return; + + nfs4_make_rec_clidname(dname, &clp->cl_name); + + status = mnt_want_write_file(nn->rec_file); + if (status) + goto out; + clear_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags); + + status = nfs4_save_creds(&original_cred); + if (status < 0) + goto out_drop_write; + + status = nfsd4_unlink_clid_dir(dname, nn); + nfs4_reset_creds(original_cred); + if (status == 0) { + vfs_fsync(nn->rec_file, 0); + if (test_bit(NFSD_NET_IN_GRACE, &nn->flags)) + __nfsd4_remove_reclaim_record_grace(dname, + HEXDIR_LEN, nn); + } +out_drop_write: + mnt_drop_write_file(nn->rec_file); +out: + if (status) + printk("NFSD: Failed to remove expired client state directory" + " %.*s\n", HEXDIR_LEN, dname); +} + +static int +purge_old(struct dentry *parent, char *cname, struct nfsd_net *nn) +{ + int status; + struct dentry *child; + struct xdr_netobj name; + + if (strlen(cname) != HEXDIR_LEN - 1) { + printk("%s: illegal name %s in recovery directory\n", + __func__, cname); + /* Keep trying; maybe the others are OK: */ + return 0; + } + name.data = kstrdup(cname, GFP_KERNEL); + if (!name.data) { + dprintk("%s: failed to allocate memory for name.data!\n", + __func__); + goto out; + } + name.len = HEXDIR_LEN; + if (nfs4_has_reclaimed_state(name, nn)) + goto out_free; + + child = start_removing_noperm(parent, &QSTR(cname)); + if (!IS_ERR(child)) { + status = vfs_rmdir(&nop_mnt_idmap, d_inode(parent), child, NULL); + if (status) + printk("failed to remove client recovery directory %pd\n", + child); + } + end_removing(child); + +out_free: + kfree(name.data); +out: + /* Keep trying, success or failure: */ + return 0; +} + +static void +nfsd4_recdir_purge_old(struct nfsd_net *nn) +{ + int status; + + clear_bit(NFSD_NET_IN_GRACE, &nn->flags); + if (!nn->rec_file) + return; + status = mnt_want_write_file(nn->rec_file); + if (status) + goto out; + status = nfsd4_list_rec_dir(purge_old, nn); + if (status == 0) + vfs_fsync(nn->rec_file, 0); + mnt_drop_write_file(nn->rec_file); +out: + nfs4_release_reclaim(nn); + if (status) + printk("nfsd4: failed to purge old clients from recovery" + " directory %pD\n", nn->rec_file); +} + +static int +load_recdir(struct dentry *parent, char *cname, struct nfsd_net *nn) +{ + struct xdr_netobj name = { .len = HEXDIR_LEN, .data = cname }; + struct xdr_netobj princhash = { .len = 0, .data = NULL }; + + if (strlen(cname) != HEXDIR_LEN - 1) { + printk("%s: illegal name %s in recovery directory\n", + __func__, cname); + /* Keep trying; maybe the others are OK: */ + return 0; + } + nfs4_client_to_reclaim(name, princhash, nn); + return 0; +} + +static int +nfsd4_recdir_load(struct net *net) { + int status; + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + if (!nn->rec_file) + return 0; + + status = nfsd4_list_rec_dir(load_recdir, nn); + if (status) + printk("nfsd4: failed loading clients from recovery" + " directory %pD\n", nn->rec_file); + return status; +} + +/* + * Hold reference to the recovery directory. + */ + +static int +nfsd4_init_recdir(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + const struct cred *original_cred; + int status; + + printk("NFSD: Using %s as the NFSv4 state recovery directory\n", + user_recovery_dirname); + + BUG_ON(nn->rec_file); + + status = nfs4_save_creds(&original_cred); + if (status < 0) { + printk("NFSD: Unable to change credentials to find recovery" + " directory: error %d\n", + status); + return status; + } + + nn->rec_file = filp_open(user_recovery_dirname, O_RDONLY | O_DIRECTORY, 0); + if (IS_ERR(nn->rec_file)) { + printk("NFSD: unable to find recovery directory %s\n", + user_recovery_dirname); + status = PTR_ERR(nn->rec_file); + nn->rec_file = NULL; + } + + nfs4_reset_creds(original_cred); + if (!status) + set_bit(NFSD_NET_IN_GRACE, &nn->flags); + return status; +} + +static void +nfsd4_shutdown_recdir(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + if (!nn->rec_file) + return; + fput(nn->rec_file); + nn->rec_file = NULL; +} + +static int +nfs4_legacy_state_init(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + int i; + + nn->reclaim_str_hashtbl = kmalloc_objs(struct list_head, + CLIENT_HASH_SIZE); + if (!nn->reclaim_str_hashtbl) + return -ENOMEM; + + for (i = 0; i < CLIENT_HASH_SIZE; i++) + INIT_LIST_HEAD(&nn->reclaim_str_hashtbl[i]); + nn->reclaim_str_hashtbl_size = 0; + init_rwsem(&nn->reclaim_str_hashtbl_lock); + + return 0; +} + +static void +nfs4_legacy_state_shutdown(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + kfree(nn->reclaim_str_hashtbl); +} + +static int +nfsd4_load_reboot_recovery_data(struct net *net) +{ + int status; + + status = nfsd4_init_recdir(net); + if (status) + return status; + + status = nfsd4_recdir_load(net); + if (status) + nfsd4_shutdown_recdir(net); + + return status; +} + +static int +nfsd4_legacy_tracking_init(struct net *net) +{ + int status; + + /* XXX: The legacy code won't work in a container */ + if (net != &init_net) { + pr_warn("NFSD: attempt to initialize legacy client tracking in a container ignored.\n"); + return -EINVAL; + } + + status = nfs4_legacy_state_init(net); + if (status) + return status; + + status = nfsd4_load_reboot_recovery_data(net); + if (status) + goto err; + pr_info("NFSD: Using legacy client tracking operations.\n"); + return 0; + +err: + nfs4_legacy_state_shutdown(net); + return status; +} + +static void +nfsd4_legacy_tracking_exit(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + nfs4_release_reclaim(nn); + nfsd4_shutdown_recdir(net); + nfs4_legacy_state_shutdown(net); +} + +/* + * Change the NFSv4 recovery directory to recdir. + */ +int +nfs4_reset_recoverydir(char *recdir) +{ + int status; + struct path path; + + status = kern_path(recdir, LOOKUP_FOLLOW, &path); + if (status) + return status; + status = -ENOTDIR; + if (d_is_dir(path.dentry)) { + strscpy(user_recovery_dirname, recdir, + sizeof(user_recovery_dirname)); + status = 0; + } + path_put(&path); + return status; +} + +char * +nfs4_recoverydir(void) +{ + return user_recovery_dirname; +} + +static int +nfsd4_check_legacy_client(struct nfs4_client *clp) +{ + char dname[HEXDIR_LEN]; + struct nfs4_client_reclaim *crp; + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + struct xdr_netobj name; + + /* did we already find that this client is stable? */ + if (test_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags)) + return 0; + + nfs4_make_rec_clidname(dname, &clp->cl_name); + + /* look for it in the reclaim hashtable otherwise */ + name.data = kmemdup(dname, HEXDIR_LEN, GFP_KERNEL); + if (!name.data) { + dprintk("%s: failed to allocate memory for name.data!\n", + __func__); + goto out_enoent; + } + name.len = HEXDIR_LEN; + down_read(&nn->reclaim_str_hashtbl_lock); + crp = nfsd4_find_reclaim_client(name, nn); + if (crp) { + set_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags); + crp->cr_clp = clp; + } + up_read(&nn->reclaim_str_hashtbl_lock); + kfree(name.data); + if (crp) + return 0; + +out_enoent: + return -ENOENT; +} + +static const struct nfsd4_client_tracking_ops nfsd4_legacy_tracking_ops = { + .init = nfsd4_legacy_tracking_init, + .exit = nfsd4_legacy_tracking_exit, + .create = nfsd4_create_clid_dir, + .remove = nfsd4_remove_clid_dir, + .check = nfsd4_check_legacy_client, + .grace_done = nfsd4_recdir_purge_old, + .version = 1, + .msglen = 0, +}; +#endif /* CONFIG_NFSD_LEGACY_CLIENT_TRACKING */ + +/* Globals */ +#define NFSD_PIPE_DIR "nfsd" +#define NFSD_CLD_PIPE "cld" + +/* per-net-ns structure for holding cld upcall info */ +struct cld_net { + struct rpc_pipe *cn_pipe; + spinlock_t cn_lock; + struct list_head cn_list; + unsigned int cn_xid; +#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING + bool cn_has_legacy; +#endif +}; + +struct cld_upcall { + struct list_head cu_list; + struct cld_net *cu_net; + struct completion cu_done; + union { + struct cld_msg_hdr cu_hdr; + struct cld_msg cu_msg; + struct cld_msg_v2 cu_msg_v2; + } cu_u; +}; + +static int +__cld_pipe_upcall(struct rpc_pipe *pipe, void *cmsg, struct nfsd_net *nn) +{ + int ret; + struct rpc_pipe_msg msg; + struct cld_upcall *cup = container_of(cmsg, struct cld_upcall, cu_u); + + memset(&msg, 0, sizeof(msg)); + msg.data = cmsg; + msg.len = nn->client_tracking_ops->msglen; + + ret = rpc_queue_upcall(pipe, &msg); + if (ret < 0) { + goto out; + } + + wait_for_completion(&cup->cu_done); + + if (msg.errno < 0) + ret = msg.errno; +out: + return ret; +} + +static int +cld_pipe_upcall(struct rpc_pipe *pipe, void *cmsg, struct nfsd_net *nn) +{ + int ret; + + /* + * -EAGAIN occurs when pipe is closed and reopened while there are + * upcalls queued. + */ + do { + ret = __cld_pipe_upcall(pipe, cmsg, nn); + } while (ret == -EAGAIN); + + return ret; +} + +static ssize_t +__cld_pipe_inprogress_downcall(const struct cld_msg_v2 __user *cmsg, + struct nfsd_net *nn) +{ + uint8_t cmd, princhashlen; + struct xdr_netobj name, princhash = { .len = 0, .data = NULL }; + char *namecopy __free(kfree) = NULL; + char *princhashcopy __free(kfree) = NULL; + uint16_t namelen; + + if (get_user(cmd, &cmsg->cm_cmd)) { + dprintk("%s: error when copying cmd from userspace", __func__); + return -EFAULT; + } + if (cmd == Cld_GraceStart) { + if (nn->client_tracking_ops->version >= 2) { + const struct cld_clntinfo __user *ci; + + ci = &cmsg->cm_u.cm_clntinfo; + if (get_user(namelen, &ci->cc_name.cn_len)) + return -EFAULT; + if (namelen == 0 || namelen > NFS4_OPAQUE_LIMIT) { + dprintk("%s: invalid namelen (%u)", __func__, namelen); + return -EINVAL; + } + namecopy = memdup_user(&ci->cc_name.cn_id, namelen); + if (IS_ERR(namecopy)) + return PTR_ERR(namecopy); + name.data = namecopy; + name.len = namelen; + if (get_user(princhashlen, &ci->cc_princhash.cp_len)) + return -EFAULT; + if (princhashlen > 0) { + princhashcopy = memdup_user( + &ci->cc_princhash.cp_data, + princhashlen); + if (IS_ERR(princhashcopy)) + return PTR_ERR(princhashcopy); + princhash.data = princhashcopy; + princhash.len = princhashlen; + } else + princhash.len = 0; + } else { + const struct cld_name __user *cnm; + + cnm = &cmsg->cm_u.cm_name; + if (get_user(namelen, &cnm->cn_len)) + return -EFAULT; + if (namelen == 0 || namelen > NFS4_OPAQUE_LIMIT) { + dprintk("%s: invalid namelen (%u)", __func__, namelen); + return -EINVAL; + } + namecopy = memdup_user(&cnm->cn_id, namelen); + if (IS_ERR(namecopy)) + return PTR_ERR(namecopy); + name.data = namecopy; + name.len = namelen; + } +#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING + if (name.len > 5 && memcmp(name.data, "hash:", 5) == 0) { + struct cld_net *cn = nn->cld_net; + + name.len = name.len - 5; + name.data = name.data + 5; + cn->cn_has_legacy = true; + } +#endif + if (!nfs4_client_to_reclaim(name, princhash, nn)) + return -EFAULT; + return nn->client_tracking_ops->msglen; + } + return -EFAULT; +} + +static ssize_t +cld_pipe_downcall(struct file *filp, const char __user *src, size_t mlen) +{ + struct cld_upcall *tmp, *cup; + struct cld_msg_hdr __user *hdr = (struct cld_msg_hdr __user *)src; + struct cld_msg_v2 __user *cmsg = (struct cld_msg_v2 __user *)src; + uint32_t xid; + struct nfsd_net *nn = net_generic(file_inode(filp)->i_sb->s_fs_info, + nfsd_net_id); + struct cld_net *cn = nn->cld_net; + int16_t status; + + if (mlen != nn->client_tracking_ops->msglen) { + dprintk("%s: got %zu bytes, expected %zu\n", __func__, mlen, + nn->client_tracking_ops->msglen); + return -EINVAL; + } + + /* copy just the xid so we can try to find that */ + if (copy_from_user(&xid, &hdr->cm_xid, sizeof(xid)) != 0) { + dprintk("%s: error when copying xid from userspace", __func__); + return -EFAULT; + } + + /* + * copy the status so we know whether to remove the upcall from the + * list (for -EINPROGRESS, we just want to make sure the xid is + * valid, not remove the upcall from the list) + */ + if (get_user(status, &hdr->cm_status)) { + dprintk("%s: error when copying status from userspace", __func__); + return -EFAULT; + } + + /* walk the list and find corresponding xid */ + cup = NULL; + spin_lock(&cn->cn_lock); + list_for_each_entry(tmp, &cn->cn_list, cu_list) { + if (get_unaligned(&tmp->cu_u.cu_hdr.cm_xid) == xid) { + cup = tmp; + if (status != -EINPROGRESS) + list_del_init(&cup->cu_list); + break; + } + } + spin_unlock(&cn->cn_lock); + + /* couldn't find upcall? */ + if (!cup) { + dprintk("%s: couldn't find upcall -- xid=%u\n", __func__, xid); + return -EINVAL; + } + + if (status == -EINPROGRESS) + return __cld_pipe_inprogress_downcall(cmsg, nn); + + if (copy_from_user(&cup->cu_u.cu_msg_v2, src, mlen) != 0) + return -EFAULT; + + complete(&cup->cu_done); + return mlen; +} + +static void +cld_pipe_destroy_msg(struct rpc_pipe_msg *msg) +{ + struct cld_msg *cmsg = msg->data; + struct cld_upcall *cup = container_of(cmsg, struct cld_upcall, + cu_u.cu_msg); + + /* errno >= 0 means we got a downcall */ + if (msg->errno >= 0) + return; + + complete(&cup->cu_done); +} + +static const struct rpc_pipe_ops cld_upcall_ops = { + .upcall = rpc_pipe_generic_upcall, + .downcall = cld_pipe_downcall, + .destroy_msg = cld_pipe_destroy_msg, +}; + +static int +nfsd4_cld_register_sb(struct super_block *sb, struct rpc_pipe *pipe) +{ + struct dentry *dir; + int err; + + dir = rpc_d_lookup_sb(sb, NFSD_PIPE_DIR); + if (dir == NULL) + return -ENOENT; + err = rpc_mkpipe_dentry(dir, NFSD_CLD_PIPE, NULL, pipe); + dput(dir); + return err; +} + +static int +nfsd4_cld_register_net(struct net *net, struct rpc_pipe *pipe) +{ + struct super_block *sb; + int err; + + sb = rpc_get_sb_net(net); + if (!sb) + return 0; + err = nfsd4_cld_register_sb(sb, pipe); + rpc_put_sb_net(net); + return err; +} + +static void +nfsd4_cld_unregister_net(struct net *net, struct rpc_pipe *pipe) +{ + struct super_block *sb; + + sb = rpc_get_sb_net(net); + if (sb) { + rpc_unlink(pipe); + rpc_put_sb_net(net); + } +} + +/* Initialize rpc_pipefs pipe for communication with client tracking daemon */ +static int +__nfsd4_init_cld_pipe(struct net *net) +{ + int ret; + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct cld_net *cn; + + if (nn->cld_net) + return 0; + + cn = kzalloc_obj(*cn); + if (!cn) { + ret = -ENOMEM; + goto err; + } + + cn->cn_pipe = rpc_mkpipe_data(&cld_upcall_ops, RPC_PIPE_WAIT_FOR_OPEN); + if (IS_ERR(cn->cn_pipe)) { + ret = PTR_ERR(cn->cn_pipe); + goto err; + } + spin_lock_init(&cn->cn_lock); + INIT_LIST_HEAD(&cn->cn_list); + + ret = nfsd4_cld_register_net(net, cn->cn_pipe); + if (unlikely(ret)) + goto err_destroy_data; + +#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING + cn->cn_has_legacy = false; +#endif + nn->cld_net = cn; + return 0; + +err_destroy_data: + rpc_destroy_pipe_data(cn->cn_pipe); +err: + kfree(cn); + printk(KERN_ERR "NFSD: unable to create nfsdcld upcall pipe (%d)\n", + ret); + return ret; +} + +static int +nfsd4_init_cld_pipe(struct net *net) +{ + int status; + + status = __nfsd4_init_cld_pipe(net); + if (!status) + pr_info("NFSD: Using old nfsdcld client tracking operations.\n"); + return status; +} + +static void +nfsd4_remove_cld_pipe(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct cld_net *cn = nn->cld_net; + + nfsd4_cld_unregister_net(net, cn->cn_pipe); + rpc_destroy_pipe_data(cn->cn_pipe); + kfree(nn->cld_net); + nn->cld_net = NULL; +} + +static struct cld_upcall * +alloc_cld_upcall(struct nfsd_net *nn) +{ + struct cld_upcall *new, *tmp; + struct cld_net *cn = nn->cld_net; + + new = kzalloc_obj(*new); + if (!new) + return new; + + /* FIXME: hard cap on number in flight? */ +restart_search: + spin_lock(&cn->cn_lock); + list_for_each_entry(tmp, &cn->cn_list, cu_list) { + if (tmp->cu_u.cu_msg.cm_xid == cn->cn_xid) { + cn->cn_xid++; + spin_unlock(&cn->cn_lock); + goto restart_search; + } + } + init_completion(&new->cu_done); + new->cu_u.cu_msg.cm_vers = nn->client_tracking_ops->version; + put_unaligned(cn->cn_xid++, &new->cu_u.cu_msg.cm_xid); + new->cu_net = cn; + list_add(&new->cu_list, &cn->cn_list); + spin_unlock(&cn->cn_lock); + + dprintk("%s: allocated xid %u\n", __func__, new->cu_u.cu_msg.cm_xid); + + return new; +} + +static void +free_cld_upcall(struct cld_upcall *victim) +{ + struct cld_net *cn = victim->cu_net; + + spin_lock(&cn->cn_lock); + list_del(&victim->cu_list); + spin_unlock(&cn->cn_lock); + kfree(victim); +} + +/* Ask daemon to create a new record */ +static void +nfsd4_cld_create(struct nfs4_client *clp) +{ + int ret; + struct cld_upcall *cup; + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + struct cld_net *cn = nn->cld_net; + + /* Don't upcall if it's already stored */ + if (test_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags)) + return; + + cup = alloc_cld_upcall(nn); + if (!cup) { + ret = -ENOMEM; + goto out_err; + } + + cup->cu_u.cu_msg.cm_cmd = Cld_Create; + cup->cu_u.cu_msg.cm_u.cm_name.cn_len = clp->cl_name.len; + memcpy(cup->cu_u.cu_msg.cm_u.cm_name.cn_id, clp->cl_name.data, + clp->cl_name.len); + + ret = cld_pipe_upcall(cn->cn_pipe, &cup->cu_u.cu_msg, nn); + if (!ret) { + ret = cup->cu_u.cu_msg.cm_status; + set_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags); + } + + free_cld_upcall(cup); +out_err: + if (ret) + printk(KERN_ERR "NFSD: Unable to create client " + "record on stable storage: %d\n", ret); +} + +/* Ask daemon to create a new record */ +static void +nfsd4_cld_create_v2(struct nfs4_client *clp) +{ + int ret; + struct cld_upcall *cup; + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + struct cld_net *cn = nn->cld_net; + struct cld_msg_v2 *cmsg; + char *principal = NULL; + + /* Don't upcall if it's already stored */ + if (test_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags)) + return; + + cup = alloc_cld_upcall(nn); + if (!cup) { + ret = -ENOMEM; + goto out_err; + } + + cmsg = &cup->cu_u.cu_msg_v2; + cmsg->cm_cmd = Cld_Create; + cmsg->cm_u.cm_clntinfo.cc_name.cn_len = clp->cl_name.len; + memcpy(cmsg->cm_u.cm_clntinfo.cc_name.cn_id, clp->cl_name.data, + clp->cl_name.len); + if (clp->cl_cred.cr_raw_principal) + principal = clp->cl_cred.cr_raw_principal; + else if (clp->cl_cred.cr_principal) + principal = clp->cl_cred.cr_principal; + if (principal) { + sha256(principal, strlen(principal), + cmsg->cm_u.cm_clntinfo.cc_princhash.cp_data); + cmsg->cm_u.cm_clntinfo.cc_princhash.cp_len = SHA256_DIGEST_SIZE; + } else + cmsg->cm_u.cm_clntinfo.cc_princhash.cp_len = 0; + + ret = cld_pipe_upcall(cn->cn_pipe, cmsg, nn); + if (!ret) { + ret = cmsg->cm_status; + set_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags); + } + + free_cld_upcall(cup); +out_err: + if (ret) + pr_err("NFSD: Unable to create client record on stable storage: %d\n", + ret); +} + +/* Ask daemon to create a new record */ +static void +nfsd4_cld_remove(struct nfs4_client *clp) +{ + int ret; + struct cld_upcall *cup; + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + struct cld_net *cn = nn->cld_net; + + /* Don't upcall if it's already removed */ + if (!test_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags)) + return; + + cup = alloc_cld_upcall(nn); + if (!cup) { + ret = -ENOMEM; + goto out_err; + } + + cup->cu_u.cu_msg.cm_cmd = Cld_Remove; + cup->cu_u.cu_msg.cm_u.cm_name.cn_len = clp->cl_name.len; + memcpy(cup->cu_u.cu_msg.cm_u.cm_name.cn_id, clp->cl_name.data, + clp->cl_name.len); + + ret = cld_pipe_upcall(cn->cn_pipe, &cup->cu_u.cu_msg, nn); + if (!ret) { + ret = cup->cu_u.cu_msg.cm_status; + clear_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags); + } + + free_cld_upcall(cup); +out_err: + if (ret) + printk(KERN_ERR "NFSD: Unable to remove client " + "record from stable storage: %d\n", ret); +} + +/* + * For older nfsdcld's that do not allow us to "slurp" the clients + * from the tracking database during startup. + * + * Check for presence of a record, and update its timestamp + */ +static int +nfsd4_cld_check_v0(struct nfs4_client *clp) +{ + int ret; + struct cld_upcall *cup; + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + struct cld_net *cn = nn->cld_net; + + /* Don't upcall if one was already stored during this grace pd */ + if (test_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags)) + return 0; + + cup = alloc_cld_upcall(nn); + if (!cup) { + printk(KERN_ERR "NFSD: Unable to check client record on " + "stable storage: %d\n", -ENOMEM); + return -ENOMEM; + } + + cup->cu_u.cu_msg.cm_cmd = Cld_Check; + cup->cu_u.cu_msg.cm_u.cm_name.cn_len = clp->cl_name.len; + memcpy(cup->cu_u.cu_msg.cm_u.cm_name.cn_id, clp->cl_name.data, + clp->cl_name.len); + + ret = cld_pipe_upcall(cn->cn_pipe, &cup->cu_u.cu_msg, nn); + if (!ret) { + ret = cup->cu_u.cu_msg.cm_status; + set_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags); + } + + free_cld_upcall(cup); + return ret; +} + +/* + * For newer nfsdcld's that allow us to "slurp" the clients + * from the tracking database during startup. + * + * Check for presence of a record in the reclaim_str_hashtbl + */ +static int +nfsd4_cld_check(struct nfs4_client *clp) +{ + struct nfs4_client_reclaim *crp; + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + /* did we already find that this client is stable? */ + if (test_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags)) + return 0; + + /* look for it in the reclaim hashtable otherwise */ + down_read(&nn->reclaim_str_hashtbl_lock); + crp = nfsd4_find_reclaim_client(clp->cl_name, nn); + if (crp) + goto found; + +#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING + if (nn->cld_net->cn_has_legacy) { + char dname[HEXDIR_LEN]; + struct xdr_netobj name; + + nfs4_make_rec_clidname(dname, &clp->cl_name); + + name.data = kmemdup(dname, HEXDIR_LEN, GFP_KERNEL); + if (!name.data) { + dprintk("%s: failed to allocate memory for name.data!\n", + __func__); + up_read(&nn->reclaim_str_hashtbl_lock); + return -ENOENT; + } + name.len = HEXDIR_LEN; + crp = nfsd4_find_reclaim_client(name, nn); + kfree(name.data); + if (crp) + goto found; + + } +#endif + up_read(&nn->reclaim_str_hashtbl_lock); + return -ENOENT; +found: + crp->cr_clp = clp; + up_read(&nn->reclaim_str_hashtbl_lock); + return 0; +} + +static int +nfsd4_cld_check_v2(struct nfs4_client *clp) +{ + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); +#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING + struct cld_net *cn = nn->cld_net; +#endif + struct nfs4_client_reclaim *crp; + unsigned int princhashlen; + char *principal = NULL; + + /* did we already find that this client is stable? */ + if (test_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags)) + return 0; + + /* look for it in the reclaim hashtable otherwise */ + down_read(&nn->reclaim_str_hashtbl_lock); + crp = nfsd4_find_reclaim_client(clp->cl_name, nn); + if (crp) + goto found; + +#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING + if (cn->cn_has_legacy) { + struct xdr_netobj name; + char dname[HEXDIR_LEN]; + + nfs4_make_rec_clidname(dname, &clp->cl_name); + + name.data = kmemdup(dname, HEXDIR_LEN, GFP_KERNEL); + if (!name.data) { + dprintk("%s: failed to allocate memory for name.data\n", + __func__); + up_read(&nn->reclaim_str_hashtbl_lock); + return -ENOENT; + } + name.len = HEXDIR_LEN; + crp = nfsd4_find_reclaim_client(name, nn); + kfree(name.data); + if (crp) + goto found; + + } +#endif + up_read(&nn->reclaim_str_hashtbl_lock); + return -ENOENT; +found: + princhashlen = crp->cr_princhash.len; + if (princhashlen) { + u8 digest[SHA256_DIGEST_SIZE]; + u8 *pdata; + + if (clp->cl_cred.cr_raw_principal) + principal = clp->cl_cred.cr_raw_principal; + else if (clp->cl_cred.cr_principal) + principal = clp->cl_cred.cr_principal; + if (principal == NULL) { + up_read(&nn->reclaim_str_hashtbl_lock); + return -ENOENT; + } + sha256(principal, strlen(principal), digest); + pdata = crp->cr_princhash.data; + if (memcmp(pdata, digest, princhashlen)) { + up_read(&nn->reclaim_str_hashtbl_lock); + return -ENOENT; + } + } + crp->cr_clp = clp; + up_read(&nn->reclaim_str_hashtbl_lock); + return 0; +} + +static int +nfsd4_cld_grace_start(struct nfsd_net *nn) +{ + int ret; + struct cld_upcall *cup; + struct cld_net *cn = nn->cld_net; + + cup = alloc_cld_upcall(nn); + if (!cup) { + ret = -ENOMEM; + goto out_err; + } + + cup->cu_u.cu_msg.cm_cmd = Cld_GraceStart; + ret = cld_pipe_upcall(cn->cn_pipe, &cup->cu_u.cu_msg, nn); + if (!ret) + ret = cup->cu_u.cu_msg.cm_status; + + free_cld_upcall(cup); +out_err: + if (ret) + dprintk("%s: Unable to get clients from userspace: %d\n", + __func__, ret); + return ret; +} + +/* For older nfsdcld's that need cm_gracetime */ +static void +nfsd4_cld_grace_done_v0(struct nfsd_net *nn) +{ + int ret; + struct cld_upcall *cup; + struct cld_net *cn = nn->cld_net; + + cup = alloc_cld_upcall(nn); + if (!cup) { + ret = -ENOMEM; + goto out_err; + } + + cup->cu_u.cu_msg.cm_cmd = Cld_GraceDone; + cup->cu_u.cu_msg.cm_u.cm_gracetime = nn->boot_time; + ret = cld_pipe_upcall(cn->cn_pipe, &cup->cu_u.cu_msg, nn); + if (!ret) + ret = cup->cu_u.cu_msg.cm_status; + + free_cld_upcall(cup); +out_err: + if (ret) + printk(KERN_ERR "NFSD: Unable to end grace period: %d\n", ret); +} + +/* + * For newer nfsdcld's that do not need cm_gracetime. We also need to call + * nfs4_release_reclaim() to clear out the reclaim_str_hashtbl. + */ +static void +nfsd4_cld_grace_done(struct nfsd_net *nn) +{ + int ret; + struct cld_upcall *cup; + struct cld_net *cn = nn->cld_net; + + cup = alloc_cld_upcall(nn); + if (!cup) { + ret = -ENOMEM; + goto out_err; + } + + cup->cu_u.cu_msg.cm_cmd = Cld_GraceDone; + ret = cld_pipe_upcall(cn->cn_pipe, &cup->cu_u.cu_msg, nn); + if (!ret) + ret = cup->cu_u.cu_msg.cm_status; + + free_cld_upcall(cup); +out_err: + nfs4_release_reclaim(nn); + if (ret) + printk(KERN_ERR "NFSD: Unable to end grace period: %d\n", ret); +} + +static int +nfs4_cld_state_init(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + int i; + + nn->reclaim_str_hashtbl = kmalloc_objs(struct list_head, + CLIENT_HASH_SIZE); + if (!nn->reclaim_str_hashtbl) + return -ENOMEM; + + for (i = 0; i < CLIENT_HASH_SIZE; i++) + INIT_LIST_HEAD(&nn->reclaim_str_hashtbl[i]); + nn->reclaim_str_hashtbl_size = 0; + init_rwsem(&nn->reclaim_str_hashtbl_lock); + set_bit(NFSD_NET_TRACK_RECLAIM_COMPLETES, &nn->flags); + atomic_set(&nn->nr_reclaim_complete, 0); + + return 0; +} + +static void +nfs4_cld_state_shutdown(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + clear_bit(NFSD_NET_TRACK_RECLAIM_COMPLETES, &nn->flags); + kfree(nn->reclaim_str_hashtbl); +} + +static bool +cld_running(struct nfsd_net *nn) +{ + struct cld_net *cn = nn->cld_net; + struct rpc_pipe *pipe = cn->cn_pipe; + + return pipe->nreaders || pipe->nwriters; +} + +static int +nfsd4_cld_get_version(struct nfsd_net *nn) +{ + int ret = 0; + struct cld_upcall *cup; + struct cld_net *cn = nn->cld_net; + uint8_t version; + + cup = alloc_cld_upcall(nn); + if (!cup) { + ret = -ENOMEM; + goto out_err; + } + cup->cu_u.cu_msg.cm_cmd = Cld_GetVersion; + ret = cld_pipe_upcall(cn->cn_pipe, &cup->cu_u.cu_msg, nn); + if (!ret) { + ret = cup->cu_u.cu_msg.cm_status; + if (ret) + goto out_free; + version = cup->cu_u.cu_msg.cm_u.cm_version; + dprintk("%s: userspace returned version %u\n", + __func__, version); + if (version < 1) + version = 1; + else if (version > CLD_UPCALL_VERSION) + version = CLD_UPCALL_VERSION; + + switch (version) { + case 1: + nn->client_tracking_ops = &nfsd4_cld_tracking_ops; + break; + case 2: + nn->client_tracking_ops = &nfsd4_cld_tracking_ops_v2; + break; + default: + break; + } + } +out_free: + free_cld_upcall(cup); +out_err: + if (ret) + dprintk("%s: Unable to get version from userspace: %d\n", + __func__, ret); + return ret; +} + +static int +nfsd4_cld_tracking_init(struct net *net) +{ + int status; + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + bool running; + int retries = 10; + + status = nfs4_cld_state_init(net); + if (status) + return status; + + status = __nfsd4_init_cld_pipe(net); + if (status) + goto err_shutdown; + + /* + * rpc pipe upcalls take 30 seconds to time out, so we don't want to + * queue an upcall unless we know that nfsdcld is running (because we + * want this to fail fast so that nfsd4_client_tracking_init() can try + * the next client tracking method). nfsdcld should already be running + * before nfsd is started, so the wait here is for nfsdcld to open the + * pipefs file we just created. + */ + while (!(running = cld_running(nn)) && retries--) + msleep(100); + + if (!running) { + status = -ETIMEDOUT; + goto err_remove; + } + + status = nfsd4_cld_get_version(nn); + if (status == -EOPNOTSUPP) + pr_warn("NFSD: nfsdcld GetVersion upcall failed. Please upgrade nfsdcld.\n"); + + status = nfsd4_cld_grace_start(nn); + if (status) { + if (status == -EOPNOTSUPP) + pr_warn("NFSD: nfsdcld GraceStart upcall failed. Please upgrade nfsdcld.\n"); + nfs4_release_reclaim(nn); + goto err_remove; + } else + pr_info("NFSD: Using nfsdcld client tracking operations.\n"); + return 0; + +err_remove: + nfsd4_remove_cld_pipe(net); +err_shutdown: + nfs4_cld_state_shutdown(net); + return status; +} + +static void +nfsd4_cld_tracking_exit(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + nfs4_release_reclaim(nn); + nfsd4_remove_cld_pipe(net); + nfs4_cld_state_shutdown(net); +} + +/* For older nfsdcld's */ +static const struct nfsd4_client_tracking_ops nfsd4_cld_tracking_ops_v0 = { + .init = nfsd4_init_cld_pipe, + .exit = nfsd4_remove_cld_pipe, + .create = nfsd4_cld_create, + .remove = nfsd4_cld_remove, + .check = nfsd4_cld_check_v0, + .grace_done = nfsd4_cld_grace_done_v0, + .version = 1, + .msglen = sizeof(struct cld_msg), +}; + +/* For newer nfsdcld's */ +static const struct nfsd4_client_tracking_ops nfsd4_cld_tracking_ops = { + .init = nfsd4_cld_tracking_init, + .exit = nfsd4_cld_tracking_exit, + .create = nfsd4_cld_create, + .remove = nfsd4_cld_remove, + .check = nfsd4_cld_check, + .grace_done = nfsd4_cld_grace_done, + .version = 1, + .msglen = sizeof(struct cld_msg), +}; + +/* v2 create/check ops include the principal, if available */ +static const struct nfsd4_client_tracking_ops nfsd4_cld_tracking_ops_v2 = { + .init = nfsd4_cld_tracking_init, + .exit = nfsd4_cld_tracking_exit, + .create = nfsd4_cld_create_v2, + .remove = nfsd4_cld_remove, + .check = nfsd4_cld_check_v2, + .grace_done = nfsd4_cld_grace_done, + .version = 2, + .msglen = sizeof(struct cld_msg_v2), +}; + +#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING +/* upcall via usermodehelper */ +static char cltrack_prog[PATH_MAX] = "/sbin/nfsdcltrack"; +module_param_string(cltrack_prog, cltrack_prog, sizeof(cltrack_prog), + S_IRUGO|S_IWUSR); +MODULE_PARM_DESC(cltrack_prog, "Path to the nfsdcltrack upcall program"); + +static bool cltrack_legacy_disable; +module_param(cltrack_legacy_disable, bool, S_IRUGO|S_IWUSR); +MODULE_PARM_DESC(cltrack_legacy_disable, + "Disable legacy recoverydir conversion. Default: false"); + +#define LEGACY_TOPDIR_ENV_PREFIX "NFSDCLTRACK_LEGACY_TOPDIR=" +#define LEGACY_RECDIR_ENV_PREFIX "NFSDCLTRACK_LEGACY_RECDIR=" +#define HAS_SESSION_ENV_PREFIX "NFSDCLTRACK_CLIENT_HAS_SESSION=" +#define GRACE_START_ENV_PREFIX "NFSDCLTRACK_GRACE_START=" + +static char * +nfsd4_cltrack_legacy_topdir(void) +{ + int copied; + size_t len; + char *result; + + if (cltrack_legacy_disable) + return NULL; + + len = strlen(LEGACY_TOPDIR_ENV_PREFIX) + + strlen(nfs4_recoverydir()) + 1; + + result = kmalloc(len, GFP_KERNEL); + if (!result) + return result; + + copied = snprintf(result, len, LEGACY_TOPDIR_ENV_PREFIX "%s", + nfs4_recoverydir()); + if (copied >= len) { + /* just return nothing if output was truncated */ + kfree(result); + return NULL; + } + + return result; +} + +static char * +nfsd4_cltrack_legacy_recdir(const struct xdr_netobj *name) +{ + int copied; + size_t len; + char *result; + + if (cltrack_legacy_disable) + return NULL; + + /* +1 is for '/' between "topdir" and "recdir" */ + len = strlen(LEGACY_RECDIR_ENV_PREFIX) + + strlen(nfs4_recoverydir()) + 1 + HEXDIR_LEN; + + result = kmalloc(len, GFP_KERNEL); + if (!result) + return result; + + copied = snprintf(result, len, LEGACY_RECDIR_ENV_PREFIX "%s/", + nfs4_recoverydir()); + if (copied > (len - HEXDIR_LEN)) { + /* just return nothing if output will be truncated */ + kfree(result); + return NULL; + } + + nfs4_make_rec_clidname(result + copied, name); + + return result; +} + +static char * +nfsd4_cltrack_client_has_session(struct nfs4_client *clp) +{ + int copied; + size_t len; + char *result; + + /* prefix + Y/N character + terminating NULL */ + len = strlen(HAS_SESSION_ENV_PREFIX) + 1 + 1; + + result = kmalloc(len, GFP_KERNEL); + if (!result) + return result; + + copied = snprintf(result, len, HAS_SESSION_ENV_PREFIX "%c", + clp->cl_minorversion ? 'Y' : 'N'); + if (copied >= len) { + /* just return nothing if output was truncated */ + kfree(result); + return NULL; + } + + return result; +} + +static char * +nfsd4_cltrack_grace_start(time64_t grace_start) +{ + int copied; + size_t len; + char *result; + + /* prefix + max width of int64_t string + terminating NULL */ + len = strlen(GRACE_START_ENV_PREFIX) + 22 + 1; + + result = kmalloc(len, GFP_KERNEL); + if (!result) + return result; + + copied = snprintf(result, len, GRACE_START_ENV_PREFIX "%lld", + grace_start); + if (copied >= len) { + /* just return nothing if output was truncated */ + kfree(result); + return NULL; + } + + return result; +} + +static int +nfsd4_umh_cltrack_upcall(char *cmd, char *arg, char *env0, char *env1) +{ + char *envp[3]; + char *argv[4]; + int ret; + + if (unlikely(!cltrack_prog[0])) { + dprintk("%s: cltrack_prog is disabled\n", __func__); + return -EACCES; + } + + dprintk("%s: cmd: %s\n", __func__, cmd); + dprintk("%s: arg: %s\n", __func__, arg ? arg : "(null)"); + dprintk("%s: env0: %s\n", __func__, env0 ? env0 : "(null)"); + dprintk("%s: env1: %s\n", __func__, env1 ? env1 : "(null)"); + + envp[0] = env0; + envp[1] = env1; + envp[2] = NULL; + + argv[0] = (char *)cltrack_prog; + argv[1] = cmd; + argv[2] = arg; + argv[3] = NULL; + + ret = call_usermodehelper(argv[0], argv, envp, UMH_WAIT_PROC); + /* + * Disable the upcall mechanism if we're getting an ENOENT or EACCES + * error. The admin can re-enable it on the fly by using sysfs + * once the problem has been fixed. + */ + if (ret == -ENOENT || ret == -EACCES) { + dprintk("NFSD: %s was not found or isn't executable (%d). " + "Setting cltrack_prog to blank string!", + cltrack_prog, ret); + cltrack_prog[0] = '\0'; + } + dprintk("%s: %s return value: %d\n", __func__, cltrack_prog, ret); + + return ret; +} + +static char * +bin_to_hex_dup(const unsigned char *src, int srclen) +{ + char *buf; + + /* +1 for terminating NULL */ + buf = kzalloc((srclen * 2) + 1, GFP_KERNEL); + if (!buf) + return buf; + + bin2hex(buf, src, srclen); + return buf; +} + +static int +nfsd4_umh_cltrack_init(struct net *net) +{ + int ret; + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + char *grace_start = nfsd4_cltrack_grace_start(nn->boot_time); + + /* XXX: The usermode helper s not working in container yet. */ + if (net != &init_net) { + pr_warn("NFSD: attempt to initialize umh client tracking in a container ignored.\n"); + kfree(grace_start); + return -EINVAL; + } + + ret = nfsd4_umh_cltrack_upcall("init", NULL, grace_start, NULL); + kfree(grace_start); + if (!ret) + pr_info("NFSD: Using UMH upcall client tracking operations.\n"); + return ret; +} + +static void +nfsd4_cltrack_upcall_lock(struct nfs4_client *clp) +{ + wait_on_bit_lock(&clp->cl_flags, NFSD4_CLIENT_UPCALL_LOCK, + TASK_UNINTERRUPTIBLE); +} + +static void +nfsd4_cltrack_upcall_unlock(struct nfs4_client *clp) +{ + clear_and_wake_up_bit(NFSD4_CLIENT_UPCALL_LOCK, &clp->cl_flags); +} + +static void +nfsd4_umh_cltrack_create(struct nfs4_client *clp) +{ + char *hexid, *has_session, *grace_start; + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + /* + * With v4.0 clients, there's little difference in outcome between a + * create and check operation, and we can end up calling into this + * function multiple times per client (once for each openowner). So, + * for v4.0 clients skip upcalling once the client has been recorded + * on stable storage. + * + * For v4.1+ clients, the outcome of the two operations is different, + * so we must ensure that we upcall for the create operation. v4.1+ + * clients call this on RECLAIM_COMPLETE though, so we should only end + * up doing a single create upcall per client. + */ + if (clp->cl_minorversion == 0 && + test_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags)) + return; + + hexid = bin_to_hex_dup(clp->cl_name.data, clp->cl_name.len); + if (!hexid) { + dprintk("%s: can't allocate memory for upcall!\n", __func__); + return; + } + + has_session = nfsd4_cltrack_client_has_session(clp); + grace_start = nfsd4_cltrack_grace_start(nn->boot_time); + + nfsd4_cltrack_upcall_lock(clp); + if (!nfsd4_umh_cltrack_upcall("create", hexid, has_session, grace_start)) + set_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags); + nfsd4_cltrack_upcall_unlock(clp); + + kfree(has_session); + kfree(grace_start); + kfree(hexid); +} + +static void +nfsd4_umh_cltrack_remove(struct nfs4_client *clp) +{ + char *hexid; + + if (!test_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags)) + return; + + hexid = bin_to_hex_dup(clp->cl_name.data, clp->cl_name.len); + if (!hexid) { + dprintk("%s: can't allocate memory for upcall!\n", __func__); + return; + } + + nfsd4_cltrack_upcall_lock(clp); + if (test_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags) && + nfsd4_umh_cltrack_upcall("remove", hexid, NULL, NULL) == 0) + clear_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags); + nfsd4_cltrack_upcall_unlock(clp); + + kfree(hexid); +} + +static int +nfsd4_umh_cltrack_check(struct nfs4_client *clp) +{ + int ret; + char *hexid, *has_session, *legacy; + + if (test_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags)) + return 0; + + hexid = bin_to_hex_dup(clp->cl_name.data, clp->cl_name.len); + if (!hexid) { + dprintk("%s: can't allocate memory for upcall!\n", __func__); + return -ENOMEM; + } + + has_session = nfsd4_cltrack_client_has_session(clp); + legacy = nfsd4_cltrack_legacy_recdir(&clp->cl_name); + + nfsd4_cltrack_upcall_lock(clp); + if (test_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags)) { + ret = 0; + } else { + ret = nfsd4_umh_cltrack_upcall("check", hexid, has_session, legacy); + if (ret == 0) + set_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags); + } + nfsd4_cltrack_upcall_unlock(clp); + kfree(has_session); + kfree(legacy); + kfree(hexid); + + return ret; +} + +static void +nfsd4_umh_cltrack_grace_done(struct nfsd_net *nn) +{ + char *legacy; + char timestr[22]; /* FIXME: better way to determine max size? */ + + sprintf(timestr, "%lld", nn->boot_time); + legacy = nfsd4_cltrack_legacy_topdir(); + nfsd4_umh_cltrack_upcall("gracedone", timestr, legacy, NULL); + kfree(legacy); +} + +static const struct nfsd4_client_tracking_ops nfsd4_umh_tracking_ops = { + .init = nfsd4_umh_cltrack_init, + .exit = NULL, + .create = nfsd4_umh_cltrack_create, + .remove = nfsd4_umh_cltrack_remove, + .check = nfsd4_umh_cltrack_check, + .grace_done = nfsd4_umh_cltrack_grace_done, + .version = 1, + .msglen = 0, +}; + +static inline int check_for_legacy_methods(int status, struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct path path; + + /* + * Next, try the UMH upcall. + */ + nn->client_tracking_ops = &nfsd4_umh_tracking_ops; + status = nn->client_tracking_ops->init(net); + if (!status) + return status; + + /* + * Finally, See if the recoverydir exists and is a directory. + * If it is, then use the legacy ops. + */ + nn->client_tracking_ops = &nfsd4_legacy_tracking_ops; + status = kern_path(nfs4_recoverydir(), LOOKUP_FOLLOW, &path); + if (!status) { + status = !d_is_dir(path.dentry); + path_put(&path); + if (status) + return -ENOTDIR; + } + return status; +} +#else +static inline int check_for_legacy_methods(int status, struct net *net) +{ + return status; +} +#endif /* CONFIG_LEGACY_NFSD_CLIENT_TRACKING */ + +int +nfsd4_client_tracking_init(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + int status; + + /* just run the init if it the method is already decided */ + if (nn->client_tracking_ops) + goto do_init; + + /* First, try to use nfsdcld */ + nn->client_tracking_ops = &nfsd4_cld_tracking_ops; + status = nn->client_tracking_ops->init(net); + if (!status) + return status; + if (status != -ETIMEDOUT) { + nn->client_tracking_ops = &nfsd4_cld_tracking_ops_v0; + status = nn->client_tracking_ops->init(net); + if (!status) + return status; + } + + status = check_for_legacy_methods(status, net); + if (status) + goto out; +do_init: + status = nn->client_tracking_ops->init(net); +out: + if (status) { + pr_warn("NFSD: Unable to initialize client recovery tracking! (%d)\n", status); + pr_warn("NFSD: Is nfsdcld running? If not, enable CONFIG_NFSD_LEGACY_CLIENT_TRACKING.\n"); + nn->client_tracking_ops = NULL; + } + return status; +} + +void +nfsd4_client_tracking_exit(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + if (nn->client_tracking_ops) { + if (nn->client_tracking_ops->exit) + nn->client_tracking_ops->exit(net); + nn->client_tracking_ops = NULL; + } +} + +void +nfsd4_client_record_create(struct nfs4_client *clp) +{ + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + if (nn->client_tracking_ops) + nn->client_tracking_ops->create(clp); +} + +void +nfsd4_client_record_remove(struct nfs4_client *clp) +{ + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + if (nn->client_tracking_ops) + nn->client_tracking_ops->remove(clp); +} + +int +nfsd4_client_record_check(struct nfs4_client *clp) +{ + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + if (nn->client_tracking_ops) + return nn->client_tracking_ops->check(clp); + + return -EOPNOTSUPP; +} + +void +nfsd4_record_grace_done(struct nfsd_net *nn) +{ + if (nn->client_tracking_ops) + nn->client_tracking_ops->grace_done(nn); +} + +static int +rpc_pipefs_event(struct notifier_block *nb, unsigned long event, void *ptr) +{ + struct super_block *sb = ptr; + struct net *net = sb->s_fs_info; + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct cld_net *cn = nn->cld_net; + int ret = 0; + + if (!try_module_get(THIS_MODULE)) + return 0; + + if (!cn) { + module_put(THIS_MODULE); + return 0; + } + + switch (event) { + case RPC_PIPEFS_MOUNT: + ret = nfsd4_cld_register_sb(sb, cn->cn_pipe); + break; + case RPC_PIPEFS_UMOUNT: + rpc_unlink(cn->cn_pipe); + break; + default: + ret = -ENOTSUPP; + break; + } + module_put(THIS_MODULE); + return ret; +} + +static struct notifier_block nfsd4_cld_block = { + .notifier_call = rpc_pipefs_event, +}; + +int +register_cld_notifier(void) +{ + WARN_ON(!nfsd_net_id); + return rpc_pipefs_notifier_register(&nfsd4_cld_block); +} + +void +unregister_cld_notifier(void) +{ + rpc_pipefs_notifier_unregister(&nfsd4_cld_block); +} diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c new file mode 100644 index 000000000..18e17232c --- /dev/null +++ b/fs/nfsd/nfs4state.c @@ -0,0 +1,10554 @@ +/* +* Copyright (c) 2001 The Regents of the University of Michigan. +* All rights reserved. +* +* Kendrick Smith +* Andy Adamson +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions +* are met: +* +* 1. Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* 2. Redistributions in binary form must reproduce the above copyright +* notice, this list of conditions and the following disclaimer in the +* documentation and/or other materials provided with the distribution. +* 3. Neither the name of the University nor the names of its +* contributors may be used to endorse or promote products derived +* from this software without specific prior written permission. +* +* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED +* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +* DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE +* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +* +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "xdr4.h" +#include "xdr4cb.h" +#include "vfs.h" +#include "current_stateid.h" +#include "stats.h" + +#include "netns.h" +#include "pnfs.h" +#include "filecache.h" +#include "nfs4xdr_gen.h" +#include "trace.h" + +#define NFSDDBG_FACILITY NFSDDBG_PROC + +#define all_ones {{ ~0, ~0}, ~0} +static const stateid_t one_stateid = { + .si_generation = ~0, + .si_opaque = all_ones, +}; +static const stateid_t zero_stateid = { + /* all fields zero */ +}; +static const stateid_t currentstateid = { + .si_generation = 1, +}; +static const stateid_t close_stateid = { + .si_generation = 0xffffffffU, +}; + +static u64 current_sessionid = 1; + +bool nfsd_delegts_enabled __read_mostly = true; + +#define ZERO_STATEID(stateid) (!memcmp((stateid), &zero_stateid, sizeof(stateid_t))) +#define ONE_STATEID(stateid) (!memcmp((stateid), &one_stateid, sizeof(stateid_t))) +#define CURRENT_STATEID(stateid) (!memcmp((stateid), ¤tstateid, sizeof(stateid_t))) +#define CLOSE_STATEID(stateid) (!memcmp((stateid), &close_stateid, sizeof(stateid_t))) + +/* forward declarations */ +static bool check_for_locks(struct nfs4_file *fp, struct nfs4_lockowner *lowner); +static void nfs4_free_ol_stateid(struct nfs4_stid *stid); +static void nfsd4_end_grace(struct nfsd_net *nn); +static void _free_cpntf_state_locked(struct nfsd_net *nn, struct nfs4_cpntf_state *cps); +static void nfsd4_file_hash_remove(struct nfs4_file *fi); +static void deleg_reaper(struct nfsd_net *nn); + +static const struct lease_manager_operations nfsd_lease_mng_ops; + +/* Locking: */ + +enum nfsd4_st_mutex_lock_subclass { + OPEN_STATEID_MUTEX = 0, + LOCK_STATEID_MUTEX = 1, +}; + +/* + * A waitqueue for all in-progress 4.0 CLOSE operations that are waiting for + * the refcount on the open stateid to drop. + */ +static DECLARE_WAIT_QUEUE_HEAD(close_wq); + +/* + * A waitqueue where a writer to clients/#/ctl destroying a client can + * wait for cl_rpc_users to drop to 0 and then for the client to be + * unhashed. + */ +static DECLARE_WAIT_QUEUE_HEAD(expiry_wq); + +static struct kmem_cache *client_slab; +static struct kmem_cache *openowner_slab; +static struct kmem_cache *lockowner_slab; +static struct kmem_cache *file_slab; +static struct kmem_cache *stateid_slab; +static struct kmem_cache *deleg_slab; +static struct kmem_cache *odstate_slab; +static struct kmem_cache *async_copy_slab; + +static void free_session(struct nfsd4_session *); + +static const struct nfsd4_callback_ops nfsd4_cb_recall_ops; +static const struct nfsd4_callback_ops nfsd4_cb_notify_lock_ops; +static const struct nfsd4_callback_ops nfsd4_cb_getattr_ops; +static const struct nfsd4_callback_ops nfsd4_cb_notify_ops; + +static struct workqueue_struct *laundry_wq; + +int nfsd4_create_laundry_wq(void) +{ + int rc = 0; + + laundry_wq = alloc_workqueue("%s", WQ_UNBOUND, 0, "nfsd4"); + if (laundry_wq == NULL) + rc = -ENOMEM; + return rc; +} + +void nfsd4_destroy_laundry_wq(void) +{ + destroy_workqueue(laundry_wq); +} + +static bool is_session_dead(struct nfsd4_session *ses) +{ + return ses->se_dead; +} + +static __be32 mark_session_dead_locked(struct nfsd4_session *ses, int ref_held_by_me) +{ + if (atomic_read(&ses->se_ref) > ref_held_by_me) + return nfserr_jukebox; + ses->se_dead = true; + return nfs_ok; +} + +static bool is_client_expired(struct nfs4_client *clp) +{ + return clp->cl_time == 0; +} + +static void nfsd4_dec_courtesy_client_count(struct nfsd_net *nn, + struct nfs4_client *clp) +{ + if (clp->cl_state != NFSD4_ACTIVE) + atomic_add_unless(&nn->nfsd_courtesy_clients, -1, 0); +} + +static __be32 get_client_locked(struct nfs4_client *clp) +{ + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + lockdep_assert_held(&nn->client_lock); + + if (is_client_expired(clp)) + return nfserr_expired; + atomic_inc(&clp->cl_rpc_users); + nfsd4_dec_courtesy_client_count(nn, clp); + clp->cl_state = NFSD4_ACTIVE; + return nfs_ok; +} + +/* must be called under the client_lock */ +static inline void +renew_client_locked(struct nfs4_client *clp) +{ + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + if (is_client_expired(clp)) { + WARN_ON(1); + printk("%s: client (clientid %08x/%08x) already expired\n", + __func__, + clp->cl_clientid.cl_boot, + clp->cl_clientid.cl_id); + return; + } + + list_move_tail(&clp->cl_lru, &nn->client_lru); + clp->cl_time = ktime_get_boottime_seconds(); + nfsd4_dec_courtesy_client_count(nn, clp); + clp->cl_state = NFSD4_ACTIVE; +} + +/* + * Finish a cl_rpc_users unpin with the client_lock held. A + * revocation walk clears @renew so the client whose state it is + * revoking is not revived; every other caller renews the lease of + * a still-active client. + */ +static void __put_client_locked(struct nfs4_client *clp, bool renew) +{ + if (is_client_expired(clp)) + wake_up_all(&expiry_wq); + else if (renew) + renew_client_locked(clp); +} + +static void put_client_renew_locked(struct nfs4_client *clp) +{ + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + lockdep_assert_held(&nn->client_lock); + + if (atomic_dec_and_test(&clp->cl_rpc_users)) + __put_client_locked(clp, true); +} + +static void put_client_renew(struct nfs4_client *clp) +{ + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + if (!atomic_dec_and_lock(&clp->cl_rpc_users, &nn->client_lock)) + return; + __put_client_locked(clp, true); + spin_unlock(&nn->client_lock); +} + +static void put_client_no_renew_locked(struct nfs4_client *clp) +{ + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + lockdep_assert_held(&nn->client_lock); + + if (atomic_dec_and_test(&clp->cl_rpc_users)) + __put_client_locked(clp, false); +} + +static void put_client_no_renew(struct nfs4_client *clp) +{ + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + if (!atomic_dec_and_lock(&clp->cl_rpc_users, &nn->client_lock)) + return; + __put_client_locked(clp, false); + spin_unlock(&nn->client_lock); +} + +static __be32 nfsd4_get_session_locked(struct nfsd4_session *ses) +{ + __be32 status; + + if (is_session_dead(ses)) + return nfserr_badsession; + status = get_client_locked(ses->se_client); + if (status) + return status; + atomic_inc(&ses->se_ref); + return nfs_ok; +} + +static void nfsd4_put_session_locked(struct nfsd4_session *ses) +{ + struct nfs4_client *clp = ses->se_client; + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + lockdep_assert_held(&nn->client_lock); + + if (atomic_dec_and_test(&ses->se_ref) && is_session_dead(ses)) + free_session(ses); + put_client_renew_locked(clp); +} + +static void nfsd4_put_session(struct nfsd4_session *ses) +{ + struct nfs4_client *clp = ses->se_client; + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + spin_lock(&nn->client_lock); + nfsd4_put_session_locked(ses); + spin_unlock(&nn->client_lock); +} + +static struct nfsd4_blocked_lock * +find_blocked_lock(struct nfs4_lockowner *lo, struct knfsd_fh *fh, + struct nfsd_net *nn) +{ + struct nfsd4_blocked_lock *cur, *found = NULL; + + spin_lock(&nn->blocked_locks_lock); + list_for_each_entry(cur, &lo->lo_blocked, nbl_list) { + if (fh_match(fh, &cur->nbl_fh)) { + list_del_init(&cur->nbl_list); + WARN_ON(list_empty(&cur->nbl_lru)); + list_del_init(&cur->nbl_lru); + found = cur; + break; + } + } + spin_unlock(&nn->blocked_locks_lock); + if (found) + locks_delete_block(&found->nbl_lock); + return found; +} + +static struct nfsd4_blocked_lock * +find_or_allocate_block(struct nfs4_lockowner *lo, struct knfsd_fh *fh, + struct nfsd_net *nn) +{ + struct nfsd4_blocked_lock *nbl; + + nbl = find_blocked_lock(lo, fh, nn); + if (!nbl) { + nbl = kmalloc_obj(*nbl); + if (nbl) { + INIT_LIST_HEAD(&nbl->nbl_list); + INIT_LIST_HEAD(&nbl->nbl_lru); + fh_copy_shallow(&nbl->nbl_fh, fh); + locks_init_lock(&nbl->nbl_lock); + kref_init(&nbl->nbl_kref); + nfsd4_init_cb(&nbl->nbl_cb, lo->lo_owner.so_client, + &nfsd4_cb_notify_lock_ops, + NFSPROC4_CLNT_CB_NOTIFY_LOCK); + } + } + return nbl; +} + +static void +free_nbl(struct kref *kref) +{ + struct nfsd4_blocked_lock *nbl; + + nbl = container_of(kref, struct nfsd4_blocked_lock, nbl_kref); + locks_release_private(&nbl->nbl_lock); + kfree(nbl); +} + +static void +free_blocked_lock(struct nfsd4_blocked_lock *nbl) +{ + locks_delete_block(&nbl->nbl_lock); + kref_put(&nbl->nbl_kref, free_nbl); +} + +/* A blocked lock's flc_owner is its nfs4_lockowner. */ +static struct nfs4_client * +nbl_client(struct nfsd4_blocked_lock *nbl) +{ + struct nfs4_lockowner *lo; + + lo = (struct nfs4_lockowner *)nbl->nbl_lock.c.flc_owner; + return lo->lo_owner.so_client; +} + +static void +remove_blocked_locks(struct nfs4_lockowner *lo) +{ + struct nfs4_client *clp = lo->lo_owner.so_client; + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + struct nfsd4_blocked_lock *nbl; + LIST_HEAD(reaplist); + + /* Dequeue all blocked locks */ + spin_lock(&nn->blocked_locks_lock); + while (!list_empty(&lo->lo_blocked)) { + nbl = list_first_entry(&lo->lo_blocked, + struct nfsd4_blocked_lock, + nbl_list); + list_del_init(&nbl->nbl_list); + WARN_ON(list_empty(&nbl->nbl_lru)); + list_move(&nbl->nbl_lru, &reaplist); + } + spin_unlock(&nn->blocked_locks_lock); + + /* Now free them */ + while (!list_empty(&reaplist)) { + nbl = list_first_entry(&reaplist, struct nfsd4_blocked_lock, + nbl_lru); + list_del_init(&nbl->nbl_lru); + free_blocked_lock(nbl); + } +} + +static bool +nfsd4_cb_notify_lock_prepare(struct nfsd4_callback *cb) +{ + struct nfsd4_blocked_lock *nbl = container_of(cb, + struct nfsd4_blocked_lock, nbl_cb); + locks_delete_block(&nbl->nbl_lock); + return true; +} + +static int +nfsd4_cb_notify_lock_done(struct nfsd4_callback *cb, struct rpc_task *task) +{ + trace_nfsd_cb_notify_lock_done(&zero_stateid, task); + + /* + * Since this is just an optimization, we don't try very hard if it + * turns out not to succeed. We'll requeue it on NFS4ERR_DELAY, and + * just quit trying on anything else. + */ + switch (task->tk_status) { + case -NFS4ERR_DELAY: + rpc_delay(task, 1 * HZ); + return 0; + default: + return 1; + } +} + +static void +nfsd4_cb_notify_lock_release(struct nfsd4_callback *cb) +{ + struct nfsd4_blocked_lock *nbl = container_of(cb, + struct nfsd4_blocked_lock, nbl_cb); + + free_blocked_lock(nbl); +} + +static const struct nfsd4_callback_ops nfsd4_cb_notify_lock_ops = { + .prepare = nfsd4_cb_notify_lock_prepare, + .done = nfsd4_cb_notify_lock_done, + .release = nfsd4_cb_notify_lock_release, + .opcode = OP_CB_NOTIFY_LOCK, +}; + +/* + * We store the NONE, READ, WRITE, and BOTH bits separately in the + * st_{access,deny}_bmap field of the stateid, in order to track not + * only what share bits are currently in force, but also what + * combinations of share bits previous opens have used. This allows us + * to enforce the recommendation in + * https://datatracker.ietf.org/doc/html/rfc7530#section-16.19.4 that + * the server return an error if the client attempt to downgrade to a + * combination of share bits not explicable by closing some of its + * previous opens. + * + * This enforcement is arguably incomplete, since we don't keep + * track of access/deny bit combinations; so, e.g., we allow: + * + * OPEN allow read, deny write + * OPEN allow both, deny none + * DOWNGRADE allow read, deny none + * + * which we should reject. + * + * But you could also argue that our current code is already overkill, + * since it only exists to return NFS4ERR_INVAL on incorrect client + * behavior. + */ +static unsigned int +bmap_to_share_mode(unsigned long bmap) +{ + int i; + unsigned int access = 0; + + for (i = 1; i < 4; i++) { + if (test_bit(i, &bmap)) + access |= i; + } + return access; +} + +/* set share access for a given stateid */ +static inline void +set_access(u32 access, struct nfs4_ol_stateid *stp) +{ + unsigned char mask = 1 << access; + + WARN_ON_ONCE(access > NFS4_SHARE_ACCESS_BOTH); + stp->st_access_bmap |= mask; +} + +/* clear share access for a given stateid */ +static inline void +clear_access(u32 access, struct nfs4_ol_stateid *stp) +{ + unsigned char mask = 1 << access; + + WARN_ON_ONCE(access > NFS4_SHARE_ACCESS_BOTH); + stp->st_access_bmap &= ~mask; +} + +/* test whether a given stateid has access */ +static inline bool +test_access(u32 access, struct nfs4_ol_stateid *stp) +{ + unsigned char mask = 1 << access; + + return (bool)(stp->st_access_bmap & mask); +} + +/* set share deny for a given stateid */ +static inline void +set_deny(u32 deny, struct nfs4_ol_stateid *stp) +{ + unsigned char mask = 1 << deny; + + WARN_ON_ONCE(deny > NFS4_SHARE_DENY_BOTH); + stp->st_deny_bmap |= mask; +} + +/* clear share deny for a given stateid */ +static inline void +clear_deny(u32 deny, struct nfs4_ol_stateid *stp) +{ + unsigned char mask = 1 << deny; + + WARN_ON_ONCE(deny > NFS4_SHARE_DENY_BOTH); + stp->st_deny_bmap &= ~mask; +} + +/* test whether a given stateid is denying specific access */ +static inline bool +test_deny(u32 deny, struct nfs4_ol_stateid *stp) +{ + unsigned char mask = 1 << deny; + + return (bool)(stp->st_deny_bmap & mask); +} + +static int nfs4_access_to_omode(u32 access) +{ + switch (access & NFS4_SHARE_ACCESS_BOTH) { + case NFS4_SHARE_ACCESS_READ: + return O_RDONLY; + case NFS4_SHARE_ACCESS_WRITE: + return O_WRONLY; + case NFS4_SHARE_ACCESS_BOTH: + return O_RDWR; + } + WARN_ON_ONCE(1); + return O_RDONLY; +} + +static inline int +access_permit_read(struct nfs4_ol_stateid *stp) +{ + return test_access(NFS4_SHARE_ACCESS_READ, stp) || + test_access(NFS4_SHARE_ACCESS_BOTH, stp) || + test_access(NFS4_SHARE_ACCESS_WRITE, stp); +} + +static inline int +access_permit_write(struct nfs4_ol_stateid *stp) +{ + return test_access(NFS4_SHARE_ACCESS_WRITE, stp) || + test_access(NFS4_SHARE_ACCESS_BOTH, stp); +} + +static inline struct nfs4_stateowner * +nfs4_get_stateowner(struct nfs4_stateowner *sop) +{ + atomic_inc(&sop->so_count); + return sop; +} + +static int +same_owner_str(struct nfs4_stateowner *sop, struct xdr_netobj *owner) +{ + return (sop->so_owner.len == owner->len) && + 0 == memcmp(sop->so_owner.data, owner->data, owner->len); +} + +static struct nfs4_openowner * +find_openstateowner_str(unsigned int hashval, struct nfsd4_open *open, + struct nfs4_client *clp) +{ + struct nfs4_stateowner *so; + + lockdep_assert_held(&clp->cl_lock); + + list_for_each_entry(so, &clp->cl_ownerstr_hashtbl[hashval], + so_strhash) { + if (!so->so_is_open_owner) + continue; + if (same_owner_str(so, &open->op_owner)) + return openowner(nfs4_get_stateowner(so)); + } + return NULL; +} + +static inline u32 +opaque_hashval(const void *ptr, int nbytes) +{ + unsigned char *cptr = (unsigned char *) ptr; + + u32 x = 0; + while (nbytes--) { + x *= 37; + x += *cptr++; + } + return x; +} + +void +put_nfs4_file(struct nfs4_file *fi) +{ + if (refcount_dec_and_test(&fi->fi_ref)) { + nfsd4_file_hash_remove(fi); + WARN_ON_ONCE(!list_empty(&fi->fi_clnt_odstate)); + WARN_ON_ONCE(!list_empty(&fi->fi_delegations)); + kfree_rcu(fi, fi_rcu); + } +} + +static struct nfsd_file * +find_writeable_file_locked(struct nfs4_file *f) +{ + struct nfsd_file *ret; + + lockdep_assert_held(&f->fi_lock); + + ret = nfsd_file_get(f->fi_fds[O_WRONLY]); + if (!ret) + ret = nfsd_file_get(f->fi_fds[O_RDWR]); + return ret; +} + +static struct nfsd_file * +find_writeable_file(struct nfs4_file *f) +{ + struct nfsd_file *ret; + + spin_lock(&f->fi_lock); + ret = find_writeable_file_locked(f); + spin_unlock(&f->fi_lock); + + return ret; +} + +static struct nfsd_file * +find_readable_file_locked(struct nfs4_file *f) +{ + struct nfsd_file *ret; + + lockdep_assert_held(&f->fi_lock); + + ret = nfsd_file_get(f->fi_fds[O_RDONLY]); + if (!ret) + ret = nfsd_file_get(f->fi_fds[O_RDWR]); + return ret; +} + +static struct nfsd_file * +find_readable_file(struct nfs4_file *f) +{ + struct nfsd_file *ret; + + spin_lock(&f->fi_lock); + ret = find_readable_file_locked(f); + spin_unlock(&f->fi_lock); + + return ret; +} + +struct nfsd_file * +find_any_file(struct nfs4_file *f) +{ + struct nfsd_file *ret; + + if (!f) + return NULL; + spin_lock(&f->fi_lock); + ret = nfsd_file_get(f->fi_fds[O_RDWR]); + if (!ret) { + ret = nfsd_file_get(f->fi_fds[O_WRONLY]); + if (!ret) + ret = nfsd_file_get(f->fi_fds[O_RDONLY]); + } + spin_unlock(&f->fi_lock); + return ret; +} + +static struct nfsd_file *find_any_file_locked(struct nfs4_file *f) +{ + lockdep_assert_held(&f->fi_lock); + + if (f->fi_fds[O_RDWR]) + return f->fi_fds[O_RDWR]; + if (f->fi_fds[O_WRONLY]) + return f->fi_fds[O_WRONLY]; + if (f->fi_fds[O_RDONLY]) + return f->fi_fds[O_RDONLY]; + return NULL; +} + +static atomic_long_t num_delegations; +unsigned long max_delegations; + +/* + * Open owner state (share locks) + */ + +/* hash tables for lock and open owners */ +#define OWNER_HASH_BITS 8 +#define OWNER_HASH_SIZE (1 << OWNER_HASH_BITS) +#define OWNER_HASH_MASK (OWNER_HASH_SIZE - 1) + +static unsigned int ownerstr_hashval(struct xdr_netobj *ownername) +{ + unsigned int ret; + + ret = opaque_hashval(ownername->data, ownername->len); + return ret & OWNER_HASH_MASK; +} + +static struct rhltable nfs4_file_rhltable ____cacheline_aligned_in_smp; + +static const struct rhashtable_params nfs4_file_rhash_params = { + .key_len = sizeof_field(struct nfs4_file, fi_inode), + .key_offset = offsetof(struct nfs4_file, fi_inode), + .head_offset = offsetof(struct nfs4_file, fi_rlist), + + /* + * Start with a single page hash table to reduce resizing churn + * on light workloads. + */ + .min_size = 256, + .automatic_shrinking = true, +}; + +/* + * Check if courtesy clients have conflicting access and resolve it if possible + * + * access: is op_share_access if share_access is true. + * Check if access mode, op_share_access, would conflict with + * the current deny mode of the file 'fp'. + * access: is op_share_deny if share_access is false. + * Check if the deny mode, op_share_deny, would conflict with + * current access of the file 'fp'. + * stp: skip checking this entry. + * new_stp: normal open, not open upgrade. + * + * Function returns: + * false - access/deny mode conflict with normal client. + * true - no conflict or conflict with courtesy client(s) is resolved. + */ +static bool +nfs4_resolve_deny_conflicts_locked(struct nfs4_file *fp, bool new_stp, + struct nfs4_ol_stateid *stp, u32 access, bool share_access) +{ + struct nfs4_ol_stateid *st; + bool resolvable = true; + unsigned char bmap; + struct nfsd_net *nn; + struct nfs4_client *clp; + + lockdep_assert_held(&fp->fi_lock); + list_for_each_entry(st, &fp->fi_stateids, st_perfile) { + /* ignore lock stateid */ + if (st->st_openstp) + continue; + if (st == stp && new_stp) + continue; + /* check file access against deny mode or vice versa */ + bmap = share_access ? st->st_deny_bmap : st->st_access_bmap; + if (!(access & bmap_to_share_mode(bmap))) + continue; + clp = st->st_stid.sc_client; + if (try_to_expire_client(clp)) + continue; + resolvable = false; + break; + } + if (resolvable) { + clp = stp->st_stid.sc_client; + nn = net_generic(clp->net, nfsd_net_id); + mod_delayed_work(laundry_wq, &nn->laundromat_work, 0); + } + return resolvable; +} + +static void +__nfs4_file_get_access(struct nfs4_file *fp, u32 access) +{ + lockdep_assert_held(&fp->fi_lock); + + if (access & NFS4_SHARE_ACCESS_WRITE) + atomic_inc(&fp->fi_access[O_WRONLY]); + if (access & NFS4_SHARE_ACCESS_READ) + atomic_inc(&fp->fi_access[O_RDONLY]); +} + +static __be32 +nfs4_file_get_access(struct nfs4_file *fp, u32 access) +{ + lockdep_assert_held(&fp->fi_lock); + + /* Does this access mode make sense? */ + if (access & ~NFS4_SHARE_ACCESS_BOTH) + return nfserr_inval; + + /* Does it conflict with a deny mode already set? */ + if ((access & fp->fi_share_deny) != 0) + return nfserr_share_denied; + + __nfs4_file_get_access(fp, access); + return nfs_ok; +} + +static __be32 nfs4_file_check_deny(struct nfs4_file *fp, u32 deny) +{ + /* Common case is that there is no deny mode. */ + if (deny) { + /* Does this deny mode make sense? */ + if (deny & ~NFS4_SHARE_DENY_BOTH) + return nfserr_inval; + + if ((deny & NFS4_SHARE_DENY_READ) && + atomic_read(&fp->fi_access[O_RDONLY])) + return nfserr_share_denied; + + if ((deny & NFS4_SHARE_DENY_WRITE) && + atomic_read(&fp->fi_access[O_WRONLY])) + return nfserr_share_denied; + } + return nfs_ok; +} + +static void __nfs4_file_put_access(struct nfs4_file *fp, int oflag) +{ + might_lock(&fp->fi_lock); + + if (atomic_dec_and_lock(&fp->fi_access[oflag], &fp->fi_lock)) { + struct nfsd_file *f1 = NULL; + struct nfsd_file *f2 = NULL; + + swap(f1, fp->fi_fds[oflag]); + if (atomic_read(&fp->fi_access[1 - oflag]) == 0) + swap(f2, fp->fi_fds[O_RDWR]); + spin_unlock(&fp->fi_lock); + if (f1) + nfsd_file_put(f1); + if (f2) + nfsd_file_put(f2); + } +} + +static void nfs4_file_put_access(struct nfs4_file *fp, u32 access) +{ + WARN_ON_ONCE(access & ~NFS4_SHARE_ACCESS_BOTH); + + if (access & NFS4_SHARE_ACCESS_WRITE) + __nfs4_file_put_access(fp, O_WRONLY); + if (access & NFS4_SHARE_ACCESS_READ) + __nfs4_file_put_access(fp, O_RDONLY); +} + +/* + * Allocate a new open/delegation state counter. This is needed for + * pNFS for proper return on close semantics. + * + * Note that we only allocate it for pNFS-enabled exports, otherwise + * all pointers to struct nfs4_clnt_odstate are always NULL. + */ +static struct nfs4_clnt_odstate * +alloc_clnt_odstate(struct nfs4_client *clp) +{ + struct nfs4_clnt_odstate *co; + + co = kmem_cache_zalloc(odstate_slab, GFP_KERNEL); + if (co) { + co->co_client = clp; + refcount_set(&co->co_odcount, 1); + } + return co; +} + +static void +hash_clnt_odstate_locked(struct nfs4_clnt_odstate *co) +{ + struct nfs4_file *fp = co->co_file; + + lockdep_assert_held(&fp->fi_lock); + list_add(&co->co_perfile, &fp->fi_clnt_odstate); +} + +static inline void +get_clnt_odstate(struct nfs4_clnt_odstate *co) +{ + if (co) + refcount_inc(&co->co_odcount); +} + +static void +put_clnt_odstate(struct nfs4_clnt_odstate *co) +{ + struct nfs4_file *fp; + + if (!co) + return; + + fp = co->co_file; + if (refcount_dec_and_lock(&co->co_odcount, &fp->fi_lock)) { + list_del(&co->co_perfile); + spin_unlock(&fp->fi_lock); + + nfsd4_return_all_file_layouts(co->co_client, fp); + kmem_cache_free(odstate_slab, co); + } +} + +static struct nfs4_clnt_odstate * +find_or_hash_clnt_odstate(struct nfs4_file *fp, struct nfs4_clnt_odstate *new) +{ + struct nfs4_clnt_odstate *co; + struct nfs4_client *cl; + + if (!new) + return NULL; + + cl = new->co_client; + + spin_lock(&fp->fi_lock); + list_for_each_entry(co, &fp->fi_clnt_odstate, co_perfile) { + if (co->co_client == cl) { + get_clnt_odstate(co); + goto out; + } + } + co = new; + co->co_file = fp; + hash_clnt_odstate_locked(new); +out: + spin_unlock(&fp->fi_lock); + return co; +} + +struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *slab, + void (*sc_free)(struct nfs4_stid *)) +{ + struct nfs4_stid *stid; + int new_id; + + stid = kmem_cache_zalloc(slab, GFP_KERNEL); + if (!stid) + return NULL; + + idr_preload(GFP_KERNEL); + spin_lock(&cl->cl_lock); + /* Reserving 0 for start of file in nfsdfs "states" file: */ + new_id = idr_alloc_cyclic(&cl->cl_stateids, stid, 1, 0, GFP_NOWAIT); + spin_unlock(&cl->cl_lock); + idr_preload_end(); + if (new_id < 0) + goto out_free; + + stid->sc_free = sc_free; + stid->sc_client = cl; + stid->sc_stateid.si_opaque.so_id = new_id; + stid->sc_stateid.si_opaque.so_clid = cl->cl_clientid; + /* Will be incremented before return to client: */ + refcount_set(&stid->sc_count, 1); + spin_lock_init(&stid->sc_lock); + INIT_LIST_HEAD(&stid->sc_cp_list); + + return stid; +out_free: + kmem_cache_free(slab, stid); + return NULL; +} + +/* + * Publish a COPY_NOTIFY stateid in nn->s2s_cp_stateids and link it onto the + * parent's sc_cp_list. That IDR holds only COPY_NOTIFY stateids. + */ +static int nfs4_init_cp_state(struct nfsd_net *nn, copy_stateid_t *stid, + struct nfs4_stid *p_stid) +{ + int new_id; + + stid->cs_stid.si_opaque.so_clid.cl_boot = (u32)nn->boot_time; + stid->cs_stid.si_opaque.so_clid.cl_id = nn->s2s_cp_cl_id; + + idr_preload(GFP_KERNEL); + spin_lock(&nn->s2s_cp_lock); + new_id = idr_alloc_cyclic(&nn->s2s_cp_stateids, stid, 0, 0, GFP_NOWAIT); + if (new_id >= 0) { + struct nfs4_cpntf_state *cps = + container_of(stid, struct nfs4_cpntf_state, cp_stateid); + + stid->cs_stid.si_opaque.so_id = new_id; + stid->cs_stid.si_generation = 1; + /* + * Set cs_type and link onto sc_cp_list under the same lock + * that installed the IDR entry, so a concurrent + * manage_cpntf_state() sees either no entry or a fully + * linked cp_list. + */ + stid->cs_type = NFS4_COPYNOTIFY_STID; + list_add(&cps->cp_list, &p_stid->sc_cp_list); + } + spin_unlock(&nn->s2s_cp_lock); + idr_preload_end(); + if (new_id < 0) + return 0; + return 1; +} + +/* sc_free for a copy offload stateid; runs from nfs4_put_stid(). */ +static void nfsd4_free_async_copy_stid(struct nfs4_stid *stid) +{ + struct nfsd4_async_copy *copy = + container_of(stid, struct nfsd4_async_copy, cp_stid); + + if (copy->copy_task) + put_task_struct(copy->copy_task); + kfree(copy->cp_copy.cp_src); + kmem_cache_free(async_copy_slab, copy); +} + +/* + * Allocate durable async COPY state. The offload stateid is a first-class + * nfs4_stid (SC_TYPE_COPY) in the client's cl_stateids, so it is per-client + * and uses the common refcounting/teardown. find_stateid_locked() hides it; + * OFFLOAD_CANCEL/OFFLOAD_STATUS find it via clp->async_copies. + */ +struct nfsd4_async_copy *nfs4_alloc_copy_stid(struct nfs4_client *clp) +{ + struct nfs4_stid *stid; + + stid = nfs4_alloc_stid(clp, async_copy_slab, nfsd4_free_async_copy_stid); + if (!stid) + return NULL; + stid->sc_type = SC_TYPE_COPY; + /* RFC 7862 Section 4.8: a copy offload stateid's seqid MUST NOT be 0 */ + stid->sc_stateid.si_generation = 1; + return container_of(stid, struct nfsd4_async_copy, cp_stid); +} + +struct nfs4_cpntf_state *nfs4_alloc_init_cpntf_state(struct nfsd_net *nn, + struct nfs4_stid *p_stid) +{ + struct nfs4_cpntf_state *cps; + + cps = kzalloc_obj(struct nfs4_cpntf_state); + if (!cps) + return NULL; + /* So a stale list_del_init() before linking is a no-op. */ + INIT_LIST_HEAD(&cps->cp_list); + cps->cpntf_time = ktime_get_boottime_seconds(); + /* + * Fully initialize the entry before nfs4_init_cp_state() publishes it, + * since a concurrent OFFLOAD_CANCEL could then free it. Take an extra + * reference for the caller (dropped with nfs4_put_cpntf_state()). + */ + memcpy(&cps->cp_p_stateid, &p_stid->sc_stateid, sizeof(stateid_t)); + memcpy(&cps->cp_p_clid, &p_stid->sc_client->cl_clientid, + sizeof(clientid_t)); + refcount_set(&cps->cp_stateid.cs_count, 2); + if (!nfs4_init_cp_state(nn, &cps->cp_stateid, p_stid)) + goto out_free; + return cps; +out_free: + kfree(cps); + return NULL; +} + +/* + * Drop the parent's reference on an already-unlinked cpntf entry. If a + * concurrent holder still owns a reference, its nfs4_put_cpntf_state() does + * the final free. + * + * nn->s2s_cp_lock must be held. + */ +static void put_cpntf_state_unlinked_locked(struct nfs4_cpntf_state *cps) +{ + WARN_ON_ONCE(cps->cp_stateid.cs_type != NFS4_COPYNOTIFY_STID); + WARN_ON_ONCE(!list_empty(&cps->cp_list)); + + if (refcount_dec_and_test(&cps->cp_stateid.cs_count)) + kfree(cps); +} + +/* + * Unhash from the IDR and sc_cp_list. Gated on list_empty() to avoid + * evicting a recycled so_id. + */ +static void nfsd4_unhash_cpntf_state(struct nfsd_net *nn, struct nfs4_cpntf_state *cps) +{ + lockdep_assert_held(&nn->s2s_cp_lock); + + if (!list_empty(&cps->cp_list)) { + list_del_init(&cps->cp_list); + idr_remove(&nn->s2s_cp_stateids, cps->cp_stateid.cs_stid.si_opaque.so_id); + } +} + +/* + * Revoke a copy-notify stateid: unlink it from the IDR and sc_cp_list first + * so no new finder can discover it, then drop the membership reference. Every + * revoke path (cancel, laundromat, drain) must use this rather than + * _free_cpntf_state_locked(), which unlinks only at refcount zero and so could + * let a second revoke free the entry under a concurrent reader. + * + * nn->s2s_cp_lock must be held. + */ +static void revoke_cpntf_state_locked(struct nfsd_net *nn, + struct nfs4_cpntf_state *cps) +{ + nfsd4_unhash_cpntf_state(nn, cps); + put_cpntf_state_unlinked_locked(cps); +} + +static void nfs4_free_cpntf_statelist(struct net *net, struct nfs4_stid *stid) +{ + struct nfs4_cpntf_state *cps, *tmp; + struct nfsd_net *nn; + + nn = net_generic(net, nfsd_net_id); + spin_lock(&nn->s2s_cp_lock); + /* + * Revoke unlinks each entry before dropping the parent's reference, so + * the drain terminates in one pass per entry regardless of cs_count; a + * concurrent holder does the final kfree via nfs4_put_cpntf_state(). + */ + list_for_each_entry_safe(cps, tmp, &stid->sc_cp_list, cp_list) + revoke_cpntf_state_locked(nn, cps); + spin_unlock(&nn->s2s_cp_lock); +} + +static struct nfs4_ol_stateid * nfs4_alloc_open_stateid(struct nfs4_client *clp) +{ + struct nfs4_stid *stid; + + stid = nfs4_alloc_stid(clp, stateid_slab, nfs4_free_ol_stateid); + if (!stid) + return NULL; + + return openlockstateid(stid); +} + +/* + * As the sc_free callback of deleg, this may be called by nfs4_put_stid + * in nfsd_break_one_deleg. + * Considering nfsd_break_one_deleg is called with the flc->flc_lock held, + * this function mustn't ever sleep. + */ +static void nfs4_free_deleg(struct nfs4_stid *stid) +{ + struct nfs4_delegation *dp = delegstateid(stid); + + WARN_ON_ONCE(!list_empty(&stid->sc_cp_list)); + WARN_ON_ONCE(!list_empty(&dp->dl_perfile)); + WARN_ON_ONCE(!list_empty(&dp->dl_perclnt)); + WARN_ON_ONCE(!list_empty(&dp->dl_recall_lru)); + kmem_cache_free(deleg_slab, stid); + atomic_long_dec(&num_delegations); +} + +/* + * When we recall a delegation, we should be careful not to hand it + * out again straight away. + * To ensure this we keep a pair of bloom filters ('new' and 'old') + * in which the filehandles of recalled delegations are "stored". + * If a filehandle appear in either filter, a delegation is blocked. + * When a delegation is recalled, the filehandle is stored in the "new" + * filter. + * Every 30 seconds we swap the filters and clear the "new" one, + * unless both are empty of course. This results in delegations for a + * given filehandle being blocked for between 30 and 60 seconds. + * + * Each filter is 256 bits. We hash the filehandle to 32bit and use the + * low 3 bytes as hash-table indices. + * + * 'blocked_delegations_lock', which is always taken in block_delegations(), + * is used to manage concurrent access. Testing does not need the lock + * except when swapping the two filters. + */ +static DEFINE_SPINLOCK(blocked_delegations_lock); +static struct bloom_pair { + int entries, old_entries; + time64_t swap_time; + int new; /* index into 'set' */ + DECLARE_BITMAP(set[2], 256); +} blocked_delegations; + +static int delegation_blocked(struct knfsd_fh *fh) +{ + u32 hash; + struct bloom_pair *bd = &blocked_delegations; + + if (bd->entries == 0) + return 0; + if (ktime_get_seconds() - bd->swap_time > 30) { + spin_lock(&blocked_delegations_lock); + if (ktime_get_seconds() - bd->swap_time > 30) { + bd->entries -= bd->old_entries; + bd->old_entries = bd->entries; + bd->new = 1-bd->new; + memset(bd->set[bd->new], 0, + sizeof(bd->set[0])); + bd->swap_time = ktime_get_seconds(); + } + spin_unlock(&blocked_delegations_lock); + } + hash = jhash(&fh->fh_raw, fh->fh_size, 0); + if (test_bit(hash&255, bd->set[0]) && + test_bit((hash>>8)&255, bd->set[0]) && + test_bit((hash>>16)&255, bd->set[0])) + return 1; + + if (test_bit(hash&255, bd->set[1]) && + test_bit((hash>>8)&255, bd->set[1]) && + test_bit((hash>>16)&255, bd->set[1])) + return 1; + + return 0; +} + +static void block_delegations(struct knfsd_fh *fh) +{ + u32 hash; + struct bloom_pair *bd = &blocked_delegations; + + hash = jhash(&fh->fh_raw, fh->fh_size, 0); + + spin_lock(&blocked_delegations_lock); + __set_bit(hash&255, bd->set[bd->new]); + __set_bit((hash>>8)&255, bd->set[bd->new]); + __set_bit((hash>>16)&255, bd->set[bd->new]); + if (bd->entries == 0) + bd->swap_time = ktime_get_seconds(); + bd->entries += 1; + spin_unlock(&blocked_delegations_lock); +} + +static struct nfs4_delegation * +__alloc_init_deleg(struct nfs4_client *clp, struct nfs4_file *fp, + struct nfs4_clnt_odstate *odstate, u32 dl_type, + void (*sc_free)(struct nfs4_stid *)) +{ + struct nfs4_delegation *dp; + struct nfs4_stid *stid; + long n; + + if (delegation_blocked(&fp->fi_fhandle)) + return NULL; + + n = atomic_long_inc_return(&num_delegations); + if (n < 0 || n > max_delegations) + goto out_dec; + + stid = nfs4_alloc_stid(clp, deleg_slab, sc_free); + if (stid == NULL) + goto out_dec; + + /* + * delegation seqid's are never incremented. The 4.1 special + * meaning of seqid 0 isn't meaningful, really, but let's avoid + * 0 anyway just for consistency and use 1. + */ + dp = delegstateid(stid); + dp->dl_stid.sc_stateid.si_generation = 1; + INIT_LIST_HEAD(&dp->dl_perfile); + INIT_LIST_HEAD(&dp->dl_perclnt); + INIT_LIST_HEAD(&dp->dl_recall_lru); + dp->dl_clnt_odstate = odstate; + get_clnt_odstate(odstate); + dp->dl_type = dl_type; + dp->dl_retries = 1; + dp->dl_recalled = false; + get_nfs4_file(fp); + dp->dl_stid.sc_file = fp; + nfsd4_init_cb(&dp->dl_recall, dp->dl_stid.sc_client, + &nfsd4_cb_recall_ops, NFSPROC4_CLNT_CB_RECALL); + return dp; +out_dec: + atomic_long_dec(&num_delegations); + return NULL; +} + +static struct nfs4_delegation * +alloc_init_deleg(struct nfs4_client *clp, struct nfs4_file *fp, + struct nfs4_clnt_odstate *odstate, u32 dl_type) +{ + struct nfs4_delegation *dp; + + dp = __alloc_init_deleg(clp, fp, odstate, dl_type, nfs4_free_deleg); + if (!dp) + return NULL; + + nfsd4_init_cb(&dp->dl_cb_fattr.ncf_getattr, dp->dl_stid.sc_client, + &nfsd4_cb_getattr_ops, NFSPROC4_CLNT_CB_GETATTR); + dp->dl_cb_fattr.ncf_file_modified = false; + return dp; +} + +static void nfs4_free_dir_deleg(struct nfs4_stid *stid) +{ + struct nfs4_delegation *dp = delegstateid(stid); + struct nfsd4_cb_notify *ncn = &dp->dl_cb_notify; + int i; + + for (i = 0; i < ncn->ncn_evt_cnt; ++i) + nfsd_notify_event_put(ncn->ncn_evt[i]); + kfree(ncn->ncn_nf); + for (i = 0; i < NOTIFY4_PAGE_ARRAY_SIZE; i++) { + if (!ncn->ncn_pages[i]) + break; + put_page(ncn->ncn_pages[i]); + } + nfs4_free_deleg(stid); +} + +static struct nfs4_delegation * +alloc_init_dir_deleg(struct nfs4_client *clp, struct nfs4_file *fp) +{ + struct nfs4_delegation *dp; + struct nfsd4_cb_notify *ncn; + int npages; + + dp = __alloc_init_deleg(clp, fp, NULL, NFS4_OPEN_DELEGATE_READ, nfs4_free_dir_deleg); + if (!dp) + return NULL; + + ncn = &dp->dl_cb_notify; + + npages = alloc_pages_bulk(GFP_KERNEL, NOTIFY4_PAGE_ARRAY_SIZE, ncn->ncn_pages); + if (npages != NOTIFY4_PAGE_ARRAY_SIZE) { + nfs4_put_stid(&dp->dl_stid); + return NULL; + } + + ncn->ncn_nf = kcalloc(NOTIFY4_EVENT_QUEUE_SIZE, sizeof(*ncn->ncn_nf), GFP_KERNEL); + if (!ncn->ncn_nf) { + nfs4_put_stid(&dp->dl_stid); + return NULL; + } + spin_lock_init(&ncn->ncn_lock); + nfsd4_init_cb(&ncn->ncn_cb, dp->dl_stid.sc_client, + &nfsd4_cb_notify_ops, NFSPROC4_CLNT_CB_NOTIFY); + return dp; +} + +void +nfs4_put_stid(struct nfs4_stid *s) +{ + struct nfs4_file *fp = s->sc_file; + struct nfs4_client *clp = s->sc_client; + struct svc_export *exp; + + might_lock(&clp->cl_lock); + + if (!refcount_dec_and_lock(&s->sc_count, &clp->cl_lock)) { + wake_up_all(&close_wq); + return; + } + idr_remove(&clp->cl_stateids, s->sc_stateid.si_opaque.so_id); + if (s->sc_status & SC_STATUS_ADMIN_REVOKED) + atomic_dec(&s->sc_client->cl_admin_revoked); + /* Read under cl_lock to serialize with drop_stid_export(). */ + exp = s->sc_export; + nfs4_free_cpntf_statelist(clp->net, s); + spin_unlock(&clp->cl_lock); + s->sc_free(s); + if (exp) + exp_put(exp); + if (fp) + put_nfs4_file(fp); +} + +void +nfs4_inc_and_copy_stateid(stateid_t *dst, struct nfs4_stid *stid) +{ + stateid_t *src = &stid->sc_stateid; + + spin_lock(&stid->sc_lock); + if (unlikely(++src->si_generation == 0)) + src->si_generation = 1; + memcpy(dst, src, sizeof(*dst)); + spin_unlock(&stid->sc_lock); +} + +static void put_deleg_file(struct nfs4_file *fp) +{ + struct nfsd_file *rnf = NULL; + struct nfsd_file *nf = NULL; + + spin_lock(&fp->fi_lock); + if (--fp->fi_delegees == 0) { + nf = rcu_dereference_protected(fp->fi_deleg_file, + lockdep_is_held(&fp->fi_lock)); + RCU_INIT_POINTER(fp->fi_deleg_file, NULL); + swap(rnf, fp->fi_rdeleg_file); + } + spin_unlock(&fp->fi_lock); + + if (nf) + nfsd_file_put(nf); + if (rnf) { + nfsd_file_put(rnf); + nfs4_file_put_access(fp, NFS4_SHARE_ACCESS_READ); + } +} + +static void nfsd4_finalize_deleg_timestamps(struct nfs4_delegation *dp, struct file *f) +{ + /* don't do anything if FMODE_NOCMTIME isn't set */ + if ((READ_ONCE(f->f_mode) & FMODE_NOCMTIME) == 0) + return; + + spin_lock(&f->f_lock); + f->f_mode &= ~FMODE_NOCMTIME; + spin_unlock(&f->f_lock); + + /* was it never written? */ + if (!dp->dl_written) + return; + + /* did it get a setattr for the timestamps at some point? */ + if (dp->dl_setattr) + return; + + /* Stamp everything to "now" */ + nfsd_update_cmtime_attr(f, ATTR_ATIME); +} + +static void nfs4_unlock_deleg_lease(struct nfs4_delegation *dp) +{ + struct nfs4_file *fp = dp->dl_stid.sc_file; + struct nfsd_file *nf = rcu_dereference_protected(fp->fi_deleg_file, 1); + + WARN_ON_ONCE(!fp->fi_delegees); + + nfsd4_finalize_deleg_timestamps(dp, nf->nf_file); + kernel_setlease(nf->nf_file, F_UNLCK, NULL, (void **)&dp); + nfsd_fsnotify_recalc_mask(nf); + put_deleg_file(fp); +} + +static void destroy_unhashed_deleg(struct nfs4_delegation *dp) +{ + put_clnt_odstate(dp->dl_clnt_odstate); + nfs4_unlock_deleg_lease(dp); + nfs4_put_stid(&dp->dl_stid); +} + +/** + * nfs4_delegation_exists - Discover if this delegation already exists + * @clp: a pointer to the nfs4_client we're granting a delegation to + * @fp: a pointer to the nfs4_file we're granting a delegation on + * + * Return: + * On success: true iff an existing delegation is found + */ + +static bool +nfs4_delegation_exists(struct nfs4_client *clp, struct nfs4_file *fp) +{ + struct nfs4_delegation *searchdp = NULL; + struct nfs4_client *searchclp = NULL; + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + lockdep_assert_held(&nn->deleg_lock); + lockdep_assert_held(&fp->fi_lock); + + list_for_each_entry(searchdp, &fp->fi_delegations, dl_perfile) { + searchclp = searchdp->dl_stid.sc_client; + if (clp == searchclp) { + return true; + } + } + return false; +} + +/** + * hash_delegation_locked - Add a delegation to the appropriate lists + * @dp: a pointer to the nfs4_delegation we are adding. + * @fp: a pointer to the nfs4_file we're granting a delegation on + * + * Return: + * On success: NULL if the delegation was successfully hashed. + * + * On error: -EAGAIN if one was previously granted to this + * nfs4_client for this nfs4_file. Delegation is not hashed. + * + */ + +static int +hash_delegation_locked(struct nfs4_delegation *dp, struct nfs4_file *fp) +{ + struct nfs4_client *clp = dp->dl_stid.sc_client; + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + lockdep_assert_held(&nn->deleg_lock); + lockdep_assert_held(&fp->fi_lock); + lockdep_assert_held(&clp->cl_lock); + + if (nfs4_delegation_exists(clp, fp)) + return -EAGAIN; + refcount_inc(&dp->dl_stid.sc_count); + dp->dl_stid.sc_type = SC_TYPE_DELEG; + list_add(&dp->dl_perfile, &fp->fi_delegations); + list_add(&dp->dl_perclnt, &clp->cl_delegations); + return 0; +} + +static bool delegation_hashed(struct nfs4_delegation *dp) +{ + return !(list_empty(&dp->dl_perfile)); +} + +static bool +unhash_delegation_locked(struct nfs4_delegation *dp, unsigned short statusmask) +{ + struct nfs4_file *fp = dp->dl_stid.sc_file; + struct nfsd_net *nn = net_generic(dp->dl_stid.sc_client->net, + nfsd_net_id); + + lockdep_assert_held(&nn->deleg_lock); + + if (!delegation_hashed(dp)) + return false; + + if (statusmask == SC_STATUS_REVOKED && + dp->dl_stid.sc_client->cl_minorversion == 0) + statusmask = SC_STATUS_CLOSED; + dp->dl_stid.sc_status |= statusmask; + if (statusmask & SC_STATUS_ADMIN_REVOKED) + atomic_inc(&dp->dl_stid.sc_client->cl_admin_revoked); + + /* Ensure that deleg break won't try to requeue it */ + ++dp->dl_time; + spin_lock(&fp->fi_lock); + list_del_init(&dp->dl_perclnt); + list_del_init(&dp->dl_recall_lru); + list_del_init(&dp->dl_perfile); + spin_unlock(&fp->fi_lock); + return true; +} + +static void destroy_delegation(struct nfs4_delegation *dp) +{ + bool unhashed; + struct nfsd_net *nn = net_generic(dp->dl_stid.sc_client->net, + nfsd_net_id); + + spin_lock(&nn->deleg_lock); + unhashed = unhash_delegation_locked(dp, SC_STATUS_CLOSED); + spin_unlock(&nn->deleg_lock); + if (unhashed) + destroy_unhashed_deleg(dp); +} + +/** + * revoke_delegation - perform nfs4 delegation structure cleanup + * @dp: pointer to the delegation + * + * This function assumes that it's called either from the administrative + * interface (nfsd4_revoke_states()) that's revoking a specific delegation + * stateid or it's called from a laundromat thread (nfsd4_landromat()) that + * determined that this specific state has expired and needs to be revoked + * (both mark state with the appropriate stid sc_status mode). It is also + * assumed that a reference was taken on the @dp state. This function + * consumes that reference. + * + * If this function finds that the @dp state is SC_STATUS_FREED it means + * that a FREE_STATEID operation for this stateid has been processed and + * we can proceed to removing it from recalled list. However, if @dp state + * isn't marked SC_STATUS_FREED, it means we need place it on the cl_revoked + * list and wait for the FREE_STATEID to arrive from the client. At the same + * time, we need to mark it as SC_STATUS_FREEABLE to indicate to the + * nfsd4_free_stateid() function that this stateid has already been added + * to the cl_revoked list and that nfsd4_free_stateid() is now responsible + * for removing it from the list. Inspection of where the delegation state + * in the revocation process is protected by the clp->cl_lock. + */ +static void revoke_delegation(struct nfs4_delegation *dp) +{ + struct nfs4_client *clp = dp->dl_stid.sc_client; + + WARN_ON(!list_empty(&dp->dl_recall_lru)); + WARN_ON_ONCE(dp->dl_stid.sc_client->cl_minorversion > 0 && + !(dp->dl_stid.sc_status & + (SC_STATUS_REVOKED | SC_STATUS_ADMIN_REVOKED))); + + trace_nfsd_stid_revoke(&dp->dl_stid); + + spin_lock(&clp->cl_lock); + if (dp->dl_stid.sc_status & SC_STATUS_FREED) { + list_del_init(&dp->dl_recall_lru); + goto out; + } + list_add(&dp->dl_recall_lru, &clp->cl_revoked); + dp->dl_stid.sc_status |= SC_STATUS_FREEABLE; +out: + spin_unlock(&clp->cl_lock); + destroy_unhashed_deleg(dp); +} + +/* + * SETCLIENTID state + */ + +static unsigned int clientid_hashval(u32 id) +{ + return id & CLIENT_HASH_MASK; +} + +static unsigned int clientstr_hashval(struct xdr_netobj name) +{ + return opaque_hashval(name.data, 8) & CLIENT_HASH_MASK; +} + +/* + * A stateid that had a deny mode associated with it is being released + * or downgraded. Recalculate the deny mode on the file. + */ +static void +recalculate_deny_mode(struct nfs4_file *fp) +{ + struct nfs4_ol_stateid *stp; + u32 old_deny; + + spin_lock(&fp->fi_lock); + old_deny = fp->fi_share_deny; + fp->fi_share_deny = 0; + list_for_each_entry(stp, &fp->fi_stateids, st_perfile) { + fp->fi_share_deny |= bmap_to_share_mode(stp->st_deny_bmap); + if (fp->fi_share_deny == old_deny) + break; + } + spin_unlock(&fp->fi_lock); +} + +static void +reset_union_bmap_deny(u32 deny, struct nfs4_ol_stateid *stp) +{ + int i; + bool change = false; + + for (i = 1; i < 4; i++) { + if ((i & deny) != i) { + change = true; + clear_deny(i, stp); + } + } + + /* Recalculate per-file deny mode if there was a change */ + if (change) + recalculate_deny_mode(stp->st_stid.sc_file); +} + +/* release all access and file references for a given stateid */ +static void +release_all_access(struct nfs4_ol_stateid *stp) +{ + int i; + struct nfs4_file *fp = stp->st_stid.sc_file; + + if (fp && stp->st_deny_bmap != 0) + recalculate_deny_mode(fp); + + for (i = 1; i < 4; i++) { + if (test_access(i, stp)) + nfs4_file_put_access(stp->st_stid.sc_file, i); + clear_access(i, stp); + } +} + +/** + * nfs4_replay_free_cache - release dynamically allocated replay buffer + * @rp: replay cache to reset + * + * If @rp->rp_buf points to a kmalloc'd buffer, free it and reset + * rp_buf to the inline rp_ibuf. Always zeroes rp_buflen. + */ +void nfs4_replay_free_cache(struct nfs4_replay *rp) +{ + if (rp->rp_buf != rp->rp_ibuf) + kfree(rp->rp_buf); + rp->rp_buf = rp->rp_ibuf; + rp->rp_buflen = 0; +} + +static inline void nfs4_free_stateowner(struct nfs4_stateowner *sop) +{ + nfs4_replay_free_cache(&sop->so_replay); + kfree(sop->so_owner.data); + sop->so_ops->so_free(sop); +} + +static void nfs4_put_stateowner(struct nfs4_stateowner *sop) +{ + struct nfs4_client *clp = sop->so_client; + + might_lock(&clp->cl_lock); + + if (!atomic_dec_and_lock(&sop->so_count, &clp->cl_lock)) + return; + sop->so_ops->so_unhash(sop); + spin_unlock(&clp->cl_lock); + nfs4_free_stateowner(sop); +} + +static bool +nfs4_ol_stateid_unhashed(const struct nfs4_ol_stateid *stp) +{ + return list_empty(&stp->st_perfile); +} + +static bool unhash_ol_stateid(struct nfs4_ol_stateid *stp) +{ + struct nfs4_file *fp = stp->st_stid.sc_file; + + lockdep_assert_held(&stp->st_stateowner->so_client->cl_lock); + + if (list_empty(&stp->st_perfile)) + return false; + + spin_lock(&fp->fi_lock); + list_del_init(&stp->st_perfile); + spin_unlock(&fp->fi_lock); + list_del(&stp->st_perstateowner); + return true; +} + +static void nfs4_free_ol_stateid(struct nfs4_stid *stid) +{ + struct nfs4_ol_stateid *stp = openlockstateid(stid); + + put_clnt_odstate(stp->st_clnt_odstate); + release_all_access(stp); + if (stp->st_stateowner) + nfs4_put_stateowner(stp->st_stateowner); + if (!list_empty(&stid->sc_cp_list)) + nfs4_free_cpntf_statelist(stid->sc_client->net, stid); + kmem_cache_free(stateid_slab, stid); +} + +static void nfs4_free_lock_stateid(struct nfs4_stid *stid) +{ + struct nfs4_ol_stateid *stp = openlockstateid(stid); + struct nfs4_lockowner *lo = lockowner(stp->st_stateowner); + struct nfsd_file *nf; + + nf = find_any_file(stp->st_stid.sc_file); + if (nf) { + get_file(nf->nf_file); + filp_close(nf->nf_file, (fl_owner_t)lo); + nfsd_file_put(nf); + } + nfs4_free_ol_stateid(stid); +} + +/* + * Put the persistent reference to an already unhashed generic stateid, while + * holding the cl_lock. If it's the last reference, then put it onto the + * reaplist for later destruction. + */ +static void put_ol_stateid_locked(struct nfs4_ol_stateid *stp, + struct list_head *reaplist) +{ + struct nfs4_stid *s = &stp->st_stid; + struct nfs4_client *clp = s->sc_client; + + lockdep_assert_held(&clp->cl_lock); + + WARN_ON_ONCE(!list_empty(&stp->st_locks)); + + if (!refcount_dec_and_test(&s->sc_count)) { + wake_up_all(&close_wq); + return; + } + + idr_remove(&clp->cl_stateids, s->sc_stateid.si_opaque.so_id); + if (s->sc_status & SC_STATUS_ADMIN_REVOKED) + atomic_dec(&s->sc_client->cl_admin_revoked); + list_add(&stp->st_locks, reaplist); +} + +static bool unhash_lock_stateid(struct nfs4_ol_stateid *stp) +{ + lockdep_assert_held(&stp->st_stid.sc_client->cl_lock); + + if (!unhash_ol_stateid(stp)) + return false; + list_del_init(&stp->st_locks); + stp->st_stid.sc_status |= SC_STATUS_CLOSED; + return true; +} + +static void release_lock_stateid(struct nfs4_ol_stateid *stp) +{ + struct nfs4_client *clp = stp->st_stid.sc_client; + bool unhashed; + + spin_lock(&clp->cl_lock); + unhashed = unhash_lock_stateid(stp); + spin_unlock(&clp->cl_lock); + if (unhashed) + nfs4_put_stid(&stp->st_stid); +} + +static void unhash_lockowner_locked(struct nfs4_lockowner *lo) +{ + struct nfs4_client *clp = lo->lo_owner.so_client; + + lockdep_assert_held(&clp->cl_lock); + + list_del_init(&lo->lo_owner.so_strhash); +} + +/* + * Free a list of generic stateids that were collected earlier after being + * fully unhashed. + */ +static void +free_ol_stateid_reaplist(struct list_head *reaplist) +{ + struct nfs4_ol_stateid *stp; + struct svc_export *exp; + struct nfs4_file *fp; + + might_sleep(); + + while (!list_empty(reaplist)) { + stp = list_first_entry(reaplist, struct nfs4_ol_stateid, + st_locks); + list_del(&stp->st_locks); + fp = stp->st_stid.sc_file; + exp = stp->st_stid.sc_export; + stp->st_stid.sc_free(&stp->st_stid); + if (exp) + exp_put(exp); + if (fp) + put_nfs4_file(fp); + } +} + +static void release_open_stateid_locks(struct nfs4_ol_stateid *open_stp, + struct list_head *reaplist) +{ + struct nfs4_ol_stateid *stp; + + lockdep_assert_held(&open_stp->st_stid.sc_client->cl_lock); + + while (!list_empty(&open_stp->st_locks)) { + stp = list_entry(open_stp->st_locks.next, + struct nfs4_ol_stateid, st_locks); + unhash_lock_stateid(stp); + put_ol_stateid_locked(stp, reaplist); + } +} + +static bool unhash_open_stateid(struct nfs4_ol_stateid *stp, + struct list_head *reaplist) +{ + lockdep_assert_held(&stp->st_stid.sc_client->cl_lock); + + if (!unhash_ol_stateid(stp)) + return false; + release_open_stateid_locks(stp, reaplist); + return true; +} + +static void release_open_stateid(struct nfs4_ol_stateid *stp) +{ + LIST_HEAD(reaplist); + + spin_lock(&stp->st_stid.sc_client->cl_lock); + stp->st_stid.sc_status |= SC_STATUS_CLOSED; + if (unhash_open_stateid(stp, &reaplist)) + put_ol_stateid_locked(stp, &reaplist); + spin_unlock(&stp->st_stid.sc_client->cl_lock); + free_ol_stateid_reaplist(&reaplist); +} + +static bool nfs4_openowner_unhashed(struct nfs4_openowner *oo) +{ + lockdep_assert_held(&oo->oo_owner.so_client->cl_lock); + + return list_empty(&oo->oo_owner.so_strhash) && + list_empty(&oo->oo_perclient); +} + +static void unhash_openowner_locked(struct nfs4_openowner *oo) +{ + struct nfs4_client *clp = oo->oo_owner.so_client; + + lockdep_assert_held(&clp->cl_lock); + + list_del_init(&oo->oo_owner.so_strhash); + list_del_init(&oo->oo_perclient); +} + +static void release_last_closed_stateid(struct nfs4_openowner *oo) +{ + struct nfsd_net *nn = net_generic(oo->oo_owner.so_client->net, + nfsd_net_id); + struct nfs4_ol_stateid *s; + + spin_lock(&nn->client_lock); + s = oo->oo_last_closed_stid; + if (s) { + list_del_init(&oo->oo_close_lru); + oo->oo_last_closed_stid = NULL; + } + spin_unlock(&nn->client_lock); + if (s) + nfs4_put_stid(&s->st_stid); +} + +static void release_openowner(struct nfs4_openowner *oo) +{ + struct nfs4_ol_stateid *stp; + struct nfs4_client *clp = oo->oo_owner.so_client; + LIST_HEAD(reaplist); + + spin_lock(&clp->cl_lock); + unhash_openowner_locked(oo); + while (!list_empty(&oo->oo_owner.so_stateids)) { + stp = list_first_entry(&oo->oo_owner.so_stateids, + struct nfs4_ol_stateid, st_perstateowner); + if (unhash_open_stateid(stp, &reaplist)) + put_ol_stateid_locked(stp, &reaplist); + } + spin_unlock(&clp->cl_lock); + free_ol_stateid_reaplist(&reaplist); + release_last_closed_stateid(oo); + nfs4_put_stateowner(&oo->oo_owner); +} + +static struct nfs4_stid *find_one_sb_stid(struct nfs4_client *clp, + struct super_block *sb, + unsigned int sc_types) +{ + unsigned long id = 0; + struct nfs4_stid *stid; + + spin_lock(&clp->cl_lock); + while ((stid = idr_get_next_ul(&clp->cl_stateids, &id)) != NULL) { + if ((stid->sc_type & sc_types) && + stid->sc_status == 0 && + stid->sc_file->fi_inode->i_sb == sb) { + refcount_inc(&stid->sc_count); + break; + } + id++; + } + spin_unlock(&clp->cl_lock); + return stid; +} + +/* + * Release the export reference an admin-revoked stateid holds, + * so the svc_export (and its vfsmount) is not pinned until the + * client issues FREE_STATEID. sc_export is no longer consulted + * once SC_STATUS_ADMIN_REVOKED is set. + */ +static void drop_stid_export(struct nfs4_client *clp, + struct nfs4_stid *stid) +{ + struct svc_export *exp; + + spin_lock(&clp->cl_lock); + exp = stid->sc_export; + stid->sc_export = NULL; + spin_unlock(&clp->cl_lock); + if (exp) + exp_put(exp); +} + +static void revoke_ol_stid(struct nfs4_client *clp, + struct nfs4_ol_stateid *stp) +{ + struct nfs4_stid *stid = &stp->st_stid; + + lockdep_assert_held(&stp->st_mutex); + spin_lock(&clp->cl_lock); + if (stid->sc_status == 0) { + stid->sc_status |= SC_STATUS_ADMIN_REVOKED; + atomic_inc(&clp->cl_admin_revoked); + spin_unlock(&clp->cl_lock); + if (stid->sc_type == SC_TYPE_LOCK) { + struct nfs4_lockowner *lo = + lockowner(stp->st_stateowner); + struct nfsd_file *nf; + + nf = find_any_file(stp->st_stid.sc_file); + if (nf) { + get_file(nf->nf_file); + filp_close(nf->nf_file, (fl_owner_t)lo); + nfsd_file_put(nf); + } + } + release_all_access(stp); + drop_stid_export(clp, stid); + } else + spin_unlock(&clp->cl_lock); +} + +static void revoke_one_stid(struct nfsd_net *nn, struct nfs4_client *clp, + struct nfs4_stid *stid) +{ + struct nfs4_ol_stateid *stp; + struct nfs4_delegation *dp; + + switch (stid->sc_type) { + case SC_TYPE_OPEN: + stp = openlockstateid(stid); + mutex_lock_nested(&stp->st_mutex, OPEN_STATEID_MUTEX); + revoke_ol_stid(clp, stp); + mutex_unlock(&stp->st_mutex); + break; + case SC_TYPE_LOCK: + stp = openlockstateid(stid); + mutex_lock_nested(&stp->st_mutex, LOCK_STATEID_MUTEX); + revoke_ol_stid(clp, stp); + mutex_unlock(&stp->st_mutex); + break; + case SC_TYPE_DELEG: + /* + * Extra reference guards against concurrent FREE_STATEID. + */ + refcount_inc(&stid->sc_count); + dp = delegstateid(stid); + spin_lock(&nn->deleg_lock); + if (!unhash_delegation_locked(dp, SC_STATUS_ADMIN_REVOKED)) + dp = NULL; + spin_unlock(&nn->deleg_lock); + if (dp) { + revoke_delegation(dp); + drop_stid_export(clp, stid); + } else + nfs4_put_stid(stid); + break; + case SC_TYPE_LAYOUT: + spin_lock(&clp->cl_lock); + if (stid->sc_status == 0) { + stid->sc_status |= SC_STATUS_ADMIN_REVOKED; + atomic_inc(&clp->cl_admin_revoked); + } + spin_unlock(&clp->cl_lock); + nfsd4_close_layout(layoutstateid(stid)); + drop_stid_export(clp, stid); + break; + } +} + +/** + * nfsd4_revoke_states - revoke all nfsv4 states associated with given filesystem + * @nn: used to identify instance of nfsd (there is one per net namespace) + * @sb: super_block used to identify target filesystem + * + * All nfs4 states (open, lock, delegation, layout) held by the server instance + * and associated with a file on the given filesystem will be revoked resulting + * in any files being closed and so all references from nfsd to the filesystem + * being released. Thus nfsd will no longer prevent the filesystem from being + * unmounted. + * + * The clients which own the states will subsequently be notified that the + * states have been "admin-revoked". + * + * Context: Caller must hold nfsd_mutex with NFSD_NET_UP set. Outside + * that window nn->conf_id_hashtbl is unallocated or freed, + * so the walk would dereference a NULL or dangling pointer. + */ +void nfsd4_revoke_states(struct nfsd_net *nn, struct super_block *sb) +{ + unsigned int idhashval; + unsigned int sc_types; + + lockdep_assert_held(&nfsd_mutex); + + sc_types = SC_TYPE_OPEN | SC_TYPE_LOCK | SC_TYPE_DELEG | SC_TYPE_LAYOUT; + + spin_lock(&nn->client_lock); + for (idhashval = 0; idhashval < CLIENT_HASH_SIZE; idhashval++) { + struct list_head *head = &nn->conf_id_hashtbl[idhashval]; + struct nfs4_client *clp; + retry: + list_for_each_entry(clp, head, cl_idhash) { + struct nfs4_stid *stid; + + /* + * force_expire_client() ignores cl_rpc_users once + * its wait_event() has passed, so pinning cannot + * keep an already-expiring client alive; the + * expiry path revokes its states instead. + */ + if (is_client_expired(clp)) + continue; + stid = find_one_sb_stid(clp, sb, sc_types); + if (stid) { + atomic_inc(&clp->cl_rpc_users); + spin_unlock(&nn->client_lock); + revoke_one_stid(nn, clp, stid); + nfs4_put_stid(stid); + spin_lock(&nn->client_lock); + if (clp->cl_minorversion == 0) + /* Allow cleanup after a lease period. + * store_release ensures cleanup will + * see any newly revoked states if it + * sees the time updated. + */ + nn->nfs40_last_revoke = + ktime_get_boottime_seconds(); + put_client_no_renew_locked(clp); + goto retry; + } + } + } + spin_unlock(&nn->client_lock); +} + +static struct nfs4_stid *find_one_export_stid(struct nfs4_client *clp, + const struct path *path, + unsigned int sc_types) +{ + unsigned long id = 0; + struct nfs4_stid *stid; + + spin_lock(&clp->cl_lock); + while ((stid = idr_get_next_ul(&clp->cl_stateids, &id)) != NULL) { + if ((stid->sc_type & sc_types) && + stid->sc_status == 0 && + stid->sc_export && + path_equal(&stid->sc_export->ex_path, path)) { + refcount_inc(&stid->sc_count); + break; + } + id++; + } + spin_unlock(&clp->cl_lock); + return stid; +} + +/** + * nfsd4_revoke_export_states - revoke nfsv4 states acquired through an export + * @nn: used to identify instance of nfsd (there is one per net namespace) + * @path: export path whose states should be revoked + * + * All nfs4 states (open, lock, delegation, layout) acquired through any + * export matching @path are revoked, regardless of which client holds + * them. Matching is by path identity (dentry + vfsmount), so multiple + * svc_export objects for the same path -- one per auth_domain -- are + * handled correctly. + * + * Userspace (exportfs -u) sends this after removing the last client + * for a path, enabling the underlying filesystem to be unmounted. + * + * Context: Caller must hold nfsd_mutex with NFSD_NET_UP set. Outside + * that window nn->conf_id_hashtbl is unallocated or freed, + * so the walk would dereference a NULL or dangling pointer. + */ +void nfsd4_revoke_export_states(struct nfsd_net *nn, const struct path *path) +{ + unsigned int idhashval; + unsigned int sc_types; + + lockdep_assert_held(&nfsd_mutex); + + sc_types = SC_TYPE_OPEN | SC_TYPE_LOCK | SC_TYPE_DELEG | SC_TYPE_LAYOUT; + + spin_lock(&nn->client_lock); + for (idhashval = 0; idhashval < CLIENT_HASH_SIZE; idhashval++) { + struct list_head *head = &nn->conf_id_hashtbl[idhashval]; + struct nfs4_client *clp; + retry: + list_for_each_entry(clp, head, cl_idhash) { + struct nfs4_stid *stid; + + /* Skip or pin clp as in nfsd4_revoke_states(). */ + if (is_client_expired(clp)) + continue; + stid = find_one_export_stid(clp, path, sc_types); + if (stid) { + atomic_inc(&clp->cl_rpc_users); + spin_unlock(&nn->client_lock); + revoke_one_stid(nn, clp, stid); + nfs4_put_stid(stid); + spin_lock(&nn->client_lock); + if (clp->cl_minorversion == 0) + nn->nfs40_last_revoke = + ktime_get_boottime_seconds(); + put_client_no_renew_locked(clp); + goto retry; + } + } + } + spin_unlock(&nn->client_lock); +} + +static inline int +hash_sessionid(struct nfs4_sessionid *sessionid) +{ + struct nfsd4_sessionid *sid = (struct nfsd4_sessionid *)sessionid; + + return sid->sequence % SESSION_HASH_SIZE; +} + +#ifdef CONFIG_SUNRPC_DEBUG +static inline void +dump_sessionid(const char *fn, struct nfs4_sessionid *sessionid) +{ + u32 *ptr = (u32 *)(&sessionid->data[0]); + dprintk("%s: %u:%u:%u:%u\n", fn, ptr[0], ptr[1], ptr[2], ptr[3]); +} +#else +static inline void +dump_sessionid(const char *fn, struct nfs4_sessionid *sessionid) +{ +} +#endif + +/* + * Bump the seqid on cstate->replay_owner, and clear replay_owner if it + * won't be used for replay. + */ +void nfsd4_bump_seqid(struct nfsd4_compound_state *cstate, __be32 nfserr) +{ + struct nfs4_stateowner *so = cstate->replay_owner; + + if (nfserr == nfserr_replay_me) + return; + + if (!seqid_mutating_err(ntohl(nfserr))) { + nfsd4_cstate_clear_replay(cstate); + return; + } + if (!so) + return; + if (so->so_is_open_owner) + release_last_closed_stateid(openowner(so)); + so->so_seqid++; + return; +} + +static void +gen_sessionid(struct nfsd4_session *ses) +{ + struct nfs4_client *clp = ses->se_client; + struct nfsd4_sessionid *sid; + + sid = (struct nfsd4_sessionid *)ses->se_sessionid.data; + sid->clientid = clp->cl_clientid; + sid->sequence = current_sessionid++; + sid->reserved = 0; +} + +/* + * The protocol defines ca_maxresponssize_cached to include the size of + * the rpc header, but all we need to cache is the data starting after + * the end of the initial SEQUENCE operation--the rest we regenerate + * each time. Therefore we can advertise a ca_maxresponssize_cached + * value that is the number of bytes in our cache plus a few additional + * bytes. In order to stay on the safe side, and not promise more than + * we can cache, those additional bytes must be the minimum possible: 24 + * bytes of rpc header (xid through accept state, with AUTH_NULL + * verifier), 12 for the compound header (with zero-length tag), and 44 + * for the SEQUENCE op response: + */ +#define NFSD_MIN_HDR_SEQ_SZ (24 + 12 + 44) + +static struct shrinker *nfsd_slot_shrinker; +static DEFINE_SPINLOCK(nfsd_session_list_lock); +static LIST_HEAD(nfsd_session_list); +/* The sum of "target_slots" on every session, slot 0 included. */ +static atomic_t nfsd_total_target_slots = ATOMIC_INIT(0); +/* Session count, subtracted from the sum to exclude slot 0. */ +static atomic_t nfsd_total_sessions = ATOMIC_INIT(0); + +static void +free_session_slots(struct nfsd4_session *ses, int from) +{ + int i; + + if (from >= ses->se_fchannel.maxreqs) + return; + + for (i = from; i < ses->se_fchannel.maxreqs; i++) { + struct nfsd4_slot *slot = xa_load(&ses->se_slots, i); + + /* + * Save the seqid in case we reactivate this slot. + * This will never require a memory allocation so GFP + * flag is irrelevant + */ + xa_store(&ses->se_slots, i, xa_mk_value(slot->sl_seqid), 0); + free_svc_cred(&slot->sl_cred); + kfree(slot); + } + ses->se_fchannel.maxreqs = from; + if (ses->se_target_maxslots > from) { + int delta = ses->se_target_maxslots - from; + + atomic_sub(delta, &nfsd_total_target_slots); + /* Retain one slot so the session can make forward progress. */ + ses->se_target_maxslots = from ?: 1; + } +} + +/* + * This interface can be used by a shrinker to reduce the target max-slots + * for a session so that some slots can eventually be freed. + * It uses spin_trylock() as it may be called in a context where another + * spinlock is held that has a dependency on client_lock. As shrinkers are + * best-effort, skipping a session with the client_lock already held has no + * great cost. + */ +static int +reduce_session_slots(struct nfsd4_session *ses, int dec) +{ + struct nfsd_net *nn = net_generic(ses->se_client->net, + nfsd_net_id); + int ret = 0; + + if (ses->se_target_maxslots <= 1) + return ret; + if (!spin_trylock(&nn->client_lock)) + return ret; + ret = min(dec, ses->se_target_maxslots-1); + ses->se_target_maxslots -= ret; + atomic_sub(ret, &nfsd_total_target_slots); + ses->se_slot_gen += 1; + if (ses->se_slot_gen == 0) { + int i; + ses->se_slot_gen = 1; + for (i = 0; i < ses->se_fchannel.maxreqs; i++) { + struct nfsd4_slot *slot = xa_load(&ses->se_slots, i); + slot->sl_generation = 0; + } + } + spin_unlock(&nn->client_lock); + return ret; +} + +static struct nfsd4_slot *nfsd4_alloc_slot(struct nfsd4_channel_attrs *fattrs, + int index, gfp_t gfp) +{ + struct nfsd4_slot *slot; + size_t size; + + /* + * The RPC and NFS session headers are never saved in + * the slot reply cache buffer. + */ + size = fattrs->maxresp_cached < NFSD_MIN_HDR_SEQ_SZ ? + 0 : fattrs->maxresp_cached - NFSD_MIN_HDR_SEQ_SZ; + + slot = kzalloc_flex(*slot, sl_data, size, gfp); + if (!slot) + return NULL; + slot->sl_index = index; + return slot; +} + +static struct nfsd4_session *alloc_session(struct nfsd4_channel_attrs *fattrs, + struct nfsd4_channel_attrs *battrs) +{ + int numslots = fattrs->maxreqs; + struct nfsd4_session *new; + struct nfsd4_slot *slot; + int i; + + new = kzalloc_obj(*new); + if (!new) + return NULL; + xa_init(&new->se_slots); + + slot = nfsd4_alloc_slot(fattrs, 0, GFP_KERNEL); + if (!slot || xa_is_err(xa_store(&new->se_slots, 0, slot, GFP_KERNEL))) + goto out_free; + + for (i = 1; i < numslots; i++) { + const gfp_t gfp = GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN; + slot = nfsd4_alloc_slot(fattrs, i, gfp); + if (!slot) + break; + if (xa_is_err(xa_store(&new->se_slots, i, slot, gfp))) { + kfree(slot); + break; + } + } + fattrs->maxreqs = i; + memcpy(&new->se_fchannel, fattrs, sizeof(struct nfsd4_channel_attrs)); + new->se_target_maxslots = i; + atomic_add(i, &nfsd_total_target_slots); + new->se_cb_slot_avail = ~0U; + new->se_cb_highest_slot = min(battrs->maxreqs - 1, + NFSD_BC_SLOT_TABLE_SIZE - 1); + spin_lock_init(&new->se_lock); + return new; +out_free: + kfree(slot); + xa_destroy(&new->se_slots); + kfree(new); + return NULL; +} + +static void free_conn(struct nfsd4_conn *c) +{ + svc_xprt_put(c->cn_xprt); + kfree(c); +} + +static void nfsd4_conn_lost(struct svc_xpt_user *u) +{ + struct nfsd4_conn *c = container_of(u, struct nfsd4_conn, cn_xpt_user); + struct nfs4_client *clp = c->cn_session->se_client; + + trace_nfsd_cb_lost(clp); + + spin_lock(&clp->cl_lock); + if (!list_empty(&c->cn_persession)) { + list_del(&c->cn_persession); + free_conn(c); + } + nfsd4_probe_callback(clp); + spin_unlock(&clp->cl_lock); +} + +static struct nfsd4_conn *alloc_conn(struct svc_rqst *rqstp, u32 flags) +{ + struct nfsd4_conn *conn; + + conn = kmalloc_obj(struct nfsd4_conn); + if (!conn) + return NULL; + svc_xprt_get(rqstp->rq_xprt); + conn->cn_xprt = rqstp->rq_xprt; + conn->cn_flags = flags; + INIT_LIST_HEAD(&conn->cn_xpt_user.list); + return conn; +} + +static void __nfsd4_hash_conn(struct nfsd4_conn *conn, struct nfsd4_session *ses) +{ + conn->cn_session = ses; + list_add(&conn->cn_persession, &ses->se_conns); +} + +static void nfsd4_hash_conn(struct nfsd4_conn *conn, struct nfsd4_session *ses) +{ + struct nfs4_client *clp = ses->se_client; + + spin_lock(&clp->cl_lock); + __nfsd4_hash_conn(conn, ses); + spin_unlock(&clp->cl_lock); +} + +static int nfsd4_register_conn(struct nfsd4_conn *conn) +{ + conn->cn_xpt_user.callback = nfsd4_conn_lost; + return register_xpt_user(conn->cn_xprt, &conn->cn_xpt_user); +} + +static void nfsd4_init_conn(struct svc_rqst *rqstp, struct nfsd4_conn *conn, struct nfsd4_session *ses) +{ + int ret; + + nfsd4_hash_conn(conn, ses); + ret = nfsd4_register_conn(conn); + if (ret) + /* oops; xprt is already down: */ + nfsd4_conn_lost(&conn->cn_xpt_user); + /* We may have gained or lost a callback channel: */ + nfsd4_probe_callback_sync(ses->se_client); +} + +static struct nfsd4_conn *alloc_conn_from_crses(struct svc_rqst *rqstp, struct nfsd4_create_session *cses) +{ + u32 dir = NFS4_CDFC4_FORE; + + if (cses->flags & SESSION4_BACK_CHAN) + dir |= NFS4_CDFC4_BACK; + return alloc_conn(rqstp, dir); +} + +/* must be called under client_lock */ +static void nfsd4_del_conns(struct nfsd4_session *s) +{ + struct nfs4_client *clp = s->se_client; + struct nfsd4_conn *c; + + spin_lock(&clp->cl_lock); + while (!list_empty(&s->se_conns)) { + c = list_first_entry(&s->se_conns, struct nfsd4_conn, cn_persession); + list_del_init(&c->cn_persession); + spin_unlock(&clp->cl_lock); + + unregister_xpt_user(c->cn_xprt, &c->cn_xpt_user); + free_conn(c); + + spin_lock(&clp->cl_lock); + } + spin_unlock(&clp->cl_lock); +} + +static void __free_session(struct nfsd4_session *ses) +{ + free_session_slots(ses, 0); + xa_destroy(&ses->se_slots); + kfree_rcu(ses, rcu_head); +} + +static void free_session(struct nfsd4_session *ses) +{ + nfsd4_del_conns(ses); + __free_session(ses); +} + +/** + * nfsd_slot_shrinker_count - report reclaimable DRC slots + * @s: shrinker descriptor (unused) + * @sc: shrink control (unused) + * + * Return: a positive count of reclaimable slots, or SHRINK_EMPTY when + * there is nothing to reclaim. + */ +static unsigned long +nfsd_slot_shrinker_count(struct shrinker *s, struct shrink_control *sc) +{ + int count; + + /* + * To prevent session deadlock, one slot of each session (slot 0) + * is not reclaimable while the session is active. Thus the number + * of sessions is subtracted from the total number of target slots. + */ + count = atomic_read(&nfsd_total_target_slots) - + atomic_read(&nfsd_total_sessions); + + return count > 0 ? count : SHRINK_EMPTY; +} + +/** + * nfsd_slot_shrinker_scan - reclaim DRC slots under memory pressure + * @s: shrinker descriptor (unused) + * @sc: shrink control; @sc->nr_to_scan bounds the sessions visited, + * @sc->nr_scanned reports how many were visited + * + * Return: the number of session slots NFSD will release. + */ +static unsigned long +nfsd_slot_shrinker_scan(struct shrinker *s, struct shrink_control *sc) +{ + struct nfsd4_session *ses; + unsigned long scanned = 0; + unsigned long freed = 0; + + /* + * Each visited session releases at most one slot. After + * nr_to_scan sessions have been visited, the list head is + * rotated past the last visited session so the next scan + * resumes from there. + */ + spin_lock(&nfsd_session_list_lock); + list_for_each_entry(ses, &nfsd_session_list, se_all_sessions) { + freed += reduce_session_slots(ses, 1); + scanned += 1; + if (scanned >= sc->nr_to_scan) { + /* Move starting point for next scan */ + list_move(&nfsd_session_list, &ses->se_all_sessions); + break; + } + } + spin_unlock(&nfsd_session_list_lock); + sc->nr_scanned = scanned; + return freed; +} + +static void init_session(struct svc_rqst *rqstp, struct nfsd4_session *new, struct nfs4_client *clp, struct nfsd4_create_session *cses) +{ + int idx; + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + + new->se_client = clp; + gen_sessionid(new); + + INIT_LIST_HEAD(&new->se_conns); + + atomic_set(&new->se_ref, 0); + new->se_dead = false; + new->se_cb_prog = cses->callback_prog; + new->se_cb_sec = cses->cb_sec; + + for (idx = 0; idx < NFSD_BC_SLOT_TABLE_SIZE; ++idx) + new->se_cb_seq_nr[idx] = 1; + + idx = hash_sessionid(&new->se_sessionid); + list_add(&new->se_hash, &nn->sessionid_hashtbl[idx]); + spin_lock(&clp->cl_lock); + list_add(&new->se_perclnt, &clp->cl_sessions); + spin_unlock(&clp->cl_lock); + + spin_lock(&nfsd_session_list_lock); + list_add_tail(&new->se_all_sessions, &nfsd_session_list); + atomic_inc(&nfsd_total_sessions); + spin_unlock(&nfsd_session_list_lock); + + { + struct sockaddr *sa = svc_addr(rqstp); + /* + * This is a little silly; with sessions there's no real + * use for the callback address. Use the peer address + * as a reasonable default for now, but consider fixing + * the rpc client not to require an address in the + * future: + */ + rpc_copy_addr((struct sockaddr *)&clp->cl_cb_conn.cb_addr, sa); + clp->cl_cb_conn.cb_addrlen = svc_addr_len(sa); + } +} + +/* caller must hold client_lock */ +static struct nfsd4_session * +__find_in_sessionid_hashtbl(struct nfs4_sessionid *sessionid, struct net *net) +{ + struct nfsd4_session *elem; + int idx; + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + lockdep_assert_held(&nn->client_lock); + + dump_sessionid(__func__, sessionid); + idx = hash_sessionid(sessionid); + /* Search in the appropriate list */ + list_for_each_entry(elem, &nn->sessionid_hashtbl[idx], se_hash) { + if (!memcmp(elem->se_sessionid.data, sessionid->data, + NFS4_MAX_SESSIONID_LEN)) { + return elem; + } + } + + dprintk("%s: session not found\n", __func__); + return NULL; +} + +static struct nfsd4_session * +find_in_sessionid_hashtbl(struct nfs4_sessionid *sessionid, struct net *net, + __be32 *ret) +{ + struct nfsd4_session *session; + __be32 status = nfserr_badsession; + + session = __find_in_sessionid_hashtbl(sessionid, net); + if (!session) + goto out; + status = nfsd4_get_session_locked(session); + if (status) + session = NULL; +out: + *ret = status; + return session; +} + +/* caller must hold client_lock */ +static void +unhash_session(struct nfsd4_session *ses) +{ + struct nfs4_client *clp = ses->se_client; + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + lockdep_assert_held(&nn->client_lock); + + list_del(&ses->se_hash); + spin_lock(&ses->se_client->cl_lock); + list_del(&ses->se_perclnt); + spin_unlock(&ses->se_client->cl_lock); + spin_lock(&nfsd_session_list_lock); + list_del(&ses->se_all_sessions); + atomic_dec(&nfsd_total_sessions); + spin_unlock(&nfsd_session_list_lock); +} + +/* SETCLIENTID and SETCLIENTID_CONFIRM Helper functions */ +static int +STALE_CLIENTID(clientid_t *clid, struct nfsd_net *nn) +{ + /* + * We're assuming the clid was not given out from a boot + * precisely 2^32 (about 136 years) before this one. That seems + * a safe assumption: + */ + if (clid->cl_boot == (u32)nn->boot_time) + return 0; + trace_nfsd_clid_stale(clid); + return 1; +} + +static struct nfs4_client *alloc_client(struct xdr_netobj name, + struct nfsd_net *nn) +{ + struct nfs4_client *clp; + int i; + + if (atomic_read(&nn->nfs4_client_count) >= nn->nfs4_max_clients && + atomic_read(&nn->nfsd_courtesy_clients) > 0) + mod_delayed_work(laundry_wq, &nn->laundromat_work, 0); + + clp = kmem_cache_zalloc(client_slab, GFP_KERNEL); + if (clp == NULL) + return NULL; + xdr_netobj_dup(&clp->cl_name, &name, GFP_KERNEL); + if (clp->cl_name.data == NULL) + goto err_no_name; + clp->cl_ownerstr_hashtbl = kmalloc_objs(struct list_head, + OWNER_HASH_SIZE); + if (!clp->cl_ownerstr_hashtbl) + goto err_no_hashtbl; + clp->cl_callback_wq = alloc_ordered_workqueue("nfsd4_callbacks", 0); + if (!clp->cl_callback_wq) + goto err_no_callback_wq; + + for (i = 0; i < OWNER_HASH_SIZE; i++) + INIT_LIST_HEAD(&clp->cl_ownerstr_hashtbl[i]); + INIT_LIST_HEAD(&clp->cl_sessions); + idr_init(&clp->cl_stateids); + atomic_set(&clp->cl_rpc_users, 0); + clp->cl_cb_state = NFSD4_CB_UNKNOWN; + clp->cl_state = NFSD4_ACTIVE; + atomic_inc(&nn->nfs4_client_count); + atomic_set(&clp->cl_delegs_in_recall, 0); + INIT_LIST_HEAD(&clp->cl_idhash); + INIT_LIST_HEAD(&clp->cl_openowners); + INIT_LIST_HEAD(&clp->cl_delegations); + INIT_LIST_HEAD(&clp->cl_lru); + INIT_LIST_HEAD(&clp->cl_revoked); +#ifdef CONFIG_NFSD_PNFS + INIT_LIST_HEAD(&clp->cl_lo_states); +#endif +#ifdef CONFIG_NFSD_SCSILAYOUT + xa_init(&clp->cl_dev_fences); + mutex_init(&clp->cl_fence_mutex); +#endif + INIT_LIST_HEAD(&clp->async_copies); + spin_lock_init(&clp->async_lock); + spin_lock_init(&clp->cl_lock); + rpc_init_wait_queue(&clp->cl_cb_waitq, "Backchannel slot table"); + return clp; +err_no_callback_wq: + kfree(clp->cl_ownerstr_hashtbl); +err_no_hashtbl: + kfree(clp->cl_name.data); +err_no_name: + kmem_cache_free(client_slab, clp); + return NULL; +} + +static void __free_client(struct kref *k) +{ + struct nfsdfs_client *c = container_of(k, struct nfsdfs_client, cl_ref); + struct nfs4_client *clp = container_of(c, struct nfs4_client, cl_nfsdfs); + + free_svc_cred(&clp->cl_cred); + destroy_workqueue(clp->cl_callback_wq); + kfree(clp->cl_ownerstr_hashtbl); + kfree(clp->cl_name.data); + kfree(clp->cl_nii_domain.data); + kfree(clp->cl_nii_name.data); + idr_destroy(&clp->cl_stateids); + kfree(clp->cl_ra); + kmem_cache_free(client_slab, clp); +} + +/** + * nfsd4_put_client - release a reference on an nfs4_client + * @clp: the client to be released + * + * When the last reference is released, the client is freed. + */ +void nfsd4_put_client(struct nfs4_client *clp) +{ + kref_put(&clp->cl_nfsdfs.cl_ref, __free_client); +} + +static void +free_client(struct nfs4_client *clp) +{ + while (!list_empty(&clp->cl_sessions)) { + struct nfsd4_session *ses; + ses = list_entry(clp->cl_sessions.next, struct nfsd4_session, + se_perclnt); + list_del(&ses->se_perclnt); + WARN_ON_ONCE(atomic_read(&ses->se_ref)); + free_session(ses); + } + rpc_destroy_wait_queue(&clp->cl_cb_waitq); + if (clp->cl_nfsd_dentry) { + nfsd_client_rmdir(clp->cl_nfsd_dentry); + clp->cl_nfsd_dentry = NULL; + wake_up_all(&expiry_wq); + } + nfsd4_put_client(clp); +} + +/* must be called under the client_lock */ +static void +unhash_client_locked(struct nfs4_client *clp) +{ + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + struct nfsd4_session *ses; + + lockdep_assert_held(&nn->client_lock); + + /* Mark the client as expired! */ + clp->cl_time = 0; + /* Make it invisible */ + if (!list_empty(&clp->cl_idhash)) { + list_del_init(&clp->cl_idhash); + if (test_bit(NFSD4_CLIENT_CONFIRMED, &clp->cl_flags)) + rb_erase(&clp->cl_namenode, &nn->conf_name_tree); + else + rb_erase(&clp->cl_namenode, &nn->unconf_name_tree); + } + list_del_init(&clp->cl_lru); + spin_lock(&clp->cl_lock); + spin_lock(&nfsd_session_list_lock); + list_for_each_entry(ses, &clp->cl_sessions, se_perclnt) { + list_del_init(&ses->se_hash); + /* + * unhash_client_locked() can run more than once for a + * client; the session stays on cl_sessions across calls. + * The first pass empties se_all_sessions via + * list_del_init(), so skip the decrement on later passes + * to keep nfsd_total_sessions from being double-counted. + */ + if (!list_empty(&ses->se_all_sessions)) { + list_del_init(&ses->se_all_sessions); + atomic_dec(&nfsd_total_sessions); + } + } + spin_unlock(&nfsd_session_list_lock); + spin_unlock(&clp->cl_lock); +} + +static void +unhash_client(struct nfs4_client *clp) +{ + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + spin_lock(&nn->client_lock); + unhash_client_locked(clp); + spin_unlock(&nn->client_lock); +} + +static __be32 mark_client_expired_locked(struct nfs4_client *clp) +{ + int users = atomic_read(&clp->cl_rpc_users); + + trace_nfsd_mark_client_expired(clp, users); + + if (users) + return nfserr_jukebox; + unhash_client_locked(clp); + return nfs_ok; +} + +static void +__destroy_client(struct nfs4_client *clp) +{ + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + int i; + struct nfs4_openowner *oo; + struct nfs4_delegation *dp; + LIST_HEAD(reaplist); + + spin_lock(&nn->deleg_lock); + while (!list_empty(&clp->cl_delegations)) { + dp = list_entry(clp->cl_delegations.next, struct nfs4_delegation, dl_perclnt); + unhash_delegation_locked(dp, SC_STATUS_CLOSED); + list_add(&dp->dl_recall_lru, &reaplist); + } + spin_unlock(&nn->deleg_lock); + while (!list_empty(&reaplist)) { + dp = list_entry(reaplist.next, struct nfs4_delegation, dl_recall_lru); + list_del_init(&dp->dl_recall_lru); + destroy_unhashed_deleg(dp); + } + while (!list_empty(&clp->cl_revoked)) { + dp = list_entry(clp->cl_revoked.next, struct nfs4_delegation, dl_recall_lru); + list_del_init(&dp->dl_recall_lru); + nfs4_put_stid(&dp->dl_stid); + } + while (!list_empty(&clp->cl_openowners)) { + oo = list_entry(clp->cl_openowners.next, struct nfs4_openowner, oo_perclient); + nfs4_get_stateowner(&oo->oo_owner); + release_openowner(oo); + } + for (i = 0; i < OWNER_HASH_SIZE; i++) { + struct nfs4_stateowner *so; + + spin_lock(&clp->cl_lock); + while (!list_empty(&clp->cl_ownerstr_hashtbl[i])) { + so = list_first_entry(&clp->cl_ownerstr_hashtbl[i], + struct nfs4_stateowner, so_strhash); + /* Should be no openowners at this point */ + WARN_ON_ONCE(so->so_is_open_owner); + nfs4_get_stateowner(so); + unhash_lockowner_locked(lockowner(so)); + spin_unlock(&clp->cl_lock); + + remove_blocked_locks(lockowner(so)); + nfs4_put_stateowner(so); + + spin_lock(&clp->cl_lock); + } + spin_unlock(&clp->cl_lock); + } + nfsd4_return_all_client_layouts(clp); + nfsd4_shutdown_copy(clp); + nfsd4_shutdown_callback(clp); + if (clp->cl_cb_conn.cb_xprt) + svc_xprt_put(clp->cl_cb_conn.cb_xprt); + atomic_add_unless(&nn->nfs4_client_count, -1, 0); + nfsd4_dec_courtesy_client_count(nn, clp); +#ifdef CONFIG_NFSD_SCSILAYOUT + xa_destroy(&clp->cl_dev_fences); +#endif + free_client(clp); + wake_up_all(&expiry_wq); +} + +static void +destroy_client(struct nfs4_client *clp) +{ + unhash_client(clp); + __destroy_client(clp); +} + +static void inc_reclaim_complete(struct nfs4_client *clp) +{ + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + if (!test_bit(NFSD_NET_TRACK_RECLAIM_COMPLETES, &nn->flags)) + return; + + down_read(&nn->reclaim_str_hashtbl_lock); + if (!nfsd4_find_reclaim_client(clp->cl_name, nn)) { + up_read(&nn->reclaim_str_hashtbl_lock); + return; + } + if (atomic_inc_return(&nn->nr_reclaim_complete) == + nn->reclaim_str_hashtbl_size) { + up_read(&nn->reclaim_str_hashtbl_lock); + printk(KERN_INFO "NFSD: all clients done reclaiming, ending NFSv4 grace period (net %x)\n", + clp->net->ns.inum); + nfsd4_end_grace(nn); + return; + } + up_read(&nn->reclaim_str_hashtbl_lock); +} + +static void expire_client(struct nfs4_client *clp) +{ + unhash_client(clp); + nfsd4_client_record_remove(clp); + __destroy_client(clp); +} + +static void copy_verf(struct nfs4_client *target, nfs4_verifier *source) +{ + memcpy(target->cl_verifier.data, source->data, + sizeof(target->cl_verifier.data)); +} + +static void copy_clid(struct nfs4_client *target, struct nfs4_client *source) +{ + target->cl_clientid.cl_boot = source->cl_clientid.cl_boot; + target->cl_clientid.cl_id = source->cl_clientid.cl_id; +} + +static int copy_cred(struct svc_cred *target, struct svc_cred *source) +{ + target->cr_principal = kstrdup(source->cr_principal, GFP_KERNEL); + target->cr_raw_principal = kstrdup(source->cr_raw_principal, + GFP_KERNEL); + target->cr_targ_princ = kstrdup(source->cr_targ_princ, GFP_KERNEL); + if ((source->cr_principal && !target->cr_principal) || + (source->cr_raw_principal && !target->cr_raw_principal) || + (source->cr_targ_princ && !target->cr_targ_princ)) + return -ENOMEM; + + target->cr_flavor = source->cr_flavor; + target->cr_uid = source->cr_uid; + target->cr_gid = source->cr_gid; + target->cr_group_info = source->cr_group_info; + get_group_info(target->cr_group_info); + target->cr_gss_mech = source->cr_gss_mech; + if (source->cr_gss_mech) + gss_mech_get(source->cr_gss_mech); + return 0; +} + +static int +compare_blob(const struct xdr_netobj *o1, const struct xdr_netobj *o2) +{ + if (o1->len < o2->len) + return -1; + if (o1->len > o2->len) + return 1; + return memcmp(o1->data, o2->data, o1->len); +} + +static int +same_verf(nfs4_verifier *v1, nfs4_verifier *v2) +{ + return 0 == memcmp(v1->data, v2->data, sizeof(v1->data)); +} + +static int +same_clid(clientid_t *cl1, clientid_t *cl2) +{ + return (cl1->cl_boot == cl2->cl_boot) && (cl1->cl_id == cl2->cl_id); +} + +static bool groups_equal(struct group_info *g1, struct group_info *g2) +{ + int i; + + if (g1->ngroups != g2->ngroups) + return false; + for (i=0; ingroups; i++) + if (!gid_eq(g1->gid[i], g2->gid[i])) + return false; + return true; +} + +/* + * RFC 3530 language requires clid_inuse be returned when the + * "principal" associated with a requests differs from that previously + * used. We use uid, gid's, and gss principal string as our best + * approximation. We also don't want to allow non-gss use of a client + * established using gss: in theory cr_principal should catch that + * change, but in practice cr_principal can be null even in the gss case + * since gssd doesn't always pass down a principal string. + */ +static bool is_gss_cred(struct svc_cred *cr) +{ + /* Is cr_flavor one of the gss "pseudoflavors"?: */ + return (cr->cr_flavor > RPC_AUTH_MAXFLAVOR); +} + + +static bool +same_creds(struct svc_cred *cr1, struct svc_cred *cr2) +{ + if ((is_gss_cred(cr1) != is_gss_cred(cr2)) + || (!uid_eq(cr1->cr_uid, cr2->cr_uid)) + || (!gid_eq(cr1->cr_gid, cr2->cr_gid)) + || !groups_equal(cr1->cr_group_info, cr2->cr_group_info)) + return false; + /* XXX: check that cr_targ_princ fields match ? */ + if (cr1->cr_principal == cr2->cr_principal) + return true; + if (!cr1->cr_principal || !cr2->cr_principal) + return false; + return 0 == strcmp(cr1->cr_principal, cr2->cr_principal); +} + +static bool svc_rqst_integrity_protected(struct svc_rqst *rqstp) +{ + struct svc_cred *cr = &rqstp->rq_cred; + u32 service; + + if (!cr->cr_gss_mech) + return false; + service = gss_pseudoflavor_to_service(cr->cr_gss_mech, cr->cr_flavor); + return service == RPC_GSS_SVC_INTEGRITY || + service == RPC_GSS_SVC_PRIVACY; +} + +bool nfsd4_mach_creds_match(struct nfs4_client *cl, struct svc_rqst *rqstp) +{ + struct svc_cred *cr = &rqstp->rq_cred; + + if (!cl->cl_mach_cred) + return true; + if (cl->cl_cred.cr_gss_mech != cr->cr_gss_mech) + return false; + if (!svc_rqst_integrity_protected(rqstp)) + return false; + if (cl->cl_cred.cr_raw_principal) + return 0 == strcmp(cl->cl_cred.cr_raw_principal, + cr->cr_raw_principal); + if (!cr->cr_principal) + return false; + return 0 == strcmp(cl->cl_cred.cr_principal, cr->cr_principal); +} + +static void gen_confirm(struct nfs4_client *clp, struct nfsd_net *nn) +{ + __be32 verf[2]; + + /* + * This is opaque to client, so no need to byte-swap. Use + * __force to keep sparse happy + */ + verf[0] = (__force __be32)(u32)ktime_get_real_seconds(); + verf[1] = (__force __be32)nn->clverifier_counter++; + memcpy(clp->cl_confirm.data, verf, sizeof(clp->cl_confirm.data)); +} + +static void gen_clid(struct nfs4_client *clp, struct nfsd_net *nn) +{ + clp->cl_clientid.cl_boot = (u32)nn->boot_time; + clp->cl_clientid.cl_id = nn->clientid_counter++; + gen_confirm(clp, nn); +} + +static struct nfs4_stid * +find_stateid_locked(struct nfs4_client *cl, stateid_t *t) +{ + struct nfs4_stid *ret; + + ret = idr_find(&cl->cl_stateids, t->si_opaque.so_id); + if (!ret || !ret->sc_type) + return NULL; + /* + * Copy offload stateids live in cl_stateids only for id allocation and + * refcounting; per RFC 7862 they are not valid targets for generic + * stateid ops (FREE_STATEID, TEST_STATEID, I/O). Hide them so those + * paths return NFS4ERR_BAD_STATEID. + */ + if (ret->sc_type == SC_TYPE_COPY) + return NULL; + return ret; +} + +static struct nfs4_stid * +find_stateid_by_type(struct nfs4_client *cl, stateid_t *t, + unsigned short typemask, unsigned short ok_states) +{ + struct nfs4_stid *s; + + spin_lock(&cl->cl_lock); + s = find_stateid_locked(cl, t); + if (s != NULL) { + if ((s->sc_status & ~ok_states) == 0 && + (typemask & s->sc_type)) + refcount_inc(&s->sc_count); + else + s = NULL; + } + spin_unlock(&cl->cl_lock); + return s; +} + +static struct nfs4_client *get_nfsdfs_clp(struct inode *inode) +{ + struct nfsdfs_client *nc; + nc = get_nfsdfs_client(inode); + if (!nc) + return NULL; + return container_of(nc, struct nfs4_client, cl_nfsdfs); +} + +static void seq_quote_mem(struct seq_file *m, char *data, int len) +{ + seq_puts(m, "\""); + seq_escape_mem(m, data, len, ESCAPE_HEX | ESCAPE_NAP | ESCAPE_APPEND, "\"\\"); + seq_puts(m, "\""); +} + +static const char *cb_state2str(int state) +{ + switch (state) { + case NFSD4_CB_UP: + return "UP"; + case NFSD4_CB_UNKNOWN: + return "UNKNOWN"; + case NFSD4_CB_DOWN: + return "DOWN"; + case NFSD4_CB_FAULT: + return "FAULT"; + } + return "UNDEFINED"; +} + +static int client_info_show(struct seq_file *m, void *v) +{ + struct inode *inode = file_inode(m->file); + struct nfsd4_session *ses; + struct nfs4_client *clp; + u64 clid; + + clp = get_nfsdfs_clp(inode); + if (!clp) + return -ENXIO; + memcpy(&clid, &clp->cl_clientid, sizeof(clid)); + seq_printf(m, "clientid: 0x%llx\n", clid); + seq_printf(m, "address: \"%pISpc\"\n", (struct sockaddr *)&clp->cl_addr); + + if (clp->cl_state == NFSD4_COURTESY) + seq_puts(m, "status: courtesy\n"); + else if (clp->cl_state == NFSD4_EXPIRABLE) + seq_puts(m, "status: expirable\n"); + else if (test_bit(NFSD4_CLIENT_CONFIRMED, &clp->cl_flags)) + seq_puts(m, "status: confirmed\n"); + else + seq_puts(m, "status: unconfirmed\n"); + seq_printf(m, "seconds from last renew: %lld\n", + ktime_get_boottime_seconds() - clp->cl_time); + seq_puts(m, "name: "); + seq_quote_mem(m, clp->cl_name.data, clp->cl_name.len); + seq_printf(m, "\nminor version: %d\n", clp->cl_minorversion); + if (clp->cl_nii_domain.data) { + seq_puts(m, "Implementation domain: "); + seq_quote_mem(m, clp->cl_nii_domain.data, + clp->cl_nii_domain.len); + seq_puts(m, "\nImplementation name: "); + seq_quote_mem(m, clp->cl_nii_name.data, clp->cl_nii_name.len); + seq_printf(m, "\nImplementation time: [%lld, %ld]\n", + clp->cl_nii_time.tv_sec, clp->cl_nii_time.tv_nsec); + } + seq_printf(m, "callback state: %s\n", cb_state2str(clp->cl_cb_state)); + seq_printf(m, "callback address: \"%pISpc\"\n", &clp->cl_cb_conn.cb_addr); + seq_printf(m, "admin-revoked states: %d\n", + atomic_read(&clp->cl_admin_revoked)); + spin_lock(&clp->cl_lock); + seq_printf(m, "session slots:"); + list_for_each_entry(ses, &clp->cl_sessions, se_perclnt) + seq_printf(m, " %u", ses->se_fchannel.maxreqs); + seq_printf(m, "\nsession target slots:"); + list_for_each_entry(ses, &clp->cl_sessions, se_perclnt) + seq_printf(m, " %u", ses->se_target_maxslots); + spin_unlock(&clp->cl_lock); + seq_puts(m, "\n"); + + nfsd4_put_client(clp); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(client_info); + +static void *states_start(struct seq_file *s, loff_t *pos) + __acquires(&clp->cl_lock) +{ + struct nfs4_client *clp = s->private; + unsigned long id = *pos; + void *ret; + + spin_lock(&clp->cl_lock); + ret = idr_get_next_ul(&clp->cl_stateids, &id); + *pos = id; + return ret; +} + +static void *states_next(struct seq_file *s, void *v, loff_t *pos) +{ + struct nfs4_client *clp = s->private; + unsigned long id = *pos; + void *ret; + + id = *pos; + id++; + ret = idr_get_next_ul(&clp->cl_stateids, &id); + *pos = id; + return ret; +} + +static void states_stop(struct seq_file *s, void *v) + __releases(&clp->cl_lock) +{ + struct nfs4_client *clp = s->private; + + spin_unlock(&clp->cl_lock); +} + +static void nfs4_show_fname(struct seq_file *s, struct nfsd_file *f) +{ + seq_printf(s, "filename: \"%pD2\"", f->nf_file); +} + +static void nfs4_show_superblock(struct seq_file *s, struct nfsd_file *f) +{ + struct inode *inode = file_inode(f->nf_file); + + seq_printf(s, "superblock: \"%02x:%02x:%llu\"", + MAJOR(inode->i_sb->s_dev), + MINOR(inode->i_sb->s_dev), + inode->i_ino); +} + +static void nfs4_show_owner(struct seq_file *s, struct nfs4_stateowner *oo) +{ + seq_puts(s, "owner: "); + seq_quote_mem(s, oo->so_owner.data, oo->so_owner.len); +} + +static void nfs4_show_stateid(struct seq_file *s, stateid_t *stid) +{ + seq_printf(s, "0x%.8x", stid->si_generation); + seq_printf(s, "%12phN", &stid->si_opaque); +} + +static int nfs4_show_open(struct seq_file *s, struct nfs4_stid *st) +{ + struct nfs4_ol_stateid *ols; + struct nfs4_file *nf; + struct nfsd_file *file; + struct nfs4_stateowner *oo; + unsigned int access, deny; + + ols = openlockstateid(st); + oo = ols->st_stateowner; + nf = st->sc_file; + + seq_puts(s, "- "); + nfs4_show_stateid(s, &st->sc_stateid); + seq_puts(s, ": { type: open, "); + + access = bmap_to_share_mode(ols->st_access_bmap); + deny = bmap_to_share_mode(ols->st_deny_bmap); + + seq_printf(s, "access: %s%s, ", + access & NFS4_SHARE_ACCESS_READ ? "r" : "-", + access & NFS4_SHARE_ACCESS_WRITE ? "w" : "-"); + seq_printf(s, "deny: %s%s, ", + deny & NFS4_SHARE_ACCESS_READ ? "r" : "-", + deny & NFS4_SHARE_ACCESS_WRITE ? "w" : "-"); + + if (nf) { + spin_lock(&nf->fi_lock); + file = find_any_file_locked(nf); + if (file) { + nfs4_show_superblock(s, file); + seq_puts(s, ", "); + nfs4_show_fname(s, file); + seq_puts(s, ", "); + } + spin_unlock(&nf->fi_lock); + } else + seq_puts(s, "closed, "); + nfs4_show_owner(s, oo); + if (st->sc_status & SC_STATUS_ADMIN_REVOKED) + seq_puts(s, ", admin-revoked"); + seq_puts(s, " }\n"); + return 0; +} + +static int nfs4_show_lock(struct seq_file *s, struct nfs4_stid *st) +{ + struct nfs4_ol_stateid *ols; + struct nfs4_file *nf; + struct nfsd_file *file; + struct nfs4_stateowner *oo; + + ols = openlockstateid(st); + oo = ols->st_stateowner; + nf = st->sc_file; + + seq_puts(s, "- "); + nfs4_show_stateid(s, &st->sc_stateid); + seq_puts(s, ": { type: lock, "); + + spin_lock(&nf->fi_lock); + file = find_any_file_locked(nf); + if (file) { + /* + * Note: a lock stateid isn't really the same thing as a lock, + * it's the locking state held by one owner on a file, and there + * may be multiple (or no) lock ranges associated with it. + * (Same for the matter is true of open stateids.) + */ + + nfs4_show_superblock(s, file); + /* XXX: open stateid? */ + seq_puts(s, ", "); + nfs4_show_fname(s, file); + seq_puts(s, ", "); + } + nfs4_show_owner(s, oo); + if (st->sc_status & SC_STATUS_ADMIN_REVOKED) + seq_puts(s, ", admin-revoked"); + seq_puts(s, " }\n"); + spin_unlock(&nf->fi_lock); + return 0; +} + +static char *nfs4_show_deleg_type(u32 dl_type) +{ + switch (dl_type) { + case OPEN_DELEGATE_READ: + return "r"; + case OPEN_DELEGATE_WRITE: + return "w"; + case OPEN_DELEGATE_READ_ATTRS_DELEG: + return "ra"; + case OPEN_DELEGATE_WRITE_ATTRS_DELEG: + return "wa"; + } + return "?"; +} + +static int nfs4_show_deleg(struct seq_file *s, struct nfs4_stid *st) +{ + struct nfs4_delegation *ds; + struct nfs4_file *nf; + struct nfsd_file *file; + + ds = delegstateid(st); + nf = st->sc_file; + + seq_puts(s, "- "); + nfs4_show_stateid(s, &st->sc_stateid); + seq_puts(s, ": { type: deleg, "); + + seq_printf(s, "access: %s", nfs4_show_deleg_type(ds->dl_type)); + + /* XXX: lease time, whether it's being recalled. */ + + spin_lock(&nf->fi_lock); + file = rcu_dereference_protected(nf->fi_deleg_file, + lockdep_is_held(&nf->fi_lock)); + if (file) { + seq_puts(s, ", "); + nfs4_show_superblock(s, file); + seq_puts(s, ", "); + nfs4_show_fname(s, file); + } + spin_unlock(&nf->fi_lock); + if (st->sc_status & SC_STATUS_ADMIN_REVOKED) + seq_puts(s, ", admin-revoked"); + seq_puts(s, " }\n"); + return 0; +} + +static int nfs4_show_layout(struct seq_file *s, struct nfs4_stid *st) +{ + struct nfs4_layout_stateid *ls; + struct nfsd_file *file; + + ls = container_of(st, struct nfs4_layout_stateid, ls_stid); + + seq_puts(s, "- "); + nfs4_show_stateid(s, &st->sc_stateid); + seq_puts(s, ": { type: layout"); + + /* XXX: What else would be useful? */ + + spin_lock(&ls->ls_stid.sc_file->fi_lock); + file = ls->ls_file; + if (file) { + seq_puts(s, ", "); + nfs4_show_superblock(s, file); + seq_puts(s, ", "); + nfs4_show_fname(s, file); + } + spin_unlock(&ls->ls_stid.sc_file->fi_lock); + if (st->sc_status & SC_STATUS_ADMIN_REVOKED) + seq_puts(s, ", admin-revoked"); + seq_puts(s, " }\n"); + + return 0; +} + +static int states_show(struct seq_file *s, void *v) +{ + struct nfs4_stid *st = v; + + switch (st->sc_type) { + case SC_TYPE_OPEN: + return nfs4_show_open(s, st); + case SC_TYPE_LOCK: + return nfs4_show_lock(s, st); + case SC_TYPE_DELEG: + return nfs4_show_deleg(s, st); + case SC_TYPE_LAYOUT: + return nfs4_show_layout(s, st); + default: + return 0; /* XXX: or SEQ_SKIP? */ + } + /* XXX: copy stateids? */ +} + +static struct seq_operations states_seq_ops = { + .start = states_start, + .next = states_next, + .stop = states_stop, + .show = states_show +}; + +static int client_states_open(struct inode *inode, struct file *file) +{ + struct seq_file *s; + struct nfs4_client *clp; + int ret; + + clp = get_nfsdfs_clp(inode); + if (!clp) + return -ENXIO; + + ret = seq_open(file, &states_seq_ops); + if (ret) { + nfsd4_put_client(clp); + return ret; + } + s = file->private_data; + s->private = clp; + return 0; +} + +static int client_opens_release(struct inode *inode, struct file *file) +{ + struct seq_file *m = file->private_data; + struct nfs4_client *clp = m->private; + + /* XXX: alternatively, we could get/drop in seq start/stop */ + nfsd4_put_client(clp); + return seq_release(inode, file); +} + +static const struct file_operations client_states_fops = { + .open = client_states_open, + .read = seq_read, + .llseek = seq_lseek, + .release = client_opens_release, +}; + +/* + * Normally we refuse to destroy clients that are in use, but here the + * administrator is telling us to just do it. We also want to wait + * so the caller has a guarantee that the client's locks are gone by + * the time the write returns: + */ +static void force_expire_client(struct nfs4_client *clp) +{ + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + bool already_expired; + + trace_nfsd_clid_admin_expired(&clp->cl_clientid); + + /* + * cl_time is cleared under client_lock before the wait so a + * revocation walk pinning cl_rpc_users under it either skips + * this client or is seen by this wait_event(). + */ + spin_lock(&nn->client_lock); + clp->cl_time = 0; + spin_unlock(&nn->client_lock); + + wait_event(expiry_wq, atomic_read(&clp->cl_rpc_users) == 0); + spin_lock(&nn->client_lock); + already_expired = list_empty(&clp->cl_lru); + if (!already_expired) + unhash_client_locked(clp); + spin_unlock(&nn->client_lock); + + if (!already_expired) + expire_client(clp); + else + wait_event(expiry_wq, clp->cl_nfsd_dentry == NULL); +} + +static ssize_t client_ctl_write(struct file *file, const char __user *buf, + size_t size, loff_t *pos) +{ + char *data; + struct nfs4_client *clp; + + data = simple_transaction_get(file, buf, size); + if (IS_ERR(data)) + return PTR_ERR(data); + if (size != 7 || 0 != memcmp(data, "expire\n", 7)) + return -EINVAL; + clp = get_nfsdfs_clp(file_inode(file)); + if (!clp) + return -ENXIO; + force_expire_client(clp); + nfsd4_put_client(clp); + return 7; +} + +static const struct file_operations client_ctl_fops = { + .write = client_ctl_write, + .release = simple_transaction_release, +}; + +static const struct tree_descr client_files[] = { + [0] = {"info", &client_info_fops, S_IRUSR}, + [1] = {"states", &client_states_fops, S_IRUSR}, + [2] = {"ctl", &client_ctl_fops, S_IWUSR}, + [3] = {""}, +}; + +static int +nfsd4_cb_recall_any_done(struct nfsd4_callback *cb, + struct rpc_task *task) +{ + trace_nfsd_cb_recall_any_done(cb, task); + switch (task->tk_status) { + case -NFS4ERR_DELAY: + rpc_delay(task, 2 * HZ); + return 0; + default: + return 1; + } +} + +static void +nfsd4_cb_recall_any_release(struct nfsd4_callback *cb) +{ + struct nfs4_client *clp = cb->cb_clp; + + nfsd4_put_client(clp); +} + +static int +nfsd4_cb_getattr_done(struct nfsd4_callback *cb, struct rpc_task *task) +{ + struct nfs4_cb_fattr *ncf = + container_of(cb, struct nfs4_cb_fattr, ncf_getattr); + struct nfs4_delegation *dp = + container_of(ncf, struct nfs4_delegation, dl_cb_fattr); + + trace_nfsd_cb_getattr_done(&dp->dl_stid.sc_stateid, task); + ncf->ncf_cb_status = task->tk_status; + switch (task->tk_status) { + case -NFS4ERR_DELAY: + rpc_delay(task, 2 * HZ); + return 0; + default: + return 1; + } +} + +static void +nfsd4_cb_getattr_release(struct nfsd4_callback *cb) +{ + struct nfs4_cb_fattr *ncf = + container_of(cb, struct nfs4_cb_fattr, ncf_getattr); + struct nfs4_delegation *dp = + container_of(ncf, struct nfs4_delegation, dl_cb_fattr); + + nfs4_put_stid(&dp->dl_stid); +} + +static void nfsd_break_one_deleg(struct nfs4_delegation *dp) +{ + bool queued; + + if (test_and_set_bit(NFSD4_CALLBACK_RUNNING, &dp->dl_recall.cb_flags)) + return; + + /* + * When called from the lease break (nfsd_break_deleg_cb()) the state + * code is serialized by the flc_lock and the lease has not been + * removed yet, so sc_count is known to be nonzero. The CB_NOTIFY + * callback paths reach here from a workqueue without the flc_lock, + * where the delegation may already be unhashed with sc_count at zero. + * Use refcount_inc_not_zero() so both cases are safe, and bail if the + * delegation is already being torn down. + */ + if (!refcount_inc_not_zero(&dp->dl_stid.sc_count)) { + clear_bit(NFSD4_CALLBACK_RUNNING, &dp->dl_recall.cb_flags); + return; + } + queued = nfsd4_run_cb(&dp->dl_recall); + WARN_ON_ONCE(!queued); + if (!queued) { + refcount_dec(&dp->dl_stid.sc_count); + clear_bit(NFSD4_CALLBACK_RUNNING, &dp->dl_recall.cb_flags); + } +} + +static bool +nfsd4_cb_notify_prepare(struct nfsd4_callback *cb) +{ + struct nfsd4_cb_notify *ncn = container_of(cb, struct nfsd4_cb_notify, ncn_cb); + struct nfs4_delegation *dp = container_of(ncn, struct nfs4_delegation, dl_cb_notify); + struct nfsd_notify_event *events[NOTIFY4_EVENT_QUEUE_SIZE]; + struct xdr_buf xdr = { .buflen = PAGE_SIZE * NOTIFY4_PAGE_ARRAY_SIZE, + .pages = ncn->ncn_pages }; + int limit = NOTIFY4_EVENT_QUEUE_SIZE; + struct xdr_stream stream; + struct nfsd_file *nf; + bool error = false; + int count, i; + + /* Save a slot for dir attr update if requested */ + if (dp->dl_notify_mask & BIT(NOTIFY4_CHANGE_DIR_ATTRS)) + --limit; + + /* Clear any failure recorded by a previous transmit. */ + ncn->ncn_encode_err = false; + + xdr_init_encode_pages(&stream, &xdr); + + spin_lock(&ncn->ncn_lock); + count = ncn->ncn_evt_cnt; + + /* spurious queueing? */ + if (count == 0) { + spin_unlock(&ncn->ncn_lock); + return false; + } + + memcpy(events, ncn->ncn_evt, sizeof(*events) * count); + ncn->ncn_evt_cnt = 0; + spin_unlock(&ncn->ncn_lock); + + /* + * We can't keep up! Drop the queued events and recall. The queue must + * be drained here: out_recall leaves ncn_evt_cnt at 0, so the release + * op won't see leftover events and requeue this callback forever. + */ + if (count > limit) { + for (i = 0; i < count; ++i) + nfsd_notify_event_put(events[i]); + goto out_recall; + } + + rcu_read_lock(); + nf = nfsd_file_get(rcu_dereference(dp->dl_stid.sc_file->fi_deleg_file)); + rcu_read_unlock(); + if (!nf) { + for (i = 0; i < count; ++i) + nfsd_notify_event_put(events[i]); + goto out_recall; + } + + for (i = 0; i < count; ++i) { + struct nfsd_notify_event *nne = events[i]; + + if (!error) { + u32 *maskp = (u32 *)xdr_reserve_space(&stream, sizeof(*maskp)); + u8 *p; + + if (!maskp) { + error = true; + goto put_event; + } + + p = nfsd4_encode_notify_event(&stream, nne, dp, nf, maskp); + if (!p) { + pr_notice("Could not generate CB_NOTIFY from fsnotify mask 0x%x\n", + nne->ne_mask); + error = true; + goto put_event; + } + + ncn->ncn_nf[i].notify_mask.count = 1; + ncn->ncn_nf[i].notify_mask.element = maskp; + ncn->ncn_nf[i].notify_vals.data = p; + ncn->ncn_nf[i].notify_vals.len = (u8 *)stream.p - p; + } +put_event: + nfsd_notify_event_put(nne); + } + if (!error && (dp->dl_notify_mask & BIT(NOTIFY4_CHANGE_DIR_ATTRS))) { + u32 *maskp = (u32 *)xdr_reserve_space(&stream, sizeof(*maskp)); + u8 *p; + + if (maskp) + p = nfsd4_encode_dir_attr_change(&stream, dp, nf); + else + p = ERR_PTR(-ENOBUFS); + + if (IS_ERR(p)) { + /* + * The client asked to be told about dir attr changes + * but the change could not be encoded. RFC 8881 + * s10.9.4 requires the server to recall the delegation + * rather than drop a requested notification, so fall + * through to recall. A NULL return instead means there + * were no attributes to report, so omit the event in + * that case. + */ + error = true; + } else if (p) { + *maskp = BIT(NOTIFY4_CHANGE_DIR_ATTRS); + ncn->ncn_nf[count].notify_mask.count = 1; + ncn->ncn_nf[count].notify_mask.element = maskp; + ncn->ncn_nf[count].notify_vals.data = p; + ncn->ncn_nf[count].notify_vals.len = (u8 *)stream.p - p; + ++count; + } + } + if (!error) { + ncn->ncn_nf_cnt = count; + nfsd_file_put(nf); + return true; + } + nfsd_file_put(nf); +out_recall: + nfsd_break_one_deleg(dp); + return false; +} + +static int +nfsd4_cb_notify_done(struct nfsd4_callback *cb, + struct rpc_task *task) +{ + struct nfsd4_cb_notify *ncn = container_of(cb, struct nfsd4_cb_notify, ncn_cb); + struct nfs4_delegation *dp = container_of(ncn, struct nfs4_delegation, dl_cb_notify); + + if (dp->dl_stid.sc_status) + return 1; + + /* + * The CB_NOTIFY op overflowed the send buffer and was dropped from the + * compound. The notification is lost, so recall the delegation rather + * than leaving the client unaware of the directory change. + */ + if (ncn->ncn_encode_err) { + nfsd_break_one_deleg(dp); + return 1; + } + + switch (task->tk_status) { + case -NFS4ERR_DELAY: + rpc_delay(task, 2 * HZ); + return 0; + default: + /* For any other hard error, recall the deleg */ + nfsd_break_one_deleg(dp); + fallthrough; + case 0: + return 1; + } +} + +static void nfsd4_run_cb_notify(struct nfsd4_cb_notify *ncn); + +static void +nfsd4_cb_notify_release(struct nfsd4_callback *cb) +{ + struct nfsd4_cb_notify *ncn = + container_of(cb, struct nfsd4_cb_notify, ncn_cb); + struct nfs4_delegation *dp = + container_of(ncn, struct nfs4_delegation, dl_cb_notify); + + /* + * Drain events that arrived while this callback was in flight, but + * don't requeue against a revoked delegation: there's no point in + * notifying a client that no longer holds it, and doing so can pin the + * stid and spin the workqueue. + */ + if (!dp->dl_stid.sc_status && READ_ONCE(ncn->ncn_evt_cnt) > 0) + nfsd4_run_cb_notify(ncn); + nfs4_put_stid(&dp->dl_stid); +} + +static const struct nfsd4_callback_ops nfsd4_cb_recall_any_ops = { + .done = nfsd4_cb_recall_any_done, + .release = nfsd4_cb_recall_any_release, + .opcode = OP_CB_RECALL_ANY, +}; + +static const struct nfsd4_callback_ops nfsd4_cb_getattr_ops = { + .done = nfsd4_cb_getattr_done, + .release = nfsd4_cb_getattr_release, + .opcode = OP_CB_GETATTR, +}; + +static const struct nfsd4_callback_ops nfsd4_cb_notify_ops = { + .prepare = nfsd4_cb_notify_prepare, + .done = nfsd4_cb_notify_done, + .release = nfsd4_cb_notify_release, + .opcode = OP_CB_NOTIFY, +}; + +static void nfs4_cb_getattr(struct nfs4_cb_fattr *ncf) +{ + struct nfs4_delegation *dp = + container_of(ncf, struct nfs4_delegation, dl_cb_fattr); + + if (test_and_set_bit(NFSD4_CALLBACK_RUNNING, &ncf->ncf_getattr.cb_flags)) + return; + + /* set to proper status when nfsd4_cb_getattr_done runs */ + ncf->ncf_cb_status = NFS4ERR_IO; + + /* ensure that wake_bit is done when RUNNING is cleared */ + set_bit(NFSD4_CALLBACK_WAKE, &ncf->ncf_getattr.cb_flags); + + refcount_inc(&dp->dl_stid.sc_count); + nfsd4_run_cb(&ncf->ncf_getattr); +} + +static struct nfs4_client *create_client(struct xdr_netobj name, + struct svc_rqst *rqstp, nfs4_verifier *verf) +{ + struct nfs4_client *clp; + struct sockaddr *sa = svc_addr(rqstp); + int ret; + struct net *net = SVC_NET(rqstp); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct dentry *dentries[ARRAY_SIZE(client_files)]; + + clp = alloc_client(name, nn); + if (clp == NULL) + return NULL; + + ret = copy_cred(&clp->cl_cred, &rqstp->rq_cred); + if (ret) { + free_client(clp); + return NULL; + } + gen_clid(clp, nn); + kref_init(&clp->cl_nfsdfs.cl_ref); + nfsd4_init_cb(&clp->cl_cb_null, clp, NULL, NFSPROC4_CLNT_CB_NULL); + clp->cl_time = ktime_get_boottime_seconds(); + copy_verf(clp, verf); + memcpy(&clp->cl_addr, sa, sizeof(struct sockaddr_storage)); + RCU_INIT_POINTER(clp->cl_cb_session, NULL); + clp->net = net; + clp->cl_nfsd_dentry = nfsd_client_mkdir( + nn, &clp->cl_nfsdfs, + clp->cl_clientid.cl_id - nn->clientid_base, + client_files, dentries); + clp->cl_nfsd_info_dentry = dentries[0]; + if (!clp->cl_nfsd_dentry) { + free_client(clp); + return NULL; + } + clp->cl_ra = kzalloc_obj(*clp->cl_ra); + if (!clp->cl_ra) { + free_client(clp); + return NULL; + } + clp->cl_ra_time = 0; + nfsd4_init_cb(&clp->cl_ra->ra_cb, clp, &nfsd4_cb_recall_any_ops, + NFSPROC4_CLNT_CB_RECALL_ANY); + return clp; +} + +static void +add_clp_to_name_tree(struct nfs4_client *new_clp, struct rb_root *root) +{ + struct rb_node **new = &(root->rb_node), *parent = NULL; + struct nfs4_client *clp; + + while (*new) { + clp = rb_entry(*new, struct nfs4_client, cl_namenode); + parent = *new; + + if (compare_blob(&clp->cl_name, &new_clp->cl_name) > 0) + new = &((*new)->rb_left); + else + new = &((*new)->rb_right); + } + + rb_link_node(&new_clp->cl_namenode, parent, new); + rb_insert_color(&new_clp->cl_namenode, root); +} + +static struct nfs4_client * +find_clp_in_name_tree(struct xdr_netobj *name, struct rb_root *root) +{ + int cmp; + struct rb_node *node = root->rb_node; + struct nfs4_client *clp; + + while (node) { + clp = rb_entry(node, struct nfs4_client, cl_namenode); + cmp = compare_blob(&clp->cl_name, name); + if (cmp > 0) + node = node->rb_left; + else if (cmp < 0) + node = node->rb_right; + else + return clp; + } + return NULL; +} + +static void +add_to_unconfirmed(struct nfs4_client *clp) +{ + unsigned int idhashval; + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + lockdep_assert_held(&nn->client_lock); + + clear_bit(NFSD4_CLIENT_CONFIRMED, &clp->cl_flags); + add_clp_to_name_tree(clp, &nn->unconf_name_tree); + idhashval = clientid_hashval(clp->cl_clientid.cl_id); + list_add(&clp->cl_idhash, &nn->unconf_id_hashtbl[idhashval]); + renew_client_locked(clp); +} + +static void +move_to_confirmed(struct nfs4_client *clp) +{ + unsigned int idhashval = clientid_hashval(clp->cl_clientid.cl_id); + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + lockdep_assert_held(&nn->client_lock); + + list_move(&clp->cl_idhash, &nn->conf_id_hashtbl[idhashval]); + rb_erase(&clp->cl_namenode, &nn->unconf_name_tree); + add_clp_to_name_tree(clp, &nn->conf_name_tree); + set_bit(NFSD4_CLIENT_CONFIRMED, &clp->cl_flags); + trace_nfsd_clid_confirmed(&clp->cl_clientid); + renew_client_locked(clp); +} + +static struct nfs4_client * +find_client_in_id_table(struct list_head *tbl, clientid_t *clid, bool sessions) +{ + struct nfs4_client *clp; + unsigned int idhashval = clientid_hashval(clid->cl_id); + + list_for_each_entry(clp, &tbl[idhashval], cl_idhash) { + if (same_clid(&clp->cl_clientid, clid)) { + if ((bool)clp->cl_minorversion != sessions) + return NULL; + renew_client_locked(clp); + return clp; + } + } + return NULL; +} + +static struct nfs4_client * +find_confirmed_client(clientid_t *clid, bool sessions, struct nfsd_net *nn) +{ + struct list_head *tbl = nn->conf_id_hashtbl; + + lockdep_assert_held(&nn->client_lock); + return find_client_in_id_table(tbl, clid, sessions); +} + +static struct nfs4_client * +find_unconfirmed_client(clientid_t *clid, bool sessions, struct nfsd_net *nn) +{ + struct list_head *tbl = nn->unconf_id_hashtbl; + + lockdep_assert_held(&nn->client_lock); + return find_client_in_id_table(tbl, clid, sessions); +} + +static bool clp_used_exchangeid(struct nfs4_client *clp) +{ + return clp->cl_exchange_flags != 0; +} + +static struct nfs4_client * +find_confirmed_client_by_name(struct xdr_netobj *name, struct nfsd_net *nn) +{ + lockdep_assert_held(&nn->client_lock); + return find_clp_in_name_tree(name, &nn->conf_name_tree); +} + +static struct nfs4_client * +find_unconfirmed_client_by_name(struct xdr_netobj *name, struct nfsd_net *nn) +{ + lockdep_assert_held(&nn->client_lock); + return find_clp_in_name_tree(name, &nn->unconf_name_tree); +} + +static void +gen_callback(struct nfs4_client *clp, struct nfsd4_setclientid *se, struct svc_rqst *rqstp) +{ + struct nfs4_cb_conn *conn = &clp->cl_cb_conn; + struct sockaddr *sa = svc_addr(rqstp); + u32 scopeid = rpc_get_scope_id(sa); + unsigned short expected_family; + + /* Currently, we only support tcp and tcp6 for the callback channel */ + if (se->se_callback_netid_len == 3 && + !memcmp(se->se_callback_netid_val, "tcp", 3)) + expected_family = AF_INET; + else if (se->se_callback_netid_len == 4 && + !memcmp(se->se_callback_netid_val, "tcp6", 4)) + expected_family = AF_INET6; + else + goto out_err; + + conn->cb_addrlen = rpc_uaddr2sockaddr(clp->net, se->se_callback_addr_val, + se->se_callback_addr_len, + (struct sockaddr *)&conn->cb_addr, + sizeof(conn->cb_addr)); + + if (!conn->cb_addrlen || conn->cb_addr.ss_family != expected_family) + goto out_err; + + if (conn->cb_addr.ss_family == AF_INET6) + ((struct sockaddr_in6 *)&conn->cb_addr)->sin6_scope_id = scopeid; + + conn->cb_prog = se->se_callback_prog; + conn->cb_ident = se->se_callback_ident; + memcpy(&conn->cb_saddr, &rqstp->rq_daddr, rqstp->rq_daddrlen); + trace_nfsd_cb_args(clp, conn); + return; +out_err: + conn->cb_addr.ss_family = AF_UNSPEC; + conn->cb_addrlen = 0; + trace_nfsd_cb_nodelegs(clp); + return; +} + +/* + * Cache a reply. nfsd4_check_resp_size() has bounded the cache size. + */ +static void +nfsd4_store_cache_entry(struct nfsd4_compoundres *resp) +{ + struct xdr_buf *buf = resp->xdr->buf; + struct nfsd4_slot *slot = resp->cstate.slot; + unsigned int base; + + /* + * RFC 5661 Section 2.10.6.1.2: + * + * Any time SEQUENCE ... returns an error ... [t]he replier MUST NOT + * modify the reply cache entry for the slot whenever an error is + * returned from SEQUENCE ... + * + * Because nfsd4_store_cache_entry is called only by + * nfsd4_sequence_done(), nfsd4_store_cache_entry() is called only + * when a SEQUENCE operation was part of the COMPOUND. + * nfs41_check_op_ordering() ensures SEQUENCE is the first op. + */ + if (resp->opcnt == 1 && resp->cstate.status != nfs_ok) + return; + + slot->sl_flags |= NFSD4_SLOT_INITIALIZED; + slot->sl_opcnt = resp->opcnt; + slot->sl_status = resp->cstate.status; + free_svc_cred(&slot->sl_cred); + copy_cred(&slot->sl_cred, &resp->rqstp->rq_cred); + + if (!(resp->cstate.slot->sl_flags & NFSD4_SLOT_CACHETHIS)) { + slot->sl_flags &= ~NFSD4_SLOT_CACHED; + return; + } + slot->sl_flags |= NFSD4_SLOT_CACHED; + + base = resp->cstate.data_offset; + slot->sl_datalen = buf->len - base; + if (read_bytes_from_xdr_buf(buf, base, slot->sl_data, slot->sl_datalen)) + WARN(1, "%s: sessions DRC could not cache compound\n", + __func__); + return; +} + +/* + * The sequence operation is not cached because we can use the slot and + * session values. + */ +static __be32 +nfsd4_replay_cache_entry(struct nfsd4_compoundres *resp, + struct nfsd4_sequence *seq) +{ + struct nfsd4_compoundargs *args = resp->rqstp->rq_argp; + struct nfsd4_slot *slot = resp->cstate.slot; + struct xdr_stream *xdr = resp->xdr; + __be32 *p; + + dprintk("--> %s slot %p\n", __func__, slot); + + /* Always encode the SEQUENCE response. */ + nfsd4_encode_operation(resp, &args->ops[0]); + if (args->opcnt == 1) + /* A solo SEQUENCE - nothing was cached */ + return args->ops[0].status; + + if (!(slot->sl_flags & NFSD4_SLOT_CACHED)) { + /* We weren't asked to cache this. */ + struct nfsd4_op *op; + + op = &args->ops[resp->opcnt++]; + op->status = nfserr_retry_uncached_rep; + nfsd4_encode_operation(resp, op); + return op->status; + } + + /* return reply from cache */ + p = xdr_reserve_space(xdr, slot->sl_datalen); + if (!p) { + WARN_ON_ONCE(1); + return nfserr_serverfault; + } + xdr_encode_opaque_fixed(p, slot->sl_data, slot->sl_datalen); + xdr_commit_encode(xdr); + + resp->opcnt = slot->sl_opcnt; + return slot->sl_status; +} + +/* + * Set the exchange_id flags returned by the server. + */ +static void +nfsd4_set_ex_flags(struct nfs4_client *new, struct nfsd4_exchange_id *clid) +{ +#ifdef CONFIG_NFSD_PNFS + new->cl_exchange_flags |= EXCHGID4_FLAG_USE_PNFS_MDS; +#else + new->cl_exchange_flags |= EXCHGID4_FLAG_USE_NON_PNFS; +#endif + + /* Referrals are supported, Migration is not. */ + new->cl_exchange_flags |= EXCHGID4_FLAG_SUPP_MOVED_REFER; + + /* set the wire flags to return to client. */ + clid->flags = new->cl_exchange_flags; +} + +static bool client_has_openowners(struct nfs4_client *clp) +{ + struct nfs4_openowner *oo; + + list_for_each_entry(oo, &clp->cl_openowners, oo_perclient) { + if (!list_empty(&oo->oo_owner.so_stateids)) + return true; + } + return false; +} + +static bool client_has_state(struct nfs4_client *clp) +{ + return client_has_openowners(clp) +#ifdef CONFIG_NFSD_PNFS + || !list_empty(&clp->cl_lo_states) +#endif + || !list_empty(&clp->cl_delegations) + || !list_empty(&clp->cl_sessions) + || nfsd4_has_active_async_copies(clp); +} + +static __be32 copy_impl_id(struct nfs4_client *clp, + struct nfsd4_exchange_id *exid) +{ + if (!exid->nii_domain.data) + return 0; + xdr_netobj_dup(&clp->cl_nii_domain, &exid->nii_domain, GFP_KERNEL); + if (!clp->cl_nii_domain.data) + return nfserr_jukebox; + xdr_netobj_dup(&clp->cl_nii_name, &exid->nii_name, GFP_KERNEL); + if (!clp->cl_nii_name.data) + return nfserr_jukebox; + clp->cl_nii_time = exid->nii_time; + return 0; +} + +__be32 +nfsd4_exchange_id(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_exchange_id *exid = &u->exchange_id; + struct nfs4_client *conf, *new; + struct nfs4_client *unconf = NULL; + __be32 status; + char addr_str[INET6_ADDRSTRLEN]; + nfs4_verifier verf = exid->verifier; + struct sockaddr *sa = svc_addr(rqstp); + bool update = exid->flags & EXCHGID4_FLAG_UPD_CONFIRMED_REC_A; + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + + rpc_ntop(sa, addr_str, sizeof(addr_str)); + dprintk("%s rqstp=%p exid=%p clname.len=%u clname.data=%p " + "ip_addr=%s flags %x, spa_how %u\n", + __func__, rqstp, exid, exid->clname.len, exid->clname.data, + addr_str, exid->flags, exid->spa_how); + + exid->server_impl_name = kasprintf(GFP_KERNEL, "%s %s %s %s", + utsname()->sysname, utsname()->release, + utsname()->version, utsname()->machine); + if (!exid->server_impl_name) + return nfserr_jukebox; + + if (exid->flags & ~EXCHGID4_FLAG_MASK_A) + return nfserr_inval; + + new = create_client(exid->clname, rqstp, &verf); + if (new == NULL) + return nfserr_jukebox; + status = copy_impl_id(new, exid); + if (status) + goto out_nolock; + + switch (exid->spa_how) { + case SP4_MACH_CRED: + exid->spo_must_enforce[0] = 0; + exid->spo_must_enforce[1] = ( + 1 << (OP_BIND_CONN_TO_SESSION - 32) | + 1 << (OP_EXCHANGE_ID - 32) | + 1 << (OP_CREATE_SESSION - 32) | + 1 << (OP_DESTROY_SESSION - 32) | + 1 << (OP_DESTROY_CLIENTID - 32)); + + exid->spo_must_allow[0] &= (1 << (OP_CLOSE) | + 1 << (OP_OPEN_DOWNGRADE) | + 1 << (OP_LOCKU) | + 1 << (OP_DELEGRETURN)); + + exid->spo_must_allow[1] &= ( + 1 << (OP_TEST_STATEID - 32) | + 1 << (OP_FREE_STATEID - 32)); + if (!svc_rqst_integrity_protected(rqstp)) { + status = nfserr_inval; + goto out_nolock; + } + /* + * Sometimes userspace doesn't give us a principal. + * Which is a bug, really. Anyway, we can't enforce + * MACH_CRED in that case, better to give up now: + */ + if (!new->cl_cred.cr_principal && + !new->cl_cred.cr_raw_principal) { + status = nfserr_serverfault; + goto out_nolock; + } + new->cl_mach_cred = true; + break; + case SP4_NONE: + break; + default: /* checked by xdr code */ + WARN_ON_ONCE(1); + fallthrough; + case SP4_SSV: + status = nfserr_encr_alg_unsupp; + goto out_nolock; + } + + /* Cases below refer to rfc 5661 section 18.35.4: */ + spin_lock(&nn->client_lock); + conf = find_confirmed_client_by_name(&exid->clname, nn); + if (conf) { + bool creds_match = same_creds(&conf->cl_cred, &rqstp->rq_cred); + bool verfs_match = same_verf(&verf, &conf->cl_verifier); + + if (update) { + if (!clp_used_exchangeid(conf)) { /* buggy client */ + status = nfserr_inval; + goto out; + } + if (!nfsd4_mach_creds_match(conf, rqstp)) { + status = nfserr_wrong_cred; + goto out; + } + if (!creds_match) { /* case 9 */ + status = nfserr_perm; + goto out; + } + if (!verfs_match) { /* case 8 */ + status = nfserr_not_same; + goto out; + } + /* case 6 */ + exid->flags |= EXCHGID4_FLAG_CONFIRMED_R; + trace_nfsd_clid_confirmed_r(conf); + goto out_copy; + } + if (!creds_match) { /* case 3 */ + if (client_has_state(conf)) { + status = nfserr_clid_inuse; + trace_nfsd_clid_cred_mismatch(conf, rqstp); + goto out; + } + goto out_new; + } + if (verfs_match) { /* case 2 */ + conf->cl_exchange_flags |= EXCHGID4_FLAG_CONFIRMED_R; + trace_nfsd_clid_confirmed_r(conf); + goto out_copy; + } + /* case 5, client reboot */ + trace_nfsd_clid_verf_mismatch(conf, rqstp, &verf); + conf = NULL; + goto out_new; + } + + if (update) { /* case 7 */ + status = nfserr_noent; + goto out; + } + + unconf = find_unconfirmed_client_by_name(&exid->clname, nn); + if (unconf) /* case 4, possible retry or client restart */ + unhash_client_locked(unconf); + + /* case 1, new owner ID */ + trace_nfsd_clid_fresh(new); + +out_new: + if (conf) { + status = mark_client_expired_locked(conf); + if (status) + goto out; + trace_nfsd_clid_replaced(&conf->cl_clientid); + } + new->cl_minorversion = cstate->minorversion; + new->cl_spo_must_allow.u.words[0] = exid->spo_must_allow[0]; + new->cl_spo_must_allow.u.words[1] = exid->spo_must_allow[1]; + + /* Contrived initial CREATE_SESSION response */ + new->cl_cs_slot.sl_status = nfserr_seq_misordered; + + add_to_unconfirmed(new); + swap(new, conf); +out_copy: + exid->clientid.cl_boot = conf->cl_clientid.cl_boot; + exid->clientid.cl_id = conf->cl_clientid.cl_id; + + exid->seqid = conf->cl_cs_slot.sl_seqid + 1; + nfsd4_set_ex_flags(conf, exid); + + exid->nii_domain.len = sizeof("kernel.org") - 1; + exid->nii_domain.data = "kernel.org"; + + /* + * Note that RFC 8881 places no length limit on + * nii_name, but this implementation permits no + * more than NFS4_OPAQUE_LIMIT bytes. + */ + exid->nii_name.len = strlen(exid->server_impl_name); + if (exid->nii_name.len > NFS4_OPAQUE_LIMIT) + exid->nii_name.len = NFS4_OPAQUE_LIMIT; + exid->nii_name.data = exid->server_impl_name; + + /* just send zeros - the date is in nii_name */ + exid->nii_time.tv_sec = 0; + exid->nii_time.tv_nsec = 0; + + dprintk("nfsd4_exchange_id seqid %d flags %x\n", + conf->cl_cs_slot.sl_seqid, conf->cl_exchange_flags); + status = nfs_ok; + +out: + spin_unlock(&nn->client_lock); +out_nolock: + if (new) + expire_client(new); + if (unconf) { + trace_nfsd_clid_expire_unconf(&unconf->cl_clientid); + expire_client(unconf); + } + return status; +} + +void +nfsd4_exchange_id_release(union nfsd4_op_u *u) +{ + struct nfsd4_exchange_id *exid = &u->exchange_id; + + kfree(exid->server_impl_name); +} + +static __be32 check_slot_seqid(u32 seqid, u32 slot_seqid, u8 flags) +{ + /* The slot is in use, and no response has been sent. */ + if (flags & NFSD4_SLOT_INUSE) { + if (seqid == slot_seqid) + return nfserr_jukebox; + else + return nfserr_seq_misordered; + } + /* Note unsigned 32-bit arithmetic handles wraparound: */ + if (likely(seqid == slot_seqid + 1)) + return nfs_ok; + if ((flags & NFSD4_SLOT_REUSED) && seqid == 1) + return nfs_ok; + if (seqid == slot_seqid) + return nfserr_replay_cache; + return nfserr_seq_misordered; +} + +/* + * Cache the create session result into the create session single DRC + * slot cache by saving the xdr structure. sl_seqid has been set. + * Do this for solo or embedded create session operations. + */ +static void +nfsd4_cache_create_session(struct nfsd4_create_session *cr_ses, + struct nfsd4_clid_slot *slot, __be32 nfserr) +{ + slot->sl_status = nfserr; + memcpy(&slot->sl_cr_ses, cr_ses, sizeof(*cr_ses)); +} + +static __be32 +nfsd4_replay_create_session(struct nfsd4_create_session *cr_ses, + struct nfsd4_clid_slot *slot) +{ + memcpy(cr_ses, &slot->sl_cr_ses, sizeof(*cr_ses)); + return slot->sl_status; +} + +#define NFSD_MIN_REQ_HDR_SEQ_SZ ((\ + 2 * 2 + /* credential,verifier: AUTH_NULL, length 0 */ \ + 1 + /* MIN tag is length with zero, only length */ \ + 3 + /* version, opcount, opcode */ \ + XDR_QUADLEN(NFS4_MAX_SESSIONID_LEN) + \ + /* seqid, slotID, slotID, cache */ \ + 4 ) * sizeof(__be32)) + +#define NFSD_MIN_RESP_HDR_SEQ_SZ ((\ + 2 + /* verifier: AUTH_NULL, length 0 */\ + 1 + /* status */ \ + 1 + /* MIN tag is length with zero, only length */ \ + 3 + /* opcount, opcode, opstatus*/ \ + XDR_QUADLEN(NFS4_MAX_SESSIONID_LEN) + \ + /* seqid, slotID, slotID, slotID, status */ \ + 5 ) * sizeof(__be32)) + +static __be32 check_forechannel_attrs(struct nfsd4_channel_attrs *ca, struct nfsd_net *nn) +{ + u32 maxrpc = nn->nfsd_serv->sv_max_mesg; + + if (ca->maxreq_sz < NFSD_MIN_REQ_HDR_SEQ_SZ) + return nfserr_toosmall; + if (ca->maxresp_sz < NFSD_MIN_RESP_HDR_SEQ_SZ) + return nfserr_toosmall; + ca->headerpadsz = 0; + ca->maxreq_sz = min_t(u32, ca->maxreq_sz, maxrpc); + ca->maxresp_sz = min_t(u32, ca->maxresp_sz, maxrpc); + ca->maxops = min_t(u32, ca->maxops, NFSD_MAX_OPS_PER_COMPOUND); + ca->maxresp_cached = min_t(u32, ca->maxresp_cached, + NFSD_SLOT_CACHE_SIZE + NFSD_MIN_HDR_SEQ_SZ); + ca->maxreqs = min_t(u32, ca->maxreqs, NFSD_MAX_SLOTS_PER_SESSION); + + return nfs_ok; +} + +/* + * Server's NFSv4.1 backchannel support is AUTH_SYS-only for now. + * These are based on similar macros in linux/sunrpc/msg_prot.h . + */ +#define RPC_MAX_HEADER_WITH_AUTH_SYS \ + (RPC_CALLHDRSIZE + 2 * (2 + UNX_CALLSLACK)) + +#define RPC_MAX_REPHEADER_WITH_AUTH_SYS \ + (RPC_REPHDRSIZE + (2 + NUL_REPLYSLACK)) + +#define NFSD_CB_MAX_REQ_SZ ((NFS4_enc_cb_recall_sz + \ + RPC_MAX_HEADER_WITH_AUTH_SYS) * sizeof(__be32)) +#define NFSD_CB_MAX_RESP_SZ ((NFS4_dec_cb_recall_sz + \ + RPC_MAX_REPHEADER_WITH_AUTH_SYS) * \ + sizeof(__be32)) + +static __be32 check_backchannel_attrs(struct nfsd4_channel_attrs *ca) +{ + ca->headerpadsz = 0; + + if (ca->maxreq_sz < NFSD_CB_MAX_REQ_SZ) + return nfserr_toosmall; + if (ca->maxresp_sz < NFSD_CB_MAX_RESP_SZ) + return nfserr_toosmall; + ca->maxresp_cached = 0; + if (ca->maxops < 2) + return nfserr_toosmall; + + return nfs_ok; +} + +static __be32 nfsd4_check_cb_sec(struct nfsd4_cb_sec *cbs) +{ + switch (cbs->flavor) { + case RPC_AUTH_NULL: + case RPC_AUTH_UNIX: + return nfs_ok; + default: + /* + * GSS case: the spec doesn't allow us to return this + * error. But it also doesn't allow us not to support + * GSS. + * I'd rather this fail hard than return some error the + * client might think it can already handle: + */ + return nfserr_encr_alg_unsupp; + } +} + +__be32 +nfsd4_create_session(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, union nfsd4_op_u *u) +{ + struct nfsd4_create_session *cr_ses = &u->create_session; + struct sockaddr *sa = svc_addr(rqstp); + struct nfs4_client *conf, *unconf; + struct nfsd4_clid_slot *cs_slot; + struct nfs4_client *old = NULL; + struct nfsd4_session *new; + struct nfsd4_conn *conn; + __be32 status = 0; + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + + if (cr_ses->flags & ~SESSION4_FLAG_MASK_A) + return nfserr_inval; + status = nfsd4_check_cb_sec(&cr_ses->cb_sec); + if (status) + return status; + status = check_forechannel_attrs(&cr_ses->fore_channel, nn); + if (status) + return status; + status = check_backchannel_attrs(&cr_ses->back_channel); + if (status) + goto out_err; + status = nfserr_jukebox; + new = alloc_session(&cr_ses->fore_channel, &cr_ses->back_channel); + if (!new) + goto out_err; + conn = alloc_conn_from_crses(rqstp, cr_ses); + if (!conn) + goto out_free_session; + + spin_lock(&nn->client_lock); + + /* RFC 8881 Section 18.36.4 Phase 1: Client record look-up. */ + unconf = find_unconfirmed_client(&cr_ses->clientid, true, nn); + conf = find_confirmed_client(&cr_ses->clientid, true, nn); + if (!conf && !unconf) { + status = nfserr_stale_clientid; + goto out_free_conn; + } + + /* RFC 8881 Section 18.36.4 Phase 2: Sequence ID processing. */ + if (conf) { + cs_slot = &conf->cl_cs_slot; + trace_nfsd_slot_seqid_conf(conf, cr_ses); + } else { + cs_slot = &unconf->cl_cs_slot; + trace_nfsd_slot_seqid_unconf(unconf, cr_ses); + } + status = check_slot_seqid(cr_ses->seqid, cs_slot->sl_seqid, 0); + switch (status) { + case nfs_ok: + cs_slot->sl_seqid++; + cr_ses->seqid = cs_slot->sl_seqid; + break; + case nfserr_replay_cache: + status = nfsd4_replay_create_session(cr_ses, cs_slot); + fallthrough; + case nfserr_jukebox: + /* The server MUST NOT cache NFS4ERR_DELAY */ + goto out_free_conn; + default: + goto out_cache_error; + } + + /* RFC 8881 Section 18.36.4 Phase 3: Client ID confirmation. */ + if (conf) { + status = nfserr_wrong_cred; + if (!nfsd4_mach_creds_match(conf, rqstp)) + goto out_cache_error; + } else { + status = nfserr_clid_inuse; + if (!same_creds(&unconf->cl_cred, &rqstp->rq_cred) || + !rpc_cmp_addr(sa, (struct sockaddr *) &unconf->cl_addr)) { + trace_nfsd_clid_cred_mismatch(unconf, rqstp); + goto out_cache_error; + } + status = nfserr_wrong_cred; + if (!nfsd4_mach_creds_match(unconf, rqstp)) + goto out_cache_error; + old = find_confirmed_client_by_name(&unconf->cl_name, nn); + if (old) { + status = mark_client_expired_locked(old); + if (status) + goto out_expired_error; + trace_nfsd_clid_replaced(&old->cl_clientid); + } + move_to_confirmed(unconf); + conf = unconf; + } + + /* RFC 8881 Section 18.36.4 Phase 4: Session creation. */ + status = nfs_ok; + /* Persistent sessions are not supported */ + cr_ses->flags &= ~SESSION4_PERSIST; + /* Upshifting from TCP to RDMA is not supported */ + cr_ses->flags &= ~SESSION4_RDMA; + /* Report the correct number of backchannel slots */ + cr_ses->back_channel.maxreqs = new->se_cb_highest_slot + 1; + + init_session(rqstp, new, conf, cr_ses); + nfsd4_get_session_locked(new); + + memcpy(cr_ses->sessionid.data, new->se_sessionid.data, + NFS4_MAX_SESSIONID_LEN); + + /* cache solo and embedded create sessions under the client_lock */ + nfsd4_cache_create_session(cr_ses, cs_slot, status); + spin_unlock(&nn->client_lock); + if (conf == unconf) + fsnotify_dentry(conf->cl_nfsd_info_dentry, FS_MODIFY); + /* init connection and backchannel */ + nfsd4_init_conn(rqstp, conn, new); + nfsd4_put_session(new); + if (old) + expire_client(old); + return status; + +out_expired_error: + /* + * Revert the slot seq_nr change so the server will process + * the client's resend instead of returning a cached response. + */ + if (status == nfserr_jukebox) { + cs_slot->sl_seqid--; + cr_ses->seqid = cs_slot->sl_seqid; + goto out_free_conn; + } +out_cache_error: + nfsd4_cache_create_session(cr_ses, cs_slot, status); +out_free_conn: + spin_unlock(&nn->client_lock); + free_conn(conn); +out_free_session: + __free_session(new); +out_err: + return status; +} + +static __be32 nfsd4_map_bcts_dir(u32 *dir) +{ + switch (*dir) { + case NFS4_CDFC4_FORE: + case NFS4_CDFC4_BACK: + return nfs_ok; + case NFS4_CDFC4_FORE_OR_BOTH: + case NFS4_CDFC4_BACK_OR_BOTH: + *dir = NFS4_CDFC4_BOTH; + return nfs_ok; + } + return nfserr_inval; +} + +__be32 nfsd4_backchannel_ctl(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_backchannel_ctl *bc = &u->backchannel_ctl; + struct nfsd4_session *session = cstate->session; + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + __be32 status; + + status = nfsd4_check_cb_sec(&bc->bc_cb_sec); + if (status) + return status; + spin_lock(&nn->client_lock); + session->se_cb_prog = bc->bc_cb_program; + session->se_cb_sec = bc->bc_cb_sec; + spin_unlock(&nn->client_lock); + + nfsd4_probe_callback(session->se_client); + + return nfs_ok; +} + +static struct nfsd4_conn *__nfsd4_find_conn(struct svc_xprt *xpt, struct nfsd4_session *s) +{ + struct nfsd4_conn *c; + + list_for_each_entry(c, &s->se_conns, cn_persession) { + if (c->cn_xprt == xpt) { + return c; + } + } + return NULL; +} + +static __be32 nfsd4_match_existing_connection(struct svc_rqst *rqst, + struct nfsd4_session *session, u32 req, struct nfsd4_conn **conn) +{ + struct nfs4_client *clp = session->se_client; + struct svc_xprt *xpt = rqst->rq_xprt; + struct nfsd4_conn *c; + __be32 status; + + /* Following the last paragraph of RFC 5661 Section 18.34.3: */ + spin_lock(&clp->cl_lock); + c = __nfsd4_find_conn(xpt, session); + if (!c) + status = nfserr_noent; + else if (req == c->cn_flags) + status = nfs_ok; + else if (req == NFS4_CDFC4_FORE_OR_BOTH && + c->cn_flags != NFS4_CDFC4_BACK) + status = nfs_ok; + else if (req == NFS4_CDFC4_BACK_OR_BOTH && + c->cn_flags != NFS4_CDFC4_FORE) + status = nfs_ok; + else + status = nfserr_inval; + spin_unlock(&clp->cl_lock); + if (status == nfs_ok && conn) + *conn = c; + return status; +} + +__be32 nfsd4_bind_conn_to_session(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_bind_conn_to_session *bcts = &u->bind_conn_to_session; + __be32 status; + struct nfsd4_conn *conn; + struct nfsd4_session *session; + struct net *net = SVC_NET(rqstp); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + if (!nfsd4_last_compound_op(rqstp)) + return nfserr_not_only_op; + spin_lock(&nn->client_lock); + session = find_in_sessionid_hashtbl(&bcts->sessionid, net, &status); + spin_unlock(&nn->client_lock); + if (!session) + goto out_no_session; + status = nfserr_wrong_cred; + if (!nfsd4_mach_creds_match(session->se_client, rqstp)) + goto out; + status = nfsd4_match_existing_connection(rqstp, session, + bcts->dir, &conn); + if (status == nfs_ok) { + if (bcts->dir == NFS4_CDFC4_FORE_OR_BOTH || + bcts->dir == NFS4_CDFC4_BACK) + conn->cn_flags |= NFS4_CDFC4_BACK; + nfsd4_probe_callback(session->se_client); + goto out; + } + if (status == nfserr_inval) + goto out; + status = nfsd4_map_bcts_dir(&bcts->dir); + if (status) + goto out; + conn = alloc_conn(rqstp, bcts->dir); + status = nfserr_jukebox; + if (!conn) + goto out; + nfsd4_init_conn(rqstp, conn, session); + status = nfs_ok; +out: + nfsd4_put_session(session); +out_no_session: + return status; +} + +static bool nfsd4_compound_in_session(struct nfsd4_compound_state *cstate, struct nfs4_sessionid *sid) +{ + if (!cstate->session) + return false; + return !memcmp(sid, &cstate->session->se_sessionid, sizeof(*sid)); +} + +__be32 +nfsd4_destroy_session(struct svc_rqst *r, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfs4_sessionid *sessionid = &u->destroy_session.sessionid; + struct nfsd4_session *ses; + __be32 status; + int ref_held_by_me = 0; + struct net *net = SVC_NET(r); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + status = nfserr_not_only_op; + if (nfsd4_compound_in_session(cstate, sessionid)) { + if (!nfsd4_last_compound_op(r)) + goto out; + ref_held_by_me++; + } + dump_sessionid(__func__, sessionid); + spin_lock(&nn->client_lock); + ses = find_in_sessionid_hashtbl(sessionid, net, &status); + if (!ses) + goto out_client_lock; + status = nfserr_wrong_cred; + if (!nfsd4_mach_creds_match(ses->se_client, r)) + goto out_put_session; + status = mark_session_dead_locked(ses, 1 + ref_held_by_me); + if (status) + goto out_put_session; + unhash_session(ses); + spin_unlock(&nn->client_lock); + + nfsd4_probe_callback_sync(ses->se_client); + + spin_lock(&nn->client_lock); + status = nfs_ok; +out_put_session: + nfsd4_put_session_locked(ses); +out_client_lock: + spin_unlock(&nn->client_lock); +out: + return status; +} + +static __be32 nfsd4_sequence_check_conn(struct nfsd4_conn *new, struct nfsd4_session *ses) +{ + struct nfs4_client *clp = ses->se_client; + struct nfsd4_conn *c; + __be32 status = nfs_ok; + int ret; + + spin_lock(&clp->cl_lock); + c = __nfsd4_find_conn(new->cn_xprt, ses); + if (c) + goto out_free; + status = nfserr_conn_not_bound_to_session; + if (clp->cl_mach_cred) + goto out_free; + __nfsd4_hash_conn(new, ses); + spin_unlock(&clp->cl_lock); + ret = nfsd4_register_conn(new); + if (ret) + /* oops; xprt is already down: */ + nfsd4_conn_lost(&new->cn_xpt_user); + return nfs_ok; +out_free: + spin_unlock(&clp->cl_lock); + free_conn(new); + return status; +} + +static bool nfsd4_session_too_many_ops(struct svc_rqst *rqstp, struct nfsd4_session *session) +{ + struct nfsd4_compoundargs *args = rqstp->rq_argp; + + return args->opcnt > session->se_fchannel.maxops; +} + +static bool nfsd4_request_too_big(struct svc_rqst *rqstp, + struct nfsd4_session *session) +{ + struct xdr_buf *xb = &rqstp->rq_arg; + + return xb->len > session->se_fchannel.maxreq_sz; +} + +static bool replay_matches_cache(struct svc_rqst *rqstp, + struct nfsd4_sequence *seq, struct nfsd4_slot *slot) +{ + struct nfsd4_compoundargs *argp = rqstp->rq_argp; + + if ((bool)(slot->sl_flags & NFSD4_SLOT_CACHETHIS) != + (bool)seq->cachethis) + return false; + /* + * If there's an error then the reply can have fewer ops than + * the call. + */ + if (slot->sl_opcnt < argp->opcnt && !slot->sl_status) + return false; + /* + * But if we cached a reply with *more* ops than the call you're + * sending us now, then this new call is clearly not really a + * replay of the old one: + */ + if (slot->sl_opcnt > argp->opcnt) + return false; + /* This is the only check explicitly called by spec: */ + if (!same_creds(&rqstp->rq_cred, &slot->sl_cred)) + return false; + /* + * There may be more comparisons we could actually do, but the + * spec doesn't require us to catch every case where the calls + * don't match (that would require caching the call as well as + * the reply), so we don't bother. + */ + return true; +} + +/* + * Note that the response is constructed here both for the case + * of a new SEQUENCE request and for a replayed SEQUENCE request. + * We do not cache SEQUENCE responses as SEQUENCE is idempotent. + */ +static void nfsd4_construct_sequence_response(struct nfsd4_session *session, + struct nfsd4_sequence *seq) +{ + struct nfs4_client *clp = session->se_client; + + seq->maxslots_response = max(session->se_target_maxslots, + seq->maxslots); + seq->target_maxslots = session->se_target_maxslots; + + switch (clp->cl_cb_state) { + case NFSD4_CB_DOWN: + seq->status_flags = SEQ4_STATUS_CB_PATH_DOWN; + break; + case NFSD4_CB_FAULT: + seq->status_flags = SEQ4_STATUS_BACKCHANNEL_FAULT; + break; + default: + seq->status_flags = 0; + } + if (!list_empty(&clp->cl_revoked)) + seq->status_flags |= SEQ4_STATUS_RECALLABLE_STATE_REVOKED; + if (atomic_read(&clp->cl_admin_revoked)) + seq->status_flags |= SEQ4_STATUS_ADMIN_STATE_REVOKED; +} + +static bool nfsd4_slots_inuse(struct nfsd4_session *ses, int from) +{ + int i; + + for (i = from; i < ses->se_fchannel.maxreqs; i++) { + struct nfsd4_slot *slot = xa_load(&ses->se_slots, i); + + if (slot->sl_flags & NFSD4_SLOT_INUSE) + return true; + } + return false; +} + +__be32 +nfsd4_sequence(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_sequence *seq = &u->sequence; + struct nfsd4_compoundres *resp = rqstp->rq_resp; + struct xdr_stream *xdr = resp->xdr; + struct nfsd4_session *session; + struct nfs4_client *clp; + struct nfsd4_slot *slot; + struct nfsd4_conn *conn; + __be32 status; + int buflen; + struct net *net = SVC_NET(rqstp); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + if (resp->opcnt != 1) + return nfserr_sequence_pos; + + /* + * Will be either used or freed by nfsd4_sequence_check_conn + * below. + */ + conn = alloc_conn(rqstp, NFS4_CDFC4_FORE); + if (!conn) + return nfserr_jukebox; + + spin_lock(&nn->client_lock); + session = find_in_sessionid_hashtbl(&seq->sessionid, net, &status); + if (!session) + goto out_no_session; + clp = session->se_client; + + status = nfserr_too_many_ops; + if (nfsd4_session_too_many_ops(rqstp, session)) + goto out_put_session; + + status = nfserr_req_too_big; + if (nfsd4_request_too_big(rqstp, session)) + goto out_put_session; + + status = nfserr_badslot; + if (seq->slotid >= session->se_fchannel.maxreqs) + goto out_put_session; + + slot = xa_load(&session->se_slots, seq->slotid); + dprintk("%s: slotid %d\n", __func__, seq->slotid); + + trace_nfsd_slot_seqid_sequence(clp, seq, slot); + + nfsd4_construct_sequence_response(session, seq); + + status = check_slot_seqid(seq->seqid, slot->sl_seqid, slot->sl_flags); + if (status == nfserr_replay_cache) { + status = nfserr_seq_misordered; + if (!(slot->sl_flags & NFSD4_SLOT_INITIALIZED)) + goto out_put_session; + status = nfserr_seq_false_retry; + if (!replay_matches_cache(rqstp, seq, slot)) + goto out_put_session; + cstate->slot = slot; + cstate->session = session; + cstate->clp = clp; + /* Return the cached reply status and set cstate->status + * for nfsd4_proc_compound processing */ + status = nfsd4_replay_cache_entry(resp, seq); + cstate->status = nfserr_replay_cache; + goto out; + } + if (status) + goto out_put_session; + + status = nfsd4_sequence_check_conn(conn, session); + conn = NULL; + if (status) + goto out_put_session; + + if (session->se_target_maxslots < session->se_fchannel.maxreqs && + slot->sl_generation == session->se_slot_gen && + seq->maxslots <= session->se_target_maxslots && + seq->slotid < session->se_target_maxslots && + !nfsd4_slots_inuse(session, session->se_target_maxslots)) + /* Client acknowledged our reduce maxreqs */ + free_session_slots(session, session->se_target_maxslots); + + buflen = (seq->cachethis) ? + session->se_fchannel.maxresp_cached : + session->se_fchannel.maxresp_sz; + status = (seq->cachethis) ? nfserr_rep_too_big_to_cache : + nfserr_rep_too_big; + if (xdr_restrict_buflen(xdr, buflen - rqstp->rq_auth_slack)) + goto out_put_session; + svc_reserve_auth(rqstp, buflen); + + status = nfs_ok; + /* Success! accept new slot seqid */ + slot->sl_seqid = seq->seqid; + slot->sl_flags &= ~NFSD4_SLOT_REUSED; + slot->sl_flags |= NFSD4_SLOT_INUSE; + slot->sl_generation = session->se_slot_gen; + if (seq->cachethis) + slot->sl_flags |= NFSD4_SLOT_CACHETHIS; + else + slot->sl_flags &= ~NFSD4_SLOT_CACHETHIS; + + cstate->slot = slot; + cstate->session = session; + cstate->clp = clp; + + /* + * If the client ever uses the highest available slot, + * gently try to allocate another 20%. This allows + * fairly quick growth without grossly over-shooting what + * the client might use. + * + * Bound that growth by the service's thread ceiling: + * slots beyond the nfsd thread count cannot raise this + * client's throughput, only deepen its backlog. Cap each + * session independently, since a session cannot use + * another's slots; a shared budget would let idle sessions + * pin an active client small. Compare against the + * configured maximum, not the running thread count, so a + * client resuming from idle can grow back before the pool + * scales up. + */ + if (seq->slotid == session->se_fchannel.maxreqs - 1 && + session->se_target_maxslots >= session->se_fchannel.maxreqs) { + int s = session->se_fchannel.maxreqs; + int ceiling = min_t(int, NFSD_MAX_SLOTS_PER_SESSION, + svc_serv_maxthreads(rqstp->rq_server)); + int cnt = min(DIV_ROUND_UP(s, 5), ceiling - s); + void *prev_slot; + + while (cnt-- > 0) { + /* + * GFP_NOWAIT both allows allocation under a + * spinlock, and only succeeds if there is + * plenty of memory. + */ + slot = nfsd4_alloc_slot(&session->se_fchannel, s, + GFP_NOWAIT); + if (!slot) + break; + prev_slot = xa_load(&session->se_slots, s); + if (xa_is_value(prev_slot)) { + slot->sl_seqid = xa_to_value(prev_slot); + slot->sl_flags |= NFSD4_SLOT_REUSED; + } + if (!xa_is_err(xa_store(&session->se_slots, s, slot, + GFP_NOWAIT))) { + s += 1; + session->se_fchannel.maxreqs = s; + atomic_add(s - session->se_target_maxslots, + &nfsd_total_target_slots); + session->se_target_maxslots = s; + } else { + kfree(slot); + break; + } + } + } + +out: + trace_nfsd_seq4_status(rqstp, seq); +out_no_session: + if (conn) + free_conn(conn); + spin_unlock(&nn->client_lock); + return status; +out_put_session: + nfsd4_put_session_locked(session); + goto out_no_session; +} + +void +nfsd4_sequence_done(struct nfsd4_compoundres *resp) +{ + struct nfsd4_compound_state *cs = &resp->cstate; + + if (nfsd4_has_session(cs)) { + if (cs->status != nfserr_replay_cache) { + nfsd4_store_cache_entry(resp); + cs->slot->sl_flags &= ~NFSD4_SLOT_INUSE; + } + /* Drop session reference that was taken in nfsd4_sequence() */ + nfsd4_put_session(cs->session); + } else if (cs->clp) + put_client_renew(cs->clp); +} + +__be32 +nfsd4_destroy_clientid(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_destroy_clientid *dc = &u->destroy_clientid; + struct nfs4_client *conf, *unconf; + struct nfs4_client *clp = NULL; + __be32 status = 0; + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + + spin_lock(&nn->client_lock); + unconf = find_unconfirmed_client(&dc->clientid, true, nn); + conf = find_confirmed_client(&dc->clientid, true, nn); + WARN_ON_ONCE(conf && unconf); + + if (conf) { + if (client_has_state(conf)) { + status = nfserr_clientid_busy; + goto out; + } + status = mark_client_expired_locked(conf); + if (status) + goto out; + clp = conf; + } else if (unconf) + clp = unconf; + else { + status = nfserr_stale_clientid; + goto out; + } + if (!nfsd4_mach_creds_match(clp, rqstp)) { + clp = NULL; + status = nfserr_wrong_cred; + goto out; + } + trace_nfsd_clid_destroyed(&clp->cl_clientid); + unhash_client_locked(clp); +out: + spin_unlock(&nn->client_lock); + if (clp) + expire_client(clp); + return status; +} + +__be32 +nfsd4_reclaim_complete(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, union nfsd4_op_u *u) +{ + struct nfsd4_reclaim_complete *rc = &u->reclaim_complete; + struct nfs4_client *clp = cstate->clp; + __be32 status = 0; + + if (rc->rca_one_fs) { + if (!cstate->current_fh.fh_dentry) + return nfserr_nofilehandle; + /* + * We don't take advantage of the rca_one_fs case. + * That's OK, it's optional, we can safely ignore it. + */ + return nfs_ok; + } + + status = nfserr_complete_already; + if (test_and_set_bit(NFSD4_CLIENT_RECLAIM_COMPLETE, &clp->cl_flags)) + goto out; + + status = nfserr_stale_clientid; + if (is_client_expired(clp)) + /* + * The following error isn't really legal. + * But we only get here if the client just explicitly + * destroyed the client. Surely it no longer cares what + * error it gets back on an operation for the dead + * client. + */ + goto out; + + status = nfs_ok; + trace_nfsd_clid_reclaim_complete(&clp->cl_clientid); + nfsd4_client_record_create(clp); + inc_reclaim_complete(clp); +out: + return status; +} + +__be32 +nfsd4_setclientid(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_setclientid *setclid = &u->setclientid; + struct xdr_netobj clname = setclid->se_name; + nfs4_verifier clverifier = setclid->se_verf; + struct nfs4_client *conf, *new; + struct nfs4_client *unconf = NULL; + __be32 status; + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + + new = create_client(clname, rqstp, &clverifier); + if (new == NULL) + return nfserr_jukebox; + spin_lock(&nn->client_lock); + conf = find_confirmed_client_by_name(&clname, nn); + if (conf && client_has_state(conf)) { + status = nfserr_clid_inuse; + if (clp_used_exchangeid(conf)) + goto out; + if (!same_creds(&conf->cl_cred, &rqstp->rq_cred)) { + trace_nfsd_clid_cred_mismatch(conf, rqstp); + goto out; + } + } + unconf = find_unconfirmed_client_by_name(&clname, nn); + if (unconf) + unhash_client_locked(unconf); + if (conf) { + if (same_verf(&conf->cl_verifier, &clverifier)) { + copy_clid(new, conf); + gen_confirm(new, nn); + } else + trace_nfsd_clid_verf_mismatch(conf, rqstp, + &clverifier); + } else + trace_nfsd_clid_fresh(new); + new->cl_minorversion = 0; + gen_callback(new, setclid, rqstp); + add_to_unconfirmed(new); + setclid->se_clientid.cl_boot = new->cl_clientid.cl_boot; + setclid->se_clientid.cl_id = new->cl_clientid.cl_id; + memcpy(setclid->se_confirm.data, new->cl_confirm.data, sizeof(setclid->se_confirm.data)); + new = NULL; + status = nfs_ok; +out: + spin_unlock(&nn->client_lock); + if (new) + free_client(new); + if (unconf) { + trace_nfsd_clid_expire_unconf(&unconf->cl_clientid); + expire_client(unconf); + } + return status; +} + +__be32 +nfsd4_setclientid_confirm(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_setclientid_confirm *setclientid_confirm = + &u->setclientid_confirm; + struct nfs4_client *conf, *unconf; + struct nfs4_client *old = NULL; + nfs4_verifier confirm = setclientid_confirm->sc_confirm; + clientid_t * clid = &setclientid_confirm->sc_clientid; + __be32 status; + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + + if (STALE_CLIENTID(clid, nn)) + return nfserr_stale_clientid; + + spin_lock(&nn->client_lock); + conf = find_confirmed_client(clid, false, nn); + unconf = find_unconfirmed_client(clid, false, nn); + /* + * We try hard to give out unique clientid's, so if we get an + * attempt to confirm the same clientid with a different cred, + * the client may be buggy; this should never happen. + * + * Nevertheless, RFC 7530 recommends INUSE for this case: + */ + status = nfserr_clid_inuse; + if (unconf && !same_creds(&unconf->cl_cred, &rqstp->rq_cred)) { + trace_nfsd_clid_cred_mismatch(unconf, rqstp); + goto out; + } + if (conf && !same_creds(&conf->cl_cred, &rqstp->rq_cred)) { + trace_nfsd_clid_cred_mismatch(conf, rqstp); + goto out; + } + if (!unconf || !same_verf(&confirm, &unconf->cl_confirm)) { + if (conf && same_verf(&confirm, &conf->cl_confirm)) { + status = nfs_ok; + } else + status = nfserr_stale_clientid; + goto out; + } + status = nfs_ok; + if (conf) { + if (get_client_locked(conf) == nfs_ok) { + old = unconf; + unhash_client_locked(old); + nfsd4_change_callback(conf, &unconf->cl_cb_conn); + } else { + conf = NULL; + } + } + + if (!conf) { + old = find_confirmed_client_by_name(&unconf->cl_name, nn); + if (old) { + status = nfserr_clid_inuse; + if (client_has_state(old) + && !same_creds(&unconf->cl_cred, + &old->cl_cred)) { + old = NULL; + goto out; + } + status = mark_client_expired_locked(old); + if (status) { + old = NULL; + goto out; + } + trace_nfsd_clid_replaced(&old->cl_clientid); + } + status = get_client_locked(unconf); + if (status != nfs_ok) { + old = NULL; + goto out; + } + move_to_confirmed(unconf); + conf = unconf; + } + spin_unlock(&nn->client_lock); + if (conf == unconf) + fsnotify_dentry(conf->cl_nfsd_info_dentry, FS_MODIFY); + nfsd4_probe_callback(conf); + spin_lock(&nn->client_lock); + put_client_renew_locked(conf); +out: + spin_unlock(&nn->client_lock); + if (old) + expire_client(old); + return status; +} + +static struct nfs4_file *nfsd4_alloc_file(void) +{ + return kmem_cache_alloc(file_slab, GFP_KERNEL); +} + +/* OPEN Share state helper functions */ + +static void nfsd4_file_init(const struct svc_fh *fh, struct nfs4_file *fp) +{ + refcount_set(&fp->fi_ref, 1); + spin_lock_init(&fp->fi_lock); + INIT_LIST_HEAD(&fp->fi_stateids); + INIT_LIST_HEAD(&fp->fi_delegations); + INIT_LIST_HEAD(&fp->fi_clnt_odstate); + fh_copy_shallow(&fp->fi_fhandle, &fh->fh_handle); + RCU_INIT_POINTER(fp->fi_deleg_file, NULL); + fp->fi_rdeleg_file = NULL; + fp->fi_had_conflict = false; + fp->fi_share_deny = 0; + memset(fp->fi_fds, 0, sizeof(fp->fi_fds)); + memset(fp->fi_access, 0, sizeof(fp->fi_access)); + fp->fi_aliased = false; + fp->fi_inode = d_inode(fh->fh_dentry); +#ifdef CONFIG_NFSD_PNFS + INIT_LIST_HEAD(&fp->fi_lo_states); + atomic_set(&fp->fi_lo_recalls, 0); +#endif +} + +void +nfsd4_free_slabs(void) +{ + kmem_cache_destroy(client_slab); + kmem_cache_destroy(openowner_slab); + kmem_cache_destroy(lockowner_slab); + kmem_cache_destroy(file_slab); + kmem_cache_destroy(stateid_slab); + kmem_cache_destroy(deleg_slab); + kmem_cache_destroy(odstate_slab); + kmem_cache_destroy(async_copy_slab); +} + +int +nfsd4_init_slabs(void) +{ + client_slab = KMEM_CACHE(nfs4_client, 0); + if (client_slab == NULL) + goto out; + openowner_slab = KMEM_CACHE(nfs4_openowner, 0); + if (openowner_slab == NULL) + goto out_free_client_slab; + lockowner_slab = KMEM_CACHE(nfs4_lockowner, 0); + if (lockowner_slab == NULL) + goto out_free_openowner_slab; + file_slab = KMEM_CACHE(nfs4_file, 0); + if (file_slab == NULL) + goto out_free_lockowner_slab; + stateid_slab = KMEM_CACHE(nfs4_ol_stateid, 0); + if (stateid_slab == NULL) + goto out_free_file_slab; + deleg_slab = KMEM_CACHE(nfs4_delegation, 0); + if (deleg_slab == NULL) + goto out_free_stateid_slab; + odstate_slab = KMEM_CACHE(nfs4_clnt_odstate, 0); + if (odstate_slab == NULL) + goto out_free_deleg_slab; + async_copy_slab = KMEM_CACHE(nfsd4_async_copy, 0); + if (async_copy_slab == NULL) + goto out_free_odstate_slab; + return 0; + +out_free_odstate_slab: + kmem_cache_destroy(odstate_slab); +out_free_deleg_slab: + kmem_cache_destroy(deleg_slab); +out_free_stateid_slab: + kmem_cache_destroy(stateid_slab); +out_free_file_slab: + kmem_cache_destroy(file_slab); +out_free_lockowner_slab: + kmem_cache_destroy(lockowner_slab); +out_free_openowner_slab: + kmem_cache_destroy(openowner_slab); +out_free_client_slab: + kmem_cache_destroy(client_slab); +out: + return -ENOMEM; +} + +static unsigned long +nfsd4_state_shrinker_count(struct shrinker *shrink, struct shrink_control *sc) +{ + struct nfsd_net *nn = shrink->private_data; + long count; + + count = atomic_read(&nn->nfsd_courtesy_clients); + if (!count) + count = atomic_long_read(&num_delegations); + if (count) + queue_work(laundry_wq, &nn->nfsd_shrinker_work); + return (unsigned long)count; +} + +static unsigned long +nfsd4_state_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc) +{ + return SHRINK_STOP; +} + +void +nfsd4_init_leases_net(struct nfsd_net *nn) +{ + struct sysinfo si; + u64 max_clients; + + nn->nfsd4_lease = 90; /* default lease time */ + nn->nfsd4_grace = 90; + nn->clverifier_counter = get_random_u32(); + nn->clientid_base = get_random_u32(); + nn->clientid_counter = nn->clientid_base + 1; + nn->s2s_cp_cl_id = nn->clientid_counter++; + + atomic_set(&nn->nfs4_client_count, 0); + si_meminfo(&si); + max_clients = (u64)si.totalram * si.mem_unit / (1024 * 1024 * 1024); + max_clients *= NFS4_CLIENTS_PER_GB; + nn->nfs4_max_clients = max_t(int, max_clients, NFS4_CLIENTS_PER_GB); + + atomic_set(&nn->nfsd_courtesy_clients, 0); +} + +enum rp_lock { + RP_UNLOCKED, + RP_LOCKED, + RP_UNHASHED, +}; + +static void init_nfs4_replay(struct nfs4_replay *rp) +{ + rp->rp_status = nfserr_serverfault; + rp->rp_buflen = 0; + rp->rp_buf = rp->rp_ibuf; + rp->rp_locked = RP_UNLOCKED; +} + +static int nfsd4_cstate_assign_replay(struct nfsd4_compound_state *cstate, + struct nfs4_stateowner *so) +{ + if (!nfsd4_has_session(cstate)) { + wait_var_event(&so->so_replay.rp_locked, + cmpxchg(&so->so_replay.rp_locked, + RP_UNLOCKED, RP_LOCKED) != RP_LOCKED); + if (so->so_replay.rp_locked == RP_UNHASHED) + return -EAGAIN; + cstate->replay_owner = nfs4_get_stateowner(so); + } + return 0; +} + +void nfsd4_cstate_clear_replay(struct nfsd4_compound_state *cstate) +{ + struct nfs4_stateowner *so = cstate->replay_owner; + + if (so != NULL) { + cstate->replay_owner = NULL; + store_release_wake_up(&so->so_replay.rp_locked, RP_UNLOCKED); + nfs4_put_stateowner(so); + } +} + +static inline void *alloc_stateowner(struct kmem_cache *slab, struct xdr_netobj *owner, struct nfs4_client *clp) +{ + struct nfs4_stateowner *sop; + + sop = kmem_cache_alloc(slab, GFP_KERNEL); + if (!sop) + return NULL; + + xdr_netobj_dup(&sop->so_owner, owner, GFP_KERNEL); + if (!sop->so_owner.data) { + kmem_cache_free(slab, sop); + return NULL; + } + + INIT_LIST_HEAD(&sop->so_stateids); + sop->so_client = clp; + init_nfs4_replay(&sop->so_replay); + atomic_set(&sop->so_count, 1); + return sop; +} + +static void hash_openowner(struct nfs4_openowner *oo, struct nfs4_client *clp, unsigned int strhashval) +{ + lockdep_assert_held(&clp->cl_lock); + + list_add(&oo->oo_owner.so_strhash, + &clp->cl_ownerstr_hashtbl[strhashval]); + list_add(&oo->oo_perclient, &clp->cl_openowners); +} + +static void nfs4_unhash_openowner(struct nfs4_stateowner *so) +{ + unhash_openowner_locked(openowner(so)); +} + +static void nfs4_free_openowner(struct nfs4_stateowner *so) +{ + struct nfs4_openowner *oo = openowner(so); + + kmem_cache_free(openowner_slab, oo); +} + +static const struct nfs4_stateowner_operations openowner_ops = { + .so_unhash = nfs4_unhash_openowner, + .so_free = nfs4_free_openowner, +}; + +static struct nfs4_ol_stateid * +nfsd4_find_existing_open(struct nfs4_file *fp, struct nfsd4_open *open) +{ + struct nfs4_ol_stateid *local, *ret = NULL; + struct nfs4_openowner *oo = open->op_openowner; + + lockdep_assert_held(&fp->fi_lock); + + list_for_each_entry(local, &fp->fi_stateids, st_perfile) { + /* ignore lock owners */ + if (local->st_stateowner->so_is_open_owner == 0) + continue; + if (local->st_stateowner != &oo->oo_owner) + continue; + if (local->st_stid.sc_type == SC_TYPE_OPEN && + !local->st_stid.sc_status) { + ret = local; + refcount_inc(&ret->st_stid.sc_count); + break; + } + } + return ret; +} + +static void nfsd4_drop_revoked_stid(struct nfs4_stid *s) + __releases(&s->sc_client->cl_lock) +{ + struct nfs4_client *cl = s->sc_client; + LIST_HEAD(reaplist); + struct nfs4_layout_stateid *ls; + struct nfs4_ol_stateid *stp; + struct nfs4_delegation *dp; + bool unhashed; + + switch (s->sc_type) { + case SC_TYPE_OPEN: + stp = openlockstateid(s); + if (unhash_open_stateid(stp, &reaplist)) + put_ol_stateid_locked(stp, &reaplist); + spin_unlock(&cl->cl_lock); + free_ol_stateid_reaplist(&reaplist); + break; + case SC_TYPE_LOCK: + stp = openlockstateid(s); + unhashed = unhash_lock_stateid(stp); + spin_unlock(&cl->cl_lock); + if (unhashed) + nfs4_put_stid(s); + break; + case SC_TYPE_DELEG: + dp = delegstateid(s); + list_del_init(&dp->dl_recall_lru); + s->sc_status |= SC_STATUS_FREED; + spin_unlock(&cl->cl_lock); + nfs4_put_stid(s); + break; + case SC_TYPE_LAYOUT: + ls = layoutstateid(s); + list_del_init(&ls->ls_perclnt); + spin_unlock(&cl->cl_lock); + nfs4_put_stid(s); + break; + default: + spin_unlock(&cl->cl_lock); + } +} + +static void nfsd40_drop_revoked_stid(struct nfs4_client *cl, + stateid_t *stid) +{ + /* NFSv4.0 has no way for the client to tell the server + * that it can forget an admin-revoked stateid. + * So we keep it around until the first time that the + * client uses it, and drop it the first time + * nfserr_admin_revoked is returned. + * For v4.1 and later we wait until explicitly told + * to free the stateid. + */ + if (cl->cl_minorversion == 0) { + struct nfs4_stid *st; + + spin_lock(&cl->cl_lock); + st = find_stateid_locked(cl, stid); + if (st) + nfsd4_drop_revoked_stid(st); + else + spin_unlock(&cl->cl_lock); + } +} + +static __be32 +nfsd4_verify_open_stid(struct nfs4_stid *s) +{ + __be32 ret = nfs_ok; + + if (s->sc_status & SC_STATUS_ADMIN_REVOKED) + ret = nfserr_admin_revoked; + else if (s->sc_status & SC_STATUS_REVOKED) + ret = nfserr_deleg_revoked; + else if (s->sc_status & SC_STATUS_CLOSED) + ret = nfserr_bad_stateid; + return ret; +} + +/* Lock the stateid st_mutex, and deal with races with CLOSE */ +static __be32 +nfsd4_lock_ol_stateid(struct nfs4_ol_stateid *stp) +{ + __be32 ret; + + mutex_lock_nested(&stp->st_mutex, LOCK_STATEID_MUTEX); + ret = nfsd4_verify_open_stid(&stp->st_stid); + if (ret == nfserr_admin_revoked) + nfsd40_drop_revoked_stid(stp->st_stid.sc_client, + &stp->st_stid.sc_stateid); + + if (ret != nfs_ok) + mutex_unlock(&stp->st_mutex); + return ret; +} + +static struct nfs4_ol_stateid * +nfsd4_find_and_lock_existing_open(struct nfs4_file *fp, struct nfsd4_open *open) +{ + struct nfs4_ol_stateid *stp; + for (;;) { + spin_lock(&fp->fi_lock); + stp = nfsd4_find_existing_open(fp, open); + spin_unlock(&fp->fi_lock); + if (!stp || nfsd4_lock_ol_stateid(stp) == nfs_ok) + break; + nfs4_put_stid(&stp->st_stid); + } + return stp; +} + +static struct nfs4_openowner * +find_or_alloc_open_stateowner(unsigned int strhashval, struct nfsd4_open *open, + struct nfsd4_compound_state *cstate) +{ + struct nfs4_client *clp = cstate->clp; + struct nfs4_openowner *oo, *new = NULL; + +retry: + spin_lock(&clp->cl_lock); + oo = find_openstateowner_str(strhashval, open, clp); + if (!oo && new) { + hash_openowner(new, clp, strhashval); + spin_unlock(&clp->cl_lock); + return new; + } + spin_unlock(&clp->cl_lock); + + if (oo && !(oo->oo_flags & NFS4_OO_CONFIRMED)) { + /* Replace unconfirmed owners without checking for replay. */ + release_openowner(oo); + oo = NULL; + goto retry; + } + if (oo) { + if (new) + nfs4_free_stateowner(&new->oo_owner); + return oo; + } + + new = alloc_stateowner(openowner_slab, &open->op_owner, clp); + if (!new) + return NULL; + new->oo_owner.so_ops = &openowner_ops; + new->oo_owner.so_is_open_owner = 1; + new->oo_owner.so_seqid = open->op_seqid; + new->oo_flags = 0; + if (nfsd4_has_session(cstate)) + new->oo_flags |= NFS4_OO_CONFIRMED; + new->oo_time = 0; + new->oo_last_closed_stid = NULL; + INIT_LIST_HEAD(&new->oo_close_lru); + goto retry; +} + +static struct nfs4_ol_stateid * +init_open_stateid(struct nfs4_file *fp, struct nfsd4_open *open) +{ + + struct nfs4_openowner *oo = open->op_openowner; + struct nfs4_ol_stateid *retstp = NULL; + struct nfs4_ol_stateid *stp; + + stp = open->op_stp; + /* We are moving these outside of the spinlocks to avoid the warnings */ + mutex_init(&stp->st_mutex); + mutex_lock_nested(&stp->st_mutex, OPEN_STATEID_MUTEX); + +retry: + spin_lock(&oo->oo_owner.so_client->cl_lock); + spin_lock(&fp->fi_lock); + + if (nfs4_openowner_unhashed(oo)) { + mutex_unlock(&stp->st_mutex); + stp = NULL; + goto out_unlock; + } + + retstp = nfsd4_find_existing_open(fp, open); + if (retstp) + goto out_unlock; + + open->op_stp = NULL; + refcount_inc(&stp->st_stid.sc_count); + stp->st_stid.sc_type = SC_TYPE_OPEN; + INIT_LIST_HEAD(&stp->st_locks); + stp->st_stateowner = nfs4_get_stateowner(&oo->oo_owner); + get_nfs4_file(fp); + stp->st_stid.sc_file = fp; + stp->st_access_bmap = 0; + stp->st_deny_bmap = 0; + stp->st_openstp = NULL; + list_add(&stp->st_perstateowner, &oo->oo_owner.so_stateids); + list_add(&stp->st_perfile, &fp->fi_stateids); + +out_unlock: + spin_unlock(&fp->fi_lock); + spin_unlock(&oo->oo_owner.so_client->cl_lock); + if (retstp) { + /* Handle races with CLOSE */ + if (nfsd4_lock_ol_stateid(retstp) != nfs_ok) { + nfs4_put_stid(&retstp->st_stid); + goto retry; + } + /* To keep mutex tracking happy */ + mutex_unlock(&stp->st_mutex); + stp = retstp; + } + return stp; +} + +/* + * In the 4.0 case we need to keep the owners around a little while to handle + * CLOSE replay. We still do need to release any file access that is held by + * them before returning however. + */ +static void +move_to_close_lru(struct nfs4_ol_stateid *s, struct net *net) +{ + struct nfs4_ol_stateid *last; + struct nfs4_openowner *oo = openowner(s->st_stateowner); + struct nfsd_net *nn = net_generic(s->st_stid.sc_client->net, + nfsd_net_id); + + dprintk("NFSD: move_to_close_lru nfs4_openowner %p\n", oo); + + /* + * We know that we hold one reference via nfsd4_close, and another + * "persistent" reference for the client. If the refcount is higher + * than 2, then there are still calls in progress that are using this + * stateid. We can't put the sc_file reference until they are finished. + * Wait for the refcount to drop to 2. Since it has been unhashed, + * there should be no danger of the refcount going back up again at + * this point. + * Some threads with a reference might be waiting for rp_locked, + * so tell them to stop waiting. + */ + store_release_wake_up(&oo->oo_owner.so_replay.rp_locked, RP_UNHASHED); + wait_event(close_wq, refcount_read(&s->st_stid.sc_count) == 2); + + release_all_access(s); + if (s->st_stid.sc_file) { + put_nfs4_file(s->st_stid.sc_file); + s->st_stid.sc_file = NULL; + } + + spin_lock(&nn->client_lock); + last = oo->oo_last_closed_stid; + oo->oo_last_closed_stid = s; + list_move_tail(&oo->oo_close_lru, &nn->close_lru); + oo->oo_time = ktime_get_boottime_seconds(); + spin_unlock(&nn->client_lock); + if (last) + nfs4_put_stid(&last->st_stid); +} + +static noinline_for_stack struct nfs4_file * +nfsd4_file_hash_lookup(const struct svc_fh *fhp) +{ + struct inode *inode = d_inode(fhp->fh_dentry); + struct rhlist_head *tmp, *list; + struct nfs4_file *fi; + + rcu_read_lock(); + list = rhltable_lookup(&nfs4_file_rhltable, &inode, + nfs4_file_rhash_params); + rhl_for_each_entry_rcu(fi, tmp, list, fi_rlist) { + if (fh_match(&fi->fi_fhandle, &fhp->fh_handle)) { + if (refcount_inc_not_zero(&fi->fi_ref)) { + rcu_read_unlock(); + return fi; + } + } + } + rcu_read_unlock(); + return NULL; +} + +/* + * On hash insertion, identify entries with the same inode but + * distinct filehandles. They will all be on the list returned + * by rhltable_lookup(). + * + * inode->i_lock prevents racing insertions from adding an entry + * for the same inode/fhp pair twice. + */ +static noinline_for_stack struct nfs4_file * +nfsd4_file_hash_insert(struct nfs4_file *new, const struct svc_fh *fhp) +{ + struct inode *inode = d_inode(fhp->fh_dentry); + struct rhlist_head *tmp, *list; + struct nfs4_file *ret = NULL; + bool alias_found = false; + struct nfs4_file *fi; + int err; + + rcu_read_lock(); + spin_lock(&inode->i_lock); + + list = rhltable_lookup(&nfs4_file_rhltable, &inode, + nfs4_file_rhash_params); + rhl_for_each_entry_rcu(fi, tmp, list, fi_rlist) { + if (fh_match(&fi->fi_fhandle, &fhp->fh_handle)) { + if (refcount_inc_not_zero(&fi->fi_ref)) + ret = fi; + } else + fi->fi_aliased = alias_found = true; + } + if (ret) + goto out_unlock; + + nfsd4_file_init(fhp, new); + err = rhltable_insert(&nfs4_file_rhltable, &new->fi_rlist, + nfs4_file_rhash_params); + if (err) + goto out_unlock; + + new->fi_aliased = alias_found; + ret = new; + +out_unlock: + spin_unlock(&inode->i_lock); + rcu_read_unlock(); + return ret; +} + +static noinline_for_stack void nfsd4_file_hash_remove(struct nfs4_file *fi) +{ + rhltable_remove(&nfs4_file_rhltable, &fi->fi_rlist, + nfs4_file_rhash_params); +} + +/* + * Called to check deny when READ with all zero stateid or + * WRITE with all zero or all one stateid + */ +static __be32 +nfs4_share_conflict(struct svc_fh *current_fh, unsigned int deny_type) +{ + struct nfs4_file *fp; + __be32 ret = nfs_ok; + + fp = nfsd4_file_hash_lookup(current_fh); + if (!fp) + return ret; + + /* Check for conflicting share reservations */ + spin_lock(&fp->fi_lock); + if (fp->fi_share_deny & deny_type) + ret = nfserr_locked; + spin_unlock(&fp->fi_lock); + put_nfs4_file(fp); + return ret; +} + +static bool nfsd4_deleg_present(const struct inode *inode) +{ + struct file_lock_context *ctx = locks_inode_context(inode); + + return ctx && !list_empty_careful(&ctx->flc_lease); +} + +/** + * nfsd_wait_for_delegreturn - wait for delegations to be returned + * @rqstp: the RPC transaction being executed + * @inode: in-core inode of the file being waited for + * + * The timeout prevents deadlock if all nfsd threads happen to be + * tied up waiting for returning delegations. + * + * Return values: + * %true: delegation was returned + * %false: timed out waiting for delegreturn + */ +bool nfsd_wait_for_delegreturn(struct svc_rqst *rqstp, struct inode *inode) +{ + long __maybe_unused timeo; + + timeo = wait_var_event_timeout(inode, !nfsd4_deleg_present(inode), + NFSD_DELEGRETURN_TIMEOUT); + trace_nfsd_delegret_wakeup(rqstp, inode, timeo); + return timeo > 0; +} + +static bool nfsd4_cb_recall_prepare(struct nfsd4_callback *cb) +{ + struct nfs4_delegation *dp = cb_to_delegation(cb); + struct nfsd_net *nn = net_generic(dp->dl_stid.sc_client->net, + nfsd_net_id); + + block_delegations(&dp->dl_stid.sc_file->fi_fhandle); + + /* + * We can't do this in nfsd_break_deleg_cb because it is + * already holding inode->i_lock. + * + * If the dl_time != 0, then we know that it has already been + * queued for a lease break. Don't queue it again. + */ + spin_lock(&nn->deleg_lock); + if (delegation_hashed(dp) && dp->dl_time == 0) { + dp->dl_time = ktime_get_boottime_seconds(); + list_add_tail(&dp->dl_recall_lru, &nn->del_recall_lru); + } + spin_unlock(&nn->deleg_lock); + return true; +} + +static int nfsd4_cb_recall_done(struct nfsd4_callback *cb, + struct rpc_task *task) +{ + struct nfs4_delegation *dp = cb_to_delegation(cb); + + trace_nfsd_cb_recall_done(&dp->dl_stid.sc_stateid, task); + + if (dp->dl_stid.sc_status) + /* CLOSED or REVOKED */ + return 1; + + switch (task->tk_status) { + case 0: + return 1; + case -NFS4ERR_DELAY: + rpc_delay(task, 2 * HZ); + return 0; + case -EBADHANDLE: + case -NFS4ERR_BAD_STATEID: + /* + * Race: client probably got cb_recall before open reply + * granting delegation. + */ + if (dp->dl_retries--) { + rpc_delay(task, 2 * HZ); + return 0; + } + fallthrough; + default: + return 1; + } +} + +static void nfsd4_cb_recall_release(struct nfsd4_callback *cb) +{ + struct nfs4_delegation *dp = cb_to_delegation(cb); + + nfs4_put_stid(&dp->dl_stid); +} + +static const struct nfsd4_callback_ops nfsd4_cb_recall_ops = { + .prepare = nfsd4_cb_recall_prepare, + .done = nfsd4_cb_recall_done, + .release = nfsd4_cb_recall_release, + .opcode = OP_CB_RECALL, +}; + +/* Called from break_lease() with flc_lock held. */ +static bool +nfsd_break_deleg_cb(struct file_lease *fl) +{ + struct nfs4_delegation *dp = (struct nfs4_delegation *) fl->c.flc_owner; + struct nfs4_file *fp = dp->dl_stid.sc_file; + struct nfs4_client *clp = dp->dl_stid.sc_client; + struct nfsd_net *nn; + + trace_nfsd_cb_recall(&dp->dl_stid); + + dp->dl_recalled = true; + atomic_inc(&clp->cl_delegs_in_recall); + if (try_to_expire_client(clp)) { + nn = net_generic(clp->net, nfsd_net_id); + mod_delayed_work(laundry_wq, &nn->laundromat_work, 0); + } + + /* + * We don't want the locks code to timeout the lease for us; + * we'll remove it ourself if a delegation isn't returned + * in time: + */ + fl->fl_break_time = 0; + + fp->fi_had_conflict = true; + nfsd_break_one_deleg(dp); + return false; +} + +/** + * nfsd_breaker_owns_lease - Check if lease conflict was resolved + * @fl: Lock state to check + * + * Return values: + * %true: Lease conflict was resolved + * %false: Lease conflict was not resolved. + */ +static bool nfsd_breaker_owns_lease(struct file_lease *fl) +{ + struct nfs4_delegation *dl = fl->c.flc_owner; + struct nfsd_thread_local_info *ntli; + struct svc_rqst *rqst; + struct nfs4_client *clp; + + /* Only nfsd leases */ + if (fl->fl_lmops != &nfsd_lease_mng_ops) + return false; + + rqst = nfsd_current_rqst(); + if (!nfsd_v4client(rqst)) + return false; + ntli = rqst->rq_private; + clp = *ntli->ntli_lease_breaker; + return dl->dl_stid.sc_client == clp; +} + +static int +nfsd_change_deleg_cb(struct file_lease *onlist, int arg, + struct list_head *dispose) +{ + struct nfs4_delegation *dp = (struct nfs4_delegation *) onlist->c.flc_owner; + struct nfs4_client *clp = dp->dl_stid.sc_client; + + if (arg & F_UNLCK) { + if (dp->dl_recalled) + atomic_dec(&clp->cl_delegs_in_recall); + return lease_modify(onlist, arg, dispose); + } else + return -EAGAIN; +} + +/** + * nfsd4_deleg_lm_open_conflict - see if the given file points to an inode that has + * an existing open that would conflict with the + * desired lease. + * @filp: file to check + * @arg: type of lease that we're trying to acquire + * + * The kernel will call into this operation to determine whether there + * are conflicting opens that may prevent the deleg from being granted. + * For nfsd, that check is done at a higher level, so this trivially + * returns 0. + */ +static int +nfsd4_deleg_lm_open_conflict(struct file *filp, int arg) +{ + return 0; +} + +static const struct lease_manager_operations nfsd_lease_mng_ops = { + .lm_breaker_owns_lease = nfsd_breaker_owns_lease, + .lm_break = nfsd_break_deleg_cb, + .lm_change = nfsd_change_deleg_cb, + .lm_open_conflict = nfsd4_deleg_lm_open_conflict, +}; + +static __be32 nfsd4_check_seqid(struct nfsd4_compound_state *cstate, struct nfs4_stateowner *so, u32 seqid) +{ + if (nfsd4_has_session(cstate)) + return nfs_ok; + if (seqid == so->so_seqid - 1) + return nfserr_replay_me; + if (seqid == so->so_seqid) + return nfs_ok; + return nfserr_bad_seqid; +} + +static struct nfs4_client *lookup_clientid(clientid_t *clid, bool sessions, + struct nfsd_net *nn) +{ + struct nfs4_client *found; + + spin_lock(&nn->client_lock); + found = find_confirmed_client(clid, sessions, nn); + if (found) + atomic_inc(&found->cl_rpc_users); + spin_unlock(&nn->client_lock); + return found; +} + +static __be32 set_client(clientid_t *clid, + struct nfsd4_compound_state *cstate, + struct nfsd_net *nn) +{ + if (cstate->clp) { + if (!same_clid(&cstate->clp->cl_clientid, clid)) + return nfserr_stale_clientid; + return nfs_ok; + } + if (STALE_CLIENTID(clid, nn)) + return nfserr_stale_clientid; + /* + * We're in the 4.0 case (otherwise the SEQUENCE op would have + * set cstate->clp), so session = false: + */ + cstate->clp = lookup_clientid(clid, false, nn); + if (!cstate->clp) + return nfserr_expired; + return nfs_ok; +} + +__be32 +nfsd4_process_open1(struct nfsd4_compound_state *cstate, + struct nfsd4_open *open, struct nfsd_net *nn) +{ + clientid_t *clientid = &open->op_clientid; + struct nfs4_client *clp = NULL; + unsigned int strhashval; + struct nfs4_openowner *oo = NULL; + __be32 status; + + /* + * In case we need it later, after we've already created the + * file and don't want to risk a further failure: + */ + open->op_file = nfsd4_alloc_file(); + if (open->op_file == NULL) + return nfserr_jukebox; + + status = set_client(clientid, cstate, nn); + if (status) + return status; + clp = cstate->clp; + + strhashval = ownerstr_hashval(&open->op_owner); +retry: + oo = find_or_alloc_open_stateowner(strhashval, open, cstate); + open->op_openowner = oo; + if (!oo) + return nfserr_jukebox; + if (nfsd4_cstate_assign_replay(cstate, &oo->oo_owner) == -EAGAIN) { + nfs4_put_stateowner(&oo->oo_owner); + goto retry; + } + status = nfsd4_check_seqid(cstate, &oo->oo_owner, open->op_seqid); + if (status) + return status; + + open->op_stp = nfs4_alloc_open_stateid(clp); + if (!open->op_stp) + return nfserr_jukebox; + + if (nfsd4_has_session(cstate) && + (cstate->current_fh.fh_export->ex_flags & NFSEXP_PNFS)) { + open->op_odstate = alloc_clnt_odstate(clp); + if (!open->op_odstate) + return nfserr_jukebox; + } + + return nfs_ok; +} + +static inline __be32 +nfs4_check_delegmode(struct nfs4_delegation *dp, int flags) +{ + if (!(flags & RD_STATE) && deleg_is_read(dp->dl_type)) + return nfserr_openmode; + else + return nfs_ok; +} + +static int share_access_to_flags(u32 share_access) +{ + return share_access == NFS4_SHARE_ACCESS_READ ? RD_STATE : WR_STATE; +} + +static struct nfs4_delegation *find_deleg_stateid(struct nfs4_client *cl, + stateid_t *s) +{ + struct nfs4_stid *ret; + + ret = find_stateid_by_type(cl, s, SC_TYPE_DELEG, SC_STATUS_REVOKED); + if (!ret) + return NULL; + return delegstateid(ret); +} + +static bool nfsd4_is_deleg_cur(struct nfsd4_open *open) +{ + return open->op_claim_type == NFS4_OPEN_CLAIM_DELEGATE_CUR || + open->op_claim_type == NFS4_OPEN_CLAIM_DELEG_CUR_FH; +} + +static __be32 +nfs4_check_deleg(struct nfs4_client *cl, struct nfsd4_open *open, + struct nfs4_delegation **dp) +{ + int flags; + __be32 status = nfserr_bad_stateid; + struct nfs4_delegation *deleg; + + deleg = find_deleg_stateid(cl, &open->op_delegate_stateid); + if (deleg == NULL) + goto out; + if (deleg->dl_stid.sc_status & SC_STATUS_ADMIN_REVOKED) { + nfs4_put_stid(&deleg->dl_stid); + status = nfserr_admin_revoked; + goto out; + } + if (deleg->dl_stid.sc_status & SC_STATUS_REVOKED) { + nfs4_put_stid(&deleg->dl_stid); + nfsd40_drop_revoked_stid(cl, &open->op_delegate_stateid); + status = nfserr_deleg_revoked; + goto out; + } + flags = share_access_to_flags(open->op_share_access); + status = nfs4_check_delegmode(deleg, flags); + if (status) { + nfs4_put_stid(&deleg->dl_stid); + goto out; + } + *dp = deleg; +out: + if (!nfsd4_is_deleg_cur(open)) + return nfs_ok; + if (status) + return status; + open->op_openowner->oo_flags |= NFS4_OO_CONFIRMED; + return nfs_ok; +} + +static inline int nfs4_access_to_access(u32 nfs4_access) +{ + int flags = 0; + + if (nfs4_access & NFS4_SHARE_ACCESS_READ) + flags |= NFSD_MAY_READ; + if (nfs4_access & NFS4_SHARE_ACCESS_WRITE) + flags |= NFSD_MAY_WRITE; + return flags; +} + +static inline __be32 +nfsd4_truncate(struct svc_rqst *rqstp, struct svc_fh *fh, + struct nfsd4_open *open) +{ + struct iattr iattr = { + .ia_valid = ATTR_SIZE, + .ia_size = 0, + }; + struct nfsd_attrs attrs = { + .na_iattr = &iattr, + }; + if (!open->op_truncate) + return 0; + if (!(open->op_share_access & NFS4_SHARE_ACCESS_WRITE)) + return nfserr_inval; + return nfsd_setattr(rqstp, fh, &attrs, NULL); +} + +static __be32 nfs4_get_vfs_file(struct svc_rqst *rqstp, struct nfs4_file *fp, + struct svc_fh *cur_fh, struct nfs4_ol_stateid *stp, + struct nfsd4_open *open, bool new_stp) +{ + struct nfsd_file *nf = NULL; + __be32 status; + int oflag = nfs4_access_to_omode(open->op_share_access); + int access = nfs4_access_to_access(open->op_share_access); + unsigned char old_access_bmap, old_deny_bmap; + + spin_lock(&fp->fi_lock); + + /* + * Are we trying to set a deny mode that would conflict with + * current access? + */ + status = nfs4_file_check_deny(fp, open->op_share_deny); + if (status != nfs_ok) { + if (status != nfserr_share_denied) { + spin_unlock(&fp->fi_lock); + goto out; + } + if (nfs4_resolve_deny_conflicts_locked(fp, new_stp, + stp, open->op_share_deny, false)) + status = nfserr_jukebox; + spin_unlock(&fp->fi_lock); + goto out; + } + + /* set access to the file */ + status = nfs4_file_get_access(fp, open->op_share_access); + if (status != nfs_ok) { + if (status != nfserr_share_denied) { + spin_unlock(&fp->fi_lock); + goto out; + } + if (nfs4_resolve_deny_conflicts_locked(fp, new_stp, + stp, open->op_share_access, true)) + status = nfserr_jukebox; + spin_unlock(&fp->fi_lock); + goto out; + } + + /* Set access bits in stateid */ + old_access_bmap = stp->st_access_bmap; + set_access(open->op_share_access, stp); + + /* Set new deny mask */ + old_deny_bmap = stp->st_deny_bmap; + set_deny(open->op_share_deny, stp); + fp->fi_share_deny |= (open->op_share_deny & NFS4_SHARE_DENY_BOTH); + + if (!fp->fi_fds[oflag]) { + spin_unlock(&fp->fi_lock); + + status = nfsd_file_acquire_opened(rqstp, cur_fh, access, + open->op_filp, &nf); + if (status != nfs_ok) + goto out_put_access; + + spin_lock(&fp->fi_lock); + if (!fp->fi_fds[oflag]) { + fp->fi_fds[oflag] = nf; + nf = NULL; + } + } + spin_unlock(&fp->fi_lock); + if (nf) + nfsd_file_put(nf); + + status = nfserrno(nfsd_open_break_lease(cur_fh->fh_dentry->d_inode, + access)); + if (status) + goto out_put_access; + + status = nfsd4_truncate(rqstp, cur_fh, open); + if (status) + goto out_put_access; +out: + return status; +out_put_access: + stp->st_access_bmap = old_access_bmap; + nfs4_file_put_access(fp, open->op_share_access); + reset_union_bmap_deny(bmap_to_share_mode(old_deny_bmap), stp); + goto out; +} + +static __be32 +nfs4_upgrade_open(struct svc_rqst *rqstp, struct nfs4_file *fp, + struct svc_fh *cur_fh, struct nfs4_ol_stateid *stp, + struct nfsd4_open *open) +{ + __be32 status; + unsigned char old_deny_bmap = stp->st_deny_bmap; + + if (!test_access(open->op_share_access, stp)) + return nfs4_get_vfs_file(rqstp, fp, cur_fh, stp, open, false); + + /* test and set deny mode */ + spin_lock(&fp->fi_lock); + status = nfs4_file_check_deny(fp, open->op_share_deny); + switch (status) { + case nfs_ok: + set_deny(open->op_share_deny, stp); + fp->fi_share_deny |= + (open->op_share_deny & NFS4_SHARE_DENY_BOTH); + break; + case nfserr_share_denied: + if (nfs4_resolve_deny_conflicts_locked(fp, false, + stp, open->op_share_deny, false)) + status = nfserr_jukebox; + break; + } + spin_unlock(&fp->fi_lock); + + if (status != nfs_ok) + return status; + + status = nfsd4_truncate(rqstp, cur_fh, open); + if (status != nfs_ok) + reset_union_bmap_deny(old_deny_bmap, stp); + return status; +} + +/* Should we give out recallable state?: */ +static bool nfsd4_cb_channel_good(struct nfs4_client *clp) +{ + if (clp->cl_cb_state == NFSD4_CB_UP) + return true; + /* + * In the sessions case, since we don't have to establish a + * separate connection for callbacks, we assume it's OK + * until we hear otherwise: + */ + return clp->cl_minorversion && clp->cl_cb_state == NFSD4_CB_UNKNOWN; +} + +static unsigned int +nfsd_notify_to_ignore(u32 notify) +{ + unsigned int mask = 0; + + if (notify & BIT(NOTIFY4_REMOVE_ENTRY)) + mask |= FL_IGN_DIR_DELETE; + if (notify & BIT(NOTIFY4_ADD_ENTRY)) + mask |= FL_IGN_DIR_CREATE; + if (notify & BIT(NOTIFY4_RENAME_ENTRY)) + mask |= FL_IGN_DIR_RENAME; + + return mask; +} + +static struct file_lease *nfs4_alloc_init_lease(struct nfs4_delegation *dp, u32 notify) +{ + struct file_lease *fl; + + fl = locks_alloc_lease(); + if (!fl) + return NULL; + fl->fl_lmops = &nfsd_lease_mng_ops; + fl->c.flc_flags = FL_DELEG | nfsd_notify_to_ignore(notify); + fl->c.flc_type = deleg_is_read(dp->dl_type) ? F_RDLCK : F_WRLCK; + fl->c.flc_owner = (fl_owner_t)dp; + fl->c.flc_pid = current->tgid; + fl->c.flc_file = rcu_dereference_protected(dp->dl_stid.sc_file->fi_deleg_file, 1)->nf_file; + return fl; +} + +static int nfsd4_check_conflicting_opens(struct nfs4_client *clp, + struct nfs4_file *fp) +{ + struct nfs4_ol_stateid *st; + struct file *f = rcu_dereference_protected(fp->fi_deleg_file, 1)->nf_file; + struct inode *ino = file_inode(f); + int writes; + + writes = atomic_read(&ino->i_writecount); + if (!writes) + return 0; + /* + * There could be multiple filehandles (hence multiple + * nfs4_files) referencing this file, but that's not too + * common; let's just give up in that case rather than + * trying to go look up all the clients using that other + * nfs4_file as well: + */ + if (fp->fi_aliased) + return -EAGAIN; + /* + * If there's a close in progress, make sure that we see it + * clear any fi_fds[] entries before we see it decrement + * i_writecount: + */ + smp_mb__after_atomic(); + + if (fp->fi_fds[O_WRONLY]) + writes--; + if (fp->fi_fds[O_RDWR]) + writes--; + if (writes > 0) + return -EAGAIN; /* There may be non-NFSv4 writers */ + /* + * It's possible there are non-NFSv4 write opens in progress, + * but if they haven't incremented i_writecount yet then they + * also haven't called break lease yet; so, they'll break this + * lease soon enough. So, all that's left to check for is NFSv4 + * opens: + */ + spin_lock(&fp->fi_lock); + list_for_each_entry(st, &fp->fi_stateids, st_perfile) { + if (st->st_openstp == NULL /* it's an open */ && + access_permit_write(st) && + st->st_stid.sc_client != clp) { + spin_unlock(&fp->fi_lock); + return -EAGAIN; + } + } + spin_unlock(&fp->fi_lock); + /* + * There's a small chance that we could be racing with another + * NFSv4 open. However, any open that hasn't added itself to + * the fi_stateids list also hasn't called break_lease yet; so, + * they'll break this lease soon enough. + */ + return 0; +} + +/* + * It's possible that between opening the dentry and setting the delegation, + * that it has been renamed or unlinked. Redo the lookup to verify that this + * hasn't happened. + */ +static int +nfsd4_verify_deleg_dentry(struct nfsd4_open *open, struct nfs4_file *fp, + struct svc_fh *parent) +{ + struct svc_export *exp; + struct dentry *child; + __be32 err; + + err = nfsd_lookup_dentry(open->op_rqstp, parent, + open->op_fname, open->op_fnamelen, + &exp, &child); + + if (err) + return -EAGAIN; + + exp_put(exp); + dput(child); + if (child != file_dentry(rcu_dereference_protected(fp->fi_deleg_file, 1)->nf_file)) + return -EAGAIN; + + return 0; +} + +/* + * We avoid breaking delegations held by a client due to its own activity, but + * clearing setuid/setgid bits on a write is an implicit activity and the client + * may not notice and continue using the old mode. Avoid giving out a delegation + * on setuid/setgid files when the client is requesting an open for write. + */ +static int +nfsd4_verify_setuid_write(struct nfsd4_open *open, struct nfsd_file *nf) +{ + struct inode *inode = file_inode(nf->nf_file); + + if ((open->op_share_access & NFS4_SHARE_ACCESS_WRITE) && + (inode->i_mode & (S_ISUID|S_ISGID))) + return -EAGAIN; + return 0; +} + +/* + * Timestamp delegation was introduced in RFC7862. Runtime switch for disabling + * this feature is /sys/kernel/debug/nfsd/delegated_timestamps. + */ +static bool nfsd4_want_deleg_timestamps(const struct nfsd4_open *open) +{ + if (!nfsd_delegts_enabled) + return false; + return open->op_deleg_want & OPEN4_SHARE_ACCESS_WANT_DELEG_TIMESTAMPS; +} + +static struct nfs4_delegation * +nfs4_set_delegation(struct nfsd4_open *open, struct nfs4_ol_stateid *stp, + struct svc_fh *parent) +{ + bool deleg_ts = nfsd4_want_deleg_timestamps(open); + struct nfs4_client *clp = stp->st_stid.sc_client; + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + struct nfs4_file *fp = stp->st_stid.sc_file; + struct nfs4_clnt_odstate *odstate = stp->st_clnt_odstate; + struct nfs4_delegation *dp; + struct nfsd_file *nf = NULL; + struct file_lease *fl; + int status = 0; + u32 dl_type; + + /* + * The fi_had_conflict and nfs_get_existing_delegation checks + * here are just optimizations; we'll need to recheck them at + * the end: + */ + if (fp->fi_had_conflict) + return ERR_PTR(-EAGAIN); + + /* + * Try for a write delegation first. RFC8881 section 10.4 says: + * + * "An OPEN_DELEGATE_WRITE delegation allows the client to handle, + * on its own, all opens." + * + * Furthermore, section 9.1.2 says: + * + * "In the case of READ, the server may perform the corresponding + * check on the access mode, or it may choose to allow READ for + * OPEN4_SHARE_ACCESS_WRITE, to accommodate clients whose WRITE + * implementation may unavoidably do reads (e.g., due to buffer + * cache constraints)." + * + * We choose to offer a write delegation for OPEN with the + * OPEN4_SHARE_ACCESS_WRITE access mode to accommodate such clients. + */ + if (open->op_share_access & NFS4_SHARE_ACCESS_WRITE) { + nf = find_writeable_file(fp); + dl_type = deleg_ts ? OPEN_DELEGATE_WRITE_ATTRS_DELEG : OPEN_DELEGATE_WRITE; + } + + /* + * If the file is being opened O_RDONLY or we couldn't get a O_RDWR + * file for some reason, then try for a read delegation instead. + */ + if (!nf && (open->op_share_access & NFS4_SHARE_ACCESS_READ)) { + nf = find_readable_file(fp); + dl_type = deleg_ts ? OPEN_DELEGATE_READ_ATTRS_DELEG : OPEN_DELEGATE_READ; + } + + if (!nf) + return ERR_PTR(-EAGAIN); + + /* + * File delegations and associated locks cannot be recovered if the + * export is from an NFS proxy server. + */ + if (exportfs_cannot_lock(nf->nf_file->f_path.mnt->mnt_sb->s_export_op)) { + nfsd_file_put(nf); + return ERR_PTR(-EOPNOTSUPP); + } + + spin_lock(&nn->deleg_lock); + spin_lock(&fp->fi_lock); + if (nfs4_delegation_exists(clp, fp)) + status = -EAGAIN; + else if (nfsd4_verify_setuid_write(open, nf)) + status = -EAGAIN; + else if (!rcu_dereference_protected(fp->fi_deleg_file, + lockdep_is_held(&fp->fi_lock))) { + rcu_assign_pointer(fp->fi_deleg_file, nf); + /* increment early to prevent fi_deleg_file from being + * cleared */ + fp->fi_delegees = 1; + nf = NULL; + } else + fp->fi_delegees++; + spin_unlock(&fp->fi_lock); + spin_unlock(&nn->deleg_lock); + if (nf) + nfsd_file_put(nf); + if (status) + return ERR_PTR(status); + + status = -ENOMEM; + dp = alloc_init_deleg(clp, fp, odstate, dl_type); + if (!dp) + goto out_delegees; + if (stp->st_stid.sc_export) + dp->dl_stid.sc_export = exp_get(stp->st_stid.sc_export); + + fl = nfs4_alloc_init_lease(dp, 0); + if (!fl) + goto out_clnt_odstate; + + status = kernel_setlease(rcu_dereference_protected(fp->fi_deleg_file, 1)->nf_file, + fl->c.flc_type, &fl, NULL); + if (fl) + locks_free_lease(fl); + if (status) + goto out_clnt_odstate; + + if (parent) { + status = nfsd4_verify_deleg_dentry(open, fp, parent); + if (status) + goto out_unlock; + } + + status = nfsd4_check_conflicting_opens(clp, fp); + if (status) + goto out_unlock; + + /* + * Now that the deleg is set, check again to ensure that nothing + * raced in and changed the mode while we weren't looking. + */ + status = nfsd4_verify_setuid_write(open, rcu_dereference_protected(fp->fi_deleg_file, 1)); + if (status) + goto out_unlock; + + status = -EAGAIN; + if (fp->fi_had_conflict) + goto out_unlock; + + spin_lock(&nn->deleg_lock); + spin_lock(&clp->cl_lock); + spin_lock(&fp->fi_lock); + status = hash_delegation_locked(dp, fp); + spin_unlock(&fp->fi_lock); + spin_unlock(&clp->cl_lock); + spin_unlock(&nn->deleg_lock); + + if (status) + goto out_unlock; + + return dp; +out_unlock: + kernel_setlease(rcu_dereference_protected(fp->fi_deleg_file, 1)->nf_file, + F_UNLCK, NULL, (void **)&dp); +out_clnt_odstate: + put_clnt_odstate(dp->dl_clnt_odstate); + nfs4_put_stid(&dp->dl_stid); +out_delegees: + put_deleg_file(fp); + return ERR_PTR(status); +} + +static void nfsd4_open_deleg_none_ext(struct nfsd4_open *open, int status) +{ + open->op_delegate_type = OPEN_DELEGATE_NONE_EXT; + if (status == -EAGAIN) + open->op_why_no_deleg = WND4_CONTENTION; + else { + open->op_why_no_deleg = WND4_RESOURCE; + switch (open->op_deleg_want) { + case OPEN4_SHARE_ACCESS_WANT_READ_DELEG: + case OPEN4_SHARE_ACCESS_WANT_WRITE_DELEG: + case OPEN4_SHARE_ACCESS_WANT_ANY_DELEG: + break; + case OPEN4_SHARE_ACCESS_WANT_CANCEL: + open->op_why_no_deleg = WND4_CANCELLED; + break; + case OPEN4_SHARE_ACCESS_WANT_NO_DELEG: + WARN_ON_ONCE(1); + } + } +} + +static bool +nfs4_delegation_stat(struct nfs4_delegation *dp, struct svc_fh *currentfh, + struct kstat *stat) +{ + struct nfsd_file *nf = find_writeable_file(dp->dl_stid.sc_file); + struct path path; + int rc; + + if (!nf) + return false; + + path.mnt = currentfh->fh_export->ex_path.mnt; + path.dentry = file_dentry(nf->nf_file); + + rc = vfs_getattr(&path, stat, + STATX_MODE | STATX_SIZE | STATX_ATIME | + STATX_MTIME | STATX_CTIME | STATX_CHANGE_COOKIE, + AT_STATX_SYNC_AS_STAT); + + nfsd_file_put(nf); + return rc == 0; +} + +/* + * Add NFS4_SHARE_ACCESS_READ to the write delegation granted on OPEN + * with NFS4_SHARE_ACCESS_WRITE by allocating separate nfsd_file and + * struct file to be used for read with delegation stateid. + * + */ +static bool +nfsd4_add_rdaccess_to_wrdeleg(struct svc_rqst *rqstp, struct nfsd4_open *open, + struct svc_fh *fh, struct nfs4_ol_stateid *stp) +{ + struct nfs4_file *fp; + struct nfsd_file *nf = NULL; + + if ((open->op_share_access & NFS4_SHARE_ACCESS_BOTH) == + NFS4_SHARE_ACCESS_WRITE) { + if (nfsd_file_acquire_opened(rqstp, fh, NFSD_MAY_READ, NULL, &nf)) + return (false); + fp = stp->st_stid.sc_file; + spin_lock(&fp->fi_lock); + if (!fp->fi_fds[O_RDONLY]) { + __nfs4_file_get_access(fp, NFS4_SHARE_ACCESS_READ); + fp->fi_fds[O_RDONLY] = nf; + fp->fi_rdeleg_file = nfsd_file_get(fp->fi_fds[O_RDONLY]); + nf = NULL; + } + spin_unlock(&fp->fi_lock); + if (nf) + nfsd_file_put(nf); + } + return true; +} + +/* + * The Linux NFS server does not offer write delegations to NFSv4.0 + * clients in order to avoid conflicts between write delegations and + * GETATTRs requesting CHANGE or SIZE attributes. + * + * With NFSv4.1 and later minorversions, the SEQUENCE operation that + * begins each COMPOUND contains a client ID. Delegation recall can + * be avoided when the server recognizes the client sending a + * GETATTR also holds write delegation it conflicts with. + * + * However, the NFSv4.0 protocol does not enable a server to + * determine that a GETATTR originated from the client holding the + * conflicting delegation versus coming from some other client. Per + * RFC 7530 Section 16.7.5, the server must recall or send a + * CB_GETATTR even when the GETATTR originates from the client that + * holds the conflicting delegation. + * + * An NFSv4.0 client can trigger a pathological situation if it + * always sends a DELEGRETURN preceded by a conflicting GETATTR in + * the same COMPOUND. COMPOUND execution will always stop at the + * GETATTR and the DELEGRETURN will never get executed. The server + * eventually revokes the delegation, which can result in loss of + * open or lock state. + */ +static void +nfs4_open_delegation(struct svc_rqst *rqstp, struct nfsd4_open *open, + struct nfs4_ol_stateid *stp, struct svc_fh *currentfh, + struct svc_fh *fh) +{ + struct nfs4_openowner *oo = openowner(stp->st_stateowner); + bool deleg_ts = nfsd4_want_deleg_timestamps(open); + struct nfs4_client *clp = stp->st_stid.sc_client; + struct svc_fh *parent = NULL; + struct nfs4_delegation *dp; + struct kstat stat; + int status = 0; + int cb_up; + + cb_up = nfsd4_cb_channel_good(oo->oo_owner.so_client); + open->op_recall = false; + switch (open->op_claim_type) { + case NFS4_OPEN_CLAIM_PREVIOUS: + if (!cb_up) + open->op_recall = true; + break; + case NFS4_OPEN_CLAIM_NULL: + parent = currentfh; + fallthrough; + case NFS4_OPEN_CLAIM_FH: + /* + * Let's not give out any delegations till everyone's + * had the chance to reclaim theirs, *and* until + * NLM locks have all been reclaimed: + */ + if (locks_in_grace(clp->net)) + goto out_no_deleg; + if (!cb_up || !(oo->oo_flags & NFS4_OO_CONFIRMED)) + goto out_no_deleg; + if (open->op_share_access & NFS4_SHARE_ACCESS_WRITE && + !clp->cl_minorversion) + goto out_no_deleg; + break; + default: + goto out_no_deleg; + } + dp = nfs4_set_delegation(open, stp, parent); + if (IS_ERR(dp)) + goto out_no_deleg; + + memcpy(&open->op_delegate_stateid, &dp->dl_stid.sc_stateid, sizeof(dp->dl_stid.sc_stateid)); + + if (open->op_share_access & NFS4_SHARE_ACCESS_WRITE) { + struct file *f; + + f = rcu_dereference_protected(dp->dl_stid.sc_file->fi_deleg_file, 1)->nf_file; + if (!nfsd4_add_rdaccess_to_wrdeleg(rqstp, open, fh, stp) || + !nfs4_delegation_stat(dp, currentfh, &stat)) { + nfs4_put_stid(&dp->dl_stid); + destroy_delegation(dp); + goto out_no_deleg; + } + open->op_delegate_type = deleg_ts ? OPEN_DELEGATE_WRITE_ATTRS_DELEG : + OPEN_DELEGATE_WRITE; + dp->dl_cb_fattr.ncf_initial_cinfo = nfsd4_change_attribute(&stat); + dp->dl_atime = stat.atime; + dp->dl_ctime = stat.ctime; + dp->dl_mtime = stat.mtime; + spin_lock(&f->f_lock); + if (deleg_ts) + f->f_mode |= FMODE_NOCMTIME; + spin_unlock(&f->f_lock); + trace_nfsd_deleg_write(&dp->dl_stid.sc_stateid); + } else { + open->op_delegate_type = deleg_ts && nfs4_delegation_stat(dp, currentfh, &stat) ? + OPEN_DELEGATE_READ_ATTRS_DELEG : OPEN_DELEGATE_READ; + dp->dl_atime = stat.atime; + trace_nfsd_deleg_read(&dp->dl_stid.sc_stateid); + } + nfs4_put_stid(&dp->dl_stid); + return; +out_no_deleg: + open->op_delegate_type = OPEN_DELEGATE_NONE; + + /* 4.1 client asking for a delegation? */ + if (open->op_deleg_want) + nfsd4_open_deleg_none_ext(open, status); + return; +} + +static void nfsd4_deleg_xgrade_none_ext(struct nfsd4_open *open, + struct nfs4_delegation *dp) +{ + if (deleg_is_write(dp->dl_type)) { + if (open->op_deleg_want & OPEN4_SHARE_ACCESS_WANT_READ_DELEG) { + open->op_delegate_type = OPEN_DELEGATE_NONE_EXT; + open->op_why_no_deleg = WND4_NOT_SUPP_DOWNGRADE; + } else if (open->op_deleg_want & OPEN4_SHARE_ACCESS_WANT_WRITE_DELEG) { + open->op_delegate_type = OPEN_DELEGATE_NONE_EXT; + open->op_why_no_deleg = WND4_NOT_SUPP_UPGRADE; + } + } + /* Otherwise the client must be confused wanting a delegation + * it already has, therefore we don't return + * OPEN_DELEGATE_NONE_EXT and reason. + */ +} + +/* Are we returning only a delegation stateid? */ +static bool open_xor_delegation(struct nfsd4_open *open) +{ + if (!(open->op_deleg_want & OPEN4_SHARE_ACCESS_WANT_OPEN_XOR_DELEGATION)) + return false; + /* Did we actually get a delegation? */ + if (!deleg_is_read(open->op_delegate_type) && !deleg_is_write(open->op_delegate_type)) + return false; + return true; +} + +/** + * nfsd4_process_open2 - finish open processing + * @rqstp: the RPC transaction being executed + * @current_fh: NFSv4 COMPOUND's current filehandle + * @open: OPEN arguments + * + * If successful, (1) truncate the file if open->op_truncate was + * set, (2) set open->op_stateid, (3) set open->op_delegation. + * + * Returns %nfs_ok on success; otherwise an nfs4stat value in + * network byte order is returned. + */ +__be32 +nfsd4_process_open2(struct svc_rqst *rqstp, struct svc_fh *current_fh, struct nfsd4_open *open) +{ + struct nfsd4_compoundres *resp = rqstp->rq_resp; + struct nfs4_client *cl = open->op_openowner->oo_owner.so_client; + struct nfs4_file *fp = NULL; + struct nfs4_ol_stateid *stp = NULL; + struct nfs4_delegation *dp = NULL; + __be32 status; + bool new_stp = false; + + /* + * Lookup file; if found, lookup stateid and check open request, + * and check for delegations in the process of being recalled. + * If not found, create the nfs4_file struct + */ + fp = nfsd4_file_hash_insert(open->op_file, current_fh); + if (unlikely(!fp)) + return nfserr_jukebox; + if (fp != open->op_file) { + status = nfs4_check_deleg(cl, open, &dp); + if (status) + goto out; + if (dp && nfsd4_is_deleg_cur(open) && + (dp->dl_stid.sc_file != fp)) { + /* + * RFC8881 section 8.2.4 mandates the server to return + * NFS4ERR_BAD_STATEID if the selected table entry does + * not match the current filehandle. However returning + * NFS4ERR_BAD_STATEID in the OPEN can cause the client + * to repeatedly retry the operation with the same + * stateid, since the stateid itself is valid. To avoid + * this situation NFSD returns NFS4ERR_INVAL instead. + */ + status = nfserr_inval; + goto out; + } + stp = nfsd4_find_and_lock_existing_open(fp, open); + } else { + open->op_file = NULL; + status = nfserr_bad_stateid; + if (nfsd4_is_deleg_cur(open)) + goto out; + } + + if (!stp) { + stp = init_open_stateid(fp, open); + if (!stp) { + status = nfserr_jukebox; + goto out; + } + + if (!open->op_stp) { + new_stp = true; + stp->st_stid.sc_export = + exp_get(current_fh->fh_export); + } + } + + /* + * OPEN the file, or upgrade an existing OPEN. + * If truncate fails, the OPEN fails. + * + * stp is already locked. + */ + if (!new_stp) { + /* Stateid was found, this is an OPEN upgrade */ + status = nfs4_upgrade_open(rqstp, fp, current_fh, stp, open); + if (status) { + mutex_unlock(&stp->st_mutex); + goto out; + } + } else { + status = nfs4_get_vfs_file(rqstp, fp, current_fh, stp, open, true); + if (status) { + release_open_stateid(stp); + mutex_unlock(&stp->st_mutex); + goto out; + } + + stp->st_clnt_odstate = find_or_hash_clnt_odstate(fp, + open->op_odstate); + if (stp->st_clnt_odstate == open->op_odstate) + open->op_odstate = NULL; + } + + nfs4_inc_and_copy_stateid(&open->op_stateid, &stp->st_stid); + mutex_unlock(&stp->st_mutex); + + if (nfsd4_has_session(&resp->cstate)) { + if (open->op_deleg_want & OPEN4_SHARE_ACCESS_WANT_NO_DELEG) { + open->op_delegate_type = OPEN_DELEGATE_NONE_EXT; + open->op_why_no_deleg = WND4_NOT_WANTED; + goto nodeleg; + } + } + + /* + * Attempt to hand out a delegation. No error return, because the + * OPEN succeeds even if we fail. + */ + nfs4_open_delegation(rqstp, open, stp, + &resp->cstate.current_fh, current_fh); + + /* + * If there is an existing open stateid, it must be updated and + * returned. Only respect WANT_OPEN_XOR_DELEGATION when a new + * open stateid would have to be created. + */ + if (new_stp && open_xor_delegation(open)) { + memcpy(&open->op_stateid, &zero_stateid, sizeof(open->op_stateid)); + open->op_rflags |= OPEN4_RESULT_NO_OPEN_STATEID; + release_open_stateid(stp); + } +nodeleg: + status = nfs_ok; + trace_nfsd_open(&stp->st_stid.sc_stateid); +out: + /* 4.1 client trying to upgrade/downgrade delegation? */ + if (open->op_delegate_type == OPEN_DELEGATE_NONE && dp && + open->op_deleg_want) + nfsd4_deleg_xgrade_none_ext(open, dp); + + if (fp) + put_nfs4_file(fp); + if (status == 0 && open->op_claim_type == NFS4_OPEN_CLAIM_PREVIOUS) + open->op_openowner->oo_flags |= NFS4_OO_CONFIRMED; + /* + * To finish the open response, we just need to set the rflags. + */ + open->op_rflags |= NFS4_OPEN_RESULT_LOCKTYPE_POSIX; + if (nfsd4_has_session(&resp->cstate)) + open->op_rflags |= NFS4_OPEN_RESULT_MAY_NOTIFY_LOCK; + else if (!(open->op_openowner->oo_flags & NFS4_OO_CONFIRMED)) + open->op_rflags |= NFS4_OPEN_RESULT_CONFIRM; + + if (dp) + nfs4_put_stid(&dp->dl_stid); + if (stp) + nfs4_put_stid(&stp->st_stid); + + return status; +} + +void nfsd4_cleanup_open_state(struct nfsd4_compound_state *cstate, + struct nfsd4_open *open) +{ + if (open->op_openowner) + nfs4_put_stateowner(&open->op_openowner->oo_owner); + if (open->op_file) + kmem_cache_free(file_slab, open->op_file); + if (open->op_stp) + nfs4_put_stid(&open->op_stp->st_stid); + if (open->op_odstate) + kmem_cache_free(odstate_slab, open->op_odstate); +} + +__be32 +nfsd4_renew(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + clientid_t *clid = &u->renew; + struct nfs4_client *clp; + __be32 status; + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + + trace_nfsd_clid_renew(clid); + status = set_client(clid, cstate, nn); + if (status) + return status; + clp = cstate->clp; + if (!list_empty(&clp->cl_delegations) + && clp->cl_cb_state != NFSD4_CB_UP) + return nfserr_cb_path_down; + return nfs_ok; +} + +static void +nfsd4_end_grace(struct nfsd_net *nn) +{ + /* + * nfsd4_end_grace() can be entered concurrently from the + * laundromat workqueue and from an nfsd compound thread + * handling RECLAIM_COMPLETE. Without serialization, both + * callers can observe NFSD_NET_GRACE_ENDED clear and proceed + * into nfsd4_record_grace_done(). For tracking ops whose + * grace_done drains reclaim_str_hashtbl, that results in + * list corruption and a double free of every + * nfs4_client_reclaim entry. Use an atomic test-and-set so + * exactly one caller proceeds. + */ + if (test_and_set_bit(NFSD_NET_GRACE_ENDED, &nn->flags)) + return; + + trace_nfsd_grace_complete(nn); + /* + * If the server goes down again right now, an NFSv4 + * client will still be allowed to reclaim after it comes back up, + * even if it hasn't yet had a chance to reclaim state this time. + * + */ + nfsd4_record_grace_done(nn); + /* + * At this point, NFSv4 clients can still reclaim. But if the + * server crashes, any that have not yet reclaimed will be out + * of luck on the next boot. + * + * (NFSv4.1+ clients are considered to have reclaimed once they + * call RECLAIM_COMPLETE. NFSv4.0 clients are considered to + * have reclaimed after their first OPEN.) + */ + locks_end_grace(&nn->nfsd4_manager); + /* + * At this point, and once lockd and/or any other containers + * exit their grace period, further reclaims will fail and + * regular locking can resume. + */ +} + +/** + * nfsd4_force_end_grace - forcibly end the NFSv4 grace period + * @nn: network namespace for the server instance to be updated + * + * Forces bypass of normal grace period completion, then schedules + * the laundromat to end the grace period immediately. Does not wait + * for the grace period to fully terminate before returning. + * + * Return values: + * %true: Grace termination schedule + * %false: No action was taken + */ +bool nfsd4_force_end_grace(struct nfsd_net *nn) +{ + if (!nn->client_tracking_ops) + return false; + if (test_bit(NFSD_NET_GRACE_ENDED, &nn->flags)) + return false; + /* laundromat_work must be initialised now, though it might be disabled */ + set_bit(NFSD_NET_GRACE_END_FORCED, &nn->flags); + /* mod_delayed_work() doesn't queue work after + * nfs4_state_shutdown_net() has called disable_delayed_work_sync() + */ + mod_delayed_work(laundry_wq, &nn->laundromat_work, 0); + return true; +} + +/* + * If we've waited a lease period but there are still clients trying to + * reclaim, wait a little longer to give them a chance to finish. + */ +static bool clients_still_reclaiming(struct nfsd_net *nn) +{ + time64_t double_grace_period_end = nn->boot_time_bt + + 2 * nn->nfsd4_lease; + + if (test_bit(NFSD_NET_GRACE_END_FORCED, &nn->flags)) + return false; + if (test_bit(NFSD_NET_TRACK_RECLAIM_COMPLETES, &nn->flags)) { + int size; + + down_read(&nn->reclaim_str_hashtbl_lock); + size = nn->reclaim_str_hashtbl_size; + up_read(&nn->reclaim_str_hashtbl_lock); + if (atomic_read(&nn->nr_reclaim_complete) == size) + return false; + } + if (!test_and_clear_bit(NFSD_NET_SOMEBODY_RECLAIMED, &nn->flags)) + return false; + /* + * If we've given them *two* lease times to reclaim, and they're + * still not done, give up: + */ + if (ktime_get_boottime_seconds() > double_grace_period_end) + return false; + return true; +} + +struct laundry_time { + time64_t cutoff; + time64_t new_timeo; +}; + +static bool state_expired(struct laundry_time *lt, time64_t last_refresh) +{ + time64_t time_remaining; + + if (last_refresh < lt->cutoff) + return true; + time_remaining = last_refresh - lt->cutoff; + lt->new_timeo = min(lt->new_timeo, time_remaining); + return false; +} + +#ifdef CONFIG_NFSD_V4_2_INTER_SSC +void nfsd4_ssc_init_umount_work(struct nfsd_net *nn) +{ + spin_lock_init(&nn->nfsd_ssc_lock); + INIT_LIST_HEAD(&nn->nfsd_ssc_mount_list); + init_waitqueue_head(&nn->nfsd_ssc_waitq); +} + +/* + * This is called when nfsd is being shutdown, after all inter_ssc + * cleanup were done, to destroy the ssc delayed unmount list. + */ +static void nfsd4_ssc_shutdown_umount(struct nfsd_net *nn) +{ + struct nfsd4_ssc_umount_item *ni = NULL; + struct nfsd4_ssc_umount_item *tmp; + + spin_lock(&nn->nfsd_ssc_lock); + list_for_each_entry_safe(ni, tmp, &nn->nfsd_ssc_mount_list, nsui_list) { + list_del(&ni->nsui_list); + spin_unlock(&nn->nfsd_ssc_lock); + mntput(ni->nsui_vfsmount); + kfree(ni); + spin_lock(&nn->nfsd_ssc_lock); + } + spin_unlock(&nn->nfsd_ssc_lock); +} + +static void nfsd4_ssc_expire_umount(struct nfsd_net *nn) +{ + bool do_wakeup = false; + struct nfsd4_ssc_umount_item *ni; + +restart: + spin_lock(&nn->nfsd_ssc_lock); + list_for_each_entry(ni, &nn->nfsd_ssc_mount_list, nsui_list) { + if (!time_after(jiffies, ni->nsui_expire)) + break; + if (refcount_read(&ni->nsui_refcnt) > 1) + continue; + + /* Prevent concurrent setup during unmount */ + ni->nsui_busy = true; + spin_unlock(&nn->nfsd_ssc_lock); + mntput(ni->nsui_vfsmount); + spin_lock(&nn->nfsd_ssc_lock); + + /* Force concurrent scanners to restart */ + list_del(&ni->nsui_list); + kfree(ni); + + /* wakeup ssc_connect waiters */ + do_wakeup = true; + /* + * Concurrent nfsd4_ssc_cancel_dul() can free any item + * on the list under nfsd_ssc_lock while mntput() runs + * above. Restart from the head; the list is short and + * the expire worker is periodic, so this is cheap. + */ + spin_unlock(&nn->nfsd_ssc_lock); + goto restart; + } + if (do_wakeup) + wake_up_all(&nn->nfsd_ssc_waitq); + spin_unlock(&nn->nfsd_ssc_lock); +} +#endif + +/* Check if any lock belonging to this lockowner has any blockers */ +static bool +nfs4_lockowner_has_blockers(struct nfs4_lockowner *lo) +{ + struct file_lock_context *ctx; + struct nfs4_ol_stateid *stp; + struct nfs4_file *nf; + + list_for_each_entry(stp, &lo->lo_owner.so_stateids, st_perstateowner) { + nf = stp->st_stid.sc_file; + ctx = locks_inode_context(nf->fi_inode); + if (!ctx) + continue; + if (locks_owner_has_blockers(ctx, lo)) + return true; + } + return false; +} + +static bool +nfs4_anylock_blockers(struct nfs4_client *clp) +{ + int i; + struct nfs4_stateowner *so; + struct nfs4_lockowner *lo; + + if (atomic_read(&clp->cl_delegs_in_recall)) + return true; + spin_lock(&clp->cl_lock); + for (i = 0; i < OWNER_HASH_SIZE; i++) { + list_for_each_entry(so, &clp->cl_ownerstr_hashtbl[i], + so_strhash) { + if (so->so_is_open_owner) + continue; + lo = lockowner(so); + if (nfs4_lockowner_has_blockers(lo)) { + spin_unlock(&clp->cl_lock); + return true; + } + } + } + spin_unlock(&clp->cl_lock); + return false; +} + +static void +nfs4_get_client_reaplist(struct nfsd_net *nn, struct list_head *reaplist, + struct laundry_time *lt) +{ + unsigned int maxreap, reapcnt = 0; + struct list_head *pos, *next; + struct nfs4_client *clp; + + maxreap = (atomic_read(&nn->nfs4_client_count) >= nn->nfs4_max_clients) ? + NFSD_CLIENT_MAX_TRIM_PER_RUN : 0; + INIT_LIST_HEAD(reaplist); + spin_lock(&nn->client_lock); + list_for_each_safe(pos, next, &nn->client_lru) { + clp = list_entry(pos, struct nfs4_client, cl_lru); + if (clp->cl_state == NFSD4_EXPIRABLE) + goto exp_client; + if (!state_expired(lt, clp->cl_time)) + break; + if (!atomic_read(&clp->cl_rpc_users)) { + if (clp->cl_state == NFSD4_ACTIVE) + atomic_inc(&nn->nfsd_courtesy_clients); + clp->cl_state = NFSD4_COURTESY; + } + if (!client_has_state(clp)) + goto exp_client; + if (!nfs4_anylock_blockers(clp)) + if (reapcnt >= maxreap) + continue; +exp_client: + if (!mark_client_expired_locked(clp)) { + list_add(&clp->cl_lru, reaplist); + reapcnt++; + } + } + spin_unlock(&nn->client_lock); +} + +static void +nfs4_get_courtesy_client_reaplist(struct nfsd_net *nn, + struct list_head *reaplist) +{ + unsigned int maxreap = 0, reapcnt = 0; + struct list_head *pos, *next; + struct nfs4_client *clp; + + maxreap = NFSD_CLIENT_MAX_TRIM_PER_RUN; + INIT_LIST_HEAD(reaplist); + + spin_lock(&nn->client_lock); + list_for_each_safe(pos, next, &nn->client_lru) { + clp = list_entry(pos, struct nfs4_client, cl_lru); + if (clp->cl_state == NFSD4_ACTIVE) + break; + if (reapcnt >= maxreap) + break; + if (!mark_client_expired_locked(clp)) { + list_add(&clp->cl_lru, reaplist); + reapcnt++; + } + } + spin_unlock(&nn->client_lock); +} + +static void +nfs4_process_client_reaplist(struct list_head *reaplist) +{ + struct list_head *pos, *next; + struct nfs4_client *clp; + + list_for_each_safe(pos, next, reaplist) { + clp = list_entry(pos, struct nfs4_client, cl_lru); + trace_nfsd_clid_purged(&clp->cl_clientid); + list_del_init(&clp->cl_lru); + expire_client(clp); + } +} + +static void nfs40_clean_admin_revoked(struct nfsd_net *nn, + struct laundry_time *lt) +{ + struct nfs4_client *clp; + + spin_lock(&nn->client_lock); + if (nn->nfs40_last_revoke == 0 || + nn->nfs40_last_revoke > lt->cutoff) { + spin_unlock(&nn->client_lock); + return; + } + nn->nfs40_last_revoke = 0; + +retry: + list_for_each_entry(clp, &nn->client_lru, cl_lru) { + unsigned long id, tmp; + struct nfs4_stid *stid; + + if (atomic_read(&clp->cl_admin_revoked) == 0) + continue; + if (is_client_expired(clp)) + continue; + + spin_lock(&clp->cl_lock); + idr_for_each_entry_ul(&clp->cl_stateids, stid, tmp, id) + if (stid->sc_status & SC_STATUS_ADMIN_REVOKED) { + refcount_inc(&stid->sc_count); + atomic_inc(&clp->cl_rpc_users); + spin_unlock(&nn->client_lock); + /* this function drops ->cl_lock */ + nfsd4_drop_revoked_stid(stid); + nfs4_put_stid(stid); + spin_lock(&nn->client_lock); + put_client_no_renew_locked(clp); + goto retry; + } + spin_unlock(&clp->cl_lock); + } + spin_unlock(&nn->client_lock); +} + +static time64_t +nfs4_laundromat(struct nfsd_net *nn) +{ + struct nfs4_openowner *oo; + struct nfs4_delegation *dp; + struct nfs4_ol_stateid *stp; + struct nfsd4_blocked_lock *nbl; + struct list_head *pos, *next, reaplist; + struct laundry_time lt = { + .cutoff = ktime_get_boottime_seconds() - nn->nfsd4_lease, + .new_timeo = nn->nfsd4_lease + }; + struct nfs4_cpntf_state *cps; + struct nfs4_client *clp; + copy_stateid_t *cps_t; + int i; + + if (clients_still_reclaiming(nn)) { + lt.new_timeo = 0; + goto out; + } + nfsd4_end_grace(nn); + + spin_lock(&nn->s2s_cp_lock); + /* s2s_cp_stateids holds only COPY_NOTIFY stateids */ + idr_for_each_entry(&nn->s2s_cp_stateids, cps_t, i) { + cps = container_of(cps_t, struct nfs4_cpntf_state, cp_stateid); + if (state_expired(<, cps->cpntf_time)) + revoke_cpntf_state_locked(nn, cps); + } + spin_unlock(&nn->s2s_cp_lock); + nfsd4_async_copy_reaper(nn); + nfs4_get_client_reaplist(nn, &reaplist, <); + nfs4_process_client_reaplist(&reaplist); + + nfs40_clean_admin_revoked(nn, <); + + spin_lock(&nn->deleg_lock); + list_for_each_safe(pos, next, &nn->del_recall_lru) { + dp = list_entry (pos, struct nfs4_delegation, dl_recall_lru); + if (!state_expired(<, dp->dl_time)) + break; + clp = dp->dl_stid.sc_client; + spin_lock(&nn->client_lock); + if (is_client_expired(clp)) { + spin_unlock(&nn->client_lock); + continue; + } + /* + * Pin without reviving: get_client_locked() would + * flip a courtesy client back to NFSD4_ACTIVE. + */ + atomic_inc(&clp->cl_rpc_users); + spin_unlock(&nn->client_lock); + refcount_inc(&dp->dl_stid.sc_count); + unhash_delegation_locked(dp, SC_STATUS_REVOKED); + list_add(&dp->dl_recall_lru, &reaplist); + } + spin_unlock(&nn->deleg_lock); + while (!list_empty(&reaplist)) { + dp = list_first_entry(&reaplist, struct nfs4_delegation, + dl_recall_lru); + clp = dp->dl_stid.sc_client; + list_del_init(&dp->dl_recall_lru); + revoke_delegation(dp); + put_client_no_renew(clp); + } + + spin_lock(&nn->client_lock); + while (!list_empty(&nn->close_lru)) { + oo = list_first_entry(&nn->close_lru, struct nfs4_openowner, + oo_close_lru); + if (!state_expired(<, oo->oo_time)) + break; + list_del_init(&oo->oo_close_lru); + clp = oo->oo_owner.so_client; + if (is_client_expired(clp)) + continue; + stp = oo->oo_last_closed_stid; + oo->oo_last_closed_stid = NULL; + atomic_inc(&clp->cl_rpc_users); + spin_unlock(&nn->client_lock); + nfs4_put_stid(&stp->st_stid); + spin_lock(&nn->client_lock); + put_client_no_renew_locked(clp); + } + spin_unlock(&nn->client_lock); + + /* + * It's possible for a client to try and acquire an already held lock + * that is being held for a long time, and then lose interest in it. + * So, we clean out any un-revisited request after a lease period + * under the assumption that the client is no longer interested. + * + * RFC5661, sec. 9.6 states that the client must not rely on getting + * notifications and must continue to poll for locks, even when the + * server supports them. Thus this shouldn't lead to clients blocking + * indefinitely once the lock does become free. + */ + BUG_ON(!list_empty(&reaplist)); + spin_lock(&nn->client_lock); + spin_lock(&nn->blocked_locks_lock); + list_for_each_safe(pos, next, &nn->blocked_locks_lru) { + nbl = list_entry(pos, struct nfsd4_blocked_lock, nbl_lru); + if (!state_expired(<, nbl->nbl_time)) + break; + clp = nbl_client(nbl); + if (is_client_expired(clp)) + continue; + atomic_inc(&clp->cl_rpc_users); + list_move(&nbl->nbl_lru, &reaplist); + list_del_init(&nbl->nbl_list); + } + spin_unlock(&nn->blocked_locks_lock); + spin_unlock(&nn->client_lock); + + while (!list_empty(&reaplist)) { + nbl = list_first_entry(&reaplist, + struct nfsd4_blocked_lock, nbl_lru); + clp = nbl_client(nbl); + list_del_init(&nbl->nbl_lru); + free_blocked_lock(nbl); + put_client_no_renew(clp); + } +#ifdef CONFIG_NFSD_V4_2_INTER_SSC + /* service the server-to-server copy delayed unmount list */ + nfsd4_ssc_expire_umount(nn); +#endif + if (atomic_long_read(&num_delegations) >= max_delegations) + deleg_reaper(nn); +out: + return max_t(time64_t, lt.new_timeo, NFSD_LAUNDROMAT_MINTIMEOUT); +} + +static void laundromat_main(struct work_struct *); + +static void +laundromat_main(struct work_struct *laundry) +{ + time64_t t; + struct delayed_work *dwork = to_delayed_work(laundry); + struct nfsd_net *nn = container_of(dwork, struct nfsd_net, + laundromat_work); + + t = nfs4_laundromat(nn); + queue_delayed_work(laundry_wq, &nn->laundromat_work, t*HZ); +} + +static void +courtesy_client_reaper(struct nfsd_net *nn) +{ + struct list_head reaplist; + + nfs4_get_courtesy_client_reaplist(nn, &reaplist); + nfs4_process_client_reaplist(&reaplist); +} + +static void +deleg_reaper(struct nfsd_net *nn) +{ + struct list_head *pos, *next; + struct nfs4_client *clp; + + spin_lock(&nn->client_lock); + list_for_each_safe(pos, next, &nn->client_lru) { + clp = list_entry(pos, struct nfs4_client, cl_lru); + + if (clp->cl_state != NFSD4_ACTIVE) + continue; + if (list_empty(&clp->cl_delegations)) + continue; + if (atomic_read(&clp->cl_delegs_in_recall)) + continue; + if (ktime_get_boottime_seconds() - clp->cl_ra_time < 5) + continue; + if (clp->cl_cb_state != NFSD4_CB_UP) + continue; + if (test_and_set_bit(NFSD4_CALLBACK_RUNNING, &clp->cl_ra->ra_cb.cb_flags)) + continue; + + /* release in nfsd4_cb_recall_any_release */ + kref_get(&clp->cl_nfsdfs.cl_ref); + clp->cl_ra_time = ktime_get_boottime_seconds(); + clp->cl_ra->ra_keep = 0; + clp->cl_ra->ra_bmval[0] = BIT(RCA4_TYPE_MASK_RDATA_DLG) | + BIT(RCA4_TYPE_MASK_WDATA_DLG); + trace_nfsd_cb_recall_any(clp->cl_ra); + nfsd4_run_cb(&clp->cl_ra->ra_cb); + } + spin_unlock(&nn->client_lock); +} + +static void +nfsd4_state_shrinker_worker(struct work_struct *work) +{ + struct nfsd_net *nn = container_of(work, struct nfsd_net, + nfsd_shrinker_work); + + courtesy_client_reaper(nn); + deleg_reaper(nn); +} + +static inline __be32 nfs4_check_fh(struct svc_fh *fhp, struct nfs4_stid *stp) +{ + if (!fh_match(&fhp->fh_handle, &stp->sc_file->fi_fhandle)) + return nfserr_bad_stateid; + return nfs_ok; +} + +static +__be32 nfs4_check_openmode(struct nfs4_ol_stateid *stp, int flags) +{ + __be32 status = nfserr_openmode; + + /* For lock stateid's, we test the parent open, not the lock: */ + if (stp->st_openstp) + stp = stp->st_openstp; + if ((flags & WR_STATE) && !access_permit_write(stp)) + goto out; + if ((flags & RD_STATE) && !access_permit_read(stp)) + goto out; + status = nfs_ok; +out: + return status; +} + +static inline __be32 +check_special_stateids(struct net *net, svc_fh *current_fh, stateid_t *stateid, int flags) +{ + if (ONE_STATEID(stateid) && (flags & RD_STATE)) + return nfs_ok; + else if (opens_in_grace(net)) { + /* Answer in remaining cases depends on existence of + * conflicting state; so we must wait out the grace period. */ + return nfserr_grace; + } else if (flags & WR_STATE) + return nfs4_share_conflict(current_fh, + NFS4_SHARE_DENY_WRITE); + else /* (flags & RD_STATE) && ZERO_STATEID(stateid) */ + return nfs4_share_conflict(current_fh, + NFS4_SHARE_DENY_READ); +} + +static __be32 check_stateid_generation(stateid_t *in, stateid_t *ref, bool has_session) +{ + /* + * When sessions are used the stateid generation number is ignored + * when it is zero. + */ + if (has_session && in->si_generation == 0) + return nfs_ok; + + if (in->si_generation == ref->si_generation) + return nfs_ok; + + /* If the client sends us a stateid from the future, it's buggy: */ + if (nfsd4_stateid_generation_after(in, ref)) + return nfserr_bad_stateid; + /* + * However, we could see a stateid from the past, even from a + * non-buggy client. For example, if the client sends a lock + * while some IO is outstanding, the lock may bump si_generation + * while the IO is still in flight. The client could avoid that + * situation by waiting for responses on all the IO requests, + * but better performance may result in retrying IO that + * receives an old_stateid error if requests are rarely + * reordered in flight: + */ + return nfserr_old_stateid; +} + +static __be32 nfsd4_stid_check_stateid_generation(stateid_t *in, struct nfs4_stid *s, bool has_session) +{ + __be32 ret; + + spin_lock(&s->sc_lock); + ret = nfsd4_verify_open_stid(s); + if (ret == nfs_ok) + ret = check_stateid_generation(in, &s->sc_stateid, has_session); + spin_unlock(&s->sc_lock); + if (ret == nfserr_admin_revoked) + nfsd40_drop_revoked_stid(s->sc_client, + &s->sc_stateid); + return ret; +} + +static __be32 nfsd4_check_openowner_confirmed(struct nfs4_ol_stateid *ols) +{ + if (ols->st_stateowner->so_is_open_owner && + !(openowner(ols->st_stateowner)->oo_flags & NFS4_OO_CONFIRMED)) + return nfserr_bad_stateid; + return nfs_ok; +} + +static __be32 nfsd4_validate_stateid(struct nfs4_client *cl, stateid_t *stateid) +{ + struct nfs4_stid *s; + __be32 status = nfserr_bad_stateid; + + if (ZERO_STATEID(stateid) || ONE_STATEID(stateid) || + CLOSE_STATEID(stateid)) + return status; + spin_lock(&cl->cl_lock); + s = find_stateid_locked(cl, stateid); + if (!s) + goto out_unlock; + status = nfsd4_stid_check_stateid_generation(stateid, s, 1); + if (status) + goto out_unlock; + status = nfsd4_verify_open_stid(s); + if (status) + goto out_unlock; + + switch (s->sc_type) { + case SC_TYPE_DELEG: + status = nfs_ok; + break; + case SC_TYPE_OPEN: + case SC_TYPE_LOCK: + status = nfsd4_check_openowner_confirmed(openlockstateid(s)); + break; + default: + printk("unknown stateid type %x\n", s->sc_type); + status = nfserr_bad_stateid; + } +out_unlock: + spin_unlock(&cl->cl_lock); + if (status == nfserr_admin_revoked) + nfsd40_drop_revoked_stid(cl, stateid); + return status; +} + +__be32 +nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate, + stateid_t *stateid, + unsigned short typemask, unsigned short statusmask, + struct nfs4_stid **s, struct nfsd_net *nn) +{ + __be32 status; + struct nfs4_stid *stid; + bool return_revoked = false; + + /* + * only return revoked delegations if explicitly asked. + * otherwise we report revoked or bad_stateid status. + */ + if (statusmask & SC_STATUS_REVOKED) + return_revoked = true; + if (typemask & SC_TYPE_DELEG) + /* Always allow REVOKED for DELEG so we can + * return the appropriate error. + */ + statusmask |= SC_STATUS_REVOKED; + + statusmask |= SC_STATUS_ADMIN_REVOKED | SC_STATUS_FREEABLE; + + if (ZERO_STATEID(stateid) || ONE_STATEID(stateid) || + CLOSE_STATEID(stateid)) + return nfserr_bad_stateid; + status = set_client(&stateid->si_opaque.so_clid, cstate, nn); + if (status == nfserr_stale_clientid) { + if (cstate->session) + return nfserr_bad_stateid; + return nfserr_stale_stateid; + } + if (status) + return status; + stid = find_stateid_by_type(cstate->clp, stateid, typemask, statusmask); + if (!stid) + return nfserr_bad_stateid; + if ((stid->sc_status & SC_STATUS_REVOKED) && !return_revoked) { + nfs4_put_stid(stid); + return nfserr_deleg_revoked; + } + if (stid->sc_status & SC_STATUS_ADMIN_REVOKED) { + nfsd40_drop_revoked_stid(cstate->clp, stateid); + nfs4_put_stid(stid); + return nfserr_admin_revoked; + } + *s = stid; + return nfs_ok; +} + +static struct nfsd_file * +nfs4_find_file(struct nfs4_stid *s, int flags) +{ + struct nfsd_file *ret = NULL; + + if (!s || s->sc_status) + return NULL; + + switch (s->sc_type) { + case SC_TYPE_DELEG: + case SC_TYPE_OPEN: + case SC_TYPE_LOCK: + if (flags & RD_STATE) + ret = find_readable_file(s->sc_file); + else + ret = find_writeable_file(s->sc_file); + } + + return ret; +} + +static __be32 +nfs4_check_olstateid(struct nfs4_ol_stateid *ols, int flags) +{ + __be32 status; + + status = nfsd4_check_openowner_confirmed(ols); + if (status) + return status; + return nfs4_check_openmode(ols, flags); +} + +static __be32 +nfs4_check_file(struct svc_rqst *rqstp, struct svc_fh *fhp, struct nfs4_stid *s, + struct nfsd_file **nfp, int flags) +{ + int acc = (flags & RD_STATE) ? NFSD_MAY_READ : NFSD_MAY_WRITE; + struct nfsd_file *nf; + __be32 status; + + nf = nfs4_find_file(s, flags); + if (nf) { + status = nfsd_permission(&rqstp->rq_cred, + fhp->fh_export, fhp->fh_dentry, + acc | NFSD_MAY_OWNER_OVERRIDE); + if (status) { + nfsd_file_put(nf); + goto out; + } + } else { + status = nfsd_file_acquire(rqstp, fhp, acc, &nf); + if (status) + return status; + } + *nfp = nf; +out: + return status; +} + +static void _free_cpntf_state_locked(struct nfsd_net *nn, struct nfs4_cpntf_state *cps) +{ + WARN_ON_ONCE(cps->cp_stateid.cs_type != NFS4_COPYNOTIFY_STID); + if (refcount_dec_and_test(&cps->cp_stateid.cs_count)) { + nfsd4_unhash_cpntf_state(nn, cps); + kfree(cps); + } +} +/* + * A READ from an inter server to server COPY will have a + * copy stateid. Look up the copy notify stateid from the + * idr structure and take a reference on it. + */ +__be32 manage_cpntf_state(struct nfsd_net *nn, stateid_t *st, + struct nfs4_client *clp, + struct nfs4_cpntf_state **cps) +{ + copy_stateid_t *cps_t; + struct nfs4_cpntf_state *state = NULL; + + if (st->si_opaque.so_clid.cl_id != nn->s2s_cp_cl_id) + return nfserr_bad_stateid; + spin_lock(&nn->s2s_cp_lock); + /* s2s_cp_stateids holds only COPY_NOTIFY stateids */ + cps_t = idr_find(&nn->s2s_cp_stateids, st->si_opaque.so_id); + if (cps_t) { + state = container_of(cps_t, struct nfs4_cpntf_state, + cp_stateid); + if (!clp) { + refcount_inc(&state->cp_stateid.cs_count); + } else if (memcmp(&clp->cl_clientid, &state->cp_p_clid, + sizeof(clientid_t))) { + /* + * OFFLOAD_CANCEL: only the creating client may cancel. + * so_id is guessable, so without this check any client + * could free another's cpntf state. + */ + state = NULL; + goto unlock; + } else { + revoke_cpntf_state_locked(nn, state); + } + } +unlock: + spin_unlock(&nn->s2s_cp_lock); + if (!state) + return nfserr_bad_stateid; + if (!clp) + *cps = state; + return 0; +} + +static __be32 find_cpntf_state(struct nfsd_net *nn, stateid_t *st, + struct nfs4_stid **stid) +{ + __be32 status; + struct nfs4_cpntf_state *cps = NULL; + struct nfs4_client *found; + + status = manage_cpntf_state(nn, st, NULL, &cps); + if (status) + return status; + + cps->cpntf_time = ktime_get_boottime_seconds(); + + status = nfserr_expired; + found = lookup_clientid(&cps->cp_p_clid, true, nn); + if (!found) + goto out; + + *stid = find_stateid_by_type(found, &cps->cp_p_stateid, + SC_TYPE_DELEG|SC_TYPE_OPEN|SC_TYPE_LOCK, + 0); + if (*stid) + status = nfs_ok; + else + status = nfserr_bad_stateid; + + put_client_renew(found); +out: + nfs4_put_cpntf_state(nn, cps); + return status; +} + +void nfs4_put_cpntf_state(struct nfsd_net *nn, struct nfs4_cpntf_state *cps) +{ + spin_lock(&nn->s2s_cp_lock); + _free_cpntf_state_locked(nn, cps); + spin_unlock(&nn->s2s_cp_lock); +} + +/** + * nfs4_preprocess_stateid_op - find and prep stateid for an operation + * @rqstp: incoming request from client + * @cstate: current compound state + * @fhp: filehandle associated with requested stateid + * @stateid: stateid (provided by client) + * @flags: flags describing type of operation to be done + * @nfp: optional nfsd_file return pointer (may be NULL) + * @cstid: optional returned nfs4_stid pointer (may be NULL) + * + * Given info from the client, look up a nfs4_stid for the operation. On + * success, it returns a reference to the nfs4_stid and/or the nfsd_file + * associated with it. + */ +__be32 +nfs4_preprocess_stateid_op(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, struct svc_fh *fhp, + stateid_t *stateid, int flags, struct nfsd_file **nfp, + struct nfs4_stid **cstid) +{ + struct net *net = SVC_NET(rqstp); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct nfs4_stid *s = NULL; + __be32 status; + + if (nfp) + *nfp = NULL; + + if (ZERO_STATEID(stateid) || ONE_STATEID(stateid)) { + status = check_special_stateids(net, fhp, stateid, flags); + goto done; + } + + status = nfsd4_lookup_stateid(cstate, stateid, + SC_TYPE_DELEG|SC_TYPE_OPEN|SC_TYPE_LOCK, + 0, &s, nn); + if (status == nfserr_bad_stateid) + status = find_cpntf_state(nn, stateid, &s); + if (status) + return status; + status = nfsd4_stid_check_stateid_generation(stateid, s, + nfsd4_has_session(cstate)); + if (status) + goto out; + + switch (s->sc_type) { + case SC_TYPE_DELEG: + status = nfs4_check_delegmode(delegstateid(s), flags); + break; + case SC_TYPE_OPEN: + case SC_TYPE_LOCK: + status = nfs4_check_olstateid(openlockstateid(s), flags); + break; + } + if (status) + goto out; + status = nfs4_check_fh(fhp, s); + +done: + if (status == nfs_ok && nfp) + status = nfs4_check_file(rqstp, fhp, s, nfp, flags); +out: + if (s) { + if (!status && cstid) + *cstid = s; + else + nfs4_put_stid(s); + } + return status; +} + +/* + * Test if the stateid is valid + */ +__be32 +nfsd4_test_stateid(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_test_stateid *test_stateid = &u->test_stateid; + struct nfsd4_test_stateid_id *stateid; + struct nfs4_client *cl = cstate->clp; + + list_for_each_entry(stateid, &test_stateid->ts_stateid_list, ts_id_list) + stateid->ts_id_status = + nfsd4_validate_stateid(cl, &stateid->ts_id_stateid); + + return nfs_ok; +} + +static __be32 +nfsd4_free_lock_stateid(stateid_t *stateid, struct nfs4_stid *s) +{ + struct nfs4_ol_stateid *stp = openlockstateid(s); + __be32 ret; + + ret = nfsd4_lock_ol_stateid(stp); + if (ret) + goto out_put_stid; + + ret = check_stateid_generation(stateid, &s->sc_stateid, 1); + if (ret) + goto out; + + ret = nfserr_locks_held; + if (check_for_locks(stp->st_stid.sc_file, + lockowner(stp->st_stateowner))) + goto out; + + release_lock_stateid(stp); + ret = nfs_ok; + +out: + mutex_unlock(&stp->st_mutex); +out_put_stid: + nfs4_put_stid(s); + return ret; +} + +__be32 +nfsd4_free_stateid(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_free_stateid *free_stateid = &u->free_stateid; + stateid_t *stateid = &free_stateid->fr_stateid; + struct nfs4_stid *s; + struct nfs4_delegation *dp; + struct nfs4_client *cl = cstate->clp; + __be32 ret = nfserr_bad_stateid; + + spin_lock(&cl->cl_lock); + s = find_stateid_locked(cl, stateid); + if (!s || s->sc_status & SC_STATUS_CLOSED) + goto out_unlock; + if (s->sc_status & SC_STATUS_ADMIN_REVOKED) { + nfsd4_drop_revoked_stid(s); + ret = nfs_ok; + goto out; + } + spin_lock(&s->sc_lock); + switch (s->sc_type) { + case SC_TYPE_DELEG: + if (s->sc_status & SC_STATUS_REVOKED) { + s->sc_status |= SC_STATUS_CLOSED; + spin_unlock(&s->sc_lock); + dp = delegstateid(s); + if (s->sc_status & SC_STATUS_FREEABLE) + list_del_init(&dp->dl_recall_lru); + s->sc_status |= SC_STATUS_FREED; + spin_unlock(&cl->cl_lock); + nfs4_put_stid(s); + ret = nfs_ok; + goto out; + } + ret = nfserr_locks_held; + break; + case SC_TYPE_OPEN: + ret = check_stateid_generation(stateid, &s->sc_stateid, 1); + if (ret) + break; + ret = nfserr_locks_held; + break; + case SC_TYPE_LOCK: + spin_unlock(&s->sc_lock); + refcount_inc(&s->sc_count); + spin_unlock(&cl->cl_lock); + ret = nfsd4_free_lock_stateid(stateid, s); + goto out; + } + spin_unlock(&s->sc_lock); +out_unlock: + spin_unlock(&cl->cl_lock); +out: + return ret; +} + +static inline int +setlkflg (int type) +{ + return (type == NFS4_READW_LT || type == NFS4_READ_LT) ? + RD_STATE : WR_STATE; +} + +static __be32 nfs4_seqid_op_checks(struct nfsd4_compound_state *cstate, stateid_t *stateid, u32 seqid, struct nfs4_ol_stateid *stp) +{ + struct svc_fh *current_fh = &cstate->current_fh; + struct nfs4_stateowner *sop = stp->st_stateowner; + __be32 status; + + status = nfsd4_check_seqid(cstate, sop, seqid); + if (status) + return status; + status = nfsd4_lock_ol_stateid(stp); + if (status != nfs_ok) + return status; + status = check_stateid_generation(stateid, &stp->st_stid.sc_stateid, nfsd4_has_session(cstate)); + if (status == nfs_ok) + status = nfs4_check_fh(current_fh, &stp->st_stid); + if (status != nfs_ok) + mutex_unlock(&stp->st_mutex); + return status; +} + +/** + * nfs4_preprocess_seqid_op - find and prep an ol_stateid for a seqid-morphing op + * @cstate: compund state + * @seqid: seqid (provided by client) + * @stateid: stateid (provided by client) + * @typemask: mask of allowable types for this operation + * @statusmask: mask of allowed states: 0 or STID_CLOSED + * @stpp: return pointer for the stateid found + * @nn: net namespace for request + * + * Given a stateid+seqid from a client, look up an nfs4_ol_stateid and + * return it in @stpp. On a nfs_ok return, the returned stateid will + * have its st_mutex locked. + */ +static __be32 +nfs4_preprocess_seqid_op(struct nfsd4_compound_state *cstate, u32 seqid, + stateid_t *stateid, + unsigned short typemask, unsigned short statusmask, + struct nfs4_ol_stateid **stpp, + struct nfsd_net *nn) +{ + __be32 status; + struct nfs4_stid *s; + struct nfs4_ol_stateid *stp = NULL; + + trace_nfsd_preprocess(seqid, stateid); + + *stpp = NULL; +retry: + status = nfsd4_lookup_stateid(cstate, stateid, + typemask, statusmask, &s, nn); + if (status) + return status; + stp = openlockstateid(s); + if (nfsd4_cstate_assign_replay(cstate, stp->st_stateowner) == -EAGAIN) { + nfs4_put_stid(&stp->st_stid); + goto retry; + } + + status = nfs4_seqid_op_checks(cstate, stateid, seqid, stp); + if (!status) + *stpp = stp; + else + nfs4_put_stid(&stp->st_stid); + return status; +} + +static __be32 nfs4_preprocess_confirmed_seqid_op(struct nfsd4_compound_state *cstate, u32 seqid, + stateid_t *stateid, struct nfs4_ol_stateid **stpp, struct nfsd_net *nn) +{ + __be32 status; + struct nfs4_openowner *oo; + struct nfs4_ol_stateid *stp; + + status = nfs4_preprocess_seqid_op(cstate, seqid, stateid, + SC_TYPE_OPEN, 0, &stp, nn); + if (status) + return status; + oo = openowner(stp->st_stateowner); + if (!(oo->oo_flags & NFS4_OO_CONFIRMED)) { + mutex_unlock(&stp->st_mutex); + nfs4_put_stid(&stp->st_stid); + return nfserr_bad_stateid; + } + *stpp = stp; + return nfs_ok; +} + +__be32 +nfsd4_open_confirm(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_open_confirm *oc = &u->open_confirm; + __be32 status; + struct nfs4_openowner *oo; + struct nfs4_ol_stateid *stp; + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + + dprintk("NFSD: nfsd4_open_confirm on file %pd\n", + cstate->current_fh.fh_dentry); + + status = fh_verify(rqstp, &cstate->current_fh, S_IFREG, 0); + if (status) + return status; + + status = nfs4_preprocess_seqid_op(cstate, + oc->oc_seqid, &oc->oc_req_stateid, + SC_TYPE_OPEN, 0, &stp, nn); + if (status) + goto out; + oo = openowner(stp->st_stateowner); + status = nfserr_bad_stateid; + if (oo->oo_flags & NFS4_OO_CONFIRMED) { + mutex_unlock(&stp->st_mutex); + goto put_stateid; + } + oo->oo_flags |= NFS4_OO_CONFIRMED; + nfs4_inc_and_copy_stateid(&oc->oc_resp_stateid, &stp->st_stid); + mutex_unlock(&stp->st_mutex); + trace_nfsd_open_confirm(oc->oc_seqid, &stp->st_stid.sc_stateid); + nfsd4_client_record_create(oo->oo_owner.so_client); + status = nfs_ok; +put_stateid: + nfs4_put_stid(&stp->st_stid); +out: + nfsd4_bump_seqid(cstate, status); + return status; +} + +static inline void nfs4_stateid_downgrade_bit(struct nfs4_ol_stateid *stp, u32 access) +{ + if (!test_access(access, stp)) + return; + nfs4_file_put_access(stp->st_stid.sc_file, access); + clear_access(access, stp); +} + +static inline void nfs4_stateid_downgrade(struct nfs4_ol_stateid *stp, u32 to_access) +{ + switch (to_access) { + case NFS4_SHARE_ACCESS_READ: + nfs4_stateid_downgrade_bit(stp, NFS4_SHARE_ACCESS_WRITE); + nfs4_stateid_downgrade_bit(stp, NFS4_SHARE_ACCESS_BOTH); + break; + case NFS4_SHARE_ACCESS_WRITE: + nfs4_stateid_downgrade_bit(stp, NFS4_SHARE_ACCESS_READ); + nfs4_stateid_downgrade_bit(stp, NFS4_SHARE_ACCESS_BOTH); + break; + case NFS4_SHARE_ACCESS_BOTH: + break; + default: + WARN_ON_ONCE(1); + } +} + +__be32 +nfsd4_open_downgrade(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, union nfsd4_op_u *u) +{ + struct nfsd4_open_downgrade *od = &u->open_downgrade; + __be32 status; + struct nfs4_ol_stateid *stp; + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + + dprintk("NFSD: nfsd4_open_downgrade on file %pd\n", + cstate->current_fh.fh_dentry); + + /* We don't yet support WANT bits: */ + if (od->od_deleg_want) + dprintk("NFSD: %s: od_deleg_want=0x%x ignored\n", __func__, + od->od_deleg_want); + + status = nfs4_preprocess_confirmed_seqid_op(cstate, od->od_seqid, + &od->od_stateid, &stp, nn); + if (status) + goto out; + status = nfserr_inval; + if (!test_access(od->od_share_access, stp)) { + dprintk("NFSD: access not a subset of current bitmap: 0x%hhx, input access=%08x\n", + stp->st_access_bmap, od->od_share_access); + goto put_stateid; + } + if (!test_deny(od->od_share_deny, stp)) { + dprintk("NFSD: deny not a subset of current bitmap: 0x%hhx, input deny=%08x\n", + stp->st_deny_bmap, od->od_share_deny); + goto put_stateid; + } + nfs4_stateid_downgrade(stp, od->od_share_access); + reset_union_bmap_deny(od->od_share_deny, stp); + nfs4_inc_and_copy_stateid(&od->od_stateid, &stp->st_stid); + status = nfs_ok; +put_stateid: + mutex_unlock(&stp->st_mutex); + nfs4_put_stid(&stp->st_stid); +out: + nfsd4_bump_seqid(cstate, status); + return status; +} + +static bool nfsd4_close_open_stateid(struct nfs4_ol_stateid *s) +{ + struct nfs4_client *clp = s->st_stid.sc_client; + bool unhashed; + LIST_HEAD(reaplist); + struct nfs4_ol_stateid *stp; + + spin_lock(&clp->cl_lock); + unhashed = unhash_open_stateid(s, &reaplist); + + if (clp->cl_minorversion) { + if (unhashed) + put_ol_stateid_locked(s, &reaplist); + spin_unlock(&clp->cl_lock); + list_for_each_entry(stp, &reaplist, st_locks) + nfs4_free_cpntf_statelist(clp->net, &stp->st_stid); + free_ol_stateid_reaplist(&reaplist); + return false; + } else { + spin_unlock(&clp->cl_lock); + free_ol_stateid_reaplist(&reaplist); + return unhashed; + } +} + +/* + * nfs4_unlock_state() called after encode + */ +__be32 +nfsd4_close(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_close *close = &u->close; + __be32 status; + struct nfs4_ol_stateid *stp; + struct net *net = SVC_NET(rqstp); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + bool need_move_to_close_list; + + dprintk("NFSD: nfsd4_close on file %pd\n", + cstate->current_fh.fh_dentry); + + status = nfs4_preprocess_seqid_op(cstate, close->cl_seqid, + &close->cl_stateid, + SC_TYPE_OPEN, SC_STATUS_CLOSED, + &stp, nn); + nfsd4_bump_seqid(cstate, status); + if (status) + goto out; + + spin_lock(&stp->st_stid.sc_client->cl_lock); + stp->st_stid.sc_status |= SC_STATUS_CLOSED; + spin_unlock(&stp->st_stid.sc_client->cl_lock); + + /* + * Technically we don't _really_ have to increment or copy it, since + * it should just be gone after this operation and we clobber the + * copied value below, but we continue to do so here just to ensure + * that racing ops see that there was a state change. + */ + nfs4_inc_and_copy_stateid(&close->cl_stateid, &stp->st_stid); + + need_move_to_close_list = nfsd4_close_open_stateid(stp); + mutex_unlock(&stp->st_mutex); + if (need_move_to_close_list) + move_to_close_lru(stp, net); + + /* v4.1+ suggests that we send a special stateid in here, since the + * clients should just ignore this anyway. Since this is not useful + * for v4.0 clients either, we set it to the special close_stateid + * universally. + * + * See RFC5661 section 18.2.4, and RFC7530 section 16.2.5 + */ + memcpy(&close->cl_stateid, &close_stateid, sizeof(close->cl_stateid)); + + /* put reference from nfs4_preprocess_seqid_op */ + nfs4_put_stid(&stp->st_stid); +out: + return status; +} + +__be32 +nfsd4_delegreturn(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_delegreturn *dr = &u->delegreturn; + struct nfs4_delegation *dp; + stateid_t *stateid = &dr->dr_stateid; + struct nfs4_stid *s; + __be32 status; + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + + status = fh_verify(rqstp, &cstate->current_fh, 0, 0); + if (status) + return status; + + status = nfsd4_lookup_stateid(cstate, stateid, SC_TYPE_DELEG, SC_STATUS_REVOKED, &s, nn); + if (status) + goto out; + dp = delegstateid(s); + status = nfsd4_stid_check_stateid_generation(stateid, &dp->dl_stid, nfsd4_has_session(cstate)); + if (status) + goto put_stateid; + + status = nfs4_check_fh(&cstate->current_fh, &dp->dl_stid); + if (status) + goto put_stateid; + + trace_nfsd_deleg_return(stateid); + destroy_delegation(dp); + smp_mb__after_atomic(); + wake_up_var(d_inode(cstate->current_fh.fh_dentry)); +put_stateid: + nfs4_put_stid(&dp->dl_stid); +out: + return status; +} + +/* last octet in a range */ +static inline u64 +last_byte_offset(u64 start, u64 len) +{ + u64 end; + + WARN_ON_ONCE(!len); + end = start + len; + return end > start ? end - 1: NFS4_MAX_UINT64; +} + +/* + * TODO: Linux file offsets are _signed_ 64-bit quantities, which means that + * we can't properly handle lock requests that go beyond the (2^63 - 1)-th + * byte, because of sign extension problems. Since NFSv4 calls for 64-bit + * locking, this prevents us from being completely protocol-compliant. The + * real solution to this problem is to start using unsigned file offsets in + * the VFS, but this is a very deep change! + */ +static inline void +nfs4_transform_lock_offset(struct file_lock *lock) +{ + if (lock->fl_start < 0) + lock->fl_start = OFFSET_MAX; + if (lock->fl_end < 0) + lock->fl_end = OFFSET_MAX; +} + +static fl_owner_t +nfsd4_lm_get_owner(fl_owner_t owner) +{ + struct nfs4_lockowner *lo = (struct nfs4_lockowner *)owner; + + nfs4_get_stateowner(&lo->lo_owner); + return owner; +} + +static void +nfsd4_lm_put_owner(fl_owner_t owner) +{ + struct nfs4_lockowner *lo = (struct nfs4_lockowner *)owner; + + if (lo) + nfs4_put_stateowner(&lo->lo_owner); +} + +/* return pointer to struct nfs4_client if client is expirable */ +static bool +nfsd4_lm_lock_expirable(struct file_lock *cfl) +{ + struct nfs4_lockowner *lo = (struct nfs4_lockowner *) cfl->c.flc_owner; + struct nfs4_client *clp = lo->lo_owner.so_client; + struct nfsd_net *nn; + + if (try_to_expire_client(clp)) { + nn = net_generic(clp->net, nfsd_net_id); + mod_delayed_work(laundry_wq, &nn->laundromat_work, 0); + return true; + } + return false; +} + +/* schedule laundromat to run immediately and wait for it to complete */ +static void +nfsd4_lm_expire_lock(void) +{ + flush_workqueue(laundry_wq); +} + +static void +nfsd4_lm_notify(struct file_lock *fl) +{ + struct nfs4_lockowner *lo = (struct nfs4_lockowner *) fl->c.flc_owner; + struct net *net = lo->lo_owner.so_client->net; + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct nfsd4_blocked_lock *nbl = container_of(fl, + struct nfsd4_blocked_lock, nbl_lock); + bool queue = false; + + /* An empty list means that something else is going to be using it */ + spin_lock(&nn->blocked_locks_lock); + if (!list_empty(&nbl->nbl_list)) { + list_del_init(&nbl->nbl_list); + list_del_init(&nbl->nbl_lru); + queue = true; + } + spin_unlock(&nn->blocked_locks_lock); + + if (queue) { + trace_nfsd_cb_notify_lock(lo, nbl); + nfsd4_try_run_cb(&nbl->nbl_cb); + } +} + +static const struct lock_manager_operations nfsd_posix_mng_ops = { + .lm_mod_owner = THIS_MODULE, + .lm_notify = nfsd4_lm_notify, + .lm_get_owner = nfsd4_lm_get_owner, + .lm_put_owner = nfsd4_lm_put_owner, + .lm_lock_expirable = nfsd4_lm_lock_expirable, + .lm_expire_lock = nfsd4_lm_expire_lock, +}; + +static inline void +nfs4_set_lock_denied(struct file_lock *fl, struct nfsd4_lock_denied *deny) +{ + struct nfs4_lockowner *lo; + + if (fl->fl_lmops == &nfsd_posix_mng_ops) { + lo = (struct nfs4_lockowner *) fl->c.flc_owner; + xdr_netobj_dup(&deny->ld_owner, &lo->lo_owner.so_owner, + GFP_KERNEL); + if (!deny->ld_owner.data) + /* We just don't care that much */ + goto nevermind; + deny->ld_clientid = lo->lo_owner.so_client->cl_clientid; + } else { +nevermind: + deny->ld_owner.len = 0; + deny->ld_owner.data = NULL; + deny->ld_clientid.cl_boot = 0; + deny->ld_clientid.cl_id = 0; + } + deny->ld_start = fl->fl_start; + deny->ld_length = NFS4_MAX_UINT64; + if (fl->fl_end != NFS4_MAX_UINT64) + deny->ld_length = fl->fl_end - fl->fl_start + 1; + deny->ld_type = NFS4_READ_LT; + if (fl->c.flc_type != F_RDLCK) + deny->ld_type = NFS4_WRITE_LT; +} + +static struct nfs4_lockowner * +find_lockowner_str_locked(struct nfs4_client *clp, struct xdr_netobj *owner) +{ + unsigned int strhashval = ownerstr_hashval(owner); + struct nfs4_stateowner *so; + + lockdep_assert_held(&clp->cl_lock); + + list_for_each_entry(so, &clp->cl_ownerstr_hashtbl[strhashval], + so_strhash) { + if (so->so_is_open_owner) + continue; + if (same_owner_str(so, owner)) + return lockowner(nfs4_get_stateowner(so)); + } + return NULL; +} + +static struct nfs4_lockowner * +find_lockowner_str(struct nfs4_client *clp, struct xdr_netobj *owner) +{ + struct nfs4_lockowner *lo; + + spin_lock(&clp->cl_lock); + lo = find_lockowner_str_locked(clp, owner); + spin_unlock(&clp->cl_lock); + return lo; +} + +static void nfs4_unhash_lockowner(struct nfs4_stateowner *sop) +{ + unhash_lockowner_locked(lockowner(sop)); +} + +static void nfs4_free_lockowner(struct nfs4_stateowner *sop) +{ + struct nfs4_lockowner *lo = lockowner(sop); + + kmem_cache_free(lockowner_slab, lo); +} + +static const struct nfs4_stateowner_operations lockowner_ops = { + .so_unhash = nfs4_unhash_lockowner, + .so_free = nfs4_free_lockowner, +}; + +/* + * Alloc a lock owner structure. + * Called in nfsd4_lock - therefore, OPEN and OPEN_CONFIRM (if needed) has + * occurred. + * + * strhashval = ownerstr_hashval + */ +static struct nfs4_lockowner * +alloc_init_lock_stateowner(unsigned int strhashval, struct nfs4_client *clp, + struct nfs4_ol_stateid *open_stp, + struct nfsd4_lock *lock) +{ + struct nfs4_lockowner *lo, *ret; + + lo = alloc_stateowner(lockowner_slab, &lock->lk_new_owner, clp); + if (!lo) + return NULL; + INIT_LIST_HEAD(&lo->lo_blocked); + INIT_LIST_HEAD(&lo->lo_owner.so_stateids); + lo->lo_owner.so_is_open_owner = 0; + lo->lo_owner.so_seqid = lock->lk_new_lock_seqid; + lo->lo_owner.so_ops = &lockowner_ops; + spin_lock(&clp->cl_lock); + ret = find_lockowner_str_locked(clp, &lock->lk_new_owner); + if (ret == NULL) { + list_add(&lo->lo_owner.so_strhash, + &clp->cl_ownerstr_hashtbl[strhashval]); + ret = lo; + } else + nfs4_free_stateowner(&lo->lo_owner); + + spin_unlock(&clp->cl_lock); + return ret; +} + +static struct nfs4_ol_stateid * +find_lock_stateid(const struct nfs4_lockowner *lo, + const struct nfs4_ol_stateid *ost) +{ + struct nfs4_ol_stateid *lst; + + lockdep_assert_held(&ost->st_stid.sc_client->cl_lock); + + /* If ost is not hashed, ost->st_locks will not be valid */ + if (!nfs4_ol_stateid_unhashed(ost)) + list_for_each_entry(lst, &ost->st_locks, st_locks) { + if (lst->st_stateowner == &lo->lo_owner) { + refcount_inc(&lst->st_stid.sc_count); + return lst; + } + } + return NULL; +} + +static struct nfs4_ol_stateid * +init_lock_stateid(struct nfs4_ol_stateid *stp, struct nfs4_lockowner *lo, + struct nfs4_file *fp, struct inode *inode, + struct nfs4_ol_stateid *open_stp) +{ + struct nfs4_client *clp = lo->lo_owner.so_client; + struct nfs4_ol_stateid *retstp; + + mutex_init(&stp->st_mutex); + mutex_lock_nested(&stp->st_mutex, OPEN_STATEID_MUTEX); +retry: + spin_lock(&clp->cl_lock); + if (nfs4_ol_stateid_unhashed(open_stp)) + goto out_close; + retstp = find_lock_stateid(lo, open_stp); + if (retstp) + goto out_found; + refcount_inc(&stp->st_stid.sc_count); + stp->st_stid.sc_type = SC_TYPE_LOCK; + stp->st_stateowner = nfs4_get_stateowner(&lo->lo_owner); + get_nfs4_file(fp); + stp->st_stid.sc_file = fp; + if (open_stp->st_stid.sc_export) + stp->st_stid.sc_export = + exp_get(open_stp->st_stid.sc_export); + stp->st_access_bmap = 0; + stp->st_deny_bmap = open_stp->st_deny_bmap; + stp->st_openstp = open_stp; + spin_lock(&fp->fi_lock); + list_add(&stp->st_locks, &open_stp->st_locks); + list_add(&stp->st_perstateowner, &lo->lo_owner.so_stateids); + list_add(&stp->st_perfile, &fp->fi_stateids); + spin_unlock(&fp->fi_lock); + spin_unlock(&clp->cl_lock); + return stp; +out_found: + spin_unlock(&clp->cl_lock); + if (nfsd4_lock_ol_stateid(retstp) != nfs_ok) { + nfs4_put_stid(&retstp->st_stid); + goto retry; + } + /* To keep mutex tracking happy */ + mutex_unlock(&stp->st_mutex); + return retstp; +out_close: + spin_unlock(&clp->cl_lock); + mutex_unlock(&stp->st_mutex); + return NULL; +} + +static struct nfs4_ol_stateid * +find_or_create_lock_stateid(struct nfs4_lockowner *lo, struct nfs4_file *fi, + struct inode *inode, struct nfs4_ol_stateid *ost, + bool *new) +{ + struct nfs4_stid *ns = NULL; + struct nfs4_ol_stateid *lst; + struct nfs4_openowner *oo = openowner(ost->st_stateowner); + struct nfs4_client *clp = oo->oo_owner.so_client; + + *new = false; + spin_lock(&clp->cl_lock); + lst = find_lock_stateid(lo, ost); + spin_unlock(&clp->cl_lock); + if (lst != NULL) { + if (nfsd4_lock_ol_stateid(lst) == nfs_ok) + goto out; + nfs4_put_stid(&lst->st_stid); + } + ns = nfs4_alloc_stid(clp, stateid_slab, nfs4_free_lock_stateid); + if (ns == NULL) + return NULL; + + lst = init_lock_stateid(openlockstateid(ns), lo, fi, inode, ost); + if (lst == openlockstateid(ns)) + *new = true; + else + nfs4_put_stid(ns); +out: + return lst; +} + +static int +check_lock_length(u64 offset, u64 length) +{ + return ((length == 0) || ((length != NFS4_MAX_UINT64) && + (length > ~offset))); +} + +static void get_lock_access(struct nfs4_ol_stateid *lock_stp, u32 access) +{ + struct nfs4_file *fp = lock_stp->st_stid.sc_file; + + lockdep_assert_held(&fp->fi_lock); + + if (test_access(access, lock_stp)) + return; + __nfs4_file_get_access(fp, access); + set_access(access, lock_stp); +} + +static __be32 +lookup_or_create_lock_state(struct nfsd4_compound_state *cstate, + struct nfs4_ol_stateid *ost, + struct nfsd4_lock *lock, + struct nfs4_ol_stateid **plst, bool *new) +{ + __be32 status; + struct nfs4_file *fi = ost->st_stid.sc_file; + struct nfs4_openowner *oo = openowner(ost->st_stateowner); + struct nfs4_client *cl = oo->oo_owner.so_client; + struct inode *inode = d_inode(cstate->current_fh.fh_dentry); + struct nfs4_lockowner *lo; + struct nfs4_ol_stateid *lst; + unsigned int strhashval; + + lo = find_lockowner_str(cl, &lock->lk_new_owner); + if (!lo) { + strhashval = ownerstr_hashval(&lock->lk_new_owner); + lo = alloc_init_lock_stateowner(strhashval, cl, ost, lock); + if (lo == NULL) + return nfserr_jukebox; + } else { + /* with an existing lockowner, seqids must be the same */ + status = nfserr_bad_seqid; + if (!cstate->minorversion && + lock->lk_new_lock_seqid != lo->lo_owner.so_seqid) + goto out; + } + + lst = find_or_create_lock_stateid(lo, fi, inode, ost, new); + if (lst == NULL) { + status = nfserr_jukebox; + goto out; + } + + status = nfs_ok; + *plst = lst; +out: + nfs4_put_stateowner(&lo->lo_owner); + return status; +} + +/* + * LOCK operation + */ +__be32 +nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_lock *lock = &u->lock; + struct nfs4_openowner *open_sop = NULL; + struct nfs4_lockowner *lock_sop = NULL; + struct nfs4_ol_stateid *lock_stp = NULL; + struct nfs4_ol_stateid *open_stp = NULL; + struct nfs4_file *fp; + struct nfsd_file *nf = NULL; + struct nfsd4_blocked_lock *nbl = NULL; + struct file_lock *file_lock = NULL; + struct file_lock *conflock = NULL; + __be32 status = 0; + int lkflg; + int err; + bool new = false; + unsigned char type; + unsigned int flags = FL_POSIX; + struct net *net = SVC_NET(rqstp); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + dprintk("NFSD: nfsd4_lock: start=%Ld length=%Ld\n", + (long long) lock->lk_offset, + (long long) lock->lk_length); + + if (check_lock_length(lock->lk_offset, lock->lk_length)) + return nfserr_inval; + + status = fh_verify(rqstp, &cstate->current_fh, S_IFREG, 0); + if (status != nfs_ok) + return status; + if (exportfs_cannot_lock(cstate->current_fh.fh_dentry->d_sb->s_export_op)) { + status = nfserr_notsupp; + goto out; + } + + if (lock->lk_is_new) { + if (nfsd4_has_session(cstate)) + /* See rfc 5661 18.10.3: given clientid is ignored: */ + memcpy(&lock->lk_new_clientid, + &cstate->clp->cl_clientid, + sizeof(clientid_t)); + + /* validate and update open stateid and open seqid */ + status = nfs4_preprocess_confirmed_seqid_op(cstate, + lock->lk_new_open_seqid, + &lock->lk_new_open_stateid, + &open_stp, nn); + if (status) + goto out; + mutex_unlock(&open_stp->st_mutex); + open_sop = openowner(open_stp->st_stateowner); + status = nfserr_bad_stateid; + if (!same_clid(&open_sop->oo_owner.so_client->cl_clientid, + &lock->lk_new_clientid)) + goto out; + status = lookup_or_create_lock_state(cstate, open_stp, lock, + &lock_stp, &new); + } else { + status = nfs4_preprocess_seqid_op(cstate, + lock->lk_old_lock_seqid, + &lock->lk_old_lock_stateid, + SC_TYPE_LOCK, 0, &lock_stp, + nn); + } + if (status) + goto out; + lock_sop = lockowner(lock_stp->st_stateowner); + + lkflg = setlkflg(lock->lk_type); + status = nfs4_check_openmode(lock_stp, lkflg); + if (status) + goto out; + + status = nfserr_grace; + if (locks_in_grace(net) && !lock->lk_reclaim) + goto out; + status = nfserr_no_grace; + if (!locks_in_grace(net) && lock->lk_reclaim) + goto out; + if (lock->lk_reclaim && + test_bit(NFSD4_CLIENT_RECLAIM_COMPLETE, &cstate->clp->cl_flags)) + goto out; + + if (lock->lk_reclaim) + flags |= FL_RECLAIM; + + fp = lock_stp->st_stid.sc_file; + switch (lock->lk_type) { + case NFS4_READW_LT: + fallthrough; + case NFS4_READ_LT: + spin_lock(&fp->fi_lock); + nf = find_readable_file_locked(fp); + if (nf) + get_lock_access(lock_stp, NFS4_SHARE_ACCESS_READ); + spin_unlock(&fp->fi_lock); + type = F_RDLCK; + break; + case NFS4_WRITEW_LT: + fallthrough; + case NFS4_WRITE_LT: + spin_lock(&fp->fi_lock); + nf = find_writeable_file_locked(fp); + if (nf) + get_lock_access(lock_stp, NFS4_SHARE_ACCESS_WRITE); + spin_unlock(&fp->fi_lock); + type = F_WRLCK; + break; + default: + status = nfserr_inval; + goto out; + } + + if (!nf) { + status = nfserr_openmode; + goto out; + } + + if ((lock->lk_type == NFS4_READW_LT || + lock->lk_type == NFS4_WRITEW_LT) && + nfsd4_has_session(cstate) && + locks_can_async_lock(nf->nf_file->f_op)) + flags |= FL_SLEEP; + + nbl = find_or_allocate_block(lock_sop, &fp->fi_fhandle, nn); + if (!nbl) { + dprintk("NFSD: %s: unable to allocate block!\n", __func__); + status = nfserr_jukebox; + goto out; + } + + file_lock = &nbl->nbl_lock; + file_lock->c.flc_type = type; + file_lock->c.flc_owner = (fl_owner_t)lockowner(nfs4_get_stateowner(&lock_sop->lo_owner)); + file_lock->c.flc_pid = current->tgid; + file_lock->c.flc_file = nf->nf_file; + file_lock->c.flc_flags = flags; + file_lock->fl_lmops = &nfsd_posix_mng_ops; + file_lock->fl_start = lock->lk_offset; + file_lock->fl_end = last_byte_offset(lock->lk_offset, lock->lk_length); + nfs4_transform_lock_offset(file_lock); + + conflock = locks_alloc_lock(); + if (!conflock) { + dprintk("NFSD: %s: unable to allocate lock!\n", __func__); + status = nfserr_jukebox; + goto out; + } + + if (flags & FL_SLEEP) { + nbl->nbl_time = ktime_get_boottime_seconds(); + spin_lock(&nn->blocked_locks_lock); + list_add_tail(&nbl->nbl_list, &lock_sop->lo_blocked); + list_add_tail(&nbl->nbl_lru, &nn->blocked_locks_lru); + kref_get(&nbl->nbl_kref); + spin_unlock(&nn->blocked_locks_lock); + } + + err = vfs_lock_file(nf->nf_file, F_SETLK, file_lock, conflock); + switch (err) { + case 0: /* success! */ + nfs4_inc_and_copy_stateid(&lock->lk_resp_stateid, &lock_stp->st_stid); + status = 0; + if (lock->lk_reclaim) + set_bit(NFSD_NET_SOMEBODY_RECLAIMED, &nn->flags); + break; + case FILE_LOCK_DEFERRED: + kref_put(&nbl->nbl_kref, free_nbl); + nbl = NULL; + fallthrough; + case -EAGAIN: /* conflock holds conflicting lock */ + status = nfserr_denied; + dprintk("NFSD: nfsd4_lock: conflicting lock found!\n"); + nfs4_set_lock_denied(conflock, &lock->lk_denied); + break; + case -EDEADLK: + status = nfserr_deadlock; + break; + default: + dprintk("NFSD: nfsd4_lock: vfs_lock_file() failed! status %d\n",err); + status = nfserrno(err); + break; + } +out: + if (nbl) { + /* dequeue it if we queued it before */ + if (flags & FL_SLEEP) { + spin_lock(&nn->blocked_locks_lock); + if (!list_empty(&nbl->nbl_list) && + !list_empty(&nbl->nbl_lru)) { + list_del_init(&nbl->nbl_list); + list_del_init(&nbl->nbl_lru); + kref_put(&nbl->nbl_kref, free_nbl); + } + /* nbl can use one of lists to be linked to reaplist */ + spin_unlock(&nn->blocked_locks_lock); + } + free_blocked_lock(nbl); + } + if (nf) + nfsd_file_put(nf); + if (lock_stp) { + /* Bump seqid manually if the 4.0 replay owner is openowner */ + if (cstate->replay_owner && + cstate->replay_owner != &lock_sop->lo_owner && + seqid_mutating_err(ntohl(status))) + lock_sop->lo_owner.so_seqid++; + + /* + * If this is a new, never-before-used stateid, and we are + * returning an error, then just go ahead and release it. + */ + if (status && new) + release_lock_stateid(lock_stp); + + mutex_unlock(&lock_stp->st_mutex); + + nfs4_put_stid(&lock_stp->st_stid); + } + if (open_stp) + nfs4_put_stid(&open_stp->st_stid); + nfsd4_bump_seqid(cstate, status); + if (conflock) + locks_free_lock(conflock); + return status; +} + +void nfsd4_lock_release(union nfsd4_op_u *u) +{ + struct nfsd4_lock *lock = &u->lock; + struct nfsd4_lock_denied *deny = &lock->lk_denied; + + kfree(deny->ld_owner.data); +} + +/* + * The NFSv4 spec allows a client to do a LOCKT without holding an OPEN, + * so we do a temporary open here just to get an open file to pass to + * vfs_test_lock. + */ +static __be32 nfsd_test_lock(struct svc_rqst *rqstp, struct svc_fh *fhp, struct file_lock *lock) +{ + struct nfsd_file *nf; + struct inode *inode; + __be32 err; + + err = nfsd_file_acquire(rqstp, fhp, NFSD_MAY_READ, &nf); + if (err) + return err; + inode = fhp->fh_dentry->d_inode; + inode_lock(inode); /* to block new leases till after test_lock: */ + err = nfserrno(nfsd_open_break_lease(inode, NFSD_MAY_READ)); + if (err) + goto out; + lock->c.flc_file = nf->nf_file; + err = nfserrno(vfs_test_lock(nf->nf_file, lock)); + lock->c.flc_file = NULL; +out: + inode_unlock(inode); + nfsd_file_put(nf); + return err; +} + +/* + * LOCKT operation + */ +__be32 +nfsd4_lockt(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_lockt *lockt = &u->lockt; + struct file_lock *file_lock = NULL; + struct nfs4_lockowner *lo = NULL; + __be32 status; + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + + if (locks_in_grace(SVC_NET(rqstp))) + return nfserr_grace; + + if (check_lock_length(lockt->lt_offset, lockt->lt_length)) + return nfserr_inval; + + if (!nfsd4_has_session(cstate)) { + status = set_client(&lockt->lt_clientid, cstate, nn); + if (status) + goto out; + } + + if ((status = fh_verify(rqstp, &cstate->current_fh, S_IFREG, 0))) + goto out; + + file_lock = locks_alloc_lock(); + if (!file_lock) { + dprintk("NFSD: %s: unable to allocate lock!\n", __func__); + status = nfserr_jukebox; + goto out; + } + + switch (lockt->lt_type) { + case NFS4_READ_LT: + case NFS4_READW_LT: + file_lock->c.flc_type = F_RDLCK; + break; + case NFS4_WRITE_LT: + case NFS4_WRITEW_LT: + file_lock->c.flc_type = F_WRLCK; + break; + default: + dprintk("NFSD: nfs4_lockt: bad lock type!\n"); + status = nfserr_inval; + goto out; + } + + lo = find_lockowner_str(cstate->clp, &lockt->lt_owner); + if (lo) + file_lock->c.flc_owner = (fl_owner_t)lo; + file_lock->c.flc_pid = current->tgid; + file_lock->c.flc_flags = FL_POSIX; + + file_lock->fl_start = lockt->lt_offset; + file_lock->fl_end = last_byte_offset(lockt->lt_offset, lockt->lt_length); + + nfs4_transform_lock_offset(file_lock); + + status = nfsd_test_lock(rqstp, &cstate->current_fh, file_lock); + if (status) + goto out; + + if (file_lock->c.flc_type != F_UNLCK) { + status = nfserr_denied; + nfs4_set_lock_denied(file_lock, &lockt->lt_denied); + } +out: + if (lo) + nfs4_put_stateowner(&lo->lo_owner); + if (file_lock) + locks_free_lock(file_lock); + return status; +} + +void nfsd4_lockt_release(union nfsd4_op_u *u) +{ + struct nfsd4_lockt *lockt = &u->lockt; + struct nfsd4_lock_denied *deny = &lockt->lt_denied; + + kfree(deny->ld_owner.data); +} + +__be32 +nfsd4_locku(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_locku *locku = &u->locku; + struct nfs4_ol_stateid *stp; + struct nfsd_file *nf = NULL; + struct file_lock *file_lock = NULL; + __be32 status; + int err; + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + + dprintk("NFSD: nfsd4_locku: start=%Ld length=%Ld\n", + (long long) locku->lu_offset, + (long long) locku->lu_length); + + if (check_lock_length(locku->lu_offset, locku->lu_length)) + return nfserr_inval; + + status = nfs4_preprocess_seqid_op(cstate, locku->lu_seqid, + &locku->lu_stateid, SC_TYPE_LOCK, 0, + &stp, nn); + if (status) + goto out; + nf = find_any_file(stp->st_stid.sc_file); + if (!nf) { + status = nfserr_lock_range; + goto put_stateid; + } + if (exportfs_cannot_lock(nf->nf_file->f_path.mnt->mnt_sb->s_export_op)) { + status = nfserr_notsupp; + goto put_file; + } + + file_lock = locks_alloc_lock(); + if (!file_lock) { + dprintk("NFSD: %s: unable to allocate lock!\n", __func__); + status = nfserr_jukebox; + goto put_file; + } + + file_lock->c.flc_type = F_UNLCK; + file_lock->c.flc_owner = (fl_owner_t)lockowner(nfs4_get_stateowner(stp->st_stateowner)); + file_lock->c.flc_pid = current->tgid; + file_lock->c.flc_file = nf->nf_file; + file_lock->c.flc_flags = FL_POSIX; + file_lock->fl_lmops = &nfsd_posix_mng_ops; + file_lock->fl_start = locku->lu_offset; + + file_lock->fl_end = last_byte_offset(locku->lu_offset, + locku->lu_length); + nfs4_transform_lock_offset(file_lock); + + err = vfs_lock_file(nf->nf_file, F_SETLK, file_lock, NULL); + if (err) { + dprintk("NFSD: nfs4_locku: vfs_lock_file failed!\n"); + goto out_nfserr; + } + nfs4_inc_and_copy_stateid(&locku->lu_stateid, &stp->st_stid); +put_file: + nfsd_file_put(nf); +put_stateid: + mutex_unlock(&stp->st_mutex); + nfs4_put_stid(&stp->st_stid); +out: + nfsd4_bump_seqid(cstate, status); + if (file_lock) + locks_free_lock(file_lock); + return status; + +out_nfserr: + status = nfserrno(err); + goto put_file; +} + +/* + * returns + * true: locks held by lockowner + * false: no locks held by lockowner + */ +static bool +check_for_locks(struct nfs4_file *fp, struct nfs4_lockowner *lowner) +{ + struct file_lock *fl; + int status = false; + struct nfsd_file *nf; + struct inode *inode; + struct file_lock_context *flctx; + + spin_lock(&fp->fi_lock); + nf = find_any_file_locked(fp); + if (!nf) { + /* Any valid lock stateid should have some sort of access */ + WARN_ON_ONCE(1); + goto out; + } + + inode = file_inode(nf->nf_file); + flctx = locks_inode_context(inode); + + if (flctx && !list_empty_careful(&flctx->flc_posix)) { + spin_lock(&flctx->flc_lock); + for_each_file_lock(fl, &flctx->flc_posix) { + if (fl->c.flc_owner == (fl_owner_t)lowner) { + status = true; + break; + } + } + spin_unlock(&flctx->flc_lock); + } +out: + spin_unlock(&fp->fi_lock); + return status; +} + +/** + * nfsd4_release_lockowner - process NFSv4.0 RELEASE_LOCKOWNER operations + * @rqstp: RPC transaction + * @cstate: NFSv4 COMPOUND state + * @u: RELEASE_LOCKOWNER arguments + * + * Check if there are any locks still held and if not, free the lockowner + * and any lock state that is owned. + * + * Return values: + * %nfs_ok: lockowner released or not found + * %nfserr_locks_held: lockowner still in use + * %nfserr_stale_clientid: clientid no longer active + * %nfserr_expired: clientid not recognized + */ +__be32 +nfsd4_release_lockowner(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + struct nfsd4_release_lockowner *rlockowner = &u->release_lockowner; + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + clientid_t *clid = &rlockowner->rl_clientid; + struct nfs4_ol_stateid *stp; + struct nfs4_lockowner *lo; + struct nfs4_client *clp; + LIST_HEAD(reaplist); + __be32 status; + + dprintk("nfsd4_release_lockowner clientid: (%08x/%08x):\n", + clid->cl_boot, clid->cl_id); + + status = set_client(clid, cstate, nn); + if (status) + return status; + clp = cstate->clp; + + spin_lock(&clp->cl_lock); + lo = find_lockowner_str_locked(clp, &rlockowner->rl_owner); + if (!lo) { + spin_unlock(&clp->cl_lock); + return nfs_ok; + } + + list_for_each_entry(stp, &lo->lo_owner.so_stateids, st_perstateowner) { + if (check_for_locks(stp->st_stid.sc_file, lo)) { + spin_unlock(&clp->cl_lock); + nfs4_put_stateowner(&lo->lo_owner); + return nfserr_locks_held; + } + } + unhash_lockowner_locked(lo); + while (!list_empty(&lo->lo_owner.so_stateids)) { + stp = list_first_entry(&lo->lo_owner.so_stateids, + struct nfs4_ol_stateid, + st_perstateowner); + unhash_lock_stateid(stp); + put_ol_stateid_locked(stp, &reaplist); + } + spin_unlock(&clp->cl_lock); + + free_ol_stateid_reaplist(&reaplist); + remove_blocked_locks(lo); + nfs4_put_stateowner(&lo->lo_owner); + return nfs_ok; +} + +static inline struct nfs4_client_reclaim * +alloc_reclaim(void) +{ + return kmalloc_obj(struct nfs4_client_reclaim); +} + +bool +nfs4_has_reclaimed_state(struct xdr_netobj name, struct nfsd_net *nn) +{ + struct nfs4_client_reclaim *crp; + bool found; + + down_read(&nn->reclaim_str_hashtbl_lock); + crp = nfsd4_find_reclaim_client(name, nn); + found = (crp && crp->cr_clp); + up_read(&nn->reclaim_str_hashtbl_lock); + return found; +} + +/* + * failure => all reset bets are off, nfserr_no_grace... + */ +struct nfs4_client_reclaim * +nfs4_client_to_reclaim(struct xdr_netobj name, struct xdr_netobj princhash, + struct nfsd_net *nn) +{ + unsigned int strhashval; + struct nfs4_client_reclaim *crp; + + down_write(&nn->reclaim_str_hashtbl_lock); + + /* + * A reclaim record for this client name may already exist (for + * example, populated at boot from the recovery directory before + * an in-grace RECLAIM_COMPLETE or an nfsdcld downcall delivers + * the same name). Dedup here so reclaim_str_hashtbl_size stays + * equal to the number of distinct client names; inc_reclaim_complete + * relies on that equality to end the grace period via the fast path. + */ + crp = nfsd4_find_reclaim_client(name, nn); + if (crp) { + if (princhash.len && crp->cr_princhash.len == 0) { + void *pdata = kmemdup(princhash.data, princhash.len, + GFP_KERNEL); + if (pdata) { + crp->cr_princhash.data = pdata; + crp->cr_princhash.len = princhash.len; + } else { + dprintk("%s: failed to allocate memory for princhash.data!\n", + __func__); + crp = NULL; + } + } + up_write(&nn->reclaim_str_hashtbl_lock); + return crp; + } + + name.data = kmemdup(name.data, name.len, GFP_KERNEL); + if (!name.data) { + dprintk("%s: failed to allocate memory for name.data!\n", + __func__); + up_write(&nn->reclaim_str_hashtbl_lock); + return NULL; + } + if (princhash.len) { + princhash.data = kmemdup(princhash.data, princhash.len, GFP_KERNEL); + if (!princhash.data) { + dprintk("%s: failed to allocate memory for princhash.data!\n", + __func__); + kfree(name.data); + up_write(&nn->reclaim_str_hashtbl_lock); + return NULL; + } + } else + princhash.data = NULL; + crp = alloc_reclaim(); + if (crp) { + strhashval = clientstr_hashval(name); + INIT_LIST_HEAD(&crp->cr_strhash); + list_add(&crp->cr_strhash, &nn->reclaim_str_hashtbl[strhashval]); + crp->cr_name.data = name.data; + crp->cr_name.len = name.len; + crp->cr_princhash.data = princhash.data; + crp->cr_princhash.len = princhash.len; + crp->cr_clp = NULL; + nn->reclaim_str_hashtbl_size++; + } else { + kfree(name.data); + kfree(princhash.data); + } + up_write(&nn->reclaim_str_hashtbl_lock); + return crp; +} + +void +nfs4_remove_reclaim_record(struct nfs4_client_reclaim *crp, struct nfsd_net *nn) +{ + list_del(&crp->cr_strhash); + kfree(crp->cr_name.data); + kfree(crp->cr_princhash.data); + kfree(crp); + nn->reclaim_str_hashtbl_size--; +} + +void +nfs4_release_reclaim(struct nfsd_net *nn) +{ + struct nfs4_client_reclaim *crp = NULL; + int i; + + down_write(&nn->reclaim_str_hashtbl_lock); + for (i = 0; i < CLIENT_HASH_SIZE; i++) { + while (!list_empty(&nn->reclaim_str_hashtbl[i])) { + crp = list_entry(nn->reclaim_str_hashtbl[i].next, + struct nfs4_client_reclaim, cr_strhash); + nfs4_remove_reclaim_record(crp, nn); + } + } + WARN_ON_ONCE(nn->reclaim_str_hashtbl_size); + up_write(&nn->reclaim_str_hashtbl_lock); +} + +/* + * called from OPEN, CLAIM_PREVIOUS with a new clientid. */ +struct nfs4_client_reclaim * +nfsd4_find_reclaim_client(struct xdr_netobj name, struct nfsd_net *nn) +{ + unsigned int strhashval; + struct nfs4_client_reclaim *crp = NULL; + + strhashval = clientstr_hashval(name); + list_for_each_entry(crp, &nn->reclaim_str_hashtbl[strhashval], cr_strhash) { + if (compare_blob(&crp->cr_name, &name) == 0) { + return crp; + } + } + return NULL; +} + +__be32 +nfs4_check_open_reclaim(struct nfs4_client *clp) +{ + if (test_bit(NFSD4_CLIENT_RECLAIM_COMPLETE, &clp->cl_flags)) + return nfserr_no_grace; + + if (nfsd4_client_record_check(clp)) + return nfserr_reclaim_bad; + + return nfs_ok; +} + +/* + * Since the lifetime of a delegation isn't limited to that of an open, a + * client may quite reasonably hang on to a delegation as long as it has + * the inode cached. This becomes an obvious problem the first time a + * client's inode cache approaches the size of the server's total memory. + * + * For now we avoid this problem by imposing a hard limit on the number + * of delegations, which varies according to the server's memory size. + */ +static void +set_max_delegations(void) +{ + /* + * Allow at most 4 delegations per megabyte of RAM. Quick + * estimates suggest that in the worst case (where every delegation + * is for a different inode), a delegation could take about 1.5K, + * giving a worst case usage of about 6% of memory. + */ + max_delegations = nr_free_buffer_pages() >> (20 - 2 - PAGE_SHIFT); +} + +static int nfs4_state_create_net(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + int i; + + nn->conf_id_hashtbl = kmalloc_objs(struct list_head, CLIENT_HASH_SIZE); + if (!nn->conf_id_hashtbl) + goto err; + nn->unconf_id_hashtbl = kmalloc_objs(struct list_head, CLIENT_HASH_SIZE); + if (!nn->unconf_id_hashtbl) + goto err_unconf_id; + nn->sessionid_hashtbl = kmalloc_objs(struct list_head, + SESSION_HASH_SIZE); + if (!nn->sessionid_hashtbl) + goto err_sessionid; + + for (i = 0; i < CLIENT_HASH_SIZE; i++) { + INIT_LIST_HEAD(&nn->conf_id_hashtbl[i]); + INIT_LIST_HEAD(&nn->unconf_id_hashtbl[i]); + } + for (i = 0; i < SESSION_HASH_SIZE; i++) + INIT_LIST_HEAD(&nn->sessionid_hashtbl[i]); + nn->conf_name_tree = RB_ROOT; + nn->unconf_name_tree = RB_ROOT; + nn->boot_time = ktime_get_real_seconds(); + nn->boot_time_bt = ktime_get_boottime_seconds(); + clear_bit(NFSD_NET_GRACE_ENDED, &nn->flags); + clear_bit(NFSD_NET_GRACE_END_FORCED, &nn->flags); + nn->nfsd4_manager.block_opens = true; + INIT_LIST_HEAD(&nn->nfsd4_manager.list); + INIT_LIST_HEAD(&nn->client_lru); + INIT_LIST_HEAD(&nn->close_lru); + INIT_LIST_HEAD(&nn->del_recall_lru); + spin_lock_init(&nn->deleg_lock); + spin_lock_init(&nn->client_lock); + spin_lock_init(&nn->s2s_cp_lock); + idr_init(&nn->s2s_cp_stateids); + atomic_set(&nn->pending_async_copies, 0); + + spin_lock_init(&nn->blocked_locks_lock); + INIT_LIST_HEAD(&nn->blocked_locks_lru); + + INIT_DELAYED_WORK(&nn->laundromat_work, laundromat_main); + /* Make sure this cannot run until client tracking is initialised */ + disable_delayed_work(&nn->laundromat_work); + INIT_WORK(&nn->nfsd_shrinker_work, nfsd4_state_shrinker_worker); + get_net(net); + + nn->nfsd_client_shrinker = shrinker_alloc(0, "nfsd-client"); + if (!nn->nfsd_client_shrinker) + goto err_shrinker; + + nn->nfsd_client_shrinker->scan_objects = nfsd4_state_shrinker_scan; + nn->nfsd_client_shrinker->count_objects = nfsd4_state_shrinker_count; + nn->nfsd_client_shrinker->private_data = nn; + + shrinker_register(nn->nfsd_client_shrinker); + + return 0; + +err_shrinker: + put_net(net); + kfree(nn->sessionid_hashtbl); +err_sessionid: + kfree(nn->unconf_id_hashtbl); +err_unconf_id: + kfree(nn->conf_id_hashtbl); +err: + return -ENOMEM; +} + +static void +nfs4_state_destroy_net(struct net *net) +{ + int i; + struct nfs4_client *clp = NULL; + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + for (i = 0; i < CLIENT_HASH_SIZE; i++) { + while (!list_empty(&nn->conf_id_hashtbl[i])) { + clp = list_entry(nn->conf_id_hashtbl[i].next, struct nfs4_client, cl_idhash); + destroy_client(clp); + } + } + + WARN_ON(!list_empty(&nn->blocked_locks_lru)); + + for (i = 0; i < CLIENT_HASH_SIZE; i++) { + while (!list_empty(&nn->unconf_id_hashtbl[i])) { + clp = list_entry(nn->unconf_id_hashtbl[i].next, struct nfs4_client, cl_idhash); + destroy_client(clp); + } + } + + kfree(nn->sessionid_hashtbl); + kfree(nn->unconf_id_hashtbl); + kfree(nn->conf_id_hashtbl); + put_net(net); +} + +int +nfs4_state_start_net(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + int ret; + + ret = nfs4_state_create_net(net); + if (ret) + return ret; + locks_start_grace(net, &nn->nfsd4_manager); + nfsd4_client_tracking_init(net); + /* safe for laundromat to run now */ + enable_delayed_work(&nn->laundromat_work); + if (test_bit(NFSD_NET_TRACK_RECLAIM_COMPLETES, &nn->flags) && + nn->reclaim_str_hashtbl_size == 0) + goto skip_grace; + printk(KERN_INFO "NFSD: starting %lld-second grace period (net %x)\n", + nn->nfsd4_grace, net->ns.inum); + trace_nfsd_grace_start(nn); + queue_delayed_work(laundry_wq, &nn->laundromat_work, nn->nfsd4_grace * HZ); + return 0; + +skip_grace: + printk(KERN_INFO "NFSD: no clients to reclaim, skipping NFSv4 grace period (net %x)\n", + net->ns.inum); + queue_delayed_work(laundry_wq, &nn->laundromat_work, nn->nfsd4_lease * HZ); + nfsd4_end_grace(nn); + return 0; +} + +/* initialization to perform when the nfsd service is started: */ +int +nfs4_state_start(void) +{ + int ret; + + ret = rhltable_init(&nfs4_file_rhltable, &nfs4_file_rhash_params); + if (ret) + return ret; + + nfsd_slot_shrinker = shrinker_alloc(0, "nfsd-DRC-slot"); + if (!nfsd_slot_shrinker) { + rhltable_destroy(&nfs4_file_rhltable); + return -ENOMEM; + } + nfsd_slot_shrinker->count_objects = nfsd_slot_shrinker_count; + nfsd_slot_shrinker->scan_objects = nfsd_slot_shrinker_scan; + shrinker_register(nfsd_slot_shrinker); + + set_max_delegations(); + return 0; +} + +void +nfs4_state_shutdown_net(struct net *net) +{ + struct nfs4_delegation *dp = NULL; + struct list_head *pos, *next, reaplist; + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + shrinker_free(nn->nfsd_client_shrinker); + cancel_work_sync(&nn->nfsd_shrinker_work); + disable_delayed_work_sync(&nn->laundromat_work); + locks_end_grace(&nn->nfsd4_manager); + + INIT_LIST_HEAD(&reaplist); + spin_lock(&nn->deleg_lock); + list_for_each_safe(pos, next, &nn->del_recall_lru) { + dp = list_entry (pos, struct nfs4_delegation, dl_recall_lru); + unhash_delegation_locked(dp, SC_STATUS_CLOSED); + list_add(&dp->dl_recall_lru, &reaplist); + } + spin_unlock(&nn->deleg_lock); + list_for_each_safe(pos, next, &reaplist) { + dp = list_entry (pos, struct nfs4_delegation, dl_recall_lru); + list_del_init(&dp->dl_recall_lru); + destroy_unhashed_deleg(dp); + } + + nfsd4_client_tracking_exit(net); + nfs4_state_destroy_net(net); +#ifdef CONFIG_NFSD_V4_2_INTER_SSC + nfsd4_ssc_shutdown_umount(nn); +#endif +} + +void +nfs4_state_shutdown(void) +{ + rhltable_destroy(&nfs4_file_rhltable); + shrinker_free(nfsd_slot_shrinker); +} + +static void +get_stateid(struct nfsd4_compound_state *cstate, stateid_t *stateid) +{ + if (HAS_CSTATE_FLAG(cstate, CURRENT_STATE_ID_FLAG) && + CURRENT_STATEID(stateid)) + memcpy(stateid, &cstate->current_stateid, sizeof(stateid_t)); +} + +static void +put_stateid(struct nfsd4_compound_state *cstate, stateid_t *stateid) +{ + if (cstate->minorversion) { + memcpy(&cstate->current_stateid, stateid, sizeof(stateid_t)); + SET_CSTATE_FLAG(cstate, CURRENT_STATE_ID_FLAG); + } +} + +void +clear_current_stateid(struct nfsd4_compound_state *cstate) +{ + CLEAR_CSTATE_FLAG(cstate, CURRENT_STATE_ID_FLAG); +} + +/* + * functions to set current state id + */ +void +nfsd4_set_opendowngradestateid(struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + put_stateid(cstate, &u->open_downgrade.od_stateid); +} + +void +nfsd4_set_openstateid(struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + put_stateid(cstate, &u->open.op_stateid); +} + +void +nfsd4_set_closestateid(struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + put_stateid(cstate, &u->close.cl_stateid); +} + +void +nfsd4_set_lockstateid(struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + put_stateid(cstate, &u->lock.lk_resp_stateid); +} + +/* + * functions to consume current state id + */ + +void +nfsd4_get_opendowngradestateid(struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + get_stateid(cstate, &u->open_downgrade.od_stateid); +} + +void +nfsd4_get_delegreturnstateid(struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + get_stateid(cstate, &u->delegreturn.dr_stateid); +} + +void +nfsd4_get_freestateid(struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + get_stateid(cstate, &u->free_stateid.fr_stateid); +} + +void +nfsd4_get_setattrstateid(struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + get_stateid(cstate, &u->setattr.sa_stateid); +} + +void +nfsd4_get_closestateid(struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + get_stateid(cstate, &u->close.cl_stateid); +} + +void +nfsd4_get_lockustateid(struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + get_stateid(cstate, &u->locku.lu_stateid); +} + +void +nfsd4_get_readstateid(struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + get_stateid(cstate, &u->read.rd_stateid); +} + +void +nfsd4_get_writestateid(struct nfsd4_compound_state *cstate, + union nfsd4_op_u *u) +{ + get_stateid(cstate, &u->write.wr_stateid); +} + +/** + * nfsd4_vet_deleg_time - vet and set the timespec for a delegated timestamp update + * @req: timestamp from the client + * @orig: original timestamp in the inode + * @now: current time + * + * Given a timestamp from the client response, check it against the + * current timestamp in the inode and the current time. Returns true + * if the inode's timestamp needs to be updated, and false otherwise. + * @req may also be changed if the timestamp needs to be clamped. + */ +bool nfsd4_vet_deleg_time(struct timespec64 *req, const struct timespec64 *orig, + const struct timespec64 *now) +{ + + /* + * "When the time presented is before the original time, then the + * update is ignored." Also no need to update if there is no change. + */ + if (timespec64_compare(req, orig) <= 0) + return false; + + /* + * "When the time presented is in the future, the server can either + * clamp the new time to the current time, or it may + * return NFS4ERR_DELAY to the client, allowing it to retry." + */ + if (timespec64_compare(req, now) > 0) + *req = *now; + + return true; +} + +static int cb_getattr_update_times(struct dentry *dentry, struct nfs4_delegation *dp) +{ + struct inode *inode = d_inode(dentry); + struct nfs4_cb_fattr *ncf = &dp->dl_cb_fattr; + struct iattr attrs = { }; + int ret; + + if (deleg_attrs_deleg(dp->dl_type)) { + struct timespec64 now = current_time(inode); + + attrs.ia_atime = ncf->ncf_cb_atime; + attrs.ia_mtime = ncf->ncf_cb_mtime; + + if (nfsd4_vet_deleg_time(&attrs.ia_atime, &dp->dl_atime, &now)) + attrs.ia_valid |= ATTR_ATIME | ATTR_ATIME_SET; + + if (nfsd4_vet_deleg_time(&attrs.ia_mtime, &dp->dl_mtime, &now)) { + attrs.ia_valid |= ATTR_MTIME | ATTR_MTIME_SET; + attrs.ia_ctime = attrs.ia_mtime; + if (nfsd4_vet_deleg_time(&attrs.ia_ctime, &dp->dl_ctime, &now)) + attrs.ia_valid |= ATTR_CTIME | ATTR_CTIME_SET; + } + } else { + attrs.ia_valid |= ATTR_MTIME | ATTR_CTIME; + } + + if (!attrs.ia_valid) + return 0; + + attrs.ia_valid |= ATTR_DELEG; + inode_lock(inode); + ret = notify_change(&nop_mnt_idmap, dentry, &attrs, NULL); + inode_unlock(inode); + return ret; +} + +/** + * nfsd4_deleg_getattr_conflict - Recall if GETATTR causes conflict + * @rqstp: RPC transaction context + * @dentry: dentry of inode to be checked for a conflict + * @pdp: returned WRITE delegation, if one was found + * + * This function is called when there is a conflict between a write + * delegation and a change/size GETATTR from another client. The server + * must either use the CB_GETATTR to get the current values of the + * attributes from the client that holds the delegation or recall the + * delegation before replying to the GETATTR. See RFC 8881 section + * 18.7.4. + * + * Returns 0 if there is no conflict; otherwise an nfs_stat + * code is returned. If @pdp is set to a non-NULL value, then the + * caller must put the reference. + */ +__be32 +nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp, struct dentry *dentry, + struct nfs4_delegation **pdp) +{ + struct nfsd_thread_local_info *ntli = rqstp->rq_private; + struct file_lock_context *ctx; + struct nfs4_delegation *dp = NULL; + struct file_lease *fl; + struct nfs4_cb_fattr *ncf; + struct inode *inode = d_inode(dentry); + __be32 status; + + ctx = locks_inode_context(inode); + if (!ctx) + return nfs_ok; + +#define NON_NFSD_LEASE ((void *)1) + + spin_lock(&ctx->flc_lock); + for_each_file_lock(fl, &ctx->flc_lease) { + if (fl->c.flc_flags == FL_LAYOUT) + continue; + if (fl->c.flc_type == F_WRLCK) { + if (fl->fl_lmops == &nfsd_lease_mng_ops) + dp = fl->c.flc_owner; + else + dp = NON_NFSD_LEASE; + } + break; + } + if (dp == NULL || dp == NON_NFSD_LEASE || + dp->dl_recall.cb_clp == *(ntli->ntli_lease_breaker)) { + spin_unlock(&ctx->flc_lock); + if (dp == NON_NFSD_LEASE) { + status = nfserrno(nfsd_open_break_lease(inode, + NFSD_MAY_READ)); + if (status != nfserr_jukebox || + !nfsd_wait_for_delegreturn(rqstp, inode)) + return status; + } + return 0; + } + + refcount_inc(&dp->dl_stid.sc_count); + ncf = &dp->dl_cb_fattr; + nfs4_cb_getattr(&dp->dl_cb_fattr); + spin_unlock(&ctx->flc_lock); + + wait_on_bit_timeout(&ncf->ncf_getattr.cb_flags, NFSD4_CALLBACK_RUNNING, + TASK_UNINTERRUPTIBLE, NFSD_CB_GETATTR_TIMEOUT); + if (ncf->ncf_cb_status) { + /* Recall delegation only if client didn't respond */ + status = nfserrno(nfsd_open_break_lease(inode, NFSD_MAY_READ)); + if (status != nfserr_jukebox || + !nfsd_wait_for_delegreturn(rqstp, inode)) + goto out_status; + status = nfs_ok; + goto out_status; + } + if (!ncf->ncf_file_modified) { + if (ncf->ncf_initial_cinfo != ncf->ncf_cb_change) + ncf->ncf_file_modified = true; + else if (i_size_read(inode) != ncf->ncf_cb_fsize) + ncf->ncf_file_modified = true; + } + if (ncf->ncf_file_modified) { + int err; + + /* + * Per section 10.4.3 of RFC 8881, the server would + * not update the file's metadata with the client's + * modified size + */ + err = cb_getattr_update_times(dentry, dp); + if (err) { + status = nfserrno(err); + goto out_status; + } + ncf->ncf_cur_fsize = ncf->ncf_cb_fsize; + *pdp = dp; + return nfs_ok; + } + status = nfs_ok; +out_status: + nfs4_put_stid(&dp->dl_stid); + return status; +} + +#define GDD_WORD0_CHILD_ATTRS (FATTR4_WORD0_TYPE | \ + FATTR4_WORD0_CHANGE | \ + FATTR4_WORD0_SIZE | \ + FATTR4_WORD0_FILEID | \ + FATTR4_WORD0_FILEHANDLE) + +#define GDD_WORD1_CHILD_ATTRS (FATTR4_WORD1_MODE | \ + FATTR4_WORD1_NUMLINKS | \ + FATTR4_WORD1_RAWDEV | \ + FATTR4_WORD1_SPACE_USED | \ + FATTR4_WORD1_TIME_ACCESS | \ + FATTR4_WORD1_TIME_METADATA | \ + FATTR4_WORD1_TIME_MODIFY | \ + FATTR4_WORD1_TIME_CREATE) + +#define GDD_WORD0_DIR_ATTRS (FATTR4_WORD0_CHANGE | \ + FATTR4_WORD0_SIZE) + +#define GDD_WORD1_DIR_ATTRS (FATTR4_WORD1_NUMLINKS | \ + FATTR4_WORD1_SPACE_USED | \ + FATTR4_WORD1_TIME_ACCESS | \ + FATTR4_WORD1_TIME_METADATA | \ + FATTR4_WORD1_TIME_MODIFY) + +/** + * nfsd_get_dir_deleg - attempt to get a directory delegation + * @cstate: compound state + * @gdd: GET_DIR_DELEGATION arg/resp structure + * @nf: nfsd_file opened on the directory + * + * Given a GET_DIR_DELEGATION request @gdd, attempt to acquire a delegation + * on the directory to which @nf refers. + */ +struct nfs4_delegation * +nfsd_get_dir_deleg(struct nfsd4_compound_state *cstate, + struct nfsd4_get_dir_delegation *gdd, + struct nfsd_file *nf) +{ + struct nfs4_client *clp = cstate->clp; + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + struct nfs4_delegation *dp; + struct file_lease *fl; + struct nfs4_file *fp, *rfp; + int status = 0; + + fp = nfsd4_alloc_file(); + if (!fp) + return ERR_PTR(-ENOMEM); + + nfsd4_file_init(&cstate->current_fh, fp); + + rfp = nfsd4_file_hash_insert(fp, &cstate->current_fh); + if (unlikely(!rfp)) { + put_nfs4_file(fp); + return ERR_PTR(-ENOMEM); + } + + if (rfp != fp) { + put_nfs4_file(fp); + fp = rfp; + } + + /* if this client already has one, return that it's unavailable */ + spin_lock(&nn->deleg_lock); + spin_lock(&fp->fi_lock); + /* existing delegation? */ + if (nfs4_delegation_exists(clp, fp)) { + status = -EAGAIN; + } else if (!rcu_dereference_protected(fp->fi_deleg_file, + lockdep_is_held(&fp->fi_lock))) { + rcu_assign_pointer(fp->fi_deleg_file, nfsd_file_get(nf)); + fp->fi_delegees = 1; + } else { + ++fp->fi_delegees; + } + spin_unlock(&fp->fi_lock); + spin_unlock(&nn->deleg_lock); + + if (status) { + put_nfs4_file(fp); + return ERR_PTR(status); + } + + /* Try to set up the lease */ + status = -ENOMEM; + dp = alloc_init_dir_deleg(clp, fp); + if (!dp) + goto out_delegees; + if (cstate->current_fh.fh_export) + dp->dl_stid.sc_export = + exp_get(cstate->current_fh.fh_export); + + /* + * NB: gddr_notification[0] represents the notifications that + * will be granted to the client + */ + dp->dl_notify_mask = gdd->gddr_notification[0]; + dp->dl_child_attrs[0] = gdd->gdda_child_attributes[0] & GDD_WORD0_CHILD_ATTRS; + dp->dl_child_attrs[1] = gdd->gdda_child_attributes[1] & GDD_WORD1_CHILD_ATTRS; + dp->dl_dir_attrs[0] = gdd->gdda_dir_attributes[0] & GDD_WORD0_DIR_ATTRS; + dp->dl_dir_attrs[1] = gdd->gdda_dir_attributes[1] & GDD_WORD1_DIR_ATTRS; + + fl = nfs4_alloc_init_lease(dp, dp->dl_notify_mask); + if (!fl) + goto out_put_stid; + + status = kernel_setlease(nf->nf_file, + fl->c.flc_type, &fl, NULL); + if (fl) + locks_free_lease(fl); + if (status) + goto out_put_stid; + + /* + * Now, try to hash it. This can fail if we race another nfsd task + * trying to set a delegation on the same file. If that happens, + * then just say UNAVAIL. + */ + spin_lock(&nn->deleg_lock); + spin_lock(&clp->cl_lock); + spin_lock(&fp->fi_lock); + status = hash_delegation_locked(dp, fp); + spin_unlock(&fp->fi_lock); + spin_unlock(&clp->cl_lock); + spin_unlock(&nn->deleg_lock); + + if (!status) { + put_nfs4_file(fp); + nfsd_fsnotify_recalc_mask(nf); + return dp; + } + + /* + * Something failed after the lease was set. Drop the lease and clean + * up the stid. The lease's flc_file is the fi_deleg_file (see + * nfs4_alloc_init_lease()), which is not necessarily this client's + * @nf when an earlier client already holds a delegation on @fp. + * generic_delete_lease() matches on flc_file, so unlock against + * fi_deleg_file or the lease will be leaked (and later freed with the + * stid, leading to a use-after-free when it's eventually broken). + */ + kernel_setlease(rcu_dereference_protected(fp->fi_deleg_file, 1)->nf_file, + F_UNLCK, NULL, (void **)&dp); + nfsd_fsnotify_recalc_mask(nf); +out_put_stid: + nfs4_put_stid(&dp->dl_stid); +out_delegees: + put_deleg_file(fp); + put_nfs4_file(fp); + return ERR_PTR(status); +} + +/** + * nfsd_update_cmtime_attr - update file's delegated ctime/mtime, + * and optionally other attributes (ie ATTR_ATIME). + * @f: pointer to an opened file + * @flags: any additional flags that should be updated + * + * Given upon opening a file delegated attributes were issues, update + * @f attributes to current times. + */ +void nfsd_update_cmtime_attr(struct file *f, unsigned int flags) +{ + int ret; + struct inode *inode = file_inode(f); + struct iattr attr = { + .ia_valid = ATTR_CTIME | ATTR_MTIME | ATTR_DELEG | flags, + }; + + inode_lock(inode); + ret = notify_change(&nop_mnt_idmap, f->f_path.dentry, &attr, NULL); + inode_unlock(inode); + if (ret) + pr_notice_ratelimited("nfsd: Unable to update timestamps on " + "inode %02x:%02x:%llu: %d\n", + MAJOR(inode->i_sb->s_dev), + MINOR(inode->i_sb->s_dev), + inode->i_ino, ret); +} + +static void +nfsd4_run_cb_notify(struct nfsd4_cb_notify *ncn) +{ + struct nfs4_delegation *dp = container_of(ncn, struct nfs4_delegation, dl_cb_notify); + + if (test_and_set_bit(NFSD4_CALLBACK_RUNNING, &ncn->ncn_cb.cb_flags)) + return; + + if (!refcount_inc_not_zero(&dp->dl_stid.sc_count)) + clear_bit(NFSD4_CALLBACK_RUNNING, &ncn->ncn_cb.cb_flags); + else + nfsd4_run_cb(&ncn->ncn_cb); +} + +static struct nfsd_notify_event * +alloc_nfsd_notify_event(u32 mask, const struct qstr *q, struct dentry *dentry, + struct inode *target) +{ + struct nfsd_notify_event *ne; + struct name_snapshot newname; + u32 newnamelen = 0; + + /* + * For a rename, @q is the old name and the live dentry carries the new + * name. Snapshot the new name now, while it is guaranteed to describe + * this event: the dentry can be renamed again before the CB_NOTIFY work + * runs, which would corrupt a late read in nfsd4_encode_notify_event(). + */ + if (mask & FS_RENAME) { + take_dentry_name_snapshot(&newname, dentry); + newnamelen = newname.name.len; + } + + ne = kmalloc(struct_size(ne, ne_name, q->len + 1 + + (newnamelen ? newnamelen + 1 : 0)), GFP_NOFS); + if (!ne) + goto out; + + memcpy(ne->ne_name, q->name, q->len); + ne->ne_name[q->len] = '\0'; + ne->ne_namelen = q->len; + + ne->ne_newnamelen = newnamelen; + if (newnamelen) { + char *p = nfsd_notify_event_newname(ne); + + memcpy(p, newname.name.name, newnamelen); + p[newnamelen] = '\0'; + } + + refcount_set(&ne->ne_ref, 1); + ne->ne_mask = mask; + ne->ne_dentry = dget(dentry); + ne->ne_target = target; + if (ne->ne_target) + ihold(ne->ne_target); +out: + if (mask & FS_RENAME) + release_dentry_name_snapshot(&newname); + return ne; +} + +static bool +should_notify_deleg(u32 mask, struct file_lease *fl) +{ + /* Don't notify the client generating the event */ + if (nfsd_breaker_owns_lease(fl)) + return false; + + /* Skip if this event wasn't ignored by the lease */ + if ((mask & FS_DELETE) && !(fl->c.flc_flags & FL_IGN_DIR_DELETE)) + return false; + if ((mask & FS_CREATE) && !(fl->c.flc_flags & FL_IGN_DIR_CREATE)) + return false; + if ((mask & FS_RENAME) && !(fl->c.flc_flags & FL_IGN_DIR_RENAME)) + return false; + + return true; +} + +static void +nfsd_recall_all_dir_delegs(const struct inode *dir) +{ + struct file_lock_context *ctx = locks_inode_context(dir); + struct file_lock_core *flc; + + spin_lock(&ctx->flc_lock); + list_for_each_entry(flc, &ctx->flc_lease, flc_list) { + struct file_lease *fl = container_of(flc, struct file_lease, c); + + if (fl->fl_lmops == &nfsd_lease_mng_ops) + nfsd_break_deleg_cb(fl); + } + spin_unlock(&ctx->flc_lock); +} + +int +nfsd_handle_dir_event(u32 mask, const struct inode *dir, const void *data, + int data_type, const struct qstr *name) +{ + struct dentry *dentry = fsnotify_data_dentry(data, data_type); + struct inode *target = fsnotify_data_rename_target(data, data_type); + struct file_lock_context *ctx; + struct file_lock_core *flc; + struct nfsd_notify_event *evt; + + trace_nfsd_handle_dir_event(mask, dir, name); + + /* Normalize cross-dir rename events to create/delete */ + if (mask & FS_MOVED_FROM) { + mask &= ~FS_MOVED_FROM; + mask |= FS_DELETE; + } + if (mask & FS_MOVED_TO) { + mask &= ~FS_MOVED_TO; + mask |= FS_CREATE; + } + + /* + * FS_RENAME fires on the source directory even for a cross-dir + * rename, where the moved entry now lives under a different parent. + * NOTIFY4_RENAME_ENTRY describes an in-place rename, so reporting it + * here would advertise a name absent from this directory. + */ + if ((mask & FS_RENAME) && dentry && d_inode(dentry->d_parent) != dir) + mask &= ~FS_RENAME; + + /* Don't do anything if this is not an expected event */ + if (!(mask & (FS_CREATE|FS_DELETE|FS_RENAME))) + return 0; + + ctx = locks_inode_context(dir); + if (!ctx || list_empty(&ctx->flc_lease)) + return 0; + + evt = alloc_nfsd_notify_event(mask, name, dentry, target); + if (!evt) { + nfsd_recall_all_dir_delegs(dir); + return 0; + } + + spin_lock(&ctx->flc_lock); + list_for_each_entry(flc, &ctx->flc_lease, flc_list) { + struct file_lease *fl = container_of(flc, struct file_lease, c); + struct nfs4_delegation *dp = flc->flc_owner; + struct nfsd4_cb_notify *ncn = &dp->dl_cb_notify; + + if (!should_notify_deleg(mask, fl)) + continue; + + spin_lock(&ncn->ncn_lock); + if (ncn->ncn_evt_cnt >= NOTIFY4_EVENT_QUEUE_SIZE) { + /* We're generating notifications too fast. Recall. */ + spin_unlock(&ncn->ncn_lock); + nfsd_break_deleg_cb(fl); + continue; + } + ncn->ncn_evt[ncn->ncn_evt_cnt++] = nfsd_notify_event_get(evt); + spin_unlock(&ncn->ncn_lock); + + nfsd4_run_cb_notify(ncn); + } + spin_unlock(&ctx->flc_lock); + nfsd_notify_event_put(evt); + return 0; +} diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c new file mode 100644 index 000000000..606ddcb08 --- /dev/null +++ b/fs/nfsd/nfs4xdr.c @@ -0,0 +1,6810 @@ +/* + * Server-side XDR for NFSv4 + * + * Copyright (c) 2002 The Regents of the University of Michigan. + * All rights reserved. + * + * Kendrick Smith + * Andy Adamson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "attr4.h" +#include "auth.h" +#include "idmap.h" +#include "acl.h" +#include "xdr4.h" +#include "vfs.h" +#include "state.h" +#include "cache.h" +#include "netns.h" +#include "pnfs.h" +#include "filecache.h" +#include "nfs4xdr_gen.h" + +#include "trace.h" + +#ifdef CONFIG_NFSD_V4_SECURITY_LABEL +#include +#endif + + +#define NFSDDBG_FACILITY NFSDDBG_XDR + +const u32 nfsd_suppattrs[3][3] = { + {NFSD4_SUPPORTED_ATTRS_WORD0, + NFSD4_SUPPORTED_ATTRS_WORD1, + NFSD4_SUPPORTED_ATTRS_WORD2}, + + {NFSD4_1_SUPPORTED_ATTRS_WORD0, + NFSD4_1_SUPPORTED_ATTRS_WORD1, + NFSD4_1_SUPPORTED_ATTRS_WORD2}, + + {NFSD4_1_SUPPORTED_ATTRS_WORD0, + NFSD4_1_SUPPORTED_ATTRS_WORD1, + NFSD4_2_SUPPORTED_ATTRS_WORD2}, +}; + +/* + * As per referral draft, the fsid for a referral MUST be different from the fsid of the containing + * directory in order to indicate to the client that a filesystem boundary is present + * We use a fixed fsid for a referral + */ +#define NFS4_REFERRAL_FSID_MAJOR 0x8000000ULL +#define NFS4_REFERRAL_FSID_MINOR 0x8000000ULL + +static __be32 +check_filename(char *str, int len) +{ + int i; + + if (len == 0) + return nfserr_inval; + if (len > NFS4_MAXNAMLEN) + return nfserr_nametoolong; + if (name_is_dot_dotdot(str, len)) + return nfserr_badname; + for (i = 0; i < len; i++) + if (str[i] == '/') + return nfserr_badname; + return 0; +} + +static int zero_clientid(clientid_t *clid) +{ + return (clid->cl_boot == 0) && (clid->cl_id == 0); +} + +/** + * svcxdr_tmpalloc - allocate memory to be freed after compound processing + * @argp: NFSv4 compound argument structure + * @len: length of buffer to allocate + * + * Allocates a buffer of size @len to be freed when processing the compound + * operation described in @argp finishes. + */ +static void * +svcxdr_tmpalloc(struct nfsd4_compoundargs *argp, size_t len) +{ + struct svcxdr_tmpbuf *tb; + + tb = kmalloc_flex(*tb, buf, len); + if (!tb) + return NULL; + tb->next = argp->to_free; + argp->to_free = tb; + return tb->buf; +} + +/* + * For xdr strings that need to be passed to other kernel api's + * as null-terminated strings. + * + * Note null-terminating in place usually isn't safe since the + * buffer might end on a page boundary. + */ +static char * +svcxdr_dupstr(struct nfsd4_compoundargs *argp, void *buf, size_t len) +{ + char *p = svcxdr_tmpalloc(argp, size_add(len, 1)); + + if (!p) + return NULL; + memcpy(p, buf, len); + p[len] = '\0'; + return p; +} + +static void * +svcxdr_savemem(struct nfsd4_compoundargs *argp, __be32 *p, size_t len) +{ + __be32 *tmp; + + /* + * The location of the decoded data item is stable, + * so @p is OK to use. This is the common case. + */ + if (p != argp->xdr->scratch.iov_base) + return p; + + tmp = svcxdr_tmpalloc(argp, len); + if (!tmp) + return NULL; + memcpy(tmp, p, len); + return tmp; +} + +/* + * NFSv4 basic data type decoders + */ + +/* + * This helper handles variable-length opaques which belong to protocol + * elements that this implementation does not support. + */ +static __be32 +nfsd4_decode_ignored_string(struct nfsd4_compoundargs *argp, u32 maxlen) +{ + u32 len; + + if (xdr_stream_decode_u32(argp->xdr, &len) < 0) + return nfserr_bad_xdr; + if (maxlen && len > maxlen) + return nfserr_bad_xdr; + if (!xdr_inline_decode(argp->xdr, len)) + return nfserr_bad_xdr; + + return nfs_ok; +} + +static __be32 +nfsd4_decode_opaque(struct nfsd4_compoundargs *argp, struct xdr_netobj *o) +{ + __be32 *p; + u32 len; + + if (xdr_stream_decode_u32(argp->xdr, &len) < 0) + return nfserr_bad_xdr; + if (len == 0 || len > NFS4_OPAQUE_LIMIT) + return nfserr_bad_xdr; + p = xdr_inline_decode(argp->xdr, len); + if (!p) + return nfserr_bad_xdr; + o->data = svcxdr_savemem(argp, p, len); + if (!o->data) + return nfserr_jukebox; + o->len = len; + + return nfs_ok; +} + +static __be32 +nfsd4_decode_component4(struct nfsd4_compoundargs *argp, char **namp, u32 *lenp) +{ + __be32 *p, status; + + if (xdr_stream_decode_u32(argp->xdr, lenp) < 0) + return nfserr_bad_xdr; + p = xdr_inline_decode(argp->xdr, *lenp); + if (!p) + return nfserr_bad_xdr; + status = check_filename((char *)p, *lenp); + if (status) + return status; + *namp = svcxdr_savemem(argp, p, *lenp); + if (!*namp) + return nfserr_jukebox; + + return nfs_ok; +} + +static __be32 +nfsd4_decode_nfstime4(struct nfsd4_compoundargs *argp, struct timespec64 *tv) +{ + __be32 *p; + + p = xdr_inline_decode(argp->xdr, XDR_UNIT * 3); + if (!p) + return nfserr_bad_xdr; + p = xdr_decode_hyper(p, &tv->tv_sec); + tv->tv_nsec = be32_to_cpup(p++); + if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC) + return nfserr_inval; + return nfs_ok; +} + +static __be32 +nfsd4_decode_verifier4(struct nfsd4_compoundargs *argp, nfs4_verifier *verf) +{ + __be32 *p; + + p = xdr_inline_decode(argp->xdr, NFS4_VERIFIER_SIZE); + if (!p) + return nfserr_bad_xdr; + memcpy(verf->data, p, sizeof(verf->data)); + return nfs_ok; +} + +/** + * nfsd4_decode_bitmap4 - Decode an NFSv4 bitmap4 + * @argp: NFSv4 compound argument structure + * @bmval: pointer to an array of u32's to decode into + * @bmlen: size of the @bmval array + * + * The server needs to return nfs_ok rather than nfserr_bad_xdr when + * encountering bitmaps containing bits it does not recognize. This + * includes bits in bitmap words past WORDn, where WORDn is the last + * bitmap WORD the implementation currently supports. Thus we are + * careful here to simply ignore bits in bitmap words that this + * implementation has yet to support explicitly. + * + * Return values: + * %nfs_ok: @bmval populated successfully + * %nfserr_bad_xdr: the encoded bitmap was invalid + */ +static __be32 +nfsd4_decode_bitmap4(struct nfsd4_compoundargs *argp, u32 *bmval, u32 bmlen) +{ + ssize_t status; + + status = xdr_stream_decode_uint32_array(argp->xdr, bmval, bmlen); + return status == -EBADMSG ? nfserr_bad_xdr : nfs_ok; +} + +static __be32 +nfsd4_decode_nfsace4(struct nfsd4_compoundargs *argp, struct nfs4_ace *ace) +{ + __be32 *p, status; + u32 length; + + if (xdr_stream_decode_u32(argp->xdr, &ace->type) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u32(argp->xdr, &ace->flag) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u32(argp->xdr, &ace->access_mask) < 0) + return nfserr_bad_xdr; + + if (xdr_stream_decode_u32(argp->xdr, &length) < 0) + return nfserr_bad_xdr; + p = xdr_inline_decode(argp->xdr, length); + if (!p) + return nfserr_bad_xdr; + ace->whotype = nfs4_acl_get_whotype((char *)p, length); + if (ace->whotype != NFS4_ACL_WHO_NAMED) + status = nfs_ok; + else if (ace->flag & NFS4_ACE_IDENTIFIER_GROUP) + status = nfsd_map_name_to_gid(argp->rqstp, + (char *)p, length, &ace->who_gid); + else + status = nfsd_map_name_to_uid(argp->rqstp, + (char *)p, length, &ace->who_uid); + + return status; +} + +/* A counted array of nfsace4's */ +static noinline __be32 +nfsd4_decode_acl(struct nfsd4_compoundargs *argp, struct nfs4_acl **acl) +{ + struct nfs4_ace *ace; + __be32 status; + u32 count; + + if (xdr_stream_decode_u32(argp->xdr, &count) < 0) + return nfserr_bad_xdr; + + if (count > xdr_stream_remaining(argp->xdr) / 20) + /* + * Even with 4-byte names there wouldn't be + * space for that many aces; something fishy is + * going on: + */ + return nfserr_fbig; + + *acl = svcxdr_tmpalloc(argp, nfs4_acl_bytes(count)); + if (*acl == NULL) + return nfserr_jukebox; + + (*acl)->naces = count; + for (ace = (*acl)->aces; ace < (*acl)->aces + count; ace++) { + status = nfsd4_decode_nfsace4(argp, ace); + if (status) + return status; + } + + return nfs_ok; +} + +static noinline __be32 +nfsd4_decode_security_label(struct nfsd4_compoundargs *argp, + struct xdr_netobj *label) +{ + u32 lfs, pi, length; + __be32 *p; + + if (xdr_stream_decode_u32(argp->xdr, &lfs) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u32(argp->xdr, &pi) < 0) + return nfserr_bad_xdr; + + if (xdr_stream_decode_u32(argp->xdr, &length) < 0) + return nfserr_bad_xdr; + if (length > NFS4_MAXLABELLEN) + return nfserr_badlabel; + p = xdr_inline_decode(argp->xdr, length); + if (!p) + return nfserr_bad_xdr; + label->len = length; + label->data = svcxdr_dupstr(argp, p, length); + if (!label->data) + return nfserr_jukebox; + + return nfs_ok; +} + +#ifdef CONFIG_NFSD_V4_POSIX_ACLS + +static short nfsd4_posixacetag4_to_tag(posixacetag4 tag) +{ + switch (tag) { + case POSIXACE4_TAG_USER_OBJ: return ACL_USER_OBJ; + case POSIXACE4_TAG_GROUP_OBJ: return ACL_GROUP_OBJ; + case POSIXACE4_TAG_USER: return ACL_USER; + case POSIXACE4_TAG_GROUP: return ACL_GROUP; + case POSIXACE4_TAG_MASK: return ACL_MASK; + case POSIXACE4_TAG_OTHER: return ACL_OTHER; + } + return ACL_OTHER; +} + +static __be32 +nfsd4_decode_posixace4(struct nfsd4_compoundargs *argp, + struct posix_acl_entry *ace) +{ + posixaceperm4 perm; + __be32 *p, status; + posixacetag4 tag; + u32 len; + + if (!xdrgen_decode_posixacetag4(argp->xdr, &tag)) + return nfserr_bad_xdr; + ace->e_tag = nfsd4_posixacetag4_to_tag(tag); + + if (!xdrgen_decode_posixaceperm4(argp->xdr, &perm)) + return nfserr_bad_xdr; + if (perm & ~S_IRWXO) + return nfserr_bad_xdr; + ace->e_perm = perm; + + if (xdr_stream_decode_u32(argp->xdr, &len) < 0) + return nfserr_bad_xdr; + p = xdr_inline_decode(argp->xdr, len); + if (!p) + return nfserr_bad_xdr; + switch (tag) { + case POSIXACE4_TAG_USER: + if (len > 0) + status = nfsd_map_name_to_uid(argp->rqstp, + (char *)p, len, &ace->e_uid); + else + status = nfserr_bad_xdr; + break; + case POSIXACE4_TAG_GROUP: + if (len > 0) + status = nfsd_map_name_to_gid(argp->rqstp, + (char *)p, len, &ace->e_gid); + else + status = nfserr_bad_xdr; + break; + default: + status = nfs_ok; + } + + return status; +} + +static noinline __be32 +nfsd4_decode_posixacl(struct nfsd4_compoundargs *argp, struct posix_acl **acl) +{ + struct posix_acl_entry *ace; + __be32 status; + u32 count; + + if (xdr_stream_decode_u32(argp->xdr, &count) < 0) + return nfserr_bad_xdr; + + /* + * The NFSv4 POSIX ACL draft doesn't define a max number of ACE's, but + * the NFSACL spec does. For NFSv4, cap the number of entries to the v3 + * limit, as we want to ensure that ACLs set via NFSv4 POSIX ACL + * extensions are retrievable via NFSACL. + */ + if (count > NFS_ACL_MAX_ENTRIES) + return nfserr_inval; + + *acl = posix_acl_alloc(count, GFP_KERNEL); + if (*acl == NULL) + return nfserr_jukebox; + + (*acl)->a_count = count; + for (ace = (*acl)->a_entries; ace < (*acl)->a_entries + count; ace++) { + status = nfsd4_decode_posixace4(argp, ace); + if (status) { + posix_acl_release(*acl); + *acl = NULL; + return status; + } + } + + /* + * posix_acl_valid() requires the ACEs to be sorted. + * If they are already sorted, sort_pacl_range() will return + * after one pass through the ACEs, since it implements bubble sort. + * Note that a count == 0 is used to delete a POSIX ACL and a count + * of 1 or 2 will always be found invalid by posix_acl_valid(). + */ + if (count >= 3) + sort_pacl_range(*acl, 0, count - 1); + + return nfs_ok; +} + +#endif /* CONFIG_NFSD_V4_POSIX_ACLS */ + +static __be32 +nfsd4_decode_fattr4(struct nfsd4_compoundargs *argp, u32 *bmval, u32 bmlen, + struct iattr *iattr, struct nfs4_acl **acl, + struct xdr_netobj *label, int *umask, + struct posix_acl **dpaclp, struct posix_acl **paclp) +{ + unsigned int starting_pos; + u32 attrlist4_count; + __be32 *p, status; + + iattr->ia_valid = 0; + status = nfsd4_decode_bitmap4(argp, bmval, bmlen); + if (status) + return nfserr_bad_xdr; + + if (bmval[0] & ~NFSD_WRITEABLE_ATTRS_WORD0 + || bmval[1] & ~NFSD_WRITEABLE_ATTRS_WORD1 + || bmval[2] & ~NFSD_WRITEABLE_ATTRS_WORD2) { + if (nfsd_attrs_supported(argp->minorversion, bmval)) + return nfserr_inval; + return nfserr_attrnotsupp; + } + + if (xdr_stream_decode_u32(argp->xdr, &attrlist4_count) < 0) + return nfserr_bad_xdr; + starting_pos = xdr_stream_pos(argp->xdr); + + if (bmval[0] & FATTR4_WORD0_SIZE) { + u64 size; + + if (xdr_stream_decode_u64(argp->xdr, &size) < 0) + return nfserr_bad_xdr; + iattr->ia_size = size; + iattr->ia_valid |= ATTR_SIZE; + } + if (bmval[0] & FATTR4_WORD0_ACL) { + status = nfsd4_decode_acl(argp, acl); + if (status) + return status; + } else + *acl = NULL; + if (bmval[1] & FATTR4_WORD1_MODE) { + u32 mode; + + if (xdr_stream_decode_u32(argp->xdr, &mode) < 0) + return nfserr_bad_xdr; + iattr->ia_mode = mode; + iattr->ia_mode &= (S_IFMT | S_IALLUGO); + iattr->ia_valid |= ATTR_MODE; + } + if (bmval[1] & FATTR4_WORD1_OWNER) { + u32 length; + + if (xdr_stream_decode_u32(argp->xdr, &length) < 0) + return nfserr_bad_xdr; + p = xdr_inline_decode(argp->xdr, length); + if (!p) + return nfserr_bad_xdr; + status = nfsd_map_name_to_uid(argp->rqstp, (char *)p, length, + &iattr->ia_uid); + if (status) + return status; + iattr->ia_valid |= ATTR_UID; + } + if (bmval[1] & FATTR4_WORD1_OWNER_GROUP) { + u32 length; + + if (xdr_stream_decode_u32(argp->xdr, &length) < 0) + return nfserr_bad_xdr; + p = xdr_inline_decode(argp->xdr, length); + if (!p) + return nfserr_bad_xdr; + status = nfsd_map_name_to_gid(argp->rqstp, (char *)p, length, + &iattr->ia_gid); + if (status) + return status; + iattr->ia_valid |= ATTR_GID; + } + if (bmval[1] & FATTR4_WORD1_TIME_ACCESS_SET) { + u32 set_it; + + if (xdr_stream_decode_u32(argp->xdr, &set_it) < 0) + return nfserr_bad_xdr; + switch (set_it) { + case NFS4_SET_TO_CLIENT_TIME: + status = nfsd4_decode_nfstime4(argp, &iattr->ia_atime); + if (status) + return status; + iattr->ia_valid |= (ATTR_ATIME | ATTR_ATIME_SET); + break; + case NFS4_SET_TO_SERVER_TIME: + iattr->ia_valid |= ATTR_ATIME; + break; + default: + return nfserr_bad_xdr; + } + } + if (bmval[1] & FATTR4_WORD1_TIME_CREATE) { + struct timespec64 ts; + + /* No Linux filesystem supports setting this attribute. */ + bmval[1] &= ~FATTR4_WORD1_TIME_CREATE; + status = nfsd4_decode_nfstime4(argp, &ts); + if (status) + return status; + } + if (bmval[1] & FATTR4_WORD1_TIME_MODIFY_SET) { + u32 set_it; + + if (xdr_stream_decode_u32(argp->xdr, &set_it) < 0) + return nfserr_bad_xdr; + switch (set_it) { + case NFS4_SET_TO_CLIENT_TIME: + status = nfsd4_decode_nfstime4(argp, &iattr->ia_mtime); + if (status) + return status; + iattr->ia_valid |= (ATTR_MTIME | ATTR_MTIME_SET); + break; + case NFS4_SET_TO_SERVER_TIME: + iattr->ia_valid |= ATTR_MTIME; + break; + default: + return nfserr_bad_xdr; + } + } + label->len = 0; + if (IS_ENABLED(CONFIG_NFSD_V4_SECURITY_LABEL) && + bmval[2] & FATTR4_WORD2_SECURITY_LABEL) { + status = nfsd4_decode_security_label(argp, label); + if (status) + return status; + } + if (bmval[2] & FATTR4_WORD2_MODE_UMASK) { + u32 mode, mask; + + if (!umask) + return nfserr_bad_xdr; + if (xdr_stream_decode_u32(argp->xdr, &mode) < 0) + return nfserr_bad_xdr; + iattr->ia_mode = mode & (S_IFMT | S_IALLUGO); + if (xdr_stream_decode_u32(argp->xdr, &mask) < 0) + return nfserr_bad_xdr; + *umask = mask & S_IRWXUGO; + iattr->ia_valid |= ATTR_MODE; + } + if (bmval[2] & FATTR4_WORD2_TIME_DELEG_ACCESS) { + fattr4_time_deleg_access access; + + if (!xdrgen_decode_fattr4_time_deleg_access(argp->xdr, &access)) + return nfserr_bad_xdr; + if (access.nseconds >= NSEC_PER_SEC) + return nfserr_inval; + iattr->ia_atime.tv_sec = access.seconds; + iattr->ia_atime.tv_nsec = access.nseconds; + iattr->ia_valid |= ATTR_ATIME | ATTR_ATIME_SET | ATTR_DELEG; + } + if (bmval[2] & FATTR4_WORD2_TIME_DELEG_MODIFY) { + fattr4_time_deleg_modify modify; + + if (!xdrgen_decode_fattr4_time_deleg_modify(argp->xdr, &modify)) + return nfserr_bad_xdr; + if (modify.nseconds >= NSEC_PER_SEC) + return nfserr_inval; + iattr->ia_mtime.tv_sec = modify.seconds; + iattr->ia_mtime.tv_nsec = modify.nseconds; + iattr->ia_ctime.tv_sec = modify.seconds; + iattr->ia_ctime.tv_nsec = modify.nseconds; + iattr->ia_valid |= ATTR_CTIME | ATTR_CTIME_SET | + ATTR_MTIME | ATTR_MTIME_SET | ATTR_DELEG; + } + + *dpaclp = NULL; + *paclp = NULL; +#ifdef CONFIG_NFSD_V4_POSIX_ACLS + if (bmval[2] & FATTR4_WORD2_POSIX_DEFAULT_ACL) { + struct posix_acl *dpacl; + + status = nfsd4_decode_posixacl(argp, &dpacl); + if (status) + return status; + *dpaclp = dpacl; + } + if (bmval[2] & FATTR4_WORD2_POSIX_ACCESS_ACL) { + struct posix_acl *pacl; + + status = nfsd4_decode_posixacl(argp, &pacl); + if (status) { + posix_acl_release(*dpaclp); + *dpaclp = NULL; + return status; + } + *paclp = pacl; + } +#endif /* CONFIG_NFSD_V4_POSIX_ACLS */ + + /* request sanity: did attrlist4 contain the expected number of words? */ + if (attrlist4_count != xdr_stream_pos(argp->xdr) - starting_pos) { +#ifdef CONFIG_NFSD_V4_POSIX_ACLS + posix_acl_release(*dpaclp); + posix_acl_release(*paclp); + *dpaclp = NULL; + *paclp = NULL; +#endif + return nfserr_bad_xdr; + } + + return nfs_ok; +} + +static __be32 +nfsd4_decode_stateid4(struct nfsd4_compoundargs *argp, stateid_t *sid) +{ + __be32 *p; + + p = xdr_inline_decode(argp->xdr, NFS4_STATEID_SIZE); + if (!p) + return nfserr_bad_xdr; + sid->si_generation = be32_to_cpup(p++); + memcpy(&sid->si_opaque, p, sizeof(sid->si_opaque)); + return nfs_ok; +} + +static __be32 +nfsd4_decode_clientid4(struct nfsd4_compoundargs *argp, clientid_t *clientid) +{ + __be32 *p; + + p = xdr_inline_decode(argp->xdr, sizeof(__be64)); + if (!p) + return nfserr_bad_xdr; + memcpy(clientid, p, sizeof(*clientid)); + return nfs_ok; +} + +static __be32 +nfsd4_decode_state_owner4(struct nfsd4_compoundargs *argp, + clientid_t *clientid, struct xdr_netobj *owner) +{ + __be32 status; + + status = nfsd4_decode_clientid4(argp, clientid); + if (status) + return status; + return nfsd4_decode_opaque(argp, owner); +} + +#ifdef CONFIG_NFSD_PNFS + +static __be32 +nfsd4_decode_layoutupdate4(struct nfsd4_compoundargs *argp, + struct nfsd4_layoutcommit *lcp) +{ + u32 len; + + if (xdr_stream_decode_u32(argp->xdr, &lcp->lc_layout_type) < 0) + return nfserr_bad_xdr; + if (lcp->lc_layout_type < LAYOUT_NFSV4_1_FILES) + return nfserr_bad_xdr; + if (lcp->lc_layout_type >= LAYOUT_TYPE_MAX) + return nfserr_bad_xdr; + + if (xdr_stream_decode_u32(argp->xdr, &len) < 0) + return nfserr_bad_xdr; + if (!xdr_stream_subsegment(argp->xdr, &lcp->lc_up_layout, len)) + return nfserr_bad_xdr; + + return nfs_ok; +} + +static __be32 +nfsd4_decode_layoutreturn4(struct nfsd4_compoundargs *argp, + struct nfsd4_layoutreturn *lrp) +{ + __be32 status; + + if (xdr_stream_decode_u32(argp->xdr, &lrp->lr_return_type) < 0) + return nfserr_bad_xdr; + switch (lrp->lr_return_type) { + case RETURN_FILE: + if (xdr_stream_decode_u64(argp->xdr, &lrp->lr_seg.offset) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u64(argp->xdr, &lrp->lr_seg.length) < 0) + return nfserr_bad_xdr; + status = nfsd4_decode_stateid4(argp, &lrp->lr_sid); + if (status) + return status; + if (xdr_stream_decode_u32(argp->xdr, &lrp->lrf_body_len) < 0) + return nfserr_bad_xdr; + if (lrp->lrf_body_len > 0) { + lrp->lrf_body = xdr_inline_decode(argp->xdr, lrp->lrf_body_len); + if (!lrp->lrf_body) + return nfserr_bad_xdr; + } + break; + case RETURN_FSID: + case RETURN_ALL: + lrp->lr_seg.offset = 0; + lrp->lr_seg.length = NFS4_MAX_UINT64; + break; + default: + return nfserr_bad_xdr; + } + + return nfs_ok; +} + +#endif /* CONFIG_NFSD_PNFS */ + +static __be32 +nfsd4_decode_sessionid4(struct nfsd4_compoundargs *argp, + struct nfs4_sessionid *sessionid) +{ + __be32 *p; + + p = xdr_inline_decode(argp->xdr, NFS4_MAX_SESSIONID_LEN); + if (!p) + return nfserr_bad_xdr; + memcpy(sessionid->data, p, sizeof(sessionid->data)); + return nfs_ok; +} + +/* Defined in Appendix A of RFC 5531 */ +static __be32 +nfsd4_decode_authsys_parms(struct nfsd4_compoundargs *argp, + struct nfsd4_cb_sec *cbs) +{ + u32 stamp, gidcount, uid, gid; + __be32 *p, status; + + if (xdr_stream_decode_u32(argp->xdr, &stamp) < 0) + return nfserr_bad_xdr; + /* machine name */ + status = nfsd4_decode_ignored_string(argp, 255); + if (status) + return status; + if (xdr_stream_decode_u32(argp->xdr, &uid) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u32(argp->xdr, &gid) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u32(argp->xdr, &gidcount) < 0) + return nfserr_bad_xdr; + if (gidcount > 16) + return nfserr_bad_xdr; + p = xdr_inline_decode(argp->xdr, gidcount << 2); + if (!p) + return nfserr_bad_xdr; + if (cbs->flavor == (u32)(-1)) { + struct user_namespace *userns = nfsd_user_namespace(argp->rqstp); + + kuid_t kuid = make_kuid(userns, uid); + kgid_t kgid = make_kgid(userns, gid); + if (uid_valid(kuid) && gid_valid(kgid)) { + cbs->uid = kuid; + cbs->gid = kgid; + cbs->flavor = RPC_AUTH_UNIX; + } else { + dprintk("RPC_AUTH_UNIX with invalid uid or gid, ignoring!\n"); + } + } + + return nfs_ok; +} + +static __be32 +nfsd4_decode_gss_cb_handles4(struct nfsd4_compoundargs *argp, + struct nfsd4_cb_sec *cbs) +{ + __be32 status; + u32 service; + + dprintk("RPC_AUTH_GSS callback secflavor not supported!\n"); + + if (xdr_stream_decode_u32(argp->xdr, &service) < 0) + return nfserr_bad_xdr; + if (service < RPC_GSS_SVC_NONE || service > RPC_GSS_SVC_PRIVACY) + return nfserr_bad_xdr; + /* gcbp_handle_from_server */ + status = nfsd4_decode_ignored_string(argp, 0); + if (status) + return status; + /* gcbp_handle_from_client */ + status = nfsd4_decode_ignored_string(argp, 0); + if (status) + return status; + + return nfs_ok; +} + +/* a counted array of callback_sec_parms4 items */ +static __be32 +nfsd4_decode_cb_sec(struct nfsd4_compoundargs *argp, struct nfsd4_cb_sec *cbs) +{ + u32 i, secflavor, nr_secflavs; + __be32 status; + + /* callback_sec_params4 */ + if (xdr_stream_decode_u32(argp->xdr, &nr_secflavs) < 0) + return nfserr_bad_xdr; + if (nr_secflavs) + cbs->flavor = (u32)(-1); + else + /* Is this legal? Be generous, take it to mean AUTH_NONE: */ + cbs->flavor = 0; + + for (i = 0; i < nr_secflavs; ++i) { + if (xdr_stream_decode_u32(argp->xdr, &secflavor) < 0) + return nfserr_bad_xdr; + switch (secflavor) { + case RPC_AUTH_NULL: + /* void */ + if (cbs->flavor == (u32)(-1)) + cbs->flavor = RPC_AUTH_NULL; + break; + case RPC_AUTH_UNIX: + status = nfsd4_decode_authsys_parms(argp, cbs); + if (status) + return status; + break; + case RPC_AUTH_GSS: + status = nfsd4_decode_gss_cb_handles4(argp, cbs); + if (status) + return status; + break; + default: + return nfserr_inval; + } + } + + return nfs_ok; +} + + +/* + * NFSv4 operation argument decoders + */ + +static __be32 +nfsd4_decode_access(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_access *access = &u->access; + if (xdr_stream_decode_u32(argp->xdr, &access->ac_req_access) < 0) + return nfserr_bad_xdr; + return nfs_ok; +} + +static __be32 +nfsd4_decode_close(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) +{ + struct nfsd4_close *close = &u->close; + if (xdr_stream_decode_u32(argp->xdr, &close->cl_seqid) < 0) + return nfserr_bad_xdr; + return nfsd4_decode_stateid4(argp, &close->cl_stateid); +} + + +static __be32 +nfsd4_decode_commit(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) +{ + struct nfsd4_commit *commit = &u->commit; + if (xdr_stream_decode_u64(argp->xdr, &commit->co_offset) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u32(argp->xdr, &commit->co_count) < 0) + return nfserr_bad_xdr; + memset(&commit->co_verf, 0, sizeof(commit->co_verf)); + return nfs_ok; +} + +static __be32 +nfsd4_decode_create(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) +{ + struct nfsd4_create *create = &u->create; + __be32 *p, status; + + memset(create, 0, sizeof(*create)); + if (xdr_stream_decode_u32(argp->xdr, &create->cr_type) < 0) + return nfserr_bad_xdr; + switch (create->cr_type) { + case NF4LNK: + if (xdr_stream_decode_u32(argp->xdr, &create->cr_datalen) < 0) + return nfserr_bad_xdr; + if (create->cr_datalen == 0) + return nfserr_inval; + if (create->cr_datalen > NFS4_MAXPATHLEN) + return nfserr_nametoolong; + p = xdr_inline_decode(argp->xdr, create->cr_datalen); + if (!p) + return nfserr_bad_xdr; + create->cr_data = svcxdr_dupstr(argp, p, create->cr_datalen); + if (!create->cr_data) + return nfserr_jukebox; + break; + case NF4BLK: + case NF4CHR: + if (xdr_stream_decode_u32(argp->xdr, &create->cr_specdata1) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u32(argp->xdr, &create->cr_specdata2) < 0) + return nfserr_bad_xdr; + break; + case NF4SOCK: + case NF4FIFO: + case NF4DIR: + default: + break; + } + status = nfsd4_decode_component4(argp, &create->cr_name, + &create->cr_namelen); + if (status) + return status; + status = nfsd4_decode_fattr4(argp, create->cr_bmval, + ARRAY_SIZE(create->cr_bmval), + &create->cr_iattr, &create->cr_acl, + &create->cr_label, &create->cr_umask, + &create->cr_dpacl, &create->cr_pacl); + if (status) + return status; + + return nfs_ok; +} + +static inline __be32 +nfsd4_decode_delegreturn(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) +{ + struct nfsd4_delegreturn *dr = &u->delegreturn; + return nfsd4_decode_stateid4(argp, &dr->dr_stateid); +} + +static inline __be32 +nfsd4_decode_getattr(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) +{ + struct nfsd4_getattr *getattr = &u->getattr; + memset(getattr, 0, sizeof(*getattr)); + return nfsd4_decode_bitmap4(argp, getattr->ga_bmval, + ARRAY_SIZE(getattr->ga_bmval)); +} + +static __be32 +nfsd4_decode_link(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) +{ + struct nfsd4_link *link = &u->link; + memset(link, 0, sizeof(*link)); + return nfsd4_decode_component4(argp, &link->li_name, &link->li_namelen); +} + +static __be32 +nfsd4_decode_open_to_lock_owner4(struct nfsd4_compoundargs *argp, + struct nfsd4_lock *lock) +{ + __be32 status; + + if (xdr_stream_decode_u32(argp->xdr, &lock->lk_new_open_seqid) < 0) + return nfserr_bad_xdr; + status = nfsd4_decode_stateid4(argp, &lock->lk_new_open_stateid); + if (status) + return status; + if (xdr_stream_decode_u32(argp->xdr, &lock->lk_new_lock_seqid) < 0) + return nfserr_bad_xdr; + return nfsd4_decode_state_owner4(argp, &lock->lk_new_clientid, + &lock->lk_new_owner); +} + +static __be32 +nfsd4_decode_exist_lock_owner4(struct nfsd4_compoundargs *argp, + struct nfsd4_lock *lock) +{ + __be32 status; + + status = nfsd4_decode_stateid4(argp, &lock->lk_old_lock_stateid); + if (status) + return status; + if (xdr_stream_decode_u32(argp->xdr, &lock->lk_old_lock_seqid) < 0) + return nfserr_bad_xdr; + + return nfs_ok; +} + +static __be32 +nfsd4_decode_locker4(struct nfsd4_compoundargs *argp, struct nfsd4_lock *lock) +{ + if (xdr_stream_decode_bool(argp->xdr, &lock->lk_is_new) < 0) + return nfserr_bad_xdr; + if (lock->lk_is_new) + return nfsd4_decode_open_to_lock_owner4(argp, lock); + return nfsd4_decode_exist_lock_owner4(argp, lock); +} + +static __be32 +nfsd4_decode_lock(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) +{ + struct nfsd4_lock *lock = &u->lock; + memset(lock, 0, sizeof(*lock)); + if (xdr_stream_decode_u32(argp->xdr, &lock->lk_type) < 0) + return nfserr_bad_xdr; + if ((lock->lk_type < NFS4_READ_LT) || (lock->lk_type > NFS4_WRITEW_LT)) + return nfserr_bad_xdr; + if (xdr_stream_decode_bool(argp->xdr, &lock->lk_reclaim) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u64(argp->xdr, &lock->lk_offset) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u64(argp->xdr, &lock->lk_length) < 0) + return nfserr_bad_xdr; + return nfsd4_decode_locker4(argp, lock); +} + +static __be32 +nfsd4_decode_lockt(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) +{ + struct nfsd4_lockt *lockt = &u->lockt; + memset(lockt, 0, sizeof(*lockt)); + if (xdr_stream_decode_u32(argp->xdr, &lockt->lt_type) < 0) + return nfserr_bad_xdr; + if ((lockt->lt_type < NFS4_READ_LT) || (lockt->lt_type > NFS4_WRITEW_LT)) + return nfserr_bad_xdr; + if (xdr_stream_decode_u64(argp->xdr, &lockt->lt_offset) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u64(argp->xdr, &lockt->lt_length) < 0) + return nfserr_bad_xdr; + return nfsd4_decode_state_owner4(argp, &lockt->lt_clientid, + &lockt->lt_owner); +} + +static __be32 +nfsd4_decode_locku(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) +{ + struct nfsd4_locku *locku = &u->locku; + __be32 status; + + if (xdr_stream_decode_u32(argp->xdr, &locku->lu_type) < 0) + return nfserr_bad_xdr; + if ((locku->lu_type < NFS4_READ_LT) || (locku->lu_type > NFS4_WRITEW_LT)) + return nfserr_bad_xdr; + if (xdr_stream_decode_u32(argp->xdr, &locku->lu_seqid) < 0) + return nfserr_bad_xdr; + status = nfsd4_decode_stateid4(argp, &locku->lu_stateid); + if (status) + return status; + if (xdr_stream_decode_u64(argp->xdr, &locku->lu_offset) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u64(argp->xdr, &locku->lu_length) < 0) + return nfserr_bad_xdr; + + return nfs_ok; +} + +static __be32 +nfsd4_decode_lookup(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) +{ + struct nfsd4_lookup *lookup = &u->lookup; + return nfsd4_decode_component4(argp, &lookup->lo_name, &lookup->lo_len); +} + +static __be32 +nfsd4_decode_createhow4(struct nfsd4_compoundargs *argp, struct nfsd4_open *open) +{ + __be32 status; + + if (xdr_stream_decode_u32(argp->xdr, &open->op_createmode) < 0) + return nfserr_bad_xdr; + switch (open->op_createmode) { + case NFS4_CREATE_UNCHECKED: + case NFS4_CREATE_GUARDED: + status = nfsd4_decode_fattr4(argp, open->op_bmval, + ARRAY_SIZE(open->op_bmval), + &open->op_iattr, &open->op_acl, + &open->op_label, &open->op_umask, + &open->op_dpacl, &open->op_pacl); + if (status) + return status; + break; + case NFS4_CREATE_EXCLUSIVE: + status = nfsd4_decode_verifier4(argp, &open->op_verf); + if (status) + return status; + break; + case NFS4_CREATE_EXCLUSIVE4_1: + if (argp->minorversion < 1) + return nfserr_bad_xdr; + status = nfsd4_decode_verifier4(argp, &open->op_verf); + if (status) + return status; + status = nfsd4_decode_fattr4(argp, open->op_bmval, + ARRAY_SIZE(open->op_bmval), + &open->op_iattr, &open->op_acl, + &open->op_label, &open->op_umask, + &open->op_dpacl, &open->op_pacl); + if (status) + return status; + break; + default: + return nfserr_bad_xdr; + } + + return nfs_ok; +} + +static __be32 +nfsd4_decode_openflag4(struct nfsd4_compoundargs *argp, struct nfsd4_open *open) +{ + __be32 status; + + if (xdr_stream_decode_u32(argp->xdr, &open->op_create) < 0) + return nfserr_bad_xdr; + switch (open->op_create) { + case NFS4_OPEN_NOCREATE: + break; + case NFS4_OPEN_CREATE: + status = nfsd4_decode_createhow4(argp, open); + if (status) + return status; + break; + default: + return nfserr_bad_xdr; + } + + return nfs_ok; +} + +static __be32 nfsd4_decode_share_access(struct nfsd4_compoundargs *argp, u32 *share_access, u32 *deleg_want, u32 *deleg_when) +{ + u32 w; + + if (xdr_stream_decode_u32(argp->xdr, &w) < 0) + return nfserr_bad_xdr; + *share_access = w & NFS4_SHARE_ACCESS_MASK; + *deleg_want = w & NFS4_SHARE_WANT_MASK; + if (deleg_when) + *deleg_when = w & NFS4_SHARE_WHEN_MASK; + + switch (w & NFS4_SHARE_ACCESS_MASK) { + case NFS4_SHARE_ACCESS_READ: + case NFS4_SHARE_ACCESS_WRITE: + case NFS4_SHARE_ACCESS_BOTH: + break; + default: + return nfserr_bad_xdr; + } + w &= ~NFS4_SHARE_ACCESS_MASK; + if (!w) + return nfs_ok; + if (!argp->minorversion) + return nfserr_bad_xdr; + switch (w & NFS4_SHARE_WANT_TYPE_MASK) { + case OPEN4_SHARE_ACCESS_WANT_NO_PREFERENCE: + case OPEN4_SHARE_ACCESS_WANT_READ_DELEG: + case OPEN4_SHARE_ACCESS_WANT_WRITE_DELEG: + case OPEN4_SHARE_ACCESS_WANT_ANY_DELEG: + case OPEN4_SHARE_ACCESS_WANT_NO_DELEG: + case OPEN4_SHARE_ACCESS_WANT_CANCEL: + break; + default: + return nfserr_bad_xdr; + } + w &= ~NFS4_SHARE_WANT_MASK; + if (!w) + return nfs_ok; + + if (!deleg_when) /* open_downgrade */ + return nfserr_inval; + switch (w) { + case NFS4_SHARE_SIGNAL_DELEG_WHEN_RESRC_AVAIL: + case NFS4_SHARE_PUSH_DELEG_WHEN_UNCONTENDED: + case (NFS4_SHARE_SIGNAL_DELEG_WHEN_RESRC_AVAIL | + NFS4_SHARE_PUSH_DELEG_WHEN_UNCONTENDED): + return nfs_ok; + } + return nfserr_bad_xdr; +} + +static __be32 nfsd4_decode_share_deny(struct nfsd4_compoundargs *argp, u32 *x) +{ + if (xdr_stream_decode_u32(argp->xdr, x) < 0) + return nfserr_bad_xdr; + /* Note: unlike access bits, deny bits may be zero. */ + if (*x & ~NFS4_SHARE_DENY_BOTH) + return nfserr_bad_xdr; + + return nfs_ok; +} + +static __be32 +nfsd4_decode_open_claim4(struct nfsd4_compoundargs *argp, + struct nfsd4_open *open) +{ + __be32 status; + + if (xdr_stream_decode_u32(argp->xdr, &open->op_claim_type) < 0) + return nfserr_bad_xdr; + switch (open->op_claim_type) { + case NFS4_OPEN_CLAIM_NULL: + case NFS4_OPEN_CLAIM_DELEGATE_PREV: + status = nfsd4_decode_component4(argp, &open->op_fname, + &open->op_fnamelen); + if (status) + return status; + break; + case NFS4_OPEN_CLAIM_PREVIOUS: + if (xdr_stream_decode_u32(argp->xdr, &open->op_delegate_type) < 0) + return nfserr_bad_xdr; + break; + case NFS4_OPEN_CLAIM_DELEGATE_CUR: + status = nfsd4_decode_stateid4(argp, &open->op_delegate_stateid); + if (status) + return status; + status = nfsd4_decode_component4(argp, &open->op_fname, + &open->op_fnamelen); + if (status) + return status; + break; + case NFS4_OPEN_CLAIM_FH: + case NFS4_OPEN_CLAIM_DELEG_PREV_FH: + if (argp->minorversion < 1) + return nfserr_bad_xdr; + /* void */ + break; + case NFS4_OPEN_CLAIM_DELEG_CUR_FH: + if (argp->minorversion < 1) + return nfserr_bad_xdr; + status = nfsd4_decode_stateid4(argp, &open->op_delegate_stateid); + if (status) + return status; + break; + default: + return nfserr_bad_xdr; + } + + return nfs_ok; +} + +static __be32 +nfsd4_decode_open(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) +{ + struct nfsd4_open *open = &u->open; + __be32 status; + u32 dummy; + + memset(open, 0, sizeof(*open)); + + if (xdr_stream_decode_u32(argp->xdr, &open->op_seqid) < 0) + return nfserr_bad_xdr; + /* deleg_want is ignored */ + status = nfsd4_decode_share_access(argp, &open->op_share_access, + &open->op_deleg_want, &dummy); + if (status) + return status; + status = nfsd4_decode_share_deny(argp, &open->op_share_deny); + if (status) + return status; + status = nfsd4_decode_state_owner4(argp, &open->op_clientid, + &open->op_owner); + if (status) + return status; + status = nfsd4_decode_openflag4(argp, open); + if (status) + return status; + return nfsd4_decode_open_claim4(argp, open); +} + +static __be32 +nfsd4_decode_open_confirm(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_open_confirm *open_conf = &u->open_confirm; + __be32 status; + + if (argp->minorversion >= 1) + return nfserr_notsupp; + + status = nfsd4_decode_stateid4(argp, &open_conf->oc_req_stateid); + if (status) + return status; + if (xdr_stream_decode_u32(argp->xdr, &open_conf->oc_seqid) < 0) + return nfserr_bad_xdr; + + memset(&open_conf->oc_resp_stateid, 0, + sizeof(open_conf->oc_resp_stateid)); + return nfs_ok; +} + +static __be32 +nfsd4_decode_open_downgrade(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_open_downgrade *open_down = &u->open_downgrade; + __be32 status; + + memset(open_down, 0, sizeof(*open_down)); + status = nfsd4_decode_stateid4(argp, &open_down->od_stateid); + if (status) + return status; + if (xdr_stream_decode_u32(argp->xdr, &open_down->od_seqid) < 0) + return nfserr_bad_xdr; + /* deleg_want is ignored */ + status = nfsd4_decode_share_access(argp, &open_down->od_share_access, + &open_down->od_deleg_want, NULL); + if (status) + return status; + return nfsd4_decode_share_deny(argp, &open_down->od_share_deny); +} + +static __be32 +nfsd4_decode_putfh(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) +{ + struct nfsd4_putfh *putfh = &u->putfh; + __be32 *p; + + if (xdr_stream_decode_u32(argp->xdr, &putfh->pf_fhlen) < 0) + return nfserr_bad_xdr; + if (putfh->pf_fhlen > NFS4_FHSIZE) + return nfserr_bad_xdr; + p = xdr_inline_decode(argp->xdr, putfh->pf_fhlen); + if (!p) + return nfserr_bad_xdr; + putfh->pf_fhval = svcxdr_savemem(argp, p, putfh->pf_fhlen); + if (!putfh->pf_fhval) + return nfserr_jukebox; + + putfh->no_verify = false; + return nfs_ok; +} + +static __be32 +nfsd4_decode_read(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) +{ + struct nfsd4_read *read = &u->read; + __be32 status; + + memset(read, 0, sizeof(*read)); + status = nfsd4_decode_stateid4(argp, &read->rd_stateid); + if (status) + return status; + if (xdr_stream_decode_u64(argp->xdr, &read->rd_offset) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u32(argp->xdr, &read->rd_length) < 0) + return nfserr_bad_xdr; + + return nfs_ok; +} + +static __be32 +nfsd4_decode_readdir(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) +{ + struct nfsd4_readdir *readdir = &u->readdir; + __be32 status; + + memset(readdir, 0, sizeof(*readdir)); + if (xdr_stream_decode_u64(argp->xdr, &readdir->rd_cookie) < 0) + return nfserr_bad_xdr; + status = nfsd4_decode_verifier4(argp, &readdir->rd_verf); + if (status) + return status; + if (xdr_stream_decode_u32(argp->xdr, &readdir->rd_dircount) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u32(argp->xdr, &readdir->rd_maxcount) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_uint32_array(argp->xdr, readdir->rd_bmval, + ARRAY_SIZE(readdir->rd_bmval)) < 0) + return nfserr_bad_xdr; + + return nfs_ok; +} + +static __be32 +nfsd4_decode_remove(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) +{ + struct nfsd4_remove *remove = &u->remove; + memset(&remove->rm_cinfo, 0, sizeof(remove->rm_cinfo)); + return nfsd4_decode_component4(argp, &remove->rm_name, &remove->rm_namelen); +} + +static __be32 +nfsd4_decode_rename(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) +{ + struct nfsd4_rename *rename = &u->rename; + __be32 status; + + memset(rename, 0, sizeof(*rename)); + status = nfsd4_decode_component4(argp, &rename->rn_sname, &rename->rn_snamelen); + if (status) + return status; + return nfsd4_decode_component4(argp, &rename->rn_tname, &rename->rn_tnamelen); +} + +static __be32 +nfsd4_decode_renew(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) +{ + clientid_t *clientid = &u->renew; + return nfsd4_decode_clientid4(argp, clientid); +} + +static __be32 +nfsd4_decode_secinfo(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_secinfo *secinfo = &u->secinfo; + secinfo->si_exp = NULL; + return nfsd4_decode_component4(argp, &secinfo->si_name, &secinfo->si_namelen); +} + +static __be32 +nfsd4_decode_setattr(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) +{ + struct nfsd4_setattr *setattr = &u->setattr; + __be32 status; + + memset(setattr, 0, sizeof(*setattr)); + status = nfsd4_decode_stateid4(argp, &setattr->sa_stateid); + if (status) + return status; + return nfsd4_decode_fattr4(argp, setattr->sa_bmval, + ARRAY_SIZE(setattr->sa_bmval), + &setattr->sa_iattr, &setattr->sa_acl, + &setattr->sa_label, NULL, &setattr->sa_dpacl, + &setattr->sa_pacl); +} + +static __be32 +nfsd4_decode_setclientid(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) +{ + struct nfsd4_setclientid *setclientid = &u->setclientid; + __be32 *p, status; + + memset(setclientid, 0, sizeof(*setclientid)); + + if (argp->minorversion >= 1) + return nfserr_notsupp; + + status = nfsd4_decode_verifier4(argp, &setclientid->se_verf); + if (status) + return status; + status = nfsd4_decode_opaque(argp, &setclientid->se_name); + if (status) + return status; + if (xdr_stream_decode_u32(argp->xdr, &setclientid->se_callback_prog) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u32(argp->xdr, &setclientid->se_callback_netid_len) < 0) + return nfserr_bad_xdr; + p = xdr_inline_decode(argp->xdr, setclientid->se_callback_netid_len); + if (!p) + return nfserr_bad_xdr; + setclientid->se_callback_netid_val = svcxdr_savemem(argp, p, + setclientid->se_callback_netid_len); + if (!setclientid->se_callback_netid_val) + return nfserr_jukebox; + + if (xdr_stream_decode_u32(argp->xdr, &setclientid->se_callback_addr_len) < 0) + return nfserr_bad_xdr; + p = xdr_inline_decode(argp->xdr, setclientid->se_callback_addr_len); + if (!p) + return nfserr_bad_xdr; + setclientid->se_callback_addr_val = svcxdr_savemem(argp, p, + setclientid->se_callback_addr_len); + if (!setclientid->se_callback_addr_val) + return nfserr_jukebox; + if (xdr_stream_decode_u32(argp->xdr, &setclientid->se_callback_ident) < 0) + return nfserr_bad_xdr; + + return nfs_ok; +} + +static __be32 +nfsd4_decode_setclientid_confirm(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_setclientid_confirm *scd_c = &u->setclientid_confirm; + __be32 status; + + if (argp->minorversion >= 1) + return nfserr_notsupp; + + status = nfsd4_decode_clientid4(argp, &scd_c->sc_clientid); + if (status) + return status; + return nfsd4_decode_verifier4(argp, &scd_c->sc_confirm); +} + +/* Also used for NVERIFY */ +static __be32 +nfsd4_decode_verify(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) +{ + struct nfsd4_verify *verify = &u->verify; + __be32 *p, status; + + memset(verify, 0, sizeof(*verify)); + + status = nfsd4_decode_bitmap4(argp, verify->ve_bmval, + ARRAY_SIZE(verify->ve_bmval)); + if (status) + return status; + + /* For convenience's sake, we compare raw xdr'd attributes in + * nfsd4_proc_verify */ + + if (xdr_stream_decode_u32(argp->xdr, &verify->ve_attrlen) < 0) + return nfserr_bad_xdr; + p = xdr_inline_decode(argp->xdr, verify->ve_attrlen); + if (!p) + return nfserr_bad_xdr; + verify->ve_attrval = svcxdr_savemem(argp, p, verify->ve_attrlen); + if (!verify->ve_attrval) + return nfserr_jukebox; + + return nfs_ok; +} + +static __be32 +nfsd4_decode_write(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) +{ + struct nfsd4_write *write = &u->write; + __be32 status; + + status = nfsd4_decode_stateid4(argp, &write->wr_stateid); + if (status) + return status; + if (xdr_stream_decode_u64(argp->xdr, &write->wr_offset) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u32(argp->xdr, &write->wr_stable_how) < 0) + return nfserr_bad_xdr; + if (write->wr_stable_how > NFS_FILE_SYNC) + return nfserr_bad_xdr; + if (xdr_stream_decode_u32(argp->xdr, &write->wr_buflen) < 0) + return nfserr_bad_xdr; + if (!xdr_stream_subsegment(argp->xdr, &write->wr_payload, write->wr_buflen)) + return nfserr_bad_xdr; + + write->wr_bytes_written = 0; + write->wr_how_written = 0; + memset(&write->wr_verifier, 0, sizeof(write->wr_verifier)); + return nfs_ok; +} + +static __be32 +nfsd4_decode_release_lockowner(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_release_lockowner *rlockowner = &u->release_lockowner; + __be32 status; + + if (argp->minorversion >= 1) + return nfserr_notsupp; + + status = nfsd4_decode_state_owner4(argp, &rlockowner->rl_clientid, + &rlockowner->rl_owner); + if (status) + return status; + + if (argp->minorversion && !zero_clientid(&rlockowner->rl_clientid)) + return nfserr_inval; + + return nfs_ok; +} + +static __be32 nfsd4_decode_backchannel_ctl(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_backchannel_ctl *bc = &u->backchannel_ctl; + memset(bc, 0, sizeof(*bc)); + if (xdr_stream_decode_u32(argp->xdr, &bc->bc_cb_program) < 0) + return nfserr_bad_xdr; + return nfsd4_decode_cb_sec(argp, &bc->bc_cb_sec); +} + +static __be32 nfsd4_decode_bind_conn_to_session(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_bind_conn_to_session *bcts = &u->bind_conn_to_session; + u32 use_conn_in_rdma_mode; + __be32 status; + + memset(bcts, 0, sizeof(*bcts)); + status = nfsd4_decode_sessionid4(argp, &bcts->sessionid); + if (status) + return status; + if (xdr_stream_decode_u32(argp->xdr, &bcts->dir) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u32(argp->xdr, &use_conn_in_rdma_mode) < 0) + return nfserr_bad_xdr; + + return nfs_ok; +} + +static __be32 +nfsd4_decode_state_protect_ops(struct nfsd4_compoundargs *argp, + struct nfsd4_exchange_id *exid) +{ + __be32 status; + + status = nfsd4_decode_bitmap4(argp, exid->spo_must_enforce, + ARRAY_SIZE(exid->spo_must_enforce)); + if (status) + return nfserr_bad_xdr; + status = nfsd4_decode_bitmap4(argp, exid->spo_must_allow, + ARRAY_SIZE(exid->spo_must_allow)); + if (status) + return nfserr_bad_xdr; + + return nfs_ok; +} + +/* + * This implementation currently does not support SP4_SSV. + * This decoder simply skips over these arguments. + */ +static noinline __be32 +nfsd4_decode_ssv_sp_parms(struct nfsd4_compoundargs *argp, + struct nfsd4_exchange_id *exid) +{ + u32 count, window, num_gss_handles; + __be32 status; + + /* ssp_ops */ + status = nfsd4_decode_state_protect_ops(argp, exid); + if (status) + return status; + + /* ssp_hash_algs<> */ + if (xdr_stream_decode_u32(argp->xdr, &count) < 0) + return nfserr_bad_xdr; + while (count--) { + status = nfsd4_decode_ignored_string(argp, 0); + if (status) + return status; + } + + /* ssp_encr_algs<> */ + if (xdr_stream_decode_u32(argp->xdr, &count) < 0) + return nfserr_bad_xdr; + while (count--) { + status = nfsd4_decode_ignored_string(argp, 0); + if (status) + return status; + } + + if (xdr_stream_decode_u32(argp->xdr, &window) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u32(argp->xdr, &num_gss_handles) < 0) + return nfserr_bad_xdr; + + return nfs_ok; +} + +static __be32 +nfsd4_decode_state_protect4_a(struct nfsd4_compoundargs *argp, + struct nfsd4_exchange_id *exid) +{ + __be32 status; + + if (xdr_stream_decode_u32(argp->xdr, &exid->spa_how) < 0) + return nfserr_bad_xdr; + switch (exid->spa_how) { + case SP4_NONE: + break; + case SP4_MACH_CRED: + status = nfsd4_decode_state_protect_ops(argp, exid); + if (status) + return status; + break; + case SP4_SSV: + status = nfsd4_decode_ssv_sp_parms(argp, exid); + if (status) + return status; + break; + default: + return nfserr_bad_xdr; + } + + return nfs_ok; +} + +static __be32 +nfsd4_decode_nfs_impl_id4(struct nfsd4_compoundargs *argp, + struct nfsd4_exchange_id *exid) +{ + __be32 status; + u32 count; + + if (xdr_stream_decode_u32(argp->xdr, &count) < 0) + return nfserr_bad_xdr; + switch (count) { + case 0: + break; + case 1: + /* Note that RFC 8881 places no length limit on + * nii_domain, but this implementation permits no + * more than NFS4_OPAQUE_LIMIT bytes */ + status = nfsd4_decode_opaque(argp, &exid->nii_domain); + if (status) + return status; + /* Note that RFC 8881 places no length limit on + * nii_name, but this implementation permits no + * more than NFS4_OPAQUE_LIMIT bytes */ + status = nfsd4_decode_opaque(argp, &exid->nii_name); + if (status) + return status; + status = nfsd4_decode_nfstime4(argp, &exid->nii_time); + if (status) + return status; + break; + default: + return nfserr_bad_xdr; + } + + return nfs_ok; +} + +static __be32 +nfsd4_decode_exchange_id(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_exchange_id *exid = &u->exchange_id; + __be32 status; + + memset(exid, 0, sizeof(*exid)); + status = nfsd4_decode_verifier4(argp, &exid->verifier); + if (status) + return status; + status = nfsd4_decode_opaque(argp, &exid->clname); + if (status) + return status; + if (xdr_stream_decode_u32(argp->xdr, &exid->flags) < 0) + return nfserr_bad_xdr; + status = nfsd4_decode_state_protect4_a(argp, exid); + if (status) + return status; + return nfsd4_decode_nfs_impl_id4(argp, exid); +} + +static __be32 +nfsd4_decode_channel_attrs4(struct nfsd4_compoundargs *argp, + struct nfsd4_channel_attrs *ca) +{ + __be32 *p; + + p = xdr_inline_decode(argp->xdr, XDR_UNIT * 7); + if (!p) + return nfserr_bad_xdr; + + /* headerpadsz is ignored */ + p++; + ca->maxreq_sz = be32_to_cpup(p++); + ca->maxresp_sz = be32_to_cpup(p++); + ca->maxresp_cached = be32_to_cpup(p++); + ca->maxops = be32_to_cpup(p++); + ca->maxreqs = be32_to_cpup(p++); + ca->nr_rdma_attrs = be32_to_cpup(p); + switch (ca->nr_rdma_attrs) { + case 0: + break; + case 1: + if (xdr_stream_decode_u32(argp->xdr, &ca->rdma_attrs) < 0) + return nfserr_bad_xdr; + break; + default: + return nfserr_bad_xdr; + } + + return nfs_ok; +} + +static __be32 +nfsd4_decode_create_session(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_create_session *sess = &u->create_session; + __be32 status; + + memset(sess, 0, sizeof(*sess)); + status = nfsd4_decode_clientid4(argp, &sess->clientid); + if (status) + return status; + if (xdr_stream_decode_u32(argp->xdr, &sess->seqid) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u32(argp->xdr, &sess->flags) < 0) + return nfserr_bad_xdr; + status = nfsd4_decode_channel_attrs4(argp, &sess->fore_channel); + if (status) + return status; + status = nfsd4_decode_channel_attrs4(argp, &sess->back_channel); + if (status) + return status; + if (xdr_stream_decode_u32(argp->xdr, &sess->callback_prog) < 0) + return nfserr_bad_xdr; + return nfsd4_decode_cb_sec(argp, &sess->cb_sec); +} + +static __be32 +nfsd4_decode_destroy_session(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_destroy_session *destroy_session = &u->destroy_session; + return nfsd4_decode_sessionid4(argp, &destroy_session->sessionid); +} + +static __be32 +nfsd4_decode_free_stateid(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_free_stateid *free_stateid = &u->free_stateid; + return nfsd4_decode_stateid4(argp, &free_stateid->fr_stateid); +} + +static __be32 +nfsd4_decode_get_dir_delegation(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_get_dir_delegation *gdd = &u->get_dir_delegation; + __be32 status; + + memset(gdd, 0, sizeof(*gdd)); + + if (xdr_stream_decode_bool(argp->xdr, &gdd->gdda_signal_deleg_avail) < 0) + return nfserr_bad_xdr; + status = nfsd4_decode_bitmap4(argp, gdd->gdda_notification_types, + ARRAY_SIZE(gdd->gdda_notification_types)); + if (status) + return status; + status = nfsd4_decode_nfstime4(argp, &gdd->gdda_child_attr_delay); + if (status) + return status; + status = nfsd4_decode_nfstime4(argp, &gdd->gdda_dir_attr_delay); + if (status) + return status; + status = nfsd4_decode_bitmap4(argp, gdd->gdda_child_attributes, + ARRAY_SIZE(gdd->gdda_child_attributes)); + if (status) + return status; + return nfsd4_decode_bitmap4(argp, gdd->gdda_dir_attributes, + ARRAY_SIZE(gdd->gdda_dir_attributes)); +} + +#ifdef CONFIG_NFSD_PNFS +static __be32 +nfsd4_decode_getdeviceinfo(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_getdeviceinfo *gdev = &u->getdeviceinfo; + __be32 status; + + memset(gdev, 0, sizeof(*gdev)); + status = nfsd4_decode_deviceid4(argp->xdr, &gdev->gd_devid); + if (status) + return status; + if (xdr_stream_decode_u32(argp->xdr, &gdev->gd_layout_type) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u32(argp->xdr, &gdev->gd_maxcount) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_uint32_array(argp->xdr, + &gdev->gd_notify_types, 1) < 0) + return nfserr_bad_xdr; + + return nfs_ok; +} + +static __be32 +nfsd4_decode_layoutcommit(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_layoutcommit *lcp = &u->layoutcommit; + __be32 *p, status; + + memset(lcp, 0, sizeof(*lcp)); + if (xdr_stream_decode_u64(argp->xdr, &lcp->lc_seg.offset) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u64(argp->xdr, &lcp->lc_seg.length) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_bool(argp->xdr, &lcp->lc_reclaim) < 0) + return nfserr_bad_xdr; + status = nfsd4_decode_stateid4(argp, &lcp->lc_sid); + if (status) + return status; + if (xdr_stream_decode_bool(argp->xdr, &lcp->lc_newoffset) < 0) + return nfserr_bad_xdr; + if (lcp->lc_newoffset) { + if (xdr_stream_decode_u64(argp->xdr, &lcp->lc_last_wr) < 0) + return nfserr_bad_xdr; + } else + lcp->lc_last_wr = 0; + p = xdr_inline_decode(argp->xdr, XDR_UNIT); + if (!p) + return nfserr_bad_xdr; + if (xdr_item_is_present(p)) { + status = nfsd4_decode_nfstime4(argp, &lcp->lc_mtime); + if (status) + return status; + } else { + lcp->lc_mtime.tv_nsec = UTIME_NOW; + } + return nfsd4_decode_layoutupdate4(argp, lcp); +} + +static __be32 +nfsd4_decode_layoutget(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_layoutget *lgp = &u->layoutget; + __be32 status; + + memset(lgp, 0, sizeof(*lgp)); + if (xdr_stream_decode_u32(argp->xdr, &lgp->lg_signal) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u32(argp->xdr, &lgp->lg_layout_type) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u32(argp->xdr, &lgp->lg_seg.iomode) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u64(argp->xdr, &lgp->lg_seg.offset) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u64(argp->xdr, &lgp->lg_seg.length) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u64(argp->xdr, &lgp->lg_minlength) < 0) + return nfserr_bad_xdr; + status = nfsd4_decode_stateid4(argp, &lgp->lg_sid); + if (status) + return status; + if (xdr_stream_decode_u32(argp->xdr, &lgp->lg_maxcount) < 0) + return nfserr_bad_xdr; + + return nfs_ok; +} + +static __be32 +nfsd4_decode_layoutreturn(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_layoutreturn *lrp = &u->layoutreturn; + memset(lrp, 0, sizeof(*lrp)); + if (xdr_stream_decode_bool(argp->xdr, &lrp->lr_reclaim) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u32(argp->xdr, &lrp->lr_layout_type) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u32(argp->xdr, &lrp->lr_seg.iomode) < 0) + return nfserr_bad_xdr; + return nfsd4_decode_layoutreturn4(argp, lrp); +} +#endif /* CONFIG_NFSD_PNFS */ + +static __be32 nfsd4_decode_secinfo_no_name(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_secinfo_no_name *sin = &u->secinfo_no_name; + + sin->sin_exp = NULL; + if (xdr_stream_decode_u32(argp->xdr, &sin->sin_style) < 0) + return nfserr_bad_xdr; + + return nfs_ok; +} + +static __be32 +nfsd4_decode_sequence(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_sequence *seq = &u->sequence; + __be32 *p, status; + + status = nfsd4_decode_sessionid4(argp, &seq->sessionid); + if (status) + return status; + p = xdr_inline_decode(argp->xdr, XDR_UNIT * 4); + if (!p) + return nfserr_bad_xdr; + seq->seqid = be32_to_cpup(p++); + seq->slotid = be32_to_cpup(p++); + /* sa_highest_slotid counts from 0 but maxslots counts from 1 ... */ + seq->maxslots = be32_to_cpup(p++) + 1; + seq->cachethis = be32_to_cpup(p); + + seq->status_flags = 0; + return nfs_ok; +} + +static __be32 +nfsd4_decode_test_stateid(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_test_stateid *test_stateid = &u->test_stateid; + struct nfsd4_test_stateid_id *stateid; + __be32 status; + u32 i; + + memset(test_stateid, 0, sizeof(*test_stateid)); + if (xdr_stream_decode_u32(argp->xdr, &test_stateid->ts_num_ids) < 0) + return nfserr_bad_xdr; + + INIT_LIST_HEAD(&test_stateid->ts_stateid_list); + for (i = 0; i < test_stateid->ts_num_ids; i++) { + stateid = svcxdr_tmpalloc(argp, sizeof(*stateid)); + if (!stateid) + return nfserr_jukebox; + INIT_LIST_HEAD(&stateid->ts_id_list); + list_add_tail(&stateid->ts_id_list, &test_stateid->ts_stateid_list); + status = nfsd4_decode_stateid4(argp, &stateid->ts_id_stateid); + if (status) + return status; + } + + return nfs_ok; +} + +static __be32 nfsd4_decode_destroy_clientid(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_destroy_clientid *dc = &u->destroy_clientid; + return nfsd4_decode_clientid4(argp, &dc->clientid); +} + +static __be32 nfsd4_decode_reclaim_complete(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_reclaim_complete *rc = &u->reclaim_complete; + if (xdr_stream_decode_bool(argp->xdr, &rc->rca_one_fs) < 0) + return nfserr_bad_xdr; + return nfs_ok; +} + +static __be32 +nfsd4_decode_fallocate(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_fallocate *fallocate = &u->allocate; + __be32 status; + + status = nfsd4_decode_stateid4(argp, &fallocate->falloc_stateid); + if (status) + return status; + if (xdr_stream_decode_u64(argp->xdr, &fallocate->falloc_offset) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u64(argp->xdr, &fallocate->falloc_length) < 0) + return nfserr_bad_xdr; + + return nfs_ok; +} + +static __be32 nfsd4_decode_nl4_server(struct nfsd4_compoundargs *argp, + struct nl4_server *ns) +{ + struct nfs42_netaddr *naddr; + __be32 *p; + u32 str_len; + + if (xdr_stream_decode_u32(argp->xdr, &ns->nl4_type) < 0) + return nfserr_bad_xdr; + + /* currently support for 1 inter-server source server */ + switch (ns->nl4_type) { + case NL4_NETADDR: + naddr = &ns->u.nl4_addr; + + if (xdr_stream_decode_u32(argp->xdr, &naddr->netid_len) < 0) + return nfserr_bad_xdr; + if (naddr->netid_len > RPCBIND_MAXNETIDLEN) + return nfserr_bad_xdr; + + p = xdr_inline_decode(argp->xdr, naddr->netid_len); + if (!p) + return nfserr_bad_xdr; + memcpy(naddr->netid, p, naddr->netid_len); + + if (xdr_stream_decode_u32(argp->xdr, &naddr->addr_len) < 0) + return nfserr_bad_xdr; + if (naddr->addr_len > RPCBIND_MAXUADDRLEN) + return nfserr_bad_xdr; + + p = xdr_inline_decode(argp->xdr, naddr->addr_len); + if (!p) + return nfserr_bad_xdr; + memcpy(naddr->addr, p, naddr->addr_len); + break; + case NL4_NAME: + case NL4_URL: + /* + * Well-formed XDR, but only NL4_NETADDR is supported. Consume + * the utf8str_cis to keep the stream aligned, then return + * NFS4ERR_NOTSUPP rather than the misleading NFS4ERR_BADXDR. + */ + if (xdr_stream_decode_u32(argp->xdr, &str_len) < 0) + return nfserr_bad_xdr; + if (!xdr_inline_decode(argp->xdr, str_len)) + return nfserr_bad_xdr; + return nfserr_notsupp; + default: + return nfserr_bad_xdr; + } + + return nfs_ok; +} + +static __be32 +nfsd4_decode_copy(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) +{ + struct nfsd4_copy *copy = &u->copy; + u32 consecutive, i, count, sync; + struct nl4_server *ns_dummy; + __be32 status; + + memset(copy, 0, sizeof(*copy)); + status = nfsd4_decode_stateid4(argp, ©->cp_src_stateid); + if (status) + return status; + status = nfsd4_decode_stateid4(argp, ©->cp_dst_stateid); + if (status) + return status; + if (xdr_stream_decode_u64(argp->xdr, ©->cp_src_pos) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u64(argp->xdr, ©->cp_dst_pos) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u64(argp->xdr, ©->cp_count) < 0) + return nfserr_bad_xdr; + /* ca_consecutive: we always do consecutive copies */ + if (xdr_stream_decode_u32(argp->xdr, &consecutive) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_bool(argp->xdr, &sync) < 0) + return nfserr_bad_xdr; + nfsd4_copy_set_sync(copy, sync); + + if (xdr_stream_decode_u32(argp->xdr, &count) < 0) + return nfserr_bad_xdr; + copy->cp_src = svcxdr_tmpalloc(argp, sizeof(*copy->cp_src)); + if (copy->cp_src == NULL) + return nfserr_jukebox; + if (count == 0) { /* intra-server copy */ + __set_bit(NFSD4_COPY_F_INTRA, ©->cp_flags); + return nfs_ok; + } + + /* decode all the supplied server addresses but use only the first */ + status = nfsd4_decode_nl4_server(argp, copy->cp_src); + if (status) + return status; + + ns_dummy = kmalloc_obj(struct nl4_server); + if (ns_dummy == NULL) + return nfserr_jukebox; + for (i = 0; i < count - 1; i++) { + status = nfsd4_decode_nl4_server(argp, ns_dummy); + if (status) { + kfree(ns_dummy); + return status; + } + } + kfree(ns_dummy); + + return nfs_ok; +} + +static __be32 +nfsd4_decode_copy_notify(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_copy_notify *cn = &u->copy_notify; + __be32 status; + + memset(cn, 0, sizeof(*cn)); + cn->cpn_src = svcxdr_tmpalloc(argp, sizeof(*cn->cpn_src)); + if (cn->cpn_src == NULL) + return nfserr_jukebox; + cn->cpn_dst = svcxdr_tmpalloc(argp, sizeof(*cn->cpn_dst)); + if (cn->cpn_dst == NULL) + return nfserr_jukebox; + + status = nfsd4_decode_stateid4(argp, &cn->cpn_src_stateid); + if (status) + return status; + return nfsd4_decode_nl4_server(argp, cn->cpn_dst); +} + +static __be32 +nfsd4_decode_offload_status(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_offload_status *os = &u->offload_status; + os->count = 0; + os->status = 0; + return nfsd4_decode_stateid4(argp, &os->stateid); +} + +static __be32 +nfsd4_decode_seek(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) +{ + struct nfsd4_seek *seek = &u->seek; + __be32 status; + + status = nfsd4_decode_stateid4(argp, &seek->seek_stateid); + if (status) + return status; + if (xdr_stream_decode_u64(argp->xdr, &seek->seek_offset) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u32(argp->xdr, &seek->seek_whence) < 0) + return nfserr_bad_xdr; + + seek->seek_eof = 0; + seek->seek_pos = 0; + return nfs_ok; +} + +static __be32 +nfsd4_decode_clone(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) +{ + struct nfsd4_clone *clone = &u->clone; + __be32 status; + + status = nfsd4_decode_stateid4(argp, &clone->cl_src_stateid); + if (status) + return status; + status = nfsd4_decode_stateid4(argp, &clone->cl_dst_stateid); + if (status) + return status; + if (xdr_stream_decode_u64(argp->xdr, &clone->cl_src_pos) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u64(argp->xdr, &clone->cl_dst_pos) < 0) + return nfserr_bad_xdr; + if (xdr_stream_decode_u64(argp->xdr, &clone->cl_count) < 0) + return nfserr_bad_xdr; + + return nfs_ok; +} + +/* + * XDR data that is more than PAGE_SIZE in size is normally part of a + * read or write. However, the size of extended attributes is limited + * by the maximum request size, and then further limited by the underlying + * filesystem limits. This can exceed PAGE_SIZE (currently, XATTR_SIZE_MAX + * is 64k). Since there is no kvec- or page-based interface to xattrs, + * and we're not dealing with contiguous pages, we need to do some copying. + */ + +/* + * Decode data into buffer. + */ +static __be32 +nfsd4_vbuf_from_vector(struct nfsd4_compoundargs *argp, struct xdr_buf *xdr, + char **bufp, size_t buflen) +{ + struct page **pages = xdr->pages; + struct kvec *head = xdr->head; + char *tmp, *dp; + u32 len; + + if (buflen <= head->iov_len) { + /* + * We're in luck, the head has enough space. Just return + * the head, no need for copying. + */ + *bufp = head->iov_base; + return 0; + } + + tmp = svcxdr_tmpalloc(argp, buflen); + if (tmp == NULL) + return nfserr_jukebox; + + dp = tmp; + memcpy(dp, head->iov_base, head->iov_len); + buflen -= head->iov_len; + dp += head->iov_len; + + while (buflen > 0) { + len = min_t(u32, buflen, PAGE_SIZE); + memcpy(dp, page_address(*pages), len); + + buflen -= len; + dp += len; + pages++; + } + + *bufp = tmp; + return 0; +} + +/* + * Get a user extended attribute name from the XDR buffer. + * It will not have the "user." prefix, so prepend it. + * Lastly, check for nul characters in the name. + */ +static __be32 +nfsd4_decode_xattr_name(struct nfsd4_compoundargs *argp, char **namep) +{ + char *name, *sp, *dp; + u32 namelen, cnt; + __be32 *p; + + if (xdr_stream_decode_u32(argp->xdr, &namelen) < 0) + return nfserr_bad_xdr; + if (namelen > (XATTR_NAME_MAX - XATTR_USER_PREFIX_LEN)) + return nfserr_nametoolong; + if (namelen == 0) + return nfserr_bad_xdr; + p = xdr_inline_decode(argp->xdr, namelen); + if (!p) + return nfserr_bad_xdr; + name = svcxdr_tmpalloc(argp, namelen + XATTR_USER_PREFIX_LEN + 1); + if (!name) + return nfserr_jukebox; + memcpy(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN); + + /* + * Copy the extended attribute name over while checking for 0 + * characters. + */ + sp = (char *)p; + dp = name + XATTR_USER_PREFIX_LEN; + cnt = namelen; + + while (cnt-- > 0) { + if (*sp == '\0') + return nfserr_bad_xdr; + *dp++ = *sp++; + } + *dp = '\0'; + + *namep = name; + + return nfs_ok; +} + +/* + * A GETXATTR op request comes without a length specifier. We just set the + * maximum length for the reply based on XATTR_SIZE_MAX and the maximum + * channel reply size. nfsd_getxattr will probe the length of the xattr, + * check it against getxa_len, and allocate + return the value. + */ +static __be32 +nfsd4_decode_getxattr(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_getxattr *getxattr = &u->getxattr; + __be32 status; + u32 maxcount; + + memset(getxattr, 0, sizeof(*getxattr)); + status = nfsd4_decode_xattr_name(argp, &getxattr->getxa_name); + if (status) + return status; + + maxcount = svc_max_payload(argp->rqstp); + maxcount = min_t(u32, XATTR_SIZE_MAX, maxcount); + + getxattr->getxa_len = maxcount; + return nfs_ok; +} + +static __be32 +nfsd4_decode_setxattr(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_setxattr *setxattr = &u->setxattr; + u32 flags, maxcount, size; + __be32 status; + + memset(setxattr, 0, sizeof(*setxattr)); + + if (xdr_stream_decode_u32(argp->xdr, &flags) < 0) + return nfserr_bad_xdr; + + if (flags > SETXATTR4_REPLACE) + return nfserr_inval; + setxattr->setxa_flags = flags; + + status = nfsd4_decode_xattr_name(argp, &setxattr->setxa_name); + if (status) + return status; + + maxcount = svc_max_payload(argp->rqstp); + maxcount = min_t(u32, XATTR_SIZE_MAX, maxcount); + + if (xdr_stream_decode_u32(argp->xdr, &size) < 0) + return nfserr_bad_xdr; + if (size > maxcount) + return nfserr_xattr2big; + + setxattr->setxa_len = size; + if (size > 0) { + struct xdr_buf payload; + + if (!xdr_stream_subsegment(argp->xdr, &payload, size)) + return nfserr_bad_xdr; + status = nfsd4_vbuf_from_vector(argp, &payload, + &setxattr->setxa_buf, size); + } + + return nfs_ok; +} + +static __be32 +nfsd4_decode_listxattrs(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_listxattrs *listxattrs = &u->listxattrs; + u32 maxcount; + + memset(listxattrs, 0, sizeof(*listxattrs)); + + if (xdr_stream_decode_u64(argp->xdr, &listxattrs->lsxa_cookie) < 0) + return nfserr_bad_xdr; + + /* + * If the cookie is too large to have even one user.x attribute + * plus trailing '\0' left in a maximum size buffer, it's invalid. + */ + if (listxattrs->lsxa_cookie >= + (XATTR_LIST_MAX / (XATTR_USER_PREFIX_LEN + 2))) + return nfserr_badcookie; + + if (xdr_stream_decode_u32(argp->xdr, &maxcount) < 0) + return nfserr_bad_xdr; + if (maxcount < 8) + /* Always need at least 2 words (length and one character) */ + return nfserr_inval; + + maxcount = min(maxcount, svc_max_payload(argp->rqstp)); + listxattrs->lsxa_maxcount = maxcount; + + return nfs_ok; +} + +static __be32 +nfsd4_decode_removexattr(struct nfsd4_compoundargs *argp, + union nfsd4_op_u *u) +{ + struct nfsd4_removexattr *removexattr = &u->removexattr; + memset(removexattr, 0, sizeof(*removexattr)); + return nfsd4_decode_xattr_name(argp, &removexattr->rmxa_name); +} + +static __be32 +nfsd4_decode_noop(struct nfsd4_compoundargs *argp, union nfsd4_op_u *p) +{ + return nfs_ok; +} + +static __be32 +nfsd4_decode_notsupp(struct nfsd4_compoundargs *argp, union nfsd4_op_u *p) +{ + return nfserr_notsupp; +} + +typedef __be32(*nfsd4_dec)(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u); + +static const nfsd4_dec nfsd4_dec_ops[] = { + [OP_ACCESS] = nfsd4_decode_access, + [OP_CLOSE] = nfsd4_decode_close, + [OP_COMMIT] = nfsd4_decode_commit, + [OP_CREATE] = nfsd4_decode_create, + [OP_DELEGPURGE] = nfsd4_decode_notsupp, + [OP_DELEGRETURN] = nfsd4_decode_delegreturn, + [OP_GETATTR] = nfsd4_decode_getattr, + [OP_GETFH] = nfsd4_decode_noop, + [OP_LINK] = nfsd4_decode_link, + [OP_LOCK] = nfsd4_decode_lock, + [OP_LOCKT] = nfsd4_decode_lockt, + [OP_LOCKU] = nfsd4_decode_locku, + [OP_LOOKUP] = nfsd4_decode_lookup, + [OP_LOOKUPP] = nfsd4_decode_noop, + [OP_NVERIFY] = nfsd4_decode_verify, + [OP_OPEN] = nfsd4_decode_open, + [OP_OPENATTR] = nfsd4_decode_notsupp, + [OP_OPEN_CONFIRM] = nfsd4_decode_open_confirm, + [OP_OPEN_DOWNGRADE] = nfsd4_decode_open_downgrade, + [OP_PUTFH] = nfsd4_decode_putfh, + [OP_PUTPUBFH] = nfsd4_decode_noop, + [OP_PUTROOTFH] = nfsd4_decode_noop, + [OP_READ] = nfsd4_decode_read, + [OP_READDIR] = nfsd4_decode_readdir, + [OP_READLINK] = nfsd4_decode_noop, + [OP_REMOVE] = nfsd4_decode_remove, + [OP_RENAME] = nfsd4_decode_rename, + [OP_RENEW] = nfsd4_decode_renew, + [OP_RESTOREFH] = nfsd4_decode_noop, + [OP_SAVEFH] = nfsd4_decode_noop, + [OP_SECINFO] = nfsd4_decode_secinfo, + [OP_SETATTR] = nfsd4_decode_setattr, + [OP_SETCLIENTID] = nfsd4_decode_setclientid, + [OP_SETCLIENTID_CONFIRM] = nfsd4_decode_setclientid_confirm, + [OP_VERIFY] = nfsd4_decode_verify, + [OP_WRITE] = nfsd4_decode_write, + [OP_RELEASE_LOCKOWNER] = nfsd4_decode_release_lockowner, + + /* new operations for NFSv4.1 */ + [OP_BACKCHANNEL_CTL] = nfsd4_decode_backchannel_ctl, + [OP_BIND_CONN_TO_SESSION] = nfsd4_decode_bind_conn_to_session, + [OP_EXCHANGE_ID] = nfsd4_decode_exchange_id, + [OP_CREATE_SESSION] = nfsd4_decode_create_session, + [OP_DESTROY_SESSION] = nfsd4_decode_destroy_session, + [OP_FREE_STATEID] = nfsd4_decode_free_stateid, + [OP_GET_DIR_DELEGATION] = nfsd4_decode_get_dir_delegation, +#ifdef CONFIG_NFSD_PNFS + [OP_GETDEVICEINFO] = nfsd4_decode_getdeviceinfo, + [OP_GETDEVICELIST] = nfsd4_decode_notsupp, + [OP_LAYOUTCOMMIT] = nfsd4_decode_layoutcommit, + [OP_LAYOUTGET] = nfsd4_decode_layoutget, + [OP_LAYOUTRETURN] = nfsd4_decode_layoutreturn, +#else + [OP_GETDEVICEINFO] = nfsd4_decode_notsupp, + [OP_GETDEVICELIST] = nfsd4_decode_notsupp, + [OP_LAYOUTCOMMIT] = nfsd4_decode_notsupp, + [OP_LAYOUTGET] = nfsd4_decode_notsupp, + [OP_LAYOUTRETURN] = nfsd4_decode_notsupp, +#endif + [OP_SECINFO_NO_NAME] = nfsd4_decode_secinfo_no_name, + [OP_SEQUENCE] = nfsd4_decode_sequence, + [OP_SET_SSV] = nfsd4_decode_notsupp, + [OP_TEST_STATEID] = nfsd4_decode_test_stateid, + [OP_WANT_DELEGATION] = nfsd4_decode_notsupp, + [OP_DESTROY_CLIENTID] = nfsd4_decode_destroy_clientid, + [OP_RECLAIM_COMPLETE] = nfsd4_decode_reclaim_complete, + + /* new operations for NFSv4.2 */ + [OP_ALLOCATE] = nfsd4_decode_fallocate, + [OP_COPY] = nfsd4_decode_copy, + [OP_COPY_NOTIFY] = nfsd4_decode_copy_notify, + [OP_DEALLOCATE] = nfsd4_decode_fallocate, + [OP_IO_ADVISE] = nfsd4_decode_notsupp, + [OP_LAYOUTERROR] = nfsd4_decode_notsupp, + [OP_LAYOUTSTATS] = nfsd4_decode_notsupp, + [OP_OFFLOAD_CANCEL] = nfsd4_decode_offload_status, + [OP_OFFLOAD_STATUS] = nfsd4_decode_offload_status, + [OP_READ_PLUS] = nfsd4_decode_read, + [OP_SEEK] = nfsd4_decode_seek, + [OP_WRITE_SAME] = nfsd4_decode_notsupp, + [OP_CLONE] = nfsd4_decode_clone, + /* RFC 8276 extended atributes operations */ + [OP_GETXATTR] = nfsd4_decode_getxattr, + [OP_SETXATTR] = nfsd4_decode_setxattr, + [OP_LISTXATTRS] = nfsd4_decode_listxattrs, + [OP_REMOVEXATTR] = nfsd4_decode_removexattr, +}; + +static inline bool +nfsd4_opnum_in_range(struct nfsd4_compoundargs *argp, struct nfsd4_op *op) +{ + if (op->opnum < FIRST_NFS4_OP) + return false; + else if (argp->minorversion == 0 && op->opnum > LAST_NFS40_OP) + return false; + else if (argp->minorversion == 1 && op->opnum > LAST_NFS41_OP) + return false; + else if (argp->minorversion == 2 && op->opnum > LAST_NFS42_OP) + return false; + return true; +} + +static bool +nfsd4_decode_compound(struct nfsd4_compoundargs *argp) +{ + struct nfsd_thread_local_info *ntli = argp->rqstp->rq_private; + struct nfsd4_op *op; + bool cachethis = false; + int auth_slack= argp->rqstp->rq_auth_slack; + int max_reply = auth_slack + 8; /* opcnt, status */ + int readcount = 0; + int readbytes = 0; + __be32 *p; + int i; + + if (xdr_stream_decode_u32(argp->xdr, &argp->taglen) < 0) + return false; + max_reply += XDR_UNIT; + argp->tag = NULL; + if (unlikely(argp->taglen)) { + if (argp->taglen > NFSD4_MAX_TAGLEN) + return false; + p = xdr_inline_decode(argp->xdr, argp->taglen); + if (!p) + return false; + argp->tag = svcxdr_savemem(argp, p, argp->taglen); + if (!argp->tag) + return false; + max_reply += xdr_align_size(argp->taglen); + } + + if (xdr_stream_decode_u32(argp->xdr, &argp->minorversion) < 0) + return false; + if (xdr_stream_decode_u32(argp->xdr, &argp->client_opcnt) < 0) + return false; + argp->opcnt = min_t(u32, argp->client_opcnt, + NFSD_MAX_OPS_PER_COMPOUND); + + if (argp->opcnt > ARRAY_SIZE(argp->iops)) { + argp->ops = vcalloc(argp->opcnt, sizeof(*argp->ops)); + if (!argp->ops) { + argp->ops = argp->iops; + return false; + } + } + + if (argp->minorversion > NFSD_SUPPORTED_MINOR_VERSION) + argp->opcnt = 0; + + for (i = 0; i < argp->opcnt; i++) { + op = &argp->ops[i]; + op->replay = NULL; + op->opdesc = NULL; + + if (xdr_stream_decode_u32(argp->xdr, &op->opnum) < 0) + return false; + if (nfsd4_opnum_in_range(argp, op)) { + op->opdesc = OPDESC(op); + op->status = nfsd4_dec_ops[op->opnum](argp, &op->u); + if (op->status != nfs_ok) + trace_nfsd_compound_decode_err(argp->rqstp, + argp->opcnt, i, + op->opnum, + op->status); + } else { + op->opnum = OP_ILLEGAL; + op->status = nfserr_op_illegal; + } + + /* + * We'll try to cache the result in the DRC if any one + * op in the compound wants to be cached: + */ + cachethis |= nfsd4_cache_this_op(op); + + if (op->opnum == OP_READ || op->opnum == OP_READ_PLUS) { + readcount++; + readbytes += nfsd4_max_reply(argp->rqstp, op); + } else + max_reply += nfsd4_max_reply(argp->rqstp, op); + /* + * OP_LOCK and OP_LOCKT may return a conflicting lock. + * (Special case because it will just skip encoding this + * if it runs out of xdr buffer space, and it is the only + * operation that behaves this way.) + */ + if (op->opnum == OP_LOCK || op->opnum == OP_LOCKT) + max_reply += NFS4_OPAQUE_LIMIT; + + if (op->status) { + argp->opcnt = i+1; + break; + } + } + /* Sessions make the DRC unnecessary: */ + if (argp->minorversion) + cachethis = false; + svc_reserve_auth(argp->rqstp, max_reply + readbytes); + ntli->ntli_cachetype = cachethis ? RC_REPLBUFF : RC_NOCACHE; + + argp->splice_ok = nfsd_read_splice_ok(argp->rqstp); + if (readcount > 1 || max_reply > PAGE_SIZE - auth_slack) + argp->splice_ok = false; + + return true; +} + +static __be32 nfsd4_encode_nfs_fh4(struct xdr_stream *xdr, + const struct knfsd_fh *fh_handle) +{ + return nfsd4_encode_opaque(xdr, fh_handle->fh_raw, fh_handle->fh_size); +} + +/* This is a frequently-encoded type; open-coded for speed */ +static __be32 nfsd4_encode_nfstime4(struct xdr_stream *xdr, + const struct timespec64 *tv) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, XDR_UNIT * 3); + if (!p) + return nfserr_resource; + p = xdr_encode_hyper(p, tv->tv_sec); + *p = cpu_to_be32(tv->tv_nsec); + return nfs_ok; +} + +static __be32 nfsd4_encode_specdata4(struct xdr_stream *xdr, + unsigned int major, unsigned int minor) +{ + __be32 status; + + status = nfsd4_encode_uint32_t(xdr, major); + if (status != nfs_ok) + return status; + return nfsd4_encode_uint32_t(xdr, minor); +} + +static __be32 +nfsd4_encode_change_info4(struct xdr_stream *xdr, const struct nfsd4_change_info *c) +{ + __be32 status; + + status = nfsd4_encode_bool(xdr, c->atomic); + if (status != nfs_ok) + return status; + status = nfsd4_encode_changeid4(xdr, c->before_change); + if (status != nfs_ok) + return status; + return nfsd4_encode_changeid4(xdr, c->after_change); +} + +static __be32 nfsd4_encode_netaddr4(struct xdr_stream *xdr, + const struct nfs42_netaddr *addr) +{ + __be32 status; + + /* na_r_netid */ + status = nfsd4_encode_opaque(xdr, addr->netid, addr->netid_len); + if (status != nfs_ok) + return status; + /* na_r_addr */ + return nfsd4_encode_opaque(xdr, addr->addr, addr->addr_len); +} + +/* Encode as an array of strings the string given with components + * separated @sep, escaped with esc_enter and esc_exit. + */ +static __be32 nfsd4_encode_components_esc(struct xdr_stream *xdr, char sep, + char *components, char esc_enter, + char esc_exit) +{ + __be32 *p; + __be32 pathlen; + int pathlen_offset; + char *str, *end, *next; + int count = 0; + + pathlen_offset = xdr->buf->len; + p = xdr_reserve_space(xdr, 4); + if (!p) + return nfserr_resource; + p++; /* We will fill this in with @count later */ + + end = str = components; + while (*end) { + bool found_esc = false; + + /* try to parse as esc_start, ..., esc_end, sep */ + if (*str == esc_enter) { + for (; *end && (*end != esc_exit); end++) + /* find esc_exit or end of string */; + next = end + 1; + if (*end && (!*next || *next == sep)) { + str++; + found_esc = true; + } + } + + if (!found_esc) + for (; *end && (*end != sep); end++) + /* find sep or end of string */; + + if (end > str) { + if (xdr_stream_encode_opaque(xdr, str, end - str) < 0) + return nfserr_resource; + count++; + } else + end++; + if (found_esc) + end = next; + + str = end; + } + pathlen = htonl(count); + write_bytes_to_xdr_buf(xdr->buf, pathlen_offset, &pathlen, 4); + return 0; +} + +/* Encode as an array of strings the string given with components + * separated @sep. + */ +static __be32 nfsd4_encode_components(struct xdr_stream *xdr, char sep, + char *components) +{ + return nfsd4_encode_components_esc(xdr, sep, components, 0, 0); +} + +static __be32 nfsd4_encode_fs_location4(struct xdr_stream *xdr, + struct nfsd4_fs_location *location) +{ + __be32 status; + + status = nfsd4_encode_components_esc(xdr, ':', location->hosts, + '[', ']'); + if (status) + return status; + status = nfsd4_encode_components(xdr, '/', location->path); + if (status) + return status; + return nfs_ok; +} + +static __be32 nfsd4_encode_pathname4(struct xdr_stream *xdr, + const struct path *root, + const struct path *path) +{ + struct path cur = *path; + struct dentry **components = NULL; + unsigned int ncomponents = 0; + __be32 err = nfserr_jukebox; + + dprintk("nfsd4_encode_components("); + + path_get(&cur); + /* First walk the path up to the nfsd root, and store the + * dentries/path components in an array. + */ + for (;;) { + if (path_equal(&cur, root)) + break; + if (cur.dentry == cur.mnt->mnt_root) { + if (follow_up(&cur)) + continue; + goto out_free; + } + if ((ncomponents & 15) == 0) { + struct dentry **new; + new = krealloc(components, + sizeof(*new) * (ncomponents + 16), + GFP_KERNEL); + if (!new) + goto out_free; + components = new; + } + components[ncomponents++] = cur.dentry; + cur.dentry = dget_parent(cur.dentry); + } + + err = nfserr_resource; + if (xdr_stream_encode_u32(xdr, ncomponents) != XDR_UNIT) + goto out_free; + while (ncomponents) { + struct dentry *dentry = components[ncomponents - 1]; + + spin_lock(&dentry->d_lock); + if (xdr_stream_encode_opaque(xdr, dentry->d_name.name, + dentry->d_name.len) < 0) { + spin_unlock(&dentry->d_lock); + goto out_free; + } + dprintk("/%pd", dentry); + spin_unlock(&dentry->d_lock); + dput(dentry); + ncomponents--; + } + + err = 0; +out_free: + dprintk(")\n"); + while (ncomponents) + dput(components[--ncomponents]); + kfree(components); + path_put(&cur); + return err; +} + +static __be32 nfsd4_encode_fs_locations4(struct xdr_stream *xdr, + struct svc_rqst *rqstp, + struct svc_export *exp) +{ + struct nfsd4_fs_locations *fslocs = &exp->ex_fslocs; + struct svc_export *exp_ps; + unsigned int i; + __be32 status; + + /* fs_root */ + exp_ps = rqst_find_fsidzero_export(rqstp); + if (IS_ERR(exp_ps)) + return nfserrno(PTR_ERR(exp_ps)); + status = nfsd4_encode_pathname4(xdr, &exp_ps->ex_path, &exp->ex_path); + exp_put(exp_ps); + if (status != nfs_ok) + return status; + + /* locations<> */ + if (xdr_stream_encode_u32(xdr, fslocs->locations_count) != XDR_UNIT) + return nfserr_resource; + for (i = 0; i < fslocs->locations_count; i++) { + status = nfsd4_encode_fs_location4(xdr, &fslocs->locations[i]); + if (status != nfs_ok) + return status; + } + + return nfs_ok; +} + +static __be32 nfsd4_encode_nfsace4(struct xdr_stream *xdr, struct svc_rqst *rqstp, + struct nfs4_ace *ace) +{ + __be32 status; + + /* type */ + status = nfsd4_encode_acetype4(xdr, ace->type); + if (status != nfs_ok) + return nfserr_resource; + /* flag */ + status = nfsd4_encode_aceflag4(xdr, ace->flag); + if (status != nfs_ok) + return nfserr_resource; + /* access mask */ + status = nfsd4_encode_acemask4(xdr, ace->access_mask & NFS4_ACE_MASK_ALL); + if (status != nfs_ok) + return nfserr_resource; + /* who */ + if (ace->whotype != NFS4_ACL_WHO_NAMED) + return nfs4_acl_write_who(xdr, ace->whotype); + if (ace->flag & NFS4_ACE_IDENTIFIER_GROUP) + return nfsd4_encode_group(xdr, rqstp, ace->who_gid); + return nfsd4_encode_user(xdr, rqstp, ace->who_uid); +} + +#define WORD0_ABSENT_FS_ATTRS (FATTR4_WORD0_FS_LOCATIONS | FATTR4_WORD0_FSID | \ + FATTR4_WORD0_RDATTR_ERROR) +#define WORD1_ABSENT_FS_ATTRS FATTR4_WORD1_MOUNTED_ON_FILEID +#define WORD2_ABSENT_FS_ATTRS 0 + +#ifdef CONFIG_NFSD_V4_SECURITY_LABEL +static inline __be32 +nfsd4_encode_security_label(struct xdr_stream *xdr, struct svc_rqst *rqstp, + const struct lsm_context *context) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, context->len + 4 + 4 + 4); + if (!p) + return nfserr_resource; + + /* + * For now we use a 0 here to indicate the null translation; in + * the future we may place a call to translation code here. + */ + *p++ = cpu_to_be32(0); /* lfs */ + *p++ = cpu_to_be32(0); /* pi */ + p = xdr_encode_opaque(p, context->context, context->len); + return 0; +} +#else +static inline __be32 +nfsd4_encode_security_label(struct xdr_stream *xdr, struct svc_rqst *rqstp, + struct lsm_context *context) +{ return 0; } +#endif + +#ifdef CONFIG_NFSD_V4_POSIX_ACLS + +static int nfsd4_posix_tagtotype(short tag) +{ + switch (tag) { + case ACL_USER_OBJ: return POSIXACE4_TAG_USER_OBJ; + case ACL_GROUP_OBJ: return POSIXACE4_TAG_GROUP_OBJ; + case ACL_USER: return POSIXACE4_TAG_USER; + case ACL_GROUP: return POSIXACE4_TAG_GROUP; + case ACL_MASK: return POSIXACE4_TAG_MASK; + case ACL_OTHER: return POSIXACE4_TAG_OTHER; + default: return -EINVAL; + } +} + +static __be32 +nfsd4_encode_posixace4(struct xdr_stream *xdr, struct svc_rqst *rqstp, + struct posix_acl_entry *acep) +{ + __be32 status; + int type; + + type = nfsd4_posix_tagtotype(acep->e_tag); + if (type < 0) + return nfserr_resource; + if (!xdrgen_encode_posixacetag4(xdr, type)) + return nfserr_resource; + if (!xdrgen_encode_posixaceperm4(xdr, acep->e_perm)) + return nfserr_resource; + + /* who */ + switch (acep->e_tag) { + case ACL_USER_OBJ: + case ACL_GROUP_OBJ: + case ACL_MASK: + case ACL_OTHER: + if (xdr_stream_encode_u32(xdr, 0) != XDR_UNIT) + return nfserr_resource; + break; + case ACL_USER: + status = nfsd4_encode_user(xdr, rqstp, acep->e_uid); + if (status != nfs_ok) + return status; + break; + case ACL_GROUP: + status = nfsd4_encode_group(xdr, rqstp, acep->e_gid); + if (status != nfs_ok) + return status; + break; + default: + return nfserr_resource; + } + return nfs_ok; +} + +static __be32 +nfsd4_encode_posixacl(struct xdr_stream *xdr, struct svc_rqst *rqstp, + struct posix_acl *acl) +{ + __be32 status; + int i; + + if (!acl) { + if (xdr_stream_encode_u32(xdr, 0) != XDR_UNIT) + return nfserr_resource; + return nfs_ok; + } + + if (acl->a_count > NFS_ACL_MAX_ENTRIES) + return nfserr_resource; + if (xdr_stream_encode_u32(xdr, acl->a_count) != XDR_UNIT) + return nfserr_resource; + for (i = 0; i < acl->a_count; i++) { + status = nfsd4_encode_posixace4(xdr, rqstp, &acl->a_entries[i]); + if (status != nfs_ok) + return status; + } + + return nfs_ok; +} + +#endif /* CONFIG_NFSD_V4_POSIX_ACL */ + +static __be32 fattr_handle_absent_fs(u32 *bmval0, u32 *bmval1, u32 *bmval2, u32 *rdattr_err) +{ + /* As per referral draft: */ + if (*bmval0 & ~WORD0_ABSENT_FS_ATTRS || + *bmval1 & ~WORD1_ABSENT_FS_ATTRS) { + if (*bmval0 & FATTR4_WORD0_RDATTR_ERROR || + *bmval0 & FATTR4_WORD0_FS_LOCATIONS) + *rdattr_err = NFSERR_MOVED; + else + return nfserr_moved; + } + *bmval0 &= WORD0_ABSENT_FS_ATTRS; + *bmval1 &= WORD1_ABSENT_FS_ATTRS; + *bmval2 &= WORD2_ABSENT_FS_ATTRS; + return 0; +} + + +static int nfsd4_get_mounted_on_ino(struct svc_export *exp, u64 *pino) +{ + struct path path = exp->ex_path; + struct kstat stat; + int err; + + path_get(&path); + while (follow_up(&path)) { + if (path.dentry != path.mnt->mnt_root) + break; + } + err = vfs_getattr(&path, &stat, STATX_INO, AT_STATX_SYNC_AS_STAT); + path_put(&path); + if (!err) + *pino = stat.ino; + return err; +} + +static __be32 +nfsd4_encode_bitmap4(struct xdr_stream *xdr, u32 bmval0, u32 bmval1, u32 bmval2) +{ + __be32 *p; + + if (bmval2) { + p = xdr_reserve_space(xdr, XDR_UNIT * 4); + if (!p) + goto out_resource; + *p++ = cpu_to_be32(3); + *p++ = cpu_to_be32(bmval0); + *p++ = cpu_to_be32(bmval1); + *p++ = cpu_to_be32(bmval2); + } else if (bmval1) { + p = xdr_reserve_space(xdr, XDR_UNIT * 3); + if (!p) + goto out_resource; + *p++ = cpu_to_be32(2); + *p++ = cpu_to_be32(bmval0); + *p++ = cpu_to_be32(bmval1); + } else { + p = xdr_reserve_space(xdr, XDR_UNIT * 2); + if (!p) + goto out_resource; + *p++ = cpu_to_be32(1); + *p++ = cpu_to_be32(bmval0); + } + + return nfs_ok; +out_resource: + return nfserr_resource; +} + +struct nfsd4_fattr_args { + struct svc_rqst *rqstp; + struct svc_export *exp; + struct dentry *dentry; + struct knfsd_fh fhandle; + struct kstat stat; + struct kstatfs statfs; + struct nfs4_acl *acl; + u64 change_attr; +#ifdef CONFIG_NFSD_V4_SECURITY_LABEL + struct lsm_context context; +#endif +#ifdef CONFIG_NFSD_V4_POSIX_ACLS + struct posix_acl *dpacl; + struct posix_acl *pacl; +#endif + u32 rdattr_err; + bool contextsupport; + bool ignore_crossmnt; + bool case_insensitive; + bool case_preserving; +}; + +typedef __be32(*nfsd4_enc_attr)(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args); + +static __be32 nfsd4_encode_fattr4__inval(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfserr_inval; +} + +static __be32 nfsd4_encode_fattr4__noop(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfs_ok; +} + +static __be32 nfsd4_encode_fattr4__true(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_bool(xdr, true); +} + +static __be32 nfsd4_encode_fattr4__false(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_bool(xdr, false); +} + +static __be32 nfsd4_encode_fattr4_supported_attrs(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + struct nfsd4_compoundres *resp = args->rqstp->rq_resp; + u32 minorversion = resp->cstate.minorversion; + u32 supp[3]; + + memcpy(supp, nfsd_suppattrs[minorversion], sizeof(supp)); + if (!IS_POSIXACL(d_inode(args->dentry))) + supp[0] &= ~FATTR4_WORD0_ACL; + if (!args->contextsupport) + supp[2] &= ~FATTR4_WORD2_SECURITY_LABEL; + + return nfsd4_encode_bitmap4(xdr, supp[0], supp[1], supp[2]); +} + +static __be32 nfsd4_encode_fattr4_type(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, XDR_UNIT); + if (!p) + return nfserr_resource; + + switch (args->stat.mode & S_IFMT) { + case S_IFIFO: + *p = cpu_to_be32(NF4FIFO); + break; + case S_IFCHR: + *p = cpu_to_be32(NF4CHR); + break; + case S_IFDIR: + *p = cpu_to_be32(NF4DIR); + break; + case S_IFBLK: + *p = cpu_to_be32(NF4BLK); + break; + case S_IFLNK: + *p = cpu_to_be32(NF4LNK); + break; + case S_IFREG: + *p = cpu_to_be32(NF4REG); + break; + case S_IFSOCK: + *p = cpu_to_be32(NF4SOCK); + break; + default: + return nfserr_serverfault; + } + + return nfs_ok; +} + +static __be32 nfsd4_encode_fattr4_fh_expire_type(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + u32 mask; + + mask = NFS4_FH_PERSISTENT; + if (!(args->exp->ex_flags & NFSEXP_NOSUBTREECHECK)) + mask |= NFS4_FH_VOL_RENAME; + return nfsd4_encode_uint32_t(xdr, mask); +} + +static __be32 nfsd4_encode_fattr4_change(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + const struct svc_export *exp = args->exp; + + if (exp && unlikely(exp->ex_flags & NFSEXP_V4ROOT)) { + u32 flush_time = convert_to_wallclock(exp->cd->flush_time); + + if (xdr_stream_encode_u32(xdr, flush_time) != XDR_UNIT) + return nfserr_resource; + if (xdr_stream_encode_u32(xdr, 0) != XDR_UNIT) + return nfserr_resource; + return nfs_ok; + } + return nfsd4_encode_changeid4(xdr, args->change_attr); +} + +static __be32 nfsd4_encode_fattr4_size(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_uint64_t(xdr, args->stat.size); +} + +static __be32 nfsd4_encode_fattr4_fsid(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, XDR_UNIT * 2 + XDR_UNIT * 2); + if (!p) + return nfserr_resource; + + if (unlikely(args->exp->ex_fslocs.migrated)) { + p = xdr_encode_hyper(p, NFS4_REFERRAL_FSID_MAJOR); + xdr_encode_hyper(p, NFS4_REFERRAL_FSID_MINOR); + return nfs_ok; + } + switch (fsid_source_fh(&args->fhandle, args->exp)) { + case FSIDSOURCE_FSID: + p = xdr_encode_hyper(p, (u64)args->exp->ex_fsid); + xdr_encode_hyper(p, (u64)0); + break; + case FSIDSOURCE_DEV: + *p++ = xdr_zero; + *p++ = cpu_to_be32(MAJOR(args->stat.dev)); + *p++ = xdr_zero; + *p = cpu_to_be32(MINOR(args->stat.dev)); + break; + case FSIDSOURCE_UUID: + xdr_encode_opaque_fixed(p, args->exp->ex_uuid, EX_UUID_LEN); + break; + } + + return nfs_ok; +} + +static __be32 nfsd4_encode_fattr4_lease_time(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + struct nfsd_net *nn = net_generic(SVC_NET(args->rqstp), nfsd_net_id); + + return nfsd4_encode_nfs_lease4(xdr, nn->nfsd4_lease); +} + +static __be32 nfsd4_encode_fattr4_rdattr_error(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_uint32_t(xdr, args->rdattr_err); +} + +static __be32 nfsd4_encode_fattr4_aclsupport(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + u32 mask; + + mask = 0; + if (IS_POSIXACL(d_inode(args->dentry))) + mask = ACL4_SUPPORT_ALLOW_ACL | ACL4_SUPPORT_DENY_ACL; + return nfsd4_encode_uint32_t(xdr, mask); +} + +static __be32 nfsd4_encode_fattr4_acl(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + struct nfs4_acl *acl = args->acl; + struct nfs4_ace *ace; + __be32 status; + + /* nfsace4<> */ + if (!acl) { + if (xdr_stream_encode_u32(xdr, 0) != XDR_UNIT) + return nfserr_resource; + } else { + if (xdr_stream_encode_u32(xdr, acl->naces) != XDR_UNIT) + return nfserr_resource; + for (ace = acl->aces; ace < acl->aces + acl->naces; ace++) { + status = nfsd4_encode_nfsace4(xdr, args->rqstp, ace); + if (status != nfs_ok) + return status; + } + } + return nfs_ok; +} + +static __be32 nfsd4_encode_fattr4_case_insensitive(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_bool(xdr, args->case_insensitive); +} + +static __be32 nfsd4_encode_fattr4_case_preserving(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_bool(xdr, args->case_preserving); +} + +static __be32 nfsd4_encode_fattr4_homogeneous(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + /* + * Casefold-capable filesystems (e.g. ext4 or f2fs with the + * casefold feature) attach a Unicode encoding at mount time + * but apply case folding per directory. The per-file-system + * case_insensitive and case_preserving values can therefore + * legitimately differ across objects that share the same fsid. + * Report FATTR4_HOMOGENEOUS = FALSE on such filesystems to + * keep that variation consistent with RFC 8881 Section 5.8.2.16. + */ + return nfsd4_encode_bool(xdr, !sb_has_encoding(args->dentry->d_sb)); +} + +static __be32 nfsd4_encode_fattr4_filehandle(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_nfs_fh4(xdr, &args->fhandle); +} + +static __be32 nfsd4_encode_fattr4_fileid(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_uint64_t(xdr, args->stat.ino); +} + +static __be32 nfsd4_encode_fattr4_files_avail(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_uint64_t(xdr, args->statfs.f_ffree); +} + +static __be32 nfsd4_encode_fattr4_files_free(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_uint64_t(xdr, args->statfs.f_ffree); +} + +static __be32 nfsd4_encode_fattr4_files_total(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_uint64_t(xdr, args->statfs.f_files); +} + +static __be32 nfsd4_encode_fattr4_fs_locations(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_fs_locations4(xdr, args->rqstp, args->exp); +} + +static __be32 nfsd4_encode_fattr4_maxfilesize(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + struct super_block *sb = args->exp->ex_path.mnt->mnt_sb; + + return nfsd4_encode_uint64_t(xdr, sb->s_maxbytes); +} + +static __be32 nfsd4_encode_fattr4_maxlink(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_uint32_t(xdr, 255); +} + +static __be32 nfsd4_encode_fattr4_maxname(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_uint32_t(xdr, args->statfs.f_namelen); +} + +static __be32 nfsd4_encode_fattr4_maxread(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_uint64_t(xdr, svc_max_payload(args->rqstp)); +} + +static __be32 nfsd4_encode_fattr4_maxwrite(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_uint64_t(xdr, svc_max_payload(args->rqstp)); +} + +static __be32 nfsd4_encode_fattr4_mode(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_mode4(xdr, args->stat.mode & S_IALLUGO); +} + +static __be32 nfsd4_encode_fattr4_numlinks(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_uint32_t(xdr, args->stat.nlink); +} + +static __be32 nfsd4_encode_fattr4_owner(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_user(xdr, args->rqstp, args->stat.uid); +} + +static __be32 nfsd4_encode_fattr4_owner_group(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_group(xdr, args->rqstp, args->stat.gid); +} + +static __be32 nfsd4_encode_fattr4_rawdev(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_specdata4(xdr, MAJOR(args->stat.rdev), + MINOR(args->stat.rdev)); +} + +static __be32 nfsd4_encode_fattr4_space_avail(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + u64 avail = (u64)args->statfs.f_bavail * (u64)args->statfs.f_bsize; + + return nfsd4_encode_uint64_t(xdr, avail); +} + +static __be32 nfsd4_encode_fattr4_space_free(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + u64 free = (u64)args->statfs.f_bfree * (u64)args->statfs.f_bsize; + + return nfsd4_encode_uint64_t(xdr, free); +} + +static __be32 nfsd4_encode_fattr4_space_total(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + u64 total = (u64)args->statfs.f_blocks * (u64)args->statfs.f_bsize; + + return nfsd4_encode_uint64_t(xdr, total); +} + +static __be32 nfsd4_encode_fattr4_space_used(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_uint64_t(xdr, (u64)args->stat.blocks << 9); +} + +static __be32 nfsd4_encode_fattr4_time_access(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_nfstime4(xdr, &args->stat.atime); +} + +static __be32 nfsd4_encode_fattr4_time_create(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_nfstime4(xdr, &args->stat.btime); +} + +/* + * ctime (in NFSv4, time_metadata) is not writeable, and the client + * doesn't really care what resolution could theoretically be stored by + * the filesystem. + * + * The client cares how close together changes can be while still + * guaranteeing ctime changes. For most filesystems (which have + * timestamps with nanosecond fields) that is limited by the resolution + * of the time returned from current_time() (which I'm assuming to be + * 1/HZ). + */ +static __be32 nfsd4_encode_fattr4_time_delta(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + const struct inode *inode = d_inode(args->dentry); + u32 ns = max_t(u32, NSEC_PER_SEC/HZ, inode->i_sb->s_time_gran); + struct timespec64 ts = ns_to_timespec64(ns); + + return nfsd4_encode_nfstime4(xdr, &ts); +} + +static __be32 nfsd4_encode_fattr4_time_metadata(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_nfstime4(xdr, &args->stat.ctime); +} + +static __be32 nfsd4_encode_fattr4_time_modify(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_nfstime4(xdr, &args->stat.mtime); +} + +static __be32 nfsd4_encode_fattr4_mounted_on_fileid(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + u64 ino; + int err; + + if (!args->ignore_crossmnt && + args->dentry == args->exp->ex_path.mnt->mnt_root) { + err = nfsd4_get_mounted_on_ino(args->exp, &ino); + if (err) + return nfserrno(err); + } else + ino = args->stat.ino; + + return nfsd4_encode_uint64_t(xdr, ino); +} + +#ifdef CONFIG_NFSD_PNFS + +static __be32 nfsd4_encode_fattr4_fs_layout_types(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + unsigned long mask = args->exp->ex_layout_types; + int i; + + /* Hamming weight of @mask is the number of layout types to return */ + if (xdr_stream_encode_u32(xdr, hweight_long(mask)) != XDR_UNIT) + return nfserr_resource; + for (i = LAYOUT_NFSV4_1_FILES; i < LAYOUT_TYPE_MAX; ++i) + if (mask & BIT(i)) { + /* layouttype4 */ + if (xdr_stream_encode_u32(xdr, i) != XDR_UNIT) + return nfserr_resource; + } + return nfs_ok; +} + +static __be32 nfsd4_encode_fattr4_layout_types(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + unsigned long mask = args->exp->ex_layout_types; + int i; + + /* Hamming weight of @mask is the number of layout types to return */ + if (xdr_stream_encode_u32(xdr, hweight_long(mask)) != XDR_UNIT) + return nfserr_resource; + for (i = LAYOUT_NFSV4_1_FILES; i < LAYOUT_TYPE_MAX; ++i) + if (mask & BIT(i)) { + /* layouttype4 */ + if (xdr_stream_encode_u32(xdr, i) != XDR_UNIT) + return nfserr_resource; + } + return nfs_ok; +} + +static __be32 nfsd4_encode_fattr4_layout_blksize(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_uint32_t(xdr, args->stat.blksize); +} + +#endif + +static __be32 nfsd4_encode_fattr4_suppattr_exclcreat(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + struct nfsd4_compoundres *resp = args->rqstp->rq_resp; + u32 supp[3]; + + memcpy(supp, nfsd_suppattrs[resp->cstate.minorversion], sizeof(supp)); + if (!IS_POSIXACL(d_inode(args->dentry))) + supp[0] &= ~FATTR4_WORD0_ACL; + if (!args->contextsupport) + supp[2] &= ~FATTR4_WORD2_SECURITY_LABEL; + + supp[0] &= NFSD_SUPPATTR_EXCLCREAT_WORD0; + supp[1] &= NFSD_SUPPATTR_EXCLCREAT_WORD1; + supp[2] &= NFSD_SUPPATTR_EXCLCREAT_WORD2; + + return nfsd4_encode_bitmap4(xdr, supp[0], supp[1], supp[2]); +} + +/* + * Copied from generic_remap_checks/generic_remap_file_range_prep. + * + * These generic functions use the file system's s_blocksize, but + * individual file systems aren't required to use + * generic_remap_file_range_prep. Until there is a mechanism for + * determining a particular file system's (or file's) clone block + * size, this is the best NFSD can do. + */ +static __be32 nfsd4_encode_fattr4_clone_blksize(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + struct inode *inode = d_inode(args->dentry); + + return nfsd4_encode_uint32_t(xdr, inode->i_sb->s_blocksize); +} + +#ifdef CONFIG_NFSD_V4_SECURITY_LABEL +static __be32 nfsd4_encode_fattr4_sec_label(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_security_label(xdr, args->rqstp, &args->context); +} +#endif + +static __be32 nfsd4_encode_fattr4_xattr_support(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + int err = xattr_supports_user_prefix(d_inode(args->dentry)); + + return nfsd4_encode_bool(xdr, err == 0); +} + +#define NFSD_OA_SHARE_ACCESS (BIT(OPEN_ARGS_SHARE_ACCESS_READ) | \ + BIT(OPEN_ARGS_SHARE_ACCESS_WRITE) | \ + BIT(OPEN_ARGS_SHARE_ACCESS_BOTH)) + +#define NFSD_OA_SHARE_DENY (BIT(OPEN_ARGS_SHARE_DENY_NONE) | \ + BIT(OPEN_ARGS_SHARE_DENY_READ) | \ + BIT(OPEN_ARGS_SHARE_DENY_WRITE) | \ + BIT(OPEN_ARGS_SHARE_DENY_BOTH)) + +#define NFSD_OA_SHARE_ACCESS_WANT (BIT(OPEN_ARGS_SHARE_ACCESS_WANT_ANY_DELEG) | \ + BIT(OPEN_ARGS_SHARE_ACCESS_WANT_NO_DELEG) | \ + BIT(OPEN_ARGS_SHARE_ACCESS_WANT_CANCEL) | \ + BIT(OPEN_ARGS_SHARE_ACCESS_WANT_DELEG_TIMESTAMPS) | \ + BIT(OPEN_ARGS_SHARE_ACCESS_WANT_OPEN_XOR_DELEGATION)) + +#define NFSD_OA_OPEN_CLAIM (BIT(OPEN_ARGS_OPEN_CLAIM_NULL) | \ + BIT(OPEN_ARGS_OPEN_CLAIM_PREVIOUS) | \ + BIT(OPEN_ARGS_OPEN_CLAIM_DELEGATE_CUR) | \ + BIT(OPEN_ARGS_OPEN_CLAIM_DELEGATE_PREV)| \ + BIT(OPEN_ARGS_OPEN_CLAIM_FH) | \ + BIT(OPEN_ARGS_OPEN_CLAIM_DELEG_CUR_FH) | \ + BIT(OPEN_ARGS_OPEN_CLAIM_DELEG_PREV_FH)) + +#define NFSD_OA_CREATE_MODE (BIT(OPEN_ARGS_CREATEMODE_UNCHECKED4) | \ + BIT(OPEN_ARGS_CREATE_MODE_GUARDED) | \ + BIT(OPEN_ARGS_CREATEMODE_EXCLUSIVE4) | \ + BIT(OPEN_ARGS_CREATE_MODE_EXCLUSIVE4_1)) + +static uint32_t oa_share_access = NFSD_OA_SHARE_ACCESS; +static uint32_t oa_share_deny = NFSD_OA_SHARE_DENY; +static uint32_t oa_share_access_want = NFSD_OA_SHARE_ACCESS_WANT; +static uint32_t oa_open_claim = NFSD_OA_OPEN_CLAIM; +static uint32_t oa_create_mode = NFSD_OA_CREATE_MODE; + +static const struct open_arguments4 nfsd_open_arguments = { + .oa_share_access = { .count = 1, .element = &oa_share_access }, + .oa_share_deny = { .count = 1, .element = &oa_share_deny }, + .oa_share_access_want = { .count = 1, .element = &oa_share_access_want }, + .oa_open_claim = { .count = 1, .element = &oa_open_claim }, + .oa_create_mode = { .count = 1, .element = &oa_create_mode }, +}; + +static __be32 nfsd4_encode_fattr4_open_arguments(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + if (!xdrgen_encode_fattr4_open_arguments(xdr, &nfsd_open_arguments)) + return nfserr_resource; + return nfs_ok; +} + +#ifdef CONFIG_NFSD_V4_POSIX_ACLS + +static __be32 nfsd4_encode_fattr4_acl_trueform(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + aclmodel4 trueform = ACL_MODEL_NONE; + + if (IS_POSIXACL(d_inode(args->dentry))) + trueform = ACL_MODEL_POSIX_DRAFT; + if (!xdrgen_encode_aclmodel4(xdr, trueform)) + return nfserr_resource; + return nfs_ok; +} + +static __be32 nfsd4_encode_fattr4_acl_trueform_scope(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + if (!xdrgen_encode_aclscope4(xdr, ACL_SCOPE_FILE_SYSTEM)) + return nfserr_resource; + return nfs_ok; +} + +static __be32 nfsd4_encode_fattr4_posix_default_acl(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_posixacl(xdr, args->rqstp, args->dpacl); +} + +static __be32 nfsd4_encode_fattr4_posix_access_acl(struct xdr_stream *xdr, + const struct nfsd4_fattr_args *args) +{ + return nfsd4_encode_posixacl(xdr, args->rqstp, args->pacl); +} + +#endif /* CONFIG_NFSD_V4_POSIX_ACLS */ + +static const nfsd4_enc_attr nfsd4_enc_fattr4_encode_ops[] = { + [FATTR4_SUPPORTED_ATTRS] = nfsd4_encode_fattr4_supported_attrs, + [FATTR4_TYPE] = nfsd4_encode_fattr4_type, + [FATTR4_FH_EXPIRE_TYPE] = nfsd4_encode_fattr4_fh_expire_type, + [FATTR4_CHANGE] = nfsd4_encode_fattr4_change, + [FATTR4_SIZE] = nfsd4_encode_fattr4_size, + [FATTR4_LINK_SUPPORT] = nfsd4_encode_fattr4__true, + [FATTR4_SYMLINK_SUPPORT] = nfsd4_encode_fattr4__true, + [FATTR4_NAMED_ATTR] = nfsd4_encode_fattr4__false, + [FATTR4_FSID] = nfsd4_encode_fattr4_fsid, + [FATTR4_UNIQUE_HANDLES] = nfsd4_encode_fattr4__true, + [FATTR4_LEASE_TIME] = nfsd4_encode_fattr4_lease_time, + [FATTR4_RDATTR_ERROR] = nfsd4_encode_fattr4_rdattr_error, + [FATTR4_ACL] = nfsd4_encode_fattr4_acl, + [FATTR4_ACLSUPPORT] = nfsd4_encode_fattr4_aclsupport, + [FATTR4_ARCHIVE] = nfsd4_encode_fattr4__noop, + [FATTR4_CANSETTIME] = nfsd4_encode_fattr4__true, + [FATTR4_CASE_INSENSITIVE] = nfsd4_encode_fattr4_case_insensitive, + [FATTR4_CASE_PRESERVING] = nfsd4_encode_fattr4_case_preserving, + [FATTR4_CHOWN_RESTRICTED] = nfsd4_encode_fattr4__true, + [FATTR4_FILEHANDLE] = nfsd4_encode_fattr4_filehandle, + [FATTR4_FILEID] = nfsd4_encode_fattr4_fileid, + [FATTR4_FILES_AVAIL] = nfsd4_encode_fattr4_files_avail, + [FATTR4_FILES_FREE] = nfsd4_encode_fattr4_files_free, + [FATTR4_FILES_TOTAL] = nfsd4_encode_fattr4_files_total, + [FATTR4_FS_LOCATIONS] = nfsd4_encode_fattr4_fs_locations, + [FATTR4_HIDDEN] = nfsd4_encode_fattr4__noop, + [FATTR4_HOMOGENEOUS] = nfsd4_encode_fattr4_homogeneous, + [FATTR4_MAXFILESIZE] = nfsd4_encode_fattr4_maxfilesize, + [FATTR4_MAXLINK] = nfsd4_encode_fattr4_maxlink, + [FATTR4_MAXNAME] = nfsd4_encode_fattr4_maxname, + [FATTR4_MAXREAD] = nfsd4_encode_fattr4_maxread, + [FATTR4_MAXWRITE] = nfsd4_encode_fattr4_maxwrite, + [FATTR4_MIMETYPE] = nfsd4_encode_fattr4__noop, + [FATTR4_MODE] = nfsd4_encode_fattr4_mode, + [FATTR4_NO_TRUNC] = nfsd4_encode_fattr4__true, + [FATTR4_NUMLINKS] = nfsd4_encode_fattr4_numlinks, + [FATTR4_OWNER] = nfsd4_encode_fattr4_owner, + [FATTR4_OWNER_GROUP] = nfsd4_encode_fattr4_owner_group, + [FATTR4_QUOTA_AVAIL_HARD] = nfsd4_encode_fattr4__noop, + [FATTR4_QUOTA_AVAIL_SOFT] = nfsd4_encode_fattr4__noop, + [FATTR4_QUOTA_USED] = nfsd4_encode_fattr4__noop, + [FATTR4_RAWDEV] = nfsd4_encode_fattr4_rawdev, + [FATTR4_SPACE_AVAIL] = nfsd4_encode_fattr4_space_avail, + [FATTR4_SPACE_FREE] = nfsd4_encode_fattr4_space_free, + [FATTR4_SPACE_TOTAL] = nfsd4_encode_fattr4_space_total, + [FATTR4_SPACE_USED] = nfsd4_encode_fattr4_space_used, + [FATTR4_SYSTEM] = nfsd4_encode_fattr4__noop, + [FATTR4_TIME_ACCESS] = nfsd4_encode_fattr4_time_access, + [FATTR4_TIME_ACCESS_SET] = nfsd4_encode_fattr4__noop, + [FATTR4_TIME_BACKUP] = nfsd4_encode_fattr4__noop, + [FATTR4_TIME_CREATE] = nfsd4_encode_fattr4_time_create, + [FATTR4_TIME_DELTA] = nfsd4_encode_fattr4_time_delta, + [FATTR4_TIME_METADATA] = nfsd4_encode_fattr4_time_metadata, + [FATTR4_TIME_MODIFY] = nfsd4_encode_fattr4_time_modify, + [FATTR4_TIME_MODIFY_SET] = nfsd4_encode_fattr4__noop, + [FATTR4_MOUNTED_ON_FILEID] = nfsd4_encode_fattr4_mounted_on_fileid, + [FATTR4_DIR_NOTIF_DELAY] = nfsd4_encode_fattr4__noop, + [FATTR4_DIRENT_NOTIF_DELAY] = nfsd4_encode_fattr4__noop, + [FATTR4_DACL] = nfsd4_encode_fattr4__noop, + [FATTR4_SACL] = nfsd4_encode_fattr4__noop, + [FATTR4_CHANGE_POLICY] = nfsd4_encode_fattr4__noop, + [FATTR4_FS_STATUS] = nfsd4_encode_fattr4__noop, + +#ifdef CONFIG_NFSD_PNFS + [FATTR4_FS_LAYOUT_TYPES] = nfsd4_encode_fattr4_fs_layout_types, + [FATTR4_LAYOUT_HINT] = nfsd4_encode_fattr4__noop, + [FATTR4_LAYOUT_TYPES] = nfsd4_encode_fattr4_layout_types, + [FATTR4_LAYOUT_BLKSIZE] = nfsd4_encode_fattr4_layout_blksize, + [FATTR4_LAYOUT_ALIGNMENT] = nfsd4_encode_fattr4__noop, +#else + [FATTR4_FS_LAYOUT_TYPES] = nfsd4_encode_fattr4__noop, + [FATTR4_LAYOUT_HINT] = nfsd4_encode_fattr4__noop, + [FATTR4_LAYOUT_TYPES] = nfsd4_encode_fattr4__noop, + [FATTR4_LAYOUT_BLKSIZE] = nfsd4_encode_fattr4__noop, + [FATTR4_LAYOUT_ALIGNMENT] = nfsd4_encode_fattr4__noop, +#endif + + [FATTR4_FS_LOCATIONS_INFO] = nfsd4_encode_fattr4__noop, + [FATTR4_MDSTHRESHOLD] = nfsd4_encode_fattr4__noop, + [FATTR4_RETENTION_GET] = nfsd4_encode_fattr4__noop, + [FATTR4_RETENTION_SET] = nfsd4_encode_fattr4__noop, + [FATTR4_RETENTEVT_GET] = nfsd4_encode_fattr4__noop, + [FATTR4_RETENTEVT_SET] = nfsd4_encode_fattr4__noop, + [FATTR4_RETENTION_HOLD] = nfsd4_encode_fattr4__noop, + [FATTR4_MODE_SET_MASKED] = nfsd4_encode_fattr4__noop, + [FATTR4_SUPPATTR_EXCLCREAT] = nfsd4_encode_fattr4_suppattr_exclcreat, + [FATTR4_FS_CHARSET_CAP] = nfsd4_encode_fattr4__noop, + [FATTR4_CLONE_BLKSIZE] = nfsd4_encode_fattr4_clone_blksize, + [FATTR4_SPACE_FREED] = nfsd4_encode_fattr4__noop, + [FATTR4_CHANGE_ATTR_TYPE] = nfsd4_encode_fattr4__noop, + +#ifdef CONFIG_NFSD_V4_SECURITY_LABEL + [FATTR4_SEC_LABEL] = nfsd4_encode_fattr4_sec_label, +#else + [FATTR4_SEC_LABEL] = nfsd4_encode_fattr4__noop, +#endif + + [FATTR4_MODE_UMASK] = nfsd4_encode_fattr4__noop, + [FATTR4_XATTR_SUPPORT] = nfsd4_encode_fattr4_xattr_support, + [FATTR4_TIME_DELEG_ACCESS] = nfsd4_encode_fattr4__inval, + [FATTR4_TIME_DELEG_MODIFY] = nfsd4_encode_fattr4__inval, + [FATTR4_OPEN_ARGUMENTS] = nfsd4_encode_fattr4_open_arguments, + + /* Reserved */ + [87] = nfsd4_encode_fattr4__inval, + [88] = nfsd4_encode_fattr4__inval, + +#ifdef CONFIG_NFSD_V4_POSIX_ACLS + [FATTR4_ACL_TRUEFORM] = nfsd4_encode_fattr4_acl_trueform, + [FATTR4_ACL_TRUEFORM_SCOPE] = nfsd4_encode_fattr4_acl_trueform_scope, + [FATTR4_POSIX_DEFAULT_ACL] = nfsd4_encode_fattr4_posix_default_acl, + [FATTR4_POSIX_ACCESS_ACL] = nfsd4_encode_fattr4_posix_access_acl, +#else + [FATTR4_ACL_TRUEFORM] = nfsd4_encode_fattr4__noop, + [FATTR4_ACL_TRUEFORM_SCOPE] = nfsd4_encode_fattr4__noop, + [FATTR4_POSIX_DEFAULT_ACL] = nfsd4_encode_fattr4__noop, + [FATTR4_POSIX_ACCESS_ACL] = nfsd4_encode_fattr4__noop, +#endif +}; + +static __be32 +nfsd4_encode_attr_vals(struct xdr_stream *xdr, u32 *attrmask, struct nfsd4_fattr_args *args) +{ + DECLARE_BITMAP(attr_bitmap, ARRAY_SIZE(nfsd4_enc_fattr4_encode_ops)); + unsigned long bit; + __be32 status; + + bitmap_from_arr32(attr_bitmap, attrmask, ARRAY_SIZE(nfsd4_enc_fattr4_encode_ops)); + for_each_set_bit(bit, attr_bitmap, ARRAY_SIZE(nfsd4_enc_fattr4_encode_ops)) { + status = nfsd4_enc_fattr4_encode_ops[bit](xdr, args); + if (status != nfs_ok) + return status; + } + return nfs_ok; +} + +/* + * Note: @fhp can be NULL; in this case, we might have to compose the filehandle + * ourselves. @case_cache is NULL for callers that encode a single dentry + * (GETATTR, the buffer wrapper); READDIR passes a per-request cache so + * non-directory children share the parent's case-folding probe result. + */ +static __be32 +nfsd4_encode_fattr4(struct svc_rqst *rqstp, struct xdr_stream *xdr, + struct svc_fh *fhp, struct svc_export *exp, + struct dentry *dentry, const u32 *bmval, + int ignore_crossmnt, + struct nfsd_case_attrs_cache *case_cache) +{ + struct nfs4_delegation *dp = NULL; + struct nfsd4_fattr_args args; + struct svc_fh *tempfh = NULL; + int starting_len = xdr->buf->len; + unsigned int attrlen_offset; + __be32 attrlen, status; + u32 attrmask[3]; + int err; + struct nfsd4_compoundres *resp = rqstp->rq_resp; + u32 minorversion = resp->cstate.minorversion; + struct path path = { + .mnt = exp->ex_path.mnt, + .dentry = dentry, + }; + + WARN_ON_ONCE(bmval[1] & NFSD_WRITEONLY_ATTRS_WORD1); + WARN_ON_ONCE(!nfsd_attrs_supported(minorversion, bmval)); + + args.rqstp = rqstp; + args.exp = exp; + args.dentry = dentry; + args.ignore_crossmnt = (ignore_crossmnt != 0); + args.acl = NULL; +#ifdef CONFIG_NFSD_V4_SECURITY_LABEL + args.context.context = NULL; +#endif +#ifdef CONFIG_NFSD_V4_POSIX_ACLS + args.dpacl = NULL; + args.pacl = NULL; +#endif + + /* + * Make a local copy of the attribute bitmap that can be modified. + */ + attrmask[0] = bmval[0]; + attrmask[1] = bmval[1]; + attrmask[2] = bmval[2]; + + args.rdattr_err = 0; + if (exp->ex_fslocs.migrated) { + status = fattr_handle_absent_fs(&attrmask[0], &attrmask[1], + &attrmask[2], &args.rdattr_err); + if (status) + goto out; + } + if ((attrmask[0] & (FATTR4_WORD0_CHANGE | + FATTR4_WORD0_SIZE)) || + (attrmask[1] & (FATTR4_WORD1_TIME_ACCESS | + FATTR4_WORD1_TIME_MODIFY | + FATTR4_WORD1_TIME_METADATA))) { + status = nfsd4_deleg_getattr_conflict(rqstp, dentry, &dp); + if (status) + goto out; + } + + err = vfs_getattr(&path, &args.stat, + STATX_BASIC_STATS | STATX_BTIME | STATX_CHANGE_COOKIE, + AT_STATX_SYNC_AS_STAT); + if (dp) { + struct nfs4_cb_fattr *ncf = &dp->dl_cb_fattr; + + if (ncf->ncf_file_modified) { + ++ncf->ncf_initial_cinfo; + args.stat.size = ncf->ncf_cur_fsize; + if (!timespec64_is_epoch(&ncf->ncf_cb_mtime)) + args.stat.mtime = ncf->ncf_cb_mtime; + } + args.change_attr = ncf->ncf_initial_cinfo; + + if (!timespec64_is_epoch(&ncf->ncf_cb_atime)) + args.stat.atime = ncf->ncf_cb_atime; + + nfs4_put_stid(&dp->dl_stid); + } else { + args.change_attr = nfsd4_change_attribute(&args.stat); + } + + if (err) + goto out_nfserr; + + if (!(args.stat.result_mask & STATX_BTIME)) + /* underlying FS does not offer btime so we can't share it */ + attrmask[1] &= ~FATTR4_WORD1_TIME_CREATE; + if ((attrmask[0] & (FATTR4_WORD0_FILES_AVAIL | FATTR4_WORD0_FILES_FREE | + FATTR4_WORD0_FILES_TOTAL | FATTR4_WORD0_MAXNAME)) || + (attrmask[1] & (FATTR4_WORD1_SPACE_AVAIL | FATTR4_WORD1_SPACE_FREE | + FATTR4_WORD1_SPACE_TOTAL))) { + err = vfs_statfs(&path, &args.statfs); + if (err) + goto out_nfserr; + } + + if ((attrmask[0] & (FATTR4_WORD0_FILEHANDLE | FATTR4_WORD0_FSID))) { + if (!fhp) { + tempfh = kmalloc_obj(struct svc_fh); + status = nfserr_jukebox; + if (!tempfh) + goto out; + fh_init(tempfh, NFS4_FHSIZE); + status = fh_compose(tempfh, exp, dentry, NULL); + if (status) + goto out; + fhp = tempfh; + } + fh_copy_shallow(&args.fhandle, &fhp->fh_handle); + } + + if (attrmask[0] & (FATTR4_WORD0_CASE_INSENSITIVE | + FATTR4_WORD0_CASE_PRESERVING)) { + /* + * In a batched encoder (READDIR) every non-directory + * child shares the same case-folding answer, so the + * directory being read is probed once and the result is + * cached. The probe targets case_cache->dir, the held + * readdir filehandle's dentry, instead of the child's + * locklessly-acquired dentry, which a concurrent rename + * could move under an unrelated parent. Directory + * entries are queried directly because casefold-capable + * filesystems answer per directory. + * + * Per RFC 8881 Section 18.7.3, an attribute advertised + * in SUPPORTED_ATTRS must come back with a value or the + * GETATTR must fail. nfsd_get_case_info() fills POSIX + * defaults and returns -EOPNOTSUPP when the underlying + * filesystem does not expose case state; encode those + * defaults so the reply agrees with what SUPPORTED_ATTRS + * advertises. Other errors fail the operation as the + * spec requires. + */ + if (case_cache && !d_is_dir(dentry)) { + if (!case_cache->valid) { + err = nfsd_get_case_info(case_cache->dir, + &case_cache->insensitive, + &case_cache->preserving); + if (err && err != -EOPNOTSUPP) + goto out_nfserr; + case_cache->valid = true; + } + args.case_insensitive = case_cache->insensitive; + args.case_preserving = case_cache->preserving; + } else { + err = nfsd_get_case_info(dentry, + &args.case_insensitive, + &args.case_preserving); + if (err && err != -EOPNOTSUPP) + goto out_nfserr; + } + } + + if (attrmask[0] & FATTR4_WORD0_ACL) { + err = nfsd4_get_nfs4_acl(rqstp, dentry, &args.acl); + if (err == -EOPNOTSUPP) + attrmask[0] &= ~FATTR4_WORD0_ACL; + else if (err == -EINVAL) { + status = nfserr_attrnotsupp; + goto out; + } else if (err != 0) + goto out_nfserr; + } + + args.contextsupport = false; + +#ifdef CONFIG_NFSD_V4_SECURITY_LABEL + if ((attrmask[2] & FATTR4_WORD2_SECURITY_LABEL) || + attrmask[0] & FATTR4_WORD0_SUPPORTED_ATTRS) { + if (exp->ex_flags & NFSEXP_SECURITY_LABEL) + err = security_inode_getsecctx(d_inode(dentry), + &args.context); + else + err = -EOPNOTSUPP; + args.contextsupport = (err == 0); + if (attrmask[2] & FATTR4_WORD2_SECURITY_LABEL) { + if (err == -EOPNOTSUPP) + attrmask[2] &= ~FATTR4_WORD2_SECURITY_LABEL; + else if (err) + goto out_nfserr; + } + } +#endif /* CONFIG_NFSD_V4_SECURITY_LABEL */ + +#ifdef CONFIG_NFSD_V4_POSIX_ACLS + if (attrmask[2] & FATTR4_WORD2_POSIX_DEFAULT_ACL) { + struct inode *inode = d_inode(dentry); + struct posix_acl *dpacl; + + if (S_ISDIR(inode->i_mode)) { + dpacl = get_inode_acl(inode, ACL_TYPE_DEFAULT); + if (IS_ERR(dpacl)) { + switch (PTR_ERR(dpacl)) { + case -EOPNOTSUPP: + attrmask[2] &= ~FATTR4_WORD2_POSIX_DEFAULT_ACL; + break; + case -EINVAL: + status = nfserr_attrnotsupp; + goto out; + default: + err = PTR_ERR(dpacl); + goto out_nfserr; + } + } else { + args.dpacl = dpacl; + } + } + } + if (attrmask[2] & FATTR4_WORD2_POSIX_ACCESS_ACL) { + struct inode *inode = d_inode(dentry); + struct posix_acl *pacl; + + pacl = get_inode_acl(inode, ACL_TYPE_ACCESS); + if (!pacl) + pacl = posix_acl_from_mode(inode->i_mode, GFP_KERNEL); + if (IS_ERR(pacl)) { + switch (PTR_ERR(pacl)) { + case -EOPNOTSUPP: + attrmask[2] &= ~FATTR4_WORD2_POSIX_ACCESS_ACL; + break; + case -EINVAL: + status = nfserr_attrnotsupp; + goto out; + default: + err = PTR_ERR(pacl); + goto out_nfserr; + } + } else { + args.pacl = pacl; + } + } +#endif /* CONFIG_NFSD_V4_POSIX_ACLS */ + + /* attrmask */ + status = nfsd4_encode_bitmap4(xdr, attrmask[0], attrmask[1], attrmask[2]); + if (status) + goto out; + + /* attr_vals */ + attrlen_offset = xdr->buf->len; + if (unlikely(!xdr_reserve_space(xdr, XDR_UNIT))) { + status = nfserr_resource; + goto out; + } + + status = nfsd4_encode_attr_vals(xdr, attrmask, &args); + if (status == nfs_ok) { + attrlen = cpu_to_be32(xdr->buf->len - attrlen_offset - XDR_UNIT); + write_bytes_to_xdr_buf(xdr->buf, attrlen_offset, &attrlen, XDR_UNIT); + } +out: +#ifdef CONFIG_NFSD_V4_POSIX_ACLS + if (args.dpacl) + posix_acl_release(args.dpacl); + if (args.pacl) + posix_acl_release(args.pacl); +#endif /* CONFIG_NFSD_V4_POSIX_ACLS */ +#ifdef CONFIG_NFSD_V4_SECURITY_LABEL + if (args.context.context) + security_release_secctx(&args.context); +#endif /* CONFIG_NFSD_V4_SECURITY_LABEL */ + kfree(args.acl); + if (tempfh) { + fh_put(tempfh); + kfree(tempfh); + } + if (status) + xdr_truncate_encode(xdr, starting_len); + return status; +out_nfserr: + status = nfserrno(err); + goto out; +} + +static bool +setup_notify_fhandle(struct dentry *dentry, struct nfs4_delegation *dp, + struct nfsd_file *nf, struct nfsd4_fattr_args *args) +{ + struct nfs4_file *fi = dp->dl_stid.sc_file; + struct nfs4_client *clp = dp->dl_stid.sc_client; + int fileid_type, fsid_len, maxsize, flags = 0; + struct knfsd_fh *fhp = &args->fhandle; + struct inode *inode = d_inode(dentry); + struct inode *parent = NULL; + struct svc_export *exp; + struct fid *fid; + bool ret = false; + + /* + * drop_stid_export() can clear sc_export under cl_lock and drop its + * reference when the delegation is admin-revoked, concurrently with + * this callback. Grab our own reference under cl_lock so the export + * can be neither NULL-raced nor freed while we encode. + */ + spin_lock(&clp->cl_lock); + exp = dp->dl_stid.sc_export; + if (exp) + exp_get(exp); + spin_unlock(&clp->cl_lock); + + fsid_len = key_len(fi->fi_fhandle.fh_fsid_type); + fhp->fh_size = 4 + fsid_len; + + /* Copy first 4 bytes + fsid */ + memcpy(&fhp->fh_raw, &fi->fi_fhandle.fh_raw, fhp->fh_size); + + fid = (struct fid *)(fh_fsid(fhp) + fsid_len/4); + maxsize = (NFS4_FHSIZE - fhp->fh_size)/4; + + /* + * Subtree-checking exports need a connectable filehandle so the + * parent can be resolved at decode time. Derive this from the + * delegation's export rather than the shared nfs4_file, which may + * have been initialized under a different export. + */ + if (exp && !(exp->ex_flags & NFSEXP_NOSUBTREECHECK) && + !S_ISDIR(inode->i_mode)) { + parent = d_inode(nf->nf_file->f_path.dentry); + flags = EXPORT_FH_CONNECTABLE; + } + + fileid_type = exportfs_encode_inode_fh(inode, fid, &maxsize, parent, flags); + if (fileid_type < 0 || fileid_type == FILEID_INVALID) + goto out; + + fhp->fh_fileid_type = fileid_type; + fhp->fh_size += maxsize * 4; + + if (exp && (exp->ex_flags & NFSEXP_SIGN_FH)) + if (!fh_append_mac(fhp, NFS4_FHSIZE, exp->cd->net)) + goto out; + + ret = true; +out: + if (exp) + exp_put(exp); + return ret; +} + +#define CB_NOTIFY_STATX_REQUEST_MASK (STATX_BASIC_STATS | \ + STATX_BTIME | \ + STATX_CHANGE_COOKIE) + +static bool +nfsd4_setup_notify_entry4(struct notify_entry4 *ne, struct xdr_stream *xdr, + struct dentry *dentry, struct nfs4_delegation *dp, + struct nfsd_file *nf, char *name, u32 namelen) +{ + struct path path = nf->nf_file->f_path; + struct nfsd4_fattr_args args = { }; + const u32 *reqmask; + uint32_t *attrmask; + __be32 status; + bool parent; + int ret; + + /* Reserve space for attrmask */ + attrmask = xdr_reserve_space(xdr, 3 * sizeof(uint32_t)); + if (!attrmask) + return false; + + ne->ne_file.data = name; + ne->ne_file.len = namelen; + ne->ne_attrs.attrmask.element = attrmask; + + parent = (dentry == path.dentry); + path.dentry = dentry; + reqmask = parent ? dp->dl_dir_attrs : dp->dl_child_attrs; + + /* + * A NULL or negative dentry has no attributes to report (expected, + * e.g. for the old entry of a rename or an entry already removed). + * The client may also have been granted the notification while + * requesting no attributes for this entry. Both cases encode an + * empty attribute set rather than failing: the vfs_getattr() and + * nfsd4_encode_attr_vals() failures below recall the delegation, so + * a case with nothing to fetch must short-circuit ahead of them. + */ + if (!path.dentry || !d_inode(path.dentry) || + (!reqmask[0] && !reqmask[1])) { + attrmask[0] = 0; + attrmask[1] = 0; + attrmask[2] = 0; + ne->ne_attrs.attr_vals.data = NULL; + ne->ne_attrs.attr_vals.len = 0; + ne->ne_attrs.attrmask.count = 1; + return true; + } + + /* + * It is possible that the client was granted a delegation when a file + * was created. Note that we don't issue a CB_GETATTR here since stale + * attributes are presumably ok. + */ + ret = vfs_getattr(&path, &args.stat, CB_NOTIFY_STATX_REQUEST_MASK, AT_STATX_SYNC_AS_STAT); + if (ret) + return false; + + args.change_attr = nfsd4_change_attribute(&args.stat); + + if (parent) { + attrmask[0] = dp->dl_dir_attrs[0]; + attrmask[1] = dp->dl_dir_attrs[1]; + } else { + attrmask[0] = dp->dl_child_attrs[0]; + attrmask[1] = dp->dl_child_attrs[1]; + + if (!setup_notify_fhandle(dentry, dp, nf, &args)) + attrmask[0] &= ~FATTR4_WORD0_FILEHANDLE; + + if (!(args.stat.result_mask & STATX_BTIME)) + attrmask[1] &= ~FATTR4_WORD1_TIME_CREATE; + } + attrmask[2] = 0; + + ne->ne_attrs.attrmask.count = 2; + ne->ne_attrs.attr_vals.data = (u8 *)xdr->p; + + status = nfsd4_encode_attr_vals(xdr, attrmask, &args); + if (status != nfs_ok) + return false; + + ne->ne_attrs.attr_vals.len = (u8 *)xdr->p - ne->ne_attrs.attr_vals.data; + return true; +} + +/** + * nfsd4_encode_notify_event - encode a notify + * @xdr: stream to which to encode the fattr4 + * @nne: nfsd_notify_event to encode + * @dp: delegation where the event occurred + * @nf: nfsd_file on which event occurred + * @notify_mask: pointer to word where notification mask should be set + * + * Encode @nne into @xdr. The matching bit in @notify_mask is set on + * success. + * + * Return: pointer to the start of the encoded event, or NULL if the + * event could not be encoded. + */ +u8 *nfsd4_encode_notify_event(struct xdr_stream *xdr, struct nfsd_notify_event *nne, + struct nfs4_delegation *dp, struct nfsd_file *nf, + u32 *notify_mask) +{ + u8 *p = NULL; + + *notify_mask = 0; + + if (nne->ne_mask & FS_DELETE) { + struct notify_remove4 nr = { }; + + if (!nfsd4_setup_notify_entry4(&nr.nrm_old_entry, xdr, nne->ne_dentry, dp, + nf, nne->ne_name, nne->ne_namelen)) + goto out_err; + p = (u8 *)xdr->p; + if (!xdrgen_encode_notify_remove4(xdr, &nr)) + goto out_err; + *notify_mask |= BIT(NOTIFY4_REMOVE_ENTRY); + } else if (nne->ne_mask & FS_CREATE) { + struct notify_add4 na = { }; + struct notify_remove4 old = { }; + + if (!nfsd4_setup_notify_entry4(&na.nad_new_entry, xdr, nne->ne_dentry, dp, + nf, nne->ne_name, nne->ne_namelen)) + goto out_err; + + /* If a file was overwritten, report it in nad_old_entry */ + if (nne->ne_target) { + if (!nfsd4_setup_notify_entry4(&old.nrm_old_entry, xdr, + NULL, dp, nf, + nne->ne_name, nne->ne_namelen)) + goto out_err; + na.nad_old_entry.count = 1; + na.nad_old_entry.element = &old; + } + + p = (u8 *)xdr->p; + if (!xdrgen_encode_notify_add4(xdr, &na)) + goto out_err; + + *notify_mask |= BIT(NOTIFY4_ADD_ENTRY); + } else if (nne->ne_mask & FS_RENAME) { + struct notify_rename4 nr = { }; + struct notify_remove4 old = { }; + char *newname = nfsd_notify_event_newname(nne); + + /* Don't send any attributes in the old_entry since they're the same in new */ + if (!nfsd4_setup_notify_entry4(&nr.nrn_old_entry.nrm_old_entry, xdr, + NULL, dp, nf, nne->ne_name, + nne->ne_namelen)) + goto out_err; + + if (!nfsd4_setup_notify_entry4(&nr.nrn_new_entry.nad_new_entry, xdr, + nne->ne_dentry, dp, nf, newname, + nne->ne_newnamelen)) + goto out_err; + + /* If a file was overwritten, report it in nad_old_entry */ + if (nne->ne_target) { + if (!nfsd4_setup_notify_entry4(&old.nrm_old_entry, xdr, + NULL, dp, nf, newname, + nne->ne_newnamelen)) + goto out_err; + nr.nrn_new_entry.nad_old_entry.count = 1; + nr.nrn_new_entry.nad_old_entry.element = &old; + } + + p = (u8 *)xdr->p; + if (!xdrgen_encode_notify_rename4(xdr, &nr)) + goto out_err; + *notify_mask |= BIT(NOTIFY4_RENAME_ENTRY); + } + return p; +out_err: + pr_warn("nfsd: unable to marshal notify event to xdr stream\n"); + return NULL; +} + +/** + * nfsd4_encode_dir_attr_change + * @xdr: stream to which to encode the fattr4 + * @dp: delegation where the event occurred + * @nf: nfsd_file opened on the directory + * + * Encode a dir attr change event. + * + * Return: a pointer to the start of the encoded event on success; NULL + * if there were no requested attributes to report, in which case the + * caller should omit the event; or an ERR_PTR if the event was requested + * but could not be marshalled into @xdr, in which case the caller should + * recall the delegation. + */ +u8 *nfsd4_encode_dir_attr_change(struct xdr_stream *xdr, struct nfs4_delegation *dp, + struct nfsd_file *nf) +{ + struct dentry *dentry = nf->nf_file->f_path.dentry; + struct notify_attr4 na = { }; + u8 *p; + + /* RFC 8881 s10.4.3: ne_file must be a zero-length string for dir attrs */ + if (!nfsd4_setup_notify_entry4(&na.na_changed_entry, xdr, + dentry, dp, nf, "", 0)) + return ERR_PTR(-ENOBUFS); + + /* No requested attributes to report; omit the event */ + if (!na.na_changed_entry.ne_attrs.attr_vals.len) + return NULL; + + p = (u8 *)xdr->p; + if (!xdrgen_encode_notify_attr4(xdr, &na)) + return ERR_PTR(-ENOBUFS); + return p; +} + +static void svcxdr_init_encode_from_buffer(struct xdr_stream *xdr, + struct xdr_buf *buf, __be32 *p, int bytes) +{ + xdr->scratch.iov_len = 0; + memset(buf, 0, sizeof(struct xdr_buf)); + buf->head[0].iov_base = p; + buf->head[0].iov_len = 0; + buf->len = 0; + xdr->buf = buf; + xdr->iov = buf->head; + xdr->p = p; + xdr->end = (void *)p + bytes; + buf->buflen = bytes; +} + +__be32 nfsd4_encode_fattr_to_buf(__be32 **p, int words, + struct svc_fh *fhp, struct svc_export *exp, + struct dentry *dentry, u32 *bmval, + struct svc_rqst *rqstp, int ignore_crossmnt) +{ + struct xdr_buf dummy; + struct xdr_stream xdr; + __be32 ret; + + svcxdr_init_encode_from_buffer(&xdr, &dummy, *p, words << 2); + ret = nfsd4_encode_fattr4(rqstp, &xdr, fhp, exp, dentry, bmval, + ignore_crossmnt, NULL); + *p = xdr.p; + return ret; +} + +/* + * The buffer space for this field was reserved during a previous + * call to nfsd4_encode_entry4(). + */ +static void nfsd4_encode_entry4_nfs_cookie4(const struct nfsd4_readdir *readdir, + u64 offset) +{ + __be64 cookie = cpu_to_be64(offset); + struct xdr_stream *xdr = readdir->xdr; + + if (!readdir->cookie_offset) + return; + write_bytes_to_xdr_buf(xdr->buf, readdir->cookie_offset, &cookie, + sizeof(cookie)); +} + +static inline int attributes_need_mount(u32 *bmval) +{ + if (bmval[0] & ~(FATTR4_WORD0_RDATTR_ERROR | FATTR4_WORD0_LEASE_TIME)) + return 1; + if (bmval[1] & ~FATTR4_WORD1_MOUNTED_ON_FILEID) + return 1; + return 0; +} + +static __be32 +nfsd4_encode_entry4_fattr(struct nfsd4_readdir *cd, const char *name, + int namlen) +{ + struct svc_export *exp = cd->rd_fhp->fh_export; + struct dentry *dentry; + __be32 nfserr; + int ignore_crossmnt = 0; + bool crossed = false; + + dentry = lookup_one_positive_unlocked(&nop_mnt_idmap, + &QSTR_LEN(name, namlen), + cd->rd_fhp->fh_dentry); + if (IS_ERR(dentry)) + return nfserrno(PTR_ERR(dentry)); + + exp_get(exp); + /* + * In the case of a mountpoint, the client may be asking for + * attributes that are only properties of the underlying filesystem + * as opposed to the cross-mounted file system. In such a case, + * we will not follow the cross mount and will fill the attribtutes + * directly from the mountpoint dentry. + */ + if (nfsd_mountpoint(dentry, exp)) { + int err; + + if (!(exp->ex_flags & NFSEXP_V4ROOT) + && !attributes_need_mount(cd->rd_bmval)) { + ignore_crossmnt = 1; + goto out_encode; + } + /* + * Why the heck aren't we just using nfsd_lookup?? + * Different "."/".." handling? Something else? + * At least, add a comment here to explain.... + */ + err = nfsd_cross_mnt(cd->rd_rqstp, &dentry, &exp); + if (err) { + nfserr = nfserrno(err); + goto out_put; + } + nfserr = check_nfsd_access(exp, cd->rd_rqstp, false); + if (nfserr) + goto out_put; + crossed = true; + + } +out_encode: + /* + * A crossed entry no longer shares a parent with the directory + * being read, so it must neither consume nor populate the + * per-readdir case-folding cache. + */ + nfserr = nfsd4_encode_fattr4(cd->rd_rqstp, cd->xdr, NULL, exp, dentry, + cd->rd_bmval, ignore_crossmnt, + crossed ? NULL : &cd->rd_case_cache); +out_put: + dput(dentry); + exp_put(exp); + return nfserr; +} + +static __be32 +nfsd4_encode_entry4_rdattr_error(struct xdr_stream *xdr, __be32 nfserr) +{ + __be32 status; + + /* attrmask */ + status = nfsd4_encode_bitmap4(xdr, FATTR4_WORD0_RDATTR_ERROR, 0, 0); + if (status != nfs_ok) + return status; + /* attr_vals */ + if (xdr_stream_encode_u32(xdr, XDR_UNIT) != XDR_UNIT) + return nfserr_resource; + /* rdattr_error */ + if (xdr_stream_encode_be32(xdr, nfserr) != XDR_UNIT) + return nfserr_resource; + return nfs_ok; +} + +static int +nfsd4_encode_entry4(void *ccdv, const char *name, int namlen, + loff_t offset, u64 ino, unsigned int d_type) +{ + struct readdir_cd *ccd = ccdv; + struct nfsd4_readdir *cd = container_of(ccd, struct nfsd4_readdir, common); + struct xdr_stream *xdr = cd->xdr; + int start_offset = xdr->buf->len; + int cookie_offset; + u32 name_and_cookie; + int entry_bytes; + __be32 nfserr = nfserr_toosmall; + + /* In nfsv4, "." and ".." never make it onto the wire.. */ + if (name && name_is_dot_dotdot(name, namlen)) { + cd->common.err = nfs_ok; + return 0; + } + + /* Encode the previous entry's cookie value */ + nfsd4_encode_entry4_nfs_cookie4(cd, offset); + + if (xdr_stream_encode_item_present(xdr) != XDR_UNIT) + goto fail; + + /* Reserve send buffer space for this entry's cookie value. */ + cookie_offset = xdr->buf->len; + if (nfsd4_encode_nfs_cookie4(xdr, OFFSET_MAX) != nfs_ok) + goto fail; + if (nfsd4_encode_component4(xdr, name, namlen) != nfs_ok) + goto fail; + nfserr = nfsd4_encode_entry4_fattr(cd, name, namlen); + switch (nfserr) { + case nfs_ok: + break; + case nfserr_resource: + nfserr = nfserr_toosmall; + goto fail; + case nfserr_noent: + xdr_truncate_encode(xdr, start_offset); + goto skip_entry; + case nfserr_jukebox: + /* + * The pseudoroot should only display dentries that lead to + * exports. If we get EJUKEBOX here, then we can't tell whether + * this entry should be included. Just fail the whole READDIR + * with NFS4ERR_DELAY in that case, and hope that the situation + * will resolve itself by the client's next attempt. + */ + if (cd->rd_fhp->fh_export->ex_flags & NFSEXP_V4ROOT) + goto fail; + fallthrough; + default: + /* + * If the client requested the RDATTR_ERROR attribute, + * we stuff the error code into this attribute + * and continue. If this attribute was not requested, + * then in accordance with the spec, we fail the + * entire READDIR operation(!) + */ + if (!(cd->rd_bmval[0] & FATTR4_WORD0_RDATTR_ERROR)) + goto fail; + if (nfsd4_encode_entry4_rdattr_error(xdr, nfserr)) { + nfserr = nfserr_toosmall; + goto fail; + } + } + nfserr = nfserr_toosmall; + entry_bytes = xdr->buf->len - start_offset; + if (entry_bytes > cd->rd_maxcount) + goto fail; + cd->rd_maxcount -= entry_bytes; + /* + * RFC 3530 14.2.24 describes rd_dircount as only a "hint", and + * notes that it could be zero. If it is zero, then the server + * should enforce only the rd_maxcount value. + */ + if (cd->rd_dircount) { + name_and_cookie = 4 + 4 * XDR_QUADLEN(namlen) + 8; + if (name_and_cookie > cd->rd_dircount && cd->cookie_offset) + goto fail; + cd->rd_dircount -= min(cd->rd_dircount, name_and_cookie); + if (!cd->rd_dircount) + cd->rd_maxcount = 0; + } + + cd->cookie_offset = cookie_offset; +skip_entry: + cd->common.err = nfs_ok; + return 0; +fail: + xdr_truncate_encode(xdr, start_offset); + cd->common.err = nfserr; + return -EINVAL; +} + +static __be32 +nfsd4_encode_verifier4(struct xdr_stream *xdr, const nfs4_verifier *verf) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, NFS4_VERIFIER_SIZE); + if (!p) + return nfserr_resource; + memcpy(p, verf->data, sizeof(verf->data)); + return nfs_ok; +} + +static __be32 +nfsd4_encode_clientid4(struct xdr_stream *xdr, const clientid_t *clientid) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, sizeof(__be64)); + if (!p) + return nfserr_resource; + memcpy(p, clientid, sizeof(*clientid)); + return nfs_ok; +} + +/* This is a frequently-encoded item; open-coded for speed */ +static __be32 +nfsd4_encode_stateid4(struct xdr_stream *xdr, const stateid_t *sid) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, NFS4_STATEID_SIZE); + if (!p) + return nfserr_resource; + *p++ = cpu_to_be32(sid->si_generation); + memcpy(p, &sid->si_opaque, sizeof(sid->si_opaque)); + return nfs_ok; +} + +static __be32 +nfsd4_encode_sessionid4(struct xdr_stream *xdr, + const struct nfs4_sessionid *sessionid) +{ + return nfsd4_encode_opaque_fixed(xdr, sessionid->data, + NFS4_MAX_SESSIONID_LEN); +} + +static __be32 +nfsd4_encode_access(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_access *access = &u->access; + struct xdr_stream *xdr = resp->xdr; + __be32 status; + + /* supported */ + status = nfsd4_encode_uint32_t(xdr, access->ac_supported); + if (status != nfs_ok) + return status; + /* access */ + return nfsd4_encode_uint32_t(xdr, access->ac_resp_access); +} + +static __be32 nfsd4_encode_bind_conn_to_session(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_bind_conn_to_session *bcts = &u->bind_conn_to_session; + struct xdr_stream *xdr = resp->xdr; + + /* bctsr_sessid */ + nfserr = nfsd4_encode_sessionid4(xdr, &bcts->sessionid); + if (nfserr != nfs_ok) + return nfserr; + /* bctsr_dir */ + if (xdr_stream_encode_u32(xdr, bcts->dir) != XDR_UNIT) + return nfserr_resource; + /* bctsr_use_conn_in_rdma_mode */ + return nfsd4_encode_bool(xdr, false); +} + +static __be32 +nfsd4_encode_close(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_close *close = &u->close; + struct xdr_stream *xdr = resp->xdr; + + /* open_stateid */ + return nfsd4_encode_stateid4(xdr, &close->cl_stateid); +} + + +static __be32 +nfsd4_encode_commit(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_commit *commit = &u->commit; + + return nfsd4_encode_verifier4(resp->xdr, &commit->co_verf); +} + +static __be32 +nfsd4_encode_create(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_create *create = &u->create; + struct xdr_stream *xdr = resp->xdr; + + /* cinfo */ + nfserr = nfsd4_encode_change_info4(xdr, &create->cr_cinfo); + if (nfserr) + return nfserr; + /* attrset */ + return nfsd4_encode_bitmap4(xdr, create->cr_bmval[0], + create->cr_bmval[1], create->cr_bmval[2]); +} + +static __be32 +nfsd4_encode_getattr(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_getattr *getattr = &u->getattr; + struct svc_fh *fhp = getattr->ga_fhp; + struct xdr_stream *xdr = resp->xdr; + + /* obj_attributes */ + return nfsd4_encode_fattr4(resp->rqstp, xdr, fhp, fhp->fh_export, + fhp->fh_dentry, getattr->ga_bmval, 0, NULL); +} + +static __be32 +nfsd4_encode_getfh(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct xdr_stream *xdr = resp->xdr; + struct svc_fh *fhp = u->getfh; + + /* object */ + return nfsd4_encode_nfs_fh4(xdr, &fhp->fh_handle); +} + +static __be32 +nfsd4_encode_lock_owner4(struct xdr_stream *xdr, const clientid_t *clientid, + const struct xdr_netobj *owner) +{ + __be32 status; + + /* clientid */ + status = nfsd4_encode_clientid4(xdr, clientid); + if (status != nfs_ok) + return status; + /* owner */ + return nfsd4_encode_opaque(xdr, owner->data, owner->len); +} + +static __be32 +nfsd4_encode_lock4denied(struct xdr_stream *xdr, + const struct nfsd4_lock_denied *ld) +{ + __be32 status; + + /* offset */ + status = nfsd4_encode_offset4(xdr, ld->ld_start); + if (status != nfs_ok) + return status; + /* length */ + status = nfsd4_encode_length4(xdr, ld->ld_length); + if (status != nfs_ok) + return status; + /* locktype */ + if (xdr_stream_encode_u32(xdr, ld->ld_type) != XDR_UNIT) + return nfserr_resource; + /* owner */ + return nfsd4_encode_lock_owner4(xdr, &ld->ld_clientid, + &ld->ld_owner); +} + +static __be32 +nfsd4_encode_lock(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_lock *lock = &u->lock; + struct xdr_stream *xdr = resp->xdr; + __be32 status; + + switch (nfserr) { + case nfs_ok: + /* resok4 */ + status = nfsd4_encode_stateid4(xdr, &lock->lk_resp_stateid); + break; + case nfserr_denied: + /* denied */ + status = nfsd4_encode_lock4denied(xdr, &lock->lk_denied); + break; + default: + return nfserr; + } + return status != nfs_ok ? status : nfserr; +} + +static __be32 +nfsd4_encode_lockt(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_lockt *lockt = &u->lockt; + struct xdr_stream *xdr = resp->xdr; + __be32 status; + + if (nfserr == nfserr_denied) { + /* denied */ + status = nfsd4_encode_lock4denied(xdr, &lockt->lt_denied); + if (status != nfs_ok) + return status; + } + return nfserr; +} + +static __be32 +nfsd4_encode_locku(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_locku *locku = &u->locku; + struct xdr_stream *xdr = resp->xdr; + + /* lock_stateid */ + return nfsd4_encode_stateid4(xdr, &locku->lu_stateid); +} + + +static __be32 +nfsd4_encode_link(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_link *link = &u->link; + struct xdr_stream *xdr = resp->xdr; + + return nfsd4_encode_change_info4(xdr, &link->li_cinfo); +} + +/* + * This implementation does not yet support returning an ACE in an + * OPEN that offers a delegation. + */ +static __be32 +nfsd4_encode_open_nfsace4(struct xdr_stream *xdr) +{ + __be32 status; + + /* type */ + status = nfsd4_encode_acetype4(xdr, NFS4_ACE_ACCESS_ALLOWED_ACE_TYPE); + if (status != nfs_ok) + return nfserr_resource; + /* flag */ + status = nfsd4_encode_aceflag4(xdr, 0); + if (status != nfs_ok) + return nfserr_resource; + /* access mask */ + status = nfsd4_encode_acemask4(xdr, 0); + if (status != nfs_ok) + return nfserr_resource; + /* who - empty for now */ + if (xdr_stream_encode_u32(xdr, 0) != XDR_UNIT) + return nfserr_resource; + return nfs_ok; +} + +static __be32 +nfsd4_encode_open_read_delegation4(struct xdr_stream *xdr, struct nfsd4_open *open) +{ + __be32 status; + + /* stateid */ + status = nfsd4_encode_stateid4(xdr, &open->op_delegate_stateid); + if (status != nfs_ok) + return status; + /* recall */ + status = nfsd4_encode_bool(xdr, open->op_recall); + if (status != nfs_ok) + return status; + /* permissions */ + return nfsd4_encode_open_nfsace4(xdr); +} + +static __be32 +nfsd4_encode_nfs_space_limit4(struct xdr_stream *xdr, u64 filesize) +{ + /* limitby */ + if (xdr_stream_encode_u32(xdr, NFS4_LIMIT_SIZE) != XDR_UNIT) + return nfserr_resource; + /* filesize */ + return nfsd4_encode_uint64_t(xdr, filesize); +} + +static __be32 +nfsd4_encode_open_write_delegation4(struct xdr_stream *xdr, + struct nfsd4_open *open) +{ + __be32 status; + + /* stateid */ + status = nfsd4_encode_stateid4(xdr, &open->op_delegate_stateid); + if (status != nfs_ok) + return status; + /* recall */ + status = nfsd4_encode_bool(xdr, open->op_recall); + if (status != nfs_ok) + return status; + /* space_limit */ + status = nfsd4_encode_nfs_space_limit4(xdr, 0); + if (status != nfs_ok) + return status; + return nfsd4_encode_open_nfsace4(xdr); +} + +static __be32 +nfsd4_encode_open_none_delegation4(struct xdr_stream *xdr, + struct nfsd4_open *open) +{ + __be32 status = nfs_ok; + + /* ond_why */ + if (xdr_stream_encode_u32(xdr, open->op_why_no_deleg) != XDR_UNIT) + return nfserr_resource; + switch (open->op_why_no_deleg) { + case WND4_CONTENTION: + /* ond_server_will_push_deleg */ + status = nfsd4_encode_bool(xdr, false); + break; + case WND4_RESOURCE: + /* ond_server_will_signal_avail */ + status = nfsd4_encode_bool(xdr, false); + } + return status; +} + +static __be32 +nfsd4_encode_open_delegation4(struct xdr_stream *xdr, struct nfsd4_open *open) +{ + __be32 status; + + /* delegation_type */ + if (xdr_stream_encode_u32(xdr, open->op_delegate_type) != XDR_UNIT) + return nfserr_resource; + switch (open->op_delegate_type) { + case OPEN_DELEGATE_NONE: + status = nfs_ok; + break; + case OPEN_DELEGATE_READ: + case OPEN_DELEGATE_READ_ATTRS_DELEG: + /* read */ + status = nfsd4_encode_open_read_delegation4(xdr, open); + break; + case OPEN_DELEGATE_WRITE: + case OPEN_DELEGATE_WRITE_ATTRS_DELEG: + /* write */ + status = nfsd4_encode_open_write_delegation4(xdr, open); + break; + case OPEN_DELEGATE_NONE_EXT: + /* od_whynone */ + status = nfsd4_encode_open_none_delegation4(xdr, open); + break; + default: + status = nfserr_serverfault; + } + + return status; +} + +static __be32 +nfsd4_encode_open(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_open *open = &u->open; + struct xdr_stream *xdr = resp->xdr; + + /* stateid */ + nfserr = nfsd4_encode_stateid4(xdr, &open->op_stateid); + if (nfserr != nfs_ok) + return nfserr; + /* cinfo */ + nfserr = nfsd4_encode_change_info4(xdr, &open->op_cinfo); + if (nfserr != nfs_ok) + return nfserr; + /* rflags */ + nfserr = nfsd4_encode_uint32_t(xdr, open->op_rflags); + if (nfserr != nfs_ok) + return nfserr; + /* attrset */ + nfserr = nfsd4_encode_bitmap4(xdr, open->op_bmval[0], + open->op_bmval[1], open->op_bmval[2]); + if (nfserr != nfs_ok) + return nfserr; + /* delegation */ + return nfsd4_encode_open_delegation4(xdr, open); +} + +static __be32 +nfsd4_encode_open_confirm(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_open_confirm *oc = &u->open_confirm; + struct xdr_stream *xdr = resp->xdr; + + /* open_stateid */ + return nfsd4_encode_stateid4(xdr, &oc->oc_resp_stateid); +} + +static __be32 +nfsd4_encode_open_downgrade(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_open_downgrade *od = &u->open_downgrade; + struct xdr_stream *xdr = resp->xdr; + + /* open_stateid */ + return nfsd4_encode_stateid4(xdr, &od->od_stateid); +} + +/* + * The operation of this function assumes that this is the only + * READ operation in the COMPOUND. If there are multiple READs, + * we use nfsd4_encode_readv(). + */ +static __be32 nfsd4_encode_splice_read( + struct nfsd4_compoundres *resp, + struct nfsd4_read *read, + struct file *file, unsigned long maxcount) +{ + struct xdr_stream *xdr = resp->xdr; + struct xdr_buf *buf = xdr->buf; + int status, space_left; + __be32 nfserr; + + /* + * Splice read doesn't work if encoding has already wandered + * into the XDR buf's page array. + */ + if (unlikely(xdr->buf->page_len)) { + WARN_ON_ONCE(1); + return nfserr_serverfault; + } + + /* + * Make sure there is room at the end of buf->head for + * svcxdr_encode_opaque_pages() to create a tail buffer + * to XDR-pad the payload. + */ + if (xdr->iov != xdr->buf->head || xdr->end - xdr->p < 1) + return nfserr_resource; + + nfserr = nfsd_splice_read(read->rd_rqstp, read->rd_fhp, + file, read->rd_offset, &maxcount, + &read->rd_eof); + read->rd_length = maxcount; + if (nfserr) + goto out_err; + svcxdr_encode_opaque_pages(read->rd_rqstp, xdr, buf->pages, + buf->page_base, maxcount); + status = svc_encode_result_payload(read->rd_rqstp, + buf->head[0].iov_len, maxcount); + if (status) { + nfserr = nfserrno(status); + goto out_err; + } + + /* + * Prepare to encode subsequent operations. + * + * xdr_truncate_encode() is not safe to use after a successful + * splice read has been done, so the following stream + * manipulations are open-coded. + */ + space_left = min_t(int, (void *)xdr->end - (void *)xdr->p, + buf->buflen - buf->len); + buf->buflen = buf->len + space_left; + xdr->end = (__be32 *)((void *)xdr->end + space_left); + + return nfs_ok; + +out_err: + /* + * nfsd_splice_actor may have already messed with the + * page length; reset it so as not to confuse + * xdr_truncate_encode in our caller. + */ + buf->page_len = 0; + return nfserr; +} + +static __be32 nfsd4_encode_readv(struct nfsd4_compoundres *resp, + struct nfsd4_read *read, + unsigned long maxcount) +{ + struct xdr_stream *xdr = resp->xdr; + unsigned int base = xdr->buf->page_len & ~PAGE_MASK; + unsigned int starting_len = xdr->buf->len; + __be32 zero = xdr_zero; + __be32 nfserr; + + nfserr = nfsd_iter_read(resp->rqstp, read->rd_fhp, read->rd_nf, + read->rd_offset, &maxcount, base, + &read->rd_eof); + read->rd_length = maxcount; + if (nfserr) + return nfserr; + + /* + * svcxdr_encode_opaque_pages() is not used here because + * we don't want to encode subsequent results in this + * COMPOUND into the xdr->buf's tail, but rather those + * results should follow the NFS READ payload in the + * buf's pages. + */ + if (xdr_reserve_space_vec(xdr, maxcount) < 0) + return nfserr_resource; + + /* + * Mark the buffer location of the NFS READ payload so that + * direct placement-capable transports send only the + * payload bytes out-of-band. + */ + if (svc_encode_result_payload(resp->rqstp, starting_len, maxcount)) + return nfserr_io; + + write_bytes_to_xdr_buf(xdr->buf, starting_len + maxcount, &zero, + xdr_pad_size(maxcount)); + return nfs_ok; +} + +static __be32 +nfsd4_encode_read(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_compoundargs *argp = resp->rqstp->rq_argp; + struct nfsd4_read *read = &u->read; + struct xdr_stream *xdr = resp->xdr; + bool splice_ok = argp->splice_ok; + unsigned int eof_offset; + unsigned long maxcount; + __be32 wire_data[2]; + struct file *file; + + if (nfserr) + return nfserr; + + eof_offset = xdr->buf->len; + file = read->rd_nf->nf_file; + + /* Reserve space for the eof flag and byte count */ + if (unlikely(!xdr_reserve_space(xdr, XDR_UNIT * 2))) { + WARN_ON_ONCE(splice_ok); + return nfserr_resource; + } + xdr_commit_encode(xdr); + + maxcount = min_t(unsigned long, read->rd_length, + (xdr->buf->buflen - xdr->buf->len)); + + if (file->f_op->splice_read && splice_ok) + nfserr = nfsd4_encode_splice_read(resp, read, file, maxcount); + else + nfserr = nfsd4_encode_readv(resp, read, maxcount); + if (nfserr) { + xdr_truncate_encode(xdr, eof_offset); + return nfserr; + } + + wire_data[0] = read->rd_eof ? xdr_one : xdr_zero; + wire_data[1] = cpu_to_be32(read->rd_length); + write_bytes_to_xdr_buf(xdr->buf, eof_offset, &wire_data, XDR_UNIT * 2); + return nfs_ok; +} + +static __be32 +nfsd4_encode_readlink(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_readlink *readlink = &u->readlink; + __be32 *p, wire_count, zero = xdr_zero; + struct xdr_stream *xdr = resp->xdr; + unsigned int length_offset; + int maxcount, status; + + /* linktext4.count */ + length_offset = xdr->buf->len; + if (unlikely(!xdr_reserve_space(xdr, XDR_UNIT))) + return nfserr_resource; + + /* linktext4.data */ + maxcount = PAGE_SIZE; + p = xdr_reserve_space(xdr, maxcount); + if (!p) + return nfserr_resource; + nfserr = nfsd_readlink(readlink->rl_rqstp, readlink->rl_fhp, + (char *)p, &maxcount); + if (nfserr == nfserr_isdir) + nfserr = nfserr_inval; + if (nfserr) + goto out_err; + status = svc_encode_result_payload(readlink->rl_rqstp, length_offset, + maxcount); + if (status) { + nfserr = nfserrno(status); + goto out_err; + } + + wire_count = cpu_to_be32(maxcount); + write_bytes_to_xdr_buf(xdr->buf, length_offset, &wire_count, XDR_UNIT); + xdr_truncate_encode(xdr, length_offset + 4 + xdr_align_size(maxcount)); + write_bytes_to_xdr_buf(xdr->buf, length_offset + 4 + maxcount, &zero, + xdr_pad_size(maxcount)); + return nfs_ok; + +out_err: + xdr_truncate_encode(xdr, length_offset); + return nfserr; +} + +static __be32 nfsd4_encode_dirlist4(struct xdr_stream *xdr, + struct nfsd4_readdir *readdir, + u32 max_payload) +{ + int bytes_left, maxcount, starting_len = xdr->buf->len; + loff_t offset; + __be32 status; + + /* + * Number of bytes left for directory entries allowing for the + * final 8 bytes of the readdir and a following failed op. + */ + bytes_left = xdr->buf->buflen - xdr->buf->len - + COMPOUND_ERR_SLACK_SPACE - XDR_UNIT * 2; + if (bytes_left < 0) + return nfserr_resource; + maxcount = min_t(u32, readdir->rd_maxcount, max_payload); + + /* + * The RFC defines rd_maxcount as the size of the + * READDIR4resok structure, which includes the verifier + * and the 8 bytes encoded at the end of this function. + */ + if (maxcount < XDR_UNIT * 4) + return nfserr_toosmall; + maxcount = min_t(int, maxcount - XDR_UNIT * 4, bytes_left); + + /* RFC 3530 14.2.24 allows us to ignore dircount when it's 0 */ + if (!readdir->rd_dircount) + readdir->rd_dircount = max_payload; + + /* *entries */ + readdir->xdr = xdr; + readdir->rd_maxcount = maxcount; + readdir->common.err = 0; + readdir->cookie_offset = 0; + readdir->rd_case_cache.dir = readdir->rd_fhp->fh_dentry; + readdir->rd_case_cache.valid = false; + offset = readdir->rd_cookie; + status = nfsd_readdir(readdir->rd_rqstp, readdir->rd_fhp, &offset, + &readdir->common, nfsd4_encode_entry4); + if (status) + return status; + if (readdir->common.err == nfserr_toosmall && + xdr->buf->len == starting_len) { + /* No entries were encoded. Which limit did we hit? */ + if (maxcount - XDR_UNIT * 4 < bytes_left) + /* It was the fault of rd_maxcount */ + return nfserr_toosmall; + /* We ran out of buffer space */ + return nfserr_resource; + } + /* Encode the final entry's cookie value */ + nfsd4_encode_entry4_nfs_cookie4(readdir, offset); + /* No entries follow */ + if (xdr_stream_encode_item_absent(xdr) != XDR_UNIT) + return nfserr_resource; + + /* eof */ + return nfsd4_encode_bool(xdr, readdir->common.err == nfserr_eof); +} + +static __be32 +nfsd4_encode_readdir(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_readdir *readdir = &u->readdir; + struct xdr_stream *xdr = resp->xdr; + int starting_len = xdr->buf->len; + + /* cookieverf */ + nfserr = nfsd4_encode_verifier4(xdr, &readdir->rd_verf); + if (nfserr != nfs_ok) + return nfserr; + + /* reply */ + nfserr = nfsd4_encode_dirlist4(xdr, readdir, svc_max_payload(resp->rqstp)); + if (nfserr != nfs_ok) + xdr_truncate_encode(xdr, starting_len); + return nfserr; +} + +static __be32 +nfsd4_encode_remove(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_remove *remove = &u->remove; + struct xdr_stream *xdr = resp->xdr; + + return nfsd4_encode_change_info4(xdr, &remove->rm_cinfo); +} + +static __be32 +nfsd4_encode_rename(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_rename *rename = &u->rename; + struct xdr_stream *xdr = resp->xdr; + + nfserr = nfsd4_encode_change_info4(xdr, &rename->rn_sinfo); + if (nfserr) + return nfserr; + return nfsd4_encode_change_info4(xdr, &rename->rn_tinfo); +} + +static __be32 +nfsd4_encode_rpcsec_gss_info(struct xdr_stream *xdr, + struct rpcsec_gss_info *info) +{ + __be32 status; + + /* oid */ + if (xdr_stream_encode_opaque(xdr, info->oid.data, info->oid.len) < 0) + return nfserr_resource; + /* qop */ + status = nfsd4_encode_qop4(xdr, info->qop); + if (status != nfs_ok) + return status; + /* service */ + if (xdr_stream_encode_u32(xdr, info->service) != XDR_UNIT) + return nfserr_resource; + + return nfs_ok; +} + +static __be32 +nfsd4_encode_secinfo4(struct xdr_stream *xdr, rpc_authflavor_t pf, + u32 *supported) +{ + struct rpcsec_gss_info info; + __be32 status; + + if (rpcauth_get_gssinfo(pf, &info) == 0) { + (*supported)++; + + /* flavor */ + status = nfsd4_encode_uint32_t(xdr, RPC_AUTH_GSS); + if (status != nfs_ok) + return status; + /* flavor_info */ + status = nfsd4_encode_rpcsec_gss_info(xdr, &info); + if (status != nfs_ok) + return status; + } else if (pf < RPC_AUTH_MAXFLAVOR) { + (*supported)++; + + /* flavor */ + status = nfsd4_encode_uint32_t(xdr, pf); + if (status != nfs_ok) + return status; + } + return nfs_ok; +} + +static __be32 +nfsd4_encode_SECINFO4resok(struct xdr_stream *xdr, struct svc_export *exp) +{ + u32 i, nflavs, supported; + struct exp_flavor_info *flavs; + struct exp_flavor_info def_flavs[2]; + unsigned int count_offset; + __be32 status, wire_count; + + if (exp->ex_nflavors) { + flavs = exp->ex_flavors; + nflavs = exp->ex_nflavors; + } else { /* Handling of some defaults in absence of real secinfo: */ + flavs = def_flavs; + if (exp->ex_client->flavour->flavour == RPC_AUTH_UNIX) { + nflavs = 2; + flavs[0].pseudoflavor = RPC_AUTH_UNIX; + flavs[1].pseudoflavor = RPC_AUTH_NULL; + } else if (exp->ex_client->flavour->flavour == RPC_AUTH_GSS) { + nflavs = 1; + flavs[0].pseudoflavor + = svcauth_gss_flavor(exp->ex_client); + } else { + nflavs = 1; + flavs[0].pseudoflavor + = exp->ex_client->flavour->flavour; + } + } + + count_offset = xdr->buf->len; + if (unlikely(!xdr_reserve_space(xdr, XDR_UNIT))) + return nfserr_resource; + + for (i = 0, supported = 0; i < nflavs; i++) { + status = nfsd4_encode_secinfo4(xdr, flavs[i].pseudoflavor, + &supported); + if (status != nfs_ok) + return status; + } + + wire_count = cpu_to_be32(supported); + write_bytes_to_xdr_buf(xdr->buf, count_offset, &wire_count, + XDR_UNIT); + return 0; +} + +static __be32 +nfsd4_encode_secinfo(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_secinfo *secinfo = &u->secinfo; + struct xdr_stream *xdr = resp->xdr; + + return nfsd4_encode_SECINFO4resok(xdr, secinfo->si_exp); +} + +static __be32 +nfsd4_encode_secinfo_no_name(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_secinfo_no_name *secinfo = &u->secinfo_no_name; + struct xdr_stream *xdr = resp->xdr; + + return nfsd4_encode_SECINFO4resok(xdr, secinfo->sin_exp); +} + +static __be32 +nfsd4_encode_setattr(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_setattr *setattr = &u->setattr; + __be32 status; + + switch (nfserr) { + case nfs_ok: + /* attrsset */ + status = nfsd4_encode_bitmap4(resp->xdr, setattr->sa_bmval[0], + setattr->sa_bmval[1], + setattr->sa_bmval[2]); + break; + default: + /* attrsset */ + status = nfsd4_encode_bitmap4(resp->xdr, 0, 0, 0); + } + return status != nfs_ok ? status : nfserr; +} + +static __be32 +nfsd4_encode_setclientid(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_setclientid *scd = &u->setclientid; + struct xdr_stream *xdr = resp->xdr; + + if (!nfserr) { + nfserr = nfsd4_encode_clientid4(xdr, &scd->se_clientid); + if (nfserr != nfs_ok) + goto out; + nfserr = nfsd4_encode_verifier4(xdr, &scd->se_confirm); + } else if (nfserr == nfserr_clid_inuse) { + /* empty network id */ + if (xdr_stream_encode_u32(xdr, 0) < 0) { + nfserr = nfserr_resource; + goto out; + } + /* empty universal address */ + if (xdr_stream_encode_u32(xdr, 0) < 0) { + nfserr = nfserr_resource; + goto out; + } + } +out: + return nfserr; +} + +static __be32 +nfsd4_encode_write(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_write *write = &u->write; + struct xdr_stream *xdr = resp->xdr; + + /* count */ + nfserr = nfsd4_encode_count4(xdr, write->wr_bytes_written); + if (nfserr) + return nfserr; + /* committed */ + if (xdr_stream_encode_u32(xdr, write->wr_how_written) != XDR_UNIT) + return nfserr_resource; + /* writeverf */ + return nfsd4_encode_verifier4(xdr, &write->wr_verifier); +} + +static __be32 +nfsd4_encode_state_protect_ops4(struct xdr_stream *xdr, + struct nfsd4_exchange_id *exid) +{ + __be32 status; + + /* spo_must_enforce */ + status = nfsd4_encode_bitmap4(xdr, exid->spo_must_enforce[0], + exid->spo_must_enforce[1], + exid->spo_must_enforce[2]); + if (status != nfs_ok) + return status; + /* spo_must_allow */ + return nfsd4_encode_bitmap4(xdr, exid->spo_must_allow[0], + exid->spo_must_allow[1], + exid->spo_must_allow[2]); +} + +static __be32 +nfsd4_encode_state_protect4_r(struct xdr_stream *xdr, struct nfsd4_exchange_id *exid) +{ + __be32 status; + + if (xdr_stream_encode_u32(xdr, exid->spa_how) != XDR_UNIT) + return nfserr_resource; + switch (exid->spa_how) { + case SP4_NONE: + status = nfs_ok; + break; + case SP4_MACH_CRED: + /* spr_mach_ops */ + status = nfsd4_encode_state_protect_ops4(xdr, exid); + break; + default: + status = nfserr_serverfault; + } + return status; +} + +static __be32 +nfsd4_encode_server_owner4(struct xdr_stream *xdr, struct svc_rqst *rqstp) +{ + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + __be32 status; + + /* so_minor_id */ + status = nfsd4_encode_uint64_t(xdr, 0); + if (status != nfs_ok) + return status; + /* so_major_id */ + return nfsd4_encode_opaque(xdr, nn->nfsd_name, strlen(nn->nfsd_name)); +} + +static __be32 +nfsd4_encode_nfs_impl_id4(struct xdr_stream *xdr, struct nfsd4_exchange_id *exid) +{ + __be32 status; + + /* nii_domain */ + status = nfsd4_encode_opaque(xdr, exid->nii_domain.data, + exid->nii_domain.len); + if (status != nfs_ok) + return status; + /* nii_name */ + status = nfsd4_encode_opaque(xdr, exid->nii_name.data, + exid->nii_name.len); + if (status != nfs_ok) + return status; + /* nii_time */ + return nfsd4_encode_nfstime4(xdr, &exid->nii_time); +} + +static __be32 +nfsd4_encode_exchange_id(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd_net *nn = net_generic(SVC_NET(resp->rqstp), nfsd_net_id); + struct nfsd4_exchange_id *exid = &u->exchange_id; + struct xdr_stream *xdr = resp->xdr; + + /* eir_clientid */ + nfserr = nfsd4_encode_clientid4(xdr, &exid->clientid); + if (nfserr != nfs_ok) + return nfserr; + /* eir_sequenceid */ + nfserr = nfsd4_encode_sequenceid4(xdr, exid->seqid); + if (nfserr != nfs_ok) + return nfserr; + /* eir_flags */ + nfserr = nfsd4_encode_uint32_t(xdr, exid->flags); + if (nfserr != nfs_ok) + return nfserr; + /* eir_state_protect */ + nfserr = nfsd4_encode_state_protect4_r(xdr, exid); + if (nfserr != nfs_ok) + return nfserr; + /* eir_server_owner */ + nfserr = nfsd4_encode_server_owner4(xdr, resp->rqstp); + if (nfserr != nfs_ok) + return nfserr; + /* eir_server_scope */ + nfserr = nfsd4_encode_opaque(xdr, nn->nfsd_name, + strlen(nn->nfsd_name)); + if (nfserr != nfs_ok) + return nfserr; + /* eir_server_impl_id<1> */ + if (xdr_stream_encode_u32(xdr, 1) != XDR_UNIT) + return nfserr_resource; + nfserr = nfsd4_encode_nfs_impl_id4(xdr, exid); + if (nfserr != nfs_ok) + return nfserr; + + return nfs_ok; +} + +static __be32 +nfsd4_encode_channel_attrs4(struct xdr_stream *xdr, + const struct nfsd4_channel_attrs *attrs) +{ + __be32 status; + + /* ca_headerpadsize */ + status = nfsd4_encode_count4(xdr, 0); + if (status != nfs_ok) + return status; + /* ca_maxrequestsize */ + status = nfsd4_encode_count4(xdr, attrs->maxreq_sz); + if (status != nfs_ok) + return status; + /* ca_maxresponsesize */ + status = nfsd4_encode_count4(xdr, attrs->maxresp_sz); + if (status != nfs_ok) + return status; + /* ca_maxresponsesize_cached */ + status = nfsd4_encode_count4(xdr, attrs->maxresp_cached); + if (status != nfs_ok) + return status; + /* ca_maxoperations */ + status = nfsd4_encode_count4(xdr, attrs->maxops); + if (status != nfs_ok) + return status; + /* ca_maxrequests */ + status = nfsd4_encode_count4(xdr, attrs->maxreqs); + if (status != nfs_ok) + return status; + /* ca_rdma_ird<1> */ + if (xdr_stream_encode_u32(xdr, attrs->nr_rdma_attrs) != XDR_UNIT) + return nfserr_resource; + if (attrs->nr_rdma_attrs) + return nfsd4_encode_uint32_t(xdr, attrs->rdma_attrs); + return nfs_ok; +} + +static __be32 +nfsd4_encode_create_session(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_create_session *sess = &u->create_session; + struct xdr_stream *xdr = resp->xdr; + + /* csr_sessionid */ + nfserr = nfsd4_encode_sessionid4(xdr, &sess->sessionid); + if (nfserr != nfs_ok) + return nfserr; + /* csr_sequence */ + nfserr = nfsd4_encode_sequenceid4(xdr, sess->seqid); + if (nfserr != nfs_ok) + return nfserr; + /* csr_flags */ + nfserr = nfsd4_encode_uint32_t(xdr, sess->flags); + if (nfserr != nfs_ok) + return nfserr; + /* csr_fore_chan_attrs */ + nfserr = nfsd4_encode_channel_attrs4(xdr, &sess->fore_channel); + if (nfserr != nfs_ok) + return nfserr; + /* csr_back_chan_attrs */ + return nfsd4_encode_channel_attrs4(xdr, &sess->back_channel); +} + +static __be32 +nfsd4_encode_sequence(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_sequence *seq = &u->sequence; + struct xdr_stream *xdr = resp->xdr; + + /* sr_sessionid */ + nfserr = nfsd4_encode_sessionid4(xdr, &seq->sessionid); + if (nfserr != nfs_ok) + return nfserr; + /* sr_sequenceid */ + nfserr = nfsd4_encode_sequenceid4(xdr, seq->seqid); + if (nfserr != nfs_ok) + return nfserr; + /* sr_slotid */ + nfserr = nfsd4_encode_slotid4(xdr, seq->slotid); + if (nfserr != nfs_ok) + return nfserr; + /* Note slotid's are numbered from zero: */ + /* sr_highest_slotid */ + nfserr = nfsd4_encode_slotid4(xdr, seq->maxslots_response - 1); + if (nfserr != nfs_ok) + return nfserr; + /* sr_target_highest_slotid */ + nfserr = nfsd4_encode_slotid4(xdr, seq->target_maxslots - 1); + if (nfserr != nfs_ok) + return nfserr; + /* sr_status_flags */ + nfserr = nfsd4_encode_uint32_t(xdr, seq->status_flags); + if (nfserr != nfs_ok) + return nfserr; + + resp->cstate.data_offset = xdr->buf->len; /* DRC cache data pointer */ + return nfs_ok; +} + +static __be32 +nfsd4_encode_test_stateid(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_test_stateid *test_stateid = &u->test_stateid; + struct nfsd4_test_stateid_id *stateid, *next; + struct xdr_stream *xdr = resp->xdr; + + /* tsr_status_codes<> */ + if (xdr_stream_encode_u32(xdr, test_stateid->ts_num_ids) != XDR_UNIT) + return nfserr_resource; + list_for_each_entry_safe(stateid, next, + &test_stateid->ts_stateid_list, ts_id_list) { + if (xdr_stream_encode_be32(xdr, stateid->ts_id_status) != XDR_UNIT) + return nfserr_resource; + } + return nfs_ok; +} + +static __be32 +nfsd4_encode_get_dir_delegation(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_get_dir_delegation *gdd = &u->get_dir_delegation; + struct xdr_stream *xdr = resp->xdr; + __be32 status = nfserr_resource; + + switch(gdd->gddrnf_status) { + case GDD4_OK: + if (xdr_stream_encode_u32(xdr, GDD4_OK) != XDR_UNIT) + break; + status = nfsd4_encode_verifier4(xdr, &gdd->gddr_cookieverf); + if (status) + break; + status = nfsd4_encode_stateid4(xdr, &gdd->gddr_stateid); + if (status) + break; + status = nfsd4_encode_bitmap4(xdr, gdd->gddr_notification[0], 0, 0); + if (status) + break; + status = nfsd4_encode_bitmap4(xdr, gdd->gddr_child_attributes[0], + gdd->gddr_child_attributes[1], + gdd->gddr_child_attributes[2]); + if (status) + break; + status = nfsd4_encode_bitmap4(xdr, gdd->gddr_dir_attributes[0], + gdd->gddr_dir_attributes[1], + gdd->gddr_dir_attributes[2]); + break; + default: + pr_warn("nfsd: bad gddrnf_status (%u)\n", gdd->gddrnf_status); + gdd->gddrnf_will_signal_deleg_avail = 0; + fallthrough; + case GDD4_UNAVAIL: + if (xdr_stream_encode_u32(xdr, GDD4_UNAVAIL) != XDR_UNIT) + break; + status = nfsd4_encode_bool(xdr, gdd->gddrnf_will_signal_deleg_avail); + break; + } + return status; +} + +#ifdef CONFIG_NFSD_PNFS +static __be32 +nfsd4_encode_device_addr4(struct xdr_stream *xdr, + const struct nfsd4_getdeviceinfo *gdev) +{ + u32 needed_len, starting_len = xdr->buf->len; + const struct nfsd4_layout_ops *ops; + __be32 status; + + /* da_layout_type */ + if (xdr_stream_encode_u32(xdr, gdev->gd_layout_type) != XDR_UNIT) + return nfserr_resource; + /* da_addr_body */ + ops = nfsd4_layout_ops[gdev->gd_layout_type]; + status = ops->encode_getdeviceinfo(xdr, gdev); + if (status != nfs_ok) { + /* + * Don't burden the layout drivers with enforcing + * gd_maxcount. Just tell the client to come back + * with a bigger buffer if it's not enough. + */ + if (xdr->buf->len + XDR_UNIT > gdev->gd_maxcount) + goto toosmall; + return status; + } + + return nfs_ok; + +toosmall: + needed_len = xdr->buf->len + XDR_UNIT; /* notifications */ + xdr_truncate_encode(xdr, starting_len); + + status = nfsd4_encode_count4(xdr, needed_len); + if (status != nfs_ok) + return status; + return nfserr_toosmall; +} + +static __be32 +nfsd4_encode_getdeviceinfo(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_getdeviceinfo *gdev = &u->getdeviceinfo; + struct xdr_stream *xdr = resp->xdr; + + /* gdir_device_addr */ + nfserr = nfsd4_encode_device_addr4(xdr, gdev); + if (nfserr) + return nfserr; + /* gdir_notification */ + return nfsd4_encode_bitmap4(xdr, gdev->gd_notify_types, 0, 0); +} + +static __be32 +nfsd4_encode_layout4(struct xdr_stream *xdr, const struct nfsd4_layoutget *lgp) +{ + const struct nfsd4_layout_ops *ops = nfsd4_layout_ops[lgp->lg_layout_type]; + __be32 status; + + /* lo_offset */ + status = nfsd4_encode_offset4(xdr, lgp->lg_seg.offset); + if (status != nfs_ok) + return status; + /* lo_length */ + status = nfsd4_encode_length4(xdr, lgp->lg_seg.length); + if (status != nfs_ok) + return status; + /* lo_iomode */ + if (xdr_stream_encode_u32(xdr, lgp->lg_seg.iomode) != XDR_UNIT) + return nfserr_resource; + /* lo_content */ + if (xdr_stream_encode_u32(xdr, lgp->lg_layout_type) != XDR_UNIT) + return nfserr_resource; + return ops->encode_layoutget(xdr, lgp); +} + +static __be32 +nfsd4_encode_layoutget(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_layoutget *lgp = &u->layoutget; + struct xdr_stream *xdr = resp->xdr; + + /* logr_return_on_close */ + nfserr = nfsd4_encode_bool(xdr, true); + if (nfserr != nfs_ok) + return nfserr; + /* logr_stateid */ + nfserr = nfsd4_encode_stateid4(xdr, &lgp->lg_sid); + if (nfserr != nfs_ok) + return nfserr; + /* logr_layout<> */ + if (xdr_stream_encode_u32(xdr, 1) != XDR_UNIT) + return nfserr_resource; + return nfsd4_encode_layout4(xdr, lgp); +} + +static __be32 +nfsd4_encode_layoutcommit(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_layoutcommit *lcp = &u->layoutcommit; + struct xdr_stream *xdr = resp->xdr; + + /* ns_sizechanged */ + nfserr = nfsd4_encode_bool(xdr, lcp->lc_size_chg); + if (nfserr != nfs_ok) + return nfserr; + if (lcp->lc_size_chg) + /* ns_size */ + return nfsd4_encode_length4(xdr, lcp->lc_newsize); + return nfs_ok; +} + +static __be32 +nfsd4_encode_layoutreturn(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_layoutreturn *lrp = &u->layoutreturn; + struct xdr_stream *xdr = resp->xdr; + + /* lrs_present */ + nfserr = nfsd4_encode_bool(xdr, lrp->lrs_present); + if (nfserr != nfs_ok) + return nfserr; + if (lrp->lrs_present) + /* lrs_stateid */ + return nfsd4_encode_stateid4(xdr, &lrp->lr_sid); + return nfs_ok; +} +#endif /* CONFIG_NFSD_PNFS */ + +static __be32 +nfsd4_encode_write_response4(struct xdr_stream *xdr, + const struct nfsd4_copy *copy) +{ + const struct nfsd42_write_res *write = ©->cp_res; + u32 count = nfsd4_copy_is_sync(copy) ? 0 : 1; + __be32 status; + + /* wr_callback_id<1> */ + if (xdr_stream_encode_u32(xdr, count) != XDR_UNIT) + return nfserr_resource; + if (count) { + status = nfsd4_encode_stateid4(xdr, &write->cb_stateid); + if (status != nfs_ok) + return status; + } + + /* wr_count */ + status = nfsd4_encode_length4(xdr, write->wr_bytes_written); + if (status != nfs_ok) + return status; + /* wr_committed */ + if (xdr_stream_encode_u32(xdr, write->wr_stable_how) != XDR_UNIT) + return nfserr_resource; + /* wr_writeverf */ + return nfsd4_encode_verifier4(xdr, &write->wr_verifier); +} + +static __be32 nfsd4_encode_copy_requirements4(struct xdr_stream *xdr, + const struct nfsd4_copy *copy) +{ + __be32 status; + + /* cr_consecutive */ + status = nfsd4_encode_bool(xdr, true); + if (status != nfs_ok) + return status; + /* cr_synchronous */ + return nfsd4_encode_bool(xdr, nfsd4_copy_is_sync(copy)); +} + +static __be32 +nfsd4_encode_copy(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_copy *copy = &u->copy; + + nfserr = nfsd4_encode_write_response4(resp->xdr, copy); + if (nfserr != nfs_ok) + return nfserr; + return nfsd4_encode_copy_requirements4(resp->xdr, copy); +} + +static __be32 +nfsd4_encode_netloc4(struct xdr_stream *xdr, const struct nl4_server *ns) +{ + __be32 status; + + if (xdr_stream_encode_u32(xdr, ns->nl4_type) != XDR_UNIT) + return nfserr_resource; + switch (ns->nl4_type) { + case NL4_NETADDR: + /* nl_addr */ + status = nfsd4_encode_netaddr4(xdr, &ns->u.nl4_addr); + break; + default: + status = nfserr_serverfault; + } + return status; +} + +static __be32 +nfsd4_encode_copy_notify(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_copy_notify *cn = &u->copy_notify; + struct xdr_stream *xdr = resp->xdr; + + /* cnr_lease_time */ + nfserr = nfsd4_encode_nfstime4(xdr, &cn->cpn_lease_time); + if (nfserr) + return nfserr; + /* cnr_stateid */ + nfserr = nfsd4_encode_stateid4(xdr, &cn->cpn_cnr_stateid); + if (nfserr) + return nfserr; + /* cnr_source_server<> */ + if (xdr_stream_encode_u32(xdr, 1) != XDR_UNIT) + return nfserr_resource; + return nfsd4_encode_netloc4(xdr, cn->cpn_src); +} + +static __be32 +nfsd4_encode_offload_status(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_offload_status *os = &u->offload_status; + struct xdr_stream *xdr = resp->xdr; + + /* osr_count */ + nfserr = nfsd4_encode_length4(xdr, os->count); + if (nfserr != nfs_ok) + return nfserr; + /* osr_complete<1> */ + if (os->completed) { + if (xdr_stream_encode_u32(xdr, 1) != XDR_UNIT) + return nfserr_resource; + if (xdr_stream_encode_be32(xdr, os->status) != XDR_UNIT) + return nfserr_resource; + } else if (xdr_stream_encode_u32(xdr, 0) != XDR_UNIT) + return nfserr_resource; + return nfs_ok; +} + +static __be32 +nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp, + struct nfsd4_read *read) +{ + struct nfsd4_compoundargs *argp = resp->rqstp->rq_argp; + struct file *file = read->rd_nf->nf_file; + struct xdr_stream *xdr = resp->xdr; + bool splice_ok = argp->splice_ok; + unsigned int offset_offset; + __be32 nfserr, wire_count; + unsigned long maxcount; + __be64 wire_offset; + + if (xdr_stream_encode_u32(xdr, NFS4_CONTENT_DATA) != XDR_UNIT) + return nfserr_io; + + offset_offset = xdr->buf->len; + + /* Reserve space for the byte offset and count */ + if (unlikely(!xdr_reserve_space(xdr, XDR_UNIT * 3))) + return nfserr_io; + xdr_commit_encode(xdr); + + maxcount = min_t(unsigned long, read->rd_length, + (xdr->buf->buflen - xdr->buf->len)); + + if (file->f_op->splice_read && splice_ok) + nfserr = nfsd4_encode_splice_read(resp, read, file, maxcount); + else + nfserr = nfsd4_encode_readv(resp, read, maxcount); + if (nfserr) + return nfserr; + + wire_offset = cpu_to_be64(read->rd_offset); + write_bytes_to_xdr_buf(xdr->buf, offset_offset, &wire_offset, + XDR_UNIT * 2); + wire_count = cpu_to_be32(read->rd_length); + write_bytes_to_xdr_buf(xdr->buf, offset_offset + XDR_UNIT * 2, + &wire_count, XDR_UNIT); + return nfs_ok; +} + +static __be32 +nfsd4_encode_read_plus(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_read *read = &u->read; + struct file *file = read->rd_nf->nf_file; + struct xdr_stream *xdr = resp->xdr; + unsigned int eof_offset; + __be32 wire_data[2]; + u32 segments = 0; + + if (nfserr) + return nfserr; + + eof_offset = xdr->buf->len; + + /* Reserve space for the eof flag and segment count */ + if (unlikely(!xdr_reserve_space(xdr, XDR_UNIT * 2))) + return nfserr_io; + xdr_commit_encode(xdr); + + read->rd_eof = read->rd_offset >= i_size_read(file_inode(file)); + if (read->rd_eof) + goto out; + + nfserr = nfsd4_encode_read_plus_data(resp, read); + if (nfserr) { + xdr_truncate_encode(xdr, eof_offset); + return nfserr; + } + + segments++; + +out: + wire_data[0] = read->rd_eof ? xdr_one : xdr_zero; + wire_data[1] = cpu_to_be32(segments); + write_bytes_to_xdr_buf(xdr->buf, eof_offset, &wire_data, XDR_UNIT * 2); + return nfserr; +} + +static __be32 +nfsd4_encode_seek(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_seek *seek = &u->seek; + struct xdr_stream *xdr = resp->xdr; + + /* sr_eof */ + nfserr = nfsd4_encode_bool(xdr, seek->seek_eof); + if (nfserr != nfs_ok) + return nfserr; + /* sr_offset */ + return nfsd4_encode_offset4(xdr, seek->seek_pos); +} + +static __be32 +nfsd4_encode_noop(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *p) +{ + return nfserr; +} + +/* + * Encode kmalloc-ed buffer in to XDR stream. + */ +static __be32 +nfsd4_vbuf_to_stream(struct xdr_stream *xdr, char *buf, u32 buflen) +{ + u32 cplen; + __be32 *p; + + cplen = min_t(unsigned long, buflen, + ((void *)xdr->end - (void *)xdr->p)); + p = xdr_reserve_space(xdr, cplen); + if (!p) + return nfserr_resource; + + memcpy(p, buf, cplen); + buf += cplen; + buflen -= cplen; + + while (buflen) { + cplen = min_t(u32, buflen, PAGE_SIZE); + p = xdr_reserve_space(xdr, cplen); + if (!p) + return nfserr_resource; + + memcpy(p, buf, cplen); + + if (cplen < PAGE_SIZE) { + /* + * We're done, with a length that wasn't page + * aligned, so possibly not word aligned. Pad + * any trailing bytes with 0. + */ + xdr_encode_opaque_fixed(p, NULL, cplen); + break; + } + + buflen -= PAGE_SIZE; + buf += PAGE_SIZE; + } + + return 0; +} + +static __be32 +nfsd4_encode_getxattr(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_getxattr *getxattr = &u->getxattr; + struct xdr_stream *xdr = resp->xdr; + __be32 *p, err; + + p = xdr_reserve_space(xdr, 4); + if (!p) + return nfserr_resource; + + *p = cpu_to_be32(getxattr->getxa_len); + + if (getxattr->getxa_len == 0) + return 0; + + err = nfsd4_vbuf_to_stream(xdr, getxattr->getxa_buf, + getxattr->getxa_len); + + kvfree(getxattr->getxa_buf); + + return err; +} + +static __be32 +nfsd4_encode_setxattr(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_setxattr *setxattr = &u->setxattr; + struct xdr_stream *xdr = resp->xdr; + + return nfsd4_encode_change_info4(xdr, &setxattr->setxa_cinfo); +} + +/* + * See if there are cookie values that can be rejected outright. + */ +static __be32 +nfsd4_listxattr_validate_cookie(struct nfsd4_listxattrs *listxattrs, + u32 *offsetp) +{ + u64 cookie = listxattrs->lsxa_cookie; + + /* + * If the cookie is larger than the maximum number we can fit + * in the buffer we just got back from vfs_listxattr, it's invalid. + */ + if (cookie > (listxattrs->lsxa_len) / (XATTR_USER_PREFIX_LEN + 2)) + return nfserr_badcookie; + + *offsetp = (u32)cookie; + return 0; +} + +static __be32 +nfsd4_encode_listxattrs(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_listxattrs *listxattrs = &u->listxattrs; + struct xdr_stream *xdr = resp->xdr; + u32 cookie_offset, count_offset, eof; + u32 left, xdrleft, slen, count; + u32 xdrlen, offset; + u64 cookie; + char *sp; + __be32 status, tmp; + __be64 wire_cookie; + __be32 *p; + u32 nuser; + + eof = 1; + + status = nfsd4_listxattr_validate_cookie(listxattrs, &offset); + if (status) + goto out; + + /* + * Reserve space for the cookie and the name array count. Record + * the offsets to save them later. + */ + cookie_offset = xdr->buf->len; + count_offset = cookie_offset + 8; + p = xdr_reserve_space(xdr, XDR_UNIT * 3); + if (!p) { + status = nfserr_resource; + goto out; + } + + count = 0; + left = listxattrs->lsxa_len; + sp = listxattrs->lsxa_buf; + nuser = 0; + + /* Bytes left is maxcount - 8 (cookie) - 4 (array count) */ + xdrleft = listxattrs->lsxa_maxcount - XDR_UNIT * 3; + + while (left > 0 && xdrleft > 0) { + slen = strlen(sp); + + /* + * Check if this is a "user." attribute, skip it if not. + */ + if (strncmp(sp, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN)) + goto contloop; + + slen -= XATTR_USER_PREFIX_LEN; + xdrlen = 4 + ((slen + 3) & ~3); + /* Check if both entry and eof can fit in the XDR buffer */ + if (xdrlen + XDR_UNIT > xdrleft) { + if (count == 0) { + /* + * Can't even fit the first attribute name. + */ + status = nfserr_toosmall; + goto out; + } + eof = 0; + goto wreof; + } + + left -= XATTR_USER_PREFIX_LEN; + sp += XATTR_USER_PREFIX_LEN; + if (nuser++ < offset) + goto contloop; + + + p = xdr_reserve_space(xdr, xdrlen); + if (!p) { + status = nfserr_resource; + goto out; + } + + xdr_encode_opaque(p, sp, slen); + + xdrleft -= xdrlen; + count++; +contloop: + sp += slen + 1; + left -= slen + 1; + } + + /* + * If there were user attributes to copy, but we didn't copy + * any, the offset was too large (e.g. the cookie was invalid). + */ + if (nuser > 0 && count == 0) { + status = nfserr_badcookie; + goto out; + } + +wreof: + p = xdr_reserve_space(xdr, 4); + if (!p) { + status = nfserr_resource; + goto out; + } + *p = cpu_to_be32(eof); + + cookie = offset + count; + + wire_cookie = cpu_to_be64(cookie); + write_bytes_to_xdr_buf(xdr->buf, cookie_offset, &wire_cookie, 8); + tmp = cpu_to_be32(count); + write_bytes_to_xdr_buf(xdr->buf, count_offset, &tmp, 4); +out: + if (listxattrs->lsxa_len) + kvfree(listxattrs->lsxa_buf); + return status; +} + +static __be32 +nfsd4_encode_removexattr(struct nfsd4_compoundres *resp, __be32 nfserr, + union nfsd4_op_u *u) +{ + struct nfsd4_removexattr *removexattr = &u->removexattr; + struct xdr_stream *xdr = resp->xdr; + + return nfsd4_encode_change_info4(xdr, &removexattr->rmxa_cinfo); +} + +typedef __be32(*nfsd4_enc)(struct nfsd4_compoundres *, __be32, union nfsd4_op_u *u); + +/* + * Note: nfsd4_enc_ops vector is shared for v4.0 and v4.1 + * since we don't need to filter out obsolete ops as this is + * done in the decoding phase. + */ +static const nfsd4_enc nfsd4_enc_ops[] = { + [OP_ACCESS] = nfsd4_encode_access, + [OP_CLOSE] = nfsd4_encode_close, + [OP_COMMIT] = nfsd4_encode_commit, + [OP_CREATE] = nfsd4_encode_create, + [OP_DELEGPURGE] = nfsd4_encode_noop, + [OP_DELEGRETURN] = nfsd4_encode_noop, + [OP_GETATTR] = nfsd4_encode_getattr, + [OP_GETFH] = nfsd4_encode_getfh, + [OP_LINK] = nfsd4_encode_link, + [OP_LOCK] = nfsd4_encode_lock, + [OP_LOCKT] = nfsd4_encode_lockt, + [OP_LOCKU] = nfsd4_encode_locku, + [OP_LOOKUP] = nfsd4_encode_noop, + [OP_LOOKUPP] = nfsd4_encode_noop, + [OP_NVERIFY] = nfsd4_encode_noop, + [OP_OPEN] = nfsd4_encode_open, + [OP_OPENATTR] = nfsd4_encode_noop, + [OP_OPEN_CONFIRM] = nfsd4_encode_open_confirm, + [OP_OPEN_DOWNGRADE] = nfsd4_encode_open_downgrade, + [OP_PUTFH] = nfsd4_encode_noop, + [OP_PUTPUBFH] = nfsd4_encode_noop, + [OP_PUTROOTFH] = nfsd4_encode_noop, + [OP_READ] = nfsd4_encode_read, + [OP_READDIR] = nfsd4_encode_readdir, + [OP_READLINK] = nfsd4_encode_readlink, + [OP_REMOVE] = nfsd4_encode_remove, + [OP_RENAME] = nfsd4_encode_rename, + [OP_RENEW] = nfsd4_encode_noop, + [OP_RESTOREFH] = nfsd4_encode_noop, + [OP_SAVEFH] = nfsd4_encode_noop, + [OP_SECINFO] = nfsd4_encode_secinfo, + [OP_SETATTR] = nfsd4_encode_setattr, + [OP_SETCLIENTID] = nfsd4_encode_setclientid, + [OP_SETCLIENTID_CONFIRM] = nfsd4_encode_noop, + [OP_VERIFY] = nfsd4_encode_noop, + [OP_WRITE] = nfsd4_encode_write, + [OP_RELEASE_LOCKOWNER] = nfsd4_encode_noop, + + /* NFSv4.1 operations */ + [OP_BACKCHANNEL_CTL] = nfsd4_encode_noop, + [OP_BIND_CONN_TO_SESSION] = nfsd4_encode_bind_conn_to_session, + [OP_EXCHANGE_ID] = nfsd4_encode_exchange_id, + [OP_CREATE_SESSION] = nfsd4_encode_create_session, + [OP_DESTROY_SESSION] = nfsd4_encode_noop, + [OP_FREE_STATEID] = nfsd4_encode_noop, + [OP_GET_DIR_DELEGATION] = nfsd4_encode_get_dir_delegation, +#ifdef CONFIG_NFSD_PNFS + [OP_GETDEVICEINFO] = nfsd4_encode_getdeviceinfo, + [OP_GETDEVICELIST] = nfsd4_encode_noop, + [OP_LAYOUTCOMMIT] = nfsd4_encode_layoutcommit, + [OP_LAYOUTGET] = nfsd4_encode_layoutget, + [OP_LAYOUTRETURN] = nfsd4_encode_layoutreturn, +#else + [OP_GETDEVICEINFO] = nfsd4_encode_noop, + [OP_GETDEVICELIST] = nfsd4_encode_noop, + [OP_LAYOUTCOMMIT] = nfsd4_encode_noop, + [OP_LAYOUTGET] = nfsd4_encode_noop, + [OP_LAYOUTRETURN] = nfsd4_encode_noop, +#endif + [OP_SECINFO_NO_NAME] = nfsd4_encode_secinfo_no_name, + [OP_SEQUENCE] = nfsd4_encode_sequence, + [OP_SET_SSV] = nfsd4_encode_noop, + [OP_TEST_STATEID] = nfsd4_encode_test_stateid, + [OP_WANT_DELEGATION] = nfsd4_encode_noop, + [OP_DESTROY_CLIENTID] = nfsd4_encode_noop, + [OP_RECLAIM_COMPLETE] = nfsd4_encode_noop, + + /* NFSv4.2 operations */ + [OP_ALLOCATE] = nfsd4_encode_noop, + [OP_COPY] = nfsd4_encode_copy, + [OP_COPY_NOTIFY] = nfsd4_encode_copy_notify, + [OP_DEALLOCATE] = nfsd4_encode_noop, + [OP_IO_ADVISE] = nfsd4_encode_noop, + [OP_LAYOUTERROR] = nfsd4_encode_noop, + [OP_LAYOUTSTATS] = nfsd4_encode_noop, + [OP_OFFLOAD_CANCEL] = nfsd4_encode_noop, + [OP_OFFLOAD_STATUS] = nfsd4_encode_offload_status, + [OP_READ_PLUS] = nfsd4_encode_read_plus, + [OP_SEEK] = nfsd4_encode_seek, + [OP_WRITE_SAME] = nfsd4_encode_noop, + [OP_CLONE] = nfsd4_encode_noop, + + /* RFC 8276 extended atributes operations */ + [OP_GETXATTR] = nfsd4_encode_getxattr, + [OP_SETXATTR] = nfsd4_encode_setxattr, + [OP_LISTXATTRS] = nfsd4_encode_listxattrs, + [OP_REMOVEXATTR] = nfsd4_encode_removexattr, +}; + +/* + * Calculate whether we still have space to encode repsize bytes. + * There are two considerations: + * - For NFS versions >=4.1, the size of the reply must stay within + * session limits + * - For all NFS versions, we must stay within limited preallocated + * buffer space. + * + * This is called before the operation is processed, so can only provide + * an upper estimate. For some nonidempotent operations (such as + * getattr), it's not necessarily a problem if that estimate is wrong, + * as we can fail it after processing without significant side effects. + */ +__be32 nfsd4_check_resp_size(struct nfsd4_compoundres *resp, u32 respsize) +{ + struct xdr_buf *buf = &resp->rqstp->rq_res; + struct nfsd4_slot *slot = resp->cstate.slot; + + if (buf->len + respsize <= buf->buflen) + return nfs_ok; + if (!nfsd4_has_session(&resp->cstate)) + return nfserr_resource; + if (slot->sl_flags & NFSD4_SLOT_CACHETHIS) { + WARN_ON_ONCE(1); + return nfserr_rep_too_big_to_cache; + } + return nfserr_rep_too_big; +} + +static __be32 nfsd4_map_status(__be32 status, u32 minor) +{ + switch (status) { + case nfs_ok: + break; + case nfserr_wrong_type: + /* RFC 8881 - 15.1.2.9 */ + if (minor == 0) + status = nfserr_inval; + break; + case nfserr_symlink_not_dir: + status = nfserr_symlink; + break; + } + return status; +} + +void +nfsd4_encode_operation(struct nfsd4_compoundres *resp, struct nfsd4_op *op) +{ + struct xdr_stream *xdr = resp->xdr; + struct nfs4_stateowner *so = resp->cstate.replay_owner; + struct svc_rqst *rqstp = resp->rqstp; + const struct nfsd4_operation *opdesc = op->opdesc; + unsigned int op_status_offset; + nfsd4_enc encoder; + + if (xdr_stream_encode_u32(xdr, op->opnum) != XDR_UNIT) + goto release; + op_status_offset = xdr->buf->len; + if (!xdr_reserve_space(xdr, XDR_UNIT)) + goto release; + + if (op->opnum == OP_ILLEGAL) + goto status; + if (op->status && opdesc && + !(opdesc->op_flags & OP_NONTRIVIAL_ERROR_ENCODE)) + goto status; + BUG_ON(op->opnum >= ARRAY_SIZE(nfsd4_enc_ops) || + !nfsd4_enc_ops[op->opnum]); + encoder = nfsd4_enc_ops[op->opnum]; + op->status = encoder(resp, op->status, &op->u); + if (op->status) + trace_nfsd_compound_encode_err(rqstp, op->opnum, op->status); + xdr_commit_encode(xdr); + + /* nfsd4_check_resp_size guarantees enough room for error status */ + if (!op->status) { + int space_needed = 0; + if (!nfsd4_last_compound_op(rqstp)) + space_needed = COMPOUND_ERR_SLACK_SPACE; + op->status = nfsd4_check_resp_size(resp, space_needed); + } + if (op->status == nfserr_resource && nfsd4_has_session(&resp->cstate)) { + struct nfsd4_slot *slot = resp->cstate.slot; + + if (slot->sl_flags & NFSD4_SLOT_CACHETHIS) + op->status = nfserr_rep_too_big_to_cache; + else + op->status = nfserr_rep_too_big; + } + if (op->status == nfserr_resource || + op->status == nfserr_rep_too_big || + op->status == nfserr_rep_too_big_to_cache) { + /* + * The operation may have already been encoded or + * partially encoded. No op returns anything additional + * in the case of one of these three errors, so we can + * just truncate back to after the status. But it's a + * bug if we had to do this on a non-idempotent op: + */ + warn_on_nonidempotent_op(op); + xdr_truncate_encode(xdr, op_status_offset + XDR_UNIT); + } else if (so) { + int len = xdr->buf->len - (op_status_offset + XDR_UNIT); + + so->so_replay.rp_status = op->status; + if (len > NFSD4_REPLAY_ISIZE) { + char *buf = kmalloc(len, GFP_KERNEL); + + nfs4_replay_free_cache(&so->so_replay); + if (buf) { + so->so_replay.rp_buf = buf; + } else { + /* rp_buflen already zeroed; skip caching */ + goto status; + } + } else if (so->so_replay.rp_buf != so->so_replay.rp_ibuf) { + nfs4_replay_free_cache(&so->so_replay); + } + so->so_replay.rp_buflen = len; + read_bytes_from_xdr_buf(xdr->buf, + op_status_offset + XDR_UNIT, + so->so_replay.rp_buf, len); + } +status: + op->status = nfsd4_map_status(op->status, + resp->cstate.minorversion); + write_bytes_to_xdr_buf(xdr->buf, op_status_offset, + &op->status, XDR_UNIT); +release: + /* + * Account for pages consumed while encoding this operation. + * The xdr_stream primitives don't manage rq_next_page. + */ + rqstp->rq_next_page = xdr->page_ptr + 1; +} + +/** + * nfsd4_encode_replay - encode a result stored in the stateowner reply cache + * @xdr: send buffer's XDR stream + * @op: operation being replayed + * + * @op->replay->rp_buf contains the previously-sent already-encoded result. + */ +void nfsd4_encode_replay(struct xdr_stream *xdr, struct nfsd4_op *op) +{ + struct nfs4_replay *rp = op->replay; + + trace_nfsd_stateowner_replay(op->opnum, rp); + + if (xdr_stream_encode_u32(xdr, op->opnum) != XDR_UNIT) + return; + if (xdr_stream_encode_be32(xdr, rp->rp_status) != XDR_UNIT) + return; + xdr_stream_encode_opaque_fixed(xdr, rp->rp_buf, rp->rp_buflen); +} + +void nfsd4_release_compoundargs(struct svc_rqst *rqstp) +{ + struct nfsd4_compoundargs *args = rqstp->rq_argp; + + args->opcnt = 0; + if (args->ops != args->iops) { + void *old_ops = args->ops; + + args->ops = args->iops; + kvfree_rcu_mightsleep(old_ops); + } + while (args->to_free) { + struct svcxdr_tmpbuf *tb = args->to_free; + args->to_free = tb->next; + kfree(tb); + } +} + +bool +nfs4svc_decode_compoundargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd4_compoundargs *args = rqstp->rq_argp; + + /* svcxdr_tmp_alloc */ + args->to_free = NULL; + + args->xdr = xdr; + args->ops = args->iops; + args->rqstp = rqstp; + + /* + * NFSv4 operation decoders can invoke svc cache lookups + * that trigger svc_defer() when RQ_USEDEFERRAL is set, + * setting RQ_DROPME. This creates two problems: + * + * 1. Non-idempotency: Compounds make it too hard to avoid + * problems if a request is deferred and replayed. + * + * 2. Session slot leakage (NFSv4.1+): If RQ_DROPME is set + * during decode but SEQUENCE executes successfully, the + * session slot will be marked INUSE. The request is then + * dropped before encoding, so the slot is never released, + * rendering it permanently unusable by the client. + */ + clear_bit(RQ_USEDEFERRAL, &rqstp->rq_flags); + + return nfsd4_decode_compound(args); +} + +bool +nfs4svc_encode_compoundres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd4_compoundres *resp = rqstp->rq_resp; + __be32 *p; + + /* + * Send buffer space for the following items is reserved + * at the top of nfsd4_proc_compound(). + */ + p = resp->statusp; + + *p++ = resp->cstate.status; + *p++ = htonl(resp->taglen); + memcpy(p, resp->tag, resp->taglen); + p += XDR_QUADLEN(resp->taglen); + *p++ = htonl(resp->opcnt); + + nfsd4_sequence_done(resp); + return true; +} diff --git a/fs/nfsd/nfs4xdr_gen.c b/fs/nfsd/nfs4xdr_gen.c new file mode 100644 index 000000000..d77835033 --- /dev/null +++ b/fs/nfsd/nfs4xdr_gen.c @@ -0,0 +1,1159 @@ +// SPDX-License-Identifier: GPL-2.0 +// Generated by xdrgen. Manual edits will be lost. +// XDR specification file: ../../Documentation/sunrpc/xdr/nfs4_1.x +// XDR specification modification time: Tue Jun 30 11:57:21 2026 + +#include + +#include "nfs4xdr_gen.h" + +static bool __maybe_unused +xdrgen_decode_int32_t(struct xdr_stream *xdr, int32_t *ptr) +{ + return xdrgen_decode_int(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_uint32_t(struct xdr_stream *xdr, uint32_t *ptr) +{ + return xdrgen_decode_unsigned_int(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_int64_t(struct xdr_stream *xdr, int64_t *ptr) +{ + return xdrgen_decode_hyper(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_uint64_t(struct xdr_stream *xdr, uint64_t *ptr) +{ + return xdrgen_decode_unsigned_hyper(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_nfsstat4(struct xdr_stream *xdr, nfsstat4 *ptr) +{ + u32 val; + + if (xdr_stream_decode_u32(xdr, &val) < 0) + return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case NFS4_OK: + case NFS4ERR_PERM: + case NFS4ERR_NOENT: + case NFS4ERR_IO: + case NFS4ERR_NXIO: + case NFS4ERR_ACCESS: + case NFS4ERR_EXIST: + case NFS4ERR_XDEV: + case NFS4ERR_NOTDIR: + case NFS4ERR_ISDIR: + case NFS4ERR_INVAL: + case NFS4ERR_FBIG: + case NFS4ERR_NOSPC: + case NFS4ERR_ROFS: + case NFS4ERR_MLINK: + case NFS4ERR_NAMETOOLONG: + case NFS4ERR_NOTEMPTY: + case NFS4ERR_DQUOT: + case NFS4ERR_STALE: + case NFS4ERR_BADHANDLE: + case NFS4ERR_BAD_COOKIE: + case NFS4ERR_NOTSUPP: + case NFS4ERR_TOOSMALL: + case NFS4ERR_SERVERFAULT: + case NFS4ERR_BADTYPE: + case NFS4ERR_DELAY: + case NFS4ERR_SAME: + case NFS4ERR_DENIED: + case NFS4ERR_EXPIRED: + case NFS4ERR_LOCKED: + case NFS4ERR_GRACE: + case NFS4ERR_FHEXPIRED: + case NFS4ERR_SHARE_DENIED: + case NFS4ERR_WRONGSEC: + case NFS4ERR_CLID_INUSE: + case NFS4ERR_RESOURCE: + case NFS4ERR_MOVED: + case NFS4ERR_NOFILEHANDLE: + case NFS4ERR_MINOR_VERS_MISMATCH: + case NFS4ERR_STALE_CLIENTID: + case NFS4ERR_STALE_STATEID: + case NFS4ERR_OLD_STATEID: + case NFS4ERR_BAD_STATEID: + case NFS4ERR_BAD_SEQID: + case NFS4ERR_NOT_SAME: + case NFS4ERR_LOCK_RANGE: + case NFS4ERR_SYMLINK: + case NFS4ERR_RESTOREFH: + case NFS4ERR_LEASE_MOVED: + case NFS4ERR_ATTRNOTSUPP: + case NFS4ERR_NO_GRACE: + case NFS4ERR_RECLAIM_BAD: + case NFS4ERR_RECLAIM_CONFLICT: + case NFS4ERR_BADXDR: + case NFS4ERR_LOCKS_HELD: + case NFS4ERR_OPENMODE: + case NFS4ERR_BADOWNER: + case NFS4ERR_BADCHAR: + case NFS4ERR_BADNAME: + case NFS4ERR_BAD_RANGE: + case NFS4ERR_LOCK_NOTSUPP: + case NFS4ERR_OP_ILLEGAL: + case NFS4ERR_DEADLOCK: + case NFS4ERR_FILE_OPEN: + case NFS4ERR_ADMIN_REVOKED: + case NFS4ERR_CB_PATH_DOWN: + case NFS4ERR_BADIOMODE: + case NFS4ERR_BADLAYOUT: + case NFS4ERR_BAD_SESSION_DIGEST: + case NFS4ERR_BADSESSION: + case NFS4ERR_BADSLOT: + case NFS4ERR_COMPLETE_ALREADY: + case NFS4ERR_CONN_NOT_BOUND_TO_SESSION: + case NFS4ERR_DELEG_ALREADY_WANTED: + case NFS4ERR_BACK_CHAN_BUSY: + case NFS4ERR_LAYOUTTRYLATER: + case NFS4ERR_LAYOUTUNAVAILABLE: + case NFS4ERR_NOMATCHING_LAYOUT: + case NFS4ERR_RECALLCONFLICT: + case NFS4ERR_UNKNOWN_LAYOUTTYPE: + case NFS4ERR_SEQ_MISORDERED: + case NFS4ERR_SEQUENCE_POS: + case NFS4ERR_REQ_TOO_BIG: + case NFS4ERR_REP_TOO_BIG: + case NFS4ERR_REP_TOO_BIG_TO_CACHE: + case NFS4ERR_RETRY_UNCACHED_REP: + case NFS4ERR_UNSAFE_COMPOUND: + case NFS4ERR_TOO_MANY_OPS: + case NFS4ERR_OP_NOT_IN_SESSION: + case NFS4ERR_HASH_ALG_UNSUPP: + case NFS4ERR_CLIENTID_BUSY: + case NFS4ERR_PNFS_IO_HOLE: + case NFS4ERR_SEQ_FALSE_RETRY: + case NFS4ERR_BAD_HIGH_SLOT: + case NFS4ERR_DEADSESSION: + case NFS4ERR_ENCR_ALG_UNSUPP: + case NFS4ERR_PNFS_NO_LAYOUT: + case NFS4ERR_NOT_ONLY_OP: + case NFS4ERR_WRONG_CRED: + case NFS4ERR_WRONG_TYPE: + case NFS4ERR_DIRDELEG_UNAVAIL: + case NFS4ERR_REJECT_DELEG: + case NFS4ERR_RETURNCONFLICT: + case NFS4ERR_DELEG_REVOKED: + case NFS4ERR_PARTNER_NOTSUPP: + case NFS4ERR_PARTNER_NO_AUTH: + case NFS4ERR_UNION_NOTSUPP: + case NFS4ERR_OFFLOAD_DENIED: + case NFS4ERR_WRONG_LFS: + case NFS4ERR_BADLABEL: + case NFS4ERR_OFFLOAD_NO_REQS: + case NFS4ERR_NOXATTR: + case NFS4ERR_XATTR2BIG: + break; + default: + return false; + } + *ptr = val; + return true; +} + +static bool __maybe_unused +xdrgen_decode_attrlist4(struct xdr_stream *xdr, attrlist4 *ptr) +{ + return xdrgen_decode_opaque(xdr, ptr, 0); +} + +static bool __maybe_unused +xdrgen_decode_bitmap4(struct xdr_stream *xdr, bitmap4 *ptr) +{ + if (xdr_stream_decode_u32(xdr, &ptr->count) < 0) + return false; + for (u32 i = 0; i < ptr->count; i++) + if (!xdrgen_decode_uint32_t(xdr, &ptr->element[i])) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_verifier4(struct xdr_stream *xdr, verifier4 *ptr) +{ + return xdr_stream_decode_opaque_fixed(xdr, ptr, NFS4_VERIFIER_SIZE) == 0; +} + +static bool __maybe_unused +xdrgen_decode_nfs_cookie4(struct xdr_stream *xdr, nfs_cookie4 *ptr) +{ + return xdrgen_decode_uint64_t(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_nfs_fh4(struct xdr_stream *xdr, nfs_fh4 *ptr) +{ + return xdrgen_decode_opaque(xdr, ptr, NFS4_FHSIZE); +} + +static bool __maybe_unused +xdrgen_decode_utf8string(struct xdr_stream *xdr, utf8string *ptr) +{ + return xdrgen_decode_opaque(xdr, ptr, 0); +} + +static bool __maybe_unused +xdrgen_decode_utf8str_cis(struct xdr_stream *xdr, utf8str_cis *ptr) +{ + return xdrgen_decode_utf8string(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_utf8str_cs(struct xdr_stream *xdr, utf8str_cs *ptr) +{ + return xdrgen_decode_utf8string(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_utf8str_mixed(struct xdr_stream *xdr, utf8str_mixed *ptr) +{ + return xdrgen_decode_utf8string(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_component4(struct xdr_stream *xdr, component4 *ptr) +{ + return xdrgen_decode_utf8str_cs(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_linktext4(struct xdr_stream *xdr, linktext4 *ptr) +{ + return xdrgen_decode_utf8str_cs(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_pathname4(struct xdr_stream *xdr, pathname4 *ptr) +{ + if (xdr_stream_decode_u32(xdr, &ptr->count) < 0) + return false; + for (u32 i = 0; i < ptr->count; i++) + if (!xdrgen_decode_component4(xdr, &ptr->element[i])) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_nfstime4(struct xdr_stream *xdr, struct nfstime4 *ptr) +{ + if (!xdrgen_decode_int64_t(xdr, &ptr->seconds)) + return false; + if (!xdrgen_decode_uint32_t(xdr, &ptr->nseconds)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_fattr4(struct xdr_stream *xdr, struct fattr4 *ptr) +{ + if (!xdrgen_decode_bitmap4(xdr, &ptr->attrmask)) + return false; + if (!xdrgen_decode_attrlist4(xdr, &ptr->attr_vals)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_stateid4(struct xdr_stream *xdr, struct stateid4 *ptr) +{ + if (!xdrgen_decode_uint32_t(xdr, &ptr->seqid)) + return false; + if (xdr_stream_decode_opaque_fixed(xdr, ptr->other, 12) < 0) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_fattr4_offline(struct xdr_stream *xdr, fattr4_offline *ptr) +{ + return xdrgen_decode_bool(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_open_arguments4(struct xdr_stream *xdr, struct open_arguments4 *ptr) +{ + if (!xdrgen_decode_bitmap4(xdr, &ptr->oa_share_access)) + return false; + if (!xdrgen_decode_bitmap4(xdr, &ptr->oa_share_deny)) + return false; + if (!xdrgen_decode_bitmap4(xdr, &ptr->oa_share_access_want)) + return false; + if (!xdrgen_decode_bitmap4(xdr, &ptr->oa_open_claim)) + return false; + if (!xdrgen_decode_bitmap4(xdr, &ptr->oa_create_mode)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_open_args_share_access4(struct xdr_stream *xdr, open_args_share_access4 *ptr) +{ + u32 val; + + if (xdr_stream_decode_u32(xdr, &val) < 0) + return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case OPEN_ARGS_SHARE_ACCESS_READ: + case OPEN_ARGS_SHARE_ACCESS_WRITE: + case OPEN_ARGS_SHARE_ACCESS_BOTH: + break; + default: + return false; + } + *ptr = val; + return true; +} + +static bool __maybe_unused +xdrgen_decode_open_args_share_deny4(struct xdr_stream *xdr, open_args_share_deny4 *ptr) +{ + u32 val; + + if (xdr_stream_decode_u32(xdr, &val) < 0) + return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case OPEN_ARGS_SHARE_DENY_NONE: + case OPEN_ARGS_SHARE_DENY_READ: + case OPEN_ARGS_SHARE_DENY_WRITE: + case OPEN_ARGS_SHARE_DENY_BOTH: + break; + default: + return false; + } + *ptr = val; + return true; +} + +static bool __maybe_unused +xdrgen_decode_open_args_share_access_want4(struct xdr_stream *xdr, open_args_share_access_want4 *ptr) +{ + u32 val; + + if (xdr_stream_decode_u32(xdr, &val) < 0) + return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case OPEN_ARGS_SHARE_ACCESS_WANT_ANY_DELEG: + case OPEN_ARGS_SHARE_ACCESS_WANT_NO_DELEG: + case OPEN_ARGS_SHARE_ACCESS_WANT_CANCEL: + case OPEN_ARGS_SHARE_ACCESS_WANT_SIGNAL_DELEG_WHEN_RESRC_AVAIL: + case OPEN_ARGS_SHARE_ACCESS_WANT_PUSH_DELEG_WHEN_UNCONTENDED: + case OPEN_ARGS_SHARE_ACCESS_WANT_DELEG_TIMESTAMPS: + case OPEN_ARGS_SHARE_ACCESS_WANT_OPEN_XOR_DELEGATION: + break; + default: + return false; + } + *ptr = val; + return true; +} + +static bool __maybe_unused +xdrgen_decode_open_args_open_claim4(struct xdr_stream *xdr, open_args_open_claim4 *ptr) +{ + u32 val; + + if (xdr_stream_decode_u32(xdr, &val) < 0) + return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case OPEN_ARGS_OPEN_CLAIM_NULL: + case OPEN_ARGS_OPEN_CLAIM_PREVIOUS: + case OPEN_ARGS_OPEN_CLAIM_DELEGATE_CUR: + case OPEN_ARGS_OPEN_CLAIM_DELEGATE_PREV: + case OPEN_ARGS_OPEN_CLAIM_FH: + case OPEN_ARGS_OPEN_CLAIM_DELEG_CUR_FH: + case OPEN_ARGS_OPEN_CLAIM_DELEG_PREV_FH: + break; + default: + return false; + } + *ptr = val; + return true; +} + +static bool __maybe_unused +xdrgen_decode_open_args_createmode4(struct xdr_stream *xdr, open_args_createmode4 *ptr) +{ + u32 val; + + if (xdr_stream_decode_u32(xdr, &val) < 0) + return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case OPEN_ARGS_CREATEMODE_UNCHECKED4: + case OPEN_ARGS_CREATE_MODE_GUARDED: + case OPEN_ARGS_CREATEMODE_EXCLUSIVE4: + case OPEN_ARGS_CREATE_MODE_EXCLUSIVE4_1: + break; + default: + return false; + } + *ptr = val; + return true; +} + +bool +xdrgen_decode_fattr4_open_arguments(struct xdr_stream *xdr, fattr4_open_arguments *ptr) +{ + return xdrgen_decode_open_arguments4(xdr, ptr); +} + +/* + * Determine what OPEN supports. + */ + +bool +xdrgen_decode_fattr4_time_deleg_access(struct xdr_stream *xdr, fattr4_time_deleg_access *ptr) +{ + return xdrgen_decode_nfstime4(xdr, ptr); +} + +bool +xdrgen_decode_fattr4_time_deleg_modify(struct xdr_stream *xdr, fattr4_time_deleg_modify *ptr) +{ + return xdrgen_decode_nfstime4(xdr, ptr); +} + +/* + * New RECOMMENDED Attribute for + * delegation caching of times + */ + +static bool __maybe_unused +xdrgen_decode_open_delegation_type4(struct xdr_stream *xdr, open_delegation_type4 *ptr) +{ + u32 val; + + if (xdr_stream_decode_u32(xdr, &val) < 0) + return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case OPEN_DELEGATE_NONE: + case OPEN_DELEGATE_READ: + case OPEN_DELEGATE_WRITE: + case OPEN_DELEGATE_NONE_EXT: + case OPEN_DELEGATE_READ_ATTRS_DELEG: + case OPEN_DELEGATE_WRITE_ATTRS_DELEG: + break; + default: + return false; + } + *ptr = val; + return true; +} + +bool +xdrgen_decode_aclmodel4(struct xdr_stream *xdr, aclmodel4 *ptr) +{ + u32 val; + + if (xdr_stream_decode_u32(xdr, &val) < 0) + return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case ACL_MODEL_NFS4: + case ACL_MODEL_POSIX_DRAFT: + case ACL_MODEL_NONE: + break; + default: + return false; + } + *ptr = val; + return true; +} + +bool +xdrgen_decode_aclscope4(struct xdr_stream *xdr, aclscope4 *ptr) +{ + u32 val; + + if (xdr_stream_decode_u32(xdr, &val) < 0) + return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case ACL_SCOPE_FILE_OBJECT: + case ACL_SCOPE_FILE_SYSTEM: + case ACL_SCOPE_SERVER: + break; + default: + return false; + } + *ptr = val; + return true; +} + +bool +xdrgen_decode_posixacetag4(struct xdr_stream *xdr, posixacetag4 *ptr) +{ + u32 val; + + if (xdr_stream_decode_u32(xdr, &val) < 0) + return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case POSIXACE4_TAG_USER_OBJ: + case POSIXACE4_TAG_USER: + case POSIXACE4_TAG_GROUP_OBJ: + case POSIXACE4_TAG_GROUP: + case POSIXACE4_TAG_MASK: + case POSIXACE4_TAG_OTHER: + break; + default: + return false; + } + *ptr = val; + return true; +} + +bool +xdrgen_decode_posixaceperm4(struct xdr_stream *xdr, posixaceperm4 *ptr) +{ + return xdrgen_decode_uint32_t(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_posixace4(struct xdr_stream *xdr, struct posixace4 *ptr) +{ + if (!xdrgen_decode_posixacetag4(xdr, &ptr->tag)) + return false; + if (!xdrgen_decode_posixaceperm4(xdr, &ptr->perm)) + return false; + if (!xdrgen_decode_utf8str_mixed(xdr, &ptr->who)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_fattr4_acl_trueform(struct xdr_stream *xdr, fattr4_acl_trueform *ptr) +{ + return xdrgen_decode_aclmodel4(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_fattr4_acl_trueform_scope(struct xdr_stream *xdr, fattr4_acl_trueform_scope *ptr) +{ + return xdrgen_decode_aclscope4(xdr, ptr); +} + +static bool __maybe_unused +xdrgen_decode_fattr4_posix_default_acl(struct xdr_stream *xdr, fattr4_posix_default_acl *ptr) +{ + if (xdr_stream_decode_u32(xdr, &ptr->count) < 0) + return false; + for (u32 i = 0; i < ptr->count; i++) + if (!xdrgen_decode_posixace4(xdr, &ptr->element[i])) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_fattr4_posix_access_acl(struct xdr_stream *xdr, fattr4_posix_access_acl *ptr) +{ + if (xdr_stream_decode_u32(xdr, &ptr->count) < 0) + return false; + for (u32 i = 0; i < ptr->count; i++) + if (!xdrgen_decode_posixace4(xdr, &ptr->element[i])) + return false; + return true; +} + +/* + * New for POSIX ACL extension + */ + +static bool __maybe_unused +xdrgen_decode_notify_type4(struct xdr_stream *xdr, notify_type4 *ptr) +{ + u32 val; + + if (xdr_stream_decode_u32(xdr, &val) < 0) + return false; + /* Compiler may optimize to a range check for dense enums */ + switch (val) { + case NOTIFY4_CHANGE_CHILD_ATTRS: + case NOTIFY4_CHANGE_DIR_ATTRS: + case NOTIFY4_REMOVE_ENTRY: + case NOTIFY4_ADD_ENTRY: + case NOTIFY4_RENAME_ENTRY: + case NOTIFY4_CHANGE_COOKIE_VERIFIER: + case NOTIFY4_GFLAG_EXTEND: + case NOTIFY4_AUFLAG_VALID: + case NOTIFY4_AUFLAG_USER: + case NOTIFY4_AUFLAG_GROUP: + case NOTIFY4_AUFLAG_OTHER: + case NOTIFY4_CHANGE_AUTH: + case NOTIFY4_CFLAG_ORDER: + case NOTIFY4_AUFLAG_GANOW: + case NOTIFY4_AUFLAG_GALATER: + case NOTIFY4_CHANGE_GA: + case NOTIFY4_CHANGE_AMASK: + break; + default: + return false; + } + *ptr = val; + return true; +} + +static bool __maybe_unused +xdrgen_decode_notify_entry4(struct xdr_stream *xdr, struct notify_entry4 *ptr) +{ + if (!xdrgen_decode_component4(xdr, &ptr->ne_file)) + return false; + if (!xdrgen_decode_fattr4(xdr, &ptr->ne_attrs)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_prev_entry4(struct xdr_stream *xdr, struct prev_entry4 *ptr) +{ + if (!xdrgen_decode_notify_entry4(xdr, &ptr->pe_prev_entry)) + return false; + if (!xdrgen_decode_nfs_cookie4(xdr, &ptr->pe_prev_entry_cookie)) + return false; + return true; +} + +bool +xdrgen_decode_notify_remove4(struct xdr_stream *xdr, struct notify_remove4 *ptr) +{ + if (!xdrgen_decode_notify_entry4(xdr, &ptr->nrm_old_entry)) + return false; + if (!xdrgen_decode_nfs_cookie4(xdr, &ptr->nrm_old_entry_cookie)) + return false; + return true; +} + +bool +xdrgen_decode_notify_add4(struct xdr_stream *xdr, struct notify_add4 *ptr) +{ + if (xdr_stream_decode_u32(xdr, &ptr->nad_old_entry.count) < 0) + return false; + if (ptr->nad_old_entry.count > 1) + return false; + for (u32 i = 0; i < ptr->nad_old_entry.count; i++) + if (!xdrgen_decode_notify_remove4(xdr, &ptr->nad_old_entry.element[i])) + return false; + if (!xdrgen_decode_notify_entry4(xdr, &ptr->nad_new_entry)) + return false; + if (xdr_stream_decode_u32(xdr, &ptr->nad_new_entry_cookie.count) < 0) + return false; + if (ptr->nad_new_entry_cookie.count > 1) + return false; + for (u32 i = 0; i < ptr->nad_new_entry_cookie.count; i++) + if (!xdrgen_decode_nfs_cookie4(xdr, &ptr->nad_new_entry_cookie.element[i])) + return false; + if (xdr_stream_decode_u32(xdr, &ptr->nad_prev_entry.count) < 0) + return false; + if (ptr->nad_prev_entry.count > 1) + return false; + for (u32 i = 0; i < ptr->nad_prev_entry.count; i++) + if (!xdrgen_decode_prev_entry4(xdr, &ptr->nad_prev_entry.element[i])) + return false; + if (!xdrgen_decode_bool(xdr, &ptr->nad_last_entry)) + return false; + return true; +} + +bool +xdrgen_decode_notify_attr4(struct xdr_stream *xdr, struct notify_attr4 *ptr) +{ + if (!xdrgen_decode_notify_entry4(xdr, &ptr->na_changed_entry)) + return false; + return true; +} + +bool +xdrgen_decode_notify_rename4(struct xdr_stream *xdr, struct notify_rename4 *ptr) +{ + if (!xdrgen_decode_notify_remove4(xdr, &ptr->nrn_old_entry)) + return false; + if (!xdrgen_decode_notify_add4(xdr, &ptr->nrn_new_entry)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_notify_verifier4(struct xdr_stream *xdr, struct notify_verifier4 *ptr) +{ + if (!xdrgen_decode_verifier4(xdr, &ptr->nv_old_cookieverf)) + return false; + if (!xdrgen_decode_verifier4(xdr, &ptr->nv_new_cookieverf)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_notifylist4(struct xdr_stream *xdr, notifylist4 *ptr) +{ + return xdrgen_decode_opaque(xdr, ptr, 0); +} + +static bool __maybe_unused +xdrgen_decode_notify4(struct xdr_stream *xdr, struct notify4 *ptr) +{ + if (!xdrgen_decode_bitmap4(xdr, &ptr->notify_mask)) + return false; + if (!xdrgen_decode_notifylist4(xdr, &ptr->notify_vals)) + return false; + return true; +} + +bool +xdrgen_decode_CB_NOTIFY4args(struct xdr_stream *xdr, struct CB_NOTIFY4args *ptr) +{ + if (!xdrgen_decode_stateid4(xdr, &ptr->cna_stateid)) + return false; + if (!xdrgen_decode_nfs_fh4(xdr, &ptr->cna_fh)) + return false; + if (xdr_stream_decode_u32(xdr, &ptr->cna_changes.count) < 0) + return false; + for (u32 i = 0; i < ptr->cna_changes.count; i++) + if (!xdrgen_decode_notify4(xdr, &ptr->cna_changes.element[i])) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_decode_CB_NOTIFY4res(struct xdr_stream *xdr, struct CB_NOTIFY4res *ptr) +{ + if (!xdrgen_decode_nfsstat4(xdr, &ptr->cnr_status)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_int32_t(struct xdr_stream *xdr, const int32_t value) +{ + return xdrgen_encode_int(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_uint32_t(struct xdr_stream *xdr, const uint32_t value) +{ + return xdrgen_encode_unsigned_int(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_int64_t(struct xdr_stream *xdr, const int64_t value) +{ + return xdrgen_encode_hyper(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_uint64_t(struct xdr_stream *xdr, const uint64_t value) +{ + return xdrgen_encode_unsigned_hyper(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_nfsstat4(struct xdr_stream *xdr, nfsstat4 value) +{ + return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; +} + +static bool __maybe_unused +xdrgen_encode_attrlist4(struct xdr_stream *xdr, const attrlist4 value) +{ + return xdr_stream_encode_opaque(xdr, value.data, value.len) >= 0; +} + +static bool __maybe_unused +xdrgen_encode_bitmap4(struct xdr_stream *xdr, const bitmap4 value) +{ + if (xdr_stream_encode_u32(xdr, value.count) != XDR_UNIT) + return false; + for (u32 i = 0; i < value.count; i++) + if (!xdrgen_encode_uint32_t(xdr, value.element[i])) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_verifier4(struct xdr_stream *xdr, const verifier4 value) +{ + return xdr_stream_encode_opaque_fixed(xdr, value, NFS4_VERIFIER_SIZE) >= 0; +} + +static bool __maybe_unused +xdrgen_encode_nfs_cookie4(struct xdr_stream *xdr, const nfs_cookie4 value) +{ + return xdrgen_encode_uint64_t(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_nfs_fh4(struct xdr_stream *xdr, const nfs_fh4 value) +{ + return xdr_stream_encode_opaque(xdr, value.data, value.len) >= 0; +} + +static bool __maybe_unused +xdrgen_encode_utf8string(struct xdr_stream *xdr, const utf8string value) +{ + return xdr_stream_encode_opaque(xdr, value.data, value.len) >= 0; +} + +static bool __maybe_unused +xdrgen_encode_utf8str_cis(struct xdr_stream *xdr, const utf8str_cis value) +{ + return xdrgen_encode_utf8string(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_utf8str_cs(struct xdr_stream *xdr, const utf8str_cs value) +{ + return xdrgen_encode_utf8string(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_utf8str_mixed(struct xdr_stream *xdr, const utf8str_mixed value) +{ + return xdrgen_encode_utf8string(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_component4(struct xdr_stream *xdr, const component4 value) +{ + return xdrgen_encode_utf8str_cs(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_linktext4(struct xdr_stream *xdr, const linktext4 value) +{ + return xdrgen_encode_utf8str_cs(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_pathname4(struct xdr_stream *xdr, const pathname4 value) +{ + if (xdr_stream_encode_u32(xdr, value.count) != XDR_UNIT) + return false; + for (u32 i = 0; i < value.count; i++) + if (!xdrgen_encode_component4(xdr, value.element[i])) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_nfstime4(struct xdr_stream *xdr, const struct nfstime4 *value) +{ + if (!xdrgen_encode_int64_t(xdr, value->seconds)) + return false; + if (!xdrgen_encode_uint32_t(xdr, value->nseconds)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_fattr4(struct xdr_stream *xdr, const struct fattr4 *value) +{ + if (!xdrgen_encode_bitmap4(xdr, value->attrmask)) + return false; + if (!xdrgen_encode_attrlist4(xdr, value->attr_vals)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_stateid4(struct xdr_stream *xdr, const struct stateid4 *value) +{ + if (!xdrgen_encode_uint32_t(xdr, value->seqid)) + return false; + if (xdr_stream_encode_opaque_fixed(xdr, value->other, 12) < 0) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_fattr4_offline(struct xdr_stream *xdr, const fattr4_offline value) +{ + return xdrgen_encode_bool(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_open_arguments4(struct xdr_stream *xdr, const struct open_arguments4 *value) +{ + if (!xdrgen_encode_bitmap4(xdr, value->oa_share_access)) + return false; + if (!xdrgen_encode_bitmap4(xdr, value->oa_share_deny)) + return false; + if (!xdrgen_encode_bitmap4(xdr, value->oa_share_access_want)) + return false; + if (!xdrgen_encode_bitmap4(xdr, value->oa_open_claim)) + return false; + if (!xdrgen_encode_bitmap4(xdr, value->oa_create_mode)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_open_args_share_access4(struct xdr_stream *xdr, open_args_share_access4 value) +{ + return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; +} + +static bool __maybe_unused +xdrgen_encode_open_args_share_deny4(struct xdr_stream *xdr, open_args_share_deny4 value) +{ + return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; +} + +static bool __maybe_unused +xdrgen_encode_open_args_share_access_want4(struct xdr_stream *xdr, open_args_share_access_want4 value) +{ + return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; +} + +static bool __maybe_unused +xdrgen_encode_open_args_open_claim4(struct xdr_stream *xdr, open_args_open_claim4 value) +{ + return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; +} + +static bool __maybe_unused +xdrgen_encode_open_args_createmode4(struct xdr_stream *xdr, open_args_createmode4 value) +{ + return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; +} + +bool +xdrgen_encode_fattr4_open_arguments(struct xdr_stream *xdr, const fattr4_open_arguments *value) +{ + return xdrgen_encode_open_arguments4(xdr, value); +} + +bool +xdrgen_encode_fattr4_time_deleg_access(struct xdr_stream *xdr, const fattr4_time_deleg_access *value) +{ + return xdrgen_encode_nfstime4(xdr, value); +} + +bool +xdrgen_encode_fattr4_time_deleg_modify(struct xdr_stream *xdr, const fattr4_time_deleg_modify *value) +{ + return xdrgen_encode_nfstime4(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_open_delegation_type4(struct xdr_stream *xdr, open_delegation_type4 value) +{ + return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; +} + +bool +xdrgen_encode_aclmodel4(struct xdr_stream *xdr, aclmodel4 value) +{ + return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; +} + +bool +xdrgen_encode_aclscope4(struct xdr_stream *xdr, aclscope4 value) +{ + return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; +} + +bool +xdrgen_encode_posixacetag4(struct xdr_stream *xdr, posixacetag4 value) +{ + return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; +} + +bool +xdrgen_encode_posixaceperm4(struct xdr_stream *xdr, const posixaceperm4 value) +{ + return xdrgen_encode_uint32_t(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_posixace4(struct xdr_stream *xdr, const struct posixace4 *value) +{ + if (!xdrgen_encode_posixacetag4(xdr, value->tag)) + return false; + if (!xdrgen_encode_posixaceperm4(xdr, value->perm)) + return false; + if (!xdrgen_encode_utf8str_mixed(xdr, value->who)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_fattr4_acl_trueform(struct xdr_stream *xdr, const fattr4_acl_trueform value) +{ + return xdrgen_encode_aclmodel4(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_fattr4_acl_trueform_scope(struct xdr_stream *xdr, const fattr4_acl_trueform_scope value) +{ + return xdrgen_encode_aclscope4(xdr, value); +} + +static bool __maybe_unused +xdrgen_encode_fattr4_posix_default_acl(struct xdr_stream *xdr, const fattr4_posix_default_acl value) +{ + if (xdr_stream_encode_u32(xdr, value.count) != XDR_UNIT) + return false; + for (u32 i = 0; i < value.count; i++) + if (!xdrgen_encode_posixace4(xdr, &value.element[i])) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_fattr4_posix_access_acl(struct xdr_stream *xdr, const fattr4_posix_access_acl value) +{ + if (xdr_stream_encode_u32(xdr, value.count) != XDR_UNIT) + return false; + for (u32 i = 0; i < value.count; i++) + if (!xdrgen_encode_posixace4(xdr, &value.element[i])) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_notify_type4(struct xdr_stream *xdr, notify_type4 value) +{ + return xdr_stream_encode_u32(xdr, value) == XDR_UNIT; +} + +static bool __maybe_unused +xdrgen_encode_notify_entry4(struct xdr_stream *xdr, const struct notify_entry4 *value) +{ + if (!xdrgen_encode_component4(xdr, value->ne_file)) + return false; + if (!xdrgen_encode_fattr4(xdr, &value->ne_attrs)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_prev_entry4(struct xdr_stream *xdr, const struct prev_entry4 *value) +{ + if (!xdrgen_encode_notify_entry4(xdr, &value->pe_prev_entry)) + return false; + if (!xdrgen_encode_nfs_cookie4(xdr, value->pe_prev_entry_cookie)) + return false; + return true; +} + +bool +xdrgen_encode_notify_remove4(struct xdr_stream *xdr, const struct notify_remove4 *value) +{ + if (!xdrgen_encode_notify_entry4(xdr, &value->nrm_old_entry)) + return false; + if (!xdrgen_encode_nfs_cookie4(xdr, value->nrm_old_entry_cookie)) + return false; + return true; +} + +bool +xdrgen_encode_notify_add4(struct xdr_stream *xdr, const struct notify_add4 *value) +{ + if (value->nad_old_entry.count > 1) + return false; + if (xdr_stream_encode_u32(xdr, value->nad_old_entry.count) != XDR_UNIT) + return false; + for (u32 i = 0; i < value->nad_old_entry.count; i++) + if (!xdrgen_encode_notify_remove4(xdr, &value->nad_old_entry.element[i])) + return false; + if (!xdrgen_encode_notify_entry4(xdr, &value->nad_new_entry)) + return false; + if (value->nad_new_entry_cookie.count > 1) + return false; + if (xdr_stream_encode_u32(xdr, value->nad_new_entry_cookie.count) != XDR_UNIT) + return false; + for (u32 i = 0; i < value->nad_new_entry_cookie.count; i++) + if (!xdrgen_encode_nfs_cookie4(xdr, value->nad_new_entry_cookie.element[i])) + return false; + if (value->nad_prev_entry.count > 1) + return false; + if (xdr_stream_encode_u32(xdr, value->nad_prev_entry.count) != XDR_UNIT) + return false; + for (u32 i = 0; i < value->nad_prev_entry.count; i++) + if (!xdrgen_encode_prev_entry4(xdr, &value->nad_prev_entry.element[i])) + return false; + if (!xdrgen_encode_bool(xdr, value->nad_last_entry)) + return false; + return true; +} + +bool +xdrgen_encode_notify_attr4(struct xdr_stream *xdr, const struct notify_attr4 *value) +{ + if (!xdrgen_encode_notify_entry4(xdr, &value->na_changed_entry)) + return false; + return true; +} + +bool +xdrgen_encode_notify_rename4(struct xdr_stream *xdr, const struct notify_rename4 *value) +{ + if (!xdrgen_encode_notify_remove4(xdr, &value->nrn_old_entry)) + return false; + if (!xdrgen_encode_notify_add4(xdr, &value->nrn_new_entry)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_notify_verifier4(struct xdr_stream *xdr, const struct notify_verifier4 *value) +{ + if (!xdrgen_encode_verifier4(xdr, value->nv_old_cookieverf)) + return false; + if (!xdrgen_encode_verifier4(xdr, value->nv_new_cookieverf)) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_notifylist4(struct xdr_stream *xdr, const notifylist4 value) +{ + return xdr_stream_encode_opaque(xdr, value.data, value.len) >= 0; +} + +static bool __maybe_unused +xdrgen_encode_notify4(struct xdr_stream *xdr, const struct notify4 *value) +{ + if (!xdrgen_encode_bitmap4(xdr, value->notify_mask)) + return false; + if (!xdrgen_encode_notifylist4(xdr, value->notify_vals)) + return false; + return true; +} + +bool +xdrgen_encode_CB_NOTIFY4args(struct xdr_stream *xdr, const struct CB_NOTIFY4args *value) +{ + if (!xdrgen_encode_stateid4(xdr, &value->cna_stateid)) + return false; + if (!xdrgen_encode_nfs_fh4(xdr, value->cna_fh)) + return false; + if (xdr_stream_encode_u32(xdr, value->cna_changes.count) != XDR_UNIT) + return false; + for (u32 i = 0; i < value->cna_changes.count; i++) + if (!xdrgen_encode_notify4(xdr, &value->cna_changes.element[i])) + return false; + return true; +} + +static bool __maybe_unused +xdrgen_encode_CB_NOTIFY4res(struct xdr_stream *xdr, const struct CB_NOTIFY4res *value) +{ + if (!xdrgen_encode_nfsstat4(xdr, value->cnr_status)) + return false; + return true; +} diff --git a/fs/nfsd/nfs4xdr_gen.h b/fs/nfsd/nfs4xdr_gen.h new file mode 100644 index 000000000..21ca82078 --- /dev/null +++ b/fs/nfsd/nfs4xdr_gen.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Generated by xdrgen. Manual edits will be lost. */ +/* XDR specification file: ../../Documentation/sunrpc/xdr/nfs4_1.x */ +/* XDR specification modification time: Tue Jun 30 11:57:21 2026 */ + +#ifndef _LINUX_XDRGEN_NFS4_1_DECL_H +#define _LINUX_XDRGEN_NFS4_1_DECL_H + +#include + +#include +#include +#include +#include + +bool xdrgen_decode_fattr4_open_arguments(struct xdr_stream *xdr, fattr4_open_arguments *ptr); +bool xdrgen_encode_fattr4_open_arguments(struct xdr_stream *xdr, const fattr4_open_arguments *value); + +bool xdrgen_decode_fattr4_time_deleg_access(struct xdr_stream *xdr, fattr4_time_deleg_access *ptr); +bool xdrgen_encode_fattr4_time_deleg_access(struct xdr_stream *xdr, const fattr4_time_deleg_access *value); + +bool xdrgen_decode_fattr4_time_deleg_modify(struct xdr_stream *xdr, fattr4_time_deleg_modify *ptr); +bool xdrgen_encode_fattr4_time_deleg_modify(struct xdr_stream *xdr, const fattr4_time_deleg_modify *value); + +bool xdrgen_decode_aclmodel4(struct xdr_stream *xdr, aclmodel4 *ptr); +bool xdrgen_encode_aclmodel4(struct xdr_stream *xdr, aclmodel4 value); + +bool xdrgen_decode_aclscope4(struct xdr_stream *xdr, aclscope4 *ptr); +bool xdrgen_encode_aclscope4(struct xdr_stream *xdr, aclscope4 value); + +bool xdrgen_decode_posixacetag4(struct xdr_stream *xdr, posixacetag4 *ptr); +bool xdrgen_encode_posixacetag4(struct xdr_stream *xdr, posixacetag4 value); + +bool xdrgen_decode_posixaceperm4(struct xdr_stream *xdr, posixaceperm4 *ptr); +bool xdrgen_encode_posixaceperm4(struct xdr_stream *xdr, const posixaceperm4 value); + + +bool xdrgen_decode_notify_remove4(struct xdr_stream *xdr, struct notify_remove4 *ptr); +bool xdrgen_encode_notify_remove4(struct xdr_stream *xdr, const struct notify_remove4 *value); + +bool xdrgen_decode_notify_add4(struct xdr_stream *xdr, struct notify_add4 *ptr); +bool xdrgen_encode_notify_add4(struct xdr_stream *xdr, const struct notify_add4 *value); + +bool xdrgen_decode_notify_attr4(struct xdr_stream *xdr, struct notify_attr4 *ptr); +bool xdrgen_encode_notify_attr4(struct xdr_stream *xdr, const struct notify_attr4 *value); + +bool xdrgen_decode_notify_rename4(struct xdr_stream *xdr, struct notify_rename4 *ptr); +bool xdrgen_encode_notify_rename4(struct xdr_stream *xdr, const struct notify_rename4 *value); + +bool xdrgen_decode_CB_NOTIFY4args(struct xdr_stream *xdr, struct CB_NOTIFY4args *ptr); +bool xdrgen_encode_CB_NOTIFY4args(struct xdr_stream *xdr, const struct CB_NOTIFY4args *value); + +#endif /* _LINUX_XDRGEN_NFS4_1_DECL_H */ diff --git a/fs/nfsd/nfscache.c b/fs/nfsd/nfscache.c new file mode 100644 index 000000000..c7db532c8 --- /dev/null +++ b/fs/nfsd/nfscache.c @@ -0,0 +1,669 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Request reply cache. This is currently a global cache, but this may + * change in the future and be a per-client cache. + * + * This code is heavily inspired by the 44BSD implementation, although + * it does things a bit differently. + * + * Copyright (C) 1995, 1996 Olaf Kirch + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nfsd.h" +#include "netns.h" +#include "stats.h" +#include "cache.h" +#include "trace.h" + +/* + * We use this value to determine the number of hash buckets from the max + * cache size, the idea being that when the cache is at its maximum number + * of entries, then this should be the average number of entries per bucket. + */ +#define TARGET_BUCKET_SIZE 8 + +struct nfsd_drc_bucket { + struct rb_root rb_head; + struct list_head lru_head; + spinlock_t cache_lock; +}; + +static struct kmem_cache *drc_slab; + +static int nfsd_cache_append(struct svc_rqst *rqstp, struct kvec *vec); +static unsigned long nfsd_reply_cache_count(struct shrinker *shrink, + struct shrink_control *sc); +static unsigned long nfsd_reply_cache_scan(struct shrinker *shrink, + struct shrink_control *sc); + +/* + * Put a cap on the size of the DRC based on the amount of available + * low memory in the machine. + * + * 64MB: 8192 + * 128MB: 11585 + * 256MB: 16384 + * 512MB: 23170 + * 1GB: 32768 + * 2GB: 46340 + * 4GB: 65536 + * 8GB: 92681 + * 16GB: 131072 + * + * ...with a hard cap of 256k entries. In the worst case, each entry will be + * ~1k, so the above numbers should give a rough max of the amount of memory + * used in k. + * + * XXX: these limits are per-container, so memory used will increase + * linearly with number of containers. Maybe that's OK. + */ +static unsigned int +nfsd_cache_size_limit(void) +{ + unsigned int limit; + unsigned long low_pages = totalram_pages() - totalhigh_pages(); + + limit = (16 * int_sqrt(low_pages)) << (PAGE_SHIFT-10); + return min_t(unsigned int, limit, 256*1024); +} + +/* + * Compute the number of hash buckets we need. Divide the max cachesize by + * the "target" max bucket size, and round up to next power of two. + */ +static unsigned int +nfsd_hashsize(unsigned int limit) +{ + return roundup_pow_of_two(limit / TARGET_BUCKET_SIZE); +} + +static struct nfsd_cacherep * +nfsd_cacherep_alloc(struct svc_rqst *rqstp, __wsum csum, + struct nfsd_net *nn) +{ + struct nfsd_cacherep *rp; + + rp = kmem_cache_alloc(drc_slab, GFP_KERNEL); + if (rp) { + rp->c_state = RC_UNUSED; + rp->c_type = RC_NOCACHE; + RB_CLEAR_NODE(&rp->c_node); + INIT_LIST_HEAD(&rp->c_lru); + + memset(&rp->c_key, 0, sizeof(rp->c_key)); + rp->c_key.k_xid = rqstp->rq_xid; + rp->c_key.k_proc = rqstp->rq_proc; + rpc_copy_addr((struct sockaddr *)&rp->c_key.k_addr, svc_addr(rqstp)); + rpc_set_port((struct sockaddr *)&rp->c_key.k_addr, rpc_get_port(svc_addr(rqstp))); + rp->c_key.k_prot = rqstp->rq_prot; + rp->c_key.k_vers = rqstp->rq_vers; + rp->c_key.k_len = rqstp->rq_arg.len; + rp->c_key.k_csum = csum; + } + return rp; +} + +static void nfsd_cacherep_free(struct nfsd_cacherep *rp) +{ + if (rp->c_type == RC_REPLBUFF) + kfree(rp->c_replvec.iov_base); + kmem_cache_free(drc_slab, rp); +} + +static unsigned long +nfsd_cacherep_dispose(struct list_head *dispose) +{ + struct nfsd_cacherep *rp; + unsigned long freed = 0; + + while (!list_empty(dispose)) { + rp = list_first_entry(dispose, struct nfsd_cacherep, c_lru); + list_del(&rp->c_lru); + nfsd_cacherep_free(rp); + freed++; + } + return freed; +} + +static void +nfsd_cacherep_unlink_locked(struct nfsd_net *nn, struct nfsd_drc_bucket *b, + struct nfsd_cacherep *rp) +{ + if (rp->c_type == RC_REPLBUFF && rp->c_replvec.iov_base) + nfsd_stats_drc_mem_usage_sub(nn, rp->c_replvec.iov_len); + if (rp->c_state != RC_UNUSED) { + rb_erase(&rp->c_node, &b->rb_head); + list_del(&rp->c_lru); + atomic_dec(&nn->num_drc_entries); + nfsd_stats_drc_mem_usage_sub(nn, sizeof(*rp)); + } +} + +static void +nfsd_reply_cache_free_locked(struct nfsd_drc_bucket *b, struct nfsd_cacherep *rp, + struct nfsd_net *nn) +{ + nfsd_cacherep_unlink_locked(nn, b, rp); + nfsd_cacherep_free(rp); +} + +static void +nfsd_reply_cache_free(struct nfsd_drc_bucket *b, struct nfsd_cacherep *rp, + struct nfsd_net *nn) +{ + spin_lock(&b->cache_lock); + nfsd_cacherep_unlink_locked(nn, b, rp); + spin_unlock(&b->cache_lock); + nfsd_cacherep_free(rp); +} + +int nfsd_drc_slab_create(void) +{ + drc_slab = KMEM_CACHE(nfsd_cacherep, 0); + return drc_slab ? 0: -ENOMEM; +} + +void nfsd_drc_slab_free(void) +{ + kmem_cache_destroy(drc_slab); +} + +int nfsd_reply_cache_init(struct nfsd_net *nn) +{ + unsigned int hashsize; + unsigned int i; + + nn->max_drc_entries = nfsd_cache_size_limit(); + atomic_set(&nn->num_drc_entries, 0); + hashsize = nfsd_hashsize(nn->max_drc_entries); + nn->maskbits = ilog2(hashsize); + + nn->drc_hashtbl = kvzalloc(array_size(hashsize, + sizeof(*nn->drc_hashtbl)), GFP_KERNEL); + if (!nn->drc_hashtbl) + return -ENOMEM; + + nn->nfsd_reply_cache_shrinker = shrinker_alloc(0, "nfsd-reply:%s", + nn->nfsd_name); + if (!nn->nfsd_reply_cache_shrinker) + goto out_shrinker; + + nn->nfsd_reply_cache_shrinker->scan_objects = nfsd_reply_cache_scan; + nn->nfsd_reply_cache_shrinker->count_objects = nfsd_reply_cache_count; + nn->nfsd_reply_cache_shrinker->seeks = 1; + nn->nfsd_reply_cache_shrinker->private_data = nn; + + for (i = 0; i < hashsize; i++) { + INIT_LIST_HEAD(&nn->drc_hashtbl[i].lru_head); + spin_lock_init(&nn->drc_hashtbl[i].cache_lock); + } + nn->drc_hashsize = hashsize; + + shrinker_register(nn->nfsd_reply_cache_shrinker); + + return 0; +out_shrinker: + kvfree(nn->drc_hashtbl); + printk(KERN_ERR "nfsd: failed to allocate reply cache\n"); + return -ENOMEM; +} + +void nfsd_reply_cache_shutdown(struct nfsd_net *nn) +{ + struct nfsd_cacherep *rp; + unsigned int i; + + shrinker_free(nn->nfsd_reply_cache_shrinker); + + for (i = 0; i < nn->drc_hashsize; i++) { + struct list_head *head = &nn->drc_hashtbl[i].lru_head; + while (!list_empty(head)) { + rp = list_first_entry(head, struct nfsd_cacherep, c_lru); + nfsd_reply_cache_free_locked(&nn->drc_hashtbl[i], + rp, nn); + } + } + + kvfree(nn->drc_hashtbl); + nn->drc_hashtbl = NULL; + nn->drc_hashsize = 0; + +} + +static void +lru_put_end(struct nfsd_drc_bucket *b, struct nfsd_cacherep *rp) +{ + rp->c_timestamp = jiffies; + list_move_tail(&rp->c_lru, &b->lru_head); +} + +static noinline struct nfsd_drc_bucket * +nfsd_cache_bucket_find(__be32 xid, struct nfsd_net *nn) +{ + unsigned int hash = hash_32((__force u32)xid, nn->maskbits); + + return &nn->drc_hashtbl[hash]; +} + +/* + * Remove and return no more than @max expired entries in bucket @b. + * If @max is zero, do not limit the number of removed entries. + */ +static void +nfsd_prune_bucket_locked(struct nfsd_net *nn, struct nfsd_drc_bucket *b, + unsigned int max, struct list_head *dispose) +{ + unsigned long expiry = jiffies - RC_EXPIRE; + struct nfsd_cacherep *rp, *tmp; + unsigned int freed = 0; + + lockdep_assert_held(&b->cache_lock); + + /* The bucket LRU is ordered oldest-first. */ + list_for_each_entry_safe(rp, tmp, &b->lru_head, c_lru) { + if (atomic_read(&nn->num_drc_entries) <= nn->max_drc_entries && + time_before(expiry, rp->c_timestamp)) + break; + + nfsd_cacherep_unlink_locked(nn, b, rp); + list_add(&rp->c_lru, dispose); + + if (max && ++freed >= max) + break; + } +} + +/** + * nfsd_reply_cache_count - count_objects method for the DRC shrinker + * @shrink: our registered shrinker context + * @sc: garbage collection parameters + * + * Returns the total number of entries in the duplicate reply cache. To + * keep things simple and quick, this is not the number of expired entries + * in the cache (ie, the number that would be removed by a call to + * nfsd_reply_cache_scan). + */ +static unsigned long +nfsd_reply_cache_count(struct shrinker *shrink, struct shrink_control *sc) +{ + struct nfsd_net *nn = shrink->private_data; + + return atomic_read(&nn->num_drc_entries); +} + +/** + * nfsd_reply_cache_scan - scan_objects method for the DRC shrinker + * @shrink: our registered shrinker context + * @sc: garbage collection parameters + * + * Free expired entries on each bucket's LRU list until we've released + * nr_to_scan freed objects. Nothing will be released if the cache + * has not exceeded it's max_drc_entries limit. + * + * Returns the number of entries released by this call. + */ +static unsigned long +nfsd_reply_cache_scan(struct shrinker *shrink, struct shrink_control *sc) +{ + struct nfsd_net *nn = shrink->private_data; + unsigned long freed = 0; + LIST_HEAD(dispose); + unsigned int i; + + for (i = 0; i < nn->drc_hashsize; i++) { + struct nfsd_drc_bucket *b = &nn->drc_hashtbl[i]; + + if (list_empty(&b->lru_head)) + continue; + + spin_lock(&b->cache_lock); + nfsd_prune_bucket_locked(nn, b, 0, &dispose); + spin_unlock(&b->cache_lock); + + freed += nfsd_cacherep_dispose(&dispose); + if (freed > sc->nr_to_scan) + break; + } + return freed; +} + +/** + * nfsd_cache_csum - Checksum incoming NFS Call arguments + * @buf: buffer containing a whole RPC Call message + * @start: starting byte of the NFS Call header + * @remaining: size of the NFS Call header, in bytes + * + * Compute a weak checksum of the leading bytes of an NFS procedure + * call header to help verify that a retransmitted Call matches an + * entry in the duplicate reply cache. + * + * To avoid assumptions about how the RPC message is laid out in + * @buf and what else it might contain (eg, a GSS MIC suffix), the + * caller passes us the exact location and length of the NFS Call + * header. + * + * Returns a 32-bit checksum value, as defined in RFC 793. + */ +static __wsum nfsd_cache_csum(struct xdr_buf *buf, unsigned int start, + unsigned int remaining) +{ + unsigned int base, len; + struct xdr_buf subbuf; + __wsum csum = 0; + void *p; + int idx; + + if (remaining > RC_CSUMLEN) + remaining = RC_CSUMLEN; + if (xdr_buf_subsegment(buf, &subbuf, start, remaining)) + return csum; + + /* rq_arg.head first */ + if (subbuf.head[0].iov_len) { + len = min_t(unsigned int, subbuf.head[0].iov_len, remaining); + csum = csum_partial(subbuf.head[0].iov_base, len, csum); + remaining -= len; + } + + /* Continue into page array */ + idx = subbuf.page_base / PAGE_SIZE; + base = subbuf.page_base & ~PAGE_MASK; + while (remaining) { + p = page_address(subbuf.pages[idx]) + base; + len = min_t(unsigned int, PAGE_SIZE - base, remaining); + csum = csum_partial(p, len, csum); + remaining -= len; + base = 0; + ++idx; + } + return csum; +} + +static int +nfsd_cache_key_cmp(const struct nfsd_cacherep *key, + const struct nfsd_cacherep *rp, struct nfsd_net *nn) +{ + if (key->c_key.k_xid == rp->c_key.k_xid && + key->c_key.k_csum != rp->c_key.k_csum) { + nfsd_stats_payload_misses_inc(nn); + trace_nfsd_drc_mismatch(nn, key, rp); + } + + return memcmp(&key->c_key, &rp->c_key, sizeof(key->c_key)); +} + +/* + * Search the request hash for an entry that matches the given rqstp. + * Must be called with cache_lock held. Returns the found entry or + * inserts an empty key on failure. + */ +static struct nfsd_cacherep * +nfsd_cache_insert(struct nfsd_drc_bucket *b, struct nfsd_cacherep *key, + struct nfsd_net *nn) +{ + struct nfsd_cacherep *rp, *ret = key; + struct rb_node **p = &b->rb_head.rb_node, + *parent = NULL; + unsigned int entries = 0; + int cmp; + + while (*p != NULL) { + ++entries; + parent = *p; + rp = rb_entry(parent, struct nfsd_cacherep, c_node); + + cmp = nfsd_cache_key_cmp(key, rp, nn); + if (cmp < 0) + p = &parent->rb_left; + else if (cmp > 0) + p = &parent->rb_right; + else { + ret = rp; + goto out; + } + } + rb_link_node(&key->c_node, parent, p); + rb_insert_color(&key->c_node, &b->rb_head); +out: + /* tally hash chain length stats */ + if (entries > nn->longest_chain) { + nn->longest_chain = entries; + nn->longest_chain_cachesize = atomic_read(&nn->num_drc_entries); + } else if (entries == nn->longest_chain) { + /* prefer to keep the smallest cachesize possible here */ + nn->longest_chain_cachesize = min_t(unsigned int, + nn->longest_chain_cachesize, + atomic_read(&nn->num_drc_entries)); + } + return ret; +} + +/** + * nfsd_cache_lookup - Find an entry in the duplicate reply cache + * @rqstp: Incoming Call to find + * @start: starting byte in @rqstp->rq_arg of the NFS Call header + * @len: size of the NFS Call header, in bytes + * @cacherep: OUT: DRC entry for this request + * + * Try to find an entry matching the current call in the cache. When none + * is found, we try to grab the oldest expired entry off the LRU list. If + * a suitable one isn't there, then drop the cache_lock and allocate a + * new one, then search again in case one got inserted while this thread + * didn't hold the lock. + * + * Return values: + * %RC_DOIT: Process the request normally + * %RC_REPLY: Reply from cache + * %RC_DROPIT: Do not process the request further + */ +int nfsd_cache_lookup(struct svc_rqst *rqstp, unsigned int start, + unsigned int len, struct nfsd_cacherep **cacherep) +{ + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + struct nfsd_thread_local_info *ntli = rqstp->rq_private; + struct nfsd_cacherep *rp, *found; + __wsum csum; + struct nfsd_drc_bucket *b; + int type = ntli->ntli_cachetype; + LIST_HEAD(dispose); + int rtn = RC_DOIT; + + if (type == RC_NOCACHE) { + nfsd_stats_rc_nocache_inc(nn); + goto out; + } + + csum = nfsd_cache_csum(&rqstp->rq_arg, start, len); + + /* + * Since the common case is a cache miss followed by an insert, + * preallocate an entry. + */ + rp = nfsd_cacherep_alloc(rqstp, csum, nn); + if (!rp) + goto out; + + b = nfsd_cache_bucket_find(rqstp->rq_xid, nn); + spin_lock(&b->cache_lock); + found = nfsd_cache_insert(b, rp, nn); + if (found != rp) + goto found_entry; + *cacherep = rp; + rp->c_state = RC_INPROG; + nfsd_prune_bucket_locked(nn, b, 3, &dispose); + spin_unlock(&b->cache_lock); + + nfsd_cacherep_dispose(&dispose); + + nfsd_stats_rc_misses_inc(nn); + atomic_inc(&nn->num_drc_entries); + nfsd_stats_drc_mem_usage_add(nn, sizeof(*rp)); + goto out; + +found_entry: + /* We found a matching entry which is either in progress or done. */ + nfsd_reply_cache_free_locked(NULL, rp, nn); + nfsd_stats_rc_hits_inc(nn); + rtn = RC_DROPIT; + rp = found; + + /* Request being processed */ + if (rp->c_state == RC_INPROG) + goto out_trace; + + /* From the hall of fame of impractical attacks: + * Is this a user who tries to snoop on the cache? */ + rtn = RC_DOIT; + if (!test_bit(RQ_SECURE, &rqstp->rq_flags) && rp->c_secure) + goto out_trace; + + /* Compose RPC reply header */ + switch (rp->c_type) { + case RC_NOCACHE: + break; + case RC_REPLSTAT: + xdr_stream_encode_be32(&rqstp->rq_res_stream, rp->c_replstat); + rtn = RC_REPLY; + break; + case RC_REPLBUFF: + if (!nfsd_cache_append(rqstp, &rp->c_replvec)) + goto out_unlock; /* should not happen */ + rtn = RC_REPLY; + break; + default: + WARN_ONCE(1, "nfsd: bad repcache type %d\n", rp->c_type); + } + +out_trace: + trace_nfsd_drc_found(nn, rqstp, rtn); +out_unlock: + spin_unlock(&b->cache_lock); +out: + return rtn; +} + +/** + * nfsd_cache_update - Update an entry in the duplicate reply cache. + * @rqstp: svc_rqst with a finished Reply + * @rp: IN: DRC entry for this request + * @cachetype: which cache to update + * @statp: pointer to Reply's NFS status code, or NULL + * + * This is called from nfsd_dispatch when the procedure has been + * executed and the complete reply is in rqstp->rq_res. + * + * We're copying around data here rather than swapping buffers because + * the toplevel loop requires max-sized buffers, which would be a waste + * of memory for a cache with a max reply size of 100 bytes (diropokres). + * + * If we should start to use different types of cache entries tailored + * specifically for attrstat and fh's, we may save even more space. + * + * Also note that a cachetype of RC_NOCACHE can legally be passed when + * nfsd failed to encode a reply that otherwise would have been cached. + * In this case, nfsd_cache_update is called with statp == NULL. + */ +void nfsd_cache_update(struct svc_rqst *rqstp, struct nfsd_cacherep *rp, + int cachetype, __be32 *statp) +{ + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + struct kvec *resv = &rqstp->rq_res.head[0], *cachv; + struct nfsd_drc_bucket *b; + int len; + size_t bufsize = 0; + + if (!rp) + return; + + b = nfsd_cache_bucket_find(rp->c_key.k_xid, nn); + + len = resv->iov_len - ((char*)statp - (char*)resv->iov_base); + len >>= 2; + + /* Don't cache excessive amounts of data and XDR failures */ + if (!statp || len > (256 >> 2)) { + nfsd_reply_cache_free(b, rp, nn); + return; + } + + switch (cachetype) { + case RC_REPLSTAT: + if (len != 1) + printk("nfsd: RC_REPLSTAT/reply len %d!\n",len); + rp->c_replstat = *statp; + break; + case RC_REPLBUFF: + cachv = &rp->c_replvec; + bufsize = len << 2; + cachv->iov_base = kmalloc(bufsize, GFP_KERNEL); + if (!cachv->iov_base) { + nfsd_reply_cache_free(b, rp, nn); + return; + } + cachv->iov_len = bufsize; + memcpy(cachv->iov_base, statp, bufsize); + break; + case RC_NOCACHE: + nfsd_reply_cache_free(b, rp, nn); + return; + } + spin_lock(&b->cache_lock); + nfsd_stats_drc_mem_usage_add(nn, bufsize); + lru_put_end(b, rp); + rp->c_secure = test_bit(RQ_SECURE, &rqstp->rq_flags); + rp->c_type = cachetype; + rp->c_state = RC_DONE; + spin_unlock(&b->cache_lock); + return; +} + +static int +nfsd_cache_append(struct svc_rqst *rqstp, struct kvec *data) +{ + __be32 *p; + + p = xdr_reserve_space(&rqstp->rq_res_stream, data->iov_len); + if (unlikely(!p)) + return false; + memcpy(p, data->iov_base, data->iov_len); + xdr_commit_encode(&rqstp->rq_res_stream); + return true; +} + +/* + * Note that fields may be added, removed or reordered in the future. Programs + * scraping this file for info should test the labels to ensure they're + * getting the correct field. + */ +int nfsd_reply_cache_stats_show(struct seq_file *m, void *v) +{ + struct nfsd_net *nn = net_generic(file_inode(m->file)->i_sb->s_fs_info, + nfsd_net_id); + + seq_printf(m, "max entries: %u\n", nn->max_drc_entries); + seq_printf(m, "num entries: %u\n", + atomic_read(&nn->num_drc_entries)); + seq_printf(m, "hash buckets: %u\n", 1 << nn->maskbits); + seq_printf(m, "mem usage: %lld\n", + percpu_counter_sum_positive(&nn->counter[NFSD_STATS_DRC_MEM_USAGE])); + seq_printf(m, "cache hits: %lld\n", + percpu_counter_sum_positive(&nn->counter[NFSD_STATS_RC_HITS])); + seq_printf(m, "cache misses: %lld\n", + percpu_counter_sum_positive(&nn->counter[NFSD_STATS_RC_MISSES])); + seq_printf(m, "not cached: %lld\n", + percpu_counter_sum_positive(&nn->counter[NFSD_STATS_RC_NOCACHE])); + seq_printf(m, "payload misses: %lld\n", + percpu_counter_sum_positive(&nn->counter[NFSD_STATS_PAYLOAD_MISSES])); + seq_printf(m, "longest chain len: %u\n", nn->longest_chain); + seq_printf(m, "cachesize at longest: %u\n", nn->longest_chain_cachesize); + return 0; +} diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c new file mode 100644 index 000000000..adb032b73 --- /dev/null +++ b/fs/nfsd/nfsctl.c @@ -0,0 +1,2942 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Syscall interface to knfsd. + * + * Copyright (C) 1995, 1996 Olaf Kirch + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "idmap.h" +#include "nfsd.h" +#include "netns.h" +#include "stats.h" +#include "cache.h" +#include "state.h" +#include "netns.h" +#include "pnfs.h" +#include "filecache.h" +#include "trace.h" +#include "netlink.h" + +/* + * We have a single directory with several nodes in it. + */ +enum { + NFSD_Root = 1, + NFSD_List, + NFSD_Export_Stats, + NFSD_Export_features, + NFSD_Fh, + NFSD_FO_UnlockIP, + NFSD_FO_UnlockFS, + NFSD_Threads, + NFSD_Pool_Threads, + NFSD_Pool_Stats, + NFSD_Reply_Cache_Stats, + NFSD_Versions, + NFSD_Ports, + NFSD_MaxBlkSize, + NFSD_Filecache, + NFSD_Leasetime, + NFSD_Gracetime, + NFSD_RecoveryDir, + NFSD_V4EndGrace, + NFSD_MaxReserved +}; + +/* + * write() for these nodes. + */ +static ssize_t write_filehandle(struct file *file, char *buf, size_t size); +static ssize_t write_unlock_ip(struct file *file, char *buf, size_t size); +static ssize_t write_unlock_fs(struct file *file, char *buf, size_t size); +static ssize_t write_threads(struct file *file, char *buf, size_t size); +static ssize_t write_pool_threads(struct file *file, char *buf, size_t size); +static ssize_t write_versions(struct file *file, char *buf, size_t size); +static ssize_t write_ports(struct file *file, char *buf, size_t size); +static ssize_t write_maxblksize(struct file *file, char *buf, size_t size); +#ifdef CONFIG_NFSD_V4 +static ssize_t write_leasetime(struct file *file, char *buf, size_t size); +static ssize_t write_gracetime(struct file *file, char *buf, size_t size); +#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING +static ssize_t write_recoverydir(struct file *file, char *buf, size_t size); +#endif +static ssize_t write_v4_end_grace(struct file *file, char *buf, size_t size); +#endif + +static ssize_t (*const write_op[])(struct file *, char *, size_t) = { + [NFSD_Fh] = write_filehandle, + [NFSD_FO_UnlockIP] = write_unlock_ip, + [NFSD_FO_UnlockFS] = write_unlock_fs, + [NFSD_Threads] = write_threads, + [NFSD_Pool_Threads] = write_pool_threads, + [NFSD_Versions] = write_versions, + [NFSD_Ports] = write_ports, + [NFSD_MaxBlkSize] = write_maxblksize, +#ifdef CONFIG_NFSD_V4 + [NFSD_Leasetime] = write_leasetime, + [NFSD_Gracetime] = write_gracetime, +#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING + [NFSD_RecoveryDir] = write_recoverydir, +#endif + [NFSD_V4EndGrace] = write_v4_end_grace, +#endif +}; + +static ssize_t nfsctl_transaction_write(struct file *file, const char __user *buf, size_t size, loff_t *pos) +{ + ino_t ino = file_inode(file)->i_ino; + char *data; + ssize_t rv; + + if (ino >= ARRAY_SIZE(write_op) || !write_op[ino]) + return -EINVAL; + + data = simple_transaction_get(file, buf, size); + if (IS_ERR(data)) + return PTR_ERR(data); + + rv = write_op[ino](file, data, size); + if (rv < 0) + return rv; + + simple_transaction_set(file, rv); + return size; +} + +static ssize_t nfsctl_transaction_read(struct file *file, char __user *buf, size_t size, loff_t *pos) +{ + if (! file->private_data) { + /* An attempt to read a transaction file without writing + * causes a 0-byte write so that the file can return + * state information + */ + ssize_t rv = nfsctl_transaction_write(file, buf, 0, pos); + if (rv < 0) + return rv; + } + return simple_transaction_read(file, buf, size, pos); +} + +static const struct file_operations transaction_ops = { + .write = nfsctl_transaction_write, + .read = nfsctl_transaction_read, + .release = simple_transaction_release, + .llseek = default_llseek, +}; + +static int exports_net_open(struct net *net, struct file *file) +{ + int err; + struct seq_file *seq; + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + err = seq_open(file, &nfs_exports_op); + if (err) + return err; + + seq = file->private_data; + seq->private = nn->svc_export_cache; + get_net(net); + return 0; +} + +static int exports_release(struct inode *inode, struct file *file) +{ + struct seq_file *seq = file->private_data; + struct cache_detail *cd = seq->private; + + put_net(cd->net); + return seq_release(inode, file); +} + +static int exports_nfsd_open(struct inode *inode, struct file *file) +{ + return exports_net_open(inode->i_sb->s_fs_info, file); +} + +static const struct file_operations exports_nfsd_operations = { + .open = exports_nfsd_open, + .read = seq_read, + .llseek = seq_lseek, + .release = exports_release, +}; + +static int export_features_show(struct seq_file *m, void *v) +{ + seq_printf(m, "0x%x 0x%x\n", NFSEXP_ALLFLAGS, NFSEXP_SECINFO_FLAGS); + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(export_features); + +static int nfsd_pool_stats_open(struct inode *inode, struct file *file) +{ + struct nfsd_net *nn = net_generic(inode->i_sb->s_fs_info, nfsd_net_id); + + return svc_pool_stats_open(&nn->nfsd_info, file); +} + +static const struct file_operations pool_stats_operations = { + .open = nfsd_pool_stats_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; + +DEFINE_SHOW_ATTRIBUTE(nfsd_reply_cache_stats); + +DEFINE_SHOW_ATTRIBUTE(nfsd_file_cache_stats); + +/*----------------------------------------------------------------------------*/ +/* + * payload - write methods + */ + +static inline struct net *netns(struct file *file) +{ + return file_inode(file)->i_sb->s_fs_info; +} + +/* + * write_unlock_ip - Release all locks used by a client + * + * Experimental. + * + * Input: + * buf: '\n'-terminated C string containing a + * presentation format IP address + * size: length of C string in @buf + * Output: + * On success: returns zero if all specified locks were released; + * returns one if one or more locks were not released + * On error: return code is negative errno value + */ +static ssize_t write_unlock_ip(struct file *file, char *buf, size_t size) +{ + struct sockaddr_storage address; + struct sockaddr *sap = (struct sockaddr *)&address; + size_t salen = sizeof(address); + char *fo_path; + struct net *net = netns(file); + + /* sanity check */ + if (size == 0) + return -EINVAL; + + if (buf[size-1] != '\n') + return -EINVAL; + + fo_path = buf; + if (qword_get(&buf, fo_path, size) < 0) + return -EINVAL; + + if (rpc_pton(net, fo_path, size, sap, salen) == 0) + return -EINVAL; + + trace_nfsd_ctl_unlock_ip(net, sap, svc_addr_len(sap)); + return nlmsvc_unlock_all_by_ip(sap); +} + +/* + * write_unlock_fs - Release all locks on a local file system + * + * Experimental. + * + * Input: + * buf: '\n'-terminated C string containing the + * absolute pathname of a local file system + * size: length of C string in @buf + * Output: + * On success: returns zero if all specified locks were released; + * returns one if one or more locks were not released + * On error: return code is negative errno value + */ +static ssize_t write_unlock_fs(struct file *file, char *buf, size_t size) +{ + struct path path; + char *fo_path; + int error; + struct nfsd_net *nn; + + /* sanity check */ + if (size == 0) + return -EINVAL; + + if (buf[size-1] != '\n') + return -EINVAL; + + fo_path = buf; + if (qword_get(&buf, fo_path, size) < 0) + return -EINVAL; + trace_nfsd_ctl_unlock_fs(netns(file), fo_path); + error = kern_path(fo_path, 0, &path); + if (error) + return error; + + /* + * XXX: Needs better sanity checking. Otherwise we could end up + * releasing locks on the wrong file system. + * + * For example: + * 1. Does the path refer to a directory? + * 2. Is that directory a mount point, or + * 3. Is that directory the root of an exported file system? + */ + error = nlmsvc_unlock_all_by_sb(path.dentry->d_sb); + mutex_lock(&nfsd_mutex); + nn = net_generic(netns(file), nfsd_net_id); + if (test_bit(NFSD_NET_UP, &nn->flags)) { + nfsd4_cancel_copy_by_sb(netns(file), path.dentry->d_sb); + nfsd4_revoke_states(nn, path.dentry->d_sb); + } else { + error = -EINVAL; + } + mutex_unlock(&nfsd_mutex); + + path_put(&path); + return error; +} + +/* + * write_filehandle - Get a variable-length NFS file handle by path + * + * On input, the buffer contains a '\n'-terminated C string comprised of + * three alphanumeric words separated by whitespace. The string may + * contain escape sequences. + * + * Input: + * buf: + * domain: client domain name + * path: export pathname + * maxsize: numeric maximum size of + * @buf + * size: length of C string in @buf + * Output: + * On success: passed-in buffer filled with '\n'-terminated C + * string containing a ASCII hex text version + * of the NFS file handle; + * return code is the size in bytes of the string + * On error: return code is negative errno value + */ +static ssize_t write_filehandle(struct file *file, char *buf, size_t size) +{ + char *dname, *path; + int maxsize; + char *mesg = buf; + int len; + struct auth_domain *dom; + struct knfsd_fh fh; + + if (size == 0) + return -EINVAL; + + if (buf[size-1] != '\n') + return -EINVAL; + buf[size-1] = 0; + + dname = mesg; + len = qword_get(&mesg, dname, size); + if (len <= 0) + return -EINVAL; + + path = dname+len+1; + len = qword_get(&mesg, path, size); + if (len <= 0) + return -EINVAL; + + len = get_int(&mesg, &maxsize); + if (len) + return len; + + if (maxsize < NFS_FHSIZE) + return -EINVAL; + maxsize = min(maxsize, NFS3_FHSIZE); + + if (qword_get(&mesg, mesg, size) > 0) + return -EINVAL; + + trace_nfsd_ctl_filehandle(netns(file), dname, path, maxsize); + + /* we have all the words, they are in buf.. */ + dom = unix_domain_find(dname); + if (!dom) + return -ENOMEM; + + len = exp_rootfh(netns(file), dom, path, &fh, maxsize); + auth_domain_put(dom); + if (len) + return len; + + mesg = buf; + len = SIMPLE_TRANSACTION_LIMIT; + qword_addhex(&mesg, &len, fh.fh_raw, fh.fh_size); + mesg[-1] = '\n'; + return mesg - buf; +} + +/* + * write_threads - Start NFSD, or report the configured number of threads + * + * Input: + * buf: ignored + * size: zero + * Output: + * On success: passed-in buffer filled with '\n'-terminated C + * string numeric value representing the configured + * number of NFSD threads; + * return code is the size in bytes of the string + * On error: return code is zero + * + * OR + * + * Input: + * buf: C string containing an unsigned + * integer value representing the + * number of NFSD threads to start + * size: non-zero length of C string in @buf + * Output: + * On success: NFS service is started; + * passed-in buffer filled with '\n'-terminated C + * string numeric value representing the configured + * number of NFSD threads; + * return code is the size in bytes of the string + * On error: return code is zero or a negative errno value + */ +static ssize_t write_threads(struct file *file, char *buf, size_t size) +{ + char *mesg = buf; + int rv; + struct net *net = netns(file); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + if (size > 0) { + int newthreads; + rv = get_int(&mesg, &newthreads); + if (rv) + return rv; + if (newthreads < 0) + return -EINVAL; + trace_nfsd_ctl_threads(net, newthreads); + mutex_lock(&nfsd_mutex); + if (newthreads > 0 || nn->nfsd_serv != NULL) + rv = nfsd_svc(1, &newthreads, net, file->f_cred, NULL); + else + rv = 0; + mutex_unlock(&nfsd_mutex); + if (rv < 0) + return rv; + } else + rv = nfsd_nrthreads(net); + + return scnprintf(buf, SIMPLE_TRANSACTION_LIMIT, "%d\n", rv); +} + +/* + * write_pool_threads - Set or report the configured number of threads per pool + * + * Input: + * buf: ignored + * size: zero + * + * OR + * + * Input: + * buf: C string containing whitespace- + * separated unsigned integer values + * representing the number of NFSD + * threads to start in each pool + * size: non-zero length of C string in @buf + * Output: + * On success: passed-in buffer filled with '\n'-terminated C + * string containing integer values representing the + * configured number of NFSD threads in each pool; + * return code is the size in bytes of the string + * On error: return code is zero or a negative errno value + */ +static ssize_t write_pool_threads(struct file *file, char *buf, size_t size) +{ + /* if size > 0, look for an array of number of threads per node + * and apply them then write out number of threads per node as reply + */ + char *mesg = buf; + int i; + int rv; + int len; + int npools; + int *nthreads; + struct net *net = netns(file); + + mutex_lock(&nfsd_mutex); + npools = nfsd_nrpools(net); + if (npools == 0) { + /* + * NFS is shut down. The admin can start it by + * writing to the threads file but NOT the pool_threads + * file, sorry. Report zero threads. + */ + mutex_unlock(&nfsd_mutex); + strcpy(buf, "0\n"); + return strlen(buf); + } + + nthreads = kzalloc_objs(int, npools); + rv = -ENOMEM; + if (nthreads == NULL) + goto out_free; + + if (size > 0) { + for (i = 0; i < npools; i++) { + rv = get_int(&mesg, &nthreads[i]); + if (rv == -ENOENT) + break; /* fewer numbers than pools */ + if (rv) + goto out_free; /* syntax error */ + rv = -EINVAL; + if (nthreads[i] < 0) + goto out_free; + trace_nfsd_ctl_pool_threads(net, i, nthreads[i]); + } + + /* + * There must always be a thread in pool 0; the admin + * can't shut down NFS completely using pool_threads. + */ + if (nthreads[0] == 0) + nthreads[0] = 1; + + rv = nfsd_set_nrthreads(i, nthreads, net); + if (rv) + goto out_free; + } + + rv = nfsd_get_nrthreads(npools, nthreads, net); + if (rv) + goto out_free; + + mesg = buf; + size = SIMPLE_TRANSACTION_LIMIT; + for (i = 0; i < npools && size > 0; i++) { + snprintf(mesg, size, "%d%c", nthreads[i], (i == npools-1 ? '\n' : ' ')); + len = strlen(mesg); + size -= len; + mesg += len; + } + rv = mesg - buf; +out_free: + kfree(nthreads); + mutex_unlock(&nfsd_mutex); + return rv; +} + +static ssize_t +nfsd_print_version_support(struct nfsd_net *nn, char *buf, int remaining, + const char *sep, unsigned vers, int minor) +{ + const char *format = minor < 0 ? "%s%c%u" : "%s%c%u.%u"; + bool supported = !!nfsd_vers(nn, vers, NFSD_TEST); + + if (vers == 4 && minor >= 0 && + !nfsd_minorversion(nn, minor, NFSD_TEST)) + supported = false; + if (minor == 0 && supported) + /* + * special case for backward compatability. + * +4.0 is never reported, it is implied by + * +4, unless -4.0 is present. + */ + return 0; + return snprintf(buf, remaining, format, sep, + supported ? '+' : '-', vers, minor); +} + +static ssize_t __write_versions(struct file *file, char *buf, size_t size) +{ + char *mesg = buf; + char *vers, *minorp, sign; + int len, num, remaining; + ssize_t tlen = 0; + char *sep; + struct nfsd_net *nn = net_generic(netns(file), nfsd_net_id); + + if (size > 0) { + if (nn->nfsd_serv) + /* Cannot change versions without updating + * nn->nfsd_serv->sv_xdrsize, and reallocing + * rq_argp and rq_resp + */ + return -EBUSY; + if (buf[size-1] != '\n') + return -EINVAL; + buf[size-1] = 0; + trace_nfsd_ctl_version(netns(file), buf); + + vers = mesg; + len = qword_get(&mesg, vers, size); + if (len <= 0) return -EINVAL; + do { + enum vers_op cmd; + unsigned minor; + sign = *vers; + if (sign == '+' || sign == '-') + num = simple_strtol((vers+1), &minorp, 0); + else + num = simple_strtol(vers, &minorp, 0); + if (*minorp == '.') { + if (num != 4) + return -EINVAL; + if (kstrtouint(minorp+1, 0, &minor) < 0) + return -EINVAL; + } + + cmd = sign == '-' ? NFSD_CLEAR : NFSD_SET; + switch(num) { +#ifdef CONFIG_NFSD_V2 + case 2: +#endif + case 3: + nfsd_vers(nn, num, cmd); + break; + case 4: + if (*minorp == '.') { + if (nfsd_minorversion(nn, minor, cmd) < 0) + return -EINVAL; + } else if ((cmd == NFSD_SET) != nfsd_vers(nn, num, NFSD_TEST)) { + /* + * Either we have +4 and no minors are enabled, + * or we have -4 and at least one minor is enabled. + * In either case, propagate 'cmd' to all minors. + */ + minor = 0; + while (nfsd_minorversion(nn, minor, cmd) >= 0) + minor++; + } + break; + default: + /* Ignore requests to disable non-existent versions */ + if (cmd == NFSD_SET) + return -EINVAL; + } + vers += len + 1; + } while ((len = qword_get(&mesg, vers, size)) > 0); + /* If all get turned off, turn them back on, as + * having no versions is BAD + */ + nfsd_reset_versions(nn); + } + + /* Now write current state into reply buffer */ + sep = ""; + remaining = SIMPLE_TRANSACTION_LIMIT; + for (num=2 ; num <= 4 ; num++) { + int minor; + if (!nfsd_vers(nn, num, NFSD_AVAIL)) + continue; + + minor = -1; + do { + len = nfsd_print_version_support(nn, buf, remaining, + sep, num, minor); + if (len >= remaining) + goto out; + remaining -= len; + buf += len; + tlen += len; + minor++; + if (len) + sep = " "; + } while (num == 4 && minor <= NFSD_SUPPORTED_MINOR_VERSION); + } +out: + len = snprintf(buf, remaining, "\n"); + if (len >= remaining) + return -EINVAL; + return tlen + len; +} + +/* + * write_versions - Set or report the available NFS protocol versions + * + * Input: + * buf: ignored + * size: zero + * Output: + * On success: passed-in buffer filled with '\n'-terminated C + * string containing positive or negative integer + * values representing the current status of each + * protocol version; + * return code is the size in bytes of the string + * On error: return code is zero or a negative errno value + * + * OR + * + * Input: + * buf: C string containing whitespace- + * separated positive or negative + * integer values representing NFS + * protocol versions to enable ("+n") + * or disable ("-n") + * size: non-zero length of C string in @buf + * Output: + * On success: status of zero or more protocol versions has + * been updated; passed-in buffer filled with + * '\n'-terminated C string containing positive + * or negative integer values representing the + * current status of each protocol version; + * return code is the size in bytes of the string + * On error: return code is zero or a negative errno value + */ +static ssize_t write_versions(struct file *file, char *buf, size_t size) +{ + ssize_t rv; + + mutex_lock(&nfsd_mutex); + rv = __write_versions(file, buf, size); + mutex_unlock(&nfsd_mutex); + return rv; +} + +/* + * Zero-length write. Return a list of NFSD's current listener + * transports. + */ +static ssize_t __write_ports_names(char *buf, struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + if (nn->nfsd_serv == NULL) + return 0; + return svc_xprt_names(nn->nfsd_serv, buf, SIMPLE_TRANSACTION_LIMIT); +} + +/* + * A single 'fd' number was written, in which case it must be for + * a socket of a supported family/protocol, and we use it as an + * nfsd listener. + */ +static ssize_t __write_ports_addfd(char *buf, struct net *net, const struct cred *cred) +{ + char *mesg = buf; + int fd, err; + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct svc_serv *serv; + + err = get_int(&mesg, &fd); + if (err != 0 || fd < 0) + return -EINVAL; + trace_nfsd_ctl_ports_addfd(net, fd); + + err = nfsd_create_serv(net); + if (err != 0) + return err; + + serv = nn->nfsd_serv; + err = svc_addsock(serv, net, fd, buf, SIMPLE_TRANSACTION_LIMIT, cred); + + if (!serv->sv_nrthreads && list_empty(&nn->nfsd_serv->sv_permsocks)) + nfsd_destroy_serv(net); + + return err; +} + +/* + * A transport listener is added by writing its transport name and + * a port number. + */ +static ssize_t __write_ports_addxprt(char *buf, struct net *net, const struct cred *cred) +{ + char transport[16]; + struct svc_xprt *xprt; + int port, err; + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct svc_serv *serv; + + if (sscanf(buf, "%15s %5u", transport, &port) != 2) + return -EINVAL; + + if (port < 1 || port > USHRT_MAX) + return -EINVAL; + trace_nfsd_ctl_ports_addxprt(net, transport, port); + + err = nfsd_create_serv(net); + if (err != 0) + return err; + + serv = nn->nfsd_serv; + err = svc_xprt_create(serv, transport, net, + PF_INET, port, SVC_SOCK_ANONYMOUS, cred); + if (err < 0) + goto out_err; + + err = svc_xprt_create(serv, transport, net, + PF_INET6, port, SVC_SOCK_ANONYMOUS, cred); + if (err < 0 && err != -EAFNOSUPPORT) + goto out_close; + + return 0; +out_close: + xprt = svc_find_xprt(serv, transport, net, PF_INET, port); + if (xprt != NULL) { + svc_xprt_close(xprt); + svc_xprt_put(xprt); + } +out_err: + if (!serv->sv_nrthreads && list_empty(&nn->nfsd_serv->sv_permsocks)) + nfsd_destroy_serv(net); + + return err; +} + +static ssize_t __write_ports(struct file *file, char *buf, size_t size, + struct net *net) +{ + if (size == 0) + return __write_ports_names(buf, net); + + if (isdigit(buf[0])) + return __write_ports_addfd(buf, net, file->f_cred); + + if (isalpha(buf[0])) + return __write_ports_addxprt(buf, net, file->f_cred); + + return -EINVAL; +} + +/* + * write_ports - Pass a socket file descriptor or transport name to listen on + * + * Input: + * buf: ignored + * size: zero + * Output: + * On success: passed-in buffer filled with a '\n'-terminated C + * string containing a whitespace-separated list of + * named NFSD listeners; + * return code is the size in bytes of the string + * On error: return code is zero or a negative errno value + * + * OR + * + * Input: + * buf: C string containing an unsigned + * integer value representing a bound + * but unconnected socket that is to be + * used as an NFSD listener; listen(3) + * must be called for a SOCK_STREAM + * socket, otherwise it is ignored + * size: non-zero length of C string in @buf + * Output: + * On success: NFS service is started; + * passed-in buffer filled with a '\n'-terminated C + * string containing a unique alphanumeric name of + * the listener; + * return code is the size in bytes of the string + * On error: return code is a negative errno value + * + * OR + * + * Input: + * buf: C string containing a transport + * name and an unsigned integer value + * representing the port to listen on, + * separated by whitespace + * size: non-zero length of C string in @buf + * Output: + * On success: returns zero; NFS service is started + * On error: return code is a negative errno value + */ +static ssize_t write_ports(struct file *file, char *buf, size_t size) +{ + ssize_t rv; + + mutex_lock(&nfsd_mutex); + rv = __write_ports(file, buf, size, netns(file)); + mutex_unlock(&nfsd_mutex); + return rv; +} + + +int nfsd_max_blksize; + +/* + * write_maxblksize - Set or report the current NFS blksize + * + * Input: + * buf: ignored + * size: zero + * + * OR + * + * Input: + * buf: C string containing an unsigned + * integer value representing the new + * NFS blksize + * size: non-zero length of C string in @buf + * Output: + * On success: passed-in buffer filled with '\n'-terminated C string + * containing numeric value of the current NFS blksize + * setting; + * return code is the size in bytes of the string + * On error: return code is zero or a negative errno value + */ +static ssize_t write_maxblksize(struct file *file, char *buf, size_t size) +{ + char *mesg = buf; + struct nfsd_net *nn = net_generic(netns(file), nfsd_net_id); + + if (size > 0) { + int bsize; + int rv = get_int(&mesg, &bsize); + if (rv) + return rv; + trace_nfsd_ctl_maxblksize(netns(file), bsize); + + /* force bsize into allowed range and + * required alignment. + */ + bsize = max_t(int, bsize, 1024); + bsize = min_t(int, bsize, NFSSVC_MAXBLKSIZE); + bsize &= ~(1024-1); + mutex_lock(&nfsd_mutex); + if (nn->nfsd_serv) { + mutex_unlock(&nfsd_mutex); + return -EBUSY; + } + nfsd_max_blksize = bsize; + mutex_unlock(&nfsd_mutex); + } + + return scnprintf(buf, SIMPLE_TRANSACTION_LIMIT, "%d\n", + nfsd_max_blksize); +} + +#ifdef CONFIG_NFSD_V4 +static ssize_t __nfsd4_write_time(struct file *file, char *buf, size_t size, + time64_t *time, struct nfsd_net *nn) +{ + struct dentry *dentry = file_dentry(file); + char *mesg = buf; + int rv, i; + + if (size > 0) { + if (nn->nfsd_serv) + return -EBUSY; + rv = get_int(&mesg, &i); + if (rv) + return rv; + trace_nfsd_ctl_time(netns(file), dentry->d_name.name, + dentry->d_name.len, i); + + /* + * Some sanity checking. We don't have a reason for + * these particular numbers, but problems with the + * extremes are: + * - Too short: the briefest network outage may + * cause clients to lose all their locks. Also, + * the frequent polling may be wasteful. + * - Too long: do you really want reboot recovery + * to take more than an hour? Or to make other + * clients wait an hour before being able to + * revoke a dead client's locks? + */ + if (i < 10 || i > 3600) + return -EINVAL; + *time = i; + } + + return scnprintf(buf, SIMPLE_TRANSACTION_LIMIT, "%lld\n", *time); +} + +static ssize_t nfsd4_write_time(struct file *file, char *buf, size_t size, + time64_t *time, struct nfsd_net *nn) +{ + ssize_t rv; + + mutex_lock(&nfsd_mutex); + rv = __nfsd4_write_time(file, buf, size, time, nn); + mutex_unlock(&nfsd_mutex); + return rv; +} + +/* + * write_leasetime - Set or report the current NFSv4 lease time + * + * Input: + * buf: ignored + * size: zero + * + * OR + * + * Input: + * buf: C string containing an unsigned + * integer value representing the new + * NFSv4 lease expiry time + * size: non-zero length of C string in @buf + * Output: + * On success: passed-in buffer filled with '\n'-terminated C + * string containing unsigned integer value of the + * current lease expiry time; + * return code is the size in bytes of the string + * On error: return code is zero or a negative errno value + */ +static ssize_t write_leasetime(struct file *file, char *buf, size_t size) +{ + struct nfsd_net *nn = net_generic(netns(file), nfsd_net_id); + return nfsd4_write_time(file, buf, size, &nn->nfsd4_lease, nn); +} + +/* + * write_gracetime - Set or report current NFSv4 grace period time + * + * As above, but sets the time of the NFSv4 grace period. + * + * Note this should never be set to less than the *previous* + * lease-period time, but we don't try to enforce this. (In the common + * case (a new boot), we don't know what the previous lease time was + * anyway.) + */ +static ssize_t write_gracetime(struct file *file, char *buf, size_t size) +{ + struct nfsd_net *nn = net_generic(netns(file), nfsd_net_id); + return nfsd4_write_time(file, buf, size, &nn->nfsd4_grace, nn); +} + +#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING +static ssize_t __write_recoverydir(struct file *file, char *buf, size_t size, + struct nfsd_net *nn) +{ + char *mesg = buf; + char *recdir; + int len, status; + + if (size > 0) { + if (nn->nfsd_serv) + return -EBUSY; + if (size > PATH_MAX || buf[size-1] != '\n') + return -EINVAL; + buf[size-1] = 0; + + recdir = mesg; + len = qword_get(&mesg, recdir, size); + if (len <= 0) + return -EINVAL; + trace_nfsd_ctl_recoverydir(netns(file), recdir); + + status = nfs4_reset_recoverydir(recdir); + if (status) + return status; + } + + return scnprintf(buf, SIMPLE_TRANSACTION_LIMIT, "%s\n", + nfs4_recoverydir()); +} + +/* + * write_recoverydir - Set or report the pathname of the recovery directory + * + * Input: + * buf: ignored + * size: zero + * + * OR + * + * Input: + * buf: C string containing the pathname + * of the directory on a local file + * system containing permanent NFSv4 + * recovery data + * size: non-zero length of C string in @buf + * Output: + * On success: passed-in buffer filled with '\n'-terminated C string + * containing the current recovery pathname setting; + * return code is the size in bytes of the string + * On error: return code is zero or a negative errno value + */ +static ssize_t write_recoverydir(struct file *file, char *buf, size_t size) +{ + ssize_t rv; + struct nfsd_net *nn = net_generic(netns(file), nfsd_net_id); + + mutex_lock(&nfsd_mutex); + rv = __write_recoverydir(file, buf, size, nn); + mutex_unlock(&nfsd_mutex); + return rv; +} +#endif + +/* + * write_v4_end_grace - release grace period for nfsd's v4.x lock manager + * + * Input: + * buf: ignored + * size: zero + * OR + * + * Input: + * buf: any value + * size: non-zero length of C string in @buf + * Output: + * passed-in buffer filled with "Y" or "N" with a newline + * and NULL-terminated C string. This indicates whether + * the grace period has ended in the current net + * namespace. Return code is the size in bytes of the + * string. Writing a string that starts with 'Y', 'y', or + * '1' to the file will end the grace period for nfsd's v4 + * lock manager. + */ +static ssize_t write_v4_end_grace(struct file *file, char *buf, size_t size) +{ + struct nfsd_net *nn = net_generic(netns(file), nfsd_net_id); + + if (size > 0) { + switch(buf[0]) { + case 'Y': + case 'y': + case '1': + if (!nfsd4_force_end_grace(nn)) + return -EBUSY; + trace_nfsd_end_grace(netns(file)); + break; + default: + return -EINVAL; + } + } + + return scnprintf(buf, SIMPLE_TRANSACTION_LIMIT, "%c\n", + test_bit(NFSD_NET_GRACE_ENDED, &nn->flags) ? 'Y' : 'N'); +} + +#endif + +/*----------------------------------------------------------------------------*/ +/* + * populating the filesystem. + */ + +static struct inode *nfsd_get_inode(struct super_block *sb, umode_t mode) +{ + struct inode *inode = new_inode(sb); + if (inode) { + /* Following advice from simple_fill_super documentation: */ + inode->i_ino = iunique(sb, NFSD_MaxReserved); + inode->i_mode = mode; + simple_inode_init_ts(inode); + } + return inode; +} + +static struct dentry *nfsd_mkdir(struct dentry *parent, struct nfsdfs_client *ncl, char *name) +{ + struct inode *dir = parent->d_inode; + struct dentry *dentry; + struct inode *inode; + + inode = nfsd_get_inode(parent->d_sb, S_IFDIR | 0600); + if (!inode) + return ERR_PTR(-ENOMEM); + + dentry = simple_start_creating(parent, name); + if (IS_ERR(dentry)) { + iput(inode); + return dentry; + } + inode->i_fop = &simple_dir_operations; + inode->i_op = &simple_dir_inode_operations; + inc_nlink(inode); + if (ncl) { + inode->i_private = ncl; + kref_get(&ncl->cl_ref); + } + d_make_persistent(dentry, inode); + inc_nlink(dir); + fsnotify_mkdir(dir, dentry); + simple_done_creating(dentry); + return dentry; // borrowed +} + +#if IS_ENABLED(CONFIG_SUNRPC_GSS) +/* + * @content is assumed to be a NUL-terminated string that lives + * longer than the symlink itself. + */ +static void _nfsd_symlink(struct dentry *parent, const char *name, + const char *content) +{ + struct inode *dir = parent->d_inode; + struct inode *inode; + struct dentry *dentry; + + inode = nfsd_get_inode(dir->i_sb, S_IFLNK | 0777); + if (!inode) + return; + + dentry = simple_start_creating(parent, name); + if (IS_ERR(dentry)) { + iput(inode); + return; + } + + inode->i_op = &simple_symlink_inode_operations; + inode->i_link = (char *)content; + inode->i_size = strlen(content); + + d_make_persistent(dentry, inode); + fsnotify_create(dir, dentry); + simple_done_creating(dentry); +} +#else +static inline void _nfsd_symlink(struct dentry *parent, const char *name, + const char *content) +{ +} + +#endif + +static void clear_ncl(struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + struct nfsdfs_client *ncl = inode->i_private; + + spin_lock(&inode->i_lock); + inode->i_private = NULL; + spin_unlock(&inode->i_lock); + kref_put(&ncl->cl_ref, ncl->cl_release); +} + +struct nfsdfs_client *get_nfsdfs_client(struct inode *inode) +{ + struct nfsdfs_client *nc; + + spin_lock(&inode->i_lock); + nc = inode->i_private; + if (nc) + kref_get(&nc->cl_ref); + spin_unlock(&inode->i_lock); + return nc; +} + +/* XXX: cut'n'paste from simple_fill_super; figure out if we could share + * code instead. */ +static int nfsdfs_create_files(struct dentry *root, + const struct tree_descr *files, + struct nfsdfs_client *ncl, + struct dentry **fdentries) +{ + struct inode *dir = d_inode(root); + struct dentry *dentry; + + for (int i = 0; files->name && files->name[0]; i++, files++) { + struct inode *inode = nfsd_get_inode(root->d_sb, + S_IFREG | files->mode); + if (!inode) + return -ENOMEM; + dentry = simple_start_creating(root, files->name); + if (IS_ERR(dentry)) { + iput(inode); + return PTR_ERR(dentry); + } + kref_get(&ncl->cl_ref); + inode->i_fop = files->ops; + inode->i_private = ncl; + d_make_persistent(dentry, inode); + fsnotify_create(dir, dentry); + if (fdentries) + fdentries[i] = dentry; // borrowed + simple_done_creating(dentry); + } + return 0; +} + +/* on success, returns positive number unique to that client. */ +struct dentry *nfsd_client_mkdir(struct nfsd_net *nn, + struct nfsdfs_client *ncl, u32 id, + const struct tree_descr *files, + struct dentry **fdentries) +{ + struct dentry *dentry; + char name[11]; + int ret; + + sprintf(name, "%u", id); + + dentry = nfsd_mkdir(nn->nfsd_client_dir, ncl, name); + if (IS_ERR(dentry)) /* XXX: tossing errors? */ + return NULL; + ret = nfsdfs_create_files(dentry, files, ncl, fdentries); + if (ret) { + nfsd_client_rmdir(dentry); + return NULL; + } + return dentry; +} + +/* Taken from __rpc_rmdir: */ +void nfsd_client_rmdir(struct dentry *dentry) +{ + simple_recursive_removal(dentry, clear_ncl); +} + +static int nfsd_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct nfsd_net *nn = net_generic(current->nsproxy->net_ns, + nfsd_net_id); + struct dentry *dentry; + int ret; + + static const struct tree_descr nfsd_files[] = { + [NFSD_List] = {"exports", &exports_nfsd_operations, S_IRUGO}, + /* Per-export io stats use same ops as exports file */ + [NFSD_Export_Stats] = {"export_stats", &exports_nfsd_operations, S_IRUGO}, + [NFSD_Export_features] = {"export_features", + &export_features_fops, S_IRUGO}, + [NFSD_FO_UnlockIP] = {"unlock_ip", + &transaction_ops, S_IWUSR|S_IRUSR}, + [NFSD_FO_UnlockFS] = {"unlock_filesystem", + &transaction_ops, S_IWUSR|S_IRUSR}, + [NFSD_Fh] = {"filehandle", &transaction_ops, S_IWUSR|S_IRUSR}, + [NFSD_Threads] = {"threads", &transaction_ops, S_IWUSR|S_IRUSR}, + [NFSD_Pool_Threads] = {"pool_threads", &transaction_ops, S_IWUSR|S_IRUSR}, + [NFSD_Pool_Stats] = {"pool_stats", &pool_stats_operations, S_IRUGO}, + [NFSD_Reply_Cache_Stats] = {"reply_cache_stats", + &nfsd_reply_cache_stats_fops, S_IRUGO}, + [NFSD_Versions] = {"versions", &transaction_ops, S_IWUSR|S_IRUSR}, + [NFSD_Ports] = {"portlist", &transaction_ops, S_IWUSR|S_IRUGO}, + [NFSD_MaxBlkSize] = {"max_block_size", &transaction_ops, S_IWUSR|S_IRUGO}, + [NFSD_Filecache] = {"filecache", &nfsd_file_cache_stats_fops, S_IRUGO}, +#ifdef CONFIG_NFSD_V4 + [NFSD_Leasetime] = {"nfsv4leasetime", &transaction_ops, S_IWUSR|S_IRUSR}, + [NFSD_Gracetime] = {"nfsv4gracetime", &transaction_ops, S_IWUSR|S_IRUSR}, +#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING + [NFSD_RecoveryDir] = {"nfsv4recoverydir", &transaction_ops, S_IWUSR|S_IRUSR}, +#endif + [NFSD_V4EndGrace] = {"v4_end_grace", &transaction_ops, S_IWUSR|S_IRUGO}, +#endif + /* last one */ {""} + }; + + ret = simple_fill_super(sb, 0x6e667364, nfsd_files); + if (ret) + return ret; + _nfsd_symlink(sb->s_root, "supported_krb5_enctypes", + "/proc/net/rpc/gss_krb5_enctypes"); + dentry = nfsd_mkdir(sb->s_root, NULL, "clients"); + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + nn->nfsd_client_dir = dentry; + return 0; +} + +static int nfsd_fs_get_tree(struct fs_context *fc) +{ + return get_tree_keyed(fc, nfsd_fill_super, get_net(fc->net_ns)); +} + +static void nfsd_fs_free_fc(struct fs_context *fc) +{ + if (fc->s_fs_info) + put_net(fc->s_fs_info); +} + +static const struct fs_context_operations nfsd_fs_context_ops = { + .free = nfsd_fs_free_fc, + .get_tree = nfsd_fs_get_tree, +}; + +static int nfsd_init_fs_context(struct fs_context *fc) +{ + put_user_ns(fc->user_ns); + fc->user_ns = get_user_ns(fc->net_ns->user_ns); + fc->ops = &nfsd_fs_context_ops; + return 0; +} + +static void nfsd_umount(struct super_block *sb) +{ + struct net *net = sb->s_fs_info; + + nfsd_shutdown_threads(net); + + kill_anon_super(sb); + put_net(net); +} + +static struct file_system_type nfsd_fs_type = { + .owner = THIS_MODULE, + .name = "nfsd", + .init_fs_context = nfsd_init_fs_context, + .kill_sb = nfsd_umount, +}; +MODULE_ALIAS_FS("nfsd"); + +#ifdef CONFIG_PROC_FS + +static int exports_proc_open(struct inode *inode, struct file *file) +{ + return exports_net_open(current->nsproxy->net_ns, file); +} + +static const struct proc_ops exports_proc_ops = { + .proc_open = exports_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = exports_release, +}; + +static int create_proc_exports_entry(void) +{ + struct proc_dir_entry *entry; + + entry = proc_mkdir("fs/nfs", NULL); + if (!entry) + return -ENOMEM; + entry = proc_create("exports", 0, entry, &exports_proc_ops); + if (!entry) { + remove_proc_entry("fs/nfs", NULL); + return -ENOMEM; + } + return 0; +} +#else /* CONFIG_PROC_FS */ +static int create_proc_exports_entry(void) +{ + return 0; +} +#endif + +unsigned int nfsd_net_id; + +struct nfsd_genl_rqstp { + struct sockaddr_storage rq_daddr; + struct sockaddr_storage rq_saddr; + unsigned long rq_flags; + ktime_t rq_stime; + __be32 rq_xid; + u32 rq_vers; + u32 rq_prog; + u32 rq_proc; + + /* NFSv4 compound */ + u32 rq_opcnt; + u32 rq_opnum[16]; +}; + +static int nfsd_genl_rpc_status_compose_msg(struct sk_buff *skb, + struct netlink_callback *cb, + struct nfsd_genl_rqstp *genl_rqstp) +{ + void *hdr; + u32 i; + + hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, + &nfsd_nl_family, 0, NFSD_CMD_RPC_STATUS_GET); + if (!hdr) + return -ENOBUFS; + + if (nla_put_be32(skb, NFSD_A_RPC_STATUS_XID, genl_rqstp->rq_xid) || + nla_put_u32(skb, NFSD_A_RPC_STATUS_FLAGS, genl_rqstp->rq_flags) || + nla_put_u32(skb, NFSD_A_RPC_STATUS_PROG, genl_rqstp->rq_prog) || + nla_put_u32(skb, NFSD_A_RPC_STATUS_PROC, genl_rqstp->rq_proc) || + nla_put_u8(skb, NFSD_A_RPC_STATUS_VERSION, genl_rqstp->rq_vers) || + nla_put_s64(skb, NFSD_A_RPC_STATUS_SERVICE_TIME, + ktime_to_us(genl_rqstp->rq_stime), + NFSD_A_RPC_STATUS_PAD)) + goto out_cancel; + + switch (genl_rqstp->rq_saddr.ss_family) { + case AF_INET: { + const struct sockaddr_in *s_in, *d_in; + + s_in = (const struct sockaddr_in *)&genl_rqstp->rq_saddr; + d_in = (const struct sockaddr_in *)&genl_rqstp->rq_daddr; + if (nla_put_in_addr(skb, NFSD_A_RPC_STATUS_SADDR4, + s_in->sin_addr.s_addr) || + nla_put_in_addr(skb, NFSD_A_RPC_STATUS_DADDR4, + d_in->sin_addr.s_addr) || + nla_put_be16(skb, NFSD_A_RPC_STATUS_SPORT, + s_in->sin_port) || + nla_put_be16(skb, NFSD_A_RPC_STATUS_DPORT, + d_in->sin_port)) + goto out_cancel; + break; + } + case AF_INET6: { + const struct sockaddr_in6 *s_in, *d_in; + + s_in = (const struct sockaddr_in6 *)&genl_rqstp->rq_saddr; + d_in = (const struct sockaddr_in6 *)&genl_rqstp->rq_daddr; + if (nla_put_in6_addr(skb, NFSD_A_RPC_STATUS_SADDR6, + &s_in->sin6_addr) || + nla_put_in6_addr(skb, NFSD_A_RPC_STATUS_DADDR6, + &d_in->sin6_addr) || + nla_put_be16(skb, NFSD_A_RPC_STATUS_SPORT, + s_in->sin6_port) || + nla_put_be16(skb, NFSD_A_RPC_STATUS_DPORT, + d_in->sin6_port)) + goto out_cancel; + break; + } + } + + for (i = 0; i < genl_rqstp->rq_opcnt; i++) + if (nla_put_u32(skb, NFSD_A_RPC_STATUS_COMPOUND_OPS, + genl_rqstp->rq_opnum[i])) + goto out_cancel; + + genlmsg_end(skb, hdr); + return 0; + +out_cancel: + genlmsg_cancel(skb, hdr); + return -ENOBUFS; +} + +/** + * nfsd_nl_rpc_status_get_dumpit - Handle rpc_status_get dumpit + * @skb: reply buffer + * @cb: netlink metadata and command arguments + * + * Returns the size of the reply or a negative errno. + */ +int nfsd_nl_rpc_status_get_dumpit(struct sk_buff *skb, + struct netlink_callback *cb) +{ + int i, ret, rqstp_index = 0; + struct nfsd_net *nn; + + mutex_lock(&nfsd_mutex); + + nn = net_generic(sock_net(skb->sk), nfsd_net_id); + if (!nn->nfsd_serv) { + ret = -ENODEV; + goto out_unlock; + } + + rcu_read_lock(); + + for (i = 0; i < svc_serv_nrpools(nn->nfsd_serv); i++) { + struct svc_rqst *rqstp; + long thread_skip = 0; + + if (i < cb->args[0]) /* already consumed */ + continue; + + /* + * The saved thread index only applies to the pool the dump + * was resumed in. Subsequent pools must start from thread 0, + * otherwise their first cb->args[1] threads are silently + * skipped. + */ + if (i == cb->args[0]) + thread_skip = cb->args[1]; + + rqstp_index = 0; + list_for_each_entry_rcu(rqstp, + &nn->nfsd_serv->sv_pools[i].sp_all_threads, + rq_all) { + struct nfsd_genl_rqstp genl_rqstp = {}; + unsigned int status_counter; + + if (rqstp_index++ < thread_skip) /* already consumed */ + continue; + /* + * Acquire rq_status_counter before parsing the rqst + * fields. rq_status_counter is set to an odd value in + * order to notify the consumers the rqstp fields are + * meaningful. + */ + status_counter = + smp_load_acquire(&rqstp->rq_status_counter); + if (!(status_counter & 1)) + continue; + + genl_rqstp.rq_xid = rqstp->rq_xid; + genl_rqstp.rq_flags = rqstp->rq_flags; + genl_rqstp.rq_vers = rqstp->rq_vers; + genl_rqstp.rq_prog = rqstp->rq_prog; + genl_rqstp.rq_proc = rqstp->rq_proc; + genl_rqstp.rq_stime = rqstp->rq_stime; + genl_rqstp.rq_opcnt = 0; + memcpy(&genl_rqstp.rq_daddr, svc_daddr(rqstp), + sizeof(struct sockaddr_storage)); + memcpy(&genl_rqstp.rq_saddr, svc_addr(rqstp), + sizeof(struct sockaddr_storage)); + +#ifdef CONFIG_NFSD_V4 + if (rqstp->rq_vers == NFS4_VERSION && + rqstp->rq_proc == NFSPROC4_COMPOUND) { + /* NFSv4 compound */ + struct nfsd4_compoundargs *args; + int j; + + args = rqstp->rq_argp; + genl_rqstp.rq_opcnt = min_t(u32, args->opcnt, + ARRAY_SIZE(genl_rqstp.rq_opnum)); + for (j = 0; j < genl_rqstp.rq_opcnt; j++) + genl_rqstp.rq_opnum[j] = + args->ops[j].opnum; + } +#endif /* CONFIG_NFSD_V4 */ + + /* + * Read-side load-load fence: order the field reads + * above before the counter re-read below, mirroring + * the smp_rmb() in the standard seqcount retry. The + * begin-side smp_load_acquire() above pairs with the + * smp_store_release() in nfsd_dispatch(). + */ + smp_rmb(); + if (READ_ONCE(rqstp->rq_status_counter) != status_counter) + continue; + + ret = nfsd_genl_rpc_status_compose_msg(skb, cb, + &genl_rqstp); + if (ret) { + if (skb->len) { + cb->args[0] = i; + cb->args[1] = rqstp_index - 1; + ret = skb->len; + } + goto out; + } + } + } + + cb->args[0] = i; + cb->args[1] = rqstp_index; + ret = skb->len; +out: + rcu_read_unlock(); +out_unlock: + mutex_unlock(&nfsd_mutex); + + return ret; +} + +/** + * nfsd_nl_fh_key_set - helper to copy fh_key from userspace + * @attr: nlattr NFSD_A_SERVER_FH_KEY + * @nn: nfsd_net + * + * Callers should hold nfsd_mutex, returns 0 on success or negative errno. + * Callers must ensure the server is shut down (sv_nrthreads == 0), + * userspace documentation asserts the key may only be set when the server + * is not running. + */ +static int nfsd_nl_fh_key_set(const struct nlattr *attr, struct nfsd_net *nn) +{ + siphash_key_t *fh_key = nn->fh_key; + u64 k0, k1; + bool changed; + + k0 = get_unaligned_le64(nla_data(attr)); + k1 = get_unaligned_le64(nla_data(attr) + 8); + + if (!fh_key) { + fh_key = kmalloc(sizeof(siphash_key_t), GFP_KERNEL); + if (!fh_key) { + trace_nfsd_ctl_fh_key_set(false, -ENOMEM); + return -ENOMEM; + } + nn->fh_key = fh_key; + changed = true; + } else { + changed = fh_key->key[0] != k0 || fh_key->key[1] != k1; + } + + fh_key->key[0] = k0; + fh_key->key[1] = k1; + trace_nfsd_ctl_fh_key_set(changed, 0); + return 0; +} + +/** + * nfsd_nl_threads_set_doit - set the number of running threads + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Return 0 on success or a negative errno. + */ +int nfsd_nl_threads_set_doit(struct sk_buff *skb, struct genl_info *info) +{ + int *nthreads, nrpools = 0, i, ret = -EOPNOTSUPP, rem; + struct net *net = genl_info_net(info); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + const struct nlattr *attr; + const char *scope = NULL; + + if (GENL_REQ_ATTR_CHECK(info, NFSD_A_SERVER_THREADS)) + return -EINVAL; + + /* count number of SERVER_THREADS values */ + nlmsg_for_each_attr_type(attr, NFSD_A_SERVER_THREADS, info->nlhdr, + GENL_HDRLEN, rem) + nrpools++; + + mutex_lock(&nfsd_mutex); + + nthreads = kzalloc_objs(int, nrpools); + if (!nthreads) { + ret = -ENOMEM; + goto out_unlock; + } + + i = 0; + nlmsg_for_each_attr_type(attr, NFSD_A_SERVER_THREADS, info->nlhdr, + GENL_HDRLEN, rem) { + nthreads[i++] = nla_get_u32(attr); + if (i >= nrpools) + break; + } + + if (info->attrs[NFSD_A_SERVER_GRACETIME] || + info->attrs[NFSD_A_SERVER_LEASETIME] || + info->attrs[NFSD_A_SERVER_SCOPE] || + info->attrs[NFSD_A_SERVER_FH_KEY]) { + ret = -EBUSY; + if (nn->nfsd_serv && nn->nfsd_serv->sv_nrthreads) + goto out_unlock; + + ret = -EINVAL; + attr = info->attrs[NFSD_A_SERVER_GRACETIME]; + if (attr) { + u32 gracetime = nla_get_u32(attr); + + if (gracetime < 10 || gracetime > 3600) + goto out_unlock; + + nn->nfsd4_grace = gracetime; + } + + attr = info->attrs[NFSD_A_SERVER_LEASETIME]; + if (attr) { + u32 leasetime = nla_get_u32(attr); + + if (leasetime < 10 || leasetime > 3600) + goto out_unlock; + + nn->nfsd4_lease = leasetime; + } + + attr = info->attrs[NFSD_A_SERVER_SCOPE]; + if (attr) + scope = nla_data(attr); + + attr = info->attrs[NFSD_A_SERVER_FH_KEY]; + if (attr) { + ret = nfsd_nl_fh_key_set(attr, nn); + if (ret) + goto out_unlock; + } + } + + attr = info->attrs[NFSD_A_SERVER_MIN_THREADS]; + if (attr) + nn->min_threads = nla_get_u32(attr); + + ret = nfsd_svc(nrpools, nthreads, net, current_cred(), scope); + if (ret > 0) + ret = 0; +out_unlock: + mutex_unlock(&nfsd_mutex); + kfree(nthreads); + return ret; +} + +/** + * nfsd_nl_threads_get_doit - get the maximum number of running threads + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Return 0 on success or a negative errno. + */ +int nfsd_nl_threads_get_doit(struct sk_buff *skb, struct genl_info *info) +{ + struct net *net = genl_info_net(info); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + void *hdr; + int err; + + skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); + if (!skb) + return -ENOMEM; + + hdr = genlmsg_iput(skb, info); + if (!hdr) { + err = -EMSGSIZE; + goto err_free_msg; + } + + mutex_lock(&nfsd_mutex); + + err = nla_put_u32(skb, NFSD_A_SERVER_GRACETIME, + nn->nfsd4_grace) || + nla_put_u32(skb, NFSD_A_SERVER_LEASETIME, + nn->nfsd4_lease) || + nla_put_u32(skb, NFSD_A_SERVER_MIN_THREADS, + nn->min_threads) || + nla_put_string(skb, NFSD_A_SERVER_SCOPE, + nn->nfsd_name); + if (err) + goto err_unlock; + + if (nn->nfsd_serv) { + int i; + + for (i = 0; i < nfsd_nrpools(net); ++i) { + struct svc_pool *sp = &nn->nfsd_serv->sv_pools[i]; + + err = nla_put_u32(skb, NFSD_A_SERVER_THREADS, + sp->sp_nrthrmax); + if (err) + goto err_unlock; + } + } else { + err = nla_put_u32(skb, NFSD_A_SERVER_THREADS, 0); + if (err) + goto err_unlock; + } + + mutex_unlock(&nfsd_mutex); + + genlmsg_end(skb, hdr); + + return genlmsg_reply(skb, info); + +err_unlock: + mutex_unlock(&nfsd_mutex); +err_free_msg: + nlmsg_free(skb); + + return err; +} + +/** + * nfsd_nl_version_set_doit - set the nfs enabled versions + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Return 0 on success or a negative errno. + */ +int nfsd_nl_version_set_doit(struct sk_buff *skb, struct genl_info *info) +{ + const struct nlattr *attr; + struct nfsd_net *nn; + int i, rem; + + if (GENL_REQ_ATTR_CHECK(info, NFSD_A_SERVER_PROTO_VERSION)) + return -EINVAL; + + mutex_lock(&nfsd_mutex); + + nn = net_generic(genl_info_net(info), nfsd_net_id); + if (nn->nfsd_serv) { + mutex_unlock(&nfsd_mutex); + return -EBUSY; + } + + /* clear current supported versions. */ + nfsd_vers(nn, 2, NFSD_CLEAR); + nfsd_vers(nn, 3, NFSD_CLEAR); + for (i = 0; i <= NFSD_SUPPORTED_MINOR_VERSION; i++) + nfsd_minorversion(nn, i, NFSD_CLEAR); + + nlmsg_for_each_attr_type(attr, NFSD_A_SERVER_PROTO_VERSION, info->nlhdr, + GENL_HDRLEN, rem) { + struct nlattr *tb[NFSD_A_VERSION_MAX + 1]; + u32 major, minor = 0; + bool enabled; + + if (nla_parse_nested(tb, NFSD_A_VERSION_MAX, attr, + nfsd_version_nl_policy, info->extack) < 0) + continue; + + if (!tb[NFSD_A_VERSION_MAJOR]) + continue; + + major = nla_get_u32(tb[NFSD_A_VERSION_MAJOR]); + if (tb[NFSD_A_VERSION_MINOR]) + minor = nla_get_u32(tb[NFSD_A_VERSION_MINOR]); + + enabled = nla_get_flag(tb[NFSD_A_VERSION_ENABLED]); + + switch (major) { + case 4: + nfsd_minorversion(nn, minor, enabled ? NFSD_SET : NFSD_CLEAR); + break; + case 3: + case 2: + if (!minor) + nfsd_vers(nn, major, enabled ? NFSD_SET : NFSD_CLEAR); + break; + default: + break; + } + } + + mutex_unlock(&nfsd_mutex); + + return 0; +} + +/** + * nfsd_nl_version_get_doit - get the enabled status for all supported nfs versions + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Return 0 on success or a negative errno. + */ +int nfsd_nl_version_get_doit(struct sk_buff *skb, struct genl_info *info) +{ + struct nfsd_net *nn; + int i, err; + void *hdr; + + skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); + if (!skb) + return -ENOMEM; + + hdr = genlmsg_iput(skb, info); + if (!hdr) { + err = -EMSGSIZE; + goto err_free_msg; + } + + mutex_lock(&nfsd_mutex); + nn = net_generic(genl_info_net(info), nfsd_net_id); + + for (i = 2; i <= 4; i++) { + int j; + + for (j = 0; j <= NFSD_SUPPORTED_MINOR_VERSION; j++) { + struct nlattr *attr; + + /* Don't record any versions the kernel doesn't have + * compiled in + */ + if (!nfsd_support_version(i)) + continue; + + /* NFSv{2,3} does not support minor numbers */ + if (i < 4 && j) + continue; + + attr = nla_nest_start(skb, + NFSD_A_SERVER_PROTO_VERSION); + if (!attr) { + err = -EINVAL; + goto err_nfsd_unlock; + } + + if (nla_put_u32(skb, NFSD_A_VERSION_MAJOR, i) || + nla_put_u32(skb, NFSD_A_VERSION_MINOR, j)) { + err = -EINVAL; + goto err_nfsd_unlock; + } + + /* Set the enabled flag if the version is enabled */ + if (nfsd_vers(nn, i, NFSD_TEST) && + (i < 4 || nfsd_minorversion(nn, j, NFSD_TEST)) && + nla_put_flag(skb, NFSD_A_VERSION_ENABLED)) { + err = -EINVAL; + goto err_nfsd_unlock; + } + + nla_nest_end(skb, attr); + } + } + + mutex_unlock(&nfsd_mutex); + genlmsg_end(skb, hdr); + + return genlmsg_reply(skb, info); + +err_nfsd_unlock: + mutex_unlock(&nfsd_mutex); +err_free_msg: + nlmsg_free(skb); + + return err; +} + +/** + * nfsd_nl_validate_listeners - sanity-check the listener list from userland + * @info: netlink metadata and command arguments + * + * Walk every NFSD_A_SERVER_SOCK_ADDR attribute and confirm that each entry + * is well-formed: it parses against the policy, carries both an address and + * a transport name, and the address is long enough for its family. Doing + * this up front lets the callers below assume every entry is valid and + * guarantees we make no changes when the request is malformed. + * + * Return: 0 if every entry is valid, or a negative errno otherwise. + */ +static int nfsd_nl_validate_listeners(struct genl_info *info) +{ + const struct nlattr *attr; + int rem; + + nlmsg_for_each_attr_type(attr, NFSD_A_SERVER_SOCK_ADDR, info->nlhdr, + GENL_HDRLEN, rem) { + struct nlattr *tb[NFSD_A_SOCK_MAX + 1]; + struct sockaddr *sa; + int err; + + err = nla_parse_nested(tb, NFSD_A_SOCK_MAX, attr, + nfsd_sock_nl_policy, info->extack); + if (err < 0) + return err; + + if (!tb[NFSD_A_SOCK_ADDR] || !tb[NFSD_A_SOCK_TRANSPORT_NAME]) + return -EINVAL; + + sa = nla_data(tb[NFSD_A_SOCK_ADDR]); + if (nla_len(tb[NFSD_A_SOCK_ADDR]) < sizeof(sa->sa_family)) + return -EINVAL; + + switch (sa->sa_family) { + case AF_INET: + if (nla_len(tb[NFSD_A_SOCK_ADDR]) < + sizeof(struct sockaddr_in)) + return -EINVAL; + break; + case AF_INET6: + if (nla_len(tb[NFSD_A_SOCK_ADDR]) < + sizeof(struct sockaddr_in6)) + return -EINVAL; + break; + default: + return -EAFNOSUPPORT; + } + } + + return 0; +} + +/** + * nfsd_nl_listener_set_doit - set the nfs running sockets + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Return 0 on success or a negative errno. + */ +int nfsd_nl_listener_set_doit(struct sk_buff *skb, struct genl_info *info) +{ + struct net *net = genl_info_net(info); + struct svc_xprt *xprt, *tmp; + const struct nlattr *attr; + struct svc_serv *serv; + LIST_HEAD(permsocks); + struct nfsd_net *nn; + bool delete = false; + int err, rem; + + /* + * Validate the entire listener list before making any changes, so a + * malformed request fails cleanly without creating a serv or touching + * the existing listeners. + */ + err = nfsd_nl_validate_listeners(info); + if (err) + return err; + + mutex_lock(&nfsd_mutex); + + err = nfsd_create_serv(net); + if (err) { + mutex_unlock(&nfsd_mutex); + return err; + } + + nn = net_generic(net, nfsd_net_id); + serv = nn->nfsd_serv; + + spin_lock_bh(&serv->sv_lock); + + /* Move all of the old listener sockets to a temp list */ + list_splice_init(&serv->sv_permsocks, &permsocks); + + /* + * Walk the list of server_socks from userland and move any that match + * back to sv_permsocks + */ + nlmsg_for_each_attr_type(attr, NFSD_A_SERVER_SOCK_ADDR, info->nlhdr, + GENL_HDRLEN, rem) { + struct nlattr *tb[NFSD_A_SOCK_MAX + 1]; + const char *xcl_name; + struct sockaddr *sa; + + /* validated up front in nfsd_nl_validate_listeners() */ + if (nla_parse_nested(tb, NFSD_A_SOCK_MAX, attr, + nfsd_sock_nl_policy, info->extack) < 0) + continue; + + xcl_name = nla_data(tb[NFSD_A_SOCK_TRANSPORT_NAME]); + sa = nla_data(tb[NFSD_A_SOCK_ADDR]); + + /* Put back any matching sockets */ + list_for_each_entry_safe(xprt, tmp, &permsocks, xpt_list) { + /* This shouldn't be possible */ + if (WARN_ON_ONCE(xprt->xpt_net != net)) { + list_move(&xprt->xpt_list, &serv->sv_permsocks); + continue; + } + + /* If everything matches, put it back */ + if (!strcmp(xprt->xpt_class->xcl_name, xcl_name) && + rpc_cmp_addr_port(sa, (struct sockaddr *)&xprt->xpt_local)) { + list_move(&xprt->xpt_list, &serv->sv_permsocks); + break; + } + } + } + + /* + * If there are listener transports remaining on the permsocks list, + * it means we were asked to remove a listener. + */ + if (!list_empty(&permsocks)) { + list_splice_init(&permsocks, &serv->sv_permsocks); + delete = true; + } + spin_unlock_bh(&serv->sv_lock); + + /* Do not remove listeners while there are active threads. */ + if (serv->sv_nrthreads) { + err = -EBUSY; + goto out_unlock_mtx; + } + + /* + * Since we can't delete an arbitrary llist entry, destroy the + * remaining listeners and recreate the list. + */ + if (delete) + svc_xprt_destroy_all(serv, net, false); + + /* walk list of addrs again, open any that still don't exist */ + nlmsg_for_each_attr_type(attr, NFSD_A_SERVER_SOCK_ADDR, info->nlhdr, + GENL_HDRLEN, rem) { + struct nlattr *tb[NFSD_A_SOCK_MAX + 1]; + const char *xcl_name; + struct sockaddr *sa; + int ret; + + /* validated up front in nfsd_nl_validate_listeners() */ + if (nla_parse_nested(tb, NFSD_A_SOCK_MAX, attr, + nfsd_sock_nl_policy, info->extack) < 0) + continue; + + xcl_name = nla_data(tb[NFSD_A_SOCK_TRANSPORT_NAME]); + sa = nla_data(tb[NFSD_A_SOCK_ADDR]); + + xprt = svc_find_listener(serv, xcl_name, net, sa); + if (xprt) { + if (delete) + WARN_ONCE(1, "Transport type=%s already exists\n", + xcl_name); + svc_xprt_put(xprt); + continue; + } + + ret = svc_xprt_create_from_sa(serv, xcl_name, net, sa, 0, + current_cred()); + /* always save the latest error */ + if (ret < 0) + err = ret; + } + + if (!serv->sv_nrthreads && list_empty(&nn->nfsd_serv->sv_permsocks)) + nfsd_destroy_serv(net); + +out_unlock_mtx: + mutex_unlock(&nfsd_mutex); + + return err; +} + +/** + * nfsd_nl_listener_get_doit - get the nfs running listeners + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Return 0 on success or a negative errno. + */ +int nfsd_nl_listener_get_doit(struct sk_buff *skb, struct genl_info *info) +{ + struct svc_xprt *xprt; + struct svc_serv *serv; + struct nfsd_net *nn; + void *hdr; + int err; + + skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); + if (!skb) + return -ENOMEM; + + hdr = genlmsg_iput(skb, info); + if (!hdr) { + err = -EMSGSIZE; + goto err_free_msg; + } + + mutex_lock(&nfsd_mutex); + nn = net_generic(genl_info_net(info), nfsd_net_id); + + /* no nfs server? Just send empty socket list */ + if (!nn->nfsd_serv) + goto out_unlock_mtx; + + serv = nn->nfsd_serv; + spin_lock_bh(&serv->sv_lock); + list_for_each_entry(xprt, &serv->sv_permsocks, xpt_list) { + struct nlattr *attr; + + attr = nla_nest_start(skb, NFSD_A_SERVER_SOCK_ADDR); + if (!attr) { + err = -EINVAL; + goto err_serv_unlock; + } + + if (nla_put_string(skb, NFSD_A_SOCK_TRANSPORT_NAME, + xprt->xpt_class->xcl_name) || + nla_put(skb, NFSD_A_SOCK_ADDR, + sizeof(struct sockaddr_storage), + &xprt->xpt_local)) { + err = -EINVAL; + goto err_serv_unlock; + } + + nla_nest_end(skb, attr); + } + spin_unlock_bh(&serv->sv_lock); +out_unlock_mtx: + mutex_unlock(&nfsd_mutex); + genlmsg_end(skb, hdr); + + return genlmsg_reply(skb, info); + +err_serv_unlock: + spin_unlock_bh(&serv->sv_lock); + mutex_unlock(&nfsd_mutex); +err_free_msg: + nlmsg_free(skb); + + return err; +} + +/** + * nfsd_nl_pool_mode_set_doit - set the number of running threads + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Return 0 on success or a negative errno. + */ +int nfsd_nl_pool_mode_set_doit(struct sk_buff *skb, struct genl_info *info) +{ + const struct nlattr *attr; + + if (GENL_REQ_ATTR_CHECK(info, NFSD_A_POOL_MODE_MODE)) + return -EINVAL; + + attr = info->attrs[NFSD_A_POOL_MODE_MODE]; + return sunrpc_set_pool_mode(nla_data(attr)); +} + +/** + * nfsd_nl_pool_mode_get_doit - get info about pool_mode + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Return 0 on success or a negative errno. + */ +int nfsd_nl_pool_mode_get_doit(struct sk_buff *skb, struct genl_info *info) +{ + struct net *net = genl_info_net(info); + char buf[16]; + void *hdr; + int err; + + if (sunrpc_get_pool_mode(buf, ARRAY_SIZE(buf)) >= ARRAY_SIZE(buf)) + return -ERANGE; + + skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); + if (!skb) + return -ENOMEM; + + err = -EMSGSIZE; + hdr = genlmsg_iput(skb, info); + if (!hdr) + goto err_free_msg; + + err = nla_put_string(skb, NFSD_A_POOL_MODE_MODE, buf) | + nla_put_u32(skb, NFSD_A_POOL_MODE_NPOOLS, nfsd_nrpools(net)); + if (err) + goto err_free_msg; + + genlmsg_end(skb, hdr); + return genlmsg_reply(skb, info); + +err_free_msg: + nlmsg_free(skb); + return err; +} + +/** + * nfsd_nl_cache_flush_doit - flush nfsd caches via netlink + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Flush the svc_export and/or expkey caches. If NFSD_A_CACHE_FLUSH_MASK + * is provided, only flush the caches indicated by the bitmask (bit 0 = + * svc_export, bit 1 = expkey). If omitted, flush both. + * + * Return 0 on success or a negative errno. + */ +int nfsd_nl_cache_flush_doit(struct sk_buff *skb, struct genl_info *info) +{ + struct net *net = genl_info_net(info); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + u32 mask = ~0U; + + if (info->attrs[NFSD_A_CACHE_FLUSH_MASK]) + mask = nla_get_u32(info->attrs[NFSD_A_CACHE_FLUSH_MASK]); + + mutex_lock(&nfsd_mutex); + + if ((mask & NFSD_CACHE_TYPE_SVC_EXPORT) && + nn->svc_export_cache) + cache_purge(nn->svc_export_cache); + + if ((mask & NFSD_CACHE_TYPE_EXPKEY) && + nn->svc_expkey_cache) + cache_purge(nn->svc_expkey_cache); + + mutex_unlock(&nfsd_mutex); + + return 0; +} + +/* Emit a single server-proc-entry nest: { op, count }. */ +static int nfsd_nl_put_proc_entry(struct sk_buff *skb, int attr, + u32 op, u64 count) +{ + struct nlattr *nest; + + nest = nla_nest_start(skb, attr); + if (!nest) + return -EMSGSIZE; + if (nla_put_u32(skb, NFSD_A_SERVER_PROC_ENTRY_OP, op) || + nla_put_u64_64bit(skb, NFSD_A_SERVER_PROC_ENTRY_COUNT, + count, NFSD_A_SERVER_PROC_ENTRY_PAD)) { + nla_nest_cancel(skb, nest); + return -EMSGSIZE; + } + nla_nest_end(skb, nest); + return 0; +} + +/* Emit the scalar server-stats counters. Only ever called on a fresh skb. */ +static int nfsd_nl_server_stats_scalars(struct sk_buff *skb, + struct nfsd_net *nn, + struct svc_stat *statp) +{ + if (nla_put_u64_64bit(skb, NFSD_A_SERVER_STATS_RC_HITS, + percpu_counter_sum_positive(&nn->counter[NFSD_STATS_RC_HITS]), + NFSD_A_SERVER_STATS_PAD) || + nla_put_u64_64bit(skb, NFSD_A_SERVER_STATS_RC_MISSES, + percpu_counter_sum_positive(&nn->counter[NFSD_STATS_RC_MISSES]), + NFSD_A_SERVER_STATS_PAD) || + nla_put_u64_64bit(skb, NFSD_A_SERVER_STATS_RC_NOCACHE, + percpu_counter_sum_positive(&nn->counter[NFSD_STATS_RC_NOCACHE]), + NFSD_A_SERVER_STATS_PAD)) + return -EMSGSIZE; + + if (nla_put_u64_64bit(skb, NFSD_A_SERVER_STATS_FH_STALE, + percpu_counter_sum_positive(&nn->counter[NFSD_STATS_FH_STALE]), + NFSD_A_SERVER_STATS_PAD)) + return -EMSGSIZE; + + if (nla_put_u64_64bit(skb, NFSD_A_SERVER_STATS_IO_READ, + percpu_counter_sum_positive(&nn->counter[NFSD_STATS_IO_READ]), + NFSD_A_SERVER_STATS_PAD) || + nla_put_u64_64bit(skb, NFSD_A_SERVER_STATS_IO_WRITE, + percpu_counter_sum_positive(&nn->counter[NFSD_STATS_IO_WRITE]), + NFSD_A_SERVER_STATS_PAD)) + return -EMSGSIZE; + + if (nla_put_u32(skb, NFSD_A_SERVER_STATS_NETCNT, statp->netcnt) || + nla_put_u32(skb, NFSD_A_SERVER_STATS_NETUDPCNT, statp->netudpcnt) || + nla_put_u32(skb, NFSD_A_SERVER_STATS_NETTCPCNT, statp->nettcpcnt) || + nla_put_u32(skb, NFSD_A_SERVER_STATS_NETTCPCONN, statp->nettcpconn)) + return -EMSGSIZE; + + if (nla_put_u32(skb, NFSD_A_SERVER_STATS_RPCCNT, statp->rpccnt) || + nla_put_u32(skb, NFSD_A_SERVER_STATS_RPCBADFMT, statp->rpcbadfmt) || + nla_put_u32(skb, NFSD_A_SERVER_STATS_RPCBADAUTH, statp->rpcbadauth) || + nla_put_u32(skb, NFSD_A_SERVER_STATS_RPCBADCLNT, statp->rpcbadclnt)) + return -EMSGSIZE; + + return 0; +} + +/* + * Emit per-version procedure counts for one NFS version, resuming at *idx. + * Returns 0 when the version has been fully emitted (or is not present), or + * -EMSGSIZE when @skb filled up, leaving *idx at the entry still to emit. + */ +static int nfsd_nl_server_stats_proc(struct sk_buff *skb, + struct svc_stat *statp, + struct svc_program *prog, + unsigned int ver, int attr, int *idx) +{ + unsigned long __percpu *counts; + unsigned int nproc; + + if (!statp->vs_count || ver >= prog->pg_nvers || + !prog->pg_vers[ver] || !statp->vs_count[ver]) + return 0; + + counts = statp->vs_count[ver]; + nproc = prog->pg_vers[ver]->vs_nproc; + + for (; *idx < nproc; (*idx)++) { + unsigned long count = 0; + int cpu; + + for_each_possible_cpu(cpu) + count += per_cpu(counts[*idx], cpu); + + if (!count) + continue; + if (nfsd_nl_put_proc_entry(skb, attr, *idx, count)) + return -EMSGSIZE; + } + + return 0; +} + +#ifdef CONFIG_NFSD_V4 +/* + * Emit NFSv4 per-operation counts, resuming at *idx. Same return convention + * as nfsd_nl_server_stats_proc(). + */ +static int nfsd_nl_server_stats_nfs4ops(struct sk_buff *skb, + struct nfsd_net *nn, int *idx) +{ + for (; *idx <= LAST_NFS4_OP; (*idx)++) { + u64 cnt = percpu_counter_sum_positive( + &nn->counter[NFSD_STATS_NFS4_OP(*idx)]); + + if (!cnt) + continue; + if (nfsd_nl_put_proc_entry(skb, NFSD_A_SERVER_STATS_PROC4OPS_OPS, + *idx, cnt)) + return -EMSGSIZE; + } + + return 0; +} + +/* + * Emit NFSv4 callback (backchannel) per-operation counts, resuming at *idx, + * which counts from OP_CB_GETATTR. Same return convention as + * nfsd_nl_server_stats_proc(). + */ +static int nfsd_nl_server_stats_cbops(struct sk_buff *skb, + struct nfsd_net *nn, int *idx) +{ + int op; + + for (op = OP_CB_GETATTR + *idx; op <= OP_CB_OFFLOAD; op++, (*idx)++) { + u64 cnt = percpu_counter_sum_positive(&nn->cb_counter[op]); + + if (!cnt) + continue; + if (nfsd_nl_put_proc_entry(skb, NFSD_A_SERVER_STATS_PROC4CB_OPS, + op, cnt)) + return -EMSGSIZE; + } + + return 0; +} +#endif + +/* Sections of the server-stats dump, emitted in order across messages. */ +enum { + NFSD_SERVER_STATS_SCALARS = 0, + NFSD_SERVER_STATS_PROC2, + NFSD_SERVER_STATS_PROC3, + NFSD_SERVER_STATS_PROC4, + NFSD_SERVER_STATS_PROC4CB, + NFSD_SERVER_STATS_PROC4OPS, + NFSD_SERVER_STATS_DONE, +}; + +/** + * nfsd_nl_server_stats_get_dumpit - dump NFS server statistics + * @skb: reply buffer + * @cb: netlink metadata and command arguments + * + * The server-stats object is emitted across one or more netlink messages. + * cb->args[0] tracks the current section and cb->args[1] the entry index + * within it, so a section that does not fit in the current message is resumed + * in the next one. The scalar counters are small and emitted once, in the + * first message; userspace merges the attributes from every message. + * + * Returns the size of the reply or a negative errno. + */ +int nfsd_nl_server_stats_get_dumpit(struct sk_buff *skb, + struct netlink_callback *cb) +{ + struct net *net = sock_net(skb->sk); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct svc_stat *statp = &nn->nfsd_svcstats; + struct svc_program *prog = statp->program; + int section = cb->args[0]; + int idx = cb->args[1]; + void *hdr; + + if (section >= NFSD_SERVER_STATS_DONE) + return 0; + + hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, + cb->nlh->nlmsg_seq, &nfsd_nl_family, + NLM_F_MULTI, NFSD_CMD_SERVER_STATS_GET); + if (!hdr) + return -ENOBUFS; + + /* Scalar stats fit easily and are emitted in the first message. */ + if (section == NFSD_SERVER_STATS_SCALARS) { + if (nfsd_nl_server_stats_scalars(skb, nn, statp)) + goto err_cancel; + section = NFSD_SERVER_STATS_PROC2; + idx = 0; + } + + /* + * Emit as many of the remaining sections as fit. A section returning + * -EMSGSIZE means the message is full: close it and resume from the + * same section/index on the next call with a fresh skb. Each entry is + * small enough to fit in a fresh skb, so forward progress is assured. + */ + while (section < NFSD_SERVER_STATS_DONE) { + int ret = 0; + + switch (section) { + case NFSD_SERVER_STATS_PROC2: + ret = nfsd_nl_server_stats_proc(skb, statp, prog, 2, + NFSD_A_SERVER_STATS_PROC2_OPS, &idx); + break; + case NFSD_SERVER_STATS_PROC3: + ret = nfsd_nl_server_stats_proc(skb, statp, prog, 3, + NFSD_A_SERVER_STATS_PROC3_OPS, &idx); + break; + case NFSD_SERVER_STATS_PROC4: + ret = nfsd_nl_server_stats_proc(skb, statp, prog, 4, + NFSD_A_SERVER_STATS_PROC4_OPS, &idx); + break; +#ifdef CONFIG_NFSD_V4 + case NFSD_SERVER_STATS_PROC4CB: + ret = nfsd_nl_server_stats_cbops(skb, nn, &idx); + break; + case NFSD_SERVER_STATS_PROC4OPS: + ret = nfsd_nl_server_stats_nfs4ops(skb, nn, &idx); + break; +#endif + } + + if (ret == -EMSGSIZE) + goto out; + if (ret) + goto err_cancel; + + section++; + idx = 0; + } + +out: + genlmsg_end(skb, hdr); + cb->args[0] = section; + cb->args[1] = idx; + return skb->len; + +err_cancel: + genlmsg_cancel(skb, hdr); + return -EMSGSIZE; +} + +int nfsd_cache_notify(struct cache_detail *cd, struct cache_head *h, u32 cache_type) +{ + struct genlmsghdr *hdr; + struct sk_buff *msg; + + if (!genl_has_listeners(&nfsd_nl_family, cd->net, NFSD_NLGRP_EXPORTD)) + return -ENOLINK; + + msg = genlmsg_new(nla_total_size(sizeof(u32)), GFP_KERNEL); + if (!msg) + return -ENOMEM; + + hdr = genlmsg_put(msg, 0, 0, &nfsd_nl_family, 0, NFSD_CMD_CACHE_NOTIFY); + if (!hdr) { + nlmsg_free(msg); + return -ENOMEM; + } + + if (nla_put_u32(msg, NFSD_A_CACHE_NOTIFY_CACHE_TYPE, cache_type)) { + nlmsg_free(msg); + return -ENOMEM; + } + + genlmsg_end(msg, hdr); + return genlmsg_multicast_netns(&nfsd_nl_family, cd->net, msg, 0, + NFSD_NLGRP_EXPORTD, GFP_KERNEL); +} + +/** + * nfsd_nl_unlock_ip_doit - release NLM locks held by an IP address + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Return: 0 on success or a negative errno. + */ +int nfsd_nl_unlock_ip_doit(struct sk_buff *skb, struct genl_info *info) +{ + struct sockaddr *sap; + + if (GENL_REQ_ATTR_CHECK(info, NFSD_A_UNLOCK_IP_ADDRESS)) + return -EINVAL; + sap = nla_data(info->attrs[NFSD_A_UNLOCK_IP_ADDRESS]); + switch (sap->sa_family) { + case AF_INET: + if (nla_len(info->attrs[NFSD_A_UNLOCK_IP_ADDRESS]) < + sizeof(struct sockaddr_in)) + return -EINVAL; + break; + case AF_INET6: + if (nla_len(info->attrs[NFSD_A_UNLOCK_IP_ADDRESS]) < + sizeof(struct sockaddr_in6)) + return -EINVAL; + break; + default: + return -EAFNOSUPPORT; + } + /* + * nlmsvc_unlock_all_by_ip() releases matching locks + * across all network namespaces because lockd operates + * a single global instance. + */ + trace_nfsd_ctl_unlock_ip(genl_info_net(info), sap, + svc_addr_len(sap)); + return nlmsvc_unlock_all_by_ip(sap); +} + +/** + * nfsd_nl_unlock_filesystem_doit - revoke NFS state under a filesystem path + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Return: 0 on success or a negative errno. + */ +int nfsd_nl_unlock_filesystem_doit(struct sk_buff *skb, + struct genl_info *info) +{ + struct net *net = genl_info_net(info); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct path path; + int error; + + if (GENL_REQ_ATTR_CHECK(info, NFSD_A_UNLOCK_FILESYSTEM_PATH)) + return -EINVAL; + + trace_nfsd_ctl_unlock_fs(net, + nla_data(info->attrs[NFSD_A_UNLOCK_FILESYSTEM_PATH])); + error = kern_path( + nla_data(info->attrs[NFSD_A_UNLOCK_FILESYSTEM_PATH]), + 0, &path); + if (error) + return error; + + error = nlmsvc_unlock_all_by_sb(path.dentry->d_sb); + + mutex_lock(&nfsd_mutex); + if (test_bit(NFSD_NET_UP, &nn->flags)) { + nfsd4_cancel_copy_by_sb(net, path.dentry->d_sb); + nfsd4_revoke_states(nn, path.dentry->d_sb); + } else { + error = -EINVAL; + } + mutex_unlock(&nfsd_mutex); + + path_put(&path); + return error; +} + +/** + * nfsd_nl_unlock_export_doit - revoke NFSv4 state for an export path + * @skb: reply buffer + * @info: netlink metadata and command arguments + * + * Revokes all NFSv4 state (opens, locks, delegations, layouts) acquired + * through any export of the given path, regardless of which client holds + * the state. Userspace (exportfs -u) sends this after removing the last + * client for a path so the underlying filesystem can be unmounted. + * + * Unlike NFSD_CMD_UNLOCK_FILESYSTEM, which operates at superblock + * granularity, this command revokes only the state associated with + * exports of a specific path. + * + * Return: 0 on success or a negative errno. + */ +int nfsd_nl_unlock_export_doit(struct sk_buff *skb, struct genl_info *info) +{ + struct net *net = genl_info_net(info); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct path path; + int error; + + if (GENL_REQ_ATTR_CHECK(info, NFSD_A_UNLOCK_EXPORT_PATH)) + return -EINVAL; + + trace_nfsd_ctl_unlock_export(net, + nla_data(info->attrs[NFSD_A_UNLOCK_EXPORT_PATH])); + error = kern_path( + nla_data(info->attrs[NFSD_A_UNLOCK_EXPORT_PATH]), + 0, &path); + if (error) + return error; + + mutex_lock(&nfsd_mutex); + if (test_bit(NFSD_NET_UP, &nn->flags)) { + nfsd_file_close_export(net, &path); + nfsd4_revoke_export_states(nn, &path); + } else + error = -EINVAL; + mutex_unlock(&nfsd_mutex); + + path_put(&path); + return error; +} + +/** + * nfsd_net_init - Prepare the nfsd_net portion of a new net namespace + * @net: a freshly-created network namespace + * + * This information stays around as long as the network namespace is + * alive whether or not there is an NFSD instance running in the + * namespace. + * + * Returns zero on success, or a negative errno otherwise. + */ +static __net_init int nfsd_net_init(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + int retval; + int i; + + retval = nfsd_net_cb_init(nn); + if (retval) + return retval; + retval = nfsd_export_init(net); + if (retval) + goto out_export_error; + retval = nfsd_idmap_init(net); + if (retval) + goto out_idmap_error; + retval = percpu_counter_init_many(nn->counter, 0, GFP_KERNEL, + NFSD_STATS_COUNTERS_NUM); + if (retval) + goto out_repcache_error; + +#ifdef CONFIG_NFSD_V4 + retval = percpu_counter_init_many(nn->cb_counter, 0, GFP_KERNEL, + NFSD_STATS_CB_OPS_NUM); + if (retval) + goto out_cb_counter_error; +#endif + + memset(&nn->nfsd_svcstats, 0, sizeof(nn->nfsd_svcstats)); + nn->nfsd_svcstats.program = &nfsd_programs[0]; + retval = svc_stat_alloc_counts(&nn->nfsd_svcstats); + if (retval) + goto out_proc_error; + if (!nfsd_proc_stat_init(net)) { + retval = -ENOMEM; + goto out_svcstats_error; + } + + for (i = 0; i < sizeof(nn->nfsd_versions); i++) + nn->nfsd_versions[i] = nfsd_support_version(i); + for (i = 0; i < sizeof(nn->nfsd4_minorversions); i++) + nn->nfsd4_minorversions[i] = nfsd_support_version(4); + nn->nfsd_info.mutex = &nfsd_mutex; + nn->nfsd_serv = NULL; + nfsd4_init_leases_net(nn); + get_random_bytes(&nn->siphash_key, sizeof(nn->siphash_key)); + seqlock_init(&nn->writeverf_lock); +#if IS_ENABLED(CONFIG_NFS_LOCALIO) + spin_lock_init(&nn->local_clients_lock); + INIT_LIST_HEAD(&nn->local_clients); +#endif + return 0; + +out_svcstats_error: + svc_stat_free_counts(&nn->nfsd_svcstats); +out_proc_error: +#ifdef CONFIG_NFSD_V4 + percpu_counter_destroy_many(nn->cb_counter, NFSD_STATS_CB_OPS_NUM); +out_cb_counter_error: +#endif + percpu_counter_destroy_many(nn->counter, NFSD_STATS_COUNTERS_NUM); +out_repcache_error: + nfsd_idmap_shutdown(net); +out_idmap_error: + nfsd_export_shutdown(net); +out_export_error: + nfsd_net_cb_shutdown(nn); + return retval; +} + +#if IS_ENABLED(CONFIG_NFS_LOCALIO) +/** + * nfsd_net_pre_exit - Disconnect localio clients from net namespace + * @net: a network namespace that is about to be destroyed + * + * This invalidates ->net pointers held by localio clients + * while they can still safely access nn->counter. + */ +static __net_exit void nfsd_net_pre_exit(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + nfs_localio_invalidate_clients(&nn->local_clients, + &nn->local_clients_lock); +} +#endif + +/** + * nfsd_net_exit - Release the nfsd_net portion of a net namespace + * @net: a network namespace that is about to be destroyed + * + */ +static __net_exit void nfsd_net_exit(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + kfree_sensitive(nn->fh_key); + nfsd_net_cb_shutdown(nn); + nfsd_proc_stat_shutdown(net); + svc_stat_free_counts(&nn->nfsd_svcstats); +#ifdef CONFIG_NFSD_V4 + percpu_counter_destroy_many(nn->cb_counter, NFSD_STATS_CB_OPS_NUM); +#endif + percpu_counter_destroy_many(nn->counter, NFSD_STATS_COUNTERS_NUM); + nfsd_idmap_shutdown(net); + nfsd_export_shutdown(net); +} + +static struct pernet_operations nfsd_net_ops = { + .init = nfsd_net_init, +#if IS_ENABLED(CONFIG_NFS_LOCALIO) + .pre_exit = nfsd_net_pre_exit, +#endif + .exit = nfsd_net_exit, + .id = &nfsd_net_id, + .size = sizeof(struct nfsd_net), +}; + +static int __init init_nfsd(void) +{ + int retval; + + retval = nfsd4_init_slabs(); + if (retval) + return retval; + + nfsd_debugfs_init(); + + retval = nfsd4_init_pnfs(); + if (retval) + goto out_free_slabs; + retval = nfsd_drc_slab_create(); + if (retval) + goto out_free_pnfs; + nfsd_lockd_init(); /* lockd->nfsd callbacks */ + retval = register_pernet_subsys(&nfsd_net_ops); + if (retval < 0) + goto out_free_lockd; + retval = register_cld_notifier(); + if (retval) + goto out_free_subsys; + retval = nfsd4_create_laundry_wq(); + if (retval) + goto out_free_cld; + retval = register_filesystem(&nfsd_fs_type); + if (retval) + goto out_free_nfsd4; + retval = genl_register_family(&nfsd_nl_family); + if (retval) + goto out_free_filesystem; + retval = create_proc_exports_entry(); + if (retval) + goto out_free_all; + nfsd_localio_ops_init(); + + return 0; +out_free_all: + genl_unregister_family(&nfsd_nl_family); +out_free_filesystem: + unregister_filesystem(&nfsd_fs_type); +out_free_nfsd4: + nfsd4_destroy_laundry_wq(); +out_free_cld: + unregister_cld_notifier(); +out_free_subsys: + unregister_pernet_subsys(&nfsd_net_ops); +out_free_lockd: + nfsd_lockd_shutdown(); + nfsd_drc_slab_free(); +out_free_pnfs: + nfsd4_exit_pnfs(); +out_free_slabs: + nfsd_debugfs_exit(); + nfsd4_free_slabs(); + return retval; +} + +static void __exit exit_nfsd(void) +{ + remove_proc_entry("fs/nfs/exports", NULL); + remove_proc_entry("fs/nfs", NULL); + genl_unregister_family(&nfsd_nl_family); + unregister_filesystem(&nfsd_fs_type); + nfsd4_destroy_laundry_wq(); + unregister_cld_notifier(); + unregister_pernet_subsys(&nfsd_net_ops); + nfsd_drc_slab_free(); + nfsd_lockd_shutdown(); + nfsd4_exit_pnfs(); + nfsd_debugfs_exit(); + nfsd4_free_slabs(); +} + +MODULE_AUTHOR("Olaf Kirch "); +MODULE_DESCRIPTION("In-kernel NFS server"); +MODULE_LICENSE("GPL"); +module_init(init_nfsd) +module_exit(exit_nfsd) diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h new file mode 100644 index 000000000..76a69d9a4 --- /dev/null +++ b/fs/nfsd/nfsd.h @@ -0,0 +1,392 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Hodge-podge collection of knfsd-related stuff. + * I will sort this out later. + * + * Copyright (C) 1995-1997 Olaf Kirch + */ + +#ifndef LINUX_NFSD_NFSD_H +#define LINUX_NFSD_NFSD_H + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +#undef ifdebug +#ifdef CONFIG_SUNRPC_DEBUG +# define ifdebug(flag) if (nfsd_debug & NFSDDBG_##flag) +#else +# define ifdebug(flag) if (0) +#endif + +/* + * nfsd version + */ +#define NFSD_MINVERS 2 +#define NFSD_MAXVERS 4 +#define NFSD_SUPPORTED_MINOR_VERSION 2 +bool nfsd_support_version(int vers); + +/* + * Default and maximum payload size (NFS READ or WRITE), in bytes. + * The maximum is an implementation limit. + */ +enum { + NFSSVC_DEFBLKSIZE = 4 * 1024 * 1024, + NFSSVC_MAXBLKSIZE = RPCSVC_MAXPAYLOAD, +}; + +/* Maximum number of operations per session compound */ +#define NFSD_MAX_OPS_PER_COMPOUND 200 + +extern struct svc_program nfsd_programs[]; +extern const struct svc_version nfsd_version2, nfsd_version3, nfsd_version4; +extern struct mutex nfsd_mutex; +extern atomic_t nfsd_th_cnt; /* number of available threads */ + +extern const struct seq_operations nfs_exports_op; + +struct nfsd_thread_local_info { + struct nfs4_client **ntli_lease_breaker; + int ntli_cachetype; +}; + +/* + * Common void argument and result helpers + */ +struct nfsd_voidargs { }; +struct nfsd_voidres { }; +bool nfssvc_decode_voidarg(struct svc_rqst *rqstp, + struct xdr_stream *xdr); +bool nfssvc_encode_voidres(struct svc_rqst *rqstp, + struct xdr_stream *xdr); + +/* + * Function prototypes. + */ +int nfsd_svc(int n, int *nservers, struct net *net, + const struct cred *cred, const char *scope); +int nfsd_dispatch(struct svc_rqst *rqstp); + +int nfsd_nrthreads(struct net *); +int nfsd_nrpools(struct net *); +int nfsd_get_nrthreads(int n, int *, struct net *); +int nfsd_set_nrthreads(int n, int *, struct net *); +void nfsd_shutdown_threads(struct net *net); + +struct svc_rqst *nfsd_current_rqst(void); + +struct nfsdfs_client { + struct kref cl_ref; + void (*cl_release)(struct kref *kref); +}; + +struct nfsd_net; + +struct nfsdfs_client *get_nfsdfs_client(struct inode *); +struct dentry *nfsd_client_mkdir(struct nfsd_net *nn, + struct nfsdfs_client *ncl, u32 id, + const struct tree_descr *, + struct dentry **fdentries); +void nfsd_client_rmdir(struct dentry *dentry); +int nfsd_cache_notify(struct cache_detail *cd, struct cache_head *h, u32 cache_type); + +#if defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL) +#ifdef CONFIG_NFSD_V2_ACL +extern const struct svc_version nfsd_acl_version2; +#else +#define nfsd_acl_version2 NULL +#endif +#ifdef CONFIG_NFSD_V3_ACL +extern const struct svc_version nfsd_acl_version3; +#else +#define nfsd_acl_version3 NULL +#endif +#endif + +#if IS_ENABLED(CONFIG_NFS_LOCALIO) +extern const struct svc_version localio_version1; +#endif + +enum vers_op {NFSD_SET, NFSD_CLEAR, NFSD_TEST, NFSD_AVAIL }; +int nfsd_vers(struct nfsd_net *nn, int vers, enum vers_op change); +int nfsd_minorversion(struct nfsd_net *nn, u32 minorversion, enum vers_op change); +void nfsd_reset_versions(struct nfsd_net *nn); +int nfsd_create_serv(struct net *net); +void nfsd_destroy_serv(struct net *net); + +#ifdef CONFIG_DEBUG_FS +void nfsd_debugfs_init(void); +void nfsd_debugfs_exit(void); +#else +static inline void nfsd_debugfs_init(void) {} +static inline void nfsd_debugfs_exit(void) {} +#endif + +extern bool nfsd_disable_splice_read __read_mostly; +extern bool nfsd_delegts_enabled __read_mostly; + +enum { + /* Any new NFSD_IO enum value must be added at the end */ + NFSD_IO_BUFFERED, + NFSD_IO_DONTCACHE, + NFSD_IO_DIRECT, +}; + +extern u64 nfsd_io_cache_read __read_mostly; +extern u64 nfsd_io_cache_write __read_mostly; + +extern int nfsd_max_blksize; + +static inline int nfsd_v4client(struct svc_rqst *rq) +{ + return rq && rq->rq_prog == NFS_PROGRAM && rq->rq_vers == 4; +} + +/* + * NFSv4 State + */ +#ifdef CONFIG_NFSD_V4 +extern unsigned long max_delegations; +int nfsd4_init_slabs(void); +void nfsd4_free_slabs(void); +int nfs4_state_start(void); +int nfs4_state_start_net(struct net *net); +void nfs4_state_shutdown(void); +void nfs4_state_shutdown_net(struct net *net); +int nfs4_reset_recoverydir(char *recdir); +char * nfs4_recoverydir(void); +bool nfsd4_spo_must_allow(struct svc_rqst *rqstp); +int nfsd4_create_laundry_wq(void); +void nfsd4_destroy_laundry_wq(void); +bool nfsd_wait_for_delegreturn(struct svc_rqst *rqstp, struct inode *inode); +#else +static inline int nfsd4_init_slabs(void) { return 0; } +static inline void nfsd4_free_slabs(void) { } +static inline int nfs4_state_start(void) { return 0; } +static inline int nfs4_state_start_net(struct net *net) { return 0; } +static inline void nfs4_state_shutdown(void) { } +static inline void nfs4_state_shutdown_net(struct net *net) { } +static inline int nfs4_reset_recoverydir(char *recdir) { return 0; } +static inline char * nfs4_recoverydir(void) {return NULL; } +static inline bool nfsd4_spo_must_allow(struct svc_rqst *rqstp) +{ + return false; +} +static inline int nfsd4_create_laundry_wq(void) { return 0; }; +static inline void nfsd4_destroy_laundry_wq(void) {}; +static inline bool nfsd_wait_for_delegreturn(struct svc_rqst *rqstp, + struct inode *inode) +{ + return false; +} +#endif + +/* + * lockd binding + */ +void nfsd_lockd_init(void); +void nfsd_lockd_shutdown(void); + + +/* + * These macros provide pre-xdr'ed values for faster operation. + */ +#define nfs_ok cpu_to_be32(NFS_OK) +#define nfserr_perm cpu_to_be32(NFSERR_PERM) +#define nfserr_noent cpu_to_be32(NFSERR_NOENT) +#define nfserr_io cpu_to_be32(NFSERR_IO) +#define nfserr_nxio cpu_to_be32(NFSERR_NXIO) +#define nfserr_acces cpu_to_be32(NFSERR_ACCES) +#define nfserr_exist cpu_to_be32(NFSERR_EXIST) +#define nfserr_xdev cpu_to_be32(NFSERR_XDEV) +#define nfserr_nodev cpu_to_be32(NFSERR_NODEV) +#define nfserr_notdir cpu_to_be32(NFSERR_NOTDIR) +#define nfserr_isdir cpu_to_be32(NFSERR_ISDIR) +#define nfserr_inval cpu_to_be32(NFSERR_INVAL) +#define nfserr_fbig cpu_to_be32(NFSERR_FBIG) +#define nfserr_nospc cpu_to_be32(NFSERR_NOSPC) +#define nfserr_rofs cpu_to_be32(NFSERR_ROFS) +#define nfserr_mlink cpu_to_be32(NFSERR_MLINK) +#define nfserr_nametoolong cpu_to_be32(NFSERR_NAMETOOLONG) +#define nfserr_notempty cpu_to_be32(NFSERR_NOTEMPTY) +#define nfserr_dquot cpu_to_be32(NFSERR_DQUOT) +#define nfserr_stale cpu_to_be32(NFSERR_STALE) +#define nfserr_remote cpu_to_be32(NFSERR_REMOTE) +#define nfserr_wflush cpu_to_be32(NFSERR_WFLUSH) +#define nfserr_badhandle cpu_to_be32(NFSERR_BADHANDLE) +#define nfserr_notsync cpu_to_be32(NFSERR_NOT_SYNC) +#define nfserr_badcookie cpu_to_be32(NFSERR_BAD_COOKIE) +#define nfserr_notsupp cpu_to_be32(NFSERR_NOTSUPP) +#define nfserr_toosmall cpu_to_be32(NFSERR_TOOSMALL) +#define nfserr_serverfault cpu_to_be32(NFSERR_SERVERFAULT) +#define nfserr_badtype cpu_to_be32(NFSERR_BADTYPE) +#define nfserr_jukebox cpu_to_be32(NFSERR_JUKEBOX) +#define nfserr_denied cpu_to_be32(NFSERR_DENIED) +#define nfserr_deadlock cpu_to_be32(NFSERR_DEADLOCK) +#define nfserr_expired cpu_to_be32(NFSERR_EXPIRED) +#define nfserr_bad_cookie cpu_to_be32(NFSERR_BAD_COOKIE) +#define nfserr_same cpu_to_be32(NFSERR_SAME) +#define nfserr_clid_inuse cpu_to_be32(NFSERR_CLID_INUSE) +#define nfserr_stale_clientid cpu_to_be32(NFSERR_STALE_CLIENTID) +#define nfserr_resource cpu_to_be32(NFSERR_RESOURCE) +#define nfserr_moved cpu_to_be32(NFSERR_MOVED) +#define nfserr_nofilehandle cpu_to_be32(NFSERR_NOFILEHANDLE) +#define nfserr_minor_vers_mismatch cpu_to_be32(NFSERR_MINOR_VERS_MISMATCH) +#define nfserr_share_denied cpu_to_be32(NFSERR_SHARE_DENIED) +#define nfserr_stale_stateid cpu_to_be32(NFSERR_STALE_STATEID) +#define nfserr_old_stateid cpu_to_be32(NFSERR_OLD_STATEID) +#define nfserr_bad_stateid cpu_to_be32(NFSERR_BAD_STATEID) +#define nfserr_bad_seqid cpu_to_be32(NFSERR_BAD_SEQID) +#define nfserr_symlink cpu_to_be32(NFSERR_SYMLINK) +#define nfserr_not_same cpu_to_be32(NFSERR_NOT_SAME) +#define nfserr_lock_range cpu_to_be32(NFSERR_LOCK_RANGE) +#define nfserr_restorefh cpu_to_be32(NFSERR_RESTOREFH) +#define nfserr_attrnotsupp cpu_to_be32(NFSERR_ATTRNOTSUPP) +#define nfserr_bad_xdr cpu_to_be32(NFSERR_BAD_XDR) +#define nfserr_openmode cpu_to_be32(NFSERR_OPENMODE) +#define nfserr_badowner cpu_to_be32(NFSERR_BADOWNER) +#define nfserr_locks_held cpu_to_be32(NFSERR_LOCKS_HELD) +#define nfserr_op_illegal cpu_to_be32(NFSERR_OP_ILLEGAL) +#define nfserr_grace cpu_to_be32(NFSERR_GRACE) +#define nfserr_no_grace cpu_to_be32(NFSERR_NO_GRACE) +#define nfserr_reclaim_bad cpu_to_be32(NFSERR_RECLAIM_BAD) +#define nfserr_badname cpu_to_be32(NFSERR_BADNAME) +#define nfserr_admin_revoked cpu_to_be32(NFS4ERR_ADMIN_REVOKED) +#define nfserr_cb_path_down cpu_to_be32(NFSERR_CB_PATH_DOWN) +#define nfserr_locked cpu_to_be32(NFSERR_LOCKED) +#define nfserr_wrongsec cpu_to_be32(NFSERR_WRONGSEC) +#define nfserr_delay cpu_to_be32(NFS4ERR_DELAY) +#define nfserr_badiomode cpu_to_be32(NFS4ERR_BADIOMODE) +#define nfserr_badlayout cpu_to_be32(NFS4ERR_BADLAYOUT) +#define nfserr_bad_session_digest cpu_to_be32(NFS4ERR_BAD_SESSION_DIGEST) +#define nfserr_badsession cpu_to_be32(NFS4ERR_BADSESSION) +#define nfserr_badslot cpu_to_be32(NFS4ERR_BADSLOT) +#define nfserr_complete_already cpu_to_be32(NFS4ERR_COMPLETE_ALREADY) +#define nfserr_conn_not_bound_to_session cpu_to_be32(NFS4ERR_CONN_NOT_BOUND_TO_SESSION) +#define nfserr_deleg_already_wanted cpu_to_be32(NFS4ERR_DELEG_ALREADY_WANTED) +#define nfserr_back_chan_busy cpu_to_be32(NFS4ERR_BACK_CHAN_BUSY) +#define nfserr_layouttrylater cpu_to_be32(NFS4ERR_LAYOUTTRYLATER) +#define nfserr_layoutunavailable cpu_to_be32(NFS4ERR_LAYOUTUNAVAILABLE) +#define nfserr_nomatching_layout cpu_to_be32(NFS4ERR_NOMATCHING_LAYOUT) +#define nfserr_recallconflict cpu_to_be32(NFS4ERR_RECALLCONFLICT) +#define nfserr_unknown_layouttype cpu_to_be32(NFS4ERR_UNKNOWN_LAYOUTTYPE) +#define nfserr_seq_misordered cpu_to_be32(NFS4ERR_SEQ_MISORDERED) +#define nfserr_sequence_pos cpu_to_be32(NFS4ERR_SEQUENCE_POS) +#define nfserr_req_too_big cpu_to_be32(NFS4ERR_REQ_TOO_BIG) +#define nfserr_rep_too_big cpu_to_be32(NFS4ERR_REP_TOO_BIG) +#define nfserr_rep_too_big_to_cache cpu_to_be32(NFS4ERR_REP_TOO_BIG_TO_CACHE) +#define nfserr_retry_uncached_rep cpu_to_be32(NFS4ERR_RETRY_UNCACHED_REP) +#define nfserr_unsafe_compound cpu_to_be32(NFS4ERR_UNSAFE_COMPOUND) +#define nfserr_too_many_ops cpu_to_be32(NFS4ERR_TOO_MANY_OPS) +#define nfserr_op_not_in_session cpu_to_be32(NFS4ERR_OP_NOT_IN_SESSION) +#define nfserr_hash_alg_unsupp cpu_to_be32(NFS4ERR_HASH_ALG_UNSUPP) +#define nfserr_clientid_busy cpu_to_be32(NFS4ERR_CLIENTID_BUSY) +#define nfserr_pnfs_io_hole cpu_to_be32(NFS4ERR_PNFS_IO_HOLE) +#define nfserr_seq_false_retry cpu_to_be32(NFS4ERR_SEQ_FALSE_RETRY) +#define nfserr_bad_high_slot cpu_to_be32(NFS4ERR_BAD_HIGH_SLOT) +#define nfserr_deadsession cpu_to_be32(NFS4ERR_DEADSESSION) +#define nfserr_encr_alg_unsupp cpu_to_be32(NFS4ERR_ENCR_ALG_UNSUPP) +#define nfserr_pnfs_no_layout cpu_to_be32(NFS4ERR_PNFS_NO_LAYOUT) +#define nfserr_not_only_op cpu_to_be32(NFS4ERR_NOT_ONLY_OP) +#define nfserr_wrong_cred cpu_to_be32(NFS4ERR_WRONG_CRED) +#define nfserr_wrong_type cpu_to_be32(NFS4ERR_WRONG_TYPE) +#define nfserr_dirdeleg_unavail cpu_to_be32(NFS4ERR_DIRDELEG_UNAVAIL) +#define nfserr_reject_deleg cpu_to_be32(NFS4ERR_REJECT_DELEG) +#define nfserr_returnconflict cpu_to_be32(NFS4ERR_RETURNCONFLICT) +#define nfserr_deleg_revoked cpu_to_be32(NFS4ERR_DELEG_REVOKED) +#define nfserr_partner_notsupp cpu_to_be32(NFS4ERR_PARTNER_NOTSUPP) +#define nfserr_partner_no_auth cpu_to_be32(NFS4ERR_PARTNER_NO_AUTH) +#define nfserr_union_notsupp cpu_to_be32(NFS4ERR_UNION_NOTSUPP) +#define nfserr_offload_denied cpu_to_be32(NFS4ERR_OFFLOAD_DENIED) +#define nfserr_wrong_lfs cpu_to_be32(NFS4ERR_WRONG_LFS) +#define nfserr_badlabel cpu_to_be32(NFS4ERR_BADLABEL) +#define nfserr_file_open cpu_to_be32(NFS4ERR_FILE_OPEN) +#define nfserr_xattr2big cpu_to_be32(NFS4ERR_XATTR2BIG) +#define nfserr_noxattr cpu_to_be32(NFS4ERR_NOXATTR) + +/* + * Error codes for internal use. These are based at an impossible + * nfsstat4 value so that, once converted to be32, they cannot conflict + * with any value defined by the protocol (compare the nlm__int__* codes + * in fs/lockd/lockd.h). + */ +enum { +/* end-of-file indicator in readdir */ + NFSERR_EOF = 30000, +#define nfserr_eof cpu_to_be32(NFSERR_EOF) + +/* replay detected */ + NFSERR_REPLAY_ME, +#define nfserr_replay_me cpu_to_be32(NFSERR_REPLAY_ME) + +/* nfs41 replay detected */ + NFSERR_REPLAY_CACHE, +#define nfserr_replay_cache cpu_to_be32(NFSERR_REPLAY_CACHE) + +/* symlink found where dir expected - handled differently to + * other symlink found errors by NFSv3. + */ + NFSERR_SYMLINK_NOT_DIR, +#define nfserr_symlink_not_dir cpu_to_be32(NFSERR_SYMLINK_NOT_DIR) +}; + +#ifdef CONFIG_NFSD_V4 + +/* before processing a COMPOUND operation, we have to check that there + * is enough space in the buffer for XDR encode to succeed. otherwise, + * we might process an operation with side effects, and be unable to + * tell the client that the operation succeeded. + * + * COMPOUND_SLACK_SPACE - this is the minimum bytes of buffer space + * needed to encode an "ordinary" _successful_ operation. (GETATTR, + * READ, READDIR, and READLINK have their own buffer checks.) if we + * fall below this level, we fail the next operation with NFS4ERR_RESOURCE. + * + * COMPOUND_ERR_SLACK_SPACE - this is the minimum bytes of buffer space + * needed to encode an operation which has failed with NFS4ERR_RESOURCE. + * care is taken to ensure that we never fall below this level for any + * reason. + */ +#define COMPOUND_SLACK_SPACE 140 /* OP_GETFH */ +#define COMPOUND_ERR_SLACK_SPACE 16 /* OP_SETATTR */ + +#define NFSD_LAUNDROMAT_MINTIMEOUT 1 /* seconds */ +#define NFSD_COURTESY_CLIENT_TIMEOUT (24 * 60 * 60) /* seconds */ +#define NFSD_CLIENT_MAX_TRIM_PER_RUN 128 +#define NFS4_CLIENTS_PER_GB 1024 +#define NFSD_DELEGRETURN_TIMEOUT (HZ / 34) /* 30ms */ +#define NFSD_CB_GETATTR_TIMEOUT NFSD_DELEGRETURN_TIMEOUT + +extern int nfsd4_is_junction(struct dentry *dentry); +extern int register_cld_notifier(void); +extern void unregister_cld_notifier(void); +#ifdef CONFIG_NFSD_V4_2_INTER_SSC +extern void nfsd4_ssc_init_umount_work(struct nfsd_net *nn); +#endif + +extern void nfsd4_init_leases_net(struct nfsd_net *nn); + +#else /* CONFIG_NFSD_V4 */ +static inline int nfsd4_is_junction(struct dentry *dentry) +{ + return 0; +} + +static inline void nfsd4_init_leases_net(struct nfsd_net *nn) { }; + +#define register_cld_notifier() 0 +#define unregister_cld_notifier() do { } while(0) + +#endif /* CONFIG_NFSD_V4 */ + +#endif /* LINUX_NFSD_NFSD_H */ diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c new file mode 100644 index 000000000..c7c60c35b --- /dev/null +++ b/fs/nfsd/nfsfh.c @@ -0,0 +1,972 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * NFS server file handle treatment. + * + * Copyright (C) 1995, 1996 Olaf Kirch + * Portions Copyright (C) 1999 G. Allen Morris III + * Extensive rewrite by Neil Brown Southern-Spring 1999 + * ... and again Southern-Winter 2001 to support export_operations + */ + +#include + +#include +#include +#include "nfsd.h" +#include "netns.h" +#include "stats.h" +#include "vfs.h" +#include "auth.h" +#include "trace.h" + +#define NFSDDBG_FACILITY NFSDDBG_FH + + +/* + * our acceptability function. + * if NOSUBTREECHECK, accept anything + * if not, require that we can walk up to exp->ex_dentry + * doing some checks on the 'x' bits + */ +static int nfsd_acceptable(void *expv, struct dentry *dentry) +{ + struct svc_export *exp = expv; + int rv; + struct dentry *tdentry; + struct dentry *parent; + + if (exp->ex_flags & NFSEXP_NOSUBTREECHECK) + return 1; + + tdentry = dget(dentry); + while (tdentry != exp->ex_path.dentry && !IS_ROOT(tdentry)) { + /* make sure parents give x permission to user */ + int err; + parent = dget_parent(tdentry); + err = inode_permission(&nop_mnt_idmap, + d_inode(parent), MAY_EXEC); + if (err < 0) { + dput(parent); + break; + } + dput(tdentry); + tdentry = parent; + } + if (tdentry != exp->ex_path.dentry) + dprintk("nfsd_acceptable failed at %p %pd\n", tdentry, tdentry); + rv = (tdentry == exp->ex_path.dentry); + dput(tdentry); + return rv; +} + +/* Type check. The correct error return for type mismatches does not seem to be + * generally agreed upon. SunOS seems to use EISDIR if file isn't S_IFREG; a + * comment in the NFSv3 spec says this is incorrect (implementation notes for + * the write call). + */ +static inline __be32 +nfsd_mode_check(struct dentry *dentry, umode_t requested) +{ + umode_t mode = d_inode(dentry)->i_mode & S_IFMT; + + if (requested == 0) /* the caller doesn't care */ + return nfs_ok; + if (mode == requested) { + if (mode == S_IFDIR && !d_can_lookup(dentry)) + return nfserr_notdir; + return nfs_ok; + } + if (mode == S_IFLNK) { + if (requested == S_IFDIR) + return nfserr_symlink_not_dir; + return nfserr_symlink; + } + if (requested == S_IFDIR) + return nfserr_notdir; + if (mode == S_IFDIR) + return nfserr_isdir; + return nfserr_wrong_type; +} + +static bool nfsd_originating_port_ok(struct svc_rqst *rqstp, + struct svc_cred *cred, + struct svc_export *exp) +{ + if (nfsexp_flags(cred, exp) & NFSEXP_INSECURE_PORT) + return true; + /* We don't require gss requests to use low ports: */ + if (cred->cr_flavor >= RPC_AUTH_GSS) + return true; + return test_bit(RQ_SECURE, &rqstp->rq_flags); +} + +static __be32 nfsd_setuser_and_check_port(struct svc_rqst *rqstp, + struct svc_cred *cred, + struct svc_export *exp) +{ + /* Check if the request originated from a secure port. */ + if (rqstp && !nfsd_originating_port_ok(rqstp, cred, exp)) { + if (IS_ENABLED(CONFIG_SUNRPC_DEBUG)) { + char buf[RPC_MAX_ADDRBUFLEN]; + + dprintk("nfsd: request from insecure port %s!\n", + svc_print_addr(rqstp, buf, sizeof(buf))); + } + return nfserr_perm; + } + + /* Set user creds for this exportpoint */ + return nfserrno(nfsd_setuser(cred, exp)); +} + +static inline __be32 check_pseudo_root(struct dentry *dentry, + struct svc_export *exp) +{ + if (!(exp->ex_flags & NFSEXP_V4ROOT)) + return nfs_ok; + /* + * We're exposing only the directories and symlinks that have to be + * traversed on the way to real exports: + */ + if (unlikely(!d_is_dir(dentry) && + !d_is_symlink(dentry))) + return nfserr_stale; + /* + * A pseudoroot export gives permission to access only one + * single directory; the kernel has to make another upcall + * before granting access to anything else under it: + */ + if (unlikely(dentry != exp->ex_path.dentry)) + return nfserr_stale; + return nfs_ok; +} + +/* Size of a file handle MAC, in 4-octet words */ +#define FH_MAC_WORDS (sizeof(__le64) / 4) + +bool fh_append_mac(struct knfsd_fh *fh, int fh_maxsize, struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + siphash_key_t *fh_key = nn->fh_key; + __le64 hash; + + if (!fh_key) + goto out_no_key; + if (fh->fh_size + sizeof(hash) > fh_maxsize) + goto out_no_space; + + hash = cpu_to_le64(siphash(&fh->fh_raw, fh->fh_size, fh_key)); + memcpy(&fh->fh_raw[fh->fh_size], &hash, sizeof(hash)); + fh->fh_size += sizeof(hash); + return true; + +out_no_key: + pr_warn_ratelimited("NFSD: unable to sign filehandles, fh_key not set.\n"); + return false; + +out_no_space: + pr_warn_ratelimited("NFSD: unable to sign filehandles, fh_size %zu would be greater than fh_maxsize %d.\n", + fh->fh_size + sizeof(hash), fh_maxsize); + return false; +} + +/* + * Verify that the filehandle's MAC was hashed from this filehandle + * given the server's fh_key: + */ +static bool fh_verify_mac(struct svc_fh *fhp, struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct knfsd_fh *fh = &fhp->fh_handle; + siphash_key_t *fh_key = nn->fh_key; + __le64 hash; + + if (!fh_key) { + pr_warn_ratelimited("NFSD: unable to verify signed filehandles, fh_key not set.\n"); + return false; + } + + hash = cpu_to_le64(siphash(&fh->fh_raw, fh->fh_size - sizeof(hash), fh_key)); + return crypto_memneq(&fh->fh_raw[fh->fh_size - sizeof(hash)], + &hash, sizeof(hash)) == 0; +} + +/* + * Use the given filehandle to look up the corresponding export and + * dentry. On success, the results are used to set fh_export and + * fh_dentry. + */ +static __be32 nfsd_set_fh_dentry(struct svc_rqst *rqstp, struct net *net, + struct svc_cred *cred, + struct auth_domain *client, + struct auth_domain *gssclient, + struct svc_fh *fhp) +{ + struct knfsd_fh *fh = &fhp->fh_handle; + struct fid *fid = NULL; + struct svc_export *exp; + struct dentry *dentry; + int fileid_type; + int data_left = fh->fh_size/4; + int len; + __be32 error; + + error = nfserr_badhandle; + if (fh->fh_size == 0) + return nfserr_nofilehandle; + + if (fh->fh_version != 1) + return error; + + if (--data_left < 0) + return error; + if (fh->fh_auth_type != 0) + return error; + len = key_len(fh->fh_fsid_type) / 4; + if (len == 0) + return error; + if (fh->fh_fsid_type == FSID_MAJOR_MINOR) { + u32 *fsid = fh_fsid(fh); + + /* deprecated, convert to type 3 */ + len = key_len(FSID_ENCODE_DEV)/4; + fh->fh_fsid_type = FSID_ENCODE_DEV; + /* + * struct knfsd_fh uses host-endian fields, which are + * sometimes used to hold net-endian values. This + * confuses sparse, so we must use __force here to + * keep it from complaining. + */ + fsid[0] = new_encode_dev(MKDEV(ntohl((__force __be32)fsid[0]), + ntohl((__force __be32)fsid[1]))); + fsid[1] = fsid[2]; + } + data_left -= len; + if (data_left < 0) + return error; + exp = rqst_exp_find(rqstp ? &rqstp->rq_chandle : NULL, + net, client, gssclient, + fh->fh_fsid_type, fh_fsid(fh)); + fid = (struct fid *)(fh_fsid(fh) + len); + + error = nfserr_stale; + if (IS_ERR(exp)) { + trace_nfsd_set_fh_dentry_badexport(rqstp, fhp, PTR_ERR(exp)); + + if (PTR_ERR(exp) == -ENOENT) + return error; + + return nfserrno(PTR_ERR(exp)); + } + + if (exp->ex_flags & NFSEXP_NOSUBTREECHECK) { + /* Elevate privileges so that the lack of 'r' or 'x' + * permission on some parent directory will + * not stop exportfs_decode_fh from being able + * to reconnect a directory into the dentry cache. + * The same problem can affect "SUBTREECHECK" exports, + * but as nfsd_acceptable depends on correct + * access control settings being in effect, we cannot + * fix that case easily. + */ + struct cred *new = prepare_creds(); + if (!new) { + error = nfserrno(-ENOMEM); + goto out; + } + new->cap_effective = + cap_raise_nfsd_set(new->cap_effective, + new->cap_permitted); + put_cred(override_creds(new)); + } else { + error = nfsd_setuser_and_check_port(rqstp, cred, exp); + if (error) + goto out; + } + + /* + * Look up the dentry using the NFS file handle. + */ + fileid_type = fh->fh_fileid_type; + error = nfserr_stale; + + if (fileid_type == FILEID_ROOT) { + /* We don't sign or verify the root, no per-file identity */ + dentry = dget(exp->ex_path.dentry); + } else { + if (exp->ex_flags & NFSEXP_SIGN_FH) { + if (!fh_verify_mac(fhp, net)) { + trace_nfsd_set_fh_dentry_badmac(rqstp, fhp, -ESTALE); + goto out; + } + data_left -= FH_MAC_WORDS; + } + + dentry = exportfs_decode_fh_raw(exp->ex_path.mnt, fid, + data_left, fileid_type, 0, + nfsd_acceptable, exp); + if (IS_ERR_OR_NULL(dentry)) { + trace_nfsd_set_fh_dentry_badhandle(rqstp, fhp, + dentry ? PTR_ERR(dentry) : -ESTALE); + switch (PTR_ERR(dentry)) { + case -ENOMEM: + case -ETIMEDOUT: + break; + default: + dentry = ERR_PTR(-ESTALE); + } + } + } + + error = nfserr_badhandle; + if (dentry == NULL) + goto out; + if (IS_ERR(dentry)) { + if (PTR_ERR(dentry) != -EINVAL) + error = nfserrno(PTR_ERR(dentry)); + goto out; + } + + if (d_is_dir(dentry) && + (dentry->d_flags & DCACHE_DISCONNECTED)) { + printk("nfsd: find_fh_dentry returned a DISCONNECTED directory: %pd2\n", + dentry); + } + + switch (fhp->fh_maxsize) { + case NFS4_FHSIZE: + if (dentry->d_sb->s_export_op->flags & EXPORT_OP_NOATOMIC_ATTR) + fhp->fh_no_atomic_attr = true; + fhp->fh_64bit_cookies = true; + break; + case NFS3_FHSIZE: + if (dentry->d_sb->s_export_op->flags & EXPORT_OP_NOWCC) + fhp->fh_no_wcc = true; + fhp->fh_64bit_cookies = true; + if (exp->ex_flags & NFSEXP_V4ROOT) { + dput(dentry); + goto out; + } + break; + case NFS_FHSIZE: + fhp->fh_no_wcc = true; + if (EX_WGATHER(exp)) + fhp->fh_use_wgather = true; + if (exp->ex_flags & NFSEXP_V4ROOT) { + dput(dentry); + goto out; + } + } + + fhp->fh_dentry = dentry; + fhp->fh_export = exp; + + return 0; +out: + exp_put(exp); + return error; +} + +/** + * __fh_verify - filehandle lookup and access checking + * @rqstp: RPC transaction context, or NULL + * @net: net namespace in which to perform the export lookup + * @cred: RPC user credential + * @client: RPC auth domain + * @gssclient: RPC GSS auth domain, or NULL + * @fhp: filehandle to be verified + * @type: expected type of object pointed to by filehandle + * @access: type of access needed to object + * + * See fh_verify() for further descriptions of @fhp, @type, and @access. + */ +static __be32 +__fh_verify(struct svc_rqst *rqstp, + struct net *net, struct svc_cred *cred, + struct auth_domain *client, + struct auth_domain *gssclient, + struct svc_fh *fhp, umode_t type, int access) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct svc_export *exp = NULL; + bool may_bypass_gss = false; + struct dentry *dentry; + __be32 error; + + if (!fhp->fh_dentry) { + error = nfsd_set_fh_dentry(rqstp, net, cred, client, + gssclient, fhp); + if (error) + goto out; + } + dentry = fhp->fh_dentry; + exp = fhp->fh_export; + + trace_nfsd_fh_verify(rqstp, fhp, type, access); + + /* + * We still have to do all these permission checks, even when + * fh_dentry is already set: + * - fh_verify may be called multiple times with different + * "access" arguments (e.g. nfsd_proc_create calls + * fh_verify(...,NFSD_MAY_EXEC) first, then later (in + * nfsd_create) calls fh_verify(...,NFSD_MAY_CREATE). + * - in the NFSv4 case, the filehandle may have been filled + * in by fh_compose, and given a dentry, but further + * compound operations performed with that filehandle + * still need permissions checks. In the worst case, a + * mountpoint crossing may have changed the export + * options, and we may now need to use a different uid + * (for example, if different id-squashing options are in + * effect on the new filesystem). + */ + error = check_pseudo_root(dentry, exp); + if (error) + goto out; + + error = nfsd_setuser_and_check_port(rqstp, cred, exp); + if (error) + goto out; + + error = nfsd_mode_check(dentry, type); + if (error) + goto out; + + /* + * If rqstp is NULL, this is a LOCALIO request which will only + * ever use a filehandle/credential pair for which access has + * been affirmed (by ACCESS or OPEN NFS requests) over the + * wire. Skip both the xprtsec policy and the security flavor + * checks. + */ + if (!rqstp) + goto check_permissions; + + if ((access & NFSD_MAY_NLM) && (exp->ex_flags & NFSEXP_NOAUTHNLM)) + /* NLM is allowed to fully bypass authentication */ + goto out; + + /* + * NLM is allowed to bypass the xprtsec policy check because lockd + * doesn't support xprtsec. + */ + if (!(access & NFSD_MAY_NLM)) { + error = check_xprtsec_policy(exp, rqstp); + if (error) + goto out; + } + + if (access & NFSD_MAY_BYPASS_GSS) + may_bypass_gss = true; + /* + * Clients may expect to be able to use auth_sys during mount, + * even if they use gss for everything else; see section 2.3.2 + * of rfc 2623. + */ + if (access & NFSD_MAY_BYPASS_GSS_ON_ROOT + && exp->ex_path.dentry == dentry) + may_bypass_gss = true; + + error = check_security_flavor(exp, rqstp, may_bypass_gss); + if (error) + goto out; + + svc_xprt_set_valid(rqstp->rq_xprt); + +check_permissions: + /* Finally, check access permissions. */ + error = nfsd_permission(cred, exp, dentry, access); +out: + trace_nfsd_fh_verify_err(rqstp, fhp, type, access, error); + if (error == nfserr_stale) + nfsd_stats_fh_stale_inc(nn, exp); + return error; +} + +/** + * fh_verify_local - filehandle lookup and access checking + * @net: net namespace in which to perform the export lookup + * @cred: RPC user credential + * @client: RPC auth domain + * @fhp: filehandle to be verified + * @type: expected type of object pointed to by filehandle + * @access: type of access needed to object + * + * This API can be used by callers who do not have an RPC + * transaction context (ie are not running in an nfsd thread). + * + * See fh_verify() for further descriptions of @fhp, @type, and @access. + */ +__be32 +fh_verify_local(struct net *net, struct svc_cred *cred, + struct auth_domain *client, struct svc_fh *fhp, + umode_t type, int access) +{ + return __fh_verify(NULL, net, cred, client, NULL, + fhp, type, access); +} + +/** + * fh_verify - filehandle lookup and access checking + * @rqstp: pointer to current rpc request + * @fhp: filehandle to be verified + * @type: expected type of object pointed to by filehandle + * @access: type of access needed to object + * + * Look up a dentry from the on-the-wire filehandle, check the client's + * access to the export, and set the current task's credentials. + * + * Regardless of success or failure of fh_verify(), fh_put() should be + * called on @fhp when the caller is finished with the filehandle. + * + * fh_verify() may be called multiple times on a given filehandle, for + * example, when processing an NFSv4 compound. The first call will look + * up a dentry using the on-the-wire filehandle. Subsequent calls will + * skip the lookup and just perform the other checks and possibly change + * the current task's credentials. + * + * @type specifies the type of object expected using one of the S_IF* + * constants defined in include/linux/stat.h. The caller may use zero + * to indicate that it doesn't care, or a negative integer to indicate + * that it expects something not of the given type. + * + * @access is formed from the NFSD_MAY_* constants defined in + * fs/nfsd/vfs.h. + */ +__be32 +fh_verify(struct svc_rqst *rqstp, struct svc_fh *fhp, umode_t type, int access) +{ + return __fh_verify(rqstp, SVC_NET(rqstp), &rqstp->rq_cred, + rqstp->rq_client, rqstp->rq_gssclient, + fhp, type, access); +} + +/* + * Compose a file handle for an NFS reply. + * + * Note that when first composed, the dentry may not yet have + * an inode. In this case a call to fh_update should be made + * before the fh goes out on the wire ... + */ +static void _fh_update(struct svc_fh *fhp, struct svc_export *exp, + struct dentry *dentry) +{ + if (dentry != exp->ex_path.dentry) { + struct fid *fid = (struct fid *) + (fh_fsid(&fhp->fh_handle) + fhp->fh_handle.fh_size/4 - 1); + int maxsize = (fhp->fh_maxsize - fhp->fh_handle.fh_size)/4; + int fh_flags = (exp->ex_flags & NFSEXP_NOSUBTREECHECK) ? 0 : + EXPORT_FH_CONNECTABLE; + int fileid_type = + exportfs_encode_fh(dentry, fid, &maxsize, fh_flags); + + fhp->fh_handle.fh_fileid_type = + fileid_type > 0 ? fileid_type : FILEID_INVALID; + fhp->fh_handle.fh_size += maxsize * 4; + + if (exp->ex_flags & NFSEXP_SIGN_FH) + if (!fh_append_mac(&fhp->fh_handle, fhp->fh_maxsize, + exp->cd->net)) + fhp->fh_handle.fh_fileid_type = FILEID_INVALID; + } else { + fhp->fh_handle.fh_fileid_type = FILEID_ROOT; + } +} + +static bool is_root_export(struct svc_export *exp) +{ + return exp->ex_path.dentry == exp->ex_path.dentry->d_sb->s_root; +} + +static struct super_block *exp_sb(struct svc_export *exp) +{ + return exp->ex_path.dentry->d_sb; +} + +static bool fsid_type_ok_for_exp(u8 fsid_type, struct svc_export *exp) +{ + switch (fsid_type) { + case FSID_DEV: + if (!old_valid_dev(exp_sb(exp)->s_dev)) + return false; + fallthrough; + case FSID_MAJOR_MINOR: + case FSID_ENCODE_DEV: + return exp_sb(exp)->s_type->fs_flags & FS_REQUIRES_DEV; + case FSID_NUM: + return exp->ex_flags & NFSEXP_FSID; + case FSID_UUID8: + case FSID_UUID16: + if (!is_root_export(exp)) + return false; + fallthrough; + case FSID_UUID4_INUM: + case FSID_UUID16_INUM: + return exp->ex_uuid != NULL; + } + return true; +} + + +static void set_version_and_fsid_type(struct svc_fh *fhp, struct svc_export *exp, struct svc_fh *ref_fh) +{ + u8 version; + u8 fsid_type; +retry: + version = 1; + if (ref_fh && ref_fh->fh_export == exp) { + version = ref_fh->fh_handle.fh_version; + fsid_type = ref_fh->fh_handle.fh_fsid_type; + + ref_fh = NULL; + + switch (version) { + case 0xca: + fsid_type = FSID_DEV; + break; + case 1: + break; + default: + goto retry; + } + + /* + * As the fsid -> filesystem mapping was guided by + * user-space, there is no guarantee that the filesystem + * actually supports that fsid type. If it doesn't we + * loop around again without ref_fh set. + */ + if (!fsid_type_ok_for_exp(fsid_type, exp)) + goto retry; + } else if (exp->ex_flags & NFSEXP_FSID) { + fsid_type = FSID_NUM; + } else if (exp->ex_uuid) { + if (fhp->fh_maxsize >= 64) { + if (is_root_export(exp)) + fsid_type = FSID_UUID16; + else + fsid_type = FSID_UUID16_INUM; + } else { + if (is_root_export(exp)) + fsid_type = FSID_UUID8; + else + fsid_type = FSID_UUID4_INUM; + } + } else if (!old_valid_dev(exp_sb(exp)->s_dev)) + /* for newer device numbers, we must use a newer fsid format */ + fsid_type = FSID_ENCODE_DEV; + else + fsid_type = FSID_DEV; + fhp->fh_handle.fh_version = version; + if (version) + fhp->fh_handle.fh_fsid_type = fsid_type; +} + +__be32 +fh_compose(struct svc_fh *fhp, struct svc_export *exp, struct dentry *dentry, + struct svc_fh *ref_fh) +{ + /* ref_fh is a reference file handle. + * if it is non-null and for the same filesystem, then we should compose + * a filehandle which is of the same version, where possible. + */ + + struct inode * inode = d_inode(dentry); + dev_t ex_dev = exp_sb(exp)->s_dev; + + dprintk("nfsd: fh_compose(exp %02x:%02x/%llu %pd2, ino=%llu)\n", + MAJOR(ex_dev), MINOR(ex_dev), + d_inode(exp->ex_path.dentry)->i_ino, + dentry, + (inode ? inode->i_ino : 0)); + + /* Choose filehandle version and fsid type based on + * the reference filehandle (if it is in the same export) + * or the export options. + */ + set_version_and_fsid_type(fhp, exp, ref_fh); + + /* If we have a ref_fh, then copy the fh_no_wcc setting from it. */ + fhp->fh_no_wcc = ref_fh ? ref_fh->fh_no_wcc : false; + + if (ref_fh == fhp) + fh_put(ref_fh); + + if (fhp->fh_dentry) { + printk(KERN_ERR "fh_compose: fh %pd2 not initialized!\n", + dentry); + } + if (fhp->fh_maxsize < NFS_FHSIZE) + printk(KERN_ERR "fh_compose: called with maxsize %d! %pd2\n", + fhp->fh_maxsize, + dentry); + + fhp->fh_dentry = dget(dentry); /* our internal copy */ + fhp->fh_export = exp_get(exp); + + fhp->fh_handle.fh_size = + key_len(fhp->fh_handle.fh_fsid_type) + 4; + fhp->fh_handle.fh_auth_type = 0; + + mk_fsid(fhp->fh_handle.fh_fsid_type, + fh_fsid(&fhp->fh_handle), + ex_dev, + d_inode(exp->ex_path.dentry)->i_ino, + exp->ex_fsid, exp->ex_uuid); + + if (inode) + _fh_update(fhp, exp, dentry); + if (fhp->fh_handle.fh_fileid_type == FILEID_INVALID) { + fh_put(fhp); + return nfserr_stale; + } + + return 0; +} + +/* + * Update file handle information after changing a dentry. + * This is only called by nfsd_create, nfsd_create_v3 and nfsd_proc_create + */ +__be32 +fh_update(struct svc_fh *fhp) +{ + struct dentry *dentry; + + if (!fhp->fh_dentry) + goto out_bad; + + dentry = fhp->fh_dentry; + if (d_really_is_negative(dentry)) + goto out_negative; + if (fhp->fh_handle.fh_fileid_type != FILEID_ROOT) + return 0; + + _fh_update(fhp, fhp->fh_export, dentry); + if (fhp->fh_handle.fh_fileid_type == FILEID_INVALID) + return nfserr_stale; + return 0; +out_bad: + printk(KERN_ERR "fh_update: fh not verified!\n"); + return nfserr_serverfault; +out_negative: + printk(KERN_ERR "fh_update: %pd2 still negative!\n", + dentry); + return nfserr_serverfault; +} + +/** + * fh_getattr - Retrieve attributes on a local file + * @fhp: File handle of target file + * @stat: Caller-supplied kstat buffer to be filled in + * + * Returns nfs_ok on success, otherwise an NFS status code is + * returned. + */ +__be32 fh_getattr(const struct svc_fh *fhp, struct kstat *stat) +{ + struct path p = { + .mnt = fhp->fh_export->ex_path.mnt, + .dentry = fhp->fh_dentry, + }; + struct inode *inode = d_inode(p.dentry); + u32 request_mask = STATX_BASIC_STATS; + + if (S_ISREG(inode->i_mode)) + request_mask |= (STATX_DIOALIGN | STATX_DIO_READ_ALIGN); + + if (fhp->fh_maxsize == NFS4_FHSIZE) + request_mask |= (STATX_BTIME | STATX_CHANGE_COOKIE); + + return nfserrno(vfs_getattr(&p, stat, request_mask, + AT_STATX_SYNC_AS_STAT)); +} + +/** + * fh_fill_pre_attrs - Fill in pre-op attributes + * @fhp: file handle to be updated + * + */ +__be32 __must_check fh_fill_pre_attrs(struct svc_fh *fhp) +{ + bool v4 = (fhp->fh_maxsize == NFS4_FHSIZE); + struct kstat stat; + __be32 err; + + if (fhp->fh_no_wcc || fhp->fh_pre_saved) + return nfs_ok; + + err = fh_getattr(fhp, &stat); + if (err) + return err; + + if (v4) + fhp->fh_pre_change = nfsd4_change_attribute(&stat); + + fhp->fh_pre_mtime = stat.mtime; + fhp->fh_pre_ctime = stat.ctime; + fhp->fh_pre_size = stat.size; + fhp->fh_pre_saved = true; + return nfs_ok; +} + +/** + * fh_fill_post_attrs - Fill in post-op attributes + * @fhp: file handle to be updated + * + */ +__be32 fh_fill_post_attrs(struct svc_fh *fhp) +{ + bool v4 = (fhp->fh_maxsize == NFS4_FHSIZE); + __be32 err; + + if (fhp->fh_no_wcc) + return nfs_ok; + + if (fhp->fh_post_saved) + printk("nfsd: inode locked twice during operation.\n"); + + err = fh_getattr(fhp, &fhp->fh_post_attr); + if (err) + return err; + + fhp->fh_post_saved = true; + if (v4) + fhp->fh_post_change = + nfsd4_change_attribute(&fhp->fh_post_attr); + return nfs_ok; +} + +/** + * fh_fill_both_attrs - Fill pre-op and post-op attributes + * @fhp: file handle to be updated + * + * This is used when the directory wasn't changed, but wcc attributes + * are needed anyway. + */ +__be32 __must_check fh_fill_both_attrs(struct svc_fh *fhp) +{ + __be32 err; + + err = fh_fill_post_attrs(fhp); + if (err) + return err; + + fhp->fh_pre_change = fhp->fh_post_change; + fhp->fh_pre_mtime = fhp->fh_post_attr.mtime; + fhp->fh_pre_ctime = fhp->fh_post_attr.ctime; + fhp->fh_pre_size = fhp->fh_post_attr.size; + fhp->fh_pre_saved = true; + return nfs_ok; +} + +/* + * Release a file handle. + */ +void +fh_put(struct svc_fh *fhp) +{ + struct dentry * dentry = fhp->fh_dentry; + struct svc_export * exp = fhp->fh_export; + if (dentry) { + fhp->fh_dentry = NULL; + dput(dentry); + fh_clear_pre_post_attrs(fhp); + } + fh_drop_write(fhp); + if (exp) { + exp_put(exp); + fhp->fh_export = NULL; + } + fhp->fh_no_wcc = false; + return; +} + +/* + * Shorthand for dprintk()'s + */ +char * SVCFH_fmt(struct svc_fh *fhp) +{ + struct knfsd_fh *fh = &fhp->fh_handle; + static char buf[2+1+1+64*3+1]; + + if (fh->fh_size > 64) + return "bad-fh"; + sprintf(buf, "%d: %*ph", fh->fh_size, fh->fh_size, fh->fh_raw); + return buf; +} + +enum fsid_source fsid_source_fh(const struct knfsd_fh *fh, + struct svc_export *exp) +{ + if (fh->fh_version != 1) + return FSIDSOURCE_DEV; + switch (fh->fh_fsid_type) { + case FSID_DEV: + case FSID_ENCODE_DEV: + case FSID_MAJOR_MINOR: + if (exp_sb(exp)->s_type->fs_flags & FS_REQUIRES_DEV) + return FSIDSOURCE_DEV; + break; + case FSID_NUM: + if (exp->ex_flags & NFSEXP_FSID) + return FSIDSOURCE_FSID; + break; + default: + break; + } + /* either a UUID type filehandle, or the filehandle doesn't + * match the export. + */ + if (exp->ex_flags & NFSEXP_FSID) + return FSIDSOURCE_FSID; + if (exp->ex_uuid) + return FSIDSOURCE_UUID; + return FSIDSOURCE_DEV; +} + +enum fsid_source fsid_source(const struct svc_fh *fhp) +{ + return fsid_source_fh(&fhp->fh_handle, fhp->fh_export); +} + +/** + * nfsd4_change_attribute - Generate an NFSv4 change_attribute value + * @stat: inode attributes + * + * Caller must fill in @stat before calling, typically by invoking + * vfs_getattr() with STATX_MODE, STATX_CTIME, and STATX_CHANGE_COOKIE. + * Returns an unsigned 64-bit changeid4 value (RFC 8881 Section 3.2). + * + * We could use i_version alone as the change attribute. However, i_version + * can go backwards on a regular file after an unclean shutdown. On its own + * that doesn't necessarily cause a problem, but if i_version goes backwards + * and then is incremented again it could reuse a value that was previously + * used before boot, and a client who queried the two values might incorrectly + * assume nothing changed. + * + * By using both ctime and the i_version counter we guarantee that as long as + * time doesn't go backwards we never reuse an old value. If the filesystem + * advertises STATX_ATTR_CHANGE_MONOTONIC, then this mitigation is not + * needed. + * + * We only need to do this for regular files as well. For directories, we + * assume that the new change attr is always logged to stable storage in some + * fashion before the results can be seen. + */ +u64 nfsd4_change_attribute(const struct kstat *stat) +{ + u64 chattr; + + if (stat->result_mask & STATX_CHANGE_COOKIE) { + chattr = stat->change_cookie; + if (S_ISREG(stat->mode) && + !(stat->attributes & STATX_ATTR_CHANGE_MONOTONIC)) { + chattr += (u64)stat->ctime.tv_sec << 30; + chattr += stat->ctime.tv_nsec; + } + } else { + chattr = time_to_chattr(&stat->ctime); + } + return chattr; +} diff --git a/fs/nfsd/nfsfh.h b/fs/nfsd/nfsfh.h new file mode 100644 index 000000000..cdeb5eea6 --- /dev/null +++ b/fs/nfsd/nfsfh.h @@ -0,0 +1,342 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 1995, 1996, 1997 Olaf Kirch + * + * This file describes the layout of the file handles as passed + * over the wire. + */ +#ifndef _LINUX_NFSD_NFSFH_H +#define _LINUX_NFSD_NFSFH_H + +#include +#include +#include +#include +#include + +#include "export.h" + +/* + * The file handle starts with a sequence of four-byte words. + * The first word contains a version number (1) and three descriptor bytes + * that tell how the remaining 3 variable length fields should be handled. + * These three bytes are auth_type, fsid_type and fileid_type. + * + * All four-byte values are in host-byte-order. + * + * The auth_type field is deprecated and must be set to 0. + * + * The fsid_type identifies how the filesystem (or export point) is + * encoded. + * Current values: + * 0 - 4 byte device id (ms-2-bytes major, ls-2-bytes minor), 4byte inode number + * NOTE: we cannot use the kdev_t device id value, because kdev_t.h + * says we mustn't. We must break it up and reassemble. + * 1 - 4 byte user specified identifier + * 2 - 4 byte major, 4 byte minor, 4 byte inode number - DEPRECATED + * 3 - 4 byte device id, encoded for user-space, 4 byte inode number + * 4 - 4 byte inode number and 4 byte uuid + * 5 - 8 byte uuid + * 6 - 16 byte uuid + * 7 - 8 byte inode number and 16 byte uuid + * + * The fileid_type identifies how the file within the filesystem is encoded. + * The values for this field are filesystem specific, exccept that + * filesystems must not use the values '0' or '0xff'. 'See enum fid_type' + * in include/linux/exportfs.h for currently registered values. + */ + +struct knfsd_fh { + unsigned int fh_size; /* + * Points to the current size while + * building a new file handle. + */ + u8 fh_raw[NFS4_FHSIZE]; +}; + +#define fh_version fh_raw[0] +#define fh_auth_type fh_raw[1] +#define fh_fsid_type fh_raw[2] +#define fh_fileid_type fh_raw[3] + +static inline u32 *fh_fsid(const struct knfsd_fh *fh) +{ + return (u32 *)&fh->fh_raw[4]; +} + +static inline __u32 ino_t_to_u32(ino_t ino) +{ + return (__u32) ino; +} + +static inline ino_t u32_to_ino_t(__u32 uino) +{ + return (ino_t) uino; +} + +/* + * This is the internal representation of an NFS handle used in knfsd. + * pre_mtime/post_version will be used to support wcc_attr's in NFSv3. + */ +typedef struct svc_fh { + struct knfsd_fh fh_handle; /* FH data */ + int fh_maxsize; /* max size for fh_handle */ + struct dentry * fh_dentry; /* validated dentry */ + struct svc_export * fh_export; /* export pointer */ + + bool fh_want_write; /* remount protection taken */ + bool fh_no_wcc; /* no wcc data needed */ + bool fh_no_atomic_attr; + /* + * wcc data is not atomic with + * operation + */ + bool fh_use_wgather; /* NFSv2 wgather option */ + bool fh_64bit_cookies;/* readdir cookie size */ + int fh_flags; /* FH flags */ + bool fh_post_saved; /* post-op attrs saved */ + bool fh_pre_saved; /* pre-op attrs saved */ + + /* Pre-op attributes saved when inode is locked */ + __u64 fh_pre_size; /* size before operation */ + struct timespec64 fh_pre_mtime; /* mtime before oper */ + struct timespec64 fh_pre_ctime; /* ctime before oper */ + /* + * pre-op nfsv4 change attr: note must check IS_I_VERSION(inode) + * to find out if it is valid. + */ + u64 fh_pre_change; + + /* Post-op attributes saved in fh_fill_post_attrs() */ + struct kstat fh_post_attr; /* full attrs after operation */ + u64 fh_post_change; /* nfsv4 change; see above */ +} svc_fh; +#define NFSD4_FH_FOREIGN (1<<0) +#define SET_FH_FLAG(c, f) ((c)->fh_flags |= (f)) +#define HAS_FH_FLAG(c, f) ((c)->fh_flags & (f)) + +enum nfsd_fsid { + FSID_DEV = 0, + FSID_NUM, + FSID_MAJOR_MINOR, + FSID_ENCODE_DEV, + FSID_UUID4_INUM, + FSID_UUID8, + FSID_UUID16, + FSID_UUID16_INUM, +}; + +enum fsid_source { + FSIDSOURCE_DEV, + FSIDSOURCE_FSID, + FSIDSOURCE_UUID, +}; +extern enum fsid_source fsid_source_fh(const struct knfsd_fh *fh, + struct svc_export *exp); +extern enum fsid_source fsid_source(const struct svc_fh *fhp); + + +/* + * This might look a little large to "inline" but in all calls except + * one, 'vers' is constant so moste of the function disappears. + * + * In some cases the values are considered to be host endian and in + * others, net endian. fsidv is always considered to be u32 as the + * callers don't know which it will be. So we must use __force to keep + * sparse from complaining. Since these values are opaque to the + * client, that shouldn't be a problem. + */ +static inline void mk_fsid(int vers, u32 *fsidv, dev_t dev, ino_t ino, + u32 fsid, unsigned char *uuid) +{ + u32 *up; + switch(vers) { + case FSID_DEV: + fsidv[0] = (__force __u32)htonl((MAJOR(dev)<<16) | + MINOR(dev)); + fsidv[1] = ino_t_to_u32(ino); + break; + case FSID_NUM: + fsidv[0] = fsid; + break; + case FSID_MAJOR_MINOR: + fsidv[0] = (__force __u32)htonl(MAJOR(dev)); + fsidv[1] = (__force __u32)htonl(MINOR(dev)); + fsidv[2] = ino_t_to_u32(ino); + break; + + case FSID_ENCODE_DEV: + fsidv[0] = new_encode_dev(dev); + fsidv[1] = ino_t_to_u32(ino); + break; + + case FSID_UUID4_INUM: + /* 4 byte fsid and inode number */ + up = (u32*)uuid; + fsidv[0] = ino_t_to_u32(ino); + fsidv[1] = up[0] ^ up[1] ^ up[2] ^ up[3]; + break; + + case FSID_UUID8: + /* 8 byte fsid */ + up = (u32*)uuid; + fsidv[0] = up[0] ^ up[2]; + fsidv[1] = up[1] ^ up[3]; + break; + + case FSID_UUID16: + /* 16 byte fsid - NFSv3+ only */ + memcpy(fsidv, uuid, 16); + break; + + case FSID_UUID16_INUM: + /* 8 byte inode and 16 byte fsid */ + *(u64*)fsidv = (u64)ino; + memcpy(fsidv+2, uuid, 16); + break; + default: BUG(); + } +} + +static inline int key_len(int type) +{ + switch(type) { + case FSID_DEV: return 8; + case FSID_NUM: return 4; + case FSID_MAJOR_MINOR: return 12; + case FSID_ENCODE_DEV: return 8; + case FSID_UUID4_INUM: return 8; + case FSID_UUID8: return 8; + case FSID_UUID16: return 16; + case FSID_UUID16_INUM: return 24; + default: return 0; + } +} + +/* + * Shorthand for dprintk()'s + */ +extern char * SVCFH_fmt(struct svc_fh *fhp); + +/* + * Function prototypes + */ +__be32 fh_verify(struct svc_rqst *, struct svc_fh *, umode_t, int); +__be32 fh_verify_local(struct net *, struct svc_cred *, struct auth_domain *, + struct svc_fh *, umode_t, int); +__be32 fh_getattr(const struct svc_fh *fhp, struct kstat *stat); +__be32 fh_compose(struct svc_fh *, struct svc_export *, struct dentry *, struct svc_fh *); +__be32 fh_update(struct svc_fh *); +void fh_put(struct svc_fh *); +bool fh_append_mac(struct knfsd_fh *fh, int fh_maxsize, struct net *net); + +static __inline__ struct svc_fh * +fh_copy(struct svc_fh *dst, const struct svc_fh *src) +{ + WARN_ON(src->fh_dentry); + + *dst = *src; + return dst; +} + +static inline void +fh_copy_shallow(struct knfsd_fh *dst, const struct knfsd_fh *src) +{ + dst->fh_size = src->fh_size; + memcpy(&dst->fh_raw, &src->fh_raw, src->fh_size); +} + +static __inline__ struct svc_fh * +fh_init(struct svc_fh *fhp, int maxsize) +{ + memset(fhp, 0, sizeof(*fhp)); + fhp->fh_maxsize = maxsize; + return fhp; +} + +static inline bool fh_match(const struct knfsd_fh *fh1, + const struct knfsd_fh *fh2) +{ + if (fh1->fh_size != fh2->fh_size) + return false; + if (memcmp(fh1->fh_raw, fh2->fh_raw, fh1->fh_size) != 0) + return false; + return true; +} + +static inline bool fh_fsid_match(const struct knfsd_fh *fh1, + const struct knfsd_fh *fh2) +{ + u32 *fsid1 = fh_fsid(fh1); + u32 *fsid2 = fh_fsid(fh2); + + if (fh1->fh_fsid_type != fh2->fh_fsid_type) + return false; + if (memcmp(fsid1, fsid2, key_len(fh1->fh_fsid_type)) != 0) + return false; + return true; +} + +/** + * fh_want_write - Get write access to an export + * @fhp: File handle of file to be written + * + * Caller must invoke fh_drop_write() when its write operation + * is complete. + * + * Returns 0 if the file handle's export can be written to. Otherwise + * the export is not prepared for updates, and the returned negative + * errno value reflects the reason for the failure. + */ +static inline int fh_want_write(struct svc_fh *fhp) +{ + int ret; + + if (fhp->fh_want_write) + return 0; + ret = mnt_want_write(fhp->fh_export->ex_path.mnt); + if (!ret) + fhp->fh_want_write = true; + return ret; +} + +/** + * fh_drop_write - Release write access on an export + * @fhp: File handle of file on which fh_want_write() was previously called + */ +static inline void fh_drop_write(struct svc_fh *fhp) +{ + if (fhp->fh_want_write) { + fhp->fh_want_write = false; + mnt_drop_write(fhp->fh_export->ex_path.mnt); + } +} + +/** + * knfsd_fh_hash - calculate the crc32 hash for the filehandle + * @fh - pointer to filehandle + * + * returns a crc32 hash for the filehandle that is compatible with + * the one displayed by "wireshark". + */ +static inline u32 knfsd_fh_hash(const struct knfsd_fh *fh) +{ + return ~crc32_le(0xFFFFFFFF, fh->fh_raw, fh->fh_size); +} + +/** + * fh_clear_pre_post_attrs - Reset pre/post attributes + * @fhp: file handle to be updated + * + */ +static inline void fh_clear_pre_post_attrs(struct svc_fh *fhp) +{ + fhp->fh_post_saved = false; + fhp->fh_pre_saved = false; +} + +u64 nfsd4_change_attribute(const struct kstat *stat); +__be32 __must_check fh_fill_pre_attrs(struct svc_fh *fhp); +__be32 fh_fill_post_attrs(struct svc_fh *fhp); +__be32 __must_check fh_fill_both_attrs(struct svc_fh *fhp); +#endif /* _LINUX_NFSD_NFSFH_H */ diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c new file mode 100644 index 000000000..e2b5f8a24 --- /dev/null +++ b/fs/nfsd/nfsproc.c @@ -0,0 +1,854 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Process version 2 NFS requests. + * + * Copyright (C) 1995-1997 Olaf Kirch + */ + +#include + +#include "cache.h" +#include "xdr.h" +#include "vfs.h" +#include "trace.h" + +#define NFSDDBG_FACILITY NFSDDBG_PROC + +static __be32 nfsd_map_status(__be32 status) +{ + switch (status) { + case nfs_ok: + break; + case nfserr_nofilehandle: + case nfserr_badhandle: + status = nfserr_stale; + break; + case nfserr_wrongsec: + case nfserr_xdev: + case nfserr_file_open: + status = nfserr_acces; + break; + case nfserr_symlink_not_dir: + status = nfserr_notdir; + break; + case nfserr_symlink: + case nfserr_wrong_type: + status = nfserr_io; + break; + } + return status; +} + +static __be32 +nfsd_proc_null(struct svc_rqst *rqstp) +{ + return rpc_success; +} + +/* + * Get a file's attributes + * N.B. After this call resp->fh needs an fh_put + */ +static __be32 +nfsd_proc_getattr(struct svc_rqst *rqstp) +{ + struct nfsd_fhandle *argp = rqstp->rq_argp; + struct nfsd_attrstat *resp = rqstp->rq_resp; + + trace_nfsd_vfs_getattr(rqstp, &argp->fh); + + fh_copy(&resp->fh, &argp->fh); + resp->status = fh_verify(rqstp, &resp->fh, 0, + NFSD_MAY_NOP | NFSD_MAY_BYPASS_GSS_ON_ROOT); + if (resp->status != nfs_ok) + goto out; + resp->status = fh_getattr(&resp->fh, &resp->stat); +out: + resp->status = nfsd_map_status(resp->status); + return rpc_success; +} + +/* + * Set a file's attributes + * N.B. After this call resp->fh needs an fh_put + */ +static __be32 +nfsd_proc_setattr(struct svc_rqst *rqstp) +{ + struct nfsd_sattrargs *argp = rqstp->rq_argp; + struct nfsd_attrstat *resp = rqstp->rq_resp; + struct iattr *iap = &argp->attrs; + struct nfsd_attrs attrs = { + .na_iattr = iap, + }; + struct svc_fh *fhp; + int hosterr; + + dprintk("nfsd: SETATTR %s, valid=%x, size=%ld\n", + SVCFH_fmt(&argp->fh), + argp->attrs.ia_valid, (long) argp->attrs.ia_size); + + fhp = fh_copy(&resp->fh, &argp->fh); + + /* + * NFSv2 does not differentiate between "set-[ac]time-to-now" + * which only requires access, and "set-[ac]time-to-X" which + * requires ownership. + * So if it looks like it might be "set both to the same time which + * is close to now", and if setattr_prepare fails, then we + * convert to "set to now" instead of "set to explicit time" + * + * We only call setattr_prepare as the last test as technically + * it is not an interface that we should be using. + */ +#define BOTH_TIME_SET (ATTR_ATIME_SET | ATTR_MTIME_SET) +#define MAX_TOUCH_TIME_ERROR (30*60) + if ((iap->ia_valid & BOTH_TIME_SET) == BOTH_TIME_SET && + iap->ia_mtime.tv_sec == iap->ia_atime.tv_sec) { + /* + * Looks probable. + * + * Now just make sure time is in the right ballpark. + * Solaris, at least, doesn't seem to care what the time + * request is. We require it be within 30 minutes of now. + */ + time64_t delta = iap->ia_atime.tv_sec - ktime_get_real_seconds(); + + resp->status = fh_verify(rqstp, fhp, 0, NFSD_MAY_NOP); + if (resp->status != nfs_ok) + goto out; + + hosterr = fh_want_write(fhp); + if (hosterr) { + resp->status = nfserrno(hosterr); + goto out; + } + + if (delta < 0) + delta = -delta; + if (delta < MAX_TOUCH_TIME_ERROR && + setattr_prepare(&nop_mnt_idmap, fhp->fh_dentry, iap) != 0) { + /* + * Turn off ATTR_[AM]TIME_SET but leave ATTR_[AM]TIME. + * This will cause notify_change to set these times + * to "now" + */ + iap->ia_valid &= ~BOTH_TIME_SET; + } + } + + resp->status = nfsd_setattr(rqstp, fhp, &attrs, NULL); + if (resp->status != nfs_ok) + goto out; + + resp->status = fh_getattr(&resp->fh, &resp->stat); +out: + resp->status = nfsd_map_status(resp->status); + return rpc_success; +} + +/* Obsolete, replaced by MNTPROC_MNT. */ +static __be32 +nfsd_proc_root(struct svc_rqst *rqstp) +{ + return rpc_success; +} + +/* + * Look up a path name component + * Note: the dentry in the resp->fh may be negative if the file + * doesn't exist yet. + * N.B. After this call resp->fh needs an fh_put + */ +static __be32 +nfsd_proc_lookup(struct svc_rqst *rqstp) +{ + struct nfsd_diropargs *argp = rqstp->rq_argp; + struct nfsd_diropres *resp = rqstp->rq_resp; + + dprintk("nfsd: LOOKUP %s %.*s\n", + SVCFH_fmt(&argp->fh), argp->len, argp->name); + + fh_init(&resp->fh, NFS_FHSIZE); + resp->status = nfsd_lookup(rqstp, &argp->fh, argp->name, argp->len, + &resp->fh); + fh_put(&argp->fh); + if (resp->status != nfs_ok) + goto out; + + resp->status = fh_getattr(&resp->fh, &resp->stat); +out: + resp->status = nfsd_map_status(resp->status); + return rpc_success; +} + +/* + * Read a symlink. + */ +static __be32 +nfsd_proc_readlink(struct svc_rqst *rqstp) +{ + struct nfsd_fhandle *argp = rqstp->rq_argp; + struct nfsd_readlinkres *resp = rqstp->rq_resp; + + dprintk("nfsd: READLINK %s\n", SVCFH_fmt(&argp->fh)); + + /* Read the symlink. */ + resp->len = NFS_MAXPATHLEN; + resp->page = *(rqstp->rq_next_page++); + resp->status = nfsd_readlink(rqstp, &argp->fh, + page_address(resp->page), &resp->len); + + fh_put(&argp->fh); + resp->status = nfsd_map_status(resp->status); + return rpc_success; +} + +/* + * Read a portion of a file. + * N.B. After this call resp->fh needs an fh_put + */ +static __be32 +nfsd_proc_read(struct svc_rqst *rqstp) +{ + struct nfsd_readargs *argp = rqstp->rq_argp; + struct nfsd_readres *resp = rqstp->rq_resp; + u32 eof; + + dprintk("nfsd: READ %s %d bytes at %d\n", + SVCFH_fmt(&argp->fh), + argp->count, argp->offset); + + argp->count = min_t(u32, argp->count, NFS_MAXDATA); + argp->count = min_t(u32, argp->count, rqstp->rq_res.buflen); + + resp->pages = rqstp->rq_next_page; + + /* Obtain buffer pointer for payload. 19 is 1 word for + * status, 17 words for fattr, and 1 word for the byte count. + */ + svc_reserve_auth(rqstp, (19<<2) + argp->count + 4); + + resp->count = argp->count; + fh_copy(&resp->fh, &argp->fh); + resp->status = nfsd_read(rqstp, &resp->fh, argp->offset, + &resp->count, &eof); + if (resp->status == nfs_ok) + resp->status = fh_getattr(&resp->fh, &resp->stat); + else if (resp->status == nfserr_jukebox) + set_bit(RQ_DROPME, &rqstp->rq_flags); + resp->status = nfsd_map_status(resp->status); + return rpc_success; +} + +/* Reserved */ +static __be32 +nfsd_proc_writecache(struct svc_rqst *rqstp) +{ + return rpc_success; +} + +/* + * Write data to a file + * N.B. After this call resp->fh needs an fh_put + */ +static __be32 +nfsd_proc_write(struct svc_rqst *rqstp) +{ + struct nfsd_writeargs *argp = rqstp->rq_argp; + struct nfsd_attrstat *resp = rqstp->rq_resp; + unsigned long cnt = argp->len; + + dprintk("nfsd: WRITE %s %u bytes at %d\n", + SVCFH_fmt(&argp->fh), + argp->len, argp->offset); + + fh_copy(&resp->fh, &argp->fh); + resp->status = nfsd_write(rqstp, &resp->fh, argp->offset, + &argp->payload, &cnt, NFS_DATA_SYNC, NULL); + if (resp->status == nfs_ok) + resp->status = fh_getattr(&resp->fh, &resp->stat); + else if (resp->status == nfserr_jukebox) + set_bit(RQ_DROPME, &rqstp->rq_flags); + resp->status = nfsd_map_status(resp->status); + return rpc_success; +} + +/* + * CREATE processing is complicated. The keyword here is `overloaded.' + * The parent directory is kept locked between the check for existence + * and the actual create() call in compliance with VFS protocols. + * N.B. After this call _both_ argp->fh and resp->fh need an fh_put + */ +static __be32 +nfsd_proc_create(struct svc_rqst *rqstp) +{ + struct nfsd_createargs *argp = rqstp->rq_argp; + struct nfsd_diropres *resp = rqstp->rq_resp; + svc_fh *dirfhp = &argp->fh; + svc_fh *newfhp = &resp->fh; + struct iattr *attr = &argp->attrs; + struct nfsd_attrs attrs = { + .na_iattr = attr, + }; + struct inode *inode; + struct dentry *dchild; + int type, mode; + int hosterr; + dev_t rdev = 0, wanted = new_decode_dev(attr->ia_size); + + /* First verify the parent file handle */ + resp->status = fh_verify(rqstp, dirfhp, S_IFDIR, NFSD_MAY_EXEC); + if (resp->status != nfs_ok) + goto done; /* must fh_put dirfhp even on error */ + + /* Check for NFSD_MAY_WRITE in nfsd_create if necessary */ + + resp->status = nfserr_exist; + if (name_is_dot_dotdot(argp->name, argp->len)) + goto done; + hosterr = fh_want_write(dirfhp); + if (hosterr) { + resp->status = nfserrno(hosterr); + goto done; + } + + dchild = start_creating(&nop_mnt_idmap, dirfhp->fh_dentry, + &QSTR_LEN(argp->name, argp->len)); + if (IS_ERR(dchild)) { + resp->status = nfserrno(PTR_ERR(dchild)); + goto out_write; + } + fh_init(newfhp, NFS_FHSIZE); + resp->status = fh_compose(newfhp, dirfhp->fh_export, dchild, dirfhp); + if (!resp->status && d_really_is_negative(dchild)) + resp->status = nfserr_noent; + if (resp->status) { + if (resp->status != nfserr_noent) + goto out_unlock; + /* + * If the new file handle wasn't verified, we can't tell + * whether the file exists or not. Time to bail ... + */ + resp->status = nfserr_acces; + if (!newfhp->fh_dentry) { + printk(KERN_WARNING + "nfsd_proc_create: file handle not verified\n"); + goto out_unlock; + } + } + + inode = d_inode(newfhp->fh_dentry); + + /* Unfudge the mode bits */ + if (attr->ia_valid & ATTR_MODE) { + type = attr->ia_mode & S_IFMT; + mode = attr->ia_mode & ~S_IFMT; + if (!type) { + /* no type, so if target exists, assume same as that, + * else assume a file */ + if (inode) { + type = inode->i_mode & S_IFMT; + switch(type) { + case S_IFCHR: + case S_IFBLK: + /* reserve rdev for later checking */ + rdev = inode->i_rdev; + attr->ia_valid |= ATTR_SIZE; + + fallthrough; + case S_IFIFO: + /* this is probably a permission check.. + * at least IRIX implements perm checking on + * echo thing > device-special-file-or-pipe + * by doing a CREATE with type==0 + */ + resp->status = nfsd_permission( + &rqstp->rq_cred, + newfhp->fh_export, + newfhp->fh_dentry, + NFSD_MAY_WRITE|NFSD_MAY_LOCAL_ACCESS); + if (resp->status && resp->status != nfserr_rofs) + goto out_unlock; + } + } else + type = S_IFREG; + } + } else if (inode) { + type = inode->i_mode & S_IFMT; + mode = inode->i_mode & ~S_IFMT; + } else { + type = S_IFREG; + mode = 0; /* ??? */ + } + + attr->ia_valid |= ATTR_MODE; + attr->ia_mode = mode; + + /* Special treatment for non-regular files according to the + * gospel of sun micro + */ + if (type != S_IFREG) { + if (type != S_IFBLK && type != S_IFCHR) { + rdev = 0; + } else if (type == S_IFCHR && !(attr->ia_valid & ATTR_SIZE)) { + /* If you think you've seen the worst, grok this. */ + type = S_IFIFO; + } else { + /* Okay, char or block special */ + if (!rdev) + rdev = wanted; + } + + /* we've used the SIZE information, so discard it */ + attr->ia_valid &= ~ATTR_SIZE; + + /* Make sure the type and device matches */ + resp->status = nfserr_exist; + if (inode && inode_wrong_type(inode, type)) + goto out_unlock; + } + + resp->status = nfs_ok; + if (!inode) { + /* File doesn't exist. Create it and set attrs */ + resp->status = nfsd_create_locked(rqstp, dirfhp, &attrs, type, + rdev, newfhp); + /* nfsd_create_locked() unlocked the parent */ + dput(dchild); + goto out_write; + } else if (type == S_IFREG) { + dprintk("nfsd: existing %s, valid=%x, size=%ld\n", + argp->name, attr->ia_valid, (long) attr->ia_size); + /* File already exists. We ignore all attributes except + * size, so that creat() behaves exactly like + * open(..., O_CREAT|O_TRUNC|O_WRONLY). + */ + attr->ia_valid &= ATTR_SIZE; + if (attr->ia_valid) + resp->status = nfsd_setattr(rqstp, newfhp, &attrs, + NULL); + } + +out_unlock: + end_creating(dchild); +out_write: + fh_drop_write(dirfhp); +done: + fh_put(dirfhp); + if (resp->status != nfs_ok) + goto out; + resp->status = fh_getattr(&resp->fh, &resp->stat); +out: + resp->status = nfsd_map_status(resp->status); + return rpc_success; +} + +static __be32 +nfsd_proc_remove(struct svc_rqst *rqstp) +{ + struct nfsd_diropargs *argp = rqstp->rq_argp; + struct nfsd_stat *resp = rqstp->rq_resp; + + /* Unlink. -SIFDIR means file must not be a directory */ + resp->status = nfsd_unlink(rqstp, &argp->fh, -S_IFDIR, + argp->name, argp->len); + fh_put(&argp->fh); + resp->status = nfsd_map_status(resp->status); + return rpc_success; +} + +static __be32 +nfsd_proc_rename(struct svc_rqst *rqstp) +{ + struct nfsd_renameargs *argp = rqstp->rq_argp; + struct nfsd_stat *resp = rqstp->rq_resp; + + resp->status = nfsd_rename(rqstp, &argp->ffh, argp->fname, argp->flen, + &argp->tfh, argp->tname, argp->tlen); + fh_put(&argp->ffh); + fh_put(&argp->tfh); + resp->status = nfsd_map_status(resp->status); + return rpc_success; +} + +static __be32 +nfsd_proc_link(struct svc_rqst *rqstp) +{ + struct nfsd_linkargs *argp = rqstp->rq_argp; + struct nfsd_stat *resp = rqstp->rq_resp; + + resp->status = nfsd_link(rqstp, &argp->tfh, argp->tname, argp->tlen, + &argp->ffh); + fh_put(&argp->ffh); + fh_put(&argp->tfh); + resp->status = nfsd_map_status(resp->status); + return rpc_success; +} + +static __be32 +nfsd_proc_symlink(struct svc_rqst *rqstp) +{ + struct nfsd_symlinkargs *argp = rqstp->rq_argp; + struct nfsd_stat *resp = rqstp->rq_resp; + struct nfsd_attrs attrs = { + .na_iattr = &argp->attrs, + }; + struct svc_fh newfh; + + if (argp->tlen > NFS_MAXPATHLEN) { + resp->status = nfserr_nametoolong; + goto out; + } + + argp->tname = svc_fill_symlink_pathname(rqstp, &argp->first, + page_address(rqstp->rq_arg.pages[0]), + argp->tlen); + if (IS_ERR(argp->tname)) { + resp->status = nfserrno(PTR_ERR(argp->tname)); + goto out; + } + + fh_init(&newfh, NFS_FHSIZE); + resp->status = nfsd_symlink(rqstp, &argp->ffh, argp->fname, argp->flen, + argp->tname, &attrs, &newfh); + + kfree(argp->tname); + fh_put(&argp->ffh); + fh_put(&newfh); +out: + resp->status = nfsd_map_status(resp->status); + return rpc_success; +} + +/* + * Make directory. This operation is not idempotent. + * N.B. After this call resp->fh needs an fh_put + */ +static __be32 +nfsd_proc_mkdir(struct svc_rqst *rqstp) +{ + struct nfsd_createargs *argp = rqstp->rq_argp; + struct nfsd_diropres *resp = rqstp->rq_resp; + struct nfsd_attrs attrs = { + .na_iattr = &argp->attrs, + }; + + if (resp->fh.fh_dentry) { + printk(KERN_WARNING + "nfsd_proc_mkdir: response already verified??\n"); + } + + argp->attrs.ia_valid &= ~ATTR_SIZE; + fh_init(&resp->fh, NFS_FHSIZE); + resp->status = nfsd_create(rqstp, &argp->fh, argp->name, argp->len, + &attrs, S_IFDIR, 0, &resp->fh); + fh_put(&argp->fh); + if (resp->status != nfs_ok) + goto out; + + resp->status = fh_getattr(&resp->fh, &resp->stat); +out: + resp->status = nfsd_map_status(resp->status); + return rpc_success; +} + +/* + * Remove a directory + */ +static __be32 +nfsd_proc_rmdir(struct svc_rqst *rqstp) +{ + struct nfsd_diropargs *argp = rqstp->rq_argp; + struct nfsd_stat *resp = rqstp->rq_resp; + + resp->status = nfsd_unlink(rqstp, &argp->fh, S_IFDIR, + argp->name, argp->len); + fh_put(&argp->fh); + resp->status = nfsd_map_status(resp->status); + return rpc_success; +} + +static void nfsd_init_dirlist_pages(struct svc_rqst *rqstp, + struct nfsd_readdirres *resp, + u32 count) +{ + struct xdr_buf *buf = &resp->dirlist; + struct xdr_stream *xdr = &resp->xdr; + + memset(buf, 0, sizeof(*buf)); + + /* Reserve room for the NULL ptr & eof flag (-2 words) */ + buf->buflen = clamp(count, (u32)(XDR_UNIT * 2), (u32)PAGE_SIZE); + buf->buflen -= XDR_UNIT * 2; + buf->pages = rqstp->rq_next_page; + rqstp->rq_next_page++; + + xdr_init_encode_pages(xdr, buf); +} + +/* + * Read a portion of a directory. + */ +static __be32 +nfsd_proc_readdir(struct svc_rqst *rqstp) +{ + struct nfsd_readdirargs *argp = rqstp->rq_argp; + struct nfsd_readdirres *resp = rqstp->rq_resp; + loff_t offset; + + trace_nfsd_vfs_readdir(rqstp, &argp->fh, argp->count, argp->cookie); + + nfsd_init_dirlist_pages(rqstp, resp, argp->count); + + resp->common.err = nfs_ok; + resp->cookie_offset = 0; + offset = argp->cookie; + resp->status = nfsd_readdir(rqstp, &argp->fh, &offset, + &resp->common, nfssvc_encode_entry); + nfssvc_encode_nfscookie(resp, offset); + + fh_put(&argp->fh); + resp->status = nfsd_map_status(resp->status); + return rpc_success; +} + +/* + * Get file system info + */ +static __be32 +nfsd_proc_statfs(struct svc_rqst *rqstp) +{ + struct nfsd_fhandle *argp = rqstp->rq_argp; + struct nfsd_statfsres *resp = rqstp->rq_resp; + + resp->status = nfsd_statfs(rqstp, &argp->fh, &resp->stats, + NFSD_MAY_BYPASS_GSS_ON_ROOT); + fh_put(&argp->fh); + resp->status = nfsd_map_status(resp->status); + return rpc_success; +} + +/* + * NFSv2 Server procedures. + * Only the results of non-idempotent operations are cached. + */ + +#define ST 1 /* status */ +#define FH 8 /* filehandle */ +#define AT 18 /* attributes */ + +static const struct svc_procedure nfsd_procedures2[18] = { + [NFSPROC_NULL] = { + .pc_func = nfsd_proc_null, + .pc_decode = nfssvc_decode_voidarg, + .pc_encode = nfssvc_encode_voidres, + .pc_argsize = sizeof(struct nfsd_voidargs), + .pc_argzero = sizeof(struct nfsd_voidargs), + .pc_ressize = sizeof(struct nfsd_voidres), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = 0, + .pc_name = "NULL", + }, + [NFSPROC_GETATTR] = { + .pc_func = nfsd_proc_getattr, + .pc_decode = nfssvc_decode_fhandleargs, + .pc_encode = nfssvc_encode_attrstatres, + .pc_release = nfssvc_release_attrstat, + .pc_argsize = sizeof(struct nfsd_fhandle), + .pc_argzero = sizeof(struct nfsd_fhandle), + .pc_ressize = sizeof(struct nfsd_attrstat), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = ST+AT, + .pc_name = "GETATTR", + }, + [NFSPROC_SETATTR] = { + .pc_func = nfsd_proc_setattr, + .pc_decode = nfssvc_decode_sattrargs, + .pc_encode = nfssvc_encode_attrstatres, + .pc_release = nfssvc_release_attrstat, + .pc_argsize = sizeof(struct nfsd_sattrargs), + .pc_argzero = sizeof(struct nfsd_sattrargs), + .pc_ressize = sizeof(struct nfsd_attrstat), + .pc_cachetype = RC_REPLBUFF, + .pc_xdrressize = ST+AT, + .pc_name = "SETATTR", + }, + [NFSPROC_ROOT] = { + .pc_func = nfsd_proc_root, + .pc_decode = nfssvc_decode_voidarg, + .pc_encode = nfssvc_encode_voidres, + .pc_argsize = sizeof(struct nfsd_voidargs), + .pc_argzero = sizeof(struct nfsd_voidargs), + .pc_ressize = sizeof(struct nfsd_voidres), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = 0, + .pc_name = "ROOT", + }, + [NFSPROC_LOOKUP] = { + .pc_func = nfsd_proc_lookup, + .pc_decode = nfssvc_decode_diropargs, + .pc_encode = nfssvc_encode_diropres, + .pc_release = nfssvc_release_diropres, + .pc_argsize = sizeof(struct nfsd_diropargs), + .pc_argzero = sizeof(struct nfsd_diropargs), + .pc_ressize = sizeof(struct nfsd_diropres), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = ST+FH+AT, + .pc_name = "LOOKUP", + }, + [NFSPROC_READLINK] = { + .pc_func = nfsd_proc_readlink, + .pc_decode = nfssvc_decode_fhandleargs, + .pc_encode = nfssvc_encode_readlinkres, + .pc_argsize = sizeof(struct nfsd_fhandle), + .pc_argzero = sizeof(struct nfsd_fhandle), + .pc_ressize = sizeof(struct nfsd_readlinkres), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = ST+1+NFS_MAXPATHLEN/4, + .pc_name = "READLINK", + }, + [NFSPROC_READ] = { + .pc_func = nfsd_proc_read, + .pc_decode = nfssvc_decode_readargs, + .pc_encode = nfssvc_encode_readres, + .pc_release = nfssvc_release_readres, + .pc_argsize = sizeof(struct nfsd_readargs), + .pc_argzero = sizeof(struct nfsd_readargs), + .pc_ressize = sizeof(struct nfsd_readres), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = ST+AT+1+NFS_MAXDATA/4, + .pc_name = "READ", + }, + [NFSPROC_WRITECACHE] = { + .pc_func = nfsd_proc_writecache, + .pc_decode = nfssvc_decode_voidarg, + .pc_encode = nfssvc_encode_voidres, + .pc_argsize = sizeof(struct nfsd_voidargs), + .pc_argzero = sizeof(struct nfsd_voidargs), + .pc_ressize = sizeof(struct nfsd_voidres), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = 0, + .pc_name = "WRITECACHE", + }, + [NFSPROC_WRITE] = { + .pc_func = nfsd_proc_write, + .pc_decode = nfssvc_decode_writeargs, + .pc_encode = nfssvc_encode_attrstatres, + .pc_release = nfssvc_release_attrstat, + .pc_argsize = sizeof(struct nfsd_writeargs), + .pc_argzero = sizeof(struct nfsd_writeargs), + .pc_ressize = sizeof(struct nfsd_attrstat), + .pc_cachetype = RC_REPLBUFF, + .pc_xdrressize = ST+AT, + .pc_name = "WRITE", + }, + [NFSPROC_CREATE] = { + .pc_func = nfsd_proc_create, + .pc_decode = nfssvc_decode_createargs, + .pc_encode = nfssvc_encode_diropres, + .pc_release = nfssvc_release_diropres, + .pc_argsize = sizeof(struct nfsd_createargs), + .pc_argzero = sizeof(struct nfsd_createargs), + .pc_ressize = sizeof(struct nfsd_diropres), + .pc_cachetype = RC_REPLBUFF, + .pc_xdrressize = ST+FH+AT, + .pc_name = "CREATE", + }, + [NFSPROC_REMOVE] = { + .pc_func = nfsd_proc_remove, + .pc_decode = nfssvc_decode_diropargs, + .pc_encode = nfssvc_encode_statres, + .pc_argsize = sizeof(struct nfsd_diropargs), + .pc_argzero = sizeof(struct nfsd_diropargs), + .pc_ressize = sizeof(struct nfsd_stat), + .pc_cachetype = RC_REPLSTAT, + .pc_xdrressize = ST, + .pc_name = "REMOVE", + }, + [NFSPROC_RENAME] = { + .pc_func = nfsd_proc_rename, + .pc_decode = nfssvc_decode_renameargs, + .pc_encode = nfssvc_encode_statres, + .pc_argsize = sizeof(struct nfsd_renameargs), + .pc_argzero = sizeof(struct nfsd_renameargs), + .pc_ressize = sizeof(struct nfsd_stat), + .pc_cachetype = RC_REPLSTAT, + .pc_xdrressize = ST, + .pc_name = "RENAME", + }, + [NFSPROC_LINK] = { + .pc_func = nfsd_proc_link, + .pc_decode = nfssvc_decode_linkargs, + .pc_encode = nfssvc_encode_statres, + .pc_argsize = sizeof(struct nfsd_linkargs), + .pc_argzero = sizeof(struct nfsd_linkargs), + .pc_ressize = sizeof(struct nfsd_stat), + .pc_cachetype = RC_REPLSTAT, + .pc_xdrressize = ST, + .pc_name = "LINK", + }, + [NFSPROC_SYMLINK] = { + .pc_func = nfsd_proc_symlink, + .pc_decode = nfssvc_decode_symlinkargs, + .pc_encode = nfssvc_encode_statres, + .pc_argsize = sizeof(struct nfsd_symlinkargs), + .pc_argzero = sizeof(struct nfsd_symlinkargs), + .pc_ressize = sizeof(struct nfsd_stat), + .pc_cachetype = RC_REPLSTAT, + .pc_xdrressize = ST, + .pc_name = "SYMLINK", + }, + [NFSPROC_MKDIR] = { + .pc_func = nfsd_proc_mkdir, + .pc_decode = nfssvc_decode_createargs, + .pc_encode = nfssvc_encode_diropres, + .pc_release = nfssvc_release_diropres, + .pc_argsize = sizeof(struct nfsd_createargs), + .pc_argzero = sizeof(struct nfsd_createargs), + .pc_ressize = sizeof(struct nfsd_diropres), + .pc_cachetype = RC_REPLBUFF, + .pc_xdrressize = ST+FH+AT, + .pc_name = "MKDIR", + }, + [NFSPROC_RMDIR] = { + .pc_func = nfsd_proc_rmdir, + .pc_decode = nfssvc_decode_diropargs, + .pc_encode = nfssvc_encode_statres, + .pc_argsize = sizeof(struct nfsd_diropargs), + .pc_argzero = sizeof(struct nfsd_diropargs), + .pc_ressize = sizeof(struct nfsd_stat), + .pc_cachetype = RC_REPLSTAT, + .pc_xdrressize = ST, + .pc_name = "RMDIR", + }, + [NFSPROC_READDIR] = { + .pc_func = nfsd_proc_readdir, + .pc_decode = nfssvc_decode_readdirargs, + .pc_encode = nfssvc_encode_readdirres, + .pc_argsize = sizeof(struct nfsd_readdirargs), + .pc_argzero = sizeof(struct nfsd_readdirargs), + .pc_ressize = sizeof(struct nfsd_readdirres), + .pc_cachetype = RC_NOCACHE, + .pc_name = "READDIR", + }, + [NFSPROC_STATFS] = { + .pc_func = nfsd_proc_statfs, + .pc_decode = nfssvc_decode_fhandleargs, + .pc_encode = nfssvc_encode_statfsres, + .pc_argsize = sizeof(struct nfsd_fhandle), + .pc_argzero = sizeof(struct nfsd_fhandle), + .pc_ressize = sizeof(struct nfsd_statfsres), + .pc_cachetype = RC_NOCACHE, + .pc_xdrressize = ST+5, + .pc_name = "STATFS", + }, +}; + +const struct svc_version nfsd_version2 = { + .vs_vers = 2, + .vs_nproc = ARRAY_SIZE(nfsd_procedures2), + .vs_proc = nfsd_procedures2, + .vs_dispatch = nfsd_dispatch, + .vs_xdrsize = NFS2_SVC_XDRSIZE, +}; diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c new file mode 100644 index 000000000..2edf716ea --- /dev/null +++ b/fs/nfsd/nfssvc.c @@ -0,0 +1,1095 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Central processing for nfsd. + * + * Authors: Olaf Kirch (okir@monad.swb.de) + * + * Copyright (C) 1995, 1996, 1997 Olaf Kirch + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "nfsd.h" +#include "cache.h" +#include "vfs.h" +#include "netns.h" +#include "filecache.h" + +#include "trace.h" + +#define NFSDDBG_FACILITY NFSDDBG_SVC + +atomic_t nfsd_th_cnt = ATOMIC_INIT(0); +static int nfsd(void *vrqstp); +#if defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL) +static int nfsd_acl_rpcbind_set(struct net *, + const struct svc_program *, + u32, int, + unsigned short, + unsigned short); +static __be32 nfsd_acl_init_request(struct svc_rqst *, + const struct svc_program *, + struct svc_process_info *); +#endif +static int nfsd_rpcbind_set(struct net *, + const struct svc_program *, + u32, int, + unsigned short, + unsigned short); +static __be32 nfsd_init_request(struct svc_rqst *, + const struct svc_program *, + struct svc_process_info *); + +/* + * nfsd_mutex protects nn->nfsd_serv -- both the pointer itself and some members + * of the svc_serv struct such as ->sv_temp_socks and ->sv_permsocks. + * + * Finally, the nfsd_mutex also protects some of the global variables that are + * accessed when nfsd starts and that are settable via the write_* routines in + * nfsctl.c. In particular: + * + * user_recovery_dirname + * user_lease_time + * nfsd_versions + */ +DEFINE_MUTEX(nfsd_mutex); + +#if IS_ENABLED(CONFIG_NFS_LOCALIO) +static const struct svc_version *localio_versions[] = { + [1] = &localio_version1, +}; + +#define NFSD_LOCALIO_NRVERS ARRAY_SIZE(localio_versions) + +#endif /* CONFIG_NFS_LOCALIO */ + +#if defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL) +static const struct svc_version *nfsd_acl_version[] = { +# if defined(CONFIG_NFSD_V2_ACL) + [2] = &nfsd_acl_version2, +# endif +# if defined(CONFIG_NFSD_V3_ACL) + [3] = &nfsd_acl_version3, +# endif +}; + +#define NFSD_ACL_MINVERS 2 +#define NFSD_ACL_NRVERS ARRAY_SIZE(nfsd_acl_version) + +#endif /* defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL) */ + +static const struct svc_version *nfsd_version[NFSD_MAXVERS+1] = { +#if defined(CONFIG_NFSD_V2) + [2] = &nfsd_version2, +#endif + [3] = &nfsd_version3, +#if defined(CONFIG_NFSD_V4) + [4] = &nfsd_version4, +#endif +}; + +struct svc_program nfsd_programs[] = { + { + .pg_prog = NFS_PROGRAM, /* program number */ + .pg_nvers = NFSD_MAXVERS+1, /* nr of entries in nfsd_version */ + .pg_vers = nfsd_version, /* version table */ + .pg_name = "nfsd", /* program name */ + .pg_class = "nfsd", /* authentication class */ + .pg_authenticate = svc_set_client, /* export authentication */ + .pg_init_request = nfsd_init_request, + .pg_rpcbind_set = nfsd_rpcbind_set, + }, +#if defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL) + { + .pg_prog = NFS_ACL_PROGRAM, + .pg_nvers = NFSD_ACL_NRVERS, + .pg_vers = nfsd_acl_version, + .pg_name = "nfsacl", + .pg_class = "nfsd", + .pg_authenticate = svc_set_client, + .pg_init_request = nfsd_acl_init_request, + .pg_rpcbind_set = nfsd_acl_rpcbind_set, + }, +#endif /* defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL) */ +#if IS_ENABLED(CONFIG_NFS_LOCALIO) + { + .pg_prog = NFS_LOCALIO_PROGRAM, + .pg_nvers = NFSD_LOCALIO_NRVERS, + .pg_vers = localio_versions, + .pg_name = "nfslocalio", + .pg_class = "nfsd", + .pg_authenticate = svc_set_client, + .pg_init_request = svc_generic_init_request, + .pg_rpcbind_set = svc_generic_rpcbind_set, + } +#endif /* CONFIG_NFS_LOCALIO */ +}; + +bool nfsd_support_version(int vers) +{ + if (vers >= NFSD_MINVERS && vers <= NFSD_MAXVERS) + return nfsd_version[vers] != NULL; + return false; +} + +int nfsd_vers(struct nfsd_net *nn, int vers, enum vers_op change) +{ + if (vers < NFSD_MINVERS || vers > NFSD_MAXVERS) + return 0; + switch(change) { + case NFSD_SET: + nn->nfsd_versions[vers] = nfsd_support_version(vers); + break; + case NFSD_CLEAR: + nn->nfsd_versions[vers] = false; + break; + case NFSD_TEST: + return nn->nfsd_versions[vers]; + case NFSD_AVAIL: + return nfsd_support_version(vers); + } + return 0; +} + +static void +nfsd_adjust_nfsd_versions4(struct nfsd_net *nn) +{ + unsigned i; + + for (i = 0; i <= NFSD_SUPPORTED_MINOR_VERSION; i++) { + if (nn->nfsd4_minorversions[i]) + return; + } + nfsd_vers(nn, 4, NFSD_CLEAR); +} + +int nfsd_minorversion(struct nfsd_net *nn, u32 minorversion, enum vers_op change) +{ + if (minorversion > NFSD_SUPPORTED_MINOR_VERSION && + change != NFSD_AVAIL) + return -1; + + switch(change) { + case NFSD_SET: + nfsd_vers(nn, 4, NFSD_SET); + nn->nfsd4_minorversions[minorversion] = + nfsd_vers(nn, 4, NFSD_TEST); + break; + case NFSD_CLEAR: + nn->nfsd4_minorversions[minorversion] = false; + nfsd_adjust_nfsd_versions4(nn); + break; + case NFSD_TEST: + return nn->nfsd4_minorversions[minorversion]; + case NFSD_AVAIL: + return minorversion <= NFSD_SUPPORTED_MINOR_VERSION && + nfsd_vers(nn, 4, NFSD_AVAIL); + } + return 0; +} + +bool nfsd_net_try_get(struct net *net) __must_hold(rcu) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + return (nn && percpu_ref_tryget_live(&nn->nfsd_net_ref)); +} + +void nfsd_net_put(struct net *net) __must_hold(rcu) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + percpu_ref_put(&nn->nfsd_net_ref); +} + +static void nfsd_net_done(struct percpu_ref *ref) +{ + struct nfsd_net *nn = container_of(ref, struct nfsd_net, nfsd_net_ref); + + complete(&nn->nfsd_net_confirm_done); +} + +static void nfsd_net_free(struct percpu_ref *ref) +{ + struct nfsd_net *nn = container_of(ref, struct nfsd_net, nfsd_net_ref); + + complete(&nn->nfsd_net_free_done); +} + +/* + * Maximum number of nfsd processes + */ +#define NFSD_MAXSERVS 8192 + +/** + * nfsd_nrthreads - report a namespace's configured nfsd thread count + * @net: network namespace to query + * + * Return: the configured thread ceiling, or 0 when no service runs. + */ +int nfsd_nrthreads(struct net *net) +{ + int rv = 0; + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + /* nfsd_mutex keeps nn->nfsd_serv valid across the read. */ + mutex_lock(&nfsd_mutex); + if (nn->nfsd_serv) + rv = svc_serv_maxthreads(nn->nfsd_serv); + mutex_unlock(&nfsd_mutex); + return rv; +} + +static int nfsd_users = 0; + +static int nfsd_startup_generic(void) +{ + int ret; + + if (nfsd_users++) + return 0; + + ret = nfsd_file_cache_init(); + if (ret) + goto dec_users; + + ret = nfs4_state_start(); + if (ret) + goto out_file_cache; + return 0; + +out_file_cache: + nfsd_file_cache_shutdown(); +dec_users: + nfsd_users--; + return ret; +} + +static void nfsd_shutdown_generic(void) +{ + if (--nfsd_users) + return; + + nfs4_state_shutdown(); + nfsd_file_cache_shutdown(); +} + +static bool nfsd_needs_lockd(struct nfsd_net *nn) +{ + return nfsd_vers(nn, 2, NFSD_TEST) || nfsd_vers(nn, 3, NFSD_TEST); +} + +/** + * nfsd_copy_write_verifier - Atomically copy a write verifier + * @verf: buffer in which to receive the verifier cookie + * @nn: NFS net namespace + * + * This function provides a wait-free mechanism for copying the + * namespace's write verifier without tearing it. + */ +void nfsd_copy_write_verifier(__be32 verf[2], struct nfsd_net *nn) +{ + unsigned int seq; + + do { + seq = read_seqbegin(&nn->writeverf_lock); + memcpy(verf, nn->writeverf, sizeof(nn->writeverf)); + } while (read_seqretry(&nn->writeverf_lock, seq)); +} + +static void nfsd_reset_write_verifier_locked(struct nfsd_net *nn) +{ + struct timespec64 now; + u64 verf; + + /* + * Because the time value is hashed, y2038 time_t overflow + * is irrelevant in this usage. + */ + ktime_get_raw_ts64(&now); + verf = siphash_2u64(now.tv_sec, now.tv_nsec, &nn->siphash_key); + memcpy(nn->writeverf, &verf, sizeof(nn->writeverf)); +} + +/** + * nfsd_reset_write_verifier - Generate a new write verifier + * @nn: NFS net namespace + * + * This function updates the ->writeverf field of @nn. This field + * contains an opaque cookie that, according to Section 18.32.3 of + * RFC 8881, "the client can use to determine whether a server has + * changed instance state (e.g., server restart) between a call to + * WRITE and a subsequent call to either WRITE or COMMIT. This + * cookie MUST be unchanged during a single instance of the NFSv4.1 + * server and MUST be unique between instances of the NFSv4.1 + * server." + */ +void nfsd_reset_write_verifier(struct nfsd_net *nn) +{ + write_seqlock(&nn->writeverf_lock); + nfsd_reset_write_verifier_locked(nn); + write_sequnlock(&nn->writeverf_lock); +} + +/* + * Crank up a set of per-namespace resources for a new NFSD instance, + * including lockd, a duplicate reply cache, an open file cache + * instance, and a cache of NFSv4 state objects. + */ +static int nfsd_startup_net(struct net *net, const struct cred *cred) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + int ret; + + if (test_bit(NFSD_NET_UP, &nn->flags)) + return 0; + + ret = nfsd_startup_generic(); + if (ret) + return ret; + + if (list_empty(&nn->nfsd_serv->sv_permsocks)) { + pr_warn("NFSD: Failed to start, no listeners configured.\n"); + ret = -EIO; + goto out_socks; + } + + if (nfsd_needs_lockd(nn) && !test_bit(NFSD_NET_LOCKD_UP, &nn->flags)) { + ret = lockd_up(net, cred); + if (ret) + goto out_socks; + set_bit(NFSD_NET_LOCKD_UP, &nn->flags); + } + + ret = nfsd_file_cache_start_net(net); + if (ret) + goto out_lockd; + + ret = nfsd_reply_cache_init(nn); + if (ret) + goto out_filecache; + +#ifdef CONFIG_NFSD_V4_2_INTER_SSC + nfsd4_ssc_init_umount_work(nn); +#endif + ret = nfs4_state_start_net(net); + if (ret) + goto out_reply_cache; + + set_bit(NFSD_NET_UP, &nn->flags); + return 0; + +out_reply_cache: + nfsd_reply_cache_shutdown(nn); +out_filecache: + nfsd_file_cache_shutdown_net(net); +out_lockd: + if (test_bit(NFSD_NET_LOCKD_UP, &nn->flags)) { + lockd_down(net); + clear_bit(NFSD_NET_LOCKD_UP, &nn->flags); + } +out_socks: + nfsd_shutdown_generic(); + return ret; +} + +static void nfsd_shutdown_net(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + if (test_bit(NFSD_NET_UP, &nn->flags)) { + percpu_ref_kill_and_confirm(&nn->nfsd_net_ref, nfsd_net_done); + wait_for_completion(&nn->nfsd_net_confirm_done); + + nfsd_export_flush(net); + nfs4_state_shutdown_net(net); + nfsd_reply_cache_shutdown(nn); + nfsd_file_cache_shutdown_net(net); + if (test_bit(NFSD_NET_LOCKD_UP, &nn->flags)) { + lockd_down(net); + clear_bit(NFSD_NET_LOCKD_UP, &nn->flags); + } + wait_for_completion(&nn->nfsd_net_free_done); + } + + percpu_ref_exit(&nn->nfsd_net_ref); + + if (test_bit(NFSD_NET_UP, &nn->flags)) + nfsd_shutdown_generic(); + clear_bit(NFSD_NET_UP, &nn->flags); +} + +static DEFINE_SPINLOCK(nfsd_notifier_lock); +static int nfsd_inetaddr_event(struct notifier_block *this, unsigned long event, + void *ptr) +{ + struct in_ifaddr *ifa = (struct in_ifaddr *)ptr; + struct net_device *dev = ifa->ifa_dev->dev; + struct net *net = dev_net(dev); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct sockaddr_in sin; + + if (event != NETDEV_DOWN || !nn->nfsd_serv) + goto out; + + spin_lock(&nfsd_notifier_lock); + if (nn->nfsd_serv) { + dprintk("nfsd_inetaddr_event: removed %pI4\n", &ifa->ifa_local); + sin.sin_family = AF_INET; + sin.sin_addr.s_addr = ifa->ifa_local; + svc_age_temp_xprts_now(nn->nfsd_serv, (struct sockaddr *)&sin); + } + spin_unlock(&nfsd_notifier_lock); + +out: + return NOTIFY_DONE; +} + +static struct notifier_block nfsd_inetaddr_notifier = { + .notifier_call = nfsd_inetaddr_event, +}; + +#if IS_ENABLED(CONFIG_IPV6) +static int nfsd_inet6addr_event(struct notifier_block *this, + unsigned long event, void *ptr) +{ + struct inet6_ifaddr *ifa = (struct inet6_ifaddr *)ptr; + struct net_device *dev = ifa->idev->dev; + struct net *net = dev_net(dev); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct sockaddr_in6 sin6; + + if (event != NETDEV_DOWN || !nn->nfsd_serv) + goto out; + + spin_lock(&nfsd_notifier_lock); + if (nn->nfsd_serv) { + dprintk("nfsd_inet6addr_event: removed %pI6\n", &ifa->addr); + sin6.sin6_family = AF_INET6; + sin6.sin6_addr = ifa->addr; + if (ipv6_addr_type(&sin6.sin6_addr) & IPV6_ADDR_LINKLOCAL) + sin6.sin6_scope_id = ifa->idev->dev->ifindex; + svc_age_temp_xprts_now(nn->nfsd_serv, (struct sockaddr *)&sin6); + } + spin_unlock(&nfsd_notifier_lock); + +out: + return NOTIFY_DONE; +} + +static struct notifier_block nfsd_inet6addr_notifier = { + .notifier_call = nfsd_inet6addr_event, +}; +#endif + +/* Only used under nfsd_mutex, so this atomic may be overkill: */ +static atomic_t nfsd_notifier_refcount = ATOMIC_INIT(0); + +/** + * nfsd_destroy_serv - tear down NFSD's svc_serv for a namespace + * @net: network namespace the NFS service is associated with + */ +void nfsd_destroy_serv(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct svc_serv *serv = nn->nfsd_serv; + + lockdep_assert_held(&nfsd_mutex); + + spin_lock(&nfsd_notifier_lock); + nn->nfsd_serv = NULL; + spin_unlock(&nfsd_notifier_lock); + + /* check if the notifier still has clients */ + if (atomic_dec_return(&nfsd_notifier_refcount) == 0) { + unregister_inetaddr_notifier(&nfsd_inetaddr_notifier); +#if IS_ENABLED(CONFIG_IPV6) + unregister_inet6addr_notifier(&nfsd_inet6addr_notifier); +#endif + } + + /* + * write_ports can create the server without actually starting + * any threads. If we get shut down before any threads are + * started, then nfsd_destroy_serv will be run before any of this + * other initialization has been done except the rpcb information. + */ + svc_xprt_destroy_all(serv, net, true); + nfsd_shutdown_net(net); + svc_destroy(&serv); +} + +void nfsd_reset_versions(struct nfsd_net *nn) +{ + int i; + + for (i = 0; i <= NFSD_MAXVERS; i++) + if (nfsd_vers(nn, i, NFSD_TEST)) + return; + + for (i = 0; i <= NFSD_MAXVERS; i++) + if (i != 4) + nfsd_vers(nn, i, NFSD_SET); + else { + int minor = 0; + while (nfsd_minorversion(nn, minor, NFSD_SET) >= 0) + minor++; + } +} + +static int nfsd_get_default_max_blksize(void) +{ + struct sysinfo i; + unsigned long long target; + unsigned long ret; + + si_meminfo(&i); + target = (i.totalram - i.totalhigh) << PAGE_SHIFT; + /* + * Aim for 1/4096 of memory per thread This gives 1MB on 4Gig + * machines, but only uses 32K on 128M machines. Bottom out at + * 8K on 32M and smaller. Of course, this is only a default. + */ + target >>= 12; + + ret = NFSSVC_DEFBLKSIZE; + while (ret > target && ret >= 8*1024*2) + ret /= 2; + return ret; +} + +void nfsd_shutdown_threads(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct svc_serv *serv; + + mutex_lock(&nfsd_mutex); + serv = nn->nfsd_serv; + if (serv == NULL) { + mutex_unlock(&nfsd_mutex); + return; + } + + /* Kill outstanding nfsd threads */ + svc_set_num_threads(serv, 0, 0); + nfsd_destroy_serv(net); + mutex_unlock(&nfsd_mutex); +} + +struct svc_rqst *nfsd_current_rqst(void) +{ + if (kthread_func(current) == nfsd) + return kthread_data(current); + return NULL; +} + +int nfsd_create_serv(struct net *net) +{ + int error; + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct svc_serv *serv; + + WARN_ON(!mutex_is_locked(&nfsd_mutex)); + if (nn->nfsd_serv) + return 0; + + error = percpu_ref_init(&nn->nfsd_net_ref, nfsd_net_free, + 0, GFP_KERNEL); + if (error) + return error; + init_completion(&nn->nfsd_net_free_done); + init_completion(&nn->nfsd_net_confirm_done); + + if (nfsd_max_blksize == 0) + nfsd_max_blksize = nfsd_get_default_max_blksize(); + nfsd_reset_versions(nn); + serv = svc_create_pooled(nfsd_programs, ARRAY_SIZE(nfsd_programs), + &nn->nfsd_svcstats, + nfsd_max_blksize, nfsd); + if (serv == NULL) { + percpu_ref_exit(&nn->nfsd_net_ref); + return -ENOMEM; + } + + error = svc_bind(serv, net); + if (error < 0) { + svc_destroy(&serv); + percpu_ref_exit(&nn->nfsd_net_ref); + return error; + } + spin_lock(&nfsd_notifier_lock); + nn->nfsd_serv = serv; + spin_unlock(&nfsd_notifier_lock); + + /* check if the notifier is already set */ + if (atomic_inc_return(&nfsd_notifier_refcount) == 1) { + register_inetaddr_notifier(&nfsd_inetaddr_notifier); +#if IS_ENABLED(CONFIG_IPV6) + register_inet6addr_notifier(&nfsd_inet6addr_notifier); +#endif + } + nfsd_reset_write_verifier(nn); + return 0; +} + +int nfsd_nrpools(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + if (nn->nfsd_serv == NULL) + return 0; + else + return svc_serv_nrpools(nn->nfsd_serv); +} + +int nfsd_get_nrthreads(int n, int *nthreads, struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct svc_serv *serv = nn->nfsd_serv; + int i; + + if (serv) + for (i = 0; i < svc_serv_nrpools(serv) && i < n; i++) + nthreads[i] = serv->sv_pools[i].sp_nrthrmax; + return 0; +} + +/** + * nfsd_set_nrthreads - set the number of running threads in the net's service + * @n: number of array members in @nthreads + * @nthreads: array of thread counts for each pool + * @net: network namespace to operate within + * + * This function alters the number of running threads for the given network + * namespace in each pool. If passed an array longer then the number of pools + * the extra pool settings are ignored. If passed an array shorter than the + * number of pools, the missing values are interpreted as 0's. + * + * Returns 0 on success or a negative errno on error. + */ +int nfsd_set_nrthreads(int n, int *nthreads, struct net *net) +{ + int i = 0; + int tot = 0; + int err = 0; + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + lockdep_assert_held(&nfsd_mutex); + + if (nn->nfsd_serv == NULL || n <= 0) + return 0; + + /* Special case: When n == 1, distribute threads equally among pools. */ + if (n == 1) + return svc_set_num_threads(nn->nfsd_serv, nn->min_threads, nthreads[0]); + + if (n > svc_serv_nrpools(nn->nfsd_serv)) + n = svc_serv_nrpools(nn->nfsd_serv); + + /* enforce a global maximum number of threads */ + tot = 0; + for (i = 0; i < n; i++) { + nthreads[i] = min(nthreads[i], NFSD_MAXSERVS); + tot += nthreads[i]; + } + if (tot > NFSD_MAXSERVS) { + /* total too large: scale down requested numbers */ + for (i = 0; i < n && tot > 0; i++) { + int new = nthreads[i] * NFSD_MAXSERVS / tot; + tot -= (nthreads[i] - new); + nthreads[i] = new; + } + for (i = 0; i < n && tot > 0; i++) { + nthreads[i]--; + tot--; + } + } + + /* apply the new numbers */ + for (i = 0; i < n; i++) { + err = svc_set_pool_threads(nn->nfsd_serv, + &nn->nfsd_serv->sv_pools[i], + nn->min_threads, nthreads[i]); + if (err) + goto out; + } + + /* Anything undefined in array is considered to be 0 */ + for (i = n; i < svc_serv_nrpools(nn->nfsd_serv); ++i) { + err = svc_set_pool_threads(nn->nfsd_serv, + &nn->nfsd_serv->sv_pools[i], + 0, 0); + if (err) + goto out; + } +out: + return err; +} + +/** + * nfsd_svc: start up or shut down the nfsd server + * @n: number of array members in @nthreads + * @nthreads: array of thread counts for each pool + * @net: network namespace to operate within + * @cred: credentials to use for xprt creation + * @scope: server scope value (defaults to nodename) + * + * Adjust the number of threads in each pool and return the new + * total number of threads in the service. + */ +int +nfsd_svc(int n, int *nthreads, struct net *net, const struct cred *cred, const char *scope) +{ + int error; + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct svc_serv *serv; + + lockdep_assert_held(&nfsd_mutex); + + dprintk("nfsd: creating service\n"); + + strscpy(nn->nfsd_name, scope ? scope : utsname()->nodename, + sizeof(nn->nfsd_name)); + + error = nfsd_create_serv(net); + if (error) + goto out; + serv = nn->nfsd_serv; + + error = nfsd_startup_net(net, cred); + if (error) + goto out_put; + error = nfsd_set_nrthreads(n, nthreads, net); + if (error) + goto out_put; + error = serv->sv_nrthreads; +out_put: + if (serv->sv_nrthreads == 0) + nfsd_destroy_serv(net); +out: + return error; +} + +#if defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL) +static bool +nfsd_support_acl_version(int vers) +{ + if (vers >= NFSD_ACL_MINVERS && vers < NFSD_ACL_NRVERS) + return nfsd_acl_version[vers] != NULL; + return false; +} + +static int +nfsd_acl_rpcbind_set(struct net *net, const struct svc_program *progp, + u32 version, int family, unsigned short proto, + unsigned short port) +{ + if (!nfsd_support_acl_version(version) || + !nfsd_vers(net_generic(net, nfsd_net_id), version, NFSD_TEST)) + return 0; + return svc_generic_rpcbind_set(net, progp, version, family, + proto, port); +} + +static __be32 +nfsd_acl_init_request(struct svc_rqst *rqstp, + const struct svc_program *progp, + struct svc_process_info *ret) +{ + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + int i; + + if (likely(nfsd_support_acl_version(rqstp->rq_vers) && + nfsd_vers(nn, rqstp->rq_vers, NFSD_TEST))) + return svc_generic_init_request(rqstp, progp, ret); + + ret->mismatch.lovers = NFSD_ACL_NRVERS; + for (i = NFSD_ACL_MINVERS; i < NFSD_ACL_NRVERS; i++) { + if (nfsd_support_acl_version(i) && + nfsd_vers(nn, i, NFSD_TEST)) { + ret->mismatch.lovers = i; + break; + } + } + if (ret->mismatch.lovers == NFSD_ACL_NRVERS) + return rpc_prog_unavail; + ret->mismatch.hivers = NFSD_ACL_MINVERS; + for (i = NFSD_ACL_NRVERS - 1; i >= NFSD_ACL_MINVERS; i--) { + if (nfsd_support_acl_version(i) && + nfsd_vers(nn, i, NFSD_TEST)) { + ret->mismatch.hivers = i; + break; + } + } + return rpc_prog_mismatch; +} +#endif + +static int +nfsd_rpcbind_set(struct net *net, const struct svc_program *progp, + u32 version, int family, unsigned short proto, + unsigned short port) +{ + if (!nfsd_vers(net_generic(net, nfsd_net_id), version, NFSD_TEST)) + return 0; + return svc_generic_rpcbind_set(net, progp, version, family, + proto, port); +} + +static __be32 +nfsd_init_request(struct svc_rqst *rqstp, + const struct svc_program *progp, + struct svc_process_info *ret) +{ + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + int i; + + if (likely(nfsd_vers(nn, rqstp->rq_vers, NFSD_TEST))) + return svc_generic_init_request(rqstp, progp, ret); + + ret->mismatch.lovers = NFSD_MAXVERS + 1; + for (i = NFSD_MINVERS; i <= NFSD_MAXVERS; i++) { + if (nfsd_vers(nn, i, NFSD_TEST)) { + ret->mismatch.lovers = i; + break; + } + } + if (ret->mismatch.lovers > NFSD_MAXVERS) + return rpc_prog_unavail; + ret->mismatch.hivers = NFSD_MINVERS; + for (i = NFSD_MAXVERS; i >= NFSD_MINVERS; i--) { + if (nfsd_vers(nn, i, NFSD_TEST)) { + ret->mismatch.hivers = i; + break; + } + } + return rpc_prog_mismatch; +} + +/* + * This is the NFS server kernel thread + */ +static int +nfsd(void *vrqstp) +{ + struct svc_rqst *rqstp = (struct svc_rqst *) vrqstp; + struct svc_pool *pool = rqstp->rq_pool; + struct svc_xprt *perm_sock = list_entry(rqstp->rq_server->sv_permsocks.next, typeof(struct svc_xprt), xpt_list); + struct net *net = perm_sock->xpt_net; + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + struct nfsd_thread_local_info ntli = { }; + bool have_mutex = false; + + /* At this point, the thread shares current->fs + * with the init process. We need to create files with the + * umask as defined by the client instead of init's umask. + */ + svc_thread_init_status(rqstp, unshare_fs_struct()); + + current->fs->umask = 0; + + atomic_inc(&nfsd_th_cnt); + + set_freezable(); + + /* use dynamic allocation if ntli should ever become large */ + static_assert(sizeof(struct nfsd_thread_local_info) < 256); + rqstp->rq_private = &ntli; + + /* + * The main request loop + */ + while (!svc_thread_should_stop(rqstp)) { + switch (svc_recv(rqstp, 5 * HZ)) { + case -ETIMEDOUT: + /* No work arrived within the timeout window */ + if (mutex_trylock(&nfsd_mutex)) { + if (pool->sp_nrthreads > pool->sp_nrthrmin) { + trace_nfsd_dynthread_kill(net, pool); + set_bit(RQ_VICTIM, &rqstp->rq_flags); + have_mutex = true; + } else { + mutex_unlock(&nfsd_mutex); + } + } else { + trace_nfsd_dynthread_trylock_fail(net, pool); + } + break; + case -EBUSY: + /* No idle threads; consider spawning another */ + if (pool->sp_nrthreads < pool->sp_nrthrmax) { + if (mutex_trylock(&nfsd_mutex)) { + if (pool->sp_nrthreads < pool->sp_nrthrmax) { + int ret; + + trace_nfsd_dynthread_start(net, pool); + ret = svc_new_thread(rqstp->rq_server, pool); + if (ret) + pr_notice_ratelimited("%s: unable to spawn new thread: %d\n", + __func__, ret); + } + mutex_unlock(&nfsd_mutex); + } else { + trace_nfsd_dynthread_trylock_fail(net, pool); + } + } + clear_bit(SP_TASK_STARTING, &pool->sp_flags); + break; + default: + break; + } + nfsd_file_net_dispose(nn); + } + + atomic_dec(&nfsd_th_cnt); + + /* Release the thread */ + svc_exit_thread(rqstp); + if (have_mutex) + mutex_unlock(&nfsd_mutex); + return 0; +} + +/* + * Set rq_status_counter back to an even value, indicating that the rqstp + * fields are no longer meaningful to a lockless reader. This pairs with the + * odd-valued store made once the request has been decoded, and must run on + * every return path that follows it so that the seq-lock like protocol used + * by nfsd_nl_rpc_status_get_dumpit() is not left permanently odd. The store + * also advances the counter so a concurrent reader detects the transition. + */ +static void nfsd_status_counter_set_idle(struct svc_rqst *rqstp) +{ + smp_store_release(&rqstp->rq_status_counter, + (rqstp->rq_status_counter | 1) + 1); +} + +/** + * nfsd_dispatch - Process an NFS or NFSACL or LOCALIO Request + * @rqstp: incoming request + * + * This RPC dispatcher integrates the NFS server's duplicate reply cache. + * + * Return values: + * %0: Processing complete; do not send a Reply + * %1: Processing complete; send Reply in rqstp->rq_res + */ +int nfsd_dispatch(struct svc_rqst *rqstp) +{ + struct nfsd_thread_local_info *ntli = rqstp->rq_private; + const struct svc_procedure *proc = rqstp->rq_procinfo; + __be32 *statp = rqstp->rq_accept_statp; + struct nfsd_cacherep *rp; + unsigned int start, len; + __be32 *nfs_reply; + + /* + * Give the xdr decoder a chance to change this if it wants + * (necessary in the NFSv4.0 compound case) + */ + ntli->ntli_cachetype = proc->pc_cachetype; + + /* + * ->pc_decode advances the argument stream past the NFS + * Call header, so grab the header's starting location and + * size now for the call to nfsd_cache_lookup(). + */ + start = xdr_stream_pos(&rqstp->rq_arg_stream); + len = xdr_stream_remaining(&rqstp->rq_arg_stream); + if (!proc->pc_decode(rqstp, &rqstp->rq_arg_stream)) + goto out_decode_err; + + /* + * Release rq_status_counter setting it to an odd value after the rpc + * request has been properly parsed. rq_status_counter is used to + * notify the consumers if the rqstp fields are stable + * (rq_status_counter is odd) or not meaningful (rq_status_counter + * is even). + */ + smp_store_release(&rqstp->rq_status_counter, rqstp->rq_status_counter | 1); + + rp = NULL; + switch (nfsd_cache_lookup(rqstp, start, len, &rp)) { + case RC_DOIT: + break; + case RC_REPLY: + goto out_cached_reply; + case RC_DROPIT: + goto out_dropit; + } + + nfs_reply = xdr_inline_decode(&rqstp->rq_res_stream, 0); + *statp = proc->pc_func(rqstp); + if (test_bit(RQ_DROPME, &rqstp->rq_flags)) + goto out_update_drop; + + if (!proc->pc_encode(rqstp, &rqstp->rq_res_stream)) + goto out_encode_err; + + nfsd_cache_update(rqstp, rp, ntli->ntli_cachetype, nfs_reply); +out_cached_reply: + nfsd_status_counter_set_idle(rqstp); + return 1; + +out_decode_err: + trace_nfsd_garbage_args_err(rqstp); + *statp = rpc_garbage_args; + return 1; + +out_update_drop: + nfsd_cache_update(rqstp, rp, RC_NOCACHE, NULL); +out_dropit: + nfsd_status_counter_set_idle(rqstp); + return 0; + +out_encode_err: + trace_nfsd_cant_encode_err(rqstp); + nfsd_cache_update(rqstp, rp, RC_NOCACHE, NULL); + *statp = rpc_system_err; + nfsd_status_counter_set_idle(rqstp); + return 1; +} + +/** + * nfssvc_decode_voidarg - Decode void arguments + * @rqstp: Server RPC transaction context + * @xdr: XDR stream positioned at arguments to decode + * + * Return values: + * %false: Arguments were not valid + * %true: Decoding was successful + */ +bool nfssvc_decode_voidarg(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + return true; +} + +/** + * nfssvc_encode_voidres - Encode void results + * @rqstp: Server RPC transaction context + * @xdr: XDR stream into which to encode results + * + * Return values: + * %false: Local error while encoding + * %true: Encoding was successful + */ +bool nfssvc_encode_voidres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + return true; +} diff --git a/fs/nfsd/nfsxdr.c b/fs/nfsd/nfsxdr.c new file mode 100644 index 000000000..019f0cc97 --- /dev/null +++ b/fs/nfsd/nfsxdr.c @@ -0,0 +1,677 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * XDR support for nfsd + * + * Copyright (C) 1995, 1996 Olaf Kirch + */ + +#include + +#include "vfs.h" +#include "xdr.h" +#include "auth.h" + +/* + * Sun convention: a sattr time-useconds field of one full second (an + * otherwise out-of-range value) means "set this time to the current + * server time." It's needed to make permissions checks for the "touch" + * program across NFSv2 mounts work correctly. See description of + * sattr in section 6.1 of "NFS Illustrated" by Brent Callaghan, + * Addison-Wesley, ISBN 0-201-32750-5 + */ +#define NFS2_SATTR_SET_TO_SERVER_TIME (1000000) + +/* + * Mapping of S_IF* types to NFS file types + */ +static const u32 nfs_ftypes[] = { + NFNON, NFCHR, NFCHR, NFBAD, + NFDIR, NFBAD, NFBLK, NFBAD, + NFREG, NFBAD, NFLNK, NFBAD, + NFSOCK, NFBAD, NFLNK, NFBAD, +}; + + +/* + * Basic NFSv2 data types (RFC 1094 Section 2.3) + */ + +/** + * svcxdr_encode_stat - Encode an NFSv2 status code + * @xdr: XDR stream + * @status: status value to encode + * + * Return values: + * %false: Send buffer space was exhausted + * %true: Success + */ +bool +svcxdr_encode_stat(struct xdr_stream *xdr, __be32 status) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, sizeof(status)); + if (!p) + return false; + *p = status; + + return true; +} + +/** + * svcxdr_decode_fhandle - Decode an NFSv2 file handle + * @xdr: XDR stream positioned at an encoded NFSv2 FH + * @fhp: OUT: filled-in server file handle + * + * Return values: + * %false: The encoded file handle was not valid + * %true: @fhp has been initialized + */ +bool +svcxdr_decode_fhandle(struct xdr_stream *xdr, struct svc_fh *fhp) +{ + __be32 *p; + + p = xdr_inline_decode(xdr, NFS_FHSIZE); + if (!p) + return false; + fh_init(fhp, NFS_FHSIZE); + memcpy(&fhp->fh_handle.fh_raw, p, NFS_FHSIZE); + fhp->fh_handle.fh_size = NFS_FHSIZE; + + return true; +} + +static bool +svcxdr_encode_fhandle(struct xdr_stream *xdr, const struct svc_fh *fhp) +{ + __be32 *p; + + p = xdr_reserve_space(xdr, NFS_FHSIZE); + if (!p) + return false; + memcpy(p, &fhp->fh_handle.fh_raw, NFS_FHSIZE); + + return true; +} + +static __be32 * +encode_timeval(__be32 *p, const struct timespec64 *time) +{ + *p++ = cpu_to_be32((u32)time->tv_sec); + if (time->tv_nsec) + *p++ = cpu_to_be32(time->tv_nsec / NSEC_PER_USEC); + else + *p++ = xdr_zero; + return p; +} + +static bool +svcxdr_decode_filename(struct xdr_stream *xdr, char **name, unsigned int *len) +{ + u32 size, i; + __be32 *p; + char *c; + + if (xdr_stream_decode_u32(xdr, &size) < 0) + return false; + if (size == 0 || size > NFS_MAXNAMLEN) + return false; + p = xdr_inline_decode(xdr, size); + if (!p) + return false; + + *len = size; + *name = (char *)p; + for (i = 0, c = *name; i < size; i++, c++) + if (*c == '\0' || *c == '/') + return false; + + return true; +} + +static bool +svcxdr_decode_diropargs(struct xdr_stream *xdr, struct svc_fh *fhp, + char **name, unsigned int *len) +{ + return svcxdr_decode_fhandle(xdr, fhp) && + svcxdr_decode_filename(xdr, name, len); +} + +static bool +svcxdr_decode_sattr(struct svc_rqst *rqstp, struct xdr_stream *xdr, + struct iattr *iap) +{ + u32 tmp1, tmp2; + __be32 *p; + + p = xdr_inline_decode(xdr, XDR_UNIT * 8); + if (!p) + return false; + + iap->ia_valid = 0; + + /* + * Some Sun clients put 0xffff in the mode field when they + * mean 0xffffffff. + */ + tmp1 = be32_to_cpup(p++); + if (tmp1 != (u32)-1 && tmp1 != 0xffff) { + iap->ia_valid |= ATTR_MODE; + iap->ia_mode = tmp1; + } + + tmp1 = be32_to_cpup(p++); + if (tmp1 != (u32)-1) { + iap->ia_uid = make_kuid(nfsd_user_namespace(rqstp), tmp1); + if (uid_valid(iap->ia_uid)) + iap->ia_valid |= ATTR_UID; + } + + tmp1 = be32_to_cpup(p++); + if (tmp1 != (u32)-1) { + iap->ia_gid = make_kgid(nfsd_user_namespace(rqstp), tmp1); + if (gid_valid(iap->ia_gid)) + iap->ia_valid |= ATTR_GID; + } + + tmp1 = be32_to_cpup(p++); + if (tmp1 != (u32)-1) { + iap->ia_valid |= ATTR_SIZE; + iap->ia_size = tmp1; + } + + tmp1 = be32_to_cpup(p++); + tmp2 = be32_to_cpup(p++); + if (tmp1 != (u32)-1 && tmp2 != (u32)-1) { + /* + * Range test here to prevent the multiplication from + * wrapping to a valid (but incorrect) value on 32-bit + * platforms. + */ + if (tmp2 > NFS2_SATTR_SET_TO_SERVER_TIME) + return false; + iap->ia_valid |= ATTR_ATIME | ATTR_ATIME_SET; + iap->ia_atime.tv_sec = tmp1; + iap->ia_atime.tv_nsec = tmp2 * NSEC_PER_USEC; + if (tmp2 == NFS2_SATTR_SET_TO_SERVER_TIME) + iap->ia_valid &= ~ATTR_ATIME_SET; + } + + tmp1 = be32_to_cpup(p++); + tmp2 = be32_to_cpup(p++); + if (tmp1 != (u32)-1 && tmp2 != (u32)-1) { + if (tmp2 > NFS2_SATTR_SET_TO_SERVER_TIME) + return false; + iap->ia_valid |= ATTR_MTIME | ATTR_MTIME_SET; + iap->ia_mtime.tv_sec = tmp1; + iap->ia_mtime.tv_nsec = tmp2 * NSEC_PER_USEC; + if (tmp2 == NFS2_SATTR_SET_TO_SERVER_TIME) + iap->ia_valid &= ~(ATTR_ATIME_SET|ATTR_MTIME_SET); + } + + return true; +} + +/** + * svcxdr_encode_fattr - Encode NFSv2 file attributes + * @rqstp: Context of a completed RPC transaction + * @xdr: XDR stream + * @fhp: File handle to encode + * @stat: Attributes to encode + * + * Return values: + * %false: Send buffer space was exhausted + * %true: Success + */ +bool +svcxdr_encode_fattr(struct svc_rqst *rqstp, struct xdr_stream *xdr, + const struct svc_fh *fhp, const struct kstat *stat) +{ + struct user_namespace *userns = nfsd_user_namespace(rqstp); + struct dentry *dentry = fhp->fh_dentry; + int type = stat->mode & S_IFMT; + struct timespec64 time; + __be32 *p; + u32 fsid; + + p = xdr_reserve_space(xdr, XDR_UNIT * 17); + if (!p) + return false; + + *p++ = cpu_to_be32(nfs_ftypes[type >> 12]); + *p++ = cpu_to_be32((u32)stat->mode); + *p++ = cpu_to_be32((u32)stat->nlink); + *p++ = cpu_to_be32((u32)from_kuid_munged(userns, stat->uid)); + *p++ = cpu_to_be32((u32)from_kgid_munged(userns, stat->gid)); + + if (S_ISLNK(type) && stat->size > NFS_MAXPATHLEN) + *p++ = cpu_to_be32(NFS_MAXPATHLEN); + else + *p++ = cpu_to_be32((u32) stat->size); + *p++ = cpu_to_be32((u32) stat->blksize); + if (S_ISCHR(type) || S_ISBLK(type)) + *p++ = cpu_to_be32(new_encode_dev(stat->rdev)); + else + *p++ = cpu_to_be32(0xffffffff); + *p++ = cpu_to_be32((u32)stat->blocks); + + switch (fsid_source(fhp)) { + case FSIDSOURCE_FSID: + fsid = (u32)fhp->fh_export->ex_fsid; + break; + case FSIDSOURCE_UUID: + fsid = ((u32 *)fhp->fh_export->ex_uuid)[0]; + fsid ^= ((u32 *)fhp->fh_export->ex_uuid)[1]; + fsid ^= ((u32 *)fhp->fh_export->ex_uuid)[2]; + fsid ^= ((u32 *)fhp->fh_export->ex_uuid)[3]; + break; + default: + fsid = new_encode_dev(stat->dev); + break; + } + *p++ = cpu_to_be32(fsid); + + *p++ = cpu_to_be32((u32)stat->ino); + p = encode_timeval(p, &stat->atime); + time = stat->mtime; + lease_get_mtime(d_inode(dentry), &time); + p = encode_timeval(p, &time); + encode_timeval(p, &stat->ctime); + + return true; +} + +/* + * XDR decode functions + */ + +bool +nfssvc_decode_fhandleargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd_fhandle *args = rqstp->rq_argp; + + return svcxdr_decode_fhandle(xdr, &args->fh); +} + +bool +nfssvc_decode_sattrargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd_sattrargs *args = rqstp->rq_argp; + + return svcxdr_decode_fhandle(xdr, &args->fh) && + svcxdr_decode_sattr(rqstp, xdr, &args->attrs); +} + +bool +nfssvc_decode_diropargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd_diropargs *args = rqstp->rq_argp; + + return svcxdr_decode_diropargs(xdr, &args->fh, &args->name, &args->len); +} + +bool +nfssvc_decode_readargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd_readargs *args = rqstp->rq_argp; + u32 totalcount; + + if (!svcxdr_decode_fhandle(xdr, &args->fh)) + return false; + if (xdr_stream_decode_u32(xdr, &args->offset) < 0) + return false; + if (xdr_stream_decode_u32(xdr, &args->count) < 0) + return false; + /* totalcount is ignored */ + if (xdr_stream_decode_u32(xdr, &totalcount) < 0) + return false; + + return true; +} + +bool +nfssvc_decode_writeargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd_writeargs *args = rqstp->rq_argp; + u32 beginoffset, totalcount; + + if (!svcxdr_decode_fhandle(xdr, &args->fh)) + return false; + /* beginoffset is ignored */ + if (xdr_stream_decode_u32(xdr, &beginoffset) < 0) + return false; + if (xdr_stream_decode_u32(xdr, &args->offset) < 0) + return false; + /* totalcount is ignored */ + if (xdr_stream_decode_u32(xdr, &totalcount) < 0) + return false; + + /* opaque data */ + if (xdr_stream_decode_u32(xdr, &args->len) < 0) + return false; + if (args->len > NFS_MAXDATA) + return false; + + return xdr_stream_subsegment(xdr, &args->payload, args->len); +} + +bool +nfssvc_decode_createargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd_createargs *args = rqstp->rq_argp; + + return svcxdr_decode_diropargs(xdr, &args->fh, + &args->name, &args->len) && + svcxdr_decode_sattr(rqstp, xdr, &args->attrs); +} + +bool +nfssvc_decode_renameargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd_renameargs *args = rqstp->rq_argp; + + return svcxdr_decode_diropargs(xdr, &args->ffh, + &args->fname, &args->flen) && + svcxdr_decode_diropargs(xdr, &args->tfh, + &args->tname, &args->tlen); +} + +bool +nfssvc_decode_linkargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd_linkargs *args = rqstp->rq_argp; + + return svcxdr_decode_fhandle(xdr, &args->ffh) && + svcxdr_decode_diropargs(xdr, &args->tfh, + &args->tname, &args->tlen); +} + +bool +nfssvc_decode_symlinkargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd_symlinkargs *args = rqstp->rq_argp; + struct kvec *head = rqstp->rq_arg.head; + + if (!svcxdr_decode_diropargs(xdr, &args->ffh, &args->fname, &args->flen)) + return false; + if (xdr_stream_decode_u32(xdr, &args->tlen) < 0) + return false; + if (args->tlen == 0) + return false; + + args->first.iov_len = head->iov_len - xdr_stream_pos(xdr); + args->first.iov_base = xdr_inline_decode(xdr, args->tlen); + if (!args->first.iov_base) + return false; + return svcxdr_decode_sattr(rqstp, xdr, &args->attrs); +} + +bool +nfssvc_decode_readdirargs(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd_readdirargs *args = rqstp->rq_argp; + + if (!svcxdr_decode_fhandle(xdr, &args->fh)) + return false; + if (xdr_stream_decode_u32(xdr, &args->cookie) < 0) + return false; + if (xdr_stream_decode_u32(xdr, &args->count) < 0) + return false; + + return true; +} + +/* + * XDR encode functions + */ + +bool +nfssvc_encode_statres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd_stat *resp = rqstp->rq_resp; + + return svcxdr_encode_stat(xdr, resp->status); +} + +bool +nfssvc_encode_attrstatres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd_attrstat *resp = rqstp->rq_resp; + + if (!svcxdr_encode_stat(xdr, resp->status)) + return false; + switch (resp->status) { + case nfs_ok: + if (!svcxdr_encode_fattr(rqstp, xdr, &resp->fh, &resp->stat)) + return false; + break; + } + + return true; +} + +bool +nfssvc_encode_diropres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd_diropres *resp = rqstp->rq_resp; + + if (!svcxdr_encode_stat(xdr, resp->status)) + return false; + switch (resp->status) { + case nfs_ok: + if (!svcxdr_encode_fhandle(xdr, &resp->fh)) + return false; + if (!svcxdr_encode_fattr(rqstp, xdr, &resp->fh, &resp->stat)) + return false; + break; + } + + return true; +} + +bool +nfssvc_encode_readlinkres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd_readlinkres *resp = rqstp->rq_resp; + struct kvec *head = rqstp->rq_res.head; + + if (!svcxdr_encode_stat(xdr, resp->status)) + return false; + switch (resp->status) { + case nfs_ok: + if (xdr_stream_encode_u32(xdr, resp->len) < 0) + return false; + svcxdr_encode_opaque_pages(rqstp, xdr, &resp->page, 0, + resp->len); + if (svc_encode_result_payload(rqstp, head->iov_len, resp->len) < 0) + return false; + break; + } + + return true; +} + +bool +nfssvc_encode_readres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd_readres *resp = rqstp->rq_resp; + struct kvec *head = rqstp->rq_res.head; + + if (!svcxdr_encode_stat(xdr, resp->status)) + return false; + switch (resp->status) { + case nfs_ok: + if (!svcxdr_encode_fattr(rqstp, xdr, &resp->fh, &resp->stat)) + return false; + if (xdr_stream_encode_u32(xdr, resp->count) < 0) + return false; + svcxdr_encode_opaque_pages(rqstp, xdr, resp->pages, + rqstp->rq_res.page_base, + resp->count); + if (svc_encode_result_payload(rqstp, head->iov_len, resp->count) < 0) + return false; + break; + } + + return true; +} + +bool +nfssvc_encode_readdirres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd_readdirres *resp = rqstp->rq_resp; + struct xdr_buf *dirlist = &resp->dirlist; + + if (!svcxdr_encode_stat(xdr, resp->status)) + return false; + switch (resp->status) { + case nfs_ok: + svcxdr_encode_opaque_pages(rqstp, xdr, dirlist->pages, 0, + dirlist->len); + /* no more entries */ + if (xdr_stream_encode_item_absent(xdr) < 0) + return false; + if (xdr_stream_encode_bool(xdr, resp->common.err == nfserr_eof) < 0) + return false; + break; + } + + return true; +} + +bool +nfssvc_encode_statfsres(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + struct nfsd_statfsres *resp = rqstp->rq_resp; + struct kstatfs *stat = &resp->stats; + __be32 *p; + + if (!svcxdr_encode_stat(xdr, resp->status)) + return false; + switch (resp->status) { + case nfs_ok: + p = xdr_reserve_space(xdr, XDR_UNIT * 5); + if (!p) + return false; + *p++ = cpu_to_be32(NFS_MAXDATA); + *p++ = cpu_to_be32(stat->f_bsize); + *p++ = cpu_to_be32(stat->f_blocks); + *p++ = cpu_to_be32(stat->f_bfree); + *p = cpu_to_be32(stat->f_bavail); + break; + } + + return true; +} + +/** + * nfssvc_encode_nfscookie - Encode a directory offset cookie + * @resp: readdir result context + * @offset: offset cookie to encode + * + * The buffer space for the offset cookie has already been reserved + * by svcxdr_encode_entry_common(). + */ +void nfssvc_encode_nfscookie(struct nfsd_readdirres *resp, u32 offset) +{ + __be32 cookie = cpu_to_be32(offset); + + if (!resp->cookie_offset) + return; + + write_bytes_to_xdr_buf(&resp->dirlist, resp->cookie_offset, &cookie, + sizeof(cookie)); + resp->cookie_offset = 0; +} + +static bool +svcxdr_encode_entry_common(struct nfsd_readdirres *resp, const char *name, + int namlen, loff_t offset, u64 ino) +{ + struct xdr_buf *dirlist = &resp->dirlist; + struct xdr_stream *xdr = &resp->xdr; + + if (xdr_stream_encode_item_present(xdr) < 0) + return false; + /* fileid */ + if (xdr_stream_encode_u32(xdr, (u32)ino) < 0) + return false; + /* name */ + if (xdr_stream_encode_opaque(xdr, name, min(namlen, NFS2_MAXNAMLEN)) < 0) + return false; + /* cookie */ + resp->cookie_offset = dirlist->len; + if (xdr_stream_encode_u32(xdr, ~0U) < 0) + return false; + + return true; +} + +/** + * nfssvc_encode_entry - encode one NFSv2 READDIR entry + * @data: directory context + * @name: name of the object to be encoded + * @namlen: length of that name, in bytes + * @offset: the offset of the previous entry + * @ino: the fileid of this entry + * @d_type: unused + * + * Return values: + * %0: Entry was successfully encoded. + * %-EINVAL: An encoding problem occurred, secondary status code in resp->common.err + * + * On exit, the following fields are updated: + * - resp->xdr + * - resp->common.err + * - resp->cookie_offset + */ +int nfssvc_encode_entry(void *data, const char *name, int namlen, + loff_t offset, u64 ino, unsigned int d_type) +{ + struct readdir_cd *ccd = data; + struct nfsd_readdirres *resp = container_of(ccd, + struct nfsd_readdirres, + common); + unsigned int starting_length = resp->dirlist.len; + + /* The offset cookie for the previous entry */ + nfssvc_encode_nfscookie(resp, offset); + + if (!svcxdr_encode_entry_common(resp, name, namlen, offset, ino)) + goto out_toosmall; + + xdr_commit_encode(&resp->xdr); + resp->common.err = nfs_ok; + return 0; + +out_toosmall: + resp->cookie_offset = 0; + resp->common.err = nfserr_toosmall; + resp->dirlist.len = starting_length; + return -EINVAL; +} + +/* + * XDR release functions + */ +void nfssvc_release_attrstat(struct svc_rqst *rqstp) +{ + struct nfsd_attrstat *resp = rqstp->rq_resp; + + fh_put(&resp->fh); +} + +void nfssvc_release_diropres(struct svc_rqst *rqstp) +{ + struct nfsd_diropres *resp = rqstp->rq_resp; + + fh_put(&resp->fh); +} + +void nfssvc_release_readres(struct svc_rqst *rqstp) +{ + struct nfsd_readres *resp = rqstp->rq_resp; + + fh_put(&resp->fh); +} diff --git a/fs/nfsd/pnfs.h b/fs/nfsd/pnfs.h new file mode 100644 index 000000000..f7bee4dc5 --- /dev/null +++ b/fs/nfsd/pnfs.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _FS_NFSD_PNFS_H +#define _FS_NFSD_PNFS_H 1 + +#ifdef CONFIG_NFSD_V4 +#include +#include + +#include "state.h" +#include "xdr4.h" + +struct xdr_stream; + +/* Cap exponential backoff between fence retries at 3 minutes */ +#define MAX_FENCE_DELAY ((unsigned int)(3 * 60 * HZ)) + +struct nfsd4_deviceid_map { + struct list_head hash; + u64 idx; + int fsid_type; + u32 fsid[]; +}; + +struct nfsd4_layout_ops { + u32 notify_types; + bool disable_recalls; + + __be32 (*proc_getdeviceinfo)(struct super_block *sb, + struct svc_rqst *rqstp, + struct nfs4_client *clp, + struct nfsd4_getdeviceinfo *gdevp); + __be32 (*encode_getdeviceinfo)(struct xdr_stream *xdr, + const struct nfsd4_getdeviceinfo *gdevp); + + __be32 (*proc_layoutget)(struct svc_rqst *rqstp, struct inode *inode, + const struct svc_fh *fhp, struct nfsd4_layoutget *lgp); + __be32 (*encode_layoutget)(struct xdr_stream *xdr, + const struct nfsd4_layoutget *lgp); + + __be32 (*proc_layoutcommit)(struct inode *inode, + struct svc_rqst *rqstp, + struct nfsd4_layoutcommit *lcp); + + bool (*fence_client)(struct nfs4_layout_stateid *ls, + struct nfsd_file *file); +}; + +extern const struct nfsd4_layout_ops *nfsd4_layout_ops[]; +#ifdef CONFIG_NFSD_BLOCKLAYOUT +extern const struct nfsd4_layout_ops bl_layout_ops; +#endif +#ifdef CONFIG_NFSD_SCSILAYOUT +extern const struct nfsd4_layout_ops scsi_layout_ops; +#endif +#ifdef CONFIG_NFSD_FLEXFILELAYOUT +extern const struct nfsd4_layout_ops ff_layout_ops; +#endif + +__be32 nfsd4_preprocess_layout_stateid(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, stateid_t *stateid, + bool create, u32 layout_type, struct nfs4_layout_stateid **lsp); +__be32 nfsd4_insert_layout(struct nfsd4_layoutget *lgp, + struct nfs4_layout_stateid *ls); +__be32 nfsd4_return_file_layouts(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, + struct nfsd4_layoutreturn *lrp); +__be32 nfsd4_return_client_layouts(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, + struct nfsd4_layoutreturn *lrp); +int nfsd4_set_deviceid(struct nfsd4_deviceid *id, const struct svc_fh *fhp, + u32 device_generation); +struct nfsd4_deviceid_map *nfsd4_find_devid_map(int idx); +#endif /* CONFIG_NFSD_V4 */ + +#ifdef CONFIG_NFSD_PNFS +void nfsd4_setup_layout_type(struct svc_export *exp); +void nfsd4_return_all_client_layouts(struct nfs4_client *); +void nfsd4_return_all_file_layouts(struct nfs4_client *clp, + struct nfs4_file *fp); +void nfsd4_close_layout(struct nfs4_layout_stateid *ls); +int nfsd4_init_pnfs(void); +void nfsd4_exit_pnfs(void); +#else +struct nfs4_client; +struct nfs4_file; +struct nfs4_layout_stateid; + +static inline void nfsd4_setup_layout_type(struct svc_export *exp) +{ +} + +static inline void nfsd4_return_all_client_layouts(struct nfs4_client *clp) +{ +} +static inline void nfsd4_return_all_file_layouts(struct nfs4_client *clp, + struct nfs4_file *fp) +{ +} +static inline void nfsd4_close_layout(struct nfs4_layout_stateid *ls) +{ +} +static inline void nfsd4_exit_pnfs(void) +{ +} +static inline int nfsd4_init_pnfs(void) +{ + return 0; +} +#endif /* CONFIG_NFSD_PNFS */ +#endif /* _FS_NFSD_PNFS_H */ diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h new file mode 100644 index 000000000..2d00a411c --- /dev/null +++ b/fs/nfsd/state.h @@ -0,0 +1,991 @@ +/* + * Copyright (c) 2001 The Regents of the University of Michigan. + * All rights reserved. + * + * Kendrick Smith + * Andy Adamson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef _NFSD4_STATE_H +#define _NFSD4_STATE_H + +#include + +#include +#include +#include +#include + +#include "nfsfh.h" +#include "nfsd.h" + +typedef struct { + u32 cl_boot; + u32 cl_id; +} clientid_t; + +typedef struct { + clientid_t so_clid; + u32 so_id; +} stateid_opaque_t; + +typedef struct { + u32 si_generation; + stateid_opaque_t si_opaque; +} stateid_t; + +typedef struct { + stateid_t cs_stid; +#define NFS4_COPYNOTIFY_STID 2 + unsigned char cs_type; + refcount_t cs_count; +} copy_stateid_t; + +struct nfsd4_referring_call { + struct list_head __list; + + u32 rc_sequenceid; + u32 rc_slotid; +}; + +struct nfsd4_referring_call_list { + struct list_head __list; + + struct nfs4_sessionid rcl_sessionid; + int __nr_referring_calls; + struct list_head rcl_referring_calls; +}; + +struct nfsd4_callback { + struct nfs4_client *cb_clp; + struct rpc_message cb_msg; +#define NFSD4_CALLBACK_RUNNING (0) +#define NFSD4_CALLBACK_WAKE (1) +#define NFSD4_CALLBACK_REQUEUE (2) + unsigned long cb_flags; + const struct nfsd4_callback_ops *cb_ops; + struct work_struct cb_work; + int cb_seq_status; + int cb_status; + int cb_held_slot; + + int cb_nr_referring_call_list; + struct list_head cb_referring_call_list; +}; + +struct nfsd4_callback_ops { + bool (*prepare)(struct nfsd4_callback *cb); + int (*done)(struct nfsd4_callback *cb, struct rpc_task *task); + void (*release)(struct nfsd4_callback *cb); + uint32_t opcode; +}; + +/* + * A core object that represents a "common" stateid. These are generally + * embedded within the different (more specific) stateid objects and contain + * fields that are of general use to any stateid. + */ +struct nfs4_stid { + refcount_t sc_count; + + /* A new stateid is added to the cl_stateids idr early before it + * is fully initialised. Its sc_type is then zero. After + * initialisation the sc_type it set under cl_lock, and then + * never changes. + */ +#define SC_TYPE_OPEN BIT(0) +#define SC_TYPE_LOCK BIT(1) +#define SC_TYPE_DELEG BIT(2) +#define SC_TYPE_LAYOUT BIT(3) +#define SC_TYPE_COPY BIT(4) + unsigned short sc_type; + +/* nn->deleg_lock protects sc_status for delegation stateids. + * ->cl_lock protects sc_status for open and lock stateids. + * ->st_mutex also protect sc_status for open stateids. + * ->ls_lock protects sc_status for layout stateids. + */ +/* + * For an open stateid kept around *only* to process close replays. + * For deleg stateid, kept in idr until last reference is dropped. + */ +#define SC_STATUS_CLOSED BIT(0) +/* For a deleg stateid kept around only to process free_stateid's: */ +#define SC_STATUS_REVOKED BIT(1) +#define SC_STATUS_ADMIN_REVOKED BIT(2) +#define SC_STATUS_FREEABLE BIT(3) +#define SC_STATUS_FREED BIT(4) + unsigned short sc_status; + + struct list_head sc_cp_list; + stateid_t sc_stateid; + spinlock_t sc_lock; + struct nfs4_client *sc_client; + struct nfs4_file *sc_file; + struct svc_export *sc_export; + void (*sc_free)(struct nfs4_stid *); +}; + +/* Keep a list of stateids issued by the COPY_NOTIFY, associate it with the + * parent OPEN/LOCK/DELEG stateid. + */ +struct nfs4_cpntf_state { + copy_stateid_t cp_stateid; + struct list_head cp_list; /* per parent nfs4_stid */ + stateid_t cp_p_stateid; /* copy of parent's stateid */ + clientid_t cp_p_clid; /* copy of parent's clid */ + time64_t cpntf_time; /* last time stateid used */ +}; + +/* + * RFC 7862 Section 4.8 states: + * + * | A copy offload stateid will be valid until either (A) the client + * | or server restarts or (B) the client returns the resource by + * | issuing an OFFLOAD_CANCEL operation or the client replies to a + * | CB_OFFLOAD operation. + * + * Because a client might not reply to a CB_OFFLOAD, or a reply + * might get lost due to connection loss, NFSD purges async copy + * state after a short period to prevent it from accumulating + * over time. + */ +#define NFSD_COPY_INITIAL_TTL 10 + +struct nfs4_cb_fattr { + struct nfsd4_callback ncf_getattr; + u32 ncf_cb_status; + + /* from CB_GETATTR reply */ + u64 ncf_cb_change; + u64 ncf_cb_fsize; + struct timespec64 ncf_cb_mtime; + struct timespec64 ncf_cb_atime; + + bool ncf_file_modified; + u64 ncf_initial_cinfo; + u64 ncf_cur_fsize; +}; + +/* + * FIXME: the current backchannel encoder can't handle a send buffer longer + * than a single page (see bc_malloc/bc_free). + */ +#define NOTIFY4_EVENT_QUEUE_SIZE 3 +#define NOTIFY4_PAGE_ARRAY_SIZE 1 + +struct nfsd_notify_event { + refcount_t ne_ref; // refcount + u32 ne_mask; // FS_* mask from fsnotify callback + struct dentry *ne_dentry; // dentry reference to target + struct inode *ne_target; // inode overwritten by rename, or NULL + u32 ne_namelen; // length of ne_name (old name for a rename) + u32 ne_newnamelen; // length of new name (rename only), else 0 + char ne_name[]; // entry name, then new name (rename only) +}; + +/* + * For a rename, the new name is snapshotted at event-alloc time and stored + * immediately after the (NUL-terminated) old name in ne_name[]. ne_dentry can + * be renamed again before the CB_NOTIFY work runs, so the new name must not be + * read from the live dentry at encode time. + */ +static inline char *nfsd_notify_event_newname(struct nfsd_notify_event *ne) +{ + return ne->ne_name + ne->ne_namelen + 1; +} + +static inline struct nfsd_notify_event *nfsd_notify_event_get(struct nfsd_notify_event *ne) +{ + refcount_inc(&ne->ne_ref); + return ne; +} + +static inline void nfsd_notify_event_put(struct nfsd_notify_event *ne) +{ + if (refcount_dec_and_test(&ne->ne_ref)) { + iput(ne->ne_target); + dput(ne->ne_dentry); + kfree(ne); + } +} + +/* + * Represents a directory delegation. The callback is for handling CB_NOTIFYs. + * As notifications from fsnotify come in, allocate a new event, take the ncn_lock, + * and add it to the ncn_evt queue. The CB_NOTIFY prepare handler will take the + * lock, clean out the list and process it. + */ +struct nfsd4_cb_notify { + spinlock_t ncn_lock; // protects the evt queue and count + int ncn_evt_cnt; // count of events in ncn_evt + int ncn_nf_cnt; // count of valid entries in ncn_nf + struct nfsd_notify_event *ncn_evt[NOTIFY4_EVENT_QUEUE_SIZE]; // list of events + struct page *ncn_pages[NOTIFY4_PAGE_ARRAY_SIZE]; // for encoding + struct notify4 *ncn_nf; // array of notify4's to be sent + bool ncn_encode_err; // did encoding fail? + struct nfsd4_callback ncn_cb; // notify4 callback +}; + +/* + * Represents a delegation stateid. The nfs4_client holds references to these + * and they are put when it is being destroyed or when the delegation is + * returned by the client: + * + * o 1 reference as long as a delegation is still in force (taken when it's + * alloc'd, put when it's returned or revoked) + * + * o 1 reference as long as a recall rpc is in progress (taken when the lease + * is broken, put when the rpc exits) + * + * o 1 more ephemeral reference for each nfsd thread currently doing something + * with that delegation without holding the cl_lock + * + * If the server attempts to recall a delegation and the client doesn't do so + * before a timeout, the server may also revoke the delegation. In that case, + * the object will either be destroyed (v4.0) or moved to a per-client list of + * revoked delegations (v4.1+). + * + * This object is a superset of the nfs4_stid. + */ +struct nfs4_delegation { + struct nfs4_stid dl_stid; /* must be first field */ + struct list_head dl_perfile; + struct list_head dl_perclnt; + struct list_head dl_recall_lru; /* delegation recalled */ + struct nfs4_clnt_odstate *dl_clnt_odstate; + time64_t dl_time; + u32 dl_type; + /* For recall: */ + int dl_retries; + struct nfsd4_callback dl_recall; + bool dl_recalled; + bool dl_written; + bool dl_setattr; + + union { + /* for CB_GETATTR */ + struct nfs4_cb_fattr dl_cb_fattr; + /* for CB_NOTIFY */ + struct nfsd4_cb_notify dl_cb_notify; + }; + + /* For delegated timestamps */ + struct timespec64 dl_atime; + struct timespec64 dl_mtime; + struct timespec64 dl_ctime; + + /* For dir delegations */ + u32 dl_notify_mask; + u32 dl_child_attrs[2]; + u32 dl_dir_attrs[2]; +}; + +static inline bool deleg_is_read(u32 dl_type) +{ + return (dl_type == OPEN_DELEGATE_READ || dl_type == OPEN_DELEGATE_READ_ATTRS_DELEG); +} + +static inline bool deleg_is_write(u32 dl_type) +{ + return (dl_type == OPEN_DELEGATE_WRITE || dl_type == OPEN_DELEGATE_WRITE_ATTRS_DELEG); +} + +static inline bool deleg_attrs_deleg(u32 dl_type) +{ + return dl_type == OPEN_DELEGATE_READ_ATTRS_DELEG || + dl_type == OPEN_DELEGATE_WRITE_ATTRS_DELEG; +} + +bool nfsd4_vet_deleg_time(struct timespec64 *cb, const struct timespec64 *orig, + const struct timespec64 *now); + +#define cb_to_delegation(cb) \ + container_of(cb, struct nfs4_delegation, dl_recall) + +/* client delegation callback info */ +struct nfs4_cb_conn { + /* SETCLIENTID info */ + struct sockaddr_storage cb_addr; + struct sockaddr_storage cb_saddr; + size_t cb_addrlen; + u32 cb_prog; /* used only in 4.0 case; + per-session otherwise */ + u32 cb_ident; /* minorversion 0 only */ + struct svc_xprt *cb_xprt; /* minorversion 1 only */ +}; + +static inline struct nfs4_delegation *delegstateid(struct nfs4_stid *s) +{ + return container_of(s, struct nfs4_delegation, dl_stid); +} + +/* Maximum number of slots per session. This is for sanity-check only. + * It could be increased if we had a mechanism to shutdown misbehaving clients. + * A large number can be needed to get good throughput on high-latency servers. + */ +#define NFSD_MAX_SLOTS_PER_SESSION 2048 +/* Maximum session per slot cache size */ +#define NFSD_SLOT_CACHE_SIZE 2048 +/* Maximum number of NFSD_SLOT_CACHE_SIZE slots per session */ +#define NFSD_CACHE_SIZE_SLOTS_PER_SESSION 32 +#define NFSD_MAX_MEM_PER_SESSION \ + (NFSD_CACHE_SIZE_SLOTS_PER_SESSION * NFSD_SLOT_CACHE_SIZE) + +struct nfsd4_slot { + u32 sl_seqid; + __be32 sl_status; + struct svc_cred sl_cred; + u32 sl_index; + u32 sl_datalen; + u16 sl_opcnt; + u16 sl_generation; +#define NFSD4_SLOT_INUSE (1 << 0) +#define NFSD4_SLOT_CACHETHIS (1 << 1) +#define NFSD4_SLOT_INITIALIZED (1 << 2) +#define NFSD4_SLOT_CACHED (1 << 3) +#define NFSD4_SLOT_REUSED (1 << 4) + u8 sl_flags; + char sl_data[]; +}; + +struct nfsd4_channel_attrs { + u32 headerpadsz; + u32 maxreq_sz; + u32 maxresp_sz; + u32 maxresp_cached; + u32 maxops; + u32 maxreqs; + u32 nr_rdma_attrs; + u32 rdma_attrs; +}; + +struct nfsd4_cb_sec { + u32 flavor; /* (u32)(-1) used to mean "no valid flavor" */ + kuid_t uid; + kgid_t gid; +}; + +struct nfsd4_create_session { + clientid_t clientid; + struct nfs4_sessionid sessionid; + u32 seqid; + u32 flags; + struct nfsd4_channel_attrs fore_channel; + struct nfsd4_channel_attrs back_channel; + u32 callback_prog; + struct nfsd4_cb_sec cb_sec; +}; + +struct nfsd4_backchannel_ctl { + u32 bc_cb_program; + struct nfsd4_cb_sec bc_cb_sec; +}; + +struct nfsd4_bind_conn_to_session { + struct nfs4_sessionid sessionid; + u32 dir; +}; + +/* The single slot clientid cache structure */ +struct nfsd4_clid_slot { + u32 sl_seqid; + __be32 sl_status; + struct nfsd4_create_session sl_cr_ses; +}; + +struct nfsd4_conn { + struct list_head cn_persession; + struct svc_xprt *cn_xprt; + struct svc_xpt_user cn_xpt_user; + struct nfsd4_session *cn_session; +/* CDFC4_FORE, CDFC4_BACK: */ + unsigned char cn_flags; +}; + +/* Maximum number of slots that nfsd will use in the backchannel */ +#define NFSD_BC_SLOT_TABLE_SIZE (sizeof(u32) * 8) + +/* + * Representation of a v4.1+ session. These are refcounted in a similar fashion + * to the nfs4_client. References are only taken when the server is actively + * working on the object (primarily during the processing of compounds). + */ +struct nfsd4_session { + atomic_t se_ref; + spinlock_t se_lock; + u32 se_cb_slot_avail; /* bitmap of available slots */ + u32 se_cb_highest_slot; /* highest slot client wants */ + u32 se_cb_prog; + struct list_head se_hash; /* hash by sessionid */ + struct list_head se_perclnt; + struct list_head se_all_sessions;/* global list of sessions */ + struct nfs4_client *se_client; + struct nfs4_sessionid se_sessionid; + struct nfsd4_channel_attrs se_fchannel; + struct nfsd4_cb_sec se_cb_sec; + struct list_head se_conns; + u32 se_cb_seq_nr[NFSD_BC_SLOT_TABLE_SIZE]; + struct xarray se_slots; /* forward channel slots */ + u16 se_slot_gen; + bool se_dead; + u32 se_target_maxslots; + struct rcu_head rcu_head; +}; + +/* formatted contents of nfs4_sessionid */ +struct nfsd4_sessionid { + clientid_t clientid; + u32 sequence; + u32 reserved; +}; + +/* Length of MD5 digest as hex, plus terminating '\0' */ +#define HEXDIR_LEN (2 * MD5_DIGEST_SIZE + 1) + +/* + * State Meaning Where set + * -------------------------------------------------------------------------- + * | NFSD4_ACTIVE | Confirmed, active | Default | + * |------------------- ----------------------------------------------------| + * | NFSD4_COURTESY | Courtesy state. | nfs4_get_client_reaplist | + * | | Lease/lock/share | | + * | | reservation conflict | | + * | | can cause Courtesy | | + * | | client to be expired | | + * |------------------------------------------------------------------------| + * | NFSD4_EXPIRABLE | Courtesy client to be| nfs4_laundromat | + * | | expired by Laundromat| try_to_expire_client | + * | | due to conflict | | + * |------------------------------------------------------------------------| + */ +enum { + NFSD4_ACTIVE = 0, + NFSD4_COURTESY, + NFSD4_EXPIRABLE, +}; + +/* + * struct nfs4_client - one per client. Clientids live here. + * + * The initial object created by an NFS client using SETCLIENTID (for NFSv4.0) + * or EXCHANGE_ID (for NFSv4.1+). These objects are refcounted and timestamped. + * Each nfsd_net_ns object contains a set of these and they are tracked via + * short and long form clientid. They are hashed and searched for under the + * per-nfsd_net client_lock spinlock. + * + * References to it are only held during the processing of compounds, and in + * certain other operations. In their "resting state" they have a refcount of + * 0. If they are not renewed within a lease period, they become eligible for + * destruction by the laundromat. + * + * These objects can also be destroyed if the client sends certain forms of + * SETCLIENTID or EXCHANGE_ID operations. + * + * Care is taken *not* to do this however when the objects have an elevated + * refcount. + * + * o Each nfs4_client is hashed by clientid + * + * o Each nfs4_clients is also hashed by name (the opaque quantity initially + * sent by the client to identify itself). + * + * o cl_perclient list is used to ensure no dangling stateowner references + * when we expire the nfs4_client + */ +struct nfs4_client { + struct list_head cl_idhash; /* hash by cl_clientid.id */ + struct rb_node cl_namenode; /* link into by-name trees */ + struct list_head *cl_ownerstr_hashtbl; + struct list_head cl_openowners; + struct idr cl_stateids; /* stateid lookup */ + struct list_head cl_delegations; + struct list_head cl_revoked; /* unacknowledged, revoked 4.1 state */ + struct list_head cl_lru; /* tail queue */ +#ifdef CONFIG_NFSD_PNFS + struct list_head cl_lo_states; /* outstanding layout states */ + bool cl_fence_retry_warn; +#endif + struct xdr_netobj cl_name; /* id generated by client */ + nfs4_verifier cl_verifier; /* generated by client */ + time64_t cl_time; /* time of last lease renewal */ + struct sockaddr_storage cl_addr; /* client ipaddress */ + bool cl_mach_cred; /* SP4_MACH_CRED in force */ + struct svc_cred cl_cred; /* setclientid principal */ + clientid_t cl_clientid; /* generated by server */ + nfs4_verifier cl_confirm; /* generated by server */ + u32 cl_minorversion; + atomic_t cl_admin_revoked; /* count of admin-revoked states */ + /* NFSv4.1 client implementation id: */ + struct xdr_netobj cl_nii_domain; + struct xdr_netobj cl_nii_name; + struct timespec64 cl_nii_time; + + /* for v4.0 and v4.1 callbacks: */ + struct nfs4_cb_conn cl_cb_conn; +#define NFSD4_CLIENT_CB_UPDATE (0) +#define NFSD4_CLIENT_CB_KILL (1) +#define NFSD4_CLIENT_STABLE (2) /* client on stable storage */ +#define NFSD4_CLIENT_RECLAIM_COMPLETE (3) /* reclaim_complete done */ +#define NFSD4_CLIENT_CONFIRMED (4) /* client is confirmed */ +#define NFSD4_CLIENT_UPCALL_LOCK (5) /* upcall serialization */ +#define NFSD4_CLIENT_CB_FLAG_MASK (1 << NFSD4_CLIENT_CB_UPDATE | \ + 1 << NFSD4_CLIENT_CB_KILL) + unsigned long cl_flags; + + struct workqueue_struct *cl_callback_wq; + const struct cred *cl_cb_cred; + struct rpc_clnt *cl_cb_client; + u32 cl_cb_ident; +#define NFSD4_CB_UP 0 +#define NFSD4_CB_UNKNOWN 1 +#define NFSD4_CB_DOWN 2 +#define NFSD4_CB_FAULT 3 + int cl_cb_state; + struct nfsd4_callback cl_cb_null; + struct nfsd4_session __rcu *cl_cb_session; + + /* for all client information that callback code might need: */ + spinlock_t cl_lock; + + /* for nfs41 */ + struct list_head cl_sessions; + struct nfsd4_clid_slot cl_cs_slot; /* create_session slot */ + u32 cl_exchange_flags; + /* number of rpc's in progress over an associated session: */ + atomic_t cl_rpc_users; + struct nfsdfs_client cl_nfsdfs; + struct nfs4_op_map cl_spo_must_allow; + + /* debugging info directory under nfsd/clients/ : */ + struct dentry *cl_nfsd_dentry; + /* 'info' file within that directory. Ref is not counted, + * but will remain valid iff cl_nfsd_dentry != NULL + */ + struct dentry *cl_nfsd_info_dentry; + + struct rpc_wait_queue cl_cb_waitq; /* backchannel callers may */ + /* wait here for slots */ + struct net *net; + struct list_head async_copies; /* list of async copies */ + spinlock_t async_lock; /* lock for async copies */ + atomic_t cl_cb_inflight; /* Outstanding callbacks */ + + unsigned int cl_state; + atomic_t cl_delegs_in_recall; + + struct nfsd4_cb_recall_any *cl_ra; + time64_t cl_ra_time; +#ifdef CONFIG_NFSD_SCSILAYOUT + struct xarray cl_dev_fences; + struct mutex cl_fence_mutex; +#endif +}; + +/* struct nfs4_client_reset + * one per old client. Populates reset_str_hashtbl. Filled from conf_id_hashtbl + * upon lease reset, or from upcall to state_daemon (to read in state + * from non-volitile storage) upon reboot. + */ +struct nfs4_client_reclaim { + struct list_head cr_strhash; /* hash by cr_name */ + struct nfs4_client *cr_clp; /* pointer to associated clp */ + struct xdr_netobj cr_name; /* recovery dir name */ + struct xdr_netobj cr_princhash; +}; + +/* + * REPLAY_ISIZE is sized for an OPEN response with delegation: + * 4(status) + 8(stateid) + 20(changeinfo) + 4(rflags) + + * 8(verifier) + 4(deleg. type) + 8(deleg. stateid) + + * 4(deleg. recall flag) + 20(deleg. space limit) + + * ~32(deleg. ace) = 112 bytes + * + * Some responses can exceed this. A LOCK denial includes the conflicting + * lock owner, which can be up to 1024 bytes (NFS4_OPAQUE_LIMIT). When a + * response exceeds REPLAY_ISIZE, a buffer is dynamically allocated. If + * that allocation fails, only rp_status is saved. Enlarging this constant + * increases the size of every nfs4_stateowner. + */ + +#define NFSD4_REPLAY_ISIZE 112 + +/* + * Replay buffer, where the result of the last seqid-mutating operation + * is cached. + */ +struct nfs4_replay { + __be32 rp_status; + unsigned int rp_buflen; + char *rp_buf; /* rp_ibuf or kmalloc'd */ + struct knfsd_fh rp_openfh; + int rp_locked; + char rp_ibuf[NFSD4_REPLAY_ISIZE]; +}; + +extern void nfs4_replay_free_cache(struct nfs4_replay *rp); + +struct nfs4_stateowner; + +struct nfs4_stateowner_operations { + void (*so_unhash)(struct nfs4_stateowner *); + void (*so_free)(struct nfs4_stateowner *); +}; + +/* + * A core object that represents either an open or lock owner. The object and + * lock owner objects have one of these embedded within them. Refcounts and + * other fields common to both owner types are contained within these + * structures. + */ +struct nfs4_stateowner { + struct list_head so_strhash; + struct list_head so_stateids; + struct nfs4_client *so_client; + const struct nfs4_stateowner_operations *so_ops; + /* after increment in nfsd4_bump_seqid, represents the next + * sequence id expected from the client: */ + atomic_t so_count; + u32 so_seqid; + struct xdr_netobj so_owner; /* open owner name */ + struct nfs4_replay so_replay; + bool so_is_open_owner; +}; + +/* + * When a file is opened, the client provides an open state owner opaque string + * that indicates the "owner" of that open. These objects are refcounted. + * References to it are held by each open state associated with it. This object + * is a superset of the nfs4_stateowner struct. + */ +struct nfs4_openowner { + struct nfs4_stateowner oo_owner; /* must be first field */ + struct list_head oo_perclient; + /* + * We keep around openowners a little while after last close, + * which saves clients from having to confirm, and allows us to + * handle close replays if they come soon enough. The close_lru + * is a list of such openowners, to be reaped by the laundromat + * thread eventually if they remain unused: + */ + struct list_head oo_close_lru; + struct nfs4_ol_stateid *oo_last_closed_stid; + time64_t oo_time; /* time of placement on so_close_lru */ +#define NFS4_OO_CONFIRMED 1 + unsigned char oo_flags; +}; + +/* + * Represents a generic "lockowner". Similar to an openowner. References to it + * are held by the lock stateids that are created on its behalf. This object is + * a superset of the nfs4_stateowner struct. + */ +struct nfs4_lockowner { + struct nfs4_stateowner lo_owner; /* must be first element */ + struct list_head lo_blocked; /* blocked file_locks */ +}; + +static inline struct nfs4_openowner * openowner(struct nfs4_stateowner *so) +{ + return container_of(so, struct nfs4_openowner, oo_owner); +} + +static inline struct nfs4_lockowner * lockowner(struct nfs4_stateowner *so) +{ + return container_of(so, struct nfs4_lockowner, lo_owner); +} + +/* + * Per-client state indicating no. of opens and outstanding delegations + * on a file from a particular client.'od' stands for 'open & delegation' + */ +struct nfs4_clnt_odstate { + struct nfs4_client *co_client; + struct nfs4_file *co_file; + struct list_head co_perfile; + refcount_t co_odcount; +}; + +/* + * nfs4_file: a file opened by some number of (open) nfs4_stateowners. + * + * These objects are global. nfsd keeps one instance of a nfs4_file per + * filehandle (though it may keep multiple file descriptors for each). Each + * inode can have multiple filehandles associated with it, so there is + * (potentially) a many to one relationship between this struct and struct + * inode. + */ +struct nfs4_file { + refcount_t fi_ref; + struct inode * fi_inode; + bool fi_aliased; + spinlock_t fi_lock; + struct rhlist_head fi_rlist; + struct list_head fi_stateids; + union { + struct list_head fi_delegations; + struct rcu_head fi_rcu; + }; + struct list_head fi_clnt_odstate; + /* One each for O_RDONLY, O_WRONLY, O_RDWR: */ + struct nfsd_file *fi_fds[3]; + /* + * Each open or lock stateid contributes 0-4 to the counts + * below depending on which bits are set in st_access_bitmap: + * 1 to fi_access[O_RDONLY] if NFS4_SHARE_ACCES_READ is set + * + 1 to fi_access[O_WRONLY] if NFS4_SHARE_ACCESS_WRITE is set + * + 1 to both of the above if NFS4_SHARE_ACCESS_BOTH is set. + */ + atomic_t fi_access[2]; + u32 fi_share_deny; + struct nfsd_file __rcu *fi_deleg_file; + struct nfsd_file *fi_rdeleg_file; + int fi_delegees; + struct knfsd_fh fi_fhandle; + bool fi_had_conflict; +#ifdef CONFIG_NFSD_PNFS + struct list_head fi_lo_states; + atomic_t fi_lo_recalls; +#endif +}; + +/* + * A generic struct representing either a open or lock stateid. The nfs4_client + * holds a reference to each of these objects, and they in turn hold a + * reference to their respective stateowners. The client's reference is + * released in response to a close or unlock (depending on whether it's an open + * or lock stateid) or when the client is being destroyed. + * + * In the case of v4.0 open stateids, these objects are preserved for a little + * while after close in order to handle CLOSE replays. Those are eventually + * reclaimed via a LRU scheme by the laundromat. + * + * This object is a superset of the nfs4_stid. "ol" stands for "Open or Lock". + * Better suggestions welcome. + */ +struct nfs4_ol_stateid { + struct nfs4_stid st_stid; + struct list_head st_perfile; + struct list_head st_perstateowner; + struct list_head st_locks; + struct nfs4_stateowner *st_stateowner; + struct nfs4_clnt_odstate *st_clnt_odstate; +/* + * These bitmasks use 3 separate bits for READ, ALLOW, and BOTH; see the + * comment above bmap_to_share_mode() for explanation: + */ + unsigned char st_access_bmap; + unsigned char st_deny_bmap; + struct nfs4_ol_stateid *st_openstp; + struct mutex st_mutex; +}; + +static inline struct nfs4_ol_stateid *openlockstateid(struct nfs4_stid *s) +{ + return container_of(s, struct nfs4_ol_stateid, st_stid); +} + +struct nfs4_layout_stateid { + struct nfs4_stid ls_stid; + struct list_head ls_perclnt; + struct list_head ls_perfile; + spinlock_t ls_lock; + struct list_head ls_layouts; + u32 ls_layout_type; + struct nfsd_file *ls_file; + struct nfsd4_callback ls_recall; + stateid_t ls_recall_sid; + bool ls_recalled; + struct mutex ls_mutex; + + struct delayed_work ls_fence_work; + unsigned int ls_fence_delay; + bool ls_fenced; + bool ls_fence_inflight; +}; + +static inline struct nfs4_layout_stateid *layoutstateid(struct nfs4_stid *s) +{ + return container_of(s, struct nfs4_layout_stateid, ls_stid); +} + +/* flags for preprocess_seqid_op() */ +#define RD_STATE 0x00000010 +#define WR_STATE 0x00000020 + +enum nfsd4_cb_op { + NFSPROC4_CLNT_CB_NULL = 0, + NFSPROC4_CLNT_CB_RECALL, + NFSPROC4_CLNT_CB_LAYOUT, + NFSPROC4_CLNT_CB_OFFLOAD, + NFSPROC4_CLNT_CB_SEQUENCE, + NFSPROC4_CLNT_CB_NOTIFY_LOCK, + NFSPROC4_CLNT_CB_RECALL_ANY, + NFSPROC4_CLNT_CB_GETATTR, + NFSPROC4_CLNT_CB_NOTIFY, +}; + +/* Returns true iff a is later than b: */ +static inline bool nfsd4_stateid_generation_after(stateid_t *a, stateid_t *b) +{ + return (s32)(a->si_generation - b->si_generation) > 0; +} + +/* + * When a client tries to get a lock on a file, we set one of these objects + * on the blocking lock. When the lock becomes free, we can then issue a + * CB_NOTIFY_LOCK to the server. + */ +struct nfsd4_blocked_lock { + struct list_head nbl_list; + struct list_head nbl_lru; + time64_t nbl_time; + struct file_lock nbl_lock; + struct knfsd_fh nbl_fh; + struct nfsd4_callback nbl_cb; + struct kref nbl_kref; +}; + +struct nfsd4_compound_state; +struct nfsd_net; +struct nfsd4_copy; +struct nfsd4_async_copy; + +extern __be32 nfs4_preprocess_stateid_op(struct svc_rqst *rqstp, + struct nfsd4_compound_state *cstate, struct svc_fh *fhp, + stateid_t *stateid, int flags, struct nfsd_file **filp, + struct nfs4_stid **cstid); +__be32 nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate, + stateid_t *stateid, unsigned short typemask, + unsigned short statusmask, + struct nfs4_stid **s, struct nfsd_net *nn); +struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *slab, + void (*sc_free)(struct nfs4_stid *)); +struct nfsd4_async_copy *nfs4_alloc_copy_stid(struct nfs4_client *clp); +struct nfs4_cpntf_state *nfs4_alloc_init_cpntf_state(struct nfsd_net *nn, + struct nfs4_stid *p_stid); +void nfs4_put_stid(struct nfs4_stid *s); +void nfs4_inc_and_copy_stateid(stateid_t *dst, struct nfs4_stid *stid); +void nfs4_remove_reclaim_record(struct nfs4_client_reclaim *, struct nfsd_net *); +extern void nfs4_release_reclaim(struct nfsd_net *); +extern struct nfs4_client_reclaim *nfsd4_find_reclaim_client(struct xdr_netobj name, + struct nfsd_net *nn); +extern __be32 nfs4_check_open_reclaim(struct nfs4_client *); +extern void nfsd4_probe_callback(struct nfs4_client *clp); +extern void nfsd4_probe_callback_sync(struct nfs4_client *clp); +extern void nfsd4_change_callback(struct nfs4_client *clp, struct nfs4_cb_conn *); +extern void nfsd41_cb_referring_call(struct nfsd4_callback *cb, + struct nfs4_sessionid *sessionid, + u32 slotid, u32 seqno); +extern void nfsd41_cb_destroy_referring_call_list(struct nfsd4_callback *cb); +extern void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp, + const struct nfsd4_callback_ops *ops, enum nfsd4_cb_op op); +extern bool nfsd4_run_cb(struct nfsd4_callback *cb); + +static inline void nfsd4_try_run_cb(struct nfsd4_callback *cb) +{ + if (!test_and_set_bit(NFSD4_CALLBACK_RUNNING, &cb->cb_flags)) + WARN_ON_ONCE(!nfsd4_run_cb(cb)); +} + +extern void nfsd4_shutdown_callback(struct nfs4_client *); +extern void nfsd4_shutdown_copy(struct nfs4_client *clp); +void nfsd4_put_client(struct nfs4_client *clp); +void nfsd4_async_copy_reaper(struct nfsd_net *nn); +bool nfsd4_has_active_async_copies(struct nfs4_client *clp); +void nfsd_update_cmtime_attr(struct file *f, unsigned int flags); +extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(struct xdr_netobj name, + struct xdr_netobj princhash, struct nfsd_net *nn); +extern bool nfs4_has_reclaimed_state(struct xdr_netobj name, struct nfsd_net *nn); +int nfsd_handle_dir_event(u32 mask, const struct inode *dir, const void *data, + int data_type, const struct qstr *name); + +void put_nfs4_file(struct nfs4_file *fi); +extern void nfs4_put_cpntf_state(struct nfsd_net *nn, + struct nfs4_cpntf_state *cps); +extern __be32 manage_cpntf_state(struct nfsd_net *nn, stateid_t *st, + struct nfs4_client *clp, + struct nfs4_cpntf_state **cps); +static inline void get_nfs4_file(struct nfs4_file *fi) +{ + refcount_inc(&fi->fi_ref); +} +struct nfsd_file *find_any_file(struct nfs4_file *f); + +#ifdef CONFIG_NFSD_V4 +void nfsd4_revoke_states(struct nfsd_net *nn, struct super_block *sb); +void nfsd4_revoke_export_states(struct nfsd_net *nn, const struct path *path); +void nfsd4_cancel_copy_by_sb(struct net *net, struct super_block *sb); +int nfsd_net_cb_init(struct nfsd_net *nn); +void nfsd_net_cb_shutdown(struct nfsd_net *nn); +#else +static inline void nfsd4_revoke_states(struct nfsd_net *nn, struct super_block *sb) +{ +} +static inline void nfsd4_revoke_export_states(struct nfsd_net *nn, + const struct path *path) +{ +} +static inline void nfsd4_cancel_copy_by_sb(struct net *net, struct super_block *sb) +{ +} +static inline int nfsd_net_cb_init(struct nfsd_net *nn) +{ + return 0; +} +static inline void nfsd_net_cb_shutdown(struct nfsd_net *nn) +{ +} +#endif + +/* grace period management */ +bool nfsd4_force_end_grace(struct nfsd_net *nn); + +/* nfs4recover operations */ +extern int nfsd4_client_tracking_init(struct net *net); +extern void nfsd4_client_tracking_exit(struct net *net); +extern void nfsd4_client_record_create(struct nfs4_client *clp); +extern void nfsd4_client_record_remove(struct nfs4_client *clp); +extern int nfsd4_client_record_check(struct nfs4_client *clp); +extern void nfsd4_record_grace_done(struct nfsd_net *nn); + +static inline bool try_to_expire_client(struct nfs4_client *clp) +{ + cmpxchg(&clp->cl_state, NFSD4_COURTESY, NFSD4_EXPIRABLE); + return clp->cl_state == NFSD4_EXPIRABLE; +} + +extern __be32 nfsd4_deleg_getattr_conflict(struct svc_rqst *rqstp, + struct dentry *dentry, struct nfs4_delegation **pdp); + +struct nfsd4_get_dir_delegation; +struct nfs4_delegation *nfsd_get_dir_deleg(struct nfsd4_compound_state *cstate, + struct nfsd4_get_dir_delegation *gdd, + struct nfsd_file *nf); +#endif /* NFSD4_STATE_H */ diff --git a/fs/nfsd/stats.c b/fs/nfsd/stats.c new file mode 100644 index 000000000..f16559813 --- /dev/null +++ b/fs/nfsd/stats.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * procfs-based user access to knfsd statistics + * + * /proc/net/rpc/nfsd + * + * Format: + * rc + * Statistsics for the reply cache + * fh + * statistics for filehandle lookup + * io + * statistics for IO throughput + * th + * number of threads + * ra + * + * plus generic RPC stats (see net/sunrpc/stats.c) + * + * Copyright (C) 1995, 1996, 1997 Olaf Kirch + */ + +#include +#include +#include +#include + +#include "nfsd.h" +#include "netns.h" +#include "stats.h" + +static int nfsd_show(struct seq_file *seq, void *v) +{ + struct net *net = pde_data(file_inode(seq->file)); + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + int i; + + seq_printf(seq, "rc %lld %lld %lld\nfh %lld 0 0 0 0\nio %lld %lld\n", + percpu_counter_sum_positive(&nn->counter[NFSD_STATS_RC_HITS]), + percpu_counter_sum_positive(&nn->counter[NFSD_STATS_RC_MISSES]), + percpu_counter_sum_positive(&nn->counter[NFSD_STATS_RC_NOCACHE]), + percpu_counter_sum_positive(&nn->counter[NFSD_STATS_FH_STALE]), + percpu_counter_sum_positive(&nn->counter[NFSD_STATS_IO_READ]), + percpu_counter_sum_positive(&nn->counter[NFSD_STATS_IO_WRITE])); + + /* thread usage: */ + seq_printf(seq, "th %u 0", atomic_read(&nfsd_th_cnt)); + + /* deprecated thread usage histogram stats */ + for (i = 0; i < 10; i++) + seq_puts(seq, " 0.000"); + + /* deprecated ra-cache stats */ + seq_puts(seq, "\nra 0 0 0 0 0 0 0 0 0 0 0 0\n"); + + /* show my rpc info */ + svc_seq_show(seq, &nn->nfsd_svcstats); + +#ifdef CONFIG_NFSD_V4 + /* Show count for individual nfsv4 operations */ + /* Writing operation numbers 0 1 2 also for maintaining uniformity */ + seq_printf(seq, "proc4ops %u", LAST_NFS4_OP + 1); + for (i = 0; i <= LAST_NFS4_OP; i++) { + seq_printf(seq, " %lld", + percpu_counter_sum_positive(&nn->counter[NFSD_STATS_NFS4_OP(i)])); + } + seq_printf(seq, "\nwdeleg_getattr %lld", + percpu_counter_sum_positive(&nn->cb_counter[OP_CB_GETATTR])); + + seq_putc(seq, '\n'); +#endif + + return 0; +} + +DEFINE_PROC_SHOW_ATTRIBUTE(nfsd); + +struct proc_dir_entry *nfsd_proc_stat_init(struct net *net) +{ + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + + return svc_proc_register(net, &nn->nfsd_svcstats, &nfsd_proc_ops); +} + +void nfsd_proc_stat_shutdown(struct net *net) +{ + svc_proc_unregister(net, "nfsd"); +} diff --git a/fs/nfsd/stats.h b/fs/nfsd/stats.h new file mode 100644 index 000000000..aabfbb1a9 --- /dev/null +++ b/fs/nfsd/stats.h @@ -0,0 +1,140 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Statistics for NFS server. + * + * Copyright (C) 1995, 1996 Olaf Kirch + */ +#ifndef _NFSD_STATS_H +#define _NFSD_STATS_H + +#include +#include + +#include "export.h" +#include "netns.h" + +struct proc_dir_entry *nfsd_proc_stat_init(struct net *net); +void nfsd_proc_stat_shutdown(struct net *net); + +/** + * nfsd_stats_rc_hits_inc - Count a duplicate reply cache hit + * @nn: target network namespace + * + * These reply cache counters are updated once per RPC. Readers use + * percpu_counter_sum_positive(), so local batching does not affect + * read accuracy. + */ +static inline void nfsd_stats_rc_hits_inc(struct nfsd_net *nn) +{ + percpu_counter_add_local(&nn->counter[NFSD_STATS_RC_HITS], 1); +} + +/** + * nfsd_stats_rc_misses_inc - Count a duplicate reply cache miss + * @nn: target network namespace + * + * See nfsd_stats_rc_hits_inc() for batching rationale. + */ +static inline void nfsd_stats_rc_misses_inc(struct nfsd_net *nn) +{ + percpu_counter_add_local(&nn->counter[NFSD_STATS_RC_MISSES], 1); +} + +/** + * nfsd_stats_rc_nocache_inc - Count a request not cached in the reply cache + * @nn: target network namespace + * + * See nfsd_stats_rc_hits_inc() for batching rationale. + */ +static inline void nfsd_stats_rc_nocache_inc(struct nfsd_net *nn) +{ + percpu_counter_add_local(&nn->counter[NFSD_STATS_RC_NOCACHE], 1); +} + +static inline void nfsd_stats_fh_stale_inc(struct nfsd_net *nn, + struct svc_export *exp) +{ + percpu_counter_inc(&nn->counter[NFSD_STATS_FH_STALE]); + if (exp && exp->ex_stats) + percpu_counter_inc(&exp->ex_stats->counter[EXP_STATS_FH_STALE]); +} + +/** + * nfsd_stats_io_read_add - Count number of bytes for an NFS READ + * @nn: target network namespace + * @exp: target export + * @amount: byte count + * + * These counters are updated on every READ request. Readers use + * percpu_counter_sum_positive(), so local batching does not affect + * read accuracy. + */ +static inline void nfsd_stats_io_read_add(struct nfsd_net *nn, + struct svc_export *exp, s64 amount) +{ + percpu_counter_add_local(&nn->counter[NFSD_STATS_IO_READ], amount); + if (exp && exp->ex_stats) + percpu_counter_add_local(&exp->ex_stats->counter[EXP_STATS_IO_READ], + amount); +} + +/** + * nfsd_stats_io_write_add - Count number of bytes for an NFS WRITE + * @nn: target network namespace + * @exp: target export + * @amount: byte count + * + * These counters are updated on every WRITE request. Readers use + * percpu_counter_sum_positive(), so local batching does not affect + * read accuracy. + */ +static inline void nfsd_stats_io_write_add(struct nfsd_net *nn, + struct svc_export *exp, s64 amount) +{ + percpu_counter_add_local(&nn->counter[NFSD_STATS_IO_WRITE], amount); + if (exp && exp->ex_stats) + percpu_counter_add_local(&exp->ex_stats->counter[EXP_STATS_IO_WRITE], + amount); +} + +static inline void nfsd_stats_payload_misses_inc(struct nfsd_net *nn) +{ + percpu_counter_inc(&nn->counter[NFSD_STATS_PAYLOAD_MISSES]); +} + +/** + * nfsd_stats_drc_mem_usage_add - Add memory used by a cache item + * @nn: target network namespace + * @amount: byte count + * + * percpu_counter_add_local() keeps updates on the per-CPU fast + * path. The sole reader, percpu_counter_sum_positive(), sums the + * per-CPU deltas, so batching locally does not lose accuracy. + */ +static inline void nfsd_stats_drc_mem_usage_add(struct nfsd_net *nn, s64 amount) +{ + percpu_counter_add_local(&nn->counter[NFSD_STATS_DRC_MEM_USAGE], + amount); +} + +/** + * nfsd_stats_drc_mem_usage_sub - Subtract memory used by a cache item + * @nn: target network namespace + * @amount: byte count + * + * See nfsd_stats_drc_mem_usage_add() for batching rationale. + */ +static inline void nfsd_stats_drc_mem_usage_sub(struct nfsd_net *nn, s64 amount) +{ + percpu_counter_sub_local(&nn->counter[NFSD_STATS_DRC_MEM_USAGE], + amount); +} + +#ifdef CONFIG_NFSD_V4 +static inline void nfsd_stats_cb_op_inc(struct nfsd_net *nn, u32 opcode) +{ + if (opcode >= OP_CB_GETATTR && opcode <= OP_CB_OFFLOAD) + percpu_counter_inc(&nn->cb_counter[opcode]); +} +#endif +#endif /* _NFSD_STATS_H */ diff --git a/fs/nfsd/trace.c b/fs/nfsd/trace.c new file mode 100644 index 000000000..f008b95ce --- /dev/null +++ b/fs/nfsd/trace.c @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define CREATE_TRACE_POINTS +#include "trace.h" diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h new file mode 100644 index 000000000..7d7a14831 --- /dev/null +++ b/fs/nfsd/trace.h @@ -0,0 +1,2780 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2014 Christoph Hellwig. + */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM nfsd + +#if !defined(_NFSD_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _NFSD_TRACE_H + +#include +#include +#include +#include +#include +#include +#include + +#include "export.h" +#include "nfsfh.h" +#include "xdr4.h" +#include "netns.h" + +#define NFSD_TRACE_PROC_CALL_FIELDS(r) \ + __field(unsigned int, netns_ino) \ + __field(u32, xid) \ + __sockaddr(server, (r)->rq_xprt->xpt_locallen) \ + __sockaddr(client, (r)->rq_xprt->xpt_remotelen) + +#define NFSD_TRACE_PROC_CALL_ASSIGNMENTS(r) \ + do { \ + struct svc_xprt *xprt = (r)->rq_xprt; \ + __entry->netns_ino = SVC_NET(r)->ns.inum; \ + __entry->xid = be32_to_cpu((r)->rq_xid); \ + __assign_sockaddr(server, &xprt->xpt_local, \ + xprt->xpt_locallen); \ + __assign_sockaddr(client, &xprt->xpt_remote, \ + xprt->xpt_remotelen); \ + } while (0) + +#define NFSD_TRACE_PROC_RES_FIELDS(r) \ + __field(unsigned int, netns_ino) \ + __field(u32, xid) \ + __field(unsigned long, status) \ + __sockaddr(server, (r)->rq_xprt->xpt_locallen) \ + __sockaddr(client, (r)->rq_xprt->xpt_remotelen) + +#define NFSD_TRACE_PROC_RES_ASSIGNMENTS(r, error) \ + do { \ + struct svc_xprt *xprt = (r)->rq_xprt; \ + __entry->netns_ino = SVC_NET(r)->ns.inum; \ + __entry->xid = be32_to_cpu((r)->rq_xid); \ + __entry->status = be32_to_cpu(error); \ + __assign_sockaddr(server, &xprt->xpt_local, \ + xprt->xpt_locallen); \ + __assign_sockaddr(client, &xprt->xpt_remote, \ + xprt->xpt_remotelen); \ + } while (0); + +DECLARE_EVENT_CLASS(nfsd_xdr_err_class, + TP_PROTO( + const struct svc_rqst *rqstp + ), + TP_ARGS(rqstp), + TP_STRUCT__entry( + __field(unsigned int, netns_ino) + __field(u32, xid) + __field(u32, vers) + __field(u32, proc) + __sockaddr(server, rqstp->rq_xprt->xpt_locallen) + __sockaddr(client, rqstp->rq_xprt->xpt_remotelen) + ), + TP_fast_assign( + const struct svc_xprt *xprt = rqstp->rq_xprt; + + __entry->netns_ino = xprt->xpt_net->ns.inum; + __entry->xid = be32_to_cpu(rqstp->rq_xid); + __entry->vers = rqstp->rq_vers; + __entry->proc = rqstp->rq_proc; + __assign_sockaddr(server, &xprt->xpt_local, xprt->xpt_locallen); + __assign_sockaddr(client, &xprt->xpt_remote, xprt->xpt_remotelen); + ), + TP_printk("xid=0x%08x vers=%u proc=%u", + __entry->xid, __entry->vers, __entry->proc + ) +); + +#define DEFINE_NFSD_XDR_ERR_EVENT(name) \ +DEFINE_EVENT(nfsd_xdr_err_class, nfsd_##name##_err, \ + TP_PROTO(const struct svc_rqst *rqstp), \ + TP_ARGS(rqstp)) + +DEFINE_NFSD_XDR_ERR_EVENT(garbage_args); +DEFINE_NFSD_XDR_ERR_EVENT(cant_encode); + +DECLARE_EVENT_CLASS(nfsd_dynthread_class, + TP_PROTO( + const struct net *net, + const struct svc_pool *pool + ), + TP_ARGS(net, pool), + TP_STRUCT__entry( + __field(unsigned int, netns_ino) + __field(unsigned int, pool_id) + __field(unsigned int, nrthreads) + __field(unsigned int, nrthrmin) + __field(unsigned int, nrthrmax) + ), + TP_fast_assign( + __entry->netns_ino = net->ns.inum; + __entry->pool_id = pool->sp_id; + __entry->nrthreads = pool->sp_nrthreads; + __entry->nrthrmin = pool->sp_nrthrmin; + __entry->nrthrmax = pool->sp_nrthrmax; + ), + TP_printk("pool=%u nrthreads=%u nrthrmin=%u nrthrmax=%u", + __entry->pool_id, __entry->nrthreads, + __entry->nrthrmin, __entry->nrthrmax + ) +); + +#define DEFINE_NFSD_DYNTHREAD_EVENT(name) \ +DEFINE_EVENT(nfsd_dynthread_class, nfsd_dynthread_##name, \ + TP_PROTO(const struct net *net, const struct svc_pool *pool), \ + TP_ARGS(net, pool)) + +DEFINE_NFSD_DYNTHREAD_EVENT(start); +DEFINE_NFSD_DYNTHREAD_EVENT(kill); +DEFINE_NFSD_DYNTHREAD_EVENT(trylock_fail); + +#define show_nfsd_may_flags(x) \ + __print_flags(x, "|", \ + { NFSD_MAY_EXEC, "EXEC" }, \ + { NFSD_MAY_WRITE, "WRITE" }, \ + { NFSD_MAY_READ, "READ" }, \ + { NFSD_MAY_SATTR, "SATTR" }, \ + { NFSD_MAY_TRUNC, "TRUNC" }, \ + { NFSD_MAY_NLM, "NLM" }, \ + { NFSD_MAY_OWNER_OVERRIDE, "OWNER_OVERRIDE" }, \ + { NFSD_MAY_LOCAL_ACCESS, "LOCAL_ACCESS" }, \ + { NFSD_MAY_BYPASS_GSS_ON_ROOT, "BYPASS_GSS_ON_ROOT" }, \ + { NFSD_MAY_NOT_BREAK_LEASE, "NOT_BREAK_LEASE" }, \ + { NFSD_MAY_BYPASS_GSS, "BYPASS_GSS" }, \ + { NFSD_MAY_READ_IF_EXEC, "READ_IF_EXEC" }, \ + { NFSD_MAY_64BIT_COOKIE, "64BIT_COOKIE" }, \ + { NFSD_MAY_LOCALIO, "LOCALIO" }) + +TRACE_EVENT(nfsd_compound, + TP_PROTO( + const struct svc_rqst *rqst, + const char *tag, + u32 taglen, + u32 opcnt + ), + TP_ARGS(rqst, tag, taglen, opcnt), + TP_STRUCT__entry( + __field(u32, xid) + __field(u32, opcnt) + __string_len(tag, tag, taglen) + ), + TP_fast_assign( + __entry->xid = be32_to_cpu(rqst->rq_xid); + __entry->opcnt = opcnt; + __assign_str(tag); + ), + TP_printk("xid=0x%08x opcnt=%u tag=%s", + __entry->xid, __entry->opcnt, __get_str(tag) + ) +) + +TRACE_EVENT(nfsd_compound_status, + TP_PROTO(u32 args_opcnt, + u32 resp_opcnt, + __be32 status, + const char *name), + TP_ARGS(args_opcnt, resp_opcnt, status, name), + TP_STRUCT__entry( + __field(u32, args_opcnt) + __field(u32, resp_opcnt) + __field(int, status) + __string(name, name) + ), + TP_fast_assign( + __entry->args_opcnt = args_opcnt; + __entry->resp_opcnt = resp_opcnt; + __entry->status = be32_to_cpu(status); + __assign_str(name); + ), + TP_printk("op=%u/%u %s status=%d", + __entry->resp_opcnt, __entry->args_opcnt, + __get_str(name), __entry->status) +) + +TRACE_EVENT(nfsd_compound_decode_err, + TP_PROTO( + const struct svc_rqst *rqstp, + u32 args_opcnt, + u32 resp_opcnt, + u32 opnum, + __be32 status + ), + TP_ARGS(rqstp, args_opcnt, resp_opcnt, opnum, status), + TP_STRUCT__entry( + NFSD_TRACE_PROC_RES_FIELDS(rqstp) + + __field(u32, args_opcnt) + __field(u32, resp_opcnt) + __field(u32, opnum) + ), + TP_fast_assign( + NFSD_TRACE_PROC_RES_ASSIGNMENTS(rqstp, status) + + __entry->args_opcnt = args_opcnt; + __entry->resp_opcnt = resp_opcnt; + __entry->opnum = opnum; + ), + TP_printk("op=%u/%u opnum=%u status=%lu", + __entry->resp_opcnt, __entry->args_opcnt, + __entry->opnum, __entry->status) +); + +DECLARE_EVENT_CLASS(nfsd_compound_err_class, + TP_PROTO( + const struct svc_rqst *rqstp, + u32 opnum, + __be32 status + ), + TP_ARGS(rqstp, opnum, status), + TP_STRUCT__entry( + NFSD_TRACE_PROC_RES_FIELDS(rqstp) + + __field(u32, opnum) + ), + TP_fast_assign( + NFSD_TRACE_PROC_RES_ASSIGNMENTS(rqstp, status) + + __entry->opnum = opnum; + ), + TP_printk("opnum=%u status=%lu", + __entry->opnum, __entry->status) +); + +#define DEFINE_NFSD_COMPOUND_ERR_EVENT(name) \ +DEFINE_EVENT(nfsd_compound_err_class, nfsd_compound_##name##_err, \ + TP_PROTO( \ + const struct svc_rqst *rqstp, \ + u32 opnum, \ + __be32 status \ + ), \ + TP_ARGS(rqstp, opnum, status)) + +DEFINE_NFSD_COMPOUND_ERR_EVENT(op); +DEFINE_NFSD_COMPOUND_ERR_EVENT(encode); + +#define show_fs_file_type(x) \ + __print_symbolic(x, \ + { S_IFLNK, "LNK" }, \ + { S_IFREG, "REG" }, \ + { S_IFDIR, "DIR" }, \ + { S_IFCHR, "CHR" }, \ + { S_IFBLK, "BLK" }, \ + { S_IFIFO, "FIFO" }, \ + { S_IFSOCK, "SOCK" }) + +TRACE_EVENT_CONDITION(nfsd_fh_verify, + TP_PROTO( + const struct svc_rqst *rqstp, + const struct svc_fh *fhp, + umode_t type, + int access + ), + TP_ARGS(rqstp, fhp, type, access), + TP_CONDITION(rqstp != NULL), + TP_STRUCT__entry( + __field(unsigned int, netns_ino) + __sockaddr(server, rqstp->rq_xprt->xpt_locallen) + __sockaddr(client, rqstp->rq_xprt->xpt_remotelen) + __field(u32, xid) + __field(u32, fh_hash) + __field(const void *, inode) + __field(unsigned long, type) + __field(unsigned long, access) + ), + TP_fast_assign( + __entry->netns_ino = SVC_NET(rqstp)->ns.inum; + __assign_sockaddr(server, &rqstp->rq_xprt->xpt_local, + rqstp->rq_xprt->xpt_locallen); + __assign_sockaddr(client, &rqstp->rq_xprt->xpt_remote, + rqstp->rq_xprt->xpt_remotelen); + __entry->xid = be32_to_cpu(rqstp->rq_xid); + __entry->fh_hash = knfsd_fh_hash(&fhp->fh_handle); + __entry->inode = d_inode(fhp->fh_dentry); + __entry->type = type; + __entry->access = access; + ), + TP_printk("xid=0x%08x fh_hash=0x%08x type=%s access=%s", + __entry->xid, __entry->fh_hash, + show_fs_file_type(__entry->type), + show_nfsd_may_flags(__entry->access) + ) +); + +TRACE_EVENT_CONDITION(nfsd_fh_verify_err, + TP_PROTO( + const struct svc_rqst *rqstp, + const struct svc_fh *fhp, + umode_t type, + int access, + __be32 error + ), + TP_ARGS(rqstp, fhp, type, access, error), + TP_CONDITION(rqstp != NULL && error), + TP_STRUCT__entry( + __field(unsigned int, netns_ino) + __sockaddr(server, rqstp->rq_xprt->xpt_locallen) + __sockaddr(client, rqstp->rq_xprt->xpt_remotelen) + __field(u32, xid) + __field(u32, fh_hash) + __field(const void *, inode) + __field(unsigned long, type) + __field(unsigned long, access) + __field(int, error) + ), + TP_fast_assign( + __entry->netns_ino = SVC_NET(rqstp)->ns.inum; + __assign_sockaddr(server, &rqstp->rq_xprt->xpt_local, + rqstp->rq_xprt->xpt_locallen); + __assign_sockaddr(client, &rqstp->rq_xprt->xpt_remote, + rqstp->rq_xprt->xpt_remotelen); + __entry->xid = be32_to_cpu(rqstp->rq_xid); + __entry->fh_hash = knfsd_fh_hash(&fhp->fh_handle); + if (fhp->fh_dentry) + __entry->inode = d_inode(fhp->fh_dentry); + else + __entry->inode = NULL; + __entry->type = type; + __entry->access = access; + __entry->error = be32_to_cpu(error); + ), + TP_printk("xid=0x%08x fh_hash=0x%08x type=%s access=%s error=%d", + __entry->xid, __entry->fh_hash, + show_fs_file_type(__entry->type), + show_nfsd_may_flags(__entry->access), + __entry->error + ) +); + +DECLARE_EVENT_CLASS(nfsd_fh_err_class, + TP_PROTO(struct svc_rqst *rqstp, + struct svc_fh *fhp, + int status), + TP_ARGS(rqstp, fhp, status), + TP_STRUCT__entry( + __field(u32, xid) + __field(u32, fh_hash) + __field(int, status) + ), + TP_fast_assign( + __entry->xid = be32_to_cpu(rqstp->rq_xid); + __entry->fh_hash = knfsd_fh_hash(&fhp->fh_handle); + __entry->status = status; + ), + TP_printk("xid=0x%08x fh_hash=0x%08x status=%d", + __entry->xid, __entry->fh_hash, + __entry->status) +) + +#define DEFINE_NFSD_FH_ERR_EVENT(name) \ +DEFINE_EVENT_CONDITION(nfsd_fh_err_class, nfsd_##name, \ + TP_PROTO(struct svc_rqst *rqstp, \ + struct svc_fh *fhp, \ + int status), \ + TP_ARGS(rqstp, fhp, status), \ + TP_CONDITION(rqstp != NULL)) + +DEFINE_NFSD_FH_ERR_EVENT(set_fh_dentry_badexport); +DEFINE_NFSD_FH_ERR_EVENT(set_fh_dentry_badhandle); +DEFINE_NFSD_FH_ERR_EVENT(set_fh_dentry_badmac); + +TRACE_EVENT(nfsd_exp_find_key, + TP_PROTO(const struct svc_expkey *key, + int status), + TP_ARGS(key, status), + TP_STRUCT__entry( + __field(u8, fsidtype) + __array(u32, fsid, 6) + __string(auth_domain, key->ek_client->name) + __field(int, status) + ), + TP_fast_assign( + __entry->fsidtype = key->ek_fsidtype; + memcpy(__entry->fsid, key->ek_fsid, 4*6); + __assign_str(auth_domain); + __entry->status = status; + ), + TP_printk("fsid=%x::%s domain=%s status=%d", + __entry->fsidtype, + __print_array(__entry->fsid, 6, 4), + __get_str(auth_domain), + __entry->status + ) +); + +TRACE_EVENT(nfsd_expkey_update, + TP_PROTO(const struct svc_expkey *key, const char *exp_path), + TP_ARGS(key, exp_path), + TP_STRUCT__entry( + __field(u8, fsidtype) + __array(u32, fsid, 6) + __string(auth_domain, key->ek_client->name) + __string(path, exp_path) + __field(bool, cache) + ), + TP_fast_assign( + __entry->fsidtype = key->ek_fsidtype; + memcpy(__entry->fsid, key->ek_fsid, 4*6); + __assign_str(auth_domain); + __assign_str(path); + __entry->cache = !test_bit(CACHE_NEGATIVE, &key->h.flags); + ), + TP_printk("fsid=%x::%s domain=%s path=%s cache=%s", + __entry->fsidtype, + __print_array(__entry->fsid, 6, 4), + __get_str(auth_domain), + __get_str(path), + __entry->cache ? "pos" : "neg" + ) +); + +TRACE_EVENT(nfsd_exp_get_by_name, + TP_PROTO(const struct svc_export *key, + int status), + TP_ARGS(key, status), + TP_STRUCT__entry( + __string(path, key->ex_path.dentry->d_name.name) + __string(auth_domain, key->ex_client->name) + __field(int, status) + ), + TP_fast_assign( + __assign_str(path); + __assign_str(auth_domain); + __entry->status = status; + ), + TP_printk("path=%s domain=%s status=%d", + __get_str(path), + __get_str(auth_domain), + __entry->status + ) +); + +TRACE_EVENT(nfsd_export_update, + TP_PROTO(const struct svc_export *key), + TP_ARGS(key), + TP_STRUCT__entry( + __string(path, key->ex_path.dentry->d_name.name) + __string(auth_domain, key->ex_client->name) + __field(bool, cache) + ), + TP_fast_assign( + __assign_str(path); + __assign_str(auth_domain); + __entry->cache = !test_bit(CACHE_NEGATIVE, &key->h.flags); + ), + TP_printk("path=%s domain=%s cache=%s", + __get_str(path), + __get_str(auth_domain), + __entry->cache ? "pos" : "neg" + ) +); + +DECLARE_EVENT_CLASS(nfsd_io_class, + TP_PROTO(struct svc_rqst *rqstp, + struct svc_fh *fhp, + u64 offset, + u32 len), + TP_ARGS(rqstp, fhp, offset, len), + TP_STRUCT__entry( + __field(u32, xid) + __field(u32, fh_hash) + __field(u64, offset) + __field(u32, len) + ), + TP_fast_assign( + __entry->xid = be32_to_cpu(rqstp->rq_xid); + __entry->fh_hash = knfsd_fh_hash(&fhp->fh_handle); + __entry->offset = offset; + __entry->len = len; + ), + TP_printk("xid=0x%08x fh_hash=0x%08x offset=%llu len=%u", + __entry->xid, __entry->fh_hash, + __entry->offset, __entry->len) +) + +#define DEFINE_NFSD_IO_EVENT(name) \ +DEFINE_EVENT(nfsd_io_class, nfsd_##name, \ + TP_PROTO(struct svc_rqst *rqstp, \ + struct svc_fh *fhp, \ + u64 offset, \ + u32 len), \ + TP_ARGS(rqstp, fhp, offset, len)) + +DEFINE_NFSD_IO_EVENT(read_start); +DEFINE_NFSD_IO_EVENT(read_splice); +DEFINE_NFSD_IO_EVENT(read_vector); +DEFINE_NFSD_IO_EVENT(read_direct); +DEFINE_NFSD_IO_EVENT(read_io_done); +DEFINE_NFSD_IO_EVENT(read_done); +DEFINE_NFSD_IO_EVENT(write_start); +DEFINE_NFSD_IO_EVENT(write_opened); +DEFINE_NFSD_IO_EVENT(write_direct); +DEFINE_NFSD_IO_EVENT(write_vector); +DEFINE_NFSD_IO_EVENT(write_io_done); +DEFINE_NFSD_IO_EVENT(write_done); +DEFINE_NFSD_IO_EVENT(commit_start); +DEFINE_NFSD_IO_EVENT(commit_done); + +DECLARE_EVENT_CLASS(nfsd_err_class, + TP_PROTO(struct svc_rqst *rqstp, + struct svc_fh *fhp, + loff_t offset, + int status), + TP_ARGS(rqstp, fhp, offset, status), + TP_STRUCT__entry( + __field(u32, xid) + __field(u32, fh_hash) + __field(loff_t, offset) + __field(int, status) + ), + TP_fast_assign( + __entry->xid = be32_to_cpu(rqstp->rq_xid); + __entry->fh_hash = knfsd_fh_hash(&fhp->fh_handle); + __entry->offset = offset; + __entry->status = status; + ), + TP_printk("xid=0x%08x fh_hash=0x%08x offset=%lld status=%d", + __entry->xid, __entry->fh_hash, + __entry->offset, __entry->status) +) + +#define DEFINE_NFSD_ERR_EVENT(name) \ +DEFINE_EVENT(nfsd_err_class, nfsd_##name, \ + TP_PROTO(struct svc_rqst *rqstp, \ + struct svc_fh *fhp, \ + loff_t offset, \ + int len), \ + TP_ARGS(rqstp, fhp, offset, len)) + +DEFINE_NFSD_ERR_EVENT(read_err); +DEFINE_NFSD_ERR_EVENT(write_err); + +TRACE_EVENT(nfsd_dirent, + TP_PROTO(struct svc_fh *fhp, + u64 ino, + const char *name, + int namlen), + TP_ARGS(fhp, ino, name, namlen), + TP_STRUCT__entry( + __field(u32, fh_hash) + __field(u64, ino) + __string_len(name, name, namlen) + ), + TP_fast_assign( + __entry->fh_hash = fhp ? knfsd_fh_hash(&fhp->fh_handle) : 0; + __entry->ino = ino; + __assign_str(name); + ), + TP_printk("fh_hash=0x%08x ino=%llu name=%s", + __entry->fh_hash, __entry->ino, __get_str(name) + ) +) + +DECLARE_EVENT_CLASS(nfsd_copy_err_class, + TP_PROTO(struct svc_rqst *rqstp, + struct svc_fh *src_fhp, + loff_t src_offset, + struct svc_fh *dst_fhp, + loff_t dst_offset, + u64 count, + int status), + TP_ARGS(rqstp, src_fhp, src_offset, dst_fhp, dst_offset, count, status), + TP_STRUCT__entry( + __field(u32, xid) + __field(u32, src_fh_hash) + __field(loff_t, src_offset) + __field(u32, dst_fh_hash) + __field(loff_t, dst_offset) + __field(u64, count) + __field(int, status) + ), + TP_fast_assign( + __entry->xid = be32_to_cpu(rqstp->rq_xid); + __entry->src_fh_hash = knfsd_fh_hash(&src_fhp->fh_handle); + __entry->src_offset = src_offset; + __entry->dst_fh_hash = knfsd_fh_hash(&dst_fhp->fh_handle); + __entry->dst_offset = dst_offset; + __entry->count = count; + __entry->status = status; + ), + TP_printk("xid=0x%08x src_fh_hash=0x%08x src_offset=%lld " + "dst_fh_hash=0x%08x dst_offset=%lld " + "count=%llu status=%d", + __entry->xid, __entry->src_fh_hash, __entry->src_offset, + __entry->dst_fh_hash, __entry->dst_offset, + (unsigned long long)__entry->count, + __entry->status) +) + +#define DEFINE_NFSD_COPY_ERR_EVENT(name) \ +DEFINE_EVENT(nfsd_copy_err_class, nfsd_##name, \ + TP_PROTO(struct svc_rqst *rqstp, \ + struct svc_fh *src_fhp, \ + loff_t src_offset, \ + struct svc_fh *dst_fhp, \ + loff_t dst_offset, \ + u64 count, \ + int status), \ + TP_ARGS(rqstp, src_fhp, src_offset, dst_fhp, dst_offset, \ + count, status)) + +DEFINE_NFSD_COPY_ERR_EVENT(clone_file_range_err); + +#include "state.h" +#include "filecache.h" +#include "vfs.h" + +TRACE_EVENT(nfsd_delegret_wakeup, + TP_PROTO( + const struct svc_rqst *rqstp, + const struct inode *inode, + long timeo + ), + TP_ARGS(rqstp, inode, timeo), + TP_STRUCT__entry( + __field(u32, xid) + __field(const void *, inode) + __field(long, timeo) + ), + TP_fast_assign( + __entry->xid = be32_to_cpu(rqstp->rq_xid); + __entry->inode = inode; + __entry->timeo = timeo; + ), + TP_printk("xid=0x%08x inode=%p%s", + __entry->xid, __entry->inode, + __entry->timeo == 0 ? " (timed out)" : "" + ) +); + +DECLARE_EVENT_CLASS(nfsd_stateid_class, + TP_PROTO(stateid_t *stp), + TP_ARGS(stp), + TP_STRUCT__entry( + __field(u32, cl_boot) + __field(u32, cl_id) + __field(u32, si_id) + __field(u32, si_generation) + ), + TP_fast_assign( + __entry->cl_boot = stp->si_opaque.so_clid.cl_boot; + __entry->cl_id = stp->si_opaque.so_clid.cl_id; + __entry->si_id = stp->si_opaque.so_id; + __entry->si_generation = stp->si_generation; + ), + TP_printk("client %08x:%08x stateid %08x:%08x", + __entry->cl_boot, + __entry->cl_id, + __entry->si_id, + __entry->si_generation) +) + +#define DEFINE_STATEID_EVENT(name) \ +DEFINE_EVENT(nfsd_stateid_class, nfsd_##name, \ + TP_PROTO(stateid_t *stp), \ + TP_ARGS(stp)) + +DEFINE_STATEID_EVENT(layoutstate_alloc); +DEFINE_STATEID_EVENT(layoutstate_unhash); +DEFINE_STATEID_EVENT(layoutstate_free); +DEFINE_STATEID_EVENT(layout_get_lookup_fail); +DEFINE_STATEID_EVENT(layout_commit_lookup_fail); +DEFINE_STATEID_EVENT(layout_return_lookup_fail); +DEFINE_STATEID_EVENT(layout_recall); +DEFINE_STATEID_EVENT(layout_recall_done); +DEFINE_STATEID_EVENT(layout_recall_fail); +DEFINE_STATEID_EVENT(layout_recall_release); + +DEFINE_STATEID_EVENT(open); +DEFINE_STATEID_EVENT(deleg_read); +DEFINE_STATEID_EVENT(deleg_write); +DEFINE_STATEID_EVENT(deleg_return); + +DECLARE_EVENT_CLASS(nfsd_stateseqid_class, + TP_PROTO(u32 seqid, const stateid_t *stp), + TP_ARGS(seqid, stp), + TP_STRUCT__entry( + __field(u32, seqid) + __field(u32, cl_boot) + __field(u32, cl_id) + __field(u32, si_id) + __field(u32, si_generation) + ), + TP_fast_assign( + __entry->seqid = seqid; + __entry->cl_boot = stp->si_opaque.so_clid.cl_boot; + __entry->cl_id = stp->si_opaque.so_clid.cl_id; + __entry->si_id = stp->si_opaque.so_id; + __entry->si_generation = stp->si_generation; + ), + TP_printk("seqid=%u client %08x:%08x stateid %08x:%08x", + __entry->seqid, __entry->cl_boot, __entry->cl_id, + __entry->si_id, __entry->si_generation) +) + +#define DEFINE_STATESEQID_EVENT(name) \ +DEFINE_EVENT(nfsd_stateseqid_class, nfsd_##name, \ + TP_PROTO(u32 seqid, const stateid_t *stp), \ + TP_ARGS(seqid, stp)) + +DEFINE_STATESEQID_EVENT(preprocess); +DEFINE_STATESEQID_EVENT(open_confirm); + +#define show_stid_type(x) \ + __print_flags(x, "|", \ + { SC_TYPE_OPEN, "OPEN" }, \ + { SC_TYPE_LOCK, "LOCK" }, \ + { SC_TYPE_DELEG, "DELEG" }, \ + { SC_TYPE_LAYOUT, "LAYOUT" }) + +#define show_stid_status(x) \ + __print_flags(x, "|", \ + { SC_STATUS_CLOSED, "CLOSED" }, \ + { SC_STATUS_REVOKED, "REVOKED" }, \ + { SC_STATUS_ADMIN_REVOKED, "ADMIN_REVOKED" }) + +DECLARE_EVENT_CLASS(nfsd_stid_class, + TP_PROTO( + const struct nfs4_stid *stid + ), + TP_ARGS(stid), + TP_STRUCT__entry( + __field(unsigned long, sc_type) + __field(unsigned long, sc_status) + __field(int, sc_count) + __field(u32, cl_boot) + __field(u32, cl_id) + __field(u32, si_id) + __field(u32, si_generation) + ), + TP_fast_assign( + const stateid_t *stp = &stid->sc_stateid; + + __entry->sc_type = stid->sc_type; + __entry->sc_status = stid->sc_status; + __entry->sc_count = refcount_read(&stid->sc_count); + __entry->cl_boot = stp->si_opaque.so_clid.cl_boot; + __entry->cl_id = stp->si_opaque.so_clid.cl_id; + __entry->si_id = stp->si_opaque.so_id; + __entry->si_generation = stp->si_generation; + ), + TP_printk("client %08x:%08x stateid %08x:%08x ref=%d type=%s state=%s", + __entry->cl_boot, __entry->cl_id, + __entry->si_id, __entry->si_generation, + __entry->sc_count, show_stid_type(__entry->sc_type), + show_stid_status(__entry->sc_status) + ) +); + +#define DEFINE_STID_EVENT(name) \ +DEFINE_EVENT(nfsd_stid_class, nfsd_stid_##name, \ + TP_PROTO(const struct nfs4_stid *stid), \ + TP_ARGS(stid)) + +DEFINE_STID_EVENT(revoke); + +TRACE_EVENT(nfsd_stateowner_replay, + TP_PROTO( + u32 opnum, + const struct nfs4_replay *rp + ), + TP_ARGS(opnum, rp), + TP_STRUCT__entry( + __field(unsigned long, status) + __field(u32, opnum) + ), + TP_fast_assign( + __entry->status = be32_to_cpu(rp->rp_status); + __entry->opnum = opnum; + ), + TP_printk("opnum=%u status=%lu", + __entry->opnum, __entry->status) +); + +TRACE_EVENT_CONDITION(nfsd_seq4_status, + TP_PROTO( + const struct svc_rqst *rqstp, + const struct nfsd4_sequence *sequence + ), + TP_ARGS(rqstp, sequence), + TP_CONDITION(sequence->status_flags), + TP_STRUCT__entry( + __field(unsigned int, netns_ino) + __field(u32, xid) + __field(u32, cl_boot) + __field(u32, cl_id) + __field(u32, seqno) + __field(u32, reserved) + __field(unsigned long, status_flags) + ), + TP_fast_assign( + const struct nfsd4_sessionid *sid = + (struct nfsd4_sessionid *)&sequence->sessionid; + + __entry->netns_ino = SVC_NET(rqstp)->ns.inum; + __entry->xid = be32_to_cpu(rqstp->rq_xid); + __entry->cl_boot = sid->clientid.cl_boot; + __entry->cl_id = sid->clientid.cl_id; + __entry->seqno = sid->sequence; + __entry->reserved = sid->reserved; + __entry->status_flags = sequence->status_flags; + ), + TP_printk("xid=0x%08x sessionid=%08x:%08x:%08x:%08x status_flags=%s", + __entry->xid, __entry->cl_boot, __entry->cl_id, + __entry->seqno, __entry->reserved, + show_nfs4_seq4_status(__entry->status_flags) + ) +); + +DECLARE_EVENT_CLASS(nfsd_cs_slot_class, + TP_PROTO( + const struct nfs4_client *clp, + const struct nfsd4_create_session *cs + ), + TP_ARGS(clp, cs), + TP_STRUCT__entry( + __field(u32, seqid) + __field(u32, slot_seqid) + __field(u32, cl_boot) + __field(u32, cl_id) + __sockaddr(addr, clp->cl_cb_conn.cb_addrlen) + ), + TP_fast_assign( + const struct nfsd4_clid_slot *slot = &clp->cl_cs_slot; + + __entry->cl_boot = clp->cl_clientid.cl_boot; + __entry->cl_id = clp->cl_clientid.cl_id; + __assign_sockaddr(addr, &clp->cl_cb_conn.cb_addr, + clp->cl_cb_conn.cb_addrlen); + __entry->seqid = cs->seqid; + __entry->slot_seqid = slot->sl_seqid; + ), + TP_printk("addr=%pISpc client %08x:%08x seqid=%u slot_seqid=%u", + __get_sockaddr(addr), __entry->cl_boot, __entry->cl_id, + __entry->seqid, __entry->slot_seqid + ) +); + +#define DEFINE_CS_SLOT_EVENT(name) \ +DEFINE_EVENT(nfsd_cs_slot_class, nfsd_##name, \ + TP_PROTO( \ + const struct nfs4_client *clp, \ + const struct nfsd4_create_session *cs \ + ), \ + TP_ARGS(clp, cs)) + +DEFINE_CS_SLOT_EVENT(slot_seqid_conf); +DEFINE_CS_SLOT_EVENT(slot_seqid_unconf); + +#define show_nfs_slot_flags(val) \ + __print_flags(val, "|", \ + { NFSD4_SLOT_INUSE, "INUSE" }, \ + { NFSD4_SLOT_CACHETHIS, "CACHETHIS" }, \ + { NFSD4_SLOT_INITIALIZED, "INITIALIZED" }, \ + { NFSD4_SLOT_CACHED, "CACHED" }, \ + { NFSD4_SLOT_REUSED, "REUSED" }) + +TRACE_EVENT(nfsd_slot_seqid_sequence, + TP_PROTO( + const struct nfs4_client *clp, + const struct nfsd4_sequence *seq, + const struct nfsd4_slot *slot + ), + TP_ARGS(clp, seq, slot), + TP_STRUCT__entry( + __field(u32, seqid) + __field(u32, slot_seqid) + __field(u32, slot_index) + __field(unsigned long, slot_flags) + __field(u32, cl_boot) + __field(u32, cl_id) + __sockaddr(addr, clp->cl_cb_conn.cb_addrlen) + ), + TP_fast_assign( + __entry->cl_boot = clp->cl_clientid.cl_boot; + __entry->cl_id = clp->cl_clientid.cl_id; + __assign_sockaddr(addr, &clp->cl_cb_conn.cb_addr, + clp->cl_cb_conn.cb_addrlen); + __entry->seqid = seq->seqid; + __entry->slot_seqid = slot->sl_seqid; + __entry->slot_index = seq->slotid; + __entry->slot_flags = slot->sl_flags; + ), + TP_printk("addr=%pISpc client %08x:%08x idx=%u seqid=%u slot_seqid=%u flags=%s", + __get_sockaddr(addr), __entry->cl_boot, __entry->cl_id, + __entry->slot_index, __entry->seqid, __entry->slot_seqid, + show_nfs_slot_flags(__entry->slot_flags) + ) +); + +DECLARE_EVENT_CLASS(nfsd_clientid_class, + TP_PROTO(const clientid_t *clid), + TP_ARGS(clid), + TP_STRUCT__entry( + __field(u32, cl_boot) + __field(u32, cl_id) + ), + TP_fast_assign( + __entry->cl_boot = clid->cl_boot; + __entry->cl_id = clid->cl_id; + ), + TP_printk("client %08x:%08x", __entry->cl_boot, __entry->cl_id) +) + +#define DEFINE_CLIENTID_EVENT(name) \ +DEFINE_EVENT(nfsd_clientid_class, nfsd_clid_##name, \ + TP_PROTO(const clientid_t *clid), \ + TP_ARGS(clid)) + +DEFINE_CLIENTID_EVENT(expire_unconf); +DEFINE_CLIENTID_EVENT(reclaim_complete); +DEFINE_CLIENTID_EVENT(confirmed); +DEFINE_CLIENTID_EVENT(destroyed); +DEFINE_CLIENTID_EVENT(admin_expired); +DEFINE_CLIENTID_EVENT(replaced); +DEFINE_CLIENTID_EVENT(purged); +DEFINE_CLIENTID_EVENT(renew); +DEFINE_CLIENTID_EVENT(stale); + +TRACE_EVENT(nfsd_mark_client_expired, + TP_PROTO( + const struct nfs4_client *clp, + int cl_rpc_users + ), + TP_ARGS(clp, cl_rpc_users), + TP_STRUCT__entry( + __field(int, cl_rpc_users) + __field(u32, cl_boot) + __field(u32, cl_id) + __sockaddr(addr, clp->cl_cb_conn.cb_addrlen) + ), + TP_fast_assign( + __entry->cl_rpc_users = cl_rpc_users; + __entry->cl_boot = clp->cl_clientid.cl_boot; + __entry->cl_id = clp->cl_clientid.cl_id; + __assign_sockaddr(addr, &clp->cl_cb_conn.cb_addr, + clp->cl_cb_conn.cb_addrlen) + ), + TP_printk("addr=%pISpc client %08x:%08x cl_rpc_users=%d", + __get_sockaddr(addr), __entry->cl_boot, __entry->cl_id, + __entry->cl_rpc_users) +); + +DECLARE_EVENT_CLASS(nfsd_net_class, + TP_PROTO(const struct nfsd_net *nn), + TP_ARGS(nn), + TP_STRUCT__entry( + __field(unsigned long long, boot_time) + ), + TP_fast_assign( + __entry->boot_time = nn->boot_time; + ), + TP_printk("boot_time=%16llx", __entry->boot_time) +) + +#define DEFINE_NET_EVENT(name) \ +DEFINE_EVENT(nfsd_net_class, nfsd_##name, \ + TP_PROTO(const struct nfsd_net *nn), \ + TP_ARGS(nn)) + +DEFINE_NET_EVENT(grace_start); +DEFINE_NET_EVENT(grace_complete); + +TRACE_EVENT(nfsd_writeverf_reset, + TP_PROTO( + const struct nfsd_net *nn, + const struct svc_rqst *rqstp, + int error + ), + TP_ARGS(nn, rqstp, error), + TP_STRUCT__entry( + __field(unsigned long long, boot_time) + __field(u32, xid) + __field(int, error) + __array(unsigned char, verifier, NFS4_VERIFIER_SIZE) + ), + TP_fast_assign( + __entry->boot_time = nn->boot_time; + __entry->xid = be32_to_cpu(rqstp->rq_xid); + __entry->error = error; + + /* avoid seqlock inside TP_fast_assign */ + memcpy(__entry->verifier, nn->writeverf, + NFS4_VERIFIER_SIZE); + ), + TP_printk("boot_time=%16llx xid=0x%08x error=%d new verifier=0x%s", + __entry->boot_time, __entry->xid, __entry->error, + __print_hex_str(__entry->verifier, NFS4_VERIFIER_SIZE) + ) +); + +TRACE_EVENT(nfsd_clid_cred_mismatch, + TP_PROTO( + const struct nfs4_client *clp, + const struct svc_rqst *rqstp + ), + TP_ARGS(clp, rqstp), + TP_STRUCT__entry( + __field(u32, cl_boot) + __field(u32, cl_id) + __field(unsigned long, cl_flavor) + __field(unsigned long, new_flavor) + __sockaddr(addr, rqstp->rq_xprt->xpt_remotelen) + ), + TP_fast_assign( + __entry->cl_boot = clp->cl_clientid.cl_boot; + __entry->cl_id = clp->cl_clientid.cl_id; + __entry->cl_flavor = clp->cl_cred.cr_flavor; + __entry->new_flavor = rqstp->rq_cred.cr_flavor; + __assign_sockaddr(addr, &rqstp->rq_xprt->xpt_remote, + rqstp->rq_xprt->xpt_remotelen); + ), + TP_printk("client %08x:%08x flavor=%s, conflict=%s from addr=%pISpc", + __entry->cl_boot, __entry->cl_id, + show_nfsd_authflavor(__entry->cl_flavor), + show_nfsd_authflavor(__entry->new_flavor), + __get_sockaddr(addr) + ) +) + +TRACE_EVENT(nfsd_clid_verf_mismatch, + TP_PROTO( + const struct nfs4_client *clp, + const struct svc_rqst *rqstp, + const nfs4_verifier *verf + ), + TP_ARGS(clp, rqstp, verf), + TP_STRUCT__entry( + __field(u32, cl_boot) + __field(u32, cl_id) + __array(unsigned char, cl_verifier, NFS4_VERIFIER_SIZE) + __array(unsigned char, new_verifier, NFS4_VERIFIER_SIZE) + __sockaddr(addr, rqstp->rq_xprt->xpt_remotelen) + ), + TP_fast_assign( + __entry->cl_boot = clp->cl_clientid.cl_boot; + __entry->cl_id = clp->cl_clientid.cl_id; + memcpy(__entry->cl_verifier, (void *)&clp->cl_verifier, + NFS4_VERIFIER_SIZE); + memcpy(__entry->new_verifier, (void *)verf, + NFS4_VERIFIER_SIZE); + __assign_sockaddr(addr, &rqstp->rq_xprt->xpt_remote, + rqstp->rq_xprt->xpt_remotelen); + ), + TP_printk("client %08x:%08x verf=0x%s, updated=0x%s from addr=%pISpc", + __entry->cl_boot, __entry->cl_id, + __print_hex_str(__entry->cl_verifier, NFS4_VERIFIER_SIZE), + __print_hex_str(__entry->new_verifier, NFS4_VERIFIER_SIZE), + __get_sockaddr(addr) + ) +); + +DECLARE_EVENT_CLASS(nfsd_clid_class, + TP_PROTO(const struct nfs4_client *clp), + TP_ARGS(clp), + TP_STRUCT__entry( + __field(u32, cl_boot) + __field(u32, cl_id) + __array(unsigned char, addr, sizeof(struct sockaddr_in6)) + __field(unsigned long, flavor) + __array(unsigned char, verifier, NFS4_VERIFIER_SIZE) + __string_len(name, clp->cl_name.data, clp->cl_name.len) + ), + TP_fast_assign( + __entry->cl_boot = clp->cl_clientid.cl_boot; + __entry->cl_id = clp->cl_clientid.cl_id; + memcpy(__entry->addr, &clp->cl_addr, + sizeof(struct sockaddr_in6)); + __entry->flavor = clp->cl_cred.cr_flavor; + memcpy(__entry->verifier, (void *)&clp->cl_verifier, + NFS4_VERIFIER_SIZE); + __assign_str(name); + ), + TP_printk("addr=%pISpc name='%s' verifier=0x%s flavor=%s client=%08x:%08x", + __entry->addr, __get_str(name), + __print_hex_str(__entry->verifier, NFS4_VERIFIER_SIZE), + show_nfsd_authflavor(__entry->flavor), + __entry->cl_boot, __entry->cl_id) +); + +#define DEFINE_CLID_EVENT(name) \ +DEFINE_EVENT(nfsd_clid_class, nfsd_clid_##name, \ + TP_PROTO(const struct nfs4_client *clp), \ + TP_ARGS(clp)) + +DEFINE_CLID_EVENT(fresh); +DEFINE_CLID_EVENT(confirmed_r); + +/* + * from fs/nfsd/filecache.h + */ +#define show_nf_flags(val) \ + __print_flags(val, "|", \ + { 1 << NFSD_FILE_HASHED, "HASHED" }, \ + { 1 << NFSD_FILE_PENDING, "PENDING" }, \ + { 1 << NFSD_FILE_REFERENCED, "REFERENCED" }, \ + { 1 << NFSD_FILE_RECENT, "RECENT" }, \ + { 1 << NFSD_FILE_GC, "GC" }) + +DECLARE_EVENT_CLASS(nfsd_file_class, + TP_PROTO(struct nfsd_file *nf), + TP_ARGS(nf), + TP_STRUCT__entry( + __field(void *, nf_inode) + __field(int, nf_ref) + __field(unsigned long, nf_flags) + __field(unsigned char, nf_may) + __field(struct file *, nf_file) + ), + TP_fast_assign( + __entry->nf_inode = nf->nf_inode; + __entry->nf_ref = refcount_read(&nf->nf_ref); + __entry->nf_flags = nf->nf_flags; + __entry->nf_may = nf->nf_may; + __entry->nf_file = nf->nf_file; + ), + TP_printk("inode=%p ref=%d flags=%s may=%s nf_file=%p", + __entry->nf_inode, + __entry->nf_ref, + show_nf_flags(__entry->nf_flags), + show_nfsd_may_flags(__entry->nf_may), + __entry->nf_file) +) + +#define DEFINE_NFSD_FILE_EVENT(name) \ +DEFINE_EVENT(nfsd_file_class, name, \ + TP_PROTO(struct nfsd_file *nf), \ + TP_ARGS(nf)) + +DEFINE_NFSD_FILE_EVENT(nfsd_file_free); +DEFINE_NFSD_FILE_EVENT(nfsd_file_unhash); +DEFINE_NFSD_FILE_EVENT(nfsd_file_put); +DEFINE_NFSD_FILE_EVENT(nfsd_file_closing); + +TRACE_EVENT(nfsd_file_alloc, + TP_PROTO( + const struct nfsd_file *nf + ), + TP_ARGS(nf), + TP_STRUCT__entry( + __field(const void *, nf_inode) + __field(unsigned long, nf_flags) + __field(unsigned long, nf_may) + __field(unsigned int, nf_ref) + ), + TP_fast_assign( + __entry->nf_inode = nf->nf_inode; + __entry->nf_flags = nf->nf_flags; + __entry->nf_ref = refcount_read(&nf->nf_ref); + __entry->nf_may = nf->nf_may; + ), + TP_printk("inode=%p ref=%u flags=%s may=%s", + __entry->nf_inode, __entry->nf_ref, + show_nf_flags(__entry->nf_flags), + show_nfsd_may_flags(__entry->nf_may) + ) +); + +TRACE_EVENT(nfsd_file_get_dio_attrs, + TP_PROTO( + const struct inode *inode, + const struct kstat *stat + ), + TP_ARGS(inode, stat), + TP_STRUCT__entry( + __field(const void *, inode) + __field(unsigned long, mask) + __field(u32, mem_align) + __field(u32, offset_align) + __field(u32, read_offset_align) + ), + TP_fast_assign( + __entry->inode = inode; + __entry->mask = stat->result_mask; + __entry->mem_align = stat->dio_mem_align; + __entry->offset_align = stat->dio_offset_align; + __entry->read_offset_align = stat->dio_read_offset_align; + ), + TP_printk("inode=%p flags=%s mem_align=%u offset_align=%u read_offset_align=%u", + __entry->inode, show_statx_mask(__entry->mask), + __entry->mem_align, __entry->offset_align, + __entry->read_offset_align + ) +); + +TRACE_EVENT(nfsd_file_acquire, + TP_PROTO( + const struct svc_rqst *rqstp, + const struct inode *inode, + unsigned int may_flags, + const struct nfsd_file *nf, + __be32 status + ), + + TP_ARGS(rqstp, inode, may_flags, nf, status), + + TP_STRUCT__entry( + __field(u32, xid) + __field(const void *, inode) + __field(unsigned long, may_flags) + __field(unsigned int, nf_ref) + __field(unsigned long, nf_flags) + __field(unsigned long, nf_may) + __field(const void *, nf_file) + __field(u32, status) + ), + + TP_fast_assign( + __entry->xid = rqstp ? be32_to_cpu(rqstp->rq_xid) : 0; + __entry->inode = inode; + __entry->may_flags = may_flags; + __entry->nf_ref = nf ? refcount_read(&nf->nf_ref) : 0; + __entry->nf_flags = nf ? nf->nf_flags : 0; + __entry->nf_may = nf ? nf->nf_may : 0; + __entry->nf_file = nf ? nf->nf_file : NULL; + __entry->status = be32_to_cpu(status); + ), + + TP_printk("xid=0x%x inode=%p may_flags=%s ref=%u nf_flags=%s nf_may=%s nf_file=%p status=%u", + __entry->xid, __entry->inode, + show_nfsd_may_flags(__entry->may_flags), + __entry->nf_ref, show_nf_flags(__entry->nf_flags), + show_nfsd_may_flags(__entry->nf_may), + __entry->nf_file, __entry->status + ) +); + +TRACE_EVENT(nfsd_file_insert_err, + TP_PROTO( + const struct svc_rqst *rqstp, + const struct inode *inode, + unsigned int may_flags, + long error + ), + TP_ARGS(rqstp, inode, may_flags, error), + TP_STRUCT__entry( + __field(u32, xid) + __field(const void *, inode) + __field(unsigned long, may_flags) + __field(long, error) + ), + TP_fast_assign( + __entry->xid = rqstp ? be32_to_cpu(rqstp->rq_xid) : 0; + __entry->inode = inode; + __entry->may_flags = may_flags; + __entry->error = error; + ), + TP_printk("xid=0x%x inode=%p may_flags=%s error=%ld", + __entry->xid, __entry->inode, + show_nfsd_may_flags(__entry->may_flags), + __entry->error + ) +); + +TRACE_EVENT(nfsd_file_cons_err, + TP_PROTO( + const struct svc_rqst *rqstp, + const struct inode *inode, + unsigned int may_flags, + const struct nfsd_file *nf + ), + TP_ARGS(rqstp, inode, may_flags, nf), + TP_STRUCT__entry( + __field(u32, xid) + __field(const void *, inode) + __field(unsigned long, may_flags) + __field(unsigned int, nf_ref) + __field(unsigned long, nf_flags) + __field(unsigned long, nf_may) + __field(const void *, nf_file) + ), + TP_fast_assign( + __entry->xid = rqstp ? be32_to_cpu(rqstp->rq_xid) : 0; + __entry->inode = inode; + __entry->may_flags = may_flags; + __entry->nf_ref = refcount_read(&nf->nf_ref); + __entry->nf_flags = nf->nf_flags; + __entry->nf_may = nf->nf_may; + __entry->nf_file = nf->nf_file; + ), + TP_printk("xid=0x%x inode=%p may_flags=%s ref=%u nf_flags=%s nf_may=%s nf_file=%p", + __entry->xid, __entry->inode, + show_nfsd_may_flags(__entry->may_flags), __entry->nf_ref, + show_nf_flags(__entry->nf_flags), + show_nfsd_may_flags(__entry->nf_may), __entry->nf_file + ) +); + +DECLARE_EVENT_CLASS(nfsd_file_open_class, + TP_PROTO(const struct nfsd_file *nf, __be32 status), + TP_ARGS(nf, status), + TP_STRUCT__entry( + __field(void *, nf_inode) /* cannot be dereferenced */ + __field(int, nf_ref) + __field(unsigned long, nf_flags) + __field(unsigned long, nf_may) + __field(void *, nf_file) /* cannot be dereferenced */ + ), + TP_fast_assign( + __entry->nf_inode = nf->nf_inode; + __entry->nf_ref = refcount_read(&nf->nf_ref); + __entry->nf_flags = nf->nf_flags; + __entry->nf_may = nf->nf_may; + __entry->nf_file = nf->nf_file; + ), + TP_printk("inode=%p ref=%d flags=%s may=%s file=%p", + __entry->nf_inode, + __entry->nf_ref, + show_nf_flags(__entry->nf_flags), + show_nfsd_may_flags(__entry->nf_may), + __entry->nf_file) +) + +#define DEFINE_NFSD_FILE_OPEN_EVENT(name) \ +DEFINE_EVENT(nfsd_file_open_class, name, \ + TP_PROTO( \ + const struct nfsd_file *nf, \ + __be32 status \ + ), \ + TP_ARGS(nf, status)) + +DEFINE_NFSD_FILE_OPEN_EVENT(nfsd_file_open); +DEFINE_NFSD_FILE_OPEN_EVENT(nfsd_file_opened); + +TRACE_EVENT(nfsd_file_is_cached, + TP_PROTO( + const struct inode *inode, + int found + ), + TP_ARGS(inode, found), + TP_STRUCT__entry( + __field(const struct inode *, inode) + __field(int, found) + ), + TP_fast_assign( + __entry->inode = inode; + __entry->found = found; + ), + TP_printk("inode=%p is %scached", + __entry->inode, + __entry->found ? "" : "not " + ) +); + +TRACE_EVENT(nfsd_file_fsnotify_handle_event, + TP_PROTO(struct inode *inode, u32 mask), + TP_ARGS(inode, mask), + TP_STRUCT__entry( + __field(struct inode *, inode) + __field(unsigned int, nlink) + __field(umode_t, mode) + __field(u32, mask) + ), + TP_fast_assign( + __entry->inode = inode; + __entry->nlink = inode->i_nlink; + __entry->mode = inode->i_mode; + __entry->mask = mask; + ), + TP_printk("inode=%p nlink=%u mode=0%ho mask=0x%x", __entry->inode, + __entry->nlink, __entry->mode, __entry->mask) +); + +TRACE_EVENT(nfsd_handle_dir_event, + TP_PROTO(u32 mask, const struct inode *dir, const struct qstr *name), + TP_ARGS(mask, dir, name), + TP_STRUCT__entry( + __field(u32, mask) + __field(dev_t, s_dev) + __field(u64, i_ino) + __string_len(name, name ? name->name : NULL, + name ? name->len : 0) + ), + TP_fast_assign( + __entry->mask = mask; + __entry->s_dev = dir ? dir->i_sb->s_dev : 0; + __entry->i_ino = dir ? dir->i_ino : 0; + __assign_str(name); + ), + TP_printk("inode=0x%x:0x%x:0x%llx mask=%s name=%s", + MAJOR(__entry->s_dev), MINOR(__entry->s_dev), + __entry->i_ino, show_fsnotify_mask(__entry->mask), + __get_str(name)) +); + +DECLARE_EVENT_CLASS(nfsd_file_gc_class, + TP_PROTO( + const struct nfsd_file *nf + ), + TP_ARGS(nf), + TP_STRUCT__entry( + __field(void *, nf_inode) + __field(void *, nf_file) + __field(int, nf_ref) + __field(unsigned long, nf_flags) + ), + TP_fast_assign( + __entry->nf_inode = nf->nf_inode; + __entry->nf_file = nf->nf_file; + __entry->nf_ref = refcount_read(&nf->nf_ref); + __entry->nf_flags = nf->nf_flags; + ), + TP_printk("inode=%p ref=%d nf_flags=%s nf_file=%p", + __entry->nf_inode, __entry->nf_ref, + show_nf_flags(__entry->nf_flags), + __entry->nf_file + ) +); + +#define DEFINE_NFSD_FILE_GC_EVENT(name) \ +DEFINE_EVENT(nfsd_file_gc_class, name, \ + TP_PROTO( \ + const struct nfsd_file *nf \ + ), \ + TP_ARGS(nf)) + +DEFINE_NFSD_FILE_GC_EVENT(nfsd_file_lru_add); +DEFINE_NFSD_FILE_GC_EVENT(nfsd_file_lru_del); +DEFINE_NFSD_FILE_GC_EVENT(nfsd_file_gc_in_use); +DEFINE_NFSD_FILE_GC_EVENT(nfsd_file_gc_writeback); +DEFINE_NFSD_FILE_GC_EVENT(nfsd_file_gc_referenced); +DEFINE_NFSD_FILE_GC_EVENT(nfsd_file_gc_aged); +DEFINE_NFSD_FILE_GC_EVENT(nfsd_file_gc_disposed); + +DECLARE_EVENT_CLASS(nfsd_file_lruwalk_class, + TP_PROTO( + unsigned long removed, + unsigned long remaining + ), + TP_ARGS(removed, remaining), + TP_STRUCT__entry( + __field(unsigned long, removed) + __field(unsigned long, remaining) + ), + TP_fast_assign( + __entry->removed = removed; + __entry->remaining = remaining; + ), + TP_printk("%lu entries removed, %lu remaining", + __entry->removed, __entry->remaining) +); + +#define DEFINE_NFSD_FILE_LRUWALK_EVENT(name) \ +DEFINE_EVENT(nfsd_file_lruwalk_class, name, \ + TP_PROTO( \ + unsigned long removed, \ + unsigned long remaining \ + ), \ + TP_ARGS(removed, remaining)) + +DEFINE_NFSD_FILE_LRUWALK_EVENT(nfsd_file_gc_removed); +DEFINE_NFSD_FILE_LRUWALK_EVENT(nfsd_file_shrinker_removed); + +TRACE_EVENT(nfsd_file_close, + TP_PROTO( + const struct inode *inode + ), + TP_ARGS(inode), + TP_STRUCT__entry( + __field(const void *, inode) + ), + TP_fast_assign( + __entry->inode = inode; + ), + TP_printk("inode=%p", + __entry->inode + ) +); + +#include "cache.h" + +TRACE_DEFINE_ENUM(RC_DROPIT); +TRACE_DEFINE_ENUM(RC_REPLY); +TRACE_DEFINE_ENUM(RC_DOIT); + +#define show_drc_retval(x) \ + __print_symbolic(x, \ + { RC_DROPIT, "DROPIT" }, \ + { RC_REPLY, "REPLY" }, \ + { RC_DOIT, "DOIT" }) + +TRACE_EVENT(nfsd_drc_found, + TP_PROTO( + const struct nfsd_net *nn, + const struct svc_rqst *rqstp, + int result + ), + TP_ARGS(nn, rqstp, result), + TP_STRUCT__entry( + __field(unsigned long long, boot_time) + __field(unsigned long, result) + __field(u32, xid) + ), + TP_fast_assign( + __entry->boot_time = nn->boot_time; + __entry->result = result; + __entry->xid = be32_to_cpu(rqstp->rq_xid); + ), + TP_printk("boot_time=%16llx xid=0x%08x result=%s", + __entry->boot_time, __entry->xid, + show_drc_retval(__entry->result)) + +); + +TRACE_EVENT(nfsd_drc_mismatch, + TP_PROTO( + const struct nfsd_net *nn, + const struct nfsd_cacherep *key, + const struct nfsd_cacherep *rp + ), + TP_ARGS(nn, key, rp), + TP_STRUCT__entry( + __field(unsigned long long, boot_time) + __field(u32, xid) + __field(u32, cached) + __field(u32, ingress) + ), + TP_fast_assign( + __entry->boot_time = nn->boot_time; + __entry->xid = be32_to_cpu(key->c_key.k_xid); + __entry->cached = (__force u32)key->c_key.k_csum; + __entry->ingress = (__force u32)rp->c_key.k_csum; + ), + TP_printk("boot_time=%16llx xid=0x%08x cached-csum=0x%08x ingress-csum=0x%08x", + __entry->boot_time, __entry->xid, __entry->cached, + __entry->ingress) +); + +TRACE_EVENT(nfsd_cb_args, + TP_PROTO( + const struct nfs4_client *clp, + const struct nfs4_cb_conn *conn + ), + TP_ARGS(clp, conn), + TP_STRUCT__entry( + __field(u32, cl_boot) + __field(u32, cl_id) + __field(u32, prog) + __field(u32, ident) + __sockaddr(addr, conn->cb_addrlen) + ), + TP_fast_assign( + __entry->cl_boot = clp->cl_clientid.cl_boot; + __entry->cl_id = clp->cl_clientid.cl_id; + __entry->prog = conn->cb_prog; + __entry->ident = conn->cb_ident; + __assign_sockaddr(addr, &conn->cb_addr, conn->cb_addrlen); + ), + TP_printk("addr=%pISpc client %08x:%08x prog=%u ident=%u", + __get_sockaddr(addr), __entry->cl_boot, __entry->cl_id, + __entry->prog, __entry->ident) +); + +TRACE_EVENT(nfsd_cb_nodelegs, + TP_PROTO(const struct nfs4_client *clp), + TP_ARGS(clp), + TP_STRUCT__entry( + __field(u32, cl_boot) + __field(u32, cl_id) + ), + TP_fast_assign( + __entry->cl_boot = clp->cl_clientid.cl_boot; + __entry->cl_id = clp->cl_clientid.cl_id; + ), + TP_printk("client %08x:%08x", __entry->cl_boot, __entry->cl_id) +) + +#define show_cb_state(val) \ + __print_symbolic(val, \ + { NFSD4_CB_UP, "UP" }, \ + { NFSD4_CB_UNKNOWN, "UNKNOWN" }, \ + { NFSD4_CB_DOWN, "DOWN" }, \ + { NFSD4_CB_FAULT, "FAULT"}) + +DECLARE_EVENT_CLASS(nfsd_cb_class, + TP_PROTO(const struct nfs4_client *clp), + TP_ARGS(clp), + TP_STRUCT__entry( + __field(unsigned long, state) + __field(u32, cl_boot) + __field(u32, cl_id) + __sockaddr(addr, clp->cl_cb_conn.cb_addrlen) + ), + TP_fast_assign( + __entry->state = clp->cl_cb_state; + __entry->cl_boot = clp->cl_clientid.cl_boot; + __entry->cl_id = clp->cl_clientid.cl_id; + __assign_sockaddr(addr, &clp->cl_cb_conn.cb_addr, + clp->cl_cb_conn.cb_addrlen) + ), + TP_printk("addr=%pISpc client %08x:%08x state=%s", + __get_sockaddr(addr), __entry->cl_boot, __entry->cl_id, + show_cb_state(__entry->state)) +); + +#define DEFINE_NFSD_CB_EVENT(name) \ +DEFINE_EVENT(nfsd_cb_class, nfsd_cb_##name, \ + TP_PROTO(const struct nfs4_client *clp), \ + TP_ARGS(clp)) + +DEFINE_NFSD_CB_EVENT(start); +DEFINE_NFSD_CB_EVENT(new_state); +DEFINE_NFSD_CB_EVENT(probe); +DEFINE_NFSD_CB_EVENT(lost); +DEFINE_NFSD_CB_EVENT(shutdown); +DEFINE_NFSD_CB_EVENT(rpc_prepare); +DEFINE_NFSD_CB_EVENT(rpc_done); +DEFINE_NFSD_CB_EVENT(rpc_release); + +TRACE_DEFINE_ENUM(RPC_AUTH_NULL); +TRACE_DEFINE_ENUM(RPC_AUTH_UNIX); +TRACE_DEFINE_ENUM(RPC_AUTH_GSS); +TRACE_DEFINE_ENUM(RPC_AUTH_GSS_KRB5); +TRACE_DEFINE_ENUM(RPC_AUTH_GSS_KRB5I); +TRACE_DEFINE_ENUM(RPC_AUTH_GSS_KRB5P); + +#define show_nfsd_authflavor(val) \ + __print_symbolic(val, \ + { RPC_AUTH_NULL, "none" }, \ + { RPC_AUTH_UNIX, "sys" }, \ + { RPC_AUTH_GSS, "gss" }, \ + { RPC_AUTH_GSS_KRB5, "krb5" }, \ + { RPC_AUTH_GSS_KRB5I, "krb5i" }, \ + { RPC_AUTH_GSS_KRB5P, "krb5p" }) + +TRACE_EVENT(nfsd_cb_setup, + TP_PROTO(const struct nfs4_client *clp, + const char *netid, + rpc_authflavor_t authflavor + ), + TP_ARGS(clp, netid, authflavor), + TP_STRUCT__entry( + __field(u32, cl_boot) + __field(u32, cl_id) + __field(unsigned long, authflavor) + __sockaddr(addr, clp->cl_cb_conn.cb_addrlen) + __string(netid, netid) + ), + TP_fast_assign( + __entry->cl_boot = clp->cl_clientid.cl_boot; + __entry->cl_id = clp->cl_clientid.cl_id; + __assign_str(netid); + __entry->authflavor = authflavor; + __assign_sockaddr(addr, &clp->cl_cb_conn.cb_addr, + clp->cl_cb_conn.cb_addrlen) + ), + TP_printk("addr=%pISpc client %08x:%08x proto=%s flavor=%s", + __get_sockaddr(addr), __entry->cl_boot, __entry->cl_id, + __get_str(netid), show_nfsd_authflavor(__entry->authflavor)) +); + +TRACE_EVENT(nfsd_cb_setup_err, + TP_PROTO( + const struct nfs4_client *clp, + long error + ), + TP_ARGS(clp, error), + TP_STRUCT__entry( + __field(long, error) + __field(u32, cl_boot) + __field(u32, cl_id) + __sockaddr(addr, clp->cl_cb_conn.cb_addrlen) + ), + TP_fast_assign( + __entry->error = error; + __entry->cl_boot = clp->cl_clientid.cl_boot; + __entry->cl_id = clp->cl_clientid.cl_id; + __assign_sockaddr(addr, &clp->cl_cb_conn.cb_addr, + clp->cl_cb_conn.cb_addrlen) + ), + TP_printk("addr=%pISpc client %08x:%08x error=%ld", + __get_sockaddr(addr), __entry->cl_boot, __entry->cl_id, + __entry->error) +); + +/* Not a real opcode, but there is no 0 operation. */ +#define _CB_NULL 0 + +#define show_nfsd_cb_opcode(val) \ + __print_symbolic(val, \ + { _CB_NULL, "CB_NULL" }, \ + { OP_CB_GETATTR, "CB_GETATTR" }, \ + { OP_CB_RECALL, "CB_RECALL" }, \ + { OP_CB_LAYOUTRECALL, "CB_LAYOUTRECALL" }, \ + { OP_CB_RECALL_ANY, "CB_RECALL_ANY" }, \ + { OP_CB_NOTIFY, "CB_NOTIFY" }, \ + { OP_CB_NOTIFY_LOCK, "CB_NOTIFY_LOCK" }, \ + { OP_CB_OFFLOAD, "CB_OFFLOAD" }) + +DECLARE_EVENT_CLASS(nfsd_cb_lifetime_class, + TP_PROTO( + const struct nfs4_client *clp, + const struct nfsd4_callback *cb + ), + TP_ARGS(clp, cb), + TP_STRUCT__entry( + __field(u32, cl_boot) + __field(u32, cl_id) + __field(const void *, cb) + __field(unsigned long, opcode) + __field(bool, need_restart) + __sockaddr(addr, clp->cl_cb_conn.cb_addrlen) + ), + TP_fast_assign( + __entry->cl_boot = clp->cl_clientid.cl_boot; + __entry->cl_id = clp->cl_clientid.cl_id; + __entry->cb = cb; + __entry->opcode = cb->cb_ops ? cb->cb_ops->opcode : _CB_NULL; + __entry->need_restart = test_bit(NFSD4_CALLBACK_REQUEUE, &cb->cb_flags); + __assign_sockaddr(addr, &clp->cl_cb_conn.cb_addr, + clp->cl_cb_conn.cb_addrlen) + ), + TP_printk("addr=%pISpc client %08x:%08x cb=%p%s opcode=%s", + __get_sockaddr(addr), __entry->cl_boot, __entry->cl_id, __entry->cb, + __entry->need_restart ? " (need restart)" : " (first try)", + show_nfsd_cb_opcode(__entry->opcode) + ) +); + +#define DEFINE_NFSD_CB_LIFETIME_EVENT(name) \ +DEFINE_EVENT(nfsd_cb_lifetime_class, nfsd_cb_##name, \ + TP_PROTO( \ + const struct nfs4_client *clp, \ + const struct nfsd4_callback *cb \ + ), \ + TP_ARGS(clp, cb)) + +DEFINE_NFSD_CB_LIFETIME_EVENT(queue); +DEFINE_NFSD_CB_LIFETIME_EVENT(destroy); +DEFINE_NFSD_CB_LIFETIME_EVENT(restart); +DEFINE_NFSD_CB_LIFETIME_EVENT(bc_update); +DEFINE_NFSD_CB_LIFETIME_EVENT(bc_shutdown); + +TRACE_EVENT(nfsd_cb_seq_status, + TP_PROTO( + const struct rpc_task *task, + const struct nfsd4_callback *cb, + const struct nfsd4_session *session + ), + TP_ARGS(task, cb, session), + TP_STRUCT__entry( + __field(unsigned int, task_id) + __field(unsigned int, client_id) + __field(u32, cl_boot) + __field(u32, cl_id) + __field(u32, seqno) + __field(u32, reserved) + __field(int, tk_status) + __field(int, seq_status) + ), + TP_fast_assign( + const struct nfsd4_sessionid *sid = + (struct nfsd4_sessionid *)&session->se_sessionid; + + __entry->task_id = task->tk_pid; + __entry->client_id = task->tk_client ? + task->tk_client->cl_clid : -1; + __entry->cl_boot = sid->clientid.cl_boot; + __entry->cl_id = sid->clientid.cl_id; + __entry->seqno = sid->sequence; + __entry->reserved = sid->reserved; + __entry->tk_status = task->tk_status; + __entry->seq_status = cb->cb_seq_status; + ), + TP_printk(SUNRPC_TRACE_TASK_SPECIFIER + " sessionid=%08x:%08x:%08x:%08x tk_status=%d seq_status=%d", + __entry->task_id, __entry->client_id, + __entry->cl_boot, __entry->cl_id, + __entry->seqno, __entry->reserved, + __entry->tk_status, __entry->seq_status + ) +); + +TRACE_EVENT(nfsd_cb_free_slot, + TP_PROTO( + const struct rpc_task *task, + const struct nfsd4_callback *cb, + const struct nfsd4_session *session + ), + TP_ARGS(task, cb, session), + TP_STRUCT__entry( + __field(unsigned int, task_id) + __field(unsigned int, client_id) + __field(u32, cl_boot) + __field(u32, cl_id) + __field(u32, seqno) + __field(u32, reserved) + __field(u32, slot_seqno) + ), + TP_fast_assign( + const struct nfsd4_sessionid *sid = + (struct nfsd4_sessionid *)&session->se_sessionid; + + __entry->task_id = task->tk_pid; + __entry->client_id = task->tk_client ? + task->tk_client->cl_clid : -1; + __entry->cl_boot = sid->clientid.cl_boot; + __entry->cl_id = sid->clientid.cl_id; + __entry->seqno = sid->sequence; + __entry->reserved = sid->reserved; + __entry->slot_seqno = session->se_cb_seq_nr[cb->cb_held_slot]; + ), + TP_printk(SUNRPC_TRACE_TASK_SPECIFIER + " sessionid=%08x:%08x:%08x:%08x new slot seqno=%u", + __entry->task_id, __entry->client_id, + __entry->cl_boot, __entry->cl_id, + __entry->seqno, __entry->reserved, + __entry->slot_seqno + ) +); + +TRACE_EVENT_CONDITION(nfsd_cb_recall, + TP_PROTO( + const struct nfs4_stid *stid + ), + TP_ARGS(stid), + TP_CONDITION(stid->sc_client), + TP_STRUCT__entry( + __field(u32, cl_boot) + __field(u32, cl_id) + __field(u32, si_id) + __field(u32, si_generation) + __sockaddr(addr, stid->sc_client->cl_cb_conn.cb_addrlen) + ), + TP_fast_assign( + const stateid_t *stp = &stid->sc_stateid; + const struct nfs4_client *clp = stid->sc_client; + + __entry->cl_boot = stp->si_opaque.so_clid.cl_boot; + __entry->cl_id = stp->si_opaque.so_clid.cl_id; + __entry->si_id = stp->si_opaque.so_id; + __entry->si_generation = stp->si_generation; + __assign_sockaddr(addr, &clp->cl_cb_conn.cb_addr, + clp->cl_cb_conn.cb_addrlen) + ), + TP_printk("addr=%pISpc client %08x:%08x stateid %08x:%08x", + __get_sockaddr(addr), __entry->cl_boot, __entry->cl_id, + __entry->si_id, __entry->si_generation) +); + +TRACE_EVENT(nfsd_cb_notify_lock, + TP_PROTO( + const struct nfs4_lockowner *lo, + const struct nfsd4_blocked_lock *nbl + ), + TP_ARGS(lo, nbl), + TP_STRUCT__entry( + __field(u32, cl_boot) + __field(u32, cl_id) + __field(u32, fh_hash) + __sockaddr(addr, lo->lo_owner.so_client->cl_cb_conn.cb_addrlen) + ), + TP_fast_assign( + const struct nfs4_client *clp = lo->lo_owner.so_client; + + __entry->cl_boot = clp->cl_clientid.cl_boot; + __entry->cl_id = clp->cl_clientid.cl_id; + __entry->fh_hash = knfsd_fh_hash(&nbl->nbl_fh); + __assign_sockaddr(addr, &clp->cl_cb_conn.cb_addr, + clp->cl_cb_conn.cb_addrlen) + ), + TP_printk("addr=%pISpc client %08x:%08x fh_hash=0x%08x", + __get_sockaddr(addr), __entry->cl_boot, __entry->cl_id, + __entry->fh_hash) +); + +TRACE_EVENT(nfsd_cb_offload, + TP_PROTO( + const struct nfs4_client *clp, + const stateid_t *stp, + const struct knfsd_fh *fh, + u64 count, + __be32 status + ), + TP_ARGS(clp, stp, fh, count, status), + TP_STRUCT__entry( + __field(u32, cl_boot) + __field(u32, cl_id) + __field(u32, si_id) + __field(u32, si_generation) + __field(u32, fh_hash) + __field(int, status) + __field(u64, count) + __sockaddr(addr, clp->cl_cb_conn.cb_addrlen) + ), + TP_fast_assign( + __entry->cl_boot = stp->si_opaque.so_clid.cl_boot; + __entry->cl_id = stp->si_opaque.so_clid.cl_id; + __entry->si_id = stp->si_opaque.so_id; + __entry->si_generation = stp->si_generation; + __entry->fh_hash = knfsd_fh_hash(fh); + __entry->status = be32_to_cpu(status); + __entry->count = count; + __assign_sockaddr(addr, &clp->cl_cb_conn.cb_addr, + clp->cl_cb_conn.cb_addrlen) + ), + TP_printk("addr=%pISpc client %08x:%08x stateid %08x:%08x fh_hash=0x%08x count=%llu status=%d", + __get_sockaddr(addr), __entry->cl_boot, __entry->cl_id, + __entry->si_id, __entry->si_generation, + __entry->fh_hash, __entry->count, __entry->status) +); + +TRACE_EVENT(nfsd_cb_recall_any, + TP_PROTO( + const struct nfsd4_cb_recall_any *ra + ), + TP_ARGS(ra), + TP_STRUCT__entry( + __field(u32, cl_boot) + __field(u32, cl_id) + __field(u32, keep) + __field(unsigned long, bmval0) + __sockaddr(addr, ra->ra_cb.cb_clp->cl_cb_conn.cb_addrlen) + ), + TP_fast_assign( + __entry->cl_boot = ra->ra_cb.cb_clp->cl_clientid.cl_boot; + __entry->cl_id = ra->ra_cb.cb_clp->cl_clientid.cl_id; + __entry->keep = ra->ra_keep; + __entry->bmval0 = ra->ra_bmval[0]; + __assign_sockaddr(addr, &ra->ra_cb.cb_clp->cl_addr, + ra->ra_cb.cb_clp->cl_cb_conn.cb_addrlen); + ), + TP_printk("addr=%pISpc client %08x:%08x keep=%u bmval0=%s", + __get_sockaddr(addr), __entry->cl_boot, __entry->cl_id, + __entry->keep, show_rca_mask(__entry->bmval0) + ) +); + +DECLARE_EVENT_CLASS(nfsd_cb_done_class, + TP_PROTO( + const stateid_t *stp, + const struct rpc_task *task + ), + TP_ARGS(stp, task), + TP_STRUCT__entry( + __field(u32, cl_boot) + __field(u32, cl_id) + __field(u32, si_id) + __field(u32, si_generation) + __field(int, status) + ), + TP_fast_assign( + __entry->cl_boot = stp->si_opaque.so_clid.cl_boot; + __entry->cl_id = stp->si_opaque.so_clid.cl_id; + __entry->si_id = stp->si_opaque.so_id; + __entry->si_generation = stp->si_generation; + __entry->status = task->tk_status; + ), + TP_printk("client %08x:%08x stateid %08x:%08x status=%d", + __entry->cl_boot, __entry->cl_id, __entry->si_id, + __entry->si_generation, __entry->status + ) +); + +#define DEFINE_NFSD_CB_DONE_EVENT(name) \ +DEFINE_EVENT(nfsd_cb_done_class, name, \ + TP_PROTO( \ + const stateid_t *stp, \ + const struct rpc_task *task \ + ), \ + TP_ARGS(stp, task)) + +DEFINE_NFSD_CB_DONE_EVENT(nfsd_cb_recall_done); +DEFINE_NFSD_CB_DONE_EVENT(nfsd_cb_notify_lock_done); +DEFINE_NFSD_CB_DONE_EVENT(nfsd_cb_layout_done); +DEFINE_NFSD_CB_DONE_EVENT(nfsd_cb_offload_done); +DEFINE_NFSD_CB_DONE_EVENT(nfsd_cb_getattr_done); + +TRACE_EVENT(nfsd_cb_recall_any_done, + TP_PROTO( + const struct nfsd4_callback *cb, + const struct rpc_task *task + ), + TP_ARGS(cb, task), + TP_STRUCT__entry( + __field(u32, cl_boot) + __field(u32, cl_id) + __field(int, status) + ), + TP_fast_assign( + __entry->status = task->tk_status; + __entry->cl_boot = cb->cb_clp->cl_clientid.cl_boot; + __entry->cl_id = cb->cb_clp->cl_clientid.cl_id; + ), + TP_printk("client %08x:%08x status=%d", + __entry->cl_boot, __entry->cl_id, __entry->status + ) +); + +TRACE_EVENT(nfsd_ctl_unlock_ip, + TP_PROTO( + const struct net *net, + const struct sockaddr *addr, + const unsigned int addrlen + ), + TP_ARGS(net, addr, addrlen), + TP_STRUCT__entry( + __field(unsigned int, netns_ino) + __sockaddr(addr, addrlen) + ), + TP_fast_assign( + __entry->netns_ino = net->ns.inum; + __assign_sockaddr(addr, addr, addrlen); + ), + TP_printk("addr=%pISpc", + __get_sockaddr(addr) + ) +); + +TRACE_EVENT(nfsd_ctl_unlock_fs, + TP_PROTO( + const struct net *net, + const char *path + ), + TP_ARGS(net, path), + TP_STRUCT__entry( + __field(unsigned int, netns_ino) + __string(path, path) + ), + TP_fast_assign( + __entry->netns_ino = net->ns.inum; + __assign_str(path); + ), + TP_printk("path=%s", + __get_str(path) + ) +); + +TRACE_EVENT(nfsd_ctl_unlock_export, + TP_PROTO( + const struct net *net, + const char *path + ), + TP_ARGS(net, path), + TP_STRUCT__entry( + __field(unsigned int, netns_ino) + __string(path, path) + ), + TP_fast_assign( + __entry->netns_ino = net->ns.inum; + __assign_str(path); + ), + TP_printk("path=%s", + __get_str(path) + ) +); + +TRACE_EVENT(nfsd_ctl_filehandle, + TP_PROTO( + const struct net *net, + const char *domain, + const char *path, + int maxsize + ), + TP_ARGS(net, domain, path, maxsize), + TP_STRUCT__entry( + __field(unsigned int, netns_ino) + __field(int, maxsize) + __string(domain, domain) + __string(path, path) + ), + TP_fast_assign( + __entry->netns_ino = net->ns.inum; + __entry->maxsize = maxsize; + __assign_str(domain); + __assign_str(path); + ), + TP_printk("domain=%s path=%s maxsize=%d", + __get_str(domain), __get_str(path), __entry->maxsize + ) +); + +TRACE_EVENT(nfsd_ctl_threads, + TP_PROTO( + const struct net *net, + int newthreads + ), + TP_ARGS(net, newthreads), + TP_STRUCT__entry( + __field(unsigned int, netns_ino) + __field(int, newthreads) + ), + TP_fast_assign( + __entry->netns_ino = net->ns.inum; + __entry->newthreads = newthreads; + ), + TP_printk("newthreads=%d", + __entry->newthreads + ) +); + +TRACE_EVENT(nfsd_ctl_pool_threads, + TP_PROTO( + const struct net *net, + int pool, + int nrthreads + ), + TP_ARGS(net, pool, nrthreads), + TP_STRUCT__entry( + __field(unsigned int, netns_ino) + __field(int, pool) + __field(int, nrthreads) + ), + TP_fast_assign( + __entry->netns_ino = net->ns.inum; + __entry->pool = pool; + __entry->nrthreads = nrthreads; + ), + TP_printk("pool=%d nrthreads=%d", + __entry->pool, __entry->nrthreads + ) +); + +TRACE_EVENT(nfsd_ctl_version, + TP_PROTO( + const struct net *net, + const char *mesg + ), + TP_ARGS(net, mesg), + TP_STRUCT__entry( + __field(unsigned int, netns_ino) + __string(mesg, mesg) + ), + TP_fast_assign( + __entry->netns_ino = net->ns.inum; + __assign_str(mesg); + ), + TP_printk("%s", + __get_str(mesg) + ) +); + +TRACE_EVENT(nfsd_ctl_ports_addfd, + TP_PROTO( + const struct net *net, + int fd + ), + TP_ARGS(net, fd), + TP_STRUCT__entry( + __field(unsigned int, netns_ino) + __field(int, fd) + ), + TP_fast_assign( + __entry->netns_ino = net->ns.inum; + __entry->fd = fd; + ), + TP_printk("fd=%d", + __entry->fd + ) +); + +TRACE_EVENT(nfsd_ctl_ports_addxprt, + TP_PROTO( + const struct net *net, + const char *transport, + int port + ), + TP_ARGS(net, transport, port), + TP_STRUCT__entry( + __field(unsigned int, netns_ino) + __field(int, port) + __string(transport, transport) + ), + TP_fast_assign( + __entry->netns_ino = net->ns.inum; + __entry->port = port; + __assign_str(transport); + ), + TP_printk("transport=%s port=%d", + __get_str(transport), __entry->port + ) +); + +TRACE_EVENT(nfsd_ctl_maxblksize, + TP_PROTO( + const struct net *net, + int bsize + ), + TP_ARGS(net, bsize), + TP_STRUCT__entry( + __field(unsigned int, netns_ino) + __field(int, bsize) + ), + TP_fast_assign( + __entry->netns_ino = net->ns.inum; + __entry->bsize = bsize; + ), + TP_printk("bsize=%d", + __entry->bsize + ) +); + +TRACE_EVENT(nfsd_ctl_minthreads, + TP_PROTO( + const struct net *net, + int minthreads + ), + TP_ARGS(net, minthreads), + TP_STRUCT__entry( + __field(unsigned int, netns_ino) + __field(int, minthreads) + ), + TP_fast_assign( + __entry->netns_ino = net->ns.inum; + __entry->minthreads = minthreads + ), + TP_printk("minthreads=%d", + __entry->minthreads + ) +); + +TRACE_EVENT(nfsd_ctl_time, + TP_PROTO( + const struct net *net, + const char *name, + size_t namelen, + int time + ), + TP_ARGS(net, name, namelen, time), + TP_STRUCT__entry( + __field(unsigned int, netns_ino) + __field(int, time) + __string_len(name, name, namelen) + ), + TP_fast_assign( + __entry->netns_ino = net->ns.inum; + __entry->time = time; + __assign_str(name); + ), + TP_printk("file=%s time=%d", + __get_str(name), __entry->time + ) +); + +TRACE_EVENT(nfsd_ctl_recoverydir, + TP_PROTO( + const struct net *net, + const char *recdir + ), + TP_ARGS(net, recdir), + TP_STRUCT__entry( + __field(unsigned int, netns_ino) + __string(recdir, recdir) + ), + TP_fast_assign( + __entry->netns_ino = net->ns.inum; + __assign_str(recdir); + ), + TP_printk("recdir=%s", + __get_str(recdir) + ) +); + +TRACE_EVENT(nfsd_end_grace, + TP_PROTO( + const struct net *net + ), + TP_ARGS(net), + TP_STRUCT__entry( + __field(unsigned int, netns_ino) + ), + TP_fast_assign( + __entry->netns_ino = net->ns.inum; + ), + TP_printk("nn=%d", __entry->netns_ino + ) +); + +TRACE_EVENT(nfsd_ctl_fh_key_set, + TP_PROTO( + bool changed, + int result + ), + TP_ARGS(changed, result), + TP_STRUCT__entry( + __field(bool, changed) + __field(int, result) + ), + TP_fast_assign( + __entry->changed = changed; + __entry->result = result; + ), + TP_printk("key %s, result=%d", + __entry->changed ? "updated" : "unmodified", + __entry->result + ) +); + +DECLARE_EVENT_CLASS(nfsd_copy_class, + TP_PROTO( + const struct nfsd4_copy *copy + ), + TP_ARGS(copy), + TP_STRUCT__entry( + __field(bool, intra) + __field(bool, async) + __field(u32, src_cl_boot) + __field(u32, src_cl_id) + __field(u32, src_so_id) + __field(u32, src_si_generation) + __field(u32, dst_cl_boot) + __field(u32, dst_cl_id) + __field(u32, dst_so_id) + __field(u32, dst_si_generation) + __field(u32, cb_cl_boot) + __field(u32, cb_cl_id) + __field(u32, cb_so_id) + __field(u32, cb_si_generation) + __field(u64, src_cp_pos) + __field(u64, dst_cp_pos) + __field(u64, cp_count) + __sockaddr(addr, sizeof(struct sockaddr_in6)) + ), + TP_fast_assign( + const stateid_t *src_stp = ©->cp_src_stateid; + const stateid_t *dst_stp = ©->cp_dst_stateid; + const stateid_t *cb_stp = ©->cp_res.cb_stateid; + + __entry->intra = test_bit(NFSD4_COPY_F_INTRA, ©->cp_flags); + __entry->async = !test_bit(NFSD4_COPY_F_SYNCHRONOUS, ©->cp_flags); + __entry->src_cl_boot = src_stp->si_opaque.so_clid.cl_boot; + __entry->src_cl_id = src_stp->si_opaque.so_clid.cl_id; + __entry->src_so_id = src_stp->si_opaque.so_id; + __entry->src_si_generation = src_stp->si_generation; + __entry->dst_cl_boot = dst_stp->si_opaque.so_clid.cl_boot; + __entry->dst_cl_id = dst_stp->si_opaque.so_clid.cl_id; + __entry->dst_so_id = dst_stp->si_opaque.so_id; + __entry->dst_si_generation = dst_stp->si_generation; + __entry->cb_cl_boot = cb_stp->si_opaque.so_clid.cl_boot; + __entry->cb_cl_id = cb_stp->si_opaque.so_clid.cl_id; + __entry->cb_so_id = cb_stp->si_opaque.so_id; + __entry->cb_si_generation = cb_stp->si_generation; + __entry->src_cp_pos = copy->cp_src_pos; + __entry->dst_cp_pos = copy->cp_dst_pos; + __entry->cp_count = copy->cp_count; + __assign_sockaddr(addr, ©->cp_clp->cl_addr, + sizeof(struct sockaddr_in6)); + ), + TP_printk("client=%pISpc intra=%d async=%d " + "src_client %08x:%08x src_stateid %08x:%08x " + "dst_client %08x:%08x dst_stateid %08x:%08x " + "cb_client %08x:%08x cb_stateid %08x:%08x " + "cp_src_pos=%llu cp_dst_pos=%llu cp_count=%llu", + __get_sockaddr(addr), __entry->intra, __entry->async, + __entry->src_cl_boot, __entry->src_cl_id, + __entry->src_so_id, __entry->src_si_generation, + __entry->dst_cl_boot, __entry->dst_cl_id, + __entry->dst_so_id, __entry->dst_si_generation, + __entry->cb_cl_boot, __entry->cb_cl_id, + __entry->cb_so_id, __entry->cb_si_generation, + __entry->src_cp_pos, __entry->dst_cp_pos, __entry->cp_count + ) +); + +#define DEFINE_COPY_EVENT(name) \ +DEFINE_EVENT(nfsd_copy_class, nfsd_copy_##name, \ + TP_PROTO(const struct nfsd4_copy *copy), \ + TP_ARGS(copy)) + +DEFINE_COPY_EVENT(inter); +DEFINE_COPY_EVENT(intra); +DEFINE_COPY_EVENT(async); + +TRACE_EVENT(nfsd_copy_done, + TP_PROTO( + const struct nfsd4_copy *copy, + __be32 status + ), + TP_ARGS(copy, status), + TP_STRUCT__entry( + __field(int, status) + __field(bool, intra) + __field(bool, async) + __sockaddr(addr, sizeof(struct sockaddr_in6)) + ), + TP_fast_assign( + __entry->status = be32_to_cpu(status); + __entry->intra = test_bit(NFSD4_COPY_F_INTRA, ©->cp_flags); + __entry->async = !test_bit(NFSD4_COPY_F_SYNCHRONOUS, ©->cp_flags); + __assign_sockaddr(addr, ©->cp_clp->cl_addr, + sizeof(struct sockaddr_in6)); + ), + TP_printk("addr=%pISpc status=%d intra=%d async=%d", + __get_sockaddr(addr), __entry->status, __entry->intra, __entry->async + ) +); + +DECLARE_EVENT_CLASS(nfsd_copy_async_done_class, + TP_PROTO( + const struct nfsd4_copy *copy + ), + TP_ARGS(copy), + TP_STRUCT__entry( + __field(int, status) + __field(bool, intra) + __field(bool, async) + __field(u32, src_cl_boot) + __field(u32, src_cl_id) + __field(u32, src_so_id) + __field(u32, src_si_generation) + __field(u32, dst_cl_boot) + __field(u32, dst_cl_id) + __field(u32, dst_so_id) + __field(u32, dst_si_generation) + __field(u32, cb_cl_boot) + __field(u32, cb_cl_id) + __field(u32, cb_so_id) + __field(u32, cb_si_generation) + __field(u64, src_cp_pos) + __field(u64, dst_cp_pos) + __field(u64, cp_count) + __sockaddr(addr, sizeof(struct sockaddr_in6)) + ), + TP_fast_assign( + const stateid_t *src_stp = ©->cp_src_stateid; + const stateid_t *dst_stp = ©->cp_dst_stateid; + const stateid_t *cb_stp = ©->cp_res.cb_stateid; + + __entry->status = be32_to_cpu(copy->nfserr); + __entry->intra = test_bit(NFSD4_COPY_F_INTRA, ©->cp_flags); + __entry->async = !test_bit(NFSD4_COPY_F_SYNCHRONOUS, ©->cp_flags); + __entry->src_cl_boot = src_stp->si_opaque.so_clid.cl_boot; + __entry->src_cl_id = src_stp->si_opaque.so_clid.cl_id; + __entry->src_so_id = src_stp->si_opaque.so_id; + __entry->src_si_generation = src_stp->si_generation; + __entry->dst_cl_boot = dst_stp->si_opaque.so_clid.cl_boot; + __entry->dst_cl_id = dst_stp->si_opaque.so_clid.cl_id; + __entry->dst_so_id = dst_stp->si_opaque.so_id; + __entry->dst_si_generation = dst_stp->si_generation; + __entry->cb_cl_boot = cb_stp->si_opaque.so_clid.cl_boot; + __entry->cb_cl_id = cb_stp->si_opaque.so_clid.cl_id; + __entry->cb_so_id = cb_stp->si_opaque.so_id; + __entry->cb_si_generation = cb_stp->si_generation; + __entry->src_cp_pos = copy->cp_src_pos; + __entry->dst_cp_pos = copy->cp_dst_pos; + __entry->cp_count = copy->cp_count; + __assign_sockaddr(addr, ©->cp_clp->cl_addr, + sizeof(struct sockaddr_in6)); + ), + TP_printk("client=%pISpc status=%d intra=%d async=%d " + "src_client %08x:%08x src_stateid %08x:%08x " + "dst_client %08x:%08x dst_stateid %08x:%08x " + "cb_client %08x:%08x cb_stateid %08x:%08x " + "cp_src_pos=%llu cp_dst_pos=%llu cp_count=%llu", + __get_sockaddr(addr), + __entry->status, __entry->intra, __entry->async, + __entry->src_cl_boot, __entry->src_cl_id, + __entry->src_so_id, __entry->src_si_generation, + __entry->dst_cl_boot, __entry->dst_cl_id, + __entry->dst_so_id, __entry->dst_si_generation, + __entry->cb_cl_boot, __entry->cb_cl_id, + __entry->cb_so_id, __entry->cb_si_generation, + __entry->src_cp_pos, __entry->dst_cp_pos, __entry->cp_count + ) +); + +#define DEFINE_COPY_ASYNC_DONE_EVENT(name) \ +DEFINE_EVENT(nfsd_copy_async_done_class, \ + nfsd_copy_async_##name, \ + TP_PROTO(const struct nfsd4_copy *copy), \ + TP_ARGS(copy)) + +DEFINE_COPY_ASYNC_DONE_EVENT(done); +DEFINE_COPY_ASYNC_DONE_EVENT(cancel); + +TRACE_EVENT(nfsd_vfs_setattr, + TP_PROTO( + const struct svc_rqst *rqstp, + const struct svc_fh *fhp, + const struct iattr *iap, + const struct timespec64 *guardtime + ), + TP_ARGS(rqstp, fhp, iap, guardtime), + TP_STRUCT__entry( + NFSD_TRACE_PROC_CALL_FIELDS(rqstp) + __field(u32, fh_hash) + __field(s64, gtime_tv_sec) + __field(u32, gtime_tv_nsec) + __field(unsigned int, ia_valid) + __field(loff_t, ia_size) + __field(uid_t, ia_uid) + __field(gid_t, ia_gid) + __field(umode_t, ia_mode) + ), + TP_fast_assign( + NFSD_TRACE_PROC_CALL_ASSIGNMENTS(rqstp); + __entry->fh_hash = knfsd_fh_hash(&fhp->fh_handle); + __entry->gtime_tv_sec = guardtime ? guardtime->tv_sec : 0; + __entry->gtime_tv_nsec = guardtime ? guardtime->tv_nsec : 0; + __entry->ia_valid = iap->ia_valid; + __entry->ia_size = iap->ia_size; + __entry->ia_uid = __kuid_val(iap->ia_uid); + __entry->ia_gid = __kgid_val(iap->ia_gid); + __entry->ia_mode = iap->ia_mode; + ), + TP_printk( + "xid=0x%08x fh_hash=0x%08x ia_valid=%s ia_size=%llu ia_mode=0%o ia_uid=%u ia_gid=%u guard_time=%lld.%u", + __entry->xid, __entry->fh_hash, show_ia_valid_flags(__entry->ia_valid), + __entry->ia_size, __entry->ia_mode, __entry->ia_uid, __entry->ia_gid, + __entry->gtime_tv_sec, __entry->gtime_tv_nsec + ) +) + +TRACE_EVENT(nfsd_vfs_lookup, + TP_PROTO( + const struct svc_rqst *rqstp, + const struct svc_fh *fhp, + const char *name, + unsigned int len + ), + TP_ARGS(rqstp, fhp, name, len), + TP_STRUCT__entry( + NFSD_TRACE_PROC_CALL_FIELDS(rqstp) + __field(u32, fh_hash) + __string_len(name, name, len) + ), + TP_fast_assign( + NFSD_TRACE_PROC_CALL_ASSIGNMENTS(rqstp); + __entry->fh_hash = knfsd_fh_hash(&fhp->fh_handle); + __assign_str(name); + ), + TP_printk("xid=0x%08x fh_hash=0x%08x name=%s", + __entry->xid, __entry->fh_hash, __get_str(name) + ) +); + +TRACE_EVENT(nfsd_vfs_create, + TP_PROTO( + const struct svc_rqst *rqstp, + const struct svc_fh *fhp, + umode_t type, + const char *name, + unsigned int len + ), + TP_ARGS(rqstp, fhp, type, name, len), + TP_STRUCT__entry( + NFSD_TRACE_PROC_CALL_FIELDS(rqstp) + __field(u32, fh_hash) + __field(umode_t, type) + __string_len(name, name, len) + ), + TP_fast_assign( + NFSD_TRACE_PROC_CALL_ASSIGNMENTS(rqstp); + __entry->fh_hash = knfsd_fh_hash(&fhp->fh_handle); + __entry->type = type; + __assign_str(name); + ), + TP_printk("xid=0x%08x fh_hash=0x%08x type=%s name=%s", + __entry->xid, __entry->fh_hash, + show_fs_file_type(__entry->type), __get_str(name) + ) +); + +TRACE_EVENT(nfsd_vfs_symlink, + TP_PROTO( + const struct svc_rqst *rqstp, + const struct svc_fh *fhp, + const char *name, + unsigned int namelen, + const char *target + ), + TP_ARGS(rqstp, fhp, name, namelen, target), + TP_STRUCT__entry( + NFSD_TRACE_PROC_CALL_FIELDS(rqstp) + __field(u32, fh_hash) + __string_len(name, name, namelen) + __string(target, target) + ), + TP_fast_assign( + NFSD_TRACE_PROC_CALL_ASSIGNMENTS(rqstp); + __entry->fh_hash = knfsd_fh_hash(&fhp->fh_handle); + __assign_str(name); + __assign_str(target); + ), + TP_printk("xid=0x%08x fh_hash=0x%08x name=%s target=%s", + __entry->xid, __entry->fh_hash, + __get_str(name), __get_str(target) + ) +); + +TRACE_EVENT(nfsd_vfs_link, + TP_PROTO( + const struct svc_rqst *rqstp, + const struct svc_fh *sfhp, + const struct svc_fh *tfhp, + const char *name, + unsigned int namelen + ), + TP_ARGS(rqstp, sfhp, tfhp, name, namelen), + TP_STRUCT__entry( + NFSD_TRACE_PROC_CALL_FIELDS(rqstp) + __field(u32, sfh_hash) + __field(u32, tfh_hash) + __string_len(name, name, namelen) + ), + TP_fast_assign( + NFSD_TRACE_PROC_CALL_ASSIGNMENTS(rqstp); + __entry->sfh_hash = knfsd_fh_hash(&sfhp->fh_handle); + __entry->tfh_hash = knfsd_fh_hash(&tfhp->fh_handle); + __assign_str(name); + ), + TP_printk("xid=0x%08x src_fh=0x%08x tgt_fh=0x%08x name=%s", + __entry->xid, __entry->sfh_hash, __entry->tfh_hash, + __get_str(name) + ) +); + +TRACE_EVENT(nfsd_vfs_unlink, + TP_PROTO( + const struct svc_rqst *rqstp, + const struct svc_fh *fhp, + const char *name, + unsigned int len + ), + TP_ARGS(rqstp, fhp, name, len), + TP_STRUCT__entry( + NFSD_TRACE_PROC_CALL_FIELDS(rqstp) + __field(u32, fh_hash) + __string_len(name, name, len) + ), + TP_fast_assign( + NFSD_TRACE_PROC_CALL_ASSIGNMENTS(rqstp); + __entry->fh_hash = knfsd_fh_hash(&fhp->fh_handle); + __assign_str(name); + ), + TP_printk("xid=0x%08x fh_hash=0x%08x name=%s", + __entry->xid, __entry->fh_hash, + __get_str(name) + ) +); + +TRACE_EVENT(nfsd_vfs_rename, + TP_PROTO( + const struct svc_rqst *rqstp, + const struct svc_fh *sfhp, + const struct svc_fh *tfhp, + const char *source, + unsigned int sourcelen, + const char *target, + unsigned int targetlen + ), + TP_ARGS(rqstp, sfhp, tfhp, source, sourcelen, target, targetlen), + TP_STRUCT__entry( + NFSD_TRACE_PROC_CALL_FIELDS(rqstp) + __field(u32, sfh_hash) + __field(u32, tfh_hash) + __string_len(source, source, sourcelen) + __string_len(target, target, targetlen) + ), + TP_fast_assign( + NFSD_TRACE_PROC_CALL_ASSIGNMENTS(rqstp); + __entry->sfh_hash = knfsd_fh_hash(&sfhp->fh_handle); + __entry->tfh_hash = knfsd_fh_hash(&tfhp->fh_handle); + __assign_str(source); + __assign_str(target); + ), + TP_printk("xid=0x%08x sfh_hash=0x%08x tfh_hash=0x%08x source=%s target=%s", + __entry->xid, __entry->sfh_hash, __entry->tfh_hash, + __get_str(source), __get_str(target) + ) +); + +TRACE_EVENT(nfsd_vfs_readdir, + TP_PROTO( + const struct svc_rqst *rqstp, + const struct svc_fh *fhp, + u32 count, + u64 offset + ), + TP_ARGS(rqstp, fhp, count, offset), + TP_STRUCT__entry( + NFSD_TRACE_PROC_CALL_FIELDS(rqstp) + __field(u32, fh_hash) + __field(u32, count) + __field(u64, offset) + ), + TP_fast_assign( + NFSD_TRACE_PROC_CALL_ASSIGNMENTS(rqstp); + __entry->fh_hash = knfsd_fh_hash(&fhp->fh_handle); + __entry->count = count; + __entry->offset = offset; + ), + TP_printk("xid=0x%08x fh_hash=0x%08x offset=%llu count=%u", + __entry->xid, __entry->fh_hash, + __entry->offset, __entry->count + ) +); + +DECLARE_EVENT_CLASS(nfsd_vfs_getattr_class, + TP_PROTO( + const struct svc_rqst *rqstp, + const struct svc_fh *fhp + ), + TP_ARGS(rqstp, fhp), + TP_STRUCT__entry( + NFSD_TRACE_PROC_CALL_FIELDS(rqstp) + __field(u32, fh_hash) + ), + TP_fast_assign( + NFSD_TRACE_PROC_CALL_ASSIGNMENTS(rqstp); + __entry->fh_hash = knfsd_fh_hash(&fhp->fh_handle); + ), + TP_printk("xid=0x%08x fh_hash=0x%08x", + __entry->xid, __entry->fh_hash + ) +); + +#define DEFINE_NFSD_VFS_GETATTR_EVENT(__name) \ +DEFINE_EVENT(nfsd_vfs_getattr_class, __name, \ + TP_PROTO( \ + const struct svc_rqst *rqstp, \ + const struct svc_fh *fhp \ + ), \ + TP_ARGS(rqstp, fhp)) + +DEFINE_NFSD_VFS_GETATTR_EVENT(nfsd_vfs_getattr); +DEFINE_NFSD_VFS_GETATTR_EVENT(nfsd_vfs_statfs); + +DECLARE_EVENT_CLASS(nfsd_pnfs_class, + TP_PROTO( + const struct nfs4_client *clp, + const char *dev, + int error + ), + TP_ARGS(clp, dev, error), + TP_STRUCT__entry( + __sockaddr(addr, sizeof(struct sockaddr_in6)) + __field(unsigned int, netns_ino) + __string(dev, dev) + __field(int, error) + ), + TP_fast_assign( + __assign_sockaddr(addr, &clp->cl_addr, + sizeof(struct sockaddr_in6)); + __entry->netns_ino = clp->net->ns.inum; + __assign_str(dev); + __entry->error = error; + ), + TP_printk("client=%pISpc nn=%d dev=%s error=%d", + __get_sockaddr(addr), + __entry->netns_ino, + __get_str(dev), + __entry->error + ) +); + +#define DEFINE_NFSD_PNFS_ERR_EVENT(name) \ +DEFINE_EVENT(nfsd_pnfs_class, nfsd_pnfs_##name, \ + TP_PROTO( \ + const struct nfs4_client *clp, \ + const char *dev, \ + int error \ + ), \ + TP_ARGS(clp, dev, error)) + +DEFINE_NFSD_PNFS_ERR_EVENT(fence); +#endif /* _NFSD_TRACE_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE trace +#include diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c new file mode 100644 index 000000000..8923a9910 --- /dev/null +++ b/fs/nfsd/vfs.c @@ -0,0 +1,2998 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * File operations used by nfsd. Some of these have been ripped from + * other parts of the kernel because they weren't exported, others + * are partial duplicates with added or changed functionality. + * + * Note that several functions dget() the dentry upon which they want + * to act, most notably those that create directory entries. Response + * dentry's are dput()'d if necessary in the release callback. + * So if you notice code paths that apparently fail to dput() the + * dentry, don't worry--they have been taken care of. + * + * Copyright (C) 1995-1999 Olaf Kirch + * Zerocpy NFS support (C) 2002 Hirokazu Takahashi + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "xdr3.h" + +#ifdef CONFIG_NFSD_V4 +#include "acl.h" +#include "idmap.h" +#include "xdr4.h" +#endif /* CONFIG_NFSD_V4 */ + +#include "nfsd.h" +#include "netns.h" +#include "stats.h" +#include "vfs.h" +#include "filecache.h" +#include "trace.h" + +#define NFSDDBG_FACILITY NFSDDBG_FILEOP + +bool nfsd_disable_splice_read __read_mostly; +u64 nfsd_io_cache_read __read_mostly = NFSD_IO_BUFFERED; +u64 nfsd_io_cache_write __read_mostly = NFSD_IO_BUFFERED; + +/** + * nfserrno - Map Linux errnos to NFS errnos + * @errno: POSIX(-ish) error code to be mapped + * + * Returns the appropriate (net-endian) nfserr_* (or nfs_ok if errno is 0). If + * it's an error we don't expect, log it once and return nfserr_io. + */ +__be32 +nfserrno (int errno) +{ + static struct { + __be32 nfserr; + int syserr; + } nfs_errtbl[] = { + { nfs_ok, 0 }, + { nfserr_perm, -EPERM }, + { nfserr_noent, -ENOENT }, + { nfserr_io, -EIO }, + { nfserr_nxio, -ENXIO }, + { nfserr_fbig, -E2BIG }, + { nfserr_stale, -EBADF }, + { nfserr_acces, -EACCES }, + { nfserr_exist, -EEXIST }, + { nfserr_xdev, -EXDEV }, + { nfserr_nodev, -ENODEV }, + { nfserr_notdir, -ENOTDIR }, + { nfserr_isdir, -EISDIR }, + { nfserr_inval, -EINVAL }, + { nfserr_fbig, -EFBIG }, + { nfserr_nospc, -ENOSPC }, + { nfserr_rofs, -EROFS }, + { nfserr_mlink, -EMLINK }, + { nfserr_nametoolong, -ENAMETOOLONG }, + { nfserr_notempty, -ENOTEMPTY }, + { nfserr_dquot, -EDQUOT }, + { nfserr_stale, -ESTALE }, + { nfserr_jukebox, -ETIMEDOUT }, + { nfserr_jukebox, -ERESTARTSYS }, + { nfserr_jukebox, -EAGAIN }, + { nfserr_jukebox, -EWOULDBLOCK }, + { nfserr_jukebox, -ENOMEM }, + { nfserr_io, -ETXTBSY }, + { nfserr_notsupp, -EOPNOTSUPP }, + { nfserr_toosmall, -ETOOSMALL }, + { nfserr_serverfault, -ESERVERFAULT }, + { nfserr_serverfault, -ENFILE }, + { nfserr_io, -EREMOTEIO }, + { nfserr_stale, -EOPENSTALE }, + { nfserr_io, -EUCLEAN }, + { nfserr_perm, -ENOKEY }, + { nfserr_no_grace, -ENOGRACE}, + { nfserr_io, -EBADMSG }, + }; + int i; + + for (i = 0; i < ARRAY_SIZE(nfs_errtbl); i++) { + if (nfs_errtbl[i].syserr == errno) + return nfs_errtbl[i].nfserr; + } + WARN_ONCE(1, "nfsd: non-standard errno: %d\n", errno); + return nfserr_io; +} + +/* + * Called from nfsd_lookup and encode_dirent. Check if we have crossed + * a mount point. + * Returns -EAGAIN or -ETIMEDOUT leaving *dpp and *expp unchanged, + * or nfs_ok having possibly changed *dpp and *expp + */ +int +nfsd_cross_mnt(struct svc_rqst *rqstp, struct dentry **dpp, + struct svc_export **expp) +{ + struct svc_export *exp = *expp, *exp2 = NULL; + struct dentry *dentry = *dpp; + struct path path = {.mnt = mntget(exp->ex_path.mnt), + .dentry = dget(dentry)}; + unsigned int follow_flags = 0; + int err = 0; + + if (exp->ex_flags & NFSEXP_CROSSMOUNT) + follow_flags = LOOKUP_AUTOMOUNT; + + err = follow_down(&path, follow_flags); + if (err < 0) + goto out; + + if (path.mnt == exp->ex_path.mnt && path.dentry == dentry && + nfsd_mountpoint(dentry, exp) == 2) { + /* This is only a mountpoint in some other namespace */ + goto out; + } + + exp2 = rqst_exp_get_by_name(rqstp, &path); + if (IS_ERR(exp2)) { + err = PTR_ERR(exp2); + exp2 = NULL; + /* + * We normally allow NFS clients to continue + * "underneath" a mountpoint that is not exported. + * The exception is V4ROOT, where no traversal is ever + * allowed without an explicit export of the new + * directory. + */ + if (err == -ENOENT && !(exp->ex_flags & NFSEXP_V4ROOT)) + err = 0; + } else if (nfsd_v4client(rqstp) || + (exp->ex_flags & NFSEXP_CROSSMOUNT) || EX_NOHIDE(exp2)) { + /* successfully crossed mount point */ + /* + * This is subtle: path.dentry is *not* on path.mnt + * at this point. The only reason we are safe is that + * original mnt is pinned down by exp, so we should + * put path *before* putting exp + */ + *dpp = path.dentry; + path.dentry = dentry; + *expp = exp2; + exp2 = exp; + } +out: + path_put(&path); + if (exp2) + exp_put(exp2); + return err; +} + +static void follow_to_parent(struct path *path) +{ + struct dentry *dp; + + while (path->dentry == path->mnt->mnt_root && follow_up(path)) + ; + dp = dget_parent(path->dentry); + dput(path->dentry); + path->dentry = dp; +} + +static int nfsd_lookup_parent(struct svc_rqst *rqstp, struct dentry *dparent, struct svc_export **exp, struct dentry **dentryp) +{ + struct svc_export *exp2; + struct path path = {.mnt = mntget((*exp)->ex_path.mnt), + .dentry = dget(dparent)}; + + follow_to_parent(&path); + + exp2 = rqst_exp_parent(rqstp, &path); + if (PTR_ERR(exp2) == -ENOENT) { + *dentryp = dget(dparent); + } else if (IS_ERR(exp2)) { + path_put(&path); + return PTR_ERR(exp2); + } else { + *dentryp = dget(path.dentry); + exp_put(*exp); + *exp = exp2; + } + path_put(&path); + return 0; +} + +/* + * For nfsd purposes, we treat V4ROOT exports as though there was an + * export at *every* directory. + * We return: + * '1' if this dentry *must* be an export point, + * '2' if it might be, if there is really a mount here, and + * '0' if there is no chance of an export point here. + */ +int nfsd_mountpoint(struct dentry *dentry, struct svc_export *exp) +{ + if (!d_inode(dentry)) + return 0; + if (exp->ex_flags & NFSEXP_V4ROOT) + return 1; + if (nfsd4_is_junction(dentry)) + return 1; + if (d_managed(dentry)) + /* + * Might only be a mountpoint in a different namespace, + * but we need to check. + */ + return 2; + return 0; +} + +__be32 +nfsd_lookup_dentry(struct svc_rqst *rqstp, struct svc_fh *fhp, + const char *name, unsigned int len, + struct svc_export **exp_ret, struct dentry **dentry_ret) +{ + struct svc_export *exp; + struct dentry *dparent; + struct dentry *dentry; + int host_err; + + trace_nfsd_vfs_lookup(rqstp, fhp, name, len); + + dparent = fhp->fh_dentry; + exp = exp_get(fhp->fh_export); + + /* Lookup the name, but don't follow links */ + if (name_is_dot_dotdot(name, len)) { + if (len==1) + dentry = dget(dparent); + else if (dparent != exp->ex_path.dentry) + dentry = dget_parent(dparent); + else if (!EX_NOHIDE(exp) && !nfsd_v4client(rqstp)) + dentry = dget(dparent); /* .. == . just like at / */ + else { + /* checking mountpoint crossing is very different when stepping up */ + host_err = nfsd_lookup_parent(rqstp, dparent, &exp, &dentry); + if (host_err) + goto out_nfserr; + } + } else { + dentry = lookup_one_unlocked(&nop_mnt_idmap, + &QSTR_LEN(name, len), dparent); + host_err = PTR_ERR(dentry); + if (IS_ERR(dentry)) + goto out_nfserr; + if (nfsd_mountpoint(dentry, exp)) { + host_err = nfsd_cross_mnt(rqstp, &dentry, &exp); + if (host_err) { + dput(dentry); + goto out_nfserr; + } + } + } + *dentry_ret = dentry; + *exp_ret = exp; + return 0; + +out_nfserr: + exp_put(exp); + return nfserrno(host_err); +} + +/** + * nfsd_lookup - look up a single path component for nfsd + * + * @rqstp: the request context + * @fhp: the file handle of the directory + * @name: the component name, or %NULL to look up parent + * @len: length of name to examine + * @resfh: pointer to pre-initialised filehandle to hold result. + * + * Look up one component of a pathname. + * N.B. After this call _both_ fhp and resfh need an fh_put + * + * If the lookup would cross a mountpoint, and the mounted filesystem + * is exported to the client with NFSEXP_NOHIDE, then the lookup is + * accepted as it stands and the mounted directory is + * returned. Otherwise the covered directory is returned. + * NOTE: this mountpoint crossing is not supported properly by all + * clients and is explicitly disallowed for NFSv3 + * + */ +__be32 +nfsd_lookup(struct svc_rqst *rqstp, struct svc_fh *fhp, const char *name, + unsigned int len, struct svc_fh *resfh) +{ + struct svc_export *exp; + struct dentry *dentry; + __be32 err; + + err = fh_verify(rqstp, fhp, S_IFDIR, NFSD_MAY_EXEC); + if (err) + return err; + err = nfsd_lookup_dentry(rqstp, fhp, name, len, &exp, &dentry); + if (err) + return err; + err = check_nfsd_access(exp, rqstp, false); + if (err) + goto out; + /* + * Note: we compose the file handle now, but as the + * dentry may be negative, it may need to be updated. + */ + err = fh_compose(resfh, exp, dentry, fhp); + if (!err && d_really_is_negative(dentry)) + err = nfserr_noent; +out: + dput(dentry); + exp_put(exp); + return err; +} + +static void +commit_reset_write_verifier(struct nfsd_net *nn, struct svc_rqst *rqstp, + int err) +{ + switch (err) { + case -EAGAIN: + case -ESTALE: + /* + * Neither of these are the result of a problem with + * durable storage, so avoid a write verifier reset. + */ + break; + default: + nfsd_reset_write_verifier(nn); + trace_nfsd_writeverf_reset(nn, rqstp, err); + } +} + +/* + * Commit metadata changes to stable storage. + */ +static int +commit_inode_metadata(struct inode *inode) +{ + const struct export_operations *export_ops = inode->i_sb->s_export_op; + + if (export_ops->commit_metadata) + return export_ops->commit_metadata(inode); + return sync_inode_metadata(inode, 1); +} + +static int +commit_metadata(struct svc_fh *fhp) +{ + struct inode *inode = d_inode(fhp->fh_dentry); + + if (!EX_ISSYNC(fhp->fh_export)) + return 0; + return commit_inode_metadata(inode); +} + +/* + * Go over the attributes and take care of the small differences between + * NFS semantics and what Linux expects. + */ +static void +nfsd_sanitize_attrs(struct inode *inode, struct iattr *iap) +{ + /* Ignore mode updates on symlinks */ + if (S_ISLNK(inode->i_mode)) + iap->ia_valid &= ~ATTR_MODE; + + /* sanitize the mode change */ + if (iap->ia_valid & ATTR_MODE) { + iap->ia_mode &= S_IALLUGO; + iap->ia_mode |= (inode->i_mode & ~S_IALLUGO); + } + + /* Revoke setuid/setgid on chown */ + if (!S_ISDIR(inode->i_mode) && + ((iap->ia_valid & ATTR_UID) || (iap->ia_valid & ATTR_GID))) { + iap->ia_valid |= ATTR_KILL_PRIV; + if (iap->ia_valid & ATTR_MODE) { + /* we're setting mode too, just clear the s*id bits */ + iap->ia_mode &= ~S_ISUID; + if (iap->ia_mode & S_IXGRP) + iap->ia_mode &= ~S_ISGID; + } else { + /* set ATTR_KILL_* bits and let VFS handle it */ + iap->ia_valid |= ATTR_KILL_SUID; + iap->ia_valid |= + setattr_should_drop_sgid(&nop_mnt_idmap, inode); + } + } +} + +static __be32 +nfsd_may_truncate(struct svc_rqst *rqstp, struct svc_fh *fhp, + struct iattr *iap) +{ + struct inode *inode = d_inode(fhp->fh_dentry); + + if (iap->ia_size >= i_size_read(inode)) + return nfs_ok; + + return nfsd_permission(&rqstp->rq_cred, fhp->fh_export, fhp->fh_dentry, + NFSD_MAY_TRUNC | NFSD_MAY_OWNER_OVERRIDE); +} + +static __be32 +nfsd_get_write_access(struct svc_fh *fhp) +{ + return nfserrno(get_write_access(d_inode(fhp->fh_dentry))); +} + +static int __nfsd_setattr(struct dentry *dentry, struct iattr *iap) +{ + int host_err; + + if (iap->ia_valid & ATTR_SIZE) { + /* + * RFC5661, Section 18.30.4: + * Changing the size of a file with SETATTR indirectly + * changes the time_modify and change attributes. + * + * (and similar for the older RFCs) + */ + struct iattr size_attr = { + .ia_valid = ATTR_SIZE | ATTR_CTIME | ATTR_MTIME, + .ia_size = iap->ia_size, + }; + + if (iap->ia_size < 0) + return -EFBIG; + + host_err = notify_change(&nop_mnt_idmap, dentry, &size_attr, NULL); + if (host_err) + return host_err; + iap->ia_valid &= ~ATTR_SIZE; + + /* + * Avoid the additional setattr call below if the only other + * attribute that the client sends is the mtime, as we update + * it as part of the size change above. + */ + if ((iap->ia_valid & ~ATTR_MTIME) == 0) + return 0; + } + + if ((iap->ia_valid & ~ATTR_DELEG) == 0) + return 0; + + /* + * If ATTR_DELEG is set, then this is an update from a client that + * holds a delegation. If this is an update for only the atime, the + * ctime should not be changed. If the update contains the mtime + * too, then ATTR_CTIME should already be set. + */ + if (!(iap->ia_valid & ATTR_DELEG)) + iap->ia_valid |= ATTR_CTIME; + + return notify_change(&nop_mnt_idmap, dentry, iap, NULL); +} + +/** + * nfsd_setattr - Set various file attributes. + * @rqstp: controlling RPC transaction + * @fhp: filehandle of target + * @attr: attributes to set + * @guardtime: do not act if ctime.tv_sec does not match this timestamp + * + * This call may adjust the contents of @attr (in particular, this + * call may change the bits in the na_iattr.ia_valid field). + * + * Returns nfs_ok on success, otherwise an NFS status code is + * returned. Caller must release @fhp by calling fh_put in either + * case. + */ +__be32 +nfsd_setattr(struct svc_rqst *rqstp, struct svc_fh *fhp, + struct nfsd_attrs *attr, const struct timespec64 *guardtime) +{ + struct dentry *dentry; + struct inode *inode; + struct iattr *iap = attr->na_iattr; + int accmode = NFSD_MAY_SATTR; + umode_t ftype = 0; + __be32 err; + int host_err = 0; + bool get_write_count; + bool size_change = (iap->ia_valid & ATTR_SIZE); + int retries; + + trace_nfsd_vfs_setattr(rqstp, fhp, iap, guardtime); + + if (iap->ia_valid & ATTR_SIZE) { + accmode |= NFSD_MAY_WRITE|NFSD_MAY_OWNER_OVERRIDE; + ftype = S_IFREG; + } + + /* + * If utimes(2) and friends are called with times not NULL, we should + * not set NFSD_MAY_WRITE bit. Otherwise fh_verify->nfsd_permission + * will return EACCES, when the caller's effective UID does not match + * the owner of the file, and the caller is not privileged. In this + * situation, we should return EPERM(notify_change will return this). + */ + if (iap->ia_valid & (ATTR_ATIME | ATTR_MTIME)) { + accmode |= NFSD_MAY_OWNER_OVERRIDE; + if (!(iap->ia_valid & (ATTR_ATIME_SET | ATTR_MTIME_SET))) + accmode |= NFSD_MAY_WRITE; + } + + /* Callers that do fh_verify should do the fh_want_write: */ + get_write_count = !fhp->fh_dentry; + + /* Get inode */ + err = fh_verify(rqstp, fhp, ftype, accmode); + if (err) + return err; + if (get_write_count) { + host_err = fh_want_write(fhp); + if (host_err) + goto out; + } + + dentry = fhp->fh_dentry; + inode = d_inode(dentry); + + nfsd_sanitize_attrs(inode, iap); + + /* + * The size case is special, it changes the file in addition to the + * attributes, and file systems don't expect it to be mixed with + * "random" attribute changes. We thus split out the size change + * into a separate call to ->setattr, and do the rest as a separate + * setattr call. + */ + if (size_change) { + err = nfsd_get_write_access(fhp); + if (err) + return err; + } + + inode_lock(inode); + if (size_change) { + err = nfsd_may_truncate(rqstp, fhp, iap); + if (err) + goto out_unlock; + } + err = fh_fill_pre_attrs(fhp); + if (err) + goto out_unlock; + + if (guardtime) { + struct timespec64 ctime = inode_get_ctime(inode); + if ((u32)guardtime->tv_sec != (u32)ctime.tv_sec || + guardtime->tv_nsec != ctime.tv_nsec) { + err = nfserr_notsync; + goto out_fill_attrs; + } + } + + for (retries = 1;;) { + struct iattr attrs; + + /* + * notify_change() can alter its iattr argument, making + * @iap unsuitable for submission multiple times. Make a + * copy for every loop iteration. + */ + attrs = *iap; + host_err = __nfsd_setattr(dentry, &attrs); + if (host_err != -EAGAIN || !retries--) + break; + if (!nfsd_wait_for_delegreturn(rqstp, inode)) + break; + } + if (attr->na_seclabel && attr->na_seclabel->len) + attr->na_labelerr = security_inode_setsecctx(dentry, + attr->na_seclabel->data, attr->na_seclabel->len); + if (IS_ENABLED(CONFIG_FS_POSIX_ACL) && attr->na_dpacl) { + if (!S_ISDIR(inode->i_mode)) + attr->na_dpaclerr = -EINVAL; + else if (attr->na_dpacl->a_count > 0) + /* a_count == 0 means delete the ACL. */ + attr->na_dpaclerr = set_posix_acl(&nop_mnt_idmap, + dentry, ACL_TYPE_DEFAULT, + attr->na_dpacl); + else + attr->na_dpaclerr = set_posix_acl(&nop_mnt_idmap, + dentry, ACL_TYPE_DEFAULT, + NULL); + } + if (IS_ENABLED(CONFIG_FS_POSIX_ACL) && attr->na_pacl) { + /* + * For any file system that is not ACL_SCOPE_FILE_OBJECT, + * a_count == 0 MUST reply nfserr_inval. + * For a file system that is ACL_SCOPE_FILE_OBJECT, + * a_count == 0 deletes the ACL. + * XXX File systems that are ACL_SCOPE_FILE_OBJECT + * are not yet supported. + */ + if (attr->na_pacl->a_count > 0) + attr->na_paclerr = set_posix_acl(&nop_mnt_idmap, + dentry, ACL_TYPE_ACCESS, + attr->na_pacl); + else + attr->na_paclerr = -EINVAL; + } +out_fill_attrs: + /* + * RFC 1813 Section 3.3.2 does not mandate that an NFS server + * returns wcc_data for SETATTR. Some client implementations + * depend on receiving wcc_data, however, to sort out partial + * updates (eg., the client requested that size and mode be + * modified, but the server changed only the file mode). + */ + fh_fill_post_attrs(fhp); +out_unlock: + inode_unlock(inode); + if (size_change) + put_write_access(inode); +out: + if (!host_err) + host_err = commit_metadata(fhp); + return err != 0 ? err : nfserrno(host_err); +} + +#if defined(CONFIG_NFSD_V4) +/* + * NFS junction information is stored in an extended attribute. + */ +#define NFSD_JUNCTION_XATTR_NAME XATTR_TRUSTED_PREFIX "junction.nfs" + +/** + * nfsd4_is_junction - Test if an object could be an NFS junction + * + * @dentry: object to test + * + * Returns 1 if "dentry" appears to contain NFS junction information. + * Otherwise 0 is returned. + */ +int nfsd4_is_junction(struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + + if (inode == NULL) + return 0; + if (inode->i_mode & S_IXUGO) + return 0; + if (!(inode->i_mode & S_ISVTX)) + return 0; + if (vfs_getxattr(&nop_mnt_idmap, dentry, NFSD_JUNCTION_XATTR_NAME, + NULL, 0) <= 0) + return 0; + return 1; +} + +static struct nfsd4_compound_state *nfsd4_get_cstate(struct svc_rqst *rqstp) +{ + return &((struct nfsd4_compoundres *)rqstp->rq_resp)->cstate; +} + +__be32 nfsd4_clone_file_range(struct svc_rqst *rqstp, + struct nfsd_file *nf_src, u64 src_pos, + struct nfsd_file *nf_dst, u64 dst_pos, + u64 count, bool sync) +{ + struct file *src = nf_src->nf_file; + struct file *dst = nf_dst->nf_file; + errseq_t since; + loff_t cloned; + __be32 ret = 0; + + since = READ_ONCE(dst->f_wb_err); + cloned = vfs_clone_file_range(src, src_pos, dst, dst_pos, count, 0); + if (cloned < 0) { + ret = nfserrno(cloned); + goto out_err; + } + if (count && cloned != count) { + ret = nfserrno(-EINVAL); + goto out_err; + } + if (sync) { + loff_t dst_end = count ? dst_pos + count - 1 : LLONG_MAX; + int status = vfs_fsync_range(dst, dst_pos, dst_end, 0); + + if (!status) + status = filemap_check_wb_err(dst->f_mapping, since); + if (!status) + status = commit_inode_metadata(file_inode(src)); + if (status < 0) { + struct nfsd_net *nn = net_generic(nf_dst->nf_net, + nfsd_net_id); + + trace_nfsd_clone_file_range_err(rqstp, + &nfsd4_get_cstate(rqstp)->save_fh, + src_pos, + &nfsd4_get_cstate(rqstp)->current_fh, + dst_pos, + count, status); + commit_reset_write_verifier(nn, rqstp, status); + ret = nfserrno(status); + } + } +out_err: + return ret; +} + +ssize_t nfsd_copy_file_range(struct file *src, u64 src_pos, struct file *dst, + u64 dst_pos, u64 count) +{ + ssize_t ret; + + /* + * Limit copy to 4MB to prevent indefinitely blocking an nfsd + * thread and client rpc slot. The choice of 4MB is somewhat + * arbitrary. We might instead base this on r/wsize, or make it + * tunable, or use a time instead of a byte limit, or implement + * asynchronous copy. In theory a client could also recognize a + * limit like this and pipeline multiple COPY requests. + */ + count = min_t(u64, count, 1 << 22); + ret = vfs_copy_file_range(src, src_pos, dst, dst_pos, count, 0); + + if (ret == -EOPNOTSUPP || ret == -EXDEV) + ret = vfs_copy_file_range(src, src_pos, dst, dst_pos, count, + COPY_FILE_SPLICE); + return ret; +} + +__be32 nfsd4_vfs_fallocate(struct svc_rqst *rqstp, struct svc_fh *fhp, + struct file *file, loff_t offset, loff_t len, + int flags) +{ + int error; + + if (!S_ISREG(file_inode(file)->i_mode)) + return nfserr_inval; + + error = vfs_fallocate(file, flags, offset, len); + if (!error) + error = commit_metadata(fhp); + + return nfserrno(error); +} +#endif /* defined(CONFIG_NFSD_V4) */ + +/* + * Check server access rights to a file system object + */ +struct accessmap { + u32 access; + int how; +}; +static struct accessmap nfs3_regaccess[] = { + { NFS3_ACCESS_READ, NFSD_MAY_READ }, + { NFS3_ACCESS_EXECUTE, NFSD_MAY_EXEC }, + { NFS3_ACCESS_MODIFY, NFSD_MAY_WRITE|NFSD_MAY_TRUNC }, + { NFS3_ACCESS_EXTEND, NFSD_MAY_WRITE }, + +#ifdef CONFIG_NFSD_V4 + { NFS4_ACCESS_XAREAD, NFSD_MAY_READ }, + { NFS4_ACCESS_XAWRITE, NFSD_MAY_WRITE }, + { NFS4_ACCESS_XALIST, NFSD_MAY_READ }, +#endif + + { 0, 0 } +}; + +static struct accessmap nfs3_diraccess[] = { + { NFS3_ACCESS_READ, NFSD_MAY_READ }, + { NFS3_ACCESS_LOOKUP, NFSD_MAY_EXEC }, + { NFS3_ACCESS_MODIFY, NFSD_MAY_EXEC|NFSD_MAY_WRITE|NFSD_MAY_TRUNC}, + { NFS3_ACCESS_EXTEND, NFSD_MAY_EXEC|NFSD_MAY_WRITE }, + { NFS3_ACCESS_DELETE, NFSD_MAY_REMOVE }, + +#ifdef CONFIG_NFSD_V4 + { NFS4_ACCESS_XAREAD, NFSD_MAY_READ }, + { NFS4_ACCESS_XAWRITE, NFSD_MAY_WRITE }, + { NFS4_ACCESS_XALIST, NFSD_MAY_READ }, +#endif + + { 0, 0 } +}; + +static struct accessmap nfs3_anyaccess[] = { + /* Some clients - Solaris 2.6 at least, make an access call + * to the server to check for access for things like /dev/null + * (which really, the server doesn't care about). So + * We provide simple access checking for them, looking + * mainly at mode bits, and we make sure to ignore read-only + * filesystem checks + */ + { NFS3_ACCESS_READ, NFSD_MAY_READ }, + { NFS3_ACCESS_EXECUTE, NFSD_MAY_EXEC }, + { NFS3_ACCESS_MODIFY, NFSD_MAY_WRITE|NFSD_MAY_LOCAL_ACCESS }, + { NFS3_ACCESS_EXTEND, NFSD_MAY_WRITE|NFSD_MAY_LOCAL_ACCESS }, + + { 0, 0 } +}; + +__be32 +nfsd_access(struct svc_rqst *rqstp, struct svc_fh *fhp, u32 *access, u32 *supported) +{ + struct accessmap *map; + struct svc_export *export; + struct dentry *dentry; + u32 query, result = 0, sresult = 0; + __be32 error; + + error = fh_verify(rqstp, fhp, 0, NFSD_MAY_NOP); + if (error) + goto out; + + export = fhp->fh_export; + dentry = fhp->fh_dentry; + + if (d_is_reg(dentry)) + map = nfs3_regaccess; + else if (d_is_dir(dentry)) + map = nfs3_diraccess; + else + map = nfs3_anyaccess; + + + query = *access; + for (; map->access; map++) { + if (map->access & query) { + __be32 err2; + + sresult |= map->access; + + err2 = nfsd_permission(&rqstp->rq_cred, export, + dentry, map->how); + switch (err2) { + case nfs_ok: + result |= map->access; + break; + + /* the following error codes just mean the access was not allowed, + * rather than an error occurred */ + case nfserr_rofs: + case nfserr_acces: + case nfserr_perm: + /* simply don't "or" in the access bit. */ + break; + default: + error = err2; + goto out; + } + } + } + *access = result; + if (supported) + *supported = sresult; + + out: + return error; +} + +int nfsd_open_break_lease(struct inode *inode, int access) +{ + unsigned int mode; + + if (access & NFSD_MAY_NOT_BREAK_LEASE) + return 0; + mode = (access & NFSD_MAY_WRITE) ? O_WRONLY : O_RDONLY; + return break_lease(inode, mode | O_NONBLOCK); +} + +/* + * Open an existing file or directory. + * The may_flags argument indicates the type of open (read/write/lock) + * and additional flags. + * N.B. After this call fhp needs an fh_put + */ +static int +__nfsd_open(struct svc_fh *fhp, umode_t type, int may_flags, struct file **filp) +{ + struct path path; + struct inode *inode; + struct file *file; + int flags = O_RDONLY|O_LARGEFILE; + int host_err = -EPERM; + + path.mnt = fhp->fh_export->ex_path.mnt; + path.dentry = fhp->fh_dentry; + inode = d_inode(path.dentry); + + if (IS_APPEND(inode) && (may_flags & NFSD_MAY_WRITE)) + goto out; + + if (!inode->i_fop) + goto out; + + host_err = nfsd_open_break_lease(inode, may_flags); + if (host_err) /* NOMEM or WOULDBLOCK */ + goto out; + + if (may_flags & NFSD_MAY_WRITE) { + if (may_flags & NFSD_MAY_READ) + flags = O_RDWR|O_LARGEFILE; + else + flags = O_WRONLY|O_LARGEFILE; + } + + file = dentry_open(&path, flags, current_cred()); + if (IS_ERR(file)) { + host_err = PTR_ERR(file); + goto out; + } + + host_err = security_file_post_open(file, may_flags); + if (host_err) { + fput(file); + goto out; + } + + *filp = file; +out: + return host_err; +} + +__be32 +nfsd_open(struct svc_rqst *rqstp, struct svc_fh *fhp, umode_t type, + int may_flags, struct file **filp) +{ + __be32 err; + int host_err; + bool retried = false; + + /* + * If we get here, then the client has already done an "open", + * and (hopefully) checked permission - so allow OWNER_OVERRIDE + * in case a chmod has now revoked permission. + * + * Arguably we should also allow the owner override for + * directories, but we never have and it doesn't seem to have + * caused anyone a problem. If we were to change this, note + * also that our filldir callbacks would need a variant of + * lookup_one_positive_unlocked() that doesn't check permissions. + */ + if (type == S_IFREG) + may_flags |= NFSD_MAY_OWNER_OVERRIDE; +retry: + err = fh_verify(rqstp, fhp, type, may_flags); + if (!err) { + host_err = __nfsd_open(fhp, type, may_flags, filp); + if (host_err == -EOPENSTALE && !retried) { + retried = true; + fh_put(fhp); + goto retry; + } + err = nfserrno(host_err); + } + return err; +} + +/** + * nfsd_open_verified - Open a regular file for the filecache + * @fhp: NFS filehandle of the file to open + * @type: S_IFMT inode type allowed (0 means any type is allowed) + * @may_flags: internal permission flags + * @filp: OUT: open "struct file *" + * + * Returns zero on success, or a negative errno value. + */ +int +nfsd_open_verified(struct svc_fh *fhp, umode_t type, int may_flags, struct file **filp) +{ + return __nfsd_open(fhp, type, may_flags, filp); +} + +/* + * Grab and keep cached pages associated with a file in the svc_rqst + * so that they can be passed to the network sendmsg routines + * directly. They will be released after the sending has completed. + * + * Return values: Number of bytes consumed, or -EIO if there are no + * remaining pages in rqstp->rq_pages. + */ +static int +nfsd_splice_actor(struct pipe_inode_info *pipe, struct pipe_buffer *buf, + struct splice_desc *sd) +{ + struct svc_rqst *rqstp = sd->u.data; + struct page *page = buf->page; // may be a compound one + unsigned offset = buf->offset; + struct page *last_page; + + last_page = page + (offset + sd->len - 1) / PAGE_SIZE; + for (page += offset / PAGE_SIZE; page <= last_page; page++) { + /* + * Skip page replacement when extending the contents of the + * current page. But note that we may get two zero_pages in a + * row from shmem. + */ + if (page == *(rqstp->rq_next_page - 1) && + offset_in_page(rqstp->rq_res.page_base + + rqstp->rq_res.page_len)) + continue; + if (unlikely(!svc_rqst_replace_page(rqstp, page))) + return -EIO; + } + if (rqstp->rq_res.page_len == 0) // first call + rqstp->rq_res.page_base = offset % PAGE_SIZE; + rqstp->rq_res.page_len += sd->len; + return sd->len; +} + +static int nfsd_direct_splice_actor(struct pipe_inode_info *pipe, + struct splice_desc *sd) +{ + return __splice_from_pipe(pipe, sd, nfsd_splice_actor); +} + +static u32 nfsd_eof_on_read(struct file *file, loff_t offset, ssize_t len, + size_t expected) +{ + if (expected != 0 && len == 0) + return 1; + if (offset+len >= i_size_read(file_inode(file))) + return 1; + return 0; +} + +static __be32 nfsd_finish_read(struct svc_rqst *rqstp, struct svc_fh *fhp, + struct file *file, loff_t offset, + unsigned long *count, u32 *eof, ssize_t host_err) +{ + if (host_err >= 0) { + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + + nfsd_stats_io_read_add(nn, fhp->fh_export, host_err); + *eof = nfsd_eof_on_read(file, offset, host_err, *count); + *count = host_err; + fsnotify_access(file); + trace_nfsd_read_io_done(rqstp, fhp, offset, *count); + return 0; + } else { + trace_nfsd_read_err(rqstp, fhp, offset, host_err); + return nfserrno(host_err); + } +} + +/** + * nfsd_splice_read - Perform a VFS read using a splice pipe + * @rqstp: RPC transaction context + * @fhp: file handle of file to be read + * @file: opened struct file of file to be read + * @offset: starting byte offset + * @count: IN: requested number of bytes; OUT: number of bytes read + * @eof: OUT: set non-zero if operation reached the end of the file + * + * Returns nfs_ok on success, otherwise an nfserr stat value is + * returned. + */ +__be32 nfsd_splice_read(struct svc_rqst *rqstp, struct svc_fh *fhp, + struct file *file, loff_t offset, unsigned long *count, + u32 *eof) +{ + struct splice_desc sd = { + .len = 0, + .total_len = *count, + .pos = offset, + .u.data = rqstp, + }; + ssize_t host_err; + + trace_nfsd_read_splice(rqstp, fhp, offset, *count); + host_err = rw_verify_area(READ, file, &offset, *count); + if (!host_err) + host_err = splice_direct_to_actor(file, &sd, + nfsd_direct_splice_actor); + return nfsd_finish_read(rqstp, fhp, file, offset, count, eof, host_err); +} + +/* + * The byte range of the client's READ request is expanded on both ends + * until it meets the underlying file system's direct I/O alignment + * requirements. After the internal read is complete, the byte range of + * the NFS READ payload is reduced to the byte range that was originally + * requested. + * + * Note that a direct read can be done only when the xdr_buf containing + * the NFS READ reply does not already have contents in its .pages array. + * This is due to potentially restrictive alignment requirements on the + * read buffer. When .page_len and @base are zero, the .pages array is + * guaranteed to be page-aligned. + */ +static noinline_for_stack __be32 +nfsd_direct_read(struct svc_rqst *rqstp, struct svc_fh *fhp, + struct nfsd_file *nf, loff_t offset, unsigned long *count, + u32 *eof) +{ + u64 dio_start, dio_end; + unsigned long v, total; + struct iov_iter iter; + struct kiocb kiocb; + ssize_t host_err; + size_t len; + + init_sync_kiocb(&kiocb, nf->nf_file); + kiocb.ki_flags |= IOCB_DIRECT; + + /* Read a properly-aligned region of bytes into rq_bvec */ + dio_start = round_down(offset, nf->nf_dio_read_offset_align); + dio_end = round_up((u64)offset + *count, nf->nf_dio_read_offset_align); + + kiocb.ki_pos = dio_start; + + v = 0; + total = dio_end - dio_start; + while (total && v < rqstp->rq_maxpages && + rqstp->rq_next_page < rqstp->rq_page_end) { + len = min_t(size_t, total, PAGE_SIZE); + bvec_set_page(&rqstp->rq_bvec[v], *rqstp->rq_next_page, + len, 0); + + total -= len; + ++rqstp->rq_next_page; + ++v; + } + + trace_nfsd_read_direct(rqstp, fhp, offset, *count - total); + iov_iter_bvec(&iter, ITER_DEST, rqstp->rq_bvec, v, + dio_end - dio_start - total); + + host_err = vfs_iocb_iter_read(nf->nf_file, &kiocb, &iter); + if (host_err >= 0) { + unsigned int pad = offset - dio_start; + + /* The returned payload starts after the pad */ + rqstp->rq_res.page_base = pad; + + /* Compute the count of bytes to be returned */ + if (host_err > pad + *count) + host_err = *count; + else if (host_err > pad) + host_err -= pad; + else + host_err = 0; + } else if (unlikely(host_err == -EINVAL)) { + struct inode *inode = d_inode(fhp->fh_dentry); + + pr_info_ratelimited("nfsd: Direct I/O alignment failure on %s/%llu\n", + inode->i_sb->s_id, inode->i_ino); + host_err = -ESERVERFAULT; + } + + return nfsd_finish_read(rqstp, fhp, nf->nf_file, offset, count, + eof, host_err); +} + +/** + * nfsd_iter_read - Perform a VFS read using an iterator + * @rqstp: RPC transaction context + * @fhp: file handle of file to be read + * @nf: opened struct nfsd_file of file to be read + * @offset: starting byte offset + * @count: IN: requested number of bytes; OUT: number of bytes read + * @base: offset in first page of read buffer + * @eof: OUT: set non-zero if operation reached the end of the file + * + * Some filesystems or situations cannot use nfsd_splice_read. This + * function is the slightly less-performant fallback for those cases. + * + * Returns nfs_ok on success, otherwise an nfserr stat value is + * returned. + */ +__be32 nfsd_iter_read(struct svc_rqst *rqstp, struct svc_fh *fhp, + struct nfsd_file *nf, loff_t offset, unsigned long *count, + unsigned int base, u32 *eof) +{ + struct file *file = nf->nf_file; + unsigned long v, total; + struct iov_iter iter; + struct kiocb kiocb; + ssize_t host_err; + size_t len; + + init_sync_kiocb(&kiocb, file); + + switch (nfsd_io_cache_read) { + case NFSD_IO_BUFFERED: + break; + case NFSD_IO_DIRECT: + /* When dio_read_offset_align is zero, dio is not supported */ + if (nf->nf_dio_read_offset_align && !rqstp->rq_res.page_len) + return nfsd_direct_read(rqstp, fhp, nf, offset, + count, eof); + fallthrough; + case NFSD_IO_DONTCACHE: + if (file->f_op->fop_flags & FOP_DONTCACHE) + kiocb.ki_flags = IOCB_DONTCACHE; + break; + } + + kiocb.ki_pos = offset; + + v = 0; + total = *count; + while (total && v < rqstp->rq_maxpages && + rqstp->rq_next_page < rqstp->rq_page_end) { + len = min_t(size_t, total, PAGE_SIZE - base); + bvec_set_page(&rqstp->rq_bvec[v], *rqstp->rq_next_page, + len, base); + + total -= len; + ++rqstp->rq_next_page; + ++v; + base = 0; + } + + trace_nfsd_read_vector(rqstp, fhp, offset, *count - total); + iov_iter_bvec(&iter, ITER_DEST, rqstp->rq_bvec, v, *count - total); + host_err = vfs_iocb_iter_read(file, &kiocb, &iter); + return nfsd_finish_read(rqstp, fhp, file, offset, count, eof, host_err); +} + +/* + * Gathered writes: If another process is currently writing to the file, + * there's a high chance this is another nfsd (triggered by a bulk write + * from a client's biod). Rather than syncing the file with each write + * request, we sleep for 10 msec. + * + * I don't know if this roughly approximates C. Juszak's idea of + * gathered writes, but it's a nice and simple solution (IMHO), and it + * seems to work:-) + * + * Note: we do this only in the NFSv2 case, since v3 and higher have a + * better tool (separate unstable writes and commits) for solving this + * problem. + */ +static int wait_for_concurrent_writes(struct file *file) +{ + struct inode *inode = file_inode(file); + static ino_t last_ino; + static dev_t last_dev; + int err = 0; + + if (atomic_read(&inode->i_writecount) > 1 + || (last_ino == inode->i_ino && last_dev == inode->i_sb->s_dev)) { + dprintk("nfsd: write defer %d\n", task_pid_nr(current)); + msleep(10); + dprintk("nfsd: write resume %d\n", task_pid_nr(current)); + } + + if (inode_state_read_once(inode) & I_DIRTY) { + dprintk("nfsd: write sync %d\n", task_pid_nr(current)); + err = vfs_fsync(file, 0); + } + last_ino = inode->i_ino; + last_dev = inode->i_sb->s_dev; + return err; +} + +struct nfsd_write_dio_seg { + struct iov_iter iter; + int flags; +}; + +static unsigned long +iov_iter_bvec_offset(const struct iov_iter *iter) +{ + return (unsigned long)(iter->bvec->bv_offset + iter->iov_offset); +} + +static void +nfsd_write_dio_seg_init(struct nfsd_write_dio_seg *segment, + struct bio_vec *bvec, unsigned int nvecs, + unsigned long total, size_t start, size_t len, + struct kiocb *iocb) +{ + iov_iter_bvec(&segment->iter, ITER_SOURCE, bvec, nvecs, total); + if (start) + iov_iter_advance(&segment->iter, start); + iov_iter_truncate(&segment->iter, len); + segment->flags = iocb->ki_flags; +} + +static unsigned int +nfsd_write_dio_iters_init(struct nfsd_file *nf, struct bio_vec *bvec, + unsigned int nvecs, struct kiocb *iocb, + unsigned long total, + struct nfsd_write_dio_seg segments[3]) +{ + u32 offset_align = nf->nf_dio_offset_align; + loff_t prefix_end, orig_end, middle_end; + u32 mem_align = nf->nf_dio_mem_align; + size_t prefix, middle, suffix; + loff_t offset = iocb->ki_pos; + unsigned int nsegs = 0; + + /* + * Check if direct I/O is feasible for this write request. + * If alignments are not available, the write is too small, + * or no alignment can be found, fall back to buffered I/O. + */ + if (unlikely(!mem_align || !offset_align) || + unlikely(total < max(offset_align, mem_align))) + goto no_dio; + + prefix_end = round_up(offset, offset_align); + orig_end = offset + total; + middle_end = round_down(orig_end, offset_align); + + prefix = prefix_end - offset; + middle = middle_end - prefix_end; + suffix = orig_end - middle_end; + + if (!middle) + goto no_dio; + + if (prefix) + nfsd_write_dio_seg_init(&segments[nsegs++], bvec, + nvecs, total, 0, prefix, iocb); + + nfsd_write_dio_seg_init(&segments[nsegs], bvec, nvecs, + total, prefix, middle, iocb); + + /* + * Check if the bvec iterator is aligned for direct I/O. + * + * bvecs generated from RPC receive buffers are contiguous: After + * the first bvec, all subsequent bvecs start at bv_offset zero + * (page-aligned). Therefore, only the first bvec is checked. + */ + if (iov_iter_bvec_offset(&segments[nsegs].iter) & (mem_align - 1)) + goto no_dio; + segments[nsegs].flags |= IOCB_DIRECT; + nsegs++; + + if (suffix) + nfsd_write_dio_seg_init(&segments[nsegs++], bvec, nvecs, total, + prefix + middle, suffix, iocb); + + return nsegs; + +no_dio: + /* No DIO alignment possible - pack into single non-DIO segment. */ + nfsd_write_dio_seg_init(&segments[0], bvec, nvecs, total, 0, + total, iocb); + return 1; +} + +static noinline_for_stack int +nfsd_direct_write(struct svc_rqst *rqstp, struct svc_fh *fhp, + struct nfsd_file *nf, unsigned int nvecs, + unsigned long *cnt, struct kiocb *kiocb) +{ + struct nfsd_write_dio_seg segments[3]; + struct file *file = nf->nf_file; + unsigned int nsegs, i; + ssize_t host_err; + size_t expected; + + nsegs = nfsd_write_dio_iters_init(nf, rqstp->rq_bvec, nvecs, + kiocb, *cnt, segments); + + *cnt = 0; + for (i = 0; i < nsegs; i++) { + kiocb->ki_flags = segments[i].flags; + if (kiocb->ki_flags & IOCB_DIRECT) + trace_nfsd_write_direct(rqstp, fhp, kiocb->ki_pos, + segments[i].iter.count); + else { + trace_nfsd_write_vector(rqstp, fhp, kiocb->ki_pos, + segments[i].iter.count); + /* + * Mark the I/O buffer as evict-able to reduce + * memory contention. + */ + if (nf->nf_file->f_op->fop_flags & FOP_DONTCACHE) + kiocb->ki_flags |= IOCB_DONTCACHE; + } + + expected = iov_iter_count(&segments[i].iter); + + host_err = vfs_iocb_iter_write(file, kiocb, &segments[i].iter); + if (host_err < 0) + return host_err; + *cnt += host_err; + if (host_err < (ssize_t)expected) + break; /* partial write */ + } + + return 0; +} + +/** + * nfsd_vfs_write - write data to an already-open file + * @rqstp: RPC execution context + * @fhp: File handle of file to write into + * @nf: An open file matching @fhp + * @offset: Byte offset of start + * @payload: xdr_buf containing the write payload + * @cnt: IN: number of bytes to write, OUT: number of bytes actually written + * @stable: An NFS stable_how value + * @verf: NFS WRITE verifier + * + * Upon return, caller must invoke fh_put on @fhp. + * + * Return values: + * An nfsstat value in network byte order. + */ +__be32 +nfsd_vfs_write(struct svc_rqst *rqstp, struct svc_fh *fhp, + struct nfsd_file *nf, loff_t offset, + const struct xdr_buf *payload, unsigned long *cnt, + int stable, __be32 *verf) +{ + struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); + struct file *file = nf->nf_file; + struct super_block *sb = file_inode(file)->i_sb; + struct kiocb kiocb; + struct svc_export *exp; + struct iov_iter iter; + errseq_t since; + __be32 nfserr; + int host_err; + unsigned long exp_op_flags = 0; + unsigned int pflags = current->flags; + bool restore_flags = false; + int nvecs; + + trace_nfsd_write_opened(rqstp, fhp, offset, *cnt); + + if (sb->s_export_op) + exp_op_flags = sb->s_export_op->flags; + + if (test_bit(RQ_LOCAL, &rqstp->rq_flags) && + !(exp_op_flags & EXPORT_OP_REMOTE_FS)) { + /* + * We want throttling in balance_dirty_pages() + * and shrink_inactive_list() to only consider + * the backingdev we are writing to, so that nfs to + * localhost doesn't cause nfsd to lock up due to all + * the client's dirty pages or its congested queue. + */ + current->flags |= PF_LOCAL_THROTTLE; + restore_flags = true; + } + + exp = fhp->fh_export; + + if (!EX_ISSYNC(exp)) + stable = NFS_UNSTABLE; + init_sync_kiocb(&kiocb, file); + kiocb.ki_pos = offset; + if (likely(!fhp->fh_use_wgather)) { + switch (stable) { + case NFS_FILE_SYNC: + /* persist data and timestamps */ + kiocb.ki_flags |= IOCB_DSYNC | IOCB_SYNC; + break; + case NFS_DATA_SYNC: + /* persist data only */ + kiocb.ki_flags |= IOCB_DSYNC; + break; + } + } + + nvecs = xdr_buf_to_bvec(rqstp->rq_bvec, rqstp->rq_maxpages, payload); + if (nvecs < 0) { + host_err = nvecs; + goto out_nfserr; + } + + since = READ_ONCE(file->f_wb_err); + if (verf) + nfsd_copy_write_verifier(verf, nn); + + switch (nfsd_io_cache_write) { + case NFSD_IO_DIRECT: + host_err = nfsd_direct_write(rqstp, fhp, nf, nvecs, + cnt, &kiocb); + break; + case NFSD_IO_DONTCACHE: + if (file->f_op->fop_flags & FOP_DONTCACHE) + kiocb.ki_flags |= IOCB_DONTCACHE; + fallthrough; + case NFSD_IO_BUFFERED: + iov_iter_bvec(&iter, ITER_SOURCE, rqstp->rq_bvec, nvecs, *cnt); + host_err = vfs_iocb_iter_write(file, &kiocb, &iter); + if (host_err < 0) + break; + *cnt = host_err; + break; + } + if (host_err < 0) { + commit_reset_write_verifier(nn, rqstp, host_err); + goto out_nfserr; + } + nfsd_stats_io_write_add(nn, exp, *cnt); + fsnotify_modify(file); + host_err = filemap_check_wb_err(file->f_mapping, since); + if (host_err < 0) { + commit_reset_write_verifier(nn, rqstp, host_err); + goto out_nfserr; + } + + if (stable && fhp->fh_use_wgather) { + host_err = wait_for_concurrent_writes(file); + if (host_err < 0) + commit_reset_write_verifier(nn, rqstp, host_err); + } + +out_nfserr: + if (host_err >= 0) { + trace_nfsd_write_io_done(rqstp, fhp, offset, *cnt); + nfserr = nfs_ok; + } else { + trace_nfsd_write_err(rqstp, fhp, offset, host_err); + nfserr = nfserrno(host_err); + } + if (restore_flags) + current_restore_flags(pflags, PF_LOCAL_THROTTLE); + return nfserr; +} + +/** + * nfsd_read_splice_ok - check if spliced reading is supported + * @rqstp: RPC transaction context + * + * Return values: + * %true: nfsd_splice_read() may be used + * %false: nfsd_splice_read() must not be used + * + * NFS READ normally uses splice to send data in-place. However the + * data in cache can change after the reply's MIC is computed but + * before the RPC reply is sent. To prevent the client from + * rejecting the server-computed MIC in this somewhat rare case, do + * not use splice with the GSS integrity and privacy services. + */ +bool nfsd_read_splice_ok(struct svc_rqst *rqstp) +{ + if (nfsd_disable_splice_read) + return false; + switch (svc_auth_flavor(rqstp)) { + case RPC_AUTH_GSS_KRB5I: + case RPC_AUTH_GSS_KRB5P: + return false; + } + return true; +} + +/** + * nfsd_read - Read data from a file + * @rqstp: RPC transaction context + * @fhp: file handle of file to be read + * @offset: starting byte offset + * @count: IN: requested number of bytes; OUT: number of bytes read + * @eof: OUT: set non-zero if operation reached the end of the file + * + * The caller must verify that there is enough space in @rqstp.rq_res + * to perform this operation. + * + * N.B. After this call fhp needs an fh_put + * + * Returns nfs_ok on success, otherwise an nfserr stat value is + * returned. + */ +__be32 nfsd_read(struct svc_rqst *rqstp, struct svc_fh *fhp, + loff_t offset, unsigned long *count, u32 *eof) +{ + struct nfsd_file *nf; + struct file *file; + __be32 err; + + trace_nfsd_read_start(rqstp, fhp, offset, *count); + err = nfsd_file_acquire_gc(rqstp, fhp, NFSD_MAY_READ, &nf); + if (err) + return err; + + file = nf->nf_file; + if (file->f_op->splice_read && nfsd_read_splice_ok(rqstp)) + err = nfsd_splice_read(rqstp, fhp, file, offset, count, eof); + else + err = nfsd_iter_read(rqstp, fhp, nf, offset, count, 0, eof); + + nfsd_file_put(nf); + trace_nfsd_read_done(rqstp, fhp, offset, *count); + return err; +} + +/** + * nfsd_write - open a file and write data to it + * @rqstp: RPC execution context + * @fhp: File handle of file to write into; nfsd_write() may modify it + * @offset: Byte offset of start + * @payload: xdr_buf containing the write payload + * @cnt: IN: number of bytes to write, OUT: number of bytes actually written + * @stable: An NFS stable_how value + * @verf: NFS WRITE verifier + * + * Upon return, caller must invoke fh_put on @fhp. + * + * Return values: + * An nfsstat value in network byte order. + */ +__be32 +nfsd_write(struct svc_rqst *rqstp, struct svc_fh *fhp, loff_t offset, + const struct xdr_buf *payload, unsigned long *cnt, int stable, + __be32 *verf) +{ + struct nfsd_file *nf; + __be32 err; + + trace_nfsd_write_start(rqstp, fhp, offset, *cnt); + + err = nfsd_file_acquire_gc(rqstp, fhp, NFSD_MAY_WRITE, &nf); + if (err) + goto out; + + err = nfsd_vfs_write(rqstp, fhp, nf, offset, payload, cnt, + stable, verf); + nfsd_file_put(nf); +out: + trace_nfsd_write_done(rqstp, fhp, offset, *cnt); + return err; +} + +/** + * nfsd_commit - Commit pending writes to stable storage + * @rqstp: RPC request being processed + * @fhp: NFS filehandle + * @nf: target file + * @offset: raw offset from beginning of file + * @count: raw count of bytes to sync + * @verf: filled in with the server's current write verifier + * + * Note: we guarantee that data that lies within the range specified + * by the 'offset' and 'count' parameters will be synced. The server + * is permitted to sync data that lies outside this range at the + * same time. + * + * Unfortunately we cannot lock the file to make sure we return full WCC + * data to the client, as locking happens lower down in the filesystem. + * + * Return values: + * An nfsstat value in network byte order. + */ +__be32 +nfsd_commit(struct svc_rqst *rqstp, struct svc_fh *fhp, struct nfsd_file *nf, + u64 offset, u32 count, __be32 *verf) +{ + __be32 err = nfs_ok; + u64 maxbytes; + loff_t start, end; + struct nfsd_net *nn; + + trace_nfsd_commit_start(rqstp, fhp, offset, count); + + /* + * Convert the client-provided (offset, count) range to a + * (start, end) range. If the client-provided range falls + * outside the maximum file size of the underlying FS, + * clamp the sync range appropriately. + */ + start = 0; + end = LLONG_MAX; + maxbytes = (u64)fhp->fh_dentry->d_sb->s_maxbytes; + if (offset < maxbytes) { + start = offset; + if (count && (offset + count - 1 < maxbytes)) + end = offset + count - 1; + } + + nn = net_generic(nf->nf_net, nfsd_net_id); + if (EX_ISSYNC(fhp->fh_export)) { + errseq_t since = READ_ONCE(nf->nf_file->f_wb_err); + int err2; + + err2 = vfs_fsync_range(nf->nf_file, start, end, 0); + switch (err2) { + case 0: + nfsd_copy_write_verifier(verf, nn); + err2 = filemap_check_wb_err(nf->nf_file->f_mapping, + since); + if (err2 < 0) + commit_reset_write_verifier(nn, rqstp, err2); + err = nfserrno(err2); + break; + case -EINVAL: + err = nfserr_notsupp; + break; + default: + commit_reset_write_verifier(nn, rqstp, err2); + err = nfserrno(err2); + } + } else + nfsd_copy_write_verifier(verf, nn); + + trace_nfsd_commit_done(rqstp, fhp, offset, count); + return err; +} + +/** + * nfsd_create_setattr - Set a created file's attributes + * @rqstp: RPC transaction being executed + * @fhp: NFS filehandle of parent directory + * @resfhp: NFS filehandle of new object + * @attrs: requested attributes of new object + * + * Returns nfs_ok on success, or an nfsstat in network byte order. + */ +__be32 +nfsd_create_setattr(struct svc_rqst *rqstp, struct svc_fh *fhp, + struct svc_fh *resfhp, struct nfsd_attrs *attrs) +{ + struct iattr *iap = attrs->na_iattr; + __be32 status; + + /* + * Mode has already been set by file creation. + */ + iap->ia_valid &= ~ATTR_MODE; + + /* + * Setting uid/gid works only for root. Irix appears to + * send along the gid on create when it tries to implement + * setgid directories via NFS: + */ + if (!uid_eq(current_fsuid(), GLOBAL_ROOT_UID)) + iap->ia_valid &= ~(ATTR_UID|ATTR_GID); + + /* + * Callers expect new file metadata to be committed even + * if the attributes have not changed. + */ + if (nfsd_attrs_valid(attrs)) + status = nfsd_setattr(rqstp, resfhp, attrs, NULL); + else + status = nfserrno(commit_metadata(resfhp)); + + /* + * Transactional filesystems had a chance to commit changes + * for both parent and child simultaneously making the + * following commit_metadata a noop in many cases. + */ + if (!status) + status = nfserrno(commit_metadata(fhp)); + + /* + * Update the new filehandle to pick up the new attributes. + */ + if (!status) + status = fh_update(resfhp); + + return status; +} + +/* HPUX client sometimes creates a file in mode 000, and sets size to 0. + * setting size to 0 may fail for some specific file systems by the permission + * checking which requires WRITE permission but the mode is 000. + * we ignore the resizing(to 0) on the just new created file, since the size is + * 0 after file created. + * + * call this only after vfs_create() is called. + * */ +static void +nfsd_check_ignore_resizing(struct iattr *iap) +{ + if ((iap->ia_valid & ATTR_SIZE) && (iap->ia_size == 0)) + iap->ia_valid &= ~ATTR_SIZE; +} + +/* The parent directory should already be locked - we will unlock */ +__be32 +nfsd_create_locked(struct svc_rqst *rqstp, struct svc_fh *fhp, + struct nfsd_attrs *attrs, + int type, dev_t rdev, struct svc_fh *resfhp) +{ + struct dentry *dentry, *dchild; + struct inode *dirp; + struct iattr *iap = attrs->na_iattr; + __be32 err; + int host_err = 0; + + dentry = fhp->fh_dentry; + dirp = d_inode(dentry); + + dchild = dget(resfhp->fh_dentry); + err = nfsd_permission(&rqstp->rq_cred, fhp->fh_export, dentry, + NFSD_MAY_CREATE); + if (err) + goto out; + + if (!(iap->ia_valid & ATTR_MODE)) + iap->ia_mode = 0; + iap->ia_mode = (iap->ia_mode & S_IALLUGO) | type; + + if (!IS_POSIXACL(dirp)) + iap->ia_mode &= ~current_umask(); + + err = 0; + switch (type) { + case S_IFREG: + host_err = vfs_create(&nop_mnt_idmap, dchild, iap->ia_mode, NULL); + if (!host_err) + nfsd_check_ignore_resizing(iap); + break; + case S_IFDIR: + dchild = vfs_mkdir(&nop_mnt_idmap, dirp, dchild, iap->ia_mode, NULL); + if (IS_ERR(dchild)) { + host_err = PTR_ERR(dchild); + } else if (d_is_negative(dchild)) { + err = nfserr_serverfault; + goto out; + } else if (unlikely(dchild != resfhp->fh_dentry)) { + dput(resfhp->fh_dentry); + resfhp->fh_dentry = dget(dchild); + } + break; + case S_IFCHR: + case S_IFBLK: + case S_IFIFO: + case S_IFSOCK: + host_err = vfs_mknod(&nop_mnt_idmap, dirp, dchild, + iap->ia_mode, rdev, NULL); + break; + default: + printk(KERN_WARNING "nfsd: bad file type %o in nfsd_create\n", + type); + host_err = -EINVAL; + } + if (host_err < 0) + goto out_nfserr; + + err = nfsd_create_setattr(rqstp, fhp, resfhp, attrs); + +out: + if (!err) + fh_fill_post_attrs(fhp); + end_creating(dchild); + return err; + +out_nfserr: + err = nfserrno(host_err); + goto out; +} + +/* + * Create a filesystem object (regular, directory, special). + * Note that the parent directory is left locked. + * + * N.B. Every call to nfsd_create needs an fh_put for _both_ fhp and resfhp + */ +__be32 +nfsd_create(struct svc_rqst *rqstp, struct svc_fh *fhp, + char *fname, int flen, struct nfsd_attrs *attrs, + int type, dev_t rdev, struct svc_fh *resfhp) +{ + struct dentry *dentry, *dchild = NULL; + __be32 err; + int host_err; + + trace_nfsd_vfs_create(rqstp, fhp, type, fname, flen); + + if (name_is_dot_dotdot(fname, flen)) + return nfserr_exist; + + err = fh_verify(rqstp, fhp, S_IFDIR, NFSD_MAY_NOP); + if (err) + return err; + + dentry = fhp->fh_dentry; + + host_err = fh_want_write(fhp); + if (host_err) + return nfserrno(host_err); + + dchild = start_creating(&nop_mnt_idmap, dentry, &QSTR_LEN(fname, flen)); + host_err = PTR_ERR(dchild); + if (IS_ERR(dchild)) + return nfserrno(host_err); + + err = fh_compose(resfhp, fhp->fh_export, dchild, fhp); + if (err) + goto out_unlock; + err = fh_fill_pre_attrs(fhp); + if (err != nfs_ok) + goto out_unlock; + err = nfsd_create_locked(rqstp, fhp, attrs, type, rdev, resfhp); + /* nfsd_create_locked() unlocked the parent */ + dput(dchild); + return err; + +out_unlock: + end_creating(dchild); + return err; +} + +/* + * Read a symlink. On entry, *lenp must contain the maximum path length that + * fits into the buffer. On return, it contains the true length. + * N.B. After this call fhp needs an fh_put + */ +__be32 +nfsd_readlink(struct svc_rqst *rqstp, struct svc_fh *fhp, char *buf, int *lenp) +{ + __be32 err; + const char *link; + struct path path; + DEFINE_DELAYED_CALL(done); + int len; + + err = fh_verify(rqstp, fhp, S_IFLNK, NFSD_MAY_NOP); + if (unlikely(err)) + return err; + + path.mnt = fhp->fh_export->ex_path.mnt; + path.dentry = fhp->fh_dentry; + + if (unlikely(!d_is_symlink(path.dentry))) + return nfserr_inval; + + touch_atime(&path); + + link = vfs_get_link(path.dentry, &done); + if (IS_ERR(link)) + return nfserrno(PTR_ERR(link)); + + len = strlen(link); + if (len < *lenp) + *lenp = len; + memcpy(buf, link, *lenp); + do_delayed_call(&done); + return 0; +} + +/** + * nfsd_symlink - Create a symlink and look up its inode + * @rqstp: RPC transaction being executed + * @fhp: NFS filehandle of parent directory + * @fname: filename of the new symlink + * @flen: length of @fname + * @path: content of the new symlink (NUL-terminated) + * @attrs: requested attributes of new object + * @resfhp: NFS filehandle of new object + * + * N.B. After this call _both_ fhp and resfhp need an fh_put + * + * Returns nfs_ok on success, or an nfsstat in network byte order. + */ +__be32 +nfsd_symlink(struct svc_rqst *rqstp, struct svc_fh *fhp, + char *fname, int flen, + char *path, struct nfsd_attrs *attrs, + struct svc_fh *resfhp) +{ + struct dentry *dentry, *dnew; + __be32 err, cerr; + int host_err; + + trace_nfsd_vfs_symlink(rqstp, fhp, fname, flen, path); + + err = nfserr_noent; + if (!flen || path[0] == '\0') + goto out; + err = nfserr_exist; + if (name_is_dot_dotdot(fname, flen)) + goto out; + + err = fh_verify(rqstp, fhp, S_IFDIR, NFSD_MAY_CREATE); + if (err) + goto out; + + host_err = fh_want_write(fhp); + if (host_err) { + err = nfserrno(host_err); + goto out; + } + + dentry = fhp->fh_dentry; + dnew = start_creating(&nop_mnt_idmap, dentry, &QSTR_LEN(fname, flen)); + if (IS_ERR(dnew)) { + err = nfserrno(PTR_ERR(dnew)); + goto out_drop_write; + } + err = fh_fill_pre_attrs(fhp); + if (err != nfs_ok) + goto out_unlock; + host_err = vfs_symlink(&nop_mnt_idmap, d_inode(dentry), dnew, path, NULL); + err = nfserrno(host_err); + cerr = fh_compose(resfhp, fhp->fh_export, dnew, fhp); + if (!err) + nfsd_create_setattr(rqstp, fhp, resfhp, attrs); + fh_fill_post_attrs(fhp); +out_unlock: + end_creating(dnew); + if (!err) + err = nfserrno(commit_metadata(fhp)); + if (!err) + err = cerr; +out_drop_write: + fh_drop_write(fhp); +out: + return err; +} + +/** + * nfsd_link - create a link + * @rqstp: RPC transaction context + * @ffhp: the file handle of the directory where the new link is to be created + * @name: the filename of the new link + * @len: the length of @name in octets + * @tfhp: the file handle of an existing file object + * + * After this call _both_ ffhp and tfhp need an fh_put. + * + * Returns a generic NFS status code in network byte-order. + */ +__be32 +nfsd_link(struct svc_rqst *rqstp, struct svc_fh *ffhp, + char *name, int len, struct svc_fh *tfhp) +{ + struct dentry *ddir, *dnew, *dold; + struct inode *dirp; + int type; + __be32 err; + int host_err; + + trace_nfsd_vfs_link(rqstp, ffhp, tfhp, name, len); + + err = fh_verify(rqstp, ffhp, S_IFDIR, NFSD_MAY_CREATE); + if (err) + goto out; + err = fh_verify(rqstp, tfhp, 0, NFSD_MAY_NOP); + if (err) + goto out; + err = nfserr_isdir; + if (d_is_dir(tfhp->fh_dentry)) + goto out; + err = nfserr_perm; + if (!len) + goto out; + err = nfserr_exist; + if (name_is_dot_dotdot(name, len)) + goto out; + + err = nfs_ok; + type = d_inode(tfhp->fh_dentry)->i_mode & S_IFMT; + host_err = fh_want_write(tfhp); + if (host_err) + goto out; + + ddir = ffhp->fh_dentry; + dirp = d_inode(ddir); + dnew = start_creating(&nop_mnt_idmap, ddir, &QSTR_LEN(name, len)); + + if (IS_ERR(dnew)) { + host_err = PTR_ERR(dnew); + goto out_drop_write; + } + + dold = tfhp->fh_dentry; + + err = nfserr_noent; + if (d_really_is_negative(dold)) + goto out_unlock; + err = fh_fill_pre_attrs(ffhp); + if (err != nfs_ok) + goto out_unlock; + host_err = vfs_link(dold, &nop_mnt_idmap, dirp, dnew, NULL); + fh_fill_post_attrs(ffhp); +out_unlock: + end_creating(dnew); + if (!host_err) { + host_err = commit_metadata(ffhp); + if (!host_err) + host_err = commit_metadata(tfhp); + } + +out_drop_write: + fh_drop_write(tfhp); + if (host_err == -EBUSY) { + /* + * See RFC 8881 Section 18.9.4 para 1-2: NFSv4 LINK + * wants a status unique to the object type. + */ + if (type != S_IFDIR) + err = nfserr_file_open; + else + err = nfserr_acces; + } +out: + return err != nfs_ok ? err : nfserrno(host_err); +} + +static void +nfsd_close_cached_files(struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + + if (inode && S_ISREG(inode->i_mode)) + nfsd_file_close_inode_sync(inode); +} + +static bool +nfsd_has_cached_files(struct dentry *dentry) +{ + bool ret = false; + struct inode *inode = d_inode(dentry); + + if (inode && S_ISREG(inode->i_mode)) + ret = nfsd_file_is_cached(inode); + return ret; +} + +/** + * nfsd_rename - rename a directory entry + * @rqstp: RPC transaction context + * @ffhp: the file handle of parent directory containing the entry to be renamed + * @fname: the filename of directory entry to be renamed + * @flen: the length of @fname in octets + * @tfhp: the file handle of parent directory to contain the renamed entry + * @tname: the filename of the new entry + * @tlen: the length of @tlen in octets + * + * After this call _both_ ffhp and tfhp need an fh_put. + * + * Returns a generic NFS status code in network byte-order. + */ +__be32 +nfsd_rename(struct svc_rqst *rqstp, struct svc_fh *ffhp, char *fname, int flen, + struct svc_fh *tfhp, char *tname, int tlen) +{ + struct dentry *fdentry, *tdentry; + int type = S_IFDIR; + struct renamedata rd = {}; + __be32 err; + int host_err; + struct dentry *close_cached; + + trace_nfsd_vfs_rename(rqstp, ffhp, tfhp, fname, flen, tname, tlen); + + err = fh_verify(rqstp, ffhp, S_IFDIR, NFSD_MAY_REMOVE); + if (err) + goto out; + err = fh_verify(rqstp, tfhp, S_IFDIR, NFSD_MAY_CREATE); + if (err) + goto out; + + fdentry = ffhp->fh_dentry; + + tdentry = tfhp->fh_dentry; + + err = nfserr_perm; + if (!flen || name_is_dot_dotdot(fname, flen) || + !tlen || name_is_dot_dotdot(tname, tlen)) + goto out; + + err = nfserr_xdev; + if (ffhp->fh_export->ex_path.mnt != tfhp->fh_export->ex_path.mnt) + goto out; + if (ffhp->fh_export->ex_path.dentry != tfhp->fh_export->ex_path.dentry) + goto out; + +retry: + close_cached = NULL; + host_err = fh_want_write(ffhp); + if (host_err) { + err = nfserrno(host_err); + goto out; + } + + rd.mnt_idmap = &nop_mnt_idmap; + rd.old_parent = fdentry; + rd.new_parent = tdentry; + + host_err = start_renaming(&rd, 0, &QSTR_LEN(fname, flen), + &QSTR_LEN(tname, tlen)); + + if (host_err) { + err = nfserrno(host_err); + goto out_want_write; + } + err = fh_fill_pre_attrs(ffhp); + if (err != nfs_ok) + goto out_unlock; + err = fh_fill_pre_attrs(tfhp); + if (err != nfs_ok) + goto out_unlock; + + type = d_inode(rd.old_dentry)->i_mode & S_IFMT; + + if (d_inode(rd.new_dentry)) + type = d_inode(rd.new_dentry)->i_mode & S_IFMT; + + if ((rd.new_dentry->d_sb->s_export_op->flags & EXPORT_OP_CLOSE_BEFORE_UNLINK) && + nfsd_has_cached_files(rd.new_dentry)) { + close_cached = dget(rd.new_dentry); + goto out_unlock; + } else { + int retries; + + for (retries = 1;;) { + host_err = vfs_rename(&rd); + if (host_err != -EAGAIN || !retries--) + break; + if (!nfsd_wait_for_delegreturn(rqstp, d_inode(rd.old_dentry))) + break; + } + if (!host_err) { + host_err = commit_metadata(tfhp); + if (!host_err) + host_err = commit_metadata(ffhp); + } + } + if (host_err == -EBUSY) { + /* + * See RFC 8881 Section 18.26.4 para 1-3: NFSv4 RENAME + * wants a status unique to the object type. + */ + if (type != S_IFDIR) + err = nfserr_file_open; + else + err = nfserr_acces; + } else { + err = nfserrno(host_err); + } + + if (!close_cached) { + fh_fill_post_attrs(ffhp); + fh_fill_post_attrs(tfhp); + } +out_unlock: + end_renaming(&rd); +out_want_write: + fh_drop_write(ffhp); + + /* + * If the target dentry has cached open files, then we need to + * try to close them prior to doing the rename. Final fput + * shouldn't be done with locks held however, so we delay it + * until this point and then reattempt the whole shebang. + */ + if (close_cached) { + nfsd_close_cached_files(close_cached); + dput(close_cached); + goto retry; + } +out: + return err; +} + +/** + * nfsd_unlink - remove a directory entry + * @rqstp: RPC transaction context + * @fhp: the file handle of the parent directory to be modified + * @type: enforced file type of the object to be removed + * @fname: the name of directory entry to be removed + * @flen: length of @fname in octets + * + * After this call fhp needs an fh_put. + * + * Returns a generic NFS status code in network byte-order. + */ +__be32 +nfsd_unlink(struct svc_rqst *rqstp, struct svc_fh *fhp, int type, + char *fname, int flen) +{ + struct dentry *dentry, *rdentry; + struct inode *dirp; + struct inode *rinode = NULL; + __be32 err; + int host_err; + + trace_nfsd_vfs_unlink(rqstp, fhp, fname, flen); + + err = nfserr_acces; + if (!flen || name_is_dot_dotdot(fname, flen)) + goto out; + err = fh_verify(rqstp, fhp, S_IFDIR, NFSD_MAY_REMOVE); + if (err) + goto out; + + host_err = fh_want_write(fhp); + if (host_err) + goto out_nfserr; + + dentry = fhp->fh_dentry; + dirp = d_inode(dentry); + + rdentry = start_removing(&nop_mnt_idmap, dentry, &QSTR_LEN(fname, flen)); + + host_err = PTR_ERR(rdentry); + if (IS_ERR(rdentry)) + goto out_drop_write; + + err = fh_fill_pre_attrs(fhp); + if (err != nfs_ok) + goto out_unlock; + + rinode = d_inode(rdentry); + /* Prevent truncation until after locks dropped */ + ihold(rinode); + + if (!type) + type = d_inode(rdentry)->i_mode & S_IFMT; + + if (type != S_IFDIR) { + int retries; + + if (rdentry->d_sb->s_export_op->flags & EXPORT_OP_CLOSE_BEFORE_UNLINK) + nfsd_close_cached_files(rdentry); + + for (retries = 1;;) { + host_err = vfs_unlink(&nop_mnt_idmap, dirp, rdentry, NULL); + if (host_err != -EAGAIN || !retries--) + break; + if (!nfsd_wait_for_delegreturn(rqstp, rinode)) + break; + } + } else { + host_err = vfs_rmdir(&nop_mnt_idmap, dirp, rdentry, NULL); + } + fh_fill_post_attrs(fhp); + +out_unlock: + end_removing(rdentry); + if (!err && !host_err) + host_err = commit_metadata(fhp); + iput(rinode); /* truncate the inode here */ + +out_drop_write: + fh_drop_write(fhp); +out_nfserr: + if (host_err == -EBUSY) { + /* + * See RFC 8881 Section 18.25.4 para 4: NFSv4 REMOVE + * wants a status unique to the object type. + */ + if (type != S_IFDIR) + err = nfserr_file_open; + else + err = nfserr_acces; + } +out: + return err != nfs_ok ? err : nfserrno(host_err); +} + +/* + * We do this buffering because we must not call back into the file + * system's ->lookup() method from the filldir callback. That may well + * deadlock a number of file systems. + * + * This is based heavily on the implementation of same in XFS. + */ +struct buffered_dirent { + u64 ino; + loff_t offset; + int namlen; + unsigned int d_type; + char name[]; +}; + +struct readdir_data { + struct dir_context ctx; + char *dirent; + size_t used; + int full; +}; + +static bool nfsd_buffered_filldir(struct dir_context *ctx, const char *name, + int namlen, loff_t offset, u64 ino, + unsigned int d_type) +{ + struct readdir_data *buf = + container_of(ctx, struct readdir_data, ctx); + struct buffered_dirent *de = (void *)(buf->dirent + buf->used); + unsigned int reclen; + + reclen = ALIGN(sizeof(struct buffered_dirent) + namlen, sizeof(u64)); + if (buf->used + reclen > PAGE_SIZE) { + buf->full = 1; + return false; + } + + de->namlen = namlen; + de->offset = offset; + de->ino = ino; + de->d_type = d_type; + memcpy(de->name, name, namlen); + buf->used += reclen; + + return true; +} + +static __be32 nfsd_buffered_readdir(struct file *file, struct svc_fh *fhp, + nfsd_filldir_t func, struct readdir_cd *cdp, + loff_t *offsetp) +{ + struct buffered_dirent *de; + int host_err; + int size; + loff_t offset; + struct readdir_data buf = { + .ctx.actor = nfsd_buffered_filldir, + .dirent = kmalloc(PAGE_SIZE, GFP_KERNEL) + }; + + if (!buf.dirent) + return nfserrno(-ENOMEM); + + offset = *offsetp; + + while (1) { + unsigned int reclen; + + cdp->err = nfserr_eof; /* will be cleared on successful read */ + buf.used = 0; + buf.full = 0; + + host_err = iterate_dir(file, &buf.ctx); + if (buf.full) + host_err = 0; + + if (host_err < 0) + break; + + size = buf.used; + + if (!size) + break; + + de = (struct buffered_dirent *)buf.dirent; + while (size > 0) { + offset = de->offset; + + if (func(cdp, de->name, de->namlen, de->offset, + de->ino, de->d_type)) + break; + + if (cdp->err != nfs_ok) + break; + + trace_nfsd_dirent(fhp, de->ino, de->name, de->namlen); + + reclen = ALIGN(sizeof(*de) + de->namlen, + sizeof(u64)); + size -= reclen; + de = (struct buffered_dirent *)((char *)de + reclen); + } + if (size > 0) /* We bailed out early */ + break; + + offset = vfs_llseek(file, 0, SEEK_CUR); + } + + kfree((buf.dirent)); + + if (host_err) + return nfserrno(host_err); + + *offsetp = offset; + return cdp->err; +} + +/** + * nfsd_readdir - Read entries from a directory + * @rqstp: RPC transaction context + * @fhp: NFS file handle of directory to be read + * @offsetp: OUT: seek offset of final entry that was read + * @cdp: OUT: an eof error value + * @func: entry filler actor + * + * This implementation ignores the NFSv3/4 verifier cookie. + * + * NB: normal system calls hold file->f_pos_lock when calling + * ->iterate_shared and ->llseek, but nfsd_readdir() does not. + * Because the struct file acquired here is not visible to other + * threads, it's internal state does not need mutex protection. + * + * Returns nfs_ok on success, otherwise an nfsstat code is + * returned. + */ +__be32 +nfsd_readdir(struct svc_rqst *rqstp, struct svc_fh *fhp, loff_t *offsetp, + struct readdir_cd *cdp, nfsd_filldir_t func) +{ + __be32 err; + struct file *file; + loff_t offset = *offsetp; + int may_flags = NFSD_MAY_READ; + + err = nfsd_open(rqstp, fhp, S_IFDIR, may_flags, &file); + if (err) + goto out; + + if (fhp->fh_64bit_cookies) + file->f_mode |= FMODE_64BITHASH; + else + file->f_mode |= FMODE_32BITHASH; + + offset = vfs_llseek(file, offset, SEEK_SET); + if (offset < 0) { + err = nfserrno((int)offset); + goto out_close; + } + + err = nfsd_buffered_readdir(file, fhp, func, cdp, offsetp); + + if (err == nfserr_eof || err == nfserr_toosmall) + err = nfs_ok; /* can still be found in ->err */ +out_close: + nfsd_filp_close(file); +out: + return err; +} + +/** + * nfsd_filp_close: close a file synchronously + * @fp: the file to close + * + * nfsd_filp_close() is similar in behaviour to filp_close(). + * The difference is that if this is the final close on the + * file, the that finalisation happens immediately, rather then + * being handed over to a work_queue, as it the case for + * filp_close(). + * When a user-space process closes a file (even when using + * filp_close() the finalisation happens before returning to + * userspace, so it is effectively synchronous. When a kernel thread + * uses file_close(), on the other hand, the handling is completely + * asynchronous. This means that any cost imposed by that finalisation + * is not imposed on the nfsd thread, and nfsd could potentually + * close files more quickly than the work queue finalises the close, + * which would lead to unbounded growth in the queue. + * + * In some contexts is it not safe to synchronously wait for + * close finalisation (see comment for __fput_sync()), but nfsd + * does not match those contexts. In partcilarly it does not, at the + * time that this function is called, hold and locks and no finalisation + * of any file, socket, or device driver would have any cause to wait + * for nfsd to make progress. + */ +void nfsd_filp_close(struct file *fp) +{ + get_file(fp); + filp_close(fp, NULL); + __fput_sync(fp); +} + +/* + * Get file system stats + * N.B. After this call fhp needs an fh_put + */ +__be32 +nfsd_statfs(struct svc_rqst *rqstp, struct svc_fh *fhp, struct kstatfs *stat, int access) +{ + __be32 err; + + trace_nfsd_vfs_statfs(rqstp, fhp); + + err = fh_verify(rqstp, fhp, 0, NFSD_MAY_NOP | access); + if (!err) { + struct path path = { + .mnt = fhp->fh_export->ex_path.mnt, + .dentry = fhp->fh_dentry, + }; + if (vfs_statfs(&path, stat)) + err = nfserr_io; + } + return err; +} + +static int exp_rdonly(struct svc_cred *cred, struct svc_export *exp) +{ + return nfsexp_flags(cred, exp) & NFSEXP_READONLY; +} + +#ifdef CONFIG_NFSD_V4 +/* + * Helper function to translate error numbers. In the case of xattr operations, + * some error codes need to be translated outside of the standard translations. + * + * ENODATA needs to be translated to nfserr_noxattr. + * E2BIG to nfserr_xattr2big. + * + * Additionally, vfs_listxattr can return -ERANGE. This means that the + * file has too many extended attributes to retrieve inside an + * XATTR_LIST_MAX sized buffer. This is a bug in the xattr implementation: + * filesystems will allow the adding of extended attributes until they hit + * their own internal limit. This limit may be larger than XATTR_LIST_MAX. + * So, at that point, the attributes are present and valid, but can't + * be retrieved using listxattr, since the upper level xattr code enforces + * the XATTR_LIST_MAX limit. + * + * This bug means that we need to deal with listxattr returning -ERANGE. The + * best mapping is to return TOOSMALL. + */ +static __be32 +nfsd_xattr_errno(int err) +{ + switch (err) { + case -ENODATA: + return nfserr_noxattr; + case -E2BIG: + return nfserr_xattr2big; + case -ERANGE: + return nfserr_toosmall; + } + return nfserrno(err); +} + +/* + * Retrieve the specified user extended attribute. To avoid always + * having to allocate the maximum size (since we are not getting + * a maximum size from the RPC), do a probe + alloc. Hold a reader + * lock on i_rwsem to prevent the extended attribute from changing + * size while we're doing this. + */ +__be32 +nfsd_getxattr(struct svc_rqst *rqstp, struct svc_fh *fhp, char *name, + void **bufp, int *lenp) +{ + ssize_t len; + __be32 err; + char *buf; + struct inode *inode; + struct dentry *dentry; + + err = fh_verify(rqstp, fhp, 0, NFSD_MAY_READ); + if (err) + return err; + + err = nfs_ok; + dentry = fhp->fh_dentry; + inode = d_inode(dentry); + + inode_lock_shared(inode); + + len = vfs_getxattr(&nop_mnt_idmap, dentry, name, NULL, 0); + + /* + * Zero-length attribute, just return. + */ + if (len == 0) { + *bufp = NULL; + *lenp = 0; + goto out; + } + + if (len < 0) { + err = nfsd_xattr_errno(len); + goto out; + } + + if (len > *lenp) { + err = nfserr_toosmall; + goto out; + } + + buf = kvmalloc(len, GFP_KERNEL); + if (buf == NULL) { + err = nfserr_jukebox; + goto out; + } + + len = vfs_getxattr(&nop_mnt_idmap, dentry, name, buf, len); + if (len <= 0) { + kvfree(buf); + buf = NULL; + err = nfsd_xattr_errno(len); + } + + *lenp = len; + *bufp = buf; + +out: + inode_unlock_shared(inode); + + return err; +} + +/* + * Retrieve the xattr names. Since we can't know how many are + * user extended attributes, we must get all attributes here, + * and have the XDR encode filter out the "user." ones. + * + * While this could always just allocate an XATTR_LIST_MAX + * buffer, that's a waste, so do a probe + allocate. To + * avoid any changes between the probe and allocate, wrap + * this in inode_lock. + */ +__be32 +nfsd_listxattr(struct svc_rqst *rqstp, struct svc_fh *fhp, char **bufp, + int *lenp) +{ + ssize_t len; + __be32 err; + char *buf; + struct inode *inode; + struct dentry *dentry; + + err = fh_verify(rqstp, fhp, 0, NFSD_MAY_READ); + if (err) + return err; + + dentry = fhp->fh_dentry; + inode = d_inode(dentry); + *lenp = 0; + + inode_lock_shared(inode); + + len = vfs_listxattr(dentry, NULL, 0); + if (len <= 0) { + err = nfsd_xattr_errno(len); + goto out; + } + + if (len > XATTR_LIST_MAX) { + err = nfserr_xattr2big; + goto out; + } + + buf = kvmalloc(len, GFP_KERNEL); + if (buf == NULL) { + err = nfserr_jukebox; + goto out; + } + + len = vfs_listxattr(dentry, buf, len); + if (len <= 0) { + kvfree(buf); + err = nfsd_xattr_errno(len); + goto out; + } + + *lenp = len; + *bufp = buf; + + err = nfs_ok; +out: + inode_unlock_shared(inode); + + return err; +} + +/** + * nfsd_removexattr - Remove an extended attribute + * @rqstp: RPC transaction being executed + * @fhp: NFS filehandle of object with xattr to remove + * @name: name of xattr to remove (NUL-terminate) + * + * Pass in a NULL pointer for delegated_inode, and let the client deal + * with NFS4ERR_DELAY (same as with e.g. setattr and remove). + * + * Returns nfs_ok on success, or an nfsstat in network byte order. + */ +__be32 +nfsd_removexattr(struct svc_rqst *rqstp, struct svc_fh *fhp, char *name) +{ + __be32 err; + int ret; + + err = fh_verify(rqstp, fhp, 0, NFSD_MAY_WRITE); + if (err) + return err; + + ret = fh_want_write(fhp); + if (ret) + return nfserrno(ret); + + inode_lock(fhp->fh_dentry->d_inode); + err = fh_fill_pre_attrs(fhp); + if (err != nfs_ok) + goto out_unlock; + ret = __vfs_removexattr_locked(&nop_mnt_idmap, fhp->fh_dentry, + name, NULL); + err = nfsd_xattr_errno(ret); + fh_fill_post_attrs(fhp); +out_unlock: + inode_unlock(fhp->fh_dentry->d_inode); + fh_drop_write(fhp); + + return err; +} + +__be32 +nfsd_setxattr(struct svc_rqst *rqstp, struct svc_fh *fhp, char *name, + void *buf, u32 len, u32 flags) +{ + __be32 err; + int ret; + + err = fh_verify(rqstp, fhp, 0, NFSD_MAY_WRITE); + if (err) + return err; + + ret = fh_want_write(fhp); + if (ret) + return nfserrno(ret); + inode_lock(fhp->fh_dentry->d_inode); + err = fh_fill_pre_attrs(fhp); + if (err != nfs_ok) + goto out_unlock; + ret = __vfs_setxattr_locked(&nop_mnt_idmap, fhp->fh_dentry, + name, buf, len, flags, NULL); + fh_fill_post_attrs(fhp); + err = nfsd_xattr_errno(ret); +out_unlock: + inode_unlock(fhp->fh_dentry->d_inode); + fh_drop_write(fhp); + return err; +} +#endif + +/* + * Check for a user's access permissions to this inode. + */ +__be32 +nfsd_permission(struct svc_cred *cred, struct svc_export *exp, + struct dentry *dentry, int acc) +{ + struct inode *inode = d_inode(dentry); + int err; + + if ((acc & NFSD_MAY_MASK) == NFSD_MAY_NOP) + return 0; +#if 0 + dprintk("nfsd: permission 0x%x%s%s%s%s%s%s%s mode 0%o%s%s%s\n", + acc, + (acc & NFSD_MAY_READ)? " read" : "", + (acc & NFSD_MAY_WRITE)? " write" : "", + (acc & NFSD_MAY_EXEC)? " exec" : "", + (acc & NFSD_MAY_SATTR)? " sattr" : "", + (acc & NFSD_MAY_TRUNC)? " trunc" : "", + (acc & NFSD_MAY_NLM)? " nlm" : "", + (acc & NFSD_MAY_OWNER_OVERRIDE)? " owneroverride" : "", + inode->i_mode, + IS_IMMUTABLE(inode)? " immut" : "", + IS_APPEND(inode)? " append" : "", + __mnt_is_readonly(exp->ex_path.mnt)? " ro" : ""); + dprintk(" owner %d/%d user %d/%d\n", + inode->i_uid, inode->i_gid, current_fsuid(), current_fsgid()); +#endif + + /* Normally we reject any write/sattr etc access on a read-only file + * system. But if it is IRIX doing check on write-access for a + * device special file, we ignore rofs. + */ + if (!(acc & NFSD_MAY_LOCAL_ACCESS)) + if (acc & (NFSD_MAY_WRITE | NFSD_MAY_SATTR | NFSD_MAY_TRUNC)) { + if (exp_rdonly(cred, exp) || + __mnt_is_readonly(exp->ex_path.mnt)) + return nfserr_rofs; + if (/* (acc & NFSD_MAY_WRITE) && */ IS_IMMUTABLE(inode)) + return nfserr_perm; + } + if ((acc & NFSD_MAY_TRUNC) && IS_APPEND(inode)) + return nfserr_perm; + + /* + * The file owner always gets access permission for accesses that + * would normally be checked at open time. This is to make + * file access work even when the client has done a fchmod(fd, 0). + * + * However, `cp foo bar' should fail nevertheless when bar is + * readonly. A sensible way to do this might be to reject all + * attempts to truncate a read-only file, because a creat() call + * always implies file truncation. + * ... but this isn't really fair. A process may reasonably call + * ftruncate on an open file descriptor on a file with perm 000. + * We must trust the client to do permission checking - using "ACCESS" + * with NFSv3. + */ + if ((acc & NFSD_MAY_OWNER_OVERRIDE) && + uid_eq(inode->i_uid, current_fsuid())) + return 0; + + /* This assumes NFSD_MAY_{READ,WRITE,EXEC} == MAY_{READ,WRITE,EXEC} */ + err = inode_permission(&nop_mnt_idmap, inode, + acc & (MAY_READ | MAY_WRITE | MAY_EXEC)); + + /* Allow read access to binaries even when mode 111 */ + if (err == -EACCES && S_ISREG(inode->i_mode) && + (((acc & NFSD_MAY_MASK) == NFSD_MAY_READ) && + (acc & (NFSD_MAY_OWNER_OVERRIDE | NFSD_MAY_READ_IF_EXEC)))) + err = inode_permission(&nop_mnt_idmap, inode, MAY_EXEC); + + return err? nfserrno(err) : 0; +} + +/** + * nfsd_get_case_info - get case sensitivity info for a dentry + * @dentry: dentry to query + * @case_insensitive: set to true if name comparison ignores case + * @case_preserving: set to true if case is preserved on disk + * + * On casefold-capable filesystems the flag lives on the directory, + * not on its entries, so for a non-directory @dentry the parent is + * queried instead. A directory (including an export root, whose + * parent lies outside the export) is queried as-is so its own + * contents' lookup behavior is reported. NFSD advertises + * fattr4_homogeneous as FALSE, so per-directory answers may differ + * within an export. + * + * The probe runs with kernel credentials. case_insensitive and + * case_preserving describe the directory's structural lookup + * behavior, not the caller's identity; running under the calling + * client's mapped credentials would let per-client MAC policy on + * the parent directory turn this query into NFS4ERR_ACCESS even + * though the underlying property is the same for every client. + * + * When the filesystem does not expose case-folding state (no + * ->fileattr_get, or the callback returns -EOPNOTSUPP / + * -ENOIOCTLCMD / -ENOTTY / -EINVAL), the outputs are filled with + * POSIX defaults (case-sensitive, case-preserving) on the premise + * that a filesystem with case-folding support wires up + * fileattr_get. + * + * Return: 0 with outputs filled, -EOPNOTSUPP with outputs filled + * to POSIX defaults, or a negative errno (e.g., -EIO, + * -ESTALE, -ENOMEM) with outputs unmodified. + */ +int +nfsd_get_case_info(struct dentry *dentry, bool *case_insensitive, + bool *case_preserving) +{ + struct file_kattr fa = {}; + const struct cred *saved; + struct cred *probe; + struct dentry *cd; + bool put = false; + int err; + + if (d_is_dir(dentry)) { + cd = dentry; + } else { + cd = dget_parent(dentry); + put = true; + } + + probe = prepare_kernel_cred(&init_task); + if (!probe) { + err = -ENOMEM; + goto out; + } + saved = override_creds(probe); + + err = vfs_fileattr_get(cd, &fa); + + put_cred(revert_creds(saved)); +out: + if (put) + dput(cd); + switch (err) { + case 0: + *case_insensitive = fa.fsx_xflags & FS_XFLAG_CASEFOLD; + *case_preserving = + !(fa.fsx_xflags & FS_XFLAG_CASENONPRESERVING); + return 0; + case -EINVAL: + case -ENOTTY: + case -ENOIOCTLCMD: + case -EOPNOTSUPP: + /* + * Filesystem does not expose case state. + * Report POSIX defaults. + */ + *case_insensitive = false; + *case_preserving = true; + return -EOPNOTSUPP; + default: + return err; + } +} diff --git a/fs/nfsd/vfs.h b/fs/nfsd/vfs.h new file mode 100644 index 000000000..4af2ff9e9 --- /dev/null +++ b/fs/nfsd/vfs.h @@ -0,0 +1,171 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 1995-1997 Olaf Kirch + */ + +#ifndef LINUX_NFSD_VFS_H +#define LINUX_NFSD_VFS_H + +#include +#include +#include "nfsfh.h" + +/* + * Flags for nfsd_permission + */ +#define NFSD_MAY_NOP 0 +#define NFSD_MAY_EXEC 0x001 /* == MAY_EXEC */ +#define NFSD_MAY_WRITE 0x002 /* == MAY_WRITE */ +#define NFSD_MAY_READ 0x004 /* == MAY_READ */ +#define NFSD_MAY_SATTR 0x008 +#define NFSD_MAY_TRUNC 0x010 +#define NFSD_MAY_NLM 0x020 /* request is from lockd */ +#define NFSD_MAY_MASK 0x03f + +/* extra hints to permission and open routines: */ +#define NFSD_MAY_OWNER_OVERRIDE 0x040 +#define NFSD_MAY_LOCAL_ACCESS 0x080 /* for device special files */ +#define NFSD_MAY_BYPASS_GSS_ON_ROOT 0x100 +#define NFSD_MAY_NOT_BREAK_LEASE 0x200 +#define NFSD_MAY_BYPASS_GSS 0x400 +#define NFSD_MAY_READ_IF_EXEC 0x800 + +#define NFSD_MAY_64BIT_COOKIE 0x1000 /* 64 bit readdir cookies for >= NFSv3 */ + +#define NFSD_MAY_LOCALIO 0x2000 /* for tracing, reflects when localio used */ + +#define NFSD_MAY_CREATE (NFSD_MAY_EXEC|NFSD_MAY_WRITE) +#define NFSD_MAY_REMOVE (NFSD_MAY_EXEC|NFSD_MAY_WRITE|NFSD_MAY_TRUNC) + +struct nfsd_file; + +/* + * Callback function for readdir + */ +typedef int (*nfsd_filldir_t)(void *, const char *, int, loff_t, u64, unsigned); + +struct readdir_cd { + __be32 err; /* nfs_ok, nfserr, or nfserr_eof */ +}; + +/* nfsd/vfs.c */ +struct nfsd_attrs { + struct iattr *na_iattr; /* input */ + struct xdr_netobj *na_seclabel; /* input */ + struct posix_acl *na_pacl; /* input */ + struct posix_acl *na_dpacl; /* input */ + + int na_labelerr; /* output */ + int na_dpaclerr; /* output */ + int na_paclerr; /* output */ +}; + +static inline void nfsd_attrs_free(struct nfsd_attrs *attrs) +{ + posix_acl_release(attrs->na_pacl); + posix_acl_release(attrs->na_dpacl); +} + +static inline bool nfsd_attrs_valid(struct nfsd_attrs *attrs) +{ + struct iattr *iap = attrs->na_iattr; + + return (iap->ia_valid || (attrs->na_seclabel && + attrs->na_seclabel->len) || + attrs->na_pacl || attrs->na_dpacl); +} + +__be32 nfserrno (int errno); +int nfsd_cross_mnt(struct svc_rqst *rqstp, struct dentry **dpp, + struct svc_export **expp); +__be32 nfsd_lookup(struct svc_rqst *, struct svc_fh *, + const char *, unsigned int, struct svc_fh *); +__be32 nfsd_lookup_dentry(struct svc_rqst *, struct svc_fh *, + const char *, unsigned int, + struct svc_export **, struct dentry **); +__be32 nfsd_setattr(struct svc_rqst *, struct svc_fh *, + struct nfsd_attrs *, const struct timespec64 *); +int nfsd_mountpoint(struct dentry *, struct svc_export *); +#ifdef CONFIG_NFSD_V4 +__be32 nfsd4_vfs_fallocate(struct svc_rqst *, struct svc_fh *, + struct file *, loff_t, loff_t, int); +__be32 nfsd4_clone_file_range(struct svc_rqst *rqstp, + struct nfsd_file *nf_src, u64 src_pos, + struct nfsd_file *nf_dst, u64 dst_pos, + u64 count, bool sync); +#endif /* CONFIG_NFSD_V4 */ +__be32 nfsd_create_locked(struct svc_rqst *, struct svc_fh *, + struct nfsd_attrs *attrs, int type, dev_t rdev, + struct svc_fh *res); +__be32 nfsd_create(struct svc_rqst *, struct svc_fh *, + char *name, int len, struct nfsd_attrs *attrs, + int type, dev_t rdev, struct svc_fh *res); +__be32 nfsd_access(struct svc_rqst *, struct svc_fh *, u32 *, u32 *); +__be32 nfsd_create_setattr(struct svc_rqst *rqstp, struct svc_fh *fhp, + struct svc_fh *resfhp, struct nfsd_attrs *iap); +__be32 nfsd_commit(struct svc_rqst *rqst, struct svc_fh *fhp, + struct nfsd_file *nf, u64 offset, u32 count, + __be32 *verf); +#ifdef CONFIG_NFSD_V4 +__be32 nfsd_getxattr(struct svc_rqst *rqstp, struct svc_fh *fhp, + char *name, void **bufp, int *lenp); +__be32 nfsd_listxattr(struct svc_rqst *rqstp, struct svc_fh *fhp, + char **bufp, int *lenp); +__be32 nfsd_removexattr(struct svc_rqst *rqstp, struct svc_fh *fhp, + char *name); +__be32 nfsd_setxattr(struct svc_rqst *rqstp, struct svc_fh *fhp, + char *name, void *buf, u32 len, u32 flags); +#endif +int nfsd_open_break_lease(struct inode *, int); +__be32 nfsd_open(struct svc_rqst *, struct svc_fh *, umode_t, + int, struct file **); +int nfsd_open_verified(struct svc_fh *fhp, umode_t type, int may_flags, + struct file **filp); +__be32 nfsd_splice_read(struct svc_rqst *rqstp, struct svc_fh *fhp, + struct file *file, loff_t offset, + unsigned long *count, + u32 *eof); +__be32 nfsd_iter_read(struct svc_rqst *rqstp, struct svc_fh *fhp, + struct nfsd_file *nf, loff_t offset, + unsigned long *count, unsigned int base, + u32 *eof); +bool nfsd_read_splice_ok(struct svc_rqst *rqstp); +__be32 nfsd_read(struct svc_rqst *rqstp, struct svc_fh *fhp, + loff_t offset, unsigned long *count, + u32 *eof); +__be32 nfsd_write(struct svc_rqst *rqstp, struct svc_fh *fhp, + loff_t offset, const struct xdr_buf *payload, + unsigned long *cnt, int stable, __be32 *verf); +__be32 nfsd_vfs_write(struct svc_rqst *rqstp, struct svc_fh *fhp, + struct nfsd_file *nf, loff_t offset, + const struct xdr_buf *payload, + unsigned long *cnt, int stable, __be32 *verf); +__be32 nfsd_readlink(struct svc_rqst *, struct svc_fh *, + char *, int *); +__be32 nfsd_symlink(struct svc_rqst *, struct svc_fh *, + char *name, int len, char *path, + struct nfsd_attrs *attrs, + struct svc_fh *res); +__be32 nfsd_link(struct svc_rqst *, struct svc_fh *, + char *, int, struct svc_fh *); +ssize_t nfsd_copy_file_range(struct file *, u64, + struct file *, u64, u64); +__be32 nfsd_rename(struct svc_rqst *, + struct svc_fh *, char *, int, + struct svc_fh *, char *, int); +__be32 nfsd_unlink(struct svc_rqst *, struct svc_fh *, int type, + char *name, int len); +__be32 nfsd_readdir(struct svc_rqst *, struct svc_fh *, + loff_t *, struct readdir_cd *, nfsd_filldir_t); +__be32 nfsd_statfs(struct svc_rqst *, struct svc_fh *, + struct kstatfs *, int access); +int nfsd_get_case_info(struct dentry *dentry, + bool *case_insensitive, + bool *case_preserving); + +__be32 nfsd_permission(struct svc_cred *cred, struct svc_export *exp, + struct dentry *dentry, int acc); + +void nfsd_filp_close(struct file *fp); + +#endif /* LINUX_NFSD_VFS_H */ diff --git a/fs/nfsd/xdr.h b/fs/nfsd/xdr.h new file mode 100644 index 000000000..df540c940 --- /dev/null +++ b/fs/nfsd/xdr.h @@ -0,0 +1,178 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* XDR types for nfsd. This is mainly a typing exercise. */ + +#ifndef LINUX_NFSD_H +#define LINUX_NFSD_H + +#include +#include "nfsd.h" +#include "nfsfh.h" +#include "vfs.h" + +struct nfsd_fhandle { + struct svc_fh fh; +}; + +struct nfsd_sattrargs { + struct svc_fh fh; + struct iattr attrs; +}; + +struct nfsd_diropargs { + struct svc_fh fh; + char * name; + unsigned int len; +}; + +struct nfsd_readargs { + struct svc_fh fh; + __u32 offset; + __u32 count; +}; + +struct nfsd_writeargs { + svc_fh fh; + __u32 offset; + __u32 len; + struct xdr_buf payload; +}; + +struct nfsd_createargs { + struct svc_fh fh; + char * name; + unsigned int len; + struct iattr attrs; +}; + +struct nfsd_renameargs { + struct svc_fh ffh; + char * fname; + unsigned int flen; + struct svc_fh tfh; + char * tname; + unsigned int tlen; +}; + +struct nfsd_linkargs { + struct svc_fh ffh; + struct svc_fh tfh; + char * tname; + unsigned int tlen; +}; + +struct nfsd_symlinkargs { + struct svc_fh ffh; + char * fname; + unsigned int flen; + char * tname; + unsigned int tlen; + struct iattr attrs; + struct kvec first; +}; + +struct nfsd_readdirargs { + struct svc_fh fh; + __u32 cookie; + __u32 count; +}; + +struct nfsd_stat { + __be32 status; +}; + +struct nfsd_attrstat { + __be32 status; + struct svc_fh fh; + struct kstat stat; +}; + +struct nfsd_diropres { + __be32 status; + struct svc_fh fh; + struct kstat stat; +}; + +struct nfsd_readlinkres { + __be32 status; + int len; + struct page *page; +}; + +struct nfsd_readres { + __be32 status; + struct svc_fh fh; + unsigned long count; + struct kstat stat; + struct page **pages; +}; + +struct nfsd_readdirres { + /* Components of the reply */ + __be32 status; + + int count; + + /* Used to encode the reply's entry list */ + struct xdr_stream xdr; + struct xdr_buf dirlist; + struct readdir_cd common; + unsigned int cookie_offset; +}; + +struct nfsd_statfsres { + __be32 status; + struct kstatfs stats; +}; + +/* + * Storage requirements for XDR arguments and results. + */ +union nfsd_xdrstore { + struct nfsd_sattrargs sattr; + struct nfsd_diropargs dirop; + struct nfsd_readargs read; + struct nfsd_writeargs write; + struct nfsd_createargs create; + struct nfsd_renameargs rename; + struct nfsd_linkargs link; + struct nfsd_symlinkargs symlink; + struct nfsd_readdirargs readdir; +}; + +#define NFS2_SVC_XDRSIZE sizeof(union nfsd_xdrstore) + + +bool nfssvc_decode_fhandleargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfssvc_decode_sattrargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfssvc_decode_diropargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfssvc_decode_readargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfssvc_decode_writeargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfssvc_decode_createargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfssvc_decode_renameargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfssvc_decode_linkargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfssvc_decode_symlinkargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfssvc_decode_readdirargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); + +bool nfssvc_encode_statres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfssvc_encode_attrstatres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfssvc_encode_diropres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfssvc_encode_readlinkres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfssvc_encode_readres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfssvc_encode_statfsres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfssvc_encode_readdirres(struct svc_rqst *rqstp, struct xdr_stream *xdr); + +void nfssvc_encode_nfscookie(struct nfsd_readdirres *resp, u32 offset); +int nfssvc_encode_entry(void *data, const char *name, int namlen, + loff_t offset, u64 ino, unsigned int d_type); + +void nfssvc_release_attrstat(struct svc_rqst *rqstp); +void nfssvc_release_diropres(struct svc_rqst *rqstp); +void nfssvc_release_readres(struct svc_rqst *rqstp); + +/* Helper functions for NFSv2 ACL code */ +bool svcxdr_decode_fhandle(struct xdr_stream *xdr, struct svc_fh *fhp); +bool svcxdr_encode_stat(struct xdr_stream *xdr, __be32 status); +bool svcxdr_encode_fattr(struct svc_rqst *rqstp, struct xdr_stream *xdr, + const struct svc_fh *fhp, const struct kstat *stat); + +#endif /* LINUX_NFSD_H */ diff --git a/fs/nfsd/xdr3.h b/fs/nfsd/xdr3.h new file mode 100644 index 000000000..344203874 --- /dev/null +++ b/fs/nfsd/xdr3.h @@ -0,0 +1,315 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * XDR types for NFSv3 in nfsd. + * + * Copyright (C) 1996-1998, Olaf Kirch + */ + +#ifndef _LINUX_NFSD_XDR3_H +#define _LINUX_NFSD_XDR3_H + +#include "xdr.h" +#include "vfs.h" + +struct nfsd3_sattrargs { + struct svc_fh fh; + struct iattr attrs; + int check_guard; + struct timespec64 guardtime; +}; + +struct nfsd3_diropargs { + struct svc_fh fh; + char * name; + unsigned int len; +}; + +struct nfsd3_accessargs { + struct svc_fh fh; + __u32 access; +}; + +struct nfsd3_readargs { + struct svc_fh fh; + __u64 offset; + __u32 count; +}; + +struct nfsd3_writeargs { + svc_fh fh; + __u64 offset; + __u32 count; + int stable; + __u32 len; + struct xdr_buf payload; +}; + +struct nfsd3_createargs { + struct svc_fh fh; + char * name; + unsigned int len; + int createmode; + struct iattr attrs; + __be32 * verf; +}; + +struct nfsd3_mknodargs { + struct svc_fh fh; + char * name; + unsigned int len; + __u32 ftype; + __u32 major, minor; + struct iattr attrs; +}; + +struct nfsd3_renameargs { + struct svc_fh ffh; + char * fname; + unsigned int flen; + struct svc_fh tfh; + char * tname; + unsigned int tlen; +}; + +struct nfsd3_linkargs { + struct svc_fh ffh; + struct svc_fh tfh; + char * tname; + unsigned int tlen; +}; + +struct nfsd3_symlinkargs { + struct svc_fh ffh; + char * fname; + unsigned int flen; + char * tname; + unsigned int tlen; + struct iattr attrs; + struct kvec first; +}; + +struct nfsd3_readdirargs { + struct svc_fh fh; + __u64 cookie; + __u32 count; + __be32 * verf; +}; + +struct nfsd3_commitargs { + struct svc_fh fh; + __u64 offset; + __u32 count; +}; + +struct nfsd3_getaclargs { + struct svc_fh fh; + __u32 mask; +}; + +struct posix_acl; +struct nfsd3_setaclargs { + struct svc_fh fh; + __u32 mask; + struct posix_acl *acl_access; + struct posix_acl *acl_default; +}; + +struct nfsd3_attrstat { + __be32 status; + struct svc_fh fh; + struct kstat stat; +}; + +/* LOOKUP, CREATE, MKDIR, SYMLINK, MKNOD */ +struct nfsd3_diropres { + __be32 status; + struct svc_fh dirfh; + struct svc_fh fh; +}; + +struct nfsd3_accessres { + __be32 status; + struct svc_fh fh; + __u32 access; + struct kstat stat; +}; + +struct nfsd3_readlinkres { + __be32 status; + struct svc_fh fh; + __u32 len; + struct page **pages; +}; + +struct nfsd3_readres { + __be32 status; + struct svc_fh fh; + unsigned long count; + __u32 eof; + struct page **pages; +}; + +struct nfsd3_writeres { + __be32 status; + struct svc_fh fh; + unsigned long count; + int committed; + __be32 verf[2]; +}; + +struct nfsd3_renameres { + __be32 status; + struct svc_fh ffh; + struct svc_fh tfh; +}; + +struct nfsd3_linkres { + __be32 status; + struct svc_fh tfh; + struct svc_fh fh; +}; + +struct nfsd3_readdirres { + /* Components of the reply */ + __be32 status; + struct svc_fh fh; + __be32 verf[2]; + + /* Used to encode the reply's entry list */ + struct xdr_stream xdr; + struct xdr_buf dirlist; + struct svc_fh scratch; + struct readdir_cd common; + unsigned int cookie_offset; + struct svc_rqst * rqstp; + +}; + +struct nfsd3_fsstatres { + __be32 status; + struct kstatfs stats; + __u32 invarsec; +}; + +struct nfsd3_fsinfores { + __be32 status; + __u32 f_rtmax; + __u32 f_rtpref; + __u32 f_rtmult; + __u32 f_wtmax; + __u32 f_wtpref; + __u32 f_wtmult; + __u32 f_dtpref; + __u64 f_maxfilesize; + __u32 f_properties; +}; + +struct nfsd3_pathconfres { + __be32 status; + __u32 p_link_max; + __u32 p_name_max; + __u32 p_no_trunc; + __u32 p_chown_restricted; + bool p_case_insensitive; + bool p_case_preserving; +}; + +struct nfsd3_commitres { + __be32 status; + struct svc_fh fh; + __be32 verf[2]; +}; + +struct nfsd3_getaclres { + __be32 status; + struct svc_fh fh; + int mask; + struct posix_acl *acl_access; + struct posix_acl *acl_default; + struct kstat stat; +}; + +/* dummy type for release */ +struct nfsd3_fhandle_pair { + __u32 dummy; + struct svc_fh fh1; + struct svc_fh fh2; +}; + +/* + * Storage requirements for XDR arguments and results. + */ +union nfsd3_xdrstore { + struct nfsd3_sattrargs sattrargs; + struct nfsd3_diropargs diropargs; + struct nfsd3_readargs readargs; + struct nfsd3_writeargs writeargs; + struct nfsd3_createargs createargs; + struct nfsd3_renameargs renameargs; + struct nfsd3_linkargs linkargs; + struct nfsd3_symlinkargs symlinkargs; + struct nfsd3_readdirargs readdirargs; + struct nfsd3_diropres diropres; + struct nfsd3_accessres accessres; + struct nfsd3_readlinkres readlinkres; + struct nfsd3_readres readres; + struct nfsd3_writeres writeres; + struct nfsd3_renameres renameres; + struct nfsd3_linkres linkres; + struct nfsd3_readdirres readdirres; + struct nfsd3_fsstatres fsstatres; + struct nfsd3_fsinfores fsinfores; + struct nfsd3_pathconfres pathconfres; + struct nfsd3_commitres commitres; + struct nfsd3_getaclres getaclres; +}; + +#define NFS3_SVC_XDRSIZE sizeof(union nfsd3_xdrstore) + +bool nfs3svc_decode_fhandleargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_decode_sattrargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_decode_diropargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_decode_accessargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_decode_readargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_decode_writeargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_decode_createargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_decode_mkdirargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_decode_mknodargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_decode_renameargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_decode_linkargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_decode_symlinkargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_decode_readdirargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_decode_readdirplusargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_decode_commitargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); + +bool nfs3svc_encode_getattrres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_encode_wccstat(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_encode_lookupres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_encode_accessres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_encode_readlinkres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_encode_readres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_encode_writeres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_encode_createres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_encode_renameres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_encode_linkres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_encode_readdirres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_encode_fsstatres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_encode_fsinfores(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_encode_pathconfres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs3svc_encode_commitres(struct svc_rqst *rqstp, struct xdr_stream *xdr); + +void nfs3svc_release_fhandle(struct svc_rqst *); +void nfs3svc_release_fhandle2(struct svc_rqst *); + +void nfs3svc_encode_cookie3(struct nfsd3_readdirres *resp, u64 offset); +int nfs3svc_encode_entry3(void *data, const char *name, int namlen, + loff_t offset, u64 ino, unsigned int d_type); +int nfs3svc_encode_entryplus3(void *data, const char *name, int namlen, + loff_t offset, u64 ino, unsigned int d_type); +/* Helper functions for NFSv3 ACL code */ +bool svcxdr_decode_nfs_fh3(struct xdr_stream *xdr, struct svc_fh *fhp); +bool svcxdr_encode_nfsstat3(struct xdr_stream *xdr, __be32 status); +bool svcxdr_encode_post_op_attr(struct svc_rqst *rqstp, struct xdr_stream *xdr, + const struct svc_fh *fhp); + +#endif /* _LINUX_NFSD_XDR3_H */ diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h new file mode 100644 index 000000000..c7eda5bc8 --- /dev/null +++ b/fs/nfsd/xdr4.h @@ -0,0 +1,1098 @@ +/* + * Server-side types for NFSv4. + * + * Copyright (c) 2002 The Regents of the University of Michigan. + * All rights reserved. + * + * Kendrick Smith + * Andy Adamson + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef _LINUX_NFSD_XDR4_H +#define _LINUX_NFSD_XDR4_H + +#include "state.h" +#include "vfs.h" + +#define NFSD4_MAX_TAGLEN 128 +#define XDR_LEN(n) (((n) + 3) & ~3) + +#define CURRENT_STATE_ID_FLAG (1<<0) +#define SAVED_STATE_ID_FLAG (1<<1) + +#define SET_CSTATE_FLAG(c, f) ((c)->sid_flags |= (f)) +#define HAS_CSTATE_FLAG(c, f) ((c)->sid_flags & (f)) +#define CLEAR_CSTATE_FLAG(c, f) ((c)->sid_flags &= ~(f)) + +/** + * nfsd4_encode_bool - Encode an XDR bool type result + * @xdr: target XDR stream + * @val: boolean value to encode + * + * Return values: + * %nfs_ok: @val encoded; @xdr advanced to next position + * %nfserr_resource: stream buffer space exhausted + */ +static __always_inline __be32 +nfsd4_encode_bool(struct xdr_stream *xdr, bool val) +{ + __be32 *p = xdr_reserve_space(xdr, XDR_UNIT); + + if (unlikely(p == NULL)) + return nfserr_resource; + *p = val ? xdr_one : xdr_zero; + return nfs_ok; +} + +/** + * nfsd4_encode_uint32_t - Encode an XDR uint32_t type result + * @xdr: target XDR stream + * @val: integer value to encode + * + * Return values: + * %nfs_ok: @val encoded; @xdr advanced to next position + * %nfserr_resource: stream buffer space exhausted + */ +static __always_inline __be32 +nfsd4_encode_uint32_t(struct xdr_stream *xdr, u32 val) +{ + __be32 *p = xdr_reserve_space(xdr, XDR_UNIT); + + if (unlikely(p == NULL)) + return nfserr_resource; + *p = cpu_to_be32(val); + return nfs_ok; +} + +#define nfsd4_encode_aceflag4(x, v) nfsd4_encode_uint32_t(x, v) +#define nfsd4_encode_acemask4(x, v) nfsd4_encode_uint32_t(x, v) +#define nfsd4_encode_acetype4(x, v) nfsd4_encode_uint32_t(x, v) +#define nfsd4_encode_count4(x, v) nfsd4_encode_uint32_t(x, v) +#define nfsd4_encode_mode4(x, v) nfsd4_encode_uint32_t(x, v) +#define nfsd4_encode_nfs_lease4(x, v) nfsd4_encode_uint32_t(x, v) +#define nfsd4_encode_qop4(x, v) nfsd4_encode_uint32_t(x, v) +#define nfsd4_encode_sequenceid4(x, v) nfsd4_encode_uint32_t(x, v) +#define nfsd4_encode_slotid4(x, v) nfsd4_encode_uint32_t(x, v) + +/** + * nfsd4_encode_uint64_t - Encode an XDR uint64_t type result + * @xdr: target XDR stream + * @val: integer value to encode + * + * Return values: + * %nfs_ok: @val encoded; @xdr advanced to next position + * %nfserr_resource: stream buffer space exhausted + */ +static __always_inline __be32 +nfsd4_encode_uint64_t(struct xdr_stream *xdr, u64 val) +{ + __be32 *p = xdr_reserve_space(xdr, XDR_UNIT * 2); + + if (unlikely(p == NULL)) + return nfserr_resource; + put_unaligned_be64(val, p); + return nfs_ok; +} + +#define nfsd4_encode_changeid4(x, v) nfsd4_encode_uint64_t(x, v) +#define nfsd4_encode_nfs_cookie4(x, v) nfsd4_encode_uint64_t(x, v) +#define nfsd4_encode_length4(x, v) nfsd4_encode_uint64_t(x, v) +#define nfsd4_encode_offset4(x, v) nfsd4_encode_uint64_t(x, v) + +/** + * nfsd4_encode_opaque_fixed - Encode a fixed-length XDR opaque type result + * @xdr: target XDR stream + * @data: pointer to data + * @size: length of data in bytes + * + * Return values: + * %nfs_ok: @data encoded; @xdr advanced to next position + * %nfserr_resource: stream buffer space exhausted + */ +static __always_inline __be32 +nfsd4_encode_opaque_fixed(struct xdr_stream *xdr, const void *data, + size_t size) +{ + __be32 *p = xdr_reserve_space(xdr, xdr_align_size(size)); + size_t pad = xdr_pad_size(size); + + if (unlikely(p == NULL)) + return nfserr_resource; + memcpy(p, data, size); + if (pad) + memset((char *)p + size, 0, pad); + return nfs_ok; +} + +/** + * nfsd4_encode_opaque - Encode a variable-length XDR opaque type result + * @xdr: target XDR stream + * @data: pointer to data + * @size: length of data in bytes + * + * Return values: + * %nfs_ok: @data encoded; @xdr advanced to next position + * %nfserr_resource: stream buffer space exhausted + */ +static __always_inline __be32 +nfsd4_encode_opaque(struct xdr_stream *xdr, const void *data, size_t size) +{ + size_t pad = xdr_pad_size(size); + __be32 *p; + + p = xdr_reserve_space(xdr, XDR_UNIT + xdr_align_size(size)); + if (unlikely(p == NULL)) + return nfserr_resource; + *p++ = cpu_to_be32(size); + memcpy(p, data, size); + if (pad) + memset((char *)p + size, 0, pad); + return nfs_ok; +} + +#define nfsd4_encode_component4(x, d, s) nfsd4_encode_opaque(x, d, s) + +struct nfsd4_compound_state { + struct svc_fh current_fh; + struct svc_fh save_fh; + struct nfs4_stateowner *replay_owner; + struct nfs4_client *clp; + /* For sessions DRC */ + struct nfsd4_session *session; + struct nfsd4_slot *slot; + int data_offset; + bool spo_must_allowed; + size_t iovlen; + u32 minorversion; + __be32 status; + stateid_t current_stateid; + stateid_t save_stateid; + /* to indicate current and saved state id presents */ + u32 sid_flags; +}; + +static inline bool nfsd4_has_session(struct nfsd4_compound_state *cs) +{ + return cs->slot != NULL; +} + +struct nfsd4_change_info { + u32 atomic; + u64 before_change; + u64 after_change; +}; + +struct nfsd4_access { + u32 ac_req_access; /* request */ + u32 ac_supported; /* response */ + u32 ac_resp_access; /* response */ +}; + +struct nfsd4_close { + u32 cl_seqid; /* request */ + stateid_t cl_stateid; /* request+response */ +}; + +struct nfsd4_commit { + u64 co_offset; /* request */ + u32 co_count; /* request */ + nfs4_verifier co_verf; /* response */ +}; + +struct nfsd4_create { + u32 cr_namelen; /* request */ + char * cr_name; /* request */ + u32 cr_type; /* request */ + union { /* request */ + struct { + u32 datalen; + char *data; + struct kvec first; + } link; /* NF4LNK */ + struct { + u32 specdata1; + u32 specdata2; + } dev; /* NF4BLK, NF4CHR */ + } u; + u32 cr_bmval[3]; /* request */ + struct iattr cr_iattr; /* request */ + int cr_umask; /* request */ + struct nfsd4_change_info cr_cinfo; /* response */ + struct nfs4_acl *cr_acl; + struct posix_acl *cr_dpacl; + struct posix_acl *cr_pacl; + struct xdr_netobj cr_label; +}; +#define cr_datalen u.link.datalen +#define cr_data u.link.data +#define cr_first u.link.first +#define cr_specdata1 u.dev.specdata1 +#define cr_specdata2 u.dev.specdata2 + +struct nfsd4_delegreturn { + stateid_t dr_stateid; +}; + +struct nfsd4_getattr { + u32 ga_bmval[3]; /* request */ + struct svc_fh *ga_fhp; /* response */ +}; + +struct nfsd4_link { + u32 li_namelen; /* request */ + char * li_name; /* request */ + struct nfsd4_change_info li_cinfo; /* response */ +}; + +struct nfsd4_lock_denied { + clientid_t ld_clientid; + struct xdr_netobj ld_owner; + u64 ld_start; + u64 ld_length; + u32 ld_type; +}; + +struct nfsd4_lock { + /* request */ + u32 lk_type; + u32 lk_reclaim; /* boolean */ + u64 lk_offset; + u64 lk_length; + u32 lk_is_new; + union { + struct { + u32 open_seqid; + stateid_t open_stateid; + u32 lock_seqid; + clientid_t clientid; + struct xdr_netobj owner; + } new; + struct { + stateid_t lock_stateid; + u32 lock_seqid; + } old; + } v; + + /* response */ + stateid_t lk_resp_stateid; + struct nfsd4_lock_denied lk_denied; +}; +#define lk_new_open_seqid v.new.open_seqid +#define lk_new_open_stateid v.new.open_stateid +#define lk_new_lock_seqid v.new.lock_seqid +#define lk_new_clientid v.new.clientid +#define lk_new_owner v.new.owner +#define lk_old_lock_stateid v.old.lock_stateid +#define lk_old_lock_seqid v.old.lock_seqid + +struct nfsd4_lockt { + u32 lt_type; + clientid_t lt_clientid; + struct xdr_netobj lt_owner; + u64 lt_offset; + u64 lt_length; + struct nfsd4_lock_denied lt_denied; +}; + +struct nfsd4_locku { + u32 lu_type; + u32 lu_seqid; + stateid_t lu_stateid; + u64 lu_offset; + u64 lu_length; +}; + + +struct nfsd4_lookup { + u32 lo_len; /* request */ + char * lo_name; /* request */ +}; + +struct nfsd4_putfh { + u32 pf_fhlen; /* request */ + char *pf_fhval; /* request */ + bool no_verify; /* represents foreigh fh */ +}; + +struct nfsd4_getxattr { + char *getxa_name; /* request */ + u32 getxa_len; /* request */ + void *getxa_buf; +}; + +struct nfsd4_setxattr { + u32 setxa_flags; /* request */ + char *setxa_name; /* request */ + char *setxa_buf; /* request */ + u32 setxa_len; /* request */ + struct nfsd4_change_info setxa_cinfo; /* response */ +}; + +struct nfsd4_removexattr { + char *rmxa_name; /* request */ + struct nfsd4_change_info rmxa_cinfo; /* response */ +}; + +struct nfsd4_listxattrs { + u64 lsxa_cookie; /* request */ + u32 lsxa_maxcount; /* request */ + char *lsxa_buf; /* unfiltered buffer (reply) */ + u32 lsxa_len; /* unfiltered len (reply) */ +}; + +struct nfsd4_open { + u32 op_claim_type; /* request */ + u32 op_fnamelen; + char * op_fname; /* request - everything but CLAIM_PREV */ + u32 op_delegate_type; /* request - CLAIM_PREV only */ + stateid_t op_delegate_stateid; /* request - response */ + u32 op_why_no_deleg; /* response - DELEG_NONE_EXT only */ + u32 op_create; /* request */ + u32 op_createmode; /* request */ + int op_umask; /* request */ + u32 op_bmval[3]; /* request */ + struct iattr op_iattr; /* UNCHECKED4, GUARDED4, EXCLUSIVE4_1 */ + nfs4_verifier op_verf __attribute__((aligned(32))); + /* EXCLUSIVE4 */ + clientid_t op_clientid; /* request */ + struct xdr_netobj op_owner; /* request */ + u32 op_seqid; /* request */ + u32 op_share_access; /* request */ + u32 op_share_deny; /* request */ + u32 op_deleg_want; /* request */ + stateid_t op_stateid; /* response */ + __be32 op_xdr_error; /* see nfsd4_open_omfg() */ + struct nfsd4_change_info op_cinfo; /* response */ + u32 op_rflags; /* response */ + bool op_recall; /* response */ + bool op_truncate; /* used during processing */ + bool op_created; /* used during processing */ + struct nfs4_openowner *op_openowner; /* used during processing */ + struct file *op_filp; /* used during processing */ + struct nfs4_file *op_file; /* used during processing */ + struct nfs4_ol_stateid *op_stp; /* used during processing */ + struct nfs4_clnt_odstate *op_odstate; /* used during processing */ + struct nfs4_acl *op_acl; + struct posix_acl *op_dpacl; + struct posix_acl *op_pacl; + struct xdr_netobj op_label; + struct svc_rqst *op_rqstp; +}; + +struct nfsd4_open_confirm { + stateid_t oc_req_stateid /* request */; + u32 oc_seqid /* request */; + stateid_t oc_resp_stateid /* response */; +}; + +struct nfsd4_open_downgrade { + stateid_t od_stateid; + u32 od_seqid; + u32 od_share_access; /* request */ + u32 od_deleg_want; /* request */ + u32 od_share_deny; /* request */ +}; + + +struct nfsd4_read { + stateid_t rd_stateid; /* request */ + u64 rd_offset; /* request */ + u32 rd_length; /* request */ + int rd_vlen; + struct nfsd_file *rd_nf; + + struct svc_rqst *rd_rqstp; /* response */ + struct svc_fh *rd_fhp; /* response */ + u32 rd_eof; /* response */ +}; + +/* + * Cache the case-folding properties of @dir so a batched encoder + * (e.g., READDIR) does not re-probe per child. @dir is the + * directory being read, held by the request, so it is stable + * against rename for the duration of the cache's lifetime. + */ +struct nfsd_case_attrs_cache { + struct dentry *dir; + bool valid; + bool insensitive; + bool preserving; +}; + +struct nfsd4_readdir { + u64 rd_cookie; /* request */ + nfs4_verifier rd_verf; /* request */ + u32 rd_dircount; /* request */ + u32 rd_maxcount; /* request */ + u32 rd_bmval[3]; /* request */ + struct svc_rqst *rd_rqstp; /* response */ + struct svc_fh * rd_fhp; /* response */ + + struct readdir_cd common; + struct xdr_stream *xdr; + int cookie_offset; + struct nfsd_case_attrs_cache rd_case_cache; +}; + +struct nfsd4_release_lockowner { + clientid_t rl_clientid; + struct xdr_netobj rl_owner; +}; +struct nfsd4_readlink { + struct svc_rqst *rl_rqstp; /* request */ + struct svc_fh * rl_fhp; /* request */ +}; + +struct nfsd4_remove { + u32 rm_namelen; /* request */ + char * rm_name; /* request */ + struct nfsd4_change_info rm_cinfo; /* response */ +}; + +struct nfsd4_rename { + u32 rn_snamelen; /* request */ + char * rn_sname; /* request */ + u32 rn_tnamelen; /* request */ + char * rn_tname; /* request */ + struct nfsd4_change_info rn_sinfo; /* response */ + struct nfsd4_change_info rn_tinfo; /* response */ +}; + +struct nfsd4_secinfo { + u32 si_namelen; /* request */ + char *si_name; /* request */ + struct svc_export *si_exp; /* response */ +}; + +struct nfsd4_secinfo_no_name { + u32 sin_style; /* request */ + struct svc_export *sin_exp; /* response */ +}; + +struct nfsd4_setattr { + stateid_t sa_stateid; /* request */ + u32 sa_bmval[3]; /* request */ + struct iattr sa_iattr; /* request */ + struct nfs4_acl *sa_acl; + struct xdr_netobj sa_label; + struct posix_acl *sa_dpacl; + struct posix_acl *sa_pacl; +}; + +struct nfsd4_setclientid { + nfs4_verifier se_verf; /* request */ + struct xdr_netobj se_name; + u32 se_callback_prog; /* request */ + u32 se_callback_netid_len; /* request */ + char * se_callback_netid_val; /* request */ + u32 se_callback_addr_len; /* request */ + char * se_callback_addr_val; /* request */ + u32 se_callback_ident; /* request */ + clientid_t se_clientid; /* response */ + nfs4_verifier se_confirm; /* response */ +}; + +struct nfsd4_setclientid_confirm { + clientid_t sc_clientid; + nfs4_verifier sc_confirm; +}; + +struct nfsd4_test_stateid_id { + __be32 ts_id_status; + stateid_t ts_id_stateid; + struct list_head ts_id_list; +}; + +struct nfsd4_test_stateid { + u32 ts_num_ids; + struct list_head ts_stateid_list; +}; + +struct nfsd4_free_stateid { + stateid_t fr_stateid; /* request */ +}; + +struct nfsd4_get_dir_delegation { + /* request */ + u32 gdda_signal_deleg_avail; + u32 gdda_notification_types[1]; + struct timespec64 gdda_child_attr_delay; + struct timespec64 gdda_dir_attr_delay; + u32 gdda_child_attributes[3]; + u32 gdda_dir_attributes[3]; + /* response */ + u32 gddrnf_status; + nfs4_verifier gddr_cookieverf; + stateid_t gddr_stateid; + u32 gddr_notification[1]; + u32 gddr_child_attributes[3]; + u32 gddr_dir_attributes[3]; + bool gddrnf_will_signal_deleg_avail; +}; + +/* also used for NVERIFY */ +struct nfsd4_verify { + u32 ve_bmval[3]; /* request */ + u32 ve_attrlen; /* request */ + char * ve_attrval; /* request */ +}; + +struct nfsd4_write { + stateid_t wr_stateid; /* request */ + u64 wr_offset; /* request */ + u32 wr_stable_how; /* request */ + u32 wr_buflen; /* request */ + struct xdr_buf wr_payload; /* request */ + + u32 wr_bytes_written; /* response */ + u32 wr_how_written; /* response */ + nfs4_verifier wr_verifier; /* response */ +}; + +struct nfsd4_exchange_id { + nfs4_verifier verifier; + struct xdr_netobj clname; + u32 flags; + clientid_t clientid; + u32 seqid; + u32 spa_how; + u32 spo_must_enforce[3]; + u32 spo_must_allow[3]; + struct xdr_netobj nii_domain; + struct xdr_netobj nii_name; + struct timespec64 nii_time; + char *server_impl_name; +}; + +struct nfsd4_sequence { + struct nfs4_sessionid sessionid; /* request/response */ + u32 seqid; /* request/response */ + u32 slotid; /* request/response */ + u32 maxslots; /* request */ + u32 cachethis; /* request */ + u32 maxslots_response; /* response */ + u32 target_maxslots; /* response */ + u32 status_flags; /* response */ +}; + +struct nfsd4_destroy_session { + struct nfs4_sessionid sessionid; +}; + +struct nfsd4_destroy_clientid { + clientid_t clientid; +}; + +struct nfsd4_reclaim_complete { + u32 rca_one_fs; +}; + +struct nfsd4_deviceid { + u64 fsid_idx; + u32 generation; +}; + +static inline __be32 * +svcxdr_encode_deviceid4(__be32 *p, const struct nfsd4_deviceid *devid) +{ + __be64 *q = (__be64 *)p; + + *q = (__force __be64)devid->fsid_idx; + p += 2; + *p++ = (__force __be32)devid->generation; + *p++ = xdr_zero; + return p; +} + +static inline __be32 * +svcxdr_decode_deviceid4(__be32 *p, struct nfsd4_deviceid *devid) +{ + __be64 *q = (__be64 *)p; + + devid->fsid_idx = (__force u64)(*q); + p += 2; + devid->generation = (__force u32)(*p++); + p++; /* NFSD does not use the remaining octets */ + return p; +} + +static inline __be32 +nfsd4_decode_deviceid4(struct xdr_stream *xdr, struct nfsd4_deviceid *devid) +{ + __be32 *p = xdr_inline_decode(xdr, NFS4_DEVICEID4_SIZE); + + if (unlikely(!p)) + return nfserr_bad_xdr; + svcxdr_decode_deviceid4(p, devid); + return nfs_ok; +} + +struct nfsd4_layout_seg { + u32 iomode; + u64 offset; + u64 length; +}; + +struct nfsd4_getdeviceinfo { + struct nfsd4_deviceid gd_devid; /* request */ + u32 gd_layout_type; /* request */ + u32 gd_maxcount; /* request */ + u32 gd_notify_types;/* request - response */ + void *gd_device; /* response */ +}; + +struct nfsd4_layoutget { + u64 lg_minlength; /* request */ + u32 lg_signal; /* request */ + u32 lg_layout_type; /* request */ + u32 lg_maxcount; /* request */ + stateid_t lg_sid; /* request/response */ + struct nfsd4_layout_seg lg_seg; /* request/response */ + void *lg_content; /* response */ +}; + +struct nfsd4_layoutcommit { + stateid_t lc_sid; /* request */ + struct nfsd4_layout_seg lc_seg; /* request */ + u32 lc_reclaim; /* request */ + u32 lc_newoffset; /* request */ + u64 lc_last_wr; /* request */ + struct timespec64 lc_mtime; /* request */ + u32 lc_layout_type; /* request */ + struct xdr_buf lc_up_layout; /* decoded by callback */ + bool lc_size_chg; /* response */ + u64 lc_newsize; /* response */ +}; + +struct nfsd4_layoutreturn { + u32 lr_return_type; /* request */ + u32 lr_layout_type; /* request */ + struct nfsd4_layout_seg lr_seg; /* request */ + u32 lr_reclaim; /* request */ + u32 lrf_body_len; /* request */ + void *lrf_body; /* request */ + stateid_t lr_sid; /* request/response */ + bool lrs_present; /* response */ +}; + +struct nfsd4_fallocate { + /* request */ + stateid_t falloc_stateid; + loff_t falloc_offset; + u64 falloc_length; +}; + +struct nfsd4_clone { + /* request */ + stateid_t cl_src_stateid; + stateid_t cl_dst_stateid; + u64 cl_src_pos; + u64 cl_dst_pos; + u64 cl_count; +}; + +struct nfsd42_write_res { + u64 wr_bytes_written; + u32 wr_stable_how; + nfs4_verifier wr_verifier; + stateid_t cb_stateid; +}; + +struct nfsd4_cb_offload { + struct nfsd4_callback co_cb; + struct nfsd42_write_res co_res; + __be32 co_nfserr; + unsigned int co_retries; + struct knfsd_fh co_fh; + + struct nfs4_sessionid co_referring_sessionid; + u32 co_referring_slotid; + u32 co_referring_seqno; +}; + +struct nfsd4_copy { + /* request */ + stateid_t cp_src_stateid; + stateid_t cp_dst_stateid; + u64 cp_src_pos; + u64 cp_dst_pos; + u64 cp_count; + struct nl4_server *cp_src; + + unsigned long cp_flags; +#define NFSD4_COPY_F_STOPPED (0) +#define NFSD4_COPY_F_INTRA (1) +#define NFSD4_COPY_F_SYNCHRONOUS (2) +#define NFSD4_COPY_F_COMMITTED (3) +#define NFSD4_COPY_F_COMPLETED (4) +#define NFSD4_COPY_F_OFFLOAD_DONE (5) +#define NFSD4_COPY_F_CB_ERROR (6) + + /* response */ + __be32 nfserr; + struct nfsd42_write_res cp_res; + struct knfsd_fh fh; + + struct nfs4_client *cp_clp; + + struct nfsd_file *nf_src; + struct nfsd_file *nf_dst; + bool attr_update; + + struct nfsd4_ssc_umount_item *ss_nsui; + struct nfs_fh c_fh; + nfs4_stateid stateid; + struct nfsd_net *cp_nn; +}; + +/* + * Durable state for an async (background) server-side COPY. + * + * struct nfsd4_copy is transient: it lives in the COMPOUND argument buffer + * and is reused once the op returns. An async COPY outlives the COMPOUND + * (worker kthread, reaper linkage, CB_OFFLOAD), so its params and result are + * snapshotted into the embedded cp_copy and it never points into the request + * buffer. + */ +struct nfsd4_async_copy { + struct nfs4_stid cp_stid; /* SC_TYPE_COPY, in cl_stateids */ + struct nfsd4_copy cp_copy; /* operation params + result */ + + struct list_head copies; /* nfs4_client.async_copies */ + struct task_struct *copy_task; + refcount_t refcount; + unsigned int cp_ttl; + struct nfsd4_cb_offload cp_cb_offload; +}; + +static inline void nfsd4_copy_set_sync(struct nfsd4_copy *copy, bool sync) +{ + if (sync) + set_bit(NFSD4_COPY_F_SYNCHRONOUS, ©->cp_flags); + else + clear_bit(NFSD4_COPY_F_SYNCHRONOUS, ©->cp_flags); +} + +static inline bool nfsd4_copy_is_sync(const struct nfsd4_copy *copy) +{ + return test_bit(NFSD4_COPY_F_SYNCHRONOUS, ©->cp_flags); +} + +static inline bool nfsd4_copy_is_async(const struct nfsd4_copy *copy) +{ + return !test_bit(NFSD4_COPY_F_SYNCHRONOUS, ©->cp_flags); +} + +static inline bool nfsd4_ssc_is_inter(const struct nfsd4_copy *copy) +{ + return !test_bit(NFSD4_COPY_F_INTRA, ©->cp_flags); +} + +struct nfsd4_seek { + /* request */ + stateid_t seek_stateid; + loff_t seek_offset; + u32 seek_whence; + + /* response */ + u32 seek_eof; + loff_t seek_pos; +}; + +struct nfsd4_offload_status { + /* request */ + stateid_t stateid; + + /* response */ + u64 count; + __be32 status; + bool completed; +}; + +struct nfsd4_copy_notify { + /* request */ + stateid_t cpn_src_stateid; + struct nl4_server *cpn_dst; + + /* response */ + stateid_t cpn_cnr_stateid; + struct timespec64 cpn_lease_time; + struct nl4_server *cpn_src; +}; + +struct nfsd4_op { + u32 opnum; + __be32 status; + const struct nfsd4_operation *opdesc; + struct nfs4_replay *replay; + union nfsd4_op_u { + struct nfsd4_access access; + struct nfsd4_close close; + struct nfsd4_commit commit; + struct nfsd4_create create; + struct nfsd4_delegreturn delegreturn; + struct nfsd4_getattr getattr; + struct svc_fh * getfh; + struct nfsd4_link link; + struct nfsd4_lock lock; + struct nfsd4_lockt lockt; + struct nfsd4_locku locku; + struct nfsd4_lookup lookup; + struct nfsd4_verify nverify; + struct nfsd4_open open; + struct nfsd4_open_confirm open_confirm; + struct nfsd4_open_downgrade open_downgrade; + struct nfsd4_putfh putfh; + struct nfsd4_read read; + struct nfsd4_readdir readdir; + struct nfsd4_readlink readlink; + struct nfsd4_remove remove; + struct nfsd4_rename rename; + clientid_t renew; + struct nfsd4_secinfo secinfo; + struct nfsd4_setattr setattr; + struct nfsd4_setclientid setclientid; + struct nfsd4_setclientid_confirm setclientid_confirm; + struct nfsd4_verify verify; + struct nfsd4_write write; + struct nfsd4_release_lockowner release_lockowner; + + /* NFSv4.1 */ + struct nfsd4_exchange_id exchange_id; + struct nfsd4_backchannel_ctl backchannel_ctl; + struct nfsd4_bind_conn_to_session bind_conn_to_session; + struct nfsd4_create_session create_session; + struct nfsd4_destroy_session destroy_session; + struct nfsd4_destroy_clientid destroy_clientid; + struct nfsd4_sequence sequence; + struct nfsd4_reclaim_complete reclaim_complete; + struct nfsd4_test_stateid test_stateid; + struct nfsd4_free_stateid free_stateid; + struct nfsd4_get_dir_delegation get_dir_delegation; + struct nfsd4_getdeviceinfo getdeviceinfo; + struct nfsd4_layoutget layoutget; + struct nfsd4_layoutcommit layoutcommit; + struct nfsd4_layoutreturn layoutreturn; + struct nfsd4_secinfo_no_name secinfo_no_name; + + /* NFSv4.2 */ + struct nfsd4_fallocate allocate; + struct nfsd4_fallocate deallocate; + struct nfsd4_clone clone; + struct nfsd4_copy copy; + struct nfsd4_offload_status offload_status; + struct nfsd4_copy_notify copy_notify; + struct nfsd4_seek seek; + + struct nfsd4_getxattr getxattr; + struct nfsd4_setxattr setxattr; + struct nfsd4_listxattrs listxattrs; + struct nfsd4_removexattr removexattr; + } u; +}; + +bool nfsd4_cache_this_op(struct nfsd4_op *); + +/* + * Memory needed just for the duration of processing one compound: + */ +struct svcxdr_tmpbuf { + struct svcxdr_tmpbuf *next; + char buf[]; +}; + +struct nfsd4_compoundargs { + /* scratch variables for XDR decode */ + struct xdr_stream *xdr; + struct svcxdr_tmpbuf *to_free; + struct svc_rqst *rqstp; + + char * tag; + u32 taglen; + u32 minorversion; + u32 client_opcnt; + u32 opcnt; + bool splice_ok; + struct nfsd4_op *ops; + struct nfsd4_op iops[8]; +}; + +struct nfsd4_compoundres { + /* scratch variables for XDR encode */ + struct xdr_stream *xdr; + struct svc_rqst * rqstp; + + __be32 *statusp; + char * tag; + u32 taglen; + u32 opcnt; + + struct nfsd4_compound_state cstate; +}; + +static inline bool nfsd4_last_compound_op(struct svc_rqst *rqstp) +{ + struct nfsd4_compoundres *resp = rqstp->rq_resp; + struct nfsd4_compoundargs *argp = rqstp->rq_argp; + + return argp->opcnt == resp->opcnt; +} + +const struct nfsd4_operation *OPDESC(struct nfsd4_op *op); +int nfsd4_max_reply(struct svc_rqst *rqstp, struct nfsd4_op *op); +void warn_on_nonidempotent_op(struct nfsd4_op *op); + +#define NFS4_SVC_XDRSIZE sizeof(struct nfsd4_compoundargs) + +bool nfsd4_mach_creds_match(struct nfs4_client *cl, struct svc_rqst *rqstp); +bool nfs4svc_decode_compoundargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); +bool nfs4svc_encode_compoundres(struct svc_rqst *rqstp, struct xdr_stream *xdr); +__be32 nfsd4_check_resp_size(struct nfsd4_compoundres *, u32); +void nfsd4_encode_operation(struct nfsd4_compoundres *, struct nfsd4_op *); +void nfsd4_encode_replay(struct xdr_stream *xdr, struct nfsd4_op *op); +__be32 nfsd4_encode_fattr_to_buf(__be32 **p, int words, + struct svc_fh *fhp, struct svc_export *exp, + struct dentry *dentry, + u32 *bmval, struct svc_rqst *, int ignore_crossmnt); +u8 *nfsd4_encode_notify_event(struct xdr_stream *xdr, struct nfsd_notify_event *nne, + struct nfs4_delegation *dd, struct nfsd_file *nf, + u32 *notify_mask); +u8 *nfsd4_encode_dir_attr_change(struct xdr_stream *xdr, struct nfs4_delegation *dp, + struct nfsd_file *nf); +extern __be32 nfsd4_setclientid(struct svc_rqst *rqstp, + struct nfsd4_compound_state *, union nfsd4_op_u *u); +extern __be32 nfsd4_setclientid_confirm(struct svc_rqst *rqstp, + struct nfsd4_compound_state *, union nfsd4_op_u *u); +void nfsd4_exchange_id_release(union nfsd4_op_u *u); +extern __be32 nfsd4_exchange_id(struct svc_rqst *rqstp, + struct nfsd4_compound_state *, union nfsd4_op_u *u); +extern __be32 nfsd4_backchannel_ctl(struct svc_rqst *, + struct nfsd4_compound_state *, union nfsd4_op_u *u); +extern __be32 nfsd4_bind_conn_to_session(struct svc_rqst *, + struct nfsd4_compound_state *, union nfsd4_op_u *u); +extern __be32 nfsd4_create_session(struct svc_rqst *, + struct nfsd4_compound_state *, union nfsd4_op_u *u); +extern __be32 nfsd4_sequence(struct svc_rqst *, + struct nfsd4_compound_state *, union nfsd4_op_u *u); +extern void nfsd4_sequence_done(struct nfsd4_compoundres *resp); +extern __be32 nfsd4_destroy_session(struct svc_rqst *, + struct nfsd4_compound_state *, union nfsd4_op_u *u); +extern __be32 nfsd4_destroy_clientid(struct svc_rqst *, struct nfsd4_compound_state *, + union nfsd4_op_u *u); +__be32 nfsd4_reclaim_complete(struct svc_rqst *, struct nfsd4_compound_state *, + union nfsd4_op_u *u); +extern __be32 nfsd4_process_open1(struct nfsd4_compound_state *, + struct nfsd4_open *open, struct nfsd_net *nn); +extern __be32 nfsd4_process_open2(struct svc_rqst *rqstp, + struct svc_fh *current_fh, struct nfsd4_open *open); +extern void nfsd4_cstate_clear_replay(struct nfsd4_compound_state *cstate); +extern void nfsd4_cleanup_open_state(struct nfsd4_compound_state *cstate, + struct nfsd4_open *open); +extern __be32 nfsd4_open_confirm(struct svc_rqst *rqstp, + struct nfsd4_compound_state *, union nfsd4_op_u *u); +extern __be32 nfsd4_close(struct svc_rqst *rqstp, struct nfsd4_compound_state *, + union nfsd4_op_u *u); +extern __be32 nfsd4_open_downgrade(struct svc_rqst *rqstp, + struct nfsd4_compound_state *, union nfsd4_op_u *u); +extern __be32 nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *, + union nfsd4_op_u *u); +extern void nfsd4_lock_release(union nfsd4_op_u *u); +extern __be32 nfsd4_lockt(struct svc_rqst *rqstp, struct nfsd4_compound_state *, + union nfsd4_op_u *u); +extern void nfsd4_lockt_release(union nfsd4_op_u *u); +extern __be32 nfsd4_locku(struct svc_rqst *rqstp, struct nfsd4_compound_state *, + union nfsd4_op_u *u); +extern __be32 +nfsd4_release_lockowner(struct svc_rqst *rqstp, + struct nfsd4_compound_state *, union nfsd4_op_u *u); +extern void nfsd4_release_compoundargs(struct svc_rqst *rqstp); +extern __be32 nfsd4_delegreturn(struct svc_rqst *rqstp, + struct nfsd4_compound_state *, union nfsd4_op_u *u); +extern __be32 nfsd4_renew(struct svc_rqst *rqstp, struct nfsd4_compound_state *, + union nfsd4_op_u *u); +extern __be32 nfsd4_test_stateid(struct svc_rqst *rqstp, + struct nfsd4_compound_state *, union nfsd4_op_u *); +extern __be32 nfsd4_free_stateid(struct svc_rqst *rqstp, + struct nfsd4_compound_state *, union nfsd4_op_u *); +extern void nfsd4_bump_seqid(struct nfsd4_compound_state *, __be32 nfserr); + +enum nfsd4_op_flags { + ALLOWED_WITHOUT_FH = 1 << 0, /* No current filehandle required */ + ALLOWED_ON_ABSENT_FS = 1 << 1, /* ops processed on absent fs */ + ALLOWED_AS_FIRST_OP = 1 << 2, /* ops reqired first in compound */ + /* For rfc 5661 section 2.6.3.1.1: */ + OP_HANDLES_WRONGSEC = 1 << 3, + OP_IS_PUTFH_LIKE = 1 << 4, + /* + * These are the ops whose result size we estimate before + * encoding, to avoid performing an op then not being able to + * respond or cache a response. This includes writes and setattrs + * as well as the operations usually called "nonidempotent": + */ + OP_MODIFIES_SOMETHING = 1 << 5, + /* + * Cache compounds containing these ops in the xid-based drc: + * We use the DRC for compounds containing non-idempotent + * operations, *except* those that are 4.1-specific (since + * sessions provide their own EOS), and except for stateful + * operations other than setclientid and setclientid_confirm + * (since sequence numbers provide EOS for open, lock, etc in + * the v4.0 case). + */ + OP_CACHEME = 1 << 6, + /* + * These are ops which clear current state id. + */ + OP_CLEAR_STATEID = 1 << 7, + /* Most ops return only an error on failure; some may do more: */ + OP_NONTRIVIAL_ERROR_ENCODE = 1 << 8, +}; + +struct nfsd4_operation { + __be32 (*op_func)(struct svc_rqst *, struct nfsd4_compound_state *, + union nfsd4_op_u *); + void (*op_release)(union nfsd4_op_u *); + u32 op_flags; + char *op_name; + /* Try to get response size before operation */ + u32 (*op_rsize_bop)(const struct svc_rqst *rqstp, + const struct nfsd4_op *op); + void (*op_get_currentstateid)(struct nfsd4_compound_state *, + union nfsd4_op_u *); + void (*op_set_currentstateid)(struct nfsd4_compound_state *, + union nfsd4_op_u *); +}; + +struct nfsd4_cb_recall_any { + struct nfsd4_callback ra_cb; + u32 ra_keep; + u32 ra_bmval[1]; +}; + +#endif diff --git a/fs/nfsd/xdr4cb.h b/fs/nfsd/xdr4cb.h new file mode 100644 index 000000000..b06d0170d --- /dev/null +++ b/fs/nfsd/xdr4cb.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#define NFS4_MAXTAGLEN 20 + +#define NFS4_enc_cb_null_sz 0 +#define NFS4_dec_cb_null_sz 0 +#define cb_compound_enc_hdr_sz 4 +#define cb_compound_dec_hdr_sz (3 + (NFS4_MAXTAGLEN >> 2)) +#define sessionid_sz (NFS4_MAX_SESSIONID_LEN >> 2) +#define enc_referring_call4_sz (1 + 1) +#define enc_referring_call_list4_sz (sessionid_sz + 1 + \ + enc_referring_call4_sz) +#define cb_sequence_enc_sz (sessionid_sz + 4 + \ + enc_referring_call_list4_sz) +#define cb_sequence_dec_sz (op_dec_sz + sessionid_sz + 4) + +#define op_enc_sz 1 +#define op_dec_sz 2 +#define enc_nfs4_fh_sz (1 + (NFS4_FHSIZE >> 2)) +#define enc_stateid_sz (NFS4_STATEID_SIZE >> 2) +#define NFS4_enc_cb_recall_sz (cb_compound_enc_hdr_sz + \ + cb_sequence_enc_sz + \ + 1 + enc_stateid_sz + \ + enc_nfs4_fh_sz) + +#define NFS4_dec_cb_recall_sz (cb_compound_dec_hdr_sz + \ + cb_sequence_dec_sz + \ + op_dec_sz) +#define NFS4_enc_cb_layout_sz (cb_compound_enc_hdr_sz + \ + cb_sequence_enc_sz + \ + 1 + 3 + \ + enc_nfs4_fh_sz + 4) +#define NFS4_dec_cb_layout_sz (cb_compound_dec_hdr_sz + \ + cb_sequence_dec_sz + \ + op_dec_sz) + +#define NFS4_enc_cb_notify_sz (cb_compound_enc_hdr_sz + \ + cb_sequence_enc_sz + \ + 1 + enc_stateid_sz + \ + enc_nfs4_fh_sz + \ + 1 + \ + NOTIFY4_EVENT_QUEUE_SIZE * \ + (2 + (NFS4_OPAQUE_LIMIT >> 2))) + +#define NFS4_dec_cb_notify_sz (cb_compound_dec_hdr_sz + \ + cb_sequence_dec_sz + \ + op_dec_sz) + +#define NFS4_enc_cb_notify_lock_sz (cb_compound_enc_hdr_sz + \ + cb_sequence_enc_sz + \ + 2 + 1 + \ + XDR_QUADLEN(NFS4_OPAQUE_LIMIT) + \ + enc_nfs4_fh_sz) +#define NFS4_dec_cb_notify_lock_sz (cb_compound_dec_hdr_sz + \ + cb_sequence_dec_sz + \ + op_dec_sz) +#define enc_cb_offload_info_sz (1 + 1 + 2 + 1 + \ + XDR_QUADLEN(NFS4_VERIFIER_SIZE)) +#define NFS4_enc_cb_offload_sz (cb_compound_enc_hdr_sz + \ + cb_sequence_enc_sz + \ + enc_nfs4_fh_sz + \ + enc_stateid_sz + \ + enc_cb_offload_info_sz) +#define NFS4_dec_cb_offload_sz (cb_compound_dec_hdr_sz + \ + cb_sequence_dec_sz + \ + op_dec_sz) +#define NFS4_enc_cb_recall_any_sz (cb_compound_enc_hdr_sz + \ + cb_sequence_enc_sz + \ + 1 + 1 + 1) +#define NFS4_dec_cb_recall_any_sz (cb_compound_dec_hdr_sz + \ + cb_sequence_dec_sz + \ + op_dec_sz) + +/* + * 1: CB_GETATTR opcode (32-bit) + * N: file_handle + * 1: number of entry in attribute array (32-bit) + * 3: entry 0-2 in attribute array (32-bit * 3) + */ +#define NFS4_enc_cb_getattr_sz (cb_compound_enc_hdr_sz + \ + cb_sequence_enc_sz + \ + 1 + enc_nfs4_fh_sz + 1 + 3) +/* + * 4: fattr_bitmap_maxsz + * 1: attribute array len + * 2: change attr (64-bit) + * 2: size (64-bit) + * 2: atime.seconds (64-bit) + * 1: atime.nanoseconds (32-bit) + * 2: mtime.seconds (64-bit) + * 1: mtime.nanoseconds (32-bit) + */ +#define NFS4_dec_cb_getattr_sz (cb_compound_dec_hdr_sz + \ + cb_sequence_dec_sz + 4 + 1 + 2 + 2 + 2 + 1 + 2 + 1 + op_dec_sz) diff --git a/fs/nilfs2/Kconfig b/fs/nilfs2/Kconfig new file mode 100644 index 000000000..7dae168e3 --- /dev/null +++ b/fs/nilfs2/Kconfig @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: GPL-2.0-only +config NILFS2_FS + tristate "NILFS2 file system support" + select BUFFER_HEAD + select CRC32 + select LEGACY_DIRECT_IO + help + NILFS2 is a log-structured file system (LFS) supporting continuous + snapshotting. In addition to versioning capability of the entire + file system, users can even restore files mistakenly overwritten or + destroyed just a few seconds ago. Since this file system can keep + consistency like conventional LFS, it achieves quick recovery after + system crashes. + + NILFS2 creates a number of checkpoints every few seconds or per + synchronous write basis (unless there is no change). Users can + select significant versions among continuously created checkpoints, + and can change them into snapshots which will be preserved for long + periods until they are changed back to checkpoints. Each + snapshot is mountable as a read-only file system concurrently with + its writable mount, and this feature is convenient for online backup. + + Some features including atime, extended attributes, and POSIX ACLs, + are not supported yet. + + To compile this file system support as a module, choose M here: the + module will be called nilfs2. If unsure, say N. diff --git a/fs/nilfs2/Makefile b/fs/nilfs2/Makefile new file mode 100644 index 000000000..43b60b8a4 --- /dev/null +++ b/fs/nilfs2/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_NILFS2_FS) += nilfs2.o +nilfs2-y := inode.o file.o dir.o super.o namei.o page.o mdt.o \ + btnode.o bmap.o btree.o direct.o dat.o recovery.o \ + the_nilfs.o segbuf.o segment.o cpfile.o sufile.o \ + ifile.o alloc.o gcinode.o ioctl.o sysfs.o diff --git a/fs/nilfs2/alloc.c b/fs/nilfs2/alloc.c new file mode 100644 index 000000000..7b1cd2bae --- /dev/null +++ b/fs/nilfs2/alloc.c @@ -0,0 +1,958 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * NILFS dat/inode allocator + * + * Copyright (C) 2006-2008 Nippon Telegraph and Telephone Corporation. + * + * Originally written by Koji Sato. + * Two allocators were unified by Ryusuke Konishi and Amagai Yoshiji. + */ + +#include +#include +#include +#include +#include +#include "mdt.h" +#include "alloc.h" + + +/** + * nilfs_palloc_groups_per_desc_block - get the number of groups that a group + * descriptor block can maintain + * @inode: inode of metadata file using this allocator + * + * Return: Number of groups that a group descriptor block can maintain. + */ +static inline unsigned long +nilfs_palloc_groups_per_desc_block(const struct inode *inode) +{ + return i_blocksize(inode) / + sizeof(struct nilfs_palloc_group_desc); +} + +/** + * nilfs_palloc_groups_count - get maximum number of groups + * @inode: inode of metadata file using this allocator + * + * Return: Maximum number of groups. + */ +static inline unsigned long +nilfs_palloc_groups_count(const struct inode *inode) +{ + return 1UL << (BITS_PER_LONG - (inode->i_blkbits + 3 /* log2(8) */)); +} + +/** + * nilfs_palloc_init_blockgroup - initialize private variables for allocator + * @inode: inode of metadata file using this allocator + * @entry_size: size of the persistent object + * + * Return: 0 on success, or a negative error code on failure. + */ +int nilfs_palloc_init_blockgroup(struct inode *inode, unsigned int entry_size) +{ + struct nilfs_mdt_info *mi = NILFS_MDT(inode); + + mi->mi_bgl = kmalloc_obj(*mi->mi_bgl, GFP_NOFS); + if (!mi->mi_bgl) + return -ENOMEM; + + bgl_lock_init(mi->mi_bgl); + + nilfs_mdt_set_entry_size(inode, entry_size, 0); + + mi->mi_blocks_per_group = + DIV_ROUND_UP(nilfs_palloc_entries_per_group(inode), + mi->mi_entries_per_block) + 1; + /* + * Number of blocks in a group including entry blocks + * and a bitmap block + */ + mi->mi_blocks_per_desc_block = + nilfs_palloc_groups_per_desc_block(inode) * + mi->mi_blocks_per_group + 1; + /* + * Number of blocks per descriptor including the + * descriptor block + */ + return 0; +} + +/** + * nilfs_palloc_group - get group number and offset from an entry number + * @inode: inode of metadata file using this allocator + * @nr: serial number of the entry (e.g. inode number) + * @offset: pointer to store offset number in the group + * + * Return: Number of the group that contains the entry with the index + * specified by @nr. + */ +static unsigned long nilfs_palloc_group(const struct inode *inode, __u64 nr, + unsigned long *offset) +{ + __u64 group = nr; + + *offset = do_div(group, nilfs_palloc_entries_per_group(inode)); + return group; +} + +/** + * nilfs_palloc_desc_blkoff - get block offset of a group descriptor block + * @inode: inode of metadata file using this allocator + * @group: group number + * + * Return: Index number in the metadata file of the descriptor block of + * the group specified by @group. + */ +static unsigned long +nilfs_palloc_desc_blkoff(const struct inode *inode, unsigned long group) +{ + unsigned long desc_block = + group / nilfs_palloc_groups_per_desc_block(inode); + return desc_block * NILFS_MDT(inode)->mi_blocks_per_desc_block; +} + +/** + * nilfs_palloc_bitmap_blkoff - get block offset of a bitmap block + * @inode: inode of metadata file using this allocator + * @group: group number + * + * nilfs_palloc_bitmap_blkoff() returns block offset of the bitmap + * block used to allocate/deallocate entries in the specified group. + * + * Return: Index number in the metadata file of the bitmap block of + * the group specified by @group. + */ +static unsigned long +nilfs_palloc_bitmap_blkoff(const struct inode *inode, unsigned long group) +{ + unsigned long desc_offset = + group % nilfs_palloc_groups_per_desc_block(inode); + return nilfs_palloc_desc_blkoff(inode, group) + 1 + + desc_offset * NILFS_MDT(inode)->mi_blocks_per_group; +} + +/** + * nilfs_palloc_group_desc_nfrees - get the number of free entries in a group + * @desc: pointer to descriptor structure for the group + * @lock: spin lock protecting @desc + * + * Return: Number of free entries written in the group descriptor @desc. + */ +static unsigned long +nilfs_palloc_group_desc_nfrees(const struct nilfs_palloc_group_desc *desc, + spinlock_t *lock) +{ + unsigned long nfree; + + spin_lock(lock); + nfree = le32_to_cpu(desc->pg_nfrees); + spin_unlock(lock); + return nfree; +} + +/** + * nilfs_palloc_group_desc_add_entries - adjust count of free entries + * @desc: pointer to descriptor structure for the group + * @lock: spin lock protecting @desc + * @n: delta to be added + * + * Return: Number of free entries after adjusting the group descriptor + * @desc. + */ +static u32 +nilfs_palloc_group_desc_add_entries(struct nilfs_palloc_group_desc *desc, + spinlock_t *lock, u32 n) +{ + u32 nfree; + + spin_lock(lock); + le32_add_cpu(&desc->pg_nfrees, n); + nfree = le32_to_cpu(desc->pg_nfrees); + spin_unlock(lock); + return nfree; +} + +/** + * nilfs_palloc_entry_blkoff - get block offset of an entry block + * @inode: inode of metadata file using this allocator + * @nr: serial number of the entry (e.g. inode number) + * + * Return: Index number in the metadata file of the block containing + * the entry specified by @nr. + */ +static unsigned long +nilfs_palloc_entry_blkoff(const struct inode *inode, __u64 nr) +{ + unsigned long group, group_offset; + + group = nilfs_palloc_group(inode, nr, &group_offset); + + return nilfs_palloc_bitmap_blkoff(inode, group) + 1 + + group_offset / NILFS_MDT(inode)->mi_entries_per_block; +} + +/** + * nilfs_palloc_desc_block_init - initialize buffer of a group descriptor block + * @inode: inode of metadata file + * @bh: buffer head of the buffer to be initialized + * @from: kernel address mapped for a chunk of the block + * + * This function does not yet support the case where block size > PAGE_SIZE. + */ +static void nilfs_palloc_desc_block_init(struct inode *inode, + struct buffer_head *bh, void *from) +{ + struct nilfs_palloc_group_desc *desc = from; + unsigned long n = nilfs_palloc_groups_per_desc_block(inode); + __le32 nfrees; + + nfrees = cpu_to_le32(nilfs_palloc_entries_per_group(inode)); + while (n-- > 0) { + desc->pg_nfrees = nfrees; + desc++; + } +} + +static int nilfs_palloc_get_block(struct inode *inode, unsigned long blkoff, + int create, + void (*init_block)(struct inode *, + struct buffer_head *, + void *), + struct buffer_head **bhp, + struct nilfs_bh_assoc *prev, + spinlock_t *lock) +{ + int ret; + + spin_lock(lock); + if (prev->bh && blkoff == prev->blkoff && + likely(buffer_uptodate(prev->bh))) { + get_bh(prev->bh); + *bhp = prev->bh; + spin_unlock(lock); + return 0; + } + spin_unlock(lock); + + ret = nilfs_mdt_get_block(inode, blkoff, create, init_block, bhp); + if (!ret) { + spin_lock(lock); + /* + * The following code must be safe for change of the + * cache contents during the get block call. + */ + brelse(prev->bh); + get_bh(*bhp); + prev->bh = *bhp; + prev->blkoff = blkoff; + spin_unlock(lock); + } + return ret; +} + +/** + * nilfs_palloc_delete_block - delete a block on the persistent allocator file + * @inode: inode of metadata file using this allocator + * @blkoff: block offset + * @prev: nilfs_bh_assoc struct of the last used buffer + * @lock: spin lock protecting @prev + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOENT - Non-existent block. + * * %-ENOMEM - Insufficient memory available. + */ +static int nilfs_palloc_delete_block(struct inode *inode, unsigned long blkoff, + struct nilfs_bh_assoc *prev, + spinlock_t *lock) +{ + spin_lock(lock); + if (prev->bh && blkoff == prev->blkoff) { + brelse(prev->bh); + prev->bh = NULL; + } + spin_unlock(lock); + return nilfs_mdt_delete_block(inode, blkoff); +} + +/** + * nilfs_palloc_get_desc_block - get buffer head of a group descriptor block + * @inode: inode of metadata file using this allocator + * @group: group number + * @create: create flag + * @bhp: pointer to store the resultant buffer head + * + * Return: 0 on success, or a negative error code on failure. + */ +static int nilfs_palloc_get_desc_block(struct inode *inode, + unsigned long group, + int create, struct buffer_head **bhp) +{ + struct nilfs_palloc_cache *cache = NILFS_MDT(inode)->mi_palloc_cache; + + return nilfs_palloc_get_block(inode, + nilfs_palloc_desc_blkoff(inode, group), + create, nilfs_palloc_desc_block_init, + bhp, &cache->prev_desc, &cache->lock); +} + +/** + * nilfs_palloc_get_bitmap_block - get buffer head of a bitmap block + * @inode: inode of metadata file using this allocator + * @group: group number + * @create: create flag + * @bhp: pointer to store the resultant buffer head + * + * Return: 0 on success, or a negative error code on failure. + */ +static int nilfs_palloc_get_bitmap_block(struct inode *inode, + unsigned long group, + int create, struct buffer_head **bhp) +{ + struct nilfs_palloc_cache *cache = NILFS_MDT(inode)->mi_palloc_cache; + + return nilfs_palloc_get_block(inode, + nilfs_palloc_bitmap_blkoff(inode, group), + create, NULL, bhp, + &cache->prev_bitmap, &cache->lock); +} + +/** + * nilfs_palloc_delete_bitmap_block - delete a bitmap block + * @inode: inode of metadata file using this allocator + * @group: group number + * + * Return: 0 on success, or a negative error code on failure. + */ +static int nilfs_palloc_delete_bitmap_block(struct inode *inode, + unsigned long group) +{ + struct nilfs_palloc_cache *cache = NILFS_MDT(inode)->mi_palloc_cache; + + return nilfs_palloc_delete_block(inode, + nilfs_palloc_bitmap_blkoff(inode, + group), + &cache->prev_bitmap, &cache->lock); +} + +/** + * nilfs_palloc_get_entry_block - get buffer head of an entry block + * @inode: inode of metadata file using this allocator + * @nr: serial number of the entry (e.g. inode number) + * @create: create flag + * @bhp: pointer to store the resultant buffer head + * + * Return: 0 on success, or a negative error code on failure. + */ +int nilfs_palloc_get_entry_block(struct inode *inode, __u64 nr, + int create, struct buffer_head **bhp) +{ + struct nilfs_palloc_cache *cache = NILFS_MDT(inode)->mi_palloc_cache; + + return nilfs_palloc_get_block(inode, + nilfs_palloc_entry_blkoff(inode, nr), + create, NULL, bhp, + &cache->prev_entry, &cache->lock); +} + +/** + * nilfs_palloc_delete_entry_block - delete an entry block + * @inode: inode of metadata file using this allocator + * @nr: serial number of the entry + * + * Return: 0 on success, or a negative error code on failure. + */ +static int nilfs_palloc_delete_entry_block(struct inode *inode, __u64 nr) +{ + struct nilfs_palloc_cache *cache = NILFS_MDT(inode)->mi_palloc_cache; + + return nilfs_palloc_delete_block(inode, + nilfs_palloc_entry_blkoff(inode, nr), + &cache->prev_entry, &cache->lock); +} + +/** + * nilfs_palloc_group_desc_offset - calculate the byte offset of a group + * descriptor in the folio containing it + * @inode: inode of metadata file using this allocator + * @group: group number + * @bh: buffer head of the group descriptor block + * + * Return: Byte offset in the folio of the group descriptor for @group. + */ +static size_t nilfs_palloc_group_desc_offset(const struct inode *inode, + unsigned long group, + const struct buffer_head *bh) +{ + return offset_in_folio(bh->b_folio, bh->b_data) + + sizeof(struct nilfs_palloc_group_desc) * + (group % nilfs_palloc_groups_per_desc_block(inode)); +} + +/** + * nilfs_palloc_bitmap_offset - calculate the byte offset of a bitmap block + * in the folio containing it + * @bh: buffer head of the bitmap block + * + * Return: Byte offset in the folio of the bitmap block for @bh. + */ +static size_t nilfs_palloc_bitmap_offset(const struct buffer_head *bh) +{ + return offset_in_folio(bh->b_folio, bh->b_data); +} + +/** + * nilfs_palloc_entry_offset - calculate the byte offset of an entry in the + * folio containing it + * @inode: inode of metadata file using this allocator + * @nr: serial number of the entry (e.g. inode number) + * @bh: buffer head of the entry block + * + * Return: Byte offset in the folio of the entry @nr. + */ +size_t nilfs_palloc_entry_offset(const struct inode *inode, __u64 nr, + const struct buffer_head *bh) +{ + unsigned long entry_index_in_group, entry_index_in_block; + + nilfs_palloc_group(inode, nr, &entry_index_in_group); + entry_index_in_block = entry_index_in_group % + NILFS_MDT(inode)->mi_entries_per_block; + + return offset_in_folio(bh->b_folio, bh->b_data) + + entry_index_in_block * NILFS_MDT(inode)->mi_entry_size; +} + +/** + * nilfs_palloc_find_available_slot - find available slot in a group + * @bitmap: bitmap of the group + * @target: offset number of an entry in the group (start point) + * @bsize: size in bits + * @lock: spin lock protecting @bitmap + * @wrap: whether to wrap around + * + * Return: Offset number within the group of the found free entry, or + * %-ENOSPC if not found. + */ +static int nilfs_palloc_find_available_slot(unsigned char *bitmap, + unsigned long target, + unsigned int bsize, + spinlock_t *lock, bool wrap) +{ + int pos, end = bsize; + + if (likely(target < bsize)) { + pos = target; + do { + pos = nilfs_find_next_zero_bit(bitmap, end, pos); + if (pos >= end) + break; + if (!nilfs_set_bit_atomic(lock, pos, bitmap)) + return pos; + } while (++pos < end); + + end = target; + } + if (!wrap) + return -ENOSPC; + + /* wrap around */ + for (pos = 0; pos < end; pos++) { + pos = nilfs_find_next_zero_bit(bitmap, end, pos); + if (pos >= end) + break; + if (!nilfs_set_bit_atomic(lock, pos, bitmap)) + return pos; + } + + return -ENOSPC; +} + +/** + * nilfs_palloc_rest_groups_in_desc_block - get the remaining number of groups + * in a group descriptor block + * @inode: inode of metadata file using this allocator + * @curr: current group number + * @max: maximum number of groups + * + * Return: Number of remaining descriptors (= groups) managed by the descriptor + * block. + */ +static unsigned long +nilfs_palloc_rest_groups_in_desc_block(const struct inode *inode, + unsigned long curr, unsigned long max) +{ + return min_t(unsigned long, + nilfs_palloc_groups_per_desc_block(inode) - + curr % nilfs_palloc_groups_per_desc_block(inode), + max - curr + 1); +} + +/** + * nilfs_palloc_count_desc_blocks - count descriptor blocks number + * @inode: inode of metadata file using this allocator + * @desc_blocks: descriptor blocks number [out] + * + * Return: 0 on success, or a negative error code on failure. + */ +static int nilfs_palloc_count_desc_blocks(struct inode *inode, + unsigned long *desc_blocks) +{ + __u64 blknum; + int ret; + + ret = nilfs_bmap_last_key(NILFS_I(inode)->i_bmap, &blknum); + if (likely(!ret)) + *desc_blocks = DIV_ROUND_UP( + (unsigned long)blknum, + NILFS_MDT(inode)->mi_blocks_per_desc_block); + return ret; +} + +/** + * nilfs_palloc_mdt_file_can_grow - check potential opportunity for + * MDT file growing + * @inode: inode of metadata file using this allocator + * @desc_blocks: known current descriptor blocks count + * + * Return: true if a group can be added in the metadata file, false if not. + */ +static inline bool nilfs_palloc_mdt_file_can_grow(struct inode *inode, + unsigned long desc_blocks) +{ + return (nilfs_palloc_groups_per_desc_block(inode) * desc_blocks) < + nilfs_palloc_groups_count(inode); +} + +/** + * nilfs_palloc_count_max_entries - count max number of entries that can be + * described by descriptor blocks count + * @inode: inode of metadata file using this allocator + * @nused: current number of used entries + * @nmaxp: max number of entries [out] + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOMEM - Insufficient memory available. + * * %-ERANGE - Number of entries in use is out of range. + */ +int nilfs_palloc_count_max_entries(struct inode *inode, u64 nused, u64 *nmaxp) +{ + unsigned long desc_blocks = 0; + u64 entries_per_desc_block, nmax; + int err; + + err = nilfs_palloc_count_desc_blocks(inode, &desc_blocks); + if (unlikely(err)) + return err; + + entries_per_desc_block = (u64)nilfs_palloc_entries_per_group(inode) * + nilfs_palloc_groups_per_desc_block(inode); + nmax = entries_per_desc_block * desc_blocks; + + if (nused == nmax && + nilfs_palloc_mdt_file_can_grow(inode, desc_blocks)) + nmax += entries_per_desc_block; + + if (nused > nmax) + return -ERANGE; + + *nmaxp = nmax; + return 0; +} + +/** + * nilfs_palloc_prepare_alloc_entry - prepare to allocate a persistent object + * @inode: inode of metadata file using this allocator + * @req: nilfs_palloc_req structure exchanged for the allocation + * @wrap: whether to wrap around + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOMEM - Insufficient memory available. + * * %-ENOSPC - Entries exhausted (No entries available for allocation). + * * %-EROFS - Read only filesystem + */ +int nilfs_palloc_prepare_alloc_entry(struct inode *inode, + struct nilfs_palloc_req *req, bool wrap) +{ + struct buffer_head *desc_bh, *bitmap_bh; + struct nilfs_palloc_group_desc *desc; + unsigned char *bitmap; + size_t doff, boff; + unsigned long group, maxgroup, ngroups; + unsigned long group_offset, maxgroup_offset; + unsigned long n, entries_per_group; + unsigned long i, j; + spinlock_t *lock; + int pos, ret; + + ngroups = nilfs_palloc_groups_count(inode); + maxgroup = ngroups - 1; + group = nilfs_palloc_group(inode, req->pr_entry_nr, &group_offset); + entries_per_group = nilfs_palloc_entries_per_group(inode); + + for (i = 0; i < ngroups; i += n) { + if (group >= ngroups && wrap) { + /* wrap around */ + group = 0; + maxgroup = nilfs_palloc_group(inode, req->pr_entry_nr, + &maxgroup_offset) - 1; + } + ret = nilfs_palloc_get_desc_block(inode, group, 1, &desc_bh); + if (ret < 0) + return ret; + + doff = nilfs_palloc_group_desc_offset(inode, group, desc_bh); + desc = kmap_local_folio(desc_bh->b_folio, doff); + n = nilfs_palloc_rest_groups_in_desc_block(inode, group, + maxgroup); + for (j = 0; j < n; j++, group++, group_offset = 0) { + lock = nilfs_mdt_bgl_lock(inode, group); + if (nilfs_palloc_group_desc_nfrees(&desc[j], lock) == 0) + continue; + + kunmap_local(desc); + ret = nilfs_palloc_get_bitmap_block(inode, group, 1, + &bitmap_bh); + if (unlikely(ret < 0)) { + brelse(desc_bh); + return ret; + } + + /* + * Re-kmap the folio containing the first (and + * subsequent) group descriptors. + */ + desc = kmap_local_folio(desc_bh->b_folio, doff); + + boff = nilfs_palloc_bitmap_offset(bitmap_bh); + bitmap = kmap_local_folio(bitmap_bh->b_folio, boff); + pos = nilfs_palloc_find_available_slot( + bitmap, group_offset, entries_per_group, lock, + wrap); + /* + * Since the search for a free slot in the second and + * subsequent bitmap blocks always starts from the + * beginning, the wrap flag only has an effect on the + * first search. + */ + kunmap_local(bitmap); + if (pos >= 0) + goto found; + + brelse(bitmap_bh); + } + + kunmap_local(desc); + brelse(desc_bh); + } + + /* no entries left */ + return -ENOSPC; + +found: + /* found a free entry */ + nilfs_palloc_group_desc_add_entries(&desc[j], lock, -1); + req->pr_entry_nr = entries_per_group * group + pos; + kunmap_local(desc); + + req->pr_desc_bh = desc_bh; + req->pr_bitmap_bh = bitmap_bh; + return 0; +} + +/** + * nilfs_palloc_commit_alloc_entry - finish allocation of a persistent object + * @inode: inode of metadata file using this allocator + * @req: nilfs_palloc_req structure exchanged for the allocation + */ +void nilfs_palloc_commit_alloc_entry(struct inode *inode, + struct nilfs_palloc_req *req) +{ + mark_buffer_dirty(req->pr_bitmap_bh); + mark_buffer_dirty(req->pr_desc_bh); + nilfs_mdt_mark_dirty(inode); + + brelse(req->pr_bitmap_bh); + brelse(req->pr_desc_bh); +} + +/** + * nilfs_palloc_commit_free_entry - finish deallocating a persistent object + * @inode: inode of metadata file using this allocator + * @req: nilfs_palloc_req structure exchanged for the removal + */ +void nilfs_palloc_commit_free_entry(struct inode *inode, + struct nilfs_palloc_req *req) +{ + unsigned long group, group_offset; + size_t doff, boff; + struct nilfs_palloc_group_desc *desc; + unsigned char *bitmap; + spinlock_t *lock; + + group = nilfs_palloc_group(inode, req->pr_entry_nr, &group_offset); + doff = nilfs_palloc_group_desc_offset(inode, group, req->pr_desc_bh); + desc = kmap_local_folio(req->pr_desc_bh->b_folio, doff); + + boff = nilfs_palloc_bitmap_offset(req->pr_bitmap_bh); + bitmap = kmap_local_folio(req->pr_bitmap_bh->b_folio, boff); + lock = nilfs_mdt_bgl_lock(inode, group); + + if (!nilfs_clear_bit_atomic(lock, group_offset, bitmap)) + nilfs_warn(inode->i_sb, + "%s (ino=%llu): entry number %llu already freed", + __func__, inode->i_ino, + (unsigned long long)req->pr_entry_nr); + else + nilfs_palloc_group_desc_add_entries(desc, lock, 1); + + kunmap_local(bitmap); + kunmap_local(desc); + + mark_buffer_dirty(req->pr_desc_bh); + mark_buffer_dirty(req->pr_bitmap_bh); + nilfs_mdt_mark_dirty(inode); + + brelse(req->pr_bitmap_bh); + brelse(req->pr_desc_bh); +} + +/** + * nilfs_palloc_abort_alloc_entry - cancel allocation of a persistent object + * @inode: inode of metadata file using this allocator + * @req: nilfs_palloc_req structure exchanged for the allocation + */ +void nilfs_palloc_abort_alloc_entry(struct inode *inode, + struct nilfs_palloc_req *req) +{ + struct nilfs_palloc_group_desc *desc; + size_t doff, boff; + unsigned char *bitmap; + unsigned long group, group_offset; + spinlock_t *lock; + + group = nilfs_palloc_group(inode, req->pr_entry_nr, &group_offset); + doff = nilfs_palloc_group_desc_offset(inode, group, req->pr_desc_bh); + desc = kmap_local_folio(req->pr_desc_bh->b_folio, doff); + + boff = nilfs_palloc_bitmap_offset(req->pr_bitmap_bh); + bitmap = kmap_local_folio(req->pr_bitmap_bh->b_folio, boff); + lock = nilfs_mdt_bgl_lock(inode, group); + + if (!nilfs_clear_bit_atomic(lock, group_offset, bitmap)) + nilfs_warn(inode->i_sb, + "%s (ino=%llu): entry number %llu already freed", + __func__, inode->i_ino, + (unsigned long long)req->pr_entry_nr); + else + nilfs_palloc_group_desc_add_entries(desc, lock, 1); + + kunmap_local(bitmap); + kunmap_local(desc); + + brelse(req->pr_bitmap_bh); + brelse(req->pr_desc_bh); + + req->pr_entry_nr = 0; + req->pr_bitmap_bh = NULL; + req->pr_desc_bh = NULL; +} + +/** + * nilfs_palloc_prepare_free_entry - prepare to deallocate a persistent object + * @inode: inode of metadata file using this allocator + * @req: nilfs_palloc_req structure exchanged for the removal + * + * Return: 0 on success, or a negative error code on failure. + */ +int nilfs_palloc_prepare_free_entry(struct inode *inode, + struct nilfs_palloc_req *req) +{ + struct buffer_head *desc_bh, *bitmap_bh; + unsigned long group, group_offset; + int ret; + + group = nilfs_palloc_group(inode, req->pr_entry_nr, &group_offset); + ret = nilfs_palloc_get_desc_block(inode, group, 1, &desc_bh); + if (ret < 0) + return ret; + ret = nilfs_palloc_get_bitmap_block(inode, group, 1, &bitmap_bh); + if (ret < 0) { + brelse(desc_bh); + return ret; + } + + req->pr_desc_bh = desc_bh; + req->pr_bitmap_bh = bitmap_bh; + return 0; +} + +/** + * nilfs_palloc_abort_free_entry - cancel deallocating a persistent object + * @inode: inode of metadata file using this allocator + * @req: nilfs_palloc_req structure exchanged for the removal + */ +void nilfs_palloc_abort_free_entry(struct inode *inode, + struct nilfs_palloc_req *req) +{ + brelse(req->pr_bitmap_bh); + brelse(req->pr_desc_bh); + + req->pr_entry_nr = 0; + req->pr_bitmap_bh = NULL; + req->pr_desc_bh = NULL; +} + +/** + * nilfs_palloc_freev - deallocate a set of persistent objects + * @inode: inode of metadata file using this allocator + * @entry_nrs: array of entry numbers to be deallocated + * @nitems: number of entries stored in @entry_nrs + * + * Return: 0 on success, or a negative error code on failure. + */ +int nilfs_palloc_freev(struct inode *inode, __u64 *entry_nrs, size_t nitems) +{ + struct buffer_head *desc_bh, *bitmap_bh; + struct nilfs_palloc_group_desc *desc; + unsigned char *bitmap; + size_t doff, boff; + unsigned long group, group_offset; + __u64 group_min_nr, last_nrs[8]; + const unsigned long epg = nilfs_palloc_entries_per_group(inode); + const unsigned int epb = NILFS_MDT(inode)->mi_entries_per_block; + unsigned int entry_start, end, pos; + spinlock_t *lock; + int i, j, k, ret; + u32 nfree; + + for (i = 0; i < nitems; i = j) { + int change_group = false; + int nempties = 0, n = 0; + + group = nilfs_palloc_group(inode, entry_nrs[i], &group_offset); + ret = nilfs_palloc_get_desc_block(inode, group, 0, &desc_bh); + if (ret < 0) + return ret; + ret = nilfs_palloc_get_bitmap_block(inode, group, 0, + &bitmap_bh); + if (ret < 0) { + brelse(desc_bh); + return ret; + } + + /* Get the first entry number of the group */ + group_min_nr = (__u64)group * epg; + + boff = nilfs_palloc_bitmap_offset(bitmap_bh); + bitmap = kmap_local_folio(bitmap_bh->b_folio, boff); + lock = nilfs_mdt_bgl_lock(inode, group); + + j = i; + entry_start = rounddown(group_offset, epb); + do { + if (!nilfs_clear_bit_atomic(lock, group_offset, + bitmap)) { + nilfs_warn(inode->i_sb, + "%s (ino=%llu): entry number %llu already freed", + __func__, inode->i_ino, + (unsigned long long)entry_nrs[j]); + } else { + n++; + } + + j++; + if (j >= nitems || entry_nrs[j] < group_min_nr || + entry_nrs[j] >= group_min_nr + epg) { + change_group = true; + } else { + group_offset = entry_nrs[j] - group_min_nr; + if (group_offset >= entry_start && + group_offset < entry_start + epb) { + /* This entry is in the same block */ + continue; + } + } + + /* Test if the entry block is empty or not */ + end = entry_start + epb; + pos = nilfs_find_next_bit(bitmap, end, entry_start); + if (pos >= end) { + last_nrs[nempties++] = entry_nrs[j - 1]; + if (nempties >= ARRAY_SIZE(last_nrs)) + break; + } + + if (change_group) + break; + + /* Go on to the next entry block */ + entry_start = rounddown(group_offset, epb); + } while (true); + + kunmap_local(bitmap); + mark_buffer_dirty(bitmap_bh); + brelse(bitmap_bh); + + for (k = 0; k < nempties; k++) { + ret = nilfs_palloc_delete_entry_block(inode, + last_nrs[k]); + if (ret && ret != -ENOENT) + nilfs_warn(inode->i_sb, + "error %d deleting block that object (entry=%llu, ino=%llu) belongs to", + ret, (unsigned long long)last_nrs[k], + inode->i_ino); + } + + doff = nilfs_palloc_group_desc_offset(inode, group, desc_bh); + desc = kmap_local_folio(desc_bh->b_folio, doff); + nfree = nilfs_palloc_group_desc_add_entries(desc, lock, n); + kunmap_local(desc); + mark_buffer_dirty(desc_bh); + nilfs_mdt_mark_dirty(inode); + brelse(desc_bh); + + if (nfree == nilfs_palloc_entries_per_group(inode)) { + ret = nilfs_palloc_delete_bitmap_block(inode, group); + if (ret && ret != -ENOENT) + nilfs_warn(inode->i_sb, + "error %d deleting bitmap block of group=%lu, ino=%llu", + ret, group, inode->i_ino); + } + } + return 0; +} + +void nilfs_palloc_setup_cache(struct inode *inode, + struct nilfs_palloc_cache *cache) +{ + NILFS_MDT(inode)->mi_palloc_cache = cache; + spin_lock_init(&cache->lock); +} + +void nilfs_palloc_clear_cache(struct inode *inode) +{ + struct nilfs_palloc_cache *cache = NILFS_MDT(inode)->mi_palloc_cache; + + spin_lock(&cache->lock); + brelse(cache->prev_desc.bh); + brelse(cache->prev_bitmap.bh); + brelse(cache->prev_entry.bh); + cache->prev_desc.bh = NULL; + cache->prev_bitmap.bh = NULL; + cache->prev_entry.bh = NULL; + spin_unlock(&cache->lock); +} + +void nilfs_palloc_destroy_cache(struct inode *inode) +{ + nilfs_palloc_clear_cache(inode); + NILFS_MDT(inode)->mi_palloc_cache = NULL; +} diff --git a/fs/nilfs2/alloc.h b/fs/nilfs2/alloc.h new file mode 100644 index 000000000..046d876ea --- /dev/null +++ b/fs/nilfs2/alloc.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Persistent object (dat entry/disk inode) allocator/deallocator + * + * Copyright (C) 2006-2008 Nippon Telegraph and Telephone Corporation. + * + * Originally written by Koji Sato. + * Two allocators were unified by Ryusuke Konishi and Amagai Yoshiji. + */ + +#ifndef _NILFS_ALLOC_H +#define _NILFS_ALLOC_H + +#include +#include +#include + +/** + * nilfs_palloc_entries_per_group - get the number of entries per group + * @inode: inode of metadata file using this allocator + * + * The number of entries per group is defined by the number of bits + * that a bitmap block can maintain. + * + * Return: Number of entries per group. + */ +static inline unsigned long +nilfs_palloc_entries_per_group(const struct inode *inode) +{ + return 1UL << (inode->i_blkbits + 3 /* log2(8 = CHAR_BITS) */); +} + +int nilfs_palloc_init_blockgroup(struct inode *, unsigned int); +int nilfs_palloc_get_entry_block(struct inode *, __u64, int, + struct buffer_head **); +size_t nilfs_palloc_entry_offset(const struct inode *inode, __u64 nr, + const struct buffer_head *bh); + +int nilfs_palloc_count_max_entries(struct inode *, u64, u64 *); + +/** + * struct nilfs_palloc_req - persistent allocator request and reply + * @pr_entry_nr: entry number (vblocknr or inode number) + * @pr_desc_bh: buffer head of the buffer containing block group descriptors + * @pr_bitmap_bh: buffer head of the buffer containing a block group bitmap + * @pr_entry_bh: buffer head of the buffer containing translation entries + */ +struct nilfs_palloc_req { + __u64 pr_entry_nr; + struct buffer_head *pr_desc_bh; + struct buffer_head *pr_bitmap_bh; + struct buffer_head *pr_entry_bh; +}; + +int nilfs_palloc_prepare_alloc_entry(struct inode *inode, + struct nilfs_palloc_req *req, bool wrap); +void nilfs_palloc_commit_alloc_entry(struct inode *, + struct nilfs_palloc_req *); +void nilfs_palloc_abort_alloc_entry(struct inode *, struct nilfs_palloc_req *); +void nilfs_palloc_commit_free_entry(struct inode *, struct nilfs_palloc_req *); +int nilfs_palloc_prepare_free_entry(struct inode *, struct nilfs_palloc_req *); +void nilfs_palloc_abort_free_entry(struct inode *, struct nilfs_palloc_req *); +int nilfs_palloc_freev(struct inode *, __u64 *, size_t); + +#define nilfs_set_bit_atomic ext2_set_bit_atomic +#define nilfs_clear_bit_atomic ext2_clear_bit_atomic +#define nilfs_find_next_zero_bit find_next_zero_bit_le +#define nilfs_find_next_bit find_next_bit_le + +/** + * struct nilfs_bh_assoc - block offset and buffer head association + * @blkoff: block offset + * @bh: buffer head + */ +struct nilfs_bh_assoc { + unsigned long blkoff; + struct buffer_head *bh; +}; + +/** + * struct nilfs_palloc_cache - persistent object allocator cache + * @lock: cache protecting lock + * @prev_desc: blockgroup descriptors cache + * @prev_bitmap: blockgroup bitmap cache + * @prev_entry: translation entries cache + */ +struct nilfs_palloc_cache { + spinlock_t lock; + struct nilfs_bh_assoc prev_desc; + struct nilfs_bh_assoc prev_bitmap; + struct nilfs_bh_assoc prev_entry; +}; + +void nilfs_palloc_setup_cache(struct inode *inode, + struct nilfs_palloc_cache *cache); +void nilfs_palloc_clear_cache(struct inode *inode); +void nilfs_palloc_destroy_cache(struct inode *inode); + +#endif /* _NILFS_ALLOC_H */ diff --git a/fs/nilfs2/bmap.c b/fs/nilfs2/bmap.c new file mode 100644 index 000000000..83f6ea30c --- /dev/null +++ b/fs/nilfs2/bmap.c @@ -0,0 +1,568 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * NILFS block mapping. + * + * Copyright (C) 2006-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Koji Sato. + */ + +#include +#include +#include +#include "nilfs.h" +#include "bmap.h" +#include "btree.h" +#include "direct.h" +#include "btnode.h" +#include "mdt.h" +#include "dat.h" +#include "alloc.h" + +struct inode *nilfs_bmap_get_dat(const struct nilfs_bmap *bmap) +{ + struct the_nilfs *nilfs = bmap->b_inode->i_sb->s_fs_info; + + return nilfs->ns_dat; +} + +static int nilfs_bmap_convert_error(struct nilfs_bmap *bmap, + const char *fname, int err) +{ + struct inode *inode = bmap->b_inode; + + if (err == -EINVAL) { + __nilfs_error(inode->i_sb, fname, + "broken bmap (inode number=%llu)", inode->i_ino); + err = -EIO; + } + return err; +} + +/** + * nilfs_bmap_lookup_at_level - find a data block or node block + * @bmap: bmap + * @key: key + * @level: level + * @ptrp: place to store the value associated to @key + * + * Description: nilfs_bmap_lookup_at_level() finds a record whose key + * matches @key in the block at @level of the bmap. The record associated + * with @key is stored in the place pointed to by @ptrp. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOENT - A record associated with @key does not exist. + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_bmap_lookup_at_level(struct nilfs_bmap *bmap, __u64 key, int level, + __u64 *ptrp) +{ + sector_t blocknr; + int ret; + + down_read(&bmap->b_sem); + ret = bmap->b_ops->bop_lookup(bmap, key, level, ptrp); + if (ret < 0) + goto out; + + if (NILFS_BMAP_USE_VBN(bmap)) { + ret = nilfs_dat_translate(nilfs_bmap_get_dat(bmap), *ptrp, + &blocknr); + if (!ret) + *ptrp = blocknr; + else if (ret == -ENOENT) { + /* + * If there was no valid entry in DAT for the block + * address obtained by b_ops->bop_lookup, then pass + * internal code -EINVAL to nilfs_bmap_convert_error + * to treat it as metadata corruption. + */ + ret = -EINVAL; + } + } + + out: + up_read(&bmap->b_sem); + return nilfs_bmap_convert_error(bmap, __func__, ret); +} + +int nilfs_bmap_lookup_contig(struct nilfs_bmap *bmap, __u64 key, __u64 *ptrp, + unsigned int maxblocks) +{ + int ret; + + down_read(&bmap->b_sem); + ret = bmap->b_ops->bop_lookup_contig(bmap, key, ptrp, maxblocks); + up_read(&bmap->b_sem); + + return nilfs_bmap_convert_error(bmap, __func__, ret); +} + +static int nilfs_bmap_do_insert(struct nilfs_bmap *bmap, __u64 key, __u64 ptr) +{ + __u64 keys[NILFS_BMAP_SMALL_HIGH + 1]; + __u64 ptrs[NILFS_BMAP_SMALL_HIGH + 1]; + int ret, n; + + if (bmap->b_ops->bop_check_insert != NULL) { + ret = bmap->b_ops->bop_check_insert(bmap, key); + if (ret > 0) { + n = bmap->b_ops->bop_gather_data( + bmap, keys, ptrs, NILFS_BMAP_SMALL_HIGH + 1); + if (n < 0) + return n; + ret = nilfs_btree_convert_and_insert( + bmap, key, ptr, keys, ptrs, n); + if (ret == 0) + bmap->b_u.u_flags |= NILFS_BMAP_LARGE; + + return ret; + } else if (ret < 0) + return ret; + } + + return bmap->b_ops->bop_insert(bmap, key, ptr); +} + +/** + * nilfs_bmap_insert - insert a new key-record pair into a bmap + * @bmap: bmap + * @key: key + * @rec: record + * + * Description: nilfs_bmap_insert() inserts the new key-record pair specified + * by @key and @rec into @bmap. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EEXIST - A record associated with @key already exists. + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_bmap_insert(struct nilfs_bmap *bmap, __u64 key, unsigned long rec) +{ + int ret; + + down_write(&bmap->b_sem); + ret = nilfs_bmap_do_insert(bmap, key, rec); + up_write(&bmap->b_sem); + + return nilfs_bmap_convert_error(bmap, __func__, ret); +} + +static int nilfs_bmap_do_delete(struct nilfs_bmap *bmap, __u64 key) +{ + __u64 keys[NILFS_BMAP_LARGE_LOW + 1]; + __u64 ptrs[NILFS_BMAP_LARGE_LOW + 1]; + int ret, n; + + if (bmap->b_ops->bop_check_delete != NULL) { + ret = bmap->b_ops->bop_check_delete(bmap, key); + if (ret > 0) { + n = bmap->b_ops->bop_gather_data( + bmap, keys, ptrs, NILFS_BMAP_LARGE_LOW + 1); + if (n < 0) + return n; + ret = nilfs_direct_delete_and_convert( + bmap, key, keys, ptrs, n); + if (ret == 0) + bmap->b_u.u_flags &= ~NILFS_BMAP_LARGE; + + return ret; + } else if (ret < 0) + return ret; + } + + return bmap->b_ops->bop_delete(bmap, key, false); +} + +/** + * nilfs_bmap_seek_key - seek a valid entry and return its key + * @bmap: bmap struct + * @start: start key number + * @keyp: place to store valid key + * + * Description: nilfs_bmap_seek_key() seeks a valid key on @bmap + * starting from @start, and stores it to @keyp if found. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOENT - No valid entry was found. + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_bmap_seek_key(struct nilfs_bmap *bmap, __u64 start, __u64 *keyp) +{ + int ret; + + down_read(&bmap->b_sem); + ret = bmap->b_ops->bop_seek_key(bmap, start, keyp); + up_read(&bmap->b_sem); + + if (ret < 0) + ret = nilfs_bmap_convert_error(bmap, __func__, ret); + return ret; +} + +int nilfs_bmap_last_key(struct nilfs_bmap *bmap, __u64 *keyp) +{ + int ret; + + down_read(&bmap->b_sem); + ret = bmap->b_ops->bop_last_key(bmap, keyp); + up_read(&bmap->b_sem); + + if (ret < 0) + ret = nilfs_bmap_convert_error(bmap, __func__, ret); + return ret; +} + +/** + * nilfs_bmap_delete - delete a key-record pair from a bmap + * @bmap: bmap + * @key: key + * + * Description: nilfs_bmap_delete() deletes the key-record pair specified by + * @key from @bmap. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOENT - A record associated with @key does not exist. + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_bmap_delete(struct nilfs_bmap *bmap, __u64 key) +{ + int ret; + + down_write(&bmap->b_sem); + ret = nilfs_bmap_do_delete(bmap, key); + up_write(&bmap->b_sem); + + return nilfs_bmap_convert_error(bmap, __func__, ret); +} + +static int nilfs_bmap_do_truncate(struct nilfs_bmap *bmap, __u64 key) +{ + __u64 lastkey; + int ret; + + ret = bmap->b_ops->bop_last_key(bmap, &lastkey); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + return ret; + } + + while (key <= lastkey) { + ret = nilfs_bmap_do_delete(bmap, lastkey); + if (ret < 0) + return ret; + ret = bmap->b_ops->bop_last_key(bmap, &lastkey); + if (ret < 0) { + if (ret == -ENOENT) + ret = 0; + return ret; + } + } + return 0; +} + +/** + * nilfs_bmap_truncate - truncate a bmap to a specified key + * @bmap: bmap + * @key: key + * + * Description: nilfs_bmap_truncate() removes key-record pairs whose keys are + * greater than or equal to @key from @bmap. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_bmap_truncate(struct nilfs_bmap *bmap, __u64 key) +{ + int ret; + + down_write(&bmap->b_sem); + ret = nilfs_bmap_do_truncate(bmap, key); + up_write(&bmap->b_sem); + + return nilfs_bmap_convert_error(bmap, __func__, ret); +} + +/** + * nilfs_bmap_clear - free resources a bmap holds + * @bmap: bmap + * + * Description: nilfs_bmap_clear() frees resources associated with @bmap. + */ +void nilfs_bmap_clear(struct nilfs_bmap *bmap) +{ + down_write(&bmap->b_sem); + if (bmap->b_ops->bop_clear != NULL) + bmap->b_ops->bop_clear(bmap); + up_write(&bmap->b_sem); +} + +/** + * nilfs_bmap_propagate - propagate dirty state + * @bmap: bmap + * @bh: buffer head + * + * Description: nilfs_bmap_propagate() marks the buffers that directly or + * indirectly refer to the block specified by @bh dirty. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_bmap_propagate(struct nilfs_bmap *bmap, struct buffer_head *bh) +{ + int ret; + + down_write(&bmap->b_sem); + ret = bmap->b_ops->bop_propagate(bmap, bh); + up_write(&bmap->b_sem); + + return nilfs_bmap_convert_error(bmap, __func__, ret); +} + +/** + * nilfs_bmap_lookup_dirty_buffers - collect dirty block buffers + * @bmap: bmap + * @listp: pointer to buffer head list + */ +void nilfs_bmap_lookup_dirty_buffers(struct nilfs_bmap *bmap, + struct list_head *listp) +{ + if (bmap->b_ops->bop_lookup_dirty_buffers != NULL) + bmap->b_ops->bop_lookup_dirty_buffers(bmap, listp); +} + +/** + * nilfs_bmap_assign - assign a new block number to a block + * @bmap: bmap + * @bh: place to store a pointer to the buffer head to which a block + * address is assigned (in/out) + * @blocknr: block number + * @binfo: block information + * + * Description: nilfs_bmap_assign() assigns the block number @blocknr to the + * buffer specified by @bh. The block information is stored in the memory + * pointed to by @binfo, and the buffer head may be replaced as a block + * address is assigned, in which case a pointer to the new buffer head is + * stored in the memory pointed to by @bh. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_bmap_assign(struct nilfs_bmap *bmap, + struct buffer_head **bh, + unsigned long blocknr, + union nilfs_binfo *binfo) +{ + int ret; + + down_write(&bmap->b_sem); + ret = bmap->b_ops->bop_assign(bmap, bh, blocknr, binfo); + up_write(&bmap->b_sem); + + return nilfs_bmap_convert_error(bmap, __func__, ret); +} + +/** + * nilfs_bmap_mark - mark block dirty + * @bmap: bmap + * @key: key + * @level: level + * + * Description: nilfs_bmap_mark() marks the block specified by @key and @level + * as dirty. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_bmap_mark(struct nilfs_bmap *bmap, __u64 key, int level) +{ + int ret; + + if (bmap->b_ops->bop_mark == NULL) + return 0; + + down_write(&bmap->b_sem); + ret = bmap->b_ops->bop_mark(bmap, key, level); + up_write(&bmap->b_sem); + + return nilfs_bmap_convert_error(bmap, __func__, ret); +} + +/** + * nilfs_bmap_test_and_clear_dirty - test and clear a bmap dirty state + * @bmap: bmap + * + * Description: nilfs_test_and_clear() is the atomic operation to test and + * clear the dirty state of @bmap. + * + * Return: 1 if @bmap is dirty, or 0 if clear. + */ +int nilfs_bmap_test_and_clear_dirty(struct nilfs_bmap *bmap) +{ + int ret; + + down_write(&bmap->b_sem); + ret = nilfs_bmap_dirty(bmap); + nilfs_bmap_clear_dirty(bmap); + up_write(&bmap->b_sem); + return ret; +} + + +/* + * Internal use only + */ +__u64 nilfs_bmap_data_get_key(const struct nilfs_bmap *bmap, + const struct buffer_head *bh) +{ + loff_t pos = folio_pos(bh->b_folio) + bh_offset(bh); + + return pos >> bmap->b_inode->i_blkbits; +} + +__u64 nilfs_bmap_find_target_seq(const struct nilfs_bmap *bmap, __u64 key) +{ + __s64 diff; + + diff = key - bmap->b_last_allocated_key; + if ((nilfs_bmap_keydiff_abs(diff) < NILFS_INODE_BMAP_SIZE) && + (bmap->b_last_allocated_ptr != NILFS_BMAP_INVALID_PTR) && + (bmap->b_last_allocated_ptr + diff > 0)) + return bmap->b_last_allocated_ptr + diff; + else + return NILFS_BMAP_INVALID_PTR; +} + +#define NILFS_BMAP_GROUP_DIV 8 /* must be power of 2 */ + +__u64 nilfs_bmap_find_target_in_group(const struct nilfs_bmap *bmap) +{ + struct inode *dat = nilfs_bmap_get_dat(bmap); + unsigned long entries_per_group = nilfs_palloc_entries_per_group(dat); + unsigned long group; + u32 index; + + BUILD_BUG_ON_NOT_POWER_OF_2(NILFS_BMAP_GROUP_DIV); + + group = div_u64(bmap->b_inode->i_ino, entries_per_group); + index = bmap->b_inode->i_ino & (NILFS_BMAP_GROUP_DIV - 1); + + return group * entries_per_group + + index * (entries_per_group / NILFS_BMAP_GROUP_DIV); +} + + +static struct lock_class_key nilfs_bmap_dat_lock_key; +static struct lock_class_key nilfs_bmap_mdt_lock_key; + +/** + * nilfs_bmap_read - read a bmap from an inode + * @bmap: bmap + * @raw_inode: on-disk inode + * + * Description: nilfs_bmap_read() initializes the bmap @bmap. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (corrupted bmap). + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_bmap_read(struct nilfs_bmap *bmap, struct nilfs_inode *raw_inode) +{ + if (raw_inode == NULL) + memset(bmap->b_u.u_data, 0, NILFS_BMAP_SIZE); + else + memcpy(bmap->b_u.u_data, raw_inode->i_bmap, NILFS_BMAP_SIZE); + + init_rwsem(&bmap->b_sem); + bmap->b_state = 0; + bmap->b_inode = &NILFS_BMAP_I(bmap)->vfs_inode; + switch (bmap->b_inode->i_ino) { + case NILFS_DAT_INO: + bmap->b_ptr_type = NILFS_BMAP_PTR_P; + bmap->b_last_allocated_key = 0; + bmap->b_last_allocated_ptr = NILFS_BMAP_NEW_PTR_INIT; + lockdep_set_class(&bmap->b_sem, &nilfs_bmap_dat_lock_key); + break; + case NILFS_CPFILE_INO: + case NILFS_SUFILE_INO: + bmap->b_ptr_type = NILFS_BMAP_PTR_VS; + bmap->b_last_allocated_key = 0; + bmap->b_last_allocated_ptr = NILFS_BMAP_INVALID_PTR; + lockdep_set_class(&bmap->b_sem, &nilfs_bmap_mdt_lock_key); + break; + case NILFS_IFILE_INO: + lockdep_set_class(&bmap->b_sem, &nilfs_bmap_mdt_lock_key); + fallthrough; + default: + bmap->b_ptr_type = NILFS_BMAP_PTR_VM; + bmap->b_last_allocated_key = 0; + bmap->b_last_allocated_ptr = NILFS_BMAP_INVALID_PTR; + break; + } + + return (bmap->b_u.u_flags & NILFS_BMAP_LARGE) ? + nilfs_btree_init(bmap) : nilfs_direct_init(bmap); +} + +/** + * nilfs_bmap_write - write back a bmap to an inode + * @bmap: bmap + * @raw_inode: on-disk inode + * + * Description: nilfs_bmap_write() stores @bmap in @raw_inode. + */ +void nilfs_bmap_write(struct nilfs_bmap *bmap, struct nilfs_inode *raw_inode) +{ + memcpy(raw_inode->i_bmap, bmap->b_u.u_data, + NILFS_INODE_BMAP_SIZE * sizeof(__le64)); + if (bmap->b_inode->i_ino == NILFS_DAT_INO) + bmap->b_last_allocated_ptr = NILFS_BMAP_NEW_PTR_INIT; +} + +void nilfs_bmap_init_gc(struct nilfs_bmap *bmap) +{ + memset(&bmap->b_u, 0, NILFS_BMAP_SIZE); + init_rwsem(&bmap->b_sem); + bmap->b_inode = &NILFS_BMAP_I(bmap)->vfs_inode; + bmap->b_ptr_type = NILFS_BMAP_PTR_U; + bmap->b_last_allocated_key = 0; + bmap->b_last_allocated_ptr = NILFS_BMAP_INVALID_PTR; + bmap->b_state = 0; + nilfs_btree_init_gc(bmap); +} + +void nilfs_bmap_save(const struct nilfs_bmap *bmap, + struct nilfs_bmap_store *store) +{ + memcpy(store->data, bmap->b_u.u_data, sizeof(store->data)); + store->last_allocated_key = bmap->b_last_allocated_key; + store->last_allocated_ptr = bmap->b_last_allocated_ptr; + store->state = bmap->b_state; +} + +void nilfs_bmap_restore(struct nilfs_bmap *bmap, + const struct nilfs_bmap_store *store) +{ + memcpy(bmap->b_u.u_data, store->data, sizeof(store->data)); + bmap->b_last_allocated_key = store->last_allocated_key; + bmap->b_last_allocated_ptr = store->last_allocated_ptr; + bmap->b_state = store->state; +} diff --git a/fs/nilfs2/bmap.h b/fs/nilfs2/bmap.h new file mode 100644 index 000000000..a72f3c308 --- /dev/null +++ b/fs/nilfs2/bmap.h @@ -0,0 +1,283 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * NILFS block mapping. + * + * Copyright (C) 2006-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Koji Sato. + */ + +#ifndef _NILFS_BMAP_H +#define _NILFS_BMAP_H + +#include +#include +#include +#include /* nilfs_binfo, nilfs_inode, etc */ +#include "alloc.h" +#include "dat.h" + +#define NILFS_BMAP_INVALID_PTR 0 + +#define nilfs_bmap_keydiff_abs(diff) ((diff) < 0 ? -(diff) : (diff)) + + +struct nilfs_bmap; + +/** + * union nilfs_bmap_ptr_req - request for bmap ptr + * @bpr_ptr: bmap pointer + * @bpr_req: request for persistent allocator + */ +union nilfs_bmap_ptr_req { + __u64 bpr_ptr; + struct nilfs_palloc_req bpr_req; +}; + +/** + * struct nilfs_bmap_stats - bmap statistics + * @bs_nblocks: number of blocks created or deleted + */ +struct nilfs_bmap_stats { + unsigned int bs_nblocks; +}; + +/** + * struct nilfs_bmap_operations - bmap operation table + * @bop_lookup: single block search operation + * @bop_lookup_contig: consecutive block search operation + * @bop_insert: block insertion operation + * @bop_delete: block delete operation + * @bop_clear: block mapping resource release operation + * @bop_propagate: operation to propagate dirty state towards the + * mapping root + * @bop_lookup_dirty_buffers: operation to collect dirty block buffers + * @bop_assign: disk block address assignment operation + * @bop_mark: operation to mark in-use blocks as dirty for + * relocation by GC + * @bop_seek_key: find valid block key operation + * @bop_last_key: find last valid block key operation + */ +struct nilfs_bmap_operations { + int (*bop_lookup)(const struct nilfs_bmap *, __u64, int, __u64 *); + int (*bop_lookup_contig)(const struct nilfs_bmap *, __u64, __u64 *, + unsigned int); + int (*bop_insert)(struct nilfs_bmap *, __u64, __u64); + int (*bop_delete)(struct nilfs_bmap *bmap, __u64 key, bool deform); + void (*bop_clear)(struct nilfs_bmap *); + + int (*bop_propagate)(struct nilfs_bmap *, struct buffer_head *); + void (*bop_lookup_dirty_buffers)(struct nilfs_bmap *, + struct list_head *); + + int (*bop_assign)(struct nilfs_bmap *, + struct buffer_head **, + sector_t, + union nilfs_binfo *); + int (*bop_mark)(struct nilfs_bmap *, __u64, int); + + int (*bop_seek_key)(const struct nilfs_bmap *, __u64, __u64 *); + int (*bop_last_key)(const struct nilfs_bmap *, __u64 *); + + /* private: internal use only */ + int (*bop_check_insert)(const struct nilfs_bmap *, __u64); + int (*bop_check_delete)(struct nilfs_bmap *, __u64); + int (*bop_gather_data)(struct nilfs_bmap *, __u64 *, __u64 *, int); +}; + + +#define NILFS_BMAP_SIZE (NILFS_INODE_BMAP_SIZE * sizeof(__le64)) +#define NILFS_BMAP_KEY_BIT BITS_PER_LONG +#define NILFS_BMAP_NEW_PTR_INIT (1UL << (BITS_PER_LONG - 1)) + +static inline int nilfs_bmap_is_new_ptr(unsigned long ptr) +{ + return !!(ptr & NILFS_BMAP_NEW_PTR_INIT); +} + + +/** + * struct nilfs_bmap - bmap structure + * @b_u: raw data + * @b_sem: semaphore + * @b_inode: owner of bmap + * @b_ops: bmap operation table + * @b_last_allocated_key: last allocated key for data block + * @b_last_allocated_ptr: last allocated ptr for data block + * @b_ptr_type: pointer type + * @b_state: state + * @b_nchildren_per_block: maximum number of child nodes for non-root nodes + */ +struct nilfs_bmap { + union { + __u8 u_flags; + __le64 u_data[NILFS_BMAP_SIZE / sizeof(__le64)]; + } b_u; + struct rw_semaphore b_sem; + struct inode *b_inode; + const struct nilfs_bmap_operations *b_ops; + __u64 b_last_allocated_key; + __u64 b_last_allocated_ptr; + int b_ptr_type; + int b_state; + __u16 b_nchildren_per_block; +}; + +/* pointer type */ +#define NILFS_BMAP_PTR_P 0 /* physical block number (i.e. LBN) */ +#define NILFS_BMAP_PTR_VS 1 /* + * virtual block number (single + * version) + */ +#define NILFS_BMAP_PTR_VM 2 /* + * virtual block number (has multiple + * versions) + */ +#define NILFS_BMAP_PTR_U (-1) /* never perform pointer operations */ + +#define NILFS_BMAP_USE_VBN(bmap) ((bmap)->b_ptr_type > 0) + +/* state */ +#define NILFS_BMAP_DIRTY 0x00000001 + +/** + * struct nilfs_bmap_store - shadow copy of bmap state + * @data: cached raw block mapping of on-disk inode + * @last_allocated_key: cached value of last allocated key for data block + * @last_allocated_ptr: cached value of last allocated ptr for data block + * @state: cached value of state field of bmap structure + */ +struct nilfs_bmap_store { + __le64 data[NILFS_BMAP_SIZE / sizeof(__le64)]; + __u64 last_allocated_key; + __u64 last_allocated_ptr; + int state; +}; + +int nilfs_bmap_test_and_clear_dirty(struct nilfs_bmap *); +int nilfs_bmap_read(struct nilfs_bmap *, struct nilfs_inode *); +void nilfs_bmap_write(struct nilfs_bmap *, struct nilfs_inode *); +int nilfs_bmap_lookup_contig(struct nilfs_bmap *, __u64, __u64 *, unsigned int); +int nilfs_bmap_insert(struct nilfs_bmap *bmap, __u64 key, unsigned long rec); +int nilfs_bmap_delete(struct nilfs_bmap *bmap, __u64 key); +int nilfs_bmap_seek_key(struct nilfs_bmap *bmap, __u64 start, __u64 *keyp); +int nilfs_bmap_last_key(struct nilfs_bmap *bmap, __u64 *keyp); +int nilfs_bmap_truncate(struct nilfs_bmap *bmap, __u64 key); +void nilfs_bmap_clear(struct nilfs_bmap *); +int nilfs_bmap_propagate(struct nilfs_bmap *, struct buffer_head *); +void nilfs_bmap_lookup_dirty_buffers(struct nilfs_bmap *, struct list_head *); +int nilfs_bmap_assign(struct nilfs_bmap *, struct buffer_head **, + unsigned long, union nilfs_binfo *); +int nilfs_bmap_lookup_at_level(struct nilfs_bmap *, __u64, int, __u64 *); +int nilfs_bmap_mark(struct nilfs_bmap *, __u64, int); + +void nilfs_bmap_init_gc(struct nilfs_bmap *); + +void nilfs_bmap_save(const struct nilfs_bmap *, struct nilfs_bmap_store *); +void nilfs_bmap_restore(struct nilfs_bmap *, const struct nilfs_bmap_store *); + +static inline int nilfs_bmap_lookup(struct nilfs_bmap *bmap, __u64 key, + __u64 *ptr) +{ + return nilfs_bmap_lookup_at_level(bmap, key, 1, ptr); +} + +/* + * Internal use only + */ +struct inode *nilfs_bmap_get_dat(const struct nilfs_bmap *); + +static inline int nilfs_bmap_prepare_alloc_ptr(struct nilfs_bmap *bmap, + union nilfs_bmap_ptr_req *req, + struct inode *dat) +{ + if (dat) + return nilfs_dat_prepare_alloc(dat, &req->bpr_req); + /* ignore target ptr */ + req->bpr_ptr = bmap->b_last_allocated_ptr++; + return 0; +} + +static inline void nilfs_bmap_commit_alloc_ptr(struct nilfs_bmap *bmap, + union nilfs_bmap_ptr_req *req, + struct inode *dat) +{ + if (dat) + nilfs_dat_commit_alloc(dat, &req->bpr_req); +} + +static inline void nilfs_bmap_abort_alloc_ptr(struct nilfs_bmap *bmap, + union nilfs_bmap_ptr_req *req, + struct inode *dat) +{ + if (dat) + nilfs_dat_abort_alloc(dat, &req->bpr_req); + else + bmap->b_last_allocated_ptr--; +} + +static inline int nilfs_bmap_prepare_end_ptr(struct nilfs_bmap *bmap, + union nilfs_bmap_ptr_req *req, + struct inode *dat) +{ + return dat ? nilfs_dat_prepare_end(dat, &req->bpr_req) : 0; +} + +static inline void nilfs_bmap_commit_end_ptr(struct nilfs_bmap *bmap, + union nilfs_bmap_ptr_req *req, + struct inode *dat) +{ + if (dat) + nilfs_dat_commit_end(dat, &req->bpr_req, + bmap->b_ptr_type == NILFS_BMAP_PTR_VS); +} + +static inline void nilfs_bmap_abort_end_ptr(struct nilfs_bmap *bmap, + union nilfs_bmap_ptr_req *req, + struct inode *dat) +{ + if (dat) + nilfs_dat_abort_end(dat, &req->bpr_req); +} + +static inline void nilfs_bmap_set_target_v(struct nilfs_bmap *bmap, __u64 key, + __u64 ptr) +{ + bmap->b_last_allocated_key = key; + bmap->b_last_allocated_ptr = ptr; +} + +__u64 nilfs_bmap_data_get_key(const struct nilfs_bmap *, + const struct buffer_head *); + +__u64 nilfs_bmap_find_target_seq(const struct nilfs_bmap *, __u64); +__u64 nilfs_bmap_find_target_in_group(const struct nilfs_bmap *); + + +/* Assume that bmap semaphore is locked. */ +static inline int nilfs_bmap_dirty(const struct nilfs_bmap *bmap) +{ + return !!(bmap->b_state & NILFS_BMAP_DIRTY); +} + +/* Assume that bmap semaphore is locked. */ +static inline void nilfs_bmap_set_dirty(struct nilfs_bmap *bmap) +{ + bmap->b_state |= NILFS_BMAP_DIRTY; +} + +/* Assume that bmap semaphore is locked. */ +static inline void nilfs_bmap_clear_dirty(struct nilfs_bmap *bmap) +{ + bmap->b_state &= ~NILFS_BMAP_DIRTY; +} + + +#define NILFS_BMAP_LARGE 0x1 + +#define NILFS_BMAP_SMALL_LOW NILFS_DIRECT_KEY_MIN +#define NILFS_BMAP_SMALL_HIGH NILFS_DIRECT_KEY_MAX +#define NILFS_BMAP_LARGE_LOW NILFS_BTREE_ROOT_NCHILDREN_MAX +#define NILFS_BMAP_LARGE_HIGH NILFS_BTREE_KEY_MAX + +#endif /* _NILFS_BMAP_H */ diff --git a/fs/nilfs2/btnode.c b/fs/nilfs2/btnode.c new file mode 100644 index 000000000..680e4009a --- /dev/null +++ b/fs/nilfs2/btnode.c @@ -0,0 +1,356 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * NILFS B-tree node cache + * + * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation. + * + * Originally written by Seiji Kihara. + * Fully revised by Ryusuke Konishi for stabilization and simplification. + * + */ + +#include +#include +#include +#include +#include +#include "nilfs.h" +#include "mdt.h" +#include "dat.h" +#include "page.h" +#include "btnode.h" + + +/** + * nilfs_init_btnc_inode - initialize B-tree node cache inode + * @btnc_inode: inode to be initialized + * + * nilfs_init_btnc_inode() sets up an inode for B-tree node cache. + */ +void nilfs_init_btnc_inode(struct inode *btnc_inode) +{ + struct nilfs_inode_info *ii = NILFS_I(btnc_inode); + + btnc_inode->i_mode = S_IFREG; + ii->i_flags = 0; + memset(&ii->i_bmap_data, 0, sizeof(struct nilfs_bmap)); + mapping_set_gfp_mask(btnc_inode->i_mapping, GFP_NOFS); + btnc_inode->i_mapping->a_ops = &nilfs_buffer_cache_aops; +} + +void nilfs_btnode_cache_clear(struct address_space *btnc) +{ + invalidate_mapping_pages(btnc, 0, -1); + truncate_inode_pages(btnc, 0); +} + +struct buffer_head * +nilfs_btnode_create_block(struct address_space *btnc, __u64 blocknr) +{ + struct inode *inode = btnc->host; + struct buffer_head *bh; + + bh = nilfs_grab_buffer(inode, btnc, blocknr, BIT(BH_NILFS_Node)); + if (unlikely(!bh)) + return ERR_PTR(-ENOMEM); + + if (unlikely(buffer_mapped(bh) || buffer_uptodate(bh) || + buffer_dirty(bh))) { + /* + * The block buffer at the specified new address was already + * in use. This can happen if it is a virtual block number + * and has been reallocated due to corruption of the bitmap + * used to manage its allocation state (if not, the buffer + * clearing of an abandoned b-tree node is missing somewhere). + */ + nilfs_error(inode->i_sb, + "state inconsistency probably due to duplicate use of b-tree node block address %llu (ino=%llu)", + (unsigned long long)blocknr, inode->i_ino); + goto failed; + } + memset(bh->b_data, 0, i_blocksize(inode)); + bh->b_blocknr = blocknr; + set_buffer_mapped(bh); + set_buffer_uptodate(bh); + + folio_unlock(bh->b_folio); + folio_put(bh->b_folio); + return bh; + +failed: + folio_unlock(bh->b_folio); + folio_put(bh->b_folio); + brelse(bh); + return ERR_PTR(-EIO); +} + +int nilfs_btnode_submit_block(struct address_space *btnc, __u64 blocknr, + sector_t pblocknr, blk_opf_t opf, + struct buffer_head **pbh, sector_t *submit_ptr) +{ + struct buffer_head *bh; + struct inode *inode = btnc->host; + struct folio *folio; + int err; + + bh = nilfs_grab_buffer(inode, btnc, blocknr, BIT(BH_NILFS_Node)); + if (unlikely(!bh)) + return -ENOMEM; + + err = -EEXIST; /* internal code */ + folio = bh->b_folio; + + if (buffer_uptodate(bh) || buffer_dirty(bh)) + goto found; + + if (pblocknr == 0) { + pblocknr = blocknr; + if (inode->i_ino != NILFS_DAT_INO) { + struct the_nilfs *nilfs = inode->i_sb->s_fs_info; + + /* blocknr is a virtual block number */ + err = nilfs_dat_translate(nilfs->ns_dat, blocknr, + &pblocknr); + if (unlikely(err)) { + brelse(bh); + goto out_locked; + } + } + } + + if (opf & REQ_RAHEAD) { + if (pblocknr != *submit_ptr + 1 || !trylock_buffer(bh)) { + err = -EBUSY; /* internal code */ + brelse(bh); + goto out_locked; + } + } else { /* opf == REQ_OP_READ */ + lock_buffer(bh); + } + if (buffer_uptodate(bh)) { + unlock_buffer(bh); + err = -EEXIST; /* internal code */ + goto found; + } + set_buffer_mapped(bh); + bh->b_blocknr = pblocknr; /* set block address for read */ + bh_submit(bh, opf, bh_end_read); + bh->b_blocknr = blocknr; /* set back to the given block address */ + *submit_ptr = pblocknr; + err = 0; +found: + *pbh = bh; + +out_locked: + folio_unlock(folio); + folio_put(folio); + return err; +} + +/** + * nilfs_btnode_delete - delete B-tree node buffer + * @bh: buffer to be deleted + * + * nilfs_btnode_delete() invalidates the specified buffer and delete the page + * including the buffer if the page gets unbusy. + */ +void nilfs_btnode_delete(struct buffer_head *bh) +{ + struct address_space *mapping; + struct folio *folio = bh->b_folio; + pgoff_t index = folio->index; + int still_dirty; + + folio_get(folio); + folio_lock(folio); + folio_wait_writeback(folio); + + nilfs_forget_buffer(bh); + still_dirty = folio_test_dirty(folio); + mapping = folio->mapping; + folio_unlock(folio); + folio_put(folio); + + if (!still_dirty && mapping) + invalidate_inode_pages2_range(mapping, index, index); +} + +/** + * nilfs_btnode_prepare_change_key - prepare to change the search key of a + * b-tree node block + * @btnc: page cache in which the b-tree node block is buffered + * @ctxt: structure for exchanging context information for key change + * + * nilfs_btnode_prepare_change_key() prepares to move the contents of the + * b-tree node block of the old key given in the "oldkey" member of @ctxt to + * the position of the new key given in the "newkey" member of @ctxt in the + * page cache @btnc. Here, the key of the block is an index in units of + * blocks, and if the page and block sizes match, it matches the page index + * in the page cache. + * + * If the page size and block size match, this function attempts to move the + * entire folio, and in preparation for this, inserts the original folio into + * the new index of the cache. If this insertion fails or if the page size + * and block size are different, it falls back to a copy preparation using + * nilfs_btnode_create_block(), inserts a new block at the position + * corresponding to "newkey", and stores the buffer head pointer in the + * "newbh" member of @ctxt. + * + * Note that the current implementation does not support folio sizes larger + * than the page size. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (metadata corruption). + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_btnode_prepare_change_key(struct address_space *btnc, + struct nilfs_btnode_chkey_ctxt *ctxt) +{ + struct buffer_head *obh, *nbh; + struct inode *inode = btnc->host; + __u64 oldkey = ctxt->oldkey, newkey = ctxt->newkey; + int err; + + if (oldkey == newkey) + return 0; + + obh = ctxt->bh; + ctxt->newbh = NULL; + + if (inode->i_blkbits == PAGE_SHIFT) { + struct folio *ofolio = obh->b_folio; + folio_lock(ofolio); +retry: + /* BUG_ON(oldkey != obh->b_folio->index); */ + if (unlikely(oldkey != ofolio->index)) + NILFS_FOLIO_BUG(ofolio, + "invalid oldkey %lld (newkey=%lld)", + (unsigned long long)oldkey, + (unsigned long long)newkey); + + xa_lock_irq(&btnc->i_pages); + err = __xa_insert(&btnc->i_pages, newkey, ofolio, GFP_NOFS); + xa_unlock_irq(&btnc->i_pages); + /* + * Note: folio->index will not change to newkey until + * nilfs_btnode_commit_change_key() will be called. + * To protect the folio in intermediate state, the folio lock + * is held. + */ + if (!err) + return 0; + else if (err != -EBUSY) + goto failed_unlock; + + err = invalidate_inode_pages2_range(btnc, newkey, newkey); + if (!err) + goto retry; + /* fallback to copy mode */ + folio_unlock(ofolio); + } + + nbh = nilfs_btnode_create_block(btnc, newkey); + if (IS_ERR(nbh)) + return PTR_ERR(nbh); + + BUG_ON(nbh == obh); + ctxt->newbh = nbh; + return 0; + + failed_unlock: + folio_unlock(obh->b_folio); + return err; +} + +/** + * nilfs_btnode_commit_change_key - commit the change of the search key of + * a b-tree node block + * @btnc: page cache in which the b-tree node block is buffered + * @ctxt: structure for exchanging context information for key change + * + * nilfs_btnode_commit_change_key() executes the key change based on the + * context @ctxt prepared by nilfs_btnode_prepare_change_key(). If no valid + * block buffer is prepared in "newbh" of @ctxt (i.e., a full folio move), + * this function removes the folio from the old index and completes the move. + * Otherwise, it copies the block data and inherited flag states of "oldbh" + * to "newbh" and clears the "oldbh" from the cache. In either case, the + * relocated buffer is marked as dirty. + * + * As with nilfs_btnode_prepare_change_key(), the current implementation does + * not support folio sizes larger than the page size. + */ +void nilfs_btnode_commit_change_key(struct address_space *btnc, + struct nilfs_btnode_chkey_ctxt *ctxt) +{ + struct buffer_head *obh = ctxt->bh, *nbh = ctxt->newbh; + __u64 oldkey = ctxt->oldkey, newkey = ctxt->newkey; + struct folio *ofolio; + + if (oldkey == newkey) + return; + + if (nbh == NULL) { /* blocksize == pagesize */ + ofolio = obh->b_folio; + if (unlikely(oldkey != ofolio->index)) + NILFS_FOLIO_BUG(ofolio, + "invalid oldkey %lld (newkey=%lld)", + (unsigned long long)oldkey, + (unsigned long long)newkey); + mark_buffer_dirty(obh); + + xa_lock_irq(&btnc->i_pages); + __xa_erase(&btnc->i_pages, oldkey); + __xa_set_mark(&btnc->i_pages, newkey, PAGECACHE_TAG_DIRTY); + xa_unlock_irq(&btnc->i_pages); + + ofolio->index = obh->b_blocknr = newkey; + folio_unlock(ofolio); + } else { + nilfs_copy_buffer(nbh, obh); + mark_buffer_dirty(nbh); + + nbh->b_blocknr = newkey; + ctxt->bh = nbh; + nilfs_btnode_delete(obh); /* will decrement bh->b_count */ + } +} + +/** + * nilfs_btnode_abort_change_key - abort the change of the search key of a + * b-tree node block + * @btnc: page cache in which the b-tree node block is buffered + * @ctxt: structure for exchanging context information for key change + * + * nilfs_btnode_abort_change_key() cancels the key change associated with the + * context @ctxt prepared via nilfs_btnode_prepare_change_key() and performs + * any necessary cleanup. If no valid block buffer is prepared in "newbh" of + * @ctxt, this function removes the folio from the destination index and aborts + * the move. Otherwise, it clears "newbh" from the cache. + * + * As with nilfs_btnode_prepare_change_key(), the current implementation does + * not support folio sizes larger than the page size. + */ +void nilfs_btnode_abort_change_key(struct address_space *btnc, + struct nilfs_btnode_chkey_ctxt *ctxt) +{ + struct buffer_head *nbh = ctxt->newbh; + __u64 oldkey = ctxt->oldkey, newkey = ctxt->newkey; + + if (oldkey == newkey) + return; + + if (nbh == NULL) { /* blocksize == pagesize */ + xa_erase_irq(&btnc->i_pages, newkey); + folio_unlock(ctxt->bh->b_folio); + } else { + /* + * When canceling a buffer that a prepare operation has + * allocated to copy a node block to another location, use + * nilfs_btnode_delete() to initialize and release the buffer + * so that the buffer flags will not be in an inconsistent + * state when it is reallocated. + */ + nilfs_btnode_delete(nbh); + } +} diff --git a/fs/nilfs2/btnode.h b/fs/nilfs2/btnode.h new file mode 100644 index 000000000..4bc5612df --- /dev/null +++ b/fs/nilfs2/btnode.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * NILFS B-tree node cache + * + * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Seiji Kihara. + * Revised by Ryusuke Konishi. + */ + +#ifndef _NILFS_BTNODE_H +#define _NILFS_BTNODE_H + +#include +#include +#include +#include + +/** + * struct nilfs_btnode_chkey_ctxt - change key context + * @oldkey: old key of block's moving content + * @newkey: new key for block's content + * @bh: buffer head of old buffer + * @newbh: buffer head of new buffer + */ +struct nilfs_btnode_chkey_ctxt { + __u64 oldkey; + __u64 newkey; + struct buffer_head *bh; + struct buffer_head *newbh; +}; + +void nilfs_init_btnc_inode(struct inode *btnc_inode); +void nilfs_btnode_cache_clear(struct address_space *); +struct buffer_head *nilfs_btnode_create_block(struct address_space *btnc, + __u64 blocknr); +int nilfs_btnode_submit_block(struct address_space *, __u64, sector_t, + blk_opf_t, struct buffer_head **, sector_t *); +void nilfs_btnode_delete(struct buffer_head *); +int nilfs_btnode_prepare_change_key(struct address_space *, + struct nilfs_btnode_chkey_ctxt *); +void nilfs_btnode_commit_change_key(struct address_space *, + struct nilfs_btnode_chkey_ctxt *); +void nilfs_btnode_abort_change_key(struct address_space *, + struct nilfs_btnode_chkey_ctxt *); + +#endif /* _NILFS_BTNODE_H */ diff --git a/fs/nilfs2/btree.c b/fs/nilfs2/btree.c new file mode 100644 index 000000000..6b8332e8c --- /dev/null +++ b/fs/nilfs2/btree.c @@ -0,0 +1,2485 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * NILFS B-tree. + * + * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Koji Sato. + */ + +#include +#include +#include +#include +#include "nilfs.h" +#include "page.h" +#include "btnode.h" +#include "btree.h" +#include "alloc.h" +#include "dat.h" + +static void __nilfs_btree_init(struct nilfs_bmap *bmap); + +static struct nilfs_btree_path *nilfs_btree_alloc_path(void) +{ + struct nilfs_btree_path *path; + int level = NILFS_BTREE_LEVEL_DATA; + + path = kmem_cache_alloc(nilfs_btree_path_cache, GFP_NOFS); + if (path == NULL) + goto out; + + for (; level < NILFS_BTREE_LEVEL_MAX; level++) { + path[level].bp_bh = NULL; + path[level].bp_sib_bh = NULL; + path[level].bp_index = 0; + path[level].bp_oldreq.bpr_ptr = NILFS_BMAP_INVALID_PTR; + path[level].bp_newreq.bpr_ptr = NILFS_BMAP_INVALID_PTR; + path[level].bp_op = NULL; + } + +out: + return path; +} + +static void nilfs_btree_free_path(struct nilfs_btree_path *path) +{ + int level = NILFS_BTREE_LEVEL_DATA; + + for (; level < NILFS_BTREE_LEVEL_MAX; level++) + brelse(path[level].bp_bh); + + kmem_cache_free(nilfs_btree_path_cache, path); +} + +/* + * B-tree node operations + */ +static int nilfs_btree_get_new_block(const struct nilfs_bmap *btree, + __u64 ptr, struct buffer_head **bhp) +{ + struct inode *btnc_inode = NILFS_BMAP_I(btree)->i_assoc_inode; + struct address_space *btnc = btnc_inode->i_mapping; + struct buffer_head *bh; + + bh = nilfs_btnode_create_block(btnc, ptr); + if (IS_ERR(bh)) + return PTR_ERR(bh); + + set_buffer_nilfs_volatile(bh); + *bhp = bh; + return 0; +} + +static int nilfs_btree_node_get_flags(const struct nilfs_btree_node *node) +{ + return node->bn_flags; +} + +static void +nilfs_btree_node_set_flags(struct nilfs_btree_node *node, int flags) +{ + node->bn_flags = flags; +} + +static int nilfs_btree_node_root(const struct nilfs_btree_node *node) +{ + return nilfs_btree_node_get_flags(node) & NILFS_BTREE_NODE_ROOT; +} + +static int nilfs_btree_node_get_level(const struct nilfs_btree_node *node) +{ + return node->bn_level; +} + +static void +nilfs_btree_node_set_level(struct nilfs_btree_node *node, int level) +{ + node->bn_level = level; +} + +static int nilfs_btree_node_get_nchildren(const struct nilfs_btree_node *node) +{ + return le16_to_cpu(node->bn_nchildren); +} + +static void +nilfs_btree_node_set_nchildren(struct nilfs_btree_node *node, int nchildren) +{ + node->bn_nchildren = cpu_to_le16(nchildren); +} + +static int nilfs_btree_node_size(const struct nilfs_bmap *btree) +{ + return i_blocksize(btree->b_inode); +} + +static int nilfs_btree_nchildren_per_block(const struct nilfs_bmap *btree) +{ + return btree->b_nchildren_per_block; +} + +static __le64 * +nilfs_btree_node_dkeys(const struct nilfs_btree_node *node) +{ + return (__le64 *)((char *)(node + 1) + + (nilfs_btree_node_root(node) ? + 0 : NILFS_BTREE_NODE_EXTRA_PAD_SIZE)); +} + +static __le64 * +nilfs_btree_node_dptrs(const struct nilfs_btree_node *node, int ncmax) +{ + return (__le64 *)(nilfs_btree_node_dkeys(node) + ncmax); +} + +static __u64 +nilfs_btree_node_get_key(const struct nilfs_btree_node *node, int index) +{ + return le64_to_cpu(*(nilfs_btree_node_dkeys(node) + index)); +} + +static void +nilfs_btree_node_set_key(struct nilfs_btree_node *node, int index, __u64 key) +{ + *(nilfs_btree_node_dkeys(node) + index) = cpu_to_le64(key); +} + +static __u64 +nilfs_btree_node_get_ptr(const struct nilfs_btree_node *node, int index, + int ncmax) +{ + return le64_to_cpu(*(nilfs_btree_node_dptrs(node, ncmax) + index)); +} + +static void +nilfs_btree_node_set_ptr(struct nilfs_btree_node *node, int index, __u64 ptr, + int ncmax) +{ + *(nilfs_btree_node_dptrs(node, ncmax) + index) = cpu_to_le64(ptr); +} + +static void nilfs_btree_node_init(struct nilfs_btree_node *node, int flags, + int level, int nchildren, int ncmax, + const __u64 *keys, const __u64 *ptrs) +{ + __le64 *dkeys; + __le64 *dptrs; + int i; + + nilfs_btree_node_set_flags(node, flags); + nilfs_btree_node_set_level(node, level); + nilfs_btree_node_set_nchildren(node, nchildren); + + dkeys = nilfs_btree_node_dkeys(node); + dptrs = nilfs_btree_node_dptrs(node, ncmax); + for (i = 0; i < nchildren; i++) { + dkeys[i] = cpu_to_le64(keys[i]); + dptrs[i] = cpu_to_le64(ptrs[i]); + } +} + +/* Assume the buffer heads corresponding to left and right are locked. */ +static void nilfs_btree_node_move_left(struct nilfs_btree_node *left, + struct nilfs_btree_node *right, + int n, int lncmax, int rncmax) +{ + __le64 *ldkeys, *rdkeys; + __le64 *ldptrs, *rdptrs; + int lnchildren, rnchildren; + + ldkeys = nilfs_btree_node_dkeys(left); + ldptrs = nilfs_btree_node_dptrs(left, lncmax); + lnchildren = nilfs_btree_node_get_nchildren(left); + + rdkeys = nilfs_btree_node_dkeys(right); + rdptrs = nilfs_btree_node_dptrs(right, rncmax); + rnchildren = nilfs_btree_node_get_nchildren(right); + + memcpy(ldkeys + lnchildren, rdkeys, n * sizeof(*rdkeys)); + memcpy(ldptrs + lnchildren, rdptrs, n * sizeof(*rdptrs)); + memmove(rdkeys, rdkeys + n, (rnchildren - n) * sizeof(*rdkeys)); + memmove(rdptrs, rdptrs + n, (rnchildren - n) * sizeof(*rdptrs)); + + lnchildren += n; + rnchildren -= n; + nilfs_btree_node_set_nchildren(left, lnchildren); + nilfs_btree_node_set_nchildren(right, rnchildren); +} + +/* Assume that the buffer heads corresponding to left and right are locked. */ +static void nilfs_btree_node_move_right(struct nilfs_btree_node *left, + struct nilfs_btree_node *right, + int n, int lncmax, int rncmax) +{ + __le64 *ldkeys, *rdkeys; + __le64 *ldptrs, *rdptrs; + int lnchildren, rnchildren; + + ldkeys = nilfs_btree_node_dkeys(left); + ldptrs = nilfs_btree_node_dptrs(left, lncmax); + lnchildren = nilfs_btree_node_get_nchildren(left); + + rdkeys = nilfs_btree_node_dkeys(right); + rdptrs = nilfs_btree_node_dptrs(right, rncmax); + rnchildren = nilfs_btree_node_get_nchildren(right); + + memmove(rdkeys + n, rdkeys, rnchildren * sizeof(*rdkeys)); + memmove(rdptrs + n, rdptrs, rnchildren * sizeof(*rdptrs)); + memcpy(rdkeys, ldkeys + lnchildren - n, n * sizeof(*rdkeys)); + memcpy(rdptrs, ldptrs + lnchildren - n, n * sizeof(*rdptrs)); + + lnchildren -= n; + rnchildren += n; + nilfs_btree_node_set_nchildren(left, lnchildren); + nilfs_btree_node_set_nchildren(right, rnchildren); +} + +/* Assume that the buffer head corresponding to node is locked. */ +static void nilfs_btree_node_insert(struct nilfs_btree_node *node, int index, + __u64 key, __u64 ptr, int ncmax) +{ + __le64 *dkeys; + __le64 *dptrs; + int nchildren; + + dkeys = nilfs_btree_node_dkeys(node); + dptrs = nilfs_btree_node_dptrs(node, ncmax); + nchildren = nilfs_btree_node_get_nchildren(node); + if (index < nchildren) { + memmove(dkeys + index + 1, dkeys + index, + (nchildren - index) * sizeof(*dkeys)); + memmove(dptrs + index + 1, dptrs + index, + (nchildren - index) * sizeof(*dptrs)); + } + dkeys[index] = cpu_to_le64(key); + dptrs[index] = cpu_to_le64(ptr); + nchildren++; + nilfs_btree_node_set_nchildren(node, nchildren); +} + +/* Assume that the buffer head corresponding to node is locked. */ +static void nilfs_btree_node_delete(struct nilfs_btree_node *node, int index, + __u64 *keyp, __u64 *ptrp, int ncmax) +{ + __u64 key; + __u64 ptr; + __le64 *dkeys; + __le64 *dptrs; + int nchildren; + + dkeys = nilfs_btree_node_dkeys(node); + dptrs = nilfs_btree_node_dptrs(node, ncmax); + key = le64_to_cpu(dkeys[index]); + ptr = le64_to_cpu(dptrs[index]); + nchildren = nilfs_btree_node_get_nchildren(node); + if (keyp != NULL) + *keyp = key; + if (ptrp != NULL) + *ptrp = ptr; + + if (index < nchildren - 1) { + memmove(dkeys + index, dkeys + index + 1, + (nchildren - index - 1) * sizeof(*dkeys)); + memmove(dptrs + index, dptrs + index + 1, + (nchildren - index - 1) * sizeof(*dptrs)); + } + nchildren--; + nilfs_btree_node_set_nchildren(node, nchildren); +} + +static int nilfs_btree_node_lookup(const struct nilfs_btree_node *node, + __u64 key, int *indexp) +{ + __u64 nkey; + int index, low, high, s; + + /* binary search */ + low = 0; + high = nilfs_btree_node_get_nchildren(node) - 1; + index = 0; + s = 0; + while (low <= high) { + index = (low + high) / 2; + nkey = nilfs_btree_node_get_key(node, index); + if (nkey == key) { + s = 0; + goto out; + } else if (nkey < key) { + low = index + 1; + s = -1; + } else { + high = index - 1; + s = 1; + } + } + + /* adjust index */ + if (nilfs_btree_node_get_level(node) > NILFS_BTREE_LEVEL_NODE_MIN) { + if (s > 0 && index > 0) + index--; + } else if (s < 0) + index++; + + out: + *indexp = index; + + return s == 0; +} + +/** + * nilfs_btree_node_broken - verify consistency of btree node + * @node: btree node block to be examined + * @size: node size (in bytes) + * @inode: host inode of btree + * @blocknr: block number + * + * Return: 0 if normal, 1 if the node is broken. + */ +static int nilfs_btree_node_broken(const struct nilfs_btree_node *node, + size_t size, struct inode *inode, + sector_t blocknr) +{ + int level, flags, nchildren; + __u64 key, prev_key; + int i; + + level = nilfs_btree_node_get_level(node); + flags = nilfs_btree_node_get_flags(node); + nchildren = nilfs_btree_node_get_nchildren(node); + + if (unlikely(level < NILFS_BTREE_LEVEL_NODE_MIN || + level >= NILFS_BTREE_LEVEL_MAX || + (flags & NILFS_BTREE_NODE_ROOT) || + nchildren <= 0 || + nchildren > NILFS_BTREE_NODE_NCHILDREN_MAX(size))) { + nilfs_crit(inode->i_sb, + "bad btree node (ino=%llu, blocknr=%llu): level = %d, flags = 0x%x, nchildren = %d", + inode->i_ino, (unsigned long long)blocknr, level, + flags, nchildren); + return 1; + } + + for (i = 1, prev_key = nilfs_btree_node_get_key(node, 0); + i < nchildren; i++, prev_key = key) { + key = nilfs_btree_node_get_key(node, i); + if (unlikely(key <= prev_key)) { + nilfs_crit(inode->i_sb, + "bad btree node (ino=%llu, blocknr=%llu): unsorted keys at index %d (%llu) and %d (%llu)", + inode->i_ino, (unsigned long long)blocknr, + i - 1, prev_key, i, key); + return 1; + } + } + return 0; +} + +/** + * nilfs_btree_root_broken - verify consistency of btree root node + * @node: btree root node to be examined + * @inode: host inode of btree + * + * Return: 0 if normal, 1 if the root node is broken. + */ +static int nilfs_btree_root_broken(const struct nilfs_btree_node *node, + struct inode *inode) +{ + int level, flags, nchildren; + int ret = 0; + + level = nilfs_btree_node_get_level(node); + flags = nilfs_btree_node_get_flags(node); + nchildren = nilfs_btree_node_get_nchildren(node); + + if (unlikely(level < NILFS_BTREE_LEVEL_NODE_MIN || + level >= NILFS_BTREE_LEVEL_MAX || + nchildren < 0 || + nchildren > NILFS_BTREE_ROOT_NCHILDREN_MAX || + (nchildren == 0 && level > NILFS_BTREE_LEVEL_NODE_MIN))) { + nilfs_crit(inode->i_sb, + "bad btree root (ino=%llu): level = %d, flags = 0x%x, nchildren = %d", + inode->i_ino, level, flags, nchildren); + ret = 1; + } + return ret; +} + +int nilfs_btree_broken_node_block(struct buffer_head *bh) +{ + struct inode *inode; + int ret; + + if (buffer_nilfs_checked(bh)) + return 0; + + inode = bh->b_folio->mapping->host; + ret = nilfs_btree_node_broken((struct nilfs_btree_node *)bh->b_data, + bh->b_size, inode, bh->b_blocknr); + if (likely(!ret)) + set_buffer_nilfs_checked(bh); + return ret; +} + +static struct nilfs_btree_node * +nilfs_btree_get_root(const struct nilfs_bmap *btree) +{ + return (struct nilfs_btree_node *)btree->b_u.u_data; +} + +static struct nilfs_btree_node * +nilfs_btree_get_nonroot_node(const struct nilfs_btree_path *path, int level) +{ + return (struct nilfs_btree_node *)path[level].bp_bh->b_data; +} + +static struct nilfs_btree_node * +nilfs_btree_get_sib_node(const struct nilfs_btree_path *path, int level) +{ + return (struct nilfs_btree_node *)path[level].bp_sib_bh->b_data; +} + +static int nilfs_btree_height(const struct nilfs_bmap *btree) +{ + return nilfs_btree_node_get_level(nilfs_btree_get_root(btree)) + 1; +} + +static struct nilfs_btree_node * +nilfs_btree_get_node(const struct nilfs_bmap *btree, + const struct nilfs_btree_path *path, + int level, int *ncmaxp) +{ + struct nilfs_btree_node *node; + + if (level == nilfs_btree_height(btree) - 1) { + node = nilfs_btree_get_root(btree); + *ncmaxp = NILFS_BTREE_ROOT_NCHILDREN_MAX; + } else { + node = nilfs_btree_get_nonroot_node(path, level); + *ncmaxp = nilfs_btree_nchildren_per_block(btree); + } + return node; +} + +static int nilfs_btree_bad_node(const struct nilfs_bmap *btree, + struct nilfs_btree_node *node, int level) +{ + if (unlikely(nilfs_btree_node_get_level(node) != level)) { + dump_stack(); + nilfs_crit(btree->b_inode->i_sb, + "btree level mismatch (ino=%llu): %d != %d", + btree->b_inode->i_ino, + nilfs_btree_node_get_level(node), level); + return 1; + } + return 0; +} + +struct nilfs_btree_readahead_info { + struct nilfs_btree_node *node; /* parent node */ + int max_ra_blocks; /* max nof blocks to read ahead */ + int index; /* current index on the parent node */ + int ncmax; /* nof children in the parent node */ +}; + +static int __nilfs_btree_get_block(const struct nilfs_bmap *btree, __u64 ptr, + struct buffer_head **bhp, + const struct nilfs_btree_readahead_info *ra) +{ + struct inode *btnc_inode = NILFS_BMAP_I(btree)->i_assoc_inode; + struct address_space *btnc = btnc_inode->i_mapping; + struct buffer_head *bh, *ra_bh; + sector_t submit_ptr = 0; + int ret; + + ret = nilfs_btnode_submit_block(btnc, ptr, 0, REQ_OP_READ, &bh, + &submit_ptr); + if (ret) { + if (likely(ret == -EEXIST)) + goto out_check; + if (ret == -ENOENT) { + /* + * Block address translation failed due to invalid + * value of 'ptr'. In this case, return internal code + * -EINVAL (broken bmap) to notify bmap layer of fatal + * metadata corruption. + */ + ret = -EINVAL; + } + return ret; + } + + if (ra) { + int i, n; + __u64 ptr2; + + /* read ahead sibling nodes */ + for (n = ra->max_ra_blocks, i = ra->index + 1; + n > 0 && i < ra->ncmax; n--, i++) { + ptr2 = nilfs_btree_node_get_ptr(ra->node, i, ra->ncmax); + + ret = nilfs_btnode_submit_block(btnc, ptr2, 0, + REQ_OP_READ | REQ_RAHEAD, + &ra_bh, &submit_ptr); + if (likely(!ret || ret == -EEXIST)) + brelse(ra_bh); + else if (ret != -EBUSY) + break; + if (!buffer_locked(bh)) + goto out_no_wait; + } + } + + wait_on_buffer(bh); + + out_no_wait: + if (!buffer_uptodate(bh)) { + nilfs_err(btree->b_inode->i_sb, + "I/O error reading b-tree node block (ino=%llu, blocknr=%llu)", + btree->b_inode->i_ino, (unsigned long long)ptr); + brelse(bh); + return -EIO; + } + + out_check: + if (nilfs_btree_broken_node_block(bh)) { + clear_buffer_uptodate(bh); + brelse(bh); + return -EINVAL; + } + + *bhp = bh; + return 0; +} + +static int nilfs_btree_get_block(const struct nilfs_bmap *btree, __u64 ptr, + struct buffer_head **bhp) +{ + return __nilfs_btree_get_block(btree, ptr, bhp, NULL); +} + +static int nilfs_btree_do_lookup(const struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + __u64 key, __u64 *ptrp, int minlevel, + int readahead) +{ + struct nilfs_btree_node *node; + struct nilfs_btree_readahead_info p, *ra; + __u64 ptr; + int level, index, found, ncmax, ret; + + node = nilfs_btree_get_root(btree); + level = nilfs_btree_node_get_level(node); + if (level < minlevel || nilfs_btree_node_get_nchildren(node) <= 0) + return -ENOENT; + + found = nilfs_btree_node_lookup(node, key, &index); + ptr = nilfs_btree_node_get_ptr(node, index, + NILFS_BTREE_ROOT_NCHILDREN_MAX); + path[level].bp_bh = NULL; + path[level].bp_index = index; + + ncmax = nilfs_btree_nchildren_per_block(btree); + + while (--level >= minlevel) { + ra = NULL; + if (level == NILFS_BTREE_LEVEL_NODE_MIN && readahead) { + p.node = nilfs_btree_get_node(btree, path, level + 1, + &p.ncmax); + p.index = index; + p.max_ra_blocks = 7; + ra = &p; + } + ret = __nilfs_btree_get_block(btree, ptr, &path[level].bp_bh, + ra); + if (ret < 0) + return ret; + + node = nilfs_btree_get_nonroot_node(path, level); + if (nilfs_btree_bad_node(btree, node, level)) + return -EINVAL; + if (!found) + found = nilfs_btree_node_lookup(node, key, &index); + else + index = 0; + if (index < ncmax) { + ptr = nilfs_btree_node_get_ptr(node, index, ncmax); + } else { + WARN_ON(found || level != NILFS_BTREE_LEVEL_NODE_MIN); + /* insert */ + ptr = NILFS_BMAP_INVALID_PTR; + } + path[level].bp_index = index; + } + if (!found) + return -ENOENT; + + if (ptrp != NULL) + *ptrp = ptr; + + return 0; +} + +static int nilfs_btree_do_lookup_last(const struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + __u64 *keyp, __u64 *ptrp) +{ + struct nilfs_btree_node *node; + __u64 ptr; + int index, level, ncmax, ret; + + node = nilfs_btree_get_root(btree); + index = nilfs_btree_node_get_nchildren(node) - 1; + if (index < 0) + return -ENOENT; + level = nilfs_btree_node_get_level(node); + ptr = nilfs_btree_node_get_ptr(node, index, + NILFS_BTREE_ROOT_NCHILDREN_MAX); + path[level].bp_bh = NULL; + path[level].bp_index = index; + ncmax = nilfs_btree_nchildren_per_block(btree); + + for (level--; level > 0; level--) { + ret = nilfs_btree_get_block(btree, ptr, &path[level].bp_bh); + if (ret < 0) + return ret; + node = nilfs_btree_get_nonroot_node(path, level); + if (nilfs_btree_bad_node(btree, node, level)) + return -EINVAL; + index = nilfs_btree_node_get_nchildren(node) - 1; + ptr = nilfs_btree_node_get_ptr(node, index, ncmax); + path[level].bp_index = index; + } + + if (keyp != NULL) + *keyp = nilfs_btree_node_get_key(node, index); + if (ptrp != NULL) + *ptrp = ptr; + + return 0; +} + +/** + * nilfs_btree_get_next_key - get next valid key from btree path array + * @btree: bmap struct of btree + * @path: array of nilfs_btree_path struct + * @minlevel: start level + * @nextkey: place to store the next valid key + * + * Return: 0 if the next key was found, %-ENOENT if not found. + */ +static int nilfs_btree_get_next_key(const struct nilfs_bmap *btree, + const struct nilfs_btree_path *path, + int minlevel, __u64 *nextkey) +{ + struct nilfs_btree_node *node; + int maxlevel = nilfs_btree_height(btree) - 1; + int index, next_adj, level; + + /* Next index is already set to bp_index for leaf nodes. */ + next_adj = 0; + for (level = minlevel; level <= maxlevel; level++) { + if (level == maxlevel) + node = nilfs_btree_get_root(btree); + else + node = nilfs_btree_get_nonroot_node(path, level); + + index = path[level].bp_index + next_adj; + if (index < nilfs_btree_node_get_nchildren(node)) { + /* Next key is in this node */ + *nextkey = nilfs_btree_node_get_key(node, index); + return 0; + } + /* For non-leaf nodes, next index is stored at bp_index + 1. */ + next_adj = 1; + } + return -ENOENT; +} + +static int nilfs_btree_lookup(const struct nilfs_bmap *btree, + __u64 key, int level, __u64 *ptrp) +{ + struct nilfs_btree_path *path; + int ret; + + path = nilfs_btree_alloc_path(); + if (path == NULL) + return -ENOMEM; + + ret = nilfs_btree_do_lookup(btree, path, key, ptrp, level, 0); + + nilfs_btree_free_path(path); + + return ret; +} + +static int nilfs_btree_lookup_contig(const struct nilfs_bmap *btree, + __u64 key, __u64 *ptrp, + unsigned int maxblocks) +{ + struct nilfs_btree_path *path; + struct nilfs_btree_node *node; + struct inode *dat = NULL; + __u64 ptr, ptr2; + sector_t blocknr; + int level = NILFS_BTREE_LEVEL_NODE_MIN; + int ret, cnt, index, maxlevel, ncmax; + struct nilfs_btree_readahead_info p; + + path = nilfs_btree_alloc_path(); + if (path == NULL) + return -ENOMEM; + + ret = nilfs_btree_do_lookup(btree, path, key, &ptr, level, 1); + if (ret < 0) + goto out; + + if (NILFS_BMAP_USE_VBN(btree)) { + dat = nilfs_bmap_get_dat(btree); + ret = nilfs_dat_translate(dat, ptr, &blocknr); + if (ret < 0) + goto dat_error; + ptr = blocknr; + } + cnt = 1; + if (cnt == maxblocks) + goto end; + + maxlevel = nilfs_btree_height(btree) - 1; + node = nilfs_btree_get_node(btree, path, level, &ncmax); + index = path[level].bp_index + 1; + for (;;) { + while (index < nilfs_btree_node_get_nchildren(node)) { + if (nilfs_btree_node_get_key(node, index) != + key + cnt) + goto end; + ptr2 = nilfs_btree_node_get_ptr(node, index, ncmax); + if (dat) { + ret = nilfs_dat_translate(dat, ptr2, &blocknr); + if (ret < 0) + goto dat_error; + ptr2 = blocknr; + } + if (ptr2 != ptr + cnt || ++cnt == maxblocks) + goto end; + index++; + } + if (level == maxlevel) + break; + + /* look-up right sibling node */ + p.node = nilfs_btree_get_node(btree, path, level + 1, &p.ncmax); + p.index = path[level + 1].bp_index + 1; + p.max_ra_blocks = 7; + if (p.index >= nilfs_btree_node_get_nchildren(p.node) || + nilfs_btree_node_get_key(p.node, p.index) != key + cnt) + break; + ptr2 = nilfs_btree_node_get_ptr(p.node, p.index, p.ncmax); + path[level + 1].bp_index = p.index; + + brelse(path[level].bp_bh); + path[level].bp_bh = NULL; + + ret = __nilfs_btree_get_block(btree, ptr2, &path[level].bp_bh, + &p); + if (ret < 0) + goto out; + node = nilfs_btree_get_nonroot_node(path, level); + ncmax = nilfs_btree_nchildren_per_block(btree); + index = 0; + path[level].bp_index = index; + } + end: + *ptrp = ptr; + ret = cnt; + out: + nilfs_btree_free_path(path); + return ret; + + dat_error: + if (ret == -ENOENT) + ret = -EINVAL; /* Notify bmap layer of metadata corruption */ + goto out; +} + +static void nilfs_btree_promote_key(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + int level, __u64 key) +{ + if (level < nilfs_btree_height(btree) - 1) { + do { + nilfs_btree_node_set_key( + nilfs_btree_get_nonroot_node(path, level), + path[level].bp_index, key); + if (!buffer_dirty(path[level].bp_bh)) + mark_buffer_dirty(path[level].bp_bh); + } while ((path[level].bp_index == 0) && + (++level < nilfs_btree_height(btree) - 1)); + } + + /* root */ + if (level == nilfs_btree_height(btree) - 1) { + nilfs_btree_node_set_key(nilfs_btree_get_root(btree), + path[level].bp_index, key); + } +} + +static void nilfs_btree_do_insert(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + int level, __u64 *keyp, __u64 *ptrp) +{ + struct nilfs_btree_node *node; + int ncblk; + + if (level < nilfs_btree_height(btree) - 1) { + node = nilfs_btree_get_nonroot_node(path, level); + ncblk = nilfs_btree_nchildren_per_block(btree); + nilfs_btree_node_insert(node, path[level].bp_index, + *keyp, *ptrp, ncblk); + if (!buffer_dirty(path[level].bp_bh)) + mark_buffer_dirty(path[level].bp_bh); + + if (path[level].bp_index == 0) + nilfs_btree_promote_key(btree, path, level + 1, + nilfs_btree_node_get_key(node, + 0)); + } else { + node = nilfs_btree_get_root(btree); + nilfs_btree_node_insert(node, path[level].bp_index, + *keyp, *ptrp, + NILFS_BTREE_ROOT_NCHILDREN_MAX); + } +} + +static void nilfs_btree_carry_left(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + int level, __u64 *keyp, __u64 *ptrp) +{ + struct nilfs_btree_node *node, *left; + int nchildren, lnchildren, n, move, ncblk; + + node = nilfs_btree_get_nonroot_node(path, level); + left = nilfs_btree_get_sib_node(path, level); + nchildren = nilfs_btree_node_get_nchildren(node); + lnchildren = nilfs_btree_node_get_nchildren(left); + ncblk = nilfs_btree_nchildren_per_block(btree); + move = 0; + + n = (nchildren + lnchildren + 1) / 2 - lnchildren; + if (n > path[level].bp_index) { + /* move insert point */ + n--; + move = 1; + } + + nilfs_btree_node_move_left(left, node, n, ncblk, ncblk); + + if (!buffer_dirty(path[level].bp_bh)) + mark_buffer_dirty(path[level].bp_bh); + if (!buffer_dirty(path[level].bp_sib_bh)) + mark_buffer_dirty(path[level].bp_sib_bh); + + nilfs_btree_promote_key(btree, path, level + 1, + nilfs_btree_node_get_key(node, 0)); + + if (move) { + brelse(path[level].bp_bh); + path[level].bp_bh = path[level].bp_sib_bh; + path[level].bp_sib_bh = NULL; + path[level].bp_index += lnchildren; + path[level + 1].bp_index--; + } else { + brelse(path[level].bp_sib_bh); + path[level].bp_sib_bh = NULL; + path[level].bp_index -= n; + } + + nilfs_btree_do_insert(btree, path, level, keyp, ptrp); +} + +static void nilfs_btree_carry_right(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + int level, __u64 *keyp, __u64 *ptrp) +{ + struct nilfs_btree_node *node, *right; + int nchildren, rnchildren, n, move, ncblk; + + node = nilfs_btree_get_nonroot_node(path, level); + right = nilfs_btree_get_sib_node(path, level); + nchildren = nilfs_btree_node_get_nchildren(node); + rnchildren = nilfs_btree_node_get_nchildren(right); + ncblk = nilfs_btree_nchildren_per_block(btree); + move = 0; + + n = (nchildren + rnchildren + 1) / 2 - rnchildren; + if (n > nchildren - path[level].bp_index) { + /* move insert point */ + n--; + move = 1; + } + + nilfs_btree_node_move_right(node, right, n, ncblk, ncblk); + + if (!buffer_dirty(path[level].bp_bh)) + mark_buffer_dirty(path[level].bp_bh); + if (!buffer_dirty(path[level].bp_sib_bh)) + mark_buffer_dirty(path[level].bp_sib_bh); + + path[level + 1].bp_index++; + nilfs_btree_promote_key(btree, path, level + 1, + nilfs_btree_node_get_key(right, 0)); + path[level + 1].bp_index--; + + if (move) { + brelse(path[level].bp_bh); + path[level].bp_bh = path[level].bp_sib_bh; + path[level].bp_sib_bh = NULL; + path[level].bp_index -= nilfs_btree_node_get_nchildren(node); + path[level + 1].bp_index++; + } else { + brelse(path[level].bp_sib_bh); + path[level].bp_sib_bh = NULL; + } + + nilfs_btree_do_insert(btree, path, level, keyp, ptrp); +} + +static void nilfs_btree_split(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + int level, __u64 *keyp, __u64 *ptrp) +{ + struct nilfs_btree_node *node, *right; + int nchildren, n, move, ncblk; + + node = nilfs_btree_get_nonroot_node(path, level); + right = nilfs_btree_get_sib_node(path, level); + nchildren = nilfs_btree_node_get_nchildren(node); + ncblk = nilfs_btree_nchildren_per_block(btree); + move = 0; + + n = (nchildren + 1) / 2; + if (n > nchildren - path[level].bp_index) { + n--; + move = 1; + } + + nilfs_btree_node_move_right(node, right, n, ncblk, ncblk); + + if (!buffer_dirty(path[level].bp_bh)) + mark_buffer_dirty(path[level].bp_bh); + if (!buffer_dirty(path[level].bp_sib_bh)) + mark_buffer_dirty(path[level].bp_sib_bh); + + if (move) { + path[level].bp_index -= nilfs_btree_node_get_nchildren(node); + nilfs_btree_node_insert(right, path[level].bp_index, + *keyp, *ptrp, ncblk); + + *keyp = nilfs_btree_node_get_key(right, 0); + *ptrp = path[level].bp_newreq.bpr_ptr; + + brelse(path[level].bp_bh); + path[level].bp_bh = path[level].bp_sib_bh; + path[level].bp_sib_bh = NULL; + } else { + nilfs_btree_do_insert(btree, path, level, keyp, ptrp); + + *keyp = nilfs_btree_node_get_key(right, 0); + *ptrp = path[level].bp_newreq.bpr_ptr; + + brelse(path[level].bp_sib_bh); + path[level].bp_sib_bh = NULL; + } + + path[level + 1].bp_index++; +} + +static void nilfs_btree_grow(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + int level, __u64 *keyp, __u64 *ptrp) +{ + struct nilfs_btree_node *root, *child; + int n, ncblk; + + root = nilfs_btree_get_root(btree); + child = nilfs_btree_get_sib_node(path, level); + ncblk = nilfs_btree_nchildren_per_block(btree); + + n = nilfs_btree_node_get_nchildren(root); + + nilfs_btree_node_move_right(root, child, n, + NILFS_BTREE_ROOT_NCHILDREN_MAX, ncblk); + nilfs_btree_node_set_level(root, level + 1); + + if (!buffer_dirty(path[level].bp_sib_bh)) + mark_buffer_dirty(path[level].bp_sib_bh); + + path[level].bp_bh = path[level].bp_sib_bh; + path[level].bp_sib_bh = NULL; + + nilfs_btree_do_insert(btree, path, level, keyp, ptrp); + + *keyp = nilfs_btree_node_get_key(child, 0); + *ptrp = path[level].bp_newreq.bpr_ptr; +} + +static __u64 nilfs_btree_find_near(const struct nilfs_bmap *btree, + const struct nilfs_btree_path *path) +{ + struct nilfs_btree_node *node; + int level, ncmax; + + if (path == NULL) + return NILFS_BMAP_INVALID_PTR; + + /* left sibling */ + level = NILFS_BTREE_LEVEL_NODE_MIN; + if (path[level].bp_index > 0) { + node = nilfs_btree_get_node(btree, path, level, &ncmax); + return nilfs_btree_node_get_ptr(node, + path[level].bp_index - 1, + ncmax); + } + + /* parent */ + level = NILFS_BTREE_LEVEL_NODE_MIN + 1; + if (level <= nilfs_btree_height(btree) - 1) { + node = nilfs_btree_get_node(btree, path, level, &ncmax); + return nilfs_btree_node_get_ptr(node, path[level].bp_index, + ncmax); + } + + return NILFS_BMAP_INVALID_PTR; +} + +static __u64 nilfs_btree_find_target_v(const struct nilfs_bmap *btree, + const struct nilfs_btree_path *path, + __u64 key) +{ + __u64 ptr; + + ptr = nilfs_bmap_find_target_seq(btree, key); + if (ptr != NILFS_BMAP_INVALID_PTR) + /* sequential access */ + return ptr; + + ptr = nilfs_btree_find_near(btree, path); + if (ptr != NILFS_BMAP_INVALID_PTR) + /* near */ + return ptr; + + /* block group */ + return nilfs_bmap_find_target_in_group(btree); +} + +static int nilfs_btree_prepare_insert(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + int *levelp, __u64 key, __u64 ptr, + struct nilfs_bmap_stats *stats) +{ + struct buffer_head *bh; + struct nilfs_btree_node *node, *parent, *sib; + __u64 sibptr; + int pindex, level, ncmax, ncblk, ret; + struct inode *dat = NULL; + + stats->bs_nblocks = 0; + level = NILFS_BTREE_LEVEL_DATA; + + /* allocate a new ptr for data block */ + if (NILFS_BMAP_USE_VBN(btree)) { + path[level].bp_newreq.bpr_ptr = + nilfs_btree_find_target_v(btree, path, key); + dat = nilfs_bmap_get_dat(btree); + } + + ret = nilfs_bmap_prepare_alloc_ptr(btree, &path[level].bp_newreq, dat); + if (ret < 0) + goto err_out_data; + + ncblk = nilfs_btree_nchildren_per_block(btree); + + for (level = NILFS_BTREE_LEVEL_NODE_MIN; + level < nilfs_btree_height(btree) - 1; + level++) { + node = nilfs_btree_get_nonroot_node(path, level); + if (nilfs_btree_node_get_nchildren(node) < ncblk) { + path[level].bp_op = nilfs_btree_do_insert; + stats->bs_nblocks++; + goto out; + } + + parent = nilfs_btree_get_node(btree, path, level + 1, &ncmax); + pindex = path[level + 1].bp_index; + + /* left sibling */ + if (pindex > 0) { + sibptr = nilfs_btree_node_get_ptr(parent, pindex - 1, + ncmax); + ret = nilfs_btree_get_block(btree, sibptr, &bh); + if (ret < 0) + goto err_out_child_node; + sib = (struct nilfs_btree_node *)bh->b_data; + if (nilfs_btree_node_get_nchildren(sib) < ncblk) { + path[level].bp_sib_bh = bh; + path[level].bp_op = nilfs_btree_carry_left; + stats->bs_nblocks++; + goto out; + } else { + brelse(bh); + } + } + + /* right sibling */ + if (pindex < nilfs_btree_node_get_nchildren(parent) - 1) { + sibptr = nilfs_btree_node_get_ptr(parent, pindex + 1, + ncmax); + ret = nilfs_btree_get_block(btree, sibptr, &bh); + if (ret < 0) + goto err_out_child_node; + sib = (struct nilfs_btree_node *)bh->b_data; + if (nilfs_btree_node_get_nchildren(sib) < ncblk) { + path[level].bp_sib_bh = bh; + path[level].bp_op = nilfs_btree_carry_right; + stats->bs_nblocks++; + goto out; + } else { + brelse(bh); + } + } + + /* split */ + path[level].bp_newreq.bpr_ptr = + path[level - 1].bp_newreq.bpr_ptr + 1; + ret = nilfs_bmap_prepare_alloc_ptr(btree, + &path[level].bp_newreq, dat); + if (ret < 0) + goto err_out_child_node; + ret = nilfs_btree_get_new_block(btree, + path[level].bp_newreq.bpr_ptr, + &bh); + if (ret < 0) + goto err_out_curr_node; + + stats->bs_nblocks++; + + sib = (struct nilfs_btree_node *)bh->b_data; + nilfs_btree_node_init(sib, 0, level, 0, ncblk, NULL, NULL); + path[level].bp_sib_bh = bh; + path[level].bp_op = nilfs_btree_split; + } + + /* root */ + node = nilfs_btree_get_root(btree); + if (nilfs_btree_node_get_nchildren(node) < + NILFS_BTREE_ROOT_NCHILDREN_MAX) { + path[level].bp_op = nilfs_btree_do_insert; + stats->bs_nblocks++; + goto out; + } + + /* grow */ + path[level].bp_newreq.bpr_ptr = path[level - 1].bp_newreq.bpr_ptr + 1; + ret = nilfs_bmap_prepare_alloc_ptr(btree, &path[level].bp_newreq, dat); + if (ret < 0) + goto err_out_child_node; + ret = nilfs_btree_get_new_block(btree, path[level].bp_newreq.bpr_ptr, + &bh); + if (ret < 0) + goto err_out_curr_node; + + nilfs_btree_node_init((struct nilfs_btree_node *)bh->b_data, + 0, level, 0, ncblk, NULL, NULL); + path[level].bp_sib_bh = bh; + path[level].bp_op = nilfs_btree_grow; + + level++; + path[level].bp_op = nilfs_btree_do_insert; + + /* a newly-created node block and a data block are added */ + stats->bs_nblocks += 2; + + /* success */ + out: + *levelp = level; + return ret; + + /* error */ + err_out_curr_node: + nilfs_bmap_abort_alloc_ptr(btree, &path[level].bp_newreq, dat); + err_out_child_node: + for (level--; level > NILFS_BTREE_LEVEL_DATA; level--) { + nilfs_btnode_delete(path[level].bp_sib_bh); + nilfs_bmap_abort_alloc_ptr(btree, &path[level].bp_newreq, dat); + + } + + nilfs_bmap_abort_alloc_ptr(btree, &path[level].bp_newreq, dat); + err_out_data: + *levelp = level; + stats->bs_nblocks = 0; + return ret; +} + +static void nilfs_btree_commit_insert(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + int maxlevel, __u64 key, __u64 ptr) +{ + struct inode *dat = NULL; + int level; + + set_buffer_nilfs_volatile((struct buffer_head *)((unsigned long)ptr)); + ptr = path[NILFS_BTREE_LEVEL_DATA].bp_newreq.bpr_ptr; + if (NILFS_BMAP_USE_VBN(btree)) { + nilfs_bmap_set_target_v(btree, key, ptr); + dat = nilfs_bmap_get_dat(btree); + } + + for (level = NILFS_BTREE_LEVEL_NODE_MIN; level <= maxlevel; level++) { + nilfs_bmap_commit_alloc_ptr(btree, + &path[level - 1].bp_newreq, dat); + path[level].bp_op(btree, path, level, &key, &ptr); + } + + if (!nilfs_bmap_dirty(btree)) + nilfs_bmap_set_dirty(btree); +} + +static int nilfs_btree_insert(struct nilfs_bmap *btree, __u64 key, __u64 ptr) +{ + struct nilfs_btree_path *path; + struct nilfs_bmap_stats stats; + int level, ret; + + path = nilfs_btree_alloc_path(); + if (path == NULL) + return -ENOMEM; + + ret = nilfs_btree_do_lookup(btree, path, key, NULL, + NILFS_BTREE_LEVEL_NODE_MIN, 0); + if (ret != -ENOENT) { + if (ret == 0) + ret = -EEXIST; + goto out; + } + + ret = nilfs_btree_prepare_insert(btree, path, &level, key, ptr, &stats); + if (ret < 0) + goto out; + nilfs_btree_commit_insert(btree, path, level, key, ptr); + nilfs_inode_add_blocks(btree->b_inode, stats.bs_nblocks); + + out: + nilfs_btree_free_path(path); + return ret; +} + +static void nilfs_btree_do_delete(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + int level, __u64 *keyp, __u64 *ptrp) +{ + struct nilfs_btree_node *node; + int ncblk; + + if (level < nilfs_btree_height(btree) - 1) { + node = nilfs_btree_get_nonroot_node(path, level); + ncblk = nilfs_btree_nchildren_per_block(btree); + nilfs_btree_node_delete(node, path[level].bp_index, + keyp, ptrp, ncblk); + if (!buffer_dirty(path[level].bp_bh)) + mark_buffer_dirty(path[level].bp_bh); + if (path[level].bp_index == 0) + nilfs_btree_promote_key(btree, path, level + 1, + nilfs_btree_node_get_key(node, 0)); + } else { + node = nilfs_btree_get_root(btree); + nilfs_btree_node_delete(node, path[level].bp_index, + keyp, ptrp, + NILFS_BTREE_ROOT_NCHILDREN_MAX); + } +} + +static void nilfs_btree_borrow_left(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + int level, __u64 *keyp, __u64 *ptrp) +{ + struct nilfs_btree_node *node, *left; + int nchildren, lnchildren, n, ncblk; + + nilfs_btree_do_delete(btree, path, level, keyp, ptrp); + + node = nilfs_btree_get_nonroot_node(path, level); + left = nilfs_btree_get_sib_node(path, level); + nchildren = nilfs_btree_node_get_nchildren(node); + lnchildren = nilfs_btree_node_get_nchildren(left); + ncblk = nilfs_btree_nchildren_per_block(btree); + + n = (nchildren + lnchildren) / 2 - nchildren; + + nilfs_btree_node_move_right(left, node, n, ncblk, ncblk); + + if (!buffer_dirty(path[level].bp_bh)) + mark_buffer_dirty(path[level].bp_bh); + if (!buffer_dirty(path[level].bp_sib_bh)) + mark_buffer_dirty(path[level].bp_sib_bh); + + nilfs_btree_promote_key(btree, path, level + 1, + nilfs_btree_node_get_key(node, 0)); + + brelse(path[level].bp_sib_bh); + path[level].bp_sib_bh = NULL; + path[level].bp_index += n; +} + +static void nilfs_btree_borrow_right(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + int level, __u64 *keyp, __u64 *ptrp) +{ + struct nilfs_btree_node *node, *right; + int nchildren, rnchildren, n, ncblk; + + nilfs_btree_do_delete(btree, path, level, keyp, ptrp); + + node = nilfs_btree_get_nonroot_node(path, level); + right = nilfs_btree_get_sib_node(path, level); + nchildren = nilfs_btree_node_get_nchildren(node); + rnchildren = nilfs_btree_node_get_nchildren(right); + ncblk = nilfs_btree_nchildren_per_block(btree); + + n = (nchildren + rnchildren) / 2 - nchildren; + + nilfs_btree_node_move_left(node, right, n, ncblk, ncblk); + + if (!buffer_dirty(path[level].bp_bh)) + mark_buffer_dirty(path[level].bp_bh); + if (!buffer_dirty(path[level].bp_sib_bh)) + mark_buffer_dirty(path[level].bp_sib_bh); + + path[level + 1].bp_index++; + nilfs_btree_promote_key(btree, path, level + 1, + nilfs_btree_node_get_key(right, 0)); + path[level + 1].bp_index--; + + brelse(path[level].bp_sib_bh); + path[level].bp_sib_bh = NULL; +} + +static void nilfs_btree_concat_left(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + int level, __u64 *keyp, __u64 *ptrp) +{ + struct nilfs_btree_node *node, *left; + int n, ncblk; + + nilfs_btree_do_delete(btree, path, level, keyp, ptrp); + + node = nilfs_btree_get_nonroot_node(path, level); + left = nilfs_btree_get_sib_node(path, level); + ncblk = nilfs_btree_nchildren_per_block(btree); + + n = nilfs_btree_node_get_nchildren(node); + + nilfs_btree_node_move_left(left, node, n, ncblk, ncblk); + + if (!buffer_dirty(path[level].bp_sib_bh)) + mark_buffer_dirty(path[level].bp_sib_bh); + + nilfs_btnode_delete(path[level].bp_bh); + path[level].bp_bh = path[level].bp_sib_bh; + path[level].bp_sib_bh = NULL; + path[level].bp_index += nilfs_btree_node_get_nchildren(left); +} + +static void nilfs_btree_concat_right(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + int level, __u64 *keyp, __u64 *ptrp) +{ + struct nilfs_btree_node *node, *right; + int n, ncblk; + + nilfs_btree_do_delete(btree, path, level, keyp, ptrp); + + node = nilfs_btree_get_nonroot_node(path, level); + right = nilfs_btree_get_sib_node(path, level); + ncblk = nilfs_btree_nchildren_per_block(btree); + + n = nilfs_btree_node_get_nchildren(right); + + nilfs_btree_node_move_left(node, right, n, ncblk, ncblk); + + if (!buffer_dirty(path[level].bp_bh)) + mark_buffer_dirty(path[level].bp_bh); + + nilfs_btnode_delete(path[level].bp_sib_bh); + path[level].bp_sib_bh = NULL; + path[level + 1].bp_index++; +} + +static void nilfs_btree_shrink(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + int level, __u64 *keyp, __u64 *ptrp) +{ + struct nilfs_btree_node *root, *child; + int n, ncblk; + + nilfs_btree_do_delete(btree, path, level, keyp, ptrp); + + root = nilfs_btree_get_root(btree); + child = nilfs_btree_get_nonroot_node(path, level); + ncblk = nilfs_btree_nchildren_per_block(btree); + + nilfs_btree_node_delete(root, 0, NULL, NULL, + NILFS_BTREE_ROOT_NCHILDREN_MAX); + nilfs_btree_node_set_level(root, level); + n = nilfs_btree_node_get_nchildren(child); + nilfs_btree_node_move_left(root, child, n, + NILFS_BTREE_ROOT_NCHILDREN_MAX, ncblk); + + nilfs_btnode_delete(path[level].bp_bh); + path[level].bp_bh = NULL; +} + +/** + * nilfs_btree_discard - discard the last node for the mapping transformation + * @btree: bmap struct of btree + * @path: array of nilfs_btree_path struct + * @level: level of the B-tree node being operated on + * @keyp: argument for passing a key (unused) + * @ptrp: argument for passing a pointer (unused) + */ +static void nilfs_btree_discard(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, int level, + __u64 *keyp, __u64 *ptrp) +{ + struct nilfs_btree_node *root = nilfs_btree_get_root(btree); + + nilfs_btree_node_delete(root, 0, NULL, NULL, + NILFS_BTREE_ROOT_NCHILDREN_MAX); + nilfs_btree_node_set_level(root, level); + + nilfs_btnode_delete(path[level].bp_bh); + path[level].bp_bh = NULL; +} + +static void nilfs_btree_nop(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + int level, __u64 *keyp, __u64 *ptrp) +{ +} + +static int nilfs_btree_prepare_delete(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + int *levelp, + struct nilfs_bmap_stats *stats, + struct inode *dat, bool deform) +{ + struct buffer_head *bh; + struct nilfs_btree_node *node, *parent, *sib; + __u64 sibptr; + int pindex, dindex, level, ncmin, ncmax, ncblk, ret; + + ret = 0; + stats->bs_nblocks = 0; + ncmin = NILFS_BTREE_NODE_NCHILDREN_MIN(nilfs_btree_node_size(btree)); + ncblk = nilfs_btree_nchildren_per_block(btree); + + for (level = NILFS_BTREE_LEVEL_NODE_MIN, dindex = path[level].bp_index; + level < nilfs_btree_height(btree) - 1; + level++) { + node = nilfs_btree_get_nonroot_node(path, level); + path[level].bp_oldreq.bpr_ptr = + nilfs_btree_node_get_ptr(node, dindex, ncblk); + ret = nilfs_bmap_prepare_end_ptr(btree, + &path[level].bp_oldreq, dat); + if (ret < 0) + goto err_out_child_node; + + if (nilfs_btree_node_get_nchildren(node) > ncmin) { + path[level].bp_op = nilfs_btree_do_delete; + stats->bs_nblocks++; + goto out; + } + + parent = nilfs_btree_get_node(btree, path, level + 1, &ncmax); + pindex = path[level + 1].bp_index; + dindex = pindex; + + if (pindex > 0) { + /* left sibling */ + sibptr = nilfs_btree_node_get_ptr(parent, pindex - 1, + ncmax); + ret = nilfs_btree_get_block(btree, sibptr, &bh); + if (ret < 0) + goto err_out_curr_node; + sib = (struct nilfs_btree_node *)bh->b_data; + if (nilfs_btree_node_get_nchildren(sib) > ncmin) { + path[level].bp_sib_bh = bh; + path[level].bp_op = nilfs_btree_borrow_left; + stats->bs_nblocks++; + goto out; + } else { + path[level].bp_sib_bh = bh; + path[level].bp_op = nilfs_btree_concat_left; + stats->bs_nblocks++; + /* continue; */ + } + } else if (pindex < + nilfs_btree_node_get_nchildren(parent) - 1) { + /* right sibling */ + sibptr = nilfs_btree_node_get_ptr(parent, pindex + 1, + ncmax); + ret = nilfs_btree_get_block(btree, sibptr, &bh); + if (ret < 0) + goto err_out_curr_node; + sib = (struct nilfs_btree_node *)bh->b_data; + if (nilfs_btree_node_get_nchildren(sib) > ncmin) { + path[level].bp_sib_bh = bh; + path[level].bp_op = nilfs_btree_borrow_right; + stats->bs_nblocks++; + goto out; + } else { + path[level].bp_sib_bh = bh; + path[level].bp_op = nilfs_btree_concat_right; + stats->bs_nblocks++; + /* + * When merging right sibling node + * into the current node, pointer to + * the right sibling node must be + * terminated instead. The adjustment + * below is required for that. + */ + dindex = pindex + 1; + /* continue; */ + } + } else { + /* no siblings */ + /* the only child of the root node */ + WARN_ON(level != nilfs_btree_height(btree) - 2); + if (nilfs_btree_node_get_nchildren(node) - 1 <= + NILFS_BTREE_ROOT_NCHILDREN_MAX) { + path[level].bp_op = nilfs_btree_shrink; + } else if (deform) { + path[level].bp_op = nilfs_btree_discard; + } else { + path[level].bp_op = nilfs_btree_do_delete; + stats->bs_nblocks++; + goto out; + } + stats->bs_nblocks += 2; + level++; + path[level].bp_op = nilfs_btree_nop; + goto shrink_root_child; + } + } + + /* child of the root node is deleted */ + path[level].bp_op = nilfs_btree_do_delete; + stats->bs_nblocks++; + +shrink_root_child: + node = nilfs_btree_get_root(btree); + path[level].bp_oldreq.bpr_ptr = + nilfs_btree_node_get_ptr(node, dindex, + NILFS_BTREE_ROOT_NCHILDREN_MAX); + + ret = nilfs_bmap_prepare_end_ptr(btree, &path[level].bp_oldreq, dat); + if (ret < 0) + goto err_out_child_node; + + /* success */ + out: + *levelp = level; + return ret; + + /* error */ + err_out_curr_node: + nilfs_bmap_abort_end_ptr(btree, &path[level].bp_oldreq, dat); + err_out_child_node: + for (level--; level >= NILFS_BTREE_LEVEL_NODE_MIN; level--) { + brelse(path[level].bp_sib_bh); + nilfs_bmap_abort_end_ptr(btree, &path[level].bp_oldreq, dat); + } + *levelp = level; + stats->bs_nblocks = 0; + return ret; +} + +static void nilfs_btree_commit_delete(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + int maxlevel, struct inode *dat) +{ + int level; + + for (level = NILFS_BTREE_LEVEL_NODE_MIN; level <= maxlevel; level++) { + nilfs_bmap_commit_end_ptr(btree, &path[level].bp_oldreq, dat); + path[level].bp_op(btree, path, level, NULL, NULL); + } + + if (!nilfs_bmap_dirty(btree)) + nilfs_bmap_set_dirty(btree); +} + +static int nilfs_btree_delete(struct nilfs_bmap *btree, __u64 key, bool deform) + +{ + struct nilfs_btree_path *path; + struct nilfs_bmap_stats stats; + struct inode *dat; + int level, ret; + + path = nilfs_btree_alloc_path(); + if (path == NULL) + return -ENOMEM; + + ret = nilfs_btree_do_lookup(btree, path, key, NULL, + NILFS_BTREE_LEVEL_NODE_MIN, 0); + if (ret < 0) + goto out; + + + dat = NILFS_BMAP_USE_VBN(btree) ? nilfs_bmap_get_dat(btree) : NULL; + + ret = nilfs_btree_prepare_delete(btree, path, &level, &stats, dat, + deform); + if (ret < 0) + goto out; + nilfs_btree_commit_delete(btree, path, level, dat); + nilfs_inode_sub_blocks(btree->b_inode, stats.bs_nblocks); + +out: + nilfs_btree_free_path(path); + return ret; +} + +static int nilfs_btree_seek_key(const struct nilfs_bmap *btree, __u64 start, + __u64 *keyp) +{ + struct nilfs_btree_path *path; + const int minlevel = NILFS_BTREE_LEVEL_NODE_MIN; + int ret; + + path = nilfs_btree_alloc_path(); + if (!path) + return -ENOMEM; + + ret = nilfs_btree_do_lookup(btree, path, start, NULL, minlevel, 0); + if (!ret) + *keyp = start; + else if (ret == -ENOENT) + ret = nilfs_btree_get_next_key(btree, path, minlevel, keyp); + + nilfs_btree_free_path(path); + return ret; +} + +static int nilfs_btree_last_key(const struct nilfs_bmap *btree, __u64 *keyp) +{ + struct nilfs_btree_path *path; + int ret; + + path = nilfs_btree_alloc_path(); + if (path == NULL) + return -ENOMEM; + + ret = nilfs_btree_do_lookup_last(btree, path, keyp, NULL); + + nilfs_btree_free_path(path); + + return ret; +} + +static int nilfs_btree_check_delete(struct nilfs_bmap *btree, __u64 key) +{ + struct buffer_head *bh; + struct nilfs_btree_node *root, *node; + __u64 maxkey, nextmaxkey; + __u64 ptr; + int nchildren, ret; + + root = nilfs_btree_get_root(btree); + nchildren = nilfs_btree_node_get_nchildren(root); + if (unlikely(nchildren == 0)) + return 0; + + switch (nilfs_btree_height(btree)) { + case 2: + bh = NULL; + node = root; + break; + case 3: + if (nchildren > 1) + return 0; + ptr = nilfs_btree_node_get_ptr(root, nchildren - 1, + NILFS_BTREE_ROOT_NCHILDREN_MAX); + ret = nilfs_btree_get_block(btree, ptr, &bh); + if (ret < 0) + return ret; + node = (struct nilfs_btree_node *)bh->b_data; + nchildren = nilfs_btree_node_get_nchildren(node); + break; + default: + return 0; + } + + maxkey = nilfs_btree_node_get_key(node, nchildren - 1); + nextmaxkey = (nchildren > 1) ? + nilfs_btree_node_get_key(node, nchildren - 2) : 0; + brelse(bh); + + return (maxkey == key) && (nextmaxkey < NILFS_BMAP_LARGE_LOW); +} + +static int nilfs_btree_gather_data(struct nilfs_bmap *btree, + __u64 *keys, __u64 *ptrs, int nitems) +{ + struct buffer_head *bh; + struct nilfs_btree_node *node, *root; + __le64 *dkeys; + __le64 *dptrs; + __u64 ptr; + int nchildren, ncmax, i, ret; + + root = nilfs_btree_get_root(btree); + switch (nilfs_btree_height(btree)) { + case 2: + bh = NULL; + node = root; + ncmax = NILFS_BTREE_ROOT_NCHILDREN_MAX; + break; + case 3: + nchildren = nilfs_btree_node_get_nchildren(root); + WARN_ON(nchildren > 1); + ptr = nilfs_btree_node_get_ptr(root, nchildren - 1, + NILFS_BTREE_ROOT_NCHILDREN_MAX); + ret = nilfs_btree_get_block(btree, ptr, &bh); + if (ret < 0) + return ret; + node = (struct nilfs_btree_node *)bh->b_data; + ncmax = nilfs_btree_nchildren_per_block(btree); + break; + default: + node = NULL; + return -EINVAL; + } + + nchildren = nilfs_btree_node_get_nchildren(node); + if (nchildren < nitems) + nitems = nchildren; + dkeys = nilfs_btree_node_dkeys(node); + dptrs = nilfs_btree_node_dptrs(node, ncmax); + for (i = 0; i < nitems; i++) { + keys[i] = le64_to_cpu(dkeys[i]); + ptrs[i] = le64_to_cpu(dptrs[i]); + } + + brelse(bh); + + return nitems; +} + +static int +nilfs_btree_prepare_convert_and_insert(struct nilfs_bmap *btree, __u64 key, + union nilfs_bmap_ptr_req *dreq, + union nilfs_bmap_ptr_req *nreq, + struct buffer_head **bhp, + struct nilfs_bmap_stats *stats) +{ + struct buffer_head *bh; + struct inode *dat = NULL; + int ret; + + stats->bs_nblocks = 0; + + /* for data */ + /* cannot find near ptr */ + if (NILFS_BMAP_USE_VBN(btree)) { + dreq->bpr_ptr = nilfs_btree_find_target_v(btree, NULL, key); + dat = nilfs_bmap_get_dat(btree); + } + + ret = nilfs_attach_btree_node_cache(&NILFS_BMAP_I(btree)->vfs_inode); + if (ret < 0) + return ret; + + ret = nilfs_bmap_prepare_alloc_ptr(btree, dreq, dat); + if (ret < 0) + return ret; + + *bhp = NULL; + stats->bs_nblocks++; + if (nreq != NULL) { + nreq->bpr_ptr = dreq->bpr_ptr + 1; + ret = nilfs_bmap_prepare_alloc_ptr(btree, nreq, dat); + if (ret < 0) + goto err_out_dreq; + + ret = nilfs_btree_get_new_block(btree, nreq->bpr_ptr, &bh); + if (ret < 0) + goto err_out_nreq; + + *bhp = bh; + stats->bs_nblocks++; + } + + /* success */ + return 0; + + /* error */ + err_out_nreq: + nilfs_bmap_abort_alloc_ptr(btree, nreq, dat); + err_out_dreq: + nilfs_bmap_abort_alloc_ptr(btree, dreq, dat); + stats->bs_nblocks = 0; + return ret; + +} + +static void +nilfs_btree_commit_convert_and_insert(struct nilfs_bmap *btree, + __u64 key, __u64 ptr, + const __u64 *keys, const __u64 *ptrs, + int n, + union nilfs_bmap_ptr_req *dreq, + union nilfs_bmap_ptr_req *nreq, + struct buffer_head *bh) +{ + struct nilfs_btree_node *node; + struct inode *dat; + __u64 tmpptr; + int ncblk; + + /* free resources */ + if (btree->b_ops->bop_clear != NULL) + btree->b_ops->bop_clear(btree); + + /* ptr must be a pointer to a buffer head. */ + set_buffer_nilfs_volatile((struct buffer_head *)((unsigned long)ptr)); + + /* convert and insert */ + dat = NILFS_BMAP_USE_VBN(btree) ? nilfs_bmap_get_dat(btree) : NULL; + __nilfs_btree_init(btree); + if (nreq != NULL) { + nilfs_bmap_commit_alloc_ptr(btree, dreq, dat); + nilfs_bmap_commit_alloc_ptr(btree, nreq, dat); + + /* create child node at level 1 */ + node = (struct nilfs_btree_node *)bh->b_data; + ncblk = nilfs_btree_nchildren_per_block(btree); + nilfs_btree_node_init(node, 0, 1, n, ncblk, keys, ptrs); + nilfs_btree_node_insert(node, n, key, dreq->bpr_ptr, ncblk); + if (!buffer_dirty(bh)) + mark_buffer_dirty(bh); + if (!nilfs_bmap_dirty(btree)) + nilfs_bmap_set_dirty(btree); + + brelse(bh); + + /* create root node at level 2 */ + node = nilfs_btree_get_root(btree); + tmpptr = nreq->bpr_ptr; + nilfs_btree_node_init(node, NILFS_BTREE_NODE_ROOT, 2, 1, + NILFS_BTREE_ROOT_NCHILDREN_MAX, + &keys[0], &tmpptr); + } else { + nilfs_bmap_commit_alloc_ptr(btree, dreq, dat); + + /* create root node at level 1 */ + node = nilfs_btree_get_root(btree); + nilfs_btree_node_init(node, NILFS_BTREE_NODE_ROOT, 1, n, + NILFS_BTREE_ROOT_NCHILDREN_MAX, + keys, ptrs); + nilfs_btree_node_insert(node, n, key, dreq->bpr_ptr, + NILFS_BTREE_ROOT_NCHILDREN_MAX); + if (!nilfs_bmap_dirty(btree)) + nilfs_bmap_set_dirty(btree); + } + + if (NILFS_BMAP_USE_VBN(btree)) + nilfs_bmap_set_target_v(btree, key, dreq->bpr_ptr); +} + +/** + * nilfs_btree_convert_and_insert - Convert and insert entries into a B-tree + * @btree: NILFS B-tree structure + * @key: Key of the new entry to be inserted + * @ptr: Pointer (block number) associated with the key to be inserted + * @keys: Array of keys to be inserted in addition to @key + * @ptrs: Array of pointers associated with @keys + * @n: Number of keys and pointers in @keys and @ptrs + * + * This function is used to insert a new entry specified by @key and @ptr, + * along with additional entries specified by @keys and @ptrs arrays, into a + * NILFS B-tree. + * It prepares the necessary changes by allocating the required blocks and any + * necessary intermediate nodes. It converts configurations from other forms of + * block mapping (the one that currently exists is direct mapping) to a B-tree. + * + * Return: 0 on success or a negative error code on failure. + */ +int nilfs_btree_convert_and_insert(struct nilfs_bmap *btree, + __u64 key, __u64 ptr, + const __u64 *keys, const __u64 *ptrs, int n) +{ + struct buffer_head *bh = NULL; + union nilfs_bmap_ptr_req dreq, nreq, *di, *ni; + struct nilfs_bmap_stats stats; + int ret; + + if (n + 1 <= NILFS_BTREE_ROOT_NCHILDREN_MAX) { + di = &dreq; + ni = NULL; + } else if ((n + 1) <= NILFS_BTREE_NODE_NCHILDREN_MAX( + nilfs_btree_node_size(btree))) { + di = &dreq; + ni = &nreq; + } else { + di = NULL; + ni = NULL; + BUG(); + } + + ret = nilfs_btree_prepare_convert_and_insert(btree, key, di, ni, &bh, + &stats); + if (ret < 0) + return ret; + nilfs_btree_commit_convert_and_insert(btree, key, ptr, keys, ptrs, n, + di, ni, bh); + nilfs_inode_add_blocks(btree->b_inode, stats.bs_nblocks); + return 0; +} + +static int nilfs_btree_propagate_p(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + int level, + struct buffer_head *bh) +{ + while ((++level < nilfs_btree_height(btree) - 1) && + !buffer_dirty(path[level].bp_bh)) + mark_buffer_dirty(path[level].bp_bh); + + return 0; +} + +static int nilfs_btree_prepare_update_v(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + int level, struct inode *dat) +{ + struct nilfs_btree_node *parent; + int ncmax, ret; + + parent = nilfs_btree_get_node(btree, path, level + 1, &ncmax); + path[level].bp_oldreq.bpr_ptr = + nilfs_btree_node_get_ptr(parent, path[level + 1].bp_index, + ncmax); + path[level].bp_newreq.bpr_ptr = path[level].bp_oldreq.bpr_ptr + 1; + ret = nilfs_dat_prepare_update(dat, &path[level].bp_oldreq.bpr_req, + &path[level].bp_newreq.bpr_req); + if (ret < 0) + return ret; + + if (buffer_nilfs_node(path[level].bp_bh)) { + path[level].bp_ctxt.oldkey = path[level].bp_oldreq.bpr_ptr; + path[level].bp_ctxt.newkey = path[level].bp_newreq.bpr_ptr; + path[level].bp_ctxt.bh = path[level].bp_bh; + ret = nilfs_btnode_prepare_change_key( + NILFS_BMAP_I(btree)->i_assoc_inode->i_mapping, + &path[level].bp_ctxt); + if (ret < 0) { + nilfs_dat_abort_update(dat, + &path[level].bp_oldreq.bpr_req, + &path[level].bp_newreq.bpr_req); + return ret; + } + } + + return 0; +} + +static void nilfs_btree_commit_update_v(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + int level, struct inode *dat) +{ + struct nilfs_btree_node *parent; + int ncmax; + + nilfs_dat_commit_update(dat, &path[level].bp_oldreq.bpr_req, + &path[level].bp_newreq.bpr_req, + btree->b_ptr_type == NILFS_BMAP_PTR_VS); + + if (buffer_nilfs_node(path[level].bp_bh)) { + nilfs_btnode_commit_change_key( + NILFS_BMAP_I(btree)->i_assoc_inode->i_mapping, + &path[level].bp_ctxt); + path[level].bp_bh = path[level].bp_ctxt.bh; + } + set_buffer_nilfs_volatile(path[level].bp_bh); + + parent = nilfs_btree_get_node(btree, path, level + 1, &ncmax); + nilfs_btree_node_set_ptr(parent, path[level + 1].bp_index, + path[level].bp_newreq.bpr_ptr, ncmax); +} + +static void nilfs_btree_abort_update_v(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + int level, struct inode *dat) +{ + nilfs_dat_abort_update(dat, &path[level].bp_oldreq.bpr_req, + &path[level].bp_newreq.bpr_req); + if (buffer_nilfs_node(path[level].bp_bh)) + nilfs_btnode_abort_change_key( + NILFS_BMAP_I(btree)->i_assoc_inode->i_mapping, + &path[level].bp_ctxt); +} + +static int nilfs_btree_prepare_propagate_v(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + int minlevel, int *maxlevelp, + struct inode *dat) +{ + int level, ret; + + level = minlevel; + if (!buffer_nilfs_volatile(path[level].bp_bh)) { + ret = nilfs_btree_prepare_update_v(btree, path, level, dat); + if (ret < 0) + return ret; + } + while ((++level < nilfs_btree_height(btree) - 1) && + !buffer_dirty(path[level].bp_bh)) { + + WARN_ON(buffer_nilfs_volatile(path[level].bp_bh)); + ret = nilfs_btree_prepare_update_v(btree, path, level, dat); + if (ret < 0) + goto out; + } + + /* success */ + *maxlevelp = level - 1; + return 0; + + /* error */ + out: + while (--level > minlevel) + nilfs_btree_abort_update_v(btree, path, level, dat); + if (!buffer_nilfs_volatile(path[level].bp_bh)) + nilfs_btree_abort_update_v(btree, path, level, dat); + return ret; +} + +static void nilfs_btree_commit_propagate_v(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + int minlevel, int maxlevel, + struct buffer_head *bh, + struct inode *dat) +{ + int level; + + if (!buffer_nilfs_volatile(path[minlevel].bp_bh)) + nilfs_btree_commit_update_v(btree, path, minlevel, dat); + + for (level = minlevel + 1; level <= maxlevel; level++) + nilfs_btree_commit_update_v(btree, path, level, dat); +} + +static int nilfs_btree_propagate_v(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + int level, struct buffer_head *bh) +{ + int maxlevel = 0, ret; + struct nilfs_btree_node *parent; + struct inode *dat = nilfs_bmap_get_dat(btree); + __u64 ptr; + int ncmax; + + get_bh(bh); + path[level].bp_bh = bh; + ret = nilfs_btree_prepare_propagate_v(btree, path, level, &maxlevel, + dat); + if (ret < 0) + goto out; + + if (buffer_nilfs_volatile(path[level].bp_bh)) { + parent = nilfs_btree_get_node(btree, path, level + 1, &ncmax); + ptr = nilfs_btree_node_get_ptr(parent, + path[level + 1].bp_index, + ncmax); + ret = nilfs_dat_mark_dirty(dat, ptr); + if (ret < 0) + goto out; + } + + nilfs_btree_commit_propagate_v(btree, path, level, maxlevel, bh, dat); + + out: + brelse(path[level].bp_bh); + path[level].bp_bh = NULL; + return ret; +} + +static int nilfs_btree_propagate(struct nilfs_bmap *btree, + struct buffer_head *bh) +{ + struct nilfs_btree_path *path; + struct nilfs_btree_node *node; + __u64 key; + int level, ret; + + WARN_ON(!buffer_dirty(bh)); + + path = nilfs_btree_alloc_path(); + if (path == NULL) + return -ENOMEM; + + if (buffer_nilfs_node(bh)) { + node = (struct nilfs_btree_node *)bh->b_data; + key = nilfs_btree_node_get_key(node, 0); + level = nilfs_btree_node_get_level(node); + } else { + key = nilfs_bmap_data_get_key(btree, bh); + level = NILFS_BTREE_LEVEL_DATA; + } + + ret = nilfs_btree_do_lookup(btree, path, key, NULL, level + 1, 0); + if (ret < 0) { + if (unlikely(ret == -ENOENT)) { + nilfs_crit(btree->b_inode->i_sb, + "writing node/leaf block does not appear in b-tree (ino=%llu) at key=%llu, level=%d", + btree->b_inode->i_ino, + (unsigned long long)key, level); + ret = -EINVAL; + } + goto out; + } + + ret = NILFS_BMAP_USE_VBN(btree) ? + nilfs_btree_propagate_v(btree, path, level, bh) : + nilfs_btree_propagate_p(btree, path, level, bh); + + out: + nilfs_btree_free_path(path); + + return ret; +} + +static int nilfs_btree_propagate_gc(struct nilfs_bmap *btree, + struct buffer_head *bh) +{ + return nilfs_dat_mark_dirty(nilfs_bmap_get_dat(btree), bh->b_blocknr); +} + +static void nilfs_btree_add_dirty_buffer(struct nilfs_bmap *btree, + struct list_head *lists, + struct buffer_head *bh) +{ + struct list_head *head; + struct buffer_head *cbh; + struct nilfs_btree_node *node, *cnode; + __u64 key, ckey; + int level; + + get_bh(bh); + node = (struct nilfs_btree_node *)bh->b_data; + key = nilfs_btree_node_get_key(node, 0); + level = nilfs_btree_node_get_level(node); + if (level < NILFS_BTREE_LEVEL_NODE_MIN || + level >= NILFS_BTREE_LEVEL_MAX) { + dump_stack(); + nilfs_warn(btree->b_inode->i_sb, + "invalid btree level: %d (key=%llu, ino=%llu, blocknr=%llu)", + level, (unsigned long long)key, + btree->b_inode->i_ino, + (unsigned long long)bh->b_blocknr); + return; + } + + list_for_each(head, &lists[level]) { + cbh = list_entry(head, struct buffer_head, b_assoc_buffers); + cnode = (struct nilfs_btree_node *)cbh->b_data; + ckey = nilfs_btree_node_get_key(cnode, 0); + if (key < ckey) + break; + } + list_add_tail(&bh->b_assoc_buffers, head); +} + +static void nilfs_btree_lookup_dirty_buffers(struct nilfs_bmap *btree, + struct list_head *listp) +{ + struct inode *btnc_inode = NILFS_BMAP_I(btree)->i_assoc_inode; + struct address_space *btcache = btnc_inode->i_mapping; + struct list_head lists[NILFS_BTREE_LEVEL_MAX]; + struct folio_batch fbatch; + struct buffer_head *bh, *head; + pgoff_t index = 0; + int level, i; + + for (level = NILFS_BTREE_LEVEL_NODE_MIN; + level < NILFS_BTREE_LEVEL_MAX; + level++) + INIT_LIST_HEAD(&lists[level]); + + folio_batch_init(&fbatch); + + while (filemap_get_folios_tag(btcache, &index, (pgoff_t)-1, + PAGECACHE_TAG_DIRTY, &fbatch)) { + for (i = 0; i < folio_batch_count(&fbatch); i++) { + bh = head = folio_buffers(fbatch.folios[i]); + do { + if (buffer_dirty(bh)) + nilfs_btree_add_dirty_buffer(btree, + lists, bh); + } while ((bh = bh->b_this_page) != head); + } + folio_batch_release(&fbatch); + cond_resched(); + } + + for (level = NILFS_BTREE_LEVEL_NODE_MIN; + level < NILFS_BTREE_LEVEL_MAX; + level++) + list_splice_tail(&lists[level], listp); +} + +static int nilfs_btree_assign_p(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + int level, + struct buffer_head **bh, + sector_t blocknr, + union nilfs_binfo *binfo) +{ + struct nilfs_btree_node *parent; + __u64 key; + __u64 ptr; + int ncmax, ret; + + parent = nilfs_btree_get_node(btree, path, level + 1, &ncmax); + ptr = nilfs_btree_node_get_ptr(parent, path[level + 1].bp_index, + ncmax); + if (buffer_nilfs_node(*bh)) { + path[level].bp_ctxt.oldkey = ptr; + path[level].bp_ctxt.newkey = blocknr; + path[level].bp_ctxt.bh = *bh; + ret = nilfs_btnode_prepare_change_key( + NILFS_BMAP_I(btree)->i_assoc_inode->i_mapping, + &path[level].bp_ctxt); + if (ret < 0) + return ret; + nilfs_btnode_commit_change_key( + NILFS_BMAP_I(btree)->i_assoc_inode->i_mapping, + &path[level].bp_ctxt); + *bh = path[level].bp_ctxt.bh; + } + + nilfs_btree_node_set_ptr(parent, path[level + 1].bp_index, blocknr, + ncmax); + + key = nilfs_btree_node_get_key(parent, path[level + 1].bp_index); + /* on-disk format */ + binfo->bi_dat.bi_blkoff = cpu_to_le64(key); + binfo->bi_dat.bi_level = level; + memset(binfo->bi_dat.bi_pad, 0, sizeof(binfo->bi_dat.bi_pad)); + + return 0; +} + +static int nilfs_btree_assign_v(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, + int level, + struct buffer_head **bh, + sector_t blocknr, + union nilfs_binfo *binfo) +{ + struct nilfs_btree_node *parent; + struct inode *dat = nilfs_bmap_get_dat(btree); + __u64 key; + __u64 ptr; + union nilfs_bmap_ptr_req req; + int ncmax, ret; + + parent = nilfs_btree_get_node(btree, path, level + 1, &ncmax); + ptr = nilfs_btree_node_get_ptr(parent, path[level + 1].bp_index, + ncmax); + req.bpr_ptr = ptr; + ret = nilfs_dat_prepare_start(dat, &req.bpr_req); + if (ret < 0) + return ret; + nilfs_dat_commit_start(dat, &req.bpr_req, blocknr); + + key = nilfs_btree_node_get_key(parent, path[level + 1].bp_index); + /* on-disk format */ + binfo->bi_v.bi_vblocknr = cpu_to_le64(ptr); + binfo->bi_v.bi_blkoff = cpu_to_le64(key); + + return 0; +} + +static int nilfs_btree_assign(struct nilfs_bmap *btree, + struct buffer_head **bh, + sector_t blocknr, + union nilfs_binfo *binfo) +{ + struct nilfs_btree_path *path; + struct nilfs_btree_node *node; + __u64 key; + int level, ret; + + path = nilfs_btree_alloc_path(); + if (path == NULL) + return -ENOMEM; + + if (buffer_nilfs_node(*bh)) { + node = (struct nilfs_btree_node *)(*bh)->b_data; + key = nilfs_btree_node_get_key(node, 0); + level = nilfs_btree_node_get_level(node); + } else { + key = nilfs_bmap_data_get_key(btree, *bh); + level = NILFS_BTREE_LEVEL_DATA; + } + + ret = nilfs_btree_do_lookup(btree, path, key, NULL, level + 1, 0); + if (ret < 0) { + WARN_ON(ret == -ENOENT); + goto out; + } + + ret = NILFS_BMAP_USE_VBN(btree) ? + nilfs_btree_assign_v(btree, path, level, bh, blocknr, binfo) : + nilfs_btree_assign_p(btree, path, level, bh, blocknr, binfo); + + out: + nilfs_btree_free_path(path); + + return ret; +} + +static int nilfs_btree_assign_gc(struct nilfs_bmap *btree, + struct buffer_head **bh, + sector_t blocknr, + union nilfs_binfo *binfo) +{ + struct nilfs_btree_node *node; + __u64 key; + int ret; + + ret = nilfs_dat_move(nilfs_bmap_get_dat(btree), (*bh)->b_blocknr, + blocknr); + if (ret < 0) + return ret; + + if (buffer_nilfs_node(*bh)) { + node = (struct nilfs_btree_node *)(*bh)->b_data; + key = nilfs_btree_node_get_key(node, 0); + } else + key = nilfs_bmap_data_get_key(btree, *bh); + + /* on-disk format */ + binfo->bi_v.bi_vblocknr = cpu_to_le64((*bh)->b_blocknr); + binfo->bi_v.bi_blkoff = cpu_to_le64(key); + + return 0; +} + +static int nilfs_btree_mark(struct nilfs_bmap *btree, __u64 key, int level) +{ + struct buffer_head *bh; + struct nilfs_btree_path *path; + __u64 ptr; + int ret; + + path = nilfs_btree_alloc_path(); + if (path == NULL) + return -ENOMEM; + + ret = nilfs_btree_do_lookup(btree, path, key, &ptr, level + 1, 0); + if (ret < 0) { + WARN_ON(ret == -ENOENT); + goto out; + } + ret = nilfs_btree_get_block(btree, ptr, &bh); + if (ret < 0) { + WARN_ON(ret == -ENOENT); + goto out; + } + + if (!buffer_dirty(bh)) + mark_buffer_dirty(bh); + brelse(bh); + if (!nilfs_bmap_dirty(btree)) + nilfs_bmap_set_dirty(btree); + + out: + nilfs_btree_free_path(path); + return ret; +} + +static const struct nilfs_bmap_operations nilfs_btree_ops = { + .bop_lookup = nilfs_btree_lookup, + .bop_lookup_contig = nilfs_btree_lookup_contig, + .bop_insert = nilfs_btree_insert, + .bop_delete = nilfs_btree_delete, + .bop_clear = NULL, + + .bop_propagate = nilfs_btree_propagate, + + .bop_lookup_dirty_buffers = nilfs_btree_lookup_dirty_buffers, + + .bop_assign = nilfs_btree_assign, + .bop_mark = nilfs_btree_mark, + + .bop_seek_key = nilfs_btree_seek_key, + .bop_last_key = nilfs_btree_last_key, + + .bop_check_insert = NULL, + .bop_check_delete = nilfs_btree_check_delete, + .bop_gather_data = nilfs_btree_gather_data, +}; + +static const struct nilfs_bmap_operations nilfs_btree_ops_gc = { + .bop_lookup = NULL, + .bop_lookup_contig = NULL, + .bop_insert = NULL, + .bop_delete = NULL, + .bop_clear = NULL, + + .bop_propagate = nilfs_btree_propagate_gc, + + .bop_lookup_dirty_buffers = nilfs_btree_lookup_dirty_buffers, + + .bop_assign = nilfs_btree_assign_gc, + .bop_mark = NULL, + + .bop_seek_key = NULL, + .bop_last_key = NULL, + + .bop_check_insert = NULL, + .bop_check_delete = NULL, + .bop_gather_data = NULL, +}; + +static void __nilfs_btree_init(struct nilfs_bmap *bmap) +{ + bmap->b_ops = &nilfs_btree_ops; + bmap->b_nchildren_per_block = + NILFS_BTREE_NODE_NCHILDREN_MAX(nilfs_btree_node_size(bmap)); +} + +int nilfs_btree_init(struct nilfs_bmap *bmap) +{ + int ret = 0; + + __nilfs_btree_init(bmap); + + if (nilfs_btree_root_broken(nilfs_btree_get_root(bmap), bmap->b_inode)) + ret = -EIO; + else + ret = nilfs_attach_btree_node_cache( + &NILFS_BMAP_I(bmap)->vfs_inode); + + return ret; +} + +void nilfs_btree_init_gc(struct nilfs_bmap *bmap) +{ + bmap->b_ops = &nilfs_btree_ops_gc; + bmap->b_nchildren_per_block = + NILFS_BTREE_NODE_NCHILDREN_MAX(nilfs_btree_node_size(bmap)); +} diff --git a/fs/nilfs2/btree.h b/fs/nilfs2/btree.h new file mode 100644 index 000000000..2a220f716 --- /dev/null +++ b/fs/nilfs2/btree.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * NILFS B-tree. + * + * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Koji Sato. + */ + +#ifndef _NILFS_BTREE_H +#define _NILFS_BTREE_H + +#include +#include +#include +#include /* nilfs_btree_node */ +#include "btnode.h" +#include "bmap.h" + +/** + * struct nilfs_btree_path - A path on which B-tree operations are executed + * @bp_bh: buffer head of node block + * @bp_sib_bh: buffer head of sibling node block + * @bp_index: index of child node + * @bp_oldreq: ptr end request for old ptr + * @bp_newreq: ptr alloc request for new ptr + * @bp_ctxt: context information for changing the key of a b-tree node block + * @bp_op: rebalance operation + */ +struct nilfs_btree_path { + struct buffer_head *bp_bh; + struct buffer_head *bp_sib_bh; + int bp_index; + union nilfs_bmap_ptr_req bp_oldreq; + union nilfs_bmap_ptr_req bp_newreq; + struct nilfs_btnode_chkey_ctxt bp_ctxt; + void (*bp_op)(struct nilfs_bmap *, struct nilfs_btree_path *, + int, __u64 *, __u64 *); +}; + +#define NILFS_BTREE_ROOT_SIZE NILFS_BMAP_SIZE +#define NILFS_BTREE_ROOT_NCHILDREN_MAX \ + ((NILFS_BTREE_ROOT_SIZE - sizeof(struct nilfs_btree_node)) / \ + (sizeof(__le64 /* dkey */) + sizeof(__le64 /* dptr */))) +#define NILFS_BTREE_ROOT_NCHILDREN_MIN 0 +#define NILFS_BTREE_NODE_EXTRA_PAD_SIZE (sizeof(__le64)) +#define NILFS_BTREE_NODE_NCHILDREN_MAX(nodesize) \ + (((nodesize) - sizeof(struct nilfs_btree_node) - \ + NILFS_BTREE_NODE_EXTRA_PAD_SIZE) / \ + (sizeof(__le64 /* dkey */) + sizeof(__le64 /* dptr */))) +#define NILFS_BTREE_NODE_NCHILDREN_MIN(nodesize) \ + ((NILFS_BTREE_NODE_NCHILDREN_MAX(nodesize) - 1) / 2 + 1) +#define NILFS_BTREE_KEY_MIN ((__u64)0) +#define NILFS_BTREE_KEY_MAX (~(__u64)0) + +extern struct kmem_cache *nilfs_btree_path_cache; + +int nilfs_btree_init(struct nilfs_bmap *); +int nilfs_btree_convert_and_insert(struct nilfs_bmap *, __u64, __u64, + const __u64 *, const __u64 *, int); +void nilfs_btree_init_gc(struct nilfs_bmap *); + +int nilfs_btree_broken_node_block(struct buffer_head *bh); + +#endif /* _NILFS_BTREE_H */ diff --git a/fs/nilfs2/cpfile.c b/fs/nilfs2/cpfile.c new file mode 100644 index 000000000..d3349fa58 --- /dev/null +++ b/fs/nilfs2/cpfile.c @@ -0,0 +1,1184 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * NILFS checkpoint file. + * + * Copyright (C) 2006-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Koji Sato. + */ + +#include +#include +#include +#include +#include +#include "mdt.h" +#include "cpfile.h" + + +static inline unsigned long +nilfs_cpfile_checkpoints_per_block(const struct inode *cpfile) +{ + return NILFS_MDT(cpfile)->mi_entries_per_block; +} + +/* block number from the beginning of the file */ +static unsigned long +nilfs_cpfile_get_blkoff(const struct inode *cpfile, __u64 cno) +{ + __u64 tcno = cno + NILFS_MDT(cpfile)->mi_first_entry_offset - 1; + + tcno = div64_ul(tcno, nilfs_cpfile_checkpoints_per_block(cpfile)); + return (unsigned long)tcno; +} + +/* offset in block */ +static unsigned long +nilfs_cpfile_get_offset(const struct inode *cpfile, __u64 cno) +{ + __u64 tcno = cno + NILFS_MDT(cpfile)->mi_first_entry_offset - 1; + + return do_div(tcno, nilfs_cpfile_checkpoints_per_block(cpfile)); +} + +static __u64 nilfs_cpfile_first_checkpoint_in_block(const struct inode *cpfile, + unsigned long blkoff) +{ + return (__u64)nilfs_cpfile_checkpoints_per_block(cpfile) * blkoff + + 1 - NILFS_MDT(cpfile)->mi_first_entry_offset; +} + +static unsigned long +nilfs_cpfile_checkpoints_in_block(const struct inode *cpfile, + __u64 curr, + __u64 max) +{ + return min_t(__u64, + nilfs_cpfile_checkpoints_per_block(cpfile) - + nilfs_cpfile_get_offset(cpfile, curr), + max - curr); +} + +static inline int nilfs_cpfile_is_in_first(const struct inode *cpfile, + __u64 cno) +{ + return nilfs_cpfile_get_blkoff(cpfile, cno) == 0; +} + +static unsigned int +nilfs_cpfile_block_add_valid_checkpoints(const struct inode *cpfile, + struct buffer_head *bh, + unsigned int n) +{ + struct nilfs_checkpoint *cp; + unsigned int count; + + cp = kmap_local_folio(bh->b_folio, + offset_in_folio(bh->b_folio, bh->b_data)); + count = le32_to_cpu(cp->cp_checkpoints_count) + n; + cp->cp_checkpoints_count = cpu_to_le32(count); + kunmap_local(cp); + return count; +} + +static int +nilfs_cpfile_block_sub_valid_checkpoints(const struct inode *cpfile, + struct buffer_head *bh, + unsigned int n) +{ + struct nilfs_checkpoint *cp; + unsigned int checkpoints_count; + int count; + + cp = kmap_local_folio(bh->b_folio, + offset_in_folio(bh->b_folio, bh->b_data)); + checkpoints_count = le32_to_cpu(cp->cp_checkpoints_count); + if (unlikely(checkpoints_count < n)) { + nilfs_error(cpfile->i_sb, + "deleted checkpoints count %u exceeds block count %u", + n, checkpoints_count); + kunmap_local(cp); + return -EIO; + } + count = checkpoints_count - n; + cp->cp_checkpoints_count = cpu_to_le32(count); + kunmap_local(cp); + return count; +} + +static void nilfs_cpfile_block_init(struct inode *cpfile, + struct buffer_head *bh, + void *from) +{ + struct nilfs_checkpoint *cp = from; + size_t cpsz = NILFS_MDT(cpfile)->mi_entry_size; + int n = nilfs_cpfile_checkpoints_per_block(cpfile); + + while (n-- > 0) { + nilfs_checkpoint_set_invalid(cp); + cp = (void *)cp + cpsz; + } +} + +/** + * nilfs_cpfile_checkpoint_offset - calculate the byte offset of a checkpoint + * entry in the folio containing it + * @cpfile: checkpoint file inode + * @cno: checkpoint number + * @bh: buffer head of block containing checkpoint indexed by @cno + * + * Return: Byte offset in the folio of the checkpoint specified by @cno. + */ +static size_t nilfs_cpfile_checkpoint_offset(const struct inode *cpfile, + __u64 cno, + struct buffer_head *bh) +{ + return offset_in_folio(bh->b_folio, bh->b_data) + + nilfs_cpfile_get_offset(cpfile, cno) * + NILFS_MDT(cpfile)->mi_entry_size; +} + +/** + * nilfs_cpfile_cp_snapshot_list_offset - calculate the byte offset of a + * checkpoint snapshot list in the folio + * containing it + * @cpfile: checkpoint file inode + * @cno: checkpoint number + * @bh: buffer head of block containing checkpoint indexed by @cno + * + * Return: Byte offset in the folio of the checkpoint snapshot list specified + * by @cno. + */ +static size_t nilfs_cpfile_cp_snapshot_list_offset(const struct inode *cpfile, + __u64 cno, + struct buffer_head *bh) +{ + return nilfs_cpfile_checkpoint_offset(cpfile, cno, bh) + + offsetof(struct nilfs_checkpoint, cp_snapshot_list); +} + +/** + * nilfs_cpfile_ch_snapshot_list_offset - calculate the byte offset of the + * snapshot list in the header + * + * Return: Byte offset in the folio of the checkpoint snapshot list + */ +static size_t nilfs_cpfile_ch_snapshot_list_offset(void) +{ + return offsetof(struct nilfs_cpfile_header, ch_snapshot_list); +} + +static int nilfs_cpfile_get_header_block(struct inode *cpfile, + struct buffer_head **bhp) +{ + int err = nilfs_mdt_get_block(cpfile, 0, 0, NULL, bhp); + + if (unlikely(err == -ENOENT)) { + nilfs_error(cpfile->i_sb, + "missing header block in checkpoint metadata"); + err = -EIO; + } + return err; +} + +static inline int nilfs_cpfile_get_checkpoint_block(struct inode *cpfile, + __u64 cno, + int create, + struct buffer_head **bhp) +{ + return nilfs_mdt_get_block(cpfile, + nilfs_cpfile_get_blkoff(cpfile, cno), + create, nilfs_cpfile_block_init, bhp); +} + +/** + * nilfs_cpfile_find_checkpoint_block - find and get a buffer on cpfile + * @cpfile: inode of cpfile + * @start_cno: start checkpoint number (inclusive) + * @end_cno: end checkpoint number (inclusive) + * @cnop: place to store the next checkpoint number + * @bhp: place to store a pointer to buffer_head struct + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOENT - no block exists in the range. + * * %-ENOMEM - Insufficient memory available. + */ +static int nilfs_cpfile_find_checkpoint_block(struct inode *cpfile, + __u64 start_cno, __u64 end_cno, + __u64 *cnop, + struct buffer_head **bhp) +{ + unsigned long start, end, blkoff; + int ret; + + if (unlikely(start_cno > end_cno)) + return -ENOENT; + + start = nilfs_cpfile_get_blkoff(cpfile, start_cno); + end = nilfs_cpfile_get_blkoff(cpfile, end_cno); + + ret = nilfs_mdt_find_block(cpfile, start, end, &blkoff, bhp); + if (!ret) + *cnop = (blkoff == start) ? start_cno : + nilfs_cpfile_first_checkpoint_in_block(cpfile, blkoff); + return ret; +} + +static inline int nilfs_cpfile_delete_checkpoint_block(struct inode *cpfile, + __u64 cno) +{ + return nilfs_mdt_delete_block(cpfile, + nilfs_cpfile_get_blkoff(cpfile, cno)); +} + +/** + * nilfs_cpfile_read_checkpoint - read a checkpoint entry in cpfile + * @cpfile: checkpoint file inode + * @cno: number of checkpoint entry to read + * @root: nilfs root object + * @ifile: ifile's inode to read and attach to @root + * + * This function imports checkpoint information from the checkpoint file and + * stores it to the inode file given by @ifile and the nilfs root object + * given by @root. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EINVAL - Invalid checkpoint. + * * %-ENOMEM - Insufficient memory available. + * * %-EIO - I/O error (including metadata corruption). + */ +int nilfs_cpfile_read_checkpoint(struct inode *cpfile, __u64 cno, + struct nilfs_root *root, struct inode *ifile) +{ + struct buffer_head *cp_bh; + struct nilfs_checkpoint *cp; + size_t offset; + int ret; + + if (cno < 1 || cno > nilfs_mdt_cno(cpfile)) + return -EINVAL; + + down_read(&NILFS_MDT(cpfile)->mi_sem); + ret = nilfs_cpfile_get_checkpoint_block(cpfile, cno, 0, &cp_bh); + if (unlikely(ret < 0)) { + if (ret == -ENOENT) + ret = -EINVAL; + goto out_sem; + } + + offset = nilfs_cpfile_checkpoint_offset(cpfile, cno, cp_bh); + cp = kmap_local_folio(cp_bh->b_folio, offset); + if (nilfs_checkpoint_invalid(cp)) { + ret = -EINVAL; + goto put_cp; + } + + ret = nilfs_read_inode_common(ifile, &cp->cp_ifile_inode); + if (unlikely(ret)) { + /* + * Since this inode is on a checkpoint entry, treat errors + * as metadata corruption. + */ + nilfs_err(cpfile->i_sb, + "ifile inode (checkpoint number=%llu) corrupted", + (unsigned long long)cno); + ret = -EIO; + goto put_cp; + } + + /* Configure the nilfs root object */ + atomic64_set(&root->inodes_count, le64_to_cpu(cp->cp_inodes_count)); + atomic64_set(&root->blocks_count, le64_to_cpu(cp->cp_blocks_count)); + root->ifile = ifile; + +put_cp: + kunmap_local(cp); + brelse(cp_bh); +out_sem: + up_read(&NILFS_MDT(cpfile)->mi_sem); + return ret; +} + +/** + * nilfs_cpfile_create_checkpoint - create a checkpoint entry on cpfile + * @cpfile: checkpoint file inode + * @cno: number of checkpoint to set up + * + * This function creates a checkpoint with the number specified by @cno on + * cpfile. If the specified checkpoint entry already exists due to a past + * failure, it will be reused without returning an error. + * In either case, the buffer of the block containing the checkpoint entry + * and the cpfile inode are made dirty for inclusion in the write log. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-ENOMEM - Insufficient memory available. + * * %-EIO - I/O error (including metadata corruption). + * * %-EROFS - Read only filesystem + */ +int nilfs_cpfile_create_checkpoint(struct inode *cpfile, __u64 cno) +{ + struct buffer_head *header_bh, *cp_bh; + struct nilfs_cpfile_header *header; + struct nilfs_checkpoint *cp; + size_t offset; + int ret; + + if (WARN_ON_ONCE(cno < 1)) + return -EIO; + + down_write(&NILFS_MDT(cpfile)->mi_sem); + ret = nilfs_cpfile_get_header_block(cpfile, &header_bh); + if (unlikely(ret < 0)) + goto out_sem; + + ret = nilfs_cpfile_get_checkpoint_block(cpfile, cno, 1, &cp_bh); + if (unlikely(ret < 0)) + goto out_header; + + offset = nilfs_cpfile_checkpoint_offset(cpfile, cno, cp_bh); + cp = kmap_local_folio(cp_bh->b_folio, offset); + if (nilfs_checkpoint_invalid(cp)) { + /* a newly-created checkpoint */ + nilfs_checkpoint_clear_invalid(cp); + kunmap_local(cp); + if (!nilfs_cpfile_is_in_first(cpfile, cno)) + nilfs_cpfile_block_add_valid_checkpoints(cpfile, cp_bh, + 1); + + header = kmap_local_folio(header_bh->b_folio, 0); + le64_add_cpu(&header->ch_ncheckpoints, 1); + kunmap_local(header); + mark_buffer_dirty(header_bh); + } else { + kunmap_local(cp); + } + + /* Force the buffer and the inode to become dirty */ + mark_buffer_dirty(cp_bh); + brelse(cp_bh); + nilfs_mdt_mark_dirty(cpfile); + +out_header: + brelse(header_bh); + +out_sem: + up_write(&NILFS_MDT(cpfile)->mi_sem); + return ret; +} + +/** + * nilfs_cpfile_finalize_checkpoint - fill in a checkpoint entry in cpfile + * @cpfile: checkpoint file inode + * @cno: checkpoint number + * @root: nilfs root object + * @blkinc: number of blocks added by this checkpoint + * @ctime: checkpoint creation time + * @minor: minor checkpoint flag + * + * This function completes the checkpoint entry numbered by @cno in the + * cpfile with the data given by the arguments @root, @blkinc, @ctime, and + * @minor. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-ENOMEM - Insufficient memory available. + * * %-EIO - I/O error (including metadata corruption). + */ +int nilfs_cpfile_finalize_checkpoint(struct inode *cpfile, __u64 cno, + struct nilfs_root *root, __u64 blkinc, + time64_t ctime, bool minor) +{ + struct buffer_head *cp_bh; + struct nilfs_checkpoint *cp; + size_t offset; + int ret; + + if (WARN_ON_ONCE(cno < 1)) + return -EIO; + + down_write(&NILFS_MDT(cpfile)->mi_sem); + ret = nilfs_cpfile_get_checkpoint_block(cpfile, cno, 0, &cp_bh); + if (unlikely(ret < 0)) { + if (ret == -ENOENT) + goto error; + goto out_sem; + } + + offset = nilfs_cpfile_checkpoint_offset(cpfile, cno, cp_bh); + cp = kmap_local_folio(cp_bh->b_folio, offset); + if (unlikely(nilfs_checkpoint_invalid(cp))) { + kunmap_local(cp); + brelse(cp_bh); + goto error; + } + + cp->cp_snapshot_list.ssl_next = 0; + cp->cp_snapshot_list.ssl_prev = 0; + cp->cp_inodes_count = cpu_to_le64(atomic64_read(&root->inodes_count)); + cp->cp_blocks_count = cpu_to_le64(atomic64_read(&root->blocks_count)); + cp->cp_nblk_inc = cpu_to_le64(blkinc); + cp->cp_create = cpu_to_le64(ctime); + cp->cp_cno = cpu_to_le64(cno); + + if (minor) + nilfs_checkpoint_set_minor(cp); + else + nilfs_checkpoint_clear_minor(cp); + + nilfs_write_inode_common(root->ifile, &cp->cp_ifile_inode); + nilfs_bmap_write(NILFS_I(root->ifile)->i_bmap, &cp->cp_ifile_inode); + + kunmap_local(cp); + brelse(cp_bh); +out_sem: + up_write(&NILFS_MDT(cpfile)->mi_sem); + return ret; + +error: + nilfs_error(cpfile->i_sb, + "checkpoint finalization failed due to metadata corruption."); + ret = -EIO; + goto out_sem; +} + +/** + * nilfs_cpfile_delete_checkpoints - delete checkpoints + * @cpfile: inode of checkpoint file + * @start: start checkpoint number + * @end: end checkpoint number + * + * Description: nilfs_cpfile_delete_checkpoints() deletes the checkpoints in + * the period from @start to @end, excluding @end itself. The checkpoints + * which have been already deleted are ignored. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EINVAL - Invalid checkpoints. + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_cpfile_delete_checkpoints(struct inode *cpfile, + __u64 start, + __u64 end) +{ + struct buffer_head *header_bh, *cp_bh; + struct nilfs_cpfile_header *header; + struct nilfs_checkpoint *cp; + size_t cpsz = NILFS_MDT(cpfile)->mi_entry_size; + __u64 cno; + size_t offset; + void *kaddr; + unsigned long tnicps; + int ret, ncps, nicps, nss, count, i; + + if (unlikely(start == 0 || start > end)) { + nilfs_err(cpfile->i_sb, + "cannot delete checkpoints: invalid range [%llu, %llu)", + (unsigned long long)start, (unsigned long long)end); + return -EINVAL; + } + + down_write(&NILFS_MDT(cpfile)->mi_sem); + + ret = nilfs_cpfile_get_header_block(cpfile, &header_bh); + if (ret < 0) + goto out_sem; + tnicps = 0; + nss = 0; + + for (cno = start; cno < end; cno += ncps) { + ncps = nilfs_cpfile_checkpoints_in_block(cpfile, cno, end); + ret = nilfs_cpfile_get_checkpoint_block(cpfile, cno, 0, &cp_bh); + if (ret < 0) { + if (ret != -ENOENT) + break; + /* skip hole */ + ret = 0; + continue; + } + + offset = nilfs_cpfile_checkpoint_offset(cpfile, cno, cp_bh); + cp = kaddr = kmap_local_folio(cp_bh->b_folio, offset); + nicps = 0; + for (i = 0; i < ncps; i++, cp = (void *)cp + cpsz) { + if (nilfs_checkpoint_snapshot(cp)) { + nss++; + } else if (!nilfs_checkpoint_invalid(cp)) { + nilfs_checkpoint_set_invalid(cp); + nicps++; + } + } + kunmap_local(kaddr); + + if (nicps <= 0) { + brelse(cp_bh); + continue; + } + + tnicps += nicps; + mark_buffer_dirty(cp_bh); + nilfs_mdt_mark_dirty(cpfile); + if (nilfs_cpfile_is_in_first(cpfile, cno)) { + brelse(cp_bh); + continue; + } + + count = nilfs_cpfile_block_sub_valid_checkpoints(cpfile, cp_bh, + nicps); + brelse(cp_bh); + if (unlikely(count < 0)) { + ret = count; + break; + } + if (count) + continue; + + /* Delete the block if there are no more valid checkpoints */ + ret = nilfs_cpfile_delete_checkpoint_block(cpfile, cno); + if (unlikely(ret)) { + nilfs_err(cpfile->i_sb, + "error %d deleting checkpoint block", ret); + break; + } + } + + if (tnicps > 0) { + header = kmap_local_folio(header_bh->b_folio, 0); + le64_add_cpu(&header->ch_ncheckpoints, -(u64)tnicps); + mark_buffer_dirty(header_bh); + nilfs_mdt_mark_dirty(cpfile); + kunmap_local(header); + } + + brelse(header_bh); + if (nss > 0) + ret = -EBUSY; + + out_sem: + up_write(&NILFS_MDT(cpfile)->mi_sem); + return ret; +} + +static void nilfs_cpfile_checkpoint_to_cpinfo(struct inode *cpfile, + struct nilfs_checkpoint *cp, + struct nilfs_cpinfo *ci) +{ + ci->ci_flags = le32_to_cpu(cp->cp_flags); + ci->ci_cno = le64_to_cpu(cp->cp_cno); + ci->ci_create = le64_to_cpu(cp->cp_create); + ci->ci_nblk_inc = le64_to_cpu(cp->cp_nblk_inc); + ci->ci_inodes_count = le64_to_cpu(cp->cp_inodes_count); + ci->ci_blocks_count = le64_to_cpu(cp->cp_blocks_count); + ci->ci_next = le64_to_cpu(cp->cp_snapshot_list.ssl_next); +} + +static ssize_t nilfs_cpfile_do_get_cpinfo(struct inode *cpfile, __u64 *cnop, + void *buf, unsigned int cisz, + size_t nci) +{ + struct nilfs_checkpoint *cp; + struct nilfs_cpinfo *ci = buf; + struct buffer_head *bh; + size_t cpsz = NILFS_MDT(cpfile)->mi_entry_size; + __u64 cur_cno = nilfs_mdt_cno(cpfile), cno = *cnop; + size_t offset; + void *kaddr; + int n, ret; + int ncps, i; + + if (cno == 0) + return -ENOENT; /* checkpoint number 0 is invalid */ + down_read(&NILFS_MDT(cpfile)->mi_sem); + + for (n = 0; n < nci; cno += ncps) { + ret = nilfs_cpfile_find_checkpoint_block( + cpfile, cno, cur_cno - 1, &cno, &bh); + if (ret < 0) { + if (likely(ret == -ENOENT)) + break; + goto out; + } + ncps = nilfs_cpfile_checkpoints_in_block(cpfile, cno, cur_cno); + + offset = nilfs_cpfile_checkpoint_offset(cpfile, cno, bh); + cp = kaddr = kmap_local_folio(bh->b_folio, offset); + for (i = 0; i < ncps && n < nci; i++, cp = (void *)cp + cpsz) { + if (!nilfs_checkpoint_invalid(cp)) { + nilfs_cpfile_checkpoint_to_cpinfo(cpfile, cp, + ci); + ci = (void *)ci + cisz; + n++; + } + } + kunmap_local(kaddr); + brelse(bh); + } + + ret = n; + if (n > 0) { + ci = (void *)ci - cisz; + *cnop = ci->ci_cno + 1; + } + + out: + up_read(&NILFS_MDT(cpfile)->mi_sem); + return ret; +} + +static ssize_t nilfs_cpfile_do_get_ssinfo(struct inode *cpfile, __u64 *cnop, + void *buf, unsigned int cisz, + size_t nci) +{ + struct buffer_head *bh; + struct nilfs_cpfile_header *header; + struct nilfs_checkpoint *cp; + struct nilfs_cpinfo *ci = buf; + __u64 curr = *cnop, next; + unsigned long curr_blkoff, next_blkoff; + size_t offset; + int n = 0, ret; + + down_read(&NILFS_MDT(cpfile)->mi_sem); + + if (curr == 0) { + ret = nilfs_cpfile_get_header_block(cpfile, &bh); + if (ret < 0) + goto out; + header = kmap_local_folio(bh->b_folio, 0); + curr = le64_to_cpu(header->ch_snapshot_list.ssl_next); + kunmap_local(header); + brelse(bh); + if (curr == 0) { + ret = 0; + goto out; + } + } else if (unlikely(curr == ~(__u64)0)) { + ret = 0; + goto out; + } + + curr_blkoff = nilfs_cpfile_get_blkoff(cpfile, curr); + ret = nilfs_cpfile_get_checkpoint_block(cpfile, curr, 0, &bh); + if (unlikely(ret < 0)) { + if (ret == -ENOENT) + ret = 0; /* No snapshots (started from a hole block) */ + goto out; + } + offset = nilfs_cpfile_checkpoint_offset(cpfile, curr, bh); + cp = kmap_local_folio(bh->b_folio, offset); + while (n < nci) { + curr = ~(__u64)0; /* Terminator */ + if (unlikely(nilfs_checkpoint_invalid(cp) || + !nilfs_checkpoint_snapshot(cp))) + break; + nilfs_cpfile_checkpoint_to_cpinfo(cpfile, cp, ci); + ci = (void *)ci + cisz; + n++; + next = le64_to_cpu(cp->cp_snapshot_list.ssl_next); + if (next == 0) + break; /* reach end of the snapshot list */ + + kunmap_local(cp); + next_blkoff = nilfs_cpfile_get_blkoff(cpfile, next); + if (curr_blkoff != next_blkoff) { + brelse(bh); + ret = nilfs_cpfile_get_checkpoint_block(cpfile, next, + 0, &bh); + if (unlikely(ret < 0)) { + WARN_ON(ret == -ENOENT); + goto out; + } + } + offset = nilfs_cpfile_checkpoint_offset(cpfile, next, bh); + cp = kmap_local_folio(bh->b_folio, offset); + curr = next; + curr_blkoff = next_blkoff; + } + kunmap_local(cp); + brelse(bh); + *cnop = curr; + ret = n; + + out: + up_read(&NILFS_MDT(cpfile)->mi_sem); + return ret; +} + +/** + * nilfs_cpfile_get_cpinfo - get information on checkpoints + * @cpfile: checkpoint file inode + * @cnop: place to pass a starting checkpoint number and receive a + * checkpoint number to continue the search + * @mode: mode of checkpoints that the caller wants to retrieve + * @buf: buffer for storing checkpoints' information + * @cisz: byte size of one checkpoint info item in array + * @nci: number of checkpoint info items to retrieve + * + * nilfs_cpfile_get_cpinfo() searches for checkpoints in @mode state + * starting from the checkpoint number stored in @cnop, and stores + * information about found checkpoints in @buf. + * The buffer pointed to by @buf must be large enough to store information + * for @nci checkpoints. If at least one checkpoint information is + * successfully retrieved, @cnop is updated to point to the checkpoint + * number to continue searching. + * + * Return: Count of checkpoint info items stored in the output buffer on + * success, or one of the following negative error codes on failure: + * * %-EINVAL - Invalid checkpoint mode. + * * %-ENOMEM - Insufficient memory available. + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOENT - Invalid checkpoint number specified. + */ + +ssize_t nilfs_cpfile_get_cpinfo(struct inode *cpfile, __u64 *cnop, int mode, + void *buf, unsigned int cisz, size_t nci) +{ + switch (mode) { + case NILFS_CHECKPOINT: + return nilfs_cpfile_do_get_cpinfo(cpfile, cnop, buf, cisz, nci); + case NILFS_SNAPSHOT: + return nilfs_cpfile_do_get_ssinfo(cpfile, cnop, buf, cisz, nci); + default: + return -EINVAL; + } +} + +/** + * nilfs_cpfile_delete_checkpoint - delete a checkpoint + * @cpfile: checkpoint file inode + * @cno: checkpoint number to delete + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EBUSY - Checkpoint in use (snapshot specified). + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOENT - No valid checkpoint found. + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_cpfile_delete_checkpoint(struct inode *cpfile, __u64 cno) +{ + struct nilfs_cpinfo ci; + __u64 tcno = cno; + ssize_t nci; + + nci = nilfs_cpfile_do_get_cpinfo(cpfile, &tcno, &ci, sizeof(ci), 1); + if (nci < 0) + return nci; + else if (nci == 0 || ci.ci_cno != cno) + return -ENOENT; + else if (nilfs_cpinfo_snapshot(&ci)) + return -EBUSY; + + return nilfs_cpfile_delete_checkpoints(cpfile, cno, cno + 1); +} + +static int nilfs_cpfile_set_snapshot(struct inode *cpfile, __u64 cno) +{ + struct buffer_head *header_bh, *curr_bh, *prev_bh, *cp_bh; + struct nilfs_cpfile_header *header; + struct nilfs_checkpoint *cp; + struct nilfs_snapshot_list *list; + __u64 curr, prev; + unsigned long curr_blkoff, prev_blkoff; + size_t offset, curr_list_offset, prev_list_offset; + int ret; + + if (cno == 0) + return -ENOENT; /* checkpoint number 0 is invalid */ + down_write(&NILFS_MDT(cpfile)->mi_sem); + + ret = nilfs_cpfile_get_header_block(cpfile, &header_bh); + if (unlikely(ret < 0)) + goto out_sem; + + ret = nilfs_cpfile_get_checkpoint_block(cpfile, cno, 0, &cp_bh); + if (ret < 0) + goto out_header; + + offset = nilfs_cpfile_checkpoint_offset(cpfile, cno, cp_bh); + cp = kmap_local_folio(cp_bh->b_folio, offset); + if (nilfs_checkpoint_invalid(cp)) { + ret = -ENOENT; + kunmap_local(cp); + goto out_cp; + } + if (nilfs_checkpoint_snapshot(cp)) { + ret = 0; + kunmap_local(cp); + goto out_cp; + } + kunmap_local(cp); + + /* + * Find the last snapshot before the checkpoint being changed to + * snapshot mode by going backwards through the snapshot list. + * Set "prev" to its checkpoint number, or 0 if not found. + */ + header = kmap_local_folio(header_bh->b_folio, 0); + list = &header->ch_snapshot_list; + curr_bh = header_bh; + get_bh(curr_bh); + curr = 0; + curr_blkoff = 0; + curr_list_offset = nilfs_cpfile_ch_snapshot_list_offset(); + prev = le64_to_cpu(list->ssl_prev); + while (prev > cno) { + prev_blkoff = nilfs_cpfile_get_blkoff(cpfile, prev); + curr = prev; + kunmap_local(list); + if (curr_blkoff != prev_blkoff) { + brelse(curr_bh); + ret = nilfs_cpfile_get_checkpoint_block(cpfile, curr, + 0, &curr_bh); + if (unlikely(ret < 0)) + goto out_cp; + } + curr_list_offset = nilfs_cpfile_cp_snapshot_list_offset( + cpfile, curr, curr_bh); + list = kmap_local_folio(curr_bh->b_folio, curr_list_offset); + curr_blkoff = prev_blkoff; + prev = le64_to_cpu(list->ssl_prev); + } + kunmap_local(list); + + if (prev != 0) { + ret = nilfs_cpfile_get_checkpoint_block(cpfile, prev, 0, + &prev_bh); + if (ret < 0) + goto out_curr; + + prev_list_offset = nilfs_cpfile_cp_snapshot_list_offset( + cpfile, prev, prev_bh); + } else { + prev_bh = header_bh; + get_bh(prev_bh); + prev_list_offset = nilfs_cpfile_ch_snapshot_list_offset(); + } + + /* Update the list entry for the next snapshot */ + list = kmap_local_folio(curr_bh->b_folio, curr_list_offset); + list->ssl_prev = cpu_to_le64(cno); + kunmap_local(list); + + /* Update the checkpoint being changed to a snapshot */ + offset = nilfs_cpfile_checkpoint_offset(cpfile, cno, cp_bh); + cp = kmap_local_folio(cp_bh->b_folio, offset); + cp->cp_snapshot_list.ssl_next = cpu_to_le64(curr); + cp->cp_snapshot_list.ssl_prev = cpu_to_le64(prev); + nilfs_checkpoint_set_snapshot(cp); + kunmap_local(cp); + + /* Update the list entry for the previous snapshot */ + list = kmap_local_folio(prev_bh->b_folio, prev_list_offset); + list->ssl_next = cpu_to_le64(cno); + kunmap_local(list); + + /* Update the statistics in the header */ + header = kmap_local_folio(header_bh->b_folio, 0); + le64_add_cpu(&header->ch_nsnapshots, 1); + kunmap_local(header); + + mark_buffer_dirty(prev_bh); + mark_buffer_dirty(curr_bh); + mark_buffer_dirty(cp_bh); + mark_buffer_dirty(header_bh); + nilfs_mdt_mark_dirty(cpfile); + + brelse(prev_bh); + + out_curr: + brelse(curr_bh); + + out_cp: + brelse(cp_bh); + + out_header: + brelse(header_bh); + + out_sem: + up_write(&NILFS_MDT(cpfile)->mi_sem); + return ret; +} + +static int nilfs_cpfile_clear_snapshot(struct inode *cpfile, __u64 cno) +{ + struct buffer_head *header_bh, *next_bh, *prev_bh, *cp_bh; + struct nilfs_cpfile_header *header; + struct nilfs_checkpoint *cp; + struct nilfs_snapshot_list *list; + __u64 next, prev; + size_t offset, next_list_offset, prev_list_offset; + int ret; + + if (cno == 0) + return -ENOENT; /* checkpoint number 0 is invalid */ + down_write(&NILFS_MDT(cpfile)->mi_sem); + + ret = nilfs_cpfile_get_header_block(cpfile, &header_bh); + if (unlikely(ret < 0)) + goto out_sem; + + ret = nilfs_cpfile_get_checkpoint_block(cpfile, cno, 0, &cp_bh); + if (ret < 0) + goto out_header; + + offset = nilfs_cpfile_checkpoint_offset(cpfile, cno, cp_bh); + cp = kmap_local_folio(cp_bh->b_folio, offset); + if (nilfs_checkpoint_invalid(cp)) { + ret = -ENOENT; + kunmap_local(cp); + goto out_cp; + } + if (!nilfs_checkpoint_snapshot(cp)) { + ret = 0; + kunmap_local(cp); + goto out_cp; + } + + list = &cp->cp_snapshot_list; + next = le64_to_cpu(list->ssl_next); + prev = le64_to_cpu(list->ssl_prev); + kunmap_local(cp); + + if (next != 0) { + ret = nilfs_cpfile_get_checkpoint_block(cpfile, next, 0, + &next_bh); + if (ret < 0) + goto out_cp; + + next_list_offset = nilfs_cpfile_cp_snapshot_list_offset( + cpfile, next, next_bh); + } else { + next_bh = header_bh; + get_bh(next_bh); + next_list_offset = nilfs_cpfile_ch_snapshot_list_offset(); + } + if (prev != 0) { + ret = nilfs_cpfile_get_checkpoint_block(cpfile, prev, 0, + &prev_bh); + if (ret < 0) + goto out_next; + + prev_list_offset = nilfs_cpfile_cp_snapshot_list_offset( + cpfile, prev, prev_bh); + } else { + prev_bh = header_bh; + get_bh(prev_bh); + prev_list_offset = nilfs_cpfile_ch_snapshot_list_offset(); + } + + /* Update the list entry for the next snapshot */ + list = kmap_local_folio(next_bh->b_folio, next_list_offset); + list->ssl_prev = cpu_to_le64(prev); + kunmap_local(list); + + /* Update the list entry for the previous snapshot */ + list = kmap_local_folio(prev_bh->b_folio, prev_list_offset); + list->ssl_next = cpu_to_le64(next); + kunmap_local(list); + + /* Update the snapshot being changed back to a plain checkpoint */ + cp = kmap_local_folio(cp_bh->b_folio, offset); + cp->cp_snapshot_list.ssl_next = cpu_to_le64(0); + cp->cp_snapshot_list.ssl_prev = cpu_to_le64(0); + nilfs_checkpoint_clear_snapshot(cp); + kunmap_local(cp); + + /* Update the statistics in the header */ + header = kmap_local_folio(header_bh->b_folio, 0); + le64_add_cpu(&header->ch_nsnapshots, -1); + kunmap_local(header); + + mark_buffer_dirty(next_bh); + mark_buffer_dirty(prev_bh); + mark_buffer_dirty(cp_bh); + mark_buffer_dirty(header_bh); + nilfs_mdt_mark_dirty(cpfile); + + brelse(prev_bh); + + out_next: + brelse(next_bh); + + out_cp: + brelse(cp_bh); + + out_header: + brelse(header_bh); + + out_sem: + up_write(&NILFS_MDT(cpfile)->mi_sem); + return ret; +} + +/** + * nilfs_cpfile_is_snapshot - determine if checkpoint is a snapshot + * @cpfile: inode of checkpoint file + * @cno: checkpoint number + * + * Return: 1 if the checkpoint specified by @cno is a snapshot, 0 if not, or + * one of the following negative error codes on failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOENT - No such checkpoint. + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_cpfile_is_snapshot(struct inode *cpfile, __u64 cno) +{ + struct buffer_head *bh; + struct nilfs_checkpoint *cp; + size_t offset; + int ret; + + /* + * CP number is invalid if it's zero or larger than the + * largest existing one. + */ + if (cno == 0 || cno >= nilfs_mdt_cno(cpfile)) + return -ENOENT; + down_read(&NILFS_MDT(cpfile)->mi_sem); + + ret = nilfs_cpfile_get_checkpoint_block(cpfile, cno, 0, &bh); + if (ret < 0) + goto out; + + offset = nilfs_cpfile_checkpoint_offset(cpfile, cno, bh); + cp = kmap_local_folio(bh->b_folio, offset); + if (nilfs_checkpoint_invalid(cp)) + ret = -ENOENT; + else + ret = nilfs_checkpoint_snapshot(cp); + kunmap_local(cp); + brelse(bh); + + out: + up_read(&NILFS_MDT(cpfile)->mi_sem); + return ret; +} + +/** + * nilfs_cpfile_change_cpmode - change checkpoint mode + * @cpfile: inode of checkpoint file + * @cno: checkpoint number + * @mode: mode of checkpoint + * + * Description: nilfs_change_cpmode() changes the mode of the checkpoint + * specified by @cno. The mode @mode is NILFS_CHECKPOINT or NILFS_SNAPSHOT. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOENT - No such checkpoint. + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_cpfile_change_cpmode(struct inode *cpfile, __u64 cno, int mode) +{ + int ret; + + switch (mode) { + case NILFS_CHECKPOINT: + if (nilfs_checkpoint_is_mounted(cpfile->i_sb, cno)) + /* + * Current implementation does not have to protect + * plain read-only mounts since they are exclusive + * with a read/write mount and are protected from the + * cleaner. + */ + ret = -EBUSY; + else + ret = nilfs_cpfile_clear_snapshot(cpfile, cno); + return ret; + case NILFS_SNAPSHOT: + return nilfs_cpfile_set_snapshot(cpfile, cno); + default: + return -EINVAL; + } +} + +/** + * nilfs_cpfile_get_stat - get checkpoint statistics + * @cpfile: inode of checkpoint file + * @cpstat: pointer to a structure of checkpoint statistics + * + * Description: nilfs_cpfile_get_stat() returns information about checkpoints. + * The checkpoint statistics are stored in the location pointed to by @cpstat. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_cpfile_get_stat(struct inode *cpfile, struct nilfs_cpstat *cpstat) +{ + struct buffer_head *bh; + struct nilfs_cpfile_header *header; + int ret; + + down_read(&NILFS_MDT(cpfile)->mi_sem); + + ret = nilfs_cpfile_get_header_block(cpfile, &bh); + if (ret < 0) + goto out_sem; + header = kmap_local_folio(bh->b_folio, 0); + cpstat->cs_cno = nilfs_mdt_cno(cpfile); + cpstat->cs_ncps = le64_to_cpu(header->ch_ncheckpoints); + cpstat->cs_nsss = le64_to_cpu(header->ch_nsnapshots); + kunmap_local(header); + brelse(bh); + + out_sem: + up_read(&NILFS_MDT(cpfile)->mi_sem); + return ret; +} + +/** + * nilfs_cpfile_read - read or get cpfile inode + * @sb: super block instance + * @cpsize: size of a checkpoint entry + * @raw_inode: on-disk cpfile inode + * @inodep: buffer to store the inode + * + * Return: 0 on success, or a negative error code on failure. + */ +int nilfs_cpfile_read(struct super_block *sb, size_t cpsize, + struct nilfs_inode *raw_inode, struct inode **inodep) +{ + struct inode *cpfile; + int err; + + if (cpsize > sb->s_blocksize) { + nilfs_err(sb, "too large checkpoint size: %zu bytes", cpsize); + return -EINVAL; + } else if (cpsize < NILFS_MIN_CHECKPOINT_SIZE) { + nilfs_err(sb, "too small checkpoint size: %zu bytes", cpsize); + return -EINVAL; + } + + cpfile = nilfs_iget_locked(sb, NULL, NILFS_CPFILE_INO); + if (unlikely(!cpfile)) + return -ENOMEM; + if (!(inode_state_read_once(cpfile) & I_NEW)) + goto out; + + err = nilfs_mdt_init(cpfile, NILFS_MDT_GFP, 0); + if (err) + goto failed; + + nilfs_mdt_set_entry_size(cpfile, cpsize, + sizeof(struct nilfs_cpfile_header)); + + err = nilfs_read_inode_common(cpfile, raw_inode); + if (err) + goto failed; + + unlock_new_inode(cpfile); + out: + *inodep = cpfile; + return 0; + failed: + iget_failed(cpfile); + return err; +} diff --git a/fs/nilfs2/cpfile.h b/fs/nilfs2/cpfile.h new file mode 100644 index 000000000..f5b1d5928 --- /dev/null +++ b/fs/nilfs2/cpfile.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * NILFS checkpoint file. + * + * Copyright (C) 2006-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Koji Sato. + */ + +#ifndef _NILFS_CPFILE_H +#define _NILFS_CPFILE_H + +#include +#include +#include /* nilfs_cpstat */ +#include /* nilfs_inode, nilfs_checkpoint */ + + +int nilfs_cpfile_read_checkpoint(struct inode *cpfile, __u64 cno, + struct nilfs_root *root, struct inode *ifile); +int nilfs_cpfile_create_checkpoint(struct inode *cpfile, __u64 cno); +int nilfs_cpfile_finalize_checkpoint(struct inode *cpfile, __u64 cno, + struct nilfs_root *root, __u64 blkinc, + time64_t ctime, bool minor); +int nilfs_cpfile_delete_checkpoints(struct inode *, __u64, __u64); +int nilfs_cpfile_delete_checkpoint(struct inode *, __u64); +int nilfs_cpfile_change_cpmode(struct inode *, __u64, int); +int nilfs_cpfile_is_snapshot(struct inode *, __u64); +int nilfs_cpfile_get_stat(struct inode *, struct nilfs_cpstat *); +ssize_t nilfs_cpfile_get_cpinfo(struct inode *, __u64 *, int, void *, + unsigned int, size_t); + +int nilfs_cpfile_read(struct super_block *sb, size_t cpsize, + struct nilfs_inode *raw_inode, struct inode **inodep); + +#endif /* _NILFS_CPFILE_H */ diff --git a/fs/nilfs2/dat.c b/fs/nilfs2/dat.c new file mode 100644 index 000000000..888dc1831 --- /dev/null +++ b/fs/nilfs2/dat.c @@ -0,0 +1,541 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * NILFS disk address translation. + * + * Copyright (C) 2006-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Koji Sato. + */ + +#include +#include +#include +#include +#include "nilfs.h" +#include "mdt.h" +#include "alloc.h" +#include "dat.h" + + +#define NILFS_CNO_MIN ((__u64)1) +#define NILFS_CNO_MAX (~(__u64)0) + +/** + * struct nilfs_dat_info - on-memory private data of DAT file + * @mi: on-memory private data of metadata file + * @palloc_cache: persistent object allocator cache of DAT file + * @shadow: shadow map of DAT file + */ +struct nilfs_dat_info { + struct nilfs_mdt_info mi; + struct nilfs_palloc_cache palloc_cache; + struct nilfs_shadow_map shadow; +}; + +static inline struct nilfs_dat_info *NILFS_DAT_I(struct inode *dat) +{ + return (struct nilfs_dat_info *)NILFS_MDT(dat); +} + +static int nilfs_dat_prepare_entry(struct inode *dat, + struct nilfs_palloc_req *req, int create) +{ + int ret; + + ret = nilfs_palloc_get_entry_block(dat, req->pr_entry_nr, + create, &req->pr_entry_bh); + if (unlikely(ret == -ENOENT)) { + nilfs_err(dat->i_sb, + "DAT doesn't have a block to manage vblocknr = %llu", + (unsigned long long)req->pr_entry_nr); + /* + * Return internal code -EINVAL to notify bmap layer of + * metadata corruption. + */ + ret = -EINVAL; + } + return ret; +} + +static void nilfs_dat_commit_entry(struct inode *dat, + struct nilfs_palloc_req *req) +{ + mark_buffer_dirty(req->pr_entry_bh); + nilfs_mdt_mark_dirty(dat); + brelse(req->pr_entry_bh); +} + +static void nilfs_dat_abort_entry(struct inode *dat, + struct nilfs_palloc_req *req) +{ + brelse(req->pr_entry_bh); +} + +int nilfs_dat_prepare_alloc(struct inode *dat, struct nilfs_palloc_req *req) +{ + int ret; + + ret = nilfs_palloc_prepare_alloc_entry(dat, req, true); + if (ret < 0) + return ret; + + ret = nilfs_dat_prepare_entry(dat, req, 1); + if (ret < 0) + nilfs_palloc_abort_alloc_entry(dat, req); + + return ret; +} + +void nilfs_dat_commit_alloc(struct inode *dat, struct nilfs_palloc_req *req) +{ + struct nilfs_dat_entry *entry; + size_t offset; + + offset = nilfs_palloc_entry_offset(dat, req->pr_entry_nr, + req->pr_entry_bh); + entry = kmap_local_folio(req->pr_entry_bh->b_folio, offset); + entry->de_start = cpu_to_le64(NILFS_CNO_MIN); + entry->de_end = cpu_to_le64(NILFS_CNO_MAX); + entry->de_blocknr = cpu_to_le64(0); + kunmap_local(entry); + + nilfs_palloc_commit_alloc_entry(dat, req); + nilfs_dat_commit_entry(dat, req); +} + +void nilfs_dat_abort_alloc(struct inode *dat, struct nilfs_palloc_req *req) +{ + nilfs_dat_abort_entry(dat, req); + nilfs_palloc_abort_alloc_entry(dat, req); +} + +static void nilfs_dat_commit_free(struct inode *dat, + struct nilfs_palloc_req *req) +{ + struct nilfs_dat_entry *entry; + size_t offset; + + offset = nilfs_palloc_entry_offset(dat, req->pr_entry_nr, + req->pr_entry_bh); + entry = kmap_local_folio(req->pr_entry_bh->b_folio, offset); + entry->de_start = cpu_to_le64(NILFS_CNO_MIN); + entry->de_end = cpu_to_le64(NILFS_CNO_MIN); + entry->de_blocknr = cpu_to_le64(0); + kunmap_local(entry); + + nilfs_dat_commit_entry(dat, req); + + if (unlikely(req->pr_desc_bh == NULL || req->pr_bitmap_bh == NULL)) { + nilfs_error(dat->i_sb, + "state inconsistency probably due to duplicate use of vblocknr = %llu", + (unsigned long long)req->pr_entry_nr); + return; + } + nilfs_palloc_commit_free_entry(dat, req); +} + +int nilfs_dat_prepare_start(struct inode *dat, struct nilfs_palloc_req *req) +{ + return nilfs_dat_prepare_entry(dat, req, 0); +} + +void nilfs_dat_commit_start(struct inode *dat, struct nilfs_palloc_req *req, + sector_t blocknr) +{ + struct nilfs_dat_entry *entry; + size_t offset; + + offset = nilfs_palloc_entry_offset(dat, req->pr_entry_nr, + req->pr_entry_bh); + entry = kmap_local_folio(req->pr_entry_bh->b_folio, offset); + entry->de_start = cpu_to_le64(nilfs_mdt_cno(dat)); + entry->de_blocknr = cpu_to_le64(blocknr); + kunmap_local(entry); + + nilfs_dat_commit_entry(dat, req); +} + +int nilfs_dat_prepare_end(struct inode *dat, struct nilfs_palloc_req *req) +{ + struct nilfs_dat_entry *entry; + __u64 start; + sector_t blocknr; + size_t offset; + int ret; + + ret = nilfs_dat_prepare_entry(dat, req, 0); + if (ret < 0) + return ret; + + offset = nilfs_palloc_entry_offset(dat, req->pr_entry_nr, + req->pr_entry_bh); + entry = kmap_local_folio(req->pr_entry_bh->b_folio, offset); + start = le64_to_cpu(entry->de_start); + blocknr = le64_to_cpu(entry->de_blocknr); + kunmap_local(entry); + + if (blocknr == 0) { + ret = nilfs_palloc_prepare_free_entry(dat, req); + if (ret < 0) { + nilfs_dat_abort_entry(dat, req); + return ret; + } + } + if (unlikely(start > nilfs_mdt_cno(dat))) { + nilfs_err(dat->i_sb, + "vblocknr = %llu has abnormal lifetime: start cno (= %llu) > current cno (= %llu)", + (unsigned long long)req->pr_entry_nr, + (unsigned long long)start, + (unsigned long long)nilfs_mdt_cno(dat)); + nilfs_dat_abort_entry(dat, req); + return -EINVAL; + } + + return 0; +} + +void nilfs_dat_commit_end(struct inode *dat, struct nilfs_palloc_req *req, + int dead) +{ + struct nilfs_dat_entry *entry; + __u64 start, end; + sector_t blocknr; + size_t offset; + + offset = nilfs_palloc_entry_offset(dat, req->pr_entry_nr, + req->pr_entry_bh); + entry = kmap_local_folio(req->pr_entry_bh->b_folio, offset); + end = start = le64_to_cpu(entry->de_start); + if (!dead) { + end = nilfs_mdt_cno(dat); + WARN_ON(start > end); + } + entry->de_end = cpu_to_le64(end); + blocknr = le64_to_cpu(entry->de_blocknr); + kunmap_local(entry); + + if (blocknr == 0) + nilfs_dat_commit_free(dat, req); + else + nilfs_dat_commit_entry(dat, req); +} + +void nilfs_dat_abort_end(struct inode *dat, struct nilfs_palloc_req *req) +{ + struct nilfs_dat_entry *entry; + __u64 start; + sector_t blocknr; + size_t offset; + + offset = nilfs_palloc_entry_offset(dat, req->pr_entry_nr, + req->pr_entry_bh); + entry = kmap_local_folio(req->pr_entry_bh->b_folio, offset); + start = le64_to_cpu(entry->de_start); + blocknr = le64_to_cpu(entry->de_blocknr); + kunmap_local(entry); + + if (start == nilfs_mdt_cno(dat) && blocknr == 0) + nilfs_palloc_abort_free_entry(dat, req); + nilfs_dat_abort_entry(dat, req); +} + +int nilfs_dat_prepare_update(struct inode *dat, + struct nilfs_palloc_req *oldreq, + struct nilfs_palloc_req *newreq) +{ + int ret; + + ret = nilfs_dat_prepare_end(dat, oldreq); + if (!ret) { + ret = nilfs_dat_prepare_alloc(dat, newreq); + if (ret < 0) + nilfs_dat_abort_end(dat, oldreq); + } + return ret; +} + +void nilfs_dat_commit_update(struct inode *dat, + struct nilfs_palloc_req *oldreq, + struct nilfs_palloc_req *newreq, int dead) +{ + nilfs_dat_commit_end(dat, oldreq, dead); + nilfs_dat_commit_alloc(dat, newreq); +} + +void nilfs_dat_abort_update(struct inode *dat, + struct nilfs_palloc_req *oldreq, + struct nilfs_palloc_req *newreq) +{ + nilfs_dat_abort_end(dat, oldreq); + nilfs_dat_abort_alloc(dat, newreq); +} + +/** + * nilfs_dat_mark_dirty - mark the DAT block buffer containing the specified + * virtual block address entry as dirty + * @dat: DAT file inode + * @vblocknr: virtual block number + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EINVAL - Invalid DAT entry (internal code). + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_dat_mark_dirty(struct inode *dat, __u64 vblocknr) +{ + struct nilfs_palloc_req req; + int ret; + + req.pr_entry_nr = vblocknr; + ret = nilfs_dat_prepare_entry(dat, &req, 0); + if (ret == 0) + nilfs_dat_commit_entry(dat, &req); + return ret; +} + +/** + * nilfs_dat_freev - free virtual block numbers + * @dat: DAT file inode + * @vblocknrs: array of virtual block numbers + * @nitems: number of virtual block numbers + * + * Description: nilfs_dat_freev() frees the virtual block numbers specified by + * @vblocknrs and @nitems. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOENT - The virtual block number have not been allocated. + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_dat_freev(struct inode *dat, __u64 *vblocknrs, size_t nitems) +{ + return nilfs_palloc_freev(dat, vblocknrs, nitems); +} + +/** + * nilfs_dat_move - change a block number + * @dat: DAT file inode + * @vblocknr: virtual block number + * @blocknr: block number + * + * Description: nilfs_dat_move() changes the block number associated with + * @vblocknr to @blocknr. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_dat_move(struct inode *dat, __u64 vblocknr, sector_t blocknr) +{ + struct buffer_head *entry_bh; + struct nilfs_dat_entry *entry; + size_t offset; + int ret; + + ret = nilfs_palloc_get_entry_block(dat, vblocknr, 0, &entry_bh); + if (ret < 0) + return ret; + + /* + * The given disk block number (blocknr) is not yet written to + * the device at this point. + * + * To prevent nilfs_dat_translate() from returning the + * uncommitted block number, this makes a copy of the entry + * buffer and redirects nilfs_dat_translate() to the copy. + */ + if (!buffer_nilfs_redirected(entry_bh)) { + ret = nilfs_mdt_freeze_buffer(dat, entry_bh); + if (ret) { + brelse(entry_bh); + return ret; + } + } + + offset = nilfs_palloc_entry_offset(dat, vblocknr, entry_bh); + entry = kmap_local_folio(entry_bh->b_folio, offset); + if (unlikely(entry->de_blocknr == cpu_to_le64(0))) { + nilfs_crit(dat->i_sb, + "%s: invalid vblocknr = %llu, [%llu, %llu)", + __func__, (unsigned long long)vblocknr, + (unsigned long long)le64_to_cpu(entry->de_start), + (unsigned long long)le64_to_cpu(entry->de_end)); + kunmap_local(entry); + brelse(entry_bh); + return -EINVAL; + } + WARN_ON(blocknr == 0); + entry->de_blocknr = cpu_to_le64(blocknr); + kunmap_local(entry); + + mark_buffer_dirty(entry_bh); + nilfs_mdt_mark_dirty(dat); + + brelse(entry_bh); + + return 0; +} + +/** + * nilfs_dat_translate - translate a virtual block number to a block number + * @dat: DAT file inode + * @vblocknr: virtual block number + * @blocknrp: pointer to a block number + * + * Description: nilfs_dat_translate() maps the virtual block number @vblocknr + * to the corresponding block number. The block number associated with + * @vblocknr is stored in the place pointed to by @blocknrp. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOENT - A block number associated with @vblocknr does not exist. + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_dat_translate(struct inode *dat, __u64 vblocknr, sector_t *blocknrp) +{ + struct buffer_head *entry_bh, *bh; + struct nilfs_dat_entry *entry; + sector_t blocknr; + size_t offset; + int ret; + + ret = nilfs_palloc_get_entry_block(dat, vblocknr, 0, &entry_bh); + if (ret < 0) + return ret; + + if (!nilfs_doing_gc() && buffer_nilfs_redirected(entry_bh)) { + bh = nilfs_mdt_get_frozen_buffer(dat, entry_bh); + if (bh) { + WARN_ON(!buffer_uptodate(bh)); + brelse(entry_bh); + entry_bh = bh; + } + } + + offset = nilfs_palloc_entry_offset(dat, vblocknr, entry_bh); + entry = kmap_local_folio(entry_bh->b_folio, offset); + blocknr = le64_to_cpu(entry->de_blocknr); + if (blocknr == 0) { + ret = -ENOENT; + goto out; + } + *blocknrp = blocknr; + + out: + kunmap_local(entry); + brelse(entry_bh); + return ret; +} + +ssize_t nilfs_dat_get_vinfo(struct inode *dat, void *buf, unsigned int visz, + size_t nvi) +{ + struct buffer_head *entry_bh; + struct nilfs_dat_entry *entry, *first_entry; + struct nilfs_vinfo *vinfo = buf; + __u64 first, last; + size_t offset; + unsigned long entries_per_block = NILFS_MDT(dat)->mi_entries_per_block; + unsigned int entry_size = NILFS_MDT(dat)->mi_entry_size; + int i, j, n, ret; + + for (i = 0; i < nvi; i += n) { + ret = nilfs_palloc_get_entry_block(dat, vinfo->vi_vblocknr, + 0, &entry_bh); + if (ret < 0) + return ret; + + first = vinfo->vi_vblocknr; + first = div64_ul(first, entries_per_block); + first *= entries_per_block; + /* first virtual block number in this block */ + + last = first + entries_per_block - 1; + /* last virtual block number in this block */ + + offset = nilfs_palloc_entry_offset(dat, first, entry_bh); + first_entry = kmap_local_folio(entry_bh->b_folio, offset); + for (j = i, n = 0; + j < nvi && vinfo->vi_vblocknr >= first && + vinfo->vi_vblocknr <= last; + j++, n++, vinfo = (void *)vinfo + visz) { + entry = (void *)first_entry + + (vinfo->vi_vblocknr - first) * entry_size; + vinfo->vi_start = le64_to_cpu(entry->de_start); + vinfo->vi_end = le64_to_cpu(entry->de_end); + vinfo->vi_blocknr = le64_to_cpu(entry->de_blocknr); + } + kunmap_local(first_entry); + brelse(entry_bh); + } + + return nvi; +} + +/** + * nilfs_dat_read - read or get dat inode + * @sb: super block instance + * @entry_size: size of a dat entry + * @raw_inode: on-disk dat inode + * @inodep: buffer to store the inode + * + * Return: 0 on success, or a negative error code on failure. + */ +int nilfs_dat_read(struct super_block *sb, size_t entry_size, + struct nilfs_inode *raw_inode, struct inode **inodep) +{ + static struct lock_class_key dat_lock_key; + struct inode *dat; + struct nilfs_dat_info *di; + int err; + + if (entry_size > sb->s_blocksize) { + nilfs_err(sb, "too large DAT entry size: %zu bytes", + entry_size); + return -EINVAL; + } else if (entry_size < NILFS_MIN_DAT_ENTRY_SIZE) { + nilfs_err(sb, "too small DAT entry size: %zu bytes", + entry_size); + return -EINVAL; + } + + dat = nilfs_iget_locked(sb, NULL, NILFS_DAT_INO); + if (unlikely(!dat)) + return -ENOMEM; + if (!(inode_state_read_once(dat) & I_NEW)) + goto out; + + err = nilfs_mdt_init(dat, NILFS_MDT_GFP, sizeof(*di)); + if (err) + goto failed; + + err = nilfs_palloc_init_blockgroup(dat, entry_size); + if (err) + goto failed; + + di = NILFS_DAT_I(dat); + lockdep_set_class(&di->mi.mi_sem, &dat_lock_key); + nilfs_palloc_setup_cache(dat, &di->palloc_cache); + err = nilfs_mdt_setup_shadow_map(dat, &di->shadow); + if (err) + goto failed; + + err = nilfs_attach_btree_node_cache(dat); + if (err) + goto failed; + err = nilfs_read_inode_common(dat, raw_inode); + if (err) + goto failed; + + unlock_new_inode(dat); + out: + *inodep = dat; + return 0; + failed: + iget_failed(dat); + return err; +} diff --git a/fs/nilfs2/dat.h b/fs/nilfs2/dat.h new file mode 100644 index 000000000..468c82d26 --- /dev/null +++ b/fs/nilfs2/dat.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * NILFS disk address translation. + * + * Copyright (C) 2006-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Koji Sato. + */ + +#ifndef _NILFS_DAT_H +#define _NILFS_DAT_H + +#include +#include +#include +#include /* nilfs_inode, nilfs_checkpoint */ + + +struct nilfs_palloc_req; + +int nilfs_dat_translate(struct inode *, __u64, sector_t *); + +int nilfs_dat_prepare_alloc(struct inode *, struct nilfs_palloc_req *); +void nilfs_dat_commit_alloc(struct inode *, struct nilfs_palloc_req *); +void nilfs_dat_abort_alloc(struct inode *, struct nilfs_palloc_req *); +int nilfs_dat_prepare_start(struct inode *, struct nilfs_palloc_req *); +void nilfs_dat_commit_start(struct inode *, struct nilfs_palloc_req *, + sector_t); +int nilfs_dat_prepare_end(struct inode *, struct nilfs_palloc_req *); +void nilfs_dat_commit_end(struct inode *, struct nilfs_palloc_req *, int); +void nilfs_dat_abort_end(struct inode *, struct nilfs_palloc_req *); +int nilfs_dat_prepare_update(struct inode *, struct nilfs_palloc_req *, + struct nilfs_palloc_req *); +void nilfs_dat_commit_update(struct inode *, struct nilfs_palloc_req *, + struct nilfs_palloc_req *, int); +void nilfs_dat_abort_update(struct inode *, struct nilfs_palloc_req *, + struct nilfs_palloc_req *); + +int nilfs_dat_mark_dirty(struct inode *, __u64); +int nilfs_dat_freev(struct inode *, __u64 *, size_t); +int nilfs_dat_move(struct inode *, __u64, sector_t); +ssize_t nilfs_dat_get_vinfo(struct inode *, void *, unsigned int, size_t); + +int nilfs_dat_read(struct super_block *sb, size_t entry_size, + struct nilfs_inode *raw_inode, struct inode **inodep); + +#endif /* _NILFS_DAT_H */ diff --git a/fs/nilfs2/dir.c b/fs/nilfs2/dir.c new file mode 100644 index 000000000..8b53802b6 --- /dev/null +++ b/fs/nilfs2/dir.c @@ -0,0 +1,665 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * NILFS directory entry operations + * + * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation. + * + * Modified for NILFS by Amagai Yoshiji. + */ +/* + * linux/fs/ext2/dir.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * from + * + * linux/fs/minix/dir.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * ext2 directory handling functions + * + * Big-endian to little-endian byte-swapping/bitmaps by + * David S. Miller (davem@caip.rutgers.edu), 1995 + * + * All code that works with directory layout had been switched to pagecache + * and moved here. AV + */ + +#include +#include +#include "nilfs.h" +#include "page.h" + +static inline unsigned int nilfs_rec_len_from_disk(__le16 dlen) +{ + unsigned int len = le16_to_cpu(dlen); + +#if (PAGE_SIZE >= 65536) + if (len == NILFS_MAX_REC_LEN) + return 1 << 16; +#endif + return len; +} + +static inline __le16 nilfs_rec_len_to_disk(unsigned int len) +{ +#if (PAGE_SIZE >= 65536) + if (len == (1 << 16)) + return cpu_to_le16(NILFS_MAX_REC_LEN); + + BUG_ON(len > (1 << 16)); +#endif + return cpu_to_le16(len); +} + +/* + * nilfs uses block-sized chunks. Arguably, sector-sized ones would be + * more robust, but we have what we have + */ +static inline unsigned int nilfs_chunk_size(struct inode *inode) +{ + return inode->i_sb->s_blocksize; +} + +/* + * Return the offset into page `page_nr' of the last valid + * byte in that page, plus one. + */ +static unsigned int nilfs_last_byte(struct inode *inode, unsigned long page_nr) +{ + u64 last_byte = inode->i_size; + + last_byte -= page_nr << PAGE_SHIFT; + if (last_byte > PAGE_SIZE) + last_byte = PAGE_SIZE; + return last_byte; +} + +static int nilfs_prepare_chunk(struct folio *folio, unsigned int from, + unsigned int to) +{ + loff_t pos = folio_pos(folio) + from; + + return __block_write_begin(folio, pos, to - from, nilfs_get_block); +} + +static void nilfs_commit_chunk(struct folio *folio, + struct address_space *mapping, size_t from, size_t to) +{ + struct inode *dir = mapping->host; + loff_t pos = folio_pos(folio) + from; + size_t copied, len = to - from; + unsigned int nr_dirty; + int err; + + nr_dirty = nilfs_page_count_clean_buffers(folio, from, to); + copied = block_write_end(pos, len, len, folio); + if (pos + copied > dir->i_size) + i_size_write(dir, pos + copied); + if (IS_DIRSYNC(dir)) + nilfs_set_transaction_flag(NILFS_TI_SYNC); + err = nilfs_set_file_dirty(dir, nr_dirty); + WARN_ON(err); /* do not happen */ + folio_unlock(folio); +} + +static bool nilfs_check_folio(struct folio *folio, char *kaddr) +{ + struct inode *dir = folio->mapping->host; + struct super_block *sb = dir->i_sb; + unsigned int chunk_size = nilfs_chunk_size(dir); + size_t offs, rec_len; + size_t limit = folio_size(folio); + struct nilfs_dir_entry *p; + char *error; + + if (dir->i_size < folio_pos(folio) + limit) { + limit = dir->i_size - folio_pos(folio); + if (limit & (chunk_size - 1)) + goto Ebadsize; + if (!limit) + goto out; + } + for (offs = 0; offs <= limit - NILFS_DIR_REC_LEN(1); offs += rec_len) { + p = (struct nilfs_dir_entry *)(kaddr + offs); + rec_len = nilfs_rec_len_from_disk(p->rec_len); + + if (rec_len < NILFS_DIR_REC_LEN(1)) + goto Eshort; + if (rec_len & 3) + goto Ealign; + if (rec_len < NILFS_DIR_REC_LEN(p->name_len)) + goto Enamelen; + if (((offs + rec_len - 1) ^ offs) & ~(chunk_size-1)) + goto Espan; + if (unlikely(p->inode && + NILFS_PRIVATE_INODE(le64_to_cpu(p->inode)))) + goto Einumber; + } + if (offs != limit) + goto Eend; +out: + folio_set_checked(folio); + return true; + + /* Too bad, we had an error */ + +Ebadsize: + nilfs_error(sb, + "size of directory #%llu is not a multiple of chunk size", + dir->i_ino); + goto fail; +Eshort: + error = "rec_len is smaller than minimal"; + goto bad_entry; +Ealign: + error = "unaligned directory entry"; + goto bad_entry; +Enamelen: + error = "rec_len is too small for name_len"; + goto bad_entry; +Espan: + error = "directory entry across blocks"; + goto bad_entry; +Einumber: + error = "disallowed inode number"; +bad_entry: + nilfs_error(sb, + "bad entry in directory #%llu: %s - offset=%lu, inode=%llu, rec_len=%zd, name_len=%d", + dir->i_ino, error, (folio->index << PAGE_SHIFT) + offs, + le64_to_cpu(p->inode), rec_len, p->name_len); + goto fail; +Eend: + p = (struct nilfs_dir_entry *)(kaddr + offs); + nilfs_error(sb, + "entry in directory #%llu spans the page boundary offset=%lu, inode=%llu", + dir->i_ino, (folio->index << PAGE_SHIFT) + offs, + le64_to_cpu(p->inode)); +fail: + return false; +} + +static void *nilfs_get_folio(struct inode *dir, unsigned long n, + struct folio **foliop) +{ + struct address_space *mapping = dir->i_mapping; + struct folio *folio = read_mapping_folio(mapping, n, NULL); + void *kaddr; + + if (IS_ERR(folio)) + return folio; + + kaddr = kmap_local_folio(folio, 0); + if (unlikely(!folio_test_checked(folio))) { + if (!nilfs_check_folio(folio, kaddr)) + goto fail; + } + + *foliop = folio; + return kaddr; + +fail: + folio_release_kmap(folio, kaddr); + return ERR_PTR(-EIO); +} + +/* + * NOTE! unlike strncmp, nilfs_match returns 1 for success, 0 for failure. + * + * len <= NILFS_NAME_LEN and de != NULL are guaranteed by caller. + */ +static int +nilfs_match(int len, const unsigned char *name, struct nilfs_dir_entry *de) +{ + if (len != de->name_len) + return 0; + if (!de->inode) + return 0; + return !memcmp(name, de->name, len); +} + +/* + * p is at least 6 bytes before the end of page + */ +static struct nilfs_dir_entry *nilfs_next_entry(struct nilfs_dir_entry *p) +{ + return (struct nilfs_dir_entry *)((char *)p + + nilfs_rec_len_from_disk(p->rec_len)); +} + +static int nilfs_readdir(struct file *file, struct dir_context *ctx) +{ + loff_t pos = ctx->pos; + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + unsigned int offset = pos & ~PAGE_MASK; + unsigned long n = pos >> PAGE_SHIFT; + unsigned long npages = dir_pages(inode); + + if (pos > inode->i_size - NILFS_DIR_REC_LEN(1)) + return 0; + + for ( ; n < npages; n++, offset = 0) { + char *kaddr, *limit; + struct nilfs_dir_entry *de; + struct folio *folio; + + kaddr = nilfs_get_folio(inode, n, &folio); + if (IS_ERR(kaddr)) { + nilfs_error(sb, "bad page in #%llu", inode->i_ino); + ctx->pos += PAGE_SIZE - offset; + return -EIO; + } + de = (struct nilfs_dir_entry *)(kaddr + offset); + limit = kaddr + nilfs_last_byte(inode, n) - + NILFS_DIR_REC_LEN(1); + for ( ; (char *)de <= limit; de = nilfs_next_entry(de)) { + if (de->rec_len == 0) { + nilfs_error(sb, "zero-length directory entry"); + folio_release_kmap(folio, kaddr); + return -EIO; + } + if (de->inode) { + unsigned char t; + + t = fs_ftype_to_dtype(de->file_type); + + if (!dir_emit(ctx, de->name, de->name_len, + le64_to_cpu(de->inode), t)) { + folio_release_kmap(folio, kaddr); + return 0; + } + } + ctx->pos += nilfs_rec_len_from_disk(de->rec_len); + } + folio_release_kmap(folio, kaddr); + } + return 0; +} + +/* + * nilfs_find_entry() + * + * Finds an entry in the specified directory with the wanted name. It + * returns the folio in which the entry was found, and the entry itself. + * The folio is mapped and unlocked. When the caller is finished with + * the entry, it should call folio_release_kmap(). + * + * On failure, returns an error pointer and the caller should ignore foliop. + */ +struct nilfs_dir_entry *nilfs_find_entry(struct inode *dir, + const struct qstr *qstr, struct folio **foliop) +{ + const unsigned char *name = qstr->name; + int namelen = qstr->len; + unsigned int reclen = NILFS_DIR_REC_LEN(namelen); + unsigned long start, n; + unsigned long npages = dir_pages(dir); + struct nilfs_inode_info *ei = NILFS_I(dir); + struct nilfs_dir_entry *de; + + if (npages == 0) + goto out; + + start = ei->i_dir_start_lookup; + if (start >= npages) + start = 0; + n = start; + do { + char *kaddr = nilfs_get_folio(dir, n, foliop); + + if (IS_ERR(kaddr)) + return ERR_CAST(kaddr); + + de = (struct nilfs_dir_entry *)kaddr; + kaddr += nilfs_last_byte(dir, n) - reclen; + while ((char *)de <= kaddr) { + if (de->rec_len == 0) { + nilfs_error(dir->i_sb, + "zero-length directory entry"); + folio_release_kmap(*foliop, kaddr); + goto out; + } + if (nilfs_match(namelen, name, de)) + goto found; + de = nilfs_next_entry(de); + } + folio_release_kmap(*foliop, kaddr); + + if (++n >= npages) + n = 0; + /* next folio is past the blocks we've got */ + if (unlikely(n > (dir->i_blocks >> (PAGE_SHIFT - 9)))) { + nilfs_error(dir->i_sb, + "dir %llu size %lld exceeds block count %llu", + dir->i_ino, dir->i_size, + (unsigned long long)dir->i_blocks); + goto out; + } + } while (n != start); +out: + return ERR_PTR(-ENOENT); + +found: + ei->i_dir_start_lookup = n; + return de; +} + +struct nilfs_dir_entry *nilfs_dotdot(struct inode *dir, struct folio **foliop) +{ + struct folio *folio; + struct nilfs_dir_entry *de, *next_de; + size_t limit; + char *msg; + + de = nilfs_get_folio(dir, 0, &folio); + if (IS_ERR(de)) + return NULL; + + limit = nilfs_last_byte(dir, 0); /* is a multiple of chunk size */ + if (unlikely(!limit || le64_to_cpu(de->inode) != dir->i_ino || + !nilfs_match(1, ".", de))) { + msg = "missing '.'"; + goto fail; + } + + next_de = nilfs_next_entry(de); + /* + * If "next_de" has not reached the end of the chunk, there is + * at least one more record. Check whether it matches "..". + */ + if (unlikely((char *)next_de == (char *)de + nilfs_chunk_size(dir) || + !nilfs_match(2, "..", next_de))) { + msg = "missing '..'"; + goto fail; + } + *foliop = folio; + return next_de; + +fail: + nilfs_error(dir->i_sb, "directory #%llu %s", dir->i_ino, msg); + folio_release_kmap(folio, de); + return NULL; +} + +int nilfs_inode_by_name(struct inode *dir, const struct qstr *qstr, u64 *ino) +{ + struct nilfs_dir_entry *de; + struct folio *folio; + + de = nilfs_find_entry(dir, qstr, &folio); + if (IS_ERR(de)) + return PTR_ERR(de); + + *ino = le64_to_cpu(de->inode); + folio_release_kmap(folio, de); + return 0; +} + +int nilfs_set_link(struct inode *dir, struct nilfs_dir_entry *de, + struct folio *folio, struct inode *inode) +{ + size_t from = offset_in_folio(folio, de); + size_t to = from + nilfs_rec_len_from_disk(de->rec_len); + struct address_space *mapping = folio->mapping; + int err; + + folio_lock(folio); + err = nilfs_prepare_chunk(folio, from, to); + if (unlikely(err)) { + folio_unlock(folio); + return err; + } + de->inode = cpu_to_le64(inode->i_ino); + de->file_type = fs_umode_to_ftype(inode->i_mode); + nilfs_commit_chunk(folio, mapping, from, to); + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + return 0; +} + +/* + * Parent is locked. + */ +int nilfs_add_link(struct dentry *dentry, struct inode *inode) +{ + struct inode *dir = d_inode(dentry->d_parent); + const unsigned char *name = dentry->d_name.name; + int namelen = dentry->d_name.len; + unsigned int chunk_size = nilfs_chunk_size(dir); + unsigned int reclen = NILFS_DIR_REC_LEN(namelen); + unsigned short rec_len, name_len; + struct folio *folio = NULL; + struct nilfs_dir_entry *de; + unsigned long npages = dir_pages(dir); + unsigned long n; + size_t from, to; + int err; + + /* + * We take care of directory expansion in the same loop. + * This code plays outside i_size, so it locks the folio + * to protect that region. + */ + for (n = 0; n <= npages; n++) { + char *kaddr = nilfs_get_folio(dir, n, &folio); + char *dir_end; + + if (IS_ERR(kaddr)) + return PTR_ERR(kaddr); + folio_lock(folio); + dir_end = kaddr + nilfs_last_byte(dir, n); + de = (struct nilfs_dir_entry *)kaddr; + kaddr += folio_size(folio) - reclen; + while ((char *)de <= kaddr) { + if ((char *)de == dir_end) { + /* We hit i_size */ + name_len = 0; + rec_len = chunk_size; + de->rec_len = nilfs_rec_len_to_disk(chunk_size); + de->inode = 0; + goto got_it; + } + if (de->rec_len == 0) { + nilfs_error(dir->i_sb, + "zero-length directory entry"); + err = -EIO; + goto out_unlock; + } + err = -EEXIST; + if (nilfs_match(namelen, name, de)) + goto out_unlock; + name_len = NILFS_DIR_REC_LEN(de->name_len); + rec_len = nilfs_rec_len_from_disk(de->rec_len); + if (!de->inode && rec_len >= reclen) + goto got_it; + if (rec_len >= name_len + reclen) + goto got_it; + de = (struct nilfs_dir_entry *)((char *)de + rec_len); + } + folio_unlock(folio); + folio_release_kmap(folio, kaddr); + } + BUG(); + return -EINVAL; + +got_it: + from = offset_in_folio(folio, de); + to = from + rec_len; + err = nilfs_prepare_chunk(folio, from, to); + if (err) + goto out_unlock; + if (de->inode) { + struct nilfs_dir_entry *de1; + + de1 = (struct nilfs_dir_entry *)((char *)de + name_len); + de1->rec_len = nilfs_rec_len_to_disk(rec_len - name_len); + de->rec_len = nilfs_rec_len_to_disk(name_len); + de = de1; + } + de->name_len = namelen; + memcpy(de->name, name, namelen); + de->inode = cpu_to_le64(inode->i_ino); + de->file_type = fs_umode_to_ftype(inode->i_mode); + nilfs_commit_chunk(folio, folio->mapping, from, to); + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + nilfs_mark_inode_dirty(dir); + /* OFFSET_CACHE */ +out_put: + folio_release_kmap(folio, de); + return err; +out_unlock: + folio_unlock(folio); + goto out_put; +} + +/* + * nilfs_delete_entry deletes a directory entry by merging it with the + * previous entry. Folio is up-to-date. + */ +int nilfs_delete_entry(struct nilfs_dir_entry *dir, struct folio *folio) +{ + struct address_space *mapping = folio->mapping; + struct inode *inode = mapping->host; + char *kaddr = (char *)((unsigned long)dir & ~(folio_size(folio) - 1)); + size_t from, to; + struct nilfs_dir_entry *de, *pde = NULL; + int err; + + from = ((char *)dir - kaddr) & ~(nilfs_chunk_size(inode) - 1); + to = ((char *)dir - kaddr) + nilfs_rec_len_from_disk(dir->rec_len); + de = (struct nilfs_dir_entry *)(kaddr + from); + + while ((char *)de < (char *)dir) { + if (de->rec_len == 0) { + nilfs_error(inode->i_sb, + "zero-length directory entry"); + err = -EIO; + goto out; + } + pde = de; + de = nilfs_next_entry(de); + } + if (pde) + from = (char *)pde - kaddr; + folio_lock(folio); + err = nilfs_prepare_chunk(folio, from, to); + if (unlikely(err)) { + folio_unlock(folio); + goto out; + } + if (pde) + pde->rec_len = nilfs_rec_len_to_disk(to - from); + dir->inode = 0; + nilfs_commit_chunk(folio, mapping, from, to); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); +out: + return err; +} + +/* + * Set the first fragment of directory. + */ +int nilfs_make_empty(struct inode *inode, struct inode *parent) +{ + struct address_space *mapping = inode->i_mapping; + struct folio *folio = filemap_grab_folio(mapping, 0); + unsigned int chunk_size = nilfs_chunk_size(inode); + struct nilfs_dir_entry *de; + int err; + void *kaddr; + + if (IS_ERR(folio)) + return PTR_ERR(folio); + + err = nilfs_prepare_chunk(folio, 0, chunk_size); + if (unlikely(err)) { + folio_unlock(folio); + goto fail; + } + kaddr = kmap_local_folio(folio, 0); + memset(kaddr, 0, chunk_size); + de = (struct nilfs_dir_entry *)kaddr; + de->name_len = 1; + de->rec_len = nilfs_rec_len_to_disk(NILFS_DIR_REC_LEN(1)); + memcpy(de->name, ".\0\0", 4); + de->inode = cpu_to_le64(inode->i_ino); + de->file_type = fs_umode_to_ftype(inode->i_mode); + + de = (struct nilfs_dir_entry *)(kaddr + NILFS_DIR_REC_LEN(1)); + de->name_len = 2; + de->rec_len = nilfs_rec_len_to_disk(chunk_size - NILFS_DIR_REC_LEN(1)); + de->inode = cpu_to_le64(parent->i_ino); + memcpy(de->name, "..\0", 4); + de->file_type = fs_umode_to_ftype(inode->i_mode); + kunmap_local(kaddr); + nilfs_commit_chunk(folio, mapping, 0, chunk_size); +fail: + folio_put(folio); + return err; +} + +/* + * routine to check that the specified directory is empty (for rmdir) + */ +int nilfs_empty_dir(struct inode *inode) +{ + struct folio *folio = NULL; + char *kaddr; + unsigned long i, npages = dir_pages(inode); + + for (i = 0; i < npages; i++) { + struct nilfs_dir_entry *de; + + kaddr = nilfs_get_folio(inode, i, &folio); + if (IS_ERR(kaddr)) + return 0; + + de = (struct nilfs_dir_entry *)kaddr; + kaddr += nilfs_last_byte(inode, i) - NILFS_DIR_REC_LEN(1); + + while ((char *)de <= kaddr) { + if (de->rec_len == 0) { + nilfs_error(inode->i_sb, + "zero-length directory entry (kaddr=%p, de=%p)", + kaddr, de); + goto not_empty; + } + if (de->inode != 0) { + /* check for . and .. */ + if (de->name[0] != '.') + goto not_empty; + if (de->name_len > 2) + goto not_empty; + if (de->name_len < 2) { + if (de->inode != + cpu_to_le64(inode->i_ino)) + goto not_empty; + } else if (de->name[1] != '.') + goto not_empty; + } + de = nilfs_next_entry(de); + } + folio_release_kmap(folio, kaddr); + } + return 1; + +not_empty: + folio_release_kmap(folio, kaddr); + return 0; +} + +const struct file_operations nilfs_dir_operations = { + .llseek = generic_file_llseek, + .read = generic_read_dir, + .iterate_shared = nilfs_readdir, + .unlocked_ioctl = nilfs_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = nilfs_compat_ioctl, +#endif /* CONFIG_COMPAT */ + .fsync = nilfs_sync_file, + .setlease = generic_setlease, +}; diff --git a/fs/nilfs2/direct.c b/fs/nilfs2/direct.c new file mode 100644 index 000000000..b8643d3aa --- /dev/null +++ b/fs/nilfs2/direct.c @@ -0,0 +1,387 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * NILFS direct block pointer. + * + * Copyright (C) 2006-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Koji Sato. + */ + +#include +#include "nilfs.h" +#include "page.h" +#include "direct.h" +#include "alloc.h" +#include "dat.h" + +static inline __le64 *nilfs_direct_dptrs(const struct nilfs_bmap *direct) +{ + return (__le64 *) + ((struct nilfs_direct_node *)direct->b_u.u_data + 1); +} + +static inline __u64 +nilfs_direct_get_ptr(const struct nilfs_bmap *direct, __u64 key) +{ + return le64_to_cpu(*(nilfs_direct_dptrs(direct) + key)); +} + +static inline void nilfs_direct_set_ptr(struct nilfs_bmap *direct, + __u64 key, __u64 ptr) +{ + *(nilfs_direct_dptrs(direct) + key) = cpu_to_le64(ptr); +} + +static int nilfs_direct_lookup(const struct nilfs_bmap *direct, + __u64 key, int level, __u64 *ptrp) +{ + __u64 ptr; + + if (key > NILFS_DIRECT_KEY_MAX || level != 1) + return -ENOENT; + ptr = nilfs_direct_get_ptr(direct, key); + if (ptr == NILFS_BMAP_INVALID_PTR) + return -ENOENT; + + *ptrp = ptr; + return 0; +} + +static int nilfs_direct_lookup_contig(const struct nilfs_bmap *direct, + __u64 key, __u64 *ptrp, + unsigned int maxblocks) +{ + struct inode *dat = NULL; + __u64 ptr, ptr2; + sector_t blocknr; + int ret, cnt; + + if (key > NILFS_DIRECT_KEY_MAX) + return -ENOENT; + ptr = nilfs_direct_get_ptr(direct, key); + if (ptr == NILFS_BMAP_INVALID_PTR) + return -ENOENT; + + if (NILFS_BMAP_USE_VBN(direct)) { + dat = nilfs_bmap_get_dat(direct); + ret = nilfs_dat_translate(dat, ptr, &blocknr); + if (ret < 0) + goto dat_error; + ptr = blocknr; + } + + maxblocks = min_t(unsigned int, maxblocks, + NILFS_DIRECT_KEY_MAX - key + 1); + for (cnt = 1; cnt < maxblocks && + (ptr2 = nilfs_direct_get_ptr(direct, key + cnt)) != + NILFS_BMAP_INVALID_PTR; + cnt++) { + if (dat) { + ret = nilfs_dat_translate(dat, ptr2, &blocknr); + if (ret < 0) + goto dat_error; + ptr2 = blocknr; + } + if (ptr2 != ptr + cnt) + break; + } + *ptrp = ptr; + return cnt; + + dat_error: + if (ret == -ENOENT) + ret = -EINVAL; /* Notify bmap layer of metadata corruption */ + return ret; +} + +static __u64 +nilfs_direct_find_target_v(const struct nilfs_bmap *direct, __u64 key) +{ + __u64 ptr; + + ptr = nilfs_bmap_find_target_seq(direct, key); + if (ptr != NILFS_BMAP_INVALID_PTR) + /* sequential access */ + return ptr; + + /* block group */ + return nilfs_bmap_find_target_in_group(direct); +} + +static int nilfs_direct_insert(struct nilfs_bmap *bmap, __u64 key, __u64 ptr) +{ + union nilfs_bmap_ptr_req req; + struct inode *dat = NULL; + struct buffer_head *bh; + int ret; + + if (key > NILFS_DIRECT_KEY_MAX) + return -ENOENT; + if (nilfs_direct_get_ptr(bmap, key) != NILFS_BMAP_INVALID_PTR) + return -EEXIST; + + if (NILFS_BMAP_USE_VBN(bmap)) { + req.bpr_ptr = nilfs_direct_find_target_v(bmap, key); + dat = nilfs_bmap_get_dat(bmap); + } + ret = nilfs_bmap_prepare_alloc_ptr(bmap, &req, dat); + if (!ret) { + /* ptr must be a pointer to a buffer head. */ + bh = (struct buffer_head *)((unsigned long)ptr); + set_buffer_nilfs_volatile(bh); + + nilfs_bmap_commit_alloc_ptr(bmap, &req, dat); + nilfs_direct_set_ptr(bmap, key, req.bpr_ptr); + + if (!nilfs_bmap_dirty(bmap)) + nilfs_bmap_set_dirty(bmap); + + if (NILFS_BMAP_USE_VBN(bmap)) + nilfs_bmap_set_target_v(bmap, key, req.bpr_ptr); + + nilfs_inode_add_blocks(bmap->b_inode, 1); + } + return ret; +} + +static int nilfs_direct_delete(struct nilfs_bmap *bmap, __u64 key, bool deform) +{ + union nilfs_bmap_ptr_req req; + struct inode *dat; + int ret; + + if (key > NILFS_DIRECT_KEY_MAX || + nilfs_direct_get_ptr(bmap, key) == NILFS_BMAP_INVALID_PTR) + return -ENOENT; + + dat = NILFS_BMAP_USE_VBN(bmap) ? nilfs_bmap_get_dat(bmap) : NULL; + req.bpr_ptr = nilfs_direct_get_ptr(bmap, key); + + ret = nilfs_bmap_prepare_end_ptr(bmap, &req, dat); + if (!ret) { + nilfs_bmap_commit_end_ptr(bmap, &req, dat); + nilfs_direct_set_ptr(bmap, key, NILFS_BMAP_INVALID_PTR); + nilfs_inode_sub_blocks(bmap->b_inode, 1); + } + return ret; +} + +static int nilfs_direct_seek_key(const struct nilfs_bmap *direct, __u64 start, + __u64 *keyp) +{ + __u64 key; + + for (key = start; key <= NILFS_DIRECT_KEY_MAX; key++) { + if (nilfs_direct_get_ptr(direct, key) != + NILFS_BMAP_INVALID_PTR) { + *keyp = key; + return 0; + } + } + return -ENOENT; +} + +static int nilfs_direct_last_key(const struct nilfs_bmap *direct, __u64 *keyp) +{ + __u64 key, lastkey; + + lastkey = NILFS_DIRECT_KEY_MAX + 1; + for (key = NILFS_DIRECT_KEY_MIN; key <= NILFS_DIRECT_KEY_MAX; key++) + if (nilfs_direct_get_ptr(direct, key) != + NILFS_BMAP_INVALID_PTR) + lastkey = key; + + if (lastkey == NILFS_DIRECT_KEY_MAX + 1) + return -ENOENT; + + *keyp = lastkey; + + return 0; +} + +static int nilfs_direct_check_insert(const struct nilfs_bmap *bmap, __u64 key) +{ + return key > NILFS_DIRECT_KEY_MAX; +} + +static int nilfs_direct_gather_data(struct nilfs_bmap *direct, + __u64 *keys, __u64 *ptrs, int nitems) +{ + __u64 key; + __u64 ptr; + int n; + + if (nitems > NILFS_DIRECT_NBLOCKS) + nitems = NILFS_DIRECT_NBLOCKS; + n = 0; + for (key = 0; key < nitems; key++) { + ptr = nilfs_direct_get_ptr(direct, key); + if (ptr != NILFS_BMAP_INVALID_PTR) { + keys[n] = key; + ptrs[n] = ptr; + n++; + } + } + return n; +} + +int nilfs_direct_delete_and_convert(struct nilfs_bmap *bmap, + __u64 key, __u64 *keys, __u64 *ptrs, int n) +{ + __le64 *dptrs; + int ret, i, j; + + /* no need to allocate any resource for conversion */ + + /* delete */ + ret = bmap->b_ops->bop_delete(bmap, key, true); + if (ret < 0) + return ret; + + /* free resources */ + if (bmap->b_ops->bop_clear != NULL) + bmap->b_ops->bop_clear(bmap); + + /* convert */ + dptrs = nilfs_direct_dptrs(bmap); + for (i = 0, j = 0; i < NILFS_DIRECT_NBLOCKS; i++) { + if ((j < n) && (i == keys[j])) { + dptrs[i] = (i != key) ? + cpu_to_le64(ptrs[j]) : + NILFS_BMAP_INVALID_PTR; + j++; + } else + dptrs[i] = NILFS_BMAP_INVALID_PTR; + } + + nilfs_direct_init(bmap); + return 0; +} + +static int nilfs_direct_propagate(struct nilfs_bmap *bmap, + struct buffer_head *bh) +{ + struct nilfs_palloc_req oldreq, newreq; + struct inode *dat; + __u64 key; + __u64 ptr; + int ret; + + if (!NILFS_BMAP_USE_VBN(bmap)) + return 0; + + dat = nilfs_bmap_get_dat(bmap); + key = nilfs_bmap_data_get_key(bmap, bh); + ptr = nilfs_direct_get_ptr(bmap, key); + if (ptr == NILFS_BMAP_INVALID_PTR) + return -EINVAL; + + if (!buffer_nilfs_volatile(bh)) { + oldreq.pr_entry_nr = ptr; + newreq.pr_entry_nr = ptr; + ret = nilfs_dat_prepare_update(dat, &oldreq, &newreq); + if (ret < 0) + return ret; + nilfs_dat_commit_update(dat, &oldreq, &newreq, + bmap->b_ptr_type == NILFS_BMAP_PTR_VS); + set_buffer_nilfs_volatile(bh); + nilfs_direct_set_ptr(bmap, key, newreq.pr_entry_nr); + } else + ret = nilfs_dat_mark_dirty(dat, ptr); + + return ret; +} + +static int nilfs_direct_assign_v(struct nilfs_bmap *direct, + __u64 key, __u64 ptr, + struct buffer_head **bh, + sector_t blocknr, + union nilfs_binfo *binfo) +{ + struct inode *dat = nilfs_bmap_get_dat(direct); + union nilfs_bmap_ptr_req req; + int ret; + + req.bpr_ptr = ptr; + ret = nilfs_dat_prepare_start(dat, &req.bpr_req); + if (!ret) { + nilfs_dat_commit_start(dat, &req.bpr_req, blocknr); + binfo->bi_v.bi_vblocknr = cpu_to_le64(ptr); + binfo->bi_v.bi_blkoff = cpu_to_le64(key); + } + return ret; +} + +static int nilfs_direct_assign_p(struct nilfs_bmap *direct, + __u64 key, __u64 ptr, + struct buffer_head **bh, + sector_t blocknr, + union nilfs_binfo *binfo) +{ + nilfs_direct_set_ptr(direct, key, blocknr); + + binfo->bi_dat.bi_blkoff = cpu_to_le64(key); + binfo->bi_dat.bi_level = 0; + memset(binfo->bi_dat.bi_pad, 0, sizeof(binfo->bi_dat.bi_pad)); + + return 0; +} + +static int nilfs_direct_assign(struct nilfs_bmap *bmap, + struct buffer_head **bh, + sector_t blocknr, + union nilfs_binfo *binfo) +{ + __u64 key; + __u64 ptr; + + key = nilfs_bmap_data_get_key(bmap, *bh); + if (unlikely(key > NILFS_DIRECT_KEY_MAX)) { + nilfs_crit(bmap->b_inode->i_sb, + "%s (ino=%llu): invalid key: %llu", + __func__, + bmap->b_inode->i_ino, (unsigned long long)key); + return -EINVAL; + } + ptr = nilfs_direct_get_ptr(bmap, key); + if (unlikely(ptr == NILFS_BMAP_INVALID_PTR)) { + nilfs_crit(bmap->b_inode->i_sb, + "%s (ino=%llu): invalid pointer: %llu", + __func__, + bmap->b_inode->i_ino, (unsigned long long)ptr); + return -EINVAL; + } + + return NILFS_BMAP_USE_VBN(bmap) ? + nilfs_direct_assign_v(bmap, key, ptr, bh, blocknr, binfo) : + nilfs_direct_assign_p(bmap, key, ptr, bh, blocknr, binfo); +} + +static const struct nilfs_bmap_operations nilfs_direct_ops = { + .bop_lookup = nilfs_direct_lookup, + .bop_lookup_contig = nilfs_direct_lookup_contig, + .bop_insert = nilfs_direct_insert, + .bop_delete = nilfs_direct_delete, + .bop_clear = NULL, + + .bop_propagate = nilfs_direct_propagate, + + .bop_lookup_dirty_buffers = NULL, + + .bop_assign = nilfs_direct_assign, + .bop_mark = NULL, + + .bop_seek_key = nilfs_direct_seek_key, + .bop_last_key = nilfs_direct_last_key, + + .bop_check_insert = nilfs_direct_check_insert, + .bop_check_delete = NULL, + .bop_gather_data = nilfs_direct_gather_data, +}; + + +int nilfs_direct_init(struct nilfs_bmap *bmap) +{ + bmap->b_ops = &nilfs_direct_ops; + return 0; +} diff --git a/fs/nilfs2/direct.h b/fs/nilfs2/direct.h new file mode 100644 index 000000000..b7ca89626 --- /dev/null +++ b/fs/nilfs2/direct.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * NILFS direct block pointer. + * + * Copyright (C) 2006-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Koji Sato. + */ + +#ifndef _NILFS_DIRECT_H +#define _NILFS_DIRECT_H + +#include +#include +#include "bmap.h" + + +#define NILFS_DIRECT_NBLOCKS (NILFS_BMAP_SIZE / sizeof(__le64) - 1) +#define NILFS_DIRECT_KEY_MIN 0 +#define NILFS_DIRECT_KEY_MAX (NILFS_DIRECT_NBLOCKS - 1) + + +int nilfs_direct_init(struct nilfs_bmap *); +int nilfs_direct_delete_and_convert(struct nilfs_bmap *, __u64, __u64 *, + __u64 *, int); + + +#endif /* _NILFS_DIRECT_H */ diff --git a/fs/nilfs2/export.h b/fs/nilfs2/export.h new file mode 100644 index 000000000..d29fd837c --- /dev/null +++ b/fs/nilfs2/export.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef NILFS_EXPORT_H +#define NILFS_EXPORT_H + +#include + +extern const struct export_operations nilfs_export_ops; + +/** + * struct nilfs_fid - NILFS file id type + * @cno: checkpoint number + * @ino: inode number + * @gen: file generation (version) for NFS + * @parent_gen: parent generation (version) for NFS + * @parent_ino: parent inode number + */ +struct nilfs_fid { + u64 cno; + u64 ino; + u32 gen; + + u32 parent_gen; + u64 parent_ino; +} __packed; + +#endif diff --git a/fs/nilfs2/file.c b/fs/nilfs2/file.c new file mode 100644 index 000000000..f93b68c48 --- /dev/null +++ b/fs/nilfs2/file.c @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * NILFS regular file handling primitives including fsync(). + * + * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Amagai Yoshiji and Ryusuke Konishi. + */ + +#include +#include +#include +#include +#include "nilfs.h" +#include "segment.h" + +int nilfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync) +{ + /* + * Called from fsync() system call + * This is the only entry point that can catch write and synch + * timing for both data blocks and intermediate blocks. + * + * This function should be implemented when the writeback function + * will be implemented. + */ + struct the_nilfs *nilfs; + struct inode *inode = file->f_mapping->host; + int err = 0; + + if (nilfs_inode_dirty(inode)) { + if (datasync) + err = nilfs_construct_dsync_segment(inode->i_sb, inode, + start, end); + else + err = nilfs_construct_segment(inode->i_sb); + } + + nilfs = inode->i_sb->s_fs_info; + if (!err) + err = nilfs_flush_device(nilfs); + + return err; +} + +static vm_fault_t nilfs_page_mkwrite(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct folio *folio = page_folio(vmf->page); + struct inode *inode = file_inode(vma->vm_file); + struct nilfs_transaction_info ti; + struct buffer_head *bh, *head; + int ret = 0; + + if (unlikely(nilfs_near_disk_full(inode->i_sb->s_fs_info))) + return VM_FAULT_SIGBUS; /* -ENOSPC */ + + sb_start_pagefault(inode->i_sb); + folio_lock(folio); + if (folio->mapping != inode->i_mapping || + folio_pos(folio) >= i_size_read(inode) || + !folio_test_uptodate(folio)) { + folio_unlock(folio); + ret = -EFAULT; /* make the VM retry the fault */ + goto out; + } + + /* + * check to see if the folio is mapped already (no holes) + */ + if (folio_test_mappedtodisk(folio)) + goto mapped; + + head = folio_buffers(folio); + if (head) { + int fully_mapped = 1; + + bh = head; + do { + if (!buffer_mapped(bh)) { + fully_mapped = 0; + break; + } + } while (bh = bh->b_this_page, bh != head); + + if (fully_mapped) { + folio_set_mappedtodisk(folio); + goto mapped; + } + } + folio_unlock(folio); + + /* + * fill hole blocks + */ + ret = nilfs_transaction_begin(inode->i_sb, &ti, 1); + /* never returns -ENOMEM, but may return -ENOSPC */ + if (unlikely(ret)) + goto out; + + file_update_time(vma->vm_file); + ret = block_page_mkwrite(vma, vmf, nilfs_get_block); + if (ret) { + nilfs_transaction_abort(inode->i_sb); + goto out; + } + nilfs_set_file_dirty(inode, 1 << (PAGE_SHIFT - inode->i_blkbits)); + nilfs_transaction_commit(inode->i_sb); + + mapped: + /* + * Since checksumming including data blocks is performed to determine + * the validity of the log to be written and used for recovery, it is + * necessary to wait for writeback to finish here, regardless of the + * stable write requirement of the backing device. + */ + folio_wait_writeback(folio); + out: + sb_end_pagefault(inode->i_sb); + return vmf_fs_error(ret); +} + +static const struct vm_operations_struct nilfs_file_vm_ops = { + .fault = filemap_fault, + .map_pages = filemap_map_pages, + .page_mkwrite = nilfs_page_mkwrite, +}; + +static int nilfs_file_mmap_prepare(struct vm_area_desc *desc) +{ + file_accessed(desc->file); + desc->vm_ops = &nilfs_file_vm_ops; + return 0; +} + +/* + * We have mostly NULL's here: the current defaults are ok for + * the nilfs filesystem. + */ +const struct file_operations nilfs_file_operations = { + .llseek = generic_file_llseek, + .read_iter = generic_file_read_iter, + .write_iter = generic_file_write_iter, + .unlocked_ioctl = nilfs_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = nilfs_compat_ioctl, +#endif /* CONFIG_COMPAT */ + .mmap_prepare = nilfs_file_mmap_prepare, + .open = generic_file_open, + /* .release = nilfs_release_file, */ + .fsync = nilfs_sync_file, + .splice_read = filemap_splice_read, + .splice_write = iter_file_splice_write, + .setlease = generic_setlease, +}; + +const struct inode_operations nilfs_file_inode_operations = { + .setattr = nilfs_setattr, + .permission = nilfs_permission, + .fiemap = nilfs_fiemap, + .fileattr_get = nilfs_fileattr_get, + .fileattr_set = nilfs_fileattr_set, +}; + +/* end of file */ diff --git a/fs/nilfs2/gcinode.c b/fs/nilfs2/gcinode.c new file mode 100644 index 000000000..85379ac23 --- /dev/null +++ b/fs/nilfs2/gcinode.c @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Dummy inodes to buffer blocks for garbage collection + * + * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Seiji Kihara, Amagai Yoshiji, and Ryusuke Konishi. + * Revised by Ryusuke Konishi. + * + */ +/* + * This file adds the cache of on-disk blocks to be moved in garbage + * collection. The disk blocks are held with dummy inodes (called + * gcinodes), and this file provides lookup function of the dummy + * inodes and their buffer read function. + * + * Buffers and pages held by the dummy inodes will be released each + * time after they are copied to a new log. Dirty blocks made on the + * current generation and the blocks to be moved by GC never overlap + * because the dirty blocks make a new generation; they rather must be + * written individually. + */ + +#include +#include +#include +#include +#include +#include "nilfs.h" +#include "btree.h" +#include "btnode.h" +#include "page.h" +#include "mdt.h" +#include "dat.h" +#include "ifile.h" + +/* + * nilfs_gccache_submit_read_data() - add data buffer and submit read request + * @inode - gc inode + * @blkoff - dummy offset treated as the key for the page cache + * @pbn - physical block number of the block + * @vbn - virtual block number of the block, 0 for non-virtual block + * @out_bh - indirect pointer to a buffer_head struct to receive the results + * + * Description: nilfs_gccache_submit_read_data() registers the data buffer + * specified by @pbn to the GC pagecache with the key @blkoff. + * This function sets @vbn (@pbn if @vbn is zero) in b_blocknr of the buffer. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOENT - The block specified with @pbn does not exist. + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_gccache_submit_read_data(struct inode *inode, sector_t blkoff, + sector_t pbn, __u64 vbn, + struct buffer_head **out_bh) +{ + struct buffer_head *bh; + int err; + + bh = nilfs_grab_buffer(inode, inode->i_mapping, blkoff, 0); + if (unlikely(!bh)) + return -ENOMEM; + + if (buffer_uptodate(bh)) + goto out; + + if (pbn == 0) { + struct the_nilfs *nilfs = inode->i_sb->s_fs_info; + + err = nilfs_dat_translate(nilfs->ns_dat, vbn, &pbn); + if (unlikely(err)) /* -EIO, -ENOMEM, -ENOENT */ + goto failed; + } + + lock_buffer(bh); + if (buffer_uptodate(bh)) { + unlock_buffer(bh); + goto out; + } + + if (!buffer_mapped(bh)) + set_buffer_mapped(bh); + bh->b_blocknr = pbn; + bh_submit(bh, REQ_OP_READ, bh_end_read); + if (vbn) + bh->b_blocknr = vbn; + out: + err = 0; + *out_bh = bh; + + failed: + folio_unlock(bh->b_folio); + folio_put(bh->b_folio); + if (unlikely(err)) + brelse(bh); + return err; +} + +/* + * nilfs_gccache_submit_read_node() - add node buffer and submit read request + * @inode - gc inode + * @pbn - physical block number for the block + * @vbn - virtual block number for the block + * @out_bh - indirect pointer to a buffer_head struct to receive the results + * + * Description: nilfs_gccache_submit_read_node() registers the node buffer + * specified by @vbn to the GC pagecache. @pbn can be supplied by the + * caller to avoid translation of the disk block address. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOENT - Invalid virtual block address. + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_gccache_submit_read_node(struct inode *inode, sector_t pbn, + __u64 vbn, struct buffer_head **out_bh) +{ + struct inode *btnc_inode = NILFS_I(inode)->i_assoc_inode; + int ret; + + ret = nilfs_btnode_submit_block(btnc_inode->i_mapping, vbn ? : pbn, pbn, + REQ_OP_READ, out_bh, &pbn); + if (ret == -EEXIST) /* internal code (cache hit) */ + ret = 0; + return ret; +} + +int nilfs_gccache_wait_and_mark_dirty(struct buffer_head *bh) +{ + wait_on_buffer(bh); + if (!buffer_uptodate(bh)) { + struct inode *inode = bh->b_folio->mapping->host; + + nilfs_err(inode->i_sb, + "I/O error reading %s block for GC (ino=%llu, vblocknr=%llu)", + buffer_nilfs_node(bh) ? "node" : "data", + inode->i_ino, (unsigned long long)bh->b_blocknr); + return -EIO; + } + if (buffer_dirty(bh)) + return -EEXIST; + + if (buffer_nilfs_node(bh) && nilfs_btree_broken_node_block(bh)) { + clear_buffer_uptodate(bh); + return -EIO; + } + mark_buffer_dirty(bh); + return 0; +} + +int nilfs_init_gcinode(struct inode *inode) +{ + struct nilfs_inode_info *ii = NILFS_I(inode); + + inode->i_mode = S_IFREG; + mapping_set_gfp_mask(inode->i_mapping, GFP_NOFS); + inode->i_mapping->a_ops = &nilfs_buffer_cache_aops; + + ii->i_flags = 0; + nilfs_bmap_init_gc(ii->i_bmap); + + return nilfs_attach_btree_node_cache(inode); +} + +/** + * nilfs_remove_all_gcinodes() - remove all unprocessed gc inodes + * @nilfs: NILFS filesystem instance + */ +void nilfs_remove_all_gcinodes(struct the_nilfs *nilfs) +{ + struct list_head *head = &nilfs->ns_gc_inodes; + struct nilfs_inode_info *ii; + + while (!list_empty(head)) { + ii = list_first_entry(head, struct nilfs_inode_info, i_dirty); + list_del_init(&ii->i_dirty); + truncate_inode_pages(&ii->vfs_inode.i_data, 0); + nilfs_btnode_cache_clear(ii->i_assoc_inode->i_mapping); + iput(&ii->vfs_inode); + } +} diff --git a/fs/nilfs2/ifile.c b/fs/nilfs2/ifile.c new file mode 100644 index 000000000..c99e1e635 --- /dev/null +++ b/fs/nilfs2/ifile.c @@ -0,0 +1,215 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * NILFS inode file + * + * Copyright (C) 2006-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Amagai Yoshiji. + * Revised by Ryusuke Konishi. + * + */ + +#include +#include +#include "nilfs.h" +#include "mdt.h" +#include "alloc.h" +#include "ifile.h" +#include "cpfile.h" + +/** + * struct nilfs_ifile_info - on-memory private data of ifile + * @mi: on-memory private data of metadata file + * @palloc_cache: persistent object allocator cache of ifile + */ +struct nilfs_ifile_info { + struct nilfs_mdt_info mi; + struct nilfs_palloc_cache palloc_cache; +}; + +static inline struct nilfs_ifile_info *NILFS_IFILE_I(struct inode *ifile) +{ + return (struct nilfs_ifile_info *)NILFS_MDT(ifile); +} + +/** + * nilfs_ifile_create_inode - create a new disk inode + * @ifile: ifile inode + * @out_ino: pointer to a variable to store inode number + * @out_bh: buffer_head contains newly allocated disk inode + * + * nilfs_ifile_create_inode() allocates a new inode in the ifile metadata + * file and stores the inode number in the variable pointed to by @out_ino, + * as well as storing the ifile's buffer with the disk inode in the location + * pointed to by @out_bh. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOMEM - Insufficient memory available. + * * %-ENOSPC - No inode left. + */ +int nilfs_ifile_create_inode(struct inode *ifile, u64 *out_ino, + struct buffer_head **out_bh) +{ + struct nilfs_palloc_req req; + int ret; + + req.pr_entry_nr = NILFS_FIRST_INO(ifile->i_sb); + req.pr_entry_bh = NULL; + + ret = nilfs_palloc_prepare_alloc_entry(ifile, &req, false); + if (!ret) { + ret = nilfs_palloc_get_entry_block(ifile, req.pr_entry_nr, 1, + &req.pr_entry_bh); + if (ret < 0) + nilfs_palloc_abort_alloc_entry(ifile, &req); + } + if (ret < 0) { + brelse(req.pr_entry_bh); + return ret; + } + nilfs_palloc_commit_alloc_entry(ifile, &req); + mark_buffer_dirty(req.pr_entry_bh); + nilfs_mdt_mark_dirty(ifile); + *out_ino = req.pr_entry_nr; + *out_bh = req.pr_entry_bh; + return 0; +} + +/** + * nilfs_ifile_delete_inode - delete a disk inode + * @ifile: ifile inode + * @ino: inode number + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOENT - Inode number unallocated. + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_ifile_delete_inode(struct inode *ifile, u64 ino) +{ + struct nilfs_palloc_req req = { + .pr_entry_nr = ino, .pr_entry_bh = NULL + }; + struct nilfs_inode *raw_inode; + size_t offset; + int ret; + + ret = nilfs_palloc_prepare_free_entry(ifile, &req); + if (!ret) { + ret = nilfs_palloc_get_entry_block(ifile, req.pr_entry_nr, 0, + &req.pr_entry_bh); + if (ret < 0) + nilfs_palloc_abort_free_entry(ifile, &req); + } + if (ret < 0) { + brelse(req.pr_entry_bh); + return ret; + } + + offset = nilfs_palloc_entry_offset(ifile, req.pr_entry_nr, + req.pr_entry_bh); + raw_inode = kmap_local_folio(req.pr_entry_bh->b_folio, offset); + raw_inode->i_flags = 0; + kunmap_local(raw_inode); + + mark_buffer_dirty(req.pr_entry_bh); + brelse(req.pr_entry_bh); + + nilfs_palloc_commit_free_entry(ifile, &req); + + return 0; +} + +int nilfs_ifile_get_inode_block(struct inode *ifile, u64 ino, + struct buffer_head **out_bh) +{ + struct super_block *sb = ifile->i_sb; + int err; + + if (unlikely(!NILFS_VALID_INODE(sb, ino))) { + nilfs_error(sb, "bad inode number: %llu", ino); + return -EINVAL; + } + + err = nilfs_palloc_get_entry_block(ifile, ino, 0, out_bh); + if (unlikely(err)) + nilfs_warn(sb, "error %d reading inode: ino=%llu", err, ino); + return err; +} + +/** + * nilfs_ifile_count_free_inodes - calculate free inodes count + * @ifile: ifile inode + * @nmaxinodes: current maximum of available inodes count [out] + * @nfreeinodes: free inodes count [out] + * + * Return: 0 on success, or a negative error code on failure. + */ +int nilfs_ifile_count_free_inodes(struct inode *ifile, + u64 *nmaxinodes, u64 *nfreeinodes) +{ + u64 nused; + int err; + + *nmaxinodes = 0; + *nfreeinodes = 0; + + nused = atomic64_read(&NILFS_I(ifile)->i_root->inodes_count); + err = nilfs_palloc_count_max_entries(ifile, nused, nmaxinodes); + if (likely(!err)) + *nfreeinodes = *nmaxinodes - nused; + return err; +} + +/** + * nilfs_ifile_read - read or get ifile inode + * @sb: super block instance + * @root: root object + * @cno: number of checkpoint entry to read + * @inode_size: size of an inode + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EINVAL - Invalid checkpoint. + * * %-ENOMEM - Insufficient memory available. + * * %-EIO - I/O error (including metadata corruption). + */ +int nilfs_ifile_read(struct super_block *sb, struct nilfs_root *root, + __u64 cno, size_t inode_size) +{ + struct the_nilfs *nilfs; + struct inode *ifile; + int err; + + ifile = nilfs_iget_locked(sb, root, NILFS_IFILE_INO); + if (unlikely(!ifile)) + return -ENOMEM; + if (!(inode_state_read_once(ifile) & I_NEW)) + goto out; + + err = nilfs_mdt_init(ifile, NILFS_MDT_GFP, + sizeof(struct nilfs_ifile_info)); + if (err) + goto failed; + + err = nilfs_palloc_init_blockgroup(ifile, inode_size); + if (err) + goto failed; + + nilfs_palloc_setup_cache(ifile, &NILFS_IFILE_I(ifile)->palloc_cache); + + nilfs = sb->s_fs_info; + err = nilfs_cpfile_read_checkpoint(nilfs->ns_cpfile, cno, root, ifile); + if (err) + goto failed; + + unlock_new_inode(ifile); + out: + return 0; + failed: + iget_failed(ifile); + return err; +} diff --git a/fs/nilfs2/ifile.h b/fs/nilfs2/ifile.h new file mode 100644 index 000000000..d38a46f5a --- /dev/null +++ b/fs/nilfs2/ifile.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * NILFS inode file + * + * Copyright (C) 2006-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Amagai Yoshiji. + * Revised by Ryusuke Konishi. + * + */ + +#ifndef _NILFS_IFILE_H +#define _NILFS_IFILE_H + +#include +#include +#include "mdt.h" +#include "alloc.h" + + +static inline struct nilfs_inode * +nilfs_ifile_map_inode(struct inode *ifile, u64 ino, struct buffer_head *ibh) +{ + size_t __offset_in_folio = nilfs_palloc_entry_offset(ifile, ino, ibh); + + return kmap_local_folio(ibh->b_folio, __offset_in_folio); +} + +static inline void nilfs_ifile_unmap_inode(struct nilfs_inode *raw_inode) +{ + kunmap_local(raw_inode); +} + +int nilfs_ifile_create_inode(struct inode *ifile, u64 *out_ino, + struct buffer_head **out_bh); +int nilfs_ifile_delete_inode(struct inode *ifile, u64 ino); +int nilfs_ifile_get_inode_block(struct inode *ifile, u64 ino, + struct buffer_head **out_bh); + +int nilfs_ifile_count_free_inodes(struct inode *, u64 *, u64 *); + +int nilfs_ifile_read(struct super_block *sb, struct nilfs_root *root, + __u64 cno, size_t inode_size); + +#endif /* _NILFS_IFILE_H */ diff --git a/fs/nilfs2/inode.c b/fs/nilfs2/inode.c new file mode 100644 index 000000000..34e609606 --- /dev/null +++ b/fs/nilfs2/inode.c @@ -0,0 +1,1229 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * NILFS inode operations. + * + * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Ryusuke Konishi. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "nilfs.h" +#include "btnode.h" +#include "segment.h" +#include "page.h" +#include "mdt.h" +#include "cpfile.h" +#include "ifile.h" + +/** + * struct nilfs_iget_args - arguments used during comparison between inodes + * @ino: inode number + * @cno: checkpoint number + * @root: pointer on NILFS root object (mounted checkpoint) + * @type: inode type + */ +struct nilfs_iget_args { + u64 ino; + __u64 cno; + struct nilfs_root *root; + unsigned int type; +}; + +static int nilfs_iget_test(struct inode *inode, void *opaque); + +void nilfs_inode_add_blocks(struct inode *inode, int n) +{ + struct nilfs_root *root = NILFS_I(inode)->i_root; + + inode_add_bytes(inode, i_blocksize(inode) * n); + if (root) + atomic64_add(n, &root->blocks_count); +} + +void nilfs_inode_sub_blocks(struct inode *inode, int n) +{ + struct nilfs_root *root = NILFS_I(inode)->i_root; + + inode_sub_bytes(inode, i_blocksize(inode) * n); + if (root) + atomic64_sub(n, &root->blocks_count); +} + +/** + * nilfs_get_block() - get a file block on the filesystem (callback function) + * @inode: inode struct of the target file + * @blkoff: file block number + * @bh_result: buffer head to be mapped on + * @create: indicate whether allocating the block or not when it has not + * been allocated yet. + * + * This function does not issue actual read request of the specified data + * block. It is done by VFS. + * + * Return: 0 on success, or a negative error code on failure. + */ +int nilfs_get_block(struct inode *inode, sector_t blkoff, + struct buffer_head *bh_result, int create) +{ + struct nilfs_inode_info *ii = NILFS_I(inode); + struct the_nilfs *nilfs = inode->i_sb->s_fs_info; + __u64 blknum = 0; + int err = 0, ret; + unsigned int maxblocks = bh_result->b_size >> inode->i_blkbits; + + down_read(&NILFS_MDT(nilfs->ns_dat)->mi_sem); + ret = nilfs_bmap_lookup_contig(ii->i_bmap, blkoff, &blknum, maxblocks); + up_read(&NILFS_MDT(nilfs->ns_dat)->mi_sem); + if (ret >= 0) { /* found */ + map_bh(bh_result, inode->i_sb, blknum); + if (ret > 0) + bh_result->b_size = (ret << inode->i_blkbits); + goto out; + } + /* data block was not found */ + if (ret == -ENOENT && create) { + struct nilfs_transaction_info ti; + + bh_result->b_blocknr = 0; + err = nilfs_transaction_begin(inode->i_sb, &ti, 1); + if (unlikely(err)) + goto out; + err = nilfs_bmap_insert(ii->i_bmap, blkoff, + (unsigned long)bh_result); + if (unlikely(err != 0)) { + if (err == -EEXIST) { + /* + * The get_block() function could be called + * from multiple callers for an inode. + * However, the page having this block must + * be locked in this case. + */ + nilfs_warn(inode->i_sb, + "%s (ino=%llu): a race condition while inserting a data block at offset=%llu", + __func__, inode->i_ino, + (unsigned long long)blkoff); + err = -EAGAIN; + } + nilfs_transaction_abort(inode->i_sb); + goto out; + } + nilfs_mark_inode_dirty_sync(inode); + nilfs_transaction_commit(inode->i_sb); /* never fails */ + /* Error handling should be detailed */ + set_buffer_new(bh_result); + set_buffer_delay(bh_result); + map_bh(bh_result, inode->i_sb, 0); + /* Disk block number must be changed to proper value */ + + } else if (ret == -ENOENT) { + /* + * not found is not error (e.g. hole); must return without + * the mapped state flag. + */ + ; + } else { + err = ret; + } + + out: + return err; +} + +/** + * nilfs_read_folio() - implement read_folio() method of nilfs_aops {} + * address_space_operations. + * @file: file struct of the file to be read + * @folio: the folio to be read + * + * Return: 0 on success, or a negative error code on failure. + */ +static int nilfs_read_folio(struct file *file, struct folio *folio) +{ + return mpage_read_folio(folio, nilfs_get_block); +} + +static void nilfs_readahead(struct readahead_control *rac) +{ + mpage_readahead(rac, nilfs_get_block); +} + +static int nilfs_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct inode *inode = mapping->host; + int err = 0; + + if (sb_rdonly(inode->i_sb)) { + nilfs_clear_dirty_pages(mapping); + return -EROFS; + } + + if (wbc->sync_mode == WB_SYNC_ALL) + err = nilfs_construct_dsync_segment(inode->i_sb, inode, + wbc->range_start, + wbc->range_end); + return err; +} + +static bool nilfs_dirty_folio(struct address_space *mapping, + struct folio *folio) +{ + struct inode *inode = mapping->host; + struct buffer_head *head; + unsigned int nr_dirty = 0; + bool ret = filemap_dirty_folio(mapping, folio); + + /* + * The page may not be locked, eg if called from try_to_unmap_one() + */ + spin_lock(&mapping->i_private_lock); + head = folio_buffers(folio); + if (head) { + struct buffer_head *bh = head; + + do { + /* Do not mark hole blocks dirty */ + if (buffer_dirty(bh) || !buffer_mapped(bh)) + continue; + + set_buffer_dirty(bh); + nr_dirty++; + } while (bh = bh->b_this_page, bh != head); + } else if (ret) { + nr_dirty = 1 << (folio_shift(folio) - inode->i_blkbits); + } + spin_unlock(&mapping->i_private_lock); + + if (nr_dirty) + nilfs_set_file_dirty(inode, nr_dirty); + return ret; +} + +void nilfs_write_failed(struct address_space *mapping, loff_t to) +{ + struct inode *inode = mapping->host; + + if (to > inode->i_size) { + truncate_pagecache(inode, inode->i_size); + nilfs_truncate(inode); + } +} + +static int nilfs_write_begin(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, + struct folio **foliop, void **fsdata) + +{ + struct inode *inode = mapping->host; + int err = nilfs_transaction_begin(inode->i_sb, NULL, 1); + + if (unlikely(err)) + return err; + + err = block_write_begin(mapping, pos, len, foliop, nilfs_get_block); + if (unlikely(err)) { + nilfs_write_failed(mapping, pos + len); + nilfs_transaction_abort(inode->i_sb); + } + return err; +} + +static int nilfs_write_end(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct folio *folio, void *fsdata) +{ + struct inode *inode = mapping->host; + unsigned int start = pos & (PAGE_SIZE - 1); + unsigned int nr_dirty; + int err; + + nr_dirty = nilfs_page_count_clean_buffers(folio, start, + start + copied); + copied = generic_write_end(iocb, mapping, pos, len, copied, folio, + fsdata); + nilfs_set_file_dirty(inode, nr_dirty); + err = nilfs_transaction_commit(inode->i_sb); + return err ? : copied; +} + +static ssize_t +nilfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) +{ + struct inode *inode = file_inode(iocb->ki_filp); + + if (iov_iter_rw(iter) == WRITE) + return 0; + + /* Needs synchronization with the cleaner */ + return blockdev_direct_IO(iocb, inode, iter, nilfs_get_block); +} + +const struct address_space_operations nilfs_aops = { + .read_folio = nilfs_read_folio, + .writepages = nilfs_writepages, + .dirty_folio = nilfs_dirty_folio, + .readahead = nilfs_readahead, + .write_begin = nilfs_write_begin, + .write_end = nilfs_write_end, + .invalidate_folio = block_invalidate_folio, + .direct_IO = nilfs_direct_IO, + .migrate_folio = buffer_migrate_folio_norefs, + .is_partially_uptodate = block_is_partially_uptodate, +}; + +const struct address_space_operations nilfs_buffer_cache_aops = { + .invalidate_folio = block_invalidate_folio, +}; + +static int nilfs_insert_inode_locked(struct inode *inode, + struct nilfs_root *root, u64 ino) +{ + struct nilfs_iget_args args = { + .ino = ino, .root = root, .cno = 0, .type = NILFS_I_TYPE_NORMAL + }; + + return insert_inode_locked4(inode, ino, nilfs_iget_test, &args); +} + +struct inode *nilfs_new_inode(struct inode *dir, umode_t mode) +{ + struct super_block *sb = dir->i_sb; + struct inode *inode; + struct nilfs_inode_info *ii; + struct nilfs_root *root; + struct buffer_head *bh; + int err = -ENOMEM; + u64 ino; + + inode = new_inode(sb); + if (unlikely(!inode)) + goto failed; + + mapping_set_gfp_mask(inode->i_mapping, + mapping_gfp_constraint(inode->i_mapping, ~__GFP_FS)); + + root = NILFS_I(dir)->i_root; + ii = NILFS_I(inode); + ii->i_state = BIT(NILFS_I_NEW); + ii->i_type = NILFS_I_TYPE_NORMAL; + ii->i_root = root; + + err = nilfs_ifile_create_inode(root->ifile, &ino, &bh); + if (unlikely(err)) + goto failed_ifile_create_inode; + /* reference count of i_bh inherits from nilfs_mdt_read_block() */ + ii->i_bh = bh; + + atomic64_inc(&root->inodes_count); + inode_init_owner(&nop_mnt_idmap, inode, dir, mode); + inode->i_ino = ino; + simple_inode_init_ts(inode); + + if (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode)) { + err = nilfs_bmap_read(ii->i_bmap, NULL); + if (err < 0) + goto failed_after_creation; + + set_bit(NILFS_I_BMAP, &ii->i_state); + /* No lock is needed; iget() ensures it. */ + } + + ii->i_flags = nilfs_mask_flags( + mode, NILFS_I(dir)->i_flags & NILFS_FL_INHERITED); + + /* ii->i_file_acl = 0; */ + /* ii->i_dir_acl = 0; */ + ii->i_dir_start_lookup = 0; + nilfs_set_inode_flags(inode); + inode->i_generation = get_random_u32(); + if (nilfs_insert_inode_locked(inode, root, ino) < 0) { + err = -EIO; + goto failed_after_creation; + } + + err = nilfs_init_acl(inode, dir); + if (unlikely(err)) + /* + * Never occur. When supporting nilfs_init_acl(), + * proper cancellation of above jobs should be considered. + */ + goto failed_after_creation; + + return inode; + + failed_after_creation: + clear_nlink(inode); + if (inode_state_read_once(inode) & I_NEW) + unlock_new_inode(inode); + iput(inode); /* + * raw_inode will be deleted through + * nilfs_evict_inode(). + */ + goto failed; + + failed_ifile_create_inode: + make_bad_inode(inode); + iput(inode); + failed: + return ERR_PTR(err); +} + +void nilfs_set_inode_flags(struct inode *inode) +{ + unsigned int flags = NILFS_I(inode)->i_flags; + unsigned int new_fl = 0; + + if (flags & FS_SYNC_FL) + new_fl |= S_SYNC; + if (flags & FS_APPEND_FL) + new_fl |= S_APPEND; + if (flags & FS_IMMUTABLE_FL) + new_fl |= S_IMMUTABLE; + if (flags & FS_NOATIME_FL) + new_fl |= S_NOATIME; + if (flags & FS_DIRSYNC_FL) + new_fl |= S_DIRSYNC; + inode_set_flags(inode, new_fl, S_SYNC | S_APPEND | S_IMMUTABLE | + S_NOATIME | S_DIRSYNC); +} + +int nilfs_read_inode_common(struct inode *inode, + struct nilfs_inode *raw_inode) +{ + struct nilfs_inode_info *ii = NILFS_I(inode); + int err; + + inode->i_mode = le16_to_cpu(raw_inode->i_mode); + i_uid_write(inode, le32_to_cpu(raw_inode->i_uid)); + i_gid_write(inode, le32_to_cpu(raw_inode->i_gid)); + set_nlink(inode, le16_to_cpu(raw_inode->i_links_count)); + inode->i_size = le64_to_cpu(raw_inode->i_size); + inode_set_atime(inode, le64_to_cpu(raw_inode->i_mtime), + le32_to_cpu(raw_inode->i_mtime_nsec)); + inode_set_ctime(inode, le64_to_cpu(raw_inode->i_ctime), + le32_to_cpu(raw_inode->i_ctime_nsec)); + inode_set_mtime(inode, le64_to_cpu(raw_inode->i_mtime), + le32_to_cpu(raw_inode->i_mtime_nsec)); + if (nilfs_is_metadata_file_inode(inode) && !S_ISREG(inode->i_mode)) + return -EIO; /* this inode is for metadata and corrupted */ + if (inode->i_nlink == 0) + return -ESTALE; /* this inode is deleted */ + + inode->i_blocks = le64_to_cpu(raw_inode->i_blocks); + ii->i_flags = le32_to_cpu(raw_inode->i_flags); +#if 0 + ii->i_file_acl = le32_to_cpu(raw_inode->i_file_acl); + ii->i_dir_acl = S_ISREG(inode->i_mode) ? + 0 : le32_to_cpu(raw_inode->i_dir_acl); +#endif + ii->i_dir_start_lookup = 0; + inode->i_generation = le32_to_cpu(raw_inode->i_generation); + + if (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) || + S_ISLNK(inode->i_mode)) { + err = nilfs_bmap_read(ii->i_bmap, raw_inode); + if (err < 0) + return err; + set_bit(NILFS_I_BMAP, &ii->i_state); + /* No lock is needed; iget() ensures it. */ + } + return 0; +} + +static int __nilfs_read_inode(struct super_block *sb, + struct nilfs_root *root, u64 ino, + struct inode *inode) +{ + struct the_nilfs *nilfs = sb->s_fs_info; + struct buffer_head *bh; + struct nilfs_inode *raw_inode; + int err; + + down_read(&NILFS_MDT(nilfs->ns_dat)->mi_sem); + err = nilfs_ifile_get_inode_block(root->ifile, ino, &bh); + if (unlikely(err)) + goto bad_inode; + + raw_inode = nilfs_ifile_map_inode(root->ifile, ino, bh); + + err = nilfs_read_inode_common(inode, raw_inode); + if (err) + goto failed_unmap; + + if (S_ISREG(inode->i_mode)) { + inode->i_op = &nilfs_file_inode_operations; + inode->i_fop = &nilfs_file_operations; + inode->i_mapping->a_ops = &nilfs_aops; + } else if (S_ISDIR(inode->i_mode)) { + inode->i_op = &nilfs_dir_inode_operations; + inode->i_fop = &nilfs_dir_operations; + inode->i_mapping->a_ops = &nilfs_aops; + } else if (S_ISLNK(inode->i_mode)) { + inode->i_op = &nilfs_symlink_inode_operations; + inode_nohighmem(inode); + inode->i_mapping->a_ops = &nilfs_aops; + } else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) || + S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) { + inode->i_op = &nilfs_special_inode_operations; + init_special_inode( + inode, inode->i_mode, + huge_decode_dev(le64_to_cpu(raw_inode->i_device_code))); + } else { + nilfs_error(sb, + "invalid file type bits in mode 0%o for inode %llu", + inode->i_mode, ino); + err = -EIO; + goto failed_unmap; + } + nilfs_ifile_unmap_inode(raw_inode); + brelse(bh); + up_read(&NILFS_MDT(nilfs->ns_dat)->mi_sem); + nilfs_set_inode_flags(inode); + mapping_set_gfp_mask(inode->i_mapping, + mapping_gfp_constraint(inode->i_mapping, ~__GFP_FS)); + return 0; + + failed_unmap: + nilfs_ifile_unmap_inode(raw_inode); + brelse(bh); + + bad_inode: + up_read(&NILFS_MDT(nilfs->ns_dat)->mi_sem); + return err; +} + +static int nilfs_iget_test(struct inode *inode, void *opaque) +{ + struct nilfs_iget_args *args = opaque; + struct nilfs_inode_info *ii; + + if (args->ino != inode->i_ino || args->root != NILFS_I(inode)->i_root) + return 0; + + ii = NILFS_I(inode); + if (ii->i_type != args->type) + return 0; + + return !(args->type & NILFS_I_TYPE_GC) || args->cno == ii->i_cno; +} + +static int nilfs_iget_set(struct inode *inode, void *opaque) +{ + struct nilfs_iget_args *args = opaque; + + inode->i_ino = args->ino; + NILFS_I(inode)->i_cno = args->cno; + NILFS_I(inode)->i_root = args->root; + NILFS_I(inode)->i_type = args->type; + if (args->root && args->ino == NILFS_ROOT_INO) + nilfs_get_root(args->root); + return 0; +} + +struct inode *nilfs_ilookup(struct super_block *sb, struct nilfs_root *root, + u64 ino) +{ + struct nilfs_iget_args args = { + .ino = ino, .root = root, .cno = 0, .type = NILFS_I_TYPE_NORMAL + }; + + return ilookup5(sb, ino, nilfs_iget_test, &args); +} + +struct inode *nilfs_iget_locked(struct super_block *sb, + struct nilfs_root *root, u64 ino) +{ + struct nilfs_iget_args args = { + .ino = ino, .root = root, .cno = 0, .type = NILFS_I_TYPE_NORMAL + }; + + return iget5_locked(sb, ino, nilfs_iget_test, nilfs_iget_set, &args); +} + +struct inode *nilfs_iget(struct super_block *sb, struct nilfs_root *root, + u64 ino) +{ + struct inode *inode; + int err; + + inode = nilfs_iget_locked(sb, root, ino); + if (unlikely(!inode)) + return ERR_PTR(-ENOMEM); + + if (!(inode_state_read_once(inode) & I_NEW)) { + if (!inode->i_nlink) { + iput(inode); + return ERR_PTR(-ESTALE); + } + return inode; + } + + err = __nilfs_read_inode(sb, root, ino, inode); + if (unlikely(err)) { + iget_failed(inode); + return ERR_PTR(err); + } + unlock_new_inode(inode); + return inode; +} + +struct inode *nilfs_iget_for_gc(struct super_block *sb, u64 ino, __u64 cno) +{ + struct nilfs_iget_args args = { + .ino = ino, .root = NULL, .cno = cno, .type = NILFS_I_TYPE_GC + }; + struct inode *inode; + int err; + + inode = iget5_locked(sb, ino, nilfs_iget_test, nilfs_iget_set, &args); + if (unlikely(!inode)) + return ERR_PTR(-ENOMEM); + if (!(inode_state_read_once(inode) & I_NEW)) + return inode; + + err = nilfs_init_gcinode(inode); + if (unlikely(err)) { + iget_failed(inode); + return ERR_PTR(err); + } + unlock_new_inode(inode); + return inode; +} + +/** + * nilfs_attach_btree_node_cache - attach a B-tree node cache to the inode + * @inode: inode object + * + * nilfs_attach_btree_node_cache() attaches a B-tree node cache to @inode, + * or does nothing if the inode already has it. This function allocates + * an additional inode to maintain page cache of B-tree nodes one-on-one. + * + * Return: 0 on success, or %-ENOMEM if memory is insufficient. + */ +int nilfs_attach_btree_node_cache(struct inode *inode) +{ + struct nilfs_inode_info *ii = NILFS_I(inode); + struct inode *btnc_inode; + struct nilfs_iget_args args; + + if (ii->i_assoc_inode) + return 0; + + args.ino = inode->i_ino; + args.root = ii->i_root; + args.cno = ii->i_cno; + args.type = ii->i_type | NILFS_I_TYPE_BTNC; + + btnc_inode = iget5_locked(inode->i_sb, inode->i_ino, nilfs_iget_test, + nilfs_iget_set, &args); + if (unlikely(!btnc_inode)) + return -ENOMEM; + if (inode_state_read_once(btnc_inode) & I_NEW) { + nilfs_init_btnc_inode(btnc_inode); + unlock_new_inode(btnc_inode); + } + NILFS_I(btnc_inode)->i_assoc_inode = inode; + NILFS_I(btnc_inode)->i_bmap = ii->i_bmap; + ii->i_assoc_inode = btnc_inode; + + return 0; +} + +/** + * nilfs_detach_btree_node_cache - detach the B-tree node cache from the inode + * @inode: inode object + * + * nilfs_detach_btree_node_cache() detaches the B-tree node cache and its + * holder inode bound to @inode, or does nothing if @inode doesn't have it. + */ +void nilfs_detach_btree_node_cache(struct inode *inode) +{ + struct nilfs_inode_info *ii = NILFS_I(inode); + struct inode *btnc_inode = ii->i_assoc_inode; + + if (btnc_inode) { + NILFS_I(btnc_inode)->i_assoc_inode = NULL; + ii->i_assoc_inode = NULL; + iput(btnc_inode); + } +} + +/** + * nilfs_iget_for_shadow - obtain inode for shadow mapping + * @inode: inode object that uses shadow mapping + * + * nilfs_iget_for_shadow() allocates a pair of inodes that holds page + * caches for shadow mapping. The page cache for data pages is set up + * in one inode and the one for b-tree node pages is set up in the + * other inode, which is attached to the former inode. + * + * Return: a pointer to the inode for data pages on success, or %-ENOMEM + * if memory is insufficient. + */ +struct inode *nilfs_iget_for_shadow(struct inode *inode) +{ + struct nilfs_iget_args args = { + .ino = inode->i_ino, .root = NULL, .cno = 0, + .type = NILFS_I_TYPE_SHADOW + }; + struct inode *s_inode; + int err; + + s_inode = iget5_locked(inode->i_sb, inode->i_ino, nilfs_iget_test, + nilfs_iget_set, &args); + if (unlikely(!s_inode)) + return ERR_PTR(-ENOMEM); + if (!(inode_state_read_once(s_inode) & I_NEW)) + return inode; + + NILFS_I(s_inode)->i_flags = 0; + memset(NILFS_I(s_inode)->i_bmap, 0, sizeof(struct nilfs_bmap)); + mapping_set_gfp_mask(s_inode->i_mapping, GFP_NOFS); + s_inode->i_mapping->a_ops = &nilfs_buffer_cache_aops; + + err = nilfs_attach_btree_node_cache(s_inode); + if (unlikely(err)) { + iget_failed(s_inode); + return ERR_PTR(err); + } + unlock_new_inode(s_inode); + return s_inode; +} + +/** + * nilfs_write_inode_common - export common inode information to on-disk inode + * @inode: inode object + * @raw_inode: on-disk inode + * + * This function writes standard information from the on-memory inode @inode + * to @raw_inode on ifile, cpfile or a super root block. Since inode bmap + * data is not exported, nilfs_bmap_write() must be called separately during + * log writing. + */ +void nilfs_write_inode_common(struct inode *inode, + struct nilfs_inode *raw_inode) +{ + struct nilfs_inode_info *ii = NILFS_I(inode); + + raw_inode->i_mode = cpu_to_le16(inode->i_mode); + raw_inode->i_uid = cpu_to_le32(i_uid_read(inode)); + raw_inode->i_gid = cpu_to_le32(i_gid_read(inode)); + raw_inode->i_links_count = cpu_to_le16(inode->i_nlink); + raw_inode->i_size = cpu_to_le64(inode->i_size); + raw_inode->i_ctime = cpu_to_le64(inode_get_ctime_sec(inode)); + raw_inode->i_mtime = cpu_to_le64(inode_get_mtime_sec(inode)); + raw_inode->i_ctime_nsec = cpu_to_le32(inode_get_ctime_nsec(inode)); + raw_inode->i_mtime_nsec = cpu_to_le32(inode_get_mtime_nsec(inode)); + raw_inode->i_blocks = cpu_to_le64(inode->i_blocks); + + raw_inode->i_flags = cpu_to_le32(ii->i_flags); + raw_inode->i_generation = cpu_to_le32(inode->i_generation); + + /* + * When extending inode, nilfs->ns_inode_size should be checked + * for substitutions of appended fields. + */ +} + +void nilfs_update_inode(struct inode *inode, struct buffer_head *ibh, int flags) +{ + u64 ino = inode->i_ino; + struct nilfs_inode_info *ii = NILFS_I(inode); + struct inode *ifile = ii->i_root->ifile; + struct nilfs_inode *raw_inode; + + raw_inode = nilfs_ifile_map_inode(ifile, ino, ibh); + + if (test_and_clear_bit(NILFS_I_NEW, &ii->i_state)) + memset(raw_inode, 0, NILFS_MDT(ifile)->mi_entry_size); + if (flags & I_DIRTY_DATASYNC) + set_bit(NILFS_I_INODE_SYNC, &ii->i_state); + + nilfs_write_inode_common(inode, raw_inode); + + if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) + raw_inode->i_device_code = + cpu_to_le64(huge_encode_dev(inode->i_rdev)); + + nilfs_ifile_unmap_inode(raw_inode); +} + +#define NILFS_MAX_TRUNCATE_BLOCKS 16384 /* 64MB for 4KB block */ + +static void nilfs_truncate_bmap(struct nilfs_inode_info *ii, + unsigned long from) +{ + __u64 b; + int ret; + + if (!test_bit(NILFS_I_BMAP, &ii->i_state)) + return; +repeat: + ret = nilfs_bmap_last_key(ii->i_bmap, &b); + if (ret == -ENOENT) + return; + else if (ret < 0) + goto failed; + + if (b < from) + return; + + b -= min_t(__u64, NILFS_MAX_TRUNCATE_BLOCKS, b - from); + ret = nilfs_bmap_truncate(ii->i_bmap, b); + nilfs_relax_pressure_in_lock(ii->vfs_inode.i_sb); + if (!ret || (ret == -ENOMEM && + nilfs_bmap_truncate(ii->i_bmap, b) == 0)) + goto repeat; + +failed: + nilfs_warn(ii->vfs_inode.i_sb, "error %d truncating bmap (ino=%llu)", + ret, ii->vfs_inode.i_ino); +} + +void nilfs_truncate(struct inode *inode) +{ + unsigned long blkoff; + unsigned int blocksize; + struct nilfs_transaction_info ti; + struct super_block *sb = inode->i_sb; + struct nilfs_inode_info *ii = NILFS_I(inode); + + if (!test_bit(NILFS_I_BMAP, &ii->i_state)) + return; + if (IS_APPEND(inode) || IS_IMMUTABLE(inode)) + return; + + blocksize = sb->s_blocksize; + blkoff = (inode->i_size + blocksize - 1) >> sb->s_blocksize_bits; + nilfs_transaction_begin(sb, &ti, 0); /* never fails */ + + block_truncate_page(inode->i_mapping, inode->i_size, nilfs_get_block); + + nilfs_truncate_bmap(ii, blkoff); + + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + if (IS_SYNC(inode)) + nilfs_set_transaction_flag(NILFS_TI_SYNC); + + nilfs_mark_inode_dirty(inode); + nilfs_set_file_dirty(inode, 0); + nilfs_transaction_commit(sb); + /* + * May construct a logical segment and may fail in sync mode. + * But truncate has no return value. + */ +} + +static void nilfs_clear_inode(struct inode *inode) +{ + struct nilfs_inode_info *ii = NILFS_I(inode); + + /* + * Free resources allocated in nilfs_read_inode(), here. + */ + BUG_ON(!list_empty(&ii->i_dirty)); + brelse(ii->i_bh); + ii->i_bh = NULL; + + if (nilfs_is_metadata_file_inode(inode)) + nilfs_mdt_clear(inode); + + if (test_bit(NILFS_I_BMAP, &ii->i_state)) + nilfs_bmap_clear(ii->i_bmap); + + if (!(ii->i_type & NILFS_I_TYPE_BTNC)) + nilfs_detach_btree_node_cache(inode); + + if (ii->i_root && inode->i_ino == NILFS_ROOT_INO) + nilfs_put_root(ii->i_root); +} + +void nilfs_evict_inode(struct inode *inode) +{ + struct nilfs_transaction_info ti; + struct super_block *sb = inode->i_sb; + struct nilfs_inode_info *ii = NILFS_I(inode); + struct the_nilfs *nilfs; + int ret; + + if (inode->i_nlink || !ii->i_root || unlikely(is_bad_inode(inode))) { + truncate_inode_pages_final(&inode->i_data); + clear_inode(inode); + nilfs_clear_inode(inode); + return; + } + nilfs_transaction_begin(sb, &ti, 0); /* never fails */ + + truncate_inode_pages_final(&inode->i_data); + + nilfs = sb->s_fs_info; + if (unlikely(sb_rdonly(sb) || !nilfs->ns_writer)) { + /* + * If this inode is about to be disposed after the file system + * has been degraded to read-only due to file system corruption + * or after the writer has been detached, do not make any + * changes that cause writes, just clear it. + * Do this check after read-locking ns_segctor_sem by + * nilfs_transaction_begin() in order to avoid a race with + * the writer detach operation. + */ + clear_inode(inode); + nilfs_clear_inode(inode); + nilfs_transaction_abort(sb); + return; + } + + /* TODO: some of the following operations may fail. */ + nilfs_truncate_bmap(ii, 0); + nilfs_mark_inode_dirty(inode); + clear_inode(inode); + + ret = nilfs_ifile_delete_inode(ii->i_root->ifile, inode->i_ino); + if (!ret) + atomic64_dec(&ii->i_root->inodes_count); + + nilfs_clear_inode(inode); + + if (IS_SYNC(inode)) + nilfs_set_transaction_flag(NILFS_TI_SYNC); + nilfs_transaction_commit(sb); + /* + * May construct a logical segment and may fail in sync mode. + * But delete_inode has no return value. + */ +} + +int nilfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *iattr) +{ + struct nilfs_transaction_info ti; + struct inode *inode = d_inode(dentry); + struct super_block *sb = inode->i_sb; + int err; + + err = setattr_prepare(&nop_mnt_idmap, dentry, iattr); + if (err) + return err; + + err = nilfs_transaction_begin(sb, &ti, 0); + if (unlikely(err)) + return err; + + if ((iattr->ia_valid & ATTR_SIZE) && + iattr->ia_size != i_size_read(inode)) { + inode_dio_wait(inode); + truncate_setsize(inode, iattr->ia_size); + nilfs_truncate(inode); + } + + setattr_copy(&nop_mnt_idmap, inode, iattr); + mark_inode_dirty(inode); + + if (iattr->ia_valid & ATTR_MODE) { + err = nilfs_acl_chmod(inode); + if (unlikely(err)) + goto out_err; + } + + return nilfs_transaction_commit(sb); + +out_err: + nilfs_transaction_abort(sb); + return err; +} + +int nilfs_permission(struct mnt_idmap *idmap, struct inode *inode, + int mask) +{ + struct nilfs_root *root = NILFS_I(inode)->i_root; + + if ((mask & MAY_WRITE) && root && + root->cno != NILFS_CPTREE_CURRENT_CNO) + return -EROFS; /* snapshot is not writable */ + + return generic_permission(&nop_mnt_idmap, inode, mask); +} + +int nilfs_load_inode_block(struct inode *inode, struct buffer_head **pbh) +{ + struct the_nilfs *nilfs = inode->i_sb->s_fs_info; + struct nilfs_inode_info *ii = NILFS_I(inode); + int err; + + spin_lock(&nilfs->ns_inode_lock); + if (ii->i_bh == NULL || unlikely(!buffer_uptodate(ii->i_bh))) { + spin_unlock(&nilfs->ns_inode_lock); + err = nilfs_ifile_get_inode_block(ii->i_root->ifile, + inode->i_ino, pbh); + if (unlikely(err)) + return err; + spin_lock(&nilfs->ns_inode_lock); + if (ii->i_bh == NULL) + ii->i_bh = *pbh; + else if (unlikely(!buffer_uptodate(ii->i_bh))) { + __brelse(ii->i_bh); + ii->i_bh = *pbh; + } else { + brelse(*pbh); + *pbh = ii->i_bh; + } + } else + *pbh = ii->i_bh; + + get_bh(*pbh); + spin_unlock(&nilfs->ns_inode_lock); + return 0; +} + +int nilfs_inode_dirty(struct inode *inode) +{ + struct nilfs_inode_info *ii = NILFS_I(inode); + struct the_nilfs *nilfs = inode->i_sb->s_fs_info; + int ret = 0; + + if (!list_empty(&ii->i_dirty)) { + spin_lock(&nilfs->ns_inode_lock); + ret = test_bit(NILFS_I_DIRTY, &ii->i_state) || + test_bit(NILFS_I_BUSY, &ii->i_state); + spin_unlock(&nilfs->ns_inode_lock); + } + return ret; +} + +int nilfs_set_file_dirty(struct inode *inode, unsigned int nr_dirty) +{ + struct nilfs_inode_info *ii = NILFS_I(inode); + struct the_nilfs *nilfs = inode->i_sb->s_fs_info; + + atomic_add(nr_dirty, &nilfs->ns_ndirtyblks); + + if (test_and_set_bit(NILFS_I_DIRTY, &ii->i_state)) + return 0; + + spin_lock(&nilfs->ns_inode_lock); + if (!test_bit(NILFS_I_QUEUED, &ii->i_state) && + !test_bit(NILFS_I_BUSY, &ii->i_state)) { + /* + * Because this routine may race with nilfs_dispose_list(), + * we have to check NILFS_I_QUEUED here, too. + */ + if (list_empty(&ii->i_dirty) && igrab(inode) == NULL) { + /* + * This will happen when somebody is freeing + * this inode. + */ + nilfs_warn(inode->i_sb, + "cannot set file dirty (ino=%llu): the file is being freed", + inode->i_ino); + spin_unlock(&nilfs->ns_inode_lock); + return -EINVAL; /* + * NILFS_I_DIRTY may remain for + * freeing inode. + */ + } + list_move_tail(&ii->i_dirty, &nilfs->ns_dirty_files); + set_bit(NILFS_I_QUEUED, &ii->i_state); + } + spin_unlock(&nilfs->ns_inode_lock); + return 0; +} + +int __nilfs_mark_inode_dirty(struct inode *inode, int flags) +{ + struct the_nilfs *nilfs = inode->i_sb->s_fs_info; + struct buffer_head *ibh; + int err; + + /* + * Do not dirty inodes after the log writer has been detached + * and its nilfs_root struct has been freed. + */ + if (unlikely(nilfs_purging(nilfs))) + return 0; + + err = nilfs_load_inode_block(inode, &ibh); + if (unlikely(err)) { + nilfs_warn(inode->i_sb, + "cannot mark inode dirty (ino=%llu): error %d loading inode block", + inode->i_ino, err); + return err; + } + nilfs_update_inode(inode, ibh, flags); + mark_buffer_dirty(ibh); + nilfs_mdt_mark_dirty(NILFS_I(inode)->i_root->ifile); + brelse(ibh); + return 0; +} + +/** + * nilfs_dirty_inode - reflect changes on given inode to an inode block. + * @inode: inode of the file to be registered. + * @flags: flags to determine the dirty state of the inode + * + * nilfs_dirty_inode() loads a inode block containing the specified + * @inode and copies data from a nilfs_inode to a corresponding inode + * entry in the inode block. This operation is excluded from the segment + * construction. This function can be called both as a single operation + * and as a part of indivisible file operations. + */ +void nilfs_dirty_inode(struct inode *inode, int flags) +{ + struct nilfs_transaction_info ti; + struct nilfs_mdt_info *mdi = NILFS_MDT(inode); + + if (is_bad_inode(inode)) { + nilfs_warn(inode->i_sb, + "tried to mark bad_inode dirty. ignored."); + dump_stack(); + return; + } + if (mdi) { + nilfs_mdt_mark_dirty(inode); + return; + } + nilfs_transaction_begin(inode->i_sb, &ti, 0); + __nilfs_mark_inode_dirty(inode, flags); + nilfs_transaction_commit(inode->i_sb); /* never fails */ +} + +int nilfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + __u64 start, __u64 len) +{ + struct the_nilfs *nilfs = inode->i_sb->s_fs_info; + __u64 logical = 0, phys = 0, size = 0; + __u32 flags = 0; + loff_t isize; + sector_t blkoff, end_blkoff; + sector_t delalloc_blkoff; + unsigned long delalloc_blklen; + unsigned int blkbits = inode->i_blkbits; + int ret, n; + + ret = fiemap_prep(inode, fieinfo, start, &len, 0); + if (ret) + return ret; + + inode_lock(inode); + + isize = i_size_read(inode); + + blkoff = start >> blkbits; + end_blkoff = (start + len - 1) >> blkbits; + + delalloc_blklen = nilfs_find_uncommitted_extent(inode, blkoff, + &delalloc_blkoff); + + do { + __u64 blkphy; + unsigned int maxblocks; + + if (delalloc_blklen && blkoff == delalloc_blkoff) { + if (size) { + /* End of the current extent */ + ret = fiemap_fill_next_extent( + fieinfo, logical, phys, size, flags); + if (ret) + break; + } + if (blkoff > end_blkoff) + break; + + flags = FIEMAP_EXTENT_MERGED | FIEMAP_EXTENT_DELALLOC; + logical = blkoff << blkbits; + phys = 0; + size = delalloc_blklen << blkbits; + + blkoff = delalloc_blkoff + delalloc_blklen; + delalloc_blklen = nilfs_find_uncommitted_extent( + inode, blkoff, &delalloc_blkoff); + continue; + } + + /* + * Limit the number of blocks that we look up so as + * not to get into the next delayed allocation extent. + */ + maxblocks = INT_MAX; + if (delalloc_blklen) + maxblocks = min_t(sector_t, delalloc_blkoff - blkoff, + maxblocks); + blkphy = 0; + + down_read(&NILFS_MDT(nilfs->ns_dat)->mi_sem); + n = nilfs_bmap_lookup_contig( + NILFS_I(inode)->i_bmap, blkoff, &blkphy, maxblocks); + up_read(&NILFS_MDT(nilfs->ns_dat)->mi_sem); + + if (n < 0) { + int past_eof; + + if (unlikely(n != -ENOENT)) + break; /* error */ + + /* HOLE */ + blkoff++; + past_eof = ((blkoff << blkbits) >= isize); + + if (size) { + /* End of the current extent */ + + if (past_eof) + flags |= FIEMAP_EXTENT_LAST; + + ret = fiemap_fill_next_extent( + fieinfo, logical, phys, size, flags); + if (ret) + break; + size = 0; + } + if (blkoff > end_blkoff || past_eof) + break; + } else { + if (size) { + if (phys && blkphy << blkbits == phys + size) { + /* The current extent goes on */ + size += (u64)n << blkbits; + } else { + /* Terminate the current extent */ + ret = fiemap_fill_next_extent( + fieinfo, logical, phys, size, + flags); + if (ret || blkoff > end_blkoff) + break; + + /* Start another extent */ + flags = FIEMAP_EXTENT_MERGED; + logical = blkoff << blkbits; + phys = blkphy << blkbits; + size = (u64)n << blkbits; + } + } else { + /* Start a new extent */ + flags = FIEMAP_EXTENT_MERGED; + logical = blkoff << blkbits; + phys = blkphy << blkbits; + size = (u64)n << blkbits; + } + blkoff += n; + } + cond_resched(); + } while (true); + + /* If ret is 1 then we just hit the end of the extent array */ + if (ret == 1) + ret = 0; + + inode_unlock(inode); + return ret; +} diff --git a/fs/nilfs2/ioctl.c b/fs/nilfs2/ioctl.c new file mode 100644 index 000000000..01a04080e --- /dev/null +++ b/fs/nilfs2/ioctl.c @@ -0,0 +1,1415 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * NILFS ioctl operations. + * + * Copyright (C) 2007, 2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Koji Sato. + */ + +#include +#include +#include +#include /* capable() */ +#include /* copy_from_user(), copy_to_user() */ +#include +#include /* compat_ptr() */ +#include /* mnt_want_write_file(), mnt_drop_write_file() */ +#include +#include +#include +#include "nilfs.h" +#include "segment.h" +#include "bmap.h" +#include "cpfile.h" +#include "sufile.h" +#include "dat.h" + +/** + * nilfs_ioctl_wrap_copy - wrapping function of get/set metadata info + * @nilfs: nilfs object + * @argv: vector of arguments from userspace + * @dir: set of direction flags + * @dofunc: concrete function of get/set metadata info + * + * Description: nilfs_ioctl_wrap_copy() gets/sets metadata info by means of + * calling dofunc() function on the basis of @argv argument. If successful, + * the requested metadata information is copied to userspace memory. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EFAULT - Failure during execution of requested operation. + * * %-EINVAL - Invalid arguments from userspace. + * * %-ENOMEM - Insufficient memory available. + */ +static int nilfs_ioctl_wrap_copy(struct the_nilfs *nilfs, + struct nilfs_argv *argv, int dir, + ssize_t (*dofunc)(struct the_nilfs *, + __u64 *, int, + void *, size_t, size_t)) +{ + void *buf; + void __user *base = u64_to_user_ptr(argv->v_base); + size_t maxmembs, total, n; + ssize_t nr; + int ret, i; + __u64 pos, ppos; + + if (argv->v_nmembs == 0) + return 0; + + if ((size_t)argv->v_size > PAGE_SIZE) + return -EINVAL; + + /* + * Reject pairs of a start item position (argv->v_index) and a + * total count (argv->v_nmembs) which leads position 'pos' to + * overflow by the increment at the end of the loop. + */ + if (argv->v_index > ~(__u64)0 - argv->v_nmembs) + return -EINVAL; + + buf = kzalloc(PAGE_SIZE, GFP_NOFS); + if (unlikely(!buf)) + return -ENOMEM; + maxmembs = PAGE_SIZE / argv->v_size; + + ret = 0; + total = 0; + pos = argv->v_index; + for (i = 0; i < argv->v_nmembs; i += n) { + n = (argv->v_nmembs - i < maxmembs) ? + argv->v_nmembs - i : maxmembs; + if ((dir & _IOC_WRITE) && + copy_from_user(buf, base + argv->v_size * i, + argv->v_size * n)) { + ret = -EFAULT; + break; + } + ppos = pos; + nr = dofunc(nilfs, &pos, argv->v_flags, buf, argv->v_size, + n); + if (nr < 0) { + ret = nr; + break; + } + if ((dir & _IOC_READ) && + copy_to_user(base + argv->v_size * i, buf, + argv->v_size * nr)) { + ret = -EFAULT; + break; + } + total += nr; + if ((size_t)nr < n) + break; + if (pos == ppos) + pos += n; + } + argv->v_nmembs = total; + + kfree(buf); + return ret; +} + +/** + * nilfs_fileattr_get - retrieve miscellaneous file attributes + * @dentry: the object to retrieve from + * @fa: fileattr pointer + * + * Return: always 0 as success. + */ +int nilfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + + fileattr_fill_flags(fa, NILFS_I(inode)->i_flags & FS_FL_USER_VISIBLE); + + return 0; +} + +/** + * nilfs_fileattr_set - change miscellaneous file attributes + * @idmap: idmap of the mount + * @dentry: the object to change + * @fa: fileattr pointer + * + * Return: 0 on success, or a negative error code on failure. + */ +int nilfs_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + struct nilfs_transaction_info ti; + unsigned int flags, oldflags; + int ret; + + if (fileattr_has_fsx(fa)) + return -EOPNOTSUPP; + + flags = nilfs_mask_flags(inode->i_mode, fa->flags); + + ret = nilfs_transaction_begin(inode->i_sb, &ti, 0); + if (ret) + return ret; + + oldflags = NILFS_I(inode)->i_flags & ~FS_FL_USER_MODIFIABLE; + NILFS_I(inode)->i_flags = oldflags | (flags & FS_FL_USER_MODIFIABLE); + + nilfs_set_inode_flags(inode); + inode_set_ctime_current(inode); + if (IS_SYNC(inode)) + nilfs_set_transaction_flag(NILFS_TI_SYNC); + + nilfs_mark_inode_dirty(inode); + return nilfs_transaction_commit(inode->i_sb); +} + +/** + * nilfs_ioctl_getversion - get info about a file's version (generation number) + * @inode: inode object + * @argp: userspace memory where the generation number of @inode is stored + * + * Return: 0 on success, or %-EFAULT on error. + */ +static int nilfs_ioctl_getversion(struct inode *inode, void __user *argp) +{ + return put_user(inode->i_generation, (int __user *)argp); +} + +/** + * nilfs_ioctl_change_cpmode - change checkpoint mode (checkpoint/snapshot) + * @inode: inode object + * @filp: file object + * @cmd: ioctl's request code + * @argp: pointer on argument from userspace + * + * Description: nilfs_ioctl_change_cpmode() function changes mode of + * given checkpoint between checkpoint and snapshot state. This ioctl + * is used in chcp and mkcp utilities. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * %-EFAULT - Failure during checkpoint mode changing. + * %-EPERM - Operation not permitted. + */ +static int nilfs_ioctl_change_cpmode(struct inode *inode, struct file *filp, + unsigned int cmd, void __user *argp) +{ + struct the_nilfs *nilfs = inode->i_sb->s_fs_info; + struct nilfs_transaction_info ti; + struct nilfs_cpmode cpmode; + int ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + + ret = -EFAULT; + if (copy_from_user(&cpmode, argp, sizeof(cpmode))) + goto out; + + mutex_lock(&nilfs->ns_snapshot_mount_mutex); + + nilfs_transaction_begin(inode->i_sb, &ti, 0); + ret = nilfs_cpfile_change_cpmode( + nilfs->ns_cpfile, cpmode.cm_cno, cpmode.cm_mode); + if (unlikely(ret < 0)) + nilfs_transaction_abort(inode->i_sb); + else + nilfs_transaction_commit(inode->i_sb); /* never fails */ + + mutex_unlock(&nilfs->ns_snapshot_mount_mutex); +out: + mnt_drop_write_file(filp); + return ret; +} + +/** + * nilfs_ioctl_delete_checkpoint - remove checkpoint + * @inode: inode object + * @filp: file object + * @cmd: ioctl's request code + * @argp: pointer on argument from userspace + * + * Description: nilfs_ioctl_delete_checkpoint() function removes + * checkpoint from NILFS2 file system. This ioctl is used in rmcp + * utility. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * %-EFAULT - Failure during checkpoint removing. + * %-EPERM - Operation not permitted. + */ +static int +nilfs_ioctl_delete_checkpoint(struct inode *inode, struct file *filp, + unsigned int cmd, void __user *argp) +{ + struct the_nilfs *nilfs = inode->i_sb->s_fs_info; + struct nilfs_transaction_info ti; + __u64 cno; + int ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + + ret = -EFAULT; + if (copy_from_user(&cno, argp, sizeof(cno))) + goto out; + + nilfs_transaction_begin(inode->i_sb, &ti, 0); + ret = nilfs_cpfile_delete_checkpoint(nilfs->ns_cpfile, cno); + if (unlikely(ret < 0)) + nilfs_transaction_abort(inode->i_sb); + else + nilfs_transaction_commit(inode->i_sb); /* never fails */ +out: + mnt_drop_write_file(filp); + return ret; +} + +/** + * nilfs_ioctl_do_get_cpinfo - callback method getting info about checkpoints + * @nilfs: nilfs object + * @posp: pointer on array of checkpoint's numbers + * @flags: checkpoint mode (checkpoint or snapshot) + * @buf: buffer for storing checkponts' info + * @size: size in bytes of one checkpoint info item in array + * @nmembs: number of checkpoints in array (numbers and infos) + * + * Description: nilfs_ioctl_do_get_cpinfo() function returns info about + * requested checkpoints. The NILFS_IOCTL_GET_CPINFO ioctl is used in + * lscp utility and by nilfs_cleanerd daemon. + * + * Return: Count of nilfs_cpinfo structures in output buffer. + */ +static ssize_t +nilfs_ioctl_do_get_cpinfo(struct the_nilfs *nilfs, __u64 *posp, int flags, + void *buf, size_t size, size_t nmembs) +{ + int ret; + + down_read(&nilfs->ns_segctor_sem); + ret = nilfs_cpfile_get_cpinfo(nilfs->ns_cpfile, posp, flags, buf, + size, nmembs); + up_read(&nilfs->ns_segctor_sem); + return ret; +} + +/** + * nilfs_ioctl_get_cpstat - get checkpoints statistics + * @inode: inode object + * @filp: file object + * @cmd: ioctl's request code + * @argp: pointer on argument from userspace + * + * Description: nilfs_ioctl_get_cpstat() returns information about checkpoints. + * The NILFS_IOCTL_GET_CPSTAT ioctl is used by lscp, rmcp utilities + * and by nilfs_cleanerd daemon. The checkpoint statistics are copied to + * the userspace memory pointed to by @argp. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EFAULT - Failure during getting checkpoints statistics. + * * %-EIO - I/O error. + * * %-ENOMEM - Insufficient memory available. + */ +static int nilfs_ioctl_get_cpstat(struct inode *inode, struct file *filp, + unsigned int cmd, void __user *argp) +{ + struct the_nilfs *nilfs = inode->i_sb->s_fs_info; + struct nilfs_cpstat cpstat; + int ret; + + down_read(&nilfs->ns_segctor_sem); + ret = nilfs_cpfile_get_stat(nilfs->ns_cpfile, &cpstat); + up_read(&nilfs->ns_segctor_sem); + if (ret < 0) + return ret; + + if (copy_to_user(argp, &cpstat, sizeof(cpstat))) + ret = -EFAULT; + return ret; +} + +/** + * nilfs_ioctl_do_get_suinfo - callback method getting segment usage info + * @nilfs: nilfs object + * @posp: pointer on array of segment numbers + * @flags: *not used* + * @buf: buffer for storing suinfo array + * @size: size in bytes of one suinfo item in array + * @nmembs: count of segment numbers and suinfos in array + * + * Description: nilfs_ioctl_do_get_suinfo() function returns segment usage + * info about requested segments. The NILFS_IOCTL_GET_SUINFO ioctl is used + * in lssu, nilfs_resize utilities and by nilfs_cleanerd daemon. + * + * Return: Count of nilfs_suinfo structures in output buffer on success, + * or a negative error code on failure. + */ +static ssize_t +nilfs_ioctl_do_get_suinfo(struct the_nilfs *nilfs, __u64 *posp, int flags, + void *buf, size_t size, size_t nmembs) +{ + int ret; + + down_read(&nilfs->ns_segctor_sem); + ret = nilfs_sufile_get_suinfo(nilfs->ns_sufile, *posp, buf, size, + nmembs); + up_read(&nilfs->ns_segctor_sem); + return ret; +} + +/** + * nilfs_ioctl_get_sustat - get segment usage statistics + * @inode: inode object + * @filp: file object + * @cmd: ioctl's request code + * @argp: pointer on argument from userspace + * + * Description: nilfs_ioctl_get_sustat() returns segment usage statistics. + * The NILFS_IOCTL_GET_SUSTAT ioctl is used in lssu, nilfs_resize utilities + * and by nilfs_cleanerd daemon. The requested segment usage information is + * copied to the userspace memory pointed to by @argp. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EFAULT - Failure during getting segment usage statistics. + * * %-EIO - I/O error. + * * %-ENOMEM - Insufficient memory available. + */ +static int nilfs_ioctl_get_sustat(struct inode *inode, struct file *filp, + unsigned int cmd, void __user *argp) +{ + struct the_nilfs *nilfs = inode->i_sb->s_fs_info; + struct nilfs_sustat sustat; + int ret; + + down_read(&nilfs->ns_segctor_sem); + ret = nilfs_sufile_get_stat(nilfs->ns_sufile, &sustat); + up_read(&nilfs->ns_segctor_sem); + if (ret < 0) + return ret; + + if (copy_to_user(argp, &sustat, sizeof(sustat))) + ret = -EFAULT; + return ret; +} + +/** + * nilfs_ioctl_do_get_vinfo - callback method getting virtual blocks info + * @nilfs: nilfs object + * @posp: *not used* + * @flags: *not used* + * @buf: buffer for storing array of nilfs_vinfo structures + * @size: size in bytes of one vinfo item in array + * @nmembs: count of vinfos in array + * + * Description: nilfs_ioctl_do_get_vinfo() function returns information + * on virtual block addresses. The NILFS_IOCTL_GET_VINFO ioctl is used + * by nilfs_cleanerd daemon. + * + * Return: Count of nilfs_vinfo structures in output buffer on success, or + * a negative error code on failure. + */ +static ssize_t +nilfs_ioctl_do_get_vinfo(struct the_nilfs *nilfs, __u64 *posp, int flags, + void *buf, size_t size, size_t nmembs) +{ + int ret; + + down_read(&nilfs->ns_segctor_sem); + ret = nilfs_dat_get_vinfo(nilfs->ns_dat, buf, size, nmembs); + up_read(&nilfs->ns_segctor_sem); + return ret; +} + +/** + * nilfs_ioctl_do_get_bdescs - callback method getting disk block descriptors + * @nilfs: nilfs object + * @posp: *not used* + * @flags: *not used* + * @buf: buffer for storing array of nilfs_bdesc structures + * @size: size in bytes of one bdesc item in array + * @nmembs: count of bdescs in array + * + * Description: nilfs_ioctl_do_get_bdescs() function returns information + * about descriptors of disk block numbers. The NILFS_IOCTL_GET_BDESCS ioctl + * is used by nilfs_cleanerd daemon. + * + * Return: Count of nilfs_bdescs structures in output buffer on success, or + * a negative error code on failure. + */ +static ssize_t +nilfs_ioctl_do_get_bdescs(struct the_nilfs *nilfs, __u64 *posp, int flags, + void *buf, size_t size, size_t nmembs) +{ + struct nilfs_bmap *bmap = NILFS_I(nilfs->ns_dat)->i_bmap; + struct nilfs_bdesc *bdescs = buf; + int ret, i; + + down_read(&nilfs->ns_segctor_sem); + for (i = 0; i < nmembs; i++) { + ret = nilfs_bmap_lookup_at_level(bmap, + bdescs[i].bd_offset, + bdescs[i].bd_level + 1, + &bdescs[i].bd_blocknr); + if (ret < 0) { + if (ret != -ENOENT) { + up_read(&nilfs->ns_segctor_sem); + return ret; + } + bdescs[i].bd_blocknr = 0; + } + } + up_read(&nilfs->ns_segctor_sem); + return nmembs; +} + +/** + * nilfs_ioctl_get_bdescs - get disk block descriptors + * @inode: inode object + * @filp: file object + * @cmd: ioctl's request code + * @argp: pointer on argument from userspace + * + * Description: nilfs_ioctl_do_get_bdescs() function returns information + * about descriptors of disk block numbers. The NILFS_IOCTL_GET_BDESCS ioctl + * is used by nilfs_cleanerd daemon. If successful, disk block descriptors + * are copied to userspace pointer @argp. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EFAULT - Failure during getting disk block descriptors. + * * %-EINVAL - Invalid arguments from userspace. + * * %-EIO - I/O error. + * * %-ENOMEM - Insufficient memory available. + */ +static int nilfs_ioctl_get_bdescs(struct inode *inode, struct file *filp, + unsigned int cmd, void __user *argp) +{ + struct the_nilfs *nilfs = inode->i_sb->s_fs_info; + struct nilfs_argv argv; + int ret; + + if (copy_from_user(&argv, argp, sizeof(argv))) + return -EFAULT; + + if (argv.v_size != sizeof(struct nilfs_bdesc)) + return -EINVAL; + + ret = nilfs_ioctl_wrap_copy(nilfs, &argv, _IOC_DIR(cmd), + nilfs_ioctl_do_get_bdescs); + if (ret < 0) + return ret; + + if (copy_to_user(argp, &argv, sizeof(argv))) + ret = -EFAULT; + return ret; +} + +/** + * nilfs_ioctl_move_inode_block - prepare data/node block for moving by GC + * @inode: inode object + * @vdesc: descriptor of virtual block number + * @buffers: list of moving buffers + * + * Description: nilfs_ioctl_move_inode_block() function registers data/node + * buffer in the GC pagecache and submit read request. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EEXIST - Block conflict detected. + * * %-EINVAL - Invalid virtual block descriptor. + * * %-EIO - I/O error. + * * %-ENOENT - Requested block doesn't exist. + * * %-ENOMEM - Insufficient memory available. + */ +static int nilfs_ioctl_move_inode_block(struct inode *inode, + struct nilfs_vdesc *vdesc, + struct list_head *buffers) +{ + struct buffer_head *bh; + __u64 limit_blkidx = (__u64)inode->i_sb->s_maxbytes >> inode->i_blkbits; + int ret; + + /* + * vblocknr 0 is reserved as an invalid pointer. Also, limit_blkidx + * ensures that the page index converted from vd_vblocknr never + * overflows the page cache limit and respects the architecture's bmap + * key width. + */ + if (unlikely(vdesc->vd_vblocknr == 0 || + vdesc->vd_vblocknr >= limit_blkidx)) + return -EINVAL; + + if (vdesc->vd_flags == 0) { + if (unlikely(vdesc->vd_offset >= limit_blkidx)) + return -EINVAL; + + ret = nilfs_gccache_submit_read_data( + inode, vdesc->vd_offset, vdesc->vd_blocknr, + vdesc->vd_vblocknr, &bh); + } else { + ret = nilfs_gccache_submit_read_node( + inode, vdesc->vd_blocknr, vdesc->vd_vblocknr, &bh); + } + + if (unlikely(ret < 0)) { + if (ret == -ENOENT) + nilfs_crit(inode->i_sb, + "%s: invalid virtual block address (%s): ino=%llu, cno=%llu, offset=%llu, blocknr=%llu, vblocknr=%llu", + __func__, vdesc->vd_flags ? "node" : "data", + (unsigned long long)vdesc->vd_ino, + (unsigned long long)vdesc->vd_cno, + (unsigned long long)vdesc->vd_offset, + (unsigned long long)vdesc->vd_blocknr, + (unsigned long long)vdesc->vd_vblocknr); + return ret; + } + if (unlikely(!list_empty(&bh->b_assoc_buffers))) { + nilfs_crit(inode->i_sb, + "%s: conflicting %s buffer: ino=%llu, cno=%llu, offset=%llu, blocknr=%llu, vblocknr=%llu", + __func__, vdesc->vd_flags ? "node" : "data", + (unsigned long long)vdesc->vd_ino, + (unsigned long long)vdesc->vd_cno, + (unsigned long long)vdesc->vd_offset, + (unsigned long long)vdesc->vd_blocknr, + (unsigned long long)vdesc->vd_vblocknr); + brelse(bh); + return -EEXIST; + } + list_add_tail(&bh->b_assoc_buffers, buffers); + return 0; +} + +/** + * nilfs_ioctl_move_blocks - move valid inode's blocks during garbage collection + * @sb: superblock object + * @argv: vector of arguments from userspace + * @buf: array of nilfs_vdesc structures + * + * Description: nilfs_ioctl_move_blocks() function reads valid data/node + * blocks that garbage collector specified with the array of nilfs_vdesc + * structures and stores them into page caches of GC inodes. + * + * Return: Number of processed nilfs_vdesc structures on success, or + * a negative error code on failure. + */ +static int nilfs_ioctl_move_blocks(struct super_block *sb, + struct nilfs_argv *argv, void *buf) +{ + size_t nmembs = argv->v_nmembs; + struct the_nilfs *nilfs = sb->s_fs_info; + struct inode *inode; + struct nilfs_vdesc *vdesc; + struct buffer_head *bh, *n; + LIST_HEAD(buffers); + u64 ino; + __u64 cno; + int i, ret; + + for (i = 0, vdesc = buf; i < nmembs; ) { + ino = vdesc->vd_ino; + cno = vdesc->vd_cno; + inode = nilfs_iget_for_gc(sb, ino, cno); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + goto failed; + } + if (list_empty(&NILFS_I(inode)->i_dirty)) { + /* + * Add the inode to GC inode list. Garbage Collection + * is serialized and no two processes manipulate the + * list simultaneously. + */ + igrab(inode); + list_add(&NILFS_I(inode)->i_dirty, + &nilfs->ns_gc_inodes); + } + + do { + ret = nilfs_ioctl_move_inode_block(inode, vdesc, + &buffers); + if (unlikely(ret < 0)) { + iput(inode); + goto failed; + } + vdesc++; + } while (++i < nmembs && + vdesc->vd_ino == ino && vdesc->vd_cno == cno); + + iput(inode); /* The inode still remains in GC inode list */ + } + + list_for_each_entry_safe(bh, n, &buffers, b_assoc_buffers) { + ret = nilfs_gccache_wait_and_mark_dirty(bh); + if (unlikely(ret < 0)) { + WARN_ON(ret == -EEXIST); + goto failed; + } + list_del_init(&bh->b_assoc_buffers); + brelse(bh); + } + return nmembs; + + failed: + list_for_each_entry_safe(bh, n, &buffers, b_assoc_buffers) { + list_del_init(&bh->b_assoc_buffers); + brelse(bh); + } + return ret; +} + +/** + * nilfs_ioctl_delete_checkpoints - delete checkpoints + * @nilfs: nilfs object + * @argv: vector of arguments from userspace + * @buf: array of periods of checkpoints numbers + * + * Description: nilfs_ioctl_delete_checkpoints() function deletes checkpoints + * in the period from p_start to p_end, excluding p_end itself. The checkpoints + * which have been already deleted are ignored. + * + * Return: Number of processed nilfs_period structures on success, or one of + * the following negative error codes on failure: + * * %-EINVAL - invalid checkpoints. + * * %-EIO - I/O error. + * * %-ENOMEM - Insufficient memory available. + */ +static int nilfs_ioctl_delete_checkpoints(struct the_nilfs *nilfs, + struct nilfs_argv *argv, void *buf) +{ + size_t nmembs = argv->v_nmembs; + struct inode *cpfile = nilfs->ns_cpfile; + struct nilfs_period *periods = buf; + int ret, i; + + for (i = 0; i < nmembs; i++) { + ret = nilfs_cpfile_delete_checkpoints( + cpfile, periods[i].p_start, periods[i].p_end); + if (ret < 0) + return ret; + } + return nmembs; +} + +/** + * nilfs_ioctl_free_vblocknrs - free virtual block numbers + * @nilfs: nilfs object + * @argv: vector of arguments from userspace + * @buf: array of virtual block numbers + * + * Description: nilfs_ioctl_free_vblocknrs() function frees + * the virtual block numbers specified by @buf and @argv->v_nmembs. + * + * Return: Number of processed virtual block numbers on success, or one of the + * following negative error codes on failure: + * * %-EIO - I/O error. + * * %-ENOENT - Unallocated virtual block number. + * * %-ENOMEM - Insufficient memory available. + */ +static int nilfs_ioctl_free_vblocknrs(struct the_nilfs *nilfs, + struct nilfs_argv *argv, void *buf) +{ + size_t nmembs = argv->v_nmembs; + int ret; + + ret = nilfs_dat_freev(nilfs->ns_dat, buf, nmembs); + + return (ret < 0) ? ret : nmembs; +} + +/** + * nilfs_ioctl_mark_blocks_dirty - mark blocks dirty + * @nilfs: nilfs object + * @argv: vector of arguments from userspace + * @buf: array of block descriptors + * + * Description: nilfs_ioctl_mark_blocks_dirty() function marks + * metadata file or data blocks as dirty. + * + * Return: Number of processed block descriptors on success, or one of the + * following negative error codes on failure: + * * %-EIO - I/O error. + * * %-ENOENT - Non-existent block (hole block). + * * %-ENOMEM - Insufficient memory available. + */ +static int nilfs_ioctl_mark_blocks_dirty(struct the_nilfs *nilfs, + struct nilfs_argv *argv, void *buf) +{ + size_t nmembs = argv->v_nmembs; + struct nilfs_bmap *bmap = NILFS_I(nilfs->ns_dat)->i_bmap; + struct nilfs_bdesc *bdescs = buf; + struct buffer_head *bh; + int ret, i; + + for (i = 0; i < nmembs; i++) { + /* + * bd_oblocknr must never be 0 as block 0 + * is never a valid GC target block + */ + if (unlikely(!bdescs[i].bd_oblocknr)) + return -EINVAL; + /* XXX: use macro or inline func to check liveness */ + ret = nilfs_bmap_lookup_at_level(bmap, + bdescs[i].bd_offset, + bdescs[i].bd_level + 1, + &bdescs[i].bd_blocknr); + if (ret < 0) { + if (ret != -ENOENT) + return ret; + bdescs[i].bd_blocknr = 0; + } + if (bdescs[i].bd_blocknr != bdescs[i].bd_oblocknr) + /* skip dead block */ + continue; + if (bdescs[i].bd_level == 0) { + ret = nilfs_mdt_get_block(nilfs->ns_dat, + bdescs[i].bd_offset, + false, NULL, &bh); + if (unlikely(ret)) { + WARN_ON(ret == -ENOENT); + return ret; + } + mark_buffer_dirty(bh); + nilfs_mdt_mark_dirty(nilfs->ns_dat); + put_bh(bh); + } else { + ret = nilfs_bmap_mark(bmap, bdescs[i].bd_offset, + bdescs[i].bd_level); + if (ret < 0) { + WARN_ON(ret == -ENOENT); + return ret; + } + } + } + return nmembs; +} + +int nilfs_ioctl_prepare_clean_segments(struct the_nilfs *nilfs, + struct nilfs_argv *argv, void **kbufs) +{ + const char *msg; + int ret; + + ret = nilfs_ioctl_delete_checkpoints(nilfs, &argv[1], kbufs[1]); + if (ret < 0) { + /* + * can safely abort because checkpoints can be removed + * independently. + */ + msg = "cannot delete checkpoints"; + goto failed; + } + ret = nilfs_ioctl_free_vblocknrs(nilfs, &argv[2], kbufs[2]); + if (ret < 0) { + /* + * can safely abort because DAT file is updated atomically + * using a copy-on-write technique. + */ + msg = "cannot delete virtual blocks from DAT file"; + goto failed; + } + ret = nilfs_ioctl_mark_blocks_dirty(nilfs, &argv[3], kbufs[3]); + if (ret < 0) { + /* + * can safely abort because the operation is nondestructive. + */ + msg = "cannot mark copying blocks dirty"; + goto failed; + } + return 0; + + failed: + nilfs_err(nilfs->ns_sb, "error %d preparing GC: %s", ret, msg); + return ret; +} + +/** + * nilfs_ioctl_clean_segments - clean segments + * @inode: inode object + * @filp: file object + * @cmd: ioctl's request code + * @argp: pointer on argument from userspace + * + * Description: nilfs_ioctl_clean_segments() function makes garbage + * collection operation in the environment of requested parameters + * from userspace. The NILFS_IOCTL_CLEAN_SEGMENTS ioctl is used by + * nilfs_cleanerd daemon. + * + * Return: 0 on success, or a negative error code on failure. + */ +static int nilfs_ioctl_clean_segments(struct inode *inode, struct file *filp, + unsigned int cmd, void __user *argp) +{ + struct nilfs_argv argv[5]; + static const size_t argsz[5] = { + sizeof(struct nilfs_vdesc), + sizeof(struct nilfs_period), + sizeof(__u64), + sizeof(struct nilfs_bdesc), + sizeof(__u64), + }; + void *kbufs[5]; + struct the_nilfs *nilfs; + size_t len, nsegs; + int n, ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + + ret = -EFAULT; + if (copy_from_user(argv, argp, sizeof(argv))) + goto out; + + ret = -EINVAL; + nsegs = argv[4].v_nmembs; + if (argv[4].v_size != argsz[4]) + goto out; + + /* + * argv[4] points to segment numbers this ioctl cleans. We + * use kmalloc() for its buffer because the memory used for the + * segment numbers is small enough. + */ + kbufs[4] = memdup_array_user(u64_to_user_ptr(argv[4].v_base), + nsegs, sizeof(__u64)); + if (IS_ERR(kbufs[4])) { + ret = PTR_ERR(kbufs[4]); + goto out; + } + nilfs = inode->i_sb->s_fs_info; + + for (n = 0; n < 4; n++) { + ret = -EINVAL; + if (argv[n].v_size != argsz[n]) + goto out_free; + + if (argv[n].v_nmembs > nsegs * nilfs->ns_blocks_per_segment) + goto out_free; + + if (argv[n].v_nmembs >= UINT_MAX / argv[n].v_size) + goto out_free; + + len = argv[n].v_size * argv[n].v_nmembs; + if (len == 0) { + kbufs[n] = NULL; + continue; + } + + kbufs[n] = vmemdup_user(u64_to_user_ptr(argv[n].v_base), len); + if (IS_ERR(kbufs[n])) { + ret = PTR_ERR(kbufs[n]); + goto out_free; + } + } + + /* + * nilfs_ioctl_move_blocks() will call nilfs_iget_for_gc(), + * which will operates an inode list without blocking. + * To protect the list from concurrent operations, + * nilfs_ioctl_move_blocks should be atomic operation. + */ + if (test_and_set_bit(THE_NILFS_GC_RUNNING, &nilfs->ns_flags)) { + ret = -EBUSY; + goto out_free; + } + + ret = nilfs_ioctl_move_blocks(inode->i_sb, &argv[0], kbufs[0]); + if (ret < 0) { + nilfs_err(inode->i_sb, + "error %d preparing GC: cannot read source blocks", + ret); + } else { + if (nilfs_sb_need_update(nilfs)) + set_nilfs_discontinued(nilfs); + ret = nilfs_clean_segments(inode->i_sb, argv, kbufs); + } + + nilfs_remove_all_gcinodes(nilfs); + clear_nilfs_gc_running(nilfs); + +out_free: + while (--n >= 0) + kvfree(kbufs[n]); + kfree(kbufs[4]); +out: + mnt_drop_write_file(filp); + return ret; +} + +/** + * nilfs_ioctl_sync - make a checkpoint + * @inode: inode object + * @filp: file object + * @cmd: ioctl's request code + * @argp: pointer on argument from userspace + * + * Description: nilfs_ioctl_sync() function constructs a logical segment + * for checkpointing. This function guarantees that all modified data + * and metadata are written out to the device when it successfully + * returned. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EFAULT - Failure during execution of requested operation. + * * %-EIO - I/O error. + * * %-ENOMEM - Insufficient memory available. + * * %-ENOSPC - No space left on device (only in a panic state). + * * %-ERESTARTSYS - Interrupted. + * * %-EROFS - Read only filesystem. + */ +static int nilfs_ioctl_sync(struct inode *inode, struct file *filp, + unsigned int cmd, void __user *argp) +{ + __u64 cno; + int ret; + struct the_nilfs *nilfs; + + ret = nilfs_construct_segment(inode->i_sb); + if (ret < 0) + return ret; + + nilfs = inode->i_sb->s_fs_info; + ret = nilfs_flush_device(nilfs); + if (ret < 0) + return ret; + + if (argp != NULL) { + down_read(&nilfs->ns_segctor_sem); + cno = nilfs->ns_cno - 1; + up_read(&nilfs->ns_segctor_sem); + if (copy_to_user(argp, &cno, sizeof(cno))) + return -EFAULT; + } + return 0; +} + +/** + * nilfs_ioctl_resize - resize NILFS2 volume + * @inode: inode object + * @filp: file object + * @argp: pointer on argument from userspace + * + * Return: 0 on success, or a negative error code on failure. + */ +static int nilfs_ioctl_resize(struct inode *inode, struct file *filp, + void __user *argp) +{ + __u64 newsize; + int ret = -EPERM; + + if (!capable(CAP_SYS_ADMIN)) + goto out; + + ret = mnt_want_write_file(filp); + if (ret) + goto out; + + ret = -EFAULT; + if (copy_from_user(&newsize, argp, sizeof(newsize))) + goto out_drop_write; + + ret = nilfs_resize_fs(inode->i_sb, newsize); + +out_drop_write: + mnt_drop_write_file(filp); +out: + return ret; +} + +/** + * nilfs_ioctl_trim_fs() - trim ioctl handle function + * @inode: inode object + * @argp: pointer on argument from userspace + * + * Description: nilfs_ioctl_trim_fs is the FITRIM ioctl handle function. It + * checks the arguments from userspace and calls nilfs_sufile_trim_fs, which + * performs the actual trim operation. + * + * Return: 0 on success, or a negative error code on failure. + */ +static int nilfs_ioctl_trim_fs(struct inode *inode, void __user *argp) +{ + struct the_nilfs *nilfs = inode->i_sb->s_fs_info; + struct fstrim_range range; + int ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (!bdev_max_discard_sectors(nilfs->ns_bdev)) + return -EOPNOTSUPP; + + if (copy_from_user(&range, argp, sizeof(range))) + return -EFAULT; + + range.minlen = max_t(u64, range.minlen, + bdev_discard_granularity(nilfs->ns_bdev)); + + down_read(&nilfs->ns_segctor_sem); + ret = nilfs_sufile_trim_fs(nilfs->ns_sufile, &range); + up_read(&nilfs->ns_segctor_sem); + + if (ret < 0) + return ret; + + if (copy_to_user(argp, &range, sizeof(range))) + return -EFAULT; + + return 0; +} + +/** + * nilfs_ioctl_set_alloc_range - limit range of segments to be allocated + * @inode: inode object + * @argp: pointer on argument from userspace + * + * Description: nilfs_ioctl_set_alloc_range() function defines lower limit + * of segments in bytes and upper limit of segments in bytes. + * The NILFS_IOCTL_SET_ALLOC_RANGE is used by nilfs_resize utility. + * + * Return: 0 on success, or a negative error code on failure. + */ +static int nilfs_ioctl_set_alloc_range(struct inode *inode, void __user *argp) +{ + struct the_nilfs *nilfs = inode->i_sb->s_fs_info; + __u64 range[2]; + __u64 minseg, maxseg; + unsigned long segbytes; + int ret = -EPERM; + + if (!capable(CAP_SYS_ADMIN)) + goto out; + + ret = -EFAULT; + if (copy_from_user(range, argp, sizeof(__u64[2]))) + goto out; + + ret = -ERANGE; + if (range[1] > bdev_nr_bytes(inode->i_sb->s_bdev)) + goto out; + + segbytes = nilfs->ns_blocks_per_segment * nilfs->ns_blocksize; + + minseg = range[0] + segbytes - 1; + minseg = div64_ul(minseg, segbytes); + + if (range[1] < 4096) + goto out; + + maxseg = NILFS_SB2_OFFSET_BYTES(range[1]); + if (maxseg < segbytes) + goto out; + + maxseg = div64_ul(maxseg, segbytes); + maxseg--; + + ret = nilfs_sufile_set_alloc_range(nilfs->ns_sufile, minseg, maxseg); +out: + return ret; +} + +/** + * nilfs_ioctl_get_info - wrapping function of get metadata info + * @inode: inode object + * @filp: file object + * @cmd: ioctl's request code + * @argp: pointer on argument from userspace + * @membsz: size of an item in bytes + * @dofunc: concrete function of getting metadata info + * + * Description: nilfs_ioctl_get_info() gets metadata info by means of + * calling dofunc() function. The requested metadata information is copied + * to userspace memory @argp. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EFAULT - Failure during execution of requested operation. + * * %-EINVAL - Invalid arguments from userspace. + * * %-EIO - I/O error. + * * %-ENOMEM - Insufficient memory available. + */ +static int nilfs_ioctl_get_info(struct inode *inode, struct file *filp, + unsigned int cmd, void __user *argp, + size_t membsz, + ssize_t (*dofunc)(struct the_nilfs *, + __u64 *, int, + void *, size_t, size_t)) + +{ + struct the_nilfs *nilfs = inode->i_sb->s_fs_info; + struct nilfs_argv argv; + int ret; + + if (copy_from_user(&argv, argp, sizeof(argv))) + return -EFAULT; + + if (argv.v_size < membsz) + return -EINVAL; + + ret = nilfs_ioctl_wrap_copy(nilfs, &argv, _IOC_DIR(cmd), dofunc); + if (ret < 0) + return ret; + + if (copy_to_user(argp, &argv, sizeof(argv))) + ret = -EFAULT; + return ret; +} + +/** + * nilfs_ioctl_set_suinfo - set segment usage info + * @inode: inode object + * @filp: file object + * @cmd: ioctl's request code + * @argp: pointer on argument from userspace + * + * Description: Expects an array of nilfs_suinfo_update structures + * encapsulated in nilfs_argv and updates the segment usage info + * according to the flags in nilfs_suinfo_update. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EEXIST - Block conflict detected. + * * %-EFAULT - Error copying input data. + * * %-EINVAL - Invalid values in input (segment number, flags or nblocks). + * * %-EIO - I/O error. + * * %-ENOMEM - Insufficient memory available. + * * %-EPERM - Not enough permissions. + */ +static int nilfs_ioctl_set_suinfo(struct inode *inode, struct file *filp, + unsigned int cmd, void __user *argp) +{ + struct the_nilfs *nilfs = inode->i_sb->s_fs_info; + struct nilfs_transaction_info ti; + struct nilfs_argv argv; + size_t len; + void *kbuf; + int ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + + ret = -EFAULT; + if (copy_from_user(&argv, argp, sizeof(argv))) + goto out; + + ret = -EINVAL; + if (argv.v_size < sizeof(struct nilfs_suinfo_update)) + goto out; + + if (argv.v_nmembs > nilfs->ns_nsegments) + goto out; + + if (argv.v_nmembs >= UINT_MAX / argv.v_size) + goto out; + + len = argv.v_size * argv.v_nmembs; + if (!len) { + ret = 0; + goto out; + } + + kbuf = vmemdup_user(u64_to_user_ptr(argv.v_base), len); + if (IS_ERR(kbuf)) { + ret = PTR_ERR(kbuf); + goto out; + } + + nilfs_transaction_begin(inode->i_sb, &ti, 0); + ret = nilfs_sufile_set_suinfo(nilfs->ns_sufile, kbuf, argv.v_size, + argv.v_nmembs); + if (unlikely(ret < 0)) + nilfs_transaction_abort(inode->i_sb); + else + nilfs_transaction_commit(inode->i_sb); /* never fails */ + + kvfree(kbuf); +out: + mnt_drop_write_file(filp); + return ret; +} + +/** + * nilfs_ioctl_get_fslabel - get the volume name of the file system + * @sb: super block instance + * @argp: pointer to userspace memory where the volume name should be stored + * + * Return: 0 on success, %-EFAULT if copying to userspace memory fails. + */ +static int nilfs_ioctl_get_fslabel(struct super_block *sb, void __user *argp) +{ + struct the_nilfs *nilfs = sb->s_fs_info; + char label[NILFS_MAX_VOLUME_NAME + 1]; + + BUILD_BUG_ON(NILFS_MAX_VOLUME_NAME >= FSLABEL_MAX); + + down_read(&nilfs->ns_sem); + memtostr_pad(label, nilfs->ns_sbp[0]->s_volume_name); + up_read(&nilfs->ns_sem); + + if (copy_to_user(argp, label, sizeof(label))) + return -EFAULT; + return 0; +} + +/** + * nilfs_ioctl_set_fslabel - set the volume name of the file system + * @sb: super block instance + * @filp: file object + * @argp: pointer to userspace memory that contains the volume name + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EFAULT - Error copying input data. + * * %-EINVAL - Label length exceeds record size in superblock. + * * %-EIO - I/O error. + * * %-EPERM - Operation not permitted (insufficient permissions). + * * %-EROFS - Read only file system. + */ +static int nilfs_ioctl_set_fslabel(struct super_block *sb, struct file *filp, + void __user *argp) +{ + char label[NILFS_MAX_VOLUME_NAME + 1]; + struct the_nilfs *nilfs = sb->s_fs_info; + struct nilfs_super_block **sbp; + size_t len; + int ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + ret = mnt_want_write_file(filp); + if (ret) + return ret; + + if (copy_from_user(label, argp, NILFS_MAX_VOLUME_NAME + 1)) { + ret = -EFAULT; + goto out_drop_write; + } + + len = strnlen(label, NILFS_MAX_VOLUME_NAME + 1); + if (len > NILFS_MAX_VOLUME_NAME) { + nilfs_err(sb, "unable to set label with more than %zu bytes", + NILFS_MAX_VOLUME_NAME); + ret = -EINVAL; + goto out_drop_write; + } + + down_write(&nilfs->ns_sem); + sbp = nilfs_prepare_super(sb, false); + if (unlikely(!sbp)) { + ret = -EIO; + goto out_unlock; + } + + strtomem_pad(sbp[0]->s_volume_name, label, 0); + if (sbp[1]) + strtomem_pad(sbp[1]->s_volume_name, label, 0); + + ret = nilfs_commit_super(sb, NILFS_SB_COMMIT_ALL); + +out_unlock: + up_write(&nilfs->ns_sem); +out_drop_write: + mnt_drop_write_file(filp); + return ret; +} + +long nilfs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + void __user *argp = (void __user *)arg; + + switch (cmd) { + case FS_IOC_GETVERSION: + return nilfs_ioctl_getversion(inode, argp); + case NILFS_IOCTL_CHANGE_CPMODE: + return nilfs_ioctl_change_cpmode(inode, filp, cmd, argp); + case NILFS_IOCTL_DELETE_CHECKPOINT: + return nilfs_ioctl_delete_checkpoint(inode, filp, cmd, argp); + case NILFS_IOCTL_GET_CPINFO: + return nilfs_ioctl_get_info(inode, filp, cmd, argp, + sizeof(struct nilfs_cpinfo), + nilfs_ioctl_do_get_cpinfo); + case NILFS_IOCTL_GET_CPSTAT: + return nilfs_ioctl_get_cpstat(inode, filp, cmd, argp); + case NILFS_IOCTL_GET_SUINFO: + return nilfs_ioctl_get_info(inode, filp, cmd, argp, + sizeof(struct nilfs_suinfo), + nilfs_ioctl_do_get_suinfo); + case NILFS_IOCTL_SET_SUINFO: + return nilfs_ioctl_set_suinfo(inode, filp, cmd, argp); + case NILFS_IOCTL_GET_SUSTAT: + return nilfs_ioctl_get_sustat(inode, filp, cmd, argp); + case NILFS_IOCTL_GET_VINFO: + return nilfs_ioctl_get_info(inode, filp, cmd, argp, + sizeof(struct nilfs_vinfo), + nilfs_ioctl_do_get_vinfo); + case NILFS_IOCTL_GET_BDESCS: + return nilfs_ioctl_get_bdescs(inode, filp, cmd, argp); + case NILFS_IOCTL_CLEAN_SEGMENTS: + return nilfs_ioctl_clean_segments(inode, filp, cmd, argp); + case NILFS_IOCTL_SYNC: + return nilfs_ioctl_sync(inode, filp, cmd, argp); + case NILFS_IOCTL_RESIZE: + return nilfs_ioctl_resize(inode, filp, argp); + case NILFS_IOCTL_SET_ALLOC_RANGE: + return nilfs_ioctl_set_alloc_range(inode, argp); + case FITRIM: + return nilfs_ioctl_trim_fs(inode, argp); + case FS_IOC_GETFSLABEL: + return nilfs_ioctl_get_fslabel(inode->i_sb, argp); + case FS_IOC_SETFSLABEL: + return nilfs_ioctl_set_fslabel(inode->i_sb, filp, argp); + default: + return -ENOTTY; + } +} + +#ifdef CONFIG_COMPAT +long nilfs_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + switch (cmd) { + case FS_IOC32_GETVERSION: + cmd = FS_IOC_GETVERSION; + break; + case NILFS_IOCTL_CHANGE_CPMODE: + case NILFS_IOCTL_DELETE_CHECKPOINT: + case NILFS_IOCTL_GET_CPINFO: + case NILFS_IOCTL_GET_CPSTAT: + case NILFS_IOCTL_GET_SUINFO: + case NILFS_IOCTL_SET_SUINFO: + case NILFS_IOCTL_GET_SUSTAT: + case NILFS_IOCTL_GET_VINFO: + case NILFS_IOCTL_GET_BDESCS: + case NILFS_IOCTL_CLEAN_SEGMENTS: + case NILFS_IOCTL_SYNC: + case NILFS_IOCTL_RESIZE: + case NILFS_IOCTL_SET_ALLOC_RANGE: + case FITRIM: + case FS_IOC_GETFSLABEL: + case FS_IOC_SETFSLABEL: + break; + default: + return -ENOIOCTLCMD; + } + return nilfs_ioctl(filp, cmd, (unsigned long)compat_ptr(arg)); +} +#endif diff --git a/fs/nilfs2/mdt.c b/fs/nilfs2/mdt.c new file mode 100644 index 000000000..2a435349f --- /dev/null +++ b/fs/nilfs2/mdt.c @@ -0,0 +1,678 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Meta data file for NILFS + * + * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Ryusuke Konishi. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "nilfs.h" +#include "btnode.h" +#include "segment.h" +#include "page.h" +#include "mdt.h" +#include "alloc.h" /* nilfs_palloc_destroy_cache() */ + +#include + +#define NILFS_MDT_MAX_RA_BLOCKS (16 - 1) + + +static int +nilfs_mdt_insert_new_block(struct inode *inode, unsigned long block, + struct buffer_head *bh, + void (*init_block)(struct inode *, + struct buffer_head *, void *)) +{ + struct nilfs_inode_info *ii = NILFS_I(inode); + struct folio *folio = bh->b_folio; + void *from; + int ret; + + /* Caller exclude read accesses using page lock */ + + /* set_buffer_new(bh); */ + bh->b_blocknr = 0; + + ret = nilfs_bmap_insert(ii->i_bmap, block, (unsigned long)bh); + if (unlikely(ret)) + return ret; + + set_buffer_mapped(bh); + + /* Initialize block (block size > PAGE_SIZE not yet supported) */ + from = kmap_local_folio(folio, offset_in_folio(folio, bh->b_data)); + memset(from, 0, bh->b_size); + if (init_block) + init_block(inode, bh, from); + kunmap_local(from); + + flush_dcache_folio(folio); + + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + nilfs_mdt_mark_dirty(inode); + + trace_nilfs2_mdt_insert_new_block(inode, inode->i_ino, block); + + return 0; +} + +static int nilfs_mdt_create_block(struct inode *inode, unsigned long block, + struct buffer_head **out_bh, + void (*init_block)(struct inode *, + struct buffer_head *, + void *)) +{ + struct super_block *sb = inode->i_sb; + struct nilfs_transaction_info ti; + struct buffer_head *bh; + int err; + + nilfs_transaction_begin(sb, &ti, 0); + + err = -ENOMEM; + bh = nilfs_grab_buffer(inode, inode->i_mapping, block, 0); + if (unlikely(!bh)) + goto failed_unlock; + + err = -EEXIST; + if (buffer_uptodate(bh)) + goto failed_bh; + + wait_on_buffer(bh); + if (buffer_uptodate(bh)) + goto failed_bh; + + err = nilfs_mdt_insert_new_block(inode, block, bh, init_block); + if (likely(!err)) { + get_bh(bh); + *out_bh = bh; + } + + failed_bh: + folio_unlock(bh->b_folio); + folio_put(bh->b_folio); + brelse(bh); + + failed_unlock: + if (likely(!err)) + err = nilfs_transaction_commit(sb); + else + nilfs_transaction_abort(sb); + + return err; +} + +static int +nilfs_mdt_submit_block(struct inode *inode, unsigned long blkoff, blk_opf_t opf, + struct buffer_head **out_bh) +{ + struct buffer_head *bh; + __u64 blknum = 0; + int ret = -ENOMEM; + + bh = nilfs_grab_buffer(inode, inode->i_mapping, blkoff, 0); + if (unlikely(!bh)) + goto failed; + + ret = -EEXIST; /* internal code */ + if (buffer_uptodate(bh)) + goto out; + + if (opf & REQ_RAHEAD) { + if (!trylock_buffer(bh)) { + ret = -EBUSY; + goto failed_bh; + } + } else /* opf == REQ_OP_READ */ + lock_buffer(bh); + + if (buffer_uptodate(bh)) { + unlock_buffer(bh); + goto out; + } + + ret = nilfs_bmap_lookup(NILFS_I(inode)->i_bmap, blkoff, &blknum); + if (unlikely(ret)) { + unlock_buffer(bh); + goto failed_bh; + } + map_bh(bh, inode->i_sb, (sector_t)blknum); + + bh_submit(bh, opf, bh_end_read); + ret = 0; + + trace_nilfs2_mdt_submit_block(inode, inode->i_ino, blkoff, + opf & REQ_OP_MASK); + out: + get_bh(bh); + *out_bh = bh; + + failed_bh: + folio_unlock(bh->b_folio); + folio_put(bh->b_folio); + brelse(bh); + failed: + return ret; +} + +static int nilfs_mdt_read_block(struct inode *inode, unsigned long block, + int readahead, struct buffer_head **out_bh) +{ + struct buffer_head *first_bh, *bh; + unsigned long blkoff; + int i, nr_ra_blocks = NILFS_MDT_MAX_RA_BLOCKS; + int err; + + err = nilfs_mdt_submit_block(inode, block, REQ_OP_READ, &first_bh); + if (err == -EEXIST) /* internal code */ + goto out; + + if (unlikely(err)) + goto failed; + + if (readahead) { + blkoff = block + 1; + for (i = 0; i < nr_ra_blocks; i++, blkoff++) { + err = nilfs_mdt_submit_block(inode, blkoff, + REQ_OP_READ | REQ_RAHEAD, &bh); + if (likely(!err || err == -EEXIST)) + brelse(bh); + else if (err != -EBUSY) + break; + /* abort readahead if bmap lookup failed */ + if (!buffer_locked(first_bh)) + goto out_no_wait; + } + } + + wait_on_buffer(first_bh); + + out_no_wait: + err = -EIO; + if (!buffer_uptodate(first_bh)) { + nilfs_err(inode->i_sb, + "I/O error reading meta-data file (ino=%llu, block-offset=%lu)", + inode->i_ino, block); + goto failed_bh; + } + out: + *out_bh = first_bh; + return 0; + + failed_bh: + brelse(first_bh); + failed: + return err; +} + +/** + * nilfs_mdt_get_block - read or create a buffer on meta data file. + * @inode: inode of the meta data file + * @blkoff: block offset + * @create: create flag + * @init_block: initializer used for newly allocated block + * @out_bh: output of a pointer to the buffer_head + * + * nilfs_mdt_get_block() looks up the specified buffer and tries to create + * a new buffer if @create is not zero. If (and only if) this function + * succeeds, it stores a pointer to the retrieved buffer head in the location + * pointed to by @out_bh. + * + * The retrieved buffer may be either an existing one or a newly allocated one. + * For a newly created buffer, if the callback function argument @init_block + * is non-NULL, the callback will be called with the buffer locked to format + * the block. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOENT - The specified block does not exist (hole block). + * * %-ENOMEM - Insufficient memory available. + * * %-EROFS - Read only filesystem (for create mode). + */ +int nilfs_mdt_get_block(struct inode *inode, unsigned long blkoff, int create, + void (*init_block)(struct inode *, + struct buffer_head *, void *), + struct buffer_head **out_bh) +{ + int ret; + + /* Should be rewritten with merging nilfs_mdt_read_block() */ + retry: + ret = nilfs_mdt_read_block(inode, blkoff, !create, out_bh); + if (!create || ret != -ENOENT) + return ret; + + ret = nilfs_mdt_create_block(inode, blkoff, out_bh, init_block); + if (unlikely(ret == -EEXIST)) { + /* create = 0; */ /* limit read-create loop retries */ + goto retry; + } + return ret; +} + +/** + * nilfs_mdt_find_block - find and get a buffer on meta data file. + * @inode: inode of the meta data file + * @start: start block offset (inclusive) + * @end: end block offset (inclusive) + * @blkoff: block offset + * @out_bh: place to store a pointer to buffer_head struct + * + * nilfs_mdt_find_block() looks up an existing block in range of + * [@start, @end] and stores pointer to a buffer head of the block to + * @out_bh, and block offset to @blkoff, respectively. @out_bh and + * @blkoff are substituted only when zero is returned. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOENT - No block was found in the range. + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_mdt_find_block(struct inode *inode, unsigned long start, + unsigned long end, unsigned long *blkoff, + struct buffer_head **out_bh) +{ + __u64 next; + int ret; + + if (unlikely(start > end)) + return -ENOENT; + + ret = nilfs_mdt_read_block(inode, start, true, out_bh); + if (!ret) { + *blkoff = start; + goto out; + } + if (unlikely(ret != -ENOENT || start == ULONG_MAX)) + goto out; + + ret = nilfs_bmap_seek_key(NILFS_I(inode)->i_bmap, start + 1, &next); + if (!ret) { + if (next <= end) { + ret = nilfs_mdt_read_block(inode, next, true, out_bh); + if (!ret) + *blkoff = next; + } else { + ret = -ENOENT; + } + } +out: + return ret; +} + +/** + * nilfs_mdt_delete_block - make a hole on the meta data file. + * @inode: inode of the meta data file + * @block: block offset + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOENT - Non-existent block. + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_mdt_delete_block(struct inode *inode, unsigned long block) +{ + struct nilfs_inode_info *ii = NILFS_I(inode); + int err; + + err = nilfs_bmap_delete(ii->i_bmap, block); + if (!err || err == -ENOENT) { + nilfs_mdt_mark_dirty(inode); + nilfs_mdt_forget_block(inode, block); + } + return err; +} + +/** + * nilfs_mdt_forget_block - discard dirty state and try to remove the page + * @inode: inode of the meta data file + * @block: block offset + * + * nilfs_mdt_forget_block() clears a dirty flag of the specified buffer, and + * tries to release the page including the buffer from a page cache. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EBUSY - Page has an active buffer. + * * %-ENOENT - Page cache has no page addressed by the offset. + */ +int nilfs_mdt_forget_block(struct inode *inode, unsigned long block) +{ + pgoff_t index = block >> (PAGE_SHIFT - inode->i_blkbits); + struct folio *folio; + struct buffer_head *bh; + int ret = 0; + int still_dirty; + + folio = filemap_lock_folio(inode->i_mapping, index); + if (IS_ERR(folio)) + return -ENOENT; + + folio_wait_writeback(folio); + + bh = folio_buffers(folio); + if (bh) { + unsigned long first_block = index << + (PAGE_SHIFT - inode->i_blkbits); + bh = get_nth_bh(bh, block - first_block); + nilfs_forget_buffer(bh); + } + still_dirty = folio_test_dirty(folio); + folio_unlock(folio); + folio_put(folio); + + if (still_dirty || + invalidate_inode_pages2_range(inode->i_mapping, index, index) != 0) + ret = -EBUSY; + return ret; +} + +int nilfs_mdt_fetch_dirty(struct inode *inode) +{ + struct nilfs_inode_info *ii = NILFS_I(inode); + + if (nilfs_bmap_test_and_clear_dirty(ii->i_bmap)) { + set_bit(NILFS_I_DIRTY, &ii->i_state); + return 1; + } + return test_bit(NILFS_I_DIRTY, &ii->i_state); +} + +static int nilfs_mdt_write_folio(struct folio *folio, + struct writeback_control *wbc) +{ + struct inode *inode = folio->mapping->host; + struct super_block *sb; + int err = 0; + + if (inode && sb_rdonly(inode->i_sb)) { + /* + * It means that filesystem was remounted in read-only + * mode because of error or metadata corruption. But we + * have dirty folios that try to be flushed in background. + * So, here we simply discard this dirty folio. + */ + nilfs_clear_folio_dirty(folio); + folio_unlock(folio); + return -EROFS; + } + + folio_redirty_for_writepage(wbc, folio); + folio_unlock(folio); + + if (!inode) + return 0; + + sb = inode->i_sb; + + if (wbc->sync_mode == WB_SYNC_ALL) + err = nilfs_construct_segment(sb); + + return err; +} + +static int nilfs_mdt_writeback(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct folio *folio = NULL; + int error; + + while ((folio = writeback_iter(mapping, wbc, folio, &error))) + error = nilfs_mdt_write_folio(folio, wbc); + + return error; +} + +static const struct address_space_operations def_mdt_aops = { + .dirty_folio = block_dirty_folio, + .invalidate_folio = block_invalidate_folio, + .writepages = nilfs_mdt_writeback, + .migrate_folio = buffer_migrate_folio_norefs, +}; + +static const struct inode_operations def_mdt_iops; +static const struct file_operations def_mdt_fops; + + +int nilfs_mdt_init(struct inode *inode, gfp_t gfp_mask, size_t objsz) +{ + struct nilfs_mdt_info *mi; + + mi = kzalloc(max(sizeof(*mi), objsz), GFP_NOFS); + if (!mi) + return -ENOMEM; + + init_rwsem(&mi->mi_sem); + inode->i_private = mi; + + inode->i_mode = S_IFREG; + mapping_set_gfp_mask(inode->i_mapping, gfp_mask); + + inode->i_op = &def_mdt_iops; + inode->i_fop = &def_mdt_fops; + inode->i_mapping->a_ops = &def_mdt_aops; + + return 0; +} + +/** + * nilfs_mdt_clear - do cleanup for the metadata file + * @inode: inode of the metadata file + */ +void nilfs_mdt_clear(struct inode *inode) +{ + struct nilfs_mdt_info *mdi = NILFS_MDT(inode); + struct nilfs_shadow_map *shadow = mdi->mi_shadow; + + if (mdi->mi_palloc_cache) + nilfs_palloc_destroy_cache(inode); + + if (shadow) { + struct inode *s_inode = shadow->inode; + + shadow->inode = NULL; + iput(s_inode); + mdi->mi_shadow = NULL; + } +} + +/** + * nilfs_mdt_destroy - release resources used by the metadata file + * @inode: inode of the metadata file + */ +void nilfs_mdt_destroy(struct inode *inode) +{ + struct nilfs_mdt_info *mdi = NILFS_MDT(inode); + + kfree(mdi->mi_bgl); /* kfree(NULL) is safe */ + kfree(mdi); +} + +void nilfs_mdt_set_entry_size(struct inode *inode, unsigned int entry_size, + unsigned int header_size) +{ + struct nilfs_mdt_info *mi = NILFS_MDT(inode); + + mi->mi_entry_size = entry_size; + mi->mi_entries_per_block = i_blocksize(inode) / entry_size; + mi->mi_first_entry_offset = DIV_ROUND_UP(header_size, entry_size); +} + +/** + * nilfs_mdt_setup_shadow_map - setup shadow map and bind it to metadata file + * @inode: inode of the metadata file + * @shadow: shadow mapping + * + * Return: 0 on success, or a negative error code on failure. + */ +int nilfs_mdt_setup_shadow_map(struct inode *inode, + struct nilfs_shadow_map *shadow) +{ + struct nilfs_mdt_info *mi = NILFS_MDT(inode); + struct inode *s_inode; + + INIT_LIST_HEAD(&shadow->frozen_buffers); + + s_inode = nilfs_iget_for_shadow(inode); + if (IS_ERR(s_inode)) + return PTR_ERR(s_inode); + + shadow->inode = s_inode; + mi->mi_shadow = shadow; + return 0; +} + +/** + * nilfs_mdt_save_to_shadow_map - copy bmap and dirty pages to shadow map + * @inode: inode of the metadata file + * + * Return: 0 on success, or a negative error code on failure. + */ +int nilfs_mdt_save_to_shadow_map(struct inode *inode) +{ + struct nilfs_mdt_info *mi = NILFS_MDT(inode); + struct nilfs_inode_info *ii = NILFS_I(inode); + struct nilfs_shadow_map *shadow = mi->mi_shadow; + struct inode *s_inode = shadow->inode; + int ret; + + ret = nilfs_copy_dirty_pages(s_inode->i_mapping, inode->i_mapping); + if (ret) + goto out; + + ret = nilfs_copy_dirty_pages(NILFS_I(s_inode)->i_assoc_inode->i_mapping, + ii->i_assoc_inode->i_mapping); + if (ret) + goto out; + + nilfs_bmap_save(ii->i_bmap, &shadow->bmap_store); + out: + return ret; +} + +int nilfs_mdt_freeze_buffer(struct inode *inode, struct buffer_head *bh) +{ + struct nilfs_shadow_map *shadow = NILFS_MDT(inode)->mi_shadow; + struct buffer_head *bh_frozen; + struct folio *folio; + int blkbits = inode->i_blkbits; + + folio = filemap_grab_folio(shadow->inode->i_mapping, + bh->b_folio->index); + if (IS_ERR(folio)) + return PTR_ERR(folio); + + bh_frozen = folio_buffers(folio); + if (!bh_frozen) + bh_frozen = create_empty_buffers(folio, 1 << blkbits, 0); + + bh_frozen = get_nth_bh(bh_frozen, + offset_in_folio(folio, bh->b_data) >> blkbits); + + if (!buffer_uptodate(bh_frozen)) + nilfs_copy_buffer(bh_frozen, bh); + if (list_empty(&bh_frozen->b_assoc_buffers)) { + list_add_tail(&bh_frozen->b_assoc_buffers, + &shadow->frozen_buffers); + set_buffer_nilfs_redirected(bh); + } else { + brelse(bh_frozen); /* already frozen */ + } + + folio_unlock(folio); + folio_put(folio); + return 0; +} + +struct buffer_head * +nilfs_mdt_get_frozen_buffer(struct inode *inode, struct buffer_head *bh) +{ + struct nilfs_shadow_map *shadow = NILFS_MDT(inode)->mi_shadow; + struct buffer_head *bh_frozen = NULL; + struct folio *folio; + int n; + + folio = filemap_lock_folio(shadow->inode->i_mapping, + bh->b_folio->index); + if (!IS_ERR(folio)) { + bh_frozen = folio_buffers(folio); + if (bh_frozen) { + n = offset_in_folio(folio, bh->b_data) >> + inode->i_blkbits; + bh_frozen = get_nth_bh(bh_frozen, n); + } + folio_unlock(folio); + folio_put(folio); + } + return bh_frozen; +} + +static void nilfs_release_frozen_buffers(struct nilfs_shadow_map *shadow) +{ + struct list_head *head = &shadow->frozen_buffers; + struct buffer_head *bh; + + while (!list_empty(head)) { + bh = list_first_entry(head, struct buffer_head, + b_assoc_buffers); + list_del_init(&bh->b_assoc_buffers); + brelse(bh); /* drop ref-count to make it releasable */ + } +} + +/** + * nilfs_mdt_restore_from_shadow_map - restore dirty pages and bmap state + * @inode: inode of the metadata file + */ +void nilfs_mdt_restore_from_shadow_map(struct inode *inode) +{ + struct nilfs_mdt_info *mi = NILFS_MDT(inode); + struct nilfs_inode_info *ii = NILFS_I(inode); + struct nilfs_shadow_map *shadow = mi->mi_shadow; + + down_write(&mi->mi_sem); + + if (mi->mi_palloc_cache) + nilfs_palloc_clear_cache(inode); + + nilfs_clear_dirty_pages(inode->i_mapping); + nilfs_copy_back_pages(inode->i_mapping, shadow->inode->i_mapping); + + nilfs_clear_dirty_pages(ii->i_assoc_inode->i_mapping); + nilfs_copy_back_pages(ii->i_assoc_inode->i_mapping, + NILFS_I(shadow->inode)->i_assoc_inode->i_mapping); + + nilfs_bmap_restore(ii->i_bmap, &shadow->bmap_store); + + up_write(&mi->mi_sem); +} + +/** + * nilfs_mdt_clear_shadow_map - truncate pages in shadow map caches + * @inode: inode of the metadata file + */ +void nilfs_mdt_clear_shadow_map(struct inode *inode) +{ + struct nilfs_mdt_info *mi = NILFS_MDT(inode); + struct nilfs_shadow_map *shadow = mi->mi_shadow; + struct inode *shadow_btnc_inode = NILFS_I(shadow->inode)->i_assoc_inode; + + down_write(&mi->mi_sem); + nilfs_release_frozen_buffers(shadow); + truncate_inode_pages(shadow->inode->i_mapping, 0); + truncate_inode_pages(shadow_btnc_inode->i_mapping, 0); + up_write(&mi->mi_sem); +} diff --git a/fs/nilfs2/mdt.h b/fs/nilfs2/mdt.h new file mode 100644 index 000000000..9e23bab3f --- /dev/null +++ b/fs/nilfs2/mdt.h @@ -0,0 +1,115 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * NILFS meta data file prototype and definitions + * + * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Ryusuke Konishi. + */ + +#ifndef _NILFS_MDT_H +#define _NILFS_MDT_H + +#include +#include +#include "nilfs.h" +#include "page.h" + +/** + * struct nilfs_shadow_map - shadow mapping of meta data file + * @bmap_store: shadow copy of bmap state + * @inode: holder of page caches used in shadow mapping + * @frozen_buffers: list of frozen buffers + */ +struct nilfs_shadow_map { + struct nilfs_bmap_store bmap_store; + struct inode *inode; + struct list_head frozen_buffers; +}; + +/** + * struct nilfs_mdt_info - on-memory private data of meta data files + * @mi_sem: reader/writer semaphore for meta data operations + * @mi_bgl: per-blockgroup locking + * @mi_entry_size: size of an entry + * @mi_first_entry_offset: offset to the first entry + * @mi_entries_per_block: number of entries in a block + * @mi_palloc_cache: persistent object allocator cache + * @mi_shadow: shadow of bmap and page caches + * @mi_blocks_per_group: number of blocks in a group + * @mi_blocks_per_desc_block: number of blocks per descriptor block + */ +struct nilfs_mdt_info { + struct rw_semaphore mi_sem; + struct blockgroup_lock *mi_bgl; + unsigned int mi_entry_size; + unsigned int mi_first_entry_offset; + unsigned long mi_entries_per_block; + struct nilfs_palloc_cache *mi_palloc_cache; + struct nilfs_shadow_map *mi_shadow; + unsigned long mi_blocks_per_group; + unsigned long mi_blocks_per_desc_block; +}; + +static inline struct nilfs_mdt_info *NILFS_MDT(const struct inode *inode) +{ + return inode->i_private; +} + +static inline int nilfs_is_metadata_file_inode(const struct inode *inode) +{ + return inode->i_private != NULL; +} + +/* Default GFP flags using highmem */ +#define NILFS_MDT_GFP (__GFP_RECLAIM | __GFP_IO | __GFP_HIGHMEM) + +int nilfs_mdt_get_block(struct inode *, unsigned long, int, + void (*init_block)(struct inode *, + struct buffer_head *, void *), + struct buffer_head **); +int nilfs_mdt_find_block(struct inode *inode, unsigned long start, + unsigned long end, unsigned long *blkoff, + struct buffer_head **out_bh); +int nilfs_mdt_delete_block(struct inode *, unsigned long); +int nilfs_mdt_forget_block(struct inode *, unsigned long); +int nilfs_mdt_fetch_dirty(struct inode *); + +int nilfs_mdt_init(struct inode *inode, gfp_t gfp_mask, size_t objsz); +void nilfs_mdt_clear(struct inode *inode); +void nilfs_mdt_destroy(struct inode *inode); + +void nilfs_mdt_set_entry_size(struct inode *, unsigned int, unsigned int); + +int nilfs_mdt_setup_shadow_map(struct inode *inode, + struct nilfs_shadow_map *shadow); +int nilfs_mdt_save_to_shadow_map(struct inode *inode); +void nilfs_mdt_restore_from_shadow_map(struct inode *inode); +void nilfs_mdt_clear_shadow_map(struct inode *inode); +int nilfs_mdt_freeze_buffer(struct inode *inode, struct buffer_head *bh); +struct buffer_head *nilfs_mdt_get_frozen_buffer(struct inode *inode, + struct buffer_head *bh); + +static inline void nilfs_mdt_mark_dirty(struct inode *inode) +{ + if (!test_bit(NILFS_I_DIRTY, &NILFS_I(inode)->i_state)) + set_bit(NILFS_I_DIRTY, &NILFS_I(inode)->i_state); +} + +static inline void nilfs_mdt_clear_dirty(struct inode *inode) +{ + clear_bit(NILFS_I_DIRTY, &NILFS_I(inode)->i_state); +} + +static inline __u64 nilfs_mdt_cno(struct inode *inode) +{ + return ((struct the_nilfs *)inode->i_sb->s_fs_info)->ns_cno; +} + +static inline spinlock_t * +nilfs_mdt_bgl_lock(struct inode *inode, unsigned int block_group) +{ + return bgl_lock_ptr(NILFS_MDT(inode)->mi_bgl, block_group); +} + +#endif /* _NILFS_MDT_H */ diff --git a/fs/nilfs2/namei.c b/fs/nilfs2/namei.c new file mode 100644 index 000000000..e037e0c6e --- /dev/null +++ b/fs/nilfs2/namei.c @@ -0,0 +1,594 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * NILFS pathname lookup operations. + * + * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation. + * + * Modified for NILFS by Amagai Yoshiji and Ryusuke Konishi. + */ +/* + * linux/fs/ext2/namei.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * from + * + * linux/fs/minix/namei.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * Big-endian to little-endian byte-swapping/bitmaps by + * David S. Miller (davem@caip.rutgers.edu), 1995 + */ + +#include +#include "nilfs.h" +#include "export.h" + +#define NILFS_FID_SIZE_NON_CONNECTABLE \ + (offsetof(struct nilfs_fid, parent_gen) / 4) +#define NILFS_FID_SIZE_CONNECTABLE (sizeof(struct nilfs_fid) / 4) + +static inline int nilfs_add_nondir(struct dentry *dentry, struct inode *inode) +{ + int err = nilfs_add_link(dentry, inode); + + if (!err) { + d_instantiate_new(dentry, inode); + return 0; + } + inode_dec_link_count(inode); + unlock_new_inode(inode); + iput(inode); + return err; +} + +/* + * Methods themselves. + */ + +static struct dentry * +nilfs_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) +{ + struct inode *inode; + u64 ino; + int res; + + if (dentry->d_name.len > NILFS_NAME_LEN) + return ERR_PTR(-ENAMETOOLONG); + + res = nilfs_inode_by_name(dir, &dentry->d_name, &ino); + if (res) { + if (res != -ENOENT) + return ERR_PTR(res); + inode = NULL; + } else { + inode = nilfs_iget(dir->i_sb, NILFS_I(dir)->i_root, ino); + if (inode == ERR_PTR(-ESTALE)) { + nilfs_error(dir->i_sb, + "deleted inode referenced: %llu", ino); + return ERR_PTR(-EIO); + } + } + + return d_splice_alias(inode, dentry); +} + +/* + * By the time this is called, we already have created + * the directory cache entry for the new file, but it + * is so far negative - it has no inode. + * + * If the create succeeds, we fill in the inode information + * with d_instantiate(). + */ +static int nilfs_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct inode *inode; + struct nilfs_transaction_info ti; + int err; + + err = nilfs_transaction_begin(dir->i_sb, &ti, 1); + if (err) + return err; + inode = nilfs_new_inode(dir, mode); + err = PTR_ERR(inode); + if (!IS_ERR(inode)) { + inode->i_op = &nilfs_file_inode_operations; + inode->i_fop = &nilfs_file_operations; + inode->i_mapping->a_ops = &nilfs_aops; + nilfs_mark_inode_dirty(inode); + err = nilfs_add_nondir(dentry, inode); + } + if (!err) + err = nilfs_transaction_commit(dir->i_sb); + else + nilfs_transaction_abort(dir->i_sb); + + return err; +} + +static int +nilfs_mknod(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) +{ + struct inode *inode; + struct nilfs_transaction_info ti; + int err; + + err = nilfs_transaction_begin(dir->i_sb, &ti, 1); + if (err) + return err; + inode = nilfs_new_inode(dir, mode); + err = PTR_ERR(inode); + if (!IS_ERR(inode)) { + init_special_inode(inode, inode->i_mode, rdev); + nilfs_mark_inode_dirty(inode); + err = nilfs_add_nondir(dentry, inode); + } + if (!err) + err = nilfs_transaction_commit(dir->i_sb); + else + nilfs_transaction_abort(dir->i_sb); + + return err; +} + +static int nilfs_symlink(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const char *symname) +{ + struct nilfs_transaction_info ti; + struct super_block *sb = dir->i_sb; + unsigned int l = strlen(symname) + 1; + struct inode *inode; + int err; + + if (l > sb->s_blocksize) + return -ENAMETOOLONG; + + err = nilfs_transaction_begin(dir->i_sb, &ti, 1); + if (err) + return err; + + inode = nilfs_new_inode(dir, S_IFLNK | 0777); + err = PTR_ERR(inode); + if (IS_ERR(inode)) + goto out; + + /* slow symlink */ + inode->i_op = &nilfs_symlink_inode_operations; + inode_nohighmem(inode); + mapping_set_gfp_mask(inode->i_mapping, + mapping_gfp_constraint(inode->i_mapping, + ~__GFP_FS)); + inode->i_mapping->a_ops = &nilfs_aops; + err = page_symlink(inode, symname, l); + if (err) + goto out_fail; + + /* mark_inode_dirty(inode); */ + /* page_symlink() do this */ + + err = nilfs_add_nondir(dentry, inode); +out: + if (!err) + err = nilfs_transaction_commit(dir->i_sb); + else + nilfs_transaction_abort(dir->i_sb); + + return err; + +out_fail: + drop_nlink(inode); + nilfs_mark_inode_dirty(inode); + unlock_new_inode(inode); + iput(inode); + goto out; +} + +static int nilfs_link(struct dentry *old_dentry, struct inode *dir, + struct dentry *dentry) +{ + struct inode *inode = d_inode(old_dentry); + struct nilfs_transaction_info ti; + int err; + + err = nilfs_transaction_begin(dir->i_sb, &ti, 1); + if (err) + return err; + + inode_set_ctime_current(inode); + inode_inc_link_count(inode); + ihold(inode); + + err = nilfs_add_link(dentry, inode); + if (!err) { + d_instantiate(dentry, inode); + err = nilfs_transaction_commit(dir->i_sb); + } else { + inode_dec_link_count(inode); + iput(inode); + nilfs_transaction_abort(dir->i_sb); + } + + return err; +} + +static struct dentry *nilfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct inode *inode; + struct nilfs_transaction_info ti; + int err; + + err = nilfs_transaction_begin(dir->i_sb, &ti, 1); + if (err) + return ERR_PTR(err); + + inc_nlink(dir); + + inode = nilfs_new_inode(dir, mode); + err = PTR_ERR(inode); + if (IS_ERR(inode)) + goto out_dir; + + inode->i_op = &nilfs_dir_inode_operations; + inode->i_fop = &nilfs_dir_operations; + inode->i_mapping->a_ops = &nilfs_aops; + + inc_nlink(inode); + + err = nilfs_make_empty(inode, dir); + if (err) + goto out_fail; + + err = nilfs_add_link(dentry, inode); + if (err) + goto out_fail; + + nilfs_mark_inode_dirty(inode); + d_instantiate_new(dentry, inode); +out: + if (!err) + err = nilfs_transaction_commit(dir->i_sb); + else + nilfs_transaction_abort(dir->i_sb); + + return err ? ERR_PTR(err) : NULL; + +out_fail: + drop_nlink(inode); + drop_nlink(inode); + nilfs_mark_inode_dirty(inode); + unlock_new_inode(inode); + iput(inode); +out_dir: + drop_nlink(dir); + nilfs_mark_inode_dirty(dir); + goto out; +} + +static int nilfs_do_unlink(struct inode *dir, struct dentry *dentry) +{ + struct inode *inode; + struct nilfs_dir_entry *de; + struct folio *folio; + int err; + + de = nilfs_find_entry(dir, &dentry->d_name, &folio); + if (IS_ERR(de)) { + err = PTR_ERR(de); + goto out; + } + + inode = d_inode(dentry); + err = -EIO; + if (le64_to_cpu(de->inode) != inode->i_ino) + goto out; + + if (!inode->i_nlink) { + nilfs_warn(inode->i_sb, + "deleting nonexistent file (ino=%llu), %d", + inode->i_ino, inode->i_nlink); + set_nlink(inode, 1); + } + err = nilfs_delete_entry(de, folio); + folio_release_kmap(folio, de); + if (err) + goto out; + + inode_set_ctime_to_ts(inode, inode_get_ctime(dir)); + drop_nlink(inode); + err = 0; +out: + return err; +} + +static int nilfs_unlink(struct inode *dir, struct dentry *dentry) +{ + struct nilfs_transaction_info ti; + int err; + + err = nilfs_transaction_begin(dir->i_sb, &ti, 0); + if (err) + return err; + + err = nilfs_do_unlink(dir, dentry); + + if (!err) { + nilfs_mark_inode_dirty(dir); + nilfs_mark_inode_dirty(d_inode(dentry)); + err = nilfs_transaction_commit(dir->i_sb); + } else + nilfs_transaction_abort(dir->i_sb); + + return err; +} + +static int nilfs_rmdir(struct inode *dir, struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + struct nilfs_transaction_info ti; + int err; + + err = nilfs_transaction_begin(dir->i_sb, &ti, 0); + if (err) + return err; + + err = -ENOTEMPTY; + if (nilfs_empty_dir(inode)) { + err = nilfs_do_unlink(dir, dentry); + if (!err) { + inode->i_size = 0; + drop_nlink(inode); + nilfs_mark_inode_dirty(inode); + drop_nlink(dir); + nilfs_mark_inode_dirty(dir); + } + } + if (!err) + err = nilfs_transaction_commit(dir->i_sb); + else + nilfs_transaction_abort(dir->i_sb); + + return err; +} + +static int nilfs_rename(struct mnt_idmap *idmap, + struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry, + unsigned int flags) +{ + struct inode *old_inode = d_inode(old_dentry); + struct inode *new_inode = d_inode(new_dentry); + struct folio *dir_folio = NULL; + struct nilfs_dir_entry *dir_de = NULL; + struct folio *old_folio; + struct nilfs_dir_entry *old_de; + struct nilfs_transaction_info ti; + bool old_is_dir = S_ISDIR(old_inode->i_mode); + int err; + + if (flags & ~RENAME_NOREPLACE) + return -EINVAL; + + err = nilfs_transaction_begin(old_dir->i_sb, &ti, 1); + if (unlikely(err)) + return err; + + old_de = nilfs_find_entry(old_dir, &old_dentry->d_name, &old_folio); + if (IS_ERR(old_de)) { + err = PTR_ERR(old_de); + goto out; + } + + if (old_is_dir && old_dir != new_dir) { + err = -EIO; + dir_de = nilfs_dotdot(old_inode, &dir_folio); + if (!dir_de) + goto out_old; + } + + if (new_inode) { + struct folio *new_folio; + struct nilfs_dir_entry *new_de; + + err = -ENOTEMPTY; + if (old_is_dir && !nilfs_empty_dir(new_inode)) + goto out_dir; + + new_de = nilfs_find_entry(new_dir, &new_dentry->d_name, + &new_folio); + if (IS_ERR(new_de)) { + err = PTR_ERR(new_de); + goto out_dir; + } + err = nilfs_set_link(new_dir, new_de, new_folio, old_inode); + folio_release_kmap(new_folio, new_de); + if (unlikely(err)) + goto out_dir; + nilfs_mark_inode_dirty(new_dir); + inode_set_ctime_current(new_inode); + if (old_is_dir) + drop_nlink(new_inode); + drop_nlink(new_inode); + nilfs_mark_inode_dirty(new_inode); + } else { + err = nilfs_add_link(new_dentry, old_inode); + if (err) + goto out_dir; + if (old_is_dir) { + inc_nlink(new_dir); + nilfs_mark_inode_dirty(new_dir); + } + } + + /* + * Like most other Unix systems, set the ctime for inodes on a + * rename. + */ + inode_set_ctime_current(old_inode); + + err = nilfs_delete_entry(old_de, old_folio); + if (likely(!err)) { + if (old_is_dir) { + if (old_dir != new_dir) + err = nilfs_set_link(old_inode, dir_de, + dir_folio, new_dir); + drop_nlink(old_dir); + } + nilfs_mark_inode_dirty(old_dir); + } + nilfs_mark_inode_dirty(old_inode); + +out_dir: + if (dir_de) + folio_release_kmap(dir_folio, dir_de); +out_old: + folio_release_kmap(old_folio, old_de); +out: + if (likely(!err)) + err = nilfs_transaction_commit(old_dir->i_sb); + else + nilfs_transaction_abort(old_dir->i_sb); + return err; +} + +/* + * Export operations + */ +static struct dentry *nilfs_get_parent(struct dentry *child) +{ + u64 ino; + int res; + struct nilfs_root *root; + + res = nilfs_inode_by_name(d_inode(child), &dotdot_name, &ino); + if (res) + return ERR_PTR(res); + + root = NILFS_I(d_inode(child))->i_root; + + return d_obtain_alias(nilfs_iget(child->d_sb, root, ino)); +} + +static struct dentry *nilfs_get_dentry(struct super_block *sb, u64 cno, + u64 ino, u32 gen) +{ + struct nilfs_root *root; + struct inode *inode; + + if (ino < NILFS_FIRST_INO(sb) && ino != NILFS_ROOT_INO) + return ERR_PTR(-ESTALE); + + root = nilfs_lookup_root(sb->s_fs_info, cno); + if (!root) + return ERR_PTR(-ESTALE); + + inode = nilfs_iget(sb, root, ino); + nilfs_put_root(root); + + if (IS_ERR(inode)) + return ERR_CAST(inode); + if (gen && inode->i_generation != gen) { + iput(inode); + return ERR_PTR(-ESTALE); + } + return d_obtain_alias(inode); +} + +static struct dentry *nilfs_fh_to_dentry(struct super_block *sb, struct fid *fh, + int fh_len, int fh_type) +{ + struct nilfs_fid *fid = (struct nilfs_fid *)fh; + + if (fh_len < NILFS_FID_SIZE_NON_CONNECTABLE || + (fh_type != FILEID_NILFS_WITH_PARENT && + fh_type != FILEID_NILFS_WITHOUT_PARENT)) + return NULL; + + return nilfs_get_dentry(sb, fid->cno, fid->ino, fid->gen); +} + +static struct dentry *nilfs_fh_to_parent(struct super_block *sb, struct fid *fh, + int fh_len, int fh_type) +{ + struct nilfs_fid *fid = (struct nilfs_fid *)fh; + + if (fh_len < NILFS_FID_SIZE_CONNECTABLE || + fh_type != FILEID_NILFS_WITH_PARENT) + return NULL; + + return nilfs_get_dentry(sb, fid->cno, fid->parent_ino, fid->parent_gen); +} + +static int nilfs_encode_fh(struct inode *inode, __u32 *fh, int *lenp, + struct inode *parent) +{ + struct nilfs_fid *fid = (struct nilfs_fid *)fh; + struct nilfs_root *root = NILFS_I(inode)->i_root; + int type; + + if (parent && *lenp < NILFS_FID_SIZE_CONNECTABLE) { + *lenp = NILFS_FID_SIZE_CONNECTABLE; + return FILEID_INVALID; + } + if (*lenp < NILFS_FID_SIZE_NON_CONNECTABLE) { + *lenp = NILFS_FID_SIZE_NON_CONNECTABLE; + return FILEID_INVALID; + } + + fid->cno = root->cno; + fid->ino = inode->i_ino; + fid->gen = inode->i_generation; + + if (parent) { + fid->parent_ino = parent->i_ino; + fid->parent_gen = parent->i_generation; + type = FILEID_NILFS_WITH_PARENT; + *lenp = NILFS_FID_SIZE_CONNECTABLE; + } else { + type = FILEID_NILFS_WITHOUT_PARENT; + *lenp = NILFS_FID_SIZE_NON_CONNECTABLE; + } + + return type; +} + +const struct inode_operations nilfs_dir_inode_operations = { + .create = nilfs_create, + .lookup = nilfs_lookup, + .link = nilfs_link, + .unlink = nilfs_unlink, + .symlink = nilfs_symlink, + .mkdir = nilfs_mkdir, + .rmdir = nilfs_rmdir, + .mknod = nilfs_mknod, + .rename = nilfs_rename, + .setattr = nilfs_setattr, + .permission = nilfs_permission, + .fiemap = nilfs_fiemap, + .fileattr_get = nilfs_fileattr_get, + .fileattr_set = nilfs_fileattr_set, +}; + +const struct inode_operations nilfs_special_inode_operations = { + .setattr = nilfs_setattr, + .permission = nilfs_permission, +}; + +const struct inode_operations nilfs_symlink_inode_operations = { + .get_link = page_get_link, + .permission = nilfs_permission, +}; + +const struct export_operations nilfs_export_ops = { + .encode_fh = nilfs_encode_fh, + .fh_to_dentry = nilfs_fh_to_dentry, + .fh_to_parent = nilfs_fh_to_parent, + .get_parent = nilfs_get_parent, +}; diff --git a/fs/nilfs2/nilfs.h b/fs/nilfs2/nilfs.h new file mode 100644 index 000000000..4fc42d378 --- /dev/null +++ b/fs/nilfs2/nilfs.h @@ -0,0 +1,415 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * NILFS local header file. + * + * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Koji Sato and Ryusuke Konishi. + */ + +#ifndef _NILFS_H +#define _NILFS_H + +#include +#include +#include +#include +#include +#include +#include +#include "the_nilfs.h" +#include "bmap.h" + +/** + * struct nilfs_inode_info - nilfs inode data in memory + * @i_flags: inode flags + * @i_type: inode type (combination of flags that inidicate usage) + * @i_state: dynamic state flags + * @i_bmap: pointer on i_bmap_data + * @i_bmap_data: raw block mapping + * @i_xattr: + * @i_dir_start_lookup: page index of last successful search + * @i_cno: checkpoint number for GC inode + * @i_assoc_inode: associated inode (B-tree node cache holder or back pointer) + * @i_dirty: list for connecting dirty files + * @xattr_sem: semaphore for extended attributes processing + * @i_bh: buffer contains disk inode + * @i_root: root object of the current filesystem tree + * @vfs_inode: VFS inode object + */ +struct nilfs_inode_info { + __u32 i_flags; + unsigned int i_type; + unsigned long i_state; /* Dynamic state flags */ + struct nilfs_bmap *i_bmap; + struct nilfs_bmap i_bmap_data; + __u64 i_xattr; /* sector_t ??? */ + __u32 i_dir_start_lookup; + __u64 i_cno; /* check point number for GC inode */ + struct inode *i_assoc_inode; + struct list_head i_dirty; /* List for connecting dirty files */ + +#ifdef CONFIG_NILFS_XATTR + /* + * Extended attributes can be read independently of the main file + * data. Taking i_sem even when reading would cause contention + * between readers of EAs and writers of regular file data, so + * instead we synchronize on xattr_sem when reading or changing + * EAs. + */ + struct rw_semaphore xattr_sem; +#endif + struct buffer_head *i_bh; /* + * i_bh contains a new or dirty + * disk inode. + */ + struct nilfs_root *i_root; + struct inode vfs_inode; +}; + +static inline struct nilfs_inode_info *NILFS_I(const struct inode *inode) +{ + return container_of(inode, struct nilfs_inode_info, vfs_inode); +} + +static inline struct nilfs_inode_info * +NILFS_BMAP_I(const struct nilfs_bmap *bmap) +{ + return container_of(bmap, struct nilfs_inode_info, i_bmap_data); +} + +/* + * Dynamic state flags of NILFS on-memory inode (i_state) + */ +enum { + NILFS_I_NEW = 0, /* Inode is newly created */ + NILFS_I_DIRTY, /* The file is dirty */ + NILFS_I_QUEUED, /* inode is in dirty_files list */ + NILFS_I_BUSY, /* + * Inode is grabbed by a segment + * constructor + */ + NILFS_I_COLLECTED, /* All dirty blocks are collected */ + NILFS_I_UPDATED, /* The file has been written back */ + NILFS_I_INODE_SYNC, /* dsync is not allowed for inode */ + NILFS_I_BMAP, /* has bmap and btnode_cache */ +}; + +/* + * Flags to identify the usage of on-memory inodes (i_type) + */ +enum { + NILFS_I_TYPE_NORMAL = 0, + NILFS_I_TYPE_GC = 0x0001, /* For data caching during GC */ + NILFS_I_TYPE_BTNC = 0x0002, /* For btree node cache */ + NILFS_I_TYPE_SHADOW = 0x0004, /* For shadowed page cache */ +}; + +/* + * commit flags for nilfs_commit_super and nilfs_sync_super + */ +enum { + NILFS_SB_COMMIT = 0, /* Commit a super block alternately */ + NILFS_SB_COMMIT_ALL /* Commit both super blocks */ +}; + +/** + * define NILFS_MAX_VOLUME_NAME - maximum number of characters (bytes) in a + * file system volume name + * + * Defined by the size of the volume name field in the on-disk superblocks. + * This volume name does not include the terminating NULL byte if the string + * length matches the field size, so use (NILFS_MAX_VOLUME_NAME + 1) for the + * size of the buffer that requires a NULL byte termination. + */ +#define NILFS_MAX_VOLUME_NAME \ + sizeof_field(struct nilfs_super_block, s_volume_name) + +/* + * Macros to check inode numbers + */ +#define NILFS_MDT_INO_BITS \ + (BIT(NILFS_DAT_INO) | BIT(NILFS_CPFILE_INO) | \ + BIT(NILFS_SUFILE_INO) | BIT(NILFS_IFILE_INO) | \ + BIT(NILFS_ATIME_INO) | BIT(NILFS_SKETCH_INO)) + +#define NILFS_SYS_INO_BITS (BIT(NILFS_ROOT_INO) | NILFS_MDT_INO_BITS) + +#define NILFS_FIRST_INO(sb) (((struct the_nilfs *)sb->s_fs_info)->ns_first_ino) + +#define NILFS_MDT_INODE(sb, ino) \ + ((ino) < NILFS_USER_INO && (NILFS_MDT_INO_BITS & BIT(ino))) +#define NILFS_VALID_INODE(sb, ino) \ + ((ino) >= NILFS_FIRST_INO(sb) || \ + ((ino) < NILFS_USER_INO && (NILFS_SYS_INO_BITS & BIT(ino)))) + +#define NILFS_PRIVATE_INODE(ino) ({ \ + u64 __ino = (ino); \ + ((__ino) < NILFS_USER_INO && (__ino) != NILFS_ROOT_INO && \ + (__ino) != NILFS_SKETCH_INO); }) + +/** + * struct nilfs_transaction_info: context information for synchronization + * @ti_magic: Magic number + * @ti_save: Backup of journal_info field of task_struct + * @ti_flags: Flags + * @ti_count: Nest level + */ +struct nilfs_transaction_info { + u32 ti_magic; + void *ti_save; + /* + * This should never be used. If it happens, + * one of other filesystems has a bug. + */ + unsigned short ti_flags; + unsigned short ti_count; +}; + +/* ti_magic */ +#define NILFS_TI_MAGIC 0xd9e392fb + +/* ti_flags */ +#define NILFS_TI_DYNAMIC_ALLOC 0x0001 /* Allocated from slab */ +#define NILFS_TI_SYNC 0x0002 /* + * Force to construct segment at the + * end of transaction. + */ +#define NILFS_TI_GC 0x0004 /* GC context */ +#define NILFS_TI_COMMIT 0x0008 /* Change happened or not */ +#define NILFS_TI_WRITER 0x0010 /* Constructor context */ + + +int nilfs_transaction_begin(struct super_block *, + struct nilfs_transaction_info *, int); +int nilfs_transaction_commit(struct super_block *); +void nilfs_transaction_abort(struct super_block *); + +static inline void nilfs_set_transaction_flag(unsigned int flag) +{ + struct nilfs_transaction_info *ti = current->journal_info; + + ti->ti_flags |= flag; +} + +static inline int nilfs_test_transaction_flag(unsigned int flag) +{ + struct nilfs_transaction_info *ti = current->journal_info; + + if (ti == NULL || ti->ti_magic != NILFS_TI_MAGIC) + return 0; + return !!(ti->ti_flags & flag); +} + +static inline int nilfs_doing_gc(void) +{ + return nilfs_test_transaction_flag(NILFS_TI_GC); +} + +static inline int nilfs_doing_construction(void) +{ + return nilfs_test_transaction_flag(NILFS_TI_WRITER); +} + +/* + * function prototype + */ +#ifdef CONFIG_NILFS_POSIX_ACL +#error "NILFS: not yet supported POSIX ACL" +extern int nilfs_acl_chmod(struct inode *); +extern int nilfs_init_acl(struct inode *, struct inode *); +#else +static inline int nilfs_acl_chmod(struct inode *inode) +{ + return 0; +} + +static inline int nilfs_init_acl(struct inode *inode, struct inode *dir) +{ + if (S_ISLNK(inode->i_mode)) + return 0; + + inode->i_mode &= ~current_umask(); + return 0; +} +#endif + +#define NILFS_ATIME_DISABLE + +/* Flags that should be inherited by new inodes from their parent. */ +#define NILFS_FL_INHERITED \ + (FS_SECRM_FL | FS_UNRM_FL | FS_COMPR_FL | FS_SYNC_FL | \ + FS_IMMUTABLE_FL | FS_APPEND_FL | FS_NODUMP_FL | FS_NOATIME_FL |\ + FS_COMPRBLK_FL | FS_NOCOMP_FL | FS_NOTAIL_FL | FS_DIRSYNC_FL) + +/* Mask out flags that are inappropriate for the given type of inode. */ +static inline __u32 nilfs_mask_flags(umode_t mode, __u32 flags) +{ + if (S_ISDIR(mode)) + return flags; + else if (S_ISREG(mode)) + return flags & ~(FS_DIRSYNC_FL | FS_TOPDIR_FL); + else + return flags & (FS_NODUMP_FL | FS_NOATIME_FL); +} + +/* dir.c */ +int nilfs_add_link(struct dentry *, struct inode *); +int nilfs_inode_by_name(struct inode *dir, const struct qstr *qstr, u64 *ino); +int nilfs_make_empty(struct inode *, struct inode *); +struct nilfs_dir_entry *nilfs_find_entry(struct inode *, const struct qstr *, + struct folio **); +int nilfs_delete_entry(struct nilfs_dir_entry *, struct folio *); +int nilfs_empty_dir(struct inode *); +struct nilfs_dir_entry *nilfs_dotdot(struct inode *, struct folio **); +int nilfs_set_link(struct inode *dir, struct nilfs_dir_entry *de, + struct folio *folio, struct inode *inode); + +/* file.c */ +extern int nilfs_sync_file(struct file *, loff_t, loff_t, int); + +/* ioctl.c */ +int nilfs_fileattr_get(struct dentry *dentry, struct file_kattr *m); +int nilfs_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa); +long nilfs_ioctl(struct file *, unsigned int, unsigned long); +long nilfs_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg); +int nilfs_ioctl_prepare_clean_segments(struct the_nilfs *, struct nilfs_argv *, + void **); + +/* inode.c */ +void nilfs_inode_add_blocks(struct inode *inode, int n); +void nilfs_inode_sub_blocks(struct inode *inode, int n); +extern struct inode *nilfs_new_inode(struct inode *, umode_t); +extern int nilfs_get_block(struct inode *, sector_t, struct buffer_head *, int); +extern void nilfs_set_inode_flags(struct inode *); +extern int nilfs_read_inode_common(struct inode *, struct nilfs_inode *); +void nilfs_write_inode_common(struct inode *inode, + struct nilfs_inode *raw_inode); +struct inode *nilfs_ilookup(struct super_block *sb, struct nilfs_root *root, + u64 ino); +struct inode *nilfs_iget_locked(struct super_block *sb, + struct nilfs_root *root, u64 ino); +struct inode *nilfs_iget(struct super_block *sb, struct nilfs_root *root, + u64 ino); +struct inode *nilfs_iget_for_gc(struct super_block *sb, u64 ino, __u64 cno); +int nilfs_attach_btree_node_cache(struct inode *inode); +void nilfs_detach_btree_node_cache(struct inode *inode); +struct inode *nilfs_iget_for_shadow(struct inode *inode); +extern void nilfs_update_inode(struct inode *, struct buffer_head *, int); +extern void nilfs_truncate(struct inode *); +extern void nilfs_evict_inode(struct inode *); +extern int nilfs_setattr(struct mnt_idmap *, struct dentry *, + struct iattr *); +extern void nilfs_write_failed(struct address_space *mapping, loff_t to); +int nilfs_permission(struct mnt_idmap *idmap, struct inode *inode, + int mask); +int nilfs_load_inode_block(struct inode *inode, struct buffer_head **pbh); +extern int nilfs_inode_dirty(struct inode *); +int nilfs_set_file_dirty(struct inode *inode, unsigned int nr_dirty); +extern int __nilfs_mark_inode_dirty(struct inode *, int); +extern void nilfs_dirty_inode(struct inode *, int flags); +int nilfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + __u64 start, __u64 len); +static inline int nilfs_mark_inode_dirty(struct inode *inode) +{ + return __nilfs_mark_inode_dirty(inode, I_DIRTY); +} +static inline int nilfs_mark_inode_dirty_sync(struct inode *inode) +{ + return __nilfs_mark_inode_dirty(inode, I_DIRTY_SYNC); +} + +/* super.c */ +extern struct inode *nilfs_alloc_inode(struct super_block *); + +__printf(2, 3) +void __nilfs_msg(struct super_block *sb, const char *fmt, ...); +extern __printf(3, 4) +void __nilfs_error(struct super_block *sb, const char *function, + const char *fmt, ...); + +#ifdef CONFIG_PRINTK + +#define nilfs_msg(sb, level, fmt, ...) \ + __nilfs_msg(sb, level fmt, ##__VA_ARGS__) +#define nilfs_error(sb, fmt, ...) \ + __nilfs_error(sb, __func__, fmt, ##__VA_ARGS__) + +#else + +#define nilfs_msg(sb, level, fmt, ...) \ + do { \ + no_printk(level fmt, ##__VA_ARGS__); \ + (void)(sb); \ + } while (0) +#define nilfs_error(sb, fmt, ...) \ + do { \ + no_printk(fmt, ##__VA_ARGS__); \ + __nilfs_error(sb, "", " "); \ + } while (0) + +#endif /* CONFIG_PRINTK */ + +#define nilfs_crit(sb, fmt, ...) \ + nilfs_msg(sb, KERN_CRIT, fmt, ##__VA_ARGS__) +#define nilfs_err(sb, fmt, ...) \ + nilfs_msg(sb, KERN_ERR, fmt, ##__VA_ARGS__) +#define nilfs_warn(sb, fmt, ...) \ + nilfs_msg(sb, KERN_WARNING, fmt, ##__VA_ARGS__) +#define nilfs_info(sb, fmt, ...) \ + nilfs_msg(sb, KERN_INFO, fmt, ##__VA_ARGS__) + +extern struct nilfs_super_block * +nilfs_read_super_block(struct super_block *, u64, int, struct buffer_head **); +extern int nilfs_store_magic(struct super_block *sb, + struct nilfs_super_block *sbp); +extern int nilfs_check_feature_compatibility(struct super_block *, + struct nilfs_super_block *); +extern void nilfs_set_log_cursor(struct nilfs_super_block *, + struct the_nilfs *); +struct nilfs_super_block **nilfs_prepare_super(struct super_block *sb, + int flip); +int nilfs_commit_super(struct super_block *sb, int flag); +int nilfs_cleanup_super(struct super_block *sb); +int nilfs_resize_fs(struct super_block *sb, __u64 newsize); +int nilfs_attach_checkpoint(struct super_block *sb, __u64 cno, int curr_mnt, + struct nilfs_root **root); +int nilfs_checkpoint_is_mounted(struct super_block *sb, __u64 cno); + +/* gcinode.c */ +int nilfs_gccache_submit_read_data(struct inode *, sector_t, sector_t, __u64, + struct buffer_head **); +int nilfs_gccache_submit_read_node(struct inode *, sector_t, __u64, + struct buffer_head **); +int nilfs_gccache_wait_and_mark_dirty(struct buffer_head *); +int nilfs_init_gcinode(struct inode *inode); +void nilfs_remove_all_gcinodes(struct the_nilfs *nilfs); + +/* sysfs.c */ +int __init nilfs_sysfs_init(void); +void nilfs_sysfs_exit(void); +int nilfs_sysfs_create_device_group(struct super_block *); +void nilfs_sysfs_delete_device_group(struct the_nilfs *); +int nilfs_sysfs_create_snapshot_group(struct nilfs_root *); +void nilfs_sysfs_delete_snapshot_group(struct nilfs_root *); + +/* + * Inodes and files operations + */ +extern const struct file_operations nilfs_dir_operations; +extern const struct inode_operations nilfs_file_inode_operations; +extern const struct file_operations nilfs_file_operations; +extern const struct address_space_operations nilfs_aops; +extern const struct address_space_operations nilfs_buffer_cache_aops; +extern const struct inode_operations nilfs_dir_inode_operations; +extern const struct inode_operations nilfs_special_inode_operations; +extern const struct inode_operations nilfs_symlink_inode_operations; + +/* + * filesystem type + */ +extern struct file_system_type nilfs_fs_type; + + +#endif /* _NILFS_H */ diff --git a/fs/nilfs2/page.c b/fs/nilfs2/page.c new file mode 100644 index 000000000..cf4f1c679 --- /dev/null +++ b/fs/nilfs2/page.c @@ -0,0 +1,579 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Buffer/page management specific to NILFS + * + * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Ryusuke Konishi and Seiji Kihara. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "nilfs.h" +#include "page.h" +#include "mdt.h" + + +#define NILFS_BUFFER_INHERENT_BITS \ + (BIT(BH_Uptodate) | BIT(BH_Mapped) | BIT(BH_NILFS_Node) | \ + BIT(BH_NILFS_Volatile) | BIT(BH_NILFS_Checked)) + +static struct buffer_head *__nilfs_get_folio_block(struct folio *folio, + unsigned long block, pgoff_t index, int blkbits, + unsigned long b_state) + +{ + unsigned long first_block; + struct buffer_head *bh = folio_buffers(folio); + + if (!bh) + bh = create_empty_buffers(folio, 1 << blkbits, b_state); + + first_block = (unsigned long)index << (PAGE_SHIFT - blkbits); + bh = get_nth_bh(bh, block - first_block); + + wait_on_buffer(bh); + return bh; +} + +struct buffer_head *nilfs_grab_buffer(struct inode *inode, + struct address_space *mapping, + unsigned long blkoff, + unsigned long b_state) +{ + int blkbits = inode->i_blkbits; + pgoff_t index = blkoff >> (PAGE_SHIFT - blkbits); + struct folio *folio; + struct buffer_head *bh; + + folio = filemap_grab_folio(mapping, index); + if (IS_ERR(folio)) + return NULL; + + bh = __nilfs_get_folio_block(folio, blkoff, index, blkbits, b_state); + if (unlikely(!bh)) { + folio_unlock(folio); + folio_put(folio); + return NULL; + } + bh->b_bdev = inode->i_sb->s_bdev; + return bh; +} + +/** + * nilfs_forget_buffer - discard dirty state + * @bh: buffer head of the buffer to be discarded + */ +void nilfs_forget_buffer(struct buffer_head *bh) +{ + struct folio *folio = bh->b_folio; + const unsigned long clear_bits = + (BIT(BH_Uptodate) | BIT(BH_Dirty) | BIT(BH_Mapped) | + BIT(BH_Async_Write) | BIT(BH_NILFS_Volatile) | + BIT(BH_NILFS_Checked) | BIT(BH_NILFS_Redirected) | + BIT(BH_Delay)); + + lock_buffer(bh); + set_mask_bits(&bh->b_state, clear_bits, 0); + if (nilfs_folio_buffers_clean(folio)) + __nilfs_clear_folio_dirty(folio); + + bh->b_blocknr = -1; + folio_clear_uptodate(folio); + folio_clear_mappedtodisk(folio); + unlock_buffer(bh); + brelse(bh); +} + +/** + * nilfs_copy_buffer -- copy buffer data and flags + * @dbh: destination buffer + * @sbh: source buffer + */ +void nilfs_copy_buffer(struct buffer_head *dbh, struct buffer_head *sbh) +{ + void *saddr, *daddr; + unsigned long bits; + struct folio *sfolio = sbh->b_folio, *dfolio = dbh->b_folio; + struct buffer_head *bh; + + saddr = kmap_local_folio(sfolio, bh_offset(sbh)); + daddr = kmap_local_folio(dfolio, bh_offset(dbh)); + memcpy(daddr, saddr, sbh->b_size); + kunmap_local(daddr); + kunmap_local(saddr); + + dbh->b_state = sbh->b_state & NILFS_BUFFER_INHERENT_BITS; + dbh->b_blocknr = sbh->b_blocknr; + dbh->b_bdev = sbh->b_bdev; + + bh = dbh; + bits = sbh->b_state & (BIT(BH_Uptodate) | BIT(BH_Mapped)); + while ((bh = bh->b_this_page) != dbh) { + lock_buffer(bh); + bits &= bh->b_state; + unlock_buffer(bh); + } + if (bits & BIT(BH_Uptodate)) + folio_mark_uptodate(dfolio); + else + folio_clear_uptodate(dfolio); + if (bits & BIT(BH_Mapped)) + folio_set_mappedtodisk(dfolio); + else + folio_clear_mappedtodisk(dfolio); +} + +/** + * nilfs_folio_buffers_clean - Check if a folio has dirty buffers or not. + * @folio: Folio to be checked. + * + * Return: false if the folio has dirty buffers, true otherwise. + */ +bool nilfs_folio_buffers_clean(struct folio *folio) +{ + struct buffer_head *bh, *head; + + bh = head = folio_buffers(folio); + do { + if (buffer_dirty(bh)) + return false; + bh = bh->b_this_page; + } while (bh != head); + return true; +} + +void nilfs_folio_bug(struct folio *folio) +{ + struct buffer_head *bh, *head; + struct address_space *m; + u64 ino; + + if (unlikely(!folio)) { + printk(KERN_CRIT "NILFS_FOLIO_BUG(NULL)\n"); + return; + } + + m = folio->mapping; + ino = m ? m->host->i_ino : 0; + + printk(KERN_CRIT "NILFS_FOLIO_BUG(%p): cnt=%d index#=%lu flags=0x%lx mapping=%p ino=%llu\n", + folio, folio_ref_count(folio), folio->index, folio->flags.f, + m, ino); + + head = folio_buffers(folio); + if (head) { + int i = 0; + + bh = head; + do { + printk(KERN_CRIT + " BH[%d] %p: cnt=%d block#=%llu state=0x%lx\n", + i++, bh, atomic_read(&bh->b_count), + (unsigned long long)bh->b_blocknr, bh->b_state); + bh = bh->b_this_page; + } while (bh != head); + } +} + +/** + * nilfs_copy_folio -- copy the folio with buffers + * @dst: destination folio + * @src: source folio + * @copy_dirty: flag whether to copy dirty states on the folio's buffer heads. + * + * This function is for both data folios and btnode folios. The dirty flag + * should be treated by caller. The folio must not be under i/o. + * Both src and dst folio must be locked + */ +static void nilfs_copy_folio(struct folio *dst, struct folio *src, + bool copy_dirty) +{ + struct buffer_head *dbh, *dbufs, *sbh; + unsigned long mask = NILFS_BUFFER_INHERENT_BITS; + + BUG_ON(folio_test_writeback(dst)); + + sbh = folio_buffers(src); + dbh = folio_buffers(dst); + if (!dbh) + dbh = create_empty_buffers(dst, sbh->b_size, 0); + + if (copy_dirty) + mask |= BIT(BH_Dirty); + + dbufs = dbh; + do { + lock_buffer(sbh); + lock_buffer(dbh); + dbh->b_state = sbh->b_state & mask; + dbh->b_blocknr = sbh->b_blocknr; + dbh->b_bdev = sbh->b_bdev; + sbh = sbh->b_this_page; + dbh = dbh->b_this_page; + } while (dbh != dbufs); + + folio_copy(dst, src); + + if (folio_test_uptodate(src) && !folio_test_uptodate(dst)) + folio_mark_uptodate(dst); + else if (!folio_test_uptodate(src) && folio_test_uptodate(dst)) + folio_clear_uptodate(dst); + if (folio_test_mappedtodisk(src) && !folio_test_mappedtodisk(dst)) + folio_set_mappedtodisk(dst); + else if (!folio_test_mappedtodisk(src) && folio_test_mappedtodisk(dst)) + folio_clear_mappedtodisk(dst); + + do { + unlock_buffer(sbh); + unlock_buffer(dbh); + sbh = sbh->b_this_page; + dbh = dbh->b_this_page; + } while (dbh != dbufs); +} + +int nilfs_copy_dirty_pages(struct address_space *dmap, + struct address_space *smap) +{ + struct inode *smap_inode = smap->host; + struct folio_batch fbatch; + unsigned int i; + pgoff_t index = 0; + int err = 0; + + folio_batch_init(&fbatch); +repeat: + if (!filemap_get_folios_tag(smap, &index, (pgoff_t)-1, + PAGECACHE_TAG_DIRTY, &fbatch)) + return 0; + + for (i = 0; i < folio_batch_count(&fbatch); i++) { + struct folio *folio = fbatch.folios[i], *dfolio; + + folio_lock(folio); + if (unlikely(!folio_test_dirty(folio))) { + if (WARN_ONCE(!sb_rdonly(smap_inode->i_sb), + "inconsistent dirty state\n")) + goto unlock_folio; + + /* + * If the filesystem has been forced to read-only + * due to metadata corruption. + */ + folio_unlock(folio); + err = -EROFS; + break; + } + + dfolio = filemap_grab_folio(dmap, folio->index); + if (IS_ERR(dfolio)) { + /* No empty page is added to the page cache */ + folio_unlock(folio); + err = PTR_ERR(dfolio); + break; + } + if (unlikely(!folio_buffers(folio))) + NILFS_FOLIO_BUG(folio, + "found empty page in dat page cache"); + + nilfs_copy_folio(dfolio, folio, true); + filemap_dirty_folio(folio_mapping(dfolio), dfolio); + + folio_unlock(dfolio); + folio_put(dfolio); +unlock_folio: + folio_unlock(folio); + } + folio_batch_release(&fbatch); + cond_resched(); + + if (likely(!err)) + goto repeat; + return err; +} + +/** + * nilfs_copy_back_pages -- copy back pages to original cache from shadow cache + * @dmap: destination page cache + * @smap: source page cache + * + * No pages must be added to the cache during this process. + * This must be ensured by the caller. + */ +void nilfs_copy_back_pages(struct address_space *dmap, + struct address_space *smap) +{ + struct folio_batch fbatch; + unsigned int i, n; + pgoff_t start = 0; + + folio_batch_init(&fbatch); +repeat: + n = filemap_get_folios(smap, &start, ~0UL, &fbatch); + if (!n) + return; + + for (i = 0; i < folio_batch_count(&fbatch); i++) { + struct folio *folio = fbatch.folios[i], *dfolio; + pgoff_t index = folio->index; + + folio_lock(folio); + dfolio = filemap_lock_folio(dmap, index); + if (!IS_ERR(dfolio)) { + /* overwrite existing folio in the destination cache */ + WARN_ON(folio_test_dirty(dfolio)); + nilfs_copy_folio(dfolio, folio, false); + folio_unlock(dfolio); + folio_put(dfolio); + /* Do we not need to remove folio from smap here? */ + } else { + struct folio *f; + + /* move the folio to the destination cache */ + xa_lock_irq(&smap->i_pages); + f = __xa_erase(&smap->i_pages, index); + WARN_ON(folio != f); + smap->nrpages--; + xa_unlock_irq(&smap->i_pages); + + xa_lock_irq(&dmap->i_pages); + f = __xa_store(&dmap->i_pages, index, folio, GFP_NOFS); + if (unlikely(f)) { + /* Probably -ENOMEM */ + folio->mapping = NULL; + folio_put(folio); + } else { + folio->mapping = dmap; + dmap->nrpages++; + if (folio_test_dirty(folio)) + __xa_set_mark(&dmap->i_pages, index, + PAGECACHE_TAG_DIRTY); + } + xa_unlock_irq(&dmap->i_pages); + } + folio_unlock(folio); + } + folio_batch_release(&fbatch); + cond_resched(); + + goto repeat; +} + +/** + * nilfs_clear_dirty_pages - discard dirty pages in address space + * @mapping: address space with dirty pages for discarding + */ +void nilfs_clear_dirty_pages(struct address_space *mapping) +{ + struct folio_batch fbatch; + unsigned int i; + pgoff_t index = 0; + + folio_batch_init(&fbatch); + + while (filemap_get_folios_tag(mapping, &index, (pgoff_t)-1, + PAGECACHE_TAG_DIRTY, &fbatch)) { + for (i = 0; i < folio_batch_count(&fbatch); i++) { + struct folio *folio = fbatch.folios[i]; + + folio_lock(folio); + + /* + * This folio may have been removed from the address + * space by truncation or invalidation when the lock + * was acquired. Skip processing in that case. + */ + if (likely(folio->mapping == mapping)) + nilfs_clear_folio_dirty(folio); + + folio_unlock(folio); + } + folio_batch_release(&fbatch); + cond_resched(); + } +} + +/** + * nilfs_clear_folio_dirty - discard dirty folio + * @folio: dirty folio that will be discarded + * + * nilfs_clear_folio_dirty() clears working states including dirty state for + * the folio and its buffers. If the folio has buffers, clear only if it is + * confirmed that none of the buffer heads are busy (none have valid + * references and none are locked). + */ +void nilfs_clear_folio_dirty(struct folio *folio) +{ + struct buffer_head *bh, *head; + + BUG_ON(!folio_test_locked(folio)); + + head = folio_buffers(folio); + if (head) { + const unsigned long clear_bits = + (BIT(BH_Uptodate) | BIT(BH_Dirty) | BIT(BH_Mapped) | + BIT(BH_Async_Write) | BIT(BH_NILFS_Volatile) | + BIT(BH_NILFS_Checked) | BIT(BH_NILFS_Redirected) | + BIT(BH_Delay)); + bool busy, invalidated = false; + +recheck_buffers: + busy = false; + bh = head; + do { + if (atomic_read(&bh->b_count) | buffer_locked(bh)) { + busy = true; + break; + } + } while (bh = bh->b_this_page, bh != head); + + if (busy) { + if (invalidated) + return; + invalidate_bh_lrus(); + invalidated = true; + goto recheck_buffers; + } + + bh = head; + do { + lock_buffer(bh); + set_mask_bits(&bh->b_state, clear_bits, 0); + unlock_buffer(bh); + } while (bh = bh->b_this_page, bh != head); + } + + folio_clear_uptodate(folio); + folio_clear_mappedtodisk(folio); + folio_clear_checked(folio); + __nilfs_clear_folio_dirty(folio); +} + +unsigned int nilfs_page_count_clean_buffers(struct folio *folio, + unsigned int from, unsigned int to) +{ + unsigned int block_start, block_end; + struct buffer_head *bh, *head; + unsigned int nc = 0; + + for (bh = head = folio_buffers(folio), block_start = 0; + bh != head || !block_start; + block_start = block_end, bh = bh->b_this_page) { + block_end = block_start + bh->b_size; + if (block_end > from && block_start < to && !buffer_dirty(bh)) + nc++; + } + return nc; +} + +/* + * NILFS2 needs clear_page_dirty() in the following two cases: + * + * 1) For B-tree node pages and data pages of DAT file, NILFS2 clears dirty + * flag of pages when it copies back pages from shadow cache to the + * original cache. + * + * 2) Some B-tree operations like insertion or deletion may dispose buffers + * in dirty state, and this needs to cancel the dirty state of their pages. + */ +void __nilfs_clear_folio_dirty(struct folio *folio) +{ + struct address_space *mapping = folio->mapping; + + if (mapping) { + xa_lock_irq(&mapping->i_pages); + if (folio_test_dirty(folio)) { + __xa_clear_mark(&mapping->i_pages, folio->index, + PAGECACHE_TAG_DIRTY); + xa_unlock_irq(&mapping->i_pages); + folio_clear_dirty_for_io(folio); + return; + } + xa_unlock_irq(&mapping->i_pages); + return; + } + folio_clear_dirty(folio); +} + +/** + * nilfs_find_uncommitted_extent - find extent of uncommitted data + * @inode: inode + * @start_blk: start block offset (in) + * @blkoff: start offset of the found extent (out) + * + * This function searches an extent of buffers marked "delayed" which + * starts from a block offset equal to or larger than @start_blk. If + * such an extent was found, this will store the start offset in + * @blkoff and return its length in blocks. + * + * Return: Length in blocks of found extent, 0 otherwise. + */ +unsigned long nilfs_find_uncommitted_extent(struct inode *inode, + sector_t start_blk, + sector_t *blkoff) +{ + unsigned int i, nr_folios; + pgoff_t index; + unsigned long length = 0; + struct folio_batch fbatch; + struct folio *folio; + + if (inode->i_mapping->nrpages == 0) + return 0; + + index = start_blk >> (PAGE_SHIFT - inode->i_blkbits); + + folio_batch_init(&fbatch); + +repeat: + nr_folios = filemap_get_folios_contig(inode->i_mapping, &index, ULONG_MAX, + &fbatch); + if (nr_folios == 0) + return length; + + i = 0; + do { + folio = fbatch.folios[i]; + + folio_lock(folio); + if (folio_buffers(folio)) { + struct buffer_head *bh, *head; + sector_t b; + + b = folio->index << (PAGE_SHIFT - inode->i_blkbits); + bh = head = folio_buffers(folio); + do { + if (b < start_blk) + continue; + if (buffer_delay(bh)) { + if (length == 0) + *blkoff = b; + length++; + } else if (length > 0) { + goto out_locked; + } + } while (++b, bh = bh->b_this_page, bh != head); + } else { + if (length > 0) + goto out_locked; + } + folio_unlock(folio); + + } while (++i < nr_folios); + + folio_batch_release(&fbatch); + cond_resched(); + goto repeat; + +out_locked: + folio_unlock(folio); + folio_batch_release(&fbatch); + return length; +} diff --git a/fs/nilfs2/page.h b/fs/nilfs2/page.h new file mode 100644 index 000000000..136cd1c14 --- /dev/null +++ b/fs/nilfs2/page.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Buffer/page management specific to NILFS + * + * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Ryusuke Konishi and Seiji Kihara. + */ + +#ifndef _NILFS_PAGE_H +#define _NILFS_PAGE_H + +#include +#include "nilfs.h" + +/* + * Extended buffer state bits + */ +enum { + BH_NILFS_Allocated = BH_PrivateStart, + BH_NILFS_Node, + BH_NILFS_Volatile, + BH_NILFS_Checked, + BH_NILFS_Redirected, +}; + +BUFFER_FNS(NILFS_Node, nilfs_node) /* nilfs node buffers */ +BUFFER_FNS(NILFS_Volatile, nilfs_volatile) +BUFFER_FNS(NILFS_Checked, nilfs_checked) /* buffer is verified */ +BUFFER_FNS(NILFS_Redirected, nilfs_redirected) /* redirected to a copy */ + + +void __nilfs_clear_folio_dirty(struct folio *); + +struct buffer_head *nilfs_grab_buffer(struct inode *, struct address_space *, + unsigned long, unsigned long); +void nilfs_forget_buffer(struct buffer_head *); +void nilfs_copy_buffer(struct buffer_head *, struct buffer_head *); +bool nilfs_folio_buffers_clean(struct folio *); +void nilfs_folio_bug(struct folio *); + +int nilfs_copy_dirty_pages(struct address_space *, struct address_space *); +void nilfs_copy_back_pages(struct address_space *, struct address_space *); +void nilfs_clear_folio_dirty(struct folio *folio); +void nilfs_clear_dirty_pages(struct address_space *mapping); +unsigned int nilfs_page_count_clean_buffers(struct folio *folio, + unsigned int from, unsigned int to); +unsigned long nilfs_find_uncommitted_extent(struct inode *inode, + sector_t start_blk, + sector_t *blkoff); + +#define NILFS_FOLIO_BUG(folio, m, a...) \ + do { nilfs_folio_bug(folio); BUG(); } while (0) + +#endif /* _NILFS_PAGE_H */ diff --git a/fs/nilfs2/recovery.c b/fs/nilfs2/recovery.c new file mode 100644 index 000000000..45fb37215 --- /dev/null +++ b/fs/nilfs2/recovery.c @@ -0,0 +1,1016 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * NILFS recovery logic + * + * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Ryusuke Konishi. + */ + +#include +#include +#include +#include +#include +#include "nilfs.h" +#include "segment.h" +#include "sufile.h" +#include "page.h" +#include "segbuf.h" + +/* + * Segment check result + */ +enum { + NILFS_SEG_VALID, + NILFS_SEG_NO_SUPER_ROOT, + NILFS_SEG_FAIL_IO, + NILFS_SEG_FAIL_MAGIC, + NILFS_SEG_FAIL_SEQ, + NILFS_SEG_FAIL_CHECKSUM_SUPER_ROOT, + NILFS_SEG_FAIL_CHECKSUM_FULL, + NILFS_SEG_FAIL_CONSISTENCY, +}; + +/* work structure for recovery */ +struct nilfs_recovery_block { + u64 ino; /* + * Inode number of the file that this block + * belongs to + */ + sector_t blocknr; /* block number */ + __u64 vblocknr; /* virtual block number */ + unsigned long blkoff; /* File offset of the data block (per block) */ + struct list_head list; +}; + + +static int nilfs_warn_segment_error(struct super_block *sb, int err) +{ + const char *msg = NULL; + + switch (err) { + case NILFS_SEG_FAIL_IO: + nilfs_err(sb, "I/O error reading segment"); + return -EIO; + case NILFS_SEG_FAIL_MAGIC: + msg = "Magic number mismatch"; + break; + case NILFS_SEG_FAIL_SEQ: + msg = "Sequence number mismatch"; + break; + case NILFS_SEG_FAIL_CHECKSUM_SUPER_ROOT: + msg = "Checksum error in super root"; + break; + case NILFS_SEG_FAIL_CHECKSUM_FULL: + msg = "Checksum error in segment payload"; + break; + case NILFS_SEG_FAIL_CONSISTENCY: + msg = "Inconsistency found"; + break; + case NILFS_SEG_NO_SUPER_ROOT: + msg = "No super root in the last segment"; + break; + default: + nilfs_err(sb, "unrecognized segment error %d", err); + return -EINVAL; + } + nilfs_warn(sb, "invalid segment: %s", msg); + return -EINVAL; +} + +/** + * nilfs_compute_checksum - compute checksum of blocks continuously + * @nilfs: nilfs object + * @bhs: buffer head of start block + * @sum: place to store result + * @offset: offset bytes in the first block + * @check_bytes: number of bytes to be checked + * @start: DBN of start block + * @nblock: number of blocks to be checked + * + * Return: 0 on success, or %-EIO if an I/O error occurs. + */ +static int nilfs_compute_checksum(struct the_nilfs *nilfs, + struct buffer_head *bhs, u32 *sum, + unsigned long offset, u64 check_bytes, + sector_t start, unsigned long nblock) +{ + unsigned int blocksize = nilfs->ns_blocksize; + unsigned long size; + u32 crc; + + BUG_ON(offset >= blocksize); + check_bytes -= offset; + size = min_t(u64, check_bytes, blocksize - offset); + crc = crc32_le(nilfs->ns_crc_seed, + (unsigned char *)bhs->b_data + offset, size); + if (--nblock > 0) { + do { + struct buffer_head *bh; + + bh = __bread(nilfs->ns_bdev, ++start, blocksize); + if (!bh) + return -EIO; + check_bytes -= size; + size = min_t(u64, check_bytes, blocksize); + crc = crc32_le(crc, bh->b_data, size); + brelse(bh); + } while (--nblock > 0); + } + *sum = crc; + return 0; +} + +/** + * nilfs_read_super_root_block - read super root block + * @nilfs: nilfs object + * @sr_block: disk block number of the super root block + * @pbh: address of a buffer_head pointer to return super root buffer + * @check: CRC check flag + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EINVAL - Super root block corrupted. + * * %-EIO - I/O error. + */ +int nilfs_read_super_root_block(struct the_nilfs *nilfs, sector_t sr_block, + struct buffer_head **pbh, int check) +{ + struct buffer_head *bh_sr; + struct nilfs_super_root *sr; + u32 crc; + int ret; + + *pbh = NULL; + bh_sr = __bread(nilfs->ns_bdev, sr_block, nilfs->ns_blocksize); + if (unlikely(!bh_sr)) { + ret = NILFS_SEG_FAIL_IO; + goto failed; + } + + sr = (struct nilfs_super_root *)bh_sr->b_data; + if (check) { + unsigned int bytes = le16_to_cpu(sr->sr_bytes); + + if (bytes == 0 || bytes > nilfs->ns_blocksize) { + ret = NILFS_SEG_FAIL_CHECKSUM_SUPER_ROOT; + goto failed_bh; + } + if (nilfs_compute_checksum( + nilfs, bh_sr, &crc, sizeof(sr->sr_sum), bytes, + sr_block, 1)) { + ret = NILFS_SEG_FAIL_IO; + goto failed_bh; + } + if (crc != le32_to_cpu(sr->sr_sum)) { + ret = NILFS_SEG_FAIL_CHECKSUM_SUPER_ROOT; + goto failed_bh; + } + } + *pbh = bh_sr; + return 0; + + failed_bh: + brelse(bh_sr); + + failed: + return nilfs_warn_segment_error(nilfs->ns_sb, ret); +} + +/** + * nilfs_read_log_header - read summary header of the specified log + * @nilfs: nilfs object + * @start_blocknr: start block number of the log + * @sum: pointer to return segment summary structure + * + * Return: Buffer head pointer, or NULL if an I/O error occurs. + */ +static struct buffer_head * +nilfs_read_log_header(struct the_nilfs *nilfs, sector_t start_blocknr, + struct nilfs_segment_summary **sum) +{ + struct buffer_head *bh_sum; + + bh_sum = __bread(nilfs->ns_bdev, start_blocknr, nilfs->ns_blocksize); + if (bh_sum) + *sum = (struct nilfs_segment_summary *)bh_sum->b_data; + return bh_sum; +} + +/** + * nilfs_validate_log - verify consistency of log + * @nilfs: nilfs object + * @seg_seq: sequence number of segment + * @bh_sum: buffer head of summary block + * @sum: segment summary struct + * + * Return: 0 on success, or one of the following internal codes on failure: + * * %NILFS_SEG_FAIL_MAGIC - Magic number mismatch. + * * %NILFS_SEG_FAIL_SEQ - Sequence number mismatch. + * * %NIFLS_SEG_FAIL_CONSISTENCY - Block count out of range. + * * %NILFS_SEG_FAIL_IO - I/O error. + * * %NILFS_SEG_FAIL_CHECKSUM_FULL - Full log checksum verification failed. + */ +static int nilfs_validate_log(struct the_nilfs *nilfs, u64 seg_seq, + struct buffer_head *bh_sum, + struct nilfs_segment_summary *sum) +{ + unsigned long nblock; + u32 crc; + int ret; + + ret = NILFS_SEG_FAIL_MAGIC; + if (le32_to_cpu(sum->ss_magic) != NILFS_SEGSUM_MAGIC) + goto out; + + ret = NILFS_SEG_FAIL_SEQ; + if (le64_to_cpu(sum->ss_seq) != seg_seq) + goto out; + + nblock = le32_to_cpu(sum->ss_nblocks); + ret = NILFS_SEG_FAIL_CONSISTENCY; + if (unlikely(nblock == 0 || nblock > nilfs->ns_blocks_per_segment)) + /* This limits the number of blocks read in the CRC check */ + goto out; + + ret = NILFS_SEG_FAIL_IO; + if (nilfs_compute_checksum(nilfs, bh_sum, &crc, sizeof(sum->ss_datasum), + ((u64)nblock << nilfs->ns_blocksize_bits), + bh_sum->b_blocknr, nblock)) + goto out; + + ret = NILFS_SEG_FAIL_CHECKSUM_FULL; + if (crc != le32_to_cpu(sum->ss_datasum)) + goto out; + ret = 0; +out: + return ret; +} + +/** + * nilfs_read_summary_info - read an item on summary blocks of a log + * @nilfs: nilfs object + * @pbh: the current buffer head on summary blocks [in, out] + * @offset: the current byte offset on summary blocks [in, out] + * @bytes: byte size of the item to be read + * + * Return: Kernel space address of current segment summary entry, or + * NULL if an I/O error occurs. + */ +static void *nilfs_read_summary_info(struct the_nilfs *nilfs, + struct buffer_head **pbh, + unsigned int *offset, unsigned int bytes) +{ + void *ptr; + sector_t blocknr; + + BUG_ON((*pbh)->b_size < *offset); + if (bytes > (*pbh)->b_size - *offset) { + blocknr = (*pbh)->b_blocknr; + brelse(*pbh); + *pbh = __bread(nilfs->ns_bdev, blocknr + 1, + nilfs->ns_blocksize); + if (unlikely(!*pbh)) + return NULL; + *offset = 0; + } + ptr = (*pbh)->b_data + *offset; + *offset += bytes; + return ptr; +} + +/** + * nilfs_skip_summary_info - skip items on summary blocks of a log + * @nilfs: nilfs object + * @pbh: the current buffer head on summary blocks [in, out] + * @offset: the current byte offset on summary blocks [in, out] + * @bytes: byte size of the item to be skipped + * @count: number of items to be skipped + */ +static void nilfs_skip_summary_info(struct the_nilfs *nilfs, + struct buffer_head **pbh, + unsigned int *offset, unsigned int bytes, + unsigned long count) +{ + unsigned int rest_item_in_current_block + = ((*pbh)->b_size - *offset) / bytes; + + if (count <= rest_item_in_current_block) { + *offset += bytes * count; + } else { + sector_t blocknr = (*pbh)->b_blocknr; + unsigned int nitem_per_block = (*pbh)->b_size / bytes; + unsigned int bcnt; + + count -= rest_item_in_current_block; + bcnt = DIV_ROUND_UP(count, nitem_per_block); + *offset = bytes * (count - (bcnt - 1) * nitem_per_block); + + brelse(*pbh); + *pbh = __bread(nilfs->ns_bdev, blocknr + bcnt, + nilfs->ns_blocksize); + } +} + +/** + * nilfs_scan_dsync_log - get block information of a log written for data sync + * @nilfs: nilfs object + * @start_blocknr: start block number of the log + * @sum: log summary information + * @head: list head to add nilfs_recovery_block struct + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error. + * * %-ENOMEM - Insufficient memory available. + */ +static int nilfs_scan_dsync_log(struct the_nilfs *nilfs, sector_t start_blocknr, + struct nilfs_segment_summary *sum, + struct list_head *head) +{ + struct buffer_head *bh; + unsigned int offset; + u32 nfinfo, sumbytes; + sector_t blocknr; + u64 ino; + int err = -EIO; + + nfinfo = le32_to_cpu(sum->ss_nfinfo); + if (!nfinfo) + return 0; + + sumbytes = le32_to_cpu(sum->ss_sumbytes); + blocknr = start_blocknr + DIV_ROUND_UP(sumbytes, nilfs->ns_blocksize); + bh = __bread(nilfs->ns_bdev, start_blocknr, nilfs->ns_blocksize); + if (unlikely(!bh)) + goto out; + + offset = le16_to_cpu(sum->ss_bytes); + for (;;) { + unsigned long nblocks, ndatablk, nnodeblk; + struct nilfs_finfo *finfo; + + finfo = nilfs_read_summary_info(nilfs, &bh, &offset, + sizeof(*finfo)); + if (unlikely(!finfo)) + goto out; + + ino = le64_to_cpu(finfo->fi_ino); + nblocks = le32_to_cpu(finfo->fi_nblocks); + ndatablk = le32_to_cpu(finfo->fi_ndatablk); + nnodeblk = nblocks - ndatablk; + + while (ndatablk-- > 0) { + struct nilfs_recovery_block *rb; + struct nilfs_binfo_v *binfo; + + binfo = nilfs_read_summary_info(nilfs, &bh, &offset, + sizeof(*binfo)); + if (unlikely(!binfo)) + goto out; + + rb = kmalloc_obj(*rb, GFP_NOFS); + if (unlikely(!rb)) { + err = -ENOMEM; + goto out; + } + rb->ino = ino; + rb->blocknr = blocknr++; + rb->vblocknr = le64_to_cpu(binfo->bi_vblocknr); + rb->blkoff = le64_to_cpu(binfo->bi_blkoff); + /* INIT_LIST_HEAD(&rb->list); */ + list_add_tail(&rb->list, head); + } + if (--nfinfo == 0) + break; + blocknr += nnodeblk; /* always 0 for data sync logs */ + nilfs_skip_summary_info(nilfs, &bh, &offset, sizeof(__le64), + nnodeblk); + if (unlikely(!bh)) + goto out; + } + err = 0; + out: + brelse(bh); /* brelse(NULL) is just ignored */ + return err; +} + +static void dispose_recovery_list(struct list_head *head) +{ + while (!list_empty(head)) { + struct nilfs_recovery_block *rb; + + rb = list_first_entry(head, struct nilfs_recovery_block, list); + list_del(&rb->list); + kfree(rb); + } +} + +struct nilfs_segment_entry { + struct list_head list; + __u64 segnum; +}; + +static int nilfs_segment_list_add(struct list_head *head, __u64 segnum) +{ + struct nilfs_segment_entry *ent = kmalloc_obj(*ent, GFP_NOFS); + + if (unlikely(!ent)) + return -ENOMEM; + + ent->segnum = segnum; + INIT_LIST_HEAD(&ent->list); + list_add_tail(&ent->list, head); + return 0; +} + +void nilfs_dispose_segment_list(struct list_head *head) +{ + while (!list_empty(head)) { + struct nilfs_segment_entry *ent; + + ent = list_first_entry(head, struct nilfs_segment_entry, list); + list_del(&ent->list); + kfree(ent); + } +} + +static int nilfs_prepare_segment_for_recovery(struct the_nilfs *nilfs, + struct super_block *sb, + struct nilfs_recovery_info *ri) +{ + struct list_head *head = &ri->ri_used_segments; + struct nilfs_segment_entry *ent, *n; + struct inode *sufile = nilfs->ns_sufile; + __u64 segnum[4]; + int err; + int i; + + segnum[0] = nilfs->ns_segnum; + segnum[1] = nilfs->ns_nextnum; + segnum[2] = ri->ri_segnum; + segnum[3] = ri->ri_nextnum; + + /* + * Releasing the next segment of the latest super root. + * The next segment is invalidated by this recovery. + */ + err = nilfs_sufile_free(sufile, segnum[1]); + if (unlikely(err)) { + if (err == -ENOENT) { + nilfs_err(sb, + "checkpoint log inconsistency at block %llu (segment %llu): next segment %llu is unallocated", + (unsigned long long)nilfs->ns_last_pseg, + (unsigned long long)nilfs->ns_segnum, + (unsigned long long)segnum[1]); + err = -EINVAL; + } + goto failed; + } + + for (i = 1; i < 4; i++) { + err = nilfs_segment_list_add(head, segnum[i]); + if (unlikely(err)) + goto failed; + } + + /* + * Collecting segments written after the latest super root. + * These are marked dirty to avoid being reallocated in the next write. + */ + list_for_each_entry_safe(ent, n, head, list) { + if (ent->segnum != segnum[0]) { + err = nilfs_sufile_scrap(sufile, ent->segnum); + if (unlikely(err)) + goto failed; + } + list_del(&ent->list); + kfree(ent); + } + + /* Allocate new segments for recovery */ + err = nilfs_sufile_alloc(sufile, &segnum[0]); + if (unlikely(err)) + goto failed; + + nilfs->ns_pseg_offset = 0; + nilfs->ns_seg_seq = ri->ri_seq + 2; + nilfs->ns_nextnum = nilfs->ns_segnum = segnum[0]; + + failed: + /* No need to recover sufile because it will be destroyed on error */ + return err; +} + +static int nilfs_recovery_copy_block(struct the_nilfs *nilfs, + struct nilfs_recovery_block *rb, + loff_t pos, struct folio *folio) +{ + struct buffer_head *bh_org; + size_t from = offset_in_folio(folio, pos); + + bh_org = __bread(nilfs->ns_bdev, rb->blocknr, nilfs->ns_blocksize); + if (unlikely(!bh_org)) + return -EIO; + + memcpy_to_folio(folio, from, bh_org->b_data, bh_org->b_size); + brelse(bh_org); + return 0; +} + +static int nilfs_recover_dsync_blocks(struct the_nilfs *nilfs, + struct super_block *sb, + struct nilfs_root *root, + struct list_head *head, + unsigned long *nr_salvaged_blocks) +{ + struct inode *inode; + struct nilfs_recovery_block *rb, *n; + unsigned int blocksize = nilfs->ns_blocksize; + struct folio *folio; + loff_t pos; + int err = 0, err2 = 0; + + list_for_each_entry_safe(rb, n, head, list) { + inode = nilfs_iget(sb, root, rb->ino); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + inode = NULL; + goto failed_inode; + } + + pos = rb->blkoff << inode->i_blkbits; + err = block_write_begin(inode->i_mapping, pos, blocksize, + &folio, nilfs_get_block); + if (unlikely(err)) { + loff_t isize = inode->i_size; + + if (pos + blocksize > isize) + nilfs_write_failed(inode->i_mapping, + pos + blocksize); + goto failed_inode; + } + + err = nilfs_recovery_copy_block(nilfs, rb, pos, folio); + if (unlikely(err)) + goto failed_folio; + + err = nilfs_set_file_dirty(inode, 1); + if (unlikely(err)) + goto failed_folio; + + block_write_end(pos, blocksize, blocksize, folio); + + folio_unlock(folio); + folio_put(folio); + + (*nr_salvaged_blocks)++; + goto next; + + failed_folio: + folio_unlock(folio); + folio_put(folio); + + failed_inode: + nilfs_warn(sb, + "error %d recovering data block (ino=%lu, block-offset=%llu)", + err, (unsigned long)rb->ino, + (unsigned long long)rb->blkoff); + if (!err2) + err2 = err; + next: + iput(inode); /* iput(NULL) is just ignored */ + list_del_init(&rb->list); + kfree(rb); + } + return err2; +} + +/** + * nilfs_do_roll_forward - salvage logical segments newer than the latest + * checkpoint + * @nilfs: nilfs object + * @sb: super block instance + * @root: NILFS root instance + * @ri: pointer to a nilfs_recovery_info + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EINVAL - Log format error. + * * %-EIO - I/O error. + * * %-ENOMEM - Insufficient memory available. + */ +static int nilfs_do_roll_forward(struct the_nilfs *nilfs, + struct super_block *sb, + struct nilfs_root *root, + struct nilfs_recovery_info *ri) +{ + struct buffer_head *bh_sum = NULL; + struct nilfs_segment_summary *sum = NULL; + sector_t pseg_start; + sector_t seg_start, seg_end; /* Starting/ending DBN of full segment */ + unsigned long nsalvaged_blocks = 0; + unsigned int flags; + u64 seg_seq; + __u64 segnum, nextnum = 0; + int empty_seg = 0; + int err = 0, ret; + LIST_HEAD(dsync_blocks); /* list of data blocks to be recovered */ + enum { + RF_INIT_ST, + RF_DSYNC_ST, /* scanning data-sync segments */ + }; + int state = RF_INIT_ST; + + pseg_start = ri->ri_lsegs_start; + seg_seq = ri->ri_lsegs_start_seq; + segnum = nilfs_get_segnum_of_block(nilfs, pseg_start); + nilfs_get_segment_range(nilfs, segnum, &seg_start, &seg_end); + + while (segnum != ri->ri_segnum || pseg_start <= ri->ri_pseg_start) { + brelse(bh_sum); + bh_sum = nilfs_read_log_header(nilfs, pseg_start, &sum); + if (!bh_sum) { + err = -EIO; + goto failed; + } + + ret = nilfs_validate_log(nilfs, seg_seq, bh_sum, sum); + if (ret) { + if (ret == NILFS_SEG_FAIL_IO) { + err = -EIO; + goto failed; + } + goto strayed; + } + + flags = le16_to_cpu(sum->ss_flags); + if (flags & NILFS_SS_SR) + goto confused; + + /* Found a valid partial segment; do recovery actions */ + nextnum = nilfs_get_segnum_of_block(nilfs, + le64_to_cpu(sum->ss_next)); + empty_seg = 0; + nilfs->ns_ctime = le64_to_cpu(sum->ss_create); + if (!(flags & NILFS_SS_GC)) + nilfs->ns_nongc_ctime = nilfs->ns_ctime; + + switch (state) { + case RF_INIT_ST: + if (!(flags & NILFS_SS_LOGBGN) || + !(flags & NILFS_SS_SYNDT)) + goto try_next_pseg; + state = RF_DSYNC_ST; + fallthrough; + case RF_DSYNC_ST: + if (!(flags & NILFS_SS_SYNDT)) + goto confused; + + err = nilfs_scan_dsync_log(nilfs, pseg_start, sum, + &dsync_blocks); + if (unlikely(err)) + goto failed; + if (flags & NILFS_SS_LOGEND) { + err = nilfs_recover_dsync_blocks( + nilfs, sb, root, &dsync_blocks, + &nsalvaged_blocks); + if (unlikely(err)) + goto failed; + state = RF_INIT_ST; + } + break; /* Fall through to try_next_pseg */ + } + + try_next_pseg: + if (pseg_start == ri->ri_lsegs_end) + break; + pseg_start += le32_to_cpu(sum->ss_nblocks); + if (pseg_start < seg_end) + continue; + goto feed_segment; + + strayed: + if (pseg_start == ri->ri_lsegs_end) + break; + + feed_segment: + /* Looking to the next full segment */ + if (empty_seg++) + break; + seg_seq++; + segnum = nextnum; + nilfs_get_segment_range(nilfs, segnum, &seg_start, &seg_end); + pseg_start = seg_start; + } + + if (nsalvaged_blocks) { + nilfs_info(sb, "salvaged %lu blocks", nsalvaged_blocks); + ri->ri_need_recovery = NILFS_RECOVERY_ROLLFORWARD_DONE; + } + out: + brelse(bh_sum); + dispose_recovery_list(&dsync_blocks); + return err; + + confused: + err = -EINVAL; + failed: + nilfs_err(sb, + "error %d roll-forwarding partial segment at blocknr = %llu", + err, (unsigned long long)pseg_start); + goto out; +} + +static void nilfs_finish_roll_forward(struct the_nilfs *nilfs, + struct nilfs_recovery_info *ri) +{ + struct buffer_head *bh; + int err; + + if (nilfs_get_segnum_of_block(nilfs, ri->ri_lsegs_start) != + nilfs_get_segnum_of_block(nilfs, ri->ri_super_root)) + return; + + bh = __getblk(nilfs->ns_bdev, ri->ri_lsegs_start, nilfs->ns_blocksize); + if (WARN_ON(!bh)) + return; /* should never happen */ + + lock_buffer(bh); + memset(bh->b_data, 0, bh->b_size); + set_buffer_uptodate(bh); + set_buffer_dirty(bh); + unlock_buffer(bh); + + err = sync_dirty_buffer(bh); + if (unlikely(err)) + nilfs_warn(nilfs->ns_sb, + "buffer sync write failed during post-cleaning of recovery."); + brelse(bh); +} + +/** + * nilfs_abort_roll_forward - cleaning up after a failed rollforward recovery + * @nilfs: nilfs object + */ +static void nilfs_abort_roll_forward(struct the_nilfs *nilfs) +{ + struct nilfs_inode_info *ii, *n; + LIST_HEAD(head); + + /* Abandon inodes that have read recovery data */ + spin_lock(&nilfs->ns_inode_lock); + list_splice_init(&nilfs->ns_dirty_files, &head); + spin_unlock(&nilfs->ns_inode_lock); + if (list_empty(&head)) + return; + + set_nilfs_purging(nilfs); + list_for_each_entry_safe(ii, n, &head, i_dirty) { + spin_lock(&nilfs->ns_inode_lock); + list_del_init(&ii->i_dirty); + spin_unlock(&nilfs->ns_inode_lock); + + iput(&ii->vfs_inode); + } + clear_nilfs_purging(nilfs); +} + +/** + * nilfs_salvage_orphan_logs - salvage logs written after the latest checkpoint + * @nilfs: nilfs object + * @sb: super block instance + * @ri: pointer to a nilfs_recovery_info struct to store search results. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EINVAL - Inconsistent filesystem state. + * * %-EIO - I/O error. + * * %-ENOMEM - Insufficient memory available. + * * %-ENOSPC - No space left on device (only in a panic state). + * * %-ERESTARTSYS - Interrupted. + */ +int nilfs_salvage_orphan_logs(struct the_nilfs *nilfs, + struct super_block *sb, + struct nilfs_recovery_info *ri) +{ + struct nilfs_root *root; + int err; + + if (ri->ri_lsegs_start == 0 || ri->ri_lsegs_end == 0) + return 0; + + err = nilfs_attach_checkpoint(sb, ri->ri_cno, true, &root); + if (unlikely(err)) { + nilfs_err(sb, "error %d loading the latest checkpoint", err); + return err; + } + + err = nilfs_do_roll_forward(nilfs, sb, root, ri); + if (unlikely(err)) + goto failed; + + if (ri->ri_need_recovery == NILFS_RECOVERY_ROLLFORWARD_DONE) { + err = nilfs_prepare_segment_for_recovery(nilfs, sb, ri); + if (unlikely(err)) { + nilfs_err(sb, "error %d preparing segment for recovery", + err); + goto failed; + } + + err = nilfs_attach_log_writer(sb, root); + if (unlikely(err)) + goto failed; + + set_nilfs_discontinued(nilfs); + err = nilfs_construct_segment(sb); + nilfs_detach_log_writer(sb); + + if (unlikely(err)) { + nilfs_err(sb, "error %d writing segment for recovery", + err); + goto put_root; + } + + nilfs_finish_roll_forward(nilfs, ri); + } + +put_root: + nilfs_put_root(root); + return err; + +failed: + nilfs_abort_roll_forward(nilfs); + goto put_root; +} + +/** + * nilfs_search_super_root - search the latest valid super root + * @nilfs: the_nilfs + * @ri: pointer to a nilfs_recovery_info struct to store search results. + * + * nilfs_search_super_root() looks for the latest super-root from a partial + * segment pointed by the superblock. It sets up struct the_nilfs through + * this search. It fills nilfs_recovery_info (ri) required for recovery. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EINVAL - No valid segment found. + * * %-EIO - I/O error. + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_search_super_root(struct the_nilfs *nilfs, + struct nilfs_recovery_info *ri) +{ + struct buffer_head *bh_sum = NULL; + struct nilfs_segment_summary *sum = NULL; + sector_t pseg_start, pseg_end, sr_pseg_start = 0; + sector_t seg_start, seg_end; /* range of full segment (block number) */ + sector_t b, end; + unsigned long nblocks; + unsigned int flags; + u64 seg_seq; + __u64 segnum, nextnum = 0; + __u64 cno; + LIST_HEAD(segments); + int empty_seg = 0, scan_newer = 0; + int ret; + + pseg_start = nilfs->ns_last_pseg; + seg_seq = nilfs->ns_last_seq; + cno = nilfs->ns_last_cno; + segnum = nilfs_get_segnum_of_block(nilfs, pseg_start); + + /* Calculate range of segment */ + nilfs_get_segment_range(nilfs, segnum, &seg_start, &seg_end); + + /* Read ahead segment */ + b = seg_start; + while (b <= seg_end) + __breadahead(nilfs->ns_bdev, b++, nilfs->ns_blocksize); + + for (;;) { + brelse(bh_sum); + ret = NILFS_SEG_FAIL_IO; + bh_sum = nilfs_read_log_header(nilfs, pseg_start, &sum); + if (!bh_sum) + goto failed; + + ret = nilfs_validate_log(nilfs, seg_seq, bh_sum, sum); + if (ret) { + if (ret == NILFS_SEG_FAIL_IO) + goto failed; + goto strayed; + } + + nblocks = le32_to_cpu(sum->ss_nblocks); + pseg_end = pseg_start + nblocks - 1; + if (unlikely(pseg_end > seg_end)) { + ret = NILFS_SEG_FAIL_CONSISTENCY; + goto strayed; + } + + /* A valid partial segment */ + ri->ri_pseg_start = pseg_start; + ri->ri_seq = seg_seq; + ri->ri_segnum = segnum; + nextnum = nilfs_get_segnum_of_block(nilfs, + le64_to_cpu(sum->ss_next)); + ri->ri_nextnum = nextnum; + empty_seg = 0; + + flags = le16_to_cpu(sum->ss_flags); + if (!(flags & NILFS_SS_SR) && !scan_newer) { + /* + * This will never happen because a superblock + * (last_segment) always points to a pseg with + * a super root. + */ + ret = NILFS_SEG_FAIL_CONSISTENCY; + goto failed; + } + + if (pseg_start == seg_start) { + nilfs_get_segment_range(nilfs, nextnum, &b, &end); + while (b <= end) + __breadahead(nilfs->ns_bdev, b++, + nilfs->ns_blocksize); + } + if (!(flags & NILFS_SS_SR)) { + if (!ri->ri_lsegs_start && (flags & NILFS_SS_LOGBGN)) { + ri->ri_lsegs_start = pseg_start; + ri->ri_lsegs_start_seq = seg_seq; + } + if (flags & NILFS_SS_LOGEND) + ri->ri_lsegs_end = pseg_start; + goto try_next_pseg; + } + + /* A valid super root was found. */ + ri->ri_cno = cno++; + ri->ri_super_root = pseg_end; + ri->ri_lsegs_start = ri->ri_lsegs_end = 0; + + nilfs_dispose_segment_list(&segments); + sr_pseg_start = pseg_start; + nilfs->ns_pseg_offset = pseg_start + nblocks - seg_start; + nilfs->ns_seg_seq = seg_seq; + nilfs->ns_segnum = segnum; + nilfs->ns_cno = cno; /* nilfs->ns_cno = ri->ri_cno + 1 */ + nilfs->ns_ctime = le64_to_cpu(sum->ss_create); + nilfs->ns_nextnum = nextnum; + + if (scan_newer) + ri->ri_need_recovery = NILFS_RECOVERY_SR_UPDATED; + else { + if (nilfs->ns_mount_state & NILFS_VALID_FS) + goto super_root_found; + scan_newer = 1; + } + + try_next_pseg: + /* Standing on a course, or met an inconsistent state */ + pseg_start += nblocks; + if (pseg_start < seg_end) + continue; + goto feed_segment; + + strayed: + /* Off the trail */ + if (!scan_newer) + /* + * This can happen if a checkpoint was written without + * barriers, or as a result of an I/O failure. + */ + goto failed; + + feed_segment: + /* Looking to the next full segment */ + if (empty_seg++) + goto super_root_found; /* found a valid super root */ + + ret = nilfs_segment_list_add(&segments, segnum); + if (unlikely(ret)) + goto failed; + + seg_seq++; + segnum = nextnum; + nilfs_get_segment_range(nilfs, segnum, &seg_start, &seg_end); + pseg_start = seg_start; + } + + super_root_found: + /* Updating pointers relating to the latest checkpoint */ + brelse(bh_sum); + list_splice_tail(&segments, &ri->ri_used_segments); + nilfs->ns_last_pseg = sr_pseg_start; + nilfs->ns_last_seq = nilfs->ns_seg_seq; + nilfs->ns_last_cno = ri->ri_cno; + return 0; + + failed: + brelse(bh_sum); + nilfs_dispose_segment_list(&segments); + return ret < 0 ? ret : nilfs_warn_segment_error(nilfs->ns_sb, ret); +} diff --git a/fs/nilfs2/segbuf.c b/fs/nilfs2/segbuf.c new file mode 100644 index 000000000..a8bdf3d31 --- /dev/null +++ b/fs/nilfs2/segbuf.c @@ -0,0 +1,472 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * NILFS segment buffer + * + * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Ryusuke Konishi. + * + */ + +#include +#include +#include +#include +#include +#include "page.h" +#include "segbuf.h" + + +struct nilfs_write_info { + struct the_nilfs *nilfs; + struct bio *bio; + int start, end; /* The region to be submitted */ + int rest_blocks; + int max_pages; + int nr_vecs; + sector_t blocknr; +}; + +static int nilfs_segbuf_write(struct nilfs_segment_buffer *segbuf, + struct the_nilfs *nilfs); +static int nilfs_segbuf_wait(struct nilfs_segment_buffer *segbuf); + +struct nilfs_segment_buffer *nilfs_segbuf_new(struct super_block *sb) +{ + struct nilfs_segment_buffer *segbuf; + + segbuf = kmem_cache_alloc(nilfs_segbuf_cachep, GFP_NOFS); + if (unlikely(!segbuf)) + return NULL; + + segbuf->sb_super = sb; + INIT_LIST_HEAD(&segbuf->sb_list); + INIT_LIST_HEAD(&segbuf->sb_segsum_buffers); + INIT_LIST_HEAD(&segbuf->sb_payload_buffers); + segbuf->sb_super_root = NULL; + + init_completion(&segbuf->sb_bio_event); + atomic_set(&segbuf->sb_err, 0); + segbuf->sb_nbio = 0; + + return segbuf; +} + +void nilfs_segbuf_free(struct nilfs_segment_buffer *segbuf) +{ + kmem_cache_free(nilfs_segbuf_cachep, segbuf); +} + +void nilfs_segbuf_map(struct nilfs_segment_buffer *segbuf, __u64 segnum, + unsigned long offset, struct the_nilfs *nilfs) +{ + segbuf->sb_segnum = segnum; + nilfs_get_segment_range(nilfs, segnum, &segbuf->sb_fseg_start, + &segbuf->sb_fseg_end); + + segbuf->sb_pseg_start = segbuf->sb_fseg_start + offset; + segbuf->sb_rest_blocks = + segbuf->sb_fseg_end - segbuf->sb_pseg_start + 1; +} + +/** + * nilfs_segbuf_map_cont - map a new log behind a given log + * @segbuf: new segment buffer + * @prev: segment buffer containing a log to be continued + */ +void nilfs_segbuf_map_cont(struct nilfs_segment_buffer *segbuf, + struct nilfs_segment_buffer *prev) +{ + segbuf->sb_segnum = prev->sb_segnum; + segbuf->sb_fseg_start = prev->sb_fseg_start; + segbuf->sb_fseg_end = prev->sb_fseg_end; + segbuf->sb_pseg_start = prev->sb_pseg_start + prev->sb_sum.nblocks; + segbuf->sb_rest_blocks = + segbuf->sb_fseg_end - segbuf->sb_pseg_start + 1; +} + +void nilfs_segbuf_set_next_segnum(struct nilfs_segment_buffer *segbuf, + __u64 nextnum, struct the_nilfs *nilfs) +{ + segbuf->sb_nextnum = nextnum; + segbuf->sb_sum.next = nilfs_get_segment_start_blocknr(nilfs, nextnum); +} + +int nilfs_segbuf_extend_segsum(struct nilfs_segment_buffer *segbuf) +{ + struct buffer_head *bh; + + bh = sb_getblk(segbuf->sb_super, + segbuf->sb_pseg_start + segbuf->sb_sum.nsumblk); + if (unlikely(!bh)) + return -ENOMEM; + + lock_buffer(bh); + if (!buffer_uptodate(bh)) { + memset(bh->b_data, 0, bh->b_size); + set_buffer_uptodate(bh); + } + unlock_buffer(bh); + nilfs_segbuf_add_segsum_buffer(segbuf, bh); + return 0; +} + +int nilfs_segbuf_extend_payload(struct nilfs_segment_buffer *segbuf, + struct buffer_head **bhp) +{ + struct buffer_head *bh; + + bh = sb_getblk(segbuf->sb_super, + segbuf->sb_pseg_start + segbuf->sb_sum.nblocks); + if (unlikely(!bh)) + return -ENOMEM; + + nilfs_segbuf_add_payload_buffer(segbuf, bh); + *bhp = bh; + return 0; +} + +int nilfs_segbuf_reset(struct nilfs_segment_buffer *segbuf, unsigned int flags, + time64_t ctime, __u64 cno) +{ + int err; + + segbuf->sb_sum.nblocks = segbuf->sb_sum.nsumblk = 0; + err = nilfs_segbuf_extend_segsum(segbuf); + if (unlikely(err)) + return err; + + segbuf->sb_sum.flags = flags; + segbuf->sb_sum.sumbytes = sizeof(struct nilfs_segment_summary); + segbuf->sb_sum.nfinfo = segbuf->sb_sum.nfileblk = 0; + segbuf->sb_sum.ctime = ctime; + segbuf->sb_sum.cno = cno; + return 0; +} + +/* + * Setup segment summary + */ +void nilfs_segbuf_fill_in_segsum(struct nilfs_segment_buffer *segbuf) +{ + struct nilfs_segment_summary *raw_sum; + struct buffer_head *bh_sum; + + bh_sum = list_entry(segbuf->sb_segsum_buffers.next, + struct buffer_head, b_assoc_buffers); + raw_sum = (struct nilfs_segment_summary *)bh_sum->b_data; + + raw_sum->ss_magic = cpu_to_le32(NILFS_SEGSUM_MAGIC); + raw_sum->ss_bytes = cpu_to_le16(sizeof(*raw_sum)); + raw_sum->ss_flags = cpu_to_le16(segbuf->sb_sum.flags); + raw_sum->ss_seq = cpu_to_le64(segbuf->sb_sum.seg_seq); + raw_sum->ss_create = cpu_to_le64(segbuf->sb_sum.ctime); + raw_sum->ss_next = cpu_to_le64(segbuf->sb_sum.next); + raw_sum->ss_nblocks = cpu_to_le32(segbuf->sb_sum.nblocks); + raw_sum->ss_nfinfo = cpu_to_le32(segbuf->sb_sum.nfinfo); + raw_sum->ss_sumbytes = cpu_to_le32(segbuf->sb_sum.sumbytes); + raw_sum->ss_pad = 0; + raw_sum->ss_cno = cpu_to_le64(segbuf->sb_sum.cno); +} + +/* + * CRC calculation routines + */ +static void +nilfs_segbuf_fill_in_segsum_crc(struct nilfs_segment_buffer *segbuf, u32 seed) +{ + struct buffer_head *bh; + struct nilfs_segment_summary *raw_sum; + unsigned long size, bytes = segbuf->sb_sum.sumbytes; + u32 crc; + + bh = list_entry(segbuf->sb_segsum_buffers.next, struct buffer_head, + b_assoc_buffers); + + raw_sum = (struct nilfs_segment_summary *)bh->b_data; + size = min_t(unsigned long, bytes, bh->b_size); + crc = crc32_le(seed, + (unsigned char *)raw_sum + + sizeof(raw_sum->ss_datasum) + sizeof(raw_sum->ss_sumsum), + size - (sizeof(raw_sum->ss_datasum) + + sizeof(raw_sum->ss_sumsum))); + + list_for_each_entry_continue(bh, &segbuf->sb_segsum_buffers, + b_assoc_buffers) { + bytes -= size; + size = min_t(unsigned long, bytes, bh->b_size); + crc = crc32_le(crc, bh->b_data, size); + } + raw_sum->ss_sumsum = cpu_to_le32(crc); +} + +static void nilfs_segbuf_fill_in_data_crc(struct nilfs_segment_buffer *segbuf, + u32 seed) +{ + struct buffer_head *bh; + struct nilfs_segment_summary *raw_sum; + u32 crc; + + bh = list_entry(segbuf->sb_segsum_buffers.next, struct buffer_head, + b_assoc_buffers); + raw_sum = (struct nilfs_segment_summary *)bh->b_data; + crc = crc32_le(seed, + (unsigned char *)raw_sum + sizeof(raw_sum->ss_datasum), + bh->b_size - sizeof(raw_sum->ss_datasum)); + + list_for_each_entry_continue(bh, &segbuf->sb_segsum_buffers, + b_assoc_buffers) { + crc = crc32_le(crc, bh->b_data, bh->b_size); + } + list_for_each_entry(bh, &segbuf->sb_payload_buffers, b_assoc_buffers) { + size_t offset = offset_in_folio(bh->b_folio, bh->b_data); + unsigned char *from; + + /* Do not support block sizes larger than PAGE_SIZE */ + from = kmap_local_folio(bh->b_folio, offset); + crc = crc32_le(crc, from, bh->b_size); + kunmap_local(from); + } + raw_sum->ss_datasum = cpu_to_le32(crc); +} + +static void +nilfs_segbuf_fill_in_super_root_crc(struct nilfs_segment_buffer *segbuf, + u32 seed) +{ + struct nilfs_super_root *raw_sr; + struct the_nilfs *nilfs = segbuf->sb_super->s_fs_info; + unsigned int srsize; + u32 crc; + + raw_sr = (struct nilfs_super_root *)segbuf->sb_super_root->b_data; + srsize = NILFS_SR_BYTES(nilfs->ns_inode_size); + crc = crc32_le(seed, + (unsigned char *)raw_sr + sizeof(raw_sr->sr_sum), + srsize - sizeof(raw_sr->sr_sum)); + raw_sr->sr_sum = cpu_to_le32(crc); +} + +static void nilfs_release_buffers(struct list_head *list) +{ + struct buffer_head *bh, *n; + + list_for_each_entry_safe(bh, n, list, b_assoc_buffers) { + list_del_init(&bh->b_assoc_buffers); + brelse(bh); + } +} + +static void nilfs_segbuf_clear(struct nilfs_segment_buffer *segbuf) +{ + nilfs_release_buffers(&segbuf->sb_segsum_buffers); + nilfs_release_buffers(&segbuf->sb_payload_buffers); + segbuf->sb_super_root = NULL; +} + +/* + * Iterators for segment buffers + */ +void nilfs_clear_logs(struct list_head *logs) +{ + struct nilfs_segment_buffer *segbuf; + + list_for_each_entry(segbuf, logs, sb_list) + nilfs_segbuf_clear(segbuf); +} + +void nilfs_truncate_logs(struct list_head *logs, + struct nilfs_segment_buffer *last) +{ + struct nilfs_segment_buffer *n, *segbuf; + + segbuf = list_prepare_entry(last, logs, sb_list); + list_for_each_entry_safe_continue(segbuf, n, logs, sb_list) { + list_del_init(&segbuf->sb_list); + nilfs_segbuf_clear(segbuf); + nilfs_segbuf_free(segbuf); + } +} + +int nilfs_write_logs(struct list_head *logs, struct the_nilfs *nilfs) +{ + struct nilfs_segment_buffer *segbuf; + int ret = 0; + + list_for_each_entry(segbuf, logs, sb_list) { + ret = nilfs_segbuf_write(segbuf, nilfs); + if (ret) + break; + } + return ret; +} + +int nilfs_wait_on_logs(struct list_head *logs) +{ + struct nilfs_segment_buffer *segbuf; + int err, ret = 0; + + list_for_each_entry(segbuf, logs, sb_list) { + err = nilfs_segbuf_wait(segbuf); + if (err && !ret) + ret = err; + } + return ret; +} + +/** + * nilfs_add_checksums_on_logs - add checksums on the logs + * @logs: list of segment buffers storing target logs + * @seed: checksum seed value + */ +void nilfs_add_checksums_on_logs(struct list_head *logs, u32 seed) +{ + struct nilfs_segment_buffer *segbuf; + + list_for_each_entry(segbuf, logs, sb_list) { + if (segbuf->sb_super_root) + nilfs_segbuf_fill_in_super_root_crc(segbuf, seed); + nilfs_segbuf_fill_in_segsum_crc(segbuf, seed); + nilfs_segbuf_fill_in_data_crc(segbuf, seed); + } +} + +/* + * BIO operations + */ +static void nilfs_end_bio_write(struct bio *bio) +{ + struct nilfs_segment_buffer *segbuf = bio->bi_private; + + if (bio->bi_status) + atomic_inc(&segbuf->sb_err); + + bio_put(bio); + complete(&segbuf->sb_bio_event); +} + +static int nilfs_segbuf_submit_bio(struct nilfs_segment_buffer *segbuf, + struct nilfs_write_info *wi) +{ + struct bio *bio = wi->bio; + + bio->bi_end_io = nilfs_end_bio_write; + bio->bi_private = segbuf; + submit_bio(bio); + segbuf->sb_nbio++; + + wi->bio = NULL; + wi->rest_blocks -= wi->end - wi->start; + wi->nr_vecs = min(wi->max_pages, wi->rest_blocks); + wi->start = wi->end; + return 0; +} + +static void nilfs_segbuf_prepare_write(struct nilfs_segment_buffer *segbuf, + struct nilfs_write_info *wi) +{ + wi->bio = NULL; + wi->rest_blocks = segbuf->sb_sum.nblocks; + wi->max_pages = BIO_MAX_VECS; + wi->nr_vecs = min(wi->max_pages, wi->rest_blocks); + wi->start = wi->end = 0; + wi->blocknr = segbuf->sb_pseg_start; +} + +static int nilfs_segbuf_submit_bh(struct nilfs_segment_buffer *segbuf, + struct nilfs_write_info *wi, + struct buffer_head *bh) +{ + int err; + + BUG_ON(wi->nr_vecs <= 0); + repeat: + if (!wi->bio) { + wi->bio = bio_alloc(wi->nilfs->ns_bdev, wi->nr_vecs, + REQ_OP_WRITE, GFP_NOIO); + wi->bio->bi_iter.bi_sector = (wi->blocknr + wi->end) << + (wi->nilfs->ns_blocksize_bits - 9); + } + + if (bio_add_folio(wi->bio, bh->b_folio, bh->b_size, + offset_in_folio(bh->b_folio, bh->b_data))) { + wi->end++; + return 0; + } + /* bio is FULL */ + err = nilfs_segbuf_submit_bio(segbuf, wi); + /* never submit current bh */ + if (likely(!err)) + goto repeat; + return err; +} + +/** + * nilfs_segbuf_write - submit write requests of a log + * @segbuf: buffer storing a log to be written + * @nilfs: nilfs object + * + * Return: Always 0. + */ +static int nilfs_segbuf_write(struct nilfs_segment_buffer *segbuf, + struct the_nilfs *nilfs) +{ + struct nilfs_write_info wi; + struct buffer_head *bh; + int res = 0; + + wi.nilfs = nilfs; + nilfs_segbuf_prepare_write(segbuf, &wi); + + list_for_each_entry(bh, &segbuf->sb_segsum_buffers, b_assoc_buffers) { + res = nilfs_segbuf_submit_bh(segbuf, &wi, bh); + if (unlikely(res)) + goto failed_bio; + } + + list_for_each_entry(bh, &segbuf->sb_payload_buffers, b_assoc_buffers) { + res = nilfs_segbuf_submit_bh(segbuf, &wi, bh); + if (unlikely(res)) + goto failed_bio; + } + + if (wi.bio) { + /* + * Last BIO is always sent through the following + * submission. + */ + wi.bio->bi_opf |= REQ_SYNC; + res = nilfs_segbuf_submit_bio(segbuf, &wi); + } + + failed_bio: + return res; +} + +/** + * nilfs_segbuf_wait - wait for completion of requested BIOs + * @segbuf: segment buffer + * + * Return: 0 on success, or %-EIO if I/O error is detected. + */ +static int nilfs_segbuf_wait(struct nilfs_segment_buffer *segbuf) +{ + int err = 0; + + if (!segbuf->sb_nbio) + return 0; + + do { + wait_for_completion(&segbuf->sb_bio_event); + } while (--segbuf->sb_nbio > 0); + + if (unlikely(atomic_read(&segbuf->sb_err) > 0)) { + nilfs_err(segbuf->sb_super, + "I/O error writing log (start-blocknr=%llu, block-count=%lu) in segment %llu", + (unsigned long long)segbuf->sb_pseg_start, + segbuf->sb_sum.nblocks, + (unsigned long long)segbuf->sb_segnum); + err = -EIO; + } + return err; +} diff --git a/fs/nilfs2/segbuf.h b/fs/nilfs2/segbuf.h new file mode 100644 index 000000000..e20091ede --- /dev/null +++ b/fs/nilfs2/segbuf.h @@ -0,0 +1,172 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * NILFS Segment buffer prototypes and definitions + * + * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Ryusuke Konishi. + * + */ +#ifndef _NILFS_SEGBUF_H +#define _NILFS_SEGBUF_H + +#include +#include +#include +#include + +/** + * struct nilfs_segsum_info - On-memory segment summary + * @flags: Flags + * @nfinfo: Number of file information structures + * @nblocks: Number of blocks included in the partial segment + * @nsumblk: Number of summary blocks + * @sumbytes: Byte count of segment summary + * @nfileblk: Total number of file blocks + * @seg_seq: Segment sequence number + * @cno: Checkpoint number + * @ctime: Creation time + * @next: Block number of the next full segment + */ +struct nilfs_segsum_info { + unsigned int flags; + unsigned long nfinfo; + unsigned long nblocks; + unsigned long nsumblk; + unsigned long sumbytes; + unsigned long nfileblk; + u64 seg_seq; + __u64 cno; + time64_t ctime; + sector_t next; +}; + +/** + * struct nilfs_segment_buffer - Segment buffer + * @sb_super: back pointer to a superblock struct + * @sb_list: List head to chain this structure + * @sb_sum: On-memory segment summary + * @sb_segnum: Index number of the full segment + * @sb_nextnum: Index number of the next full segment + * @sb_fseg_start: Start block number of the full segment + * @sb_fseg_end: End block number of the full segment + * @sb_pseg_start: Disk block number of partial segment + * @sb_rest_blocks: Number of residual blocks in the current segment + * @sb_segsum_buffers: List of buffers for segment summaries + * @sb_payload_buffers: List of buffers for segment payload + * @sb_super_root: Pointer to buffer storing a super root block (if exists) + * @sb_nbio: Number of flying bio requests + * @sb_err: I/O error status + * @sb_bio_event: Completion event of log writing + */ +struct nilfs_segment_buffer { + struct super_block *sb_super; + struct list_head sb_list; + + /* Segment information */ + struct nilfs_segsum_info sb_sum; + __u64 sb_segnum; + __u64 sb_nextnum; + sector_t sb_fseg_start, sb_fseg_end; + sector_t sb_pseg_start; + unsigned int sb_rest_blocks; + + /* Buffers */ + struct list_head sb_segsum_buffers; + struct list_head sb_payload_buffers; /* including super root */ + struct buffer_head *sb_super_root; + + /* io status */ + int sb_nbio; + atomic_t sb_err; + struct completion sb_bio_event; +}; + +#define NILFS_LIST_SEGBUF(head) \ + list_entry((head), struct nilfs_segment_buffer, sb_list) +#define NILFS_NEXT_SEGBUF(segbuf) NILFS_LIST_SEGBUF((segbuf)->sb_list.next) +#define NILFS_PREV_SEGBUF(segbuf) NILFS_LIST_SEGBUF((segbuf)->sb_list.prev) +#define NILFS_LAST_SEGBUF(head) NILFS_LIST_SEGBUF((head)->prev) +#define NILFS_FIRST_SEGBUF(head) NILFS_LIST_SEGBUF((head)->next) +#define NILFS_SEGBUF_IS_LAST(segbuf, head) ((segbuf)->sb_list.next == (head)) + +#define nilfs_for_each_segbuf_before(s, t, h) \ + for ((s) = NILFS_FIRST_SEGBUF(h); (s) != (t); \ + (s) = NILFS_NEXT_SEGBUF(s)) + +#define NILFS_SEGBUF_FIRST_BH(head) \ + (list_entry((head)->next, struct buffer_head, b_assoc_buffers)) +#define NILFS_SEGBUF_NEXT_BH(bh) \ + (list_entry((bh)->b_assoc_buffers.next, struct buffer_head, \ + b_assoc_buffers)) +#define NILFS_SEGBUF_BH_IS_LAST(bh, head) ((bh)->b_assoc_buffers.next == head) + +extern struct kmem_cache *nilfs_segbuf_cachep; + +struct nilfs_segment_buffer *nilfs_segbuf_new(struct super_block *); +void nilfs_segbuf_free(struct nilfs_segment_buffer *); +void nilfs_segbuf_map(struct nilfs_segment_buffer *, __u64, unsigned long, + struct the_nilfs *); +void nilfs_segbuf_map_cont(struct nilfs_segment_buffer *segbuf, + struct nilfs_segment_buffer *prev); +void nilfs_segbuf_set_next_segnum(struct nilfs_segment_buffer *, __u64, + struct the_nilfs *); +int nilfs_segbuf_reset(struct nilfs_segment_buffer *, unsigned int, time64_t, + __u64); +int nilfs_segbuf_extend_segsum(struct nilfs_segment_buffer *); +int nilfs_segbuf_extend_payload(struct nilfs_segment_buffer *, + struct buffer_head **); +void nilfs_segbuf_fill_in_segsum(struct nilfs_segment_buffer *); + +static inline int nilfs_segbuf_simplex(struct nilfs_segment_buffer *segbuf) +{ + unsigned int flags = segbuf->sb_sum.flags; + + return (flags & (NILFS_SS_LOGBGN | NILFS_SS_LOGEND)) == + (NILFS_SS_LOGBGN | NILFS_SS_LOGEND); +} + +static inline int nilfs_segbuf_empty(struct nilfs_segment_buffer *segbuf) +{ + return segbuf->sb_sum.nblocks == segbuf->sb_sum.nsumblk; +} + +static inline void +nilfs_segbuf_add_segsum_buffer(struct nilfs_segment_buffer *segbuf, + struct buffer_head *bh) +{ + list_add_tail(&bh->b_assoc_buffers, &segbuf->sb_segsum_buffers); + segbuf->sb_sum.nblocks++; + segbuf->sb_sum.nsumblk++; +} + +static inline void +nilfs_segbuf_add_payload_buffer(struct nilfs_segment_buffer *segbuf, + struct buffer_head *bh) +{ + list_add_tail(&bh->b_assoc_buffers, &segbuf->sb_payload_buffers); + segbuf->sb_sum.nblocks++; +} + +static inline void +nilfs_segbuf_add_file_buffer(struct nilfs_segment_buffer *segbuf, + struct buffer_head *bh) +{ + get_bh(bh); + nilfs_segbuf_add_payload_buffer(segbuf, bh); + segbuf->sb_sum.nfileblk++; +} + +void nilfs_clear_logs(struct list_head *logs); +void nilfs_truncate_logs(struct list_head *logs, + struct nilfs_segment_buffer *last); +int nilfs_write_logs(struct list_head *logs, struct the_nilfs *nilfs); +int nilfs_wait_on_logs(struct list_head *logs); +void nilfs_add_checksums_on_logs(struct list_head *logs, u32 seed); + +static inline void nilfs_destroy_logs(struct list_head *logs) +{ + nilfs_truncate_logs(logs, NULL); +} + +#endif /* _NILFS_SEGBUF_H */ diff --git a/fs/nilfs2/segment.c b/fs/nilfs2/segment.c new file mode 100644 index 000000000..829573cb6 --- /dev/null +++ b/fs/nilfs2/segment.c @@ -0,0 +1,2929 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * NILFS segment constructor. + * + * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Ryusuke Konishi. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nilfs.h" +#include "btnode.h" +#include "page.h" +#include "segment.h" +#include "sufile.h" +#include "cpfile.h" +#include "ifile.h" +#include "segbuf.h" + + +/* + * Segment constructor + */ +#define SC_N_INODEVEC 16 /* Size of locally allocated inode vector */ + +#define SC_MAX_SEGDELTA 64 /* + * Upper limit of the number of segments + * appended in collection retry loop + */ + +/* Construction mode */ +enum { + SC_LSEG_SR = 1, /* Make a logical segment having a super root */ + SC_LSEG_DSYNC, /* + * Flush data blocks of a given file and make + * a logical segment without a super root. + */ + SC_FLUSH_FILE, /* + * Flush data files, leads to segment writes without + * creating a checkpoint. + */ + SC_FLUSH_DAT, /* + * Flush DAT file. This also creates segments + * without a checkpoint. + */ +}; + +/* Stage numbers of dirty block collection */ +enum { + NILFS_ST_INIT = 0, + NILFS_ST_GC, /* Collecting dirty blocks for GC */ + NILFS_ST_FILE, + NILFS_ST_IFILE, + NILFS_ST_CPFILE, + NILFS_ST_SUFILE, + NILFS_ST_DAT, + NILFS_ST_SR, /* Super root */ + NILFS_ST_DSYNC, /* Data sync blocks */ + NILFS_ST_DONE, +}; + +#define CREATE_TRACE_POINTS +#include + +/* + * nilfs_sc_cstage_inc(), nilfs_sc_cstage_set(), nilfs_sc_cstage_get() are + * wrapper functions of stage count (nilfs_sc_info->sc_stage.scnt). Users of + * the variable must use them because transition of stage count must involve + * trace events (trace_nilfs2_collection_stage_transition). + * + * nilfs_sc_cstage_get() isn't required for the above purpose because it doesn't + * produce tracepoint events. It is provided just for making the intention + * clear. + */ +static inline void nilfs_sc_cstage_inc(struct nilfs_sc_info *sci) +{ + sci->sc_stage.scnt++; + trace_nilfs2_collection_stage_transition(sci); +} + +static inline void nilfs_sc_cstage_set(struct nilfs_sc_info *sci, int next_scnt) +{ + sci->sc_stage.scnt = next_scnt; + trace_nilfs2_collection_stage_transition(sci); +} + +static inline int nilfs_sc_cstage_get(struct nilfs_sc_info *sci) +{ + return sci->sc_stage.scnt; +} + +/* State flags of collection */ +#define NILFS_CF_NODE 0x0001 /* Collecting node blocks */ +#define NILFS_CF_IFILE_STARTED 0x0002 /* IFILE stage has started */ +#define NILFS_CF_SUFREED 0x0004 /* segment usages has been freed */ +#define NILFS_CF_HISTORY_MASK (NILFS_CF_IFILE_STARTED | NILFS_CF_SUFREED) + +/* Operations depending on the construction mode and file type */ +struct nilfs_sc_operations { + int (*collect_data)(struct nilfs_sc_info *, struct buffer_head *, + struct inode *); + int (*collect_node)(struct nilfs_sc_info *, struct buffer_head *, + struct inode *); + int (*collect_bmap)(struct nilfs_sc_info *, struct buffer_head *, + struct inode *); + void (*write_data_binfo)(struct nilfs_sc_info *, + struct nilfs_segsum_pointer *, + union nilfs_binfo *); + void (*write_node_binfo)(struct nilfs_sc_info *, + struct nilfs_segsum_pointer *, + union nilfs_binfo *); +}; + +/* + * Other definitions + */ +static void nilfs_segctor_start_timer(struct nilfs_sc_info *); +static void nilfs_segctor_do_flush(struct nilfs_sc_info *, int); +static void nilfs_segctor_do_immediate_flush(struct nilfs_sc_info *); +static void nilfs_dispose_list(struct the_nilfs *, struct list_head *, int); + +#define nilfs_cnt32_ge(a, b) \ + (typecheck(__u32, a) && typecheck(__u32, b) && \ + ((__s32)((a) - (b)) >= 0)) + +static int nilfs_prepare_segment_lock(struct super_block *sb, + struct nilfs_transaction_info *ti) +{ + struct nilfs_transaction_info *cur_ti = current->journal_info; + void *save = NULL; + + if (cur_ti) { + if (cur_ti->ti_magic == NILFS_TI_MAGIC) + return ++cur_ti->ti_count; + + /* + * If journal_info field is occupied by other FS, + * it is saved and will be restored on + * nilfs_transaction_commit(). + */ + nilfs_warn(sb, "journal info from a different FS"); + save = current->journal_info; + } + if (!ti) { + ti = kmem_cache_alloc(nilfs_transaction_cachep, GFP_NOFS); + if (!ti) + return -ENOMEM; + ti->ti_flags = NILFS_TI_DYNAMIC_ALLOC; + } else { + ti->ti_flags = 0; + } + ti->ti_count = 0; + ti->ti_save = save; + ti->ti_magic = NILFS_TI_MAGIC; + current->journal_info = ti; + return 0; +} + +/** + * nilfs_transaction_begin - start indivisible file operations. + * @sb: super block + * @ti: nilfs_transaction_info + * @vacancy_check: flags for vacancy rate checks + * + * nilfs_transaction_begin() acquires a reader/writer semaphore, called + * the segment semaphore, to make a segment construction and write tasks + * exclusive. The function is used with nilfs_transaction_commit() in pairs. + * The region enclosed by these two functions can be nested. To avoid a + * deadlock, the semaphore is only acquired or released in the outermost call. + * + * This function allocates a nilfs_transaction_info struct to keep context + * information on it. It is initialized and hooked onto the current task in + * the outermost call. If a pre-allocated struct is given to @ti, it is used + * instead; otherwise a new struct is assigned from a slab. + * + * When @vacancy_check flag is set, this function will check the amount of + * free space, and will wait for the GC to reclaim disk space if low capacity. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-ENOMEM - Insufficient memory available. + * * %-ENOSPC - No space left on device (if checking free space). + */ +int nilfs_transaction_begin(struct super_block *sb, + struct nilfs_transaction_info *ti, + int vacancy_check) +{ + struct the_nilfs *nilfs; + int ret = nilfs_prepare_segment_lock(sb, ti); + struct nilfs_transaction_info *trace_ti; + + if (unlikely(ret < 0)) + return ret; + if (ret > 0) { + trace_ti = current->journal_info; + + trace_nilfs2_transaction_transition(sb, trace_ti, + trace_ti->ti_count, trace_ti->ti_flags, + TRACE_NILFS2_TRANSACTION_BEGIN); + return 0; + } + + sb_start_intwrite(sb); + + nilfs = sb->s_fs_info; + down_read(&nilfs->ns_segctor_sem); + if (vacancy_check && nilfs_near_disk_full(nilfs)) { + up_read(&nilfs->ns_segctor_sem); + ret = -ENOSPC; + goto failed; + } + + trace_ti = current->journal_info; + trace_nilfs2_transaction_transition(sb, trace_ti, trace_ti->ti_count, + trace_ti->ti_flags, + TRACE_NILFS2_TRANSACTION_BEGIN); + return 0; + + failed: + ti = current->journal_info; + current->journal_info = ti->ti_save; + if (ti->ti_flags & NILFS_TI_DYNAMIC_ALLOC) + kmem_cache_free(nilfs_transaction_cachep, ti); + sb_end_intwrite(sb); + return ret; +} + +/** + * nilfs_transaction_commit - commit indivisible file operations. + * @sb: super block + * + * nilfs_transaction_commit() releases the read semaphore which is + * acquired by nilfs_transaction_begin(). This is only performed + * in outermost call of this function. If a commit flag is set, + * nilfs_transaction_commit() sets a timer to start the segment + * constructor. If a sync flag is set, it starts construction + * directly. + * + * Return: 0 on success, or a negative error code on failure. + */ +int nilfs_transaction_commit(struct super_block *sb) +{ + struct nilfs_transaction_info *ti = current->journal_info; + struct the_nilfs *nilfs = sb->s_fs_info; + int err = 0; + + BUG_ON(ti == NULL || ti->ti_magic != NILFS_TI_MAGIC); + ti->ti_flags |= NILFS_TI_COMMIT; + if (ti->ti_count > 0) { + ti->ti_count--; + trace_nilfs2_transaction_transition(sb, ti, ti->ti_count, + ti->ti_flags, TRACE_NILFS2_TRANSACTION_COMMIT); + return 0; + } + if (nilfs->ns_writer) { + struct nilfs_sc_info *sci = nilfs->ns_writer; + + if (ti->ti_flags & NILFS_TI_COMMIT) + nilfs_segctor_start_timer(sci); + if (atomic_read(&nilfs->ns_ndirtyblks) > sci->sc_watermark) + nilfs_segctor_do_flush(sci, 0); + } + up_read(&nilfs->ns_segctor_sem); + trace_nilfs2_transaction_transition(sb, ti, ti->ti_count, + ti->ti_flags, TRACE_NILFS2_TRANSACTION_COMMIT); + + current->journal_info = ti->ti_save; + + if (ti->ti_flags & NILFS_TI_SYNC) + err = nilfs_construct_segment(sb); + if (ti->ti_flags & NILFS_TI_DYNAMIC_ALLOC) + kmem_cache_free(nilfs_transaction_cachep, ti); + sb_end_intwrite(sb); + return err; +} + +void nilfs_transaction_abort(struct super_block *sb) +{ + struct nilfs_transaction_info *ti = current->journal_info; + struct the_nilfs *nilfs = sb->s_fs_info; + + BUG_ON(ti == NULL || ti->ti_magic != NILFS_TI_MAGIC); + if (ti->ti_count > 0) { + ti->ti_count--; + trace_nilfs2_transaction_transition(sb, ti, ti->ti_count, + ti->ti_flags, TRACE_NILFS2_TRANSACTION_ABORT); + return; + } + up_read(&nilfs->ns_segctor_sem); + + trace_nilfs2_transaction_transition(sb, ti, ti->ti_count, + ti->ti_flags, TRACE_NILFS2_TRANSACTION_ABORT); + + current->journal_info = ti->ti_save; + if (ti->ti_flags & NILFS_TI_DYNAMIC_ALLOC) + kmem_cache_free(nilfs_transaction_cachep, ti); + sb_end_intwrite(sb); +} + +void nilfs_relax_pressure_in_lock(struct super_block *sb) +{ + struct the_nilfs *nilfs = sb->s_fs_info; + struct nilfs_sc_info *sci = nilfs->ns_writer; + + if (sb_rdonly(sb) || unlikely(!sci) || !sci->sc_flush_request) + return; + + set_bit(NILFS_SC_PRIOR_FLUSH, &sci->sc_flags); + up_read(&nilfs->ns_segctor_sem); + + down_write(&nilfs->ns_segctor_sem); + if (sci->sc_flush_request && + test_bit(NILFS_SC_PRIOR_FLUSH, &sci->sc_flags)) { + struct nilfs_transaction_info *ti = current->journal_info; + + ti->ti_flags |= NILFS_TI_WRITER; + nilfs_segctor_do_immediate_flush(sci); + ti->ti_flags &= ~NILFS_TI_WRITER; + } + downgrade_write(&nilfs->ns_segctor_sem); +} + +static void nilfs_transaction_lock(struct super_block *sb, + struct nilfs_transaction_info *ti, + int gcflag) +{ + struct nilfs_transaction_info *cur_ti = current->journal_info; + struct the_nilfs *nilfs = sb->s_fs_info; + struct nilfs_sc_info *sci = nilfs->ns_writer; + + WARN_ON(cur_ti); + ti->ti_flags = NILFS_TI_WRITER; + ti->ti_count = 0; + ti->ti_save = cur_ti; + ti->ti_magic = NILFS_TI_MAGIC; + current->journal_info = ti; + + for (;;) { + trace_nilfs2_transaction_transition(sb, ti, ti->ti_count, + ti->ti_flags, TRACE_NILFS2_TRANSACTION_TRYLOCK); + + down_write(&nilfs->ns_segctor_sem); + if (!test_bit(NILFS_SC_PRIOR_FLUSH, &sci->sc_flags)) + break; + + nilfs_segctor_do_immediate_flush(sci); + + up_write(&nilfs->ns_segctor_sem); + cond_resched(); + } + if (gcflag) + ti->ti_flags |= NILFS_TI_GC; + + trace_nilfs2_transaction_transition(sb, ti, ti->ti_count, + ti->ti_flags, TRACE_NILFS2_TRANSACTION_LOCK); +} + +static void nilfs_transaction_unlock(struct super_block *sb) +{ + struct nilfs_transaction_info *ti = current->journal_info; + struct the_nilfs *nilfs = sb->s_fs_info; + + BUG_ON(ti == NULL || ti->ti_magic != NILFS_TI_MAGIC); + BUG_ON(ti->ti_count > 0); + + up_write(&nilfs->ns_segctor_sem); + current->journal_info = ti->ti_save; + + trace_nilfs2_transaction_transition(sb, ti, ti->ti_count, + ti->ti_flags, TRACE_NILFS2_TRANSACTION_UNLOCK); +} + +static void *nilfs_segctor_map_segsum_entry(struct nilfs_sc_info *sci, + struct nilfs_segsum_pointer *ssp, + unsigned int bytes) +{ + struct nilfs_segment_buffer *segbuf = sci->sc_curseg; + unsigned int blocksize = sci->sc_super->s_blocksize; + void *p; + + if (unlikely(ssp->offset + bytes > blocksize)) { + ssp->offset = 0; + BUG_ON(NILFS_SEGBUF_BH_IS_LAST(ssp->bh, + &segbuf->sb_segsum_buffers)); + ssp->bh = NILFS_SEGBUF_NEXT_BH(ssp->bh); + } + p = ssp->bh->b_data + ssp->offset; + ssp->offset += bytes; + return p; +} + +/** + * nilfs_segctor_reset_segment_buffer - reset the current segment buffer + * @sci: nilfs_sc_info + * + * Return: 0 on success, or a negative error code on failure. + */ +static int nilfs_segctor_reset_segment_buffer(struct nilfs_sc_info *sci) +{ + struct nilfs_segment_buffer *segbuf = sci->sc_curseg; + struct buffer_head *sumbh; + unsigned int sumbytes; + unsigned int flags = 0; + int err; + + if (nilfs_doing_gc()) + flags = NILFS_SS_GC; + err = nilfs_segbuf_reset(segbuf, flags, sci->sc_seg_ctime, sci->sc_cno); + if (unlikely(err)) + return err; + + sumbh = NILFS_SEGBUF_FIRST_BH(&segbuf->sb_segsum_buffers); + sumbytes = segbuf->sb_sum.sumbytes; + sci->sc_finfo_ptr.bh = sumbh; sci->sc_finfo_ptr.offset = sumbytes; + sci->sc_binfo_ptr.bh = sumbh; sci->sc_binfo_ptr.offset = sumbytes; + sci->sc_blk_cnt = sci->sc_datablk_cnt = 0; + return 0; +} + +/** + * nilfs_segctor_zeropad_segsum - zero pad the rest of the segment summary area + * @sci: segment constructor object + * + * nilfs_segctor_zeropad_segsum() zero-fills unallocated space at the end of + * the current segment summary block. + */ +static void nilfs_segctor_zeropad_segsum(struct nilfs_sc_info *sci) +{ + struct nilfs_segsum_pointer *ssp; + + ssp = sci->sc_blk_cnt > 0 ? &sci->sc_binfo_ptr : &sci->sc_finfo_ptr; + if (ssp->offset < ssp->bh->b_size) + memset(ssp->bh->b_data + ssp->offset, 0, + ssp->bh->b_size - ssp->offset); +} + +static int nilfs_segctor_feed_segment(struct nilfs_sc_info *sci) +{ + sci->sc_nblk_this_inc += sci->sc_curseg->sb_sum.nblocks; + if (NILFS_SEGBUF_IS_LAST(sci->sc_curseg, &sci->sc_segbufs)) + return -E2BIG; /* + * The current segment is filled up + * (internal code) + */ + nilfs_segctor_zeropad_segsum(sci); + sci->sc_curseg = NILFS_NEXT_SEGBUF(sci->sc_curseg); + return nilfs_segctor_reset_segment_buffer(sci); +} + +static int nilfs_segctor_add_super_root(struct nilfs_sc_info *sci) +{ + struct nilfs_segment_buffer *segbuf = sci->sc_curseg; + int err; + + if (segbuf->sb_sum.nblocks >= segbuf->sb_rest_blocks) { + err = nilfs_segctor_feed_segment(sci); + if (err) + return err; + segbuf = sci->sc_curseg; + } + err = nilfs_segbuf_extend_payload(segbuf, &segbuf->sb_super_root); + if (likely(!err)) + segbuf->sb_sum.flags |= NILFS_SS_SR; + return err; +} + +/* + * Functions for making segment summary and payloads + */ +static int nilfs_segctor_segsum_block_required( + struct nilfs_sc_info *sci, const struct nilfs_segsum_pointer *ssp, + unsigned int binfo_size) +{ + unsigned int blocksize = sci->sc_super->s_blocksize; + /* Size of finfo and binfo is enough small against blocksize */ + + return ssp->offset + binfo_size + + (!sci->sc_blk_cnt ? sizeof(struct nilfs_finfo) : 0) > + blocksize; +} + +static void nilfs_segctor_begin_finfo(struct nilfs_sc_info *sci, + struct inode *inode) +{ + sci->sc_curseg->sb_sum.nfinfo++; + sci->sc_binfo_ptr = sci->sc_finfo_ptr; + nilfs_segctor_map_segsum_entry( + sci, &sci->sc_binfo_ptr, sizeof(struct nilfs_finfo)); + + if (NILFS_I(inode)->i_root && + !test_bit(NILFS_SC_HAVE_DELTA, &sci->sc_flags)) + set_bit(NILFS_SC_HAVE_DELTA, &sci->sc_flags); + /* skip finfo */ +} + +static void nilfs_segctor_end_finfo(struct nilfs_sc_info *sci, + struct inode *inode) +{ + struct nilfs_finfo *finfo; + struct nilfs_inode_info *ii; + struct nilfs_segment_buffer *segbuf; + __u64 cno; + + if (sci->sc_blk_cnt == 0) + return; + + ii = NILFS_I(inode); + + if (ii->i_type & NILFS_I_TYPE_GC) + cno = ii->i_cno; + else if (NILFS_ROOT_METADATA_FILE(inode->i_ino)) + cno = 0; + else + cno = sci->sc_cno; + + finfo = nilfs_segctor_map_segsum_entry(sci, &sci->sc_finfo_ptr, + sizeof(*finfo)); + finfo->fi_ino = cpu_to_le64(inode->i_ino); + finfo->fi_nblocks = cpu_to_le32(sci->sc_blk_cnt); + finfo->fi_ndatablk = cpu_to_le32(sci->sc_datablk_cnt); + finfo->fi_cno = cpu_to_le64(cno); + + segbuf = sci->sc_curseg; + segbuf->sb_sum.sumbytes = sci->sc_binfo_ptr.offset + + sci->sc_super->s_blocksize * (segbuf->sb_sum.nsumblk - 1); + sci->sc_finfo_ptr = sci->sc_binfo_ptr; + sci->sc_blk_cnt = sci->sc_datablk_cnt = 0; +} + +static int nilfs_segctor_add_file_block(struct nilfs_sc_info *sci, + struct buffer_head *bh, + struct inode *inode, + unsigned int binfo_size) +{ + struct nilfs_segment_buffer *segbuf; + int required, err = 0; + + retry: + segbuf = sci->sc_curseg; + required = nilfs_segctor_segsum_block_required( + sci, &sci->sc_binfo_ptr, binfo_size); + if (segbuf->sb_sum.nblocks + required + 1 > segbuf->sb_rest_blocks) { + nilfs_segctor_end_finfo(sci, inode); + err = nilfs_segctor_feed_segment(sci); + if (err) + return err; + goto retry; + } + if (unlikely(required)) { + nilfs_segctor_zeropad_segsum(sci); + err = nilfs_segbuf_extend_segsum(segbuf); + if (unlikely(err)) + goto failed; + } + if (sci->sc_blk_cnt == 0) + nilfs_segctor_begin_finfo(sci, inode); + + nilfs_segctor_map_segsum_entry(sci, &sci->sc_binfo_ptr, binfo_size); + /* Substitution to vblocknr is delayed until update_blocknr() */ + nilfs_segbuf_add_file_buffer(segbuf, bh); + sci->sc_blk_cnt++; + failed: + return err; +} + +/* + * Callback functions that enumerate, mark, and collect dirty blocks + */ +static int nilfs_collect_file_data(struct nilfs_sc_info *sci, + struct buffer_head *bh, struct inode *inode) +{ + int err; + + err = nilfs_bmap_propagate(NILFS_I(inode)->i_bmap, bh); + if (err < 0) + return err; + + err = nilfs_segctor_add_file_block(sci, bh, inode, + sizeof(struct nilfs_binfo_v)); + if (!err) + sci->sc_datablk_cnt++; + return err; +} + +static int nilfs_collect_file_node(struct nilfs_sc_info *sci, + struct buffer_head *bh, + struct inode *inode) +{ + return nilfs_bmap_propagate(NILFS_I(inode)->i_bmap, bh); +} + +static int nilfs_collect_file_bmap(struct nilfs_sc_info *sci, + struct buffer_head *bh, + struct inode *inode) +{ + WARN_ON(!buffer_dirty(bh)); + return nilfs_segctor_add_file_block(sci, bh, inode, sizeof(__le64)); +} + +static void nilfs_write_file_data_binfo(struct nilfs_sc_info *sci, + struct nilfs_segsum_pointer *ssp, + union nilfs_binfo *binfo) +{ + struct nilfs_binfo_v *binfo_v = nilfs_segctor_map_segsum_entry( + sci, ssp, sizeof(*binfo_v)); + *binfo_v = binfo->bi_v; +} + +static void nilfs_write_file_node_binfo(struct nilfs_sc_info *sci, + struct nilfs_segsum_pointer *ssp, + union nilfs_binfo *binfo) +{ + __le64 *vblocknr = nilfs_segctor_map_segsum_entry( + sci, ssp, sizeof(*vblocknr)); + *vblocknr = binfo->bi_v.bi_vblocknr; +} + +static const struct nilfs_sc_operations nilfs_sc_file_ops = { + .collect_data = nilfs_collect_file_data, + .collect_node = nilfs_collect_file_node, + .collect_bmap = nilfs_collect_file_bmap, + .write_data_binfo = nilfs_write_file_data_binfo, + .write_node_binfo = nilfs_write_file_node_binfo, +}; + +static int nilfs_collect_dat_data(struct nilfs_sc_info *sci, + struct buffer_head *bh, struct inode *inode) +{ + int err; + + err = nilfs_bmap_propagate(NILFS_I(inode)->i_bmap, bh); + if (err < 0) + return err; + + err = nilfs_segctor_add_file_block(sci, bh, inode, sizeof(__le64)); + if (!err) + sci->sc_datablk_cnt++; + return err; +} + +static int nilfs_collect_dat_bmap(struct nilfs_sc_info *sci, + struct buffer_head *bh, struct inode *inode) +{ + WARN_ON(!buffer_dirty(bh)); + return nilfs_segctor_add_file_block(sci, bh, inode, + sizeof(struct nilfs_binfo_dat)); +} + +static void nilfs_write_dat_data_binfo(struct nilfs_sc_info *sci, + struct nilfs_segsum_pointer *ssp, + union nilfs_binfo *binfo) +{ + __le64 *blkoff = nilfs_segctor_map_segsum_entry(sci, ssp, + sizeof(*blkoff)); + *blkoff = binfo->bi_dat.bi_blkoff; +} + +static void nilfs_write_dat_node_binfo(struct nilfs_sc_info *sci, + struct nilfs_segsum_pointer *ssp, + union nilfs_binfo *binfo) +{ + struct nilfs_binfo_dat *binfo_dat = + nilfs_segctor_map_segsum_entry(sci, ssp, sizeof(*binfo_dat)); + *binfo_dat = binfo->bi_dat; +} + +static const struct nilfs_sc_operations nilfs_sc_dat_ops = { + .collect_data = nilfs_collect_dat_data, + .collect_node = nilfs_collect_file_node, + .collect_bmap = nilfs_collect_dat_bmap, + .write_data_binfo = nilfs_write_dat_data_binfo, + .write_node_binfo = nilfs_write_dat_node_binfo, +}; + +static const struct nilfs_sc_operations nilfs_sc_dsync_ops = { + .collect_data = nilfs_collect_file_data, + .collect_node = NULL, + .collect_bmap = NULL, + .write_data_binfo = nilfs_write_file_data_binfo, + .write_node_binfo = NULL, +}; + +static size_t nilfs_lookup_dirty_data_buffers(struct inode *inode, + struct list_head *listp, + size_t nlimit, + loff_t start, loff_t end) +{ + struct address_space *mapping = inode->i_mapping; + struct folio_batch fbatch; + pgoff_t index = 0, last = ULONG_MAX; + size_t ndirties = 0; + int i; + + if (unlikely(start != 0 || end != LLONG_MAX)) { + /* + * A valid range is given for sync-ing data pages. The + * range is rounded to per-page; extra dirty buffers + * may be included if blocksize < pagesize. + */ + index = start >> PAGE_SHIFT; + last = end >> PAGE_SHIFT; + } + folio_batch_init(&fbatch); + repeat: + if (unlikely(index > last) || + !filemap_get_folios_tag(mapping, &index, last, + PAGECACHE_TAG_DIRTY, &fbatch)) + return ndirties; + + for (i = 0; i < folio_batch_count(&fbatch); i++) { + struct buffer_head *bh, *head; + struct folio *folio = fbatch.folios[i]; + + folio_lock(folio); + if (unlikely(folio->mapping != mapping)) { + /* Exclude folios removed from the address space */ + folio_unlock(folio); + continue; + } + head = folio_buffers(folio); + if (!head) + head = create_empty_buffers(folio, + i_blocksize(inode), 0); + + bh = head; + do { + if (!buffer_dirty(bh) || buffer_async_write(bh)) + continue; + get_bh(bh); + list_add_tail(&bh->b_assoc_buffers, listp); + ndirties++; + if (unlikely(ndirties >= nlimit)) { + folio_unlock(folio); + folio_batch_release(&fbatch); + cond_resched(); + return ndirties; + } + } while (bh = bh->b_this_page, bh != head); + + folio_unlock(folio); + } + folio_batch_release(&fbatch); + cond_resched(); + goto repeat; +} + +static void nilfs_lookup_dirty_node_buffers(struct inode *inode, + struct list_head *listp) +{ + struct nilfs_inode_info *ii = NILFS_I(inode); + struct inode *btnc_inode = ii->i_assoc_inode; + struct folio_batch fbatch; + struct buffer_head *bh, *head; + unsigned int i; + pgoff_t index = 0; + + if (!btnc_inode) + return; + folio_batch_init(&fbatch); + + while (filemap_get_folios_tag(btnc_inode->i_mapping, &index, + (pgoff_t)-1, PAGECACHE_TAG_DIRTY, &fbatch)) { + for (i = 0; i < folio_batch_count(&fbatch); i++) { + bh = head = folio_buffers(fbatch.folios[i]); + do { + if (buffer_dirty(bh) && + !buffer_async_write(bh)) { + get_bh(bh); + list_add_tail(&bh->b_assoc_buffers, + listp); + } + bh = bh->b_this_page; + } while (bh != head); + } + folio_batch_release(&fbatch); + cond_resched(); + } +} + +static void nilfs_dispose_list(struct the_nilfs *nilfs, + struct list_head *head, int force) +{ + struct nilfs_inode_info *ii, *n; + struct nilfs_inode_info *ivec[SC_N_INODEVEC], **pii; + unsigned int nv = 0; + + while (!list_empty(head)) { + spin_lock(&nilfs->ns_inode_lock); + list_for_each_entry_safe(ii, n, head, i_dirty) { + list_del_init(&ii->i_dirty); + if (force) { + if (unlikely(ii->i_bh)) { + brelse(ii->i_bh); + ii->i_bh = NULL; + } + } else if (test_bit(NILFS_I_DIRTY, &ii->i_state)) { + set_bit(NILFS_I_QUEUED, &ii->i_state); + list_add_tail(&ii->i_dirty, + &nilfs->ns_dirty_files); + continue; + } + ivec[nv++] = ii; + if (nv == SC_N_INODEVEC) + break; + } + spin_unlock(&nilfs->ns_inode_lock); + + for (pii = ivec; nv > 0; pii++, nv--) + iput(&(*pii)->vfs_inode); + } +} + +static void nilfs_iput_work_func(struct work_struct *work) +{ + struct nilfs_sc_info *sci = container_of(work, struct nilfs_sc_info, + sc_iput_work); + struct the_nilfs *nilfs = sci->sc_super->s_fs_info; + + nilfs_dispose_list(nilfs, &sci->sc_iput_queue, 0); +} + +static int nilfs_test_metadata_dirty(struct the_nilfs *nilfs, + struct nilfs_root *root) +{ + int ret = 0; + + if (nilfs_mdt_fetch_dirty(root->ifile)) + ret++; + if (nilfs_mdt_fetch_dirty(nilfs->ns_cpfile)) + ret++; + if (nilfs_mdt_fetch_dirty(nilfs->ns_sufile)) + ret++; + if ((ret || nilfs_doing_gc()) && nilfs_mdt_fetch_dirty(nilfs->ns_dat)) + ret++; + return ret; +} + +static int nilfs_segctor_clean(struct nilfs_sc_info *sci) +{ + return list_empty(&sci->sc_dirty_files) && + !test_bit(NILFS_SC_DIRTY, &sci->sc_flags) && + sci->sc_nfreesegs == 0 && + (!nilfs_doing_gc() || list_empty(&sci->sc_gc_inodes)); +} + +static int nilfs_segctor_confirm(struct nilfs_sc_info *sci) +{ + struct the_nilfs *nilfs = sci->sc_super->s_fs_info; + int ret = 0; + + if (nilfs_test_metadata_dirty(nilfs, sci->sc_root)) + set_bit(NILFS_SC_DIRTY, &sci->sc_flags); + + spin_lock(&nilfs->ns_inode_lock); + if (list_empty(&nilfs->ns_dirty_files) && nilfs_segctor_clean(sci)) + ret++; + + spin_unlock(&nilfs->ns_inode_lock); + return ret; +} + +static void nilfs_segctor_clear_metadata_dirty(struct nilfs_sc_info *sci) +{ + struct the_nilfs *nilfs = sci->sc_super->s_fs_info; + + nilfs_mdt_clear_dirty(sci->sc_root->ifile); + nilfs_mdt_clear_dirty(nilfs->ns_cpfile); + nilfs_mdt_clear_dirty(nilfs->ns_sufile); + nilfs_mdt_clear_dirty(nilfs->ns_dat); +} + +static void nilfs_fill_in_file_bmap(struct inode *ifile, + struct nilfs_inode_info *ii) + +{ + struct buffer_head *ibh; + struct nilfs_inode *raw_inode; + + if (test_bit(NILFS_I_BMAP, &ii->i_state)) { + ibh = ii->i_bh; + BUG_ON(!ibh); + raw_inode = nilfs_ifile_map_inode(ifile, ii->vfs_inode.i_ino, + ibh); + nilfs_bmap_write(ii->i_bmap, raw_inode); + nilfs_ifile_unmap_inode(raw_inode); + } +} + +static void nilfs_segctor_fill_in_file_bmap(struct nilfs_sc_info *sci) +{ + struct nilfs_inode_info *ii; + + list_for_each_entry(ii, &sci->sc_dirty_files, i_dirty) { + nilfs_fill_in_file_bmap(sci->sc_root->ifile, ii); + set_bit(NILFS_I_COLLECTED, &ii->i_state); + } +} + +/** + * nilfs_write_root_mdt_inode - export root metadata inode information to + * the on-disk inode + * @inode: inode object of the root metadata file + * @raw_inode: on-disk inode + * + * nilfs_write_root_mdt_inode() writes inode information and bmap data of + * @inode to the inode area of the metadata file allocated on the super root + * block created to finalize the log. Since super root blocks are configured + * each time, this function zero-fills the unused area of @raw_inode. + */ +static void nilfs_write_root_mdt_inode(struct inode *inode, + struct nilfs_inode *raw_inode) +{ + struct the_nilfs *nilfs = inode->i_sb->s_fs_info; + + nilfs_write_inode_common(inode, raw_inode); + + /* zero-fill unused portion of raw_inode */ + raw_inode->i_xattr = 0; + raw_inode->i_pad = 0; + memset((void *)raw_inode + sizeof(*raw_inode), 0, + nilfs->ns_inode_size - sizeof(*raw_inode)); + + nilfs_bmap_write(NILFS_I(inode)->i_bmap, raw_inode); +} + +static void nilfs_segctor_fill_in_super_root(struct nilfs_sc_info *sci, + struct the_nilfs *nilfs) +{ + struct buffer_head *bh_sr; + struct nilfs_super_root *raw_sr; + unsigned int isz, srsz; + + bh_sr = NILFS_LAST_SEGBUF(&sci->sc_segbufs)->sb_super_root; + + lock_buffer(bh_sr); + raw_sr = (struct nilfs_super_root *)bh_sr->b_data; + isz = nilfs->ns_inode_size; + srsz = NILFS_SR_BYTES(isz); + + raw_sr->sr_sum = 0; /* Ensure initialization within this update */ + raw_sr->sr_bytes = cpu_to_le16(srsz); + raw_sr->sr_nongc_ctime + = cpu_to_le64(nilfs_doing_gc() ? + nilfs->ns_nongc_ctime : sci->sc_seg_ctime); + raw_sr->sr_flags = 0; + + nilfs_write_root_mdt_inode(nilfs->ns_dat, (void *)raw_sr + + NILFS_SR_DAT_OFFSET(isz)); + nilfs_write_root_mdt_inode(nilfs->ns_cpfile, (void *)raw_sr + + NILFS_SR_CPFILE_OFFSET(isz)); + nilfs_write_root_mdt_inode(nilfs->ns_sufile, (void *)raw_sr + + NILFS_SR_SUFILE_OFFSET(isz)); + + memset((void *)raw_sr + srsz, 0, nilfs->ns_blocksize - srsz); + set_buffer_uptodate(bh_sr); + unlock_buffer(bh_sr); +} + +static void nilfs_redirty_inodes(struct list_head *head) +{ + struct nilfs_inode_info *ii; + + list_for_each_entry(ii, head, i_dirty) { + if (test_bit(NILFS_I_COLLECTED, &ii->i_state)) + clear_bit(NILFS_I_COLLECTED, &ii->i_state); + } +} + +static void nilfs_drop_collected_inodes(struct list_head *head) +{ + struct nilfs_inode_info *ii; + + list_for_each_entry(ii, head, i_dirty) { + if (!test_and_clear_bit(NILFS_I_COLLECTED, &ii->i_state)) + continue; + + clear_bit(NILFS_I_INODE_SYNC, &ii->i_state); + set_bit(NILFS_I_UPDATED, &ii->i_state); + } +} + +static int nilfs_segctor_apply_buffers(struct nilfs_sc_info *sci, + struct inode *inode, + struct list_head *listp, + int (*collect)(struct nilfs_sc_info *, + struct buffer_head *, + struct inode *)) +{ + struct buffer_head *bh, *n; + int err = 0; + + if (collect) { + list_for_each_entry_safe(bh, n, listp, b_assoc_buffers) { + list_del_init(&bh->b_assoc_buffers); + err = collect(sci, bh, inode); + brelse(bh); + if (unlikely(err)) + goto dispose_buffers; + } + return 0; + } + + dispose_buffers: + while (!list_empty(listp)) { + bh = list_first_entry(listp, struct buffer_head, + b_assoc_buffers); + list_del_init(&bh->b_assoc_buffers); + brelse(bh); + } + return err; +} + +static size_t nilfs_segctor_buffer_rest(struct nilfs_sc_info *sci) +{ + /* Remaining number of blocks within segment buffer */ + return sci->sc_segbuf_nblocks - + (sci->sc_nblk_this_inc + sci->sc_curseg->sb_sum.nblocks); +} + +static int nilfs_segctor_scan_file(struct nilfs_sc_info *sci, + struct inode *inode, + const struct nilfs_sc_operations *sc_ops) +{ + LIST_HEAD(data_buffers); + LIST_HEAD(node_buffers); + int err; + + if (!(sci->sc_stage.flags & NILFS_CF_NODE)) { + size_t n, rest = nilfs_segctor_buffer_rest(sci); + + n = nilfs_lookup_dirty_data_buffers( + inode, &data_buffers, rest + 1, 0, LLONG_MAX); + if (n > rest) { + err = nilfs_segctor_apply_buffers( + sci, inode, &data_buffers, + sc_ops->collect_data); + BUG_ON(!err); /* always receive -E2BIG or true error */ + goto break_or_fail; + } + } + nilfs_lookup_dirty_node_buffers(inode, &node_buffers); + + if (!(sci->sc_stage.flags & NILFS_CF_NODE)) { + err = nilfs_segctor_apply_buffers( + sci, inode, &data_buffers, sc_ops->collect_data); + if (unlikely(err)) { + /* dispose node list */ + nilfs_segctor_apply_buffers( + sci, inode, &node_buffers, NULL); + goto break_or_fail; + } + sci->sc_stage.flags |= NILFS_CF_NODE; + } + /* Collect node */ + err = nilfs_segctor_apply_buffers( + sci, inode, &node_buffers, sc_ops->collect_node); + if (unlikely(err)) + goto break_or_fail; + + nilfs_bmap_lookup_dirty_buffers(NILFS_I(inode)->i_bmap, &node_buffers); + err = nilfs_segctor_apply_buffers( + sci, inode, &node_buffers, sc_ops->collect_bmap); + if (unlikely(err)) + goto break_or_fail; + + nilfs_segctor_end_finfo(sci, inode); + sci->sc_stage.flags &= ~NILFS_CF_NODE; + + break_or_fail: + return err; +} + +static int nilfs_segctor_scan_file_dsync(struct nilfs_sc_info *sci, + struct inode *inode) +{ + LIST_HEAD(data_buffers); + size_t n, rest = nilfs_segctor_buffer_rest(sci); + int err; + + n = nilfs_lookup_dirty_data_buffers(inode, &data_buffers, rest + 1, + sci->sc_dsync_start, + sci->sc_dsync_end); + + err = nilfs_segctor_apply_buffers(sci, inode, &data_buffers, + nilfs_collect_file_data); + if (!err) { + nilfs_segctor_end_finfo(sci, inode); + BUG_ON(n > rest); + /* always receive -E2BIG or true error if n > rest */ + } + return err; +} + +/** + * nilfs_free_segments - free the segments given by an array of segment numbers + * @nilfs: nilfs object + * @segnumv: array of segment numbers to be freed + * @nsegs: number of segments to be freed in @segnumv + * + * nilfs_free_segments() wraps nilfs_sufile_freev() and + * nilfs_sufile_cancel_freev(), and edits the segment usage metadata file + * (sufile) to free all segments given by @segnumv and @nsegs at once. If + * it fails midway, it cancels the changes so that none of the segments are + * freed. If @nsegs is 0, this function does nothing. + * + * The freeing of segments is not finalized until the writing of a log with + * a super root block containing this sufile change is complete, and it can + * be canceled with nilfs_sufile_cancel_freev() until then. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EINVAL - Invalid segment number. + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOMEM - Insufficient memory available. + */ +static int nilfs_free_segments(struct the_nilfs *nilfs, __u64 *segnumv, + size_t nsegs) +{ + size_t ndone; + int ret; + + if (!nsegs) + return 0; + + ret = nilfs_sufile_freev(nilfs->ns_sufile, segnumv, nsegs, &ndone); + if (unlikely(ret)) { + nilfs_sufile_cancel_freev(nilfs->ns_sufile, segnumv, ndone, + NULL); + /* + * If a segment usage of the segments to be freed is in a + * hole block, nilfs_sufile_freev() will return -ENOENT. + * In this case, -EINVAL should be returned to the caller + * since there is something wrong with the given segment + * number array. This error can only occur during GC, so + * there is no need to worry about it propagating to other + * callers (such as fsync). + */ + if (ret == -ENOENT) { + nilfs_err(nilfs->ns_sb, + "The segment usage entry %llu to be freed is invalid (in a hole)", + (unsigned long long)segnumv[ndone]); + ret = -EINVAL; + } + } + return ret; +} + +static int nilfs_segctor_collect_blocks(struct nilfs_sc_info *sci, int mode) +{ + struct the_nilfs *nilfs = sci->sc_super->s_fs_info; + struct list_head *head; + struct nilfs_inode_info *ii; + int err = 0; + + switch (nilfs_sc_cstage_get(sci)) { + case NILFS_ST_INIT: + /* Pre-processes */ + sci->sc_stage.flags = 0; + + if (!test_bit(NILFS_SC_UNCLOSED, &sci->sc_flags)) { + sci->sc_nblk_inc = 0; + sci->sc_curseg->sb_sum.flags = NILFS_SS_LOGBGN; + if (mode == SC_LSEG_DSYNC) { + nilfs_sc_cstage_set(sci, NILFS_ST_DSYNC); + goto dsync_mode; + } + } + + sci->sc_stage.dirty_file_ptr = NULL; + sci->sc_stage.gc_inode_ptr = NULL; + if (mode == SC_FLUSH_DAT) { + nilfs_sc_cstage_set(sci, NILFS_ST_DAT); + goto dat_stage; + } + nilfs_sc_cstage_inc(sci); + fallthrough; + case NILFS_ST_GC: + if (nilfs_doing_gc()) { + head = &sci->sc_gc_inodes; + ii = list_prepare_entry(sci->sc_stage.gc_inode_ptr, + head, i_dirty); + list_for_each_entry_continue(ii, head, i_dirty) { + err = nilfs_segctor_scan_file( + sci, &ii->vfs_inode, + &nilfs_sc_file_ops); + if (unlikely(err)) { + sci->sc_stage.gc_inode_ptr = list_entry( + ii->i_dirty.prev, + struct nilfs_inode_info, + i_dirty); + goto break_or_fail; + } + set_bit(NILFS_I_COLLECTED, &ii->i_state); + } + sci->sc_stage.gc_inode_ptr = NULL; + } + nilfs_sc_cstage_inc(sci); + fallthrough; + case NILFS_ST_FILE: + head = &sci->sc_dirty_files; + ii = list_prepare_entry(sci->sc_stage.dirty_file_ptr, head, + i_dirty); + list_for_each_entry_continue(ii, head, i_dirty) { + clear_bit(NILFS_I_DIRTY, &ii->i_state); + + err = nilfs_segctor_scan_file(sci, &ii->vfs_inode, + &nilfs_sc_file_ops); + if (unlikely(err)) { + sci->sc_stage.dirty_file_ptr = + list_entry(ii->i_dirty.prev, + struct nilfs_inode_info, + i_dirty); + goto break_or_fail; + } + /* sci->sc_stage.dirty_file_ptr = NILFS_I(inode); */ + /* XXX: required ? */ + } + sci->sc_stage.dirty_file_ptr = NULL; + if (mode == SC_FLUSH_FILE) { + nilfs_sc_cstage_set(sci, NILFS_ST_DONE); + return 0; + } + nilfs_sc_cstage_inc(sci); + sci->sc_stage.flags |= NILFS_CF_IFILE_STARTED; + fallthrough; + case NILFS_ST_IFILE: + err = nilfs_segctor_scan_file(sci, sci->sc_root->ifile, + &nilfs_sc_file_ops); + if (unlikely(err)) + break; + nilfs_sc_cstage_inc(sci); + /* Creating a checkpoint */ + err = nilfs_cpfile_create_checkpoint(nilfs->ns_cpfile, + nilfs->ns_cno); + if (unlikely(err)) + break; + fallthrough; + case NILFS_ST_CPFILE: + err = nilfs_segctor_scan_file(sci, nilfs->ns_cpfile, + &nilfs_sc_file_ops); + if (unlikely(err)) + break; + nilfs_sc_cstage_inc(sci); + fallthrough; + case NILFS_ST_SUFILE: + err = nilfs_free_segments(nilfs, sci->sc_freesegs, + sci->sc_nfreesegs); + if (unlikely(err)) + break; + sci->sc_stage.flags |= NILFS_CF_SUFREED; + + err = nilfs_segctor_scan_file(sci, nilfs->ns_sufile, + &nilfs_sc_file_ops); + if (unlikely(err)) + break; + nilfs_sc_cstage_inc(sci); + fallthrough; + case NILFS_ST_DAT: + dat_stage: + err = nilfs_segctor_scan_file(sci, nilfs->ns_dat, + &nilfs_sc_dat_ops); + if (unlikely(err)) + break; + if (mode == SC_FLUSH_DAT) { + nilfs_sc_cstage_set(sci, NILFS_ST_DONE); + return 0; + } + nilfs_sc_cstage_inc(sci); + fallthrough; + case NILFS_ST_SR: + if (mode == SC_LSEG_SR) { + /* Appending a super root */ + err = nilfs_segctor_add_super_root(sci); + if (unlikely(err)) + break; + } + /* End of a logical segment */ + sci->sc_curseg->sb_sum.flags |= NILFS_SS_LOGEND; + nilfs_sc_cstage_set(sci, NILFS_ST_DONE); + return 0; + case NILFS_ST_DSYNC: + dsync_mode: + sci->sc_curseg->sb_sum.flags |= NILFS_SS_SYNDT; + ii = sci->sc_dsync_inode; + if (!test_bit(NILFS_I_BUSY, &ii->i_state)) + break; + + err = nilfs_segctor_scan_file_dsync(sci, &ii->vfs_inode); + if (unlikely(err)) + break; + sci->sc_curseg->sb_sum.flags |= NILFS_SS_LOGEND; + nilfs_sc_cstage_set(sci, NILFS_ST_DONE); + return 0; + case NILFS_ST_DONE: + return 0; + default: + BUG(); + } + + break_or_fail: + return err; +} + +/** + * nilfs_segctor_begin_construction - setup segment buffer to make a new log + * @sci: nilfs_sc_info + * @nilfs: nilfs object + * + * Return: 0 on success, or a negative error code on failure. + */ +static int nilfs_segctor_begin_construction(struct nilfs_sc_info *sci, + struct the_nilfs *nilfs) +{ + struct nilfs_segment_buffer *segbuf, *prev; + __u64 nextnum; + int err, alloc = 0; + + segbuf = nilfs_segbuf_new(sci->sc_super); + if (unlikely(!segbuf)) + return -ENOMEM; + + if (list_empty(&sci->sc_write_logs)) { + nilfs_segbuf_map(segbuf, nilfs->ns_segnum, + nilfs->ns_pseg_offset, nilfs); + if (segbuf->sb_rest_blocks < NILFS_PSEG_MIN_BLOCKS) { + nilfs_shift_to_next_segment(nilfs); + nilfs_segbuf_map(segbuf, nilfs->ns_segnum, 0, nilfs); + } + + segbuf->sb_sum.seg_seq = nilfs->ns_seg_seq; + nextnum = nilfs->ns_nextnum; + + if (nilfs->ns_segnum == nilfs->ns_nextnum) + /* Start from the head of a new full segment */ + alloc++; + } else { + /* Continue logs */ + prev = NILFS_LAST_SEGBUF(&sci->sc_write_logs); + nilfs_segbuf_map_cont(segbuf, prev); + segbuf->sb_sum.seg_seq = prev->sb_sum.seg_seq; + nextnum = prev->sb_nextnum; + + if (segbuf->sb_rest_blocks < NILFS_PSEG_MIN_BLOCKS) { + nilfs_segbuf_map(segbuf, prev->sb_nextnum, 0, nilfs); + segbuf->sb_sum.seg_seq++; + alloc++; + } + } + + err = nilfs_sufile_mark_dirty(nilfs->ns_sufile, segbuf->sb_segnum); + if (err) + goto failed; + + if (alloc) { + err = nilfs_sufile_alloc(nilfs->ns_sufile, &nextnum); + if (err) + goto failed; + } + nilfs_segbuf_set_next_segnum(segbuf, nextnum, nilfs); + + BUG_ON(!list_empty(&sci->sc_segbufs)); + list_add_tail(&segbuf->sb_list, &sci->sc_segbufs); + sci->sc_segbuf_nblocks = segbuf->sb_rest_blocks; + return 0; + + failed: + nilfs_segbuf_free(segbuf); + return err; +} + +static int nilfs_segctor_extend_segments(struct nilfs_sc_info *sci, + struct the_nilfs *nilfs, int nadd) +{ + struct nilfs_segment_buffer *segbuf, *prev; + struct inode *sufile = nilfs->ns_sufile; + __u64 nextnextnum; + LIST_HEAD(list); + int err, ret, i; + + prev = NILFS_LAST_SEGBUF(&sci->sc_segbufs); + /* + * Since the segment specified with nextnum might be allocated during + * the previous construction, the buffer including its segusage may + * not be dirty. The following call ensures that the buffer is dirty + * and will pin the buffer on memory until the sufile is written. + */ + err = nilfs_sufile_mark_dirty(sufile, prev->sb_nextnum); + if (unlikely(err)) + return err; + + for (i = 0; i < nadd; i++) { + /* extend segment info */ + err = -ENOMEM; + segbuf = nilfs_segbuf_new(sci->sc_super); + if (unlikely(!segbuf)) + goto failed; + + /* map this buffer to region of segment on-disk */ + nilfs_segbuf_map(segbuf, prev->sb_nextnum, 0, nilfs); + sci->sc_segbuf_nblocks += segbuf->sb_rest_blocks; + + /* allocate the next next full segment */ + err = nilfs_sufile_alloc(sufile, &nextnextnum); + if (unlikely(err)) + goto failed_segbuf; + + segbuf->sb_sum.seg_seq = prev->sb_sum.seg_seq + 1; + nilfs_segbuf_set_next_segnum(segbuf, nextnextnum, nilfs); + + list_add_tail(&segbuf->sb_list, &list); + prev = segbuf; + } + list_splice_tail(&list, &sci->sc_segbufs); + return 0; + + failed_segbuf: + nilfs_segbuf_free(segbuf); + failed: + list_for_each_entry(segbuf, &list, sb_list) { + ret = nilfs_sufile_free(sufile, segbuf->sb_nextnum); + nilfs_sufile_warn_on_error(sufile, ret); + } + nilfs_destroy_logs(&list); + return err; +} + +static void nilfs_free_incomplete_logs(struct list_head *logs, + struct the_nilfs *nilfs) +{ + struct nilfs_segment_buffer *segbuf, *prev; + struct inode *sufile = nilfs->ns_sufile; + int ret; + + segbuf = NILFS_FIRST_SEGBUF(logs); + if (nilfs->ns_nextnum != segbuf->sb_nextnum) { + ret = nilfs_sufile_free(sufile, segbuf->sb_nextnum); + nilfs_sufile_warn_on_error(sufile, ret); + } + if (atomic_read(&segbuf->sb_err)) { + /* Case 1: The first segment failed */ + if (segbuf->sb_pseg_start != segbuf->sb_fseg_start) + /* + * Case 1a: Partial segment appended into an existing + * segment + */ + nilfs_terminate_segment(nilfs, segbuf->sb_fseg_start, + segbuf->sb_fseg_end); + else /* Case 1b: New full segment */ + set_nilfs_discontinued(nilfs); + } + + prev = segbuf; + list_for_each_entry_continue(segbuf, logs, sb_list) { + if (prev->sb_nextnum != segbuf->sb_nextnum) { + ret = nilfs_sufile_free(sufile, segbuf->sb_nextnum); + nilfs_sufile_warn_on_error(sufile, ret); + } + if (atomic_read(&segbuf->sb_err) && + segbuf->sb_segnum != nilfs->ns_nextnum) + /* Case 2: extended segment (!= next) failed */ + nilfs_sufile_set_error(sufile, segbuf->sb_segnum); + prev = segbuf; + } +} + +static void nilfs_segctor_update_segusage(struct nilfs_sc_info *sci, + struct inode *sufile) +{ + struct nilfs_segment_buffer *segbuf; + unsigned long live_blocks; + int ret; + + list_for_each_entry(segbuf, &sci->sc_segbufs, sb_list) { + live_blocks = segbuf->sb_sum.nblocks + + (segbuf->sb_pseg_start - segbuf->sb_fseg_start); + ret = nilfs_sufile_set_segment_usage(sufile, segbuf->sb_segnum, + live_blocks, + sci->sc_seg_ctime); + nilfs_sufile_warn_on_error(sufile, ret); + } +} + +static void nilfs_cancel_segusage(struct list_head *logs, struct inode *sufile) +{ + struct nilfs_segment_buffer *segbuf; + int ret; + + segbuf = NILFS_FIRST_SEGBUF(logs); + ret = nilfs_sufile_set_segment_usage(sufile, segbuf->sb_segnum, + segbuf->sb_pseg_start - + segbuf->sb_fseg_start, 0); + nilfs_sufile_warn_on_error(sufile, ret); + + list_for_each_entry_continue(segbuf, logs, sb_list) { + ret = nilfs_sufile_set_segment_usage(sufile, segbuf->sb_segnum, + 0, 0); + nilfs_sufile_warn_on_error(sufile, ret); + } +} + +static int nilfs_segctor_truncate_segments(struct nilfs_sc_info *sci, + struct nilfs_segment_buffer *last, + struct inode *sufile) +{ + struct nilfs_segment_buffer *segbuf = last; + int ret, err = 0; + + list_for_each_entry_continue(segbuf, &sci->sc_segbufs, sb_list) { + sci->sc_segbuf_nblocks -= segbuf->sb_rest_blocks; + + ret = nilfs_sufile_warn_on_error( + sufile, nilfs_sufile_free(sufile, segbuf->sb_nextnum)); + if (unlikely(ret) && err != -EROFS) + err = ret; + } + nilfs_truncate_logs(&sci->sc_segbufs, last); + return err; +} + + +static int nilfs_segctor_collect(struct nilfs_sc_info *sci, + struct the_nilfs *nilfs, int mode) +{ + struct nilfs_cstage prev_stage = sci->sc_stage; + int err, nadd = 1; + + /* Collection retry loop */ + for (;;) { + sci->sc_nblk_this_inc = 0; + sci->sc_curseg = NILFS_FIRST_SEGBUF(&sci->sc_segbufs); + + err = nilfs_segctor_reset_segment_buffer(sci); + if (unlikely(err)) + goto failed; + + err = nilfs_segctor_collect_blocks(sci, mode); + sci->sc_nblk_this_inc += sci->sc_curseg->sb_sum.nblocks; + if (!err) + break; + + if (unlikely(err != -E2BIG)) + goto failed; + + /* The current segment is filled up */ + if (mode != SC_LSEG_SR || + nilfs_sc_cstage_get(sci) < NILFS_ST_CPFILE) + break; + + nilfs_clear_logs(&sci->sc_segbufs); + + if (sci->sc_stage.flags & NILFS_CF_SUFREED) { + err = nilfs_sufile_cancel_freev(nilfs->ns_sufile, + sci->sc_freesegs, + sci->sc_nfreesegs, + NULL); + nilfs_sufile_warn_on_error(nilfs->ns_sufile, err); + sci->sc_stage.flags &= ~NILFS_CF_SUFREED; + } + + err = nilfs_segctor_extend_segments(sci, nilfs, nadd); + if (unlikely(err)) + return err; + + nadd = min_t(int, nadd << 1, SC_MAX_SEGDELTA); + sci->sc_stage = prev_stage; + } + nilfs_segctor_zeropad_segsum(sci); + err = nilfs_segctor_truncate_segments(sci, sci->sc_curseg, + nilfs->ns_sufile); + + failed: + return err; +} + +static void nilfs_list_replace_buffer(struct buffer_head *old_bh, + struct buffer_head *new_bh) +{ + BUG_ON(!list_empty(&new_bh->b_assoc_buffers)); + + list_replace_init(&old_bh->b_assoc_buffers, &new_bh->b_assoc_buffers); + /* The caller must release old_bh */ +} + +static int +nilfs_segctor_update_payload_blocknr(struct nilfs_sc_info *sci, + struct nilfs_segment_buffer *segbuf, + int mode) +{ + struct inode *inode = NULL; + sector_t blocknr; + unsigned long nfinfo = segbuf->sb_sum.nfinfo; + unsigned long nblocks = 0, ndatablk = 0; + const struct nilfs_sc_operations *sc_op = NULL; + struct nilfs_segsum_pointer ssp; + struct nilfs_finfo *finfo = NULL; + union nilfs_binfo binfo; + struct buffer_head *bh, *bh_org; + u64 ino = 0; + int err = 0; + + if (!nfinfo) + goto out; + + blocknr = segbuf->sb_pseg_start + segbuf->sb_sum.nsumblk; + ssp.bh = NILFS_SEGBUF_FIRST_BH(&segbuf->sb_segsum_buffers); + ssp.offset = sizeof(struct nilfs_segment_summary); + + list_for_each_entry(bh, &segbuf->sb_payload_buffers, b_assoc_buffers) { + if (bh == segbuf->sb_super_root) + break; + if (!finfo) { + finfo = nilfs_segctor_map_segsum_entry( + sci, &ssp, sizeof(*finfo)); + ino = le64_to_cpu(finfo->fi_ino); + nblocks = le32_to_cpu(finfo->fi_nblocks); + ndatablk = le32_to_cpu(finfo->fi_ndatablk); + + inode = bh->b_folio->mapping->host; + + if (mode == SC_LSEG_DSYNC) + sc_op = &nilfs_sc_dsync_ops; + else if (ino == NILFS_DAT_INO) + sc_op = &nilfs_sc_dat_ops; + else /* file blocks */ + sc_op = &nilfs_sc_file_ops; + } + bh_org = bh; + get_bh(bh_org); + err = nilfs_bmap_assign(NILFS_I(inode)->i_bmap, &bh, blocknr, + &binfo); + if (bh != bh_org) + nilfs_list_replace_buffer(bh_org, bh); + brelse(bh_org); + if (unlikely(err)) + goto failed_bmap; + + if (ndatablk > 0) + sc_op->write_data_binfo(sci, &ssp, &binfo); + else + sc_op->write_node_binfo(sci, &ssp, &binfo); + + blocknr++; + if (--nblocks == 0) { + finfo = NULL; + if (--nfinfo == 0) + break; + } else if (ndatablk > 0) + ndatablk--; + } + out: + return 0; + + failed_bmap: + return err; +} + +static int nilfs_segctor_assign(struct nilfs_sc_info *sci, int mode) +{ + struct nilfs_segment_buffer *segbuf; + int err; + + list_for_each_entry(segbuf, &sci->sc_segbufs, sb_list) { + err = nilfs_segctor_update_payload_blocknr(sci, segbuf, mode); + if (unlikely(err)) + return err; + nilfs_segbuf_fill_in_segsum(segbuf); + } + return 0; +} + +static void nilfs_begin_folio_io(struct folio *folio) +{ + if (!folio || folio_test_writeback(folio)) + /* + * For split b-tree node pages, this function may be called + * twice. We ignore the 2nd or later calls by this check. + */ + return; + + folio_lock(folio); + folio_clear_dirty_for_io(folio); + folio_start_writeback(folio); + folio_unlock(folio); +} + +/** + * nilfs_prepare_write_logs - prepare to write logs + * @logs: logs to prepare for writing + * @seed: checksum seed value + * + * nilfs_prepare_write_logs() adds checksums and prepares the block + * buffers/folios for writing logs. In order to stabilize folios of + * memory-mapped file blocks by putting them in writeback state before + * calculating the checksums, first prepare to write payload blocks other + * than segment summary and super root blocks in which the checksums will + * be embedded. + */ +static void nilfs_prepare_write_logs(struct list_head *logs, u32 seed) +{ + struct nilfs_segment_buffer *segbuf; + struct folio *bd_folio = NULL, *fs_folio = NULL; + struct buffer_head *bh; + + /* Prepare to write payload blocks */ + list_for_each_entry(segbuf, logs, sb_list) { + list_for_each_entry(bh, &segbuf->sb_payload_buffers, + b_assoc_buffers) { + if (bh == segbuf->sb_super_root) + break; + set_buffer_async_write(bh); + if (bh->b_folio != fs_folio) { + nilfs_begin_folio_io(fs_folio); + fs_folio = bh->b_folio; + } + } + } + nilfs_begin_folio_io(fs_folio); + + nilfs_add_checksums_on_logs(logs, seed); + + /* Prepare to write segment summary blocks */ + list_for_each_entry(segbuf, logs, sb_list) { + list_for_each_entry(bh, &segbuf->sb_segsum_buffers, + b_assoc_buffers) { + mark_buffer_dirty(bh); + if (bh->b_folio == bd_folio) + continue; + if (bd_folio) { + folio_lock(bd_folio); + folio_wait_writeback(bd_folio); + folio_clear_dirty_for_io(bd_folio); + folio_start_writeback(bd_folio); + folio_unlock(bd_folio); + } + bd_folio = bh->b_folio; + } + } + + /* Prepare to write super root block */ + bh = NILFS_LAST_SEGBUF(logs)->sb_super_root; + if (bh) { + mark_buffer_dirty(bh); + if (bh->b_folio != bd_folio) { + folio_lock(bd_folio); + folio_wait_writeback(bd_folio); + folio_clear_dirty_for_io(bd_folio); + folio_start_writeback(bd_folio); + folio_unlock(bd_folio); + bd_folio = bh->b_folio; + } + } + + if (bd_folio) { + folio_lock(bd_folio); + folio_wait_writeback(bd_folio); + folio_clear_dirty_for_io(bd_folio); + folio_start_writeback(bd_folio); + folio_unlock(bd_folio); + } +} + +static int nilfs_segctor_write(struct nilfs_sc_info *sci, + struct the_nilfs *nilfs) +{ + int ret; + + ret = nilfs_write_logs(&sci->sc_segbufs, nilfs); + list_splice_tail_init(&sci->sc_segbufs, &sci->sc_write_logs); + return ret; +} + +static void nilfs_end_folio_io(struct folio *folio, int err) +{ + if (!folio) + return; + + if (buffer_nilfs_node(folio_buffers(folio)) && + !folio_test_writeback(folio)) { + /* + * For b-tree node pages, this function may be called twice + * or more because they might be split in a segment. + */ + if (folio_test_dirty(folio)) { + /* + * For pages holding split b-tree node buffers, dirty + * flag on the buffers may be cleared discretely. + * In that case, the page is once redirtied for + * remaining buffers, and it must be cancelled if + * all the buffers get cleaned later. + */ + folio_lock(folio); + if (nilfs_folio_buffers_clean(folio)) + __nilfs_clear_folio_dirty(folio); + folio_unlock(folio); + } + return; + } + + if (err || !nilfs_folio_buffers_clean(folio)) + filemap_dirty_folio(folio->mapping, folio); + + folio_end_writeback(folio); +} + +static void nilfs_abort_logs(struct list_head *logs, int err) +{ + struct nilfs_segment_buffer *segbuf; + struct folio *bd_folio = NULL, *fs_folio = NULL; + struct buffer_head *bh; + + if (list_empty(logs)) + return; + + list_for_each_entry(segbuf, logs, sb_list) { + list_for_each_entry(bh, &segbuf->sb_segsum_buffers, + b_assoc_buffers) { + clear_buffer_uptodate(bh); + if (bh->b_folio != bd_folio) { + if (bd_folio) + folio_end_writeback(bd_folio); + bd_folio = bh->b_folio; + } + } + + list_for_each_entry(bh, &segbuf->sb_payload_buffers, + b_assoc_buffers) { + if (bh == segbuf->sb_super_root) { + clear_buffer_uptodate(bh); + if (bh->b_folio != bd_folio) { + folio_end_writeback(bd_folio); + bd_folio = bh->b_folio; + } + break; + } + clear_buffer_async_write(bh); + if (bh->b_folio != fs_folio) { + nilfs_end_folio_io(fs_folio, err); + fs_folio = bh->b_folio; + } + } + } + if (bd_folio) + folio_end_writeback(bd_folio); + + nilfs_end_folio_io(fs_folio, err); +} + +static void nilfs_segctor_abort_construction(struct nilfs_sc_info *sci, + struct the_nilfs *nilfs, int err) +{ + LIST_HEAD(logs); + int ret; + + list_splice_tail_init(&sci->sc_write_logs, &logs); + ret = nilfs_wait_on_logs(&logs); + nilfs_abort_logs(&logs, ret ? : err); + + list_splice_tail_init(&sci->sc_segbufs, &logs); + if (list_empty(&logs)) + return; /* if the first segment buffer preparation failed */ + + nilfs_cancel_segusage(&logs, nilfs->ns_sufile); + nilfs_free_incomplete_logs(&logs, nilfs); + + if (sci->sc_stage.flags & NILFS_CF_SUFREED) { + ret = nilfs_sufile_cancel_freev(nilfs->ns_sufile, + sci->sc_freesegs, + sci->sc_nfreesegs, + NULL); + nilfs_sufile_warn_on_error(nilfs->ns_sufile, ret); + } + + nilfs_destroy_logs(&logs); +} + +static void nilfs_set_next_segment(struct the_nilfs *nilfs, + struct nilfs_segment_buffer *segbuf) +{ + nilfs->ns_segnum = segbuf->sb_segnum; + nilfs->ns_nextnum = segbuf->sb_nextnum; + nilfs->ns_pseg_offset = segbuf->sb_pseg_start - segbuf->sb_fseg_start + + segbuf->sb_sum.nblocks; + nilfs->ns_seg_seq = segbuf->sb_sum.seg_seq; + nilfs->ns_ctime = segbuf->sb_sum.ctime; +} + +static void nilfs_segctor_complete_write(struct nilfs_sc_info *sci) +{ + struct nilfs_segment_buffer *segbuf; + struct folio *bd_folio = NULL, *fs_folio = NULL; + struct the_nilfs *nilfs = sci->sc_super->s_fs_info; + int update_sr = false; + + list_for_each_entry(segbuf, &sci->sc_write_logs, sb_list) { + struct buffer_head *bh; + + list_for_each_entry(bh, &segbuf->sb_segsum_buffers, + b_assoc_buffers) { + set_buffer_uptodate(bh); + clear_buffer_dirty(bh); + if (bh->b_folio != bd_folio) { + if (bd_folio) + folio_end_writeback(bd_folio); + bd_folio = bh->b_folio; + } + } + /* + * We assume that the buffers which belong to the same folio + * continue over the buffer list. + * Under this assumption, the last BHs of folios is + * identifiable by the discontinuity of bh->b_folio + * (folio != fs_folio). + * + * For B-tree node blocks, however, this assumption is not + * guaranteed. The cleanup code of B-tree node folios needs + * special care. + */ + list_for_each_entry(bh, &segbuf->sb_payload_buffers, + b_assoc_buffers) { + const unsigned long set_bits = BIT(BH_Uptodate); + const unsigned long clear_bits = + (BIT(BH_Dirty) | BIT(BH_Async_Write) | + BIT(BH_Delay) | BIT(BH_NILFS_Volatile) | + BIT(BH_NILFS_Redirected)); + + if (bh == segbuf->sb_super_root) { + set_buffer_uptodate(bh); + clear_buffer_dirty(bh); + if (bh->b_folio != bd_folio) { + folio_end_writeback(bd_folio); + bd_folio = bh->b_folio; + } + update_sr = true; + break; + } + set_mask_bits(&bh->b_state, clear_bits, set_bits); + if (bh->b_folio != fs_folio) { + nilfs_end_folio_io(fs_folio, 0); + fs_folio = bh->b_folio; + } + } + + if (!nilfs_segbuf_simplex(segbuf)) { + if (segbuf->sb_sum.flags & NILFS_SS_LOGBGN) { + set_bit(NILFS_SC_UNCLOSED, &sci->sc_flags); + sci->sc_lseg_stime = jiffies; + } + if (segbuf->sb_sum.flags & NILFS_SS_LOGEND) + clear_bit(NILFS_SC_UNCLOSED, &sci->sc_flags); + } + } + /* + * Since folios may continue over multiple segment buffers, + * end of the last folio must be checked outside of the loop. + */ + if (bd_folio) + folio_end_writeback(bd_folio); + + nilfs_end_folio_io(fs_folio, 0); + + nilfs_drop_collected_inodes(&sci->sc_dirty_files); + + if (nilfs_doing_gc()) + nilfs_drop_collected_inodes(&sci->sc_gc_inodes); + else + nilfs->ns_nongc_ctime = sci->sc_seg_ctime; + + sci->sc_nblk_inc += sci->sc_nblk_this_inc; + + segbuf = NILFS_LAST_SEGBUF(&sci->sc_write_logs); + nilfs_set_next_segment(nilfs, segbuf); + + if (update_sr) { + nilfs->ns_flushed_device = 0; + nilfs_set_last_segment(nilfs, segbuf->sb_pseg_start, + segbuf->sb_sum.seg_seq, nilfs->ns_cno++); + + clear_bit(NILFS_SC_HAVE_DELTA, &sci->sc_flags); + clear_bit(NILFS_SC_DIRTY, &sci->sc_flags); + set_bit(NILFS_SC_SUPER_ROOT, &sci->sc_flags); + nilfs_segctor_clear_metadata_dirty(sci); + } else + clear_bit(NILFS_SC_SUPER_ROOT, &sci->sc_flags); +} + +static int nilfs_segctor_wait(struct nilfs_sc_info *sci) +{ + int ret; + + ret = nilfs_wait_on_logs(&sci->sc_write_logs); + if (!ret) { + nilfs_segctor_complete_write(sci); + nilfs_destroy_logs(&sci->sc_write_logs); + } + return ret; +} + +static int nilfs_segctor_collect_dirty_files(struct nilfs_sc_info *sci, + struct the_nilfs *nilfs) +{ + struct nilfs_inode_info *ii, *n; + struct inode *ifile = sci->sc_root->ifile; + + spin_lock(&nilfs->ns_inode_lock); + retry: + list_for_each_entry_safe(ii, n, &nilfs->ns_dirty_files, i_dirty) { + if (!ii->i_bh) { + struct buffer_head *ibh; + int err; + + spin_unlock(&nilfs->ns_inode_lock); + err = nilfs_ifile_get_inode_block( + ifile, ii->vfs_inode.i_ino, &ibh); + if (unlikely(err)) { + nilfs_warn(sci->sc_super, + "log writer: error %d getting inode block (ino=%llu)", + err, ii->vfs_inode.i_ino); + return err; + } + spin_lock(&nilfs->ns_inode_lock); + if (likely(!ii->i_bh)) + ii->i_bh = ibh; + else + brelse(ibh); + goto retry; + } + + // Always redirty the buffer to avoid race condition + mark_buffer_dirty(ii->i_bh); + nilfs_mdt_mark_dirty(ifile); + + clear_bit(NILFS_I_QUEUED, &ii->i_state); + set_bit(NILFS_I_BUSY, &ii->i_state); + list_move_tail(&ii->i_dirty, &sci->sc_dirty_files); + } + spin_unlock(&nilfs->ns_inode_lock); + + return 0; +} + +static void nilfs_segctor_drop_written_files(struct nilfs_sc_info *sci, + struct the_nilfs *nilfs) +{ + struct nilfs_inode_info *ii, *n; + int during_mount = !(sci->sc_super->s_flags & SB_ACTIVE); + int defer_iput = false; + + spin_lock(&nilfs->ns_inode_lock); + list_for_each_entry_safe(ii, n, &sci->sc_dirty_files, i_dirty) { + if (!test_and_clear_bit(NILFS_I_UPDATED, &ii->i_state) || + test_bit(NILFS_I_DIRTY, &ii->i_state)) + continue; + + clear_bit(NILFS_I_BUSY, &ii->i_state); + brelse(ii->i_bh); + ii->i_bh = NULL; + list_del_init(&ii->i_dirty); + if (!ii->vfs_inode.i_nlink || during_mount) { + /* + * Defer calling iput() to avoid deadlocks if + * i_nlink == 0 or mount is not yet finished. + */ + list_add_tail(&ii->i_dirty, &sci->sc_iput_queue); + defer_iput = true; + } else { + spin_unlock(&nilfs->ns_inode_lock); + iput(&ii->vfs_inode); + spin_lock(&nilfs->ns_inode_lock); + } + } + spin_unlock(&nilfs->ns_inode_lock); + + if (defer_iput) + schedule_work(&sci->sc_iput_work); +} + +/* + * Main procedure of segment constructor + */ +static int nilfs_segctor_do_construct(struct nilfs_sc_info *sci, int mode) +{ + struct the_nilfs *nilfs = sci->sc_super->s_fs_info; + int err; + + if (sb_rdonly(sci->sc_super)) + return -EROFS; + + nilfs_sc_cstage_set(sci, NILFS_ST_INIT); + sci->sc_cno = nilfs->ns_cno; + + err = nilfs_segctor_collect_dirty_files(sci, nilfs); + if (unlikely(err)) + goto out; + + if (nilfs_test_metadata_dirty(nilfs, sci->sc_root)) + set_bit(NILFS_SC_DIRTY, &sci->sc_flags); + + if (nilfs_segctor_clean(sci)) + goto out; + + do { + sci->sc_stage.flags &= ~NILFS_CF_HISTORY_MASK; + + err = nilfs_segctor_begin_construction(sci, nilfs); + if (unlikely(err)) + goto failed; + + /* Update time stamp */ + sci->sc_seg_ctime = ktime_get_real_seconds(); + + err = nilfs_segctor_collect(sci, nilfs, mode); + if (unlikely(err)) + goto failed; + + /* Avoid empty segment */ + if (nilfs_sc_cstage_get(sci) == NILFS_ST_DONE && + nilfs_segbuf_empty(sci->sc_curseg)) { + nilfs_segctor_abort_construction(sci, nilfs, 1); + goto out; + } + + err = nilfs_segctor_assign(sci, mode); + if (unlikely(err)) + goto failed; + + if (sci->sc_stage.flags & NILFS_CF_IFILE_STARTED) + nilfs_segctor_fill_in_file_bmap(sci); + + if (mode == SC_LSEG_SR && + nilfs_sc_cstage_get(sci) >= NILFS_ST_CPFILE) { + err = nilfs_cpfile_finalize_checkpoint( + nilfs->ns_cpfile, nilfs->ns_cno, sci->sc_root, + sci->sc_nblk_inc + sci->sc_nblk_this_inc, + sci->sc_seg_ctime, + !test_bit(NILFS_SC_HAVE_DELTA, &sci->sc_flags)); + if (unlikely(err)) + goto failed_to_write; + + nilfs_segctor_fill_in_super_root(sci, nilfs); + } + nilfs_segctor_update_segusage(sci, nilfs->ns_sufile); + + /* Write partial segments */ + nilfs_prepare_write_logs(&sci->sc_segbufs, nilfs->ns_crc_seed); + + err = nilfs_segctor_write(sci, nilfs); + if (unlikely(err)) + goto failed_to_write; + + if (nilfs_sc_cstage_get(sci) == NILFS_ST_DONE || + nilfs->ns_blocksize_bits != PAGE_SHIFT) { + /* + * At this point, we avoid double buffering + * for blocksize < pagesize because page dirty + * flag is turned off during write and dirty + * buffers are not properly collected for + * pages crossing over segments. + */ + err = nilfs_segctor_wait(sci); + if (err) + goto failed_to_write; + } + } while (nilfs_sc_cstage_get(sci) != NILFS_ST_DONE); + + out: + nilfs_segctor_drop_written_files(sci, nilfs); + return err; + + failed_to_write: + failed: + if (mode == SC_LSEG_SR && nilfs_sc_cstage_get(sci) >= NILFS_ST_IFILE) + nilfs_redirty_inodes(&sci->sc_dirty_files); + if (nilfs_doing_gc()) + nilfs_redirty_inodes(&sci->sc_gc_inodes); + nilfs_segctor_abort_construction(sci, nilfs, err); + goto out; +} + +/** + * nilfs_segctor_start_timer - set timer of background write + * @sci: nilfs_sc_info + * + * If the timer has already been set, it ignores the new request. + * This function MUST be called within a section locking the segment + * semaphore. + */ +static void nilfs_segctor_start_timer(struct nilfs_sc_info *sci) +{ + spin_lock(&sci->sc_state_lock); + if (!(sci->sc_state & NILFS_SEGCTOR_COMMIT)) { + if (sci->sc_task) { + sci->sc_timer.expires = jiffies + sci->sc_interval; + add_timer(&sci->sc_timer); + } + sci->sc_state |= NILFS_SEGCTOR_COMMIT; + } + spin_unlock(&sci->sc_state_lock); +} + +static void nilfs_segctor_do_flush(struct nilfs_sc_info *sci, int bn) +{ + spin_lock(&sci->sc_state_lock); + if (!(sci->sc_flush_request & BIT(bn))) { + unsigned long prev_req = sci->sc_flush_request; + + sci->sc_flush_request |= BIT(bn); + if (!prev_req) + wake_up(&sci->sc_wait_daemon); + } + spin_unlock(&sci->sc_state_lock); +} + +struct nilfs_segctor_wait_request { + wait_queue_entry_t wq; + __u32 seq; + int err; + atomic_t done; +}; + +static int nilfs_segctor_sync(struct nilfs_sc_info *sci) +{ + struct nilfs_segctor_wait_request wait_req; + int err = 0; + + init_wait(&wait_req.wq); + wait_req.err = 0; + atomic_set(&wait_req.done, 0); + init_waitqueue_entry(&wait_req.wq, current); + + /* + * To prevent a race issue where completion notifications from the + * log writer thread are missed, increment the request sequence count + * "sc_seq_request" and insert a wait queue entry using the current + * sequence number into the "sc_wait_request" queue at the same time + * within the lock section of "sc_state_lock". + */ + spin_lock(&sci->sc_state_lock); + wait_req.seq = ++sci->sc_seq_request; + add_wait_queue(&sci->sc_wait_request, &wait_req.wq); + spin_unlock(&sci->sc_state_lock); + + wake_up(&sci->sc_wait_daemon); + + for (;;) { + set_current_state(TASK_INTERRUPTIBLE); + + /* + * Synchronize only while the log writer thread is alive. + * Leave flushing out after the log writer thread exits to + * the cleanup work in nilfs_segctor_destroy(). + */ + if (!sci->sc_task) + break; + + if (atomic_read(&wait_req.done)) { + err = wait_req.err; + break; + } + if (!signal_pending(current)) { + schedule(); + continue; + } + err = -ERESTARTSYS; + break; + } + finish_wait(&sci->sc_wait_request, &wait_req.wq); + return err; +} + +static void nilfs_segctor_wakeup(struct nilfs_sc_info *sci, int err, bool force) +{ + struct nilfs_segctor_wait_request *wrq, *n; + unsigned long flags; + + spin_lock_irqsave(&sci->sc_wait_request.lock, flags); + list_for_each_entry_safe(wrq, n, &sci->sc_wait_request.head, wq.entry) { + if (!atomic_read(&wrq->done) && + (force || nilfs_cnt32_ge(sci->sc_seq_done, wrq->seq))) { + wrq->err = err; + atomic_set(&wrq->done, 1); + } + if (atomic_read(&wrq->done)) { + wrq->wq.func(&wrq->wq, + TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE, + 0, NULL); + } + } + spin_unlock_irqrestore(&sci->sc_wait_request.lock, flags); +} + +/** + * nilfs_construct_segment - construct a logical segment + * @sb: super block + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOMEM - Insufficient memory available. + * * %-ENOSPC - No space left on device (only in a panic state). + * * %-ERESTARTSYS - Interrupted. + * * %-EROFS - Read only filesystem. + */ +int nilfs_construct_segment(struct super_block *sb) +{ + struct the_nilfs *nilfs = sb->s_fs_info; + struct nilfs_sc_info *sci = nilfs->ns_writer; + struct nilfs_transaction_info *ti; + + if (sb_rdonly(sb) || unlikely(!sci)) + return -EROFS; + + /* A call inside transactions causes a deadlock. */ + BUG_ON((ti = current->journal_info) && ti->ti_magic == NILFS_TI_MAGIC); + + return nilfs_segctor_sync(sci); +} + +/** + * nilfs_construct_dsync_segment - construct a data-only logical segment + * @sb: super block + * @inode: inode whose data blocks should be written out + * @start: start byte offset + * @end: end byte offset (inclusive) + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOMEM - Insufficient memory available. + * * %-ENOSPC - No space left on device (only in a panic state). + * * %-ERESTARTSYS - Interrupted. + * * %-EROFS - Read only filesystem. + */ +int nilfs_construct_dsync_segment(struct super_block *sb, struct inode *inode, + loff_t start, loff_t end) +{ + struct the_nilfs *nilfs = sb->s_fs_info; + struct nilfs_sc_info *sci = nilfs->ns_writer; + struct nilfs_inode_info *ii; + struct nilfs_transaction_info ti; + int err = 0; + + if (sb_rdonly(sb) || unlikely(!sci)) + return -EROFS; + + nilfs_transaction_lock(sb, &ti, 0); + + ii = NILFS_I(inode); + if (test_bit(NILFS_I_INODE_SYNC, &ii->i_state) || + nilfs_test_opt(nilfs, STRICT_ORDER) || + test_bit(NILFS_SC_UNCLOSED, &sci->sc_flags) || + nilfs_discontinued(nilfs)) { + nilfs_transaction_unlock(sb); + err = nilfs_segctor_sync(sci); + return err; + } + + spin_lock(&nilfs->ns_inode_lock); + if (!test_bit(NILFS_I_QUEUED, &ii->i_state) && + !test_bit(NILFS_I_BUSY, &ii->i_state)) { + spin_unlock(&nilfs->ns_inode_lock); + nilfs_transaction_unlock(sb); + return 0; + } + spin_unlock(&nilfs->ns_inode_lock); + sci->sc_dsync_inode = ii; + sci->sc_dsync_start = start; + sci->sc_dsync_end = end; + + err = nilfs_segctor_do_construct(sci, SC_LSEG_DSYNC); + if (!err) + nilfs->ns_flushed_device = 0; + + nilfs_transaction_unlock(sb); + return err; +} + +#define FLUSH_FILE_BIT (0x1) /* data file only */ +#define FLUSH_DAT_BIT BIT(NILFS_DAT_INO) /* DAT only */ + +/** + * nilfs_segctor_accept - record accepted sequence count of log-write requests + * @sci: segment constructor object + */ +static void nilfs_segctor_accept(struct nilfs_sc_info *sci) +{ + bool thread_is_alive; + + spin_lock(&sci->sc_state_lock); + sci->sc_seq_accepted = sci->sc_seq_request; + thread_is_alive = (bool)sci->sc_task; + spin_unlock(&sci->sc_state_lock); + + /* + * This function does not race with the log writer thread's + * termination. Therefore, deleting sc_timer, which should not be + * done after the log writer thread exits, can be done safely outside + * the area protected by sc_state_lock. + */ + if (thread_is_alive) + timer_delete_sync(&sci->sc_timer); +} + +/** + * nilfs_segctor_notify - notify the result of request to caller threads + * @sci: segment constructor object + * @mode: mode of log forming + * @err: error code to be notified + */ +static void nilfs_segctor_notify(struct nilfs_sc_info *sci, int mode, int err) +{ + /* Clear requests (even when the construction failed) */ + spin_lock(&sci->sc_state_lock); + + if (mode == SC_LSEG_SR) { + sci->sc_state &= ~NILFS_SEGCTOR_COMMIT; + sci->sc_seq_done = sci->sc_seq_accepted; + nilfs_segctor_wakeup(sci, err, false); + sci->sc_flush_request = 0; + } else { + if (mode == SC_FLUSH_FILE) + sci->sc_flush_request &= ~FLUSH_FILE_BIT; + else if (mode == SC_FLUSH_DAT) + sci->sc_flush_request &= ~FLUSH_DAT_BIT; + + /* re-enable timer if checkpoint creation was not done */ + if ((sci->sc_state & NILFS_SEGCTOR_COMMIT) && sci->sc_task && + time_before(jiffies, sci->sc_timer.expires)) + add_timer(&sci->sc_timer); + } + spin_unlock(&sci->sc_state_lock); +} + +/** + * nilfs_segctor_construct - form logs and write them to disk + * @sci: segment constructor object + * @mode: mode of log forming + * + * Return: 0 on success, or a negative error code on failure. + */ +static int nilfs_segctor_construct(struct nilfs_sc_info *sci, int mode) +{ + struct the_nilfs *nilfs = sci->sc_super->s_fs_info; + struct nilfs_super_block **sbp; + int err = 0; + + nilfs_segctor_accept(sci); + + if (nilfs_discontinued(nilfs)) + mode = SC_LSEG_SR; + if (!nilfs_segctor_confirm(sci)) + err = nilfs_segctor_do_construct(sci, mode); + + if (likely(!err)) { + if (mode != SC_FLUSH_DAT) + atomic_set(&nilfs->ns_ndirtyblks, 0); + if (test_bit(NILFS_SC_SUPER_ROOT, &sci->sc_flags) && + nilfs_discontinued(nilfs)) { + down_write(&nilfs->ns_sem); + err = -EIO; + sbp = nilfs_prepare_super(sci->sc_super, + nilfs_sb_will_flip(nilfs)); + if (likely(sbp)) { + nilfs_set_log_cursor(sbp[0], nilfs); + err = nilfs_commit_super(sci->sc_super, + NILFS_SB_COMMIT); + } + up_write(&nilfs->ns_sem); + } + } + + nilfs_segctor_notify(sci, mode, err); + return err; +} + +static void nilfs_construction_timeout(struct timer_list *t) +{ + struct nilfs_sc_info *sci = timer_container_of(sci, t, sc_timer); + + wake_up_process(sci->sc_task); +} + +static void +nilfs_remove_written_gcinodes(struct the_nilfs *nilfs, struct list_head *head) +{ + struct nilfs_inode_info *ii, *n; + + list_for_each_entry_safe(ii, n, head, i_dirty) { + if (!test_bit(NILFS_I_UPDATED, &ii->i_state)) + continue; + list_del_init(&ii->i_dirty); + truncate_inode_pages(&ii->vfs_inode.i_data, 0); + nilfs_btnode_cache_clear(ii->i_assoc_inode->i_mapping); + iput(&ii->vfs_inode); + } +} + +int nilfs_clean_segments(struct super_block *sb, struct nilfs_argv *argv, + void **kbufs) +{ + struct the_nilfs *nilfs = sb->s_fs_info; + struct nilfs_sc_info *sci = nilfs->ns_writer; + struct nilfs_transaction_info ti; + int err; + size_t i, nfreesegs = argv[4].v_nmembs; + __u64 *segnumv = kbufs[4]; + + if (unlikely(!sci)) + return -EROFS; + + nilfs_transaction_lock(sb, &ti, 1); + + /* + * Validate segment numbers under ns_segctor_sem (held for write + * by nilfs_transaction_lock above) so the check is serialized + * against nilfs_ioctl_resize(), which can modify ns_nsegments. + * Rejecting bad input here, before any segment-cleaning work + * begins, avoids the per-element diagnostic path inside + * nilfs_sufile_updatev() that would otherwise run under this + * same lock and stall concurrent readers. + */ + for (i = 0; i < nfreesegs; i++) { + if (segnumv[i] >= nilfs->ns_nsegments) { + nilfs_err(sb, + "Segment number %llu to be freed is out of range", + (unsigned long long)segnumv[i]); + err = -EINVAL; + goto bail_unlock; + } + } + + err = nilfs_mdt_save_to_shadow_map(nilfs->ns_dat); + if (unlikely(err)) + goto out_unlock; + + err = nilfs_ioctl_prepare_clean_segments(nilfs, argv, kbufs); + if (unlikely(err)) { + nilfs_mdt_restore_from_shadow_map(nilfs->ns_dat); + goto out_unlock; + } + + sci->sc_freesegs = kbufs[4]; + sci->sc_nfreesegs = argv[4].v_nmembs; + list_splice_tail_init(&nilfs->ns_gc_inodes, &sci->sc_gc_inodes); + + for (;;) { + err = nilfs_segctor_construct(sci, SC_LSEG_SR); + nilfs_remove_written_gcinodes(nilfs, &sci->sc_gc_inodes); + + if (likely(!err)) + break; + + nilfs_warn(sb, "error %d cleaning segments", err); + + if (unlikely(err == -EROFS)) + goto out_unlock; + + set_current_state(TASK_INTERRUPTIBLE); + schedule_timeout(sci->sc_interval); + } + if (nilfs_test_opt(nilfs, DISCARD)) { + int ret = nilfs_discard_segments(nilfs, sci->sc_freesegs, + sci->sc_nfreesegs); + if (ret) { + nilfs_warn(sb, + "error %d on discard request, turning discards off for the device", + ret); + nilfs_clear_opt(nilfs, DISCARD); + } + } + + out_unlock: + sci->sc_freesegs = NULL; + sci->sc_nfreesegs = 0; + nilfs_mdt_clear_shadow_map(nilfs->ns_dat); + bail_unlock: + nilfs_transaction_unlock(sb); + return err; +} + +static void nilfs_segctor_thread_construct(struct nilfs_sc_info *sci, int mode) +{ + struct nilfs_transaction_info ti; + + nilfs_transaction_lock(sci->sc_super, &ti, 0); + nilfs_segctor_construct(sci, mode); + + /* + * Unclosed segment should be retried. We do this using sc_timer. + * Timeout of sc_timer will invoke complete construction which leads + * to close the current logical segment. + */ + if (test_bit(NILFS_SC_UNCLOSED, &sci->sc_flags)) + nilfs_segctor_start_timer(sci); + + nilfs_transaction_unlock(sci->sc_super); +} + +static void nilfs_segctor_do_immediate_flush(struct nilfs_sc_info *sci) +{ + int mode = 0; + + spin_lock(&sci->sc_state_lock); + mode = (sci->sc_flush_request & FLUSH_DAT_BIT) ? + SC_FLUSH_DAT : SC_FLUSH_FILE; + spin_unlock(&sci->sc_state_lock); + + if (mode) { + nilfs_segctor_do_construct(sci, mode); + + spin_lock(&sci->sc_state_lock); + sci->sc_flush_request &= (mode == SC_FLUSH_FILE) ? + ~FLUSH_FILE_BIT : ~FLUSH_DAT_BIT; + spin_unlock(&sci->sc_state_lock); + } + clear_bit(NILFS_SC_PRIOR_FLUSH, &sci->sc_flags); +} + +static int nilfs_segctor_flush_mode(struct nilfs_sc_info *sci) +{ + if (!test_bit(NILFS_SC_UNCLOSED, &sci->sc_flags) || + time_before(jiffies, sci->sc_lseg_stime + sci->sc_mjcp_freq)) { + if (!(sci->sc_flush_request & ~FLUSH_FILE_BIT)) + return SC_FLUSH_FILE; + else if (!(sci->sc_flush_request & ~FLUSH_DAT_BIT)) + return SC_FLUSH_DAT; + } + return SC_LSEG_SR; +} + +/** + * nilfs_log_write_required - determine whether log writing is required + * @sci: nilfs_sc_info struct + * @modep: location for storing log writing mode + * + * Return: true if log writing is required, false otherwise. If log writing + * is required, the mode is stored in the location pointed to by @modep. + */ +static bool nilfs_log_write_required(struct nilfs_sc_info *sci, int *modep) +{ + bool timedout, ret = true; + + spin_lock(&sci->sc_state_lock); + timedout = ((sci->sc_state & NILFS_SEGCTOR_COMMIT) && + time_after_eq(jiffies, sci->sc_timer.expires)); + if (timedout || sci->sc_seq_request != sci->sc_seq_done) + *modep = SC_LSEG_SR; + else if (sci->sc_flush_request) + *modep = nilfs_segctor_flush_mode(sci); + else + ret = false; + + spin_unlock(&sci->sc_state_lock); + return ret; +} + +/** + * nilfs_segctor_thread - main loop of the log writer thread + * @arg: pointer to a struct nilfs_sc_info. + * + * nilfs_segctor_thread() is the main loop function of the log writer kernel + * thread, which determines whether log writing is necessary, and if so, + * performs the log write in the background, or waits if not. It is also + * used to decide the background writeback of the superblock. + * + * Return: Always 0. + */ +static int nilfs_segctor_thread(void *arg) +{ + struct nilfs_sc_info *sci = (struct nilfs_sc_info *)arg; + struct the_nilfs *nilfs = sci->sc_super->s_fs_info; + + nilfs_info(sci->sc_super, + "segctord starting. Construction interval = %lu seconds, CP frequency < %lu seconds", + sci->sc_interval / HZ, sci->sc_mjcp_freq / HZ); + + set_freezable(); + + while (!kthread_should_stop()) { + DEFINE_WAIT(wait); + bool should_write; + int mode; + + if (freezing(current)) { + try_to_freeze(); + continue; + } + + prepare_to_wait(&sci->sc_wait_daemon, &wait, + TASK_INTERRUPTIBLE); + should_write = nilfs_log_write_required(sci, &mode); + if (!should_write) + schedule(); + finish_wait(&sci->sc_wait_daemon, &wait); + + if (nilfs_sb_dirty(nilfs) && nilfs_sb_need_update(nilfs)) + set_nilfs_discontinued(nilfs); + + if (should_write) + nilfs_segctor_thread_construct(sci, mode); + } + + /* end sync. */ + spin_lock(&sci->sc_state_lock); + sci->sc_task = NULL; + timer_shutdown_sync(&sci->sc_timer); + spin_unlock(&sci->sc_state_lock); + return 0; +} + +/* + * Setup & clean-up functions + */ +static struct nilfs_sc_info *nilfs_segctor_new(struct super_block *sb, + struct nilfs_root *root) +{ + struct the_nilfs *nilfs = sb->s_fs_info; + struct nilfs_sc_info *sci; + + sci = kzalloc_obj(*sci); + if (!sci) + return NULL; + + sci->sc_super = sb; + + nilfs_get_root(root); + sci->sc_root = root; + + init_waitqueue_head(&sci->sc_wait_request); + init_waitqueue_head(&sci->sc_wait_daemon); + spin_lock_init(&sci->sc_state_lock); + INIT_LIST_HEAD(&sci->sc_dirty_files); + INIT_LIST_HEAD(&sci->sc_segbufs); + INIT_LIST_HEAD(&sci->sc_write_logs); + INIT_LIST_HEAD(&sci->sc_gc_inodes); + INIT_LIST_HEAD(&sci->sc_iput_queue); + INIT_WORK(&sci->sc_iput_work, nilfs_iput_work_func); + + sci->sc_interval = HZ * NILFS_SC_DEFAULT_TIMEOUT; + sci->sc_mjcp_freq = HZ * NILFS_SC_DEFAULT_SR_FREQ; + sci->sc_watermark = NILFS_SC_DEFAULT_WATERMARK; + + if (nilfs->ns_interval) + sci->sc_interval = HZ * nilfs->ns_interval; + if (nilfs->ns_watermark) + sci->sc_watermark = nilfs->ns_watermark; + return sci; +} + +static void nilfs_segctor_write_out(struct nilfs_sc_info *sci) +{ + int ret, retrycount = NILFS_SC_CLEANUP_RETRY; + + /* + * The segctord thread was stopped and its timer was removed. + * But some tasks remain. + */ + do { + struct nilfs_transaction_info ti; + + nilfs_transaction_lock(sci->sc_super, &ti, 0); + ret = nilfs_segctor_construct(sci, SC_LSEG_SR); + nilfs_transaction_unlock(sci->sc_super); + + flush_work(&sci->sc_iput_work); + + } while (ret && ret != -EROFS && retrycount-- > 0); +} + +/** + * nilfs_segctor_destroy - destroy the segment constructor. + * @sci: nilfs_sc_info + * + * nilfs_segctor_destroy() kills the segctord thread and frees + * the nilfs_sc_info struct. + * Caller must hold the segment semaphore. + */ +static void nilfs_segctor_destroy(struct nilfs_sc_info *sci) +{ + struct the_nilfs *nilfs = sci->sc_super->s_fs_info; + int flag; + + up_write(&nilfs->ns_segctor_sem); + + if (sci->sc_task) { + wake_up(&sci->sc_wait_daemon); + if (kthread_stop(sci->sc_task)) { + spin_lock(&sci->sc_state_lock); + sci->sc_task = NULL; + timer_shutdown_sync(&sci->sc_timer); + spin_unlock(&sci->sc_state_lock); + } + } + + spin_lock(&sci->sc_state_lock); + flag = ((sci->sc_state & NILFS_SEGCTOR_COMMIT) || sci->sc_flush_request + || sci->sc_seq_request != sci->sc_seq_done); + spin_unlock(&sci->sc_state_lock); + + /* + * Forcibly wake up tasks waiting in nilfs_segctor_sync(), which can + * be called from delayed iput() via nilfs_evict_inode() and can race + * with the above log writer thread termination. + */ + nilfs_segctor_wakeup(sci, 0, true); + + if (flush_work(&sci->sc_iput_work)) + flag = true; + + if (flag || !nilfs_segctor_confirm(sci)) + nilfs_segctor_write_out(sci); + + if (!list_empty(&sci->sc_dirty_files)) { + nilfs_warn(sci->sc_super, + "disposed unprocessed dirty file(s) when stopping log writer"); + nilfs_dispose_list(nilfs, &sci->sc_dirty_files, 1); + } + + if (!list_empty(&sci->sc_iput_queue)) { + nilfs_warn(sci->sc_super, + "disposed unprocessed inode(s) in iput queue when stopping log writer"); + nilfs_dispose_list(nilfs, &sci->sc_iput_queue, 1); + } + + WARN_ON(!list_empty(&sci->sc_segbufs)); + WARN_ON(!list_empty(&sci->sc_write_logs)); + + nilfs_put_root(sci->sc_root); + + down_write(&nilfs->ns_segctor_sem); + + kfree(sci); +} + +/** + * nilfs_attach_log_writer - attach log writer + * @sb: super block instance + * @root: root object of the current filesystem tree + * + * This allocates a log writer object, initializes it, and starts the + * log writer. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EINTR - Log writer thread creation failed due to interruption. + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_attach_log_writer(struct super_block *sb, struct nilfs_root *root) +{ + struct the_nilfs *nilfs = sb->s_fs_info; + struct nilfs_sc_info *sci; + struct task_struct *t; + int err; + + if (nilfs->ns_writer) { + /* + * This happens if the filesystem is made read-only by + * __nilfs_error or nilfs_remount and then remounted + * read/write. In these cases, reuse the existing + * writer. + */ + return 0; + } + + sci = nilfs_segctor_new(sb, root); + if (unlikely(!sci)) + return -ENOMEM; + + nilfs->ns_writer = sci; + t = kthread_create(nilfs_segctor_thread, sci, "segctord"); + if (IS_ERR(t)) { + err = PTR_ERR(t); + nilfs_err(sb, "error %d creating segctord thread", err); + nilfs_detach_log_writer(sb); + return err; + } + sci->sc_task = t; + timer_setup(&sci->sc_timer, nilfs_construction_timeout, 0); + + wake_up_process(sci->sc_task); + return 0; +} + +/** + * nilfs_detach_log_writer - destroy log writer + * @sb: super block instance + * + * This kills log writer daemon, frees the log writer object, and + * destroys list of dirty files. + */ +void nilfs_detach_log_writer(struct super_block *sb) +{ + struct the_nilfs *nilfs = sb->s_fs_info; + LIST_HEAD(garbage_list); + + down_write(&nilfs->ns_segctor_sem); + if (nilfs->ns_writer) { + nilfs_segctor_destroy(nilfs->ns_writer); + nilfs->ns_writer = NULL; + } + set_nilfs_purging(nilfs); + + /* Force to free the list of dirty files */ + spin_lock(&nilfs->ns_inode_lock); + if (!list_empty(&nilfs->ns_dirty_files)) { + list_splice_init(&nilfs->ns_dirty_files, &garbage_list); + nilfs_warn(sb, + "disposed unprocessed dirty file(s) when detaching log writer"); + } + spin_unlock(&nilfs->ns_inode_lock); + up_write(&nilfs->ns_segctor_sem); + + nilfs_dispose_list(nilfs, &garbage_list, 1); + clear_nilfs_purging(nilfs); +} diff --git a/fs/nilfs2/segment.h b/fs/nilfs2/segment.h new file mode 100644 index 000000000..4b39ed43a --- /dev/null +++ b/fs/nilfs2/segment.h @@ -0,0 +1,244 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * NILFS Segment constructor prototypes and definitions + * + * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Ryusuke Konishi. + * + */ +#ifndef _NILFS_SEGMENT_H +#define _NILFS_SEGMENT_H + +#include +#include +#include +#include +#include "nilfs.h" + +struct nilfs_root; + +/** + * struct nilfs_recovery_info - Recovery information + * @ri_need_recovery: Recovery status + * @ri_super_root: Block number of the last super root + * @ri_cno: Number of the last checkpoint + * @ri_lsegs_start: Region for roll-forwarding (start block number) + * @ri_lsegs_end: Region for roll-forwarding (end block number) + * @ri_lsegs_start_seq: Sequence value of the segment at ri_lsegs_start + * @ri_used_segments: List of segments to be mark active + * @ri_pseg_start: Block number of the last partial segment + * @ri_seq: Sequence number on the last partial segment + * @ri_segnum: Segment number on the last partial segment + * @ri_nextnum: Next segment number on the last partial segment + */ +struct nilfs_recovery_info { + int ri_need_recovery; + sector_t ri_super_root; + __u64 ri_cno; + + sector_t ri_lsegs_start; + sector_t ri_lsegs_end; + u64 ri_lsegs_start_seq; + struct list_head ri_used_segments; + sector_t ri_pseg_start; + u64 ri_seq; + __u64 ri_segnum; + __u64 ri_nextnum; +}; + +/* ri_need_recovery */ +#define NILFS_RECOVERY_SR_UPDATED 1 /* The super root was updated */ +#define NILFS_RECOVERY_ROLLFORWARD_DONE 2 /* Rollforward was carried out */ + +/** + * struct nilfs_cstage - Context of collection stage + * @scnt: Stage count, must be accessed via wrappers: + * nilfs_sc_cstage_inc(), nilfs_sc_cstage_set(), nilfs_sc_cstage_get() + * @flags: State flags + * @dirty_file_ptr: Pointer on dirty_files list, or inode of a target file + * @gc_inode_ptr: Pointer on the list of gc-inodes + */ +struct nilfs_cstage { + int scnt; + unsigned int flags; + struct nilfs_inode_info *dirty_file_ptr; + struct nilfs_inode_info *gc_inode_ptr; +}; + +struct nilfs_segment_buffer; + +struct nilfs_segsum_pointer { + struct buffer_head *bh; + unsigned int offset; /* offset in bytes */ +}; + +/** + * struct nilfs_sc_info - Segment constructor information + * @sc_super: Back pointer to super_block struct + * @sc_root: root object of the current filesystem tree + * @sc_nblk_inc: Block count of current generation + * @sc_dirty_files: List of files to be written + * @sc_gc_inodes: List of GC inodes having blocks to be written + * @sc_iput_queue: list of inodes for which iput should be done + * @sc_iput_work: work struct to defer iput call + * @sc_freesegs: array of segment numbers to be freed + * @sc_nfreesegs: number of segments on @sc_freesegs + * @sc_dsync_inode: inode whose data pages are written for a sync operation + * @sc_dsync_start: start byte offset of data pages + * @sc_dsync_end: end byte offset of data pages (inclusive) + * @sc_segbufs: List of segment buffers + * @sc_write_logs: List of segment buffers to hold logs under writing + * @sc_segbuf_nblocks: Number of available blocks in segment buffers. + * @sc_curseg: Current segment buffer + * @sc_stage: Collection stage + * @sc_finfo_ptr: pointer to the current finfo struct in the segment summary + * @sc_binfo_ptr: pointer to the current binfo struct in the segment summary + * @sc_blk_cnt: Block count of a file + * @sc_datablk_cnt: Data block count of a file + * @sc_nblk_this_inc: Number of blocks included in the current logical segment + * @sc_seg_ctime: Creation time + * @sc_cno: checkpoint number of current log + * @sc_flags: Internal flags + * @sc_state_lock: spinlock for sc_state and so on + * @sc_state: Segctord state flags + * @sc_flush_request: inode bitmap of metadata files to be flushed + * @sc_wait_request: Client request queue + * @sc_wait_daemon: Daemon wait queue + * @sc_seq_request: Request counter + * @sc_seq_accepted: Accepted request count + * @sc_seq_done: Completion counter + * @sc_sync: Request of explicit sync operation + * @sc_interval: Timeout value of background construction + * @sc_mjcp_freq: Frequency of creating checkpoints + * @sc_lseg_stime: Start time of the latest logical segment + * @sc_watermark: Watermark for the number of dirty buffers + * @sc_timer: Timer for segctord + * @sc_task: current thread of segctord + */ +struct nilfs_sc_info { + struct super_block *sc_super; + struct nilfs_root *sc_root; + + unsigned long sc_nblk_inc; + + struct list_head sc_dirty_files; + struct list_head sc_gc_inodes; + struct list_head sc_iput_queue; + struct work_struct sc_iput_work; + + __u64 *sc_freesegs; + size_t sc_nfreesegs; + + struct nilfs_inode_info *sc_dsync_inode; + loff_t sc_dsync_start; + loff_t sc_dsync_end; + + /* Segment buffers */ + struct list_head sc_segbufs; + struct list_head sc_write_logs; + unsigned long sc_segbuf_nblocks; + struct nilfs_segment_buffer *sc_curseg; + + struct nilfs_cstage sc_stage; + + struct nilfs_segsum_pointer sc_finfo_ptr; + struct nilfs_segsum_pointer sc_binfo_ptr; + unsigned long sc_blk_cnt; + unsigned long sc_datablk_cnt; + unsigned long sc_nblk_this_inc; + time64_t sc_seg_ctime; + __u64 sc_cno; + unsigned long sc_flags; + + spinlock_t sc_state_lock; + unsigned long sc_state; + unsigned long sc_flush_request; + + wait_queue_head_t sc_wait_request; + wait_queue_head_t sc_wait_daemon; + + __u32 sc_seq_request; + __u32 sc_seq_accepted; + __u32 sc_seq_done; + + int sc_sync; + unsigned long sc_interval; + unsigned long sc_mjcp_freq; + unsigned long sc_lseg_stime; /* in 1/HZ seconds */ + unsigned long sc_watermark; + + struct timer_list sc_timer; + struct task_struct *sc_task; +}; + +/* sc_flags */ +enum { + NILFS_SC_DIRTY, /* One or more dirty meta-data blocks exist */ + NILFS_SC_UNCLOSED, /* Logical segment is not closed */ + NILFS_SC_SUPER_ROOT, /* The latest segment has a super root */ + NILFS_SC_PRIOR_FLUSH, /* + * Requesting immediate flush without making a + * checkpoint + */ + NILFS_SC_HAVE_DELTA, /* + * Next checkpoint will have update of files + * other than DAT, cpfile, sufile, or files + * moved by GC. + */ +}; + +/* sc_state */ +#define NILFS_SEGCTOR_COMMIT 0x0004 /* committed transaction exists */ + +/* + * Constant parameters + */ +#define NILFS_SC_CLEANUP_RETRY 3 /* + * Retry count of construction when + * destroying segctord + */ + +/* + * Default values of timeout, in seconds. + */ +#define NILFS_SC_DEFAULT_TIMEOUT 5 /* + * Timeout value of dirty blocks. + * It triggers construction of a + * logical segment with a super root. + */ +#define NILFS_SC_DEFAULT_SR_FREQ 30 /* + * Maximum frequency of super root + * creation + */ + +/* + * The default threshold amount of data, in block counts. + */ +#define NILFS_SC_DEFAULT_WATERMARK 3600 + +/* super.c */ +extern struct kmem_cache *nilfs_transaction_cachep; + +/* segment.c */ +extern void nilfs_relax_pressure_in_lock(struct super_block *); + +extern int nilfs_construct_segment(struct super_block *); +extern int nilfs_construct_dsync_segment(struct super_block *, struct inode *, + loff_t, loff_t); +extern int nilfs_clean_segments(struct super_block *, struct nilfs_argv *, + void **); + +int nilfs_attach_log_writer(struct super_block *sb, struct nilfs_root *root); +void nilfs_detach_log_writer(struct super_block *sb); + +/* recovery.c */ +extern int nilfs_read_super_root_block(struct the_nilfs *, sector_t, + struct buffer_head **, int); +extern int nilfs_search_super_root(struct the_nilfs *, + struct nilfs_recovery_info *); +int nilfs_salvage_orphan_logs(struct the_nilfs *nilfs, struct super_block *sb, + struct nilfs_recovery_info *ri); +extern void nilfs_dispose_segment_list(struct list_head *); + +#endif /* _NILFS_SEGMENT_H */ diff --git a/fs/nilfs2/sufile.c b/fs/nilfs2/sufile.c new file mode 100644 index 000000000..eceedca02 --- /dev/null +++ b/fs/nilfs2/sufile.c @@ -0,0 +1,1273 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * NILFS segment usage file. + * + * Copyright (C) 2006-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Koji Sato. + * Revised by Ryusuke Konishi. + */ + +#include +#include +#include +#include +#include +#include "mdt.h" +#include "sufile.h" + +#include + +/** + * struct nilfs_sufile_info - on-memory private data of sufile + * @mi: on-memory private data of metadata file + * @ncleansegs: number of clean segments + * @allocmin: lower limit of allocatable segment range + * @allocmax: upper limit of allocatable segment range + */ +struct nilfs_sufile_info { + struct nilfs_mdt_info mi; + unsigned long ncleansegs;/* number of clean segments */ + __u64 allocmin; /* lower limit of allocatable segment range */ + __u64 allocmax; /* upper limit of allocatable segment range */ +}; + +static inline struct nilfs_sufile_info *NILFS_SUI(struct inode *sufile) +{ + return (struct nilfs_sufile_info *)NILFS_MDT(sufile); +} + +static inline unsigned long +nilfs_sufile_segment_usages_per_block(const struct inode *sufile) +{ + return NILFS_MDT(sufile)->mi_entries_per_block; +} + +static unsigned long +nilfs_sufile_get_blkoff(const struct inode *sufile, __u64 segnum) +{ + __u64 t = segnum + NILFS_MDT(sufile)->mi_first_entry_offset; + + t = div64_ul(t, nilfs_sufile_segment_usages_per_block(sufile)); + return (unsigned long)t; +} + +static unsigned long +nilfs_sufile_get_offset(const struct inode *sufile, __u64 segnum) +{ + __u64 t = segnum + NILFS_MDT(sufile)->mi_first_entry_offset; + + return do_div(t, nilfs_sufile_segment_usages_per_block(sufile)); +} + +static unsigned long +nilfs_sufile_segment_usages_in_block(const struct inode *sufile, __u64 curr, + __u64 max) +{ + return min_t(unsigned long, + nilfs_sufile_segment_usages_per_block(sufile) - + nilfs_sufile_get_offset(sufile, curr), + max - curr + 1); +} + +/** + * nilfs_sufile_segment_usage_offset - calculate the byte offset of a segment + * usage entry in the folio containing it + * @sufile: segment usage file inode + * @segnum: number of segment usage + * @bh: buffer head of block containing segment usage indexed by @segnum + * + * Return: Byte offset in the folio of the segment usage entry. + */ +static size_t nilfs_sufile_segment_usage_offset(const struct inode *sufile, + __u64 segnum, + struct buffer_head *bh) +{ + return offset_in_folio(bh->b_folio, bh->b_data) + + nilfs_sufile_get_offset(sufile, segnum) * + NILFS_MDT(sufile)->mi_entry_size; +} + +static int nilfs_sufile_get_header_block(struct inode *sufile, + struct buffer_head **bhp) +{ + int err = nilfs_mdt_get_block(sufile, 0, 0, NULL, bhp); + + if (unlikely(err == -ENOENT)) { + nilfs_error(sufile->i_sb, + "missing header block in segment usage metadata"); + err = -EIO; + } + return err; +} + +static inline int +nilfs_sufile_get_segment_usage_block(struct inode *sufile, __u64 segnum, + int create, struct buffer_head **bhp) +{ + return nilfs_mdt_get_block(sufile, + nilfs_sufile_get_blkoff(sufile, segnum), + create, NULL, bhp); +} + +static int nilfs_sufile_delete_segment_usage_block(struct inode *sufile, + __u64 segnum) +{ + return nilfs_mdt_delete_block(sufile, + nilfs_sufile_get_blkoff(sufile, segnum)); +} + +static void nilfs_sufile_mod_counter(struct buffer_head *header_bh, + u64 ncleanadd, u64 ndirtyadd) +{ + struct nilfs_sufile_header *header; + + header = kmap_local_folio(header_bh->b_folio, 0); + le64_add_cpu(&header->sh_ncleansegs, ncleanadd); + le64_add_cpu(&header->sh_ndirtysegs, ndirtyadd); + kunmap_local(header); + + mark_buffer_dirty(header_bh); +} + +/** + * nilfs_sufile_get_ncleansegs - return the number of clean segments + * @sufile: inode of segment usage file + * + * Return: Number of clean segments. + */ +unsigned long nilfs_sufile_get_ncleansegs(struct inode *sufile) +{ + return NILFS_SUI(sufile)->ncleansegs; +} + +/** + * nilfs_sufile_updatev - modify multiple segment usages at a time + * @sufile: inode of segment usage file + * @segnumv: array of segment numbers + * @nsegs: size of @segnumv array + * @create: creation flag + * @ndone: place to store number of modified segments on @segnumv + * @dofunc: primitive operation for the update + * + * Description: nilfs_sufile_updatev() repeatedly calls @dofunc + * against the given array of segments. The @dofunc is called with + * buffers of a header block and the sufile block in which the target + * segment usage entry is contained. If @ndone is given, the number + * of successfully modified segments from the head is stored in the + * place @ndone points to. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EINVAL - Invalid segment usage number + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOENT - Given segment usage is in hole block (may be returned if + * @create is zero) + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_sufile_updatev(struct inode *sufile, __u64 *segnumv, size_t nsegs, + int create, size_t *ndone, + void (*dofunc)(struct inode *, __u64, + struct buffer_head *, + struct buffer_head *)) +{ + struct buffer_head *header_bh, *bh; + unsigned long blkoff, prev_blkoff; + __u64 *seg; + size_t nerr = 0, n = 0; + int ret = 0; + + if (unlikely(nsegs == 0)) + goto out; + + down_write(&NILFS_MDT(sufile)->mi_sem); + for (seg = segnumv; seg < segnumv + nsegs; seg++) { + if (unlikely(*seg >= nilfs_sufile_get_nsegments(sufile))) { + nilfs_warn(sufile->i_sb, + "%s: invalid segment number: %llu", + __func__, (unsigned long long)*seg); + nerr++; + } + } + if (nerr > 0) { + ret = -EINVAL; + goto out_sem; + } + + ret = nilfs_sufile_get_header_block(sufile, &header_bh); + if (ret < 0) + goto out_sem; + + seg = segnumv; + blkoff = nilfs_sufile_get_blkoff(sufile, *seg); + ret = nilfs_mdt_get_block(sufile, blkoff, create, NULL, &bh); + if (ret < 0) + goto out_header; + + for (;;) { + dofunc(sufile, *seg, header_bh, bh); + + if (++seg >= segnumv + nsegs) + break; + prev_blkoff = blkoff; + blkoff = nilfs_sufile_get_blkoff(sufile, *seg); + if (blkoff == prev_blkoff) + continue; + + /* get different block */ + brelse(bh); + ret = nilfs_mdt_get_block(sufile, blkoff, create, NULL, &bh); + if (unlikely(ret < 0)) + goto out_header; + } + brelse(bh); + + out_header: + n = seg - segnumv; + brelse(header_bh); + out_sem: + up_write(&NILFS_MDT(sufile)->mi_sem); + out: + if (ndone) + *ndone = n; + return ret; +} + +int nilfs_sufile_update(struct inode *sufile, __u64 segnum, int create, + void (*dofunc)(struct inode *, __u64, + struct buffer_head *, + struct buffer_head *)) +{ + struct buffer_head *header_bh, *bh; + int ret; + + if (unlikely(segnum >= nilfs_sufile_get_nsegments(sufile))) { + nilfs_warn(sufile->i_sb, "%s: invalid segment number: %llu", + __func__, (unsigned long long)segnum); + return -EINVAL; + } + down_write(&NILFS_MDT(sufile)->mi_sem); + + ret = nilfs_sufile_get_header_block(sufile, &header_bh); + if (ret < 0) + goto out_sem; + + ret = nilfs_sufile_get_segment_usage_block(sufile, segnum, create, &bh); + if (!ret) { + dofunc(sufile, segnum, header_bh, bh); + brelse(bh); + } + brelse(header_bh); + + out_sem: + up_write(&NILFS_MDT(sufile)->mi_sem); + return ret; +} + +/** + * nilfs_sufile_set_alloc_range - limit range of segment to be allocated + * @sufile: inode of segment usage file + * @start: minimum segment number of allocatable region (inclusive) + * @end: maximum segment number of allocatable region (inclusive) + * + * Return: 0 on success, or %-ERANGE if segment range is invalid. + */ +int nilfs_sufile_set_alloc_range(struct inode *sufile, __u64 start, __u64 end) +{ + struct nilfs_sufile_info *sui = NILFS_SUI(sufile); + __u64 nsegs; + int ret = -ERANGE; + + down_write(&NILFS_MDT(sufile)->mi_sem); + nsegs = nilfs_sufile_get_nsegments(sufile); + + if (start <= end && end < nsegs) { + sui->allocmin = start; + sui->allocmax = end; + ret = 0; + } + up_write(&NILFS_MDT(sufile)->mi_sem); + return ret; +} + +/** + * nilfs_sufile_alloc - allocate a segment + * @sufile: inode of segment usage file + * @segnump: pointer to segment number + * + * Description: nilfs_sufile_alloc() allocates a clean segment, and stores + * its segment number in the place pointed to by @segnump. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOMEM - Insufficient memory available. + * * %-ENOSPC - No clean segment left. + */ +int nilfs_sufile_alloc(struct inode *sufile, __u64 *segnump) +{ + struct buffer_head *header_bh, *su_bh; + struct nilfs_sufile_header *header; + struct nilfs_segment_usage *su; + struct nilfs_sufile_info *sui = NILFS_SUI(sufile); + size_t susz = NILFS_MDT(sufile)->mi_entry_size; + __u64 segnum, maxsegnum, last_alloc; + size_t offset; + void *kaddr; + unsigned long nsegments, nsus, cnt; + int ret, j; + + down_write(&NILFS_MDT(sufile)->mi_sem); + + ret = nilfs_sufile_get_header_block(sufile, &header_bh); + if (ret < 0) + goto out_sem; + header = kmap_local_folio(header_bh->b_folio, 0); + last_alloc = le64_to_cpu(header->sh_last_alloc); + kunmap_local(header); + + nsegments = nilfs_sufile_get_nsegments(sufile); + maxsegnum = sui->allocmax; + segnum = last_alloc + 1; + if (segnum < sui->allocmin || segnum > sui->allocmax) + segnum = sui->allocmin; + + for (cnt = 0; cnt < nsegments; cnt += nsus) { + if (segnum > maxsegnum) { + if (cnt < sui->allocmax - sui->allocmin + 1) { + /* + * wrap around in the limited region. + * if allocation started from + * sui->allocmin, this never happens. + */ + segnum = sui->allocmin; + maxsegnum = last_alloc; + } else if (segnum > sui->allocmin && + sui->allocmax + 1 < nsegments) { + segnum = sui->allocmax + 1; + maxsegnum = nsegments - 1; + } else if (sui->allocmin > 0) { + segnum = 0; + maxsegnum = sui->allocmin - 1; + } else { + break; /* never happens */ + } + } + trace_nilfs2_segment_usage_check(sufile, segnum, cnt); + ret = nilfs_sufile_get_segment_usage_block(sufile, segnum, 1, + &su_bh); + if (ret < 0) + goto out_header; + + offset = nilfs_sufile_segment_usage_offset(sufile, segnum, + su_bh); + su = kaddr = kmap_local_folio(su_bh->b_folio, offset); + + nsus = nilfs_sufile_segment_usages_in_block( + sufile, segnum, maxsegnum); + for (j = 0; j < nsus; j++, su = (void *)su + susz, segnum++) { + if (!nilfs_segment_usage_clean(su)) + continue; + /* found a clean segment */ + nilfs_segment_usage_set_dirty(su); + kunmap_local(kaddr); + + header = kmap_local_folio(header_bh->b_folio, 0); + le64_add_cpu(&header->sh_ncleansegs, -1); + le64_add_cpu(&header->sh_ndirtysegs, 1); + header->sh_last_alloc = cpu_to_le64(segnum); + kunmap_local(header); + + sui->ncleansegs--; + mark_buffer_dirty(header_bh); + mark_buffer_dirty(su_bh); + nilfs_mdt_mark_dirty(sufile); + brelse(su_bh); + *segnump = segnum; + + trace_nilfs2_segment_usage_allocated(sufile, segnum); + + goto out_header; + } + + kunmap_local(kaddr); + brelse(su_bh); + } + + /* no segments left */ + ret = -ENOSPC; + + out_header: + brelse(header_bh); + + out_sem: + up_write(&NILFS_MDT(sufile)->mi_sem); + return ret; +} + +void nilfs_sufile_do_cancel_free(struct inode *sufile, __u64 segnum, + struct buffer_head *header_bh, + struct buffer_head *su_bh) +{ + struct nilfs_segment_usage *su; + size_t offset; + + offset = nilfs_sufile_segment_usage_offset(sufile, segnum, su_bh); + su = kmap_local_folio(su_bh->b_folio, offset); + if (unlikely(!nilfs_segment_usage_clean(su))) { + nilfs_warn(sufile->i_sb, "%s: segment %llu must be clean", + __func__, (unsigned long long)segnum); + kunmap_local(su); + return; + } + nilfs_segment_usage_set_dirty(su); + kunmap_local(su); + + nilfs_sufile_mod_counter(header_bh, -1, 1); + NILFS_SUI(sufile)->ncleansegs--; + + mark_buffer_dirty(su_bh); + nilfs_mdt_mark_dirty(sufile); +} + +void nilfs_sufile_do_scrap(struct inode *sufile, __u64 segnum, + struct buffer_head *header_bh, + struct buffer_head *su_bh) +{ + struct nilfs_segment_usage *su; + size_t offset; + int clean, dirty; + + offset = nilfs_sufile_segment_usage_offset(sufile, segnum, su_bh); + su = kmap_local_folio(su_bh->b_folio, offset); + if (su->su_flags == cpu_to_le32(BIT(NILFS_SEGMENT_USAGE_DIRTY)) && + su->su_nblocks == cpu_to_le32(0)) { + kunmap_local(su); + return; + } + clean = nilfs_segment_usage_clean(su); + dirty = nilfs_segment_usage_dirty(su); + + /* make the segment garbage */ + su->su_lastmod = cpu_to_le64(0); + su->su_nblocks = cpu_to_le32(0); + su->su_flags = cpu_to_le32(BIT(NILFS_SEGMENT_USAGE_DIRTY)); + kunmap_local(su); + + nilfs_sufile_mod_counter(header_bh, clean ? (u64)-1 : 0, dirty ? 0 : 1); + NILFS_SUI(sufile)->ncleansegs -= clean; + + mark_buffer_dirty(su_bh); + nilfs_mdt_mark_dirty(sufile); +} + +void nilfs_sufile_do_free(struct inode *sufile, __u64 segnum, + struct buffer_head *header_bh, + struct buffer_head *su_bh) +{ + struct nilfs_segment_usage *su; + size_t offset; + int sudirty; + + offset = nilfs_sufile_segment_usage_offset(sufile, segnum, su_bh); + su = kmap_local_folio(su_bh->b_folio, offset); + if (nilfs_segment_usage_clean(su)) { + nilfs_warn(sufile->i_sb, "%s: segment %llu is already clean", + __func__, (unsigned long long)segnum); + kunmap_local(su); + return; + } + if (unlikely(nilfs_segment_usage_error(su))) + nilfs_warn(sufile->i_sb, "free segment %llu marked in error", + (unsigned long long)segnum); + + sudirty = nilfs_segment_usage_dirty(su); + if (unlikely(!sudirty)) + nilfs_warn(sufile->i_sb, "free unallocated segment %llu", + (unsigned long long)segnum); + + nilfs_segment_usage_set_clean(su); + kunmap_local(su); + mark_buffer_dirty(su_bh); + + nilfs_sufile_mod_counter(header_bh, 1, sudirty ? (u64)-1 : 0); + NILFS_SUI(sufile)->ncleansegs++; + + nilfs_mdt_mark_dirty(sufile); + + trace_nilfs2_segment_usage_freed(sufile, segnum); +} + +/** + * nilfs_sufile_mark_dirty - mark the buffer having a segment usage dirty + * @sufile: inode of segment usage file + * @segnum: segment number + * + * Return: 0 on success, or a negative error code on failure. + */ +int nilfs_sufile_mark_dirty(struct inode *sufile, __u64 segnum) +{ + struct buffer_head *bh; + size_t offset; + struct nilfs_segment_usage *su; + int ret; + + down_write(&NILFS_MDT(sufile)->mi_sem); + ret = nilfs_sufile_get_segment_usage_block(sufile, segnum, 0, &bh); + if (unlikely(ret)) { + if (ret == -ENOENT) { + nilfs_error(sufile->i_sb, + "segment usage for segment %llu is unreadable due to a hole block", + (unsigned long long)segnum); + ret = -EIO; + } + goto out_sem; + } + + offset = nilfs_sufile_segment_usage_offset(sufile, segnum, bh); + su = kmap_local_folio(bh->b_folio, offset); + if (unlikely(nilfs_segment_usage_error(su))) { + struct the_nilfs *nilfs = sufile->i_sb->s_fs_info; + + kunmap_local(su); + brelse(bh); + if (nilfs_segment_is_active(nilfs, segnum)) { + nilfs_error(sufile->i_sb, + "active segment %llu is erroneous", + (unsigned long long)segnum); + } else { + /* + * Segments marked erroneous are never allocated by + * nilfs_sufile_alloc(); only active segments, ie, + * the segments indexed by ns_segnum or ns_nextnum, + * can be erroneous here. + */ + WARN_ON_ONCE(1); + } + ret = -EIO; + } else { + nilfs_segment_usage_set_dirty(su); + kunmap_local(su); + mark_buffer_dirty(bh); + nilfs_mdt_mark_dirty(sufile); + brelse(bh); + } +out_sem: + up_write(&NILFS_MDT(sufile)->mi_sem); + return ret; +} + +/** + * nilfs_sufile_set_segment_usage - set usage of a segment + * @sufile: inode of segment usage file + * @segnum: segment number + * @nblocks: number of live blocks in the segment + * @modtime: modification time (option) + * + * Return: 0 on success, or a negative error code on failure. + */ +int nilfs_sufile_set_segment_usage(struct inode *sufile, __u64 segnum, + unsigned long nblocks, time64_t modtime) +{ + struct buffer_head *bh; + struct nilfs_segment_usage *su; + size_t offset; + int ret; + + down_write(&NILFS_MDT(sufile)->mi_sem); + ret = nilfs_sufile_get_segment_usage_block(sufile, segnum, 0, &bh); + if (ret < 0) + goto out_sem; + + offset = nilfs_sufile_segment_usage_offset(sufile, segnum, bh); + su = kmap_local_folio(bh->b_folio, offset); + if (modtime) { + /* + * Check segusage error and set su_lastmod only when updating + * this entry with a valid timestamp, not for cancellation. + */ + WARN_ON_ONCE(nilfs_segment_usage_error(su)); + su->su_lastmod = cpu_to_le64(modtime); + } + su->su_nblocks = cpu_to_le32(nblocks); + kunmap_local(su); + + mark_buffer_dirty(bh); + nilfs_mdt_mark_dirty(sufile); + brelse(bh); + + out_sem: + up_write(&NILFS_MDT(sufile)->mi_sem); + return ret; +} + +/** + * nilfs_sufile_get_stat - get segment usage statistics + * @sufile: inode of segment usage file + * @sustat: pointer to a structure of segment usage statistics + * + * Description: nilfs_sufile_get_stat() retrieves segment usage statistics + * and stores them in the location pointed to by @sustat. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOMEM - Insufficient memory available. + */ +int nilfs_sufile_get_stat(struct inode *sufile, struct nilfs_sustat *sustat) +{ + struct buffer_head *header_bh; + struct nilfs_sufile_header *header; + struct the_nilfs *nilfs = sufile->i_sb->s_fs_info; + int ret; + + down_read(&NILFS_MDT(sufile)->mi_sem); + + ret = nilfs_sufile_get_header_block(sufile, &header_bh); + if (ret < 0) + goto out_sem; + + header = kmap_local_folio(header_bh->b_folio, 0); + sustat->ss_nsegs = nilfs_sufile_get_nsegments(sufile); + sustat->ss_ncleansegs = le64_to_cpu(header->sh_ncleansegs); + sustat->ss_ndirtysegs = le64_to_cpu(header->sh_ndirtysegs); + sustat->ss_ctime = nilfs->ns_ctime; + sustat->ss_nongc_ctime = nilfs->ns_nongc_ctime; + spin_lock(&nilfs->ns_last_segment_lock); + sustat->ss_prot_seq = nilfs->ns_prot_seq; + spin_unlock(&nilfs->ns_last_segment_lock); + kunmap_local(header); + brelse(header_bh); + + out_sem: + up_read(&NILFS_MDT(sufile)->mi_sem); + return ret; +} + +void nilfs_sufile_do_set_error(struct inode *sufile, __u64 segnum, + struct buffer_head *header_bh, + struct buffer_head *su_bh) +{ + struct nilfs_segment_usage *su; + size_t offset; + int suclean; + + offset = nilfs_sufile_segment_usage_offset(sufile, segnum, su_bh); + su = kmap_local_folio(su_bh->b_folio, offset); + if (nilfs_segment_usage_error(su)) { + kunmap_local(su); + return; + } + suclean = nilfs_segment_usage_clean(su); + nilfs_segment_usage_set_error(su); + kunmap_local(su); + + if (suclean) { + nilfs_sufile_mod_counter(header_bh, -1, 0); + NILFS_SUI(sufile)->ncleansegs--; + } + mark_buffer_dirty(su_bh); + nilfs_mdt_mark_dirty(sufile); +} + +/** + * nilfs_sufile_truncate_range - truncate range of segment array + * @sufile: inode of segment usage file + * @start: start segment number (inclusive) + * @end: end segment number (inclusive) + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EBUSY - Dirty or active segments are present in the range. + * * %-EINVAL - Invalid number of segments specified. + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOMEM - Insufficient memory available. + */ +static int nilfs_sufile_truncate_range(struct inode *sufile, + __u64 start, __u64 end) +{ + struct the_nilfs *nilfs = sufile->i_sb->s_fs_info; + struct buffer_head *header_bh; + struct buffer_head *su_bh; + struct nilfs_segment_usage *su, *su2; + size_t susz = NILFS_MDT(sufile)->mi_entry_size; + unsigned long segusages_per_block; + unsigned long nsegs, ncleaned; + __u64 segnum; + size_t offset; + ssize_t n, nc; + int ret; + int j; + + nsegs = nilfs_sufile_get_nsegments(sufile); + + ret = -EINVAL; + if (start > end || start >= nsegs) + goto out; + + ret = nilfs_sufile_get_header_block(sufile, &header_bh); + if (ret < 0) + goto out; + + segusages_per_block = nilfs_sufile_segment_usages_per_block(sufile); + ncleaned = 0; + + for (segnum = start; segnum <= end; segnum += n) { + n = min_t(unsigned long, + segusages_per_block - + nilfs_sufile_get_offset(sufile, segnum), + end - segnum + 1); + ret = nilfs_sufile_get_segment_usage_block(sufile, segnum, 0, + &su_bh); + if (ret < 0) { + if (ret != -ENOENT) + goto out_header; + /* hole */ + continue; + } + offset = nilfs_sufile_segment_usage_offset(sufile, segnum, + su_bh); + su = kmap_local_folio(su_bh->b_folio, offset); + su2 = su; + for (j = 0; j < n; j++, su = (void *)su + susz) { + if ((le32_to_cpu(su->su_flags) & + ~BIT(NILFS_SEGMENT_USAGE_ERROR)) || + nilfs_segment_is_active(nilfs, segnum + j)) { + ret = -EBUSY; + kunmap_local(su2); + brelse(su_bh); + goto out_header; + } + } + nc = 0; + for (su = su2, j = 0; j < n; j++, su = (void *)su + susz) { + if (nilfs_segment_usage_error(su)) { + nilfs_segment_usage_set_clean(su); + nc++; + } + } + kunmap_local(su2); + if (nc > 0) { + mark_buffer_dirty(su_bh); + ncleaned += nc; + } + brelse(su_bh); + + if (n == segusages_per_block) { + /* make hole */ + nilfs_sufile_delete_segment_usage_block(sufile, segnum); + } + } + ret = 0; + +out_header: + if (ncleaned > 0) { + NILFS_SUI(sufile)->ncleansegs += ncleaned; + nilfs_sufile_mod_counter(header_bh, ncleaned, 0); + nilfs_mdt_mark_dirty(sufile); + } + brelse(header_bh); +out: + return ret; +} + +/** + * nilfs_sufile_resize - resize segment array + * @sufile: inode of segment usage file + * @newnsegs: new number of segments + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EBUSY - Dirty or active segments exist in the region to be truncated. + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOMEM - Insufficient memory available. + * * %-ENOSPC - Enough free space is not left for shrinking. + */ +int nilfs_sufile_resize(struct inode *sufile, __u64 newnsegs) +{ + struct the_nilfs *nilfs = sufile->i_sb->s_fs_info; + struct buffer_head *header_bh; + struct nilfs_sufile_header *header; + struct nilfs_sufile_info *sui = NILFS_SUI(sufile); + unsigned long nsegs, nrsvsegs; + int ret = 0; + + down_write(&NILFS_MDT(sufile)->mi_sem); + + nsegs = nilfs_sufile_get_nsegments(sufile); + if (nsegs == newnsegs) + goto out; + + ret = -ENOSPC; + nrsvsegs = nilfs_nrsvsegs(nilfs, newnsegs); + if (newnsegs < nsegs && nsegs - newnsegs + nrsvsegs > sui->ncleansegs) + goto out; + + ret = nilfs_sufile_get_header_block(sufile, &header_bh); + if (ret < 0) + goto out; + + if (newnsegs > nsegs) { + sui->ncleansegs += newnsegs - nsegs; + } else /* newnsegs < nsegs */ { + ret = nilfs_sufile_truncate_range(sufile, newnsegs, nsegs - 1); + if (ret < 0) + goto out_header; + + sui->ncleansegs -= nsegs - newnsegs; + + /* + * If the sufile is successfully truncated, immediately adjust + * the segment allocation space while locking the semaphore + * "mi_sem" so that nilfs_sufile_alloc() never allocates + * segments in the truncated space. + */ + sui->allocmax = newnsegs - 1; + sui->allocmin = 0; + } + + header = kmap_local_folio(header_bh->b_folio, 0); + header->sh_ncleansegs = cpu_to_le64(sui->ncleansegs); + kunmap_local(header); + + mark_buffer_dirty(header_bh); + nilfs_mdt_mark_dirty(sufile); + nilfs_set_nsegments(nilfs, newnsegs); + +out_header: + brelse(header_bh); +out: + up_write(&NILFS_MDT(sufile)->mi_sem); + return ret; +} + +/** + * nilfs_sufile_get_suinfo - get segment usage information + * @sufile: inode of segment usage file + * @segnum: segment number to start looking + * @buf: array of suinfo + * @sisz: byte size of suinfo + * @nsi: size of suinfo array + * + * Return: Count of segment usage info items stored in the output buffer on + * success, or one of the following negative error codes on failure: + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOMEM - Insufficient memory available. + */ +ssize_t nilfs_sufile_get_suinfo(struct inode *sufile, __u64 segnum, void *buf, + unsigned int sisz, size_t nsi) +{ + struct buffer_head *su_bh; + struct nilfs_segment_usage *su; + struct nilfs_suinfo *si = buf; + size_t susz = NILFS_MDT(sufile)->mi_entry_size; + struct the_nilfs *nilfs = sufile->i_sb->s_fs_info; + size_t offset; + void *kaddr; + unsigned long nsegs, segusages_per_block; + ssize_t n; + int ret, i, j; + + down_read(&NILFS_MDT(sufile)->mi_sem); + + segusages_per_block = nilfs_sufile_segment_usages_per_block(sufile); + nsegs = min_t(unsigned long, + nilfs_sufile_get_nsegments(sufile) - segnum, + nsi); + for (i = 0; i < nsegs; i += n, segnum += n) { + n = min_t(unsigned long, + segusages_per_block - + nilfs_sufile_get_offset(sufile, segnum), + nsegs - i); + ret = nilfs_sufile_get_segment_usage_block(sufile, segnum, 0, + &su_bh); + if (ret < 0) { + if (ret != -ENOENT) + goto out; + /* hole */ + memset(si, 0, sisz * n); + si = (void *)si + sisz * n; + continue; + } + + offset = nilfs_sufile_segment_usage_offset(sufile, segnum, + su_bh); + su = kaddr = kmap_local_folio(su_bh->b_folio, offset); + for (j = 0; j < n; + j++, su = (void *)su + susz, si = (void *)si + sisz) { + si->sui_lastmod = le64_to_cpu(su->su_lastmod); + si->sui_nblocks = le32_to_cpu(su->su_nblocks); + si->sui_flags = le32_to_cpu(su->su_flags) & + ~BIT(NILFS_SEGMENT_USAGE_ACTIVE); + if (nilfs_segment_is_active(nilfs, segnum + j)) + si->sui_flags |= + BIT(NILFS_SEGMENT_USAGE_ACTIVE); + } + kunmap_local(kaddr); + brelse(su_bh); + } + ret = nsegs; + + out: + up_read(&NILFS_MDT(sufile)->mi_sem); + return ret; +} + +/** + * nilfs_sufile_set_suinfo - sets segment usage info + * @sufile: inode of segment usage file + * @buf: array of suinfo_update + * @supsz: byte size of suinfo_update + * @nsup: size of suinfo_update array + * + * Description: Takes an array of nilfs_suinfo_update structs and updates + * segment usage accordingly. Only the fields indicated by the sup_flags + * are updated. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EINVAL - Invalid values in input (segment number, flags or nblocks). + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOMEM - Insufficient memory available. + */ +ssize_t nilfs_sufile_set_suinfo(struct inode *sufile, void *buf, + unsigned int supsz, size_t nsup) +{ + struct the_nilfs *nilfs = sufile->i_sb->s_fs_info; + struct buffer_head *header_bh, *bh; + struct nilfs_suinfo_update *sup, *supend = buf + supsz * nsup; + struct nilfs_segment_usage *su; + size_t offset; + unsigned long blkoff, prev_blkoff; + int cleansi, cleansu, dirtysi, dirtysu; + long ncleaned = 0, ndirtied = 0; + int ret = 0; + + if (unlikely(nsup == 0)) + return ret; + + for (sup = buf; sup < supend; sup = (void *)sup + supsz) { + if (sup->sup_segnum >= nilfs->ns_nsegments + || (sup->sup_flags & + (~0UL << __NR_NILFS_SUINFO_UPDATE_FIELDS)) + || (nilfs_suinfo_update_nblocks(sup) && + sup->sup_sui.sui_nblocks > + nilfs->ns_blocks_per_segment)) + return -EINVAL; + } + + down_write(&NILFS_MDT(sufile)->mi_sem); + + ret = nilfs_sufile_get_header_block(sufile, &header_bh); + if (ret < 0) + goto out_sem; + + sup = buf; + blkoff = nilfs_sufile_get_blkoff(sufile, sup->sup_segnum); + ret = nilfs_mdt_get_block(sufile, blkoff, 1, NULL, &bh); + if (ret < 0) + goto out_header; + + for (;;) { + offset = nilfs_sufile_segment_usage_offset( + sufile, sup->sup_segnum, bh); + su = kmap_local_folio(bh->b_folio, offset); + + if (nilfs_suinfo_update_lastmod(sup)) + su->su_lastmod = cpu_to_le64(sup->sup_sui.sui_lastmod); + + if (nilfs_suinfo_update_nblocks(sup)) + su->su_nblocks = cpu_to_le32(sup->sup_sui.sui_nblocks); + + if (nilfs_suinfo_update_flags(sup)) { + /* + * Active flag is a virtual flag projected by running + * nilfs kernel code - drop it not to write it to + * disk. + */ + sup->sup_sui.sui_flags &= + ~BIT(NILFS_SEGMENT_USAGE_ACTIVE); + + cleansi = nilfs_suinfo_clean(&sup->sup_sui); + cleansu = nilfs_segment_usage_clean(su); + dirtysi = nilfs_suinfo_dirty(&sup->sup_sui); + dirtysu = nilfs_segment_usage_dirty(su); + + if (cleansi && !cleansu) + ++ncleaned; + else if (!cleansi && cleansu) + --ncleaned; + + if (dirtysi && !dirtysu) + ++ndirtied; + else if (!dirtysi && dirtysu) + --ndirtied; + + su->su_flags = cpu_to_le32(sup->sup_sui.sui_flags); + } + + kunmap_local(su); + + sup = (void *)sup + supsz; + if (sup >= supend) + break; + + prev_blkoff = blkoff; + blkoff = nilfs_sufile_get_blkoff(sufile, sup->sup_segnum); + if (blkoff == prev_blkoff) + continue; + + /* get different block */ + mark_buffer_dirty(bh); + put_bh(bh); + ret = nilfs_mdt_get_block(sufile, blkoff, 1, NULL, &bh); + if (unlikely(ret < 0)) + goto out_mark; + } + mark_buffer_dirty(bh); + put_bh(bh); + + out_mark: + if (ncleaned || ndirtied) { + nilfs_sufile_mod_counter(header_bh, (u64)ncleaned, + (u64)ndirtied); + NILFS_SUI(sufile)->ncleansegs += ncleaned; + } + nilfs_mdt_mark_dirty(sufile); + out_header: + put_bh(header_bh); + out_sem: + up_write(&NILFS_MDT(sufile)->mi_sem); + return ret; +} + +/** + * nilfs_sufile_trim_fs() - trim ioctl handle function + * @sufile: inode of segment usage file + * @range: fstrim_range structure + * + * start: First Byte to trim + * len: number of Bytes to trim from start + * minlen: minimum extent length in Bytes + * + * Decription: nilfs_sufile_trim_fs goes through all segments containing bytes + * from start to start+len. start is rounded up to the next block boundary + * and start+len is rounded down. For each clean segment blkdev_issue_discard + * function is invoked. + * + * Return: 0 on success, or a negative error code on failure. + */ +int nilfs_sufile_trim_fs(struct inode *sufile, struct fstrim_range *range) +{ + struct the_nilfs *nilfs = sufile->i_sb->s_fs_info; + struct buffer_head *su_bh; + struct nilfs_segment_usage *su; + size_t offset; + void *kaddr; + size_t n, i, susz = NILFS_MDT(sufile)->mi_entry_size; + sector_t seg_start, seg_end, start_block, end_block; + sector_t start = 0, nblocks = 0; + u64 segnum, segnum_end, minlen, len, max_blocks, ndiscarded = 0; + int ret = 0; + unsigned int sects_per_block; + + sects_per_block = (1 << nilfs->ns_blocksize_bits) / + bdev_logical_block_size(nilfs->ns_bdev); + len = range->len >> nilfs->ns_blocksize_bits; + minlen = range->minlen >> nilfs->ns_blocksize_bits; + max_blocks = ((u64)nilfs->ns_nsegments * nilfs->ns_blocks_per_segment); + + if (!len || range->start >= max_blocks << nilfs->ns_blocksize_bits) + return -EINVAL; + + start_block = (range->start + nilfs->ns_blocksize - 1) >> + nilfs->ns_blocksize_bits; + + /* + * range->len can be very large (actually, it is set to + * ULLONG_MAX by default) - truncate upper end of the range + * carefully so as not to overflow. + */ + if (max_blocks - start_block < len) + end_block = max_blocks - 1; + else + end_block = start_block + len - 1; + + if (end_block < nilfs->ns_first_data_block) + goto out; + + segnum = nilfs_get_segnum_of_block(nilfs, start_block); + segnum_end = nilfs_get_segnum_of_block(nilfs, end_block); + + down_read(&NILFS_MDT(sufile)->mi_sem); + + while (segnum <= segnum_end) { + n = nilfs_sufile_segment_usages_in_block(sufile, segnum, + segnum_end); + + ret = nilfs_sufile_get_segment_usage_block(sufile, segnum, 0, + &su_bh); + if (ret < 0) { + if (ret != -ENOENT) + goto out_sem; + /* hole */ + segnum += n; + continue; + } + + offset = nilfs_sufile_segment_usage_offset(sufile, segnum, + su_bh); + su = kaddr = kmap_local_folio(su_bh->b_folio, offset); + for (i = 0; i < n; ++i, ++segnum, su = (void *)su + susz) { + if (!nilfs_segment_usage_clean(su)) + continue; + + nilfs_get_segment_range(nilfs, segnum, &seg_start, + &seg_end); + + if (!nblocks) { + /* start new extent */ + start = seg_start; + nblocks = seg_end - seg_start + 1; + continue; + } + + if (start + nblocks == seg_start) { + /* add to previous extent */ + nblocks += seg_end - seg_start + 1; + continue; + } + + /* discard previous extent */ + if (start < start_block) { + nblocks -= start_block - start; + start = start_block; + } + + if (nblocks >= minlen) { + kunmap_local(kaddr); + + ret = blkdev_issue_discard(nilfs->ns_bdev, + start * sects_per_block, + nblocks * sects_per_block, + GFP_NOFS); + if (ret < 0) { + put_bh(su_bh); + goto out_sem; + } + + ndiscarded += nblocks; + offset = nilfs_sufile_segment_usage_offset( + sufile, segnum, su_bh); + su = kaddr = kmap_local_folio(su_bh->b_folio, + offset); + } + + /* start new extent */ + start = seg_start; + nblocks = seg_end - seg_start + 1; + } + kunmap_local(kaddr); + put_bh(su_bh); + } + + + if (nblocks) { + /* discard last extent */ + if (start < start_block) { + nblocks -= start_block - start; + start = start_block; + } + if (start + nblocks > end_block + 1) + nblocks = end_block - start + 1; + + if (nblocks >= minlen) { + ret = blkdev_issue_discard(nilfs->ns_bdev, + start * sects_per_block, + nblocks * sects_per_block, + GFP_NOFS); + if (!ret) + ndiscarded += nblocks; + } + } + +out_sem: + up_read(&NILFS_MDT(sufile)->mi_sem); + +out: + range->len = ndiscarded << nilfs->ns_blocksize_bits; + return ret; +} + +/** + * nilfs_sufile_read - read or get sufile inode + * @sb: super block instance + * @susize: size of a segment usage entry + * @raw_inode: on-disk sufile inode + * @inodep: buffer to store the inode + * + * Return: 0 on success, or a negative error code on failure. + */ +int nilfs_sufile_read(struct super_block *sb, size_t susize, + struct nilfs_inode *raw_inode, struct inode **inodep) +{ + struct inode *sufile; + struct nilfs_sufile_info *sui; + struct buffer_head *header_bh; + struct nilfs_sufile_header *header; + int err; + + if (susize > sb->s_blocksize) { + nilfs_err(sb, "too large segment usage size: %zu bytes", + susize); + return -EINVAL; + } else if (susize < NILFS_MIN_SEGMENT_USAGE_SIZE) { + nilfs_err(sb, "too small segment usage size: %zu bytes", + susize); + return -EINVAL; + } + + sufile = nilfs_iget_locked(sb, NULL, NILFS_SUFILE_INO); + if (unlikely(!sufile)) + return -ENOMEM; + if (!(inode_state_read_once(sufile) & I_NEW)) + goto out; + + err = nilfs_mdt_init(sufile, NILFS_MDT_GFP, sizeof(*sui)); + if (err) + goto failed; + + nilfs_mdt_set_entry_size(sufile, susize, + sizeof(struct nilfs_sufile_header)); + + err = nilfs_read_inode_common(sufile, raw_inode); + if (err) + goto failed; + + err = nilfs_mdt_get_block(sufile, 0, 0, NULL, &header_bh); + if (unlikely(err)) { + if (err == -ENOENT) { + nilfs_err(sb, + "missing header block in segment usage metadata"); + err = -EINVAL; + } + goto failed; + } + + sui = NILFS_SUI(sufile); + header = kmap_local_folio(header_bh->b_folio, 0); + sui->ncleansegs = le64_to_cpu(header->sh_ncleansegs); + kunmap_local(header); + brelse(header_bh); + + sui->allocmax = nilfs_sufile_get_nsegments(sufile) - 1; + sui->allocmin = 0; + + unlock_new_inode(sufile); + out: + *inodep = sufile; + return 0; + failed: + iget_failed(sufile); + return err; +} diff --git a/fs/nilfs2/sufile.h b/fs/nilfs2/sufile.h new file mode 100644 index 000000000..5888ed479 --- /dev/null +++ b/fs/nilfs2/sufile.h @@ -0,0 +1,162 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * NILFS segment usage file. + * + * Copyright (C) 2006-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Koji Sato. + */ + +#ifndef _NILFS_SUFILE_H +#define _NILFS_SUFILE_H + +#include +#include +#include +#include "mdt.h" + + +static inline unsigned long nilfs_sufile_get_nsegments(struct inode *sufile) +{ + return ((struct the_nilfs *)sufile->i_sb->s_fs_info)->ns_nsegments; +} + +unsigned long nilfs_sufile_get_ncleansegs(struct inode *sufile); + +int nilfs_sufile_set_alloc_range(struct inode *sufile, __u64 start, __u64 end); +int nilfs_sufile_alloc(struct inode *, __u64 *); +int nilfs_sufile_mark_dirty(struct inode *sufile, __u64 segnum); +int nilfs_sufile_set_segment_usage(struct inode *sufile, __u64 segnum, + unsigned long nblocks, time64_t modtime); +int nilfs_sufile_get_stat(struct inode *, struct nilfs_sustat *); +ssize_t nilfs_sufile_get_suinfo(struct inode *, __u64, void *, unsigned int, + size_t); +ssize_t nilfs_sufile_set_suinfo(struct inode *, void *, unsigned int, size_t); + +int nilfs_sufile_updatev(struct inode *, __u64 *, size_t, int, size_t *, + void (*dofunc)(struct inode *, __u64, + struct buffer_head *, + struct buffer_head *)); +int nilfs_sufile_update(struct inode *, __u64, int, + void (*dofunc)(struct inode *, __u64, + struct buffer_head *, + struct buffer_head *)); +void nilfs_sufile_do_scrap(struct inode *, __u64, struct buffer_head *, + struct buffer_head *); +void nilfs_sufile_do_free(struct inode *, __u64, struct buffer_head *, + struct buffer_head *); +void nilfs_sufile_do_cancel_free(struct inode *, __u64, struct buffer_head *, + struct buffer_head *); +void nilfs_sufile_do_set_error(struct inode *, __u64, struct buffer_head *, + struct buffer_head *); + +int nilfs_sufile_resize(struct inode *sufile, __u64 newnsegs); +int nilfs_sufile_read(struct super_block *sb, size_t susize, + struct nilfs_inode *raw_inode, struct inode **inodep); +int nilfs_sufile_trim_fs(struct inode *sufile, struct fstrim_range *range); + +/** + * nilfs_sufile_warn_on_error - warn on unexpected sufile error + * @sufile: inode of segment usage file + * @err: status code returned by a sufile function + * + * Even if buffer heads of blocks containing segment usage entries have + * been dirtied in advance by calling functions such as + * nilfs_sufile_mark_dirty() or nilfs_sufile_{alloc,free}(), those buffers + * can be discarded from memory after the file system detects corruption and + * degrades to read-only mode, which may cause sufile operations, including + * cancel operations, to return errors. nilfs_sufile_warn_on_error() is used + * to detect unexpected errors other than during read-only degradation. + * + * Return: 0 if @err is 0, %-EROFS if in read-only degraded mode, and %-EIO + * otherwise. + */ +#define nilfs_sufile_warn_on_error(sufile, err) \ + ({ \ + int _err = (err); \ + \ + if (unlikely(_err)) \ + _err = WARN_ONCE(!sb_rdonly((sufile)->i_sb), \ + "unexpected sufile error %d\n", _err) ? \ + -EIO : -EROFS; \ + _err; \ + }) + +/** + * nilfs_sufile_scrap - make a segment garbage + * @sufile: inode of segment usage file + * @segnum: segment number to be freed + * + * Return: 0 on success, or a negative error code on failure. + */ +static inline int nilfs_sufile_scrap(struct inode *sufile, __u64 segnum) +{ + return nilfs_sufile_update(sufile, segnum, 1, nilfs_sufile_do_scrap); +} + +/** + * nilfs_sufile_free - free segment + * @sufile: inode of segment usage file + * @segnum: segment number to be freed + * + * Return: 0 on success, or a negative error code on failure. + */ +static inline int nilfs_sufile_free(struct inode *sufile, __u64 segnum) +{ + return nilfs_sufile_update(sufile, segnum, 0, nilfs_sufile_do_free); +} + +/** + * nilfs_sufile_freev - free segments + * @sufile: inode of segment usage file + * @segnumv: array of segment numbers + * @nsegs: size of @segnumv array + * @ndone: place to store the number of freed segments + * + * Return: 0 on success, or a negative error code on failure. + */ +static inline int nilfs_sufile_freev(struct inode *sufile, __u64 *segnumv, + size_t nsegs, size_t *ndone) +{ + return nilfs_sufile_updatev(sufile, segnumv, nsegs, 0, ndone, + nilfs_sufile_do_free); +} + +/** + * nilfs_sufile_cancel_freev - reallocate freeing segments + * @sufile: inode of segment usage file + * @segnumv: array of segment numbers + * @nsegs: size of @segnumv array + * @ndone: place to store the number of cancelled segments + * + * Return: 0 on success, or a negative error code on failure. + */ +static inline int nilfs_sufile_cancel_freev(struct inode *sufile, + __u64 *segnumv, size_t nsegs, + size_t *ndone) +{ + return nilfs_sufile_updatev(sufile, segnumv, nsegs, 0, ndone, + nilfs_sufile_do_cancel_free); +} + +/** + * nilfs_sufile_set_error - mark a segment as erroneous + * @sufile: inode of segment usage file + * @segnum: segment number + * + * Description: nilfs_sufile_set_error() marks the segment specified by + * @segnum as erroneous. The error segment will never be used again. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EINVAL - Invalid segment usage number. + * * %-EIO - I/O error (including metadata corruption). + * * %-ENOMEM - Insufficient memory available. + */ +static inline int nilfs_sufile_set_error(struct inode *sufile, __u64 segnum) +{ + return nilfs_sufile_update(sufile, segnum, 0, + nilfs_sufile_do_set_error); +} + +#endif /* _NILFS_SUFILE_H */ diff --git a/fs/nilfs2/super.c b/fs/nilfs2/super.c new file mode 100644 index 000000000..893a504cb --- /dev/null +++ b/fs/nilfs2/super.c @@ -0,0 +1,1413 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * NILFS module and super block management. + * + * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Ryusuke Konishi. + */ +/* + * linux/fs/ext2/super.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * from + * + * linux/fs/minix/inode.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * Big-endian to little-endian byte-swapping/bitmaps by + * David S. Miller (davem@caip.rutgers.edu), 1995 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "nilfs.h" +#include "export.h" +#include "mdt.h" +#include "alloc.h" +#include "btree.h" +#include "btnode.h" +#include "page.h" +#include "cpfile.h" +#include "sufile.h" /* nilfs_sufile_resize(), nilfs_sufile_set_alloc_range() */ +#include "ifile.h" +#include "dat.h" +#include "segment.h" +#include "segbuf.h" + +MODULE_AUTHOR("NTT Corp."); +MODULE_DESCRIPTION("A New Implementation of the Log-structured Filesystem " + "(NILFS)"); +MODULE_LICENSE("GPL"); + +static struct kmem_cache *nilfs_inode_cachep; +struct kmem_cache *nilfs_transaction_cachep; +struct kmem_cache *nilfs_segbuf_cachep; +struct kmem_cache *nilfs_btree_path_cache; + +static int nilfs_setup_super(struct super_block *sb, int is_mount); + +void __nilfs_msg(struct super_block *sb, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + int level; + + va_start(args, fmt); + + level = printk_get_level(fmt); + vaf.fmt = printk_skip_level(fmt); + vaf.va = &args; + + if (sb) + printk("%c%cNILFS (%s): %pV\n", + KERN_SOH_ASCII, level, sb->s_id, &vaf); + else + printk("%c%cNILFS: %pV\n", + KERN_SOH_ASCII, level, &vaf); + + va_end(args); +} + +static void nilfs_set_error(struct super_block *sb) +{ + struct the_nilfs *nilfs = sb->s_fs_info; + struct nilfs_super_block **sbp; + + down_write(&nilfs->ns_sem); + if (!(nilfs->ns_mount_state & NILFS_ERROR_FS)) { + nilfs->ns_mount_state |= NILFS_ERROR_FS; + sbp = nilfs_prepare_super(sb, 0); + if (likely(sbp)) { + sbp[0]->s_state |= cpu_to_le16(NILFS_ERROR_FS); + if (sbp[1]) + sbp[1]->s_state |= cpu_to_le16(NILFS_ERROR_FS); + nilfs_commit_super(sb, NILFS_SB_COMMIT_ALL); + } + } + up_write(&nilfs->ns_sem); +} + +/** + * __nilfs_error() - report failure condition on a filesystem + * @sb: super block instance + * @function: name of calling function + * @fmt: format string for message to be output + * @...: optional arguments to @fmt + * + * __nilfs_error() sets an ERROR_FS flag on the superblock as well as + * reporting an error message. This function should be called when + * NILFS detects incoherences or defects of meta data on disk. + * + * This implements the body of nilfs_error() macro. Normally, + * nilfs_error() should be used. As for sustainable errors such as a + * single-shot I/O error, nilfs_err() should be used instead. + * + * Callers should not add a trailing newline since this will do it. + */ +void __nilfs_error(struct super_block *sb, const char *function, + const char *fmt, ...) +{ + struct the_nilfs *nilfs = sb->s_fs_info; + struct va_format vaf; + va_list args; + + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + printk(KERN_CRIT "NILFS error (device %s): %s: %pV\n", + sb->s_id, function, &vaf); + + va_end(args); + + if (!sb_rdonly(sb)) { + nilfs_set_error(sb); + + if (nilfs_test_opt(nilfs, ERRORS_RO)) { + printk(KERN_CRIT "Remounting filesystem read-only\n"); + sb->s_flags |= SB_RDONLY; + } + } + + if (nilfs_test_opt(nilfs, ERRORS_PANIC)) + panic("NILFS (device %s): panic forced after error\n", + sb->s_id); +} + +struct inode *nilfs_alloc_inode(struct super_block *sb) +{ + struct nilfs_inode_info *ii; + + ii = alloc_inode_sb(sb, nilfs_inode_cachep, GFP_NOFS); + if (!ii) + return NULL; + ii->i_bh = NULL; + ii->i_state = 0; + ii->i_type = 0; + ii->i_cno = 0; + ii->i_assoc_inode = NULL; + ii->i_bmap = &ii->i_bmap_data; + return &ii->vfs_inode; +} + +static void nilfs_free_inode(struct inode *inode) +{ + if (nilfs_is_metadata_file_inode(inode)) + nilfs_mdt_destroy(inode); + + kmem_cache_free(nilfs_inode_cachep, NILFS_I(inode)); +} + +static int nilfs_sync_super(struct super_block *sb, int flag) +{ + struct the_nilfs *nilfs = sb->s_fs_info; + int err; + + retry: + set_buffer_dirty(nilfs->ns_sbh[0]); + if (nilfs_test_opt(nilfs, BARRIER)) { + err = __sync_dirty_buffer(nilfs->ns_sbh[0], + REQ_SYNC | REQ_PREFLUSH | REQ_FUA); + } else { + err = sync_dirty_buffer(nilfs->ns_sbh[0]); + } + + if (unlikely(err)) { + nilfs_err(sb, "unable to write superblock: err=%d", err); + if (err == -EIO && nilfs->ns_sbh[1]) { + /* + * sbp[0] points to newer log than sbp[1], + * so copy sbp[0] to sbp[1] to take over sbp[0]. + */ + memcpy(nilfs->ns_sbp[1], nilfs->ns_sbp[0], + nilfs->ns_sbsize); + nilfs_fall_back_super_block(nilfs); + goto retry; + } + } else { + struct nilfs_super_block *sbp = nilfs->ns_sbp[0]; + + nilfs->ns_sbwcount++; + + /* + * The latest segment becomes trailable from the position + * written in superblock. + */ + clear_nilfs_discontinued(nilfs); + + /* update GC protection for recent segments */ + if (nilfs->ns_sbh[1]) { + if (flag == NILFS_SB_COMMIT_ALL) { + set_buffer_dirty(nilfs->ns_sbh[1]); + if (sync_dirty_buffer(nilfs->ns_sbh[1]) < 0) + goto out; + } + if (le64_to_cpu(nilfs->ns_sbp[1]->s_last_cno) < + le64_to_cpu(nilfs->ns_sbp[0]->s_last_cno)) + sbp = nilfs->ns_sbp[1]; + } + + spin_lock(&nilfs->ns_last_segment_lock); + nilfs->ns_prot_seq = le64_to_cpu(sbp->s_last_seq); + spin_unlock(&nilfs->ns_last_segment_lock); + } + out: + return err; +} + +void nilfs_set_log_cursor(struct nilfs_super_block *sbp, + struct the_nilfs *nilfs) +{ + sector_t nfreeblocks; + + /* nilfs->ns_sem must be locked by the caller. */ + nilfs_count_free_blocks(nilfs, &nfreeblocks); + sbp->s_free_blocks_count = cpu_to_le64(nfreeblocks); + + spin_lock(&nilfs->ns_last_segment_lock); + sbp->s_last_seq = cpu_to_le64(nilfs->ns_last_seq); + sbp->s_last_pseg = cpu_to_le64(nilfs->ns_last_pseg); + sbp->s_last_cno = cpu_to_le64(nilfs->ns_last_cno); + spin_unlock(&nilfs->ns_last_segment_lock); +} + +struct nilfs_super_block **nilfs_prepare_super(struct super_block *sb, + int flip) +{ + struct the_nilfs *nilfs = sb->s_fs_info; + struct nilfs_super_block **sbp = nilfs->ns_sbp; + + /* nilfs->ns_sem must be locked by the caller. */ + if (sbp[0]->s_magic != cpu_to_le16(NILFS_SUPER_MAGIC)) { + if (sbp[1] && + sbp[1]->s_magic == cpu_to_le16(NILFS_SUPER_MAGIC)) { + memcpy(sbp[0], sbp[1], nilfs->ns_sbsize); + } else { + nilfs_crit(sb, "superblock broke"); + return NULL; + } + } else if (sbp[1] && + sbp[1]->s_magic != cpu_to_le16(NILFS_SUPER_MAGIC)) { + memcpy(sbp[1], sbp[0], nilfs->ns_sbsize); + } + + if (flip && sbp[1]) + nilfs_swap_super_block(nilfs); + + return sbp; +} + +int nilfs_commit_super(struct super_block *sb, int flag) +{ + struct the_nilfs *nilfs = sb->s_fs_info; + struct nilfs_super_block **sbp = nilfs->ns_sbp; + time64_t t; + + /* nilfs->ns_sem must be locked by the caller. */ + t = ktime_get_real_seconds(); + nilfs->ns_sbwtime = t; + sbp[0]->s_wtime = cpu_to_le64(t); + sbp[0]->s_sum = 0; + sbp[0]->s_sum = cpu_to_le32(crc32_le(nilfs->ns_crc_seed, + (unsigned char *)sbp[0], + nilfs->ns_sbsize)); + if (flag == NILFS_SB_COMMIT_ALL && sbp[1]) { + sbp[1]->s_wtime = sbp[0]->s_wtime; + sbp[1]->s_sum = 0; + sbp[1]->s_sum = cpu_to_le32(crc32_le(nilfs->ns_crc_seed, + (unsigned char *)sbp[1], + nilfs->ns_sbsize)); + } + clear_nilfs_sb_dirty(nilfs); + nilfs->ns_flushed_device = 1; + /* make sure store to ns_flushed_device cannot be reordered */ + smp_wmb(); + return nilfs_sync_super(sb, flag); +} + +/** + * nilfs_cleanup_super() - write filesystem state for cleanup + * @sb: super block instance to be unmounted or degraded to read-only + * + * This function restores state flags in the on-disk super block. + * This will set "clean" flag (i.e. NILFS_VALID_FS) unless the + * filesystem was not clean previously. + * + * Return: 0 on success, %-EIO if I/O error or superblock is corrupted. + */ +int nilfs_cleanup_super(struct super_block *sb) +{ + struct the_nilfs *nilfs = sb->s_fs_info; + struct nilfs_super_block **sbp; + int flag = NILFS_SB_COMMIT; + int ret = -EIO; + + sbp = nilfs_prepare_super(sb, 0); + if (sbp) { + sbp[0]->s_state = cpu_to_le16(nilfs->ns_mount_state); + nilfs_set_log_cursor(sbp[0], nilfs); + if (sbp[1] && sbp[0]->s_last_cno == sbp[1]->s_last_cno) { + /* + * make the "clean" flag also to the opposite + * super block if both super blocks point to + * the same checkpoint. + */ + sbp[1]->s_state = sbp[0]->s_state; + flag = NILFS_SB_COMMIT_ALL; + } + ret = nilfs_commit_super(sb, flag); + } + return ret; +} + +/** + * nilfs_move_2nd_super - relocate secondary super block + * @sb: super block instance + * @sb2off: new offset of the secondary super block (in bytes) + * + * Return: 0 on success, or a negative error code on failure. + */ +static int nilfs_move_2nd_super(struct super_block *sb, loff_t sb2off) +{ + struct the_nilfs *nilfs = sb->s_fs_info; + struct buffer_head *nsbh; + struct nilfs_super_block *nsbp; + sector_t blocknr, newblocknr; + unsigned long offset; + int sb2i; /* array index of the secondary superblock */ + int ret = 0; + + /* nilfs->ns_sem must be locked by the caller. */ + if (nilfs->ns_sbh[1] && + nilfs->ns_sbh[1]->b_blocknr > nilfs->ns_first_data_block) { + sb2i = 1; + blocknr = nilfs->ns_sbh[1]->b_blocknr; + } else if (nilfs->ns_sbh[0]->b_blocknr > nilfs->ns_first_data_block) { + sb2i = 0; + blocknr = nilfs->ns_sbh[0]->b_blocknr; + } else { + sb2i = -1; + blocknr = 0; + } + if (sb2i >= 0 && (u64)blocknr << nilfs->ns_blocksize_bits == sb2off) + goto out; /* super block location is unchanged */ + + /* Get new super block buffer */ + newblocknr = sb2off >> nilfs->ns_blocksize_bits; + offset = sb2off & (nilfs->ns_blocksize - 1); + nsbh = sb_getblk(sb, newblocknr); + if (!nsbh) { + nilfs_warn(sb, + "unable to move secondary superblock to block %llu", + (unsigned long long)newblocknr); + ret = -EIO; + goto out; + } + nsbp = (void *)nsbh->b_data + offset; + + lock_buffer(nsbh); + if (sb2i >= 0) { + /* + * The position of the second superblock only changes by 4KiB, + * which is larger than the maximum superblock data size + * (= 1KiB), so there is no need to use memmove() to allow + * overlap between source and destination. + */ + memcpy(nsbp, nilfs->ns_sbp[sb2i], nilfs->ns_sbsize); + + /* + * Zero fill after copy to avoid overwriting in case of move + * within the same block. + */ + memset(nsbh->b_data, 0, offset); + memset((void *)nsbp + nilfs->ns_sbsize, 0, + nsbh->b_size - offset - nilfs->ns_sbsize); + } else { + memset(nsbh->b_data, 0, nsbh->b_size); + } + set_buffer_uptodate(nsbh); + unlock_buffer(nsbh); + + if (sb2i >= 0) { + brelse(nilfs->ns_sbh[sb2i]); + nilfs->ns_sbh[sb2i] = nsbh; + nilfs->ns_sbp[sb2i] = nsbp; + } else if (nilfs->ns_sbh[0]->b_blocknr < nilfs->ns_first_data_block) { + /* secondary super block will be restored to index 1 */ + nilfs->ns_sbh[1] = nsbh; + nilfs->ns_sbp[1] = nsbp; + } else { + brelse(nsbh); + } +out: + return ret; +} + +/** + * nilfs_resize_fs - resize the filesystem + * @sb: super block instance + * @newsize: new size of the filesystem (in bytes) + * + * Return: 0 on success, or a negative error code on failure. + */ +int nilfs_resize_fs(struct super_block *sb, __u64 newsize) +{ + struct the_nilfs *nilfs = sb->s_fs_info; + struct nilfs_super_block **sbp; + __u64 devsize, newnsegs; + loff_t sb2off; + int ret; + + ret = -ERANGE; + devsize = bdev_nr_bytes(sb->s_bdev); + if (newsize > devsize) + goto out; + + /* + * Prevent underflow in second superblock position calculation. + * The exact minimum size check is done in nilfs_sufile_resize(). + */ + if (newsize < 4096) { + ret = -ENOSPC; + goto out; + } + + /* + * Write lock is required to protect some functions depending + * on the number of segments, the number of reserved segments, + * and so forth. + */ + down_write(&nilfs->ns_segctor_sem); + + sb2off = NILFS_SB2_OFFSET_BYTES(newsize); + newnsegs = sb2off >> nilfs->ns_blocksize_bits; + newnsegs = div64_ul(newnsegs, nilfs->ns_blocks_per_segment); + + ret = nilfs_sufile_resize(nilfs->ns_sufile, newnsegs); + up_write(&nilfs->ns_segctor_sem); + if (ret < 0) + goto out; + + ret = nilfs_construct_segment(sb); + if (ret < 0) + goto out; + + down_write(&nilfs->ns_sem); + nilfs_move_2nd_super(sb, sb2off); + ret = -EIO; + sbp = nilfs_prepare_super(sb, 0); + if (likely(sbp)) { + nilfs_set_log_cursor(sbp[0], nilfs); + /* + * Drop NILFS_RESIZE_FS flag for compatibility with + * mount-time resize which may be implemented in a + * future release. + */ + sbp[0]->s_state = cpu_to_le16(le16_to_cpu(sbp[0]->s_state) & + ~NILFS_RESIZE_FS); + sbp[0]->s_dev_size = cpu_to_le64(newsize); + sbp[0]->s_nsegments = cpu_to_le64(nilfs->ns_nsegments); + if (sbp[1]) + memcpy(sbp[1], sbp[0], nilfs->ns_sbsize); + ret = nilfs_commit_super(sb, NILFS_SB_COMMIT_ALL); + } + up_write(&nilfs->ns_sem); + + /* + * Reset the range of allocatable segments last. This order + * is important in the case of expansion because the secondary + * superblock must be protected from log write until migration + * completes. + */ + if (!ret) + nilfs_sufile_set_alloc_range(nilfs->ns_sufile, 0, newnsegs - 1); +out: + return ret; +} + +static void nilfs_put_super(struct super_block *sb) +{ + struct the_nilfs *nilfs = sb->s_fs_info; + + nilfs_detach_log_writer(sb); + + if (!sb_rdonly(sb)) { + down_write(&nilfs->ns_sem); + nilfs_cleanup_super(sb); + up_write(&nilfs->ns_sem); + } + + nilfs_sysfs_delete_device_group(nilfs); + iput(nilfs->ns_sufile); + iput(nilfs->ns_cpfile); + iput(nilfs->ns_dat); + + destroy_nilfs(nilfs); + sb->s_fs_info = NULL; +} + +static int nilfs_sync_fs(struct super_block *sb, int wait) +{ + struct the_nilfs *nilfs = sb->s_fs_info; + struct nilfs_super_block **sbp; + int err = 0; + + /* This function is called when super block should be written back */ + if (wait) + err = nilfs_construct_segment(sb); + + down_write(&nilfs->ns_sem); + if (nilfs_sb_dirty(nilfs)) { + sbp = nilfs_prepare_super(sb, nilfs_sb_will_flip(nilfs)); + if (likely(sbp)) { + nilfs_set_log_cursor(sbp[0], nilfs); + nilfs_commit_super(sb, NILFS_SB_COMMIT); + } + } + up_write(&nilfs->ns_sem); + + if (!err) + err = nilfs_flush_device(nilfs); + + return err; +} + +int nilfs_attach_checkpoint(struct super_block *sb, __u64 cno, int curr_mnt, + struct nilfs_root **rootp) +{ + struct the_nilfs *nilfs = sb->s_fs_info; + struct nilfs_root *root; + int err = -ENOMEM; + + root = nilfs_find_or_create_root( + nilfs, curr_mnt ? NILFS_CPTREE_CURRENT_CNO : cno); + if (!root) + return err; + + if (root->ifile) + goto reuse; /* already attached checkpoint */ + + down_read(&nilfs->ns_segctor_sem); + err = nilfs_ifile_read(sb, root, cno, nilfs->ns_inode_size); + up_read(&nilfs->ns_segctor_sem); + if (unlikely(err)) + goto failed; + + reuse: + *rootp = root; + return 0; + + failed: + if (err == -EINVAL) + nilfs_err(sb, "Invalid checkpoint (checkpoint number=%llu)", + (unsigned long long)cno); + nilfs_put_root(root); + + return err; +} + +static int nilfs_freeze(struct super_block *sb) +{ + struct the_nilfs *nilfs = sb->s_fs_info; + int err; + + if (sb_rdonly(sb)) + return 0; + + /* Mark super block clean */ + down_write(&nilfs->ns_sem); + err = nilfs_cleanup_super(sb); + up_write(&nilfs->ns_sem); + return err; +} + +static int nilfs_unfreeze(struct super_block *sb) +{ + struct the_nilfs *nilfs = sb->s_fs_info; + + if (sb_rdonly(sb)) + return 0; + + down_write(&nilfs->ns_sem); + nilfs_setup_super(sb, false); + up_write(&nilfs->ns_sem); + return 0; +} + +static int nilfs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + struct nilfs_root *root = NILFS_I(d_inode(dentry))->i_root; + struct the_nilfs *nilfs = root->nilfs; + u64 id = huge_encode_dev(sb->s_bdev->bd_dev); + unsigned long long blocks; + unsigned long overhead; + unsigned long nrsvblocks; + sector_t nfreeblocks; + u64 nmaxinodes, nfreeinodes; + int err; + + /* + * Compute all of the segment blocks + * + * The blocks before first segment and after last segment + * are excluded. + */ + blocks = nilfs->ns_blocks_per_segment * nilfs->ns_nsegments + - nilfs->ns_first_data_block; + nrsvblocks = nilfs->ns_nrsvsegs * nilfs->ns_blocks_per_segment; + + /* + * Compute the overhead + * + * When distributing meta data blocks outside segment structure, + * We must count them as the overhead. + */ + overhead = 0; + + err = nilfs_count_free_blocks(nilfs, &nfreeblocks); + if (unlikely(err)) + return err; + + err = nilfs_ifile_count_free_inodes(root->ifile, + &nmaxinodes, &nfreeinodes); + if (unlikely(err)) { + nilfs_warn(sb, "failed to count free inodes: err=%d", err); + if (err == -ERANGE) { + /* + * If nilfs_palloc_count_max_entries() returns + * -ERANGE error code then we simply treat + * curent inodes count as maximum possible and + * zero as free inodes value. + */ + nmaxinodes = atomic64_read(&root->inodes_count); + nfreeinodes = 0; + err = 0; + } else + return err; + } + + buf->f_type = NILFS_SUPER_MAGIC; + buf->f_bsize = sb->s_blocksize; + buf->f_blocks = blocks - overhead; + buf->f_bfree = nfreeblocks; + buf->f_bavail = (buf->f_bfree >= nrsvblocks) ? + (buf->f_bfree - nrsvblocks) : 0; + buf->f_files = nmaxinodes; + buf->f_ffree = nfreeinodes; + buf->f_namelen = NILFS_NAME_LEN; + buf->f_fsid = u64_to_fsid(id); + + return 0; +} + +static int nilfs_show_options(struct seq_file *seq, struct dentry *dentry) +{ + struct super_block *sb = dentry->d_sb; + struct the_nilfs *nilfs = sb->s_fs_info; + struct nilfs_root *root = NILFS_I(d_inode(dentry))->i_root; + + if (!nilfs_test_opt(nilfs, BARRIER)) + seq_puts(seq, ",nobarrier"); + if (root->cno != NILFS_CPTREE_CURRENT_CNO) + seq_printf(seq, ",cp=%llu", (unsigned long long)root->cno); + if (nilfs_test_opt(nilfs, ERRORS_PANIC)) + seq_puts(seq, ",errors=panic"); + if (nilfs_test_opt(nilfs, ERRORS_CONT)) + seq_puts(seq, ",errors=continue"); + if (nilfs_test_opt(nilfs, STRICT_ORDER)) + seq_puts(seq, ",order=strict"); + if (nilfs_test_opt(nilfs, NORECOVERY)) + seq_puts(seq, ",norecovery"); + if (nilfs_test_opt(nilfs, DISCARD)) + seq_puts(seq, ",discard"); + + return 0; +} + +static const struct super_operations nilfs_sops = { + .alloc_inode = nilfs_alloc_inode, + .free_inode = nilfs_free_inode, + .dirty_inode = nilfs_dirty_inode, + .evict_inode = nilfs_evict_inode, + .put_super = nilfs_put_super, + .sync_fs = nilfs_sync_fs, + .freeze_fs = nilfs_freeze, + .unfreeze_fs = nilfs_unfreeze, + .statfs = nilfs_statfs, + .show_options = nilfs_show_options +}; + +enum { + Opt_err, Opt_barrier, Opt_snapshot, Opt_order, Opt_norecovery, + Opt_discard, +}; + +static const struct constant_table nilfs_param_err[] = { + {"continue", NILFS_MOUNT_ERRORS_CONT}, + {"panic", NILFS_MOUNT_ERRORS_PANIC}, + {"remount-ro", NILFS_MOUNT_ERRORS_RO}, + {} +}; + +static const struct fs_parameter_spec nilfs_param_spec[] = { + fsparam_enum ("errors", Opt_err, nilfs_param_err), + fsparam_flag_no ("barrier", Opt_barrier), + fsparam_u64 ("cp", Opt_snapshot), + fsparam_string ("order", Opt_order), + fsparam_flag ("norecovery", Opt_norecovery), + fsparam_flag_no ("discard", Opt_discard), + {} +}; + +struct nilfs_fs_context { + unsigned long ns_mount_opt; + __u64 cno; +}; + +static int nilfs_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct nilfs_fs_context *nilfs = fc->fs_private; + int is_remount = fc->purpose == FS_CONTEXT_FOR_RECONFIGURE; + struct fs_parse_result result; + int opt; + + opt = fs_parse(fc, nilfs_param_spec, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_barrier: + if (result.negated) + nilfs_clear_opt(nilfs, BARRIER); + else + nilfs_set_opt(nilfs, BARRIER); + break; + case Opt_order: + if (strcmp(param->string, "relaxed") == 0) + /* Ordered data semantics */ + nilfs_clear_opt(nilfs, STRICT_ORDER); + else if (strcmp(param->string, "strict") == 0) + /* Strict in-order semantics */ + nilfs_set_opt(nilfs, STRICT_ORDER); + else + return -EINVAL; + break; + case Opt_err: + nilfs->ns_mount_opt &= ~NILFS_MOUNT_ERROR_MODE; + nilfs->ns_mount_opt |= result.uint_32; + break; + case Opt_snapshot: + if (is_remount) { + struct super_block *sb = fc->root->d_sb; + + nilfs_err(sb, + "\"%s\" option is invalid for remount", + param->key); + return -EINVAL; + } + if (result.uint_64 == 0) { + nilfs_err(NULL, + "invalid option \"cp=0\": invalid checkpoint number 0"); + return -EINVAL; + } + nilfs->cno = result.uint_64; + break; + case Opt_norecovery: + nilfs_set_opt(nilfs, NORECOVERY); + break; + case Opt_discard: + if (result.negated) + nilfs_clear_opt(nilfs, DISCARD); + else + nilfs_set_opt(nilfs, DISCARD); + break; + default: + return -EINVAL; + } + + return 0; +} + +static int nilfs_setup_super(struct super_block *sb, int is_mount) +{ + struct the_nilfs *nilfs = sb->s_fs_info; + struct nilfs_super_block **sbp; + int max_mnt_count; + int mnt_count; + + /* nilfs->ns_sem must be locked by the caller. */ + sbp = nilfs_prepare_super(sb, 0); + if (!sbp) + return -EIO; + + if (!is_mount) + goto skip_mount_setup; + + max_mnt_count = le16_to_cpu(sbp[0]->s_max_mnt_count); + mnt_count = le16_to_cpu(sbp[0]->s_mnt_count); + + if (nilfs->ns_mount_state & NILFS_ERROR_FS) { + nilfs_warn(sb, "mounting fs with errors"); +#if 0 + } else if (max_mnt_count >= 0 && mnt_count >= max_mnt_count) { + nilfs_warn(sb, "maximal mount count reached"); +#endif + } + if (!max_mnt_count) + sbp[0]->s_max_mnt_count = cpu_to_le16(NILFS_DFL_MAX_MNT_COUNT); + + sbp[0]->s_mnt_count = cpu_to_le16(mnt_count + 1); + sbp[0]->s_mtime = cpu_to_le64(ktime_get_real_seconds()); + +skip_mount_setup: + sbp[0]->s_state = + cpu_to_le16(le16_to_cpu(sbp[0]->s_state) & ~NILFS_VALID_FS); + /* synchronize sbp[1] with sbp[0] */ + if (sbp[1]) + memcpy(sbp[1], sbp[0], nilfs->ns_sbsize); + return nilfs_commit_super(sb, NILFS_SB_COMMIT_ALL); +} + +struct nilfs_super_block *nilfs_read_super_block(struct super_block *sb, + u64 pos, int blocksize, + struct buffer_head **pbh) +{ + unsigned long long sb_index = pos; + unsigned long offset; + + offset = do_div(sb_index, blocksize); + *pbh = sb_bread(sb, sb_index); + if (!*pbh) + return NULL; + return (struct nilfs_super_block *)((char *)(*pbh)->b_data + offset); +} + +int nilfs_store_magic(struct super_block *sb, + struct nilfs_super_block *sbp) +{ + struct the_nilfs *nilfs = sb->s_fs_info; + + sb->s_magic = le16_to_cpu(sbp->s_magic); + + /* FS independent flags */ +#ifdef NILFS_ATIME_DISABLE + sb->s_flags |= SB_NOATIME; +#endif + + nilfs->ns_resuid = le16_to_cpu(sbp->s_def_resuid); + nilfs->ns_resgid = le16_to_cpu(sbp->s_def_resgid); + nilfs->ns_interval = le32_to_cpu(sbp->s_c_interval); + nilfs->ns_watermark = le32_to_cpu(sbp->s_c_block_max); + + return 0; +} + +int nilfs_check_feature_compatibility(struct super_block *sb, + struct nilfs_super_block *sbp) +{ + __u64 features; + + features = le64_to_cpu(sbp->s_feature_incompat) & + ~NILFS_FEATURE_INCOMPAT_SUPP; + if (features) { + nilfs_err(sb, + "couldn't mount because of unsupported optional features (%llx)", + (unsigned long long)features); + return -EINVAL; + } + features = le64_to_cpu(sbp->s_feature_compat_ro) & + ~NILFS_FEATURE_COMPAT_RO_SUPP; + if (!sb_rdonly(sb) && features) { + nilfs_err(sb, + "couldn't mount RDWR because of unsupported optional features (%llx)", + (unsigned long long)features); + return -EINVAL; + } + return 0; +} + +static int nilfs_get_root_dentry(struct super_block *sb, + struct nilfs_root *root, + struct dentry **root_dentry) +{ + struct inode *inode; + struct dentry *dentry; + int ret = 0; + + inode = nilfs_iget(sb, root, NILFS_ROOT_INO); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + nilfs_err(sb, "error %d getting root inode", ret); + goto out; + } + if (!S_ISDIR(inode->i_mode) || !inode->i_blocks || !inode->i_size) { + iput(inode); + nilfs_err(sb, "corrupt root inode"); + ret = -EINVAL; + goto out; + } + + if (root->cno == NILFS_CPTREE_CURRENT_CNO) { + dentry = d_find_alias(inode); + if (!dentry) { + dentry = d_make_root(inode); + if (!dentry) { + ret = -ENOMEM; + goto failed_dentry; + } + } else { + iput(inode); + } + } else { + dentry = d_obtain_root(inode); + if (IS_ERR(dentry)) { + ret = PTR_ERR(dentry); + goto failed_dentry; + } + } + *root_dentry = dentry; + out: + return ret; + + failed_dentry: + nilfs_err(sb, "error %d getting root dentry", ret); + goto out; +} + +static int nilfs_attach_snapshot(struct super_block *s, __u64 cno, + struct dentry **root_dentry) +{ + struct the_nilfs *nilfs = s->s_fs_info; + struct nilfs_root *root; + int ret; + + mutex_lock(&nilfs->ns_snapshot_mount_mutex); + + down_read(&nilfs->ns_segctor_sem); + ret = nilfs_cpfile_is_snapshot(nilfs->ns_cpfile, cno); + up_read(&nilfs->ns_segctor_sem); + if (ret < 0) { + ret = (ret == -ENOENT) ? -EINVAL : ret; + goto out; + } else if (!ret) { + nilfs_err(s, + "The specified checkpoint is not a snapshot (checkpoint number=%llu)", + (unsigned long long)cno); + ret = -EINVAL; + goto out; + } + + ret = nilfs_attach_checkpoint(s, cno, false, &root); + if (ret) { + nilfs_err(s, + "error %d while loading snapshot (checkpoint number=%llu)", + ret, (unsigned long long)cno); + goto out; + } + ret = nilfs_get_root_dentry(s, root, root_dentry); + nilfs_put_root(root); + out: + mutex_unlock(&nilfs->ns_snapshot_mount_mutex); + return ret; +} + +/** + * nilfs_tree_is_busy() - try to shrink dentries of a checkpoint + * @root_dentry: root dentry of the tree to be shrunk + * + * Return: true if the tree was in-use, false otherwise. + */ +static bool nilfs_tree_is_busy(struct dentry *root_dentry) +{ + shrink_dcache_parent(root_dentry); + return d_count(root_dentry) > 1; +} + +int nilfs_checkpoint_is_mounted(struct super_block *sb, __u64 cno) +{ + struct the_nilfs *nilfs = sb->s_fs_info; + struct nilfs_root *root; + struct inode *inode; + struct dentry *dentry; + int ret; + + if (cno > nilfs->ns_cno) + return false; + + if (cno >= nilfs_last_cno(nilfs)) + return true; /* protect recent checkpoints */ + + ret = false; + root = nilfs_lookup_root(nilfs, cno); + if (root) { + inode = nilfs_ilookup(sb, root, NILFS_ROOT_INO); + if (inode) { + dentry = d_find_alias(inode); + if (dentry) { + ret = nilfs_tree_is_busy(dentry); + dput(dentry); + } + iput(inode); + } + nilfs_put_root(root); + } + return ret; +} + +/** + * nilfs_fill_super() - initialize a super block instance + * @sb: super_block + * @fc: filesystem context + * + * This function is called exclusively by nilfs->ns_mount_mutex. + * So, the recovery process is protected from other simultaneous mounts. + * + * Return: 0 on success, or a negative error code on failure. + */ +static int +nilfs_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct the_nilfs *nilfs; + struct nilfs_root *fsroot; + struct nilfs_fs_context *ctx = fc->fs_private; + __u64 cno; + int err; + + nilfs = alloc_nilfs(sb); + if (!nilfs) + return -ENOMEM; + + sb->s_fs_info = nilfs; + + err = init_nilfs(nilfs, sb); + if (err) + goto failed_nilfs; + + /* Copy in parsed mount options */ + nilfs->ns_mount_opt = ctx->ns_mount_opt; + + sb->s_op = &nilfs_sops; + sb->s_export_op = &nilfs_export_ops; + sb->s_root = NULL; + sb->s_time_gran = 1; + sb->s_max_links = NILFS_LINK_MAX; + + err = load_nilfs(nilfs, sb); + if (err) + goto failed_nilfs; + + super_set_uuid(sb, nilfs->ns_sbp[0]->s_uuid, + sizeof(nilfs->ns_sbp[0]->s_uuid)); + super_set_sysfs_name_bdev(sb); + + cno = nilfs_last_cno(nilfs); + err = nilfs_attach_checkpoint(sb, cno, true, &fsroot); + if (err) { + nilfs_err(sb, + "error %d while loading last checkpoint (checkpoint number=%llu)", + err, (unsigned long long)cno); + goto failed_unload; + } + + if (!sb_rdonly(sb)) { + err = nilfs_attach_log_writer(sb, fsroot); + if (err) + goto failed_checkpoint; + } + + err = nilfs_get_root_dentry(sb, fsroot, &sb->s_root); + if (err) + goto failed_segctor; + + nilfs_put_root(fsroot); + + if (!sb_rdonly(sb)) { + down_write(&nilfs->ns_sem); + nilfs_setup_super(sb, true); + up_write(&nilfs->ns_sem); + } + + return 0; + + failed_segctor: + nilfs_detach_log_writer(sb); + + failed_checkpoint: + nilfs_put_root(fsroot); + + failed_unload: + nilfs_sysfs_delete_device_group(nilfs); + iput(nilfs->ns_sufile); + iput(nilfs->ns_cpfile); + iput(nilfs->ns_dat); + + failed_nilfs: + destroy_nilfs(nilfs); + return err; +} + +static int nilfs_reconfigure(struct fs_context *fc) +{ + struct nilfs_fs_context *ctx = fc->fs_private; + struct super_block *sb = fc->root->d_sb; + struct the_nilfs *nilfs = sb->s_fs_info; + int err; + + sync_filesystem(sb); + + err = -EINVAL; + + if (!nilfs_valid_fs(nilfs)) { + nilfs_warn(sb, + "couldn't remount because the filesystem is in an incomplete recovery state"); + goto ignore_opts; + } + if ((bool)(fc->sb_flags & SB_RDONLY) == sb_rdonly(sb)) + goto out; + if (fc->sb_flags & SB_RDONLY) { + sb->s_flags |= SB_RDONLY; + + /* + * Remounting a valid RW partition RDONLY, so set + * the RDONLY flag and then mark the partition as valid again. + */ + down_write(&nilfs->ns_sem); + nilfs_cleanup_super(sb); + up_write(&nilfs->ns_sem); + } else { + __u64 features; + struct nilfs_root *root; + + /* + * Mounting a RDONLY partition read-write, so reread and + * store the current valid flag. (It may have been changed + * by fsck since we originally mounted the partition.) + */ + down_read(&nilfs->ns_sem); + features = le64_to_cpu(nilfs->ns_sbp[0]->s_feature_compat_ro) & + ~NILFS_FEATURE_COMPAT_RO_SUPP; + up_read(&nilfs->ns_sem); + if (features) { + nilfs_warn(sb, + "couldn't remount RDWR because of unsupported optional features (%llx)", + (unsigned long long)features); + err = -EROFS; + goto ignore_opts; + } + + sb->s_flags &= ~SB_RDONLY; + + root = NILFS_I(d_inode(sb->s_root))->i_root; + err = nilfs_attach_log_writer(sb, root); + if (err) { + sb->s_flags |= SB_RDONLY; + goto ignore_opts; + } + + down_write(&nilfs->ns_sem); + nilfs_setup_super(sb, true); + up_write(&nilfs->ns_sem); + } + out: + sb->s_flags = (sb->s_flags & ~SB_POSIXACL); + /* Copy over parsed remount options */ + nilfs->ns_mount_opt = ctx->ns_mount_opt; + + return 0; + + ignore_opts: + return err; +} + +static int +nilfs_get_tree(struct fs_context *fc) +{ + struct nilfs_fs_context *ctx = fc->fs_private; + struct super_block *s; + dev_t dev; + int err; + + if (ctx->cno && !(fc->sb_flags & SB_RDONLY)) { + nilfs_err(NULL, + "invalid option \"cp=%llu\": read-only option is not specified", + ctx->cno); + return -EINVAL; + } + + err = lookup_bdev(fc->source, &dev); + if (err) + return err; + + s = sget_dev(fc, dev); + if (IS_ERR(s)) + return PTR_ERR(s); + + if (!s->s_root) { + err = setup_bdev_super(s, fc->sb_flags, fc); + if (!err) + err = nilfs_fill_super(s, fc); + if (err) + goto failed_super; + + s->s_flags |= SB_ACTIVE; + } else if (!ctx->cno) { + if (nilfs_tree_is_busy(s->s_root)) { + if ((fc->sb_flags ^ s->s_flags) & SB_RDONLY) { + nilfs_err(s, + "the device already has a %s mount.", + sb_rdonly(s) ? "read-only" : "read/write"); + err = -EBUSY; + goto failed_super; + } + } else { + /* + * Try reconfigure to setup mount states if the current + * tree is not mounted and only snapshots use this sb. + * + * Since nilfs_reconfigure() requires fc->root to be + * set, set it first and release it on failure. + */ + fc->root = dget(s->s_root); + err = nilfs_reconfigure(fc); + if (err) { + dput(fc->root); + fc->root = NULL; /* prevent double release */ + goto failed_super; + } + return 0; + } + } + + if (ctx->cno) { + struct dentry *root_dentry; + + err = nilfs_attach_snapshot(s, ctx->cno, &root_dentry); + if (err) + goto failed_super; + fc->root = root_dentry; + return 0; + } + + fc->root = dget(s->s_root); + return 0; + + failed_super: + deactivate_locked_super(s); + return err; +} + +static void nilfs_free_fc(struct fs_context *fc) +{ + kfree(fc->fs_private); +} + +static const struct fs_context_operations nilfs_context_ops = { + .parse_param = nilfs_parse_param, + .get_tree = nilfs_get_tree, + .reconfigure = nilfs_reconfigure, + .free = nilfs_free_fc, +}; + +static int nilfs_init_fs_context(struct fs_context *fc) +{ + struct nilfs_fs_context *ctx; + + ctx = kzalloc_obj(*ctx); + if (!ctx) + return -ENOMEM; + + ctx->ns_mount_opt = NILFS_MOUNT_ERRORS_RO | NILFS_MOUNT_BARRIER; + fc->fs_private = ctx; + fc->ops = &nilfs_context_ops; + + return 0; +} + +struct file_system_type nilfs_fs_type = { + .owner = THIS_MODULE, + .name = "nilfs2", + .kill_sb = kill_block_super, + .fs_flags = FS_REQUIRES_DEV, + .init_fs_context = nilfs_init_fs_context, + .parameters = nilfs_param_spec, +}; +MODULE_ALIAS_FS("nilfs2"); + +static void nilfs_inode_init_once(void *obj) +{ + struct nilfs_inode_info *ii = obj; + + INIT_LIST_HEAD(&ii->i_dirty); +#ifdef CONFIG_NILFS_XATTR + init_rwsem(&ii->xattr_sem); +#endif + inode_init_once(&ii->vfs_inode); +} + +static void nilfs_segbuf_init_once(void *obj) +{ + memset(obj, 0, sizeof(struct nilfs_segment_buffer)); +} + +static void nilfs_destroy_cachep(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + + kmem_cache_destroy(nilfs_inode_cachep); + kmem_cache_destroy(nilfs_transaction_cachep); + kmem_cache_destroy(nilfs_segbuf_cachep); + kmem_cache_destroy(nilfs_btree_path_cache); +} + +static int __init nilfs_init_cachep(void) +{ + nilfs_inode_cachep = kmem_cache_create("nilfs2_inode_cache", + sizeof(struct nilfs_inode_info), 0, + SLAB_RECLAIM_ACCOUNT|SLAB_ACCOUNT, + nilfs_inode_init_once); + if (!nilfs_inode_cachep) + goto fail; + + nilfs_transaction_cachep = kmem_cache_create("nilfs2_transaction_cache", + sizeof(struct nilfs_transaction_info), 0, + SLAB_RECLAIM_ACCOUNT, NULL); + if (!nilfs_transaction_cachep) + goto fail; + + nilfs_segbuf_cachep = kmem_cache_create("nilfs2_segbuf_cache", + sizeof(struct nilfs_segment_buffer), 0, + SLAB_RECLAIM_ACCOUNT, nilfs_segbuf_init_once); + if (!nilfs_segbuf_cachep) + goto fail; + + nilfs_btree_path_cache = kmem_cache_create("nilfs2_btree_path_cache", + sizeof(struct nilfs_btree_path) * NILFS_BTREE_LEVEL_MAX, + 0, 0, NULL); + if (!nilfs_btree_path_cache) + goto fail; + + return 0; + +fail: + nilfs_destroy_cachep(); + return -ENOMEM; +} + +static int __init init_nilfs_fs(void) +{ + int err; + + err = nilfs_init_cachep(); + if (err) + goto fail; + + err = nilfs_sysfs_init(); + if (err) + goto free_cachep; + + err = register_filesystem(&nilfs_fs_type); + if (err) + goto deinit_sysfs_entry; + + printk(KERN_INFO "NILFS version 2 loaded\n"); + return 0; + +deinit_sysfs_entry: + nilfs_sysfs_exit(); +free_cachep: + nilfs_destroy_cachep(); +fail: + return err; +} + +static void __exit exit_nilfs_fs(void) +{ + nilfs_destroy_cachep(); + nilfs_sysfs_exit(); + unregister_filesystem(&nilfs_fs_type); +} + +module_init(init_nilfs_fs) +module_exit(exit_nilfs_fs) diff --git a/fs/nilfs2/sysfs.c b/fs/nilfs2/sysfs.c new file mode 100644 index 000000000..bc52afbfc --- /dev/null +++ b/fs/nilfs2/sysfs.c @@ -0,0 +1,1140 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Sysfs support implementation. + * + * Copyright (C) 2005-2014 Nippon Telegraph and Telephone Corporation. + * Copyright (C) 2014 HGST, Inc., a Western Digital Company. + * + * Written by Vyacheslav Dubeyko + */ + +#include + +#include "nilfs.h" +#include "mdt.h" +#include "sufile.h" +#include "cpfile.h" +#include "sysfs.h" + +/* /sys/fs// */ +static struct kset *nilfs_kset; + +#define NILFS_DEV_INT_GROUP_OPS(name, parent_name) \ +static ssize_t nilfs_##name##_attr_show(struct kobject *kobj, \ + struct attribute *attr, char *buf) \ +{ \ + struct the_nilfs *nilfs = container_of(kobj->parent, \ + struct the_nilfs, \ + ns_##parent_name##_kobj); \ + struct nilfs_##name##_attr *a = container_of(attr, \ + struct nilfs_##name##_attr, \ + attr); \ + return a->show ? a->show(a, nilfs, buf) : 0; \ +} \ +static ssize_t nilfs_##name##_attr_store(struct kobject *kobj, \ + struct attribute *attr, \ + const char *buf, size_t len) \ +{ \ + struct the_nilfs *nilfs = container_of(kobj->parent, \ + struct the_nilfs, \ + ns_##parent_name##_kobj); \ + struct nilfs_##name##_attr *a = container_of(attr, \ + struct nilfs_##name##_attr, \ + attr); \ + return a->store ? a->store(a, nilfs, buf, len) : 0; \ +} \ +static const struct sysfs_ops nilfs_##name##_attr_ops = { \ + .show = nilfs_##name##_attr_show, \ + .store = nilfs_##name##_attr_store, \ +} + +#define NILFS_DEV_INT_GROUP_TYPE(name, parent_name) \ +static void nilfs_##name##_attr_release(struct kobject *kobj) \ +{ \ + struct nilfs_sysfs_##parent_name##_subgroups *subgroups = container_of(kobj, \ + struct nilfs_sysfs_##parent_name##_subgroups, \ + sg_##name##_kobj); \ + complete(&subgroups->sg_##name##_kobj_unregister); \ +} \ +static const struct kobj_type nilfs_##name##_ktype = { \ + .default_groups = nilfs_##name##_groups, \ + .sysfs_ops = &nilfs_##name##_attr_ops, \ + .release = nilfs_##name##_attr_release, \ +} + +#define NILFS_DEV_INT_GROUP_FNS(name, parent_name) \ +static int nilfs_sysfs_create_##name##_group(struct the_nilfs *nilfs) \ +{ \ + struct kobject *parent; \ + struct kobject *kobj; \ + struct completion *kobj_unregister; \ + struct nilfs_sysfs_##parent_name##_subgroups *subgroups; \ + int err; \ + subgroups = nilfs->ns_##parent_name##_subgroups; \ + kobj = &subgroups->sg_##name##_kobj; \ + kobj_unregister = &subgroups->sg_##name##_kobj_unregister; \ + parent = &nilfs->ns_##parent_name##_kobj; \ + kobj->kset = nilfs_kset; \ + init_completion(kobj_unregister); \ + err = kobject_init_and_add(kobj, &nilfs_##name##_ktype, parent, \ + #name); \ + if (err) \ + kobject_put(kobj); \ + return err; \ +} \ +static void nilfs_sysfs_delete_##name##_group(struct the_nilfs *nilfs) \ +{ \ + kobject_put(&nilfs->ns_##parent_name##_subgroups->sg_##name##_kobj); \ +} + +/************************************************************************ + * NILFS snapshot attrs * + ************************************************************************/ + +static ssize_t +nilfs_snapshot_inodes_count_show(struct nilfs_snapshot_attr *attr, + struct nilfs_root *root, char *buf) +{ + return sysfs_emit(buf, "%llu\n", + (unsigned long long)atomic64_read(&root->inodes_count)); +} + +static ssize_t +nilfs_snapshot_blocks_count_show(struct nilfs_snapshot_attr *attr, + struct nilfs_root *root, char *buf) +{ + return sysfs_emit(buf, "%llu\n", + (unsigned long long)atomic64_read(&root->blocks_count)); +} + +static const char snapshot_readme_str[] = + "The group contains details about mounted snapshot.\n\n" + "(1) inodes_count\n\tshow number of inodes for snapshot.\n\n" + "(2) blocks_count\n\tshow number of blocks for snapshot.\n\n"; + +static ssize_t +nilfs_snapshot_README_show(struct nilfs_snapshot_attr *attr, + struct nilfs_root *root, char *buf) +{ + return sysfs_emit(buf, snapshot_readme_str); +} + +NILFS_SNAPSHOT_RO_ATTR(inodes_count); +NILFS_SNAPSHOT_RO_ATTR(blocks_count); +NILFS_SNAPSHOT_RO_ATTR(README); + +static struct attribute *nilfs_snapshot_attrs[] = { + NILFS_SNAPSHOT_ATTR_LIST(inodes_count), + NILFS_SNAPSHOT_ATTR_LIST(blocks_count), + NILFS_SNAPSHOT_ATTR_LIST(README), + NULL, +}; +ATTRIBUTE_GROUPS(nilfs_snapshot); + +static ssize_t nilfs_snapshot_attr_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct nilfs_root *root = + container_of(kobj, struct nilfs_root, snapshot_kobj); + struct nilfs_snapshot_attr *a = + container_of(attr, struct nilfs_snapshot_attr, attr); + + return a->show ? a->show(a, root, buf) : 0; +} + +static ssize_t nilfs_snapshot_attr_store(struct kobject *kobj, + struct attribute *attr, + const char *buf, size_t len) +{ + struct nilfs_root *root = + container_of(kobj, struct nilfs_root, snapshot_kobj); + struct nilfs_snapshot_attr *a = + container_of(attr, struct nilfs_snapshot_attr, attr); + + return a->store ? a->store(a, root, buf, len) : 0; +} + +static void nilfs_snapshot_attr_release(struct kobject *kobj) +{ + struct nilfs_root *root = container_of(kobj, struct nilfs_root, + snapshot_kobj); + complete(&root->snapshot_kobj_unregister); +} + +static const struct sysfs_ops nilfs_snapshot_attr_ops = { + .show = nilfs_snapshot_attr_show, + .store = nilfs_snapshot_attr_store, +}; + +static const struct kobj_type nilfs_snapshot_ktype = { + .default_groups = nilfs_snapshot_groups, + .sysfs_ops = &nilfs_snapshot_attr_ops, + .release = nilfs_snapshot_attr_release, +}; + +int nilfs_sysfs_create_snapshot_group(struct nilfs_root *root) +{ + struct the_nilfs *nilfs; + struct kobject *parent; + int err; + + nilfs = root->nilfs; + parent = &nilfs->ns_dev_subgroups->sg_mounted_snapshots_kobj; + root->snapshot_kobj.kset = nilfs_kset; + init_completion(&root->snapshot_kobj_unregister); + + if (root->cno == NILFS_CPTREE_CURRENT_CNO) { + err = kobject_init_and_add(&root->snapshot_kobj, + &nilfs_snapshot_ktype, + &nilfs->ns_dev_kobj, + "current_checkpoint"); + } else { + err = kobject_init_and_add(&root->snapshot_kobj, + &nilfs_snapshot_ktype, + parent, + "%llu", root->cno); + } + + if (err) + kobject_put(&root->snapshot_kobj); + + return err; +} + +void nilfs_sysfs_delete_snapshot_group(struct nilfs_root *root) +{ + kobject_put(&root->snapshot_kobj); +} + +/************************************************************************ + * NILFS mounted snapshots attrs * + ************************************************************************/ + +static const char mounted_snapshots_readme_str[] = + "The mounted_snapshots group contains group for\n" + "every mounted snapshot.\n"; + +static ssize_t +nilfs_mounted_snapshots_README_show(struct nilfs_mounted_snapshots_attr *attr, + struct the_nilfs *nilfs, char *buf) +{ + return sysfs_emit(buf, mounted_snapshots_readme_str); +} + +NILFS_MOUNTED_SNAPSHOTS_RO_ATTR(README); + +static struct attribute *nilfs_mounted_snapshots_attrs[] = { + NILFS_MOUNTED_SNAPSHOTS_ATTR_LIST(README), + NULL, +}; +ATTRIBUTE_GROUPS(nilfs_mounted_snapshots); + +NILFS_DEV_INT_GROUP_OPS(mounted_snapshots, dev); +NILFS_DEV_INT_GROUP_TYPE(mounted_snapshots, dev); +NILFS_DEV_INT_GROUP_FNS(mounted_snapshots, dev); + +/************************************************************************ + * NILFS checkpoints attrs * + ************************************************************************/ + +static ssize_t +nilfs_checkpoints_checkpoints_number_show(struct nilfs_checkpoints_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + __u64 ncheckpoints; + struct nilfs_cpstat cpstat; + int err; + + down_read(&nilfs->ns_segctor_sem); + err = nilfs_cpfile_get_stat(nilfs->ns_cpfile, &cpstat); + up_read(&nilfs->ns_segctor_sem); + if (err < 0) { + nilfs_err(nilfs->ns_sb, "unable to get checkpoint stat: err=%d", + err); + return err; + } + + ncheckpoints = cpstat.cs_ncps; + + return sysfs_emit(buf, "%llu\n", ncheckpoints); +} + +static ssize_t +nilfs_checkpoints_snapshots_number_show(struct nilfs_checkpoints_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + __u64 nsnapshots; + struct nilfs_cpstat cpstat; + int err; + + down_read(&nilfs->ns_segctor_sem); + err = nilfs_cpfile_get_stat(nilfs->ns_cpfile, &cpstat); + up_read(&nilfs->ns_segctor_sem); + if (err < 0) { + nilfs_err(nilfs->ns_sb, "unable to get checkpoint stat: err=%d", + err); + return err; + } + + nsnapshots = cpstat.cs_nsss; + + return sysfs_emit(buf, "%llu\n", nsnapshots); +} + +static ssize_t +nilfs_checkpoints_last_seg_checkpoint_show(struct nilfs_checkpoints_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + __u64 last_cno; + + spin_lock(&nilfs->ns_last_segment_lock); + last_cno = nilfs->ns_last_cno; + spin_unlock(&nilfs->ns_last_segment_lock); + + return sysfs_emit(buf, "%llu\n", last_cno); +} + +static ssize_t +nilfs_checkpoints_next_checkpoint_show(struct nilfs_checkpoints_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + __u64 cno; + + down_read(&nilfs->ns_segctor_sem); + cno = nilfs->ns_cno; + up_read(&nilfs->ns_segctor_sem); + + return sysfs_emit(buf, "%llu\n", cno); +} + +static const char checkpoints_readme_str[] = + "The checkpoints group contains attributes that describe\n" + "details about volume's checkpoints.\n\n" + "(1) checkpoints_number\n\tshow number of checkpoints on volume.\n\n" + "(2) snapshots_number\n\tshow number of snapshots on volume.\n\n" + "(3) last_seg_checkpoint\n" + "\tshow checkpoint number of the latest segment.\n\n" + "(4) next_checkpoint\n\tshow next checkpoint number.\n\n"; + +static ssize_t +nilfs_checkpoints_README_show(struct nilfs_checkpoints_attr *attr, + struct the_nilfs *nilfs, char *buf) +{ + return sysfs_emit(buf, checkpoints_readme_str); +} + +NILFS_CHECKPOINTS_RO_ATTR(checkpoints_number); +NILFS_CHECKPOINTS_RO_ATTR(snapshots_number); +NILFS_CHECKPOINTS_RO_ATTR(last_seg_checkpoint); +NILFS_CHECKPOINTS_RO_ATTR(next_checkpoint); +NILFS_CHECKPOINTS_RO_ATTR(README); + +static struct attribute *nilfs_checkpoints_attrs[] = { + NILFS_CHECKPOINTS_ATTR_LIST(checkpoints_number), + NILFS_CHECKPOINTS_ATTR_LIST(snapshots_number), + NILFS_CHECKPOINTS_ATTR_LIST(last_seg_checkpoint), + NILFS_CHECKPOINTS_ATTR_LIST(next_checkpoint), + NILFS_CHECKPOINTS_ATTR_LIST(README), + NULL, +}; +ATTRIBUTE_GROUPS(nilfs_checkpoints); + +NILFS_DEV_INT_GROUP_OPS(checkpoints, dev); +NILFS_DEV_INT_GROUP_TYPE(checkpoints, dev); +NILFS_DEV_INT_GROUP_FNS(checkpoints, dev); + +/************************************************************************ + * NILFS segments attrs * + ************************************************************************/ + +static ssize_t +nilfs_segments_segments_number_show(struct nilfs_segments_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + return sysfs_emit(buf, "%lu\n", nilfs->ns_nsegments); +} + +static ssize_t +nilfs_segments_blocks_per_segment_show(struct nilfs_segments_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + return sysfs_emit(buf, "%lu\n", nilfs->ns_blocks_per_segment); +} + +static ssize_t +nilfs_segments_clean_segments_show(struct nilfs_segments_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + unsigned long ncleansegs; + + down_read(&NILFS_MDT(nilfs->ns_dat)->mi_sem); + ncleansegs = nilfs_sufile_get_ncleansegs(nilfs->ns_sufile); + up_read(&NILFS_MDT(nilfs->ns_dat)->mi_sem); + + return sysfs_emit(buf, "%lu\n", ncleansegs); +} + +static ssize_t +nilfs_segments_dirty_segments_show(struct nilfs_segments_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + struct nilfs_sustat sustat; + int err; + + down_read(&nilfs->ns_segctor_sem); + err = nilfs_sufile_get_stat(nilfs->ns_sufile, &sustat); + up_read(&nilfs->ns_segctor_sem); + if (err < 0) { + nilfs_err(nilfs->ns_sb, "unable to get segment stat: err=%d", + err); + return err; + } + + return sysfs_emit(buf, "%llu\n", sustat.ss_ndirtysegs); +} + +static const char segments_readme_str[] = + "The segments group contains attributes that describe\n" + "details about volume's segments.\n\n" + "(1) segments_number\n\tshow number of segments on volume.\n\n" + "(2) blocks_per_segment\n\tshow number of blocks in segment.\n\n" + "(3) clean_segments\n\tshow count of clean segments.\n\n" + "(4) dirty_segments\n\tshow count of dirty segments.\n\n"; + +static ssize_t +nilfs_segments_README_show(struct nilfs_segments_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + return sysfs_emit(buf, segments_readme_str); +} + +NILFS_SEGMENTS_RO_ATTR(segments_number); +NILFS_SEGMENTS_RO_ATTR(blocks_per_segment); +NILFS_SEGMENTS_RO_ATTR(clean_segments); +NILFS_SEGMENTS_RO_ATTR(dirty_segments); +NILFS_SEGMENTS_RO_ATTR(README); + +static struct attribute *nilfs_segments_attrs[] = { + NILFS_SEGMENTS_ATTR_LIST(segments_number), + NILFS_SEGMENTS_ATTR_LIST(blocks_per_segment), + NILFS_SEGMENTS_ATTR_LIST(clean_segments), + NILFS_SEGMENTS_ATTR_LIST(dirty_segments), + NILFS_SEGMENTS_ATTR_LIST(README), + NULL, +}; +ATTRIBUTE_GROUPS(nilfs_segments); + +NILFS_DEV_INT_GROUP_OPS(segments, dev); +NILFS_DEV_INT_GROUP_TYPE(segments, dev); +NILFS_DEV_INT_GROUP_FNS(segments, dev); + +/************************************************************************ + * NILFS segctor attrs * + ************************************************************************/ + +static ssize_t +nilfs_segctor_last_pseg_block_show(struct nilfs_segctor_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + sector_t last_pseg; + + spin_lock(&nilfs->ns_last_segment_lock); + last_pseg = nilfs->ns_last_pseg; + spin_unlock(&nilfs->ns_last_segment_lock); + + return sysfs_emit(buf, "%llu\n", + (unsigned long long)last_pseg); +} + +static ssize_t +nilfs_segctor_last_seg_sequence_show(struct nilfs_segctor_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + u64 last_seq; + + spin_lock(&nilfs->ns_last_segment_lock); + last_seq = nilfs->ns_last_seq; + spin_unlock(&nilfs->ns_last_segment_lock); + + return sysfs_emit(buf, "%llu\n", last_seq); +} + +static ssize_t +nilfs_segctor_last_seg_checkpoint_show(struct nilfs_segctor_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + __u64 last_cno; + + spin_lock(&nilfs->ns_last_segment_lock); + last_cno = nilfs->ns_last_cno; + spin_unlock(&nilfs->ns_last_segment_lock); + + return sysfs_emit(buf, "%llu\n", last_cno); +} + +static ssize_t +nilfs_segctor_current_seg_sequence_show(struct nilfs_segctor_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + u64 seg_seq; + + down_read(&nilfs->ns_segctor_sem); + seg_seq = nilfs->ns_seg_seq; + up_read(&nilfs->ns_segctor_sem); + + return sysfs_emit(buf, "%llu\n", seg_seq); +} + +static ssize_t +nilfs_segctor_current_last_full_seg_show(struct nilfs_segctor_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + __u64 segnum; + + down_read(&nilfs->ns_segctor_sem); + segnum = nilfs->ns_segnum; + up_read(&nilfs->ns_segctor_sem); + + return sysfs_emit(buf, "%llu\n", segnum); +} + +static ssize_t +nilfs_segctor_next_full_seg_show(struct nilfs_segctor_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + __u64 nextnum; + + down_read(&nilfs->ns_segctor_sem); + nextnum = nilfs->ns_nextnum; + up_read(&nilfs->ns_segctor_sem); + + return sysfs_emit(buf, "%llu\n", nextnum); +} + +static ssize_t +nilfs_segctor_next_pseg_offset_show(struct nilfs_segctor_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + unsigned long pseg_offset; + + down_read(&nilfs->ns_segctor_sem); + pseg_offset = nilfs->ns_pseg_offset; + up_read(&nilfs->ns_segctor_sem); + + return sysfs_emit(buf, "%lu\n", pseg_offset); +} + +static ssize_t +nilfs_segctor_next_checkpoint_show(struct nilfs_segctor_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + __u64 cno; + + down_read(&nilfs->ns_segctor_sem); + cno = nilfs->ns_cno; + up_read(&nilfs->ns_segctor_sem); + + return sysfs_emit(buf, "%llu\n", cno); +} + +static ssize_t +nilfs_segctor_last_seg_write_time_show(struct nilfs_segctor_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + time64_t ctime; + + down_read(&nilfs->ns_segctor_sem); + ctime = nilfs->ns_ctime; + up_read(&nilfs->ns_segctor_sem); + + return sysfs_emit(buf, "%ptTs\n", &ctime); +} + +static ssize_t +nilfs_segctor_last_seg_write_time_secs_show(struct nilfs_segctor_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + time64_t ctime; + + down_read(&nilfs->ns_segctor_sem); + ctime = nilfs->ns_ctime; + up_read(&nilfs->ns_segctor_sem); + + return sysfs_emit(buf, "%llu\n", ctime); +} + +static ssize_t +nilfs_segctor_last_nongc_write_time_show(struct nilfs_segctor_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + time64_t nongc_ctime; + + down_read(&nilfs->ns_segctor_sem); + nongc_ctime = nilfs->ns_nongc_ctime; + up_read(&nilfs->ns_segctor_sem); + + return sysfs_emit(buf, "%ptTs\n", &nongc_ctime); +} + +static ssize_t +nilfs_segctor_last_nongc_write_time_secs_show(struct nilfs_segctor_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + time64_t nongc_ctime; + + down_read(&nilfs->ns_segctor_sem); + nongc_ctime = nilfs->ns_nongc_ctime; + up_read(&nilfs->ns_segctor_sem); + + return sysfs_emit(buf, "%llu\n", nongc_ctime); +} + +static ssize_t +nilfs_segctor_dirty_data_blocks_count_show(struct nilfs_segctor_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + u32 ndirtyblks; + + down_read(&nilfs->ns_segctor_sem); + ndirtyblks = atomic_read(&nilfs->ns_ndirtyblks); + up_read(&nilfs->ns_segctor_sem); + + return sysfs_emit(buf, "%u\n", ndirtyblks); +} + +static const char segctor_readme_str[] = + "The segctor group contains attributes that describe\n" + "segctor thread activity details.\n\n" + "(1) last_pseg_block\n" + "\tshow start block number of the latest segment.\n\n" + "(2) last_seg_sequence\n" + "\tshow sequence value of the latest segment.\n\n" + "(3) last_seg_checkpoint\n" + "\tshow checkpoint number of the latest segment.\n\n" + "(4) current_seg_sequence\n\tshow segment sequence counter.\n\n" + "(5) current_last_full_seg\n" + "\tshow index number of the latest full segment.\n\n" + "(6) next_full_seg\n" + "\tshow index number of the full segment index to be used next.\n\n" + "(7) next_pseg_offset\n" + "\tshow offset of next partial segment in the current full segment.\n\n" + "(8) next_checkpoint\n\tshow next checkpoint number.\n\n" + "(9) last_seg_write_time\n" + "\tshow write time of the last segment in human-readable format.\n\n" + "(10) last_seg_write_time_secs\n" + "\tshow write time of the last segment in seconds.\n\n" + "(11) last_nongc_write_time\n" + "\tshow write time of the last segment not for cleaner operation " + "in human-readable format.\n\n" + "(12) last_nongc_write_time_secs\n" + "\tshow write time of the last segment not for cleaner operation " + "in seconds.\n\n" + "(13) dirty_data_blocks_count\n" + "\tshow number of dirty data blocks.\n\n"; + +static ssize_t +nilfs_segctor_README_show(struct nilfs_segctor_attr *attr, + struct the_nilfs *nilfs, char *buf) +{ + return sysfs_emit(buf, segctor_readme_str); +} + +NILFS_SEGCTOR_RO_ATTR(last_pseg_block); +NILFS_SEGCTOR_RO_ATTR(last_seg_sequence); +NILFS_SEGCTOR_RO_ATTR(last_seg_checkpoint); +NILFS_SEGCTOR_RO_ATTR(current_seg_sequence); +NILFS_SEGCTOR_RO_ATTR(current_last_full_seg); +NILFS_SEGCTOR_RO_ATTR(next_full_seg); +NILFS_SEGCTOR_RO_ATTR(next_pseg_offset); +NILFS_SEGCTOR_RO_ATTR(next_checkpoint); +NILFS_SEGCTOR_RO_ATTR(last_seg_write_time); +NILFS_SEGCTOR_RO_ATTR(last_seg_write_time_secs); +NILFS_SEGCTOR_RO_ATTR(last_nongc_write_time); +NILFS_SEGCTOR_RO_ATTR(last_nongc_write_time_secs); +NILFS_SEGCTOR_RO_ATTR(dirty_data_blocks_count); +NILFS_SEGCTOR_RO_ATTR(README); + +static struct attribute *nilfs_segctor_attrs[] = { + NILFS_SEGCTOR_ATTR_LIST(last_pseg_block), + NILFS_SEGCTOR_ATTR_LIST(last_seg_sequence), + NILFS_SEGCTOR_ATTR_LIST(last_seg_checkpoint), + NILFS_SEGCTOR_ATTR_LIST(current_seg_sequence), + NILFS_SEGCTOR_ATTR_LIST(current_last_full_seg), + NILFS_SEGCTOR_ATTR_LIST(next_full_seg), + NILFS_SEGCTOR_ATTR_LIST(next_pseg_offset), + NILFS_SEGCTOR_ATTR_LIST(next_checkpoint), + NILFS_SEGCTOR_ATTR_LIST(last_seg_write_time), + NILFS_SEGCTOR_ATTR_LIST(last_seg_write_time_secs), + NILFS_SEGCTOR_ATTR_LIST(last_nongc_write_time), + NILFS_SEGCTOR_ATTR_LIST(last_nongc_write_time_secs), + NILFS_SEGCTOR_ATTR_LIST(dirty_data_blocks_count), + NILFS_SEGCTOR_ATTR_LIST(README), + NULL, +}; +ATTRIBUTE_GROUPS(nilfs_segctor); + +NILFS_DEV_INT_GROUP_OPS(segctor, dev); +NILFS_DEV_INT_GROUP_TYPE(segctor, dev); +NILFS_DEV_INT_GROUP_FNS(segctor, dev); + +/************************************************************************ + * NILFS superblock attrs * + ************************************************************************/ + +static ssize_t +nilfs_superblock_sb_write_time_show(struct nilfs_superblock_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + time64_t sbwtime; + + down_read(&nilfs->ns_sem); + sbwtime = nilfs->ns_sbwtime; + up_read(&nilfs->ns_sem); + + return sysfs_emit(buf, "%ptTs\n", &sbwtime); +} + +static ssize_t +nilfs_superblock_sb_write_time_secs_show(struct nilfs_superblock_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + time64_t sbwtime; + + down_read(&nilfs->ns_sem); + sbwtime = nilfs->ns_sbwtime; + up_read(&nilfs->ns_sem); + + return sysfs_emit(buf, "%llu\n", sbwtime); +} + +static ssize_t +nilfs_superblock_sb_write_count_show(struct nilfs_superblock_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + unsigned int sbwcount; + + down_read(&nilfs->ns_sem); + sbwcount = nilfs->ns_sbwcount; + up_read(&nilfs->ns_sem); + + return sysfs_emit(buf, "%u\n", sbwcount); +} + +static ssize_t +nilfs_superblock_sb_update_frequency_show(struct nilfs_superblock_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + unsigned int sb_update_freq; + + down_read(&nilfs->ns_sem); + sb_update_freq = nilfs->ns_sb_update_freq; + up_read(&nilfs->ns_sem); + + return sysfs_emit(buf, "%u\n", sb_update_freq); +} + +static ssize_t +nilfs_superblock_sb_update_frequency_store(struct nilfs_superblock_attr *attr, + struct the_nilfs *nilfs, + const char *buf, size_t count) +{ + unsigned int val; + int err; + + err = kstrtouint(skip_spaces(buf), 0, &val); + if (err) { + nilfs_err(nilfs->ns_sb, "unable to convert string: err=%d", + err); + return err; + } + + if (val < NILFS_SB_FREQ) { + val = NILFS_SB_FREQ; + nilfs_warn(nilfs->ns_sb, + "superblock update frequency cannot be lesser than 10 seconds"); + } + + down_write(&nilfs->ns_sem); + nilfs->ns_sb_update_freq = val; + up_write(&nilfs->ns_sem); + + return count; +} + +static const char sb_readme_str[] = + "The superblock group contains attributes that describe\n" + "superblock's details.\n\n" + "(1) sb_write_time\n\tshow previous write time of super block " + "in human-readable format.\n\n" + "(2) sb_write_time_secs\n\tshow previous write time of super block " + "in seconds.\n\n" + "(3) sb_write_count\n\tshow write count of super block.\n\n" + "(4) sb_update_frequency\n" + "\tshow/set interval of periodical update of superblock (in seconds).\n\n" + "\tYou can set preferable frequency of superblock update by command:\n\n" + "\t'echo > /sys/fs///superblock/sb_update_frequency'\n"; + +static ssize_t +nilfs_superblock_README_show(struct nilfs_superblock_attr *attr, + struct the_nilfs *nilfs, char *buf) +{ + return sysfs_emit(buf, sb_readme_str); +} + +NILFS_SUPERBLOCK_RO_ATTR(sb_write_time); +NILFS_SUPERBLOCK_RO_ATTR(sb_write_time_secs); +NILFS_SUPERBLOCK_RO_ATTR(sb_write_count); +NILFS_SUPERBLOCK_RW_ATTR(sb_update_frequency); +NILFS_SUPERBLOCK_RO_ATTR(README); + +static struct attribute *nilfs_superblock_attrs[] = { + NILFS_SUPERBLOCK_ATTR_LIST(sb_write_time), + NILFS_SUPERBLOCK_ATTR_LIST(sb_write_time_secs), + NILFS_SUPERBLOCK_ATTR_LIST(sb_write_count), + NILFS_SUPERBLOCK_ATTR_LIST(sb_update_frequency), + NILFS_SUPERBLOCK_ATTR_LIST(README), + NULL, +}; +ATTRIBUTE_GROUPS(nilfs_superblock); + +NILFS_DEV_INT_GROUP_OPS(superblock, dev); +NILFS_DEV_INT_GROUP_TYPE(superblock, dev); +NILFS_DEV_INT_GROUP_FNS(superblock, dev); + +/************************************************************************ + * NILFS device attrs * + ************************************************************************/ + +static +ssize_t nilfs_dev_revision_show(struct nilfs_dev_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + struct nilfs_super_block *raw_sb; + u32 major; + u16 minor; + + down_read(&nilfs->ns_sem); + raw_sb = nilfs->ns_sbp[0]; + major = le32_to_cpu(raw_sb->s_rev_level); + minor = le16_to_cpu(raw_sb->s_minor_rev_level); + up_read(&nilfs->ns_sem); + + return sysfs_emit(buf, "%d.%d\n", major, minor); +} + +static +ssize_t nilfs_dev_blocksize_show(struct nilfs_dev_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + return sysfs_emit(buf, "%u\n", nilfs->ns_blocksize); +} + +static +ssize_t nilfs_dev_device_size_show(struct nilfs_dev_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + struct nilfs_super_block *raw_sb; + u64 dev_size; + + down_read(&nilfs->ns_sem); + raw_sb = nilfs->ns_sbp[0]; + dev_size = le64_to_cpu(raw_sb->s_dev_size); + up_read(&nilfs->ns_sem); + + return sysfs_emit(buf, "%llu\n", dev_size); +} + +static +ssize_t nilfs_dev_free_blocks_show(struct nilfs_dev_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + sector_t free_blocks = 0; + + nilfs_count_free_blocks(nilfs, &free_blocks); + return sysfs_emit(buf, "%llu\n", + (unsigned long long)free_blocks); +} + +static +ssize_t nilfs_dev_uuid_show(struct nilfs_dev_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + struct nilfs_super_block *raw_sb; + ssize_t len; + + down_read(&nilfs->ns_sem); + raw_sb = nilfs->ns_sbp[0]; + len = sysfs_emit(buf, "%pUb\n", raw_sb->s_uuid); + up_read(&nilfs->ns_sem); + + return len; +} + +static +ssize_t nilfs_dev_volume_name_show(struct nilfs_dev_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + struct nilfs_super_block *raw_sb; + ssize_t len; + + down_read(&nilfs->ns_sem); + raw_sb = nilfs->ns_sbp[0]; + len = scnprintf(buf, sizeof(raw_sb->s_volume_name), "%s\n", + raw_sb->s_volume_name); + up_read(&nilfs->ns_sem); + + return len; +} + +static const char dev_readme_str[] = + "The group contains attributes that describe file system\n" + "partition's details.\n\n" + "(1) revision\n\tshow NILFS file system revision.\n\n" + "(2) blocksize\n\tshow volume block size in bytes.\n\n" + "(3) device_size\n\tshow volume size in bytes.\n\n" + "(4) free_blocks\n\tshow count of free blocks on volume.\n\n" + "(5) uuid\n\tshow volume's UUID.\n\n" + "(6) volume_name\n\tshow volume's name.\n\n"; + +static ssize_t nilfs_dev_README_show(struct nilfs_dev_attr *attr, + struct the_nilfs *nilfs, + char *buf) +{ + return sysfs_emit(buf, dev_readme_str); +} + +NILFS_DEV_RO_ATTR(revision); +NILFS_DEV_RO_ATTR(blocksize); +NILFS_DEV_RO_ATTR(device_size); +NILFS_DEV_RO_ATTR(free_blocks); +NILFS_DEV_RO_ATTR(uuid); +NILFS_DEV_RO_ATTR(volume_name); +NILFS_DEV_RO_ATTR(README); + +static struct attribute *nilfs_dev_attrs[] = { + NILFS_DEV_ATTR_LIST(revision), + NILFS_DEV_ATTR_LIST(blocksize), + NILFS_DEV_ATTR_LIST(device_size), + NILFS_DEV_ATTR_LIST(free_blocks), + NILFS_DEV_ATTR_LIST(uuid), + NILFS_DEV_ATTR_LIST(volume_name), + NILFS_DEV_ATTR_LIST(README), + NULL, +}; +ATTRIBUTE_GROUPS(nilfs_dev); + +static ssize_t nilfs_dev_attr_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct the_nilfs *nilfs = container_of(kobj, struct the_nilfs, + ns_dev_kobj); + struct nilfs_dev_attr *a = container_of(attr, struct nilfs_dev_attr, + attr); + + return a->show ? a->show(a, nilfs, buf) : 0; +} + +static ssize_t nilfs_dev_attr_store(struct kobject *kobj, + struct attribute *attr, + const char *buf, size_t len) +{ + struct the_nilfs *nilfs = container_of(kobj, struct the_nilfs, + ns_dev_kobj); + struct nilfs_dev_attr *a = container_of(attr, struct nilfs_dev_attr, + attr); + + return a->store ? a->store(a, nilfs, buf, len) : 0; +} + +static void nilfs_dev_attr_release(struct kobject *kobj) +{ + struct the_nilfs *nilfs = container_of(kobj, struct the_nilfs, + ns_dev_kobj); + complete(&nilfs->ns_dev_kobj_unregister); +} + +static const struct sysfs_ops nilfs_dev_attr_ops = { + .show = nilfs_dev_attr_show, + .store = nilfs_dev_attr_store, +}; + +static const struct kobj_type nilfs_dev_ktype = { + .default_groups = nilfs_dev_groups, + .sysfs_ops = &nilfs_dev_attr_ops, + .release = nilfs_dev_attr_release, +}; + +int nilfs_sysfs_create_device_group(struct super_block *sb) +{ + struct the_nilfs *nilfs = sb->s_fs_info; + size_t devgrp_size = sizeof(struct nilfs_sysfs_dev_subgroups); + int err; + + nilfs->ns_dev_subgroups = kzalloc(devgrp_size, GFP_KERNEL); + if (unlikely(!nilfs->ns_dev_subgroups)) { + err = -ENOMEM; + nilfs_err(sb, "unable to allocate memory for device group"); + goto failed_create_device_group; + } + + nilfs->ns_dev_kobj.kset = nilfs_kset; + init_completion(&nilfs->ns_dev_kobj_unregister); + err = kobject_init_and_add(&nilfs->ns_dev_kobj, &nilfs_dev_ktype, NULL, + "%s", sb->s_id); + if (err) + goto cleanup_dev_kobject; + + err = nilfs_sysfs_create_mounted_snapshots_group(nilfs); + if (err) + goto cleanup_dev_kobject; + + err = nilfs_sysfs_create_checkpoints_group(nilfs); + if (err) + goto delete_mounted_snapshots_group; + + err = nilfs_sysfs_create_segments_group(nilfs); + if (err) + goto delete_checkpoints_group; + + err = nilfs_sysfs_create_superblock_group(nilfs); + if (err) + goto delete_segments_group; + + err = nilfs_sysfs_create_segctor_group(nilfs); + if (err) + goto delete_superblock_group; + + return 0; + +delete_superblock_group: + nilfs_sysfs_delete_superblock_group(nilfs); + +delete_segments_group: + nilfs_sysfs_delete_segments_group(nilfs); + +delete_checkpoints_group: + nilfs_sysfs_delete_checkpoints_group(nilfs); + +delete_mounted_snapshots_group: + nilfs_sysfs_delete_mounted_snapshots_group(nilfs); + +cleanup_dev_kobject: + kobject_put(&nilfs->ns_dev_kobj); + kfree(nilfs->ns_dev_subgroups); + +failed_create_device_group: + return err; +} + +void nilfs_sysfs_delete_device_group(struct the_nilfs *nilfs) +{ + nilfs_sysfs_delete_mounted_snapshots_group(nilfs); + nilfs_sysfs_delete_checkpoints_group(nilfs); + nilfs_sysfs_delete_segments_group(nilfs); + nilfs_sysfs_delete_superblock_group(nilfs); + nilfs_sysfs_delete_segctor_group(nilfs); + kobject_del(&nilfs->ns_dev_kobj); + kobject_put(&nilfs->ns_dev_kobj); + kfree(nilfs->ns_dev_subgroups); +} + +/************************************************************************ + * NILFS feature attrs * + ************************************************************************/ + +static ssize_t nilfs_feature_revision_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%d.%d\n", + NILFS_CURRENT_REV, NILFS_MINOR_REV); +} + +static const char features_readme_str[] = + "The features group contains attributes that describe NILFS file\n" + "system driver features.\n\n" + "(1) revision\n\tshow current revision of NILFS file system driver.\n"; + +static ssize_t nilfs_feature_README_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + return sysfs_emit(buf, features_readme_str); +} + +NILFS_FEATURE_RO_ATTR(revision); +NILFS_FEATURE_RO_ATTR(README); + +static struct attribute *nilfs_feature_attrs[] = { + NILFS_FEATURE_ATTR_LIST(revision), + NILFS_FEATURE_ATTR_LIST(README), + NULL, +}; + +static const struct attribute_group nilfs_feature_attr_group = { + .name = "features", + .attrs = nilfs_feature_attrs, +}; + +int __init nilfs_sysfs_init(void) +{ + int err; + + nilfs_kset = kset_create_and_add(NILFS_ROOT_GROUP_NAME, NULL, fs_kobj); + if (!nilfs_kset) { + err = -ENOMEM; + nilfs_err(NULL, "unable to create sysfs entry: err=%d", err); + goto failed_sysfs_init; + } + + err = sysfs_create_group(&nilfs_kset->kobj, &nilfs_feature_attr_group); + if (unlikely(err)) { + nilfs_err(NULL, "unable to create feature group: err=%d", err); + goto cleanup_sysfs_init; + } + + return 0; + +cleanup_sysfs_init: + kset_unregister(nilfs_kset); + +failed_sysfs_init: + return err; +} + +void nilfs_sysfs_exit(void) +{ + sysfs_remove_group(&nilfs_kset->kobj, &nilfs_feature_attr_group); + kset_unregister(nilfs_kset); +} diff --git a/fs/nilfs2/sysfs.h b/fs/nilfs2/sysfs.h new file mode 100644 index 000000000..d370cd5cc --- /dev/null +++ b/fs/nilfs2/sysfs.h @@ -0,0 +1,167 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Sysfs support declarations. + * + * Copyright (C) 2005-2014 Nippon Telegraph and Telephone Corporation. + * Copyright (C) 2014 HGST, Inc., a Western Digital Company. + * + * Written by Vyacheslav Dubeyko + */ + +#ifndef _NILFS_SYSFS_H +#define _NILFS_SYSFS_H + +#include + +#define NILFS_ROOT_GROUP_NAME "nilfs2" + +/* + * struct nilfs_sysfs_dev_subgroups - device subgroup kernel objects + * @sg_superblock_kobj: /sys/fs///superblock + * @sg_superblock_kobj_unregister: completion state + * @sg_segctor_kobj: /sys/fs///segctor + * @sg_segctor_kobj_unregister: completion state + * @sg_mounted_snapshots_kobj: /sys/fs///mounted_snapshots + * @sg_mounted_snapshots_kobj_unregister: completion state + * @sg_checkpoints_kobj: /sys/fs///checkpoints + * @sg_checkpoints_kobj_unregister: completion state + * @sg_segments_kobj: /sys/fs///segments + * @sg_segments_kobj_unregister: completion state + */ +struct nilfs_sysfs_dev_subgroups { + /* /sys/fs///superblock */ + struct kobject sg_superblock_kobj; + struct completion sg_superblock_kobj_unregister; + + /* /sys/fs///segctor */ + struct kobject sg_segctor_kobj; + struct completion sg_segctor_kobj_unregister; + + /* /sys/fs///mounted_snapshots */ + struct kobject sg_mounted_snapshots_kobj; + struct completion sg_mounted_snapshots_kobj_unregister; + + /* /sys/fs///checkpoints */ + struct kobject sg_checkpoints_kobj; + struct completion sg_checkpoints_kobj_unregister; + + /* /sys/fs///segments */ + struct kobject sg_segments_kobj; + struct completion sg_segments_kobj_unregister; +}; + +#define NILFS_KOBJ_ATTR_STRUCT(name) \ +struct nilfs_##name##_attr { \ + struct attribute attr; \ + ssize_t (*show)(struct kobject *, struct kobj_attribute *, \ + char *); \ + ssize_t (*store)(struct kobject *, struct kobj_attribute *, \ + const char *, size_t); \ +} + +NILFS_KOBJ_ATTR_STRUCT(feature); + +#define NILFS_DEV_ATTR_STRUCT(name) \ +struct nilfs_##name##_attr { \ + struct attribute attr; \ + ssize_t (*show)(struct nilfs_##name##_attr *, struct the_nilfs *, \ + char *); \ + ssize_t (*store)(struct nilfs_##name##_attr *, struct the_nilfs *, \ + const char *, size_t); \ +} + +NILFS_DEV_ATTR_STRUCT(dev); +NILFS_DEV_ATTR_STRUCT(segments); +NILFS_DEV_ATTR_STRUCT(mounted_snapshots); +NILFS_DEV_ATTR_STRUCT(checkpoints); +NILFS_DEV_ATTR_STRUCT(superblock); +NILFS_DEV_ATTR_STRUCT(segctor); + +#define NILFS_CP_ATTR_STRUCT(name) \ +struct nilfs_##name##_attr { \ + struct attribute attr; \ + ssize_t (*show)(struct nilfs_##name##_attr *, struct nilfs_root *, \ + char *); \ + ssize_t (*store)(struct nilfs_##name##_attr *, struct nilfs_root *, \ + const char *, size_t); \ +} + +NILFS_CP_ATTR_STRUCT(snapshot); + +#define NILFS_ATTR(type, name, mode, show, store) \ + static struct nilfs_##type##_attr nilfs_##type##_attr_##name = \ + __ATTR(name, mode, show, store) + +#define NILFS_INFO_ATTR(type, name) \ + NILFS_ATTR(type, name, 0444, NULL, NULL) +#define NILFS_RO_ATTR(type, name) \ + NILFS_ATTR(type, name, 0444, nilfs_##type##_##name##_show, NULL) +#define NILFS_RW_ATTR(type, name) \ + NILFS_ATTR(type, name, 0644, \ + nilfs_##type##_##name##_show, \ + nilfs_##type##_##name##_store) + +#define NILFS_FEATURE_INFO_ATTR(name) \ + NILFS_INFO_ATTR(feature, name) +#define NILFS_FEATURE_RO_ATTR(name) \ + NILFS_RO_ATTR(feature, name) +#define NILFS_FEATURE_RW_ATTR(name) \ + NILFS_RW_ATTR(feature, name) + +#define NILFS_DEV_INFO_ATTR(name) \ + NILFS_INFO_ATTR(dev, name) +#define NILFS_DEV_RO_ATTR(name) \ + NILFS_RO_ATTR(dev, name) +#define NILFS_DEV_RW_ATTR(name) \ + NILFS_RW_ATTR(dev, name) + +#define NILFS_SEGMENTS_RO_ATTR(name) \ + NILFS_RO_ATTR(segments, name) +#define NILFS_SEGMENTS_RW_ATTR(name) \ + NILFS_RW_ATTR(segs_info, name) + +#define NILFS_MOUNTED_SNAPSHOTS_RO_ATTR(name) \ + NILFS_RO_ATTR(mounted_snapshots, name) + +#define NILFS_CHECKPOINTS_RO_ATTR(name) \ + NILFS_RO_ATTR(checkpoints, name) +#define NILFS_CHECKPOINTS_RW_ATTR(name) \ + NILFS_RW_ATTR(checkpoints, name) + +#define NILFS_SNAPSHOT_INFO_ATTR(name) \ + NILFS_INFO_ATTR(snapshot, name) +#define NILFS_SNAPSHOT_RO_ATTR(name) \ + NILFS_RO_ATTR(snapshot, name) +#define NILFS_SNAPSHOT_RW_ATTR(name) \ + NILFS_RW_ATTR(snapshot, name) + +#define NILFS_SUPERBLOCK_RO_ATTR(name) \ + NILFS_RO_ATTR(superblock, name) +#define NILFS_SUPERBLOCK_RW_ATTR(name) \ + NILFS_RW_ATTR(superblock, name) + +#define NILFS_SEGCTOR_INFO_ATTR(name) \ + NILFS_INFO_ATTR(segctor, name) +#define NILFS_SEGCTOR_RO_ATTR(name) \ + NILFS_RO_ATTR(segctor, name) +#define NILFS_SEGCTOR_RW_ATTR(name) \ + NILFS_RW_ATTR(segctor, name) + +#define NILFS_FEATURE_ATTR_LIST(name) \ + (&nilfs_feature_attr_##name.attr) +#define NILFS_DEV_ATTR_LIST(name) \ + (&nilfs_dev_attr_##name.attr) +#define NILFS_SEGMENTS_ATTR_LIST(name) \ + (&nilfs_segments_attr_##name.attr) +#define NILFS_MOUNTED_SNAPSHOTS_ATTR_LIST(name) \ + (&nilfs_mounted_snapshots_attr_##name.attr) +#define NILFS_CHECKPOINTS_ATTR_LIST(name) \ + (&nilfs_checkpoints_attr_##name.attr) +#define NILFS_SNAPSHOT_ATTR_LIST(name) \ + (&nilfs_snapshot_attr_##name.attr) +#define NILFS_SUPERBLOCK_ATTR_LIST(name) \ + (&nilfs_superblock_attr_##name.attr) +#define NILFS_SEGCTOR_ATTR_LIST(name) \ + (&nilfs_segctor_attr_##name.attr) + +#endif /* _NILFS_SYSFS_H */ diff --git a/fs/nilfs2/the_nilfs.c b/fs/nilfs2/the_nilfs.c new file mode 100644 index 000000000..f3805e7aa --- /dev/null +++ b/fs/nilfs2/the_nilfs.c @@ -0,0 +1,945 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * the_nilfs shared structure. + * + * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Ryusuke Konishi. + * + */ + +#include +#include +#include +#include +#include +#include +#include "nilfs.h" +#include "segment.h" +#include "alloc.h" +#include "cpfile.h" +#include "sufile.h" +#include "dat.h" +#include "segbuf.h" + + +static int nilfs_valid_sb(struct nilfs_super_block *sbp); + +void nilfs_set_last_segment(struct the_nilfs *nilfs, + sector_t start_blocknr, u64 seq, __u64 cno) +{ + spin_lock(&nilfs->ns_last_segment_lock); + nilfs->ns_last_pseg = start_blocknr; + nilfs->ns_last_seq = seq; + nilfs->ns_last_cno = cno; + + if (!nilfs_sb_dirty(nilfs)) { + if (nilfs->ns_prev_seq == nilfs->ns_last_seq) + goto stay_cursor; + + set_nilfs_sb_dirty(nilfs); + } + nilfs->ns_prev_seq = nilfs->ns_last_seq; + + stay_cursor: + spin_unlock(&nilfs->ns_last_segment_lock); +} + +/** + * alloc_nilfs - allocate a nilfs object + * @sb: super block instance + * + * Return: a pointer to the allocated nilfs object on success, or NULL on + * failure. + */ +struct the_nilfs *alloc_nilfs(struct super_block *sb) +{ + struct the_nilfs *nilfs; + + nilfs = kzalloc_obj(*nilfs); + if (!nilfs) + return NULL; + + nilfs->ns_sb = sb; + nilfs->ns_bdev = sb->s_bdev; + atomic_set(&nilfs->ns_ndirtyblks, 0); + init_rwsem(&nilfs->ns_sem); + mutex_init(&nilfs->ns_snapshot_mount_mutex); + INIT_LIST_HEAD(&nilfs->ns_dirty_files); + INIT_LIST_HEAD(&nilfs->ns_gc_inodes); + spin_lock_init(&nilfs->ns_inode_lock); + spin_lock_init(&nilfs->ns_last_segment_lock); + nilfs->ns_cptree = RB_ROOT; + spin_lock_init(&nilfs->ns_cptree_lock); + init_rwsem(&nilfs->ns_segctor_sem); + nilfs->ns_sb_update_freq = NILFS_SB_FREQ; + + return nilfs; +} + +/** + * destroy_nilfs - destroy nilfs object + * @nilfs: nilfs object to be released + */ +void destroy_nilfs(struct the_nilfs *nilfs) +{ + might_sleep(); + if (nilfs_init(nilfs)) { + brelse(nilfs->ns_sbh[0]); + brelse(nilfs->ns_sbh[1]); + } + kfree(nilfs); +} + +static int nilfs_load_super_root(struct the_nilfs *nilfs, + struct super_block *sb, sector_t sr_block) +{ + struct buffer_head *bh_sr; + struct nilfs_super_root *raw_sr; + struct nilfs_super_block **sbp = nilfs->ns_sbp; + struct nilfs_inode *rawi; + unsigned int dat_entry_size, segment_usage_size, checkpoint_size; + unsigned int inode_size; + int err; + + err = nilfs_read_super_root_block(nilfs, sr_block, &bh_sr, 1); + if (unlikely(err)) + return err; + + down_read(&nilfs->ns_sem); + dat_entry_size = le16_to_cpu(sbp[0]->s_dat_entry_size); + checkpoint_size = le16_to_cpu(sbp[0]->s_checkpoint_size); + segment_usage_size = le16_to_cpu(sbp[0]->s_segment_usage_size); + up_read(&nilfs->ns_sem); + + inode_size = nilfs->ns_inode_size; + + rawi = (void *)bh_sr->b_data + NILFS_SR_DAT_OFFSET(inode_size); + err = nilfs_dat_read(sb, dat_entry_size, rawi, &nilfs->ns_dat); + if (err) + goto failed; + + rawi = (void *)bh_sr->b_data + NILFS_SR_CPFILE_OFFSET(inode_size); + err = nilfs_cpfile_read(sb, checkpoint_size, rawi, &nilfs->ns_cpfile); + if (err) + goto failed_dat; + + rawi = (void *)bh_sr->b_data + NILFS_SR_SUFILE_OFFSET(inode_size); + err = nilfs_sufile_read(sb, segment_usage_size, rawi, + &nilfs->ns_sufile); + if (err) + goto failed_cpfile; + + raw_sr = (struct nilfs_super_root *)bh_sr->b_data; + nilfs->ns_nongc_ctime = le64_to_cpu(raw_sr->sr_nongc_ctime); + + failed: + brelse(bh_sr); + return err; + + failed_cpfile: + iput(nilfs->ns_cpfile); + + failed_dat: + iput(nilfs->ns_dat); + goto failed; +} + +static void nilfs_init_recovery_info(struct nilfs_recovery_info *ri) +{ + memset(ri, 0, sizeof(*ri)); + INIT_LIST_HEAD(&ri->ri_used_segments); +} + +static void nilfs_clear_recovery_info(struct nilfs_recovery_info *ri) +{ + nilfs_dispose_segment_list(&ri->ri_used_segments); +} + +/** + * nilfs_store_log_cursor - load log cursor from a super block + * @nilfs: nilfs object + * @sbp: buffer storing super block to be read + * + * nilfs_store_log_cursor() reads the last position of the log + * containing a super root from a given super block, and initializes + * relevant information on the nilfs object preparatory for log + * scanning and recovery. + * + * Return: 0 on success, or %-EINVAL if current segment number is out + * of range. + */ +static int nilfs_store_log_cursor(struct the_nilfs *nilfs, + struct nilfs_super_block *sbp) +{ + int ret = 0; + + nilfs->ns_last_pseg = le64_to_cpu(sbp->s_last_pseg); + nilfs->ns_last_cno = le64_to_cpu(sbp->s_last_cno); + nilfs->ns_last_seq = le64_to_cpu(sbp->s_last_seq); + + nilfs->ns_prev_seq = nilfs->ns_last_seq; + nilfs->ns_seg_seq = nilfs->ns_last_seq; + nilfs->ns_segnum = + nilfs_get_segnum_of_block(nilfs, nilfs->ns_last_pseg); + nilfs->ns_cno = nilfs->ns_last_cno + 1; + if (nilfs->ns_segnum >= nilfs->ns_nsegments) { + nilfs_err(nilfs->ns_sb, + "pointed segment number is out of range: segnum=%llu, nsegments=%lu", + (unsigned long long)nilfs->ns_segnum, + nilfs->ns_nsegments); + ret = -EINVAL; + } + return ret; +} + +/** + * nilfs_get_blocksize - get block size from raw superblock data + * @sb: super block instance + * @sbp: superblock raw data buffer + * @blocksize: place to store block size + * + * nilfs_get_blocksize() calculates the block size from the block size + * exponent information written in @sbp and stores it in @blocksize, + * or aborts with an error message if it's too large. + * + * Return: 0 on success, or %-EINVAL if the block size is too large. + */ +static int nilfs_get_blocksize(struct super_block *sb, + struct nilfs_super_block *sbp, int *blocksize) +{ + unsigned int shift_bits = le32_to_cpu(sbp->s_log_block_size); + + if (unlikely(shift_bits > + ilog2(NILFS_MAX_BLOCK_SIZE) - BLOCK_SIZE_BITS)) { + nilfs_err(sb, "too large filesystem blocksize: 2 ^ %u KiB", + shift_bits); + return -EINVAL; + } + *blocksize = BLOCK_SIZE << shift_bits; + return 0; +} + +/** + * load_nilfs - load and recover the nilfs + * @nilfs: the_nilfs structure to be released + * @sb: super block instance used to recover past segment + * + * load_nilfs() searches and load the latest super root, + * attaches the last segment, and does recovery if needed. + * The caller must call this exclusively for simultaneous mounts. + * + * Return: 0 on success, or one of the following negative error codes on + * failure: + * * %-EINVAL - No valid segment found. + * * %-EIO - I/O error. + * * %-ENOMEM - Insufficient memory available. + * * %-EROFS - Read only device or RO compat mode (if recovery is required) + */ +int load_nilfs(struct the_nilfs *nilfs, struct super_block *sb) +{ + struct nilfs_recovery_info ri; + unsigned int s_flags = sb->s_flags; + int really_read_only = bdev_read_only(nilfs->ns_bdev); + int valid_fs = nilfs_valid_fs(nilfs); + int err; + + if (!valid_fs) { + nilfs_warn(sb, "mounting unchecked fs"); + if (s_flags & SB_RDONLY) { + nilfs_info(sb, + "recovery required for readonly filesystem"); + nilfs_info(sb, + "write access will be enabled during recovery"); + } + } + + nilfs_init_recovery_info(&ri); + + err = nilfs_search_super_root(nilfs, &ri); + if (unlikely(err)) { + struct nilfs_super_block **sbp = nilfs->ns_sbp; + int blocksize; + + if (err != -EINVAL) + goto scan_error; + + if (!nilfs_valid_sb(sbp[1])) { + nilfs_warn(sb, + "unable to fall back to spare super block"); + goto scan_error; + } + nilfs_info(sb, "trying rollback from an earlier position"); + + /* + * restore super block with its spare and reconfigure + * relevant states of the nilfs object. + */ + memcpy(sbp[0], sbp[1], nilfs->ns_sbsize); + nilfs->ns_crc_seed = le32_to_cpu(sbp[0]->s_crc_seed); + nilfs->ns_sbwtime = le64_to_cpu(sbp[0]->s_wtime); + + /* verify consistency between two super blocks */ + err = nilfs_get_blocksize(sb, sbp[0], &blocksize); + if (err) + goto scan_error; + + if (blocksize != nilfs->ns_blocksize) { + nilfs_warn(sb, + "blocksize differs between two super blocks (%d != %d)", + blocksize, nilfs->ns_blocksize); + err = -EINVAL; + goto scan_error; + } + + err = nilfs_store_log_cursor(nilfs, sbp[0]); + if (err) + goto scan_error; + + /* drop clean flag to allow roll-forward and recovery */ + nilfs->ns_mount_state &= ~NILFS_VALID_FS; + valid_fs = 0; + + err = nilfs_search_super_root(nilfs, &ri); + if (err) + goto scan_error; + } + + err = nilfs_load_super_root(nilfs, sb, ri.ri_super_root); + if (unlikely(err)) { + nilfs_err(sb, "error %d while loading super root", err); + goto failed; + } + + err = nilfs_sysfs_create_device_group(sb); + if (unlikely(err)) + goto sysfs_error; + + if (valid_fs) + goto skip_recovery; + + if (s_flags & SB_RDONLY) { + __u64 features; + + if (nilfs_test_opt(nilfs, NORECOVERY)) { + nilfs_info(sb, + "norecovery option specified, skipping roll-forward recovery"); + goto skip_recovery; + } + features = le64_to_cpu(nilfs->ns_sbp[0]->s_feature_compat_ro) & + ~NILFS_FEATURE_COMPAT_RO_SUPP; + if (features) { + nilfs_err(sb, + "couldn't proceed with recovery because of unsupported optional features (%llx)", + (unsigned long long)features); + err = -EROFS; + goto failed_unload; + } + if (really_read_only) { + nilfs_err(sb, + "write access unavailable, cannot proceed"); + err = -EROFS; + goto failed_unload; + } + sb->s_flags &= ~SB_RDONLY; + } else if (nilfs_test_opt(nilfs, NORECOVERY)) { + nilfs_err(sb, + "recovery cancelled because norecovery option was specified for a read/write mount"); + err = -EINVAL; + goto failed_unload; + } + + err = nilfs_salvage_orphan_logs(nilfs, sb, &ri); + if (err) + goto failed_unload; + + down_write(&nilfs->ns_sem); + nilfs->ns_mount_state |= NILFS_VALID_FS; /* set "clean" flag */ + err = nilfs_cleanup_super(sb); + up_write(&nilfs->ns_sem); + + if (err) { + nilfs_err(sb, + "error %d updating super block. recovery unfinished.", + err); + goto failed_unload; + } + nilfs_info(sb, "recovery complete"); + + skip_recovery: + nilfs_clear_recovery_info(&ri); + sb->s_flags = s_flags; + return 0; + + scan_error: + nilfs_err(sb, "error %d while searching super root", err); + goto failed; + + failed_unload: + nilfs_sysfs_delete_device_group(nilfs); + + sysfs_error: + iput(nilfs->ns_cpfile); + iput(nilfs->ns_sufile); + iput(nilfs->ns_dat); + + failed: + nilfs_clear_recovery_info(&ri); + sb->s_flags = s_flags; + return err; +} + +static unsigned long long nilfs_max_size(unsigned int blkbits) +{ + unsigned int max_bits; + unsigned long long res = MAX_LFS_FILESIZE; /* page cache limit */ + + max_bits = blkbits + NILFS_BMAP_KEY_BIT; /* bmap size limit */ + if (max_bits < 64) + res = min_t(unsigned long long, res, (1ULL << max_bits) - 1); + return res; +} + +/** + * nilfs_nrsvsegs - calculate the number of reserved segments + * @nilfs: nilfs object + * @nsegs: total number of segments + * + * Return: Number of reserved segments. + */ +unsigned long nilfs_nrsvsegs(struct the_nilfs *nilfs, unsigned long nsegs) +{ + return max_t(unsigned long, NILFS_MIN_NRSVSEGS, + DIV_ROUND_UP(nsegs * nilfs->ns_r_segments_percentage, + 100)); +} + +/** + * nilfs_max_segment_count - calculate the maximum number of segments + * @nilfs: nilfs object + * + * Return: Maximum number of segments + */ +static u64 nilfs_max_segment_count(struct the_nilfs *nilfs) +{ + u64 max_count = U64_MAX; + + max_count = div64_ul(max_count, nilfs->ns_blocks_per_segment); + return min_t(u64, max_count, ULONG_MAX); +} + +void nilfs_set_nsegments(struct the_nilfs *nilfs, unsigned long nsegs) +{ + nilfs->ns_nsegments = nsegs; + nilfs->ns_nrsvsegs = nilfs_nrsvsegs(nilfs, nsegs); +} + +static int nilfs_store_disk_layout(struct the_nilfs *nilfs, + struct nilfs_super_block *sbp) +{ + u64 nsegments, nblocks; + + if (le32_to_cpu(sbp->s_rev_level) < NILFS_MIN_SUPP_REV) { + nilfs_err(nilfs->ns_sb, + "unsupported revision (superblock rev.=%d.%d, current rev.=%d.%d). Please check the version of mkfs.nilfs(2).", + le32_to_cpu(sbp->s_rev_level), + le16_to_cpu(sbp->s_minor_rev_level), + NILFS_CURRENT_REV, NILFS_MINOR_REV); + return -EINVAL; + } + nilfs->ns_sbsize = le16_to_cpu(sbp->s_bytes); + if (nilfs->ns_sbsize > BLOCK_SIZE) + return -EINVAL; + + nilfs->ns_inode_size = le16_to_cpu(sbp->s_inode_size); + if (nilfs->ns_inode_size > nilfs->ns_blocksize) { + nilfs_err(nilfs->ns_sb, "too large inode size: %d bytes", + nilfs->ns_inode_size); + return -EINVAL; + } else if (nilfs->ns_inode_size < NILFS_MIN_INODE_SIZE) { + nilfs_err(nilfs->ns_sb, "too small inode size: %d bytes", + nilfs->ns_inode_size); + return -EINVAL; + } + if (NILFS_SR_BYTES(nilfs->ns_inode_size) > nilfs->ns_blocksize) { + nilfs_err(nilfs->ns_sb, + "too large inode size for super root: %d bytes", + nilfs->ns_inode_size); + return -EINVAL; + } + + nilfs->ns_first_ino = le32_to_cpu(sbp->s_first_ino); + if (nilfs->ns_first_ino < NILFS_USER_INO) { + nilfs_err(nilfs->ns_sb, + "too small lower limit for non-reserved inode numbers: %u", + nilfs->ns_first_ino); + return -EINVAL; + } + + nilfs->ns_blocks_per_segment = le32_to_cpu(sbp->s_blocks_per_segment); + if (nilfs->ns_blocks_per_segment < NILFS_SEG_MIN_BLOCKS) { + nilfs_err(nilfs->ns_sb, "too short segment: %lu blocks", + nilfs->ns_blocks_per_segment); + return -EINVAL; + } + + nilfs->ns_first_data_block = le64_to_cpu(sbp->s_first_data_block); + nilfs->ns_r_segments_percentage = + le32_to_cpu(sbp->s_r_segments_percentage); + if (nilfs->ns_r_segments_percentage < 1 || + nilfs->ns_r_segments_percentage > 99) { + nilfs_err(nilfs->ns_sb, + "invalid reserved segments percentage: %lu", + nilfs->ns_r_segments_percentage); + return -EINVAL; + } + + nsegments = le64_to_cpu(sbp->s_nsegments); + if (nsegments > nilfs_max_segment_count(nilfs)) { + nilfs_err(nilfs->ns_sb, + "segment count %llu exceeds upper limit (%llu segments)", + (unsigned long long)nsegments, + (unsigned long long)nilfs_max_segment_count(nilfs)); + return -EINVAL; + } + + nblocks = sb_bdev_nr_blocks(nilfs->ns_sb); + if (nblocks) { + u64 min_block_count = nsegments * nilfs->ns_blocks_per_segment; + /* + * To avoid failing to mount early device images without a + * second superblock, exclude that block count from the + * "min_block_count" calculation. + */ + + if (nblocks < min_block_count) { + nilfs_err(nilfs->ns_sb, + "total number of segment blocks %llu exceeds device size (%llu blocks)", + (unsigned long long)min_block_count, + (unsigned long long)nblocks); + return -EINVAL; + } + } + + nilfs_set_nsegments(nilfs, nsegments); + nilfs->ns_crc_seed = le32_to_cpu(sbp->s_crc_seed); + return 0; +} + +static int nilfs_valid_sb(struct nilfs_super_block *sbp) +{ + static unsigned char sum[4]; + const int sumoff = offsetof(struct nilfs_super_block, s_sum); + size_t bytes; + u32 crc; + + if (!sbp || le16_to_cpu(sbp->s_magic) != NILFS_SUPER_MAGIC) + return 0; + bytes = le16_to_cpu(sbp->s_bytes); + if (bytes < sumoff + 4 || bytes > BLOCK_SIZE) + return 0; + crc = crc32_le(le32_to_cpu(sbp->s_crc_seed), (unsigned char *)sbp, + sumoff); + crc = crc32_le(crc, sum, 4); + crc = crc32_le(crc, (unsigned char *)sbp + sumoff + 4, + bytes - sumoff - 4); + return crc == le32_to_cpu(sbp->s_sum); +} + +/** + * nilfs_sb2_bad_offset - check the location of the second superblock + * @sbp: superblock raw data buffer + * @offset: byte offset of second superblock calculated from device size + * + * nilfs_sb2_bad_offset() checks if the position on the second + * superblock is valid or not based on the filesystem parameters + * stored in @sbp. If @offset points to a location within the segment + * area, or if the parameters themselves are not normal, it is + * determined to be invalid. + * + * Return: true if invalid, false if valid. + */ +static bool nilfs_sb2_bad_offset(struct nilfs_super_block *sbp, u64 offset) +{ + unsigned int shift_bits = le32_to_cpu(sbp->s_log_block_size); + u32 blocks_per_segment = le32_to_cpu(sbp->s_blocks_per_segment); + u64 nsegments = le64_to_cpu(sbp->s_nsegments); + u64 index; + + if (blocks_per_segment < NILFS_SEG_MIN_BLOCKS || + shift_bits > ilog2(NILFS_MAX_BLOCK_SIZE) - BLOCK_SIZE_BITS) + return true; + + index = offset >> (shift_bits + BLOCK_SIZE_BITS); + do_div(index, blocks_per_segment); + return index < nsegments; +} + +static void nilfs_release_super_block(struct the_nilfs *nilfs) +{ + int i; + + for (i = 0; i < 2; i++) { + if (nilfs->ns_sbp[i]) { + brelse(nilfs->ns_sbh[i]); + nilfs->ns_sbh[i] = NULL; + nilfs->ns_sbp[i] = NULL; + } + } +} + +void nilfs_fall_back_super_block(struct the_nilfs *nilfs) +{ + brelse(nilfs->ns_sbh[0]); + nilfs->ns_sbh[0] = nilfs->ns_sbh[1]; + nilfs->ns_sbp[0] = nilfs->ns_sbp[1]; + nilfs->ns_sbh[1] = NULL; + nilfs->ns_sbp[1] = NULL; +} + +void nilfs_swap_super_block(struct the_nilfs *nilfs) +{ + struct buffer_head *tsbh = nilfs->ns_sbh[0]; + struct nilfs_super_block *tsbp = nilfs->ns_sbp[0]; + + nilfs->ns_sbh[0] = nilfs->ns_sbh[1]; + nilfs->ns_sbp[0] = nilfs->ns_sbp[1]; + nilfs->ns_sbh[1] = tsbh; + nilfs->ns_sbp[1] = tsbp; +} + +static int nilfs_load_super_block(struct the_nilfs *nilfs, + struct super_block *sb, int blocksize, + struct nilfs_super_block **sbpp) +{ + struct nilfs_super_block **sbp = nilfs->ns_sbp; + struct buffer_head **sbh = nilfs->ns_sbh; + u64 sb2off, devsize = bdev_nr_bytes(nilfs->ns_bdev); + int valid[2], swp = 0, older; + + if (devsize < NILFS_SEG_MIN_BLOCKS * NILFS_MIN_BLOCK_SIZE + 4096) { + nilfs_err(sb, "device size too small"); + return -EINVAL; + } + sb2off = NILFS_SB2_OFFSET_BYTES(devsize); + + sbp[0] = nilfs_read_super_block(sb, NILFS_SB_OFFSET_BYTES, blocksize, + &sbh[0]); + sbp[1] = nilfs_read_super_block(sb, sb2off, blocksize, &sbh[1]); + + if (!sbp[0]) { + if (!sbp[1]) { + nilfs_err(sb, "unable to read superblock"); + return -EIO; + } + nilfs_warn(sb, + "unable to read primary superblock (blocksize = %d)", + blocksize); + } else if (!sbp[1]) { + nilfs_warn(sb, + "unable to read secondary superblock (blocksize = %d)", + blocksize); + } + + /* + * Compare two super blocks and set 1 in swp if the secondary + * super block is valid and newer. Otherwise, set 0 in swp. + */ + valid[0] = nilfs_valid_sb(sbp[0]); + valid[1] = nilfs_valid_sb(sbp[1]); + swp = valid[1] && (!valid[0] || + le64_to_cpu(sbp[1]->s_last_cno) > + le64_to_cpu(sbp[0]->s_last_cno)); + + if (valid[swp] && nilfs_sb2_bad_offset(sbp[swp], sb2off)) { + brelse(sbh[1]); + sbh[1] = NULL; + sbp[1] = NULL; + valid[1] = 0; + swp = 0; + } + if (!valid[swp]) { + nilfs_release_super_block(nilfs); + nilfs_err(sb, "couldn't find nilfs on the device"); + return -EINVAL; + } + + if (!valid[!swp]) + nilfs_warn(sb, + "broken superblock, retrying with spare superblock (blocksize = %d)", + blocksize); + if (swp) + nilfs_swap_super_block(nilfs); + + /* + * Calculate the array index of the older superblock data. + * If one has been dropped, set index 0 pointing to the remaining one, + * otherwise set index 1 pointing to the old one (including if both + * are the same). + * + * Divided case valid[0] valid[1] swp -> older + * ------------------------------------------------------------- + * Both SBs are invalid 0 0 N/A (Error) + * SB1 is invalid 0 1 1 0 + * SB2 is invalid 1 0 0 0 + * SB2 is newer 1 1 1 0 + * SB2 is older or the same 1 1 0 1 + */ + older = valid[1] ^ swp; + + nilfs->ns_sbwcount = 0; + nilfs->ns_sbwtime = le64_to_cpu(sbp[0]->s_wtime); + nilfs->ns_prot_seq = le64_to_cpu(sbp[older]->s_last_seq); + *sbpp = sbp[0]; + return 0; +} + +/** + * init_nilfs - initialize a NILFS instance. + * @nilfs: the_nilfs structure + * @sb: super block + * + * init_nilfs() performs common initialization per block device (e.g. + * reading the super block, getting disk layout information, initializing + * shared fields in the_nilfs). + * + * Return: 0 on success, or a negative error code on failure. + */ +int init_nilfs(struct the_nilfs *nilfs, struct super_block *sb) +{ + struct nilfs_super_block *sbp; + int blocksize; + int err; + + blocksize = sb_min_blocksize(sb, NILFS_MIN_BLOCK_SIZE); + if (!blocksize) { + nilfs_err(sb, "unable to set blocksize"); + err = -EINVAL; + goto out; + } + err = nilfs_load_super_block(nilfs, sb, blocksize, &sbp); + if (err) + goto out; + + err = nilfs_store_magic(sb, sbp); + if (err) + goto failed_sbh; + + err = nilfs_check_feature_compatibility(sb, sbp); + if (err) + goto failed_sbh; + + err = nilfs_get_blocksize(sb, sbp, &blocksize); + if (err) + goto failed_sbh; + + if (blocksize < NILFS_MIN_BLOCK_SIZE) { + nilfs_err(sb, + "couldn't mount because of unsupported filesystem blocksize %d", + blocksize); + err = -EINVAL; + goto failed_sbh; + } + if (sb->s_blocksize != blocksize) { + int hw_blocksize = bdev_logical_block_size(sb->s_bdev); + + if (blocksize < hw_blocksize) { + nilfs_err(sb, + "blocksize %d too small for device (sector-size = %d)", + blocksize, hw_blocksize); + err = -EINVAL; + goto failed_sbh; + } + nilfs_release_super_block(nilfs); + if (!sb_set_blocksize(sb, blocksize)) { + nilfs_err(sb, "bad blocksize %d", blocksize); + err = -EINVAL; + goto out; + } + + err = nilfs_load_super_block(nilfs, sb, blocksize, &sbp); + if (err) + goto out; + /* + * Not to failed_sbh; sbh is released automatically + * when reloading fails. + */ + } + nilfs->ns_blocksize_bits = sb->s_blocksize_bits; + nilfs->ns_blocksize = blocksize; + + err = nilfs_store_disk_layout(nilfs, sbp); + if (err) + goto failed_sbh; + + sb->s_maxbytes = nilfs_max_size(sb->s_blocksize_bits); + + nilfs->ns_mount_state = le16_to_cpu(sbp->s_state); + + err = nilfs_store_log_cursor(nilfs, sbp); + if (err) + goto failed_sbh; + + set_nilfs_init(nilfs); + err = 0; + out: + return err; + + failed_sbh: + nilfs_release_super_block(nilfs); + goto out; +} + +int nilfs_discard_segments(struct the_nilfs *nilfs, __u64 *segnump, + size_t nsegs) +{ + sector_t seg_start, seg_end; + sector_t start = 0, nblocks = 0; + unsigned int sects_per_block; + __u64 *sn; + int ret = 0; + + sects_per_block = (1 << nilfs->ns_blocksize_bits) / + bdev_logical_block_size(nilfs->ns_bdev); + for (sn = segnump; sn < segnump + nsegs; sn++) { + nilfs_get_segment_range(nilfs, *sn, &seg_start, &seg_end); + + if (!nblocks) { + start = seg_start; + nblocks = seg_end - seg_start + 1; + } else if (start + nblocks == seg_start) { + nblocks += seg_end - seg_start + 1; + } else { + ret = blkdev_issue_discard(nilfs->ns_bdev, + start * sects_per_block, + nblocks * sects_per_block, + GFP_NOFS); + if (ret < 0) + return ret; + nblocks = 0; + } + } + if (nblocks) + ret = blkdev_issue_discard(nilfs->ns_bdev, + start * sects_per_block, + nblocks * sects_per_block, + GFP_NOFS); + return ret; +} + +int nilfs_count_free_blocks(struct the_nilfs *nilfs, sector_t *nblocks) +{ + unsigned long ncleansegs; + + ncleansegs = nilfs_sufile_get_ncleansegs(nilfs->ns_sufile); + *nblocks = (sector_t)ncleansegs * nilfs->ns_blocks_per_segment; + return 0; +} + +int nilfs_near_disk_full(struct the_nilfs *nilfs) +{ + unsigned long ncleansegs, nincsegs; + + ncleansegs = nilfs_sufile_get_ncleansegs(nilfs->ns_sufile); + nincsegs = atomic_read(&nilfs->ns_ndirtyblks) / + nilfs->ns_blocks_per_segment + 1; + + return ncleansegs <= nilfs->ns_nrsvsegs + nincsegs; +} + +struct nilfs_root *nilfs_lookup_root(struct the_nilfs *nilfs, __u64 cno) +{ + struct rb_node *n; + struct nilfs_root *root; + + spin_lock(&nilfs->ns_cptree_lock); + n = nilfs->ns_cptree.rb_node; + while (n) { + root = rb_entry(n, struct nilfs_root, rb_node); + + if (cno < root->cno) { + n = n->rb_left; + } else if (cno > root->cno) { + n = n->rb_right; + } else { + refcount_inc(&root->count); + spin_unlock(&nilfs->ns_cptree_lock); + return root; + } + } + spin_unlock(&nilfs->ns_cptree_lock); + + return NULL; +} + +struct nilfs_root * +nilfs_find_or_create_root(struct the_nilfs *nilfs, __u64 cno) +{ + struct rb_node **p, *parent; + struct nilfs_root *root, *new; + int err; + + root = nilfs_lookup_root(nilfs, cno); + if (root) + return root; + + new = kzalloc_obj(*root); + if (!new) + return NULL; + + spin_lock(&nilfs->ns_cptree_lock); + + p = &nilfs->ns_cptree.rb_node; + parent = NULL; + + while (*p) { + parent = *p; + root = rb_entry(parent, struct nilfs_root, rb_node); + + if (cno < root->cno) { + p = &(*p)->rb_left; + } else if (cno > root->cno) { + p = &(*p)->rb_right; + } else { + refcount_inc(&root->count); + spin_unlock(&nilfs->ns_cptree_lock); + kfree(new); + return root; + } + } + + new->cno = cno; + new->ifile = NULL; + new->nilfs = nilfs; + refcount_set(&new->count, 1); + atomic64_set(&new->inodes_count, 0); + atomic64_set(&new->blocks_count, 0); + + rb_link_node(&new->rb_node, parent, p); + rb_insert_color(&new->rb_node, &nilfs->ns_cptree); + + spin_unlock(&nilfs->ns_cptree_lock); + + err = nilfs_sysfs_create_snapshot_group(new); + if (err) { + kfree(new); + new = NULL; + } + + return new; +} + +void nilfs_put_root(struct nilfs_root *root) +{ + struct the_nilfs *nilfs = root->nilfs; + + if (refcount_dec_and_lock(&root->count, &nilfs->ns_cptree_lock)) { + rb_erase(&root->rb_node, &nilfs->ns_cptree); + spin_unlock(&nilfs->ns_cptree_lock); + + nilfs_sysfs_delete_snapshot_group(root); + iput(root->ifile); + + kfree(root); + } +} diff --git a/fs/nilfs2/the_nilfs.h b/fs/nilfs2/the_nilfs.h new file mode 100644 index 000000000..4776a70f0 --- /dev/null +++ b/fs/nilfs2/the_nilfs.h @@ -0,0 +1,376 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * the_nilfs shared structure. + * + * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation. + * + * Written by Ryusuke Konishi. + * + */ + +#ifndef _THE_NILFS_H +#define _THE_NILFS_H + +#include +#include +#include +#include +#include +#include +#include +#include + +struct nilfs_sc_info; +struct nilfs_sysfs_dev_subgroups; + +/* the_nilfs struct */ +enum { + THE_NILFS_INIT = 0, /* Information from super_block is set */ + THE_NILFS_DISCONTINUED, /* 'next' pointer chain has broken */ + THE_NILFS_GC_RUNNING, /* gc process is running */ + THE_NILFS_SB_DIRTY, /* super block is dirty */ + THE_NILFS_PURGING, /* disposing dirty files for cleanup */ +}; + +/** + * struct the_nilfs - struct to supervise multiple nilfs mount points + * @ns_flags: flags + * @ns_flushed_device: flag indicating if all volatile data was flushed + * @ns_sb: back pointer to super block instance + * @ns_bdev: block device + * @ns_sem: semaphore for shared states + * @ns_snapshot_mount_mutex: mutex to protect snapshot mounts + * @ns_sbh: buffer heads of on-disk super blocks + * @ns_sbp: pointers to super block data + * @ns_sbwtime: previous write time of super block + * @ns_sbwcount: write count of super block + * @ns_sbsize: size of valid data in super block + * @ns_mount_state: file system state + * @ns_sb_update_freq: interval of periodical update of superblocks (in seconds) + * @ns_seg_seq: segment sequence counter + * @ns_segnum: index number of the latest full segment. + * @ns_nextnum: index number of the full segment index to be used next + * @ns_pseg_offset: offset of next partial segment in the current full segment + * @ns_cno: next checkpoint number + * @ns_ctime: write time of the last segment + * @ns_nongc_ctime: write time of the last segment not for cleaner operation + * @ns_ndirtyblks: Number of dirty data blocks + * @ns_last_segment_lock: lock protecting fields for the latest segment + * @ns_last_pseg: start block number of the latest segment + * @ns_last_seq: sequence value of the latest segment + * @ns_last_cno: checkpoint number of the latest segment + * @ns_prot_seq: least sequence number of segments which must not be reclaimed + * @ns_prev_seq: base sequence number used to decide if advance log cursor + * @ns_writer: log writer + * @ns_segctor_sem: semaphore protecting log write + * @ns_dat: DAT file inode + * @ns_cpfile: checkpoint file inode + * @ns_sufile: segusage file inode + * @ns_cptree: rb-tree of all mounted checkpoints (nilfs_root) + * @ns_cptree_lock: lock protecting @ns_cptree + * @ns_dirty_files: list of dirty files + * @ns_inode_lock: lock protecting @ns_dirty_files + * @ns_gc_inodes: dummy inodes to keep live blocks + * @ns_mount_opt: mount options + * @ns_resuid: uid for reserved blocks + * @ns_resgid: gid for reserved blocks + * @ns_interval: checkpoint creation interval + * @ns_watermark: watermark for the number of dirty buffers + * @ns_blocksize_bits: bit length of block size + * @ns_blocksize: block size + * @ns_nsegments: number of segments in filesystem + * @ns_blocks_per_segment: number of blocks per segment + * @ns_r_segments_percentage: reserved segments percentage + * @ns_nrsvsegs: number of reserved segments + * @ns_first_data_block: block number of first data block + * @ns_inode_size: size of on-disk inode + * @ns_first_ino: first not-special inode number + * @ns_crc_seed: seed value of CRC32 calculation + * @ns_dev_kobj: /sys/fs// + * @ns_dev_kobj_unregister: completion state + * @ns_dev_subgroups: subgroups pointer + */ +struct the_nilfs { + unsigned long ns_flags; + int ns_flushed_device; + + struct super_block *ns_sb; + struct block_device *ns_bdev; + struct rw_semaphore ns_sem; + struct mutex ns_snapshot_mount_mutex; + + /* + * used for + * - loading the latest checkpoint exclusively. + * - allocating a new full segment. + */ + struct buffer_head *ns_sbh[2]; + struct nilfs_super_block *ns_sbp[2]; + time64_t ns_sbwtime; + unsigned int ns_sbwcount; + unsigned int ns_sbsize; + unsigned int ns_mount_state; + unsigned int ns_sb_update_freq; + + /* + * The following fields are updated by a writable FS-instance. + * These fields are protected by ns_segctor_sem outside load_nilfs(). + */ + u64 ns_seg_seq; + __u64 ns_segnum; + __u64 ns_nextnum; + unsigned long ns_pseg_offset; + __u64 ns_cno; + time64_t ns_ctime; + time64_t ns_nongc_ctime; + atomic_t ns_ndirtyblks; + + /* + * The following fields hold information on the latest partial segment + * written to disk with a super root. These fields are protected by + * ns_last_segment_lock. + */ + spinlock_t ns_last_segment_lock; + sector_t ns_last_pseg; + u64 ns_last_seq; + __u64 ns_last_cno; + u64 ns_prot_seq; + u64 ns_prev_seq; + + struct nilfs_sc_info *ns_writer; + struct rw_semaphore ns_segctor_sem; + + /* + * Following fields are lock free except for the period before + * the_nilfs is initialized. + */ + struct inode *ns_dat; + struct inode *ns_cpfile; + struct inode *ns_sufile; + + /* Checkpoint tree */ + struct rb_root ns_cptree; + spinlock_t ns_cptree_lock; + + /* Dirty inode list */ + struct list_head ns_dirty_files; + spinlock_t ns_inode_lock; + + /* GC inode list */ + struct list_head ns_gc_inodes; + + /* Mount options */ + unsigned long ns_mount_opt; + + uid_t ns_resuid; + gid_t ns_resgid; + unsigned long ns_interval; + unsigned long ns_watermark; + + /* Disk layout information (static) */ + unsigned int ns_blocksize_bits; + unsigned int ns_blocksize; + unsigned long ns_nsegments; + unsigned long ns_blocks_per_segment; + unsigned long ns_r_segments_percentage; + unsigned long ns_nrsvsegs; + unsigned long ns_first_data_block; + int ns_inode_size; + unsigned int ns_first_ino; + u32 ns_crc_seed; + + /* /sys/fs// */ + struct kobject ns_dev_kobj; + struct completion ns_dev_kobj_unregister; + struct nilfs_sysfs_dev_subgroups *ns_dev_subgroups; +}; + +#define THE_NILFS_FNS(bit, name) \ +static inline void set_nilfs_##name(struct the_nilfs *nilfs) \ +{ \ + set_bit(THE_NILFS_##bit, &(nilfs)->ns_flags); \ +} \ +static inline void clear_nilfs_##name(struct the_nilfs *nilfs) \ +{ \ + clear_bit(THE_NILFS_##bit, &(nilfs)->ns_flags); \ +} \ +static inline int nilfs_##name(struct the_nilfs *nilfs) \ +{ \ + return test_bit(THE_NILFS_##bit, &(nilfs)->ns_flags); \ +} + +THE_NILFS_FNS(INIT, init) +THE_NILFS_FNS(DISCONTINUED, discontinued) +THE_NILFS_FNS(GC_RUNNING, gc_running) +THE_NILFS_FNS(SB_DIRTY, sb_dirty) +THE_NILFS_FNS(PURGING, purging) + +/* + * Mount option operations + */ +#define nilfs_clear_opt(nilfs, opt) \ + ((nilfs)->ns_mount_opt &= ~NILFS_MOUNT_##opt) +#define nilfs_set_opt(nilfs, opt) \ + ((nilfs)->ns_mount_opt |= NILFS_MOUNT_##opt) +#define nilfs_test_opt(nilfs, opt) ((nilfs)->ns_mount_opt & NILFS_MOUNT_##opt) + +/** + * struct nilfs_root - nilfs root object + * @cno: checkpoint number + * @rb_node: red-black tree node + * @count: refcount of this structure + * @nilfs: nilfs object + * @ifile: inode file + * @inodes_count: number of inodes + * @blocks_count: number of blocks + * @snapshot_kobj: /sys/fs///mounted_snapshots/ + * @snapshot_kobj_unregister: completion state for kernel object + */ +struct nilfs_root { + __u64 cno; + struct rb_node rb_node; + + refcount_t count; + struct the_nilfs *nilfs; + struct inode *ifile; + + atomic64_t inodes_count; + atomic64_t blocks_count; + + /* /sys/fs///mounted_snapshots/ */ + struct kobject snapshot_kobj; + struct completion snapshot_kobj_unregister; +}; + +/* Special checkpoint number */ +#define NILFS_CPTREE_CURRENT_CNO 0 + +/* Minimum interval of periodical update of superblocks (in seconds) */ +#define NILFS_SB_FREQ 10 + +static inline int nilfs_sb_need_update(struct the_nilfs *nilfs) +{ + u64 t = ktime_get_real_seconds(); + + return t < nilfs->ns_sbwtime || + t > nilfs->ns_sbwtime + nilfs->ns_sb_update_freq; +} + +static inline int nilfs_sb_will_flip(struct the_nilfs *nilfs) +{ + int flip_bits = nilfs->ns_sbwcount & 0x0FL; + + return (flip_bits != 0x08 && flip_bits != 0x0F); +} + +void nilfs_set_last_segment(struct the_nilfs *, sector_t, u64, __u64); +struct the_nilfs *alloc_nilfs(struct super_block *sb); +void destroy_nilfs(struct the_nilfs *nilfs); +int init_nilfs(struct the_nilfs *nilfs, struct super_block *sb); +int load_nilfs(struct the_nilfs *nilfs, struct super_block *sb); +unsigned long nilfs_nrsvsegs(struct the_nilfs *nilfs, unsigned long nsegs); +void nilfs_set_nsegments(struct the_nilfs *nilfs, unsigned long nsegs); +int nilfs_discard_segments(struct the_nilfs *, __u64 *, size_t); +int nilfs_count_free_blocks(struct the_nilfs *, sector_t *); +struct nilfs_root *nilfs_lookup_root(struct the_nilfs *nilfs, __u64 cno); +struct nilfs_root *nilfs_find_or_create_root(struct the_nilfs *nilfs, + __u64 cno); +void nilfs_put_root(struct nilfs_root *root); +int nilfs_near_disk_full(struct the_nilfs *); +void nilfs_fall_back_super_block(struct the_nilfs *); +void nilfs_swap_super_block(struct the_nilfs *); + + +static inline void nilfs_get_root(struct nilfs_root *root) +{ + refcount_inc(&root->count); +} + +static inline int nilfs_valid_fs(struct the_nilfs *nilfs) +{ + unsigned int valid_fs; + + down_read(&nilfs->ns_sem); + valid_fs = (nilfs->ns_mount_state & NILFS_VALID_FS); + up_read(&nilfs->ns_sem); + return valid_fs; +} + +static inline void +nilfs_get_segment_range(struct the_nilfs *nilfs, __u64 segnum, + sector_t *seg_start, sector_t *seg_end) +{ + *seg_start = (sector_t)nilfs->ns_blocks_per_segment * segnum; + *seg_end = *seg_start + nilfs->ns_blocks_per_segment - 1; + if (segnum == 0) + *seg_start = nilfs->ns_first_data_block; +} + +static inline sector_t +nilfs_get_segment_start_blocknr(struct the_nilfs *nilfs, __u64 segnum) +{ + return (segnum == 0) ? nilfs->ns_first_data_block : + (sector_t)nilfs->ns_blocks_per_segment * segnum; +} + +static inline __u64 +nilfs_get_segnum_of_block(struct the_nilfs *nilfs, sector_t blocknr) +{ + sector_t segnum = blocknr; + + sector_div(segnum, nilfs->ns_blocks_per_segment); + return segnum; +} + +static inline void +nilfs_terminate_segment(struct the_nilfs *nilfs, sector_t seg_start, + sector_t seg_end) +{ + /* terminate the current full segment (used in case of I/O-error) */ + nilfs->ns_pseg_offset = seg_end - seg_start + 1; +} + +static inline void nilfs_shift_to_next_segment(struct the_nilfs *nilfs) +{ + /* move forward with a full segment */ + nilfs->ns_segnum = nilfs->ns_nextnum; + nilfs->ns_pseg_offset = 0; + nilfs->ns_seg_seq++; +} + +static inline __u64 nilfs_last_cno(struct the_nilfs *nilfs) +{ + __u64 cno; + + spin_lock(&nilfs->ns_last_segment_lock); + cno = nilfs->ns_last_cno; + spin_unlock(&nilfs->ns_last_segment_lock); + return cno; +} + +static inline int nilfs_segment_is_active(struct the_nilfs *nilfs, __u64 n) +{ + return n == nilfs->ns_segnum || n == nilfs->ns_nextnum; +} + +static inline int nilfs_flush_device(struct the_nilfs *nilfs) +{ + int err; + + if (!nilfs_test_opt(nilfs, BARRIER) || nilfs->ns_flushed_device) + return 0; + + nilfs->ns_flushed_device = 1; + /* + * the store to ns_flushed_device must not be reordered after + * blkdev_issue_flush(). + */ + smp_wmb(); + + err = blkdev_issue_flush(nilfs->ns_bdev); + if (err != -EIO) + err = 0; + return err; +} + +#endif /* _THE_NILFS_H */ diff --git a/fs/nls/Kconfig b/fs/nls/Kconfig new file mode 100644 index 000000000..2a601af6f --- /dev/null +++ b/fs/nls/Kconfig @@ -0,0 +1,623 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Native language support configuration +# + +menuconfig NLS + tristate "Native language support" + help + The base Native Language Support. A number of filesystems + depend on it (e.g. FAT, JOLIET, NT, BEOS filesystems), as well + as the ability of some filesystems to use native languages + (NCP, SMB). + + If unsure, say Y. + + To compile this code as a module, choose M here: the module + will be called nls_base. + +if NLS + +config NLS_DEFAULT + string "Default NLS Option" + default "iso8859-1" + help + The default NLS used when mounting file system. Note, that this is + the NLS used by your console, not the NLS used by a specific file + system (if different) to store data (filenames) on a disk. + Currently, the valid values are: + big5, cp437, cp737, cp775, cp850, cp852, cp855, cp857, cp860, cp861, + cp862, cp863, cp864, cp865, cp866, cp869, cp874, cp932, cp936, + cp949, cp950, cp1251, cp1255, euc-jp, euc-kr, gb2312, iso8859-1, + iso8859-2, iso8859-3, iso8859-4, iso8859-5, iso8859-6, iso8859-7, + iso8859-8, iso8859-9, iso8859-13, iso8859-14, iso8859-15, + koi8-r, koi8-ru, koi8-u, sjis, tis-620, macroman, utf8. + If you specify a wrong value, it will use the built-in NLS; + compatible with iso8859-1. + + If unsure, specify it as "iso8859-1". + +config NLS_CODEPAGE_437 + tristate "Codepage 437 (United States, Canada)" + help + The Microsoft FAT file system family can deal with filenames in + native language character sets. These character sets are stored + in so-called DOS codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + DOS/Windows partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the DOS codepage that is used in + the United States and parts of Canada. This is recommended. + +config NLS_CODEPAGE_737 + tristate "Codepage 737 (Greek)" + help + The Microsoft FAT file system family can deal with filenames in + native language character sets. These character sets are stored + in so-called DOS codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + DOS/Windows partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the DOS codepage that is used for + Greek. If unsure, say N. + +config NLS_CODEPAGE_775 + tristate "Codepage 775 (Baltic Rim)" + help + The Microsoft FAT file system family can deal with filenames in + native language character sets. These character sets are stored + in so-called DOS codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + DOS/Windows partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the DOS codepage that is used + for the Baltic Rim Languages (Latvian and Lithuanian). If unsure, + say N. + +config NLS_CODEPAGE_850 + tristate "Codepage 850 (Europe)" + help + The Microsoft FAT file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called DOS codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + DOS/Windows partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the DOS codepage that is used for + much of Europe -- United Kingdom, Germany, Spain, Italy, and [add + more countries here]. It has some characters useful to many European + languages that are not part of the US codepage 437. + + If unsure, say Y. + +config NLS_CODEPAGE_852 + tristate "Codepage 852 (Central/Eastern Europe)" + help + The Microsoft FAT file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called DOS codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + DOS/Windows partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the Latin 2 codepage used by DOS + for much of Central and Eastern Europe. It has all the required + characters for these languages: Albanian, Croatian, Czech, English, + Finnish, Hungarian, Irish, German, Polish, Romanian, Serbian (Latin + transcription), Slovak, Slovenian, and Sorbian. + +config NLS_CODEPAGE_855 + tristate "Codepage 855 (Cyrillic)" + help + The Microsoft FAT file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called DOS codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + DOS/Windows partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the DOS codepage for Cyrillic. + +config NLS_CODEPAGE_857 + tristate "Codepage 857 (Turkish)" + help + The Microsoft FAT file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called DOS codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + DOS/Windows partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the DOS codepage for Turkish. + +config NLS_CODEPAGE_860 + tristate "Codepage 860 (Portuguese)" + help + The Microsoft FAT file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called DOS codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + DOS/Windows partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the DOS codepage for Portuguese. + +config NLS_CODEPAGE_861 + tristate "Codepage 861 (Icelandic)" + help + The Microsoft FAT file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called DOS codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + DOS/Windows partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the DOS codepage for Icelandic. + +config NLS_CODEPAGE_862 + tristate "Codepage 862 (Hebrew)" + help + The Microsoft FAT file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called DOS codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + DOS/Windows partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the DOS codepage for Hebrew. + +config NLS_CODEPAGE_863 + tristate "Codepage 863 (Canadian French)" + help + The Microsoft FAT file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called DOS codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + DOS/Windows partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the DOS codepage for Canadian + French. + +config NLS_CODEPAGE_864 + tristate "Codepage 864 (Arabic)" + help + The Microsoft FAT file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called DOS codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + DOS/Windows partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the DOS codepage for Arabic. + +config NLS_CODEPAGE_865 + tristate "Codepage 865 (Norwegian, Danish)" + help + The Microsoft FAT file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called DOS codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + DOS/Windows partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the DOS codepage for the Nordic + European countries. + +config NLS_CODEPAGE_866 + tristate "Codepage 866 (Cyrillic/Russian)" + help + The Microsoft FAT file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called DOS codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + DOS/Windows partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the DOS codepage for + Cyrillic/Russian. + +config NLS_CODEPAGE_869 + tristate "Codepage 869 (Greek)" + help + The Microsoft FAT file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called DOS codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + DOS/Windows partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the DOS codepage for Greek. + +config NLS_CODEPAGE_936 + tristate "Simplified Chinese charset (CP936, GB2312)" + help + The Microsoft FAT file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called DOS codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + DOS/Windows partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the DOS codepage for Simplified + Chinese(GBK). + +config NLS_CODEPAGE_950 + tristate "Traditional Chinese charset (Big5)" + help + The Microsoft FAT file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called DOS codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + DOS/Windows partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the DOS codepage for Traditional + Chinese(Big5). + +config NLS_CODEPAGE_932 + tristate "Japanese charsets (Shift-JIS, EUC-JP)" + help + The Microsoft FAT file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called DOS codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + DOS/Windows partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the DOS codepage for Shift-JIS + or EUC-JP. To use EUC-JP, you can use 'euc-jp' as mount option or + NLS Default value during kernel configuration, instead of 'cp932'. + +config NLS_CODEPAGE_949 + tristate "Korean charset (CP949, EUC-KR)" + help + The Microsoft FAT file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called DOS codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + DOS/Windows partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the DOS codepage for UHC. + +config NLS_CODEPAGE_874 + tristate "Thai charset (CP874, TIS-620)" + help + The Microsoft FAT file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called DOS codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + DOS/Windows partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the DOS codepage for Thai. + +config NLS_ISO8859_8 + tristate "Hebrew charsets (ISO-8859-8, CP1255)" + help + If you want to display filenames with native language characters + from the Microsoft FAT file system family or from JOLIET CD-ROMs + correctly on the screen, you need to include the appropriate + input/output character sets. Say Y here for ISO8859-8, the Hebrew + character set. + +config NLS_CODEPAGE_1250 + tristate "Windows CP1250 (Slavic/Central European Languages)" + help + If you want to display filenames with native language characters + from the Microsoft FAT file system family or from JOLIET CDROMs + correctly on the screen, you need to include the appropriate + input/output character sets. Say Y here for the Windows CP-1250 + character set, which works for most Latin-written Slavic and Central + European languages: Czech, German, Hungarian, Polish, Rumanian, Croatian, + Slovak, Slovene. + +config NLS_CODEPAGE_1251 + tristate "Windows CP1251 (Bulgarian, Belarusian)" + help + The Microsoft FAT file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called DOS codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + DOS/Windows partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the DOS codepage for Russian and + Bulgarian and Belarusian. + +config NLS_ASCII + tristate "ASCII (United States)" + help + An ASCII NLS module is needed if you want to override the + DEFAULT NLS with this very basic charset and don't want any + non-ASCII characters to be translated. + +config NLS_ISO8859_1 + tristate "NLS ISO 8859-1 (Latin 1; Western European Languages)" + help + If you want to display filenames with native language characters + from the Microsoft FAT file system family or from JOLIET CD-ROMs + correctly on the screen, you need to include the appropriate + input/output character sets. Say Y here for the Latin 1 character + set, which covers most West European languages such as Albanian, + Catalan, Danish, Dutch, English, Faeroese, Finnish, French, German, + Galician, Irish, Icelandic, Italian, Norwegian, Portuguese, Spanish, + and Swedish. It is also the default for the US. If unsure, say Y. + +config NLS_ISO8859_2 + tristate "NLS ISO 8859-2 (Latin 2; Slavic/Central European Languages)" + help + If you want to display filenames with native language characters + from the Microsoft FAT file system family or from JOLIET CD-ROMs + correctly on the screen, you need to include the appropriate + input/output character sets. Say Y here for the Latin 2 character + set, which works for most Latin-written Slavic and Central European + languages: Czech, German, Hungarian, Polish, Rumanian, Croatian, + Slovak, Slovene. + +config NLS_ISO8859_3 + tristate "NLS ISO 8859-3 (Latin 3; Esperanto, Galician, Maltese, Turkish)" + help + If you want to display filenames with native language characters + from the Microsoft FAT file system family or from JOLIET CD-ROMs + correctly on the screen, you need to include the appropriate + input/output character sets. Say Y here for the Latin 3 character + set, which is popular with authors of Esperanto, Galician, Maltese, + and Turkish. + +config NLS_ISO8859_4 + tristate "NLS ISO 8859-4 (Latin 4; old Baltic charset)" + help + If you want to display filenames with native language characters + from the Microsoft FAT file system family or from JOLIET CD-ROMs + correctly on the screen, you need to include the appropriate + input/output character sets. Say Y here for the Latin 4 character + set which introduces letters for Estonian, Latvian, and + Lithuanian. It is an incomplete predecessor of Latin 7. + +config NLS_ISO8859_5 + tristate "NLS ISO 8859-5 (Cyrillic)" + help + If you want to display filenames with native language characters + from the Microsoft FAT file system family or from JOLIET CD-ROMs + correctly on the screen, you need to include the appropriate + input/output character sets. Say Y here for ISO8859-5, a Cyrillic + character set with which you can type Bulgarian, Belarusian, + Macedonian, Russian, Serbian, and Ukrainian. Note that the charset + KOI8-R is preferred in Russia. + +config NLS_ISO8859_6 + tristate "NLS ISO 8859-6 (Arabic)" + help + If you want to display filenames with native language characters + from the Microsoft FAT file system family or from JOLIET CD-ROMs + correctly on the screen, you need to include the appropriate + input/output character sets. Say Y here for ISO8859-6, the Arabic + character set. + +config NLS_ISO8859_7 + tristate "NLS ISO 8859-7 (Modern Greek)" + help + If you want to display filenames with native language characters + from the Microsoft FAT file system family or from JOLIET CD-ROMs + correctly on the screen, you need to include the appropriate + input/output character sets. Say Y here for ISO8859-7, the Modern + Greek character set. + +config NLS_ISO8859_9 + tristate "NLS ISO 8859-9 (Latin 5; Turkish)" + help + If you want to display filenames with native language characters + from the Microsoft FAT file system family or from JOLIET CD-ROMs + correctly on the screen, you need to include the appropriate + input/output character sets. Say Y here for the Latin 5 character + set, and it replaces the rarely needed Icelandic letters in Latin 1 + with the Turkish ones. Useful in Turkey. + +config NLS_ISO8859_13 + tristate "NLS ISO 8859-13 (Latin 7; Baltic)" + help + If you want to display filenames with native language characters + from the Microsoft FAT file system family or from JOLIET CD-ROMs + correctly on the screen, you need to include the appropriate + input/output character sets. Say Y here for the Latin 7 character + set, which supports modern Baltic languages including Latvian + and Lithuanian. + +config NLS_ISO8859_14 + tristate "NLS ISO 8859-14 (Latin 8; Celtic)" + help + If you want to display filenames with native language characters + from the Microsoft FAT file system family or from JOLIET CD-ROMs + correctly on the screen, you need to include the appropriate + input/output character sets. Say Y here for the Latin 8 character + set, which adds the last accented vowels for Welsh (aka Cymraeg) + (and Manx Gaelic) that were missing in Latin 1. + has further information. + +config NLS_ISO8859_15 + tristate "NLS ISO 8859-15 (Latin 9; Western European Languages with Euro)" + help + If you want to display filenames with native language characters + from the Microsoft FAT file system family or from JOLIET CD-ROMs + correctly on the screen, you need to include the appropriate + input/output character sets. Say Y here for the Latin 9 character + set, which covers most West European languages such as Albanian, + Catalan, Danish, Dutch, English, Estonian, Faeroese, Finnish, + French, German, Galician, Irish, Icelandic, Italian, Norwegian, + Portuguese, Spanish, and Swedish. Latin 9 is an update to + Latin 1 (ISO 8859-1) that removes a handful of rarely used + characters and instead adds support for Estonian, corrects the + support for French and Finnish, and adds the new Euro character. + If unsure, say Y. + +config NLS_KOI8_R + tristate "NLS KOI8-R (Russian)" + help + If you want to display filenames with native language characters + from the Microsoft FAT file system family or from JOLIET CD-ROMs + correctly on the screen, you need to include the appropriate + input/output character sets. Say Y here for the preferred Russian + character set. + +config NLS_KOI8_U + tristate "NLS KOI8-U/RU (Ukrainian, Belarusian)" + help + If you want to display filenames with native language characters + from the Microsoft FAT file system family or from JOLIET CD-ROMs + correctly on the screen, you need to include the appropriate + input/output character sets. Say Y here for the preferred Ukrainian + (koi8-u) and Belarusian (koi8-ru) character sets. + +config NLS_MAC_ROMAN + tristate "Codepage macroman" + help + The Apple HFS file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called MAC codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + Mac partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the Mac codepage that is used for + much of Europe -- United Kingdom, Germany, Spain, Italy, and [add + more countries here]. + + If unsure, say Y. + +config NLS_MAC_CELTIC + tristate "Codepage macceltic" + help + The Apple HFS file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called MAC codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + Mac partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the Mac codepage that is used for + Celtic. + + If unsure, say Y. + +config NLS_MAC_CENTEURO + tristate "Codepage maccenteuro" + help + The Apple HFS file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called MAC codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + Mac partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the Mac codepage that is used for + Central Europe. + + If unsure, say Y. + +config NLS_MAC_CROATIAN + tristate "Codepage maccroatian" + help + The Apple HFS file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called MAC codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + Mac partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the Mac codepage that is used for + Croatian. + + If unsure, say Y. + +config NLS_MAC_CYRILLIC + tristate "Codepage maccyrillic" + help + The Apple HFS file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called MAC codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + Mac partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the Mac codepage that is used for + Cyrillic. + + If unsure, say Y. + +config NLS_MAC_GAELIC + tristate "Codepage macgaelic" + help + The Apple HFS file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called MAC codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + Mac partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the Mac codepage that is used for + Gaelic. + + If unsure, say Y. + +config NLS_MAC_GREEK + tristate "Codepage macgreek" + help + The Apple HFS file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called MAC codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + Mac partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the Mac codepage that is used for + Greek. + + If unsure, say Y. + +config NLS_MAC_ICELAND + tristate "Codepage maciceland" + help + The Apple HFS file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called MAC codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + Mac partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the Mac codepage that is used for + Iceland. + + If unsure, say Y. + +config NLS_MAC_INUIT + tristate "Codepage macinuit" + help + The Apple HFS file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called MAC codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + Mac partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the Mac codepage that is used for + Inuit. + + If unsure, say Y. + +config NLS_MAC_ROMANIAN + tristate "Codepage macromanian" + help + The Apple HFS file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called MAC codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + Mac partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the Mac codepage that is used for + Romanian. + + If unsure, say Y. + +config NLS_MAC_TURKISH + tristate "Codepage macturkish" + help + The Apple HFS file system family can deal with filenames in + native language character sets. These character sets are stored in + so-called MAC codepages. You need to include the appropriate + codepage if you want to be able to read/write these filenames on + Mac partitions correctly. This does apply to the filenames + only, not to the file contents. You can include several codepages; + say Y here if you want to include the Mac codepage that is used for + Turkish. + + If unsure, say Y. + +config NLS_UTF8 + tristate "NLS UTF-8" + help + If you want to display filenames with native language characters + from the Microsoft FAT file system family or from JOLIET CD-ROMs + correctly on the screen, you need to include the appropriate + input/output character sets. Say Y here for the UTF-8 encoding of + the Unicode/ISO9646 universal character set. + +config NLS_UCS2_UTILS + tristate + +endif # NLS diff --git a/fs/nls/Makefile b/fs/nls/Makefile new file mode 100644 index 000000000..5062c699d --- /dev/null +++ b/fs/nls/Makefile @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for native language support +# + +obj-$(CONFIG_NLS) += nls_base.o + +obj-$(CONFIG_NLS_CODEPAGE_437) += nls_cp437.o +obj-$(CONFIG_NLS_CODEPAGE_737) += nls_cp737.o +obj-$(CONFIG_NLS_CODEPAGE_775) += nls_cp775.o +obj-$(CONFIG_NLS_CODEPAGE_850) += nls_cp850.o +obj-$(CONFIG_NLS_CODEPAGE_852) += nls_cp852.o +obj-$(CONFIG_NLS_CODEPAGE_855) += nls_cp855.o +obj-$(CONFIG_NLS_CODEPAGE_857) += nls_cp857.o +obj-$(CONFIG_NLS_CODEPAGE_860) += nls_cp860.o +obj-$(CONFIG_NLS_CODEPAGE_861) += nls_cp861.o +obj-$(CONFIG_NLS_CODEPAGE_862) += nls_cp862.o +obj-$(CONFIG_NLS_CODEPAGE_863) += nls_cp863.o +obj-$(CONFIG_NLS_CODEPAGE_864) += nls_cp864.o +obj-$(CONFIG_NLS_CODEPAGE_865) += nls_cp865.o +obj-$(CONFIG_NLS_CODEPAGE_866) += nls_cp866.o +obj-$(CONFIG_NLS_CODEPAGE_869) += nls_cp869.o +obj-$(CONFIG_NLS_CODEPAGE_874) += nls_cp874.o +obj-$(CONFIG_NLS_CODEPAGE_932) += nls_cp932.o nls_euc-jp.o +obj-$(CONFIG_NLS_CODEPAGE_936) += nls_cp936.o +obj-$(CONFIG_NLS_CODEPAGE_949) += nls_cp949.o +obj-$(CONFIG_NLS_CODEPAGE_950) += nls_cp950.o +obj-$(CONFIG_NLS_CODEPAGE_1250) += nls_cp1250.o +obj-$(CONFIG_NLS_CODEPAGE_1251) += nls_cp1251.o +obj-$(CONFIG_NLS_ASCII) += nls_ascii.o +obj-$(CONFIG_NLS_ISO8859_1) += nls_iso8859-1.o +obj-$(CONFIG_NLS_ISO8859_2) += nls_iso8859-2.o +obj-$(CONFIG_NLS_ISO8859_3) += nls_iso8859-3.o +obj-$(CONFIG_NLS_ISO8859_4) += nls_iso8859-4.o +obj-$(CONFIG_NLS_ISO8859_5) += nls_iso8859-5.o +obj-$(CONFIG_NLS_ISO8859_6) += nls_iso8859-6.o +obj-$(CONFIG_NLS_ISO8859_7) += nls_iso8859-7.o +obj-$(CONFIG_NLS_ISO8859_8) += nls_cp1255.o +obj-$(CONFIG_NLS_ISO8859_9) += nls_iso8859-9.o +obj-$(CONFIG_NLS_ISO8859_13) += nls_iso8859-13.o +obj-$(CONFIG_NLS_ISO8859_14) += nls_iso8859-14.o +obj-$(CONFIG_NLS_ISO8859_15) += nls_iso8859-15.o +obj-$(CONFIG_NLS_KOI8_R) += nls_koi8-r.o +obj-$(CONFIG_NLS_KOI8_U) += nls_koi8-u.o nls_koi8-ru.o +obj-$(CONFIG_NLS_UTF8) += nls_utf8.o +obj-$(CONFIG_NLS_MAC_CELTIC) += mac-celtic.o +obj-$(CONFIG_NLS_MAC_CENTEURO) += mac-centeuro.o +obj-$(CONFIG_NLS_MAC_CROATIAN) += mac-croatian.o +obj-$(CONFIG_NLS_MAC_CYRILLIC) += mac-cyrillic.o +obj-$(CONFIG_NLS_MAC_GAELIC) += mac-gaelic.o +obj-$(CONFIG_NLS_MAC_GREEK) += mac-greek.o +obj-$(CONFIG_NLS_MAC_ICELAND) += mac-iceland.o +obj-$(CONFIG_NLS_MAC_INUIT) += mac-inuit.o +obj-$(CONFIG_NLS_MAC_ROMANIAN) += mac-romanian.o +obj-$(CONFIG_NLS_MAC_ROMAN) += mac-roman.o +obj-$(CONFIG_NLS_MAC_TURKISH) += mac-turkish.o +obj-$(CONFIG_NLS_UCS2_UTILS) += nls_ucs2_utils.o diff --git a/fs/nls/mac-celtic.c b/fs/nls/mac-celtic.c new file mode 100644 index 000000000..2963f3299 --- /dev/null +++ b/fs/nls/mac-celtic.c @@ -0,0 +1,602 @@ +/* + * linux/fs/nls/mac-celtic.c + * + * Charset macceltic translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +/* + * COPYRIGHT AND PERMISSION NOTICE + * + * Copyright 1991-2012 Unicode, Inc. All rights reserved. Distributed under + * the Terms of Use in http://www.unicode.org/copyright.html. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of the Unicode data files and any associated documentation (the "Data + * Files") or Unicode software and any associated documentation (the + * "Software") to deal in the Data Files or Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, and/or sell copies of the Data Files or Software, and + * to permit persons to whom the Data Files or Software are furnished to do + * so, provided that (a) the above copyright notice(s) and this permission + * notice appear with all copies of the Data Files or Software, (b) both the + * above copyright notice(s) and this permission notice appear in associated + * documentation, and (c) there is clear notice in each modified Data File or + * in the Software as well as in the documentation associated with the Data + * File(s) or Software that the data or software has been modified. + * + * THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF + * THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS + * INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT + * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF + * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THE DATA FILES OR SOFTWARE. + * + * Except as contained in this notice, the name of a copyright holder shall + * not be used in advertising or otherwise to promote the sale, use or other + * dealings in these Data Files or Software without prior written + * authorization of the copyright holder. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00 */ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10 */ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20 */ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30 */ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40 */ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50 */ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60 */ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70 */ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80 */ + 0x00c4, 0x00c5, 0x00c7, 0x00c9, + 0x00d1, 0x00d6, 0x00dc, 0x00e1, + 0x00e0, 0x00e2, 0x00e4, 0x00e3, + 0x00e5, 0x00e7, 0x00e9, 0x00e8, + /* 0x90 */ + 0x00ea, 0x00eb, 0x00ed, 0x00ec, + 0x00ee, 0x00ef, 0x00f1, 0x00f3, + 0x00f2, 0x00f4, 0x00f6, 0x00f5, + 0x00fa, 0x00f9, 0x00fb, 0x00fc, + /* 0xa0 */ + 0x2020, 0x00b0, 0x00a2, 0x00a3, + 0x00a7, 0x2022, 0x00b6, 0x00df, + 0x00ae, 0x00a9, 0x2122, 0x00b4, + 0x00a8, 0x2260, 0x00c6, 0x00d8, + /* 0xb0 */ + 0x221e, 0x00b1, 0x2264, 0x2265, + 0x00a5, 0x00b5, 0x2202, 0x2211, + 0x220f, 0x03c0, 0x222b, 0x00aa, + 0x00ba, 0x03a9, 0x00e6, 0x00f8, + /* 0xc0 */ + 0x00bf, 0x00a1, 0x00ac, 0x221a, + 0x0192, 0x2248, 0x2206, 0x00ab, + 0x00bb, 0x2026, 0x00a0, 0x00c0, + 0x00c3, 0x00d5, 0x0152, 0x0153, + /* 0xd0 */ + 0x2013, 0x2014, 0x201c, 0x201d, + 0x2018, 0x2019, 0x00f7, 0x25ca, + 0x00ff, 0x0178, 0x2044, 0x20ac, + 0x2039, 0x203a, 0x0176, 0x0177, + /* 0xe0 */ + 0x2021, 0x00b7, 0x1ef2, 0x1ef3, + 0x2030, 0x00c2, 0x00ca, 0x00c1, + 0x00cb, 0x00c8, 0x00cd, 0x00ce, + 0x00cf, 0x00cc, 0x00d3, 0x00d4, + /* 0xf0 */ + 0x2663, 0x00d2, 0x00da, 0x00db, + 0x00d9, 0x0131, 0x00dd, 0x00fd, + 0x0174, 0x0175, 0x1e84, 0x1e85, + 0x1e80, 0x1e81, 0x1e82, 0x1e83, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xca, 0xc1, 0xa2, 0xa3, 0x00, 0xb4, 0x00, 0xa4, /* 0xa0-0xa7 */ + 0xac, 0xa9, 0xbb, 0xc7, 0xc2, 0x00, 0xa8, 0x00, /* 0xa8-0xaf */ + 0xa1, 0xb1, 0x00, 0x00, 0xab, 0xb5, 0xa6, 0xe1, /* 0xb0-0xb7 */ + 0x00, 0x00, 0xbc, 0xc8, 0x00, 0x00, 0x00, 0xc0, /* 0xb8-0xbf */ + 0xcb, 0xe7, 0xe5, 0xcc, 0x80, 0x81, 0xae, 0x82, /* 0xc0-0xc7 */ + 0xe9, 0x83, 0xe6, 0xe8, 0xed, 0xea, 0xeb, 0xec, /* 0xc8-0xcf */ + 0x00, 0x84, 0xf1, 0xee, 0xef, 0xcd, 0x85, 0x00, /* 0xd0-0xd7 */ + 0xaf, 0xf4, 0xf2, 0xf3, 0x86, 0xf6, 0x00, 0xa7, /* 0xd8-0xdf */ + 0x88, 0x87, 0x89, 0x8b, 0x8a, 0x8c, 0xbe, 0x8d, /* 0xe0-0xe7 */ + 0x8f, 0x8e, 0x90, 0x91, 0x93, 0x92, 0x94, 0x95, /* 0xe8-0xef */ + 0x00, 0x96, 0x98, 0x97, 0x99, 0x9b, 0x9a, 0xd6, /* 0xf0-0xf7 */ + 0xbf, 0x9d, 0x9c, 0x9e, 0x9f, 0xf7, 0x00, 0xd8, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0xf5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0xce, 0xcf, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0xf8, 0xf9, 0xde, 0xdf, /* 0x70-0x77 */ + 0xd9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0xc4, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page03[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0xbd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0xb9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page1e[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0xfc, 0xfd, 0xfe, 0xff, 0xfa, 0xfb, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0xe2, 0xe3, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0xd0, 0xd1, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xd4, 0xd5, 0x00, 0x00, 0xd2, 0xd3, 0x00, 0x00, /* 0x18-0x1f */ + 0xa0, 0xe0, 0xa5, 0x00, 0x00, 0x00, 0xc9, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0xe4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0xdc, 0xdd, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0xdb, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page21[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0xaa, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page22[256] = { + 0x00, 0x00, 0xb6, 0x00, 0x00, 0x00, 0xc6, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, /* 0x08-0x0f */ + 0x00, 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, 0xb0, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0xba, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0xc5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0xad, 0x00, 0x00, 0x00, 0xb2, 0xb3, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page25[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0xd7, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page26[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0xf0, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, NULL, page03, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, page1e, NULL, + page20, page21, page22, NULL, NULL, page25, page26, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x00-0x07 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x08-0x0f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x10-0x17 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x18-0x1f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x20-0x27 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x28-0x2f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30-0x37 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x38-0x3f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x40-0x47 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x48-0x4f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x50-0x57 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x58-0x5f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60-0x67 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x68-0x6f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70-0x77 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x78-0x7f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x80-0x87 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x88-0x8f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90-0x97 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x98-0x9f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa0-0xa7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa8-0xaf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0-0xb7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb8-0xbf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc0-0xc7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc8-0xcf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0-0xd7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd8-0xdf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe0-0xe7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe8-0xef */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf0-0xf7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x00-0x07 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x08-0x0f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x10-0x17 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x18-0x1f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x20-0x27 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x28-0x2f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x30-0x37 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x38-0x3f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x40-0x47 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x48-0x4f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x50-0x57 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x58-0x5f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x60-0x67 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x68-0x6f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x70-0x77 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x78-0x7f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x80-0x87 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x88-0x8f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x90-0x97 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x98-0x9f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xa0-0xa7 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xa8-0xaf */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xb0-0xb7 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xb8-0xbf */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xc0-0xc7 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xc8-0xcf */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xd0-0xd7 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xd8-0xdf */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xe0-0xe7 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xe8-0xef */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xf0-0xf7 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "macceltic", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_macceltic(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_macceltic(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_macceltic) +module_exit(exit_nls_macceltic) + +MODULE_DESCRIPTION("NLS Codepage macceltic"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/mac-centeuro.c b/fs/nls/mac-centeuro.c new file mode 100644 index 000000000..43b20f4bd --- /dev/null +++ b/fs/nls/mac-centeuro.c @@ -0,0 +1,532 @@ +/* + * linux/fs/nls/mac-centeuro.c + * + * Charset maccenteuro translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +/* + * COPYRIGHT AND PERMISSION NOTICE + * + * Copyright 1991-2012 Unicode, Inc. All rights reserved. Distributed under + * the Terms of Use in http://www.unicode.org/copyright.html. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of the Unicode data files and any associated documentation (the "Data + * Files") or Unicode software and any associated documentation (the + * "Software") to deal in the Data Files or Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, and/or sell copies of the Data Files or Software, and + * to permit persons to whom the Data Files or Software are furnished to do + * so, provided that (a) the above copyright notice(s) and this permission + * notice appear with all copies of the Data Files or Software, (b) both the + * above copyright notice(s) and this permission notice appear in associated + * documentation, and (c) there is clear notice in each modified Data File or + * in the Software as well as in the documentation associated with the Data + * File(s) or Software that the data or software has been modified. + * + * THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF + * THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS + * INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT + * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF + * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THE DATA FILES OR SOFTWARE. + * + * Except as contained in this notice, the name of a copyright holder shall + * not be used in advertising or otherwise to promote the sale, use or other + * dealings in these Data Files or Software without prior written + * authorization of the copyright holder. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00 */ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10 */ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20 */ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30 */ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40 */ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50 */ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60 */ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70 */ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80 */ + 0x00c4, 0x0100, 0x0101, 0x00c9, + 0x0104, 0x00d6, 0x00dc, 0x00e1, + 0x0105, 0x010c, 0x00e4, 0x010d, + 0x0106, 0x0107, 0x00e9, 0x0179, + /* 0x90 */ + 0x017a, 0x010e, 0x00ed, 0x010f, + 0x0112, 0x0113, 0x0116, 0x00f3, + 0x0117, 0x00f4, 0x00f6, 0x00f5, + 0x00fa, 0x011a, 0x011b, 0x00fc, + /* 0xa0 */ + 0x2020, 0x00b0, 0x0118, 0x00a3, + 0x00a7, 0x2022, 0x00b6, 0x00df, + 0x00ae, 0x00a9, 0x2122, 0x0119, + 0x00a8, 0x2260, 0x0123, 0x012e, + /* 0xb0 */ + 0x012f, 0x012a, 0x2264, 0x2265, + 0x012b, 0x0136, 0x2202, 0x2211, + 0x0142, 0x013b, 0x013c, 0x013d, + 0x013e, 0x0139, 0x013a, 0x0145, + /* 0xc0 */ + 0x0146, 0x0143, 0x00ac, 0x221a, + 0x0144, 0x0147, 0x2206, 0x00ab, + 0x00bb, 0x2026, 0x00a0, 0x0148, + 0x0150, 0x00d5, 0x0151, 0x014c, + /* 0xd0 */ + 0x2013, 0x2014, 0x201c, 0x201d, + 0x2018, 0x2019, 0x00f7, 0x25ca, + 0x014d, 0x0154, 0x0155, 0x0158, + 0x2039, 0x203a, 0x0159, 0x0156, + /* 0xe0 */ + 0x0157, 0x0160, 0x201a, 0x201e, + 0x0161, 0x015a, 0x015b, 0x00c1, + 0x0164, 0x0165, 0x00cd, 0x017d, + 0x017e, 0x016a, 0x00d3, 0x00d4, + /* 0xf0 */ + 0x016b, 0x016e, 0x00da, 0x016f, + 0x0170, 0x0171, 0x0172, 0x0173, + 0x00dd, 0x00fd, 0x0137, 0x017b, + 0x0141, 0x017c, 0x0122, 0x02c7, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xca, 0x00, 0x00, 0xa3, 0x00, 0x00, 0x00, 0xa4, /* 0xa0-0xa7 */ + 0xac, 0xa9, 0x00, 0xc7, 0xc2, 0x00, 0xa8, 0x00, /* 0xa8-0xaf */ + 0xa1, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa6, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0xe7, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x83, 0x00, 0x00, 0x00, 0xea, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0xee, 0xef, 0xcd, 0x85, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0xf2, 0x00, 0x86, 0xf8, 0x00, 0xa7, /* 0xd8-0xdf */ + 0x00, 0x87, 0x00, 0x00, 0x8a, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x8e, 0x00, 0x00, 0x00, 0x92, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x97, 0x99, 0x9b, 0x9a, 0xd6, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x9c, 0x00, 0x9f, 0xf9, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0x81, 0x82, 0x00, 0x00, 0x84, 0x88, 0x8c, 0x8d, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0x8b, 0x91, 0x93, /* 0x08-0x0f */ + 0x00, 0x00, 0x94, 0x95, 0x00, 0x00, 0x96, 0x98, /* 0x10-0x17 */ + 0xa2, 0xab, 0x9d, 0x9e, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0xfe, 0xae, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0xb1, 0xb4, 0x00, 0x00, 0xaf, 0xb0, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb5, 0xfa, /* 0x30-0x37 */ + 0x00, 0xbd, 0xbe, 0xb9, 0xba, 0xbb, 0xbc, 0x00, /* 0x38-0x3f */ + 0x00, 0xfc, 0xb8, 0xc1, 0xc4, 0xbf, 0xc0, 0xc5, /* 0x40-0x47 */ + 0xcb, 0x00, 0x00, 0x00, 0xcf, 0xd8, 0x00, 0x00, /* 0x48-0x4f */ + 0xcc, 0xce, 0x00, 0x00, 0xd9, 0xda, 0xdf, 0xe0, /* 0x50-0x57 */ + 0xdb, 0xde, 0xe5, 0xe6, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0xe1, 0xe4, 0x00, 0x00, 0xe8, 0xe9, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0xed, 0xf0, 0x00, 0x00, 0xf1, 0xf3, /* 0x68-0x6f */ + 0xf4, 0xf5, 0xf6, 0xf7, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x8f, 0x90, 0xfb, 0xfd, 0xeb, 0xec, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page02[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0xd0, 0xd1, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xd4, 0xd5, 0xe2, 0x00, 0xd2, 0xd3, 0xe3, 0x00, /* 0x18-0x1f */ + 0xa0, 0x00, 0xa5, 0x00, 0x00, 0x00, 0xc9, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0xdc, 0xdd, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page21[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0xaa, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page22[256] = { + 0x00, 0x00, 0xb6, 0x00, 0x00, 0x00, 0xc6, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0xad, 0x00, 0x00, 0x00, 0xb2, 0xb3, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page25[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0xd7, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, page02, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + page20, page21, page22, NULL, NULL, page25, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x00-0x07 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x08-0x0f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x10-0x17 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x18-0x1f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x20-0x27 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x28-0x2f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30-0x37 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x38-0x3f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x40-0x47 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x48-0x4f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x50-0x57 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x58-0x5f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60-0x67 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x68-0x6f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70-0x77 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x78-0x7f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x80-0x87 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x88-0x8f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90-0x97 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x98-0x9f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa0-0xa7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa8-0xaf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0-0xb7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb8-0xbf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc0-0xc7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc8-0xcf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0-0xd7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd8-0xdf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe0-0xe7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe8-0xef */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf0-0xf7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x00-0x07 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x08-0x0f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x10-0x17 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x18-0x1f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x20-0x27 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x28-0x2f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30-0x37 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x38-0x3f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x40-0x47 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x48-0x4f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x50-0x57 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x58-0x5f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60-0x67 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x68-0x6f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70-0x77 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x78-0x7f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x80-0x87 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x88-0x8f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90-0x97 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x98-0x9f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa0-0xa7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa8-0xaf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0-0xb7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb8-0xbf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc0-0xc7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc8-0xcf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0-0xd7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd8-0xdf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe0-0xe7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe8-0xef */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf0-0xf7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "maccenteuro", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_maccenteuro(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_maccenteuro(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_maccenteuro) +module_exit(exit_nls_maccenteuro) + +MODULE_DESCRIPTION("NLS Codepage maccenteuro"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/mac-croatian.c b/fs/nls/mac-croatian.c new file mode 100644 index 000000000..62730d6a6 --- /dev/null +++ b/fs/nls/mac-croatian.c @@ -0,0 +1,602 @@ +/* + * linux/fs/nls/mac-croatian.c + * + * Charset maccroatian translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +/* + * COPYRIGHT AND PERMISSION NOTICE + * + * Copyright 1991-2012 Unicode, Inc. All rights reserved. Distributed under + * the Terms of Use in http://www.unicode.org/copyright.html. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of the Unicode data files and any associated documentation (the "Data + * Files") or Unicode software and any associated documentation (the + * "Software") to deal in the Data Files or Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, and/or sell copies of the Data Files or Software, and + * to permit persons to whom the Data Files or Software are furnished to do + * so, provided that (a) the above copyright notice(s) and this permission + * notice appear with all copies of the Data Files or Software, (b) both the + * above copyright notice(s) and this permission notice appear in associated + * documentation, and (c) there is clear notice in each modified Data File or + * in the Software as well as in the documentation associated with the Data + * File(s) or Software that the data or software has been modified. + * + * THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF + * THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS + * INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT + * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF + * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THE DATA FILES OR SOFTWARE. + * + * Except as contained in this notice, the name of a copyright holder shall + * not be used in advertising or otherwise to promote the sale, use or other + * dealings in these Data Files or Software without prior written + * authorization of the copyright holder. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00 */ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10 */ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20 */ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30 */ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40 */ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50 */ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60 */ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70 */ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80 */ + 0x00c4, 0x00c5, 0x00c7, 0x00c9, + 0x00d1, 0x00d6, 0x00dc, 0x00e1, + 0x00e0, 0x00e2, 0x00e4, 0x00e3, + 0x00e5, 0x00e7, 0x00e9, 0x00e8, + /* 0x90 */ + 0x00ea, 0x00eb, 0x00ed, 0x00ec, + 0x00ee, 0x00ef, 0x00f1, 0x00f3, + 0x00f2, 0x00f4, 0x00f6, 0x00f5, + 0x00fa, 0x00f9, 0x00fb, 0x00fc, + /* 0xa0 */ + 0x2020, 0x00b0, 0x00a2, 0x00a3, + 0x00a7, 0x2022, 0x00b6, 0x00df, + 0x00ae, 0x0160, 0x2122, 0x00b4, + 0x00a8, 0x2260, 0x017d, 0x00d8, + /* 0xb0 */ + 0x221e, 0x00b1, 0x2264, 0x2265, + 0x2206, 0x00b5, 0x2202, 0x2211, + 0x220f, 0x0161, 0x222b, 0x00aa, + 0x00ba, 0x03a9, 0x017e, 0x00f8, + /* 0xc0 */ + 0x00bf, 0x00a1, 0x00ac, 0x221a, + 0x0192, 0x2248, 0x0106, 0x00ab, + 0x010c, 0x2026, 0x00a0, 0x00c0, + 0x00c3, 0x00d5, 0x0152, 0x0153, + /* 0xd0 */ + 0x0110, 0x2014, 0x201c, 0x201d, + 0x2018, 0x2019, 0x00f7, 0x25ca, + 0xf8ff, 0x00a9, 0x2044, 0x20ac, + 0x2039, 0x203a, 0x00c6, 0x00bb, + /* 0xe0 */ + 0x2013, 0x00b7, 0x201a, 0x201e, + 0x2030, 0x00c2, 0x0107, 0x00c1, + 0x010d, 0x00c8, 0x00cd, 0x00ce, + 0x00cf, 0x00cc, 0x00d3, 0x00d4, + /* 0xf0 */ + 0x0111, 0x00d2, 0x00da, 0x00db, + 0x00d9, 0x0131, 0x02c6, 0x02dc, + 0x00af, 0x03c0, 0x00cb, 0x02da, + 0x00b8, 0x00ca, 0x00e6, 0x02c7, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xca, 0xc1, 0xa2, 0xa3, 0x00, 0x00, 0x00, 0xa4, /* 0xa0-0xa7 */ + 0xac, 0xd9, 0xbb, 0xc7, 0xc2, 0x00, 0xa8, 0xf8, /* 0xa8-0xaf */ + 0xa1, 0xb1, 0x00, 0x00, 0xab, 0xb5, 0xa6, 0xe1, /* 0xb0-0xb7 */ + 0xfc, 0x00, 0xbc, 0xdf, 0x00, 0x00, 0x00, 0xc0, /* 0xb8-0xbf */ + 0xcb, 0xe7, 0xe5, 0xcc, 0x80, 0x81, 0xde, 0x82, /* 0xc0-0xc7 */ + 0xe9, 0x83, 0xfd, 0xfa, 0xed, 0xea, 0xeb, 0xec, /* 0xc8-0xcf */ + 0x00, 0x84, 0xf1, 0xee, 0xef, 0xcd, 0x85, 0x00, /* 0xd0-0xd7 */ + 0xaf, 0xf4, 0xf2, 0xf3, 0x86, 0x00, 0x00, 0xa7, /* 0xd8-0xdf */ + 0x88, 0x87, 0x89, 0x8b, 0x8a, 0x8c, 0xfe, 0x8d, /* 0xe0-0xe7 */ + 0x8f, 0x8e, 0x90, 0x91, 0x93, 0x92, 0x94, 0x95, /* 0xe8-0xef */ + 0x00, 0x96, 0x98, 0x97, 0x99, 0x9b, 0x9a, 0xd6, /* 0xf0-0xf7 */ + 0xbf, 0x9d, 0x9c, 0x9e, 0x9f, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0xe6, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xc8, 0xe8, 0x00, 0x00, /* 0x08-0x0f */ + 0xd0, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0xf5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0xce, 0xcf, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0xa9, 0xb9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0xae, 0xbe, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0xc4, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page02[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf6, 0xff, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0xfb, 0x00, 0xf7, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page03[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0xbd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0xf9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0xe0, 0xd1, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xd4, 0xd5, 0xe2, 0x00, 0xd2, 0xd3, 0xe3, 0x00, /* 0x18-0x1f */ + 0xa0, 0x00, 0xa5, 0x00, 0x00, 0x00, 0xc9, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0xe4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0xdc, 0xdd, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0xdb, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page21[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0xaa, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page22[256] = { + 0x00, 0x00, 0xb6, 0x00, 0x00, 0x00, 0xb4, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, /* 0x08-0x0f */ + 0x00, 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, 0xb0, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0xba, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0xc5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0xad, 0x00, 0x00, 0x00, 0xb2, 0xb3, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page25[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0xd7, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char pagef8[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd8, /* 0xf8-0xff */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, page02, page03, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + page20, page21, page22, NULL, NULL, page25, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + pagef8, NULL, NULL, NULL, NULL, NULL, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x00-0x07 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x08-0x0f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x10-0x17 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x18-0x1f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x20-0x27 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x28-0x2f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30-0x37 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x38-0x3f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x40-0x47 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x48-0x4f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x50-0x57 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x58-0x5f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60-0x67 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x68-0x6f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70-0x77 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x78-0x7f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x80-0x87 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x88-0x8f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90-0x97 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x98-0x9f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa0-0xa7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa8-0xaf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0-0xb7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb8-0xbf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc0-0xc7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc8-0xcf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0-0xd7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd8-0xdf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe0-0xe7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe8-0xef */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf0-0xf7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x00-0x07 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x08-0x0f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x10-0x17 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x18-0x1f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x20-0x27 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x28-0x2f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30-0x37 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x38-0x3f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x40-0x47 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x48-0x4f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x50-0x57 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x58-0x5f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60-0x67 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x68-0x6f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70-0x77 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x78-0x7f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x80-0x87 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x88-0x8f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90-0x97 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x98-0x9f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa0-0xa7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa8-0xaf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0-0xb7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb8-0xbf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc0-0xc7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc8-0xcf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0-0xd7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd8-0xdf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe0-0xe7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe8-0xef */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf0-0xf7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "maccroatian", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_maccroatian(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_maccroatian(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_maccroatian) +module_exit(exit_nls_maccroatian) + +MODULE_DESCRIPTION("NLS Codepage maccroatian"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/mac-cyrillic.c b/fs/nls/mac-cyrillic.c new file mode 100644 index 000000000..7a5c4d16a --- /dev/null +++ b/fs/nls/mac-cyrillic.c @@ -0,0 +1,497 @@ +/* + * linux/fs/nls/mac-cyrillic.c + * + * Charset maccyrillic translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +/* + * COPYRIGHT AND PERMISSION NOTICE + * + * Copyright 1991-2012 Unicode, Inc. All rights reserved. Distributed under + * the Terms of Use in http://www.unicode.org/copyright.html. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of the Unicode data files and any associated documentation (the "Data + * Files") or Unicode software and any associated documentation (the + * "Software") to deal in the Data Files or Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, and/or sell copies of the Data Files or Software, and + * to permit persons to whom the Data Files or Software are furnished to do + * so, provided that (a) the above copyright notice(s) and this permission + * notice appear with all copies of the Data Files or Software, (b) both the + * above copyright notice(s) and this permission notice appear in associated + * documentation, and (c) there is clear notice in each modified Data File or + * in the Software as well as in the documentation associated with the Data + * File(s) or Software that the data or software has been modified. + * + * THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF + * THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS + * INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT + * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF + * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THE DATA FILES OR SOFTWARE. + * + * Except as contained in this notice, the name of a copyright holder shall + * not be used in advertising or otherwise to promote the sale, use or other + * dealings in these Data Files or Software without prior written + * authorization of the copyright holder. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00 */ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10 */ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20 */ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30 */ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40 */ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50 */ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60 */ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70 */ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80 */ + 0x0410, 0x0411, 0x0412, 0x0413, + 0x0414, 0x0415, 0x0416, 0x0417, + 0x0418, 0x0419, 0x041a, 0x041b, + 0x041c, 0x041d, 0x041e, 0x041f, + /* 0x90 */ + 0x0420, 0x0421, 0x0422, 0x0423, + 0x0424, 0x0425, 0x0426, 0x0427, + 0x0428, 0x0429, 0x042a, 0x042b, + 0x042c, 0x042d, 0x042e, 0x042f, + /* 0xa0 */ + 0x2020, 0x00b0, 0x0490, 0x00a3, + 0x00a7, 0x2022, 0x00b6, 0x0406, + 0x00ae, 0x00a9, 0x2122, 0x0402, + 0x0452, 0x2260, 0x0403, 0x0453, + /* 0xb0 */ + 0x221e, 0x00b1, 0x2264, 0x2265, + 0x0456, 0x00b5, 0x0491, 0x0408, + 0x0404, 0x0454, 0x0407, 0x0457, + 0x0409, 0x0459, 0x040a, 0x045a, + /* 0xc0 */ + 0x0458, 0x0405, 0x00ac, 0x221a, + 0x0192, 0x2248, 0x2206, 0x00ab, + 0x00bb, 0x2026, 0x00a0, 0x040b, + 0x045b, 0x040c, 0x045c, 0x0455, + /* 0xd0 */ + 0x2013, 0x2014, 0x201c, 0x201d, + 0x2018, 0x2019, 0x00f7, 0x201e, + 0x040e, 0x045e, 0x040f, 0x045f, + 0x2116, 0x0401, 0x0451, 0x044f, + /* 0xe0 */ + 0x0430, 0x0431, 0x0432, 0x0433, + 0x0434, 0x0435, 0x0436, 0x0437, + 0x0438, 0x0439, 0x043a, 0x043b, + 0x043c, 0x043d, 0x043e, 0x043f, + /* 0xf0 */ + 0x0440, 0x0441, 0x0442, 0x0443, + 0x0444, 0x0445, 0x0446, 0x0447, + 0x0448, 0x0449, 0x044a, 0x044b, + 0x044c, 0x044d, 0x044e, 0x20ac, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xca, 0x00, 0x00, 0xa3, 0x00, 0x00, 0x00, 0xa4, /* 0xa0-0xa7 */ + 0x00, 0xa9, 0x00, 0xc7, 0xc2, 0x00, 0xa8, 0x00, /* 0xa8-0xaf */ + 0xa1, 0xb1, 0x00, 0x00, 0x00, 0xb5, 0xa6, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd6, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0xc4, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page04[256] = { + 0x00, 0xdd, 0xab, 0xae, 0xb8, 0xc1, 0xa7, 0xba, /* 0x00-0x07 */ + 0xb7, 0xbc, 0xbe, 0xcb, 0xcd, 0x00, 0xd8, 0xda, /* 0x08-0x0f */ + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x10-0x17 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x18-0x1f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x20-0x27 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x28-0x2f */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0x30-0x37 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0x38-0x3f */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0x40-0x47 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, /* 0x48-0x4f */ + 0x00, 0xde, 0xac, 0xaf, 0xb9, 0xcf, 0xb4, 0xbb, /* 0x50-0x57 */ + 0xc0, 0xbd, 0xbf, 0xcc, 0xce, 0x00, 0xd9, 0xdb, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0xa2, 0xb6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0xd0, 0xd1, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xd4, 0xd5, 0x00, 0x00, 0xd2, 0xd3, 0xd7, 0x00, /* 0x18-0x1f */ + 0xa0, 0x00, 0xa5, 0x00, 0x00, 0x00, 0xc9, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page21[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdc, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0xaa, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page22[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc6, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, 0xb0, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0xc5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0xad, 0x00, 0x00, 0x00, 0xb2, 0xb3, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, NULL, NULL, page04, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + page20, page21, page22, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x00-0x07 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x08-0x0f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x10-0x17 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x18-0x1f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x20-0x27 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x28-0x2f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30-0x37 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x38-0x3f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x40-0x47 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x48-0x4f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x50-0x57 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x58-0x5f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60-0x67 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x68-0x6f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70-0x77 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x78-0x7f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x80-0x87 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x88-0x8f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90-0x97 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x98-0x9f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa0-0xa7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa8-0xaf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0-0xb7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb8-0xbf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc0-0xc7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc8-0xcf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0-0xd7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd8-0xdf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe0-0xe7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe8-0xef */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf0-0xf7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x00-0x07 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x08-0x0f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x10-0x17 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x18-0x1f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x20-0x27 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x28-0x2f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30-0x37 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x38-0x3f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x40-0x47 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x48-0x4f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x50-0x57 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x58-0x5f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60-0x67 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x68-0x6f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70-0x77 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x78-0x7f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x80-0x87 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x88-0x8f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90-0x97 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x98-0x9f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa0-0xa7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa8-0xaf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0-0xb7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb8-0xbf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc0-0xc7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc8-0xcf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0-0xd7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd8-0xdf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe0-0xe7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe8-0xef */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf0-0xf7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "maccyrillic", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_maccyrillic(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_maccyrillic(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_maccyrillic) +module_exit(exit_nls_maccyrillic) + +MODULE_DESCRIPTION("NLS Codepage maccyrillic"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/mac-gaelic.c b/fs/nls/mac-gaelic.c new file mode 100644 index 000000000..3d22f03a9 --- /dev/null +++ b/fs/nls/mac-gaelic.c @@ -0,0 +1,567 @@ +/* + * linux/fs/nls/mac-gaelic.c + * + * Charset macgaelic translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +/* + * COPYRIGHT AND PERMISSION NOTICE + * + * Copyright 1991-2012 Unicode, Inc. All rights reserved. Distributed under + * the Terms of Use in http://www.unicode.org/copyright.html. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of the Unicode data files and any associated documentation (the "Data + * Files") or Unicode software and any associated documentation (the + * "Software") to deal in the Data Files or Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, and/or sell copies of the Data Files or Software, and + * to permit persons to whom the Data Files or Software are furnished to do + * so, provided that (a) the above copyright notice(s) and this permission + * notice appear with all copies of the Data Files or Software, (b) both the + * above copyright notice(s) and this permission notice appear in associated + * documentation, and (c) there is clear notice in each modified Data File or + * in the Software as well as in the documentation associated with the Data + * File(s) or Software that the data or software has been modified. + * + * THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF + * THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS + * INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT + * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF + * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THE DATA FILES OR SOFTWARE. + * + * Except as contained in this notice, the name of a copyright holder shall + * not be used in advertising or otherwise to promote the sale, use or other + * dealings in these Data Files or Software without prior written + * authorization of the copyright holder. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00 */ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10 */ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20 */ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30 */ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40 */ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50 */ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60 */ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70 */ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80 */ + 0x00c4, 0x00c5, 0x00c7, 0x00c9, + 0x00d1, 0x00d6, 0x00dc, 0x00e1, + 0x00e0, 0x00e2, 0x00e4, 0x00e3, + 0x00e5, 0x00e7, 0x00e9, 0x00e8, + /* 0x90 */ + 0x00ea, 0x00eb, 0x00ed, 0x00ec, + 0x00ee, 0x00ef, 0x00f1, 0x00f3, + 0x00f2, 0x00f4, 0x00f6, 0x00f5, + 0x00fa, 0x00f9, 0x00fb, 0x00fc, + /* 0xa0 */ + 0x2020, 0x00b0, 0x00a2, 0x00a3, + 0x00a7, 0x2022, 0x00b6, 0x00df, + 0x00ae, 0x00a9, 0x2122, 0x00b4, + 0x00a8, 0x2260, 0x00c6, 0x00d8, + /* 0xb0 */ + 0x1e02, 0x00b1, 0x2264, 0x2265, + 0x1e03, 0x010a, 0x010b, 0x1e0a, + 0x1e0b, 0x1e1e, 0x1e1f, 0x0120, + 0x0121, 0x1e40, 0x00e6, 0x00f8, + /* 0xc0 */ + 0x1e41, 0x1e56, 0x1e57, 0x027c, + 0x0192, 0x017f, 0x1e60, 0x00ab, + 0x00bb, 0x2026, 0x00a0, 0x00c0, + 0x00c3, 0x00d5, 0x0152, 0x0153, + /* 0xd0 */ + 0x2013, 0x2014, 0x201c, 0x201d, + 0x2018, 0x2019, 0x1e61, 0x1e9b, + 0x00ff, 0x0178, 0x1e6a, 0x20ac, + 0x2039, 0x203a, 0x0176, 0x0177, + /* 0xe0 */ + 0x1e6b, 0x00b7, 0x1ef2, 0x1ef3, + 0x204a, 0x00c2, 0x00ca, 0x00c1, + 0x00cb, 0x00c8, 0x00cd, 0x00ce, + 0x00cf, 0x00cc, 0x00d3, 0x00d4, + /* 0xf0 */ + 0x2663, 0x00d2, 0x00da, 0x00db, + 0x00d9, 0x0131, 0x00dd, 0x00fd, + 0x0174, 0x0175, 0x1e84, 0x1e85, + 0x1e80, 0x1e81, 0x1e82, 0x1e83, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xca, 0x00, 0xa2, 0xa3, 0x00, 0x00, 0x00, 0xa4, /* 0xa0-0xa7 */ + 0xac, 0xa9, 0x00, 0xc7, 0x00, 0x00, 0xa8, 0x00, /* 0xa8-0xaf */ + 0xa1, 0xb1, 0x00, 0x00, 0xab, 0x00, 0xa6, 0xe1, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0xc8, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0xcb, 0xe7, 0xe5, 0xcc, 0x80, 0x81, 0xae, 0x82, /* 0xc0-0xc7 */ + 0xe9, 0x83, 0xe6, 0xe8, 0xed, 0xea, 0xeb, 0xec, /* 0xc8-0xcf */ + 0x00, 0x84, 0xf1, 0xee, 0xef, 0xcd, 0x85, 0x00, /* 0xd0-0xd7 */ + 0xaf, 0xf4, 0xf2, 0xf3, 0x86, 0xf6, 0x00, 0xa7, /* 0xd8-0xdf */ + 0x88, 0x87, 0x89, 0x8b, 0x8a, 0x8c, 0xbe, 0x8d, /* 0xe0-0xe7 */ + 0x8f, 0x8e, 0x90, 0x91, 0x93, 0x92, 0x94, 0x95, /* 0xe8-0xef */ + 0x00, 0x96, 0x98, 0x97, 0x99, 0x9b, 0x9a, 0x00, /* 0xf0-0xf7 */ + 0xbf, 0x9d, 0x9c, 0x9e, 0x9f, 0xf7, 0x00, 0xd8, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0xb5, 0xb6, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0xbb, 0xbc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0xf5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0xce, 0xcf, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0xf8, 0xf9, 0xde, 0xdf, /* 0x70-0x77 */ + 0xd9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc5, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0xc4, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page02[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page1e[256] = { + 0x00, 0x00, 0xb0, 0xb4, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0xb7, 0xb8, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb9, 0xba, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0xbd, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc1, 0xc2, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0xc6, 0xd6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0xda, 0xe0, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0xfc, 0xfd, 0xfe, 0xff, 0xfa, 0xfb, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0xd7, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0xe2, 0xe3, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0xd0, 0xd1, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xd4, 0xd5, 0x00, 0x00, 0xd2, 0xd3, 0x00, 0x00, /* 0x18-0x1f */ + 0xa0, 0x00, 0xa5, 0x00, 0x00, 0x00, 0xc9, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0xdc, 0xdd, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0xe4, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0xdb, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page21[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0xaa, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page22[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0xad, 0x00, 0x00, 0x00, 0xb2, 0xb3, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page26[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0xf0, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, page02, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, page1e, NULL, + page20, page21, page22, NULL, NULL, NULL, page26, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x00-0x07 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x08-0x0f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x10-0x17 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x18-0x1f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x20-0x27 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x28-0x2f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30-0x37 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x38-0x3f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x40-0x47 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x48-0x4f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x50-0x57 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x58-0x5f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60-0x67 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x68-0x6f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70-0x77 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x78-0x7f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x80-0x87 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x88-0x8f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90-0x97 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x98-0x9f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa0-0xa7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa8-0xaf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0-0xb7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb8-0xbf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc0-0xc7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc8-0xcf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0-0xd7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd8-0xdf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe0-0xe7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe8-0xef */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf0-0xf7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x00-0x07 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x08-0x0f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x10-0x17 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x18-0x1f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x20-0x27 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x28-0x2f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x30-0x37 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x38-0x3f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x40-0x47 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x48-0x4f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x50-0x57 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x58-0x5f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x60-0x67 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x68-0x6f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x70-0x77 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x78-0x7f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x80-0x87 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x88-0x8f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x90-0x97 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x98-0x9f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xa0-0xa7 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xa8-0xaf */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xb0-0xb7 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xb8-0xbf */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xc0-0xc7 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xc8-0xcf */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xd0-0xd7 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xd8-0xdf */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xe0-0xe7 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xe8-0xef */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xf0-0xf7 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "macgaelic", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_macgaelic(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_macgaelic(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_macgaelic) +module_exit(exit_nls_macgaelic) + +MODULE_DESCRIPTION("NLS Codepage macgaelic"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/mac-greek.c b/fs/nls/mac-greek.c new file mode 100644 index 000000000..de3aa9ddb --- /dev/null +++ b/fs/nls/mac-greek.c @@ -0,0 +1,497 @@ +/* + * linux/fs/nls/mac-greek.c + * + * Charset macgreek translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +/* + * COPYRIGHT AND PERMISSION NOTICE + * + * Copyright 1991-2012 Unicode, Inc. All rights reserved. Distributed under + * the Terms of Use in http://www.unicode.org/copyright.html. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of the Unicode data files and any associated documentation (the "Data + * Files") or Unicode software and any associated documentation (the + * "Software") to deal in the Data Files or Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, and/or sell copies of the Data Files or Software, and + * to permit persons to whom the Data Files or Software are furnished to do + * so, provided that (a) the above copyright notice(s) and this permission + * notice appear with all copies of the Data Files or Software, (b) both the + * above copyright notice(s) and this permission notice appear in associated + * documentation, and (c) there is clear notice in each modified Data File or + * in the Software as well as in the documentation associated with the Data + * File(s) or Software that the data or software has been modified. + * + * THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF + * THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS + * INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT + * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF + * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THE DATA FILES OR SOFTWARE. + * + * Except as contained in this notice, the name of a copyright holder shall + * not be used in advertising or otherwise to promote the sale, use or other + * dealings in these Data Files or Software without prior written + * authorization of the copyright holder. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00 */ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10 */ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20 */ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30 */ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40 */ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50 */ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60 */ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70 */ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80 */ + 0x00c4, 0x00b9, 0x00b2, 0x00c9, + 0x00b3, 0x00d6, 0x00dc, 0x0385, + 0x00e0, 0x00e2, 0x00e4, 0x0384, + 0x00a8, 0x00e7, 0x00e9, 0x00e8, + /* 0x90 */ + 0x00ea, 0x00eb, 0x00a3, 0x2122, + 0x00ee, 0x00ef, 0x2022, 0x00bd, + 0x2030, 0x00f4, 0x00f6, 0x00a6, + 0x20ac, 0x00f9, 0x00fb, 0x00fc, + /* 0xa0 */ + 0x2020, 0x0393, 0x0394, 0x0398, + 0x039b, 0x039e, 0x03a0, 0x00df, + 0x00ae, 0x00a9, 0x03a3, 0x03aa, + 0x00a7, 0x2260, 0x00b0, 0x00b7, + /* 0xb0 */ + 0x0391, 0x00b1, 0x2264, 0x2265, + 0x00a5, 0x0392, 0x0395, 0x0396, + 0x0397, 0x0399, 0x039a, 0x039c, + 0x03a6, 0x03ab, 0x03a8, 0x03a9, + /* 0xc0 */ + 0x03ac, 0x039d, 0x00ac, 0x039f, + 0x03a1, 0x2248, 0x03a4, 0x00ab, + 0x00bb, 0x2026, 0x00a0, 0x03a5, + 0x03a7, 0x0386, 0x0388, 0x0153, + /* 0xd0 */ + 0x2013, 0x2015, 0x201c, 0x201d, + 0x2018, 0x2019, 0x00f7, 0x0389, + 0x038a, 0x038c, 0x038e, 0x03ad, + 0x03ae, 0x03af, 0x03cc, 0x038f, + /* 0xe0 */ + 0x03cd, 0x03b1, 0x03b2, 0x03c8, + 0x03b4, 0x03b5, 0x03c6, 0x03b3, + 0x03b7, 0x03b9, 0x03be, 0x03ba, + 0x03bb, 0x03bc, 0x03bd, 0x03bf, + /* 0xf0 */ + 0x03c0, 0x03ce, 0x03c1, 0x03c3, + 0x03c4, 0x03b8, 0x03c9, 0x03c2, + 0x03c7, 0x03c5, 0x03b6, 0x03ca, + 0x03cb, 0x0390, 0x03b0, 0x00ad, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xca, 0x00, 0x00, 0x92, 0x00, 0xb4, 0x9b, 0xac, /* 0xa0-0xa7 */ + 0x8c, 0xa9, 0x00, 0xc7, 0xc2, 0xff, 0xa8, 0x00, /* 0xa8-0xaf */ + 0xae, 0xb1, 0x82, 0x84, 0x00, 0x00, 0x00, 0xaf, /* 0xb0-0xb7 */ + 0x00, 0x81, 0x00, 0xc8, 0x00, 0x97, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x83, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x85, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x86, 0x00, 0x00, 0xa7, /* 0xd8-0xdf */ + 0x88, 0x00, 0x89, 0x00, 0x8a, 0x00, 0x00, 0x8d, /* 0xe0-0xe7 */ + 0x8f, 0x8e, 0x90, 0x91, 0x00, 0x00, 0x94, 0x95, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x99, 0x00, 0x9a, 0xd6, /* 0xf0-0xf7 */ + 0x00, 0x9d, 0x00, 0x9e, 0x9f, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0xcf, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page03[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x8b, 0x87, 0xcd, 0x00, /* 0x80-0x87 */ + 0xce, 0xd7, 0xd8, 0x00, 0xd9, 0x00, 0xda, 0xdf, /* 0x88-0x8f */ + 0xfd, 0xb0, 0xb5, 0xa1, 0xa2, 0xb6, 0xb7, 0xb8, /* 0x90-0x97 */ + 0xa3, 0xb9, 0xba, 0xa4, 0xbb, 0xc1, 0xa5, 0xc3, /* 0x98-0x9f */ + 0xa6, 0xc4, 0x00, 0xaa, 0xc6, 0xcb, 0xbc, 0xcc, /* 0xa0-0xa7 */ + 0xbe, 0xbf, 0xab, 0xbd, 0xc0, 0xdb, 0xdc, 0xdd, /* 0xa8-0xaf */ + 0xfe, 0xe1, 0xe2, 0xe7, 0xe4, 0xe5, 0xfa, 0xe8, /* 0xb0-0xb7 */ + 0xf5, 0xe9, 0xeb, 0xec, 0xed, 0xee, 0xea, 0xef, /* 0xb8-0xbf */ + 0xf0, 0xf2, 0xf7, 0xf3, 0xf4, 0xf9, 0xe6, 0xf8, /* 0xc0-0xc7 */ + 0xe3, 0xf6, 0xfb, 0xfc, 0xde, 0xe0, 0xf1, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0xd0, 0x00, 0xd1, 0x00, 0x00, /* 0x10-0x17 */ + 0xd4, 0xd5, 0x00, 0x00, 0xd2, 0xd3, 0x00, 0x00, /* 0x18-0x1f */ + 0xa0, 0x00, 0x96, 0x00, 0x00, 0x00, 0xc9, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x98, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x9c, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page21[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x93, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page22[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0xc5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0xad, 0x00, 0x00, 0x00, 0xb2, 0xb3, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, NULL, page03, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + page20, page21, page22, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x00-0x07 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x08-0x0f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x10-0x17 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x18-0x1f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x20-0x27 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x28-0x2f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30-0x37 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x38-0x3f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x40-0x47 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x48-0x4f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x50-0x57 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x58-0x5f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60-0x67 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x68-0x6f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70-0x77 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x78-0x7f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x80-0x87 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x88-0x8f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90-0x97 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x98-0x9f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa0-0xa7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa8-0xaf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0-0xb7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb8-0xbf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc0-0xc7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc8-0xcf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0-0xd7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd8-0xdf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe0-0xe7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe8-0xef */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf0-0xf7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x00-0x07 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x08-0x0f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x10-0x17 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x18-0x1f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x20-0x27 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x28-0x2f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30-0x37 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x38-0x3f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x40-0x47 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x48-0x4f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x50-0x57 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x58-0x5f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60-0x67 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x68-0x6f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70-0x77 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x78-0x7f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x80-0x87 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x88-0x8f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90-0x97 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x98-0x9f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa0-0xa7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa8-0xaf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0-0xb7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb8-0xbf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc0-0xc7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc8-0xcf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0-0xd7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd8-0xdf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe0-0xe7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe8-0xef */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf0-0xf7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "macgreek", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_macgreek(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_macgreek(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_macgreek) +module_exit(exit_nls_macgreek) + +MODULE_DESCRIPTION("NLS Codepage macgreek"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/mac-iceland.c b/fs/nls/mac-iceland.c new file mode 100644 index 000000000..0bba83f9d --- /dev/null +++ b/fs/nls/mac-iceland.c @@ -0,0 +1,602 @@ +/* + * linux/fs/nls/mac-iceland.c + * + * Charset maciceland translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +/* + * COPYRIGHT AND PERMISSION NOTICE + * + * Copyright 1991-2012 Unicode, Inc. All rights reserved. Distributed under + * the Terms of Use in http://www.unicode.org/copyright.html. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of the Unicode data files and any associated documentation (the "Data + * Files") or Unicode software and any associated documentation (the + * "Software") to deal in the Data Files or Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, and/or sell copies of the Data Files or Software, and + * to permit persons to whom the Data Files or Software are furnished to do + * so, provided that (a) the above copyright notice(s) and this permission + * notice appear with all copies of the Data Files or Software, (b) both the + * above copyright notice(s) and this permission notice appear in associated + * documentation, and (c) there is clear notice in each modified Data File or + * in the Software as well as in the documentation associated with the Data + * File(s) or Software that the data or software has been modified. + * + * THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF + * THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS + * INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT + * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF + * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THE DATA FILES OR SOFTWARE. + * + * Except as contained in this notice, the name of a copyright holder shall + * not be used in advertising or otherwise to promote the sale, use or other + * dealings in these Data Files or Software without prior written + * authorization of the copyright holder. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00 */ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10 */ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20 */ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30 */ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40 */ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50 */ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60 */ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70 */ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80 */ + 0x00c4, 0x00c5, 0x00c7, 0x00c9, + 0x00d1, 0x00d6, 0x00dc, 0x00e1, + 0x00e0, 0x00e2, 0x00e4, 0x00e3, + 0x00e5, 0x00e7, 0x00e9, 0x00e8, + /* 0x90 */ + 0x00ea, 0x00eb, 0x00ed, 0x00ec, + 0x00ee, 0x00ef, 0x00f1, 0x00f3, + 0x00f2, 0x00f4, 0x00f6, 0x00f5, + 0x00fa, 0x00f9, 0x00fb, 0x00fc, + /* 0xa0 */ + 0x00dd, 0x00b0, 0x00a2, 0x00a3, + 0x00a7, 0x2022, 0x00b6, 0x00df, + 0x00ae, 0x00a9, 0x2122, 0x00b4, + 0x00a8, 0x2260, 0x00c6, 0x00d8, + /* 0xb0 */ + 0x221e, 0x00b1, 0x2264, 0x2265, + 0x00a5, 0x00b5, 0x2202, 0x2211, + 0x220f, 0x03c0, 0x222b, 0x00aa, + 0x00ba, 0x03a9, 0x00e6, 0x00f8, + /* 0xc0 */ + 0x00bf, 0x00a1, 0x00ac, 0x221a, + 0x0192, 0x2248, 0x2206, 0x00ab, + 0x00bb, 0x2026, 0x00a0, 0x00c0, + 0x00c3, 0x00d5, 0x0152, 0x0153, + /* 0xd0 */ + 0x2013, 0x2014, 0x201c, 0x201d, + 0x2018, 0x2019, 0x00f7, 0x25ca, + 0x00ff, 0x0178, 0x2044, 0x20ac, + 0x00d0, 0x00f0, 0x00de, 0x00fe, + /* 0xe0 */ + 0x00fd, 0x00b7, 0x201a, 0x201e, + 0x2030, 0x00c2, 0x00ca, 0x00c1, + 0x00cb, 0x00c8, 0x00cd, 0x00ce, + 0x00cf, 0x00cc, 0x00d3, 0x00d4, + /* 0xf0 */ + 0xf8ff, 0x00d2, 0x00da, 0x00db, + 0x00d9, 0x0131, 0x02c6, 0x02dc, + 0x00af, 0x02d8, 0x02d9, 0x02da, + 0x00b8, 0x02dd, 0x02db, 0x02c7, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xca, 0xc1, 0xa2, 0xa3, 0x00, 0xb4, 0x00, 0xa4, /* 0xa0-0xa7 */ + 0xac, 0xa9, 0xbb, 0xc7, 0xc2, 0x00, 0xa8, 0xf8, /* 0xa8-0xaf */ + 0xa1, 0xb1, 0x00, 0x00, 0xab, 0xb5, 0xa6, 0xe1, /* 0xb0-0xb7 */ + 0xfc, 0x00, 0xbc, 0xc8, 0x00, 0x00, 0x00, 0xc0, /* 0xb8-0xbf */ + 0xcb, 0xe7, 0xe5, 0xcc, 0x80, 0x81, 0xae, 0x82, /* 0xc0-0xc7 */ + 0xe9, 0x83, 0xe6, 0xe8, 0xed, 0xea, 0xeb, 0xec, /* 0xc8-0xcf */ + 0xdc, 0x84, 0xf1, 0xee, 0xef, 0xcd, 0x85, 0x00, /* 0xd0-0xd7 */ + 0xaf, 0xf4, 0xf2, 0xf3, 0x86, 0xa0, 0xde, 0xa7, /* 0xd8-0xdf */ + 0x88, 0x87, 0x89, 0x8b, 0x8a, 0x8c, 0xbe, 0x8d, /* 0xe0-0xe7 */ + 0x8f, 0x8e, 0x90, 0x91, 0x93, 0x92, 0x94, 0x95, /* 0xe8-0xef */ + 0xdd, 0x96, 0x98, 0x97, 0x99, 0x9b, 0x9a, 0xd6, /* 0xf0-0xf7 */ + 0xbf, 0x9d, 0x9c, 0x9e, 0x9f, 0xe0, 0xdf, 0xd8, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0xf5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0xce, 0xcf, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0xd9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0xc4, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page02[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf6, 0xff, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0xf9, 0xfa, 0xfb, 0xfe, 0xf7, 0xfd, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page03[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0xbd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0xb9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0xd0, 0xd1, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xd4, 0xd5, 0xe2, 0x00, 0xd2, 0xd3, 0xe3, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0xa5, 0x00, 0x00, 0x00, 0xc9, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0xe4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0xdb, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page21[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0xaa, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page22[256] = { + 0x00, 0x00, 0xb6, 0x00, 0x00, 0x00, 0xc6, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, /* 0x08-0x0f */ + 0x00, 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, 0xb0, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0xba, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0xc5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0xad, 0x00, 0x00, 0x00, 0xb2, 0xb3, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page25[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0xd7, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char pagef8[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, /* 0xf8-0xff */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, page02, page03, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + page20, page21, page22, NULL, NULL, page25, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + pagef8, NULL, NULL, NULL, NULL, NULL, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x00-0x07 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x08-0x0f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x10-0x17 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x18-0x1f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x20-0x27 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x28-0x2f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30-0x37 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x38-0x3f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x40-0x47 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x48-0x4f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x50-0x57 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x58-0x5f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60-0x67 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x68-0x6f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70-0x77 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x78-0x7f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x80-0x87 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x88-0x8f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90-0x97 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x98-0x9f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa0-0xa7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa8-0xaf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0-0xb7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb8-0xbf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc0-0xc7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc8-0xcf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0-0xd7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd8-0xdf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe0-0xe7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe8-0xef */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf0-0xf7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x00-0x07 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x08-0x0f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x10-0x17 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x18-0x1f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x20-0x27 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x28-0x2f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30-0x37 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x38-0x3f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x40-0x47 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x48-0x4f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x50-0x57 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x58-0x5f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60-0x67 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x68-0x6f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70-0x77 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x78-0x7f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x80-0x87 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x88-0x8f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90-0x97 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x98-0x9f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa0-0xa7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa8-0xaf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0-0xb7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb8-0xbf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc0-0xc7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc8-0xcf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0-0xd7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd8-0xdf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe0-0xe7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe8-0xef */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf0-0xf7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "maciceland", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_maciceland(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_maciceland(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_maciceland) +module_exit(exit_nls_maciceland) + +MODULE_DESCRIPTION("NLS Codepage maciceland"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/mac-inuit.c b/fs/nls/mac-inuit.c new file mode 100644 index 000000000..493386832 --- /dev/null +++ b/fs/nls/mac-inuit.c @@ -0,0 +1,532 @@ +/* + * linux/fs/nls/mac-inuit.c + * + * Charset macinuit translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +/* + * COPYRIGHT AND PERMISSION NOTICE + * + * Copyright 1991-2012 Unicode, Inc. All rights reserved. Distributed under + * the Terms of Use in http://www.unicode.org/copyright.html. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of the Unicode data files and any associated documentation (the "Data + * Files") or Unicode software and any associated documentation (the + * "Software") to deal in the Data Files or Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, and/or sell copies of the Data Files or Software, and + * to permit persons to whom the Data Files or Software are furnished to do + * so, provided that (a) the above copyright notice(s) and this permission + * notice appear with all copies of the Data Files or Software, (b) both the + * above copyright notice(s) and this permission notice appear in associated + * documentation, and (c) there is clear notice in each modified Data File or + * in the Software as well as in the documentation associated with the Data + * File(s) or Software that the data or software has been modified. + * + * THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF + * THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS + * INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT + * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF + * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THE DATA FILES OR SOFTWARE. + * + * Except as contained in this notice, the name of a copyright holder shall + * not be used in advertising or otherwise to promote the sale, use or other + * dealings in these Data Files or Software without prior written + * authorization of the copyright holder. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00 */ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10 */ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20 */ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30 */ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40 */ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50 */ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60 */ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70 */ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80 */ + 0x1403, 0x1404, 0x1405, 0x1406, + 0x140a, 0x140b, 0x1431, 0x1432, + 0x1433, 0x1434, 0x1438, 0x1439, + 0x1449, 0x144e, 0x144f, 0x1450, + /* 0x90 */ + 0x1451, 0x1455, 0x1456, 0x1466, + 0x146d, 0x146e, 0x146f, 0x1470, + 0x1472, 0x1473, 0x1483, 0x148b, + 0x148c, 0x148d, 0x148e, 0x1490, + /* 0xa0 */ + 0x1491, 0x00b0, 0x14a1, 0x14a5, + 0x14a6, 0x2022, 0x00b6, 0x14a7, + 0x00ae, 0x00a9, 0x2122, 0x14a8, + 0x14aa, 0x14ab, 0x14bb, 0x14c2, + /* 0xb0 */ + 0x14c3, 0x14c4, 0x14c5, 0x14c7, + 0x14c8, 0x14d0, 0x14ef, 0x14f0, + 0x14f1, 0x14f2, 0x14f4, 0x14f5, + 0x1505, 0x14d5, 0x14d6, 0x14d7, + /* 0xc0 */ + 0x14d8, 0x14da, 0x14db, 0x14ea, + 0x1528, 0x1529, 0x152a, 0x152b, + 0x152d, 0x2026, 0x00a0, 0x152e, + 0x153e, 0x1555, 0x1556, 0x1557, + /* 0xd0 */ + 0x2013, 0x2014, 0x201c, 0x201d, + 0x2018, 0x2019, 0x1558, 0x1559, + 0x155a, 0x155d, 0x1546, 0x1547, + 0x1548, 0x1549, 0x154b, 0x154c, + /* 0xe0 */ + 0x1550, 0x157f, 0x1580, 0x1581, + 0x1582, 0x1583, 0x1584, 0x1585, + 0x158f, 0x1590, 0x1591, 0x1592, + 0x1593, 0x1594, 0x1595, 0x1671, + /* 0xf0 */ + 0x1672, 0x1673, 0x1674, 0x1675, + 0x1676, 0x1596, 0x15a0, 0x15a1, + 0x15a2, 0x15a3, 0x15a4, 0x15a5, + 0x15a6, 0x157c, 0x0141, 0x0142, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xca, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0xa9, 0x00, 0x00, 0x00, 0x00, 0xa8, 0x00, /* 0xa8-0xaf */ + 0xa1, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa6, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0xfe, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page14[256] = { + 0x00, 0x00, 0x00, 0x80, 0x81, 0x82, 0x83, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x84, 0x85, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x86, 0x87, 0x88, 0x89, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x8a, 0x8b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x8c, 0x00, 0x00, 0x00, 0x00, 0x8d, 0x8e, /* 0x48-0x4f */ + 0x8f, 0x90, 0x00, 0x00, 0x00, 0x91, 0x92, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x94, 0x95, 0x96, /* 0x68-0x6f */ + 0x97, 0x00, 0x98, 0x99, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x9a, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x9b, 0x9c, 0x9d, 0x9e, 0x00, /* 0x88-0x8f */ + 0x9f, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0xa2, 0x00, 0x00, 0x00, 0xa3, 0xa4, 0xa7, /* 0xa0-0xa7 */ + 0xab, 0x00, 0xac, 0xad, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0xae, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0xaf, 0xb0, 0xb1, 0xb2, 0x00, 0xb3, /* 0xc0-0xc7 */ + 0xb4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0xb5, 0x00, 0x00, 0x00, 0x00, 0xbd, 0xbe, 0xbf, /* 0xd0-0xd7 */ + 0xc0, 0x00, 0xc1, 0xc2, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, 0x00, 0xb6, /* 0xe8-0xef */ + 0xb7, 0xb8, 0xb9, 0x00, 0xba, 0xbb, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page15[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0xbc, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0xc4, 0xc5, 0xc6, 0xc7, 0x00, 0xc8, 0xcb, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcc, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xda, 0xdb, /* 0x40-0x47 */ + 0xdc, 0xdd, 0x00, 0xde, 0xdf, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0xe0, 0x00, 0x00, 0x00, 0x00, 0xcd, 0xce, 0xcf, /* 0x50-0x57 */ + 0xd6, 0xd7, 0xd8, 0x00, 0x00, 0xd9, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x00, 0xe1, /* 0x78-0x7f */ + 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, /* 0x88-0x8f */ + 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xf5, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page16[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0xd0, 0xd1, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xd4, 0xd5, 0x00, 0x00, 0xd2, 0xd3, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0xa5, 0x00, 0x00, 0x00, 0xc9, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page21[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0xaa, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, page14, page15, page16, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + page20, page21, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x00-0x07 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x08-0x0f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x10-0x17 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x18-0x1f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x20-0x27 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x28-0x2f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30-0x37 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x38-0x3f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x40-0x47 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x48-0x4f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x50-0x57 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x58-0x5f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60-0x67 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x68-0x6f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70-0x77 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x78-0x7f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x80-0x87 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x88-0x8f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90-0x97 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x98-0x9f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa0-0xa7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa8-0xaf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0-0xb7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb8-0xbf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc0-0xc7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc8-0xcf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0-0xd7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd8-0xdf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe0-0xe7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe8-0xef */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf0-0xf7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x00-0x07 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x08-0x0f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x10-0x17 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x18-0x1f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x20-0x27 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x28-0x2f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x30-0x37 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x38-0x3f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x40-0x47 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x48-0x4f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x50-0x57 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x58-0x5f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x60-0x67 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x68-0x6f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x70-0x77 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x78-0x7f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x80-0x87 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x88-0x8f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x90-0x97 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0x98-0x9f */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xa0-0xa7 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xa8-0xaf */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xb0-0xb7 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xb8-0xbf */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xc0-0xc7 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xc8-0xcf */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xd0-0xd7 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xd8-0xdf */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xe0-0xe7 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xe8-0xef */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xf0-0xf7 */ + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "macinuit", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_macinuit(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_macinuit(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_macinuit) +module_exit(exit_nls_macinuit) + +MODULE_DESCRIPTION("NLS Codepage macinuit"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/mac-roman.c b/fs/nls/mac-roman.c new file mode 100644 index 000000000..d3c082173 --- /dev/null +++ b/fs/nls/mac-roman.c @@ -0,0 +1,637 @@ +/* + * linux/fs/nls/mac-roman.c + * + * Charset macroman translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +/* + * COPYRIGHT AND PERMISSION NOTICE + * + * Copyright 1991-2012 Unicode, Inc. All rights reserved. Distributed under + * the Terms of Use in http://www.unicode.org/copyright.html. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of the Unicode data files and any associated documentation (the "Data + * Files") or Unicode software and any associated documentation (the + * "Software") to deal in the Data Files or Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, and/or sell copies of the Data Files or Software, and + * to permit persons to whom the Data Files or Software are furnished to do + * so, provided that (a) the above copyright notice(s) and this permission + * notice appear with all copies of the Data Files or Software, (b) both the + * above copyright notice(s) and this permission notice appear in associated + * documentation, and (c) there is clear notice in each modified Data File or + * in the Software as well as in the documentation associated with the Data + * File(s) or Software that the data or software has been modified. + * + * THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF + * THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS + * INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT + * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF + * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THE DATA FILES OR SOFTWARE. + * + * Except as contained in this notice, the name of a copyright holder shall + * not be used in advertising or otherwise to promote the sale, use or other + * dealings in these Data Files or Software without prior written + * authorization of the copyright holder. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00 */ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10 */ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20 */ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30 */ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40 */ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50 */ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60 */ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70 */ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80 */ + 0x00c4, 0x00c5, 0x00c7, 0x00c9, + 0x00d1, 0x00d6, 0x00dc, 0x00e1, + 0x00e0, 0x00e2, 0x00e4, 0x00e3, + 0x00e5, 0x00e7, 0x00e9, 0x00e8, + /* 0x90 */ + 0x00ea, 0x00eb, 0x00ed, 0x00ec, + 0x00ee, 0x00ef, 0x00f1, 0x00f3, + 0x00f2, 0x00f4, 0x00f6, 0x00f5, + 0x00fa, 0x00f9, 0x00fb, 0x00fc, + /* 0xa0 */ + 0x2020, 0x00b0, 0x00a2, 0x00a3, + 0x00a7, 0x2022, 0x00b6, 0x00df, + 0x00ae, 0x00a9, 0x2122, 0x00b4, + 0x00a8, 0x2260, 0x00c6, 0x00d8, + /* 0xb0 */ + 0x221e, 0x00b1, 0x2264, 0x2265, + 0x00a5, 0x00b5, 0x2202, 0x2211, + 0x220f, 0x03c0, 0x222b, 0x00aa, + 0x00ba, 0x03a9, 0x00e6, 0x00f8, + /* 0xc0 */ + 0x00bf, 0x00a1, 0x00ac, 0x221a, + 0x0192, 0x2248, 0x2206, 0x00ab, + 0x00bb, 0x2026, 0x00a0, 0x00c0, + 0x00c3, 0x00d5, 0x0152, 0x0153, + /* 0xd0 */ + 0x2013, 0x2014, 0x201c, 0x201d, + 0x2018, 0x2019, 0x00f7, 0x25ca, + 0x00ff, 0x0178, 0x2044, 0x20ac, + 0x2039, 0x203a, 0xfb01, 0xfb02, + /* 0xe0 */ + 0x2021, 0x00b7, 0x201a, 0x201e, + 0x2030, 0x00c2, 0x00ca, 0x00c1, + 0x00cb, 0x00c8, 0x00cd, 0x00ce, + 0x00cf, 0x00cc, 0x00d3, 0x00d4, + /* 0xf0 */ + 0xf8ff, 0x00d2, 0x00da, 0x00db, + 0x00d9, 0x0131, 0x02c6, 0x02dc, + 0x00af, 0x02d8, 0x02d9, 0x02da, + 0x00b8, 0x02dd, 0x02db, 0x02c7, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xca, 0xc1, 0xa2, 0xa3, 0x00, 0xb4, 0x00, 0xa4, /* 0xa0-0xa7 */ + 0xac, 0xa9, 0xbb, 0xc7, 0xc2, 0x00, 0xa8, 0xf8, /* 0xa8-0xaf */ + 0xa1, 0xb1, 0x00, 0x00, 0xab, 0xb5, 0xa6, 0xe1, /* 0xb0-0xb7 */ + 0xfc, 0x00, 0xbc, 0xc8, 0x00, 0x00, 0x00, 0xc0, /* 0xb8-0xbf */ + 0xcb, 0xe7, 0xe5, 0xcc, 0x80, 0x81, 0xae, 0x82, /* 0xc0-0xc7 */ + 0xe9, 0x83, 0xe6, 0xe8, 0xed, 0xea, 0xeb, 0xec, /* 0xc8-0xcf */ + 0x00, 0x84, 0xf1, 0xee, 0xef, 0xcd, 0x85, 0x00, /* 0xd0-0xd7 */ + 0xaf, 0xf4, 0xf2, 0xf3, 0x86, 0x00, 0x00, 0xa7, /* 0xd8-0xdf */ + 0x88, 0x87, 0x89, 0x8b, 0x8a, 0x8c, 0xbe, 0x8d, /* 0xe0-0xe7 */ + 0x8f, 0x8e, 0x90, 0x91, 0x93, 0x92, 0x94, 0x95, /* 0xe8-0xef */ + 0x00, 0x96, 0x98, 0x97, 0x99, 0x9b, 0x9a, 0xd6, /* 0xf0-0xf7 */ + 0xbf, 0x9d, 0x9c, 0x9e, 0x9f, 0x00, 0x00, 0xd8, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0xf5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0xce, 0xcf, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0xd9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0xc4, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page02[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf6, 0xff, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0xf9, 0xfa, 0xfb, 0xfe, 0xf7, 0xfd, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page03[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0xbd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0xb9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0xd0, 0xd1, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xd4, 0xd5, 0xe2, 0x00, 0xd2, 0xd3, 0xe3, 0x00, /* 0x18-0x1f */ + 0xa0, 0xe0, 0xa5, 0x00, 0x00, 0x00, 0xc9, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0xe4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0xdc, 0xdd, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0xdb, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page21[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0xaa, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page22[256] = { + 0x00, 0x00, 0xb6, 0x00, 0x00, 0x00, 0xc6, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, /* 0x08-0x0f */ + 0x00, 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, 0xb0, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0xba, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0xc5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0xad, 0x00, 0x00, 0x00, 0xb2, 0xb3, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page25[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0xd7, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char pagef8[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, /* 0xf8-0xff */ +}; + +static const unsigned char pagefb[256] = { + 0x00, 0xde, 0xdf, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, page02, page03, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + page20, page21, page22, NULL, NULL, page25, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + pagef8, NULL, NULL, pagefb, NULL, NULL, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x00-0x07 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x08-0x0f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x10-0x17 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x18-0x1f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x20-0x27 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x28-0x2f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30-0x37 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x38-0x3f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x40-0x47 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x48-0x4f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x50-0x57 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x58-0x5f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60-0x67 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x68-0x6f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70-0x77 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x78-0x7f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x80-0x87 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x88-0x8f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90-0x97 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x98-0x9f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa0-0xa7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa8-0xaf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0-0xb7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb8-0xbf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc0-0xc7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc8-0xcf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0-0xd7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd8-0xdf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe0-0xe7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe8-0xef */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf0-0xf7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x00-0x07 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x08-0x0f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x10-0x17 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x18-0x1f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x20-0x27 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x28-0x2f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30-0x37 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x38-0x3f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x40-0x47 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x48-0x4f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x50-0x57 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x58-0x5f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60-0x67 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x68-0x6f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70-0x77 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x78-0x7f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x80-0x87 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x88-0x8f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90-0x97 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x98-0x9f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa0-0xa7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa8-0xaf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0-0xb7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb8-0xbf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc0-0xc7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc8-0xcf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0-0xd7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd8-0xdf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe0-0xe7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe8-0xef */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf0-0xf7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "macroman", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_macroman(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_macroman(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_macroman) +module_exit(exit_nls_macroman) + +MODULE_DESCRIPTION("NLS Codepage macroman"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/mac-romanian.c b/fs/nls/mac-romanian.c new file mode 100644 index 000000000..a7735852f --- /dev/null +++ b/fs/nls/mac-romanian.c @@ -0,0 +1,602 @@ +/* + * linux/fs/nls/mac-romanian.c + * + * Charset macromanian translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +/* + * COPYRIGHT AND PERMISSION NOTICE + * + * Copyright 1991-2012 Unicode, Inc. All rights reserved. Distributed under + * the Terms of Use in http://www.unicode.org/copyright.html. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of the Unicode data files and any associated documentation (the "Data + * Files") or Unicode software and any associated documentation (the + * "Software") to deal in the Data Files or Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, and/or sell copies of the Data Files or Software, and + * to permit persons to whom the Data Files or Software are furnished to do + * so, provided that (a) the above copyright notice(s) and this permission + * notice appear with all copies of the Data Files or Software, (b) both the + * above copyright notice(s) and this permission notice appear in associated + * documentation, and (c) there is clear notice in each modified Data File or + * in the Software as well as in the documentation associated with the Data + * File(s) or Software that the data or software has been modified. + * + * THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF + * THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS + * INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT + * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF + * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THE DATA FILES OR SOFTWARE. + * + * Except as contained in this notice, the name of a copyright holder shall + * not be used in advertising or otherwise to promote the sale, use or other + * dealings in these Data Files or Software without prior written + * authorization of the copyright holder. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00 */ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10 */ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20 */ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30 */ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40 */ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50 */ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60 */ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70 */ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80 */ + 0x00c4, 0x00c5, 0x00c7, 0x00c9, + 0x00d1, 0x00d6, 0x00dc, 0x00e1, + 0x00e0, 0x00e2, 0x00e4, 0x00e3, + 0x00e5, 0x00e7, 0x00e9, 0x00e8, + /* 0x90 */ + 0x00ea, 0x00eb, 0x00ed, 0x00ec, + 0x00ee, 0x00ef, 0x00f1, 0x00f3, + 0x00f2, 0x00f4, 0x00f6, 0x00f5, + 0x00fa, 0x00f9, 0x00fb, 0x00fc, + /* 0xa0 */ + 0x2020, 0x00b0, 0x00a2, 0x00a3, + 0x00a7, 0x2022, 0x00b6, 0x00df, + 0x00ae, 0x00a9, 0x2122, 0x00b4, + 0x00a8, 0x2260, 0x0102, 0x0218, + /* 0xb0 */ + 0x221e, 0x00b1, 0x2264, 0x2265, + 0x00a5, 0x00b5, 0x2202, 0x2211, + 0x220f, 0x03c0, 0x222b, 0x00aa, + 0x00ba, 0x03a9, 0x0103, 0x0219, + /* 0xc0 */ + 0x00bf, 0x00a1, 0x00ac, 0x221a, + 0x0192, 0x2248, 0x2206, 0x00ab, + 0x00bb, 0x2026, 0x00a0, 0x00c0, + 0x00c3, 0x00d5, 0x0152, 0x0153, + /* 0xd0 */ + 0x2013, 0x2014, 0x201c, 0x201d, + 0x2018, 0x2019, 0x00f7, 0x25ca, + 0x00ff, 0x0178, 0x2044, 0x20ac, + 0x2039, 0x203a, 0x021a, 0x021b, + /* 0xe0 */ + 0x2021, 0x00b7, 0x201a, 0x201e, + 0x2030, 0x00c2, 0x00ca, 0x00c1, + 0x00cb, 0x00c8, 0x00cd, 0x00ce, + 0x00cf, 0x00cc, 0x00d3, 0x00d4, + /* 0xf0 */ + 0xf8ff, 0x00d2, 0x00da, 0x00db, + 0x00d9, 0x0131, 0x02c6, 0x02dc, + 0x00af, 0x02d8, 0x02d9, 0x02da, + 0x00b8, 0x02dd, 0x02db, 0x02c7, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xca, 0xc1, 0xa2, 0xa3, 0x00, 0xb4, 0x00, 0xa4, /* 0xa0-0xa7 */ + 0xac, 0xa9, 0xbb, 0xc7, 0xc2, 0x00, 0xa8, 0xf8, /* 0xa8-0xaf */ + 0xa1, 0xb1, 0x00, 0x00, 0xab, 0xb5, 0xa6, 0xe1, /* 0xb0-0xb7 */ + 0xfc, 0x00, 0xbc, 0xc8, 0x00, 0x00, 0x00, 0xc0, /* 0xb8-0xbf */ + 0xcb, 0xe7, 0xe5, 0xcc, 0x80, 0x81, 0x00, 0x82, /* 0xc0-0xc7 */ + 0xe9, 0x83, 0xe6, 0xe8, 0xed, 0xea, 0xeb, 0xec, /* 0xc8-0xcf */ + 0x00, 0x84, 0xf1, 0xee, 0xef, 0xcd, 0x85, 0x00, /* 0xd0-0xd7 */ + 0x00, 0xf4, 0xf2, 0xf3, 0x86, 0x00, 0x00, 0xa7, /* 0xd8-0xdf */ + 0x88, 0x87, 0x89, 0x8b, 0x8a, 0x8c, 0x00, 0x8d, /* 0xe0-0xe7 */ + 0x8f, 0x8e, 0x90, 0x91, 0x93, 0x92, 0x94, 0x95, /* 0xe8-0xef */ + 0x00, 0x96, 0x98, 0x97, 0x99, 0x9b, 0x9a, 0xd6, /* 0xf0-0xf7 */ + 0x00, 0x9d, 0x9c, 0x9e, 0x9f, 0x00, 0x00, 0xd8, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0x00, 0x00, 0xae, 0xbe, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0xf5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0xce, 0xcf, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0xd9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0xc4, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page02[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xaf, 0xbf, 0xde, 0xdf, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf6, 0xff, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0xf9, 0xfa, 0xfb, 0xfe, 0xf7, 0xfd, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page03[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0xbd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0xb9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0xd0, 0xd1, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xd4, 0xd5, 0xe2, 0x00, 0xd2, 0xd3, 0xe3, 0x00, /* 0x18-0x1f */ + 0xa0, 0xe0, 0xa5, 0x00, 0x00, 0x00, 0xc9, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0xe4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0xdc, 0xdd, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0xdb, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page21[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0xaa, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page22[256] = { + 0x00, 0x00, 0xb6, 0x00, 0x00, 0x00, 0xc6, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, /* 0x08-0x0f */ + 0x00, 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, 0xb0, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0xba, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0xc5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0xad, 0x00, 0x00, 0x00, 0xb2, 0xb3, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page25[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0xd7, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char pagef8[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, /* 0xf8-0xff */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, page02, page03, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + page20, page21, page22, NULL, NULL, page25, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + pagef8, NULL, NULL, NULL, NULL, NULL, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x00-0x07 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x08-0x0f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x10-0x17 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x18-0x1f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x20-0x27 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x28-0x2f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30-0x37 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x38-0x3f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x40-0x47 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x48-0x4f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x50-0x57 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x58-0x5f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60-0x67 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x68-0x6f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70-0x77 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x78-0x7f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x80-0x87 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x88-0x8f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90-0x97 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x98-0x9f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa0-0xa7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa8-0xaf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0-0xb7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb8-0xbf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc0-0xc7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc8-0xcf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0-0xd7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd8-0xdf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe0-0xe7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe8-0xef */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf0-0xf7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x00-0x07 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x08-0x0f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x10-0x17 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x18-0x1f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x20-0x27 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x28-0x2f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30-0x37 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x38-0x3f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x40-0x47 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x48-0x4f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x50-0x57 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x58-0x5f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60-0x67 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x68-0x6f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70-0x77 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x78-0x7f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x80-0x87 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x88-0x8f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90-0x97 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x98-0x9f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa0-0xa7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa8-0xaf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0-0xb7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb8-0xbf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc0-0xc7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc8-0xcf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0-0xd7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd8-0xdf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe0-0xe7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe8-0xef */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf0-0xf7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "macromanian", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_macromanian(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_macromanian(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_macromanian) +module_exit(exit_nls_macromanian) + +MODULE_DESCRIPTION("NLS Codepage macromanian"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/mac-turkish.c b/fs/nls/mac-turkish.c new file mode 100644 index 000000000..d77e9b6b7 --- /dev/null +++ b/fs/nls/mac-turkish.c @@ -0,0 +1,602 @@ +/* + * linux/fs/nls/mac-turkish.c + * + * Charset macturkish translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +/* + * COPYRIGHT AND PERMISSION NOTICE + * + * Copyright 1991-2012 Unicode, Inc. All rights reserved. Distributed under + * the Terms of Use in http://www.unicode.org/copyright.html. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of the Unicode data files and any associated documentation (the "Data + * Files") or Unicode software and any associated documentation (the + * "Software") to deal in the Data Files or Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, and/or sell copies of the Data Files or Software, and + * to permit persons to whom the Data Files or Software are furnished to do + * so, provided that (a) the above copyright notice(s) and this permission + * notice appear with all copies of the Data Files or Software, (b) both the + * above copyright notice(s) and this permission notice appear in associated + * documentation, and (c) there is clear notice in each modified Data File or + * in the Software as well as in the documentation associated with the Data + * File(s) or Software that the data or software has been modified. + * + * THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF + * THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS + * INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT + * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF + * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THE DATA FILES OR SOFTWARE. + * + * Except as contained in this notice, the name of a copyright holder shall + * not be used in advertising or otherwise to promote the sale, use or other + * dealings in these Data Files or Software without prior written + * authorization of the copyright holder. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00 */ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10 */ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20 */ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30 */ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40 */ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50 */ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60 */ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70 */ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80 */ + 0x00c4, 0x00c5, 0x00c7, 0x00c9, + 0x00d1, 0x00d6, 0x00dc, 0x00e1, + 0x00e0, 0x00e2, 0x00e4, 0x00e3, + 0x00e5, 0x00e7, 0x00e9, 0x00e8, + /* 0x90 */ + 0x00ea, 0x00eb, 0x00ed, 0x00ec, + 0x00ee, 0x00ef, 0x00f1, 0x00f3, + 0x00f2, 0x00f4, 0x00f6, 0x00f5, + 0x00fa, 0x00f9, 0x00fb, 0x00fc, + /* 0xa0 */ + 0x2020, 0x00b0, 0x00a2, 0x00a3, + 0x00a7, 0x2022, 0x00b6, 0x00df, + 0x00ae, 0x00a9, 0x2122, 0x00b4, + 0x00a8, 0x2260, 0x00c6, 0x00d8, + /* 0xb0 */ + 0x221e, 0x00b1, 0x2264, 0x2265, + 0x00a5, 0x00b5, 0x2202, 0x2211, + 0x220f, 0x03c0, 0x222b, 0x00aa, + 0x00ba, 0x03a9, 0x00e6, 0x00f8, + /* 0xc0 */ + 0x00bf, 0x00a1, 0x00ac, 0x221a, + 0x0192, 0x2248, 0x2206, 0x00ab, + 0x00bb, 0x2026, 0x00a0, 0x00c0, + 0x00c3, 0x00d5, 0x0152, 0x0153, + /* 0xd0 */ + 0x2013, 0x2014, 0x201c, 0x201d, + 0x2018, 0x2019, 0x00f7, 0x25ca, + 0x00ff, 0x0178, 0x011e, 0x011f, + 0x0130, 0x0131, 0x015e, 0x015f, + /* 0xe0 */ + 0x2021, 0x00b7, 0x201a, 0x201e, + 0x2030, 0x00c2, 0x00ca, 0x00c1, + 0x00cb, 0x00c8, 0x00cd, 0x00ce, + 0x00cf, 0x00cc, 0x00d3, 0x00d4, + /* 0xf0 */ + 0xf8ff, 0x00d2, 0x00da, 0x00db, + 0x00d9, 0xf8a0, 0x02c6, 0x02dc, + 0x00af, 0x02d8, 0x02d9, 0x02da, + 0x00b8, 0x02dd, 0x02db, 0x02c7, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xca, 0xc1, 0xa2, 0xa3, 0x00, 0xb4, 0x00, 0xa4, /* 0xa0-0xa7 */ + 0xac, 0xa9, 0xbb, 0xc7, 0xc2, 0x00, 0xa8, 0xf8, /* 0xa8-0xaf */ + 0xa1, 0xb1, 0x00, 0x00, 0xab, 0xb5, 0xa6, 0xe1, /* 0xb0-0xb7 */ + 0xfc, 0x00, 0xbc, 0xc8, 0x00, 0x00, 0x00, 0xc0, /* 0xb8-0xbf */ + 0xcb, 0xe7, 0xe5, 0xcc, 0x80, 0x81, 0xae, 0x82, /* 0xc0-0xc7 */ + 0xe9, 0x83, 0xe6, 0xe8, 0xed, 0xea, 0xeb, 0xec, /* 0xc8-0xcf */ + 0x00, 0x84, 0xf1, 0xee, 0xef, 0xcd, 0x85, 0x00, /* 0xd0-0xd7 */ + 0xaf, 0xf4, 0xf2, 0xf3, 0x86, 0x00, 0x00, 0xa7, /* 0xd8-0xdf */ + 0x88, 0x87, 0x89, 0x8b, 0x8a, 0x8c, 0xbe, 0x8d, /* 0xe0-0xe7 */ + 0x8f, 0x8e, 0x90, 0x91, 0x93, 0x92, 0x94, 0x95, /* 0xe8-0xef */ + 0x00, 0x96, 0x98, 0x97, 0x99, 0x9b, 0x9a, 0xd6, /* 0xf0-0xf7 */ + 0xbf, 0x9d, 0x9c, 0x9e, 0x9f, 0x00, 0x00, 0xd8, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xda, 0xdb, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0xdc, 0xdd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0xce, 0xcf, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xde, 0xdf, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0xd9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0xc4, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page02[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf6, 0xff, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0xf9, 0xfa, 0xfb, 0xfe, 0xf7, 0xfd, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page03[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0xbd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0xb9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0xd0, 0xd1, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xd4, 0xd5, 0xe2, 0x00, 0xd2, 0xd3, 0xe3, 0x00, /* 0x18-0x1f */ + 0xa0, 0xe0, 0xa5, 0x00, 0x00, 0x00, 0xc9, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0xe4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page21[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0xaa, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page22[256] = { + 0x00, 0x00, 0xb6, 0x00, 0x00, 0x00, 0xc6, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb8, /* 0x08-0x0f */ + 0x00, 0xb7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, 0xb0, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0xba, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0xc5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0xad, 0x00, 0x00, 0x00, 0xb2, 0xb3, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page25[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0xd7, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char pagef8[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xf5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, /* 0xf8-0xff */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, page02, page03, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + page20, page21, page22, NULL, NULL, page25, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + pagef8, NULL, NULL, NULL, NULL, NULL, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x00-0x07 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x08-0x0f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x10-0x17 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x18-0x1f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x20-0x27 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x28-0x2f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30-0x37 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x38-0x3f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x40-0x47 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x48-0x4f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x50-0x57 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x58-0x5f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60-0x67 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x68-0x6f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70-0x77 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x78-0x7f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x80-0x87 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x88-0x8f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90-0x97 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x98-0x9f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa0-0xa7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa8-0xaf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0-0xb7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb8-0xbf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc0-0xc7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc8-0xcf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0-0xd7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd8-0xdf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe0-0xe7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe8-0xef */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf0-0xf7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x00-0x07 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x08-0x0f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x10-0x17 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x18-0x1f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x20-0x27 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x28-0x2f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x30-0x37 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x38-0x3f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x40-0x47 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x48-0x4f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x50-0x57 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x58-0x5f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x60-0x67 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x68-0x6f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x70-0x77 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x78-0x7f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x80-0x87 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x88-0x8f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x90-0x97 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0x98-0x9f */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa0-0xa7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xa8-0xaf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb0-0xb7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xb8-0xbf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc0-0xc7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xc8-0xcf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd0-0xd7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xd8-0xdf */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe0-0xe7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xe8-0xef */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf0-0xf7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "macturkish", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_macturkish(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_macturkish(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_macturkish) +module_exit(exit_nls_macturkish) + +MODULE_DESCRIPTION("NLS Codepage macturkish"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_ascii.c b/fs/nls/nls_ascii.c new file mode 100644 index 000000000..068143d71 --- /dev/null +++ b/fs/nls/nls_ascii.c @@ -0,0 +1,167 @@ +/* + * linux/fs/nls/nls_ascii.c + * + * Charset ascii translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "ascii", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_ascii(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_ascii(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_ascii) +module_exit(exit_nls_ascii) + +MODULE_DESCRIPTION("NLS ASCII (United States)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_base.c b/fs/nls/nls_base.c new file mode 100644 index 000000000..a5c3a9f1b --- /dev/null +++ b/fs/nls/nls_base.c @@ -0,0 +1,560 @@ +/* + * linux/fs/nls/nls_base.c + * + * Native language support--charsets and unicode translations. + * By Gordon Chaffee 1996, 1997 + * + * Unicode based case conversion 1999 by Wolfram Pienkoss + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +static struct nls_table default_table; +static struct nls_table *tables = &default_table; +static DEFINE_SPINLOCK(nls_lock); + +/* + * Sample implementation from Unicode home page. + * http://www.stonehand.com/unicode/standard/fss-utf.html + */ +struct utf8_table { + int cmask; + int cval; + int shift; + long lmask; + long lval; +}; + +static const struct utf8_table utf8_table[] = +{ + {0x80, 0x00, 0*6, 0x7F, 0, /* 1 byte sequence */}, + {0xE0, 0xC0, 1*6, 0x7FF, 0x80, /* 2 byte sequence */}, + {0xF0, 0xE0, 2*6, 0xFFFF, 0x800, /* 3 byte sequence */}, + {0xF8, 0xF0, 3*6, 0x1FFFFF, 0x10000, /* 4 byte sequence */}, + {0xFC, 0xF8, 4*6, 0x3FFFFFF, 0x200000, /* 5 byte sequence */}, + {0xFE, 0xFC, 5*6, 0x7FFFFFFF, 0x4000000, /* 6 byte sequence */}, + {0, /* end of table */} +}; + +#define UNICODE_MAX 0x0010ffff +#define PLANE_SIZE 0x00010000 + +#define SURROGATE_MASK 0xfffff800 +#define SURROGATE_PAIR 0x0000d800 +#define SURROGATE_LOW 0x00000400 +#define SURROGATE_BITS 0x000003ff + +int utf8_to_utf32(const u8 *s, int inlen, unicode_t *pu) +{ + unsigned long l; + int c0, c, nc; + const struct utf8_table *t; + + nc = 0; + c0 = *s; + l = c0; + for (t = utf8_table; t->cmask; t++) { + nc++; + if ((c0 & t->cmask) == t->cval) { + l &= t->lmask; + if (l < t->lval || l > UNICODE_MAX || + (l & SURROGATE_MASK) == SURROGATE_PAIR) + return -EILSEQ; + + *pu = (unicode_t) l; + return nc; + } + if (inlen <= nc) + return -EOVERFLOW; + + s++; + c = (*s ^ 0x80) & 0xFF; + if (c & 0xC0) + return -EILSEQ; + + l = (l << 6) | c; + } + return -EILSEQ; +} +EXPORT_SYMBOL(utf8_to_utf32); + +int utf32_to_utf8(unicode_t u, u8 *s, int maxout) +{ + unsigned long l; + int c, nc; + const struct utf8_table *t; + + if (!s) + return 0; + + l = u; + if (l > UNICODE_MAX || (l & SURROGATE_MASK) == SURROGATE_PAIR) + return -EILSEQ; + + nc = 0; + for (t = utf8_table; t->cmask && maxout; t++, maxout--) { + nc++; + if (l <= t->lmask) { + c = t->shift; + *s = (u8) (t->cval | (l >> c)); + while (c > 0) { + c -= 6; + s++; + *s = (u8) (0x80 | ((l >> c) & 0x3F)); + } + return nc; + } + } + return -EOVERFLOW; +} +EXPORT_SYMBOL(utf32_to_utf8); + +static inline void put_utf16(wchar_t *s, unsigned c, enum utf16_endian endian) +{ + switch (endian) { + default: + *s = (wchar_t) c; + break; + case UTF16_LITTLE_ENDIAN: + *s = __cpu_to_le16(c); + break; + case UTF16_BIG_ENDIAN: + *s = __cpu_to_be16(c); + break; + } +} + +int utf8s_to_utf16s(const u8 *s, int inlen, enum utf16_endian endian, + wchar_t *pwcs, int maxout) +{ + u16 *op; + int size; + unicode_t u; + + op = pwcs; + while (inlen > 0 && maxout > 0 && *s) { + if (*s & 0x80) { + size = utf8_to_utf32(s, inlen, &u); + if (size < 0) + return -EINVAL; + s += size; + inlen -= size; + + if (u >= PLANE_SIZE) { + if (maxout < 2) + break; + u -= PLANE_SIZE; + put_utf16(op++, SURROGATE_PAIR | + ((u >> 10) & SURROGATE_BITS), + endian); + put_utf16(op++, SURROGATE_PAIR | + SURROGATE_LOW | + (u & SURROGATE_BITS), + endian); + maxout -= 2; + } else { + put_utf16(op++, u, endian); + maxout--; + } + } else { + put_utf16(op++, *s++, endian); + inlen--; + maxout--; + } + } + return op - pwcs; +} +EXPORT_SYMBOL(utf8s_to_utf16s); + +static inline unsigned long get_utf16(unsigned c, enum utf16_endian endian) +{ + switch (endian) { + default: + return c; + case UTF16_LITTLE_ENDIAN: + return __le16_to_cpu(c); + case UTF16_BIG_ENDIAN: + return __be16_to_cpu(c); + } +} + +int utf16s_to_utf8s(const wchar_t *pwcs, int inlen, enum utf16_endian endian, + u8 *s, int maxout) +{ + u8 *op; + int size; + unsigned long u, v; + + op = s; + while (inlen > 0 && maxout > 0) { + u = get_utf16(*pwcs, endian); + if (!u) + break; + pwcs++; + inlen--; + if (u > 0x7f) { + if ((u & SURROGATE_MASK) == SURROGATE_PAIR) { + if (u & SURROGATE_LOW) { + /* Ignore character and move on */ + continue; + } + if (inlen <= 0) + break; + v = get_utf16(*pwcs, endian); + if ((v & SURROGATE_MASK) != SURROGATE_PAIR || + !(v & SURROGATE_LOW)) { + /* Ignore character and move on */ + continue; + } + u = PLANE_SIZE + ((u & SURROGATE_BITS) << 10) + + (v & SURROGATE_BITS); + pwcs++; + inlen--; + } + size = utf32_to_utf8(u, op, maxout); + if (size < 0) { + if (size == -EILSEQ) { + /* Ignore character and move on */ + continue; + } + /* + * Stop filling the buffer with data once a character + * does not fit anymore. + */ + break; + } else { + op += size; + maxout -= size; + } + } else { + *op++ = (u8) u; + maxout--; + } + } + return op - s; +} +EXPORT_SYMBOL(utf16s_to_utf8s); + +int __register_nls(struct nls_table *nls, struct module *owner) +{ + struct nls_table ** tmp = &tables; + + if (nls->next) + return -EBUSY; + + nls->owner = owner; + spin_lock(&nls_lock); + while (*tmp) { + if (nls == *tmp) { + spin_unlock(&nls_lock); + return -EBUSY; + } + tmp = &(*tmp)->next; + } + nls->next = tables; + tables = nls; + spin_unlock(&nls_lock); + return 0; +} +EXPORT_SYMBOL(__register_nls); + +int unregister_nls(struct nls_table * nls) +{ + struct nls_table ** tmp = &tables; + + spin_lock(&nls_lock); + while (*tmp) { + if (nls == *tmp) { + *tmp = nls->next; + spin_unlock(&nls_lock); + return 0; + } + tmp = &(*tmp)->next; + } + spin_unlock(&nls_lock); + return -EINVAL; +} + +static struct nls_table *find_nls(const char *charset) +{ + struct nls_table *nls; + spin_lock(&nls_lock); + for (nls = tables; nls; nls = nls->next) { + if (!strcmp(nls->charset, charset)) + break; + if (nls->alias && !strcmp(nls->alias, charset)) + break; + } + if (nls && !try_module_get(nls->owner)) + nls = NULL; + spin_unlock(&nls_lock); + return nls; +} + +struct nls_table *load_nls(const char *charset) +{ + return try_then_request_module(find_nls(charset), "nls_%s", charset); +} + +void unload_nls(struct nls_table *nls) +{ + if (nls) + module_put(nls->owner); +} + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x0080, 0x0081, 0x0082, 0x0083, + 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, + 0x008c, 0x008d, 0x008e, 0x008f, + /* 0x90*/ + 0x0090, 0x0091, 0x0092, 0x0093, + 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, + 0x009c, 0x009d, 0x009e, 0x009f, + /* 0xa0*/ + 0x00a0, 0x00a1, 0x00a2, 0x00a3, + 0x00a4, 0x00a5, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0x00aa, 0x00ab, + 0x00ac, 0x00ad, 0x00ae, 0x00af, + /* 0xb0*/ + 0x00b0, 0x00b1, 0x00b2, 0x00b3, + 0x00b4, 0x00b5, 0x00b6, 0x00b7, + 0x00b8, 0x00b9, 0x00ba, 0x00bb, + 0x00bc, 0x00bd, 0x00be, 0x00bf, + /* 0xc0*/ + 0x00c0, 0x00c1, 0x00c2, 0x00c3, + 0x00c4, 0x00c5, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, + 0x00cc, 0x00cd, 0x00ce, 0x00cf, + /* 0xd0*/ + 0x00d0, 0x00d1, 0x00d2, 0x00d3, + 0x00d4, 0x00d5, 0x00d6, 0x00d7, + 0x00d8, 0x00d9, 0x00da, 0x00db, + 0x00dc, 0x00dd, 0x00de, 0x00df, + /* 0xe0*/ + 0x00e0, 0x00e1, 0x00e2, 0x00e3, + 0x00e4, 0x00e5, 0x00e6, 0x00e7, + 0x00e8, 0x00e9, 0x00ea, 0x00eb, + 0x00ec, 0x00ed, 0x00ee, 0x00ef, + /* 0xf0*/ + 0x00f0, 0x00f1, 0x00f2, 0x00f3, + 0x00f4, 0x00f5, 0x00f6, 0x00f7, + 0x00f8, 0x00f9, 0x00fa, 0x00fb, + 0x00fc, 0x00fd, 0x00fe, 0x00ff, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00 +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table default_table = { + .charset = "default", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +/* Returns a simple default translation table */ +struct nls_table *load_nls_default(void) +{ + struct nls_table *default_nls; + + default_nls = load_nls(CONFIG_NLS_DEFAULT); + if (default_nls != NULL) + return default_nls; + else + return &default_table; +} + +EXPORT_SYMBOL(unregister_nls); +EXPORT_SYMBOL(unload_nls); +EXPORT_SYMBOL(load_nls); +EXPORT_SYMBOL(load_nls_default); + +MODULE_DESCRIPTION("Base file system native language support"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_cp1250.c b/fs/nls/nls_cp1250.c new file mode 100644 index 000000000..e22a57a4b --- /dev/null +++ b/fs/nls/nls_cp1250.c @@ -0,0 +1,347 @@ +/* + * linux/fs/nls/nls_cp1250.c + * + * Charset cp1250 translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x20ac, 0x0000, 0x201a, 0x0000, + 0x201e, 0x2026, 0x2020, 0x2021, + 0x0000, 0x2030, 0x0160, 0x2039, + 0x015a, 0x0164, 0x017d, 0x0179, + /* 0x90*/ + 0x0000, 0x2018, 0x2019, 0x201c, + 0x201d, 0x2022, 0x2013, 0x2014, + 0x0000, 0x2122, 0x0161, 0x203a, + 0x015b, 0x0165, 0x017e, 0x017a, + /* 0xa0*/ + 0x00a0, 0x02c7, 0x02d8, 0x0141, + 0x00a4, 0x0104, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0x015e, 0x00ab, + 0x00ac, 0x00ad, 0x00ae, 0x017b, + /* 0xb0*/ + 0x00b0, 0x00b1, 0x02db, 0x0142, + 0x00b4, 0x00b5, 0x00b6, 0x00b7, + 0x00b8, 0x0105, 0x015f, 0x00bb, + 0x013d, 0x02dd, 0x013e, 0x017c, + /* 0xc0*/ + 0x0154, 0x00c1, 0x00c2, 0x0102, + 0x00c4, 0x0139, 0x0106, 0x00c7, + 0x010c, 0x00c9, 0x0118, 0x00cb, + 0x011a, 0x00cd, 0x00ce, 0x010e, + /* 0xd0*/ + 0x0110, 0x0143, 0x0147, 0x00d3, + 0x00d4, 0x0150, 0x00d6, 0x00d7, + 0x0158, 0x016e, 0x00da, 0x0170, + 0x00dc, 0x00dd, 0x0162, 0x00df, + /* 0xe0*/ + 0x0155, 0x00e1, 0x00e2, 0x0103, + 0x00e4, 0x013a, 0x0107, 0x00e7, + 0x010d, 0x00e9, 0x0119, 0x00eb, + 0x011b, 0x00ed, 0x00ee, 0x010f, + /* 0xf0*/ + 0x0111, 0x0144, 0x0148, 0x00f3, + 0x00f4, 0x0151, 0x00f6, 0x00f7, + 0x0159, 0x016f, 0x00fa, 0x0171, + 0x00fc, 0x00fd, 0x0163, 0x02d9, + }; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xa0, 0x00, 0x00, 0x00, 0xa4, 0x00, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0x00, 0xab, 0xac, 0xad, 0xae, 0x00, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0x00, 0x00, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0x00, 0x00, 0xbb, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0xc1, 0xc2, 0x00, 0xc4, 0x00, 0x00, 0xc7, /* 0xc0-0xc7 */ + 0x00, 0xc9, 0x00, 0xcb, 0x00, 0xcd, 0xce, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0xd3, 0xd4, 0x00, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0x00, 0x00, 0xda, 0x00, 0xdc, 0xdd, 0x00, 0xdf, /* 0xd8-0xdf */ + 0x00, 0xe1, 0xe2, 0x00, 0xe4, 0x00, 0x00, 0xe7, /* 0xe0-0xe7 */ + 0x00, 0xe9, 0x00, 0xeb, 0x00, 0xed, 0xee, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0xf3, 0xf4, 0x00, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0x00, 0x00, 0xfa, 0x00, 0xfc, 0xfd, 0x00, 0x00, /* 0xf8-0xff */ + }; + +static const unsigned char page01[256] = { + 0x00, 0x00, 0xc3, 0xe3, 0xa5, 0xb9, 0xc6, 0xe6, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xc8, 0xe8, 0xcf, 0xef, /* 0x08-0x0f */ + 0xd0, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xca, 0xea, 0xcc, 0xec, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0xc5, 0xe5, 0x00, 0x00, 0xbc, 0xbe, 0x00, /* 0x38-0x3f */ + 0x00, 0xa3, 0xb3, 0xd1, 0xf1, 0x00, 0x00, 0xd2, /* 0x40-0x47 */ + 0xf2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0xd5, 0xf5, 0x00, 0x00, 0xc0, 0xe0, 0x00, 0x00, /* 0x50-0x57 */ + 0xd8, 0xf8, 0x8c, 0x9c, 0x00, 0x00, 0xaa, 0xba, /* 0x58-0x5f */ + 0x8a, 0x9a, 0xde, 0xfe, 0x8d, 0x9d, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd9, 0xf9, /* 0x68-0x6f */ + 0xdb, 0xfb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x8f, 0x9f, 0xaf, 0xbf, 0x8e, 0x9e, 0x00, /* 0x78-0x7f */ + + }; + +static const unsigned char page02[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa1, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0xa2, 0xff, 0x00, 0xb2, 0x00, 0xbd, 0x00, 0x00, /* 0xd8-0xdf */ + }; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x96, 0x97, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x91, 0x92, 0x82, 0x00, 0x93, 0x94, 0x84, 0x00, /* 0x18-0x1f */ + 0x86, 0x87, 0x95, 0x00, 0x00, 0x00, 0x85, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x89, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x8b, 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + }; + +static const unsigned char page21[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x99, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + }; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, page02, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + page20, page21, NULL, NULL, NULL, NULL, NULL, NULL, + }; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x00, 0x82, 0x00, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x00, 0x89, 0x9a, 0x8b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x88-0x8f */ + 0x00, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x00, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xb3, 0xa4, 0xb9, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xba, 0xab, 0xac, 0xad, 0xae, 0xbf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbe, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xc0-0xc7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xc8-0xcf */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xd7, /* 0xd0-0xd7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ + }; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x00, 0x82, 0x00, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x00, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x00, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x00, 0x99, 0x8a, 0x9b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xa3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xa5, 0xaa, 0xbb, 0xbc, 0xbd, 0xbc, 0xaf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0x00, /* 0xd8-0xdf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xe0-0xe7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xe8-0xef */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xf7, /* 0xf0-0xf7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xff, /* 0xf8-0xff */ + }; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "cp1250", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_cp1250(void) +{ + return register_nls(&table); +} +static void __exit exit_nls_cp1250(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_cp1250) +module_exit(exit_nls_cp1250) + +MODULE_DESCRIPTION("NLS Windows CP1250 (Slavic/Central European Languages)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_cp1251.c b/fs/nls/nls_cp1251.c new file mode 100644 index 000000000..6f46d339f --- /dev/null +++ b/fs/nls/nls_cp1251.c @@ -0,0 +1,302 @@ +/* + * linux/fs/nls/nls_cp1251.c + * + * Charset cp1251 translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x0402, 0x0403, 0x201a, 0x0453, + 0x201e, 0x2026, 0x2020, 0x2021, + 0x20ac, 0x2030, 0x0409, 0x2039, + 0x040a, 0x040c, 0x040b, 0x040f, + /* 0x90*/ + 0x0452, 0x2018, 0x2019, 0x201c, + 0x201d, 0x2022, 0x2013, 0x2014, + 0x0000, 0x2122, 0x0459, 0x203a, + 0x045a, 0x045c, 0x045b, 0x045f, + /* 0xa0*/ + 0x00a0, 0x040e, 0x045e, 0x0408, + 0x00a4, 0x0490, 0x00a6, 0x00a7, + 0x0401, 0x00a9, 0x0404, 0x00ab, + 0x00ac, 0x00ad, 0x00ae, 0x0407, + /* 0xb0*/ + 0x00b0, 0x00b1, 0x0406, 0x0456, + 0x0491, 0x00b5, 0x00b6, 0x00b7, + 0x0451, 0x2116, 0x0454, 0x00bb, + 0x0458, 0x0405, 0x0455, 0x0457, + /* 0xc0*/ + 0x0410, 0x0411, 0x0412, 0x0413, + 0x0414, 0x0415, 0x0416, 0x0417, + 0x0418, 0x0419, 0x041a, 0x041b, + 0x041c, 0x041d, 0x041e, 0x041f, + /* 0xd0*/ + 0x0420, 0x0421, 0x0422, 0x0423, + 0x0424, 0x0425, 0x0426, 0x0427, + 0x0428, 0x0429, 0x042a, 0x042b, + 0x042c, 0x042d, 0x042e, 0x042f, + /* 0xe0*/ + 0x0430, 0x0431, 0x0432, 0x0433, + 0x0434, 0x0435, 0x0436, 0x0437, + 0x0438, 0x0439, 0x043a, 0x043b, + 0x043c, 0x043d, 0x043e, 0x043f, + /* 0xf0*/ + 0x0440, 0x0441, 0x0442, 0x0443, + 0x0444, 0x0445, 0x0446, 0x0447, + 0x0448, 0x0449, 0x044a, 0x044b, + 0x044c, 0x044d, 0x044e, 0x044f, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xa0, 0x00, 0x00, 0x00, 0xa4, 0x00, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0x00, 0xa9, 0x00, 0xab, 0xac, 0xad, 0xae, 0x00, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0x00, 0x00, 0x00, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0xbb, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0xbb, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0xbb, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0xbb, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0xbb, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0xbb, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0xbb, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0xbb, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0xbb, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page04[256] = { + 0x00, 0xa8, 0x80, 0x81, 0xaa, 0xbd, 0xb2, 0xaf, /* 0x00-0x07 */ + 0xa3, 0x8a, 0x8c, 0x8e, 0x8d, 0x00, 0xa1, 0x8f, /* 0x08-0x0f */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0x10-0x17 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0x18-0x1f */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0x20-0x27 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0x28-0x2f */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0x30-0x37 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0x38-0x3f */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0x40-0x47 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0x48-0x4f */ + 0x00, 0xb8, 0x90, 0x83, 0xba, 0xbe, 0xb3, 0xbf, /* 0x50-0x57 */ + 0xbc, 0x9a, 0x9c, 0x9e, 0x9d, 0x00, 0xa2, 0x9f, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0xa5, 0xb4, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x96, 0x97, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x91, 0x92, 0x82, 0x00, 0x93, 0x94, 0x84, 0x00, /* 0x18-0x1f */ + 0x86, 0x87, 0x95, 0x00, 0x00, 0x00, 0x85, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x89, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x8b, 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ +}; + +static const unsigned char page21[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb9, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x99, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, NULL, NULL, NULL, page04, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + page20, page21, NULL, NULL, NULL, NULL, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + 0x90, 0x83, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x9a, 0x8b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa2, 0xa2, 0xbc, 0xa4, 0xb4, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xb8, 0xa9, 0xba, 0xab, 0xac, 0xad, 0xae, 0xbf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb3, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbe, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xc0-0xc7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xc8-0xcf */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xd0-0xd7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + 0x80, 0x81, 0x82, 0x81, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x80, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x8a, 0x9b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa1, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb2, 0xa5, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xa8, 0xb9, 0xaa, 0xbb, 0xa3, 0xbd, 0xbd, 0xaf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xe0-0xe7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xe8-0xef */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xf0-0xf7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "cp1251", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_cp1251(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_cp1251(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_cp1251) +module_exit(exit_nls_cp1251) + +MODULE_DESCRIPTION("NLS Windows CP1251 (Bulgarian, Belarusian)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_cp1255.c b/fs/nls/nls_cp1255.c new file mode 100644 index 000000000..299e089d4 --- /dev/null +++ b/fs/nls/nls_cp1255.c @@ -0,0 +1,385 @@ +/* + * linux/fs/nls/nls_cp1255.c + * + * Charset cp1255 translation tables. + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x20ac, 0x0000, 0x201a, 0x0192, + 0x201e, 0x2026, 0x2020, 0x2021, + 0x02c6, 0x2030, 0x0000, 0x2039, + 0x0000, 0x0000, 0x0000, 0x0000, + /* 0x90*/ + 0x0000, 0x2018, 0x2019, 0x201c, + 0x201d, 0x2022, 0x2013, 0x2014, + 0x02dc, 0x2122, 0x0000, 0x203a, + 0x0000, 0x0000, 0x0000, 0x0000, + /* 0xa0*/ + 0x00a0, 0x00a1, 0x00a2, 0x00a3, + 0x20aa, 0x00a5, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0x00d7, 0x00ab, + 0x00ac, 0x00ad, 0x00ae, 0x203e, + /* 0xb0*/ + 0x00b0, 0x00b1, 0x00b2, 0x00b3, + 0x00b4, 0x00b5, 0x00b6, 0x00b7, + 0x00b8, 0x00b9, 0x00f7, 0x00bb, + 0x00bc, 0x00bd, 0x00be, 0x00bf, + /* 0xc0*/ + 0x05b0, 0x05b1, 0x05b2, 0x05b3, + 0x05b4, 0x05b5, 0x05b6, 0x05b7, + 0x05b8, 0x05b9, 0x0000, 0x05bb, + 0x05bc, 0x05bd, 0x05be, 0x05bf, + /* 0xd0*/ + 0x05c0, 0x05c1, 0x05c2, 0x05c3, + 0x05f0, 0x05f1, 0x05f2, 0x05f3, + 0x05f4, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x2017, + /* 0xe0*/ + 0x05d0, 0x05d1, 0x05d2, 0x05d3, + 0x05d4, 0x05d5, 0x05d6, 0x05d7, + 0x05d8, 0x05d9, 0x05da, 0x05db, + 0x05dc, 0x05dd, 0x05de, 0x05df, + /* 0xf0*/ + 0x05e0, 0x05e1, 0x05e2, 0x05e3, + 0x05e4, 0x05e5, 0x05e6, 0x05e7, + 0x05e8, 0x05e9, 0x05ea, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xa0, 0x00, 0xa2, 0xa3, 0x00, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0x00, 0xab, 0xac, 0xad, 0xae, 0x00, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0x00, 0xbb, 0xbc, 0xbd, 0xbe, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xaa, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xba, /* 0xf0-0xf7 */ +}; + +static const unsigned char page01[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x83, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ +}; + +static const unsigned char page02[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x98, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ +}; + +static const unsigned char page05[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xb0-0xb7 */ + 0xc8, 0xc9, 0x00, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xb8-0xbf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xd0-0xd7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xd8-0xdf */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xe0-0xe7 */ + 0xf8, 0xf9, 0xfa, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfd, 0xfe, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x96, 0x97, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x91, 0x92, 0x82, 0x00, 0x93, 0x94, 0x84, 0x00, /* 0x18-0x1f */ + 0x86, 0x87, 0x95, 0x00, 0x00, 0x00, 0x85, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x89, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x8b, 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0xa4, 0x00, 0x80, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ +}; + +static const unsigned char page21[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb9, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x99, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, page02, NULL, NULL, page05, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + page20, page21, NULL, NULL, NULL, NULL, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xa0, 0x00, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xa0, 0x00, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0x00, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "cp1255", + .alias = "iso8859-8", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_cp1255(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_cp1255(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_cp1255) +module_exit(exit_nls_cp1255) + +MODULE_DESCRIPTION("NLS Hebrew charsets (ISO-8859-8, CP1255)"); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_ALIAS_NLS(iso8859-8); diff --git a/fs/nls/nls_cp437.c b/fs/nls/nls_cp437.c new file mode 100644 index 000000000..ab880499e --- /dev/null +++ b/fs/nls/nls_cp437.c @@ -0,0 +1,388 @@ +/* + * linux/fs/nls/nls_cp437.c + * + * Charset cp437 translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x00c7, 0x00fc, 0x00e9, 0x00e2, + 0x00e4, 0x00e0, 0x00e5, 0x00e7, + 0x00ea, 0x00eb, 0x00e8, 0x00ef, + 0x00ee, 0x00ec, 0x00c4, 0x00c5, + /* 0x90*/ + 0x00c9, 0x00e6, 0x00c6, 0x00f4, + 0x00f6, 0x00f2, 0x00fb, 0x00f9, + 0x00ff, 0x00d6, 0x00dc, 0x00a2, + 0x00a3, 0x00a5, 0x20a7, 0x0192, + /* 0xa0*/ + 0x00e1, 0x00ed, 0x00f3, 0x00fa, + 0x00f1, 0x00d1, 0x00aa, 0x00ba, + 0x00bf, 0x2310, 0x00ac, 0x00bd, + 0x00bc, 0x00a1, 0x00ab, 0x00bb, + /* 0xb0*/ + 0x2591, 0x2592, 0x2593, 0x2502, + 0x2524, 0x2561, 0x2562, 0x2556, + 0x2555, 0x2563, 0x2551, 0x2557, + 0x255d, 0x255c, 0x255b, 0x2510, + /* 0xc0*/ + 0x2514, 0x2534, 0x252c, 0x251c, + 0x2500, 0x253c, 0x255e, 0x255f, + 0x255a, 0x2554, 0x2569, 0x2566, + 0x2560, 0x2550, 0x256c, 0x2567, + /* 0xd0*/ + 0x2568, 0x2564, 0x2565, 0x2559, + 0x2558, 0x2552, 0x2553, 0x256b, + 0x256a, 0x2518, 0x250c, 0x2588, + 0x2584, 0x258c, 0x2590, 0x2580, + /* 0xe0*/ + 0x03b1, 0x00df, 0x0393, 0x03c0, + 0x03a3, 0x03c3, 0x00b5, 0x03c4, + 0x03a6, 0x0398, 0x03a9, 0x03b4, + 0x221e, 0x03c6, 0x03b5, 0x2229, + /* 0xf0*/ + 0x2261, 0x00b1, 0x2265, 0x2264, + 0x2320, 0x2321, 0x00f7, 0x2248, + 0x00b0, 0x2219, 0x00b7, 0x221a, + 0x207f, 0x00b2, 0x25a0, 0x00a0, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xff, 0xad, 0x9b, 0x9c, 0x00, 0x9d, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0xa6, 0xae, 0xaa, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0xf8, 0xf1, 0xfd, 0x00, 0x00, 0xe6, 0x00, 0xfa, /* 0xb0-0xb7 */ + 0x00, 0x00, 0xa7, 0xaf, 0xac, 0xab, 0x00, 0xa8, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x8e, 0x8f, 0x92, 0x80, /* 0xc0-0xc7 */ + 0x00, 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0xa5, 0x00, 0x00, 0x00, 0x00, 0x99, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x9a, 0x00, 0x00, 0xe1, /* 0xd8-0xdf */ + 0x85, 0xa0, 0x83, 0x00, 0x84, 0x86, 0x91, 0x87, /* 0xe0-0xe7 */ + 0x8a, 0x82, 0x88, 0x89, 0x8d, 0xa1, 0x8c, 0x8b, /* 0xe8-0xef */ + 0x00, 0xa4, 0x95, 0xa2, 0x93, 0x00, 0x94, 0xf6, /* 0xf0-0xf7 */ + 0x00, 0x97, 0xa3, 0x96, 0x81, 0x00, 0x00, 0x98, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x9f, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ +}; + +static const unsigned char page03[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0xe2, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0xe9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0xe4, 0x00, 0x00, 0xe8, 0x00, /* 0xa0-0xa7 */ + 0x00, 0xea, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0xe0, 0x00, 0x00, 0xeb, 0xee, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0xe3, 0x00, 0x00, 0xe5, 0xe7, 0x00, 0xed, 0x00, /* 0xc0-0xc7 */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfc, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9e, /* 0xa0-0xa7 */ +}; + +static const unsigned char page22[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0xf9, 0xfb, 0x00, 0x00, 0x00, 0xec, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0xef, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0xf7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0xf0, 0x00, 0x00, 0xf3, 0xf2, 0x00, 0x00, /* 0x60-0x67 */ +}; + +static const unsigned char page23[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0xa9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0xf4, 0xf5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ +}; + +static const unsigned char page25[256] = { + 0xc4, 0x00, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0xbf, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xd9, 0x00, 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xc2, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xc5, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0xcd, 0xba, 0xd5, 0xd6, 0xc9, 0xb8, 0xb7, 0xbb, /* 0x50-0x57 */ + 0xd4, 0xd3, 0xc8, 0xbe, 0xbd, 0xbc, 0xc6, 0xc7, /* 0x58-0x5f */ + 0xcc, 0xb5, 0xb6, 0xb9, 0xd1, 0xd2, 0xcb, 0xcf, /* 0x60-0x67 */ + 0xd0, 0xca, 0xd8, 0xd7, 0xce, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0xdf, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0xdb, 0x00, 0x00, 0x00, 0xdd, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0xde, 0xb0, 0xb1, 0xb2, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, NULL, page03, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + page20, NULL, page22, page23, NULL, page25, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x87, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x84, 0x86, /* 0x88-0x8f */ + 0x82, 0x91, 0x91, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x94, 0x81, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa4, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0x00, 0xe3, 0xe5, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xed, 0x00, 0x00, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x9a, 0x90, 0x00, 0x8e, 0x00, 0x8f, 0x80, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x92, 0x92, 0x00, 0x99, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0xa5, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0x00, 0xe1, 0xe2, 0x00, 0xe4, 0xe4, 0x00, 0x00, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0x00, 0xec, 0xe8, 0x00, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "cp437", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_cp437(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_cp437(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_cp437) +module_exit(exit_nls_cp437) + +MODULE_DESCRIPTION("NLS Codepage 437 (United States, Canada)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_cp737.c b/fs/nls/nls_cp737.c new file mode 100644 index 000000000..5c3761829 --- /dev/null +++ b/fs/nls/nls_cp737.c @@ -0,0 +1,351 @@ +/* + * linux/fs/nls/nls_cp737.c + * + * Charset cp737 translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x0391, 0x0392, 0x0393, 0x0394, + 0x0395, 0x0396, 0x0397, 0x0398, + 0x0399, 0x039a, 0x039b, 0x039c, + 0x039d, 0x039e, 0x039f, 0x03a0, + /* 0x90*/ + 0x03a1, 0x03a3, 0x03a4, 0x03a5, + 0x03a6, 0x03a7, 0x03a8, 0x03a9, + 0x03b1, 0x03b2, 0x03b3, 0x03b4, + 0x03b5, 0x03b6, 0x03b7, 0x03b8, + /* 0xa0*/ + 0x03b9, 0x03ba, 0x03bb, 0x03bc, + 0x03bd, 0x03be, 0x03bf, 0x03c0, + 0x03c1, 0x03c3, 0x03c2, 0x03c4, + 0x03c5, 0x03c6, 0x03c7, 0x03c8, + /* 0xb0*/ + 0x2591, 0x2592, 0x2593, 0x2502, + 0x2524, 0x2561, 0x2562, 0x2556, + 0x2555, 0x2563, 0x2551, 0x2557, + 0x255d, 0x255c, 0x255b, 0x2510, + /* 0xc0*/ + 0x2514, 0x2534, 0x252c, 0x251c, + 0x2500, 0x253c, 0x255e, 0x255f, + 0x255a, 0x2554, 0x2569, 0x2566, + 0x2560, 0x2550, 0x256c, 0x2567, + /* 0xd0*/ + 0x2568, 0x2564, 0x2565, 0x2559, + 0x2558, 0x2552, 0x2553, 0x256b, + 0x256a, 0x2518, 0x250c, 0x2588, + 0x2584, 0x258c, 0x2590, 0x2580, + /* 0xe0*/ + 0x03c9, 0x03ac, 0x03ad, 0x03ae, + 0x03ca, 0x03af, 0x03cc, 0x03cd, + 0x03cb, 0x03ce, 0x0386, 0x0388, + 0x0389, 0x038a, 0x038c, 0x038e, + /* 0xf0*/ + 0x038f, 0x00b1, 0x2265, 0x2264, + 0x03aa, 0x03ab, 0x00f7, 0x2248, + 0x00b0, 0x2219, 0x00b7, 0x221a, + 0x207f, 0x00b2, 0x25a0, 0x00a0, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0xf8, 0xf1, 0xfd, 0x00, 0x00, 0x00, 0x00, 0xfa, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf6, /* 0xf0-0xf7 */ +}; + +static const unsigned char page03[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xea, 0x00, /* 0x80-0x87 */ + 0xeb, 0xec, 0xed, 0x00, 0xee, 0x00, 0xef, 0xf0, /* 0x88-0x8f */ + 0x00, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, /* 0x90-0x97 */ + 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, /* 0x98-0x9f */ + 0x8f, 0x90, 0x00, 0x91, 0x92, 0x93, 0x94, 0x95, /* 0xa0-0xa7 */ + 0x96, 0x97, 0xf4, 0xf5, 0xe1, 0xe2, 0xe3, 0xe5, /* 0xa8-0xaf */ + 0x00, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, /* 0xb0-0xb7 */ + 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, /* 0xb8-0xbf */ + 0xa7, 0xa8, 0xaa, 0xa9, 0xab, 0xac, 0xad, 0xae, /* 0xc0-0xc7 */ + 0xaf, 0xe0, 0xe4, 0xe8, 0xe6, 0xe7, 0xe9, 0x00, /* 0xc8-0xcf */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfc, /* 0x78-0x7f */ +}; + +static const unsigned char page22[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0xf9, 0xfb, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0xf7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0xf3, 0xf2, 0x00, 0x00, /* 0x60-0x67 */ +}; + +static const unsigned char page25[256] = { + 0xc4, 0x00, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0xbf, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xd9, 0x00, 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xc2, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xc5, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0xcd, 0xba, 0xd5, 0xd6, 0xc9, 0xb8, 0xb7, 0xbb, /* 0x50-0x57 */ + 0xd4, 0xd3, 0xc8, 0xbe, 0xbd, 0xbc, 0xc6, 0xc7, /* 0x58-0x5f */ + 0xcc, 0xb5, 0xb6, 0xb9, 0xd1, 0xd2, 0xcb, 0xcf, /* 0x60-0x67 */ + 0xd0, 0xca, 0xd8, 0xd7, 0xce, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0xdf, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0xdb, 0x00, 0x00, 0x00, 0xdd, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0xde, 0xb0, 0xb1, 0xb2, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, NULL, NULL, page03, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + page20, NULL, page22, NULL, NULL, page25, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x80-0x87 */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0x88-0x8f */ + 0xa8, 0xa9, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xe0, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xe1, 0xe2, 0xe3, 0xe5, 0xe6, 0xe7, /* 0xe8-0xef */ + 0xe9, 0xf1, 0xf2, 0xf3, 0xe4, 0xe8, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x98-0x9f */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0xa0-0xa7 */ + 0x90, 0x91, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0x97, 0xea, 0xeb, 0xec, 0xf4, 0xed, 0xee, 0xef, /* 0xe0-0xe7 */ + 0xf5, 0xf0, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "cp737", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_cp737(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_cp737(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_cp737) +module_exit(exit_nls_cp737) + +MODULE_DESCRIPTION("NLS Codepage 737 (Greek)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_cp775.c b/fs/nls/nls_cp775.c new file mode 100644 index 000000000..51ccc9089 --- /dev/null +++ b/fs/nls/nls_cp775.c @@ -0,0 +1,320 @@ +/* + * linux/fs/nls/nls_cp775.c + * + * Charset cp775 translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x0106, 0x00fc, 0x00e9, 0x0101, + 0x00e4, 0x0123, 0x00e5, 0x0107, + 0x0142, 0x0113, 0x0156, 0x0157, + 0x012b, 0x0179, 0x00c4, 0x00c5, + /* 0x90*/ + 0x00c9, 0x00e6, 0x00c6, 0x014d, + 0x00f6, 0x0122, 0x00a2, 0x015a, + 0x015b, 0x00d6, 0x00dc, 0x00f8, + 0x00a3, 0x00d8, 0x00d7, 0x00a4, + /* 0xa0*/ + 0x0100, 0x012a, 0x00f3, 0x017b, + 0x017c, 0x017a, 0x201d, 0x00a6, + 0x00a9, 0x00ae, 0x00ac, 0x00bd, + 0x00bc, 0x0141, 0x00ab, 0x00bb, + /* 0xb0*/ + 0x2591, 0x2592, 0x2593, 0x2502, + 0x2524, 0x0104, 0x010c, 0x0118, + 0x0116, 0x2563, 0x2551, 0x2557, + 0x255d, 0x012e, 0x0160, 0x2510, + /* 0xc0*/ + 0x2514, 0x2534, 0x252c, 0x251c, + 0x2500, 0x253c, 0x0172, 0x016a, + 0x255a, 0x2554, 0x2569, 0x2566, + 0x2560, 0x2550, 0x256c, 0x017d, + /* 0xd0*/ + 0x0105, 0x010d, 0x0119, 0x0117, + 0x012f, 0x0161, 0x0173, 0x016b, + 0x017e, 0x2518, 0x250c, 0x2588, + 0x2584, 0x258c, 0x2590, 0x2580, + /* 0xe0*/ + 0x00d3, 0x00df, 0x014c, 0x0143, + 0x00f5, 0x00d5, 0x00b5, 0x0144, + 0x0136, 0x0137, 0x013b, 0x013c, + 0x0146, 0x0112, 0x0145, 0x2019, + /* 0xf0*/ + 0x00ad, 0x00b1, 0x201c, 0x00be, + 0x00b6, 0x00a7, 0x00f7, 0x201e, + 0x00b0, 0x2219, 0x00b7, 0x00b9, + 0x00b3, 0x00b2, 0x25a0, 0x00a0, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xff, 0x00, 0x96, 0x9c, 0x9f, 0x00, 0xa7, 0xf5, /* 0xa0-0xa7 */ + 0x00, 0xa8, 0x00, 0xae, 0xaa, 0xf0, 0xa9, 0x00, /* 0xa8-0xaf */ + 0xf8, 0xf1, 0xfd, 0xfc, 0x00, 0xe6, 0xf4, 0xfa, /* 0xb0-0xb7 */ + 0x00, 0xfb, 0x00, 0xaf, 0xac, 0xab, 0xf3, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x8e, 0x8f, 0x92, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0xe0, 0x00, 0xe5, 0x99, 0x9e, /* 0xd0-0xd7 */ + 0x9d, 0x00, 0x00, 0x00, 0x9a, 0x00, 0x00, 0xe1, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x84, 0x86, 0x91, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x82, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0xa2, 0x00, 0xe4, 0x94, 0xf6, /* 0xf0-0xf7 */ + 0x9b, 0x00, 0x00, 0x00, 0x81, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0xa0, 0x83, 0x00, 0x00, 0xb5, 0xd0, 0x80, 0x87, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xb6, 0xd1, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0xed, 0x89, 0x00, 0x00, 0xb8, 0xd3, /* 0x10-0x17 */ + 0xb7, 0xd2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x95, 0x85, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0xa1, 0x8c, 0x00, 0x00, 0xbd, 0xd4, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe8, 0xe9, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0xea, 0xeb, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0xad, 0x88, 0xe3, 0xe7, 0xee, 0xec, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0xe2, 0x93, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8a, 0x8b, /* 0x50-0x57 */ + 0x00, 0x00, 0x97, 0x98, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0xbe, 0xd5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0xc7, 0xd7, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0xc6, 0xd6, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x8d, 0xa5, 0xa3, 0xa4, 0xcf, 0xd8, 0x00, /* 0x78-0x7f */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0xef, 0x00, 0x00, 0xf2, 0xa6, 0xf7, 0x00, /* 0x18-0x1f */ +}; + +static const unsigned char page22[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0xf9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ +}; + +static const unsigned char page25[256] = { + 0xc4, 0x00, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0xbf, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xd9, 0x00, 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xc2, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xc5, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0xcd, 0xba, 0x00, 0x00, 0xc9, 0x00, 0x00, 0xbb, /* 0x50-0x57 */ + 0x00, 0x00, 0xc8, 0x00, 0x00, 0xbc, 0x00, 0x00, /* 0x58-0x5f */ + 0xcc, 0x00, 0x00, 0xb9, 0x00, 0x00, 0xcb, 0x00, /* 0x60-0x67 */ + 0x00, 0xca, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0xdf, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0xdb, 0x00, 0x00, 0x00, 0xdd, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0xde, 0xb0, 0xb1, 0xb2, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + page20, NULL, page22, NULL, NULL, page25, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x87, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8b, 0x8b, 0x8c, 0xa5, 0x84, 0x86, /* 0x88-0x8f */ + 0x82, 0x91, 0x91, 0x93, 0x94, 0x85, 0x96, 0x98, /* 0x90-0x97 */ + 0x98, 0x94, 0x81, 0x9b, 0x9c, 0x9b, 0x9e, 0x9f, /* 0x98-0x9f */ + 0x83, 0x8c, 0xa2, 0xa4, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0x88, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xd0, 0xd1, 0xd2, /* 0xb0-0xb7 */ + 0xd3, 0xb9, 0xba, 0xbb, 0xbc, 0xd4, 0xd5, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xd6, 0xd7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xd8, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xa2, 0xe1, 0x93, 0xe7, 0xe4, 0xe4, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe9, 0xe9, 0xeb, 0xeb, 0xec, 0x89, 0xec, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x9a, 0x90, 0xa0, 0x8e, 0x95, 0x8f, 0x80, /* 0x80-0x87 */ + 0xad, 0xed, 0x8a, 0x8a, 0xa1, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x92, 0x92, 0xe2, 0x99, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x97, 0x99, 0x9a, 0x9d, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xe0, 0xa3, 0xa3, 0x8d, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xb5, 0xb6, 0xb7, 0xb8, 0xbd, 0xbe, 0xc6, 0xc7, /* 0xd0-0xd7 */ + 0xcf, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe5, 0xe5, 0x00, 0xe3, /* 0xe0-0xe7 */ + 0xe8, 0xe8, 0xea, 0xea, 0xee, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "cp775", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_cp775(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_cp775(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_cp775) +module_exit(exit_nls_cp775) + +MODULE_DESCRIPTION("NLS Codepage 775 (Baltic Rim)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_cp850.c b/fs/nls/nls_cp850.c new file mode 100644 index 000000000..5f9b9507a --- /dev/null +++ b/fs/nls/nls_cp850.c @@ -0,0 +1,316 @@ +/* + * linux/fs/nls/nls_cp850.c + * + * Charset cp850 translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x00c7, 0x00fc, 0x00e9, 0x00e2, + 0x00e4, 0x00e0, 0x00e5, 0x00e7, + 0x00ea, 0x00eb, 0x00e8, 0x00ef, + 0x00ee, 0x00ec, 0x00c4, 0x00c5, + /* 0x90*/ + 0x00c9, 0x00e6, 0x00c6, 0x00f4, + 0x00f6, 0x00f2, 0x00fb, 0x00f9, + 0x00ff, 0x00d6, 0x00dc, 0x00f8, + 0x00a3, 0x00d8, 0x00d7, 0x0192, + /* 0xa0*/ + 0x00e1, 0x00ed, 0x00f3, 0x00fa, + 0x00f1, 0x00d1, 0x00aa, 0x00ba, + 0x00bf, 0x00ae, 0x00ac, 0x00bd, + 0x00bc, 0x00a1, 0x00ab, 0x00bb, + /* 0xb0*/ + 0x2591, 0x2592, 0x2593, 0x2502, + 0x2524, 0x00c1, 0x00c2, 0x00c0, + 0x00a9, 0x2563, 0x2551, 0x2557, + 0x255d, 0x00a2, 0x00a5, 0x2510, + /* 0xc0*/ + 0x2514, 0x2534, 0x252c, 0x251c, + 0x2500, 0x253c, 0x00e3, 0x00c3, + 0x255a, 0x2554, 0x2569, 0x2566, + 0x2560, 0x2550, 0x256c, 0x00a4, + /* 0xd0*/ + 0x00f0, 0x00d0, 0x00ca, 0x00cb, + 0x00c8, 0x0131, 0x00cd, 0x00ce, + 0x00cf, 0x2518, 0x250c, 0x2588, + 0x2584, 0x00a6, 0x00cc, 0x2580, + /* 0xe0*/ + 0x00d3, 0x00df, 0x00d4, 0x00d2, + 0x00f5, 0x00d5, 0x00b5, 0x00fe, + 0x00de, 0x00da, 0x00db, 0x00d9, + 0x00fd, 0x00dd, 0x00af, 0x00b4, + /* 0xf0*/ + 0x00ad, 0x00b1, 0x2017, 0x00be, + 0x00b6, 0x00a7, 0x00f7, 0x00b8, + 0x00b0, 0x00a8, 0x00b7, 0x00b9, + 0x00b3, 0x00b2, 0x25a0, 0x00a0, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xff, 0xad, 0xbd, 0x9c, 0xcf, 0xbe, 0xdd, 0xf5, /* 0xa0-0xa7 */ + 0xf9, 0xb8, 0xa6, 0xae, 0xaa, 0xf0, 0xa9, 0xee, /* 0xa8-0xaf */ + 0xf8, 0xf1, 0xfd, 0xfc, 0xef, 0xe6, 0xf4, 0xfa, /* 0xb0-0xb7 */ + 0xf7, 0xfb, 0xa7, 0xaf, 0xac, 0xab, 0xf3, 0xa8, /* 0xb8-0xbf */ + 0xb7, 0xb5, 0xb6, 0xc7, 0x8e, 0x8f, 0x92, 0x80, /* 0xc0-0xc7 */ + 0xd4, 0x90, 0xd2, 0xd3, 0xde, 0xd6, 0xd7, 0xd8, /* 0xc8-0xcf */ + 0xd1, 0xa5, 0xe3, 0xe0, 0xe2, 0xe5, 0x99, 0x9e, /* 0xd0-0xd7 */ + 0x9d, 0xeb, 0xe9, 0xea, 0x9a, 0xed, 0xe8, 0xe1, /* 0xd8-0xdf */ + 0x85, 0xa0, 0x83, 0xc6, 0x84, 0x86, 0x91, 0x87, /* 0xe0-0xe7 */ + 0x8a, 0x82, 0x88, 0x89, 0x8d, 0xa1, 0x8c, 0x8b, /* 0xe8-0xef */ + 0xd0, 0xa4, 0x95, 0xa2, 0x93, 0xe4, 0x94, 0xf6, /* 0xf0-0xf7 */ + 0x9b, 0x97, 0xa3, 0x96, 0x81, 0xec, 0xe7, 0x98, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0xd5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x9f, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf2, /* 0x10-0x17 */ +}; + +static const unsigned char page25[256] = { + 0xc4, 0x00, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0xbf, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xd9, 0x00, 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xc2, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xc5, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0xcd, 0xba, 0x00, 0x00, 0xc9, 0x00, 0x00, 0xbb, /* 0x50-0x57 */ + 0x00, 0x00, 0xc8, 0x00, 0x00, 0xbc, 0x00, 0x00, /* 0x58-0x5f */ + 0xcc, 0x00, 0x00, 0xb9, 0x00, 0x00, 0xcb, 0x00, /* 0x60-0x67 */ + 0x00, 0xca, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0xdf, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0xdb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0xb0, 0xb1, 0xb2, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + page20, NULL, NULL, NULL, NULL, page25, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x87, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x84, 0x86, /* 0x88-0x8f */ + 0x82, 0x91, 0x91, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x94, 0x81, 0x9b, 0x9c, 0x9b, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa4, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xa0, 0x83, 0x85, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc6, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd0, 0x88, 0x89, 0x8a, 0xd5, 0xa1, 0x8c, /* 0xd0-0xd7 */ + 0x8b, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0x8d, 0xdf, /* 0xd8-0xdf */ + 0xa2, 0xe1, 0x93, 0x95, 0xe4, 0xe4, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe7, 0xa3, 0x96, 0x97, 0xec, 0xec, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x9a, 0x90, 0xb6, 0x8e, 0xb7, 0x8f, 0x80, /* 0x80-0x87 */ + 0xd2, 0xd3, 0xd4, 0xd8, 0xd7, 0xde, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x92, 0x92, 0xe2, 0x99, 0xe3, 0xea, 0xeb, /* 0x90-0x97 */ + 0x00, 0x99, 0x9a, 0x9d, 0x9c, 0x9d, 0x9e, 0x00, /* 0x98-0x9f */ + 0xb5, 0xd6, 0xe0, 0xe9, 0xa5, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc7, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd1, 0xd1, 0xd2, 0xd3, 0xd4, 0x49, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe5, 0xe5, 0x00, 0xe8, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xed, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "cp850", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_cp850(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_cp850(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_cp850) +module_exit(exit_nls_cp850) + +MODULE_DESCRIPTION("NLS Codepage 850 (Europe)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_cp852.c b/fs/nls/nls_cp852.c new file mode 100644 index 000000000..fc513a5e8 --- /dev/null +++ b/fs/nls/nls_cp852.c @@ -0,0 +1,338 @@ +/* + * linux/fs/nls/nls_cp852.c + * + * Charset cp852 translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x00c7, 0x00fc, 0x00e9, 0x00e2, + 0x00e4, 0x016f, 0x0107, 0x00e7, + 0x0142, 0x00eb, 0x0150, 0x0151, + 0x00ee, 0x0179, 0x00c4, 0x0106, + /* 0x90*/ + 0x00c9, 0x0139, 0x013a, 0x00f4, + 0x00f6, 0x013d, 0x013e, 0x015a, + 0x015b, 0x00d6, 0x00dc, 0x0164, + 0x0165, 0x0141, 0x00d7, 0x010d, + /* 0xa0*/ + 0x00e1, 0x00ed, 0x00f3, 0x00fa, + 0x0104, 0x0105, 0x017d, 0x017e, + 0x0118, 0x0119, 0x00ac, 0x017a, + 0x010c, 0x015f, 0x00ab, 0x00bb, + /* 0xb0*/ + 0x2591, 0x2592, 0x2593, 0x2502, + 0x2524, 0x00c1, 0x00c2, 0x011a, + 0x015e, 0x2563, 0x2551, 0x2557, + 0x255d, 0x017b, 0x017c, 0x2510, + /* 0xc0*/ + 0x2514, 0x2534, 0x252c, 0x251c, + 0x2500, 0x253c, 0x0102, 0x0103, + 0x255a, 0x2554, 0x2569, 0x2566, + 0x2560, 0x2550, 0x256c, 0x00a4, + /* 0xd0*/ + 0x0111, 0x0110, 0x010e, 0x00cb, + 0x010f, 0x0147, 0x00cd, 0x00ce, + 0x011b, 0x2518, 0x250c, 0x2588, + 0x2584, 0x0162, 0x016e, 0x2580, + /* 0xe0*/ + 0x00d3, 0x00df, 0x00d4, 0x0143, + 0x0144, 0x0148, 0x0160, 0x0161, + 0x0154, 0x00da, 0x0155, 0x0170, + 0x00fd, 0x00dd, 0x0163, 0x00b4, + /* 0xf0*/ + 0x00ad, 0x02dd, 0x02db, 0x02c7, + 0x02d8, 0x00a7, 0x00f7, 0x00b8, + 0x00b0, 0x00a8, 0x02d9, 0x0171, + 0x0158, 0x0159, 0x25a0, 0x00a0, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xff, 0x00, 0x00, 0x00, 0xcf, 0x00, 0x00, 0xf5, /* 0xa0-0xa7 */ + 0xf9, 0x00, 0x00, 0xae, 0xaa, 0xf0, 0x00, 0x00, /* 0xa8-0xaf */ + 0xf8, 0x00, 0x00, 0x00, 0xef, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0xf7, 0x00, 0x00, 0xaf, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0xb5, 0xb6, 0x00, 0x8e, 0x00, 0x00, 0x80, /* 0xc0-0xc7 */ + 0x00, 0x90, 0x00, 0xd3, 0x00, 0xd6, 0xd7, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0xe0, 0xe2, 0x00, 0x99, 0x9e, /* 0xd0-0xd7 */ + 0x00, 0x00, 0xe9, 0x00, 0x9a, 0xed, 0x00, 0xe1, /* 0xd8-0xdf */ + 0x00, 0xa0, 0x83, 0x00, 0x84, 0x00, 0x00, 0x87, /* 0xe0-0xe7 */ + 0x00, 0x82, 0x00, 0x89, 0x00, 0xa1, 0x8c, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0xa2, 0x93, 0x00, 0x94, 0xf6, /* 0xf0-0xf7 */ + 0x00, 0x00, 0xa3, 0x00, 0x81, 0xec, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0x00, 0x00, 0xc6, 0xc7, 0xa4, 0xa5, 0x8f, 0x86, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xac, 0x9f, 0xd2, 0xd4, /* 0x08-0x0f */ + 0xd1, 0xd0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xa8, 0xa9, 0xb7, 0xd8, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x91, 0x92, 0x00, 0x00, 0x95, 0x96, 0x00, /* 0x38-0x3f */ + 0x00, 0x9d, 0x88, 0xe3, 0xe4, 0x00, 0x00, 0xd5, /* 0x40-0x47 */ + 0xe5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x8a, 0x8b, 0x00, 0x00, 0xe8, 0xea, 0x00, 0x00, /* 0x50-0x57 */ + 0xfc, 0xfd, 0x97, 0x98, 0x00, 0x00, 0xb8, 0xad, /* 0x58-0x5f */ + 0xe6, 0xe7, 0xdd, 0xee, 0x9b, 0x9c, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xde, 0x85, /* 0x68-0x6f */ + 0xeb, 0xfb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x8d, 0xab, 0xbd, 0xbe, 0xa6, 0xa7, 0x00, /* 0x78-0x7f */ +}; + +static const unsigned char page02[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf3, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0xf4, 0xfa, 0x00, 0xf2, 0x00, 0xf1, 0x00, 0x00, /* 0xd8-0xdf */ +}; + +static const unsigned char page25[256] = { + 0xc4, 0x00, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0xbf, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xd9, 0x00, 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xc2, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xc5, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0xcd, 0xba, 0x00, 0x00, 0xc9, 0x00, 0x00, 0xbb, /* 0x50-0x57 */ + 0x00, 0x00, 0xc8, 0x00, 0x00, 0xbc, 0x00, 0x00, /* 0x58-0x5f */ + 0xcc, 0x00, 0x00, 0xb9, 0x00, 0x00, 0xcb, 0x00, /* 0x60-0x67 */ + 0x00, 0xca, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0xdf, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0xdb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0xb0, 0xb1, 0xb2, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, page02, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, page25, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x87, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8b, 0x8b, 0x8c, 0xab, 0x84, 0x86, /* 0x88-0x8f */ + 0x82, 0x92, 0x92, 0x93, 0x94, 0x96, 0x96, 0x98, /* 0x90-0x97 */ + 0x98, 0x94, 0x81, 0x9c, 0x9c, 0x88, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa5, 0xa5, 0xa7, 0xa7, /* 0xa0-0xa7 */ + 0xa9, 0xa9, 0xaa, 0xab, 0x9f, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xa0, 0x83, 0xd8, /* 0xb0-0xb7 */ + 0xad, 0xb9, 0xba, 0xbb, 0xbc, 0xbe, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc7, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd0, 0xd4, 0x89, 0xd4, 0xe5, 0xa1, 0x8c, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xee, 0x85, 0xdf, /* 0xd8-0xdf */ + 0xa2, 0xe1, 0x93, 0xe4, 0xe4, 0xe5, 0xe7, 0xe7, /* 0xe0-0xe7 */ + 0xea, 0xa3, 0xea, 0xfb, 0xec, 0xec, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfd, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x9a, 0x90, 0xb6, 0x8e, 0xde, 0x8f, 0x80, /* 0x80-0x87 */ + 0x9d, 0xd3, 0x8a, 0x8a, 0xd7, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x91, 0xe2, 0x99, 0x95, 0x95, 0x97, /* 0x90-0x97 */ + 0x97, 0x99, 0x9a, 0x9b, 0x9b, 0x9d, 0x9e, 0xac, /* 0x98-0x9f */ + 0xb5, 0xd6, 0xe0, 0xe9, 0xa4, 0xa4, 0xa6, 0xa6, /* 0xa0-0xa7 */ + 0xa8, 0xa8, 0xaa, 0x8d, 0xac, 0xb8, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbd, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc6, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd1, 0xd1, 0xd2, 0xd3, 0xd2, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xb7, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe3, 0xd5, 0xe6, 0xe6, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xe8, 0xeb, 0xed, 0xed, 0xdd, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xeb, 0xfc, 0xfc, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "cp852", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_cp852(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_cp852(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_cp852) +module_exit(exit_nls_cp852) + +MODULE_DESCRIPTION("NLS Codepage 852 (Central/Eastern Europe)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_cp855.c b/fs/nls/nls_cp855.c new file mode 100644 index 000000000..a43be58ad --- /dev/null +++ b/fs/nls/nls_cp855.c @@ -0,0 +1,300 @@ +/* + * linux/fs/nls/nls_cp855.c + * + * Charset cp855 translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x0452, 0x0402, 0x0453, 0x0403, + 0x0451, 0x0401, 0x0454, 0x0404, + 0x0455, 0x0405, 0x0456, 0x0406, + 0x0457, 0x0407, 0x0458, 0x0408, + /* 0x90*/ + 0x0459, 0x0409, 0x045a, 0x040a, + 0x045b, 0x040b, 0x045c, 0x040c, + 0x045e, 0x040e, 0x045f, 0x040f, + 0x044e, 0x042e, 0x044a, 0x042a, + /* 0xa0*/ + 0x0430, 0x0410, 0x0431, 0x0411, + 0x0446, 0x0426, 0x0434, 0x0414, + 0x0435, 0x0415, 0x0444, 0x0424, + 0x0433, 0x0413, 0x00ab, 0x00bb, + /* 0xb0*/ + 0x2591, 0x2592, 0x2593, 0x2502, + 0x2524, 0x0445, 0x0425, 0x0438, + 0x0418, 0x2563, 0x2551, 0x2557, + 0x255d, 0x0439, 0x0419, 0x2510, + /* 0xc0*/ + 0x2514, 0x2534, 0x252c, 0x251c, + 0x2500, 0x253c, 0x043a, 0x041a, + 0x255a, 0x2554, 0x2569, 0x2566, + 0x2560, 0x2550, 0x256c, 0x00a4, + /* 0xd0*/ + 0x043b, 0x041b, 0x043c, 0x041c, + 0x043d, 0x041d, 0x043e, 0x041e, + 0x043f, 0x2518, 0x250c, 0x2588, + 0x2584, 0x041f, 0x044f, 0x2580, + /* 0xe0*/ + 0x042f, 0x0440, 0x0420, 0x0441, + 0x0421, 0x0442, 0x0422, 0x0443, + 0x0423, 0x0436, 0x0416, 0x0432, + 0x0412, 0x044c, 0x042c, 0x2116, + /* 0xf0*/ + 0x00ad, 0x044b, 0x042b, 0x0437, + 0x0417, 0x0448, 0x0428, 0x044d, + 0x042d, 0x0449, 0x0429, 0x0447, + 0x0427, 0x00a7, 0x25a0, 0x00a0, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xff, 0x00, 0x00, 0x00, 0xcf, 0x00, 0x00, 0xfd, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0xae, 0x00, 0xf0, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0xaf, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ +}; + +static const unsigned char page04[256] = { + 0x00, 0x85, 0x81, 0x83, 0x87, 0x89, 0x8b, 0x8d, /* 0x00-0x07 */ + 0x8f, 0x91, 0x93, 0x95, 0x97, 0x00, 0x99, 0x9b, /* 0x08-0x0f */ + 0xa1, 0xa3, 0xec, 0xad, 0xa7, 0xa9, 0xea, 0xf4, /* 0x10-0x17 */ + 0xb8, 0xbe, 0xc7, 0xd1, 0xd3, 0xd5, 0xd7, 0xdd, /* 0x18-0x1f */ + 0xe2, 0xe4, 0xe6, 0xe8, 0xab, 0xb6, 0xa5, 0xfc, /* 0x20-0x27 */ + 0xf6, 0xfa, 0x9f, 0xf2, 0xee, 0xf8, 0x9d, 0xe0, /* 0x28-0x2f */ + 0xa0, 0xa2, 0xeb, 0xac, 0xa6, 0xa8, 0xe9, 0xf3, /* 0x30-0x37 */ + 0xb7, 0xbd, 0xc6, 0xd0, 0xd2, 0xd4, 0xd6, 0xd8, /* 0x38-0x3f */ + 0xe1, 0xe3, 0xe5, 0xe7, 0xaa, 0xb5, 0xa4, 0xfb, /* 0x40-0x47 */ + 0xf5, 0xf9, 0x9e, 0xf1, 0xed, 0xf7, 0x9c, 0xde, /* 0x48-0x4f */ + 0x00, 0x84, 0x80, 0x82, 0x86, 0x88, 0x8a, 0x8c, /* 0x50-0x57 */ + 0x8e, 0x90, 0x92, 0x94, 0x96, 0x00, 0x98, 0x9a, /* 0x58-0x5f */ +}; + +static const unsigned char page21[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xef, 0x00, /* 0x10-0x17 */ +}; + +static const unsigned char page25[256] = { + 0xc4, 0x00, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0xbf, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xd9, 0x00, 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xc2, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xc5, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0xcd, 0xba, 0x00, 0x00, 0xc9, 0x00, 0x00, 0xbb, /* 0x50-0x57 */ + 0x00, 0x00, 0xc8, 0x00, 0x00, 0xbc, 0x00, 0x00, /* 0x58-0x5f */ + 0xcc, 0x00, 0x00, 0xb9, 0x00, 0x00, 0xcb, 0x00, /* 0x60-0x67 */ + 0x00, 0xca, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0xdf, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0xdb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0xb0, 0xb1, 0xb2, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, NULL, NULL, NULL, page04, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, page21, NULL, NULL, NULL, page25, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x80, 0x82, 0x82, 0x84, 0x84, 0x86, 0x86, /* 0x80-0x87 */ + 0x88, 0x88, 0x8a, 0x8a, 0x8c, 0x8c, 0x8e, 0x8e, /* 0x88-0x8f */ + 0x90, 0x90, 0x92, 0x92, 0x94, 0x94, 0x96, 0x96, /* 0x90-0x97 */ + 0x98, 0x98, 0x9a, 0x9a, 0x9c, 0x9c, 0x9e, 0x9e, /* 0x98-0x9f */ + 0xa0, 0xa0, 0xa2, 0xa2, 0xa4, 0xa4, 0xa6, 0xa6, /* 0xa0-0xa7 */ + 0xa8, 0xa8, 0xaa, 0xaa, 0xac, 0xac, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb5, 0xb7, /* 0xb0-0xb7 */ + 0xb7, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbd, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc6, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd0, 0xd2, 0xd2, 0xd4, 0xd4, 0xd6, 0xd6, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xd8, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xde, 0xe1, 0xe1, 0xe3, 0xe3, 0xe5, 0xe5, 0xe7, /* 0xe0-0xe7 */ + 0xe7, 0xe9, 0xe9, 0xeb, 0xeb, 0xed, 0xed, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf1, 0xf3, 0xf3, 0xf5, 0xf5, 0xf7, /* 0xf0-0xf7 */ + 0xf7, 0xf9, 0xf9, 0xfb, 0xfb, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x81, 0x81, 0x83, 0x83, 0x85, 0x85, 0x87, 0x87, /* 0x80-0x87 */ + 0x89, 0x89, 0x8b, 0x8b, 0x8d, 0x8d, 0x8f, 0x8f, /* 0x88-0x8f */ + 0x91, 0x91, 0x93, 0x93, 0x95, 0x95, 0x97, 0x97, /* 0x90-0x97 */ + 0x99, 0x99, 0x9b, 0x9b, 0x9d, 0x9d, 0x9f, 0x9f, /* 0x98-0x9f */ + 0xa1, 0xa1, 0xa3, 0xa3, 0xa5, 0xa5, 0xa7, 0xa7, /* 0xa0-0xa7 */ + 0xa9, 0xa9, 0xab, 0xab, 0xad, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb6, 0xb6, 0xb8, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbe, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc7, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd1, 0xd1, 0xd3, 0xd3, 0xd5, 0xd5, 0xd7, 0xd7, /* 0xd0-0xd7 */ + 0xdd, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xe0, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe2, 0xe2, 0xe4, 0xe4, 0xe6, 0xe6, 0xe8, /* 0xe0-0xe7 */ + 0xe8, 0xea, 0xea, 0xec, 0xec, 0xee, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf2, 0xf2, 0xf4, 0xf4, 0xf6, 0xf6, 0xf8, /* 0xf0-0xf7 */ + 0xf8, 0xfa, 0xfa, 0xfc, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "cp855", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_cp855(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_cp855(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_cp855) +module_exit(exit_nls_cp855) + +MODULE_DESCRIPTION("NLS Codepage 855 (Cyrillic)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_cp857.c b/fs/nls/nls_cp857.c new file mode 100644 index 000000000..772cd4195 --- /dev/null +++ b/fs/nls/nls_cp857.c @@ -0,0 +1,302 @@ +/* + * linux/fs/nls/nls_cp857.c + * + * Charset cp857 translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x00c7, 0x00fc, 0x00e9, 0x00e2, + 0x00e4, 0x00e0, 0x00e5, 0x00e7, + 0x00ea, 0x00eb, 0x00e8, 0x00ef, + 0x00ee, 0x0131, 0x00c4, 0x00c5, + /* 0x90*/ + 0x00c9, 0x00e6, 0x00c6, 0x00f4, + 0x00f6, 0x00f2, 0x00fb, 0x00f9, + 0x0130, 0x00d6, 0x00dc, 0x00f8, + 0x00a3, 0x00d8, 0x015e, 0x015f, + /* 0xa0*/ + 0x00e1, 0x00ed, 0x00f3, 0x00fa, + 0x00f1, 0x00d1, 0x011e, 0x011f, + 0x00bf, 0x00ae, 0x00ac, 0x00bd, + 0x00bc, 0x00a1, 0x00ab, 0x00bb, + /* 0xb0*/ + 0x2591, 0x2592, 0x2593, 0x2502, + 0x2524, 0x00c1, 0x00c2, 0x00c0, + 0x00a9, 0x2563, 0x2551, 0x2557, + 0x255d, 0x00a2, 0x00a5, 0x2510, + /* 0xc0*/ + 0x2514, 0x2534, 0x252c, 0x251c, + 0x2500, 0x253c, 0x00e3, 0x00c3, + 0x255a, 0x2554, 0x2569, 0x2566, + 0x2560, 0x2550, 0x256c, 0x00a4, + /* 0xd0*/ + 0x00ba, 0x00aa, 0x00ca, 0x00cb, + 0x00c8, 0x0000, 0x00cd, 0x00ce, + 0x00cf, 0x2518, 0x250c, 0x2588, + 0x2584, 0x00a6, 0x00cc, 0x2580, + /* 0xe0*/ + 0x00d3, 0x00df, 0x00d4, 0x00d2, + 0x00f5, 0x00d5, 0x00b5, 0x0000, + 0x00d7, 0x00da, 0x00db, 0x00d9, + 0x00ec, 0x00ff, 0x00af, 0x00b4, + /* 0xf0*/ + 0x00ad, 0x00b1, 0x0000, 0x00be, + 0x00b6, 0x00a7, 0x00f7, 0x00b8, + 0x00b0, 0x00a8, 0x00b7, 0x00b9, + 0x00b3, 0x00b2, 0x25a0, 0x00a0, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xff, 0xad, 0xbd, 0x9c, 0xcf, 0xbe, 0xdd, 0xf5, /* 0xa0-0xa7 */ + 0xf9, 0xb8, 0xd1, 0xae, 0xaa, 0xf0, 0xa9, 0xee, /* 0xa8-0xaf */ + 0xf8, 0xf1, 0xfd, 0xfc, 0xef, 0xe6, 0xf4, 0xfa, /* 0xb0-0xb7 */ + 0xf7, 0xfb, 0xd0, 0xaf, 0xac, 0xab, 0xf3, 0xa8, /* 0xb8-0xbf */ + 0xb7, 0xb5, 0xb6, 0xc7, 0x8e, 0x8f, 0x92, 0x80, /* 0xc0-0xc7 */ + 0xd4, 0x90, 0xd2, 0xd3, 0xde, 0xd6, 0xd7, 0xd8, /* 0xc8-0xcf */ + 0x00, 0xa5, 0xe3, 0xe0, 0xe2, 0xe5, 0x99, 0xe8, /* 0xd0-0xd7 */ + 0x9d, 0xeb, 0xe9, 0xea, 0x9a, 0x00, 0x00, 0xe1, /* 0xd8-0xdf */ + 0x85, 0xa0, 0x83, 0xc6, 0x84, 0x86, 0x91, 0x87, /* 0xe0-0xe7 */ + 0x8a, 0x82, 0x88, 0x89, 0xec, 0xa1, 0x8c, 0x8b, /* 0xe8-0xef */ + 0x00, 0xa4, 0x95, 0xa2, 0x93, 0xe4, 0x94, 0xf6, /* 0xf0-0xf7 */ + 0x9b, 0x97, 0xa3, 0x96, 0x81, 0x00, 0x00, 0xed, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa6, 0xa7, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x98, 0x8d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9e, 0x9f, /* 0x58-0x5f */ +}; + +static const unsigned char page25[256] = { + 0xc4, 0x00, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0xbf, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xd9, 0x00, 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xc2, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xc5, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0xcd, 0xba, 0x00, 0x00, 0xc9, 0x00, 0x00, 0xbb, /* 0x50-0x57 */ + 0x00, 0x00, 0xc8, 0x00, 0x00, 0xbc, 0x00, 0x00, /* 0x58-0x5f */ + 0xcc, 0x00, 0x00, 0xb9, 0x00, 0x00, 0xcb, 0x00, /* 0x60-0x67 */ + 0x00, 0xca, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0xdf, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0xdb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0xb0, 0xb1, 0xb2, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, page25, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x87, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x84, 0x86, /* 0x88-0x8f */ + 0x82, 0x91, 0x91, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x69, 0x94, 0x81, 0x9b, 0x9c, 0x9b, 0x9f, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa4, 0xa7, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xa0, 0x83, 0x85, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc6, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0x88, 0x89, 0x8a, 0x00, 0xa1, 0x8c, /* 0xd0-0xd7 */ + 0x8b, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xec, 0xdf, /* 0xd8-0xdf */ + 0xa2, 0xe1, 0x93, 0x95, 0xe4, 0xe4, 0xe6, 0x00, /* 0xe0-0xe7 */ + 0xe8, 0xa3, 0x96, 0x97, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0x00, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x9a, 0x90, 0xb6, 0x8e, 0xb7, 0x8f, 0x80, /* 0x80-0x87 */ + 0xd2, 0xd3, 0xd4, 0xd8, 0xd7, 0x49, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x92, 0x92, 0xe2, 0x99, 0xe3, 0xea, 0xeb, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9d, 0x9c, 0x9d, 0x9e, 0x9e, /* 0x98-0x9f */ + 0xb5, 0xd6, 0xe0, 0xe9, 0xa5, 0xa5, 0xa6, 0xa6, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc7, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0x00, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe5, 0xe5, 0x00, 0x00, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xde, 0x00, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0x00, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "cp857", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_cp857(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_cp857(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_cp857) +module_exit(exit_nls_cp857) + +MODULE_DESCRIPTION("NLS Codepage 857 (Turkish)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_cp860.c b/fs/nls/nls_cp860.c new file mode 100644 index 000000000..36cf4ca11 --- /dev/null +++ b/fs/nls/nls_cp860.c @@ -0,0 +1,365 @@ +/* + * linux/fs/nls/nls_cp860.c + * + * Charset cp860 translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x00c7, 0x00fc, 0x00e9, 0x00e2, + 0x00e3, 0x00e0, 0x00c1, 0x00e7, + 0x00ea, 0x00ca, 0x00e8, 0x00cd, + 0x00d4, 0x00ec, 0x00c3, 0x00c2, + /* 0x90*/ + 0x00c9, 0x00c0, 0x00c8, 0x00f4, + 0x00f5, 0x00f2, 0x00da, 0x00f9, + 0x00cc, 0x00d5, 0x00dc, 0x00a2, + 0x00a3, 0x00d9, 0x20a7, 0x00d3, + /* 0xa0*/ + 0x00e1, 0x00ed, 0x00f3, 0x00fa, + 0x00f1, 0x00d1, 0x00aa, 0x00ba, + 0x00bf, 0x00d2, 0x00ac, 0x00bd, + 0x00bc, 0x00a1, 0x00ab, 0x00bb, + /* 0xb0*/ + 0x2591, 0x2592, 0x2593, 0x2502, + 0x2524, 0x2561, 0x2562, 0x2556, + 0x2555, 0x2563, 0x2551, 0x2557, + 0x255d, 0x255c, 0x255b, 0x2510, + /* 0xc0*/ + 0x2514, 0x2534, 0x252c, 0x251c, + 0x2500, 0x253c, 0x255e, 0x255f, + 0x255a, 0x2554, 0x2569, 0x2566, + 0x2560, 0x2550, 0x256c, 0x2567, + /* 0xd0*/ + 0x2568, 0x2564, 0x2565, 0x2559, + 0x2558, 0x2552, 0x2553, 0x256b, + 0x256a, 0x2518, 0x250c, 0x2588, + 0x2584, 0x258c, 0x2590, 0x2580, + /* 0xe0*/ + 0x03b1, 0x00df, 0x0393, 0x03c0, + 0x03a3, 0x03c3, 0x00b5, 0x03c4, + 0x03a6, 0x0398, 0x03a9, 0x03b4, + 0x221e, 0x03c6, 0x03b5, 0x2229, + /* 0xf0*/ + 0x2261, 0x00b1, 0x2265, 0x2264, + 0x2320, 0x2321, 0x00f7, 0x2248, + 0x00b0, 0x2219, 0x00b7, 0x221a, + 0x207f, 0x00b2, 0x25a0, 0x00a0, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xff, 0xad, 0x9b, 0x9c, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0xa6, 0xae, 0xaa, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0xf8, 0xf1, 0xfd, 0x00, 0x00, 0xe6, 0x00, 0xfa, /* 0xb0-0xb7 */ + 0x00, 0x00, 0xa7, 0xaf, 0xac, 0xab, 0x00, 0xa8, /* 0xb8-0xbf */ + 0x91, 0x86, 0x8f, 0x8e, 0x00, 0x00, 0x00, 0x80, /* 0xc0-0xc7 */ + 0x92, 0x90, 0x89, 0x00, 0x98, 0x8b, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0xa5, 0xa9, 0x9f, 0x8c, 0x99, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x9d, 0x96, 0x00, 0x9a, 0x00, 0x00, 0xe1, /* 0xd8-0xdf */ + 0x85, 0xa0, 0x83, 0x84, 0x00, 0x00, 0x00, 0x87, /* 0xe0-0xe7 */ + 0x8a, 0x82, 0x88, 0x00, 0x8d, 0xa1, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0xa4, 0x95, 0xa2, 0x93, 0x94, 0x00, 0xf6, /* 0xf0-0xf7 */ + 0x00, 0x97, 0xa3, 0x00, 0x81, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page03[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0xe2, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0xe9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0xe4, 0x00, 0x00, 0xe8, 0x00, /* 0xa0-0xa7 */ + 0x00, 0xea, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0xe0, 0x00, 0x00, 0xeb, 0xee, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0xe3, 0x00, 0x00, 0xe5, 0xe7, 0x00, 0xed, 0x00, /* 0xc0-0xc7 */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfc, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9e, /* 0xa0-0xa7 */ +}; + +static const unsigned char page22[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0xf9, 0xfb, 0x00, 0x00, 0x00, 0xec, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0xef, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0xf7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0xf0, 0x00, 0x00, 0xf3, 0xf2, 0x00, 0x00, /* 0x60-0x67 */ +}; + +static const unsigned char page23[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0xf4, 0xf5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ +}; + +static const unsigned char page25[256] = { + 0xc4, 0x00, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0xbf, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xd9, 0x00, 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xc2, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xc5, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0xcd, 0xba, 0xd5, 0xd6, 0xc9, 0xb8, 0xb7, 0xbb, /* 0x50-0x57 */ + 0xd4, 0xd3, 0xc8, 0xbe, 0xbd, 0xbc, 0xc6, 0xc7, /* 0x58-0x5f */ + 0xcc, 0xb5, 0xb6, 0xb9, 0xd1, 0xd2, 0xcb, 0xcf, /* 0x60-0x67 */ + 0xd0, 0xca, 0xd8, 0xd7, 0xce, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0xdf, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0xdb, 0x00, 0x00, 0x00, 0xdd, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0xde, 0xb0, 0xb1, 0xb2, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, NULL, NULL, page03, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + page20, NULL, page22, page23, NULL, page25, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x87, 0x81, 0x82, 0x83, 0x84, 0x85, 0xa0, 0x87, /* 0x80-0x87 */ + 0x88, 0x88, 0x8a, 0xa1, 0x93, 0x8d, 0x84, 0x83, /* 0x88-0x8f */ + 0x82, 0x85, 0x8a, 0x93, 0x94, 0x95, 0xa3, 0x97, /* 0x90-0x97 */ + 0x8d, 0x94, 0x81, 0x9b, 0x9c, 0x97, 0x9e, 0xa2, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa4, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0x95, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0x00, 0xe3, 0xe5, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xed, 0x00, 0x00, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x9a, 0x90, 0x8f, 0x8e, 0x91, 0x86, 0x80, /* 0x80-0x87 */ + 0x89, 0x89, 0x92, 0x8b, 0x8c, 0x98, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x8c, 0x99, 0xa9, 0x96, 0x9d, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0x86, 0x8b, 0x9f, 0x96, 0xa5, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0x00, 0xe1, 0xe2, 0x00, 0xe4, 0xe4, 0x00, 0x00, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0x00, 0xec, 0xe8, 0x00, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "cp860", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_cp860(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_cp860(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_cp860) +module_exit(exit_nls_cp860) + +MODULE_DESCRIPTION("NLS Codepage 860 (Portuguese)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_cp861.c b/fs/nls/nls_cp861.c new file mode 100644 index 000000000..b7397d079 --- /dev/null +++ b/fs/nls/nls_cp861.c @@ -0,0 +1,388 @@ +/* + * linux/fs/nls/nls_cp861.c + * + * Charset cp861 translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x00c7, 0x00fc, 0x00e9, 0x00e2, + 0x00e4, 0x00e0, 0x00e5, 0x00e7, + 0x00ea, 0x00eb, 0x00e8, 0x00d0, + 0x00f0, 0x00de, 0x00c4, 0x00c5, + /* 0x90*/ + 0x00c9, 0x00e6, 0x00c6, 0x00f4, + 0x00f6, 0x00fe, 0x00fb, 0x00dd, + 0x00fd, 0x00d6, 0x00dc, 0x00f8, + 0x00a3, 0x00d8, 0x20a7, 0x0192, + /* 0xa0*/ + 0x00e1, 0x00ed, 0x00f3, 0x00fa, + 0x00c1, 0x00cd, 0x00d3, 0x00da, + 0x00bf, 0x2310, 0x00ac, 0x00bd, + 0x00bc, 0x00a1, 0x00ab, 0x00bb, + /* 0xb0*/ + 0x2591, 0x2592, 0x2593, 0x2502, + 0x2524, 0x2561, 0x2562, 0x2556, + 0x2555, 0x2563, 0x2551, 0x2557, + 0x255d, 0x255c, 0x255b, 0x2510, + /* 0xc0*/ + 0x2514, 0x2534, 0x252c, 0x251c, + 0x2500, 0x253c, 0x255e, 0x255f, + 0x255a, 0x2554, 0x2569, 0x2566, + 0x2560, 0x2550, 0x256c, 0x2567, + /* 0xd0*/ + 0x2568, 0x2564, 0x2565, 0x2559, + 0x2558, 0x2552, 0x2553, 0x256b, + 0x256a, 0x2518, 0x250c, 0x2588, + 0x2584, 0x258c, 0x2590, 0x2580, + /* 0xe0*/ + 0x03b1, 0x00df, 0x0393, 0x03c0, + 0x03a3, 0x03c3, 0x00b5, 0x03c4, + 0x03a6, 0x0398, 0x03a9, 0x03b4, + 0x221e, 0x03c6, 0x03b5, 0x2229, + /* 0xf0*/ + 0x2261, 0x00b1, 0x2265, 0x2264, + 0x2320, 0x2321, 0x00f7, 0x2248, + 0x00b0, 0x2219, 0x00b7, 0x221a, + 0x207f, 0x00b2, 0x25a0, 0x00a0, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xff, 0xad, 0x00, 0x9c, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0xae, 0xaa, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0xf8, 0xf1, 0xfd, 0x00, 0x00, 0xe6, 0x00, 0xfa, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0xaf, 0xac, 0xab, 0x00, 0xa8, /* 0xb8-0xbf */ + 0x00, 0xa4, 0x00, 0x00, 0x8e, 0x8f, 0x92, 0x80, /* 0xc0-0xc7 */ + 0x00, 0x90, 0x00, 0x00, 0x00, 0xa5, 0x00, 0x00, /* 0xc8-0xcf */ + 0x8b, 0x00, 0x00, 0xa6, 0x00, 0x00, 0x99, 0x00, /* 0xd0-0xd7 */ + 0x9d, 0x00, 0xa7, 0x00, 0x9a, 0x97, 0x8d, 0xe1, /* 0xd8-0xdf */ + 0x85, 0xa0, 0x83, 0x00, 0x84, 0x86, 0x91, 0x87, /* 0xe0-0xe7 */ + 0x8a, 0x82, 0x88, 0x89, 0x00, 0xa1, 0x00, 0x00, /* 0xe8-0xef */ + 0x8c, 0x00, 0x00, 0xa2, 0x93, 0x00, 0x94, 0xf6, /* 0xf0-0xf7 */ + 0x9b, 0x00, 0xa3, 0x96, 0x81, 0x98, 0x95, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x9f, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ +}; + +static const unsigned char page03[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0xe2, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0xe9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0xe4, 0x00, 0x00, 0xe8, 0x00, /* 0xa0-0xa7 */ + 0x00, 0xea, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0xe0, 0x00, 0x00, 0xeb, 0xee, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0xe3, 0x00, 0x00, 0xe5, 0xe7, 0x00, 0xed, 0x00, /* 0xc0-0xc7 */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfc, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9e, /* 0xa0-0xa7 */ +}; + +static const unsigned char page22[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0xf9, 0xfb, 0x00, 0x00, 0x00, 0xec, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0xef, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0xf7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0xf0, 0x00, 0x00, 0xf3, 0xf2, 0x00, 0x00, /* 0x60-0x67 */ +}; + +static const unsigned char page23[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0xa9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0xf4, 0xf5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ +}; + +static const unsigned char page25[256] = { + 0xc4, 0x00, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0xbf, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xd9, 0x00, 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xc2, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xc5, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0xcd, 0xba, 0xd5, 0xd6, 0xc9, 0xb8, 0xb7, 0xbb, /* 0x50-0x57 */ + 0xd4, 0xd3, 0xc8, 0xbe, 0xbd, 0xbc, 0xc6, 0xc7, /* 0x58-0x5f */ + 0xcc, 0xb5, 0xb6, 0xb9, 0xd1, 0xd2, 0xcb, 0xcf, /* 0x60-0x67 */ + 0xd0, 0xca, 0xd8, 0xd7, 0xce, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0xdf, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0xdb, 0x00, 0x00, 0x00, 0xdd, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0xde, 0xb0, 0xb1, 0xb2, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, NULL, page03, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + page20, NULL, page22, page23, NULL, page25, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x87, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8c, 0x8c, 0x95, 0x84, 0x86, /* 0x88-0x8f */ + 0x82, 0x91, 0x91, 0x93, 0x94, 0x95, 0x96, 0x98, /* 0x90-0x97 */ + 0x98, 0x94, 0x81, 0x9b, 0x9c, 0x9b, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa0, 0xa1, 0xa2, 0xa3, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0x00, 0xe3, 0xe5, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xed, 0x00, 0x00, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x9a, 0x90, 0x00, 0x8e, 0x00, 0x8f, 0x80, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x8b, 0x8b, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x92, 0x92, 0x00, 0x99, 0x8d, 0x00, 0x97, /* 0x90-0x97 */ + 0x97, 0x99, 0x9a, 0x9d, 0x9c, 0x9d, 0x9e, 0x00, /* 0x98-0x9f */ + 0xa4, 0xa5, 0xa6, 0xa7, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0x00, 0xe1, 0xe2, 0x00, 0xe4, 0xe4, 0x00, 0x00, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0x00, 0xec, 0xe8, 0x00, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "cp861", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_cp861(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_cp861(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_cp861) +module_exit(exit_nls_cp861) + +MODULE_DESCRIPTION("NLS Codepage 861 (Icelandic)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_cp862.c b/fs/nls/nls_cp862.c new file mode 100644 index 000000000..fd3b95d1e --- /dev/null +++ b/fs/nls/nls_cp862.c @@ -0,0 +1,422 @@ +/* + * linux/fs/nls/nls_cp862.c + * + * Charset cp862 translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x05d0, 0x05d1, 0x05d2, 0x05d3, + 0x05d4, 0x05d5, 0x05d6, 0x05d7, + 0x05d8, 0x05d9, 0x05da, 0x05db, + 0x05dc, 0x05dd, 0x05de, 0x05df, + /* 0x90*/ + 0x05e0, 0x05e1, 0x05e2, 0x05e3, + 0x05e4, 0x05e5, 0x05e6, 0x05e7, + 0x05e8, 0x05e9, 0x05ea, 0x00a2, + 0x00a3, 0x00a5, 0x20a7, 0x0192, + /* 0xa0*/ + 0x00e1, 0x00ed, 0x00f3, 0x00fa, + 0x00f1, 0x00d1, 0x00aa, 0x00ba, + 0x00bf, 0x2310, 0x00ac, 0x00bd, + 0x00bc, 0x00a1, 0x00ab, 0x00bb, + /* 0xb0*/ + 0x2591, 0x2592, 0x2593, 0x2502, + 0x2524, 0x2561, 0x2562, 0x2556, + 0x2555, 0x2563, 0x2551, 0x2557, + 0x255d, 0x255c, 0x255b, 0x2510, + /* 0xc0*/ + 0x2514, 0x2534, 0x252c, 0x251c, + 0x2500, 0x253c, 0x255e, 0x255f, + 0x255a, 0x2554, 0x2569, 0x2566, + 0x2560, 0x2550, 0x256c, 0x2567, + /* 0xd0*/ + 0x2568, 0x2564, 0x2565, 0x2559, + 0x2558, 0x2552, 0x2553, 0x256b, + 0x256a, 0x2518, 0x250c, 0x2588, + 0x2584, 0x258c, 0x2590, 0x2580, + /* 0xe0*/ + 0x03b1, 0x00df, 0x0393, 0x03c0, + 0x03a3, 0x03c3, 0x00b5, 0x03c4, + 0x03a6, 0x0398, 0x03a9, 0x03b4, + 0x221e, 0x03c6, 0x03b5, 0x2229, + /* 0xf0*/ + 0x2261, 0x00b1, 0x2265, 0x2264, + 0x2320, 0x2321, 0x00f7, 0x2248, + 0x00b0, 0x2219, 0x00b7, 0x221a, + 0x207f, 0x00b2, 0x25a0, 0x00a0, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xff, 0xad, 0x9b, 0x9c, 0x00, 0x9d, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0xa6, 0xae, 0xaa, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0xf8, 0xf1, 0xfd, 0x00, 0x00, 0xe6, 0x00, 0xfa, /* 0xb0-0xb7 */ + 0x00, 0x00, 0xa7, 0xaf, 0xac, 0xab, 0x00, 0xa8, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0xa5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe1, /* 0xd8-0xdf */ + 0x00, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0xa1, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0xa4, 0x00, 0xa2, 0x00, 0x00, 0x00, 0xf6, /* 0xf0-0xf7 */ + 0x00, 0x00, 0xa3, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x9f, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ +}; + +static const unsigned char page03[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0xe2, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0xe9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0xe4, 0x00, 0x00, 0xe8, 0x00, /* 0xa0-0xa7 */ + 0x00, 0xea, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0xe0, 0x00, 0x00, 0xeb, 0xee, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0xe3, 0x00, 0x00, 0xe5, 0xe7, 0x00, 0xed, 0x00, /* 0xc0-0xc7 */ +}; + +static const unsigned char page05[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0xd0-0xd7 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0xd8-0xdf */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0xe0-0xe7 */ + 0x98, 0x99, 0x9a, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfc, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9e, /* 0xa0-0xa7 */ +}; + +static const unsigned char page22[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0xf9, 0xfb, 0x00, 0x00, 0x00, 0xec, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0xef, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0xf7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0xf0, 0x00, 0x00, 0xf3, 0xf2, 0x00, 0x00, /* 0x60-0x67 */ +}; + +static const unsigned char page23[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0xa9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0xf4, 0xf5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ +}; + +static const unsigned char page25[256] = { + 0xc4, 0x00, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0xbf, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xd9, 0x00, 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xc2, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xc5, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0xcd, 0xba, 0xd5, 0xd6, 0xc9, 0xb8, 0xb7, 0xbb, /* 0x50-0x57 */ + 0xd4, 0xd3, 0xc8, 0xbe, 0xbd, 0xbc, 0xc6, 0xc7, /* 0x58-0x5f */ + 0xcc, 0xb5, 0xb6, 0xb9, 0xd1, 0xd2, 0xcb, 0xcf, /* 0x60-0x67 */ + 0xd0, 0xca, 0xd8, 0xd7, 0xce, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0xdf, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0xdb, 0x00, 0x00, 0x00, 0xdd, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0xde, 0xb0, 0xb1, 0xb2, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, NULL, page03, NULL, page05, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + page20, NULL, page22, page23, NULL, page25, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa4, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0x00, 0xe3, 0xe5, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xed, 0x00, 0x00, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0xa5, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0x00, 0xe1, 0xe2, 0x00, 0xe4, 0xe4, 0x00, 0x00, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0x00, 0xec, 0xe8, 0x00, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "cp862", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_cp862(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_cp862(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_cp862) +module_exit(exit_nls_cp862) + +MODULE_DESCRIPTION("NLS Codepage 862 (Hebrew)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_cp863.c b/fs/nls/nls_cp863.c new file mode 100644 index 000000000..813ae7944 --- /dev/null +++ b/fs/nls/nls_cp863.c @@ -0,0 +1,382 @@ +/* + * linux/fs/nls/nls_cp863.c + * + * Charset cp863 translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x00c7, 0x00fc, 0x00e9, 0x00e2, + 0x00c2, 0x00e0, 0x00b6, 0x00e7, + 0x00ea, 0x00eb, 0x00e8, 0x00ef, + 0x00ee, 0x2017, 0x00c0, 0x00a7, + /* 0x90*/ + 0x00c9, 0x00c8, 0x00ca, 0x00f4, + 0x00cb, 0x00cf, 0x00fb, 0x00f9, + 0x00a4, 0x00d4, 0x00dc, 0x00a2, + 0x00a3, 0x00d9, 0x00db, 0x0192, + /* 0xa0*/ + 0x00a6, 0x00b4, 0x00f3, 0x00fa, + 0x00a8, 0x00b8, 0x00b3, 0x00af, + 0x00ce, 0x2310, 0x00ac, 0x00bd, + 0x00bc, 0x00be, 0x00ab, 0x00bb, + /* 0xb0*/ + 0x2591, 0x2592, 0x2593, 0x2502, + 0x2524, 0x2561, 0x2562, 0x2556, + 0x2555, 0x2563, 0x2551, 0x2557, + 0x255d, 0x255c, 0x255b, 0x2510, + /* 0xc0*/ + 0x2514, 0x2534, 0x252c, 0x251c, + 0x2500, 0x253c, 0x255e, 0x255f, + 0x255a, 0x2554, 0x2569, 0x2566, + 0x2560, 0x2550, 0x256c, 0x2567, + /* 0xd0*/ + 0x2568, 0x2564, 0x2565, 0x2559, + 0x2558, 0x2552, 0x2553, 0x256b, + 0x256a, 0x2518, 0x250c, 0x2588, + 0x2584, 0x258c, 0x2590, 0x2580, + /* 0xe0*/ + 0x03b1, 0x00df, 0x0393, 0x03c0, + 0x03a3, 0x03c3, 0x00b5, 0x03c4, + 0x03a6, 0x0398, 0x03a9, 0x03b4, + 0x221e, 0x03c6, 0x03b5, 0x2229, + /* 0xf0*/ + 0x2261, 0x00b1, 0x2265, 0x2264, + 0x2320, 0x2321, 0x00f7, 0x2248, + 0x00b0, 0x2219, 0x00b7, 0x221a, + 0x207f, 0x00b2, 0x25a0, 0x00a0, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xff, 0x00, 0x9b, 0x9c, 0x98, 0x00, 0xa0, 0x8f, /* 0xa0-0xa7 */ + 0xa4, 0x00, 0x00, 0xae, 0xaa, 0x00, 0x00, 0xa7, /* 0xa8-0xaf */ + 0xf8, 0xf1, 0xfd, 0xa6, 0xa1, 0xe6, 0x86, 0xfa, /* 0xb0-0xb7 */ + 0xa5, 0x00, 0x00, 0xaf, 0xac, 0xab, 0xad, 0x00, /* 0xb8-0xbf */ + 0x8e, 0x00, 0x84, 0x00, 0x00, 0x00, 0x00, 0x80, /* 0xc0-0xc7 */ + 0x91, 0x90, 0x92, 0x94, 0x00, 0x00, 0xa8, 0x95, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x99, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x9d, 0x00, 0x9e, 0x9a, 0x00, 0x00, 0xe1, /* 0xd8-0xdf */ + 0x85, 0x00, 0x83, 0x00, 0x00, 0x00, 0x00, 0x87, /* 0xe0-0xe7 */ + 0x8a, 0x82, 0x88, 0x89, 0x00, 0x00, 0x8c, 0x8b, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0xa2, 0x93, 0x00, 0x00, 0xf6, /* 0xf0-0xf7 */ + 0x00, 0x97, 0xa3, 0x96, 0x81, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x9f, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ +}; + +static const unsigned char page03[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0xe2, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0xe9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0xe4, 0x00, 0x00, 0xe8, 0x00, /* 0xa0-0xa7 */ + 0x00, 0xea, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0xe0, 0x00, 0x00, 0xeb, 0xee, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0xe3, 0x00, 0x00, 0xe5, 0xe7, 0x00, 0xed, 0x00, /* 0xc0-0xc7 */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8d, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfc, /* 0x78-0x7f */ +}; + +static const unsigned char page22[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0xf9, 0xfb, 0x00, 0x00, 0x00, 0xec, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0xef, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0xf7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0xf0, 0x00, 0x00, 0xf3, 0xf2, 0x00, 0x00, /* 0x60-0x67 */ +}; + +static const unsigned char page23[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0xa9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0xf4, 0xf5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ +}; + +static const unsigned char page25[256] = { + 0xc4, 0x00, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0xbf, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xd9, 0x00, 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xc2, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xc5, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0xcd, 0xba, 0xd5, 0xd6, 0xc9, 0xb8, 0xb7, 0xbb, /* 0x50-0x57 */ + 0xd4, 0xd3, 0xc8, 0xbe, 0xbd, 0xbc, 0xc6, 0xc7, /* 0x58-0x5f */ + 0xcc, 0xb5, 0xb6, 0xb9, 0xd1, 0xd2, 0xcb, 0xcf, /* 0x60-0x67 */ + 0xd0, 0xca, 0xd8, 0xd7, 0xce, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0xdf, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0xdb, 0x00, 0x00, 0x00, 0xdd, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0xde, 0xb0, 0xb1, 0xb2, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, NULL, page03, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + page20, NULL, page22, page23, NULL, page25, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x87, 0x81, 0x82, 0x83, 0x83, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x85, 0x8f, /* 0x88-0x8f */ + 0x82, 0x8a, 0x88, 0x93, 0x89, 0x8b, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x93, 0x81, 0x9b, 0x9c, 0x97, 0x96, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0x8c, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0x00, 0xe3, 0xe5, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xed, 0x00, 0x00, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x9a, 0x90, 0x84, 0x84, 0x8e, 0x86, 0x80, /* 0x80-0x87 */ + 0x92, 0x94, 0x91, 0x95, 0xa8, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x99, 0x94, 0x95, 0x9e, 0x9d, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x00, /* 0x98-0x9f */ + 0xa0, 0xa1, 0x00, 0x00, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0x00, 0xe1, 0xe2, 0x00, 0xe4, 0xe4, 0x00, 0x00, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0x00, 0xec, 0xe8, 0x00, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "cp863", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_cp863(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_cp863(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_cp863) +module_exit(exit_nls_cp863) + +MODULE_DESCRIPTION("NLS Codepage 863 (Canadian French)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_cp864.c b/fs/nls/nls_cp864.c new file mode 100644 index 000000000..d9eb6d5cd --- /dev/null +++ b/fs/nls/nls_cp864.c @@ -0,0 +1,408 @@ +/* + * linux/fs/nls/nls_cp864.c + * + * Charset cp864 translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x066a, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x00b0, 0x00b7, 0x2219, 0x221a, + 0x2592, 0x2500, 0x2502, 0x253c, + 0x2524, 0x252c, 0x251c, 0x2534, + 0x2510, 0x250c, 0x2514, 0x2518, + /* 0x90*/ + 0x03b2, 0x221e, 0x03c6, 0x00b1, + 0x00bd, 0x00bc, 0x2248, 0x00ab, + 0x00bb, 0xfef7, 0xfef8, 0x0000, + 0x0000, 0xfefb, 0xfefc, 0x0000, + /* 0xa0*/ + 0x00a0, 0x00ad, 0xfe82, 0x00a3, + 0x00a4, 0xfe84, 0x0000, 0x0000, + 0xfe8e, 0xfe8f, 0xfe95, 0xfe99, + 0x060c, 0xfe9d, 0xfea1, 0xfea5, + /* 0xb0*/ + 0x0660, 0x0661, 0x0662, 0x0663, + 0x0664, 0x0665, 0x0666, 0x0667, + 0x0668, 0x0669, 0xfed1, 0x061b, + 0xfeb1, 0xfeb5, 0xfeb9, 0x061f, + /* 0xc0*/ + 0x00a2, 0xfe80, 0xfe81, 0xfe83, + 0xfe85, 0xfeca, 0xfe8b, 0xfe8d, + 0xfe91, 0xfe93, 0xfe97, 0xfe9b, + 0xfe9f, 0xfea3, 0xfea7, 0xfea9, + /* 0xd0*/ + 0xfeab, 0xfead, 0xfeaf, 0xfeb3, + 0xfeb7, 0xfebb, 0xfebf, 0xfec1, + 0xfec5, 0xfecb, 0xfecf, 0x00a6, + 0x00ac, 0x00f7, 0x00d7, 0xfec9, + /* 0xe0*/ + 0x0640, 0xfed3, 0xfed7, 0xfedb, + 0xfedf, 0xfee3, 0xfee7, 0xfeeb, + 0xfeed, 0xfeef, 0xfef3, 0xfebd, + 0xfecc, 0xfece, 0xfecd, 0xfee1, + /* 0xf0*/ + 0xfe7d, 0x0651, 0xfee5, 0xfee9, + 0xfeec, 0xfef0, 0xfef2, 0xfed0, + 0xfed5, 0xfef5, 0xfef6, 0xfedd, + 0xfed9, 0xfef1, 0x25a0, 0x0000, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x00, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xa0, 0x00, 0xc0, 0xa3, 0xa4, 0x00, 0xdb, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x97, 0xdc, 0xa1, 0x00, 0x00, /* 0xa8-0xaf */ + 0x80, 0x93, 0x00, 0x00, 0x00, 0x00, 0x00, 0x81, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x98, 0x95, 0x94, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xde, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xdd, /* 0xf0-0xf7 */ +}; + +static const unsigned char page03[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0x00, /* 0xc0-0xc7 */ +}; + +static const unsigned char page06[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xac, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0xbb, 0x00, 0x00, 0x00, 0xbf, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0xf1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0x60-0x67 */ + 0xb8, 0xb9, 0x25, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ +}; + +static const unsigned char page22[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x82, 0x83, 0x00, 0x00, 0x00, 0x91, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x96, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ +}; + +static const unsigned char page25[256] = { + 0x85, 0x00, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x8d, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x8c, 0x00, 0x00, 0x00, 0x8e, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x8f, 0x00, 0x00, 0x00, 0x8a, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x88, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x8b, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x87, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ +}; + +static const unsigned char pagefe[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x00, 0x00, /* 0x78-0x7f */ + + 0xc1, 0xc2, 0xa2, 0xc3, 0xa5, 0xc4, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0xc6, 0x00, 0xc7, 0xa8, 0xa9, /* 0x88-0x8f */ + 0x00, 0xc8, 0x00, 0xc9, 0x00, 0xaa, 0x00, 0xca, /* 0x90-0x97 */ + 0x00, 0xab, 0x00, 0xcb, 0x00, 0xad, 0x00, 0xcc, /* 0x98-0x9f */ + 0x00, 0xae, 0x00, 0xcd, 0x00, 0xaf, 0x00, 0xce, /* 0xa0-0xa7 */ + 0x00, 0xcf, 0x00, 0xd0, 0x00, 0xd1, 0x00, 0xd2, /* 0xa8-0xaf */ + 0x00, 0xbc, 0x00, 0xd3, 0x00, 0xbd, 0x00, 0xd4, /* 0xb0-0xb7 */ + 0x00, 0xbe, 0x00, 0xd5, 0x00, 0xeb, 0x00, 0xd6, /* 0xb8-0xbf */ + 0x00, 0xd7, 0x00, 0x00, 0x00, 0xd8, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0xdf, 0xc5, 0xd9, 0xec, 0xee, 0xed, 0xda, /* 0xc8-0xcf */ + 0xf7, 0xba, 0x00, 0xe1, 0x00, 0xf8, 0x00, 0xe2, /* 0xd0-0xd7 */ + 0x00, 0xfc, 0x00, 0xe3, 0x00, 0xfb, 0x00, 0xe4, /* 0xd8-0xdf */ + 0x00, 0xef, 0x00, 0xe5, 0x00, 0xf2, 0x00, 0xe6, /* 0xe0-0xe7 */ + 0x00, 0xf3, 0x00, 0xe7, 0xf4, 0xe8, 0x00, 0xe9, /* 0xe8-0xef */ + 0xf5, 0xfd, 0xf6, 0xea, 0x00, 0xf9, 0xfa, 0x99, /* 0xf0-0xf7 */ + 0x9a, 0x00, 0x00, 0x9d, 0x9e, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, NULL, NULL, page03, NULL, NULL, page06, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, page22, NULL, NULL, page25, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, pagefe, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x00, 0x00, 0x9d, 0x9e, 0x00, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0x00, 0x00, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x00, 0x91, 0x00, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x00, 0x00, 0x9d, 0x9e, 0x00, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0x00, 0x00, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0x00, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "cp864", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_cp864(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_cp864(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_cp864) +module_exit(exit_nls_cp864) + +MODULE_DESCRIPTION("NLS Codepage 864 (Arabic)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_cp865.c b/fs/nls/nls_cp865.c new file mode 100644 index 000000000..2678ffd98 --- /dev/null +++ b/fs/nls/nls_cp865.c @@ -0,0 +1,388 @@ +/* + * linux/fs/nls/nls_cp865.c + * + * Charset cp865 translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x00c7, 0x00fc, 0x00e9, 0x00e2, + 0x00e4, 0x00e0, 0x00e5, 0x00e7, + 0x00ea, 0x00eb, 0x00e8, 0x00ef, + 0x00ee, 0x00ec, 0x00c4, 0x00c5, + /* 0x90*/ + 0x00c9, 0x00e6, 0x00c6, 0x00f4, + 0x00f6, 0x00f2, 0x00fb, 0x00f9, + 0x00ff, 0x00d6, 0x00dc, 0x00f8, + 0x00a3, 0x00d8, 0x20a7, 0x0192, + /* 0xa0*/ + 0x00e1, 0x00ed, 0x00f3, 0x00fa, + 0x00f1, 0x00d1, 0x00aa, 0x00ba, + 0x00bf, 0x2310, 0x00ac, 0x00bd, + 0x00bc, 0x00a1, 0x00ab, 0x00a4, + /* 0xb0*/ + 0x2591, 0x2592, 0x2593, 0x2502, + 0x2524, 0x2561, 0x2562, 0x2556, + 0x2555, 0x2563, 0x2551, 0x2557, + 0x255d, 0x255c, 0x255b, 0x2510, + /* 0xc0*/ + 0x2514, 0x2534, 0x252c, 0x251c, + 0x2500, 0x253c, 0x255e, 0x255f, + 0x255a, 0x2554, 0x2569, 0x2566, + 0x2560, 0x2550, 0x256c, 0x2567, + /* 0xd0*/ + 0x2568, 0x2564, 0x2565, 0x2559, + 0x2558, 0x2552, 0x2553, 0x256b, + 0x256a, 0x2518, 0x250c, 0x2588, + 0x2584, 0x258c, 0x2590, 0x2580, + /* 0xe0*/ + 0x03b1, 0x00df, 0x0393, 0x03c0, + 0x03a3, 0x03c3, 0x00b5, 0x03c4, + 0x03a6, 0x0398, 0x03a9, 0x03b4, + 0x221e, 0x03c6, 0x03b5, 0x2229, + /* 0xf0*/ + 0x2261, 0x00b1, 0x2265, 0x2264, + 0x2320, 0x2321, 0x00f7, 0x2248, + 0x00b0, 0x2219, 0x00b7, 0x221a, + 0x207f, 0x00b2, 0x25a0, 0x00a0, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xff, 0xad, 0x00, 0x9c, 0xaf, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0xa6, 0xae, 0xaa, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0xf8, 0xf1, 0xfd, 0x00, 0x00, 0xe6, 0x00, 0xfa, /* 0xb0-0xb7 */ + 0x00, 0x00, 0xa7, 0x00, 0xac, 0xab, 0x00, 0xa8, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x8e, 0x8f, 0x92, 0x80, /* 0xc0-0xc7 */ + 0x00, 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0xa5, 0x00, 0x00, 0x00, 0x00, 0x99, 0x00, /* 0xd0-0xd7 */ + 0x9d, 0x00, 0x00, 0x00, 0x9a, 0x00, 0x00, 0xe1, /* 0xd8-0xdf */ + 0x85, 0xa0, 0x83, 0x00, 0x84, 0x86, 0x91, 0x87, /* 0xe0-0xe7 */ + 0x8a, 0x82, 0x88, 0x89, 0x8d, 0xa1, 0x8c, 0x8b, /* 0xe8-0xef */ + 0x00, 0xa4, 0x95, 0xa2, 0x93, 0x00, 0x94, 0xf6, /* 0xf0-0xf7 */ + 0x9b, 0x97, 0xa3, 0x96, 0x81, 0x00, 0x00, 0x98, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x9f, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ +}; + +static const unsigned char page03[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0xe2, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0xe9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0xe4, 0x00, 0x00, 0xe8, 0x00, /* 0xa0-0xa7 */ + 0x00, 0xea, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0xe0, 0x00, 0x00, 0xeb, 0xee, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0xe3, 0x00, 0x00, 0xe5, 0xe7, 0x00, 0xed, 0x00, /* 0xc0-0xc7 */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfc, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9e, /* 0xa0-0xa7 */ +}; + +static const unsigned char page22[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0xf9, 0xfb, 0x00, 0x00, 0x00, 0xec, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0xef, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0xf7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0xf0, 0x00, 0x00, 0xf3, 0xf2, 0x00, 0x00, /* 0x60-0x67 */ +}; + +static const unsigned char page23[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0xa9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0xf4, 0xf5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ +}; + +static const unsigned char page25[256] = { + 0xc4, 0x00, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0xbf, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xd9, 0x00, 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xc2, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xc5, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0xcd, 0xba, 0xd5, 0xd6, 0xc9, 0xb8, 0xb7, 0xbb, /* 0x50-0x57 */ + 0xd4, 0xd3, 0xc8, 0xbe, 0xbd, 0xbc, 0xc6, 0xc7, /* 0x58-0x5f */ + 0xcc, 0xb5, 0xb6, 0xb9, 0xd1, 0xd2, 0xcb, 0xcf, /* 0x60-0x67 */ + 0xd0, 0xca, 0xd8, 0xd7, 0xce, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0xdf, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0xdb, 0x00, 0x00, 0x00, 0xdd, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0xde, 0xb0, 0xb1, 0xb2, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, NULL, page03, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + page20, NULL, page22, page23, NULL, page25, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x87, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x84, 0x86, /* 0x88-0x8f */ + 0x82, 0x91, 0x91, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x94, 0x81, 0x9b, 0x9c, 0x9b, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa4, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0x00, 0xe3, 0xe5, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xed, 0x00, 0x00, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x9a, 0x90, 0x00, 0x8e, 0x00, 0x8f, 0x80, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x92, 0x92, 0x00, 0x99, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x99, 0x9a, 0x9d, 0x9c, 0x9d, 0x9e, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0xa5, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0x00, 0xe1, 0xe2, 0x00, 0xe4, 0xe4, 0x00, 0x00, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0x00, 0xec, 0xe8, 0x00, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "cp865", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_cp865(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_cp865(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_cp865) +module_exit(exit_nls_cp865) + +MODULE_DESCRIPTION("NLS Codepage 865 (Norwegian, Danish)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_cp866.c b/fs/nls/nls_cp866.c new file mode 100644 index 000000000..7e93d0a38 --- /dev/null +++ b/fs/nls/nls_cp866.c @@ -0,0 +1,306 @@ +/* + * linux/fs/nls/nls_cp866.c + * + * Charset cp866 translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x0410, 0x0411, 0x0412, 0x0413, + 0x0414, 0x0415, 0x0416, 0x0417, + 0x0418, 0x0419, 0x041a, 0x041b, + 0x041c, 0x041d, 0x041e, 0x041f, + /* 0x90*/ + 0x0420, 0x0421, 0x0422, 0x0423, + 0x0424, 0x0425, 0x0426, 0x0427, + 0x0428, 0x0429, 0x042a, 0x042b, + 0x042c, 0x042d, 0x042e, 0x042f, + /* 0xa0*/ + 0x0430, 0x0431, 0x0432, 0x0433, + 0x0434, 0x0435, 0x0436, 0x0437, + 0x0438, 0x0439, 0x043a, 0x043b, + 0x043c, 0x043d, 0x043e, 0x043f, + /* 0xb0*/ + 0x2591, 0x2592, 0x2593, 0x2502, + 0x2524, 0x2561, 0x2562, 0x2556, + 0x2555, 0x2563, 0x2551, 0x2557, + 0x255d, 0x255c, 0x255b, 0x2510, + /* 0xc0*/ + 0x2514, 0x2534, 0x252c, 0x251c, + 0x2500, 0x253c, 0x255e, 0x255f, + 0x255a, 0x2554, 0x2569, 0x2566, + 0x2560, 0x2550, 0x256c, 0x2567, + /* 0xd0*/ + 0x2568, 0x2564, 0x2565, 0x2559, + 0x2558, 0x2552, 0x2553, 0x256b, + 0x256a, 0x2518, 0x250c, 0x2588, + 0x2584, 0x258c, 0x2590, 0x2580, + /* 0xe0*/ + 0x0440, 0x0441, 0x0442, 0x0443, + 0x0444, 0x0445, 0x0446, 0x0447, + 0x0448, 0x0449, 0x044a, 0x044b, + 0x044c, 0x044d, 0x044e, 0x044f, + /* 0xf0*/ + 0x0401, 0x0451, 0x0404, 0x0454, + 0x0407, 0x0457, 0x040e, 0x045e, + 0x00b0, 0x2219, 0x00b7, 0x221a, + 0x2116, 0x00a4, 0x25a0, 0x00a0, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xff, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfa, /* 0xb0-0xb7 */ +}; + +static const unsigned char page04[256] = { + 0x00, 0xf0, 0x00, 0x00, 0xf2, 0x00, 0x00, 0xf4, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf6, 0x00, /* 0x08-0x0f */ + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x10-0x17 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x18-0x1f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x20-0x27 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x28-0x2f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0x30-0x37 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0x38-0x3f */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0x40-0x47 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0x48-0x4f */ + 0x00, 0xf1, 0x00, 0x00, 0xf3, 0x00, 0x00, 0xf5, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf7, 0x00, /* 0x58-0x5f */ +}; + +static const unsigned char page21[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfc, 0x00, /* 0x10-0x17 */ +}; + +static const unsigned char page22[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0xf9, 0xfb, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ +}; + +static const unsigned char page25[256] = { + 0xc4, 0x00, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0xbf, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xd9, 0x00, 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xc2, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xc5, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0xcd, 0xba, 0xd5, 0xd6, 0xc9, 0xb8, 0xb7, 0xbb, /* 0x50-0x57 */ + 0xd4, 0xd3, 0xc8, 0xbe, 0xbd, 0xbc, 0xc6, 0xc7, /* 0x58-0x5f */ + 0xcc, 0xb5, 0xb6, 0xb9, 0xd1, 0xd2, 0xcb, 0xcf, /* 0x60-0x67 */ + 0xd0, 0xca, 0xd8, 0xd7, 0xce, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0xdf, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0xdb, 0x00, 0x00, 0x00, 0xdd, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0xde, 0xb0, 0xb1, 0xb2, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, NULL, NULL, NULL, page04, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, page21, page22, NULL, NULL, page25, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0x80-0x87 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0x88-0x8f */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0x90-0x97 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf1, 0xf1, 0xf3, 0xf3, 0xf5, 0xf5, 0xf7, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0xa0-0xa7 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0xe0-0xe7 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0xe8-0xef */ + 0xf0, 0xf0, 0xf2, 0xf2, 0xf4, 0xf4, 0xf6, 0xf6, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "cp866", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_cp866(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_cp866(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_cp866) +module_exit(exit_nls_cp866) + +MODULE_DESCRIPTION("NLS Codepage 866 (Cyrillic/Russian)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_cp869.c b/fs/nls/nls_cp869.c new file mode 100644 index 000000000..4491737dd --- /dev/null +++ b/fs/nls/nls_cp869.c @@ -0,0 +1,316 @@ +/* + * linux/fs/nls/nls_cp869.c + * + * Charset cp869 translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0386, 0x0000, + 0x00b7, 0x00ac, 0x00a6, 0x2018, + 0x2019, 0x0388, 0x2015, 0x0389, + /* 0x90*/ + 0x038a, 0x03aa, 0x038c, 0x0000, + 0x0000, 0x038e, 0x03ab, 0x00a9, + 0x038f, 0x00b2, 0x00b3, 0x03ac, + 0x00a3, 0x03ad, 0x03ae, 0x03af, + /* 0xa0*/ + 0x03ca, 0x0390, 0x03cc, 0x03cd, + 0x0391, 0x0392, 0x0393, 0x0394, + 0x0395, 0x0396, 0x0397, 0x00bd, + 0x0398, 0x0399, 0x00ab, 0x00bb, + /* 0xb0*/ + 0x2591, 0x2592, 0x2593, 0x2502, + 0x2524, 0x039a, 0x039b, 0x039c, + 0x039d, 0x2563, 0x2551, 0x2557, + 0x255d, 0x039e, 0x039f, 0x2510, + /* 0xc0*/ + 0x2514, 0x2534, 0x252c, 0x251c, + 0x2500, 0x253c, 0x03a0, 0x03a1, + 0x255a, 0x2554, 0x2569, 0x2566, + 0x2560, 0x2550, 0x256c, 0x03a3, + /* 0xd0*/ + 0x03a4, 0x03a5, 0x03a6, 0x03a7, + 0x03a8, 0x03a9, 0x03b1, 0x03b2, + 0x03b3, 0x2518, 0x250c, 0x2588, + 0x2584, 0x03b4, 0x03b5, 0x2580, + /* 0xe0*/ + 0x03b6, 0x03b7, 0x03b8, 0x03b9, + 0x03ba, 0x03bb, 0x03bc, 0x03bd, + 0x03be, 0x03bf, 0x03c0, 0x03c1, + 0x03c3, 0x03c2, 0x03c4, 0x0384, + /* 0xf0*/ + 0x00ad, 0x00b1, 0x03c5, 0x03c6, + 0x03c7, 0x00a7, 0x03c8, 0x0385, + 0x00b0, 0x00a8, 0x03c9, 0x03cb, + 0x03b0, 0x03ce, 0x25a0, 0x00a0, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xff, 0x00, 0x00, 0x9c, 0x00, 0x00, 0x8a, 0xf5, /* 0xa0-0xa7 */ + 0xf9, 0x97, 0x00, 0xae, 0x89, 0xf0, 0x00, 0x00, /* 0xa8-0xaf */ + 0xf8, 0xf1, 0x99, 0x9a, 0x00, 0x00, 0x00, 0x88, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0xaf, 0x00, 0xab, 0x00, 0x00, /* 0xb8-0xbf */ +}; + +static const unsigned char page03[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0xef, 0xf7, 0x86, 0x00, /* 0x80-0x87 */ + 0x8d, 0x8f, 0x90, 0x00, 0x92, 0x00, 0x95, 0x98, /* 0x88-0x8f */ + 0xa1, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, /* 0x90-0x97 */ + 0xac, 0xad, 0xb5, 0xb6, 0xb7, 0xb8, 0xbd, 0xbe, /* 0x98-0x9f */ + 0xc6, 0xc7, 0x00, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, /* 0xa0-0xa7 */ + 0xd4, 0xd5, 0x91, 0x96, 0x9b, 0x9d, 0x9e, 0x9f, /* 0xa8-0xaf */ + 0xfc, 0xd6, 0xd7, 0xd8, 0xdd, 0xde, 0xe0, 0xe1, /* 0xb0-0xb7 */ + 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, /* 0xb8-0xbf */ + 0xea, 0xeb, 0xed, 0xec, 0xee, 0xf2, 0xf3, 0xf4, /* 0xc0-0xc7 */ + 0xf6, 0xfa, 0xa0, 0xfb, 0xa2, 0xa3, 0xfd, 0x00, /* 0xc8-0xcf */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x8e, 0x00, 0x00, /* 0x10-0x17 */ + 0x8b, 0x8c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ +}; + +static const unsigned char page25[256] = { + 0xc4, 0x00, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0xbf, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xd9, 0x00, 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xc2, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xc5, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0xcd, 0xba, 0x00, 0x00, 0xc9, 0x00, 0x00, 0xbb, /* 0x50-0x57 */ + 0x00, 0x00, 0xc8, 0x00, 0x00, 0xbc, 0x00, 0x00, /* 0x58-0x5f */ + 0xcc, 0x00, 0x00, 0xb9, 0x00, 0x00, 0xcb, 0x00, /* 0x60-0x67 */ + 0x00, 0xca, 0x00, 0x00, 0xce, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0xdf, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0xdb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0xb0, 0xb1, 0xb2, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, NULL, NULL, page03, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + page20, NULL, NULL, NULL, NULL, page25, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9b, 0x00, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x9d, 0x8e, 0x9e, /* 0x88-0x8f */ + 0x9f, 0xa0, 0xa2, 0x00, 0x00, 0xa3, 0xfb, 0x97, /* 0x90-0x97 */ + 0xfd, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xd6, 0xd7, 0xd8, 0xdd, /* 0xa0-0xa7 */ + 0xde, 0xe0, 0xe1, 0xab, 0xe2, 0xe3, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xe4, 0xe5, 0xe6, /* 0xb0-0xb7 */ + 0xe7, 0xb9, 0xba, 0xbb, 0xbc, 0xe8, 0xe9, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xea, 0xeb, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xec, /* 0xc8-0xcf */ + 0xee, 0xf2, 0xf3, 0xf4, 0xf6, 0xfa, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x86, 0x00, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x00, 0x00, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x86, 0x9c, 0x8d, 0x8f, 0x90, /* 0x98-0x9f */ + 0x91, 0xa1, 0x92, 0x95, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xa4, 0xa5, /* 0xd0-0xd7 */ + 0xa6, 0xd9, 0xda, 0xdb, 0xdc, 0xa7, 0xa8, 0xdf, /* 0xd8-0xdf */ + 0xa9, 0xaa, 0xac, 0xad, 0xb5, 0xb6, 0xb7, 0xb8, /* 0xe0-0xe7 */ + 0xbd, 0xbe, 0xc6, 0xc7, 0xcf, 0xcf, 0xd0, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xd1, 0xd2, 0xd3, 0xf5, 0xd4, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xd5, 0x96, 0xfc, 0x98, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "cp869", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_cp869(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_cp869(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_cp869) +module_exit(exit_nls_cp869) + +MODULE_DESCRIPTION("NLS Codepage 869 (Greek)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_cp874.c b/fs/nls/nls_cp874.c new file mode 100644 index 000000000..4fcfbf8ca --- /dev/null +++ b/fs/nls/nls_cp874.c @@ -0,0 +1,276 @@ +/* + * linux/fs/nls/nls_cp874.c + * + * Charset cp874 translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x2026, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, + /* 0x90*/ + 0x0000, 0x2018, 0x2019, 0x201c, + 0x201d, 0x2022, 0x2013, 0x2014, + 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, + /* 0xa0*/ + 0x00a0, 0x0e01, 0x0e02, 0x0e03, + 0x0e04, 0x0e05, 0x0e06, 0x0e07, + 0x0e08, 0x0e09, 0x0e0a, 0x0e0b, + 0x0e0c, 0x0e0d, 0x0e0e, 0x0e0f, + /* 0xb0*/ + 0x0e10, 0x0e11, 0x0e12, 0x0e13, + 0x0e14, 0x0e15, 0x0e16, 0x0e17, + 0x0e18, 0x0e19, 0x0e1a, 0x0e1b, + 0x0e1c, 0x0e1d, 0x0e1e, 0x0e1f, + /* 0xc0*/ + 0x0e20, 0x0e21, 0x0e22, 0x0e23, + 0x0e24, 0x0e25, 0x0e26, 0x0e27, + 0x0e28, 0x0e29, 0x0e2a, 0x0e2b, + 0x0e2c, 0x0e2d, 0x0e2e, 0x0e2f, + /* 0xd0*/ + 0x0e30, 0x0e31, 0x0e32, 0x0e33, + 0x0e34, 0x0e35, 0x0e36, 0x0e37, + 0x0e38, 0x0e39, 0x0e3a, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0e3f, + /* 0xe0*/ + 0x0e40, 0x0e41, 0x0e42, 0x0e43, + 0x0e44, 0x0e45, 0x0e46, 0x0e47, + 0x0e48, 0x0e49, 0x0e4a, 0x0e4b, + 0x0e4c, 0x0e4d, 0x0e4e, 0x0e4f, + /* 0xf0*/ + 0x0e50, 0x0e51, 0x0e52, 0x0e53, + 0x0e54, 0x0e55, 0x0e56, 0x0e57, + 0x0e58, 0x0e59, 0x0e5a, 0x0e5b, + 0x0000, 0x0000, 0x0000, 0x0000, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xa0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ +}; + +static const unsigned char page0e[256] = { + 0x00, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0x00-0x07 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0x08-0x0f */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0x10-0x17 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0x18-0x1f */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0x20-0x27 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0x28-0x2f */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0x30-0x37 */ + 0xd8, 0xd9, 0xda, 0x00, 0x00, 0x00, 0x00, 0xdf, /* 0x38-0x3f */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0x40-0x47 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0x48-0x4f */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0x50-0x57 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x96, 0x97, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x91, 0x92, 0x00, 0x00, 0x93, 0x94, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x95, 0x00, 0x00, 0x00, 0x85, 0x00, /* 0x20-0x27 */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, page0e, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + page20, NULL, NULL, NULL, NULL, NULL, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x85, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0x00, 0x00, 0x00, 0x00, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x85, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0x00, 0x00, 0x00, 0x00, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "cp874", + .alias = "tis-620", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_cp874(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_cp874(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_cp874) +module_exit(exit_nls_cp874) + +MODULE_DESCRIPTION("NLS Thai charset (CP874, TIS-620)"); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_ALIAS_NLS(tis-620); diff --git a/fs/nls/nls_cp932.c b/fs/nls/nls_cp932.c new file mode 100644 index 000000000..e5e6270fc --- /dev/null +++ b/fs/nls/nls_cp932.c @@ -0,0 +1,7934 @@ +/* + * linux/fs/nls/nls_cp932.c + * + * Charset cp932 translation tables. + * This translation table was generated automatically, the + * original table can be download from the Microsoft website. + * (http://www.microsoft.com/typography/unicode/unicodecp.htm) + */ + +#include +#include +#include +#include +#include + +static const wchar_t c2u_81[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x3000,0x3001,0x3002,0xFF0C,0xFF0E,0x30FB,0xFF1A,0xFF1B,/* 0x40-0x47 */ + 0xFF1F,0xFF01,0x309B,0x309C,0x00B4,0xFF40,0x00A8,0xFF3E,/* 0x48-0x4F */ + 0xFFE3,0xFF3F,0x30FD,0x30FE,0x309D,0x309E,0x3003,0x4EDD,/* 0x50-0x57 */ + 0x3005,0x3006,0x3007,0x30FC,0x2015,0x2010,0xFF0F,0xFF3C,/* 0x58-0x5F */ + 0xFF5E,0x2225,0xFF5C,0x2026,0x2025,0x2018,0x2019,0x201C,/* 0x60-0x67 */ + 0x201D,0xFF08,0xFF09,0x3014,0x3015,0xFF3B,0xFF3D,0xFF5B,/* 0x68-0x6F */ + 0xFF5D,0x3008,0x3009,0x300A,0x300B,0x300C,0x300D,0x300E,/* 0x70-0x77 */ + 0x300F,0x3010,0x3011,0xFF0B,0xFF0D,0x00B1,0x00D7,0x0000,/* 0x78-0x7F */ + + 0x00F7,0xFF1D,0x2260,0xFF1C,0xFF1E,0x2266,0x2267,0x221E,/* 0x80-0x87 */ + 0x2234,0x2642,0x2640,0x00B0,0x2032,0x2033,0x2103,0xFFE5,/* 0x88-0x8F */ + 0xFF04,0xFFE0,0xFFE1,0xFF05,0xFF03,0xFF06,0xFF0A,0xFF20,/* 0x90-0x97 */ + 0x00A7,0x2606,0x2605,0x25CB,0x25CF,0x25CE,0x25C7,0x25C6,/* 0x98-0x9F */ + 0x25A1,0x25A0,0x25B3,0x25B2,0x25BD,0x25BC,0x203B,0x3012,/* 0xA0-0xA7 */ + 0x2192,0x2190,0x2191,0x2193,0x3013,0x0000,0x0000,0x0000,/* 0xA8-0xAF */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xB0-0xB7 */ + 0x2208,0x220B,0x2286,0x2287,0x2282,0x2283,0x222A,0x2229,/* 0xB8-0xBF */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xC0-0xC7 */ + 0x2227,0x2228,0xFFE2,0x21D2,0x21D4,0x2200,0x2203,0x0000,/* 0xC8-0xCF */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xD0-0xD7 */ + 0x0000,0x0000,0x2220,0x22A5,0x2312,0x2202,0x2207,0x2261,/* 0xD8-0xDF */ + 0x2252,0x226A,0x226B,0x221A,0x223D,0x221D,0x2235,0x222B,/* 0xE0-0xE7 */ + 0x222C,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xE8-0xEF */ + 0x212B,0x2030,0x266F,0x266D,0x266A,0x2020,0x2021,0x00B6,/* 0xF0-0xF7 */ + 0x0000,0x0000,0x0000,0x0000,0x25EF,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_82[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0xFF10,/* 0x48-0x4F */ + 0xFF11,0xFF12,0xFF13,0xFF14,0xFF15,0xFF16,0xFF17,0xFF18,/* 0x50-0x57 */ + 0xFF19,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0xFF21,0xFF22,0xFF23,0xFF24,0xFF25,0xFF26,0xFF27,0xFF28,/* 0x60-0x67 */ + 0xFF29,0xFF2A,0xFF2B,0xFF2C,0xFF2D,0xFF2E,0xFF2F,0xFF30,/* 0x68-0x6F */ + 0xFF31,0xFF32,0xFF33,0xFF34,0xFF35,0xFF36,0xFF37,0xFF38,/* 0x70-0x77 */ + 0xFF39,0xFF3A,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xFF41,0xFF42,0xFF43,0xFF44,0xFF45,0xFF46,0xFF47,/* 0x80-0x87 */ + 0xFF48,0xFF49,0xFF4A,0xFF4B,0xFF4C,0xFF4D,0xFF4E,0xFF4F,/* 0x88-0x8F */ + 0xFF50,0xFF51,0xFF52,0xFF53,0xFF54,0xFF55,0xFF56,0xFF57,/* 0x90-0x97 */ + 0xFF58,0xFF59,0xFF5A,0x0000,0x0000,0x0000,0x0000,0x3041,/* 0x98-0x9F */ + 0x3042,0x3043,0x3044,0x3045,0x3046,0x3047,0x3048,0x3049,/* 0xA0-0xA7 */ + 0x304A,0x304B,0x304C,0x304D,0x304E,0x304F,0x3050,0x3051,/* 0xA8-0xAF */ + 0x3052,0x3053,0x3054,0x3055,0x3056,0x3057,0x3058,0x3059,/* 0xB0-0xB7 */ + 0x305A,0x305B,0x305C,0x305D,0x305E,0x305F,0x3060,0x3061,/* 0xB8-0xBF */ + 0x3062,0x3063,0x3064,0x3065,0x3066,0x3067,0x3068,0x3069,/* 0xC0-0xC7 */ + 0x306A,0x306B,0x306C,0x306D,0x306E,0x306F,0x3070,0x3071,/* 0xC8-0xCF */ + 0x3072,0x3073,0x3074,0x3075,0x3076,0x3077,0x3078,0x3079,/* 0xD0-0xD7 */ + 0x307A,0x307B,0x307C,0x307D,0x307E,0x307F,0x3080,0x3081,/* 0xD8-0xDF */ + 0x3082,0x3083,0x3084,0x3085,0x3086,0x3087,0x3088,0x3089,/* 0xE0-0xE7 */ + 0x308A,0x308B,0x308C,0x308D,0x308E,0x308F,0x3090,0x3091,/* 0xE8-0xEF */ + 0x3092,0x3093,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xF0-0xF7 */ +}; + +static const wchar_t c2u_83[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x30A1,0x30A2,0x30A3,0x30A4,0x30A5,0x30A6,0x30A7,0x30A8,/* 0x40-0x47 */ + 0x30A9,0x30AA,0x30AB,0x30AC,0x30AD,0x30AE,0x30AF,0x30B0,/* 0x48-0x4F */ + 0x30B1,0x30B2,0x30B3,0x30B4,0x30B5,0x30B6,0x30B7,0x30B8,/* 0x50-0x57 */ + 0x30B9,0x30BA,0x30BB,0x30BC,0x30BD,0x30BE,0x30BF,0x30C0,/* 0x58-0x5F */ + 0x30C1,0x30C2,0x30C3,0x30C4,0x30C5,0x30C6,0x30C7,0x30C8,/* 0x60-0x67 */ + 0x30C9,0x30CA,0x30CB,0x30CC,0x30CD,0x30CE,0x30CF,0x30D0,/* 0x68-0x6F */ + 0x30D1,0x30D2,0x30D3,0x30D4,0x30D5,0x30D6,0x30D7,0x30D8,/* 0x70-0x77 */ + 0x30D9,0x30DA,0x30DB,0x30DC,0x30DD,0x30DE,0x30DF,0x0000,/* 0x78-0x7F */ + + 0x30E0,0x30E1,0x30E2,0x30E3,0x30E4,0x30E5,0x30E6,0x30E7,/* 0x80-0x87 */ + 0x30E8,0x30E9,0x30EA,0x30EB,0x30EC,0x30ED,0x30EE,0x30EF,/* 0x88-0x8F */ + 0x30F0,0x30F1,0x30F2,0x30F3,0x30F4,0x30F5,0x30F6,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0391,/* 0x98-0x9F */ + 0x0392,0x0393,0x0394,0x0395,0x0396,0x0397,0x0398,0x0399,/* 0xA0-0xA7 */ + 0x039A,0x039B,0x039C,0x039D,0x039E,0x039F,0x03A0,0x03A1,/* 0xA8-0xAF */ + 0x03A3,0x03A4,0x03A5,0x03A6,0x03A7,0x03A8,0x03A9,0x0000,/* 0xB0-0xB7 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x03B1,/* 0xB8-0xBF */ + 0x03B2,0x03B3,0x03B4,0x03B5,0x03B6,0x03B7,0x03B8,0x03B9,/* 0xC0-0xC7 */ + 0x03BA,0x03BB,0x03BC,0x03BD,0x03BE,0x03BF,0x03C0,0x03C1,/* 0xC8-0xCF */ + 0x03C3,0x03C4,0x03C5,0x03C6,0x03C7,0x03C8,0x03C9,0x0000,/* 0xD0-0xD7 */ +}; + +static const wchar_t c2u_84[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0410,0x0411,0x0412,0x0413,0x0414,0x0415,0x0401,0x0416,/* 0x40-0x47 */ + 0x0417,0x0418,0x0419,0x041A,0x041B,0x041C,0x041D,0x041E,/* 0x48-0x4F */ + 0x041F,0x0420,0x0421,0x0422,0x0423,0x0424,0x0425,0x0426,/* 0x50-0x57 */ + 0x0427,0x0428,0x0429,0x042A,0x042B,0x042C,0x042D,0x042E,/* 0x58-0x5F */ + 0x042F,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0430,0x0431,0x0432,0x0433,0x0434,0x0435,0x0451,0x0436,/* 0x70-0x77 */ + 0x0437,0x0438,0x0439,0x043A,0x043B,0x043C,0x043D,0x0000,/* 0x78-0x7F */ + + 0x043E,0x043F,0x0440,0x0441,0x0442,0x0443,0x0444,0x0445,/* 0x80-0x87 */ + 0x0446,0x0447,0x0448,0x0449,0x044A,0x044B,0x044C,0x044D,/* 0x88-0x8F */ + 0x044E,0x044F,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x2500,/* 0x98-0x9F */ + 0x2502,0x250C,0x2510,0x2518,0x2514,0x251C,0x252C,0x2524,/* 0xA0-0xA7 */ + 0x2534,0x253C,0x2501,0x2503,0x250F,0x2513,0x251B,0x2517,/* 0xA8-0xAF */ + 0x2523,0x2533,0x252B,0x253B,0x254B,0x2520,0x252F,0x2528,/* 0xB0-0xB7 */ + 0x2537,0x253F,0x251D,0x2530,0x2525,0x2538,0x2542,0x0000,/* 0xB8-0xBF */ +}; + +static const wchar_t c2u_87[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x2460,0x2461,0x2462,0x2463,0x2464,0x2465,0x2466,0x2467,/* 0x40-0x47 */ + 0x2468,0x2469,0x246A,0x246B,0x246C,0x246D,0x246E,0x246F,/* 0x48-0x4F */ + 0x2470,0x2471,0x2472,0x2473,0x2160,0x2161,0x2162,0x2163,/* 0x50-0x57 */ + 0x2164,0x2165,0x2166,0x2167,0x2168,0x2169,0x0000,0x3349,/* 0x58-0x5F */ + 0x3314,0x3322,0x334D,0x3318,0x3327,0x3303,0x3336,0x3351,/* 0x60-0x67 */ + 0x3357,0x330D,0x3326,0x3323,0x332B,0x334A,0x333B,0x339C,/* 0x68-0x6F */ + 0x339D,0x339E,0x338E,0x338F,0x33C4,0x33A1,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x337B,0x0000,/* 0x78-0x7F */ + + 0x301D,0x301F,0x2116,0x33CD,0x2121,0x32A4,0x32A5,0x32A6,/* 0x80-0x87 */ + 0x32A7,0x32A8,0x3231,0x3232,0x3239,0x337E,0x337D,0x337C,/* 0x88-0x8F */ + 0x2252,0x2261,0x222B,0x222E,0x2211,0x221A,0x22A5,0x2220,/* 0x90-0x97 */ + 0x221F,0x22BF,0x2235,0x2229,0x222A,0x0000,0x0000,0x0000,/* 0x98-0x9F */ +}; + +static const wchar_t c2u_88[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x4E9C,/* 0x98-0x9F */ + 0x5516,0x5A03,0x963F,0x54C0,0x611B,0x6328,0x59F6,0x9022,/* 0xA0-0xA7 */ + 0x8475,0x831C,0x7A50,0x60AA,0x63E1,0x6E25,0x65ED,0x8466,/* 0xA8-0xAF */ + 0x82A6,0x9BF5,0x6893,0x5727,0x65A1,0x6271,0x5B9B,0x59D0,/* 0xB0-0xB7 */ + 0x867B,0x98F4,0x7D62,0x7DBE,0x9B8E,0x6216,0x7C9F,0x88B7,/* 0xB8-0xBF */ + 0x5B89,0x5EB5,0x6309,0x6697,0x6848,0x95C7,0x978D,0x674F,/* 0xC0-0xC7 */ + 0x4EE5,0x4F0A,0x4F4D,0x4F9D,0x5049,0x56F2,0x5937,0x59D4,/* 0xC8-0xCF */ + 0x5A01,0x5C09,0x60DF,0x610F,0x6170,0x6613,0x6905,0x70BA,/* 0xD0-0xD7 */ + 0x754F,0x7570,0x79FB,0x7DAD,0x7DEF,0x80C3,0x840E,0x8863,/* 0xD8-0xDF */ + 0x8B02,0x9055,0x907A,0x533B,0x4E95,0x4EA5,0x57DF,0x80B2,/* 0xE0-0xE7 */ + 0x90C1,0x78EF,0x4E00,0x58F1,0x6EA2,0x9038,0x7A32,0x8328,/* 0xE8-0xEF */ + 0x828B,0x9C2F,0x5141,0x5370,0x54BD,0x54E1,0x56E0,0x59FB,/* 0xF0-0xF7 */ + 0x5F15,0x98F2,0x6DEB,0x80E4,0x852D,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_89[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9662,0x9670,0x96A0,0x97FB,0x540B,0x53F3,0x5B87,0x70CF,/* 0x40-0x47 */ + 0x7FBD,0x8FC2,0x96E8,0x536F,0x9D5C,0x7ABA,0x4E11,0x7893,/* 0x48-0x4F */ + 0x81FC,0x6E26,0x5618,0x5504,0x6B1D,0x851A,0x9C3B,0x59E5,/* 0x50-0x57 */ + 0x53A9,0x6D66,0x74DC,0x958F,0x5642,0x4E91,0x904B,0x96F2,/* 0x58-0x5F */ + 0x834F,0x990C,0x53E1,0x55B6,0x5B30,0x5F71,0x6620,0x66F3,/* 0x60-0x67 */ + 0x6804,0x6C38,0x6CF3,0x6D29,0x745B,0x76C8,0x7A4E,0x9834,/* 0x68-0x6F */ + 0x82F1,0x885B,0x8A60,0x92ED,0x6DB2,0x75AB,0x76CA,0x99C5,/* 0x70-0x77 */ + 0x60A6,0x8B01,0x8D8A,0x95B2,0x698E,0x53AD,0x5186,0x0000,/* 0x78-0x7F */ + + 0x5712,0x5830,0x5944,0x5BB4,0x5EF6,0x6028,0x63A9,0x63F4,/* 0x80-0x87 */ + 0x6CBF,0x6F14,0x708E,0x7114,0x7159,0x71D5,0x733F,0x7E01,/* 0x88-0x8F */ + 0x8276,0x82D1,0x8597,0x9060,0x925B,0x9D1B,0x5869,0x65BC,/* 0x90-0x97 */ + 0x6C5A,0x7525,0x51F9,0x592E,0x5965,0x5F80,0x5FDC,0x62BC,/* 0x98-0x9F */ + 0x65FA,0x6A2A,0x6B27,0x6BB4,0x738B,0x7FC1,0x8956,0x9D2C,/* 0xA0-0xA7 */ + 0x9D0E,0x9EC4,0x5CA1,0x6C96,0x837B,0x5104,0x5C4B,0x61B6,/* 0xA8-0xAF */ + 0x81C6,0x6876,0x7261,0x4E59,0x4FFA,0x5378,0x6069,0x6E29,/* 0xB0-0xB7 */ + 0x7A4F,0x97F3,0x4E0B,0x5316,0x4EEE,0x4F55,0x4F3D,0x4FA1,/* 0xB8-0xBF */ + 0x4F73,0x52A0,0x53EF,0x5609,0x590F,0x5AC1,0x5BB6,0x5BE1,/* 0xC0-0xC7 */ + 0x79D1,0x6687,0x679C,0x67B6,0x6B4C,0x6CB3,0x706B,0x73C2,/* 0xC8-0xCF */ + 0x798D,0x79BE,0x7A3C,0x7B87,0x82B1,0x82DB,0x8304,0x8377,/* 0xD0-0xD7 */ + 0x83EF,0x83D3,0x8766,0x8AB2,0x5629,0x8CA8,0x8FE6,0x904E,/* 0xD8-0xDF */ + 0x971E,0x868A,0x4FC4,0x5CE8,0x6211,0x7259,0x753B,0x81E5,/* 0xE0-0xE7 */ + 0x82BD,0x86FE,0x8CC0,0x96C5,0x9913,0x99D5,0x4ECB,0x4F1A,/* 0xE8-0xEF */ + 0x89E3,0x56DE,0x584A,0x58CA,0x5EFB,0x5FEB,0x602A,0x6094,/* 0xF0-0xF7 */ + 0x6062,0x61D0,0x6212,0x62D0,0x6539,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_8A[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9B41,0x6666,0x68B0,0x6D77,0x7070,0x754C,0x7686,0x7D75,/* 0x40-0x47 */ + 0x82A5,0x87F9,0x958B,0x968E,0x8C9D,0x51F1,0x52BE,0x5916,/* 0x48-0x4F */ + 0x54B3,0x5BB3,0x5D16,0x6168,0x6982,0x6DAF,0x788D,0x84CB,/* 0x50-0x57 */ + 0x8857,0x8A72,0x93A7,0x9AB8,0x6D6C,0x99A8,0x86D9,0x57A3,/* 0x58-0x5F */ + 0x67FF,0x86CE,0x920E,0x5283,0x5687,0x5404,0x5ED3,0x62E1,/* 0x60-0x67 */ + 0x64B9,0x683C,0x6838,0x6BBB,0x7372,0x78BA,0x7A6B,0x899A,/* 0x68-0x6F */ + 0x89D2,0x8D6B,0x8F03,0x90ED,0x95A3,0x9694,0x9769,0x5B66,/* 0x70-0x77 */ + 0x5CB3,0x697D,0x984D,0x984E,0x639B,0x7B20,0x6A2B,0x0000,/* 0x78-0x7F */ + + 0x6A7F,0x68B6,0x9C0D,0x6F5F,0x5272,0x559D,0x6070,0x62EC,/* 0x80-0x87 */ + 0x6D3B,0x6E07,0x6ED1,0x845B,0x8910,0x8F44,0x4E14,0x9C39,/* 0x88-0x8F */ + 0x53F6,0x691B,0x6A3A,0x9784,0x682A,0x515C,0x7AC3,0x84B2,/* 0x90-0x97 */ + 0x91DC,0x938C,0x565B,0x9D28,0x6822,0x8305,0x8431,0x7CA5,/* 0x98-0x9F */ + 0x5208,0x82C5,0x74E6,0x4E7E,0x4F83,0x51A0,0x5BD2,0x520A,/* 0xA0-0xA7 */ + 0x52D8,0x52E7,0x5DFB,0x559A,0x582A,0x59E6,0x5B8C,0x5B98,/* 0xA8-0xAF */ + 0x5BDB,0x5E72,0x5E79,0x60A3,0x611F,0x6163,0x61BE,0x63DB,/* 0xB0-0xB7 */ + 0x6562,0x67D1,0x6853,0x68FA,0x6B3E,0x6B53,0x6C57,0x6F22,/* 0xB8-0xBF */ + 0x6F97,0x6F45,0x74B0,0x7518,0x76E3,0x770B,0x7AFF,0x7BA1,/* 0xC0-0xC7 */ + 0x7C21,0x7DE9,0x7F36,0x7FF0,0x809D,0x8266,0x839E,0x89B3,/* 0xC8-0xCF */ + 0x8ACC,0x8CAB,0x9084,0x9451,0x9593,0x9591,0x95A2,0x9665,/* 0xD0-0xD7 */ + 0x97D3,0x9928,0x8218,0x4E38,0x542B,0x5CB8,0x5DCC,0x73A9,/* 0xD8-0xDF */ + 0x764C,0x773C,0x5CA9,0x7FEB,0x8D0B,0x96C1,0x9811,0x9854,/* 0xE0-0xE7 */ + 0x9858,0x4F01,0x4F0E,0x5371,0x559C,0x5668,0x57FA,0x5947,/* 0xE8-0xEF */ + 0x5B09,0x5BC4,0x5C90,0x5E0C,0x5E7E,0x5FCC,0x63EE,0x673A,/* 0xF0-0xF7 */ + 0x65D7,0x65E2,0x671F,0x68CB,0x68C4,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_8B[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6A5F,0x5E30,0x6BC5,0x6C17,0x6C7D,0x757F,0x7948,0x5B63,/* 0x40-0x47 */ + 0x7A00,0x7D00,0x5FBD,0x898F,0x8A18,0x8CB4,0x8D77,0x8ECC,/* 0x48-0x4F */ + 0x8F1D,0x98E2,0x9A0E,0x9B3C,0x4E80,0x507D,0x5100,0x5993,/* 0x50-0x57 */ + 0x5B9C,0x622F,0x6280,0x64EC,0x6B3A,0x72A0,0x7591,0x7947,/* 0x58-0x5F */ + 0x7FA9,0x87FB,0x8ABC,0x8B70,0x63AC,0x83CA,0x97A0,0x5409,/* 0x60-0x67 */ + 0x5403,0x55AB,0x6854,0x6A58,0x8A70,0x7827,0x6775,0x9ECD,/* 0x68-0x6F */ + 0x5374,0x5BA2,0x811A,0x8650,0x9006,0x4E18,0x4E45,0x4EC7,/* 0x70-0x77 */ + 0x4F11,0x53CA,0x5438,0x5BAE,0x5F13,0x6025,0x6551,0x0000,/* 0x78-0x7F */ + + 0x673D,0x6C42,0x6C72,0x6CE3,0x7078,0x7403,0x7A76,0x7AAE,/* 0x80-0x87 */ + 0x7B08,0x7D1A,0x7CFE,0x7D66,0x65E7,0x725B,0x53BB,0x5C45,/* 0x88-0x8F */ + 0x5DE8,0x62D2,0x62E0,0x6319,0x6E20,0x865A,0x8A31,0x8DDD,/* 0x90-0x97 */ + 0x92F8,0x6F01,0x79A6,0x9B5A,0x4EA8,0x4EAB,0x4EAC,0x4F9B,/* 0x98-0x9F */ + 0x4FA0,0x50D1,0x5147,0x7AF6,0x5171,0x51F6,0x5354,0x5321,/* 0xA0-0xA7 */ + 0x537F,0x53EB,0x55AC,0x5883,0x5CE1,0x5F37,0x5F4A,0x602F,/* 0xA8-0xAF */ + 0x6050,0x606D,0x631F,0x6559,0x6A4B,0x6CC1,0x72C2,0x72ED,/* 0xB0-0xB7 */ + 0x77EF,0x80F8,0x8105,0x8208,0x854E,0x90F7,0x93E1,0x97FF,/* 0xB8-0xBF */ + 0x9957,0x9A5A,0x4EF0,0x51DD,0x5C2D,0x6681,0x696D,0x5C40,/* 0xC0-0xC7 */ + 0x66F2,0x6975,0x7389,0x6850,0x7C81,0x50C5,0x52E4,0x5747,/* 0xC8-0xCF */ + 0x5DFE,0x9326,0x65A4,0x6B23,0x6B3D,0x7434,0x7981,0x79BD,/* 0xD0-0xD7 */ + 0x7B4B,0x7DCA,0x82B9,0x83CC,0x887F,0x895F,0x8B39,0x8FD1,/* 0xD8-0xDF */ + 0x91D1,0x541F,0x9280,0x4E5D,0x5036,0x53E5,0x533A,0x72D7,/* 0xE0-0xE7 */ + 0x7396,0x77E9,0x82E6,0x8EAF,0x99C6,0x99C8,0x99D2,0x5177,/* 0xE8-0xEF */ + 0x611A,0x865E,0x55B0,0x7A7A,0x5076,0x5BD3,0x9047,0x9685,/* 0xF0-0xF7 */ + 0x4E32,0x6ADB,0x91E7,0x5C51,0x5C48,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_8C[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6398,0x7A9F,0x6C93,0x9774,0x8F61,0x7AAA,0x718A,0x9688,/* 0x40-0x47 */ + 0x7C82,0x6817,0x7E70,0x6851,0x936C,0x52F2,0x541B,0x85AB,/* 0x48-0x4F */ + 0x8A13,0x7FA4,0x8ECD,0x90E1,0x5366,0x8888,0x7941,0x4FC2,/* 0x50-0x57 */ + 0x50BE,0x5211,0x5144,0x5553,0x572D,0x73EA,0x578B,0x5951,/* 0x58-0x5F */ + 0x5F62,0x5F84,0x6075,0x6176,0x6167,0x61A9,0x63B2,0x643A,/* 0x60-0x67 */ + 0x656C,0x666F,0x6842,0x6E13,0x7566,0x7A3D,0x7CFB,0x7D4C,/* 0x68-0x6F */ + 0x7D99,0x7E4B,0x7F6B,0x830E,0x834A,0x86CD,0x8A08,0x8A63,/* 0x70-0x77 */ + 0x8B66,0x8EFD,0x981A,0x9D8F,0x82B8,0x8FCE,0x9BE8,0x0000,/* 0x78-0x7F */ + + 0x5287,0x621F,0x6483,0x6FC0,0x9699,0x6841,0x5091,0x6B20,/* 0x80-0x87 */ + 0x6C7A,0x6F54,0x7A74,0x7D50,0x8840,0x8A23,0x6708,0x4EF6,/* 0x88-0x8F */ + 0x5039,0x5026,0x5065,0x517C,0x5238,0x5263,0x55A7,0x570F,/* 0x90-0x97 */ + 0x5805,0x5ACC,0x5EFA,0x61B2,0x61F8,0x62F3,0x6372,0x691C,/* 0x98-0x9F */ + 0x6A29,0x727D,0x72AC,0x732E,0x7814,0x786F,0x7D79,0x770C,/* 0xA0-0xA7 */ + 0x80A9,0x898B,0x8B19,0x8CE2,0x8ED2,0x9063,0x9375,0x967A,/* 0xA8-0xAF */ + 0x9855,0x9A13,0x9E78,0x5143,0x539F,0x53B3,0x5E7B,0x5F26,/* 0xB0-0xB7 */ + 0x6E1B,0x6E90,0x7384,0x73FE,0x7D43,0x8237,0x8A00,0x8AFA,/* 0xB8-0xBF */ + 0x9650,0x4E4E,0x500B,0x53E4,0x547C,0x56FA,0x59D1,0x5B64,/* 0xC0-0xC7 */ + 0x5DF1,0x5EAB,0x5F27,0x6238,0x6545,0x67AF,0x6E56,0x72D0,/* 0xC8-0xCF */ + 0x7CCA,0x88B4,0x80A1,0x80E1,0x83F0,0x864E,0x8A87,0x8DE8,/* 0xD0-0xD7 */ + 0x9237,0x96C7,0x9867,0x9F13,0x4E94,0x4E92,0x4F0D,0x5348,/* 0xD8-0xDF */ + 0x5449,0x543E,0x5A2F,0x5F8C,0x5FA1,0x609F,0x68A7,0x6A8E,/* 0xE0-0xE7 */ + 0x745A,0x7881,0x8A9E,0x8AA4,0x8B77,0x9190,0x4E5E,0x9BC9,/* 0xE8-0xEF */ + 0x4EA4,0x4F7C,0x4FAF,0x5019,0x5016,0x5149,0x516C,0x529F,/* 0xF0-0xF7 */ + 0x52B9,0x52FE,0x539A,0x53E3,0x5411,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_8D[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x540E,0x5589,0x5751,0x57A2,0x597D,0x5B54,0x5B5D,0x5B8F,/* 0x40-0x47 */ + 0x5DE5,0x5DE7,0x5DF7,0x5E78,0x5E83,0x5E9A,0x5EB7,0x5F18,/* 0x48-0x4F */ + 0x6052,0x614C,0x6297,0x62D8,0x63A7,0x653B,0x6602,0x6643,/* 0x50-0x57 */ + 0x66F4,0x676D,0x6821,0x6897,0x69CB,0x6C5F,0x6D2A,0x6D69,/* 0x58-0x5F */ + 0x6E2F,0x6E9D,0x7532,0x7687,0x786C,0x7A3F,0x7CE0,0x7D05,/* 0x60-0x67 */ + 0x7D18,0x7D5E,0x7DB1,0x8015,0x8003,0x80AF,0x80B1,0x8154,/* 0x68-0x6F */ + 0x818F,0x822A,0x8352,0x884C,0x8861,0x8B1B,0x8CA2,0x8CFC,/* 0x70-0x77 */ + 0x90CA,0x9175,0x9271,0x783F,0x92FC,0x95A4,0x964D,0x0000,/* 0x78-0x7F */ + + 0x9805,0x9999,0x9AD8,0x9D3B,0x525B,0x52AB,0x53F7,0x5408,/* 0x80-0x87 */ + 0x58D5,0x62F7,0x6FE0,0x8C6A,0x8F5F,0x9EB9,0x514B,0x523B,/* 0x88-0x8F */ + 0x544A,0x56FD,0x7A40,0x9177,0x9D60,0x9ED2,0x7344,0x6F09,/* 0x90-0x97 */ + 0x8170,0x7511,0x5FFD,0x60DA,0x9AA8,0x72DB,0x8FBC,0x6B64,/* 0x98-0x9F */ + 0x9803,0x4ECA,0x56F0,0x5764,0x58BE,0x5A5A,0x6068,0x61C7,/* 0xA0-0xA7 */ + 0x660F,0x6606,0x6839,0x68B1,0x6DF7,0x75D5,0x7D3A,0x826E,/* 0xA8-0xAF */ + 0x9B42,0x4E9B,0x4F50,0x53C9,0x5506,0x5D6F,0x5DE6,0x5DEE,/* 0xB0-0xB7 */ + 0x67FB,0x6C99,0x7473,0x7802,0x8A50,0x9396,0x88DF,0x5750,/* 0xB8-0xBF */ + 0x5EA7,0x632B,0x50B5,0x50AC,0x518D,0x6700,0x54C9,0x585E,/* 0xC0-0xC7 */ + 0x59BB,0x5BB0,0x5F69,0x624D,0x63A1,0x683D,0x6B73,0x6E08,/* 0xC8-0xCF */ + 0x707D,0x91C7,0x7280,0x7815,0x7826,0x796D,0x658E,0x7D30,/* 0xD0-0xD7 */ + 0x83DC,0x88C1,0x8F09,0x969B,0x5264,0x5728,0x6750,0x7F6A,/* 0xD8-0xDF */ + 0x8CA1,0x51B4,0x5742,0x962A,0x583A,0x698A,0x80B4,0x54B2,/* 0xE0-0xE7 */ + 0x5D0E,0x57FC,0x7895,0x9DFA,0x4F5C,0x524A,0x548B,0x643E,/* 0xE8-0xEF */ + 0x6628,0x6714,0x67F5,0x7A84,0x7B56,0x7D22,0x932F,0x685C,/* 0xF0-0xF7 */ + 0x9BAD,0x7B39,0x5319,0x518A,0x5237,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_8E[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x5BDF,0x62F6,0x64AE,0x64E6,0x672D,0x6BBA,0x85A9,0x96D1,/* 0x40-0x47 */ + 0x7690,0x9BD6,0x634C,0x9306,0x9BAB,0x76BF,0x6652,0x4E09,/* 0x48-0x4F */ + 0x5098,0x53C2,0x5C71,0x60E8,0x6492,0x6563,0x685F,0x71E6,/* 0x50-0x57 */ + 0x73CA,0x7523,0x7B97,0x7E82,0x8695,0x8B83,0x8CDB,0x9178,/* 0x58-0x5F */ + 0x9910,0x65AC,0x66AB,0x6B8B,0x4ED5,0x4ED4,0x4F3A,0x4F7F,/* 0x60-0x67 */ + 0x523A,0x53F8,0x53F2,0x55E3,0x56DB,0x58EB,0x59CB,0x59C9,/* 0x68-0x6F */ + 0x59FF,0x5B50,0x5C4D,0x5E02,0x5E2B,0x5FD7,0x601D,0x6307,/* 0x70-0x77 */ + 0x652F,0x5B5C,0x65AF,0x65BD,0x65E8,0x679D,0x6B62,0x0000,/* 0x78-0x7F */ + + 0x6B7B,0x6C0F,0x7345,0x7949,0x79C1,0x7CF8,0x7D19,0x7D2B,/* 0x80-0x87 */ + 0x80A2,0x8102,0x81F3,0x8996,0x8A5E,0x8A69,0x8A66,0x8A8C,/* 0x88-0x8F */ + 0x8AEE,0x8CC7,0x8CDC,0x96CC,0x98FC,0x6B6F,0x4E8B,0x4F3C,/* 0x90-0x97 */ + 0x4F8D,0x5150,0x5B57,0x5BFA,0x6148,0x6301,0x6642,0x6B21,/* 0x98-0x9F */ + 0x6ECB,0x6CBB,0x723E,0x74BD,0x75D4,0x78C1,0x793A,0x800C,/* 0xA0-0xA7 */ + 0x8033,0x81EA,0x8494,0x8F9E,0x6C50,0x9E7F,0x5F0F,0x8B58,/* 0xA8-0xAF */ + 0x9D2B,0x7AFA,0x8EF8,0x5B8D,0x96EB,0x4E03,0x53F1,0x57F7,/* 0xB0-0xB7 */ + 0x5931,0x5AC9,0x5BA4,0x6089,0x6E7F,0x6F06,0x75BE,0x8CEA,/* 0xB8-0xBF */ + 0x5B9F,0x8500,0x7BE0,0x5072,0x67F4,0x829D,0x5C61,0x854A,/* 0xC0-0xC7 */ + 0x7E1E,0x820E,0x5199,0x5C04,0x6368,0x8D66,0x659C,0x716E,/* 0xC8-0xCF */ + 0x793E,0x7D17,0x8005,0x8B1D,0x8ECA,0x906E,0x86C7,0x90AA,/* 0xD0-0xD7 */ + 0x501F,0x52FA,0x5C3A,0x6753,0x707C,0x7235,0x914C,0x91C8,/* 0xD8-0xDF */ + 0x932B,0x82E5,0x5BC2,0x5F31,0x60F9,0x4E3B,0x53D6,0x5B88,/* 0xE0-0xE7 */ + 0x624B,0x6731,0x6B8A,0x72E9,0x73E0,0x7A2E,0x816B,0x8DA3,/* 0xE8-0xEF */ + 0x9152,0x9996,0x5112,0x53D7,0x546A,0x5BFF,0x6388,0x6A39,/* 0xF0-0xF7 */ + 0x7DAC,0x9700,0x56DA,0x53CE,0x5468,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_8F[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x5B97,0x5C31,0x5DDE,0x4FEE,0x6101,0x62FE,0x6D32,0x79C0,/* 0x40-0x47 */ + 0x79CB,0x7D42,0x7E4D,0x7FD2,0x81ED,0x821F,0x8490,0x8846,/* 0x48-0x4F */ + 0x8972,0x8B90,0x8E74,0x8F2F,0x9031,0x914B,0x916C,0x96C6,/* 0x50-0x57 */ + 0x919C,0x4EC0,0x4F4F,0x5145,0x5341,0x5F93,0x620E,0x67D4,/* 0x58-0x5F */ + 0x6C41,0x6E0B,0x7363,0x7E26,0x91CD,0x9283,0x53D4,0x5919,/* 0x60-0x67 */ + 0x5BBF,0x6DD1,0x795D,0x7E2E,0x7C9B,0x587E,0x719F,0x51FA,/* 0x68-0x6F */ + 0x8853,0x8FF0,0x4FCA,0x5CFB,0x6625,0x77AC,0x7AE3,0x821C,/* 0x70-0x77 */ + 0x99FF,0x51C6,0x5FAA,0x65EC,0x696F,0x6B89,0x6DF3,0x0000,/* 0x78-0x7F */ + + 0x6E96,0x6F64,0x76FE,0x7D14,0x5DE1,0x9075,0x9187,0x9806,/* 0x80-0x87 */ + 0x51E6,0x521D,0x6240,0x6691,0x66D9,0x6E1A,0x5EB6,0x7DD2,/* 0x88-0x8F */ + 0x7F72,0x66F8,0x85AF,0x85F7,0x8AF8,0x52A9,0x53D9,0x5973,/* 0x90-0x97 */ + 0x5E8F,0x5F90,0x6055,0x92E4,0x9664,0x50B7,0x511F,0x52DD,/* 0x98-0x9F */ + 0x5320,0x5347,0x53EC,0x54E8,0x5546,0x5531,0x5617,0x5968,/* 0xA0-0xA7 */ + 0x59BE,0x5A3C,0x5BB5,0x5C06,0x5C0F,0x5C11,0x5C1A,0x5E84,/* 0xA8-0xAF */ + 0x5E8A,0x5EE0,0x5F70,0x627F,0x6284,0x62DB,0x638C,0x6377,/* 0xB0-0xB7 */ + 0x6607,0x660C,0x662D,0x6676,0x677E,0x68A2,0x6A1F,0x6A35,/* 0xB8-0xBF */ + 0x6CBC,0x6D88,0x6E09,0x6E58,0x713C,0x7126,0x7167,0x75C7,/* 0xC0-0xC7 */ + 0x7701,0x785D,0x7901,0x7965,0x79F0,0x7AE0,0x7B11,0x7CA7,/* 0xC8-0xCF */ + 0x7D39,0x8096,0x83D6,0x848B,0x8549,0x885D,0x88F3,0x8A1F,/* 0xD0-0xD7 */ + 0x8A3C,0x8A54,0x8A73,0x8C61,0x8CDE,0x91A4,0x9266,0x937E,/* 0xD8-0xDF */ + 0x9418,0x969C,0x9798,0x4E0A,0x4E08,0x4E1E,0x4E57,0x5197,/* 0xE0-0xE7 */ + 0x5270,0x57CE,0x5834,0x58CC,0x5B22,0x5E38,0x60C5,0x64FE,/* 0xE8-0xEF */ + 0x6761,0x6756,0x6D44,0x72B6,0x7573,0x7A63,0x84B8,0x8B72,/* 0xF0-0xF7 */ + 0x91B8,0x9320,0x5631,0x57F4,0x98FE,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_90[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x62ED,0x690D,0x6B96,0x71ED,0x7E54,0x8077,0x8272,0x89E6,/* 0x40-0x47 */ + 0x98DF,0x8755,0x8FB1,0x5C3B,0x4F38,0x4FE1,0x4FB5,0x5507,/* 0x48-0x4F */ + 0x5A20,0x5BDD,0x5BE9,0x5FC3,0x614E,0x632F,0x65B0,0x664B,/* 0x50-0x57 */ + 0x68EE,0x699B,0x6D78,0x6DF1,0x7533,0x75B9,0x771F,0x795E,/* 0x58-0x5F */ + 0x79E6,0x7D33,0x81E3,0x82AF,0x85AA,0x89AA,0x8A3A,0x8EAB,/* 0x60-0x67 */ + 0x8F9B,0x9032,0x91DD,0x9707,0x4EBA,0x4EC1,0x5203,0x5875,/* 0x68-0x6F */ + 0x58EC,0x5C0B,0x751A,0x5C3D,0x814E,0x8A0A,0x8FC5,0x9663,/* 0x70-0x77 */ + 0x976D,0x7B25,0x8ACF,0x9808,0x9162,0x56F3,0x53A8,0x0000,/* 0x78-0x7F */ + + 0x9017,0x5439,0x5782,0x5E25,0x63A8,0x6C34,0x708A,0x7761,/* 0x80-0x87 */ + 0x7C8B,0x7FE0,0x8870,0x9042,0x9154,0x9310,0x9318,0x968F,/* 0x88-0x8F */ + 0x745E,0x9AC4,0x5D07,0x5D69,0x6570,0x67A2,0x8DA8,0x96DB,/* 0x90-0x97 */ + 0x636E,0x6749,0x6919,0x83C5,0x9817,0x96C0,0x88FE,0x6F84,/* 0x98-0x9F */ + 0x647A,0x5BF8,0x4E16,0x702C,0x755D,0x662F,0x51C4,0x5236,/* 0xA0-0xA7 */ + 0x52E2,0x59D3,0x5F81,0x6027,0x6210,0x653F,0x6574,0x661F,/* 0xA8-0xAF */ + 0x6674,0x68F2,0x6816,0x6B63,0x6E05,0x7272,0x751F,0x76DB,/* 0xB0-0xB7 */ + 0x7CBE,0x8056,0x58F0,0x88FD,0x897F,0x8AA0,0x8A93,0x8ACB,/* 0xB8-0xBF */ + 0x901D,0x9192,0x9752,0x9759,0x6589,0x7A0E,0x8106,0x96BB,/* 0xC0-0xC7 */ + 0x5E2D,0x60DC,0x621A,0x65A5,0x6614,0x6790,0x77F3,0x7A4D,/* 0xC8-0xCF */ + 0x7C4D,0x7E3E,0x810A,0x8CAC,0x8D64,0x8DE1,0x8E5F,0x78A9,/* 0xD0-0xD7 */ + 0x5207,0x62D9,0x63A5,0x6442,0x6298,0x8A2D,0x7A83,0x7BC0,/* 0xD8-0xDF */ + 0x8AAC,0x96EA,0x7D76,0x820C,0x8749,0x4ED9,0x5148,0x5343,/* 0xE0-0xE7 */ + 0x5360,0x5BA3,0x5C02,0x5C16,0x5DDD,0x6226,0x6247,0x64B0,/* 0xE8-0xEF */ + 0x6813,0x6834,0x6CC9,0x6D45,0x6D17,0x67D3,0x6F5C,0x714E,/* 0xF0-0xF7 */ + 0x717D,0x65CB,0x7A7F,0x7BAD,0x7DDA,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_91[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7E4A,0x7FA8,0x817A,0x821B,0x8239,0x85A6,0x8A6E,0x8CCE,/* 0x40-0x47 */ + 0x8DF5,0x9078,0x9077,0x92AD,0x9291,0x9583,0x9BAE,0x524D,/* 0x48-0x4F */ + 0x5584,0x6F38,0x7136,0x5168,0x7985,0x7E55,0x81B3,0x7CCE,/* 0x50-0x57 */ + 0x564C,0x5851,0x5CA8,0x63AA,0x66FE,0x66FD,0x695A,0x72D9,/* 0x58-0x5F */ + 0x758F,0x758E,0x790E,0x7956,0x79DF,0x7C97,0x7D20,0x7D44,/* 0x60-0x67 */ + 0x8607,0x8A34,0x963B,0x9061,0x9F20,0x50E7,0x5275,0x53CC,/* 0x68-0x6F */ + 0x53E2,0x5009,0x55AA,0x58EE,0x594F,0x723D,0x5B8B,0x5C64,/* 0x70-0x77 */ + 0x531D,0x60E3,0x60F3,0x635C,0x6383,0x633F,0x63BB,0x0000,/* 0x78-0x7F */ + + 0x64CD,0x65E9,0x66F9,0x5DE3,0x69CD,0x69FD,0x6F15,0x71E5,/* 0x80-0x87 */ + 0x4E89,0x75E9,0x76F8,0x7A93,0x7CDF,0x7DCF,0x7D9C,0x8061,/* 0x88-0x8F */ + 0x8349,0x8358,0x846C,0x84BC,0x85FB,0x88C5,0x8D70,0x9001,/* 0x90-0x97 */ + 0x906D,0x9397,0x971C,0x9A12,0x50CF,0x5897,0x618E,0x81D3,/* 0x98-0x9F */ + 0x8535,0x8D08,0x9020,0x4FC3,0x5074,0x5247,0x5373,0x606F,/* 0xA0-0xA7 */ + 0x6349,0x675F,0x6E2C,0x8DB3,0x901F,0x4FD7,0x5C5E,0x8CCA,/* 0xA8-0xAF */ + 0x65CF,0x7D9A,0x5352,0x8896,0x5176,0x63C3,0x5B58,0x5B6B,/* 0xB0-0xB7 */ + 0x5C0A,0x640D,0x6751,0x905C,0x4ED6,0x591A,0x592A,0x6C70,/* 0xB8-0xBF */ + 0x8A51,0x553E,0x5815,0x59A5,0x60F0,0x6253,0x67C1,0x8235,/* 0xC0-0xC7 */ + 0x6955,0x9640,0x99C4,0x9A28,0x4F53,0x5806,0x5BFE,0x8010,/* 0xC8-0xCF */ + 0x5CB1,0x5E2F,0x5F85,0x6020,0x614B,0x6234,0x66FF,0x6CF0,/* 0xD0-0xD7 */ + 0x6EDE,0x80CE,0x817F,0x82D4,0x888B,0x8CB8,0x9000,0x902E,/* 0xD8-0xDF */ + 0x968A,0x9EDB,0x9BDB,0x4EE3,0x53F0,0x5927,0x7B2C,0x918D,/* 0xE0-0xE7 */ + 0x984C,0x9DF9,0x6EDD,0x7027,0x5353,0x5544,0x5B85,0x6258,/* 0xE8-0xEF */ + 0x629E,0x62D3,0x6CA2,0x6FEF,0x7422,0x8A17,0x9438,0x6FC1,/* 0xF0-0xF7 */ + 0x8AFE,0x8338,0x51E7,0x86F8,0x53EA,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_92[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x53E9,0x4F46,0x9054,0x8FB0,0x596A,0x8131,0x5DFD,0x7AEA,/* 0x40-0x47 */ + 0x8FBF,0x68DA,0x8C37,0x72F8,0x9C48,0x6A3D,0x8AB0,0x4E39,/* 0x48-0x4F */ + 0x5358,0x5606,0x5766,0x62C5,0x63A2,0x65E6,0x6B4E,0x6DE1,/* 0x50-0x57 */ + 0x6E5B,0x70AD,0x77ED,0x7AEF,0x7BAA,0x7DBB,0x803D,0x80C6,/* 0x58-0x5F */ + 0x86CB,0x8A95,0x935B,0x56E3,0x58C7,0x5F3E,0x65AD,0x6696,/* 0x60-0x67 */ + 0x6A80,0x6BB5,0x7537,0x8AC7,0x5024,0x77E5,0x5730,0x5F1B,/* 0x68-0x6F */ + 0x6065,0x667A,0x6C60,0x75F4,0x7A1A,0x7F6E,0x81F4,0x8718,/* 0x70-0x77 */ + 0x9045,0x99B3,0x7BC9,0x755C,0x7AF9,0x7B51,0x84C4,0x0000,/* 0x78-0x7F */ + + 0x9010,0x79E9,0x7A92,0x8336,0x5AE1,0x7740,0x4E2D,0x4EF2,/* 0x80-0x87 */ + 0x5B99,0x5FE0,0x62BD,0x663C,0x67F1,0x6CE8,0x866B,0x8877,/* 0x88-0x8F */ + 0x8A3B,0x914E,0x92F3,0x99D0,0x6A17,0x7026,0x732A,0x82E7,/* 0x90-0x97 */ + 0x8457,0x8CAF,0x4E01,0x5146,0x51CB,0x558B,0x5BF5,0x5E16,/* 0x98-0x9F */ + 0x5E33,0x5E81,0x5F14,0x5F35,0x5F6B,0x5FB4,0x61F2,0x6311,/* 0xA0-0xA7 */ + 0x66A2,0x671D,0x6F6E,0x7252,0x753A,0x773A,0x8074,0x8139,/* 0xA8-0xAF */ + 0x8178,0x8776,0x8ABF,0x8ADC,0x8D85,0x8DF3,0x929A,0x9577,/* 0xB0-0xB7 */ + 0x9802,0x9CE5,0x52C5,0x6357,0x76F4,0x6715,0x6C88,0x73CD,/* 0xB8-0xBF */ + 0x8CC3,0x93AE,0x9673,0x6D25,0x589C,0x690E,0x69CC,0x8FFD,/* 0xC0-0xC7 */ + 0x939A,0x75DB,0x901A,0x585A,0x6802,0x63B4,0x69FB,0x4F43,/* 0xC8-0xCF */ + 0x6F2C,0x67D8,0x8FBB,0x8526,0x7DB4,0x9354,0x693F,0x6F70,/* 0xD0-0xD7 */ + 0x576A,0x58F7,0x5B2C,0x7D2C,0x722A,0x540A,0x91E3,0x9DB4,/* 0xD8-0xDF */ + 0x4EAD,0x4F4E,0x505C,0x5075,0x5243,0x8C9E,0x5448,0x5824,/* 0xE0-0xE7 */ + 0x5B9A,0x5E1D,0x5E95,0x5EAD,0x5EF7,0x5F1F,0x608C,0x62B5,/* 0xE8-0xEF */ + 0x633A,0x63D0,0x68AF,0x6C40,0x7887,0x798E,0x7A0B,0x7DE0,/* 0xF0-0xF7 */ + 0x8247,0x8A02,0x8AE6,0x8E44,0x9013,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_93[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x90B8,0x912D,0x91D8,0x9F0E,0x6CE5,0x6458,0x64E2,0x6575,/* 0x40-0x47 */ + 0x6EF4,0x7684,0x7B1B,0x9069,0x93D1,0x6EBA,0x54F2,0x5FB9,/* 0x48-0x4F */ + 0x64A4,0x8F4D,0x8FED,0x9244,0x5178,0x586B,0x5929,0x5C55,/* 0x50-0x57 */ + 0x5E97,0x6DFB,0x7E8F,0x751C,0x8CBC,0x8EE2,0x985B,0x70B9,/* 0x58-0x5F */ + 0x4F1D,0x6BBF,0x6FB1,0x7530,0x96FB,0x514E,0x5410,0x5835,/* 0x60-0x67 */ + 0x5857,0x59AC,0x5C60,0x5F92,0x6597,0x675C,0x6E21,0x767B,/* 0x68-0x6F */ + 0x83DF,0x8CED,0x9014,0x90FD,0x934D,0x7825,0x783A,0x52AA,/* 0x70-0x77 */ + 0x5EA6,0x571F,0x5974,0x6012,0x5012,0x515A,0x51AC,0x0000,/* 0x78-0x7F */ + + 0x51CD,0x5200,0x5510,0x5854,0x5858,0x5957,0x5B95,0x5CF6,/* 0x80-0x87 */ + 0x5D8B,0x60BC,0x6295,0x642D,0x6771,0x6843,0x68BC,0x68DF,/* 0x88-0x8F */ + 0x76D7,0x6DD8,0x6E6F,0x6D9B,0x706F,0x71C8,0x5F53,0x75D8,/* 0x90-0x97 */ + 0x7977,0x7B49,0x7B54,0x7B52,0x7CD6,0x7D71,0x5230,0x8463,/* 0x98-0x9F */ + 0x8569,0x85E4,0x8A0E,0x8B04,0x8C46,0x8E0F,0x9003,0x900F,/* 0xA0-0xA7 */ + 0x9419,0x9676,0x982D,0x9A30,0x95D8,0x50CD,0x52D5,0x540C,/* 0xA8-0xAF */ + 0x5802,0x5C0E,0x61A7,0x649E,0x6D1E,0x77B3,0x7AE5,0x80F4,/* 0xB0-0xB7 */ + 0x8404,0x9053,0x9285,0x5CE0,0x9D07,0x533F,0x5F97,0x5FB3,/* 0xB8-0xBF */ + 0x6D9C,0x7279,0x7763,0x79BF,0x7BE4,0x6BD2,0x72EC,0x8AAD,/* 0xC0-0xC7 */ + 0x6803,0x6A61,0x51F8,0x7A81,0x6934,0x5C4A,0x9CF6,0x82EB,/* 0xC8-0xCF */ + 0x5BC5,0x9149,0x701E,0x5678,0x5C6F,0x60C7,0x6566,0x6C8C,/* 0xD0-0xD7 */ + 0x8C5A,0x9041,0x9813,0x5451,0x66C7,0x920D,0x5948,0x90A3,/* 0xD8-0xDF */ + 0x5185,0x4E4D,0x51EA,0x8599,0x8B0E,0x7058,0x637A,0x934B,/* 0xE0-0xE7 */ + 0x6962,0x99B4,0x7E04,0x7577,0x5357,0x6960,0x8EDF,0x96E3,/* 0xE8-0xEF */ + 0x6C5D,0x4E8C,0x5C3C,0x5F10,0x8FE9,0x5302,0x8CD1,0x8089,/* 0xF0-0xF7 */ + 0x8679,0x5EFF,0x65E5,0x4E73,0x5165,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_94[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x5982,0x5C3F,0x97EE,0x4EFB,0x598A,0x5FCD,0x8A8D,0x6FE1,/* 0x40-0x47 */ + 0x79B0,0x7962,0x5BE7,0x8471,0x732B,0x71B1,0x5E74,0x5FF5,/* 0x48-0x4F */ + 0x637B,0x649A,0x71C3,0x7C98,0x4E43,0x5EFC,0x4E4B,0x57DC,/* 0x50-0x57 */ + 0x56A2,0x60A9,0x6FC3,0x7D0D,0x80FD,0x8133,0x81BF,0x8FB2,/* 0x58-0x5F */ + 0x8997,0x86A4,0x5DF4,0x628A,0x64AD,0x8987,0x6777,0x6CE2,/* 0x60-0x67 */ + 0x6D3E,0x7436,0x7834,0x5A46,0x7F75,0x82AD,0x99AC,0x4FF3,/* 0x68-0x6F */ + 0x5EC3,0x62DD,0x6392,0x6557,0x676F,0x76C3,0x724C,0x80CC,/* 0x70-0x77 */ + 0x80BA,0x8F29,0x914D,0x500D,0x57F9,0x5A92,0x6885,0x0000,/* 0x78-0x7F */ + + 0x6973,0x7164,0x72FD,0x8CB7,0x58F2,0x8CE0,0x966A,0x9019,/* 0x80-0x87 */ + 0x877F,0x79E4,0x77E7,0x8429,0x4F2F,0x5265,0x535A,0x62CD,/* 0x88-0x8F */ + 0x67CF,0x6CCA,0x767D,0x7B94,0x7C95,0x8236,0x8584,0x8FEB,/* 0x90-0x97 */ + 0x66DD,0x6F20,0x7206,0x7E1B,0x83AB,0x99C1,0x9EA6,0x51FD,/* 0x98-0x9F */ + 0x7BB1,0x7872,0x7BB8,0x8087,0x7B48,0x6AE8,0x5E61,0x808C,/* 0xA0-0xA7 */ + 0x7551,0x7560,0x516B,0x9262,0x6E8C,0x767A,0x9197,0x9AEA,/* 0xA8-0xAF */ + 0x4F10,0x7F70,0x629C,0x7B4F,0x95A5,0x9CE9,0x567A,0x5859,/* 0xB0-0xB7 */ + 0x86E4,0x96BC,0x4F34,0x5224,0x534A,0x53CD,0x53DB,0x5E06,/* 0xB8-0xBF */ + 0x642C,0x6591,0x677F,0x6C3E,0x6C4E,0x7248,0x72AF,0x73ED,/* 0xC0-0xC7 */ + 0x7554,0x7E41,0x822C,0x85E9,0x8CA9,0x7BC4,0x91C6,0x7169,/* 0xC8-0xCF */ + 0x9812,0x98EF,0x633D,0x6669,0x756A,0x76E4,0x78D0,0x8543,/* 0xD0-0xD7 */ + 0x86EE,0x532A,0x5351,0x5426,0x5983,0x5E87,0x5F7C,0x60B2,/* 0xD8-0xDF */ + 0x6249,0x6279,0x62AB,0x6590,0x6BD4,0x6CCC,0x75B2,0x76AE,/* 0xE0-0xE7 */ + 0x7891,0x79D8,0x7DCB,0x7F77,0x80A5,0x88AB,0x8AB9,0x8CBB,/* 0xE8-0xEF */ + 0x907F,0x975E,0x98DB,0x6A0B,0x7C38,0x5099,0x5C3E,0x5FAE,/* 0xF0-0xF7 */ + 0x6787,0x6BD8,0x7435,0x7709,0x7F8E,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_95[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9F3B,0x67CA,0x7A17,0x5339,0x758B,0x9AED,0x5F66,0x819D,/* 0x40-0x47 */ + 0x83F1,0x8098,0x5F3C,0x5FC5,0x7562,0x7B46,0x903C,0x6867,/* 0x48-0x4F */ + 0x59EB,0x5A9B,0x7D10,0x767E,0x8B2C,0x4FF5,0x5F6A,0x6A19,/* 0x50-0x57 */ + 0x6C37,0x6F02,0x74E2,0x7968,0x8868,0x8A55,0x8C79,0x5EDF,/* 0x58-0x5F */ + 0x63CF,0x75C5,0x79D2,0x82D7,0x9328,0x92F2,0x849C,0x86ED,/* 0x60-0x67 */ + 0x9C2D,0x54C1,0x5F6C,0x658C,0x6D5C,0x7015,0x8CA7,0x8CD3,/* 0x68-0x6F */ + 0x983B,0x654F,0x74F6,0x4E0D,0x4ED8,0x57E0,0x592B,0x5A66,/* 0x70-0x77 */ + 0x5BCC,0x51A8,0x5E03,0x5E9C,0x6016,0x6276,0x6577,0x0000,/* 0x78-0x7F */ + + 0x65A7,0x666E,0x6D6E,0x7236,0x7B26,0x8150,0x819A,0x8299,/* 0x80-0x87 */ + 0x8B5C,0x8CA0,0x8CE6,0x8D74,0x961C,0x9644,0x4FAE,0x64AB,/* 0x88-0x8F */ + 0x6B66,0x821E,0x8461,0x856A,0x90E8,0x5C01,0x6953,0x98A8,/* 0x90-0x97 */ + 0x847A,0x8557,0x4F0F,0x526F,0x5FA9,0x5E45,0x670D,0x798F,/* 0x98-0x9F */ + 0x8179,0x8907,0x8986,0x6DF5,0x5F17,0x6255,0x6CB8,0x4ECF,/* 0xA0-0xA7 */ + 0x7269,0x9B92,0x5206,0x543B,0x5674,0x58B3,0x61A4,0x626E,/* 0xA8-0xAF */ + 0x711A,0x596E,0x7C89,0x7CDE,0x7D1B,0x96F0,0x6587,0x805E,/* 0xB0-0xB7 */ + 0x4E19,0x4F75,0x5175,0x5840,0x5E63,0x5E73,0x5F0A,0x67C4,/* 0xB8-0xBF */ + 0x4E26,0x853D,0x9589,0x965B,0x7C73,0x9801,0x50FB,0x58C1,/* 0xC0-0xC7 */ + 0x7656,0x78A7,0x5225,0x77A5,0x8511,0x7B86,0x504F,0x5909,/* 0xC8-0xCF */ + 0x7247,0x7BC7,0x7DE8,0x8FBA,0x8FD4,0x904D,0x4FBF,0x52C9,/* 0xD0-0xD7 */ + 0x5A29,0x5F01,0x97AD,0x4FDD,0x8217,0x92EA,0x5703,0x6355,/* 0xD8-0xDF */ + 0x6B69,0x752B,0x88DC,0x8F14,0x7A42,0x52DF,0x5893,0x6155,/* 0xE0-0xE7 */ + 0x620A,0x66AE,0x6BCD,0x7C3F,0x83E9,0x5023,0x4FF8,0x5305,/* 0xE8-0xEF */ + 0x5446,0x5831,0x5949,0x5B9D,0x5CF0,0x5CEF,0x5D29,0x5E96,/* 0xF0-0xF7 */ + 0x62B1,0x6367,0x653E,0x65B9,0x670B,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_96[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6CD5,0x6CE1,0x70F9,0x7832,0x7E2B,0x80DE,0x82B3,0x840C,/* 0x40-0x47 */ + 0x84EC,0x8702,0x8912,0x8A2A,0x8C4A,0x90A6,0x92D2,0x98FD,/* 0x48-0x4F */ + 0x9CF3,0x9D6C,0x4E4F,0x4EA1,0x508D,0x5256,0x574A,0x59A8,/* 0x50-0x57 */ + 0x5E3D,0x5FD8,0x5FD9,0x623F,0x66B4,0x671B,0x67D0,0x68D2,/* 0x58-0x5F */ + 0x5192,0x7D21,0x80AA,0x81A8,0x8B00,0x8C8C,0x8CBF,0x927E,/* 0x60-0x67 */ + 0x9632,0x5420,0x982C,0x5317,0x50D5,0x535C,0x58A8,0x64B2,/* 0x68-0x6F */ + 0x6734,0x7267,0x7766,0x7A46,0x91E6,0x52C3,0x6CA1,0x6B86,/* 0x70-0x77 */ + 0x5800,0x5E4C,0x5954,0x672C,0x7FFB,0x51E1,0x76C6,0x0000,/* 0x78-0x7F */ + + 0x6469,0x78E8,0x9B54,0x9EBB,0x57CB,0x59B9,0x6627,0x679A,/* 0x80-0x87 */ + 0x6BCE,0x54E9,0x69D9,0x5E55,0x819C,0x6795,0x9BAA,0x67FE,/* 0x88-0x8F */ + 0x9C52,0x685D,0x4EA6,0x4FE3,0x53C8,0x62B9,0x672B,0x6CAB,/* 0x90-0x97 */ + 0x8FC4,0x4FAD,0x7E6D,0x9EBF,0x4E07,0x6162,0x6E80,0x6F2B,/* 0x98-0x9F */ + 0x8513,0x5473,0x672A,0x9B45,0x5DF3,0x7B95,0x5CAC,0x5BC6,/* 0xA0-0xA7 */ + 0x871C,0x6E4A,0x84D1,0x7A14,0x8108,0x5999,0x7C8D,0x6C11,/* 0xA8-0xAF */ + 0x7720,0x52D9,0x5922,0x7121,0x725F,0x77DB,0x9727,0x9D61,/* 0xB0-0xB7 */ + 0x690B,0x5A7F,0x5A18,0x51A5,0x540D,0x547D,0x660E,0x76DF,/* 0xB8-0xBF */ + 0x8FF7,0x9298,0x9CF4,0x59EA,0x725D,0x6EC5,0x514D,0x68C9,/* 0xC0-0xC7 */ + 0x7DBF,0x7DEC,0x9762,0x9EBA,0x6478,0x6A21,0x8302,0x5984,/* 0xC8-0xCF */ + 0x5B5F,0x6BDB,0x731B,0x76F2,0x7DB2,0x8017,0x8499,0x5132,/* 0xD0-0xD7 */ + 0x6728,0x9ED9,0x76EE,0x6762,0x52FF,0x9905,0x5C24,0x623B,/* 0xD8-0xDF */ + 0x7C7E,0x8CB0,0x554F,0x60B6,0x7D0B,0x9580,0x5301,0x4E5F,/* 0xE0-0xE7 */ + 0x51B6,0x591C,0x723A,0x8036,0x91CE,0x5F25,0x77E2,0x5384,/* 0xE8-0xEF */ + 0x5F79,0x7D04,0x85AC,0x8A33,0x8E8D,0x9756,0x67F3,0x85AE,/* 0xF0-0xF7 */ + 0x9453,0x6109,0x6108,0x6CB9,0x7652,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_97[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8AED,0x8F38,0x552F,0x4F51,0x512A,0x52C7,0x53CB,0x5BA5,/* 0x40-0x47 */ + 0x5E7D,0x60A0,0x6182,0x63D6,0x6709,0x67DA,0x6E67,0x6D8C,/* 0x48-0x4F */ + 0x7336,0x7337,0x7531,0x7950,0x88D5,0x8A98,0x904A,0x9091,/* 0x50-0x57 */ + 0x90F5,0x96C4,0x878D,0x5915,0x4E88,0x4F59,0x4E0E,0x8A89,/* 0x58-0x5F */ + 0x8F3F,0x9810,0x50AD,0x5E7C,0x5996,0x5BB9,0x5EB8,0x63DA,/* 0x60-0x67 */ + 0x63FA,0x64C1,0x66DC,0x694A,0x69D8,0x6D0B,0x6EB6,0x7194,/* 0x68-0x6F */ + 0x7528,0x7AAF,0x7F8A,0x8000,0x8449,0x84C9,0x8981,0x8B21,/* 0x70-0x77 */ + 0x8E0A,0x9065,0x967D,0x990A,0x617E,0x6291,0x6B32,0x0000,/* 0x78-0x7F */ + + 0x6C83,0x6D74,0x7FCC,0x7FFC,0x6DC0,0x7F85,0x87BA,0x88F8,/* 0x80-0x87 */ + 0x6765,0x83B1,0x983C,0x96F7,0x6D1B,0x7D61,0x843D,0x916A,/* 0x88-0x8F */ + 0x4E71,0x5375,0x5D50,0x6B04,0x6FEB,0x85CD,0x862D,0x89A7,/* 0x90-0x97 */ + 0x5229,0x540F,0x5C65,0x674E,0x68A8,0x7406,0x7483,0x75E2,/* 0x98-0x9F */ + 0x88CF,0x88E1,0x91CC,0x96E2,0x9678,0x5F8B,0x7387,0x7ACB,/* 0xA0-0xA7 */ + 0x844E,0x63A0,0x7565,0x5289,0x6D41,0x6E9C,0x7409,0x7559,/* 0xA8-0xAF */ + 0x786B,0x7C92,0x9686,0x7ADC,0x9F8D,0x4FB6,0x616E,0x65C5,/* 0xB0-0xB7 */ + 0x865C,0x4E86,0x4EAE,0x50DA,0x4E21,0x51CC,0x5BEE,0x6599,/* 0xB8-0xBF */ + 0x6881,0x6DBC,0x731F,0x7642,0x77AD,0x7A1C,0x7CE7,0x826F,/* 0xC0-0xC7 */ + 0x8AD2,0x907C,0x91CF,0x9675,0x9818,0x529B,0x7DD1,0x502B,/* 0xC8-0xCF */ + 0x5398,0x6797,0x6DCB,0x71D0,0x7433,0x81E8,0x8F2A,0x96A3,/* 0xD0-0xD7 */ + 0x9C57,0x9E9F,0x7460,0x5841,0x6D99,0x7D2F,0x985E,0x4EE4,/* 0xD8-0xDF */ + 0x4F36,0x4F8B,0x51B7,0x52B1,0x5DBA,0x601C,0x73B2,0x793C,/* 0xE0-0xE7 */ + 0x82D3,0x9234,0x96B7,0x96F6,0x970A,0x9E97,0x9F62,0x66A6,/* 0xE8-0xEF */ + 0x6B74,0x5217,0x52A3,0x70C8,0x88C2,0x5EC9,0x604B,0x6190,/* 0xF0-0xF7 */ + 0x6F23,0x7149,0x7C3E,0x7DF4,0x806F,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_98[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x84EE,0x9023,0x932C,0x5442,0x9B6F,0x6AD3,0x7089,0x8CC2,/* 0x40-0x47 */ + 0x8DEF,0x9732,0x52B4,0x5A41,0x5ECA,0x5F04,0x6717,0x697C,/* 0x48-0x4F */ + 0x6994,0x6D6A,0x6F0F,0x7262,0x72FC,0x7BED,0x8001,0x807E,/* 0x50-0x57 */ + 0x874B,0x90CE,0x516D,0x9E93,0x7984,0x808B,0x9332,0x8AD6,/* 0x58-0x5F */ + 0x502D,0x548C,0x8A71,0x6B6A,0x8CC4,0x8107,0x60D1,0x67A0,/* 0x60-0x67 */ + 0x9DF2,0x4E99,0x4E98,0x9C10,0x8A6B,0x85C1,0x8568,0x6900,/* 0x68-0x6F */ + 0x6E7E,0x7897,0x8155,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x5F0C,/* 0x98-0x9F */ + 0x4E10,0x4E15,0x4E2A,0x4E31,0x4E36,0x4E3C,0x4E3F,0x4E42,/* 0xA0-0xA7 */ + 0x4E56,0x4E58,0x4E82,0x4E85,0x8C6B,0x4E8A,0x8212,0x5F0D,/* 0xA8-0xAF */ + 0x4E8E,0x4E9E,0x4E9F,0x4EA0,0x4EA2,0x4EB0,0x4EB3,0x4EB6,/* 0xB0-0xB7 */ + 0x4ECE,0x4ECD,0x4EC4,0x4EC6,0x4EC2,0x4ED7,0x4EDE,0x4EED,/* 0xB8-0xBF */ + 0x4EDF,0x4EF7,0x4F09,0x4F5A,0x4F30,0x4F5B,0x4F5D,0x4F57,/* 0xC0-0xC7 */ + 0x4F47,0x4F76,0x4F88,0x4F8F,0x4F98,0x4F7B,0x4F69,0x4F70,/* 0xC8-0xCF */ + 0x4F91,0x4F6F,0x4F86,0x4F96,0x5118,0x4FD4,0x4FDF,0x4FCE,/* 0xD0-0xD7 */ + 0x4FD8,0x4FDB,0x4FD1,0x4FDA,0x4FD0,0x4FE4,0x4FE5,0x501A,/* 0xD8-0xDF */ + 0x5028,0x5014,0x502A,0x5025,0x5005,0x4F1C,0x4FF6,0x5021,/* 0xE0-0xE7 */ + 0x5029,0x502C,0x4FFE,0x4FEF,0x5011,0x5006,0x5043,0x5047,/* 0xE8-0xEF */ + 0x6703,0x5055,0x5050,0x5048,0x505A,0x5056,0x506C,0x5078,/* 0xF0-0xF7 */ + 0x5080,0x509A,0x5085,0x50B4,0x50B2,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_99[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x50C9,0x50CA,0x50B3,0x50C2,0x50D6,0x50DE,0x50E5,0x50ED,/* 0x40-0x47 */ + 0x50E3,0x50EE,0x50F9,0x50F5,0x5109,0x5101,0x5102,0x5116,/* 0x48-0x4F */ + 0x5115,0x5114,0x511A,0x5121,0x513A,0x5137,0x513C,0x513B,/* 0x50-0x57 */ + 0x513F,0x5140,0x5152,0x514C,0x5154,0x5162,0x7AF8,0x5169,/* 0x58-0x5F */ + 0x516A,0x516E,0x5180,0x5182,0x56D8,0x518C,0x5189,0x518F,/* 0x60-0x67 */ + 0x5191,0x5193,0x5195,0x5196,0x51A4,0x51A6,0x51A2,0x51A9,/* 0x68-0x6F */ + 0x51AA,0x51AB,0x51B3,0x51B1,0x51B2,0x51B0,0x51B5,0x51BD,/* 0x70-0x77 */ + 0x51C5,0x51C9,0x51DB,0x51E0,0x8655,0x51E9,0x51ED,0x0000,/* 0x78-0x7F */ + + 0x51F0,0x51F5,0x51FE,0x5204,0x520B,0x5214,0x520E,0x5227,/* 0x80-0x87 */ + 0x522A,0x522E,0x5233,0x5239,0x524F,0x5244,0x524B,0x524C,/* 0x88-0x8F */ + 0x525E,0x5254,0x526A,0x5274,0x5269,0x5273,0x527F,0x527D,/* 0x90-0x97 */ + 0x528D,0x5294,0x5292,0x5271,0x5288,0x5291,0x8FA8,0x8FA7,/* 0x98-0x9F */ + 0x52AC,0x52AD,0x52BC,0x52B5,0x52C1,0x52CD,0x52D7,0x52DE,/* 0xA0-0xA7 */ + 0x52E3,0x52E6,0x98ED,0x52E0,0x52F3,0x52F5,0x52F8,0x52F9,/* 0xA8-0xAF */ + 0x5306,0x5308,0x7538,0x530D,0x5310,0x530F,0x5315,0x531A,/* 0xB0-0xB7 */ + 0x5323,0x532F,0x5331,0x5333,0x5338,0x5340,0x5346,0x5345,/* 0xB8-0xBF */ + 0x4E17,0x5349,0x534D,0x51D6,0x535E,0x5369,0x536E,0x5918,/* 0xC0-0xC7 */ + 0x537B,0x5377,0x5382,0x5396,0x53A0,0x53A6,0x53A5,0x53AE,/* 0xC8-0xCF */ + 0x53B0,0x53B6,0x53C3,0x7C12,0x96D9,0x53DF,0x66FC,0x71EE,/* 0xD0-0xD7 */ + 0x53EE,0x53E8,0x53ED,0x53FA,0x5401,0x543D,0x5440,0x542C,/* 0xD8-0xDF */ + 0x542D,0x543C,0x542E,0x5436,0x5429,0x541D,0x544E,0x548F,/* 0xE0-0xE7 */ + 0x5475,0x548E,0x545F,0x5471,0x5477,0x5470,0x5492,0x547B,/* 0xE8-0xEF */ + 0x5480,0x5476,0x5484,0x5490,0x5486,0x54C7,0x54A2,0x54B8,/* 0xF0-0xF7 */ + 0x54A5,0x54AC,0x54C4,0x54C8,0x54A8,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_9A[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x54AB,0x54C2,0x54A4,0x54BE,0x54BC,0x54D8,0x54E5,0x54E6,/* 0x40-0x47 */ + 0x550F,0x5514,0x54FD,0x54EE,0x54ED,0x54FA,0x54E2,0x5539,/* 0x48-0x4F */ + 0x5540,0x5563,0x554C,0x552E,0x555C,0x5545,0x5556,0x5557,/* 0x50-0x57 */ + 0x5538,0x5533,0x555D,0x5599,0x5580,0x54AF,0x558A,0x559F,/* 0x58-0x5F */ + 0x557B,0x557E,0x5598,0x559E,0x55AE,0x557C,0x5583,0x55A9,/* 0x60-0x67 */ + 0x5587,0x55A8,0x55DA,0x55C5,0x55DF,0x55C4,0x55DC,0x55E4,/* 0x68-0x6F */ + 0x55D4,0x5614,0x55F7,0x5616,0x55FE,0x55FD,0x561B,0x55F9,/* 0x70-0x77 */ + 0x564E,0x5650,0x71DF,0x5634,0x5636,0x5632,0x5638,0x0000,/* 0x78-0x7F */ + + 0x566B,0x5664,0x562F,0x566C,0x566A,0x5686,0x5680,0x568A,/* 0x80-0x87 */ + 0x56A0,0x5694,0x568F,0x56A5,0x56AE,0x56B6,0x56B4,0x56C2,/* 0x88-0x8F */ + 0x56BC,0x56C1,0x56C3,0x56C0,0x56C8,0x56CE,0x56D1,0x56D3,/* 0x90-0x97 */ + 0x56D7,0x56EE,0x56F9,0x5700,0x56FF,0x5704,0x5709,0x5708,/* 0x98-0x9F */ + 0x570B,0x570D,0x5713,0x5718,0x5716,0x55C7,0x571C,0x5726,/* 0xA0-0xA7 */ + 0x5737,0x5738,0x574E,0x573B,0x5740,0x574F,0x5769,0x57C0,/* 0xA8-0xAF */ + 0x5788,0x5761,0x577F,0x5789,0x5793,0x57A0,0x57B3,0x57A4,/* 0xB0-0xB7 */ + 0x57AA,0x57B0,0x57C3,0x57C6,0x57D4,0x57D2,0x57D3,0x580A,/* 0xB8-0xBF */ + 0x57D6,0x57E3,0x580B,0x5819,0x581D,0x5872,0x5821,0x5862,/* 0xC0-0xC7 */ + 0x584B,0x5870,0x6BC0,0x5852,0x583D,0x5879,0x5885,0x58B9,/* 0xC8-0xCF */ + 0x589F,0x58AB,0x58BA,0x58DE,0x58BB,0x58B8,0x58AE,0x58C5,/* 0xD0-0xD7 */ + 0x58D3,0x58D1,0x58D7,0x58D9,0x58D8,0x58E5,0x58DC,0x58E4,/* 0xD8-0xDF */ + 0x58DF,0x58EF,0x58FA,0x58F9,0x58FB,0x58FC,0x58FD,0x5902,/* 0xE0-0xE7 */ + 0x590A,0x5910,0x591B,0x68A6,0x5925,0x592C,0x592D,0x5932,/* 0xE8-0xEF */ + 0x5938,0x593E,0x7AD2,0x5955,0x5950,0x594E,0x595A,0x5958,/* 0xF0-0xF7 */ + 0x5962,0x5960,0x5967,0x596C,0x5969,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_9B[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x5978,0x5981,0x599D,0x4F5E,0x4FAB,0x59A3,0x59B2,0x59C6,/* 0x40-0x47 */ + 0x59E8,0x59DC,0x598D,0x59D9,0x59DA,0x5A25,0x5A1F,0x5A11,/* 0x48-0x4F */ + 0x5A1C,0x5A09,0x5A1A,0x5A40,0x5A6C,0x5A49,0x5A35,0x5A36,/* 0x50-0x57 */ + 0x5A62,0x5A6A,0x5A9A,0x5ABC,0x5ABE,0x5ACB,0x5AC2,0x5ABD,/* 0x58-0x5F */ + 0x5AE3,0x5AD7,0x5AE6,0x5AE9,0x5AD6,0x5AFA,0x5AFB,0x5B0C,/* 0x60-0x67 */ + 0x5B0B,0x5B16,0x5B32,0x5AD0,0x5B2A,0x5B36,0x5B3E,0x5B43,/* 0x68-0x6F */ + 0x5B45,0x5B40,0x5B51,0x5B55,0x5B5A,0x5B5B,0x5B65,0x5B69,/* 0x70-0x77 */ + 0x5B70,0x5B73,0x5B75,0x5B78,0x6588,0x5B7A,0x5B80,0x0000,/* 0x78-0x7F */ + + 0x5B83,0x5BA6,0x5BB8,0x5BC3,0x5BC7,0x5BC9,0x5BD4,0x5BD0,/* 0x80-0x87 */ + 0x5BE4,0x5BE6,0x5BE2,0x5BDE,0x5BE5,0x5BEB,0x5BF0,0x5BF6,/* 0x88-0x8F */ + 0x5BF3,0x5C05,0x5C07,0x5C08,0x5C0D,0x5C13,0x5C20,0x5C22,/* 0x90-0x97 */ + 0x5C28,0x5C38,0x5C39,0x5C41,0x5C46,0x5C4E,0x5C53,0x5C50,/* 0x98-0x9F */ + 0x5C4F,0x5B71,0x5C6C,0x5C6E,0x4E62,0x5C76,0x5C79,0x5C8C,/* 0xA0-0xA7 */ + 0x5C91,0x5C94,0x599B,0x5CAB,0x5CBB,0x5CB6,0x5CBC,0x5CB7,/* 0xA8-0xAF */ + 0x5CC5,0x5CBE,0x5CC7,0x5CD9,0x5CE9,0x5CFD,0x5CFA,0x5CED,/* 0xB0-0xB7 */ + 0x5D8C,0x5CEA,0x5D0B,0x5D15,0x5D17,0x5D5C,0x5D1F,0x5D1B,/* 0xB8-0xBF */ + 0x5D11,0x5D14,0x5D22,0x5D1A,0x5D19,0x5D18,0x5D4C,0x5D52,/* 0xC0-0xC7 */ + 0x5D4E,0x5D4B,0x5D6C,0x5D73,0x5D76,0x5D87,0x5D84,0x5D82,/* 0xC8-0xCF */ + 0x5DA2,0x5D9D,0x5DAC,0x5DAE,0x5DBD,0x5D90,0x5DB7,0x5DBC,/* 0xD0-0xD7 */ + 0x5DC9,0x5DCD,0x5DD3,0x5DD2,0x5DD6,0x5DDB,0x5DEB,0x5DF2,/* 0xD8-0xDF */ + 0x5DF5,0x5E0B,0x5E1A,0x5E19,0x5E11,0x5E1B,0x5E36,0x5E37,/* 0xE0-0xE7 */ + 0x5E44,0x5E43,0x5E40,0x5E4E,0x5E57,0x5E54,0x5E5F,0x5E62,/* 0xE8-0xEF */ + 0x5E64,0x5E47,0x5E75,0x5E76,0x5E7A,0x9EBC,0x5E7F,0x5EA0,/* 0xF0-0xF7 */ + 0x5EC1,0x5EC2,0x5EC8,0x5ED0,0x5ECF,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_9C[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x5ED6,0x5EE3,0x5EDD,0x5EDA,0x5EDB,0x5EE2,0x5EE1,0x5EE8,/* 0x40-0x47 */ + 0x5EE9,0x5EEC,0x5EF1,0x5EF3,0x5EF0,0x5EF4,0x5EF8,0x5EFE,/* 0x48-0x4F */ + 0x5F03,0x5F09,0x5F5D,0x5F5C,0x5F0B,0x5F11,0x5F16,0x5F29,/* 0x50-0x57 */ + 0x5F2D,0x5F38,0x5F41,0x5F48,0x5F4C,0x5F4E,0x5F2F,0x5F51,/* 0x58-0x5F */ + 0x5F56,0x5F57,0x5F59,0x5F61,0x5F6D,0x5F73,0x5F77,0x5F83,/* 0x60-0x67 */ + 0x5F82,0x5F7F,0x5F8A,0x5F88,0x5F91,0x5F87,0x5F9E,0x5F99,/* 0x68-0x6F */ + 0x5F98,0x5FA0,0x5FA8,0x5FAD,0x5FBC,0x5FD6,0x5FFB,0x5FE4,/* 0x70-0x77 */ + 0x5FF8,0x5FF1,0x5FDD,0x60B3,0x5FFF,0x6021,0x6060,0x0000,/* 0x78-0x7F */ + + 0x6019,0x6010,0x6029,0x600E,0x6031,0x601B,0x6015,0x602B,/* 0x80-0x87 */ + 0x6026,0x600F,0x603A,0x605A,0x6041,0x606A,0x6077,0x605F,/* 0x88-0x8F */ + 0x604A,0x6046,0x604D,0x6063,0x6043,0x6064,0x6042,0x606C,/* 0x90-0x97 */ + 0x606B,0x6059,0x6081,0x608D,0x60E7,0x6083,0x609A,0x6084,/* 0x98-0x9F */ + 0x609B,0x6096,0x6097,0x6092,0x60A7,0x608B,0x60E1,0x60B8,/* 0xA0-0xA7 */ + 0x60E0,0x60D3,0x60B4,0x5FF0,0x60BD,0x60C6,0x60B5,0x60D8,/* 0xA8-0xAF */ + 0x614D,0x6115,0x6106,0x60F6,0x60F7,0x6100,0x60F4,0x60FA,/* 0xB0-0xB7 */ + 0x6103,0x6121,0x60FB,0x60F1,0x610D,0x610E,0x6147,0x613E,/* 0xB8-0xBF */ + 0x6128,0x6127,0x614A,0x613F,0x613C,0x612C,0x6134,0x613D,/* 0xC0-0xC7 */ + 0x6142,0x6144,0x6173,0x6177,0x6158,0x6159,0x615A,0x616B,/* 0xC8-0xCF */ + 0x6174,0x616F,0x6165,0x6171,0x615F,0x615D,0x6153,0x6175,/* 0xD0-0xD7 */ + 0x6199,0x6196,0x6187,0x61AC,0x6194,0x619A,0x618A,0x6191,/* 0xD8-0xDF */ + 0x61AB,0x61AE,0x61CC,0x61CA,0x61C9,0x61F7,0x61C8,0x61C3,/* 0xE0-0xE7 */ + 0x61C6,0x61BA,0x61CB,0x7F79,0x61CD,0x61E6,0x61E3,0x61F6,/* 0xE8-0xEF */ + 0x61FA,0x61F4,0x61FF,0x61FD,0x61FC,0x61FE,0x6200,0x6208,/* 0xF0-0xF7 */ + 0x6209,0x620D,0x620C,0x6214,0x621B,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_9D[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x621E,0x6221,0x622A,0x622E,0x6230,0x6232,0x6233,0x6241,/* 0x40-0x47 */ + 0x624E,0x625E,0x6263,0x625B,0x6260,0x6268,0x627C,0x6282,/* 0x48-0x4F */ + 0x6289,0x627E,0x6292,0x6293,0x6296,0x62D4,0x6283,0x6294,/* 0x50-0x57 */ + 0x62D7,0x62D1,0x62BB,0x62CF,0x62FF,0x62C6,0x64D4,0x62C8,/* 0x58-0x5F */ + 0x62DC,0x62CC,0x62CA,0x62C2,0x62C7,0x629B,0x62C9,0x630C,/* 0x60-0x67 */ + 0x62EE,0x62F1,0x6327,0x6302,0x6308,0x62EF,0x62F5,0x6350,/* 0x68-0x6F */ + 0x633E,0x634D,0x641C,0x634F,0x6396,0x638E,0x6380,0x63AB,/* 0x70-0x77 */ + 0x6376,0x63A3,0x638F,0x6389,0x639F,0x63B5,0x636B,0x0000,/* 0x78-0x7F */ + + 0x6369,0x63BE,0x63E9,0x63C0,0x63C6,0x63E3,0x63C9,0x63D2,/* 0x80-0x87 */ + 0x63F6,0x63C4,0x6416,0x6434,0x6406,0x6413,0x6426,0x6436,/* 0x88-0x8F */ + 0x651D,0x6417,0x6428,0x640F,0x6467,0x646F,0x6476,0x644E,/* 0x90-0x97 */ + 0x652A,0x6495,0x6493,0x64A5,0x64A9,0x6488,0x64BC,0x64DA,/* 0x98-0x9F */ + 0x64D2,0x64C5,0x64C7,0x64BB,0x64D8,0x64C2,0x64F1,0x64E7,/* 0xA0-0xA7 */ + 0x8209,0x64E0,0x64E1,0x62AC,0x64E3,0x64EF,0x652C,0x64F6,/* 0xA8-0xAF */ + 0x64F4,0x64F2,0x64FA,0x6500,0x64FD,0x6518,0x651C,0x6505,/* 0xB0-0xB7 */ + 0x6524,0x6523,0x652B,0x6534,0x6535,0x6537,0x6536,0x6538,/* 0xB8-0xBF */ + 0x754B,0x6548,0x6556,0x6555,0x654D,0x6558,0x655E,0x655D,/* 0xC0-0xC7 */ + 0x6572,0x6578,0x6582,0x6583,0x8B8A,0x659B,0x659F,0x65AB,/* 0xC8-0xCF */ + 0x65B7,0x65C3,0x65C6,0x65C1,0x65C4,0x65CC,0x65D2,0x65DB,/* 0xD0-0xD7 */ + 0x65D9,0x65E0,0x65E1,0x65F1,0x6772,0x660A,0x6603,0x65FB,/* 0xD8-0xDF */ + 0x6773,0x6635,0x6636,0x6634,0x661C,0x664F,0x6644,0x6649,/* 0xE0-0xE7 */ + 0x6641,0x665E,0x665D,0x6664,0x6667,0x6668,0x665F,0x6662,/* 0xE8-0xEF */ + 0x6670,0x6683,0x6688,0x668E,0x6689,0x6684,0x6698,0x669D,/* 0xF0-0xF7 */ + 0x66C1,0x66B9,0x66C9,0x66BE,0x66BC,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_9E[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x66C4,0x66B8,0x66D6,0x66DA,0x66E0,0x663F,0x66E6,0x66E9,/* 0x40-0x47 */ + 0x66F0,0x66F5,0x66F7,0x670F,0x6716,0x671E,0x6726,0x6727,/* 0x48-0x4F */ + 0x9738,0x672E,0x673F,0x6736,0x6741,0x6738,0x6737,0x6746,/* 0x50-0x57 */ + 0x675E,0x6760,0x6759,0x6763,0x6764,0x6789,0x6770,0x67A9,/* 0x58-0x5F */ + 0x677C,0x676A,0x678C,0x678B,0x67A6,0x67A1,0x6785,0x67B7,/* 0x60-0x67 */ + 0x67EF,0x67B4,0x67EC,0x67B3,0x67E9,0x67B8,0x67E4,0x67DE,/* 0x68-0x6F */ + 0x67DD,0x67E2,0x67EE,0x67B9,0x67CE,0x67C6,0x67E7,0x6A9C,/* 0x70-0x77 */ + 0x681E,0x6846,0x6829,0x6840,0x684D,0x6832,0x684E,0x0000,/* 0x78-0x7F */ + + 0x68B3,0x682B,0x6859,0x6863,0x6877,0x687F,0x689F,0x688F,/* 0x80-0x87 */ + 0x68AD,0x6894,0x689D,0x689B,0x6883,0x6AAE,0x68B9,0x6874,/* 0x88-0x8F */ + 0x68B5,0x68A0,0x68BA,0x690F,0x688D,0x687E,0x6901,0x68CA,/* 0x90-0x97 */ + 0x6908,0x68D8,0x6922,0x6926,0x68E1,0x690C,0x68CD,0x68D4,/* 0x98-0x9F */ + 0x68E7,0x68D5,0x6936,0x6912,0x6904,0x68D7,0x68E3,0x6925,/* 0xA0-0xA7 */ + 0x68F9,0x68E0,0x68EF,0x6928,0x692A,0x691A,0x6923,0x6921,/* 0xA8-0xAF */ + 0x68C6,0x6979,0x6977,0x695C,0x6978,0x696B,0x6954,0x697E,/* 0xB0-0xB7 */ + 0x696E,0x6939,0x6974,0x693D,0x6959,0x6930,0x6961,0x695E,/* 0xB8-0xBF */ + 0x695D,0x6981,0x696A,0x69B2,0x69AE,0x69D0,0x69BF,0x69C1,/* 0xC0-0xC7 */ + 0x69D3,0x69BE,0x69CE,0x5BE8,0x69CA,0x69DD,0x69BB,0x69C3,/* 0xC8-0xCF */ + 0x69A7,0x6A2E,0x6991,0x69A0,0x699C,0x6995,0x69B4,0x69DE,/* 0xD0-0xD7 */ + 0x69E8,0x6A02,0x6A1B,0x69FF,0x6B0A,0x69F9,0x69F2,0x69E7,/* 0xD8-0xDF */ + 0x6A05,0x69B1,0x6A1E,0x69ED,0x6A14,0x69EB,0x6A0A,0x6A12,/* 0xE0-0xE7 */ + 0x6AC1,0x6A23,0x6A13,0x6A44,0x6A0C,0x6A72,0x6A36,0x6A78,/* 0xE8-0xEF */ + 0x6A47,0x6A62,0x6A59,0x6A66,0x6A48,0x6A38,0x6A22,0x6A90,/* 0xF0-0xF7 */ + 0x6A8D,0x6AA0,0x6A84,0x6AA2,0x6AA3,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_9F[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6A97,0x8617,0x6ABB,0x6AC3,0x6AC2,0x6AB8,0x6AB3,0x6AAC,/* 0x40-0x47 */ + 0x6ADE,0x6AD1,0x6ADF,0x6AAA,0x6ADA,0x6AEA,0x6AFB,0x6B05,/* 0x48-0x4F */ + 0x8616,0x6AFA,0x6B12,0x6B16,0x9B31,0x6B1F,0x6B38,0x6B37,/* 0x50-0x57 */ + 0x76DC,0x6B39,0x98EE,0x6B47,0x6B43,0x6B49,0x6B50,0x6B59,/* 0x58-0x5F */ + 0x6B54,0x6B5B,0x6B5F,0x6B61,0x6B78,0x6B79,0x6B7F,0x6B80,/* 0x60-0x67 */ + 0x6B84,0x6B83,0x6B8D,0x6B98,0x6B95,0x6B9E,0x6BA4,0x6BAA,/* 0x68-0x6F */ + 0x6BAB,0x6BAF,0x6BB2,0x6BB1,0x6BB3,0x6BB7,0x6BBC,0x6BC6,/* 0x70-0x77 */ + 0x6BCB,0x6BD3,0x6BDF,0x6BEC,0x6BEB,0x6BF3,0x6BEF,0x0000,/* 0x78-0x7F */ + + 0x9EBE,0x6C08,0x6C13,0x6C14,0x6C1B,0x6C24,0x6C23,0x6C5E,/* 0x80-0x87 */ + 0x6C55,0x6C62,0x6C6A,0x6C82,0x6C8D,0x6C9A,0x6C81,0x6C9B,/* 0x88-0x8F */ + 0x6C7E,0x6C68,0x6C73,0x6C92,0x6C90,0x6CC4,0x6CF1,0x6CD3,/* 0x90-0x97 */ + 0x6CBD,0x6CD7,0x6CC5,0x6CDD,0x6CAE,0x6CB1,0x6CBE,0x6CBA,/* 0x98-0x9F */ + 0x6CDB,0x6CEF,0x6CD9,0x6CEA,0x6D1F,0x884D,0x6D36,0x6D2B,/* 0xA0-0xA7 */ + 0x6D3D,0x6D38,0x6D19,0x6D35,0x6D33,0x6D12,0x6D0C,0x6D63,/* 0xA8-0xAF */ + 0x6D93,0x6D64,0x6D5A,0x6D79,0x6D59,0x6D8E,0x6D95,0x6FE4,/* 0xB0-0xB7 */ + 0x6D85,0x6DF9,0x6E15,0x6E0A,0x6DB5,0x6DC7,0x6DE6,0x6DB8,/* 0xB8-0xBF */ + 0x6DC6,0x6DEC,0x6DDE,0x6DCC,0x6DE8,0x6DD2,0x6DC5,0x6DFA,/* 0xC0-0xC7 */ + 0x6DD9,0x6DE4,0x6DD5,0x6DEA,0x6DEE,0x6E2D,0x6E6E,0x6E2E,/* 0xC8-0xCF */ + 0x6E19,0x6E72,0x6E5F,0x6E3E,0x6E23,0x6E6B,0x6E2B,0x6E76,/* 0xD0-0xD7 */ + 0x6E4D,0x6E1F,0x6E43,0x6E3A,0x6E4E,0x6E24,0x6EFF,0x6E1D,/* 0xD8-0xDF */ + 0x6E38,0x6E82,0x6EAA,0x6E98,0x6EC9,0x6EB7,0x6ED3,0x6EBD,/* 0xE0-0xE7 */ + 0x6EAF,0x6EC4,0x6EB2,0x6ED4,0x6ED5,0x6E8F,0x6EA5,0x6EC2,/* 0xE8-0xEF */ + 0x6E9F,0x6F41,0x6F11,0x704C,0x6EEC,0x6EF8,0x6EFE,0x6F3F,/* 0xF0-0xF7 */ + 0x6EF2,0x6F31,0x6EEF,0x6F32,0x6ECC,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E0[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6F3E,0x6F13,0x6EF7,0x6F86,0x6F7A,0x6F78,0x6F81,0x6F80,/* 0x40-0x47 */ + 0x6F6F,0x6F5B,0x6FF3,0x6F6D,0x6F82,0x6F7C,0x6F58,0x6F8E,/* 0x48-0x4F */ + 0x6F91,0x6FC2,0x6F66,0x6FB3,0x6FA3,0x6FA1,0x6FA4,0x6FB9,/* 0x50-0x57 */ + 0x6FC6,0x6FAA,0x6FDF,0x6FD5,0x6FEC,0x6FD4,0x6FD8,0x6FF1,/* 0x58-0x5F */ + 0x6FEE,0x6FDB,0x7009,0x700B,0x6FFA,0x7011,0x7001,0x700F,/* 0x60-0x67 */ + 0x6FFE,0x701B,0x701A,0x6F74,0x701D,0x7018,0x701F,0x7030,/* 0x68-0x6F */ + 0x703E,0x7032,0x7051,0x7063,0x7099,0x7092,0x70AF,0x70F1,/* 0x70-0x77 */ + 0x70AC,0x70B8,0x70B3,0x70AE,0x70DF,0x70CB,0x70DD,0x0000,/* 0x78-0x7F */ + + 0x70D9,0x7109,0x70FD,0x711C,0x7119,0x7165,0x7155,0x7188,/* 0x80-0x87 */ + 0x7166,0x7162,0x714C,0x7156,0x716C,0x718F,0x71FB,0x7184,/* 0x88-0x8F */ + 0x7195,0x71A8,0x71AC,0x71D7,0x71B9,0x71BE,0x71D2,0x71C9,/* 0x90-0x97 */ + 0x71D4,0x71CE,0x71E0,0x71EC,0x71E7,0x71F5,0x71FC,0x71F9,/* 0x98-0x9F */ + 0x71FF,0x720D,0x7210,0x721B,0x7228,0x722D,0x722C,0x7230,/* 0xA0-0xA7 */ + 0x7232,0x723B,0x723C,0x723F,0x7240,0x7246,0x724B,0x7258,/* 0xA8-0xAF */ + 0x7274,0x727E,0x7282,0x7281,0x7287,0x7292,0x7296,0x72A2,/* 0xB0-0xB7 */ + 0x72A7,0x72B9,0x72B2,0x72C3,0x72C6,0x72C4,0x72CE,0x72D2,/* 0xB8-0xBF */ + 0x72E2,0x72E0,0x72E1,0x72F9,0x72F7,0x500F,0x7317,0x730A,/* 0xC0-0xC7 */ + 0x731C,0x7316,0x731D,0x7334,0x732F,0x7329,0x7325,0x733E,/* 0xC8-0xCF */ + 0x734E,0x734F,0x9ED8,0x7357,0x736A,0x7368,0x7370,0x7378,/* 0xD0-0xD7 */ + 0x7375,0x737B,0x737A,0x73C8,0x73B3,0x73CE,0x73BB,0x73C0,/* 0xD8-0xDF */ + 0x73E5,0x73EE,0x73DE,0x74A2,0x7405,0x746F,0x7425,0x73F8,/* 0xE0-0xE7 */ + 0x7432,0x743A,0x7455,0x743F,0x745F,0x7459,0x7441,0x745C,/* 0xE8-0xEF */ + 0x7469,0x7470,0x7463,0x746A,0x7476,0x747E,0x748B,0x749E,/* 0xF0-0xF7 */ + 0x74A7,0x74CA,0x74CF,0x74D4,0x73F1,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E1[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x74E0,0x74E3,0x74E7,0x74E9,0x74EE,0x74F2,0x74F0,0x74F1,/* 0x40-0x47 */ + 0x74F8,0x74F7,0x7504,0x7503,0x7505,0x750C,0x750E,0x750D,/* 0x48-0x4F */ + 0x7515,0x7513,0x751E,0x7526,0x752C,0x753C,0x7544,0x754D,/* 0x50-0x57 */ + 0x754A,0x7549,0x755B,0x7546,0x755A,0x7569,0x7564,0x7567,/* 0x58-0x5F */ + 0x756B,0x756D,0x7578,0x7576,0x7586,0x7587,0x7574,0x758A,/* 0x60-0x67 */ + 0x7589,0x7582,0x7594,0x759A,0x759D,0x75A5,0x75A3,0x75C2,/* 0x68-0x6F */ + 0x75B3,0x75C3,0x75B5,0x75BD,0x75B8,0x75BC,0x75B1,0x75CD,/* 0x70-0x77 */ + 0x75CA,0x75D2,0x75D9,0x75E3,0x75DE,0x75FE,0x75FF,0x0000,/* 0x78-0x7F */ + + 0x75FC,0x7601,0x75F0,0x75FA,0x75F2,0x75F3,0x760B,0x760D,/* 0x80-0x87 */ + 0x7609,0x761F,0x7627,0x7620,0x7621,0x7622,0x7624,0x7634,/* 0x88-0x8F */ + 0x7630,0x763B,0x7647,0x7648,0x7646,0x765C,0x7658,0x7661,/* 0x90-0x97 */ + 0x7662,0x7668,0x7669,0x766A,0x7667,0x766C,0x7670,0x7672,/* 0x98-0x9F */ + 0x7676,0x7678,0x767C,0x7680,0x7683,0x7688,0x768B,0x768E,/* 0xA0-0xA7 */ + 0x7696,0x7693,0x7699,0x769A,0x76B0,0x76B4,0x76B8,0x76B9,/* 0xA8-0xAF */ + 0x76BA,0x76C2,0x76CD,0x76D6,0x76D2,0x76DE,0x76E1,0x76E5,/* 0xB0-0xB7 */ + 0x76E7,0x76EA,0x862F,0x76FB,0x7708,0x7707,0x7704,0x7729,/* 0xB8-0xBF */ + 0x7724,0x771E,0x7725,0x7726,0x771B,0x7737,0x7738,0x7747,/* 0xC0-0xC7 */ + 0x775A,0x7768,0x776B,0x775B,0x7765,0x777F,0x777E,0x7779,/* 0xC8-0xCF */ + 0x778E,0x778B,0x7791,0x77A0,0x779E,0x77B0,0x77B6,0x77B9,/* 0xD0-0xD7 */ + 0x77BF,0x77BC,0x77BD,0x77BB,0x77C7,0x77CD,0x77D7,0x77DA,/* 0xD8-0xDF */ + 0x77DC,0x77E3,0x77EE,0x77FC,0x780C,0x7812,0x7926,0x7820,/* 0xE0-0xE7 */ + 0x792A,0x7845,0x788E,0x7874,0x7886,0x787C,0x789A,0x788C,/* 0xE8-0xEF */ + 0x78A3,0x78B5,0x78AA,0x78AF,0x78D1,0x78C6,0x78CB,0x78D4,/* 0xF0-0xF7 */ + 0x78BE,0x78BC,0x78C5,0x78CA,0x78EC,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E2[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x78E7,0x78DA,0x78FD,0x78F4,0x7907,0x7912,0x7911,0x7919,/* 0x40-0x47 */ + 0x792C,0x792B,0x7940,0x7960,0x7957,0x795F,0x795A,0x7955,/* 0x48-0x4F */ + 0x7953,0x797A,0x797F,0x798A,0x799D,0x79A7,0x9F4B,0x79AA,/* 0x50-0x57 */ + 0x79AE,0x79B3,0x79B9,0x79BA,0x79C9,0x79D5,0x79E7,0x79EC,/* 0x58-0x5F */ + 0x79E1,0x79E3,0x7A08,0x7A0D,0x7A18,0x7A19,0x7A20,0x7A1F,/* 0x60-0x67 */ + 0x7980,0x7A31,0x7A3B,0x7A3E,0x7A37,0x7A43,0x7A57,0x7A49,/* 0x68-0x6F */ + 0x7A61,0x7A62,0x7A69,0x9F9D,0x7A70,0x7A79,0x7A7D,0x7A88,/* 0x70-0x77 */ + 0x7A97,0x7A95,0x7A98,0x7A96,0x7AA9,0x7AC8,0x7AB0,0x0000,/* 0x78-0x7F */ + + 0x7AB6,0x7AC5,0x7AC4,0x7ABF,0x9083,0x7AC7,0x7ACA,0x7ACD,/* 0x80-0x87 */ + 0x7ACF,0x7AD5,0x7AD3,0x7AD9,0x7ADA,0x7ADD,0x7AE1,0x7AE2,/* 0x88-0x8F */ + 0x7AE6,0x7AED,0x7AF0,0x7B02,0x7B0F,0x7B0A,0x7B06,0x7B33,/* 0x90-0x97 */ + 0x7B18,0x7B19,0x7B1E,0x7B35,0x7B28,0x7B36,0x7B50,0x7B7A,/* 0x98-0x9F */ + 0x7B04,0x7B4D,0x7B0B,0x7B4C,0x7B45,0x7B75,0x7B65,0x7B74,/* 0xA0-0xA7 */ + 0x7B67,0x7B70,0x7B71,0x7B6C,0x7B6E,0x7B9D,0x7B98,0x7B9F,/* 0xA8-0xAF */ + 0x7B8D,0x7B9C,0x7B9A,0x7B8B,0x7B92,0x7B8F,0x7B5D,0x7B99,/* 0xB0-0xB7 */ + 0x7BCB,0x7BC1,0x7BCC,0x7BCF,0x7BB4,0x7BC6,0x7BDD,0x7BE9,/* 0xB8-0xBF */ + 0x7C11,0x7C14,0x7BE6,0x7BE5,0x7C60,0x7C00,0x7C07,0x7C13,/* 0xC0-0xC7 */ + 0x7BF3,0x7BF7,0x7C17,0x7C0D,0x7BF6,0x7C23,0x7C27,0x7C2A,/* 0xC8-0xCF */ + 0x7C1F,0x7C37,0x7C2B,0x7C3D,0x7C4C,0x7C43,0x7C54,0x7C4F,/* 0xD0-0xD7 */ + 0x7C40,0x7C50,0x7C58,0x7C5F,0x7C64,0x7C56,0x7C65,0x7C6C,/* 0xD8-0xDF */ + 0x7C75,0x7C83,0x7C90,0x7CA4,0x7CAD,0x7CA2,0x7CAB,0x7CA1,/* 0xE0-0xE7 */ + 0x7CA8,0x7CB3,0x7CB2,0x7CB1,0x7CAE,0x7CB9,0x7CBD,0x7CC0,/* 0xE8-0xEF */ + 0x7CC5,0x7CC2,0x7CD8,0x7CD2,0x7CDC,0x7CE2,0x9B3B,0x7CEF,/* 0xF0-0xF7 */ + 0x7CF2,0x7CF4,0x7CF6,0x7CFA,0x7D06,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E3[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7D02,0x7D1C,0x7D15,0x7D0A,0x7D45,0x7D4B,0x7D2E,0x7D32,/* 0x40-0x47 */ + 0x7D3F,0x7D35,0x7D46,0x7D73,0x7D56,0x7D4E,0x7D72,0x7D68,/* 0x48-0x4F */ + 0x7D6E,0x7D4F,0x7D63,0x7D93,0x7D89,0x7D5B,0x7D8F,0x7D7D,/* 0x50-0x57 */ + 0x7D9B,0x7DBA,0x7DAE,0x7DA3,0x7DB5,0x7DC7,0x7DBD,0x7DAB,/* 0x58-0x5F */ + 0x7E3D,0x7DA2,0x7DAF,0x7DDC,0x7DB8,0x7D9F,0x7DB0,0x7DD8,/* 0x60-0x67 */ + 0x7DDD,0x7DE4,0x7DDE,0x7DFB,0x7DF2,0x7DE1,0x7E05,0x7E0A,/* 0x68-0x6F */ + 0x7E23,0x7E21,0x7E12,0x7E31,0x7E1F,0x7E09,0x7E0B,0x7E22,/* 0x70-0x77 */ + 0x7E46,0x7E66,0x7E3B,0x7E35,0x7E39,0x7E43,0x7E37,0x0000,/* 0x78-0x7F */ + + 0x7E32,0x7E3A,0x7E67,0x7E5D,0x7E56,0x7E5E,0x7E59,0x7E5A,/* 0x80-0x87 */ + 0x7E79,0x7E6A,0x7E69,0x7E7C,0x7E7B,0x7E83,0x7DD5,0x7E7D,/* 0x88-0x8F */ + 0x8FAE,0x7E7F,0x7E88,0x7E89,0x7E8C,0x7E92,0x7E90,0x7E93,/* 0x90-0x97 */ + 0x7E94,0x7E96,0x7E8E,0x7E9B,0x7E9C,0x7F38,0x7F3A,0x7F45,/* 0x98-0x9F */ + 0x7F4C,0x7F4D,0x7F4E,0x7F50,0x7F51,0x7F55,0x7F54,0x7F58,/* 0xA0-0xA7 */ + 0x7F5F,0x7F60,0x7F68,0x7F69,0x7F67,0x7F78,0x7F82,0x7F86,/* 0xA8-0xAF */ + 0x7F83,0x7F88,0x7F87,0x7F8C,0x7F94,0x7F9E,0x7F9D,0x7F9A,/* 0xB0-0xB7 */ + 0x7FA3,0x7FAF,0x7FB2,0x7FB9,0x7FAE,0x7FB6,0x7FB8,0x8B71,/* 0xB8-0xBF */ + 0x7FC5,0x7FC6,0x7FCA,0x7FD5,0x7FD4,0x7FE1,0x7FE6,0x7FE9,/* 0xC0-0xC7 */ + 0x7FF3,0x7FF9,0x98DC,0x8006,0x8004,0x800B,0x8012,0x8018,/* 0xC8-0xCF */ + 0x8019,0x801C,0x8021,0x8028,0x803F,0x803B,0x804A,0x8046,/* 0xD0-0xD7 */ + 0x8052,0x8058,0x805A,0x805F,0x8062,0x8068,0x8073,0x8072,/* 0xD8-0xDF */ + 0x8070,0x8076,0x8079,0x807D,0x807F,0x8084,0x8086,0x8085,/* 0xE0-0xE7 */ + 0x809B,0x8093,0x809A,0x80AD,0x5190,0x80AC,0x80DB,0x80E5,/* 0xE8-0xEF */ + 0x80D9,0x80DD,0x80C4,0x80DA,0x80D6,0x8109,0x80EF,0x80F1,/* 0xF0-0xF7 */ + 0x811B,0x8129,0x8123,0x812F,0x814B,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E4[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x968B,0x8146,0x813E,0x8153,0x8151,0x80FC,0x8171,0x816E,/* 0x40-0x47 */ + 0x8165,0x8166,0x8174,0x8183,0x8188,0x818A,0x8180,0x8182,/* 0x48-0x4F */ + 0x81A0,0x8195,0x81A4,0x81A3,0x815F,0x8193,0x81A9,0x81B0,/* 0x50-0x57 */ + 0x81B5,0x81BE,0x81B8,0x81BD,0x81C0,0x81C2,0x81BA,0x81C9,/* 0x58-0x5F */ + 0x81CD,0x81D1,0x81D9,0x81D8,0x81C8,0x81DA,0x81DF,0x81E0,/* 0x60-0x67 */ + 0x81E7,0x81FA,0x81FB,0x81FE,0x8201,0x8202,0x8205,0x8207,/* 0x68-0x6F */ + 0x820A,0x820D,0x8210,0x8216,0x8229,0x822B,0x8238,0x8233,/* 0x70-0x77 */ + 0x8240,0x8259,0x8258,0x825D,0x825A,0x825F,0x8264,0x0000,/* 0x78-0x7F */ + + 0x8262,0x8268,0x826A,0x826B,0x822E,0x8271,0x8277,0x8278,/* 0x80-0x87 */ + 0x827E,0x828D,0x8292,0x82AB,0x829F,0x82BB,0x82AC,0x82E1,/* 0x88-0x8F */ + 0x82E3,0x82DF,0x82D2,0x82F4,0x82F3,0x82FA,0x8393,0x8303,/* 0x90-0x97 */ + 0x82FB,0x82F9,0x82DE,0x8306,0x82DC,0x8309,0x82D9,0x8335,/* 0x98-0x9F */ + 0x8334,0x8316,0x8332,0x8331,0x8340,0x8339,0x8350,0x8345,/* 0xA0-0xA7 */ + 0x832F,0x832B,0x8317,0x8318,0x8385,0x839A,0x83AA,0x839F,/* 0xA8-0xAF */ + 0x83A2,0x8396,0x8323,0x838E,0x8387,0x838A,0x837C,0x83B5,/* 0xB0-0xB7 */ + 0x8373,0x8375,0x83A0,0x8389,0x83A8,0x83F4,0x8413,0x83EB,/* 0xB8-0xBF */ + 0x83CE,0x83FD,0x8403,0x83D8,0x840B,0x83C1,0x83F7,0x8407,/* 0xC0-0xC7 */ + 0x83E0,0x83F2,0x840D,0x8422,0x8420,0x83BD,0x8438,0x8506,/* 0xC8-0xCF */ + 0x83FB,0x846D,0x842A,0x843C,0x855A,0x8484,0x8477,0x846B,/* 0xD0-0xD7 */ + 0x84AD,0x846E,0x8482,0x8469,0x8446,0x842C,0x846F,0x8479,/* 0xD8-0xDF */ + 0x8435,0x84CA,0x8462,0x84B9,0x84BF,0x849F,0x84D9,0x84CD,/* 0xE0-0xE7 */ + 0x84BB,0x84DA,0x84D0,0x84C1,0x84C6,0x84D6,0x84A1,0x8521,/* 0xE8-0xEF */ + 0x84FF,0x84F4,0x8517,0x8518,0x852C,0x851F,0x8515,0x8514,/* 0xF0-0xF7 */ + 0x84FC,0x8540,0x8563,0x8558,0x8548,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E5[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8541,0x8602,0x854B,0x8555,0x8580,0x85A4,0x8588,0x8591,/* 0x40-0x47 */ + 0x858A,0x85A8,0x856D,0x8594,0x859B,0x85EA,0x8587,0x859C,/* 0x48-0x4F */ + 0x8577,0x857E,0x8590,0x85C9,0x85BA,0x85CF,0x85B9,0x85D0,/* 0x50-0x57 */ + 0x85D5,0x85DD,0x85E5,0x85DC,0x85F9,0x860A,0x8613,0x860B,/* 0x58-0x5F */ + 0x85FE,0x85FA,0x8606,0x8622,0x861A,0x8630,0x863F,0x864D,/* 0x60-0x67 */ + 0x4E55,0x8654,0x865F,0x8667,0x8671,0x8693,0x86A3,0x86A9,/* 0x68-0x6F */ + 0x86AA,0x868B,0x868C,0x86B6,0x86AF,0x86C4,0x86C6,0x86B0,/* 0x70-0x77 */ + 0x86C9,0x8823,0x86AB,0x86D4,0x86DE,0x86E9,0x86EC,0x0000,/* 0x78-0x7F */ + + 0x86DF,0x86DB,0x86EF,0x8712,0x8706,0x8708,0x8700,0x8703,/* 0x80-0x87 */ + 0x86FB,0x8711,0x8709,0x870D,0x86F9,0x870A,0x8734,0x873F,/* 0x88-0x8F */ + 0x8737,0x873B,0x8725,0x8729,0x871A,0x8760,0x875F,0x8778,/* 0x90-0x97 */ + 0x874C,0x874E,0x8774,0x8757,0x8768,0x876E,0x8759,0x8753,/* 0x98-0x9F */ + 0x8763,0x876A,0x8805,0x87A2,0x879F,0x8782,0x87AF,0x87CB,/* 0xA0-0xA7 */ + 0x87BD,0x87C0,0x87D0,0x96D6,0x87AB,0x87C4,0x87B3,0x87C7,/* 0xA8-0xAF */ + 0x87C6,0x87BB,0x87EF,0x87F2,0x87E0,0x880F,0x880D,0x87FE,/* 0xB0-0xB7 */ + 0x87F6,0x87F7,0x880E,0x87D2,0x8811,0x8816,0x8815,0x8822,/* 0xB8-0xBF */ + 0x8821,0x8831,0x8836,0x8839,0x8827,0x883B,0x8844,0x8842,/* 0xC0-0xC7 */ + 0x8852,0x8859,0x885E,0x8862,0x886B,0x8881,0x887E,0x889E,/* 0xC8-0xCF */ + 0x8875,0x887D,0x88B5,0x8872,0x8882,0x8897,0x8892,0x88AE,/* 0xD0-0xD7 */ + 0x8899,0x88A2,0x888D,0x88A4,0x88B0,0x88BF,0x88B1,0x88C3,/* 0xD8-0xDF */ + 0x88C4,0x88D4,0x88D8,0x88D9,0x88DD,0x88F9,0x8902,0x88FC,/* 0xE0-0xE7 */ + 0x88F4,0x88E8,0x88F2,0x8904,0x890C,0x890A,0x8913,0x8943,/* 0xE8-0xEF */ + 0x891E,0x8925,0x892A,0x892B,0x8941,0x8944,0x893B,0x8936,/* 0xF0-0xF7 */ + 0x8938,0x894C,0x891D,0x8960,0x895E,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E6[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8966,0x8964,0x896D,0x896A,0x896F,0x8974,0x8977,0x897E,/* 0x40-0x47 */ + 0x8983,0x8988,0x898A,0x8993,0x8998,0x89A1,0x89A9,0x89A6,/* 0x48-0x4F */ + 0x89AC,0x89AF,0x89B2,0x89BA,0x89BD,0x89BF,0x89C0,0x89DA,/* 0x50-0x57 */ + 0x89DC,0x89DD,0x89E7,0x89F4,0x89F8,0x8A03,0x8A16,0x8A10,/* 0x58-0x5F */ + 0x8A0C,0x8A1B,0x8A1D,0x8A25,0x8A36,0x8A41,0x8A5B,0x8A52,/* 0x60-0x67 */ + 0x8A46,0x8A48,0x8A7C,0x8A6D,0x8A6C,0x8A62,0x8A85,0x8A82,/* 0x68-0x6F */ + 0x8A84,0x8AA8,0x8AA1,0x8A91,0x8AA5,0x8AA6,0x8A9A,0x8AA3,/* 0x70-0x77 */ + 0x8AC4,0x8ACD,0x8AC2,0x8ADA,0x8AEB,0x8AF3,0x8AE7,0x0000,/* 0x78-0x7F */ + + 0x8AE4,0x8AF1,0x8B14,0x8AE0,0x8AE2,0x8AF7,0x8ADE,0x8ADB,/* 0x80-0x87 */ + 0x8B0C,0x8B07,0x8B1A,0x8AE1,0x8B16,0x8B10,0x8B17,0x8B20,/* 0x88-0x8F */ + 0x8B33,0x97AB,0x8B26,0x8B2B,0x8B3E,0x8B28,0x8B41,0x8B4C,/* 0x90-0x97 */ + 0x8B4F,0x8B4E,0x8B49,0x8B56,0x8B5B,0x8B5A,0x8B6B,0x8B5F,/* 0x98-0x9F */ + 0x8B6C,0x8B6F,0x8B74,0x8B7D,0x8B80,0x8B8C,0x8B8E,0x8B92,/* 0xA0-0xA7 */ + 0x8B93,0x8B96,0x8B99,0x8B9A,0x8C3A,0x8C41,0x8C3F,0x8C48,/* 0xA8-0xAF */ + 0x8C4C,0x8C4E,0x8C50,0x8C55,0x8C62,0x8C6C,0x8C78,0x8C7A,/* 0xB0-0xB7 */ + 0x8C82,0x8C89,0x8C85,0x8C8A,0x8C8D,0x8C8E,0x8C94,0x8C7C,/* 0xB8-0xBF */ + 0x8C98,0x621D,0x8CAD,0x8CAA,0x8CBD,0x8CB2,0x8CB3,0x8CAE,/* 0xC0-0xC7 */ + 0x8CB6,0x8CC8,0x8CC1,0x8CE4,0x8CE3,0x8CDA,0x8CFD,0x8CFA,/* 0xC8-0xCF */ + 0x8CFB,0x8D04,0x8D05,0x8D0A,0x8D07,0x8D0F,0x8D0D,0x8D10,/* 0xD0-0xD7 */ + 0x9F4E,0x8D13,0x8CCD,0x8D14,0x8D16,0x8D67,0x8D6D,0x8D71,/* 0xD8-0xDF */ + 0x8D73,0x8D81,0x8D99,0x8DC2,0x8DBE,0x8DBA,0x8DCF,0x8DDA,/* 0xE0-0xE7 */ + 0x8DD6,0x8DCC,0x8DDB,0x8DCB,0x8DEA,0x8DEB,0x8DDF,0x8DE3,/* 0xE8-0xEF */ + 0x8DFC,0x8E08,0x8E09,0x8DFF,0x8E1D,0x8E1E,0x8E10,0x8E1F,/* 0xF0-0xF7 */ + 0x8E42,0x8E35,0x8E30,0x8E34,0x8E4A,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E7[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8E47,0x8E49,0x8E4C,0x8E50,0x8E48,0x8E59,0x8E64,0x8E60,/* 0x40-0x47 */ + 0x8E2A,0x8E63,0x8E55,0x8E76,0x8E72,0x8E7C,0x8E81,0x8E87,/* 0x48-0x4F */ + 0x8E85,0x8E84,0x8E8B,0x8E8A,0x8E93,0x8E91,0x8E94,0x8E99,/* 0x50-0x57 */ + 0x8EAA,0x8EA1,0x8EAC,0x8EB0,0x8EC6,0x8EB1,0x8EBE,0x8EC5,/* 0x58-0x5F */ + 0x8EC8,0x8ECB,0x8EDB,0x8EE3,0x8EFC,0x8EFB,0x8EEB,0x8EFE,/* 0x60-0x67 */ + 0x8F0A,0x8F05,0x8F15,0x8F12,0x8F19,0x8F13,0x8F1C,0x8F1F,/* 0x68-0x6F */ + 0x8F1B,0x8F0C,0x8F26,0x8F33,0x8F3B,0x8F39,0x8F45,0x8F42,/* 0x70-0x77 */ + 0x8F3E,0x8F4C,0x8F49,0x8F46,0x8F4E,0x8F57,0x8F5C,0x0000,/* 0x78-0x7F */ + + 0x8F62,0x8F63,0x8F64,0x8F9C,0x8F9F,0x8FA3,0x8FAD,0x8FAF,/* 0x80-0x87 */ + 0x8FB7,0x8FDA,0x8FE5,0x8FE2,0x8FEA,0x8FEF,0x9087,0x8FF4,/* 0x88-0x8F */ + 0x9005,0x8FF9,0x8FFA,0x9011,0x9015,0x9021,0x900D,0x901E,/* 0x90-0x97 */ + 0x9016,0x900B,0x9027,0x9036,0x9035,0x9039,0x8FF8,0x904F,/* 0x98-0x9F */ + 0x9050,0x9051,0x9052,0x900E,0x9049,0x903E,0x9056,0x9058,/* 0xA0-0xA7 */ + 0x905E,0x9068,0x906F,0x9076,0x96A8,0x9072,0x9082,0x907D,/* 0xA8-0xAF */ + 0x9081,0x9080,0x908A,0x9089,0x908F,0x90A8,0x90AF,0x90B1,/* 0xB0-0xB7 */ + 0x90B5,0x90E2,0x90E4,0x6248,0x90DB,0x9102,0x9112,0x9119,/* 0xB8-0xBF */ + 0x9132,0x9130,0x914A,0x9156,0x9158,0x9163,0x9165,0x9169,/* 0xC0-0xC7 */ + 0x9173,0x9172,0x918B,0x9189,0x9182,0x91A2,0x91AB,0x91AF,/* 0xC8-0xCF */ + 0x91AA,0x91B5,0x91B4,0x91BA,0x91C0,0x91C1,0x91C9,0x91CB,/* 0xD0-0xD7 */ + 0x91D0,0x91D6,0x91DF,0x91E1,0x91DB,0x91FC,0x91F5,0x91F6,/* 0xD8-0xDF */ + 0x921E,0x91FF,0x9214,0x922C,0x9215,0x9211,0x925E,0x9257,/* 0xE0-0xE7 */ + 0x9245,0x9249,0x9264,0x9248,0x9295,0x923F,0x924B,0x9250,/* 0xE8-0xEF */ + 0x929C,0x9296,0x9293,0x929B,0x925A,0x92CF,0x92B9,0x92B7,/* 0xF0-0xF7 */ + 0x92E9,0x930F,0x92FA,0x9344,0x932E,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E8[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9319,0x9322,0x931A,0x9323,0x933A,0x9335,0x933B,0x935C,/* 0x40-0x47 */ + 0x9360,0x937C,0x936E,0x9356,0x93B0,0x93AC,0x93AD,0x9394,/* 0x48-0x4F */ + 0x93B9,0x93D6,0x93D7,0x93E8,0x93E5,0x93D8,0x93C3,0x93DD,/* 0x50-0x57 */ + 0x93D0,0x93C8,0x93E4,0x941A,0x9414,0x9413,0x9403,0x9407,/* 0x58-0x5F */ + 0x9410,0x9436,0x942B,0x9435,0x9421,0x943A,0x9441,0x9452,/* 0x60-0x67 */ + 0x9444,0x945B,0x9460,0x9462,0x945E,0x946A,0x9229,0x9470,/* 0x68-0x6F */ + 0x9475,0x9477,0x947D,0x945A,0x947C,0x947E,0x9481,0x947F,/* 0x70-0x77 */ + 0x9582,0x9587,0x958A,0x9594,0x9596,0x9598,0x9599,0x0000,/* 0x78-0x7F */ + + 0x95A0,0x95A8,0x95A7,0x95AD,0x95BC,0x95BB,0x95B9,0x95BE,/* 0x80-0x87 */ + 0x95CA,0x6FF6,0x95C3,0x95CD,0x95CC,0x95D5,0x95D4,0x95D6,/* 0x88-0x8F */ + 0x95DC,0x95E1,0x95E5,0x95E2,0x9621,0x9628,0x962E,0x962F,/* 0x90-0x97 */ + 0x9642,0x964C,0x964F,0x964B,0x9677,0x965C,0x965E,0x965D,/* 0x98-0x9F */ + 0x965F,0x9666,0x9672,0x966C,0x968D,0x9698,0x9695,0x9697,/* 0xA0-0xA7 */ + 0x96AA,0x96A7,0x96B1,0x96B2,0x96B0,0x96B4,0x96B6,0x96B8,/* 0xA8-0xAF */ + 0x96B9,0x96CE,0x96CB,0x96C9,0x96CD,0x894D,0x96DC,0x970D,/* 0xB0-0xB7 */ + 0x96D5,0x96F9,0x9704,0x9706,0x9708,0x9713,0x970E,0x9711,/* 0xB8-0xBF */ + 0x970F,0x9716,0x9719,0x9724,0x972A,0x9730,0x9739,0x973D,/* 0xC0-0xC7 */ + 0x973E,0x9744,0x9746,0x9748,0x9742,0x9749,0x975C,0x9760,/* 0xC8-0xCF */ + 0x9764,0x9766,0x9768,0x52D2,0x976B,0x9771,0x9779,0x9785,/* 0xD0-0xD7 */ + 0x977C,0x9781,0x977A,0x9786,0x978B,0x978F,0x9790,0x979C,/* 0xD8-0xDF */ + 0x97A8,0x97A6,0x97A3,0x97B3,0x97B4,0x97C3,0x97C6,0x97C8,/* 0xE0-0xE7 */ + 0x97CB,0x97DC,0x97ED,0x9F4F,0x97F2,0x7ADF,0x97F6,0x97F5,/* 0xE8-0xEF */ + 0x980F,0x980C,0x9838,0x9824,0x9821,0x9837,0x983D,0x9846,/* 0xF0-0xF7 */ + 0x984F,0x984B,0x986B,0x986F,0x9870,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E9[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9871,0x9874,0x9873,0x98AA,0x98AF,0x98B1,0x98B6,0x98C4,/* 0x40-0x47 */ + 0x98C3,0x98C6,0x98E9,0x98EB,0x9903,0x9909,0x9912,0x9914,/* 0x48-0x4F */ + 0x9918,0x9921,0x991D,0x991E,0x9924,0x9920,0x992C,0x992E,/* 0x50-0x57 */ + 0x993D,0x993E,0x9942,0x9949,0x9945,0x9950,0x994B,0x9951,/* 0x58-0x5F */ + 0x9952,0x994C,0x9955,0x9997,0x9998,0x99A5,0x99AD,0x99AE,/* 0x60-0x67 */ + 0x99BC,0x99DF,0x99DB,0x99DD,0x99D8,0x99D1,0x99ED,0x99EE,/* 0x68-0x6F */ + 0x99F1,0x99F2,0x99FB,0x99F8,0x9A01,0x9A0F,0x9A05,0x99E2,/* 0x70-0x77 */ + 0x9A19,0x9A2B,0x9A37,0x9A45,0x9A42,0x9A40,0x9A43,0x0000,/* 0x78-0x7F */ + + 0x9A3E,0x9A55,0x9A4D,0x9A5B,0x9A57,0x9A5F,0x9A62,0x9A65,/* 0x80-0x87 */ + 0x9A64,0x9A69,0x9A6B,0x9A6A,0x9AAD,0x9AB0,0x9ABC,0x9AC0,/* 0x88-0x8F */ + 0x9ACF,0x9AD1,0x9AD3,0x9AD4,0x9ADE,0x9ADF,0x9AE2,0x9AE3,/* 0x90-0x97 */ + 0x9AE6,0x9AEF,0x9AEB,0x9AEE,0x9AF4,0x9AF1,0x9AF7,0x9AFB,/* 0x98-0x9F */ + 0x9B06,0x9B18,0x9B1A,0x9B1F,0x9B22,0x9B23,0x9B25,0x9B27,/* 0xA0-0xA7 */ + 0x9B28,0x9B29,0x9B2A,0x9B2E,0x9B2F,0x9B32,0x9B44,0x9B43,/* 0xA8-0xAF */ + 0x9B4F,0x9B4D,0x9B4E,0x9B51,0x9B58,0x9B74,0x9B93,0x9B83,/* 0xB0-0xB7 */ + 0x9B91,0x9B96,0x9B97,0x9B9F,0x9BA0,0x9BA8,0x9BB4,0x9BC0,/* 0xB8-0xBF */ + 0x9BCA,0x9BB9,0x9BC6,0x9BCF,0x9BD1,0x9BD2,0x9BE3,0x9BE2,/* 0xC0-0xC7 */ + 0x9BE4,0x9BD4,0x9BE1,0x9C3A,0x9BF2,0x9BF1,0x9BF0,0x9C15,/* 0xC8-0xCF */ + 0x9C14,0x9C09,0x9C13,0x9C0C,0x9C06,0x9C08,0x9C12,0x9C0A,/* 0xD0-0xD7 */ + 0x9C04,0x9C2E,0x9C1B,0x9C25,0x9C24,0x9C21,0x9C30,0x9C47,/* 0xD8-0xDF */ + 0x9C32,0x9C46,0x9C3E,0x9C5A,0x9C60,0x9C67,0x9C76,0x9C78,/* 0xE0-0xE7 */ + 0x9CE7,0x9CEC,0x9CF0,0x9D09,0x9D08,0x9CEB,0x9D03,0x9D06,/* 0xE8-0xEF */ + 0x9D2A,0x9D26,0x9DAF,0x9D23,0x9D1F,0x9D44,0x9D15,0x9D12,/* 0xF0-0xF7 */ + 0x9D41,0x9D3F,0x9D3E,0x9D46,0x9D48,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_EA[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9D5D,0x9D5E,0x9D64,0x9D51,0x9D50,0x9D59,0x9D72,0x9D89,/* 0x40-0x47 */ + 0x9D87,0x9DAB,0x9D6F,0x9D7A,0x9D9A,0x9DA4,0x9DA9,0x9DB2,/* 0x48-0x4F */ + 0x9DC4,0x9DC1,0x9DBB,0x9DB8,0x9DBA,0x9DC6,0x9DCF,0x9DC2,/* 0x50-0x57 */ + 0x9DD9,0x9DD3,0x9DF8,0x9DE6,0x9DED,0x9DEF,0x9DFD,0x9E1A,/* 0x58-0x5F */ + 0x9E1B,0x9E1E,0x9E75,0x9E79,0x9E7D,0x9E81,0x9E88,0x9E8B,/* 0x60-0x67 */ + 0x9E8C,0x9E92,0x9E95,0x9E91,0x9E9D,0x9EA5,0x9EA9,0x9EB8,/* 0x68-0x6F */ + 0x9EAA,0x9EAD,0x9761,0x9ECC,0x9ECE,0x9ECF,0x9ED0,0x9ED4,/* 0x70-0x77 */ + 0x9EDC,0x9EDE,0x9EDD,0x9EE0,0x9EE5,0x9EE8,0x9EEF,0x0000,/* 0x78-0x7F */ + + 0x9EF4,0x9EF6,0x9EF7,0x9EF9,0x9EFB,0x9EFC,0x9EFD,0x9F07,/* 0x80-0x87 */ + 0x9F08,0x76B7,0x9F15,0x9F21,0x9F2C,0x9F3E,0x9F4A,0x9F52,/* 0x88-0x8F */ + 0x9F54,0x9F63,0x9F5F,0x9F60,0x9F61,0x9F66,0x9F67,0x9F6C,/* 0x90-0x97 */ + 0x9F6A,0x9F77,0x9F72,0x9F76,0x9F95,0x9F9C,0x9FA0,0x582F,/* 0x98-0x9F */ + 0x69C7,0x9059,0x7464,0x51DC,0x7199,0x0000,0x0000,0x0000,/* 0xA0-0xA7 */ +}; + +static const wchar_t c2u_ED[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7E8A,0x891C,0x9348,0x9288,0x84DC,0x4FC9,0x70BB,0x6631,/* 0x40-0x47 */ + 0x68C8,0x92F9,0x66FB,0x5F45,0x4E28,0x4EE1,0x4EFC,0x4F00,/* 0x48-0x4F */ + 0x4F03,0x4F39,0x4F56,0x4F92,0x4F8A,0x4F9A,0x4F94,0x4FCD,/* 0x50-0x57 */ + 0x5040,0x5022,0x4FFF,0x501E,0x5046,0x5070,0x5042,0x5094,/* 0x58-0x5F */ + 0x50F4,0x50D8,0x514A,0x5164,0x519D,0x51BE,0x51EC,0x5215,/* 0x60-0x67 */ + 0x529C,0x52A6,0x52C0,0x52DB,0x5300,0x5307,0x5324,0x5372,/* 0x68-0x6F */ + 0x5393,0x53B2,0x53DD,0xFA0E,0x549C,0x548A,0x54A9,0x54FF,/* 0x70-0x77 */ + 0x5586,0x5759,0x5765,0x57AC,0x57C8,0x57C7,0xFA0F,0x0000,/* 0x78-0x7F */ + + 0xFA10,0x589E,0x58B2,0x590B,0x5953,0x595B,0x595D,0x5963,/* 0x80-0x87 */ + 0x59A4,0x59BA,0x5B56,0x5BC0,0x752F,0x5BD8,0x5BEC,0x5C1E,/* 0x88-0x8F */ + 0x5CA6,0x5CBA,0x5CF5,0x5D27,0x5D53,0xFA11,0x5D42,0x5D6D,/* 0x90-0x97 */ + 0x5DB8,0x5DB9,0x5DD0,0x5F21,0x5F34,0x5F67,0x5FB7,0x5FDE,/* 0x98-0x9F */ + 0x605D,0x6085,0x608A,0x60DE,0x60D5,0x6120,0x60F2,0x6111,/* 0xA0-0xA7 */ + 0x6137,0x6130,0x6198,0x6213,0x62A6,0x63F5,0x6460,0x649D,/* 0xA8-0xAF */ + 0x64CE,0x654E,0x6600,0x6615,0x663B,0x6609,0x662E,0x661E,/* 0xB0-0xB7 */ + 0x6624,0x6665,0x6657,0x6659,0xFA12,0x6673,0x6699,0x66A0,/* 0xB8-0xBF */ + 0x66B2,0x66BF,0x66FA,0x670E,0xF929,0x6766,0x67BB,0x6852,/* 0xC0-0xC7 */ + 0x67C0,0x6801,0x6844,0x68CF,0xFA13,0x6968,0xFA14,0x6998,/* 0xC8-0xCF */ + 0x69E2,0x6A30,0x6A6B,0x6A46,0x6A73,0x6A7E,0x6AE2,0x6AE4,/* 0xD0-0xD7 */ + 0x6BD6,0x6C3F,0x6C5C,0x6C86,0x6C6F,0x6CDA,0x6D04,0x6D87,/* 0xD8-0xDF */ + 0x6D6F,0x6D96,0x6DAC,0x6DCF,0x6DF8,0x6DF2,0x6DFC,0x6E39,/* 0xE0-0xE7 */ + 0x6E5C,0x6E27,0x6E3C,0x6EBF,0x6F88,0x6FB5,0x6FF5,0x7005,/* 0xE8-0xEF */ + 0x7007,0x7028,0x7085,0x70AB,0x710F,0x7104,0x715C,0x7146,/* 0xF0-0xF7 */ + 0x7147,0xFA15,0x71C1,0x71FE,0x72B1,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_EE[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x72BE,0x7324,0xFA16,0x7377,0x73BD,0x73C9,0x73D6,0x73E3,/* 0x40-0x47 */ + 0x73D2,0x7407,0x73F5,0x7426,0x742A,0x7429,0x742E,0x7462,/* 0x48-0x4F */ + 0x7489,0x749F,0x7501,0x756F,0x7682,0x769C,0x769E,0x769B,/* 0x50-0x57 */ + 0x76A6,0xFA17,0x7746,0x52AF,0x7821,0x784E,0x7864,0x787A,/* 0x58-0x5F */ + 0x7930,0xFA18,0xFA19,0xFA1A,0x7994,0xFA1B,0x799B,0x7AD1,/* 0x60-0x67 */ + 0x7AE7,0xFA1C,0x7AEB,0x7B9E,0xFA1D,0x7D48,0x7D5C,0x7DB7,/* 0x68-0x6F */ + 0x7DA0,0x7DD6,0x7E52,0x7F47,0x7FA1,0xFA1E,0x8301,0x8362,/* 0x70-0x77 */ + 0x837F,0x83C7,0x83F6,0x8448,0x84B4,0x8553,0x8559,0x0000,/* 0x78-0x7F */ + + 0x856B,0xFA1F,0x85B0,0xFA20,0xFA21,0x8807,0x88F5,0x8A12,/* 0x80-0x87 */ + 0x8A37,0x8A79,0x8AA7,0x8ABE,0x8ADF,0xFA22,0x8AF6,0x8B53,/* 0x88-0x8F */ + 0x8B7F,0x8CF0,0x8CF4,0x8D12,0x8D76,0xFA23,0x8ECF,0xFA24,/* 0x90-0x97 */ + 0xFA25,0x9067,0x90DE,0xFA26,0x9115,0x9127,0x91DA,0x91D7,/* 0x98-0x9F */ + 0x91DE,0x91ED,0x91EE,0x91E4,0x91E5,0x9206,0x9210,0x920A,/* 0xA0-0xA7 */ + 0x923A,0x9240,0x923C,0x924E,0x9259,0x9251,0x9239,0x9267,/* 0xA8-0xAF */ + 0x92A7,0x9277,0x9278,0x92E7,0x92D7,0x92D9,0x92D0,0xFA27,/* 0xB0-0xB7 */ + 0x92D5,0x92E0,0x92D3,0x9325,0x9321,0x92FB,0xFA28,0x931E,/* 0xB8-0xBF */ + 0x92FF,0x931D,0x9302,0x9370,0x9357,0x93A4,0x93C6,0x93DE,/* 0xC0-0xC7 */ + 0x93F8,0x9431,0x9445,0x9448,0x9592,0xF9DC,0xFA29,0x969D,/* 0xC8-0xCF */ + 0x96AF,0x9733,0x973B,0x9743,0x974D,0x974F,0x9751,0x9755,/* 0xD0-0xD7 */ + 0x9857,0x9865,0xFA2A,0xFA2B,0x9927,0xFA2C,0x999E,0x9A4E,/* 0xD8-0xDF */ + 0x9AD9,0x9ADC,0x9B75,0x9B72,0x9B8F,0x9BB1,0x9BBB,0x9C00,/* 0xE0-0xE7 */ + 0x9D70,0x9D6B,0xFA2D,0x9E19,0x9ED1,0x0000,0x0000,0x2170,/* 0xE8-0xEF */ + 0x2171,0x2172,0x2173,0x2174,0x2175,0x2176,0x2177,0x2178,/* 0xF0-0xF7 */ + 0x2179,0xFFE2,0xFFE4,0xFF07,0xFF02,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_FA[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x2170,0x2171,0x2172,0x2173,0x2174,0x2175,0x2176,0x2177,/* 0x40-0x47 */ + 0x2178,0x2179,0x2160,0x2161,0x2162,0x2163,0x2164,0x2165,/* 0x48-0x4F */ + 0x2166,0x2167,0x2168,0x2169,0xFFE2,0xFFE4,0xFF07,0xFF02,/* 0x50-0x57 */ + 0x3231,0x2116,0x2121,0x2235,0x7E8A,0x891C,0x9348,0x9288,/* 0x58-0x5F */ + 0x84DC,0x4FC9,0x70BB,0x6631,0x68C8,0x92F9,0x66FB,0x5F45,/* 0x60-0x67 */ + 0x4E28,0x4EE1,0x4EFC,0x4F00,0x4F03,0x4F39,0x4F56,0x4F92,/* 0x68-0x6F */ + 0x4F8A,0x4F9A,0x4F94,0x4FCD,0x5040,0x5022,0x4FFF,0x501E,/* 0x70-0x77 */ + 0x5046,0x5070,0x5042,0x5094,0x50F4,0x50D8,0x514A,0x0000,/* 0x78-0x7F */ + + 0x5164,0x519D,0x51BE,0x51EC,0x5215,0x529C,0x52A6,0x52C0,/* 0x80-0x87 */ + 0x52DB,0x5300,0x5307,0x5324,0x5372,0x5393,0x53B2,0x53DD,/* 0x88-0x8F */ + 0xFA0E,0x549C,0x548A,0x54A9,0x54FF,0x5586,0x5759,0x5765,/* 0x90-0x97 */ + 0x57AC,0x57C8,0x57C7,0xFA0F,0xFA10,0x589E,0x58B2,0x590B,/* 0x98-0x9F */ + 0x5953,0x595B,0x595D,0x5963,0x59A4,0x59BA,0x5B56,0x5BC0,/* 0xA0-0xA7 */ + 0x752F,0x5BD8,0x5BEC,0x5C1E,0x5CA6,0x5CBA,0x5CF5,0x5D27,/* 0xA8-0xAF */ + 0x5D53,0xFA11,0x5D42,0x5D6D,0x5DB8,0x5DB9,0x5DD0,0x5F21,/* 0xB0-0xB7 */ + 0x5F34,0x5F67,0x5FB7,0x5FDE,0x605D,0x6085,0x608A,0x60DE,/* 0xB8-0xBF */ + 0x60D5,0x6120,0x60F2,0x6111,0x6137,0x6130,0x6198,0x6213,/* 0xC0-0xC7 */ + 0x62A6,0x63F5,0x6460,0x649D,0x64CE,0x654E,0x6600,0x6615,/* 0xC8-0xCF */ + 0x663B,0x6609,0x662E,0x661E,0x6624,0x6665,0x6657,0x6659,/* 0xD0-0xD7 */ + 0xFA12,0x6673,0x6699,0x66A0,0x66B2,0x66BF,0x66FA,0x670E,/* 0xD8-0xDF */ + 0xF929,0x6766,0x67BB,0x6852,0x67C0,0x6801,0x6844,0x68CF,/* 0xE0-0xE7 */ + 0xFA13,0x6968,0xFA14,0x6998,0x69E2,0x6A30,0x6A6B,0x6A46,/* 0xE8-0xEF */ + 0x6A73,0x6A7E,0x6AE2,0x6AE4,0x6BD6,0x6C3F,0x6C5C,0x6C86,/* 0xF0-0xF7 */ + 0x6C6F,0x6CDA,0x6D04,0x6D87,0x6D6F,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_FB[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6D96,0x6DAC,0x6DCF,0x6DF8,0x6DF2,0x6DFC,0x6E39,0x6E5C,/* 0x40-0x47 */ + 0x6E27,0x6E3C,0x6EBF,0x6F88,0x6FB5,0x6FF5,0x7005,0x7007,/* 0x48-0x4F */ + 0x7028,0x7085,0x70AB,0x710F,0x7104,0x715C,0x7146,0x7147,/* 0x50-0x57 */ + 0xFA15,0x71C1,0x71FE,0x72B1,0x72BE,0x7324,0xFA16,0x7377,/* 0x58-0x5F */ + 0x73BD,0x73C9,0x73D6,0x73E3,0x73D2,0x7407,0x73F5,0x7426,/* 0x60-0x67 */ + 0x742A,0x7429,0x742E,0x7462,0x7489,0x749F,0x7501,0x756F,/* 0x68-0x6F */ + 0x7682,0x769C,0x769E,0x769B,0x76A6,0xFA17,0x7746,0x52AF,/* 0x70-0x77 */ + 0x7821,0x784E,0x7864,0x787A,0x7930,0xFA18,0xFA19,0x0000,/* 0x78-0x7F */ + + 0xFA1A,0x7994,0xFA1B,0x799B,0x7AD1,0x7AE7,0xFA1C,0x7AEB,/* 0x80-0x87 */ + 0x7B9E,0xFA1D,0x7D48,0x7D5C,0x7DB7,0x7DA0,0x7DD6,0x7E52,/* 0x88-0x8F */ + 0x7F47,0x7FA1,0xFA1E,0x8301,0x8362,0x837F,0x83C7,0x83F6,/* 0x90-0x97 */ + 0x8448,0x84B4,0x8553,0x8559,0x856B,0xFA1F,0x85B0,0xFA20,/* 0x98-0x9F */ + 0xFA21,0x8807,0x88F5,0x8A12,0x8A37,0x8A79,0x8AA7,0x8ABE,/* 0xA0-0xA7 */ + 0x8ADF,0xFA22,0x8AF6,0x8B53,0x8B7F,0x8CF0,0x8CF4,0x8D12,/* 0xA8-0xAF */ + 0x8D76,0xFA23,0x8ECF,0xFA24,0xFA25,0x9067,0x90DE,0xFA26,/* 0xB0-0xB7 */ + 0x9115,0x9127,0x91DA,0x91D7,0x91DE,0x91ED,0x91EE,0x91E4,/* 0xB8-0xBF */ + 0x91E5,0x9206,0x9210,0x920A,0x923A,0x9240,0x923C,0x924E,/* 0xC0-0xC7 */ + 0x9259,0x9251,0x9239,0x9267,0x92A7,0x9277,0x9278,0x92E7,/* 0xC8-0xCF */ + 0x92D7,0x92D9,0x92D0,0xFA27,0x92D5,0x92E0,0x92D3,0x9325,/* 0xD0-0xD7 */ + 0x9321,0x92FB,0xFA28,0x931E,0x92FF,0x931D,0x9302,0x9370,/* 0xD8-0xDF */ + 0x9357,0x93A4,0x93C6,0x93DE,0x93F8,0x9431,0x9445,0x9448,/* 0xE0-0xE7 */ + 0x9592,0xF9DC,0xFA29,0x969D,0x96AF,0x9733,0x973B,0x9743,/* 0xE8-0xEF */ + 0x974D,0x974F,0x9751,0x9755,0x9857,0x9865,0xFA2A,0xFA2B,/* 0xF0-0xF7 */ + 0x9927,0xFA2C,0x999E,0x9A4E,0x9AD9,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_FC[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9ADC,0x9B75,0x9B72,0x9B8F,0x9BB1,0x9BBB,0x9C00,0x9D70,/* 0x40-0x47 */ + 0x9D6B,0xFA2D,0x9E19,0x9ED1,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ +}; + +static const wchar_t *page_charset2uni[256] = { + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, c2u_81, c2u_82, c2u_83, c2u_84, NULL, NULL, c2u_87, + c2u_88, c2u_89, c2u_8A, c2u_8B, c2u_8C, c2u_8D, c2u_8E, c2u_8F, + c2u_90, c2u_91, c2u_92, c2u_93, c2u_94, c2u_95, c2u_96, c2u_97, + c2u_98, c2u_99, c2u_9A, c2u_9B, c2u_9C, c2u_9D, c2u_9E, c2u_9F, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + c2u_E0, c2u_E1, c2u_E2, c2u_E3, c2u_E4, c2u_E5, c2u_E6, c2u_E7, + c2u_E8, c2u_E9, c2u_EA, NULL, NULL, c2u_ED, c2u_EE, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, c2u_FA, c2u_FB, c2u_FC, NULL, NULL, NULL, +}; + +static const unsigned char u2c_00hi[256 - 0xA0][2] = { + {0x00, 0x00}, {0x00, 0x00}, {0x81, 0x91}, {0x81, 0x92},/* 0xA0-0xA3 */ + {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00}, {0x81, 0x98},/* 0xA4-0xA7 */ + {0x81, 0x4E}, {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00},/* 0xA8-0xAB */ + {0x81, 0xCA}, {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00},/* 0xAC-0xAF */ + {0x81, 0x8B}, {0x81, 0x7D}, {0x00, 0x00}, {0x00, 0x00},/* 0xB0-0xB3 */ + {0x81, 0x4C}, {0x00, 0x00}, {0x81, 0xF7}, {0x00, 0x00},/* 0xB4-0xB7 */ + {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00},/* 0xB8-0xBB */ + {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00},/* 0xBC-0xBF */ + {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00},/* 0xC0-0xC3 */ + {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00},/* 0xC4-0xC7 */ + {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00},/* 0xC8-0xCB */ + {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00},/* 0xCC-0xCF */ + {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00},/* 0xD0-0xD3 */ + {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00}, {0x81, 0x7E},/* 0xD4-0xD7 */ + {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00},/* 0xD8-0xDB */ + {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00},/* 0xDC-0xDF */ + {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00},/* 0xE0-0xE3 */ + {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00},/* 0xE4-0xE7 */ + {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00},/* 0xE8-0xEB */ + {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00},/* 0xEC-0xEF */ + {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00},/* 0xF0-0xF3 */ + {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00}, {0x81, 0x80},/* 0xF4-0xF7 */ + {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00},/* 0xF8-0xFB */ + {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00}, {0x00, 0x00},/* 0xFC-0xFF */ +}; + +static const unsigned char u2c_03[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x83, 0x9F, 0x83, 0xA0, 0x83, 0xA1, /* 0x90-0x93 */ + 0x83, 0xA2, 0x83, 0xA3, 0x83, 0xA4, 0x83, 0xA5, /* 0x94-0x97 */ + 0x83, 0xA6, 0x83, 0xA7, 0x83, 0xA8, 0x83, 0xA9, /* 0x98-0x9B */ + 0x83, 0xAA, 0x83, 0xAB, 0x83, 0xAC, 0x83, 0xAD, /* 0x9C-0x9F */ + 0x83, 0xAE, 0x83, 0xAF, 0x00, 0x00, 0x83, 0xB0, /* 0xA0-0xA3 */ + 0x83, 0xB1, 0x83, 0xB2, 0x83, 0xB3, 0x83, 0xB4, /* 0xA4-0xA7 */ + 0x83, 0xB5, 0x83, 0xB6, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x83, 0xBF, 0x83, 0xC0, 0x83, 0xC1, /* 0xB0-0xB3 */ + 0x83, 0xC2, 0x83, 0xC3, 0x83, 0xC4, 0x83, 0xC5, /* 0xB4-0xB7 */ + 0x83, 0xC6, 0x83, 0xC7, 0x83, 0xC8, 0x83, 0xC9, /* 0xB8-0xBB */ + 0x83, 0xCA, 0x83, 0xCB, 0x83, 0xCC, 0x83, 0xCD, /* 0xBC-0xBF */ + 0x83, 0xCE, 0x83, 0xCF, 0x00, 0x00, 0x83, 0xD0, /* 0xC0-0xC3 */ + 0x83, 0xD1, 0x83, 0xD2, 0x83, 0xD3, 0x83, 0xD4, /* 0xC4-0xC7 */ + 0x83, 0xD5, 0x83, 0xD6, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ +}; + +static const unsigned char u2c_04[512] = { + 0x00, 0x00, 0x84, 0x46, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x84, 0x40, 0x84, 0x41, 0x84, 0x42, 0x84, 0x43, /* 0x10-0x13 */ + 0x84, 0x44, 0x84, 0x45, 0x84, 0x47, 0x84, 0x48, /* 0x14-0x17 */ + 0x84, 0x49, 0x84, 0x4A, 0x84, 0x4B, 0x84, 0x4C, /* 0x18-0x1B */ + 0x84, 0x4D, 0x84, 0x4E, 0x84, 0x4F, 0x84, 0x50, /* 0x1C-0x1F */ + 0x84, 0x51, 0x84, 0x52, 0x84, 0x53, 0x84, 0x54, /* 0x20-0x23 */ + 0x84, 0x55, 0x84, 0x56, 0x84, 0x57, 0x84, 0x58, /* 0x24-0x27 */ + 0x84, 0x59, 0x84, 0x5A, 0x84, 0x5B, 0x84, 0x5C, /* 0x28-0x2B */ + 0x84, 0x5D, 0x84, 0x5E, 0x84, 0x5F, 0x84, 0x60, /* 0x2C-0x2F */ + 0x84, 0x70, 0x84, 0x71, 0x84, 0x72, 0x84, 0x73, /* 0x30-0x33 */ + 0x84, 0x74, 0x84, 0x75, 0x84, 0x77, 0x84, 0x78, /* 0x34-0x37 */ + 0x84, 0x79, 0x84, 0x7A, 0x84, 0x7B, 0x84, 0x7C, /* 0x38-0x3B */ + 0x84, 0x7D, 0x84, 0x7E, 0x84, 0x80, 0x84, 0x81, /* 0x3C-0x3F */ + 0x84, 0x82, 0x84, 0x83, 0x84, 0x84, 0x84, 0x85, /* 0x40-0x43 */ + 0x84, 0x86, 0x84, 0x87, 0x84, 0x88, 0x84, 0x89, /* 0x44-0x47 */ + 0x84, 0x8A, 0x84, 0x8B, 0x84, 0x8C, 0x84, 0x8D, /* 0x48-0x4B */ + 0x84, 0x8E, 0x84, 0x8F, 0x84, 0x90, 0x84, 0x91, /* 0x4C-0x4F */ + 0x00, 0x00, 0x84, 0x76, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ +}; + +static const unsigned char u2c_20[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x81, 0x5D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x81, 0x5C, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x81, 0x65, 0x81, 0x66, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x81, 0x67, 0x81, 0x68, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x81, 0xF5, 0x81, 0xF6, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x81, 0x64, 0x81, 0x63, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x81, 0xF1, 0x00, 0x00, 0x81, 0x8C, 0x81, 0x8D, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x81, 0xA6, /* 0x38-0x3B */ +}; + +static const unsigned char u2c_21[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x81, 0x8E, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xFA, 0x59, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0xFA, 0x5A, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x81, 0xF0, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0xFA, 0x4A, 0xFA, 0x4B, 0xFA, 0x4C, 0xFA, 0x4D, /* 0x60-0x63 */ + 0xFA, 0x4E, 0xFA, 0x4F, 0xFA, 0x50, 0xFA, 0x51, /* 0x64-0x67 */ + 0xFA, 0x52, 0xFA, 0x53, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0xEE, 0xEF, 0xEE, 0xF0, 0xEE, 0xF1, 0xEE, 0xF2, /* 0x70-0x73 */ + 0xEE, 0xF3, 0xEE, 0xF4, 0xEE, 0xF5, 0xEE, 0xF6, /* 0x74-0x77 */ + 0xEE, 0xF7, 0xEE, 0xF8, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x81, 0xA9, 0x81, 0xAA, 0x81, 0xA8, 0x81, 0xAB, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x81, 0xCB, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x81, 0xCC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ +}; + +static const unsigned char u2c_22[512] = { + 0x81, 0xCD, 0x00, 0x00, 0x81, 0xDD, 0x81, 0xCE, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x81, 0xDE, /* 0x04-0x07 */ + 0x81, 0xB8, 0x00, 0x00, 0x00, 0x00, 0x81, 0xB9, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x87, 0x94, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x87, 0x95, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x81, 0xE5, 0x81, 0x87, 0x87, 0x98, /* 0x1C-0x1F */ + 0x87, 0x97, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x81, 0x61, 0x00, 0x00, 0x81, 0xC8, /* 0x24-0x27 */ + 0x81, 0xC9, 0x87, 0x9B, 0x87, 0x9C, 0x87, 0x92, /* 0x28-0x2B */ + 0x81, 0xE8, 0x00, 0x00, 0x87, 0x93, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x81, 0x88, 0xFA, 0x5B, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x81, 0xE4, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x87, 0x90, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x81, 0x82, 0x87, 0x91, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x81, 0x85, 0x81, 0x86, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x81, 0xE1, 0x81, 0xE2, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x81, 0xBC, 0x81, 0xBD, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x81, 0xBA, 0x81, 0xBB, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x87, 0x96, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x87, 0x99, /* 0xBC-0xBF */ +}; + +static const unsigned char u2c_23[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x81, 0xDC, 0x00, 0x00, /* 0x10-0x13 */ +}; + +static const unsigned char u2c_24[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x87, 0x40, 0x87, 0x41, 0x87, 0x42, 0x87, 0x43, /* 0x60-0x63 */ + 0x87, 0x44, 0x87, 0x45, 0x87, 0x46, 0x87, 0x47, /* 0x64-0x67 */ + 0x87, 0x48, 0x87, 0x49, 0x87, 0x4A, 0x87, 0x4B, /* 0x68-0x6B */ + 0x87, 0x4C, 0x87, 0x4D, 0x87, 0x4E, 0x87, 0x4F, /* 0x6C-0x6F */ + 0x87, 0x50, 0x87, 0x51, 0x87, 0x52, 0x87, 0x53, /* 0x70-0x73 */ +}; + +static const unsigned char u2c_25[512] = { + 0x84, 0x9F, 0x84, 0xAA, 0x84, 0xA0, 0x84, 0xAB, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x84, 0xA1, 0x00, 0x00, 0x00, 0x00, 0x84, 0xAC, /* 0x0C-0x0F */ + 0x84, 0xA2, 0x00, 0x00, 0x00, 0x00, 0x84, 0xAD, /* 0x10-0x13 */ + 0x84, 0xA4, 0x00, 0x00, 0x00, 0x00, 0x84, 0xAF, /* 0x14-0x17 */ + 0x84, 0xA3, 0x00, 0x00, 0x00, 0x00, 0x84, 0xAE, /* 0x18-0x1B */ + 0x84, 0xA5, 0x84, 0xBA, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x84, 0xB5, 0x00, 0x00, 0x00, 0x00, 0x84, 0xB0, /* 0x20-0x23 */ + 0x84, 0xA7, 0x84, 0xBC, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x84, 0xB7, 0x00, 0x00, 0x00, 0x00, 0x84, 0xB2, /* 0x28-0x2B */ + 0x84, 0xA6, 0x00, 0x00, 0x00, 0x00, 0x84, 0xB6, /* 0x2C-0x2F */ + 0x84, 0xBB, 0x00, 0x00, 0x00, 0x00, 0x84, 0xB1, /* 0x30-0x33 */ + 0x84, 0xA8, 0x00, 0x00, 0x00, 0x00, 0x84, 0xB8, /* 0x34-0x37 */ + 0x84, 0xBD, 0x00, 0x00, 0x00, 0x00, 0x84, 0xB3, /* 0x38-0x3B */ + 0x84, 0xA9, 0x00, 0x00, 0x00, 0x00, 0x84, 0xB9, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x84, 0xBE, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x84, 0xB4, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x81, 0xA1, 0x81, 0xA0, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x81, 0xA3, 0x81, 0xA2, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x81, 0xA5, 0x81, 0xA4, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x81, 0x9F, 0x81, 0x9E, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x81, 0x9B, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x81, 0x9D, 0x81, 0x9C, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x81, 0xFC, /* 0xEC-0xEF */ +}; + +static const unsigned char u2c_26[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x81, 0x9A, 0x81, 0x99, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x81, 0x8A, 0x00, 0x00, 0x81, 0x89, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x81, 0xF4, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x81, 0xF3, 0x00, 0x00, 0x81, 0xF2, /* 0x6C-0x6F */ +}; + +static const unsigned char u2c_30[512] = { + 0x81, 0x40, 0x81, 0x41, 0x81, 0x42, 0x81, 0x56, /* 0x00-0x03 */ + 0x00, 0x00, 0x81, 0x58, 0x81, 0x59, 0x81, 0x5A, /* 0x04-0x07 */ + 0x81, 0x71, 0x81, 0x72, 0x81, 0x73, 0x81, 0x74, /* 0x08-0x0B */ + 0x81, 0x75, 0x81, 0x76, 0x81, 0x77, 0x81, 0x78, /* 0x0C-0x0F */ + 0x81, 0x79, 0x81, 0x7A, 0x81, 0xA7, 0x81, 0xAC, /* 0x10-0x13 */ + 0x81, 0x6B, 0x81, 0x6C, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x87, 0x80, 0x00, 0x00, 0x87, 0x81, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x82, 0x9F, 0x82, 0xA0, 0x82, 0xA1, /* 0x40-0x43 */ + 0x82, 0xA2, 0x82, 0xA3, 0x82, 0xA4, 0x82, 0xA5, /* 0x44-0x47 */ + 0x82, 0xA6, 0x82, 0xA7, 0x82, 0xA8, 0x82, 0xA9, /* 0x48-0x4B */ + 0x82, 0xAA, 0x82, 0xAB, 0x82, 0xAC, 0x82, 0xAD, /* 0x4C-0x4F */ + 0x82, 0xAE, 0x82, 0xAF, 0x82, 0xB0, 0x82, 0xB1, /* 0x50-0x53 */ + 0x82, 0xB2, 0x82, 0xB3, 0x82, 0xB4, 0x82, 0xB5, /* 0x54-0x57 */ + 0x82, 0xB6, 0x82, 0xB7, 0x82, 0xB8, 0x82, 0xB9, /* 0x58-0x5B */ + 0x82, 0xBA, 0x82, 0xBB, 0x82, 0xBC, 0x82, 0xBD, /* 0x5C-0x5F */ + 0x82, 0xBE, 0x82, 0xBF, 0x82, 0xC0, 0x82, 0xC1, /* 0x60-0x63 */ + 0x82, 0xC2, 0x82, 0xC3, 0x82, 0xC4, 0x82, 0xC5, /* 0x64-0x67 */ + 0x82, 0xC6, 0x82, 0xC7, 0x82, 0xC8, 0x82, 0xC9, /* 0x68-0x6B */ + 0x82, 0xCA, 0x82, 0xCB, 0x82, 0xCC, 0x82, 0xCD, /* 0x6C-0x6F */ + 0x82, 0xCE, 0x82, 0xCF, 0x82, 0xD0, 0x82, 0xD1, /* 0x70-0x73 */ + 0x82, 0xD2, 0x82, 0xD3, 0x82, 0xD4, 0x82, 0xD5, /* 0x74-0x77 */ + 0x82, 0xD6, 0x82, 0xD7, 0x82, 0xD8, 0x82, 0xD9, /* 0x78-0x7B */ + 0x82, 0xDA, 0x82, 0xDB, 0x82, 0xDC, 0x82, 0xDD, /* 0x7C-0x7F */ + + 0x82, 0xDE, 0x82, 0xDF, 0x82, 0xE0, 0x82, 0xE1, /* 0x80-0x83 */ + 0x82, 0xE2, 0x82, 0xE3, 0x82, 0xE4, 0x82, 0xE5, /* 0x84-0x87 */ + 0x82, 0xE6, 0x82, 0xE7, 0x82, 0xE8, 0x82, 0xE9, /* 0x88-0x8B */ + 0x82, 0xEA, 0x82, 0xEB, 0x82, 0xEC, 0x82, 0xED, /* 0x8C-0x8F */ + 0x82, 0xEE, 0x82, 0xEF, 0x82, 0xF0, 0x82, 0xF1, /* 0x90-0x93 */ + 0x83, 0x94, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x81, 0x4A, /* 0x98-0x9B */ + 0x81, 0x4B, 0x81, 0x54, 0x81, 0x55, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x83, 0x40, 0x83, 0x41, 0x83, 0x42, /* 0xA0-0xA3 */ + 0x83, 0x43, 0x83, 0x44, 0x83, 0x45, 0x83, 0x46, /* 0xA4-0xA7 */ + 0x83, 0x47, 0x83, 0x48, 0x83, 0x49, 0x83, 0x4A, /* 0xA8-0xAB */ + 0x83, 0x4B, 0x83, 0x4C, 0x83, 0x4D, 0x83, 0x4E, /* 0xAC-0xAF */ + 0x83, 0x4F, 0x83, 0x50, 0x83, 0x51, 0x83, 0x52, /* 0xB0-0xB3 */ + 0x83, 0x53, 0x83, 0x54, 0x83, 0x55, 0x83, 0x56, /* 0xB4-0xB7 */ + 0x83, 0x57, 0x83, 0x58, 0x83, 0x59, 0x83, 0x5A, /* 0xB8-0xBB */ + 0x83, 0x5B, 0x83, 0x5C, 0x83, 0x5D, 0x83, 0x5E, /* 0xBC-0xBF */ + 0x83, 0x5F, 0x83, 0x60, 0x83, 0x61, 0x83, 0x62, /* 0xC0-0xC3 */ + 0x83, 0x63, 0x83, 0x64, 0x83, 0x65, 0x83, 0x66, /* 0xC4-0xC7 */ + 0x83, 0x67, 0x83, 0x68, 0x83, 0x69, 0x83, 0x6A, /* 0xC8-0xCB */ + 0x83, 0x6B, 0x83, 0x6C, 0x83, 0x6D, 0x83, 0x6E, /* 0xCC-0xCF */ + 0x83, 0x6F, 0x83, 0x70, 0x83, 0x71, 0x83, 0x72, /* 0xD0-0xD3 */ + 0x83, 0x73, 0x83, 0x74, 0x83, 0x75, 0x83, 0x76, /* 0xD4-0xD7 */ + 0x83, 0x77, 0x83, 0x78, 0x83, 0x79, 0x83, 0x7A, /* 0xD8-0xDB */ + 0x83, 0x7B, 0x83, 0x7C, 0x83, 0x7D, 0x83, 0x7E, /* 0xDC-0xDF */ + 0x83, 0x80, 0x83, 0x81, 0x83, 0x82, 0x83, 0x83, /* 0xE0-0xE3 */ + 0x83, 0x84, 0x83, 0x85, 0x83, 0x86, 0x83, 0x87, /* 0xE4-0xE7 */ + 0x83, 0x88, 0x83, 0x89, 0x83, 0x8A, 0x83, 0x8B, /* 0xE8-0xEB */ + 0x83, 0x8C, 0x83, 0x8D, 0x83, 0x8E, 0x83, 0x8F, /* 0xEC-0xEF */ + 0x83, 0x90, 0x83, 0x91, 0x83, 0x92, 0x83, 0x93, /* 0xF0-0xF3 */ + 0x83, 0x94, 0x83, 0x95, 0x83, 0x96, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x81, 0x45, /* 0xF8-0xFB */ + 0x81, 0x5B, 0x81, 0x52, 0x81, 0x53, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_32[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0xFA, 0x58, 0x87, 0x8B, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x87, 0x8C, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x87, 0x85, 0x87, 0x86, 0x87, 0x87, 0x87, 0x88, /* 0xA4-0xA7 */ + 0x87, 0x89, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ +}; + +static const unsigned char u2c_33[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x87, 0x65, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x87, 0x69, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x87, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x87, 0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x87, 0x61, 0x87, 0x6B, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x87, 0x6A, 0x87, 0x64, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x87, 0x6C, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x87, 0x66, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x87, 0x6E, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x87, 0x5F, 0x87, 0x6D, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x87, 0x62, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x87, 0x67, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x87, 0x68, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x87, 0x7E, /* 0x78-0x7B */ + 0x87, 0x8F, 0x87, 0x8E, 0x87, 0x8D, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x87, 0x72, 0x87, 0x73, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x87, 0x6F, 0x87, 0x70, 0x87, 0x71, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x87, 0x75, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x87, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x87, 0x83, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ +}; + +static const unsigned char u2c_4E[512] = { + 0x88, 0xEA, 0x92, 0x9A, 0x00, 0x00, 0x8E, 0xB5, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x96, 0x9C, /* 0x04-0x07 */ + 0x8F, 0xE4, 0x8E, 0x4F, 0x8F, 0xE3, 0x89, 0xBA, /* 0x08-0x0B */ + 0x00, 0x00, 0x95, 0x73, 0x97, 0x5E, 0x00, 0x00, /* 0x0C-0x0F */ + 0x98, 0xA0, 0x89, 0x4E, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x8A, 0x8E, 0x98, 0xA1, 0x90, 0xA2, 0x99, 0xC0, /* 0x14-0x17 */ + 0x8B, 0x75, 0x95, 0xB8, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x8F, 0xE5, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x97, 0xBC, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x95, 0xC0, 0x00, 0x00, /* 0x24-0x27 */ + 0xED, 0x4C, 0x00, 0x00, 0x98, 0xA2, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x92, 0x86, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x98, 0xA3, 0x8B, 0xF8, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x98, 0xA4, 0x00, 0x00, /* 0x34-0x37 */ + 0x8A, 0xDB, 0x92, 0x4F, 0x00, 0x00, 0x8E, 0xE5, /* 0x38-0x3B */ + 0x98, 0xA5, 0x00, 0x00, 0x00, 0x00, 0x98, 0xA6, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x98, 0xA7, 0x94, 0x54, /* 0x40-0x43 */ + 0x00, 0x00, 0x8B, 0x76, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x94, 0x56, /* 0x48-0x4B */ + 0x00, 0x00, 0x93, 0xE1, 0x8C, 0xC1, 0x96, 0x52, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0xE5, 0x68, 0x98, 0xA8, 0x8F, 0xE6, /* 0x54-0x57 */ + 0x98, 0xA9, 0x89, 0xB3, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x8B, 0xE3, 0x8C, 0xEE, 0x96, 0xE7, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x9B, 0xA4, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x97, 0x90, 0x00, 0x00, 0x93, 0xFB, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x8A, 0xA3, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x8B, 0x54, 0x00, 0x00, 0x98, 0xAA, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x98, 0xAB, 0x97, 0xB9, 0x00, 0x00, /* 0x84-0x87 */ + 0x97, 0x5C, 0x91, 0x88, 0x98, 0xAD, 0x8E, 0x96, /* 0x88-0x8B */ + 0x93, 0xF1, 0x00, 0x00, 0x98, 0xB0, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x89, 0x5D, 0x8C, 0xDD, 0x00, 0x00, /* 0x90-0x93 */ + 0x8C, 0xDC, 0x88, 0xE4, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x98, 0x6A, 0x98, 0x69, 0x00, 0x00, 0x8D, 0xB1, /* 0x98-0x9B */ + 0x88, 0x9F, 0x00, 0x00, 0x98, 0xB1, 0x98, 0xB2, /* 0x9C-0x9F */ + 0x98, 0xB3, 0x96, 0x53, 0x98, 0xB4, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x8C, 0xF0, 0x88, 0xE5, 0x96, 0x92, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x8B, 0x9C, 0x00, 0x00, 0x00, 0x00, 0x8B, 0x9D, /* 0xA8-0xAB */ + 0x8B, 0x9E, 0x92, 0xE0, 0x97, 0xBA, 0x00, 0x00, /* 0xAC-0xAF */ + 0x98, 0xB5, 0x00, 0x00, 0x00, 0x00, 0x98, 0xB6, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x98, 0xB7, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x90, 0x6C, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x8F, 0x59, 0x90, 0x6D, 0x98, 0xBC, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x98, 0xBA, 0x00, 0x00, 0x98, 0xBB, 0x8B, 0x77, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x8D, 0xA1, 0x89, 0xEE, /* 0xC8-0xCB */ + 0x00, 0x00, 0x98, 0xB9, 0x98, 0xB8, 0x95, 0xA7, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x8E, 0x65, 0x8E, 0x64, 0x91, 0xBC, 0x98, 0xBD, /* 0xD4-0xD7 */ + 0x95, 0x74, 0x90, 0xE5, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x81, 0x57, 0x98, 0xBE, 0x98, 0xC0, /* 0xDC-0xDF */ + 0x00, 0x00, 0xED, 0x4D, 0x00, 0x00, 0x91, 0xE3, /* 0xE0-0xE3 */ + 0x97, 0xDF, 0x88, 0xC8, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x98, 0xBF, 0x89, 0xBC, 0x00, 0x00, /* 0xEC-0xEF */ + 0x8B, 0xC2, 0x00, 0x00, 0x92, 0x87, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x8C, 0x8F, 0x98, 0xC1, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x94, 0x43, /* 0xF8-0xFB */ + 0xED, 0x4E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_4F[512] = { + 0xED, 0x4F, 0x8A, 0xE9, 0x00, 0x00, 0xED, 0x50, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x98, 0xC2, 0x88, 0xC9, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x8C, 0xDE, 0x8A, 0xEA, 0x95, 0x9A, /* 0x0C-0x0F */ + 0x94, 0xB0, 0x8B, 0x78, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0xEF, 0x00, 0x00, /* 0x18-0x1B */ + 0x98, 0xE5, 0x93, 0x60, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x94, 0x8C, /* 0x2C-0x2F */ + 0x98, 0xC4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x94, 0xBA, 0x00, 0x00, 0x97, 0xE0, 0x00, 0x00, /* 0x34-0x37 */ + 0x90, 0x4C, 0xED, 0x51, 0x8E, 0x66, 0x00, 0x00, /* 0x38-0x3B */ + 0x8E, 0x97, 0x89, 0xBE, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0xCF, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x92, 0x41, 0x98, 0xC8, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x88, 0xCA, 0x92, 0xE1, 0x8F, 0x5A, /* 0x4C-0x4F */ + 0x8D, 0xB2, 0x97, 0x43, 0x00, 0x00, 0x91, 0xCC, /* 0x50-0x53 */ + 0x00, 0x00, 0x89, 0xBD, 0xED, 0x52, 0x98, 0xC7, /* 0x54-0x57 */ + 0x00, 0x00, 0x97, 0x5D, 0x98, 0xC3, 0x98, 0xC5, /* 0x58-0x5B */ + 0x8D, 0xEC, 0x98, 0xC6, 0x9B, 0x43, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x98, 0xCE, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0xD1, /* 0x6C-0x6F */ + 0x98, 0xCF, 0x00, 0x00, 0x00, 0x00, 0x89, 0xC0, /* 0x70-0x73 */ + 0x00, 0x00, 0x95, 0xB9, 0x98, 0xC9, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0xCD, /* 0x78-0x7B */ + 0x8C, 0xF1, 0x00, 0x00, 0x00, 0x00, 0x8E, 0x67, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8A, 0xA4, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x98, 0xD2, 0x00, 0x00, /* 0x84-0x87 */ + 0x98, 0xCA, 0x00, 0x00, 0xED, 0x54, 0x97, 0xE1, /* 0x88-0x8B */ + 0x00, 0x00, 0x8E, 0x98, 0x00, 0x00, 0x98, 0xCB, /* 0x8C-0x8F */ + 0x00, 0x00, 0x98, 0xD0, 0xED, 0x53, 0x00, 0x00, /* 0x90-0x93 */ + 0xED, 0x56, 0x00, 0x00, 0x98, 0xD3, 0x00, 0x00, /* 0x94-0x97 */ + 0x98, 0xCC, 0x00, 0x00, 0xED, 0x55, 0x8B, 0x9F, /* 0x98-0x9B */ + 0x00, 0x00, 0x88, 0xCB, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x8B, 0xA0, 0x89, 0xBF, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9B, 0x44, /* 0xA8-0xAB */ + 0x00, 0x00, 0x96, 0x99, 0x95, 0x8E, 0x8C, 0xF2, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x90, 0x4E, 0x97, 0xB5, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0xD6, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x8C, 0x57, 0x91, 0xA3, /* 0xC0-0xC3 */ + 0x89, 0xE2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xED, 0x45, 0x8F, 0x72, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0xED, 0x57, 0x98, 0xD7, 0x00, 0x00, /* 0xCC-0xCF */ + 0x98, 0xDC, 0x98, 0xDA, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x98, 0xD5, 0x00, 0x00, 0x00, 0x00, 0x91, 0xAD, /* 0xD4-0xD7 */ + 0x98, 0xD8, 0x00, 0x00, 0x98, 0xDB, 0x98, 0xD9, /* 0xD8-0xDB */ + 0x00, 0x00, 0x95, 0xDB, 0x00, 0x00, 0x98, 0xD6, /* 0xDC-0xDF */ + 0x00, 0x00, 0x90, 0x4D, 0x00, 0x00, 0x96, 0x93, /* 0xE0-0xE3 */ + 0x98, 0xDD, 0x98, 0xDE, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x8F, 0x43, 0x98, 0xEB, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x94, 0x6F, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x95, 0x55, 0x98, 0xE6, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x95, 0xEE, 0x00, 0x00, 0x89, 0xB4, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0x98, 0xEA, 0xED, 0x5A, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_50[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x98, 0xE4, 0x98, 0xED, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x91, 0x71, 0x00, 0x00, 0x8C, 0xC2, /* 0x08-0x0B */ + 0x00, 0x00, 0x94, 0x7B, 0x00, 0x00, 0xE0, 0xC5, /* 0x0C-0x0F */ + 0x00, 0x00, 0x98, 0xEC, 0x93, 0x7C, 0x00, 0x00, /* 0x10-0x13 */ + 0x98, 0xE1, 0x00, 0x00, 0x8C, 0xF4, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x8C, 0xF3, 0x98, 0xDF, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xED, 0x5B, 0x8E, 0xD8, /* 0x1C-0x1F */ + 0x00, 0x00, 0x98, 0xE7, 0xED, 0x59, 0x95, 0xED, /* 0x20-0x23 */ + 0x92, 0x6C, 0x98, 0xE3, 0x8C, 0x91, 0x00, 0x00, /* 0x24-0x27 */ + 0x98, 0xE0, 0x98, 0xE8, 0x98, 0xE2, 0x97, 0xCF, /* 0x28-0x2B */ + 0x98, 0xE9, 0x98, 0x60, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x8B, 0xE4, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x8C, 0x90, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0xED, 0x58, 0x00, 0x00, 0xED, 0x5E, 0x98, 0xEE, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xED, 0x5C, 0x98, 0xEF, /* 0x44-0x47 */ + 0x98, 0xF3, 0x88, 0xCC, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0xCE, /* 0x4C-0x4F */ + 0x98, 0xF2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x98, 0xF1, 0x98, 0xF5, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x98, 0xF4, 0x00, 0x00, /* 0x58-0x5B */ + 0x92, 0xE2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x8C, 0x92, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x98, 0xF6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0xED, 0x5D, 0x00, 0x00, 0x8E, 0xC3, 0x00, 0x00, /* 0x70-0x73 */ + 0x91, 0xA4, 0x92, 0xE3, 0x8B, 0xF4, 0x00, 0x00, /* 0x74-0x77 */ + 0x98, 0xF7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x8B, 0x55, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x98, 0xF8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x98, 0xFA, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x96, 0x54, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x8C, 0x86, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0xED, 0x5F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x8E, 0x50, 0x94, 0xF5, 0x98, 0xF9, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x8D, 0xC3, 0x97, 0x62, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x98, 0xFC, 0x99, 0x42, /* 0xB0-0xB3 */ + 0x98, 0xFB, 0x8D, 0xC2, 0x00, 0x00, 0x8F, 0x9D, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x8C, 0x58, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x99, 0x43, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x8B, 0xCD, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x99, 0x40, 0x99, 0x41, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x93, 0xAD, 0x00, 0x00, 0x91, 0x9C, /* 0xCC-0xCF */ + 0x00, 0x00, 0x8B, 0xA1, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x96, 0x6C, 0x99, 0x44, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xED, 0x61, 0x00, 0x00, 0x97, 0xBB, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x99, 0x45, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x99, 0x48, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x99, 0x46, 0x00, 0x00, 0x91, 0x6D, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x99, 0x47, 0x99, 0x49, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xED, 0x60, 0x99, 0x4B, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x99, 0x4A, 0x00, 0x00, 0x95, 0xC6, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_51[512] = { + 0x8B, 0x56, 0x99, 0x4D, 0x99, 0x4E, 0x00, 0x00, /* 0x00-0x03 */ + 0x89, 0xAD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x99, 0x4C, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x8E, 0xF2, 0x00, 0x00, /* 0x10-0x13 */ + 0x99, 0x51, 0x99, 0x50, 0x99, 0x4F, 0x00, 0x00, /* 0x14-0x17 */ + 0x98, 0xD4, 0x00, 0x00, 0x99, 0x52, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8F, 0x9E, /* 0x1C-0x1F */ + 0x00, 0x00, 0x99, 0x53, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x97, 0x44, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x96, 0xD7, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x99, 0x55, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x99, 0x54, 0x99, 0x57, /* 0x38-0x3B */ + 0x99, 0x56, 0x00, 0x00, 0x00, 0x00, 0x99, 0x58, /* 0x3C-0x3F */ + 0x99, 0x59, 0x88, 0xF2, 0x00, 0x00, 0x8C, 0xB3, /* 0x40-0x43 */ + 0x8C, 0x5A, 0x8F, 0x5B, 0x92, 0x9B, 0x8B, 0xA2, /* 0x44-0x47 */ + 0x90, 0xE6, 0x8C, 0xF5, 0xED, 0x62, 0x8D, 0x8E, /* 0x48-0x4B */ + 0x99, 0x5B, 0x96, 0xC6, 0x93, 0x65, 0x00, 0x00, /* 0x4C-0x4F */ + 0x8E, 0x99, 0x00, 0x00, 0x99, 0x5A, 0x00, 0x00, /* 0x50-0x53 */ + 0x99, 0x5C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x93, 0x7D, 0x00, 0x00, /* 0x58-0x5B */ + 0x8A, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x99, 0x5D, 0x00, 0x00, /* 0x60-0x63 */ + 0xED, 0x63, 0x93, 0xFC, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x91, 0x53, 0x99, 0x5F, 0x99, 0x60, 0x94, 0xAA, /* 0x68-0x6B */ + 0x8C, 0xF6, 0x98, 0x5A, 0x99, 0x61, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x8B, 0xA4, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x95, 0xBA, 0x91, 0xB4, 0x8B, 0xEF, /* 0x74-0x77 */ + 0x93, 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x8C, 0x93, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x99, 0x62, 0x00, 0x00, 0x99, 0x63, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x93, 0xE0, 0x89, 0x7E, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x99, 0x66, 0x8D, 0xFB, 0x00, 0x00, /* 0x88-0x8B */ + 0x99, 0x65, 0x8D, 0xC4, 0x00, 0x00, 0x99, 0x67, /* 0x8C-0x8F */ + 0xE3, 0xEC, 0x99, 0x68, 0x96, 0x60, 0x99, 0x69, /* 0x90-0x93 */ + 0x00, 0x00, 0x99, 0x6A, 0x99, 0x6B, 0x8F, 0xE7, /* 0x94-0x97 */ + 0x00, 0x00, 0x8E, 0xCA, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0xED, 0x64, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x8A, 0xA5, 0x00, 0x00, 0x99, 0x6E, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x99, 0x6C, 0x96, 0xBB, 0x99, 0x6D, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x95, 0x79, 0x99, 0x6F, 0x99, 0x70, 0x99, 0x71, /* 0xA8-0xAB */ + 0x93, 0x7E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x99, 0x75, 0x99, 0x73, 0x99, 0x74, 0x99, 0x72, /* 0xB0-0xB3 */ + 0x8D, 0xE1, 0x99, 0x76, 0x96, 0xE8, 0x97, 0xE2, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x99, 0x77, 0xED, 0x65, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x90, 0xA6, 0x99, 0x78, 0x8F, 0x79, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x99, 0x79, 0x00, 0x00, 0x92, 0x9C, /* 0xC8-0xCB */ + 0x97, 0xBD, 0x93, 0x80, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x99, 0xC3, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x99, 0x7A, /* 0xD8-0xDB */ + 0xEA, 0xA3, 0x8B, 0xC3, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x99, 0x7B, 0x96, 0x7D, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x8F, 0x88, 0x91, 0xFA, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x99, 0x7D, 0x93, 0xE2, 0x00, 0x00, /* 0xE8-0xEB */ + 0xED, 0x66, 0x99, 0x7E, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x99, 0x80, 0x8A, 0x4D, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x99, 0x81, 0x8B, 0xA5, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x93, 0xCA, 0x89, 0x9A, 0x8F, 0x6F, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x94, 0x9F, 0x99, 0x82, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_52[512] = { + 0x93, 0x81, 0x00, 0x00, 0x00, 0x00, 0x90, 0x6E, /* 0x00-0x03 */ + 0x99, 0x83, 0x00, 0x00, 0x95, 0xAA, 0x90, 0xD8, /* 0x04-0x07 */ + 0x8A, 0xA0, 0x00, 0x00, 0x8A, 0xA7, 0x99, 0x84, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x99, 0x86, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x8C, 0x59, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x99, 0x85, 0xED, 0x67, 0x00, 0x00, 0x97, 0xF1, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x8F, 0x89, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x94, 0xBB, 0x95, 0xCA, 0x00, 0x00, 0x99, 0x87, /* 0x24-0x27 */ + 0x00, 0x00, 0x97, 0x98, 0x99, 0x88, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x99, 0x89, 0x00, 0x00, /* 0x2C-0x2F */ + 0x93, 0x9E, 0x00, 0x00, 0x00, 0x00, 0x99, 0x8A, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x90, 0xA7, 0x8D, 0xFC, /* 0x34-0x37 */ + 0x8C, 0x94, 0x99, 0x8B, 0x8E, 0x68, 0x8D, 0x8F, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0xE4, /* 0x40-0x43 */ + 0x99, 0x8D, 0x00, 0x00, 0x00, 0x00, 0x91, 0xA5, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x8D, 0xED, 0x99, 0x8E, /* 0x48-0x4B */ + 0x99, 0x8F, 0x91, 0x4F, 0x00, 0x00, 0x99, 0x8C, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x99, 0x91, 0x00, 0x00, 0x96, 0x55, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8D, 0x84, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x99, 0x90, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8C, 0x95, /* 0x60-0x63 */ + 0x8D, 0xDC, 0x94, 0x8D, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x99, 0x94, 0x99, 0x92, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0x9B, /* 0x6C-0x6F */ + 0x8F, 0xE8, 0x99, 0x9B, 0x8A, 0x84, 0x99, 0x95, /* 0x70-0x73 */ + 0x99, 0x93, 0x91, 0x6E, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x99, 0x97, 0x00, 0x00, 0x99, 0x96, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8A, 0x63, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8C, 0x80, /* 0x84-0x87 */ + 0x99, 0x9C, 0x97, 0xAB, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x99, 0x98, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x99, 0x9D, 0x99, 0x9A, 0x00, 0x00, /* 0x90-0x93 */ + 0x99, 0x99, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x97, 0xCD, /* 0x98-0x9B */ + 0xED, 0x68, 0x00, 0x00, 0x00, 0x00, 0x8C, 0xF7, /* 0x9C-0x9F */ + 0x89, 0xC1, 0x00, 0x00, 0x00, 0x00, 0x97, 0xF2, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0xED, 0x69, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x8F, 0x95, 0x93, 0x77, 0x8D, 0x85, /* 0xA8-0xAB */ + 0x99, 0xA0, 0x99, 0xA1, 0x00, 0x00, 0xEE, 0x5B, /* 0xAC-0xAF */ + 0x00, 0x00, 0x97, 0xE3, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x98, 0x4A, 0x99, 0xA3, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x8C, 0xF8, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x99, 0xA2, 0x00, 0x00, 0x8A, 0x4E, 0x00, 0x00, /* 0xBC-0xBF */ + 0xED, 0x6A, 0x99, 0xA4, 0x00, 0x00, 0x96, 0x75, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x92, 0xBA, 0x00, 0x00, 0x97, 0x45, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x95, 0xD7, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x99, 0xA5, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xE8, 0xD3, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x93, 0xAE, 0x00, 0x00, 0x99, 0xA6, /* 0xD4-0xD7 */ + 0x8A, 0xA8, 0x96, 0xB1, 0x00, 0x00, 0xED, 0x6B, /* 0xD8-0xDB */ + 0x00, 0x00, 0x8F, 0x9F, 0x99, 0xA7, 0x95, 0xE5, /* 0xDC-0xDF */ + 0x99, 0xAB, 0x00, 0x00, 0x90, 0xA8, 0x99, 0xA8, /* 0xE0-0xE3 */ + 0x8B, 0xCE, 0x00, 0x00, 0x99, 0xA9, 0x8A, 0xA9, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x8C, 0x4D, 0x99, 0xAC, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x99, 0xAD, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x99, 0xAE, 0x99, 0xAF, 0x8E, 0xD9, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0x8C, 0xF9, 0x96, 0xDC, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_53[512] = { + 0xED, 0x6C, 0x96, 0xE6, 0x93, 0xF5, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x95, 0xEF, 0x99, 0xB0, 0xED, 0x6D, /* 0x04-0x07 */ + 0x99, 0xB1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x99, 0xB3, 0x00, 0x00, 0x99, 0xB5, /* 0x0C-0x0F */ + 0x99, 0xB4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x99, 0xB6, 0x89, 0xBB, 0x96, 0x6B, /* 0x14-0x17 */ + 0x00, 0x00, 0x8D, 0xFA, 0x99, 0xB7, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x91, 0x78, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x8F, 0xA0, 0x8B, 0xA7, 0x00, 0x00, 0x99, 0xB8, /* 0x20-0x23 */ + 0xED, 0x6E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x94, 0xD9, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x99, 0xB9, /* 0x2C-0x2F */ + 0x00, 0x00, 0x99, 0xBA, 0x00, 0x00, 0x99, 0xBB, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x99, 0xBC, 0x95, 0x43, 0x8B, 0xE6, 0x88, 0xE3, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0xBD, /* 0x3C-0x3F */ + 0x99, 0xBD, 0x8F, 0x5C, 0x00, 0x00, 0x90, 0xE7, /* 0x40-0x43 */ + 0x00, 0x00, 0x99, 0xBF, 0x99, 0xBE, 0x8F, 0xA1, /* 0x44-0x47 */ + 0x8C, 0xDF, 0x99, 0xC1, 0x94, 0xBC, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x99, 0xC2, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x94, 0xDA, 0x91, 0xB2, 0x91, 0xEC, /* 0x50-0x53 */ + 0x8B, 0xA6, 0x00, 0x00, 0x00, 0x00, 0x93, 0xEC, /* 0x54-0x57 */ + 0x92, 0x50, 0x00, 0x00, 0x94, 0x8E, 0x00, 0x00, /* 0x58-0x5B */ + 0x96, 0x6D, 0x00, 0x00, 0x99, 0xC4, 0x00, 0x00, /* 0x5C-0x5F */ + 0x90, 0xE8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x8C, 0x54, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x99, 0xC5, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x99, 0xC6, 0x89, 0x4B, /* 0x6C-0x6F */ + 0x88, 0xF3, 0x8A, 0xEB, 0xED, 0x6F, 0x91, 0xA6, /* 0x70-0x73 */ + 0x8B, 0x70, 0x97, 0x91, 0x00, 0x00, 0x99, 0xC9, /* 0x74-0x77 */ + 0x89, 0xB5, 0x00, 0x00, 0x00, 0x00, 0x99, 0xC8, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8B, 0xA8, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x99, 0xCA, 0x00, 0x00, /* 0x80-0x83 */ + 0x96, 0xEF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xED, 0x70, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x99, 0xCB, 0x00, 0x00, /* 0x94-0x97 */ + 0x97, 0xD0, 0x00, 0x00, 0x8C, 0xFA, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8C, 0xB4, /* 0x9C-0x9F */ + 0x99, 0xCC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x99, 0xCE, 0x99, 0xCD, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x90, 0x7E, 0x89, 0x58, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x89, 0x7D, 0x99, 0xCF, 0x00, 0x00, /* 0xAC-0xAF */ + 0x99, 0xD0, 0x00, 0x00, 0xED, 0x71, 0x8C, 0xB5, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x99, 0xD1, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8B, 0x8E, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x8E, 0x51, 0x99, 0xD2, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x96, 0x94, 0x8D, 0xB3, 0x8B, 0x79, 0x97, 0x46, /* 0xC8-0xCB */ + 0x91, 0x6F, 0x94, 0xBD, 0x8E, 0xFB, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x8F, 0x66, 0x00, 0x00, 0x8E, 0xE6, 0x8E, 0xF3, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x8F, 0x96, 0x00, 0x00, 0x94, 0xBE, /* 0xD8-0xDB */ + 0x00, 0x00, 0xED, 0x72, 0x00, 0x00, 0x99, 0xD5, /* 0xDC-0xDF */ + 0x00, 0x00, 0x89, 0x62, 0x91, 0x70, 0x8C, 0xFB, /* 0xE0-0xE3 */ + 0x8C, 0xC3, 0x8B, 0xE5, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x99, 0xD9, 0x92, 0x40, 0x91, 0xFC, 0x8B, 0xA9, /* 0xE8-0xEB */ + 0x8F, 0xA2, 0x99, 0xDA, 0x99, 0xD8, 0x89, 0xC2, /* 0xEC-0xEF */ + 0x91, 0xE4, 0x8E, 0xB6, 0x8E, 0x6A, 0x89, 0x45, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x8A, 0x90, 0x8D, 0x86, /* 0xF4-0xF7 */ + 0x8E, 0x69, 0x00, 0x00, 0x99, 0xDB, 0x00, 0x00, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_54[512] = { + 0x00, 0x00, 0x99, 0xDC, 0x00, 0x00, 0x8B, 0x68, /* 0x00-0x03 */ + 0x8A, 0x65, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x8D, 0x87, 0x8B, 0x67, 0x92, 0xDD, 0x89, 0x44, /* 0x08-0x0B */ + 0x93, 0xAF, 0x96, 0xBC, 0x8D, 0x40, 0x97, 0x99, /* 0x0C-0x0F */ + 0x93, 0x66, 0x8C, 0xFC, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8C, 0x4E, /* 0x18-0x1B */ + 0x00, 0x00, 0x99, 0xE5, 0x00, 0x00, 0x8B, 0xE1, /* 0x1C-0x1F */ + 0x96, 0x69, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x94, 0xDB, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x99, 0xE4, 0x00, 0x00, 0x8A, 0xDC, /* 0x28-0x2B */ + 0x99, 0xDF, 0x99, 0xE0, 0x99, 0xE2, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x99, 0xE3, 0x00, 0x00, /* 0x34-0x37 */ + 0x8B, 0x7A, 0x90, 0x81, 0x00, 0x00, 0x95, 0xAB, /* 0x38-0x3B */ + 0x99, 0xE1, 0x99, 0xDD, 0x8C, 0xE1, 0x00, 0x00, /* 0x3C-0x3F */ + 0x99, 0xDE, 0x00, 0x00, 0x98, 0x43, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x95, 0xF0, 0x00, 0x00, /* 0x44-0x47 */ + 0x92, 0xE6, 0x8C, 0xE0, 0x8D, 0x90, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x99, 0xE6, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x93, 0xDB, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x99, 0xEA, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x8E, 0xFC, 0x00, 0x00, 0x8E, 0xF4, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x99, 0xED, 0x99, 0xEB, 0x00, 0x00, 0x96, 0xA1, /* 0x70-0x73 */ + 0x00, 0x00, 0x99, 0xE8, 0x99, 0xF1, 0x99, 0xEC, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x99, 0xEF, /* 0x78-0x7B */ + 0x8C, 0xC4, 0x96, 0xBD, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x99, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x99, 0xF2, 0x00, 0x00, 0x99, 0xF4, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0xED, 0x75, 0x8D, 0xEE, /* 0x88-0x8B */ + 0x98, 0x61, 0x00, 0x00, 0x99, 0xE9, 0x99, 0xE7, /* 0x8C-0x8F */ + 0x99, 0xF3, 0x00, 0x00, 0x99, 0xEE, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0xED, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x99, 0xF6, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x9A, 0x42, 0x99, 0xF8, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x99, 0xFC, 0xED, 0x76, 0x00, 0x00, 0x9A, 0x40, /* 0xA8-0xAB */ + 0x99, 0xF9, 0x00, 0x00, 0x00, 0x00, 0x9A, 0x5D, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x8D, 0xE7, 0x8A, 0x50, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x99, 0xF7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x9A, 0x44, 0x88, 0xF4, 0x9A, 0x43, 0x00, 0x00, /* 0xBC-0xBF */ + 0x88, 0xA3, 0x95, 0x69, 0x9A, 0x41, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x99, 0xFA, 0x00, 0x00, 0x00, 0x00, 0x99, 0xF5, /* 0xC4-0xC7 */ + 0x99, 0xFB, 0x8D, 0xC6, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x9A, 0x45, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x88, 0xF5, 0x9A, 0x4E, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x9A, 0x46, 0x9A, 0x47, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x8F, 0xA3, 0x96, 0x89, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x9A, 0x4C, 0x9A, 0x4B, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x93, 0x4E, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x9A, 0x4D, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x9A, 0x4A, 0x00, 0x00, 0xED, 0x77, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_55[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x89, 0x53, 0x00, 0x00, 0x8D, 0xB4, 0x90, 0x4F, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9A, 0x48, /* 0x0C-0x0F */ + 0x93, 0x82, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x9A, 0x49, 0x00, 0x00, 0x88, 0xA0, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x9A, 0x53, 0x97, 0x42, /* 0x2C-0x2F */ + 0x00, 0x00, 0x8F, 0xA5, 0x00, 0x00, 0x9A, 0x59, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x9A, 0x58, 0x9A, 0x4F, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x91, 0xC1, 0x00, 0x00, /* 0x3C-0x3F */ + 0x9A, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x91, 0xED, 0x9A, 0x55, 0x8F, 0xA4, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x9A, 0x52, 0x00, 0x00, 0x00, 0x00, 0x96, 0xE2, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8C, 0x5B, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x9A, 0x56, 0x9A, 0x57, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x9A, 0x54, 0x9A, 0x5A, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9A, 0x51, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9A, 0x60, /* 0x78-0x7B */ + 0x9A, 0x65, 0x00, 0x00, 0x9A, 0x61, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x9A, 0x5C, 0x00, 0x00, 0x00, 0x00, 0x9A, 0x66, /* 0x80-0x83 */ + 0x91, 0x50, 0x00, 0x00, 0xED, 0x78, 0x9A, 0x68, /* 0x84-0x87 */ + 0x00, 0x00, 0x8D, 0x41, 0x9A, 0x5E, 0x92, 0x9D, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x9A, 0x62, 0x9A, 0x5B, 0x8A, 0xAB, 0x00, 0x00, /* 0x98-0x9B */ + 0x8A, 0xEC, 0x8A, 0x85, 0x9A, 0x63, 0x9A, 0x5F, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8C, 0x96, /* 0xA4-0xA7 */ + 0x9A, 0x69, 0x9A, 0x67, 0x91, 0x72, 0x8B, 0x69, /* 0xA8-0xAB */ + 0x8B, 0xAA, 0x00, 0x00, 0x9A, 0x64, 0x00, 0x00, /* 0xAC-0xAF */ + 0x8B, 0xF2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0x63, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x9A, 0x6D, 0x9A, 0x6B, 0x00, 0x00, 0x9A, 0xA5, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x9A, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x9A, 0x6A, 0x00, 0x00, /* 0xD8-0xDB */ + 0x9A, 0x6E, 0x00, 0x00, 0x00, 0x00, 0x9A, 0x6C, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8E, 0x6B, /* 0xE0-0xE3 */ + 0x9A, 0x6F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9A, 0x72, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x9A, 0x77, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x9A, 0x75, 0x9A, 0x74, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_56[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x92, 0x51, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x89, 0xC3, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x9A, 0x71, 0x00, 0x00, 0x9A, 0x73, 0x8F, 0xA6, /* 0x14-0x17 */ + 0x89, 0x52, 0x00, 0x00, 0x00, 0x00, 0x9A, 0x76, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x89, 0xDC, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9A, 0x82, /* 0x2C-0x2F */ + 0x00, 0x00, 0x8F, 0xFA, 0x9A, 0x7D, 0x00, 0x00, /* 0x30-0x33 */ + 0x9A, 0x7B, 0x00, 0x00, 0x9A, 0x7C, 0x00, 0x00, /* 0x34-0x37 */ + 0x9A, 0x7E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0x5C, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x91, 0x58, 0x00, 0x00, 0x9A, 0x78, 0x00, 0x00, /* 0x4C-0x4F */ + 0x9A, 0x79, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8A, 0x9A, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x9A, 0x81, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x8A, 0xED, 0x00, 0x00, 0x9A, 0x84, 0x9A, 0x80, /* 0x68-0x6B */ + 0x9A, 0x83, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x95, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x93, 0xD3, 0x00, 0x00, 0x94, 0xB6, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x9A, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x9A, 0x85, 0x8A, 0x64, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x9A, 0x87, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9A, 0x8A, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x9A, 0x89, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x9A, 0x88, 0x00, 0x00, 0x94, 0x58, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x9A, 0x8B, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x9A, 0x8C, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x9A, 0x8E, 0x00, 0x00, 0x9A, 0x8D, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x9A, 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x9A, 0x93, 0x9A, 0x91, 0x9A, 0x8F, 0x9A, 0x92, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x9A, 0x94, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x9A, 0x95, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x9A, 0x96, 0x00, 0x00, 0x9A, 0x97, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9A, 0x98, /* 0xD4-0xD7 */ + 0x99, 0x64, 0x00, 0x00, 0x8E, 0xFA, 0x8E, 0x6C, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0xF1, 0x00, 0x00, /* 0xDC-0xDF */ + 0x88, 0xF6, 0x00, 0x00, 0x00, 0x00, 0x92, 0x63, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x9A, 0x99, 0x00, 0x00, /* 0xEC-0xEF */ + 0x8D, 0xA2, 0x00, 0x00, 0x88, 0xCD, 0x90, 0x7D, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x9A, 0x9A, 0x8C, 0xC5, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x8D, 0x91, 0x00, 0x00, 0x9A, 0x9C, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_57[512] = { + 0x9A, 0x9B, 0x00, 0x00, 0x00, 0x00, 0x95, 0xDE, /* 0x00-0x03 */ + 0x9A, 0x9D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x9A, 0x9F, 0x9A, 0x9E, 0x00, 0x00, 0x9A, 0xA0, /* 0x08-0x0B */ + 0x00, 0x00, 0x9A, 0xA1, 0x00, 0x00, 0x8C, 0x97, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0x80, 0x9A, 0xA2, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x9A, 0xA4, 0x00, 0x00, /* 0x14-0x17 */ + 0x9A, 0xA3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x9A, 0xA6, 0x00, 0x00, 0x00, 0x00, 0x93, 0x79, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x9A, 0xA7, 0x88, 0xB3, /* 0x24-0x27 */ + 0x8D, 0xDD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x8C, 0x5C, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x92, 0x6E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9A, 0xA8, /* 0x34-0x37 */ + 0x9A, 0xA9, 0x00, 0x00, 0x00, 0x00, 0x9A, 0xAB, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x9A, 0xAC, 0x00, 0x00, 0x8D, 0xE2, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8B, 0xCF, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x96, 0x56, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x9A, 0xAA, 0x9A, 0xAD, /* 0x4C-0x4F */ + 0x8D, 0xBF, 0x8D, 0x42, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0xED, 0x79, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x9A, 0xB1, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x8D, 0xA3, 0xED, 0x7A, 0x92, 0x52, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x9A, 0xAE, 0x92, 0xD8, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9A, 0xB2, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x90, 0x82, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x9A, 0xB0, 0x9A, 0xB3, 0x00, 0x00, 0x8C, 0x5E, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9A, 0xB4, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x9A, 0xB5, 0x00, 0x00, 0x8D, 0x43, 0x8A, 0x5F, /* 0xA0-0xA3 */ + 0x9A, 0xB7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x9A, 0xB8, 0x00, 0x00, /* 0xA8-0xAB */ + 0xED, 0x7B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x9A, 0xB9, 0x00, 0x00, 0x00, 0x00, 0x9A, 0xB6, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x9A, 0xAF, 0x00, 0x00, 0x00, 0x00, 0x9A, 0xBA, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x9A, 0xBB, 0xED, 0x7D, /* 0xC4-0xC7 */ + 0xED, 0x7C, 0x00, 0x00, 0x00, 0x00, 0x96, 0x84, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x8F, 0xE9, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x9A, 0xBD, 0x9A, 0xBE, /* 0xD0-0xD3 */ + 0x9A, 0xBC, 0x00, 0x00, 0x9A, 0xC0, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x94, 0x57, 0x00, 0x00, 0x00, 0x00, 0x88, 0xE6, /* 0xDC-0xDF */ + 0x95, 0x75, 0x00, 0x00, 0x00, 0x00, 0x9A, 0xC1, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x8F, 0xFB, 0x00, 0x00, 0x00, 0x00, 0x8E, 0xB7, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x94, 0x7C, 0x8A, 0xEE, 0x00, 0x00, /* 0xF8-0xFB */ + 0x8D, 0xE9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_58[512] = { + 0x96, 0x78, 0x00, 0x00, 0x93, 0xB0, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x8C, 0x98, 0x91, 0xCD, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x9A, 0xBF, 0x9A, 0xC2, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x91, 0xC2, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x9A, 0xC3, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x9A, 0xC4, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x9A, 0xC6, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x92, 0xE7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x8A, 0xAC, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEA, 0x9F, /* 0x2C-0x2F */ + 0x89, 0x81, 0x95, 0xF1, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x8F, 0xEA, 0x93, 0x67, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x8D, 0xE4, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x9A, 0xCC, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x95, 0xBB, 0x97, 0xDB, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0xF2, 0x9A, 0xC8, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x91, 0x59, 0x9A, 0xCB, 0x00, 0x00, /* 0x50-0x53 */ + 0x93, 0x83, 0x00, 0x00, 0x00, 0x00, 0x93, 0x68, /* 0x54-0x57 */ + 0x93, 0x84, 0x94, 0xB7, 0x92, 0xCB, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x8D, 0xC7, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x9A, 0xC7, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x89, 0x96, 0x00, 0x00, 0x93, 0x55, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x9A, 0xC9, 0x00, 0x00, 0x9A, 0xC5, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x90, 0x6F, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x9A, 0xCD, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x8F, 0x6D, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8B, 0xAB, /* 0x80-0x83 */ + 0x00, 0x00, 0x9A, 0xCE, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0xE6, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x91, 0x9D, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x92, 0xC4, 0x00, 0x00, 0xED, 0x81, 0x9A, 0xD0, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x96, 0x6E, 0x00, 0x00, 0x00, 0x00, 0x9A, 0xD1, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x9A, 0xD6, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xED, 0x82, 0x95, 0xAD, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x9A, 0xD5, 0x9A, 0xCF, 0x9A, 0xD2, 0x9A, 0xD4, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x8D, 0xA4, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x95, 0xC7, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x9A, 0xD7, 0x00, 0x00, 0x92, 0x64, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0xF3, 0x00, 0x00, /* 0xC8-0xCB */ + 0x8F, 0xEB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x9A, 0xD9, 0x00, 0x00, 0x9A, 0xD8, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x8D, 0x88, 0x00, 0x00, 0x9A, 0xDA, /* 0xD4-0xD7 */ + 0x9A, 0xDC, 0x9A, 0xDB, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x9A, 0xDE, 0x00, 0x00, 0x9A, 0xD3, 0x9A, 0xE0, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x9A, 0xDF, 0x9A, 0xDD, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8E, 0x6D, /* 0xE8-0xEB */ + 0x90, 0x70, 0x00, 0x00, 0x91, 0x73, 0x9A, 0xE1, /* 0xEC-0xEF */ + 0x90, 0xBA, 0x88, 0xEB, 0x94, 0x84, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0xD9, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x9A, 0xE3, 0x9A, 0xE2, 0x9A, 0xE4, /* 0xF8-0xFB */ + 0x9A, 0xE5, 0x9A, 0xE6, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_59[512] = { + 0x00, 0x00, 0x00, 0x00, 0x9A, 0xE7, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x95, 0xCF, 0x9A, 0xE8, 0xED, 0x83, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x89, 0xC4, /* 0x0C-0x0F */ + 0x9A, 0xE9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x97, 0x5B, 0x8A, 0x4F, 0x00, 0x00, /* 0x14-0x17 */ + 0x99, 0xC7, 0x8F, 0x67, 0x91, 0xBD, 0x9A, 0xEA, /* 0x18-0x1B */ + 0x96, 0xE9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x96, 0xB2, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x9A, 0xEC, 0x00, 0x00, 0x91, 0xE5, /* 0x24-0x27 */ + 0x00, 0x00, 0x93, 0x56, 0x91, 0xBE, 0x95, 0x76, /* 0x28-0x2B */ + 0x9A, 0xED, 0x9A, 0xEE, 0x89, 0x9B, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x8E, 0xB8, 0x9A, 0xEF, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0xCE, /* 0x34-0x37 */ + 0x9A, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x9A, 0xF1, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x89, 0x82, 0x00, 0x00, 0x00, 0x00, 0x8A, 0xEF, /* 0x44-0x47 */ + 0x93, 0xDE, 0x95, 0xF2, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x9A, 0xF5, 0x91, 0x74, /* 0x4C-0x4F */ + 0x9A, 0xF4, 0x8C, 0x5F, 0x00, 0x00, 0xED, 0x84, /* 0x50-0x53 */ + 0x96, 0x7A, 0x9A, 0xF3, 0x00, 0x00, 0x93, 0x85, /* 0x54-0x57 */ + 0x9A, 0xF7, 0x00, 0x00, 0x9A, 0xF6, 0xED, 0x85, /* 0x58-0x5B */ + 0x00, 0x00, 0xED, 0x86, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x9A, 0xF9, 0x00, 0x00, 0x9A, 0xF8, 0xED, 0x87, /* 0x60-0x63 */ + 0x00, 0x00, 0x89, 0x9C, 0x00, 0x00, 0x9A, 0xFA, /* 0x64-0x67 */ + 0x8F, 0xA7, 0x9A, 0xFC, 0x92, 0x44, 0x00, 0x00, /* 0x68-0x6B */ + 0x9A, 0xFB, 0x00, 0x00, 0x95, 0xB1, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8F, 0x97, /* 0x70-0x73 */ + 0x93, 0x7A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x9B, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x8D, 0x44, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x9B, 0x41, 0x94, 0x40, 0x94, 0xDC, /* 0x80-0x83 */ + 0x96, 0xCF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x94, 0x44, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x9B, 0x4A, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8B, 0x57, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x97, 0x64, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x96, 0xAD, 0x00, 0x00, 0x9B, 0xAA, /* 0x98-0x9B */ + 0x00, 0x00, 0x9B, 0x42, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9B, 0x45, /* 0xA0-0xA3 */ + 0xED, 0x88, 0x91, 0xC3, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x96, 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x93, 0x69, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x9B, 0x46, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x96, 0x85, 0xED, 0x89, 0x8D, 0xC8, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x8F, 0xA8, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x9B, 0x47, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x8E, 0x6F, 0x00, 0x00, 0x8E, 0x6E, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x88, 0xB7, 0x8C, 0xC6, 0x00, 0x00, 0x90, 0xA9, /* 0xD0-0xD3 */ + 0x88, 0xCF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x9B, 0x4B, 0x9B, 0x4C, 0x00, 0x00, /* 0xD8-0xDB */ + 0x9B, 0x49, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x89, 0x57, 0x8A, 0xAD, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x9B, 0x48, 0x00, 0x00, 0x96, 0xC3, 0x95, 0x50, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x88, 0xA6, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0xF7, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8E, 0x70, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_5A[512] = { + 0x00, 0x00, 0x88, 0xD0, 0x00, 0x00, 0x88, 0xA1, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x9B, 0x51, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x9B, 0x4F, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x96, 0xBA, 0x00, 0x00, 0x9B, 0x52, 0x00, 0x00, /* 0x18-0x1B */ + 0x9B, 0x50, 0x00, 0x00, 0x00, 0x00, 0x9B, 0x4E, /* 0x1C-0x1F */ + 0x90, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x9B, 0x4D, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x95, 0xD8, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8C, 0xE2, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x9B, 0x56, 0x9B, 0x57, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x8F, 0xA9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x9B, 0x53, 0x98, 0x4B, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x94, 0x6B, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x9B, 0x55, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x8D, 0xA5, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x9B, 0x58, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x95, 0x77, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x9B, 0x59, 0x00, 0x00, /* 0x68-0x6B */ + 0x9B, 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x96, 0xB9, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x94, 0x7D, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x9B, 0x5A, 0x95, 0x51, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x9B, 0x5B, 0x9B, 0x5F, 0x9B, 0x5C, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x89, 0xC5, 0x9B, 0x5E, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x8E, 0xB9, 0x00, 0x00, 0x9B, 0x5D, /* 0xC8-0xCB */ + 0x8C, 0x99, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x9B, 0x6B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x9B, 0x64, 0x9B, 0x61, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x92, 0x84, 0x00, 0x00, 0x9B, 0x60, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x9B, 0x62, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x9B, 0x63, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x9B, 0x65, 0x9B, 0x66, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_5B[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x8A, 0xF0, 0x00, 0x00, 0x9B, 0x68, /* 0x08-0x0B */ + 0x9B, 0x67, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x9B, 0x69, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x8F, 0xEC, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x9B, 0x6C, 0x00, 0x00, /* 0x28-0x2B */ + 0x92, 0xDA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x89, 0x64, 0x00, 0x00, 0x9B, 0x6A, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x9B, 0x6D, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x9B, 0x6E, 0x00, 0x00, /* 0x3C-0x3F */ + 0x9B, 0x71, 0x00, 0x00, 0x00, 0x00, 0x9B, 0x6F, /* 0x40-0x43 */ + 0x00, 0x00, 0x9B, 0x70, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x8E, 0x71, 0x9B, 0x72, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x8D, 0x45, 0x9B, 0x73, 0xED, 0x8A, 0x8E, 0x9A, /* 0x54-0x57 */ + 0x91, 0xB6, 0x00, 0x00, 0x9B, 0x74, 0x9B, 0x75, /* 0x58-0x5B */ + 0x8E, 0x79, 0x8D, 0x46, 0x00, 0x00, 0x96, 0xD0, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8B, 0x47, /* 0x60-0x63 */ + 0x8C, 0xC7, 0x9B, 0x76, 0x8A, 0x77, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x9B, 0x77, 0x00, 0x00, 0x91, 0xB7, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x9B, 0x78, 0x9B, 0xA1, 0x00, 0x00, 0x9B, 0x79, /* 0x70-0x73 */ + 0x00, 0x00, 0x9B, 0x7A, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x9B, 0x7B, 0x00, 0x00, 0x9B, 0x7D, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x9B, 0x7E, 0x00, 0x00, 0x00, 0x00, 0x9B, 0x80, /* 0x80-0x83 */ + 0x00, 0x00, 0x91, 0xEE, 0x00, 0x00, 0x89, 0x46, /* 0x84-0x87 */ + 0x8E, 0xE7, 0x88, 0xC0, 0x00, 0x00, 0x91, 0x76, /* 0x88-0x8B */ + 0x8A, 0xAE, 0x8E, 0xB3, 0x00, 0x00, 0x8D, 0x47, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x93, 0x86, 0x00, 0x00, 0x8F, 0x40, /* 0x94-0x97 */ + 0x8A, 0xAF, 0x92, 0x88, 0x92, 0xE8, 0x88, 0xB6, /* 0x98-0x9B */ + 0x8B, 0x58, 0x95, 0xF3, 0x00, 0x00, 0x8E, 0xC0, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x8B, 0x71, 0x90, 0xE9, /* 0xA0-0xA3 */ + 0x8E, 0xBA, 0x97, 0x47, 0x9B, 0x81, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x8B, 0x7B, 0x00, 0x00, /* 0xAC-0xAF */ + 0x8D, 0xC9, 0x00, 0x00, 0x00, 0x00, 0x8A, 0x51, /* 0xB0-0xB3 */ + 0x89, 0x83, 0x8F, 0xAA, 0x89, 0xC6, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x9B, 0x82, 0x97, 0x65, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8F, 0x68, /* 0xBC-0xBF */ + 0xED, 0x8B, 0x00, 0x00, 0x8E, 0xE2, 0x9B, 0x83, /* 0xC0-0xC3 */ + 0x8A, 0xF1, 0x93, 0xD0, 0x96, 0xA7, 0x9B, 0x84, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x9B, 0x85, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x95, 0x78, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x9B, 0x87, 0x00, 0x00, 0x8A, 0xA6, 0x8B, 0xF5, /* 0xD0-0xD3 */ + 0x9B, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xED, 0x8D, 0x00, 0x00, 0x00, 0x00, 0x8A, 0xB0, /* 0xD8-0xDB */ + 0x00, 0x00, 0x90, 0x51, 0x9B, 0x8B, 0x8E, 0x40, /* 0xDC-0xDF */ + 0x00, 0x00, 0x89, 0xC7, 0x9B, 0x8A, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x9B, 0x88, 0x9B, 0x8C, 0x9B, 0x89, 0x94, 0x4A, /* 0xE4-0xE7 */ + 0x9E, 0xCB, 0x90, 0x52, 0x00, 0x00, 0x9B, 0x8D, /* 0xE8-0xEB */ + 0xED, 0x8E, 0x00, 0x00, 0x97, 0xBE, 0x00, 0x00, /* 0xEC-0xEF */ + 0x9B, 0x8E, 0x00, 0x00, 0x00, 0x00, 0x9B, 0x90, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x92, 0x9E, 0x9B, 0x8F, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x90, 0xA1, 0x00, 0x00, 0x8E, 0x9B, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0x91, 0xCE, 0x8E, 0xF5, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_5C[512] = { + 0x00, 0x00, 0x95, 0x95, 0x90, 0xEA, 0x00, 0x00, /* 0x00-0x03 */ + 0x8E, 0xCB, 0x9B, 0x91, 0x8F, 0xAB, 0x9B, 0x92, /* 0x04-0x07 */ + 0x9B, 0x93, 0x88, 0xD1, 0x91, 0xB8, 0x90, 0x71, /* 0x08-0x0B */ + 0x00, 0x00, 0x9B, 0x94, 0x93, 0xB1, 0x8F, 0xAC, /* 0x0C-0x0F */ + 0x00, 0x00, 0x8F, 0xAD, 0x00, 0x00, 0x9B, 0x95, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x90, 0xEB, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x8F, 0xAE, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xED, 0x8F, 0x00, 0x00, /* 0x1C-0x1F */ + 0x9B, 0x96, 0x00, 0x00, 0x9B, 0x97, 0x00, 0x00, /* 0x20-0x23 */ + 0x96, 0xDE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x9B, 0x98, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x8B, 0xC4, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x8F, 0x41, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x9B, 0x99, 0x9B, 0x9A, 0x8E, 0xDA, 0x90, 0x4B, /* 0x38-0x3B */ + 0x93, 0xF2, 0x90, 0x73, 0x94, 0xF6, 0x94, 0x41, /* 0x3C-0x3F */ + 0x8B, 0xC7, 0x9B, 0x9B, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x8B, 0x8F, 0x9B, 0x9C, 0x00, 0x00, /* 0x44-0x47 */ + 0x8B, 0xFC, 0x00, 0x00, 0x93, 0xCD, 0x89, 0xAE, /* 0x48-0x4B */ + 0x00, 0x00, 0x8E, 0x72, 0x9B, 0x9D, 0x9B, 0xA0, /* 0x4C-0x4F */ + 0x9B, 0x9F, 0x8B, 0xFB, 0x00, 0x00, 0x9B, 0x9E, /* 0x50-0x53 */ + 0x00, 0x00, 0x93, 0x57, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x91, 0xAE, 0x00, 0x00, /* 0x5C-0x5F */ + 0x93, 0x6A, 0x8E, 0xC6, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x91, 0x77, 0x97, 0x9A, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x9B, 0xA2, 0x00, 0x00, 0x9B, 0xA3, 0x93, 0xD4, /* 0x6C-0x6F */ + 0x00, 0x00, 0x8E, 0x52, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x9B, 0xA5, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x9B, 0xA6, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x9B, 0xA7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x8A, 0xF2, 0x9B, 0xA8, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x9B, 0xA9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x89, 0xAA, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0xED, 0x90, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x91, 0x5A, 0x8A, 0xE2, 0x00, 0x00, 0x9B, 0xAB, /* 0xA8-0xAB */ + 0x96, 0xA6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x91, 0xD0, 0x00, 0x00, 0x8A, 0x78, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x9B, 0xAD, 0x9B, 0xAF, /* 0xB4-0xB7 */ + 0x8A, 0xDD, 0x00, 0x00, 0xED, 0x91, 0x9B, 0xAC, /* 0xB8-0xBB */ + 0x9B, 0xAE, 0x00, 0x00, 0x9B, 0xB1, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x9B, 0xB0, 0x00, 0x00, 0x9B, 0xB2, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x9B, 0xB3, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x93, 0xBB, 0x8B, 0xAC, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x89, 0xE3, 0x9B, 0xB4, 0x9B, 0xB9, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x9B, 0xB7, 0x00, 0x00, 0x95, 0xF5, /* 0xEC-0xEF */ + 0x95, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0xED, 0x92, 0x93, 0x87, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x9B, 0xB6, 0x8F, 0x73, /* 0xF8-0xFB */ + 0x00, 0x00, 0x9B, 0xB5, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_5D[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x92, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9B, 0xBA, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x8D, 0xE8, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x9B, 0xC0, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x9B, 0xC1, 0x9B, 0xBB, 0x8A, 0x52, 0x9B, 0xBC, /* 0x14-0x17 */ + 0x9B, 0xC5, 0x9B, 0xC4, 0x9B, 0xC3, 0x9B, 0xBF, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9B, 0xBE, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x9B, 0xC2, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xED, 0x93, /* 0x24-0x27 */ + 0x00, 0x00, 0x95, 0xF6, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0xED, 0x96, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9B, 0xC9, /* 0x48-0x4B */ + 0x9B, 0xC6, 0x00, 0x00, 0x9B, 0xC8, 0x00, 0x00, /* 0x4C-0x4F */ + 0x97, 0x92, 0x00, 0x00, 0x9B, 0xC7, 0xED, 0x94, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x9B, 0xBD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x90, 0x93, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x9B, 0xCA, 0xED, 0x97, 0x00, 0x00, 0x8D, 0xB5, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9B, 0xCB, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x9B, 0xCC, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x9B, 0xCF, 0x00, 0x00, /* 0x80-0x83 */ + 0x9B, 0xCE, 0x00, 0x00, 0x00, 0x00, 0x9B, 0xCD, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0x88, /* 0x88-0x8B */ + 0x9B, 0xB8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x9B, 0xD5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x9B, 0xD1, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x9B, 0xD0, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x9B, 0xD2, 0x00, 0x00, 0x9B, 0xD3, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9B, 0xD6, /* 0xB4-0xB7 */ + 0xED, 0x98, 0xED, 0x99, 0x97, 0xE4, 0x00, 0x00, /* 0xB8-0xBB */ + 0x9B, 0xD7, 0x9B, 0xD4, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x9B, 0xD8, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x8A, 0xDE, 0x9B, 0xD9, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0xED, 0x9A, 0x00, 0x00, 0x9B, 0xDB, 0x9B, 0xDA, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x9B, 0xDC, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9B, 0xDD, /* 0xD8-0xDB */ + 0x00, 0x00, 0x90, 0xEC, 0x8F, 0x42, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x8F, 0x84, 0x00, 0x00, 0x91, 0x83, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x8D, 0x48, 0x8D, 0xB6, 0x8D, 0x49, /* 0xE4-0xE7 */ + 0x8B, 0x90, 0x00, 0x00, 0x00, 0x00, 0x9B, 0xDE, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x8D, 0xB7, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x8C, 0xC8, 0x9B, 0xDF, 0x96, 0xA4, /* 0xF0-0xF3 */ + 0x94, 0x62, 0x9B, 0xE0, 0x00, 0x00, 0x8D, 0x4A, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8A, 0xAA, /* 0xF8-0xFB */ + 0x00, 0x00, 0x92, 0x46, 0x8B, 0xD0, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_5E[512] = { + 0x00, 0x00, 0x00, 0x00, 0x8E, 0x73, 0x95, 0x7A, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x94, 0xBF, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9B, 0xE1, /* 0x08-0x0B */ + 0x8A, 0xF3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x9B, 0xE4, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x92, 0x9F, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x9B, 0xE3, 0x9B, 0xE2, 0x9B, 0xE5, /* 0x18-0x1B */ + 0x00, 0x00, 0x92, 0xE9, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x90, 0x83, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8E, 0x74, /* 0x28-0x2B */ + 0x00, 0x00, 0x90, 0xC8, 0x00, 0x00, 0x91, 0xD1, /* 0x2C-0x2F */ + 0x8B, 0x41, 0x00, 0x00, 0x00, 0x00, 0x92, 0xA0, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x9B, 0xE6, 0x9B, 0xE7, /* 0x34-0x37 */ + 0x8F, 0xED, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x96, 0x58, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x9B, 0xEA, 0x00, 0x00, 0x00, 0x00, 0x9B, 0xE9, /* 0x40-0x43 */ + 0x9B, 0xE8, 0x95, 0x9D, 0x00, 0x00, 0x9B, 0xF1, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x96, 0x79, 0x00, 0x00, 0x9B, 0xEB, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x9B, 0xED, 0x96, 0x8B, 0x00, 0x00, 0x9B, 0xEC, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9B, 0xEE, /* 0x5C-0x5F */ + 0x00, 0x00, 0x94, 0xA6, 0x9B, 0xEF, 0x95, 0xBC, /* 0x60-0x63 */ + 0x9B, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x8A, 0xB1, 0x95, 0xBD, /* 0x70-0x73 */ + 0x94, 0x4E, 0x9B, 0xF2, 0x9B, 0xF3, 0x00, 0x00, /* 0x74-0x77 */ + 0x8D, 0x4B, 0x8A, 0xB2, 0x9B, 0xF4, 0x8C, 0xB6, /* 0x78-0x7B */ + 0x97, 0x63, 0x97, 0x48, 0x8A, 0xF4, 0x9B, 0xF6, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x92, 0xA1, 0x00, 0x00, 0x8D, 0x4C, /* 0x80-0x83 */ + 0x8F, 0xAF, 0x00, 0x00, 0x00, 0x00, 0x94, 0xDD, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x8F, 0xB0, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8F, 0x98, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x92, 0xEA, 0x95, 0xF7, 0x93, 0x58, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x8D, 0x4D, 0x00, 0x00, /* 0x98-0x9B */ + 0x95, 0x7B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x9B, 0xF7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x93, 0x78, 0x8D, 0xC0, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8C, 0xC9, /* 0xA8-0xAB */ + 0x00, 0x00, 0x92, 0xEB, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x88, 0xC1, 0x8F, 0x8E, 0x8D, 0x4E, /* 0xB4-0xB7 */ + 0x97, 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x9B, 0xF8, 0x9B, 0xF9, 0x94, 0x70, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x9B, 0xFA, 0x97, 0xF5, 0x98, 0x4C, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9B, 0xFC, /* 0xCC-0xCF */ + 0x9B, 0xFB, 0x00, 0x00, 0x00, 0x00, 0x8A, 0x66, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x9C, 0x40, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x9C, 0x43, 0x9C, 0x44, /* 0xD8-0xDB */ + 0x00, 0x00, 0x9C, 0x42, 0x00, 0x00, 0x95, 0x5F, /* 0xDC-0xDF */ + 0x8F, 0xB1, 0x9C, 0x46, 0x9C, 0x45, 0x9C, 0x41, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x9C, 0x47, 0x9C, 0x48, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x9C, 0x49, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x9C, 0x4C, 0x9C, 0x4A, 0x00, 0x00, 0x9C, 0x4B, /* 0xF0-0xF3 */ + 0x9C, 0x4D, 0x00, 0x00, 0x89, 0x84, 0x92, 0xEC, /* 0xF4-0xF7 */ + 0x9C, 0x4E, 0x00, 0x00, 0x8C, 0x9A, 0x89, 0xF4, /* 0xF8-0xFB */ + 0x94, 0x55, 0x00, 0x00, 0x9C, 0x4F, 0x93, 0xF9, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_5F[512] = { + 0x00, 0x00, 0x95, 0xD9, 0x00, 0x00, 0x9C, 0x50, /* 0x00-0x03 */ + 0x98, 0x4D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x9C, 0x51, 0x95, 0xBE, 0x9C, 0x54, /* 0x08-0x0B */ + 0x98, 0x9F, 0x98, 0xAF, 0x00, 0x00, 0x8E, 0xAE, /* 0x0C-0x0F */ + 0x93, 0xF3, 0x9C, 0x55, 0x00, 0x00, 0x8B, 0x7C, /* 0x10-0x13 */ + 0x92, 0xA2, 0x88, 0xF8, 0x9C, 0x56, 0x95, 0xA4, /* 0x14-0x17 */ + 0x8D, 0x4F, 0x00, 0x00, 0x00, 0x00, 0x92, 0x6F, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0xED, /* 0x1C-0x1F */ + 0x00, 0x00, 0xED, 0x9B, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x96, 0xED, 0x8C, 0xB7, 0x8C, 0xCA, /* 0x24-0x27 */ + 0x00, 0x00, 0x9C, 0x57, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x9C, 0x58, 0x00, 0x00, 0x9C, 0x5E, /* 0x2C-0x2F */ + 0x00, 0x00, 0x8E, 0xE3, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0xED, 0x9C, 0x92, 0xA3, 0x00, 0x00, 0x8B, 0xAD, /* 0x34-0x37 */ + 0x9C, 0x59, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x95, 0x4A, 0x00, 0x00, 0x92, 0x65, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x9C, 0x5A, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0xED, 0x4B, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x9C, 0x5B, 0x00, 0x00, 0x8B, 0xAE, 0x00, 0x00, /* 0x48-0x4B */ + 0x9C, 0x5C, 0x00, 0x00, 0x9C, 0x5D, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x9C, 0x5F, 0x00, 0x00, 0x93, 0x96, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x9C, 0x60, 0x9C, 0x61, /* 0x54-0x57 */ + 0x00, 0x00, 0x9C, 0x62, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x9C, 0x53, 0x9C, 0x52, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x9C, 0x63, 0x8C, 0x60, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x95, 0x46, 0xED, 0x9D, /* 0x64-0x67 */ + 0x00, 0x00, 0x8D, 0xCA, 0x95, 0x56, 0x92, 0xA4, /* 0x68-0x6B */ + 0x95, 0x6A, 0x9C, 0x64, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x8F, 0xB2, 0x89, 0x65, 0x00, 0x00, 0x9C, 0x65, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9C, 0x66, /* 0x74-0x77 */ + 0x00, 0x00, 0x96, 0xF0, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x94, 0xDE, 0x00, 0x00, 0x00, 0x00, 0x9C, 0x69, /* 0x7C-0x7F */ + + 0x89, 0x9D, 0x90, 0xAA, 0x9C, 0x68, 0x9C, 0x67, /* 0x80-0x83 */ + 0x8C, 0x61, 0x91, 0xD2, 0x00, 0x00, 0x9C, 0x6D, /* 0x84-0x87 */ + 0x9C, 0x6B, 0x00, 0x00, 0x9C, 0x6A, 0x97, 0xA5, /* 0x88-0x8B */ + 0x8C, 0xE3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x8F, 0x99, 0x9C, 0x6C, 0x93, 0x6B, 0x8F, 0x5D, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0xBE, /* 0x94-0x97 */ + 0x9C, 0x70, 0x9C, 0x6F, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x9C, 0x6E, 0x00, 0x00, /* 0x9C-0x9F */ + 0x9C, 0x71, 0x8C, 0xE4, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x9C, 0x72, 0x95, 0x9C, 0x8F, 0x7A, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x9C, 0x73, 0x94, 0xF7, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0xBF, /* 0xB0-0xB3 */ + 0x92, 0xA5, 0x00, 0x00, 0x00, 0x00, 0xED, 0x9E, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x93, 0x4F, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x9C, 0x74, 0x8B, 0x4A, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x53, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x95, 0x4B, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x8A, 0xF5, 0x94, 0x45, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x9C, 0x75, 0x8E, 0x75, /* 0xD4-0xD7 */ + 0x96, 0x59, 0x96, 0x5A, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x89, 0x9E, 0x9C, 0x7A, 0xED, 0x9F, 0x00, 0x00, /* 0xDC-0xDF */ + 0x92, 0x89, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x9C, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x89, 0xF5, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x9C, 0xAB, 0x9C, 0x79, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x94, 0x4F, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x9C, 0x78, 0x00, 0x00, 0x00, 0x00, 0x9C, 0x76, /* 0xF8-0xFB */ + 0x00, 0x00, 0x8D, 0x9A, 0x00, 0x00, 0x9C, 0x7C, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_60[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x9C, 0x83, 0x9C, 0x89, /* 0x0C-0x0F */ + 0x9C, 0x81, 0x00, 0x00, 0x93, 0x7B, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x9C, 0x86, 0x95, 0x7C, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x9C, 0x80, 0x00, 0x00, 0x9C, 0x85, /* 0x18-0x1B */ + 0x97, 0xE5, 0x8E, 0x76, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x91, 0xD3, 0x9C, 0x7D, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x8B, 0x7D, 0x9C, 0x88, 0x90, 0xAB, /* 0x24-0x27 */ + 0x89, 0x85, 0x9C, 0x82, 0x89, 0xF6, 0x9C, 0x87, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8B, 0xAF, /* 0x2C-0x2F */ + 0x00, 0x00, 0x9C, 0x84, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x9C, 0x8A, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x9C, 0x8C, 0x9C, 0x96, 0x9C, 0x94, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x9C, 0x91, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x9C, 0x90, 0x97, 0xF6, /* 0x48-0x4B */ + 0x00, 0x00, 0x9C, 0x92, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x8B, 0xB0, 0x00, 0x00, 0x8D, 0x50, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x8F, 0x9A, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x9C, 0x99, 0x9C, 0x8B, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0xED, 0xA0, 0x00, 0x00, 0x9C, 0x8F, /* 0x5C-0x5F */ + 0x9C, 0x7E, 0x00, 0x00, 0x89, 0xF8, 0x9C, 0x93, /* 0x60-0x63 */ + 0x9C, 0x95, 0x92, 0x70, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x8D, 0xA6, 0x89, 0xB6, 0x9C, 0x8D, 0x9C, 0x98, /* 0x68-0x6B */ + 0x9C, 0x97, 0x8B, 0xB1, 0x00, 0x00, 0x91, 0xA7, /* 0x6C-0x6F */ + 0x8A, 0x86, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x8C, 0x62, 0x00, 0x00, 0x9C, 0x8E, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x9C, 0x9A, 0x00, 0x00, 0x9C, 0x9D, /* 0x80-0x83 */ + 0x9C, 0x9F, 0xED, 0xA1, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x8E, 0xBB, 0xED, 0xA2, 0x9C, 0xA5, /* 0x88-0x8B */ + 0x92, 0xEE, 0x9C, 0x9B, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x9C, 0xA3, 0x00, 0x00, /* 0x90-0x93 */ + 0x89, 0xF7, 0x00, 0x00, 0x9C, 0xA1, 0x9C, 0xA2, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x9C, 0x9E, 0x9C, 0xA0, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8C, 0xE5, /* 0x9C-0x9F */ + 0x97, 0x49, 0x00, 0x00, 0x00, 0x00, 0x8A, 0xB3, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0x78, 0x9C, 0xA4, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x94, 0x59, 0x88, 0xAB, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x94, 0xDF, 0x9C, 0x7B, /* 0xB0-0xB3 */ + 0x9C, 0xAA, 0x9C, 0xAE, 0x96, 0xE3, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x9C, 0xA7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x93, 0x89, 0x9C, 0xAC, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x8F, 0xEE, 0x9C, 0xAD, 0x93, 0xD5, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x98, 0x66, 0x00, 0x00, 0x9C, 0xA9, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xED, 0xA4, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x9C, 0xAF, 0x00, 0x00, 0x8D, 0x9B, 0x00, 0x00, /* 0xD8-0xDB */ + 0x90, 0xC9, 0x00, 0x00, 0xED, 0xA3, 0x88, 0xD2, /* 0xDC-0xDF */ + 0x9C, 0xA8, 0x9C, 0xA6, 0x00, 0x00, 0x91, 0x79, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9C, 0x9C, /* 0xE4-0xE7 */ + 0x8E, 0x53, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x91, 0xC4, 0x9C, 0xBB, 0xED, 0xA6, 0x91, 0x7A, /* 0xF0-0xF3 */ + 0x9C, 0xB6, 0x00, 0x00, 0x9C, 0xB3, 0x9C, 0xB4, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x8E, 0xE4, 0x9C, 0xB7, 0x9C, 0xBA, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_61[512] = { + 0x9C, 0xB5, 0x8F, 0x44, 0x00, 0x00, 0x9C, 0xB8, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x9C, 0xB2, 0x00, 0x00, /* 0x04-0x07 */ + 0x96, 0xFA, 0x96, 0xF9, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x9C, 0xBC, 0x9C, 0xBD, 0x88, 0xD3, /* 0x0C-0x0F */ + 0x00, 0x00, 0xED, 0xA7, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x9C, 0xB1, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x8B, 0xF0, 0x88, 0xA4, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8A, 0xB4, /* 0x1C-0x1F */ + 0xED, 0xA5, 0x9C, 0xB9, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9C, 0xC1, /* 0x24-0x27 */ + 0x9C, 0xC0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x9C, 0xC5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xED, 0xA9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x9C, 0xC6, 0x00, 0x00, 0x00, 0x00, 0xED, 0xA8, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x9C, 0xC4, 0x9C, 0xC7, 0x9C, 0xBF, 0x9C, 0xC3, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x9C, 0xC8, 0x00, 0x00, /* 0x40-0x43 */ + 0x9C, 0xC9, 0x00, 0x00, 0x00, 0x00, 0x9C, 0xBE, /* 0x44-0x47 */ + 0x8E, 0x9C, 0x00, 0x00, 0x9C, 0xC2, 0x91, 0xD4, /* 0x48-0x4B */ + 0x8D, 0x51, 0x9C, 0xB0, 0x90, 0x54, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9C, 0xD6, /* 0x50-0x53 */ + 0x00, 0x00, 0x95, 0xE7, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x9C, 0xCC, 0x9C, 0xCD, 0x9C, 0xCE, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x9C, 0xD5, 0x00, 0x00, 0x9C, 0xD4, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x96, 0x9D, 0x8A, 0xB5, /* 0x60-0x63 */ + 0x00, 0x00, 0x9C, 0xD2, 0x00, 0x00, 0x8C, 0x64, /* 0x64-0x67 */ + 0x8A, 0x53, 0x00, 0x00, 0x00, 0x00, 0x9C, 0xCF, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x97, 0xB6, 0x9C, 0xD1, /* 0x6C-0x6F */ + 0x88, 0xD4, 0x9C, 0xD3, 0x00, 0x00, 0x9C, 0xCA, /* 0x70-0x73 */ + 0x9C, 0xD0, 0x9C, 0xD7, 0x8C, 0x63, 0x9C, 0xCB, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x97, 0x7C, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x97, 0x4A, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9C, 0xDA, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x9C, 0xDE, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x91, 0x9E, 0x00, 0x00, /* 0x8C-0x8F */ + 0x97, 0xF7, 0x9C, 0xDF, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x9C, 0xDC, 0x00, 0x00, 0x9C, 0xD9, 0x00, 0x00, /* 0x94-0x97 */ + 0xED, 0xAA, 0x9C, 0xD8, 0x9C, 0xDD, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x95, 0xAE, 0x00, 0x00, 0x00, 0x00, 0x93, 0xB2, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x8C, 0x65, 0x00, 0x00, 0x9C, 0xE0, /* 0xA8-0xAB */ + 0x9C, 0xDB, 0x00, 0x00, 0x9C, 0xE1, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x8C, 0x9B, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0xAF, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x9C, 0xE9, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x8A, 0xB6, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9C, 0xE7, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x9C, 0xE8, 0x8D, 0xA7, /* 0xC4-0xC7 */ + 0x9C, 0xE6, 0x9C, 0xE4, 0x9C, 0xE3, 0x9C, 0xEA, /* 0xC8-0xCB */ + 0x9C, 0xE2, 0x9C, 0xEC, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x89, 0xF9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9C, 0xEE, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x9C, 0xED, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x92, 0xA6, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x9C, 0xF1, 0x00, 0x00, 0x9C, 0xEF, 0x9C, 0xE5, /* 0xF4-0xF7 */ + 0x8C, 0x9C, 0x00, 0x00, 0x9C, 0xF0, 0x00, 0x00, /* 0xF8-0xFB */ + 0x9C, 0xF4, 0x9C, 0xF3, 0x9C, 0xF5, 0x9C, 0xF2, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_62[512] = { + 0x9C, 0xF6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x9C, 0xF7, 0x9C, 0xF8, 0x95, 0xE8, 0x00, 0x00, /* 0x08-0x0B */ + 0x9C, 0xFA, 0x9C, 0xF9, 0x8F, 0x5E, 0x00, 0x00, /* 0x0C-0x0F */ + 0x90, 0xAC, 0x89, 0xE4, 0x89, 0xFA, 0xED, 0xAB, /* 0x10-0x13 */ + 0x9C, 0xFB, 0x00, 0x00, 0x88, 0xBD, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x90, 0xCA, 0x9C, 0xFC, /* 0x18-0x1B */ + 0x00, 0x00, 0xE6, 0xC1, 0x9D, 0x40, 0x8C, 0x81, /* 0x1C-0x1F */ + 0x00, 0x00, 0x9D, 0x41, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x90, 0xED, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x9D, 0x42, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x9D, 0x43, 0x8B, 0x59, /* 0x2C-0x2F */ + 0x9D, 0x44, 0x00, 0x00, 0x9D, 0x45, 0x9D, 0x46, /* 0x30-0x33 */ + 0x91, 0xD5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x8C, 0xCB, 0x00, 0x00, 0x00, 0x00, 0x96, 0xDF, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x96, 0x5B, /* 0x3C-0x3F */ + 0x8F, 0x8A, 0x9D, 0x47, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0xEE, /* 0x44-0x47 */ + 0xE7, 0xBB, 0x94, 0xE0, 0x00, 0x00, 0x8E, 0xE8, /* 0x48-0x4B */ + 0x00, 0x00, 0x8D, 0xCB, 0x9D, 0x48, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x91, 0xC5, /* 0x50-0x53 */ + 0x00, 0x00, 0x95, 0xA5, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x91, 0xEF, 0x00, 0x00, 0x00, 0x00, 0x9D, 0x4B, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x9D, 0x49, 0x00, 0x00, /* 0x5C-0x5F */ + 0x9D, 0x4C, 0x00, 0x00, 0x00, 0x00, 0x9D, 0x4A, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x9D, 0x4D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x95, 0xAF, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x88, 0xB5, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x95, 0x7D, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x94, 0xE1, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x9D, 0x4E, 0x00, 0x00, 0x9D, 0x51, 0x8F, 0xB3, /* 0x7C-0x7F */ + + 0x8B, 0x5A, 0x00, 0x00, 0x9D, 0x4F, 0x9D, 0x56, /* 0x80-0x83 */ + 0x8F, 0xB4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x9D, 0x50, 0x94, 0x63, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x97, 0x7D, 0x9D, 0x52, 0x9D, 0x53, /* 0x90-0x93 */ + 0x9D, 0x57, 0x93, 0x8A, 0x9D, 0x54, 0x8D, 0x52, /* 0x94-0x97 */ + 0x90, 0xDC, 0x00, 0x00, 0x00, 0x00, 0x9D, 0x65, /* 0x98-0x9B */ + 0x94, 0xB2, 0x00, 0x00, 0x91, 0xF0, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0xED, 0xAC, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x94, 0xE2, /* 0xA8-0xAB */ + 0x9D, 0xAB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x95, 0xF8, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x92, 0xEF, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x96, 0x95, 0x00, 0x00, 0x9D, 0x5A, /* 0xB8-0xBB */ + 0x89, 0x9F, 0x92, 0x8A, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x9D, 0x63, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x92, 0x53, 0x9D, 0x5D, 0x9D, 0x64, /* 0xC4-0xC7 */ + 0x9D, 0x5F, 0x9D, 0x66, 0x9D, 0x62, 0x00, 0x00, /* 0xC8-0xCB */ + 0x9D, 0x61, 0x94, 0x8F, 0x00, 0x00, 0x9D, 0x5B, /* 0xCC-0xCF */ + 0x89, 0xFB, 0x9D, 0x59, 0x8B, 0x91, 0x91, 0xF1, /* 0xD0-0xD3 */ + 0x9D, 0x55, 0x00, 0x00, 0x00, 0x00, 0x9D, 0x58, /* 0xD4-0xD7 */ + 0x8D, 0x53, 0x90, 0xD9, 0x00, 0x00, 0x8F, 0xB5, /* 0xD8-0xDB */ + 0x9D, 0x60, 0x94, 0x71, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x8B, 0x92, 0x8A, 0x67, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x8A, 0x87, 0x90, 0x40, 0x9D, 0x68, 0x9D, 0x6D, /* 0xEC-0xEF */ + 0x00, 0x00, 0x9D, 0x69, 0x00, 0x00, 0x8C, 0x9D, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x9D, 0x6E, 0x8E, 0x41, 0x8D, 0x89, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0x8F, 0x45, 0x9D, 0x5C, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_63[512] = { + 0x00, 0x00, 0x8E, 0x9D, 0x9D, 0x6B, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8E, 0x77, /* 0x04-0x07 */ + 0x9D, 0x6C, 0x88, 0xC2, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x9D, 0x67, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x92, 0xA7, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x8B, 0x93, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8B, 0xB2, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9D, 0x6A, /* 0x24-0x27 */ + 0x88, 0xA5, 0x00, 0x00, 0x00, 0x00, 0x8D, 0xC1, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x55, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x92, 0xF0, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x94, 0xD2, 0x9D, 0x70, 0x91, 0x7D, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x91, 0xA8, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x8E, 0x4A, 0x9D, 0x71, 0x00, 0x00, 0x9D, 0x73, /* 0x4C-0x4F */ + 0x9D, 0x6F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x95, 0xDF, 0x00, 0x00, 0x92, 0xBB, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x91, 0x7B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0xF9, /* 0x64-0x67 */ + 0x8E, 0xCC, 0x9D, 0x80, 0x00, 0x00, 0x9D, 0x7E, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x90, 0x98, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x8C, 0x9E, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x9D, 0x78, 0x8F, 0xB7, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x93, 0xE6, 0x94, 0x50, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x9D, 0x76, 0x00, 0x00, 0x00, 0x00, 0x91, 0x7C, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x8E, 0xF6, 0x9D, 0x7B, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x8F, 0xB6, 0x00, 0x00, 0x9D, 0x75, 0x9D, 0x7A, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x94, 0x72, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x9D, 0x74, 0x00, 0x00, /* 0x94-0x97 */ + 0x8C, 0x40, 0x00, 0x00, 0x00, 0x00, 0x8A, 0x7C, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9D, 0x7C, /* 0x9C-0x9F */ + 0x97, 0xA9, 0x8D, 0xCC, 0x92, 0x54, 0x9D, 0x79, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x90, 0xDA, 0x00, 0x00, 0x8D, 0x54, /* 0xA4-0xA7 */ + 0x90, 0x84, 0x89, 0x86, 0x91, 0x5B, 0x9D, 0x77, /* 0xA8-0xAB */ + 0x8B, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x8C, 0x66, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x92, 0xCD, 0x9D, 0x7D, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x91, 0x7E, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x9D, 0x81, 0x00, 0x00, /* 0xBC-0xBF */ + 0x9D, 0x83, 0x00, 0x00, 0x00, 0x00, 0x91, 0xB5, /* 0xC0-0xC3 */ + 0x9D, 0x89, 0x00, 0x00, 0x9D, 0x84, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x9D, 0x86, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0x60, /* 0xCC-0xCF */ + 0x92, 0xF1, 0x00, 0x00, 0x9D, 0x87, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x97, 0x4B, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x97, 0x67, 0x8A, 0xB7, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x88, 0xAC, 0x00, 0x00, 0x9D, 0x85, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x9D, 0x82, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x8A, 0xF6, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x89, 0x87, 0xED, 0xAD, 0x9D, 0x88, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x97, 0x68, 0x00, 0x00, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_64[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x9D, 0x8C, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x91, 0xB9, 0x00, 0x00, 0x9D, 0x93, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9D, 0x8D, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x9D, 0x8A, 0x9D, 0x91, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x9D, 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x9D, 0x8E, 0x00, 0x00, /* 0x24-0x27 */ + 0x9D, 0x92, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x94, 0xC0, 0x93, 0x8B, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x9D, 0x8B, 0x00, 0x00, 0x9D, 0x8F, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x8C, 0x67, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x8D, 0xEF, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x90, 0xDB, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x9D, 0x97, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x93, 0x45, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0xED, 0xAE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9D, 0x94, /* 0x64-0x67 */ + 0x00, 0x00, 0x96, 0x80, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9D, 0x95, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x9D, 0x96, 0x00, 0x00, /* 0x74-0x77 */ + 0x96, 0xCC, 0x00, 0x00, 0x90, 0xA0, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8C, 0x82, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x9D, 0x9D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x8E, 0x54, 0x9D, 0x9A, /* 0x90-0x93 */ + 0x00, 0x00, 0x9D, 0x99, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x94, 0x51, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0xED, 0xAF, 0x93, 0xB3, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x93, 0x50, 0x9D, 0x9B, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x9D, 0x9C, 0x00, 0x00, 0x95, 0x8F, /* 0xA8-0xAB */ + 0x00, 0x00, 0x94, 0x64, 0x8E, 0x42, 0x00, 0x00, /* 0xAC-0xAF */ + 0x90, 0xEF, 0x00, 0x00, 0x96, 0x6F, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x8A, 0x68, 0x00, 0x00, 0x9D, 0xA3, /* 0xB8-0xBB */ + 0x9D, 0x9E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x97, 0x69, 0x9D, 0xA5, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x9D, 0xA1, 0x00, 0x00, 0x9D, 0xA2, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x91, 0x80, 0xED, 0xB0, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x9D, 0xA0, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x9D, 0x5E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x9D, 0xA4, 0x00, 0x00, 0x9D, 0x9F, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x9D, 0xA9, 0x9D, 0xAA, 0x93, 0x46, 0x9D, 0xAC, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x8E, 0x43, 0x9D, 0xA7, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x8B, 0x5B, 0x00, 0x00, 0x00, 0x00, 0x9D, 0xAD, /* 0xEC-0xEF */ + 0x00, 0x00, 0x9D, 0xA6, 0x9D, 0xB1, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x9D, 0xB0, 0x00, 0x00, 0x9D, 0xAF, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x9D, 0xB2, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x9D, 0xB4, 0x8F, 0xEF, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_65[512] = { + 0x9D, 0xB3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x9D, 0xB7, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x9D, 0xB5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x9D, 0xB6, 0x9D, 0x90, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9D, 0xB9, /* 0x20-0x23 */ + 0x9D, 0xB8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x9D, 0x98, 0x9D, 0xBA, /* 0x28-0x2B */ + 0x9D, 0xAE, 0x00, 0x00, 0x00, 0x00, 0x8E, 0x78, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x9D, 0xBB, 0x9D, 0xBC, 0x9D, 0xBE, 0x9D, 0xBD, /* 0x34-0x37 */ + 0x9D, 0xBF, 0x89, 0xFC, 0x00, 0x00, 0x8D, 0x55, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x95, 0xFA, 0x90, 0xAD, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x8C, 0xCC, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x9D, 0xC1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x9D, 0xC4, 0xED, 0xB1, 0x95, 0x71, /* 0x4C-0x4F */ + 0x00, 0x00, 0x8B, 0x7E, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x9D, 0xC3, 0x9D, 0xC2, 0x94, 0x73, /* 0x54-0x57 */ + 0x9D, 0xC5, 0x8B, 0xB3, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x9D, 0xC7, 0x9D, 0xC6, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x8A, 0xB8, 0x8E, 0x55, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x93, 0xD6, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x8C, 0x68, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x90, 0x94, 0x00, 0x00, 0x9D, 0xC8, 0x00, 0x00, /* 0x70-0x73 */ + 0x90, 0xAE, 0x93, 0x47, 0x00, 0x00, 0x95, 0x7E, /* 0x74-0x77 */ + 0x9D, 0xC9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x9D, 0xCA, 0x9D, 0xCB, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0xB6, /* 0x84-0x87 */ + 0x9B, 0x7C, 0x90, 0xC4, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x95, 0x6B, 0x00, 0x00, 0x8D, 0xD6, 0x00, 0x00, /* 0x8C-0x8F */ + 0x94, 0xE3, 0x94, 0xC1, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0x6C, /* 0x94-0x97 */ + 0x00, 0x00, 0x97, 0xBF, 0x00, 0x00, 0x9D, 0xCD, /* 0x98-0x9B */ + 0x8E, 0xCE, 0x00, 0x00, 0x00, 0x00, 0x9D, 0xCE, /* 0x9C-0x9F */ + 0x00, 0x00, 0x88, 0xB4, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x8B, 0xD2, 0x90, 0xCB, 0x00, 0x00, 0x95, 0x80, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9D, 0xCF, /* 0xA8-0xAB */ + 0x8E, 0x61, 0x92, 0x66, 0x00, 0x00, 0x8E, 0x7A, /* 0xAC-0xAF */ + 0x90, 0x56, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9D, 0xD0, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x95, 0xFB, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x89, 0x97, 0x8E, 0x7B, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x9D, 0xD3, 0x00, 0x00, 0x9D, 0xD1, /* 0xC0-0xC3 */ + 0x9D, 0xD4, 0x97, 0xB7, 0x9D, 0xD2, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0xF9, /* 0xC8-0xCB */ + 0x9D, 0xD5, 0x00, 0x00, 0x00, 0x00, 0x91, 0xB0, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x9D, 0xD6, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8A, 0xF8, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x9D, 0xD8, 0x00, 0x00, 0x9D, 0xD7, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x9D, 0xD9, 0x9D, 0xDA, 0x8A, 0xF9, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x93, 0xFA, 0x92, 0x55, 0x8B, 0x8C, /* 0xE4-0xE7 */ + 0x8E, 0x7C, 0x91, 0x81, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x8F, 0x7B, 0x88, 0xAE, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x9D, 0xDB, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0xA0, 0x9D, 0xDF, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_66[512] = { + 0xED, 0xB2, 0x00, 0x00, 0x8D, 0x56, 0x9D, 0xDE, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x8D, 0xA9, 0x8F, 0xB8, /* 0x04-0x07 */ + 0x00, 0x00, 0xED, 0xB5, 0x9D, 0xDD, 0x00, 0x00, /* 0x08-0x0B */ + 0x8F, 0xB9, 0x00, 0x00, 0x96, 0xBE, 0x8D, 0xA8, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0xD5, /* 0x10-0x13 */ + 0x90, 0xCC, 0xED, 0xB3, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x9D, 0xE4, 0x00, 0x00, 0xED, 0xB7, 0x90, 0xAF, /* 0x1C-0x1F */ + 0x89, 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0xED, 0xB8, 0x8F, 0x74, 0x00, 0x00, 0x96, 0x86, /* 0x24-0x27 */ + 0x8D, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x8F, 0xBA, 0xED, 0xB6, 0x90, 0xA5, /* 0x2C-0x2F */ + 0x00, 0x00, 0xED, 0x47, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x9D, 0xE3, 0x9D, 0xE1, 0x9D, 0xE2, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xED, 0xB4, /* 0x38-0x3B */ + 0x92, 0x8B, 0x00, 0x00, 0x00, 0x00, 0x9E, 0x45, /* 0x3C-0x3F */ + 0x00, 0x00, 0x9D, 0xE8, 0x8E, 0x9E, 0x8D, 0x57, /* 0x40-0x43 */ + 0x9D, 0xE6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x9D, 0xE7, 0x00, 0x00, 0x90, 0x57, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9D, 0xE5, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x8E, 0x4E, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xED, 0xBA, /* 0x54-0x57 */ + 0x00, 0x00, 0xED, 0xBB, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x9D, 0xEA, 0x9D, 0xE9, 0x9D, 0xEE, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x9D, 0xEF, 0x00, 0x00, /* 0x60-0x63 */ + 0x9D, 0xEB, 0xED, 0xB9, 0x8A, 0x41, 0x9D, 0xEC, /* 0x64-0x67 */ + 0x9D, 0xED, 0x94, 0xD3, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x95, 0x81, 0x8C, 0x69, /* 0x6C-0x6F */ + 0x9D, 0xF0, 0x00, 0x00, 0x00, 0x00, 0xED, 0xBD, /* 0x70-0x73 */ + 0x90, 0xB0, 0x00, 0x00, 0x8F, 0xBB, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x92, 0x71, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x8B, 0xC5, 0x00, 0x00, 0x9D, 0xF1, /* 0x80-0x83 */ + 0x9D, 0xF5, 0x00, 0x00, 0x00, 0x00, 0x89, 0xC9, /* 0x84-0x87 */ + 0x9D, 0xF2, 0x9D, 0xF4, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x9D, 0xF3, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x8F, 0x8B, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x92, 0x67, 0x88, 0xC3, /* 0x94-0x97 */ + 0x9D, 0xF6, 0xED, 0xBE, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x9D, 0xF7, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0xED, 0xBF, 0x00, 0x00, 0x92, 0xA8, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x97, 0xEF, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8E, 0x62, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x95, 0xE9, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xED, 0xC0, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x96, 0x5C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x9E, 0x41, 0x9D, 0xF9, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x9D, 0xFC, 0x00, 0x00, 0x9D, 0xFB, 0xED, 0xC1, /* 0xBC-0xBF */ + 0x00, 0x00, 0x9D, 0xF8, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x9E, 0x40, 0x00, 0x00, 0x00, 0x00, 0x93, 0xDC, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x9D, 0xFA, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x9E, 0x42, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x8F, 0x8C, 0x9E, 0x43, 0x00, 0x00, /* 0xD8-0xDB */ + 0x97, 0x6A, 0x94, 0x98, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x9E, 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x9E, 0x46, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x9E, 0x47, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x9E, 0x48, 0x00, 0x00, 0x8B, 0xC8, 0x89, 0x67, /* 0xF0-0xF3 */ + 0x8D, 0x58, 0x9E, 0x49, 0x00, 0x00, 0x9E, 0x4A, /* 0xF4-0xF7 */ + 0x8F, 0x91, 0x91, 0x82, 0xED, 0xC2, 0xED, 0x4A, /* 0xF8-0xFB */ + 0x99, 0xD6, 0x91, 0x5D, 0x91, 0x5C, 0x91, 0xD6, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_67[512] = { + 0x8D, 0xC5, 0x00, 0x00, 0x00, 0x00, 0x98, 0xF0, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x8C, 0x8E, 0x97, 0x4C, 0x00, 0x00, 0x95, 0xFC, /* 0x08-0x0B */ + 0x00, 0x00, 0x95, 0x9E, 0xED, 0xC3, 0x9E, 0x4B, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x8D, 0xF1, 0x92, 0xBD, 0x9E, 0x4C, 0x98, 0x4E, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x96, 0x5D, /* 0x18-0x1B */ + 0x00, 0x00, 0x92, 0xA9, 0x9E, 0x4D, 0x8A, 0xFA, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x9E, 0x4E, 0x9E, 0x4F, /* 0x24-0x27 */ + 0x96, 0xD8, 0x00, 0x00, 0x96, 0xA2, 0x96, 0x96, /* 0x28-0x2B */ + 0x96, 0x7B, 0x8E, 0x44, 0x9E, 0x51, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x8E, 0xE9, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x96, 0x70, 0x00, 0x00, 0x9E, 0x53, 0x9E, 0x56, /* 0x34-0x37 */ + 0x9E, 0x55, 0x00, 0x00, 0x8A, 0xF7, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x8B, 0x80, 0x00, 0x00, 0x9E, 0x52, /* 0x3C-0x3F */ + 0x00, 0x00, 0x9E, 0x54, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x9E, 0x57, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x90, 0x99, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x97, 0x9B, 0x88, 0xC7, /* 0x4C-0x4F */ + 0x8D, 0xDE, 0x91, 0xBA, 0x00, 0x00, 0x8E, 0xDB, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x8F, 0xF1, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x9E, 0x5A, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x93, 0x6D, 0x00, 0x00, 0x9E, 0x58, 0x91, 0xA9, /* 0x5C-0x5F */ + 0x9E, 0x59, 0x8F, 0xF0, 0x96, 0xDB, 0x9E, 0x5B, /* 0x60-0x63 */ + 0x9E, 0x5C, 0x97, 0x88, 0xED, 0xC5, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x9E, 0x61, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x8D, 0x59, 0x00, 0x00, 0x94, 0x74, /* 0x6C-0x6F */ + 0x9E, 0x5E, 0x93, 0x8C, 0x9D, 0xDC, 0x9D, 0xE0, /* 0x70-0x73 */ + 0x00, 0x00, 0x8B, 0x6E, 0x00, 0x00, 0x94, 0x66, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x9E, 0x60, 0x00, 0x00, 0x8F, 0xBC, 0x94, 0xC2, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x9E, 0x66, 0x00, 0x00, 0x94, 0xF8, /* 0x84-0x87 */ + 0x00, 0x00, 0x9E, 0x5D, 0x00, 0x00, 0x9E, 0x63, /* 0x88-0x8B */ + 0x9E, 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x90, 0xCD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x96, 0x8D, 0x00, 0x00, 0x97, 0xD1, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x96, 0x87, 0x00, 0x00, /* 0x98-0x9B */ + 0x89, 0xCA, 0x8E, 0x7D, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x98, 0x67, 0x9E, 0x65, 0x90, 0x95, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x9E, 0x64, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x9E, 0x5F, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8C, 0xCD, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9E, 0x6B, /* 0xB0-0xB3 */ + 0x9E, 0x69, 0x00, 0x00, 0x89, 0xCB, 0x9E, 0x67, /* 0xB4-0xB7 */ + 0x9E, 0x6D, 0x9E, 0x73, 0x00, 0x00, 0xED, 0xC6, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0xED, 0xC8, 0x91, 0xC6, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x95, 0xBF, 0x00, 0x00, 0x9E, 0x75, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x95, 0x41, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x9E, 0x74, 0x94, 0x90, /* 0xCC-0xCF */ + 0x96, 0x5E, 0x8A, 0xB9, 0x00, 0x00, 0x90, 0xF5, /* 0xD0-0xD3 */ + 0x8F, 0x5F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x92, 0xD1, 0x00, 0x00, 0x97, 0x4D, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x9E, 0x70, 0x9E, 0x6F, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x9E, 0x71, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x9E, 0x6E, 0x00, 0x00, 0x00, 0x00, 0x9E, 0x76, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x9E, 0x6C, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x9E, 0x6A, 0x00, 0x00, 0x9E, 0x72, 0x9E, 0x68, /* 0xEC-0xEF */ + 0x00, 0x00, 0x92, 0x8C, 0x00, 0x00, 0x96, 0xF6, /* 0xF0-0xF3 */ + 0x8E, 0xC4, 0x8D, 0xF2, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8D, 0xB8, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0x96, 0x8F, 0x8A, 0x60, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_68[512] = { + 0x00, 0x00, 0xED, 0xC9, 0x92, 0xCC, 0x93, 0xC8, /* 0x00-0x03 */ + 0x89, 0x68, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0xF0, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x90, 0xB2, 0x8C, 0x49, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x9E, 0x78, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x8D, 0x5A, 0x8A, 0x9C, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x9E, 0x7A, 0x8A, 0x94, 0x9E, 0x81, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x9E, 0x7D, 0x00, 0x00, /* 0x30-0x33 */ + 0x90, 0xF1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x8A, 0x6A, 0x8D, 0xAA, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x8A, 0x69, 0x8D, 0xCD, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x9E, 0x7B, 0x8C, 0x85, 0x8C, 0x6A, 0x93, 0x8D, /* 0x40-0x43 */ + 0xED, 0xCA, 0x00, 0x00, 0x9E, 0x79, 0x00, 0x00, /* 0x44-0x47 */ + 0x88, 0xC4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x9E, 0x7C, 0x9E, 0x7E, 0x00, 0x00, /* 0x4C-0x4F */ + 0x8B, 0xCB, 0x8C, 0x4B, 0xED, 0xC7, 0x8A, 0xBA, /* 0x50-0x53 */ + 0x8B, 0x6A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x9E, 0x82, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x8D, 0xF7, 0x96, 0x91, 0x00, 0x00, 0x8E, 0x56, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9E, 0x83, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0x4F, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x9E, 0x8F, 0x00, 0x00, 0x89, 0xB1, 0x9E, 0x84, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x9E, 0x95, 0x9E, 0x85, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x97, 0xC0, 0x00, 0x00, 0x9E, 0x8C, /* 0x80-0x83 */ + 0x00, 0x00, 0x94, 0x7E, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x9E, 0x94, 0x00, 0x00, 0x9E, 0x87, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0xB2, /* 0x90-0x93 */ + 0x9E, 0x89, 0x00, 0x00, 0x00, 0x00, 0x8D, 0x5B, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9E, 0x8B, /* 0x98-0x9B */ + 0x00, 0x00, 0x9E, 0x8A, 0x00, 0x00, 0x9E, 0x86, /* 0x9C-0x9F */ + 0x9E, 0x91, 0x00, 0x00, 0x8F, 0xBD, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x9A, 0xEB, 0x8C, 0xE6, /* 0xA4-0xA7 */ + 0x97, 0x9C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x9E, 0x88, 0x00, 0x00, 0x92, 0xF2, /* 0xAC-0xAF */ + 0x8A, 0x42, 0x8D, 0xAB, 0x00, 0x00, 0x9E, 0x80, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x9E, 0x90, 0x8A, 0x81, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x9E, 0x8E, 0x9E, 0x92, 0x00, 0x00, /* 0xB8-0xBB */ + 0x93, 0x8E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x8A, 0xFC, 0x00, 0x00, 0x9E, 0xB0, 0x00, 0x00, /* 0xC4-0xC7 */ + 0xED, 0x48, 0x96, 0xC7, 0x9E, 0x97, 0x8A, 0xFB, /* 0xC8-0xCB */ + 0x00, 0x00, 0x9E, 0x9E, 0x00, 0x00, 0xED, 0xCB, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x96, 0x5F, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x9E, 0x9F, 0x9E, 0xA1, 0x00, 0x00, 0x9E, 0xA5, /* 0xD4-0xD7 */ + 0x9E, 0x99, 0x00, 0x00, 0x92, 0x49, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0x8F, /* 0xDC-0xDF */ + 0x9E, 0xA9, 0x9E, 0x9C, 0x00, 0x00, 0x9E, 0xA6, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9E, 0xA0, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x90, 0x58, 0x9E, 0xAA, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x90, 0xB1, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x9E, 0xA8, 0x8A, 0xBB, 0x00, 0x00, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_69[512] = { + 0x98, 0x6F, 0x9E, 0x96, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x9E, 0xA4, 0x88, 0xD6, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x9E, 0x98, 0x00, 0x00, 0x00, 0x00, 0x96, 0xB8, /* 0x08-0x0B */ + 0x9E, 0x9D, 0x90, 0x41, 0x92, 0xC5, 0x9E, 0x93, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x9E, 0xA3, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x90, 0x9A, 0x9E, 0xAD, 0x8A, 0x91, /* 0x18-0x1B */ + 0x8C, 0x9F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x9E, 0xAF, 0x9E, 0x9A, 0x9E, 0xAE, /* 0x20-0x23 */ + 0x00, 0x00, 0x9E, 0xA7, 0x9E, 0x9B, 0x00, 0x00, /* 0x24-0x27 */ + 0x9E, 0xAB, 0x00, 0x00, 0x9E, 0xAC, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x9E, 0xBD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x93, 0xCC, 0x00, 0x00, 0x9E, 0xA2, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x9E, 0xB9, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x9E, 0xBB, 0x00, 0x00, 0x92, 0xD6, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x97, 0x6B, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0x96, /* 0x50-0x53 */ + 0x9E, 0xB6, 0x91, 0xC8, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x9E, 0xBC, 0x91, 0x5E, 0x00, 0x00, /* 0x58-0x5B */ + 0x9E, 0xB3, 0x9E, 0xC0, 0x9E, 0xBF, 0x00, 0x00, /* 0x5C-0x5F */ + 0x93, 0xED, 0x9E, 0xBE, 0x93, 0xE8, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0xED, 0xCD, 0x00, 0x00, 0x9E, 0xC2, 0x9E, 0xB5, /* 0x68-0x6B */ + 0x00, 0x00, 0x8B, 0xC6, 0x9E, 0xB8, 0x8F, 0x7C, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x94, 0x80, /* 0x70-0x73 */ + 0x9E, 0xBA, 0x8B, 0xC9, 0x00, 0x00, 0x9E, 0xB2, /* 0x74-0x77 */ + 0x9E, 0xB4, 0x9E, 0xB1, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x98, 0x4F, 0x8A, 0x79, 0x9E, 0xB7, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x9E, 0xC1, 0x8A, 0x54, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x8D, 0xE5, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0x7C, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x9E, 0xD2, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x98, 0x50, 0x9E, 0xD5, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0xED, 0xCF, 0x00, 0x00, 0x00, 0x00, 0x90, 0x59, /* 0x98-0x9B */ + 0x9E, 0xD4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x9E, 0xD3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9E, 0xD0, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x9E, 0xC4, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x9E, 0xE1, 0x9E, 0xC3, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x9E, 0xD6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9E, 0xCE, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x9E, 0xC9, 0x9E, 0xC6, /* 0xBC-0xBF */ + 0x00, 0x00, 0x9E, 0xC7, 0x00, 0x00, 0x9E, 0xCF, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEA, 0xA0, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x9E, 0xCC, 0x8D, 0x5C, /* 0xC8-0xCB */ + 0x92, 0xC6, 0x91, 0x84, 0x9E, 0xCA, 0x00, 0x00, /* 0xCC-0xCF */ + 0x9E, 0xC5, 0x00, 0x00, 0x00, 0x00, 0x9E, 0xC8, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x97, 0x6C, 0x96, 0x8A, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x9E, 0xCD, 0x9E, 0xD7, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0xED, 0xD0, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9E, 0xDF, /* 0xE4-0xE7 */ + 0x9E, 0xD8, 0x00, 0x00, 0x00, 0x00, 0x9E, 0xE5, /* 0xE8-0xEB */ + 0x00, 0x00, 0x9E, 0xE3, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x9E, 0xDE, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x9E, 0xDD, 0x00, 0x00, 0x92, 0xCE, /* 0xF8-0xFB */ + 0x00, 0x00, 0x91, 0x85, 0x00, 0x00, 0x9E, 0xDB, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_6A[512] = { + 0x00, 0x00, 0x00, 0x00, 0x9E, 0xD9, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x9E, 0xE0, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x9E, 0xE6, 0x94, 0xF3, /* 0x08-0x0B */ + 0x9E, 0xEC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x9E, 0xE7, 0x9E, 0xEA, /* 0x10-0x13 */ + 0x9E, 0xE4, 0x00, 0x00, 0x00, 0x00, 0x92, 0x94, /* 0x14-0x17 */ + 0x00, 0x00, 0x95, 0x57, 0x00, 0x00, 0x9E, 0xDA, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x9E, 0xE2, 0x8F, 0xBE, /* 0x1C-0x1F */ + 0x00, 0x00, 0x96, 0xCD, 0x9E, 0xF6, 0x9E, 0xE9, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x8C, 0xA0, 0x89, 0xA1, 0x8A, 0x7E, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x9E, 0xD1, 0x00, 0x00, /* 0x2C-0x2F */ + 0xED, 0xD1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x8F, 0xBF, 0x9E, 0xEE, 0x00, 0x00, /* 0x34-0x37 */ + 0x9E, 0xF5, 0x8E, 0xF7, 0x8A, 0x92, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x92, 0x4D, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x9E, 0xEB, 0x00, 0x00, 0xED, 0xD3, 0x9E, 0xF0, /* 0x44-0x47 */ + 0x9E, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x8B, 0xB4, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x8B, 0x6B, 0x9E, 0xF2, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8B, 0x40, /* 0x5C-0x5F */ + 0x00, 0x00, 0x93, 0xC9, 0x9E, 0xF1, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x9E, 0xF3, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xED, 0xD2, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x9E, 0xED, 0xED, 0xD4, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x9E, 0xEF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0xED, 0xD5, 0x8A, 0x80, /* 0x7C-0x7F */ + + 0x92, 0x68, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x9E, 0xFA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x9E, 0xF8, 0x8C, 0xE7, 0x00, 0x00, /* 0x8C-0x8F */ + 0x9E, 0xF7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9F, 0x40, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x9E, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x9E, 0xF9, 0x00, 0x00, 0x9E, 0xFB, 0x9E, 0xFC, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x9F, 0x4B, 0x00, 0x00, /* 0xA8-0xAB */ + 0x9F, 0x47, 0x00, 0x00, 0x9E, 0x8D, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9F, 0x46, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x9F, 0x45, 0x00, 0x00, 0x00, 0x00, 0x9F, 0x42, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x9E, 0xE8, 0x9F, 0x44, 0x9F, 0x43, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x9F, 0x49, 0x00, 0x00, 0x98, 0x45, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x9F, 0x4C, 0x8B, 0xF9, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x9F, 0x48, 0x9F, 0x4A, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0xED, 0xD6, 0x00, 0x00, /* 0xE0-0xE3 */ + 0xED, 0xD7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x94, 0xA5, 0x00, 0x00, 0x9F, 0x4D, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x9F, 0x51, 0x9F, 0x4E, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_6B[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x97, 0x93, 0x9F, 0x4F, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x9E, 0xDC, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x9F, 0x52, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x9F, 0x53, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x89, 0x54, 0x00, 0x00, 0x9F, 0x55, /* 0x1C-0x1F */ + 0x8C, 0x87, 0x8E, 0x9F, 0x00, 0x00, 0x8B, 0xD3, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x89, 0xA2, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x97, 0x7E, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9F, 0x57, /* 0x34-0x37 */ + 0x9F, 0x56, 0x9F, 0x59, 0x8B, 0x5C, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x8B, 0xD4, 0x8A, 0xBC, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9F, 0x5C, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9F, 0x5B, /* 0x44-0x47 */ + 0x00, 0x00, 0x9F, 0x5D, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x89, 0xCC, 0x00, 0x00, 0x92, 0x56, 0x00, 0x00, /* 0x4C-0x4F */ + 0x9F, 0x5E, 0x00, 0x00, 0x00, 0x00, 0x8A, 0xBD, /* 0x50-0x53 */ + 0x9F, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x9F, 0x5F, 0x00, 0x00, 0x9F, 0x61, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9F, 0x62, /* 0x5C-0x5F */ + 0x00, 0x00, 0x9F, 0x63, 0x8E, 0x7E, 0x90, 0xB3, /* 0x60-0x63 */ + 0x8D, 0x9F, 0x00, 0x00, 0x95, 0x90, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x95, 0xE0, 0x98, 0x63, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8E, 0x95, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8D, 0xCE, /* 0x70-0x73 */ + 0x97, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x9F, 0x64, 0x9F, 0x65, 0x00, 0x00, 0x8E, 0x80, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9F, 0x66, /* 0x7C-0x7F */ + + 0x9F, 0x67, 0x00, 0x00, 0x00, 0x00, 0x9F, 0x69, /* 0x80-0x83 */ + 0x9F, 0x68, 0x00, 0x00, 0x96, 0x77, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x8F, 0x7D, 0x8E, 0xEA, 0x8E, 0x63, /* 0x88-0x8B */ + 0x00, 0x00, 0x9F, 0x6A, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x9F, 0x6C, 0x90, 0x42, 0x00, 0x00, /* 0x94-0x97 */ + 0x9F, 0x6B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x9F, 0x6D, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x9F, 0x6E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x9F, 0x6F, 0x9F, 0x70, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9F, 0x71, /* 0xAC-0xAF */ + 0x00, 0x00, 0x9F, 0x73, 0x9F, 0x72, 0x9F, 0x74, /* 0xB0-0xB3 */ + 0x89, 0xA3, 0x92, 0x69, 0x00, 0x00, 0x9F, 0x75, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x8E, 0x45, 0x8A, 0x6B, /* 0xB8-0xBB */ + 0x9F, 0x76, 0x00, 0x00, 0x00, 0x00, 0x93, 0x61, /* 0xBC-0xBF */ + 0x9A, 0xCA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x8B, 0x42, 0x9F, 0x77, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9F, 0x78, /* 0xC8-0xCB */ + 0x00, 0x00, 0x95, 0xEA, 0x96, 0x88, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x93, 0xC5, 0x9F, 0x79, /* 0xD0-0xD3 */ + 0x94, 0xE4, 0x00, 0x00, 0xED, 0xD8, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x94, 0xF9, 0x00, 0x00, 0x00, 0x00, 0x96, 0xD1, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9F, 0x7A, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9F, 0x7C, /* 0xE8-0xEB */ + 0x9F, 0x7B, 0x00, 0x00, 0x00, 0x00, 0x9F, 0x7E, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9F, 0x7D, /* 0xF0-0xF3 */ +}; + +static const unsigned char u2c_6C[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x9F, 0x81, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8E, 0x81, /* 0x0C-0x0F */ + 0x00, 0x00, 0x96, 0xAF, 0x00, 0x00, 0x9F, 0x82, /* 0x10-0x13 */ + 0x9F, 0x83, 0x00, 0x00, 0x00, 0x00, 0x8B, 0x43, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9F, 0x84, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9F, 0x86, /* 0x20-0x23 */ + 0x9F, 0x85, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x90, 0x85, 0x00, 0x00, 0x00, 0x00, 0x95, 0x58, /* 0x34-0x37 */ + 0x89, 0x69, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x94, 0xC3, 0xED, 0xD9, /* 0x3C-0x3F */ + 0x92, 0xF3, 0x8F, 0x60, 0x8B, 0x81, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x94, 0xC4, 0x00, 0x00, /* 0x4C-0x4F */ + 0x8E, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x9F, 0x88, 0x00, 0x00, 0x8A, 0xBE, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0x98, 0x00, 0x00, /* 0x58-0x5B */ + 0xED, 0xDA, 0x93, 0xF0, 0x9F, 0x87, 0x8D, 0x5D, /* 0x5C-0x5F */ + 0x92, 0x72, 0x00, 0x00, 0x9F, 0x89, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x9F, 0x91, 0x00, 0x00, 0x9F, 0x8A, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xED, 0xDC, /* 0x6C-0x6F */ + 0x91, 0xBF, 0x00, 0x00, 0x8B, 0x82, 0x9F, 0x92, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x8C, 0x88, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x8B, 0x44, 0x9F, 0x90, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x9F, 0x8E, 0x9F, 0x8B, 0x97, 0x80, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0xED, 0xDB, 0x00, 0x00, /* 0x84-0x87 */ + 0x92, 0xBE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x93, 0xD7, 0x9F, 0x8C, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x9F, 0x94, 0x00, 0x00, 0x9F, 0x93, 0x8C, 0x42, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0xAB, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x8D, 0xB9, 0x9F, 0x8D, 0x9F, 0x8F, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x96, 0x76, 0x91, 0xF2, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x96, 0x97, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x9F, 0x9C, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x9F, 0x9D, 0x00, 0x00, 0x89, 0xCD, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x95, 0xA6, 0x96, 0xFB, 0x9F, 0x9F, 0x8E, 0xA1, /* 0xB8-0xBB */ + 0x8F, 0xC0, 0x9F, 0x98, 0x9F, 0x9E, 0x89, 0x88, /* 0xBC-0xBF */ + 0x00, 0x00, 0x8B, 0xB5, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x9F, 0x95, 0x9F, 0x9A, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x90, 0xF2, 0x94, 0x91, 0x00, 0x00, /* 0xC8-0xCB */ + 0x94, 0xE5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9F, 0x97, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x96, 0x40, 0x00, 0x00, 0x9F, 0x99, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x9F, 0xA2, 0xED, 0xDD, 0x9F, 0xA0, /* 0xD8-0xDB */ + 0x00, 0x00, 0x9F, 0x9B, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x96, 0x41, 0x94, 0x67, 0x8B, 0x83, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x93, 0x44, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x92, 0x8D, 0x00, 0x00, 0x9F, 0xA3, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9F, 0xA1, /* 0xEC-0xEF */ + 0x91, 0xD7, 0x9F, 0x96, 0x00, 0x00, 0x89, 0x6A, /* 0xF0-0xF3 */ +}; + +static const unsigned char u2c_6D[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0xED, 0xDE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x97, 0x6D, /* 0x08-0x0B */ + 0x9F, 0xAE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x9F, 0xAD, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0xF4, /* 0x14-0x17 */ + 0x00, 0x00, 0x9F, 0xAA, 0x00, 0x00, 0x97, 0x8C, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x93, 0xB4, 0x9F, 0xA4, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x92, 0xC3, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x89, 0x6B, 0x8D, 0x5E, 0x9F, 0xA7, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x8F, 0x46, 0x9F, 0xAC, /* 0x30-0x33 */ + 0x00, 0x00, 0x9F, 0xAB, 0x9F, 0xA6, 0x00, 0x00, /* 0x34-0x37 */ + 0x9F, 0xA9, 0x00, 0x00, 0x00, 0x00, 0x8A, 0x88, /* 0x38-0x3B */ + 0x00, 0x00, 0x9F, 0xA8, 0x94, 0x68, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x97, 0xAC, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x8F, 0xF2, 0x90, 0xF3, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x9F, 0xB4, 0x9F, 0xB2, 0x00, 0x00, /* 0x58-0x5B */ + 0x95, 0x6C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9F, 0xAF, /* 0x60-0x63 */ + 0x9F, 0xB1, 0x00, 0x00, 0x89, 0x59, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x8D, 0x5F, 0x98, 0x51, 0x00, 0x00, /* 0x68-0x6B */ + 0x8A, 0x5C, 0x00, 0x00, 0x95, 0x82, 0xED, 0xE0, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x97, 0x81, 0x00, 0x00, 0x00, 0x00, 0x8A, 0x43, /* 0x74-0x77 */ + 0x90, 0x5A, 0x9F, 0xB3, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x9F, 0xB8, 0x00, 0x00, 0xED, 0xDF, /* 0x84-0x87 */ + 0x8F, 0xC1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x97, 0x4F, 0x00, 0x00, 0x9F, 0xB5, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9F, 0xB0, /* 0x90-0x93 */ + 0x00, 0x00, 0x9F, 0xB6, 0xED, 0xE1, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x97, 0xDC, 0x00, 0x00, 0x93, 0x93, /* 0x98-0x9B */ + 0x93, 0xC0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0xED, 0xE2, 0x00, 0x00, 0x00, 0x00, 0x8A, 0x55, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0x74, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x9F, 0xBC, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x9F, 0xBF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x97, 0xC1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x97, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x9F, 0xC6, 0x9F, 0xC0, 0x9F, 0xBD, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x97, 0xD2, /* 0xC8-0xCB */ + 0x9F, 0xC3, 0x00, 0x00, 0x00, 0x00, 0xED, 0xE3, /* 0xCC-0xCF */ + 0x00, 0x00, 0x8F, 0x69, 0x9F, 0xC5, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x9F, 0xCA, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x93, 0x91, 0x9F, 0xC8, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x9F, 0xC2, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x92, 0x57, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x9F, 0xC9, 0x00, 0x00, 0x9F, 0xBE, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x9F, 0xC4, 0x00, 0x00, 0x9F, 0xCB, 0x88, 0xFA, /* 0xE8-0xEB */ + 0x9F, 0xC1, 0x00, 0x00, 0x9F, 0xCC, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x90, 0x5B, 0xED, 0xE5, 0x8F, 0x7E, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x95, 0xA3, 0x00, 0x00, 0x8D, 0xAC, /* 0xF4-0xF7 */ + 0xED, 0xE4, 0x9F, 0xB9, 0x9F, 0xC7, 0x93, 0x59, /* 0xF8-0xFB */ + 0xED, 0xE6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_6E[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x90, 0xB4, 0x00, 0x00, 0x8A, 0x89, /* 0x04-0x07 */ + 0x8D, 0xCF, 0x8F, 0xC2, 0x9F, 0xBB, 0x8F, 0x61, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8C, 0x6B, /* 0x10-0x13 */ + 0x00, 0x00, 0x9F, 0xBA, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x9F, 0xD0, 0x8F, 0x8D, 0x8C, 0xB8, /* 0x18-0x1B */ + 0x00, 0x00, 0x9F, 0xDF, 0x00, 0x00, 0x9F, 0xD9, /* 0x1C-0x1F */ + 0x8B, 0x94, 0x93, 0x6E, 0x00, 0x00, 0x9F, 0xD4, /* 0x20-0x23 */ + 0x9F, 0xDD, 0x88, 0xAD, 0x89, 0x51, 0xED, 0xE9, /* 0x24-0x27 */ + 0x00, 0x00, 0x89, 0xB7, 0x00, 0x00, 0x9F, 0xD6, /* 0x28-0x2B */ + 0x91, 0xAA, 0x9F, 0xCD, 0x9F, 0xCF, 0x8D, 0x60, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x9F, 0xE0, 0xED, 0xE7, 0x9F, 0xDB, 0x00, 0x00, /* 0x38-0x3B */ + 0xED, 0xEA, 0x00, 0x00, 0x9F, 0xD3, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9F, 0xDA, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x96, 0xA9, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x9F, 0xD8, 0x9F, 0xDC, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x8C, 0xCE, 0x00, 0x00, /* 0x54-0x57 */ + 0x8F, 0xC3, 0x00, 0x00, 0x00, 0x00, 0x92, 0x58, /* 0x58-0x5B */ + 0xED, 0xE8, 0x00, 0x00, 0x00, 0x00, 0x9F, 0xD2, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x97, 0x4E, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9F, 0xD5, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x9F, 0xCE, 0x93, 0x92, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x9F, 0xD1, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x9F, 0xD7, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x98, 0x70, 0x8E, 0xBC, /* 0x7C-0x7F */ + + 0x96, 0x9E, 0x00, 0x00, 0x9F, 0xE1, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x94, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x9F, 0xED, /* 0x8C-0x8F */ + 0x8C, 0xB9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x8F, 0x80, 0x00, 0x00, /* 0x94-0x97 */ + 0x9F, 0xE3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x97, 0xAD, 0x8D, 0x61, 0x00, 0x00, 0x9F, 0xF0, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x88, 0xEC, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x9F, 0xEE, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x9F, 0xE2, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9F, 0xE8, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x9F, 0xEA, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x97, 0x6E, 0x9F, 0xE5, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x93, 0x4D, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x9F, 0xE7, 0x00, 0x00, 0xED, 0xEB, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x9F, 0xEF, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x9F, 0xE9, 0x96, 0xC5, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x9F, 0xE4, 0x00, 0x00, 0x8E, 0xA0, /* 0xC8-0xCB */ + 0x9F, 0xFC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x8A, 0x8A, 0x00, 0x00, 0x9F, 0xE6, /* 0xD0-0xD3 */ + 0x9F, 0xEB, 0x9F, 0xEC, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x91, 0xEA, 0x91, 0xD8, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x9F, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x9F, 0xFA, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x9F, 0xF8, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x93, 0x48, 0x00, 0x00, 0x00, 0x00, 0xE0, 0x42, /* 0xF4-0xF7 */ + 0x9F, 0xF5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0x9F, 0xF6, 0x9F, 0xDE, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_6F[512] = { + 0x00, 0x00, 0x8B, 0x99, 0x95, 0x59, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x8E, 0xBD, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x8D, 0x97, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x52, /* 0x0C-0x0F */ + 0x00, 0x00, 0x9F, 0xF2, 0x00, 0x00, 0xE0, 0x41, /* 0x10-0x13 */ + 0x89, 0x89, 0x91, 0x86, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x94, 0x99, 0x00, 0x00, 0x8A, 0xBF, 0x97, 0xF8, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x96, 0x9F, /* 0x28-0x2B */ + 0x92, 0xD0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x9F, 0xF9, 0x9F, 0xFB, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x91, 0x51, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0x40, 0x9F, 0xF7, /* 0x3C-0x3F */ + 0x00, 0x00, 0x9F, 0xF1, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x8A, 0xC1, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x8C, 0x89, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0xE0, 0x4E, 0x00, 0x00, 0x00, 0x00, 0xE0, 0x49, /* 0x58-0x5B */ + 0x90, 0xF6, 0x00, 0x00, 0x00, 0x00, 0x8A, 0x83, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x8F, 0x81, 0x00, 0x00, 0xE0, 0x52, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0xE0, 0x4B, 0x92, 0xAA, 0xE0, 0x48, /* 0x6C-0x6F */ + 0x92, 0xD7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0xE0, 0x6B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0xE0, 0x45, 0x00, 0x00, 0xE0, 0x44, 0x00, 0x00, /* 0x78-0x7B */ + 0xE0, 0x4D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xE0, 0x47, 0xE0, 0x46, 0xE0, 0x4C, 0x00, 0x00, /* 0x80-0x83 */ + 0x90, 0x9F, 0x00, 0x00, 0xE0, 0x43, 0x00, 0x00, /* 0x84-0x87 */ + 0xED, 0xEC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0x4F, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xE0, 0x50, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8A, 0xC0, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0xE0, 0x55, 0x00, 0x00, 0xE0, 0x54, /* 0xA0-0xA3 */ + 0xE0, 0x56, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0x59, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x93, 0x62, 0x00, 0x00, 0xE0, 0x53, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xED, 0xED, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xE0, 0x57, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x8C, 0x83, 0x91, 0xF7, 0xE0, 0x51, 0x94, 0x5A, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0x58, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xE0, 0x5D, 0xE0, 0x5B, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xE0, 0x5E, 0x00, 0x00, 0x00, 0x00, 0xE0, 0x61, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0x5A, /* 0xDC-0xDF */ + 0x8D, 0x8A, 0x94, 0x47, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x9F, 0xB7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x97, 0x94, /* 0xE8-0xEB */ + 0xE0, 0x5C, 0x00, 0x00, 0xE0, 0x60, 0x91, 0xF3, /* 0xEC-0xEF */ + 0x00, 0x00, 0xE0, 0x5F, 0x00, 0x00, 0xE0, 0x4A, /* 0xF0-0xF3 */ + 0x00, 0x00, 0xED, 0xEE, 0xE8, 0x89, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0x64, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0x68, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_70[512] = { + 0x00, 0x00, 0xE0, 0x66, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0xED, 0xEF, 0x00, 0x00, 0xED, 0xF0, /* 0x04-0x07 */ + 0x00, 0x00, 0xE0, 0x62, 0x00, 0x00, 0xE0, 0x63, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0x67, /* 0x0C-0x0F */ + 0x00, 0x00, 0xE0, 0x65, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x95, 0x6D, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0xE0, 0x6D, 0x00, 0x00, 0xE0, 0x6A, 0xE0, 0x69, /* 0x18-0x1B */ + 0x00, 0x00, 0xE0, 0x6C, 0x93, 0xD2, 0xE0, 0x6E, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x92, 0x95, 0x91, 0xEB, /* 0x24-0x27 */ + 0xED, 0xF1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x90, 0xA3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xE0, 0x6F, 0x00, 0x00, 0xE0, 0x71, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0x70, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x9F, 0xF3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0xE0, 0x72, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x93, 0xE5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0x73, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x89, 0xCE, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0x94, /* 0x6C-0x6F */ + 0x8A, 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x8B, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x8E, 0xDC, 0x8D, 0xD0, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0xED, 0xF2, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x98, 0x46, 0x90, 0x86, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0x8A, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0x75, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0xE0, 0x74, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xED, 0xF3, /* 0xA8-0xAB */ + 0xE0, 0x78, 0x92, 0x59, 0xE0, 0x7B, 0xE0, 0x76, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0x7A, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xE0, 0x79, 0x93, 0x5F, 0x88, 0xD7, 0xED, 0x46, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x97, 0xF3, 0x00, 0x00, 0x00, 0x00, 0xE0, 0x7D, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x89, 0x47, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xE0, 0x80, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0xE0, 0x7E, 0x00, 0x00, 0xE0, 0x7C, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0xE0, 0x77, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x96, 0x42, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0xE0, 0x82, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_71[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0xED, 0xF5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xE0, 0x81, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xED, 0xF4, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x89, 0x8B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0xE0, 0x84, 0x95, 0xB0, 0x00, 0x00, /* 0x18-0x1B */ + 0xE0, 0x83, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x96, 0xB3, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x8F, 0xC5, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x91, 0x52, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x8F, 0xC4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xED, 0xF7, 0xED, 0xF8, /* 0x44-0x47 */ + 0x00, 0x00, 0x97, 0xF9, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0xE0, 0x8A, 0x00, 0x00, 0x90, 0xF7, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0xE0, 0x86, 0xE0, 0x8B, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x89, 0x8C, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0xED, 0xF6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0x89, 0x00, 0x00, /* 0x60-0x63 */ + 0x94, 0x81, 0xE0, 0x85, 0xE0, 0x88, 0x8F, 0xC6, /* 0x64-0x67 */ + 0x00, 0x00, 0x94, 0xCF, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0xE0, 0x8C, 0x00, 0x00, 0x8E, 0xCF, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x90, 0xF8, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0xE0, 0x8F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0xE0, 0x87, 0x00, 0x00, 0x8C, 0x46, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0x8D, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x97, 0x6F, 0xE0, 0x90, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0xEA, 0xA4, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8F, 0x6E, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xE0, 0x91, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0xE0, 0x92, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x94, 0x4D, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xE0, 0x94, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0x95, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0xED, 0xFA, 0x00, 0x00, 0x94, 0x52, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x93, 0x95, 0xE0, 0x97, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0x99, 0x00, 0x00, /* 0xCC-0xCF */ + 0x97, 0xD3, 0x00, 0x00, 0xE0, 0x96, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xE0, 0x98, 0x89, 0x8D, 0x00, 0x00, 0xE0, 0x93, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9A, 0x7A, /* 0xDC-0xDF */ + 0xE0, 0x9A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x91, 0x87, 0x8E, 0x57, 0xE0, 0x9C, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0xE0, 0x9B, 0x90, 0x43, 0x99, 0xD7, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0xE0, 0x9D, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xE0, 0x9F, 0x00, 0x00, 0xE0, 0x8E, /* 0xF8-0xFB */ + 0xE0, 0x9E, 0x00, 0x00, 0xED, 0xFB, 0xE0, 0xA0, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_72[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x94, 0x9A, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0xE0, 0xA1, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0xE0, 0xA2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xA3, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0xE0, 0xA4, 0x00, 0x00, 0x92, 0xDC, 0x00, 0x00, /* 0x28-0x2B */ + 0xE0, 0xA6, 0xE0, 0xA5, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xE0, 0xA7, 0x00, 0x00, 0xE0, 0xA8, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x8E, 0xDD, 0x95, 0x83, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x96, 0xEA, 0xE0, 0xA9, /* 0x38-0x3B */ + 0xE0, 0xAA, 0x91, 0x75, 0x8E, 0xA2, 0xE0, 0xAB, /* 0x3C-0x3F */ + 0xE0, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xAD, 0x95, 0xD0, /* 0x44-0x47 */ + 0x94, 0xC5, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xAE, /* 0x48-0x4B */ + 0x94, 0x76, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x92, 0xAB, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0xE0, 0xAF, 0x89, 0xE5, 0x00, 0x00, 0x8B, 0x8D, /* 0x58-0x5B */ + 0x00, 0x00, 0x96, 0xC4, 0x00, 0x00, 0x96, 0xB4, /* 0x5C-0x5F */ + 0x00, 0x00, 0x89, 0xB2, 0x98, 0x53, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x96, 0x71, /* 0x64-0x67 */ + 0x00, 0x00, 0x95, 0xA8, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x90, 0xB5, 0x00, 0x00, /* 0x70-0x73 */ + 0xE0, 0xB0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x93, 0xC1, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x8C, 0xA1, 0xE0, 0xB1, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x8D, 0xD2, 0xE0, 0xB3, 0xE0, 0xB2, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xB4, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xB5, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xB6, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x8B, 0x5D, 0x00, 0x00, 0xE0, 0xB7, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xB8, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x8C, 0xA2, 0x00, 0x00, 0x00, 0x00, 0x94, 0xC6, /* 0xAC-0xAF */ + 0x00, 0x00, 0xED, 0xFC, 0xE0, 0xBA, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x8F, 0xF3, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xE0, 0xB9, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0xEE, 0x40, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x8B, 0xB6, 0xE0, 0xBB, /* 0xC0-0xC3 */ + 0xE0, 0xBD, 0x00, 0x00, 0xE0, 0xBC, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xBE, 0x00, 0x00, /* 0xCC-0xCF */ + 0x8C, 0xCF, 0x00, 0x00, 0xE0, 0xBF, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8B, 0xE7, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x91, 0x5F, 0x00, 0x00, 0x8D, 0x9D, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0xE0, 0xC1, 0xE0, 0xC2, 0xE0, 0xC0, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x8E, 0xEB, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x93, 0xC6, 0x8B, 0xB7, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xC4, /* 0xF4-0xF7 */ + 0x92, 0x4B, 0xE0, 0xC3, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x98, 0x54, 0x94, 0x82, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_73[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xC7, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xC9, 0xE0, 0xC6, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x96, 0xD2, /* 0x18-0x1B */ + 0xE0, 0xC8, 0xE0, 0xCA, 0x00, 0x00, 0x97, 0xC2, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0xEE, 0x41, 0xE0, 0xCE, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0xE0, 0xCD, 0x92, 0x96, 0x94, 0x4C, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x8C, 0xA3, 0xE0, 0xCC, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0xE0, 0xCB, 0x00, 0x00, 0x97, 0x50, 0x97, 0x51, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xCF, 0x89, 0x8E, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x8D, 0x96, 0x8E, 0x82, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xD0, 0xE0, 0xD1, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xD3, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8F, 0x62, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0xE0, 0xD5, 0x00, 0x00, 0xE0, 0xD4, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0xE0, 0xD6, 0x00, 0x00, 0x8A, 0x6C, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0xE0, 0xD8, 0x00, 0x00, 0xEE, 0x43, /* 0x74-0x77 */ + 0xE0, 0xD7, 0x00, 0x00, 0xE0, 0xDA, 0xE0, 0xD9, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x8C, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x97, 0xA6, /* 0x84-0x87 */ + 0x00, 0x00, 0x8B, 0xCA, 0x00, 0x00, 0x89, 0xA4, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x8B, 0xE8, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x8A, 0xDF, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x97, 0xE6, 0xE0, 0xDC, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xDE, /* 0xB8-0xBB */ + 0x00, 0x00, 0xEE, 0x44, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0xE0, 0xDF, 0x00, 0x00, 0x89, 0xCF, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0xE0, 0xDB, 0xEE, 0x45, 0x8E, 0x58, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x92, 0xBF, 0xE0, 0xDD, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xEE, 0x48, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0xEE, 0x46, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xE2, 0x00, 0x00, /* 0xDC-0xDF */ + 0x8E, 0xEC, 0x00, 0x00, 0x00, 0x00, 0xEE, 0x47, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xE0, 0xE0, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x8C, 0x5D, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x94, 0xC7, 0xE0, 0xE1, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0xE0, 0xFC, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0xEE, 0x4A, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xE0, 0xE7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0x8C, 0xBB, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_74[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8B, 0x85, /* 0x00-0x03 */ + 0x00, 0x00, 0xE0, 0xE4, 0x97, 0x9D, 0xEE, 0x49, /* 0x04-0x07 */ + 0x00, 0x00, 0x97, 0xAE, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x91, 0xF4, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0xE0, 0xE6, 0xEE, 0x4B, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0xEE, 0x4D, 0xEE, 0x4C, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0xEE, 0x4E, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xE8, 0x97, 0xD4, /* 0x30-0x33 */ + 0x8B, 0xD5, 0x94, 0xFA, 0x94, 0x69, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xE9, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xEB, /* 0x3C-0x3F */ + 0x00, 0x00, 0xE0, 0xEE, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0xE0, 0xEA, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0xE0, 0xED, 0x8C, 0xE8, 0x89, 0x6C, /* 0x58-0x5B */ + 0xE0, 0xEF, 0x00, 0x00, 0x90, 0x90, 0xE0, 0xEC, /* 0x5C-0x5F */ + 0x97, 0xDA, 0x00, 0x00, 0xEE, 0x4F, 0xE0, 0xF2, /* 0x60-0x63 */ + 0xEA, 0xA2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0xE0, 0xF0, 0xE0, 0xF3, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xE5, /* 0x6C-0x6F */ + 0xE0, 0xF1, 0x00, 0x00, 0x00, 0x00, 0x8D, 0xBA, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xF4, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xF5, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x97, 0x9E, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0xEE, 0x50, 0x00, 0x00, 0xE0, 0xF6, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xF7, 0xEE, 0x51, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xE3, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xF8, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x8A, 0xC2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x8E, 0xA3, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xF9, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xFA, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xE0, 0xFB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x89, 0x5A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0xE1, 0x40, 0x00, 0x00, 0x95, 0x5A, 0xE1, 0x41, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x8A, 0xA2, 0xE1, 0x42, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xE1, 0x43, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0x44, 0x00, 0x00, /* 0xEC-0xEF */ + 0xE1, 0x46, 0xE1, 0x47, 0xE1, 0x45, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x95, 0x72, 0xE1, 0x49, /* 0xF4-0xF7 */ + 0xE1, 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_75[512] = { + 0x00, 0x00, 0xEE, 0x52, 0x00, 0x00, 0xE1, 0x4B, /* 0x00-0x03 */ + 0xE1, 0x4A, 0xE1, 0x4C, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0xE1, 0x4D, 0xE1, 0x4F, 0xE1, 0x4E, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x8D, 0x99, 0x00, 0x00, 0xE1, 0x51, /* 0x10-0x13 */ + 0x00, 0x00, 0xE1, 0x50, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x8A, 0xC3, 0x00, 0x00, 0x90, 0x72, 0x00, 0x00, /* 0x18-0x1B */ + 0x93, 0x5B, 0x00, 0x00, 0xE1, 0x52, 0x90, 0xB6, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8E, 0x59, /* 0x20-0x23 */ + 0x00, 0x00, 0x89, 0x99, 0xE1, 0x53, 0x00, 0x00, /* 0x24-0x27 */ + 0x97, 0x70, 0x00, 0x00, 0x00, 0x00, 0x95, 0xE1, /* 0x28-0x2B */ + 0xE1, 0x54, 0x00, 0x00, 0x00, 0x00, 0xED, 0x8C, /* 0x2C-0x2F */ + 0x93, 0x63, 0x97, 0x52, 0x8D, 0x62, 0x90, 0x5C, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0x6A, /* 0x34-0x37 */ + 0x99, 0xB2, 0x00, 0x00, 0x92, 0xAC, 0x89, 0xE6, /* 0x38-0x3B */ + 0xE1, 0x55, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0xE1, 0x56, 0x00, 0x00, 0xE1, 0x5B, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0xE1, 0x59, 0xE1, 0x58, 0x9D, 0xC0, /* 0x48-0x4B */ + 0x8A, 0x45, 0xE1, 0x57, 0x00, 0x00, 0x88, 0xD8, /* 0x4C-0x4F */ + 0x00, 0x00, 0x94, 0xA8, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x94, 0xC8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x97, 0xAF, 0xE1, 0x5C, 0xE1, 0x5A, /* 0x58-0x5B */ + 0x92, 0x7B, 0x90, 0xA4, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x94, 0xA9, 0x00, 0x00, 0x95, 0x4C, 0x00, 0x00, /* 0x60-0x63 */ + 0xE1, 0x5E, 0x97, 0xAA, 0x8C, 0x6C, 0xE1, 0x5F, /* 0x64-0x67 */ + 0x00, 0x00, 0xE1, 0x5D, 0x94, 0xD4, 0xE1, 0x60, /* 0x68-0x6B */ + 0x00, 0x00, 0xE1, 0x61, 0x00, 0x00, 0xEE, 0x53, /* 0x6C-0x6F */ + 0x88, 0xD9, 0x00, 0x00, 0x00, 0x00, 0x8F, 0xF4, /* 0x70-0x73 */ + 0xE1, 0x66, 0x00, 0x00, 0xE1, 0x63, 0x93, 0xEB, /* 0x74-0x77 */ + 0xE1, 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8B, 0x45, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0xE1, 0x69, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0x64, 0xE1, 0x65, /* 0x84-0x87 */ + 0x00, 0x00, 0xE1, 0x68, 0xE1, 0x67, 0x95, 0x44, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x91, 0x61, 0x91, 0x60, /* 0x8C-0x8F */ + 0x00, 0x00, 0x8B, 0x5E, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0xE1, 0x6A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0x6B, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0xE1, 0x6C, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE1, 0x6E, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xE1, 0x6D, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x89, 0x75, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0xE1, 0x76, 0x94, 0xE6, 0xE1, 0x70, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xE1, 0x72, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xE1, 0x74, 0x90, 0x5D, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0xE1, 0x75, 0xE1, 0x73, 0x8E, 0xBE, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0x6F, 0xE1, 0x71, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x95, 0x61, 0x00, 0x00, 0x8F, 0xC7, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0x78, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0xE1, 0x77, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0x79, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x8E, 0xA4, 0x8D, 0xAD, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x93, 0x97, 0xE1, 0x7A, 0x00, 0x00, 0x92, 0xC9, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0x7C, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x97, 0x9F, 0xE1, 0x7B, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x91, 0x89, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0xE1, 0x82, 0x00, 0x00, 0xE1, 0x84, 0xE1, 0x85, /* 0xF0-0xF3 */ + 0x92, 0x73, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0x83, 0x00, 0x00, /* 0xF8-0xFB */ + 0xE1, 0x80, 0x00, 0x00, 0xE1, 0x7D, 0xE1, 0x7E, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_76[512] = { + 0x00, 0x00, 0xE1, 0x81, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xE1, 0x88, 0x00, 0x00, 0xE1, 0x86, /* 0x08-0x0B */ + 0x00, 0x00, 0xE1, 0x87, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE1, 0x89, /* 0x1C-0x1F */ + 0xE1, 0x8B, 0xE1, 0x8C, 0xE1, 0x8D, 0x00, 0x00, /* 0x20-0x23 */ + 0xE1, 0x8E, 0x00, 0x00, 0x00, 0x00, 0xE1, 0x8A, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xE1, 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0xE1, 0x8F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE1, 0x91, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x97, 0xC3, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0x94, 0xE1, 0x92, /* 0x44-0x47 */ + 0xE1, 0x93, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x8A, 0xE0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x96, 0xFC, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x95, 0xC8, 0x00, 0x00, /* 0x54-0x57 */ + 0xE1, 0x96, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0xE1, 0x95, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0xE1, 0x97, 0xE1, 0x98, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE1, 0x9C, /* 0x64-0x67 */ + 0xE1, 0x99, 0xE1, 0x9A, 0xE1, 0x9B, 0x00, 0x00, /* 0x68-0x6B */ + 0xE1, 0x9D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0xE1, 0x9E, 0x00, 0x00, 0xE1, 0x9F, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xA0, 0x00, 0x00, /* 0x74-0x77 */ + 0xE1, 0xA1, 0x00, 0x00, 0x94, 0xAD, 0x93, 0x6F, /* 0x78-0x7B */ + 0xE1, 0xA2, 0x94, 0x92, 0x95, 0x53, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xE1, 0xA3, 0x00, 0x00, 0xEE, 0x54, 0xE1, 0xA4, /* 0x80-0x83 */ + 0x93, 0x49, 0x00, 0x00, 0x8A, 0x46, 0x8D, 0x63, /* 0x84-0x87 */ + 0xE1, 0xA5, 0x00, 0x00, 0x00, 0x00, 0xE1, 0xA6, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xA7, 0x00, 0x00, /* 0x8C-0x8F */ + 0x8E, 0x48, 0x00, 0x00, 0x00, 0x00, 0xE1, 0xA9, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xA8, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0xE1, 0xAA, 0xE1, 0xAB, 0xEE, 0x57, /* 0x98-0x9B */ + 0xEE, 0x55, 0x00, 0x00, 0xEE, 0x56, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0xEE, 0x58, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x94, 0xE7, 0x00, 0x00, /* 0xAC-0xAF */ + 0xE1, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xE1, 0xAD, 0x00, 0x00, 0x00, 0x00, 0xEA, 0x89, /* 0xB4-0xB7 */ + 0xE1, 0xAE, 0xE1, 0xAF, 0xE1, 0xB0, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8E, 0x4D, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xB1, 0x94, 0x75, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x96, 0x7E, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x89, 0x6D, 0x00, 0x00, 0x89, 0x76, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0xE1, 0xB2, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xB4, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xB3, 0x93, 0x90, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0xB7, /* 0xD8-0xDB */ + 0x9F, 0x58, 0x00, 0x00, 0xE1, 0xB5, 0x96, 0xBF, /* 0xDC-0xDF */ + 0x00, 0x00, 0xE1, 0xB6, 0x00, 0x00, 0x8A, 0xC4, /* 0xE0-0xE3 */ + 0x94, 0xD5, 0xE1, 0xB7, 0x00, 0x00, 0xE1, 0xB8, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xB9, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x96, 0xDA, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x96, 0xD3, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x92, 0xBC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x91, 0x8A, 0x00, 0x00, 0x00, 0x00, 0xE1, 0xBB, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0x8F, 0x82, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_77[512] = { + 0x00, 0x00, 0x8F, 0xC8, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0xE1, 0xBE, 0x00, 0x00, 0x00, 0x00, 0xE1, 0xBD, /* 0x04-0x07 */ + 0xE1, 0xBC, 0x94, 0xFB, 0x00, 0x00, 0x8A, 0xC5, /* 0x08-0x0B */ + 0x8C, 0xA7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE1, 0xC4, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xC1, 0x90, 0x5E, /* 0x1C-0x1F */ + 0x96, 0xB0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0xE1, 0xC0, 0xE1, 0xC2, 0xE1, 0xC3, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0xE1, 0xBF, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE1, 0xC5, /* 0x34-0x37 */ + 0xE1, 0xC6, 0x00, 0x00, 0x92, 0xAD, 0x00, 0x00, /* 0x38-0x3B */ + 0x8A, 0xE1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x92, 0x85, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xEE, 0x5A, 0xE1, 0xC7, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xC8, 0xE1, 0xCB, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x90, 0x87, 0x00, 0x00, 0x93, 0xC2, /* 0x60-0x63 */ + 0x00, 0x00, 0xE1, 0xCC, 0x96, 0x72, 0x00, 0x00, /* 0x64-0x67 */ + 0xE1, 0xC9, 0x00, 0x00, 0x00, 0x00, 0xE1, 0xCA, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0xE1, 0xCF, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xCE, 0xE1, 0xCD, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE1, 0xD1, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xD0, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xE1, 0xD2, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xD4, 0x00, 0x00, /* 0x9C-0x9F */ + 0xE1, 0xD3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x95, 0xCB, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x8F, 0x75, 0x97, 0xC4, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0xE1, 0xD5, 0x00, 0x00, 0x00, 0x00, 0x93, 0xB5, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xD6, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xE1, 0xD7, 0x00, 0x00, 0xE1, 0xDB, /* 0xB8-0xBB */ + 0xE1, 0xD9, 0xE1, 0xDA, 0x00, 0x00, 0xE1, 0xD8, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE1, 0xDC, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0xE1, 0xDD, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE1, 0xDE, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xDF, 0x96, 0xB5, /* 0xD8-0xDB */ + 0xE1, 0xE0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x96, 0xEE, 0xE1, 0xE1, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x92, 0x6D, 0x00, 0x00, 0x94, 0x8A, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x8B, 0xE9, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x92, 0x5A, 0xE1, 0xE2, 0x8B, 0xB8, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0xCE, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0xE1, 0xE3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_78[512] = { + 0x00, 0x00, 0x00, 0x00, 0x8D, 0xBB, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0xE1, 0xE4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xE5, 0x00, 0x00, /* 0x10-0x13 */ + 0x8C, 0xA4, 0x8D, 0xD3, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0xE1, 0xE7, 0xEE, 0x5C, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x93, 0x75, 0x8D, 0xD4, 0x8B, 0x6D, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x96, 0x43, 0x00, 0x00, /* 0x30-0x33 */ + 0x94, 0x6A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x93, 0x76, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8D, 0x7B, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0xE1, 0xE9, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0xEE, 0x5D, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x8F, 0xC9, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0xEE, 0x5E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x97, 0xB0, /* 0x68-0x6B */ + 0x8D, 0x64, 0x00, 0x00, 0x00, 0x00, 0x8C, 0xA5, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x94, 0xA1, 0x00, 0x00, /* 0x70-0x73 */ + 0xE1, 0xEB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0xEE, 0x5F, 0x00, 0x00, /* 0x78-0x7B */ + 0xE1, 0xED, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x8C, 0xE9, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xEC, 0x92, 0xF4, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0xE1, 0xEF, 0x8A, 0x56, 0xE1, 0xEA, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x94, 0xE8, 0x00, 0x00, 0x89, 0x4F, /* 0x90-0x93 */ + 0x00, 0x00, 0x8D, 0xEA, 0x00, 0x00, 0x98, 0x71, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xEE, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE1, 0xF0, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0xC9, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x90, 0xD7, 0xE1, 0xF2, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE1, 0xF3, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xE1, 0xF1, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x8A, 0x6D, 0x00, 0x00, /* 0xB8-0xBB */ + 0xE1, 0xF9, 0x00, 0x00, 0xE1, 0xF8, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x8E, 0xA5, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xE1, 0xFA, 0xE1, 0xF5, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xFB, 0xE1, 0xF6, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x94, 0xD6, 0xE1, 0xF4, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xE1, 0xF7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0x41, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0x40, /* 0xE4-0xE7 */ + 0x96, 0x81, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0xE1, 0xFC, 0x00, 0x00, 0x00, 0x00, 0x88, 0xE9, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xE2, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0xE2, 0x42, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_79[512] = { + 0x00, 0x00, 0x8F, 0xCA, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0x44, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x91, 0x62, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0xE2, 0x46, 0xE2, 0x45, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0xE2, 0x47, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xE6, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xE8, 0xE2, 0x49, /* 0x28-0x2B */ + 0xE2, 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xEE, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x8E, 0xA6, 0x00, 0x00, /* 0x38-0x3B */ + 0x97, 0xE7, 0x00, 0x00, 0x8E, 0xD0, 0x00, 0x00, /* 0x3C-0x3F */ + 0xE2, 0x4A, 0x8C, 0x56, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8B, 0x5F, /* 0x44-0x47 */ + 0x8B, 0x46, 0x8E, 0x83, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x97, 0x53, 0x00, 0x00, 0x00, 0x00, 0xE2, 0x50, /* 0x50-0x53 */ + 0x00, 0x00, 0xE2, 0x4F, 0x91, 0x63, 0xE2, 0x4C, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0x4E, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x8F, 0x6A, 0x90, 0x5F, 0xE2, 0x4D, /* 0x5C-0x5F */ + 0xE2, 0x4B, 0x00, 0x00, 0x94, 0x49, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x8F, 0xCB, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x95, 0x5B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x8D, 0xD5, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0x98, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0x51, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0x52, /* 0x7C-0x7F */ + + 0xE2, 0x68, 0x8B, 0xD6, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x98, 0x5C, 0x91, 0x54, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0x53, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x89, 0xD0, 0x92, 0xF5, 0x95, 0x9F, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0xEE, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEE, 0x66, /* 0x98-0x9B */ + 0x00, 0x00, 0xE2, 0x54, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x8B, 0x9A, 0xE2, 0x55, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0x57, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0x58, 0x00, 0x00, /* 0xAC-0xAF */ + 0x94, 0x48, 0x00, 0x00, 0x00, 0x00, 0xE2, 0x59, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xE2, 0x5A, 0xE2, 0x5B, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x8B, 0xD7, 0x89, 0xD1, 0x93, 0xC3, /* 0xBC-0xBF */ + 0x8F, 0x47, 0x8E, 0x84, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xE2, 0x5C, 0x00, 0x00, 0x8F, 0x48, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x89, 0xC8, 0x95, 0x62, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xE2, 0x5D, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x94, 0xE9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x91, 0x64, /* 0xDC-0xDF */ + 0x00, 0x00, 0xE2, 0x60, 0x00, 0x00, 0xE2, 0x61, /* 0xE0-0xE3 */ + 0x94, 0x89, 0x00, 0x00, 0x90, 0x60, 0xE2, 0x5E, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x92, 0x81, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0xE2, 0x5F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x8F, 0xCC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0xDA, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_7A[512] = { + 0x8B, 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0xE2, 0x62, 0x00, 0x00, 0x00, 0x00, 0x92, 0xF6, /* 0x08-0x0B */ + 0x00, 0x00, 0xE2, 0x63, 0x90, 0xC5, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x96, 0xAB, 0x00, 0x00, 0x00, 0x00, 0x95, 0x42, /* 0x14-0x17 */ + 0xE2, 0x64, 0xE2, 0x65, 0x92, 0x74, 0x00, 0x00, /* 0x18-0x1B */ + 0x97, 0xC5, 0x00, 0x00, 0x00, 0x00, 0xE2, 0x67, /* 0x1C-0x1F */ + 0xE2, 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x8E, 0xED, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0xE2, 0x69, 0x88, 0xEE, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0x6C, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0x6A, /* 0x38-0x3B */ + 0x89, 0xD2, 0x8C, 0x6D, 0xE2, 0x6B, 0x8D, 0x65, /* 0x3C-0x3F */ + 0x8D, 0x92, 0x00, 0x00, 0x95, 0xE4, 0xE2, 0x6D, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x96, 0x73, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0xE2, 0x6F, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x90, 0xCF, 0x89, 0x6E, 0x89, 0xB8, /* 0x4C-0x4F */ + 0x88, 0xAA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0x6E, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0xE2, 0x70, 0xE2, 0x71, 0x8F, 0xF5, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0xE2, 0x72, 0x00, 0x00, 0x8A, 0x6E, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0xE2, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x8C, 0x8A, 0x00, 0x00, 0x8B, 0x86, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0xE2, 0x75, 0x8B, 0xF3, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0xE2, 0x76, 0x00, 0x00, 0x90, 0xFA, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x93, 0xCB, 0x00, 0x00, 0x90, 0xDE, /* 0x80-0x83 */ + 0x8D, 0xF3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0xE2, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x92, 0x82, 0x91, 0x8B, /* 0x90-0x93 */ + 0x00, 0x00, 0xE2, 0x79, 0xE2, 0x7B, 0xE2, 0x78, /* 0x94-0x97 */ + 0xE2, 0x7A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8C, 0x41, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0xE2, 0x7C, 0x8C, 0x45, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x8B, 0x87, 0x97, 0x71, /* 0xAC-0xAF */ + 0xE2, 0x7E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0x80, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0x4D, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0x83, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8A, 0x96, /* 0xC0-0xC3 */ + 0xE2, 0x82, 0xE2, 0x81, 0x00, 0x00, 0xE2, 0x85, /* 0xC4-0xC7 */ + 0xE2, 0x7D, 0x00, 0x00, 0xE2, 0x86, 0x97, 0xA7, /* 0xC8-0xCB */ + 0x00, 0x00, 0xE2, 0x87, 0x00, 0x00, 0xE2, 0x88, /* 0xCC-0xCF */ + 0x00, 0x00, 0xEE, 0x67, 0x9A, 0xF2, 0xE2, 0x8A, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xE2, 0x89, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xE2, 0x8B, 0xE2, 0x8C, 0x00, 0x00, /* 0xD8-0xDB */ + 0x97, 0xB3, 0xE2, 0x8D, 0x00, 0x00, 0xE8, 0xED, /* 0xDC-0xDF */ + 0x8F, 0xCD, 0xE2, 0x8E, 0xE2, 0x8F, 0x8F, 0x76, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x93, 0xB6, 0xE2, 0x90, 0xEE, 0x68, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x92, 0x47, 0xEE, 0x6A, /* 0xE8-0xEB */ + 0x00, 0x00, 0xE2, 0x91, 0x00, 0x00, 0x92, 0x5B, /* 0xEC-0xEF */ + 0xE2, 0x92, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x8B, 0xA3, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x99, 0x5E, 0x92, 0x7C, 0x8E, 0xB1, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8A, 0xC6, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_7B[512] = { + 0x00, 0x00, 0x00, 0x00, 0xE2, 0x93, 0x00, 0x00, /* 0x00-0x03 */ + 0xE2, 0xA0, 0x00, 0x00, 0xE2, 0x96, 0x00, 0x00, /* 0x04-0x07 */ + 0x8B, 0x88, 0x00, 0x00, 0xE2, 0x95, 0xE2, 0xA2, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0x94, /* 0x0C-0x0F */ + 0x00, 0x00, 0x8F, 0xCE, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0xE2, 0x98, 0xE2, 0x99, 0x00, 0x00, 0x93, 0x4A, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0x9A, 0x00, 0x00, /* 0x1C-0x1F */ + 0x8A, 0x7D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x90, 0x79, 0x95, 0x84, 0x00, 0x00, /* 0x24-0x27 */ + 0xE2, 0x9C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x91, 0xE6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0x97, /* 0x30-0x33 */ + 0x00, 0x00, 0xE2, 0x9B, 0xE2, 0x9D, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x8D, 0xF9, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0xE2, 0xA4, 0x95, 0x4D, 0x00, 0x00, /* 0x44-0x47 */ + 0x94, 0xA4, 0x93, 0x99, 0x00, 0x00, 0x8B, 0xD8, /* 0x48-0x4B */ + 0xE2, 0xA3, 0xE2, 0xA1, 0x00, 0x00, 0x94, 0xB3, /* 0x4C-0x4F */ + 0xE2, 0x9E, 0x92, 0x7D, 0x93, 0x9B, 0x00, 0x00, /* 0x50-0x53 */ + 0x93, 0x9A, 0x00, 0x00, 0x8D, 0xF4, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0xE2, 0xB6, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0xE2, 0xA6, 0x00, 0x00, 0xE2, 0xA8, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0xE2, 0xAB, 0x00, 0x00, 0xE2, 0xAC, 0x00, 0x00, /* 0x6C-0x6F */ + 0xE2, 0xA9, 0xE2, 0xAA, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0xE2, 0xA7, 0xE2, 0xA5, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0x9F, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x95, 0xCD, 0x89, 0xD3, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xB3, /* 0x88-0x8B */ + 0x00, 0x00, 0xE2, 0xB0, 0x00, 0x00, 0xE2, 0xB5, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0xB4, 0x00, 0x00, /* 0x90-0x93 */ + 0x94, 0x93, 0x96, 0xA5, 0x00, 0x00, 0x8E, 0x5A, /* 0x94-0x97 */ + 0xE2, 0xAE, 0xE2, 0xB7, 0xE2, 0xB2, 0x00, 0x00, /* 0x98-0x9B */ + 0xE2, 0xB1, 0xE2, 0xAD, 0xEE, 0x6B, 0xE2, 0xAF, /* 0x9C-0x9F */ + 0x00, 0x00, 0x8A, 0xC7, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x92, 0x5C, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x90, 0xFB, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x94, 0xA0, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xE2, 0xBC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x94, 0xA2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x90, 0xDF, 0xE2, 0xB9, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x94, 0xCD, 0x00, 0x00, 0xE2, 0xBD, 0x95, 0xD1, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x92, 0x7A, 0x00, 0x00, 0xE2, 0xB8, /* 0xC8-0xCB */ + 0xE2, 0xBA, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xBB, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0xE2, 0xBE, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x8E, 0xC2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x93, 0xC4, 0xE2, 0xC3, 0xE2, 0xC2, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xE2, 0xBF, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x98, 0x55, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xC8, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0xCC, 0xE2, 0xC9, /* 0xF4-0xF7 */ +}; + +static const unsigned char u2c_7C[512] = { + 0xE2, 0xC5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xC6, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0xE2, 0xCB, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0xE2, 0xC0, 0x99, 0xD3, 0xE2, 0xC7, /* 0x10-0x13 */ + 0xE2, 0xC1, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xCA, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xD0, /* 0x1C-0x1F */ + 0x00, 0x00, 0x8A, 0xC8, 0x00, 0x00, 0xE2, 0xCD, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xCE, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0xCF, 0xE2, 0xD2, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xD1, /* 0x34-0x37 */ + 0x94, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0xE2, 0xD3, 0x97, 0xFA, 0x95, 0xEB, /* 0x3C-0x3F */ + 0xE2, 0xD8, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xD5, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0xE2, 0xD4, 0x90, 0xD0, 0x00, 0x00, 0xE2, 0xD7, /* 0x4C-0x4F */ + 0xE2, 0xD9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0xE2, 0xD6, 0x00, 0x00, 0xE2, 0xDD, 0x00, 0x00, /* 0x54-0x57 */ + 0xE2, 0xDA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xDB, /* 0x5C-0x5F */ + 0xE2, 0xC4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0xE2, 0xDC, 0xE2, 0xDE, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0xE2, 0xDF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0xC4, /* 0x70-0x73 */ + 0x00, 0x00, 0xE2, 0xE0, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x96, 0xE0, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x8B, 0xCC, 0x8C, 0x48, 0xE2, 0xE1, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x95, 0xB2, 0x00, 0x00, 0x90, 0x88, /* 0x88-0x8B */ + 0x00, 0x00, 0x96, 0xAE, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0xE2, 0xE2, 0x00, 0x00, 0x97, 0xB1, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x94, 0x94, 0x00, 0x00, 0x91, 0x65, /* 0x94-0x97 */ + 0x94, 0x53, 0x00, 0x00, 0x00, 0x00, 0x8F, 0x6C, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0xBE, /* 0x9C-0x9F */ + 0x00, 0x00, 0xE2, 0xE7, 0xE2, 0xE5, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xE2, 0xE3, 0x8A, 0x9F, 0x00, 0x00, 0x8F, 0xCF, /* 0xA4-0xA7 */ + 0xE2, 0xE8, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xE6, /* 0xA8-0xAB */ + 0x00, 0x00, 0xE2, 0xE4, 0xE2, 0xEC, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0xE2, 0xEB, 0xE2, 0xEA, 0xE2, 0xE9, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xE2, 0xED, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0xE2, 0xEE, 0x90, 0xB8, 0x00, 0x00, /* 0xBC-0xBF */ + 0xE2, 0xEF, 0x00, 0x00, 0xE2, 0xF1, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xE2, 0xF0, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x8C, 0xD0, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x91, 0x57, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0xF3, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x93, 0x9C, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xE2, 0xF2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0xE2, 0xF4, 0x00, 0x00, 0x95, 0xB3, 0x91, 0x8C, /* 0xDC-0xDF */ + 0x8D, 0x66, 0x00, 0x00, 0xE2, 0xF5, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x97, 0xC6, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xF7, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0xF8, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xE2, 0xF9, 0x00, 0x00, 0xE2, 0xFA, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x8E, 0x85, 0x00, 0x00, 0xE2, 0xFB, 0x8C, 0x6E, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0x8B, 0x8A, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_7D[512] = { + 0x8B, 0x49, 0x00, 0x00, 0xE3, 0x40, 0x00, 0x00, /* 0x00-0x03 */ + 0x96, 0xF1, 0x8D, 0x67, 0xE2, 0xFC, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0x43, 0x96, 0xE4, /* 0x08-0x0B */ + 0x00, 0x00, 0x94, 0x5B, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x95, 0x52, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x8F, 0x83, 0xE3, 0x42, 0x00, 0x00, 0x8E, 0xD1, /* 0x14-0x17 */ + 0x8D, 0x68, 0x8E, 0x86, 0x8B, 0x89, 0x95, 0xB4, /* 0x18-0x1B */ + 0xE3, 0x41, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x91, 0x66, 0x96, 0x61, 0x8D, 0xF5, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8E, 0x87, /* 0x28-0x2B */ + 0x92, 0xDB, 0x00, 0x00, 0xE3, 0x46, 0x97, 0xDD, /* 0x2C-0x2F */ + 0x8D, 0xD7, 0x00, 0x00, 0xE3, 0x47, 0x90, 0x61, /* 0x30-0x33 */ + 0x00, 0x00, 0xE3, 0x49, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x8F, 0xD0, 0x8D, 0xAE, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE3, 0x48, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x8F, 0x49, 0x8C, 0xBC, /* 0x40-0x43 */ + 0x91, 0x67, 0xE3, 0x44, 0xE3, 0x4A, 0x00, 0x00, /* 0x44-0x47 */ + 0xEE, 0x6D, 0x00, 0x00, 0x00, 0x00, 0xE3, 0x45, /* 0x48-0x4B */ + 0x8C, 0x6F, 0x00, 0x00, 0xE3, 0x4D, 0xE3, 0x51, /* 0x4C-0x4F */ + 0x8C, 0x8B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0x4C, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE3, 0x55, /* 0x58-0x5B */ + 0xEE, 0x6E, 0x00, 0x00, 0x8D, 0x69, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x97, 0x8D, 0x88, 0xBA, 0xE3, 0x52, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x8B, 0x8B, 0x00, 0x00, /* 0x64-0x67 */ + 0xE3, 0x4F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0x50, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x93, 0x9D, 0xE3, 0x4E, 0xE3, 0x4B, /* 0x70-0x73 */ + 0x00, 0x00, 0x8A, 0x47, 0x90, 0xE2, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x8C, 0xA6, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0xE3, 0x57, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0xE3, 0x54, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE3, 0x56, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE3, 0x53, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x8C, 0x70, 0x91, 0xB1, 0xE3, 0x58, /* 0x98-0x9B */ + 0x91, 0x8E, 0x00, 0x00, 0x00, 0x00, 0xE3, 0x65, /* 0x9C-0x9F */ + 0xEE, 0x70, 0x00, 0x00, 0xE3, 0x61, 0xE3, 0x5B, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE3, 0x5F, /* 0xA8-0xAB */ + 0x8E, 0xF8, 0x88, 0xDB, 0xE3, 0x5A, 0xE3, 0x62, /* 0xAC-0xAF */ + 0xE3, 0x66, 0x8D, 0x6A, 0x96, 0xD4, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x92, 0xD4, 0xE3, 0x5C, 0x00, 0x00, 0xEE, 0x6F, /* 0xB4-0xB7 */ + 0xE3, 0x64, 0x00, 0x00, 0xE3, 0x59, 0x92, 0x5D, /* 0xB8-0xBB */ + 0x00, 0x00, 0xE3, 0x5E, 0x88, 0xBB, 0x96, 0xC8, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE3, 0x5D, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x8B, 0xD9, 0x94, 0xEA, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x91, 0x8D, /* 0xCC-0xCF */ + 0x00, 0x00, 0x97, 0xCE, 0x8F, 0x8F, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xE3, 0x8E, 0xEE, 0x71, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xE3, 0x67, 0x00, 0x00, 0x90, 0xFC, 0x00, 0x00, /* 0xD8-0xDB */ + 0xE3, 0x63, 0xE3, 0x68, 0xE3, 0x6A, 0x00, 0x00, /* 0xDC-0xDF */ + 0x92, 0xF7, 0xE3, 0x6D, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0xE3, 0x69, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x95, 0xD2, 0x8A, 0xC9, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x96, 0xC9, 0x00, 0x00, 0x00, 0x00, 0x88, 0xDC, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0x6C, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x97, 0xFB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE3, 0x6B, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_7E[512] = { + 0x00, 0x00, 0x89, 0x8F, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x93, 0xEA, 0xE3, 0x6E, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xE3, 0x75, 0xE3, 0x6F, 0xE3, 0x76, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0x72, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x94, 0x9B, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x8E, 0xC8, 0xE3, 0x74, /* 0x1C-0x1F */ + 0x00, 0x00, 0xE3, 0x71, 0xE3, 0x77, 0xE3, 0x70, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x8F, 0x63, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x96, 0x44, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x8F, 0x6B, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0xE3, 0x73, 0xE3, 0x80, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0xE3, 0x7B, 0x00, 0x00, 0xE3, 0x7E, /* 0x34-0x37 */ + 0x00, 0x00, 0xE3, 0x7C, 0xE3, 0x81, 0xE3, 0x7A, /* 0x38-0x3B */ + 0x00, 0x00, 0xE3, 0x60, 0x90, 0xD1, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x94, 0xC9, 0x00, 0x00, 0xE3, 0x7D, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0x78, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x91, 0x40, 0x8C, 0x71, /* 0x48-0x4B */ + 0x00, 0x00, 0x8F, 0x4A, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0xEE, 0x72, 0x00, 0x00, /* 0x50-0x53 */ + 0x90, 0x44, 0x91, 0x55, 0xE3, 0x84, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0xE3, 0x86, 0xE3, 0x87, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0xE3, 0x83, 0xE3, 0x85, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0x79, 0xE3, 0x82, /* 0x64-0x67 */ + 0x00, 0x00, 0xE3, 0x8A, 0xE3, 0x89, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x96, 0x9A, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x8C, 0x4A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0xE3, 0x88, 0x00, 0x00, 0xE3, 0x8C, /* 0x78-0x7B */ + 0xE3, 0x8B, 0xE3, 0x8F, 0x00, 0x00, 0xE3, 0x91, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x8E, 0x5B, 0xE3, 0x8D, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0xE3, 0x92, 0xE3, 0x93, 0xED, 0x40, 0x00, 0x00, /* 0x88-0x8B */ + 0xE3, 0x94, 0x00, 0x00, 0xE3, 0x9A, 0x93, 0x5A, /* 0x8C-0x8F */ + 0xE3, 0x96, 0x00, 0x00, 0xE3, 0x95, 0xE3, 0x97, /* 0x90-0x93 */ + 0xE3, 0x98, 0x00, 0x00, 0xE3, 0x99, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE3, 0x9B, /* 0x98-0x9B */ + 0xE3, 0x9C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ +}; + +static const unsigned char u2c_7F[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x8A, 0xCA, 0x00, 0x00, /* 0x34-0x37 */ + 0xE3, 0x9D, 0x00, 0x00, 0xE3, 0x9E, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0xE3, 0x9F, 0x00, 0x00, 0xEE, 0x73, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0xE3, 0xA0, 0xE3, 0xA1, 0xE3, 0xA2, 0x00, 0x00, /* 0x4C-0x4F */ + 0xE3, 0xA3, 0xE3, 0xA4, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0xE3, 0xA6, 0xE3, 0xA5, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0xE3, 0xA7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE3, 0xA8, /* 0x5C-0x5F */ + 0xE3, 0xA9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE3, 0xAC, /* 0x64-0x67 */ + 0xE3, 0xAA, 0xE3, 0xAB, 0x8D, 0xDF, 0x8C, 0x72, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x92, 0x75, 0x00, 0x00, /* 0x6C-0x6F */ + 0x94, 0xB1, 0x00, 0x00, 0x8F, 0x90, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x94, 0x6C, 0x00, 0x00, 0x94, 0xEB, /* 0x74-0x77 */ + 0xE3, 0xAD, 0x9C, 0xEB, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xAE, 0xE3, 0xB0, /* 0x80-0x83 */ + 0x00, 0x00, 0x97, 0x85, 0xE3, 0xAF, 0xE3, 0xB2, /* 0x84-0x87 */ + 0xE3, 0xB1, 0x00, 0x00, 0x97, 0x72, 0x00, 0x00, /* 0x88-0x8B */ + 0xE3, 0xB3, 0x00, 0x00, 0x94, 0xFC, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0xE3, 0xB4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xB7, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0xE3, 0xB6, 0xE3, 0xB5, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0xEE, 0x74, 0x00, 0x00, 0xE3, 0xB8, /* 0xA0-0xA3 */ + 0x8C, 0x51, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x91, 0x41, 0x8B, 0x60, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xBC, 0xE3, 0xB9, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xBA, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xBD, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xE3, 0xBE, 0xE3, 0xBB, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x89, 0x48, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x89, 0xA5, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xE3, 0xC0, 0xE3, 0xC1, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xC2, 0x00, 0x00, /* 0xC8-0xCB */ + 0x97, 0x82, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x8F, 0x4B, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xE3, 0xC4, 0xE3, 0xC3, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x90, 0x89, 0xE3, 0xC5, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xC6, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xE3, 0xC7, 0x00, 0x00, 0x8A, 0xE3, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x8A, 0xCB, 0x00, 0x00, 0x00, 0x00, 0xE3, 0xC8, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xE3, 0xC9, 0x00, 0x00, 0x96, 0x7C, /* 0xF8-0xFB */ + 0x97, 0x83, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_80[512] = { + 0x97, 0x73, 0x98, 0x56, 0x00, 0x00, 0x8D, 0x6C, /* 0x00-0x03 */ + 0xE3, 0xCC, 0x8E, 0xD2, 0xE3, 0xCB, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE3, 0xCD, /* 0x08-0x0B */ + 0x8E, 0xA7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x91, 0xCF, 0x00, 0x00, 0xE3, 0xCE, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x8D, 0x6B, 0x00, 0x00, 0x96, 0xD5, /* 0x14-0x17 */ + 0xE3, 0xCF, 0xE3, 0xD0, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0xE3, 0xD1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0xE3, 0xD2, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0xE3, 0xD3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8E, 0xA8, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x96, 0xEB, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE3, 0xD5, /* 0x38-0x3B */ + 0x00, 0x00, 0x92, 0x5E, 0x00, 0x00, 0xE3, 0xD4, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xD7, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xD6, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xD8, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x90, 0xB9, 0x00, 0x00, /* 0x54-0x57 */ + 0xE3, 0xD9, 0x00, 0x00, 0xE3, 0xDA, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x95, 0xB7, 0xE3, 0xDB, /* 0x5C-0x5F */ + 0x00, 0x00, 0x91, 0x8F, 0xE3, 0xDC, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0xE3, 0xDD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x97, 0xFC, /* 0x6C-0x6F */ + 0xE3, 0xE0, 0x00, 0x00, 0xE3, 0xDF, 0xE3, 0xDE, /* 0x70-0x73 */ + 0x92, 0xAE, 0x00, 0x00, 0xE3, 0xE1, 0x90, 0x45, /* 0x74-0x77 */ + 0x00, 0x00, 0xE3, 0xE2, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0xE3, 0xE3, 0x98, 0x57, 0xE3, 0xE4, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0xE3, 0xE5, 0xE3, 0xE7, 0xE3, 0xE6, 0x94, 0xA3, /* 0x84-0x87 */ + 0x00, 0x00, 0x93, 0xF7, 0x00, 0x00, 0x98, 0x5D, /* 0x88-0x8B */ + 0x94, 0xA7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE3, 0xE9, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x8F, 0xD1, 0x00, 0x00, /* 0x94-0x97 */ + 0x95, 0x49, 0x00, 0x00, 0xE3, 0xEA, 0xE3, 0xE8, /* 0x98-0x9B */ + 0x00, 0x00, 0x8A, 0xCC, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x8C, 0xD2, 0x8E, 0x88, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x94, 0xEC, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x8C, 0xA8, 0x96, 0x62, 0x00, 0x00, /* 0xA8-0xAB */ + 0xE3, 0xED, 0xE3, 0xEB, 0x00, 0x00, 0x8D, 0x6D, /* 0xAC-0xAF */ + 0x00, 0x00, 0x8D, 0x6E, 0x88, 0xE7, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x8D, 0xE6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x94, 0x78, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0xDD, /* 0xC0-0xC3 */ + 0xE3, 0xF2, 0x00, 0x00, 0x92, 0x5F, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x94, 0x77, 0x00, 0x00, 0x91, 0xD9, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xF4, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xE3, 0xF0, 0xE3, 0xF3, 0xE3, 0xEE, /* 0xD8-0xDB */ + 0x00, 0x00, 0xE3, 0xF1, 0x96, 0x45, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x8C, 0xD3, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x88, 0xFB, 0xE3, 0xEF, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE3, 0xF6, /* 0xEC-0xEF */ + 0x00, 0x00, 0xE3, 0xF7, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x93, 0xB7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x8B, 0xB9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0xE4, 0x45, 0x94, 0x5C, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_81[512] = { + 0x00, 0x00, 0x00, 0x00, 0x8E, 0x89, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x8B, 0xBA, 0x90, 0xC6, 0x98, 0x65, /* 0x04-0x07 */ + 0x96, 0xAC, 0xE3, 0xF5, 0x90, 0xD2, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x8B, 0x72, 0xE3, 0xF8, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE3, 0xFA, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0xE3, 0xF9, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE3, 0xFB, /* 0x2C-0x2F */ + 0x00, 0x00, 0x92, 0x45, 0x00, 0x00, 0x94, 0x5D, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x92, 0xAF, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0x42, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0x41, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE3, 0xFC, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x90, 0x74, 0x00, 0x00, /* 0x4C-0x4F */ + 0x95, 0x85, 0xE4, 0x44, 0x00, 0x00, 0xE4, 0x43, /* 0x50-0x53 */ + 0x8D, 0x6F, 0x98, 0x72, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0x54, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0xE4, 0x48, 0xE4, 0x49, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8E, 0xEE, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0x47, 0x00, 0x00, /* 0x6C-0x6F */ + 0x8D, 0x98, 0xE4, 0x46, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0xE4, 0x4A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x92, 0xB0, 0x95, 0xA0, 0x91, 0x42, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x91, 0xDA, /* 0x7C-0x7F */ + + 0xE4, 0x4E, 0x00, 0x00, 0xE4, 0x4F, 0xE4, 0x4B, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0xE4, 0x4C, 0x00, 0x00, 0xE4, 0x4D, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8D, 0x70, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0x55, /* 0x90-0x93 */ + 0x00, 0x00, 0xE4, 0x51, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x95, 0x86, 0x00, 0x00, /* 0x98-0x9B */ + 0x96, 0x8C, 0x95, 0x47, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0xE4, 0x50, 0x00, 0x00, 0x00, 0x00, 0xE4, 0x53, /* 0xA0-0xA3 */ + 0xE4, 0x52, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x96, 0x63, 0xE4, 0x56, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0xE4, 0x57, 0x00, 0x00, 0x00, 0x00, 0x91, 0x56, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xE4, 0x58, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xE4, 0x5A, 0x00, 0x00, 0xE4, 0x5E, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0xE4, 0x5B, 0xE4, 0x59, 0x94, 0x5E, /* 0xBC-0xBF */ + 0xE4, 0x5C, 0x00, 0x00, 0xE4, 0x5D, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0xB0, 0x00, 0x00, /* 0xC4-0xC7 */ + 0xE4, 0x64, 0xE4, 0x5F, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0xE4, 0x60, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0xE4, 0x61, 0x00, 0x00, 0x91, 0x9F, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xE4, 0x63, 0xE4, 0x62, 0xE4, 0x65, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0x66, /* 0xDC-0xDF */ + 0xE4, 0x67, 0x00, 0x00, 0x00, 0x00, 0x90, 0x62, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x89, 0xE7, 0x00, 0x00, 0xE4, 0x68, /* 0xE4-0xE7 */ + 0x97, 0xD5, 0x00, 0x00, 0x8E, 0xA9, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x8F, 0x4C, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8E, 0x8A, /* 0xF0-0xF3 */ + 0x92, 0x76, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0x69, 0xE4, 0x6A, /* 0xF8-0xFB */ + 0x89, 0x50, 0x00, 0x00, 0xE4, 0x6B, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_82[512] = { + 0x00, 0x00, 0xE4, 0x6C, 0xE4, 0x6D, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0xE4, 0x6E, 0x00, 0x00, 0xE4, 0x6F, /* 0x04-0x07 */ + 0x8B, 0xBB, 0x9D, 0xA8, 0xE4, 0x70, 0x00, 0x00, /* 0x08-0x0B */ + 0x90, 0xE3, 0xE4, 0x71, 0x8E, 0xC9, 0x00, 0x00, /* 0x0C-0x0F */ + 0xE4, 0x72, 0x00, 0x00, 0x98, 0xAE, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0x73, 0x95, 0xDC, /* 0x14-0x17 */ + 0x8A, 0xDA, 0x00, 0x00, 0x00, 0x00, 0x91, 0x43, /* 0x18-0x1B */ + 0x8F, 0x77, 0x00, 0x00, 0x95, 0x91, 0x8F, 0x4D, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0xE4, 0x74, 0x8D, 0x71, 0xE4, 0x75, /* 0x28-0x2B */ + 0x94, 0xCA, 0x00, 0x00, 0xE4, 0x84, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0x77, /* 0x30-0x33 */ + 0x00, 0x00, 0x91, 0xC7, 0x94, 0x95, 0x8C, 0xBD, /* 0x34-0x37 */ + 0xE4, 0x76, 0x91, 0x44, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0xE4, 0x78, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0xF8, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0xE4, 0x7A, 0xE4, 0x79, 0xE4, 0x7C, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0xE4, 0x7B, 0x00, 0x00, 0xE4, 0x7D, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0x80, 0x00, 0x00, /* 0x60-0x63 */ + 0xE4, 0x7E, 0x00, 0x00, 0x8A, 0xCD, 0x00, 0x00, /* 0x64-0x67 */ + 0xE4, 0x81, 0x00, 0x00, 0xE4, 0x82, 0xE4, 0x83, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x8D, 0xAF, 0x97, 0xC7, /* 0x6C-0x6F */ + 0x00, 0x00, 0xE4, 0x85, 0x90, 0x46, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0x90, 0xE4, 0x86, /* 0x74-0x77 */ + 0xE4, 0x87, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0x88, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0xF0, /* 0x88-0x8B */ + 0x00, 0x00, 0xE4, 0x89, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0x8A, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x95, 0x87, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x8E, 0xC5, 0x00, 0x00, 0xE4, 0x8C, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x8A, 0x48, 0x88, 0xB0, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0x8B, /* 0xA8-0xAB */ + 0xE4, 0x8E, 0x94, 0x6D, 0x00, 0x00, 0x90, 0x63, /* 0xAC-0xAF */ + 0x00, 0x00, 0x89, 0xD4, 0x00, 0x00, 0x96, 0x46, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x8C, 0x7C, 0x8B, 0xDA, 0x00, 0x00, 0xE4, 0x8D, /* 0xB8-0xBB */ + 0x00, 0x00, 0x89, 0xE8, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x8A, 0xA1, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x89, 0x91, 0xE4, 0x92, 0x97, 0xE8, /* 0xD0-0xD3 */ + 0x91, 0xDB, 0x00, 0x00, 0x00, 0x00, 0x95, 0x63, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xE4, 0x9E, 0x00, 0x00, 0x89, 0xD5, /* 0xD8-0xDB */ + 0xE4, 0x9C, 0x00, 0x00, 0xE4, 0x9A, 0xE4, 0x91, /* 0xDC-0xDF */ + 0x00, 0x00, 0xE4, 0x8F, 0x00, 0x00, 0xE4, 0x90, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x8E, 0xE1, 0x8B, 0xEA, 0x92, 0x97, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0xCF, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x89, 0x70, 0x00, 0x00, 0xE4, 0x94, /* 0xF0-0xF3 */ + 0xE4, 0x93, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xE4, 0x99, 0xE4, 0x95, 0xE4, 0x98, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_83[512] = { + 0x00, 0x00, 0xEE, 0x76, 0x96, 0xCE, 0xE4, 0x97, /* 0x00-0x03 */ + 0x89, 0xD6, 0x8A, 0x9D, 0xE4, 0x9B, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xE4, 0x9D, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x8C, 0x73, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0xA1, 0xE4, 0xAA, /* 0x14-0x17 */ + 0xE4, 0xAB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x88, 0xA9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xB2, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x88, 0xEF, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xA9, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xA8, /* 0x2C-0x2F */ + 0x00, 0x00, 0xE4, 0xA3, 0xE4, 0xA2, 0x00, 0x00, /* 0x30-0x33 */ + 0xE4, 0xA0, 0xE4, 0x9F, 0x92, 0x83, 0x00, 0x00, /* 0x34-0x37 */ + 0x91, 0xF9, 0xE4, 0xA5, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0xE4, 0xA4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0xE4, 0xA7, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x91, 0x90, 0x8C, 0x74, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x89, 0x60, /* 0x4C-0x4F */ + 0xE4, 0xA6, 0x00, 0x00, 0x8D, 0x72, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x91, 0x91, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0xEE, 0x77, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xB8, /* 0x70-0x73 */ + 0x00, 0x00, 0xE4, 0xB9, 0x00, 0x00, 0x89, 0xD7, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x89, 0xAC, /* 0x78-0x7B */ + 0xE4, 0xB6, 0x00, 0x00, 0x00, 0x00, 0xEE, 0x78, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0xE4, 0xAC, 0x00, 0x00, 0xE4, 0xB4, /* 0x84-0x87 */ + 0x00, 0x00, 0xE4, 0xBB, 0xE4, 0xB5, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0xB3, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0x96, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0xB1, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0xAD, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x8A, 0xCE, 0xE4, 0xAF, /* 0x9C-0x9F */ + 0xE4, 0xBA, 0x00, 0x00, 0xE4, 0xB0, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xE4, 0xBC, 0x00, 0x00, 0xE4, 0xAE, 0x94, 0x9C, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x97, 0x89, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xE4, 0xB7, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0xE4, 0xCD, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0xE4, 0xC5, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x90, 0x9B, 0x00, 0x00, 0xEE, 0x79, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x8B, 0x65, 0x00, 0x00, /* 0xC8-0xCB */ + 0x8B, 0xDB, 0x00, 0x00, 0xE4, 0xC0, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x89, 0xD9, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x8F, 0xD2, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xE4, 0xC3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x8D, 0xD8, 0x00, 0x00, 0x00, 0x00, 0x93, 0x70, /* 0xDC-0xDF */ + 0xE4, 0xC8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x95, 0xEC, 0x00, 0x00, 0xE4, 0xBF, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x89, 0xD8, /* 0xEC-0xEF */ + 0x8C, 0xD4, 0x95, 0x48, 0xE4, 0xC9, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xE4, 0xBD, 0x00, 0x00, 0xEE, 0x7A, 0xE4, 0xC6, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xD0, /* 0xF8-0xFB */ + 0x00, 0x00, 0xE4, 0xC1, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_84[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xC2, /* 0x00-0x03 */ + 0x93, 0xB8, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xC7, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xC4, /* 0x08-0x0B */ + 0x96, 0x47, 0xE4, 0xCA, 0x88, 0xDE, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xBE, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0xE4, 0xCC, 0x00, 0x00, 0xE4, 0xCB, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x94, 0x8B, 0xE4, 0xD2, 0x00, 0x00, /* 0x28-0x2B */ + 0xE4, 0xDD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x8A, 0x9E, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0xE4, 0xE0, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0xE4, 0xCE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0xE4, 0xD3, 0x97, 0x8E, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0xDC, 0x00, 0x00, /* 0x44-0x47 */ + 0xEE, 0x7B, 0x97, 0x74, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x97, 0xA8, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0x98, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8A, 0x8B, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x95, 0x92, 0xE4, 0xE2, 0x93, 0x9F, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x88, 0xAF, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0xE4, 0xDB, 0x00, 0x00, 0xE4, 0xD7, /* 0x68-0x6B */ + 0x91, 0x92, 0xE4, 0xD1, 0xE4, 0xD9, 0xE4, 0xDE, /* 0x6C-0x6F */ + 0x00, 0x00, 0x94, 0x4B, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x88, 0xA8, 0x00, 0x00, 0xE4, 0xD6, /* 0x74-0x77 */ + 0x00, 0x00, 0xE4, 0xDF, 0x95, 0x98, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0xE4, 0xDA, 0x00, 0x00, /* 0x80-0x83 */ + 0xE4, 0xD5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8F, 0xD3, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x8F, 0x4E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x8E, 0xAA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x96, 0xD6, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x95, 0x66, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xE5, /* 0x9C-0x9F */ + 0x00, 0x00, 0xE4, 0xEE, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0xE4, 0xD8, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x8A, 0x97, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xEE, 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x8F, 0xF6, 0xE4, 0xE3, 0x00, 0x00, 0xE4, 0xE8, /* 0xB8-0xBB */ + 0x91, 0x93, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xE4, /* 0xBC-0xBF */ + 0x00, 0x00, 0xE4, 0xEB, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x92, 0x7E, 0x00, 0x00, 0xE4, 0xEC, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x97, 0x75, 0xE4, 0xE1, 0x8A, 0x57, /* 0xC8-0xCB */ + 0x00, 0x00, 0xE4, 0xE7, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0xE4, 0xEA, 0x96, 0xAA, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0xED, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xE4, 0xE6, 0xE4, 0xE9, 0x00, 0x00, /* 0xD8-0xDB */ + 0xED, 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x96, 0x48, 0x00, 0x00, 0x98, 0x40, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xE4, 0xF1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0xE4, 0xF8, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xF0, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_85[512] = { + 0x8E, 0xC1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0xCF, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x95, 0xCC, 0x00, 0x00, 0x96, 0xA0, /* 0x10-0x13 */ + 0xE4, 0xF7, 0xE4, 0xF6, 0x00, 0x00, 0xE4, 0xF2, /* 0x14-0x17 */ + 0xE4, 0xF3, 0x00, 0x00, 0x89, 0x55, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xF5, /* 0x1C-0x1F */ + 0x00, 0x00, 0xE4, 0xEF, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x92, 0xD3, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0xE4, 0xF4, 0x88, 0xFC, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x91, 0xA0, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x95, 0xC1, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0xE4, 0xF9, 0xE5, 0x40, 0x00, 0x00, 0x94, 0xD7, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0xE4, 0xFC, 0x8F, 0xD4, 0x8E, 0xC7, 0xE5, 0x42, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x8B, 0xBC, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEE, 0x7D, /* 0x50-0x53 */ + 0x00, 0x00, 0xE5, 0x43, 0x00, 0x00, 0x95, 0x99, /* 0x54-0x57 */ + 0xE4, 0xFB, 0xEE, 0x7E, 0xE4, 0xD4, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xFA, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x98, 0x6E, 0x93, 0xA0, 0x95, 0x93, 0xEE, 0x80, /* 0x68-0x6B */ + 0x00, 0x00, 0xE5, 0x4A, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE5, 0x50, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0x51, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xE5, 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x94, 0x96, 0x00, 0x00, 0x00, 0x00, 0xE5, 0x4E, /* 0x84-0x87 */ + 0xE5, 0x46, 0x00, 0x00, 0xE5, 0x48, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0xE5, 0x52, 0xE5, 0x47, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0xE5, 0x4B, 0x00, 0x00, 0x00, 0x00, 0x89, 0x92, /* 0x94-0x97 */ + 0x00, 0x00, 0x93, 0xE3, 0x00, 0x00, 0xE5, 0x4C, /* 0x98-0x9B */ + 0xE5, 0x4F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xE5, 0x45, 0x00, 0x00, 0x91, 0x45, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xE5, 0x49, 0x8E, 0x46, 0x90, 0x64, 0x8C, 0x4F, /* 0xA8-0xAB */ + 0x96, 0xF2, 0x00, 0x00, 0x96, 0xF7, 0x8F, 0x92, /* 0xAC-0xAF */ + 0xEE, 0x82, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xE5, 0x56, 0xE5, 0x54, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x98, 0x6D, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xE5, 0x53, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x97, 0x95, 0x00, 0x00, 0xE5, 0x55, /* 0xCC-0xCF */ + 0xE5, 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xE5, 0x58, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0xE5, 0x5B, 0xE5, 0x59, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x93, 0xA1, 0xE5, 0x5A, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x94, 0xCB, 0xE5, 0x4D, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8F, 0x93, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xE5, 0x5C, 0xE5, 0x61, 0x91, 0x94, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0x60, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_86[512] = { + 0x00, 0x00, 0x00, 0x00, 0xE5, 0x41, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0x62, 0x91, 0x68, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0x5D, 0xE5, 0x5F, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE5, 0x5E, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x9F, 0x50, 0x9F, 0x41, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0x64, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0x63, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x97, 0x96, 0x00, 0x00, 0xE1, 0xBA, /* 0x2C-0x2F */ + 0xE5, 0x65, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE5, 0x66, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0xE5, 0x67, 0x8C, 0xD5, 0x00, 0x00, /* 0x4C-0x4F */ + 0x8B, 0x73, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0xE5, 0x69, 0x99, 0x7C, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x8B, 0x95, 0x00, 0x00, /* 0x58-0x5B */ + 0x97, 0xB8, 0x00, 0x00, 0x8B, 0xF1, 0xE5, 0x6A, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE5, 0x6B, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0x8E, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0xE5, 0x6C, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x93, 0xF8, 0x00, 0x00, 0x88, 0xB8, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0xE1, 0xE5, 0x71, /* 0x88-0x8B */ + 0xE5, 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE5, 0x6D, /* 0x90-0x93 */ + 0x00, 0x00, 0x8E, 0x5C, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE5, 0x6E, /* 0xA0-0xA3 */ + 0x94, 0x61, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0xE5, 0x6F, 0xE5, 0x70, 0xE5, 0x7A, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE5, 0x74, /* 0xAC-0xAF */ + 0xE5, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0x73, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xE5, 0x75, 0x00, 0x00, 0xE5, 0x76, 0x8E, 0xD6, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xE5, 0x78, 0x00, 0x00, 0x92, 0x60, /* 0xC8-0xCB */ + 0x00, 0x00, 0x8C, 0x75, 0x8A, 0x61, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xE5, 0x7B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x8A, 0x5E, 0x00, 0x00, 0xE5, 0x81, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0x7C, 0xE5, 0x80, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x94, 0xB8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xE5, 0x7D, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0xE5, 0x7E, 0x95, 0x67, 0x94, 0xD8, 0xE5, 0x82, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x91, 0xFB, 0xE5, 0x8C, 0x00, 0x00, 0xE5, 0x88, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0xE9, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_87[512] = { + 0xE5, 0x86, 0x00, 0x00, 0x96, 0x49, 0xE5, 0x87, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0x84, 0x00, 0x00, /* 0x04-0x07 */ + 0xE5, 0x85, 0xE5, 0x8A, 0xE5, 0x8D, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0xE5, 0x8B, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0xE5, 0x89, 0xE5, 0x83, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x92, 0x77, 0x00, 0x00, 0xE5, 0x94, 0x00, 0x00, /* 0x18-0x1B */ + 0x96, 0xA8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0xE5, 0x92, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0xE5, 0x93, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0xE5, 0x8E, 0x00, 0x00, 0x00, 0x00, 0xE5, 0x90, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE5, 0x91, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE5, 0x8F, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x90, 0xE4, 0x00, 0x00, 0x98, 0x58, /* 0x48-0x4B */ + 0xE5, 0x98, 0x00, 0x00, 0xE5, 0x99, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE5, 0x9F, /* 0x50-0x53 */ + 0x00, 0x00, 0x90, 0x49, 0x00, 0x00, 0xE5, 0x9B, /* 0x54-0x57 */ + 0x00, 0x00, 0xE5, 0x9E, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE5, 0x96, /* 0x5C-0x5F */ + 0xE5, 0x95, 0x00, 0x00, 0x00, 0x00, 0xE5, 0xA0, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0xDA, 0x00, 0x00, /* 0x64-0x67 */ + 0xE5, 0x9C, 0x00, 0x00, 0xE5, 0xA1, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0x9D, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0xE5, 0x9A, 0x00, 0x00, 0x92, 0xB1, 0x00, 0x00, /* 0x74-0x77 */ + 0xE5, 0x97, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x94, 0x88, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xA5, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x97, 0x5A, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE5, 0xA4, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xA3, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE5, 0xAC, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE5, 0xA6, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE5, 0xAE, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x97, 0x86, 0xE5, 0xB1, /* 0xB8-0xBB */ + 0x00, 0x00, 0xE5, 0xA8, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0xE5, 0xA9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xE5, 0xAD, 0x00, 0x00, 0xE5, 0xB0, 0xE5, 0xAF, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE5, 0xA7, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0xE5, 0xAA, 0x00, 0x00, 0xE5, 0xBB, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0xE5, 0xB4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE5, 0xB2, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xB3, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xB8, 0xE5, 0xB9, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x8A, 0x49, 0x00, 0x00, 0x8B, 0x61, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xB7, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_88[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0xE5, 0xA2, 0x00, 0x00, 0xEE, 0x85, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0xE5, 0xB6, 0xE5, 0xBA, 0xE5, 0xB5, /* 0x0C-0x0F */ + 0x00, 0x00, 0xE5, 0xBC, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0xE5, 0xBE, 0xE5, 0xBD, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0xE5, 0xC0, 0xE5, 0xBF, 0xE5, 0x79, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE5, 0xC4, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0xE5, 0xC1, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xC2, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0xE5, 0xC3, 0x00, 0x00, 0xE5, 0xC5, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x8C, 0x8C, 0x00, 0x00, 0xE5, 0xC7, 0x00, 0x00, /* 0x40-0x43 */ + 0xE5, 0xC6, 0x00, 0x00, 0x8F, 0x4F, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x8D, 0x73, 0x9F, 0xA5, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xC8, 0x8F, 0x70, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8A, 0x58, /* 0x54-0x57 */ + 0x00, 0x00, 0xE5, 0xC9, 0x00, 0x00, 0x89, 0x71, /* 0x58-0x5B */ + 0x00, 0x00, 0x8F, 0xD5, 0xE5, 0xCA, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x8D, 0x74, 0xE5, 0xCB, 0x88, 0xDF, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x95, 0x5C, 0x00, 0x00, 0x00, 0x00, 0xE5, 0xCC, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x90, 0x8A, 0x00, 0x00, 0xE5, 0xD3, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0xE5, 0xD0, 0x00, 0x00, 0x92, 0x8F, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0xE5, 0xD1, 0xE5, 0xCE, 0x8B, 0xDC, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xE5, 0xCD, 0xE5, 0xD4, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x8C, 0x55, 0x00, 0x00, 0x00, 0x00, 0x91, 0xDC, /* 0x88-0x8B */ + 0x00, 0x00, 0xE5, 0xDA, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xD6, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x91, 0xB3, 0xE5, 0xD5, /* 0x94-0x97 */ + 0x00, 0x00, 0xE5, 0xD8, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xCF, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xD9, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xE5, 0xDB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x94, 0xED, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xD7, 0x00, 0x00, /* 0xAC-0xAF */ + 0xE5, 0xDC, 0xE5, 0xDE, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x8C, 0xD1, 0xE5, 0xD2, 0x00, 0x00, 0x88, 0xBF, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE5, 0xDD, /* 0xBC-0xBF */ + 0x00, 0x00, 0x8D, 0xD9, 0x97, 0xF4, 0xE5, 0xDF, /* 0xC0-0xC3 */ + 0xE5, 0xE0, 0x91, 0x95, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x97, 0xA0, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xE5, 0xE1, 0x97, 0x54, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xE5, 0xE2, 0xE5, 0xE3, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x95, 0xE2, 0xE5, 0xE4, 0x00, 0x00, 0x8D, 0xBE, /* 0xDC-0xDF */ + 0x00, 0x00, 0x97, 0xA1, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xE5, 0xE9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xEA, 0x8F, 0xD6, /* 0xF0-0xF3 */ + 0xE5, 0xE8, 0xEE, 0x86, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x97, 0x87, 0xE5, 0xE5, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0xE5, 0xE7, 0x90, 0xBB, 0x90, 0x9E, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_89[512] = { + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xE6, 0x00, 0x00, /* 0x00-0x03 */ + 0xE5, 0xEB, 0x00, 0x00, 0x00, 0x00, 0x95, 0xA1, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xED, 0x00, 0x00, /* 0x08-0x0B */ + 0xE5, 0xEC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x8A, 0x8C, 0x00, 0x00, 0x96, 0x4A, 0xE5, 0xEE, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0xED, 0x41, 0xE5, 0xFA, 0xE5, 0xF0, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0xE5, 0xF1, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xF2, 0xE5, 0xF3, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xF7, 0x00, 0x00, /* 0x34-0x37 */ + 0xE5, 0xF8, 0x00, 0x00, 0x00, 0x00, 0xE5, 0xF6, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0xE5, 0xF4, 0x00, 0x00, 0xE5, 0xEF, /* 0x40-0x43 */ + 0xE5, 0xF5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0xE5, 0xF9, 0xE8, 0xB5, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0xA6, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xFC, 0x8B, 0xDD, /* 0x5C-0x5F */ + 0xE5, 0xFB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0xE6, 0x41, 0x00, 0x00, 0xE6, 0x40, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0x43, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0xE6, 0x42, 0x00, 0x00, 0xE6, 0x44, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x8F, 0x50, 0x00, 0x00, /* 0x70-0x73 */ + 0xE6, 0x45, 0x00, 0x00, 0x00, 0x00, 0xE6, 0x46, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0x47, 0x90, 0xBC, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x97, 0x76, 0x00, 0x00, 0xE6, 0x48, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x95, 0xA2, 0x94, 0x65, /* 0x84-0x87 */ + 0xE6, 0x49, 0x00, 0x00, 0xE6, 0x4A, 0x8C, 0xA9, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8B, 0x4B, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE6, 0x4B, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x8E, 0x8B, 0x94, 0x60, /* 0x94-0x97 */ + 0xE6, 0x4C, 0x00, 0x00, 0x8A, 0x6F, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0xE6, 0x4D, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0x4F, 0x97, 0x97, /* 0xA4-0xA7 */ + 0x00, 0x00, 0xE6, 0x4E, 0x90, 0x65, 0x00, 0x00, /* 0xA8-0xAB */ + 0xE6, 0x50, 0x00, 0x00, 0x00, 0x00, 0xE6, 0x51, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0x52, 0x8A, 0xCF, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0x53, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0xE6, 0x54, 0x00, 0x00, 0xE6, 0x55, /* 0xBC-0xBF */ + 0xE6, 0x56, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x8A, 0x70, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0x57, 0x00, 0x00, /* 0xD8-0xDB */ + 0xE6, 0x58, 0xE6, 0x59, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x89, 0xF0, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x90, 0x47, 0xE6, 0x5A, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xE6, 0x5B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xE6, 0x5C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_8A[512] = { + 0x8C, 0xBE, 0x00, 0x00, 0x92, 0xF9, 0xE6, 0x5D, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x8C, 0x76, 0x00, 0x00, 0x90, 0x75, 0x00, 0x00, /* 0x08-0x0B */ + 0xE6, 0x60, 0x00, 0x00, 0x93, 0xA2, 0x00, 0x00, /* 0x0C-0x0F */ + 0xE6, 0x5F, 0x00, 0x00, 0xEE, 0x87, 0x8C, 0x50, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0x5E, 0x91, 0xF5, /* 0x14-0x17 */ + 0x8B, 0x4C, 0x00, 0x00, 0x00, 0x00, 0xE6, 0x61, /* 0x18-0x1B */ + 0x00, 0x00, 0xE6, 0x62, 0x00, 0x00, 0x8F, 0xD7, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8C, 0x8D, /* 0x20-0x23 */ + 0x00, 0x00, 0xE6, 0x63, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x96, 0x4B, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x90, 0xDD, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x8B, 0x96, 0x00, 0x00, 0x96, 0xF3, /* 0x30-0x33 */ + 0x91, 0x69, 0x00, 0x00, 0xE6, 0x64, 0xEE, 0x88, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x90, 0x66, 0x92, 0x90, /* 0x38-0x3B */ + 0x8F, 0xD8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0xE6, 0x65, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0x68, 0x00, 0x00, /* 0x44-0x47 */ + 0xE6, 0x69, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x8D, 0xBC, 0x91, 0xC0, 0xE6, 0x67, 0x00, 0x00, /* 0x50-0x53 */ + 0x8F, 0xD9, 0x95, 0x5D, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE6, 0x66, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x8E, 0x8C, 0x00, 0x00, /* 0x5C-0x5F */ + 0x89, 0x72, 0x00, 0x00, 0xE6, 0x6D, 0x8C, 0x77, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x8E, 0x8E, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x8E, 0x8D, 0x00, 0x00, 0x98, 0x6C, /* 0x68-0x6B */ + 0xE6, 0x6C, 0xE6, 0x6B, 0x91, 0x46, 0x00, 0x00, /* 0x6C-0x6F */ + 0x8B, 0x6C, 0x98, 0x62, 0x8A, 0x59, 0x8F, 0xDA, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0xEE, 0x89, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0xE6, 0x6A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0xE6, 0x6F, 0x00, 0x00, /* 0x80-0x83 */ + 0xE6, 0x70, 0xE6, 0x6E, 0x00, 0x00, 0x8C, 0xD6, /* 0x84-0x87 */ + 0x00, 0x00, 0x97, 0x5F, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x8E, 0x8F, 0x94, 0x46, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xE6, 0x73, 0x00, 0x00, 0x90, 0xBE, /* 0x90-0x93 */ + 0x00, 0x00, 0x92, 0x61, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x97, 0x55, 0x00, 0x00, 0xE6, 0x76, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x8C, 0xEA, 0x00, 0x00, /* 0x9C-0x9F */ + 0x90, 0xBD, 0xE6, 0x72, 0x00, 0x00, 0xE6, 0x77, /* 0xA0-0xA3 */ + 0x8C, 0xEB, 0xE6, 0x74, 0xE6, 0x75, 0xEE, 0x8A, /* 0xA4-0xA7 */ + 0xE6, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x90, 0xE0, 0x93, 0xC7, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x92, 0x4E, 0x00, 0x00, 0x89, 0xDB, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x94, 0xEE, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x8B, 0x62, 0x00, 0x00, 0xEE, 0x8B, 0x92, 0xB2, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0x7A, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xE6, 0x78, 0x00, 0x00, 0x00, 0x00, 0x92, 0x6B, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0xBF, /* 0xC8-0xCB */ + 0x8A, 0xD0, 0xE6, 0x79, 0x00, 0x00, 0x90, 0x7A, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x97, 0xC8, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x98, 0x5F, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0x7B, 0xE6, 0x87, /* 0xD8-0xDB */ + 0x92, 0xB3, 0x00, 0x00, 0xE6, 0x86, 0xEE, 0x8C, /* 0xDC-0xDF */ + 0xE6, 0x83, 0xE6, 0x8B, 0xE6, 0x84, 0x00, 0x00, /* 0xE0-0xE3 */ + 0xE6, 0x80, 0x00, 0x00, 0x92, 0xFA, 0xE6, 0x7E, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE6, 0x7C, /* 0xE8-0xEB */ + 0x00, 0x00, 0x97, 0x40, 0x8E, 0x90, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0xE6, 0x81, 0x00, 0x00, 0xE6, 0x7D, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0xEE, 0x8E, 0xE6, 0x85, /* 0xF4-0xF7 */ + 0x8F, 0x94, 0x00, 0x00, 0x8C, 0xBF, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0x91, 0xF8, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_8B[512] = { + 0x96, 0x64, 0x89, 0x79, 0x88, 0xE0, 0x00, 0x00, /* 0x00-0x03 */ + 0x93, 0xA3, 0x00, 0x00, 0x00, 0x00, 0xE6, 0x89, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0xE6, 0x88, 0x00, 0x00, 0x93, 0xE4, 0x00, 0x00, /* 0x0C-0x0F */ + 0xE6, 0x8D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0xE6, 0x82, 0x00, 0x00, 0xE6, 0x8C, 0xE6, 0x8E, /* 0x14-0x17 */ + 0x00, 0x00, 0x8C, 0xAA, 0xE6, 0x8A, 0x8D, 0x75, /* 0x18-0x1B */ + 0x00, 0x00, 0x8E, 0xD3, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0xE6, 0x8F, 0x97, 0x77, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0x92, 0x00, 0x00, /* 0x24-0x27 */ + 0xE6, 0x95, 0x00, 0x00, 0x00, 0x00, 0xE6, 0x93, /* 0x28-0x2B */ + 0x95, 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE6, 0x90, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x8B, 0xDE, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0x94, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0xE6, 0x96, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0xE6, 0x9A, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0xE6, 0x97, 0x00, 0x00, 0xE6, 0x99, 0xE6, 0x98, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEE, 0x8F, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0x9B, 0x00, 0x00, /* 0x54-0x57 */ + 0x8E, 0xAF, 0x00, 0x00, 0xE6, 0x9D, 0xE6, 0x9C, /* 0x58-0x5B */ + 0x95, 0x88, 0x00, 0x00, 0x00, 0x00, 0xE6, 0x9F, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x8C, 0x78, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE6, 0x9E, /* 0x68-0x6B */ + 0xE6, 0xA0, 0x00, 0x00, 0x00, 0x00, 0xE6, 0xA1, /* 0x6C-0x6F */ + 0x8B, 0x63, 0xE3, 0xBF, 0x8F, 0xF7, 0x00, 0x00, /* 0x70-0x73 */ + 0xE6, 0xA2, 0x00, 0x00, 0x00, 0x00, 0x8C, 0xEC, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0xE6, 0xA3, 0x00, 0x00, 0xEE, 0x90, /* 0x7C-0x7F */ + + 0xE6, 0xA4, 0x00, 0x00, 0x00, 0x00, 0x8E, 0x5D, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x9D, 0xCC, 0x00, 0x00, /* 0x88-0x8B */ + 0xE6, 0xA5, 0x00, 0x00, 0xE6, 0xA6, 0x00, 0x00, /* 0x8C-0x8F */ + 0x8F, 0x51, 0x00, 0x00, 0xE6, 0xA7, 0xE6, 0xA8, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0xA9, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0xE6, 0xAA, 0xE6, 0xAB, 0x00, 0x00, /* 0x98-0x9B */ +}; + +static const unsigned char u2c_8C[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0x4A, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0xAC, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE6, 0xAE, /* 0x3C-0x3F */ + 0x00, 0x00, 0xE6, 0xAD, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x93, 0xA4, 0x00, 0x00, /* 0x44-0x47 */ + 0xE6, 0xAF, 0x00, 0x00, 0x96, 0x4C, 0x00, 0x00, /* 0x48-0x4B */ + 0xE6, 0xB0, 0x00, 0x00, 0xE6, 0xB1, 0x00, 0x00, /* 0x4C-0x4F */ + 0xE6, 0xB2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0xE6, 0xB3, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x93, 0xD8, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x8F, 0xDB, 0xE6, 0xB4, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x8D, 0x8B, 0x98, 0xAC, /* 0x68-0x6B */ + 0xE6, 0xB5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0xE6, 0xB6, 0x95, 0x5E, 0xE6, 0xB7, 0x00, 0x00, /* 0x78-0x7B */ + 0xE6, 0xBF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0xE6, 0xB8, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0xE6, 0xBA, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0xE6, 0xB9, 0xE6, 0xBB, 0x00, 0x00, /* 0x88-0x8B */ + 0x96, 0x65, 0xE6, 0xBC, 0xE6, 0xBD, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0xE6, 0xBE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0xE6, 0xC0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x8A, 0x4C, 0x92, 0xE5, 0x00, 0x00, /* 0x9C-0x9F */ + 0x95, 0x89, 0x8D, 0xE0, 0x8D, 0x76, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0x6E, /* 0xA4-0xA7 */ + 0x89, 0xDD, 0x94, 0xCC, 0xE6, 0xC3, 0x8A, 0xD1, /* 0xA8-0xAB */ + 0x90, 0xD3, 0xE6, 0xC2, 0xE6, 0xC7, 0x92, 0x99, /* 0xAC-0xAF */ + 0x96, 0xE1, 0x00, 0x00, 0xE6, 0xC5, 0xE6, 0xC6, /* 0xB0-0xB3 */ + 0x8B, 0x4D, 0x00, 0x00, 0xE6, 0xC8, 0x94, 0x83, /* 0xB4-0xB7 */ + 0x91, 0xDD, 0x00, 0x00, 0x00, 0x00, 0x94, 0xEF, /* 0xB8-0xBB */ + 0x93, 0x5C, 0xE6, 0xC4, 0x00, 0x00, 0x96, 0x66, /* 0xBC-0xBF */ + 0x89, 0xEA, 0xE6, 0xCA, 0x98, 0x47, 0x92, 0xC0, /* 0xC0-0xC3 */ + 0x98, 0x64, 0x00, 0x00, 0x00, 0x00, 0x8E, 0x91, /* 0xC4-0xC7 */ + 0xE6, 0xC9, 0x00, 0x00, 0x91, 0xAF, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0xE6, 0xDA, 0x91, 0x47, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x93, 0xF6, 0x00, 0x00, 0x95, 0x6F, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0xCD, 0x8E, 0x5E, /* 0xD8-0xDB */ + 0x8E, 0x92, 0x00, 0x00, 0x8F, 0xDC, 0x00, 0x00, /* 0xDC-0xDF */ + 0x94, 0x85, 0x00, 0x00, 0x8C, 0xAB, 0xE6, 0xCC, /* 0xE0-0xE3 */ + 0xE6, 0xCB, 0x00, 0x00, 0x95, 0x8A, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x8E, 0xBF, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x93, 0x71, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0xEE, 0x91, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xEE, 0x92, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0xCF, 0xE6, 0xD0, /* 0xF8-0xFB */ + 0x8D, 0x77, 0xE6, 0xCE, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_8D[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0xE6, 0xD1, 0xE6, 0xD2, 0x00, 0x00, 0xE6, 0xD4, /* 0x04-0x07 */ + 0x91, 0xA1, 0x00, 0x00, 0xE6, 0xD3, 0x8A, 0xE4, /* 0x08-0x0B */ + 0x00, 0x00, 0xE6, 0xD6, 0x00, 0x00, 0xE6, 0xD5, /* 0x0C-0x0F */ + 0xE6, 0xD7, 0x00, 0x00, 0xEE, 0x93, 0xE6, 0xD9, /* 0x10-0x13 */ + 0xE6, 0xDB, 0x00, 0x00, 0xE6, 0xDC, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x90, 0xD4, 0x00, 0x00, 0x8E, 0xCD, 0xE6, 0xDD, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8A, 0x71, /* 0x68-0x6B */ + 0x00, 0x00, 0xE6, 0xDE, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x91, 0x96, 0xE6, 0xDF, 0x00, 0x00, 0xE6, 0xE0, /* 0x70-0x73 */ + 0x95, 0x8B, 0x00, 0x00, 0xEE, 0x94, 0x8B, 0x4E, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xE6, 0xE1, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x92, 0xB4, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0x7A, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0xE6, 0xE2, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8E, 0xEF, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x90, 0x96, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x91, 0xAB, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0xE5, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0xE4, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0xE3, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE6, 0xEB, /* 0xC8-0xCB */ + 0xE6, 0xE9, 0x00, 0x00, 0x00, 0x00, 0xE6, 0xE6, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0xE8, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0xE7, 0xE6, 0xEA, /* 0xD8-0xDB */ + 0x00, 0x00, 0x8B, 0x97, 0x00, 0x00, 0xE6, 0xEE, /* 0xDC-0xDF */ + 0x00, 0x00, 0x90, 0xD5, 0x00, 0x00, 0xE6, 0xEF, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x8C, 0xD7, 0x00, 0x00, 0xE6, 0xEC, 0xE6, 0xED, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x48, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0xB5, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x91, 0x48, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0xE6, 0xF0, 0x00, 0x00, 0x00, 0x00, 0xE6, 0xF3, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_8E[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0xE6, 0xF1, 0xE6, 0xF2, 0x97, 0x78, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0xA5, /* 0x0C-0x0F */ + 0xE6, 0xF6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0xE6, 0xF4, 0xE6, 0xF5, 0xE6, 0xF7, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0x48, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xE6, 0xFA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0xE6, 0xFB, 0xE6, 0xF9, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0xF8, 0x00, 0x00, /* 0x40-0x43 */ + 0x92, 0xFB, 0x00, 0x00, 0x00, 0x00, 0xE7, 0x40, /* 0x44-0x47 */ + 0xE7, 0x44, 0xE7, 0x41, 0xE6, 0xFC, 0x00, 0x00, /* 0x48-0x4B */ + 0xE7, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0xE7, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0xE7, 0x4A, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0xE7, 0x45, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0xD6, /* 0x5C-0x5F */ + 0xE7, 0x47, 0x00, 0x00, 0x00, 0x00, 0xE7, 0x49, /* 0x60-0x63 */ + 0xE7, 0x46, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0x4C, 0x00, 0x00, /* 0x70-0x73 */ + 0x8F, 0x52, 0x00, 0x00, 0xE7, 0x4B, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0xE7, 0x4D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xE7, 0x4E, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0xE7, 0x51, 0xE7, 0x50, 0x00, 0x00, 0xE7, 0x4F, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0x53, 0xE7, 0x52, /* 0x88-0x8B */ + 0x00, 0x00, 0x96, 0xF4, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xE7, 0x55, 0x00, 0x00, 0xE7, 0x54, /* 0x90-0x93 */ + 0xE7, 0x56, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0xE7, 0x57, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0xE7, 0x59, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0x58, 0x90, 0x67, /* 0xA8-0xAB */ + 0xE7, 0x5A, 0x00, 0x00, 0x00, 0x00, 0x8B, 0xEB, /* 0xAC-0xAF */ + 0xE7, 0x5B, 0xE7, 0x5D, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0x5E, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xE7, 0x5F, 0xE7, 0x5C, 0x00, 0x00, /* 0xC4-0xC7 */ + 0xE7, 0x60, 0x00, 0x00, 0x8E, 0xD4, 0xE7, 0x61, /* 0xC8-0xCB */ + 0x8B, 0x4F, 0x8C, 0x52, 0x00, 0x00, 0xEE, 0x96, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x8C, 0xAC, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0x62, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0xEE, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x93, 0x5D, 0xE7, 0x63, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0x66, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x8E, 0xB2, 0x00, 0x00, 0x00, 0x00, 0xE7, 0x65, /* 0xF8-0xFB */ + 0xE7, 0x64, 0x8C, 0x79, 0xE7, 0x67, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_8F[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8A, 0x72, /* 0x00-0x03 */ + 0x00, 0x00, 0xE7, 0x69, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x8D, 0xDA, 0xE7, 0x68, 0x00, 0x00, /* 0x08-0x0B */ + 0xE7, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0x6B, 0xE7, 0x6D, /* 0x10-0x13 */ + 0x95, 0xE3, 0xE7, 0x6A, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0xE7, 0x6C, 0x00, 0x00, 0xE7, 0x70, /* 0x18-0x1B */ + 0xE7, 0x6E, 0x8B, 0x50, 0x00, 0x00, 0xE7, 0x6F, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0x72, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x94, 0x79, 0x97, 0xD6, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8F, 0x53, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0x73, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x97, 0x41, 0xE7, 0x75, 0x00, 0x00, 0xE7, 0x74, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0x78, 0x97, 0x60, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0x77, 0x00, 0x00, /* 0x40-0x43 */ + 0x8A, 0x8D, 0xE7, 0x76, 0xE7, 0x7B, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0xE7, 0x7A, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0xE7, 0x79, 0x93, 0x51, 0xE7, 0x7C, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0x7D, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0xE7, 0x7E, 0x00, 0x00, 0x00, 0x00, 0x8D, 0x8C, /* 0x5C-0x5F */ + 0x00, 0x00, 0x8C, 0x44, 0xE7, 0x80, 0xE7, 0x81, /* 0x60-0x63 */ + 0xE7, 0x82, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x68, /* 0x98-0x9B */ + 0xE7, 0x83, 0x00, 0x00, 0x8E, 0xAB, 0xE7, 0x84, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0x85, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x99, 0x9F, /* 0xA4-0xA7 */ + 0x99, 0x9E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0xE7, 0x86, 0xE3, 0x90, 0xE7, 0x87, /* 0xAC-0xAF */ + 0x92, 0x43, 0x90, 0x4A, 0x94, 0x5F, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0x88, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x95, 0xD3, 0x92, 0xD2, /* 0xB8-0xBB */ + 0x8D, 0x9E, 0x00, 0x00, 0x00, 0x00, 0x92, 0x48, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0x49, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x96, 0x98, 0x90, 0x76, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x8C, 0x7D, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x8B, 0xDF, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x95, 0xD4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0x89, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0x8B, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xE7, 0x8A, 0x89, 0xDE, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x93, 0xF4, 0xE7, 0x8C, 0x94, 0x97, /* 0xE8-0xEB */ + 0x00, 0x00, 0x93, 0x52, 0x00, 0x00, 0xE7, 0x8D, /* 0xEC-0xEF */ + 0x8F, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xE7, 0x8F, 0x00, 0x00, 0x00, 0x00, 0x96, 0xC0, /* 0xF4-0xF7 */ + 0xE7, 0x9E, 0xE7, 0x91, 0xE7, 0x92, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x92, 0xC7, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_90[512] = { + 0x91, 0xDE, 0x91, 0x97, 0x00, 0x00, 0x93, 0xA6, /* 0x00-0x03 */ + 0x00, 0x00, 0xE7, 0x90, 0x8B, 0x74, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0x99, /* 0x08-0x0B */ + 0x00, 0x00, 0xE7, 0x96, 0xE7, 0xA3, 0x93, 0xA7, /* 0x0C-0x0F */ + 0x92, 0x80, 0xE7, 0x93, 0x00, 0x00, 0x92, 0xFC, /* 0x10-0x13 */ + 0x93, 0x72, 0xE7, 0x94, 0xE7, 0x98, 0x90, 0x80, /* 0x14-0x17 */ + 0x00, 0x00, 0x94, 0x87, 0x92, 0xCA, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x90, 0xC0, 0xE7, 0x97, 0x91, 0xAC, /* 0x1C-0x1F */ + 0x91, 0xA2, 0xE7, 0x95, 0x88, 0xA7, 0x98, 0x41, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0x9A, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x91, 0xDF, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x8F, 0x54, 0x90, 0x69, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0xE7, 0x9C, 0xE7, 0x9B, 0x00, 0x00, /* 0x34-0x37 */ + 0x88, 0xED, 0xE7, 0x9D, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x95, 0x4E, 0x00, 0x00, 0xE7, 0xA5, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x93, 0xD9, 0x90, 0x8B, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x92, 0x78, 0x00, 0x00, 0x8B, 0xF6, /* 0x44-0x47 */ + 0x00, 0x00, 0xE7, 0xA4, 0x97, 0x56, 0x89, 0x5E, /* 0x48-0x4B */ + 0x00, 0x00, 0x95, 0xD5, 0x89, 0xDF, 0xE7, 0x9F, /* 0x4C-0x4F */ + 0xE7, 0xA0, 0xE7, 0xA1, 0xE7, 0xA2, 0x93, 0xB9, /* 0x50-0x53 */ + 0x92, 0x42, 0x88, 0xE1, 0xE7, 0xA6, 0x00, 0x00, /* 0x54-0x57 */ + 0xE7, 0xA7, 0xEA, 0xA1, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x91, 0xBB, 0x00, 0x00, 0xE7, 0xA8, 0x00, 0x00, /* 0x5C-0x5F */ + 0x89, 0x93, 0x91, 0x6B, 0x00, 0x00, 0x8C, 0xAD, /* 0x60-0x63 */ + 0x00, 0x00, 0x97, 0x79, 0x00, 0x00, 0xEE, 0x99, /* 0x64-0x67 */ + 0xE7, 0xA9, 0x93, 0x4B, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x91, 0x98, 0x8E, 0xD5, 0xE7, 0xAA, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0xAD, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x8F, 0x85, 0xE7, 0xAB, 0x91, 0x4A, /* 0x74-0x77 */ + 0x91, 0x49, 0x00, 0x00, 0x88, 0xE2, 0x00, 0x00, /* 0x78-0x7B */ + 0x97, 0xC9, 0xE7, 0xAF, 0x00, 0x00, 0x94, 0xF0, /* 0x7C-0x7F */ + + 0xE7, 0xB1, 0xE7, 0xB0, 0xE7, 0xAE, 0xE2, 0x84, /* 0x80-0x83 */ + 0x8A, 0xD2, 0x00, 0x00, 0x00, 0x00, 0xE7, 0x8E, /* 0x84-0x87 */ + 0x00, 0x00, 0xE7, 0xB3, 0xE7, 0xB2, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0xB4, /* 0x8C-0x8F */ + 0x00, 0x00, 0x97, 0x57, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x93, 0xDF, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x96, 0x4D, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xE7, 0xB5, 0x00, 0x00, 0x8E, 0xD7, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0xB6, /* 0xAC-0xAF */ + 0x00, 0x00, 0xE7, 0xB7, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xE7, 0xB8, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x93, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x88, 0xE8, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x8D, 0x78, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x98, 0x59, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0xBC, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0xEE, 0x9A, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x8C, 0x53, 0xE7, 0xB9, 0x00, 0x00, /* 0xE0-0xE3 */ + 0xE7, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x95, 0x94, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x8A, 0x73, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x97, 0x58, 0x00, 0x00, 0x8B, 0xBD, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x93, 0x73, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_91[512] = { + 0x00, 0x00, 0x00, 0x00, 0xE7, 0xBD, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0xBE, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0xEE, 0x9C, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0xE7, 0xBF, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEE, 0x9D, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x93, 0x41, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xE7, 0xC1, 0x00, 0x00, 0xE7, 0xC0, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x93, 0xD1, 0xE7, 0xC2, 0x8F, 0x55, /* 0x48-0x4B */ + 0x8E, 0xDE, 0x94, 0x7A, 0x92, 0x91, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x8E, 0xF0, 0x00, 0x00, /* 0x50-0x53 */ + 0x90, 0x8C, 0x00, 0x00, 0xE7, 0xC3, 0x00, 0x00, /* 0x54-0x57 */ + 0xE7, 0xC4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x90, 0x7C, 0xE7, 0xC5, /* 0x60-0x63 */ + 0x00, 0x00, 0xE7, 0xC6, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0xE7, 0xC7, 0x97, 0x8F, 0x00, 0x00, /* 0x68-0x6B */ + 0x8F, 0x56, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0xC9, 0xE7, 0xC8, /* 0x70-0x73 */ + 0x00, 0x00, 0x8D, 0x79, 0x00, 0x00, 0x8D, 0x93, /* 0x74-0x77 */ + 0x8E, 0x5F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0xE7, 0xCC, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8F, 0x86, /* 0x84-0x87 */ + 0x00, 0x00, 0xE7, 0xCB, 0x00, 0x00, 0xE7, 0xCA, /* 0x88-0x8B */ + 0x00, 0x00, 0x91, 0xE7, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x8C, 0xED, 0x00, 0x00, 0x90, 0xC1, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x94, 0xAE, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x8F, 0x58, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0xCD, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x8F, 0xDD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0xD0, 0xE7, 0xCE, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0xCF, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xE7, 0xD2, 0xE7, 0xD1, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x8F, 0xF8, 0x00, 0x00, 0xE7, 0xD3, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0xE7, 0xD4, 0xE7, 0xD5, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x94, 0xCE, 0x8D, 0xD1, /* 0xC4-0xC7 */ + 0x8E, 0xDF, 0xE7, 0xD6, 0x00, 0x00, 0xE7, 0xD7, /* 0xC8-0xCB */ + 0x97, 0xA2, 0x8F, 0x64, 0x96, 0xEC, 0x97, 0xCA, /* 0xCC-0xCF */ + 0xE7, 0xD8, 0x8B, 0xE0, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0xD9, 0xEE, 0x9F, /* 0xD4-0xD7 */ + 0x93, 0x42, 0x00, 0x00, 0xEE, 0x9E, 0xE7, 0xDC, /* 0xD8-0xDB */ + 0x8A, 0x98, 0x90, 0x6A, 0xEE, 0xA0, 0xE7, 0xDA, /* 0xDC-0xDF */ + 0x00, 0x00, 0xE7, 0xDB, 0x00, 0x00, 0x92, 0xDE, /* 0xE0-0xE3 */ + 0xEE, 0xA3, 0xEE, 0xA4, 0x96, 0x74, 0x8B, 0xFA, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0xEE, 0xA1, 0xEE, 0xA2, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0xE7, 0xDE, 0xE7, 0xDF, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0xE7, 0xDD, 0x00, 0x00, 0x00, 0x00, 0xE7, 0xE1, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_92[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0xEE, 0xA5, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xEE, 0xA7, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x93, 0xDD, 0x8A, 0x62, 0x00, 0x00, /* 0x0C-0x0F */ + 0xEE, 0xA6, 0xE7, 0xE5, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0xE7, 0xE2, 0xE7, 0xE4, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0xE0, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0xE8, 0x6E, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0xE7, 0xE3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x97, 0xE9, 0x00, 0x00, 0x00, 0x00, 0x8C, 0xD8, /* 0x34-0x37 */ + 0x00, 0x00, 0xEE, 0xAE, 0xEE, 0xA8, 0x00, 0x00, /* 0x38-0x3B */ + 0xEE, 0xAA, 0x00, 0x00, 0x00, 0x00, 0xE7, 0xED, /* 0x3C-0x3F */ + 0xEE, 0xA9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x93, 0x53, 0xE7, 0xE8, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0xE7, 0xEB, 0xE7, 0xE9, 0x00, 0x00, 0xE7, 0xEE, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0xEE, 0xAB, 0x00, 0x00, /* 0x4C-0x4F */ + 0xE7, 0xEF, 0xEE, 0xAD, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0xE7, /* 0x54-0x57 */ + 0x00, 0x00, 0xEE, 0xAC, 0xE7, 0xF4, 0x89, 0x94, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0xE6, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x94, 0xAB, 0x00, 0x00, /* 0x60-0x63 */ + 0xE7, 0xEA, 0x00, 0x00, 0x8F, 0xDE, 0xEE, 0xAF, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x8D, 0x7A, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEE, 0xB1, /* 0x74-0x77 */ + 0xEE, 0xB2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x96, 0x67, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x8B, 0xE2, 0x00, 0x00, 0x00, 0x00, 0x8F, 0x65, /* 0x80-0x83 */ + 0x00, 0x00, 0x93, 0xBA, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0xED, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x91, 0x4C, 0x00, 0x00, 0xE7, 0xF2, /* 0x90-0x93 */ + 0x00, 0x00, 0xE7, 0xEC, 0xE7, 0xF1, 0x00, 0x00, /* 0x94-0x97 */ + 0x96, 0xC1, 0x00, 0x00, 0x92, 0xB6, 0xE7, 0xF3, /* 0x98-0x9B */ + 0xE7, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEE, 0xB0, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x91, 0x4B, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0xF7, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xE7, 0xF6, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0xF5, /* 0xCC-0xCF */ + 0xEE, 0xB6, 0x00, 0x00, 0x96, 0x4E, 0xEE, 0xBA, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xEE, 0xB8, 0x00, 0x00, 0xEE, 0xB4, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xEE, 0xB5, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0xEE, 0xB9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x8F, 0x9B, 0x00, 0x00, 0x00, 0x00, 0xEE, 0xB3, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xE7, 0xF8, 0x95, 0xDD, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x89, 0x73, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x95, 0x65, 0x92, 0x92, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x8B, 0x98, 0xED, 0x49, 0xE7, 0xFA, 0xEE, 0xBD, /* 0xF8-0xFB */ + 0x8D, 0x7C, 0x00, 0x00, 0x00, 0x00, 0xEE, 0xC0, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_93[512] = { + 0x00, 0x00, 0x00, 0x00, 0xEE, 0xC2, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x8E, 0x4B, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0xF9, /* 0x0C-0x0F */ + 0x90, 0x8D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x90, 0x8E, 0xE8, 0x40, 0xE8, 0x42, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0xEE, 0xC1, 0xEE, 0xBF, 0x00, 0x00, /* 0x1C-0x1F */ + 0x8F, 0xF9, 0xEE, 0xBC, 0xE8, 0x41, 0xE8, 0x43, /* 0x20-0x23 */ + 0x00, 0x00, 0xEE, 0xBB, 0x8B, 0xD1, 0x00, 0x00, /* 0x24-0x27 */ + 0x95, 0x64, 0x00, 0x00, 0x00, 0x00, 0x8E, 0xE0, /* 0x28-0x2B */ + 0x98, 0x42, 0x00, 0x00, 0xE7, 0xFC, 0x8D, 0xF6, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x98, 0x5E, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0xE8, 0x45, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xE8, 0x44, 0xE8, 0x46, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0xE7, 0xFB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0xED, 0x42, 0x00, 0x00, 0x00, 0x00, 0x93, 0xE7, /* 0x48-0x4B */ + 0x00, 0x00, 0x93, 0x74, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x92, 0xD5, 0x00, 0x00, 0xE8, 0x4B, 0xEE, 0xC4, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0x62, /* 0x58-0x5B */ + 0xE8, 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0xE8, 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x8C, 0x4C, 0x00, 0x00, 0xE8, 0x4A, 0x00, 0x00, /* 0x6C-0x6F */ + 0xEE, 0xC3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x8C, 0xAE, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0xE8, 0x49, 0x00, 0x00, 0x8F, 0xDF, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x8A, 0x99, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0xE8, 0x4F, 0x00, 0x00, 0x8D, 0xBD, 0x91, 0x99, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x92, 0xC8, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xEE, 0xC5, 0x00, 0x00, 0x00, 0x00, 0x8A, 0x5A, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0xE8, 0x4D, 0xE8, 0x4E, 0x92, 0xC1, 0x00, 0x00, /* 0xAC-0xAF */ + 0xE8, 0x4C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xE8, 0x50, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE8, 0x56, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0xEE, 0xC6, 0x00, 0x00, /* 0xC4-0xC7 */ + 0xE8, 0x59, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0xE8, 0x58, 0x93, 0x4C, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0xE8, 0x51, 0xE8, 0x52, /* 0xD4-0xD7 */ + 0xE8, 0x55, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0xE8, 0x57, 0xEE, 0xC7, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x8B, 0xBE, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0xE8, 0x5A, 0xE8, 0x54, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xE8, 0x53, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xEE, 0xC8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_94[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE8, 0x5E, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE8, 0x5F, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0xE8, 0x60, 0x00, 0x00, 0x00, 0x00, 0xE8, 0x5D, /* 0x10-0x13 */ + 0xE8, 0x5C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x8F, 0xE0, 0x93, 0xA8, 0xE8, 0x5B, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0xE8, 0x64, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE8, 0x62, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0xEE, 0xC9, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0xE8, 0x63, 0xE8, 0x61, 0x00, 0x00, /* 0x34-0x37 */ + 0x91, 0xF6, 0x00, 0x00, 0xE8, 0x65, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0xE8, 0x66, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0xE8, 0x68, 0xEE, 0xCA, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0xEE, 0xCB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x8A, 0xD3, 0xE8, 0x67, 0x96, 0xF8, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0xE8, 0x73, 0xE8, 0x69, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0xE8, 0x6C, 0x00, 0x00, /* 0x5C-0x5F */ + 0xE8, 0x6A, 0x00, 0x00, 0xE8, 0x6B, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0xE8, 0x6D, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0xE8, 0x6F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0xE8, 0x70, 0x00, 0x00, 0xE8, 0x71, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0xE8, 0x74, 0xE8, 0x72, 0xE8, 0x75, 0xE8, 0x77, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xE8, 0x76, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ +}; + +static const unsigned char u2c_95[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0xB7, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x96, 0xE5, 0x00, 0x00, 0xE8, 0x78, 0x91, 0x4D, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE8, 0x79, /* 0x84-0x87 */ + 0x00, 0x00, 0x95, 0xC2, 0xE8, 0x7A, 0x8A, 0x4A, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x89, 0x5B, /* 0x8C-0x8F */ + 0x00, 0x00, 0x8A, 0xD5, 0xEE, 0xCC, 0x8A, 0xD4, /* 0x90-0x93 */ + 0xE8, 0x7B, 0x00, 0x00, 0xE8, 0x7C, 0x00, 0x00, /* 0x94-0x97 */ + 0xE8, 0x7D, 0xE8, 0x7E, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0xE8, 0x80, 0x00, 0x00, 0x8A, 0xD6, 0x8A, 0x74, /* 0xA0-0xA3 */ + 0x8D, 0x7D, 0x94, 0xB4, 0x00, 0x00, 0xE8, 0x82, /* 0xA4-0xA7 */ + 0xE8, 0x81, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0xE8, 0x83, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0x7B, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xE8, 0x86, 0x00, 0x00, 0xE8, 0x85, /* 0xB8-0xBB */ + 0xE8, 0x84, 0x00, 0x00, 0xE8, 0x87, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE8, 0x8A, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0xC5, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0xE8, 0x88, 0x00, 0x00, /* 0xC8-0xCB */ + 0xE8, 0x8C, 0xE8, 0x8B, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xE8, 0x8E, 0xE8, 0x8D, 0xE8, 0x8F, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x93, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0xE8, 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0xE8, 0x91, 0xE8, 0x93, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xE8, 0x92, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ +}; + +static const unsigned char u2c_96[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x95, 0x8C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0xE8, 0x94, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0xE8, 0x95, 0x00, 0x00, 0x8D, 0xE3, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0xE8, 0x96, 0xE8, 0x97, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x96, 0x68, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x91, 0x6A, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x88, 0xA2, /* 0x3C-0x3F */ + 0x91, 0xC9, 0x00, 0x00, 0xE8, 0x98, 0x00, 0x00, /* 0x40-0x43 */ + 0x95, 0x8D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE8, 0x9B, /* 0x48-0x4B */ + 0xE8, 0x99, 0x8D, 0x7E, 0x00, 0x00, 0xE8, 0x9A, /* 0x4C-0x4F */ + 0x8C, 0xC0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0xC3, /* 0x58-0x5B */ + 0xE8, 0x9D, 0xE8, 0x9F, 0xE8, 0x9E, 0xE8, 0xA0, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0x40, 0x90, 0x77, /* 0x60-0x63 */ + 0x8F, 0x9C, 0x8A, 0xD7, 0xE8, 0xA1, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x94, 0x86, 0x00, 0x00, /* 0x68-0x6B */ + 0xE8, 0xA3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x89, 0x41, 0x00, 0x00, 0xE8, 0xA2, 0x92, 0xC2, /* 0x70-0x73 */ + 0x00, 0x00, 0x97, 0xCB, 0x93, 0xA9, 0xE8, 0x9C, /* 0x74-0x77 */ + 0x97, 0xA4, 0x00, 0x00, 0x8C, 0xAF, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x97, 0x7A, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x8B, 0xF7, 0x97, 0xB2, 0x00, 0x00, /* 0x84-0x87 */ + 0x8C, 0x47, 0x00, 0x00, 0x91, 0xE0, 0xE4, 0x40, /* 0x88-0x8B */ + 0x00, 0x00, 0xE8, 0xA4, 0x8A, 0x4B, 0x90, 0x8F, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x8A, 0x75, 0xE8, 0xA6, 0x00, 0x00, 0xE8, 0xA7, /* 0x94-0x97 */ + 0xE8, 0xA5, 0x8C, 0x84, 0x00, 0x00, 0x8D, 0xDB, /* 0x98-0x9B */ + 0x8F, 0xE1, 0xEE, 0xCF, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x89, 0x42, 0x00, 0x00, 0x00, 0x00, 0x97, 0xD7, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE8, 0xA9, /* 0xA4-0xA7 */ + 0xE7, 0xAC, 0x00, 0x00, 0xE8, 0xA8, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEE, 0xD0, /* 0xAC-0xAF */ + 0xE8, 0xAC, 0xE8, 0xAA, 0xE8, 0xAB, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xE8, 0xAD, 0x00, 0x00, 0xE8, 0xAE, 0x97, 0xEA, /* 0xB4-0xB7 */ + 0xE8, 0xAF, 0xE8, 0xB0, 0x00, 0x00, 0x90, 0xC7, /* 0xB8-0xBB */ + 0x94, 0xB9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x90, 0x9D, 0x8A, 0xE5, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x97, 0x59, 0x89, 0xEB, 0x8F, 0x57, 0x8C, 0xD9, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xE8, 0xB3, 0x00, 0x00, 0xE8, 0xB2, /* 0xC8-0xCB */ + 0x8E, 0x93, 0xE8, 0xB4, 0xE8, 0xB1, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x8E, 0x47, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xE8, 0xB8, 0xE5, 0xAB, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x99, 0xD4, 0x00, 0x00, 0x90, 0x97, /* 0xD8-0xDB */ + 0xE8, 0xB6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x97, 0xA3, 0x93, 0xEF, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x89, 0x4A, 0x00, 0x00, 0x90, 0xE1, 0x8E, 0xB4, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x95, 0xB5, 0x00, 0x00, 0x89, 0x5F, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x97, 0xEB, 0x97, 0x8B, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xE8, 0xB9, 0x00, 0x00, 0x93, 0x64, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_97[512] = { + 0x8E, 0xF9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0xE8, 0xBA, 0x00, 0x00, 0xE8, 0xBB, 0x90, 0x6B, /* 0x04-0x07 */ + 0xE8, 0xBC, 0x00, 0x00, 0x97, 0xEC, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0xE8, 0xB7, 0xE8, 0xBE, 0xE8, 0xC0, /* 0x0C-0x0F */ + 0x00, 0x00, 0xE8, 0xBF, 0x00, 0x00, 0xE8, 0xBD, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xE8, 0xC1, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0xE8, 0xC2, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x91, 0x9A, 0x00, 0x00, 0x89, 0xE0, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0xE8, 0xC3, 0x00, 0x00, 0x00, 0x00, 0x96, 0xB6, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xE8, 0xC4, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xE8, 0xC5, 0x00, 0x00, 0x98, 0x49, 0xEE, 0xD1, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x9E, 0x50, 0xE8, 0xC6, 0x00, 0x00, 0xEE, 0xD2, /* 0x38-0x3B */ + 0x00, 0x00, 0xE8, 0xC7, 0xE8, 0xC8, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0xE8, 0xCC, 0xEE, 0xD3, /* 0x40-0x43 */ + 0xE8, 0xC9, 0x00, 0x00, 0xE8, 0xCA, 0x00, 0x00, /* 0x44-0x47 */ + 0xE8, 0xCB, 0xE8, 0xCD, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0xEE, 0xD4, 0x00, 0x00, 0xEE, 0xD5, /* 0x4C-0x4F */ + 0x00, 0x00, 0xEE, 0xD6, 0x90, 0xC2, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0xEE, 0xD7, 0x96, 0xF5, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x90, 0xC3, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0xE8, 0xCE, 0x00, 0x00, 0x94, 0xF1, 0x00, 0x00, /* 0x5C-0x5F */ + 0xE8, 0xCF, 0xEA, 0x72, 0x96, 0xCA, 0x00, 0x00, /* 0x60-0x63 */ + 0xE8, 0xD0, 0x00, 0x00, 0xE8, 0xD1, 0x00, 0x00, /* 0x64-0x67 */ + 0xE8, 0xD2, 0x8A, 0x76, 0x00, 0x00, 0xE8, 0xD4, /* 0x68-0x6B */ + 0x00, 0x00, 0x90, 0x78, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0xE8, 0xD5, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x8C, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0xE8, 0xD6, 0xE8, 0xDA, 0x00, 0x00, /* 0x78-0x7B */ + 0xE8, 0xD8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xE8, 0xD9, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x8A, 0x93, 0xE8, 0xD7, 0xE8, 0xDB, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE8, 0xDC, /* 0x88-0x8B */ + 0x00, 0x00, 0x88, 0xC6, 0x00, 0x00, 0xE8, 0xDD, /* 0x8C-0x8F */ + 0xE8, 0xDE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x8F, 0xE2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0xE8, 0xDF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x8B, 0x66, 0x00, 0x00, 0x00, 0x00, 0xE8, 0xE2, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0xE8, 0xE1, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xE8, 0xE0, 0x00, 0x00, 0x00, 0x00, 0xE6, 0x91, /* 0xA8-0xAB */ + 0x00, 0x00, 0x95, 0xDA, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE8, 0xE3, /* 0xB0-0xB3 */ + 0xE8, 0xE4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE8, 0xE5, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0xE8, 0xE6, 0x00, 0x00, /* 0xC4-0xC7 */ + 0xE8, 0xE7, 0x00, 0x00, 0x00, 0x00, 0xE8, 0xE8, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8A, 0xD8, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0xE8, 0xE9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0xE8, 0xEA, 0x94, 0x42, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0xE8, 0xEC, 0x89, 0xB9, /* 0xF0-0xF3 */ + 0x00, 0x00, 0xE8, 0xEF, 0xE8, 0xEE, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x89, 0x43, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8B, 0xBF, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_98[512] = { + 0x00, 0x00, 0x95, 0xC5, 0x92, 0xB8, 0x8D, 0xA0, /* 0x00-0x03 */ + 0x00, 0x00, 0x8D, 0x80, 0x8F, 0x87, 0x00, 0x00, /* 0x04-0x07 */ + 0x90, 0x7B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0xE8, 0xF1, 0x00, 0x00, 0x00, 0x00, 0xE8, 0xF0, /* 0x0C-0x0F */ + 0x97, 0x61, 0x8A, 0xE6, 0x94, 0xD0, 0x93, 0xDA, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, 0x9C, /* 0x14-0x17 */ + 0x97, 0xCC, 0x00, 0x00, 0x8C, 0x7A, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0xE8, 0xF4, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0xE8, 0xF3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x96, 0x6A, 0x93, 0xAA, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x89, 0x6F, 0x00, 0x00, 0x00, 0x00, 0xE8, 0xF5, /* 0x34-0x37 */ + 0xE8, 0xF2, 0x00, 0x00, 0x00, 0x00, 0x95, 0x70, /* 0x38-0x3B */ + 0x97, 0x8A, 0xE8, 0xF6, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xE8, 0xF7, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE8, 0xF9, /* 0x48-0x4B */ + 0x91, 0xE8, 0x8A, 0x7A, 0x8A, 0x7B, 0xE8, 0xF8, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x8A, 0xE7, 0x8C, 0xB0, 0x00, 0x00, 0xEE, 0xD8, /* 0x54-0x57 */ + 0x8A, 0xE8, 0x00, 0x00, 0x00, 0x00, 0x93, 0x5E, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x97, 0xDE, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0xEE, 0xD9, 0x00, 0x00, 0x8C, 0xDA, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE8, 0xFA, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE8, 0xFB, /* 0x6C-0x6F */ + 0xE8, 0xFC, 0xE9, 0x40, 0x00, 0x00, 0xE9, 0x42, /* 0x70-0x73 */ + 0xE9, 0x41, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x95, 0x97, 0x00, 0x00, 0xE9, 0x43, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE9, 0x44, /* 0xAC-0xAF */ + 0x00, 0x00, 0xE9, 0x45, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0x46, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE9, 0x48, /* 0xC0-0xC3 */ + 0xE9, 0x47, 0x00, 0x00, 0xE9, 0x49, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x94, 0xF2, /* 0xD8-0xDB */ + 0xE3, 0xCA, 0x00, 0x00, 0x00, 0x00, 0x90, 0x48, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x8B, 0x51, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xE9, 0x4A, 0x00, 0x00, 0xE9, 0x4B, /* 0xE8-0xEB */ + 0x00, 0x00, 0x99, 0xAA, 0x9F, 0x5A, 0x94, 0xD1, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x88, 0xF9, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x88, 0xB9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x8E, 0x94, 0x96, 0x4F, 0x8F, 0xFC, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_99[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE9, 0x4C, /* 0x00-0x03 */ + 0x00, 0x00, 0x96, 0xDD, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xE9, 0x4D, 0x97, 0x7B, 0x00, 0x00, /* 0x08-0x0B */ + 0x89, 0x61, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x8E, 0x60, 0x00, 0x00, 0xE9, 0x4E, 0x89, 0xEC, /* 0x10-0x13 */ + 0xE9, 0x4F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0xE9, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0xE9, 0x52, 0xE9, 0x53, 0x00, 0x00, /* 0x1C-0x1F */ + 0xE9, 0x55, 0xE9, 0x51, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0xE9, 0x54, 0x00, 0x00, 0x00, 0x00, 0xEE, 0xDC, /* 0x24-0x27 */ + 0x8A, 0xD9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0xE9, 0x56, 0x00, 0x00, 0xE9, 0x57, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0xE9, 0x58, 0xE9, 0x59, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0x5A, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0xE9, 0x5C, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0xE9, 0x5B, 0x00, 0x00, 0xE9, 0x5E, /* 0x48-0x4B */ + 0xE9, 0x61, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0xE9, 0x5D, 0xE9, 0x5F, 0xE9, 0x60, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0xE9, 0x62, 0x00, 0x00, 0x8B, 0xC0, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x8E, 0xF1, 0xE9, 0x63, /* 0x94-0x97 */ + 0xE9, 0x64, 0x8D, 0x81, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0xEE, 0xDE, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xE9, 0x65, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x8A, 0x5D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x94, 0x6E, 0xE9, 0x66, 0xE9, 0x67, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x92, 0x79, /* 0xB0-0xB3 */ + 0x93, 0xE9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0xE9, 0x68, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x94, 0x9D, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x91, 0xCA, 0x89, 0x77, 0x8B, 0xEC, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x8B, 0xED, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x92, 0x93, 0xE9, 0x6D, 0x8B, 0xEE, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x89, 0xED, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xE9, 0x6C, 0x00, 0x00, 0x00, 0x00, 0xE9, 0x6A, /* 0xD8-0xDB */ + 0x00, 0x00, 0xE9, 0x6B, 0x00, 0x00, 0xE9, 0x69, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0x77, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0xE9, 0x6E, 0xE9, 0x6F, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0xE9, 0x70, 0xE9, 0x71, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xE9, 0x73, 0x00, 0x00, 0x00, 0x00, 0xE9, 0x72, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8F, 0x78, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_9A[512] = { + 0x00, 0x00, 0xE9, 0x74, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0xE9, 0x76, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x8B, 0x52, 0xE9, 0x75, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x91, 0x9B, 0x8C, 0xB1, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0xE9, 0x78, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x91, 0xCB, 0x00, 0x00, 0x00, 0x00, 0xE9, 0x79, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x93, 0xAB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE9, 0x7A, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0x80, 0x00, 0x00, /* 0x3C-0x3F */ + 0xE9, 0x7D, 0x00, 0x00, 0xE9, 0x7C, 0xE9, 0x7E, /* 0x40-0x43 */ + 0x00, 0x00, 0xE9, 0x7B, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0xE9, 0x82, 0xEE, 0xDF, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0xE9, 0x81, 0x00, 0x00, 0xE9, 0x84, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x8B, 0xC1, 0xE9, 0x83, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE9, 0x85, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0x86, 0x00, 0x00, /* 0x60-0x63 */ + 0xE9, 0x88, 0xE9, 0x87, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0xE9, 0x89, 0xE9, 0x8B, 0xE9, 0x8A, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x8D, 0x9C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0xE9, 0x8C, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0xE9, 0x8D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x8A, 0x5B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0xE9, 0x8E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0xE9, 0x8F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x90, 0x91, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE9, 0x90, /* 0xCC-0xCF */ + 0x00, 0x00, 0xE9, 0x91, 0x00, 0x00, 0xE9, 0x92, /* 0xD0-0xD3 */ + 0xE9, 0x93, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x8D, 0x82, 0xEE, 0xE0, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0xEE, 0xE1, 0x00, 0x00, 0xE9, 0x94, 0xE9, 0x95, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0x96, 0xE9, 0x97, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0x98, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x94, 0xAF, 0xE9, 0x9A, /* 0xE8-0xEB */ + 0x00, 0x00, 0x95, 0x45, 0xE9, 0x9B, 0xE9, 0x99, /* 0xEC-0xEF */ + 0x00, 0x00, 0xE9, 0x9D, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xE9, 0x9C, 0x00, 0x00, 0x00, 0x00, 0xE9, 0x9E, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE9, 0x9F, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_9B[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xA0, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0xE9, 0xA1, 0x00, 0x00, 0xE9, 0xA2, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE9, 0xA3, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xA4, 0xE9, 0xA5, /* 0x20-0x23 */ + 0x00, 0x00, 0xE9, 0xA6, 0x00, 0x00, 0xE9, 0xA7, /* 0x24-0x27 */ + 0xE9, 0xA8, 0xE9, 0xA9, 0xE9, 0xAA, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xAB, 0xE9, 0xAC, /* 0x2C-0x2F */ + 0x00, 0x00, 0x9F, 0x54, 0xE9, 0xAD, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xF6, /* 0x38-0x3B */ + 0x8B, 0x53, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x8A, 0x40, 0x8D, 0xB0, 0xE9, 0xAF, /* 0x40-0x43 */ + 0xE9, 0xAE, 0x96, 0xA3, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0xE9, 0xB1, 0xE9, 0xB2, 0xE9, 0xB0, /* 0x4C-0x4F */ + 0x00, 0x00, 0xE9, 0xB3, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x96, 0x82, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0xE9, 0xB4, 0x00, 0x00, 0x8B, 0x9B, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x44, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0xEE, 0xE3, 0x00, 0x00, /* 0x70-0x73 */ + 0xE9, 0xB5, 0xEE, 0xE2, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE9, 0xB7, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x88, 0xBC, 0xEE, 0xE4, /* 0x8C-0x8F */ + 0x00, 0x00, 0xE9, 0xB8, 0x95, 0xA9, 0xE9, 0xB6, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xB9, 0xE9, 0xBA, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE9, 0xBB, /* 0x9C-0x9F */ + 0xE9, 0xBC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xE9, 0xBD, 0x00, 0x00, 0x96, 0x8E, 0x8E, 0x4C, /* 0xA8-0xAB */ + 0x00, 0x00, 0x8D, 0xF8, 0x91, 0x4E, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0xEE, 0xE5, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xE9, 0xBE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xE9, 0xC1, 0x00, 0x00, 0xEE, 0xE6, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0xE9, 0xBF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xC2, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x8C, 0xEF, 0xE9, 0xC0, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE9, 0xC3, /* 0xCC-0xCF */ + 0x00, 0x00, 0xE9, 0xC4, 0xE9, 0xC5, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xE9, 0xC9, 0x00, 0x00, 0x8E, 0x49, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x91, 0xE2, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0xE9, 0xCA, 0xE9, 0xC7, 0xE9, 0xC6, /* 0xE0-0xE3 */ + 0xE9, 0xC8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x8C, 0x7E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0xE9, 0xCE, 0xE9, 0xCD, 0xE9, 0xCC, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x88, 0xB1, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ +}; + +static const unsigned char u2c_9C[512] = { + 0xEE, 0xE7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0xE9, 0xD8, 0x00, 0x00, 0xE9, 0xD4, 0x00, 0x00, /* 0x04-0x07 */ + 0xE9, 0xD5, 0xE9, 0xD1, 0xE9, 0xD7, 0x00, 0x00, /* 0x08-0x0B */ + 0xE9, 0xD3, 0x8A, 0x82, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x98, 0x6B, 0x00, 0x00, 0xE9, 0xD6, 0xE9, 0xD2, /* 0x10-0x13 */ + 0xE9, 0xD0, 0xE9, 0xCF, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE9, 0xDA, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0xE9, 0xDD, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0xE9, 0xDC, 0xE9, 0xDB, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x95, 0x68, 0xE9, 0xD9, 0x88, 0xF1, /* 0x2C-0x2F */ + 0xE9, 0xDE, 0x00, 0x00, 0xE9, 0xE0, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x8A, 0x8F, 0xE9, 0xCB, 0x89, 0x56, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xE2, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xE1, 0xE9, 0xDF, /* 0x44-0x47 */ + 0x92, 0x4C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x96, 0x90, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x97, 0xD8, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xE3, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0xE9, 0xE4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE9, 0xE5, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xE6, 0x00, 0x00, /* 0x74-0x77 */ + 0xE9, 0xE7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x92, 0xB9, 0x00, 0x00, 0xE9, 0xE8, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x94, 0xB5, 0x00, 0x00, 0xE9, 0xED, /* 0xE8-0xEB */ + 0xE9, 0xE9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0xE9, 0xEA, 0x00, 0x00, 0x00, 0x00, 0x96, 0x50, /* 0xF0-0xF3 */ + 0x96, 0xC2, 0x00, 0x00, 0x93, 0xCE, 0x00, 0x00, /* 0xF4-0xF7 */ +}; + +static const unsigned char u2c_9D[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE9, 0xEE, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xEF, 0x93, 0xBC, /* 0x04-0x07 */ + 0xE9, 0xEC, 0xE9, 0xEB, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0xA8, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xF7, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0xE9, 0xF6, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x89, 0x95, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE9, 0xF4, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE9, 0xF3, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xF1, 0x00, 0x00, /* 0x24-0x27 */ + 0x8A, 0x9B, 0x00, 0x00, 0xE9, 0xF0, 0x8E, 0xB0, /* 0x28-0x2B */ + 0x89, 0xA7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8D, 0x83, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xFA, 0xE9, 0xF9, /* 0x3C-0x3F */ + 0x00, 0x00, 0xE9, 0xF8, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0xE9, 0xF5, 0x00, 0x00, 0xE9, 0xFB, 0x00, 0x00, /* 0x44-0x47 */ + 0xE9, 0xFC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0xEA, 0x44, 0xEA, 0x43, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0xEA, 0x45, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x89, 0x4C, 0xEA, 0x40, 0xEA, 0x41, 0x00, 0x00, /* 0x5C-0x5F */ + 0x8D, 0x94, 0x96, 0xB7, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0xEA, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEE, 0xE9, /* 0x68-0x6B */ + 0x96, 0x51, 0x00, 0x00, 0x00, 0x00, 0xEA, 0x4A, /* 0x6C-0x6F */ + 0xEE, 0xE8, 0x00, 0x00, 0xEA, 0x46, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0xEA, 0x4B, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEA, 0x48, /* 0x84-0x87 */ + 0x00, 0x00, 0xEA, 0x47, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8C, 0x7B, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0xEA, 0x4C, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xEA, 0x4D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0xEA, 0x4E, 0x00, 0x00, 0xEA, 0x49, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE9, 0xF2, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xEA, 0x4F, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x92, 0xDF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xEA, 0x53, 0x00, 0x00, 0xEA, 0x54, 0xEA, 0x52, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0xEA, 0x51, 0xEA, 0x57, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xEA, 0x50, 0x00, 0x00, 0xEA, 0x55, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEA, 0x56, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEA, 0x59, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xEA, 0x58, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0xEA, 0x5B, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0xEA, 0x5C, 0x00, 0x00, 0xEA, 0x5D, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x98, 0x68, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xEA, 0x5A, 0x91, 0xE9, 0x8D, 0xEB, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0xEA, 0x5E, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_9E[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0xEE, 0xEB, 0xEA, 0x5F, 0xEA, 0x60, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xEA, 0x61, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0xEA, 0x62, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x8C, 0xB2, 0xEA, 0x63, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0xEA, 0x64, 0x00, 0x00, 0x8E, 0xAD, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xEA, 0x65, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0xEA, 0x66, 0x00, 0x00, 0x00, 0x00, 0xEA, 0x67, /* 0x88-0x8B */ + 0xEA, 0x68, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xEA, 0x6B, 0xEA, 0x69, 0x98, 0x5B, /* 0x90-0x93 */ + 0x00, 0x00, 0xEA, 0x6A, 0x00, 0x00, 0x97, 0xED, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0xEA, 0x6C, 0x00, 0x00, 0x97, 0xD9, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xEA, 0x6D, 0x94, 0x9E, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0xEA, 0x6E, 0xEA, 0x70, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0xEA, 0x71, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xEA, 0x6F, 0x8D, 0x8D, 0x96, 0xCB, 0x96, 0x83, /* 0xB8-0xBB */ + 0x9B, 0xF5, 0x00, 0x00, 0x9F, 0x80, 0x96, 0x9B, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x89, 0xA9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0xEA, 0x73, 0x8B, 0x6F, 0xEA, 0x74, 0xEA, 0x75, /* 0xCC-0xCF */ + 0xEA, 0x76, 0xEE, 0xEC, 0x8D, 0x95, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xEA, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xE0, 0xD2, 0x96, 0xD9, 0x00, 0x00, 0x91, 0xE1, /* 0xD8-0xDB */ + 0xEA, 0x78, 0xEA, 0x7A, 0xEA, 0x79, 0x00, 0x00, /* 0xDC-0xDF */ + 0xEA, 0x7B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xEA, 0x7C, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xEA, 0x7D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEA, 0x7E, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xEA, 0x80, 0x00, 0x00, 0xEA, 0x81, 0xEA, 0x82, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xEA, 0x83, 0x00, 0x00, 0xEA, 0x84, /* 0xF8-0xFB */ + 0xEA, 0x85, 0xEA, 0x86, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_9F[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEA, 0x87, /* 0x04-0x07 */ + 0xEA, 0x88, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x93, 0x43, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8C, 0xDB, /* 0x10-0x13 */ + 0x00, 0x00, 0xEA, 0x8A, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x91, 0x6C, 0xEA, 0x8B, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0xEA, 0x8C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x95, 0x40, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xEA, 0x8D, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0xEA, 0x8E, 0xE2, 0x56, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0xD8, 0xE8, 0xEB, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0xEA, 0x8F, 0x00, 0x00, /* 0x50-0x53 */ + 0xEA, 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEA, 0x92, /* 0x5C-0x5F */ + 0xEA, 0x93, 0xEA, 0x94, 0x97, 0xEE, 0xEA, 0x91, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0xEA, 0x95, 0xEA, 0x96, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0xEA, 0x98, 0x00, 0x00, /* 0x68-0x6B */ + 0xEA, 0x97, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0xEA, 0x9A, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0xEA, 0x9B, 0xEA, 0x99, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x97, 0xB4, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0xEA, 0x9C, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0xEA, 0x9D, 0xE2, 0x73, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0xEA, 0x9E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ +}; + +static const unsigned char u2c_DC[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ +}; + +static const unsigned char u2c_F9[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0xED, 0xC4, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0xEE, 0xCD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ +}; + +static const unsigned char u2c_FA[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0xED, 0x73, 0xED, 0x7E, /* 0x0C-0x0F */ + 0xED, 0x80, 0xED, 0x95, 0xED, 0xBC, 0xED, 0xCC, /* 0x10-0x13 */ + 0xED, 0xCE, 0xED, 0xF9, 0xEE, 0x42, 0xEE, 0x59, /* 0x14-0x17 */ + 0xEE, 0x61, 0xEE, 0x62, 0xEE, 0x63, 0xEE, 0x65, /* 0x18-0x1B */ + 0xEE, 0x69, 0xEE, 0x6C, 0xEE, 0x75, 0xEE, 0x81, /* 0x1C-0x1F */ + 0xEE, 0x83, 0xEE, 0x84, 0xEE, 0x8D, 0xEE, 0x95, /* 0x20-0x23 */ + 0xEE, 0x97, 0xEE, 0x98, 0xEE, 0x9B, 0xEE, 0xB7, /* 0x24-0x27 */ + 0xEE, 0xBE, 0xEE, 0xCE, 0xEE, 0xDA, 0xEE, 0xDB, /* 0x28-0x2B */ + 0xEE, 0xDD, 0xEE, 0xEA, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ +}; + +static const unsigned char u2c_FF[512] = { + 0x00, 0x00, 0x81, 0x49, 0xEE, 0xFC, 0x81, 0x94, /* 0x00-0x03 */ + 0x81, 0x90, 0x81, 0x93, 0x81, 0x95, 0xEE, 0xFB, /* 0x04-0x07 */ + 0x81, 0x69, 0x81, 0x6A, 0x81, 0x96, 0x81, 0x7B, /* 0x08-0x0B */ + 0x81, 0x43, 0x81, 0x7C, 0x81, 0x44, 0x81, 0x5E, /* 0x0C-0x0F */ + 0x82, 0x4F, 0x82, 0x50, 0x82, 0x51, 0x82, 0x52, /* 0x10-0x13 */ + 0x82, 0x53, 0x82, 0x54, 0x82, 0x55, 0x82, 0x56, /* 0x14-0x17 */ + 0x82, 0x57, 0x82, 0x58, 0x81, 0x46, 0x81, 0x47, /* 0x18-0x1B */ + 0x81, 0x83, 0x81, 0x81, 0x81, 0x84, 0x81, 0x48, /* 0x1C-0x1F */ + 0x81, 0x97, 0x82, 0x60, 0x82, 0x61, 0x82, 0x62, /* 0x20-0x23 */ + 0x82, 0x63, 0x82, 0x64, 0x82, 0x65, 0x82, 0x66, /* 0x24-0x27 */ + 0x82, 0x67, 0x82, 0x68, 0x82, 0x69, 0x82, 0x6A, /* 0x28-0x2B */ + 0x82, 0x6B, 0x82, 0x6C, 0x82, 0x6D, 0x82, 0x6E, /* 0x2C-0x2F */ + 0x82, 0x6F, 0x82, 0x70, 0x82, 0x71, 0x82, 0x72, /* 0x30-0x33 */ + 0x82, 0x73, 0x82, 0x74, 0x82, 0x75, 0x82, 0x76, /* 0x34-0x37 */ + 0x82, 0x77, 0x82, 0x78, 0x82, 0x79, 0x81, 0x6D, /* 0x38-0x3B */ + 0x81, 0x5F, 0x81, 0x6E, 0x81, 0x4F, 0x81, 0x51, /* 0x3C-0x3F */ + 0x81, 0x4D, 0x82, 0x81, 0x82, 0x82, 0x82, 0x83, /* 0x40-0x43 */ + 0x82, 0x84, 0x82, 0x85, 0x82, 0x86, 0x82, 0x87, /* 0x44-0x47 */ + 0x82, 0x88, 0x82, 0x89, 0x82, 0x8A, 0x82, 0x8B, /* 0x48-0x4B */ + 0x82, 0x8C, 0x82, 0x8D, 0x82, 0x8E, 0x82, 0x8F, /* 0x4C-0x4F */ + 0x82, 0x90, 0x82, 0x91, 0x82, 0x92, 0x82, 0x93, /* 0x50-0x53 */ + 0x82, 0x94, 0x82, 0x95, 0x82, 0x96, 0x82, 0x97, /* 0x54-0x57 */ + 0x82, 0x98, 0x82, 0x99, 0x82, 0x9A, 0x81, 0x6F, /* 0x58-0x5B */ + 0x81, 0x62, 0x81, 0x70, 0x81, 0x60, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0xA1, 0x00, 0xA2, 0x00, 0xA3, /* 0x60-0x63 */ + 0x00, 0xA4, 0x00, 0xA5, 0x00, 0xA6, 0x00, 0xA7, /* 0x64-0x67 */ + 0x00, 0xA8, 0x00, 0xA9, 0x00, 0xAA, 0x00, 0xAB, /* 0x68-0x6B */ + 0x00, 0xAC, 0x00, 0xAD, 0x00, 0xAE, 0x00, 0xAF, /* 0x6C-0x6F */ + 0x00, 0xB0, 0x00, 0xB1, 0x00, 0xB2, 0x00, 0xB3, /* 0x70-0x73 */ + 0x00, 0xB4, 0x00, 0xB5, 0x00, 0xB6, 0x00, 0xB7, /* 0x74-0x77 */ + 0x00, 0xB8, 0x00, 0xB9, 0x00, 0xBA, 0x00, 0xBB, /* 0x78-0x7B */ + 0x00, 0xBC, 0x00, 0xBD, 0x00, 0xBE, 0x00, 0xBF, /* 0x7C-0x7F */ + + 0x00, 0xC0, 0x00, 0xC1, 0x00, 0xC2, 0x00, 0xC3, /* 0x80-0x83 */ + 0x00, 0xC4, 0x00, 0xC5, 0x00, 0xC6, 0x00, 0xC7, /* 0x84-0x87 */ + 0x00, 0xC8, 0x00, 0xC9, 0x00, 0xCA, 0x00, 0xCB, /* 0x88-0x8B */ + 0x00, 0xCC, 0x00, 0xCD, 0x00, 0xCE, 0x00, 0xCF, /* 0x8C-0x8F */ + 0x00, 0xD0, 0x00, 0xD1, 0x00, 0xD2, 0x00, 0xD3, /* 0x90-0x93 */ + 0x00, 0xD4, 0x00, 0xD5, 0x00, 0xD6, 0x00, 0xD7, /* 0x94-0x97 */ + 0x00, 0xD8, 0x00, 0xD9, 0x00, 0xDA, 0x00, 0xDB, /* 0x98-0x9B */ + 0x00, 0xDC, 0x00, 0xDD, 0x00, 0xDE, 0x00, 0xDF, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x81, 0x91, 0x81, 0x92, 0x81, 0xCA, 0x81, 0x50, /* 0xE0-0xE3 */ + 0xEE, 0xFA, 0x81, 0x8F, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ +}; + +static const unsigned char *const page_uni2charset[256] = { + NULL, NULL, NULL, u2c_03, u2c_04, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + u2c_20, u2c_21, u2c_22, u2c_23, u2c_24, u2c_25, u2c_26, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + u2c_30, NULL, u2c_32, u2c_33, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, u2c_4E, u2c_4F, + u2c_50, u2c_51, u2c_52, u2c_53, u2c_54, u2c_55, u2c_56, u2c_57, + u2c_58, u2c_59, u2c_5A, u2c_5B, u2c_5C, u2c_5D, u2c_5E, u2c_5F, + u2c_60, u2c_61, u2c_62, u2c_63, u2c_64, u2c_65, u2c_66, u2c_67, + u2c_68, u2c_69, u2c_6A, u2c_6B, u2c_6C, u2c_6D, u2c_6E, u2c_6F, + u2c_70, u2c_71, u2c_72, u2c_73, u2c_74, u2c_75, u2c_76, u2c_77, + u2c_78, u2c_79, u2c_7A, u2c_7B, u2c_7C, u2c_7D, u2c_7E, u2c_7F, + u2c_80, u2c_81, u2c_82, u2c_83, u2c_84, u2c_85, u2c_86, u2c_87, + u2c_88, u2c_89, u2c_8A, u2c_8B, u2c_8C, u2c_8D, u2c_8E, u2c_8F, + u2c_90, u2c_91, u2c_92, u2c_93, u2c_94, u2c_95, u2c_96, u2c_97, + u2c_98, u2c_99, u2c_9A, u2c_9B, u2c_9C, u2c_9D, u2c_9E, u2c_9F, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, u2c_DC, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, u2c_F9, u2c_FA, NULL, NULL, NULL, NULL, u2c_FF, }; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(const wchar_t uni, + unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni&0xFF; + unsigned char ch = (uni>>8)&0xFF; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + if (ch == 0xFF && 0x61 <= cl && cl <= 0x9F) { + out[0] = cl + 0x40; + return 1; + } + uni2charset = page_uni2charset[ch]; + if (uni2charset) { + if (boundlen < 2) + return -ENAMETOOLONG; + + out[0] = uni2charset[cl*2]; + out[1] = uni2charset[cl*2+1]; + if (out[0] == 0x00 && out[1] == 0x00) + return -EINVAL; + return 2; + } else if (ch == 0) { + if (cl <= 0x7F) { + out[0] = cl; + return 1; + } else if (0xA0 <= cl) { + out[0] = u2c_00hi[cl - 0xA0][0]; + out[1] = u2c_00hi[cl - 0xA0][1]; + if (out[0] && out[1]) + return 2; + } + return -EINVAL; + } + else + return -EINVAL; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, + wchar_t *uni) +{ + unsigned char ch, cl; + const wchar_t *charset2uni; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + if (rawstring[0] <= 0x7F) { + *uni = rawstring[0]; + return 1; + } + if (0xA1 <= rawstring[0] && rawstring[0] <= 0xDF) { + *uni = 0xFF00 | (rawstring[0] - 0x40); + return 1; + } + + if (boundlen < 2) + return -ENAMETOOLONG; + ch = rawstring[0]; + cl = rawstring[1]; + charset2uni = page_charset2uni[ch]; + if (charset2uni && cl) { + *uni = charset2uni[cl]; + if (*uni == 0x0000) + return -EINVAL; + return 2; + } + else + return -EINVAL; +} + +static struct nls_table table = { + .charset = "cp932", + .alias = "sjis", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_cp932(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_cp932(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_cp932) +module_exit(exit_nls_cp932) + +MODULE_DESCRIPTION("NLS Japanese charset (Shift-JIS)"); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_ALIAS_NLS(sjis); diff --git a/fs/nls/nls_cp936.c b/fs/nls/nls_cp936.c new file mode 100644 index 000000000..91d0a15fd --- /dev/null +++ b/fs/nls/nls_cp936.c @@ -0,0 +1,11112 @@ +/* + * linux/fs/nls/nls_cp936.c + * + * Charset cp936 translation tables. + * This translation table was generated automatically, the + * original table can be download from the Microsoft website. + * (http://www.microsoft.com/typography/unicode/unicodecp.htm) + */ + +#include +#include +#include +#include +#include + +static const wchar_t c2u_81[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x4E02,0x4E04,0x4E05,0x4E06,0x4E0F,0x4E12,0x4E17,0x4E1F,/* 0x40-0x47 */ + 0x4E20,0x4E21,0x4E23,0x4E26,0x4E29,0x4E2E,0x4E2F,0x4E31,/* 0x48-0x4F */ + 0x4E33,0x4E35,0x4E37,0x4E3C,0x4E40,0x4E41,0x4E42,0x4E44,/* 0x50-0x57 */ + 0x4E46,0x4E4A,0x4E51,0x4E55,0x4E57,0x4E5A,0x4E5B,0x4E62,/* 0x58-0x5F */ + 0x4E63,0x4E64,0x4E65,0x4E67,0x4E68,0x4E6A,0x4E6B,0x4E6C,/* 0x60-0x67 */ + 0x4E6D,0x4E6E,0x4E6F,0x4E72,0x4E74,0x4E75,0x4E76,0x4E77,/* 0x68-0x6F */ + 0x4E78,0x4E79,0x4E7A,0x4E7B,0x4E7C,0x4E7D,0x4E7F,0x4E80,/* 0x70-0x77 */ + 0x4E81,0x4E82,0x4E83,0x4E84,0x4E85,0x4E87,0x4E8A,0x0000,/* 0x78-0x7F */ + + 0x4E90,0x4E96,0x4E97,0x4E99,0x4E9C,0x4E9D,0x4E9E,0x4EA3,/* 0x80-0x87 */ + 0x4EAA,0x4EAF,0x4EB0,0x4EB1,0x4EB4,0x4EB6,0x4EB7,0x4EB8,/* 0x88-0x8F */ + 0x4EB9,0x4EBC,0x4EBD,0x4EBE,0x4EC8,0x4ECC,0x4ECF,0x4ED0,/* 0x90-0x97 */ + 0x4ED2,0x4EDA,0x4EDB,0x4EDC,0x4EE0,0x4EE2,0x4EE6,0x4EE7,/* 0x98-0x9F */ + 0x4EE9,0x4EED,0x4EEE,0x4EEF,0x4EF1,0x4EF4,0x4EF8,0x4EF9,/* 0xA0-0xA7 */ + 0x4EFA,0x4EFC,0x4EFE,0x4F00,0x4F02,0x4F03,0x4F04,0x4F05,/* 0xA8-0xAF */ + 0x4F06,0x4F07,0x4F08,0x4F0B,0x4F0C,0x4F12,0x4F13,0x4F14,/* 0xB0-0xB7 */ + 0x4F15,0x4F16,0x4F1C,0x4F1D,0x4F21,0x4F23,0x4F28,0x4F29,/* 0xB8-0xBF */ + 0x4F2C,0x4F2D,0x4F2E,0x4F31,0x4F33,0x4F35,0x4F37,0x4F39,/* 0xC0-0xC7 */ + 0x4F3B,0x4F3E,0x4F3F,0x4F40,0x4F41,0x4F42,0x4F44,0x4F45,/* 0xC8-0xCF */ + 0x4F47,0x4F48,0x4F49,0x4F4A,0x4F4B,0x4F4C,0x4F52,0x4F54,/* 0xD0-0xD7 */ + 0x4F56,0x4F61,0x4F62,0x4F66,0x4F68,0x4F6A,0x4F6B,0x4F6D,/* 0xD8-0xDF */ + 0x4F6E,0x4F71,0x4F72,0x4F75,0x4F77,0x4F78,0x4F79,0x4F7A,/* 0xE0-0xE7 */ + 0x4F7D,0x4F80,0x4F81,0x4F82,0x4F85,0x4F86,0x4F87,0x4F8A,/* 0xE8-0xEF */ + 0x4F8C,0x4F8E,0x4F90,0x4F92,0x4F93,0x4F95,0x4F96,0x4F98,/* 0xF0-0xF7 */ + 0x4F99,0x4F9A,0x4F9C,0x4F9E,0x4F9F,0x4FA1,0x4FA2,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_82[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x4FA4,0x4FAB,0x4FAD,0x4FB0,0x4FB1,0x4FB2,0x4FB3,0x4FB4,/* 0x40-0x47 */ + 0x4FB6,0x4FB7,0x4FB8,0x4FB9,0x4FBA,0x4FBB,0x4FBC,0x4FBD,/* 0x48-0x4F */ + 0x4FBE,0x4FC0,0x4FC1,0x4FC2,0x4FC6,0x4FC7,0x4FC8,0x4FC9,/* 0x50-0x57 */ + 0x4FCB,0x4FCC,0x4FCD,0x4FD2,0x4FD3,0x4FD4,0x4FD5,0x4FD6,/* 0x58-0x5F */ + 0x4FD9,0x4FDB,0x4FE0,0x4FE2,0x4FE4,0x4FE5,0x4FE7,0x4FEB,/* 0x60-0x67 */ + 0x4FEC,0x4FF0,0x4FF2,0x4FF4,0x4FF5,0x4FF6,0x4FF7,0x4FF9,/* 0x68-0x6F */ + 0x4FFB,0x4FFC,0x4FFD,0x4FFF,0x5000,0x5001,0x5002,0x5003,/* 0x70-0x77 */ + 0x5004,0x5005,0x5006,0x5007,0x5008,0x5009,0x500A,0x0000,/* 0x78-0x7F */ + + 0x500B,0x500E,0x5010,0x5011,0x5013,0x5015,0x5016,0x5017,/* 0x80-0x87 */ + 0x501B,0x501D,0x501E,0x5020,0x5022,0x5023,0x5024,0x5027,/* 0x88-0x8F */ + 0x502B,0x502F,0x5030,0x5031,0x5032,0x5033,0x5034,0x5035,/* 0x90-0x97 */ + 0x5036,0x5037,0x5038,0x5039,0x503B,0x503D,0x503F,0x5040,/* 0x98-0x9F */ + 0x5041,0x5042,0x5044,0x5045,0x5046,0x5049,0x504A,0x504B,/* 0xA0-0xA7 */ + 0x504D,0x5050,0x5051,0x5052,0x5053,0x5054,0x5056,0x5057,/* 0xA8-0xAF */ + 0x5058,0x5059,0x505B,0x505D,0x505E,0x505F,0x5060,0x5061,/* 0xB0-0xB7 */ + 0x5062,0x5063,0x5064,0x5066,0x5067,0x5068,0x5069,0x506A,/* 0xB8-0xBF */ + 0x506B,0x506D,0x506E,0x506F,0x5070,0x5071,0x5072,0x5073,/* 0xC0-0xC7 */ + 0x5074,0x5075,0x5078,0x5079,0x507A,0x507C,0x507D,0x5081,/* 0xC8-0xCF */ + 0x5082,0x5083,0x5084,0x5086,0x5087,0x5089,0x508A,0x508B,/* 0xD0-0xD7 */ + 0x508C,0x508E,0x508F,0x5090,0x5091,0x5092,0x5093,0x5094,/* 0xD8-0xDF */ + 0x5095,0x5096,0x5097,0x5098,0x5099,0x509A,0x509B,0x509C,/* 0xE0-0xE7 */ + 0x509D,0x509E,0x509F,0x50A0,0x50A1,0x50A2,0x50A4,0x50A6,/* 0xE8-0xEF */ + 0x50AA,0x50AB,0x50AD,0x50AE,0x50AF,0x50B0,0x50B1,0x50B3,/* 0xF0-0xF7 */ + 0x50B4,0x50B5,0x50B6,0x50B7,0x50B8,0x50B9,0x50BC,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_83[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x50BD,0x50BE,0x50BF,0x50C0,0x50C1,0x50C2,0x50C3,0x50C4,/* 0x40-0x47 */ + 0x50C5,0x50C6,0x50C7,0x50C8,0x50C9,0x50CA,0x50CB,0x50CC,/* 0x48-0x4F */ + 0x50CD,0x50CE,0x50D0,0x50D1,0x50D2,0x50D3,0x50D4,0x50D5,/* 0x50-0x57 */ + 0x50D7,0x50D8,0x50D9,0x50DB,0x50DC,0x50DD,0x50DE,0x50DF,/* 0x58-0x5F */ + 0x50E0,0x50E1,0x50E2,0x50E3,0x50E4,0x50E5,0x50E8,0x50E9,/* 0x60-0x67 */ + 0x50EA,0x50EB,0x50EF,0x50F0,0x50F1,0x50F2,0x50F4,0x50F6,/* 0x68-0x6F */ + 0x50F7,0x50F8,0x50F9,0x50FA,0x50FC,0x50FD,0x50FE,0x50FF,/* 0x70-0x77 */ + 0x5100,0x5101,0x5102,0x5103,0x5104,0x5105,0x5108,0x0000,/* 0x78-0x7F */ + + 0x5109,0x510A,0x510C,0x510D,0x510E,0x510F,0x5110,0x5111,/* 0x80-0x87 */ + 0x5113,0x5114,0x5115,0x5116,0x5117,0x5118,0x5119,0x511A,/* 0x88-0x8F */ + 0x511B,0x511C,0x511D,0x511E,0x511F,0x5120,0x5122,0x5123,/* 0x90-0x97 */ + 0x5124,0x5125,0x5126,0x5127,0x5128,0x5129,0x512A,0x512B,/* 0x98-0x9F */ + 0x512C,0x512D,0x512E,0x512F,0x5130,0x5131,0x5132,0x5133,/* 0xA0-0xA7 */ + 0x5134,0x5135,0x5136,0x5137,0x5138,0x5139,0x513A,0x513B,/* 0xA8-0xAF */ + 0x513C,0x513D,0x513E,0x5142,0x5147,0x514A,0x514C,0x514E,/* 0xB0-0xB7 */ + 0x514F,0x5150,0x5152,0x5153,0x5157,0x5158,0x5159,0x515B,/* 0xB8-0xBF */ + 0x515D,0x515E,0x515F,0x5160,0x5161,0x5163,0x5164,0x5166,/* 0xC0-0xC7 */ + 0x5167,0x5169,0x516A,0x516F,0x5172,0x517A,0x517E,0x517F,/* 0xC8-0xCF */ + 0x5183,0x5184,0x5186,0x5187,0x518A,0x518B,0x518E,0x518F,/* 0xD0-0xD7 */ + 0x5190,0x5191,0x5193,0x5194,0x5198,0x519A,0x519D,0x519E,/* 0xD8-0xDF */ + 0x519F,0x51A1,0x51A3,0x51A6,0x51A7,0x51A8,0x51A9,0x51AA,/* 0xE0-0xE7 */ + 0x51AD,0x51AE,0x51B4,0x51B8,0x51B9,0x51BA,0x51BE,0x51BF,/* 0xE8-0xEF */ + 0x51C1,0x51C2,0x51C3,0x51C5,0x51C8,0x51CA,0x51CD,0x51CE,/* 0xF0-0xF7 */ + 0x51D0,0x51D2,0x51D3,0x51D4,0x51D5,0x51D6,0x51D7,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_84[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x51D8,0x51D9,0x51DA,0x51DC,0x51DE,0x51DF,0x51E2,0x51E3,/* 0x40-0x47 */ + 0x51E5,0x51E6,0x51E7,0x51E8,0x51E9,0x51EA,0x51EC,0x51EE,/* 0x48-0x4F */ + 0x51F1,0x51F2,0x51F4,0x51F7,0x51FE,0x5204,0x5205,0x5209,/* 0x50-0x57 */ + 0x520B,0x520C,0x520F,0x5210,0x5213,0x5214,0x5215,0x521C,/* 0x58-0x5F */ + 0x521E,0x521F,0x5221,0x5222,0x5223,0x5225,0x5226,0x5227,/* 0x60-0x67 */ + 0x522A,0x522C,0x522F,0x5231,0x5232,0x5234,0x5235,0x523C,/* 0x68-0x6F */ + 0x523E,0x5244,0x5245,0x5246,0x5247,0x5248,0x5249,0x524B,/* 0x70-0x77 */ + 0x524E,0x524F,0x5252,0x5253,0x5255,0x5257,0x5258,0x0000,/* 0x78-0x7F */ + + 0x5259,0x525A,0x525B,0x525D,0x525F,0x5260,0x5262,0x5263,/* 0x80-0x87 */ + 0x5264,0x5266,0x5268,0x526B,0x526C,0x526D,0x526E,0x5270,/* 0x88-0x8F */ + 0x5271,0x5273,0x5274,0x5275,0x5276,0x5277,0x5278,0x5279,/* 0x90-0x97 */ + 0x527A,0x527B,0x527C,0x527E,0x5280,0x5283,0x5284,0x5285,/* 0x98-0x9F */ + 0x5286,0x5287,0x5289,0x528A,0x528B,0x528C,0x528D,0x528E,/* 0xA0-0xA7 */ + 0x528F,0x5291,0x5292,0x5294,0x5295,0x5296,0x5297,0x5298,/* 0xA8-0xAF */ + 0x5299,0x529A,0x529C,0x52A4,0x52A5,0x52A6,0x52A7,0x52AE,/* 0xB0-0xB7 */ + 0x52AF,0x52B0,0x52B4,0x52B5,0x52B6,0x52B7,0x52B8,0x52B9,/* 0xB8-0xBF */ + 0x52BA,0x52BB,0x52BC,0x52BD,0x52C0,0x52C1,0x52C2,0x52C4,/* 0xC0-0xC7 */ + 0x52C5,0x52C6,0x52C8,0x52CA,0x52CC,0x52CD,0x52CE,0x52CF,/* 0xC8-0xCF */ + 0x52D1,0x52D3,0x52D4,0x52D5,0x52D7,0x52D9,0x52DA,0x52DB,/* 0xD0-0xD7 */ + 0x52DC,0x52DD,0x52DE,0x52E0,0x52E1,0x52E2,0x52E3,0x52E5,/* 0xD8-0xDF */ + 0x52E6,0x52E7,0x52E8,0x52E9,0x52EA,0x52EB,0x52EC,0x52ED,/* 0xE0-0xE7 */ + 0x52EE,0x52EF,0x52F1,0x52F2,0x52F3,0x52F4,0x52F5,0x52F6,/* 0xE8-0xEF */ + 0x52F7,0x52F8,0x52FB,0x52FC,0x52FD,0x5301,0x5302,0x5303,/* 0xF0-0xF7 */ + 0x5304,0x5307,0x5309,0x530A,0x530B,0x530C,0x530E,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_85[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x5311,0x5312,0x5313,0x5314,0x5318,0x531B,0x531C,0x531E,/* 0x40-0x47 */ + 0x531F,0x5322,0x5324,0x5325,0x5327,0x5328,0x5329,0x532B,/* 0x48-0x4F */ + 0x532C,0x532D,0x532F,0x5330,0x5331,0x5332,0x5333,0x5334,/* 0x50-0x57 */ + 0x5335,0x5336,0x5337,0x5338,0x533C,0x533D,0x5340,0x5342,/* 0x58-0x5F */ + 0x5344,0x5346,0x534B,0x534C,0x534D,0x5350,0x5354,0x5358,/* 0x60-0x67 */ + 0x5359,0x535B,0x535D,0x5365,0x5368,0x536A,0x536C,0x536D,/* 0x68-0x6F */ + 0x5372,0x5376,0x5379,0x537B,0x537C,0x537D,0x537E,0x5380,/* 0x70-0x77 */ + 0x5381,0x5383,0x5387,0x5388,0x538A,0x538E,0x538F,0x0000,/* 0x78-0x7F */ + + 0x5390,0x5391,0x5392,0x5393,0x5394,0x5396,0x5397,0x5399,/* 0x80-0x87 */ + 0x539B,0x539C,0x539E,0x53A0,0x53A1,0x53A4,0x53A7,0x53AA,/* 0x88-0x8F */ + 0x53AB,0x53AC,0x53AD,0x53AF,0x53B0,0x53B1,0x53B2,0x53B3,/* 0x90-0x97 */ + 0x53B4,0x53B5,0x53B7,0x53B8,0x53B9,0x53BA,0x53BC,0x53BD,/* 0x98-0x9F */ + 0x53BE,0x53C0,0x53C3,0x53C4,0x53C5,0x53C6,0x53C7,0x53CE,/* 0xA0-0xA7 */ + 0x53CF,0x53D0,0x53D2,0x53D3,0x53D5,0x53DA,0x53DC,0x53DD,/* 0xA8-0xAF */ + 0x53DE,0x53E1,0x53E2,0x53E7,0x53F4,0x53FA,0x53FE,0x53FF,/* 0xB0-0xB7 */ + 0x5400,0x5402,0x5405,0x5407,0x540B,0x5414,0x5418,0x5419,/* 0xB8-0xBF */ + 0x541A,0x541C,0x5422,0x5424,0x5425,0x542A,0x5430,0x5433,/* 0xC0-0xC7 */ + 0x5436,0x5437,0x543A,0x543D,0x543F,0x5441,0x5442,0x5444,/* 0xC8-0xCF */ + 0x5445,0x5447,0x5449,0x544C,0x544D,0x544E,0x544F,0x5451,/* 0xD0-0xD7 */ + 0x545A,0x545D,0x545E,0x545F,0x5460,0x5461,0x5463,0x5465,/* 0xD8-0xDF */ + 0x5467,0x5469,0x546A,0x546B,0x546C,0x546D,0x546E,0x546F,/* 0xE0-0xE7 */ + 0x5470,0x5474,0x5479,0x547A,0x547E,0x547F,0x5481,0x5483,/* 0xE8-0xEF */ + 0x5485,0x5487,0x5488,0x5489,0x548A,0x548D,0x5491,0x5493,/* 0xF0-0xF7 */ + 0x5497,0x5498,0x549C,0x549E,0x549F,0x54A0,0x54A1,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_86[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x54A2,0x54A5,0x54AE,0x54B0,0x54B2,0x54B5,0x54B6,0x54B7,/* 0x40-0x47 */ + 0x54B9,0x54BA,0x54BC,0x54BE,0x54C3,0x54C5,0x54CA,0x54CB,/* 0x48-0x4F */ + 0x54D6,0x54D8,0x54DB,0x54E0,0x54E1,0x54E2,0x54E3,0x54E4,/* 0x50-0x57 */ + 0x54EB,0x54EC,0x54EF,0x54F0,0x54F1,0x54F4,0x54F5,0x54F6,/* 0x58-0x5F */ + 0x54F7,0x54F8,0x54F9,0x54FB,0x54FE,0x5500,0x5502,0x5503,/* 0x60-0x67 */ + 0x5504,0x5505,0x5508,0x550A,0x550B,0x550C,0x550D,0x550E,/* 0x68-0x6F */ + 0x5512,0x5513,0x5515,0x5516,0x5517,0x5518,0x5519,0x551A,/* 0x70-0x77 */ + 0x551C,0x551D,0x551E,0x551F,0x5521,0x5525,0x5526,0x0000,/* 0x78-0x7F */ + + 0x5528,0x5529,0x552B,0x552D,0x5532,0x5534,0x5535,0x5536,/* 0x80-0x87 */ + 0x5538,0x5539,0x553A,0x553B,0x553D,0x5540,0x5542,0x5545,/* 0x88-0x8F */ + 0x5547,0x5548,0x554B,0x554C,0x554D,0x554E,0x554F,0x5551,/* 0x90-0x97 */ + 0x5552,0x5553,0x5554,0x5557,0x5558,0x5559,0x555A,0x555B,/* 0x98-0x9F */ + 0x555D,0x555E,0x555F,0x5560,0x5562,0x5563,0x5568,0x5569,/* 0xA0-0xA7 */ + 0x556B,0x556F,0x5570,0x5571,0x5572,0x5573,0x5574,0x5579,/* 0xA8-0xAF */ + 0x557A,0x557D,0x557F,0x5585,0x5586,0x558C,0x558D,0x558E,/* 0xB0-0xB7 */ + 0x5590,0x5592,0x5593,0x5595,0x5596,0x5597,0x559A,0x559B,/* 0xB8-0xBF */ + 0x559E,0x55A0,0x55A1,0x55A2,0x55A3,0x55A4,0x55A5,0x55A6,/* 0xC0-0xC7 */ + 0x55A8,0x55A9,0x55AA,0x55AB,0x55AC,0x55AD,0x55AE,0x55AF,/* 0xC8-0xCF */ + 0x55B0,0x55B2,0x55B4,0x55B6,0x55B8,0x55BA,0x55BC,0x55BF,/* 0xD0-0xD7 */ + 0x55C0,0x55C1,0x55C2,0x55C3,0x55C6,0x55C7,0x55C8,0x55CA,/* 0xD8-0xDF */ + 0x55CB,0x55CE,0x55CF,0x55D0,0x55D5,0x55D7,0x55D8,0x55D9,/* 0xE0-0xE7 */ + 0x55DA,0x55DB,0x55DE,0x55E0,0x55E2,0x55E7,0x55E9,0x55ED,/* 0xE8-0xEF */ + 0x55EE,0x55F0,0x55F1,0x55F4,0x55F6,0x55F8,0x55F9,0x55FA,/* 0xF0-0xF7 */ + 0x55FB,0x55FC,0x55FF,0x5602,0x5603,0x5604,0x5605,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_87[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x5606,0x5607,0x560A,0x560B,0x560D,0x5610,0x5611,0x5612,/* 0x40-0x47 */ + 0x5613,0x5614,0x5615,0x5616,0x5617,0x5619,0x561A,0x561C,/* 0x48-0x4F */ + 0x561D,0x5620,0x5621,0x5622,0x5625,0x5626,0x5628,0x5629,/* 0x50-0x57 */ + 0x562A,0x562B,0x562E,0x562F,0x5630,0x5633,0x5635,0x5637,/* 0x58-0x5F */ + 0x5638,0x563A,0x563C,0x563D,0x563E,0x5640,0x5641,0x5642,/* 0x60-0x67 */ + 0x5643,0x5644,0x5645,0x5646,0x5647,0x5648,0x5649,0x564A,/* 0x68-0x6F */ + 0x564B,0x564F,0x5650,0x5651,0x5652,0x5653,0x5655,0x5656,/* 0x70-0x77 */ + 0x565A,0x565B,0x565D,0x565E,0x565F,0x5660,0x5661,0x0000,/* 0x78-0x7F */ + + 0x5663,0x5665,0x5666,0x5667,0x566D,0x566E,0x566F,0x5670,/* 0x80-0x87 */ + 0x5672,0x5673,0x5674,0x5675,0x5677,0x5678,0x5679,0x567A,/* 0x88-0x8F */ + 0x567D,0x567E,0x567F,0x5680,0x5681,0x5682,0x5683,0x5684,/* 0x90-0x97 */ + 0x5687,0x5688,0x5689,0x568A,0x568B,0x568C,0x568D,0x5690,/* 0x98-0x9F */ + 0x5691,0x5692,0x5694,0x5695,0x5696,0x5697,0x5698,0x5699,/* 0xA0-0xA7 */ + 0x569A,0x569B,0x569C,0x569D,0x569E,0x569F,0x56A0,0x56A1,/* 0xA8-0xAF */ + 0x56A2,0x56A4,0x56A5,0x56A6,0x56A7,0x56A8,0x56A9,0x56AA,/* 0xB0-0xB7 */ + 0x56AB,0x56AC,0x56AD,0x56AE,0x56B0,0x56B1,0x56B2,0x56B3,/* 0xB8-0xBF */ + 0x56B4,0x56B5,0x56B6,0x56B8,0x56B9,0x56BA,0x56BB,0x56BD,/* 0xC0-0xC7 */ + 0x56BE,0x56BF,0x56C0,0x56C1,0x56C2,0x56C3,0x56C4,0x56C5,/* 0xC8-0xCF */ + 0x56C6,0x56C7,0x56C8,0x56C9,0x56CB,0x56CC,0x56CD,0x56CE,/* 0xD0-0xD7 */ + 0x56CF,0x56D0,0x56D1,0x56D2,0x56D3,0x56D5,0x56D6,0x56D8,/* 0xD8-0xDF */ + 0x56D9,0x56DC,0x56E3,0x56E5,0x56E6,0x56E7,0x56E8,0x56E9,/* 0xE0-0xE7 */ + 0x56EA,0x56EC,0x56EE,0x56EF,0x56F2,0x56F3,0x56F6,0x56F7,/* 0xE8-0xEF */ + 0x56F8,0x56FB,0x56FC,0x5700,0x5701,0x5702,0x5705,0x5707,/* 0xF0-0xF7 */ + 0x570B,0x570C,0x570D,0x570E,0x570F,0x5710,0x5711,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_88[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x5712,0x5713,0x5714,0x5715,0x5716,0x5717,0x5718,0x5719,/* 0x40-0x47 */ + 0x571A,0x571B,0x571D,0x571E,0x5720,0x5721,0x5722,0x5724,/* 0x48-0x4F */ + 0x5725,0x5726,0x5727,0x572B,0x5731,0x5732,0x5734,0x5735,/* 0x50-0x57 */ + 0x5736,0x5737,0x5738,0x573C,0x573D,0x573F,0x5741,0x5743,/* 0x58-0x5F */ + 0x5744,0x5745,0x5746,0x5748,0x5749,0x574B,0x5752,0x5753,/* 0x60-0x67 */ + 0x5754,0x5755,0x5756,0x5758,0x5759,0x5762,0x5763,0x5765,/* 0x68-0x6F */ + 0x5767,0x576C,0x576E,0x5770,0x5771,0x5772,0x5774,0x5775,/* 0x70-0x77 */ + 0x5778,0x5779,0x577A,0x577D,0x577E,0x577F,0x5780,0x0000,/* 0x78-0x7F */ + + 0x5781,0x5787,0x5788,0x5789,0x578A,0x578D,0x578E,0x578F,/* 0x80-0x87 */ + 0x5790,0x5791,0x5794,0x5795,0x5796,0x5797,0x5798,0x5799,/* 0x88-0x8F */ + 0x579A,0x579C,0x579D,0x579E,0x579F,0x57A5,0x57A8,0x57AA,/* 0x90-0x97 */ + 0x57AC,0x57AF,0x57B0,0x57B1,0x57B3,0x57B5,0x57B6,0x57B7,/* 0x98-0x9F */ + 0x57B9,0x57BA,0x57BB,0x57BC,0x57BD,0x57BE,0x57BF,0x57C0,/* 0xA0-0xA7 */ + 0x57C1,0x57C4,0x57C5,0x57C6,0x57C7,0x57C8,0x57C9,0x57CA,/* 0xA8-0xAF */ + 0x57CC,0x57CD,0x57D0,0x57D1,0x57D3,0x57D6,0x57D7,0x57DB,/* 0xB0-0xB7 */ + 0x57DC,0x57DE,0x57E1,0x57E2,0x57E3,0x57E5,0x57E6,0x57E7,/* 0xB8-0xBF */ + 0x57E8,0x57E9,0x57EA,0x57EB,0x57EC,0x57EE,0x57F0,0x57F1,/* 0xC0-0xC7 */ + 0x57F2,0x57F3,0x57F5,0x57F6,0x57F7,0x57FB,0x57FC,0x57FE,/* 0xC8-0xCF */ + 0x57FF,0x5801,0x5803,0x5804,0x5805,0x5808,0x5809,0x580A,/* 0xD0-0xD7 */ + 0x580C,0x580E,0x580F,0x5810,0x5812,0x5813,0x5814,0x5816,/* 0xD8-0xDF */ + 0x5817,0x5818,0x581A,0x581B,0x581C,0x581D,0x581F,0x5822,/* 0xE0-0xE7 */ + 0x5823,0x5825,0x5826,0x5827,0x5828,0x5829,0x582B,0x582C,/* 0xE8-0xEF */ + 0x582D,0x582E,0x582F,0x5831,0x5832,0x5833,0x5834,0x5836,/* 0xF0-0xF7 */ + 0x5837,0x5838,0x5839,0x583A,0x583B,0x583C,0x583D,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_89[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x583E,0x583F,0x5840,0x5841,0x5842,0x5843,0x5845,0x5846,/* 0x40-0x47 */ + 0x5847,0x5848,0x5849,0x584A,0x584B,0x584E,0x584F,0x5850,/* 0x48-0x4F */ + 0x5852,0x5853,0x5855,0x5856,0x5857,0x5859,0x585A,0x585B,/* 0x50-0x57 */ + 0x585C,0x585D,0x585F,0x5860,0x5861,0x5862,0x5863,0x5864,/* 0x58-0x5F */ + 0x5866,0x5867,0x5868,0x5869,0x586A,0x586D,0x586E,0x586F,/* 0x60-0x67 */ + 0x5870,0x5871,0x5872,0x5873,0x5874,0x5875,0x5876,0x5877,/* 0x68-0x6F */ + 0x5878,0x5879,0x587A,0x587B,0x587C,0x587D,0x587F,0x5882,/* 0x70-0x77 */ + 0x5884,0x5886,0x5887,0x5888,0x588A,0x588B,0x588C,0x0000,/* 0x78-0x7F */ + + 0x588D,0x588E,0x588F,0x5890,0x5891,0x5894,0x5895,0x5896,/* 0x80-0x87 */ + 0x5897,0x5898,0x589B,0x589C,0x589D,0x58A0,0x58A1,0x58A2,/* 0x88-0x8F */ + 0x58A3,0x58A4,0x58A5,0x58A6,0x58A7,0x58AA,0x58AB,0x58AC,/* 0x90-0x97 */ + 0x58AD,0x58AE,0x58AF,0x58B0,0x58B1,0x58B2,0x58B3,0x58B4,/* 0x98-0x9F */ + 0x58B5,0x58B6,0x58B7,0x58B8,0x58B9,0x58BA,0x58BB,0x58BD,/* 0xA0-0xA7 */ + 0x58BE,0x58BF,0x58C0,0x58C2,0x58C3,0x58C4,0x58C6,0x58C7,/* 0xA8-0xAF */ + 0x58C8,0x58C9,0x58CA,0x58CB,0x58CC,0x58CD,0x58CE,0x58CF,/* 0xB0-0xB7 */ + 0x58D0,0x58D2,0x58D3,0x58D4,0x58D6,0x58D7,0x58D8,0x58D9,/* 0xB8-0xBF */ + 0x58DA,0x58DB,0x58DC,0x58DD,0x58DE,0x58DF,0x58E0,0x58E1,/* 0xC0-0xC7 */ + 0x58E2,0x58E3,0x58E5,0x58E6,0x58E7,0x58E8,0x58E9,0x58EA,/* 0xC8-0xCF */ + 0x58ED,0x58EF,0x58F1,0x58F2,0x58F4,0x58F5,0x58F7,0x58F8,/* 0xD0-0xD7 */ + 0x58FA,0x58FB,0x58FC,0x58FD,0x58FE,0x58FF,0x5900,0x5901,/* 0xD8-0xDF */ + 0x5903,0x5905,0x5906,0x5908,0x5909,0x590A,0x590B,0x590C,/* 0xE0-0xE7 */ + 0x590E,0x5910,0x5911,0x5912,0x5913,0x5917,0x5918,0x591B,/* 0xE8-0xEF */ + 0x591D,0x591E,0x5920,0x5921,0x5922,0x5923,0x5926,0x5928,/* 0xF0-0xF7 */ + 0x592C,0x5930,0x5932,0x5933,0x5935,0x5936,0x593B,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_8A[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x593D,0x593E,0x593F,0x5940,0x5943,0x5945,0x5946,0x594A,/* 0x40-0x47 */ + 0x594C,0x594D,0x5950,0x5952,0x5953,0x5959,0x595B,0x595C,/* 0x48-0x4F */ + 0x595D,0x595E,0x595F,0x5961,0x5963,0x5964,0x5966,0x5967,/* 0x50-0x57 */ + 0x5968,0x5969,0x596A,0x596B,0x596C,0x596D,0x596E,0x596F,/* 0x58-0x5F */ + 0x5970,0x5971,0x5972,0x5975,0x5977,0x597A,0x597B,0x597C,/* 0x60-0x67 */ + 0x597E,0x597F,0x5980,0x5985,0x5989,0x598B,0x598C,0x598E,/* 0x68-0x6F */ + 0x598F,0x5990,0x5991,0x5994,0x5995,0x5998,0x599A,0x599B,/* 0x70-0x77 */ + 0x599C,0x599D,0x599F,0x59A0,0x59A1,0x59A2,0x59A6,0x0000,/* 0x78-0x7F */ + + 0x59A7,0x59AC,0x59AD,0x59B0,0x59B1,0x59B3,0x59B4,0x59B5,/* 0x80-0x87 */ + 0x59B6,0x59B7,0x59B8,0x59BA,0x59BC,0x59BD,0x59BF,0x59C0,/* 0x88-0x8F */ + 0x59C1,0x59C2,0x59C3,0x59C4,0x59C5,0x59C7,0x59C8,0x59C9,/* 0x90-0x97 */ + 0x59CC,0x59CD,0x59CE,0x59CF,0x59D5,0x59D6,0x59D9,0x59DB,/* 0x98-0x9F */ + 0x59DE,0x59DF,0x59E0,0x59E1,0x59E2,0x59E4,0x59E6,0x59E7,/* 0xA0-0xA7 */ + 0x59E9,0x59EA,0x59EB,0x59ED,0x59EE,0x59EF,0x59F0,0x59F1,/* 0xA8-0xAF */ + 0x59F2,0x59F3,0x59F4,0x59F5,0x59F6,0x59F7,0x59F8,0x59FA,/* 0xB0-0xB7 */ + 0x59FC,0x59FD,0x59FE,0x5A00,0x5A02,0x5A0A,0x5A0B,0x5A0D,/* 0xB8-0xBF */ + 0x5A0E,0x5A0F,0x5A10,0x5A12,0x5A14,0x5A15,0x5A16,0x5A17,/* 0xC0-0xC7 */ + 0x5A19,0x5A1A,0x5A1B,0x5A1D,0x5A1E,0x5A21,0x5A22,0x5A24,/* 0xC8-0xCF */ + 0x5A26,0x5A27,0x5A28,0x5A2A,0x5A2B,0x5A2C,0x5A2D,0x5A2E,/* 0xD0-0xD7 */ + 0x5A2F,0x5A30,0x5A33,0x5A35,0x5A37,0x5A38,0x5A39,0x5A3A,/* 0xD8-0xDF */ + 0x5A3B,0x5A3D,0x5A3E,0x5A3F,0x5A41,0x5A42,0x5A43,0x5A44,/* 0xE0-0xE7 */ + 0x5A45,0x5A47,0x5A48,0x5A4B,0x5A4C,0x5A4D,0x5A4E,0x5A4F,/* 0xE8-0xEF */ + 0x5A50,0x5A51,0x5A52,0x5A53,0x5A54,0x5A56,0x5A57,0x5A58,/* 0xF0-0xF7 */ + 0x5A59,0x5A5B,0x5A5C,0x5A5D,0x5A5E,0x5A5F,0x5A60,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_8B[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x5A61,0x5A63,0x5A64,0x5A65,0x5A66,0x5A68,0x5A69,0x5A6B,/* 0x40-0x47 */ + 0x5A6C,0x5A6D,0x5A6E,0x5A6F,0x5A70,0x5A71,0x5A72,0x5A73,/* 0x48-0x4F */ + 0x5A78,0x5A79,0x5A7B,0x5A7C,0x5A7D,0x5A7E,0x5A80,0x5A81,/* 0x50-0x57 */ + 0x5A82,0x5A83,0x5A84,0x5A85,0x5A86,0x5A87,0x5A88,0x5A89,/* 0x58-0x5F */ + 0x5A8A,0x5A8B,0x5A8C,0x5A8D,0x5A8E,0x5A8F,0x5A90,0x5A91,/* 0x60-0x67 */ + 0x5A93,0x5A94,0x5A95,0x5A96,0x5A97,0x5A98,0x5A99,0x5A9C,/* 0x68-0x6F */ + 0x5A9D,0x5A9E,0x5A9F,0x5AA0,0x5AA1,0x5AA2,0x5AA3,0x5AA4,/* 0x70-0x77 */ + 0x5AA5,0x5AA6,0x5AA7,0x5AA8,0x5AA9,0x5AAB,0x5AAC,0x0000,/* 0x78-0x7F */ + + 0x5AAD,0x5AAE,0x5AAF,0x5AB0,0x5AB1,0x5AB4,0x5AB6,0x5AB7,/* 0x80-0x87 */ + 0x5AB9,0x5ABA,0x5ABB,0x5ABC,0x5ABD,0x5ABF,0x5AC0,0x5AC3,/* 0x88-0x8F */ + 0x5AC4,0x5AC5,0x5AC6,0x5AC7,0x5AC8,0x5ACA,0x5ACB,0x5ACD,/* 0x90-0x97 */ + 0x5ACE,0x5ACF,0x5AD0,0x5AD1,0x5AD3,0x5AD5,0x5AD7,0x5AD9,/* 0x98-0x9F */ + 0x5ADA,0x5ADB,0x5ADD,0x5ADE,0x5ADF,0x5AE2,0x5AE4,0x5AE5,/* 0xA0-0xA7 */ + 0x5AE7,0x5AE8,0x5AEA,0x5AEC,0x5AED,0x5AEE,0x5AEF,0x5AF0,/* 0xA8-0xAF */ + 0x5AF2,0x5AF3,0x5AF4,0x5AF5,0x5AF6,0x5AF7,0x5AF8,0x5AF9,/* 0xB0-0xB7 */ + 0x5AFA,0x5AFB,0x5AFC,0x5AFD,0x5AFE,0x5AFF,0x5B00,0x5B01,/* 0xB8-0xBF */ + 0x5B02,0x5B03,0x5B04,0x5B05,0x5B06,0x5B07,0x5B08,0x5B0A,/* 0xC0-0xC7 */ + 0x5B0B,0x5B0C,0x5B0D,0x5B0E,0x5B0F,0x5B10,0x5B11,0x5B12,/* 0xC8-0xCF */ + 0x5B13,0x5B14,0x5B15,0x5B18,0x5B19,0x5B1A,0x5B1B,0x5B1C,/* 0xD0-0xD7 */ + 0x5B1D,0x5B1E,0x5B1F,0x5B20,0x5B21,0x5B22,0x5B23,0x5B24,/* 0xD8-0xDF */ + 0x5B25,0x5B26,0x5B27,0x5B28,0x5B29,0x5B2A,0x5B2B,0x5B2C,/* 0xE0-0xE7 */ + 0x5B2D,0x5B2E,0x5B2F,0x5B30,0x5B31,0x5B33,0x5B35,0x5B36,/* 0xE8-0xEF */ + 0x5B38,0x5B39,0x5B3A,0x5B3B,0x5B3C,0x5B3D,0x5B3E,0x5B3F,/* 0xF0-0xF7 */ + 0x5B41,0x5B42,0x5B43,0x5B44,0x5B45,0x5B46,0x5B47,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_8C[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x5B48,0x5B49,0x5B4A,0x5B4B,0x5B4C,0x5B4D,0x5B4E,0x5B4F,/* 0x40-0x47 */ + 0x5B52,0x5B56,0x5B5E,0x5B60,0x5B61,0x5B67,0x5B68,0x5B6B,/* 0x48-0x4F */ + 0x5B6D,0x5B6E,0x5B6F,0x5B72,0x5B74,0x5B76,0x5B77,0x5B78,/* 0x50-0x57 */ + 0x5B79,0x5B7B,0x5B7C,0x5B7E,0x5B7F,0x5B82,0x5B86,0x5B8A,/* 0x58-0x5F */ + 0x5B8D,0x5B8E,0x5B90,0x5B91,0x5B92,0x5B94,0x5B96,0x5B9F,/* 0x60-0x67 */ + 0x5BA7,0x5BA8,0x5BA9,0x5BAC,0x5BAD,0x5BAE,0x5BAF,0x5BB1,/* 0x68-0x6F */ + 0x5BB2,0x5BB7,0x5BBA,0x5BBB,0x5BBC,0x5BC0,0x5BC1,0x5BC3,/* 0x70-0x77 */ + 0x5BC8,0x5BC9,0x5BCA,0x5BCB,0x5BCD,0x5BCE,0x5BCF,0x0000,/* 0x78-0x7F */ + + 0x5BD1,0x5BD4,0x5BD5,0x5BD6,0x5BD7,0x5BD8,0x5BD9,0x5BDA,/* 0x80-0x87 */ + 0x5BDB,0x5BDC,0x5BE0,0x5BE2,0x5BE3,0x5BE6,0x5BE7,0x5BE9,/* 0x88-0x8F */ + 0x5BEA,0x5BEB,0x5BEC,0x5BED,0x5BEF,0x5BF1,0x5BF2,0x5BF3,/* 0x90-0x97 */ + 0x5BF4,0x5BF5,0x5BF6,0x5BF7,0x5BFD,0x5BFE,0x5C00,0x5C02,/* 0x98-0x9F */ + 0x5C03,0x5C05,0x5C07,0x5C08,0x5C0B,0x5C0C,0x5C0D,0x5C0E,/* 0xA0-0xA7 */ + 0x5C10,0x5C12,0x5C13,0x5C17,0x5C19,0x5C1B,0x5C1E,0x5C1F,/* 0xA8-0xAF */ + 0x5C20,0x5C21,0x5C23,0x5C26,0x5C28,0x5C29,0x5C2A,0x5C2B,/* 0xB0-0xB7 */ + 0x5C2D,0x5C2E,0x5C2F,0x5C30,0x5C32,0x5C33,0x5C35,0x5C36,/* 0xB8-0xBF */ + 0x5C37,0x5C43,0x5C44,0x5C46,0x5C47,0x5C4C,0x5C4D,0x5C52,/* 0xC0-0xC7 */ + 0x5C53,0x5C54,0x5C56,0x5C57,0x5C58,0x5C5A,0x5C5B,0x5C5C,/* 0xC8-0xCF */ + 0x5C5D,0x5C5F,0x5C62,0x5C64,0x5C67,0x5C68,0x5C69,0x5C6A,/* 0xD0-0xD7 */ + 0x5C6B,0x5C6C,0x5C6D,0x5C70,0x5C72,0x5C73,0x5C74,0x5C75,/* 0xD8-0xDF */ + 0x5C76,0x5C77,0x5C78,0x5C7B,0x5C7C,0x5C7D,0x5C7E,0x5C80,/* 0xE0-0xE7 */ + 0x5C83,0x5C84,0x5C85,0x5C86,0x5C87,0x5C89,0x5C8A,0x5C8B,/* 0xE8-0xEF */ + 0x5C8E,0x5C8F,0x5C92,0x5C93,0x5C95,0x5C9D,0x5C9E,0x5C9F,/* 0xF0-0xF7 */ + 0x5CA0,0x5CA1,0x5CA4,0x5CA5,0x5CA6,0x5CA7,0x5CA8,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_8D[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x5CAA,0x5CAE,0x5CAF,0x5CB0,0x5CB2,0x5CB4,0x5CB6,0x5CB9,/* 0x40-0x47 */ + 0x5CBA,0x5CBB,0x5CBC,0x5CBE,0x5CC0,0x5CC2,0x5CC3,0x5CC5,/* 0x48-0x4F */ + 0x5CC6,0x5CC7,0x5CC8,0x5CC9,0x5CCA,0x5CCC,0x5CCD,0x5CCE,/* 0x50-0x57 */ + 0x5CCF,0x5CD0,0x5CD1,0x5CD3,0x5CD4,0x5CD5,0x5CD6,0x5CD7,/* 0x58-0x5F */ + 0x5CD8,0x5CDA,0x5CDB,0x5CDC,0x5CDD,0x5CDE,0x5CDF,0x5CE0,/* 0x60-0x67 */ + 0x5CE2,0x5CE3,0x5CE7,0x5CE9,0x5CEB,0x5CEC,0x5CEE,0x5CEF,/* 0x68-0x6F */ + 0x5CF1,0x5CF2,0x5CF3,0x5CF4,0x5CF5,0x5CF6,0x5CF7,0x5CF8,/* 0x70-0x77 */ + 0x5CF9,0x5CFA,0x5CFC,0x5CFD,0x5CFE,0x5CFF,0x5D00,0x0000,/* 0x78-0x7F */ + + 0x5D01,0x5D04,0x5D05,0x5D08,0x5D09,0x5D0A,0x5D0B,0x5D0C,/* 0x80-0x87 */ + 0x5D0D,0x5D0F,0x5D10,0x5D11,0x5D12,0x5D13,0x5D15,0x5D17,/* 0x88-0x8F */ + 0x5D18,0x5D19,0x5D1A,0x5D1C,0x5D1D,0x5D1F,0x5D20,0x5D21,/* 0x90-0x97 */ + 0x5D22,0x5D23,0x5D25,0x5D28,0x5D2A,0x5D2B,0x5D2C,0x5D2F,/* 0x98-0x9F */ + 0x5D30,0x5D31,0x5D32,0x5D33,0x5D35,0x5D36,0x5D37,0x5D38,/* 0xA0-0xA7 */ + 0x5D39,0x5D3A,0x5D3B,0x5D3C,0x5D3F,0x5D40,0x5D41,0x5D42,/* 0xA8-0xAF */ + 0x5D43,0x5D44,0x5D45,0x5D46,0x5D48,0x5D49,0x5D4D,0x5D4E,/* 0xB0-0xB7 */ + 0x5D4F,0x5D50,0x5D51,0x5D52,0x5D53,0x5D54,0x5D55,0x5D56,/* 0xB8-0xBF */ + 0x5D57,0x5D59,0x5D5A,0x5D5C,0x5D5E,0x5D5F,0x5D60,0x5D61,/* 0xC0-0xC7 */ + 0x5D62,0x5D63,0x5D64,0x5D65,0x5D66,0x5D67,0x5D68,0x5D6A,/* 0xC8-0xCF */ + 0x5D6D,0x5D6E,0x5D70,0x5D71,0x5D72,0x5D73,0x5D75,0x5D76,/* 0xD0-0xD7 */ + 0x5D77,0x5D78,0x5D79,0x5D7A,0x5D7B,0x5D7C,0x5D7D,0x5D7E,/* 0xD8-0xDF */ + 0x5D7F,0x5D80,0x5D81,0x5D83,0x5D84,0x5D85,0x5D86,0x5D87,/* 0xE0-0xE7 */ + 0x5D88,0x5D89,0x5D8A,0x5D8B,0x5D8C,0x5D8D,0x5D8E,0x5D8F,/* 0xE8-0xEF */ + 0x5D90,0x5D91,0x5D92,0x5D93,0x5D94,0x5D95,0x5D96,0x5D97,/* 0xF0-0xF7 */ + 0x5D98,0x5D9A,0x5D9B,0x5D9C,0x5D9E,0x5D9F,0x5DA0,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_8E[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x5DA1,0x5DA2,0x5DA3,0x5DA4,0x5DA5,0x5DA6,0x5DA7,0x5DA8,/* 0x40-0x47 */ + 0x5DA9,0x5DAA,0x5DAB,0x5DAC,0x5DAD,0x5DAE,0x5DAF,0x5DB0,/* 0x48-0x4F */ + 0x5DB1,0x5DB2,0x5DB3,0x5DB4,0x5DB5,0x5DB6,0x5DB8,0x5DB9,/* 0x50-0x57 */ + 0x5DBA,0x5DBB,0x5DBC,0x5DBD,0x5DBE,0x5DBF,0x5DC0,0x5DC1,/* 0x58-0x5F */ + 0x5DC2,0x5DC3,0x5DC4,0x5DC6,0x5DC7,0x5DC8,0x5DC9,0x5DCA,/* 0x60-0x67 */ + 0x5DCB,0x5DCC,0x5DCE,0x5DCF,0x5DD0,0x5DD1,0x5DD2,0x5DD3,/* 0x68-0x6F */ + 0x5DD4,0x5DD5,0x5DD6,0x5DD7,0x5DD8,0x5DD9,0x5DDA,0x5DDC,/* 0x70-0x77 */ + 0x5DDF,0x5DE0,0x5DE3,0x5DE4,0x5DEA,0x5DEC,0x5DED,0x0000,/* 0x78-0x7F */ + + 0x5DF0,0x5DF5,0x5DF6,0x5DF8,0x5DF9,0x5DFA,0x5DFB,0x5DFC,/* 0x80-0x87 */ + 0x5DFF,0x5E00,0x5E04,0x5E07,0x5E09,0x5E0A,0x5E0B,0x5E0D,/* 0x88-0x8F */ + 0x5E0E,0x5E12,0x5E13,0x5E17,0x5E1E,0x5E1F,0x5E20,0x5E21,/* 0x90-0x97 */ + 0x5E22,0x5E23,0x5E24,0x5E25,0x5E28,0x5E29,0x5E2A,0x5E2B,/* 0x98-0x9F */ + 0x5E2C,0x5E2F,0x5E30,0x5E32,0x5E33,0x5E34,0x5E35,0x5E36,/* 0xA0-0xA7 */ + 0x5E39,0x5E3A,0x5E3E,0x5E3F,0x5E40,0x5E41,0x5E43,0x5E46,/* 0xA8-0xAF */ + 0x5E47,0x5E48,0x5E49,0x5E4A,0x5E4B,0x5E4D,0x5E4E,0x5E4F,/* 0xB0-0xB7 */ + 0x5E50,0x5E51,0x5E52,0x5E53,0x5E56,0x5E57,0x5E58,0x5E59,/* 0xB8-0xBF */ + 0x5E5A,0x5E5C,0x5E5D,0x5E5F,0x5E60,0x5E63,0x5E64,0x5E65,/* 0xC0-0xC7 */ + 0x5E66,0x5E67,0x5E68,0x5E69,0x5E6A,0x5E6B,0x5E6C,0x5E6D,/* 0xC8-0xCF */ + 0x5E6E,0x5E6F,0x5E70,0x5E71,0x5E75,0x5E77,0x5E79,0x5E7E,/* 0xD0-0xD7 */ + 0x5E81,0x5E82,0x5E83,0x5E85,0x5E88,0x5E89,0x5E8C,0x5E8D,/* 0xD8-0xDF */ + 0x5E8E,0x5E92,0x5E98,0x5E9B,0x5E9D,0x5EA1,0x5EA2,0x5EA3,/* 0xE0-0xE7 */ + 0x5EA4,0x5EA8,0x5EA9,0x5EAA,0x5EAB,0x5EAC,0x5EAE,0x5EAF,/* 0xE8-0xEF */ + 0x5EB0,0x5EB1,0x5EB2,0x5EB4,0x5EBA,0x5EBB,0x5EBC,0x5EBD,/* 0xF0-0xF7 */ + 0x5EBF,0x5EC0,0x5EC1,0x5EC2,0x5EC3,0x5EC4,0x5EC5,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_8F[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x5EC6,0x5EC7,0x5EC8,0x5ECB,0x5ECC,0x5ECD,0x5ECE,0x5ECF,/* 0x40-0x47 */ + 0x5ED0,0x5ED4,0x5ED5,0x5ED7,0x5ED8,0x5ED9,0x5EDA,0x5EDC,/* 0x48-0x4F */ + 0x5EDD,0x5EDE,0x5EDF,0x5EE0,0x5EE1,0x5EE2,0x5EE3,0x5EE4,/* 0x50-0x57 */ + 0x5EE5,0x5EE6,0x5EE7,0x5EE9,0x5EEB,0x5EEC,0x5EED,0x5EEE,/* 0x58-0x5F */ + 0x5EEF,0x5EF0,0x5EF1,0x5EF2,0x5EF3,0x5EF5,0x5EF8,0x5EF9,/* 0x60-0x67 */ + 0x5EFB,0x5EFC,0x5EFD,0x5F05,0x5F06,0x5F07,0x5F09,0x5F0C,/* 0x68-0x6F */ + 0x5F0D,0x5F0E,0x5F10,0x5F12,0x5F14,0x5F16,0x5F19,0x5F1A,/* 0x70-0x77 */ + 0x5F1C,0x5F1D,0x5F1E,0x5F21,0x5F22,0x5F23,0x5F24,0x0000,/* 0x78-0x7F */ + + 0x5F28,0x5F2B,0x5F2C,0x5F2E,0x5F30,0x5F32,0x5F33,0x5F34,/* 0x80-0x87 */ + 0x5F35,0x5F36,0x5F37,0x5F38,0x5F3B,0x5F3D,0x5F3E,0x5F3F,/* 0x88-0x8F */ + 0x5F41,0x5F42,0x5F43,0x5F44,0x5F45,0x5F46,0x5F47,0x5F48,/* 0x90-0x97 */ + 0x5F49,0x5F4A,0x5F4B,0x5F4C,0x5F4D,0x5F4E,0x5F4F,0x5F51,/* 0x98-0x9F */ + 0x5F54,0x5F59,0x5F5A,0x5F5B,0x5F5C,0x5F5E,0x5F5F,0x5F60,/* 0xA0-0xA7 */ + 0x5F63,0x5F65,0x5F67,0x5F68,0x5F6B,0x5F6E,0x5F6F,0x5F72,/* 0xA8-0xAF */ + 0x5F74,0x5F75,0x5F76,0x5F78,0x5F7A,0x5F7D,0x5F7E,0x5F7F,/* 0xB0-0xB7 */ + 0x5F83,0x5F86,0x5F8D,0x5F8E,0x5F8F,0x5F91,0x5F93,0x5F94,/* 0xB8-0xBF */ + 0x5F96,0x5F9A,0x5F9B,0x5F9D,0x5F9E,0x5F9F,0x5FA0,0x5FA2,/* 0xC0-0xC7 */ + 0x5FA3,0x5FA4,0x5FA5,0x5FA6,0x5FA7,0x5FA9,0x5FAB,0x5FAC,/* 0xC8-0xCF */ + 0x5FAF,0x5FB0,0x5FB1,0x5FB2,0x5FB3,0x5FB4,0x5FB6,0x5FB8,/* 0xD0-0xD7 */ + 0x5FB9,0x5FBA,0x5FBB,0x5FBE,0x5FBF,0x5FC0,0x5FC1,0x5FC2,/* 0xD8-0xDF */ + 0x5FC7,0x5FC8,0x5FCA,0x5FCB,0x5FCE,0x5FD3,0x5FD4,0x5FD5,/* 0xE0-0xE7 */ + 0x5FDA,0x5FDB,0x5FDC,0x5FDE,0x5FDF,0x5FE2,0x5FE3,0x5FE5,/* 0xE8-0xEF */ + 0x5FE6,0x5FE8,0x5FE9,0x5FEC,0x5FEF,0x5FF0,0x5FF2,0x5FF3,/* 0xF0-0xF7 */ + 0x5FF4,0x5FF6,0x5FF7,0x5FF9,0x5FFA,0x5FFC,0x6007,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_90[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6008,0x6009,0x600B,0x600C,0x6010,0x6011,0x6013,0x6017,/* 0x40-0x47 */ + 0x6018,0x601A,0x601E,0x601F,0x6022,0x6023,0x6024,0x602C,/* 0x48-0x4F */ + 0x602D,0x602E,0x6030,0x6031,0x6032,0x6033,0x6034,0x6036,/* 0x50-0x57 */ + 0x6037,0x6038,0x6039,0x603A,0x603D,0x603E,0x6040,0x6044,/* 0x58-0x5F */ + 0x6045,0x6046,0x6047,0x6048,0x6049,0x604A,0x604C,0x604E,/* 0x60-0x67 */ + 0x604F,0x6051,0x6053,0x6054,0x6056,0x6057,0x6058,0x605B,/* 0x68-0x6F */ + 0x605C,0x605E,0x605F,0x6060,0x6061,0x6065,0x6066,0x606E,/* 0x70-0x77 */ + 0x6071,0x6072,0x6074,0x6075,0x6077,0x607E,0x6080,0x0000,/* 0x78-0x7F */ + + 0x6081,0x6082,0x6085,0x6086,0x6087,0x6088,0x608A,0x608B,/* 0x80-0x87 */ + 0x608E,0x608F,0x6090,0x6091,0x6093,0x6095,0x6097,0x6098,/* 0x88-0x8F */ + 0x6099,0x609C,0x609E,0x60A1,0x60A2,0x60A4,0x60A5,0x60A7,/* 0x90-0x97 */ + 0x60A9,0x60AA,0x60AE,0x60B0,0x60B3,0x60B5,0x60B6,0x60B7,/* 0x98-0x9F */ + 0x60B9,0x60BA,0x60BD,0x60BE,0x60BF,0x60C0,0x60C1,0x60C2,/* 0xA0-0xA7 */ + 0x60C3,0x60C4,0x60C7,0x60C8,0x60C9,0x60CC,0x60CD,0x60CE,/* 0xA8-0xAF */ + 0x60CF,0x60D0,0x60D2,0x60D3,0x60D4,0x60D6,0x60D7,0x60D9,/* 0xB0-0xB7 */ + 0x60DB,0x60DE,0x60E1,0x60E2,0x60E3,0x60E4,0x60E5,0x60EA,/* 0xB8-0xBF */ + 0x60F1,0x60F2,0x60F5,0x60F7,0x60F8,0x60FB,0x60FC,0x60FD,/* 0xC0-0xC7 */ + 0x60FE,0x60FF,0x6102,0x6103,0x6104,0x6105,0x6107,0x610A,/* 0xC8-0xCF */ + 0x610B,0x610C,0x6110,0x6111,0x6112,0x6113,0x6114,0x6116,/* 0xD0-0xD7 */ + 0x6117,0x6118,0x6119,0x611B,0x611C,0x611D,0x611E,0x6121,/* 0xD8-0xDF */ + 0x6122,0x6125,0x6128,0x6129,0x612A,0x612C,0x612D,0x612E,/* 0xE0-0xE7 */ + 0x612F,0x6130,0x6131,0x6132,0x6133,0x6134,0x6135,0x6136,/* 0xE8-0xEF */ + 0x6137,0x6138,0x6139,0x613A,0x613B,0x613C,0x613D,0x613E,/* 0xF0-0xF7 */ + 0x6140,0x6141,0x6142,0x6143,0x6144,0x6145,0x6146,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_91[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6147,0x6149,0x614B,0x614D,0x614F,0x6150,0x6152,0x6153,/* 0x40-0x47 */ + 0x6154,0x6156,0x6157,0x6158,0x6159,0x615A,0x615B,0x615C,/* 0x48-0x4F */ + 0x615E,0x615F,0x6160,0x6161,0x6163,0x6164,0x6165,0x6166,/* 0x50-0x57 */ + 0x6169,0x616A,0x616B,0x616C,0x616D,0x616E,0x616F,0x6171,/* 0x58-0x5F */ + 0x6172,0x6173,0x6174,0x6176,0x6178,0x6179,0x617A,0x617B,/* 0x60-0x67 */ + 0x617C,0x617D,0x617E,0x617F,0x6180,0x6181,0x6182,0x6183,/* 0x68-0x6F */ + 0x6184,0x6185,0x6186,0x6187,0x6188,0x6189,0x618A,0x618C,/* 0x70-0x77 */ + 0x618D,0x618F,0x6190,0x6191,0x6192,0x6193,0x6195,0x0000,/* 0x78-0x7F */ + + 0x6196,0x6197,0x6198,0x6199,0x619A,0x619B,0x619C,0x619E,/* 0x80-0x87 */ + 0x619F,0x61A0,0x61A1,0x61A2,0x61A3,0x61A4,0x61A5,0x61A6,/* 0x88-0x8F */ + 0x61AA,0x61AB,0x61AD,0x61AE,0x61AF,0x61B0,0x61B1,0x61B2,/* 0x90-0x97 */ + 0x61B3,0x61B4,0x61B5,0x61B6,0x61B8,0x61B9,0x61BA,0x61BB,/* 0x98-0x9F */ + 0x61BC,0x61BD,0x61BF,0x61C0,0x61C1,0x61C3,0x61C4,0x61C5,/* 0xA0-0xA7 */ + 0x61C6,0x61C7,0x61C9,0x61CC,0x61CD,0x61CE,0x61CF,0x61D0,/* 0xA8-0xAF */ + 0x61D3,0x61D5,0x61D6,0x61D7,0x61D8,0x61D9,0x61DA,0x61DB,/* 0xB0-0xB7 */ + 0x61DC,0x61DD,0x61DE,0x61DF,0x61E0,0x61E1,0x61E2,0x61E3,/* 0xB8-0xBF */ + 0x61E4,0x61E5,0x61E7,0x61E8,0x61E9,0x61EA,0x61EB,0x61EC,/* 0xC0-0xC7 */ + 0x61ED,0x61EE,0x61EF,0x61F0,0x61F1,0x61F2,0x61F3,0x61F4,/* 0xC8-0xCF */ + 0x61F6,0x61F7,0x61F8,0x61F9,0x61FA,0x61FB,0x61FC,0x61FD,/* 0xD0-0xD7 */ + 0x61FE,0x6200,0x6201,0x6202,0x6203,0x6204,0x6205,0x6207,/* 0xD8-0xDF */ + 0x6209,0x6213,0x6214,0x6219,0x621C,0x621D,0x621E,0x6220,/* 0xE0-0xE7 */ + 0x6223,0x6226,0x6227,0x6228,0x6229,0x622B,0x622D,0x622F,/* 0xE8-0xEF */ + 0x6230,0x6231,0x6232,0x6235,0x6236,0x6238,0x6239,0x623A,/* 0xF0-0xF7 */ + 0x623B,0x623C,0x6242,0x6244,0x6245,0x6246,0x624A,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_92[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x624F,0x6250,0x6255,0x6256,0x6257,0x6259,0x625A,0x625C,/* 0x40-0x47 */ + 0x625D,0x625E,0x625F,0x6260,0x6261,0x6262,0x6264,0x6265,/* 0x48-0x4F */ + 0x6268,0x6271,0x6272,0x6274,0x6275,0x6277,0x6278,0x627A,/* 0x50-0x57 */ + 0x627B,0x627D,0x6281,0x6282,0x6283,0x6285,0x6286,0x6287,/* 0x58-0x5F */ + 0x6288,0x628B,0x628C,0x628D,0x628E,0x628F,0x6290,0x6294,/* 0x60-0x67 */ + 0x6299,0x629C,0x629D,0x629E,0x62A3,0x62A6,0x62A7,0x62A9,/* 0x68-0x6F */ + 0x62AA,0x62AD,0x62AE,0x62AF,0x62B0,0x62B2,0x62B3,0x62B4,/* 0x70-0x77 */ + 0x62B6,0x62B7,0x62B8,0x62BA,0x62BE,0x62C0,0x62C1,0x0000,/* 0x78-0x7F */ + + 0x62C3,0x62CB,0x62CF,0x62D1,0x62D5,0x62DD,0x62DE,0x62E0,/* 0x80-0x87 */ + 0x62E1,0x62E4,0x62EA,0x62EB,0x62F0,0x62F2,0x62F5,0x62F8,/* 0x88-0x8F */ + 0x62F9,0x62FA,0x62FB,0x6300,0x6303,0x6304,0x6305,0x6306,/* 0x90-0x97 */ + 0x630A,0x630B,0x630C,0x630D,0x630F,0x6310,0x6312,0x6313,/* 0x98-0x9F */ + 0x6314,0x6315,0x6317,0x6318,0x6319,0x631C,0x6326,0x6327,/* 0xA0-0xA7 */ + 0x6329,0x632C,0x632D,0x632E,0x6330,0x6331,0x6333,0x6334,/* 0xA8-0xAF */ + 0x6335,0x6336,0x6337,0x6338,0x633B,0x633C,0x633E,0x633F,/* 0xB0-0xB7 */ + 0x6340,0x6341,0x6344,0x6347,0x6348,0x634A,0x6351,0x6352,/* 0xB8-0xBF */ + 0x6353,0x6354,0x6356,0x6357,0x6358,0x6359,0x635A,0x635B,/* 0xC0-0xC7 */ + 0x635C,0x635D,0x6360,0x6364,0x6365,0x6366,0x6368,0x636A,/* 0xC8-0xCF */ + 0x636B,0x636C,0x636F,0x6370,0x6372,0x6373,0x6374,0x6375,/* 0xD0-0xD7 */ + 0x6378,0x6379,0x637C,0x637D,0x637E,0x637F,0x6381,0x6383,/* 0xD8-0xDF */ + 0x6384,0x6385,0x6386,0x638B,0x638D,0x6391,0x6393,0x6394,/* 0xE0-0xE7 */ + 0x6395,0x6397,0x6399,0x639A,0x639B,0x639C,0x639D,0x639E,/* 0xE8-0xEF */ + 0x639F,0x63A1,0x63A4,0x63A6,0x63AB,0x63AF,0x63B1,0x63B2,/* 0xF0-0xF7 */ + 0x63B5,0x63B6,0x63B9,0x63BB,0x63BD,0x63BF,0x63C0,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_93[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x63C1,0x63C2,0x63C3,0x63C5,0x63C7,0x63C8,0x63CA,0x63CB,/* 0x40-0x47 */ + 0x63CC,0x63D1,0x63D3,0x63D4,0x63D5,0x63D7,0x63D8,0x63D9,/* 0x48-0x4F */ + 0x63DA,0x63DB,0x63DC,0x63DD,0x63DF,0x63E2,0x63E4,0x63E5,/* 0x50-0x57 */ + 0x63E6,0x63E7,0x63E8,0x63EB,0x63EC,0x63EE,0x63EF,0x63F0,/* 0x58-0x5F */ + 0x63F1,0x63F3,0x63F5,0x63F7,0x63F9,0x63FA,0x63FB,0x63FC,/* 0x60-0x67 */ + 0x63FE,0x6403,0x6404,0x6406,0x6407,0x6408,0x6409,0x640A,/* 0x68-0x6F */ + 0x640D,0x640E,0x6411,0x6412,0x6415,0x6416,0x6417,0x6418,/* 0x70-0x77 */ + 0x6419,0x641A,0x641D,0x641F,0x6422,0x6423,0x6424,0x0000,/* 0x78-0x7F */ + + 0x6425,0x6427,0x6428,0x6429,0x642B,0x642E,0x642F,0x6430,/* 0x80-0x87 */ + 0x6431,0x6432,0x6433,0x6435,0x6436,0x6437,0x6438,0x6439,/* 0x88-0x8F */ + 0x643B,0x643C,0x643E,0x6440,0x6442,0x6443,0x6449,0x644B,/* 0x90-0x97 */ + 0x644C,0x644D,0x644E,0x644F,0x6450,0x6451,0x6453,0x6455,/* 0x98-0x9F */ + 0x6456,0x6457,0x6459,0x645A,0x645B,0x645C,0x645D,0x645F,/* 0xA0-0xA7 */ + 0x6460,0x6461,0x6462,0x6463,0x6464,0x6465,0x6466,0x6468,/* 0xA8-0xAF */ + 0x646A,0x646B,0x646C,0x646E,0x646F,0x6470,0x6471,0x6472,/* 0xB0-0xB7 */ + 0x6473,0x6474,0x6475,0x6476,0x6477,0x647B,0x647C,0x647D,/* 0xB8-0xBF */ + 0x647E,0x647F,0x6480,0x6481,0x6483,0x6486,0x6488,0x6489,/* 0xC0-0xC7 */ + 0x648A,0x648B,0x648C,0x648D,0x648E,0x648F,0x6490,0x6493,/* 0xC8-0xCF */ + 0x6494,0x6497,0x6498,0x649A,0x649B,0x649C,0x649D,0x649F,/* 0xD0-0xD7 */ + 0x64A0,0x64A1,0x64A2,0x64A3,0x64A5,0x64A6,0x64A7,0x64A8,/* 0xD8-0xDF */ + 0x64AA,0x64AB,0x64AF,0x64B1,0x64B2,0x64B3,0x64B4,0x64B6,/* 0xE0-0xE7 */ + 0x64B9,0x64BB,0x64BD,0x64BE,0x64BF,0x64C1,0x64C3,0x64C4,/* 0xE8-0xEF */ + 0x64C6,0x64C7,0x64C8,0x64C9,0x64CA,0x64CB,0x64CC,0x64CF,/* 0xF0-0xF7 */ + 0x64D1,0x64D3,0x64D4,0x64D5,0x64D6,0x64D9,0x64DA,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_94[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x64DB,0x64DC,0x64DD,0x64DF,0x64E0,0x64E1,0x64E3,0x64E5,/* 0x40-0x47 */ + 0x64E7,0x64E8,0x64E9,0x64EA,0x64EB,0x64EC,0x64ED,0x64EE,/* 0x48-0x4F */ + 0x64EF,0x64F0,0x64F1,0x64F2,0x64F3,0x64F4,0x64F5,0x64F6,/* 0x50-0x57 */ + 0x64F7,0x64F8,0x64F9,0x64FA,0x64FB,0x64FC,0x64FD,0x64FE,/* 0x58-0x5F */ + 0x64FF,0x6501,0x6502,0x6503,0x6504,0x6505,0x6506,0x6507,/* 0x60-0x67 */ + 0x6508,0x650A,0x650B,0x650C,0x650D,0x650E,0x650F,0x6510,/* 0x68-0x6F */ + 0x6511,0x6513,0x6514,0x6515,0x6516,0x6517,0x6519,0x651A,/* 0x70-0x77 */ + 0x651B,0x651C,0x651D,0x651E,0x651F,0x6520,0x6521,0x0000,/* 0x78-0x7F */ + + 0x6522,0x6523,0x6524,0x6526,0x6527,0x6528,0x6529,0x652A,/* 0x80-0x87 */ + 0x652C,0x652D,0x6530,0x6531,0x6532,0x6533,0x6537,0x653A,/* 0x88-0x8F */ + 0x653C,0x653D,0x6540,0x6541,0x6542,0x6543,0x6544,0x6546,/* 0x90-0x97 */ + 0x6547,0x654A,0x654B,0x654D,0x654E,0x6550,0x6552,0x6553,/* 0x98-0x9F */ + 0x6554,0x6557,0x6558,0x655A,0x655C,0x655F,0x6560,0x6561,/* 0xA0-0xA7 */ + 0x6564,0x6565,0x6567,0x6568,0x6569,0x656A,0x656D,0x656E,/* 0xA8-0xAF */ + 0x656F,0x6571,0x6573,0x6575,0x6576,0x6578,0x6579,0x657A,/* 0xB0-0xB7 */ + 0x657B,0x657C,0x657D,0x657E,0x657F,0x6580,0x6581,0x6582,/* 0xB8-0xBF */ + 0x6583,0x6584,0x6585,0x6586,0x6588,0x6589,0x658A,0x658D,/* 0xC0-0xC7 */ + 0x658E,0x658F,0x6592,0x6594,0x6595,0x6596,0x6598,0x659A,/* 0xC8-0xCF */ + 0x659D,0x659E,0x65A0,0x65A2,0x65A3,0x65A6,0x65A8,0x65AA,/* 0xD0-0xD7 */ + 0x65AC,0x65AE,0x65B1,0x65B2,0x65B3,0x65B4,0x65B5,0x65B6,/* 0xD8-0xDF */ + 0x65B7,0x65B8,0x65BA,0x65BB,0x65BE,0x65BF,0x65C0,0x65C2,/* 0xE0-0xE7 */ + 0x65C7,0x65C8,0x65C9,0x65CA,0x65CD,0x65D0,0x65D1,0x65D3,/* 0xE8-0xEF */ + 0x65D4,0x65D5,0x65D8,0x65D9,0x65DA,0x65DB,0x65DC,0x65DD,/* 0xF0-0xF7 */ + 0x65DE,0x65DF,0x65E1,0x65E3,0x65E4,0x65EA,0x65EB,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_95[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x65F2,0x65F3,0x65F4,0x65F5,0x65F8,0x65F9,0x65FB,0x65FC,/* 0x40-0x47 */ + 0x65FD,0x65FE,0x65FF,0x6601,0x6604,0x6605,0x6607,0x6608,/* 0x48-0x4F */ + 0x6609,0x660B,0x660D,0x6610,0x6611,0x6612,0x6616,0x6617,/* 0x50-0x57 */ + 0x6618,0x661A,0x661B,0x661C,0x661E,0x6621,0x6622,0x6623,/* 0x58-0x5F */ + 0x6624,0x6626,0x6629,0x662A,0x662B,0x662C,0x662E,0x6630,/* 0x60-0x67 */ + 0x6632,0x6633,0x6637,0x6638,0x6639,0x663A,0x663B,0x663D,/* 0x68-0x6F */ + 0x663F,0x6640,0x6642,0x6644,0x6645,0x6646,0x6647,0x6648,/* 0x70-0x77 */ + 0x6649,0x664A,0x664D,0x664E,0x6650,0x6651,0x6658,0x0000,/* 0x78-0x7F */ + + 0x6659,0x665B,0x665C,0x665D,0x665E,0x6660,0x6662,0x6663,/* 0x80-0x87 */ + 0x6665,0x6667,0x6669,0x666A,0x666B,0x666C,0x666D,0x6671,/* 0x88-0x8F */ + 0x6672,0x6673,0x6675,0x6678,0x6679,0x667B,0x667C,0x667D,/* 0x90-0x97 */ + 0x667F,0x6680,0x6681,0x6683,0x6685,0x6686,0x6688,0x6689,/* 0x98-0x9F */ + 0x668A,0x668B,0x668D,0x668E,0x668F,0x6690,0x6692,0x6693,/* 0xA0-0xA7 */ + 0x6694,0x6695,0x6698,0x6699,0x669A,0x669B,0x669C,0x669E,/* 0xA8-0xAF */ + 0x669F,0x66A0,0x66A1,0x66A2,0x66A3,0x66A4,0x66A5,0x66A6,/* 0xB0-0xB7 */ + 0x66A9,0x66AA,0x66AB,0x66AC,0x66AD,0x66AF,0x66B0,0x66B1,/* 0xB8-0xBF */ + 0x66B2,0x66B3,0x66B5,0x66B6,0x66B7,0x66B8,0x66BA,0x66BB,/* 0xC0-0xC7 */ + 0x66BC,0x66BD,0x66BF,0x66C0,0x66C1,0x66C2,0x66C3,0x66C4,/* 0xC8-0xCF */ + 0x66C5,0x66C6,0x66C7,0x66C8,0x66C9,0x66CA,0x66CB,0x66CC,/* 0xD0-0xD7 */ + 0x66CD,0x66CE,0x66CF,0x66D0,0x66D1,0x66D2,0x66D3,0x66D4,/* 0xD8-0xDF */ + 0x66D5,0x66D6,0x66D7,0x66D8,0x66DA,0x66DE,0x66DF,0x66E0,/* 0xE0-0xE7 */ + 0x66E1,0x66E2,0x66E3,0x66E4,0x66E5,0x66E7,0x66E8,0x66EA,/* 0xE8-0xEF */ + 0x66EB,0x66EC,0x66ED,0x66EE,0x66EF,0x66F1,0x66F5,0x66F6,/* 0xF0-0xF7 */ + 0x66F8,0x66FA,0x66FB,0x66FD,0x6701,0x6702,0x6703,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_96[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6704,0x6705,0x6706,0x6707,0x670C,0x670E,0x670F,0x6711,/* 0x40-0x47 */ + 0x6712,0x6713,0x6716,0x6718,0x6719,0x671A,0x671C,0x671E,/* 0x48-0x4F */ + 0x6720,0x6721,0x6722,0x6723,0x6724,0x6725,0x6727,0x6729,/* 0x50-0x57 */ + 0x672E,0x6730,0x6732,0x6733,0x6736,0x6737,0x6738,0x6739,/* 0x58-0x5F */ + 0x673B,0x673C,0x673E,0x673F,0x6741,0x6744,0x6745,0x6747,/* 0x60-0x67 */ + 0x674A,0x674B,0x674D,0x6752,0x6754,0x6755,0x6757,0x6758,/* 0x68-0x6F */ + 0x6759,0x675A,0x675B,0x675D,0x6762,0x6763,0x6764,0x6766,/* 0x70-0x77 */ + 0x6767,0x676B,0x676C,0x676E,0x6771,0x6774,0x6776,0x0000,/* 0x78-0x7F */ + + 0x6778,0x6779,0x677A,0x677B,0x677D,0x6780,0x6782,0x6783,/* 0x80-0x87 */ + 0x6785,0x6786,0x6788,0x678A,0x678C,0x678D,0x678E,0x678F,/* 0x88-0x8F */ + 0x6791,0x6792,0x6793,0x6794,0x6796,0x6799,0x679B,0x679F,/* 0x90-0x97 */ + 0x67A0,0x67A1,0x67A4,0x67A6,0x67A9,0x67AC,0x67AE,0x67B1,/* 0x98-0x9F */ + 0x67B2,0x67B4,0x67B9,0x67BA,0x67BB,0x67BC,0x67BD,0x67BE,/* 0xA0-0xA7 */ + 0x67BF,0x67C0,0x67C2,0x67C5,0x67C6,0x67C7,0x67C8,0x67C9,/* 0xA8-0xAF */ + 0x67CA,0x67CB,0x67CC,0x67CD,0x67CE,0x67D5,0x67D6,0x67D7,/* 0xB0-0xB7 */ + 0x67DB,0x67DF,0x67E1,0x67E3,0x67E4,0x67E6,0x67E7,0x67E8,/* 0xB8-0xBF */ + 0x67EA,0x67EB,0x67ED,0x67EE,0x67F2,0x67F5,0x67F6,0x67F7,/* 0xC0-0xC7 */ + 0x67F8,0x67F9,0x67FA,0x67FB,0x67FC,0x67FE,0x6801,0x6802,/* 0xC8-0xCF */ + 0x6803,0x6804,0x6806,0x680D,0x6810,0x6812,0x6814,0x6815,/* 0xD0-0xD7 */ + 0x6818,0x6819,0x681A,0x681B,0x681C,0x681E,0x681F,0x6820,/* 0xD8-0xDF */ + 0x6822,0x6823,0x6824,0x6825,0x6826,0x6827,0x6828,0x682B,/* 0xE0-0xE7 */ + 0x682C,0x682D,0x682E,0x682F,0x6830,0x6831,0x6834,0x6835,/* 0xE8-0xEF */ + 0x6836,0x683A,0x683B,0x683F,0x6847,0x684B,0x684D,0x684F,/* 0xF0-0xF7 */ + 0x6852,0x6856,0x6857,0x6858,0x6859,0x685A,0x685B,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_97[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x685C,0x685D,0x685E,0x685F,0x686A,0x686C,0x686D,0x686E,/* 0x40-0x47 */ + 0x686F,0x6870,0x6871,0x6872,0x6873,0x6875,0x6878,0x6879,/* 0x48-0x4F */ + 0x687A,0x687B,0x687C,0x687D,0x687E,0x687F,0x6880,0x6882,/* 0x50-0x57 */ + 0x6884,0x6887,0x6888,0x6889,0x688A,0x688B,0x688C,0x688D,/* 0x58-0x5F */ + 0x688E,0x6890,0x6891,0x6892,0x6894,0x6895,0x6896,0x6898,/* 0x60-0x67 */ + 0x6899,0x689A,0x689B,0x689C,0x689D,0x689E,0x689F,0x68A0,/* 0x68-0x6F */ + 0x68A1,0x68A3,0x68A4,0x68A5,0x68A9,0x68AA,0x68AB,0x68AC,/* 0x70-0x77 */ + 0x68AE,0x68B1,0x68B2,0x68B4,0x68B6,0x68B7,0x68B8,0x0000,/* 0x78-0x7F */ + + 0x68B9,0x68BA,0x68BB,0x68BC,0x68BD,0x68BE,0x68BF,0x68C1,/* 0x80-0x87 */ + 0x68C3,0x68C4,0x68C5,0x68C6,0x68C7,0x68C8,0x68CA,0x68CC,/* 0x88-0x8F */ + 0x68CE,0x68CF,0x68D0,0x68D1,0x68D3,0x68D4,0x68D6,0x68D7,/* 0x90-0x97 */ + 0x68D9,0x68DB,0x68DC,0x68DD,0x68DE,0x68DF,0x68E1,0x68E2,/* 0x98-0x9F */ + 0x68E4,0x68E5,0x68E6,0x68E7,0x68E8,0x68E9,0x68EA,0x68EB,/* 0xA0-0xA7 */ + 0x68EC,0x68ED,0x68EF,0x68F2,0x68F3,0x68F4,0x68F6,0x68F7,/* 0xA8-0xAF */ + 0x68F8,0x68FB,0x68FD,0x68FE,0x68FF,0x6900,0x6902,0x6903,/* 0xB0-0xB7 */ + 0x6904,0x6906,0x6907,0x6908,0x6909,0x690A,0x690C,0x690F,/* 0xB8-0xBF */ + 0x6911,0x6913,0x6914,0x6915,0x6916,0x6917,0x6918,0x6919,/* 0xC0-0xC7 */ + 0x691A,0x691B,0x691C,0x691D,0x691E,0x6921,0x6922,0x6923,/* 0xC8-0xCF */ + 0x6925,0x6926,0x6927,0x6928,0x6929,0x692A,0x692B,0x692C,/* 0xD0-0xD7 */ + 0x692E,0x692F,0x6931,0x6932,0x6933,0x6935,0x6936,0x6937,/* 0xD8-0xDF */ + 0x6938,0x693A,0x693B,0x693C,0x693E,0x6940,0x6941,0x6943,/* 0xE0-0xE7 */ + 0x6944,0x6945,0x6946,0x6947,0x6948,0x6949,0x694A,0x694B,/* 0xE8-0xEF */ + 0x694C,0x694D,0x694E,0x694F,0x6950,0x6951,0x6952,0x6953,/* 0xF0-0xF7 */ + 0x6955,0x6956,0x6958,0x6959,0x695B,0x695C,0x695F,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_98[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6961,0x6962,0x6964,0x6965,0x6967,0x6968,0x6969,0x696A,/* 0x40-0x47 */ + 0x696C,0x696D,0x696F,0x6970,0x6972,0x6973,0x6974,0x6975,/* 0x48-0x4F */ + 0x6976,0x697A,0x697B,0x697D,0x697E,0x697F,0x6981,0x6983,/* 0x50-0x57 */ + 0x6985,0x698A,0x698B,0x698C,0x698E,0x698F,0x6990,0x6991,/* 0x58-0x5F */ + 0x6992,0x6993,0x6996,0x6997,0x6999,0x699A,0x699D,0x699E,/* 0x60-0x67 */ + 0x699F,0x69A0,0x69A1,0x69A2,0x69A3,0x69A4,0x69A5,0x69A6,/* 0x68-0x6F */ + 0x69A9,0x69AA,0x69AC,0x69AE,0x69AF,0x69B0,0x69B2,0x69B3,/* 0x70-0x77 */ + 0x69B5,0x69B6,0x69B8,0x69B9,0x69BA,0x69BC,0x69BD,0x0000,/* 0x78-0x7F */ + + 0x69BE,0x69BF,0x69C0,0x69C2,0x69C3,0x69C4,0x69C5,0x69C6,/* 0x80-0x87 */ + 0x69C7,0x69C8,0x69C9,0x69CB,0x69CD,0x69CF,0x69D1,0x69D2,/* 0x88-0x8F */ + 0x69D3,0x69D5,0x69D6,0x69D7,0x69D8,0x69D9,0x69DA,0x69DC,/* 0x90-0x97 */ + 0x69DD,0x69DE,0x69E1,0x69E2,0x69E3,0x69E4,0x69E5,0x69E6,/* 0x98-0x9F */ + 0x69E7,0x69E8,0x69E9,0x69EA,0x69EB,0x69EC,0x69EE,0x69EF,/* 0xA0-0xA7 */ + 0x69F0,0x69F1,0x69F3,0x69F4,0x69F5,0x69F6,0x69F7,0x69F8,/* 0xA8-0xAF */ + 0x69F9,0x69FA,0x69FB,0x69FC,0x69FE,0x6A00,0x6A01,0x6A02,/* 0xB0-0xB7 */ + 0x6A03,0x6A04,0x6A05,0x6A06,0x6A07,0x6A08,0x6A09,0x6A0B,/* 0xB8-0xBF */ + 0x6A0C,0x6A0D,0x6A0E,0x6A0F,0x6A10,0x6A11,0x6A12,0x6A13,/* 0xC0-0xC7 */ + 0x6A14,0x6A15,0x6A16,0x6A19,0x6A1A,0x6A1B,0x6A1C,0x6A1D,/* 0xC8-0xCF */ + 0x6A1E,0x6A20,0x6A22,0x6A23,0x6A24,0x6A25,0x6A26,0x6A27,/* 0xD0-0xD7 */ + 0x6A29,0x6A2B,0x6A2C,0x6A2D,0x6A2E,0x6A30,0x6A32,0x6A33,/* 0xD8-0xDF */ + 0x6A34,0x6A36,0x6A37,0x6A38,0x6A39,0x6A3A,0x6A3B,0x6A3C,/* 0xE0-0xE7 */ + 0x6A3F,0x6A40,0x6A41,0x6A42,0x6A43,0x6A45,0x6A46,0x6A48,/* 0xE8-0xEF */ + 0x6A49,0x6A4A,0x6A4B,0x6A4C,0x6A4D,0x6A4E,0x6A4F,0x6A51,/* 0xF0-0xF7 */ + 0x6A52,0x6A53,0x6A54,0x6A55,0x6A56,0x6A57,0x6A5A,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_99[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6A5C,0x6A5D,0x6A5E,0x6A5F,0x6A60,0x6A62,0x6A63,0x6A64,/* 0x40-0x47 */ + 0x6A66,0x6A67,0x6A68,0x6A69,0x6A6A,0x6A6B,0x6A6C,0x6A6D,/* 0x48-0x4F */ + 0x6A6E,0x6A6F,0x6A70,0x6A72,0x6A73,0x6A74,0x6A75,0x6A76,/* 0x50-0x57 */ + 0x6A77,0x6A78,0x6A7A,0x6A7B,0x6A7D,0x6A7E,0x6A7F,0x6A81,/* 0x58-0x5F */ + 0x6A82,0x6A83,0x6A85,0x6A86,0x6A87,0x6A88,0x6A89,0x6A8A,/* 0x60-0x67 */ + 0x6A8B,0x6A8C,0x6A8D,0x6A8F,0x6A92,0x6A93,0x6A94,0x6A95,/* 0x68-0x6F */ + 0x6A96,0x6A98,0x6A99,0x6A9A,0x6A9B,0x6A9C,0x6A9D,0x6A9E,/* 0x70-0x77 */ + 0x6A9F,0x6AA1,0x6AA2,0x6AA3,0x6AA4,0x6AA5,0x6AA6,0x0000,/* 0x78-0x7F */ + + 0x6AA7,0x6AA8,0x6AAA,0x6AAD,0x6AAE,0x6AAF,0x6AB0,0x6AB1,/* 0x80-0x87 */ + 0x6AB2,0x6AB3,0x6AB4,0x6AB5,0x6AB6,0x6AB7,0x6AB8,0x6AB9,/* 0x88-0x8F */ + 0x6ABA,0x6ABB,0x6ABC,0x6ABD,0x6ABE,0x6ABF,0x6AC0,0x6AC1,/* 0x90-0x97 */ + 0x6AC2,0x6AC3,0x6AC4,0x6AC5,0x6AC6,0x6AC7,0x6AC8,0x6AC9,/* 0x98-0x9F */ + 0x6ACA,0x6ACB,0x6ACC,0x6ACD,0x6ACE,0x6ACF,0x6AD0,0x6AD1,/* 0xA0-0xA7 */ + 0x6AD2,0x6AD3,0x6AD4,0x6AD5,0x6AD6,0x6AD7,0x6AD8,0x6AD9,/* 0xA8-0xAF */ + 0x6ADA,0x6ADB,0x6ADC,0x6ADD,0x6ADE,0x6ADF,0x6AE0,0x6AE1,/* 0xB0-0xB7 */ + 0x6AE2,0x6AE3,0x6AE4,0x6AE5,0x6AE6,0x6AE7,0x6AE8,0x6AE9,/* 0xB8-0xBF */ + 0x6AEA,0x6AEB,0x6AEC,0x6AED,0x6AEE,0x6AEF,0x6AF0,0x6AF1,/* 0xC0-0xC7 */ + 0x6AF2,0x6AF3,0x6AF4,0x6AF5,0x6AF6,0x6AF7,0x6AF8,0x6AF9,/* 0xC8-0xCF */ + 0x6AFA,0x6AFB,0x6AFC,0x6AFD,0x6AFE,0x6AFF,0x6B00,0x6B01,/* 0xD0-0xD7 */ + 0x6B02,0x6B03,0x6B04,0x6B05,0x6B06,0x6B07,0x6B08,0x6B09,/* 0xD8-0xDF */ + 0x6B0A,0x6B0B,0x6B0C,0x6B0D,0x6B0E,0x6B0F,0x6B10,0x6B11,/* 0xE0-0xE7 */ + 0x6B12,0x6B13,0x6B14,0x6B15,0x6B16,0x6B17,0x6B18,0x6B19,/* 0xE8-0xEF */ + 0x6B1A,0x6B1B,0x6B1C,0x6B1D,0x6B1E,0x6B1F,0x6B25,0x6B26,/* 0xF0-0xF7 */ + 0x6B28,0x6B29,0x6B2A,0x6B2B,0x6B2C,0x6B2D,0x6B2E,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_9A[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6B2F,0x6B30,0x6B31,0x6B33,0x6B34,0x6B35,0x6B36,0x6B38,/* 0x40-0x47 */ + 0x6B3B,0x6B3C,0x6B3D,0x6B3F,0x6B40,0x6B41,0x6B42,0x6B44,/* 0x48-0x4F */ + 0x6B45,0x6B48,0x6B4A,0x6B4B,0x6B4D,0x6B4E,0x6B4F,0x6B50,/* 0x50-0x57 */ + 0x6B51,0x6B52,0x6B53,0x6B54,0x6B55,0x6B56,0x6B57,0x6B58,/* 0x58-0x5F */ + 0x6B5A,0x6B5B,0x6B5C,0x6B5D,0x6B5E,0x6B5F,0x6B60,0x6B61,/* 0x60-0x67 */ + 0x6B68,0x6B69,0x6B6B,0x6B6C,0x6B6D,0x6B6E,0x6B6F,0x6B70,/* 0x68-0x6F */ + 0x6B71,0x6B72,0x6B73,0x6B74,0x6B75,0x6B76,0x6B77,0x6B78,/* 0x70-0x77 */ + 0x6B7A,0x6B7D,0x6B7E,0x6B7F,0x6B80,0x6B85,0x6B88,0x0000,/* 0x78-0x7F */ + + 0x6B8C,0x6B8E,0x6B8F,0x6B90,0x6B91,0x6B94,0x6B95,0x6B97,/* 0x80-0x87 */ + 0x6B98,0x6B99,0x6B9C,0x6B9D,0x6B9E,0x6B9F,0x6BA0,0x6BA2,/* 0x88-0x8F */ + 0x6BA3,0x6BA4,0x6BA5,0x6BA6,0x6BA7,0x6BA8,0x6BA9,0x6BAB,/* 0x90-0x97 */ + 0x6BAC,0x6BAD,0x6BAE,0x6BAF,0x6BB0,0x6BB1,0x6BB2,0x6BB6,/* 0x98-0x9F */ + 0x6BB8,0x6BB9,0x6BBA,0x6BBB,0x6BBC,0x6BBD,0x6BBE,0x6BC0,/* 0xA0-0xA7 */ + 0x6BC3,0x6BC4,0x6BC6,0x6BC7,0x6BC8,0x6BC9,0x6BCA,0x6BCC,/* 0xA8-0xAF */ + 0x6BCE,0x6BD0,0x6BD1,0x6BD8,0x6BDA,0x6BDC,0x6BDD,0x6BDE,/* 0xB0-0xB7 */ + 0x6BDF,0x6BE0,0x6BE2,0x6BE3,0x6BE4,0x6BE5,0x6BE6,0x6BE7,/* 0xB8-0xBF */ + 0x6BE8,0x6BE9,0x6BEC,0x6BED,0x6BEE,0x6BF0,0x6BF1,0x6BF2,/* 0xC0-0xC7 */ + 0x6BF4,0x6BF6,0x6BF7,0x6BF8,0x6BFA,0x6BFB,0x6BFC,0x6BFE,/* 0xC8-0xCF */ + 0x6BFF,0x6C00,0x6C01,0x6C02,0x6C03,0x6C04,0x6C08,0x6C09,/* 0xD0-0xD7 */ + 0x6C0A,0x6C0B,0x6C0C,0x6C0E,0x6C12,0x6C17,0x6C1C,0x6C1D,/* 0xD8-0xDF */ + 0x6C1E,0x6C20,0x6C23,0x6C25,0x6C2B,0x6C2C,0x6C2D,0x6C31,/* 0xE0-0xE7 */ + 0x6C33,0x6C36,0x6C37,0x6C39,0x6C3A,0x6C3B,0x6C3C,0x6C3E,/* 0xE8-0xEF */ + 0x6C3F,0x6C43,0x6C44,0x6C45,0x6C48,0x6C4B,0x6C4C,0x6C4D,/* 0xF0-0xF7 */ + 0x6C4E,0x6C4F,0x6C51,0x6C52,0x6C53,0x6C56,0x6C58,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_9B[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6C59,0x6C5A,0x6C62,0x6C63,0x6C65,0x6C66,0x6C67,0x6C6B,/* 0x40-0x47 */ + 0x6C6C,0x6C6D,0x6C6E,0x6C6F,0x6C71,0x6C73,0x6C75,0x6C77,/* 0x48-0x4F */ + 0x6C78,0x6C7A,0x6C7B,0x6C7C,0x6C7F,0x6C80,0x6C84,0x6C87,/* 0x50-0x57 */ + 0x6C8A,0x6C8B,0x6C8D,0x6C8E,0x6C91,0x6C92,0x6C95,0x6C96,/* 0x58-0x5F */ + 0x6C97,0x6C98,0x6C9A,0x6C9C,0x6C9D,0x6C9E,0x6CA0,0x6CA2,/* 0x60-0x67 */ + 0x6CA8,0x6CAC,0x6CAF,0x6CB0,0x6CB4,0x6CB5,0x6CB6,0x6CB7,/* 0x68-0x6F */ + 0x6CBA,0x6CC0,0x6CC1,0x6CC2,0x6CC3,0x6CC6,0x6CC7,0x6CC8,/* 0x70-0x77 */ + 0x6CCB,0x6CCD,0x6CCE,0x6CCF,0x6CD1,0x6CD2,0x6CD8,0x0000,/* 0x78-0x7F */ + + 0x6CD9,0x6CDA,0x6CDC,0x6CDD,0x6CDF,0x6CE4,0x6CE6,0x6CE7,/* 0x80-0x87 */ + 0x6CE9,0x6CEC,0x6CED,0x6CF2,0x6CF4,0x6CF9,0x6CFF,0x6D00,/* 0x88-0x8F */ + 0x6D02,0x6D03,0x6D05,0x6D06,0x6D08,0x6D09,0x6D0A,0x6D0D,/* 0x90-0x97 */ + 0x6D0F,0x6D10,0x6D11,0x6D13,0x6D14,0x6D15,0x6D16,0x6D18,/* 0x98-0x9F */ + 0x6D1C,0x6D1D,0x6D1F,0x6D20,0x6D21,0x6D22,0x6D23,0x6D24,/* 0xA0-0xA7 */ + 0x6D26,0x6D28,0x6D29,0x6D2C,0x6D2D,0x6D2F,0x6D30,0x6D34,/* 0xA8-0xAF */ + 0x6D36,0x6D37,0x6D38,0x6D3A,0x6D3F,0x6D40,0x6D42,0x6D44,/* 0xB0-0xB7 */ + 0x6D49,0x6D4C,0x6D50,0x6D55,0x6D56,0x6D57,0x6D58,0x6D5B,/* 0xB8-0xBF */ + 0x6D5D,0x6D5F,0x6D61,0x6D62,0x6D64,0x6D65,0x6D67,0x6D68,/* 0xC0-0xC7 */ + 0x6D6B,0x6D6C,0x6D6D,0x6D70,0x6D71,0x6D72,0x6D73,0x6D75,/* 0xC8-0xCF */ + 0x6D76,0x6D79,0x6D7A,0x6D7B,0x6D7D,0x6D7E,0x6D7F,0x6D80,/* 0xD0-0xD7 */ + 0x6D81,0x6D83,0x6D84,0x6D86,0x6D87,0x6D8A,0x6D8B,0x6D8D,/* 0xD8-0xDF */ + 0x6D8F,0x6D90,0x6D92,0x6D96,0x6D97,0x6D98,0x6D99,0x6D9A,/* 0xE0-0xE7 */ + 0x6D9C,0x6DA2,0x6DA5,0x6DAC,0x6DAD,0x6DB0,0x6DB1,0x6DB3,/* 0xE8-0xEF */ + 0x6DB4,0x6DB6,0x6DB7,0x6DB9,0x6DBA,0x6DBB,0x6DBC,0x6DBD,/* 0xF0-0xF7 */ + 0x6DBE,0x6DC1,0x6DC2,0x6DC3,0x6DC8,0x6DC9,0x6DCA,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_9C[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6DCD,0x6DCE,0x6DCF,0x6DD0,0x6DD2,0x6DD3,0x6DD4,0x6DD5,/* 0x40-0x47 */ + 0x6DD7,0x6DDA,0x6DDB,0x6DDC,0x6DDF,0x6DE2,0x6DE3,0x6DE5,/* 0x48-0x4F */ + 0x6DE7,0x6DE8,0x6DE9,0x6DEA,0x6DED,0x6DEF,0x6DF0,0x6DF2,/* 0x50-0x57 */ + 0x6DF4,0x6DF5,0x6DF6,0x6DF8,0x6DFA,0x6DFD,0x6DFE,0x6DFF,/* 0x58-0x5F */ + 0x6E00,0x6E01,0x6E02,0x6E03,0x6E04,0x6E06,0x6E07,0x6E08,/* 0x60-0x67 */ + 0x6E09,0x6E0B,0x6E0F,0x6E12,0x6E13,0x6E15,0x6E18,0x6E19,/* 0x68-0x6F */ + 0x6E1B,0x6E1C,0x6E1E,0x6E1F,0x6E22,0x6E26,0x6E27,0x6E28,/* 0x70-0x77 */ + 0x6E2A,0x6E2C,0x6E2E,0x6E30,0x6E31,0x6E33,0x6E35,0x0000,/* 0x78-0x7F */ + + 0x6E36,0x6E37,0x6E39,0x6E3B,0x6E3C,0x6E3D,0x6E3E,0x6E3F,/* 0x80-0x87 */ + 0x6E40,0x6E41,0x6E42,0x6E45,0x6E46,0x6E47,0x6E48,0x6E49,/* 0x88-0x8F */ + 0x6E4A,0x6E4B,0x6E4C,0x6E4F,0x6E50,0x6E51,0x6E52,0x6E55,/* 0x90-0x97 */ + 0x6E57,0x6E59,0x6E5A,0x6E5C,0x6E5D,0x6E5E,0x6E60,0x6E61,/* 0x98-0x9F */ + 0x6E62,0x6E63,0x6E64,0x6E65,0x6E66,0x6E67,0x6E68,0x6E69,/* 0xA0-0xA7 */ + 0x6E6A,0x6E6C,0x6E6D,0x6E6F,0x6E70,0x6E71,0x6E72,0x6E73,/* 0xA8-0xAF */ + 0x6E74,0x6E75,0x6E76,0x6E77,0x6E78,0x6E79,0x6E7A,0x6E7B,/* 0xB0-0xB7 */ + 0x6E7C,0x6E7D,0x6E80,0x6E81,0x6E82,0x6E84,0x6E87,0x6E88,/* 0xB8-0xBF */ + 0x6E8A,0x6E8B,0x6E8C,0x6E8D,0x6E8E,0x6E91,0x6E92,0x6E93,/* 0xC0-0xC7 */ + 0x6E94,0x6E95,0x6E96,0x6E97,0x6E99,0x6E9A,0x6E9B,0x6E9D,/* 0xC8-0xCF */ + 0x6E9E,0x6EA0,0x6EA1,0x6EA3,0x6EA4,0x6EA6,0x6EA8,0x6EA9,/* 0xD0-0xD7 */ + 0x6EAB,0x6EAC,0x6EAD,0x6EAE,0x6EB0,0x6EB3,0x6EB5,0x6EB8,/* 0xD8-0xDF */ + 0x6EB9,0x6EBC,0x6EBE,0x6EBF,0x6EC0,0x6EC3,0x6EC4,0x6EC5,/* 0xE0-0xE7 */ + 0x6EC6,0x6EC8,0x6EC9,0x6ECA,0x6ECC,0x6ECD,0x6ECE,0x6ED0,/* 0xE8-0xEF */ + 0x6ED2,0x6ED6,0x6ED8,0x6ED9,0x6EDB,0x6EDC,0x6EDD,0x6EE3,/* 0xF0-0xF7 */ + 0x6EE7,0x6EEA,0x6EEB,0x6EEC,0x6EED,0x6EEE,0x6EEF,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_9D[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6EF0,0x6EF1,0x6EF2,0x6EF3,0x6EF5,0x6EF6,0x6EF7,0x6EF8,/* 0x40-0x47 */ + 0x6EFA,0x6EFB,0x6EFC,0x6EFD,0x6EFE,0x6EFF,0x6F00,0x6F01,/* 0x48-0x4F */ + 0x6F03,0x6F04,0x6F05,0x6F07,0x6F08,0x6F0A,0x6F0B,0x6F0C,/* 0x50-0x57 */ + 0x6F0D,0x6F0E,0x6F10,0x6F11,0x6F12,0x6F16,0x6F17,0x6F18,/* 0x58-0x5F */ + 0x6F19,0x6F1A,0x6F1B,0x6F1C,0x6F1D,0x6F1E,0x6F1F,0x6F21,/* 0x60-0x67 */ + 0x6F22,0x6F23,0x6F25,0x6F26,0x6F27,0x6F28,0x6F2C,0x6F2E,/* 0x68-0x6F */ + 0x6F30,0x6F32,0x6F34,0x6F35,0x6F37,0x6F38,0x6F39,0x6F3A,/* 0x70-0x77 */ + 0x6F3B,0x6F3C,0x6F3D,0x6F3F,0x6F40,0x6F41,0x6F42,0x0000,/* 0x78-0x7F */ + + 0x6F43,0x6F44,0x6F45,0x6F48,0x6F49,0x6F4A,0x6F4C,0x6F4E,/* 0x80-0x87 */ + 0x6F4F,0x6F50,0x6F51,0x6F52,0x6F53,0x6F54,0x6F55,0x6F56,/* 0x88-0x8F */ + 0x6F57,0x6F59,0x6F5A,0x6F5B,0x6F5D,0x6F5F,0x6F60,0x6F61,/* 0x90-0x97 */ + 0x6F63,0x6F64,0x6F65,0x6F67,0x6F68,0x6F69,0x6F6A,0x6F6B,/* 0x98-0x9F */ + 0x6F6C,0x6F6F,0x6F70,0x6F71,0x6F73,0x6F75,0x6F76,0x6F77,/* 0xA0-0xA7 */ + 0x6F79,0x6F7B,0x6F7D,0x6F7E,0x6F7F,0x6F80,0x6F81,0x6F82,/* 0xA8-0xAF */ + 0x6F83,0x6F85,0x6F86,0x6F87,0x6F8A,0x6F8B,0x6F8F,0x6F90,/* 0xB0-0xB7 */ + 0x6F91,0x6F92,0x6F93,0x6F94,0x6F95,0x6F96,0x6F97,0x6F98,/* 0xB8-0xBF */ + 0x6F99,0x6F9A,0x6F9B,0x6F9D,0x6F9E,0x6F9F,0x6FA0,0x6FA2,/* 0xC0-0xC7 */ + 0x6FA3,0x6FA4,0x6FA5,0x6FA6,0x6FA8,0x6FA9,0x6FAA,0x6FAB,/* 0xC8-0xCF */ + 0x6FAC,0x6FAD,0x6FAE,0x6FAF,0x6FB0,0x6FB1,0x6FB2,0x6FB4,/* 0xD0-0xD7 */ + 0x6FB5,0x6FB7,0x6FB8,0x6FBA,0x6FBB,0x6FBC,0x6FBD,0x6FBE,/* 0xD8-0xDF */ + 0x6FBF,0x6FC1,0x6FC3,0x6FC4,0x6FC5,0x6FC6,0x6FC7,0x6FC8,/* 0xE0-0xE7 */ + 0x6FCA,0x6FCB,0x6FCC,0x6FCD,0x6FCE,0x6FCF,0x6FD0,0x6FD3,/* 0xE8-0xEF */ + 0x6FD4,0x6FD5,0x6FD6,0x6FD7,0x6FD8,0x6FD9,0x6FDA,0x6FDB,/* 0xF0-0xF7 */ + 0x6FDC,0x6FDD,0x6FDF,0x6FE2,0x6FE3,0x6FE4,0x6FE5,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_9E[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6FE6,0x6FE7,0x6FE8,0x6FE9,0x6FEA,0x6FEB,0x6FEC,0x6FED,/* 0x40-0x47 */ + 0x6FF0,0x6FF1,0x6FF2,0x6FF3,0x6FF4,0x6FF5,0x6FF6,0x6FF7,/* 0x48-0x4F */ + 0x6FF8,0x6FF9,0x6FFA,0x6FFB,0x6FFC,0x6FFD,0x6FFE,0x6FFF,/* 0x50-0x57 */ + 0x7000,0x7001,0x7002,0x7003,0x7004,0x7005,0x7006,0x7007,/* 0x58-0x5F */ + 0x7008,0x7009,0x700A,0x700B,0x700C,0x700D,0x700E,0x700F,/* 0x60-0x67 */ + 0x7010,0x7012,0x7013,0x7014,0x7015,0x7016,0x7017,0x7018,/* 0x68-0x6F */ + 0x7019,0x701C,0x701D,0x701E,0x701F,0x7020,0x7021,0x7022,/* 0x70-0x77 */ + 0x7024,0x7025,0x7026,0x7027,0x7028,0x7029,0x702A,0x0000,/* 0x78-0x7F */ + + 0x702B,0x702C,0x702D,0x702E,0x702F,0x7030,0x7031,0x7032,/* 0x80-0x87 */ + 0x7033,0x7034,0x7036,0x7037,0x7038,0x703A,0x703B,0x703C,/* 0x88-0x8F */ + 0x703D,0x703E,0x703F,0x7040,0x7041,0x7042,0x7043,0x7044,/* 0x90-0x97 */ + 0x7045,0x7046,0x7047,0x7048,0x7049,0x704A,0x704B,0x704D,/* 0x98-0x9F */ + 0x704E,0x7050,0x7051,0x7052,0x7053,0x7054,0x7055,0x7056,/* 0xA0-0xA7 */ + 0x7057,0x7058,0x7059,0x705A,0x705B,0x705C,0x705D,0x705F,/* 0xA8-0xAF */ + 0x7060,0x7061,0x7062,0x7063,0x7064,0x7065,0x7066,0x7067,/* 0xB0-0xB7 */ + 0x7068,0x7069,0x706A,0x706E,0x7071,0x7072,0x7073,0x7074,/* 0xB8-0xBF */ + 0x7077,0x7079,0x707A,0x707B,0x707D,0x7081,0x7082,0x7083,/* 0xC0-0xC7 */ + 0x7084,0x7086,0x7087,0x7088,0x708B,0x708C,0x708D,0x708F,/* 0xC8-0xCF */ + 0x7090,0x7091,0x7093,0x7097,0x7098,0x709A,0x709B,0x709E,/* 0xD0-0xD7 */ + 0x709F,0x70A0,0x70A1,0x70A2,0x70A3,0x70A4,0x70A5,0x70A6,/* 0xD8-0xDF */ + 0x70A7,0x70A8,0x70A9,0x70AA,0x70B0,0x70B2,0x70B4,0x70B5,/* 0xE0-0xE7 */ + 0x70B6,0x70BA,0x70BE,0x70BF,0x70C4,0x70C5,0x70C6,0x70C7,/* 0xE8-0xEF */ + 0x70C9,0x70CB,0x70CC,0x70CD,0x70CE,0x70CF,0x70D0,0x70D1,/* 0xF0-0xF7 */ + 0x70D2,0x70D3,0x70D4,0x70D5,0x70D6,0x70D7,0x70DA,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_9F[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x70DC,0x70DD,0x70DE,0x70E0,0x70E1,0x70E2,0x70E3,0x70E5,/* 0x40-0x47 */ + 0x70EA,0x70EE,0x70F0,0x70F1,0x70F2,0x70F3,0x70F4,0x70F5,/* 0x48-0x4F */ + 0x70F6,0x70F8,0x70FA,0x70FB,0x70FC,0x70FE,0x70FF,0x7100,/* 0x50-0x57 */ + 0x7101,0x7102,0x7103,0x7104,0x7105,0x7106,0x7107,0x7108,/* 0x58-0x5F */ + 0x710B,0x710C,0x710D,0x710E,0x710F,0x7111,0x7112,0x7114,/* 0x60-0x67 */ + 0x7117,0x711B,0x711C,0x711D,0x711E,0x711F,0x7120,0x7121,/* 0x68-0x6F */ + 0x7122,0x7123,0x7124,0x7125,0x7127,0x7128,0x7129,0x712A,/* 0x70-0x77 */ + 0x712B,0x712C,0x712D,0x712E,0x7132,0x7133,0x7134,0x0000,/* 0x78-0x7F */ + + 0x7135,0x7137,0x7138,0x7139,0x713A,0x713B,0x713C,0x713D,/* 0x80-0x87 */ + 0x713E,0x713F,0x7140,0x7141,0x7142,0x7143,0x7144,0x7146,/* 0x88-0x8F */ + 0x7147,0x7148,0x7149,0x714B,0x714D,0x714F,0x7150,0x7151,/* 0x90-0x97 */ + 0x7152,0x7153,0x7154,0x7155,0x7156,0x7157,0x7158,0x7159,/* 0x98-0x9F */ + 0x715A,0x715B,0x715D,0x715F,0x7160,0x7161,0x7162,0x7163,/* 0xA0-0xA7 */ + 0x7165,0x7169,0x716A,0x716B,0x716C,0x716D,0x716F,0x7170,/* 0xA8-0xAF */ + 0x7171,0x7174,0x7175,0x7176,0x7177,0x7179,0x717B,0x717C,/* 0xB0-0xB7 */ + 0x717E,0x717F,0x7180,0x7181,0x7182,0x7183,0x7185,0x7186,/* 0xB8-0xBF */ + 0x7187,0x7188,0x7189,0x718B,0x718C,0x718D,0x718E,0x7190,/* 0xC0-0xC7 */ + 0x7191,0x7192,0x7193,0x7195,0x7196,0x7197,0x719A,0x719B,/* 0xC8-0xCF */ + 0x719C,0x719D,0x719E,0x71A1,0x71A2,0x71A3,0x71A4,0x71A5,/* 0xD0-0xD7 */ + 0x71A6,0x71A7,0x71A9,0x71AA,0x71AB,0x71AD,0x71AE,0x71AF,/* 0xD8-0xDF */ + 0x71B0,0x71B1,0x71B2,0x71B4,0x71B6,0x71B7,0x71B8,0x71BA,/* 0xE0-0xE7 */ + 0x71BB,0x71BC,0x71BD,0x71BE,0x71BF,0x71C0,0x71C1,0x71C2,/* 0xE8-0xEF */ + 0x71C4,0x71C5,0x71C6,0x71C7,0x71C8,0x71C9,0x71CA,0x71CB,/* 0xF0-0xF7 */ + 0x71CC,0x71CD,0x71CF,0x71D0,0x71D1,0x71D2,0x71D3,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_A0[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x71D6,0x71D7,0x71D8,0x71D9,0x71DA,0x71DB,0x71DC,0x71DD,/* 0x40-0x47 */ + 0x71DE,0x71DF,0x71E1,0x71E2,0x71E3,0x71E4,0x71E6,0x71E8,/* 0x48-0x4F */ + 0x71E9,0x71EA,0x71EB,0x71EC,0x71ED,0x71EF,0x71F0,0x71F1,/* 0x50-0x57 */ + 0x71F2,0x71F3,0x71F4,0x71F5,0x71F6,0x71F7,0x71F8,0x71FA,/* 0x58-0x5F */ + 0x71FB,0x71FC,0x71FD,0x71FE,0x71FF,0x7200,0x7201,0x7202,/* 0x60-0x67 */ + 0x7203,0x7204,0x7205,0x7207,0x7208,0x7209,0x720A,0x720B,/* 0x68-0x6F */ + 0x720C,0x720D,0x720E,0x720F,0x7210,0x7211,0x7212,0x7213,/* 0x70-0x77 */ + 0x7214,0x7215,0x7216,0x7217,0x7218,0x7219,0x721A,0x0000,/* 0x78-0x7F */ + + 0x721B,0x721C,0x721E,0x721F,0x7220,0x7221,0x7222,0x7223,/* 0x80-0x87 */ + 0x7224,0x7225,0x7226,0x7227,0x7229,0x722B,0x722D,0x722E,/* 0x88-0x8F */ + 0x722F,0x7232,0x7233,0x7234,0x723A,0x723C,0x723E,0x7240,/* 0x90-0x97 */ + 0x7241,0x7242,0x7243,0x7244,0x7245,0x7246,0x7249,0x724A,/* 0x98-0x9F */ + 0x724B,0x724E,0x724F,0x7250,0x7251,0x7253,0x7254,0x7255,/* 0xA0-0xA7 */ + 0x7257,0x7258,0x725A,0x725C,0x725E,0x7260,0x7263,0x7264,/* 0xA8-0xAF */ + 0x7265,0x7268,0x726A,0x726B,0x726C,0x726D,0x7270,0x7271,/* 0xB0-0xB7 */ + 0x7273,0x7274,0x7276,0x7277,0x7278,0x727B,0x727C,0x727D,/* 0xB8-0xBF */ + 0x7282,0x7283,0x7285,0x7286,0x7287,0x7288,0x7289,0x728C,/* 0xC0-0xC7 */ + 0x728E,0x7290,0x7291,0x7293,0x7294,0x7295,0x7296,0x7297,/* 0xC8-0xCF */ + 0x7298,0x7299,0x729A,0x729B,0x729C,0x729D,0x729E,0x72A0,/* 0xD0-0xD7 */ + 0x72A1,0x72A2,0x72A3,0x72A4,0x72A5,0x72A6,0x72A7,0x72A8,/* 0xD8-0xDF */ + 0x72A9,0x72AA,0x72AB,0x72AE,0x72B1,0x72B2,0x72B3,0x72B5,/* 0xE0-0xE7 */ + 0x72BA,0x72BB,0x72BC,0x72BD,0x72BE,0x72BF,0x72C0,0x72C5,/* 0xE8-0xEF */ + 0x72C6,0x72C7,0x72C9,0x72CA,0x72CB,0x72CC,0x72CF,0x72D1,/* 0xF0-0xF7 */ + 0x72D3,0x72D4,0x72D5,0x72D6,0x72D8,0x72DA,0x72DB,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_A1[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x3000,0x3001,0x3002,0x00B7,0x02C9,0x02C7,0x00A8,/* 0xA0-0xA7 */ + 0x3003,0x3005,0x2014,0xFF5E,0x2016,0x2026,0x2018,0x2019,/* 0xA8-0xAF */ + 0x201C,0x201D,0x3014,0x3015,0x3008,0x3009,0x300A,0x300B,/* 0xB0-0xB7 */ + 0x300C,0x300D,0x300E,0x300F,0x3016,0x3017,0x3010,0x3011,/* 0xB8-0xBF */ + 0x00B1,0x00D7,0x00F7,0x2236,0x2227,0x2228,0x2211,0x220F,/* 0xC0-0xC7 */ + 0x222A,0x2229,0x2208,0x2237,0x221A,0x22A5,0x2225,0x2220,/* 0xC8-0xCF */ + 0x2312,0x2299,0x222B,0x222E,0x2261,0x224C,0x2248,0x223D,/* 0xD0-0xD7 */ + 0x221D,0x2260,0x226E,0x226F,0x2264,0x2265,0x221E,0x2235,/* 0xD8-0xDF */ + 0x2234,0x2642,0x2640,0x00B0,0x2032,0x2033,0x2103,0xFF04,/* 0xE0-0xE7 */ + 0x00A4,0xFFE0,0xFFE1,0x2030,0x00A7,0x2116,0x2606,0x2605,/* 0xE8-0xEF */ + 0x25CB,0x25CF,0x25CE,0x25C7,0x25C6,0x25A1,0x25A0,0x25B3,/* 0xF0-0xF7 */ + 0x25B2,0x203B,0x2192,0x2190,0x2191,0x2193,0x3013,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_A2[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x2170,0x2171,0x2172,0x2173,0x2174,0x2175,0x2176,/* 0xA0-0xA7 */ + 0x2177,0x2178,0x2179,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xA8-0xAF */ + 0x0000,0x2488,0x2489,0x248A,0x248B,0x248C,0x248D,0x248E,/* 0xB0-0xB7 */ + 0x248F,0x2490,0x2491,0x2492,0x2493,0x2494,0x2495,0x2496,/* 0xB8-0xBF */ + 0x2497,0x2498,0x2499,0x249A,0x249B,0x2474,0x2475,0x2476,/* 0xC0-0xC7 */ + 0x2477,0x2478,0x2479,0x247A,0x247B,0x247C,0x247D,0x247E,/* 0xC8-0xCF */ + 0x247F,0x2480,0x2481,0x2482,0x2483,0x2484,0x2485,0x2486,/* 0xD0-0xD7 */ + 0x2487,0x2460,0x2461,0x2462,0x2463,0x2464,0x2465,0x2466,/* 0xD8-0xDF */ + 0x2467,0x2468,0x2469,0x0000,0x0000,0x3220,0x3221,0x3222,/* 0xE0-0xE7 */ + 0x3223,0x3224,0x3225,0x3226,0x3227,0x3228,0x3229,0x0000,/* 0xE8-0xEF */ + 0x0000,0x2160,0x2161,0x2162,0x2163,0x2164,0x2165,0x2166,/* 0xF0-0xF7 */ + 0x2167,0x2168,0x2169,0x216A,0x216B,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_A3[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0xFF01,0xFF02,0xFF03,0xFFE5,0xFF05,0xFF06,0xFF07,/* 0xA0-0xA7 */ + 0xFF08,0xFF09,0xFF0A,0xFF0B,0xFF0C,0xFF0D,0xFF0E,0xFF0F,/* 0xA8-0xAF */ + 0xFF10,0xFF11,0xFF12,0xFF13,0xFF14,0xFF15,0xFF16,0xFF17,/* 0xB0-0xB7 */ + 0xFF18,0xFF19,0xFF1A,0xFF1B,0xFF1C,0xFF1D,0xFF1E,0xFF1F,/* 0xB8-0xBF */ + 0xFF20,0xFF21,0xFF22,0xFF23,0xFF24,0xFF25,0xFF26,0xFF27,/* 0xC0-0xC7 */ + 0xFF28,0xFF29,0xFF2A,0xFF2B,0xFF2C,0xFF2D,0xFF2E,0xFF2F,/* 0xC8-0xCF */ + 0xFF30,0xFF31,0xFF32,0xFF33,0xFF34,0xFF35,0xFF36,0xFF37,/* 0xD0-0xD7 */ + 0xFF38,0xFF39,0xFF3A,0xFF3B,0xFF3C,0xFF3D,0xFF3E,0xFF3F,/* 0xD8-0xDF */ + 0xFF40,0xFF41,0xFF42,0xFF43,0xFF44,0xFF45,0xFF46,0xFF47,/* 0xE0-0xE7 */ + 0xFF48,0xFF49,0xFF4A,0xFF4B,0xFF4C,0xFF4D,0xFF4E,0xFF4F,/* 0xE8-0xEF */ + 0xFF50,0xFF51,0xFF52,0xFF53,0xFF54,0xFF55,0xFF56,0xFF57,/* 0xF0-0xF7 */ + 0xFF58,0xFF59,0xFF5A,0xFF5B,0xFF5C,0xFF5D,0xFFE3,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_A4[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x3041,0x3042,0x3043,0x3044,0x3045,0x3046,0x3047,/* 0xA0-0xA7 */ + 0x3048,0x3049,0x304A,0x304B,0x304C,0x304D,0x304E,0x304F,/* 0xA8-0xAF */ + 0x3050,0x3051,0x3052,0x3053,0x3054,0x3055,0x3056,0x3057,/* 0xB0-0xB7 */ + 0x3058,0x3059,0x305A,0x305B,0x305C,0x305D,0x305E,0x305F,/* 0xB8-0xBF */ + 0x3060,0x3061,0x3062,0x3063,0x3064,0x3065,0x3066,0x3067,/* 0xC0-0xC7 */ + 0x3068,0x3069,0x306A,0x306B,0x306C,0x306D,0x306E,0x306F,/* 0xC8-0xCF */ + 0x3070,0x3071,0x3072,0x3073,0x3074,0x3075,0x3076,0x3077,/* 0xD0-0xD7 */ + 0x3078,0x3079,0x307A,0x307B,0x307C,0x307D,0x307E,0x307F,/* 0xD8-0xDF */ + 0x3080,0x3081,0x3082,0x3083,0x3084,0x3085,0x3086,0x3087,/* 0xE0-0xE7 */ + 0x3088,0x3089,0x308A,0x308B,0x308C,0x308D,0x308E,0x308F,/* 0xE8-0xEF */ + 0x3090,0x3091,0x3092,0x3093,0x0000,0x0000,0x0000,0x0000,/* 0xF0-0xF7 */ +}; + +static const wchar_t c2u_A5[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x30A1,0x30A2,0x30A3,0x30A4,0x30A5,0x30A6,0x30A7,/* 0xA0-0xA7 */ + 0x30A8,0x30A9,0x30AA,0x30AB,0x30AC,0x30AD,0x30AE,0x30AF,/* 0xA8-0xAF */ + 0x30B0,0x30B1,0x30B2,0x30B3,0x30B4,0x30B5,0x30B6,0x30B7,/* 0xB0-0xB7 */ + 0x30B8,0x30B9,0x30BA,0x30BB,0x30BC,0x30BD,0x30BE,0x30BF,/* 0xB8-0xBF */ + 0x30C0,0x30C1,0x30C2,0x30C3,0x30C4,0x30C5,0x30C6,0x30C7,/* 0xC0-0xC7 */ + 0x30C8,0x30C9,0x30CA,0x30CB,0x30CC,0x30CD,0x30CE,0x30CF,/* 0xC8-0xCF */ + 0x30D0,0x30D1,0x30D2,0x30D3,0x30D4,0x30D5,0x30D6,0x30D7,/* 0xD0-0xD7 */ + 0x30D8,0x30D9,0x30DA,0x30DB,0x30DC,0x30DD,0x30DE,0x30DF,/* 0xD8-0xDF */ + 0x30E0,0x30E1,0x30E2,0x30E3,0x30E4,0x30E5,0x30E6,0x30E7,/* 0xE0-0xE7 */ + 0x30E8,0x30E9,0x30EA,0x30EB,0x30EC,0x30ED,0x30EE,0x30EF,/* 0xE8-0xEF */ + 0x30F0,0x30F1,0x30F2,0x30F3,0x30F4,0x30F5,0x30F6,0x0000,/* 0xF0-0xF7 */ +}; + +static const wchar_t c2u_A6[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x0391,0x0392,0x0393,0x0394,0x0395,0x0396,0x0397,/* 0xA0-0xA7 */ + 0x0398,0x0399,0x039A,0x039B,0x039C,0x039D,0x039E,0x039F,/* 0xA8-0xAF */ + 0x03A0,0x03A1,0x03A3,0x03A4,0x03A5,0x03A6,0x03A7,0x03A8,/* 0xB0-0xB7 */ + 0x03A9,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xB8-0xBF */ + 0x0000,0x03B1,0x03B2,0x03B3,0x03B4,0x03B5,0x03B6,0x03B7,/* 0xC0-0xC7 */ + 0x03B8,0x03B9,0x03BA,0x03BB,0x03BC,0x03BD,0x03BE,0x03BF,/* 0xC8-0xCF */ + 0x03C0,0x03C1,0x03C3,0x03C4,0x03C5,0x03C6,0x03C7,0x03C8,/* 0xD0-0xD7 */ + 0x03C9,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xD8-0xDF */ + 0xFE35,0xFE36,0xFE39,0xFE3A,0xFE3F,0xFE40,0xFE3D,0xFE3E,/* 0xE0-0xE7 */ + 0xFE41,0xFE42,0xFE43,0xFE44,0x0000,0x0000,0xFE3B,0xFE3C,/* 0xE8-0xEF */ + 0xFE37,0xFE38,0xFE31,0x0000,0xFE33,0xFE34,0x0000,0x0000,/* 0xF0-0xF7 */ +}; + +static const wchar_t c2u_A7[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x0410,0x0411,0x0412,0x0413,0x0414,0x0415,0x0401,/* 0xA0-0xA7 */ + 0x0416,0x0417,0x0418,0x0419,0x041A,0x041B,0x041C,0x041D,/* 0xA8-0xAF */ + 0x041E,0x041F,0x0420,0x0421,0x0422,0x0423,0x0424,0x0425,/* 0xB0-0xB7 */ + 0x0426,0x0427,0x0428,0x0429,0x042A,0x042B,0x042C,0x042D,/* 0xB8-0xBF */ + 0x042E,0x042F,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xC0-0xC7 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xC8-0xCF */ + 0x0000,0x0430,0x0431,0x0432,0x0433,0x0434,0x0435,0x0451,/* 0xD0-0xD7 */ + 0x0436,0x0437,0x0438,0x0439,0x043A,0x043B,0x043C,0x043D,/* 0xD8-0xDF */ + 0x043E,0x043F,0x0440,0x0441,0x0442,0x0443,0x0444,0x0445,/* 0xE0-0xE7 */ + 0x0446,0x0447,0x0448,0x0449,0x044A,0x044B,0x044C,0x044D,/* 0xE8-0xEF */ + 0x044E,0x044F,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xF0-0xF7 */ +}; + +static const wchar_t c2u_A8[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x02CA,0x02CB,0x02D9,0x2013,0x2015,0x2025,0x2035,0x2105,/* 0x40-0x47 */ + 0x2109,0x2196,0x2197,0x2198,0x2199,0x2215,0x221F,0x2223,/* 0x48-0x4F */ + 0x2252,0x2266,0x2267,0x22BF,0x2550,0x2551,0x2552,0x2553,/* 0x50-0x57 */ + 0x2554,0x2555,0x2556,0x2557,0x2558,0x2559,0x255A,0x255B,/* 0x58-0x5F */ + 0x255C,0x255D,0x255E,0x255F,0x2560,0x2561,0x2562,0x2563,/* 0x60-0x67 */ + 0x2564,0x2565,0x2566,0x2567,0x2568,0x2569,0x256A,0x256B,/* 0x68-0x6F */ + 0x256C,0x256D,0x256E,0x256F,0x2570,0x2571,0x2572,0x2573,/* 0x70-0x77 */ + 0x2581,0x2582,0x2583,0x2584,0x2585,0x2586,0x2587,0x0000,/* 0x78-0x7F */ + + 0x2588,0x2589,0x258A,0x258B,0x258C,0x258D,0x258E,0x258F,/* 0x80-0x87 */ + 0x2593,0x2594,0x2595,0x25BC,0x25BD,0x25E2,0x25E3,0x25E4,/* 0x88-0x8F */ + 0x25E5,0x2609,0x2295,0x3012,0x301D,0x301E,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x0101,0x00E1,0x01CE,0x00E0,0x0113,0x00E9,0x011B,/* 0xA0-0xA7 */ + 0x00E8,0x012B,0x00ED,0x01D0,0x00EC,0x014D,0x00F3,0x01D2,/* 0xA8-0xAF */ + 0x00F2,0x016B,0x00FA,0x01D4,0x00F9,0x01D6,0x01D8,0x01DA,/* 0xB0-0xB7 */ + 0x01DC,0x00FC,0x00EA,0x0251,0x0000,0x0144,0x0148,0x0000,/* 0xB8-0xBF */ + 0x0261,0x0000,0x0000,0x0000,0x0000,0x3105,0x3106,0x3107,/* 0xC0-0xC7 */ + 0x3108,0x3109,0x310A,0x310B,0x310C,0x310D,0x310E,0x310F,/* 0xC8-0xCF */ + 0x3110,0x3111,0x3112,0x3113,0x3114,0x3115,0x3116,0x3117,/* 0xD0-0xD7 */ + 0x3118,0x3119,0x311A,0x311B,0x311C,0x311D,0x311E,0x311F,/* 0xD8-0xDF */ + 0x3120,0x3121,0x3122,0x3123,0x3124,0x3125,0x3126,0x3127,/* 0xE0-0xE7 */ + 0x3128,0x3129,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xE8-0xEF */ +}; + +static const wchar_t c2u_A9[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x3021,0x3022,0x3023,0x3024,0x3025,0x3026,0x3027,0x3028,/* 0x40-0x47 */ + 0x3029,0x32A3,0x338E,0x338F,0x339C,0x339D,0x339E,0x33A1,/* 0x48-0x4F */ + 0x33C4,0x33CE,0x33D1,0x33D2,0x33D5,0xFE30,0xFFE2,0xFFE4,/* 0x50-0x57 */ + 0x0000,0x2121,0x3231,0x0000,0x2010,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x30FC,0x309B,0x309C,0x30FD,0x30FE,0x3006,0x309D,0x309E,/* 0x60-0x67 */ + 0xFE49,0xFE4A,0xFE4B,0xFE4C,0xFE4D,0xFE4E,0xFE4F,0xFE50,/* 0x68-0x6F */ + 0xFE51,0xFE52,0xFE54,0xFE55,0xFE56,0xFE57,0xFE59,0xFE5A,/* 0x70-0x77 */ + 0xFE5B,0xFE5C,0xFE5D,0xFE5E,0xFE5F,0xFE60,0xFE61,0x0000,/* 0x78-0x7F */ + + 0xFE62,0xFE63,0xFE64,0xFE65,0xFE66,0xFE68,0xFE69,0xFE6A,/* 0x80-0x87 */ + 0xFE6B,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x3007,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x0000,0x0000,0x0000,0x2500,0x2501,0x2502,0x2503,/* 0xA0-0xA7 */ + 0x2504,0x2505,0x2506,0x2507,0x2508,0x2509,0x250A,0x250B,/* 0xA8-0xAF */ + 0x250C,0x250D,0x250E,0x250F,0x2510,0x2511,0x2512,0x2513,/* 0xB0-0xB7 */ + 0x2514,0x2515,0x2516,0x2517,0x2518,0x2519,0x251A,0x251B,/* 0xB8-0xBF */ + 0x251C,0x251D,0x251E,0x251F,0x2520,0x2521,0x2522,0x2523,/* 0xC0-0xC7 */ + 0x2524,0x2525,0x2526,0x2527,0x2528,0x2529,0x252A,0x252B,/* 0xC8-0xCF */ + 0x252C,0x252D,0x252E,0x252F,0x2530,0x2531,0x2532,0x2533,/* 0xD0-0xD7 */ + 0x2534,0x2535,0x2536,0x2537,0x2538,0x2539,0x253A,0x253B,/* 0xD8-0xDF */ + 0x253C,0x253D,0x253E,0x253F,0x2540,0x2541,0x2542,0x2543,/* 0xE0-0xE7 */ + 0x2544,0x2545,0x2546,0x2547,0x2548,0x2549,0x254A,0x254B,/* 0xE8-0xEF */ +}; + +static const wchar_t c2u_AA[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x72DC,0x72DD,0x72DF,0x72E2,0x72E3,0x72E4,0x72E5,0x72E6,/* 0x40-0x47 */ + 0x72E7,0x72EA,0x72EB,0x72F5,0x72F6,0x72F9,0x72FD,0x72FE,/* 0x48-0x4F */ + 0x72FF,0x7300,0x7302,0x7304,0x7305,0x7306,0x7307,0x7308,/* 0x50-0x57 */ + 0x7309,0x730B,0x730C,0x730D,0x730F,0x7310,0x7311,0x7312,/* 0x58-0x5F */ + 0x7314,0x7318,0x7319,0x731A,0x731F,0x7320,0x7323,0x7324,/* 0x60-0x67 */ + 0x7326,0x7327,0x7328,0x732D,0x732F,0x7330,0x7332,0x7333,/* 0x68-0x6F */ + 0x7335,0x7336,0x733A,0x733B,0x733C,0x733D,0x7340,0x7341,/* 0x70-0x77 */ + 0x7342,0x7343,0x7344,0x7345,0x7346,0x7347,0x7348,0x0000,/* 0x78-0x7F */ + + 0x7349,0x734A,0x734B,0x734C,0x734E,0x734F,0x7351,0x7353,/* 0x80-0x87 */ + 0x7354,0x7355,0x7356,0x7358,0x7359,0x735A,0x735B,0x735C,/* 0x88-0x8F */ + 0x735D,0x735E,0x735F,0x7361,0x7362,0x7363,0x7364,0x7365,/* 0x90-0x97 */ + 0x7366,0x7367,0x7368,0x7369,0x736A,0x736B,0x736E,0x7370,/* 0x98-0x9F */ + 0x7371,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xA0-0xA7 */ +}; + +static const wchar_t c2u_AB[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7372,0x7373,0x7374,0x7375,0x7376,0x7377,0x7378,0x7379,/* 0x40-0x47 */ + 0x737A,0x737B,0x737C,0x737D,0x737F,0x7380,0x7381,0x7382,/* 0x48-0x4F */ + 0x7383,0x7385,0x7386,0x7388,0x738A,0x738C,0x738D,0x738F,/* 0x50-0x57 */ + 0x7390,0x7392,0x7393,0x7394,0x7395,0x7397,0x7398,0x7399,/* 0x58-0x5F */ + 0x739A,0x739C,0x739D,0x739E,0x73A0,0x73A1,0x73A3,0x73A4,/* 0x60-0x67 */ + 0x73A5,0x73A6,0x73A7,0x73A8,0x73AA,0x73AC,0x73AD,0x73B1,/* 0x68-0x6F */ + 0x73B4,0x73B5,0x73B6,0x73B8,0x73B9,0x73BC,0x73BD,0x73BE,/* 0x70-0x77 */ + 0x73BF,0x73C1,0x73C3,0x73C4,0x73C5,0x73C6,0x73C7,0x0000,/* 0x78-0x7F */ + + 0x73CB,0x73CC,0x73CE,0x73D2,0x73D3,0x73D4,0x73D5,0x73D6,/* 0x80-0x87 */ + 0x73D7,0x73D8,0x73DA,0x73DB,0x73DC,0x73DD,0x73DF,0x73E1,/* 0x88-0x8F */ + 0x73E2,0x73E3,0x73E4,0x73E6,0x73E8,0x73EA,0x73EB,0x73EC,/* 0x90-0x97 */ + 0x73EE,0x73EF,0x73F0,0x73F1,0x73F3,0x73F4,0x73F5,0x73F6,/* 0x98-0x9F */ + 0x73F7,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xA0-0xA7 */ +}; + +static const wchar_t c2u_AC[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x73F8,0x73F9,0x73FA,0x73FB,0x73FC,0x73FD,0x73FE,0x73FF,/* 0x40-0x47 */ + 0x7400,0x7401,0x7402,0x7404,0x7407,0x7408,0x740B,0x740C,/* 0x48-0x4F */ + 0x740D,0x740E,0x7411,0x7412,0x7413,0x7414,0x7415,0x7416,/* 0x50-0x57 */ + 0x7417,0x7418,0x7419,0x741C,0x741D,0x741E,0x741F,0x7420,/* 0x58-0x5F */ + 0x7421,0x7423,0x7424,0x7427,0x7429,0x742B,0x742D,0x742F,/* 0x60-0x67 */ + 0x7431,0x7432,0x7437,0x7438,0x7439,0x743A,0x743B,0x743D,/* 0x68-0x6F */ + 0x743E,0x743F,0x7440,0x7442,0x7443,0x7444,0x7445,0x7446,/* 0x70-0x77 */ + 0x7447,0x7448,0x7449,0x744A,0x744B,0x744C,0x744D,0x0000,/* 0x78-0x7F */ + + 0x744E,0x744F,0x7450,0x7451,0x7452,0x7453,0x7454,0x7456,/* 0x80-0x87 */ + 0x7458,0x745D,0x7460,0x7461,0x7462,0x7463,0x7464,0x7465,/* 0x88-0x8F */ + 0x7466,0x7467,0x7468,0x7469,0x746A,0x746B,0x746C,0x746E,/* 0x90-0x97 */ + 0x746F,0x7471,0x7472,0x7473,0x7474,0x7475,0x7478,0x7479,/* 0x98-0x9F */ + 0x747A,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xA0-0xA7 */ +}; + +static const wchar_t c2u_AD[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x747B,0x747C,0x747D,0x747F,0x7482,0x7484,0x7485,0x7486,/* 0x40-0x47 */ + 0x7488,0x7489,0x748A,0x748C,0x748D,0x748F,0x7491,0x7492,/* 0x48-0x4F */ + 0x7493,0x7494,0x7495,0x7496,0x7497,0x7498,0x7499,0x749A,/* 0x50-0x57 */ + 0x749B,0x749D,0x749F,0x74A0,0x74A1,0x74A2,0x74A3,0x74A4,/* 0x58-0x5F */ + 0x74A5,0x74A6,0x74AA,0x74AB,0x74AC,0x74AD,0x74AE,0x74AF,/* 0x60-0x67 */ + 0x74B0,0x74B1,0x74B2,0x74B3,0x74B4,0x74B5,0x74B6,0x74B7,/* 0x68-0x6F */ + 0x74B8,0x74B9,0x74BB,0x74BC,0x74BD,0x74BE,0x74BF,0x74C0,/* 0x70-0x77 */ + 0x74C1,0x74C2,0x74C3,0x74C4,0x74C5,0x74C6,0x74C7,0x0000,/* 0x78-0x7F */ + + 0x74C8,0x74C9,0x74CA,0x74CB,0x74CC,0x74CD,0x74CE,0x74CF,/* 0x80-0x87 */ + 0x74D0,0x74D1,0x74D3,0x74D4,0x74D5,0x74D6,0x74D7,0x74D8,/* 0x88-0x8F */ + 0x74D9,0x74DA,0x74DB,0x74DD,0x74DF,0x74E1,0x74E5,0x74E7,/* 0x90-0x97 */ + 0x74E8,0x74E9,0x74EA,0x74EB,0x74EC,0x74ED,0x74F0,0x74F1,/* 0x98-0x9F */ + 0x74F2,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xA0-0xA7 */ +}; + +static const wchar_t c2u_AE[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x74F3,0x74F5,0x74F8,0x74F9,0x74FA,0x74FB,0x74FC,0x74FD,/* 0x40-0x47 */ + 0x74FE,0x7500,0x7501,0x7502,0x7503,0x7505,0x7506,0x7507,/* 0x48-0x4F */ + 0x7508,0x7509,0x750A,0x750B,0x750C,0x750E,0x7510,0x7512,/* 0x50-0x57 */ + 0x7514,0x7515,0x7516,0x7517,0x751B,0x751D,0x751E,0x7520,/* 0x58-0x5F */ + 0x7521,0x7522,0x7523,0x7524,0x7526,0x7527,0x752A,0x752E,/* 0x60-0x67 */ + 0x7534,0x7536,0x7539,0x753C,0x753D,0x753F,0x7541,0x7542,/* 0x68-0x6F */ + 0x7543,0x7544,0x7546,0x7547,0x7549,0x754A,0x754D,0x7550,/* 0x70-0x77 */ + 0x7551,0x7552,0x7553,0x7555,0x7556,0x7557,0x7558,0x0000,/* 0x78-0x7F */ + + 0x755D,0x755E,0x755F,0x7560,0x7561,0x7562,0x7563,0x7564,/* 0x80-0x87 */ + 0x7567,0x7568,0x7569,0x756B,0x756C,0x756D,0x756E,0x756F,/* 0x88-0x8F */ + 0x7570,0x7571,0x7573,0x7575,0x7576,0x7577,0x757A,0x757B,/* 0x90-0x97 */ + 0x757C,0x757D,0x757E,0x7580,0x7581,0x7582,0x7584,0x7585,/* 0x98-0x9F */ + 0x7587,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xA0-0xA7 */ +}; + +static const wchar_t c2u_AF[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7588,0x7589,0x758A,0x758C,0x758D,0x758E,0x7590,0x7593,/* 0x40-0x47 */ + 0x7595,0x7598,0x759B,0x759C,0x759E,0x75A2,0x75A6,0x75A7,/* 0x48-0x4F */ + 0x75A8,0x75A9,0x75AA,0x75AD,0x75B6,0x75B7,0x75BA,0x75BB,/* 0x50-0x57 */ + 0x75BF,0x75C0,0x75C1,0x75C6,0x75CB,0x75CC,0x75CE,0x75CF,/* 0x58-0x5F */ + 0x75D0,0x75D1,0x75D3,0x75D7,0x75D9,0x75DA,0x75DC,0x75DD,/* 0x60-0x67 */ + 0x75DF,0x75E0,0x75E1,0x75E5,0x75E9,0x75EC,0x75ED,0x75EE,/* 0x68-0x6F */ + 0x75EF,0x75F2,0x75F3,0x75F5,0x75F6,0x75F7,0x75F8,0x75FA,/* 0x70-0x77 */ + 0x75FB,0x75FD,0x75FE,0x7602,0x7604,0x7606,0x7607,0x0000,/* 0x78-0x7F */ + + 0x7608,0x7609,0x760B,0x760D,0x760E,0x760F,0x7611,0x7612,/* 0x80-0x87 */ + 0x7613,0x7614,0x7616,0x761A,0x761C,0x761D,0x761E,0x7621,/* 0x88-0x8F */ + 0x7623,0x7627,0x7628,0x762C,0x762E,0x762F,0x7631,0x7632,/* 0x90-0x97 */ + 0x7636,0x7637,0x7639,0x763A,0x763B,0x763D,0x7641,0x7642,/* 0x98-0x9F */ + 0x7644,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xA0-0xA7 */ +}; + +static const wchar_t c2u_B0[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7645,0x7646,0x7647,0x7648,0x7649,0x764A,0x764B,0x764E,/* 0x40-0x47 */ + 0x764F,0x7650,0x7651,0x7652,0x7653,0x7655,0x7657,0x7658,/* 0x48-0x4F */ + 0x7659,0x765A,0x765B,0x765D,0x765F,0x7660,0x7661,0x7662,/* 0x50-0x57 */ + 0x7664,0x7665,0x7666,0x7667,0x7668,0x7669,0x766A,0x766C,/* 0x58-0x5F */ + 0x766D,0x766E,0x7670,0x7671,0x7672,0x7673,0x7674,0x7675,/* 0x60-0x67 */ + 0x7676,0x7677,0x7679,0x767A,0x767C,0x767F,0x7680,0x7681,/* 0x68-0x6F */ + 0x7683,0x7685,0x7689,0x768A,0x768C,0x768D,0x768F,0x7690,/* 0x70-0x77 */ + 0x7692,0x7694,0x7695,0x7697,0x7698,0x769A,0x769B,0x0000,/* 0x78-0x7F */ + + 0x769C,0x769D,0x769E,0x769F,0x76A0,0x76A1,0x76A2,0x76A3,/* 0x80-0x87 */ + 0x76A5,0x76A6,0x76A7,0x76A8,0x76A9,0x76AA,0x76AB,0x76AC,/* 0x88-0x8F */ + 0x76AD,0x76AF,0x76B0,0x76B3,0x76B5,0x76B6,0x76B7,0x76B8,/* 0x90-0x97 */ + 0x76B9,0x76BA,0x76BB,0x76BC,0x76BD,0x76BE,0x76C0,0x76C1,/* 0x98-0x9F */ + 0x76C3,0x554A,0x963F,0x57C3,0x6328,0x54CE,0x5509,0x54C0,/* 0xA0-0xA7 */ + 0x7691,0x764C,0x853C,0x77EE,0x827E,0x788D,0x7231,0x9698,/* 0xA8-0xAF */ + 0x978D,0x6C28,0x5B89,0x4FFA,0x6309,0x6697,0x5CB8,0x80FA,/* 0xB0-0xB7 */ + 0x6848,0x80AE,0x6602,0x76CE,0x51F9,0x6556,0x71AC,0x7FF1,/* 0xB8-0xBF */ + 0x8884,0x50B2,0x5965,0x61CA,0x6FB3,0x82AD,0x634C,0x6252,/* 0xC0-0xC7 */ + 0x53ED,0x5427,0x7B06,0x516B,0x75A4,0x5DF4,0x62D4,0x8DCB,/* 0xC8-0xCF */ + 0x9776,0x628A,0x8019,0x575D,0x9738,0x7F62,0x7238,0x767D,/* 0xD0-0xD7 */ + 0x67CF,0x767E,0x6446,0x4F70,0x8D25,0x62DC,0x7A17,0x6591,/* 0xD8-0xDF */ + 0x73ED,0x642C,0x6273,0x822C,0x9881,0x677F,0x7248,0x626E,/* 0xE0-0xE7 */ + 0x62CC,0x4F34,0x74E3,0x534A,0x529E,0x7ECA,0x90A6,0x5E2E,/* 0xE8-0xEF */ + 0x6886,0x699C,0x8180,0x7ED1,0x68D2,0x78C5,0x868C,0x9551,/* 0xF0-0xF7 */ + 0x508D,0x8C24,0x82DE,0x80DE,0x5305,0x8912,0x5265,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B1[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x76C4,0x76C7,0x76C9,0x76CB,0x76CC,0x76D3,0x76D5,0x76D9,/* 0x40-0x47 */ + 0x76DA,0x76DC,0x76DD,0x76DE,0x76E0,0x76E1,0x76E2,0x76E3,/* 0x48-0x4F */ + 0x76E4,0x76E6,0x76E7,0x76E8,0x76E9,0x76EA,0x76EB,0x76EC,/* 0x50-0x57 */ + 0x76ED,0x76F0,0x76F3,0x76F5,0x76F6,0x76F7,0x76FA,0x76FB,/* 0x58-0x5F */ + 0x76FD,0x76FF,0x7700,0x7702,0x7703,0x7705,0x7706,0x770A,/* 0x60-0x67 */ + 0x770C,0x770E,0x770F,0x7710,0x7711,0x7712,0x7713,0x7714,/* 0x68-0x6F */ + 0x7715,0x7716,0x7717,0x7718,0x771B,0x771C,0x771D,0x771E,/* 0x70-0x77 */ + 0x7721,0x7723,0x7724,0x7725,0x7727,0x772A,0x772B,0x0000,/* 0x78-0x7F */ + + 0x772C,0x772E,0x7730,0x7731,0x7732,0x7733,0x7734,0x7739,/* 0x80-0x87 */ + 0x773B,0x773D,0x773E,0x773F,0x7742,0x7744,0x7745,0x7746,/* 0x88-0x8F */ + 0x7748,0x7749,0x774A,0x774B,0x774C,0x774D,0x774E,0x774F,/* 0x90-0x97 */ + 0x7752,0x7753,0x7754,0x7755,0x7756,0x7757,0x7758,0x7759,/* 0x98-0x9F */ + 0x775C,0x8584,0x96F9,0x4FDD,0x5821,0x9971,0x5B9D,0x62B1,/* 0xA0-0xA7 */ + 0x62A5,0x66B4,0x8C79,0x9C8D,0x7206,0x676F,0x7891,0x60B2,/* 0xA8-0xAF */ + 0x5351,0x5317,0x8F88,0x80CC,0x8D1D,0x94A1,0x500D,0x72C8,/* 0xB0-0xB7 */ + 0x5907,0x60EB,0x7119,0x88AB,0x5954,0x82EF,0x672C,0x7B28,/* 0xB8-0xBF */ + 0x5D29,0x7EF7,0x752D,0x6CF5,0x8E66,0x8FF8,0x903C,0x9F3B,/* 0xC0-0xC7 */ + 0x6BD4,0x9119,0x7B14,0x5F7C,0x78A7,0x84D6,0x853D,0x6BD5,/* 0xC8-0xCF */ + 0x6BD9,0x6BD6,0x5E01,0x5E87,0x75F9,0x95ED,0x655D,0x5F0A,/* 0xD0-0xD7 */ + 0x5FC5,0x8F9F,0x58C1,0x81C2,0x907F,0x965B,0x97AD,0x8FB9,/* 0xD8-0xDF */ + 0x7F16,0x8D2C,0x6241,0x4FBF,0x53D8,0x535E,0x8FA8,0x8FA9,/* 0xE0-0xE7 */ + 0x8FAB,0x904D,0x6807,0x5F6A,0x8198,0x8868,0x9CD6,0x618B,/* 0xE8-0xEF */ + 0x522B,0x762A,0x5F6C,0x658C,0x6FD2,0x6EE8,0x5BBE,0x6448,/* 0xF0-0xF7 */ + 0x5175,0x51B0,0x67C4,0x4E19,0x79C9,0x997C,0x70B3,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B2[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x775D,0x775E,0x775F,0x7760,0x7764,0x7767,0x7769,0x776A,/* 0x40-0x47 */ + 0x776D,0x776E,0x776F,0x7770,0x7771,0x7772,0x7773,0x7774,/* 0x48-0x4F */ + 0x7775,0x7776,0x7777,0x7778,0x777A,0x777B,0x777C,0x7781,/* 0x50-0x57 */ + 0x7782,0x7783,0x7786,0x7787,0x7788,0x7789,0x778A,0x778B,/* 0x58-0x5F */ + 0x778F,0x7790,0x7793,0x7794,0x7795,0x7796,0x7797,0x7798,/* 0x60-0x67 */ + 0x7799,0x779A,0x779B,0x779C,0x779D,0x779E,0x77A1,0x77A3,/* 0x68-0x6F */ + 0x77A4,0x77A6,0x77A8,0x77AB,0x77AD,0x77AE,0x77AF,0x77B1,/* 0x70-0x77 */ + 0x77B2,0x77B4,0x77B6,0x77B7,0x77B8,0x77B9,0x77BA,0x0000,/* 0x78-0x7F */ + + 0x77BC,0x77BE,0x77C0,0x77C1,0x77C2,0x77C3,0x77C4,0x77C5,/* 0x80-0x87 */ + 0x77C6,0x77C7,0x77C8,0x77C9,0x77CA,0x77CB,0x77CC,0x77CE,/* 0x88-0x8F */ + 0x77CF,0x77D0,0x77D1,0x77D2,0x77D3,0x77D4,0x77D5,0x77D6,/* 0x90-0x97 */ + 0x77D8,0x77D9,0x77DA,0x77DD,0x77DE,0x77DF,0x77E0,0x77E1,/* 0x98-0x9F */ + 0x77E4,0x75C5,0x5E76,0x73BB,0x83E0,0x64AD,0x62E8,0x94B5,/* 0xA0-0xA7 */ + 0x6CE2,0x535A,0x52C3,0x640F,0x94C2,0x7B94,0x4F2F,0x5E1B,/* 0xA8-0xAF */ + 0x8236,0x8116,0x818A,0x6E24,0x6CCA,0x9A73,0x6355,0x535C,/* 0xB0-0xB7 */ + 0x54FA,0x8865,0x57E0,0x4E0D,0x5E03,0x6B65,0x7C3F,0x90E8,/* 0xB8-0xBF */ + 0x6016,0x64E6,0x731C,0x88C1,0x6750,0x624D,0x8D22,0x776C,/* 0xC0-0xC7 */ + 0x8E29,0x91C7,0x5F69,0x83DC,0x8521,0x9910,0x53C2,0x8695,/* 0xC8-0xCF */ + 0x6B8B,0x60ED,0x60E8,0x707F,0x82CD,0x8231,0x4ED3,0x6CA7,/* 0xD0-0xD7 */ + 0x85CF,0x64CD,0x7CD9,0x69FD,0x66F9,0x8349,0x5395,0x7B56,/* 0xD8-0xDF */ + 0x4FA7,0x518C,0x6D4B,0x5C42,0x8E6D,0x63D2,0x53C9,0x832C,/* 0xE0-0xE7 */ + 0x8336,0x67E5,0x78B4,0x643D,0x5BDF,0x5C94,0x5DEE,0x8BE7,/* 0xE8-0xEF */ + 0x62C6,0x67F4,0x8C7A,0x6400,0x63BA,0x8749,0x998B,0x8C17,/* 0xF0-0xF7 */ + 0x7F20,0x94F2,0x4EA7,0x9610,0x98A4,0x660C,0x7316,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B3[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x77E6,0x77E8,0x77EA,0x77EF,0x77F0,0x77F1,0x77F2,0x77F4,/* 0x40-0x47 */ + 0x77F5,0x77F7,0x77F9,0x77FA,0x77FB,0x77FC,0x7803,0x7804,/* 0x48-0x4F */ + 0x7805,0x7806,0x7807,0x7808,0x780A,0x780B,0x780E,0x780F,/* 0x50-0x57 */ + 0x7810,0x7813,0x7815,0x7819,0x781B,0x781E,0x7820,0x7821,/* 0x58-0x5F */ + 0x7822,0x7824,0x7828,0x782A,0x782B,0x782E,0x782F,0x7831,/* 0x60-0x67 */ + 0x7832,0x7833,0x7835,0x7836,0x783D,0x783F,0x7841,0x7842,/* 0x68-0x6F */ + 0x7843,0x7844,0x7846,0x7848,0x7849,0x784A,0x784B,0x784D,/* 0x70-0x77 */ + 0x784F,0x7851,0x7853,0x7854,0x7858,0x7859,0x785A,0x0000,/* 0x78-0x7F */ + + 0x785B,0x785C,0x785E,0x785F,0x7860,0x7861,0x7862,0x7863,/* 0x80-0x87 */ + 0x7864,0x7865,0x7866,0x7867,0x7868,0x7869,0x786F,0x7870,/* 0x88-0x8F */ + 0x7871,0x7872,0x7873,0x7874,0x7875,0x7876,0x7878,0x7879,/* 0x90-0x97 */ + 0x787A,0x787B,0x787D,0x787E,0x787F,0x7880,0x7881,0x7882,/* 0x98-0x9F */ + 0x7883,0x573A,0x5C1D,0x5E38,0x957F,0x507F,0x80A0,0x5382,/* 0xA0-0xA7 */ + 0x655E,0x7545,0x5531,0x5021,0x8D85,0x6284,0x949E,0x671D,/* 0xA8-0xAF */ + 0x5632,0x6F6E,0x5DE2,0x5435,0x7092,0x8F66,0x626F,0x64A4,/* 0xB0-0xB7 */ + 0x63A3,0x5F7B,0x6F88,0x90F4,0x81E3,0x8FB0,0x5C18,0x6668,/* 0xB8-0xBF */ + 0x5FF1,0x6C89,0x9648,0x8D81,0x886C,0x6491,0x79F0,0x57CE,/* 0xC0-0xC7 */ + 0x6A59,0x6210,0x5448,0x4E58,0x7A0B,0x60E9,0x6F84,0x8BDA,/* 0xC8-0xCF */ + 0x627F,0x901E,0x9A8B,0x79E4,0x5403,0x75F4,0x6301,0x5319,/* 0xD0-0xD7 */ + 0x6C60,0x8FDF,0x5F1B,0x9A70,0x803B,0x9F7F,0x4F88,0x5C3A,/* 0xD8-0xDF */ + 0x8D64,0x7FC5,0x65A5,0x70BD,0x5145,0x51B2,0x866B,0x5D07,/* 0xE0-0xE7 */ + 0x5BA0,0x62BD,0x916C,0x7574,0x8E0C,0x7A20,0x6101,0x7B79,/* 0xE8-0xEF */ + 0x4EC7,0x7EF8,0x7785,0x4E11,0x81ED,0x521D,0x51FA,0x6A71,/* 0xF0-0xF7 */ + 0x53A8,0x8E87,0x9504,0x96CF,0x6EC1,0x9664,0x695A,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B4[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7884,0x7885,0x7886,0x7888,0x788A,0x788B,0x788F,0x7890,/* 0x40-0x47 */ + 0x7892,0x7894,0x7895,0x7896,0x7899,0x789D,0x789E,0x78A0,/* 0x48-0x4F */ + 0x78A2,0x78A4,0x78A6,0x78A8,0x78A9,0x78AA,0x78AB,0x78AC,/* 0x50-0x57 */ + 0x78AD,0x78AE,0x78AF,0x78B5,0x78B6,0x78B7,0x78B8,0x78BA,/* 0x58-0x5F */ + 0x78BB,0x78BC,0x78BD,0x78BF,0x78C0,0x78C2,0x78C3,0x78C4,/* 0x60-0x67 */ + 0x78C6,0x78C7,0x78C8,0x78CC,0x78CD,0x78CE,0x78CF,0x78D1,/* 0x68-0x6F */ + 0x78D2,0x78D3,0x78D6,0x78D7,0x78D8,0x78DA,0x78DB,0x78DC,/* 0x70-0x77 */ + 0x78DD,0x78DE,0x78DF,0x78E0,0x78E1,0x78E2,0x78E3,0x0000,/* 0x78-0x7F */ + + 0x78E4,0x78E5,0x78E6,0x78E7,0x78E9,0x78EA,0x78EB,0x78ED,/* 0x80-0x87 */ + 0x78EE,0x78EF,0x78F0,0x78F1,0x78F3,0x78F5,0x78F6,0x78F8,/* 0x88-0x8F */ + 0x78F9,0x78FB,0x78FC,0x78FD,0x78FE,0x78FF,0x7900,0x7902,/* 0x90-0x97 */ + 0x7903,0x7904,0x7906,0x7907,0x7908,0x7909,0x790A,0x790B,/* 0x98-0x9F */ + 0x790C,0x7840,0x50A8,0x77D7,0x6410,0x89E6,0x5904,0x63E3,/* 0xA0-0xA7 */ + 0x5DDD,0x7A7F,0x693D,0x4F20,0x8239,0x5598,0x4E32,0x75AE,/* 0xA8-0xAF */ + 0x7A97,0x5E62,0x5E8A,0x95EF,0x521B,0x5439,0x708A,0x6376,/* 0xB0-0xB7 */ + 0x9524,0x5782,0x6625,0x693F,0x9187,0x5507,0x6DF3,0x7EAF,/* 0xB8-0xBF */ + 0x8822,0x6233,0x7EF0,0x75B5,0x8328,0x78C1,0x96CC,0x8F9E,/* 0xC0-0xC7 */ + 0x6148,0x74F7,0x8BCD,0x6B64,0x523A,0x8D50,0x6B21,0x806A,/* 0xC8-0xCF */ + 0x8471,0x56F1,0x5306,0x4ECE,0x4E1B,0x51D1,0x7C97,0x918B,/* 0xD0-0xD7 */ + 0x7C07,0x4FC3,0x8E7F,0x7BE1,0x7A9C,0x6467,0x5D14,0x50AC,/* 0xD8-0xDF */ + 0x8106,0x7601,0x7CB9,0x6DEC,0x7FE0,0x6751,0x5B58,0x5BF8,/* 0xE0-0xE7 */ + 0x78CB,0x64AE,0x6413,0x63AA,0x632B,0x9519,0x642D,0x8FBE,/* 0xE8-0xEF */ + 0x7B54,0x7629,0x6253,0x5927,0x5446,0x6B79,0x50A3,0x6234,/* 0xF0-0xF7 */ + 0x5E26,0x6B86,0x4EE3,0x8D37,0x888B,0x5F85,0x902E,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B5[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x790D,0x790E,0x790F,0x7910,0x7911,0x7912,0x7914,0x7915,/* 0x40-0x47 */ + 0x7916,0x7917,0x7918,0x7919,0x791A,0x791B,0x791C,0x791D,/* 0x48-0x4F */ + 0x791F,0x7920,0x7921,0x7922,0x7923,0x7925,0x7926,0x7927,/* 0x50-0x57 */ + 0x7928,0x7929,0x792A,0x792B,0x792C,0x792D,0x792E,0x792F,/* 0x58-0x5F */ + 0x7930,0x7931,0x7932,0x7933,0x7935,0x7936,0x7937,0x7938,/* 0x60-0x67 */ + 0x7939,0x793D,0x793F,0x7942,0x7943,0x7944,0x7945,0x7947,/* 0x68-0x6F */ + 0x794A,0x794B,0x794C,0x794D,0x794E,0x794F,0x7950,0x7951,/* 0x70-0x77 */ + 0x7952,0x7954,0x7955,0x7958,0x7959,0x7961,0x7963,0x0000,/* 0x78-0x7F */ + + 0x7964,0x7966,0x7969,0x796A,0x796B,0x796C,0x796E,0x7970,/* 0x80-0x87 */ + 0x7971,0x7972,0x7973,0x7974,0x7975,0x7976,0x7979,0x797B,/* 0x88-0x8F */ + 0x797C,0x797D,0x797E,0x797F,0x7982,0x7983,0x7986,0x7987,/* 0x90-0x97 */ + 0x7988,0x7989,0x798B,0x798C,0x798D,0x798E,0x7990,0x7991,/* 0x98-0x9F */ + 0x7992,0x6020,0x803D,0x62C5,0x4E39,0x5355,0x90F8,0x63B8,/* 0xA0-0xA7 */ + 0x80C6,0x65E6,0x6C2E,0x4F46,0x60EE,0x6DE1,0x8BDE,0x5F39,/* 0xA8-0xAF */ + 0x86CB,0x5F53,0x6321,0x515A,0x8361,0x6863,0x5200,0x6363,/* 0xB0-0xB7 */ + 0x8E48,0x5012,0x5C9B,0x7977,0x5BFC,0x5230,0x7A3B,0x60BC,/* 0xB8-0xBF */ + 0x9053,0x76D7,0x5FB7,0x5F97,0x7684,0x8E6C,0x706F,0x767B,/* 0xC0-0xC7 */ + 0x7B49,0x77AA,0x51F3,0x9093,0x5824,0x4F4E,0x6EF4,0x8FEA,/* 0xC8-0xCF */ + 0x654C,0x7B1B,0x72C4,0x6DA4,0x7FDF,0x5AE1,0x62B5,0x5E95,/* 0xD0-0xD7 */ + 0x5730,0x8482,0x7B2C,0x5E1D,0x5F1F,0x9012,0x7F14,0x98A0,/* 0xD8-0xDF */ + 0x6382,0x6EC7,0x7898,0x70B9,0x5178,0x975B,0x57AB,0x7535,/* 0xE0-0xE7 */ + 0x4F43,0x7538,0x5E97,0x60E6,0x5960,0x6DC0,0x6BBF,0x7889,/* 0xE8-0xEF */ + 0x53FC,0x96D5,0x51CB,0x5201,0x6389,0x540A,0x9493,0x8C03,/* 0xF0-0xF7 */ + 0x8DCC,0x7239,0x789F,0x8776,0x8FED,0x8C0D,0x53E0,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B6[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7993,0x7994,0x7995,0x7996,0x7997,0x7998,0x7999,0x799B,/* 0x40-0x47 */ + 0x799C,0x799D,0x799E,0x799F,0x79A0,0x79A1,0x79A2,0x79A3,/* 0x48-0x4F */ + 0x79A4,0x79A5,0x79A6,0x79A8,0x79A9,0x79AA,0x79AB,0x79AC,/* 0x50-0x57 */ + 0x79AD,0x79AE,0x79AF,0x79B0,0x79B1,0x79B2,0x79B4,0x79B5,/* 0x58-0x5F */ + 0x79B6,0x79B7,0x79B8,0x79BC,0x79BF,0x79C2,0x79C4,0x79C5,/* 0x60-0x67 */ + 0x79C7,0x79C8,0x79CA,0x79CC,0x79CE,0x79CF,0x79D0,0x79D3,/* 0x68-0x6F */ + 0x79D4,0x79D6,0x79D7,0x79D9,0x79DA,0x79DB,0x79DC,0x79DD,/* 0x70-0x77 */ + 0x79DE,0x79E0,0x79E1,0x79E2,0x79E5,0x79E8,0x79EA,0x0000,/* 0x78-0x7F */ + + 0x79EC,0x79EE,0x79F1,0x79F2,0x79F3,0x79F4,0x79F5,0x79F6,/* 0x80-0x87 */ + 0x79F7,0x79F9,0x79FA,0x79FC,0x79FE,0x79FF,0x7A01,0x7A04,/* 0x88-0x8F */ + 0x7A05,0x7A07,0x7A08,0x7A09,0x7A0A,0x7A0C,0x7A0F,0x7A10,/* 0x90-0x97 */ + 0x7A11,0x7A12,0x7A13,0x7A15,0x7A16,0x7A18,0x7A19,0x7A1B,/* 0x98-0x9F */ + 0x7A1C,0x4E01,0x76EF,0x53EE,0x9489,0x9876,0x9F0E,0x952D,/* 0xA0-0xA7 */ + 0x5B9A,0x8BA2,0x4E22,0x4E1C,0x51AC,0x8463,0x61C2,0x52A8,/* 0xA8-0xAF */ + 0x680B,0x4F97,0x606B,0x51BB,0x6D1E,0x515C,0x6296,0x6597,/* 0xB0-0xB7 */ + 0x9661,0x8C46,0x9017,0x75D8,0x90FD,0x7763,0x6BD2,0x728A,/* 0xB8-0xBF */ + 0x72EC,0x8BFB,0x5835,0x7779,0x8D4C,0x675C,0x9540,0x809A,/* 0xC0-0xC7 */ + 0x5EA6,0x6E21,0x5992,0x7AEF,0x77ED,0x953B,0x6BB5,0x65AD,/* 0xC8-0xCF */ + 0x7F0E,0x5806,0x5151,0x961F,0x5BF9,0x58A9,0x5428,0x8E72,/* 0xD0-0xD7 */ + 0x6566,0x987F,0x56E4,0x949D,0x76FE,0x9041,0x6387,0x54C6,/* 0xD8-0xDF */ + 0x591A,0x593A,0x579B,0x8EB2,0x6735,0x8DFA,0x8235,0x5241,/* 0xE0-0xE7 */ + 0x60F0,0x5815,0x86FE,0x5CE8,0x9E45,0x4FC4,0x989D,0x8BB9,/* 0xE8-0xEF */ + 0x5A25,0x6076,0x5384,0x627C,0x904F,0x9102,0x997F,0x6069,/* 0xF0-0xF7 */ + 0x800C,0x513F,0x8033,0x5C14,0x9975,0x6D31,0x4E8C,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B7[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7A1D,0x7A1F,0x7A21,0x7A22,0x7A24,0x7A25,0x7A26,0x7A27,/* 0x40-0x47 */ + 0x7A28,0x7A29,0x7A2A,0x7A2B,0x7A2C,0x7A2D,0x7A2E,0x7A2F,/* 0x48-0x4F */ + 0x7A30,0x7A31,0x7A32,0x7A34,0x7A35,0x7A36,0x7A38,0x7A3A,/* 0x50-0x57 */ + 0x7A3E,0x7A40,0x7A41,0x7A42,0x7A43,0x7A44,0x7A45,0x7A47,/* 0x58-0x5F */ + 0x7A48,0x7A49,0x7A4A,0x7A4B,0x7A4C,0x7A4D,0x7A4E,0x7A4F,/* 0x60-0x67 */ + 0x7A50,0x7A52,0x7A53,0x7A54,0x7A55,0x7A56,0x7A58,0x7A59,/* 0x68-0x6F */ + 0x7A5A,0x7A5B,0x7A5C,0x7A5D,0x7A5E,0x7A5F,0x7A60,0x7A61,/* 0x70-0x77 */ + 0x7A62,0x7A63,0x7A64,0x7A65,0x7A66,0x7A67,0x7A68,0x0000,/* 0x78-0x7F */ + + 0x7A69,0x7A6A,0x7A6B,0x7A6C,0x7A6D,0x7A6E,0x7A6F,0x7A71,/* 0x80-0x87 */ + 0x7A72,0x7A73,0x7A75,0x7A7B,0x7A7C,0x7A7D,0x7A7E,0x7A82,/* 0x88-0x8F */ + 0x7A85,0x7A87,0x7A89,0x7A8A,0x7A8B,0x7A8C,0x7A8E,0x7A8F,/* 0x90-0x97 */ + 0x7A90,0x7A93,0x7A94,0x7A99,0x7A9A,0x7A9B,0x7A9E,0x7AA1,/* 0x98-0x9F */ + 0x7AA2,0x8D30,0x53D1,0x7F5A,0x7B4F,0x4F10,0x4E4F,0x9600,/* 0xA0-0xA7 */ + 0x6CD5,0x73D0,0x85E9,0x5E06,0x756A,0x7FFB,0x6A0A,0x77FE,/* 0xA8-0xAF */ + 0x9492,0x7E41,0x51E1,0x70E6,0x53CD,0x8FD4,0x8303,0x8D29,/* 0xB0-0xB7 */ + 0x72AF,0x996D,0x6CDB,0x574A,0x82B3,0x65B9,0x80AA,0x623F,/* 0xB8-0xBF */ + 0x9632,0x59A8,0x4EFF,0x8BBF,0x7EBA,0x653E,0x83F2,0x975E,/* 0xC0-0xC7 */ + 0x5561,0x98DE,0x80A5,0x532A,0x8BFD,0x5420,0x80BA,0x5E9F,/* 0xC8-0xCF */ + 0x6CB8,0x8D39,0x82AC,0x915A,0x5429,0x6C1B,0x5206,0x7EB7,/* 0xD0-0xD7 */ + 0x575F,0x711A,0x6C7E,0x7C89,0x594B,0x4EFD,0x5FFF,0x6124,/* 0xD8-0xDF */ + 0x7CAA,0x4E30,0x5C01,0x67AB,0x8702,0x5CF0,0x950B,0x98CE,/* 0xE0-0xE7 */ + 0x75AF,0x70FD,0x9022,0x51AF,0x7F1D,0x8BBD,0x5949,0x51E4,/* 0xE8-0xEF */ + 0x4F5B,0x5426,0x592B,0x6577,0x80A4,0x5B75,0x6276,0x62C2,/* 0xF0-0xF7 */ + 0x8F90,0x5E45,0x6C1F,0x7B26,0x4F0F,0x4FD8,0x670D,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B8[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7AA3,0x7AA4,0x7AA7,0x7AA9,0x7AAA,0x7AAB,0x7AAE,0x7AAF,/* 0x40-0x47 */ + 0x7AB0,0x7AB1,0x7AB2,0x7AB4,0x7AB5,0x7AB6,0x7AB7,0x7AB8,/* 0x48-0x4F */ + 0x7AB9,0x7ABA,0x7ABB,0x7ABC,0x7ABD,0x7ABE,0x7AC0,0x7AC1,/* 0x50-0x57 */ + 0x7AC2,0x7AC3,0x7AC4,0x7AC5,0x7AC6,0x7AC7,0x7AC8,0x7AC9,/* 0x58-0x5F */ + 0x7ACA,0x7ACC,0x7ACD,0x7ACE,0x7ACF,0x7AD0,0x7AD1,0x7AD2,/* 0x60-0x67 */ + 0x7AD3,0x7AD4,0x7AD5,0x7AD7,0x7AD8,0x7ADA,0x7ADB,0x7ADC,/* 0x68-0x6F */ + 0x7ADD,0x7AE1,0x7AE2,0x7AE4,0x7AE7,0x7AE8,0x7AE9,0x7AEA,/* 0x70-0x77 */ + 0x7AEB,0x7AEC,0x7AEE,0x7AF0,0x7AF1,0x7AF2,0x7AF3,0x0000,/* 0x78-0x7F */ + + 0x7AF4,0x7AF5,0x7AF6,0x7AF7,0x7AF8,0x7AFB,0x7AFC,0x7AFE,/* 0x80-0x87 */ + 0x7B00,0x7B01,0x7B02,0x7B05,0x7B07,0x7B09,0x7B0C,0x7B0D,/* 0x88-0x8F */ + 0x7B0E,0x7B10,0x7B12,0x7B13,0x7B16,0x7B17,0x7B18,0x7B1A,/* 0x90-0x97 */ + 0x7B1C,0x7B1D,0x7B1F,0x7B21,0x7B22,0x7B23,0x7B27,0x7B29,/* 0x98-0x9F */ + 0x7B2D,0x6D6E,0x6DAA,0x798F,0x88B1,0x5F17,0x752B,0x629A,/* 0xA0-0xA7 */ + 0x8F85,0x4FEF,0x91DC,0x65A7,0x812F,0x8151,0x5E9C,0x8150,/* 0xA8-0xAF */ + 0x8D74,0x526F,0x8986,0x8D4B,0x590D,0x5085,0x4ED8,0x961C,/* 0xB0-0xB7 */ + 0x7236,0x8179,0x8D1F,0x5BCC,0x8BA3,0x9644,0x5987,0x7F1A,/* 0xB8-0xBF */ + 0x5490,0x5676,0x560E,0x8BE5,0x6539,0x6982,0x9499,0x76D6,/* 0xC0-0xC7 */ + 0x6E89,0x5E72,0x7518,0x6746,0x67D1,0x7AFF,0x809D,0x8D76,/* 0xC8-0xCF */ + 0x611F,0x79C6,0x6562,0x8D63,0x5188,0x521A,0x94A2,0x7F38,/* 0xD0-0xD7 */ + 0x809B,0x7EB2,0x5C97,0x6E2F,0x6760,0x7BD9,0x768B,0x9AD8,/* 0xD8-0xDF */ + 0x818F,0x7F94,0x7CD5,0x641E,0x9550,0x7A3F,0x544A,0x54E5,/* 0xE0-0xE7 */ + 0x6B4C,0x6401,0x6208,0x9E3D,0x80F3,0x7599,0x5272,0x9769,/* 0xE8-0xEF */ + 0x845B,0x683C,0x86E4,0x9601,0x9694,0x94EC,0x4E2A,0x5404,/* 0xF0-0xF7 */ + 0x7ED9,0x6839,0x8DDF,0x8015,0x66F4,0x5E9A,0x7FB9,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B9[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7B2F,0x7B30,0x7B32,0x7B34,0x7B35,0x7B36,0x7B37,0x7B39,/* 0x40-0x47 */ + 0x7B3B,0x7B3D,0x7B3F,0x7B40,0x7B41,0x7B42,0x7B43,0x7B44,/* 0x48-0x4F */ + 0x7B46,0x7B48,0x7B4A,0x7B4D,0x7B4E,0x7B53,0x7B55,0x7B57,/* 0x50-0x57 */ + 0x7B59,0x7B5C,0x7B5E,0x7B5F,0x7B61,0x7B63,0x7B64,0x7B65,/* 0x58-0x5F */ + 0x7B66,0x7B67,0x7B68,0x7B69,0x7B6A,0x7B6B,0x7B6C,0x7B6D,/* 0x60-0x67 */ + 0x7B6F,0x7B70,0x7B73,0x7B74,0x7B76,0x7B78,0x7B7A,0x7B7C,/* 0x68-0x6F */ + 0x7B7D,0x7B7F,0x7B81,0x7B82,0x7B83,0x7B84,0x7B86,0x7B87,/* 0x70-0x77 */ + 0x7B88,0x7B89,0x7B8A,0x7B8B,0x7B8C,0x7B8E,0x7B8F,0x0000,/* 0x78-0x7F */ + + 0x7B91,0x7B92,0x7B93,0x7B96,0x7B98,0x7B99,0x7B9A,0x7B9B,/* 0x80-0x87 */ + 0x7B9E,0x7B9F,0x7BA0,0x7BA3,0x7BA4,0x7BA5,0x7BAE,0x7BAF,/* 0x88-0x8F */ + 0x7BB0,0x7BB2,0x7BB3,0x7BB5,0x7BB6,0x7BB7,0x7BB9,0x7BBA,/* 0x90-0x97 */ + 0x7BBB,0x7BBC,0x7BBD,0x7BBE,0x7BBF,0x7BC0,0x7BC2,0x7BC3,/* 0x98-0x9F */ + 0x7BC4,0x57C2,0x803F,0x6897,0x5DE5,0x653B,0x529F,0x606D,/* 0xA0-0xA7 */ + 0x9F9A,0x4F9B,0x8EAC,0x516C,0x5BAB,0x5F13,0x5DE9,0x6C5E,/* 0xA8-0xAF */ + 0x62F1,0x8D21,0x5171,0x94A9,0x52FE,0x6C9F,0x82DF,0x72D7,/* 0xB0-0xB7 */ + 0x57A2,0x6784,0x8D2D,0x591F,0x8F9C,0x83C7,0x5495,0x7B8D,/* 0xB8-0xBF */ + 0x4F30,0x6CBD,0x5B64,0x59D1,0x9F13,0x53E4,0x86CA,0x9AA8,/* 0xC0-0xC7 */ + 0x8C37,0x80A1,0x6545,0x987E,0x56FA,0x96C7,0x522E,0x74DC,/* 0xC8-0xCF */ + 0x5250,0x5BE1,0x6302,0x8902,0x4E56,0x62D0,0x602A,0x68FA,/* 0xD0-0xD7 */ + 0x5173,0x5B98,0x51A0,0x89C2,0x7BA1,0x9986,0x7F50,0x60EF,/* 0xD8-0xDF */ + 0x704C,0x8D2F,0x5149,0x5E7F,0x901B,0x7470,0x89C4,0x572D,/* 0xE0-0xE7 */ + 0x7845,0x5F52,0x9F9F,0x95FA,0x8F68,0x9B3C,0x8BE1,0x7678,/* 0xE8-0xEF */ + 0x6842,0x67DC,0x8DEA,0x8D35,0x523D,0x8F8A,0x6EDA,0x68CD,/* 0xF0-0xF7 */ + 0x9505,0x90ED,0x56FD,0x679C,0x88F9,0x8FC7,0x54C8,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_BA[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7BC5,0x7BC8,0x7BC9,0x7BCA,0x7BCB,0x7BCD,0x7BCE,0x7BCF,/* 0x40-0x47 */ + 0x7BD0,0x7BD2,0x7BD4,0x7BD5,0x7BD6,0x7BD7,0x7BD8,0x7BDB,/* 0x48-0x4F */ + 0x7BDC,0x7BDE,0x7BDF,0x7BE0,0x7BE2,0x7BE3,0x7BE4,0x7BE7,/* 0x50-0x57 */ + 0x7BE8,0x7BE9,0x7BEB,0x7BEC,0x7BED,0x7BEF,0x7BF0,0x7BF2,/* 0x58-0x5F */ + 0x7BF3,0x7BF4,0x7BF5,0x7BF6,0x7BF8,0x7BF9,0x7BFA,0x7BFB,/* 0x60-0x67 */ + 0x7BFD,0x7BFF,0x7C00,0x7C01,0x7C02,0x7C03,0x7C04,0x7C05,/* 0x68-0x6F */ + 0x7C06,0x7C08,0x7C09,0x7C0A,0x7C0D,0x7C0E,0x7C10,0x7C11,/* 0x70-0x77 */ + 0x7C12,0x7C13,0x7C14,0x7C15,0x7C17,0x7C18,0x7C19,0x0000,/* 0x78-0x7F */ + + 0x7C1A,0x7C1B,0x7C1C,0x7C1D,0x7C1E,0x7C20,0x7C21,0x7C22,/* 0x80-0x87 */ + 0x7C23,0x7C24,0x7C25,0x7C28,0x7C29,0x7C2B,0x7C2C,0x7C2D,/* 0x88-0x8F */ + 0x7C2E,0x7C2F,0x7C30,0x7C31,0x7C32,0x7C33,0x7C34,0x7C35,/* 0x90-0x97 */ + 0x7C36,0x7C37,0x7C39,0x7C3A,0x7C3B,0x7C3C,0x7C3D,0x7C3E,/* 0x98-0x9F */ + 0x7C42,0x9AB8,0x5B69,0x6D77,0x6C26,0x4EA5,0x5BB3,0x9A87,/* 0xA0-0xA7 */ + 0x9163,0x61A8,0x90AF,0x97E9,0x542B,0x6DB5,0x5BD2,0x51FD,/* 0xA8-0xAF */ + 0x558A,0x7F55,0x7FF0,0x64BC,0x634D,0x65F1,0x61BE,0x608D,/* 0xB0-0xB7 */ + 0x710A,0x6C57,0x6C49,0x592F,0x676D,0x822A,0x58D5,0x568E,/* 0xB8-0xBF */ + 0x8C6A,0x6BEB,0x90DD,0x597D,0x8017,0x53F7,0x6D69,0x5475,/* 0xC0-0xC7 */ + 0x559D,0x8377,0x83CF,0x6838,0x79BE,0x548C,0x4F55,0x5408,/* 0xC8-0xCF */ + 0x76D2,0x8C89,0x9602,0x6CB3,0x6DB8,0x8D6B,0x8910,0x9E64,/* 0xD0-0xD7 */ + 0x8D3A,0x563F,0x9ED1,0x75D5,0x5F88,0x72E0,0x6068,0x54FC,/* 0xD8-0xDF */ + 0x4EA8,0x6A2A,0x8861,0x6052,0x8F70,0x54C4,0x70D8,0x8679,/* 0xE0-0xE7 */ + 0x9E3F,0x6D2A,0x5B8F,0x5F18,0x7EA2,0x5589,0x4FAF,0x7334,/* 0xE8-0xEF */ + 0x543C,0x539A,0x5019,0x540E,0x547C,0x4E4E,0x5FFD,0x745A,/* 0xF0-0xF7 */ + 0x58F6,0x846B,0x80E1,0x8774,0x72D0,0x7CCA,0x6E56,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_BB[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7C43,0x7C44,0x7C45,0x7C46,0x7C47,0x7C48,0x7C49,0x7C4A,/* 0x40-0x47 */ + 0x7C4B,0x7C4C,0x7C4E,0x7C4F,0x7C50,0x7C51,0x7C52,0x7C53,/* 0x48-0x4F */ + 0x7C54,0x7C55,0x7C56,0x7C57,0x7C58,0x7C59,0x7C5A,0x7C5B,/* 0x50-0x57 */ + 0x7C5C,0x7C5D,0x7C5E,0x7C5F,0x7C60,0x7C61,0x7C62,0x7C63,/* 0x58-0x5F */ + 0x7C64,0x7C65,0x7C66,0x7C67,0x7C68,0x7C69,0x7C6A,0x7C6B,/* 0x60-0x67 */ + 0x7C6C,0x7C6D,0x7C6E,0x7C6F,0x7C70,0x7C71,0x7C72,0x7C75,/* 0x68-0x6F */ + 0x7C76,0x7C77,0x7C78,0x7C79,0x7C7A,0x7C7E,0x7C7F,0x7C80,/* 0x70-0x77 */ + 0x7C81,0x7C82,0x7C83,0x7C84,0x7C85,0x7C86,0x7C87,0x0000,/* 0x78-0x7F */ + + 0x7C88,0x7C8A,0x7C8B,0x7C8C,0x7C8D,0x7C8E,0x7C8F,0x7C90,/* 0x80-0x87 */ + 0x7C93,0x7C94,0x7C96,0x7C99,0x7C9A,0x7C9B,0x7CA0,0x7CA1,/* 0x88-0x8F */ + 0x7CA3,0x7CA6,0x7CA7,0x7CA8,0x7CA9,0x7CAB,0x7CAC,0x7CAD,/* 0x90-0x97 */ + 0x7CAF,0x7CB0,0x7CB4,0x7CB5,0x7CB6,0x7CB7,0x7CB8,0x7CBA,/* 0x98-0x9F */ + 0x7CBB,0x5F27,0x864E,0x552C,0x62A4,0x4E92,0x6CAA,0x6237,/* 0xA0-0xA7 */ + 0x82B1,0x54D7,0x534E,0x733E,0x6ED1,0x753B,0x5212,0x5316,/* 0xA8-0xAF */ + 0x8BDD,0x69D0,0x5F8A,0x6000,0x6DEE,0x574F,0x6B22,0x73AF,/* 0xB0-0xB7 */ + 0x6853,0x8FD8,0x7F13,0x6362,0x60A3,0x5524,0x75EA,0x8C62,/* 0xB8-0xBF */ + 0x7115,0x6DA3,0x5BA6,0x5E7B,0x8352,0x614C,0x9EC4,0x78FA,/* 0xC0-0xC7 */ + 0x8757,0x7C27,0x7687,0x51F0,0x60F6,0x714C,0x6643,0x5E4C,/* 0xC8-0xCF */ + 0x604D,0x8C0E,0x7070,0x6325,0x8F89,0x5FBD,0x6062,0x86D4,/* 0xD0-0xD7 */ + 0x56DE,0x6BC1,0x6094,0x6167,0x5349,0x60E0,0x6666,0x8D3F,/* 0xD8-0xDF */ + 0x79FD,0x4F1A,0x70E9,0x6C47,0x8BB3,0x8BF2,0x7ED8,0x8364,/* 0xE0-0xE7 */ + 0x660F,0x5A5A,0x9B42,0x6D51,0x6DF7,0x8C41,0x6D3B,0x4F19,/* 0xE8-0xEF */ + 0x706B,0x83B7,0x6216,0x60D1,0x970D,0x8D27,0x7978,0x51FB,/* 0xF0-0xF7 */ + 0x573E,0x57FA,0x673A,0x7578,0x7A3D,0x79EF,0x7B95,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_BC[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7CBF,0x7CC0,0x7CC2,0x7CC3,0x7CC4,0x7CC6,0x7CC9,0x7CCB,/* 0x40-0x47 */ + 0x7CCE,0x7CCF,0x7CD0,0x7CD1,0x7CD2,0x7CD3,0x7CD4,0x7CD8,/* 0x48-0x4F */ + 0x7CDA,0x7CDB,0x7CDD,0x7CDE,0x7CE1,0x7CE2,0x7CE3,0x7CE4,/* 0x50-0x57 */ + 0x7CE5,0x7CE6,0x7CE7,0x7CE9,0x7CEA,0x7CEB,0x7CEC,0x7CED,/* 0x58-0x5F */ + 0x7CEE,0x7CF0,0x7CF1,0x7CF2,0x7CF3,0x7CF4,0x7CF5,0x7CF6,/* 0x60-0x67 */ + 0x7CF7,0x7CF9,0x7CFA,0x7CFC,0x7CFD,0x7CFE,0x7CFF,0x7D00,/* 0x68-0x6F */ + 0x7D01,0x7D02,0x7D03,0x7D04,0x7D05,0x7D06,0x7D07,0x7D08,/* 0x70-0x77 */ + 0x7D09,0x7D0B,0x7D0C,0x7D0D,0x7D0E,0x7D0F,0x7D10,0x0000,/* 0x78-0x7F */ + + 0x7D11,0x7D12,0x7D13,0x7D14,0x7D15,0x7D16,0x7D17,0x7D18,/* 0x80-0x87 */ + 0x7D19,0x7D1A,0x7D1B,0x7D1C,0x7D1D,0x7D1E,0x7D1F,0x7D21,/* 0x88-0x8F */ + 0x7D23,0x7D24,0x7D25,0x7D26,0x7D28,0x7D29,0x7D2A,0x7D2C,/* 0x90-0x97 */ + 0x7D2D,0x7D2E,0x7D30,0x7D31,0x7D32,0x7D33,0x7D34,0x7D35,/* 0x98-0x9F */ + 0x7D36,0x808C,0x9965,0x8FF9,0x6FC0,0x8BA5,0x9E21,0x59EC,/* 0xA0-0xA7 */ + 0x7EE9,0x7F09,0x5409,0x6781,0x68D8,0x8F91,0x7C4D,0x96C6,/* 0xA8-0xAF */ + 0x53CA,0x6025,0x75BE,0x6C72,0x5373,0x5AC9,0x7EA7,0x6324,/* 0xB0-0xB7 */ + 0x51E0,0x810A,0x5DF1,0x84DF,0x6280,0x5180,0x5B63,0x4F0E,/* 0xB8-0xBF */ + 0x796D,0x5242,0x60B8,0x6D4E,0x5BC4,0x5BC2,0x8BA1,0x8BB0,/* 0xC0-0xC7 */ + 0x65E2,0x5FCC,0x9645,0x5993,0x7EE7,0x7EAA,0x5609,0x67B7,/* 0xC8-0xCF */ + 0x5939,0x4F73,0x5BB6,0x52A0,0x835A,0x988A,0x8D3E,0x7532,/* 0xD0-0xD7 */ + 0x94BE,0x5047,0x7A3C,0x4EF7,0x67B6,0x9A7E,0x5AC1,0x6B7C,/* 0xD8-0xDF */ + 0x76D1,0x575A,0x5C16,0x7B3A,0x95F4,0x714E,0x517C,0x80A9,/* 0xE0-0xE7 */ + 0x8270,0x5978,0x7F04,0x8327,0x68C0,0x67EC,0x78B1,0x7877,/* 0xE8-0xEF */ + 0x62E3,0x6361,0x7B80,0x4FED,0x526A,0x51CF,0x8350,0x69DB,/* 0xF0-0xF7 */ + 0x9274,0x8DF5,0x8D31,0x89C1,0x952E,0x7BAD,0x4EF6,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_BD[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7D37,0x7D38,0x7D39,0x7D3A,0x7D3B,0x7D3C,0x7D3D,0x7D3E,/* 0x40-0x47 */ + 0x7D3F,0x7D40,0x7D41,0x7D42,0x7D43,0x7D44,0x7D45,0x7D46,/* 0x48-0x4F */ + 0x7D47,0x7D48,0x7D49,0x7D4A,0x7D4B,0x7D4C,0x7D4D,0x7D4E,/* 0x50-0x57 */ + 0x7D4F,0x7D50,0x7D51,0x7D52,0x7D53,0x7D54,0x7D55,0x7D56,/* 0x58-0x5F */ + 0x7D57,0x7D58,0x7D59,0x7D5A,0x7D5B,0x7D5C,0x7D5D,0x7D5E,/* 0x60-0x67 */ + 0x7D5F,0x7D60,0x7D61,0x7D62,0x7D63,0x7D64,0x7D65,0x7D66,/* 0x68-0x6F */ + 0x7D67,0x7D68,0x7D69,0x7D6A,0x7D6B,0x7D6C,0x7D6D,0x7D6F,/* 0x70-0x77 */ + 0x7D70,0x7D71,0x7D72,0x7D73,0x7D74,0x7D75,0x7D76,0x0000,/* 0x78-0x7F */ + + 0x7D78,0x7D79,0x7D7A,0x7D7B,0x7D7C,0x7D7D,0x7D7E,0x7D7F,/* 0x80-0x87 */ + 0x7D80,0x7D81,0x7D82,0x7D83,0x7D84,0x7D85,0x7D86,0x7D87,/* 0x88-0x8F */ + 0x7D88,0x7D89,0x7D8A,0x7D8B,0x7D8C,0x7D8D,0x7D8E,0x7D8F,/* 0x90-0x97 */ + 0x7D90,0x7D91,0x7D92,0x7D93,0x7D94,0x7D95,0x7D96,0x7D97,/* 0x98-0x9F */ + 0x7D98,0x5065,0x8230,0x5251,0x996F,0x6E10,0x6E85,0x6DA7,/* 0xA0-0xA7 */ + 0x5EFA,0x50F5,0x59DC,0x5C06,0x6D46,0x6C5F,0x7586,0x848B,/* 0xA8-0xAF */ + 0x6868,0x5956,0x8BB2,0x5320,0x9171,0x964D,0x8549,0x6912,/* 0xB0-0xB7 */ + 0x7901,0x7126,0x80F6,0x4EA4,0x90CA,0x6D47,0x9A84,0x5A07,/* 0xB8-0xBF */ + 0x56BC,0x6405,0x94F0,0x77EB,0x4FA5,0x811A,0x72E1,0x89D2,/* 0xC0-0xC7 */ + 0x997A,0x7F34,0x7EDE,0x527F,0x6559,0x9175,0x8F7F,0x8F83,/* 0xC8-0xCF */ + 0x53EB,0x7A96,0x63ED,0x63A5,0x7686,0x79F8,0x8857,0x9636,/* 0xD0-0xD7 */ + 0x622A,0x52AB,0x8282,0x6854,0x6770,0x6377,0x776B,0x7AED,/* 0xD8-0xDF */ + 0x6D01,0x7ED3,0x89E3,0x59D0,0x6212,0x85C9,0x82A5,0x754C,/* 0xE0-0xE7 */ + 0x501F,0x4ECB,0x75A5,0x8BEB,0x5C4A,0x5DFE,0x7B4B,0x65A4,/* 0xE8-0xEF */ + 0x91D1,0x4ECA,0x6D25,0x895F,0x7D27,0x9526,0x4EC5,0x8C28,/* 0xF0-0xF7 */ + 0x8FDB,0x9773,0x664B,0x7981,0x8FD1,0x70EC,0x6D78,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_BE[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7D99,0x7D9A,0x7D9B,0x7D9C,0x7D9D,0x7D9E,0x7D9F,0x7DA0,/* 0x40-0x47 */ + 0x7DA1,0x7DA2,0x7DA3,0x7DA4,0x7DA5,0x7DA7,0x7DA8,0x7DA9,/* 0x48-0x4F */ + 0x7DAA,0x7DAB,0x7DAC,0x7DAD,0x7DAF,0x7DB0,0x7DB1,0x7DB2,/* 0x50-0x57 */ + 0x7DB3,0x7DB4,0x7DB5,0x7DB6,0x7DB7,0x7DB8,0x7DB9,0x7DBA,/* 0x58-0x5F */ + 0x7DBB,0x7DBC,0x7DBD,0x7DBE,0x7DBF,0x7DC0,0x7DC1,0x7DC2,/* 0x60-0x67 */ + 0x7DC3,0x7DC4,0x7DC5,0x7DC6,0x7DC7,0x7DC8,0x7DC9,0x7DCA,/* 0x68-0x6F */ + 0x7DCB,0x7DCC,0x7DCD,0x7DCE,0x7DCF,0x7DD0,0x7DD1,0x7DD2,/* 0x70-0x77 */ + 0x7DD3,0x7DD4,0x7DD5,0x7DD6,0x7DD7,0x7DD8,0x7DD9,0x0000,/* 0x78-0x7F */ + + 0x7DDA,0x7DDB,0x7DDC,0x7DDD,0x7DDE,0x7DDF,0x7DE0,0x7DE1,/* 0x80-0x87 */ + 0x7DE2,0x7DE3,0x7DE4,0x7DE5,0x7DE6,0x7DE7,0x7DE8,0x7DE9,/* 0x88-0x8F */ + 0x7DEA,0x7DEB,0x7DEC,0x7DED,0x7DEE,0x7DEF,0x7DF0,0x7DF1,/* 0x90-0x97 */ + 0x7DF2,0x7DF3,0x7DF4,0x7DF5,0x7DF6,0x7DF7,0x7DF8,0x7DF9,/* 0x98-0x9F */ + 0x7DFA,0x5C3D,0x52B2,0x8346,0x5162,0x830E,0x775B,0x6676,/* 0xA0-0xA7 */ + 0x9CB8,0x4EAC,0x60CA,0x7CBE,0x7CB3,0x7ECF,0x4E95,0x8B66,/* 0xA8-0xAF */ + 0x666F,0x9888,0x9759,0x5883,0x656C,0x955C,0x5F84,0x75C9,/* 0xB0-0xB7 */ + 0x9756,0x7ADF,0x7ADE,0x51C0,0x70AF,0x7A98,0x63EA,0x7A76,/* 0xB8-0xBF */ + 0x7EA0,0x7396,0x97ED,0x4E45,0x7078,0x4E5D,0x9152,0x53A9,/* 0xC0-0xC7 */ + 0x6551,0x65E7,0x81FC,0x8205,0x548E,0x5C31,0x759A,0x97A0,/* 0xC8-0xCF */ + 0x62D8,0x72D9,0x75BD,0x5C45,0x9A79,0x83CA,0x5C40,0x5480,/* 0xD0-0xD7 */ + 0x77E9,0x4E3E,0x6CAE,0x805A,0x62D2,0x636E,0x5DE8,0x5177,/* 0xD8-0xDF */ + 0x8DDD,0x8E1E,0x952F,0x4FF1,0x53E5,0x60E7,0x70AC,0x5267,/* 0xE0-0xE7 */ + 0x6350,0x9E43,0x5A1F,0x5026,0x7737,0x5377,0x7EE2,0x6485,/* 0xE8-0xEF */ + 0x652B,0x6289,0x6398,0x5014,0x7235,0x89C9,0x51B3,0x8BC0,/* 0xF0-0xF7 */ + 0x7EDD,0x5747,0x83CC,0x94A7,0x519B,0x541B,0x5CFB,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_BF[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7DFB,0x7DFC,0x7DFD,0x7DFE,0x7DFF,0x7E00,0x7E01,0x7E02,/* 0x40-0x47 */ + 0x7E03,0x7E04,0x7E05,0x7E06,0x7E07,0x7E08,0x7E09,0x7E0A,/* 0x48-0x4F */ + 0x7E0B,0x7E0C,0x7E0D,0x7E0E,0x7E0F,0x7E10,0x7E11,0x7E12,/* 0x50-0x57 */ + 0x7E13,0x7E14,0x7E15,0x7E16,0x7E17,0x7E18,0x7E19,0x7E1A,/* 0x58-0x5F */ + 0x7E1B,0x7E1C,0x7E1D,0x7E1E,0x7E1F,0x7E20,0x7E21,0x7E22,/* 0x60-0x67 */ + 0x7E23,0x7E24,0x7E25,0x7E26,0x7E27,0x7E28,0x7E29,0x7E2A,/* 0x68-0x6F */ + 0x7E2B,0x7E2C,0x7E2D,0x7E2E,0x7E2F,0x7E30,0x7E31,0x7E32,/* 0x70-0x77 */ + 0x7E33,0x7E34,0x7E35,0x7E36,0x7E37,0x7E38,0x7E39,0x0000,/* 0x78-0x7F */ + + 0x7E3A,0x7E3C,0x7E3D,0x7E3E,0x7E3F,0x7E40,0x7E42,0x7E43,/* 0x80-0x87 */ + 0x7E44,0x7E45,0x7E46,0x7E48,0x7E49,0x7E4A,0x7E4B,0x7E4C,/* 0x88-0x8F */ + 0x7E4D,0x7E4E,0x7E4F,0x7E50,0x7E51,0x7E52,0x7E53,0x7E54,/* 0x90-0x97 */ + 0x7E55,0x7E56,0x7E57,0x7E58,0x7E59,0x7E5A,0x7E5B,0x7E5C,/* 0x98-0x9F */ + 0x7E5D,0x4FCA,0x7AE3,0x6D5A,0x90E1,0x9A8F,0x5580,0x5496,/* 0xA0-0xA7 */ + 0x5361,0x54AF,0x5F00,0x63E9,0x6977,0x51EF,0x6168,0x520A,/* 0xA8-0xAF */ + 0x582A,0x52D8,0x574E,0x780D,0x770B,0x5EB7,0x6177,0x7CE0,/* 0xB0-0xB7 */ + 0x625B,0x6297,0x4EA2,0x7095,0x8003,0x62F7,0x70E4,0x9760,/* 0xB8-0xBF */ + 0x5777,0x82DB,0x67EF,0x68F5,0x78D5,0x9897,0x79D1,0x58F3,/* 0xC0-0xC7 */ + 0x54B3,0x53EF,0x6E34,0x514B,0x523B,0x5BA2,0x8BFE,0x80AF,/* 0xC8-0xCF */ + 0x5543,0x57A6,0x6073,0x5751,0x542D,0x7A7A,0x6050,0x5B54,/* 0xD0-0xD7 */ + 0x63A7,0x62A0,0x53E3,0x6263,0x5BC7,0x67AF,0x54ED,0x7A9F,/* 0xD8-0xDF */ + 0x82E6,0x9177,0x5E93,0x88E4,0x5938,0x57AE,0x630E,0x8DE8,/* 0xE0-0xE7 */ + 0x80EF,0x5757,0x7B77,0x4FA9,0x5FEB,0x5BBD,0x6B3E,0x5321,/* 0xE8-0xEF */ + 0x7B50,0x72C2,0x6846,0x77FF,0x7736,0x65F7,0x51B5,0x4E8F,/* 0xF0-0xF7 */ + 0x76D4,0x5CBF,0x7AA5,0x8475,0x594E,0x9B41,0x5080,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_C0[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7E5E,0x7E5F,0x7E60,0x7E61,0x7E62,0x7E63,0x7E64,0x7E65,/* 0x40-0x47 */ + 0x7E66,0x7E67,0x7E68,0x7E69,0x7E6A,0x7E6B,0x7E6C,0x7E6D,/* 0x48-0x4F */ + 0x7E6E,0x7E6F,0x7E70,0x7E71,0x7E72,0x7E73,0x7E74,0x7E75,/* 0x50-0x57 */ + 0x7E76,0x7E77,0x7E78,0x7E79,0x7E7A,0x7E7B,0x7E7C,0x7E7D,/* 0x58-0x5F */ + 0x7E7E,0x7E7F,0x7E80,0x7E81,0x7E83,0x7E84,0x7E85,0x7E86,/* 0x60-0x67 */ + 0x7E87,0x7E88,0x7E89,0x7E8A,0x7E8B,0x7E8C,0x7E8D,0x7E8E,/* 0x68-0x6F */ + 0x7E8F,0x7E90,0x7E91,0x7E92,0x7E93,0x7E94,0x7E95,0x7E96,/* 0x70-0x77 */ + 0x7E97,0x7E98,0x7E99,0x7E9A,0x7E9C,0x7E9D,0x7E9E,0x0000,/* 0x78-0x7F */ + + 0x7EAE,0x7EB4,0x7EBB,0x7EBC,0x7ED6,0x7EE4,0x7EEC,0x7EF9,/* 0x80-0x87 */ + 0x7F0A,0x7F10,0x7F1E,0x7F37,0x7F39,0x7F3B,0x7F3C,0x7F3D,/* 0x88-0x8F */ + 0x7F3E,0x7F3F,0x7F40,0x7F41,0x7F43,0x7F46,0x7F47,0x7F48,/* 0x90-0x97 */ + 0x7F49,0x7F4A,0x7F4B,0x7F4C,0x7F4D,0x7F4E,0x7F4F,0x7F52,/* 0x98-0x9F */ + 0x7F53,0x9988,0x6127,0x6E83,0x5764,0x6606,0x6346,0x56F0,/* 0xA0-0xA7 */ + 0x62EC,0x6269,0x5ED3,0x9614,0x5783,0x62C9,0x5587,0x8721,/* 0xA8-0xAF */ + 0x814A,0x8FA3,0x5566,0x83B1,0x6765,0x8D56,0x84DD,0x5A6A,/* 0xB0-0xB7 */ + 0x680F,0x62E6,0x7BEE,0x9611,0x5170,0x6F9C,0x8C30,0x63FD,/* 0xB8-0xBF */ + 0x89C8,0x61D2,0x7F06,0x70C2,0x6EE5,0x7405,0x6994,0x72FC,/* 0xC0-0xC7 */ + 0x5ECA,0x90CE,0x6717,0x6D6A,0x635E,0x52B3,0x7262,0x8001,/* 0xC8-0xCF */ + 0x4F6C,0x59E5,0x916A,0x70D9,0x6D9D,0x52D2,0x4E50,0x96F7,/* 0xD0-0xD7 */ + 0x956D,0x857E,0x78CA,0x7D2F,0x5121,0x5792,0x64C2,0x808B,/* 0xD8-0xDF */ + 0x7C7B,0x6CEA,0x68F1,0x695E,0x51B7,0x5398,0x68A8,0x7281,/* 0xE0-0xE7 */ + 0x9ECE,0x7BF1,0x72F8,0x79BB,0x6F13,0x7406,0x674E,0x91CC,/* 0xE8-0xEF */ + 0x9CA4,0x793C,0x8389,0x8354,0x540F,0x6817,0x4E3D,0x5389,/* 0xF0-0xF7 */ + 0x52B1,0x783E,0x5386,0x5229,0x5088,0x4F8B,0x4FD0,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_C1[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7F56,0x7F59,0x7F5B,0x7F5C,0x7F5D,0x7F5E,0x7F60,0x7F63,/* 0x40-0x47 */ + 0x7F64,0x7F65,0x7F66,0x7F67,0x7F6B,0x7F6C,0x7F6D,0x7F6F,/* 0x48-0x4F */ + 0x7F70,0x7F73,0x7F75,0x7F76,0x7F77,0x7F78,0x7F7A,0x7F7B,/* 0x50-0x57 */ + 0x7F7C,0x7F7D,0x7F7F,0x7F80,0x7F82,0x7F83,0x7F84,0x7F85,/* 0x58-0x5F */ + 0x7F86,0x7F87,0x7F88,0x7F89,0x7F8B,0x7F8D,0x7F8F,0x7F90,/* 0x60-0x67 */ + 0x7F91,0x7F92,0x7F93,0x7F95,0x7F96,0x7F97,0x7F98,0x7F99,/* 0x68-0x6F */ + 0x7F9B,0x7F9C,0x7FA0,0x7FA2,0x7FA3,0x7FA5,0x7FA6,0x7FA8,/* 0x70-0x77 */ + 0x7FA9,0x7FAA,0x7FAB,0x7FAC,0x7FAD,0x7FAE,0x7FB1,0x0000,/* 0x78-0x7F */ + + 0x7FB3,0x7FB4,0x7FB5,0x7FB6,0x7FB7,0x7FBA,0x7FBB,0x7FBE,/* 0x80-0x87 */ + 0x7FC0,0x7FC2,0x7FC3,0x7FC4,0x7FC6,0x7FC7,0x7FC8,0x7FC9,/* 0x88-0x8F */ + 0x7FCB,0x7FCD,0x7FCF,0x7FD0,0x7FD1,0x7FD2,0x7FD3,0x7FD6,/* 0x90-0x97 */ + 0x7FD7,0x7FD9,0x7FDA,0x7FDB,0x7FDC,0x7FDD,0x7FDE,0x7FE2,/* 0x98-0x9F */ + 0x7FE3,0x75E2,0x7ACB,0x7C92,0x6CA5,0x96B6,0x529B,0x7483,/* 0xA0-0xA7 */ + 0x54E9,0x4FE9,0x8054,0x83B2,0x8FDE,0x9570,0x5EC9,0x601C,/* 0xA8-0xAF */ + 0x6D9F,0x5E18,0x655B,0x8138,0x94FE,0x604B,0x70BC,0x7EC3,/* 0xB0-0xB7 */ + 0x7CAE,0x51C9,0x6881,0x7CB1,0x826F,0x4E24,0x8F86,0x91CF,/* 0xB8-0xBF */ + 0x667E,0x4EAE,0x8C05,0x64A9,0x804A,0x50DA,0x7597,0x71CE,/* 0xC0-0xC7 */ + 0x5BE5,0x8FBD,0x6F66,0x4E86,0x6482,0x9563,0x5ED6,0x6599,/* 0xC8-0xCF */ + 0x5217,0x88C2,0x70C8,0x52A3,0x730E,0x7433,0x6797,0x78F7,/* 0xD0-0xD7 */ + 0x9716,0x4E34,0x90BB,0x9CDE,0x6DCB,0x51DB,0x8D41,0x541D,/* 0xD8-0xDF */ + 0x62CE,0x73B2,0x83F1,0x96F6,0x9F84,0x94C3,0x4F36,0x7F9A,/* 0xE0-0xE7 */ + 0x51CC,0x7075,0x9675,0x5CAD,0x9886,0x53E6,0x4EE4,0x6E9C,/* 0xE8-0xEF */ + 0x7409,0x69B4,0x786B,0x998F,0x7559,0x5218,0x7624,0x6D41,/* 0xF0-0xF7 */ + 0x67F3,0x516D,0x9F99,0x804B,0x5499,0x7B3C,0x7ABF,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_C2[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7FE4,0x7FE7,0x7FE8,0x7FEA,0x7FEB,0x7FEC,0x7FED,0x7FEF,/* 0x40-0x47 */ + 0x7FF2,0x7FF4,0x7FF5,0x7FF6,0x7FF7,0x7FF8,0x7FF9,0x7FFA,/* 0x48-0x4F */ + 0x7FFD,0x7FFE,0x7FFF,0x8002,0x8007,0x8008,0x8009,0x800A,/* 0x50-0x57 */ + 0x800E,0x800F,0x8011,0x8013,0x801A,0x801B,0x801D,0x801E,/* 0x58-0x5F */ + 0x801F,0x8021,0x8023,0x8024,0x802B,0x802C,0x802D,0x802E,/* 0x60-0x67 */ + 0x802F,0x8030,0x8032,0x8034,0x8039,0x803A,0x803C,0x803E,/* 0x68-0x6F */ + 0x8040,0x8041,0x8044,0x8045,0x8047,0x8048,0x8049,0x804E,/* 0x70-0x77 */ + 0x804F,0x8050,0x8051,0x8053,0x8055,0x8056,0x8057,0x0000,/* 0x78-0x7F */ + + 0x8059,0x805B,0x805C,0x805D,0x805E,0x805F,0x8060,0x8061,/* 0x80-0x87 */ + 0x8062,0x8063,0x8064,0x8065,0x8066,0x8067,0x8068,0x806B,/* 0x88-0x8F */ + 0x806C,0x806D,0x806E,0x806F,0x8070,0x8072,0x8073,0x8074,/* 0x90-0x97 */ + 0x8075,0x8076,0x8077,0x8078,0x8079,0x807A,0x807B,0x807C,/* 0x98-0x9F */ + 0x807D,0x9686,0x5784,0x62E2,0x9647,0x697C,0x5A04,0x6402,/* 0xA0-0xA7 */ + 0x7BD3,0x6F0F,0x964B,0x82A6,0x5362,0x9885,0x5E90,0x7089,/* 0xA8-0xAF */ + 0x63B3,0x5364,0x864F,0x9C81,0x9E93,0x788C,0x9732,0x8DEF,/* 0xB0-0xB7 */ + 0x8D42,0x9E7F,0x6F5E,0x7984,0x5F55,0x9646,0x622E,0x9A74,/* 0xB8-0xBF */ + 0x5415,0x94DD,0x4FA3,0x65C5,0x5C65,0x5C61,0x7F15,0x8651,/* 0xC0-0xC7 */ + 0x6C2F,0x5F8B,0x7387,0x6EE4,0x7EFF,0x5CE6,0x631B,0x5B6A,/* 0xC8-0xCF */ + 0x6EE6,0x5375,0x4E71,0x63A0,0x7565,0x62A1,0x8F6E,0x4F26,/* 0xD0-0xD7 */ + 0x4ED1,0x6CA6,0x7EB6,0x8BBA,0x841D,0x87BA,0x7F57,0x903B,/* 0xD8-0xDF */ + 0x9523,0x7BA9,0x9AA1,0x88F8,0x843D,0x6D1B,0x9A86,0x7EDC,/* 0xE0-0xE7 */ + 0x5988,0x9EBB,0x739B,0x7801,0x8682,0x9A6C,0x9A82,0x561B,/* 0xE8-0xEF */ + 0x5417,0x57CB,0x4E70,0x9EA6,0x5356,0x8FC8,0x8109,0x7792,/* 0xF0-0xF7 */ + 0x9992,0x86EE,0x6EE1,0x8513,0x66FC,0x6162,0x6F2B,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_C3[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x807E,0x8081,0x8082,0x8085,0x8088,0x808A,0x808D,0x808E,/* 0x40-0x47 */ + 0x808F,0x8090,0x8091,0x8092,0x8094,0x8095,0x8097,0x8099,/* 0x48-0x4F */ + 0x809E,0x80A3,0x80A6,0x80A7,0x80A8,0x80AC,0x80B0,0x80B3,/* 0x50-0x57 */ + 0x80B5,0x80B6,0x80B8,0x80B9,0x80BB,0x80C5,0x80C7,0x80C8,/* 0x58-0x5F */ + 0x80C9,0x80CA,0x80CB,0x80CF,0x80D0,0x80D1,0x80D2,0x80D3,/* 0x60-0x67 */ + 0x80D4,0x80D5,0x80D8,0x80DF,0x80E0,0x80E2,0x80E3,0x80E6,/* 0x68-0x6F */ + 0x80EE,0x80F5,0x80F7,0x80F9,0x80FB,0x80FE,0x80FF,0x8100,/* 0x70-0x77 */ + 0x8101,0x8103,0x8104,0x8105,0x8107,0x8108,0x810B,0x0000,/* 0x78-0x7F */ + + 0x810C,0x8115,0x8117,0x8119,0x811B,0x811C,0x811D,0x811F,/* 0x80-0x87 */ + 0x8120,0x8121,0x8122,0x8123,0x8124,0x8125,0x8126,0x8127,/* 0x88-0x8F */ + 0x8128,0x8129,0x812A,0x812B,0x812D,0x812E,0x8130,0x8133,/* 0x90-0x97 */ + 0x8134,0x8135,0x8137,0x8139,0x813A,0x813B,0x813C,0x813D,/* 0x98-0x9F */ + 0x813F,0x8C29,0x8292,0x832B,0x76F2,0x6C13,0x5FD9,0x83BD,/* 0xA0-0xA7 */ + 0x732B,0x8305,0x951A,0x6BDB,0x77DB,0x94C6,0x536F,0x8302,/* 0xA8-0xAF */ + 0x5192,0x5E3D,0x8C8C,0x8D38,0x4E48,0x73AB,0x679A,0x6885,/* 0xB0-0xB7 */ + 0x9176,0x9709,0x7164,0x6CA1,0x7709,0x5A92,0x9541,0x6BCF,/* 0xB8-0xBF */ + 0x7F8E,0x6627,0x5BD0,0x59B9,0x5A9A,0x95E8,0x95F7,0x4EEC,/* 0xC0-0xC7 */ + 0x840C,0x8499,0x6AAC,0x76DF,0x9530,0x731B,0x68A6,0x5B5F,/* 0xC8-0xCF */ + 0x772F,0x919A,0x9761,0x7CDC,0x8FF7,0x8C1C,0x5F25,0x7C73,/* 0xD0-0xD7 */ + 0x79D8,0x89C5,0x6CCC,0x871C,0x5BC6,0x5E42,0x68C9,0x7720,/* 0xD8-0xDF */ + 0x7EF5,0x5195,0x514D,0x52C9,0x5A29,0x7F05,0x9762,0x82D7,/* 0xE0-0xE7 */ + 0x63CF,0x7784,0x85D0,0x79D2,0x6E3A,0x5E99,0x5999,0x8511,/* 0xE8-0xEF */ + 0x706D,0x6C11,0x62BF,0x76BF,0x654F,0x60AF,0x95FD,0x660E,/* 0xF0-0xF7 */ + 0x879F,0x9E23,0x94ED,0x540D,0x547D,0x8C2C,0x6478,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_C4[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8140,0x8141,0x8142,0x8143,0x8144,0x8145,0x8147,0x8149,/* 0x40-0x47 */ + 0x814D,0x814E,0x814F,0x8152,0x8156,0x8157,0x8158,0x815B,/* 0x48-0x4F */ + 0x815C,0x815D,0x815E,0x815F,0x8161,0x8162,0x8163,0x8164,/* 0x50-0x57 */ + 0x8166,0x8168,0x816A,0x816B,0x816C,0x816F,0x8172,0x8173,/* 0x58-0x5F */ + 0x8175,0x8176,0x8177,0x8178,0x8181,0x8183,0x8184,0x8185,/* 0x60-0x67 */ + 0x8186,0x8187,0x8189,0x818B,0x818C,0x818D,0x818E,0x8190,/* 0x68-0x6F */ + 0x8192,0x8193,0x8194,0x8195,0x8196,0x8197,0x8199,0x819A,/* 0x70-0x77 */ + 0x819E,0x819F,0x81A0,0x81A1,0x81A2,0x81A4,0x81A5,0x0000,/* 0x78-0x7F */ + + 0x81A7,0x81A9,0x81AB,0x81AC,0x81AD,0x81AE,0x81AF,0x81B0,/* 0x80-0x87 */ + 0x81B1,0x81B2,0x81B4,0x81B5,0x81B6,0x81B7,0x81B8,0x81B9,/* 0x88-0x8F */ + 0x81BC,0x81BD,0x81BE,0x81BF,0x81C4,0x81C5,0x81C7,0x81C8,/* 0x90-0x97 */ + 0x81C9,0x81CB,0x81CD,0x81CE,0x81CF,0x81D0,0x81D1,0x81D2,/* 0x98-0x9F */ + 0x81D3,0x6479,0x8611,0x6A21,0x819C,0x78E8,0x6469,0x9B54,/* 0xA0-0xA7 */ + 0x62B9,0x672B,0x83AB,0x58A8,0x9ED8,0x6CAB,0x6F20,0x5BDE,/* 0xA8-0xAF */ + 0x964C,0x8C0B,0x725F,0x67D0,0x62C7,0x7261,0x4EA9,0x59C6,/* 0xB0-0xB7 */ + 0x6BCD,0x5893,0x66AE,0x5E55,0x52DF,0x6155,0x6728,0x76EE,/* 0xB8-0xBF */ + 0x7766,0x7267,0x7A46,0x62FF,0x54EA,0x5450,0x94A0,0x90A3,/* 0xC0-0xC7 */ + 0x5A1C,0x7EB3,0x6C16,0x4E43,0x5976,0x8010,0x5948,0x5357,/* 0xC8-0xCF */ + 0x7537,0x96BE,0x56CA,0x6320,0x8111,0x607C,0x95F9,0x6DD6,/* 0xD0-0xD7 */ + 0x5462,0x9981,0x5185,0x5AE9,0x80FD,0x59AE,0x9713,0x502A,/* 0xD8-0xDF */ + 0x6CE5,0x5C3C,0x62DF,0x4F60,0x533F,0x817B,0x9006,0x6EBA,/* 0xE0-0xE7 */ + 0x852B,0x62C8,0x5E74,0x78BE,0x64B5,0x637B,0x5FF5,0x5A18,/* 0xE8-0xEF */ + 0x917F,0x9E1F,0x5C3F,0x634F,0x8042,0x5B7D,0x556E,0x954A,/* 0xF0-0xF7 */ + 0x954D,0x6D85,0x60A8,0x67E0,0x72DE,0x51DD,0x5B81,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_C5[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x81D4,0x81D5,0x81D6,0x81D7,0x81D8,0x81D9,0x81DA,0x81DB,/* 0x40-0x47 */ + 0x81DC,0x81DD,0x81DE,0x81DF,0x81E0,0x81E1,0x81E2,0x81E4,/* 0x48-0x4F */ + 0x81E5,0x81E6,0x81E8,0x81E9,0x81EB,0x81EE,0x81EF,0x81F0,/* 0x50-0x57 */ + 0x81F1,0x81F2,0x81F5,0x81F6,0x81F7,0x81F8,0x81F9,0x81FA,/* 0x58-0x5F */ + 0x81FD,0x81FF,0x8203,0x8207,0x8208,0x8209,0x820A,0x820B,/* 0x60-0x67 */ + 0x820E,0x820F,0x8211,0x8213,0x8215,0x8216,0x8217,0x8218,/* 0x68-0x6F */ + 0x8219,0x821A,0x821D,0x8220,0x8224,0x8225,0x8226,0x8227,/* 0x70-0x77 */ + 0x8229,0x822E,0x8232,0x823A,0x823C,0x823D,0x823F,0x0000,/* 0x78-0x7F */ + + 0x8240,0x8241,0x8242,0x8243,0x8245,0x8246,0x8248,0x824A,/* 0x80-0x87 */ + 0x824C,0x824D,0x824E,0x8250,0x8251,0x8252,0x8253,0x8254,/* 0x88-0x8F */ + 0x8255,0x8256,0x8257,0x8259,0x825B,0x825C,0x825D,0x825E,/* 0x90-0x97 */ + 0x8260,0x8261,0x8262,0x8263,0x8264,0x8265,0x8266,0x8267,/* 0x98-0x9F */ + 0x8269,0x62E7,0x6CDE,0x725B,0x626D,0x94AE,0x7EBD,0x8113,/* 0xA0-0xA7 */ + 0x6D53,0x519C,0x5F04,0x5974,0x52AA,0x6012,0x5973,0x6696,/* 0xA8-0xAF */ + 0x8650,0x759F,0x632A,0x61E6,0x7CEF,0x8BFA,0x54E6,0x6B27,/* 0xB0-0xB7 */ + 0x9E25,0x6BB4,0x85D5,0x5455,0x5076,0x6CA4,0x556A,0x8DB4,/* 0xB8-0xBF */ + 0x722C,0x5E15,0x6015,0x7436,0x62CD,0x6392,0x724C,0x5F98,/* 0xC0-0xC7 */ + 0x6E43,0x6D3E,0x6500,0x6F58,0x76D8,0x78D0,0x76FC,0x7554,/* 0xC8-0xCF */ + 0x5224,0x53DB,0x4E53,0x5E9E,0x65C1,0x802A,0x80D6,0x629B,/* 0xD0-0xD7 */ + 0x5486,0x5228,0x70AE,0x888D,0x8DD1,0x6CE1,0x5478,0x80DA,/* 0xD8-0xDF */ + 0x57F9,0x88F4,0x8D54,0x966A,0x914D,0x4F69,0x6C9B,0x55B7,/* 0xE0-0xE7 */ + 0x76C6,0x7830,0x62A8,0x70F9,0x6F8E,0x5F6D,0x84EC,0x68DA,/* 0xE8-0xEF */ + 0x787C,0x7BF7,0x81A8,0x670B,0x9E4F,0x6367,0x78B0,0x576F,/* 0xF0-0xF7 */ + 0x7812,0x9739,0x6279,0x62AB,0x5288,0x7435,0x6BD7,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_C6[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x826A,0x826B,0x826C,0x826D,0x8271,0x8275,0x8276,0x8277,/* 0x40-0x47 */ + 0x8278,0x827B,0x827C,0x8280,0x8281,0x8283,0x8285,0x8286,/* 0x48-0x4F */ + 0x8287,0x8289,0x828C,0x8290,0x8293,0x8294,0x8295,0x8296,/* 0x50-0x57 */ + 0x829A,0x829B,0x829E,0x82A0,0x82A2,0x82A3,0x82A7,0x82B2,/* 0x58-0x5F */ + 0x82B5,0x82B6,0x82BA,0x82BB,0x82BC,0x82BF,0x82C0,0x82C2,/* 0x60-0x67 */ + 0x82C3,0x82C5,0x82C6,0x82C9,0x82D0,0x82D6,0x82D9,0x82DA,/* 0x68-0x6F */ + 0x82DD,0x82E2,0x82E7,0x82E8,0x82E9,0x82EA,0x82EC,0x82ED,/* 0x70-0x77 */ + 0x82EE,0x82F0,0x82F2,0x82F3,0x82F5,0x82F6,0x82F8,0x0000,/* 0x78-0x7F */ + + 0x82FA,0x82FC,0x82FD,0x82FE,0x82FF,0x8300,0x830A,0x830B,/* 0x80-0x87 */ + 0x830D,0x8310,0x8312,0x8313,0x8316,0x8318,0x8319,0x831D,/* 0x88-0x8F */ + 0x831E,0x831F,0x8320,0x8321,0x8322,0x8323,0x8324,0x8325,/* 0x90-0x97 */ + 0x8326,0x8329,0x832A,0x832E,0x8330,0x8332,0x8337,0x833B,/* 0x98-0x9F */ + 0x833D,0x5564,0x813E,0x75B2,0x76AE,0x5339,0x75DE,0x50FB,/* 0xA0-0xA7 */ + 0x5C41,0x8B6C,0x7BC7,0x504F,0x7247,0x9A97,0x98D8,0x6F02,/* 0xA8-0xAF */ + 0x74E2,0x7968,0x6487,0x77A5,0x62FC,0x9891,0x8D2B,0x54C1,/* 0xB0-0xB7 */ + 0x8058,0x4E52,0x576A,0x82F9,0x840D,0x5E73,0x51ED,0x74F6,/* 0xB8-0xBF */ + 0x8BC4,0x5C4F,0x5761,0x6CFC,0x9887,0x5A46,0x7834,0x9B44,/* 0xC0-0xC7 */ + 0x8FEB,0x7C95,0x5256,0x6251,0x94FA,0x4EC6,0x8386,0x8461,/* 0xC8-0xCF */ + 0x83E9,0x84B2,0x57D4,0x6734,0x5703,0x666E,0x6D66,0x8C31,/* 0xD0-0xD7 */ + 0x66DD,0x7011,0x671F,0x6B3A,0x6816,0x621A,0x59BB,0x4E03,/* 0xD8-0xDF */ + 0x51C4,0x6F06,0x67D2,0x6C8F,0x5176,0x68CB,0x5947,0x6B67,/* 0xE0-0xE7 */ + 0x7566,0x5D0E,0x8110,0x9F50,0x65D7,0x7948,0x7941,0x9A91,/* 0xE8-0xEF */ + 0x8D77,0x5C82,0x4E5E,0x4F01,0x542F,0x5951,0x780C,0x5668,/* 0xF0-0xF7 */ + 0x6C14,0x8FC4,0x5F03,0x6C7D,0x6CE3,0x8BAB,0x6390,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_C7[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x833E,0x833F,0x8341,0x8342,0x8344,0x8345,0x8348,0x834A,/* 0x40-0x47 */ + 0x834B,0x834C,0x834D,0x834E,0x8353,0x8355,0x8356,0x8357,/* 0x48-0x4F */ + 0x8358,0x8359,0x835D,0x8362,0x8370,0x8371,0x8372,0x8373,/* 0x50-0x57 */ + 0x8374,0x8375,0x8376,0x8379,0x837A,0x837E,0x837F,0x8380,/* 0x58-0x5F */ + 0x8381,0x8382,0x8383,0x8384,0x8387,0x8388,0x838A,0x838B,/* 0x60-0x67 */ + 0x838C,0x838D,0x838F,0x8390,0x8391,0x8394,0x8395,0x8396,/* 0x68-0x6F */ + 0x8397,0x8399,0x839A,0x839D,0x839F,0x83A1,0x83A2,0x83A3,/* 0x70-0x77 */ + 0x83A4,0x83A5,0x83A6,0x83A7,0x83AC,0x83AD,0x83AE,0x0000,/* 0x78-0x7F */ + + 0x83AF,0x83B5,0x83BB,0x83BE,0x83BF,0x83C2,0x83C3,0x83C4,/* 0x80-0x87 */ + 0x83C6,0x83C8,0x83C9,0x83CB,0x83CD,0x83CE,0x83D0,0x83D1,/* 0x88-0x8F */ + 0x83D2,0x83D3,0x83D5,0x83D7,0x83D9,0x83DA,0x83DB,0x83DE,/* 0x90-0x97 */ + 0x83E2,0x83E3,0x83E4,0x83E6,0x83E7,0x83E8,0x83EB,0x83EC,/* 0x98-0x9F */ + 0x83ED,0x6070,0x6D3D,0x7275,0x6266,0x948E,0x94C5,0x5343,/* 0xA0-0xA7 */ + 0x8FC1,0x7B7E,0x4EDF,0x8C26,0x4E7E,0x9ED4,0x94B1,0x94B3,/* 0xA8-0xAF */ + 0x524D,0x6F5C,0x9063,0x6D45,0x8C34,0x5811,0x5D4C,0x6B20,/* 0xB0-0xB7 */ + 0x6B49,0x67AA,0x545B,0x8154,0x7F8C,0x5899,0x8537,0x5F3A,/* 0xB8-0xBF */ + 0x62A2,0x6A47,0x9539,0x6572,0x6084,0x6865,0x77A7,0x4E54,/* 0xC0-0xC7 */ + 0x4FA8,0x5DE7,0x9798,0x64AC,0x7FD8,0x5CED,0x4FCF,0x7A8D,/* 0xC8-0xCF */ + 0x5207,0x8304,0x4E14,0x602F,0x7A83,0x94A6,0x4FB5,0x4EB2,/* 0xD0-0xD7 */ + 0x79E6,0x7434,0x52E4,0x82B9,0x64D2,0x79BD,0x5BDD,0x6C81,/* 0xD8-0xDF */ + 0x9752,0x8F7B,0x6C22,0x503E,0x537F,0x6E05,0x64CE,0x6674,/* 0xE0-0xE7 */ + 0x6C30,0x60C5,0x9877,0x8BF7,0x5E86,0x743C,0x7A77,0x79CB,/* 0xE8-0xEF */ + 0x4E18,0x90B1,0x7403,0x6C42,0x56DA,0x914B,0x6CC5,0x8D8B,/* 0xF0-0xF7 */ + 0x533A,0x86C6,0x66F2,0x8EAF,0x5C48,0x9A71,0x6E20,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_C8[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x83EE,0x83EF,0x83F3,0x83F4,0x83F5,0x83F6,0x83F7,0x83FA,/* 0x40-0x47 */ + 0x83FB,0x83FC,0x83FE,0x83FF,0x8400,0x8402,0x8405,0x8407,/* 0x48-0x4F */ + 0x8408,0x8409,0x840A,0x8410,0x8412,0x8413,0x8414,0x8415,/* 0x50-0x57 */ + 0x8416,0x8417,0x8419,0x841A,0x841B,0x841E,0x841F,0x8420,/* 0x58-0x5F */ + 0x8421,0x8422,0x8423,0x8429,0x842A,0x842B,0x842C,0x842D,/* 0x60-0x67 */ + 0x842E,0x842F,0x8430,0x8432,0x8433,0x8434,0x8435,0x8436,/* 0x68-0x6F */ + 0x8437,0x8439,0x843A,0x843B,0x843E,0x843F,0x8440,0x8441,/* 0x70-0x77 */ + 0x8442,0x8443,0x8444,0x8445,0x8447,0x8448,0x8449,0x0000,/* 0x78-0x7F */ + + 0x844A,0x844B,0x844C,0x844D,0x844E,0x844F,0x8450,0x8452,/* 0x80-0x87 */ + 0x8453,0x8454,0x8455,0x8456,0x8458,0x845D,0x845E,0x845F,/* 0x88-0x8F */ + 0x8460,0x8462,0x8464,0x8465,0x8466,0x8467,0x8468,0x846A,/* 0x90-0x97 */ + 0x846E,0x846F,0x8470,0x8472,0x8474,0x8477,0x8479,0x847B,/* 0x98-0x9F */ + 0x847C,0x53D6,0x5A36,0x9F8B,0x8DA3,0x53BB,0x5708,0x98A7,/* 0xA0-0xA7 */ + 0x6743,0x919B,0x6CC9,0x5168,0x75CA,0x62F3,0x72AC,0x5238,/* 0xA8-0xAF */ + 0x529D,0x7F3A,0x7094,0x7638,0x5374,0x9E4A,0x69B7,0x786E,/* 0xB0-0xB7 */ + 0x96C0,0x88D9,0x7FA4,0x7136,0x71C3,0x5189,0x67D3,0x74E4,/* 0xB8-0xBF */ + 0x58E4,0x6518,0x56B7,0x8BA9,0x9976,0x6270,0x7ED5,0x60F9,/* 0xC0-0xC7 */ + 0x70ED,0x58EC,0x4EC1,0x4EBA,0x5FCD,0x97E7,0x4EFB,0x8BA4,/* 0xC8-0xCF */ + 0x5203,0x598A,0x7EAB,0x6254,0x4ECD,0x65E5,0x620E,0x8338,/* 0xD0-0xD7 */ + 0x84C9,0x8363,0x878D,0x7194,0x6EB6,0x5BB9,0x7ED2,0x5197,/* 0xD8-0xDF */ + 0x63C9,0x67D4,0x8089,0x8339,0x8815,0x5112,0x5B7A,0x5982,/* 0xE0-0xE7 */ + 0x8FB1,0x4E73,0x6C5D,0x5165,0x8925,0x8F6F,0x962E,0x854A,/* 0xE8-0xEF */ + 0x745E,0x9510,0x95F0,0x6DA6,0x82E5,0x5F31,0x6492,0x6D12,/* 0xF0-0xF7 */ + 0x8428,0x816E,0x9CC3,0x585E,0x8D5B,0x4E09,0x53C1,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_C9[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x847D,0x847E,0x847F,0x8480,0x8481,0x8483,0x8484,0x8485,/* 0x40-0x47 */ + 0x8486,0x848A,0x848D,0x848F,0x8490,0x8491,0x8492,0x8493,/* 0x48-0x4F */ + 0x8494,0x8495,0x8496,0x8498,0x849A,0x849B,0x849D,0x849E,/* 0x50-0x57 */ + 0x849F,0x84A0,0x84A2,0x84A3,0x84A4,0x84A5,0x84A6,0x84A7,/* 0x58-0x5F */ + 0x84A8,0x84A9,0x84AA,0x84AB,0x84AC,0x84AD,0x84AE,0x84B0,/* 0x60-0x67 */ + 0x84B1,0x84B3,0x84B5,0x84B6,0x84B7,0x84BB,0x84BC,0x84BE,/* 0x68-0x6F */ + 0x84C0,0x84C2,0x84C3,0x84C5,0x84C6,0x84C7,0x84C8,0x84CB,/* 0x70-0x77 */ + 0x84CC,0x84CE,0x84CF,0x84D2,0x84D4,0x84D5,0x84D7,0x0000,/* 0x78-0x7F */ + + 0x84D8,0x84D9,0x84DA,0x84DB,0x84DC,0x84DE,0x84E1,0x84E2,/* 0x80-0x87 */ + 0x84E4,0x84E7,0x84E8,0x84E9,0x84EA,0x84EB,0x84ED,0x84EE,/* 0x88-0x8F */ + 0x84EF,0x84F1,0x84F2,0x84F3,0x84F4,0x84F5,0x84F6,0x84F7,/* 0x90-0x97 */ + 0x84F8,0x84F9,0x84FA,0x84FB,0x84FD,0x84FE,0x8500,0x8501,/* 0x98-0x9F */ + 0x8502,0x4F1E,0x6563,0x6851,0x55D3,0x4E27,0x6414,0x9A9A,/* 0xA0-0xA7 */ + 0x626B,0x5AC2,0x745F,0x8272,0x6DA9,0x68EE,0x50E7,0x838E,/* 0xA8-0xAF */ + 0x7802,0x6740,0x5239,0x6C99,0x7EB1,0x50BB,0x5565,0x715E,/* 0xB0-0xB7 */ + 0x7B5B,0x6652,0x73CA,0x82EB,0x6749,0x5C71,0x5220,0x717D,/* 0xB8-0xBF */ + 0x886B,0x95EA,0x9655,0x64C5,0x8D61,0x81B3,0x5584,0x6C55,/* 0xC0-0xC7 */ + 0x6247,0x7F2E,0x5892,0x4F24,0x5546,0x8D4F,0x664C,0x4E0A,/* 0xC8-0xCF */ + 0x5C1A,0x88F3,0x68A2,0x634E,0x7A0D,0x70E7,0x828D,0x52FA,/* 0xD0-0xD7 */ + 0x97F6,0x5C11,0x54E8,0x90B5,0x7ECD,0x5962,0x8D4A,0x86C7,/* 0xD8-0xDF */ + 0x820C,0x820D,0x8D66,0x6444,0x5C04,0x6151,0x6D89,0x793E,/* 0xE0-0xE7 */ + 0x8BBE,0x7837,0x7533,0x547B,0x4F38,0x8EAB,0x6DF1,0x5A20,/* 0xE8-0xEF */ + 0x7EC5,0x795E,0x6C88,0x5BA1,0x5A76,0x751A,0x80BE,0x614E,/* 0xF0-0xF7 */ + 0x6E17,0x58F0,0x751F,0x7525,0x7272,0x5347,0x7EF3,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_CA[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8503,0x8504,0x8505,0x8506,0x8507,0x8508,0x8509,0x850A,/* 0x40-0x47 */ + 0x850B,0x850D,0x850E,0x850F,0x8510,0x8512,0x8514,0x8515,/* 0x48-0x4F */ + 0x8516,0x8518,0x8519,0x851B,0x851C,0x851D,0x851E,0x8520,/* 0x50-0x57 */ + 0x8522,0x8523,0x8524,0x8525,0x8526,0x8527,0x8528,0x8529,/* 0x58-0x5F */ + 0x852A,0x852D,0x852E,0x852F,0x8530,0x8531,0x8532,0x8533,/* 0x60-0x67 */ + 0x8534,0x8535,0x8536,0x853E,0x853F,0x8540,0x8541,0x8542,/* 0x68-0x6F */ + 0x8544,0x8545,0x8546,0x8547,0x854B,0x854C,0x854D,0x854E,/* 0x70-0x77 */ + 0x854F,0x8550,0x8551,0x8552,0x8553,0x8554,0x8555,0x0000,/* 0x78-0x7F */ + + 0x8557,0x8558,0x855A,0x855B,0x855C,0x855D,0x855F,0x8560,/* 0x80-0x87 */ + 0x8561,0x8562,0x8563,0x8565,0x8566,0x8567,0x8569,0x856A,/* 0x88-0x8F */ + 0x856B,0x856C,0x856D,0x856E,0x856F,0x8570,0x8571,0x8573,/* 0x90-0x97 */ + 0x8575,0x8576,0x8577,0x8578,0x857C,0x857D,0x857F,0x8580,/* 0x98-0x9F */ + 0x8581,0x7701,0x76DB,0x5269,0x80DC,0x5723,0x5E08,0x5931,/* 0xA0-0xA7 */ + 0x72EE,0x65BD,0x6E7F,0x8BD7,0x5C38,0x8671,0x5341,0x77F3,/* 0xA8-0xAF */ + 0x62FE,0x65F6,0x4EC0,0x98DF,0x8680,0x5B9E,0x8BC6,0x53F2,/* 0xB0-0xB7 */ + 0x77E2,0x4F7F,0x5C4E,0x9A76,0x59CB,0x5F0F,0x793A,0x58EB,/* 0xB8-0xBF */ + 0x4E16,0x67FF,0x4E8B,0x62ED,0x8A93,0x901D,0x52BF,0x662F,/* 0xC0-0xC7 */ + 0x55DC,0x566C,0x9002,0x4ED5,0x4F8D,0x91CA,0x9970,0x6C0F,/* 0xC8-0xCF */ + 0x5E02,0x6043,0x5BA4,0x89C6,0x8BD5,0x6536,0x624B,0x9996,/* 0xD0-0xD7 */ + 0x5B88,0x5BFF,0x6388,0x552E,0x53D7,0x7626,0x517D,0x852C,/* 0xD8-0xDF */ + 0x67A2,0x68B3,0x6B8A,0x6292,0x8F93,0x53D4,0x8212,0x6DD1,/* 0xE0-0xE7 */ + 0x758F,0x4E66,0x8D4E,0x5B70,0x719F,0x85AF,0x6691,0x66D9,/* 0xE8-0xEF */ + 0x7F72,0x8700,0x9ECD,0x9F20,0x5C5E,0x672F,0x8FF0,0x6811,/* 0xF0-0xF7 */ + 0x675F,0x620D,0x7AD6,0x5885,0x5EB6,0x6570,0x6F31,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_CB[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8582,0x8583,0x8586,0x8588,0x8589,0x858A,0x858B,0x858C,/* 0x40-0x47 */ + 0x858D,0x858E,0x8590,0x8591,0x8592,0x8593,0x8594,0x8595,/* 0x48-0x4F */ + 0x8596,0x8597,0x8598,0x8599,0x859A,0x859D,0x859E,0x859F,/* 0x50-0x57 */ + 0x85A0,0x85A1,0x85A2,0x85A3,0x85A5,0x85A6,0x85A7,0x85A9,/* 0x58-0x5F */ + 0x85AB,0x85AC,0x85AD,0x85B1,0x85B2,0x85B3,0x85B4,0x85B5,/* 0x60-0x67 */ + 0x85B6,0x85B8,0x85BA,0x85BB,0x85BC,0x85BD,0x85BE,0x85BF,/* 0x68-0x6F */ + 0x85C0,0x85C2,0x85C3,0x85C4,0x85C5,0x85C6,0x85C7,0x85C8,/* 0x70-0x77 */ + 0x85CA,0x85CB,0x85CC,0x85CD,0x85CE,0x85D1,0x85D2,0x0000,/* 0x78-0x7F */ + + 0x85D4,0x85D6,0x85D7,0x85D8,0x85D9,0x85DA,0x85DB,0x85DD,/* 0x80-0x87 */ + 0x85DE,0x85DF,0x85E0,0x85E1,0x85E2,0x85E3,0x85E5,0x85E6,/* 0x88-0x8F */ + 0x85E7,0x85E8,0x85EA,0x85EB,0x85EC,0x85ED,0x85EE,0x85EF,/* 0x90-0x97 */ + 0x85F0,0x85F1,0x85F2,0x85F3,0x85F4,0x85F5,0x85F6,0x85F7,/* 0x98-0x9F */ + 0x85F8,0x6055,0x5237,0x800D,0x6454,0x8870,0x7529,0x5E05,/* 0xA0-0xA7 */ + 0x6813,0x62F4,0x971C,0x53CC,0x723D,0x8C01,0x6C34,0x7761,/* 0xA8-0xAF */ + 0x7A0E,0x542E,0x77AC,0x987A,0x821C,0x8BF4,0x7855,0x6714,/* 0xB0-0xB7 */ + 0x70C1,0x65AF,0x6495,0x5636,0x601D,0x79C1,0x53F8,0x4E1D,/* 0xB8-0xBF */ + 0x6B7B,0x8086,0x5BFA,0x55E3,0x56DB,0x4F3A,0x4F3C,0x9972,/* 0xC0-0xC7 */ + 0x5DF3,0x677E,0x8038,0x6002,0x9882,0x9001,0x5B8B,0x8BBC,/* 0xC8-0xCF */ + 0x8BF5,0x641C,0x8258,0x64DE,0x55FD,0x82CF,0x9165,0x4FD7,/* 0xD0-0xD7 */ + 0x7D20,0x901F,0x7C9F,0x50F3,0x5851,0x6EAF,0x5BBF,0x8BC9,/* 0xD8-0xDF */ + 0x8083,0x9178,0x849C,0x7B97,0x867D,0x968B,0x968F,0x7EE5,/* 0xE0-0xE7 */ + 0x9AD3,0x788E,0x5C81,0x7A57,0x9042,0x96A7,0x795F,0x5B59,/* 0xE8-0xEF */ + 0x635F,0x7B0B,0x84D1,0x68AD,0x5506,0x7F29,0x7410,0x7D22,/* 0xF0-0xF7 */ + 0x9501,0x6240,0x584C,0x4ED6,0x5B83,0x5979,0x5854,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_CC[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x85F9,0x85FA,0x85FC,0x85FD,0x85FE,0x8600,0x8601,0x8602,/* 0x40-0x47 */ + 0x8603,0x8604,0x8606,0x8607,0x8608,0x8609,0x860A,0x860B,/* 0x48-0x4F */ + 0x860C,0x860D,0x860E,0x860F,0x8610,0x8612,0x8613,0x8614,/* 0x50-0x57 */ + 0x8615,0x8617,0x8618,0x8619,0x861A,0x861B,0x861C,0x861D,/* 0x58-0x5F */ + 0x861E,0x861F,0x8620,0x8621,0x8622,0x8623,0x8624,0x8625,/* 0x60-0x67 */ + 0x8626,0x8628,0x862A,0x862B,0x862C,0x862D,0x862E,0x862F,/* 0x68-0x6F */ + 0x8630,0x8631,0x8632,0x8633,0x8634,0x8635,0x8636,0x8637,/* 0x70-0x77 */ + 0x8639,0x863A,0x863B,0x863D,0x863E,0x863F,0x8640,0x0000,/* 0x78-0x7F */ + + 0x8641,0x8642,0x8643,0x8644,0x8645,0x8646,0x8647,0x8648,/* 0x80-0x87 */ + 0x8649,0x864A,0x864B,0x864C,0x8652,0x8653,0x8655,0x8656,/* 0x88-0x8F */ + 0x8657,0x8658,0x8659,0x865B,0x865C,0x865D,0x865F,0x8660,/* 0x90-0x97 */ + 0x8661,0x8663,0x8664,0x8665,0x8666,0x8667,0x8668,0x8669,/* 0x98-0x9F */ + 0x866A,0x736D,0x631E,0x8E4B,0x8E0F,0x80CE,0x82D4,0x62AC,/* 0xA0-0xA7 */ + 0x53F0,0x6CF0,0x915E,0x592A,0x6001,0x6C70,0x574D,0x644A,/* 0xA8-0xAF */ + 0x8D2A,0x762B,0x6EE9,0x575B,0x6A80,0x75F0,0x6F6D,0x8C2D,/* 0xB0-0xB7 */ + 0x8C08,0x5766,0x6BEF,0x8892,0x78B3,0x63A2,0x53F9,0x70AD,/* 0xB8-0xBF */ + 0x6C64,0x5858,0x642A,0x5802,0x68E0,0x819B,0x5510,0x7CD6,/* 0xC0-0xC7 */ + 0x5018,0x8EBA,0x6DCC,0x8D9F,0x70EB,0x638F,0x6D9B,0x6ED4,/* 0xC8-0xCF */ + 0x7EE6,0x8404,0x6843,0x9003,0x6DD8,0x9676,0x8BA8,0x5957,/* 0xD0-0xD7 */ + 0x7279,0x85E4,0x817E,0x75BC,0x8A8A,0x68AF,0x5254,0x8E22,/* 0xD8-0xDF */ + 0x9511,0x63D0,0x9898,0x8E44,0x557C,0x4F53,0x66FF,0x568F,/* 0xE0-0xE7 */ + 0x60D5,0x6D95,0x5243,0x5C49,0x5929,0x6DFB,0x586B,0x7530,/* 0xE8-0xEF */ + 0x751C,0x606C,0x8214,0x8146,0x6311,0x6761,0x8FE2,0x773A,/* 0xF0-0xF7 */ + 0x8DF3,0x8D34,0x94C1,0x5E16,0x5385,0x542C,0x70C3,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_CD[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x866D,0x866F,0x8670,0x8672,0x8673,0x8674,0x8675,0x8676,/* 0x40-0x47 */ + 0x8677,0x8678,0x8683,0x8684,0x8685,0x8686,0x8687,0x8688,/* 0x48-0x4F */ + 0x8689,0x868E,0x868F,0x8690,0x8691,0x8692,0x8694,0x8696,/* 0x50-0x57 */ + 0x8697,0x8698,0x8699,0x869A,0x869B,0x869E,0x869F,0x86A0,/* 0x58-0x5F */ + 0x86A1,0x86A2,0x86A5,0x86A6,0x86AB,0x86AD,0x86AE,0x86B2,/* 0x60-0x67 */ + 0x86B3,0x86B7,0x86B8,0x86B9,0x86BB,0x86BC,0x86BD,0x86BE,/* 0x68-0x6F */ + 0x86BF,0x86C1,0x86C2,0x86C3,0x86C5,0x86C8,0x86CC,0x86CD,/* 0x70-0x77 */ + 0x86D2,0x86D3,0x86D5,0x86D6,0x86D7,0x86DA,0x86DC,0x0000,/* 0x78-0x7F */ + + 0x86DD,0x86E0,0x86E1,0x86E2,0x86E3,0x86E5,0x86E6,0x86E7,/* 0x80-0x87 */ + 0x86E8,0x86EA,0x86EB,0x86EC,0x86EF,0x86F5,0x86F6,0x86F7,/* 0x88-0x8F */ + 0x86FA,0x86FB,0x86FC,0x86FD,0x86FF,0x8701,0x8704,0x8705,/* 0x90-0x97 */ + 0x8706,0x870B,0x870C,0x870E,0x870F,0x8710,0x8711,0x8714,/* 0x98-0x9F */ + 0x8716,0x6C40,0x5EF7,0x505C,0x4EAD,0x5EAD,0x633A,0x8247,/* 0xA0-0xA7 */ + 0x901A,0x6850,0x916E,0x77B3,0x540C,0x94DC,0x5F64,0x7AE5,/* 0xA8-0xAF */ + 0x6876,0x6345,0x7B52,0x7EDF,0x75DB,0x5077,0x6295,0x5934,/* 0xB0-0xB7 */ + 0x900F,0x51F8,0x79C3,0x7A81,0x56FE,0x5F92,0x9014,0x6D82,/* 0xB8-0xBF */ + 0x5C60,0x571F,0x5410,0x5154,0x6E4D,0x56E2,0x63A8,0x9893,/* 0xC0-0xC7 */ + 0x817F,0x8715,0x892A,0x9000,0x541E,0x5C6F,0x81C0,0x62D6,/* 0xC8-0xCF */ + 0x6258,0x8131,0x9E35,0x9640,0x9A6E,0x9A7C,0x692D,0x59A5,/* 0xD0-0xD7 */ + 0x62D3,0x553E,0x6316,0x54C7,0x86D9,0x6D3C,0x5A03,0x74E6,/* 0xD8-0xDF */ + 0x889C,0x6B6A,0x5916,0x8C4C,0x5F2F,0x6E7E,0x73A9,0x987D,/* 0xE0-0xE7 */ + 0x4E38,0x70F7,0x5B8C,0x7897,0x633D,0x665A,0x7696,0x60CB,/* 0xE8-0xEF */ + 0x5B9B,0x5A49,0x4E07,0x8155,0x6C6A,0x738B,0x4EA1,0x6789,/* 0xF0-0xF7 */ + 0x7F51,0x5F80,0x65FA,0x671B,0x5FD8,0x5984,0x5A01,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_CE[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8719,0x871B,0x871D,0x871F,0x8720,0x8724,0x8726,0x8727,/* 0x40-0x47 */ + 0x8728,0x872A,0x872B,0x872C,0x872D,0x872F,0x8730,0x8732,/* 0x48-0x4F */ + 0x8733,0x8735,0x8736,0x8738,0x8739,0x873A,0x873C,0x873D,/* 0x50-0x57 */ + 0x8740,0x8741,0x8742,0x8743,0x8744,0x8745,0x8746,0x874A,/* 0x58-0x5F */ + 0x874B,0x874D,0x874F,0x8750,0x8751,0x8752,0x8754,0x8755,/* 0x60-0x67 */ + 0x8756,0x8758,0x875A,0x875B,0x875C,0x875D,0x875E,0x875F,/* 0x68-0x6F */ + 0x8761,0x8762,0x8766,0x8767,0x8768,0x8769,0x876A,0x876B,/* 0x70-0x77 */ + 0x876C,0x876D,0x876F,0x8771,0x8772,0x8773,0x8775,0x0000,/* 0x78-0x7F */ + + 0x8777,0x8778,0x8779,0x877A,0x877F,0x8780,0x8781,0x8784,/* 0x80-0x87 */ + 0x8786,0x8787,0x8789,0x878A,0x878C,0x878E,0x878F,0x8790,/* 0x88-0x8F */ + 0x8791,0x8792,0x8794,0x8795,0x8796,0x8798,0x8799,0x879A,/* 0x90-0x97 */ + 0x879B,0x879C,0x879D,0x879E,0x87A0,0x87A1,0x87A2,0x87A3,/* 0x98-0x9F */ + 0x87A4,0x5DCD,0x5FAE,0x5371,0x97E6,0x8FDD,0x6845,0x56F4,/* 0xA0-0xA7 */ + 0x552F,0x60DF,0x4E3A,0x6F4D,0x7EF4,0x82C7,0x840E,0x59D4,/* 0xA8-0xAF */ + 0x4F1F,0x4F2A,0x5C3E,0x7EAC,0x672A,0x851A,0x5473,0x754F,/* 0xB0-0xB7 */ + 0x80C3,0x5582,0x9B4F,0x4F4D,0x6E2D,0x8C13,0x5C09,0x6170,/* 0xB8-0xBF */ + 0x536B,0x761F,0x6E29,0x868A,0x6587,0x95FB,0x7EB9,0x543B,/* 0xC0-0xC7 */ + 0x7A33,0x7D0A,0x95EE,0x55E1,0x7FC1,0x74EE,0x631D,0x8717,/* 0xC8-0xCF */ + 0x6DA1,0x7A9D,0x6211,0x65A1,0x5367,0x63E1,0x6C83,0x5DEB,/* 0xD0-0xD7 */ + 0x545C,0x94A8,0x4E4C,0x6C61,0x8BEC,0x5C4B,0x65E0,0x829C,/* 0xD8-0xDF */ + 0x68A7,0x543E,0x5434,0x6BCB,0x6B66,0x4E94,0x6342,0x5348,/* 0xE0-0xE7 */ + 0x821E,0x4F0D,0x4FAE,0x575E,0x620A,0x96FE,0x6664,0x7269,/* 0xE8-0xEF */ + 0x52FF,0x52A1,0x609F,0x8BEF,0x6614,0x7199,0x6790,0x897F,/* 0xF0-0xF7 */ + 0x7852,0x77FD,0x6670,0x563B,0x5438,0x9521,0x727A,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_CF[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x87A5,0x87A6,0x87A7,0x87A9,0x87AA,0x87AE,0x87B0,0x87B1,/* 0x40-0x47 */ + 0x87B2,0x87B4,0x87B6,0x87B7,0x87B8,0x87B9,0x87BB,0x87BC,/* 0x48-0x4F */ + 0x87BE,0x87BF,0x87C1,0x87C2,0x87C3,0x87C4,0x87C5,0x87C7,/* 0x50-0x57 */ + 0x87C8,0x87C9,0x87CC,0x87CD,0x87CE,0x87CF,0x87D0,0x87D4,/* 0x58-0x5F */ + 0x87D5,0x87D6,0x87D7,0x87D8,0x87D9,0x87DA,0x87DC,0x87DD,/* 0x60-0x67 */ + 0x87DE,0x87DF,0x87E1,0x87E2,0x87E3,0x87E4,0x87E6,0x87E7,/* 0x68-0x6F */ + 0x87E8,0x87E9,0x87EB,0x87EC,0x87ED,0x87EF,0x87F0,0x87F1,/* 0x70-0x77 */ + 0x87F2,0x87F3,0x87F4,0x87F5,0x87F6,0x87F7,0x87F8,0x0000,/* 0x78-0x7F */ + + 0x87FA,0x87FB,0x87FC,0x87FD,0x87FF,0x8800,0x8801,0x8802,/* 0x80-0x87 */ + 0x8804,0x8805,0x8806,0x8807,0x8808,0x8809,0x880B,0x880C,/* 0x88-0x8F */ + 0x880D,0x880E,0x880F,0x8810,0x8811,0x8812,0x8814,0x8817,/* 0x90-0x97 */ + 0x8818,0x8819,0x881A,0x881C,0x881D,0x881E,0x881F,0x8820,/* 0x98-0x9F */ + 0x8823,0x7A00,0x606F,0x5E0C,0x6089,0x819D,0x5915,0x60DC,/* 0xA0-0xA7 */ + 0x7184,0x70EF,0x6EAA,0x6C50,0x7280,0x6A84,0x88AD,0x5E2D,/* 0xA8-0xAF */ + 0x4E60,0x5AB3,0x559C,0x94E3,0x6D17,0x7CFB,0x9699,0x620F,/* 0xB0-0xB7 */ + 0x7EC6,0x778E,0x867E,0x5323,0x971E,0x8F96,0x6687,0x5CE1,/* 0xB8-0xBF */ + 0x4FA0,0x72ED,0x4E0B,0x53A6,0x590F,0x5413,0x6380,0x9528,/* 0xC0-0xC7 */ + 0x5148,0x4ED9,0x9C9C,0x7EA4,0x54B8,0x8D24,0x8854,0x8237,/* 0xC8-0xCF */ + 0x95F2,0x6D8E,0x5F26,0x5ACC,0x663E,0x9669,0x73B0,0x732E,/* 0xD0-0xD7 */ + 0x53BF,0x817A,0x9985,0x7FA1,0x5BAA,0x9677,0x9650,0x7EBF,/* 0xD8-0xDF */ + 0x76F8,0x53A2,0x9576,0x9999,0x7BB1,0x8944,0x6E58,0x4E61,/* 0xE0-0xE7 */ + 0x7FD4,0x7965,0x8BE6,0x60F3,0x54CD,0x4EAB,0x9879,0x5DF7,/* 0xE8-0xEF */ + 0x6A61,0x50CF,0x5411,0x8C61,0x8427,0x785D,0x9704,0x524A,/* 0xF0-0xF7 */ + 0x54EE,0x56A3,0x9500,0x6D88,0x5BB5,0x6DC6,0x6653,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D0[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8824,0x8825,0x8826,0x8827,0x8828,0x8829,0x882A,0x882B,/* 0x40-0x47 */ + 0x882C,0x882D,0x882E,0x882F,0x8830,0x8831,0x8833,0x8834,/* 0x48-0x4F */ + 0x8835,0x8836,0x8837,0x8838,0x883A,0x883B,0x883D,0x883E,/* 0x50-0x57 */ + 0x883F,0x8841,0x8842,0x8843,0x8846,0x8847,0x8848,0x8849,/* 0x58-0x5F */ + 0x884A,0x884B,0x884E,0x884F,0x8850,0x8851,0x8852,0x8853,/* 0x60-0x67 */ + 0x8855,0x8856,0x8858,0x885A,0x885B,0x885C,0x885D,0x885E,/* 0x68-0x6F */ + 0x885F,0x8860,0x8866,0x8867,0x886A,0x886D,0x886F,0x8871,/* 0x70-0x77 */ + 0x8873,0x8874,0x8875,0x8876,0x8878,0x8879,0x887A,0x0000,/* 0x78-0x7F */ + + 0x887B,0x887C,0x8880,0x8883,0x8886,0x8887,0x8889,0x888A,/* 0x80-0x87 */ + 0x888C,0x888E,0x888F,0x8890,0x8891,0x8893,0x8894,0x8895,/* 0x88-0x8F */ + 0x8897,0x8898,0x8899,0x889A,0x889B,0x889D,0x889E,0x889F,/* 0x90-0x97 */ + 0x88A0,0x88A1,0x88A3,0x88A5,0x88A6,0x88A7,0x88A8,0x88A9,/* 0x98-0x9F */ + 0x88AA,0x5C0F,0x5B5D,0x6821,0x8096,0x5578,0x7B11,0x6548,/* 0xA0-0xA7 */ + 0x6954,0x4E9B,0x6B47,0x874E,0x978B,0x534F,0x631F,0x643A,/* 0xA8-0xAF */ + 0x90AA,0x659C,0x80C1,0x8C10,0x5199,0x68B0,0x5378,0x87F9,/* 0xB0-0xB7 */ + 0x61C8,0x6CC4,0x6CFB,0x8C22,0x5C51,0x85AA,0x82AF,0x950C,/* 0xB8-0xBF */ + 0x6B23,0x8F9B,0x65B0,0x5FFB,0x5FC3,0x4FE1,0x8845,0x661F,/* 0xC0-0xC7 */ + 0x8165,0x7329,0x60FA,0x5174,0x5211,0x578B,0x5F62,0x90A2,/* 0xC8-0xCF */ + 0x884C,0x9192,0x5E78,0x674F,0x6027,0x59D3,0x5144,0x51F6,/* 0xD0-0xD7 */ + 0x80F8,0x5308,0x6C79,0x96C4,0x718A,0x4F11,0x4FEE,0x7F9E,/* 0xD8-0xDF */ + 0x673D,0x55C5,0x9508,0x79C0,0x8896,0x7EE3,0x589F,0x620C,/* 0xE0-0xE7 */ + 0x9700,0x865A,0x5618,0x987B,0x5F90,0x8BB8,0x84C4,0x9157,/* 0xE8-0xEF */ + 0x53D9,0x65ED,0x5E8F,0x755C,0x6064,0x7D6E,0x5A7F,0x7EEA,/* 0xF0-0xF7 */ + 0x7EED,0x8F69,0x55A7,0x5BA3,0x60AC,0x65CB,0x7384,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D1[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x88AC,0x88AE,0x88AF,0x88B0,0x88B2,0x88B3,0x88B4,0x88B5,/* 0x40-0x47 */ + 0x88B6,0x88B8,0x88B9,0x88BA,0x88BB,0x88BD,0x88BE,0x88BF,/* 0x48-0x4F */ + 0x88C0,0x88C3,0x88C4,0x88C7,0x88C8,0x88CA,0x88CB,0x88CC,/* 0x50-0x57 */ + 0x88CD,0x88CF,0x88D0,0x88D1,0x88D3,0x88D6,0x88D7,0x88DA,/* 0x58-0x5F */ + 0x88DB,0x88DC,0x88DD,0x88DE,0x88E0,0x88E1,0x88E6,0x88E7,/* 0x60-0x67 */ + 0x88E9,0x88EA,0x88EB,0x88EC,0x88ED,0x88EE,0x88EF,0x88F2,/* 0x68-0x6F */ + 0x88F5,0x88F6,0x88F7,0x88FA,0x88FB,0x88FD,0x88FF,0x8900,/* 0x70-0x77 */ + 0x8901,0x8903,0x8904,0x8905,0x8906,0x8907,0x8908,0x0000,/* 0x78-0x7F */ + + 0x8909,0x890B,0x890C,0x890D,0x890E,0x890F,0x8911,0x8914,/* 0x80-0x87 */ + 0x8915,0x8916,0x8917,0x8918,0x891C,0x891D,0x891E,0x891F,/* 0x88-0x8F */ + 0x8920,0x8922,0x8923,0x8924,0x8926,0x8927,0x8928,0x8929,/* 0x90-0x97 */ + 0x892C,0x892D,0x892E,0x892F,0x8931,0x8932,0x8933,0x8935,/* 0x98-0x9F */ + 0x8937,0x9009,0x7663,0x7729,0x7EDA,0x9774,0x859B,0x5B66,/* 0xA0-0xA7 */ + 0x7A74,0x96EA,0x8840,0x52CB,0x718F,0x5FAA,0x65EC,0x8BE2,/* 0xA8-0xAF */ + 0x5BFB,0x9A6F,0x5DE1,0x6B89,0x6C5B,0x8BAD,0x8BAF,0x900A,/* 0xB0-0xB7 */ + 0x8FC5,0x538B,0x62BC,0x9E26,0x9E2D,0x5440,0x4E2B,0x82BD,/* 0xB8-0xBF */ + 0x7259,0x869C,0x5D16,0x8859,0x6DAF,0x96C5,0x54D1,0x4E9A,/* 0xC0-0xC7 */ + 0x8BB6,0x7109,0x54BD,0x9609,0x70DF,0x6DF9,0x76D0,0x4E25,/* 0xC8-0xCF */ + 0x7814,0x8712,0x5CA9,0x5EF6,0x8A00,0x989C,0x960E,0x708E,/* 0xD0-0xD7 */ + 0x6CBF,0x5944,0x63A9,0x773C,0x884D,0x6F14,0x8273,0x5830,/* 0xD8-0xDF */ + 0x71D5,0x538C,0x781A,0x96C1,0x5501,0x5F66,0x7130,0x5BB4,/* 0xE0-0xE7 */ + 0x8C1A,0x9A8C,0x6B83,0x592E,0x9E2F,0x79E7,0x6768,0x626C,/* 0xE8-0xEF */ + 0x4F6F,0x75A1,0x7F8A,0x6D0B,0x9633,0x6C27,0x4EF0,0x75D2,/* 0xF0-0xF7 */ + 0x517B,0x6837,0x6F3E,0x9080,0x8170,0x5996,0x7476,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D2[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8938,0x8939,0x893A,0x893B,0x893C,0x893D,0x893E,0x893F,/* 0x40-0x47 */ + 0x8940,0x8942,0x8943,0x8945,0x8946,0x8947,0x8948,0x8949,/* 0x48-0x4F */ + 0x894A,0x894B,0x894C,0x894D,0x894E,0x894F,0x8950,0x8951,/* 0x50-0x57 */ + 0x8952,0x8953,0x8954,0x8955,0x8956,0x8957,0x8958,0x8959,/* 0x58-0x5F */ + 0x895A,0x895B,0x895C,0x895D,0x8960,0x8961,0x8962,0x8963,/* 0x60-0x67 */ + 0x8964,0x8965,0x8967,0x8968,0x8969,0x896A,0x896B,0x896C,/* 0x68-0x6F */ + 0x896D,0x896E,0x896F,0x8970,0x8971,0x8972,0x8973,0x8974,/* 0x70-0x77 */ + 0x8975,0x8976,0x8977,0x8978,0x8979,0x897A,0x897C,0x0000,/* 0x78-0x7F */ + + 0x897D,0x897E,0x8980,0x8982,0x8984,0x8985,0x8987,0x8988,/* 0x80-0x87 */ + 0x8989,0x898A,0x898B,0x898C,0x898D,0x898E,0x898F,0x8990,/* 0x88-0x8F */ + 0x8991,0x8992,0x8993,0x8994,0x8995,0x8996,0x8997,0x8998,/* 0x90-0x97 */ + 0x8999,0x899A,0x899B,0x899C,0x899D,0x899E,0x899F,0x89A0,/* 0x98-0x9F */ + 0x89A1,0x6447,0x5C27,0x9065,0x7A91,0x8C23,0x59DA,0x54AC,/* 0xA0-0xA7 */ + 0x8200,0x836F,0x8981,0x8000,0x6930,0x564E,0x8036,0x7237,/* 0xA8-0xAF */ + 0x91CE,0x51B6,0x4E5F,0x9875,0x6396,0x4E1A,0x53F6,0x66F3,/* 0xB0-0xB7 */ + 0x814B,0x591C,0x6DB2,0x4E00,0x58F9,0x533B,0x63D6,0x94F1,/* 0xB8-0xBF */ + 0x4F9D,0x4F0A,0x8863,0x9890,0x5937,0x9057,0x79FB,0x4EEA,/* 0xC0-0xC7 */ + 0x80F0,0x7591,0x6C82,0x5B9C,0x59E8,0x5F5D,0x6905,0x8681,/* 0xC8-0xCF */ + 0x501A,0x5DF2,0x4E59,0x77E3,0x4EE5,0x827A,0x6291,0x6613,/* 0xD0-0xD7 */ + 0x9091,0x5C79,0x4EBF,0x5F79,0x81C6,0x9038,0x8084,0x75AB,/* 0xD8-0xDF */ + 0x4EA6,0x88D4,0x610F,0x6BC5,0x5FC6,0x4E49,0x76CA,0x6EA2,/* 0xE0-0xE7 */ + 0x8BE3,0x8BAE,0x8C0A,0x8BD1,0x5F02,0x7FFC,0x7FCC,0x7ECE,/* 0xE8-0xEF */ + 0x8335,0x836B,0x56E0,0x6BB7,0x97F3,0x9634,0x59FB,0x541F,/* 0xF0-0xF7 */ + 0x94F6,0x6DEB,0x5BC5,0x996E,0x5C39,0x5F15,0x9690,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D3[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x89A2,0x89A3,0x89A4,0x89A5,0x89A6,0x89A7,0x89A8,0x89A9,/* 0x40-0x47 */ + 0x89AA,0x89AB,0x89AC,0x89AD,0x89AE,0x89AF,0x89B0,0x89B1,/* 0x48-0x4F */ + 0x89B2,0x89B3,0x89B4,0x89B5,0x89B6,0x89B7,0x89B8,0x89B9,/* 0x50-0x57 */ + 0x89BA,0x89BB,0x89BC,0x89BD,0x89BE,0x89BF,0x89C0,0x89C3,/* 0x58-0x5F */ + 0x89CD,0x89D3,0x89D4,0x89D5,0x89D7,0x89D8,0x89D9,0x89DB,/* 0x60-0x67 */ + 0x89DD,0x89DF,0x89E0,0x89E1,0x89E2,0x89E4,0x89E7,0x89E8,/* 0x68-0x6F */ + 0x89E9,0x89EA,0x89EC,0x89ED,0x89EE,0x89F0,0x89F1,0x89F2,/* 0x70-0x77 */ + 0x89F4,0x89F5,0x89F6,0x89F7,0x89F8,0x89F9,0x89FA,0x0000,/* 0x78-0x7F */ + + 0x89FB,0x89FC,0x89FD,0x89FE,0x89FF,0x8A01,0x8A02,0x8A03,/* 0x80-0x87 */ + 0x8A04,0x8A05,0x8A06,0x8A08,0x8A09,0x8A0A,0x8A0B,0x8A0C,/* 0x88-0x8F */ + 0x8A0D,0x8A0E,0x8A0F,0x8A10,0x8A11,0x8A12,0x8A13,0x8A14,/* 0x90-0x97 */ + 0x8A15,0x8A16,0x8A17,0x8A18,0x8A19,0x8A1A,0x8A1B,0x8A1C,/* 0x98-0x9F */ + 0x8A1D,0x5370,0x82F1,0x6A31,0x5A74,0x9E70,0x5E94,0x7F28,/* 0xA0-0xA7 */ + 0x83B9,0x8424,0x8425,0x8367,0x8747,0x8FCE,0x8D62,0x76C8,/* 0xA8-0xAF */ + 0x5F71,0x9896,0x786C,0x6620,0x54DF,0x62E5,0x4F63,0x81C3,/* 0xB0-0xB7 */ + 0x75C8,0x5EB8,0x96CD,0x8E0A,0x86F9,0x548F,0x6CF3,0x6D8C,/* 0xB8-0xBF */ + 0x6C38,0x607F,0x52C7,0x7528,0x5E7D,0x4F18,0x60A0,0x5FE7,/* 0xC0-0xC7 */ + 0x5C24,0x7531,0x90AE,0x94C0,0x72B9,0x6CB9,0x6E38,0x9149,/* 0xC8-0xCF */ + 0x6709,0x53CB,0x53F3,0x4F51,0x91C9,0x8BF1,0x53C8,0x5E7C,/* 0xD0-0xD7 */ + 0x8FC2,0x6DE4,0x4E8E,0x76C2,0x6986,0x865E,0x611A,0x8206,/* 0xD8-0xDF */ + 0x4F59,0x4FDE,0x903E,0x9C7C,0x6109,0x6E1D,0x6E14,0x9685,/* 0xE0-0xE7 */ + 0x4E88,0x5A31,0x96E8,0x4E0E,0x5C7F,0x79B9,0x5B87,0x8BED,/* 0xE8-0xEF */ + 0x7FBD,0x7389,0x57DF,0x828B,0x90C1,0x5401,0x9047,0x55BB,/* 0xF0-0xF7 */ + 0x5CEA,0x5FA1,0x6108,0x6B32,0x72F1,0x80B2,0x8A89,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D4[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8A1E,0x8A1F,0x8A20,0x8A21,0x8A22,0x8A23,0x8A24,0x8A25,/* 0x40-0x47 */ + 0x8A26,0x8A27,0x8A28,0x8A29,0x8A2A,0x8A2B,0x8A2C,0x8A2D,/* 0x48-0x4F */ + 0x8A2E,0x8A2F,0x8A30,0x8A31,0x8A32,0x8A33,0x8A34,0x8A35,/* 0x50-0x57 */ + 0x8A36,0x8A37,0x8A38,0x8A39,0x8A3A,0x8A3B,0x8A3C,0x8A3D,/* 0x58-0x5F */ + 0x8A3F,0x8A40,0x8A41,0x8A42,0x8A43,0x8A44,0x8A45,0x8A46,/* 0x60-0x67 */ + 0x8A47,0x8A49,0x8A4A,0x8A4B,0x8A4C,0x8A4D,0x8A4E,0x8A4F,/* 0x68-0x6F */ + 0x8A50,0x8A51,0x8A52,0x8A53,0x8A54,0x8A55,0x8A56,0x8A57,/* 0x70-0x77 */ + 0x8A58,0x8A59,0x8A5A,0x8A5B,0x8A5C,0x8A5D,0x8A5E,0x0000,/* 0x78-0x7F */ + + 0x8A5F,0x8A60,0x8A61,0x8A62,0x8A63,0x8A64,0x8A65,0x8A66,/* 0x80-0x87 */ + 0x8A67,0x8A68,0x8A69,0x8A6A,0x8A6B,0x8A6C,0x8A6D,0x8A6E,/* 0x88-0x8F */ + 0x8A6F,0x8A70,0x8A71,0x8A72,0x8A73,0x8A74,0x8A75,0x8A76,/* 0x90-0x97 */ + 0x8A77,0x8A78,0x8A7A,0x8A7B,0x8A7C,0x8A7D,0x8A7E,0x8A7F,/* 0x98-0x9F */ + 0x8A80,0x6D74,0x5BD3,0x88D5,0x9884,0x8C6B,0x9A6D,0x9E33,/* 0xA0-0xA7 */ + 0x6E0A,0x51A4,0x5143,0x57A3,0x8881,0x539F,0x63F4,0x8F95,/* 0xA8-0xAF */ + 0x56ED,0x5458,0x5706,0x733F,0x6E90,0x7F18,0x8FDC,0x82D1,/* 0xB0-0xB7 */ + 0x613F,0x6028,0x9662,0x66F0,0x7EA6,0x8D8A,0x8DC3,0x94A5,/* 0xB8-0xBF */ + 0x5CB3,0x7CA4,0x6708,0x60A6,0x9605,0x8018,0x4E91,0x90E7,/* 0xC0-0xC7 */ + 0x5300,0x9668,0x5141,0x8FD0,0x8574,0x915D,0x6655,0x97F5,/* 0xC8-0xCF */ + 0x5B55,0x531D,0x7838,0x6742,0x683D,0x54C9,0x707E,0x5BB0,/* 0xD0-0xD7 */ + 0x8F7D,0x518D,0x5728,0x54B1,0x6512,0x6682,0x8D5E,0x8D43,/* 0xD8-0xDF */ + 0x810F,0x846C,0x906D,0x7CDF,0x51FF,0x85FB,0x67A3,0x65E9,/* 0xE0-0xE7 */ + 0x6FA1,0x86A4,0x8E81,0x566A,0x9020,0x7682,0x7076,0x71E5,/* 0xE8-0xEF */ + 0x8D23,0x62E9,0x5219,0x6CFD,0x8D3C,0x600E,0x589E,0x618E,/* 0xF0-0xF7 */ + 0x66FE,0x8D60,0x624E,0x55B3,0x6E23,0x672D,0x8F67,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D5[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8A81,0x8A82,0x8A83,0x8A84,0x8A85,0x8A86,0x8A87,0x8A88,/* 0x40-0x47 */ + 0x8A8B,0x8A8C,0x8A8D,0x8A8E,0x8A8F,0x8A90,0x8A91,0x8A92,/* 0x48-0x4F */ + 0x8A94,0x8A95,0x8A96,0x8A97,0x8A98,0x8A99,0x8A9A,0x8A9B,/* 0x50-0x57 */ + 0x8A9C,0x8A9D,0x8A9E,0x8A9F,0x8AA0,0x8AA1,0x8AA2,0x8AA3,/* 0x58-0x5F */ + 0x8AA4,0x8AA5,0x8AA6,0x8AA7,0x8AA8,0x8AA9,0x8AAA,0x8AAB,/* 0x60-0x67 */ + 0x8AAC,0x8AAD,0x8AAE,0x8AAF,0x8AB0,0x8AB1,0x8AB2,0x8AB3,/* 0x68-0x6F */ + 0x8AB4,0x8AB5,0x8AB6,0x8AB7,0x8AB8,0x8AB9,0x8ABA,0x8ABB,/* 0x70-0x77 */ + 0x8ABC,0x8ABD,0x8ABE,0x8ABF,0x8AC0,0x8AC1,0x8AC2,0x0000,/* 0x78-0x7F */ + + 0x8AC3,0x8AC4,0x8AC5,0x8AC6,0x8AC7,0x8AC8,0x8AC9,0x8ACA,/* 0x80-0x87 */ + 0x8ACB,0x8ACC,0x8ACD,0x8ACE,0x8ACF,0x8AD0,0x8AD1,0x8AD2,/* 0x88-0x8F */ + 0x8AD3,0x8AD4,0x8AD5,0x8AD6,0x8AD7,0x8AD8,0x8AD9,0x8ADA,/* 0x90-0x97 */ + 0x8ADB,0x8ADC,0x8ADD,0x8ADE,0x8ADF,0x8AE0,0x8AE1,0x8AE2,/* 0x98-0x9F */ + 0x8AE3,0x94E1,0x95F8,0x7728,0x6805,0x69A8,0x548B,0x4E4D,/* 0xA0-0xA7 */ + 0x70B8,0x8BC8,0x6458,0x658B,0x5B85,0x7A84,0x503A,0x5BE8,/* 0xA8-0xAF */ + 0x77BB,0x6BE1,0x8A79,0x7C98,0x6CBE,0x76CF,0x65A9,0x8F97,/* 0xB0-0xB7 */ + 0x5D2D,0x5C55,0x8638,0x6808,0x5360,0x6218,0x7AD9,0x6E5B,/* 0xB8-0xBF */ + 0x7EFD,0x6A1F,0x7AE0,0x5F70,0x6F33,0x5F20,0x638C,0x6DA8,/* 0xC0-0xC7 */ + 0x6756,0x4E08,0x5E10,0x8D26,0x4ED7,0x80C0,0x7634,0x969C,/* 0xC8-0xCF */ + 0x62DB,0x662D,0x627E,0x6CBC,0x8D75,0x7167,0x7F69,0x5146,/* 0xD0-0xD7 */ + 0x8087,0x53EC,0x906E,0x6298,0x54F2,0x86F0,0x8F99,0x8005,/* 0xD8-0xDF */ + 0x9517,0x8517,0x8FD9,0x6D59,0x73CD,0x659F,0x771F,0x7504,/* 0xE0-0xE7 */ + 0x7827,0x81FB,0x8D1E,0x9488,0x4FA6,0x6795,0x75B9,0x8BCA,/* 0xE8-0xEF */ + 0x9707,0x632F,0x9547,0x9635,0x84B8,0x6323,0x7741,0x5F81,/* 0xF0-0xF7 */ + 0x72F0,0x4E89,0x6014,0x6574,0x62EF,0x6B63,0x653F,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D6[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8AE4,0x8AE5,0x8AE6,0x8AE7,0x8AE8,0x8AE9,0x8AEA,0x8AEB,/* 0x40-0x47 */ + 0x8AEC,0x8AED,0x8AEE,0x8AEF,0x8AF0,0x8AF1,0x8AF2,0x8AF3,/* 0x48-0x4F */ + 0x8AF4,0x8AF5,0x8AF6,0x8AF7,0x8AF8,0x8AF9,0x8AFA,0x8AFB,/* 0x50-0x57 */ + 0x8AFC,0x8AFD,0x8AFE,0x8AFF,0x8B00,0x8B01,0x8B02,0x8B03,/* 0x58-0x5F */ + 0x8B04,0x8B05,0x8B06,0x8B08,0x8B09,0x8B0A,0x8B0B,0x8B0C,/* 0x60-0x67 */ + 0x8B0D,0x8B0E,0x8B0F,0x8B10,0x8B11,0x8B12,0x8B13,0x8B14,/* 0x68-0x6F */ + 0x8B15,0x8B16,0x8B17,0x8B18,0x8B19,0x8B1A,0x8B1B,0x8B1C,/* 0x70-0x77 */ + 0x8B1D,0x8B1E,0x8B1F,0x8B20,0x8B21,0x8B22,0x8B23,0x0000,/* 0x78-0x7F */ + + 0x8B24,0x8B25,0x8B27,0x8B28,0x8B29,0x8B2A,0x8B2B,0x8B2C,/* 0x80-0x87 */ + 0x8B2D,0x8B2E,0x8B2F,0x8B30,0x8B31,0x8B32,0x8B33,0x8B34,/* 0x88-0x8F */ + 0x8B35,0x8B36,0x8B37,0x8B38,0x8B39,0x8B3A,0x8B3B,0x8B3C,/* 0x90-0x97 */ + 0x8B3D,0x8B3E,0x8B3F,0x8B40,0x8B41,0x8B42,0x8B43,0x8B44,/* 0x98-0x9F */ + 0x8B45,0x5E27,0x75C7,0x90D1,0x8BC1,0x829D,0x679D,0x652F,/* 0xA0-0xA7 */ + 0x5431,0x8718,0x77E5,0x80A2,0x8102,0x6C41,0x4E4B,0x7EC7,/* 0xA8-0xAF */ + 0x804C,0x76F4,0x690D,0x6B96,0x6267,0x503C,0x4F84,0x5740,/* 0xB0-0xB7 */ + 0x6307,0x6B62,0x8DBE,0x53EA,0x65E8,0x7EB8,0x5FD7,0x631A,/* 0xB8-0xBF */ + 0x63B7,0x81F3,0x81F4,0x7F6E,0x5E1C,0x5CD9,0x5236,0x667A,/* 0xC0-0xC7 */ + 0x79E9,0x7A1A,0x8D28,0x7099,0x75D4,0x6EDE,0x6CBB,0x7A92,/* 0xC8-0xCF */ + 0x4E2D,0x76C5,0x5FE0,0x949F,0x8877,0x7EC8,0x79CD,0x80BF,/* 0xD0-0xD7 */ + 0x91CD,0x4EF2,0x4F17,0x821F,0x5468,0x5DDE,0x6D32,0x8BCC,/* 0xD8-0xDF */ + 0x7CA5,0x8F74,0x8098,0x5E1A,0x5492,0x76B1,0x5B99,0x663C,/* 0xE0-0xE7 */ + 0x9AA4,0x73E0,0x682A,0x86DB,0x6731,0x732A,0x8BF8,0x8BDB,/* 0xE8-0xEF */ + 0x9010,0x7AF9,0x70DB,0x716E,0x62C4,0x77A9,0x5631,0x4E3B,/* 0xF0-0xF7 */ + 0x8457,0x67F1,0x52A9,0x86C0,0x8D2E,0x94F8,0x7B51,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D7[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8B46,0x8B47,0x8B48,0x8B49,0x8B4A,0x8B4B,0x8B4C,0x8B4D,/* 0x40-0x47 */ + 0x8B4E,0x8B4F,0x8B50,0x8B51,0x8B52,0x8B53,0x8B54,0x8B55,/* 0x48-0x4F */ + 0x8B56,0x8B57,0x8B58,0x8B59,0x8B5A,0x8B5B,0x8B5C,0x8B5D,/* 0x50-0x57 */ + 0x8B5E,0x8B5F,0x8B60,0x8B61,0x8B62,0x8B63,0x8B64,0x8B65,/* 0x58-0x5F */ + 0x8B67,0x8B68,0x8B69,0x8B6A,0x8B6B,0x8B6D,0x8B6E,0x8B6F,/* 0x60-0x67 */ + 0x8B70,0x8B71,0x8B72,0x8B73,0x8B74,0x8B75,0x8B76,0x8B77,/* 0x68-0x6F */ + 0x8B78,0x8B79,0x8B7A,0x8B7B,0x8B7C,0x8B7D,0x8B7E,0x8B7F,/* 0x70-0x77 */ + 0x8B80,0x8B81,0x8B82,0x8B83,0x8B84,0x8B85,0x8B86,0x0000,/* 0x78-0x7F */ + + 0x8B87,0x8B88,0x8B89,0x8B8A,0x8B8B,0x8B8C,0x8B8D,0x8B8E,/* 0x80-0x87 */ + 0x8B8F,0x8B90,0x8B91,0x8B92,0x8B93,0x8B94,0x8B95,0x8B96,/* 0x88-0x8F */ + 0x8B97,0x8B98,0x8B99,0x8B9A,0x8B9B,0x8B9C,0x8B9D,0x8B9E,/* 0x90-0x97 */ + 0x8B9F,0x8BAC,0x8BB1,0x8BBB,0x8BC7,0x8BD0,0x8BEA,0x8C09,/* 0x98-0x9F */ + 0x8C1E,0x4F4F,0x6CE8,0x795D,0x9A7B,0x6293,0x722A,0x62FD,/* 0xA0-0xA7 */ + 0x4E13,0x7816,0x8F6C,0x64B0,0x8D5A,0x7BC6,0x6869,0x5E84,/* 0xA8-0xAF */ + 0x88C5,0x5986,0x649E,0x58EE,0x72B6,0x690E,0x9525,0x8FFD,/* 0xB0-0xB7 */ + 0x8D58,0x5760,0x7F00,0x8C06,0x51C6,0x6349,0x62D9,0x5353,/* 0xB8-0xBF */ + 0x684C,0x7422,0x8301,0x914C,0x5544,0x7740,0x707C,0x6D4A,/* 0xC0-0xC7 */ + 0x5179,0x54A8,0x8D44,0x59FF,0x6ECB,0x6DC4,0x5B5C,0x7D2B,/* 0xC8-0xCF */ + 0x4ED4,0x7C7D,0x6ED3,0x5B50,0x81EA,0x6E0D,0x5B57,0x9B03,/* 0xD0-0xD7 */ + 0x68D5,0x8E2A,0x5B97,0x7EFC,0x603B,0x7EB5,0x90B9,0x8D70,/* 0xD8-0xDF */ + 0x594F,0x63CD,0x79DF,0x8DB3,0x5352,0x65CF,0x7956,0x8BC5,/* 0xE0-0xE7 */ + 0x963B,0x7EC4,0x94BB,0x7E82,0x5634,0x9189,0x6700,0x7F6A,/* 0xE8-0xEF */ + 0x5C0A,0x9075,0x6628,0x5DE6,0x4F50,0x67DE,0x505A,0x4F5C,/* 0xF0-0xF7 */ + 0x5750,0x5EA7,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D8[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8C38,0x8C39,0x8C3A,0x8C3B,0x8C3C,0x8C3D,0x8C3E,0x8C3F,/* 0x40-0x47 */ + 0x8C40,0x8C42,0x8C43,0x8C44,0x8C45,0x8C48,0x8C4A,0x8C4B,/* 0x48-0x4F */ + 0x8C4D,0x8C4E,0x8C4F,0x8C50,0x8C51,0x8C52,0x8C53,0x8C54,/* 0x50-0x57 */ + 0x8C56,0x8C57,0x8C58,0x8C59,0x8C5B,0x8C5C,0x8C5D,0x8C5E,/* 0x58-0x5F */ + 0x8C5F,0x8C60,0x8C63,0x8C64,0x8C65,0x8C66,0x8C67,0x8C68,/* 0x60-0x67 */ + 0x8C69,0x8C6C,0x8C6D,0x8C6E,0x8C6F,0x8C70,0x8C71,0x8C72,/* 0x68-0x6F */ + 0x8C74,0x8C75,0x8C76,0x8C77,0x8C7B,0x8C7C,0x8C7D,0x8C7E,/* 0x70-0x77 */ + 0x8C7F,0x8C80,0x8C81,0x8C83,0x8C84,0x8C86,0x8C87,0x0000,/* 0x78-0x7F */ + + 0x8C88,0x8C8B,0x8C8D,0x8C8E,0x8C8F,0x8C90,0x8C91,0x8C92,/* 0x80-0x87 */ + 0x8C93,0x8C95,0x8C96,0x8C97,0x8C99,0x8C9A,0x8C9B,0x8C9C,/* 0x88-0x8F */ + 0x8C9D,0x8C9E,0x8C9F,0x8CA0,0x8CA1,0x8CA2,0x8CA3,0x8CA4,/* 0x90-0x97 */ + 0x8CA5,0x8CA6,0x8CA7,0x8CA8,0x8CA9,0x8CAA,0x8CAB,0x8CAC,/* 0x98-0x9F */ + 0x8CAD,0x4E8D,0x4E0C,0x5140,0x4E10,0x5EFF,0x5345,0x4E15,/* 0xA0-0xA7 */ + 0x4E98,0x4E1E,0x9B32,0x5B6C,0x5669,0x4E28,0x79BA,0x4E3F,/* 0xA8-0xAF */ + 0x5315,0x4E47,0x592D,0x723B,0x536E,0x6C10,0x56DF,0x80E4,/* 0xB0-0xB7 */ + 0x9997,0x6BD3,0x777E,0x9F17,0x4E36,0x4E9F,0x9F10,0x4E5C,/* 0xB8-0xBF */ + 0x4E69,0x4E93,0x8288,0x5B5B,0x556C,0x560F,0x4EC4,0x538D,/* 0xC0-0xC7 */ + 0x539D,0x53A3,0x53A5,0x53AE,0x9765,0x8D5D,0x531A,0x53F5,/* 0xC8-0xCF */ + 0x5326,0x532E,0x533E,0x8D5C,0x5366,0x5363,0x5202,0x5208,/* 0xD0-0xD7 */ + 0x520E,0x522D,0x5233,0x523F,0x5240,0x524C,0x525E,0x5261,/* 0xD8-0xDF */ + 0x525C,0x84AF,0x527D,0x5282,0x5281,0x5290,0x5293,0x5182,/* 0xE0-0xE7 */ + 0x7F54,0x4EBB,0x4EC3,0x4EC9,0x4EC2,0x4EE8,0x4EE1,0x4EEB,/* 0xE8-0xEF */ + 0x4EDE,0x4F1B,0x4EF3,0x4F22,0x4F64,0x4EF5,0x4F25,0x4F27,/* 0xF0-0xF7 */ + 0x4F09,0x4F2B,0x4F5E,0x4F67,0x6538,0x4F5A,0x4F5D,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D9[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8CAE,0x8CAF,0x8CB0,0x8CB1,0x8CB2,0x8CB3,0x8CB4,0x8CB5,/* 0x40-0x47 */ + 0x8CB6,0x8CB7,0x8CB8,0x8CB9,0x8CBA,0x8CBB,0x8CBC,0x8CBD,/* 0x48-0x4F */ + 0x8CBE,0x8CBF,0x8CC0,0x8CC1,0x8CC2,0x8CC3,0x8CC4,0x8CC5,/* 0x50-0x57 */ + 0x8CC6,0x8CC7,0x8CC8,0x8CC9,0x8CCA,0x8CCB,0x8CCC,0x8CCD,/* 0x58-0x5F */ + 0x8CCE,0x8CCF,0x8CD0,0x8CD1,0x8CD2,0x8CD3,0x8CD4,0x8CD5,/* 0x60-0x67 */ + 0x8CD6,0x8CD7,0x8CD8,0x8CD9,0x8CDA,0x8CDB,0x8CDC,0x8CDD,/* 0x68-0x6F */ + 0x8CDE,0x8CDF,0x8CE0,0x8CE1,0x8CE2,0x8CE3,0x8CE4,0x8CE5,/* 0x70-0x77 */ + 0x8CE6,0x8CE7,0x8CE8,0x8CE9,0x8CEA,0x8CEB,0x8CEC,0x0000,/* 0x78-0x7F */ + + 0x8CED,0x8CEE,0x8CEF,0x8CF0,0x8CF1,0x8CF2,0x8CF3,0x8CF4,/* 0x80-0x87 */ + 0x8CF5,0x8CF6,0x8CF7,0x8CF8,0x8CF9,0x8CFA,0x8CFB,0x8CFC,/* 0x88-0x8F */ + 0x8CFD,0x8CFE,0x8CFF,0x8D00,0x8D01,0x8D02,0x8D03,0x8D04,/* 0x90-0x97 */ + 0x8D05,0x8D06,0x8D07,0x8D08,0x8D09,0x8D0A,0x8D0B,0x8D0C,/* 0x98-0x9F */ + 0x8D0D,0x4F5F,0x4F57,0x4F32,0x4F3D,0x4F76,0x4F74,0x4F91,/* 0xA0-0xA7 */ + 0x4F89,0x4F83,0x4F8F,0x4F7E,0x4F7B,0x4FAA,0x4F7C,0x4FAC,/* 0xA8-0xAF */ + 0x4F94,0x4FE6,0x4FE8,0x4FEA,0x4FC5,0x4FDA,0x4FE3,0x4FDC,/* 0xB0-0xB7 */ + 0x4FD1,0x4FDF,0x4FF8,0x5029,0x504C,0x4FF3,0x502C,0x500F,/* 0xB8-0xBF */ + 0x502E,0x502D,0x4FFE,0x501C,0x500C,0x5025,0x5028,0x507E,/* 0xC0-0xC7 */ + 0x5043,0x5055,0x5048,0x504E,0x506C,0x507B,0x50A5,0x50A7,/* 0xC8-0xCF */ + 0x50A9,0x50BA,0x50D6,0x5106,0x50ED,0x50EC,0x50E6,0x50EE,/* 0xD0-0xD7 */ + 0x5107,0x510B,0x4EDD,0x6C3D,0x4F58,0x4F65,0x4FCE,0x9FA0,/* 0xD8-0xDF */ + 0x6C46,0x7C74,0x516E,0x5DFD,0x9EC9,0x9998,0x5181,0x5914,/* 0xE0-0xE7 */ + 0x52F9,0x530D,0x8A07,0x5310,0x51EB,0x5919,0x5155,0x4EA0,/* 0xE8-0xEF */ + 0x5156,0x4EB3,0x886E,0x88A4,0x4EB5,0x8114,0x88D2,0x7980,/* 0xF0-0xF7 */ + 0x5B34,0x8803,0x7FB8,0x51AB,0x51B1,0x51BD,0x51BC,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_DA[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8D0E,0x8D0F,0x8D10,0x8D11,0x8D12,0x8D13,0x8D14,0x8D15,/* 0x40-0x47 */ + 0x8D16,0x8D17,0x8D18,0x8D19,0x8D1A,0x8D1B,0x8D1C,0x8D20,/* 0x48-0x4F */ + 0x8D51,0x8D52,0x8D57,0x8D5F,0x8D65,0x8D68,0x8D69,0x8D6A,/* 0x50-0x57 */ + 0x8D6C,0x8D6E,0x8D6F,0x8D71,0x8D72,0x8D78,0x8D79,0x8D7A,/* 0x58-0x5F */ + 0x8D7B,0x8D7C,0x8D7D,0x8D7E,0x8D7F,0x8D80,0x8D82,0x8D83,/* 0x60-0x67 */ + 0x8D86,0x8D87,0x8D88,0x8D89,0x8D8C,0x8D8D,0x8D8E,0x8D8F,/* 0x68-0x6F */ + 0x8D90,0x8D92,0x8D93,0x8D95,0x8D96,0x8D97,0x8D98,0x8D99,/* 0x70-0x77 */ + 0x8D9A,0x8D9B,0x8D9C,0x8D9D,0x8D9E,0x8DA0,0x8DA1,0x0000,/* 0x78-0x7F */ + + 0x8DA2,0x8DA4,0x8DA5,0x8DA6,0x8DA7,0x8DA8,0x8DA9,0x8DAA,/* 0x80-0x87 */ + 0x8DAB,0x8DAC,0x8DAD,0x8DAE,0x8DAF,0x8DB0,0x8DB2,0x8DB6,/* 0x88-0x8F */ + 0x8DB7,0x8DB9,0x8DBB,0x8DBD,0x8DC0,0x8DC1,0x8DC2,0x8DC5,/* 0x90-0x97 */ + 0x8DC7,0x8DC8,0x8DC9,0x8DCA,0x8DCD,0x8DD0,0x8DD2,0x8DD3,/* 0x98-0x9F */ + 0x8DD4,0x51C7,0x5196,0x51A2,0x51A5,0x8BA0,0x8BA6,0x8BA7,/* 0xA0-0xA7 */ + 0x8BAA,0x8BB4,0x8BB5,0x8BB7,0x8BC2,0x8BC3,0x8BCB,0x8BCF,/* 0xA8-0xAF */ + 0x8BCE,0x8BD2,0x8BD3,0x8BD4,0x8BD6,0x8BD8,0x8BD9,0x8BDC,/* 0xB0-0xB7 */ + 0x8BDF,0x8BE0,0x8BE4,0x8BE8,0x8BE9,0x8BEE,0x8BF0,0x8BF3,/* 0xB8-0xBF */ + 0x8BF6,0x8BF9,0x8BFC,0x8BFF,0x8C00,0x8C02,0x8C04,0x8C07,/* 0xC0-0xC7 */ + 0x8C0C,0x8C0F,0x8C11,0x8C12,0x8C14,0x8C15,0x8C16,0x8C19,/* 0xC8-0xCF */ + 0x8C1B,0x8C18,0x8C1D,0x8C1F,0x8C20,0x8C21,0x8C25,0x8C27,/* 0xD0-0xD7 */ + 0x8C2A,0x8C2B,0x8C2E,0x8C2F,0x8C32,0x8C33,0x8C35,0x8C36,/* 0xD8-0xDF */ + 0x5369,0x537A,0x961D,0x9622,0x9621,0x9631,0x962A,0x963D,/* 0xE0-0xE7 */ + 0x963C,0x9642,0x9649,0x9654,0x965F,0x9667,0x966C,0x9672,/* 0xE8-0xEF */ + 0x9674,0x9688,0x968D,0x9697,0x96B0,0x9097,0x909B,0x909D,/* 0xF0-0xF7 */ + 0x9099,0x90AC,0x90A1,0x90B4,0x90B3,0x90B6,0x90BA,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_DB[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8DD5,0x8DD8,0x8DD9,0x8DDC,0x8DE0,0x8DE1,0x8DE2,0x8DE5,/* 0x40-0x47 */ + 0x8DE6,0x8DE7,0x8DE9,0x8DED,0x8DEE,0x8DF0,0x8DF1,0x8DF2,/* 0x48-0x4F */ + 0x8DF4,0x8DF6,0x8DFC,0x8DFE,0x8DFF,0x8E00,0x8E01,0x8E02,/* 0x50-0x57 */ + 0x8E03,0x8E04,0x8E06,0x8E07,0x8E08,0x8E0B,0x8E0D,0x8E0E,/* 0x58-0x5F */ + 0x8E10,0x8E11,0x8E12,0x8E13,0x8E15,0x8E16,0x8E17,0x8E18,/* 0x60-0x67 */ + 0x8E19,0x8E1A,0x8E1B,0x8E1C,0x8E20,0x8E21,0x8E24,0x8E25,/* 0x68-0x6F */ + 0x8E26,0x8E27,0x8E28,0x8E2B,0x8E2D,0x8E30,0x8E32,0x8E33,/* 0x70-0x77 */ + 0x8E34,0x8E36,0x8E37,0x8E38,0x8E3B,0x8E3C,0x8E3E,0x0000,/* 0x78-0x7F */ + + 0x8E3F,0x8E43,0x8E45,0x8E46,0x8E4C,0x8E4D,0x8E4E,0x8E4F,/* 0x80-0x87 */ + 0x8E50,0x8E53,0x8E54,0x8E55,0x8E56,0x8E57,0x8E58,0x8E5A,/* 0x88-0x8F */ + 0x8E5B,0x8E5C,0x8E5D,0x8E5E,0x8E5F,0x8E60,0x8E61,0x8E62,/* 0x90-0x97 */ + 0x8E63,0x8E64,0x8E65,0x8E67,0x8E68,0x8E6A,0x8E6B,0x8E6E,/* 0x98-0x9F */ + 0x8E71,0x90B8,0x90B0,0x90CF,0x90C5,0x90BE,0x90D0,0x90C4,/* 0xA0-0xA7 */ + 0x90C7,0x90D3,0x90E6,0x90E2,0x90DC,0x90D7,0x90DB,0x90EB,/* 0xA8-0xAF */ + 0x90EF,0x90FE,0x9104,0x9122,0x911E,0x9123,0x9131,0x912F,/* 0xB0-0xB7 */ + 0x9139,0x9143,0x9146,0x520D,0x5942,0x52A2,0x52AC,0x52AD,/* 0xB8-0xBF */ + 0x52BE,0x54FF,0x52D0,0x52D6,0x52F0,0x53DF,0x71EE,0x77CD,/* 0xC0-0xC7 */ + 0x5EF4,0x51F5,0x51FC,0x9B2F,0x53B6,0x5F01,0x755A,0x5DEF,/* 0xC8-0xCF */ + 0x574C,0x57A9,0x57A1,0x587E,0x58BC,0x58C5,0x58D1,0x5729,/* 0xD0-0xD7 */ + 0x572C,0x572A,0x5733,0x5739,0x572E,0x572F,0x575C,0x573B,/* 0xD8-0xDF */ + 0x5742,0x5769,0x5785,0x576B,0x5786,0x577C,0x577B,0x5768,/* 0xE0-0xE7 */ + 0x576D,0x5776,0x5773,0x57AD,0x57A4,0x578C,0x57B2,0x57CF,/* 0xE8-0xEF */ + 0x57A7,0x57B4,0x5793,0x57A0,0x57D5,0x57D8,0x57DA,0x57D9,/* 0xF0-0xF7 */ + 0x57D2,0x57B8,0x57F4,0x57EF,0x57F8,0x57E4,0x57DD,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_DC[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8E73,0x8E75,0x8E77,0x8E78,0x8E79,0x8E7A,0x8E7B,0x8E7D,/* 0x40-0x47 */ + 0x8E7E,0x8E80,0x8E82,0x8E83,0x8E84,0x8E86,0x8E88,0x8E89,/* 0x48-0x4F */ + 0x8E8A,0x8E8B,0x8E8C,0x8E8D,0x8E8E,0x8E91,0x8E92,0x8E93,/* 0x50-0x57 */ + 0x8E95,0x8E96,0x8E97,0x8E98,0x8E99,0x8E9A,0x8E9B,0x8E9D,/* 0x58-0x5F */ + 0x8E9F,0x8EA0,0x8EA1,0x8EA2,0x8EA3,0x8EA4,0x8EA5,0x8EA6,/* 0x60-0x67 */ + 0x8EA7,0x8EA8,0x8EA9,0x8EAA,0x8EAD,0x8EAE,0x8EB0,0x8EB1,/* 0x68-0x6F */ + 0x8EB3,0x8EB4,0x8EB5,0x8EB6,0x8EB7,0x8EB8,0x8EB9,0x8EBB,/* 0x70-0x77 */ + 0x8EBC,0x8EBD,0x8EBE,0x8EBF,0x8EC0,0x8EC1,0x8EC2,0x0000,/* 0x78-0x7F */ + + 0x8EC3,0x8EC4,0x8EC5,0x8EC6,0x8EC7,0x8EC8,0x8EC9,0x8ECA,/* 0x80-0x87 */ + 0x8ECB,0x8ECC,0x8ECD,0x8ECF,0x8ED0,0x8ED1,0x8ED2,0x8ED3,/* 0x88-0x8F */ + 0x8ED4,0x8ED5,0x8ED6,0x8ED7,0x8ED8,0x8ED9,0x8EDA,0x8EDB,/* 0x90-0x97 */ + 0x8EDC,0x8EDD,0x8EDE,0x8EDF,0x8EE0,0x8EE1,0x8EE2,0x8EE3,/* 0x98-0x9F */ + 0x8EE4,0x580B,0x580D,0x57FD,0x57ED,0x5800,0x581E,0x5819,/* 0xA0-0xA7 */ + 0x5844,0x5820,0x5865,0x586C,0x5881,0x5889,0x589A,0x5880,/* 0xA8-0xAF */ + 0x99A8,0x9F19,0x61FF,0x8279,0x827D,0x827F,0x828F,0x828A,/* 0xB0-0xB7 */ + 0x82A8,0x8284,0x828E,0x8291,0x8297,0x8299,0x82AB,0x82B8,/* 0xB8-0xBF */ + 0x82BE,0x82B0,0x82C8,0x82CA,0x82E3,0x8298,0x82B7,0x82AE,/* 0xC0-0xC7 */ + 0x82CB,0x82CC,0x82C1,0x82A9,0x82B4,0x82A1,0x82AA,0x829F,/* 0xC8-0xCF */ + 0x82C4,0x82CE,0x82A4,0x82E1,0x8309,0x82F7,0x82E4,0x830F,/* 0xD0-0xD7 */ + 0x8307,0x82DC,0x82F4,0x82D2,0x82D8,0x830C,0x82FB,0x82D3,/* 0xD8-0xDF */ + 0x8311,0x831A,0x8306,0x8314,0x8315,0x82E0,0x82D5,0x831C,/* 0xE0-0xE7 */ + 0x8351,0x835B,0x835C,0x8308,0x8392,0x833C,0x8334,0x8331,/* 0xE8-0xEF */ + 0x839B,0x835E,0x832F,0x834F,0x8347,0x8343,0x835F,0x8340,/* 0xF0-0xF7 */ + 0x8317,0x8360,0x832D,0x833A,0x8333,0x8366,0x8365,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_DD[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8EE5,0x8EE6,0x8EE7,0x8EE8,0x8EE9,0x8EEA,0x8EEB,0x8EEC,/* 0x40-0x47 */ + 0x8EED,0x8EEE,0x8EEF,0x8EF0,0x8EF1,0x8EF2,0x8EF3,0x8EF4,/* 0x48-0x4F */ + 0x8EF5,0x8EF6,0x8EF7,0x8EF8,0x8EF9,0x8EFA,0x8EFB,0x8EFC,/* 0x50-0x57 */ + 0x8EFD,0x8EFE,0x8EFF,0x8F00,0x8F01,0x8F02,0x8F03,0x8F04,/* 0x58-0x5F */ + 0x8F05,0x8F06,0x8F07,0x8F08,0x8F09,0x8F0A,0x8F0B,0x8F0C,/* 0x60-0x67 */ + 0x8F0D,0x8F0E,0x8F0F,0x8F10,0x8F11,0x8F12,0x8F13,0x8F14,/* 0x68-0x6F */ + 0x8F15,0x8F16,0x8F17,0x8F18,0x8F19,0x8F1A,0x8F1B,0x8F1C,/* 0x70-0x77 */ + 0x8F1D,0x8F1E,0x8F1F,0x8F20,0x8F21,0x8F22,0x8F23,0x0000,/* 0x78-0x7F */ + + 0x8F24,0x8F25,0x8F26,0x8F27,0x8F28,0x8F29,0x8F2A,0x8F2B,/* 0x80-0x87 */ + 0x8F2C,0x8F2D,0x8F2E,0x8F2F,0x8F30,0x8F31,0x8F32,0x8F33,/* 0x88-0x8F */ + 0x8F34,0x8F35,0x8F36,0x8F37,0x8F38,0x8F39,0x8F3A,0x8F3B,/* 0x90-0x97 */ + 0x8F3C,0x8F3D,0x8F3E,0x8F3F,0x8F40,0x8F41,0x8F42,0x8F43,/* 0x98-0x9F */ + 0x8F44,0x8368,0x831B,0x8369,0x836C,0x836A,0x836D,0x836E,/* 0xA0-0xA7 */ + 0x83B0,0x8378,0x83B3,0x83B4,0x83A0,0x83AA,0x8393,0x839C,/* 0xA8-0xAF */ + 0x8385,0x837C,0x83B6,0x83A9,0x837D,0x83B8,0x837B,0x8398,/* 0xB0-0xB7 */ + 0x839E,0x83A8,0x83BA,0x83BC,0x83C1,0x8401,0x83E5,0x83D8,/* 0xB8-0xBF */ + 0x5807,0x8418,0x840B,0x83DD,0x83FD,0x83D6,0x841C,0x8438,/* 0xC0-0xC7 */ + 0x8411,0x8406,0x83D4,0x83DF,0x840F,0x8403,0x83F8,0x83F9,/* 0xC8-0xCF */ + 0x83EA,0x83C5,0x83C0,0x8426,0x83F0,0x83E1,0x845C,0x8451,/* 0xD0-0xD7 */ + 0x845A,0x8459,0x8473,0x8487,0x8488,0x847A,0x8489,0x8478,/* 0xD8-0xDF */ + 0x843C,0x8446,0x8469,0x8476,0x848C,0x848E,0x8431,0x846D,/* 0xE0-0xE7 */ + 0x84C1,0x84CD,0x84D0,0x84E6,0x84BD,0x84D3,0x84CA,0x84BF,/* 0xE8-0xEF */ + 0x84BA,0x84E0,0x84A1,0x84B9,0x84B4,0x8497,0x84E5,0x84E3,/* 0xF0-0xF7 */ + 0x850C,0x750D,0x8538,0x84F0,0x8539,0x851F,0x853A,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_DE[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8F45,0x8F46,0x8F47,0x8F48,0x8F49,0x8F4A,0x8F4B,0x8F4C,/* 0x40-0x47 */ + 0x8F4D,0x8F4E,0x8F4F,0x8F50,0x8F51,0x8F52,0x8F53,0x8F54,/* 0x48-0x4F */ + 0x8F55,0x8F56,0x8F57,0x8F58,0x8F59,0x8F5A,0x8F5B,0x8F5C,/* 0x50-0x57 */ + 0x8F5D,0x8F5E,0x8F5F,0x8F60,0x8F61,0x8F62,0x8F63,0x8F64,/* 0x58-0x5F */ + 0x8F65,0x8F6A,0x8F80,0x8F8C,0x8F92,0x8F9D,0x8FA0,0x8FA1,/* 0x60-0x67 */ + 0x8FA2,0x8FA4,0x8FA5,0x8FA6,0x8FA7,0x8FAA,0x8FAC,0x8FAD,/* 0x68-0x6F */ + 0x8FAE,0x8FAF,0x8FB2,0x8FB3,0x8FB4,0x8FB5,0x8FB7,0x8FB8,/* 0x70-0x77 */ + 0x8FBA,0x8FBB,0x8FBC,0x8FBF,0x8FC0,0x8FC3,0x8FC6,0x0000,/* 0x78-0x7F */ + + 0x8FC9,0x8FCA,0x8FCB,0x8FCC,0x8FCD,0x8FCF,0x8FD2,0x8FD6,/* 0x80-0x87 */ + 0x8FD7,0x8FDA,0x8FE0,0x8FE1,0x8FE3,0x8FE7,0x8FEC,0x8FEF,/* 0x88-0x8F */ + 0x8FF1,0x8FF2,0x8FF4,0x8FF5,0x8FF6,0x8FFA,0x8FFB,0x8FFC,/* 0x90-0x97 */ + 0x8FFE,0x8FFF,0x9007,0x9008,0x900C,0x900E,0x9013,0x9015,/* 0x98-0x9F */ + 0x9018,0x8556,0x853B,0x84FF,0x84FC,0x8559,0x8548,0x8568,/* 0xA0-0xA7 */ + 0x8564,0x855E,0x857A,0x77A2,0x8543,0x8572,0x857B,0x85A4,/* 0xA8-0xAF */ + 0x85A8,0x8587,0x858F,0x8579,0x85AE,0x859C,0x8585,0x85B9,/* 0xB0-0xB7 */ + 0x85B7,0x85B0,0x85D3,0x85C1,0x85DC,0x85FF,0x8627,0x8605,/* 0xB8-0xBF */ + 0x8629,0x8616,0x863C,0x5EFE,0x5F08,0x593C,0x5941,0x8037,/* 0xC0-0xC7 */ + 0x5955,0x595A,0x5958,0x530F,0x5C22,0x5C25,0x5C2C,0x5C34,/* 0xC8-0xCF */ + 0x624C,0x626A,0x629F,0x62BB,0x62CA,0x62DA,0x62D7,0x62EE,/* 0xD0-0xD7 */ + 0x6322,0x62F6,0x6339,0x634B,0x6343,0x63AD,0x63F6,0x6371,/* 0xD8-0xDF */ + 0x637A,0x638E,0x63B4,0x636D,0x63AC,0x638A,0x6369,0x63AE,/* 0xE0-0xE7 */ + 0x63BC,0x63F2,0x63F8,0x63E0,0x63FF,0x63C4,0x63DE,0x63CE,/* 0xE8-0xEF */ + 0x6452,0x63C6,0x63BE,0x6445,0x6441,0x640B,0x641B,0x6420,/* 0xF0-0xF7 */ + 0x640C,0x6426,0x6421,0x645E,0x6484,0x646D,0x6496,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_DF[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9019,0x901C,0x9023,0x9024,0x9025,0x9027,0x9028,0x9029,/* 0x40-0x47 */ + 0x902A,0x902B,0x902C,0x9030,0x9031,0x9032,0x9033,0x9034,/* 0x48-0x4F */ + 0x9037,0x9039,0x903A,0x903D,0x903F,0x9040,0x9043,0x9045,/* 0x50-0x57 */ + 0x9046,0x9048,0x9049,0x904A,0x904B,0x904C,0x904E,0x9054,/* 0x58-0x5F */ + 0x9055,0x9056,0x9059,0x905A,0x905C,0x905D,0x905E,0x905F,/* 0x60-0x67 */ + 0x9060,0x9061,0x9064,0x9066,0x9067,0x9069,0x906A,0x906B,/* 0x68-0x6F */ + 0x906C,0x906F,0x9070,0x9071,0x9072,0x9073,0x9076,0x9077,/* 0x70-0x77 */ + 0x9078,0x9079,0x907A,0x907B,0x907C,0x907E,0x9081,0x0000,/* 0x78-0x7F */ + + 0x9084,0x9085,0x9086,0x9087,0x9089,0x908A,0x908C,0x908D,/* 0x80-0x87 */ + 0x908E,0x908F,0x9090,0x9092,0x9094,0x9096,0x9098,0x909A,/* 0x88-0x8F */ + 0x909C,0x909E,0x909F,0x90A0,0x90A4,0x90A5,0x90A7,0x90A8,/* 0x90-0x97 */ + 0x90A9,0x90AB,0x90AD,0x90B2,0x90B7,0x90BC,0x90BD,0x90BF,/* 0x98-0x9F */ + 0x90C0,0x647A,0x64B7,0x64B8,0x6499,0x64BA,0x64C0,0x64D0,/* 0xA0-0xA7 */ + 0x64D7,0x64E4,0x64E2,0x6509,0x6525,0x652E,0x5F0B,0x5FD2,/* 0xA8-0xAF */ + 0x7519,0x5F11,0x535F,0x53F1,0x53FD,0x53E9,0x53E8,0x53FB,/* 0xB0-0xB7 */ + 0x5412,0x5416,0x5406,0x544B,0x5452,0x5453,0x5454,0x5456,/* 0xB8-0xBF */ + 0x5443,0x5421,0x5457,0x5459,0x5423,0x5432,0x5482,0x5494,/* 0xC0-0xC7 */ + 0x5477,0x5471,0x5464,0x549A,0x549B,0x5484,0x5476,0x5466,/* 0xC8-0xCF */ + 0x549D,0x54D0,0x54AD,0x54C2,0x54B4,0x54D2,0x54A7,0x54A6,/* 0xD0-0xD7 */ + 0x54D3,0x54D4,0x5472,0x54A3,0x54D5,0x54BB,0x54BF,0x54CC,/* 0xD8-0xDF */ + 0x54D9,0x54DA,0x54DC,0x54A9,0x54AA,0x54A4,0x54DD,0x54CF,/* 0xE0-0xE7 */ + 0x54DE,0x551B,0x54E7,0x5520,0x54FD,0x5514,0x54F3,0x5522,/* 0xE8-0xEF */ + 0x5523,0x550F,0x5511,0x5527,0x552A,0x5567,0x558F,0x55B5,/* 0xF0-0xF7 */ + 0x5549,0x556D,0x5541,0x5555,0x553F,0x5550,0x553C,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E0[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x90C2,0x90C3,0x90C6,0x90C8,0x90C9,0x90CB,0x90CC,0x90CD,/* 0x40-0x47 */ + 0x90D2,0x90D4,0x90D5,0x90D6,0x90D8,0x90D9,0x90DA,0x90DE,/* 0x48-0x4F */ + 0x90DF,0x90E0,0x90E3,0x90E4,0x90E5,0x90E9,0x90EA,0x90EC,/* 0x50-0x57 */ + 0x90EE,0x90F0,0x90F1,0x90F2,0x90F3,0x90F5,0x90F6,0x90F7,/* 0x58-0x5F */ + 0x90F9,0x90FA,0x90FB,0x90FC,0x90FF,0x9100,0x9101,0x9103,/* 0x60-0x67 */ + 0x9105,0x9106,0x9107,0x9108,0x9109,0x910A,0x910B,0x910C,/* 0x68-0x6F */ + 0x910D,0x910E,0x910F,0x9110,0x9111,0x9112,0x9113,0x9114,/* 0x70-0x77 */ + 0x9115,0x9116,0x9117,0x9118,0x911A,0x911B,0x911C,0x0000,/* 0x78-0x7F */ + + 0x911D,0x911F,0x9120,0x9121,0x9124,0x9125,0x9126,0x9127,/* 0x80-0x87 */ + 0x9128,0x9129,0x912A,0x912B,0x912C,0x912D,0x912E,0x9130,/* 0x88-0x8F */ + 0x9132,0x9133,0x9134,0x9135,0x9136,0x9137,0x9138,0x913A,/* 0x90-0x97 */ + 0x913B,0x913C,0x913D,0x913E,0x913F,0x9140,0x9141,0x9142,/* 0x98-0x9F */ + 0x9144,0x5537,0x5556,0x5575,0x5576,0x5577,0x5533,0x5530,/* 0xA0-0xA7 */ + 0x555C,0x558B,0x55D2,0x5583,0x55B1,0x55B9,0x5588,0x5581,/* 0xA8-0xAF */ + 0x559F,0x557E,0x55D6,0x5591,0x557B,0x55DF,0x55BD,0x55BE,/* 0xB0-0xB7 */ + 0x5594,0x5599,0x55EA,0x55F7,0x55C9,0x561F,0x55D1,0x55EB,/* 0xB8-0xBF */ + 0x55EC,0x55D4,0x55E6,0x55DD,0x55C4,0x55EF,0x55E5,0x55F2,/* 0xC0-0xC7 */ + 0x55F3,0x55CC,0x55CD,0x55E8,0x55F5,0x55E4,0x8F94,0x561E,/* 0xC8-0xCF */ + 0x5608,0x560C,0x5601,0x5624,0x5623,0x55FE,0x5600,0x5627,/* 0xD0-0xD7 */ + 0x562D,0x5658,0x5639,0x5657,0x562C,0x564D,0x5662,0x5659,/* 0xD8-0xDF */ + 0x565C,0x564C,0x5654,0x5686,0x5664,0x5671,0x566B,0x567B,/* 0xE0-0xE7 */ + 0x567C,0x5685,0x5693,0x56AF,0x56D4,0x56D7,0x56DD,0x56E1,/* 0xE8-0xEF */ + 0x56F5,0x56EB,0x56F9,0x56FF,0x5704,0x570A,0x5709,0x571C,/* 0xF0-0xF7 */ + 0x5E0F,0x5E19,0x5E14,0x5E11,0x5E31,0x5E3B,0x5E3C,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E1[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9145,0x9147,0x9148,0x9151,0x9153,0x9154,0x9155,0x9156,/* 0x40-0x47 */ + 0x9158,0x9159,0x915B,0x915C,0x915F,0x9160,0x9166,0x9167,/* 0x48-0x4F */ + 0x9168,0x916B,0x916D,0x9173,0x917A,0x917B,0x917C,0x9180,/* 0x50-0x57 */ + 0x9181,0x9182,0x9183,0x9184,0x9186,0x9188,0x918A,0x918E,/* 0x58-0x5F */ + 0x918F,0x9193,0x9194,0x9195,0x9196,0x9197,0x9198,0x9199,/* 0x60-0x67 */ + 0x919C,0x919D,0x919E,0x919F,0x91A0,0x91A1,0x91A4,0x91A5,/* 0x68-0x6F */ + 0x91A6,0x91A7,0x91A8,0x91A9,0x91AB,0x91AC,0x91B0,0x91B1,/* 0x70-0x77 */ + 0x91B2,0x91B3,0x91B6,0x91B7,0x91B8,0x91B9,0x91BB,0x0000,/* 0x78-0x7F */ + + 0x91BC,0x91BD,0x91BE,0x91BF,0x91C0,0x91C1,0x91C2,0x91C3,/* 0x80-0x87 */ + 0x91C4,0x91C5,0x91C6,0x91C8,0x91CB,0x91D0,0x91D2,0x91D3,/* 0x88-0x8F */ + 0x91D4,0x91D5,0x91D6,0x91D7,0x91D8,0x91D9,0x91DA,0x91DB,/* 0x90-0x97 */ + 0x91DD,0x91DE,0x91DF,0x91E0,0x91E1,0x91E2,0x91E3,0x91E4,/* 0x98-0x9F */ + 0x91E5,0x5E37,0x5E44,0x5E54,0x5E5B,0x5E5E,0x5E61,0x5C8C,/* 0xA0-0xA7 */ + 0x5C7A,0x5C8D,0x5C90,0x5C96,0x5C88,0x5C98,0x5C99,0x5C91,/* 0xA8-0xAF */ + 0x5C9A,0x5C9C,0x5CB5,0x5CA2,0x5CBD,0x5CAC,0x5CAB,0x5CB1,/* 0xB0-0xB7 */ + 0x5CA3,0x5CC1,0x5CB7,0x5CC4,0x5CD2,0x5CE4,0x5CCB,0x5CE5,/* 0xB8-0xBF */ + 0x5D02,0x5D03,0x5D27,0x5D26,0x5D2E,0x5D24,0x5D1E,0x5D06,/* 0xC0-0xC7 */ + 0x5D1B,0x5D58,0x5D3E,0x5D34,0x5D3D,0x5D6C,0x5D5B,0x5D6F,/* 0xC8-0xCF */ + 0x5D5D,0x5D6B,0x5D4B,0x5D4A,0x5D69,0x5D74,0x5D82,0x5D99,/* 0xD0-0xD7 */ + 0x5D9D,0x8C73,0x5DB7,0x5DC5,0x5F73,0x5F77,0x5F82,0x5F87,/* 0xD8-0xDF */ + 0x5F89,0x5F8C,0x5F95,0x5F99,0x5F9C,0x5FA8,0x5FAD,0x5FB5,/* 0xE0-0xE7 */ + 0x5FBC,0x8862,0x5F61,0x72AD,0x72B0,0x72B4,0x72B7,0x72B8,/* 0xE8-0xEF */ + 0x72C3,0x72C1,0x72CE,0x72CD,0x72D2,0x72E8,0x72EF,0x72E9,/* 0xF0-0xF7 */ + 0x72F2,0x72F4,0x72F7,0x7301,0x72F3,0x7303,0x72FA,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E2[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x91E6,0x91E7,0x91E8,0x91E9,0x91EA,0x91EB,0x91EC,0x91ED,/* 0x40-0x47 */ + 0x91EE,0x91EF,0x91F0,0x91F1,0x91F2,0x91F3,0x91F4,0x91F5,/* 0x48-0x4F */ + 0x91F6,0x91F7,0x91F8,0x91F9,0x91FA,0x91FB,0x91FC,0x91FD,/* 0x50-0x57 */ + 0x91FE,0x91FF,0x9200,0x9201,0x9202,0x9203,0x9204,0x9205,/* 0x58-0x5F */ + 0x9206,0x9207,0x9208,0x9209,0x920A,0x920B,0x920C,0x920D,/* 0x60-0x67 */ + 0x920E,0x920F,0x9210,0x9211,0x9212,0x9213,0x9214,0x9215,/* 0x68-0x6F */ + 0x9216,0x9217,0x9218,0x9219,0x921A,0x921B,0x921C,0x921D,/* 0x70-0x77 */ + 0x921E,0x921F,0x9220,0x9221,0x9222,0x9223,0x9224,0x0000,/* 0x78-0x7F */ + + 0x9225,0x9226,0x9227,0x9228,0x9229,0x922A,0x922B,0x922C,/* 0x80-0x87 */ + 0x922D,0x922E,0x922F,0x9230,0x9231,0x9232,0x9233,0x9234,/* 0x88-0x8F */ + 0x9235,0x9236,0x9237,0x9238,0x9239,0x923A,0x923B,0x923C,/* 0x90-0x97 */ + 0x923D,0x923E,0x923F,0x9240,0x9241,0x9242,0x9243,0x9244,/* 0x98-0x9F */ + 0x9245,0x72FB,0x7317,0x7313,0x7321,0x730A,0x731E,0x731D,/* 0xA0-0xA7 */ + 0x7315,0x7322,0x7339,0x7325,0x732C,0x7338,0x7331,0x7350,/* 0xA8-0xAF */ + 0x734D,0x7357,0x7360,0x736C,0x736F,0x737E,0x821B,0x5925,/* 0xB0-0xB7 */ + 0x98E7,0x5924,0x5902,0x9963,0x9967,0x9968,0x9969,0x996A,/* 0xB8-0xBF */ + 0x996B,0x996C,0x9974,0x9977,0x997D,0x9980,0x9984,0x9987,/* 0xC0-0xC7 */ + 0x998A,0x998D,0x9990,0x9991,0x9993,0x9994,0x9995,0x5E80,/* 0xC8-0xCF */ + 0x5E91,0x5E8B,0x5E96,0x5EA5,0x5EA0,0x5EB9,0x5EB5,0x5EBE,/* 0xD0-0xD7 */ + 0x5EB3,0x8D53,0x5ED2,0x5ED1,0x5EDB,0x5EE8,0x5EEA,0x81BA,/* 0xD8-0xDF */ + 0x5FC4,0x5FC9,0x5FD6,0x5FCF,0x6003,0x5FEE,0x6004,0x5FE1,/* 0xE0-0xE7 */ + 0x5FE4,0x5FFE,0x6005,0x6006,0x5FEA,0x5FED,0x5FF8,0x6019,/* 0xE8-0xEF */ + 0x6035,0x6026,0x601B,0x600F,0x600D,0x6029,0x602B,0x600A,/* 0xF0-0xF7 */ + 0x603F,0x6021,0x6078,0x6079,0x607B,0x607A,0x6042,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E3[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9246,0x9247,0x9248,0x9249,0x924A,0x924B,0x924C,0x924D,/* 0x40-0x47 */ + 0x924E,0x924F,0x9250,0x9251,0x9252,0x9253,0x9254,0x9255,/* 0x48-0x4F */ + 0x9256,0x9257,0x9258,0x9259,0x925A,0x925B,0x925C,0x925D,/* 0x50-0x57 */ + 0x925E,0x925F,0x9260,0x9261,0x9262,0x9263,0x9264,0x9265,/* 0x58-0x5F */ + 0x9266,0x9267,0x9268,0x9269,0x926A,0x926B,0x926C,0x926D,/* 0x60-0x67 */ + 0x926E,0x926F,0x9270,0x9271,0x9272,0x9273,0x9275,0x9276,/* 0x68-0x6F */ + 0x9277,0x9278,0x9279,0x927A,0x927B,0x927C,0x927D,0x927E,/* 0x70-0x77 */ + 0x927F,0x9280,0x9281,0x9282,0x9283,0x9284,0x9285,0x0000,/* 0x78-0x7F */ + + 0x9286,0x9287,0x9288,0x9289,0x928A,0x928B,0x928C,0x928D,/* 0x80-0x87 */ + 0x928F,0x9290,0x9291,0x9292,0x9293,0x9294,0x9295,0x9296,/* 0x88-0x8F */ + 0x9297,0x9298,0x9299,0x929A,0x929B,0x929C,0x929D,0x929E,/* 0x90-0x97 */ + 0x929F,0x92A0,0x92A1,0x92A2,0x92A3,0x92A4,0x92A5,0x92A6,/* 0x98-0x9F */ + 0x92A7,0x606A,0x607D,0x6096,0x609A,0x60AD,0x609D,0x6083,/* 0xA0-0xA7 */ + 0x6092,0x608C,0x609B,0x60EC,0x60BB,0x60B1,0x60DD,0x60D8,/* 0xA8-0xAF */ + 0x60C6,0x60DA,0x60B4,0x6120,0x6126,0x6115,0x6123,0x60F4,/* 0xB0-0xB7 */ + 0x6100,0x610E,0x612B,0x614A,0x6175,0x61AC,0x6194,0x61A7,/* 0xB8-0xBF */ + 0x61B7,0x61D4,0x61F5,0x5FDD,0x96B3,0x95E9,0x95EB,0x95F1,/* 0xC0-0xC7 */ + 0x95F3,0x95F5,0x95F6,0x95FC,0x95FE,0x9603,0x9604,0x9606,/* 0xC8-0xCF */ + 0x9608,0x960A,0x960B,0x960C,0x960D,0x960F,0x9612,0x9615,/* 0xD0-0xD7 */ + 0x9616,0x9617,0x9619,0x961A,0x4E2C,0x723F,0x6215,0x6C35,/* 0xD8-0xDF */ + 0x6C54,0x6C5C,0x6C4A,0x6CA3,0x6C85,0x6C90,0x6C94,0x6C8C,/* 0xE0-0xE7 */ + 0x6C68,0x6C69,0x6C74,0x6C76,0x6C86,0x6CA9,0x6CD0,0x6CD4,/* 0xE8-0xEF */ + 0x6CAD,0x6CF7,0x6CF8,0x6CF1,0x6CD7,0x6CB2,0x6CE0,0x6CD6,/* 0xF0-0xF7 */ + 0x6CFA,0x6CEB,0x6CEE,0x6CB1,0x6CD3,0x6CEF,0x6CFE,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E4[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x92A8,0x92A9,0x92AA,0x92AB,0x92AC,0x92AD,0x92AF,0x92B0,/* 0x40-0x47 */ + 0x92B1,0x92B2,0x92B3,0x92B4,0x92B5,0x92B6,0x92B7,0x92B8,/* 0x48-0x4F */ + 0x92B9,0x92BA,0x92BB,0x92BC,0x92BD,0x92BE,0x92BF,0x92C0,/* 0x50-0x57 */ + 0x92C1,0x92C2,0x92C3,0x92C4,0x92C5,0x92C6,0x92C7,0x92C9,/* 0x58-0x5F */ + 0x92CA,0x92CB,0x92CC,0x92CD,0x92CE,0x92CF,0x92D0,0x92D1,/* 0x60-0x67 */ + 0x92D2,0x92D3,0x92D4,0x92D5,0x92D6,0x92D7,0x92D8,0x92D9,/* 0x68-0x6F */ + 0x92DA,0x92DB,0x92DC,0x92DD,0x92DE,0x92DF,0x92E0,0x92E1,/* 0x70-0x77 */ + 0x92E2,0x92E3,0x92E4,0x92E5,0x92E6,0x92E7,0x92E8,0x0000,/* 0x78-0x7F */ + + 0x92E9,0x92EA,0x92EB,0x92EC,0x92ED,0x92EE,0x92EF,0x92F0,/* 0x80-0x87 */ + 0x92F1,0x92F2,0x92F3,0x92F4,0x92F5,0x92F6,0x92F7,0x92F8,/* 0x88-0x8F */ + 0x92F9,0x92FA,0x92FB,0x92FC,0x92FD,0x92FE,0x92FF,0x9300,/* 0x90-0x97 */ + 0x9301,0x9302,0x9303,0x9304,0x9305,0x9306,0x9307,0x9308,/* 0x98-0x9F */ + 0x9309,0x6D39,0x6D27,0x6D0C,0x6D43,0x6D48,0x6D07,0x6D04,/* 0xA0-0xA7 */ + 0x6D19,0x6D0E,0x6D2B,0x6D4D,0x6D2E,0x6D35,0x6D1A,0x6D4F,/* 0xA8-0xAF */ + 0x6D52,0x6D54,0x6D33,0x6D91,0x6D6F,0x6D9E,0x6DA0,0x6D5E,/* 0xB0-0xB7 */ + 0x6D93,0x6D94,0x6D5C,0x6D60,0x6D7C,0x6D63,0x6E1A,0x6DC7,/* 0xB8-0xBF */ + 0x6DC5,0x6DDE,0x6E0E,0x6DBF,0x6DE0,0x6E11,0x6DE6,0x6DDD,/* 0xC0-0xC7 */ + 0x6DD9,0x6E16,0x6DAB,0x6E0C,0x6DAE,0x6E2B,0x6E6E,0x6E4E,/* 0xC8-0xCF */ + 0x6E6B,0x6EB2,0x6E5F,0x6E86,0x6E53,0x6E54,0x6E32,0x6E25,/* 0xD0-0xD7 */ + 0x6E44,0x6EDF,0x6EB1,0x6E98,0x6EE0,0x6F2D,0x6EE2,0x6EA5,/* 0xD8-0xDF */ + 0x6EA7,0x6EBD,0x6EBB,0x6EB7,0x6ED7,0x6EB4,0x6ECF,0x6E8F,/* 0xE0-0xE7 */ + 0x6EC2,0x6E9F,0x6F62,0x6F46,0x6F47,0x6F24,0x6F15,0x6EF9,/* 0xE8-0xEF */ + 0x6F2F,0x6F36,0x6F4B,0x6F74,0x6F2A,0x6F09,0x6F29,0x6F89,/* 0xF0-0xF7 */ + 0x6F8D,0x6F8C,0x6F78,0x6F72,0x6F7C,0x6F7A,0x6FD1,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E5[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x930A,0x930B,0x930C,0x930D,0x930E,0x930F,0x9310,0x9311,/* 0x40-0x47 */ + 0x9312,0x9313,0x9314,0x9315,0x9316,0x9317,0x9318,0x9319,/* 0x48-0x4F */ + 0x931A,0x931B,0x931C,0x931D,0x931E,0x931F,0x9320,0x9321,/* 0x50-0x57 */ + 0x9322,0x9323,0x9324,0x9325,0x9326,0x9327,0x9328,0x9329,/* 0x58-0x5F */ + 0x932A,0x932B,0x932C,0x932D,0x932E,0x932F,0x9330,0x9331,/* 0x60-0x67 */ + 0x9332,0x9333,0x9334,0x9335,0x9336,0x9337,0x9338,0x9339,/* 0x68-0x6F */ + 0x933A,0x933B,0x933C,0x933D,0x933F,0x9340,0x9341,0x9342,/* 0x70-0x77 */ + 0x9343,0x9344,0x9345,0x9346,0x9347,0x9348,0x9349,0x0000,/* 0x78-0x7F */ + + 0x934A,0x934B,0x934C,0x934D,0x934E,0x934F,0x9350,0x9351,/* 0x80-0x87 */ + 0x9352,0x9353,0x9354,0x9355,0x9356,0x9357,0x9358,0x9359,/* 0x88-0x8F */ + 0x935A,0x935B,0x935C,0x935D,0x935E,0x935F,0x9360,0x9361,/* 0x90-0x97 */ + 0x9362,0x9363,0x9364,0x9365,0x9366,0x9367,0x9368,0x9369,/* 0x98-0x9F */ + 0x936B,0x6FC9,0x6FA7,0x6FB9,0x6FB6,0x6FC2,0x6FE1,0x6FEE,/* 0xA0-0xA7 */ + 0x6FDE,0x6FE0,0x6FEF,0x701A,0x7023,0x701B,0x7039,0x7035,/* 0xA8-0xAF */ + 0x704F,0x705E,0x5B80,0x5B84,0x5B95,0x5B93,0x5BA5,0x5BB8,/* 0xB0-0xB7 */ + 0x752F,0x9A9E,0x6434,0x5BE4,0x5BEE,0x8930,0x5BF0,0x8E47,/* 0xB8-0xBF */ + 0x8B07,0x8FB6,0x8FD3,0x8FD5,0x8FE5,0x8FEE,0x8FE4,0x8FE9,/* 0xC0-0xC7 */ + 0x8FE6,0x8FF3,0x8FE8,0x9005,0x9004,0x900B,0x9026,0x9011,/* 0xC8-0xCF */ + 0x900D,0x9016,0x9021,0x9035,0x9036,0x902D,0x902F,0x9044,/* 0xD0-0xD7 */ + 0x9051,0x9052,0x9050,0x9068,0x9058,0x9062,0x905B,0x66B9,/* 0xD8-0xDF */ + 0x9074,0x907D,0x9082,0x9088,0x9083,0x908B,0x5F50,0x5F57,/* 0xE0-0xE7 */ + 0x5F56,0x5F58,0x5C3B,0x54AB,0x5C50,0x5C59,0x5B71,0x5C63,/* 0xE8-0xEF */ + 0x5C66,0x7FBC,0x5F2A,0x5F29,0x5F2D,0x8274,0x5F3C,0x9B3B,/* 0xF0-0xF7 */ + 0x5C6E,0x5981,0x5983,0x598D,0x59A9,0x59AA,0x59A3,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E6[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x936C,0x936D,0x936E,0x936F,0x9370,0x9371,0x9372,0x9373,/* 0x40-0x47 */ + 0x9374,0x9375,0x9376,0x9377,0x9378,0x9379,0x937A,0x937B,/* 0x48-0x4F */ + 0x937C,0x937D,0x937E,0x937F,0x9380,0x9381,0x9382,0x9383,/* 0x50-0x57 */ + 0x9384,0x9385,0x9386,0x9387,0x9388,0x9389,0x938A,0x938B,/* 0x58-0x5F */ + 0x938C,0x938D,0x938E,0x9390,0x9391,0x9392,0x9393,0x9394,/* 0x60-0x67 */ + 0x9395,0x9396,0x9397,0x9398,0x9399,0x939A,0x939B,0x939C,/* 0x68-0x6F */ + 0x939D,0x939E,0x939F,0x93A0,0x93A1,0x93A2,0x93A3,0x93A4,/* 0x70-0x77 */ + 0x93A5,0x93A6,0x93A7,0x93A8,0x93A9,0x93AA,0x93AB,0x0000,/* 0x78-0x7F */ + + 0x93AC,0x93AD,0x93AE,0x93AF,0x93B0,0x93B1,0x93B2,0x93B3,/* 0x80-0x87 */ + 0x93B4,0x93B5,0x93B6,0x93B7,0x93B8,0x93B9,0x93BA,0x93BB,/* 0x88-0x8F */ + 0x93BC,0x93BD,0x93BE,0x93BF,0x93C0,0x93C1,0x93C2,0x93C3,/* 0x90-0x97 */ + 0x93C4,0x93C5,0x93C6,0x93C7,0x93C8,0x93C9,0x93CB,0x93CC,/* 0x98-0x9F */ + 0x93CD,0x5997,0x59CA,0x59AB,0x599E,0x59A4,0x59D2,0x59B2,/* 0xA0-0xA7 */ + 0x59AF,0x59D7,0x59BE,0x5A05,0x5A06,0x59DD,0x5A08,0x59E3,/* 0xA8-0xAF */ + 0x59D8,0x59F9,0x5A0C,0x5A09,0x5A32,0x5A34,0x5A11,0x5A23,/* 0xB0-0xB7 */ + 0x5A13,0x5A40,0x5A67,0x5A4A,0x5A55,0x5A3C,0x5A62,0x5A75,/* 0xB8-0xBF */ + 0x80EC,0x5AAA,0x5A9B,0x5A77,0x5A7A,0x5ABE,0x5AEB,0x5AB2,/* 0xC0-0xC7 */ + 0x5AD2,0x5AD4,0x5AB8,0x5AE0,0x5AE3,0x5AF1,0x5AD6,0x5AE6,/* 0xC8-0xCF */ + 0x5AD8,0x5ADC,0x5B09,0x5B17,0x5B16,0x5B32,0x5B37,0x5B40,/* 0xD0-0xD7 */ + 0x5C15,0x5C1C,0x5B5A,0x5B65,0x5B73,0x5B51,0x5B53,0x5B62,/* 0xD8-0xDF */ + 0x9A75,0x9A77,0x9A78,0x9A7A,0x9A7F,0x9A7D,0x9A80,0x9A81,/* 0xE0-0xE7 */ + 0x9A85,0x9A88,0x9A8A,0x9A90,0x9A92,0x9A93,0x9A96,0x9A98,/* 0xE8-0xEF */ + 0x9A9B,0x9A9C,0x9A9D,0x9A9F,0x9AA0,0x9AA2,0x9AA3,0x9AA5,/* 0xF0-0xF7 */ + 0x9AA7,0x7E9F,0x7EA1,0x7EA3,0x7EA5,0x7EA8,0x7EA9,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E7[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x93CE,0x93CF,0x93D0,0x93D1,0x93D2,0x93D3,0x93D4,0x93D5,/* 0x40-0x47 */ + 0x93D7,0x93D8,0x93D9,0x93DA,0x93DB,0x93DC,0x93DD,0x93DE,/* 0x48-0x4F */ + 0x93DF,0x93E0,0x93E1,0x93E2,0x93E3,0x93E4,0x93E5,0x93E6,/* 0x50-0x57 */ + 0x93E7,0x93E8,0x93E9,0x93EA,0x93EB,0x93EC,0x93ED,0x93EE,/* 0x58-0x5F */ + 0x93EF,0x93F0,0x93F1,0x93F2,0x93F3,0x93F4,0x93F5,0x93F6,/* 0x60-0x67 */ + 0x93F7,0x93F8,0x93F9,0x93FA,0x93FB,0x93FC,0x93FD,0x93FE,/* 0x68-0x6F */ + 0x93FF,0x9400,0x9401,0x9402,0x9403,0x9404,0x9405,0x9406,/* 0x70-0x77 */ + 0x9407,0x9408,0x9409,0x940A,0x940B,0x940C,0x940D,0x0000,/* 0x78-0x7F */ + + 0x940E,0x940F,0x9410,0x9411,0x9412,0x9413,0x9414,0x9415,/* 0x80-0x87 */ + 0x9416,0x9417,0x9418,0x9419,0x941A,0x941B,0x941C,0x941D,/* 0x88-0x8F */ + 0x941E,0x941F,0x9420,0x9421,0x9422,0x9423,0x9424,0x9425,/* 0x90-0x97 */ + 0x9426,0x9427,0x9428,0x9429,0x942A,0x942B,0x942C,0x942D,/* 0x98-0x9F */ + 0x942E,0x7EAD,0x7EB0,0x7EBE,0x7EC0,0x7EC1,0x7EC2,0x7EC9,/* 0xA0-0xA7 */ + 0x7ECB,0x7ECC,0x7ED0,0x7ED4,0x7ED7,0x7EDB,0x7EE0,0x7EE1,/* 0xA8-0xAF */ + 0x7EE8,0x7EEB,0x7EEE,0x7EEF,0x7EF1,0x7EF2,0x7F0D,0x7EF6,/* 0xB0-0xB7 */ + 0x7EFA,0x7EFB,0x7EFE,0x7F01,0x7F02,0x7F03,0x7F07,0x7F08,/* 0xB8-0xBF */ + 0x7F0B,0x7F0C,0x7F0F,0x7F11,0x7F12,0x7F17,0x7F19,0x7F1C,/* 0xC0-0xC7 */ + 0x7F1B,0x7F1F,0x7F21,0x7F22,0x7F23,0x7F24,0x7F25,0x7F26,/* 0xC8-0xCF */ + 0x7F27,0x7F2A,0x7F2B,0x7F2C,0x7F2D,0x7F2F,0x7F30,0x7F31,/* 0xD0-0xD7 */ + 0x7F32,0x7F33,0x7F35,0x5E7A,0x757F,0x5DDB,0x753E,0x9095,/* 0xD8-0xDF */ + 0x738E,0x7391,0x73AE,0x73A2,0x739F,0x73CF,0x73C2,0x73D1,/* 0xE0-0xE7 */ + 0x73B7,0x73B3,0x73C0,0x73C9,0x73C8,0x73E5,0x73D9,0x987C,/* 0xE8-0xEF */ + 0x740A,0x73E9,0x73E7,0x73DE,0x73BA,0x73F2,0x740F,0x742A,/* 0xF0-0xF7 */ + 0x745B,0x7426,0x7425,0x7428,0x7430,0x742E,0x742C,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E8[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x942F,0x9430,0x9431,0x9432,0x9433,0x9434,0x9435,0x9436,/* 0x40-0x47 */ + 0x9437,0x9438,0x9439,0x943A,0x943B,0x943C,0x943D,0x943F,/* 0x48-0x4F */ + 0x9440,0x9441,0x9442,0x9443,0x9444,0x9445,0x9446,0x9447,/* 0x50-0x57 */ + 0x9448,0x9449,0x944A,0x944B,0x944C,0x944D,0x944E,0x944F,/* 0x58-0x5F */ + 0x9450,0x9451,0x9452,0x9453,0x9454,0x9455,0x9456,0x9457,/* 0x60-0x67 */ + 0x9458,0x9459,0x945A,0x945B,0x945C,0x945D,0x945E,0x945F,/* 0x68-0x6F */ + 0x9460,0x9461,0x9462,0x9463,0x9464,0x9465,0x9466,0x9467,/* 0x70-0x77 */ + 0x9468,0x9469,0x946A,0x946C,0x946D,0x946E,0x946F,0x0000,/* 0x78-0x7F */ + + 0x9470,0x9471,0x9472,0x9473,0x9474,0x9475,0x9476,0x9477,/* 0x80-0x87 */ + 0x9478,0x9479,0x947A,0x947B,0x947C,0x947D,0x947E,0x947F,/* 0x88-0x8F */ + 0x9480,0x9481,0x9482,0x9483,0x9484,0x9491,0x9496,0x9498,/* 0x90-0x97 */ + 0x94C7,0x94CF,0x94D3,0x94D4,0x94DA,0x94E6,0x94FB,0x951C,/* 0x98-0x9F */ + 0x9520,0x741B,0x741A,0x7441,0x745C,0x7457,0x7455,0x7459,/* 0xA0-0xA7 */ + 0x7477,0x746D,0x747E,0x749C,0x748E,0x7480,0x7481,0x7487,/* 0xA8-0xAF */ + 0x748B,0x749E,0x74A8,0x74A9,0x7490,0x74A7,0x74D2,0x74BA,/* 0xB0-0xB7 */ + 0x97EA,0x97EB,0x97EC,0x674C,0x6753,0x675E,0x6748,0x6769,/* 0xB8-0xBF */ + 0x67A5,0x6787,0x676A,0x6773,0x6798,0x67A7,0x6775,0x67A8,/* 0xC0-0xC7 */ + 0x679E,0x67AD,0x678B,0x6777,0x677C,0x67F0,0x6809,0x67D8,/* 0xC8-0xCF */ + 0x680A,0x67E9,0x67B0,0x680C,0x67D9,0x67B5,0x67DA,0x67B3,/* 0xD0-0xD7 */ + 0x67DD,0x6800,0x67C3,0x67B8,0x67E2,0x680E,0x67C1,0x67FD,/* 0xD8-0xDF */ + 0x6832,0x6833,0x6860,0x6861,0x684E,0x6862,0x6844,0x6864,/* 0xE0-0xE7 */ + 0x6883,0x681D,0x6855,0x6866,0x6841,0x6867,0x6840,0x683E,/* 0xE8-0xEF */ + 0x684A,0x6849,0x6829,0x68B5,0x688F,0x6874,0x6877,0x6893,/* 0xF0-0xF7 */ + 0x686B,0x68C2,0x696E,0x68FC,0x691F,0x6920,0x68F9,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E9[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9527,0x9533,0x953D,0x9543,0x9548,0x954B,0x9555,0x955A,/* 0x40-0x47 */ + 0x9560,0x956E,0x9574,0x9575,0x9577,0x9578,0x9579,0x957A,/* 0x48-0x4F */ + 0x957B,0x957C,0x957D,0x957E,0x9580,0x9581,0x9582,0x9583,/* 0x50-0x57 */ + 0x9584,0x9585,0x9586,0x9587,0x9588,0x9589,0x958A,0x958B,/* 0x58-0x5F */ + 0x958C,0x958D,0x958E,0x958F,0x9590,0x9591,0x9592,0x9593,/* 0x60-0x67 */ + 0x9594,0x9595,0x9596,0x9597,0x9598,0x9599,0x959A,0x959B,/* 0x68-0x6F */ + 0x959C,0x959D,0x959E,0x959F,0x95A0,0x95A1,0x95A2,0x95A3,/* 0x70-0x77 */ + 0x95A4,0x95A5,0x95A6,0x95A7,0x95A8,0x95A9,0x95AA,0x0000,/* 0x78-0x7F */ + + 0x95AB,0x95AC,0x95AD,0x95AE,0x95AF,0x95B0,0x95B1,0x95B2,/* 0x80-0x87 */ + 0x95B3,0x95B4,0x95B5,0x95B6,0x95B7,0x95B8,0x95B9,0x95BA,/* 0x88-0x8F */ + 0x95BB,0x95BC,0x95BD,0x95BE,0x95BF,0x95C0,0x95C1,0x95C2,/* 0x90-0x97 */ + 0x95C3,0x95C4,0x95C5,0x95C6,0x95C7,0x95C8,0x95C9,0x95CA,/* 0x98-0x9F */ + 0x95CB,0x6924,0x68F0,0x690B,0x6901,0x6957,0x68E3,0x6910,/* 0xA0-0xA7 */ + 0x6971,0x6939,0x6960,0x6942,0x695D,0x6984,0x696B,0x6980,/* 0xA8-0xAF */ + 0x6998,0x6978,0x6934,0x69CC,0x6987,0x6988,0x69CE,0x6989,/* 0xB0-0xB7 */ + 0x6966,0x6963,0x6979,0x699B,0x69A7,0x69BB,0x69AB,0x69AD,/* 0xB8-0xBF */ + 0x69D4,0x69B1,0x69C1,0x69CA,0x69DF,0x6995,0x69E0,0x698D,/* 0xC0-0xC7 */ + 0x69FF,0x6A2F,0x69ED,0x6A17,0x6A18,0x6A65,0x69F2,0x6A44,/* 0xC8-0xCF */ + 0x6A3E,0x6AA0,0x6A50,0x6A5B,0x6A35,0x6A8E,0x6A79,0x6A3D,/* 0xD0-0xD7 */ + 0x6A28,0x6A58,0x6A7C,0x6A91,0x6A90,0x6AA9,0x6A97,0x6AAB,/* 0xD8-0xDF */ + 0x7337,0x7352,0x6B81,0x6B82,0x6B87,0x6B84,0x6B92,0x6B93,/* 0xE0-0xE7 */ + 0x6B8D,0x6B9A,0x6B9B,0x6BA1,0x6BAA,0x8F6B,0x8F6D,0x8F71,/* 0xE8-0xEF */ + 0x8F72,0x8F73,0x8F75,0x8F76,0x8F78,0x8F77,0x8F79,0x8F7A,/* 0xF0-0xF7 */ + 0x8F7C,0x8F7E,0x8F81,0x8F82,0x8F84,0x8F87,0x8F8B,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_EA[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x95CC,0x95CD,0x95CE,0x95CF,0x95D0,0x95D1,0x95D2,0x95D3,/* 0x40-0x47 */ + 0x95D4,0x95D5,0x95D6,0x95D7,0x95D8,0x95D9,0x95DA,0x95DB,/* 0x48-0x4F */ + 0x95DC,0x95DD,0x95DE,0x95DF,0x95E0,0x95E1,0x95E2,0x95E3,/* 0x50-0x57 */ + 0x95E4,0x95E5,0x95E6,0x95E7,0x95EC,0x95FF,0x9607,0x9613,/* 0x58-0x5F */ + 0x9618,0x961B,0x961E,0x9620,0x9623,0x9624,0x9625,0x9626,/* 0x60-0x67 */ + 0x9627,0x9628,0x9629,0x962B,0x962C,0x962D,0x962F,0x9630,/* 0x68-0x6F */ + 0x9637,0x9638,0x9639,0x963A,0x963E,0x9641,0x9643,0x964A,/* 0x70-0x77 */ + 0x964E,0x964F,0x9651,0x9652,0x9653,0x9656,0x9657,0x0000,/* 0x78-0x7F */ + + 0x9658,0x9659,0x965A,0x965C,0x965D,0x965E,0x9660,0x9663,/* 0x80-0x87 */ + 0x9665,0x9666,0x966B,0x966D,0x966E,0x966F,0x9670,0x9671,/* 0x88-0x8F */ + 0x9673,0x9678,0x9679,0x967A,0x967B,0x967C,0x967D,0x967E,/* 0x90-0x97 */ + 0x967F,0x9680,0x9681,0x9682,0x9683,0x9684,0x9687,0x9689,/* 0x98-0x9F */ + 0x968A,0x8F8D,0x8F8E,0x8F8F,0x8F98,0x8F9A,0x8ECE,0x620B,/* 0xA0-0xA7 */ + 0x6217,0x621B,0x621F,0x6222,0x6221,0x6225,0x6224,0x622C,/* 0xA8-0xAF */ + 0x81E7,0x74EF,0x74F4,0x74FF,0x750F,0x7511,0x7513,0x6534,/* 0xB0-0xB7 */ + 0x65EE,0x65EF,0x65F0,0x660A,0x6619,0x6772,0x6603,0x6615,/* 0xB8-0xBF */ + 0x6600,0x7085,0x66F7,0x661D,0x6634,0x6631,0x6636,0x6635,/* 0xC0-0xC7 */ + 0x8006,0x665F,0x6654,0x6641,0x664F,0x6656,0x6661,0x6657,/* 0xC8-0xCF */ + 0x6677,0x6684,0x668C,0x66A7,0x669D,0x66BE,0x66DB,0x66DC,/* 0xD0-0xD7 */ + 0x66E6,0x66E9,0x8D32,0x8D33,0x8D36,0x8D3B,0x8D3D,0x8D40,/* 0xD8-0xDF */ + 0x8D45,0x8D46,0x8D48,0x8D49,0x8D47,0x8D4D,0x8D55,0x8D59,/* 0xE0-0xE7 */ + 0x89C7,0x89CA,0x89CB,0x89CC,0x89CE,0x89CF,0x89D0,0x89D1,/* 0xE8-0xEF */ + 0x726E,0x729F,0x725D,0x7266,0x726F,0x727E,0x727F,0x7284,/* 0xF0-0xF7 */ + 0x728B,0x728D,0x728F,0x7292,0x6308,0x6332,0x63B0,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_EB[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x968C,0x968E,0x9691,0x9692,0x9693,0x9695,0x9696,0x969A,/* 0x40-0x47 */ + 0x969B,0x969D,0x969E,0x969F,0x96A0,0x96A1,0x96A2,0x96A3,/* 0x48-0x4F */ + 0x96A4,0x96A5,0x96A6,0x96A8,0x96A9,0x96AA,0x96AB,0x96AC,/* 0x50-0x57 */ + 0x96AD,0x96AE,0x96AF,0x96B1,0x96B2,0x96B4,0x96B5,0x96B7,/* 0x58-0x5F */ + 0x96B8,0x96BA,0x96BB,0x96BF,0x96C2,0x96C3,0x96C8,0x96CA,/* 0x60-0x67 */ + 0x96CB,0x96D0,0x96D1,0x96D3,0x96D4,0x96D6,0x96D7,0x96D8,/* 0x68-0x6F */ + 0x96D9,0x96DA,0x96DB,0x96DC,0x96DD,0x96DE,0x96DF,0x96E1,/* 0x70-0x77 */ + 0x96E2,0x96E3,0x96E4,0x96E5,0x96E6,0x96E7,0x96EB,0x0000,/* 0x78-0x7F */ + + 0x96EC,0x96ED,0x96EE,0x96F0,0x96F1,0x96F2,0x96F4,0x96F5,/* 0x80-0x87 */ + 0x96F8,0x96FA,0x96FB,0x96FC,0x96FD,0x96FF,0x9702,0x9703,/* 0x88-0x8F */ + 0x9705,0x970A,0x970B,0x970C,0x9710,0x9711,0x9712,0x9714,/* 0x90-0x97 */ + 0x9715,0x9717,0x9718,0x9719,0x971A,0x971B,0x971D,0x971F,/* 0x98-0x9F */ + 0x9720,0x643F,0x64D8,0x8004,0x6BEA,0x6BF3,0x6BFD,0x6BF5,/* 0xA0-0xA7 */ + 0x6BF9,0x6C05,0x6C07,0x6C06,0x6C0D,0x6C15,0x6C18,0x6C19,/* 0xA8-0xAF */ + 0x6C1A,0x6C21,0x6C29,0x6C24,0x6C2A,0x6C32,0x6535,0x6555,/* 0xB0-0xB7 */ + 0x656B,0x724D,0x7252,0x7256,0x7230,0x8662,0x5216,0x809F,/* 0xB8-0xBF */ + 0x809C,0x8093,0x80BC,0x670A,0x80BD,0x80B1,0x80AB,0x80AD,/* 0xC0-0xC7 */ + 0x80B4,0x80B7,0x80E7,0x80E8,0x80E9,0x80EA,0x80DB,0x80C2,/* 0xC8-0xCF */ + 0x80C4,0x80D9,0x80CD,0x80D7,0x6710,0x80DD,0x80EB,0x80F1,/* 0xD0-0xD7 */ + 0x80F4,0x80ED,0x810D,0x810E,0x80F2,0x80FC,0x6715,0x8112,/* 0xD8-0xDF */ + 0x8C5A,0x8136,0x811E,0x812C,0x8118,0x8132,0x8148,0x814C,/* 0xE0-0xE7 */ + 0x8153,0x8174,0x8159,0x815A,0x8171,0x8160,0x8169,0x817C,/* 0xE8-0xEF */ + 0x817D,0x816D,0x8167,0x584D,0x5AB5,0x8188,0x8182,0x8191,/* 0xF0-0xF7 */ + 0x6ED5,0x81A3,0x81AA,0x81CC,0x6726,0x81CA,0x81BB,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_EC[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9721,0x9722,0x9723,0x9724,0x9725,0x9726,0x9727,0x9728,/* 0x40-0x47 */ + 0x9729,0x972B,0x972C,0x972E,0x972F,0x9731,0x9733,0x9734,/* 0x48-0x4F */ + 0x9735,0x9736,0x9737,0x973A,0x973B,0x973C,0x973D,0x973F,/* 0x50-0x57 */ + 0x9740,0x9741,0x9742,0x9743,0x9744,0x9745,0x9746,0x9747,/* 0x58-0x5F */ + 0x9748,0x9749,0x974A,0x974B,0x974C,0x974D,0x974E,0x974F,/* 0x60-0x67 */ + 0x9750,0x9751,0x9754,0x9755,0x9757,0x9758,0x975A,0x975C,/* 0x68-0x6F */ + 0x975D,0x975F,0x9763,0x9764,0x9766,0x9767,0x9768,0x976A,/* 0x70-0x77 */ + 0x976B,0x976C,0x976D,0x976E,0x976F,0x9770,0x9771,0x0000,/* 0x78-0x7F */ + + 0x9772,0x9775,0x9777,0x9778,0x9779,0x977A,0x977B,0x977D,/* 0x80-0x87 */ + 0x977E,0x977F,0x9780,0x9781,0x9782,0x9783,0x9784,0x9786,/* 0x88-0x8F */ + 0x9787,0x9788,0x9789,0x978A,0x978C,0x978E,0x978F,0x9790,/* 0x90-0x97 */ + 0x9793,0x9795,0x9796,0x9797,0x9799,0x979A,0x979B,0x979C,/* 0x98-0x9F */ + 0x979D,0x81C1,0x81A6,0x6B24,0x6B37,0x6B39,0x6B43,0x6B46,/* 0xA0-0xA7 */ + 0x6B59,0x98D1,0x98D2,0x98D3,0x98D5,0x98D9,0x98DA,0x6BB3,/* 0xA8-0xAF */ + 0x5F40,0x6BC2,0x89F3,0x6590,0x9F51,0x6593,0x65BC,0x65C6,/* 0xB0-0xB7 */ + 0x65C4,0x65C3,0x65CC,0x65CE,0x65D2,0x65D6,0x7080,0x709C,/* 0xB8-0xBF */ + 0x7096,0x709D,0x70BB,0x70C0,0x70B7,0x70AB,0x70B1,0x70E8,/* 0xC0-0xC7 */ + 0x70CA,0x7110,0x7113,0x7116,0x712F,0x7131,0x7173,0x715C,/* 0xC8-0xCF */ + 0x7168,0x7145,0x7172,0x714A,0x7178,0x717A,0x7198,0x71B3,/* 0xD0-0xD7 */ + 0x71B5,0x71A8,0x71A0,0x71E0,0x71D4,0x71E7,0x71F9,0x721D,/* 0xD8-0xDF */ + 0x7228,0x706C,0x7118,0x7166,0x71B9,0x623E,0x623D,0x6243,/* 0xE0-0xE7 */ + 0x6248,0x6249,0x793B,0x7940,0x7946,0x7949,0x795B,0x795C,/* 0xE8-0xEF */ + 0x7953,0x795A,0x7962,0x7957,0x7960,0x796F,0x7967,0x797A,/* 0xF0-0xF7 */ + 0x7985,0x798A,0x799A,0x79A7,0x79B3,0x5FD1,0x5FD0,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_ED[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x979E,0x979F,0x97A1,0x97A2,0x97A4,0x97A5,0x97A6,0x97A7,/* 0x40-0x47 */ + 0x97A8,0x97A9,0x97AA,0x97AC,0x97AE,0x97B0,0x97B1,0x97B3,/* 0x48-0x4F */ + 0x97B5,0x97B6,0x97B7,0x97B8,0x97B9,0x97BA,0x97BB,0x97BC,/* 0x50-0x57 */ + 0x97BD,0x97BE,0x97BF,0x97C0,0x97C1,0x97C2,0x97C3,0x97C4,/* 0x58-0x5F */ + 0x97C5,0x97C6,0x97C7,0x97C8,0x97C9,0x97CA,0x97CB,0x97CC,/* 0x60-0x67 */ + 0x97CD,0x97CE,0x97CF,0x97D0,0x97D1,0x97D2,0x97D3,0x97D4,/* 0x68-0x6F */ + 0x97D5,0x97D6,0x97D7,0x97D8,0x97D9,0x97DA,0x97DB,0x97DC,/* 0x70-0x77 */ + 0x97DD,0x97DE,0x97DF,0x97E0,0x97E1,0x97E2,0x97E3,0x0000,/* 0x78-0x7F */ + + 0x97E4,0x97E5,0x97E8,0x97EE,0x97EF,0x97F0,0x97F1,0x97F2,/* 0x80-0x87 */ + 0x97F4,0x97F7,0x97F8,0x97F9,0x97FA,0x97FB,0x97FC,0x97FD,/* 0x88-0x8F */ + 0x97FE,0x97FF,0x9800,0x9801,0x9802,0x9803,0x9804,0x9805,/* 0x90-0x97 */ + 0x9806,0x9807,0x9808,0x9809,0x980A,0x980B,0x980C,0x980D,/* 0x98-0x9F */ + 0x980E,0x603C,0x605D,0x605A,0x6067,0x6041,0x6059,0x6063,/* 0xA0-0xA7 */ + 0x60AB,0x6106,0x610D,0x615D,0x61A9,0x619D,0x61CB,0x61D1,/* 0xA8-0xAF */ + 0x6206,0x8080,0x807F,0x6C93,0x6CF6,0x6DFC,0x77F6,0x77F8,/* 0xB0-0xB7 */ + 0x7800,0x7809,0x7817,0x7818,0x7811,0x65AB,0x782D,0x781C,/* 0xB8-0xBF */ + 0x781D,0x7839,0x783A,0x783B,0x781F,0x783C,0x7825,0x782C,/* 0xC0-0xC7 */ + 0x7823,0x7829,0x784E,0x786D,0x7856,0x7857,0x7826,0x7850,/* 0xC8-0xCF */ + 0x7847,0x784C,0x786A,0x789B,0x7893,0x789A,0x7887,0x789C,/* 0xD0-0xD7 */ + 0x78A1,0x78A3,0x78B2,0x78B9,0x78A5,0x78D4,0x78D9,0x78C9,/* 0xD8-0xDF */ + 0x78EC,0x78F2,0x7905,0x78F4,0x7913,0x7924,0x791E,0x7934,/* 0xE0-0xE7 */ + 0x9F9B,0x9EF9,0x9EFB,0x9EFC,0x76F1,0x7704,0x770D,0x76F9,/* 0xE8-0xEF */ + 0x7707,0x7708,0x771A,0x7722,0x7719,0x772D,0x7726,0x7735,/* 0xF0-0xF7 */ + 0x7738,0x7750,0x7751,0x7747,0x7743,0x775A,0x7768,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_EE[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x980F,0x9810,0x9811,0x9812,0x9813,0x9814,0x9815,0x9816,/* 0x40-0x47 */ + 0x9817,0x9818,0x9819,0x981A,0x981B,0x981C,0x981D,0x981E,/* 0x48-0x4F */ + 0x981F,0x9820,0x9821,0x9822,0x9823,0x9824,0x9825,0x9826,/* 0x50-0x57 */ + 0x9827,0x9828,0x9829,0x982A,0x982B,0x982C,0x982D,0x982E,/* 0x58-0x5F */ + 0x982F,0x9830,0x9831,0x9832,0x9833,0x9834,0x9835,0x9836,/* 0x60-0x67 */ + 0x9837,0x9838,0x9839,0x983A,0x983B,0x983C,0x983D,0x983E,/* 0x68-0x6F */ + 0x983F,0x9840,0x9841,0x9842,0x9843,0x9844,0x9845,0x9846,/* 0x70-0x77 */ + 0x9847,0x9848,0x9849,0x984A,0x984B,0x984C,0x984D,0x0000,/* 0x78-0x7F */ + + 0x984E,0x984F,0x9850,0x9851,0x9852,0x9853,0x9854,0x9855,/* 0x80-0x87 */ + 0x9856,0x9857,0x9858,0x9859,0x985A,0x985B,0x985C,0x985D,/* 0x88-0x8F */ + 0x985E,0x985F,0x9860,0x9861,0x9862,0x9863,0x9864,0x9865,/* 0x90-0x97 */ + 0x9866,0x9867,0x9868,0x9869,0x986A,0x986B,0x986C,0x986D,/* 0x98-0x9F */ + 0x986E,0x7762,0x7765,0x777F,0x778D,0x777D,0x7780,0x778C,/* 0xA0-0xA7 */ + 0x7791,0x779F,0x77A0,0x77B0,0x77B5,0x77BD,0x753A,0x7540,/* 0xA8-0xAF */ + 0x754E,0x754B,0x7548,0x755B,0x7572,0x7579,0x7583,0x7F58,/* 0xB0-0xB7 */ + 0x7F61,0x7F5F,0x8A48,0x7F68,0x7F74,0x7F71,0x7F79,0x7F81,/* 0xB8-0xBF */ + 0x7F7E,0x76CD,0x76E5,0x8832,0x9485,0x9486,0x9487,0x948B,/* 0xC0-0xC7 */ + 0x948A,0x948C,0x948D,0x948F,0x9490,0x9494,0x9497,0x9495,/* 0xC8-0xCF */ + 0x949A,0x949B,0x949C,0x94A3,0x94A4,0x94AB,0x94AA,0x94AD,/* 0xD0-0xD7 */ + 0x94AC,0x94AF,0x94B0,0x94B2,0x94B4,0x94B6,0x94B7,0x94B8,/* 0xD8-0xDF */ + 0x94B9,0x94BA,0x94BC,0x94BD,0x94BF,0x94C4,0x94C8,0x94C9,/* 0xE0-0xE7 */ + 0x94CA,0x94CB,0x94CC,0x94CD,0x94CE,0x94D0,0x94D1,0x94D2,/* 0xE8-0xEF */ + 0x94D5,0x94D6,0x94D7,0x94D9,0x94D8,0x94DB,0x94DE,0x94DF,/* 0xF0-0xF7 */ + 0x94E0,0x94E2,0x94E4,0x94E5,0x94E7,0x94E8,0x94EA,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_EF[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x986F,0x9870,0x9871,0x9872,0x9873,0x9874,0x988B,0x988E,/* 0x40-0x47 */ + 0x9892,0x9895,0x9899,0x98A3,0x98A8,0x98A9,0x98AA,0x98AB,/* 0x48-0x4F */ + 0x98AC,0x98AD,0x98AE,0x98AF,0x98B0,0x98B1,0x98B2,0x98B3,/* 0x50-0x57 */ + 0x98B4,0x98B5,0x98B6,0x98B7,0x98B8,0x98B9,0x98BA,0x98BB,/* 0x58-0x5F */ + 0x98BC,0x98BD,0x98BE,0x98BF,0x98C0,0x98C1,0x98C2,0x98C3,/* 0x60-0x67 */ + 0x98C4,0x98C5,0x98C6,0x98C7,0x98C8,0x98C9,0x98CA,0x98CB,/* 0x68-0x6F */ + 0x98CC,0x98CD,0x98CF,0x98D0,0x98D4,0x98D6,0x98D7,0x98DB,/* 0x70-0x77 */ + 0x98DC,0x98DD,0x98E0,0x98E1,0x98E2,0x98E3,0x98E4,0x0000,/* 0x78-0x7F */ + + 0x98E5,0x98E6,0x98E9,0x98EA,0x98EB,0x98EC,0x98ED,0x98EE,/* 0x80-0x87 */ + 0x98EF,0x98F0,0x98F1,0x98F2,0x98F3,0x98F4,0x98F5,0x98F6,/* 0x88-0x8F */ + 0x98F7,0x98F8,0x98F9,0x98FA,0x98FB,0x98FC,0x98FD,0x98FE,/* 0x90-0x97 */ + 0x98FF,0x9900,0x9901,0x9902,0x9903,0x9904,0x9905,0x9906,/* 0x98-0x9F */ + 0x9907,0x94E9,0x94EB,0x94EE,0x94EF,0x94F3,0x94F4,0x94F5,/* 0xA0-0xA7 */ + 0x94F7,0x94F9,0x94FC,0x94FD,0x94FF,0x9503,0x9502,0x9506,/* 0xA8-0xAF */ + 0x9507,0x9509,0x950A,0x950D,0x950E,0x950F,0x9512,0x9513,/* 0xB0-0xB7 */ + 0x9514,0x9515,0x9516,0x9518,0x951B,0x951D,0x951E,0x951F,/* 0xB8-0xBF */ + 0x9522,0x952A,0x952B,0x9529,0x952C,0x9531,0x9532,0x9534,/* 0xC0-0xC7 */ + 0x9536,0x9537,0x9538,0x953C,0x953E,0x953F,0x9542,0x9535,/* 0xC8-0xCF */ + 0x9544,0x9545,0x9546,0x9549,0x954C,0x954E,0x954F,0x9552,/* 0xD0-0xD7 */ + 0x9553,0x9554,0x9556,0x9557,0x9558,0x9559,0x955B,0x955E,/* 0xD8-0xDF */ + 0x955F,0x955D,0x9561,0x9562,0x9564,0x9565,0x9566,0x9567,/* 0xE0-0xE7 */ + 0x9568,0x9569,0x956A,0x956B,0x956C,0x956F,0x9571,0x9572,/* 0xE8-0xEF */ + 0x9573,0x953A,0x77E7,0x77EC,0x96C9,0x79D5,0x79ED,0x79E3,/* 0xF0-0xF7 */ + 0x79EB,0x7A06,0x5D47,0x7A03,0x7A02,0x7A1E,0x7A14,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F0[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9908,0x9909,0x990A,0x990B,0x990C,0x990E,0x990F,0x9911,/* 0x40-0x47 */ + 0x9912,0x9913,0x9914,0x9915,0x9916,0x9917,0x9918,0x9919,/* 0x48-0x4F */ + 0x991A,0x991B,0x991C,0x991D,0x991E,0x991F,0x9920,0x9921,/* 0x50-0x57 */ + 0x9922,0x9923,0x9924,0x9925,0x9926,0x9927,0x9928,0x9929,/* 0x58-0x5F */ + 0x992A,0x992B,0x992C,0x992D,0x992F,0x9930,0x9931,0x9932,/* 0x60-0x67 */ + 0x9933,0x9934,0x9935,0x9936,0x9937,0x9938,0x9939,0x993A,/* 0x68-0x6F */ + 0x993B,0x993C,0x993D,0x993E,0x993F,0x9940,0x9941,0x9942,/* 0x70-0x77 */ + 0x9943,0x9944,0x9945,0x9946,0x9947,0x9948,0x9949,0x0000,/* 0x78-0x7F */ + + 0x994A,0x994B,0x994C,0x994D,0x994E,0x994F,0x9950,0x9951,/* 0x80-0x87 */ + 0x9952,0x9953,0x9956,0x9957,0x9958,0x9959,0x995A,0x995B,/* 0x88-0x8F */ + 0x995C,0x995D,0x995E,0x995F,0x9960,0x9961,0x9962,0x9964,/* 0x90-0x97 */ + 0x9966,0x9973,0x9978,0x9979,0x997B,0x997E,0x9982,0x9983,/* 0x98-0x9F */ + 0x9989,0x7A39,0x7A37,0x7A51,0x9ECF,0x99A5,0x7A70,0x7688,/* 0xA0-0xA7 */ + 0x768E,0x7693,0x7699,0x76A4,0x74DE,0x74E0,0x752C,0x9E20,/* 0xA8-0xAF */ + 0x9E22,0x9E28,0x9E29,0x9E2A,0x9E2B,0x9E2C,0x9E32,0x9E31,/* 0xB0-0xB7 */ + 0x9E36,0x9E38,0x9E37,0x9E39,0x9E3A,0x9E3E,0x9E41,0x9E42,/* 0xB8-0xBF */ + 0x9E44,0x9E46,0x9E47,0x9E48,0x9E49,0x9E4B,0x9E4C,0x9E4E,/* 0xC0-0xC7 */ + 0x9E51,0x9E55,0x9E57,0x9E5A,0x9E5B,0x9E5C,0x9E5E,0x9E63,/* 0xC8-0xCF */ + 0x9E66,0x9E67,0x9E68,0x9E69,0x9E6A,0x9E6B,0x9E6C,0x9E71,/* 0xD0-0xD7 */ + 0x9E6D,0x9E73,0x7592,0x7594,0x7596,0x75A0,0x759D,0x75AC,/* 0xD8-0xDF */ + 0x75A3,0x75B3,0x75B4,0x75B8,0x75C4,0x75B1,0x75B0,0x75C3,/* 0xE0-0xE7 */ + 0x75C2,0x75D6,0x75CD,0x75E3,0x75E8,0x75E6,0x75E4,0x75EB,/* 0xE8-0xEF */ + 0x75E7,0x7603,0x75F1,0x75FC,0x75FF,0x7610,0x7600,0x7605,/* 0xF0-0xF7 */ + 0x760C,0x7617,0x760A,0x7625,0x7618,0x7615,0x7619,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F1[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x998C,0x998E,0x999A,0x999B,0x999C,0x999D,0x999E,0x999F,/* 0x40-0x47 */ + 0x99A0,0x99A1,0x99A2,0x99A3,0x99A4,0x99A6,0x99A7,0x99A9,/* 0x48-0x4F */ + 0x99AA,0x99AB,0x99AC,0x99AD,0x99AE,0x99AF,0x99B0,0x99B1,/* 0x50-0x57 */ + 0x99B2,0x99B3,0x99B4,0x99B5,0x99B6,0x99B7,0x99B8,0x99B9,/* 0x58-0x5F */ + 0x99BA,0x99BB,0x99BC,0x99BD,0x99BE,0x99BF,0x99C0,0x99C1,/* 0x60-0x67 */ + 0x99C2,0x99C3,0x99C4,0x99C5,0x99C6,0x99C7,0x99C8,0x99C9,/* 0x68-0x6F */ + 0x99CA,0x99CB,0x99CC,0x99CD,0x99CE,0x99CF,0x99D0,0x99D1,/* 0x70-0x77 */ + 0x99D2,0x99D3,0x99D4,0x99D5,0x99D6,0x99D7,0x99D8,0x0000,/* 0x78-0x7F */ + + 0x99D9,0x99DA,0x99DB,0x99DC,0x99DD,0x99DE,0x99DF,0x99E0,/* 0x80-0x87 */ + 0x99E1,0x99E2,0x99E3,0x99E4,0x99E5,0x99E6,0x99E7,0x99E8,/* 0x88-0x8F */ + 0x99E9,0x99EA,0x99EB,0x99EC,0x99ED,0x99EE,0x99EF,0x99F0,/* 0x90-0x97 */ + 0x99F1,0x99F2,0x99F3,0x99F4,0x99F5,0x99F6,0x99F7,0x99F8,/* 0x98-0x9F */ + 0x99F9,0x761B,0x763C,0x7622,0x7620,0x7640,0x762D,0x7630,/* 0xA0-0xA7 */ + 0x763F,0x7635,0x7643,0x763E,0x7633,0x764D,0x765E,0x7654,/* 0xA8-0xAF */ + 0x765C,0x7656,0x766B,0x766F,0x7FCA,0x7AE6,0x7A78,0x7A79,/* 0xB0-0xB7 */ + 0x7A80,0x7A86,0x7A88,0x7A95,0x7AA6,0x7AA0,0x7AAC,0x7AA8,/* 0xB8-0xBF */ + 0x7AAD,0x7AB3,0x8864,0x8869,0x8872,0x887D,0x887F,0x8882,/* 0xC0-0xC7 */ + 0x88A2,0x88C6,0x88B7,0x88BC,0x88C9,0x88E2,0x88CE,0x88E3,/* 0xC8-0xCF */ + 0x88E5,0x88F1,0x891A,0x88FC,0x88E8,0x88FE,0x88F0,0x8921,/* 0xD0-0xD7 */ + 0x8919,0x8913,0x891B,0x890A,0x8934,0x892B,0x8936,0x8941,/* 0xD8-0xDF */ + 0x8966,0x897B,0x758B,0x80E5,0x76B2,0x76B4,0x77DC,0x8012,/* 0xE0-0xE7 */ + 0x8014,0x8016,0x801C,0x8020,0x8022,0x8025,0x8026,0x8027,/* 0xE8-0xEF */ + 0x8029,0x8028,0x8031,0x800B,0x8035,0x8043,0x8046,0x804D,/* 0xF0-0xF7 */ + 0x8052,0x8069,0x8071,0x8983,0x9878,0x9880,0x9883,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F2[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x99FA,0x99FB,0x99FC,0x99FD,0x99FE,0x99FF,0x9A00,0x9A01,/* 0x40-0x47 */ + 0x9A02,0x9A03,0x9A04,0x9A05,0x9A06,0x9A07,0x9A08,0x9A09,/* 0x48-0x4F */ + 0x9A0A,0x9A0B,0x9A0C,0x9A0D,0x9A0E,0x9A0F,0x9A10,0x9A11,/* 0x50-0x57 */ + 0x9A12,0x9A13,0x9A14,0x9A15,0x9A16,0x9A17,0x9A18,0x9A19,/* 0x58-0x5F */ + 0x9A1A,0x9A1B,0x9A1C,0x9A1D,0x9A1E,0x9A1F,0x9A20,0x9A21,/* 0x60-0x67 */ + 0x9A22,0x9A23,0x9A24,0x9A25,0x9A26,0x9A27,0x9A28,0x9A29,/* 0x68-0x6F */ + 0x9A2A,0x9A2B,0x9A2C,0x9A2D,0x9A2E,0x9A2F,0x9A30,0x9A31,/* 0x70-0x77 */ + 0x9A32,0x9A33,0x9A34,0x9A35,0x9A36,0x9A37,0x9A38,0x0000,/* 0x78-0x7F */ + + 0x9A39,0x9A3A,0x9A3B,0x9A3C,0x9A3D,0x9A3E,0x9A3F,0x9A40,/* 0x80-0x87 */ + 0x9A41,0x9A42,0x9A43,0x9A44,0x9A45,0x9A46,0x9A47,0x9A48,/* 0x88-0x8F */ + 0x9A49,0x9A4A,0x9A4B,0x9A4C,0x9A4D,0x9A4E,0x9A4F,0x9A50,/* 0x90-0x97 */ + 0x9A51,0x9A52,0x9A53,0x9A54,0x9A55,0x9A56,0x9A57,0x9A58,/* 0x98-0x9F */ + 0x9A59,0x9889,0x988C,0x988D,0x988F,0x9894,0x989A,0x989B,/* 0xA0-0xA7 */ + 0x989E,0x989F,0x98A1,0x98A2,0x98A5,0x98A6,0x864D,0x8654,/* 0xA8-0xAF */ + 0x866C,0x866E,0x867F,0x867A,0x867C,0x867B,0x86A8,0x868D,/* 0xB0-0xB7 */ + 0x868B,0x86AC,0x869D,0x86A7,0x86A3,0x86AA,0x8693,0x86A9,/* 0xB8-0xBF */ + 0x86B6,0x86C4,0x86B5,0x86CE,0x86B0,0x86BA,0x86B1,0x86AF,/* 0xC0-0xC7 */ + 0x86C9,0x86CF,0x86B4,0x86E9,0x86F1,0x86F2,0x86ED,0x86F3,/* 0xC8-0xCF */ + 0x86D0,0x8713,0x86DE,0x86F4,0x86DF,0x86D8,0x86D1,0x8703,/* 0xD0-0xD7 */ + 0x8707,0x86F8,0x8708,0x870A,0x870D,0x8709,0x8723,0x873B,/* 0xD8-0xDF */ + 0x871E,0x8725,0x872E,0x871A,0x873E,0x8748,0x8734,0x8731,/* 0xE0-0xE7 */ + 0x8729,0x8737,0x873F,0x8782,0x8722,0x877D,0x877E,0x877B,/* 0xE8-0xEF */ + 0x8760,0x8770,0x874C,0x876E,0x878B,0x8753,0x8763,0x877C,/* 0xF0-0xF7 */ + 0x8764,0x8759,0x8765,0x8793,0x87AF,0x87A8,0x87D2,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F3[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9A5A,0x9A5B,0x9A5C,0x9A5D,0x9A5E,0x9A5F,0x9A60,0x9A61,/* 0x40-0x47 */ + 0x9A62,0x9A63,0x9A64,0x9A65,0x9A66,0x9A67,0x9A68,0x9A69,/* 0x48-0x4F */ + 0x9A6A,0x9A6B,0x9A72,0x9A83,0x9A89,0x9A8D,0x9A8E,0x9A94,/* 0x50-0x57 */ + 0x9A95,0x9A99,0x9AA6,0x9AA9,0x9AAA,0x9AAB,0x9AAC,0x9AAD,/* 0x58-0x5F */ + 0x9AAE,0x9AAF,0x9AB2,0x9AB3,0x9AB4,0x9AB5,0x9AB9,0x9ABB,/* 0x60-0x67 */ + 0x9ABD,0x9ABE,0x9ABF,0x9AC3,0x9AC4,0x9AC6,0x9AC7,0x9AC8,/* 0x68-0x6F */ + 0x9AC9,0x9ACA,0x9ACD,0x9ACE,0x9ACF,0x9AD0,0x9AD2,0x9AD4,/* 0x70-0x77 */ + 0x9AD5,0x9AD6,0x9AD7,0x9AD9,0x9ADA,0x9ADB,0x9ADC,0x0000,/* 0x78-0x7F */ + + 0x9ADD,0x9ADE,0x9AE0,0x9AE2,0x9AE3,0x9AE4,0x9AE5,0x9AE7,/* 0x80-0x87 */ + 0x9AE8,0x9AE9,0x9AEA,0x9AEC,0x9AEE,0x9AF0,0x9AF1,0x9AF2,/* 0x88-0x8F */ + 0x9AF3,0x9AF4,0x9AF5,0x9AF6,0x9AF7,0x9AF8,0x9AFA,0x9AFC,/* 0x90-0x97 */ + 0x9AFD,0x9AFE,0x9AFF,0x9B00,0x9B01,0x9B02,0x9B04,0x9B05,/* 0x98-0x9F */ + 0x9B06,0x87C6,0x8788,0x8785,0x87AD,0x8797,0x8783,0x87AB,/* 0xA0-0xA7 */ + 0x87E5,0x87AC,0x87B5,0x87B3,0x87CB,0x87D3,0x87BD,0x87D1,/* 0xA8-0xAF */ + 0x87C0,0x87CA,0x87DB,0x87EA,0x87E0,0x87EE,0x8816,0x8813,/* 0xB0-0xB7 */ + 0x87FE,0x880A,0x881B,0x8821,0x8839,0x883C,0x7F36,0x7F42,/* 0xB8-0xBF */ + 0x7F44,0x7F45,0x8210,0x7AFA,0x7AFD,0x7B08,0x7B03,0x7B04,/* 0xC0-0xC7 */ + 0x7B15,0x7B0A,0x7B2B,0x7B0F,0x7B47,0x7B38,0x7B2A,0x7B19,/* 0xC8-0xCF */ + 0x7B2E,0x7B31,0x7B20,0x7B25,0x7B24,0x7B33,0x7B3E,0x7B1E,/* 0xD0-0xD7 */ + 0x7B58,0x7B5A,0x7B45,0x7B75,0x7B4C,0x7B5D,0x7B60,0x7B6E,/* 0xD8-0xDF */ + 0x7B7B,0x7B62,0x7B72,0x7B71,0x7B90,0x7BA6,0x7BA7,0x7BB8,/* 0xE0-0xE7 */ + 0x7BAC,0x7B9D,0x7BA8,0x7B85,0x7BAA,0x7B9C,0x7BA2,0x7BAB,/* 0xE8-0xEF */ + 0x7BB4,0x7BD1,0x7BC1,0x7BCC,0x7BDD,0x7BDA,0x7BE5,0x7BE6,/* 0xF0-0xF7 */ + 0x7BEA,0x7C0C,0x7BFE,0x7BFC,0x7C0F,0x7C16,0x7C0B,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F4[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9B07,0x9B09,0x9B0A,0x9B0B,0x9B0C,0x9B0D,0x9B0E,0x9B10,/* 0x40-0x47 */ + 0x9B11,0x9B12,0x9B14,0x9B15,0x9B16,0x9B17,0x9B18,0x9B19,/* 0x48-0x4F */ + 0x9B1A,0x9B1B,0x9B1C,0x9B1D,0x9B1E,0x9B20,0x9B21,0x9B22,/* 0x50-0x57 */ + 0x9B24,0x9B25,0x9B26,0x9B27,0x9B28,0x9B29,0x9B2A,0x9B2B,/* 0x58-0x5F */ + 0x9B2C,0x9B2D,0x9B2E,0x9B30,0x9B31,0x9B33,0x9B34,0x9B35,/* 0x60-0x67 */ + 0x9B36,0x9B37,0x9B38,0x9B39,0x9B3A,0x9B3D,0x9B3E,0x9B3F,/* 0x68-0x6F */ + 0x9B40,0x9B46,0x9B4A,0x9B4B,0x9B4C,0x9B4E,0x9B50,0x9B52,/* 0x70-0x77 */ + 0x9B53,0x9B55,0x9B56,0x9B57,0x9B58,0x9B59,0x9B5A,0x0000,/* 0x78-0x7F */ + + 0x9B5B,0x9B5C,0x9B5D,0x9B5E,0x9B5F,0x9B60,0x9B61,0x9B62,/* 0x80-0x87 */ + 0x9B63,0x9B64,0x9B65,0x9B66,0x9B67,0x9B68,0x9B69,0x9B6A,/* 0x88-0x8F */ + 0x9B6B,0x9B6C,0x9B6D,0x9B6E,0x9B6F,0x9B70,0x9B71,0x9B72,/* 0x90-0x97 */ + 0x9B73,0x9B74,0x9B75,0x9B76,0x9B77,0x9B78,0x9B79,0x9B7A,/* 0x98-0x9F */ + 0x9B7B,0x7C1F,0x7C2A,0x7C26,0x7C38,0x7C41,0x7C40,0x81FE,/* 0xA0-0xA7 */ + 0x8201,0x8202,0x8204,0x81EC,0x8844,0x8221,0x8222,0x8223,/* 0xA8-0xAF */ + 0x822D,0x822F,0x8228,0x822B,0x8238,0x823B,0x8233,0x8234,/* 0xB0-0xB7 */ + 0x823E,0x8244,0x8249,0x824B,0x824F,0x825A,0x825F,0x8268,/* 0xB8-0xBF */ + 0x887E,0x8885,0x8888,0x88D8,0x88DF,0x895E,0x7F9D,0x7F9F,/* 0xC0-0xC7 */ + 0x7FA7,0x7FAF,0x7FB0,0x7FB2,0x7C7C,0x6549,0x7C91,0x7C9D,/* 0xC8-0xCF */ + 0x7C9C,0x7C9E,0x7CA2,0x7CB2,0x7CBC,0x7CBD,0x7CC1,0x7CC7,/* 0xD0-0xD7 */ + 0x7CCC,0x7CCD,0x7CC8,0x7CC5,0x7CD7,0x7CE8,0x826E,0x66A8,/* 0xD8-0xDF */ + 0x7FBF,0x7FCE,0x7FD5,0x7FE5,0x7FE1,0x7FE6,0x7FE9,0x7FEE,/* 0xE0-0xE7 */ + 0x7FF3,0x7CF8,0x7D77,0x7DA6,0x7DAE,0x7E47,0x7E9B,0x9EB8,/* 0xE8-0xEF */ + 0x9EB4,0x8D73,0x8D84,0x8D94,0x8D91,0x8DB1,0x8D67,0x8D6D,/* 0xF0-0xF7 */ + 0x8C47,0x8C49,0x914A,0x9150,0x914E,0x914F,0x9164,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F5[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9B7C,0x9B7D,0x9B7E,0x9B7F,0x9B80,0x9B81,0x9B82,0x9B83,/* 0x40-0x47 */ + 0x9B84,0x9B85,0x9B86,0x9B87,0x9B88,0x9B89,0x9B8A,0x9B8B,/* 0x48-0x4F */ + 0x9B8C,0x9B8D,0x9B8E,0x9B8F,0x9B90,0x9B91,0x9B92,0x9B93,/* 0x50-0x57 */ + 0x9B94,0x9B95,0x9B96,0x9B97,0x9B98,0x9B99,0x9B9A,0x9B9B,/* 0x58-0x5F */ + 0x9B9C,0x9B9D,0x9B9E,0x9B9F,0x9BA0,0x9BA1,0x9BA2,0x9BA3,/* 0x60-0x67 */ + 0x9BA4,0x9BA5,0x9BA6,0x9BA7,0x9BA8,0x9BA9,0x9BAA,0x9BAB,/* 0x68-0x6F */ + 0x9BAC,0x9BAD,0x9BAE,0x9BAF,0x9BB0,0x9BB1,0x9BB2,0x9BB3,/* 0x70-0x77 */ + 0x9BB4,0x9BB5,0x9BB6,0x9BB7,0x9BB8,0x9BB9,0x9BBA,0x0000,/* 0x78-0x7F */ + + 0x9BBB,0x9BBC,0x9BBD,0x9BBE,0x9BBF,0x9BC0,0x9BC1,0x9BC2,/* 0x80-0x87 */ + 0x9BC3,0x9BC4,0x9BC5,0x9BC6,0x9BC7,0x9BC8,0x9BC9,0x9BCA,/* 0x88-0x8F */ + 0x9BCB,0x9BCC,0x9BCD,0x9BCE,0x9BCF,0x9BD0,0x9BD1,0x9BD2,/* 0x90-0x97 */ + 0x9BD3,0x9BD4,0x9BD5,0x9BD6,0x9BD7,0x9BD8,0x9BD9,0x9BDA,/* 0x98-0x9F */ + 0x9BDB,0x9162,0x9161,0x9170,0x9169,0x916F,0x917D,0x917E,/* 0xA0-0xA7 */ + 0x9172,0x9174,0x9179,0x918C,0x9185,0x9190,0x918D,0x9191,/* 0xA8-0xAF */ + 0x91A2,0x91A3,0x91AA,0x91AD,0x91AE,0x91AF,0x91B5,0x91B4,/* 0xB0-0xB7 */ + 0x91BA,0x8C55,0x9E7E,0x8DB8,0x8DEB,0x8E05,0x8E59,0x8E69,/* 0xB8-0xBF */ + 0x8DB5,0x8DBF,0x8DBC,0x8DBA,0x8DC4,0x8DD6,0x8DD7,0x8DDA,/* 0xC0-0xC7 */ + 0x8DDE,0x8DCE,0x8DCF,0x8DDB,0x8DC6,0x8DEC,0x8DF7,0x8DF8,/* 0xC8-0xCF */ + 0x8DE3,0x8DF9,0x8DFB,0x8DE4,0x8E09,0x8DFD,0x8E14,0x8E1D,/* 0xD0-0xD7 */ + 0x8E1F,0x8E2C,0x8E2E,0x8E23,0x8E2F,0x8E3A,0x8E40,0x8E39,/* 0xD8-0xDF */ + 0x8E35,0x8E3D,0x8E31,0x8E49,0x8E41,0x8E42,0x8E51,0x8E52,/* 0xE0-0xE7 */ + 0x8E4A,0x8E70,0x8E76,0x8E7C,0x8E6F,0x8E74,0x8E85,0x8E8F,/* 0xE8-0xEF */ + 0x8E94,0x8E90,0x8E9C,0x8E9E,0x8C78,0x8C82,0x8C8A,0x8C85,/* 0xF0-0xF7 */ + 0x8C98,0x8C94,0x659B,0x89D6,0x89DE,0x89DA,0x89DC,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F6[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9BDC,0x9BDD,0x9BDE,0x9BDF,0x9BE0,0x9BE1,0x9BE2,0x9BE3,/* 0x40-0x47 */ + 0x9BE4,0x9BE5,0x9BE6,0x9BE7,0x9BE8,0x9BE9,0x9BEA,0x9BEB,/* 0x48-0x4F */ + 0x9BEC,0x9BED,0x9BEE,0x9BEF,0x9BF0,0x9BF1,0x9BF2,0x9BF3,/* 0x50-0x57 */ + 0x9BF4,0x9BF5,0x9BF6,0x9BF7,0x9BF8,0x9BF9,0x9BFA,0x9BFB,/* 0x58-0x5F */ + 0x9BFC,0x9BFD,0x9BFE,0x9BFF,0x9C00,0x9C01,0x9C02,0x9C03,/* 0x60-0x67 */ + 0x9C04,0x9C05,0x9C06,0x9C07,0x9C08,0x9C09,0x9C0A,0x9C0B,/* 0x68-0x6F */ + 0x9C0C,0x9C0D,0x9C0E,0x9C0F,0x9C10,0x9C11,0x9C12,0x9C13,/* 0x70-0x77 */ + 0x9C14,0x9C15,0x9C16,0x9C17,0x9C18,0x9C19,0x9C1A,0x0000,/* 0x78-0x7F */ + + 0x9C1B,0x9C1C,0x9C1D,0x9C1E,0x9C1F,0x9C20,0x9C21,0x9C22,/* 0x80-0x87 */ + 0x9C23,0x9C24,0x9C25,0x9C26,0x9C27,0x9C28,0x9C29,0x9C2A,/* 0x88-0x8F */ + 0x9C2B,0x9C2C,0x9C2D,0x9C2E,0x9C2F,0x9C30,0x9C31,0x9C32,/* 0x90-0x97 */ + 0x9C33,0x9C34,0x9C35,0x9C36,0x9C37,0x9C38,0x9C39,0x9C3A,/* 0x98-0x9F */ + 0x9C3B,0x89E5,0x89EB,0x89EF,0x8A3E,0x8B26,0x9753,0x96E9,/* 0xA0-0xA7 */ + 0x96F3,0x96EF,0x9706,0x9701,0x9708,0x970F,0x970E,0x972A,/* 0xA8-0xAF */ + 0x972D,0x9730,0x973E,0x9F80,0x9F83,0x9F85,0x9F86,0x9F87,/* 0xB0-0xB7 */ + 0x9F88,0x9F89,0x9F8A,0x9F8C,0x9EFE,0x9F0B,0x9F0D,0x96B9,/* 0xB8-0xBF */ + 0x96BC,0x96BD,0x96CE,0x96D2,0x77BF,0x96E0,0x928E,0x92AE,/* 0xC0-0xC7 */ + 0x92C8,0x933E,0x936A,0x93CA,0x938F,0x943E,0x946B,0x9C7F,/* 0xC8-0xCF */ + 0x9C82,0x9C85,0x9C86,0x9C87,0x9C88,0x7A23,0x9C8B,0x9C8E,/* 0xD0-0xD7 */ + 0x9C90,0x9C91,0x9C92,0x9C94,0x9C95,0x9C9A,0x9C9B,0x9C9E,/* 0xD8-0xDF */ + 0x9C9F,0x9CA0,0x9CA1,0x9CA2,0x9CA3,0x9CA5,0x9CA6,0x9CA7,/* 0xE0-0xE7 */ + 0x9CA8,0x9CA9,0x9CAB,0x9CAD,0x9CAE,0x9CB0,0x9CB1,0x9CB2,/* 0xE8-0xEF */ + 0x9CB3,0x9CB4,0x9CB5,0x9CB6,0x9CB7,0x9CBA,0x9CBB,0x9CBC,/* 0xF0-0xF7 */ + 0x9CBD,0x9CC4,0x9CC5,0x9CC6,0x9CC7,0x9CCA,0x9CCB,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F7[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9C3C,0x9C3D,0x9C3E,0x9C3F,0x9C40,0x9C41,0x9C42,0x9C43,/* 0x40-0x47 */ + 0x9C44,0x9C45,0x9C46,0x9C47,0x9C48,0x9C49,0x9C4A,0x9C4B,/* 0x48-0x4F */ + 0x9C4C,0x9C4D,0x9C4E,0x9C4F,0x9C50,0x9C51,0x9C52,0x9C53,/* 0x50-0x57 */ + 0x9C54,0x9C55,0x9C56,0x9C57,0x9C58,0x9C59,0x9C5A,0x9C5B,/* 0x58-0x5F */ + 0x9C5C,0x9C5D,0x9C5E,0x9C5F,0x9C60,0x9C61,0x9C62,0x9C63,/* 0x60-0x67 */ + 0x9C64,0x9C65,0x9C66,0x9C67,0x9C68,0x9C69,0x9C6A,0x9C6B,/* 0x68-0x6F */ + 0x9C6C,0x9C6D,0x9C6E,0x9C6F,0x9C70,0x9C71,0x9C72,0x9C73,/* 0x70-0x77 */ + 0x9C74,0x9C75,0x9C76,0x9C77,0x9C78,0x9C79,0x9C7A,0x0000,/* 0x78-0x7F */ + + 0x9C7B,0x9C7D,0x9C7E,0x9C80,0x9C83,0x9C84,0x9C89,0x9C8A,/* 0x80-0x87 */ + 0x9C8C,0x9C8F,0x9C93,0x9C96,0x9C97,0x9C98,0x9C99,0x9C9D,/* 0x88-0x8F */ + 0x9CAA,0x9CAC,0x9CAF,0x9CB9,0x9CBE,0x9CBF,0x9CC0,0x9CC1,/* 0x90-0x97 */ + 0x9CC2,0x9CC8,0x9CC9,0x9CD1,0x9CD2,0x9CDA,0x9CDB,0x9CE0,/* 0x98-0x9F */ + 0x9CE1,0x9CCC,0x9CCD,0x9CCE,0x9CCF,0x9CD0,0x9CD3,0x9CD4,/* 0xA0-0xA7 */ + 0x9CD5,0x9CD7,0x9CD8,0x9CD9,0x9CDC,0x9CDD,0x9CDF,0x9CE2,/* 0xA8-0xAF */ + 0x977C,0x9785,0x9791,0x9792,0x9794,0x97AF,0x97AB,0x97A3,/* 0xB0-0xB7 */ + 0x97B2,0x97B4,0x9AB1,0x9AB0,0x9AB7,0x9E58,0x9AB6,0x9ABA,/* 0xB8-0xBF */ + 0x9ABC,0x9AC1,0x9AC0,0x9AC5,0x9AC2,0x9ACB,0x9ACC,0x9AD1,/* 0xC0-0xC7 */ + 0x9B45,0x9B43,0x9B47,0x9B49,0x9B48,0x9B4D,0x9B51,0x98E8,/* 0xC8-0xCF */ + 0x990D,0x992E,0x9955,0x9954,0x9ADF,0x9AE1,0x9AE6,0x9AEF,/* 0xD0-0xD7 */ + 0x9AEB,0x9AFB,0x9AED,0x9AF9,0x9B08,0x9B0F,0x9B13,0x9B1F,/* 0xD8-0xDF */ + 0x9B23,0x9EBD,0x9EBE,0x7E3B,0x9E82,0x9E87,0x9E88,0x9E8B,/* 0xE0-0xE7 */ + 0x9E92,0x93D6,0x9E9D,0x9E9F,0x9EDB,0x9EDC,0x9EDD,0x9EE0,/* 0xE8-0xEF */ + 0x9EDF,0x9EE2,0x9EE9,0x9EE7,0x9EE5,0x9EEA,0x9EEF,0x9F22,/* 0xF0-0xF7 */ + 0x9F2C,0x9F2F,0x9F39,0x9F37,0x9F3D,0x9F3E,0x9F44,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F8[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9CE3,0x9CE4,0x9CE5,0x9CE6,0x9CE7,0x9CE8,0x9CE9,0x9CEA,/* 0x40-0x47 */ + 0x9CEB,0x9CEC,0x9CED,0x9CEE,0x9CEF,0x9CF0,0x9CF1,0x9CF2,/* 0x48-0x4F */ + 0x9CF3,0x9CF4,0x9CF5,0x9CF6,0x9CF7,0x9CF8,0x9CF9,0x9CFA,/* 0x50-0x57 */ + 0x9CFB,0x9CFC,0x9CFD,0x9CFE,0x9CFF,0x9D00,0x9D01,0x9D02,/* 0x58-0x5F */ + 0x9D03,0x9D04,0x9D05,0x9D06,0x9D07,0x9D08,0x9D09,0x9D0A,/* 0x60-0x67 */ + 0x9D0B,0x9D0C,0x9D0D,0x9D0E,0x9D0F,0x9D10,0x9D11,0x9D12,/* 0x68-0x6F */ + 0x9D13,0x9D14,0x9D15,0x9D16,0x9D17,0x9D18,0x9D19,0x9D1A,/* 0x70-0x77 */ + 0x9D1B,0x9D1C,0x9D1D,0x9D1E,0x9D1F,0x9D20,0x9D21,0x0000,/* 0x78-0x7F */ + + 0x9D22,0x9D23,0x9D24,0x9D25,0x9D26,0x9D27,0x9D28,0x9D29,/* 0x80-0x87 */ + 0x9D2A,0x9D2B,0x9D2C,0x9D2D,0x9D2E,0x9D2F,0x9D30,0x9D31,/* 0x88-0x8F */ + 0x9D32,0x9D33,0x9D34,0x9D35,0x9D36,0x9D37,0x9D38,0x9D39,/* 0x90-0x97 */ + 0x9D3A,0x9D3B,0x9D3C,0x9D3D,0x9D3E,0x9D3F,0x9D40,0x9D41,/* 0x98-0x9F */ + 0x9D42,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xA0-0xA7 */ +}; + +static const wchar_t c2u_F9[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9D43,0x9D44,0x9D45,0x9D46,0x9D47,0x9D48,0x9D49,0x9D4A,/* 0x40-0x47 */ + 0x9D4B,0x9D4C,0x9D4D,0x9D4E,0x9D4F,0x9D50,0x9D51,0x9D52,/* 0x48-0x4F */ + 0x9D53,0x9D54,0x9D55,0x9D56,0x9D57,0x9D58,0x9D59,0x9D5A,/* 0x50-0x57 */ + 0x9D5B,0x9D5C,0x9D5D,0x9D5E,0x9D5F,0x9D60,0x9D61,0x9D62,/* 0x58-0x5F */ + 0x9D63,0x9D64,0x9D65,0x9D66,0x9D67,0x9D68,0x9D69,0x9D6A,/* 0x60-0x67 */ + 0x9D6B,0x9D6C,0x9D6D,0x9D6E,0x9D6F,0x9D70,0x9D71,0x9D72,/* 0x68-0x6F */ + 0x9D73,0x9D74,0x9D75,0x9D76,0x9D77,0x9D78,0x9D79,0x9D7A,/* 0x70-0x77 */ + 0x9D7B,0x9D7C,0x9D7D,0x9D7E,0x9D7F,0x9D80,0x9D81,0x0000,/* 0x78-0x7F */ + + 0x9D82,0x9D83,0x9D84,0x9D85,0x9D86,0x9D87,0x9D88,0x9D89,/* 0x80-0x87 */ + 0x9D8A,0x9D8B,0x9D8C,0x9D8D,0x9D8E,0x9D8F,0x9D90,0x9D91,/* 0x88-0x8F */ + 0x9D92,0x9D93,0x9D94,0x9D95,0x9D96,0x9D97,0x9D98,0x9D99,/* 0x90-0x97 */ + 0x9D9A,0x9D9B,0x9D9C,0x9D9D,0x9D9E,0x9D9F,0x9DA0,0x9DA1,/* 0x98-0x9F */ + 0x9DA2,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xA0-0xA7 */ +}; + +static const wchar_t c2u_FA[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9DA3,0x9DA4,0x9DA5,0x9DA6,0x9DA7,0x9DA8,0x9DA9,0x9DAA,/* 0x40-0x47 */ + 0x9DAB,0x9DAC,0x9DAD,0x9DAE,0x9DAF,0x9DB0,0x9DB1,0x9DB2,/* 0x48-0x4F */ + 0x9DB3,0x9DB4,0x9DB5,0x9DB6,0x9DB7,0x9DB8,0x9DB9,0x9DBA,/* 0x50-0x57 */ + 0x9DBB,0x9DBC,0x9DBD,0x9DBE,0x9DBF,0x9DC0,0x9DC1,0x9DC2,/* 0x58-0x5F */ + 0x9DC3,0x9DC4,0x9DC5,0x9DC6,0x9DC7,0x9DC8,0x9DC9,0x9DCA,/* 0x60-0x67 */ + 0x9DCB,0x9DCC,0x9DCD,0x9DCE,0x9DCF,0x9DD0,0x9DD1,0x9DD2,/* 0x68-0x6F */ + 0x9DD3,0x9DD4,0x9DD5,0x9DD6,0x9DD7,0x9DD8,0x9DD9,0x9DDA,/* 0x70-0x77 */ + 0x9DDB,0x9DDC,0x9DDD,0x9DDE,0x9DDF,0x9DE0,0x9DE1,0x0000,/* 0x78-0x7F */ + + 0x9DE2,0x9DE3,0x9DE4,0x9DE5,0x9DE6,0x9DE7,0x9DE8,0x9DE9,/* 0x80-0x87 */ + 0x9DEA,0x9DEB,0x9DEC,0x9DED,0x9DEE,0x9DEF,0x9DF0,0x9DF1,/* 0x88-0x8F */ + 0x9DF2,0x9DF3,0x9DF4,0x9DF5,0x9DF6,0x9DF7,0x9DF8,0x9DF9,/* 0x90-0x97 */ + 0x9DFA,0x9DFB,0x9DFC,0x9DFD,0x9DFE,0x9DFF,0x9E00,0x9E01,/* 0x98-0x9F */ + 0x9E02,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xA0-0xA7 */ +}; + +static const wchar_t c2u_FB[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9E03,0x9E04,0x9E05,0x9E06,0x9E07,0x9E08,0x9E09,0x9E0A,/* 0x40-0x47 */ + 0x9E0B,0x9E0C,0x9E0D,0x9E0E,0x9E0F,0x9E10,0x9E11,0x9E12,/* 0x48-0x4F */ + 0x9E13,0x9E14,0x9E15,0x9E16,0x9E17,0x9E18,0x9E19,0x9E1A,/* 0x50-0x57 */ + 0x9E1B,0x9E1C,0x9E1D,0x9E1E,0x9E24,0x9E27,0x9E2E,0x9E30,/* 0x58-0x5F */ + 0x9E34,0x9E3B,0x9E3C,0x9E40,0x9E4D,0x9E50,0x9E52,0x9E53,/* 0x60-0x67 */ + 0x9E54,0x9E56,0x9E59,0x9E5D,0x9E5F,0x9E60,0x9E61,0x9E62,/* 0x68-0x6F */ + 0x9E65,0x9E6E,0x9E6F,0x9E72,0x9E74,0x9E75,0x9E76,0x9E77,/* 0x70-0x77 */ + 0x9E78,0x9E79,0x9E7A,0x9E7B,0x9E7C,0x9E7D,0x9E80,0x0000,/* 0x78-0x7F */ + + 0x9E81,0x9E83,0x9E84,0x9E85,0x9E86,0x9E89,0x9E8A,0x9E8C,/* 0x80-0x87 */ + 0x9E8D,0x9E8E,0x9E8F,0x9E90,0x9E91,0x9E94,0x9E95,0x9E96,/* 0x88-0x8F */ + 0x9E97,0x9E98,0x9E99,0x9E9A,0x9E9B,0x9E9C,0x9E9E,0x9EA0,/* 0x90-0x97 */ + 0x9EA1,0x9EA2,0x9EA3,0x9EA4,0x9EA5,0x9EA7,0x9EA8,0x9EA9,/* 0x98-0x9F */ + 0x9EAA,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xA0-0xA7 */ +}; + +static const wchar_t c2u_FC[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9EAB,0x9EAC,0x9EAD,0x9EAE,0x9EAF,0x9EB0,0x9EB1,0x9EB2,/* 0x40-0x47 */ + 0x9EB3,0x9EB5,0x9EB6,0x9EB7,0x9EB9,0x9EBA,0x9EBC,0x9EBF,/* 0x48-0x4F */ + 0x9EC0,0x9EC1,0x9EC2,0x9EC3,0x9EC5,0x9EC6,0x9EC7,0x9EC8,/* 0x50-0x57 */ + 0x9ECA,0x9ECB,0x9ECC,0x9ED0,0x9ED2,0x9ED3,0x9ED5,0x9ED6,/* 0x58-0x5F */ + 0x9ED7,0x9ED9,0x9EDA,0x9EDE,0x9EE1,0x9EE3,0x9EE4,0x9EE6,/* 0x60-0x67 */ + 0x9EE8,0x9EEB,0x9EEC,0x9EED,0x9EEE,0x9EF0,0x9EF1,0x9EF2,/* 0x68-0x6F */ + 0x9EF3,0x9EF4,0x9EF5,0x9EF6,0x9EF7,0x9EF8,0x9EFA,0x9EFD,/* 0x70-0x77 */ + 0x9EFF,0x9F00,0x9F01,0x9F02,0x9F03,0x9F04,0x9F05,0x0000,/* 0x78-0x7F */ + + 0x9F06,0x9F07,0x9F08,0x9F09,0x9F0A,0x9F0C,0x9F0F,0x9F11,/* 0x80-0x87 */ + 0x9F12,0x9F14,0x9F15,0x9F16,0x9F18,0x9F1A,0x9F1B,0x9F1C,/* 0x88-0x8F */ + 0x9F1D,0x9F1E,0x9F1F,0x9F21,0x9F23,0x9F24,0x9F25,0x9F26,/* 0x90-0x97 */ + 0x9F27,0x9F28,0x9F29,0x9F2A,0x9F2B,0x9F2D,0x9F2E,0x9F30,/* 0x98-0x9F */ + 0x9F31,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xA0-0xA7 */ +}; + +static const wchar_t c2u_FD[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9F32,0x9F33,0x9F34,0x9F35,0x9F36,0x9F38,0x9F3A,0x9F3C,/* 0x40-0x47 */ + 0x9F3F,0x9F40,0x9F41,0x9F42,0x9F43,0x9F45,0x9F46,0x9F47,/* 0x48-0x4F */ + 0x9F48,0x9F49,0x9F4A,0x9F4B,0x9F4C,0x9F4D,0x9F4E,0x9F4F,/* 0x50-0x57 */ + 0x9F52,0x9F53,0x9F54,0x9F55,0x9F56,0x9F57,0x9F58,0x9F59,/* 0x58-0x5F */ + 0x9F5A,0x9F5B,0x9F5C,0x9F5D,0x9F5E,0x9F5F,0x9F60,0x9F61,/* 0x60-0x67 */ + 0x9F62,0x9F63,0x9F64,0x9F65,0x9F66,0x9F67,0x9F68,0x9F69,/* 0x68-0x6F */ + 0x9F6A,0x9F6B,0x9F6C,0x9F6D,0x9F6E,0x9F6F,0x9F70,0x9F71,/* 0x70-0x77 */ + 0x9F72,0x9F73,0x9F74,0x9F75,0x9F76,0x9F77,0x9F78,0x0000,/* 0x78-0x7F */ + + 0x9F79,0x9F7A,0x9F7B,0x9F7C,0x9F7D,0x9F7E,0x9F81,0x9F82,/* 0x80-0x87 */ + 0x9F8D,0x9F8E,0x9F8F,0x9F90,0x9F91,0x9F92,0x9F93,0x9F94,/* 0x88-0x8F */ + 0x9F95,0x9F96,0x9F97,0x9F98,0x9F9C,0x9F9D,0x9F9E,0x9FA1,/* 0x90-0x97 */ + 0x9FA2,0x9FA3,0x9FA4,0x9FA5,0xF92C,0xF979,0xF995,0xF9E7,/* 0x98-0x9F */ + 0xF9F1,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xA0-0xA7 */ +}; + +static const wchar_t c2u_FE[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0xFA0C,0xFA0D,0xFA0E,0xFA0F,0xFA11,0xFA13,0xFA14,0xFA18,/* 0x40-0x47 */ + 0xFA1F,0xFA20,0xFA21,0xFA23,0xFA24,0xFA27,0xFA28,0xFA29,/* 0x48-0x4F */ +}; + +static const wchar_t *page_charset2uni[256] = { + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, c2u_81, c2u_82, c2u_83, c2u_84, c2u_85, c2u_86, c2u_87, + c2u_88, c2u_89, c2u_8A, c2u_8B, c2u_8C, c2u_8D, c2u_8E, c2u_8F, + c2u_90, c2u_91, c2u_92, c2u_93, c2u_94, c2u_95, c2u_96, c2u_97, + c2u_98, c2u_99, c2u_9A, c2u_9B, c2u_9C, c2u_9D, c2u_9E, c2u_9F, + c2u_A0, c2u_A1, c2u_A2, c2u_A3, c2u_A4, c2u_A5, c2u_A6, c2u_A7, + c2u_A8, c2u_A9, c2u_AA, c2u_AB, c2u_AC, c2u_AD, c2u_AE, c2u_AF, + c2u_B0, c2u_B1, c2u_B2, c2u_B3, c2u_B4, c2u_B5, c2u_B6, c2u_B7, + c2u_B8, c2u_B9, c2u_BA, c2u_BB, c2u_BC, c2u_BD, c2u_BE, c2u_BF, + c2u_C0, c2u_C1, c2u_C2, c2u_C3, c2u_C4, c2u_C5, c2u_C6, c2u_C7, + c2u_C8, c2u_C9, c2u_CA, c2u_CB, c2u_CC, c2u_CD, c2u_CE, c2u_CF, + c2u_D0, c2u_D1, c2u_D2, c2u_D3, c2u_D4, c2u_D5, c2u_D6, c2u_D7, + c2u_D8, c2u_D9, c2u_DA, c2u_DB, c2u_DC, c2u_DD, c2u_DE, c2u_DF, + c2u_E0, c2u_E1, c2u_E2, c2u_E3, c2u_E4, c2u_E5, c2u_E6, c2u_E7, + c2u_E8, c2u_E9, c2u_EA, c2u_EB, c2u_EC, c2u_ED, c2u_EE, c2u_EF, + c2u_F0, c2u_F1, c2u_F2, c2u_F3, c2u_F4, c2u_F5, c2u_F6, c2u_F7, + c2u_F8, c2u_F9, c2u_FA, c2u_FB, c2u_FC, c2u_FD, c2u_FE, NULL, +}; + +static const unsigned char u2c_00[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xA1, 0xE8, 0x00, 0x00, 0x00, 0x00, 0xA1, 0xEC, /* 0xA4-0xA7 */ + 0xA1, 0xA7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0xA1, 0xE3, 0xA1, 0xC0, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA1, 0xA4, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA1, 0xC1, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0xA8, 0xA4, 0xA8, 0xA2, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xA8, 0xA8, 0xA8, 0xA6, 0xA8, 0xBA, 0x00, 0x00, /* 0xE8-0xEB */ + 0xA8, 0xAC, 0xA8, 0xAA, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0xA8, 0xB0, 0xA8, 0xAE, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA1, 0xC2, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xA8, 0xB4, 0xA8, 0xB2, 0x00, 0x00, /* 0xF8-0xFB */ + 0xA8, 0xB9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_01[512] = { + 0xA8, 0xA1, 0xA8, 0xA1, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0xA8, 0xA5, 0xA8, 0xA5, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0xA8, 0xA7, 0xA8, 0xA7, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xA8, 0xA9, 0xA8, 0xA9, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0xA8, 0xBD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0xA8, 0xBE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0xA8, 0xAD, 0xA8, 0xAD, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0xA8, 0xB1, 0xA8, 0xB1, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0xA1, 0xCE, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0xA8, 0xA3, 0xA8, 0xA3, 0xA8, 0xAB, /* 0xCC-0xCF */ + 0xA8, 0xAB, 0xA8, 0xAF, 0xA8, 0xAF, 0xA8, 0xB3, /* 0xD0-0xD3 */ + 0xA8, 0xB3, 0xA8, 0xB5, 0xA8, 0xB5, 0xA8, 0xB6, /* 0xD4-0xD7 */ + 0xA8, 0xB6, 0xA8, 0xB7, 0xA8, 0xB7, 0xA8, 0xB8, /* 0xD8-0xDB */ + 0xA8, 0xB8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ +}; + +static const unsigned char u2c_02[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0xA8, 0xBB, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0xA8, 0xC0, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA1, 0xA6, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xA1, 0xA5, 0xA8, 0x40, 0xA8, 0x41, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xA8, 0x42, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ +}; + +static const unsigned char u2c_03[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xA6, 0xA1, 0xA6, 0xA2, 0xA6, 0xA3, /* 0x90-0x93 */ + 0xA6, 0xA4, 0xA6, 0xA5, 0xA6, 0xA6, 0xA6, 0xA7, /* 0x94-0x97 */ + 0xA6, 0xA8, 0xA6, 0xA9, 0xA6, 0xAA, 0xA6, 0xAB, /* 0x98-0x9B */ + 0xA6, 0xAC, 0xA6, 0xAD, 0xA6, 0xAE, 0xA6, 0xAF, /* 0x9C-0x9F */ + 0xA6, 0xB0, 0xA6, 0xB1, 0x00, 0x00, 0xA6, 0xB2, /* 0xA0-0xA3 */ + 0xA6, 0xB3, 0xA6, 0xB4, 0xA6, 0xB5, 0xA6, 0xB6, /* 0xA4-0xA7 */ + 0xA6, 0xB7, 0xA6, 0xB8, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0xA6, 0xC1, 0xA6, 0xC2, 0xA6, 0xC3, /* 0xB0-0xB3 */ + 0xA6, 0xC4, 0xA6, 0xC5, 0xA6, 0xC6, 0xA6, 0xC7, /* 0xB4-0xB7 */ + 0xA6, 0xC8, 0xA6, 0xC9, 0xA6, 0xCA, 0xA6, 0xCB, /* 0xB8-0xBB */ + 0xA6, 0xCC, 0xA6, 0xCD, 0xA6, 0xCE, 0xA6, 0xCF, /* 0xBC-0xBF */ + 0xA6, 0xD0, 0xA6, 0xD1, 0x00, 0x00, 0xA6, 0xD2, /* 0xC0-0xC3 */ + 0xA6, 0xD3, 0xA6, 0xD4, 0xA6, 0xD5, 0xA6, 0xD6, /* 0xC4-0xC7 */ + 0xA6, 0xD7, 0xA6, 0xD8, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ +}; + +static const unsigned char u2c_04[512] = { + 0x00, 0x00, 0xA7, 0xA7, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0xA7, 0xA1, 0xA7, 0xA2, 0xA7, 0xA3, 0xA7, 0xA4, /* 0x10-0x13 */ + 0xA7, 0xA5, 0xA7, 0xA6, 0xA7, 0xA8, 0xA7, 0xA9, /* 0x14-0x17 */ + 0xA7, 0xAA, 0xA7, 0xAB, 0xA7, 0xAC, 0xA7, 0xAD, /* 0x18-0x1B */ + 0xA7, 0xAE, 0xA7, 0xAF, 0xA7, 0xB0, 0xA7, 0xB1, /* 0x1C-0x1F */ + 0xA7, 0xB2, 0xA7, 0xB3, 0xA7, 0xB4, 0xA7, 0xB5, /* 0x20-0x23 */ + 0xA7, 0xB6, 0xA7, 0xB7, 0xA7, 0xB8, 0xA7, 0xB9, /* 0x24-0x27 */ + 0xA7, 0xBA, 0xA7, 0xBB, 0xA7, 0xBC, 0xA7, 0xBD, /* 0x28-0x2B */ + 0xA7, 0xBE, 0xA7, 0xBF, 0xA7, 0xC0, 0xA7, 0xC1, /* 0x2C-0x2F */ + 0xA7, 0xD1, 0xA7, 0xD2, 0xA7, 0xD3, 0xA7, 0xD4, /* 0x30-0x33 */ + 0xA7, 0xD5, 0xA7, 0xD6, 0xA7, 0xD8, 0xA7, 0xD9, /* 0x34-0x37 */ + 0xA7, 0xDA, 0xA7, 0xDB, 0xA7, 0xDC, 0xA7, 0xDD, /* 0x38-0x3B */ + 0xA7, 0xDE, 0xA7, 0xDF, 0xA7, 0xE0, 0xA7, 0xE1, /* 0x3C-0x3F */ + 0xA7, 0xE2, 0xA7, 0xE3, 0xA7, 0xE4, 0xA7, 0xE5, /* 0x40-0x43 */ + 0xA7, 0xE6, 0xA7, 0xE7, 0xA7, 0xE8, 0xA7, 0xE9, /* 0x44-0x47 */ + 0xA7, 0xEA, 0xA7, 0xEB, 0xA7, 0xEC, 0xA7, 0xED, /* 0x48-0x4B */ + 0xA7, 0xEE, 0xA7, 0xEF, 0xA7, 0xF0, 0xA7, 0xF1, /* 0x4C-0x4F */ + 0x00, 0x00, 0xA7, 0xD7, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ +}; + +static const unsigned char u2c_20[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0xA9, 0x5C, 0x00, 0x00, 0x00, 0x00, 0xA8, 0x43, /* 0x10-0x13 */ + 0xA1, 0xAA, 0xA8, 0x44, 0xA1, 0xAC, 0x00, 0x00, /* 0x14-0x17 */ + 0xA1, 0xAE, 0xA1, 0xAF, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0xA1, 0xB0, 0xA1, 0xB1, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0xA8, 0x45, 0xA1, 0xAD, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xA1, 0xEB, 0x00, 0x00, 0xA1, 0xE4, 0xA1, 0xE5, /* 0x30-0x33 */ + 0x00, 0x00, 0xA8, 0x46, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA1, 0xF9, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xA3, 0xFE, 0x00, 0x00, /* 0x3C-0x3F */ +}; + +static const unsigned char u2c_21[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA1, 0xE6, /* 0x00-0x03 */ + 0x00, 0x00, 0xA8, 0x47, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xA8, 0x48, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xED, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0xA9, 0x59, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0xA2, 0xF1, 0xA2, 0xF2, 0xA2, 0xF3, 0xA2, 0xF4, /* 0x60-0x63 */ + 0xA2, 0xF5, 0xA2, 0xF6, 0xA2, 0xF7, 0xA2, 0xF8, /* 0x64-0x67 */ + 0xA2, 0xF9, 0xA2, 0xFA, 0xA2, 0xFB, 0xA2, 0xFC, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0xA2, 0xA1, 0xA2, 0xA2, 0xA2, 0xA3, 0xA2, 0xA4, /* 0x70-0x73 */ + 0xA2, 0xA5, 0xA2, 0xA6, 0xA2, 0xA7, 0xA2, 0xA8, /* 0x74-0x77 */ + 0xA2, 0xA9, 0xA2, 0xAA, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0xA1, 0xFB, 0xA1, 0xFC, 0xA1, 0xFA, 0xA1, 0xFD, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0xA8, 0x49, 0xA8, 0x4A, /* 0x94-0x97 */ + 0xA8, 0x4B, 0xA8, 0x4C, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ +}; + +static const unsigned char u2c_22[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0xA1, 0xCA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA1, 0xC7, /* 0x0C-0x0F */ + 0x00, 0x00, 0xA1, 0xC6, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0xA8, 0x4D, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0xA1, 0xE3, 0x00, 0x00, 0xA1, 0xCC, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0xA1, 0xD8, 0xA1, 0xDE, 0xA8, 0x4E, /* 0x1C-0x1F */ + 0xA1, 0xCF, 0x00, 0x00, 0x00, 0x00, 0xA8, 0x4F, /* 0x20-0x23 */ + 0x00, 0x00, 0xA1, 0xCE, 0x00, 0x00, 0xA1, 0xC4, /* 0x24-0x27 */ + 0xA1, 0xC5, 0xA1, 0xC9, 0xA1, 0xC8, 0xA1, 0xD2, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xD3, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0xA1, 0xE0, 0xA1, 0xDF, 0xA1, 0xC3, 0xA1, 0xCB, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0xA1, 0xAB, 0xA1, 0xD7, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0xA1, 0xD6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0xA1, 0xD5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0xA8, 0x50, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0xA1, 0xD9, 0xA1, 0xD4, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0xA1, 0xDC, 0xA1, 0xDD, 0xA8, 0x51, 0xA8, 0x52, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xDA, 0xA1, 0xDB, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0xA8, 0x92, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0xA1, 0xD1, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xA1, 0xCD, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA8, 0x53, /* 0xBC-0xBF */ +}; + +static const unsigned char u2c_23[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xD0, 0x00, 0x00, /* 0x10-0x13 */ +}; + +static const unsigned char u2c_24[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0xA2, 0xD9, 0xA2, 0xDA, 0xA2, 0xDB, 0xA2, 0xDC, /* 0x60-0x63 */ + 0xA2, 0xDD, 0xA2, 0xDE, 0xA2, 0xDF, 0xA2, 0xE0, /* 0x64-0x67 */ + 0xA2, 0xE1, 0xA2, 0xE2, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0xA2, 0xC5, 0xA2, 0xC6, 0xA2, 0xC7, 0xA2, 0xC8, /* 0x74-0x77 */ + 0xA2, 0xC9, 0xA2, 0xCA, 0xA2, 0xCB, 0xA2, 0xCC, /* 0x78-0x7B */ + 0xA2, 0xCD, 0xA2, 0xCE, 0xA2, 0xCF, 0xA2, 0xD0, /* 0x7C-0x7F */ + + 0xA2, 0xD1, 0xA2, 0xD2, 0xA2, 0xD3, 0xA2, 0xD4, /* 0x80-0x83 */ + 0xA2, 0xD5, 0xA2, 0xD6, 0xA2, 0xD7, 0xA2, 0xD8, /* 0x84-0x87 */ + 0xA2, 0xB1, 0xA2, 0xB2, 0xA2, 0xB3, 0xA2, 0xB4, /* 0x88-0x8B */ + 0xA2, 0xB5, 0xA2, 0xB6, 0xA2, 0xB7, 0xA2, 0xB8, /* 0x8C-0x8F */ + 0xA2, 0xB9, 0xA2, 0xBA, 0xA2, 0xBB, 0xA2, 0xBC, /* 0x90-0x93 */ + 0xA2, 0xBD, 0xA2, 0xBE, 0xA2, 0xBF, 0xA2, 0xC0, /* 0x94-0x97 */ + 0xA2, 0xC1, 0xA2, 0xC2, 0xA2, 0xC3, 0xA2, 0xC4, /* 0x98-0x9B */ +}; + +static const unsigned char u2c_25[512] = { + 0xA9, 0xA4, 0xA9, 0xA5, 0xA9, 0xA6, 0xA9, 0xA7, /* 0x00-0x03 */ + 0xA9, 0xA8, 0xA9, 0xA9, 0xA9, 0xAA, 0xA9, 0xAB, /* 0x04-0x07 */ + 0xA9, 0xAC, 0xA9, 0xAD, 0xA9, 0xAE, 0xA9, 0xAF, /* 0x08-0x0B */ + 0xA9, 0xB0, 0xA9, 0xB1, 0xA9, 0xB2, 0xA9, 0xB3, /* 0x0C-0x0F */ + 0xA9, 0xB4, 0xA9, 0xB5, 0xA9, 0xB6, 0xA9, 0xB7, /* 0x10-0x13 */ + 0xA9, 0xB8, 0xA9, 0xB9, 0xA9, 0xBA, 0xA9, 0xBB, /* 0x14-0x17 */ + 0xA9, 0xBC, 0xA9, 0xBD, 0xA9, 0xBE, 0xA9, 0xBF, /* 0x18-0x1B */ + 0xA9, 0xC0, 0xA9, 0xC1, 0xA9, 0xC2, 0xA9, 0xC3, /* 0x1C-0x1F */ + 0xA9, 0xC4, 0xA9, 0xC5, 0xA9, 0xC6, 0xA9, 0xC7, /* 0x20-0x23 */ + 0xA9, 0xC8, 0xA9, 0xC9, 0xA9, 0xCA, 0xA9, 0xCB, /* 0x24-0x27 */ + 0xA9, 0xCC, 0xA9, 0xCD, 0xA9, 0xCE, 0xA9, 0xCF, /* 0x28-0x2B */ + 0xA9, 0xD0, 0xA9, 0xD1, 0xA9, 0xD2, 0xA9, 0xD3, /* 0x2C-0x2F */ + 0xA9, 0xD4, 0xA9, 0xD5, 0xA9, 0xD6, 0xA9, 0xD7, /* 0x30-0x33 */ + 0xA9, 0xD8, 0xA9, 0xD9, 0xA9, 0xDA, 0xA9, 0xDB, /* 0x34-0x37 */ + 0xA9, 0xDC, 0xA9, 0xDD, 0xA9, 0xDE, 0xA9, 0xDF, /* 0x38-0x3B */ + 0xA9, 0xE0, 0xA9, 0xE1, 0xA9, 0xE2, 0xA9, 0xE3, /* 0x3C-0x3F */ + 0xA9, 0xE4, 0xA9, 0xE5, 0xA9, 0xE6, 0xA9, 0xE7, /* 0x40-0x43 */ + 0xA9, 0xE8, 0xA9, 0xE9, 0xA9, 0xEA, 0xA9, 0xEB, /* 0x44-0x47 */ + 0xA9, 0xEC, 0xA9, 0xED, 0xA9, 0xEE, 0xA9, 0xEF, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0xA8, 0x54, 0xA8, 0x55, 0xA8, 0x56, 0xA8, 0x57, /* 0x50-0x53 */ + 0xA8, 0x58, 0xA8, 0x59, 0xA8, 0x5A, 0xA8, 0x5B, /* 0x54-0x57 */ + 0xA8, 0x5C, 0xA8, 0x5D, 0xA8, 0x5E, 0xA8, 0x5F, /* 0x58-0x5B */ + 0xA8, 0x60, 0xA8, 0x61, 0xA8, 0x62, 0xA8, 0x63, /* 0x5C-0x5F */ + 0xA8, 0x64, 0xA8, 0x65, 0xA8, 0x66, 0xA8, 0x67, /* 0x60-0x63 */ + 0xA8, 0x68, 0xA8, 0x69, 0xA8, 0x6A, 0xA8, 0x6B, /* 0x64-0x67 */ + 0xA8, 0x6C, 0xA8, 0x6D, 0xA8, 0x6E, 0xA8, 0x6F, /* 0x68-0x6B */ + 0xA8, 0x70, 0xA8, 0x71, 0xA8, 0x72, 0xA8, 0x73, /* 0x6C-0x6F */ + 0xA8, 0x74, 0xA8, 0x75, 0xA8, 0x76, 0xA8, 0x77, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xA8, 0x78, 0xA8, 0x79, 0xA8, 0x7A, /* 0x80-0x83 */ + 0xA8, 0x7B, 0xA8, 0x7C, 0xA8, 0x7D, 0xA8, 0x7E, /* 0x84-0x87 */ + 0xA8, 0x80, 0xA8, 0x81, 0xA8, 0x82, 0xA8, 0x83, /* 0x88-0x8B */ + 0xA8, 0x84, 0xA8, 0x85, 0xA8, 0x86, 0xA8, 0x87, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA8, 0x88, /* 0x90-0x93 */ + 0xA8, 0x89, 0xA8, 0x8A, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0xA1, 0xF6, 0xA1, 0xF5, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xF8, 0xA1, 0xF7, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0xA8, 0x8B, 0xA8, 0x8C, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xF4, 0xA1, 0xF3, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA1, 0xF0, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xF2, 0xA1, 0xF1, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0xA8, 0x8D, 0xA8, 0x8E, /* 0xE0-0xE3 */ + 0xA8, 0x8F, 0xA8, 0x90, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_26[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0xA1, 0xEF, 0xA1, 0xEE, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xA8, 0x91, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0xA1, 0xE2, 0x00, 0x00, 0xA1, 0xE1, 0x00, 0x00, /* 0x40-0x43 */ +}; + +static const unsigned char u2c_30[512] = { + 0xA1, 0xA1, 0xA1, 0xA2, 0xA1, 0xA3, 0xA1, 0xA8, /* 0x00-0x03 */ + 0x00, 0x00, 0xA1, 0xA9, 0xA9, 0x65, 0xA9, 0x96, /* 0x04-0x07 */ + 0xA1, 0xB4, 0xA1, 0xB5, 0xA1, 0xB6, 0xA1, 0xB7, /* 0x08-0x0B */ + 0xA1, 0xB8, 0xA1, 0xB9, 0xA1, 0xBA, 0xA1, 0xBB, /* 0x0C-0x0F */ + 0xA1, 0xBE, 0xA1, 0xBF, 0xA8, 0x93, 0xA1, 0xFE, /* 0x10-0x13 */ + 0xA1, 0xB2, 0xA1, 0xB3, 0xA1, 0xBC, 0xA1, 0xBD, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0xA8, 0x94, 0xA8, 0x95, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0xA9, 0x40, 0xA9, 0x41, 0xA9, 0x42, /* 0x20-0x23 */ + 0xA9, 0x43, 0xA9, 0x44, 0xA9, 0x45, 0xA9, 0x46, /* 0x24-0x27 */ + 0xA9, 0x47, 0xA9, 0x48, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0xA4, 0xA1, 0xA4, 0xA2, 0xA4, 0xA3, /* 0x40-0x43 */ + 0xA4, 0xA4, 0xA4, 0xA5, 0xA4, 0xA6, 0xA4, 0xA7, /* 0x44-0x47 */ + 0xA4, 0xA8, 0xA4, 0xA9, 0xA4, 0xAA, 0xA4, 0xAB, /* 0x48-0x4B */ + 0xA4, 0xAC, 0xA4, 0xAD, 0xA4, 0xAE, 0xA4, 0xAF, /* 0x4C-0x4F */ + 0xA4, 0xB0, 0xA4, 0xB1, 0xA4, 0xB2, 0xA4, 0xB3, /* 0x50-0x53 */ + 0xA4, 0xB4, 0xA4, 0xB5, 0xA4, 0xB6, 0xA4, 0xB7, /* 0x54-0x57 */ + 0xA4, 0xB8, 0xA4, 0xB9, 0xA4, 0xBA, 0xA4, 0xBB, /* 0x58-0x5B */ + 0xA4, 0xBC, 0xA4, 0xBD, 0xA4, 0xBE, 0xA4, 0xBF, /* 0x5C-0x5F */ + 0xA4, 0xC0, 0xA4, 0xC1, 0xA4, 0xC2, 0xA4, 0xC3, /* 0x60-0x63 */ + 0xA4, 0xC4, 0xA4, 0xC5, 0xA4, 0xC6, 0xA4, 0xC7, /* 0x64-0x67 */ + 0xA4, 0xC8, 0xA4, 0xC9, 0xA4, 0xCA, 0xA4, 0xCB, /* 0x68-0x6B */ + 0xA4, 0xCC, 0xA4, 0xCD, 0xA4, 0xCE, 0xA4, 0xCF, /* 0x6C-0x6F */ + 0xA4, 0xD0, 0xA4, 0xD1, 0xA4, 0xD2, 0xA4, 0xD3, /* 0x70-0x73 */ + 0xA4, 0xD4, 0xA4, 0xD5, 0xA4, 0xD6, 0xA4, 0xD7, /* 0x74-0x77 */ + 0xA4, 0xD8, 0xA4, 0xD9, 0xA4, 0xDA, 0xA4, 0xDB, /* 0x78-0x7B */ + 0xA4, 0xDC, 0xA4, 0xDD, 0xA4, 0xDE, 0xA4, 0xDF, /* 0x7C-0x7F */ + + 0xA4, 0xE0, 0xA4, 0xE1, 0xA4, 0xE2, 0xA4, 0xE3, /* 0x80-0x83 */ + 0xA4, 0xE4, 0xA4, 0xE5, 0xA4, 0xE6, 0xA4, 0xE7, /* 0x84-0x87 */ + 0xA4, 0xE8, 0xA4, 0xE9, 0xA4, 0xEA, 0xA4, 0xEB, /* 0x88-0x8B */ + 0xA4, 0xEC, 0xA4, 0xED, 0xA4, 0xEE, 0xA4, 0xEF, /* 0x8C-0x8F */ + 0xA4, 0xF0, 0xA4, 0xF1, 0xA4, 0xF2, 0xA4, 0xF3, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA9, 0x61, /* 0x98-0x9B */ + 0xA9, 0x62, 0xA9, 0x66, 0xA9, 0x67, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0xA5, 0xA1, 0xA5, 0xA2, 0xA5, 0xA3, /* 0xA0-0xA3 */ + 0xA5, 0xA4, 0xA5, 0xA5, 0xA5, 0xA6, 0xA5, 0xA7, /* 0xA4-0xA7 */ + 0xA5, 0xA8, 0xA5, 0xA9, 0xA5, 0xAA, 0xA5, 0xAB, /* 0xA8-0xAB */ + 0xA5, 0xAC, 0xA5, 0xAD, 0xA5, 0xAE, 0xA5, 0xAF, /* 0xAC-0xAF */ + 0xA5, 0xB0, 0xA5, 0xB1, 0xA5, 0xB2, 0xA5, 0xB3, /* 0xB0-0xB3 */ + 0xA5, 0xB4, 0xA5, 0xB5, 0xA5, 0xB6, 0xA5, 0xB7, /* 0xB4-0xB7 */ + 0xA5, 0xB8, 0xA5, 0xB9, 0xA5, 0xBA, 0xA5, 0xBB, /* 0xB8-0xBB */ + 0xA5, 0xBC, 0xA5, 0xBD, 0xA5, 0xBE, 0xA5, 0xBF, /* 0xBC-0xBF */ + 0xA5, 0xC0, 0xA5, 0xC1, 0xA5, 0xC2, 0xA5, 0xC3, /* 0xC0-0xC3 */ + 0xA5, 0xC4, 0xA5, 0xC5, 0xA5, 0xC6, 0xA5, 0xC7, /* 0xC4-0xC7 */ + 0xA5, 0xC8, 0xA5, 0xC9, 0xA5, 0xCA, 0xA5, 0xCB, /* 0xC8-0xCB */ + 0xA5, 0xCC, 0xA5, 0xCD, 0xA5, 0xCE, 0xA5, 0xCF, /* 0xCC-0xCF */ + 0xA5, 0xD0, 0xA5, 0xD1, 0xA5, 0xD2, 0xA5, 0xD3, /* 0xD0-0xD3 */ + 0xA5, 0xD4, 0xA5, 0xD5, 0xA5, 0xD6, 0xA5, 0xD7, /* 0xD4-0xD7 */ + 0xA5, 0xD8, 0xA5, 0xD9, 0xA5, 0xDA, 0xA5, 0xDB, /* 0xD8-0xDB */ + 0xA5, 0xDC, 0xA5, 0xDD, 0xA5, 0xDE, 0xA5, 0xDF, /* 0xDC-0xDF */ + 0xA5, 0xE0, 0xA5, 0xE1, 0xA5, 0xE2, 0xA5, 0xE3, /* 0xE0-0xE3 */ + 0xA5, 0xE4, 0xA5, 0xE5, 0xA5, 0xE6, 0xA5, 0xE7, /* 0xE4-0xE7 */ + 0xA5, 0xE8, 0xA5, 0xE9, 0xA5, 0xEA, 0xA5, 0xEB, /* 0xE8-0xEB */ + 0xA5, 0xEC, 0xA5, 0xED, 0xA5, 0xEE, 0xA5, 0xEF, /* 0xEC-0xEF */ + 0xA5, 0xF0, 0xA5, 0xF1, 0xA5, 0xF2, 0xA5, 0xF3, /* 0xF0-0xF3 */ + 0xA5, 0xF4, 0xA5, 0xF5, 0xA5, 0xF6, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0xA9, 0x60, 0xA9, 0x63, 0xA9, 0x64, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_31[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0xA8, 0xC5, 0xA8, 0xC6, 0xA8, 0xC7, /* 0x04-0x07 */ + 0xA8, 0xC8, 0xA8, 0xC9, 0xA8, 0xCA, 0xA8, 0xCB, /* 0x08-0x0B */ + 0xA8, 0xCC, 0xA8, 0xCD, 0xA8, 0xCE, 0xA8, 0xCF, /* 0x0C-0x0F */ + 0xA8, 0xD0, 0xA8, 0xD1, 0xA8, 0xD2, 0xA8, 0xD3, /* 0x10-0x13 */ + 0xA8, 0xD4, 0xA8, 0xD5, 0xA8, 0xD6, 0xA8, 0xD7, /* 0x14-0x17 */ + 0xA8, 0xD8, 0xA8, 0xD9, 0xA8, 0xDA, 0xA8, 0xDB, /* 0x18-0x1B */ + 0xA8, 0xDC, 0xA8, 0xDD, 0xA8, 0xDE, 0xA8, 0xDF, /* 0x1C-0x1F */ + 0xA8, 0xE0, 0xA8, 0xE1, 0xA8, 0xE2, 0xA8, 0xE3, /* 0x20-0x23 */ + 0xA8, 0xE4, 0xA8, 0xE5, 0xA8, 0xE6, 0xA8, 0xE7, /* 0x24-0x27 */ + 0xA8, 0xE8, 0xA8, 0xE9, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0xD2, 0xBB, 0xB6, 0xFE, /* 0x90-0x93 */ + 0xC8, 0xFD, 0xCB, 0xC4, 0xC9, 0xCF, 0xD6, 0xD0, /* 0x94-0x97 */ + 0xCF, 0xC2, 0xBC, 0xD7, 0xD2, 0xD2, 0xB1, 0xFB, /* 0x98-0x9B */ + 0xB6, 0xA1, 0xCC, 0xEC, 0xB5, 0xD8, 0xC8, 0xCB, /* 0x9C-0x9F */ +}; + +static const unsigned char u2c_32[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0xA2, 0xE5, 0xA2, 0xE6, 0xA2, 0xE7, 0xA2, 0xE8, /* 0x20-0x23 */ + 0xA2, 0xE9, 0xA2, 0xEA, 0xA2, 0xEB, 0xA2, 0xEC, /* 0x24-0x27 */ + 0xA2, 0xED, 0xA2, 0xEE, 0xD4, 0xC2, 0xBB, 0xF0, /* 0x28-0x2B */ + 0xCB, 0xAE, 0xC4, 0xBE, 0xBD, 0xF0, 0xCD, 0xC1, /* 0x2C-0x2F */ + 0xC8, 0xD5, 0xA9, 0x5A, 0xD3, 0xD0, 0xC9, 0xE7, /* 0x30-0x33 */ + 0xC3, 0xFB, 0xCC, 0xD8, 0xB2, 0xC6, 0xD7, 0xA3, /* 0x34-0x37 */ + 0xC0, 0xCD, 0xB4, 0xFA, 0xBA, 0xF4, 0xD1, 0xA7, /* 0x38-0x3B */ + 0xBC, 0xE0, 0xC6, 0xF3, 0xD7, 0xCA, 0xD0, 0xAD, /* 0x3C-0x3F */ + 0xBC, 0xC0, 0xD0, 0xDD, 0xD7, 0xD4, 0xD6, 0xC1, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xD2, 0xBB, 0xB6, 0xFE, 0xC8, 0xFD, 0xCB, 0xC4, /* 0x80-0x83 */ + 0xCE, 0xE5, 0xC1, 0xF9, 0xC6, 0xDF, 0xB0, 0xCB, /* 0x84-0x87 */ + 0xBE, 0xC5, 0xCA, 0xAE, 0xD4, 0xC2, 0xBB, 0xF0, /* 0x88-0x8B */ + 0xCB, 0xAE, 0xC4, 0xBE, 0xBD, 0xF0, 0xCD, 0xC1, /* 0x8C-0x8F */ + 0xC8, 0xD5, 0xD6, 0xEA, 0xD3, 0xD0, 0xC9, 0xE7, /* 0x90-0x93 */ + 0xC3, 0xFB, 0xCC, 0xD8, 0xB2, 0xC6, 0xD7, 0xA3, /* 0x94-0x97 */ + 0xC0, 0xCD, 0xC3, 0xD8, 0xC4, 0xD0, 0xC5, 0xAE, /* 0x98-0x9B */ + 0xCA, 0xCA, 0xD3, 0xC5, 0x00, 0x00, 0xD7, 0xA2, /* 0x9C-0x9F */ + 0xCF, 0xEE, 0xD0, 0xDD, 0xD0, 0xB4, 0xA9, 0x49, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0xD2, 0xBD, 0xD7, 0xDA, 0xD1, 0xA7, /* 0xA8-0xAB */ + 0xBC, 0xE0, 0xC6, 0xF3, 0xD7, 0xCA, 0xD0, 0xAD, /* 0xAC-0xAF */ + 0xD2, 0xB9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ +}; + +static const unsigned char u2c_33[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0xA9, 0x4A, 0xA9, 0x4B, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0xA9, 0x4C, 0xA9, 0x4D, 0xA9, 0x4E, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0xA9, 0x4F, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xA9, 0x50, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0xA9, 0x51, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0xA9, 0x52, 0xA9, 0x53, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xA9, 0x54, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ +}; + +static const unsigned char u2c_4E[512] = { + 0xD2, 0xBB, 0xB6, 0xA1, 0x81, 0x40, 0xC6, 0xDF, /* 0x00-0x03 */ + 0x81, 0x41, 0x81, 0x42, 0x81, 0x43, 0xCD, 0xF2, /* 0x04-0x07 */ + 0xD5, 0xC9, 0xC8, 0xFD, 0xC9, 0xCF, 0xCF, 0xC2, /* 0x08-0x0B */ + 0xD8, 0xA2, 0xB2, 0xBB, 0xD3, 0xEB, 0x81, 0x44, /* 0x0C-0x0F */ + 0xD8, 0xA4, 0xB3, 0xF3, 0x81, 0x45, 0xD7, 0xA8, /* 0x10-0x13 */ + 0xC7, 0xD2, 0xD8, 0xA7, 0xCA, 0xC0, 0x81, 0x46, /* 0x14-0x17 */ + 0xC7, 0xF0, 0xB1, 0xFB, 0xD2, 0xB5, 0xB4, 0xD4, /* 0x18-0x1B */ + 0xB6, 0xAB, 0xCB, 0xBF, 0xD8, 0xA9, 0x81, 0x47, /* 0x1C-0x1F */ + 0x81, 0x48, 0x81, 0x49, 0xB6, 0xAA, 0x81, 0x4A, /* 0x20-0x23 */ + 0xC1, 0xBD, 0xD1, 0xCF, 0x81, 0x4B, 0xC9, 0xA5, /* 0x24-0x27 */ + 0xD8, 0xAD, 0x81, 0x4C, 0xB8, 0xF6, 0xD1, 0xBE, /* 0x28-0x2B */ + 0xE3, 0xDC, 0xD6, 0xD0, 0x81, 0x4D, 0x81, 0x4E, /* 0x2C-0x2F */ + 0xB7, 0xE1, 0x81, 0x4F, 0xB4, 0xAE, 0x81, 0x50, /* 0x30-0x33 */ + 0xC1, 0xD9, 0x81, 0x51, 0xD8, 0xBC, 0x81, 0x52, /* 0x34-0x37 */ + 0xCD, 0xE8, 0xB5, 0xA4, 0xCE, 0xAA, 0xD6, 0xF7, /* 0x38-0x3B */ + 0x81, 0x53, 0xC0, 0xF6, 0xBE, 0xD9, 0xD8, 0xAF, /* 0x3C-0x3F */ + 0x81, 0x54, 0x81, 0x55, 0x81, 0x56, 0xC4, 0xCB, /* 0x40-0x43 */ + 0x81, 0x57, 0xBE, 0xC3, 0x81, 0x58, 0xD8, 0xB1, /* 0x44-0x47 */ + 0xC3, 0xB4, 0xD2, 0xE5, 0x81, 0x59, 0xD6, 0xAE, /* 0x48-0x4B */ + 0xCE, 0xDA, 0xD5, 0xA7, 0xBA, 0xF5, 0xB7, 0xA6, /* 0x4C-0x4F */ + 0xC0, 0xD6, 0x81, 0x5A, 0xC6, 0xB9, 0xC5, 0xD2, /* 0x50-0x53 */ + 0xC7, 0xC7, 0x81, 0x5B, 0xB9, 0xD4, 0x81, 0x5C, /* 0x54-0x57 */ + 0xB3, 0xCB, 0xD2, 0xD2, 0x81, 0x5D, 0x81, 0x5E, /* 0x58-0x5B */ + 0xD8, 0xBF, 0xBE, 0xC5, 0xC6, 0xF2, 0xD2, 0xB2, /* 0x5C-0x5F */ + 0xCF, 0xB0, 0xCF, 0xE7, 0x81, 0x5F, 0x81, 0x60, /* 0x60-0x63 */ + 0x81, 0x61, 0x81, 0x62, 0xCA, 0xE9, 0x81, 0x63, /* 0x64-0x67 */ + 0x81, 0x64, 0xD8, 0xC0, 0x81, 0x65, 0x81, 0x66, /* 0x68-0x6B */ + 0x81, 0x67, 0x81, 0x68, 0x81, 0x69, 0x81, 0x6A, /* 0x6C-0x6F */ + 0xC2, 0xF2, 0xC2, 0xD2, 0x81, 0x6B, 0xC8, 0xE9, /* 0x70-0x73 */ + 0x81, 0x6C, 0x81, 0x6D, 0x81, 0x6E, 0x81, 0x6F, /* 0x74-0x77 */ + 0x81, 0x70, 0x81, 0x71, 0x81, 0x72, 0x81, 0x73, /* 0x78-0x7B */ + 0x81, 0x74, 0x81, 0x75, 0xC7, 0xAC, 0x81, 0x76, /* 0x7C-0x7F */ + + 0x81, 0x77, 0x81, 0x78, 0x81, 0x79, 0x81, 0x7A, /* 0x80-0x83 */ + 0x81, 0x7B, 0x81, 0x7C, 0xC1, 0xCB, 0x81, 0x7D, /* 0x84-0x87 */ + 0xD3, 0xE8, 0xD5, 0xF9, 0x81, 0x7E, 0xCA, 0xC2, /* 0x88-0x8B */ + 0xB6, 0xFE, 0xD8, 0xA1, 0xD3, 0xDA, 0xBF, 0xF7, /* 0x8C-0x8F */ + 0x81, 0x80, 0xD4, 0xC6, 0xBB, 0xA5, 0xD8, 0xC1, /* 0x90-0x93 */ + 0xCE, 0xE5, 0xBE, 0xAE, 0x81, 0x81, 0x81, 0x82, /* 0x94-0x97 */ + 0xD8, 0xA8, 0x81, 0x83, 0xD1, 0xC7, 0xD0, 0xA9, /* 0x98-0x9B */ + 0x81, 0x84, 0x81, 0x85, 0x81, 0x86, 0xD8, 0xBD, /* 0x9C-0x9F */ + 0xD9, 0xEF, 0xCD, 0xF6, 0xBF, 0xBA, 0x81, 0x87, /* 0xA0-0xA3 */ + 0xBD, 0xBB, 0xBA, 0xA5, 0xD2, 0xE0, 0xB2, 0xFA, /* 0xA4-0xA7 */ + 0xBA, 0xE0, 0xC4, 0xB6, 0x81, 0x88, 0xCF, 0xED, /* 0xA8-0xAB */ + 0xBE, 0xA9, 0xCD, 0xA4, 0xC1, 0xC1, 0x81, 0x89, /* 0xAC-0xAF */ + 0x81, 0x8A, 0x81, 0x8B, 0xC7, 0xD7, 0xD9, 0xF1, /* 0xB0-0xB3 */ + 0x81, 0x8C, 0xD9, 0xF4, 0x81, 0x8D, 0x81, 0x8E, /* 0xB4-0xB7 */ + 0x81, 0x8F, 0x81, 0x90, 0xC8, 0xCB, 0xD8, 0xE9, /* 0xB8-0xBB */ + 0x81, 0x91, 0x81, 0x92, 0x81, 0x93, 0xD2, 0xDA, /* 0xBC-0xBF */ + 0xCA, 0xB2, 0xC8, 0xCA, 0xD8, 0xEC, 0xD8, 0xEA, /* 0xC0-0xC3 */ + 0xD8, 0xC6, 0xBD, 0xF6, 0xC6, 0xCD, 0xB3, 0xF0, /* 0xC4-0xC7 */ + 0x81, 0x94, 0xD8, 0xEB, 0xBD, 0xF1, 0xBD, 0xE9, /* 0xC8-0xCB */ + 0x81, 0x95, 0xC8, 0xD4, 0xB4, 0xD3, 0x81, 0x96, /* 0xCC-0xCF */ + 0x81, 0x97, 0xC2, 0xD8, 0x81, 0x98, 0xB2, 0xD6, /* 0xD0-0xD3 */ + 0xD7, 0xD0, 0xCA, 0xCB, 0xCB, 0xFB, 0xD5, 0xCC, /* 0xD4-0xD7 */ + 0xB8, 0xB6, 0xCF, 0xC9, 0x81, 0x99, 0x81, 0x9A, /* 0xD8-0xDB */ + 0x81, 0x9B, 0xD9, 0xDA, 0xD8, 0xF0, 0xC7, 0xAA, /* 0xDC-0xDF */ + 0x81, 0x9C, 0xD8, 0xEE, 0x81, 0x9D, 0xB4, 0xFA, /* 0xE0-0xE3 */ + 0xC1, 0xEE, 0xD2, 0xD4, 0x81, 0x9E, 0x81, 0x9F, /* 0xE4-0xE7 */ + 0xD8, 0xED, 0x81, 0xA0, 0xD2, 0xC7, 0xD8, 0xEF, /* 0xE8-0xEB */ + 0xC3, 0xC7, 0x81, 0xA1, 0x81, 0xA2, 0x81, 0xA3, /* 0xEC-0xEF */ + 0xD1, 0xF6, 0x81, 0xA4, 0xD6, 0xD9, 0xD8, 0xF2, /* 0xF0-0xF3 */ + 0x81, 0xA5, 0xD8, 0xF5, 0xBC, 0xFE, 0xBC, 0xDB, /* 0xF4-0xF7 */ + 0x81, 0xA6, 0x81, 0xA7, 0x81, 0xA8, 0xC8, 0xCE, /* 0xF8-0xFB */ + 0x81, 0xA9, 0xB7, 0xDD, 0x81, 0xAA, 0xB7, 0xC2, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_4F[512] = { + 0x81, 0xAB, 0xC6, 0xF3, 0x81, 0xAC, 0x81, 0xAD, /* 0x00-0x03 */ + 0x81, 0xAE, 0x81, 0xAF, 0x81, 0xB0, 0x81, 0xB1, /* 0x04-0x07 */ + 0x81, 0xB2, 0xD8, 0xF8, 0xD2, 0xC1, 0x81, 0xB3, /* 0x08-0x0B */ + 0x81, 0xB4, 0xCE, 0xE9, 0xBC, 0xBF, 0xB7, 0xFC, /* 0x0C-0x0F */ + 0xB7, 0xA5, 0xD0, 0xDD, 0x81, 0xB5, 0x81, 0xB6, /* 0x10-0x13 */ + 0x81, 0xB7, 0x81, 0xB8, 0x81, 0xB9, 0xD6, 0xDA, /* 0x14-0x17 */ + 0xD3, 0xC5, 0xBB, 0xEF, 0xBB, 0xE1, 0xD8, 0xF1, /* 0x18-0x1B */ + 0x81, 0xBA, 0x81, 0xBB, 0xC9, 0xA1, 0xCE, 0xB0, /* 0x1C-0x1F */ + 0xB4, 0xAB, 0x81, 0xBC, 0xD8, 0xF3, 0x81, 0xBD, /* 0x20-0x23 */ + 0xC9, 0xCB, 0xD8, 0xF6, 0xC2, 0xD7, 0xD8, 0xF7, /* 0x24-0x27 */ + 0x81, 0xBE, 0x81, 0xBF, 0xCE, 0xB1, 0xD8, 0xF9, /* 0x28-0x2B */ + 0x81, 0xC0, 0x81, 0xC1, 0x81, 0xC2, 0xB2, 0xAE, /* 0x2C-0x2F */ + 0xB9, 0xC0, 0x81, 0xC3, 0xD9, 0xA3, 0x81, 0xC4, /* 0x30-0x33 */ + 0xB0, 0xE9, 0x81, 0xC5, 0xC1, 0xE6, 0x81, 0xC6, /* 0x34-0x37 */ + 0xC9, 0xEC, 0x81, 0xC7, 0xCB, 0xC5, 0x81, 0xC8, /* 0x38-0x3B */ + 0xCB, 0xC6, 0xD9, 0xA4, 0x81, 0xC9, 0x81, 0xCA, /* 0x3C-0x3F */ + 0x81, 0xCB, 0x81, 0xCC, 0x81, 0xCD, 0xB5, 0xE8, /* 0x40-0x43 */ + 0x81, 0xCE, 0x81, 0xCF, 0xB5, 0xAB, 0x81, 0xD0, /* 0x44-0x47 */ + 0x81, 0xD1, 0x81, 0xD2, 0x81, 0xD3, 0x81, 0xD4, /* 0x48-0x4B */ + 0x81, 0xD5, 0xCE, 0xBB, 0xB5, 0xCD, 0xD7, 0xA1, /* 0x4C-0x4F */ + 0xD7, 0xF4, 0xD3, 0xD3, 0x81, 0xD6, 0xCC, 0xE5, /* 0x50-0x53 */ + 0x81, 0xD7, 0xBA, 0xCE, 0x81, 0xD8, 0xD9, 0xA2, /* 0x54-0x57 */ + 0xD9, 0xDC, 0xD3, 0xE0, 0xD8, 0xFD, 0xB7, 0xF0, /* 0x58-0x5B */ + 0xD7, 0xF7, 0xD8, 0xFE, 0xD8, 0xFA, 0xD9, 0xA1, /* 0x5C-0x5F */ + 0xC4, 0xE3, 0x81, 0xD9, 0x81, 0xDA, 0xD3, 0xB6, /* 0x60-0x63 */ + 0xD8, 0xF4, 0xD9, 0xDD, 0x81, 0xDB, 0xD8, 0xFB, /* 0x64-0x67 */ + 0x81, 0xDC, 0xC5, 0xE5, 0x81, 0xDD, 0x81, 0xDE, /* 0x68-0x6B */ + 0xC0, 0xD0, 0x81, 0xDF, 0x81, 0xE0, 0xD1, 0xF0, /* 0x6C-0x6F */ + 0xB0, 0xDB, 0x81, 0xE1, 0x81, 0xE2, 0xBC, 0xD1, /* 0x70-0x73 */ + 0xD9, 0xA6, 0x81, 0xE3, 0xD9, 0xA5, 0x81, 0xE4, /* 0x74-0x77 */ + 0x81, 0xE5, 0x81, 0xE6, 0x81, 0xE7, 0xD9, 0xAC, /* 0x78-0x7B */ + 0xD9, 0xAE, 0x81, 0xE8, 0xD9, 0xAB, 0xCA, 0xB9, /* 0x7C-0x7F */ + + 0x81, 0xE9, 0x81, 0xEA, 0x81, 0xEB, 0xD9, 0xA9, /* 0x80-0x83 */ + 0xD6, 0xB6, 0x81, 0xEC, 0x81, 0xED, 0x81, 0xEE, /* 0x84-0x87 */ + 0xB3, 0xDE, 0xD9, 0xA8, 0x81, 0xEF, 0xC0, 0xFD, /* 0x88-0x8B */ + 0x81, 0xF0, 0xCA, 0xCC, 0x81, 0xF1, 0xD9, 0xAA, /* 0x8C-0x8F */ + 0x81, 0xF2, 0xD9, 0xA7, 0x81, 0xF3, 0x81, 0xF4, /* 0x90-0x93 */ + 0xD9, 0xB0, 0x81, 0xF5, 0x81, 0xF6, 0xB6, 0xB1, /* 0x94-0x97 */ + 0x81, 0xF7, 0x81, 0xF8, 0x81, 0xF9, 0xB9, 0xA9, /* 0x98-0x9B */ + 0x81, 0xFA, 0xD2, 0xC0, 0x81, 0xFB, 0x81, 0xFC, /* 0x9C-0x9F */ + 0xCF, 0xC0, 0x81, 0xFD, 0x81, 0xFE, 0xC2, 0xC2, /* 0xA0-0xA3 */ + 0x82, 0x40, 0xBD, 0xC4, 0xD5, 0xEC, 0xB2, 0xE0, /* 0xA4-0xA7 */ + 0xC7, 0xC8, 0xBF, 0xEB, 0xD9, 0xAD, 0x82, 0x41, /* 0xA8-0xAB */ + 0xD9, 0xAF, 0x82, 0x42, 0xCE, 0xEA, 0xBA, 0xEE, /* 0xAC-0xAF */ + 0x82, 0x43, 0x82, 0x44, 0x82, 0x45, 0x82, 0x46, /* 0xB0-0xB3 */ + 0x82, 0x47, 0xC7, 0xD6, 0x82, 0x48, 0x82, 0x49, /* 0xB4-0xB7 */ + 0x82, 0x4A, 0x82, 0x4B, 0x82, 0x4C, 0x82, 0x4D, /* 0xB8-0xBB */ + 0x82, 0x4E, 0x82, 0x4F, 0x82, 0x50, 0xB1, 0xE3, /* 0xBC-0xBF */ + 0x82, 0x51, 0x82, 0x52, 0x82, 0x53, 0xB4, 0xD9, /* 0xC0-0xC3 */ + 0xB6, 0xED, 0xD9, 0xB4, 0x82, 0x54, 0x82, 0x55, /* 0xC4-0xC7 */ + 0x82, 0x56, 0x82, 0x57, 0xBF, 0xA1, 0x82, 0x58, /* 0xC8-0xCB */ + 0x82, 0x59, 0x82, 0x5A, 0xD9, 0xDE, 0xC7, 0xCE, /* 0xCC-0xCF */ + 0xC0, 0xFE, 0xD9, 0xB8, 0x82, 0x5B, 0x82, 0x5C, /* 0xD0-0xD3 */ + 0x82, 0x5D, 0x82, 0x5E, 0x82, 0x5F, 0xCB, 0xD7, /* 0xD4-0xD7 */ + 0xB7, 0xFD, 0x82, 0x60, 0xD9, 0xB5, 0x82, 0x61, /* 0xD8-0xDB */ + 0xD9, 0xB7, 0xB1, 0xA3, 0xD3, 0xE1, 0xD9, 0xB9, /* 0xDC-0xDF */ + 0x82, 0x62, 0xD0, 0xC5, 0x82, 0x63, 0xD9, 0xB6, /* 0xE0-0xE3 */ + 0x82, 0x64, 0x82, 0x65, 0xD9, 0xB1, 0x82, 0x66, /* 0xE4-0xE7 */ + 0xD9, 0xB2, 0xC1, 0xA9, 0xD9, 0xB3, 0x82, 0x67, /* 0xE8-0xEB */ + 0x82, 0x68, 0xBC, 0xF3, 0xD0, 0xDE, 0xB8, 0xA9, /* 0xEC-0xEF */ + 0x82, 0x69, 0xBE, 0xE3, 0x82, 0x6A, 0xD9, 0xBD, /* 0xF0-0xF3 */ + 0x82, 0x6B, 0x82, 0x6C, 0x82, 0x6D, 0x82, 0x6E, /* 0xF4-0xF7 */ + 0xD9, 0xBA, 0x82, 0x6F, 0xB0, 0xB3, 0x82, 0x70, /* 0xF8-0xFB */ + 0x82, 0x71, 0x82, 0x72, 0xD9, 0xC2, 0x82, 0x73, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_50[512] = { + 0x82, 0x74, 0x82, 0x75, 0x82, 0x76, 0x82, 0x77, /* 0x00-0x03 */ + 0x82, 0x78, 0x82, 0x79, 0x82, 0x7A, 0x82, 0x7B, /* 0x04-0x07 */ + 0x82, 0x7C, 0x82, 0x7D, 0x82, 0x7E, 0x82, 0x80, /* 0x08-0x0B */ + 0xD9, 0xC4, 0xB1, 0xB6, 0x82, 0x81, 0xD9, 0xBF, /* 0x0C-0x0F */ + 0x82, 0x82, 0x82, 0x83, 0xB5, 0xB9, 0x82, 0x84, /* 0x10-0x13 */ + 0xBE, 0xF3, 0x82, 0x85, 0x82, 0x86, 0x82, 0x87, /* 0x14-0x17 */ + 0xCC, 0xC8, 0xBA, 0xF2, 0xD2, 0xD0, 0x82, 0x88, /* 0x18-0x1B */ + 0xD9, 0xC3, 0x82, 0x89, 0x82, 0x8A, 0xBD, 0xE8, /* 0x1C-0x1F */ + 0x82, 0x8B, 0xB3, 0xAB, 0x82, 0x8C, 0x82, 0x8D, /* 0x20-0x23 */ + 0x82, 0x8E, 0xD9, 0xC5, 0xBE, 0xEB, 0x82, 0x8F, /* 0x24-0x27 */ + 0xD9, 0xC6, 0xD9, 0xBB, 0xC4, 0xDF, 0x82, 0x90, /* 0x28-0x2B */ + 0xD9, 0xBE, 0xD9, 0xC1, 0xD9, 0xC0, 0x82, 0x91, /* 0x2C-0x2F */ + 0x82, 0x92, 0x82, 0x93, 0x82, 0x94, 0x82, 0x95, /* 0x30-0x33 */ + 0x82, 0x96, 0x82, 0x97, 0x82, 0x98, 0x82, 0x99, /* 0x34-0x37 */ + 0x82, 0x9A, 0x82, 0x9B, 0xD5, 0xAE, 0x82, 0x9C, /* 0x38-0x3B */ + 0xD6, 0xB5, 0x82, 0x9D, 0xC7, 0xE3, 0x82, 0x9E, /* 0x3C-0x3F */ + 0x82, 0x9F, 0x82, 0xA0, 0x82, 0xA1, 0xD9, 0xC8, /* 0x40-0x43 */ + 0x82, 0xA2, 0x82, 0xA3, 0x82, 0xA4, 0xBC, 0xD9, /* 0x44-0x47 */ + 0xD9, 0xCA, 0x82, 0xA5, 0x82, 0xA6, 0x82, 0xA7, /* 0x48-0x4B */ + 0xD9, 0xBC, 0x82, 0xA8, 0xD9, 0xCB, 0xC6, 0xAB, /* 0x4C-0x4F */ + 0x82, 0xA9, 0x82, 0xAA, 0x82, 0xAB, 0x82, 0xAC, /* 0x50-0x53 */ + 0x82, 0xAD, 0xD9, 0xC9, 0x82, 0xAE, 0x82, 0xAF, /* 0x54-0x57 */ + 0x82, 0xB0, 0x82, 0xB1, 0xD7, 0xF6, 0x82, 0xB2, /* 0x58-0x5B */ + 0xCD, 0xA3, 0x82, 0xB3, 0x82, 0xB4, 0x82, 0xB5, /* 0x5C-0x5F */ + 0x82, 0xB6, 0x82, 0xB7, 0x82, 0xB8, 0x82, 0xB9, /* 0x60-0x63 */ + 0x82, 0xBA, 0xBD, 0xA1, 0x82, 0xBB, 0x82, 0xBC, /* 0x64-0x67 */ + 0x82, 0xBD, 0x82, 0xBE, 0x82, 0xBF, 0x82, 0xC0, /* 0x68-0x6B */ + 0xD9, 0xCC, 0x82, 0xC1, 0x82, 0xC2, 0x82, 0xC3, /* 0x6C-0x6F */ + 0x82, 0xC4, 0x82, 0xC5, 0x82, 0xC6, 0x82, 0xC7, /* 0x70-0x73 */ + 0x82, 0xC8, 0x82, 0xC9, 0xC5, 0xBC, 0xCD, 0xB5, /* 0x74-0x77 */ + 0x82, 0xCA, 0x82, 0xCB, 0x82, 0xCC, 0xD9, 0xCD, /* 0x78-0x7B */ + 0x82, 0xCD, 0x82, 0xCE, 0xD9, 0xC7, 0xB3, 0xA5, /* 0x7C-0x7F */ + + 0xBF, 0xFE, 0x82, 0xCF, 0x82, 0xD0, 0x82, 0xD1, /* 0x80-0x83 */ + 0x82, 0xD2, 0xB8, 0xB5, 0x82, 0xD3, 0x82, 0xD4, /* 0x84-0x87 */ + 0xC0, 0xFC, 0x82, 0xD5, 0x82, 0xD6, 0x82, 0xD7, /* 0x88-0x8B */ + 0x82, 0xD8, 0xB0, 0xF8, 0x82, 0xD9, 0x82, 0xDA, /* 0x8C-0x8F */ + 0x82, 0xDB, 0x82, 0xDC, 0x82, 0xDD, 0x82, 0xDE, /* 0x90-0x93 */ + 0x82, 0xDF, 0x82, 0xE0, 0x82, 0xE1, 0x82, 0xE2, /* 0x94-0x97 */ + 0x82, 0xE3, 0x82, 0xE4, 0x82, 0xE5, 0x82, 0xE6, /* 0x98-0x9B */ + 0x82, 0xE7, 0x82, 0xE8, 0x82, 0xE9, 0x82, 0xEA, /* 0x9C-0x9F */ + 0x82, 0xEB, 0x82, 0xEC, 0x82, 0xED, 0xB4, 0xF6, /* 0xA0-0xA3 */ + 0x82, 0xEE, 0xD9, 0xCE, 0x82, 0xEF, 0xD9, 0xCF, /* 0xA4-0xA7 */ + 0xB4, 0xA2, 0xD9, 0xD0, 0x82, 0xF0, 0x82, 0xF1, /* 0xA8-0xAB */ + 0xB4, 0xDF, 0x82, 0xF2, 0x82, 0xF3, 0x82, 0xF4, /* 0xAC-0xAF */ + 0x82, 0xF5, 0x82, 0xF6, 0xB0, 0xC1, 0x82, 0xF7, /* 0xB0-0xB3 */ + 0x82, 0xF8, 0x82, 0xF9, 0x82, 0xFA, 0x82, 0xFB, /* 0xB4-0xB7 */ + 0x82, 0xFC, 0x82, 0xFD, 0xD9, 0xD1, 0xC9, 0xB5, /* 0xB8-0xBB */ + 0x82, 0xFE, 0x83, 0x40, 0x83, 0x41, 0x83, 0x42, /* 0xBC-0xBF */ + 0x83, 0x43, 0x83, 0x44, 0x83, 0x45, 0x83, 0x46, /* 0xC0-0xC3 */ + 0x83, 0x47, 0x83, 0x48, 0x83, 0x49, 0x83, 0x4A, /* 0xC4-0xC7 */ + 0x83, 0x4B, 0x83, 0x4C, 0x83, 0x4D, 0x83, 0x4E, /* 0xC8-0xCB */ + 0x83, 0x4F, 0x83, 0x50, 0x83, 0x51, 0xCF, 0xF1, /* 0xCC-0xCF */ + 0x83, 0x52, 0x83, 0x53, 0x83, 0x54, 0x83, 0x55, /* 0xD0-0xD3 */ + 0x83, 0x56, 0x83, 0x57, 0xD9, 0xD2, 0x83, 0x58, /* 0xD4-0xD7 */ + 0x83, 0x59, 0x83, 0x5A, 0xC1, 0xC5, 0x83, 0x5B, /* 0xD8-0xDB */ + 0x83, 0x5C, 0x83, 0x5D, 0x83, 0x5E, 0x83, 0x5F, /* 0xDC-0xDF */ + 0x83, 0x60, 0x83, 0x61, 0x83, 0x62, 0x83, 0x63, /* 0xE0-0xE3 */ + 0x83, 0x64, 0x83, 0x65, 0xD9, 0xD6, 0xC9, 0xAE, /* 0xE4-0xE7 */ + 0x83, 0x66, 0x83, 0x67, 0x83, 0x68, 0x83, 0x69, /* 0xE8-0xEB */ + 0xD9, 0xD5, 0xD9, 0xD4, 0xD9, 0xD7, 0x83, 0x6A, /* 0xEC-0xEF */ + 0x83, 0x6B, 0x83, 0x6C, 0x83, 0x6D, 0xCB, 0xDB, /* 0xF0-0xF3 */ + 0x83, 0x6E, 0xBD, 0xA9, 0x83, 0x6F, 0x83, 0x70, /* 0xF4-0xF7 */ + 0x83, 0x71, 0x83, 0x72, 0x83, 0x73, 0xC6, 0xA7, /* 0xF8-0xFB */ + 0x83, 0x74, 0x83, 0x75, 0x83, 0x76, 0x83, 0x77, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_51[512] = { + 0x83, 0x78, 0x83, 0x79, 0x83, 0x7A, 0x83, 0x7B, /* 0x00-0x03 */ + 0x83, 0x7C, 0x83, 0x7D, 0xD9, 0xD3, 0xD9, 0xD8, /* 0x04-0x07 */ + 0x83, 0x7E, 0x83, 0x80, 0x83, 0x81, 0xD9, 0xD9, /* 0x08-0x0B */ + 0x83, 0x82, 0x83, 0x83, 0x83, 0x84, 0x83, 0x85, /* 0x0C-0x0F */ + 0x83, 0x86, 0x83, 0x87, 0xC8, 0xE5, 0x83, 0x88, /* 0x10-0x13 */ + 0x83, 0x89, 0x83, 0x8A, 0x83, 0x8B, 0x83, 0x8C, /* 0x14-0x17 */ + 0x83, 0x8D, 0x83, 0x8E, 0x83, 0x8F, 0x83, 0x90, /* 0x18-0x1B */ + 0x83, 0x91, 0x83, 0x92, 0x83, 0x93, 0x83, 0x94, /* 0x1C-0x1F */ + 0x83, 0x95, 0xC0, 0xDC, 0x83, 0x96, 0x83, 0x97, /* 0x20-0x23 */ + 0x83, 0x98, 0x83, 0x99, 0x83, 0x9A, 0x83, 0x9B, /* 0x24-0x27 */ + 0x83, 0x9C, 0x83, 0x9D, 0x83, 0x9E, 0x83, 0x9F, /* 0x28-0x2B */ + 0x83, 0xA0, 0x83, 0xA1, 0x83, 0xA2, 0x83, 0xA3, /* 0x2C-0x2F */ + 0x83, 0xA4, 0x83, 0xA5, 0x83, 0xA6, 0x83, 0xA7, /* 0x30-0x33 */ + 0x83, 0xA8, 0x83, 0xA9, 0x83, 0xAA, 0x83, 0xAB, /* 0x34-0x37 */ + 0x83, 0xAC, 0x83, 0xAD, 0x83, 0xAE, 0x83, 0xAF, /* 0x38-0x3B */ + 0x83, 0xB0, 0x83, 0xB1, 0x83, 0xB2, 0xB6, 0xF9, /* 0x3C-0x3F */ + 0xD8, 0xA3, 0xD4, 0xCA, 0x83, 0xB3, 0xD4, 0xAA, /* 0x40-0x43 */ + 0xD0, 0xD6, 0xB3, 0xE4, 0xD5, 0xD7, 0x83, 0xB4, /* 0x44-0x47 */ + 0xCF, 0xC8, 0xB9, 0xE2, 0x83, 0xB5, 0xBF, 0xCB, /* 0x48-0x4B */ + 0x83, 0xB6, 0xC3, 0xE2, 0x83, 0xB7, 0x83, 0xB8, /* 0x4C-0x4F */ + 0x83, 0xB9, 0xB6, 0xD2, 0x83, 0xBA, 0x83, 0xBB, /* 0x50-0x53 */ + 0xCD, 0xC3, 0xD9, 0xEE, 0xD9, 0xF0, 0x83, 0xBC, /* 0x54-0x57 */ + 0x83, 0xBD, 0x83, 0xBE, 0xB5, 0xB3, 0x83, 0xBF, /* 0x58-0x5B */ + 0xB6, 0xB5, 0x83, 0xC0, 0x83, 0xC1, 0x83, 0xC2, /* 0x5C-0x5F */ + 0x83, 0xC3, 0x83, 0xC4, 0xBE, 0xA4, 0x83, 0xC5, /* 0x60-0x63 */ + 0x83, 0xC6, 0xC8, 0xEB, 0x83, 0xC7, 0x83, 0xC8, /* 0x64-0x67 */ + 0xC8, 0xAB, 0x83, 0xC9, 0x83, 0xCA, 0xB0, 0xCB, /* 0x68-0x6B */ + 0xB9, 0xAB, 0xC1, 0xF9, 0xD9, 0xE2, 0x83, 0xCB, /* 0x6C-0x6F */ + 0xC0, 0xBC, 0xB9, 0xB2, 0x83, 0xCC, 0xB9, 0xD8, /* 0x70-0x73 */ + 0xD0, 0xCB, 0xB1, 0xF8, 0xC6, 0xE4, 0xBE, 0xDF, /* 0x74-0x77 */ + 0xB5, 0xE4, 0xD7, 0xC8, 0x83, 0xCD, 0xD1, 0xF8, /* 0x78-0x7B */ + 0xBC, 0xE6, 0xCA, 0xDE, 0x83, 0xCE, 0x83, 0xCF, /* 0x7C-0x7F */ + + 0xBC, 0xBD, 0xD9, 0xE6, 0xD8, 0xE7, 0x83, 0xD0, /* 0x80-0x83 */ + 0x83, 0xD1, 0xC4, 0xDA, 0x83, 0xD2, 0x83, 0xD3, /* 0x84-0x87 */ + 0xB8, 0xD4, 0xC8, 0xBD, 0x83, 0xD4, 0x83, 0xD5, /* 0x88-0x8B */ + 0xB2, 0xE1, 0xD4, 0xD9, 0x83, 0xD6, 0x83, 0xD7, /* 0x8C-0x8F */ + 0x83, 0xD8, 0x83, 0xD9, 0xC3, 0xB0, 0x83, 0xDA, /* 0x90-0x93 */ + 0x83, 0xDB, 0xC3, 0xE1, 0xDA, 0xA2, 0xC8, 0xDF, /* 0x94-0x97 */ + 0x83, 0xDC, 0xD0, 0xB4, 0x83, 0xDD, 0xBE, 0xFC, /* 0x98-0x9B */ + 0xC5, 0xA9, 0x83, 0xDE, 0x83, 0xDF, 0x83, 0xE0, /* 0x9C-0x9F */ + 0xB9, 0xDA, 0x83, 0xE1, 0xDA, 0xA3, 0x83, 0xE2, /* 0xA0-0xA3 */ + 0xD4, 0xA9, 0xDA, 0xA4, 0x83, 0xE3, 0x83, 0xE4, /* 0xA4-0xA7 */ + 0x83, 0xE5, 0x83, 0xE6, 0x83, 0xE7, 0xD9, 0xFB, /* 0xA8-0xAB */ + 0xB6, 0xAC, 0x83, 0xE8, 0x83, 0xE9, 0xB7, 0xEB, /* 0xAC-0xAF */ + 0xB1, 0xF9, 0xD9, 0xFC, 0xB3, 0xE5, 0xBE, 0xF6, /* 0xB0-0xB3 */ + 0x83, 0xEA, 0xBF, 0xF6, 0xD2, 0xB1, 0xC0, 0xE4, /* 0xB4-0xB7 */ + 0x83, 0xEB, 0x83, 0xEC, 0x83, 0xED, 0xB6, 0xB3, /* 0xB8-0xBB */ + 0xD9, 0xFE, 0xD9, 0xFD, 0x83, 0xEE, 0x83, 0xEF, /* 0xBC-0xBF */ + 0xBE, 0xBB, 0x83, 0xF0, 0x83, 0xF1, 0x83, 0xF2, /* 0xC0-0xC3 */ + 0xC6, 0xE0, 0x83, 0xF3, 0xD7, 0xBC, 0xDA, 0xA1, /* 0xC4-0xC7 */ + 0x83, 0xF4, 0xC1, 0xB9, 0x83, 0xF5, 0xB5, 0xF2, /* 0xC8-0xCB */ + 0xC1, 0xE8, 0x83, 0xF6, 0x83, 0xF7, 0xBC, 0xF5, /* 0xCC-0xCF */ + 0x83, 0xF8, 0xB4, 0xD5, 0x83, 0xF9, 0x83, 0xFA, /* 0xD0-0xD3 */ + 0x83, 0xFB, 0x83, 0xFC, 0x83, 0xFD, 0x83, 0xFE, /* 0xD4-0xD7 */ + 0x84, 0x40, 0x84, 0x41, 0x84, 0x42, 0xC1, 0xDD, /* 0xD8-0xDB */ + 0x84, 0x43, 0xC4, 0xFD, 0x84, 0x44, 0x84, 0x45, /* 0xDC-0xDF */ + 0xBC, 0xB8, 0xB7, 0xB2, 0x84, 0x46, 0x84, 0x47, /* 0xE0-0xE3 */ + 0xB7, 0xEF, 0x84, 0x48, 0x84, 0x49, 0x84, 0x4A, /* 0xE4-0xE7 */ + 0x84, 0x4B, 0x84, 0x4C, 0x84, 0x4D, 0xD9, 0xEC, /* 0xE8-0xEB */ + 0x84, 0x4E, 0xC6, 0xBE, 0x84, 0x4F, 0xBF, 0xAD, /* 0xEC-0xEF */ + 0xBB, 0xCB, 0x84, 0x50, 0x84, 0x51, 0xB5, 0xCA, /* 0xF0-0xF3 */ + 0x84, 0x52, 0xDB, 0xC9, 0xD0, 0xD7, 0x84, 0x53, /* 0xF4-0xF7 */ + 0xCD, 0xB9, 0xB0, 0xBC, 0xB3, 0xF6, 0xBB, 0xF7, /* 0xF8-0xFB */ + 0xDB, 0xCA, 0xBA, 0xAF, 0x84, 0x54, 0xD4, 0xE4, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_52[512] = { + 0xB5, 0xB6, 0xB5, 0xF3, 0xD8, 0xD6, 0xC8, 0xD0, /* 0x00-0x03 */ + 0x84, 0x55, 0x84, 0x56, 0xB7, 0xD6, 0xC7, 0xD0, /* 0x04-0x07 */ + 0xD8, 0xD7, 0x84, 0x57, 0xBF, 0xAF, 0x84, 0x58, /* 0x08-0x0B */ + 0x84, 0x59, 0xDB, 0xBB, 0xD8, 0xD8, 0x84, 0x5A, /* 0x0C-0x0F */ + 0x84, 0x5B, 0xD0, 0xCC, 0xBB, 0xAE, 0x84, 0x5C, /* 0x10-0x13 */ + 0x84, 0x5D, 0x84, 0x5E, 0xEB, 0xBE, 0xC1, 0xD0, /* 0x14-0x17 */ + 0xC1, 0xF5, 0xD4, 0xF2, 0xB8, 0xD5, 0xB4, 0xB4, /* 0x18-0x1B */ + 0x84, 0x5F, 0xB3, 0xF5, 0x84, 0x60, 0x84, 0x61, /* 0x1C-0x1F */ + 0xC9, 0xBE, 0x84, 0x62, 0x84, 0x63, 0x84, 0x64, /* 0x20-0x23 */ + 0xC5, 0xD0, 0x84, 0x65, 0x84, 0x66, 0x84, 0x67, /* 0x24-0x27 */ + 0xC5, 0xD9, 0xC0, 0xFB, 0x84, 0x68, 0xB1, 0xF0, /* 0x28-0x2B */ + 0x84, 0x69, 0xD8, 0xD9, 0xB9, 0xCE, 0x84, 0x6A, /* 0x2C-0x2F */ + 0xB5, 0xBD, 0x84, 0x6B, 0x84, 0x6C, 0xD8, 0xDA, /* 0x30-0x33 */ + 0x84, 0x6D, 0x84, 0x6E, 0xD6, 0xC6, 0xCB, 0xA2, /* 0x34-0x37 */ + 0xC8, 0xAF, 0xC9, 0xB2, 0xB4, 0xCC, 0xBF, 0xCC, /* 0x38-0x3B */ + 0x84, 0x6F, 0xB9, 0xF4, 0x84, 0x70, 0xD8, 0xDB, /* 0x3C-0x3F */ + 0xD8, 0xDC, 0xB6, 0xE7, 0xBC, 0xC1, 0xCC, 0xEA, /* 0x40-0x43 */ + 0x84, 0x71, 0x84, 0x72, 0x84, 0x73, 0x84, 0x74, /* 0x44-0x47 */ + 0x84, 0x75, 0x84, 0x76, 0xCF, 0xF7, 0x84, 0x77, /* 0x48-0x4B */ + 0xD8, 0xDD, 0xC7, 0xB0, 0x84, 0x78, 0x84, 0x79, /* 0x4C-0x4F */ + 0xB9, 0xD0, 0xBD, 0xA3, 0x84, 0x7A, 0x84, 0x7B, /* 0x50-0x53 */ + 0xCC, 0xDE, 0x84, 0x7C, 0xC6, 0xCA, 0x84, 0x7D, /* 0x54-0x57 */ + 0x84, 0x7E, 0x84, 0x80, 0x84, 0x81, 0x84, 0x82, /* 0x58-0x5B */ + 0xD8, 0xE0, 0x84, 0x83, 0xD8, 0xDE, 0x84, 0x84, /* 0x5C-0x5F */ + 0x84, 0x85, 0xD8, 0xDF, 0x84, 0x86, 0x84, 0x87, /* 0x60-0x63 */ + 0x84, 0x88, 0xB0, 0xFE, 0x84, 0x89, 0xBE, 0xE7, /* 0x64-0x67 */ + 0x84, 0x8A, 0xCA, 0xA3, 0xBC, 0xF4, 0x84, 0x8B, /* 0x68-0x6B */ + 0x84, 0x8C, 0x84, 0x8D, 0x84, 0x8E, 0xB8, 0xB1, /* 0x6C-0x6F */ + 0x84, 0x8F, 0x84, 0x90, 0xB8, 0xEE, 0x84, 0x91, /* 0x70-0x73 */ + 0x84, 0x92, 0x84, 0x93, 0x84, 0x94, 0x84, 0x95, /* 0x74-0x77 */ + 0x84, 0x96, 0x84, 0x97, 0x84, 0x98, 0x84, 0x99, /* 0x78-0x7B */ + 0x84, 0x9A, 0xD8, 0xE2, 0x84, 0x9B, 0xBD, 0xCB, /* 0x7C-0x7F */ + + 0x84, 0x9C, 0xD8, 0xE4, 0xD8, 0xE3, 0x84, 0x9D, /* 0x80-0x83 */ + 0x84, 0x9E, 0x84, 0x9F, 0x84, 0xA0, 0x84, 0xA1, /* 0x84-0x87 */ + 0xC5, 0xFC, 0x84, 0xA2, 0x84, 0xA3, 0x84, 0xA4, /* 0x88-0x8B */ + 0x84, 0xA5, 0x84, 0xA6, 0x84, 0xA7, 0x84, 0xA8, /* 0x8C-0x8F */ + 0xD8, 0xE5, 0x84, 0xA9, 0x84, 0xAA, 0xD8, 0xE6, /* 0x90-0x93 */ + 0x84, 0xAB, 0x84, 0xAC, 0x84, 0xAD, 0x84, 0xAE, /* 0x94-0x97 */ + 0x84, 0xAF, 0x84, 0xB0, 0x84, 0xB1, 0xC1, 0xA6, /* 0x98-0x9B */ + 0x84, 0xB2, 0xC8, 0xB0, 0xB0, 0xEC, 0xB9, 0xA6, /* 0x9C-0x9F */ + 0xBC, 0xD3, 0xCE, 0xF1, 0xDB, 0xBD, 0xC1, 0xD3, /* 0xA0-0xA3 */ + 0x84, 0xB3, 0x84, 0xB4, 0x84, 0xB5, 0x84, 0xB6, /* 0xA4-0xA7 */ + 0xB6, 0xAF, 0xD6, 0xFA, 0xC5, 0xAC, 0xBD, 0xD9, /* 0xA8-0xAB */ + 0xDB, 0xBE, 0xDB, 0xBF, 0x84, 0xB7, 0x84, 0xB8, /* 0xAC-0xAF */ + 0x84, 0xB9, 0xC0, 0xF8, 0xBE, 0xA2, 0xC0, 0xCD, /* 0xB0-0xB3 */ + 0x84, 0xBA, 0x84, 0xBB, 0x84, 0xBC, 0x84, 0xBD, /* 0xB4-0xB7 */ + 0x84, 0xBE, 0x84, 0xBF, 0x84, 0xC0, 0x84, 0xC1, /* 0xB8-0xBB */ + 0x84, 0xC2, 0x84, 0xC3, 0xDB, 0xC0, 0xCA, 0xC6, /* 0xBC-0xBF */ + 0x84, 0xC4, 0x84, 0xC5, 0x84, 0xC6, 0xB2, 0xAA, /* 0xC0-0xC3 */ + 0x84, 0xC7, 0x84, 0xC8, 0x84, 0xC9, 0xD3, 0xC2, /* 0xC4-0xC7 */ + 0x84, 0xCA, 0xC3, 0xE3, 0x84, 0xCB, 0xD1, 0xAB, /* 0xC8-0xCB */ + 0x84, 0xCC, 0x84, 0xCD, 0x84, 0xCE, 0x84, 0xCF, /* 0xCC-0xCF */ + 0xDB, 0xC2, 0x84, 0xD0, 0xC0, 0xD5, 0x84, 0xD1, /* 0xD0-0xD3 */ + 0x84, 0xD2, 0x84, 0xD3, 0xDB, 0xC3, 0x84, 0xD4, /* 0xD4-0xD7 */ + 0xBF, 0xB1, 0x84, 0xD5, 0x84, 0xD6, 0x84, 0xD7, /* 0xD8-0xDB */ + 0x84, 0xD8, 0x84, 0xD9, 0x84, 0xDA, 0xC4, 0xBC, /* 0xDC-0xDF */ + 0x84, 0xDB, 0x84, 0xDC, 0x84, 0xDD, 0x84, 0xDE, /* 0xE0-0xE3 */ + 0xC7, 0xDA, 0x84, 0xDF, 0x84, 0xE0, 0x84, 0xE1, /* 0xE4-0xE7 */ + 0x84, 0xE2, 0x84, 0xE3, 0x84, 0xE4, 0x84, 0xE5, /* 0xE8-0xEB */ + 0x84, 0xE6, 0x84, 0xE7, 0x84, 0xE8, 0x84, 0xE9, /* 0xEC-0xEF */ + 0xDB, 0xC4, 0x84, 0xEA, 0x84, 0xEB, 0x84, 0xEC, /* 0xF0-0xF3 */ + 0x84, 0xED, 0x84, 0xEE, 0x84, 0xEF, 0x84, 0xF0, /* 0xF4-0xF7 */ + 0x84, 0xF1, 0xD9, 0xE8, 0xC9, 0xD7, 0x84, 0xF2, /* 0xF8-0xFB */ + 0x84, 0xF3, 0x84, 0xF4, 0xB9, 0xB4, 0xCE, 0xF0, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_53[512] = { + 0xD4, 0xC8, 0x84, 0xF5, 0x84, 0xF6, 0x84, 0xF7, /* 0x00-0x03 */ + 0x84, 0xF8, 0xB0, 0xFC, 0xB4, 0xD2, 0x84, 0xF9, /* 0x04-0x07 */ + 0xD0, 0xD9, 0x84, 0xFA, 0x84, 0xFB, 0x84, 0xFC, /* 0x08-0x0B */ + 0x84, 0xFD, 0xD9, 0xE9, 0x84, 0xFE, 0xDE, 0xCB, /* 0x0C-0x0F */ + 0xD9, 0xEB, 0x85, 0x40, 0x85, 0x41, 0x85, 0x42, /* 0x10-0x13 */ + 0x85, 0x43, 0xD8, 0xB0, 0xBB, 0xAF, 0xB1, 0xB1, /* 0x14-0x17 */ + 0x85, 0x44, 0xB3, 0xD7, 0xD8, 0xCE, 0x85, 0x45, /* 0x18-0x1B */ + 0x85, 0x46, 0xD4, 0xD1, 0x85, 0x47, 0x85, 0x48, /* 0x1C-0x1F */ + 0xBD, 0xB3, 0xBF, 0xEF, 0x85, 0x49, 0xCF, 0xBB, /* 0x20-0x23 */ + 0x85, 0x4A, 0x85, 0x4B, 0xD8, 0xD0, 0x85, 0x4C, /* 0x24-0x27 */ + 0x85, 0x4D, 0x85, 0x4E, 0xB7, 0xCB, 0x85, 0x4F, /* 0x28-0x2B */ + 0x85, 0x50, 0x85, 0x51, 0xD8, 0xD1, 0x85, 0x52, /* 0x2C-0x2F */ + 0x85, 0x53, 0x85, 0x54, 0x85, 0x55, 0x85, 0x56, /* 0x30-0x33 */ + 0x85, 0x57, 0x85, 0x58, 0x85, 0x59, 0x85, 0x5A, /* 0x34-0x37 */ + 0x85, 0x5B, 0xC6, 0xA5, 0xC7, 0xF8, 0xD2, 0xBD, /* 0x38-0x3B */ + 0x85, 0x5C, 0x85, 0x5D, 0xD8, 0xD2, 0xC4, 0xE4, /* 0x3C-0x3F */ + 0x85, 0x5E, 0xCA, 0xAE, 0x85, 0x5F, 0xC7, 0xA7, /* 0x40-0x43 */ + 0x85, 0x60, 0xD8, 0xA6, 0x85, 0x61, 0xC9, 0xFD, /* 0x44-0x47 */ + 0xCE, 0xE7, 0xBB, 0xDC, 0xB0, 0xEB, 0x85, 0x62, /* 0x48-0x4B */ + 0x85, 0x63, 0x85, 0x64, 0xBB, 0xAA, 0xD0, 0xAD, /* 0x4C-0x4F */ + 0x85, 0x65, 0xB1, 0xB0, 0xD7, 0xE4, 0xD7, 0xBF, /* 0x50-0x53 */ + 0x85, 0x66, 0xB5, 0xA5, 0xC2, 0xF4, 0xC4, 0xCF, /* 0x54-0x57 */ + 0x85, 0x67, 0x85, 0x68, 0xB2, 0xA9, 0x85, 0x69, /* 0x58-0x5B */ + 0xB2, 0xB7, 0x85, 0x6A, 0xB1, 0xE5, 0xDF, 0xB2, /* 0x5C-0x5F */ + 0xD5, 0xBC, 0xBF, 0xA8, 0xC2, 0xAC, 0xD8, 0xD5, /* 0x60-0x63 */ + 0xC2, 0xB1, 0x85, 0x6B, 0xD8, 0xD4, 0xCE, 0xD4, /* 0x64-0x67 */ + 0x85, 0x6C, 0xDA, 0xE0, 0x85, 0x6D, 0xCE, 0xC0, /* 0x68-0x6B */ + 0x85, 0x6E, 0x85, 0x6F, 0xD8, 0xB4, 0xC3, 0xAE, /* 0x6C-0x6F */ + 0xD3, 0xA1, 0xCE, 0xA3, 0x85, 0x70, 0xBC, 0xB4, /* 0x70-0x73 */ + 0xC8, 0xB4, 0xC2, 0xD1, 0x85, 0x71, 0xBE, 0xED, /* 0x74-0x77 */ + 0xD0, 0xB6, 0x85, 0x72, 0xDA, 0xE1, 0x85, 0x73, /* 0x78-0x7B */ + 0x85, 0x74, 0x85, 0x75, 0x85, 0x76, 0xC7, 0xE4, /* 0x7C-0x7F */ + + 0x85, 0x77, 0x85, 0x78, 0xB3, 0xA7, 0x85, 0x79, /* 0x80-0x83 */ + 0xB6, 0xF2, 0xCC, 0xFC, 0xC0, 0xFA, 0x85, 0x7A, /* 0x84-0x87 */ + 0x85, 0x7B, 0xC0, 0xF7, 0x85, 0x7C, 0xD1, 0xB9, /* 0x88-0x8B */ + 0xD1, 0xE1, 0xD8, 0xC7, 0x85, 0x7D, 0x85, 0x7E, /* 0x8C-0x8F */ + 0x85, 0x80, 0x85, 0x81, 0x85, 0x82, 0x85, 0x83, /* 0x90-0x93 */ + 0x85, 0x84, 0xB2, 0xDE, 0x85, 0x85, 0x85, 0x86, /* 0x94-0x97 */ + 0xC0, 0xE5, 0x85, 0x87, 0xBA, 0xF1, 0x85, 0x88, /* 0x98-0x9B */ + 0x85, 0x89, 0xD8, 0xC8, 0x85, 0x8A, 0xD4, 0xAD, /* 0x9C-0x9F */ + 0x85, 0x8B, 0x85, 0x8C, 0xCF, 0xE1, 0xD8, 0xC9, /* 0xA0-0xA3 */ + 0x85, 0x8D, 0xD8, 0xCA, 0xCF, 0xC3, 0x85, 0x8E, /* 0xA4-0xA7 */ + 0xB3, 0xF8, 0xBE, 0xC7, 0x85, 0x8F, 0x85, 0x90, /* 0xA8-0xAB */ + 0x85, 0x91, 0x85, 0x92, 0xD8, 0xCB, 0x85, 0x93, /* 0xAC-0xAF */ + 0x85, 0x94, 0x85, 0x95, 0x85, 0x96, 0x85, 0x97, /* 0xB0-0xB3 */ + 0x85, 0x98, 0x85, 0x99, 0xDB, 0xCC, 0x85, 0x9A, /* 0xB4-0xB7 */ + 0x85, 0x9B, 0x85, 0x9C, 0x85, 0x9D, 0xC8, 0xA5, /* 0xB8-0xBB */ + 0x85, 0x9E, 0x85, 0x9F, 0x85, 0xA0, 0xCF, 0xD8, /* 0xBC-0xBF */ + 0x85, 0xA1, 0xC8, 0xFE, 0xB2, 0xCE, 0x85, 0xA2, /* 0xC0-0xC3 */ + 0x85, 0xA3, 0x85, 0xA4, 0x85, 0xA5, 0x85, 0xA6, /* 0xC4-0xC7 */ + 0xD3, 0xD6, 0xB2, 0xE6, 0xBC, 0xB0, 0xD3, 0xD1, /* 0xC8-0xCB */ + 0xCB, 0xAB, 0xB7, 0xB4, 0x85, 0xA7, 0x85, 0xA8, /* 0xCC-0xCF */ + 0x85, 0xA9, 0xB7, 0xA2, 0x85, 0xAA, 0x85, 0xAB, /* 0xD0-0xD3 */ + 0xCA, 0xE5, 0x85, 0xAC, 0xC8, 0xA1, 0xCA, 0xDC, /* 0xD4-0xD7 */ + 0xB1, 0xE4, 0xD0, 0xF0, 0x85, 0xAD, 0xC5, 0xD1, /* 0xD8-0xDB */ + 0x85, 0xAE, 0x85, 0xAF, 0x85, 0xB0, 0xDB, 0xC5, /* 0xDC-0xDF */ + 0xB5, 0xFE, 0x85, 0xB1, 0x85, 0xB2, 0xBF, 0xDA, /* 0xE0-0xE3 */ + 0xB9, 0xC5, 0xBE, 0xE4, 0xC1, 0xED, 0x85, 0xB3, /* 0xE4-0xE7 */ + 0xDF, 0xB6, 0xDF, 0xB5, 0xD6, 0xBB, 0xBD, 0xD0, /* 0xE8-0xEB */ + 0xD5, 0xD9, 0xB0, 0xC8, 0xB6, 0xA3, 0xBF, 0xC9, /* 0xEC-0xEF */ + 0xCC, 0xA8, 0xDF, 0xB3, 0xCA, 0xB7, 0xD3, 0xD2, /* 0xF0-0xF3 */ + 0x85, 0xB4, 0xD8, 0xCF, 0xD2, 0xB6, 0xBA, 0xC5, /* 0xF4-0xF7 */ + 0xCB, 0xBE, 0xCC, 0xBE, 0x85, 0xB5, 0xDF, 0xB7, /* 0xF8-0xFB */ + 0xB5, 0xF0, 0xDF, 0xB4, 0x85, 0xB6, 0x85, 0xB7, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_54[512] = { + 0x85, 0xB8, 0xD3, 0xF5, 0x85, 0xB9, 0xB3, 0xD4, /* 0x00-0x03 */ + 0xB8, 0xF7, 0x85, 0xBA, 0xDF, 0xBA, 0x85, 0xBB, /* 0x04-0x07 */ + 0xBA, 0xCF, 0xBC, 0xAA, 0xB5, 0xF5, 0x85, 0xBC, /* 0x08-0x0B */ + 0xCD, 0xAC, 0xC3, 0xFB, 0xBA, 0xF3, 0xC0, 0xF4, /* 0x0C-0x0F */ + 0xCD, 0xC2, 0xCF, 0xF2, 0xDF, 0xB8, 0xCF, 0xC5, /* 0x10-0x13 */ + 0x85, 0xBD, 0xC2, 0xC0, 0xDF, 0xB9, 0xC2, 0xF0, /* 0x14-0x17 */ + 0x85, 0xBE, 0x85, 0xBF, 0x85, 0xC0, 0xBE, 0xFD, /* 0x18-0x1B */ + 0x85, 0xC1, 0xC1, 0xDF, 0xCD, 0xCC, 0xD2, 0xF7, /* 0x1C-0x1F */ + 0xB7, 0xCD, 0xDF, 0xC1, 0x85, 0xC2, 0xDF, 0xC4, /* 0x20-0x23 */ + 0x85, 0xC3, 0x85, 0xC4, 0xB7, 0xF1, 0xB0, 0xC9, /* 0x24-0x27 */ + 0xB6, 0xD6, 0xB7, 0xD4, 0x85, 0xC5, 0xBA, 0xAC, /* 0x28-0x2B */ + 0xCC, 0xFD, 0xBF, 0xD4, 0xCB, 0xB1, 0xC6, 0xF4, /* 0x2C-0x2F */ + 0x85, 0xC6, 0xD6, 0xA8, 0xDF, 0xC5, 0x85, 0xC7, /* 0x30-0x33 */ + 0xCE, 0xE2, 0xB3, 0xB3, 0x85, 0xC8, 0x85, 0xC9, /* 0x34-0x37 */ + 0xCE, 0xFC, 0xB4, 0xB5, 0x85, 0xCA, 0xCE, 0xC7, /* 0x38-0x3B */ + 0xBA, 0xF0, 0x85, 0xCB, 0xCE, 0xE1, 0x85, 0xCC, /* 0x3C-0x3F */ + 0xD1, 0xBD, 0x85, 0xCD, 0x85, 0xCE, 0xDF, 0xC0, /* 0x40-0x43 */ + 0x85, 0xCF, 0x85, 0xD0, 0xB4, 0xF4, 0x85, 0xD1, /* 0x44-0x47 */ + 0xB3, 0xCA, 0x85, 0xD2, 0xB8, 0xE6, 0xDF, 0xBB, /* 0x48-0x4B */ + 0x85, 0xD3, 0x85, 0xD4, 0x85, 0xD5, 0x85, 0xD6, /* 0x4C-0x4F */ + 0xC4, 0xC5, 0x85, 0xD7, 0xDF, 0xBC, 0xDF, 0xBD, /* 0x50-0x53 */ + 0xDF, 0xBE, 0xC5, 0xBB, 0xDF, 0xBF, 0xDF, 0xC2, /* 0x54-0x57 */ + 0xD4, 0xB1, 0xDF, 0xC3, 0x85, 0xD8, 0xC7, 0xBA, /* 0x58-0x5B */ + 0xCE, 0xD8, 0x85, 0xD9, 0x85, 0xDA, 0x85, 0xDB, /* 0x5C-0x5F */ + 0x85, 0xDC, 0x85, 0xDD, 0xC4, 0xD8, 0x85, 0xDE, /* 0x60-0x63 */ + 0xDF, 0xCA, 0x85, 0xDF, 0xDF, 0xCF, 0x85, 0xE0, /* 0x64-0x67 */ + 0xD6, 0xDC, 0x85, 0xE1, 0x85, 0xE2, 0x85, 0xE3, /* 0x68-0x6B */ + 0x85, 0xE4, 0x85, 0xE5, 0x85, 0xE6, 0x85, 0xE7, /* 0x6C-0x6F */ + 0x85, 0xE8, 0xDF, 0xC9, 0xDF, 0xDA, 0xCE, 0xB6, /* 0x70-0x73 */ + 0x85, 0xE9, 0xBA, 0xC7, 0xDF, 0xCE, 0xDF, 0xC8, /* 0x74-0x77 */ + 0xC5, 0xDE, 0x85, 0xEA, 0x85, 0xEB, 0xC9, 0xEB, /* 0x78-0x7B */ + 0xBA, 0xF4, 0xC3, 0xFC, 0x85, 0xEC, 0x85, 0xED, /* 0x7C-0x7F */ + + 0xBE, 0xD7, 0x85, 0xEE, 0xDF, 0xC6, 0x85, 0xEF, /* 0x80-0x83 */ + 0xDF, 0xCD, 0x85, 0xF0, 0xC5, 0xD8, 0x85, 0xF1, /* 0x84-0x87 */ + 0x85, 0xF2, 0x85, 0xF3, 0x85, 0xF4, 0xD5, 0xA6, /* 0x88-0x8B */ + 0xBA, 0xCD, 0x85, 0xF5, 0xBE, 0xCC, 0xD3, 0xBD, /* 0x8C-0x8F */ + 0xB8, 0xC0, 0x85, 0xF6, 0xD6, 0xE4, 0x85, 0xF7, /* 0x90-0x93 */ + 0xDF, 0xC7, 0xB9, 0xBE, 0xBF, 0xA7, 0x85, 0xF8, /* 0x94-0x97 */ + 0x85, 0xF9, 0xC1, 0xFC, 0xDF, 0xCB, 0xDF, 0xCC, /* 0x98-0x9B */ + 0x85, 0xFA, 0xDF, 0xD0, 0x85, 0xFB, 0x85, 0xFC, /* 0x9C-0x9F */ + 0x85, 0xFD, 0x85, 0xFE, 0x86, 0x40, 0xDF, 0xDB, /* 0xA0-0xA3 */ + 0xDF, 0xE5, 0x86, 0x41, 0xDF, 0xD7, 0xDF, 0xD6, /* 0xA4-0xA7 */ + 0xD7, 0xC9, 0xDF, 0xE3, 0xDF, 0xE4, 0xE5, 0xEB, /* 0xA8-0xAB */ + 0xD2, 0xA7, 0xDF, 0xD2, 0x86, 0x42, 0xBF, 0xA9, /* 0xAC-0xAF */ + 0x86, 0x43, 0xD4, 0xDB, 0x86, 0x44, 0xBF, 0xC8, /* 0xB0-0xB3 */ + 0xDF, 0xD4, 0x86, 0x45, 0x86, 0x46, 0x86, 0x47, /* 0xB4-0xB7 */ + 0xCF, 0xCC, 0x86, 0x48, 0x86, 0x49, 0xDF, 0xDD, /* 0xB8-0xBB */ + 0x86, 0x4A, 0xD1, 0xCA, 0x86, 0x4B, 0xDF, 0xDE, /* 0xBC-0xBF */ + 0xB0, 0xA7, 0xC6, 0xB7, 0xDF, 0xD3, 0x86, 0x4C, /* 0xC0-0xC3 */ + 0xBA, 0xE5, 0x86, 0x4D, 0xB6, 0xDF, 0xCD, 0xDB, /* 0xC4-0xC7 */ + 0xB9, 0xFE, 0xD4, 0xD5, 0x86, 0x4E, 0x86, 0x4F, /* 0xC8-0xCB */ + 0xDF, 0xDF, 0xCF, 0xEC, 0xB0, 0xA5, 0xDF, 0xE7, /* 0xCC-0xCF */ + 0xDF, 0xD1, 0xD1, 0xC6, 0xDF, 0xD5, 0xDF, 0xD8, /* 0xD0-0xD3 */ + 0xDF, 0xD9, 0xDF, 0xDC, 0x86, 0x50, 0xBB, 0xA9, /* 0xD4-0xD7 */ + 0x86, 0x51, 0xDF, 0xE0, 0xDF, 0xE1, 0x86, 0x52, /* 0xD8-0xDB */ + 0xDF, 0xE2, 0xDF, 0xE6, 0xDF, 0xE8, 0xD3, 0xB4, /* 0xDC-0xDF */ + 0x86, 0x53, 0x86, 0x54, 0x86, 0x55, 0x86, 0x56, /* 0xE0-0xE3 */ + 0x86, 0x57, 0xB8, 0xE7, 0xC5, 0xB6, 0xDF, 0xEA, /* 0xE4-0xE7 */ + 0xC9, 0xDA, 0xC1, 0xA8, 0xC4, 0xC4, 0x86, 0x58, /* 0xE8-0xEB */ + 0x86, 0x59, 0xBF, 0xDE, 0xCF, 0xF8, 0x86, 0x5A, /* 0xEC-0xEF */ + 0x86, 0x5B, 0x86, 0x5C, 0xD5, 0xDC, 0xDF, 0xEE, /* 0xF0-0xF3 */ + 0x86, 0x5D, 0x86, 0x5E, 0x86, 0x5F, 0x86, 0x60, /* 0xF4-0xF7 */ + 0x86, 0x61, 0x86, 0x62, 0xB2, 0xB8, 0x86, 0x63, /* 0xF8-0xFB */ + 0xBA, 0xDF, 0xDF, 0xEC, 0x86, 0x64, 0xDB, 0xC1, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_55[512] = { + 0x86, 0x65, 0xD1, 0xE4, 0x86, 0x66, 0x86, 0x67, /* 0x00-0x03 */ + 0x86, 0x68, 0x86, 0x69, 0xCB, 0xF4, 0xB4, 0xBD, /* 0x04-0x07 */ + 0x86, 0x6A, 0xB0, 0xA6, 0x86, 0x6B, 0x86, 0x6C, /* 0x08-0x0B */ + 0x86, 0x6D, 0x86, 0x6E, 0x86, 0x6F, 0xDF, 0xF1, /* 0x0C-0x0F */ + 0xCC, 0xC6, 0xDF, 0xF2, 0x86, 0x70, 0x86, 0x71, /* 0x10-0x13 */ + 0xDF, 0xED, 0x86, 0x72, 0x86, 0x73, 0x86, 0x74, /* 0x14-0x17 */ + 0x86, 0x75, 0x86, 0x76, 0x86, 0x77, 0xDF, 0xE9, /* 0x18-0x1B */ + 0x86, 0x78, 0x86, 0x79, 0x86, 0x7A, 0x86, 0x7B, /* 0x1C-0x1F */ + 0xDF, 0xEB, 0x86, 0x7C, 0xDF, 0xEF, 0xDF, 0xF0, /* 0x20-0x23 */ + 0xBB, 0xBD, 0x86, 0x7D, 0x86, 0x7E, 0xDF, 0xF3, /* 0x24-0x27 */ + 0x86, 0x80, 0x86, 0x81, 0xDF, 0xF4, 0x86, 0x82, /* 0x28-0x2B */ + 0xBB, 0xA3, 0x86, 0x83, 0xCA, 0xDB, 0xCE, 0xA8, /* 0x2C-0x2F */ + 0xE0, 0xA7, 0xB3, 0xAA, 0x86, 0x84, 0xE0, 0xA6, /* 0x30-0x33 */ + 0x86, 0x85, 0x86, 0x86, 0x86, 0x87, 0xE0, 0xA1, /* 0x34-0x37 */ + 0x86, 0x88, 0x86, 0x89, 0x86, 0x8A, 0x86, 0x8B, /* 0x38-0x3B */ + 0xDF, 0xFE, 0x86, 0x8C, 0xCD, 0xD9, 0xDF, 0xFC, /* 0x3C-0x3F */ + 0x86, 0x8D, 0xDF, 0xFA, 0x86, 0x8E, 0xBF, 0xD0, /* 0x40-0x43 */ + 0xD7, 0xC4, 0x86, 0x8F, 0xC9, 0xCC, 0x86, 0x90, /* 0x44-0x47 */ + 0x86, 0x91, 0xDF, 0xF8, 0xB0, 0xA1, 0x86, 0x92, /* 0x48-0x4B */ + 0x86, 0x93, 0x86, 0x94, 0x86, 0x95, 0x86, 0x96, /* 0x4C-0x4F */ + 0xDF, 0xFD, 0x86, 0x97, 0x86, 0x98, 0x86, 0x99, /* 0x50-0x53 */ + 0x86, 0x9A, 0xDF, 0xFB, 0xE0, 0xA2, 0x86, 0x9B, /* 0x54-0x57 */ + 0x86, 0x9C, 0x86, 0x9D, 0x86, 0x9E, 0x86, 0x9F, /* 0x58-0x5B */ + 0xE0, 0xA8, 0x86, 0xA0, 0x86, 0xA1, 0x86, 0xA2, /* 0x5C-0x5F */ + 0x86, 0xA3, 0xB7, 0xC8, 0x86, 0xA4, 0x86, 0xA5, /* 0x60-0x63 */ + 0xC6, 0xA1, 0xC9, 0xB6, 0xC0, 0xB2, 0xDF, 0xF5, /* 0x64-0x67 */ + 0x86, 0xA6, 0x86, 0xA7, 0xC5, 0xBE, 0x86, 0xA8, /* 0x68-0x6B */ + 0xD8, 0xC4, 0xDF, 0xF9, 0xC4, 0xF6, 0x86, 0xA9, /* 0x6C-0x6F */ + 0x86, 0xAA, 0x86, 0xAB, 0x86, 0xAC, 0x86, 0xAD, /* 0x70-0x73 */ + 0x86, 0xAE, 0xE0, 0xA3, 0xE0, 0xA4, 0xE0, 0xA5, /* 0x74-0x77 */ + 0xD0, 0xA5, 0x86, 0xAF, 0x86, 0xB0, 0xE0, 0xB4, /* 0x78-0x7B */ + 0xCC, 0xE4, 0x86, 0xB1, 0xE0, 0xB1, 0x86, 0xB2, /* 0x7C-0x7F */ + + 0xBF, 0xA6, 0xE0, 0xAF, 0xCE, 0xB9, 0xE0, 0xAB, /* 0x80-0x83 */ + 0xC9, 0xC6, 0x86, 0xB3, 0x86, 0xB4, 0xC0, 0xAE, /* 0x84-0x87 */ + 0xE0, 0xAE, 0xBA, 0xED, 0xBA, 0xB0, 0xE0, 0xA9, /* 0x88-0x8B */ + 0x86, 0xB5, 0x86, 0xB6, 0x86, 0xB7, 0xDF, 0xF6, /* 0x8C-0x8F */ + 0x86, 0xB8, 0xE0, 0xB3, 0x86, 0xB9, 0x86, 0xBA, /* 0x90-0x93 */ + 0xE0, 0xB8, 0x86, 0xBB, 0x86, 0xBC, 0x86, 0xBD, /* 0x94-0x97 */ + 0xB4, 0xAD, 0xE0, 0xB9, 0x86, 0xBE, 0x86, 0xBF, /* 0x98-0x9B */ + 0xCF, 0xB2, 0xBA, 0xC8, 0x86, 0xC0, 0xE0, 0xB0, /* 0x9C-0x9F */ + 0x86, 0xC1, 0x86, 0xC2, 0x86, 0xC3, 0x86, 0xC4, /* 0xA0-0xA3 */ + 0x86, 0xC5, 0x86, 0xC6, 0x86, 0xC7, 0xD0, 0xFA, /* 0xA4-0xA7 */ + 0x86, 0xC8, 0x86, 0xC9, 0x86, 0xCA, 0x86, 0xCB, /* 0xA8-0xAB */ + 0x86, 0xCC, 0x86, 0xCD, 0x86, 0xCE, 0x86, 0xCF, /* 0xAC-0xAF */ + 0x86, 0xD0, 0xE0, 0xAC, 0x86, 0xD1, 0xD4, 0xFB, /* 0xB0-0xB3 */ + 0x86, 0xD2, 0xDF, 0xF7, 0x86, 0xD3, 0xC5, 0xE7, /* 0xB4-0xB7 */ + 0x86, 0xD4, 0xE0, 0xAD, 0x86, 0xD5, 0xD3, 0xF7, /* 0xB8-0xBB */ + 0x86, 0xD6, 0xE0, 0xB6, 0xE0, 0xB7, 0x86, 0xD7, /* 0xBC-0xBF */ + 0x86, 0xD8, 0x86, 0xD9, 0x86, 0xDA, 0x86, 0xDB, /* 0xC0-0xC3 */ + 0xE0, 0xC4, 0xD0, 0xE1, 0x86, 0xDC, 0x86, 0xDD, /* 0xC4-0xC7 */ + 0x86, 0xDE, 0xE0, 0xBC, 0x86, 0xDF, 0x86, 0xE0, /* 0xC8-0xCB */ + 0xE0, 0xC9, 0xE0, 0xCA, 0x86, 0xE1, 0x86, 0xE2, /* 0xCC-0xCF */ + 0x86, 0xE3, 0xE0, 0xBE, 0xE0, 0xAA, 0xC9, 0xA4, /* 0xD0-0xD3 */ + 0xE0, 0xC1, 0x86, 0xE4, 0xE0, 0xB2, 0x86, 0xE5, /* 0xD4-0xD7 */ + 0x86, 0xE6, 0x86, 0xE7, 0x86, 0xE8, 0x86, 0xE9, /* 0xD8-0xDB */ + 0xCA, 0xC8, 0xE0, 0xC3, 0x86, 0xEA, 0xE0, 0xB5, /* 0xDC-0xDF */ + 0x86, 0xEB, 0xCE, 0xCB, 0x86, 0xEC, 0xCB, 0xC3, /* 0xE0-0xE3 */ + 0xE0, 0xCD, 0xE0, 0xC6, 0xE0, 0xC2, 0x86, 0xED, /* 0xE4-0xE7 */ + 0xE0, 0xCB, 0x86, 0xEE, 0xE0, 0xBA, 0xE0, 0xBF, /* 0xE8-0xEB */ + 0xE0, 0xC0, 0x86, 0xEF, 0x86, 0xF0, 0xE0, 0xC5, /* 0xEC-0xEF */ + 0x86, 0xF1, 0x86, 0xF2, 0xE0, 0xC7, 0xE0, 0xC8, /* 0xF0-0xF3 */ + 0x86, 0xF3, 0xE0, 0xCC, 0x86, 0xF4, 0xE0, 0xBB, /* 0xF4-0xF7 */ + 0x86, 0xF5, 0x86, 0xF6, 0x86, 0xF7, 0x86, 0xF8, /* 0xF8-0xFB */ + 0x86, 0xF9, 0xCB, 0xD4, 0xE0, 0xD5, 0x86, 0xFA, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_56[512] = { + 0xE0, 0xD6, 0xE0, 0xD2, 0x86, 0xFB, 0x86, 0xFC, /* 0x00-0x03 */ + 0x86, 0xFD, 0x86, 0xFE, 0x87, 0x40, 0x87, 0x41, /* 0x04-0x07 */ + 0xE0, 0xD0, 0xBC, 0xCE, 0x87, 0x42, 0x87, 0x43, /* 0x08-0x0B */ + 0xE0, 0xD1, 0x87, 0x44, 0xB8, 0xC2, 0xD8, 0xC5, /* 0x0C-0x0F */ + 0x87, 0x45, 0x87, 0x46, 0x87, 0x47, 0x87, 0x48, /* 0x10-0x13 */ + 0x87, 0x49, 0x87, 0x4A, 0x87, 0x4B, 0x87, 0x4C, /* 0x14-0x17 */ + 0xD0, 0xEA, 0x87, 0x4D, 0x87, 0x4E, 0xC2, 0xEF, /* 0x18-0x1B */ + 0x87, 0x4F, 0x87, 0x50, 0xE0, 0xCF, 0xE0, 0xBD, /* 0x1C-0x1F */ + 0x87, 0x51, 0x87, 0x52, 0x87, 0x53, 0xE0, 0xD4, /* 0x20-0x23 */ + 0xE0, 0xD3, 0x87, 0x54, 0x87, 0x55, 0xE0, 0xD7, /* 0x24-0x27 */ + 0x87, 0x56, 0x87, 0x57, 0x87, 0x58, 0x87, 0x59, /* 0x28-0x2B */ + 0xE0, 0xDC, 0xE0, 0xD8, 0x87, 0x5A, 0x87, 0x5B, /* 0x2C-0x2F */ + 0x87, 0x5C, 0xD6, 0xF6, 0xB3, 0xB0, 0x87, 0x5D, /* 0x30-0x33 */ + 0xD7, 0xEC, 0x87, 0x5E, 0xCB, 0xBB, 0x87, 0x5F, /* 0x34-0x37 */ + 0x87, 0x60, 0xE0, 0xDA, 0x87, 0x61, 0xCE, 0xFB, /* 0x38-0x3B */ + 0x87, 0x62, 0x87, 0x63, 0x87, 0x64, 0xBA, 0xD9, /* 0x3C-0x3F */ + 0x87, 0x65, 0x87, 0x66, 0x87, 0x67, 0x87, 0x68, /* 0x40-0x43 */ + 0x87, 0x69, 0x87, 0x6A, 0x87, 0x6B, 0x87, 0x6C, /* 0x44-0x47 */ + 0x87, 0x6D, 0x87, 0x6E, 0x87, 0x6F, 0x87, 0x70, /* 0x48-0x4B */ + 0xE0, 0xE1, 0xE0, 0xDD, 0xD2, 0xAD, 0x87, 0x71, /* 0x4C-0x4F */ + 0x87, 0x72, 0x87, 0x73, 0x87, 0x74, 0x87, 0x75, /* 0x50-0x53 */ + 0xE0, 0xE2, 0x87, 0x76, 0x87, 0x77, 0xE0, 0xDB, /* 0x54-0x57 */ + 0xE0, 0xD9, 0xE0, 0xDF, 0x87, 0x78, 0x87, 0x79, /* 0x58-0x5B */ + 0xE0, 0xE0, 0x87, 0x7A, 0x87, 0x7B, 0x87, 0x7C, /* 0x5C-0x5F */ + 0x87, 0x7D, 0x87, 0x7E, 0xE0, 0xDE, 0x87, 0x80, /* 0x60-0x63 */ + 0xE0, 0xE4, 0x87, 0x81, 0x87, 0x82, 0x87, 0x83, /* 0x64-0x67 */ + 0xC6, 0xF7, 0xD8, 0xAC, 0xD4, 0xEB, 0xE0, 0xE6, /* 0x68-0x6B */ + 0xCA, 0xC9, 0x87, 0x84, 0x87, 0x85, 0x87, 0x86, /* 0x6C-0x6F */ + 0x87, 0x87, 0xE0, 0xE5, 0x87, 0x88, 0x87, 0x89, /* 0x70-0x73 */ + 0x87, 0x8A, 0x87, 0x8B, 0xB8, 0xC1, 0x87, 0x8C, /* 0x74-0x77 */ + 0x87, 0x8D, 0x87, 0x8E, 0x87, 0x8F, 0xE0, 0xE7, /* 0x78-0x7B */ + 0xE0, 0xE8, 0x87, 0x90, 0x87, 0x91, 0x87, 0x92, /* 0x7C-0x7F */ + + 0x87, 0x93, 0x87, 0x94, 0x87, 0x95, 0x87, 0x96, /* 0x80-0x83 */ + 0x87, 0x97, 0xE0, 0xE9, 0xE0, 0xE3, 0x87, 0x98, /* 0x84-0x87 */ + 0x87, 0x99, 0x87, 0x9A, 0x87, 0x9B, 0x87, 0x9C, /* 0x88-0x8B */ + 0x87, 0x9D, 0x87, 0x9E, 0xBA, 0xBF, 0xCC, 0xE7, /* 0x8C-0x8F */ + 0x87, 0x9F, 0x87, 0xA0, 0x87, 0xA1, 0xE0, 0xEA, /* 0x90-0x93 */ + 0x87, 0xA2, 0x87, 0xA3, 0x87, 0xA4, 0x87, 0xA5, /* 0x94-0x97 */ + 0x87, 0xA6, 0x87, 0xA7, 0x87, 0xA8, 0x87, 0xA9, /* 0x98-0x9B */ + 0x87, 0xAA, 0x87, 0xAB, 0x87, 0xAC, 0x87, 0xAD, /* 0x9C-0x9F */ + 0x87, 0xAE, 0x87, 0xAF, 0x87, 0xB0, 0xCF, 0xF9, /* 0xA0-0xA3 */ + 0x87, 0xB1, 0x87, 0xB2, 0x87, 0xB3, 0x87, 0xB4, /* 0xA4-0xA7 */ + 0x87, 0xB5, 0x87, 0xB6, 0x87, 0xB7, 0x87, 0xB8, /* 0xA8-0xAB */ + 0x87, 0xB9, 0x87, 0xBA, 0x87, 0xBB, 0xE0, 0xEB, /* 0xAC-0xAF */ + 0x87, 0xBC, 0x87, 0xBD, 0x87, 0xBE, 0x87, 0xBF, /* 0xB0-0xB3 */ + 0x87, 0xC0, 0x87, 0xC1, 0x87, 0xC2, 0xC8, 0xC2, /* 0xB4-0xB7 */ + 0x87, 0xC3, 0x87, 0xC4, 0x87, 0xC5, 0x87, 0xC6, /* 0xB8-0xBB */ + 0xBD, 0xC0, 0x87, 0xC7, 0x87, 0xC8, 0x87, 0xC9, /* 0xBC-0xBF */ + 0x87, 0xCA, 0x87, 0xCB, 0x87, 0xCC, 0x87, 0xCD, /* 0xC0-0xC3 */ + 0x87, 0xCE, 0x87, 0xCF, 0x87, 0xD0, 0x87, 0xD1, /* 0xC4-0xC7 */ + 0x87, 0xD2, 0x87, 0xD3, 0xC4, 0xD2, 0x87, 0xD4, /* 0xC8-0xCB */ + 0x87, 0xD5, 0x87, 0xD6, 0x87, 0xD7, 0x87, 0xD8, /* 0xCC-0xCF */ + 0x87, 0xD9, 0x87, 0xDA, 0x87, 0xDB, 0x87, 0xDC, /* 0xD0-0xD3 */ + 0xE0, 0xEC, 0x87, 0xDD, 0x87, 0xDE, 0xE0, 0xED, /* 0xD4-0xD7 */ + 0x87, 0xDF, 0x87, 0xE0, 0xC7, 0xF4, 0xCB, 0xC4, /* 0xD8-0xDB */ + 0x87, 0xE1, 0xE0, 0xEE, 0xBB, 0xD8, 0xD8, 0xB6, /* 0xDC-0xDF */ + 0xD2, 0xF2, 0xE0, 0xEF, 0xCD, 0xC5, 0x87, 0xE2, /* 0xE0-0xE3 */ + 0xB6, 0xDA, 0x87, 0xE3, 0x87, 0xE4, 0x87, 0xE5, /* 0xE4-0xE7 */ + 0x87, 0xE6, 0x87, 0xE7, 0x87, 0xE8, 0xE0, 0xF1, /* 0xE8-0xEB */ + 0x87, 0xE9, 0xD4, 0xB0, 0x87, 0xEA, 0x87, 0xEB, /* 0xEC-0xEF */ + 0xC0, 0xA7, 0xB4, 0xD1, 0x87, 0xEC, 0x87, 0xED, /* 0xF0-0xF3 */ + 0xCE, 0xA7, 0xE0, 0xF0, 0x87, 0xEE, 0x87, 0xEF, /* 0xF4-0xF7 */ + 0x87, 0xF0, 0xE0, 0xF2, 0xB9, 0xCC, 0x87, 0xF1, /* 0xF8-0xFB */ + 0x87, 0xF2, 0xB9, 0xFA, 0xCD, 0xBC, 0xE0, 0xF3, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_57[512] = { + 0x87, 0xF3, 0x87, 0xF4, 0x87, 0xF5, 0xC6, 0xD4, /* 0x00-0x03 */ + 0xE0, 0xF4, 0x87, 0xF6, 0xD4, 0xB2, 0x87, 0xF7, /* 0x04-0x07 */ + 0xC8, 0xA6, 0xE0, 0xF6, 0xE0, 0xF5, 0x87, 0xF8, /* 0x08-0x0B */ + 0x87, 0xF9, 0x87, 0xFA, 0x87, 0xFB, 0x87, 0xFC, /* 0x0C-0x0F */ + 0x87, 0xFD, 0x87, 0xFE, 0x88, 0x40, 0x88, 0x41, /* 0x10-0x13 */ + 0x88, 0x42, 0x88, 0x43, 0x88, 0x44, 0x88, 0x45, /* 0x14-0x17 */ + 0x88, 0x46, 0x88, 0x47, 0x88, 0x48, 0x88, 0x49, /* 0x18-0x1B */ + 0xE0, 0xF7, 0x88, 0x4A, 0x88, 0x4B, 0xCD, 0xC1, /* 0x1C-0x1F */ + 0x88, 0x4C, 0x88, 0x4D, 0x88, 0x4E, 0xCA, 0xA5, /* 0x20-0x23 */ + 0x88, 0x4F, 0x88, 0x50, 0x88, 0x51, 0x88, 0x52, /* 0x24-0x27 */ + 0xD4, 0xDA, 0xDB, 0xD7, 0xDB, 0xD9, 0x88, 0x53, /* 0x28-0x2B */ + 0xDB, 0xD8, 0xB9, 0xE7, 0xDB, 0xDC, 0xDB, 0xDD, /* 0x2C-0x2F */ + 0xB5, 0xD8, 0x88, 0x54, 0x88, 0x55, 0xDB, 0xDA, /* 0x30-0x33 */ + 0x88, 0x56, 0x88, 0x57, 0x88, 0x58, 0x88, 0x59, /* 0x34-0x37 */ + 0x88, 0x5A, 0xDB, 0xDB, 0xB3, 0xA1, 0xDB, 0xDF, /* 0x38-0x3B */ + 0x88, 0x5B, 0x88, 0x5C, 0xBB, 0xF8, 0x88, 0x5D, /* 0x3C-0x3F */ + 0xD6, 0xB7, 0x88, 0x5E, 0xDB, 0xE0, 0x88, 0x5F, /* 0x40-0x43 */ + 0x88, 0x60, 0x88, 0x61, 0x88, 0x62, 0xBE, 0xF9, /* 0x44-0x47 */ + 0x88, 0x63, 0x88, 0x64, 0xB7, 0xBB, 0x88, 0x65, /* 0x48-0x4B */ + 0xDB, 0xD0, 0xCC, 0xAE, 0xBF, 0xB2, 0xBB, 0xB5, /* 0x4C-0x4F */ + 0xD7, 0xF8, 0xBF, 0xD3, 0x88, 0x66, 0x88, 0x67, /* 0x50-0x53 */ + 0x88, 0x68, 0x88, 0x69, 0x88, 0x6A, 0xBF, 0xE9, /* 0x54-0x57 */ + 0x88, 0x6B, 0x88, 0x6C, 0xBC, 0xE1, 0xCC, 0xB3, /* 0x58-0x5B */ + 0xDB, 0xDE, 0xB0, 0xD3, 0xCE, 0xEB, 0xB7, 0xD8, /* 0x5C-0x5F */ + 0xD7, 0xB9, 0xC6, 0xC2, 0x88, 0x6D, 0x88, 0x6E, /* 0x60-0x63 */ + 0xC0, 0xA4, 0x88, 0x6F, 0xCC, 0xB9, 0x88, 0x70, /* 0x64-0x67 */ + 0xDB, 0xE7, 0xDB, 0xE1, 0xC6, 0xBA, 0xDB, 0xE3, /* 0x68-0x6B */ + 0x88, 0x71, 0xDB, 0xE8, 0x88, 0x72, 0xC5, 0xF7, /* 0x6C-0x6F */ + 0x88, 0x73, 0x88, 0x74, 0x88, 0x75, 0xDB, 0xEA, /* 0x70-0x73 */ + 0x88, 0x76, 0x88, 0x77, 0xDB, 0xE9, 0xBF, 0xC0, /* 0x74-0x77 */ + 0x88, 0x78, 0x88, 0x79, 0x88, 0x7A, 0xDB, 0xE6, /* 0x78-0x7B */ + 0xDB, 0xE5, 0x88, 0x7B, 0x88, 0x7C, 0x88, 0x7D, /* 0x7C-0x7F */ + + 0x88, 0x7E, 0x88, 0x80, 0xB4, 0xB9, 0xC0, 0xAC, /* 0x80-0x83 */ + 0xC2, 0xA2, 0xDB, 0xE2, 0xDB, 0xE4, 0x88, 0x81, /* 0x84-0x87 */ + 0x88, 0x82, 0x88, 0x83, 0x88, 0x84, 0xD0, 0xCD, /* 0x88-0x8B */ + 0xDB, 0xED, 0x88, 0x85, 0x88, 0x86, 0x88, 0x87, /* 0x8C-0x8F */ + 0x88, 0x88, 0x88, 0x89, 0xC0, 0xDD, 0xDB, 0xF2, /* 0x90-0x93 */ + 0x88, 0x8A, 0x88, 0x8B, 0x88, 0x8C, 0x88, 0x8D, /* 0x94-0x97 */ + 0x88, 0x8E, 0x88, 0x8F, 0x88, 0x90, 0xB6, 0xE2, /* 0x98-0x9B */ + 0x88, 0x91, 0x88, 0x92, 0x88, 0x93, 0x88, 0x94, /* 0x9C-0x9F */ + 0xDB, 0xF3, 0xDB, 0xD2, 0xB9, 0xB8, 0xD4, 0xAB, /* 0xA0-0xA3 */ + 0xDB, 0xEC, 0x88, 0x95, 0xBF, 0xD1, 0xDB, 0xF0, /* 0xA4-0xA7 */ + 0x88, 0x96, 0xDB, 0xD1, 0x88, 0x97, 0xB5, 0xE6, /* 0xA8-0xAB */ + 0x88, 0x98, 0xDB, 0xEB, 0xBF, 0xE5, 0x88, 0x99, /* 0xAC-0xAF */ + 0x88, 0x9A, 0x88, 0x9B, 0xDB, 0xEE, 0x88, 0x9C, /* 0xB0-0xB3 */ + 0xDB, 0xF1, 0x88, 0x9D, 0x88, 0x9E, 0x88, 0x9F, /* 0xB4-0xB7 */ + 0xDB, 0xF9, 0x88, 0xA0, 0x88, 0xA1, 0x88, 0xA2, /* 0xB8-0xBB */ + 0x88, 0xA3, 0x88, 0xA4, 0x88, 0xA5, 0x88, 0xA6, /* 0xBC-0xBF */ + 0x88, 0xA7, 0x88, 0xA8, 0xB9, 0xA1, 0xB0, 0xA3, /* 0xC0-0xC3 */ + 0x88, 0xA9, 0x88, 0xAA, 0x88, 0xAB, 0x88, 0xAC, /* 0xC4-0xC7 */ + 0x88, 0xAD, 0x88, 0xAE, 0x88, 0xAF, 0xC2, 0xF1, /* 0xC8-0xCB */ + 0x88, 0xB0, 0x88, 0xB1, 0xB3, 0xC7, 0xDB, 0xEF, /* 0xCC-0xCF */ + 0x88, 0xB2, 0x88, 0xB3, 0xDB, 0xF8, 0x88, 0xB4, /* 0xD0-0xD3 */ + 0xC6, 0xD2, 0xDB, 0xF4, 0x88, 0xB5, 0x88, 0xB6, /* 0xD4-0xD7 */ + 0xDB, 0xF5, 0xDB, 0xF7, 0xDB, 0xF6, 0x88, 0xB7, /* 0xD8-0xDB */ + 0x88, 0xB8, 0xDB, 0xFE, 0x88, 0xB9, 0xD3, 0xF2, /* 0xDC-0xDF */ + 0xB2, 0xBA, 0x88, 0xBA, 0x88, 0xBB, 0x88, 0xBC, /* 0xE0-0xE3 */ + 0xDB, 0xFD, 0x88, 0xBD, 0x88, 0xBE, 0x88, 0xBF, /* 0xE4-0xE7 */ + 0x88, 0xC0, 0x88, 0xC1, 0x88, 0xC2, 0x88, 0xC3, /* 0xE8-0xEB */ + 0x88, 0xC4, 0xDC, 0xA4, 0x88, 0xC5, 0xDB, 0xFB, /* 0xEC-0xEF */ + 0x88, 0xC6, 0x88, 0xC7, 0x88, 0xC8, 0x88, 0xC9, /* 0xF0-0xF3 */ + 0xDB, 0xFA, 0x88, 0xCA, 0x88, 0xCB, 0x88, 0xCC, /* 0xF4-0xF7 */ + 0xDB, 0xFC, 0xC5, 0xE0, 0xBB, 0xF9, 0x88, 0xCD, /* 0xF8-0xFB */ + 0x88, 0xCE, 0xDC, 0xA3, 0x88, 0xCF, 0x88, 0xD0, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_58[512] = { + 0xDC, 0xA5, 0x88, 0xD1, 0xCC, 0xC3, 0x88, 0xD2, /* 0x00-0x03 */ + 0x88, 0xD3, 0x88, 0xD4, 0xB6, 0xD1, 0xDD, 0xC0, /* 0x04-0x07 */ + 0x88, 0xD5, 0x88, 0xD6, 0x88, 0xD7, 0xDC, 0xA1, /* 0x08-0x0B */ + 0x88, 0xD8, 0xDC, 0xA2, 0x88, 0xD9, 0x88, 0xDA, /* 0x0C-0x0F */ + 0x88, 0xDB, 0xC7, 0xB5, 0x88, 0xDC, 0x88, 0xDD, /* 0x10-0x13 */ + 0x88, 0xDE, 0xB6, 0xE9, 0x88, 0xDF, 0x88, 0xE0, /* 0x14-0x17 */ + 0x88, 0xE1, 0xDC, 0xA7, 0x88, 0xE2, 0x88, 0xE3, /* 0x18-0x1B */ + 0x88, 0xE4, 0x88, 0xE5, 0xDC, 0xA6, 0x88, 0xE6, /* 0x1C-0x1F */ + 0xDC, 0xA9, 0xB1, 0xA4, 0x88, 0xE7, 0x88, 0xE8, /* 0x20-0x23 */ + 0xB5, 0xCC, 0x88, 0xE9, 0x88, 0xEA, 0x88, 0xEB, /* 0x24-0x27 */ + 0x88, 0xEC, 0x88, 0xED, 0xBF, 0xB0, 0x88, 0xEE, /* 0x28-0x2B */ + 0x88, 0xEF, 0x88, 0xF0, 0x88, 0xF1, 0x88, 0xF2, /* 0x2C-0x2F */ + 0xD1, 0xDF, 0x88, 0xF3, 0x88, 0xF4, 0x88, 0xF5, /* 0x30-0x33 */ + 0x88, 0xF6, 0xB6, 0xC2, 0x88, 0xF7, 0x88, 0xF8, /* 0x34-0x37 */ + 0x88, 0xF9, 0x88, 0xFA, 0x88, 0xFB, 0x88, 0xFC, /* 0x38-0x3B */ + 0x88, 0xFD, 0x88, 0xFE, 0x89, 0x40, 0x89, 0x41, /* 0x3C-0x3F */ + 0x89, 0x42, 0x89, 0x43, 0x89, 0x44, 0x89, 0x45, /* 0x40-0x43 */ + 0xDC, 0xA8, 0x89, 0x46, 0x89, 0x47, 0x89, 0x48, /* 0x44-0x47 */ + 0x89, 0x49, 0x89, 0x4A, 0x89, 0x4B, 0x89, 0x4C, /* 0x48-0x4B */ + 0xCB, 0xFA, 0xEB, 0xF3, 0x89, 0x4D, 0x89, 0x4E, /* 0x4C-0x4F */ + 0x89, 0x4F, 0xCB, 0xDC, 0x89, 0x50, 0x89, 0x51, /* 0x50-0x53 */ + 0xCB, 0xFE, 0x89, 0x52, 0x89, 0x53, 0x89, 0x54, /* 0x54-0x57 */ + 0xCC, 0xC1, 0x89, 0x55, 0x89, 0x56, 0x89, 0x57, /* 0x58-0x5B */ + 0x89, 0x58, 0x89, 0x59, 0xC8, 0xFB, 0x89, 0x5A, /* 0x5C-0x5F */ + 0x89, 0x5B, 0x89, 0x5C, 0x89, 0x5D, 0x89, 0x5E, /* 0x60-0x63 */ + 0x89, 0x5F, 0xDC, 0xAA, 0x89, 0x60, 0x89, 0x61, /* 0x64-0x67 */ + 0x89, 0x62, 0x89, 0x63, 0x89, 0x64, 0xCC, 0xEE, /* 0x68-0x6B */ + 0xDC, 0xAB, 0x89, 0x65, 0x89, 0x66, 0x89, 0x67, /* 0x6C-0x6F */ + 0x89, 0x68, 0x89, 0x69, 0x89, 0x6A, 0x89, 0x6B, /* 0x70-0x73 */ + 0x89, 0x6C, 0x89, 0x6D, 0x89, 0x6E, 0x89, 0x6F, /* 0x74-0x77 */ + 0x89, 0x70, 0x89, 0x71, 0x89, 0x72, 0x89, 0x73, /* 0x78-0x7B */ + 0x89, 0x74, 0x89, 0x75, 0xDB, 0xD3, 0x89, 0x76, /* 0x7C-0x7F */ + + 0xDC, 0xAF, 0xDC, 0xAC, 0x89, 0x77, 0xBE, 0xB3, /* 0x80-0x83 */ + 0x89, 0x78, 0xCA, 0xFB, 0x89, 0x79, 0x89, 0x7A, /* 0x84-0x87 */ + 0x89, 0x7B, 0xDC, 0xAD, 0x89, 0x7C, 0x89, 0x7D, /* 0x88-0x8B */ + 0x89, 0x7E, 0x89, 0x80, 0x89, 0x81, 0x89, 0x82, /* 0x8C-0x8F */ + 0x89, 0x83, 0x89, 0x84, 0xC9, 0xCA, 0xC4, 0xB9, /* 0x90-0x93 */ + 0x89, 0x85, 0x89, 0x86, 0x89, 0x87, 0x89, 0x88, /* 0x94-0x97 */ + 0x89, 0x89, 0xC7, 0xBD, 0xDC, 0xAE, 0x89, 0x8A, /* 0x98-0x9B */ + 0x89, 0x8B, 0x89, 0x8C, 0xD4, 0xF6, 0xD0, 0xE6, /* 0x9C-0x9F */ + 0x89, 0x8D, 0x89, 0x8E, 0x89, 0x8F, 0x89, 0x90, /* 0xA0-0xA3 */ + 0x89, 0x91, 0x89, 0x92, 0x89, 0x93, 0x89, 0x94, /* 0xA4-0xA7 */ + 0xC4, 0xAB, 0xB6, 0xD5, 0x89, 0x95, 0x89, 0x96, /* 0xA8-0xAB */ + 0x89, 0x97, 0x89, 0x98, 0x89, 0x99, 0x89, 0x9A, /* 0xAC-0xAF */ + 0x89, 0x9B, 0x89, 0x9C, 0x89, 0x9D, 0x89, 0x9E, /* 0xB0-0xB3 */ + 0x89, 0x9F, 0x89, 0xA0, 0x89, 0xA1, 0x89, 0xA2, /* 0xB4-0xB7 */ + 0x89, 0xA3, 0x89, 0xA4, 0x89, 0xA5, 0x89, 0xA6, /* 0xB8-0xBB */ + 0xDB, 0xD4, 0x89, 0xA7, 0x89, 0xA8, 0x89, 0xA9, /* 0xBC-0xBF */ + 0x89, 0xAA, 0xB1, 0xDA, 0x89, 0xAB, 0x89, 0xAC, /* 0xC0-0xC3 */ + 0x89, 0xAD, 0xDB, 0xD5, 0x89, 0xAE, 0x89, 0xAF, /* 0xC4-0xC7 */ + 0x89, 0xB0, 0x89, 0xB1, 0x89, 0xB2, 0x89, 0xB3, /* 0xC8-0xCB */ + 0x89, 0xB4, 0x89, 0xB5, 0x89, 0xB6, 0x89, 0xB7, /* 0xCC-0xCF */ + 0x89, 0xB8, 0xDB, 0xD6, 0x89, 0xB9, 0x89, 0xBA, /* 0xD0-0xD3 */ + 0x89, 0xBB, 0xBA, 0xBE, 0x89, 0xBC, 0x89, 0xBD, /* 0xD4-0xD7 */ + 0x89, 0xBE, 0x89, 0xBF, 0x89, 0xC0, 0x89, 0xC1, /* 0xD8-0xDB */ + 0x89, 0xC2, 0x89, 0xC3, 0x89, 0xC4, 0x89, 0xC5, /* 0xDC-0xDF */ + 0x89, 0xC6, 0x89, 0xC7, 0x89, 0xC8, 0x89, 0xC9, /* 0xE0-0xE3 */ + 0xC8, 0xC0, 0x89, 0xCA, 0x89, 0xCB, 0x89, 0xCC, /* 0xE4-0xE7 */ + 0x89, 0xCD, 0x89, 0xCE, 0x89, 0xCF, 0xCA, 0xBF, /* 0xE8-0xEB */ + 0xC8, 0xC9, 0x89, 0xD0, 0xD7, 0xB3, 0x89, 0xD1, /* 0xEC-0xEF */ + 0xC9, 0xF9, 0x89, 0xD2, 0x89, 0xD3, 0xBF, 0xC7, /* 0xF0-0xF3 */ + 0x89, 0xD4, 0x89, 0xD5, 0xBA, 0xF8, 0x89, 0xD6, /* 0xF4-0xF7 */ + 0x89, 0xD7, 0xD2, 0xBC, 0x89, 0xD8, 0x89, 0xD9, /* 0xF8-0xFB */ + 0x89, 0xDA, 0x89, 0xDB, 0x89, 0xDC, 0x89, 0xDD, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_59[512] = { + 0x89, 0xDE, 0x89, 0xDF, 0xE2, 0xBA, 0x89, 0xE0, /* 0x00-0x03 */ + 0xB4, 0xA6, 0x89, 0xE1, 0x89, 0xE2, 0xB1, 0xB8, /* 0x04-0x07 */ + 0x89, 0xE3, 0x89, 0xE4, 0x89, 0xE5, 0x89, 0xE6, /* 0x08-0x0B */ + 0x89, 0xE7, 0xB8, 0xB4, 0x89, 0xE8, 0xCF, 0xC4, /* 0x0C-0x0F */ + 0x89, 0xE9, 0x89, 0xEA, 0x89, 0xEB, 0x89, 0xEC, /* 0x10-0x13 */ + 0xD9, 0xE7, 0xCF, 0xA6, 0xCD, 0xE2, 0x89, 0xED, /* 0x14-0x17 */ + 0x89, 0xEE, 0xD9, 0xED, 0xB6, 0xE0, 0x89, 0xEF, /* 0x18-0x1B */ + 0xD2, 0xB9, 0x89, 0xF0, 0x89, 0xF1, 0xB9, 0xBB, /* 0x1C-0x1F */ + 0x89, 0xF2, 0x89, 0xF3, 0x89, 0xF4, 0x89, 0xF5, /* 0x20-0x23 */ + 0xE2, 0xB9, 0xE2, 0xB7, 0x89, 0xF6, 0xB4, 0xF3, /* 0x24-0x27 */ + 0x89, 0xF7, 0xCC, 0xEC, 0xCC, 0xAB, 0xB7, 0xF2, /* 0x28-0x2B */ + 0x89, 0xF8, 0xD8, 0xB2, 0xD1, 0xEB, 0xBA, 0xBB, /* 0x2C-0x2F */ + 0x89, 0xF9, 0xCA, 0xA7, 0x89, 0xFA, 0x89, 0xFB, /* 0x30-0x33 */ + 0xCD, 0xB7, 0x89, 0xFC, 0x89, 0xFD, 0xD2, 0xC4, /* 0x34-0x37 */ + 0xBF, 0xE4, 0xBC, 0xD0, 0xB6, 0xE1, 0x89, 0xFE, /* 0x38-0x3B */ + 0xDE, 0xC5, 0x8A, 0x40, 0x8A, 0x41, 0x8A, 0x42, /* 0x3C-0x3F */ + 0x8A, 0x43, 0xDE, 0xC6, 0xDB, 0xBC, 0x8A, 0x44, /* 0x40-0x43 */ + 0xD1, 0xD9, 0x8A, 0x45, 0x8A, 0x46, 0xC6, 0xE6, /* 0x44-0x47 */ + 0xC4, 0xCE, 0xB7, 0xEE, 0x8A, 0x47, 0xB7, 0xDC, /* 0x48-0x4B */ + 0x8A, 0x48, 0x8A, 0x49, 0xBF, 0xFC, 0xD7, 0xE0, /* 0x4C-0x4F */ + 0x8A, 0x4A, 0xC6, 0xF5, 0x8A, 0x4B, 0x8A, 0x4C, /* 0x50-0x53 */ + 0xB1, 0xBC, 0xDE, 0xC8, 0xBD, 0xB1, 0xCC, 0xD7, /* 0x54-0x57 */ + 0xDE, 0xCA, 0x8A, 0x4D, 0xDE, 0xC9, 0x8A, 0x4E, /* 0x58-0x5B */ + 0x8A, 0x4F, 0x8A, 0x50, 0x8A, 0x51, 0x8A, 0x52, /* 0x5C-0x5F */ + 0xB5, 0xEC, 0x8A, 0x53, 0xC9, 0xDD, 0x8A, 0x54, /* 0x60-0x63 */ + 0x8A, 0x55, 0xB0, 0xC2, 0x8A, 0x56, 0x8A, 0x57, /* 0x64-0x67 */ + 0x8A, 0x58, 0x8A, 0x59, 0x8A, 0x5A, 0x8A, 0x5B, /* 0x68-0x6B */ + 0x8A, 0x5C, 0x8A, 0x5D, 0x8A, 0x5E, 0x8A, 0x5F, /* 0x6C-0x6F */ + 0x8A, 0x60, 0x8A, 0x61, 0x8A, 0x62, 0xC5, 0xAE, /* 0x70-0x73 */ + 0xC5, 0xAB, 0x8A, 0x63, 0xC4, 0xCC, 0x8A, 0x64, /* 0x74-0x77 */ + 0xBC, 0xE9, 0xCB, 0xFD, 0x8A, 0x65, 0x8A, 0x66, /* 0x78-0x7B */ + 0x8A, 0x67, 0xBA, 0xC3, 0x8A, 0x68, 0x8A, 0x69, /* 0x7C-0x7F */ + + 0x8A, 0x6A, 0xE5, 0xF9, 0xC8, 0xE7, 0xE5, 0xFA, /* 0x80-0x83 */ + 0xCD, 0xFD, 0x8A, 0x6B, 0xD7, 0xB1, 0xB8, 0xBE, /* 0x84-0x87 */ + 0xC2, 0xE8, 0x8A, 0x6C, 0xC8, 0xD1, 0x8A, 0x6D, /* 0x88-0x8B */ + 0x8A, 0x6E, 0xE5, 0xFB, 0x8A, 0x6F, 0x8A, 0x70, /* 0x8C-0x8F */ + 0x8A, 0x71, 0x8A, 0x72, 0xB6, 0xCA, 0xBC, 0xCB, /* 0x90-0x93 */ + 0x8A, 0x73, 0x8A, 0x74, 0xD1, 0xFD, 0xE6, 0xA1, /* 0x94-0x97 */ + 0x8A, 0x75, 0xC3, 0xEE, 0x8A, 0x76, 0x8A, 0x77, /* 0x98-0x9B */ + 0x8A, 0x78, 0x8A, 0x79, 0xE6, 0xA4, 0x8A, 0x7A, /* 0x9C-0x9F */ + 0x8A, 0x7B, 0x8A, 0x7C, 0x8A, 0x7D, 0xE5, 0xFE, /* 0xA0-0xA3 */ + 0xE6, 0xA5, 0xCD, 0xD7, 0x8A, 0x7E, 0x8A, 0x80, /* 0xA4-0xA7 */ + 0xB7, 0xC1, 0xE5, 0xFC, 0xE5, 0xFD, 0xE6, 0xA3, /* 0xA8-0xAB */ + 0x8A, 0x81, 0x8A, 0x82, 0xC4, 0xDD, 0xE6, 0xA8, /* 0xAC-0xAF */ + 0x8A, 0x83, 0x8A, 0x84, 0xE6, 0xA7, 0x8A, 0x85, /* 0xB0-0xB3 */ + 0x8A, 0x86, 0x8A, 0x87, 0x8A, 0x88, 0x8A, 0x89, /* 0xB4-0xB7 */ + 0x8A, 0x8A, 0xC3, 0xC3, 0x8A, 0x8B, 0xC6, 0xDE, /* 0xB8-0xBB */ + 0x8A, 0x8C, 0x8A, 0x8D, 0xE6, 0xAA, 0x8A, 0x8E, /* 0xBC-0xBF */ + 0x8A, 0x8F, 0x8A, 0x90, 0x8A, 0x91, 0x8A, 0x92, /* 0xC0-0xC3 */ + 0x8A, 0x93, 0x8A, 0x94, 0xC4, 0xB7, 0x8A, 0x95, /* 0xC4-0xC7 */ + 0x8A, 0x96, 0x8A, 0x97, 0xE6, 0xA2, 0xCA, 0xBC, /* 0xC8-0xCB */ + 0x8A, 0x98, 0x8A, 0x99, 0x8A, 0x9A, 0x8A, 0x9B, /* 0xCC-0xCF */ + 0xBD, 0xE3, 0xB9, 0xC3, 0xE6, 0xA6, 0xD0, 0xD5, /* 0xD0-0xD3 */ + 0xCE, 0xAF, 0x8A, 0x9C, 0x8A, 0x9D, 0xE6, 0xA9, /* 0xD4-0xD7 */ + 0xE6, 0xB0, 0x8A, 0x9E, 0xD2, 0xA6, 0x8A, 0x9F, /* 0xD8-0xDB */ + 0xBD, 0xAA, 0xE6, 0xAD, 0x8A, 0xA0, 0x8A, 0xA1, /* 0xDC-0xDF */ + 0x8A, 0xA2, 0x8A, 0xA3, 0x8A, 0xA4, 0xE6, 0xAF, /* 0xE0-0xE3 */ + 0x8A, 0xA5, 0xC0, 0xD1, 0x8A, 0xA6, 0x8A, 0xA7, /* 0xE4-0xE7 */ + 0xD2, 0xCC, 0x8A, 0xA8, 0x8A, 0xA9, 0x8A, 0xAA, /* 0xE8-0xEB */ + 0xBC, 0xA7, 0x8A, 0xAB, 0x8A, 0xAC, 0x8A, 0xAD, /* 0xEC-0xEF */ + 0x8A, 0xAE, 0x8A, 0xAF, 0x8A, 0xB0, 0x8A, 0xB1, /* 0xF0-0xF3 */ + 0x8A, 0xB2, 0x8A, 0xB3, 0x8A, 0xB4, 0x8A, 0xB5, /* 0xF4-0xF7 */ + 0x8A, 0xB6, 0xE6, 0xB1, 0x8A, 0xB7, 0xD2, 0xF6, /* 0xF8-0xFB */ + 0x8A, 0xB8, 0x8A, 0xB9, 0x8A, 0xBA, 0xD7, 0xCB, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_5A[512] = { + 0x8A, 0xBB, 0xCD, 0xFE, 0x8A, 0xBC, 0xCD, 0xDE, /* 0x00-0x03 */ + 0xC2, 0xA6, 0xE6, 0xAB, 0xE6, 0xAC, 0xBD, 0xBF, /* 0x04-0x07 */ + 0xE6, 0xAE, 0xE6, 0xB3, 0x8A, 0xBD, 0x8A, 0xBE, /* 0x08-0x0B */ + 0xE6, 0xB2, 0x8A, 0xBF, 0x8A, 0xC0, 0x8A, 0xC1, /* 0x0C-0x0F */ + 0x8A, 0xC2, 0xE6, 0xB6, 0x8A, 0xC3, 0xE6, 0xB8, /* 0x10-0x13 */ + 0x8A, 0xC4, 0x8A, 0xC5, 0x8A, 0xC6, 0x8A, 0xC7, /* 0x14-0x17 */ + 0xC4, 0xEF, 0x8A, 0xC8, 0x8A, 0xC9, 0x8A, 0xCA, /* 0x18-0x1B */ + 0xC4, 0xC8, 0x8A, 0xCB, 0x8A, 0xCC, 0xBE, 0xEA, /* 0x1C-0x1F */ + 0xC9, 0xEF, 0x8A, 0xCD, 0x8A, 0xCE, 0xE6, 0xB7, /* 0x20-0x23 */ + 0x8A, 0xCF, 0xB6, 0xF0, 0x8A, 0xD0, 0x8A, 0xD1, /* 0x24-0x27 */ + 0x8A, 0xD2, 0xC3, 0xE4, 0x8A, 0xD3, 0x8A, 0xD4, /* 0x28-0x2B */ + 0x8A, 0xD5, 0x8A, 0xD6, 0x8A, 0xD7, 0x8A, 0xD8, /* 0x2C-0x2F */ + 0x8A, 0xD9, 0xD3, 0xE9, 0xE6, 0xB4, 0x8A, 0xDA, /* 0x30-0x33 */ + 0xE6, 0xB5, 0x8A, 0xDB, 0xC8, 0xA2, 0x8A, 0xDC, /* 0x34-0x37 */ + 0x8A, 0xDD, 0x8A, 0xDE, 0x8A, 0xDF, 0x8A, 0xE0, /* 0x38-0x3B */ + 0xE6, 0xBD, 0x8A, 0xE1, 0x8A, 0xE2, 0x8A, 0xE3, /* 0x3C-0x3F */ + 0xE6, 0xB9, 0x8A, 0xE4, 0x8A, 0xE5, 0x8A, 0xE6, /* 0x40-0x43 */ + 0x8A, 0xE7, 0x8A, 0xE8, 0xC6, 0xC5, 0x8A, 0xE9, /* 0x44-0x47 */ + 0x8A, 0xEA, 0xCD, 0xF1, 0xE6, 0xBB, 0x8A, 0xEB, /* 0x48-0x4B */ + 0x8A, 0xEC, 0x8A, 0xED, 0x8A, 0xEE, 0x8A, 0xEF, /* 0x4C-0x4F */ + 0x8A, 0xF0, 0x8A, 0xF1, 0x8A, 0xF2, 0x8A, 0xF3, /* 0x50-0x53 */ + 0x8A, 0xF4, 0xE6, 0xBC, 0x8A, 0xF5, 0x8A, 0xF6, /* 0x54-0x57 */ + 0x8A, 0xF7, 0x8A, 0xF8, 0xBB, 0xE9, 0x8A, 0xF9, /* 0x58-0x5B */ + 0x8A, 0xFA, 0x8A, 0xFB, 0x8A, 0xFC, 0x8A, 0xFD, /* 0x5C-0x5F */ + 0x8A, 0xFE, 0x8B, 0x40, 0xE6, 0xBE, 0x8B, 0x41, /* 0x60-0x63 */ + 0x8B, 0x42, 0x8B, 0x43, 0x8B, 0x44, 0xE6, 0xBA, /* 0x64-0x67 */ + 0x8B, 0x45, 0x8B, 0x46, 0xC0, 0xB7, 0x8B, 0x47, /* 0x68-0x6B */ + 0x8B, 0x48, 0x8B, 0x49, 0x8B, 0x4A, 0x8B, 0x4B, /* 0x6C-0x6F */ + 0x8B, 0x4C, 0x8B, 0x4D, 0x8B, 0x4E, 0x8B, 0x4F, /* 0x70-0x73 */ + 0xD3, 0xA4, 0xE6, 0xBF, 0xC9, 0xF4, 0xE6, 0xC3, /* 0x74-0x77 */ + 0x8B, 0x50, 0x8B, 0x51, 0xE6, 0xC4, 0x8B, 0x52, /* 0x78-0x7B */ + 0x8B, 0x53, 0x8B, 0x54, 0x8B, 0x55, 0xD0, 0xF6, /* 0x7C-0x7F */ + + 0x8B, 0x56, 0x8B, 0x57, 0x8B, 0x58, 0x8B, 0x59, /* 0x80-0x83 */ + 0x8B, 0x5A, 0x8B, 0x5B, 0x8B, 0x5C, 0x8B, 0x5D, /* 0x84-0x87 */ + 0x8B, 0x5E, 0x8B, 0x5F, 0x8B, 0x60, 0x8B, 0x61, /* 0x88-0x8B */ + 0x8B, 0x62, 0x8B, 0x63, 0x8B, 0x64, 0x8B, 0x65, /* 0x8C-0x8F */ + 0x8B, 0x66, 0x8B, 0x67, 0xC3, 0xBD, 0x8B, 0x68, /* 0x90-0x93 */ + 0x8B, 0x69, 0x8B, 0x6A, 0x8B, 0x6B, 0x8B, 0x6C, /* 0x94-0x97 */ + 0x8B, 0x6D, 0x8B, 0x6E, 0xC3, 0xC4, 0xE6, 0xC2, /* 0x98-0x9B */ + 0x8B, 0x6F, 0x8B, 0x70, 0x8B, 0x71, 0x8B, 0x72, /* 0x9C-0x9F */ + 0x8B, 0x73, 0x8B, 0x74, 0x8B, 0x75, 0x8B, 0x76, /* 0xA0-0xA3 */ + 0x8B, 0x77, 0x8B, 0x78, 0x8B, 0x79, 0x8B, 0x7A, /* 0xA4-0xA7 */ + 0x8B, 0x7B, 0x8B, 0x7C, 0xE6, 0xC1, 0x8B, 0x7D, /* 0xA8-0xAB */ + 0x8B, 0x7E, 0x8B, 0x80, 0x8B, 0x81, 0x8B, 0x82, /* 0xAC-0xAF */ + 0x8B, 0x83, 0x8B, 0x84, 0xE6, 0xC7, 0xCF, 0xB1, /* 0xB0-0xB3 */ + 0x8B, 0x85, 0xEB, 0xF4, 0x8B, 0x86, 0x8B, 0x87, /* 0xB4-0xB7 */ + 0xE6, 0xCA, 0x8B, 0x88, 0x8B, 0x89, 0x8B, 0x8A, /* 0xB8-0xBB */ + 0x8B, 0x8B, 0x8B, 0x8C, 0xE6, 0xC5, 0x8B, 0x8D, /* 0xBC-0xBF */ + 0x8B, 0x8E, 0xBC, 0xDE, 0xC9, 0xA9, 0x8B, 0x8F, /* 0xC0-0xC3 */ + 0x8B, 0x90, 0x8B, 0x91, 0x8B, 0x92, 0x8B, 0x93, /* 0xC4-0xC7 */ + 0x8B, 0x94, 0xBC, 0xB5, 0x8B, 0x95, 0x8B, 0x96, /* 0xC8-0xCB */ + 0xCF, 0xD3, 0x8B, 0x97, 0x8B, 0x98, 0x8B, 0x99, /* 0xCC-0xCF */ + 0x8B, 0x9A, 0x8B, 0x9B, 0xE6, 0xC8, 0x8B, 0x9C, /* 0xD0-0xD3 */ + 0xE6, 0xC9, 0x8B, 0x9D, 0xE6, 0xCE, 0x8B, 0x9E, /* 0xD4-0xD7 */ + 0xE6, 0xD0, 0x8B, 0x9F, 0x8B, 0xA0, 0x8B, 0xA1, /* 0xD8-0xDB */ + 0xE6, 0xD1, 0x8B, 0xA2, 0x8B, 0xA3, 0x8B, 0xA4, /* 0xDC-0xDF */ + 0xE6, 0xCB, 0xB5, 0xD5, 0x8B, 0xA5, 0xE6, 0xCC, /* 0xE0-0xE3 */ + 0x8B, 0xA6, 0x8B, 0xA7, 0xE6, 0xCF, 0x8B, 0xA8, /* 0xE4-0xE7 */ + 0x8B, 0xA9, 0xC4, 0xDB, 0x8B, 0xAA, 0xE6, 0xC6, /* 0xE8-0xEB */ + 0x8B, 0xAB, 0x8B, 0xAC, 0x8B, 0xAD, 0x8B, 0xAE, /* 0xEC-0xEF */ + 0x8B, 0xAF, 0xE6, 0xCD, 0x8B, 0xB0, 0x8B, 0xB1, /* 0xF0-0xF3 */ + 0x8B, 0xB2, 0x8B, 0xB3, 0x8B, 0xB4, 0x8B, 0xB5, /* 0xF4-0xF7 */ + 0x8B, 0xB6, 0x8B, 0xB7, 0x8B, 0xB8, 0x8B, 0xB9, /* 0xF8-0xFB */ + 0x8B, 0xBA, 0x8B, 0xBB, 0x8B, 0xBC, 0x8B, 0xBD, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_5B[512] = { + 0x8B, 0xBE, 0x8B, 0xBF, 0x8B, 0xC0, 0x8B, 0xC1, /* 0x00-0x03 */ + 0x8B, 0xC2, 0x8B, 0xC3, 0x8B, 0xC4, 0x8B, 0xC5, /* 0x04-0x07 */ + 0x8B, 0xC6, 0xE6, 0xD2, 0x8B, 0xC7, 0x8B, 0xC8, /* 0x08-0x0B */ + 0x8B, 0xC9, 0x8B, 0xCA, 0x8B, 0xCB, 0x8B, 0xCC, /* 0x0C-0x0F */ + 0x8B, 0xCD, 0x8B, 0xCE, 0x8B, 0xCF, 0x8B, 0xD0, /* 0x10-0x13 */ + 0x8B, 0xD1, 0x8B, 0xD2, 0xE6, 0xD4, 0xE6, 0xD3, /* 0x14-0x17 */ + 0x8B, 0xD3, 0x8B, 0xD4, 0x8B, 0xD5, 0x8B, 0xD6, /* 0x18-0x1B */ + 0x8B, 0xD7, 0x8B, 0xD8, 0x8B, 0xD9, 0x8B, 0xDA, /* 0x1C-0x1F */ + 0x8B, 0xDB, 0x8B, 0xDC, 0x8B, 0xDD, 0x8B, 0xDE, /* 0x20-0x23 */ + 0x8B, 0xDF, 0x8B, 0xE0, 0x8B, 0xE1, 0x8B, 0xE2, /* 0x24-0x27 */ + 0x8B, 0xE3, 0x8B, 0xE4, 0x8B, 0xE5, 0x8B, 0xE6, /* 0x28-0x2B */ + 0x8B, 0xE7, 0x8B, 0xE8, 0x8B, 0xE9, 0x8B, 0xEA, /* 0x2C-0x2F */ + 0x8B, 0xEB, 0x8B, 0xEC, 0xE6, 0xD5, 0x8B, 0xED, /* 0x30-0x33 */ + 0xD9, 0xF8, 0x8B, 0xEE, 0x8B, 0xEF, 0xE6, 0xD6, /* 0x34-0x37 */ + 0x8B, 0xF0, 0x8B, 0xF1, 0x8B, 0xF2, 0x8B, 0xF3, /* 0x38-0x3B */ + 0x8B, 0xF4, 0x8B, 0xF5, 0x8B, 0xF6, 0x8B, 0xF7, /* 0x3C-0x3F */ + 0xE6, 0xD7, 0x8B, 0xF8, 0x8B, 0xF9, 0x8B, 0xFA, /* 0x40-0x43 */ + 0x8B, 0xFB, 0x8B, 0xFC, 0x8B, 0xFD, 0x8B, 0xFE, /* 0x44-0x47 */ + 0x8C, 0x40, 0x8C, 0x41, 0x8C, 0x42, 0x8C, 0x43, /* 0x48-0x4B */ + 0x8C, 0x44, 0x8C, 0x45, 0x8C, 0x46, 0x8C, 0x47, /* 0x4C-0x4F */ + 0xD7, 0xD3, 0xE6, 0xDD, 0x8C, 0x48, 0xE6, 0xDE, /* 0x50-0x53 */ + 0xBF, 0xD7, 0xD4, 0xD0, 0x8C, 0x49, 0xD7, 0xD6, /* 0x54-0x57 */ + 0xB4, 0xE6, 0xCB, 0xEF, 0xE6, 0xDA, 0xD8, 0xC3, /* 0x58-0x5B */ + 0xD7, 0xCE, 0xD0, 0xA2, 0x8C, 0x4A, 0xC3, 0xCF, /* 0x5C-0x5F */ + 0x8C, 0x4B, 0x8C, 0x4C, 0xE6, 0xDF, 0xBC, 0xBE, /* 0x60-0x63 */ + 0xB9, 0xC2, 0xE6, 0xDB, 0xD1, 0xA7, 0x8C, 0x4D, /* 0x64-0x67 */ + 0x8C, 0x4E, 0xBA, 0xA2, 0xC2, 0xCF, 0x8C, 0x4F, /* 0x68-0x6B */ + 0xD8, 0xAB, 0x8C, 0x50, 0x8C, 0x51, 0x8C, 0x52, /* 0x6C-0x6F */ + 0xCA, 0xEB, 0xE5, 0xEE, 0x8C, 0x53, 0xE6, 0xDC, /* 0x70-0x73 */ + 0x8C, 0x54, 0xB7, 0xF5, 0x8C, 0x55, 0x8C, 0x56, /* 0x74-0x77 */ + 0x8C, 0x57, 0x8C, 0x58, 0xC8, 0xE6, 0x8C, 0x59, /* 0x78-0x7B */ + 0x8C, 0x5A, 0xC4, 0xF5, 0x8C, 0x5B, 0x8C, 0x5C, /* 0x7C-0x7F */ + + 0xE5, 0xB2, 0xC4, 0xFE, 0x8C, 0x5D, 0xCB, 0xFC, /* 0x80-0x83 */ + 0xE5, 0xB3, 0xD5, 0xAC, 0x8C, 0x5E, 0xD3, 0xEE, /* 0x84-0x87 */ + 0xCA, 0xD8, 0xB0, 0xB2, 0x8C, 0x5F, 0xCB, 0xCE, /* 0x88-0x8B */ + 0xCD, 0xEA, 0x8C, 0x60, 0x8C, 0x61, 0xBA, 0xEA, /* 0x8C-0x8F */ + 0x8C, 0x62, 0x8C, 0x63, 0x8C, 0x64, 0xE5, 0xB5, /* 0x90-0x93 */ + 0x8C, 0x65, 0xE5, 0xB4, 0x8C, 0x66, 0xD7, 0xDA, /* 0x94-0x97 */ + 0xB9, 0xD9, 0xD6, 0xE6, 0xB6, 0xA8, 0xCD, 0xF0, /* 0x98-0x9B */ + 0xD2, 0xCB, 0xB1, 0xA6, 0xCA, 0xB5, 0x8C, 0x67, /* 0x9C-0x9F */ + 0xB3, 0xE8, 0xC9, 0xF3, 0xBF, 0xCD, 0xD0, 0xFB, /* 0xA0-0xA3 */ + 0xCA, 0xD2, 0xE5, 0xB6, 0xBB, 0xC2, 0x8C, 0x68, /* 0xA4-0xA7 */ + 0x8C, 0x69, 0x8C, 0x6A, 0xCF, 0xDC, 0xB9, 0xAC, /* 0xA8-0xAB */ + 0x8C, 0x6B, 0x8C, 0x6C, 0x8C, 0x6D, 0x8C, 0x6E, /* 0xAC-0xAF */ + 0xD4, 0xD7, 0x8C, 0x6F, 0x8C, 0x70, 0xBA, 0xA6, /* 0xB0-0xB3 */ + 0xD1, 0xE7, 0xCF, 0xFC, 0xBC, 0xD2, 0x8C, 0x71, /* 0xB4-0xB7 */ + 0xE5, 0xB7, 0xC8, 0xDD, 0x8C, 0x72, 0x8C, 0x73, /* 0xB8-0xBB */ + 0x8C, 0x74, 0xBF, 0xED, 0xB1, 0xF6, 0xCB, 0xDE, /* 0xBC-0xBF */ + 0x8C, 0x75, 0x8C, 0x76, 0xBC, 0xC5, 0x8C, 0x77, /* 0xC0-0xC3 */ + 0xBC, 0xC4, 0xD2, 0xFA, 0xC3, 0xDC, 0xBF, 0xDC, /* 0xC4-0xC7 */ + 0x8C, 0x78, 0x8C, 0x79, 0x8C, 0x7A, 0x8C, 0x7B, /* 0xC8-0xCB */ + 0xB8, 0xBB, 0x8C, 0x7C, 0x8C, 0x7D, 0x8C, 0x7E, /* 0xCC-0xCF */ + 0xC3, 0xC2, 0x8C, 0x80, 0xBA, 0xAE, 0xD4, 0xA2, /* 0xD0-0xD3 */ + 0x8C, 0x81, 0x8C, 0x82, 0x8C, 0x83, 0x8C, 0x84, /* 0xD4-0xD7 */ + 0x8C, 0x85, 0x8C, 0x86, 0x8C, 0x87, 0x8C, 0x88, /* 0xD8-0xDB */ + 0x8C, 0x89, 0xC7, 0xDE, 0xC4, 0xAF, 0xB2, 0xEC, /* 0xDC-0xDF */ + 0x8C, 0x8A, 0xB9, 0xD1, 0x8C, 0x8B, 0x8C, 0x8C, /* 0xE0-0xE3 */ + 0xE5, 0xBB, 0xC1, 0xC8, 0x8C, 0x8D, 0x8C, 0x8E, /* 0xE4-0xE7 */ + 0xD5, 0xAF, 0x8C, 0x8F, 0x8C, 0x90, 0x8C, 0x91, /* 0xE8-0xEB */ + 0x8C, 0x92, 0x8C, 0x93, 0xE5, 0xBC, 0x8C, 0x94, /* 0xEC-0xEF */ + 0xE5, 0xBE, 0x8C, 0x95, 0x8C, 0x96, 0x8C, 0x97, /* 0xF0-0xF3 */ + 0x8C, 0x98, 0x8C, 0x99, 0x8C, 0x9A, 0x8C, 0x9B, /* 0xF4-0xF7 */ + 0xB4, 0xE7, 0xB6, 0xD4, 0xCB, 0xC2, 0xD1, 0xB0, /* 0xF8-0xFB */ + 0xB5, 0xBC, 0x8C, 0x9C, 0x8C, 0x9D, 0xCA, 0xD9, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_5C[512] = { + 0x8C, 0x9E, 0xB7, 0xE2, 0x8C, 0x9F, 0x8C, 0xA0, /* 0x00-0x03 */ + 0xC9, 0xE4, 0x8C, 0xA1, 0xBD, 0xAB, 0x8C, 0xA2, /* 0x04-0x07 */ + 0x8C, 0xA3, 0xCE, 0xBE, 0xD7, 0xF0, 0x8C, 0xA4, /* 0x08-0x0B */ + 0x8C, 0xA5, 0x8C, 0xA6, 0x8C, 0xA7, 0xD0, 0xA1, /* 0x0C-0x0F */ + 0x8C, 0xA8, 0xC9, 0xD9, 0x8C, 0xA9, 0x8C, 0xAA, /* 0x10-0x13 */ + 0xB6, 0xFB, 0xE6, 0xD8, 0xBC, 0xE2, 0x8C, 0xAB, /* 0x14-0x17 */ + 0xB3, 0xBE, 0x8C, 0xAC, 0xC9, 0xD0, 0x8C, 0xAD, /* 0x18-0x1B */ + 0xE6, 0xD9, 0xB3, 0xA2, 0x8C, 0xAE, 0x8C, 0xAF, /* 0x1C-0x1F */ + 0x8C, 0xB0, 0x8C, 0xB1, 0xDE, 0xCC, 0x8C, 0xB2, /* 0x20-0x23 */ + 0xD3, 0xC8, 0xDE, 0xCD, 0x8C, 0xB3, 0xD2, 0xA2, /* 0x24-0x27 */ + 0x8C, 0xB4, 0x8C, 0xB5, 0x8C, 0xB6, 0x8C, 0xB7, /* 0x28-0x2B */ + 0xDE, 0xCE, 0x8C, 0xB8, 0x8C, 0xB9, 0x8C, 0xBA, /* 0x2C-0x2F */ + 0x8C, 0xBB, 0xBE, 0xCD, 0x8C, 0xBC, 0x8C, 0xBD, /* 0x30-0x33 */ + 0xDE, 0xCF, 0x8C, 0xBE, 0x8C, 0xBF, 0x8C, 0xC0, /* 0x34-0x37 */ + 0xCA, 0xAC, 0xD2, 0xFC, 0xB3, 0xDF, 0xE5, 0xEA, /* 0x38-0x3B */ + 0xC4, 0xE1, 0xBE, 0xA1, 0xCE, 0xB2, 0xC4, 0xF2, /* 0x3C-0x3F */ + 0xBE, 0xD6, 0xC6, 0xA8, 0xB2, 0xE3, 0x8C, 0xC1, /* 0x40-0x43 */ + 0x8C, 0xC2, 0xBE, 0xD3, 0x8C, 0xC3, 0x8C, 0xC4, /* 0x44-0x47 */ + 0xC7, 0xFC, 0xCC, 0xEB, 0xBD, 0xEC, 0xCE, 0xDD, /* 0x48-0x4B */ + 0x8C, 0xC5, 0x8C, 0xC6, 0xCA, 0xBA, 0xC6, 0xC1, /* 0x4C-0x4F */ + 0xE5, 0xEC, 0xD0, 0xBC, 0x8C, 0xC7, 0x8C, 0xC8, /* 0x50-0x53 */ + 0x8C, 0xC9, 0xD5, 0xB9, 0x8C, 0xCA, 0x8C, 0xCB, /* 0x54-0x57 */ + 0x8C, 0xCC, 0xE5, 0xED, 0x8C, 0xCD, 0x8C, 0xCE, /* 0x58-0x5B */ + 0x8C, 0xCF, 0x8C, 0xD0, 0xCA, 0xF4, 0x8C, 0xD1, /* 0x5C-0x5F */ + 0xCD, 0xC0, 0xC2, 0xC5, 0x8C, 0xD2, 0xE5, 0xEF, /* 0x60-0x63 */ + 0x8C, 0xD3, 0xC2, 0xC4, 0xE5, 0xF0, 0x8C, 0xD4, /* 0x64-0x67 */ + 0x8C, 0xD5, 0x8C, 0xD6, 0x8C, 0xD7, 0x8C, 0xD8, /* 0x68-0x6B */ + 0x8C, 0xD9, 0x8C, 0xDA, 0xE5, 0xF8, 0xCD, 0xCD, /* 0x6C-0x6F */ + 0x8C, 0xDB, 0xC9, 0xBD, 0x8C, 0xDC, 0x8C, 0xDD, /* 0x70-0x73 */ + 0x8C, 0xDE, 0x8C, 0xDF, 0x8C, 0xE0, 0x8C, 0xE1, /* 0x74-0x77 */ + 0x8C, 0xE2, 0xD2, 0xD9, 0xE1, 0xA8, 0x8C, 0xE3, /* 0x78-0x7B */ + 0x8C, 0xE4, 0x8C, 0xE5, 0x8C, 0xE6, 0xD3, 0xEC, /* 0x7C-0x7F */ + + 0x8C, 0xE7, 0xCB, 0xEA, 0xC6, 0xF1, 0x8C, 0xE8, /* 0x80-0x83 */ + 0x8C, 0xE9, 0x8C, 0xEA, 0x8C, 0xEB, 0x8C, 0xEC, /* 0x84-0x87 */ + 0xE1, 0xAC, 0x8C, 0xED, 0x8C, 0xEE, 0x8C, 0xEF, /* 0x88-0x8B */ + 0xE1, 0xA7, 0xE1, 0xA9, 0x8C, 0xF0, 0x8C, 0xF1, /* 0x8C-0x8F */ + 0xE1, 0xAA, 0xE1, 0xAF, 0x8C, 0xF2, 0x8C, 0xF3, /* 0x90-0x93 */ + 0xB2, 0xED, 0x8C, 0xF4, 0xE1, 0xAB, 0xB8, 0xDA, /* 0x94-0x97 */ + 0xE1, 0xAD, 0xE1, 0xAE, 0xE1, 0xB0, 0xB5, 0xBA, /* 0x98-0x9B */ + 0xE1, 0xB1, 0x8C, 0xF5, 0x8C, 0xF6, 0x8C, 0xF7, /* 0x9C-0x9F */ + 0x8C, 0xF8, 0x8C, 0xF9, 0xE1, 0xB3, 0xE1, 0xB8, /* 0xA0-0xA3 */ + 0x8C, 0xFA, 0x8C, 0xFB, 0x8C, 0xFC, 0x8C, 0xFD, /* 0xA4-0xA7 */ + 0x8C, 0xFE, 0xD1, 0xD2, 0x8D, 0x40, 0xE1, 0xB6, /* 0xA8-0xAB */ + 0xE1, 0xB5, 0xC1, 0xEB, 0x8D, 0x41, 0x8D, 0x42, /* 0xAC-0xAF */ + 0x8D, 0x43, 0xE1, 0xB7, 0x8D, 0x44, 0xD4, 0xC0, /* 0xB0-0xB3 */ + 0x8D, 0x45, 0xE1, 0xB2, 0x8D, 0x46, 0xE1, 0xBA, /* 0xB4-0xB7 */ + 0xB0, 0xB6, 0x8D, 0x47, 0x8D, 0x48, 0x8D, 0x49, /* 0xB8-0xBB */ + 0x8D, 0x4A, 0xE1, 0xB4, 0x8D, 0x4B, 0xBF, 0xF9, /* 0xBC-0xBF */ + 0x8D, 0x4C, 0xE1, 0xB9, 0x8D, 0x4D, 0x8D, 0x4E, /* 0xC0-0xC3 */ + 0xE1, 0xBB, 0x8D, 0x4F, 0x8D, 0x50, 0x8D, 0x51, /* 0xC4-0xC7 */ + 0x8D, 0x52, 0x8D, 0x53, 0x8D, 0x54, 0xE1, 0xBE, /* 0xC8-0xCB */ + 0x8D, 0x55, 0x8D, 0x56, 0x8D, 0x57, 0x8D, 0x58, /* 0xCC-0xCF */ + 0x8D, 0x59, 0x8D, 0x5A, 0xE1, 0xBC, 0x8D, 0x5B, /* 0xD0-0xD3 */ + 0x8D, 0x5C, 0x8D, 0x5D, 0x8D, 0x5E, 0x8D, 0x5F, /* 0xD4-0xD7 */ + 0x8D, 0x60, 0xD6, 0xC5, 0x8D, 0x61, 0x8D, 0x62, /* 0xD8-0xDB */ + 0x8D, 0x63, 0x8D, 0x64, 0x8D, 0x65, 0x8D, 0x66, /* 0xDC-0xDF */ + 0x8D, 0x67, 0xCF, 0xBF, 0x8D, 0x68, 0x8D, 0x69, /* 0xE0-0xE3 */ + 0xE1, 0xBD, 0xE1, 0xBF, 0xC2, 0xCD, 0x8D, 0x6A, /* 0xE4-0xE7 */ + 0xB6, 0xEB, 0x8D, 0x6B, 0xD3, 0xF8, 0x8D, 0x6C, /* 0xE8-0xEB */ + 0x8D, 0x6D, 0xC7, 0xCD, 0x8D, 0x6E, 0x8D, 0x6F, /* 0xEC-0xEF */ + 0xB7, 0xE5, 0x8D, 0x70, 0x8D, 0x71, 0x8D, 0x72, /* 0xF0-0xF3 */ + 0x8D, 0x73, 0x8D, 0x74, 0x8D, 0x75, 0x8D, 0x76, /* 0xF4-0xF7 */ + 0x8D, 0x77, 0x8D, 0x78, 0x8D, 0x79, 0xBE, 0xFE, /* 0xF8-0xFB */ + 0x8D, 0x7A, 0x8D, 0x7B, 0x8D, 0x7C, 0x8D, 0x7D, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_5D[512] = { + 0x8D, 0x7E, 0x8D, 0x80, 0xE1, 0xC0, 0xE1, 0xC1, /* 0x00-0x03 */ + 0x8D, 0x81, 0x8D, 0x82, 0xE1, 0xC7, 0xB3, 0xE7, /* 0x04-0x07 */ + 0x8D, 0x83, 0x8D, 0x84, 0x8D, 0x85, 0x8D, 0x86, /* 0x08-0x0B */ + 0x8D, 0x87, 0x8D, 0x88, 0xC6, 0xE9, 0x8D, 0x89, /* 0x0C-0x0F */ + 0x8D, 0x8A, 0x8D, 0x8B, 0x8D, 0x8C, 0x8D, 0x8D, /* 0x10-0x13 */ + 0xB4, 0xDE, 0x8D, 0x8E, 0xD1, 0xC2, 0x8D, 0x8F, /* 0x14-0x17 */ + 0x8D, 0x90, 0x8D, 0x91, 0x8D, 0x92, 0xE1, 0xC8, /* 0x18-0x1B */ + 0x8D, 0x93, 0x8D, 0x94, 0xE1, 0xC6, 0x8D, 0x95, /* 0x1C-0x1F */ + 0x8D, 0x96, 0x8D, 0x97, 0x8D, 0x98, 0x8D, 0x99, /* 0x20-0x23 */ + 0xE1, 0xC5, 0x8D, 0x9A, 0xE1, 0xC3, 0xE1, 0xC2, /* 0x24-0x27 */ + 0x8D, 0x9B, 0xB1, 0xC0, 0x8D, 0x9C, 0x8D, 0x9D, /* 0x28-0x2B */ + 0x8D, 0x9E, 0xD5, 0xB8, 0xE1, 0xC4, 0x8D, 0x9F, /* 0x2C-0x2F */ + 0x8D, 0xA0, 0x8D, 0xA1, 0x8D, 0xA2, 0x8D, 0xA3, /* 0x30-0x33 */ + 0xE1, 0xCB, 0x8D, 0xA4, 0x8D, 0xA5, 0x8D, 0xA6, /* 0x34-0x37 */ + 0x8D, 0xA7, 0x8D, 0xA8, 0x8D, 0xA9, 0x8D, 0xAA, /* 0x38-0x3B */ + 0x8D, 0xAB, 0xE1, 0xCC, 0xE1, 0xCA, 0x8D, 0xAC, /* 0x3C-0x3F */ + 0x8D, 0xAD, 0x8D, 0xAE, 0x8D, 0xAF, 0x8D, 0xB0, /* 0x40-0x43 */ + 0x8D, 0xB1, 0x8D, 0xB2, 0x8D, 0xB3, 0xEF, 0xFA, /* 0x44-0x47 */ + 0x8D, 0xB4, 0x8D, 0xB5, 0xE1, 0xD3, 0xE1, 0xD2, /* 0x48-0x4B */ + 0xC7, 0xB6, 0x8D, 0xB6, 0x8D, 0xB7, 0x8D, 0xB8, /* 0x4C-0x4F */ + 0x8D, 0xB9, 0x8D, 0xBA, 0x8D, 0xBB, 0x8D, 0xBC, /* 0x50-0x53 */ + 0x8D, 0xBD, 0x8D, 0xBE, 0x8D, 0xBF, 0x8D, 0xC0, /* 0x54-0x57 */ + 0xE1, 0xC9, 0x8D, 0xC1, 0x8D, 0xC2, 0xE1, 0xCE, /* 0x58-0x5B */ + 0x8D, 0xC3, 0xE1, 0xD0, 0x8D, 0xC4, 0x8D, 0xC5, /* 0x5C-0x5F */ + 0x8D, 0xC6, 0x8D, 0xC7, 0x8D, 0xC8, 0x8D, 0xC9, /* 0x60-0x63 */ + 0x8D, 0xCA, 0x8D, 0xCB, 0x8D, 0xCC, 0x8D, 0xCD, /* 0x64-0x67 */ + 0x8D, 0xCE, 0xE1, 0xD4, 0x8D, 0xCF, 0xE1, 0xD1, /* 0x68-0x6B */ + 0xE1, 0xCD, 0x8D, 0xD0, 0x8D, 0xD1, 0xE1, 0xCF, /* 0x6C-0x6F */ + 0x8D, 0xD2, 0x8D, 0xD3, 0x8D, 0xD4, 0x8D, 0xD5, /* 0x70-0x73 */ + 0xE1, 0xD5, 0x8D, 0xD6, 0x8D, 0xD7, 0x8D, 0xD8, /* 0x74-0x77 */ + 0x8D, 0xD9, 0x8D, 0xDA, 0x8D, 0xDB, 0x8D, 0xDC, /* 0x78-0x7B */ + 0x8D, 0xDD, 0x8D, 0xDE, 0x8D, 0xDF, 0x8D, 0xE0, /* 0x7C-0x7F */ + + 0x8D, 0xE1, 0x8D, 0xE2, 0xE1, 0xD6, 0x8D, 0xE3, /* 0x80-0x83 */ + 0x8D, 0xE4, 0x8D, 0xE5, 0x8D, 0xE6, 0x8D, 0xE7, /* 0x84-0x87 */ + 0x8D, 0xE8, 0x8D, 0xE9, 0x8D, 0xEA, 0x8D, 0xEB, /* 0x88-0x8B */ + 0x8D, 0xEC, 0x8D, 0xED, 0x8D, 0xEE, 0x8D, 0xEF, /* 0x8C-0x8F */ + 0x8D, 0xF0, 0x8D, 0xF1, 0x8D, 0xF2, 0x8D, 0xF3, /* 0x90-0x93 */ + 0x8D, 0xF4, 0x8D, 0xF5, 0x8D, 0xF6, 0x8D, 0xF7, /* 0x94-0x97 */ + 0x8D, 0xF8, 0xE1, 0xD7, 0x8D, 0xF9, 0x8D, 0xFA, /* 0x98-0x9B */ + 0x8D, 0xFB, 0xE1, 0xD8, 0x8D, 0xFC, 0x8D, 0xFD, /* 0x9C-0x9F */ + 0x8D, 0xFE, 0x8E, 0x40, 0x8E, 0x41, 0x8E, 0x42, /* 0xA0-0xA3 */ + 0x8E, 0x43, 0x8E, 0x44, 0x8E, 0x45, 0x8E, 0x46, /* 0xA4-0xA7 */ + 0x8E, 0x47, 0x8E, 0x48, 0x8E, 0x49, 0x8E, 0x4A, /* 0xA8-0xAB */ + 0x8E, 0x4B, 0x8E, 0x4C, 0x8E, 0x4D, 0x8E, 0x4E, /* 0xAC-0xAF */ + 0x8E, 0x4F, 0x8E, 0x50, 0x8E, 0x51, 0x8E, 0x52, /* 0xB0-0xB3 */ + 0x8E, 0x53, 0x8E, 0x54, 0x8E, 0x55, 0xE1, 0xDA, /* 0xB4-0xB7 */ + 0x8E, 0x56, 0x8E, 0x57, 0x8E, 0x58, 0x8E, 0x59, /* 0xB8-0xBB */ + 0x8E, 0x5A, 0x8E, 0x5B, 0x8E, 0x5C, 0x8E, 0x5D, /* 0xBC-0xBF */ + 0x8E, 0x5E, 0x8E, 0x5F, 0x8E, 0x60, 0x8E, 0x61, /* 0xC0-0xC3 */ + 0x8E, 0x62, 0xE1, 0xDB, 0x8E, 0x63, 0x8E, 0x64, /* 0xC4-0xC7 */ + 0x8E, 0x65, 0x8E, 0x66, 0x8E, 0x67, 0x8E, 0x68, /* 0xC8-0xCB */ + 0x8E, 0x69, 0xCE, 0xA1, 0x8E, 0x6A, 0x8E, 0x6B, /* 0xCC-0xCF */ + 0x8E, 0x6C, 0x8E, 0x6D, 0x8E, 0x6E, 0x8E, 0x6F, /* 0xD0-0xD3 */ + 0x8E, 0x70, 0x8E, 0x71, 0x8E, 0x72, 0x8E, 0x73, /* 0xD4-0xD7 */ + 0x8E, 0x74, 0x8E, 0x75, 0x8E, 0x76, 0xE7, 0xDD, /* 0xD8-0xDB */ + 0x8E, 0x77, 0xB4, 0xA8, 0xD6, 0xDD, 0x8E, 0x78, /* 0xDC-0xDF */ + 0x8E, 0x79, 0xD1, 0xB2, 0xB3, 0xB2, 0x8E, 0x7A, /* 0xE0-0xE3 */ + 0x8E, 0x7B, 0xB9, 0xA4, 0xD7, 0xF3, 0xC7, 0xC9, /* 0xE4-0xE7 */ + 0xBE, 0xDE, 0xB9, 0xAE, 0x8E, 0x7C, 0xCE, 0xD7, /* 0xE8-0xEB */ + 0x8E, 0x7D, 0x8E, 0x7E, 0xB2, 0xEE, 0xDB, 0xCF, /* 0xEC-0xEF */ + 0x8E, 0x80, 0xBC, 0xBA, 0xD2, 0xD1, 0xCB, 0xC8, /* 0xF0-0xF3 */ + 0xB0, 0xCD, 0x8E, 0x81, 0x8E, 0x82, 0xCF, 0xEF, /* 0xF4-0xF7 */ + 0x8E, 0x83, 0x8E, 0x84, 0x8E, 0x85, 0x8E, 0x86, /* 0xF8-0xFB */ + 0x8E, 0x87, 0xD9, 0xE3, 0xBD, 0xED, 0x8E, 0x88, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_5E[512] = { + 0x8E, 0x89, 0xB1, 0xD2, 0xCA, 0xD0, 0xB2, 0xBC, /* 0x00-0x03 */ + 0x8E, 0x8A, 0xCB, 0xA7, 0xB7, 0xAB, 0x8E, 0x8B, /* 0x04-0x07 */ + 0xCA, 0xA6, 0x8E, 0x8C, 0x8E, 0x8D, 0x8E, 0x8E, /* 0x08-0x0B */ + 0xCF, 0xA3, 0x8E, 0x8F, 0x8E, 0x90, 0xE0, 0xF8, /* 0x0C-0x0F */ + 0xD5, 0xCA, 0xE0, 0xFB, 0x8E, 0x91, 0x8E, 0x92, /* 0x10-0x13 */ + 0xE0, 0xFA, 0xC5, 0xC1, 0xCC, 0xFB, 0x8E, 0x93, /* 0x14-0x17 */ + 0xC1, 0xB1, 0xE0, 0xF9, 0xD6, 0xE3, 0xB2, 0xAF, /* 0x18-0x1B */ + 0xD6, 0xC4, 0xB5, 0xDB, 0x8E, 0x94, 0x8E, 0x95, /* 0x1C-0x1F */ + 0x8E, 0x96, 0x8E, 0x97, 0x8E, 0x98, 0x8E, 0x99, /* 0x20-0x23 */ + 0x8E, 0x9A, 0x8E, 0x9B, 0xB4, 0xF8, 0xD6, 0xA1, /* 0x24-0x27 */ + 0x8E, 0x9C, 0x8E, 0x9D, 0x8E, 0x9E, 0x8E, 0x9F, /* 0x28-0x2B */ + 0x8E, 0xA0, 0xCF, 0xAF, 0xB0, 0xEF, 0x8E, 0xA1, /* 0x2C-0x2F */ + 0x8E, 0xA2, 0xE0, 0xFC, 0x8E, 0xA3, 0x8E, 0xA4, /* 0x30-0x33 */ + 0x8E, 0xA5, 0x8E, 0xA6, 0x8E, 0xA7, 0xE1, 0xA1, /* 0x34-0x37 */ + 0xB3, 0xA3, 0x8E, 0xA8, 0x8E, 0xA9, 0xE0, 0xFD, /* 0x38-0x3B */ + 0xE0, 0xFE, 0xC3, 0xB1, 0x8E, 0xAA, 0x8E, 0xAB, /* 0x3C-0x3F */ + 0x8E, 0xAC, 0x8E, 0xAD, 0xC3, 0xDD, 0x8E, 0xAE, /* 0x40-0x43 */ + 0xE1, 0xA2, 0xB7, 0xF9, 0x8E, 0xAF, 0x8E, 0xB0, /* 0x44-0x47 */ + 0x8E, 0xB1, 0x8E, 0xB2, 0x8E, 0xB3, 0x8E, 0xB4, /* 0x48-0x4B */ + 0xBB, 0xCF, 0x8E, 0xB5, 0x8E, 0xB6, 0x8E, 0xB7, /* 0x4C-0x4F */ + 0x8E, 0xB8, 0x8E, 0xB9, 0x8E, 0xBA, 0x8E, 0xBB, /* 0x50-0x53 */ + 0xE1, 0xA3, 0xC4, 0xBB, 0x8E, 0xBC, 0x8E, 0xBD, /* 0x54-0x57 */ + 0x8E, 0xBE, 0x8E, 0xBF, 0x8E, 0xC0, 0xE1, 0xA4, /* 0x58-0x5B */ + 0x8E, 0xC1, 0x8E, 0xC2, 0xE1, 0xA5, 0x8E, 0xC3, /* 0x5C-0x5F */ + 0x8E, 0xC4, 0xE1, 0xA6, 0xB4, 0xB1, 0x8E, 0xC5, /* 0x60-0x63 */ + 0x8E, 0xC6, 0x8E, 0xC7, 0x8E, 0xC8, 0x8E, 0xC9, /* 0x64-0x67 */ + 0x8E, 0xCA, 0x8E, 0xCB, 0x8E, 0xCC, 0x8E, 0xCD, /* 0x68-0x6B */ + 0x8E, 0xCE, 0x8E, 0xCF, 0x8E, 0xD0, 0x8E, 0xD1, /* 0x6C-0x6F */ + 0x8E, 0xD2, 0x8E, 0xD3, 0xB8, 0xC9, 0xC6, 0xBD, /* 0x70-0x73 */ + 0xC4, 0xEA, 0x8E, 0xD4, 0xB2, 0xA2, 0x8E, 0xD5, /* 0x74-0x77 */ + 0xD0, 0xD2, 0x8E, 0xD6, 0xE7, 0xDB, 0xBB, 0xC3, /* 0x78-0x7B */ + 0xD3, 0xD7, 0xD3, 0xC4, 0x8E, 0xD7, 0xB9, 0xE3, /* 0x7C-0x7F */ + + 0xE2, 0xCF, 0x8E, 0xD8, 0x8E, 0xD9, 0x8E, 0xDA, /* 0x80-0x83 */ + 0xD7, 0xAF, 0x8E, 0xDB, 0xC7, 0xEC, 0xB1, 0xD3, /* 0x84-0x87 */ + 0x8E, 0xDC, 0x8E, 0xDD, 0xB4, 0xB2, 0xE2, 0xD1, /* 0x88-0x8B */ + 0x8E, 0xDE, 0x8E, 0xDF, 0x8E, 0xE0, 0xD0, 0xF2, /* 0x8C-0x8F */ + 0xC2, 0xAE, 0xE2, 0xD0, 0x8E, 0xE1, 0xBF, 0xE2, /* 0x90-0x93 */ + 0xD3, 0xA6, 0xB5, 0xD7, 0xE2, 0xD2, 0xB5, 0xEA, /* 0x94-0x97 */ + 0x8E, 0xE2, 0xC3, 0xED, 0xB8, 0xFD, 0x8E, 0xE3, /* 0x98-0x9B */ + 0xB8, 0xAE, 0x8E, 0xE4, 0xC5, 0xD3, 0xB7, 0xCF, /* 0x9C-0x9F */ + 0xE2, 0xD4, 0x8E, 0xE5, 0x8E, 0xE6, 0x8E, 0xE7, /* 0xA0-0xA3 */ + 0x8E, 0xE8, 0xE2, 0xD3, 0xB6, 0xC8, 0xD7, 0xF9, /* 0xA4-0xA7 */ + 0x8E, 0xE9, 0x8E, 0xEA, 0x8E, 0xEB, 0x8E, 0xEC, /* 0xA8-0xAB */ + 0x8E, 0xED, 0xCD, 0xA5, 0x8E, 0xEE, 0x8E, 0xEF, /* 0xAC-0xAF */ + 0x8E, 0xF0, 0x8E, 0xF1, 0x8E, 0xF2, 0xE2, 0xD8, /* 0xB0-0xB3 */ + 0x8E, 0xF3, 0xE2, 0xD6, 0xCA, 0xFC, 0xBF, 0xB5, /* 0xB4-0xB7 */ + 0xD3, 0xB9, 0xE2, 0xD5, 0x8E, 0xF4, 0x8E, 0xF5, /* 0xB8-0xBB */ + 0x8E, 0xF6, 0x8E, 0xF7, 0xE2, 0xD7, 0x8E, 0xF8, /* 0xBC-0xBF */ + 0x8E, 0xF9, 0x8E, 0xFA, 0x8E, 0xFB, 0x8E, 0xFC, /* 0xC0-0xC3 */ + 0x8E, 0xFD, 0x8E, 0xFE, 0x8F, 0x40, 0x8F, 0x41, /* 0xC4-0xC7 */ + 0x8F, 0x42, 0xC1, 0xAE, 0xC0, 0xC8, 0x8F, 0x43, /* 0xC8-0xCB */ + 0x8F, 0x44, 0x8F, 0x45, 0x8F, 0x46, 0x8F, 0x47, /* 0xCC-0xCF */ + 0x8F, 0x48, 0xE2, 0xDB, 0xE2, 0xDA, 0xC0, 0xAA, /* 0xD0-0xD3 */ + 0x8F, 0x49, 0x8F, 0x4A, 0xC1, 0xCE, 0x8F, 0x4B, /* 0xD4-0xD7 */ + 0x8F, 0x4C, 0x8F, 0x4D, 0x8F, 0x4E, 0xE2, 0xDC, /* 0xD8-0xDB */ + 0x8F, 0x4F, 0x8F, 0x50, 0x8F, 0x51, 0x8F, 0x52, /* 0xDC-0xDF */ + 0x8F, 0x53, 0x8F, 0x54, 0x8F, 0x55, 0x8F, 0x56, /* 0xE0-0xE3 */ + 0x8F, 0x57, 0x8F, 0x58, 0x8F, 0x59, 0x8F, 0x5A, /* 0xE4-0xE7 */ + 0xE2, 0xDD, 0x8F, 0x5B, 0xE2, 0xDE, 0x8F, 0x5C, /* 0xE8-0xEB */ + 0x8F, 0x5D, 0x8F, 0x5E, 0x8F, 0x5F, 0x8F, 0x60, /* 0xEC-0xEF */ + 0x8F, 0x61, 0x8F, 0x62, 0x8F, 0x63, 0x8F, 0x64, /* 0xF0-0xF3 */ + 0xDB, 0xC8, 0x8F, 0x65, 0xD1, 0xD3, 0xCD, 0xA2, /* 0xF4-0xF7 */ + 0x8F, 0x66, 0x8F, 0x67, 0xBD, 0xA8, 0x8F, 0x68, /* 0xF8-0xFB */ + 0x8F, 0x69, 0x8F, 0x6A, 0xDE, 0xC3, 0xD8, 0xA5, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_5F[512] = { + 0xBF, 0xAA, 0xDB, 0xCD, 0xD2, 0xEC, 0xC6, 0xFA, /* 0x00-0x03 */ + 0xC5, 0xAA, 0x8F, 0x6B, 0x8F, 0x6C, 0x8F, 0x6D, /* 0x04-0x07 */ + 0xDE, 0xC4, 0x8F, 0x6E, 0xB1, 0xD7, 0xDF, 0xAE, /* 0x08-0x0B */ + 0x8F, 0x6F, 0x8F, 0x70, 0x8F, 0x71, 0xCA, 0xBD, /* 0x0C-0x0F */ + 0x8F, 0x72, 0xDF, 0xB1, 0x8F, 0x73, 0xB9, 0xAD, /* 0x10-0x13 */ + 0x8F, 0x74, 0xD2, 0xFD, 0x8F, 0x75, 0xB8, 0xA5, /* 0x14-0x17 */ + 0xBA, 0xEB, 0x8F, 0x76, 0x8F, 0x77, 0xB3, 0xDA, /* 0x18-0x1B */ + 0x8F, 0x78, 0x8F, 0x79, 0x8F, 0x7A, 0xB5, 0xDC, /* 0x1C-0x1F */ + 0xD5, 0xC5, 0x8F, 0x7B, 0x8F, 0x7C, 0x8F, 0x7D, /* 0x20-0x23 */ + 0x8F, 0x7E, 0xC3, 0xD6, 0xCF, 0xD2, 0xBB, 0xA1, /* 0x24-0x27 */ + 0x8F, 0x80, 0xE5, 0xF3, 0xE5, 0xF2, 0x8F, 0x81, /* 0x28-0x2B */ + 0x8F, 0x82, 0xE5, 0xF4, 0x8F, 0x83, 0xCD, 0xE4, /* 0x2C-0x2F */ + 0x8F, 0x84, 0xC8, 0xF5, 0x8F, 0x85, 0x8F, 0x86, /* 0x30-0x33 */ + 0x8F, 0x87, 0x8F, 0x88, 0x8F, 0x89, 0x8F, 0x8A, /* 0x34-0x37 */ + 0x8F, 0x8B, 0xB5, 0xAF, 0xC7, 0xBF, 0x8F, 0x8C, /* 0x38-0x3B */ + 0xE5, 0xF6, 0x8F, 0x8D, 0x8F, 0x8E, 0x8F, 0x8F, /* 0x3C-0x3F */ + 0xEC, 0xB0, 0x8F, 0x90, 0x8F, 0x91, 0x8F, 0x92, /* 0x40-0x43 */ + 0x8F, 0x93, 0x8F, 0x94, 0x8F, 0x95, 0x8F, 0x96, /* 0x44-0x47 */ + 0x8F, 0x97, 0x8F, 0x98, 0x8F, 0x99, 0x8F, 0x9A, /* 0x48-0x4B */ + 0x8F, 0x9B, 0x8F, 0x9C, 0x8F, 0x9D, 0x8F, 0x9E, /* 0x4C-0x4F */ + 0xE5, 0xE6, 0x8F, 0x9F, 0xB9, 0xE9, 0xB5, 0xB1, /* 0x50-0x53 */ + 0x8F, 0xA0, 0xC2, 0xBC, 0xE5, 0xE8, 0xE5, 0xE7, /* 0x54-0x57 */ + 0xE5, 0xE9, 0x8F, 0xA1, 0x8F, 0xA2, 0x8F, 0xA3, /* 0x58-0x5B */ + 0x8F, 0xA4, 0xD2, 0xCD, 0x8F, 0xA5, 0x8F, 0xA6, /* 0x5C-0x5F */ + 0x8F, 0xA7, 0xE1, 0xEA, 0xD0, 0xCE, 0x8F, 0xA8, /* 0x60-0x63 */ + 0xCD, 0xAE, 0x8F, 0xA9, 0xD1, 0xE5, 0x8F, 0xAA, /* 0x64-0x67 */ + 0x8F, 0xAB, 0xB2, 0xCA, 0xB1, 0xEB, 0x8F, 0xAC, /* 0x68-0x6B */ + 0xB1, 0xF2, 0xC5, 0xED, 0x8F, 0xAD, 0x8F, 0xAE, /* 0x6C-0x6F */ + 0xD5, 0xC3, 0xD3, 0xB0, 0x8F, 0xAF, 0xE1, 0xDC, /* 0x70-0x73 */ + 0x8F, 0xB0, 0x8F, 0xB1, 0x8F, 0xB2, 0xE1, 0xDD, /* 0x74-0x77 */ + 0x8F, 0xB3, 0xD2, 0xDB, 0x8F, 0xB4, 0xB3, 0xB9, /* 0x78-0x7B */ + 0xB1, 0xCB, 0x8F, 0xB5, 0x8F, 0xB6, 0x8F, 0xB7, /* 0x7C-0x7F */ + + 0xCD, 0xF9, 0xD5, 0xF7, 0xE1, 0xDE, 0x8F, 0xB8, /* 0x80-0x83 */ + 0xBE, 0xB6, 0xB4, 0xFD, 0x8F, 0xB9, 0xE1, 0xDF, /* 0x84-0x87 */ + 0xBA, 0xDC, 0xE1, 0xE0, 0xBB, 0xB2, 0xC2, 0xC9, /* 0x88-0x8B */ + 0xE1, 0xE1, 0x8F, 0xBA, 0x8F, 0xBB, 0x8F, 0xBC, /* 0x8C-0x8F */ + 0xD0, 0xEC, 0x8F, 0xBD, 0xCD, 0xBD, 0x8F, 0xBE, /* 0x90-0x93 */ + 0x8F, 0xBF, 0xE1, 0xE2, 0x8F, 0xC0, 0xB5, 0xC3, /* 0x94-0x97 */ + 0xC5, 0xC7, 0xE1, 0xE3, 0x8F, 0xC1, 0x8F, 0xC2, /* 0x98-0x9B */ + 0xE1, 0xE4, 0x8F, 0xC3, 0x8F, 0xC4, 0x8F, 0xC5, /* 0x9C-0x9F */ + 0x8F, 0xC6, 0xD3, 0xF9, 0x8F, 0xC7, 0x8F, 0xC8, /* 0xA0-0xA3 */ + 0x8F, 0xC9, 0x8F, 0xCA, 0x8F, 0xCB, 0x8F, 0xCC, /* 0xA4-0xA7 */ + 0xE1, 0xE5, 0x8F, 0xCD, 0xD1, 0xAD, 0x8F, 0xCE, /* 0xA8-0xAB */ + 0x8F, 0xCF, 0xE1, 0xE6, 0xCE, 0xA2, 0x8F, 0xD0, /* 0xAC-0xAF */ + 0x8F, 0xD1, 0x8F, 0xD2, 0x8F, 0xD3, 0x8F, 0xD4, /* 0xB0-0xB3 */ + 0x8F, 0xD5, 0xE1, 0xE7, 0x8F, 0xD6, 0xB5, 0xC2, /* 0xB4-0xB7 */ + 0x8F, 0xD7, 0x8F, 0xD8, 0x8F, 0xD9, 0x8F, 0xDA, /* 0xB8-0xBB */ + 0xE1, 0xE8, 0xBB, 0xD5, 0x8F, 0xDB, 0x8F, 0xDC, /* 0xBC-0xBF */ + 0x8F, 0xDD, 0x8F, 0xDE, 0x8F, 0xDF, 0xD0, 0xC4, /* 0xC0-0xC3 */ + 0xE2, 0xE0, 0xB1, 0xD8, 0xD2, 0xE4, 0x8F, 0xE0, /* 0xC4-0xC7 */ + 0x8F, 0xE1, 0xE2, 0xE1, 0x8F, 0xE2, 0x8F, 0xE3, /* 0xC8-0xCB */ + 0xBC, 0xC9, 0xC8, 0xCC, 0x8F, 0xE4, 0xE2, 0xE3, /* 0xCC-0xCF */ + 0xEC, 0xFE, 0xEC, 0xFD, 0xDF, 0xAF, 0x8F, 0xE5, /* 0xD0-0xD3 */ + 0x8F, 0xE6, 0x8F, 0xE7, 0xE2, 0xE2, 0xD6, 0xBE, /* 0xD4-0xD7 */ + 0xCD, 0xFC, 0xC3, 0xA6, 0x8F, 0xE8, 0x8F, 0xE9, /* 0xD8-0xDB */ + 0x8F, 0xEA, 0xE3, 0xC3, 0x8F, 0xEB, 0x8F, 0xEC, /* 0xDC-0xDF */ + 0xD6, 0xD2, 0xE2, 0xE7, 0x8F, 0xED, 0x8F, 0xEE, /* 0xE0-0xE3 */ + 0xE2, 0xE8, 0x8F, 0xEF, 0x8F, 0xF0, 0xD3, 0xC7, /* 0xE4-0xE7 */ + 0x8F, 0xF1, 0x8F, 0xF2, 0xE2, 0xEC, 0xBF, 0xEC, /* 0xE8-0xEB */ + 0x8F, 0xF3, 0xE2, 0xED, 0xE2, 0xE5, 0x8F, 0xF4, /* 0xEC-0xEF */ + 0x8F, 0xF5, 0xB3, 0xC0, 0x8F, 0xF6, 0x8F, 0xF7, /* 0xF0-0xF3 */ + 0x8F, 0xF8, 0xC4, 0xEE, 0x8F, 0xF9, 0x8F, 0xFA, /* 0xF4-0xF7 */ + 0xE2, 0xEE, 0x8F, 0xFB, 0x8F, 0xFC, 0xD0, 0xC3, /* 0xF8-0xFB */ + 0x8F, 0xFD, 0xBA, 0xF6, 0xE2, 0xE9, 0xB7, 0xDE, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_60[512] = { + 0xBB, 0xB3, 0xCC, 0xAC, 0xCB, 0xCB, 0xE2, 0xE4, /* 0x00-0x03 */ + 0xE2, 0xE6, 0xE2, 0xEA, 0xE2, 0xEB, 0x8F, 0xFE, /* 0x04-0x07 */ + 0x90, 0x40, 0x90, 0x41, 0xE2, 0xF7, 0x90, 0x42, /* 0x08-0x0B */ + 0x90, 0x43, 0xE2, 0xF4, 0xD4, 0xF5, 0xE2, 0xF3, /* 0x0C-0x0F */ + 0x90, 0x44, 0x90, 0x45, 0xC5, 0xAD, 0x90, 0x46, /* 0x10-0x13 */ + 0xD5, 0xFA, 0xC5, 0xC2, 0xB2, 0xC0, 0x90, 0x47, /* 0x14-0x17 */ + 0x90, 0x48, 0xE2, 0xEF, 0x90, 0x49, 0xE2, 0xF2, /* 0x18-0x1B */ + 0xC1, 0xAF, 0xCB, 0xBC, 0x90, 0x4A, 0x90, 0x4B, /* 0x1C-0x1F */ + 0xB5, 0xA1, 0xE2, 0xF9, 0x90, 0x4C, 0x90, 0x4D, /* 0x20-0x23 */ + 0x90, 0x4E, 0xBC, 0xB1, 0xE2, 0xF1, 0xD0, 0xD4, /* 0x24-0x27 */ + 0xD4, 0xB9, 0xE2, 0xF5, 0xB9, 0xD6, 0xE2, 0xF6, /* 0x28-0x2B */ + 0x90, 0x4F, 0x90, 0x50, 0x90, 0x51, 0xC7, 0xD3, /* 0x2C-0x2F */ + 0x90, 0x52, 0x90, 0x53, 0x90, 0x54, 0x90, 0x55, /* 0x30-0x33 */ + 0x90, 0x56, 0xE2, 0xF0, 0x90, 0x57, 0x90, 0x58, /* 0x34-0x37 */ + 0x90, 0x59, 0x90, 0x5A, 0x90, 0x5B, 0xD7, 0xDC, /* 0x38-0x3B */ + 0xED, 0xA1, 0x90, 0x5C, 0x90, 0x5D, 0xE2, 0xF8, /* 0x3C-0x3F */ + 0x90, 0x5E, 0xED, 0xA5, 0xE2, 0xFE, 0xCA, 0xD1, /* 0x40-0x43 */ + 0x90, 0x5F, 0x90, 0x60, 0x90, 0x61, 0x90, 0x62, /* 0x44-0x47 */ + 0x90, 0x63, 0x90, 0x64, 0x90, 0x65, 0xC1, 0xB5, /* 0x48-0x4B */ + 0x90, 0x66, 0xBB, 0xD0, 0x90, 0x67, 0x90, 0x68, /* 0x4C-0x4F */ + 0xBF, 0xD6, 0x90, 0x69, 0xBA, 0xE3, 0x90, 0x6A, /* 0x50-0x53 */ + 0x90, 0x6B, 0xCB, 0xA1, 0x90, 0x6C, 0x90, 0x6D, /* 0x54-0x57 */ + 0x90, 0x6E, 0xED, 0xA6, 0xED, 0xA3, 0x90, 0x6F, /* 0x58-0x5B */ + 0x90, 0x70, 0xED, 0xA2, 0x90, 0x71, 0x90, 0x72, /* 0x5C-0x5F */ + 0x90, 0x73, 0x90, 0x74, 0xBB, 0xD6, 0xED, 0xA7, /* 0x60-0x63 */ + 0xD0, 0xF4, 0x90, 0x75, 0x90, 0x76, 0xED, 0xA4, /* 0x64-0x67 */ + 0xBA, 0xDE, 0xB6, 0xF7, 0xE3, 0xA1, 0xB6, 0xB2, /* 0x68-0x6B */ + 0xCC, 0xF1, 0xB9, 0xA7, 0x90, 0x77, 0xCF, 0xA2, /* 0x6C-0x6F */ + 0xC7, 0xA1, 0x90, 0x78, 0x90, 0x79, 0xBF, 0xD2, /* 0x70-0x73 */ + 0x90, 0x7A, 0x90, 0x7B, 0xB6, 0xF1, 0x90, 0x7C, /* 0x74-0x77 */ + 0xE2, 0xFA, 0xE2, 0xFB, 0xE2, 0xFD, 0xE2, 0xFC, /* 0x78-0x7B */ + 0xC4, 0xD5, 0xE3, 0xA2, 0x90, 0x7D, 0xD3, 0xC1, /* 0x7C-0x7F */ + + 0x90, 0x7E, 0x90, 0x80, 0x90, 0x81, 0xE3, 0xA7, /* 0x80-0x83 */ + 0xC7, 0xC4, 0x90, 0x82, 0x90, 0x83, 0x90, 0x84, /* 0x84-0x87 */ + 0x90, 0x85, 0xCF, 0xA4, 0x90, 0x86, 0x90, 0x87, /* 0x88-0x8B */ + 0xE3, 0xA9, 0xBA, 0xB7, 0x90, 0x88, 0x90, 0x89, /* 0x8C-0x8F */ + 0x90, 0x8A, 0x90, 0x8B, 0xE3, 0xA8, 0x90, 0x8C, /* 0x90-0x93 */ + 0xBB, 0xDA, 0x90, 0x8D, 0xE3, 0xA3, 0x90, 0x8E, /* 0x94-0x97 */ + 0x90, 0x8F, 0x90, 0x90, 0xE3, 0xA4, 0xE3, 0xAA, /* 0x98-0x9B */ + 0x90, 0x91, 0xE3, 0xA6, 0x90, 0x92, 0xCE, 0xF2, /* 0x9C-0x9F */ + 0xD3, 0xC6, 0x90, 0x93, 0x90, 0x94, 0xBB, 0xBC, /* 0xA0-0xA3 */ + 0x90, 0x95, 0x90, 0x96, 0xD4, 0xC3, 0x90, 0x97, /* 0xA4-0xA7 */ + 0xC4, 0xFA, 0x90, 0x98, 0x90, 0x99, 0xED, 0xA8, /* 0xA8-0xAB */ + 0xD0, 0xFC, 0xE3, 0xA5, 0x90, 0x9A, 0xC3, 0xF5, /* 0xAC-0xAF */ + 0x90, 0x9B, 0xE3, 0xAD, 0xB1, 0xAF, 0x90, 0x9C, /* 0xB0-0xB3 */ + 0xE3, 0xB2, 0x90, 0x9D, 0x90, 0x9E, 0x90, 0x9F, /* 0xB4-0xB7 */ + 0xBC, 0xC2, 0x90, 0xA0, 0x90, 0xA1, 0xE3, 0xAC, /* 0xB8-0xBB */ + 0xB5, 0xBF, 0x90, 0xA2, 0x90, 0xA3, 0x90, 0xA4, /* 0xBC-0xBF */ + 0x90, 0xA5, 0x90, 0xA6, 0x90, 0xA7, 0x90, 0xA8, /* 0xC0-0xC3 */ + 0x90, 0xA9, 0xC7, 0xE9, 0xE3, 0xB0, 0x90, 0xAA, /* 0xC4-0xC7 */ + 0x90, 0xAB, 0x90, 0xAC, 0xBE, 0xAA, 0xCD, 0xEF, /* 0xC8-0xCB */ + 0x90, 0xAD, 0x90, 0xAE, 0x90, 0xAF, 0x90, 0xB0, /* 0xCC-0xCF */ + 0x90, 0xB1, 0xBB, 0xF3, 0x90, 0xB2, 0x90, 0xB3, /* 0xD0-0xD3 */ + 0x90, 0xB4, 0xCC, 0xE8, 0x90, 0xB5, 0x90, 0xB6, /* 0xD4-0xD7 */ + 0xE3, 0xAF, 0x90, 0xB7, 0xE3, 0xB1, 0x90, 0xB8, /* 0xD8-0xDB */ + 0xCF, 0xA7, 0xE3, 0xAE, 0x90, 0xB9, 0xCE, 0xA9, /* 0xDC-0xDF */ + 0xBB, 0xDD, 0x90, 0xBA, 0x90, 0xBB, 0x90, 0xBC, /* 0xE0-0xE3 */ + 0x90, 0xBD, 0x90, 0xBE, 0xB5, 0xEB, 0xBE, 0xE5, /* 0xE4-0xE7 */ + 0xB2, 0xD2, 0xB3, 0xCD, 0x90, 0xBF, 0xB1, 0xB9, /* 0xE8-0xEB */ + 0xE3, 0xAB, 0xB2, 0xD1, 0xB5, 0xAC, 0xB9, 0xDF, /* 0xEC-0xEF */ + 0xB6, 0xE8, 0x90, 0xC0, 0x90, 0xC1, 0xCF, 0xEB, /* 0xF0-0xF3 */ + 0xE3, 0xB7, 0x90, 0xC2, 0xBB, 0xCC, 0x90, 0xC3, /* 0xF4-0xF7 */ + 0x90, 0xC4, 0xC8, 0xC7, 0xD0, 0xCA, 0x90, 0xC5, /* 0xF8-0xFB */ + 0x90, 0xC6, 0x90, 0xC7, 0x90, 0xC8, 0x90, 0xC9, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_61[512] = { + 0xE3, 0xB8, 0xB3, 0xEE, 0x90, 0xCA, 0x90, 0xCB, /* 0x00-0x03 */ + 0x90, 0xCC, 0x90, 0xCD, 0xED, 0xA9, 0x90, 0xCE, /* 0x04-0x07 */ + 0xD3, 0xFA, 0xD3, 0xE4, 0x90, 0xCF, 0x90, 0xD0, /* 0x08-0x0B */ + 0x90, 0xD1, 0xED, 0xAA, 0xE3, 0xB9, 0xD2, 0xE2, /* 0x0C-0x0F */ + 0x90, 0xD2, 0x90, 0xD3, 0x90, 0xD4, 0x90, 0xD5, /* 0x10-0x13 */ + 0x90, 0xD6, 0xE3, 0xB5, 0x90, 0xD7, 0x90, 0xD8, /* 0x14-0x17 */ + 0x90, 0xD9, 0x90, 0xDA, 0xD3, 0xDE, 0x90, 0xDB, /* 0x18-0x1B */ + 0x90, 0xDC, 0x90, 0xDD, 0x90, 0xDE, 0xB8, 0xD0, /* 0x1C-0x1F */ + 0xE3, 0xB3, 0x90, 0xDF, 0x90, 0xE0, 0xE3, 0xB6, /* 0x20-0x23 */ + 0xB7, 0xDF, 0x90, 0xE1, 0xE3, 0xB4, 0xC0, 0xA2, /* 0x24-0x27 */ + 0x90, 0xE2, 0x90, 0xE3, 0x90, 0xE4, 0xE3, 0xBA, /* 0x28-0x2B */ + 0x90, 0xE5, 0x90, 0xE6, 0x90, 0xE7, 0x90, 0xE8, /* 0x2C-0x2F */ + 0x90, 0xE9, 0x90, 0xEA, 0x90, 0xEB, 0x90, 0xEC, /* 0x30-0x33 */ + 0x90, 0xED, 0x90, 0xEE, 0x90, 0xEF, 0x90, 0xF0, /* 0x34-0x37 */ + 0x90, 0xF1, 0x90, 0xF2, 0x90, 0xF3, 0x90, 0xF4, /* 0x38-0x3B */ + 0x90, 0xF5, 0x90, 0xF6, 0x90, 0xF7, 0xD4, 0xB8, /* 0x3C-0x3F */ + 0x90, 0xF8, 0x90, 0xF9, 0x90, 0xFA, 0x90, 0xFB, /* 0x40-0x43 */ + 0x90, 0xFC, 0x90, 0xFD, 0x90, 0xFE, 0x91, 0x40, /* 0x44-0x47 */ + 0xB4, 0xC8, 0x91, 0x41, 0xE3, 0xBB, 0x91, 0x42, /* 0x48-0x4B */ + 0xBB, 0xC5, 0x91, 0x43, 0xC9, 0xF7, 0x91, 0x44, /* 0x4C-0x4F */ + 0x91, 0x45, 0xC9, 0xE5, 0x91, 0x46, 0x91, 0x47, /* 0x50-0x53 */ + 0x91, 0x48, 0xC4, 0xBD, 0x91, 0x49, 0x91, 0x4A, /* 0x54-0x57 */ + 0x91, 0x4B, 0x91, 0x4C, 0x91, 0x4D, 0x91, 0x4E, /* 0x58-0x5B */ + 0x91, 0x4F, 0xED, 0xAB, 0x91, 0x50, 0x91, 0x51, /* 0x5C-0x5F */ + 0x91, 0x52, 0x91, 0x53, 0xC2, 0xFD, 0x91, 0x54, /* 0x60-0x63 */ + 0x91, 0x55, 0x91, 0x56, 0x91, 0x57, 0xBB, 0xDB, /* 0x64-0x67 */ + 0xBF, 0xAE, 0x91, 0x58, 0x91, 0x59, 0x91, 0x5A, /* 0x68-0x6B */ + 0x91, 0x5B, 0x91, 0x5C, 0x91, 0x5D, 0x91, 0x5E, /* 0x6C-0x6F */ + 0xCE, 0xBF, 0x91, 0x5F, 0x91, 0x60, 0x91, 0x61, /* 0x70-0x73 */ + 0x91, 0x62, 0xE3, 0xBC, 0x91, 0x63, 0xBF, 0xB6, /* 0x74-0x77 */ + 0x91, 0x64, 0x91, 0x65, 0x91, 0x66, 0x91, 0x67, /* 0x78-0x7B */ + 0x91, 0x68, 0x91, 0x69, 0x91, 0x6A, 0x91, 0x6B, /* 0x7C-0x7F */ + + 0x91, 0x6C, 0x91, 0x6D, 0x91, 0x6E, 0x91, 0x6F, /* 0x80-0x83 */ + 0x91, 0x70, 0x91, 0x71, 0x91, 0x72, 0x91, 0x73, /* 0x84-0x87 */ + 0x91, 0x74, 0x91, 0x75, 0x91, 0x76, 0xB1, 0xEF, /* 0x88-0x8B */ + 0x91, 0x77, 0x91, 0x78, 0xD4, 0xF7, 0x91, 0x79, /* 0x8C-0x8F */ + 0x91, 0x7A, 0x91, 0x7B, 0x91, 0x7C, 0x91, 0x7D, /* 0x90-0x93 */ + 0xE3, 0xBE, 0x91, 0x7E, 0x91, 0x80, 0x91, 0x81, /* 0x94-0x97 */ + 0x91, 0x82, 0x91, 0x83, 0x91, 0x84, 0x91, 0x85, /* 0x98-0x9B */ + 0x91, 0x86, 0xED, 0xAD, 0x91, 0x87, 0x91, 0x88, /* 0x9C-0x9F */ + 0x91, 0x89, 0x91, 0x8A, 0x91, 0x8B, 0x91, 0x8C, /* 0xA0-0xA3 */ + 0x91, 0x8D, 0x91, 0x8E, 0x91, 0x8F, 0xE3, 0xBF, /* 0xA4-0xA7 */ + 0xBA, 0xA9, 0xED, 0xAC, 0x91, 0x90, 0x91, 0x91, /* 0xA8-0xAB */ + 0xE3, 0xBD, 0x91, 0x92, 0x91, 0x93, 0x91, 0x94, /* 0xAC-0xAF */ + 0x91, 0x95, 0x91, 0x96, 0x91, 0x97, 0x91, 0x98, /* 0xB0-0xB3 */ + 0x91, 0x99, 0x91, 0x9A, 0x91, 0x9B, 0xE3, 0xC0, /* 0xB4-0xB7 */ + 0x91, 0x9C, 0x91, 0x9D, 0x91, 0x9E, 0x91, 0x9F, /* 0xB8-0xBB */ + 0x91, 0xA0, 0x91, 0xA1, 0xBA, 0xB6, 0x91, 0xA2, /* 0xBC-0xBF */ + 0x91, 0xA3, 0x91, 0xA4, 0xB6, 0xAE, 0x91, 0xA5, /* 0xC0-0xC3 */ + 0x91, 0xA6, 0x91, 0xA7, 0x91, 0xA8, 0x91, 0xA9, /* 0xC4-0xC7 */ + 0xD0, 0xB8, 0x91, 0xAA, 0xB0, 0xC3, 0xED, 0xAE, /* 0xC8-0xCB */ + 0x91, 0xAB, 0x91, 0xAC, 0x91, 0xAD, 0x91, 0xAE, /* 0xCC-0xCF */ + 0x91, 0xAF, 0xED, 0xAF, 0xC0, 0xC1, 0x91, 0xB0, /* 0xD0-0xD3 */ + 0xE3, 0xC1, 0x91, 0xB1, 0x91, 0xB2, 0x91, 0xB3, /* 0xD4-0xD7 */ + 0x91, 0xB4, 0x91, 0xB5, 0x91, 0xB6, 0x91, 0xB7, /* 0xD8-0xDB */ + 0x91, 0xB8, 0x91, 0xB9, 0x91, 0xBA, 0x91, 0xBB, /* 0xDC-0xDF */ + 0x91, 0xBC, 0x91, 0xBD, 0x91, 0xBE, 0x91, 0xBF, /* 0xE0-0xE3 */ + 0x91, 0xC0, 0x91, 0xC1, 0xC5, 0xB3, 0x91, 0xC2, /* 0xE4-0xE7 */ + 0x91, 0xC3, 0x91, 0xC4, 0x91, 0xC5, 0x91, 0xC6, /* 0xE8-0xEB */ + 0x91, 0xC7, 0x91, 0xC8, 0x91, 0xC9, 0x91, 0xCA, /* 0xEC-0xEF */ + 0x91, 0xCB, 0x91, 0xCC, 0x91, 0xCD, 0x91, 0xCE, /* 0xF0-0xF3 */ + 0x91, 0xCF, 0xE3, 0xC2, 0x91, 0xD0, 0x91, 0xD1, /* 0xF4-0xF7 */ + 0x91, 0xD2, 0x91, 0xD3, 0x91, 0xD4, 0x91, 0xD5, /* 0xF8-0xFB */ + 0x91, 0xD6, 0x91, 0xD7, 0x91, 0xD8, 0xDC, 0xB2, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_62[512] = { + 0x91, 0xD9, 0x91, 0xDA, 0x91, 0xDB, 0x91, 0xDC, /* 0x00-0x03 */ + 0x91, 0xDD, 0x91, 0xDE, 0xED, 0xB0, 0x91, 0xDF, /* 0x04-0x07 */ + 0xB8, 0xEA, 0x91, 0xE0, 0xCE, 0xEC, 0xEA, 0xA7, /* 0x08-0x0B */ + 0xD0, 0xE7, 0xCA, 0xF9, 0xC8, 0xD6, 0xCF, 0xB7, /* 0x0C-0x0F */ + 0xB3, 0xC9, 0xCE, 0xD2, 0xBD, 0xE4, 0x91, 0xE1, /* 0x10-0x13 */ + 0x91, 0xE2, 0xE3, 0xDE, 0xBB, 0xF2, 0xEA, 0xA8, /* 0x14-0x17 */ + 0xD5, 0xBD, 0x91, 0xE3, 0xC6, 0xDD, 0xEA, 0xA9, /* 0x18-0x1B */ + 0x91, 0xE4, 0x91, 0xE5, 0x91, 0xE6, 0xEA, 0xAA, /* 0x1C-0x1F */ + 0x91, 0xE7, 0xEA, 0xAC, 0xEA, 0xAB, 0x91, 0xE8, /* 0x20-0x23 */ + 0xEA, 0xAE, 0xEA, 0xAD, 0x91, 0xE9, 0x91, 0xEA, /* 0x24-0x27 */ + 0x91, 0xEB, 0x91, 0xEC, 0xBD, 0xD8, 0x91, 0xED, /* 0x28-0x2B */ + 0xEA, 0xAF, 0x91, 0xEE, 0xC2, 0xBE, 0x91, 0xEF, /* 0x2C-0x2F */ + 0x91, 0xF0, 0x91, 0xF1, 0x91, 0xF2, 0xB4, 0xC1, /* 0x30-0x33 */ + 0xB4, 0xF7, 0x91, 0xF3, 0x91, 0xF4, 0xBB, 0xA7, /* 0x34-0x37 */ + 0x91, 0xF5, 0x91, 0xF6, 0x91, 0xF7, 0x91, 0xF8, /* 0x38-0x3B */ + 0x91, 0xF9, 0xEC, 0xE6, 0xEC, 0xE5, 0xB7, 0xBF, /* 0x3C-0x3F */ + 0xCB, 0xF9, 0xB1, 0xE2, 0x91, 0xFA, 0xEC, 0xE7, /* 0x40-0x43 */ + 0x91, 0xFB, 0x91, 0xFC, 0x91, 0xFD, 0xC9, 0xC8, /* 0x44-0x47 */ + 0xEC, 0xE8, 0xEC, 0xE9, 0x91, 0xFE, 0xCA, 0xD6, /* 0x48-0x4B */ + 0xDE, 0xD0, 0xB2, 0xC5, 0xD4, 0xFA, 0x92, 0x40, /* 0x4C-0x4F */ + 0x92, 0x41, 0xC6, 0xCB, 0xB0, 0xC7, 0xB4, 0xF2, /* 0x50-0x53 */ + 0xC8, 0xD3, 0x92, 0x42, 0x92, 0x43, 0x92, 0x44, /* 0x54-0x57 */ + 0xCD, 0xD0, 0x92, 0x45, 0x92, 0x46, 0xBF, 0xB8, /* 0x58-0x5B */ + 0x92, 0x47, 0x92, 0x48, 0x92, 0x49, 0x92, 0x4A, /* 0x5C-0x5F */ + 0x92, 0x4B, 0x92, 0x4C, 0x92, 0x4D, 0xBF, 0xDB, /* 0x60-0x63 */ + 0x92, 0x4E, 0x92, 0x4F, 0xC7, 0xA4, 0xD6, 0xB4, /* 0x64-0x67 */ + 0x92, 0x50, 0xC0, 0xA9, 0xDE, 0xD1, 0xC9, 0xA8, /* 0x68-0x6B */ + 0xD1, 0xEF, 0xC5, 0xA4, 0xB0, 0xE7, 0xB3, 0xB6, /* 0x6C-0x6F */ + 0xC8, 0xC5, 0x92, 0x51, 0x92, 0x52, 0xB0, 0xE2, /* 0x70-0x73 */ + 0x92, 0x53, 0x92, 0x54, 0xB7, 0xF6, 0x92, 0x55, /* 0x74-0x77 */ + 0x92, 0x56, 0xC5, 0xFA, 0x92, 0x57, 0x92, 0x58, /* 0x78-0x7B */ + 0xB6, 0xF3, 0x92, 0x59, 0xD5, 0xD2, 0xB3, 0xD0, /* 0x7C-0x7F */ + + 0xBC, 0xBC, 0x92, 0x5A, 0x92, 0x5B, 0x92, 0x5C, /* 0x80-0x83 */ + 0xB3, 0xAD, 0x92, 0x5D, 0x92, 0x5E, 0x92, 0x5F, /* 0x84-0x87 */ + 0x92, 0x60, 0xBE, 0xF1, 0xB0, 0xD1, 0x92, 0x61, /* 0x88-0x8B */ + 0x92, 0x62, 0x92, 0x63, 0x92, 0x64, 0x92, 0x65, /* 0x8C-0x8F */ + 0x92, 0x66, 0xD2, 0xD6, 0xCA, 0xE3, 0xD7, 0xA5, /* 0x90-0x93 */ + 0x92, 0x67, 0xCD, 0xB6, 0xB6, 0xB6, 0xBF, 0xB9, /* 0x94-0x97 */ + 0xD5, 0xDB, 0x92, 0x68, 0xB8, 0xA7, 0xC5, 0xD7, /* 0x98-0x9B */ + 0x92, 0x69, 0x92, 0x6A, 0x92, 0x6B, 0xDE, 0xD2, /* 0x9C-0x9F */ + 0xBF, 0xD9, 0xC2, 0xD5, 0xC7, 0xC0, 0x92, 0x6C, /* 0xA0-0xA3 */ + 0xBB, 0xA4, 0xB1, 0xA8, 0x92, 0x6D, 0x92, 0x6E, /* 0xA4-0xA7 */ + 0xC5, 0xEA, 0x92, 0x6F, 0x92, 0x70, 0xC5, 0xFB, /* 0xA8-0xAB */ + 0xCC, 0xA7, 0x92, 0x71, 0x92, 0x72, 0x92, 0x73, /* 0xAC-0xAF */ + 0x92, 0x74, 0xB1, 0xA7, 0x92, 0x75, 0x92, 0x76, /* 0xB0-0xB3 */ + 0x92, 0x77, 0xB5, 0xD6, 0x92, 0x78, 0x92, 0x79, /* 0xB4-0xB7 */ + 0x92, 0x7A, 0xC4, 0xA8, 0x92, 0x7B, 0xDE, 0xD3, /* 0xB8-0xBB */ + 0xD1, 0xBA, 0xB3, 0xE9, 0x92, 0x7C, 0xC3, 0xF2, /* 0xBC-0xBF */ + 0x92, 0x7D, 0x92, 0x7E, 0xB7, 0xF7, 0x92, 0x80, /* 0xC0-0xC3 */ + 0xD6, 0xF4, 0xB5, 0xA3, 0xB2, 0xF0, 0xC4, 0xB4, /* 0xC4-0xC7 */ + 0xC4, 0xE9, 0xC0, 0xAD, 0xDE, 0xD4, 0x92, 0x81, /* 0xC8-0xCB */ + 0xB0, 0xE8, 0xC5, 0xC4, 0xC1, 0xE0, 0x92, 0x82, /* 0xCC-0xCF */ + 0xB9, 0xD5, 0x92, 0x83, 0xBE, 0xDC, 0xCD, 0xD8, /* 0xD0-0xD3 */ + 0xB0, 0xCE, 0x92, 0x84, 0xCD, 0xCF, 0xDE, 0xD6, /* 0xD4-0xD7 */ + 0xBE, 0xD0, 0xD7, 0xBE, 0xDE, 0xD5, 0xD5, 0xD0, /* 0xD8-0xDB */ + 0xB0, 0xDD, 0x92, 0x85, 0x92, 0x86, 0xC4, 0xE2, /* 0xDC-0xDF */ + 0x92, 0x87, 0x92, 0x88, 0xC2, 0xA3, 0xBC, 0xF0, /* 0xE0-0xE3 */ + 0x92, 0x89, 0xD3, 0xB5, 0xC0, 0xB9, 0xC5, 0xA1, /* 0xE4-0xE7 */ + 0xB2, 0xA6, 0xD4, 0xF1, 0x92, 0x8A, 0x92, 0x8B, /* 0xE8-0xEB */ + 0xC0, 0xA8, 0xCA, 0xC3, 0xDE, 0xD7, 0xD5, 0xFC, /* 0xEC-0xEF */ + 0x92, 0x8C, 0xB9, 0xB0, 0x92, 0x8D, 0xC8, 0xAD, /* 0xF0-0xF3 */ + 0xCB, 0xA9, 0x92, 0x8E, 0xDE, 0xD9, 0xBF, 0xBD, /* 0xF4-0xF7 */ + 0x92, 0x8F, 0x92, 0x90, 0x92, 0x91, 0x92, 0x92, /* 0xF8-0xFB */ + 0xC6, 0xB4, 0xD7, 0xA7, 0xCA, 0xB0, 0xC4, 0xC3, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_63[512] = { + 0x92, 0x93, 0xB3, 0xD6, 0xB9, 0xD2, 0x92, 0x94, /* 0x00-0x03 */ + 0x92, 0x95, 0x92, 0x96, 0x92, 0x97, 0xD6, 0xB8, /* 0x04-0x07 */ + 0xEA, 0xFC, 0xB0, 0xB4, 0x92, 0x98, 0x92, 0x99, /* 0x08-0x0B */ + 0x92, 0x9A, 0x92, 0x9B, 0xBF, 0xE6, 0x92, 0x9C, /* 0x0C-0x0F */ + 0x92, 0x9D, 0xCC, 0xF4, 0x92, 0x9E, 0x92, 0x9F, /* 0x10-0x13 */ + 0x92, 0xA0, 0x92, 0xA1, 0xCD, 0xDA, 0x92, 0xA2, /* 0x14-0x17 */ + 0x92, 0xA3, 0x92, 0xA4, 0xD6, 0xBF, 0xC2, 0xCE, /* 0x18-0x1B */ + 0x92, 0xA5, 0xCE, 0xCE, 0xCC, 0xA2, 0xD0, 0xAE, /* 0x1C-0x1F */ + 0xC4, 0xD3, 0xB5, 0xB2, 0xDE, 0xD8, 0xD5, 0xF5, /* 0x20-0x23 */ + 0xBC, 0xB7, 0xBB, 0xD3, 0x92, 0xA6, 0x92, 0xA7, /* 0x24-0x27 */ + 0xB0, 0xA4, 0x92, 0xA8, 0xC5, 0xB2, 0xB4, 0xEC, /* 0x28-0x2B */ + 0x92, 0xA9, 0x92, 0xAA, 0x92, 0xAB, 0xD5, 0xF1, /* 0x2C-0x2F */ + 0x92, 0xAC, 0x92, 0xAD, 0xEA, 0xFD, 0x92, 0xAE, /* 0x30-0x33 */ + 0x92, 0xAF, 0x92, 0xB0, 0x92, 0xB1, 0x92, 0xB2, /* 0x34-0x37 */ + 0x92, 0xB3, 0xDE, 0xDA, 0xCD, 0xA6, 0x92, 0xB4, /* 0x38-0x3B */ + 0x92, 0xB5, 0xCD, 0xEC, 0x92, 0xB6, 0x92, 0xB7, /* 0x3C-0x3F */ + 0x92, 0xB8, 0x92, 0xB9, 0xCE, 0xE6, 0xDE, 0xDC, /* 0x40-0x43 */ + 0x92, 0xBA, 0xCD, 0xB1, 0xC0, 0xA6, 0x92, 0xBB, /* 0x44-0x47 */ + 0x92, 0xBC, 0xD7, 0xBD, 0x92, 0xBD, 0xDE, 0xDB, /* 0x48-0x4B */ + 0xB0, 0xC6, 0xBA, 0xB4, 0xC9, 0xD3, 0xC4, 0xF3, /* 0x4C-0x4F */ + 0xBE, 0xE8, 0x92, 0xBE, 0x92, 0xBF, 0x92, 0xC0, /* 0x50-0x53 */ + 0x92, 0xC1, 0xB2, 0xB6, 0x92, 0xC2, 0x92, 0xC3, /* 0x54-0x57 */ + 0x92, 0xC4, 0x92, 0xC5, 0x92, 0xC6, 0x92, 0xC7, /* 0x58-0x5B */ + 0x92, 0xC8, 0x92, 0xC9, 0xC0, 0xCC, 0xCB, 0xF0, /* 0x5C-0x5F */ + 0x92, 0xCA, 0xBC, 0xF1, 0xBB, 0xBB, 0xB5, 0xB7, /* 0x60-0x63 */ + 0x92, 0xCB, 0x92, 0xCC, 0x92, 0xCD, 0xC5, 0xF5, /* 0x64-0x67 */ + 0x92, 0xCE, 0xDE, 0xE6, 0x92, 0xCF, 0x92, 0xD0, /* 0x68-0x6B */ + 0x92, 0xD1, 0xDE, 0xE3, 0xBE, 0xDD, 0x92, 0xD2, /* 0x6C-0x6F */ + 0x92, 0xD3, 0xDE, 0xDF, 0x92, 0xD4, 0x92, 0xD5, /* 0x70-0x73 */ + 0x92, 0xD6, 0x92, 0xD7, 0xB4, 0xB7, 0xBD, 0xDD, /* 0x74-0x77 */ + 0x92, 0xD8, 0x92, 0xD9, 0xDE, 0xE0, 0xC4, 0xED, /* 0x78-0x7B */ + 0x92, 0xDA, 0x92, 0xDB, 0x92, 0xDC, 0x92, 0xDD, /* 0x7C-0x7F */ + + 0xCF, 0xC6, 0x92, 0xDE, 0xB5, 0xE0, 0x92, 0xDF, /* 0x80-0x83 */ + 0x92, 0xE0, 0x92, 0xE1, 0x92, 0xE2, 0xB6, 0xDE, /* 0x84-0x87 */ + 0xCA, 0xDA, 0xB5, 0xF4, 0xDE, 0xE5, 0x92, 0xE3, /* 0x88-0x8B */ + 0xD5, 0xC6, 0x92, 0xE4, 0xDE, 0xE1, 0xCC, 0xCD, /* 0x8C-0x8F */ + 0xC6, 0xFE, 0x92, 0xE5, 0xC5, 0xC5, 0x92, 0xE6, /* 0x90-0x93 */ + 0x92, 0xE7, 0x92, 0xE8, 0xD2, 0xB4, 0x92, 0xE9, /* 0x94-0x97 */ + 0xBE, 0xF2, 0x92, 0xEA, 0x92, 0xEB, 0x92, 0xEC, /* 0x98-0x9B */ + 0x92, 0xED, 0x92, 0xEE, 0x92, 0xEF, 0x92, 0xF0, /* 0x9C-0x9F */ + 0xC2, 0xD3, 0x92, 0xF1, 0xCC, 0xBD, 0xB3, 0xB8, /* 0xA0-0xA3 */ + 0x92, 0xF2, 0xBD, 0xD3, 0x92, 0xF3, 0xBF, 0xD8, /* 0xA4-0xA7 */ + 0xCD, 0xC6, 0xD1, 0xDA, 0xB4, 0xEB, 0x92, 0xF4, /* 0xA8-0xAB */ + 0xDE, 0xE4, 0xDE, 0xDD, 0xDE, 0xE7, 0x92, 0xF5, /* 0xAC-0xAF */ + 0xEA, 0xFE, 0x92, 0xF6, 0x92, 0xF7, 0xC2, 0xB0, /* 0xB0-0xB3 */ + 0xDE, 0xE2, 0x92, 0xF8, 0x92, 0xF9, 0xD6, 0xC0, /* 0xB4-0xB7 */ + 0xB5, 0xA7, 0x92, 0xFA, 0xB2, 0xF4, 0x92, 0xFB, /* 0xB8-0xBB */ + 0xDE, 0xE8, 0x92, 0xFC, 0xDE, 0xF2, 0x92, 0xFD, /* 0xBC-0xBF */ + 0x92, 0xFE, 0x93, 0x40, 0x93, 0x41, 0x93, 0x42, /* 0xC0-0xC3 */ + 0xDE, 0xED, 0x93, 0x43, 0xDE, 0xF1, 0x93, 0x44, /* 0xC4-0xC7 */ + 0x93, 0x45, 0xC8, 0xE0, 0x93, 0x46, 0x93, 0x47, /* 0xC8-0xCB */ + 0x93, 0x48, 0xD7, 0xE1, 0xDE, 0xEF, 0xC3, 0xE8, /* 0xCC-0xCF */ + 0xCC, 0xE1, 0x93, 0x49, 0xB2, 0xE5, 0x93, 0x4A, /* 0xD0-0xD3 */ + 0x93, 0x4B, 0x93, 0x4C, 0xD2, 0xBE, 0x93, 0x4D, /* 0xD4-0xD7 */ + 0x93, 0x4E, 0x93, 0x4F, 0x93, 0x50, 0x93, 0x51, /* 0xD8-0xDB */ + 0x93, 0x52, 0x93, 0x53, 0xDE, 0xEE, 0x93, 0x54, /* 0xDC-0xDF */ + 0xDE, 0xEB, 0xCE, 0xD5, 0x93, 0x55, 0xB4, 0xA7, /* 0xE0-0xE3 */ + 0x93, 0x56, 0x93, 0x57, 0x93, 0x58, 0x93, 0x59, /* 0xE4-0xE7 */ + 0x93, 0x5A, 0xBF, 0xAB, 0xBE, 0xBE, 0x93, 0x5B, /* 0xE8-0xEB */ + 0x93, 0x5C, 0xBD, 0xD2, 0x93, 0x5D, 0x93, 0x5E, /* 0xEC-0xEF */ + 0x93, 0x5F, 0x93, 0x60, 0xDE, 0xE9, 0x93, 0x61, /* 0xF0-0xF3 */ + 0xD4, 0xAE, 0x93, 0x62, 0xDE, 0xDE, 0x93, 0x63, /* 0xF4-0xF7 */ + 0xDE, 0xEA, 0x93, 0x64, 0x93, 0x65, 0x93, 0x66, /* 0xF8-0xFB */ + 0x93, 0x67, 0xC0, 0xBF, 0x93, 0x68, 0xDE, 0xEC, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_64[512] = { + 0xB2, 0xF3, 0xB8, 0xE9, 0xC2, 0xA7, 0x93, 0x69, /* 0x00-0x03 */ + 0x93, 0x6A, 0xBD, 0xC1, 0x93, 0x6B, 0x93, 0x6C, /* 0x04-0x07 */ + 0x93, 0x6D, 0x93, 0x6E, 0x93, 0x6F, 0xDE, 0xF5, /* 0x08-0x0B */ + 0xDE, 0xF8, 0x93, 0x70, 0x93, 0x71, 0xB2, 0xAB, /* 0x0C-0x0F */ + 0xB4, 0xA4, 0x93, 0x72, 0x93, 0x73, 0xB4, 0xEA, /* 0x10-0x13 */ + 0xC9, 0xA6, 0x93, 0x74, 0x93, 0x75, 0x93, 0x76, /* 0x14-0x17 */ + 0x93, 0x77, 0x93, 0x78, 0x93, 0x79, 0xDE, 0xF6, /* 0x18-0x1B */ + 0xCB, 0xD1, 0x93, 0x7A, 0xB8, 0xE3, 0x93, 0x7B, /* 0x1C-0x1F */ + 0xDE, 0xF7, 0xDE, 0xFA, 0x93, 0x7C, 0x93, 0x7D, /* 0x20-0x23 */ + 0x93, 0x7E, 0x93, 0x80, 0xDE, 0xF9, 0x93, 0x81, /* 0x24-0x27 */ + 0x93, 0x82, 0x93, 0x83, 0xCC, 0xC2, 0x93, 0x84, /* 0x28-0x2B */ + 0xB0, 0xE1, 0xB4, 0xEE, 0x93, 0x85, 0x93, 0x86, /* 0x2C-0x2F */ + 0x93, 0x87, 0x93, 0x88, 0x93, 0x89, 0x93, 0x8A, /* 0x30-0x33 */ + 0xE5, 0xBA, 0x93, 0x8B, 0x93, 0x8C, 0x93, 0x8D, /* 0x34-0x37 */ + 0x93, 0x8E, 0x93, 0x8F, 0xD0, 0xAF, 0x93, 0x90, /* 0x38-0x3B */ + 0x93, 0x91, 0xB2, 0xEB, 0x93, 0x92, 0xEB, 0xA1, /* 0x3C-0x3F */ + 0x93, 0x93, 0xDE, 0xF4, 0x93, 0x94, 0x93, 0x95, /* 0x40-0x43 */ + 0xC9, 0xE3, 0xDE, 0xF3, 0xB0, 0xDA, 0xD2, 0xA1, /* 0x44-0x47 */ + 0xB1, 0xF7, 0x93, 0x96, 0xCC, 0xAF, 0x93, 0x97, /* 0x48-0x4B */ + 0x93, 0x98, 0x93, 0x99, 0x93, 0x9A, 0x93, 0x9B, /* 0x4C-0x4F */ + 0x93, 0x9C, 0x93, 0x9D, 0xDE, 0xF0, 0x93, 0x9E, /* 0x50-0x53 */ + 0xCB, 0xA4, 0x93, 0x9F, 0x93, 0xA0, 0x93, 0xA1, /* 0x54-0x57 */ + 0xD5, 0xAA, 0x93, 0xA2, 0x93, 0xA3, 0x93, 0xA4, /* 0x58-0x5B */ + 0x93, 0xA5, 0x93, 0xA6, 0xDE, 0xFB, 0x93, 0xA7, /* 0x5C-0x5F */ + 0x93, 0xA8, 0x93, 0xA9, 0x93, 0xAA, 0x93, 0xAB, /* 0x60-0x63 */ + 0x93, 0xAC, 0x93, 0xAD, 0x93, 0xAE, 0xB4, 0xDD, /* 0x64-0x67 */ + 0x93, 0xAF, 0xC4, 0xA6, 0x93, 0xB0, 0x93, 0xB1, /* 0x68-0x6B */ + 0x93, 0xB2, 0xDE, 0xFD, 0x93, 0xB3, 0x93, 0xB4, /* 0x6C-0x6F */ + 0x93, 0xB5, 0x93, 0xB6, 0x93, 0xB7, 0x93, 0xB8, /* 0x70-0x73 */ + 0x93, 0xB9, 0x93, 0xBA, 0x93, 0xBB, 0x93, 0xBC, /* 0x74-0x77 */ + 0xC3, 0xFE, 0xC4, 0xA1, 0xDF, 0xA1, 0x93, 0xBD, /* 0x78-0x7B */ + 0x93, 0xBE, 0x93, 0xBF, 0x93, 0xC0, 0x93, 0xC1, /* 0x7C-0x7F */ + + 0x93, 0xC2, 0x93, 0xC3, 0xC1, 0xCC, 0x93, 0xC4, /* 0x80-0x83 */ + 0xDE, 0xFC, 0xBE, 0xEF, 0x93, 0xC5, 0xC6, 0xB2, /* 0x84-0x87 */ + 0x93, 0xC6, 0x93, 0xC7, 0x93, 0xC8, 0x93, 0xC9, /* 0x88-0x8B */ + 0x93, 0xCA, 0x93, 0xCB, 0x93, 0xCC, 0x93, 0xCD, /* 0x8C-0x8F */ + 0x93, 0xCE, 0xB3, 0xC5, 0xC8, 0xF6, 0x93, 0xCF, /* 0x90-0x93 */ + 0x93, 0xD0, 0xCB, 0xBA, 0xDE, 0xFE, 0x93, 0xD1, /* 0x94-0x97 */ + 0x93, 0xD2, 0xDF, 0xA4, 0x93, 0xD3, 0x93, 0xD4, /* 0x98-0x9B */ + 0x93, 0xD5, 0x93, 0xD6, 0xD7, 0xB2, 0x93, 0xD7, /* 0x9C-0x9F */ + 0x93, 0xD8, 0x93, 0xD9, 0x93, 0xDA, 0x93, 0xDB, /* 0xA0-0xA3 */ + 0xB3, 0xB7, 0x93, 0xDC, 0x93, 0xDD, 0x93, 0xDE, /* 0xA4-0xA7 */ + 0x93, 0xDF, 0xC1, 0xC3, 0x93, 0xE0, 0x93, 0xE1, /* 0xA8-0xAB */ + 0xC7, 0xCB, 0xB2, 0xA5, 0xB4, 0xE9, 0x93, 0xE2, /* 0xAC-0xAF */ + 0xD7, 0xAB, 0x93, 0xE3, 0x93, 0xE4, 0x93, 0xE5, /* 0xB0-0xB3 */ + 0x93, 0xE6, 0xC4, 0xEC, 0x93, 0xE7, 0xDF, 0xA2, /* 0xB4-0xB7 */ + 0xDF, 0xA3, 0x93, 0xE8, 0xDF, 0xA5, 0x93, 0xE9, /* 0xB8-0xBB */ + 0xBA, 0xB3, 0x93, 0xEA, 0x93, 0xEB, 0x93, 0xEC, /* 0xBC-0xBF */ + 0xDF, 0xA6, 0x93, 0xED, 0xC0, 0xDE, 0x93, 0xEE, /* 0xC0-0xC3 */ + 0x93, 0xEF, 0xC9, 0xC3, 0x93, 0xF0, 0x93, 0xF1, /* 0xC4-0xC7 */ + 0x93, 0xF2, 0x93, 0xF3, 0x93, 0xF4, 0x93, 0xF5, /* 0xC8-0xCB */ + 0x93, 0xF6, 0xB2, 0xD9, 0xC7, 0xE6, 0x93, 0xF7, /* 0xCC-0xCF */ + 0xDF, 0xA7, 0x93, 0xF8, 0xC7, 0xDC, 0x93, 0xF9, /* 0xD0-0xD3 */ + 0x93, 0xFA, 0x93, 0xFB, 0x93, 0xFC, 0xDF, 0xA8, /* 0xD4-0xD7 */ + 0xEB, 0xA2, 0x93, 0xFD, 0x93, 0xFE, 0x94, 0x40, /* 0xD8-0xDB */ + 0x94, 0x41, 0x94, 0x42, 0xCB, 0xD3, 0x94, 0x43, /* 0xDC-0xDF */ + 0x94, 0x44, 0x94, 0x45, 0xDF, 0xAA, 0x94, 0x46, /* 0xE0-0xE3 */ + 0xDF, 0xA9, 0x94, 0x47, 0xB2, 0xC1, 0x94, 0x48, /* 0xE4-0xE7 */ + 0x94, 0x49, 0x94, 0x4A, 0x94, 0x4B, 0x94, 0x4C, /* 0xE8-0xEB */ + 0x94, 0x4D, 0x94, 0x4E, 0x94, 0x4F, 0x94, 0x50, /* 0xEC-0xEF */ + 0x94, 0x51, 0x94, 0x52, 0x94, 0x53, 0x94, 0x54, /* 0xF0-0xF3 */ + 0x94, 0x55, 0x94, 0x56, 0x94, 0x57, 0x94, 0x58, /* 0xF4-0xF7 */ + 0x94, 0x59, 0x94, 0x5A, 0x94, 0x5B, 0x94, 0x5C, /* 0xF8-0xFB */ + 0x94, 0x5D, 0x94, 0x5E, 0x94, 0x5F, 0x94, 0x60, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_65[512] = { + 0xC5, 0xCA, 0x94, 0x61, 0x94, 0x62, 0x94, 0x63, /* 0x00-0x03 */ + 0x94, 0x64, 0x94, 0x65, 0x94, 0x66, 0x94, 0x67, /* 0x04-0x07 */ + 0x94, 0x68, 0xDF, 0xAB, 0x94, 0x69, 0x94, 0x6A, /* 0x08-0x0B */ + 0x94, 0x6B, 0x94, 0x6C, 0x94, 0x6D, 0x94, 0x6E, /* 0x0C-0x0F */ + 0x94, 0x6F, 0x94, 0x70, 0xD4, 0xDC, 0x94, 0x71, /* 0x10-0x13 */ + 0x94, 0x72, 0x94, 0x73, 0x94, 0x74, 0x94, 0x75, /* 0x14-0x17 */ + 0xC8, 0xC1, 0x94, 0x76, 0x94, 0x77, 0x94, 0x78, /* 0x18-0x1B */ + 0x94, 0x79, 0x94, 0x7A, 0x94, 0x7B, 0x94, 0x7C, /* 0x1C-0x1F */ + 0x94, 0x7D, 0x94, 0x7E, 0x94, 0x80, 0x94, 0x81, /* 0x20-0x23 */ + 0x94, 0x82, 0xDF, 0xAC, 0x94, 0x83, 0x94, 0x84, /* 0x24-0x27 */ + 0x94, 0x85, 0x94, 0x86, 0x94, 0x87, 0xBE, 0xF0, /* 0x28-0x2B */ + 0x94, 0x88, 0x94, 0x89, 0xDF, 0xAD, 0xD6, 0xA7, /* 0x2C-0x2F */ + 0x94, 0x8A, 0x94, 0x8B, 0x94, 0x8C, 0x94, 0x8D, /* 0x30-0x33 */ + 0xEA, 0xB7, 0xEB, 0xB6, 0xCA, 0xD5, 0x94, 0x8E, /* 0x34-0x37 */ + 0xD8, 0xFC, 0xB8, 0xC4, 0x94, 0x8F, 0xB9, 0xA5, /* 0x38-0x3B */ + 0x94, 0x90, 0x94, 0x91, 0xB7, 0xC5, 0xD5, 0xFE, /* 0x3C-0x3F */ + 0x94, 0x92, 0x94, 0x93, 0x94, 0x94, 0x94, 0x95, /* 0x40-0x43 */ + 0x94, 0x96, 0xB9, 0xCA, 0x94, 0x97, 0x94, 0x98, /* 0x44-0x47 */ + 0xD0, 0xA7, 0xF4, 0xCD, 0x94, 0x99, 0x94, 0x9A, /* 0x48-0x4B */ + 0xB5, 0xD0, 0x94, 0x9B, 0x94, 0x9C, 0xC3, 0xF4, /* 0x4C-0x4F */ + 0x94, 0x9D, 0xBE, 0xC8, 0x94, 0x9E, 0x94, 0x9F, /* 0x50-0x53 */ + 0x94, 0xA0, 0xEB, 0xB7, 0xB0, 0xBD, 0x94, 0xA1, /* 0x54-0x57 */ + 0x94, 0xA2, 0xBD, 0xCC, 0x94, 0xA3, 0xC1, 0xB2, /* 0x58-0x5B */ + 0x94, 0xA4, 0xB1, 0xD6, 0xB3, 0xA8, 0x94, 0xA5, /* 0x5C-0x5F */ + 0x94, 0xA6, 0x94, 0xA7, 0xB8, 0xD2, 0xC9, 0xA2, /* 0x60-0x63 */ + 0x94, 0xA8, 0x94, 0xA9, 0xB6, 0xD8, 0x94, 0xAA, /* 0x64-0x67 */ + 0x94, 0xAB, 0x94, 0xAC, 0x94, 0xAD, 0xEB, 0xB8, /* 0x68-0x6B */ + 0xBE, 0xB4, 0x94, 0xAE, 0x94, 0xAF, 0x94, 0xB0, /* 0x6C-0x6F */ + 0xCA, 0xFD, 0x94, 0xB1, 0xC7, 0xC3, 0x94, 0xB2, /* 0x70-0x73 */ + 0xD5, 0xFB, 0x94, 0xB3, 0x94, 0xB4, 0xB7, 0xF3, /* 0x74-0x77 */ + 0x94, 0xB5, 0x94, 0xB6, 0x94, 0xB7, 0x94, 0xB8, /* 0x78-0x7B */ + 0x94, 0xB9, 0x94, 0xBA, 0x94, 0xBB, 0x94, 0xBC, /* 0x7C-0x7F */ + + 0x94, 0xBD, 0x94, 0xBE, 0x94, 0xBF, 0x94, 0xC0, /* 0x80-0x83 */ + 0x94, 0xC1, 0x94, 0xC2, 0x94, 0xC3, 0xCE, 0xC4, /* 0x84-0x87 */ + 0x94, 0xC4, 0x94, 0xC5, 0x94, 0xC6, 0xD5, 0xAB, /* 0x88-0x8B */ + 0xB1, 0xF3, 0x94, 0xC7, 0x94, 0xC8, 0x94, 0xC9, /* 0x8C-0x8F */ + 0xEC, 0xB3, 0xB0, 0xDF, 0x94, 0xCA, 0xEC, 0xB5, /* 0x90-0x93 */ + 0x94, 0xCB, 0x94, 0xCC, 0x94, 0xCD, 0xB6, 0xB7, /* 0x94-0x97 */ + 0x94, 0xCE, 0xC1, 0xCF, 0x94, 0xCF, 0xF5, 0xFA, /* 0x98-0x9B */ + 0xD0, 0xB1, 0x94, 0xD0, 0x94, 0xD1, 0xD5, 0xE5, /* 0x9C-0x9F */ + 0x94, 0xD2, 0xCE, 0xD3, 0x94, 0xD3, 0x94, 0xD4, /* 0xA0-0xA3 */ + 0xBD, 0xEF, 0xB3, 0xE2, 0x94, 0xD5, 0xB8, 0xAB, /* 0xA4-0xA7 */ + 0x94, 0xD6, 0xD5, 0xB6, 0x94, 0xD7, 0xED, 0xBD, /* 0xA8-0xAB */ + 0x94, 0xD8, 0xB6, 0xCF, 0x94, 0xD9, 0xCB, 0xB9, /* 0xAC-0xAF */ + 0xD0, 0xC2, 0x94, 0xDA, 0x94, 0xDB, 0x94, 0xDC, /* 0xB0-0xB3 */ + 0x94, 0xDD, 0x94, 0xDE, 0x94, 0xDF, 0x94, 0xE0, /* 0xB4-0xB7 */ + 0x94, 0xE1, 0xB7, 0xBD, 0x94, 0xE2, 0x94, 0xE3, /* 0xB8-0xBB */ + 0xEC, 0xB6, 0xCA, 0xA9, 0x94, 0xE4, 0x94, 0xE5, /* 0xBC-0xBF */ + 0x94, 0xE6, 0xC5, 0xD4, 0x94, 0xE7, 0xEC, 0xB9, /* 0xC0-0xC3 */ + 0xEC, 0xB8, 0xC2, 0xC3, 0xEC, 0xB7, 0x94, 0xE8, /* 0xC4-0xC7 */ + 0x94, 0xE9, 0x94, 0xEA, 0x94, 0xEB, 0xD0, 0xFD, /* 0xC8-0xCB */ + 0xEC, 0xBA, 0x94, 0xEC, 0xEC, 0xBB, 0xD7, 0xE5, /* 0xCC-0xCF */ + 0x94, 0xED, 0x94, 0xEE, 0xEC, 0xBC, 0x94, 0xEF, /* 0xD0-0xD3 */ + 0x94, 0xF0, 0x94, 0xF1, 0xEC, 0xBD, 0xC6, 0xEC, /* 0xD4-0xD7 */ + 0x94, 0xF2, 0x94, 0xF3, 0x94, 0xF4, 0x94, 0xF5, /* 0xD8-0xDB */ + 0x94, 0xF6, 0x94, 0xF7, 0x94, 0xF8, 0x94, 0xF9, /* 0xDC-0xDF */ + 0xCE, 0xDE, 0x94, 0xFA, 0xBC, 0xC8, 0x94, 0xFB, /* 0xE0-0xE3 */ + 0x94, 0xFC, 0xC8, 0xD5, 0xB5, 0xA9, 0xBE, 0xC9, /* 0xE4-0xE7 */ + 0xD6, 0xBC, 0xD4, 0xE7, 0x94, 0xFD, 0x94, 0xFE, /* 0xE8-0xEB */ + 0xD1, 0xAE, 0xD0, 0xF1, 0xEA, 0xB8, 0xEA, 0xB9, /* 0xEC-0xEF */ + 0xEA, 0xBA, 0xBA, 0xB5, 0x95, 0x40, 0x95, 0x41, /* 0xF0-0xF3 */ + 0x95, 0x42, 0x95, 0x43, 0xCA, 0xB1, 0xBF, 0xF5, /* 0xF4-0xF7 */ + 0x95, 0x44, 0x95, 0x45, 0xCD, 0xFA, 0x95, 0x46, /* 0xF8-0xFB */ + 0x95, 0x47, 0x95, 0x48, 0x95, 0x49, 0x95, 0x4A, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_66[512] = { + 0xEA, 0xC0, 0x95, 0x4B, 0xB0, 0xBA, 0xEA, 0xBE, /* 0x00-0x03 */ + 0x95, 0x4C, 0x95, 0x4D, 0xC0, 0xA5, 0x95, 0x4E, /* 0x04-0x07 */ + 0x95, 0x4F, 0x95, 0x50, 0xEA, 0xBB, 0x95, 0x51, /* 0x08-0x0B */ + 0xB2, 0xFD, 0x95, 0x52, 0xC3, 0xF7, 0xBB, 0xE8, /* 0x0C-0x0F */ + 0x95, 0x53, 0x95, 0x54, 0x95, 0x55, 0xD2, 0xD7, /* 0x10-0x13 */ + 0xCE, 0xF4, 0xEA, 0xBF, 0x95, 0x56, 0x95, 0x57, /* 0x14-0x17 */ + 0x95, 0x58, 0xEA, 0xBC, 0x95, 0x59, 0x95, 0x5A, /* 0x18-0x1B */ + 0x95, 0x5B, 0xEA, 0xC3, 0x95, 0x5C, 0xD0, 0xC7, /* 0x1C-0x1F */ + 0xD3, 0xB3, 0x95, 0x5D, 0x95, 0x5E, 0x95, 0x5F, /* 0x20-0x23 */ + 0x95, 0x60, 0xB4, 0xBA, 0x95, 0x61, 0xC3, 0xC1, /* 0x24-0x27 */ + 0xD7, 0xF2, 0x95, 0x62, 0x95, 0x63, 0x95, 0x64, /* 0x28-0x2B */ + 0x95, 0x65, 0xD5, 0xD1, 0x95, 0x66, 0xCA, 0xC7, /* 0x2C-0x2F */ + 0x95, 0x67, 0xEA, 0xC5, 0x95, 0x68, 0x95, 0x69, /* 0x30-0x33 */ + 0xEA, 0xC4, 0xEA, 0xC7, 0xEA, 0xC6, 0x95, 0x6A, /* 0x34-0x37 */ + 0x95, 0x6B, 0x95, 0x6C, 0x95, 0x6D, 0x95, 0x6E, /* 0x38-0x3B */ + 0xD6, 0xE7, 0x95, 0x6F, 0xCF, 0xD4, 0x95, 0x70, /* 0x3C-0x3F */ + 0x95, 0x71, 0xEA, 0xCB, 0x95, 0x72, 0xBB, 0xCE, /* 0x40-0x43 */ + 0x95, 0x73, 0x95, 0x74, 0x95, 0x75, 0x95, 0x76, /* 0x44-0x47 */ + 0x95, 0x77, 0x95, 0x78, 0x95, 0x79, 0xBD, 0xFA, /* 0x48-0x4B */ + 0xC9, 0xCE, 0x95, 0x7A, 0x95, 0x7B, 0xEA, 0xCC, /* 0x4C-0x4F */ + 0x95, 0x7C, 0x95, 0x7D, 0xC9, 0xB9, 0xCF, 0xFE, /* 0x50-0x53 */ + 0xEA, 0xCA, 0xD4, 0xCE, 0xEA, 0xCD, 0xEA, 0xCF, /* 0x54-0x57 */ + 0x95, 0x7E, 0x95, 0x80, 0xCD, 0xED, 0x95, 0x81, /* 0x58-0x5B */ + 0x95, 0x82, 0x95, 0x83, 0x95, 0x84, 0xEA, 0xC9, /* 0x5C-0x5F */ + 0x95, 0x85, 0xEA, 0xCE, 0x95, 0x86, 0x95, 0x87, /* 0x60-0x63 */ + 0xCE, 0xEE, 0x95, 0x88, 0xBB, 0xDE, 0x95, 0x89, /* 0x64-0x67 */ + 0xB3, 0xBF, 0x95, 0x8A, 0x95, 0x8B, 0x95, 0x8C, /* 0x68-0x6B */ + 0x95, 0x8D, 0x95, 0x8E, 0xC6, 0xD5, 0xBE, 0xB0, /* 0x6C-0x6F */ + 0xCE, 0xFA, 0x95, 0x8F, 0x95, 0x90, 0x95, 0x91, /* 0x70-0x73 */ + 0xC7, 0xE7, 0x95, 0x92, 0xBE, 0xA7, 0xEA, 0xD0, /* 0x74-0x77 */ + 0x95, 0x93, 0x95, 0x94, 0xD6, 0xC7, 0x95, 0x95, /* 0x78-0x7B */ + 0x95, 0x96, 0x95, 0x97, 0xC1, 0xC0, 0x95, 0x98, /* 0x7C-0x7F */ + + 0x95, 0x99, 0x95, 0x9A, 0xD4, 0xDD, 0x95, 0x9B, /* 0x80-0x83 */ + 0xEA, 0xD1, 0x95, 0x9C, 0x95, 0x9D, 0xCF, 0xBE, /* 0x84-0x87 */ + 0x95, 0x9E, 0x95, 0x9F, 0x95, 0xA0, 0x95, 0xA1, /* 0x88-0x8B */ + 0xEA, 0xD2, 0x95, 0xA2, 0x95, 0xA3, 0x95, 0xA4, /* 0x8C-0x8F */ + 0x95, 0xA5, 0xCA, 0xEE, 0x95, 0xA6, 0x95, 0xA7, /* 0x90-0x93 */ + 0x95, 0xA8, 0x95, 0xA9, 0xC5, 0xAF, 0xB0, 0xB5, /* 0x94-0x97 */ + 0x95, 0xAA, 0x95, 0xAB, 0x95, 0xAC, 0x95, 0xAD, /* 0x98-0x9B */ + 0x95, 0xAE, 0xEA, 0xD4, 0x95, 0xAF, 0x95, 0xB0, /* 0x9C-0x9F */ + 0x95, 0xB1, 0x95, 0xB2, 0x95, 0xB3, 0x95, 0xB4, /* 0xA0-0xA3 */ + 0x95, 0xB5, 0x95, 0xB6, 0x95, 0xB7, 0xEA, 0xD3, /* 0xA4-0xA7 */ + 0xF4, 0xDF, 0x95, 0xB8, 0x95, 0xB9, 0x95, 0xBA, /* 0xA8-0xAB */ + 0x95, 0xBB, 0x95, 0xBC, 0xC4, 0xBA, 0x95, 0xBD, /* 0xAC-0xAF */ + 0x95, 0xBE, 0x95, 0xBF, 0x95, 0xC0, 0x95, 0xC1, /* 0xB0-0xB3 */ + 0xB1, 0xA9, 0x95, 0xC2, 0x95, 0xC3, 0x95, 0xC4, /* 0xB4-0xB7 */ + 0x95, 0xC5, 0xE5, 0xDF, 0x95, 0xC6, 0x95, 0xC7, /* 0xB8-0xBB */ + 0x95, 0xC8, 0x95, 0xC9, 0xEA, 0xD5, 0x95, 0xCA, /* 0xBC-0xBF */ + 0x95, 0xCB, 0x95, 0xCC, 0x95, 0xCD, 0x95, 0xCE, /* 0xC0-0xC3 */ + 0x95, 0xCF, 0x95, 0xD0, 0x95, 0xD1, 0x95, 0xD2, /* 0xC4-0xC7 */ + 0x95, 0xD3, 0x95, 0xD4, 0x95, 0xD5, 0x95, 0xD6, /* 0xC8-0xCB */ + 0x95, 0xD7, 0x95, 0xD8, 0x95, 0xD9, 0x95, 0xDA, /* 0xCC-0xCF */ + 0x95, 0xDB, 0x95, 0xDC, 0x95, 0xDD, 0x95, 0xDE, /* 0xD0-0xD3 */ + 0x95, 0xDF, 0x95, 0xE0, 0x95, 0xE1, 0x95, 0xE2, /* 0xD4-0xD7 */ + 0x95, 0xE3, 0xCA, 0xEF, 0x95, 0xE4, 0xEA, 0xD6, /* 0xD8-0xDB */ + 0xEA, 0xD7, 0xC6, 0xD8, 0x95, 0xE5, 0x95, 0xE6, /* 0xDC-0xDF */ + 0x95, 0xE7, 0x95, 0xE8, 0x95, 0xE9, 0x95, 0xEA, /* 0xE0-0xE3 */ + 0x95, 0xEB, 0x95, 0xEC, 0xEA, 0xD8, 0x95, 0xED, /* 0xE4-0xE7 */ + 0x95, 0xEE, 0xEA, 0xD9, 0x95, 0xEF, 0x95, 0xF0, /* 0xE8-0xEB */ + 0x95, 0xF1, 0x95, 0xF2, 0x95, 0xF3, 0x95, 0xF4, /* 0xEC-0xEF */ + 0xD4, 0xBB, 0x95, 0xF5, 0xC7, 0xFA, 0xD2, 0xB7, /* 0xF0-0xF3 */ + 0xB8, 0xFC, 0x95, 0xF6, 0x95, 0xF7, 0xEA, 0xC2, /* 0xF4-0xF7 */ + 0x95, 0xF8, 0xB2, 0xDC, 0x95, 0xF9, 0x95, 0xFA, /* 0xF8-0xFB */ + 0xC2, 0xFC, 0x95, 0xFB, 0xD4, 0xF8, 0xCC, 0xE6, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_67[512] = { + 0xD7, 0xEE, 0x95, 0xFC, 0x95, 0xFD, 0x95, 0xFE, /* 0x00-0x03 */ + 0x96, 0x40, 0x96, 0x41, 0x96, 0x42, 0x96, 0x43, /* 0x04-0x07 */ + 0xD4, 0xC2, 0xD3, 0xD0, 0xEB, 0xC3, 0xC5, 0xF3, /* 0x08-0x0B */ + 0x96, 0x44, 0xB7, 0xFE, 0x96, 0x45, 0x96, 0x46, /* 0x0C-0x0F */ + 0xEB, 0xD4, 0x96, 0x47, 0x96, 0x48, 0x96, 0x49, /* 0x10-0x13 */ + 0xCB, 0xB7, 0xEB, 0xDE, 0x96, 0x4A, 0xC0, 0xCA, /* 0x14-0x17 */ + 0x96, 0x4B, 0x96, 0x4C, 0x96, 0x4D, 0xCD, 0xFB, /* 0x18-0x1B */ + 0x96, 0x4E, 0xB3, 0xAF, 0x96, 0x4F, 0xC6, 0xDA, /* 0x1C-0x1F */ + 0x96, 0x50, 0x96, 0x51, 0x96, 0x52, 0x96, 0x53, /* 0x20-0x23 */ + 0x96, 0x54, 0x96, 0x55, 0xEB, 0xFC, 0x96, 0x56, /* 0x24-0x27 */ + 0xC4, 0xBE, 0x96, 0x57, 0xCE, 0xB4, 0xC4, 0xA9, /* 0x28-0x2B */ + 0xB1, 0xBE, 0xD4, 0xFD, 0x96, 0x58, 0xCA, 0xF5, /* 0x2C-0x2F */ + 0x96, 0x59, 0xD6, 0xEC, 0x96, 0x5A, 0x96, 0x5B, /* 0x30-0x33 */ + 0xC6, 0xD3, 0xB6, 0xE4, 0x96, 0x5C, 0x96, 0x5D, /* 0x34-0x37 */ + 0x96, 0x5E, 0x96, 0x5F, 0xBB, 0xFA, 0x96, 0x60, /* 0x38-0x3B */ + 0x96, 0x61, 0xD0, 0xE0, 0x96, 0x62, 0x96, 0x63, /* 0x3C-0x3F */ + 0xC9, 0xB1, 0x96, 0x64, 0xD4, 0xD3, 0xC8, 0xA8, /* 0x40-0x43 */ + 0x96, 0x65, 0x96, 0x66, 0xB8, 0xCB, 0x96, 0x67, /* 0x44-0x47 */ + 0xE8, 0xBE, 0xC9, 0xBC, 0x96, 0x68, 0x96, 0x69, /* 0x48-0x4B */ + 0xE8, 0xBB, 0x96, 0x6A, 0xC0, 0xEE, 0xD0, 0xD3, /* 0x4C-0x4F */ + 0xB2, 0xC4, 0xB4, 0xE5, 0x96, 0x6B, 0xE8, 0xBC, /* 0x50-0x53 */ + 0x96, 0x6C, 0x96, 0x6D, 0xD5, 0xC8, 0x96, 0x6E, /* 0x54-0x57 */ + 0x96, 0x6F, 0x96, 0x70, 0x96, 0x71, 0x96, 0x72, /* 0x58-0x5B */ + 0xB6, 0xC5, 0x96, 0x73, 0xE8, 0xBD, 0xCA, 0xF8, /* 0x5C-0x5F */ + 0xB8, 0xDC, 0xCC, 0xF5, 0x96, 0x74, 0x96, 0x75, /* 0x60-0x63 */ + 0x96, 0x76, 0xC0, 0xB4, 0x96, 0x77, 0x96, 0x78, /* 0x64-0x67 */ + 0xD1, 0xEE, 0xE8, 0xBF, 0xE8, 0xC2, 0x96, 0x79, /* 0x68-0x6B */ + 0x96, 0x7A, 0xBA, 0xBC, 0x96, 0x7B, 0xB1, 0xAD, /* 0x6C-0x6F */ + 0xBD, 0xDC, 0x96, 0x7C, 0xEA, 0xBD, 0xE8, 0xC3, /* 0x70-0x73 */ + 0x96, 0x7D, 0xE8, 0xC6, 0x96, 0x7E, 0xE8, 0xCB, /* 0x74-0x77 */ + 0x96, 0x80, 0x96, 0x81, 0x96, 0x82, 0x96, 0x83, /* 0x78-0x7B */ + 0xE8, 0xCC, 0x96, 0x84, 0xCB, 0xC9, 0xB0, 0xE5, /* 0x7C-0x7F */ + + 0x96, 0x85, 0xBC, 0xAB, 0x96, 0x86, 0x96, 0x87, /* 0x80-0x83 */ + 0xB9, 0xB9, 0x96, 0x88, 0x96, 0x89, 0xE8, 0xC1, /* 0x84-0x87 */ + 0x96, 0x8A, 0xCD, 0xF7, 0x96, 0x8B, 0xE8, 0xCA, /* 0x88-0x8B */ + 0x96, 0x8C, 0x96, 0x8D, 0x96, 0x8E, 0x96, 0x8F, /* 0x8C-0x8F */ + 0xCE, 0xF6, 0x96, 0x90, 0x96, 0x91, 0x96, 0x92, /* 0x90-0x93 */ + 0x96, 0x93, 0xD5, 0xED, 0x96, 0x94, 0xC1, 0xD6, /* 0x94-0x97 */ + 0xE8, 0xC4, 0x96, 0x95, 0xC3, 0xB6, 0x96, 0x96, /* 0x98-0x9B */ + 0xB9, 0xFB, 0xD6, 0xA6, 0xE8, 0xC8, 0x96, 0x97, /* 0x9C-0x9F */ + 0x96, 0x98, 0x96, 0x99, 0xCA, 0xE0, 0xD4, 0xE6, /* 0xA0-0xA3 */ + 0x96, 0x9A, 0xE8, 0xC0, 0x96, 0x9B, 0xE8, 0xC5, /* 0xA4-0xA7 */ + 0xE8, 0xC7, 0x96, 0x9C, 0xC7, 0xB9, 0xB7, 0xE3, /* 0xA8-0xAB */ + 0x96, 0x9D, 0xE8, 0xC9, 0x96, 0x9E, 0xBF, 0xDD, /* 0xAC-0xAF */ + 0xE8, 0xD2, 0x96, 0x9F, 0x96, 0xA0, 0xE8, 0xD7, /* 0xB0-0xB3 */ + 0x96, 0xA1, 0xE8, 0xD5, 0xBC, 0xDC, 0xBC, 0xCF, /* 0xB4-0xB7 */ + 0xE8, 0xDB, 0x96, 0xA2, 0x96, 0xA3, 0x96, 0xA4, /* 0xB8-0xBB */ + 0x96, 0xA5, 0x96, 0xA6, 0x96, 0xA7, 0x96, 0xA8, /* 0xBC-0xBF */ + 0x96, 0xA9, 0xE8, 0xDE, 0x96, 0xAA, 0xE8, 0xDA, /* 0xC0-0xC3 */ + 0xB1, 0xFA, 0x96, 0xAB, 0x96, 0xAC, 0x96, 0xAD, /* 0xC4-0xC7 */ + 0x96, 0xAE, 0x96, 0xAF, 0x96, 0xB0, 0x96, 0xB1, /* 0xC8-0xCB */ + 0x96, 0xB2, 0x96, 0xB3, 0x96, 0xB4, 0xB0, 0xD8, /* 0xCC-0xCF */ + 0xC4, 0xB3, 0xB8, 0xCC, 0xC6, 0xE2, 0xC8, 0xBE, /* 0xD0-0xD3 */ + 0xC8, 0xE1, 0x96, 0xB5, 0x96, 0xB6, 0x96, 0xB7, /* 0xD4-0xD7 */ + 0xE8, 0xCF, 0xE8, 0xD4, 0xE8, 0xD6, 0x96, 0xB8, /* 0xD8-0xDB */ + 0xB9, 0xF1, 0xE8, 0xD8, 0xD7, 0xF5, 0x96, 0xB9, /* 0xDC-0xDF */ + 0xC4, 0xFB, 0x96, 0xBA, 0xE8, 0xDC, 0x96, 0xBB, /* 0xE0-0xE3 */ + 0x96, 0xBC, 0xB2, 0xE9, 0x96, 0xBD, 0x96, 0xBE, /* 0xE4-0xE7 */ + 0x96, 0xBF, 0xE8, 0xD1, 0x96, 0xC0, 0x96, 0xC1, /* 0xE8-0xEB */ + 0xBC, 0xED, 0x96, 0xC2, 0x96, 0xC3, 0xBF, 0xC2, /* 0xEC-0xEF */ + 0xE8, 0xCD, 0xD6, 0xF9, 0x96, 0xC4, 0xC1, 0xF8, /* 0xF0-0xF3 */ + 0xB2, 0xF1, 0x96, 0xC5, 0x96, 0xC6, 0x96, 0xC7, /* 0xF4-0xF7 */ + 0x96, 0xC8, 0x96, 0xC9, 0x96, 0xCA, 0x96, 0xCB, /* 0xF8-0xFB */ + 0x96, 0xCC, 0xE8, 0xDF, 0x96, 0xCD, 0xCA, 0xC1, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_68[512] = { + 0xE8, 0xD9, 0x96, 0xCE, 0x96, 0xCF, 0x96, 0xD0, /* 0x00-0x03 */ + 0x96, 0xD1, 0xD5, 0xA4, 0x96, 0xD2, 0xB1, 0xEA, /* 0x04-0x07 */ + 0xD5, 0xBB, 0xE8, 0xCE, 0xE8, 0xD0, 0xB6, 0xB0, /* 0x08-0x0B */ + 0xE8, 0xD3, 0x96, 0xD3, 0xE8, 0xDD, 0xC0, 0xB8, /* 0x0C-0x0F */ + 0x96, 0xD4, 0xCA, 0xF7, 0x96, 0xD5, 0xCB, 0xA8, /* 0x10-0x13 */ + 0x96, 0xD6, 0x96, 0xD7, 0xC6, 0xDC, 0xC0, 0xF5, /* 0x14-0x17 */ + 0x96, 0xD8, 0x96, 0xD9, 0x96, 0xDA, 0x96, 0xDB, /* 0x18-0x1B */ + 0x96, 0xDC, 0xE8, 0xE9, 0x96, 0xDD, 0x96, 0xDE, /* 0x1C-0x1F */ + 0x96, 0xDF, 0xD0, 0xA3, 0x96, 0xE0, 0x96, 0xE1, /* 0x20-0x23 */ + 0x96, 0xE2, 0x96, 0xE3, 0x96, 0xE4, 0x96, 0xE5, /* 0x24-0x27 */ + 0x96, 0xE6, 0xE8, 0xF2, 0xD6, 0xEA, 0x96, 0xE7, /* 0x28-0x2B */ + 0x96, 0xE8, 0x96, 0xE9, 0x96, 0xEA, 0x96, 0xEB, /* 0x2C-0x2F */ + 0x96, 0xEC, 0x96, 0xED, 0xE8, 0xE0, 0xE8, 0xE1, /* 0x30-0x33 */ + 0x96, 0xEE, 0x96, 0xEF, 0x96, 0xF0, 0xD1, 0xF9, /* 0x34-0x37 */ + 0xBA, 0xCB, 0xB8, 0xF9, 0x96, 0xF1, 0x96, 0xF2, /* 0x38-0x3B */ + 0xB8, 0xF1, 0xD4, 0xD4, 0xE8, 0xEF, 0x96, 0xF3, /* 0x3C-0x3F */ + 0xE8, 0xEE, 0xE8, 0xEC, 0xB9, 0xF0, 0xCC, 0xD2, /* 0x40-0x43 */ + 0xE8, 0xE6, 0xCE, 0xA6, 0xBF, 0xF2, 0x96, 0xF4, /* 0x44-0x47 */ + 0xB0, 0xB8, 0xE8, 0xF1, 0xE8, 0xF0, 0x96, 0xF5, /* 0x48-0x4B */ + 0xD7, 0xC0, 0x96, 0xF6, 0xE8, 0xE4, 0x96, 0xF7, /* 0x4C-0x4F */ + 0xCD, 0xA9, 0xC9, 0xA3, 0x96, 0xF8, 0xBB, 0xB8, /* 0x50-0x53 */ + 0xBD, 0xDB, 0xE8, 0xEA, 0x96, 0xF9, 0x96, 0xFA, /* 0x54-0x57 */ + 0x96, 0xFB, 0x96, 0xFC, 0x96, 0xFD, 0x96, 0xFE, /* 0x58-0x5B */ + 0x97, 0x40, 0x97, 0x41, 0x97, 0x42, 0x97, 0x43, /* 0x5C-0x5F */ + 0xE8, 0xE2, 0xE8, 0xE3, 0xE8, 0xE5, 0xB5, 0xB5, /* 0x60-0x63 */ + 0xE8, 0xE7, 0xC7, 0xC5, 0xE8, 0xEB, 0xE8, 0xED, /* 0x64-0x67 */ + 0xBD, 0xB0, 0xD7, 0xAE, 0x97, 0x44, 0xE8, 0xF8, /* 0x68-0x6B */ + 0x97, 0x45, 0x97, 0x46, 0x97, 0x47, 0x97, 0x48, /* 0x6C-0x6F */ + 0x97, 0x49, 0x97, 0x4A, 0x97, 0x4B, 0x97, 0x4C, /* 0x70-0x73 */ + 0xE8, 0xF5, 0x97, 0x4D, 0xCD, 0xB0, 0xE8, 0xF6, /* 0x74-0x77 */ + 0x97, 0x4E, 0x97, 0x4F, 0x97, 0x50, 0x97, 0x51, /* 0x78-0x7B */ + 0x97, 0x52, 0x97, 0x53, 0x97, 0x54, 0x97, 0x55, /* 0x7C-0x7F */ + + 0x97, 0x56, 0xC1, 0xBA, 0x97, 0x57, 0xE8, 0xE8, /* 0x80-0x83 */ + 0x97, 0x58, 0xC3, 0xB7, 0xB0, 0xF0, 0x97, 0x59, /* 0x84-0x87 */ + 0x97, 0x5A, 0x97, 0x5B, 0x97, 0x5C, 0x97, 0x5D, /* 0x88-0x8B */ + 0x97, 0x5E, 0x97, 0x5F, 0x97, 0x60, 0xE8, 0xF4, /* 0x8C-0x8F */ + 0x97, 0x61, 0x97, 0x62, 0x97, 0x63, 0xE8, 0xF7, /* 0x90-0x93 */ + 0x97, 0x64, 0x97, 0x65, 0x97, 0x66, 0xB9, 0xA3, /* 0x94-0x97 */ + 0x97, 0x67, 0x97, 0x68, 0x97, 0x69, 0x97, 0x6A, /* 0x98-0x9B */ + 0x97, 0x6B, 0x97, 0x6C, 0x97, 0x6D, 0x97, 0x6E, /* 0x9C-0x9F */ + 0x97, 0x6F, 0x97, 0x70, 0xC9, 0xD2, 0x97, 0x71, /* 0xA0-0xA3 */ + 0x97, 0x72, 0x97, 0x73, 0xC3, 0xCE, 0xCE, 0xE0, /* 0xA4-0xA7 */ + 0xC0, 0xE6, 0x97, 0x74, 0x97, 0x75, 0x97, 0x76, /* 0xA8-0xAB */ + 0x97, 0x77, 0xCB, 0xF3, 0x97, 0x78, 0xCC, 0xDD, /* 0xAC-0xAF */ + 0xD0, 0xB5, 0x97, 0x79, 0x97, 0x7A, 0xCA, 0xE1, /* 0xB0-0xB3 */ + 0x97, 0x7B, 0xE8, 0xF3, 0x97, 0x7C, 0x97, 0x7D, /* 0xB4-0xB7 */ + 0x97, 0x7E, 0x97, 0x80, 0x97, 0x81, 0x97, 0x82, /* 0xB8-0xBB */ + 0x97, 0x83, 0x97, 0x84, 0x97, 0x85, 0x97, 0x86, /* 0xBC-0xBF */ + 0xBC, 0xEC, 0x97, 0x87, 0xE8, 0xF9, 0x97, 0x88, /* 0xC0-0xC3 */ + 0x97, 0x89, 0x97, 0x8A, 0x97, 0x8B, 0x97, 0x8C, /* 0xC4-0xC7 */ + 0x97, 0x8D, 0xC3, 0xDE, 0x97, 0x8E, 0xC6, 0xE5, /* 0xC8-0xCB */ + 0x97, 0x8F, 0xB9, 0xF7, 0x97, 0x90, 0x97, 0x91, /* 0xCC-0xCF */ + 0x97, 0x92, 0x97, 0x93, 0xB0, 0xF4, 0x97, 0x94, /* 0xD0-0xD3 */ + 0x97, 0x95, 0xD7, 0xD8, 0x97, 0x96, 0x97, 0x97, /* 0xD4-0xD7 */ + 0xBC, 0xAC, 0x97, 0x98, 0xC5, 0xEF, 0x97, 0x99, /* 0xD8-0xDB */ + 0x97, 0x9A, 0x97, 0x9B, 0x97, 0x9C, 0x97, 0x9D, /* 0xDC-0xDF */ + 0xCC, 0xC4, 0x97, 0x9E, 0x97, 0x9F, 0xE9, 0xA6, /* 0xE0-0xE3 */ + 0x97, 0xA0, 0x97, 0xA1, 0x97, 0xA2, 0x97, 0xA3, /* 0xE4-0xE7 */ + 0x97, 0xA4, 0x97, 0xA5, 0x97, 0xA6, 0x97, 0xA7, /* 0xE8-0xEB */ + 0x97, 0xA8, 0x97, 0xA9, 0xC9, 0xAD, 0x97, 0xAA, /* 0xEC-0xEF */ + 0xE9, 0xA2, 0xC0, 0xE2, 0x97, 0xAB, 0x97, 0xAC, /* 0xF0-0xF3 */ + 0x97, 0xAD, 0xBF, 0xC3, 0x97, 0xAE, 0x97, 0xAF, /* 0xF4-0xF7 */ + 0x97, 0xB0, 0xE8, 0xFE, 0xB9, 0xD7, 0x97, 0xB1, /* 0xF8-0xFB */ + 0xE8, 0xFB, 0x97, 0xB2, 0x97, 0xB3, 0x97, 0xB4, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_69[512] = { + 0x97, 0xB5, 0xE9, 0xA4, 0x97, 0xB6, 0x97, 0xB7, /* 0x00-0x03 */ + 0x97, 0xB8, 0xD2, 0xCE, 0x97, 0xB9, 0x97, 0xBA, /* 0x04-0x07 */ + 0x97, 0xBB, 0x97, 0xBC, 0x97, 0xBD, 0xE9, 0xA3, /* 0x08-0x0B */ + 0x97, 0xBE, 0xD6, 0xB2, 0xD7, 0xB5, 0x97, 0xBF, /* 0x0C-0x0F */ + 0xE9, 0xA7, 0x97, 0xC0, 0xBD, 0xB7, 0x97, 0xC1, /* 0x10-0x13 */ + 0x97, 0xC2, 0x97, 0xC3, 0x97, 0xC4, 0x97, 0xC5, /* 0x14-0x17 */ + 0x97, 0xC6, 0x97, 0xC7, 0x97, 0xC8, 0x97, 0xC9, /* 0x18-0x1B */ + 0x97, 0xCA, 0x97, 0xCB, 0x97, 0xCC, 0xE8, 0xFC, /* 0x1C-0x1F */ + 0xE8, 0xFD, 0x97, 0xCD, 0x97, 0xCE, 0x97, 0xCF, /* 0x20-0x23 */ + 0xE9, 0xA1, 0x97, 0xD0, 0x97, 0xD1, 0x97, 0xD2, /* 0x24-0x27 */ + 0x97, 0xD3, 0x97, 0xD4, 0x97, 0xD5, 0x97, 0xD6, /* 0x28-0x2B */ + 0x97, 0xD7, 0xCD, 0xD6, 0x97, 0xD8, 0x97, 0xD9, /* 0x2C-0x2F */ + 0xD2, 0xAC, 0x97, 0xDA, 0x97, 0xDB, 0x97, 0xDC, /* 0x30-0x33 */ + 0xE9, 0xB2, 0x97, 0xDD, 0x97, 0xDE, 0x97, 0xDF, /* 0x34-0x37 */ + 0x97, 0xE0, 0xE9, 0xA9, 0x97, 0xE1, 0x97, 0xE2, /* 0x38-0x3B */ + 0x97, 0xE3, 0xB4, 0xAA, 0x97, 0xE4, 0xB4, 0xBB, /* 0x3C-0x3F */ + 0x97, 0xE5, 0x97, 0xE6, 0xE9, 0xAB, 0x97, 0xE7, /* 0x40-0x43 */ + 0x97, 0xE8, 0x97, 0xE9, 0x97, 0xEA, 0x97, 0xEB, /* 0x44-0x47 */ + 0x97, 0xEC, 0x97, 0xED, 0x97, 0xEE, 0x97, 0xEF, /* 0x48-0x4B */ + 0x97, 0xF0, 0x97, 0xF1, 0x97, 0xF2, 0x97, 0xF3, /* 0x4C-0x4F */ + 0x97, 0xF4, 0x97, 0xF5, 0x97, 0xF6, 0x97, 0xF7, /* 0x50-0x53 */ + 0xD0, 0xA8, 0x97, 0xF8, 0x97, 0xF9, 0xE9, 0xA5, /* 0x54-0x57 */ + 0x97, 0xFA, 0x97, 0xFB, 0xB3, 0xFE, 0x97, 0xFC, /* 0x58-0x5B */ + 0x97, 0xFD, 0xE9, 0xAC, 0xC0, 0xE3, 0x97, 0xFE, /* 0x5C-0x5F */ + 0xE9, 0xAA, 0x98, 0x40, 0x98, 0x41, 0xE9, 0xB9, /* 0x60-0x63 */ + 0x98, 0x42, 0x98, 0x43, 0xE9, 0xB8, 0x98, 0x44, /* 0x64-0x67 */ + 0x98, 0x45, 0x98, 0x46, 0x98, 0x47, 0xE9, 0xAE, /* 0x68-0x6B */ + 0x98, 0x48, 0x98, 0x49, 0xE8, 0xFA, 0x98, 0x4A, /* 0x6C-0x6F */ + 0x98, 0x4B, 0xE9, 0xA8, 0x98, 0x4C, 0x98, 0x4D, /* 0x70-0x73 */ + 0x98, 0x4E, 0x98, 0x4F, 0x98, 0x50, 0xBF, 0xAC, /* 0x74-0x77 */ + 0xE9, 0xB1, 0xE9, 0xBA, 0x98, 0x51, 0x98, 0x52, /* 0x78-0x7B */ + 0xC2, 0xA5, 0x98, 0x53, 0x98, 0x54, 0x98, 0x55, /* 0x7C-0x7F */ + + 0xE9, 0xAF, 0x98, 0x56, 0xB8, 0xC5, 0x98, 0x57, /* 0x80-0x83 */ + 0xE9, 0xAD, 0x98, 0x58, 0xD3, 0xDC, 0xE9, 0xB4, /* 0x84-0x87 */ + 0xE9, 0xB5, 0xE9, 0xB7, 0x98, 0x59, 0x98, 0x5A, /* 0x88-0x8B */ + 0x98, 0x5B, 0xE9, 0xC7, 0x98, 0x5C, 0x98, 0x5D, /* 0x8C-0x8F */ + 0x98, 0x5E, 0x98, 0x5F, 0x98, 0x60, 0x98, 0x61, /* 0x90-0x93 */ + 0xC0, 0xC6, 0xE9, 0xC5, 0x98, 0x62, 0x98, 0x63, /* 0x94-0x97 */ + 0xE9, 0xB0, 0x98, 0x64, 0x98, 0x65, 0xE9, 0xBB, /* 0x98-0x9B */ + 0xB0, 0xF1, 0x98, 0x66, 0x98, 0x67, 0x98, 0x68, /* 0x9C-0x9F */ + 0x98, 0x69, 0x98, 0x6A, 0x98, 0x6B, 0x98, 0x6C, /* 0xA0-0xA3 */ + 0x98, 0x6D, 0x98, 0x6E, 0x98, 0x6F, 0xE9, 0xBC, /* 0xA4-0xA7 */ + 0xD5, 0xA5, 0x98, 0x70, 0x98, 0x71, 0xE9, 0xBE, /* 0xA8-0xAB */ + 0x98, 0x72, 0xE9, 0xBF, 0x98, 0x73, 0x98, 0x74, /* 0xAC-0xAF */ + 0x98, 0x75, 0xE9, 0xC1, 0x98, 0x76, 0x98, 0x77, /* 0xB0-0xB3 */ + 0xC1, 0xF1, 0x98, 0x78, 0x98, 0x79, 0xC8, 0xB6, /* 0xB4-0xB7 */ + 0x98, 0x7A, 0x98, 0x7B, 0x98, 0x7C, 0xE9, 0xBD, /* 0xB8-0xBB */ + 0x98, 0x7D, 0x98, 0x7E, 0x98, 0x80, 0x98, 0x81, /* 0xBC-0xBF */ + 0x98, 0x82, 0xE9, 0xC2, 0x98, 0x83, 0x98, 0x84, /* 0xC0-0xC3 */ + 0x98, 0x85, 0x98, 0x86, 0x98, 0x87, 0x98, 0x88, /* 0xC4-0xC7 */ + 0x98, 0x89, 0x98, 0x8A, 0xE9, 0xC3, 0x98, 0x8B, /* 0xC8-0xCB */ + 0xE9, 0xB3, 0x98, 0x8C, 0xE9, 0xB6, 0x98, 0x8D, /* 0xCC-0xCF */ + 0xBB, 0xB1, 0x98, 0x8E, 0x98, 0x8F, 0x98, 0x90, /* 0xD0-0xD3 */ + 0xE9, 0xC0, 0x98, 0x91, 0x98, 0x92, 0x98, 0x93, /* 0xD4-0xD7 */ + 0x98, 0x94, 0x98, 0x95, 0x98, 0x96, 0xBC, 0xF7, /* 0xD8-0xDB */ + 0x98, 0x97, 0x98, 0x98, 0x98, 0x99, 0xE9, 0xC4, /* 0xDC-0xDF */ + 0xE9, 0xC6, 0x98, 0x9A, 0x98, 0x9B, 0x98, 0x9C, /* 0xE0-0xE3 */ + 0x98, 0x9D, 0x98, 0x9E, 0x98, 0x9F, 0x98, 0xA0, /* 0xE4-0xE7 */ + 0x98, 0xA1, 0x98, 0xA2, 0x98, 0xA3, 0x98, 0xA4, /* 0xE8-0xEB */ + 0x98, 0xA5, 0xE9, 0xCA, 0x98, 0xA6, 0x98, 0xA7, /* 0xEC-0xEF */ + 0x98, 0xA8, 0x98, 0xA9, 0xE9, 0xCE, 0x98, 0xAA, /* 0xF0-0xF3 */ + 0x98, 0xAB, 0x98, 0xAC, 0x98, 0xAD, 0x98, 0xAE, /* 0xF4-0xF7 */ + 0x98, 0xAF, 0x98, 0xB0, 0x98, 0xB1, 0x98, 0xB2, /* 0xF8-0xFB */ + 0x98, 0xB3, 0xB2, 0xDB, 0x98, 0xB4, 0xE9, 0xC8, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_6A[512] = { + 0x98, 0xB5, 0x98, 0xB6, 0x98, 0xB7, 0x98, 0xB8, /* 0x00-0x03 */ + 0x98, 0xB9, 0x98, 0xBA, 0x98, 0xBB, 0x98, 0xBC, /* 0x04-0x07 */ + 0x98, 0xBD, 0x98, 0xBE, 0xB7, 0xAE, 0x98, 0xBF, /* 0x08-0x0B */ + 0x98, 0xC0, 0x98, 0xC1, 0x98, 0xC2, 0x98, 0xC3, /* 0x0C-0x0F */ + 0x98, 0xC4, 0x98, 0xC5, 0x98, 0xC6, 0x98, 0xC7, /* 0x10-0x13 */ + 0x98, 0xC8, 0x98, 0xC9, 0x98, 0xCA, 0xE9, 0xCB, /* 0x14-0x17 */ + 0xE9, 0xCC, 0x98, 0xCB, 0x98, 0xCC, 0x98, 0xCD, /* 0x18-0x1B */ + 0x98, 0xCE, 0x98, 0xCF, 0x98, 0xD0, 0xD5, 0xC1, /* 0x1C-0x1F */ + 0x98, 0xD1, 0xC4, 0xA3, 0x98, 0xD2, 0x98, 0xD3, /* 0x20-0x23 */ + 0x98, 0xD4, 0x98, 0xD5, 0x98, 0xD6, 0x98, 0xD7, /* 0x24-0x27 */ + 0xE9, 0xD8, 0x98, 0xD8, 0xBA, 0xE1, 0x98, 0xD9, /* 0x28-0x2B */ + 0x98, 0xDA, 0x98, 0xDB, 0x98, 0xDC, 0xE9, 0xC9, /* 0x2C-0x2F */ + 0x98, 0xDD, 0xD3, 0xA3, 0x98, 0xDE, 0x98, 0xDF, /* 0x30-0x33 */ + 0x98, 0xE0, 0xE9, 0xD4, 0x98, 0xE1, 0x98, 0xE2, /* 0x34-0x37 */ + 0x98, 0xE3, 0x98, 0xE4, 0x98, 0xE5, 0x98, 0xE6, /* 0x38-0x3B */ + 0x98, 0xE7, 0xE9, 0xD7, 0xE9, 0xD0, 0x98, 0xE8, /* 0x3C-0x3F */ + 0x98, 0xE9, 0x98, 0xEA, 0x98, 0xEB, 0x98, 0xEC, /* 0x40-0x43 */ + 0xE9, 0xCF, 0x98, 0xED, 0x98, 0xEE, 0xC7, 0xC1, /* 0x44-0x47 */ + 0x98, 0xEF, 0x98, 0xF0, 0x98, 0xF1, 0x98, 0xF2, /* 0x48-0x4B */ + 0x98, 0xF3, 0x98, 0xF4, 0x98, 0xF5, 0x98, 0xF6, /* 0x4C-0x4F */ + 0xE9, 0xD2, 0x98, 0xF7, 0x98, 0xF8, 0x98, 0xF9, /* 0x50-0x53 */ + 0x98, 0xFA, 0x98, 0xFB, 0x98, 0xFC, 0x98, 0xFD, /* 0x54-0x57 */ + 0xE9, 0xD9, 0xB3, 0xC8, 0x98, 0xFE, 0xE9, 0xD3, /* 0x58-0x5B */ + 0x99, 0x40, 0x99, 0x41, 0x99, 0x42, 0x99, 0x43, /* 0x5C-0x5F */ + 0x99, 0x44, 0xCF, 0xF0, 0x99, 0x45, 0x99, 0x46, /* 0x60-0x63 */ + 0x99, 0x47, 0xE9, 0xCD, 0x99, 0x48, 0x99, 0x49, /* 0x64-0x67 */ + 0x99, 0x4A, 0x99, 0x4B, 0x99, 0x4C, 0x99, 0x4D, /* 0x68-0x6B */ + 0x99, 0x4E, 0x99, 0x4F, 0x99, 0x50, 0x99, 0x51, /* 0x6C-0x6F */ + 0x99, 0x52, 0xB3, 0xF7, 0x99, 0x53, 0x99, 0x54, /* 0x70-0x73 */ + 0x99, 0x55, 0x99, 0x56, 0x99, 0x57, 0x99, 0x58, /* 0x74-0x77 */ + 0x99, 0x59, 0xE9, 0xD6, 0x99, 0x5A, 0x99, 0x5B, /* 0x78-0x7B */ + 0xE9, 0xDA, 0x99, 0x5C, 0x99, 0x5D, 0x99, 0x5E, /* 0x7C-0x7F */ + + 0xCC, 0xB4, 0x99, 0x5F, 0x99, 0x60, 0x99, 0x61, /* 0x80-0x83 */ + 0xCF, 0xAD, 0x99, 0x62, 0x99, 0x63, 0x99, 0x64, /* 0x84-0x87 */ + 0x99, 0x65, 0x99, 0x66, 0x99, 0x67, 0x99, 0x68, /* 0x88-0x8B */ + 0x99, 0x69, 0x99, 0x6A, 0xE9, 0xD5, 0x99, 0x6B, /* 0x8C-0x8F */ + 0xE9, 0xDC, 0xE9, 0xDB, 0x99, 0x6C, 0x99, 0x6D, /* 0x90-0x93 */ + 0x99, 0x6E, 0x99, 0x6F, 0x99, 0x70, 0xE9, 0xDE, /* 0x94-0x97 */ + 0x99, 0x71, 0x99, 0x72, 0x99, 0x73, 0x99, 0x74, /* 0x98-0x9B */ + 0x99, 0x75, 0x99, 0x76, 0x99, 0x77, 0x99, 0x78, /* 0x9C-0x9F */ + 0xE9, 0xD1, 0x99, 0x79, 0x99, 0x7A, 0x99, 0x7B, /* 0xA0-0xA3 */ + 0x99, 0x7C, 0x99, 0x7D, 0x99, 0x7E, 0x99, 0x80, /* 0xA4-0xA7 */ + 0x99, 0x81, 0xE9, 0xDD, 0x99, 0x82, 0xE9, 0xDF, /* 0xA8-0xAB */ + 0xC3, 0xCA, 0x99, 0x83, 0x99, 0x84, 0x99, 0x85, /* 0xAC-0xAF */ + 0x99, 0x86, 0x99, 0x87, 0x99, 0x88, 0x99, 0x89, /* 0xB0-0xB3 */ + 0x99, 0x8A, 0x99, 0x8B, 0x99, 0x8C, 0x99, 0x8D, /* 0xB4-0xB7 */ + 0x99, 0x8E, 0x99, 0x8F, 0x99, 0x90, 0x99, 0x91, /* 0xB8-0xBB */ + 0x99, 0x92, 0x99, 0x93, 0x99, 0x94, 0x99, 0x95, /* 0xBC-0xBF */ + 0x99, 0x96, 0x99, 0x97, 0x99, 0x98, 0x99, 0x99, /* 0xC0-0xC3 */ + 0x99, 0x9A, 0x99, 0x9B, 0x99, 0x9C, 0x99, 0x9D, /* 0xC4-0xC7 */ + 0x99, 0x9E, 0x99, 0x9F, 0x99, 0xA0, 0x99, 0xA1, /* 0xC8-0xCB */ + 0x99, 0xA2, 0x99, 0xA3, 0x99, 0xA4, 0x99, 0xA5, /* 0xCC-0xCF */ + 0x99, 0xA6, 0x99, 0xA7, 0x99, 0xA8, 0x99, 0xA9, /* 0xD0-0xD3 */ + 0x99, 0xAA, 0x99, 0xAB, 0x99, 0xAC, 0x99, 0xAD, /* 0xD4-0xD7 */ + 0x99, 0xAE, 0x99, 0xAF, 0x99, 0xB0, 0x99, 0xB1, /* 0xD8-0xDB */ + 0x99, 0xB2, 0x99, 0xB3, 0x99, 0xB4, 0x99, 0xB5, /* 0xDC-0xDF */ + 0x99, 0xB6, 0x99, 0xB7, 0x99, 0xB8, 0x99, 0xB9, /* 0xE0-0xE3 */ + 0x99, 0xBA, 0x99, 0xBB, 0x99, 0xBC, 0x99, 0xBD, /* 0xE4-0xE7 */ + 0x99, 0xBE, 0x99, 0xBF, 0x99, 0xC0, 0x99, 0xC1, /* 0xE8-0xEB */ + 0x99, 0xC2, 0x99, 0xC3, 0x99, 0xC4, 0x99, 0xC5, /* 0xEC-0xEF */ + 0x99, 0xC6, 0x99, 0xC7, 0x99, 0xC8, 0x99, 0xC9, /* 0xF0-0xF3 */ + 0x99, 0xCA, 0x99, 0xCB, 0x99, 0xCC, 0x99, 0xCD, /* 0xF4-0xF7 */ + 0x99, 0xCE, 0x99, 0xCF, 0x99, 0xD0, 0x99, 0xD1, /* 0xF8-0xFB */ + 0x99, 0xD2, 0x99, 0xD3, 0x99, 0xD4, 0x99, 0xD5, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_6B[512] = { + 0x99, 0xD6, 0x99, 0xD7, 0x99, 0xD8, 0x99, 0xD9, /* 0x00-0x03 */ + 0x99, 0xDA, 0x99, 0xDB, 0x99, 0xDC, 0x99, 0xDD, /* 0x04-0x07 */ + 0x99, 0xDE, 0x99, 0xDF, 0x99, 0xE0, 0x99, 0xE1, /* 0x08-0x0B */ + 0x99, 0xE2, 0x99, 0xE3, 0x99, 0xE4, 0x99, 0xE5, /* 0x0C-0x0F */ + 0x99, 0xE6, 0x99, 0xE7, 0x99, 0xE8, 0x99, 0xE9, /* 0x10-0x13 */ + 0x99, 0xEA, 0x99, 0xEB, 0x99, 0xEC, 0x99, 0xED, /* 0x14-0x17 */ + 0x99, 0xEE, 0x99, 0xEF, 0x99, 0xF0, 0x99, 0xF1, /* 0x18-0x1B */ + 0x99, 0xF2, 0x99, 0xF3, 0x99, 0xF4, 0x99, 0xF5, /* 0x1C-0x1F */ + 0xC7, 0xB7, 0xB4, 0xCE, 0xBB, 0xB6, 0xD0, 0xC0, /* 0x20-0x23 */ + 0xEC, 0xA3, 0x99, 0xF6, 0x99, 0xF7, 0xC5, 0xB7, /* 0x24-0x27 */ + 0x99, 0xF8, 0x99, 0xF9, 0x99, 0xFA, 0x99, 0xFB, /* 0x28-0x2B */ + 0x99, 0xFC, 0x99, 0xFD, 0x99, 0xFE, 0x9A, 0x40, /* 0x2C-0x2F */ + 0x9A, 0x41, 0x9A, 0x42, 0xD3, 0xFB, 0x9A, 0x43, /* 0x30-0x33 */ + 0x9A, 0x44, 0x9A, 0x45, 0x9A, 0x46, 0xEC, 0xA4, /* 0x34-0x37 */ + 0x9A, 0x47, 0xEC, 0xA5, 0xC6, 0xDB, 0x9A, 0x48, /* 0x38-0x3B */ + 0x9A, 0x49, 0x9A, 0x4A, 0xBF, 0xEE, 0x9A, 0x4B, /* 0x3C-0x3F */ + 0x9A, 0x4C, 0x9A, 0x4D, 0x9A, 0x4E, 0xEC, 0xA6, /* 0x40-0x43 */ + 0x9A, 0x4F, 0x9A, 0x50, 0xEC, 0xA7, 0xD0, 0xAA, /* 0x44-0x47 */ + 0x9A, 0x51, 0xC7, 0xB8, 0x9A, 0x52, 0x9A, 0x53, /* 0x48-0x4B */ + 0xB8, 0xE8, 0x9A, 0x54, 0x9A, 0x55, 0x9A, 0x56, /* 0x4C-0x4F */ + 0x9A, 0x57, 0x9A, 0x58, 0x9A, 0x59, 0x9A, 0x5A, /* 0x50-0x53 */ + 0x9A, 0x5B, 0x9A, 0x5C, 0x9A, 0x5D, 0x9A, 0x5E, /* 0x54-0x57 */ + 0x9A, 0x5F, 0xEC, 0xA8, 0x9A, 0x60, 0x9A, 0x61, /* 0x58-0x5B */ + 0x9A, 0x62, 0x9A, 0x63, 0x9A, 0x64, 0x9A, 0x65, /* 0x5C-0x5F */ + 0x9A, 0x66, 0x9A, 0x67, 0xD6, 0xB9, 0xD5, 0xFD, /* 0x60-0x63 */ + 0xB4, 0xCB, 0xB2, 0xBD, 0xCE, 0xE4, 0xC6, 0xE7, /* 0x64-0x67 */ + 0x9A, 0x68, 0x9A, 0x69, 0xCD, 0xE1, 0x9A, 0x6A, /* 0x68-0x6B */ + 0x9A, 0x6B, 0x9A, 0x6C, 0x9A, 0x6D, 0x9A, 0x6E, /* 0x6C-0x6F */ + 0x9A, 0x6F, 0x9A, 0x70, 0x9A, 0x71, 0x9A, 0x72, /* 0x70-0x73 */ + 0x9A, 0x73, 0x9A, 0x74, 0x9A, 0x75, 0x9A, 0x76, /* 0x74-0x77 */ + 0x9A, 0x77, 0xB4, 0xF5, 0x9A, 0x78, 0xCB, 0xC0, /* 0x78-0x7B */ + 0xBC, 0xDF, 0x9A, 0x79, 0x9A, 0x7A, 0x9A, 0x7B, /* 0x7C-0x7F */ + + 0x9A, 0x7C, 0xE9, 0xE2, 0xE9, 0xE3, 0xD1, 0xEA, /* 0x80-0x83 */ + 0xE9, 0xE5, 0x9A, 0x7D, 0xB4, 0xF9, 0xE9, 0xE4, /* 0x84-0x87 */ + 0x9A, 0x7E, 0xD1, 0xB3, 0xCA, 0xE2, 0xB2, 0xD0, /* 0x88-0x8B */ + 0x9A, 0x80, 0xE9, 0xE8, 0x9A, 0x81, 0x9A, 0x82, /* 0x8C-0x8F */ + 0x9A, 0x83, 0x9A, 0x84, 0xE9, 0xE6, 0xE9, 0xE7, /* 0x90-0x93 */ + 0x9A, 0x85, 0x9A, 0x86, 0xD6, 0xB3, 0x9A, 0x87, /* 0x94-0x97 */ + 0x9A, 0x88, 0x9A, 0x89, 0xE9, 0xE9, 0xE9, 0xEA, /* 0x98-0x9B */ + 0x9A, 0x8A, 0x9A, 0x8B, 0x9A, 0x8C, 0x9A, 0x8D, /* 0x9C-0x9F */ + 0x9A, 0x8E, 0xE9, 0xEB, 0x9A, 0x8F, 0x9A, 0x90, /* 0xA0-0xA3 */ + 0x9A, 0x91, 0x9A, 0x92, 0x9A, 0x93, 0x9A, 0x94, /* 0xA4-0xA7 */ + 0x9A, 0x95, 0x9A, 0x96, 0xE9, 0xEC, 0x9A, 0x97, /* 0xA8-0xAB */ + 0x9A, 0x98, 0x9A, 0x99, 0x9A, 0x9A, 0x9A, 0x9B, /* 0xAC-0xAF */ + 0x9A, 0x9C, 0x9A, 0x9D, 0x9A, 0x9E, 0xEC, 0xAF, /* 0xB0-0xB3 */ + 0xC5, 0xB9, 0xB6, 0xCE, 0x9A, 0x9F, 0xD2, 0xF3, /* 0xB4-0xB7 */ + 0x9A, 0xA0, 0x9A, 0xA1, 0x9A, 0xA2, 0x9A, 0xA3, /* 0xB8-0xBB */ + 0x9A, 0xA4, 0x9A, 0xA5, 0x9A, 0xA6, 0xB5, 0xEE, /* 0xBC-0xBF */ + 0x9A, 0xA7, 0xBB, 0xD9, 0xEC, 0xB1, 0x9A, 0xA8, /* 0xC0-0xC3 */ + 0x9A, 0xA9, 0xD2, 0xE3, 0x9A, 0xAA, 0x9A, 0xAB, /* 0xC4-0xC7 */ + 0x9A, 0xAC, 0x9A, 0xAD, 0x9A, 0xAE, 0xCE, 0xE3, /* 0xC8-0xCB */ + 0x9A, 0xAF, 0xC4, 0xB8, 0x9A, 0xB0, 0xC3, 0xBF, /* 0xCC-0xCF */ + 0x9A, 0xB1, 0x9A, 0xB2, 0xB6, 0xBE, 0xD8, 0xB9, /* 0xD0-0xD3 */ + 0xB1, 0xC8, 0xB1, 0xCF, 0xB1, 0xD1, 0xC5, 0xFE, /* 0xD4-0xD7 */ + 0x9A, 0xB3, 0xB1, 0xD0, 0x9A, 0xB4, 0xC3, 0xAB, /* 0xD8-0xDB */ + 0x9A, 0xB5, 0x9A, 0xB6, 0x9A, 0xB7, 0x9A, 0xB8, /* 0xDC-0xDF */ + 0x9A, 0xB9, 0xD5, 0xB1, 0x9A, 0xBA, 0x9A, 0xBB, /* 0xE0-0xE3 */ + 0x9A, 0xBC, 0x9A, 0xBD, 0x9A, 0xBE, 0x9A, 0xBF, /* 0xE4-0xE7 */ + 0x9A, 0xC0, 0x9A, 0xC1, 0xEB, 0xA4, 0xBA, 0xC1, /* 0xE8-0xEB */ + 0x9A, 0xC2, 0x9A, 0xC3, 0x9A, 0xC4, 0xCC, 0xBA, /* 0xEC-0xEF */ + 0x9A, 0xC5, 0x9A, 0xC6, 0x9A, 0xC7, 0xEB, 0xA5, /* 0xF0-0xF3 */ + 0x9A, 0xC8, 0xEB, 0xA7, 0x9A, 0xC9, 0x9A, 0xCA, /* 0xF4-0xF7 */ + 0x9A, 0xCB, 0xEB, 0xA8, 0x9A, 0xCC, 0x9A, 0xCD, /* 0xF8-0xFB */ + 0x9A, 0xCE, 0xEB, 0xA6, 0x9A, 0xCF, 0x9A, 0xD0, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_6C[512] = { + 0x9A, 0xD1, 0x9A, 0xD2, 0x9A, 0xD3, 0x9A, 0xD4, /* 0x00-0x03 */ + 0x9A, 0xD5, 0xEB, 0xA9, 0xEB, 0xAB, 0xEB, 0xAA, /* 0x04-0x07 */ + 0x9A, 0xD6, 0x9A, 0xD7, 0x9A, 0xD8, 0x9A, 0xD9, /* 0x08-0x0B */ + 0x9A, 0xDA, 0xEB, 0xAC, 0x9A, 0xDB, 0xCA, 0xCF, /* 0x0C-0x0F */ + 0xD8, 0xB5, 0xC3, 0xF1, 0x9A, 0xDC, 0xC3, 0xA5, /* 0x10-0x13 */ + 0xC6, 0xF8, 0xEB, 0xAD, 0xC4, 0xCA, 0x9A, 0xDD, /* 0x14-0x17 */ + 0xEB, 0xAE, 0xEB, 0xAF, 0xEB, 0xB0, 0xB7, 0xD5, /* 0x18-0x1B */ + 0x9A, 0xDE, 0x9A, 0xDF, 0x9A, 0xE0, 0xB7, 0xFA, /* 0x1C-0x1F */ + 0x9A, 0xE1, 0xEB, 0xB1, 0xC7, 0xE2, 0x9A, 0xE2, /* 0x20-0x23 */ + 0xEB, 0xB3, 0x9A, 0xE3, 0xBA, 0xA4, 0xD1, 0xF5, /* 0x24-0x27 */ + 0xB0, 0xB1, 0xEB, 0xB2, 0xEB, 0xB4, 0x9A, 0xE4, /* 0x28-0x2B */ + 0x9A, 0xE5, 0x9A, 0xE6, 0xB5, 0xAA, 0xC2, 0xC8, /* 0x2C-0x2F */ + 0xC7, 0xE8, 0x9A, 0xE7, 0xEB, 0xB5, 0x9A, 0xE8, /* 0x30-0x33 */ + 0xCB, 0xAE, 0xE3, 0xDF, 0x9A, 0xE9, 0x9A, 0xEA, /* 0x34-0x37 */ + 0xD3, 0xC0, 0x9A, 0xEB, 0x9A, 0xEC, 0x9A, 0xED, /* 0x38-0x3B */ + 0x9A, 0xEE, 0xD9, 0xDB, 0x9A, 0xEF, 0x9A, 0xF0, /* 0x3C-0x3F */ + 0xCD, 0xA1, 0xD6, 0xAD, 0xC7, 0xF3, 0x9A, 0xF1, /* 0x40-0x43 */ + 0x9A, 0xF2, 0x9A, 0xF3, 0xD9, 0xE0, 0xBB, 0xE3, /* 0x44-0x47 */ + 0x9A, 0xF4, 0xBA, 0xBA, 0xE3, 0xE2, 0x9A, 0xF5, /* 0x48-0x4B */ + 0x9A, 0xF6, 0x9A, 0xF7, 0x9A, 0xF8, 0x9A, 0xF9, /* 0x4C-0x4F */ + 0xCF, 0xAB, 0x9A, 0xFA, 0x9A, 0xFB, 0x9A, 0xFC, /* 0x50-0x53 */ + 0xE3, 0xE0, 0xC9, 0xC7, 0x9A, 0xFD, 0xBA, 0xB9, /* 0x54-0x57 */ + 0x9A, 0xFE, 0x9B, 0x40, 0x9B, 0x41, 0xD1, 0xB4, /* 0x58-0x5B */ + 0xE3, 0xE1, 0xC8, 0xEA, 0xB9, 0xAF, 0xBD, 0xAD, /* 0x5C-0x5F */ + 0xB3, 0xD8, 0xCE, 0xDB, 0x9B, 0x42, 0x9B, 0x43, /* 0x60-0x63 */ + 0xCC, 0xC0, 0x9B, 0x44, 0x9B, 0x45, 0x9B, 0x46, /* 0x64-0x67 */ + 0xE3, 0xE8, 0xE3, 0xE9, 0xCD, 0xF4, 0x9B, 0x47, /* 0x68-0x6B */ + 0x9B, 0x48, 0x9B, 0x49, 0x9B, 0x4A, 0x9B, 0x4B, /* 0x6C-0x6F */ + 0xCC, 0xAD, 0x9B, 0x4C, 0xBC, 0xB3, 0x9B, 0x4D, /* 0x70-0x73 */ + 0xE3, 0xEA, 0x9B, 0x4E, 0xE3, 0xEB, 0x9B, 0x4F, /* 0x74-0x77 */ + 0x9B, 0x50, 0xD0, 0xDA, 0x9B, 0x51, 0x9B, 0x52, /* 0x78-0x7B */ + 0x9B, 0x53, 0xC6, 0xFB, 0xB7, 0xDA, 0x9B, 0x54, /* 0x7C-0x7F */ + + 0x9B, 0x55, 0xC7, 0xDF, 0xD2, 0xCA, 0xCE, 0xD6, /* 0x80-0x83 */ + 0x9B, 0x56, 0xE3, 0xE4, 0xE3, 0xEC, 0x9B, 0x57, /* 0x84-0x87 */ + 0xC9, 0xF2, 0xB3, 0xC1, 0x9B, 0x58, 0x9B, 0x59, /* 0x88-0x8B */ + 0xE3, 0xE7, 0x9B, 0x5A, 0x9B, 0x5B, 0xC6, 0xE3, /* 0x8C-0x8F */ + 0xE3, 0xE5, 0x9B, 0x5C, 0x9B, 0x5D, 0xED, 0xB3, /* 0x90-0x93 */ + 0xE3, 0xE6, 0x9B, 0x5E, 0x9B, 0x5F, 0x9B, 0x60, /* 0x94-0x97 */ + 0x9B, 0x61, 0xC9, 0xB3, 0x9B, 0x62, 0xC5, 0xE6, /* 0x98-0x9B */ + 0x9B, 0x63, 0x9B, 0x64, 0x9B, 0x65, 0xB9, 0xB5, /* 0x9C-0x9F */ + 0x9B, 0x66, 0xC3, 0xBB, 0x9B, 0x67, 0xE3, 0xE3, /* 0xA0-0xA3 */ + 0xC5, 0xBD, 0xC1, 0xA4, 0xC2, 0xD9, 0xB2, 0xD7, /* 0xA4-0xA7 */ + 0x9B, 0x68, 0xE3, 0xED, 0xBB, 0xA6, 0xC4, 0xAD, /* 0xA8-0xAB */ + 0x9B, 0x69, 0xE3, 0xF0, 0xBE, 0xDA, 0x9B, 0x6A, /* 0xAC-0xAF */ + 0x9B, 0x6B, 0xE3, 0xFB, 0xE3, 0xF5, 0xBA, 0xD3, /* 0xB0-0xB3 */ + 0x9B, 0x6C, 0x9B, 0x6D, 0x9B, 0x6E, 0x9B, 0x6F, /* 0xB4-0xB7 */ + 0xB7, 0xD0, 0xD3, 0xCD, 0x9B, 0x70, 0xD6, 0xCE, /* 0xB8-0xBB */ + 0xD5, 0xD3, 0xB9, 0xC1, 0xD5, 0xB4, 0xD1, 0xD8, /* 0xBC-0xBF */ + 0x9B, 0x71, 0x9B, 0x72, 0x9B, 0x73, 0x9B, 0x74, /* 0xC0-0xC3 */ + 0xD0, 0xB9, 0xC7, 0xF6, 0x9B, 0x75, 0x9B, 0x76, /* 0xC4-0xC7 */ + 0x9B, 0x77, 0xC8, 0xAA, 0xB2, 0xB4, 0x9B, 0x78, /* 0xC8-0xCB */ + 0xC3, 0xDA, 0x9B, 0x79, 0x9B, 0x7A, 0x9B, 0x7B, /* 0xCC-0xCF */ + 0xE3, 0xEE, 0x9B, 0x7C, 0x9B, 0x7D, 0xE3, 0xFC, /* 0xD0-0xD3 */ + 0xE3, 0xEF, 0xB7, 0xA8, 0xE3, 0xF7, 0xE3, 0xF4, /* 0xD4-0xD7 */ + 0x9B, 0x7E, 0x9B, 0x80, 0x9B, 0x81, 0xB7, 0xBA, /* 0xD8-0xDB */ + 0x9B, 0x82, 0x9B, 0x83, 0xC5, 0xA2, 0x9B, 0x84, /* 0xDC-0xDF */ + 0xE3, 0xF6, 0xC5, 0xDD, 0xB2, 0xA8, 0xC6, 0xFC, /* 0xE0-0xE3 */ + 0x9B, 0x85, 0xC4, 0xE0, 0x9B, 0x86, 0x9B, 0x87, /* 0xE4-0xE7 */ + 0xD7, 0xA2, 0x9B, 0x88, 0xC0, 0xE1, 0xE3, 0xF9, /* 0xE8-0xEB */ + 0x9B, 0x89, 0x9B, 0x8A, 0xE3, 0xFA, 0xE3, 0xFD, /* 0xEC-0xEF */ + 0xCC, 0xA9, 0xE3, 0xF3, 0x9B, 0x8B, 0xD3, 0xBE, /* 0xF0-0xF3 */ + 0x9B, 0x8C, 0xB1, 0xC3, 0xED, 0xB4, 0xE3, 0xF1, /* 0xF4-0xF7 */ + 0xE3, 0xF2, 0x9B, 0x8D, 0xE3, 0xF8, 0xD0, 0xBA, /* 0xF8-0xFB */ + 0xC6, 0xC3, 0xD4, 0xF3, 0xE3, 0xFE, 0x9B, 0x8E, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_6D[512] = { + 0x9B, 0x8F, 0xBD, 0xE0, 0x9B, 0x90, 0x9B, 0x91, /* 0x00-0x03 */ + 0xE4, 0xA7, 0x9B, 0x92, 0x9B, 0x93, 0xE4, 0xA6, /* 0x04-0x07 */ + 0x9B, 0x94, 0x9B, 0x95, 0x9B, 0x96, 0xD1, 0xF3, /* 0x08-0x0B */ + 0xE4, 0xA3, 0x9B, 0x97, 0xE4, 0xA9, 0x9B, 0x98, /* 0x0C-0x0F */ + 0x9B, 0x99, 0x9B, 0x9A, 0xC8, 0xF7, 0x9B, 0x9B, /* 0x10-0x13 */ + 0x9B, 0x9C, 0x9B, 0x9D, 0x9B, 0x9E, 0xCF, 0xB4, /* 0x14-0x17 */ + 0x9B, 0x9F, 0xE4, 0xA8, 0xE4, 0xAE, 0xC2, 0xE5, /* 0x18-0x1B */ + 0x9B, 0xA0, 0x9B, 0xA1, 0xB6, 0xB4, 0x9B, 0xA2, /* 0x1C-0x1F */ + 0x9B, 0xA3, 0x9B, 0xA4, 0x9B, 0xA5, 0x9B, 0xA6, /* 0x20-0x23 */ + 0x9B, 0xA7, 0xBD, 0xF2, 0x9B, 0xA8, 0xE4, 0xA2, /* 0x24-0x27 */ + 0x9B, 0xA9, 0x9B, 0xAA, 0xBA, 0xE9, 0xE4, 0xAA, /* 0x28-0x2B */ + 0x9B, 0xAB, 0x9B, 0xAC, 0xE4, 0xAC, 0x9B, 0xAD, /* 0x2C-0x2F */ + 0x9B, 0xAE, 0xB6, 0xFD, 0xD6, 0xDE, 0xE4, 0xB2, /* 0x30-0x33 */ + 0x9B, 0xAF, 0xE4, 0xAD, 0x9B, 0xB0, 0x9B, 0xB1, /* 0x34-0x37 */ + 0x9B, 0xB2, 0xE4, 0xA1, 0x9B, 0xB3, 0xBB, 0xEE, /* 0x38-0x3B */ + 0xCD, 0xDD, 0xC7, 0xA2, 0xC5, 0xC9, 0x9B, 0xB4, /* 0x3C-0x3F */ + 0x9B, 0xB5, 0xC1, 0xF7, 0x9B, 0xB6, 0xE4, 0xA4, /* 0x40-0x43 */ + 0x9B, 0xB7, 0xC7, 0xB3, 0xBD, 0xAC, 0xBD, 0xBD, /* 0x44-0x47 */ + 0xE4, 0xA5, 0x9B, 0xB8, 0xD7, 0xC7, 0xB2, 0xE2, /* 0x48-0x4B */ + 0x9B, 0xB9, 0xE4, 0xAB, 0xBC, 0xC3, 0xE4, 0xAF, /* 0x4C-0x4F */ + 0x9B, 0xBA, 0xBB, 0xEB, 0xE4, 0xB0, 0xC5, 0xA8, /* 0x50-0x53 */ + 0xE4, 0xB1, 0x9B, 0xBB, 0x9B, 0xBC, 0x9B, 0xBD, /* 0x54-0x57 */ + 0x9B, 0xBE, 0xD5, 0xE3, 0xBF, 0xA3, 0x9B, 0xBF, /* 0x58-0x5B */ + 0xE4, 0xBA, 0x9B, 0xC0, 0xE4, 0xB7, 0x9B, 0xC1, /* 0x5C-0x5F */ + 0xE4, 0xBB, 0x9B, 0xC2, 0x9B, 0xC3, 0xE4, 0xBD, /* 0x60-0x63 */ + 0x9B, 0xC4, 0x9B, 0xC5, 0xC6, 0xD6, 0x9B, 0xC6, /* 0x64-0x67 */ + 0x9B, 0xC7, 0xBA, 0xC6, 0xC0, 0xCB, 0x9B, 0xC8, /* 0x68-0x6B */ + 0x9B, 0xC9, 0x9B, 0xCA, 0xB8, 0xA1, 0xE4, 0xB4, /* 0x6C-0x6F */ + 0x9B, 0xCB, 0x9B, 0xCC, 0x9B, 0xCD, 0x9B, 0xCE, /* 0x70-0x73 */ + 0xD4, 0xA1, 0x9B, 0xCF, 0x9B, 0xD0, 0xBA, 0xA3, /* 0x74-0x77 */ + 0xBD, 0xFE, 0x9B, 0xD1, 0x9B, 0xD2, 0x9B, 0xD3, /* 0x78-0x7B */ + 0xE4, 0xBC, 0x9B, 0xD4, 0x9B, 0xD5, 0x9B, 0xD6, /* 0x7C-0x7F */ + + 0x9B, 0xD7, 0x9B, 0xD8, 0xCD, 0xBF, 0x9B, 0xD9, /* 0x80-0x83 */ + 0x9B, 0xDA, 0xC4, 0xF9, 0x9B, 0xDB, 0x9B, 0xDC, /* 0x84-0x87 */ + 0xCF, 0xFB, 0xC9, 0xE6, 0x9B, 0xDD, 0x9B, 0xDE, /* 0x88-0x8B */ + 0xD3, 0xBF, 0x9B, 0xDF, 0xCF, 0xD1, 0x9B, 0xE0, /* 0x8C-0x8F */ + 0x9B, 0xE1, 0xE4, 0xB3, 0x9B, 0xE2, 0xE4, 0xB8, /* 0x90-0x93 */ + 0xE4, 0xB9, 0xCC, 0xE9, 0x9B, 0xE3, 0x9B, 0xE4, /* 0x94-0x97 */ + 0x9B, 0xE5, 0x9B, 0xE6, 0x9B, 0xE7, 0xCC, 0xCE, /* 0x98-0x9B */ + 0x9B, 0xE8, 0xC0, 0xD4, 0xE4, 0xB5, 0xC1, 0xB0, /* 0x9C-0x9F */ + 0xE4, 0xB6, 0xCE, 0xD0, 0x9B, 0xE9, 0xBB, 0xC1, /* 0xA0-0xA3 */ + 0xB5, 0xD3, 0x9B, 0xEA, 0xC8, 0xF3, 0xBD, 0xA7, /* 0xA4-0xA7 */ + 0xD5, 0xC7, 0xC9, 0xAC, 0xB8, 0xA2, 0xE4, 0xCA, /* 0xA8-0xAB */ + 0x9B, 0xEB, 0x9B, 0xEC, 0xE4, 0xCC, 0xD1, 0xC4, /* 0xAC-0xAF */ + 0x9B, 0xED, 0x9B, 0xEE, 0xD2, 0xBA, 0x9B, 0xEF, /* 0xB0-0xB3 */ + 0x9B, 0xF0, 0xBA, 0xAD, 0x9B, 0xF1, 0x9B, 0xF2, /* 0xB4-0xB7 */ + 0xBA, 0xD4, 0x9B, 0xF3, 0x9B, 0xF4, 0x9B, 0xF5, /* 0xB8-0xBB */ + 0x9B, 0xF6, 0x9B, 0xF7, 0x9B, 0xF8, 0xE4, 0xC3, /* 0xBC-0xBF */ + 0xB5, 0xED, 0x9B, 0xF9, 0x9B, 0xFA, 0x9B, 0xFB, /* 0xC0-0xC3 */ + 0xD7, 0xCD, 0xE4, 0xC0, 0xCF, 0xFD, 0xE4, 0xBF, /* 0xC4-0xC7 */ + 0x9B, 0xFC, 0x9B, 0xFD, 0x9B, 0xFE, 0xC1, 0xDC, /* 0xC8-0xCB */ + 0xCC, 0xCA, 0x9C, 0x40, 0x9C, 0x41, 0x9C, 0x42, /* 0xCC-0xCF */ + 0x9C, 0x43, 0xCA, 0xE7, 0x9C, 0x44, 0x9C, 0x45, /* 0xD0-0xD3 */ + 0x9C, 0x46, 0x9C, 0x47, 0xC4, 0xD7, 0x9C, 0x48, /* 0xD4-0xD7 */ + 0xCC, 0xD4, 0xE4, 0xC8, 0x9C, 0x49, 0x9C, 0x4A, /* 0xD8-0xDB */ + 0x9C, 0x4B, 0xE4, 0xC7, 0xE4, 0xC1, 0x9C, 0x4C, /* 0xDC-0xDF */ + 0xE4, 0xC4, 0xB5, 0xAD, 0x9C, 0x4D, 0x9C, 0x4E, /* 0xE0-0xE3 */ + 0xD3, 0xD9, 0x9C, 0x4F, 0xE4, 0xC6, 0x9C, 0x50, /* 0xE4-0xE7 */ + 0x9C, 0x51, 0x9C, 0x52, 0x9C, 0x53, 0xD2, 0xF9, /* 0xE8-0xEB */ + 0xB4, 0xE3, 0x9C, 0x54, 0xBB, 0xB4, 0x9C, 0x55, /* 0xEC-0xEF */ + 0x9C, 0x56, 0xC9, 0xEE, 0x9C, 0x57, 0xB4, 0xBE, /* 0xF0-0xF3 */ + 0x9C, 0x58, 0x9C, 0x59, 0x9C, 0x5A, 0xBB, 0xEC, /* 0xF4-0xF7 */ + 0x9C, 0x5B, 0xD1, 0xCD, 0x9C, 0x5C, 0xCC, 0xED, /* 0xF8-0xFB */ + 0xED, 0xB5, 0x9C, 0x5D, 0x9C, 0x5E, 0x9C, 0x5F, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_6E[512] = { + 0x9C, 0x60, 0x9C, 0x61, 0x9C, 0x62, 0x9C, 0x63, /* 0x00-0x03 */ + 0x9C, 0x64, 0xC7, 0xE5, 0x9C, 0x65, 0x9C, 0x66, /* 0x04-0x07 */ + 0x9C, 0x67, 0x9C, 0x68, 0xD4, 0xA8, 0x9C, 0x69, /* 0x08-0x0B */ + 0xE4, 0xCB, 0xD7, 0xD5, 0xE4, 0xC2, 0x9C, 0x6A, /* 0x0C-0x0F */ + 0xBD, 0xA5, 0xE4, 0xC5, 0x9C, 0x6B, 0x9C, 0x6C, /* 0x10-0x13 */ + 0xD3, 0xE6, 0x9C, 0x6D, 0xE4, 0xC9, 0xC9, 0xF8, /* 0x14-0x17 */ + 0x9C, 0x6E, 0x9C, 0x6F, 0xE4, 0xBE, 0x9C, 0x70, /* 0x18-0x1B */ + 0x9C, 0x71, 0xD3, 0xE5, 0x9C, 0x72, 0x9C, 0x73, /* 0x1C-0x1F */ + 0xC7, 0xFE, 0xB6, 0xC9, 0x9C, 0x74, 0xD4, 0xFC, /* 0x20-0x23 */ + 0xB2, 0xB3, 0xE4, 0xD7, 0x9C, 0x75, 0x9C, 0x76, /* 0x24-0x27 */ + 0x9C, 0x77, 0xCE, 0xC2, 0x9C, 0x78, 0xE4, 0xCD, /* 0x28-0x2B */ + 0x9C, 0x79, 0xCE, 0xBC, 0x9C, 0x7A, 0xB8, 0xDB, /* 0x2C-0x2F */ + 0x9C, 0x7B, 0x9C, 0x7C, 0xE4, 0xD6, 0x9C, 0x7D, /* 0x30-0x33 */ + 0xBF, 0xCA, 0x9C, 0x7E, 0x9C, 0x80, 0x9C, 0x81, /* 0x34-0x37 */ + 0xD3, 0xCE, 0x9C, 0x82, 0xC3, 0xEC, 0x9C, 0x83, /* 0x38-0x3B */ + 0x9C, 0x84, 0x9C, 0x85, 0x9C, 0x86, 0x9C, 0x87, /* 0x3C-0x3F */ + 0x9C, 0x88, 0x9C, 0x89, 0x9C, 0x8A, 0xC5, 0xC8, /* 0x40-0x43 */ + 0xE4, 0xD8, 0x9C, 0x8B, 0x9C, 0x8C, 0x9C, 0x8D, /* 0x44-0x47 */ + 0x9C, 0x8E, 0x9C, 0x8F, 0x9C, 0x90, 0x9C, 0x91, /* 0x48-0x4B */ + 0x9C, 0x92, 0xCD, 0xC4, 0xE4, 0xCF, 0x9C, 0x93, /* 0x4C-0x4F */ + 0x9C, 0x94, 0x9C, 0x95, 0x9C, 0x96, 0xE4, 0xD4, /* 0x50-0x53 */ + 0xE4, 0xD5, 0x9C, 0x97, 0xBA, 0xFE, 0x9C, 0x98, /* 0x54-0x57 */ + 0xCF, 0xE6, 0x9C, 0x99, 0x9C, 0x9A, 0xD5, 0xBF, /* 0x58-0x5B */ + 0x9C, 0x9B, 0x9C, 0x9C, 0x9C, 0x9D, 0xE4, 0xD2, /* 0x5C-0x5F */ + 0x9C, 0x9E, 0x9C, 0x9F, 0x9C, 0xA0, 0x9C, 0xA1, /* 0x60-0x63 */ + 0x9C, 0xA2, 0x9C, 0xA3, 0x9C, 0xA4, 0x9C, 0xA5, /* 0x64-0x67 */ + 0x9C, 0xA6, 0x9C, 0xA7, 0x9C, 0xA8, 0xE4, 0xD0, /* 0x68-0x6B */ + 0x9C, 0xA9, 0x9C, 0xAA, 0xE4, 0xCE, 0x9C, 0xAB, /* 0x6C-0x6F */ + 0x9C, 0xAC, 0x9C, 0xAD, 0x9C, 0xAE, 0x9C, 0xAF, /* 0x70-0x73 */ + 0x9C, 0xB0, 0x9C, 0xB1, 0x9C, 0xB2, 0x9C, 0xB3, /* 0x74-0x77 */ + 0x9C, 0xB4, 0x9C, 0xB5, 0x9C, 0xB6, 0x9C, 0xB7, /* 0x78-0x7B */ + 0x9C, 0xB8, 0x9C, 0xB9, 0xCD, 0xE5, 0xCA, 0xAA, /* 0x7C-0x7F */ + + 0x9C, 0xBA, 0x9C, 0xBB, 0x9C, 0xBC, 0xC0, 0xA3, /* 0x80-0x83 */ + 0x9C, 0xBD, 0xBD, 0xA6, 0xE4, 0xD3, 0x9C, 0xBE, /* 0x84-0x87 */ + 0x9C, 0xBF, 0xB8, 0xC8, 0x9C, 0xC0, 0x9C, 0xC1, /* 0x88-0x8B */ + 0x9C, 0xC2, 0x9C, 0xC3, 0x9C, 0xC4, 0xE4, 0xE7, /* 0x8C-0x8F */ + 0xD4, 0xB4, 0x9C, 0xC5, 0x9C, 0xC6, 0x9C, 0xC7, /* 0x90-0x93 */ + 0x9C, 0xC8, 0x9C, 0xC9, 0x9C, 0xCA, 0x9C, 0xCB, /* 0x94-0x97 */ + 0xE4, 0xDB, 0x9C, 0xCC, 0x9C, 0xCD, 0x9C, 0xCE, /* 0x98-0x9B */ + 0xC1, 0xEF, 0x9C, 0xCF, 0x9C, 0xD0, 0xE4, 0xE9, /* 0x9C-0x9F */ + 0x9C, 0xD1, 0x9C, 0xD2, 0xD2, 0xE7, 0x9C, 0xD3, /* 0xA0-0xA3 */ + 0x9C, 0xD4, 0xE4, 0xDF, 0x9C, 0xD5, 0xE4, 0xE0, /* 0xA4-0xA7 */ + 0x9C, 0xD6, 0x9C, 0xD7, 0xCF, 0xAA, 0x9C, 0xD8, /* 0xA8-0xAB */ + 0x9C, 0xD9, 0x9C, 0xDA, 0x9C, 0xDB, 0xCB, 0xDD, /* 0xAC-0xAF */ + 0x9C, 0xDC, 0xE4, 0xDA, 0xE4, 0xD1, 0x9C, 0xDD, /* 0xB0-0xB3 */ + 0xE4, 0xE5, 0x9C, 0xDE, 0xC8, 0xDC, 0xE4, 0xE3, /* 0xB4-0xB7 */ + 0x9C, 0xDF, 0x9C, 0xE0, 0xC4, 0xE7, 0xE4, 0xE2, /* 0xB8-0xBB */ + 0x9C, 0xE1, 0xE4, 0xE1, 0x9C, 0xE2, 0x9C, 0xE3, /* 0xBC-0xBF */ + 0x9C, 0xE4, 0xB3, 0xFC, 0xE4, 0xE8, 0x9C, 0xE5, /* 0xC0-0xC3 */ + 0x9C, 0xE6, 0x9C, 0xE7, 0x9C, 0xE8, 0xB5, 0xE1, /* 0xC4-0xC7 */ + 0x9C, 0xE9, 0x9C, 0xEA, 0x9C, 0xEB, 0xD7, 0xCC, /* 0xC8-0xCB */ + 0x9C, 0xEC, 0x9C, 0xED, 0x9C, 0xEE, 0xE4, 0xE6, /* 0xCC-0xCF */ + 0x9C, 0xEF, 0xBB, 0xAC, 0x9C, 0xF0, 0xD7, 0xD2, /* 0xD0-0xD3 */ + 0xCC, 0xCF, 0xEB, 0xF8, 0x9C, 0xF1, 0xE4, 0xE4, /* 0xD4-0xD7 */ + 0x9C, 0xF2, 0x9C, 0xF3, 0xB9, 0xF6, 0x9C, 0xF4, /* 0xD8-0xDB */ + 0x9C, 0xF5, 0x9C, 0xF6, 0xD6, 0xCD, 0xE4, 0xD9, /* 0xDC-0xDF */ + 0xE4, 0xDC, 0xC2, 0xFA, 0xE4, 0xDE, 0x9C, 0xF7, /* 0xE0-0xE3 */ + 0xC2, 0xCB, 0xC0, 0xC4, 0xC2, 0xD0, 0x9C, 0xF8, /* 0xE4-0xE7 */ + 0xB1, 0xF5, 0xCC, 0xB2, 0x9C, 0xF9, 0x9C, 0xFA, /* 0xE8-0xEB */ + 0x9C, 0xFB, 0x9C, 0xFC, 0x9C, 0xFD, 0x9C, 0xFE, /* 0xEC-0xEF */ + 0x9D, 0x40, 0x9D, 0x41, 0x9D, 0x42, 0x9D, 0x43, /* 0xF0-0xF3 */ + 0xB5, 0xCE, 0x9D, 0x44, 0x9D, 0x45, 0x9D, 0x46, /* 0xF4-0xF7 */ + 0x9D, 0x47, 0xE4, 0xEF, 0x9D, 0x48, 0x9D, 0x49, /* 0xF8-0xFB */ + 0x9D, 0x4A, 0x9D, 0x4B, 0x9D, 0x4C, 0x9D, 0x4D, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_6F[512] = { + 0x9D, 0x4E, 0x9D, 0x4F, 0xC6, 0xAF, 0x9D, 0x50, /* 0x00-0x03 */ + 0x9D, 0x51, 0x9D, 0x52, 0xC6, 0xE1, 0x9D, 0x53, /* 0x04-0x07 */ + 0x9D, 0x54, 0xE4, 0xF5, 0x9D, 0x55, 0x9D, 0x56, /* 0x08-0x0B */ + 0x9D, 0x57, 0x9D, 0x58, 0x9D, 0x59, 0xC2, 0xA9, /* 0x0C-0x0F */ + 0x9D, 0x5A, 0x9D, 0x5B, 0x9D, 0x5C, 0xC0, 0xEC, /* 0x10-0x13 */ + 0xD1, 0xDD, 0xE4, 0xEE, 0x9D, 0x5D, 0x9D, 0x5E, /* 0x14-0x17 */ + 0x9D, 0x5F, 0x9D, 0x60, 0x9D, 0x61, 0x9D, 0x62, /* 0x18-0x1B */ + 0x9D, 0x63, 0x9D, 0x64, 0x9D, 0x65, 0x9D, 0x66, /* 0x1C-0x1F */ + 0xC4, 0xAE, 0x9D, 0x67, 0x9D, 0x68, 0x9D, 0x69, /* 0x20-0x23 */ + 0xE4, 0xED, 0x9D, 0x6A, 0x9D, 0x6B, 0x9D, 0x6C, /* 0x24-0x27 */ + 0x9D, 0x6D, 0xE4, 0xF6, 0xE4, 0xF4, 0xC2, 0xFE, /* 0x28-0x2B */ + 0x9D, 0x6E, 0xE4, 0xDD, 0x9D, 0x6F, 0xE4, 0xF0, /* 0x2C-0x2F */ + 0x9D, 0x70, 0xCA, 0xFE, 0x9D, 0x71, 0xD5, 0xC4, /* 0x30-0x33 */ + 0x9D, 0x72, 0x9D, 0x73, 0xE4, 0xF1, 0x9D, 0x74, /* 0x34-0x37 */ + 0x9D, 0x75, 0x9D, 0x76, 0x9D, 0x77, 0x9D, 0x78, /* 0x38-0x3B */ + 0x9D, 0x79, 0x9D, 0x7A, 0xD1, 0xFA, 0x9D, 0x7B, /* 0x3C-0x3F */ + 0x9D, 0x7C, 0x9D, 0x7D, 0x9D, 0x7E, 0x9D, 0x80, /* 0x40-0x43 */ + 0x9D, 0x81, 0x9D, 0x82, 0xE4, 0xEB, 0xE4, 0xEC, /* 0x44-0x47 */ + 0x9D, 0x83, 0x9D, 0x84, 0x9D, 0x85, 0xE4, 0xF2, /* 0x48-0x4B */ + 0x9D, 0x86, 0xCE, 0xAB, 0x9D, 0x87, 0x9D, 0x88, /* 0x4C-0x4F */ + 0x9D, 0x89, 0x9D, 0x8A, 0x9D, 0x8B, 0x9D, 0x8C, /* 0x50-0x53 */ + 0x9D, 0x8D, 0x9D, 0x8E, 0x9D, 0x8F, 0x9D, 0x90, /* 0x54-0x57 */ + 0xC5, 0xCB, 0x9D, 0x91, 0x9D, 0x92, 0x9D, 0x93, /* 0x58-0x5B */ + 0xC7, 0xB1, 0x9D, 0x94, 0xC2, 0xBA, 0x9D, 0x95, /* 0x5C-0x5F */ + 0x9D, 0x96, 0x9D, 0x97, 0xE4, 0xEA, 0x9D, 0x98, /* 0x60-0x63 */ + 0x9D, 0x99, 0x9D, 0x9A, 0xC1, 0xCA, 0x9D, 0x9B, /* 0x64-0x67 */ + 0x9D, 0x9C, 0x9D, 0x9D, 0x9D, 0x9E, 0x9D, 0x9F, /* 0x68-0x6B */ + 0x9D, 0xA0, 0xCC, 0xB6, 0xB3, 0xB1, 0x9D, 0xA1, /* 0x6C-0x6F */ + 0x9D, 0xA2, 0x9D, 0xA3, 0xE4, 0xFB, 0x9D, 0xA4, /* 0x70-0x73 */ + 0xE4, 0xF3, 0x9D, 0xA5, 0x9D, 0xA6, 0x9D, 0xA7, /* 0x74-0x77 */ + 0xE4, 0xFA, 0x9D, 0xA8, 0xE4, 0xFD, 0x9D, 0xA9, /* 0x78-0x7B */ + 0xE4, 0xFC, 0x9D, 0xAA, 0x9D, 0xAB, 0x9D, 0xAC, /* 0x7C-0x7F */ + + 0x9D, 0xAD, 0x9D, 0xAE, 0x9D, 0xAF, 0x9D, 0xB0, /* 0x80-0x83 */ + 0xB3, 0xCE, 0x9D, 0xB1, 0x9D, 0xB2, 0x9D, 0xB3, /* 0x84-0x87 */ + 0xB3, 0xBA, 0xE4, 0xF7, 0x9D, 0xB4, 0x9D, 0xB5, /* 0x88-0x8B */ + 0xE4, 0xF9, 0xE4, 0xF8, 0xC5, 0xEC, 0x9D, 0xB6, /* 0x8C-0x8F */ + 0x9D, 0xB7, 0x9D, 0xB8, 0x9D, 0xB9, 0x9D, 0xBA, /* 0x90-0x93 */ + 0x9D, 0xBB, 0x9D, 0xBC, 0x9D, 0xBD, 0x9D, 0xBE, /* 0x94-0x97 */ + 0x9D, 0xBF, 0x9D, 0xC0, 0x9D, 0xC1, 0x9D, 0xC2, /* 0x98-0x9B */ + 0xC0, 0xBD, 0x9D, 0xC3, 0x9D, 0xC4, 0x9D, 0xC5, /* 0x9C-0x9F */ + 0x9D, 0xC6, 0xD4, 0xE8, 0x9D, 0xC7, 0x9D, 0xC8, /* 0xA0-0xA3 */ + 0x9D, 0xC9, 0x9D, 0xCA, 0x9D, 0xCB, 0xE5, 0xA2, /* 0xA4-0xA7 */ + 0x9D, 0xCC, 0x9D, 0xCD, 0x9D, 0xCE, 0x9D, 0xCF, /* 0xA8-0xAB */ + 0x9D, 0xD0, 0x9D, 0xD1, 0x9D, 0xD2, 0x9D, 0xD3, /* 0xAC-0xAF */ + 0x9D, 0xD4, 0x9D, 0xD5, 0x9D, 0xD6, 0xB0, 0xC4, /* 0xB0-0xB3 */ + 0x9D, 0xD7, 0x9D, 0xD8, 0xE5, 0xA4, 0x9D, 0xD9, /* 0xB4-0xB7 */ + 0x9D, 0xDA, 0xE5, 0xA3, 0x9D, 0xDB, 0x9D, 0xDC, /* 0xB8-0xBB */ + 0x9D, 0xDD, 0x9D, 0xDE, 0x9D, 0xDF, 0x9D, 0xE0, /* 0xBC-0xBF */ + 0xBC, 0xA4, 0x9D, 0xE1, 0xE5, 0xA5, 0x9D, 0xE2, /* 0xC0-0xC3 */ + 0x9D, 0xE3, 0x9D, 0xE4, 0x9D, 0xE5, 0x9D, 0xE6, /* 0xC4-0xC7 */ + 0x9D, 0xE7, 0xE5, 0xA1, 0x9D, 0xE8, 0x9D, 0xE9, /* 0xC8-0xCB */ + 0x9D, 0xEA, 0x9D, 0xEB, 0x9D, 0xEC, 0x9D, 0xED, /* 0xCC-0xCF */ + 0x9D, 0xEE, 0xE4, 0xFE, 0xB1, 0xF4, 0x9D, 0xEF, /* 0xD0-0xD3 */ + 0x9D, 0xF0, 0x9D, 0xF1, 0x9D, 0xF2, 0x9D, 0xF3, /* 0xD4-0xD7 */ + 0x9D, 0xF4, 0x9D, 0xF5, 0x9D, 0xF6, 0x9D, 0xF7, /* 0xD8-0xDB */ + 0x9D, 0xF8, 0x9D, 0xF9, 0xE5, 0xA8, 0x9D, 0xFA, /* 0xDC-0xDF */ + 0xE5, 0xA9, 0xE5, 0xA6, 0x9D, 0xFB, 0x9D, 0xFC, /* 0xE0-0xE3 */ + 0x9D, 0xFD, 0x9D, 0xFE, 0x9E, 0x40, 0x9E, 0x41, /* 0xE4-0xE7 */ + 0x9E, 0x42, 0x9E, 0x43, 0x9E, 0x44, 0x9E, 0x45, /* 0xE8-0xEB */ + 0x9E, 0x46, 0x9E, 0x47, 0xE5, 0xA7, 0xE5, 0xAA, /* 0xEC-0xEF */ + 0x9E, 0x48, 0x9E, 0x49, 0x9E, 0x4A, 0x9E, 0x4B, /* 0xF0-0xF3 */ + 0x9E, 0x4C, 0x9E, 0x4D, 0x9E, 0x4E, 0x9E, 0x4F, /* 0xF4-0xF7 */ + 0x9E, 0x50, 0x9E, 0x51, 0x9E, 0x52, 0x9E, 0x53, /* 0xF8-0xFB */ + 0x9E, 0x54, 0x9E, 0x55, 0x9E, 0x56, 0x9E, 0x57, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_70[512] = { + 0x9E, 0x58, 0x9E, 0x59, 0x9E, 0x5A, 0x9E, 0x5B, /* 0x00-0x03 */ + 0x9E, 0x5C, 0x9E, 0x5D, 0x9E, 0x5E, 0x9E, 0x5F, /* 0x04-0x07 */ + 0x9E, 0x60, 0x9E, 0x61, 0x9E, 0x62, 0x9E, 0x63, /* 0x08-0x0B */ + 0x9E, 0x64, 0x9E, 0x65, 0x9E, 0x66, 0x9E, 0x67, /* 0x0C-0x0F */ + 0x9E, 0x68, 0xC6, 0xD9, 0x9E, 0x69, 0x9E, 0x6A, /* 0x10-0x13 */ + 0x9E, 0x6B, 0x9E, 0x6C, 0x9E, 0x6D, 0x9E, 0x6E, /* 0x14-0x17 */ + 0x9E, 0x6F, 0x9E, 0x70, 0xE5, 0xAB, 0xE5, 0xAD, /* 0x18-0x1B */ + 0x9E, 0x71, 0x9E, 0x72, 0x9E, 0x73, 0x9E, 0x74, /* 0x1C-0x1F */ + 0x9E, 0x75, 0x9E, 0x76, 0x9E, 0x77, 0xE5, 0xAC, /* 0x20-0x23 */ + 0x9E, 0x78, 0x9E, 0x79, 0x9E, 0x7A, 0x9E, 0x7B, /* 0x24-0x27 */ + 0x9E, 0x7C, 0x9E, 0x7D, 0x9E, 0x7E, 0x9E, 0x80, /* 0x28-0x2B */ + 0x9E, 0x81, 0x9E, 0x82, 0x9E, 0x83, 0x9E, 0x84, /* 0x2C-0x2F */ + 0x9E, 0x85, 0x9E, 0x86, 0x9E, 0x87, 0x9E, 0x88, /* 0x30-0x33 */ + 0x9E, 0x89, 0xE5, 0xAF, 0x9E, 0x8A, 0x9E, 0x8B, /* 0x34-0x37 */ + 0x9E, 0x8C, 0xE5, 0xAE, 0x9E, 0x8D, 0x9E, 0x8E, /* 0x38-0x3B */ + 0x9E, 0x8F, 0x9E, 0x90, 0x9E, 0x91, 0x9E, 0x92, /* 0x3C-0x3F */ + 0x9E, 0x93, 0x9E, 0x94, 0x9E, 0x95, 0x9E, 0x96, /* 0x40-0x43 */ + 0x9E, 0x97, 0x9E, 0x98, 0x9E, 0x99, 0x9E, 0x9A, /* 0x44-0x47 */ + 0x9E, 0x9B, 0x9E, 0x9C, 0x9E, 0x9D, 0x9E, 0x9E, /* 0x48-0x4B */ + 0xB9, 0xE0, 0x9E, 0x9F, 0x9E, 0xA0, 0xE5, 0xB0, /* 0x4C-0x4F */ + 0x9E, 0xA1, 0x9E, 0xA2, 0x9E, 0xA3, 0x9E, 0xA4, /* 0x50-0x53 */ + 0x9E, 0xA5, 0x9E, 0xA6, 0x9E, 0xA7, 0x9E, 0xA8, /* 0x54-0x57 */ + 0x9E, 0xA9, 0x9E, 0xAA, 0x9E, 0xAB, 0x9E, 0xAC, /* 0x58-0x5B */ + 0x9E, 0xAD, 0x9E, 0xAE, 0xE5, 0xB1, 0x9E, 0xAF, /* 0x5C-0x5F */ + 0x9E, 0xB0, 0x9E, 0xB1, 0x9E, 0xB2, 0x9E, 0xB3, /* 0x60-0x63 */ + 0x9E, 0xB4, 0x9E, 0xB5, 0x9E, 0xB6, 0x9E, 0xB7, /* 0x64-0x67 */ + 0x9E, 0xB8, 0x9E, 0xB9, 0x9E, 0xBA, 0xBB, 0xF0, /* 0x68-0x6B */ + 0xEC, 0xE1, 0xC3, 0xF0, 0x9E, 0xBB, 0xB5, 0xC6, /* 0x6C-0x6F */ + 0xBB, 0xD2, 0x9E, 0xBC, 0x9E, 0xBD, 0x9E, 0xBE, /* 0x70-0x73 */ + 0x9E, 0xBF, 0xC1, 0xE9, 0xD4, 0xEE, 0x9E, 0xC0, /* 0x74-0x77 */ + 0xBE, 0xC4, 0x9E, 0xC1, 0x9E, 0xC2, 0x9E, 0xC3, /* 0x78-0x7B */ + 0xD7, 0xC6, 0x9E, 0xC4, 0xD4, 0xD6, 0xB2, 0xD3, /* 0x7C-0x7F */ + + 0xEC, 0xBE, 0x9E, 0xC5, 0x9E, 0xC6, 0x9E, 0xC7, /* 0x80-0x83 */ + 0x9E, 0xC8, 0xEA, 0xC1, 0x9E, 0xC9, 0x9E, 0xCA, /* 0x84-0x87 */ + 0x9E, 0xCB, 0xC2, 0xAF, 0xB4, 0xB6, 0x9E, 0xCC, /* 0x88-0x8B */ + 0x9E, 0xCD, 0x9E, 0xCE, 0xD1, 0xD7, 0x9E, 0xCF, /* 0x8C-0x8F */ + 0x9E, 0xD0, 0x9E, 0xD1, 0xB3, 0xB4, 0x9E, 0xD2, /* 0x90-0x93 */ + 0xC8, 0xB2, 0xBF, 0xBB, 0xEC, 0xC0, 0x9E, 0xD3, /* 0x94-0x97 */ + 0x9E, 0xD4, 0xD6, 0xCB, 0x9E, 0xD5, 0x9E, 0xD6, /* 0x98-0x9B */ + 0xEC, 0xBF, 0xEC, 0xC1, 0x9E, 0xD7, 0x9E, 0xD8, /* 0x9C-0x9F */ + 0x9E, 0xD9, 0x9E, 0xDA, 0x9E, 0xDB, 0x9E, 0xDC, /* 0xA0-0xA3 */ + 0x9E, 0xDD, 0x9E, 0xDE, 0x9E, 0xDF, 0x9E, 0xE0, /* 0xA4-0xA7 */ + 0x9E, 0xE1, 0x9E, 0xE2, 0x9E, 0xE3, 0xEC, 0xC5, /* 0xA8-0xAB */ + 0xBE, 0xE6, 0xCC, 0xBF, 0xC5, 0xDA, 0xBE, 0xBC, /* 0xAC-0xAF */ + 0x9E, 0xE4, 0xEC, 0xC6, 0x9E, 0xE5, 0xB1, 0xFE, /* 0xB0-0xB3 */ + 0x9E, 0xE6, 0x9E, 0xE7, 0x9E, 0xE8, 0xEC, 0xC4, /* 0xB4-0xB7 */ + 0xD5, 0xA8, 0xB5, 0xE3, 0x9E, 0xE9, 0xEC, 0xC2, /* 0xB8-0xBB */ + 0xC1, 0xB6, 0xB3, 0xE3, 0x9E, 0xEA, 0x9E, 0xEB, /* 0xBC-0xBF */ + 0xEC, 0xC3, 0xCB, 0xB8, 0xC0, 0xC3, 0xCC, 0xFE, /* 0xC0-0xC3 */ + 0x9E, 0xEC, 0x9E, 0xED, 0x9E, 0xEE, 0x9E, 0xEF, /* 0xC4-0xC7 */ + 0xC1, 0xD2, 0x9E, 0xF0, 0xEC, 0xC8, 0x9E, 0xF1, /* 0xC8-0xCB */ + 0x9E, 0xF2, 0x9E, 0xF3, 0x9E, 0xF4, 0x9E, 0xF5, /* 0xCC-0xCF */ + 0x9E, 0xF6, 0x9E, 0xF7, 0x9E, 0xF8, 0x9E, 0xF9, /* 0xD0-0xD3 */ + 0x9E, 0xFA, 0x9E, 0xFB, 0x9E, 0xFC, 0x9E, 0xFD, /* 0xD4-0xD7 */ + 0xBA, 0xE6, 0xC0, 0xD3, 0x9E, 0xFE, 0xD6, 0xF2, /* 0xD8-0xDB */ + 0x9F, 0x40, 0x9F, 0x41, 0x9F, 0x42, 0xD1, 0xCC, /* 0xDC-0xDF */ + 0x9F, 0x43, 0x9F, 0x44, 0x9F, 0x45, 0x9F, 0x46, /* 0xE0-0xE3 */ + 0xBF, 0xBE, 0x9F, 0x47, 0xB7, 0xB3, 0xC9, 0xD5, /* 0xE4-0xE7 */ + 0xEC, 0xC7, 0xBB, 0xE2, 0x9F, 0x48, 0xCC, 0xCC, /* 0xE8-0xEB */ + 0xBD, 0xFD, 0xC8, 0xC8, 0x9F, 0x49, 0xCF, 0xA9, /* 0xEC-0xEF */ + 0x9F, 0x4A, 0x9F, 0x4B, 0x9F, 0x4C, 0x9F, 0x4D, /* 0xF0-0xF3 */ + 0x9F, 0x4E, 0x9F, 0x4F, 0x9F, 0x50, 0xCD, 0xE9, /* 0xF4-0xF7 */ + 0x9F, 0x51, 0xC5, 0xEB, 0x9F, 0x52, 0x9F, 0x53, /* 0xF8-0xFB */ + 0x9F, 0x54, 0xB7, 0xE9, 0x9F, 0x55, 0x9F, 0x56, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_71[512] = { + 0x9F, 0x57, 0x9F, 0x58, 0x9F, 0x59, 0x9F, 0x5A, /* 0x00-0x03 */ + 0x9F, 0x5B, 0x9F, 0x5C, 0x9F, 0x5D, 0x9F, 0x5E, /* 0x04-0x07 */ + 0x9F, 0x5F, 0xD1, 0xC9, 0xBA, 0xB8, 0x9F, 0x60, /* 0x08-0x0B */ + 0x9F, 0x61, 0x9F, 0x62, 0x9F, 0x63, 0x9F, 0x64, /* 0x0C-0x0F */ + 0xEC, 0xC9, 0x9F, 0x65, 0x9F, 0x66, 0xEC, 0xCA, /* 0x10-0x13 */ + 0x9F, 0x67, 0xBB, 0xC0, 0xEC, 0xCB, 0x9F, 0x68, /* 0x14-0x17 */ + 0xEC, 0xE2, 0xB1, 0xBA, 0xB7, 0xD9, 0x9F, 0x69, /* 0x18-0x1B */ + 0x9F, 0x6A, 0x9F, 0x6B, 0x9F, 0x6C, 0x9F, 0x6D, /* 0x1C-0x1F */ + 0x9F, 0x6E, 0x9F, 0x6F, 0x9F, 0x70, 0x9F, 0x71, /* 0x20-0x23 */ + 0x9F, 0x72, 0x9F, 0x73, 0xBD, 0xB9, 0x9F, 0x74, /* 0x24-0x27 */ + 0x9F, 0x75, 0x9F, 0x76, 0x9F, 0x77, 0x9F, 0x78, /* 0x28-0x2B */ + 0x9F, 0x79, 0x9F, 0x7A, 0x9F, 0x7B, 0xEC, 0xCC, /* 0x2C-0x2F */ + 0xD1, 0xE6, 0xEC, 0xCD, 0x9F, 0x7C, 0x9F, 0x7D, /* 0x30-0x33 */ + 0x9F, 0x7E, 0x9F, 0x80, 0xC8, 0xBB, 0x9F, 0x81, /* 0x34-0x37 */ + 0x9F, 0x82, 0x9F, 0x83, 0x9F, 0x84, 0x9F, 0x85, /* 0x38-0x3B */ + 0x9F, 0x86, 0x9F, 0x87, 0x9F, 0x88, 0x9F, 0x89, /* 0x3C-0x3F */ + 0x9F, 0x8A, 0x9F, 0x8B, 0x9F, 0x8C, 0x9F, 0x8D, /* 0x40-0x43 */ + 0x9F, 0x8E, 0xEC, 0xD1, 0x9F, 0x8F, 0x9F, 0x90, /* 0x44-0x47 */ + 0x9F, 0x91, 0x9F, 0x92, 0xEC, 0xD3, 0x9F, 0x93, /* 0x48-0x4B */ + 0xBB, 0xCD, 0x9F, 0x94, 0xBC, 0xE5, 0x9F, 0x95, /* 0x4C-0x4F */ + 0x9F, 0x96, 0x9F, 0x97, 0x9F, 0x98, 0x9F, 0x99, /* 0x50-0x53 */ + 0x9F, 0x9A, 0x9F, 0x9B, 0x9F, 0x9C, 0x9F, 0x9D, /* 0x54-0x57 */ + 0x9F, 0x9E, 0x9F, 0x9F, 0x9F, 0xA0, 0x9F, 0xA1, /* 0x58-0x5B */ + 0xEC, 0xCF, 0x9F, 0xA2, 0xC9, 0xB7, 0x9F, 0xA3, /* 0x5C-0x5F */ + 0x9F, 0xA4, 0x9F, 0xA5, 0x9F, 0xA6, 0x9F, 0xA7, /* 0x60-0x63 */ + 0xC3, 0xBA, 0x9F, 0xA8, 0xEC, 0xE3, 0xD5, 0xD5, /* 0x64-0x67 */ + 0xEC, 0xD0, 0x9F, 0xA9, 0x9F, 0xAA, 0x9F, 0xAB, /* 0x68-0x6B */ + 0x9F, 0xAC, 0x9F, 0xAD, 0xD6, 0xF3, 0x9F, 0xAE, /* 0x6C-0x6F */ + 0x9F, 0xAF, 0x9F, 0xB0, 0xEC, 0xD2, 0xEC, 0xCE, /* 0x70-0x73 */ + 0x9F, 0xB1, 0x9F, 0xB2, 0x9F, 0xB3, 0x9F, 0xB4, /* 0x74-0x77 */ + 0xEC, 0xD4, 0x9F, 0xB5, 0xEC, 0xD5, 0x9F, 0xB6, /* 0x78-0x7B */ + 0x9F, 0xB7, 0xC9, 0xBF, 0x9F, 0xB8, 0x9F, 0xB9, /* 0x7C-0x7F */ + + 0x9F, 0xBA, 0x9F, 0xBB, 0x9F, 0xBC, 0x9F, 0xBD, /* 0x80-0x83 */ + 0xCF, 0xA8, 0x9F, 0xBE, 0x9F, 0xBF, 0x9F, 0xC0, /* 0x84-0x87 */ + 0x9F, 0xC1, 0x9F, 0xC2, 0xD0, 0xDC, 0x9F, 0xC3, /* 0x88-0x8B */ + 0x9F, 0xC4, 0x9F, 0xC5, 0x9F, 0xC6, 0xD1, 0xAC, /* 0x8C-0x8F */ + 0x9F, 0xC7, 0x9F, 0xC8, 0x9F, 0xC9, 0x9F, 0xCA, /* 0x90-0x93 */ + 0xC8, 0xDB, 0x9F, 0xCB, 0x9F, 0xCC, 0x9F, 0xCD, /* 0x94-0x97 */ + 0xEC, 0xD6, 0xCE, 0xF5, 0x9F, 0xCE, 0x9F, 0xCF, /* 0x98-0x9B */ + 0x9F, 0xD0, 0x9F, 0xD1, 0x9F, 0xD2, 0xCA, 0xEC, /* 0x9C-0x9F */ + 0xEC, 0xDA, 0x9F, 0xD3, 0x9F, 0xD4, 0x9F, 0xD5, /* 0xA0-0xA3 */ + 0x9F, 0xD6, 0x9F, 0xD7, 0x9F, 0xD8, 0x9F, 0xD9, /* 0xA4-0xA7 */ + 0xEC, 0xD9, 0x9F, 0xDA, 0x9F, 0xDB, 0x9F, 0xDC, /* 0xA8-0xAB */ + 0xB0, 0xBE, 0x9F, 0xDD, 0x9F, 0xDE, 0x9F, 0xDF, /* 0xAC-0xAF */ + 0x9F, 0xE0, 0x9F, 0xE1, 0x9F, 0xE2, 0xEC, 0xD7, /* 0xB0-0xB3 */ + 0x9F, 0xE3, 0xEC, 0xD8, 0x9F, 0xE4, 0x9F, 0xE5, /* 0xB4-0xB7 */ + 0x9F, 0xE6, 0xEC, 0xE4, 0x9F, 0xE7, 0x9F, 0xE8, /* 0xB8-0xBB */ + 0x9F, 0xE9, 0x9F, 0xEA, 0x9F, 0xEB, 0x9F, 0xEC, /* 0xBC-0xBF */ + 0x9F, 0xED, 0x9F, 0xEE, 0x9F, 0xEF, 0xC8, 0xBC, /* 0xC0-0xC3 */ + 0x9F, 0xF0, 0x9F, 0xF1, 0x9F, 0xF2, 0x9F, 0xF3, /* 0xC4-0xC7 */ + 0x9F, 0xF4, 0x9F, 0xF5, 0x9F, 0xF6, 0x9F, 0xF7, /* 0xC8-0xCB */ + 0x9F, 0xF8, 0x9F, 0xF9, 0xC1, 0xC7, 0x9F, 0xFA, /* 0xCC-0xCF */ + 0x9F, 0xFB, 0x9F, 0xFC, 0x9F, 0xFD, 0x9F, 0xFE, /* 0xD0-0xD3 */ + 0xEC, 0xDC, 0xD1, 0xE0, 0xA0, 0x40, 0xA0, 0x41, /* 0xD4-0xD7 */ + 0xA0, 0x42, 0xA0, 0x43, 0xA0, 0x44, 0xA0, 0x45, /* 0xD8-0xDB */ + 0xA0, 0x46, 0xA0, 0x47, 0xA0, 0x48, 0xA0, 0x49, /* 0xDC-0xDF */ + 0xEC, 0xDB, 0xA0, 0x4A, 0xA0, 0x4B, 0xA0, 0x4C, /* 0xE0-0xE3 */ + 0xA0, 0x4D, 0xD4, 0xEF, 0xA0, 0x4E, 0xEC, 0xDD, /* 0xE4-0xE7 */ + 0xA0, 0x4F, 0xA0, 0x50, 0xA0, 0x51, 0xA0, 0x52, /* 0xE8-0xEB */ + 0xA0, 0x53, 0xA0, 0x54, 0xDB, 0xC6, 0xA0, 0x55, /* 0xEC-0xEF */ + 0xA0, 0x56, 0xA0, 0x57, 0xA0, 0x58, 0xA0, 0x59, /* 0xF0-0xF3 */ + 0xA0, 0x5A, 0xA0, 0x5B, 0xA0, 0x5C, 0xA0, 0x5D, /* 0xF4-0xF7 */ + 0xA0, 0x5E, 0xEC, 0xDE, 0xA0, 0x5F, 0xA0, 0x60, /* 0xF8-0xFB */ + 0xA0, 0x61, 0xA0, 0x62, 0xA0, 0x63, 0xA0, 0x64, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_72[512] = { + 0xA0, 0x65, 0xA0, 0x66, 0xA0, 0x67, 0xA0, 0x68, /* 0x00-0x03 */ + 0xA0, 0x69, 0xA0, 0x6A, 0xB1, 0xAC, 0xA0, 0x6B, /* 0x04-0x07 */ + 0xA0, 0x6C, 0xA0, 0x6D, 0xA0, 0x6E, 0xA0, 0x6F, /* 0x08-0x0B */ + 0xA0, 0x70, 0xA0, 0x71, 0xA0, 0x72, 0xA0, 0x73, /* 0x0C-0x0F */ + 0xA0, 0x74, 0xA0, 0x75, 0xA0, 0x76, 0xA0, 0x77, /* 0x10-0x13 */ + 0xA0, 0x78, 0xA0, 0x79, 0xA0, 0x7A, 0xA0, 0x7B, /* 0x14-0x17 */ + 0xA0, 0x7C, 0xA0, 0x7D, 0xA0, 0x7E, 0xA0, 0x80, /* 0x18-0x1B */ + 0xA0, 0x81, 0xEC, 0xDF, 0xA0, 0x82, 0xA0, 0x83, /* 0x1C-0x1F */ + 0xA0, 0x84, 0xA0, 0x85, 0xA0, 0x86, 0xA0, 0x87, /* 0x20-0x23 */ + 0xA0, 0x88, 0xA0, 0x89, 0xA0, 0x8A, 0xA0, 0x8B, /* 0x24-0x27 */ + 0xEC, 0xE0, 0xA0, 0x8C, 0xD7, 0xA6, 0xA0, 0x8D, /* 0x28-0x2B */ + 0xC5, 0xC0, 0xA0, 0x8E, 0xA0, 0x8F, 0xA0, 0x90, /* 0x2C-0x2F */ + 0xEB, 0xBC, 0xB0, 0xAE, 0xA0, 0x91, 0xA0, 0x92, /* 0x30-0x33 */ + 0xA0, 0x93, 0xBE, 0xF4, 0xB8, 0xB8, 0xD2, 0xAF, /* 0x34-0x37 */ + 0xB0, 0xD6, 0xB5, 0xF9, 0xA0, 0x94, 0xD8, 0xB3, /* 0x38-0x3B */ + 0xA0, 0x95, 0xCB, 0xAC, 0xA0, 0x96, 0xE3, 0xDD, /* 0x3C-0x3F */ + 0xA0, 0x97, 0xA0, 0x98, 0xA0, 0x99, 0xA0, 0x9A, /* 0x40-0x43 */ + 0xA0, 0x9B, 0xA0, 0x9C, 0xA0, 0x9D, 0xC6, 0xAC, /* 0x44-0x47 */ + 0xB0, 0xE6, 0xA0, 0x9E, 0xA0, 0x9F, 0xA0, 0xA0, /* 0x48-0x4B */ + 0xC5, 0xC6, 0xEB, 0xB9, 0xA0, 0xA1, 0xA0, 0xA2, /* 0x4C-0x4F */ + 0xA0, 0xA3, 0xA0, 0xA4, 0xEB, 0xBA, 0xA0, 0xA5, /* 0x50-0x53 */ + 0xA0, 0xA6, 0xA0, 0xA7, 0xEB, 0xBB, 0xA0, 0xA8, /* 0x54-0x57 */ + 0xA0, 0xA9, 0xD1, 0xC0, 0xA0, 0xAA, 0xC5, 0xA3, /* 0x58-0x5B */ + 0xA0, 0xAB, 0xEA, 0xF2, 0xA0, 0xAC, 0xC4, 0xB2, /* 0x5C-0x5F */ + 0xA0, 0xAD, 0xC4, 0xB5, 0xC0, 0xCE, 0xA0, 0xAE, /* 0x60-0x63 */ + 0xA0, 0xAF, 0xA0, 0xB0, 0xEA, 0xF3, 0xC4, 0xC1, /* 0x64-0x67 */ + 0xA0, 0xB1, 0xCE, 0xEF, 0xA0, 0xB2, 0xA0, 0xB3, /* 0x68-0x6B */ + 0xA0, 0xB4, 0xA0, 0xB5, 0xEA, 0xF0, 0xEA, 0xF4, /* 0x6C-0x6F */ + 0xA0, 0xB6, 0xA0, 0xB7, 0xC9, 0xFC, 0xA0, 0xB8, /* 0x70-0x73 */ + 0xA0, 0xB9, 0xC7, 0xA3, 0xA0, 0xBA, 0xA0, 0xBB, /* 0x74-0x77 */ + 0xA0, 0xBC, 0xCC, 0xD8, 0xCE, 0xFE, 0xA0, 0xBD, /* 0x78-0x7B */ + 0xA0, 0xBE, 0xA0, 0xBF, 0xEA, 0xF5, 0xEA, 0xF6, /* 0x7C-0x7F */ + + 0xCF, 0xAC, 0xC0, 0xE7, 0xA0, 0xC0, 0xA0, 0xC1, /* 0x80-0x83 */ + 0xEA, 0xF7, 0xA0, 0xC2, 0xA0, 0xC3, 0xA0, 0xC4, /* 0x84-0x87 */ + 0xA0, 0xC5, 0xA0, 0xC6, 0xB6, 0xBF, 0xEA, 0xF8, /* 0x88-0x8B */ + 0xA0, 0xC7, 0xEA, 0xF9, 0xA0, 0xC8, 0xEA, 0xFA, /* 0x8C-0x8F */ + 0xA0, 0xC9, 0xA0, 0xCA, 0xEA, 0xFB, 0xA0, 0xCB, /* 0x90-0x93 */ + 0xA0, 0xCC, 0xA0, 0xCD, 0xA0, 0xCE, 0xA0, 0xCF, /* 0x94-0x97 */ + 0xA0, 0xD0, 0xA0, 0xD1, 0xA0, 0xD2, 0xA0, 0xD3, /* 0x98-0x9B */ + 0xA0, 0xD4, 0xA0, 0xD5, 0xA0, 0xD6, 0xEA, 0xF1, /* 0x9C-0x9F */ + 0xA0, 0xD7, 0xA0, 0xD8, 0xA0, 0xD9, 0xA0, 0xDA, /* 0xA0-0xA3 */ + 0xA0, 0xDB, 0xA0, 0xDC, 0xA0, 0xDD, 0xA0, 0xDE, /* 0xA4-0xA7 */ + 0xA0, 0xDF, 0xA0, 0xE0, 0xA0, 0xE1, 0xA0, 0xE2, /* 0xA8-0xAB */ + 0xC8, 0xAE, 0xE1, 0xEB, 0xA0, 0xE3, 0xB7, 0xB8, /* 0xAC-0xAF */ + 0xE1, 0xEC, 0xA0, 0xE4, 0xA0, 0xE5, 0xA0, 0xE6, /* 0xB0-0xB3 */ + 0xE1, 0xED, 0xA0, 0xE7, 0xD7, 0xB4, 0xE1, 0xEE, /* 0xB4-0xB7 */ + 0xE1, 0xEF, 0xD3, 0xCC, 0xA0, 0xE8, 0xA0, 0xE9, /* 0xB8-0xBB */ + 0xA0, 0xEA, 0xA0, 0xEB, 0xA0, 0xEC, 0xA0, 0xED, /* 0xBC-0xBF */ + 0xA0, 0xEE, 0xE1, 0xF1, 0xBF, 0xF1, 0xE1, 0xF0, /* 0xC0-0xC3 */ + 0xB5, 0xD2, 0xA0, 0xEF, 0xA0, 0xF0, 0xA0, 0xF1, /* 0xC4-0xC7 */ + 0xB1, 0xB7, 0xA0, 0xF2, 0xA0, 0xF3, 0xA0, 0xF4, /* 0xC8-0xCB */ + 0xA0, 0xF5, 0xE1, 0xF3, 0xE1, 0xF2, 0xA0, 0xF6, /* 0xCC-0xCF */ + 0xBA, 0xFC, 0xA0, 0xF7, 0xE1, 0xF4, 0xA0, 0xF8, /* 0xD0-0xD3 */ + 0xA0, 0xF9, 0xA0, 0xFA, 0xA0, 0xFB, 0xB9, 0xB7, /* 0xD4-0xD7 */ + 0xA0, 0xFC, 0xBE, 0xD1, 0xA0, 0xFD, 0xA0, 0xFE, /* 0xD8-0xDB */ + 0xAA, 0x40, 0xAA, 0x41, 0xC4, 0xFC, 0xAA, 0x42, /* 0xDC-0xDF */ + 0xBA, 0xDD, 0xBD, 0xC6, 0xAA, 0x43, 0xAA, 0x44, /* 0xE0-0xE3 */ + 0xAA, 0x45, 0xAA, 0x46, 0xAA, 0x47, 0xAA, 0x48, /* 0xE4-0xE7 */ + 0xE1, 0xF5, 0xE1, 0xF7, 0xAA, 0x49, 0xAA, 0x4A, /* 0xE8-0xEB */ + 0xB6, 0xC0, 0xCF, 0xC1, 0xCA, 0xA8, 0xE1, 0xF6, /* 0xEC-0xEF */ + 0xD5, 0xF8, 0xD3, 0xFC, 0xE1, 0xF8, 0xE1, 0xFC, /* 0xF0-0xF3 */ + 0xE1, 0xF9, 0xAA, 0x4B, 0xAA, 0x4C, 0xE1, 0xFA, /* 0xF4-0xF7 */ + 0xC0, 0xEA, 0xAA, 0x4D, 0xE1, 0xFE, 0xE2, 0xA1, /* 0xF8-0xFB */ + 0xC0, 0xC7, 0xAA, 0x4E, 0xAA, 0x4F, 0xAA, 0x50, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_73[512] = { + 0xAA, 0x51, 0xE1, 0xFB, 0xAA, 0x52, 0xE1, 0xFD, /* 0x00-0x03 */ + 0xAA, 0x53, 0xAA, 0x54, 0xAA, 0x55, 0xAA, 0x56, /* 0x04-0x07 */ + 0xAA, 0x57, 0xAA, 0x58, 0xE2, 0xA5, 0xAA, 0x59, /* 0x08-0x0B */ + 0xAA, 0x5A, 0xAA, 0x5B, 0xC1, 0xD4, 0xAA, 0x5C, /* 0x0C-0x0F */ + 0xAA, 0x5D, 0xAA, 0x5E, 0xAA, 0x5F, 0xE2, 0xA3, /* 0x10-0x13 */ + 0xAA, 0x60, 0xE2, 0xA8, 0xB2, 0xFE, 0xE2, 0xA2, /* 0x14-0x17 */ + 0xAA, 0x61, 0xAA, 0x62, 0xAA, 0x63, 0xC3, 0xCD, /* 0x18-0x1B */ + 0xB2, 0xC2, 0xE2, 0xA7, 0xE2, 0xA6, 0xAA, 0x64, /* 0x1C-0x1F */ + 0xAA, 0x65, 0xE2, 0xA4, 0xE2, 0xA9, 0xAA, 0x66, /* 0x20-0x23 */ + 0xAA, 0x67, 0xE2, 0xAB, 0xAA, 0x68, 0xAA, 0x69, /* 0x24-0x27 */ + 0xAA, 0x6A, 0xD0, 0xC9, 0xD6, 0xED, 0xC3, 0xA8, /* 0x28-0x2B */ + 0xE2, 0xAC, 0xAA, 0x6B, 0xCF, 0xD7, 0xAA, 0x6C, /* 0x2C-0x2F */ + 0xAA, 0x6D, 0xE2, 0xAE, 0xAA, 0x6E, 0xAA, 0x6F, /* 0x30-0x33 */ + 0xBA, 0xEF, 0xAA, 0x70, 0xAA, 0x71, 0xE9, 0xE0, /* 0x34-0x37 */ + 0xE2, 0xAD, 0xE2, 0xAA, 0xAA, 0x72, 0xAA, 0x73, /* 0x38-0x3B */ + 0xAA, 0x74, 0xAA, 0x75, 0xBB, 0xAB, 0xD4, 0xB3, /* 0x3C-0x3F */ + 0xAA, 0x76, 0xAA, 0x77, 0xAA, 0x78, 0xAA, 0x79, /* 0x40-0x43 */ + 0xAA, 0x7A, 0xAA, 0x7B, 0xAA, 0x7C, 0xAA, 0x7D, /* 0x44-0x47 */ + 0xAA, 0x7E, 0xAA, 0x80, 0xAA, 0x81, 0xAA, 0x82, /* 0x48-0x4B */ + 0xAA, 0x83, 0xE2, 0xB0, 0xAA, 0x84, 0xAA, 0x85, /* 0x4C-0x4F */ + 0xE2, 0xAF, 0xAA, 0x86, 0xE9, 0xE1, 0xAA, 0x87, /* 0x50-0x53 */ + 0xAA, 0x88, 0xAA, 0x89, 0xAA, 0x8A, 0xE2, 0xB1, /* 0x54-0x57 */ + 0xAA, 0x8B, 0xAA, 0x8C, 0xAA, 0x8D, 0xAA, 0x8E, /* 0x58-0x5B */ + 0xAA, 0x8F, 0xAA, 0x90, 0xAA, 0x91, 0xAA, 0x92, /* 0x5C-0x5F */ + 0xE2, 0xB2, 0xAA, 0x93, 0xAA, 0x94, 0xAA, 0x95, /* 0x60-0x63 */ + 0xAA, 0x96, 0xAA, 0x97, 0xAA, 0x98, 0xAA, 0x99, /* 0x64-0x67 */ + 0xAA, 0x9A, 0xAA, 0x9B, 0xAA, 0x9C, 0xAA, 0x9D, /* 0x68-0x6B */ + 0xE2, 0xB3, 0xCC, 0xA1, 0xAA, 0x9E, 0xE2, 0xB4, /* 0x6C-0x6F */ + 0xAA, 0x9F, 0xAA, 0xA0, 0xAB, 0x40, 0xAB, 0x41, /* 0x70-0x73 */ + 0xAB, 0x42, 0xAB, 0x43, 0xAB, 0x44, 0xAB, 0x45, /* 0x74-0x77 */ + 0xAB, 0x46, 0xAB, 0x47, 0xAB, 0x48, 0xAB, 0x49, /* 0x78-0x7B */ + 0xAB, 0x4A, 0xAB, 0x4B, 0xE2, 0xB5, 0xAB, 0x4C, /* 0x7C-0x7F */ + + 0xAB, 0x4D, 0xAB, 0x4E, 0xAB, 0x4F, 0xAB, 0x50, /* 0x80-0x83 */ + 0xD0, 0xFE, 0xAB, 0x51, 0xAB, 0x52, 0xC2, 0xCA, /* 0x84-0x87 */ + 0xAB, 0x53, 0xD3, 0xF1, 0xAB, 0x54, 0xCD, 0xF5, /* 0x88-0x8B */ + 0xAB, 0x55, 0xAB, 0x56, 0xE7, 0xE0, 0xAB, 0x57, /* 0x8C-0x8F */ + 0xAB, 0x58, 0xE7, 0xE1, 0xAB, 0x59, 0xAB, 0x5A, /* 0x90-0x93 */ + 0xAB, 0x5B, 0xAB, 0x5C, 0xBE, 0xC1, 0xAB, 0x5D, /* 0x94-0x97 */ + 0xAB, 0x5E, 0xAB, 0x5F, 0xAB, 0x60, 0xC2, 0xEA, /* 0x98-0x9B */ + 0xAB, 0x61, 0xAB, 0x62, 0xAB, 0x63, 0xE7, 0xE4, /* 0x9C-0x9F */ + 0xAB, 0x64, 0xAB, 0x65, 0xE7, 0xE3, 0xAB, 0x66, /* 0xA0-0xA3 */ + 0xAB, 0x67, 0xAB, 0x68, 0xAB, 0x69, 0xAB, 0x6A, /* 0xA4-0xA7 */ + 0xAB, 0x6B, 0xCD, 0xE6, 0xAB, 0x6C, 0xC3, 0xB5, /* 0xA8-0xAB */ + 0xAB, 0x6D, 0xAB, 0x6E, 0xE7, 0xE2, 0xBB, 0xB7, /* 0xAC-0xAF */ + 0xCF, 0xD6, 0xAB, 0x6F, 0xC1, 0xE1, 0xE7, 0xE9, /* 0xB0-0xB3 */ + 0xAB, 0x70, 0xAB, 0x71, 0xAB, 0x72, 0xE7, 0xE8, /* 0xB4-0xB7 */ + 0xAB, 0x73, 0xAB, 0x74, 0xE7, 0xF4, 0xB2, 0xA3, /* 0xB8-0xBB */ + 0xAB, 0x75, 0xAB, 0x76, 0xAB, 0x77, 0xAB, 0x78, /* 0xBC-0xBF */ + 0xE7, 0xEA, 0xAB, 0x79, 0xE7, 0xE6, 0xAB, 0x7A, /* 0xC0-0xC3 */ + 0xAB, 0x7B, 0xAB, 0x7C, 0xAB, 0x7D, 0xAB, 0x7E, /* 0xC4-0xC7 */ + 0xE7, 0xEC, 0xE7, 0xEB, 0xC9, 0xBA, 0xAB, 0x80, /* 0xC8-0xCB */ + 0xAB, 0x81, 0xD5, 0xE4, 0xAB, 0x82, 0xE7, 0xE5, /* 0xCC-0xCF */ + 0xB7, 0xA9, 0xE7, 0xE7, 0xAB, 0x83, 0xAB, 0x84, /* 0xD0-0xD3 */ + 0xAB, 0x85, 0xAB, 0x86, 0xAB, 0x87, 0xAB, 0x88, /* 0xD4-0xD7 */ + 0xAB, 0x89, 0xE7, 0xEE, 0xAB, 0x8A, 0xAB, 0x8B, /* 0xD8-0xDB */ + 0xAB, 0x8C, 0xAB, 0x8D, 0xE7, 0xF3, 0xAB, 0x8E, /* 0xDC-0xDF */ + 0xD6, 0xE9, 0xAB, 0x8F, 0xAB, 0x90, 0xAB, 0x91, /* 0xE0-0xE3 */ + 0xAB, 0x92, 0xE7, 0xED, 0xAB, 0x93, 0xE7, 0xF2, /* 0xE4-0xE7 */ + 0xAB, 0x94, 0xE7, 0xF1, 0xAB, 0x95, 0xAB, 0x96, /* 0xE8-0xEB */ + 0xAB, 0x97, 0xB0, 0xE0, 0xAB, 0x98, 0xAB, 0x99, /* 0xEC-0xEF */ + 0xAB, 0x9A, 0xAB, 0x9B, 0xE7, 0xF5, 0xAB, 0x9C, /* 0xF0-0xF3 */ + 0xAB, 0x9D, 0xAB, 0x9E, 0xAB, 0x9F, 0xAB, 0xA0, /* 0xF4-0xF7 */ + 0xAC, 0x40, 0xAC, 0x41, 0xAC, 0x42, 0xAC, 0x43, /* 0xF8-0xFB */ + 0xAC, 0x44, 0xAC, 0x45, 0xAC, 0x46, 0xAC, 0x47, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_74[512] = { + 0xAC, 0x48, 0xAC, 0x49, 0xAC, 0x4A, 0xC7, 0xF2, /* 0x00-0x03 */ + 0xAC, 0x4B, 0xC0, 0xC5, 0xC0, 0xED, 0xAC, 0x4C, /* 0x04-0x07 */ + 0xAC, 0x4D, 0xC1, 0xF0, 0xE7, 0xF0, 0xAC, 0x4E, /* 0x08-0x0B */ + 0xAC, 0x4F, 0xAC, 0x50, 0xAC, 0x51, 0xE7, 0xF6, /* 0x0C-0x0F */ + 0xCB, 0xF6, 0xAC, 0x52, 0xAC, 0x53, 0xAC, 0x54, /* 0x10-0x13 */ + 0xAC, 0x55, 0xAC, 0x56, 0xAC, 0x57, 0xAC, 0x58, /* 0x14-0x17 */ + 0xAC, 0x59, 0xAC, 0x5A, 0xE8, 0xA2, 0xE8, 0xA1, /* 0x18-0x1B */ + 0xAC, 0x5B, 0xAC, 0x5C, 0xAC, 0x5D, 0xAC, 0x5E, /* 0x1C-0x1F */ + 0xAC, 0x5F, 0xAC, 0x60, 0xD7, 0xC1, 0xAC, 0x61, /* 0x20-0x23 */ + 0xAC, 0x62, 0xE7, 0xFA, 0xE7, 0xF9, 0xAC, 0x63, /* 0x24-0x27 */ + 0xE7, 0xFB, 0xAC, 0x64, 0xE7, 0xF7, 0xAC, 0x65, /* 0x28-0x2B */ + 0xE7, 0xFE, 0xAC, 0x66, 0xE7, 0xFD, 0xAC, 0x67, /* 0x2C-0x2F */ + 0xE7, 0xFC, 0xAC, 0x68, 0xAC, 0x69, 0xC1, 0xD5, /* 0x30-0x33 */ + 0xC7, 0xD9, 0xC5, 0xFD, 0xC5, 0xC3, 0xAC, 0x6A, /* 0x34-0x37 */ + 0xAC, 0x6B, 0xAC, 0x6C, 0xAC, 0x6D, 0xAC, 0x6E, /* 0x38-0x3B */ + 0xC7, 0xED, 0xAC, 0x6F, 0xAC, 0x70, 0xAC, 0x71, /* 0x3C-0x3F */ + 0xAC, 0x72, 0xE8, 0xA3, 0xAC, 0x73, 0xAC, 0x74, /* 0x40-0x43 */ + 0xAC, 0x75, 0xAC, 0x76, 0xAC, 0x77, 0xAC, 0x78, /* 0x44-0x47 */ + 0xAC, 0x79, 0xAC, 0x7A, 0xAC, 0x7B, 0xAC, 0x7C, /* 0x48-0x4B */ + 0xAC, 0x7D, 0xAC, 0x7E, 0xAC, 0x80, 0xAC, 0x81, /* 0x4C-0x4F */ + 0xAC, 0x82, 0xAC, 0x83, 0xAC, 0x84, 0xAC, 0x85, /* 0x50-0x53 */ + 0xAC, 0x86, 0xE8, 0xA6, 0xAC, 0x87, 0xE8, 0xA5, /* 0x54-0x57 */ + 0xAC, 0x88, 0xE8, 0xA7, 0xBA, 0xF7, 0xE7, 0xF8, /* 0x58-0x5B */ + 0xE8, 0xA4, 0xAC, 0x89, 0xC8, 0xF0, 0xC9, 0xAA, /* 0x5C-0x5F */ + 0xAC, 0x8A, 0xAC, 0x8B, 0xAC, 0x8C, 0xAC, 0x8D, /* 0x60-0x63 */ + 0xAC, 0x8E, 0xAC, 0x8F, 0xAC, 0x90, 0xAC, 0x91, /* 0x64-0x67 */ + 0xAC, 0x92, 0xAC, 0x93, 0xAC, 0x94, 0xAC, 0x95, /* 0x68-0x6B */ + 0xAC, 0x96, 0xE8, 0xA9, 0xAC, 0x97, 0xAC, 0x98, /* 0x6C-0x6F */ + 0xB9, 0xE5, 0xAC, 0x99, 0xAC, 0x9A, 0xAC, 0x9B, /* 0x70-0x73 */ + 0xAC, 0x9C, 0xAC, 0x9D, 0xD1, 0xFE, 0xE8, 0xA8, /* 0x74-0x77 */ + 0xAC, 0x9E, 0xAC, 0x9F, 0xAC, 0xA0, 0xAD, 0x40, /* 0x78-0x7B */ + 0xAD, 0x41, 0xAD, 0x42, 0xE8, 0xAA, 0xAD, 0x43, /* 0x7C-0x7F */ + + 0xE8, 0xAD, 0xE8, 0xAE, 0xAD, 0x44, 0xC1, 0xA7, /* 0x80-0x83 */ + 0xAD, 0x45, 0xAD, 0x46, 0xAD, 0x47, 0xE8, 0xAF, /* 0x84-0x87 */ + 0xAD, 0x48, 0xAD, 0x49, 0xAD, 0x4A, 0xE8, 0xB0, /* 0x88-0x8B */ + 0xAD, 0x4B, 0xAD, 0x4C, 0xE8, 0xAC, 0xAD, 0x4D, /* 0x8C-0x8F */ + 0xE8, 0xB4, 0xAD, 0x4E, 0xAD, 0x4F, 0xAD, 0x50, /* 0x90-0x93 */ + 0xAD, 0x51, 0xAD, 0x52, 0xAD, 0x53, 0xAD, 0x54, /* 0x94-0x97 */ + 0xAD, 0x55, 0xAD, 0x56, 0xAD, 0x57, 0xAD, 0x58, /* 0x98-0x9B */ + 0xE8, 0xAB, 0xAD, 0x59, 0xE8, 0xB1, 0xAD, 0x5A, /* 0x9C-0x9F */ + 0xAD, 0x5B, 0xAD, 0x5C, 0xAD, 0x5D, 0xAD, 0x5E, /* 0xA0-0xA3 */ + 0xAD, 0x5F, 0xAD, 0x60, 0xAD, 0x61, 0xE8, 0xB5, /* 0xA4-0xA7 */ + 0xE8, 0xB2, 0xE8, 0xB3, 0xAD, 0x62, 0xAD, 0x63, /* 0xA8-0xAB */ + 0xAD, 0x64, 0xAD, 0x65, 0xAD, 0x66, 0xAD, 0x67, /* 0xAC-0xAF */ + 0xAD, 0x68, 0xAD, 0x69, 0xAD, 0x6A, 0xAD, 0x6B, /* 0xB0-0xB3 */ + 0xAD, 0x6C, 0xAD, 0x6D, 0xAD, 0x6E, 0xAD, 0x6F, /* 0xB4-0xB7 */ + 0xAD, 0x70, 0xAD, 0x71, 0xE8, 0xB7, 0xAD, 0x72, /* 0xB8-0xBB */ + 0xAD, 0x73, 0xAD, 0x74, 0xAD, 0x75, 0xAD, 0x76, /* 0xBC-0xBF */ + 0xAD, 0x77, 0xAD, 0x78, 0xAD, 0x79, 0xAD, 0x7A, /* 0xC0-0xC3 */ + 0xAD, 0x7B, 0xAD, 0x7C, 0xAD, 0x7D, 0xAD, 0x7E, /* 0xC4-0xC7 */ + 0xAD, 0x80, 0xAD, 0x81, 0xAD, 0x82, 0xAD, 0x83, /* 0xC8-0xCB */ + 0xAD, 0x84, 0xAD, 0x85, 0xAD, 0x86, 0xAD, 0x87, /* 0xCC-0xCF */ + 0xAD, 0x88, 0xAD, 0x89, 0xE8, 0xB6, 0xAD, 0x8A, /* 0xD0-0xD3 */ + 0xAD, 0x8B, 0xAD, 0x8C, 0xAD, 0x8D, 0xAD, 0x8E, /* 0xD4-0xD7 */ + 0xAD, 0x8F, 0xAD, 0x90, 0xAD, 0x91, 0xAD, 0x92, /* 0xD8-0xDB */ + 0xB9, 0xCF, 0xAD, 0x93, 0xF0, 0xAC, 0xAD, 0x94, /* 0xDC-0xDF */ + 0xF0, 0xAD, 0xAD, 0x95, 0xC6, 0xB0, 0xB0, 0xEA, /* 0xE0-0xE3 */ + 0xC8, 0xBF, 0xAD, 0x96, 0xCD, 0xDF, 0xAD, 0x97, /* 0xE4-0xE7 */ + 0xAD, 0x98, 0xAD, 0x99, 0xAD, 0x9A, 0xAD, 0x9B, /* 0xE8-0xEB */ + 0xAD, 0x9C, 0xAD, 0x9D, 0xCE, 0xCD, 0xEA, 0xB1, /* 0xEC-0xEF */ + 0xAD, 0x9E, 0xAD, 0x9F, 0xAD, 0xA0, 0xAE, 0x40, /* 0xF0-0xF3 */ + 0xEA, 0xB2, 0xAE, 0x41, 0xC6, 0xBF, 0xB4, 0xC9, /* 0xF4-0xF7 */ + 0xAE, 0x42, 0xAE, 0x43, 0xAE, 0x44, 0xAE, 0x45, /* 0xF8-0xFB */ + 0xAE, 0x46, 0xAE, 0x47, 0xAE, 0x48, 0xEA, 0xB3, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_75[512] = { + 0xAE, 0x49, 0xAE, 0x4A, 0xAE, 0x4B, 0xAE, 0x4C, /* 0x00-0x03 */ + 0xD5, 0xE7, 0xAE, 0x4D, 0xAE, 0x4E, 0xAE, 0x4F, /* 0x04-0x07 */ + 0xAE, 0x50, 0xAE, 0x51, 0xAE, 0x52, 0xAE, 0x53, /* 0x08-0x0B */ + 0xAE, 0x54, 0xDD, 0xF9, 0xAE, 0x55, 0xEA, 0xB4, /* 0x0C-0x0F */ + 0xAE, 0x56, 0xEA, 0xB5, 0xAE, 0x57, 0xEA, 0xB6, /* 0x10-0x13 */ + 0xAE, 0x58, 0xAE, 0x59, 0xAE, 0x5A, 0xAE, 0x5B, /* 0x14-0x17 */ + 0xB8, 0xCA, 0xDF, 0xB0, 0xC9, 0xF5, 0xAE, 0x5C, /* 0x18-0x1B */ + 0xCC, 0xF0, 0xAE, 0x5D, 0xAE, 0x5E, 0xC9, 0xFA, /* 0x1C-0x1F */ + 0xAE, 0x5F, 0xAE, 0x60, 0xAE, 0x61, 0xAE, 0x62, /* 0x20-0x23 */ + 0xAE, 0x63, 0xC9, 0xFB, 0xAE, 0x64, 0xAE, 0x65, /* 0x24-0x27 */ + 0xD3, 0xC3, 0xCB, 0xA6, 0xAE, 0x66, 0xB8, 0xA6, /* 0x28-0x2B */ + 0xF0, 0xAE, 0xB1, 0xC2, 0xAE, 0x67, 0xE5, 0xB8, /* 0x2C-0x2F */ + 0xCC, 0xEF, 0xD3, 0xC9, 0xBC, 0xD7, 0xC9, 0xEA, /* 0x30-0x33 */ + 0xAE, 0x68, 0xB5, 0xE7, 0xAE, 0x69, 0xC4, 0xD0, /* 0x34-0x37 */ + 0xB5, 0xE9, 0xAE, 0x6A, 0xEE, 0xAE, 0xBB, 0xAD, /* 0x38-0x3B */ + 0xAE, 0x6B, 0xAE, 0x6C, 0xE7, 0xDE, 0xAE, 0x6D, /* 0x3C-0x3F */ + 0xEE, 0xAF, 0xAE, 0x6E, 0xAE, 0x6F, 0xAE, 0x70, /* 0x40-0x43 */ + 0xAE, 0x71, 0xB3, 0xA9, 0xAE, 0x72, 0xAE, 0x73, /* 0x44-0x47 */ + 0xEE, 0xB2, 0xAE, 0x74, 0xAE, 0x75, 0xEE, 0xB1, /* 0x48-0x4B */ + 0xBD, 0xE7, 0xAE, 0x76, 0xEE, 0xB0, 0xCE, 0xB7, /* 0x4C-0x4F */ + 0xAE, 0x77, 0xAE, 0x78, 0xAE, 0x79, 0xAE, 0x7A, /* 0x50-0x53 */ + 0xC5, 0xCF, 0xAE, 0x7B, 0xAE, 0x7C, 0xAE, 0x7D, /* 0x54-0x57 */ + 0xAE, 0x7E, 0xC1, 0xF4, 0xDB, 0xCE, 0xEE, 0xB3, /* 0x58-0x5B */ + 0xD0, 0xF3, 0xAE, 0x80, 0xAE, 0x81, 0xAE, 0x82, /* 0x5C-0x5F */ + 0xAE, 0x83, 0xAE, 0x84, 0xAE, 0x85, 0xAE, 0x86, /* 0x60-0x63 */ + 0xAE, 0x87, 0xC2, 0xD4, 0xC6, 0xE8, 0xAE, 0x88, /* 0x64-0x67 */ + 0xAE, 0x89, 0xAE, 0x8A, 0xB7, 0xAC, 0xAE, 0x8B, /* 0x68-0x6B */ + 0xAE, 0x8C, 0xAE, 0x8D, 0xAE, 0x8E, 0xAE, 0x8F, /* 0x6C-0x6F */ + 0xAE, 0x90, 0xAE, 0x91, 0xEE, 0xB4, 0xAE, 0x92, /* 0x70-0x73 */ + 0xB3, 0xEB, 0xAE, 0x93, 0xAE, 0x94, 0xAE, 0x95, /* 0x74-0x77 */ + 0xBB, 0xFB, 0xEE, 0xB5, 0xAE, 0x96, 0xAE, 0x97, /* 0x78-0x7B */ + 0xAE, 0x98, 0xAE, 0x99, 0xAE, 0x9A, 0xE7, 0xDC, /* 0x7C-0x7F */ + + 0xAE, 0x9B, 0xAE, 0x9C, 0xAE, 0x9D, 0xEE, 0xB6, /* 0x80-0x83 */ + 0xAE, 0x9E, 0xAE, 0x9F, 0xBD, 0xAE, 0xAE, 0xA0, /* 0x84-0x87 */ + 0xAF, 0x40, 0xAF, 0x41, 0xAF, 0x42, 0xF1, 0xE2, /* 0x88-0x8B */ + 0xAF, 0x43, 0xAF, 0x44, 0xAF, 0x45, 0xCA, 0xE8, /* 0x8C-0x8F */ + 0xAF, 0x46, 0xD2, 0xC9, 0xF0, 0xDA, 0xAF, 0x47, /* 0x90-0x93 */ + 0xF0, 0xDB, 0xAF, 0x48, 0xF0, 0xDC, 0xC1, 0xC6, /* 0x94-0x97 */ + 0xAF, 0x49, 0xB8, 0xED, 0xBE, 0xCE, 0xAF, 0x4A, /* 0x98-0x9B */ + 0xAF, 0x4B, 0xF0, 0xDE, 0xAF, 0x4C, 0xC5, 0xB1, /* 0x9C-0x9F */ + 0xF0, 0xDD, 0xD1, 0xF1, 0xAF, 0x4D, 0xF0, 0xE0, /* 0xA0-0xA3 */ + 0xB0, 0xCC, 0xBD, 0xEA, 0xAF, 0x4E, 0xAF, 0x4F, /* 0xA4-0xA7 */ + 0xAF, 0x50, 0xAF, 0x51, 0xAF, 0x52, 0xD2, 0xDF, /* 0xA8-0xAB */ + 0xF0, 0xDF, 0xAF, 0x53, 0xB4, 0xAF, 0xB7, 0xE8, /* 0xAC-0xAF */ + 0xF0, 0xE6, 0xF0, 0xE5, 0xC6, 0xA3, 0xF0, 0xE1, /* 0xB0-0xB3 */ + 0xF0, 0xE2, 0xB4, 0xC3, 0xAF, 0x54, 0xAF, 0x55, /* 0xB4-0xB7 */ + 0xF0, 0xE3, 0xD5, 0xEE, 0xAF, 0x56, 0xAF, 0x57, /* 0xB8-0xBB */ + 0xCC, 0xDB, 0xBE, 0xD2, 0xBC, 0xB2, 0xAF, 0x58, /* 0xBC-0xBF */ + 0xAF, 0x59, 0xAF, 0x5A, 0xF0, 0xE8, 0xF0, 0xE7, /* 0xC0-0xC3 */ + 0xF0, 0xE4, 0xB2, 0xA1, 0xAF, 0x5B, 0xD6, 0xA2, /* 0xC4-0xC7 */ + 0xD3, 0xB8, 0xBE, 0xB7, 0xC8, 0xAC, 0xAF, 0x5C, /* 0xC8-0xCB */ + 0xAF, 0x5D, 0xF0, 0xEA, 0xAF, 0x5E, 0xAF, 0x5F, /* 0xCC-0xCF */ + 0xAF, 0x60, 0xAF, 0x61, 0xD1, 0xF7, 0xAF, 0x62, /* 0xD0-0xD3 */ + 0xD6, 0xCC, 0xBA, 0xDB, 0xF0, 0xE9, 0xAF, 0x63, /* 0xD4-0xD7 */ + 0xB6, 0xBB, 0xAF, 0x64, 0xAF, 0x65, 0xCD, 0xB4, /* 0xD8-0xDB */ + 0xAF, 0x66, 0xAF, 0x67, 0xC6, 0xA6, 0xAF, 0x68, /* 0xDC-0xDF */ + 0xAF, 0x69, 0xAF, 0x6A, 0xC1, 0xA1, 0xF0, 0xEB, /* 0xE0-0xE3 */ + 0xF0, 0xEE, 0xAF, 0x6B, 0xF0, 0xED, 0xF0, 0xF0, /* 0xE4-0xE7 */ + 0xF0, 0xEC, 0xAF, 0x6C, 0xBB, 0xBE, 0xF0, 0xEF, /* 0xE8-0xEB */ + 0xAF, 0x6D, 0xAF, 0x6E, 0xAF, 0x6F, 0xAF, 0x70, /* 0xEC-0xEF */ + 0xCC, 0xB5, 0xF0, 0xF2, 0xAF, 0x71, 0xAF, 0x72, /* 0xF0-0xF3 */ + 0xB3, 0xD5, 0xAF, 0x73, 0xAF, 0x74, 0xAF, 0x75, /* 0xF4-0xF7 */ + 0xAF, 0x76, 0xB1, 0xD4, 0xAF, 0x77, 0xAF, 0x78, /* 0xF8-0xFB */ + 0xF0, 0xF3, 0xAF, 0x79, 0xAF, 0x7A, 0xF0, 0xF4, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_76[512] = { + 0xF0, 0xF6, 0xB4, 0xE1, 0xAF, 0x7B, 0xF0, 0xF1, /* 0x00-0x03 */ + 0xAF, 0x7C, 0xF0, 0xF7, 0xAF, 0x7D, 0xAF, 0x7E, /* 0x04-0x07 */ + 0xAF, 0x80, 0xAF, 0x81, 0xF0, 0xFA, 0xAF, 0x82, /* 0x08-0x0B */ + 0xF0, 0xF8, 0xAF, 0x83, 0xAF, 0x84, 0xAF, 0x85, /* 0x0C-0x0F */ + 0xF0, 0xF5, 0xAF, 0x86, 0xAF, 0x87, 0xAF, 0x88, /* 0x10-0x13 */ + 0xAF, 0x89, 0xF0, 0xFD, 0xAF, 0x8A, 0xF0, 0xF9, /* 0x14-0x17 */ + 0xF0, 0xFC, 0xF0, 0xFE, 0xAF, 0x8B, 0xF1, 0xA1, /* 0x18-0x1B */ + 0xAF, 0x8C, 0xAF, 0x8D, 0xAF, 0x8E, 0xCE, 0xC1, /* 0x1C-0x1F */ + 0xF1, 0xA4, 0xAF, 0x8F, 0xF1, 0xA3, 0xAF, 0x90, /* 0x20-0x23 */ + 0xC1, 0xF6, 0xF0, 0xFB, 0xCA, 0xDD, 0xAF, 0x91, /* 0x24-0x27 */ + 0xAF, 0x92, 0xB4, 0xF1, 0xB1, 0xF1, 0xCC, 0xB1, /* 0x28-0x2B */ + 0xAF, 0x93, 0xF1, 0xA6, 0xAF, 0x94, 0xAF, 0x95, /* 0x2C-0x2F */ + 0xF1, 0xA7, 0xAF, 0x96, 0xAF, 0x97, 0xF1, 0xAC, /* 0x30-0x33 */ + 0xD5, 0xCE, 0xF1, 0xA9, 0xAF, 0x98, 0xAF, 0x99, /* 0x34-0x37 */ + 0xC8, 0xB3, 0xAF, 0x9A, 0xAF, 0x9B, 0xAF, 0x9C, /* 0x38-0x3B */ + 0xF1, 0xA2, 0xAF, 0x9D, 0xF1, 0xAB, 0xF1, 0xA8, /* 0x3C-0x3F */ + 0xF1, 0xA5, 0xAF, 0x9E, 0xAF, 0x9F, 0xF1, 0xAA, /* 0x40-0x43 */ + 0xAF, 0xA0, 0xB0, 0x40, 0xB0, 0x41, 0xB0, 0x42, /* 0x44-0x47 */ + 0xB0, 0x43, 0xB0, 0x44, 0xB0, 0x45, 0xB0, 0x46, /* 0x48-0x4B */ + 0xB0, 0xA9, 0xF1, 0xAD, 0xB0, 0x47, 0xB0, 0x48, /* 0x4C-0x4F */ + 0xB0, 0x49, 0xB0, 0x4A, 0xB0, 0x4B, 0xB0, 0x4C, /* 0x50-0x53 */ + 0xF1, 0xAF, 0xB0, 0x4D, 0xF1, 0xB1, 0xB0, 0x4E, /* 0x54-0x57 */ + 0xB0, 0x4F, 0xB0, 0x50, 0xB0, 0x51, 0xB0, 0x52, /* 0x58-0x5B */ + 0xF1, 0xB0, 0xB0, 0x53, 0xF1, 0xAE, 0xB0, 0x54, /* 0x5C-0x5F */ + 0xB0, 0x55, 0xB0, 0x56, 0xB0, 0x57, 0xD1, 0xA2, /* 0x60-0x63 */ + 0xB0, 0x58, 0xB0, 0x59, 0xB0, 0x5A, 0xB0, 0x5B, /* 0x64-0x67 */ + 0xB0, 0x5C, 0xB0, 0x5D, 0xB0, 0x5E, 0xF1, 0xB2, /* 0x68-0x6B */ + 0xB0, 0x5F, 0xB0, 0x60, 0xB0, 0x61, 0xF1, 0xB3, /* 0x6C-0x6F */ + 0xB0, 0x62, 0xB0, 0x63, 0xB0, 0x64, 0xB0, 0x65, /* 0x70-0x73 */ + 0xB0, 0x66, 0xB0, 0x67, 0xB0, 0x68, 0xB0, 0x69, /* 0x74-0x77 */ + 0xB9, 0xEF, 0xB0, 0x6A, 0xB0, 0x6B, 0xB5, 0xC7, /* 0x78-0x7B */ + 0xB0, 0x6C, 0xB0, 0xD7, 0xB0, 0xD9, 0xB0, 0x6D, /* 0x7C-0x7F */ + + 0xB0, 0x6E, 0xB0, 0x6F, 0xD4, 0xED, 0xB0, 0x70, /* 0x80-0x83 */ + 0xB5, 0xC4, 0xB0, 0x71, 0xBD, 0xD4, 0xBB, 0xCA, /* 0x84-0x87 */ + 0xF0, 0xA7, 0xB0, 0x72, 0xB0, 0x73, 0xB8, 0xDE, /* 0x88-0x8B */ + 0xB0, 0x74, 0xB0, 0x75, 0xF0, 0xA8, 0xB0, 0x76, /* 0x8C-0x8F */ + 0xB0, 0x77, 0xB0, 0xA8, 0xB0, 0x78, 0xF0, 0xA9, /* 0x90-0x93 */ + 0xB0, 0x79, 0xB0, 0x7A, 0xCD, 0xEE, 0xB0, 0x7B, /* 0x94-0x97 */ + 0xB0, 0x7C, 0xF0, 0xAA, 0xB0, 0x7D, 0xB0, 0x7E, /* 0x98-0x9B */ + 0xB0, 0x80, 0xB0, 0x81, 0xB0, 0x82, 0xB0, 0x83, /* 0x9C-0x9F */ + 0xB0, 0x84, 0xB0, 0x85, 0xB0, 0x86, 0xB0, 0x87, /* 0xA0-0xA3 */ + 0xF0, 0xAB, 0xB0, 0x88, 0xB0, 0x89, 0xB0, 0x8A, /* 0xA4-0xA7 */ + 0xB0, 0x8B, 0xB0, 0x8C, 0xB0, 0x8D, 0xB0, 0x8E, /* 0xA8-0xAB */ + 0xB0, 0x8F, 0xB0, 0x90, 0xC6, 0xA4, 0xB0, 0x91, /* 0xAC-0xAF */ + 0xB0, 0x92, 0xD6, 0xE5, 0xF1, 0xE4, 0xB0, 0x93, /* 0xB0-0xB3 */ + 0xF1, 0xE5, 0xB0, 0x94, 0xB0, 0x95, 0xB0, 0x96, /* 0xB4-0xB7 */ + 0xB0, 0x97, 0xB0, 0x98, 0xB0, 0x99, 0xB0, 0x9A, /* 0xB8-0xBB */ + 0xB0, 0x9B, 0xB0, 0x9C, 0xB0, 0x9D, 0xC3, 0xF3, /* 0xBC-0xBF */ + 0xB0, 0x9E, 0xB0, 0x9F, 0xD3, 0xDB, 0xB0, 0xA0, /* 0xC0-0xC3 */ + 0xB1, 0x40, 0xD6, 0xD1, 0xC5, 0xE8, 0xB1, 0x41, /* 0xC4-0xC7 */ + 0xD3, 0xAF, 0xB1, 0x42, 0xD2, 0xE6, 0xB1, 0x43, /* 0xC8-0xCB */ + 0xB1, 0x44, 0xEE, 0xC1, 0xB0, 0xBB, 0xD5, 0xB5, /* 0xCC-0xCF */ + 0xD1, 0xCE, 0xBC, 0xE0, 0xBA, 0xD0, 0xB1, 0x45, /* 0xD0-0xD3 */ + 0xBF, 0xF8, 0xB1, 0x46, 0xB8, 0xC7, 0xB5, 0xC1, /* 0xD4-0xD7 */ + 0xC5, 0xCC, 0xB1, 0x47, 0xB1, 0x48, 0xCA, 0xA2, /* 0xD8-0xDB */ + 0xB1, 0x49, 0xB1, 0x4A, 0xB1, 0x4B, 0xC3, 0xCB, /* 0xDC-0xDF */ + 0xB1, 0x4C, 0xB1, 0x4D, 0xB1, 0x4E, 0xB1, 0x4F, /* 0xE0-0xE3 */ + 0xB1, 0x50, 0xEE, 0xC2, 0xB1, 0x51, 0xB1, 0x52, /* 0xE4-0xE7 */ + 0xB1, 0x53, 0xB1, 0x54, 0xB1, 0x55, 0xB1, 0x56, /* 0xE8-0xEB */ + 0xB1, 0x57, 0xB1, 0x58, 0xC4, 0xBF, 0xB6, 0xA2, /* 0xEC-0xEF */ + 0xB1, 0x59, 0xED, 0xEC, 0xC3, 0xA4, 0xB1, 0x5A, /* 0xF0-0xF3 */ + 0xD6, 0xB1, 0xB1, 0x5B, 0xB1, 0x5C, 0xB1, 0x5D, /* 0xF4-0xF7 */ + 0xCF, 0xE0, 0xED, 0xEF, 0xB1, 0x5E, 0xB1, 0x5F, /* 0xF8-0xFB */ + 0xC5, 0xCE, 0xB1, 0x60, 0xB6, 0xDC, 0xB1, 0x61, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_77[512] = { + 0xB1, 0x62, 0xCA, 0xA1, 0xB1, 0x63, 0xB1, 0x64, /* 0x00-0x03 */ + 0xED, 0xED, 0xB1, 0x65, 0xB1, 0x66, 0xED, 0xF0, /* 0x04-0x07 */ + 0xED, 0xF1, 0xC3, 0xBC, 0xB1, 0x67, 0xBF, 0xB4, /* 0x08-0x0B */ + 0xB1, 0x68, 0xED, 0xEE, 0xB1, 0x69, 0xB1, 0x6A, /* 0x0C-0x0F */ + 0xB1, 0x6B, 0xB1, 0x6C, 0xB1, 0x6D, 0xB1, 0x6E, /* 0x10-0x13 */ + 0xB1, 0x6F, 0xB1, 0x70, 0xB1, 0x71, 0xB1, 0x72, /* 0x14-0x17 */ + 0xB1, 0x73, 0xED, 0xF4, 0xED, 0xF2, 0xB1, 0x74, /* 0x18-0x1B */ + 0xB1, 0x75, 0xB1, 0x76, 0xB1, 0x77, 0xD5, 0xE6, /* 0x1C-0x1F */ + 0xC3, 0xDF, 0xB1, 0x78, 0xED, 0xF3, 0xB1, 0x79, /* 0x20-0x23 */ + 0xB1, 0x7A, 0xB1, 0x7B, 0xED, 0xF6, 0xB1, 0x7C, /* 0x24-0x27 */ + 0xD5, 0xA3, 0xD1, 0xA3, 0xB1, 0x7D, 0xB1, 0x7E, /* 0x28-0x2B */ + 0xB1, 0x80, 0xED, 0xF5, 0xB1, 0x81, 0xC3, 0xD0, /* 0x2C-0x2F */ + 0xB1, 0x82, 0xB1, 0x83, 0xB1, 0x84, 0xB1, 0x85, /* 0x30-0x33 */ + 0xB1, 0x86, 0xED, 0xF7, 0xBF, 0xF4, 0xBE, 0xEC, /* 0x34-0x37 */ + 0xED, 0xF8, 0xB1, 0x87, 0xCC, 0xF7, 0xB1, 0x88, /* 0x38-0x3B */ + 0xD1, 0xDB, 0xB1, 0x89, 0xB1, 0x8A, 0xB1, 0x8B, /* 0x3C-0x3F */ + 0xD7, 0xC5, 0xD5, 0xF6, 0xB1, 0x8C, 0xED, 0xFC, /* 0x40-0x43 */ + 0xB1, 0x8D, 0xB1, 0x8E, 0xB1, 0x8F, 0xED, 0xFB, /* 0x44-0x47 */ + 0xB1, 0x90, 0xB1, 0x91, 0xB1, 0x92, 0xB1, 0x93, /* 0x48-0x4B */ + 0xB1, 0x94, 0xB1, 0x95, 0xB1, 0x96, 0xB1, 0x97, /* 0x4C-0x4F */ + 0xED, 0xF9, 0xED, 0xFA, 0xB1, 0x98, 0xB1, 0x99, /* 0x50-0x53 */ + 0xB1, 0x9A, 0xB1, 0x9B, 0xB1, 0x9C, 0xB1, 0x9D, /* 0x54-0x57 */ + 0xB1, 0x9E, 0xB1, 0x9F, 0xED, 0xFD, 0xBE, 0xA6, /* 0x58-0x5B */ + 0xB1, 0xA0, 0xB2, 0x40, 0xB2, 0x41, 0xB2, 0x42, /* 0x5C-0x5F */ + 0xB2, 0x43, 0xCB, 0xAF, 0xEE, 0xA1, 0xB6, 0xBD, /* 0x60-0x63 */ + 0xB2, 0x44, 0xEE, 0xA2, 0xC4, 0xC0, 0xB2, 0x45, /* 0x64-0x67 */ + 0xED, 0xFE, 0xB2, 0x46, 0xB2, 0x47, 0xBD, 0xDE, /* 0x68-0x6B */ + 0xB2, 0xC7, 0xB2, 0x48, 0xB2, 0x49, 0xB2, 0x4A, /* 0x6C-0x6F */ + 0xB2, 0x4B, 0xB2, 0x4C, 0xB2, 0x4D, 0xB2, 0x4E, /* 0x70-0x73 */ + 0xB2, 0x4F, 0xB2, 0x50, 0xB2, 0x51, 0xB2, 0x52, /* 0x74-0x77 */ + 0xB2, 0x53, 0xB6, 0xC3, 0xB2, 0x54, 0xB2, 0x55, /* 0x78-0x7B */ + 0xB2, 0x56, 0xEE, 0xA5, 0xD8, 0xBA, 0xEE, 0xA3, /* 0x7C-0x7F */ + + 0xEE, 0xA6, 0xB2, 0x57, 0xB2, 0x58, 0xB2, 0x59, /* 0x80-0x83 */ + 0xC3, 0xE9, 0xB3, 0xF2, 0xB2, 0x5A, 0xB2, 0x5B, /* 0x84-0x87 */ + 0xB2, 0x5C, 0xB2, 0x5D, 0xB2, 0x5E, 0xB2, 0x5F, /* 0x88-0x8B */ + 0xEE, 0xA7, 0xEE, 0xA4, 0xCF, 0xB9, 0xB2, 0x60, /* 0x8C-0x8F */ + 0xB2, 0x61, 0xEE, 0xA8, 0xC2, 0xF7, 0xB2, 0x62, /* 0x90-0x93 */ + 0xB2, 0x63, 0xB2, 0x64, 0xB2, 0x65, 0xB2, 0x66, /* 0x94-0x97 */ + 0xB2, 0x67, 0xB2, 0x68, 0xB2, 0x69, 0xB2, 0x6A, /* 0x98-0x9B */ + 0xB2, 0x6B, 0xB2, 0x6C, 0xB2, 0x6D, 0xEE, 0xA9, /* 0x9C-0x9F */ + 0xEE, 0xAA, 0xB2, 0x6E, 0xDE, 0xAB, 0xB2, 0x6F, /* 0xA0-0xA3 */ + 0xB2, 0x70, 0xC6, 0xB3, 0xB2, 0x71, 0xC7, 0xC6, /* 0xA4-0xA7 */ + 0xB2, 0x72, 0xD6, 0xF5, 0xB5, 0xC9, 0xB2, 0x73, /* 0xA8-0xAB */ + 0xCB, 0xB2, 0xB2, 0x74, 0xB2, 0x75, 0xB2, 0x76, /* 0xAC-0xAF */ + 0xEE, 0xAB, 0xB2, 0x77, 0xB2, 0x78, 0xCD, 0xAB, /* 0xB0-0xB3 */ + 0xB2, 0x79, 0xEE, 0xAC, 0xB2, 0x7A, 0xB2, 0x7B, /* 0xB4-0xB7 */ + 0xB2, 0x7C, 0xB2, 0x7D, 0xB2, 0x7E, 0xD5, 0xB0, /* 0xB8-0xBB */ + 0xB2, 0x80, 0xEE, 0xAD, 0xB2, 0x81, 0xF6, 0xC4, /* 0xBC-0xBF */ + 0xB2, 0x82, 0xB2, 0x83, 0xB2, 0x84, 0xB2, 0x85, /* 0xC0-0xC3 */ + 0xB2, 0x86, 0xB2, 0x87, 0xB2, 0x88, 0xB2, 0x89, /* 0xC4-0xC7 */ + 0xB2, 0x8A, 0xB2, 0x8B, 0xB2, 0x8C, 0xB2, 0x8D, /* 0xC8-0xCB */ + 0xB2, 0x8E, 0xDB, 0xC7, 0xB2, 0x8F, 0xB2, 0x90, /* 0xCC-0xCF */ + 0xB2, 0x91, 0xB2, 0x92, 0xB2, 0x93, 0xB2, 0x94, /* 0xD0-0xD3 */ + 0xB2, 0x95, 0xB2, 0x96, 0xB2, 0x97, 0xB4, 0xA3, /* 0xD4-0xD7 */ + 0xB2, 0x98, 0xB2, 0x99, 0xB2, 0x9A, 0xC3, 0xAC, /* 0xD8-0xDB */ + 0xF1, 0xE6, 0xB2, 0x9B, 0xB2, 0x9C, 0xB2, 0x9D, /* 0xDC-0xDF */ + 0xB2, 0x9E, 0xB2, 0x9F, 0xCA, 0xB8, 0xD2, 0xD3, /* 0xE0-0xE3 */ + 0xB2, 0xA0, 0xD6, 0xAA, 0xB3, 0x40, 0xEF, 0xF2, /* 0xE4-0xE7 */ + 0xB3, 0x41, 0xBE, 0xD8, 0xB3, 0x42, 0xBD, 0xC3, /* 0xE8-0xEB */ + 0xEF, 0xF3, 0xB6, 0xCC, 0xB0, 0xAB, 0xB3, 0x43, /* 0xEC-0xEF */ + 0xB3, 0x44, 0xB3, 0x45, 0xB3, 0x46, 0xCA, 0xAF, /* 0xF0-0xF3 */ + 0xB3, 0x47, 0xB3, 0x48, 0xED, 0xB6, 0xB3, 0x49, /* 0xF4-0xF7 */ + 0xED, 0xB7, 0xB3, 0x4A, 0xB3, 0x4B, 0xB3, 0x4C, /* 0xF8-0xFB */ + 0xB3, 0x4D, 0xCE, 0xF9, 0xB7, 0xAF, 0xBF, 0xF3, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_78[512] = { + 0xED, 0xB8, 0xC2, 0xEB, 0xC9, 0xB0, 0xB3, 0x4E, /* 0x00-0x03 */ + 0xB3, 0x4F, 0xB3, 0x50, 0xB3, 0x51, 0xB3, 0x52, /* 0x04-0x07 */ + 0xB3, 0x53, 0xED, 0xB9, 0xB3, 0x54, 0xB3, 0x55, /* 0x08-0x0B */ + 0xC6, 0xF6, 0xBF, 0xB3, 0xB3, 0x56, 0xB3, 0x57, /* 0x0C-0x0F */ + 0xB3, 0x58, 0xED, 0xBC, 0xC5, 0xF8, 0xB3, 0x59, /* 0x10-0x13 */ + 0xD1, 0xD0, 0xB3, 0x5A, 0xD7, 0xA9, 0xED, 0xBA, /* 0x14-0x17 */ + 0xED, 0xBB, 0xB3, 0x5B, 0xD1, 0xE2, 0xB3, 0x5C, /* 0x18-0x1B */ + 0xED, 0xBF, 0xED, 0xC0, 0xB3, 0x5D, 0xED, 0xC4, /* 0x1C-0x1F */ + 0xB3, 0x5E, 0xB3, 0x5F, 0xB3, 0x60, 0xED, 0xC8, /* 0x20-0x23 */ + 0xB3, 0x61, 0xED, 0xC6, 0xED, 0xCE, 0xD5, 0xE8, /* 0x24-0x27 */ + 0xB3, 0x62, 0xED, 0xC9, 0xB3, 0x63, 0xB3, 0x64, /* 0x28-0x2B */ + 0xED, 0xC7, 0xED, 0xBE, 0xB3, 0x65, 0xB3, 0x66, /* 0x2C-0x2F */ + 0xC5, 0xE9, 0xB3, 0x67, 0xB3, 0x68, 0xB3, 0x69, /* 0x30-0x33 */ + 0xC6, 0xC6, 0xB3, 0x6A, 0xB3, 0x6B, 0xC9, 0xE9, /* 0x34-0x37 */ + 0xD4, 0xD2, 0xED, 0xC1, 0xED, 0xC2, 0xED, 0xC3, /* 0x38-0x3B */ + 0xED, 0xC5, 0xB3, 0x6C, 0xC0, 0xF9, 0xB3, 0x6D, /* 0x3C-0x3F */ + 0xB4, 0xA1, 0xB3, 0x6E, 0xB3, 0x6F, 0xB3, 0x70, /* 0x40-0x43 */ + 0xB3, 0x71, 0xB9, 0xE8, 0xB3, 0x72, 0xED, 0xD0, /* 0x44-0x47 */ + 0xB3, 0x73, 0xB3, 0x74, 0xB3, 0x75, 0xB3, 0x76, /* 0x48-0x4B */ + 0xED, 0xD1, 0xB3, 0x77, 0xED, 0xCA, 0xB3, 0x78, /* 0x4C-0x4F */ + 0xED, 0xCF, 0xB3, 0x79, 0xCE, 0xF8, 0xB3, 0x7A, /* 0x50-0x53 */ + 0xB3, 0x7B, 0xCB, 0xB6, 0xED, 0xCC, 0xED, 0xCD, /* 0x54-0x57 */ + 0xB3, 0x7C, 0xB3, 0x7D, 0xB3, 0x7E, 0xB3, 0x80, /* 0x58-0x5B */ + 0xB3, 0x81, 0xCF, 0xF5, 0xB3, 0x82, 0xB3, 0x83, /* 0x5C-0x5F */ + 0xB3, 0x84, 0xB3, 0x85, 0xB3, 0x86, 0xB3, 0x87, /* 0x60-0x63 */ + 0xB3, 0x88, 0xB3, 0x89, 0xB3, 0x8A, 0xB3, 0x8B, /* 0x64-0x67 */ + 0xB3, 0x8C, 0xB3, 0x8D, 0xED, 0xD2, 0xC1, 0xF2, /* 0x68-0x6B */ + 0xD3, 0xB2, 0xED, 0xCB, 0xC8, 0xB7, 0xB3, 0x8E, /* 0x6C-0x6F */ + 0xB3, 0x8F, 0xB3, 0x90, 0xB3, 0x91, 0xB3, 0x92, /* 0x70-0x73 */ + 0xB3, 0x93, 0xB3, 0x94, 0xB3, 0x95, 0xBC, 0xEF, /* 0x74-0x77 */ + 0xB3, 0x96, 0xB3, 0x97, 0xB3, 0x98, 0xB3, 0x99, /* 0x78-0x7B */ + 0xC5, 0xF0, 0xB3, 0x9A, 0xB3, 0x9B, 0xB3, 0x9C, /* 0x7C-0x7F */ + + 0xB3, 0x9D, 0xB3, 0x9E, 0xB3, 0x9F, 0xB3, 0xA0, /* 0x80-0x83 */ + 0xB4, 0x40, 0xB4, 0x41, 0xB4, 0x42, 0xED, 0xD6, /* 0x84-0x87 */ + 0xB4, 0x43, 0xB5, 0xEF, 0xB4, 0x44, 0xB4, 0x45, /* 0x88-0x8B */ + 0xC2, 0xB5, 0xB0, 0xAD, 0xCB, 0xE9, 0xB4, 0x46, /* 0x8C-0x8F */ + 0xB4, 0x47, 0xB1, 0xAE, 0xB4, 0x48, 0xED, 0xD4, /* 0x90-0x93 */ + 0xB4, 0x49, 0xB4, 0x4A, 0xB4, 0x4B, 0xCD, 0xEB, /* 0x94-0x97 */ + 0xB5, 0xE2, 0xB4, 0x4C, 0xED, 0xD5, 0xED, 0xD3, /* 0x98-0x9B */ + 0xED, 0xD7, 0xB4, 0x4D, 0xB4, 0x4E, 0xB5, 0xFA, /* 0x9C-0x9F */ + 0xB4, 0x4F, 0xED, 0xD8, 0xB4, 0x50, 0xED, 0xD9, /* 0xA0-0xA3 */ + 0xB4, 0x51, 0xED, 0xDC, 0xB4, 0x52, 0xB1, 0xCC, /* 0xA4-0xA7 */ + 0xB4, 0x53, 0xB4, 0x54, 0xB4, 0x55, 0xB4, 0x56, /* 0xA8-0xAB */ + 0xB4, 0x57, 0xB4, 0x58, 0xB4, 0x59, 0xB4, 0x5A, /* 0xAC-0xAF */ + 0xC5, 0xF6, 0xBC, 0xEE, 0xED, 0xDA, 0xCC, 0xBC, /* 0xB0-0xB3 */ + 0xB2, 0xEA, 0xB4, 0x5B, 0xB4, 0x5C, 0xB4, 0x5D, /* 0xB4-0xB7 */ + 0xB4, 0x5E, 0xED, 0xDB, 0xB4, 0x5F, 0xB4, 0x60, /* 0xB8-0xBB */ + 0xB4, 0x61, 0xB4, 0x62, 0xC4, 0xEB, 0xB4, 0x63, /* 0xBC-0xBF */ + 0xB4, 0x64, 0xB4, 0xC5, 0xB4, 0x65, 0xB4, 0x66, /* 0xC0-0xC3 */ + 0xB4, 0x67, 0xB0, 0xF5, 0xB4, 0x68, 0xB4, 0x69, /* 0xC4-0xC7 */ + 0xB4, 0x6A, 0xED, 0xDF, 0xC0, 0xDA, 0xB4, 0xE8, /* 0xC8-0xCB */ + 0xB4, 0x6B, 0xB4, 0x6C, 0xB4, 0x6D, 0xB4, 0x6E, /* 0xCC-0xCF */ + 0xC5, 0xCD, 0xB4, 0x6F, 0xB4, 0x70, 0xB4, 0x71, /* 0xD0-0xD3 */ + 0xED, 0xDD, 0xBF, 0xC4, 0xB4, 0x72, 0xB4, 0x73, /* 0xD4-0xD7 */ + 0xB4, 0x74, 0xED, 0xDE, 0xB4, 0x75, 0xB4, 0x76, /* 0xD8-0xDB */ + 0xB4, 0x77, 0xB4, 0x78, 0xB4, 0x79, 0xB4, 0x7A, /* 0xDC-0xDF */ + 0xB4, 0x7B, 0xB4, 0x7C, 0xB4, 0x7D, 0xB4, 0x7E, /* 0xE0-0xE3 */ + 0xB4, 0x80, 0xB4, 0x81, 0xB4, 0x82, 0xB4, 0x83, /* 0xE4-0xE7 */ + 0xC4, 0xA5, 0xB4, 0x84, 0xB4, 0x85, 0xB4, 0x86, /* 0xE8-0xEB */ + 0xED, 0xE0, 0xB4, 0x87, 0xB4, 0x88, 0xB4, 0x89, /* 0xEC-0xEF */ + 0xB4, 0x8A, 0xB4, 0x8B, 0xED, 0xE1, 0xB4, 0x8C, /* 0xF0-0xF3 */ + 0xED, 0xE3, 0xB4, 0x8D, 0xB4, 0x8E, 0xC1, 0xD7, /* 0xF4-0xF7 */ + 0xB4, 0x8F, 0xB4, 0x90, 0xBB, 0xC7, 0xB4, 0x91, /* 0xF8-0xFB */ + 0xB4, 0x92, 0xB4, 0x93, 0xB4, 0x94, 0xB4, 0x95, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_79[512] = { + 0xB4, 0x96, 0xBD, 0xB8, 0xB4, 0x97, 0xB4, 0x98, /* 0x00-0x03 */ + 0xB4, 0x99, 0xED, 0xE2, 0xB4, 0x9A, 0xB4, 0x9B, /* 0x04-0x07 */ + 0xB4, 0x9C, 0xB4, 0x9D, 0xB4, 0x9E, 0xB4, 0x9F, /* 0x08-0x0B */ + 0xB4, 0xA0, 0xB5, 0x40, 0xB5, 0x41, 0xB5, 0x42, /* 0x0C-0x0F */ + 0xB5, 0x43, 0xB5, 0x44, 0xB5, 0x45, 0xED, 0xE4, /* 0x10-0x13 */ + 0xB5, 0x46, 0xB5, 0x47, 0xB5, 0x48, 0xB5, 0x49, /* 0x14-0x17 */ + 0xB5, 0x4A, 0xB5, 0x4B, 0xB5, 0x4C, 0xB5, 0x4D, /* 0x18-0x1B */ + 0xB5, 0x4E, 0xB5, 0x4F, 0xED, 0xE6, 0xB5, 0x50, /* 0x1C-0x1F */ + 0xB5, 0x51, 0xB5, 0x52, 0xB5, 0x53, 0xB5, 0x54, /* 0x20-0x23 */ + 0xED, 0xE5, 0xB5, 0x55, 0xB5, 0x56, 0xB5, 0x57, /* 0x24-0x27 */ + 0xB5, 0x58, 0xB5, 0x59, 0xB5, 0x5A, 0xB5, 0x5B, /* 0x28-0x2B */ + 0xB5, 0x5C, 0xB5, 0x5D, 0xB5, 0x5E, 0xB5, 0x5F, /* 0x2C-0x2F */ + 0xB5, 0x60, 0xB5, 0x61, 0xB5, 0x62, 0xB5, 0x63, /* 0x30-0x33 */ + 0xED, 0xE7, 0xB5, 0x64, 0xB5, 0x65, 0xB5, 0x66, /* 0x34-0x37 */ + 0xB5, 0x67, 0xB5, 0x68, 0xCA, 0xBE, 0xEC, 0xEA, /* 0x38-0x3B */ + 0xC0, 0xF1, 0xB5, 0x69, 0xC9, 0xE7, 0xB5, 0x6A, /* 0x3C-0x3F */ + 0xEC, 0xEB, 0xC6, 0xEE, 0xB5, 0x6B, 0xB5, 0x6C, /* 0x40-0x43 */ + 0xB5, 0x6D, 0xB5, 0x6E, 0xEC, 0xEC, 0xB5, 0x6F, /* 0x44-0x47 */ + 0xC6, 0xED, 0xEC, 0xED, 0xB5, 0x70, 0xB5, 0x71, /* 0x48-0x4B */ + 0xB5, 0x72, 0xB5, 0x73, 0xB5, 0x74, 0xB5, 0x75, /* 0x4C-0x4F */ + 0xB5, 0x76, 0xB5, 0x77, 0xB5, 0x78, 0xEC, 0xF0, /* 0x50-0x53 */ + 0xB5, 0x79, 0xB5, 0x7A, 0xD7, 0xE6, 0xEC, 0xF3, /* 0x54-0x57 */ + 0xB5, 0x7B, 0xB5, 0x7C, 0xEC, 0xF1, 0xEC, 0xEE, /* 0x58-0x5B */ + 0xEC, 0xEF, 0xD7, 0xA3, 0xC9, 0xF1, 0xCB, 0xEE, /* 0x5C-0x5F */ + 0xEC, 0xF4, 0xB5, 0x7D, 0xEC, 0xF2, 0xB5, 0x7E, /* 0x60-0x63 */ + 0xB5, 0x80, 0xCF, 0xE9, 0xB5, 0x81, 0xEC, 0xF6, /* 0x64-0x67 */ + 0xC6, 0xB1, 0xB5, 0x82, 0xB5, 0x83, 0xB5, 0x84, /* 0x68-0x6B */ + 0xB5, 0x85, 0xBC, 0xC0, 0xB5, 0x86, 0xEC, 0xF5, /* 0x6C-0x6F */ + 0xB5, 0x87, 0xB5, 0x88, 0xB5, 0x89, 0xB5, 0x8A, /* 0x70-0x73 */ + 0xB5, 0x8B, 0xB5, 0x8C, 0xB5, 0x8D, 0xB5, 0xBB, /* 0x74-0x77 */ + 0xBB, 0xF6, 0xB5, 0x8E, 0xEC, 0xF7, 0xB5, 0x8F, /* 0x78-0x7B */ + 0xB5, 0x90, 0xB5, 0x91, 0xB5, 0x92, 0xB5, 0x93, /* 0x7C-0x7F */ + + 0xD9, 0xF7, 0xBD, 0xFB, 0xB5, 0x94, 0xB5, 0x95, /* 0x80-0x83 */ + 0xC2, 0xBB, 0xEC, 0xF8, 0xB5, 0x96, 0xB5, 0x97, /* 0x84-0x87 */ + 0xB5, 0x98, 0xB5, 0x99, 0xEC, 0xF9, 0xB5, 0x9A, /* 0x88-0x8B */ + 0xB5, 0x9B, 0xB5, 0x9C, 0xB5, 0x9D, 0xB8, 0xA3, /* 0x8C-0x8F */ + 0xB5, 0x9E, 0xB5, 0x9F, 0xB5, 0xA0, 0xB6, 0x40, /* 0x90-0x93 */ + 0xB6, 0x41, 0xB6, 0x42, 0xB6, 0x43, 0xB6, 0x44, /* 0x94-0x97 */ + 0xB6, 0x45, 0xB6, 0x46, 0xEC, 0xFA, 0xB6, 0x47, /* 0x98-0x9B */ + 0xB6, 0x48, 0xB6, 0x49, 0xB6, 0x4A, 0xB6, 0x4B, /* 0x9C-0x9F */ + 0xB6, 0x4C, 0xB6, 0x4D, 0xB6, 0x4E, 0xB6, 0x4F, /* 0xA0-0xA3 */ + 0xB6, 0x50, 0xB6, 0x51, 0xB6, 0x52, 0xEC, 0xFB, /* 0xA4-0xA7 */ + 0xB6, 0x53, 0xB6, 0x54, 0xB6, 0x55, 0xB6, 0x56, /* 0xA8-0xAB */ + 0xB6, 0x57, 0xB6, 0x58, 0xB6, 0x59, 0xB6, 0x5A, /* 0xAC-0xAF */ + 0xB6, 0x5B, 0xB6, 0x5C, 0xB6, 0x5D, 0xEC, 0xFC, /* 0xB0-0xB3 */ + 0xB6, 0x5E, 0xB6, 0x5F, 0xB6, 0x60, 0xB6, 0x61, /* 0xB4-0xB7 */ + 0xB6, 0x62, 0xD3, 0xED, 0xD8, 0xAE, 0xC0, 0xEB, /* 0xB8-0xBB */ + 0xB6, 0x63, 0xC7, 0xDD, 0xBA, 0xCC, 0xB6, 0x64, /* 0xBC-0xBF */ + 0xD0, 0xE3, 0xCB, 0xBD, 0xB6, 0x65, 0xCD, 0xBA, /* 0xC0-0xC3 */ + 0xB6, 0x66, 0xB6, 0x67, 0xB8, 0xD1, 0xB6, 0x68, /* 0xC4-0xC7 */ + 0xB6, 0x69, 0xB1, 0xFC, 0xB6, 0x6A, 0xC7, 0xEF, /* 0xC8-0xCB */ + 0xB6, 0x6B, 0xD6, 0xD6, 0xB6, 0x6C, 0xB6, 0x6D, /* 0xCC-0xCF */ + 0xB6, 0x6E, 0xBF, 0xC6, 0xC3, 0xEB, 0xB6, 0x6F, /* 0xD0-0xD3 */ + 0xB6, 0x70, 0xEF, 0xF5, 0xB6, 0x71, 0xB6, 0x72, /* 0xD4-0xD7 */ + 0xC3, 0xD8, 0xB6, 0x73, 0xB6, 0x74, 0xB6, 0x75, /* 0xD8-0xDB */ + 0xB6, 0x76, 0xB6, 0x77, 0xB6, 0x78, 0xD7, 0xE2, /* 0xDC-0xDF */ + 0xB6, 0x79, 0xB6, 0x7A, 0xB6, 0x7B, 0xEF, 0xF7, /* 0xE0-0xE3 */ + 0xB3, 0xD3, 0xB6, 0x7C, 0xC7, 0xD8, 0xD1, 0xED, /* 0xE4-0xE7 */ + 0xB6, 0x7D, 0xD6, 0xC8, 0xB6, 0x7E, 0xEF, 0xF8, /* 0xE8-0xEB */ + 0xB6, 0x80, 0xEF, 0xF6, 0xB6, 0x81, 0xBB, 0xFD, /* 0xEC-0xEF */ + 0xB3, 0xC6, 0xB6, 0x82, 0xB6, 0x83, 0xB6, 0x84, /* 0xF0-0xF3 */ + 0xB6, 0x85, 0xB6, 0x86, 0xB6, 0x87, 0xB6, 0x88, /* 0xF4-0xF7 */ + 0xBD, 0xD5, 0xB6, 0x89, 0xB6, 0x8A, 0xD2, 0xC6, /* 0xF8-0xFB */ + 0xB6, 0x8B, 0xBB, 0xE0, 0xB6, 0x8C, 0xB6, 0x8D, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_7A[512] = { + 0xCF, 0xA1, 0xB6, 0x8E, 0xEF, 0xFC, 0xEF, 0xFB, /* 0x00-0x03 */ + 0xB6, 0x8F, 0xB6, 0x90, 0xEF, 0xF9, 0xB6, 0x91, /* 0x04-0x07 */ + 0xB6, 0x92, 0xB6, 0x93, 0xB6, 0x94, 0xB3, 0xCC, /* 0x08-0x0B */ + 0xB6, 0x95, 0xC9, 0xD4, 0xCB, 0xB0, 0xB6, 0x96, /* 0x0C-0x0F */ + 0xB6, 0x97, 0xB6, 0x98, 0xB6, 0x99, 0xB6, 0x9A, /* 0x10-0x13 */ + 0xEF, 0xFE, 0xB6, 0x9B, 0xB6, 0x9C, 0xB0, 0xDE, /* 0x14-0x17 */ + 0xB6, 0x9D, 0xB6, 0x9E, 0xD6, 0xC9, 0xB6, 0x9F, /* 0x18-0x1B */ + 0xB6, 0xA0, 0xB7, 0x40, 0xEF, 0xFD, 0xB7, 0x41, /* 0x1C-0x1F */ + 0xB3, 0xED, 0xB7, 0x42, 0xB7, 0x43, 0xF6, 0xD5, /* 0x20-0x23 */ + 0xB7, 0x44, 0xB7, 0x45, 0xB7, 0x46, 0xB7, 0x47, /* 0x24-0x27 */ + 0xB7, 0x48, 0xB7, 0x49, 0xB7, 0x4A, 0xB7, 0x4B, /* 0x28-0x2B */ + 0xB7, 0x4C, 0xB7, 0x4D, 0xB7, 0x4E, 0xB7, 0x4F, /* 0x2C-0x2F */ + 0xB7, 0x50, 0xB7, 0x51, 0xB7, 0x52, 0xCE, 0xC8, /* 0x30-0x33 */ + 0xB7, 0x53, 0xB7, 0x54, 0xB7, 0x55, 0xF0, 0xA2, /* 0x34-0x37 */ + 0xB7, 0x56, 0xF0, 0xA1, 0xB7, 0x57, 0xB5, 0xBE, /* 0x38-0x3B */ + 0xBC, 0xDA, 0xBB, 0xFC, 0xB7, 0x58, 0xB8, 0xE5, /* 0x3C-0x3F */ + 0xB7, 0x59, 0xB7, 0x5A, 0xB7, 0x5B, 0xB7, 0x5C, /* 0x40-0x43 */ + 0xB7, 0x5D, 0xB7, 0x5E, 0xC4, 0xC2, 0xB7, 0x5F, /* 0x44-0x47 */ + 0xB7, 0x60, 0xB7, 0x61, 0xB7, 0x62, 0xB7, 0x63, /* 0x48-0x4B */ + 0xB7, 0x64, 0xB7, 0x65, 0xB7, 0x66, 0xB7, 0x67, /* 0x4C-0x4F */ + 0xB7, 0x68, 0xF0, 0xA3, 0xB7, 0x69, 0xB7, 0x6A, /* 0x50-0x53 */ + 0xB7, 0x6B, 0xB7, 0x6C, 0xB7, 0x6D, 0xCB, 0xEB, /* 0x54-0x57 */ + 0xB7, 0x6E, 0xB7, 0x6F, 0xB7, 0x70, 0xB7, 0x71, /* 0x58-0x5B */ + 0xB7, 0x72, 0xB7, 0x73, 0xB7, 0x74, 0xB7, 0x75, /* 0x5C-0x5F */ + 0xB7, 0x76, 0xB7, 0x77, 0xB7, 0x78, 0xB7, 0x79, /* 0x60-0x63 */ + 0xB7, 0x7A, 0xB7, 0x7B, 0xB7, 0x7C, 0xB7, 0x7D, /* 0x64-0x67 */ + 0xB7, 0x7E, 0xB7, 0x80, 0xB7, 0x81, 0xB7, 0x82, /* 0x68-0x6B */ + 0xB7, 0x83, 0xB7, 0x84, 0xB7, 0x85, 0xB7, 0x86, /* 0x6C-0x6F */ + 0xF0, 0xA6, 0xB7, 0x87, 0xB7, 0x88, 0xB7, 0x89, /* 0x70-0x73 */ + 0xD1, 0xA8, 0xB7, 0x8A, 0xBE, 0xBF, 0xC7, 0xEE, /* 0x74-0x77 */ + 0xF1, 0xB6, 0xF1, 0xB7, 0xBF, 0xD5, 0xB7, 0x8B, /* 0x78-0x7B */ + 0xB7, 0x8C, 0xB7, 0x8D, 0xB7, 0x8E, 0xB4, 0xA9, /* 0x7C-0x7F */ + + 0xF1, 0xB8, 0xCD, 0xBB, 0xB7, 0x8F, 0xC7, 0xD4, /* 0x80-0x83 */ + 0xD5, 0xAD, 0xB7, 0x90, 0xF1, 0xB9, 0xB7, 0x91, /* 0x84-0x87 */ + 0xF1, 0xBA, 0xB7, 0x92, 0xB7, 0x93, 0xB7, 0x94, /* 0x88-0x8B */ + 0xB7, 0x95, 0xC7, 0xCF, 0xB7, 0x96, 0xB7, 0x97, /* 0x8C-0x8F */ + 0xB7, 0x98, 0xD2, 0xA4, 0xD6, 0xCF, 0xB7, 0x99, /* 0x90-0x93 */ + 0xB7, 0x9A, 0xF1, 0xBB, 0xBD, 0xD1, 0xB4, 0xB0, /* 0x94-0x97 */ + 0xBE, 0xBD, 0xB7, 0x9B, 0xB7, 0x9C, 0xB7, 0x9D, /* 0x98-0x9B */ + 0xB4, 0xDC, 0xCE, 0xD1, 0xB7, 0x9E, 0xBF, 0xDF, /* 0x9C-0x9F */ + 0xF1, 0xBD, 0xB7, 0x9F, 0xB7, 0xA0, 0xB8, 0x40, /* 0xA0-0xA3 */ + 0xB8, 0x41, 0xBF, 0xFA, 0xF1, 0xBC, 0xB8, 0x42, /* 0xA4-0xA7 */ + 0xF1, 0xBF, 0xB8, 0x43, 0xB8, 0x44, 0xB8, 0x45, /* 0xA8-0xAB */ + 0xF1, 0xBE, 0xF1, 0xC0, 0xB8, 0x46, 0xB8, 0x47, /* 0xAC-0xAF */ + 0xB8, 0x48, 0xB8, 0x49, 0xB8, 0x4A, 0xF1, 0xC1, /* 0xB0-0xB3 */ + 0xB8, 0x4B, 0xB8, 0x4C, 0xB8, 0x4D, 0xB8, 0x4E, /* 0xB4-0xB7 */ + 0xB8, 0x4F, 0xB8, 0x50, 0xB8, 0x51, 0xB8, 0x52, /* 0xB8-0xBB */ + 0xB8, 0x53, 0xB8, 0x54, 0xB8, 0x55, 0xC1, 0xFE, /* 0xBC-0xBF */ + 0xB8, 0x56, 0xB8, 0x57, 0xB8, 0x58, 0xB8, 0x59, /* 0xC0-0xC3 */ + 0xB8, 0x5A, 0xB8, 0x5B, 0xB8, 0x5C, 0xB8, 0x5D, /* 0xC4-0xC7 */ + 0xB8, 0x5E, 0xB8, 0x5F, 0xB8, 0x60, 0xC1, 0xA2, /* 0xC8-0xCB */ + 0xB8, 0x61, 0xB8, 0x62, 0xB8, 0x63, 0xB8, 0x64, /* 0xCC-0xCF */ + 0xB8, 0x65, 0xB8, 0x66, 0xB8, 0x67, 0xB8, 0x68, /* 0xD0-0xD3 */ + 0xB8, 0x69, 0xB8, 0x6A, 0xCA, 0xFA, 0xB8, 0x6B, /* 0xD4-0xD7 */ + 0xB8, 0x6C, 0xD5, 0xBE, 0xB8, 0x6D, 0xB8, 0x6E, /* 0xD8-0xDB */ + 0xB8, 0x6F, 0xB8, 0x70, 0xBE, 0xBA, 0xBE, 0xB9, /* 0xDC-0xDF */ + 0xD5, 0xC2, 0xB8, 0x71, 0xB8, 0x72, 0xBF, 0xA2, /* 0xE0-0xE3 */ + 0xB8, 0x73, 0xCD, 0xAF, 0xF1, 0xB5, 0xB8, 0x74, /* 0xE4-0xE7 */ + 0xB8, 0x75, 0xB8, 0x76, 0xB8, 0x77, 0xB8, 0x78, /* 0xE8-0xEB */ + 0xB8, 0x79, 0xBD, 0xDF, 0xB8, 0x7A, 0xB6, 0xCB, /* 0xEC-0xEF */ + 0xB8, 0x7B, 0xB8, 0x7C, 0xB8, 0x7D, 0xB8, 0x7E, /* 0xF0-0xF3 */ + 0xB8, 0x80, 0xB8, 0x81, 0xB8, 0x82, 0xB8, 0x83, /* 0xF4-0xF7 */ + 0xB8, 0x84, 0xD6, 0xF1, 0xF3, 0xC3, 0xB8, 0x85, /* 0xF8-0xFB */ + 0xB8, 0x86, 0xF3, 0xC4, 0xB8, 0x87, 0xB8, 0xCD, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_7B[512] = { + 0xB8, 0x88, 0xB8, 0x89, 0xB8, 0x8A, 0xF3, 0xC6, /* 0x00-0x03 */ + 0xF3, 0xC7, 0xB8, 0x8B, 0xB0, 0xCA, 0xB8, 0x8C, /* 0x04-0x07 */ + 0xF3, 0xC5, 0xB8, 0x8D, 0xF3, 0xC9, 0xCB, 0xF1, /* 0x08-0x0B */ + 0xB8, 0x8E, 0xB8, 0x8F, 0xB8, 0x90, 0xF3, 0xCB, /* 0x0C-0x0F */ + 0xB8, 0x91, 0xD0, 0xA6, 0xB8, 0x92, 0xB8, 0x93, /* 0x10-0x13 */ + 0xB1, 0xCA, 0xF3, 0xC8, 0xB8, 0x94, 0xB8, 0x95, /* 0x14-0x17 */ + 0xB8, 0x96, 0xF3, 0xCF, 0xB8, 0x97, 0xB5, 0xD1, /* 0x18-0x1B */ + 0xB8, 0x98, 0xB8, 0x99, 0xF3, 0xD7, 0xB8, 0x9A, /* 0x1C-0x1F */ + 0xF3, 0xD2, 0xB8, 0x9B, 0xB8, 0x9C, 0xB8, 0x9D, /* 0x20-0x23 */ + 0xF3, 0xD4, 0xF3, 0xD3, 0xB7, 0xFB, 0xB8, 0x9E, /* 0x24-0x27 */ + 0xB1, 0xBF, 0xB8, 0x9F, 0xF3, 0xCE, 0xF3, 0xCA, /* 0x28-0x2B */ + 0xB5, 0xDA, 0xB8, 0xA0, 0xF3, 0xD0, 0xB9, 0x40, /* 0x2C-0x2F */ + 0xB9, 0x41, 0xF3, 0xD1, 0xB9, 0x42, 0xF3, 0xD5, /* 0x30-0x33 */ + 0xB9, 0x43, 0xB9, 0x44, 0xB9, 0x45, 0xB9, 0x46, /* 0x34-0x37 */ + 0xF3, 0xCD, 0xB9, 0x47, 0xBC, 0xE3, 0xB9, 0x48, /* 0x38-0x3B */ + 0xC1, 0xFD, 0xB9, 0x49, 0xF3, 0xD6, 0xB9, 0x4A, /* 0x3C-0x3F */ + 0xB9, 0x4B, 0xB9, 0x4C, 0xB9, 0x4D, 0xB9, 0x4E, /* 0x40-0x43 */ + 0xB9, 0x4F, 0xF3, 0xDA, 0xB9, 0x50, 0xF3, 0xCC, /* 0x44-0x47 */ + 0xB9, 0x51, 0xB5, 0xC8, 0xB9, 0x52, 0xBD, 0xEE, /* 0x48-0x4B */ + 0xF3, 0xDC, 0xB9, 0x53, 0xB9, 0x54, 0xB7, 0xA4, /* 0x4C-0x4F */ + 0xBF, 0xF0, 0xD6, 0xFE, 0xCD, 0xB2, 0xB9, 0x55, /* 0x50-0x53 */ + 0xB4, 0xF0, 0xB9, 0x56, 0xB2, 0xDF, 0xB9, 0x57, /* 0x54-0x57 */ + 0xF3, 0xD8, 0xB9, 0x58, 0xF3, 0xD9, 0xC9, 0xB8, /* 0x58-0x5B */ + 0xB9, 0x59, 0xF3, 0xDD, 0xB9, 0x5A, 0xB9, 0x5B, /* 0x5C-0x5F */ + 0xF3, 0xDE, 0xB9, 0x5C, 0xF3, 0xE1, 0xB9, 0x5D, /* 0x60-0x63 */ + 0xB9, 0x5E, 0xB9, 0x5F, 0xB9, 0x60, 0xB9, 0x61, /* 0x64-0x67 */ + 0xB9, 0x62, 0xB9, 0x63, 0xB9, 0x64, 0xB9, 0x65, /* 0x68-0x6B */ + 0xB9, 0x66, 0xB9, 0x67, 0xF3, 0xDF, 0xB9, 0x68, /* 0x6C-0x6F */ + 0xB9, 0x69, 0xF3, 0xE3, 0xF3, 0xE2, 0xB9, 0x6A, /* 0x70-0x73 */ + 0xB9, 0x6B, 0xF3, 0xDB, 0xB9, 0x6C, 0xBF, 0xEA, /* 0x74-0x77 */ + 0xB9, 0x6D, 0xB3, 0xEF, 0xB9, 0x6E, 0xF3, 0xE0, /* 0x78-0x7B */ + 0xB9, 0x6F, 0xB9, 0x70, 0xC7, 0xA9, 0xB9, 0x71, /* 0x7C-0x7F */ + + 0xBC, 0xF2, 0xB9, 0x72, 0xB9, 0x73, 0xB9, 0x74, /* 0x80-0x83 */ + 0xB9, 0x75, 0xF3, 0xEB, 0xB9, 0x76, 0xB9, 0x77, /* 0x84-0x87 */ + 0xB9, 0x78, 0xB9, 0x79, 0xB9, 0x7A, 0xB9, 0x7B, /* 0x88-0x8B */ + 0xB9, 0x7C, 0xB9, 0xBF, 0xB9, 0x7D, 0xB9, 0x7E, /* 0x8C-0x8F */ + 0xF3, 0xE4, 0xB9, 0x80, 0xB9, 0x81, 0xB9, 0x82, /* 0x90-0x93 */ + 0xB2, 0xAD, 0xBB, 0xFE, 0xB9, 0x83, 0xCB, 0xE3, /* 0x94-0x97 */ + 0xB9, 0x84, 0xB9, 0x85, 0xB9, 0x86, 0xB9, 0x87, /* 0x98-0x9B */ + 0xF3, 0xED, 0xF3, 0xE9, 0xB9, 0x88, 0xB9, 0x89, /* 0x9C-0x9F */ + 0xB9, 0x8A, 0xB9, 0xDC, 0xF3, 0xEE, 0xB9, 0x8B, /* 0xA0-0xA3 */ + 0xB9, 0x8C, 0xB9, 0x8D, 0xF3, 0xE5, 0xF3, 0xE6, /* 0xA4-0xA7 */ + 0xF3, 0xEA, 0xC2, 0xE1, 0xF3, 0xEC, 0xF3, 0xEF, /* 0xA8-0xAB */ + 0xF3, 0xE8, 0xBC, 0xFD, 0xB9, 0x8E, 0xB9, 0x8F, /* 0xAC-0xAF */ + 0xB9, 0x90, 0xCF, 0xE4, 0xB9, 0x91, 0xB9, 0x92, /* 0xB0-0xB3 */ + 0xF3, 0xF0, 0xB9, 0x93, 0xB9, 0x94, 0xB9, 0x95, /* 0xB4-0xB7 */ + 0xF3, 0xE7, 0xB9, 0x96, 0xB9, 0x97, 0xB9, 0x98, /* 0xB8-0xBB */ + 0xB9, 0x99, 0xB9, 0x9A, 0xB9, 0x9B, 0xB9, 0x9C, /* 0xBC-0xBF */ + 0xB9, 0x9D, 0xF3, 0xF2, 0xB9, 0x9E, 0xB9, 0x9F, /* 0xC0-0xC3 */ + 0xB9, 0xA0, 0xBA, 0x40, 0xD7, 0xAD, 0xC6, 0xAA, /* 0xC4-0xC7 */ + 0xBA, 0x41, 0xBA, 0x42, 0xBA, 0x43, 0xBA, 0x44, /* 0xC8-0xCB */ + 0xF3, 0xF3, 0xBA, 0x45, 0xBA, 0x46, 0xBA, 0x47, /* 0xCC-0xCF */ + 0xBA, 0x48, 0xF3, 0xF1, 0xBA, 0x49, 0xC2, 0xA8, /* 0xD0-0xD3 */ + 0xBA, 0x4A, 0xBA, 0x4B, 0xBA, 0x4C, 0xBA, 0x4D, /* 0xD4-0xD7 */ + 0xBA, 0x4E, 0xB8, 0xDD, 0xF3, 0xF5, 0xBA, 0x4F, /* 0xD8-0xDB */ + 0xBA, 0x50, 0xF3, 0xF4, 0xBA, 0x51, 0xBA, 0x52, /* 0xDC-0xDF */ + 0xBA, 0x53, 0xB4, 0xDB, 0xBA, 0x54, 0xBA, 0x55, /* 0xE0-0xE3 */ + 0xBA, 0x56, 0xF3, 0xF6, 0xF3, 0xF7, 0xBA, 0x57, /* 0xE4-0xE7 */ + 0xBA, 0x58, 0xBA, 0x59, 0xF3, 0xF8, 0xBA, 0x5A, /* 0xE8-0xEB */ + 0xBA, 0x5B, 0xBA, 0x5C, 0xC0, 0xBA, 0xBA, 0x5D, /* 0xEC-0xEF */ + 0xBA, 0x5E, 0xC0, 0xE9, 0xBA, 0x5F, 0xBA, 0x60, /* 0xF0-0xF3 */ + 0xBA, 0x61, 0xBA, 0x62, 0xBA, 0x63, 0xC5, 0xF1, /* 0xF4-0xF7 */ + 0xBA, 0x64, 0xBA, 0x65, 0xBA, 0x66, 0xBA, 0x67, /* 0xF8-0xFB */ + 0xF3, 0xFB, 0xBA, 0x68, 0xF3, 0xFA, 0xBA, 0x69, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_7C[512] = { + 0xBA, 0x6A, 0xBA, 0x6B, 0xBA, 0x6C, 0xBA, 0x6D, /* 0x00-0x03 */ + 0xBA, 0x6E, 0xBA, 0x6F, 0xBA, 0x70, 0xB4, 0xD8, /* 0x04-0x07 */ + 0xBA, 0x71, 0xBA, 0x72, 0xBA, 0x73, 0xF3, 0xFE, /* 0x08-0x0B */ + 0xF3, 0xF9, 0xBA, 0x74, 0xBA, 0x75, 0xF3, 0xFC, /* 0x0C-0x0F */ + 0xBA, 0x76, 0xBA, 0x77, 0xBA, 0x78, 0xBA, 0x79, /* 0x10-0x13 */ + 0xBA, 0x7A, 0xBA, 0x7B, 0xF3, 0xFD, 0xBA, 0x7C, /* 0x14-0x17 */ + 0xBA, 0x7D, 0xBA, 0x7E, 0xBA, 0x80, 0xBA, 0x81, /* 0x18-0x1B */ + 0xBA, 0x82, 0xBA, 0x83, 0xBA, 0x84, 0xF4, 0xA1, /* 0x1C-0x1F */ + 0xBA, 0x85, 0xBA, 0x86, 0xBA, 0x87, 0xBA, 0x88, /* 0x20-0x23 */ + 0xBA, 0x89, 0xBA, 0x8A, 0xF4, 0xA3, 0xBB, 0xC9, /* 0x24-0x27 */ + 0xBA, 0x8B, 0xBA, 0x8C, 0xF4, 0xA2, 0xBA, 0x8D, /* 0x28-0x2B */ + 0xBA, 0x8E, 0xBA, 0x8F, 0xBA, 0x90, 0xBA, 0x91, /* 0x2C-0x2F */ + 0xBA, 0x92, 0xBA, 0x93, 0xBA, 0x94, 0xBA, 0x95, /* 0x30-0x33 */ + 0xBA, 0x96, 0xBA, 0x97, 0xBA, 0x98, 0xBA, 0x99, /* 0x34-0x37 */ + 0xF4, 0xA4, 0xBA, 0x9A, 0xBA, 0x9B, 0xBA, 0x9C, /* 0x38-0x3B */ + 0xBA, 0x9D, 0xBA, 0x9E, 0xBA, 0x9F, 0xB2, 0xBE, /* 0x3C-0x3F */ + 0xF4, 0xA6, 0xF4, 0xA5, 0xBA, 0xA0, 0xBB, 0x40, /* 0x40-0x43 */ + 0xBB, 0x41, 0xBB, 0x42, 0xBB, 0x43, 0xBB, 0x44, /* 0x44-0x47 */ + 0xBB, 0x45, 0xBB, 0x46, 0xBB, 0x47, 0xBB, 0x48, /* 0x48-0x4B */ + 0xBB, 0x49, 0xBC, 0xAE, 0xBB, 0x4A, 0xBB, 0x4B, /* 0x4C-0x4F */ + 0xBB, 0x4C, 0xBB, 0x4D, 0xBB, 0x4E, 0xBB, 0x4F, /* 0x50-0x53 */ + 0xBB, 0x50, 0xBB, 0x51, 0xBB, 0x52, 0xBB, 0x53, /* 0x54-0x57 */ + 0xBB, 0x54, 0xBB, 0x55, 0xBB, 0x56, 0xBB, 0x57, /* 0x58-0x5B */ + 0xBB, 0x58, 0xBB, 0x59, 0xBB, 0x5A, 0xBB, 0x5B, /* 0x5C-0x5F */ + 0xBB, 0x5C, 0xBB, 0x5D, 0xBB, 0x5E, 0xBB, 0x5F, /* 0x60-0x63 */ + 0xBB, 0x60, 0xBB, 0x61, 0xBB, 0x62, 0xBB, 0x63, /* 0x64-0x67 */ + 0xBB, 0x64, 0xBB, 0x65, 0xBB, 0x66, 0xBB, 0x67, /* 0x68-0x6B */ + 0xBB, 0x68, 0xBB, 0x69, 0xBB, 0x6A, 0xBB, 0x6B, /* 0x6C-0x6F */ + 0xBB, 0x6C, 0xBB, 0x6D, 0xBB, 0x6E, 0xC3, 0xD7, /* 0x70-0x73 */ + 0xD9, 0xE1, 0xBB, 0x6F, 0xBB, 0x70, 0xBB, 0x71, /* 0x74-0x77 */ + 0xBB, 0x72, 0xBB, 0x73, 0xBB, 0x74, 0xC0, 0xE0, /* 0x78-0x7B */ + 0xF4, 0xCC, 0xD7, 0xD1, 0xBB, 0x75, 0xBB, 0x76, /* 0x7C-0x7F */ + + 0xBB, 0x77, 0xBB, 0x78, 0xBB, 0x79, 0xBB, 0x7A, /* 0x80-0x83 */ + 0xBB, 0x7B, 0xBB, 0x7C, 0xBB, 0x7D, 0xBB, 0x7E, /* 0x84-0x87 */ + 0xBB, 0x80, 0xB7, 0xDB, 0xBB, 0x81, 0xBB, 0x82, /* 0x88-0x8B */ + 0xBB, 0x83, 0xBB, 0x84, 0xBB, 0x85, 0xBB, 0x86, /* 0x8C-0x8F */ + 0xBB, 0x87, 0xF4, 0xCE, 0xC1, 0xA3, 0xBB, 0x88, /* 0x90-0x93 */ + 0xBB, 0x89, 0xC6, 0xC9, 0xBB, 0x8A, 0xB4, 0xD6, /* 0x94-0x97 */ + 0xD5, 0xB3, 0xBB, 0x8B, 0xBB, 0x8C, 0xBB, 0x8D, /* 0x98-0x9B */ + 0xF4, 0xD0, 0xF4, 0xCF, 0xF4, 0xD1, 0xCB, 0xDA, /* 0x9C-0x9F */ + 0xBB, 0x8E, 0xBB, 0x8F, 0xF4, 0xD2, 0xBB, 0x90, /* 0xA0-0xA3 */ + 0xD4, 0xC1, 0xD6, 0xE0, 0xBB, 0x91, 0xBB, 0x92, /* 0xA4-0xA7 */ + 0xBB, 0x93, 0xBB, 0x94, 0xB7, 0xE0, 0xBB, 0x95, /* 0xA8-0xAB */ + 0xBB, 0x96, 0xBB, 0x97, 0xC1, 0xB8, 0xBB, 0x98, /* 0xAC-0xAF */ + 0xBB, 0x99, 0xC1, 0xBB, 0xF4, 0xD3, 0xBE, 0xAC, /* 0xB0-0xB3 */ + 0xBB, 0x9A, 0xBB, 0x9B, 0xBB, 0x9C, 0xBB, 0x9D, /* 0xB4-0xB7 */ + 0xBB, 0x9E, 0xB4, 0xE2, 0xBB, 0x9F, 0xBB, 0xA0, /* 0xB8-0xBB */ + 0xF4, 0xD4, 0xF4, 0xD5, 0xBE, 0xAB, 0xBC, 0x40, /* 0xBC-0xBF */ + 0xBC, 0x41, 0xF4, 0xD6, 0xBC, 0x42, 0xBC, 0x43, /* 0xC0-0xC3 */ + 0xBC, 0x44, 0xF4, 0xDB, 0xBC, 0x45, 0xF4, 0xD7, /* 0xC4-0xC7 */ + 0xF4, 0xDA, 0xBC, 0x46, 0xBA, 0xFD, 0xBC, 0x47, /* 0xC8-0xCB */ + 0xF4, 0xD8, 0xF4, 0xD9, 0xBC, 0x48, 0xBC, 0x49, /* 0xCC-0xCF */ + 0xBC, 0x4A, 0xBC, 0x4B, 0xBC, 0x4C, 0xBC, 0x4D, /* 0xD0-0xD3 */ + 0xBC, 0x4E, 0xB8, 0xE2, 0xCC, 0xC7, 0xF4, 0xDC, /* 0xD4-0xD7 */ + 0xBC, 0x4F, 0xB2, 0xDA, 0xBC, 0x50, 0xBC, 0x51, /* 0xD8-0xDB */ + 0xC3, 0xD3, 0xBC, 0x52, 0xBC, 0x53, 0xD4, 0xE3, /* 0xDC-0xDF */ + 0xBF, 0xB7, 0xBC, 0x54, 0xBC, 0x55, 0xBC, 0x56, /* 0xE0-0xE3 */ + 0xBC, 0x57, 0xBC, 0x58, 0xBC, 0x59, 0xBC, 0x5A, /* 0xE4-0xE7 */ + 0xF4, 0xDD, 0xBC, 0x5B, 0xBC, 0x5C, 0xBC, 0x5D, /* 0xE8-0xEB */ + 0xBC, 0x5E, 0xBC, 0x5F, 0xBC, 0x60, 0xC5, 0xB4, /* 0xEC-0xEF */ + 0xBC, 0x61, 0xBC, 0x62, 0xBC, 0x63, 0xBC, 0x64, /* 0xF0-0xF3 */ + 0xBC, 0x65, 0xBC, 0x66, 0xBC, 0x67, 0xBC, 0x68, /* 0xF4-0xF7 */ + 0xF4, 0xE9, 0xBC, 0x69, 0xBC, 0x6A, 0xCF, 0xB5, /* 0xF8-0xFB */ + 0xBC, 0x6B, 0xBC, 0x6C, 0xBC, 0x6D, 0xBC, 0x6E, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_7D[512] = { + 0xBC, 0x6F, 0xBC, 0x70, 0xBC, 0x71, 0xBC, 0x72, /* 0x00-0x03 */ + 0xBC, 0x73, 0xBC, 0x74, 0xBC, 0x75, 0xBC, 0x76, /* 0x04-0x07 */ + 0xBC, 0x77, 0xBC, 0x78, 0xCE, 0xC9, 0xBC, 0x79, /* 0x08-0x0B */ + 0xBC, 0x7A, 0xBC, 0x7B, 0xBC, 0x7C, 0xBC, 0x7D, /* 0x0C-0x0F */ + 0xBC, 0x7E, 0xBC, 0x80, 0xBC, 0x81, 0xBC, 0x82, /* 0x10-0x13 */ + 0xBC, 0x83, 0xBC, 0x84, 0xBC, 0x85, 0xBC, 0x86, /* 0x14-0x17 */ + 0xBC, 0x87, 0xBC, 0x88, 0xBC, 0x89, 0xBC, 0x8A, /* 0x18-0x1B */ + 0xBC, 0x8B, 0xBC, 0x8C, 0xBC, 0x8D, 0xBC, 0x8E, /* 0x1C-0x1F */ + 0xCB, 0xD8, 0xBC, 0x8F, 0xCB, 0xF7, 0xBC, 0x90, /* 0x20-0x23 */ + 0xBC, 0x91, 0xBC, 0x92, 0xBC, 0x93, 0xBD, 0xF4, /* 0x24-0x27 */ + 0xBC, 0x94, 0xBC, 0x95, 0xBC, 0x96, 0xD7, 0xCF, /* 0x28-0x2B */ + 0xBC, 0x97, 0xBC, 0x98, 0xBC, 0x99, 0xC0, 0xDB, /* 0x2C-0x2F */ + 0xBC, 0x9A, 0xBC, 0x9B, 0xBC, 0x9C, 0xBC, 0x9D, /* 0x30-0x33 */ + 0xBC, 0x9E, 0xBC, 0x9F, 0xBC, 0xA0, 0xBD, 0x40, /* 0x34-0x37 */ + 0xBD, 0x41, 0xBD, 0x42, 0xBD, 0x43, 0xBD, 0x44, /* 0x38-0x3B */ + 0xBD, 0x45, 0xBD, 0x46, 0xBD, 0x47, 0xBD, 0x48, /* 0x3C-0x3F */ + 0xBD, 0x49, 0xBD, 0x4A, 0xBD, 0x4B, 0xBD, 0x4C, /* 0x40-0x43 */ + 0xBD, 0x4D, 0xBD, 0x4E, 0xBD, 0x4F, 0xBD, 0x50, /* 0x44-0x47 */ + 0xBD, 0x51, 0xBD, 0x52, 0xBD, 0x53, 0xBD, 0x54, /* 0x48-0x4B */ + 0xBD, 0x55, 0xBD, 0x56, 0xBD, 0x57, 0xBD, 0x58, /* 0x4C-0x4F */ + 0xBD, 0x59, 0xBD, 0x5A, 0xBD, 0x5B, 0xBD, 0x5C, /* 0x50-0x53 */ + 0xBD, 0x5D, 0xBD, 0x5E, 0xBD, 0x5F, 0xBD, 0x60, /* 0x54-0x57 */ + 0xBD, 0x61, 0xBD, 0x62, 0xBD, 0x63, 0xBD, 0x64, /* 0x58-0x5B */ + 0xBD, 0x65, 0xBD, 0x66, 0xBD, 0x67, 0xBD, 0x68, /* 0x5C-0x5F */ + 0xBD, 0x69, 0xBD, 0x6A, 0xBD, 0x6B, 0xBD, 0x6C, /* 0x60-0x63 */ + 0xBD, 0x6D, 0xBD, 0x6E, 0xBD, 0x6F, 0xBD, 0x70, /* 0x64-0x67 */ + 0xBD, 0x71, 0xBD, 0x72, 0xBD, 0x73, 0xBD, 0x74, /* 0x68-0x6B */ + 0xBD, 0x75, 0xBD, 0x76, 0xD0, 0xF5, 0xBD, 0x77, /* 0x6C-0x6F */ + 0xBD, 0x78, 0xBD, 0x79, 0xBD, 0x7A, 0xBD, 0x7B, /* 0x70-0x73 */ + 0xBD, 0x7C, 0xBD, 0x7D, 0xBD, 0x7E, 0xF4, 0xEA, /* 0x74-0x77 */ + 0xBD, 0x80, 0xBD, 0x81, 0xBD, 0x82, 0xBD, 0x83, /* 0x78-0x7B */ + 0xBD, 0x84, 0xBD, 0x85, 0xBD, 0x86, 0xBD, 0x87, /* 0x7C-0x7F */ + + 0xBD, 0x88, 0xBD, 0x89, 0xBD, 0x8A, 0xBD, 0x8B, /* 0x80-0x83 */ + 0xBD, 0x8C, 0xBD, 0x8D, 0xBD, 0x8E, 0xBD, 0x8F, /* 0x84-0x87 */ + 0xBD, 0x90, 0xBD, 0x91, 0xBD, 0x92, 0xBD, 0x93, /* 0x88-0x8B */ + 0xBD, 0x94, 0xBD, 0x95, 0xBD, 0x96, 0xBD, 0x97, /* 0x8C-0x8F */ + 0xBD, 0x98, 0xBD, 0x99, 0xBD, 0x9A, 0xBD, 0x9B, /* 0x90-0x93 */ + 0xBD, 0x9C, 0xBD, 0x9D, 0xBD, 0x9E, 0xBD, 0x9F, /* 0x94-0x97 */ + 0xBD, 0xA0, 0xBE, 0x40, 0xBE, 0x41, 0xBE, 0x42, /* 0x98-0x9B */ + 0xBE, 0x43, 0xBE, 0x44, 0xBE, 0x45, 0xBE, 0x46, /* 0x9C-0x9F */ + 0xBE, 0x47, 0xBE, 0x48, 0xBE, 0x49, 0xBE, 0x4A, /* 0xA0-0xA3 */ + 0xBE, 0x4B, 0xBE, 0x4C, 0xF4, 0xEB, 0xBE, 0x4D, /* 0xA4-0xA7 */ + 0xBE, 0x4E, 0xBE, 0x4F, 0xBE, 0x50, 0xBE, 0x51, /* 0xA8-0xAB */ + 0xBE, 0x52, 0xBE, 0x53, 0xF4, 0xEC, 0xBE, 0x54, /* 0xAC-0xAF */ + 0xBE, 0x55, 0xBE, 0x56, 0xBE, 0x57, 0xBE, 0x58, /* 0xB0-0xB3 */ + 0xBE, 0x59, 0xBE, 0x5A, 0xBE, 0x5B, 0xBE, 0x5C, /* 0xB4-0xB7 */ + 0xBE, 0x5D, 0xBE, 0x5E, 0xBE, 0x5F, 0xBE, 0x60, /* 0xB8-0xBB */ + 0xBE, 0x61, 0xBE, 0x62, 0xBE, 0x63, 0xBE, 0x64, /* 0xBC-0xBF */ + 0xBE, 0x65, 0xBE, 0x66, 0xBE, 0x67, 0xBE, 0x68, /* 0xC0-0xC3 */ + 0xBE, 0x69, 0xBE, 0x6A, 0xBE, 0x6B, 0xBE, 0x6C, /* 0xC4-0xC7 */ + 0xBE, 0x6D, 0xBE, 0x6E, 0xBE, 0x6F, 0xBE, 0x70, /* 0xC8-0xCB */ + 0xBE, 0x71, 0xBE, 0x72, 0xBE, 0x73, 0xBE, 0x74, /* 0xCC-0xCF */ + 0xBE, 0x75, 0xBE, 0x76, 0xBE, 0x77, 0xBE, 0x78, /* 0xD0-0xD3 */ + 0xBE, 0x79, 0xBE, 0x7A, 0xBE, 0x7B, 0xBE, 0x7C, /* 0xD4-0xD7 */ + 0xBE, 0x7D, 0xBE, 0x7E, 0xBE, 0x80, 0xBE, 0x81, /* 0xD8-0xDB */ + 0xBE, 0x82, 0xBE, 0x83, 0xBE, 0x84, 0xBE, 0x85, /* 0xDC-0xDF */ + 0xBE, 0x86, 0xBE, 0x87, 0xBE, 0x88, 0xBE, 0x89, /* 0xE0-0xE3 */ + 0xBE, 0x8A, 0xBE, 0x8B, 0xBE, 0x8C, 0xBE, 0x8D, /* 0xE4-0xE7 */ + 0xBE, 0x8E, 0xBE, 0x8F, 0xBE, 0x90, 0xBE, 0x91, /* 0xE8-0xEB */ + 0xBE, 0x92, 0xBE, 0x93, 0xBE, 0x94, 0xBE, 0x95, /* 0xEC-0xEF */ + 0xBE, 0x96, 0xBE, 0x97, 0xBE, 0x98, 0xBE, 0x99, /* 0xF0-0xF3 */ + 0xBE, 0x9A, 0xBE, 0x9B, 0xBE, 0x9C, 0xBE, 0x9D, /* 0xF4-0xF7 */ + 0xBE, 0x9E, 0xBE, 0x9F, 0xBE, 0xA0, 0xBF, 0x40, /* 0xF8-0xFB */ + 0xBF, 0x41, 0xBF, 0x42, 0xBF, 0x43, 0xBF, 0x44, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_7E[512] = { + 0xBF, 0x45, 0xBF, 0x46, 0xBF, 0x47, 0xBF, 0x48, /* 0x00-0x03 */ + 0xBF, 0x49, 0xBF, 0x4A, 0xBF, 0x4B, 0xBF, 0x4C, /* 0x04-0x07 */ + 0xBF, 0x4D, 0xBF, 0x4E, 0xBF, 0x4F, 0xBF, 0x50, /* 0x08-0x0B */ + 0xBF, 0x51, 0xBF, 0x52, 0xBF, 0x53, 0xBF, 0x54, /* 0x0C-0x0F */ + 0xBF, 0x55, 0xBF, 0x56, 0xBF, 0x57, 0xBF, 0x58, /* 0x10-0x13 */ + 0xBF, 0x59, 0xBF, 0x5A, 0xBF, 0x5B, 0xBF, 0x5C, /* 0x14-0x17 */ + 0xBF, 0x5D, 0xBF, 0x5E, 0xBF, 0x5F, 0xBF, 0x60, /* 0x18-0x1B */ + 0xBF, 0x61, 0xBF, 0x62, 0xBF, 0x63, 0xBF, 0x64, /* 0x1C-0x1F */ + 0xBF, 0x65, 0xBF, 0x66, 0xBF, 0x67, 0xBF, 0x68, /* 0x20-0x23 */ + 0xBF, 0x69, 0xBF, 0x6A, 0xBF, 0x6B, 0xBF, 0x6C, /* 0x24-0x27 */ + 0xBF, 0x6D, 0xBF, 0x6E, 0xBF, 0x6F, 0xBF, 0x70, /* 0x28-0x2B */ + 0xBF, 0x71, 0xBF, 0x72, 0xBF, 0x73, 0xBF, 0x74, /* 0x2C-0x2F */ + 0xBF, 0x75, 0xBF, 0x76, 0xBF, 0x77, 0xBF, 0x78, /* 0x30-0x33 */ + 0xBF, 0x79, 0xBF, 0x7A, 0xBF, 0x7B, 0xBF, 0x7C, /* 0x34-0x37 */ + 0xBF, 0x7D, 0xBF, 0x7E, 0xBF, 0x80, 0xF7, 0xE3, /* 0x38-0x3B */ + 0xBF, 0x81, 0xBF, 0x82, 0xBF, 0x83, 0xBF, 0x84, /* 0x3C-0x3F */ + 0xBF, 0x85, 0xB7, 0xB1, 0xBF, 0x86, 0xBF, 0x87, /* 0x40-0x43 */ + 0xBF, 0x88, 0xBF, 0x89, 0xBF, 0x8A, 0xF4, 0xED, /* 0x44-0x47 */ + 0xBF, 0x8B, 0xBF, 0x8C, 0xBF, 0x8D, 0xBF, 0x8E, /* 0x48-0x4B */ + 0xBF, 0x8F, 0xBF, 0x90, 0xBF, 0x91, 0xBF, 0x92, /* 0x4C-0x4F */ + 0xBF, 0x93, 0xBF, 0x94, 0xBF, 0x95, 0xBF, 0x96, /* 0x50-0x53 */ + 0xBF, 0x97, 0xBF, 0x98, 0xBF, 0x99, 0xBF, 0x9A, /* 0x54-0x57 */ + 0xBF, 0x9B, 0xBF, 0x9C, 0xBF, 0x9D, 0xBF, 0x9E, /* 0x58-0x5B */ + 0xBF, 0x9F, 0xBF, 0xA0, 0xC0, 0x40, 0xC0, 0x41, /* 0x5C-0x5F */ + 0xC0, 0x42, 0xC0, 0x43, 0xC0, 0x44, 0xC0, 0x45, /* 0x60-0x63 */ + 0xC0, 0x46, 0xC0, 0x47, 0xC0, 0x48, 0xC0, 0x49, /* 0x64-0x67 */ + 0xC0, 0x4A, 0xC0, 0x4B, 0xC0, 0x4C, 0xC0, 0x4D, /* 0x68-0x6B */ + 0xC0, 0x4E, 0xC0, 0x4F, 0xC0, 0x50, 0xC0, 0x51, /* 0x6C-0x6F */ + 0xC0, 0x52, 0xC0, 0x53, 0xC0, 0x54, 0xC0, 0x55, /* 0x70-0x73 */ + 0xC0, 0x56, 0xC0, 0x57, 0xC0, 0x58, 0xC0, 0x59, /* 0x74-0x77 */ + 0xC0, 0x5A, 0xC0, 0x5B, 0xC0, 0x5C, 0xC0, 0x5D, /* 0x78-0x7B */ + 0xC0, 0x5E, 0xC0, 0x5F, 0xC0, 0x60, 0xC0, 0x61, /* 0x7C-0x7F */ + + 0xC0, 0x62, 0xC0, 0x63, 0xD7, 0xEB, 0xC0, 0x64, /* 0x80-0x83 */ + 0xC0, 0x65, 0xC0, 0x66, 0xC0, 0x67, 0xC0, 0x68, /* 0x84-0x87 */ + 0xC0, 0x69, 0xC0, 0x6A, 0xC0, 0x6B, 0xC0, 0x6C, /* 0x88-0x8B */ + 0xC0, 0x6D, 0xC0, 0x6E, 0xC0, 0x6F, 0xC0, 0x70, /* 0x8C-0x8F */ + 0xC0, 0x71, 0xC0, 0x72, 0xC0, 0x73, 0xC0, 0x74, /* 0x90-0x93 */ + 0xC0, 0x75, 0xC0, 0x76, 0xC0, 0x77, 0xC0, 0x78, /* 0x94-0x97 */ + 0xC0, 0x79, 0xC0, 0x7A, 0xC0, 0x7B, 0xF4, 0xEE, /* 0x98-0x9B */ + 0xC0, 0x7C, 0xC0, 0x7D, 0xC0, 0x7E, 0xE6, 0xF9, /* 0x9C-0x9F */ + 0xBE, 0xC0, 0xE6, 0xFA, 0xBA, 0xEC, 0xE6, 0xFB, /* 0xA0-0xA3 */ + 0xCF, 0xCB, 0xE6, 0xFC, 0xD4, 0xBC, 0xBC, 0xB6, /* 0xA4-0xA7 */ + 0xE6, 0xFD, 0xE6, 0xFE, 0xBC, 0xCD, 0xC8, 0xD2, /* 0xA8-0xAB */ + 0xCE, 0xB3, 0xE7, 0xA1, 0xC0, 0x80, 0xB4, 0xBF, /* 0xAC-0xAF */ + 0xE7, 0xA2, 0xC9, 0xB4, 0xB8, 0xD9, 0xC4, 0xC9, /* 0xB0-0xB3 */ + 0xC0, 0x81, 0xD7, 0xDD, 0xC2, 0xDA, 0xB7, 0xD7, /* 0xB4-0xB7 */ + 0xD6, 0xBD, 0xCE, 0xC6, 0xB7, 0xC4, 0xC0, 0x82, /* 0xB8-0xBB */ + 0xC0, 0x83, 0xC5, 0xA6, 0xE7, 0xA3, 0xCF, 0xDF, /* 0xBC-0xBF */ + 0xE7, 0xA4, 0xE7, 0xA5, 0xE7, 0xA6, 0xC1, 0xB7, /* 0xC0-0xC3 */ + 0xD7, 0xE9, 0xC9, 0xF0, 0xCF, 0xB8, 0xD6, 0xAF, /* 0xC4-0xC7 */ + 0xD6, 0xD5, 0xE7, 0xA7, 0xB0, 0xED, 0xE7, 0xA8, /* 0xC8-0xCB */ + 0xE7, 0xA9, 0xC9, 0xDC, 0xD2, 0xEF, 0xBE, 0xAD, /* 0xCC-0xCF */ + 0xE7, 0xAA, 0xB0, 0xF3, 0xC8, 0xDE, 0xBD, 0xE1, /* 0xD0-0xD3 */ + 0xE7, 0xAB, 0xC8, 0xC6, 0xC0, 0x84, 0xE7, 0xAC, /* 0xD4-0xD7 */ + 0xBB, 0xE6, 0xB8, 0xF8, 0xD1, 0xA4, 0xE7, 0xAD, /* 0xD8-0xDB */ + 0xC2, 0xE7, 0xBE, 0xF8, 0xBD, 0xCA, 0xCD, 0xB3, /* 0xDC-0xDF */ + 0xE7, 0xAE, 0xE7, 0xAF, 0xBE, 0xEE, 0xD0, 0xE5, /* 0xE0-0xE3 */ + 0xC0, 0x85, 0xCB, 0xE7, 0xCC, 0xD0, 0xBC, 0xCC, /* 0xE4-0xE7 */ + 0xE7, 0xB0, 0xBC, 0xA8, 0xD0, 0xF7, 0xE7, 0xB1, /* 0xE8-0xEB */ + 0xC0, 0x86, 0xD0, 0xF8, 0xE7, 0xB2, 0xE7, 0xB3, /* 0xEC-0xEF */ + 0xB4, 0xC2, 0xE7, 0xB4, 0xE7, 0xB5, 0xC9, 0xFE, /* 0xF0-0xF3 */ + 0xCE, 0xAC, 0xC3, 0xE0, 0xE7, 0xB7, 0xB1, 0xC1, /* 0xF4-0xF7 */ + 0xB3, 0xF1, 0xC0, 0x87, 0xE7, 0xB8, 0xE7, 0xB9, /* 0xF8-0xFB */ + 0xD7, 0xDB, 0xD5, 0xC0, 0xE7, 0xBA, 0xC2, 0xCC, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_7F[512] = { + 0xD7, 0xBA, 0xE7, 0xBB, 0xE7, 0xBC, 0xE7, 0xBD, /* 0x00-0x03 */ + 0xBC, 0xEA, 0xC3, 0xE5, 0xC0, 0xC2, 0xE7, 0xBE, /* 0x04-0x07 */ + 0xE7, 0xBF, 0xBC, 0xA9, 0xC0, 0x88, 0xE7, 0xC0, /* 0x08-0x0B */ + 0xE7, 0xC1, 0xE7, 0xB6, 0xB6, 0xD0, 0xE7, 0xC2, /* 0x0C-0x0F */ + 0xC0, 0x89, 0xE7, 0xC3, 0xE7, 0xC4, 0xBB, 0xBA, /* 0x10-0x13 */ + 0xB5, 0xDE, 0xC2, 0xC6, 0xB1, 0xE0, 0xE7, 0xC5, /* 0x14-0x17 */ + 0xD4, 0xB5, 0xE7, 0xC6, 0xB8, 0xBF, 0xE7, 0xC8, /* 0x18-0x1B */ + 0xE7, 0xC7, 0xB7, 0xEC, 0xC0, 0x8A, 0xE7, 0xC9, /* 0x1C-0x1F */ + 0xB2, 0xF8, 0xE7, 0xCA, 0xE7, 0xCB, 0xE7, 0xCC, /* 0x20-0x23 */ + 0xE7, 0xCD, 0xE7, 0xCE, 0xE7, 0xCF, 0xE7, 0xD0, /* 0x24-0x27 */ + 0xD3, 0xA7, 0xCB, 0xF5, 0xE7, 0xD1, 0xE7, 0xD2, /* 0x28-0x2B */ + 0xE7, 0xD3, 0xE7, 0xD4, 0xC9, 0xC9, 0xE7, 0xD5, /* 0x2C-0x2F */ + 0xE7, 0xD6, 0xE7, 0xD7, 0xE7, 0xD8, 0xE7, 0xD9, /* 0x30-0x33 */ + 0xBD, 0xC9, 0xE7, 0xDA, 0xF3, 0xBE, 0xC0, 0x8B, /* 0x34-0x37 */ + 0xB8, 0xD7, 0xC0, 0x8C, 0xC8, 0xB1, 0xC0, 0x8D, /* 0x38-0x3B */ + 0xC0, 0x8E, 0xC0, 0x8F, 0xC0, 0x90, 0xC0, 0x91, /* 0x3C-0x3F */ + 0xC0, 0x92, 0xC0, 0x93, 0xF3, 0xBF, 0xC0, 0x94, /* 0x40-0x43 */ + 0xF3, 0xC0, 0xF3, 0xC1, 0xC0, 0x95, 0xC0, 0x96, /* 0x44-0x47 */ + 0xC0, 0x97, 0xC0, 0x98, 0xC0, 0x99, 0xC0, 0x9A, /* 0x48-0x4B */ + 0xC0, 0x9B, 0xC0, 0x9C, 0xC0, 0x9D, 0xC0, 0x9E, /* 0x4C-0x4F */ + 0xB9, 0xDE, 0xCD, 0xF8, 0xC0, 0x9F, 0xC0, 0xA0, /* 0x50-0x53 */ + 0xD8, 0xE8, 0xBA, 0xB1, 0xC1, 0x40, 0xC2, 0xDE, /* 0x54-0x57 */ + 0xEE, 0xB7, 0xC1, 0x41, 0xB7, 0xA3, 0xC1, 0x42, /* 0x58-0x5B */ + 0xC1, 0x43, 0xC1, 0x44, 0xC1, 0x45, 0xEE, 0xB9, /* 0x5C-0x5F */ + 0xC1, 0x46, 0xEE, 0xB8, 0xB0, 0xD5, 0xC1, 0x47, /* 0x60-0x63 */ + 0xC1, 0x48, 0xC1, 0x49, 0xC1, 0x4A, 0xC1, 0x4B, /* 0x64-0x67 */ + 0xEE, 0xBB, 0xD5, 0xD6, 0xD7, 0xEF, 0xC1, 0x4C, /* 0x68-0x6B */ + 0xC1, 0x4D, 0xC1, 0x4E, 0xD6, 0xC3, 0xC1, 0x4F, /* 0x6C-0x6F */ + 0xC1, 0x50, 0xEE, 0xBD, 0xCA, 0xF0, 0xC1, 0x51, /* 0x70-0x73 */ + 0xEE, 0xBC, 0xC1, 0x52, 0xC1, 0x53, 0xC1, 0x54, /* 0x74-0x77 */ + 0xC1, 0x55, 0xEE, 0xBE, 0xC1, 0x56, 0xC1, 0x57, /* 0x78-0x7B */ + 0xC1, 0x58, 0xC1, 0x59, 0xEE, 0xC0, 0xC1, 0x5A, /* 0x7C-0x7F */ + + 0xC1, 0x5B, 0xEE, 0xBF, 0xC1, 0x5C, 0xC1, 0x5D, /* 0x80-0x83 */ + 0xC1, 0x5E, 0xC1, 0x5F, 0xC1, 0x60, 0xC1, 0x61, /* 0x84-0x87 */ + 0xC1, 0x62, 0xC1, 0x63, 0xD1, 0xF2, 0xC1, 0x64, /* 0x88-0x8B */ + 0xC7, 0xBC, 0xC1, 0x65, 0xC3, 0xC0, 0xC1, 0x66, /* 0x8C-0x8F */ + 0xC1, 0x67, 0xC1, 0x68, 0xC1, 0x69, 0xC1, 0x6A, /* 0x90-0x93 */ + 0xB8, 0xE1, 0xC1, 0x6B, 0xC1, 0x6C, 0xC1, 0x6D, /* 0x94-0x97 */ + 0xC1, 0x6E, 0xC1, 0x6F, 0xC1, 0xE7, 0xC1, 0x70, /* 0x98-0x9B */ + 0xC1, 0x71, 0xF4, 0xC6, 0xD0, 0xDF, 0xF4, 0xC7, /* 0x9C-0x9F */ + 0xC1, 0x72, 0xCF, 0xDB, 0xC1, 0x73, 0xC1, 0x74, /* 0xA0-0xA3 */ + 0xC8, 0xBA, 0xC1, 0x75, 0xC1, 0x76, 0xF4, 0xC8, /* 0xA4-0xA7 */ + 0xC1, 0x77, 0xC1, 0x78, 0xC1, 0x79, 0xC1, 0x7A, /* 0xA8-0xAB */ + 0xC1, 0x7B, 0xC1, 0x7C, 0xC1, 0x7D, 0xF4, 0xC9, /* 0xAC-0xAF */ + 0xF4, 0xCA, 0xC1, 0x7E, 0xF4, 0xCB, 0xC1, 0x80, /* 0xB0-0xB3 */ + 0xC1, 0x81, 0xC1, 0x82, 0xC1, 0x83, 0xC1, 0x84, /* 0xB4-0xB7 */ + 0xD9, 0xFA, 0xB8, 0xFE, 0xC1, 0x85, 0xC1, 0x86, /* 0xB8-0xBB */ + 0xE5, 0xF1, 0xD3, 0xF0, 0xC1, 0x87, 0xF4, 0xE0, /* 0xBC-0xBF */ + 0xC1, 0x88, 0xCE, 0xCC, 0xC1, 0x89, 0xC1, 0x8A, /* 0xC0-0xC3 */ + 0xC1, 0x8B, 0xB3, 0xE1, 0xC1, 0x8C, 0xC1, 0x8D, /* 0xC4-0xC7 */ + 0xC1, 0x8E, 0xC1, 0x8F, 0xF1, 0xB4, 0xC1, 0x90, /* 0xC8-0xCB */ + 0xD2, 0xEE, 0xC1, 0x91, 0xF4, 0xE1, 0xC1, 0x92, /* 0xCC-0xCF */ + 0xC1, 0x93, 0xC1, 0x94, 0xC1, 0x95, 0xC1, 0x96, /* 0xD0-0xD3 */ + 0xCF, 0xE8, 0xF4, 0xE2, 0xC1, 0x97, 0xC1, 0x98, /* 0xD4-0xD7 */ + 0xC7, 0xCC, 0xC1, 0x99, 0xC1, 0x9A, 0xC1, 0x9B, /* 0xD8-0xDB */ + 0xC1, 0x9C, 0xC1, 0x9D, 0xC1, 0x9E, 0xB5, 0xD4, /* 0xDC-0xDF */ + 0xB4, 0xE4, 0xF4, 0xE4, 0xC1, 0x9F, 0xC1, 0xA0, /* 0xE0-0xE3 */ + 0xC2, 0x40, 0xF4, 0xE3, 0xF4, 0xE5, 0xC2, 0x41, /* 0xE4-0xE7 */ + 0xC2, 0x42, 0xF4, 0xE6, 0xC2, 0x43, 0xC2, 0x44, /* 0xE8-0xEB */ + 0xC2, 0x45, 0xC2, 0x46, 0xF4, 0xE7, 0xC2, 0x47, /* 0xEC-0xEF */ + 0xBA, 0xB2, 0xB0, 0xBF, 0xC2, 0x48, 0xF4, 0xE8, /* 0xF0-0xF3 */ + 0xC2, 0x49, 0xC2, 0x4A, 0xC2, 0x4B, 0xC2, 0x4C, /* 0xF4-0xF7 */ + 0xC2, 0x4D, 0xC2, 0x4E, 0xC2, 0x4F, 0xB7, 0xAD, /* 0xF8-0xFB */ + 0xD2, 0xED, 0xC2, 0x50, 0xC2, 0x51, 0xC2, 0x52, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_80[512] = { + 0xD2, 0xAB, 0xC0, 0xCF, 0xC2, 0x53, 0xBF, 0xBC, /* 0x00-0x03 */ + 0xEB, 0xA3, 0xD5, 0xDF, 0xEA, 0xC8, 0xC2, 0x54, /* 0x04-0x07 */ + 0xC2, 0x55, 0xC2, 0x56, 0xC2, 0x57, 0xF1, 0xF3, /* 0x08-0x0B */ + 0xB6, 0xF8, 0xCB, 0xA3, 0xC2, 0x58, 0xC2, 0x59, /* 0x0C-0x0F */ + 0xC4, 0xCD, 0xC2, 0x5A, 0xF1, 0xE7, 0xC2, 0x5B, /* 0x10-0x13 */ + 0xF1, 0xE8, 0xB8, 0xFB, 0xF1, 0xE9, 0xBA, 0xC4, /* 0x14-0x17 */ + 0xD4, 0xC5, 0xB0, 0xD2, 0xC2, 0x5C, 0xC2, 0x5D, /* 0x18-0x1B */ + 0xF1, 0xEA, 0xC2, 0x5E, 0xC2, 0x5F, 0xC2, 0x60, /* 0x1C-0x1F */ + 0xF1, 0xEB, 0xC2, 0x61, 0xF1, 0xEC, 0xC2, 0x62, /* 0x20-0x23 */ + 0xC2, 0x63, 0xF1, 0xED, 0xF1, 0xEE, 0xF1, 0xEF, /* 0x24-0x27 */ + 0xF1, 0xF1, 0xF1, 0xF0, 0xC5, 0xD5, 0xC2, 0x64, /* 0x28-0x2B */ + 0xC2, 0x65, 0xC2, 0x66, 0xC2, 0x67, 0xC2, 0x68, /* 0x2C-0x2F */ + 0xC2, 0x69, 0xF1, 0xF2, 0xC2, 0x6A, 0xB6, 0xFA, /* 0x30-0x33 */ + 0xC2, 0x6B, 0xF1, 0xF4, 0xD2, 0xAE, 0xDE, 0xC7, /* 0x34-0x37 */ + 0xCB, 0xCA, 0xC2, 0x6C, 0xC2, 0x6D, 0xB3, 0xDC, /* 0x38-0x3B */ + 0xC2, 0x6E, 0xB5, 0xA2, 0xC2, 0x6F, 0xB9, 0xA2, /* 0x3C-0x3F */ + 0xC2, 0x70, 0xC2, 0x71, 0xC4, 0xF4, 0xF1, 0xF5, /* 0x40-0x43 */ + 0xC2, 0x72, 0xC2, 0x73, 0xF1, 0xF6, 0xC2, 0x74, /* 0x44-0x47 */ + 0xC2, 0x75, 0xC2, 0x76, 0xC1, 0xC4, 0xC1, 0xFB, /* 0x48-0x4B */ + 0xD6, 0xB0, 0xF1, 0xF7, 0xC2, 0x77, 0xC2, 0x78, /* 0x4C-0x4F */ + 0xC2, 0x79, 0xC2, 0x7A, 0xF1, 0xF8, 0xC2, 0x7B, /* 0x50-0x53 */ + 0xC1, 0xAA, 0xC2, 0x7C, 0xC2, 0x7D, 0xC2, 0x7E, /* 0x54-0x57 */ + 0xC6, 0xB8, 0xC2, 0x80, 0xBE, 0xDB, 0xC2, 0x81, /* 0x58-0x5B */ + 0xC2, 0x82, 0xC2, 0x83, 0xC2, 0x84, 0xC2, 0x85, /* 0x5C-0x5F */ + 0xC2, 0x86, 0xC2, 0x87, 0xC2, 0x88, 0xC2, 0x89, /* 0x60-0x63 */ + 0xC2, 0x8A, 0xC2, 0x8B, 0xC2, 0x8C, 0xC2, 0x8D, /* 0x64-0x67 */ + 0xC2, 0x8E, 0xF1, 0xF9, 0xB4, 0xCF, 0xC2, 0x8F, /* 0x68-0x6B */ + 0xC2, 0x90, 0xC2, 0x91, 0xC2, 0x92, 0xC2, 0x93, /* 0x6C-0x6F */ + 0xC2, 0x94, 0xF1, 0xFA, 0xC2, 0x95, 0xC2, 0x96, /* 0x70-0x73 */ + 0xC2, 0x97, 0xC2, 0x98, 0xC2, 0x99, 0xC2, 0x9A, /* 0x74-0x77 */ + 0xC2, 0x9B, 0xC2, 0x9C, 0xC2, 0x9D, 0xC2, 0x9E, /* 0x78-0x7B */ + 0xC2, 0x9F, 0xC2, 0xA0, 0xC3, 0x40, 0xED, 0xB2, /* 0x7C-0x7F */ + + 0xED, 0xB1, 0xC3, 0x41, 0xC3, 0x42, 0xCB, 0xE0, /* 0x80-0x83 */ + 0xD2, 0xDE, 0xC3, 0x43, 0xCB, 0xC1, 0xD5, 0xD8, /* 0x84-0x87 */ + 0xC3, 0x44, 0xC8, 0xE2, 0xC3, 0x45, 0xC0, 0xDF, /* 0x88-0x8B */ + 0xBC, 0xA1, 0xC3, 0x46, 0xC3, 0x47, 0xC3, 0x48, /* 0x8C-0x8F */ + 0xC3, 0x49, 0xC3, 0x4A, 0xC3, 0x4B, 0xEB, 0xC1, /* 0x90-0x93 */ + 0xC3, 0x4C, 0xC3, 0x4D, 0xD0, 0xA4, 0xC3, 0x4E, /* 0x94-0x97 */ + 0xD6, 0xE2, 0xC3, 0x4F, 0xB6, 0xC7, 0xB8, 0xD8, /* 0x98-0x9B */ + 0xEB, 0xC0, 0xB8, 0xCE, 0xC3, 0x50, 0xEB, 0xBF, /* 0x9C-0x9F */ + 0xB3, 0xA6, 0xB9, 0xC9, 0xD6, 0xAB, 0xC3, 0x51, /* 0xA0-0xA3 */ + 0xB7, 0xF4, 0xB7, 0xCA, 0xC3, 0x52, 0xC3, 0x53, /* 0xA4-0xA7 */ + 0xC3, 0x54, 0xBC, 0xE7, 0xB7, 0xBE, 0xEB, 0xC6, /* 0xA8-0xAB */ + 0xC3, 0x55, 0xEB, 0xC7, 0xB0, 0xB9, 0xBF, 0xCF, /* 0xAC-0xAF */ + 0xC3, 0x56, 0xEB, 0xC5, 0xD3, 0xFD, 0xC3, 0x57, /* 0xB0-0xB3 */ + 0xEB, 0xC8, 0xC3, 0x58, 0xC3, 0x59, 0xEB, 0xC9, /* 0xB4-0xB7 */ + 0xC3, 0x5A, 0xC3, 0x5B, 0xB7, 0xCE, 0xC3, 0x5C, /* 0xB8-0xBB */ + 0xEB, 0xC2, 0xEB, 0xC4, 0xC9, 0xF6, 0xD6, 0xD7, /* 0xBC-0xBF */ + 0xD5, 0xCD, 0xD0, 0xB2, 0xEB, 0xCF, 0xCE, 0xB8, /* 0xC0-0xC3 */ + 0xEB, 0xD0, 0xC3, 0x5D, 0xB5, 0xA8, 0xC3, 0x5E, /* 0xC4-0xC7 */ + 0xC3, 0x5F, 0xC3, 0x60, 0xC3, 0x61, 0xC3, 0x62, /* 0xC8-0xCB */ + 0xB1, 0xB3, 0xEB, 0xD2, 0xCC, 0xA5, 0xC3, 0x63, /* 0xCC-0xCF */ + 0xC3, 0x64, 0xC3, 0x65, 0xC3, 0x66, 0xC3, 0x67, /* 0xD0-0xD3 */ + 0xC3, 0x68, 0xC3, 0x69, 0xC5, 0xD6, 0xEB, 0xD3, /* 0xD4-0xD7 */ + 0xC3, 0x6A, 0xEB, 0xD1, 0xC5, 0xDF, 0xEB, 0xCE, /* 0xD8-0xDB */ + 0xCA, 0xA4, 0xEB, 0xD5, 0xB0, 0xFB, 0xC3, 0x6B, /* 0xDC-0xDF */ + 0xC3, 0x6C, 0xBA, 0xFA, 0xC3, 0x6D, 0xC3, 0x6E, /* 0xE0-0xE3 */ + 0xD8, 0xB7, 0xF1, 0xE3, 0xC3, 0x6F, 0xEB, 0xCA, /* 0xE4-0xE7 */ + 0xEB, 0xCB, 0xEB, 0xCC, 0xEB, 0xCD, 0xEB, 0xD6, /* 0xE8-0xEB */ + 0xE6, 0xC0, 0xEB, 0xD9, 0xC3, 0x70, 0xBF, 0xE8, /* 0xEC-0xEF */ + 0xD2, 0xC8, 0xEB, 0xD7, 0xEB, 0xDC, 0xB8, 0xEC, /* 0xF0-0xF3 */ + 0xEB, 0xD8, 0xC3, 0x71, 0xBD, 0xBA, 0xC3, 0x72, /* 0xF4-0xF7 */ + 0xD0, 0xD8, 0xC3, 0x73, 0xB0, 0xB7, 0xC3, 0x74, /* 0xF8-0xFB */ + 0xEB, 0xDD, 0xC4, 0xDC, 0xC3, 0x75, 0xC3, 0x76, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_81[512] = { + 0xC3, 0x77, 0xC3, 0x78, 0xD6, 0xAC, 0xC3, 0x79, /* 0x00-0x03 */ + 0xC3, 0x7A, 0xC3, 0x7B, 0xB4, 0xE0, 0xC3, 0x7C, /* 0x04-0x07 */ + 0xC3, 0x7D, 0xC2, 0xF6, 0xBC, 0xB9, 0xC3, 0x7E, /* 0x08-0x0B */ + 0xC3, 0x80, 0xEB, 0xDA, 0xEB, 0xDB, 0xD4, 0xE0, /* 0x0C-0x0F */ + 0xC6, 0xEA, 0xC4, 0xD4, 0xEB, 0xDF, 0xC5, 0xA7, /* 0x10-0x13 */ + 0xD9, 0xF5, 0xC3, 0x81, 0xB2, 0xB1, 0xC3, 0x82, /* 0x14-0x17 */ + 0xEB, 0xE4, 0xC3, 0x83, 0xBD, 0xC5, 0xC3, 0x84, /* 0x18-0x1B */ + 0xC3, 0x85, 0xC3, 0x86, 0xEB, 0xE2, 0xC3, 0x87, /* 0x1C-0x1F */ + 0xC3, 0x88, 0xC3, 0x89, 0xC3, 0x8A, 0xC3, 0x8B, /* 0x20-0x23 */ + 0xC3, 0x8C, 0xC3, 0x8D, 0xC3, 0x8E, 0xC3, 0x8F, /* 0x24-0x27 */ + 0xC3, 0x90, 0xC3, 0x91, 0xC3, 0x92, 0xC3, 0x93, /* 0x28-0x2B */ + 0xEB, 0xE3, 0xC3, 0x94, 0xC3, 0x95, 0xB8, 0xAC, /* 0x2C-0x2F */ + 0xC3, 0x96, 0xCD, 0xD1, 0xEB, 0xE5, 0xC3, 0x97, /* 0x30-0x33 */ + 0xC3, 0x98, 0xC3, 0x99, 0xEB, 0xE1, 0xC3, 0x9A, /* 0x34-0x37 */ + 0xC1, 0xB3, 0xC3, 0x9B, 0xC3, 0x9C, 0xC3, 0x9D, /* 0x38-0x3B */ + 0xC3, 0x9E, 0xC3, 0x9F, 0xC6, 0xA2, 0xC3, 0xA0, /* 0x3C-0x3F */ + 0xC4, 0x40, 0xC4, 0x41, 0xC4, 0x42, 0xC4, 0x43, /* 0x40-0x43 */ + 0xC4, 0x44, 0xC4, 0x45, 0xCC, 0xF3, 0xC4, 0x46, /* 0x44-0x47 */ + 0xEB, 0xE6, 0xC4, 0x47, 0xC0, 0xB0, 0xD2, 0xB8, /* 0x48-0x4B */ + 0xEB, 0xE7, 0xC4, 0x48, 0xC4, 0x49, 0xC4, 0x4A, /* 0x4C-0x4F */ + 0xB8, 0xAF, 0xB8, 0xAD, 0xC4, 0x4B, 0xEB, 0xE8, /* 0x50-0x53 */ + 0xC7, 0xBB, 0xCD, 0xF3, 0xC4, 0x4C, 0xC4, 0x4D, /* 0x54-0x57 */ + 0xC4, 0x4E, 0xEB, 0xEA, 0xEB, 0xEB, 0xC4, 0x4F, /* 0x58-0x5B */ + 0xC4, 0x50, 0xC4, 0x51, 0xC4, 0x52, 0xC4, 0x53, /* 0x5C-0x5F */ + 0xEB, 0xED, 0xC4, 0x54, 0xC4, 0x55, 0xC4, 0x56, /* 0x60-0x63 */ + 0xC4, 0x57, 0xD0, 0xC8, 0xC4, 0x58, 0xEB, 0xF2, /* 0x64-0x67 */ + 0xC4, 0x59, 0xEB, 0xEE, 0xC4, 0x5A, 0xC4, 0x5B, /* 0x68-0x6B */ + 0xC4, 0x5C, 0xEB, 0xF1, 0xC8, 0xF9, 0xC4, 0x5D, /* 0x6C-0x6F */ + 0xD1, 0xFC, 0xEB, 0xEC, 0xC4, 0x5E, 0xC4, 0x5F, /* 0x70-0x73 */ + 0xEB, 0xE9, 0xC4, 0x60, 0xC4, 0x61, 0xC4, 0x62, /* 0x74-0x77 */ + 0xC4, 0x63, 0xB8, 0xB9, 0xCF, 0xD9, 0xC4, 0xE5, /* 0x78-0x7B */ + 0xEB, 0xEF, 0xEB, 0xF0, 0xCC, 0xDA, 0xCD, 0xC8, /* 0x7C-0x7F */ + + 0xB0, 0xF2, 0xC4, 0x64, 0xEB, 0xF6, 0xC4, 0x65, /* 0x80-0x83 */ + 0xC4, 0x66, 0xC4, 0x67, 0xC4, 0x68, 0xC4, 0x69, /* 0x84-0x87 */ + 0xEB, 0xF5, 0xC4, 0x6A, 0xB2, 0xB2, 0xC4, 0x6B, /* 0x88-0x8B */ + 0xC4, 0x6C, 0xC4, 0x6D, 0xC4, 0x6E, 0xB8, 0xE0, /* 0x8C-0x8F */ + 0xC4, 0x6F, 0xEB, 0xF7, 0xC4, 0x70, 0xC4, 0x71, /* 0x90-0x93 */ + 0xC4, 0x72, 0xC4, 0x73, 0xC4, 0x74, 0xC4, 0x75, /* 0x94-0x97 */ + 0xB1, 0xEC, 0xC4, 0x76, 0xC4, 0x77, 0xCC, 0xC5, /* 0x98-0x9B */ + 0xC4, 0xA4, 0xCF, 0xA5, 0xC4, 0x78, 0xC4, 0x79, /* 0x9C-0x9F */ + 0xC4, 0x7A, 0xC4, 0x7B, 0xC4, 0x7C, 0xEB, 0xF9, /* 0xA0-0xA3 */ + 0xC4, 0x7D, 0xC4, 0x7E, 0xEC, 0xA2, 0xC4, 0x80, /* 0xA4-0xA7 */ + 0xC5, 0xF2, 0xC4, 0x81, 0xEB, 0xFA, 0xC4, 0x82, /* 0xA8-0xAB */ + 0xC4, 0x83, 0xC4, 0x84, 0xC4, 0x85, 0xC4, 0x86, /* 0xAC-0xAF */ + 0xC4, 0x87, 0xC4, 0x88, 0xC4, 0x89, 0xC9, 0xC5, /* 0xB0-0xB3 */ + 0xC4, 0x8A, 0xC4, 0x8B, 0xC4, 0x8C, 0xC4, 0x8D, /* 0xB4-0xB7 */ + 0xC4, 0x8E, 0xC4, 0x8F, 0xE2, 0xDF, 0xEB, 0xFE, /* 0xB8-0xBB */ + 0xC4, 0x90, 0xC4, 0x91, 0xC4, 0x92, 0xC4, 0x93, /* 0xBC-0xBF */ + 0xCD, 0xCE, 0xEC, 0xA1, 0xB1, 0xDB, 0xD3, 0xB7, /* 0xC0-0xC3 */ + 0xC4, 0x94, 0xC4, 0x95, 0xD2, 0xDC, 0xC4, 0x96, /* 0xC4-0xC7 */ + 0xC4, 0x97, 0xC4, 0x98, 0xEB, 0xFD, 0xC4, 0x99, /* 0xC8-0xCB */ + 0xEB, 0xFB, 0xC4, 0x9A, 0xC4, 0x9B, 0xC4, 0x9C, /* 0xCC-0xCF */ + 0xC4, 0x9D, 0xC4, 0x9E, 0xC4, 0x9F, 0xC4, 0xA0, /* 0xD0-0xD3 */ + 0xC5, 0x40, 0xC5, 0x41, 0xC5, 0x42, 0xC5, 0x43, /* 0xD4-0xD7 */ + 0xC5, 0x44, 0xC5, 0x45, 0xC5, 0x46, 0xC5, 0x47, /* 0xD8-0xDB */ + 0xC5, 0x48, 0xC5, 0x49, 0xC5, 0x4A, 0xC5, 0x4B, /* 0xDC-0xDF */ + 0xC5, 0x4C, 0xC5, 0x4D, 0xC5, 0x4E, 0xB3, 0xBC, /* 0xE0-0xE3 */ + 0xC5, 0x4F, 0xC5, 0x50, 0xC5, 0x51, 0xEA, 0xB0, /* 0xE4-0xE7 */ + 0xC5, 0x52, 0xC5, 0x53, 0xD7, 0xD4, 0xC5, 0x54, /* 0xE8-0xEB */ + 0xF4, 0xAB, 0xB3, 0xF4, 0xC5, 0x55, 0xC5, 0x56, /* 0xEC-0xEF */ + 0xC5, 0x57, 0xC5, 0x58, 0xC5, 0x59, 0xD6, 0xC1, /* 0xF0-0xF3 */ + 0xD6, 0xC2, 0xC5, 0x5A, 0xC5, 0x5B, 0xC5, 0x5C, /* 0xF4-0xF7 */ + 0xC5, 0x5D, 0xC5, 0x5E, 0xC5, 0x5F, 0xD5, 0xE9, /* 0xF8-0xFB */ + 0xBE, 0xCA, 0xC5, 0x60, 0xF4, 0xA7, 0xC5, 0x61, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_82[512] = { + 0xD2, 0xA8, 0xF4, 0xA8, 0xF4, 0xA9, 0xC5, 0x62, /* 0x00-0x03 */ + 0xF4, 0xAA, 0xBE, 0xCB, 0xD3, 0xDF, 0xC5, 0x63, /* 0x04-0x07 */ + 0xC5, 0x64, 0xC5, 0x65, 0xC5, 0x66, 0xC5, 0x67, /* 0x08-0x0B */ + 0xC9, 0xE0, 0xC9, 0xE1, 0xC5, 0x68, 0xC5, 0x69, /* 0x0C-0x0F */ + 0xF3, 0xC2, 0xC5, 0x6A, 0xCA, 0xE6, 0xC5, 0x6B, /* 0x10-0x13 */ + 0xCC, 0xF2, 0xC5, 0x6C, 0xC5, 0x6D, 0xC5, 0x6E, /* 0x14-0x17 */ + 0xC5, 0x6F, 0xC5, 0x70, 0xC5, 0x71, 0xE2, 0xB6, /* 0x18-0x1B */ + 0xCB, 0xB4, 0xC5, 0x72, 0xCE, 0xE8, 0xD6, 0xDB, /* 0x1C-0x1F */ + 0xC5, 0x73, 0xF4, 0xAD, 0xF4, 0xAE, 0xF4, 0xAF, /* 0x20-0x23 */ + 0xC5, 0x74, 0xC5, 0x75, 0xC5, 0x76, 0xC5, 0x77, /* 0x24-0x27 */ + 0xF4, 0xB2, 0xC5, 0x78, 0xBA, 0xBD, 0xF4, 0xB3, /* 0x28-0x2B */ + 0xB0, 0xE3, 0xF4, 0xB0, 0xC5, 0x79, 0xF4, 0xB1, /* 0x2C-0x2F */ + 0xBD, 0xA2, 0xB2, 0xD5, 0xC5, 0x7A, 0xF4, 0xB6, /* 0x30-0x33 */ + 0xF4, 0xB7, 0xB6, 0xE6, 0xB2, 0xB0, 0xCF, 0xCF, /* 0x34-0x37 */ + 0xF4, 0xB4, 0xB4, 0xAC, 0xC5, 0x7B, 0xF4, 0xB5, /* 0x38-0x3B */ + 0xC5, 0x7C, 0xC5, 0x7D, 0xF4, 0xB8, 0xC5, 0x7E, /* 0x3C-0x3F */ + 0xC5, 0x80, 0xC5, 0x81, 0xC5, 0x82, 0xC5, 0x83, /* 0x40-0x43 */ + 0xF4, 0xB9, 0xC5, 0x84, 0xC5, 0x85, 0xCD, 0xA7, /* 0x44-0x47 */ + 0xC5, 0x86, 0xF4, 0xBA, 0xC5, 0x87, 0xF4, 0xBB, /* 0x48-0x4B */ + 0xC5, 0x88, 0xC5, 0x89, 0xC5, 0x8A, 0xF4, 0xBC, /* 0x4C-0x4F */ + 0xC5, 0x8B, 0xC5, 0x8C, 0xC5, 0x8D, 0xC5, 0x8E, /* 0x50-0x53 */ + 0xC5, 0x8F, 0xC5, 0x90, 0xC5, 0x91, 0xC5, 0x92, /* 0x54-0x57 */ + 0xCB, 0xD2, 0xC5, 0x93, 0xF4, 0xBD, 0xC5, 0x94, /* 0x58-0x5B */ + 0xC5, 0x95, 0xC5, 0x96, 0xC5, 0x97, 0xF4, 0xBE, /* 0x5C-0x5F */ + 0xC5, 0x98, 0xC5, 0x99, 0xC5, 0x9A, 0xC5, 0x9B, /* 0x60-0x63 */ + 0xC5, 0x9C, 0xC5, 0x9D, 0xC5, 0x9E, 0xC5, 0x9F, /* 0x64-0x67 */ + 0xF4, 0xBF, 0xC5, 0xA0, 0xC6, 0x40, 0xC6, 0x41, /* 0x68-0x6B */ + 0xC6, 0x42, 0xC6, 0x43, 0xF4, 0xDE, 0xC1, 0xBC, /* 0x6C-0x6F */ + 0xBC, 0xE8, 0xC6, 0x44, 0xC9, 0xAB, 0xD1, 0xDE, /* 0x70-0x73 */ + 0xE5, 0xF5, 0xC6, 0x45, 0xC6, 0x46, 0xC6, 0x47, /* 0x74-0x77 */ + 0xC6, 0x48, 0xDC, 0xB3, 0xD2, 0xD5, 0xC6, 0x49, /* 0x78-0x7B */ + 0xC6, 0x4A, 0xDC, 0xB4, 0xB0, 0xAC, 0xDC, 0xB5, /* 0x7C-0x7F */ + + 0xC6, 0x4B, 0xC6, 0x4C, 0xBD, 0xDA, 0xC6, 0x4D, /* 0x80-0x83 */ + 0xDC, 0xB9, 0xC6, 0x4E, 0xC6, 0x4F, 0xC6, 0x50, /* 0x84-0x87 */ + 0xD8, 0xC2, 0xC6, 0x51, 0xDC, 0xB7, 0xD3, 0xF3, /* 0x88-0x8B */ + 0xC6, 0x52, 0xC9, 0xD6, 0xDC, 0xBA, 0xDC, 0xB6, /* 0x8C-0x8F */ + 0xC6, 0x53, 0xDC, 0xBB, 0xC3, 0xA2, 0xC6, 0x54, /* 0x90-0x93 */ + 0xC6, 0x55, 0xC6, 0x56, 0xC6, 0x57, 0xDC, 0xBC, /* 0x94-0x97 */ + 0xDC, 0xC5, 0xDC, 0xBD, 0xC6, 0x58, 0xC6, 0x59, /* 0x98-0x9B */ + 0xCE, 0xDF, 0xD6, 0xA5, 0xC6, 0x5A, 0xDC, 0xCF, /* 0x9C-0x9F */ + 0xC6, 0x5B, 0xDC, 0xCD, 0xC6, 0x5C, 0xC6, 0x5D, /* 0xA0-0xA3 */ + 0xDC, 0xD2, 0xBD, 0xE6, 0xC2, 0xAB, 0xC6, 0x5E, /* 0xA4-0xA7 */ + 0xDC, 0xB8, 0xDC, 0xCB, 0xDC, 0xCE, 0xDC, 0xBE, /* 0xA8-0xAB */ + 0xB7, 0xD2, 0xB0, 0xC5, 0xDC, 0xC7, 0xD0, 0xBE, /* 0xAC-0xAF */ + 0xDC, 0xC1, 0xBB, 0xA8, 0xC6, 0x5F, 0xB7, 0xBC, /* 0xB0-0xB3 */ + 0xDC, 0xCC, 0xC6, 0x60, 0xC6, 0x61, 0xDC, 0xC6, /* 0xB4-0xB7 */ + 0xDC, 0xBF, 0xC7, 0xDB, 0xC6, 0x62, 0xC6, 0x63, /* 0xB8-0xBB */ + 0xC6, 0x64, 0xD1, 0xBF, 0xDC, 0xC0, 0xC6, 0x65, /* 0xBC-0xBF */ + 0xC6, 0x66, 0xDC, 0xCA, 0xC6, 0x67, 0xC6, 0x68, /* 0xC0-0xC3 */ + 0xDC, 0xD0, 0xC6, 0x69, 0xC6, 0x6A, 0xCE, 0xAD, /* 0xC4-0xC7 */ + 0xDC, 0xC2, 0xC6, 0x6B, 0xDC, 0xC3, 0xDC, 0xC8, /* 0xC8-0xCB */ + 0xDC, 0xC9, 0xB2, 0xD4, 0xDC, 0xD1, 0xCB, 0xD5, /* 0xCC-0xCF */ + 0xC6, 0x6C, 0xD4, 0xB7, 0xDC, 0xDB, 0xDC, 0xDF, /* 0xD0-0xD3 */ + 0xCC, 0xA6, 0xDC, 0xE6, 0xC6, 0x6D, 0xC3, 0xE7, /* 0xD4-0xD7 */ + 0xDC, 0xDC, 0xC6, 0x6E, 0xC6, 0x6F, 0xBF, 0xC1, /* 0xD8-0xDB */ + 0xDC, 0xD9, 0xC6, 0x70, 0xB0, 0xFA, 0xB9, 0xB6, /* 0xDC-0xDF */ + 0xDC, 0xE5, 0xDC, 0xD3, 0xC6, 0x71, 0xDC, 0xC4, /* 0xE0-0xE3 */ + 0xDC, 0xD6, 0xC8, 0xF4, 0xBF, 0xE0, 0xC6, 0x72, /* 0xE4-0xE7 */ + 0xC6, 0x73, 0xC6, 0x74, 0xC6, 0x75, 0xC9, 0xBB, /* 0xE8-0xEB */ + 0xC6, 0x76, 0xC6, 0x77, 0xC6, 0x78, 0xB1, 0xBD, /* 0xEC-0xEF */ + 0xC6, 0x79, 0xD3, 0xA2, 0xC6, 0x7A, 0xC6, 0x7B, /* 0xF0-0xF3 */ + 0xDC, 0xDA, 0xC6, 0x7C, 0xC6, 0x7D, 0xDC, 0xD5, /* 0xF4-0xF7 */ + 0xC6, 0x7E, 0xC6, 0xBB, 0xC6, 0x80, 0xDC, 0xDE, /* 0xF8-0xFB */ + 0xC6, 0x81, 0xC6, 0x82, 0xC6, 0x83, 0xC6, 0x84, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_83[512] = { + 0xC6, 0x85, 0xD7, 0xC2, 0xC3, 0xAF, 0xB7, 0xB6, /* 0x00-0x03 */ + 0xC7, 0xD1, 0xC3, 0xA9, 0xDC, 0xE2, 0xDC, 0xD8, /* 0x04-0x07 */ + 0xDC, 0xEB, 0xDC, 0xD4, 0xC6, 0x86, 0xC6, 0x87, /* 0x08-0x0B */ + 0xDC, 0xDD, 0xC6, 0x88, 0xBE, 0xA5, 0xDC, 0xD7, /* 0x0C-0x0F */ + 0xC6, 0x89, 0xDC, 0xE0, 0xC6, 0x8A, 0xC6, 0x8B, /* 0x10-0x13 */ + 0xDC, 0xE3, 0xDC, 0xE4, 0xC6, 0x8C, 0xDC, 0xF8, /* 0x14-0x17 */ + 0xC6, 0x8D, 0xC6, 0x8E, 0xDC, 0xE1, 0xDD, 0xA2, /* 0x18-0x1B */ + 0xDC, 0xE7, 0xC6, 0x8F, 0xC6, 0x90, 0xC6, 0x91, /* 0x1C-0x1F */ + 0xC6, 0x92, 0xC6, 0x93, 0xC6, 0x94, 0xC6, 0x95, /* 0x20-0x23 */ + 0xC6, 0x96, 0xC6, 0x97, 0xC6, 0x98, 0xBC, 0xEB, /* 0x24-0x27 */ + 0xB4, 0xC4, 0xC6, 0x99, 0xC6, 0x9A, 0xC3, 0xA3, /* 0x28-0x2B */ + 0xB2, 0xE7, 0xDC, 0xFA, 0xC6, 0x9B, 0xDC, 0xF2, /* 0x2C-0x2F */ + 0xC6, 0x9C, 0xDC, 0xEF, 0xC6, 0x9D, 0xDC, 0xFC, /* 0x30-0x33 */ + 0xDC, 0xEE, 0xD2, 0xF0, 0xB2, 0xE8, 0xC6, 0x9E, /* 0x34-0x37 */ + 0xC8, 0xD7, 0xC8, 0xE3, 0xDC, 0xFB, 0xC6, 0x9F, /* 0x38-0x3B */ + 0xDC, 0xED, 0xC6, 0xA0, 0xC7, 0x40, 0xC7, 0x41, /* 0x3C-0x3F */ + 0xDC, 0xF7, 0xC7, 0x42, 0xC7, 0x43, 0xDC, 0xF5, /* 0x40-0x43 */ + 0xC7, 0x44, 0xC7, 0x45, 0xBE, 0xA3, 0xDC, 0xF4, /* 0x44-0x47 */ + 0xC7, 0x46, 0xB2, 0xDD, 0xC7, 0x47, 0xC7, 0x48, /* 0x48-0x4B */ + 0xC7, 0x49, 0xC7, 0x4A, 0xC7, 0x4B, 0xDC, 0xF3, /* 0x4C-0x4F */ + 0xBC, 0xF6, 0xDC, 0xE8, 0xBB, 0xC4, 0xC7, 0x4C, /* 0x50-0x53 */ + 0xC0, 0xF3, 0xC7, 0x4D, 0xC7, 0x4E, 0xC7, 0x4F, /* 0x54-0x57 */ + 0xC7, 0x50, 0xC7, 0x51, 0xBC, 0xD4, 0xDC, 0xE9, /* 0x58-0x5B */ + 0xDC, 0xEA, 0xC7, 0x52, 0xDC, 0xF1, 0xDC, 0xF6, /* 0x5C-0x5F */ + 0xDC, 0xF9, 0xB5, 0xB4, 0xC7, 0x53, 0xC8, 0xD9, /* 0x60-0x63 */ + 0xBB, 0xE7, 0xDC, 0xFE, 0xDC, 0xFD, 0xD3, 0xAB, /* 0x64-0x67 */ + 0xDD, 0xA1, 0xDD, 0xA3, 0xDD, 0xA5, 0xD2, 0xF1, /* 0x68-0x6B */ + 0xDD, 0xA4, 0xDD, 0xA6, 0xDD, 0xA7, 0xD2, 0xA9, /* 0x6C-0x6F */ + 0xC7, 0x54, 0xC7, 0x55, 0xC7, 0x56, 0xC7, 0x57, /* 0x70-0x73 */ + 0xC7, 0x58, 0xC7, 0x59, 0xC7, 0x5A, 0xBA, 0xC9, /* 0x74-0x77 */ + 0xDD, 0xA9, 0xC7, 0x5B, 0xC7, 0x5C, 0xDD, 0xB6, /* 0x78-0x7B */ + 0xDD, 0xB1, 0xDD, 0xB4, 0xC7, 0x5D, 0xC7, 0x5E, /* 0x7C-0x7F */ + + 0xC7, 0x5F, 0xC7, 0x60, 0xC7, 0x61, 0xC7, 0x62, /* 0x80-0x83 */ + 0xC7, 0x63, 0xDD, 0xB0, 0xC6, 0xCE, 0xC7, 0x64, /* 0x84-0x87 */ + 0xC7, 0x65, 0xC0, 0xF2, 0xC7, 0x66, 0xC7, 0x67, /* 0x88-0x8B */ + 0xC7, 0x68, 0xC7, 0x69, 0xC9, 0xAF, 0xC7, 0x6A, /* 0x8C-0x8F */ + 0xC7, 0x6B, 0xC7, 0x6C, 0xDC, 0xEC, 0xDD, 0xAE, /* 0x90-0x93 */ + 0xC7, 0x6D, 0xC7, 0x6E, 0xC7, 0x6F, 0xC7, 0x70, /* 0x94-0x97 */ + 0xDD, 0xB7, 0xC7, 0x71, 0xC7, 0x72, 0xDC, 0xF0, /* 0x98-0x9B */ + 0xDD, 0xAF, 0xC7, 0x73, 0xDD, 0xB8, 0xC7, 0x74, /* 0x9C-0x9F */ + 0xDD, 0xAC, 0xC7, 0x75, 0xC7, 0x76, 0xC7, 0x77, /* 0xA0-0xA3 */ + 0xC7, 0x78, 0xC7, 0x79, 0xC7, 0x7A, 0xC7, 0x7B, /* 0xA4-0xA7 */ + 0xDD, 0xB9, 0xDD, 0xB3, 0xDD, 0xAD, 0xC4, 0xAA, /* 0xA8-0xAB */ + 0xC7, 0x7C, 0xC7, 0x7D, 0xC7, 0x7E, 0xC7, 0x80, /* 0xAC-0xAF */ + 0xDD, 0xA8, 0xC0, 0xB3, 0xC1, 0xAB, 0xDD, 0xAA, /* 0xB0-0xB3 */ + 0xDD, 0xAB, 0xC7, 0x81, 0xDD, 0xB2, 0xBB, 0xF1, /* 0xB4-0xB7 */ + 0xDD, 0xB5, 0xD3, 0xA8, 0xDD, 0xBA, 0xC7, 0x82, /* 0xB8-0xBB */ + 0xDD, 0xBB, 0xC3, 0xA7, 0xC7, 0x83, 0xC7, 0x84, /* 0xBC-0xBF */ + 0xDD, 0xD2, 0xDD, 0xBC, 0xC7, 0x85, 0xC7, 0x86, /* 0xC0-0xC3 */ + 0xC7, 0x87, 0xDD, 0xD1, 0xC7, 0x88, 0xB9, 0xBD, /* 0xC4-0xC7 */ + 0xC7, 0x89, 0xC7, 0x8A, 0xBE, 0xD5, 0xC7, 0x8B, /* 0xC8-0xCB */ + 0xBE, 0xFA, 0xC7, 0x8C, 0xC7, 0x8D, 0xBA, 0xCA, /* 0xCC-0xCF */ + 0xC7, 0x8E, 0xC7, 0x8F, 0xC7, 0x90, 0xC7, 0x91, /* 0xD0-0xD3 */ + 0xDD, 0xCA, 0xC7, 0x92, 0xDD, 0xC5, 0xC7, 0x93, /* 0xD4-0xD7 */ + 0xDD, 0xBF, 0xC7, 0x94, 0xC7, 0x95, 0xC7, 0x96, /* 0xD8-0xDB */ + 0xB2, 0xCB, 0xDD, 0xC3, 0xC7, 0x97, 0xDD, 0xCB, /* 0xDC-0xDF */ + 0xB2, 0xA4, 0xDD, 0xD5, 0xC7, 0x98, 0xC7, 0x99, /* 0xE0-0xE3 */ + 0xC7, 0x9A, 0xDD, 0xBE, 0xC7, 0x9B, 0xC7, 0x9C, /* 0xE4-0xE7 */ + 0xC7, 0x9D, 0xC6, 0xD0, 0xDD, 0xD0, 0xC7, 0x9E, /* 0xE8-0xEB */ + 0xC7, 0x9F, 0xC7, 0xA0, 0xC8, 0x40, 0xC8, 0x41, /* 0xEC-0xEF */ + 0xDD, 0xD4, 0xC1, 0xE2, 0xB7, 0xC6, 0xC8, 0x42, /* 0xF0-0xF3 */ + 0xC8, 0x43, 0xC8, 0x44, 0xC8, 0x45, 0xC8, 0x46, /* 0xF4-0xF7 */ + 0xDD, 0xCE, 0xDD, 0xCF, 0xC8, 0x47, 0xC8, 0x48, /* 0xF8-0xFB */ + 0xC8, 0x49, 0xDD, 0xC4, 0xC8, 0x4A, 0xC8, 0x4B, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_84[512] = { + 0xC8, 0x4C, 0xDD, 0xBD, 0xC8, 0x4D, 0xDD, 0xCD, /* 0x00-0x03 */ + 0xCC, 0xD1, 0xC8, 0x4E, 0xDD, 0xC9, 0xC8, 0x4F, /* 0x04-0x07 */ + 0xC8, 0x50, 0xC8, 0x51, 0xC8, 0x52, 0xDD, 0xC2, /* 0x08-0x0B */ + 0xC3, 0xC8, 0xC6, 0xBC, 0xCE, 0xAE, 0xDD, 0xCC, /* 0x0C-0x0F */ + 0xC8, 0x53, 0xDD, 0xC8, 0xC8, 0x54, 0xC8, 0x55, /* 0x10-0x13 */ + 0xC8, 0x56, 0xC8, 0x57, 0xC8, 0x58, 0xC8, 0x59, /* 0x14-0x17 */ + 0xDD, 0xC1, 0xC8, 0x5A, 0xC8, 0x5B, 0xC8, 0x5C, /* 0x18-0x1B */ + 0xDD, 0xC6, 0xC2, 0xDC, 0xC8, 0x5D, 0xC8, 0x5E, /* 0x1C-0x1F */ + 0xC8, 0x5F, 0xC8, 0x60, 0xC8, 0x61, 0xC8, 0x62, /* 0x20-0x23 */ + 0xD3, 0xA9, 0xD3, 0xAA, 0xDD, 0xD3, 0xCF, 0xF4, /* 0x24-0x27 */ + 0xC8, 0xF8, 0xC8, 0x63, 0xC8, 0x64, 0xC8, 0x65, /* 0x28-0x2B */ + 0xC8, 0x66, 0xC8, 0x67, 0xC8, 0x68, 0xC8, 0x69, /* 0x2C-0x2F */ + 0xC8, 0x6A, 0xDD, 0xE6, 0xC8, 0x6B, 0xC8, 0x6C, /* 0x30-0x33 */ + 0xC8, 0x6D, 0xC8, 0x6E, 0xC8, 0x6F, 0xC8, 0x70, /* 0x34-0x37 */ + 0xDD, 0xC7, 0xC8, 0x71, 0xC8, 0x72, 0xC8, 0x73, /* 0x38-0x3B */ + 0xDD, 0xE0, 0xC2, 0xE4, 0xC8, 0x74, 0xC8, 0x75, /* 0x3C-0x3F */ + 0xC8, 0x76, 0xC8, 0x77, 0xC8, 0x78, 0xC8, 0x79, /* 0x40-0x43 */ + 0xC8, 0x7A, 0xC8, 0x7B, 0xDD, 0xE1, 0xC8, 0x7C, /* 0x44-0x47 */ + 0xC8, 0x7D, 0xC8, 0x7E, 0xC8, 0x80, 0xC8, 0x81, /* 0x48-0x4B */ + 0xC8, 0x82, 0xC8, 0x83, 0xC8, 0x84, 0xC8, 0x85, /* 0x4C-0x4F */ + 0xC8, 0x86, 0xDD, 0xD7, 0xC8, 0x87, 0xC8, 0x88, /* 0x50-0x53 */ + 0xC8, 0x89, 0xC8, 0x8A, 0xC8, 0x8B, 0xD6, 0xF8, /* 0x54-0x57 */ + 0xC8, 0x8C, 0xDD, 0xD9, 0xDD, 0xD8, 0xB8, 0xF0, /* 0x58-0x5B */ + 0xDD, 0xD6, 0xC8, 0x8D, 0xC8, 0x8E, 0xC8, 0x8F, /* 0x5C-0x5F */ + 0xC8, 0x90, 0xC6, 0xCF, 0xC8, 0x91, 0xB6, 0xAD, /* 0x60-0x63 */ + 0xC8, 0x92, 0xC8, 0x93, 0xC8, 0x94, 0xC8, 0x95, /* 0x64-0x67 */ + 0xC8, 0x96, 0xDD, 0xE2, 0xC8, 0x97, 0xBA, 0xF9, /* 0x68-0x6B */ + 0xD4, 0xE1, 0xDD, 0xE7, 0xC8, 0x98, 0xC8, 0x99, /* 0x6C-0x6F */ + 0xC8, 0x9A, 0xB4, 0xD0, 0xC8, 0x9B, 0xDD, 0xDA, /* 0x70-0x73 */ + 0xC8, 0x9C, 0xBF, 0xFB, 0xDD, 0xE3, 0xC8, 0x9D, /* 0x74-0x77 */ + 0xDD, 0xDF, 0xC8, 0x9E, 0xDD, 0xDD, 0xC8, 0x9F, /* 0x78-0x7B */ + 0xC8, 0xA0, 0xC9, 0x40, 0xC9, 0x41, 0xC9, 0x42, /* 0x7C-0x7F */ + + 0xC9, 0x43, 0xC9, 0x44, 0xB5, 0xD9, 0xC9, 0x45, /* 0x80-0x83 */ + 0xC9, 0x46, 0xC9, 0x47, 0xC9, 0x48, 0xDD, 0xDB, /* 0x84-0x87 */ + 0xDD, 0xDC, 0xDD, 0xDE, 0xC9, 0x49, 0xBD, 0xAF, /* 0x88-0x8B */ + 0xDD, 0xE4, 0xC9, 0x4A, 0xDD, 0xE5, 0xC9, 0x4B, /* 0x8C-0x8F */ + 0xC9, 0x4C, 0xC9, 0x4D, 0xC9, 0x4E, 0xC9, 0x4F, /* 0x90-0x93 */ + 0xC9, 0x50, 0xC9, 0x51, 0xC9, 0x52, 0xDD, 0xF5, /* 0x94-0x97 */ + 0xC9, 0x53, 0xC3, 0xC9, 0xC9, 0x54, 0xC9, 0x55, /* 0x98-0x9B */ + 0xCB, 0xE2, 0xC9, 0x56, 0xC9, 0x57, 0xC9, 0x58, /* 0x9C-0x9F */ + 0xC9, 0x59, 0xDD, 0xF2, 0xC9, 0x5A, 0xC9, 0x5B, /* 0xA0-0xA3 */ + 0xC9, 0x5C, 0xC9, 0x5D, 0xC9, 0x5E, 0xC9, 0x5F, /* 0xA4-0xA7 */ + 0xC9, 0x60, 0xC9, 0x61, 0xC9, 0x62, 0xC9, 0x63, /* 0xA8-0xAB */ + 0xC9, 0x64, 0xC9, 0x65, 0xC9, 0x66, 0xD8, 0xE1, /* 0xAC-0xAF */ + 0xC9, 0x67, 0xC9, 0x68, 0xC6, 0xD1, 0xC9, 0x69, /* 0xB0-0xB3 */ + 0xDD, 0xF4, 0xC9, 0x6A, 0xC9, 0x6B, 0xC9, 0x6C, /* 0xB4-0xB7 */ + 0xD5, 0xF4, 0xDD, 0xF3, 0xDD, 0xF0, 0xC9, 0x6D, /* 0xB8-0xBB */ + 0xC9, 0x6E, 0xDD, 0xEC, 0xC9, 0x6F, 0xDD, 0xEF, /* 0xBC-0xBF */ + 0xC9, 0x70, 0xDD, 0xE8, 0xC9, 0x71, 0xC9, 0x72, /* 0xC0-0xC3 */ + 0xD0, 0xEE, 0xC9, 0x73, 0xC9, 0x74, 0xC9, 0x75, /* 0xC4-0xC7 */ + 0xC9, 0x76, 0xC8, 0xD8, 0xDD, 0xEE, 0xC9, 0x77, /* 0xC8-0xCB */ + 0xC9, 0x78, 0xDD, 0xE9, 0xC9, 0x79, 0xC9, 0x7A, /* 0xCC-0xCF */ + 0xDD, 0xEA, 0xCB, 0xF2, 0xC9, 0x7B, 0xDD, 0xED, /* 0xD0-0xD3 */ + 0xC9, 0x7C, 0xC9, 0x7D, 0xB1, 0xCD, 0xC9, 0x7E, /* 0xD4-0xD7 */ + 0xC9, 0x80, 0xC9, 0x81, 0xC9, 0x82, 0xC9, 0x83, /* 0xD8-0xDB */ + 0xC9, 0x84, 0xC0, 0xB6, 0xC9, 0x85, 0xBC, 0xBB, /* 0xDC-0xDF */ + 0xDD, 0xF1, 0xC9, 0x86, 0xC9, 0x87, 0xDD, 0xF7, /* 0xE0-0xE3 */ + 0xC9, 0x88, 0xDD, 0xF6, 0xDD, 0xEB, 0xC9, 0x89, /* 0xE4-0xE7 */ + 0xC9, 0x8A, 0xC9, 0x8B, 0xC9, 0x8C, 0xC9, 0x8D, /* 0xE8-0xEB */ + 0xC5, 0xEE, 0xC9, 0x8E, 0xC9, 0x8F, 0xC9, 0x90, /* 0xEC-0xEF */ + 0xDD, 0xFB, 0xC9, 0x91, 0xC9, 0x92, 0xC9, 0x93, /* 0xF0-0xF3 */ + 0xC9, 0x94, 0xC9, 0x95, 0xC9, 0x96, 0xC9, 0x97, /* 0xF4-0xF7 */ + 0xC9, 0x98, 0xC9, 0x99, 0xC9, 0x9A, 0xC9, 0x9B, /* 0xF8-0xFB */ + 0xDE, 0xA4, 0xC9, 0x9C, 0xC9, 0x9D, 0xDE, 0xA3, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_85[512] = { + 0xC9, 0x9E, 0xC9, 0x9F, 0xC9, 0xA0, 0xCA, 0x40, /* 0x00-0x03 */ + 0xCA, 0x41, 0xCA, 0x42, 0xCA, 0x43, 0xCA, 0x44, /* 0x04-0x07 */ + 0xCA, 0x45, 0xCA, 0x46, 0xCA, 0x47, 0xCA, 0x48, /* 0x08-0x0B */ + 0xDD, 0xF8, 0xCA, 0x49, 0xCA, 0x4A, 0xCA, 0x4B, /* 0x0C-0x0F */ + 0xCA, 0x4C, 0xC3, 0xEF, 0xCA, 0x4D, 0xC2, 0xFB, /* 0x10-0x13 */ + 0xCA, 0x4E, 0xCA, 0x4F, 0xCA, 0x50, 0xD5, 0xE1, /* 0x14-0x17 */ + 0xCA, 0x51, 0xCA, 0x52, 0xCE, 0xB5, 0xCA, 0x53, /* 0x18-0x1B */ + 0xCA, 0x54, 0xCA, 0x55, 0xCA, 0x56, 0xDD, 0xFD, /* 0x1C-0x1F */ + 0xCA, 0x57, 0xB2, 0xCC, 0xCA, 0x58, 0xCA, 0x59, /* 0x20-0x23 */ + 0xCA, 0x5A, 0xCA, 0x5B, 0xCA, 0x5C, 0xCA, 0x5D, /* 0x24-0x27 */ + 0xCA, 0x5E, 0xCA, 0x5F, 0xCA, 0x60, 0xC4, 0xE8, /* 0x28-0x2B */ + 0xCA, 0xDF, 0xCA, 0x61, 0xCA, 0x62, 0xCA, 0x63, /* 0x2C-0x2F */ + 0xCA, 0x64, 0xCA, 0x65, 0xCA, 0x66, 0xCA, 0x67, /* 0x30-0x33 */ + 0xCA, 0x68, 0xCA, 0x69, 0xCA, 0x6A, 0xC7, 0xBE, /* 0x34-0x37 */ + 0xDD, 0xFA, 0xDD, 0xFC, 0xDD, 0xFE, 0xDE, 0xA2, /* 0x38-0x3B */ + 0xB0, 0xAA, 0xB1, 0xCE, 0xCA, 0x6B, 0xCA, 0x6C, /* 0x3C-0x3F */ + 0xCA, 0x6D, 0xCA, 0x6E, 0xCA, 0x6F, 0xDE, 0xAC, /* 0x40-0x43 */ + 0xCA, 0x70, 0xCA, 0x71, 0xCA, 0x72, 0xCA, 0x73, /* 0x44-0x47 */ + 0xDE, 0xA6, 0xBD, 0xB6, 0xC8, 0xEF, 0xCA, 0x74, /* 0x48-0x4B */ + 0xCA, 0x75, 0xCA, 0x76, 0xCA, 0x77, 0xCA, 0x78, /* 0x4C-0x4F */ + 0xCA, 0x79, 0xCA, 0x7A, 0xCA, 0x7B, 0xCA, 0x7C, /* 0x50-0x53 */ + 0xCA, 0x7D, 0xCA, 0x7E, 0xDE, 0xA1, 0xCA, 0x80, /* 0x54-0x57 */ + 0xCA, 0x81, 0xDE, 0xA5, 0xCA, 0x82, 0xCA, 0x83, /* 0x58-0x5B */ + 0xCA, 0x84, 0xCA, 0x85, 0xDE, 0xA9, 0xCA, 0x86, /* 0x5C-0x5F */ + 0xCA, 0x87, 0xCA, 0x88, 0xCA, 0x89, 0xCA, 0x8A, /* 0x60-0x63 */ + 0xDE, 0xA8, 0xCA, 0x8B, 0xCA, 0x8C, 0xCA, 0x8D, /* 0x64-0x67 */ + 0xDE, 0xA7, 0xCA, 0x8E, 0xCA, 0x8F, 0xCA, 0x90, /* 0x68-0x6B */ + 0xCA, 0x91, 0xCA, 0x92, 0xCA, 0x93, 0xCA, 0x94, /* 0x6C-0x6F */ + 0xCA, 0x95, 0xCA, 0x96, 0xDE, 0xAD, 0xCA, 0x97, /* 0x70-0x73 */ + 0xD4, 0xCC, 0xCA, 0x98, 0xCA, 0x99, 0xCA, 0x9A, /* 0x74-0x77 */ + 0xCA, 0x9B, 0xDE, 0xB3, 0xDE, 0xAA, 0xDE, 0xAE, /* 0x78-0x7B */ + 0xCA, 0x9C, 0xCA, 0x9D, 0xC0, 0xD9, 0xCA, 0x9E, /* 0x7C-0x7F */ + + 0xCA, 0x9F, 0xCA, 0xA0, 0xCB, 0x40, 0xCB, 0x41, /* 0x80-0x83 */ + 0xB1, 0xA1, 0xDE, 0xB6, 0xCB, 0x42, 0xDE, 0xB1, /* 0x84-0x87 */ + 0xCB, 0x43, 0xCB, 0x44, 0xCB, 0x45, 0xCB, 0x46, /* 0x88-0x8B */ + 0xCB, 0x47, 0xCB, 0x48, 0xCB, 0x49, 0xDE, 0xB2, /* 0x8C-0x8F */ + 0xCB, 0x4A, 0xCB, 0x4B, 0xCB, 0x4C, 0xCB, 0x4D, /* 0x90-0x93 */ + 0xCB, 0x4E, 0xCB, 0x4F, 0xCB, 0x50, 0xCB, 0x51, /* 0x94-0x97 */ + 0xCB, 0x52, 0xCB, 0x53, 0xCB, 0x54, 0xD1, 0xA6, /* 0x98-0x9B */ + 0xDE, 0xB5, 0xCB, 0x55, 0xCB, 0x56, 0xCB, 0x57, /* 0x9C-0x9F */ + 0xCB, 0x58, 0xCB, 0x59, 0xCB, 0x5A, 0xCB, 0x5B, /* 0xA0-0xA3 */ + 0xDE, 0xAF, 0xCB, 0x5C, 0xCB, 0x5D, 0xCB, 0x5E, /* 0xA4-0xA7 */ + 0xDE, 0xB0, 0xCB, 0x5F, 0xD0, 0xBD, 0xCB, 0x60, /* 0xA8-0xAB */ + 0xCB, 0x61, 0xCB, 0x62, 0xDE, 0xB4, 0xCA, 0xED, /* 0xAC-0xAF */ + 0xDE, 0xB9, 0xCB, 0x63, 0xCB, 0x64, 0xCB, 0x65, /* 0xB0-0xB3 */ + 0xCB, 0x66, 0xCB, 0x67, 0xCB, 0x68, 0xDE, 0xB8, /* 0xB4-0xB7 */ + 0xCB, 0x69, 0xDE, 0xB7, 0xCB, 0x6A, 0xCB, 0x6B, /* 0xB8-0xBB */ + 0xCB, 0x6C, 0xCB, 0x6D, 0xCB, 0x6E, 0xCB, 0x6F, /* 0xBC-0xBF */ + 0xCB, 0x70, 0xDE, 0xBB, 0xCB, 0x71, 0xCB, 0x72, /* 0xC0-0xC3 */ + 0xCB, 0x73, 0xCB, 0x74, 0xCB, 0x75, 0xCB, 0x76, /* 0xC4-0xC7 */ + 0xCB, 0x77, 0xBD, 0xE5, 0xCB, 0x78, 0xCB, 0x79, /* 0xC8-0xCB */ + 0xCB, 0x7A, 0xCB, 0x7B, 0xCB, 0x7C, 0xB2, 0xD8, /* 0xCC-0xCF */ + 0xC3, 0xEA, 0xCB, 0x7D, 0xCB, 0x7E, 0xDE, 0xBA, /* 0xD0-0xD3 */ + 0xCB, 0x80, 0xC5, 0xBA, 0xCB, 0x81, 0xCB, 0x82, /* 0xD4-0xD7 */ + 0xCB, 0x83, 0xCB, 0x84, 0xCB, 0x85, 0xCB, 0x86, /* 0xD8-0xDB */ + 0xDE, 0xBC, 0xCB, 0x87, 0xCB, 0x88, 0xCB, 0x89, /* 0xDC-0xDF */ + 0xCB, 0x8A, 0xCB, 0x8B, 0xCB, 0x8C, 0xCB, 0x8D, /* 0xE0-0xE3 */ + 0xCC, 0xD9, 0xCB, 0x8E, 0xCB, 0x8F, 0xCB, 0x90, /* 0xE4-0xE7 */ + 0xCB, 0x91, 0xB7, 0xAA, 0xCB, 0x92, 0xCB, 0x93, /* 0xE8-0xEB */ + 0xCB, 0x94, 0xCB, 0x95, 0xCB, 0x96, 0xCB, 0x97, /* 0xEC-0xEF */ + 0xCB, 0x98, 0xCB, 0x99, 0xCB, 0x9A, 0xCB, 0x9B, /* 0xF0-0xF3 */ + 0xCB, 0x9C, 0xCB, 0x9D, 0xCB, 0x9E, 0xCB, 0x9F, /* 0xF4-0xF7 */ + 0xCB, 0xA0, 0xCC, 0x40, 0xCC, 0x41, 0xD4, 0xE5, /* 0xF8-0xFB */ + 0xCC, 0x42, 0xCC, 0x43, 0xCC, 0x44, 0xDE, 0xBD, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_86[512] = { + 0xCC, 0x45, 0xCC, 0x46, 0xCC, 0x47, 0xCC, 0x48, /* 0x00-0x03 */ + 0xCC, 0x49, 0xDE, 0xBF, 0xCC, 0x4A, 0xCC, 0x4B, /* 0x04-0x07 */ + 0xCC, 0x4C, 0xCC, 0x4D, 0xCC, 0x4E, 0xCC, 0x4F, /* 0x08-0x0B */ + 0xCC, 0x50, 0xCC, 0x51, 0xCC, 0x52, 0xCC, 0x53, /* 0x0C-0x0F */ + 0xCC, 0x54, 0xC4, 0xA2, 0xCC, 0x55, 0xCC, 0x56, /* 0x10-0x13 */ + 0xCC, 0x57, 0xCC, 0x58, 0xDE, 0xC1, 0xCC, 0x59, /* 0x14-0x17 */ + 0xCC, 0x5A, 0xCC, 0x5B, 0xCC, 0x5C, 0xCC, 0x5D, /* 0x18-0x1B */ + 0xCC, 0x5E, 0xCC, 0x5F, 0xCC, 0x60, 0xCC, 0x61, /* 0x1C-0x1F */ + 0xCC, 0x62, 0xCC, 0x63, 0xCC, 0x64, 0xCC, 0x65, /* 0x20-0x23 */ + 0xCC, 0x66, 0xCC, 0x67, 0xCC, 0x68, 0xDE, 0xBE, /* 0x24-0x27 */ + 0xCC, 0x69, 0xDE, 0xC0, 0xCC, 0x6A, 0xCC, 0x6B, /* 0x28-0x2B */ + 0xCC, 0x6C, 0xCC, 0x6D, 0xCC, 0x6E, 0xCC, 0x6F, /* 0x2C-0x2F */ + 0xCC, 0x70, 0xCC, 0x71, 0xCC, 0x72, 0xCC, 0x73, /* 0x30-0x33 */ + 0xCC, 0x74, 0xCC, 0x75, 0xCC, 0x76, 0xCC, 0x77, /* 0x34-0x37 */ + 0xD5, 0xBA, 0xCC, 0x78, 0xCC, 0x79, 0xCC, 0x7A, /* 0x38-0x3B */ + 0xDE, 0xC2, 0xCC, 0x7B, 0xCC, 0x7C, 0xCC, 0x7D, /* 0x3C-0x3F */ + 0xCC, 0x7E, 0xCC, 0x80, 0xCC, 0x81, 0xCC, 0x82, /* 0x40-0x43 */ + 0xCC, 0x83, 0xCC, 0x84, 0xCC, 0x85, 0xCC, 0x86, /* 0x44-0x47 */ + 0xCC, 0x87, 0xCC, 0x88, 0xCC, 0x89, 0xCC, 0x8A, /* 0x48-0x4B */ + 0xCC, 0x8B, 0xF2, 0xAE, 0xBB, 0xA2, 0xC2, 0xB2, /* 0x4C-0x4F */ + 0xC5, 0xB0, 0xC2, 0xC7, 0xCC, 0x8C, 0xCC, 0x8D, /* 0x50-0x53 */ + 0xF2, 0xAF, 0xCC, 0x8E, 0xCC, 0x8F, 0xCC, 0x90, /* 0x54-0x57 */ + 0xCC, 0x91, 0xCC, 0x92, 0xD0, 0xE9, 0xCC, 0x93, /* 0x58-0x5B */ + 0xCC, 0x94, 0xCC, 0x95, 0xD3, 0xDD, 0xCC, 0x96, /* 0x5C-0x5F */ + 0xCC, 0x97, 0xCC, 0x98, 0xEB, 0xBD, 0xCC, 0x99, /* 0x60-0x63 */ + 0xCC, 0x9A, 0xCC, 0x9B, 0xCC, 0x9C, 0xCC, 0x9D, /* 0x64-0x67 */ + 0xCC, 0x9E, 0xCC, 0x9F, 0xCC, 0xA0, 0xB3, 0xE6, /* 0x68-0x6B */ + 0xF2, 0xB0, 0xCD, 0x40, 0xF2, 0xB1, 0xCD, 0x41, /* 0x6C-0x6F */ + 0xCD, 0x42, 0xCA, 0xAD, 0xCD, 0x43, 0xCD, 0x44, /* 0x70-0x73 */ + 0xCD, 0x45, 0xCD, 0x46, 0xCD, 0x47, 0xCD, 0x48, /* 0x74-0x77 */ + 0xCD, 0x49, 0xBA, 0xE7, 0xF2, 0xB3, 0xF2, 0xB5, /* 0x78-0x7B */ + 0xF2, 0xB4, 0xCB, 0xE4, 0xCF, 0xBA, 0xF2, 0xB2, /* 0x7C-0x7F */ + + 0xCA, 0xB4, 0xD2, 0xCF, 0xC2, 0xEC, 0xCD, 0x4A, /* 0x80-0x83 */ + 0xCD, 0x4B, 0xCD, 0x4C, 0xCD, 0x4D, 0xCD, 0x4E, /* 0x84-0x87 */ + 0xCD, 0x4F, 0xCD, 0x50, 0xCE, 0xC3, 0xF2, 0xB8, /* 0x88-0x8B */ + 0xB0, 0xF6, 0xF2, 0xB7, 0xCD, 0x51, 0xCD, 0x52, /* 0x8C-0x8F */ + 0xCD, 0x53, 0xCD, 0x54, 0xCD, 0x55, 0xF2, 0xBE, /* 0x90-0x93 */ + 0xCD, 0x56, 0xB2, 0xCF, 0xCD, 0x57, 0xCD, 0x58, /* 0x94-0x97 */ + 0xCD, 0x59, 0xCD, 0x5A, 0xCD, 0x5B, 0xCD, 0x5C, /* 0x98-0x9B */ + 0xD1, 0xC1, 0xF2, 0xBA, 0xCD, 0x5D, 0xCD, 0x5E, /* 0x9C-0x9F */ + 0xCD, 0x5F, 0xCD, 0x60, 0xCD, 0x61, 0xF2, 0xBC, /* 0xA0-0xA3 */ + 0xD4, 0xE9, 0xCD, 0x62, 0xCD, 0x63, 0xF2, 0xBB, /* 0xA4-0xA7 */ + 0xF2, 0xB6, 0xF2, 0xBF, 0xF2, 0xBD, 0xCD, 0x64, /* 0xA8-0xAB */ + 0xF2, 0xB9, 0xCD, 0x65, 0xCD, 0x66, 0xF2, 0xC7, /* 0xAC-0xAF */ + 0xF2, 0xC4, 0xF2, 0xC6, 0xCD, 0x67, 0xCD, 0x68, /* 0xB0-0xB3 */ + 0xF2, 0xCA, 0xF2, 0xC2, 0xF2, 0xC0, 0xCD, 0x69, /* 0xB4-0xB7 */ + 0xCD, 0x6A, 0xCD, 0x6B, 0xF2, 0xC5, 0xCD, 0x6C, /* 0xB8-0xBB */ + 0xCD, 0x6D, 0xCD, 0x6E, 0xCD, 0x6F, 0xCD, 0x70, /* 0xBC-0xBF */ + 0xD6, 0xFB, 0xCD, 0x71, 0xCD, 0x72, 0xCD, 0x73, /* 0xC0-0xC3 */ + 0xF2, 0xC1, 0xCD, 0x74, 0xC7, 0xF9, 0xC9, 0xDF, /* 0xC4-0xC7 */ + 0xCD, 0x75, 0xF2, 0xC8, 0xB9, 0xC6, 0xB5, 0xB0, /* 0xC8-0xCB */ + 0xCD, 0x76, 0xCD, 0x77, 0xF2, 0xC3, 0xF2, 0xC9, /* 0xCC-0xCF */ + 0xF2, 0xD0, 0xF2, 0xD6, 0xCD, 0x78, 0xCD, 0x79, /* 0xD0-0xD3 */ + 0xBB, 0xD7, 0xCD, 0x7A, 0xCD, 0x7B, 0xCD, 0x7C, /* 0xD4-0xD7 */ + 0xF2, 0xD5, 0xCD, 0xDC, 0xCD, 0x7D, 0xD6, 0xEB, /* 0xD8-0xDB */ + 0xCD, 0x7E, 0xCD, 0x80, 0xF2, 0xD2, 0xF2, 0xD4, /* 0xDC-0xDF */ + 0xCD, 0x81, 0xCD, 0x82, 0xCD, 0x83, 0xCD, 0x84, /* 0xE0-0xE3 */ + 0xB8, 0xF2, 0xCD, 0x85, 0xCD, 0x86, 0xCD, 0x87, /* 0xE4-0xE7 */ + 0xCD, 0x88, 0xF2, 0xCB, 0xCD, 0x89, 0xCD, 0x8A, /* 0xE8-0xEB */ + 0xCD, 0x8B, 0xF2, 0xCE, 0xC2, 0xF9, 0xCD, 0x8C, /* 0xEC-0xEF */ + 0xD5, 0xDD, 0xF2, 0xCC, 0xF2, 0xCD, 0xF2, 0xCF, /* 0xF0-0xF3 */ + 0xF2, 0xD3, 0xCD, 0x8D, 0xCD, 0x8E, 0xCD, 0x8F, /* 0xF4-0xF7 */ + 0xF2, 0xD9, 0xD3, 0xBC, 0xCD, 0x90, 0xCD, 0x91, /* 0xF8-0xFB */ + 0xCD, 0x92, 0xCD, 0x93, 0xB6, 0xEA, 0xCD, 0x94, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_87[512] = { + 0xCA, 0xF1, 0xCD, 0x95, 0xB7, 0xE4, 0xF2, 0xD7, /* 0x00-0x03 */ + 0xCD, 0x96, 0xCD, 0x97, 0xCD, 0x98, 0xF2, 0xD8, /* 0x04-0x07 */ + 0xF2, 0xDA, 0xF2, 0xDD, 0xF2, 0xDB, 0xCD, 0x99, /* 0x08-0x0B */ + 0xCD, 0x9A, 0xF2, 0xDC, 0xCD, 0x9B, 0xCD, 0x9C, /* 0x0C-0x0F */ + 0xCD, 0x9D, 0xCD, 0x9E, 0xD1, 0xD1, 0xF2, 0xD1, /* 0x10-0x13 */ + 0xCD, 0x9F, 0xCD, 0xC9, 0xCD, 0xA0, 0xCE, 0xCF, /* 0x14-0x17 */ + 0xD6, 0xA9, 0xCE, 0x40, 0xF2, 0xE3, 0xCE, 0x41, /* 0x18-0x1B */ + 0xC3, 0xDB, 0xCE, 0x42, 0xF2, 0xE0, 0xCE, 0x43, /* 0x1C-0x1F */ + 0xCE, 0x44, 0xC0, 0xAF, 0xF2, 0xEC, 0xF2, 0xDE, /* 0x20-0x23 */ + 0xCE, 0x45, 0xF2, 0xE1, 0xCE, 0x46, 0xCE, 0x47, /* 0x24-0x27 */ + 0xCE, 0x48, 0xF2, 0xE8, 0xCE, 0x49, 0xCE, 0x4A, /* 0x28-0x2B */ + 0xCE, 0x4B, 0xCE, 0x4C, 0xF2, 0xE2, 0xCE, 0x4D, /* 0x2C-0x2F */ + 0xCE, 0x4E, 0xF2, 0xE7, 0xCE, 0x4F, 0xCE, 0x50, /* 0x30-0x33 */ + 0xF2, 0xE6, 0xCE, 0x51, 0xCE, 0x52, 0xF2, 0xE9, /* 0x34-0x37 */ + 0xCE, 0x53, 0xCE, 0x54, 0xCE, 0x55, 0xF2, 0xDF, /* 0x38-0x3B */ + 0xCE, 0x56, 0xCE, 0x57, 0xF2, 0xE4, 0xF2, 0xEA, /* 0x3C-0x3F */ + 0xCE, 0x58, 0xCE, 0x59, 0xCE, 0x5A, 0xCE, 0x5B, /* 0x40-0x43 */ + 0xCE, 0x5C, 0xCE, 0x5D, 0xCE, 0x5E, 0xD3, 0xAC, /* 0x44-0x47 */ + 0xF2, 0xE5, 0xB2, 0xF5, 0xCE, 0x5F, 0xCE, 0x60, /* 0x48-0x4B */ + 0xF2, 0xF2, 0xCE, 0x61, 0xD0, 0xAB, 0xCE, 0x62, /* 0x4C-0x4F */ + 0xCE, 0x63, 0xCE, 0x64, 0xCE, 0x65, 0xF2, 0xF5, /* 0x50-0x53 */ + 0xCE, 0x66, 0xCE, 0x67, 0xCE, 0x68, 0xBB, 0xC8, /* 0x54-0x57 */ + 0xCE, 0x69, 0xF2, 0xF9, 0xCE, 0x6A, 0xCE, 0x6B, /* 0x58-0x5B */ + 0xCE, 0x6C, 0xCE, 0x6D, 0xCE, 0x6E, 0xCE, 0x6F, /* 0x5C-0x5F */ + 0xF2, 0xF0, 0xCE, 0x70, 0xCE, 0x71, 0xF2, 0xF6, /* 0x60-0x63 */ + 0xF2, 0xF8, 0xF2, 0xFA, 0xCE, 0x72, 0xCE, 0x73, /* 0x64-0x67 */ + 0xCE, 0x74, 0xCE, 0x75, 0xCE, 0x76, 0xCE, 0x77, /* 0x68-0x6B */ + 0xCE, 0x78, 0xCE, 0x79, 0xF2, 0xF3, 0xCE, 0x7A, /* 0x6C-0x6F */ + 0xF2, 0xF1, 0xCE, 0x7B, 0xCE, 0x7C, 0xCE, 0x7D, /* 0x70-0x73 */ + 0xBA, 0xFB, 0xCE, 0x7E, 0xB5, 0xFB, 0xCE, 0x80, /* 0x74-0x77 */ + 0xCE, 0x81, 0xCE, 0x82, 0xCE, 0x83, 0xF2, 0xEF, /* 0x78-0x7B */ + 0xF2, 0xF7, 0xF2, 0xED, 0xF2, 0xEE, 0xCE, 0x84, /* 0x7C-0x7F */ + + 0xCE, 0x85, 0xCE, 0x86, 0xF2, 0xEB, 0xF3, 0xA6, /* 0x80-0x83 */ + 0xCE, 0x87, 0xF3, 0xA3, 0xCE, 0x88, 0xCE, 0x89, /* 0x84-0x87 */ + 0xF3, 0xA2, 0xCE, 0x8A, 0xCE, 0x8B, 0xF2, 0xF4, /* 0x88-0x8B */ + 0xCE, 0x8C, 0xC8, 0xDA, 0xCE, 0x8D, 0xCE, 0x8E, /* 0x8C-0x8F */ + 0xCE, 0x8F, 0xCE, 0x90, 0xCE, 0x91, 0xF2, 0xFB, /* 0x90-0x93 */ + 0xCE, 0x92, 0xCE, 0x93, 0xCE, 0x94, 0xF3, 0xA5, /* 0x94-0x97 */ + 0xCE, 0x95, 0xCE, 0x96, 0xCE, 0x97, 0xCE, 0x98, /* 0x98-0x9B */ + 0xCE, 0x99, 0xCE, 0x9A, 0xCE, 0x9B, 0xC3, 0xF8, /* 0x9C-0x9F */ + 0xCE, 0x9C, 0xCE, 0x9D, 0xCE, 0x9E, 0xCE, 0x9F, /* 0xA0-0xA3 */ + 0xCE, 0xA0, 0xCF, 0x40, 0xCF, 0x41, 0xCF, 0x42, /* 0xA4-0xA7 */ + 0xF2, 0xFD, 0xCF, 0x43, 0xCF, 0x44, 0xF3, 0xA7, /* 0xA8-0xAB */ + 0xF3, 0xA9, 0xF3, 0xA4, 0xCF, 0x45, 0xF2, 0xFC, /* 0xAC-0xAF */ + 0xCF, 0x46, 0xCF, 0x47, 0xCF, 0x48, 0xF3, 0xAB, /* 0xB0-0xB3 */ + 0xCF, 0x49, 0xF3, 0xAA, 0xCF, 0x4A, 0xCF, 0x4B, /* 0xB4-0xB7 */ + 0xCF, 0x4C, 0xCF, 0x4D, 0xC2, 0xDD, 0xCF, 0x4E, /* 0xB8-0xBB */ + 0xCF, 0x4F, 0xF3, 0xAE, 0xCF, 0x50, 0xCF, 0x51, /* 0xBC-0xBF */ + 0xF3, 0xB0, 0xCF, 0x52, 0xCF, 0x53, 0xCF, 0x54, /* 0xC0-0xC3 */ + 0xCF, 0x55, 0xCF, 0x56, 0xF3, 0xA1, 0xCF, 0x57, /* 0xC4-0xC7 */ + 0xCF, 0x58, 0xCF, 0x59, 0xF3, 0xB1, 0xF3, 0xAC, /* 0xC8-0xCB */ + 0xCF, 0x5A, 0xCF, 0x5B, 0xCF, 0x5C, 0xCF, 0x5D, /* 0xCC-0xCF */ + 0xCF, 0x5E, 0xF3, 0xAF, 0xF2, 0xFE, 0xF3, 0xAD, /* 0xD0-0xD3 */ + 0xCF, 0x5F, 0xCF, 0x60, 0xCF, 0x61, 0xCF, 0x62, /* 0xD4-0xD7 */ + 0xCF, 0x63, 0xCF, 0x64, 0xCF, 0x65, 0xF3, 0xB2, /* 0xD8-0xDB */ + 0xCF, 0x66, 0xCF, 0x67, 0xCF, 0x68, 0xCF, 0x69, /* 0xDC-0xDF */ + 0xF3, 0xB4, 0xCF, 0x6A, 0xCF, 0x6B, 0xCF, 0x6C, /* 0xE0-0xE3 */ + 0xCF, 0x6D, 0xF3, 0xA8, 0xCF, 0x6E, 0xCF, 0x6F, /* 0xE4-0xE7 */ + 0xCF, 0x70, 0xCF, 0x71, 0xF3, 0xB3, 0xCF, 0x72, /* 0xE8-0xEB */ + 0xCF, 0x73, 0xCF, 0x74, 0xF3, 0xB5, 0xCF, 0x75, /* 0xEC-0xEF */ + 0xCF, 0x76, 0xCF, 0x77, 0xCF, 0x78, 0xCF, 0x79, /* 0xF0-0xF3 */ + 0xCF, 0x7A, 0xCF, 0x7B, 0xCF, 0x7C, 0xCF, 0x7D, /* 0xF4-0xF7 */ + 0xCF, 0x7E, 0xD0, 0xB7, 0xCF, 0x80, 0xCF, 0x81, /* 0xF8-0xFB */ + 0xCF, 0x82, 0xCF, 0x83, 0xF3, 0xB8, 0xCF, 0x84, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_88[512] = { + 0xCF, 0x85, 0xCF, 0x86, 0xCF, 0x87, 0xD9, 0xF9, /* 0x00-0x03 */ + 0xCF, 0x88, 0xCF, 0x89, 0xCF, 0x8A, 0xCF, 0x8B, /* 0x04-0x07 */ + 0xCF, 0x8C, 0xCF, 0x8D, 0xF3, 0xB9, 0xCF, 0x8E, /* 0x08-0x0B */ + 0xCF, 0x8F, 0xCF, 0x90, 0xCF, 0x91, 0xCF, 0x92, /* 0x0C-0x0F */ + 0xCF, 0x93, 0xCF, 0x94, 0xCF, 0x95, 0xF3, 0xB7, /* 0x10-0x13 */ + 0xCF, 0x96, 0xC8, 0xE4, 0xF3, 0xB6, 0xCF, 0x97, /* 0x14-0x17 */ + 0xCF, 0x98, 0xCF, 0x99, 0xCF, 0x9A, 0xF3, 0xBA, /* 0x18-0x1B */ + 0xCF, 0x9B, 0xCF, 0x9C, 0xCF, 0x9D, 0xCF, 0x9E, /* 0x1C-0x1F */ + 0xCF, 0x9F, 0xF3, 0xBB, 0xB4, 0xC0, 0xCF, 0xA0, /* 0x20-0x23 */ + 0xD0, 0x40, 0xD0, 0x41, 0xD0, 0x42, 0xD0, 0x43, /* 0x24-0x27 */ + 0xD0, 0x44, 0xD0, 0x45, 0xD0, 0x46, 0xD0, 0x47, /* 0x28-0x2B */ + 0xD0, 0x48, 0xD0, 0x49, 0xD0, 0x4A, 0xD0, 0x4B, /* 0x2C-0x2F */ + 0xD0, 0x4C, 0xD0, 0x4D, 0xEE, 0xC3, 0xD0, 0x4E, /* 0x30-0x33 */ + 0xD0, 0x4F, 0xD0, 0x50, 0xD0, 0x51, 0xD0, 0x52, /* 0x34-0x37 */ + 0xD0, 0x53, 0xF3, 0xBC, 0xD0, 0x54, 0xD0, 0x55, /* 0x38-0x3B */ + 0xF3, 0xBD, 0xD0, 0x56, 0xD0, 0x57, 0xD0, 0x58, /* 0x3C-0x3F */ + 0xD1, 0xAA, 0xD0, 0x59, 0xD0, 0x5A, 0xD0, 0x5B, /* 0x40-0x43 */ + 0xF4, 0xAC, 0xD0, 0xC6, 0xD0, 0x5C, 0xD0, 0x5D, /* 0x44-0x47 */ + 0xD0, 0x5E, 0xD0, 0x5F, 0xD0, 0x60, 0xD0, 0x61, /* 0x48-0x4B */ + 0xD0, 0xD0, 0xD1, 0xDC, 0xD0, 0x62, 0xD0, 0x63, /* 0x4C-0x4F */ + 0xD0, 0x64, 0xD0, 0x65, 0xD0, 0x66, 0xD0, 0x67, /* 0x50-0x53 */ + 0xCF, 0xCE, 0xD0, 0x68, 0xD0, 0x69, 0xBD, 0xD6, /* 0x54-0x57 */ + 0xD0, 0x6A, 0xD1, 0xC3, 0xD0, 0x6B, 0xD0, 0x6C, /* 0x58-0x5B */ + 0xD0, 0x6D, 0xD0, 0x6E, 0xD0, 0x6F, 0xD0, 0x70, /* 0x5C-0x5F */ + 0xD0, 0x71, 0xBA, 0xE2, 0xE1, 0xE9, 0xD2, 0xC2, /* 0x60-0x63 */ + 0xF1, 0xC2, 0xB2, 0xB9, 0xD0, 0x72, 0xD0, 0x73, /* 0x64-0x67 */ + 0xB1, 0xED, 0xF1, 0xC3, 0xD0, 0x74, 0xC9, 0xC0, /* 0x68-0x6B */ + 0xB3, 0xC4, 0xD0, 0x75, 0xD9, 0xF2, 0xD0, 0x76, /* 0x6C-0x6F */ + 0xCB, 0xA5, 0xD0, 0x77, 0xF1, 0xC4, 0xD0, 0x78, /* 0x70-0x73 */ + 0xD0, 0x79, 0xD0, 0x7A, 0xD0, 0x7B, 0xD6, 0xD4, /* 0x74-0x77 */ + 0xD0, 0x7C, 0xD0, 0x7D, 0xD0, 0x7E, 0xD0, 0x80, /* 0x78-0x7B */ + 0xD0, 0x81, 0xF1, 0xC5, 0xF4, 0xC0, 0xF1, 0xC6, /* 0x7C-0x7F */ + + 0xD0, 0x82, 0xD4, 0xAC, 0xF1, 0xC7, 0xD0, 0x83, /* 0x80-0x83 */ + 0xB0, 0xC0, 0xF4, 0xC1, 0xD0, 0x84, 0xD0, 0x85, /* 0x84-0x87 */ + 0xF4, 0xC2, 0xD0, 0x86, 0xD0, 0x87, 0xB4, 0xFC, /* 0x88-0x8B */ + 0xD0, 0x88, 0xC5, 0xDB, 0xD0, 0x89, 0xD0, 0x8A, /* 0x8C-0x8F */ + 0xD0, 0x8B, 0xD0, 0x8C, 0xCC, 0xBB, 0xD0, 0x8D, /* 0x90-0x93 */ + 0xD0, 0x8E, 0xD0, 0x8F, 0xD0, 0xE4, 0xD0, 0x90, /* 0x94-0x97 */ + 0xD0, 0x91, 0xD0, 0x92, 0xD0, 0x93, 0xD0, 0x94, /* 0x98-0x9B */ + 0xCD, 0xE0, 0xD0, 0x95, 0xD0, 0x96, 0xD0, 0x97, /* 0x9C-0x9F */ + 0xD0, 0x98, 0xD0, 0x99, 0xF1, 0xC8, 0xD0, 0x9A, /* 0xA0-0xA3 */ + 0xD9, 0xF3, 0xD0, 0x9B, 0xD0, 0x9C, 0xD0, 0x9D, /* 0xA4-0xA7 */ + 0xD0, 0x9E, 0xD0, 0x9F, 0xD0, 0xA0, 0xB1, 0xBB, /* 0xA8-0xAB */ + 0xD1, 0x40, 0xCF, 0xAE, 0xD1, 0x41, 0xD1, 0x42, /* 0xAC-0xAF */ + 0xD1, 0x43, 0xB8, 0xA4, 0xD1, 0x44, 0xD1, 0x45, /* 0xB0-0xB3 */ + 0xD1, 0x46, 0xD1, 0x47, 0xD1, 0x48, 0xF1, 0xCA, /* 0xB4-0xB7 */ + 0xD1, 0x49, 0xD1, 0x4A, 0xD1, 0x4B, 0xD1, 0x4C, /* 0xB8-0xBB */ + 0xF1, 0xCB, 0xD1, 0x4D, 0xD1, 0x4E, 0xD1, 0x4F, /* 0xBC-0xBF */ + 0xD1, 0x50, 0xB2, 0xC3, 0xC1, 0xD1, 0xD1, 0x51, /* 0xC0-0xC3 */ + 0xD1, 0x52, 0xD7, 0xB0, 0xF1, 0xC9, 0xD1, 0x53, /* 0xC4-0xC7 */ + 0xD1, 0x54, 0xF1, 0xCC, 0xD1, 0x55, 0xD1, 0x56, /* 0xC8-0xCB */ + 0xD1, 0x57, 0xD1, 0x58, 0xF1, 0xCE, 0xD1, 0x59, /* 0xCC-0xCF */ + 0xD1, 0x5A, 0xD1, 0x5B, 0xD9, 0xF6, 0xD1, 0x5C, /* 0xD0-0xD3 */ + 0xD2, 0xE1, 0xD4, 0xA3, 0xD1, 0x5D, 0xD1, 0x5E, /* 0xD4-0xD7 */ + 0xF4, 0xC3, 0xC8, 0xB9, 0xD1, 0x5F, 0xD1, 0x60, /* 0xD8-0xDB */ + 0xD1, 0x61, 0xD1, 0x62, 0xD1, 0x63, 0xF4, 0xC4, /* 0xDC-0xDF */ + 0xD1, 0x64, 0xD1, 0x65, 0xF1, 0xCD, 0xF1, 0xCF, /* 0xE0-0xE3 */ + 0xBF, 0xE3, 0xF1, 0xD0, 0xD1, 0x66, 0xD1, 0x67, /* 0xE4-0xE7 */ + 0xF1, 0xD4, 0xD1, 0x68, 0xD1, 0x69, 0xD1, 0x6A, /* 0xE8-0xEB */ + 0xD1, 0x6B, 0xD1, 0x6C, 0xD1, 0x6D, 0xD1, 0x6E, /* 0xEC-0xEF */ + 0xF1, 0xD6, 0xF1, 0xD1, 0xD1, 0x6F, 0xC9, 0xD1, /* 0xF0-0xF3 */ + 0xC5, 0xE1, 0xD1, 0x70, 0xD1, 0x71, 0xD1, 0x72, /* 0xF4-0xF7 */ + 0xC2, 0xE3, 0xB9, 0xFC, 0xD1, 0x73, 0xD1, 0x74, /* 0xF8-0xFB */ + 0xF1, 0xD3, 0xD1, 0x75, 0xF1, 0xD5, 0xD1, 0x76, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_89[512] = { + 0xD1, 0x77, 0xD1, 0x78, 0xB9, 0xD3, 0xD1, 0x79, /* 0x00-0x03 */ + 0xD1, 0x7A, 0xD1, 0x7B, 0xD1, 0x7C, 0xD1, 0x7D, /* 0x04-0x07 */ + 0xD1, 0x7E, 0xD1, 0x80, 0xF1, 0xDB, 0xD1, 0x81, /* 0x08-0x0B */ + 0xD1, 0x82, 0xD1, 0x83, 0xD1, 0x84, 0xD1, 0x85, /* 0x0C-0x0F */ + 0xBA, 0xD6, 0xD1, 0x86, 0xB0, 0xFD, 0xF1, 0xD9, /* 0x10-0x13 */ + 0xD1, 0x87, 0xD1, 0x88, 0xD1, 0x89, 0xD1, 0x8A, /* 0x14-0x17 */ + 0xD1, 0x8B, 0xF1, 0xD8, 0xF1, 0xD2, 0xF1, 0xDA, /* 0x18-0x1B */ + 0xD1, 0x8C, 0xD1, 0x8D, 0xD1, 0x8E, 0xD1, 0x8F, /* 0x1C-0x1F */ + 0xD1, 0x90, 0xF1, 0xD7, 0xD1, 0x91, 0xD1, 0x92, /* 0x20-0x23 */ + 0xD1, 0x93, 0xC8, 0xEC, 0xD1, 0x94, 0xD1, 0x95, /* 0x24-0x27 */ + 0xD1, 0x96, 0xD1, 0x97, 0xCD, 0xCA, 0xF1, 0xDD, /* 0x28-0x2B */ + 0xD1, 0x98, 0xD1, 0x99, 0xD1, 0x9A, 0xD1, 0x9B, /* 0x2C-0x2F */ + 0xE5, 0xBD, 0xD1, 0x9C, 0xD1, 0x9D, 0xD1, 0x9E, /* 0x30-0x33 */ + 0xF1, 0xDC, 0xD1, 0x9F, 0xF1, 0xDE, 0xD1, 0xA0, /* 0x34-0x37 */ + 0xD2, 0x40, 0xD2, 0x41, 0xD2, 0x42, 0xD2, 0x43, /* 0x38-0x3B */ + 0xD2, 0x44, 0xD2, 0x45, 0xD2, 0x46, 0xD2, 0x47, /* 0x3C-0x3F */ + 0xD2, 0x48, 0xF1, 0xDF, 0xD2, 0x49, 0xD2, 0x4A, /* 0x40-0x43 */ + 0xCF, 0xE5, 0xD2, 0x4B, 0xD2, 0x4C, 0xD2, 0x4D, /* 0x44-0x47 */ + 0xD2, 0x4E, 0xD2, 0x4F, 0xD2, 0x50, 0xD2, 0x51, /* 0x48-0x4B */ + 0xD2, 0x52, 0xD2, 0x53, 0xD2, 0x54, 0xD2, 0x55, /* 0x4C-0x4F */ + 0xD2, 0x56, 0xD2, 0x57, 0xD2, 0x58, 0xD2, 0x59, /* 0x50-0x53 */ + 0xD2, 0x5A, 0xD2, 0x5B, 0xD2, 0x5C, 0xD2, 0x5D, /* 0x54-0x57 */ + 0xD2, 0x5E, 0xD2, 0x5F, 0xD2, 0x60, 0xD2, 0x61, /* 0x58-0x5B */ + 0xD2, 0x62, 0xD2, 0x63, 0xF4, 0xC5, 0xBD, 0xF3, /* 0x5C-0x5F */ + 0xD2, 0x64, 0xD2, 0x65, 0xD2, 0x66, 0xD2, 0x67, /* 0x60-0x63 */ + 0xD2, 0x68, 0xD2, 0x69, 0xF1, 0xE0, 0xD2, 0x6A, /* 0x64-0x67 */ + 0xD2, 0x6B, 0xD2, 0x6C, 0xD2, 0x6D, 0xD2, 0x6E, /* 0x68-0x6B */ + 0xD2, 0x6F, 0xD2, 0x70, 0xD2, 0x71, 0xD2, 0x72, /* 0x6C-0x6F */ + 0xD2, 0x73, 0xD2, 0x74, 0xD2, 0x75, 0xD2, 0x76, /* 0x70-0x73 */ + 0xD2, 0x77, 0xD2, 0x78, 0xD2, 0x79, 0xD2, 0x7A, /* 0x74-0x77 */ + 0xD2, 0x7B, 0xD2, 0x7C, 0xD2, 0x7D, 0xF1, 0xE1, /* 0x78-0x7B */ + 0xD2, 0x7E, 0xD2, 0x80, 0xD2, 0x81, 0xCE, 0xF7, /* 0x7C-0x7F */ + + 0xD2, 0x82, 0xD2, 0xAA, 0xD2, 0x83, 0xF1, 0xFB, /* 0x80-0x83 */ + 0xD2, 0x84, 0xD2, 0x85, 0xB8, 0xB2, 0xD2, 0x86, /* 0x84-0x87 */ + 0xD2, 0x87, 0xD2, 0x88, 0xD2, 0x89, 0xD2, 0x8A, /* 0x88-0x8B */ + 0xD2, 0x8B, 0xD2, 0x8C, 0xD2, 0x8D, 0xD2, 0x8E, /* 0x8C-0x8F */ + 0xD2, 0x8F, 0xD2, 0x90, 0xD2, 0x91, 0xD2, 0x92, /* 0x90-0x93 */ + 0xD2, 0x93, 0xD2, 0x94, 0xD2, 0x95, 0xD2, 0x96, /* 0x94-0x97 */ + 0xD2, 0x97, 0xD2, 0x98, 0xD2, 0x99, 0xD2, 0x9A, /* 0x98-0x9B */ + 0xD2, 0x9B, 0xD2, 0x9C, 0xD2, 0x9D, 0xD2, 0x9E, /* 0x9C-0x9F */ + 0xD2, 0x9F, 0xD2, 0xA0, 0xD3, 0x40, 0xD3, 0x41, /* 0xA0-0xA3 */ + 0xD3, 0x42, 0xD3, 0x43, 0xD3, 0x44, 0xD3, 0x45, /* 0xA4-0xA7 */ + 0xD3, 0x46, 0xD3, 0x47, 0xD3, 0x48, 0xD3, 0x49, /* 0xA8-0xAB */ + 0xD3, 0x4A, 0xD3, 0x4B, 0xD3, 0x4C, 0xD3, 0x4D, /* 0xAC-0xAF */ + 0xD3, 0x4E, 0xD3, 0x4F, 0xD3, 0x50, 0xD3, 0x51, /* 0xB0-0xB3 */ + 0xD3, 0x52, 0xD3, 0x53, 0xD3, 0x54, 0xD3, 0x55, /* 0xB4-0xB7 */ + 0xD3, 0x56, 0xD3, 0x57, 0xD3, 0x58, 0xD3, 0x59, /* 0xB8-0xBB */ + 0xD3, 0x5A, 0xD3, 0x5B, 0xD3, 0x5C, 0xD3, 0x5D, /* 0xBC-0xBF */ + 0xD3, 0x5E, 0xBC, 0xFB, 0xB9, 0xDB, 0xD3, 0x5F, /* 0xC0-0xC3 */ + 0xB9, 0xE6, 0xC3, 0xD9, 0xCA, 0xD3, 0xEA, 0xE8, /* 0xC4-0xC7 */ + 0xC0, 0xC0, 0xBE, 0xF5, 0xEA, 0xE9, 0xEA, 0xEA, /* 0xC8-0xCB */ + 0xEA, 0xEB, 0xD3, 0x60, 0xEA, 0xEC, 0xEA, 0xED, /* 0xCC-0xCF */ + 0xEA, 0xEE, 0xEA, 0xEF, 0xBD, 0xC7, 0xD3, 0x61, /* 0xD0-0xD3 */ + 0xD3, 0x62, 0xD3, 0x63, 0xF5, 0xFB, 0xD3, 0x64, /* 0xD4-0xD7 */ + 0xD3, 0x65, 0xD3, 0x66, 0xF5, 0xFD, 0xD3, 0x67, /* 0xD8-0xDB */ + 0xF5, 0xFE, 0xD3, 0x68, 0xF5, 0xFC, 0xD3, 0x69, /* 0xDC-0xDF */ + 0xD3, 0x6A, 0xD3, 0x6B, 0xD3, 0x6C, 0xBD, 0xE2, /* 0xE0-0xE3 */ + 0xD3, 0x6D, 0xF6, 0xA1, 0xB4, 0xA5, 0xD3, 0x6E, /* 0xE4-0xE7 */ + 0xD3, 0x6F, 0xD3, 0x70, 0xD3, 0x71, 0xF6, 0xA2, /* 0xE8-0xEB */ + 0xD3, 0x72, 0xD3, 0x73, 0xD3, 0x74, 0xF6, 0xA3, /* 0xEC-0xEF */ + 0xD3, 0x75, 0xD3, 0x76, 0xD3, 0x77, 0xEC, 0xB2, /* 0xF0-0xF3 */ + 0xD3, 0x78, 0xD3, 0x79, 0xD3, 0x7A, 0xD3, 0x7B, /* 0xF4-0xF7 */ + 0xD3, 0x7C, 0xD3, 0x7D, 0xD3, 0x7E, 0xD3, 0x80, /* 0xF8-0xFB */ + 0xD3, 0x81, 0xD3, 0x82, 0xD3, 0x83, 0xD3, 0x84, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_8A[512] = { + 0xD1, 0xD4, 0xD3, 0x85, 0xD3, 0x86, 0xD3, 0x87, /* 0x00-0x03 */ + 0xD3, 0x88, 0xD3, 0x89, 0xD3, 0x8A, 0xD9, 0xEA, /* 0x04-0x07 */ + 0xD3, 0x8B, 0xD3, 0x8C, 0xD3, 0x8D, 0xD3, 0x8E, /* 0x08-0x0B */ + 0xD3, 0x8F, 0xD3, 0x90, 0xD3, 0x91, 0xD3, 0x92, /* 0x0C-0x0F */ + 0xD3, 0x93, 0xD3, 0x94, 0xD3, 0x95, 0xD3, 0x96, /* 0x10-0x13 */ + 0xD3, 0x97, 0xD3, 0x98, 0xD3, 0x99, 0xD3, 0x9A, /* 0x14-0x17 */ + 0xD3, 0x9B, 0xD3, 0x9C, 0xD3, 0x9D, 0xD3, 0x9E, /* 0x18-0x1B */ + 0xD3, 0x9F, 0xD3, 0xA0, 0xD4, 0x40, 0xD4, 0x41, /* 0x1C-0x1F */ + 0xD4, 0x42, 0xD4, 0x43, 0xD4, 0x44, 0xD4, 0x45, /* 0x20-0x23 */ + 0xD4, 0x46, 0xD4, 0x47, 0xD4, 0x48, 0xD4, 0x49, /* 0x24-0x27 */ + 0xD4, 0x4A, 0xD4, 0x4B, 0xD4, 0x4C, 0xD4, 0x4D, /* 0x28-0x2B */ + 0xD4, 0x4E, 0xD4, 0x4F, 0xD4, 0x50, 0xD4, 0x51, /* 0x2C-0x2F */ + 0xD4, 0x52, 0xD4, 0x53, 0xD4, 0x54, 0xD4, 0x55, /* 0x30-0x33 */ + 0xD4, 0x56, 0xD4, 0x57, 0xD4, 0x58, 0xD4, 0x59, /* 0x34-0x37 */ + 0xD4, 0x5A, 0xD4, 0x5B, 0xD4, 0x5C, 0xD4, 0x5D, /* 0x38-0x3B */ + 0xD4, 0x5E, 0xD4, 0x5F, 0xF6, 0xA4, 0xD4, 0x60, /* 0x3C-0x3F */ + 0xD4, 0x61, 0xD4, 0x62, 0xD4, 0x63, 0xD4, 0x64, /* 0x40-0x43 */ + 0xD4, 0x65, 0xD4, 0x66, 0xD4, 0x67, 0xD4, 0x68, /* 0x44-0x47 */ + 0xEE, 0xBA, 0xD4, 0x69, 0xD4, 0x6A, 0xD4, 0x6B, /* 0x48-0x4B */ + 0xD4, 0x6C, 0xD4, 0x6D, 0xD4, 0x6E, 0xD4, 0x6F, /* 0x4C-0x4F */ + 0xD4, 0x70, 0xD4, 0x71, 0xD4, 0x72, 0xD4, 0x73, /* 0x50-0x53 */ + 0xD4, 0x74, 0xD4, 0x75, 0xD4, 0x76, 0xD4, 0x77, /* 0x54-0x57 */ + 0xD4, 0x78, 0xD4, 0x79, 0xD4, 0x7A, 0xD4, 0x7B, /* 0x58-0x5B */ + 0xD4, 0x7C, 0xD4, 0x7D, 0xD4, 0x7E, 0xD4, 0x80, /* 0x5C-0x5F */ + 0xD4, 0x81, 0xD4, 0x82, 0xD4, 0x83, 0xD4, 0x84, /* 0x60-0x63 */ + 0xD4, 0x85, 0xD4, 0x86, 0xD4, 0x87, 0xD4, 0x88, /* 0x64-0x67 */ + 0xD4, 0x89, 0xD4, 0x8A, 0xD4, 0x8B, 0xD4, 0x8C, /* 0x68-0x6B */ + 0xD4, 0x8D, 0xD4, 0x8E, 0xD4, 0x8F, 0xD4, 0x90, /* 0x6C-0x6F */ + 0xD4, 0x91, 0xD4, 0x92, 0xD4, 0x93, 0xD4, 0x94, /* 0x70-0x73 */ + 0xD4, 0x95, 0xD4, 0x96, 0xD4, 0x97, 0xD4, 0x98, /* 0x74-0x77 */ + 0xD4, 0x99, 0xD5, 0xB2, 0xD4, 0x9A, 0xD4, 0x9B, /* 0x78-0x7B */ + 0xD4, 0x9C, 0xD4, 0x9D, 0xD4, 0x9E, 0xD4, 0x9F, /* 0x7C-0x7F */ + + 0xD4, 0xA0, 0xD5, 0x40, 0xD5, 0x41, 0xD5, 0x42, /* 0x80-0x83 */ + 0xD5, 0x43, 0xD5, 0x44, 0xD5, 0x45, 0xD5, 0x46, /* 0x84-0x87 */ + 0xD5, 0x47, 0xD3, 0xFE, 0xCC, 0xDC, 0xD5, 0x48, /* 0x88-0x8B */ + 0xD5, 0x49, 0xD5, 0x4A, 0xD5, 0x4B, 0xD5, 0x4C, /* 0x8C-0x8F */ + 0xD5, 0x4D, 0xD5, 0x4E, 0xD5, 0x4F, 0xCA, 0xC4, /* 0x90-0x93 */ + 0xD5, 0x50, 0xD5, 0x51, 0xD5, 0x52, 0xD5, 0x53, /* 0x94-0x97 */ + 0xD5, 0x54, 0xD5, 0x55, 0xD5, 0x56, 0xD5, 0x57, /* 0x98-0x9B */ + 0xD5, 0x58, 0xD5, 0x59, 0xD5, 0x5A, 0xD5, 0x5B, /* 0x9C-0x9F */ + 0xD5, 0x5C, 0xD5, 0x5D, 0xD5, 0x5E, 0xD5, 0x5F, /* 0xA0-0xA3 */ + 0xD5, 0x60, 0xD5, 0x61, 0xD5, 0x62, 0xD5, 0x63, /* 0xA4-0xA7 */ + 0xD5, 0x64, 0xD5, 0x65, 0xD5, 0x66, 0xD5, 0x67, /* 0xA8-0xAB */ + 0xD5, 0x68, 0xD5, 0x69, 0xD5, 0x6A, 0xD5, 0x6B, /* 0xAC-0xAF */ + 0xD5, 0x6C, 0xD5, 0x6D, 0xD5, 0x6E, 0xD5, 0x6F, /* 0xB0-0xB3 */ + 0xD5, 0x70, 0xD5, 0x71, 0xD5, 0x72, 0xD5, 0x73, /* 0xB4-0xB7 */ + 0xD5, 0x74, 0xD5, 0x75, 0xD5, 0x76, 0xD5, 0x77, /* 0xB8-0xBB */ + 0xD5, 0x78, 0xD5, 0x79, 0xD5, 0x7A, 0xD5, 0x7B, /* 0xBC-0xBF */ + 0xD5, 0x7C, 0xD5, 0x7D, 0xD5, 0x7E, 0xD5, 0x80, /* 0xC0-0xC3 */ + 0xD5, 0x81, 0xD5, 0x82, 0xD5, 0x83, 0xD5, 0x84, /* 0xC4-0xC7 */ + 0xD5, 0x85, 0xD5, 0x86, 0xD5, 0x87, 0xD5, 0x88, /* 0xC8-0xCB */ + 0xD5, 0x89, 0xD5, 0x8A, 0xD5, 0x8B, 0xD5, 0x8C, /* 0xCC-0xCF */ + 0xD5, 0x8D, 0xD5, 0x8E, 0xD5, 0x8F, 0xD5, 0x90, /* 0xD0-0xD3 */ + 0xD5, 0x91, 0xD5, 0x92, 0xD5, 0x93, 0xD5, 0x94, /* 0xD4-0xD7 */ + 0xD5, 0x95, 0xD5, 0x96, 0xD5, 0x97, 0xD5, 0x98, /* 0xD8-0xDB */ + 0xD5, 0x99, 0xD5, 0x9A, 0xD5, 0x9B, 0xD5, 0x9C, /* 0xDC-0xDF */ + 0xD5, 0x9D, 0xD5, 0x9E, 0xD5, 0x9F, 0xD5, 0xA0, /* 0xE0-0xE3 */ + 0xD6, 0x40, 0xD6, 0x41, 0xD6, 0x42, 0xD6, 0x43, /* 0xE4-0xE7 */ + 0xD6, 0x44, 0xD6, 0x45, 0xD6, 0x46, 0xD6, 0x47, /* 0xE8-0xEB */ + 0xD6, 0x48, 0xD6, 0x49, 0xD6, 0x4A, 0xD6, 0x4B, /* 0xEC-0xEF */ + 0xD6, 0x4C, 0xD6, 0x4D, 0xD6, 0x4E, 0xD6, 0x4F, /* 0xF0-0xF3 */ + 0xD6, 0x50, 0xD6, 0x51, 0xD6, 0x52, 0xD6, 0x53, /* 0xF4-0xF7 */ + 0xD6, 0x54, 0xD6, 0x55, 0xD6, 0x56, 0xD6, 0x57, /* 0xF8-0xFB */ + 0xD6, 0x58, 0xD6, 0x59, 0xD6, 0x5A, 0xD6, 0x5B, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_8B[512] = { + 0xD6, 0x5C, 0xD6, 0x5D, 0xD6, 0x5E, 0xD6, 0x5F, /* 0x00-0x03 */ + 0xD6, 0x60, 0xD6, 0x61, 0xD6, 0x62, 0xE5, 0xC0, /* 0x04-0x07 */ + 0xD6, 0x63, 0xD6, 0x64, 0xD6, 0x65, 0xD6, 0x66, /* 0x08-0x0B */ + 0xD6, 0x67, 0xD6, 0x68, 0xD6, 0x69, 0xD6, 0x6A, /* 0x0C-0x0F */ + 0xD6, 0x6B, 0xD6, 0x6C, 0xD6, 0x6D, 0xD6, 0x6E, /* 0x10-0x13 */ + 0xD6, 0x6F, 0xD6, 0x70, 0xD6, 0x71, 0xD6, 0x72, /* 0x14-0x17 */ + 0xD6, 0x73, 0xD6, 0x74, 0xD6, 0x75, 0xD6, 0x76, /* 0x18-0x1B */ + 0xD6, 0x77, 0xD6, 0x78, 0xD6, 0x79, 0xD6, 0x7A, /* 0x1C-0x1F */ + 0xD6, 0x7B, 0xD6, 0x7C, 0xD6, 0x7D, 0xD6, 0x7E, /* 0x20-0x23 */ + 0xD6, 0x80, 0xD6, 0x81, 0xF6, 0xA5, 0xD6, 0x82, /* 0x24-0x27 */ + 0xD6, 0x83, 0xD6, 0x84, 0xD6, 0x85, 0xD6, 0x86, /* 0x28-0x2B */ + 0xD6, 0x87, 0xD6, 0x88, 0xD6, 0x89, 0xD6, 0x8A, /* 0x2C-0x2F */ + 0xD6, 0x8B, 0xD6, 0x8C, 0xD6, 0x8D, 0xD6, 0x8E, /* 0x30-0x33 */ + 0xD6, 0x8F, 0xD6, 0x90, 0xD6, 0x91, 0xD6, 0x92, /* 0x34-0x37 */ + 0xD6, 0x93, 0xD6, 0x94, 0xD6, 0x95, 0xD6, 0x96, /* 0x38-0x3B */ + 0xD6, 0x97, 0xD6, 0x98, 0xD6, 0x99, 0xD6, 0x9A, /* 0x3C-0x3F */ + 0xD6, 0x9B, 0xD6, 0x9C, 0xD6, 0x9D, 0xD6, 0x9E, /* 0x40-0x43 */ + 0xD6, 0x9F, 0xD6, 0xA0, 0xD7, 0x40, 0xD7, 0x41, /* 0x44-0x47 */ + 0xD7, 0x42, 0xD7, 0x43, 0xD7, 0x44, 0xD7, 0x45, /* 0x48-0x4B */ + 0xD7, 0x46, 0xD7, 0x47, 0xD7, 0x48, 0xD7, 0x49, /* 0x4C-0x4F */ + 0xD7, 0x4A, 0xD7, 0x4B, 0xD7, 0x4C, 0xD7, 0x4D, /* 0x50-0x53 */ + 0xD7, 0x4E, 0xD7, 0x4F, 0xD7, 0x50, 0xD7, 0x51, /* 0x54-0x57 */ + 0xD7, 0x52, 0xD7, 0x53, 0xD7, 0x54, 0xD7, 0x55, /* 0x58-0x5B */ + 0xD7, 0x56, 0xD7, 0x57, 0xD7, 0x58, 0xD7, 0x59, /* 0x5C-0x5F */ + 0xD7, 0x5A, 0xD7, 0x5B, 0xD7, 0x5C, 0xD7, 0x5D, /* 0x60-0x63 */ + 0xD7, 0x5E, 0xD7, 0x5F, 0xBE, 0xAF, 0xD7, 0x60, /* 0x64-0x67 */ + 0xD7, 0x61, 0xD7, 0x62, 0xD7, 0x63, 0xD7, 0x64, /* 0x68-0x6B */ + 0xC6, 0xA9, 0xD7, 0x65, 0xD7, 0x66, 0xD7, 0x67, /* 0x6C-0x6F */ + 0xD7, 0x68, 0xD7, 0x69, 0xD7, 0x6A, 0xD7, 0x6B, /* 0x70-0x73 */ + 0xD7, 0x6C, 0xD7, 0x6D, 0xD7, 0x6E, 0xD7, 0x6F, /* 0x74-0x77 */ + 0xD7, 0x70, 0xD7, 0x71, 0xD7, 0x72, 0xD7, 0x73, /* 0x78-0x7B */ + 0xD7, 0x74, 0xD7, 0x75, 0xD7, 0x76, 0xD7, 0x77, /* 0x7C-0x7F */ + + 0xD7, 0x78, 0xD7, 0x79, 0xD7, 0x7A, 0xD7, 0x7B, /* 0x80-0x83 */ + 0xD7, 0x7C, 0xD7, 0x7D, 0xD7, 0x7E, 0xD7, 0x80, /* 0x84-0x87 */ + 0xD7, 0x81, 0xD7, 0x82, 0xD7, 0x83, 0xD7, 0x84, /* 0x88-0x8B */ + 0xD7, 0x85, 0xD7, 0x86, 0xD7, 0x87, 0xD7, 0x88, /* 0x8C-0x8F */ + 0xD7, 0x89, 0xD7, 0x8A, 0xD7, 0x8B, 0xD7, 0x8C, /* 0x90-0x93 */ + 0xD7, 0x8D, 0xD7, 0x8E, 0xD7, 0x8F, 0xD7, 0x90, /* 0x94-0x97 */ + 0xD7, 0x91, 0xD7, 0x92, 0xD7, 0x93, 0xD7, 0x94, /* 0x98-0x9B */ + 0xD7, 0x95, 0xD7, 0x96, 0xD7, 0x97, 0xD7, 0x98, /* 0x9C-0x9F */ + 0xDA, 0xA5, 0xBC, 0xC6, 0xB6, 0xA9, 0xB8, 0xBC, /* 0xA0-0xA3 */ + 0xC8, 0xCF, 0xBC, 0xA5, 0xDA, 0xA6, 0xDA, 0xA7, /* 0xA4-0xA7 */ + 0xCC, 0xD6, 0xC8, 0xC3, 0xDA, 0xA8, 0xC6, 0xFD, /* 0xA8-0xAB */ + 0xD7, 0x99, 0xD1, 0xB5, 0xD2, 0xE9, 0xD1, 0xB6, /* 0xAC-0xAF */ + 0xBC, 0xC7, 0xD7, 0x9A, 0xBD, 0xB2, 0xBB, 0xE4, /* 0xB0-0xB3 */ + 0xDA, 0xA9, 0xDA, 0xAA, 0xD1, 0xC8, 0xDA, 0xAB, /* 0xB4-0xB7 */ + 0xD0, 0xED, 0xB6, 0xEF, 0xC2, 0xDB, 0xD7, 0x9B, /* 0xB8-0xBB */ + 0xCB, 0xCF, 0xB7, 0xED, 0xC9, 0xE8, 0xB7, 0xC3, /* 0xBC-0xBF */ + 0xBE, 0xF7, 0xD6, 0xA4, 0xDA, 0xAC, 0xDA, 0xAD, /* 0xC0-0xC3 */ + 0xC6, 0xC0, 0xD7, 0xE7, 0xCA, 0xB6, 0xD7, 0x9C, /* 0xC4-0xC7 */ + 0xD5, 0xA9, 0xCB, 0xDF, 0xD5, 0xEF, 0xDA, 0xAE, /* 0xC8-0xCB */ + 0xD6, 0xDF, 0xB4, 0xCA, 0xDA, 0xB0, 0xDA, 0xAF, /* 0xCC-0xCF */ + 0xD7, 0x9D, 0xD2, 0xEB, 0xDA, 0xB1, 0xDA, 0xB2, /* 0xD0-0xD3 */ + 0xDA, 0xB3, 0xCA, 0xD4, 0xDA, 0xB4, 0xCA, 0xAB, /* 0xD4-0xD7 */ + 0xDA, 0xB5, 0xDA, 0xB6, 0xB3, 0xCF, 0xD6, 0xEF, /* 0xD8-0xDB */ + 0xDA, 0xB7, 0xBB, 0xB0, 0xB5, 0xAE, 0xDA, 0xB8, /* 0xDC-0xDF */ + 0xDA, 0xB9, 0xB9, 0xEE, 0xD1, 0xAF, 0xD2, 0xE8, /* 0xE0-0xE3 */ + 0xDA, 0xBA, 0xB8, 0xC3, 0xCF, 0xEA, 0xB2, 0xEF, /* 0xE4-0xE7 */ + 0xDA, 0xBB, 0xDA, 0xBC, 0xD7, 0x9E, 0xBD, 0xEB, /* 0xE8-0xEB */ + 0xCE, 0xDC, 0xD3, 0xEF, 0xDA, 0xBD, 0xCE, 0xF3, /* 0xEC-0xEF */ + 0xDA, 0xBE, 0xD3, 0xD5, 0xBB, 0xE5, 0xDA, 0xBF, /* 0xF0-0xF3 */ + 0xCB, 0xB5, 0xCB, 0xD0, 0xDA, 0xC0, 0xC7, 0xEB, /* 0xF4-0xF7 */ + 0xD6, 0xEE, 0xDA, 0xC1, 0xC5, 0xB5, 0xB6, 0xC1, /* 0xF8-0xFB */ + 0xDA, 0xC2, 0xB7, 0xCC, 0xBF, 0xCE, 0xDA, 0xC3, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_8C[512] = { + 0xDA, 0xC4, 0xCB, 0xAD, 0xDA, 0xC5, 0xB5, 0xF7, /* 0x00-0x03 */ + 0xDA, 0xC6, 0xC1, 0xC2, 0xD7, 0xBB, 0xDA, 0xC7, /* 0x04-0x07 */ + 0xCC, 0xB8, 0xD7, 0x9F, 0xD2, 0xEA, 0xC4, 0xB1, /* 0x08-0x0B */ + 0xDA, 0xC8, 0xB5, 0xFD, 0xBB, 0xD1, 0xDA, 0xC9, /* 0x0C-0x0F */ + 0xD0, 0xB3, 0xDA, 0xCA, 0xDA, 0xCB, 0xCE, 0xBD, /* 0x10-0x13 */ + 0xDA, 0xCC, 0xDA, 0xCD, 0xDA, 0xCE, 0xB2, 0xF7, /* 0x14-0x17 */ + 0xDA, 0xD1, 0xDA, 0xCF, 0xD1, 0xE8, 0xDA, 0xD0, /* 0x18-0x1B */ + 0xC3, 0xD5, 0xDA, 0xD2, 0xD7, 0xA0, 0xDA, 0xD3, /* 0x1C-0x1F */ + 0xDA, 0xD4, 0xDA, 0xD5, 0xD0, 0xBB, 0xD2, 0xA5, /* 0x20-0x23 */ + 0xB0, 0xF9, 0xDA, 0xD6, 0xC7, 0xAB, 0xDA, 0xD7, /* 0x24-0x27 */ + 0xBD, 0xF7, 0xC3, 0xA1, 0xDA, 0xD8, 0xDA, 0xD9, /* 0x28-0x2B */ + 0xC3, 0xFD, 0xCC, 0xB7, 0xDA, 0xDA, 0xDA, 0xDB, /* 0x2C-0x2F */ + 0xC0, 0xBE, 0xC6, 0xD7, 0xDA, 0xDC, 0xDA, 0xDD, /* 0x30-0x33 */ + 0xC7, 0xB4, 0xDA, 0xDE, 0xDA, 0xDF, 0xB9, 0xC8, /* 0x34-0x37 */ + 0xD8, 0x40, 0xD8, 0x41, 0xD8, 0x42, 0xD8, 0x43, /* 0x38-0x3B */ + 0xD8, 0x44, 0xD8, 0x45, 0xD8, 0x46, 0xD8, 0x47, /* 0x3C-0x3F */ + 0xD8, 0x48, 0xBB, 0xED, 0xD8, 0x49, 0xD8, 0x4A, /* 0x40-0x43 */ + 0xD8, 0x4B, 0xD8, 0x4C, 0xB6, 0xB9, 0xF4, 0xF8, /* 0x44-0x47 */ + 0xD8, 0x4D, 0xF4, 0xF9, 0xD8, 0x4E, 0xD8, 0x4F, /* 0x48-0x4B */ + 0xCD, 0xE3, 0xD8, 0x50, 0xD8, 0x51, 0xD8, 0x52, /* 0x4C-0x4F */ + 0xD8, 0x53, 0xD8, 0x54, 0xD8, 0x55, 0xD8, 0x56, /* 0x50-0x53 */ + 0xD8, 0x57, 0xF5, 0xB9, 0xD8, 0x58, 0xD8, 0x59, /* 0x54-0x57 */ + 0xD8, 0x5A, 0xD8, 0x5B, 0xEB, 0xE0, 0xD8, 0x5C, /* 0x58-0x5B */ + 0xD8, 0x5D, 0xD8, 0x5E, 0xD8, 0x5F, 0xD8, 0x60, /* 0x5C-0x5F */ + 0xD8, 0x61, 0xCF, 0xF3, 0xBB, 0xBF, 0xD8, 0x62, /* 0x60-0x63 */ + 0xD8, 0x63, 0xD8, 0x64, 0xD8, 0x65, 0xD8, 0x66, /* 0x64-0x67 */ + 0xD8, 0x67, 0xD8, 0x68, 0xBA, 0xC0, 0xD4, 0xA5, /* 0x68-0x6B */ + 0xD8, 0x69, 0xD8, 0x6A, 0xD8, 0x6B, 0xD8, 0x6C, /* 0x6C-0x6F */ + 0xD8, 0x6D, 0xD8, 0x6E, 0xD8, 0x6F, 0xE1, 0xD9, /* 0x70-0x73 */ + 0xD8, 0x70, 0xD8, 0x71, 0xD8, 0x72, 0xD8, 0x73, /* 0x74-0x77 */ + 0xF5, 0xF4, 0xB1, 0xAA, 0xB2, 0xF2, 0xD8, 0x74, /* 0x78-0x7B */ + 0xD8, 0x75, 0xD8, 0x76, 0xD8, 0x77, 0xD8, 0x78, /* 0x7C-0x7F */ + + 0xD8, 0x79, 0xD8, 0x7A, 0xF5, 0xF5, 0xD8, 0x7B, /* 0x80-0x83 */ + 0xD8, 0x7C, 0xF5, 0xF7, 0xD8, 0x7D, 0xD8, 0x7E, /* 0x84-0x87 */ + 0xD8, 0x80, 0xBA, 0xD1, 0xF5, 0xF6, 0xD8, 0x81, /* 0x88-0x8B */ + 0xC3, 0xB2, 0xD8, 0x82, 0xD8, 0x83, 0xD8, 0x84, /* 0x8C-0x8F */ + 0xD8, 0x85, 0xD8, 0x86, 0xD8, 0x87, 0xD8, 0x88, /* 0x90-0x93 */ + 0xF5, 0xF9, 0xD8, 0x89, 0xD8, 0x8A, 0xD8, 0x8B, /* 0x94-0x97 */ + 0xF5, 0xF8, 0xD8, 0x8C, 0xD8, 0x8D, 0xD8, 0x8E, /* 0x98-0x9B */ + 0xD8, 0x8F, 0xD8, 0x90, 0xD8, 0x91, 0xD8, 0x92, /* 0x9C-0x9F */ + 0xD8, 0x93, 0xD8, 0x94, 0xD8, 0x95, 0xD8, 0x96, /* 0xA0-0xA3 */ + 0xD8, 0x97, 0xD8, 0x98, 0xD8, 0x99, 0xD8, 0x9A, /* 0xA4-0xA7 */ + 0xD8, 0x9B, 0xD8, 0x9C, 0xD8, 0x9D, 0xD8, 0x9E, /* 0xA8-0xAB */ + 0xD8, 0x9F, 0xD8, 0xA0, 0xD9, 0x40, 0xD9, 0x41, /* 0xAC-0xAF */ + 0xD9, 0x42, 0xD9, 0x43, 0xD9, 0x44, 0xD9, 0x45, /* 0xB0-0xB3 */ + 0xD9, 0x46, 0xD9, 0x47, 0xD9, 0x48, 0xD9, 0x49, /* 0xB4-0xB7 */ + 0xD9, 0x4A, 0xD9, 0x4B, 0xD9, 0x4C, 0xD9, 0x4D, /* 0xB8-0xBB */ + 0xD9, 0x4E, 0xD9, 0x4F, 0xD9, 0x50, 0xD9, 0x51, /* 0xBC-0xBF */ + 0xD9, 0x52, 0xD9, 0x53, 0xD9, 0x54, 0xD9, 0x55, /* 0xC0-0xC3 */ + 0xD9, 0x56, 0xD9, 0x57, 0xD9, 0x58, 0xD9, 0x59, /* 0xC4-0xC7 */ + 0xD9, 0x5A, 0xD9, 0x5B, 0xD9, 0x5C, 0xD9, 0x5D, /* 0xC8-0xCB */ + 0xD9, 0x5E, 0xD9, 0x5F, 0xD9, 0x60, 0xD9, 0x61, /* 0xCC-0xCF */ + 0xD9, 0x62, 0xD9, 0x63, 0xD9, 0x64, 0xD9, 0x65, /* 0xD0-0xD3 */ + 0xD9, 0x66, 0xD9, 0x67, 0xD9, 0x68, 0xD9, 0x69, /* 0xD4-0xD7 */ + 0xD9, 0x6A, 0xD9, 0x6B, 0xD9, 0x6C, 0xD9, 0x6D, /* 0xD8-0xDB */ + 0xD9, 0x6E, 0xD9, 0x6F, 0xD9, 0x70, 0xD9, 0x71, /* 0xDC-0xDF */ + 0xD9, 0x72, 0xD9, 0x73, 0xD9, 0x74, 0xD9, 0x75, /* 0xE0-0xE3 */ + 0xD9, 0x76, 0xD9, 0x77, 0xD9, 0x78, 0xD9, 0x79, /* 0xE4-0xE7 */ + 0xD9, 0x7A, 0xD9, 0x7B, 0xD9, 0x7C, 0xD9, 0x7D, /* 0xE8-0xEB */ + 0xD9, 0x7E, 0xD9, 0x80, 0xD9, 0x81, 0xD9, 0x82, /* 0xEC-0xEF */ + 0xD9, 0x83, 0xD9, 0x84, 0xD9, 0x85, 0xD9, 0x86, /* 0xF0-0xF3 */ + 0xD9, 0x87, 0xD9, 0x88, 0xD9, 0x89, 0xD9, 0x8A, /* 0xF4-0xF7 */ + 0xD9, 0x8B, 0xD9, 0x8C, 0xD9, 0x8D, 0xD9, 0x8E, /* 0xF8-0xFB */ + 0xD9, 0x8F, 0xD9, 0x90, 0xD9, 0x91, 0xD9, 0x92, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_8D[512] = { + 0xD9, 0x93, 0xD9, 0x94, 0xD9, 0x95, 0xD9, 0x96, /* 0x00-0x03 */ + 0xD9, 0x97, 0xD9, 0x98, 0xD9, 0x99, 0xD9, 0x9A, /* 0x04-0x07 */ + 0xD9, 0x9B, 0xD9, 0x9C, 0xD9, 0x9D, 0xD9, 0x9E, /* 0x08-0x0B */ + 0xD9, 0x9F, 0xD9, 0xA0, 0xDA, 0x40, 0xDA, 0x41, /* 0x0C-0x0F */ + 0xDA, 0x42, 0xDA, 0x43, 0xDA, 0x44, 0xDA, 0x45, /* 0x10-0x13 */ + 0xDA, 0x46, 0xDA, 0x47, 0xDA, 0x48, 0xDA, 0x49, /* 0x14-0x17 */ + 0xDA, 0x4A, 0xDA, 0x4B, 0xDA, 0x4C, 0xDA, 0x4D, /* 0x18-0x1B */ + 0xDA, 0x4E, 0xB1, 0xB4, 0xD5, 0xEA, 0xB8, 0xBA, /* 0x1C-0x1F */ + 0xDA, 0x4F, 0xB9, 0xB1, 0xB2, 0xC6, 0xD4, 0xF0, /* 0x20-0x23 */ + 0xCF, 0xCD, 0xB0, 0xDC, 0xD5, 0xCB, 0xBB, 0xF5, /* 0x24-0x27 */ + 0xD6, 0xCA, 0xB7, 0xB7, 0xCC, 0xB0, 0xC6, 0xB6, /* 0x28-0x2B */ + 0xB1, 0xE1, 0xB9, 0xBA, 0xD6, 0xFC, 0xB9, 0xE1, /* 0x2C-0x2F */ + 0xB7, 0xA1, 0xBC, 0xFA, 0xEA, 0xDA, 0xEA, 0xDB, /* 0x30-0x33 */ + 0xCC, 0xF9, 0xB9, 0xF3, 0xEA, 0xDC, 0xB4, 0xFB, /* 0x34-0x37 */ + 0xC3, 0xB3, 0xB7, 0xD1, 0xBA, 0xD8, 0xEA, 0xDD, /* 0x38-0x3B */ + 0xD4, 0xF4, 0xEA, 0xDE, 0xBC, 0xD6, 0xBB, 0xDF, /* 0x3C-0x3F */ + 0xEA, 0xDF, 0xC1, 0xDE, 0xC2, 0xB8, 0xD4, 0xDF, /* 0x40-0x43 */ + 0xD7, 0xCA, 0xEA, 0xE0, 0xEA, 0xE1, 0xEA, 0xE4, /* 0x44-0x47 */ + 0xEA, 0xE2, 0xEA, 0xE3, 0xC9, 0xDE, 0xB8, 0xB3, /* 0x48-0x4B */ + 0xB6, 0xC4, 0xEA, 0xE5, 0xCA, 0xEA, 0xC9, 0xCD, /* 0x4C-0x4F */ + 0xB4, 0xCD, 0xDA, 0x50, 0xDA, 0x51, 0xE2, 0xD9, /* 0x50-0x53 */ + 0xC5, 0xE2, 0xEA, 0xE6, 0xC0, 0xB5, 0xDA, 0x52, /* 0x54-0x57 */ + 0xD7, 0xB8, 0xEA, 0xE7, 0xD7, 0xAC, 0xC8, 0xFC, /* 0x58-0x5B */ + 0xD8, 0xD3, 0xD8, 0xCD, 0xD4, 0xDE, 0xDA, 0x53, /* 0x5C-0x5F */ + 0xD4, 0xF9, 0xC9, 0xC4, 0xD3, 0xAE, 0xB8, 0xD3, /* 0x60-0x63 */ + 0xB3, 0xE0, 0xDA, 0x54, 0xC9, 0xE2, 0xF4, 0xF6, /* 0x64-0x67 */ + 0xDA, 0x55, 0xDA, 0x56, 0xDA, 0x57, 0xBA, 0xD5, /* 0x68-0x6B */ + 0xDA, 0x58, 0xF4, 0xF7, 0xDA, 0x59, 0xDA, 0x5A, /* 0x6C-0x6F */ + 0xD7, 0xDF, 0xDA, 0x5B, 0xDA, 0x5C, 0xF4, 0xF1, /* 0x70-0x73 */ + 0xB8, 0xB0, 0xD5, 0xD4, 0xB8, 0xCF, 0xC6, 0xF0, /* 0x74-0x77 */ + 0xDA, 0x5D, 0xDA, 0x5E, 0xDA, 0x5F, 0xDA, 0x60, /* 0x78-0x7B */ + 0xDA, 0x61, 0xDA, 0x62, 0xDA, 0x63, 0xDA, 0x64, /* 0x7C-0x7F */ + + 0xDA, 0x65, 0xB3, 0xC3, 0xDA, 0x66, 0xDA, 0x67, /* 0x80-0x83 */ + 0xF4, 0xF2, 0xB3, 0xAC, 0xDA, 0x68, 0xDA, 0x69, /* 0x84-0x87 */ + 0xDA, 0x6A, 0xDA, 0x6B, 0xD4, 0xBD, 0xC7, 0xF7, /* 0x88-0x8B */ + 0xDA, 0x6C, 0xDA, 0x6D, 0xDA, 0x6E, 0xDA, 0x6F, /* 0x8C-0x8F */ + 0xDA, 0x70, 0xF4, 0xF4, 0xDA, 0x71, 0xDA, 0x72, /* 0x90-0x93 */ + 0xF4, 0xF3, 0xDA, 0x73, 0xDA, 0x74, 0xDA, 0x75, /* 0x94-0x97 */ + 0xDA, 0x76, 0xDA, 0x77, 0xDA, 0x78, 0xDA, 0x79, /* 0x98-0x9B */ + 0xDA, 0x7A, 0xDA, 0x7B, 0xDA, 0x7C, 0xCC, 0xCB, /* 0x9C-0x9F */ + 0xDA, 0x7D, 0xDA, 0x7E, 0xDA, 0x80, 0xC8, 0xA4, /* 0xA0-0xA3 */ + 0xDA, 0x81, 0xDA, 0x82, 0xDA, 0x83, 0xDA, 0x84, /* 0xA4-0xA7 */ + 0xDA, 0x85, 0xDA, 0x86, 0xDA, 0x87, 0xDA, 0x88, /* 0xA8-0xAB */ + 0xDA, 0x89, 0xDA, 0x8A, 0xDA, 0x8B, 0xDA, 0x8C, /* 0xAC-0xAF */ + 0xDA, 0x8D, 0xF4, 0xF5, 0xDA, 0x8E, 0xD7, 0xE3, /* 0xB0-0xB3 */ + 0xC5, 0xBF, 0xF5, 0xC0, 0xDA, 0x8F, 0xDA, 0x90, /* 0xB4-0xB7 */ + 0xF5, 0xBB, 0xDA, 0x91, 0xF5, 0xC3, 0xDA, 0x92, /* 0xB8-0xBB */ + 0xF5, 0xC2, 0xDA, 0x93, 0xD6, 0xBA, 0xF5, 0xC1, /* 0xBC-0xBF */ + 0xDA, 0x94, 0xDA, 0x95, 0xDA, 0x96, 0xD4, 0xBE, /* 0xC0-0xC3 */ + 0xF5, 0xC4, 0xDA, 0x97, 0xF5, 0xCC, 0xDA, 0x98, /* 0xC4-0xC7 */ + 0xDA, 0x99, 0xDA, 0x9A, 0xDA, 0x9B, 0xB0, 0xCF, /* 0xC8-0xCB */ + 0xB5, 0xF8, 0xDA, 0x9C, 0xF5, 0xC9, 0xF5, 0xCA, /* 0xCC-0xCF */ + 0xDA, 0x9D, 0xC5, 0xDC, 0xDA, 0x9E, 0xDA, 0x9F, /* 0xD0-0xD3 */ + 0xDA, 0xA0, 0xDB, 0x40, 0xF5, 0xC5, 0xF5, 0xC6, /* 0xD4-0xD7 */ + 0xDB, 0x41, 0xDB, 0x42, 0xF5, 0xC7, 0xF5, 0xCB, /* 0xD8-0xDB */ + 0xDB, 0x43, 0xBE, 0xE0, 0xF5, 0xC8, 0xB8, 0xFA, /* 0xDC-0xDF */ + 0xDB, 0x44, 0xDB, 0x45, 0xDB, 0x46, 0xF5, 0xD0, /* 0xE0-0xE3 */ + 0xF5, 0xD3, 0xDB, 0x47, 0xDB, 0x48, 0xDB, 0x49, /* 0xE4-0xE7 */ + 0xBF, 0xE7, 0xDB, 0x4A, 0xB9, 0xF2, 0xF5, 0xBC, /* 0xE8-0xEB */ + 0xF5, 0xCD, 0xDB, 0x4B, 0xDB, 0x4C, 0xC2, 0xB7, /* 0xEC-0xEF */ + 0xDB, 0x4D, 0xDB, 0x4E, 0xDB, 0x4F, 0xCC, 0xF8, /* 0xF0-0xF3 */ + 0xDB, 0x50, 0xBC, 0xF9, 0xDB, 0x51, 0xF5, 0xCE, /* 0xF4-0xF7 */ + 0xF5, 0xCF, 0xF5, 0xD1, 0xB6, 0xE5, 0xF5, 0xD2, /* 0xF8-0xFB */ + 0xDB, 0x52, 0xF5, 0xD5, 0xDB, 0x53, 0xDB, 0x54, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_8E[512] = { + 0xDB, 0x55, 0xDB, 0x56, 0xDB, 0x57, 0xDB, 0x58, /* 0x00-0x03 */ + 0xDB, 0x59, 0xF5, 0xBD, 0xDB, 0x5A, 0xDB, 0x5B, /* 0x04-0x07 */ + 0xDB, 0x5C, 0xF5, 0xD4, 0xD3, 0xBB, 0xDB, 0x5D, /* 0x08-0x0B */ + 0xB3, 0xEC, 0xDB, 0x5E, 0xDB, 0x5F, 0xCC, 0xA4, /* 0x0C-0x0F */ + 0xDB, 0x60, 0xDB, 0x61, 0xDB, 0x62, 0xDB, 0x63, /* 0x10-0x13 */ + 0xF5, 0xD6, 0xDB, 0x64, 0xDB, 0x65, 0xDB, 0x66, /* 0x14-0x17 */ + 0xDB, 0x67, 0xDB, 0x68, 0xDB, 0x69, 0xDB, 0x6A, /* 0x18-0x1B */ + 0xDB, 0x6B, 0xF5, 0xD7, 0xBE, 0xE1, 0xF5, 0xD8, /* 0x1C-0x1F */ + 0xDB, 0x6C, 0xDB, 0x6D, 0xCC, 0xDF, 0xF5, 0xDB, /* 0x20-0x23 */ + 0xDB, 0x6E, 0xDB, 0x6F, 0xDB, 0x70, 0xDB, 0x71, /* 0x24-0x27 */ + 0xDB, 0x72, 0xB2, 0xC8, 0xD7, 0xD9, 0xDB, 0x73, /* 0x28-0x2B */ + 0xF5, 0xD9, 0xDB, 0x74, 0xF5, 0xDA, 0xF5, 0xDC, /* 0x2C-0x2F */ + 0xDB, 0x75, 0xF5, 0xE2, 0xDB, 0x76, 0xDB, 0x77, /* 0x30-0x33 */ + 0xDB, 0x78, 0xF5, 0xE0, 0xDB, 0x79, 0xDB, 0x7A, /* 0x34-0x37 */ + 0xDB, 0x7B, 0xF5, 0xDF, 0xF5, 0xDD, 0xDB, 0x7C, /* 0x38-0x3B */ + 0xDB, 0x7D, 0xF5, 0xE1, 0xDB, 0x7E, 0xDB, 0x80, /* 0x3C-0x3F */ + 0xF5, 0xDE, 0xF5, 0xE4, 0xF5, 0xE5, 0xDB, 0x81, /* 0x40-0x43 */ + 0xCC, 0xE3, 0xDB, 0x82, 0xDB, 0x83, 0xE5, 0xBF, /* 0x44-0x47 */ + 0xB5, 0xB8, 0xF5, 0xE3, 0xF5, 0xE8, 0xCC, 0xA3, /* 0x48-0x4B */ + 0xDB, 0x84, 0xDB, 0x85, 0xDB, 0x86, 0xDB, 0x87, /* 0x4C-0x4F */ + 0xDB, 0x88, 0xF5, 0xE6, 0xF5, 0xE7, 0xDB, 0x89, /* 0x50-0x53 */ + 0xDB, 0x8A, 0xDB, 0x8B, 0xDB, 0x8C, 0xDB, 0x8D, /* 0x54-0x57 */ + 0xDB, 0x8E, 0xF5, 0xBE, 0xDB, 0x8F, 0xDB, 0x90, /* 0x58-0x5B */ + 0xDB, 0x91, 0xDB, 0x92, 0xDB, 0x93, 0xDB, 0x94, /* 0x5C-0x5F */ + 0xDB, 0x95, 0xDB, 0x96, 0xDB, 0x97, 0xDB, 0x98, /* 0x60-0x63 */ + 0xDB, 0x99, 0xDB, 0x9A, 0xB1, 0xC4, 0xDB, 0x9B, /* 0x64-0x67 */ + 0xDB, 0x9C, 0xF5, 0xBF, 0xDB, 0x9D, 0xDB, 0x9E, /* 0x68-0x6B */ + 0xB5, 0xC5, 0xB2, 0xE4, 0xDB, 0x9F, 0xF5, 0xEC, /* 0x6C-0x6F */ + 0xF5, 0xE9, 0xDB, 0xA0, 0xB6, 0xD7, 0xDC, 0x40, /* 0x70-0x73 */ + 0xF5, 0xED, 0xDC, 0x41, 0xF5, 0xEA, 0xDC, 0x42, /* 0x74-0x77 */ + 0xDC, 0x43, 0xDC, 0x44, 0xDC, 0x45, 0xDC, 0x46, /* 0x78-0x7B */ + 0xF5, 0xEB, 0xDC, 0x47, 0xDC, 0x48, 0xB4, 0xDA, /* 0x7C-0x7F */ + + 0xDC, 0x49, 0xD4, 0xEA, 0xDC, 0x4A, 0xDC, 0x4B, /* 0x80-0x83 */ + 0xDC, 0x4C, 0xF5, 0xEE, 0xDC, 0x4D, 0xB3, 0xF9, /* 0x84-0x87 */ + 0xDC, 0x4E, 0xDC, 0x4F, 0xDC, 0x50, 0xDC, 0x51, /* 0x88-0x8B */ + 0xDC, 0x52, 0xDC, 0x53, 0xDC, 0x54, 0xF5, 0xEF, /* 0x8C-0x8F */ + 0xF5, 0xF1, 0xDC, 0x55, 0xDC, 0x56, 0xDC, 0x57, /* 0x90-0x93 */ + 0xF5, 0xF0, 0xDC, 0x58, 0xDC, 0x59, 0xDC, 0x5A, /* 0x94-0x97 */ + 0xDC, 0x5B, 0xDC, 0x5C, 0xDC, 0x5D, 0xDC, 0x5E, /* 0x98-0x9B */ + 0xF5, 0xF2, 0xDC, 0x5F, 0xF5, 0xF3, 0xDC, 0x60, /* 0x9C-0x9F */ + 0xDC, 0x61, 0xDC, 0x62, 0xDC, 0x63, 0xDC, 0x64, /* 0xA0-0xA3 */ + 0xDC, 0x65, 0xDC, 0x66, 0xDC, 0x67, 0xDC, 0x68, /* 0xA4-0xA7 */ + 0xDC, 0x69, 0xDC, 0x6A, 0xDC, 0x6B, 0xC9, 0xED, /* 0xA8-0xAB */ + 0xB9, 0xAA, 0xDC, 0x6C, 0xDC, 0x6D, 0xC7, 0xFB, /* 0xAC-0xAF */ + 0xDC, 0x6E, 0xDC, 0x6F, 0xB6, 0xE3, 0xDC, 0x70, /* 0xB0-0xB3 */ + 0xDC, 0x71, 0xDC, 0x72, 0xDC, 0x73, 0xDC, 0x74, /* 0xB4-0xB7 */ + 0xDC, 0x75, 0xDC, 0x76, 0xCC, 0xC9, 0xDC, 0x77, /* 0xB8-0xBB */ + 0xDC, 0x78, 0xDC, 0x79, 0xDC, 0x7A, 0xDC, 0x7B, /* 0xBC-0xBF */ + 0xDC, 0x7C, 0xDC, 0x7D, 0xDC, 0x7E, 0xDC, 0x80, /* 0xC0-0xC3 */ + 0xDC, 0x81, 0xDC, 0x82, 0xDC, 0x83, 0xDC, 0x84, /* 0xC4-0xC7 */ + 0xDC, 0x85, 0xDC, 0x86, 0xDC, 0x87, 0xDC, 0x88, /* 0xC8-0xCB */ + 0xDC, 0x89, 0xDC, 0x8A, 0xEA, 0xA6, 0xDC, 0x8B, /* 0xCC-0xCF */ + 0xDC, 0x8C, 0xDC, 0x8D, 0xDC, 0x8E, 0xDC, 0x8F, /* 0xD0-0xD3 */ + 0xDC, 0x90, 0xDC, 0x91, 0xDC, 0x92, 0xDC, 0x93, /* 0xD4-0xD7 */ + 0xDC, 0x94, 0xDC, 0x95, 0xDC, 0x96, 0xDC, 0x97, /* 0xD8-0xDB */ + 0xDC, 0x98, 0xDC, 0x99, 0xDC, 0x9A, 0xDC, 0x9B, /* 0xDC-0xDF */ + 0xDC, 0x9C, 0xDC, 0x9D, 0xDC, 0x9E, 0xDC, 0x9F, /* 0xE0-0xE3 */ + 0xDC, 0xA0, 0xDD, 0x40, 0xDD, 0x41, 0xDD, 0x42, /* 0xE4-0xE7 */ + 0xDD, 0x43, 0xDD, 0x44, 0xDD, 0x45, 0xDD, 0x46, /* 0xE8-0xEB */ + 0xDD, 0x47, 0xDD, 0x48, 0xDD, 0x49, 0xDD, 0x4A, /* 0xEC-0xEF */ + 0xDD, 0x4B, 0xDD, 0x4C, 0xDD, 0x4D, 0xDD, 0x4E, /* 0xF0-0xF3 */ + 0xDD, 0x4F, 0xDD, 0x50, 0xDD, 0x51, 0xDD, 0x52, /* 0xF4-0xF7 */ + 0xDD, 0x53, 0xDD, 0x54, 0xDD, 0x55, 0xDD, 0x56, /* 0xF8-0xFB */ + 0xDD, 0x57, 0xDD, 0x58, 0xDD, 0x59, 0xDD, 0x5A, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_8F[512] = { + 0xDD, 0x5B, 0xDD, 0x5C, 0xDD, 0x5D, 0xDD, 0x5E, /* 0x00-0x03 */ + 0xDD, 0x5F, 0xDD, 0x60, 0xDD, 0x61, 0xDD, 0x62, /* 0x04-0x07 */ + 0xDD, 0x63, 0xDD, 0x64, 0xDD, 0x65, 0xDD, 0x66, /* 0x08-0x0B */ + 0xDD, 0x67, 0xDD, 0x68, 0xDD, 0x69, 0xDD, 0x6A, /* 0x0C-0x0F */ + 0xDD, 0x6B, 0xDD, 0x6C, 0xDD, 0x6D, 0xDD, 0x6E, /* 0x10-0x13 */ + 0xDD, 0x6F, 0xDD, 0x70, 0xDD, 0x71, 0xDD, 0x72, /* 0x14-0x17 */ + 0xDD, 0x73, 0xDD, 0x74, 0xDD, 0x75, 0xDD, 0x76, /* 0x18-0x1B */ + 0xDD, 0x77, 0xDD, 0x78, 0xDD, 0x79, 0xDD, 0x7A, /* 0x1C-0x1F */ + 0xDD, 0x7B, 0xDD, 0x7C, 0xDD, 0x7D, 0xDD, 0x7E, /* 0x20-0x23 */ + 0xDD, 0x80, 0xDD, 0x81, 0xDD, 0x82, 0xDD, 0x83, /* 0x24-0x27 */ + 0xDD, 0x84, 0xDD, 0x85, 0xDD, 0x86, 0xDD, 0x87, /* 0x28-0x2B */ + 0xDD, 0x88, 0xDD, 0x89, 0xDD, 0x8A, 0xDD, 0x8B, /* 0x2C-0x2F */ + 0xDD, 0x8C, 0xDD, 0x8D, 0xDD, 0x8E, 0xDD, 0x8F, /* 0x30-0x33 */ + 0xDD, 0x90, 0xDD, 0x91, 0xDD, 0x92, 0xDD, 0x93, /* 0x34-0x37 */ + 0xDD, 0x94, 0xDD, 0x95, 0xDD, 0x96, 0xDD, 0x97, /* 0x38-0x3B */ + 0xDD, 0x98, 0xDD, 0x99, 0xDD, 0x9A, 0xDD, 0x9B, /* 0x3C-0x3F */ + 0xDD, 0x9C, 0xDD, 0x9D, 0xDD, 0x9E, 0xDD, 0x9F, /* 0x40-0x43 */ + 0xDD, 0xA0, 0xDE, 0x40, 0xDE, 0x41, 0xDE, 0x42, /* 0x44-0x47 */ + 0xDE, 0x43, 0xDE, 0x44, 0xDE, 0x45, 0xDE, 0x46, /* 0x48-0x4B */ + 0xDE, 0x47, 0xDE, 0x48, 0xDE, 0x49, 0xDE, 0x4A, /* 0x4C-0x4F */ + 0xDE, 0x4B, 0xDE, 0x4C, 0xDE, 0x4D, 0xDE, 0x4E, /* 0x50-0x53 */ + 0xDE, 0x4F, 0xDE, 0x50, 0xDE, 0x51, 0xDE, 0x52, /* 0x54-0x57 */ + 0xDE, 0x53, 0xDE, 0x54, 0xDE, 0x55, 0xDE, 0x56, /* 0x58-0x5B */ + 0xDE, 0x57, 0xDE, 0x58, 0xDE, 0x59, 0xDE, 0x5A, /* 0x5C-0x5F */ + 0xDE, 0x5B, 0xDE, 0x5C, 0xDE, 0x5D, 0xDE, 0x5E, /* 0x60-0x63 */ + 0xDE, 0x5F, 0xDE, 0x60, 0xB3, 0xB5, 0xD4, 0xFE, /* 0x64-0x67 */ + 0xB9, 0xEC, 0xD0, 0xF9, 0xDE, 0x61, 0xE9, 0xED, /* 0x68-0x6B */ + 0xD7, 0xAA, 0xE9, 0xEE, 0xC2, 0xD6, 0xC8, 0xED, /* 0x6C-0x6F */ + 0xBA, 0xE4, 0xE9, 0xEF, 0xE9, 0xF0, 0xE9, 0xF1, /* 0x70-0x73 */ + 0xD6, 0xE1, 0xE9, 0xF2, 0xE9, 0xF3, 0xE9, 0xF5, /* 0x74-0x77 */ + 0xE9, 0xF4, 0xE9, 0xF6, 0xE9, 0xF7, 0xC7, 0xE1, /* 0x78-0x7B */ + 0xE9, 0xF8, 0xD4, 0xD8, 0xE9, 0xF9, 0xBD, 0xCE, /* 0x7C-0x7F */ + + 0xDE, 0x62, 0xE9, 0xFA, 0xE9, 0xFB, 0xBD, 0xCF, /* 0x80-0x83 */ + 0xE9, 0xFC, 0xB8, 0xA8, 0xC1, 0xBE, 0xE9, 0xFD, /* 0x84-0x87 */ + 0xB1, 0xB2, 0xBB, 0xD4, 0xB9, 0xF5, 0xE9, 0xFE, /* 0x88-0x8B */ + 0xDE, 0x63, 0xEA, 0xA1, 0xEA, 0xA2, 0xEA, 0xA3, /* 0x8C-0x8F */ + 0xB7, 0xF8, 0xBC, 0xAD, 0xDE, 0x64, 0xCA, 0xE4, /* 0x90-0x93 */ + 0xE0, 0xCE, 0xD4, 0xAF, 0xCF, 0xBD, 0xD5, 0xB7, /* 0x94-0x97 */ + 0xEA, 0xA4, 0xD5, 0xDE, 0xEA, 0xA5, 0xD0, 0xC1, /* 0x98-0x9B */ + 0xB9, 0xBC, 0xDE, 0x65, 0xB4, 0xC7, 0xB1, 0xD9, /* 0x9C-0x9F */ + 0xDE, 0x66, 0xDE, 0x67, 0xDE, 0x68, 0xC0, 0xB1, /* 0xA0-0xA3 */ + 0xDE, 0x69, 0xDE, 0x6A, 0xDE, 0x6B, 0xDE, 0x6C, /* 0xA4-0xA7 */ + 0xB1, 0xE6, 0xB1, 0xE7, 0xDE, 0x6D, 0xB1, 0xE8, /* 0xA8-0xAB */ + 0xDE, 0x6E, 0xDE, 0x6F, 0xDE, 0x70, 0xDE, 0x71, /* 0xAC-0xAF */ + 0xB3, 0xBD, 0xC8, 0xE8, 0xDE, 0x72, 0xDE, 0x73, /* 0xB0-0xB3 */ + 0xDE, 0x74, 0xDE, 0x75, 0xE5, 0xC1, 0xDE, 0x76, /* 0xB4-0xB7 */ + 0xDE, 0x77, 0xB1, 0xDF, 0xDE, 0x78, 0xDE, 0x79, /* 0xB8-0xBB */ + 0xDE, 0x7A, 0xC1, 0xC9, 0xB4, 0xEF, 0xDE, 0x7B, /* 0xBC-0xBF */ + 0xDE, 0x7C, 0xC7, 0xA8, 0xD3, 0xD8, 0xDE, 0x7D, /* 0xC0-0xC3 */ + 0xC6, 0xF9, 0xD1, 0xB8, 0xDE, 0x7E, 0xB9, 0xFD, /* 0xC4-0xC7 */ + 0xC2, 0xF5, 0xDE, 0x80, 0xDE, 0x81, 0xDE, 0x82, /* 0xC8-0xCB */ + 0xDE, 0x83, 0xDE, 0x84, 0xD3, 0xAD, 0xDE, 0x85, /* 0xCC-0xCF */ + 0xD4, 0xCB, 0xBD, 0xFC, 0xDE, 0x86, 0xE5, 0xC2, /* 0xD0-0xD3 */ + 0xB7, 0xB5, 0xE5, 0xC3, 0xDE, 0x87, 0xDE, 0x88, /* 0xD4-0xD7 */ + 0xBB, 0xB9, 0xD5, 0xE2, 0xDE, 0x89, 0xBD, 0xF8, /* 0xD8-0xDB */ + 0xD4, 0xB6, 0xCE, 0xA5, 0xC1, 0xAC, 0xB3, 0xD9, /* 0xDC-0xDF */ + 0xDE, 0x8A, 0xDE, 0x8B, 0xCC, 0xF6, 0xDE, 0x8C, /* 0xE0-0xE3 */ + 0xE5, 0xC6, 0xE5, 0xC4, 0xE5, 0xC8, 0xDE, 0x8D, /* 0xE4-0xE7 */ + 0xE5, 0xCA, 0xE5, 0xC7, 0xB5, 0xCF, 0xC6, 0xC8, /* 0xE8-0xEB */ + 0xDE, 0x8E, 0xB5, 0xFC, 0xE5, 0xC5, 0xDE, 0x8F, /* 0xEC-0xEF */ + 0xCA, 0xF6, 0xDE, 0x90, 0xDE, 0x91, 0xE5, 0xC9, /* 0xF0-0xF3 */ + 0xDE, 0x92, 0xDE, 0x93, 0xDE, 0x94, 0xC3, 0xD4, /* 0xF4-0xF7 */ + 0xB1, 0xC5, 0xBC, 0xA3, 0xDE, 0x95, 0xDE, 0x96, /* 0xF8-0xFB */ + 0xDE, 0x97, 0xD7, 0xB7, 0xDE, 0x98, 0xDE, 0x99, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_90[512] = { + 0xCD, 0xCB, 0xCB, 0xCD, 0xCA, 0xCA, 0xCC, 0xD3, /* 0x00-0x03 */ + 0xE5, 0xCC, 0xE5, 0xCB, 0xC4, 0xE6, 0xDE, 0x9A, /* 0x04-0x07 */ + 0xDE, 0x9B, 0xD1, 0xA1, 0xD1, 0xB7, 0xE5, 0xCD, /* 0x08-0x0B */ + 0xDE, 0x9C, 0xE5, 0xD0, 0xDE, 0x9D, 0xCD, 0xB8, /* 0x0C-0x0F */ + 0xD6, 0xF0, 0xE5, 0xCF, 0xB5, 0xDD, 0xDE, 0x9E, /* 0x10-0x13 */ + 0xCD, 0xBE, 0xDE, 0x9F, 0xE5, 0xD1, 0xB6, 0xBA, /* 0x14-0x17 */ + 0xDE, 0xA0, 0xDF, 0x40, 0xCD, 0xA8, 0xB9, 0xE4, /* 0x18-0x1B */ + 0xDF, 0x41, 0xCA, 0xC5, 0xB3, 0xD1, 0xCB, 0xD9, /* 0x1C-0x1F */ + 0xD4, 0xEC, 0xE5, 0xD2, 0xB7, 0xEA, 0xDF, 0x42, /* 0x20-0x23 */ + 0xDF, 0x43, 0xDF, 0x44, 0xE5, 0xCE, 0xDF, 0x45, /* 0x24-0x27 */ + 0xDF, 0x46, 0xDF, 0x47, 0xDF, 0x48, 0xDF, 0x49, /* 0x28-0x2B */ + 0xDF, 0x4A, 0xE5, 0xD5, 0xB4, 0xFE, 0xE5, 0xD6, /* 0x2C-0x2F */ + 0xDF, 0x4B, 0xDF, 0x4C, 0xDF, 0x4D, 0xDF, 0x4E, /* 0x30-0x33 */ + 0xDF, 0x4F, 0xE5, 0xD3, 0xE5, 0xD4, 0xDF, 0x50, /* 0x34-0x37 */ + 0xD2, 0xDD, 0xDF, 0x51, 0xDF, 0x52, 0xC2, 0xDF, /* 0x38-0x3B */ + 0xB1, 0xC6, 0xDF, 0x53, 0xD3, 0xE2, 0xDF, 0x54, /* 0x3C-0x3F */ + 0xDF, 0x55, 0xB6, 0xDD, 0xCB, 0xEC, 0xDF, 0x56, /* 0x40-0x43 */ + 0xE5, 0xD7, 0xDF, 0x57, 0xDF, 0x58, 0xD3, 0xF6, /* 0x44-0x47 */ + 0xDF, 0x59, 0xDF, 0x5A, 0xDF, 0x5B, 0xDF, 0x5C, /* 0x48-0x4B */ + 0xDF, 0x5D, 0xB1, 0xE9, 0xDF, 0x5E, 0xB6, 0xF4, /* 0x4C-0x4F */ + 0xE5, 0xDA, 0xE5, 0xD8, 0xE5, 0xD9, 0xB5, 0xC0, /* 0x50-0x53 */ + 0xDF, 0x5F, 0xDF, 0x60, 0xDF, 0x61, 0xD2, 0xC5, /* 0x54-0x57 */ + 0xE5, 0xDC, 0xDF, 0x62, 0xDF, 0x63, 0xE5, 0xDE, /* 0x58-0x5B */ + 0xDF, 0x64, 0xDF, 0x65, 0xDF, 0x66, 0xDF, 0x67, /* 0x5C-0x5F */ + 0xDF, 0x68, 0xDF, 0x69, 0xE5, 0xDD, 0xC7, 0xB2, /* 0x60-0x63 */ + 0xDF, 0x6A, 0xD2, 0xA3, 0xDF, 0x6B, 0xDF, 0x6C, /* 0x64-0x67 */ + 0xE5, 0xDB, 0xDF, 0x6D, 0xDF, 0x6E, 0xDF, 0x6F, /* 0x68-0x6B */ + 0xDF, 0x70, 0xD4, 0xE2, 0xD5, 0xDA, 0xDF, 0x71, /* 0x6C-0x6F */ + 0xDF, 0x72, 0xDF, 0x73, 0xDF, 0x74, 0xDF, 0x75, /* 0x70-0x73 */ + 0xE5, 0xE0, 0xD7, 0xF1, 0xDF, 0x76, 0xDF, 0x77, /* 0x74-0x77 */ + 0xDF, 0x78, 0xDF, 0x79, 0xDF, 0x7A, 0xDF, 0x7B, /* 0x78-0x7B */ + 0xDF, 0x7C, 0xE5, 0xE1, 0xDF, 0x7D, 0xB1, 0xDC, /* 0x7C-0x7F */ + + 0xD1, 0xFB, 0xDF, 0x7E, 0xE5, 0xE2, 0xE5, 0xE4, /* 0x80-0x83 */ + 0xDF, 0x80, 0xDF, 0x81, 0xDF, 0x82, 0xDF, 0x83, /* 0x84-0x87 */ + 0xE5, 0xE3, 0xDF, 0x84, 0xDF, 0x85, 0xE5, 0xE5, /* 0x88-0x8B */ + 0xDF, 0x86, 0xDF, 0x87, 0xDF, 0x88, 0xDF, 0x89, /* 0x8C-0x8F */ + 0xDF, 0x8A, 0xD2, 0xD8, 0xDF, 0x8B, 0xB5, 0xCB, /* 0x90-0x93 */ + 0xDF, 0x8C, 0xE7, 0xDF, 0xDF, 0x8D, 0xDA, 0xF5, /* 0x94-0x97 */ + 0xDF, 0x8E, 0xDA, 0xF8, 0xDF, 0x8F, 0xDA, 0xF6, /* 0x98-0x9B */ + 0xDF, 0x90, 0xDA, 0xF7, 0xDF, 0x91, 0xDF, 0x92, /* 0x9C-0x9F */ + 0xDF, 0x93, 0xDA, 0xFA, 0xD0, 0xCF, 0xC4, 0xC7, /* 0xA0-0xA3 */ + 0xDF, 0x94, 0xDF, 0x95, 0xB0, 0xEE, 0xDF, 0x96, /* 0xA4-0xA7 */ + 0xDF, 0x97, 0xDF, 0x98, 0xD0, 0xB0, 0xDF, 0x99, /* 0xA8-0xAB */ + 0xDA, 0xF9, 0xDF, 0x9A, 0xD3, 0xCA, 0xBA, 0xAA, /* 0xAC-0xAF */ + 0xDB, 0xA2, 0xC7, 0xF1, 0xDF, 0x9B, 0xDA, 0xFC, /* 0xB0-0xB3 */ + 0xDA, 0xFB, 0xC9, 0xDB, 0xDA, 0xFD, 0xDF, 0x9C, /* 0xB4-0xB7 */ + 0xDB, 0xA1, 0xD7, 0xDE, 0xDA, 0xFE, 0xC1, 0xDA, /* 0xB8-0xBB */ + 0xDF, 0x9D, 0xDF, 0x9E, 0xDB, 0xA5, 0xDF, 0x9F, /* 0xBC-0xBF */ + 0xDF, 0xA0, 0xD3, 0xF4, 0xE0, 0x40, 0xE0, 0x41, /* 0xC0-0xC3 */ + 0xDB, 0xA7, 0xDB, 0xA4, 0xE0, 0x42, 0xDB, 0xA8, /* 0xC4-0xC7 */ + 0xE0, 0x43, 0xE0, 0x44, 0xBD, 0xBC, 0xE0, 0x45, /* 0xC8-0xCB */ + 0xE0, 0x46, 0xE0, 0x47, 0xC0, 0xC9, 0xDB, 0xA3, /* 0xCC-0xCF */ + 0xDB, 0xA6, 0xD6, 0xA3, 0xE0, 0x48, 0xDB, 0xA9, /* 0xD0-0xD3 */ + 0xE0, 0x49, 0xE0, 0x4A, 0xE0, 0x4B, 0xDB, 0xAD, /* 0xD4-0xD7 */ + 0xE0, 0x4C, 0xE0, 0x4D, 0xE0, 0x4E, 0xDB, 0xAE, /* 0xD8-0xDB */ + 0xDB, 0xAC, 0xBA, 0xC2, 0xE0, 0x4F, 0xE0, 0x50, /* 0xDC-0xDF */ + 0xE0, 0x51, 0xBF, 0xA4, 0xDB, 0xAB, 0xE0, 0x52, /* 0xE0-0xE3 */ + 0xE0, 0x53, 0xE0, 0x54, 0xDB, 0xAA, 0xD4, 0xC7, /* 0xE4-0xE7 */ + 0xB2, 0xBF, 0xE0, 0x55, 0xE0, 0x56, 0xDB, 0xAF, /* 0xE8-0xEB */ + 0xE0, 0x57, 0xB9, 0xF9, 0xE0, 0x58, 0xDB, 0xB0, /* 0xEC-0xEF */ + 0xE0, 0x59, 0xE0, 0x5A, 0xE0, 0x5B, 0xE0, 0x5C, /* 0xF0-0xF3 */ + 0xB3, 0xBB, 0xE0, 0x5D, 0xE0, 0x5E, 0xE0, 0x5F, /* 0xF4-0xF7 */ + 0xB5, 0xA6, 0xE0, 0x60, 0xE0, 0x61, 0xE0, 0x62, /* 0xF8-0xFB */ + 0xE0, 0x63, 0xB6, 0xBC, 0xDB, 0xB1, 0xE0, 0x64, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_91[512] = { + 0xE0, 0x65, 0xE0, 0x66, 0xB6, 0xF5, 0xE0, 0x67, /* 0x00-0x03 */ + 0xDB, 0xB2, 0xE0, 0x68, 0xE0, 0x69, 0xE0, 0x6A, /* 0x04-0x07 */ + 0xE0, 0x6B, 0xE0, 0x6C, 0xE0, 0x6D, 0xE0, 0x6E, /* 0x08-0x0B */ + 0xE0, 0x6F, 0xE0, 0x70, 0xE0, 0x71, 0xE0, 0x72, /* 0x0C-0x0F */ + 0xE0, 0x73, 0xE0, 0x74, 0xE0, 0x75, 0xE0, 0x76, /* 0x10-0x13 */ + 0xE0, 0x77, 0xE0, 0x78, 0xE0, 0x79, 0xE0, 0x7A, /* 0x14-0x17 */ + 0xE0, 0x7B, 0xB1, 0xC9, 0xE0, 0x7C, 0xE0, 0x7D, /* 0x18-0x1B */ + 0xE0, 0x7E, 0xE0, 0x80, 0xDB, 0xB4, 0xE0, 0x81, /* 0x1C-0x1F */ + 0xE0, 0x82, 0xE0, 0x83, 0xDB, 0xB3, 0xDB, 0xB5, /* 0x20-0x23 */ + 0xE0, 0x84, 0xE0, 0x85, 0xE0, 0x86, 0xE0, 0x87, /* 0x24-0x27 */ + 0xE0, 0x88, 0xE0, 0x89, 0xE0, 0x8A, 0xE0, 0x8B, /* 0x28-0x2B */ + 0xE0, 0x8C, 0xE0, 0x8D, 0xE0, 0x8E, 0xDB, 0xB7, /* 0x2C-0x2F */ + 0xE0, 0x8F, 0xDB, 0xB6, 0xE0, 0x90, 0xE0, 0x91, /* 0x30-0x33 */ + 0xE0, 0x92, 0xE0, 0x93, 0xE0, 0x94, 0xE0, 0x95, /* 0x34-0x37 */ + 0xE0, 0x96, 0xDB, 0xB8, 0xE0, 0x97, 0xE0, 0x98, /* 0x38-0x3B */ + 0xE0, 0x99, 0xE0, 0x9A, 0xE0, 0x9B, 0xE0, 0x9C, /* 0x3C-0x3F */ + 0xE0, 0x9D, 0xE0, 0x9E, 0xE0, 0x9F, 0xDB, 0xB9, /* 0x40-0x43 */ + 0xE0, 0xA0, 0xE1, 0x40, 0xDB, 0xBA, 0xE1, 0x41, /* 0x44-0x47 */ + 0xE1, 0x42, 0xD3, 0xCF, 0xF4, 0xFA, 0xC7, 0xF5, /* 0x48-0x4B */ + 0xD7, 0xC3, 0xC5, 0xE4, 0xF4, 0xFC, 0xF4, 0xFD, /* 0x4C-0x4F */ + 0xF4, 0xFB, 0xE1, 0x43, 0xBE, 0xC6, 0xE1, 0x44, /* 0x50-0x53 */ + 0xE1, 0x45, 0xE1, 0x46, 0xE1, 0x47, 0xD0, 0xEF, /* 0x54-0x57 */ + 0xE1, 0x48, 0xE1, 0x49, 0xB7, 0xD3, 0xE1, 0x4A, /* 0x58-0x5B */ + 0xE1, 0x4B, 0xD4, 0xCD, 0xCC, 0xAA, 0xE1, 0x4C, /* 0x5C-0x5F */ + 0xE1, 0x4D, 0xF5, 0xA2, 0xF5, 0xA1, 0xBA, 0xA8, /* 0x60-0x63 */ + 0xF4, 0xFE, 0xCB, 0xD6, 0xE1, 0x4E, 0xE1, 0x4F, /* 0x64-0x67 */ + 0xE1, 0x50, 0xF5, 0xA4, 0xC0, 0xD2, 0xE1, 0x51, /* 0x68-0x6B */ + 0xB3, 0xEA, 0xE1, 0x52, 0xCD, 0xAA, 0xF5, 0xA5, /* 0x6C-0x6F */ + 0xF5, 0xA3, 0xBD, 0xB4, 0xF5, 0xA8, 0xE1, 0x53, /* 0x70-0x73 */ + 0xF5, 0xA9, 0xBD, 0xCD, 0xC3, 0xB8, 0xBF, 0xE1, /* 0x74-0x77 */ + 0xCB, 0xE1, 0xF5, 0xAA, 0xE1, 0x54, 0xE1, 0x55, /* 0x78-0x7B */ + 0xE1, 0x56, 0xF5, 0xA6, 0xF5, 0xA7, 0xC4, 0xF0, /* 0x7C-0x7F */ + + 0xE1, 0x57, 0xE1, 0x58, 0xE1, 0x59, 0xE1, 0x5A, /* 0x80-0x83 */ + 0xE1, 0x5B, 0xF5, 0xAC, 0xE1, 0x5C, 0xB4, 0xBC, /* 0x84-0x87 */ + 0xE1, 0x5D, 0xD7, 0xED, 0xE1, 0x5E, 0xB4, 0xD7, /* 0x88-0x8B */ + 0xF5, 0xAB, 0xF5, 0xAE, 0xE1, 0x5F, 0xE1, 0x60, /* 0x8C-0x8F */ + 0xF5, 0xAD, 0xF5, 0xAF, 0xD0, 0xD1, 0xE1, 0x61, /* 0x90-0x93 */ + 0xE1, 0x62, 0xE1, 0x63, 0xE1, 0x64, 0xE1, 0x65, /* 0x94-0x97 */ + 0xE1, 0x66, 0xE1, 0x67, 0xC3, 0xD1, 0xC8, 0xA9, /* 0x98-0x9B */ + 0xE1, 0x68, 0xE1, 0x69, 0xE1, 0x6A, 0xE1, 0x6B, /* 0x9C-0x9F */ + 0xE1, 0x6C, 0xE1, 0x6D, 0xF5, 0xB0, 0xF5, 0xB1, /* 0xA0-0xA3 */ + 0xE1, 0x6E, 0xE1, 0x6F, 0xE1, 0x70, 0xE1, 0x71, /* 0xA4-0xA7 */ + 0xE1, 0x72, 0xE1, 0x73, 0xF5, 0xB2, 0xE1, 0x74, /* 0xA8-0xAB */ + 0xE1, 0x75, 0xF5, 0xB3, 0xF5, 0xB4, 0xF5, 0xB5, /* 0xAC-0xAF */ + 0xE1, 0x76, 0xE1, 0x77, 0xE1, 0x78, 0xE1, 0x79, /* 0xB0-0xB3 */ + 0xF5, 0xB7, 0xF5, 0xB6, 0xE1, 0x7A, 0xE1, 0x7B, /* 0xB4-0xB7 */ + 0xE1, 0x7C, 0xE1, 0x7D, 0xF5, 0xB8, 0xE1, 0x7E, /* 0xB8-0xBB */ + 0xE1, 0x80, 0xE1, 0x81, 0xE1, 0x82, 0xE1, 0x83, /* 0xBC-0xBF */ + 0xE1, 0x84, 0xE1, 0x85, 0xE1, 0x86, 0xE1, 0x87, /* 0xC0-0xC3 */ + 0xE1, 0x88, 0xE1, 0x89, 0xE1, 0x8A, 0xB2, 0xC9, /* 0xC4-0xC7 */ + 0xE1, 0x8B, 0xD3, 0xD4, 0xCA, 0xCD, 0xE1, 0x8C, /* 0xC8-0xCB */ + 0xC0, 0xEF, 0xD6, 0xD8, 0xD2, 0xB0, 0xC1, 0xBF, /* 0xCC-0xCF */ + 0xE1, 0x8D, 0xBD, 0xF0, 0xE1, 0x8E, 0xE1, 0x8F, /* 0xD0-0xD3 */ + 0xE1, 0x90, 0xE1, 0x91, 0xE1, 0x92, 0xE1, 0x93, /* 0xD4-0xD7 */ + 0xE1, 0x94, 0xE1, 0x95, 0xE1, 0x96, 0xE1, 0x97, /* 0xD8-0xDB */ + 0xB8, 0xAA, 0xE1, 0x98, 0xE1, 0x99, 0xE1, 0x9A, /* 0xDC-0xDF */ + 0xE1, 0x9B, 0xE1, 0x9C, 0xE1, 0x9D, 0xE1, 0x9E, /* 0xE0-0xE3 */ + 0xE1, 0x9F, 0xE1, 0xA0, 0xE2, 0x40, 0xE2, 0x41, /* 0xE4-0xE7 */ + 0xE2, 0x42, 0xE2, 0x43, 0xE2, 0x44, 0xE2, 0x45, /* 0xE8-0xEB */ + 0xE2, 0x46, 0xE2, 0x47, 0xE2, 0x48, 0xE2, 0x49, /* 0xEC-0xEF */ + 0xE2, 0x4A, 0xE2, 0x4B, 0xE2, 0x4C, 0xE2, 0x4D, /* 0xF0-0xF3 */ + 0xE2, 0x4E, 0xE2, 0x4F, 0xE2, 0x50, 0xE2, 0x51, /* 0xF4-0xF7 */ + 0xE2, 0x52, 0xE2, 0x53, 0xE2, 0x54, 0xE2, 0x55, /* 0xF8-0xFB */ + 0xE2, 0x56, 0xE2, 0x57, 0xE2, 0x58, 0xE2, 0x59, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_92[512] = { + 0xE2, 0x5A, 0xE2, 0x5B, 0xE2, 0x5C, 0xE2, 0x5D, /* 0x00-0x03 */ + 0xE2, 0x5E, 0xE2, 0x5F, 0xE2, 0x60, 0xE2, 0x61, /* 0x04-0x07 */ + 0xE2, 0x62, 0xE2, 0x63, 0xE2, 0x64, 0xE2, 0x65, /* 0x08-0x0B */ + 0xE2, 0x66, 0xE2, 0x67, 0xE2, 0x68, 0xE2, 0x69, /* 0x0C-0x0F */ + 0xE2, 0x6A, 0xE2, 0x6B, 0xE2, 0x6C, 0xE2, 0x6D, /* 0x10-0x13 */ + 0xE2, 0x6E, 0xE2, 0x6F, 0xE2, 0x70, 0xE2, 0x71, /* 0x14-0x17 */ + 0xE2, 0x72, 0xE2, 0x73, 0xE2, 0x74, 0xE2, 0x75, /* 0x18-0x1B */ + 0xE2, 0x76, 0xE2, 0x77, 0xE2, 0x78, 0xE2, 0x79, /* 0x1C-0x1F */ + 0xE2, 0x7A, 0xE2, 0x7B, 0xE2, 0x7C, 0xE2, 0x7D, /* 0x20-0x23 */ + 0xE2, 0x7E, 0xE2, 0x80, 0xE2, 0x81, 0xE2, 0x82, /* 0x24-0x27 */ + 0xE2, 0x83, 0xE2, 0x84, 0xE2, 0x85, 0xE2, 0x86, /* 0x28-0x2B */ + 0xE2, 0x87, 0xE2, 0x88, 0xE2, 0x89, 0xE2, 0x8A, /* 0x2C-0x2F */ + 0xE2, 0x8B, 0xE2, 0x8C, 0xE2, 0x8D, 0xE2, 0x8E, /* 0x30-0x33 */ + 0xE2, 0x8F, 0xE2, 0x90, 0xE2, 0x91, 0xE2, 0x92, /* 0x34-0x37 */ + 0xE2, 0x93, 0xE2, 0x94, 0xE2, 0x95, 0xE2, 0x96, /* 0x38-0x3B */ + 0xE2, 0x97, 0xE2, 0x98, 0xE2, 0x99, 0xE2, 0x9A, /* 0x3C-0x3F */ + 0xE2, 0x9B, 0xE2, 0x9C, 0xE2, 0x9D, 0xE2, 0x9E, /* 0x40-0x43 */ + 0xE2, 0x9F, 0xE2, 0xA0, 0xE3, 0x40, 0xE3, 0x41, /* 0x44-0x47 */ + 0xE3, 0x42, 0xE3, 0x43, 0xE3, 0x44, 0xE3, 0x45, /* 0x48-0x4B */ + 0xE3, 0x46, 0xE3, 0x47, 0xE3, 0x48, 0xE3, 0x49, /* 0x4C-0x4F */ + 0xE3, 0x4A, 0xE3, 0x4B, 0xE3, 0x4C, 0xE3, 0x4D, /* 0x50-0x53 */ + 0xE3, 0x4E, 0xE3, 0x4F, 0xE3, 0x50, 0xE3, 0x51, /* 0x54-0x57 */ + 0xE3, 0x52, 0xE3, 0x53, 0xE3, 0x54, 0xE3, 0x55, /* 0x58-0x5B */ + 0xE3, 0x56, 0xE3, 0x57, 0xE3, 0x58, 0xE3, 0x59, /* 0x5C-0x5F */ + 0xE3, 0x5A, 0xE3, 0x5B, 0xE3, 0x5C, 0xE3, 0x5D, /* 0x60-0x63 */ + 0xE3, 0x5E, 0xE3, 0x5F, 0xE3, 0x60, 0xE3, 0x61, /* 0x64-0x67 */ + 0xE3, 0x62, 0xE3, 0x63, 0xE3, 0x64, 0xE3, 0x65, /* 0x68-0x6B */ + 0xE3, 0x66, 0xE3, 0x67, 0xE3, 0x68, 0xE3, 0x69, /* 0x6C-0x6F */ + 0xE3, 0x6A, 0xE3, 0x6B, 0xE3, 0x6C, 0xE3, 0x6D, /* 0x70-0x73 */ + 0xBC, 0xF8, 0xE3, 0x6E, 0xE3, 0x6F, 0xE3, 0x70, /* 0x74-0x77 */ + 0xE3, 0x71, 0xE3, 0x72, 0xE3, 0x73, 0xE3, 0x74, /* 0x78-0x7B */ + 0xE3, 0x75, 0xE3, 0x76, 0xE3, 0x77, 0xE3, 0x78, /* 0x7C-0x7F */ + + 0xE3, 0x79, 0xE3, 0x7A, 0xE3, 0x7B, 0xE3, 0x7C, /* 0x80-0x83 */ + 0xE3, 0x7D, 0xE3, 0x7E, 0xE3, 0x80, 0xE3, 0x81, /* 0x84-0x87 */ + 0xE3, 0x82, 0xE3, 0x83, 0xE3, 0x84, 0xE3, 0x85, /* 0x88-0x8B */ + 0xE3, 0x86, 0xE3, 0x87, 0xF6, 0xC6, 0xE3, 0x88, /* 0x8C-0x8F */ + 0xE3, 0x89, 0xE3, 0x8A, 0xE3, 0x8B, 0xE3, 0x8C, /* 0x90-0x93 */ + 0xE3, 0x8D, 0xE3, 0x8E, 0xE3, 0x8F, 0xE3, 0x90, /* 0x94-0x97 */ + 0xE3, 0x91, 0xE3, 0x92, 0xE3, 0x93, 0xE3, 0x94, /* 0x98-0x9B */ + 0xE3, 0x95, 0xE3, 0x96, 0xE3, 0x97, 0xE3, 0x98, /* 0x9C-0x9F */ + 0xE3, 0x99, 0xE3, 0x9A, 0xE3, 0x9B, 0xE3, 0x9C, /* 0xA0-0xA3 */ + 0xE3, 0x9D, 0xE3, 0x9E, 0xE3, 0x9F, 0xE3, 0xA0, /* 0xA4-0xA7 */ + 0xE4, 0x40, 0xE4, 0x41, 0xE4, 0x42, 0xE4, 0x43, /* 0xA8-0xAB */ + 0xE4, 0x44, 0xE4, 0x45, 0xF6, 0xC7, 0xE4, 0x46, /* 0xAC-0xAF */ + 0xE4, 0x47, 0xE4, 0x48, 0xE4, 0x49, 0xE4, 0x4A, /* 0xB0-0xB3 */ + 0xE4, 0x4B, 0xE4, 0x4C, 0xE4, 0x4D, 0xE4, 0x4E, /* 0xB4-0xB7 */ + 0xE4, 0x4F, 0xE4, 0x50, 0xE4, 0x51, 0xE4, 0x52, /* 0xB8-0xBB */ + 0xE4, 0x53, 0xE4, 0x54, 0xE4, 0x55, 0xE4, 0x56, /* 0xBC-0xBF */ + 0xE4, 0x57, 0xE4, 0x58, 0xE4, 0x59, 0xE4, 0x5A, /* 0xC0-0xC3 */ + 0xE4, 0x5B, 0xE4, 0x5C, 0xE4, 0x5D, 0xE4, 0x5E, /* 0xC4-0xC7 */ + 0xF6, 0xC8, 0xE4, 0x5F, 0xE4, 0x60, 0xE4, 0x61, /* 0xC8-0xCB */ + 0xE4, 0x62, 0xE4, 0x63, 0xE4, 0x64, 0xE4, 0x65, /* 0xCC-0xCF */ + 0xE4, 0x66, 0xE4, 0x67, 0xE4, 0x68, 0xE4, 0x69, /* 0xD0-0xD3 */ + 0xE4, 0x6A, 0xE4, 0x6B, 0xE4, 0x6C, 0xE4, 0x6D, /* 0xD4-0xD7 */ + 0xE4, 0x6E, 0xE4, 0x6F, 0xE4, 0x70, 0xE4, 0x71, /* 0xD8-0xDB */ + 0xE4, 0x72, 0xE4, 0x73, 0xE4, 0x74, 0xE4, 0x75, /* 0xDC-0xDF */ + 0xE4, 0x76, 0xE4, 0x77, 0xE4, 0x78, 0xE4, 0x79, /* 0xE0-0xE3 */ + 0xE4, 0x7A, 0xE4, 0x7B, 0xE4, 0x7C, 0xE4, 0x7D, /* 0xE4-0xE7 */ + 0xE4, 0x7E, 0xE4, 0x80, 0xE4, 0x81, 0xE4, 0x82, /* 0xE8-0xEB */ + 0xE4, 0x83, 0xE4, 0x84, 0xE4, 0x85, 0xE4, 0x86, /* 0xEC-0xEF */ + 0xE4, 0x87, 0xE4, 0x88, 0xE4, 0x89, 0xE4, 0x8A, /* 0xF0-0xF3 */ + 0xE4, 0x8B, 0xE4, 0x8C, 0xE4, 0x8D, 0xE4, 0x8E, /* 0xF4-0xF7 */ + 0xE4, 0x8F, 0xE4, 0x90, 0xE4, 0x91, 0xE4, 0x92, /* 0xF8-0xFB */ + 0xE4, 0x93, 0xE4, 0x94, 0xE4, 0x95, 0xE4, 0x96, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_93[512] = { + 0xE4, 0x97, 0xE4, 0x98, 0xE4, 0x99, 0xE4, 0x9A, /* 0x00-0x03 */ + 0xE4, 0x9B, 0xE4, 0x9C, 0xE4, 0x9D, 0xE4, 0x9E, /* 0x04-0x07 */ + 0xE4, 0x9F, 0xE4, 0xA0, 0xE5, 0x40, 0xE5, 0x41, /* 0x08-0x0B */ + 0xE5, 0x42, 0xE5, 0x43, 0xE5, 0x44, 0xE5, 0x45, /* 0x0C-0x0F */ + 0xE5, 0x46, 0xE5, 0x47, 0xE5, 0x48, 0xE5, 0x49, /* 0x10-0x13 */ + 0xE5, 0x4A, 0xE5, 0x4B, 0xE5, 0x4C, 0xE5, 0x4D, /* 0x14-0x17 */ + 0xE5, 0x4E, 0xE5, 0x4F, 0xE5, 0x50, 0xE5, 0x51, /* 0x18-0x1B */ + 0xE5, 0x52, 0xE5, 0x53, 0xE5, 0x54, 0xE5, 0x55, /* 0x1C-0x1F */ + 0xE5, 0x56, 0xE5, 0x57, 0xE5, 0x58, 0xE5, 0x59, /* 0x20-0x23 */ + 0xE5, 0x5A, 0xE5, 0x5B, 0xE5, 0x5C, 0xE5, 0x5D, /* 0x24-0x27 */ + 0xE5, 0x5E, 0xE5, 0x5F, 0xE5, 0x60, 0xE5, 0x61, /* 0x28-0x2B */ + 0xE5, 0x62, 0xE5, 0x63, 0xE5, 0x64, 0xE5, 0x65, /* 0x2C-0x2F */ + 0xE5, 0x66, 0xE5, 0x67, 0xE5, 0x68, 0xE5, 0x69, /* 0x30-0x33 */ + 0xE5, 0x6A, 0xE5, 0x6B, 0xE5, 0x6C, 0xE5, 0x6D, /* 0x34-0x37 */ + 0xE5, 0x6E, 0xE5, 0x6F, 0xE5, 0x70, 0xE5, 0x71, /* 0x38-0x3B */ + 0xE5, 0x72, 0xE5, 0x73, 0xF6, 0xC9, 0xE5, 0x74, /* 0x3C-0x3F */ + 0xE5, 0x75, 0xE5, 0x76, 0xE5, 0x77, 0xE5, 0x78, /* 0x40-0x43 */ + 0xE5, 0x79, 0xE5, 0x7A, 0xE5, 0x7B, 0xE5, 0x7C, /* 0x44-0x47 */ + 0xE5, 0x7D, 0xE5, 0x7E, 0xE5, 0x80, 0xE5, 0x81, /* 0x48-0x4B */ + 0xE5, 0x82, 0xE5, 0x83, 0xE5, 0x84, 0xE5, 0x85, /* 0x4C-0x4F */ + 0xE5, 0x86, 0xE5, 0x87, 0xE5, 0x88, 0xE5, 0x89, /* 0x50-0x53 */ + 0xE5, 0x8A, 0xE5, 0x8B, 0xE5, 0x8C, 0xE5, 0x8D, /* 0x54-0x57 */ + 0xE5, 0x8E, 0xE5, 0x8F, 0xE5, 0x90, 0xE5, 0x91, /* 0x58-0x5B */ + 0xE5, 0x92, 0xE5, 0x93, 0xE5, 0x94, 0xE5, 0x95, /* 0x5C-0x5F */ + 0xE5, 0x96, 0xE5, 0x97, 0xE5, 0x98, 0xE5, 0x99, /* 0x60-0x63 */ + 0xE5, 0x9A, 0xE5, 0x9B, 0xE5, 0x9C, 0xE5, 0x9D, /* 0x64-0x67 */ + 0xE5, 0x9E, 0xE5, 0x9F, 0xF6, 0xCA, 0xE5, 0xA0, /* 0x68-0x6B */ + 0xE6, 0x40, 0xE6, 0x41, 0xE6, 0x42, 0xE6, 0x43, /* 0x6C-0x6F */ + 0xE6, 0x44, 0xE6, 0x45, 0xE6, 0x46, 0xE6, 0x47, /* 0x70-0x73 */ + 0xE6, 0x48, 0xE6, 0x49, 0xE6, 0x4A, 0xE6, 0x4B, /* 0x74-0x77 */ + 0xE6, 0x4C, 0xE6, 0x4D, 0xE6, 0x4E, 0xE6, 0x4F, /* 0x78-0x7B */ + 0xE6, 0x50, 0xE6, 0x51, 0xE6, 0x52, 0xE6, 0x53, /* 0x7C-0x7F */ + + 0xE6, 0x54, 0xE6, 0x55, 0xE6, 0x56, 0xE6, 0x57, /* 0x80-0x83 */ + 0xE6, 0x58, 0xE6, 0x59, 0xE6, 0x5A, 0xE6, 0x5B, /* 0x84-0x87 */ + 0xE6, 0x5C, 0xE6, 0x5D, 0xE6, 0x5E, 0xE6, 0x5F, /* 0x88-0x8B */ + 0xE6, 0x60, 0xE6, 0x61, 0xE6, 0x62, 0xF6, 0xCC, /* 0x8C-0x8F */ + 0xE6, 0x63, 0xE6, 0x64, 0xE6, 0x65, 0xE6, 0x66, /* 0x90-0x93 */ + 0xE6, 0x67, 0xE6, 0x68, 0xE6, 0x69, 0xE6, 0x6A, /* 0x94-0x97 */ + 0xE6, 0x6B, 0xE6, 0x6C, 0xE6, 0x6D, 0xE6, 0x6E, /* 0x98-0x9B */ + 0xE6, 0x6F, 0xE6, 0x70, 0xE6, 0x71, 0xE6, 0x72, /* 0x9C-0x9F */ + 0xE6, 0x73, 0xE6, 0x74, 0xE6, 0x75, 0xE6, 0x76, /* 0xA0-0xA3 */ + 0xE6, 0x77, 0xE6, 0x78, 0xE6, 0x79, 0xE6, 0x7A, /* 0xA4-0xA7 */ + 0xE6, 0x7B, 0xE6, 0x7C, 0xE6, 0x7D, 0xE6, 0x7E, /* 0xA8-0xAB */ + 0xE6, 0x80, 0xE6, 0x81, 0xE6, 0x82, 0xE6, 0x83, /* 0xAC-0xAF */ + 0xE6, 0x84, 0xE6, 0x85, 0xE6, 0x86, 0xE6, 0x87, /* 0xB0-0xB3 */ + 0xE6, 0x88, 0xE6, 0x89, 0xE6, 0x8A, 0xE6, 0x8B, /* 0xB4-0xB7 */ + 0xE6, 0x8C, 0xE6, 0x8D, 0xE6, 0x8E, 0xE6, 0x8F, /* 0xB8-0xBB */ + 0xE6, 0x90, 0xE6, 0x91, 0xE6, 0x92, 0xE6, 0x93, /* 0xBC-0xBF */ + 0xE6, 0x94, 0xE6, 0x95, 0xE6, 0x96, 0xE6, 0x97, /* 0xC0-0xC3 */ + 0xE6, 0x98, 0xE6, 0x99, 0xE6, 0x9A, 0xE6, 0x9B, /* 0xC4-0xC7 */ + 0xE6, 0x9C, 0xE6, 0x9D, 0xF6, 0xCB, 0xE6, 0x9E, /* 0xC8-0xCB */ + 0xE6, 0x9F, 0xE6, 0xA0, 0xE7, 0x40, 0xE7, 0x41, /* 0xCC-0xCF */ + 0xE7, 0x42, 0xE7, 0x43, 0xE7, 0x44, 0xE7, 0x45, /* 0xD0-0xD3 */ + 0xE7, 0x46, 0xE7, 0x47, 0xF7, 0xE9, 0xE7, 0x48, /* 0xD4-0xD7 */ + 0xE7, 0x49, 0xE7, 0x4A, 0xE7, 0x4B, 0xE7, 0x4C, /* 0xD8-0xDB */ + 0xE7, 0x4D, 0xE7, 0x4E, 0xE7, 0x4F, 0xE7, 0x50, /* 0xDC-0xDF */ + 0xE7, 0x51, 0xE7, 0x52, 0xE7, 0x53, 0xE7, 0x54, /* 0xE0-0xE3 */ + 0xE7, 0x55, 0xE7, 0x56, 0xE7, 0x57, 0xE7, 0x58, /* 0xE4-0xE7 */ + 0xE7, 0x59, 0xE7, 0x5A, 0xE7, 0x5B, 0xE7, 0x5C, /* 0xE8-0xEB */ + 0xE7, 0x5D, 0xE7, 0x5E, 0xE7, 0x5F, 0xE7, 0x60, /* 0xEC-0xEF */ + 0xE7, 0x61, 0xE7, 0x62, 0xE7, 0x63, 0xE7, 0x64, /* 0xF0-0xF3 */ + 0xE7, 0x65, 0xE7, 0x66, 0xE7, 0x67, 0xE7, 0x68, /* 0xF4-0xF7 */ + 0xE7, 0x69, 0xE7, 0x6A, 0xE7, 0x6B, 0xE7, 0x6C, /* 0xF8-0xFB */ + 0xE7, 0x6D, 0xE7, 0x6E, 0xE7, 0x6F, 0xE7, 0x70, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_94[512] = { + 0xE7, 0x71, 0xE7, 0x72, 0xE7, 0x73, 0xE7, 0x74, /* 0x00-0x03 */ + 0xE7, 0x75, 0xE7, 0x76, 0xE7, 0x77, 0xE7, 0x78, /* 0x04-0x07 */ + 0xE7, 0x79, 0xE7, 0x7A, 0xE7, 0x7B, 0xE7, 0x7C, /* 0x08-0x0B */ + 0xE7, 0x7D, 0xE7, 0x7E, 0xE7, 0x80, 0xE7, 0x81, /* 0x0C-0x0F */ + 0xE7, 0x82, 0xE7, 0x83, 0xE7, 0x84, 0xE7, 0x85, /* 0x10-0x13 */ + 0xE7, 0x86, 0xE7, 0x87, 0xE7, 0x88, 0xE7, 0x89, /* 0x14-0x17 */ + 0xE7, 0x8A, 0xE7, 0x8B, 0xE7, 0x8C, 0xE7, 0x8D, /* 0x18-0x1B */ + 0xE7, 0x8E, 0xE7, 0x8F, 0xE7, 0x90, 0xE7, 0x91, /* 0x1C-0x1F */ + 0xE7, 0x92, 0xE7, 0x93, 0xE7, 0x94, 0xE7, 0x95, /* 0x20-0x23 */ + 0xE7, 0x96, 0xE7, 0x97, 0xE7, 0x98, 0xE7, 0x99, /* 0x24-0x27 */ + 0xE7, 0x9A, 0xE7, 0x9B, 0xE7, 0x9C, 0xE7, 0x9D, /* 0x28-0x2B */ + 0xE7, 0x9E, 0xE7, 0x9F, 0xE7, 0xA0, 0xE8, 0x40, /* 0x2C-0x2F */ + 0xE8, 0x41, 0xE8, 0x42, 0xE8, 0x43, 0xE8, 0x44, /* 0x30-0x33 */ + 0xE8, 0x45, 0xE8, 0x46, 0xE8, 0x47, 0xE8, 0x48, /* 0x34-0x37 */ + 0xE8, 0x49, 0xE8, 0x4A, 0xE8, 0x4B, 0xE8, 0x4C, /* 0x38-0x3B */ + 0xE8, 0x4D, 0xE8, 0x4E, 0xF6, 0xCD, 0xE8, 0x4F, /* 0x3C-0x3F */ + 0xE8, 0x50, 0xE8, 0x51, 0xE8, 0x52, 0xE8, 0x53, /* 0x40-0x43 */ + 0xE8, 0x54, 0xE8, 0x55, 0xE8, 0x56, 0xE8, 0x57, /* 0x44-0x47 */ + 0xE8, 0x58, 0xE8, 0x59, 0xE8, 0x5A, 0xE8, 0x5B, /* 0x48-0x4B */ + 0xE8, 0x5C, 0xE8, 0x5D, 0xE8, 0x5E, 0xE8, 0x5F, /* 0x4C-0x4F */ + 0xE8, 0x60, 0xE8, 0x61, 0xE8, 0x62, 0xE8, 0x63, /* 0x50-0x53 */ + 0xE8, 0x64, 0xE8, 0x65, 0xE8, 0x66, 0xE8, 0x67, /* 0x54-0x57 */ + 0xE8, 0x68, 0xE8, 0x69, 0xE8, 0x6A, 0xE8, 0x6B, /* 0x58-0x5B */ + 0xE8, 0x6C, 0xE8, 0x6D, 0xE8, 0x6E, 0xE8, 0x6F, /* 0x5C-0x5F */ + 0xE8, 0x70, 0xE8, 0x71, 0xE8, 0x72, 0xE8, 0x73, /* 0x60-0x63 */ + 0xE8, 0x74, 0xE8, 0x75, 0xE8, 0x76, 0xE8, 0x77, /* 0x64-0x67 */ + 0xE8, 0x78, 0xE8, 0x79, 0xE8, 0x7A, 0xF6, 0xCE, /* 0x68-0x6B */ + 0xE8, 0x7B, 0xE8, 0x7C, 0xE8, 0x7D, 0xE8, 0x7E, /* 0x6C-0x6F */ + 0xE8, 0x80, 0xE8, 0x81, 0xE8, 0x82, 0xE8, 0x83, /* 0x70-0x73 */ + 0xE8, 0x84, 0xE8, 0x85, 0xE8, 0x86, 0xE8, 0x87, /* 0x74-0x77 */ + 0xE8, 0x88, 0xE8, 0x89, 0xE8, 0x8A, 0xE8, 0x8B, /* 0x78-0x7B */ + 0xE8, 0x8C, 0xE8, 0x8D, 0xE8, 0x8E, 0xE8, 0x8F, /* 0x7C-0x7F */ + + 0xE8, 0x90, 0xE8, 0x91, 0xE8, 0x92, 0xE8, 0x93, /* 0x80-0x83 */ + 0xE8, 0x94, 0xEE, 0xC4, 0xEE, 0xC5, 0xEE, 0xC6, /* 0x84-0x87 */ + 0xD5, 0xEB, 0xB6, 0xA4, 0xEE, 0xC8, 0xEE, 0xC7, /* 0x88-0x8B */ + 0xEE, 0xC9, 0xEE, 0xCA, 0xC7, 0xA5, 0xEE, 0xCB, /* 0x8C-0x8F */ + 0xEE, 0xCC, 0xE8, 0x95, 0xB7, 0xB0, 0xB5, 0xF6, /* 0x90-0x93 */ + 0xEE, 0xCD, 0xEE, 0xCF, 0xE8, 0x96, 0xEE, 0xCE, /* 0x94-0x97 */ + 0xE8, 0x97, 0xB8, 0xC6, 0xEE, 0xD0, 0xEE, 0xD1, /* 0x98-0x9B */ + 0xEE, 0xD2, 0xB6, 0xDB, 0xB3, 0xAE, 0xD6, 0xD3, /* 0x9C-0x9F */ + 0xC4, 0xC6, 0xB1, 0xB5, 0xB8, 0xD6, 0xEE, 0xD3, /* 0xA0-0xA3 */ + 0xEE, 0xD4, 0xD4, 0xBF, 0xC7, 0xD5, 0xBE, 0xFB, /* 0xA4-0xA7 */ + 0xCE, 0xD9, 0xB9, 0xB3, 0xEE, 0xD6, 0xEE, 0xD5, /* 0xA8-0xAB */ + 0xEE, 0xD8, 0xEE, 0xD7, 0xC5, 0xA5, 0xEE, 0xD9, /* 0xAC-0xAF */ + 0xEE, 0xDA, 0xC7, 0xAE, 0xEE, 0xDB, 0xC7, 0xAF, /* 0xB0-0xB3 */ + 0xEE, 0xDC, 0xB2, 0xA7, 0xEE, 0xDD, 0xEE, 0xDE, /* 0xB4-0xB7 */ + 0xEE, 0xDF, 0xEE, 0xE0, 0xEE, 0xE1, 0xD7, 0xEA, /* 0xB8-0xBB */ + 0xEE, 0xE2, 0xEE, 0xE3, 0xBC, 0xD8, 0xEE, 0xE4, /* 0xBC-0xBF */ + 0xD3, 0xCB, 0xCC, 0xFA, 0xB2, 0xAC, 0xC1, 0xE5, /* 0xC0-0xC3 */ + 0xEE, 0xE5, 0xC7, 0xA6, 0xC3, 0xAD, 0xE8, 0x98, /* 0xC4-0xC7 */ + 0xEE, 0xE6, 0xEE, 0xE7, 0xEE, 0xE8, 0xEE, 0xE9, /* 0xC8-0xCB */ + 0xEE, 0xEA, 0xEE, 0xEB, 0xEE, 0xEC, 0xE8, 0x99, /* 0xCC-0xCF */ + 0xEE, 0xED, 0xEE, 0xEE, 0xEE, 0xEF, 0xE8, 0x9A, /* 0xD0-0xD3 */ + 0xE8, 0x9B, 0xEE, 0xF0, 0xEE, 0xF1, 0xEE, 0xF2, /* 0xD4-0xD7 */ + 0xEE, 0xF4, 0xEE, 0xF3, 0xE8, 0x9C, 0xEE, 0xF5, /* 0xD8-0xDB */ + 0xCD, 0xAD, 0xC2, 0xC1, 0xEE, 0xF6, 0xEE, 0xF7, /* 0xDC-0xDF */ + 0xEE, 0xF8, 0xD5, 0xA1, 0xEE, 0xF9, 0xCF, 0xB3, /* 0xE0-0xE3 */ + 0xEE, 0xFA, 0xEE, 0xFB, 0xE8, 0x9D, 0xEE, 0xFC, /* 0xE4-0xE7 */ + 0xEE, 0xFD, 0xEF, 0xA1, 0xEE, 0xFE, 0xEF, 0xA2, /* 0xE8-0xEB */ + 0xB8, 0xF5, 0xC3, 0xFA, 0xEF, 0xA3, 0xEF, 0xA4, /* 0xEC-0xEF */ + 0xBD, 0xC2, 0xD2, 0xBF, 0xB2, 0xF9, 0xEF, 0xA5, /* 0xF0-0xF3 */ + 0xEF, 0xA6, 0xEF, 0xA7, 0xD2, 0xF8, 0xEF, 0xA8, /* 0xF4-0xF7 */ + 0xD6, 0xFD, 0xEF, 0xA9, 0xC6, 0xCC, 0xE8, 0x9E, /* 0xF8-0xFB */ + 0xEF, 0xAA, 0xEF, 0xAB, 0xC1, 0xB4, 0xEF, 0xAC, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_95[512] = { + 0xCF, 0xFA, 0xCB, 0xF8, 0xEF, 0xAE, 0xEF, 0xAD, /* 0x00-0x03 */ + 0xB3, 0xFA, 0xB9, 0xF8, 0xEF, 0xAF, 0xEF, 0xB0, /* 0x04-0x07 */ + 0xD0, 0xE2, 0xEF, 0xB1, 0xEF, 0xB2, 0xB7, 0xE6, /* 0x08-0x0B */ + 0xD0, 0xBF, 0xEF, 0xB3, 0xEF, 0xB4, 0xEF, 0xB5, /* 0x0C-0x0F */ + 0xC8, 0xF1, 0xCC, 0xE0, 0xEF, 0xB6, 0xEF, 0xB7, /* 0x10-0x13 */ + 0xEF, 0xB8, 0xEF, 0xB9, 0xEF, 0xBA, 0xD5, 0xE0, /* 0x14-0x17 */ + 0xEF, 0xBB, 0xB4, 0xED, 0xC3, 0xAA, 0xEF, 0xBC, /* 0x18-0x1B */ + 0xE8, 0x9F, 0xEF, 0xBD, 0xEF, 0xBE, 0xEF, 0xBF, /* 0x1C-0x1F */ + 0xE8, 0xA0, 0xCE, 0xFD, 0xEF, 0xC0, 0xC2, 0xE0, /* 0x20-0x23 */ + 0xB4, 0xB8, 0xD7, 0xB6, 0xBD, 0xF5, 0xE9, 0x40, /* 0x24-0x27 */ + 0xCF, 0xC7, 0xEF, 0xC3, 0xEF, 0xC1, 0xEF, 0xC2, /* 0x28-0x2B */ + 0xEF, 0xC4, 0xB6, 0xA7, 0xBC, 0xFC, 0xBE, 0xE2, /* 0x2C-0x2F */ + 0xC3, 0xCC, 0xEF, 0xC5, 0xEF, 0xC6, 0xE9, 0x41, /* 0x30-0x33 */ + 0xEF, 0xC7, 0xEF, 0xCF, 0xEF, 0xC8, 0xEF, 0xC9, /* 0x34-0x37 */ + 0xEF, 0xCA, 0xC7, 0xC2, 0xEF, 0xF1, 0xB6, 0xCD, /* 0x38-0x3B */ + 0xEF, 0xCB, 0xE9, 0x42, 0xEF, 0xCC, 0xEF, 0xCD, /* 0x3C-0x3F */ + 0xB6, 0xC6, 0xC3, 0xBE, 0xEF, 0xCE, 0xE9, 0x43, /* 0x40-0x43 */ + 0xEF, 0xD0, 0xEF, 0xD1, 0xEF, 0xD2, 0xD5, 0xF2, /* 0x44-0x47 */ + 0xE9, 0x44, 0xEF, 0xD3, 0xC4, 0xF7, 0xE9, 0x45, /* 0x48-0x4B */ + 0xEF, 0xD4, 0xC4, 0xF8, 0xEF, 0xD5, 0xEF, 0xD6, /* 0x4C-0x4F */ + 0xB8, 0xE4, 0xB0, 0xF7, 0xEF, 0xD7, 0xEF, 0xD8, /* 0x50-0x53 */ + 0xEF, 0xD9, 0xE9, 0x46, 0xEF, 0xDA, 0xEF, 0xDB, /* 0x54-0x57 */ + 0xEF, 0xDC, 0xEF, 0xDD, 0xE9, 0x47, 0xEF, 0xDE, /* 0x58-0x5B */ + 0xBE, 0xB5, 0xEF, 0xE1, 0xEF, 0xDF, 0xEF, 0xE0, /* 0x5C-0x5F */ + 0xE9, 0x48, 0xEF, 0xE2, 0xEF, 0xE3, 0xC1, 0xCD, /* 0x60-0x63 */ + 0xEF, 0xE4, 0xEF, 0xE5, 0xEF, 0xE6, 0xEF, 0xE7, /* 0x64-0x67 */ + 0xEF, 0xE8, 0xEF, 0xE9, 0xEF, 0xEA, 0xEF, 0xEB, /* 0x68-0x6B */ + 0xEF, 0xEC, 0xC0, 0xD8, 0xE9, 0x49, 0xEF, 0xED, /* 0x6C-0x6F */ + 0xC1, 0xAD, 0xEF, 0xEE, 0xEF, 0xEF, 0xEF, 0xF0, /* 0x70-0x73 */ + 0xE9, 0x4A, 0xE9, 0x4B, 0xCF, 0xE2, 0xE9, 0x4C, /* 0x74-0x77 */ + 0xE9, 0x4D, 0xE9, 0x4E, 0xE9, 0x4F, 0xE9, 0x50, /* 0x78-0x7B */ + 0xE9, 0x51, 0xE9, 0x52, 0xE9, 0x53, 0xB3, 0xA4, /* 0x7C-0x7F */ + + 0xE9, 0x54, 0xE9, 0x55, 0xE9, 0x56, 0xE9, 0x57, /* 0x80-0x83 */ + 0xE9, 0x58, 0xE9, 0x59, 0xE9, 0x5A, 0xE9, 0x5B, /* 0x84-0x87 */ + 0xE9, 0x5C, 0xE9, 0x5D, 0xE9, 0x5E, 0xE9, 0x5F, /* 0x88-0x8B */ + 0xE9, 0x60, 0xE9, 0x61, 0xE9, 0x62, 0xE9, 0x63, /* 0x8C-0x8F */ + 0xE9, 0x64, 0xE9, 0x65, 0xE9, 0x66, 0xE9, 0x67, /* 0x90-0x93 */ + 0xE9, 0x68, 0xE9, 0x69, 0xE9, 0x6A, 0xE9, 0x6B, /* 0x94-0x97 */ + 0xE9, 0x6C, 0xE9, 0x6D, 0xE9, 0x6E, 0xE9, 0x6F, /* 0x98-0x9B */ + 0xE9, 0x70, 0xE9, 0x71, 0xE9, 0x72, 0xE9, 0x73, /* 0x9C-0x9F */ + 0xE9, 0x74, 0xE9, 0x75, 0xE9, 0x76, 0xE9, 0x77, /* 0xA0-0xA3 */ + 0xE9, 0x78, 0xE9, 0x79, 0xE9, 0x7A, 0xE9, 0x7B, /* 0xA4-0xA7 */ + 0xE9, 0x7C, 0xE9, 0x7D, 0xE9, 0x7E, 0xE9, 0x80, /* 0xA8-0xAB */ + 0xE9, 0x81, 0xE9, 0x82, 0xE9, 0x83, 0xE9, 0x84, /* 0xAC-0xAF */ + 0xE9, 0x85, 0xE9, 0x86, 0xE9, 0x87, 0xE9, 0x88, /* 0xB0-0xB3 */ + 0xE9, 0x89, 0xE9, 0x8A, 0xE9, 0x8B, 0xE9, 0x8C, /* 0xB4-0xB7 */ + 0xE9, 0x8D, 0xE9, 0x8E, 0xE9, 0x8F, 0xE9, 0x90, /* 0xB8-0xBB */ + 0xE9, 0x91, 0xE9, 0x92, 0xE9, 0x93, 0xE9, 0x94, /* 0xBC-0xBF */ + 0xE9, 0x95, 0xE9, 0x96, 0xE9, 0x97, 0xE9, 0x98, /* 0xC0-0xC3 */ + 0xE9, 0x99, 0xE9, 0x9A, 0xE9, 0x9B, 0xE9, 0x9C, /* 0xC4-0xC7 */ + 0xE9, 0x9D, 0xE9, 0x9E, 0xE9, 0x9F, 0xE9, 0xA0, /* 0xC8-0xCB */ + 0xEA, 0x40, 0xEA, 0x41, 0xEA, 0x42, 0xEA, 0x43, /* 0xCC-0xCF */ + 0xEA, 0x44, 0xEA, 0x45, 0xEA, 0x46, 0xEA, 0x47, /* 0xD0-0xD3 */ + 0xEA, 0x48, 0xEA, 0x49, 0xEA, 0x4A, 0xEA, 0x4B, /* 0xD4-0xD7 */ + 0xEA, 0x4C, 0xEA, 0x4D, 0xEA, 0x4E, 0xEA, 0x4F, /* 0xD8-0xDB */ + 0xEA, 0x50, 0xEA, 0x51, 0xEA, 0x52, 0xEA, 0x53, /* 0xDC-0xDF */ + 0xEA, 0x54, 0xEA, 0x55, 0xEA, 0x56, 0xEA, 0x57, /* 0xE0-0xE3 */ + 0xEA, 0x58, 0xEA, 0x59, 0xEA, 0x5A, 0xEA, 0x5B, /* 0xE4-0xE7 */ + 0xC3, 0xC5, 0xE3, 0xC5, 0xC9, 0xC1, 0xE3, 0xC6, /* 0xE8-0xEB */ + 0xEA, 0x5C, 0xB1, 0xD5, 0xCE, 0xCA, 0xB4, 0xB3, /* 0xEC-0xEF */ + 0xC8, 0xF2, 0xE3, 0xC7, 0xCF, 0xD0, 0xE3, 0xC8, /* 0xF0-0xF3 */ + 0xBC, 0xE4, 0xE3, 0xC9, 0xE3, 0xCA, 0xC3, 0xC6, /* 0xF4-0xF7 */ + 0xD5, 0xA2, 0xC4, 0xD6, 0xB9, 0xEB, 0xCE, 0xC5, /* 0xF8-0xFB */ + 0xE3, 0xCB, 0xC3, 0xF6, 0xE3, 0xCC, 0xEA, 0x5D, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_96[512] = { + 0xB7, 0xA7, 0xB8, 0xF3, 0xBA, 0xD2, 0xE3, 0xCD, /* 0x00-0x03 */ + 0xE3, 0xCE, 0xD4, 0xC4, 0xE3, 0xCF, 0xEA, 0x5E, /* 0x04-0x07 */ + 0xE3, 0xD0, 0xD1, 0xCB, 0xE3, 0xD1, 0xE3, 0xD2, /* 0x08-0x0B */ + 0xE3, 0xD3, 0xE3, 0xD4, 0xD1, 0xD6, 0xE3, 0xD5, /* 0x0C-0x0F */ + 0xB2, 0xFB, 0xC0, 0xBB, 0xE3, 0xD6, 0xEA, 0x5F, /* 0x10-0x13 */ + 0xC0, 0xAB, 0xE3, 0xD7, 0xE3, 0xD8, 0xE3, 0xD9, /* 0x14-0x17 */ + 0xEA, 0x60, 0xE3, 0xDA, 0xE3, 0xDB, 0xEA, 0x61, /* 0x18-0x1B */ + 0xB8, 0xB7, 0xDA, 0xE2, 0xEA, 0x62, 0xB6, 0xD3, /* 0x1C-0x1F */ + 0xEA, 0x63, 0xDA, 0xE4, 0xDA, 0xE3, 0xEA, 0x64, /* 0x20-0x23 */ + 0xEA, 0x65, 0xEA, 0x66, 0xEA, 0x67, 0xEA, 0x68, /* 0x24-0x27 */ + 0xEA, 0x69, 0xEA, 0x6A, 0xDA, 0xE6, 0xEA, 0x6B, /* 0x28-0x2B */ + 0xEA, 0x6C, 0xEA, 0x6D, 0xC8, 0xEE, 0xEA, 0x6E, /* 0x2C-0x2F */ + 0xEA, 0x6F, 0xDA, 0xE5, 0xB7, 0xC0, 0xD1, 0xF4, /* 0x30-0x33 */ + 0xD2, 0xF5, 0xD5, 0xF3, 0xBD, 0xD7, 0xEA, 0x70, /* 0x34-0x37 */ + 0xEA, 0x71, 0xEA, 0x72, 0xEA, 0x73, 0xD7, 0xE8, /* 0x38-0x3B */ + 0xDA, 0xE8, 0xDA, 0xE7, 0xEA, 0x74, 0xB0, 0xA2, /* 0x3C-0x3F */ + 0xCD, 0xD3, 0xEA, 0x75, 0xDA, 0xE9, 0xEA, 0x76, /* 0x40-0x43 */ + 0xB8, 0xBD, 0xBC, 0xCA, 0xC2, 0xBD, 0xC2, 0xA4, /* 0x44-0x47 */ + 0xB3, 0xC2, 0xDA, 0xEA, 0xEA, 0x77, 0xC2, 0xAA, /* 0x48-0x4B */ + 0xC4, 0xB0, 0xBD, 0xB5, 0xEA, 0x78, 0xEA, 0x79, /* 0x4C-0x4F */ + 0xCF, 0xDE, 0xEA, 0x7A, 0xEA, 0x7B, 0xEA, 0x7C, /* 0x50-0x53 */ + 0xDA, 0xEB, 0xC9, 0xC2, 0xEA, 0x7D, 0xEA, 0x7E, /* 0x54-0x57 */ + 0xEA, 0x80, 0xEA, 0x81, 0xEA, 0x82, 0xB1, 0xDD, /* 0x58-0x5B */ + 0xEA, 0x83, 0xEA, 0x84, 0xEA, 0x85, 0xDA, 0xEC, /* 0x5C-0x5F */ + 0xEA, 0x86, 0xB6, 0xB8, 0xD4, 0xBA, 0xEA, 0x87, /* 0x60-0x63 */ + 0xB3, 0xFD, 0xEA, 0x88, 0xEA, 0x89, 0xDA, 0xED, /* 0x64-0x67 */ + 0xD4, 0xC9, 0xCF, 0xD5, 0xC5, 0xE3, 0xEA, 0x8A, /* 0x68-0x6B */ + 0xDA, 0xEE, 0xEA, 0x8B, 0xEA, 0x8C, 0xEA, 0x8D, /* 0x6C-0x6F */ + 0xEA, 0x8E, 0xEA, 0x8F, 0xDA, 0xEF, 0xEA, 0x90, /* 0x70-0x73 */ + 0xDA, 0xF0, 0xC1, 0xEA, 0xCC, 0xD5, 0xCF, 0xDD, /* 0x74-0x77 */ + 0xEA, 0x91, 0xEA, 0x92, 0xEA, 0x93, 0xEA, 0x94, /* 0x78-0x7B */ + 0xEA, 0x95, 0xEA, 0x96, 0xEA, 0x97, 0xEA, 0x98, /* 0x7C-0x7F */ + + 0xEA, 0x99, 0xEA, 0x9A, 0xEA, 0x9B, 0xEA, 0x9C, /* 0x80-0x83 */ + 0xEA, 0x9D, 0xD3, 0xE7, 0xC2, 0xA1, 0xEA, 0x9E, /* 0x84-0x87 */ + 0xDA, 0xF1, 0xEA, 0x9F, 0xEA, 0xA0, 0xCB, 0xE5, /* 0x88-0x8B */ + 0xEB, 0x40, 0xDA, 0xF2, 0xEB, 0x41, 0xCB, 0xE6, /* 0x8C-0x8F */ + 0xD2, 0xFE, 0xEB, 0x42, 0xEB, 0x43, 0xEB, 0x44, /* 0x90-0x93 */ + 0xB8, 0xF4, 0xEB, 0x45, 0xEB, 0x46, 0xDA, 0xF3, /* 0x94-0x97 */ + 0xB0, 0xAF, 0xCF, 0xB6, 0xEB, 0x47, 0xEB, 0x48, /* 0x98-0x9B */ + 0xD5, 0xCF, 0xEB, 0x49, 0xEB, 0x4A, 0xEB, 0x4B, /* 0x9C-0x9F */ + 0xEB, 0x4C, 0xEB, 0x4D, 0xEB, 0x4E, 0xEB, 0x4F, /* 0xA0-0xA3 */ + 0xEB, 0x50, 0xEB, 0x51, 0xEB, 0x52, 0xCB, 0xED, /* 0xA4-0xA7 */ + 0xEB, 0x53, 0xEB, 0x54, 0xEB, 0x55, 0xEB, 0x56, /* 0xA8-0xAB */ + 0xEB, 0x57, 0xEB, 0x58, 0xEB, 0x59, 0xEB, 0x5A, /* 0xAC-0xAF */ + 0xDA, 0xF4, 0xEB, 0x5B, 0xEB, 0x5C, 0xE3, 0xC4, /* 0xB0-0xB3 */ + 0xEB, 0x5D, 0xEB, 0x5E, 0xC1, 0xA5, 0xEB, 0x5F, /* 0xB4-0xB7 */ + 0xEB, 0x60, 0xF6, 0xBF, 0xEB, 0x61, 0xEB, 0x62, /* 0xB8-0xBB */ + 0xF6, 0xC0, 0xF6, 0xC1, 0xC4, 0xD1, 0xEB, 0x63, /* 0xBC-0xBF */ + 0xC8, 0xB8, 0xD1, 0xE3, 0xEB, 0x64, 0xEB, 0x65, /* 0xC0-0xC3 */ + 0xD0, 0xDB, 0xD1, 0xC5, 0xBC, 0xAF, 0xB9, 0xCD, /* 0xC4-0xC7 */ + 0xEB, 0x66, 0xEF, 0xF4, 0xEB, 0x67, 0xEB, 0x68, /* 0xC8-0xCB */ + 0xB4, 0xC6, 0xD3, 0xBA, 0xF6, 0xC2, 0xB3, 0xFB, /* 0xCC-0xCF */ + 0xEB, 0x69, 0xEB, 0x6A, 0xF6, 0xC3, 0xEB, 0x6B, /* 0xD0-0xD3 */ + 0xEB, 0x6C, 0xB5, 0xF1, 0xEB, 0x6D, 0xEB, 0x6E, /* 0xD4-0xD7 */ + 0xEB, 0x6F, 0xEB, 0x70, 0xEB, 0x71, 0xEB, 0x72, /* 0xD8-0xDB */ + 0xEB, 0x73, 0xEB, 0x74, 0xEB, 0x75, 0xEB, 0x76, /* 0xDC-0xDF */ + 0xF6, 0xC5, 0xEB, 0x77, 0xEB, 0x78, 0xEB, 0x79, /* 0xE0-0xE3 */ + 0xEB, 0x7A, 0xEB, 0x7B, 0xEB, 0x7C, 0xEB, 0x7D, /* 0xE4-0xE7 */ + 0xD3, 0xEA, 0xF6, 0xA7, 0xD1, 0xA9, 0xEB, 0x7E, /* 0xE8-0xEB */ + 0xEB, 0x80, 0xEB, 0x81, 0xEB, 0x82, 0xF6, 0xA9, /* 0xEC-0xEF */ + 0xEB, 0x83, 0xEB, 0x84, 0xEB, 0x85, 0xF6, 0xA8, /* 0xF0-0xF3 */ + 0xEB, 0x86, 0xEB, 0x87, 0xC1, 0xE3, 0xC0, 0xD7, /* 0xF4-0xF7 */ + 0xEB, 0x88, 0xB1, 0xA2, 0xEB, 0x89, 0xEB, 0x8A, /* 0xF8-0xFB */ + 0xEB, 0x8B, 0xEB, 0x8C, 0xCE, 0xED, 0xEB, 0x8D, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_97[512] = { + 0xD0, 0xE8, 0xF6, 0xAB, 0xEB, 0x8E, 0xEB, 0x8F, /* 0x00-0x03 */ + 0xCF, 0xF6, 0xEB, 0x90, 0xF6, 0xAA, 0xD5, 0xF0, /* 0x04-0x07 */ + 0xF6, 0xAC, 0xC3, 0xB9, 0xEB, 0x91, 0xEB, 0x92, /* 0x08-0x0B */ + 0xEB, 0x93, 0xBB, 0xF4, 0xF6, 0xAE, 0xF6, 0xAD, /* 0x0C-0x0F */ + 0xEB, 0x94, 0xEB, 0x95, 0xEB, 0x96, 0xC4, 0xDE, /* 0x10-0x13 */ + 0xEB, 0x97, 0xEB, 0x98, 0xC1, 0xD8, 0xEB, 0x99, /* 0x14-0x17 */ + 0xEB, 0x9A, 0xEB, 0x9B, 0xEB, 0x9C, 0xEB, 0x9D, /* 0x18-0x1B */ + 0xCB, 0xAA, 0xEB, 0x9E, 0xCF, 0xBC, 0xEB, 0x9F, /* 0x1C-0x1F */ + 0xEB, 0xA0, 0xEC, 0x40, 0xEC, 0x41, 0xEC, 0x42, /* 0x20-0x23 */ + 0xEC, 0x43, 0xEC, 0x44, 0xEC, 0x45, 0xEC, 0x46, /* 0x24-0x27 */ + 0xEC, 0x47, 0xEC, 0x48, 0xF6, 0xAF, 0xEC, 0x49, /* 0x28-0x2B */ + 0xEC, 0x4A, 0xF6, 0xB0, 0xEC, 0x4B, 0xEC, 0x4C, /* 0x2C-0x2F */ + 0xF6, 0xB1, 0xEC, 0x4D, 0xC2, 0xB6, 0xEC, 0x4E, /* 0x30-0x33 */ + 0xEC, 0x4F, 0xEC, 0x50, 0xEC, 0x51, 0xEC, 0x52, /* 0x34-0x37 */ + 0xB0, 0xD4, 0xC5, 0xF9, 0xEC, 0x53, 0xEC, 0x54, /* 0x38-0x3B */ + 0xEC, 0x55, 0xEC, 0x56, 0xF6, 0xB2, 0xEC, 0x57, /* 0x3C-0x3F */ + 0xEC, 0x58, 0xEC, 0x59, 0xEC, 0x5A, 0xEC, 0x5B, /* 0x40-0x43 */ + 0xEC, 0x5C, 0xEC, 0x5D, 0xEC, 0x5E, 0xEC, 0x5F, /* 0x44-0x47 */ + 0xEC, 0x60, 0xEC, 0x61, 0xEC, 0x62, 0xEC, 0x63, /* 0x48-0x4B */ + 0xEC, 0x64, 0xEC, 0x65, 0xEC, 0x66, 0xEC, 0x67, /* 0x4C-0x4F */ + 0xEC, 0x68, 0xEC, 0x69, 0xC7, 0xE0, 0xF6, 0xA6, /* 0x50-0x53 */ + 0xEC, 0x6A, 0xEC, 0x6B, 0xBE, 0xB8, 0xEC, 0x6C, /* 0x54-0x57 */ + 0xEC, 0x6D, 0xBE, 0xB2, 0xEC, 0x6E, 0xB5, 0xE5, /* 0x58-0x5B */ + 0xEC, 0x6F, 0xEC, 0x70, 0xB7, 0xC7, 0xEC, 0x71, /* 0x5C-0x5F */ + 0xBF, 0xBF, 0xC3, 0xD2, 0xC3, 0xE6, 0xEC, 0x72, /* 0x60-0x63 */ + 0xEC, 0x73, 0xD8, 0xCC, 0xEC, 0x74, 0xEC, 0x75, /* 0x64-0x67 */ + 0xEC, 0x76, 0xB8, 0xEF, 0xEC, 0x77, 0xEC, 0x78, /* 0x68-0x6B */ + 0xEC, 0x79, 0xEC, 0x7A, 0xEC, 0x7B, 0xEC, 0x7C, /* 0x6C-0x6F */ + 0xEC, 0x7D, 0xEC, 0x7E, 0xEC, 0x80, 0xBD, 0xF9, /* 0x70-0x73 */ + 0xD1, 0xA5, 0xEC, 0x81, 0xB0, 0xD0, 0xEC, 0x82, /* 0x74-0x77 */ + 0xEC, 0x83, 0xEC, 0x84, 0xEC, 0x85, 0xEC, 0x86, /* 0x78-0x7B */ + 0xF7, 0xB0, 0xEC, 0x87, 0xEC, 0x88, 0xEC, 0x89, /* 0x7C-0x7F */ + + 0xEC, 0x8A, 0xEC, 0x8B, 0xEC, 0x8C, 0xEC, 0x8D, /* 0x80-0x83 */ + 0xEC, 0x8E, 0xF7, 0xB1, 0xEC, 0x8F, 0xEC, 0x90, /* 0x84-0x87 */ + 0xEC, 0x91, 0xEC, 0x92, 0xEC, 0x93, 0xD0, 0xAC, /* 0x88-0x8B */ + 0xEC, 0x94, 0xB0, 0xB0, 0xEC, 0x95, 0xEC, 0x96, /* 0x8C-0x8F */ + 0xEC, 0x97, 0xF7, 0xB2, 0xF7, 0xB3, 0xEC, 0x98, /* 0x90-0x93 */ + 0xF7, 0xB4, 0xEC, 0x99, 0xEC, 0x9A, 0xEC, 0x9B, /* 0x94-0x97 */ + 0xC7, 0xCA, 0xEC, 0x9C, 0xEC, 0x9D, 0xEC, 0x9E, /* 0x98-0x9B */ + 0xEC, 0x9F, 0xEC, 0xA0, 0xED, 0x40, 0xED, 0x41, /* 0x9C-0x9F */ + 0xBE, 0xCF, 0xED, 0x42, 0xED, 0x43, 0xF7, 0xB7, /* 0xA0-0xA3 */ + 0xED, 0x44, 0xED, 0x45, 0xED, 0x46, 0xED, 0x47, /* 0xA4-0xA7 */ + 0xED, 0x48, 0xED, 0x49, 0xED, 0x4A, 0xF7, 0xB6, /* 0xA8-0xAB */ + 0xED, 0x4B, 0xB1, 0xDE, 0xED, 0x4C, 0xF7, 0xB5, /* 0xAC-0xAF */ + 0xED, 0x4D, 0xED, 0x4E, 0xF7, 0xB8, 0xED, 0x4F, /* 0xB0-0xB3 */ + 0xF7, 0xB9, 0xED, 0x50, 0xED, 0x51, 0xED, 0x52, /* 0xB4-0xB7 */ + 0xED, 0x53, 0xED, 0x54, 0xED, 0x55, 0xED, 0x56, /* 0xB8-0xBB */ + 0xED, 0x57, 0xED, 0x58, 0xED, 0x59, 0xED, 0x5A, /* 0xBC-0xBF */ + 0xED, 0x5B, 0xED, 0x5C, 0xED, 0x5D, 0xED, 0x5E, /* 0xC0-0xC3 */ + 0xED, 0x5F, 0xED, 0x60, 0xED, 0x61, 0xED, 0x62, /* 0xC4-0xC7 */ + 0xED, 0x63, 0xED, 0x64, 0xED, 0x65, 0xED, 0x66, /* 0xC8-0xCB */ + 0xED, 0x67, 0xED, 0x68, 0xED, 0x69, 0xED, 0x6A, /* 0xCC-0xCF */ + 0xED, 0x6B, 0xED, 0x6C, 0xED, 0x6D, 0xED, 0x6E, /* 0xD0-0xD3 */ + 0xED, 0x6F, 0xED, 0x70, 0xED, 0x71, 0xED, 0x72, /* 0xD4-0xD7 */ + 0xED, 0x73, 0xED, 0x74, 0xED, 0x75, 0xED, 0x76, /* 0xD8-0xDB */ + 0xED, 0x77, 0xED, 0x78, 0xED, 0x79, 0xED, 0x7A, /* 0xDC-0xDF */ + 0xED, 0x7B, 0xED, 0x7C, 0xED, 0x7D, 0xED, 0x7E, /* 0xE0-0xE3 */ + 0xED, 0x80, 0xED, 0x81, 0xCE, 0xA4, 0xC8, 0xCD, /* 0xE4-0xE7 */ + 0xED, 0x82, 0xBA, 0xAB, 0xE8, 0xB8, 0xE8, 0xB9, /* 0xE8-0xEB */ + 0xE8, 0xBA, 0xBE, 0xC2, 0xED, 0x83, 0xED, 0x84, /* 0xEC-0xEF */ + 0xED, 0x85, 0xED, 0x86, 0xED, 0x87, 0xD2, 0xF4, /* 0xF0-0xF3 */ + 0xED, 0x88, 0xD4, 0xCF, 0xC9, 0xD8, 0xED, 0x89, /* 0xF4-0xF7 */ + 0xED, 0x8A, 0xED, 0x8B, 0xED, 0x8C, 0xED, 0x8D, /* 0xF8-0xFB */ + 0xED, 0x8E, 0xED, 0x8F, 0xED, 0x90, 0xED, 0x91, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_98[512] = { + 0xED, 0x92, 0xED, 0x93, 0xED, 0x94, 0xED, 0x95, /* 0x00-0x03 */ + 0xED, 0x96, 0xED, 0x97, 0xED, 0x98, 0xED, 0x99, /* 0x04-0x07 */ + 0xED, 0x9A, 0xED, 0x9B, 0xED, 0x9C, 0xED, 0x9D, /* 0x08-0x0B */ + 0xED, 0x9E, 0xED, 0x9F, 0xED, 0xA0, 0xEE, 0x40, /* 0x0C-0x0F */ + 0xEE, 0x41, 0xEE, 0x42, 0xEE, 0x43, 0xEE, 0x44, /* 0x10-0x13 */ + 0xEE, 0x45, 0xEE, 0x46, 0xEE, 0x47, 0xEE, 0x48, /* 0x14-0x17 */ + 0xEE, 0x49, 0xEE, 0x4A, 0xEE, 0x4B, 0xEE, 0x4C, /* 0x18-0x1B */ + 0xEE, 0x4D, 0xEE, 0x4E, 0xEE, 0x4F, 0xEE, 0x50, /* 0x1C-0x1F */ + 0xEE, 0x51, 0xEE, 0x52, 0xEE, 0x53, 0xEE, 0x54, /* 0x20-0x23 */ + 0xEE, 0x55, 0xEE, 0x56, 0xEE, 0x57, 0xEE, 0x58, /* 0x24-0x27 */ + 0xEE, 0x59, 0xEE, 0x5A, 0xEE, 0x5B, 0xEE, 0x5C, /* 0x28-0x2B */ + 0xEE, 0x5D, 0xEE, 0x5E, 0xEE, 0x5F, 0xEE, 0x60, /* 0x2C-0x2F */ + 0xEE, 0x61, 0xEE, 0x62, 0xEE, 0x63, 0xEE, 0x64, /* 0x30-0x33 */ + 0xEE, 0x65, 0xEE, 0x66, 0xEE, 0x67, 0xEE, 0x68, /* 0x34-0x37 */ + 0xEE, 0x69, 0xEE, 0x6A, 0xEE, 0x6B, 0xEE, 0x6C, /* 0x38-0x3B */ + 0xEE, 0x6D, 0xEE, 0x6E, 0xEE, 0x6F, 0xEE, 0x70, /* 0x3C-0x3F */ + 0xEE, 0x71, 0xEE, 0x72, 0xEE, 0x73, 0xEE, 0x74, /* 0x40-0x43 */ + 0xEE, 0x75, 0xEE, 0x76, 0xEE, 0x77, 0xEE, 0x78, /* 0x44-0x47 */ + 0xEE, 0x79, 0xEE, 0x7A, 0xEE, 0x7B, 0xEE, 0x7C, /* 0x48-0x4B */ + 0xEE, 0x7D, 0xEE, 0x7E, 0xEE, 0x80, 0xEE, 0x81, /* 0x4C-0x4F */ + 0xEE, 0x82, 0xEE, 0x83, 0xEE, 0x84, 0xEE, 0x85, /* 0x50-0x53 */ + 0xEE, 0x86, 0xEE, 0x87, 0xEE, 0x88, 0xEE, 0x89, /* 0x54-0x57 */ + 0xEE, 0x8A, 0xEE, 0x8B, 0xEE, 0x8C, 0xEE, 0x8D, /* 0x58-0x5B */ + 0xEE, 0x8E, 0xEE, 0x8F, 0xEE, 0x90, 0xEE, 0x91, /* 0x5C-0x5F */ + 0xEE, 0x92, 0xEE, 0x93, 0xEE, 0x94, 0xEE, 0x95, /* 0x60-0x63 */ + 0xEE, 0x96, 0xEE, 0x97, 0xEE, 0x98, 0xEE, 0x99, /* 0x64-0x67 */ + 0xEE, 0x9A, 0xEE, 0x9B, 0xEE, 0x9C, 0xEE, 0x9D, /* 0x68-0x6B */ + 0xEE, 0x9E, 0xEE, 0x9F, 0xEE, 0xA0, 0xEF, 0x40, /* 0x6C-0x6F */ + 0xEF, 0x41, 0xEF, 0x42, 0xEF, 0x43, 0xEF, 0x44, /* 0x70-0x73 */ + 0xEF, 0x45, 0xD2, 0xB3, 0xB6, 0xA5, 0xC7, 0xEA, /* 0x74-0x77 */ + 0xF1, 0xFC, 0xCF, 0xEE, 0xCB, 0xB3, 0xD0, 0xEB, /* 0x78-0x7B */ + 0xE7, 0xEF, 0xCD, 0xE7, 0xB9, 0xCB, 0xB6, 0xD9, /* 0x7C-0x7F */ + + 0xF1, 0xFD, 0xB0, 0xE4, 0xCB, 0xCC, 0xF1, 0xFE, /* 0x80-0x83 */ + 0xD4, 0xA4, 0xC2, 0xAD, 0xC1, 0xEC, 0xC6, 0xC4, /* 0x84-0x87 */ + 0xBE, 0xB1, 0xF2, 0xA1, 0xBC, 0xD5, 0xEF, 0x46, /* 0x88-0x8B */ + 0xF2, 0xA2, 0xF2, 0xA3, 0xEF, 0x47, 0xF2, 0xA4, /* 0x8C-0x8F */ + 0xD2, 0xC3, 0xC6, 0xB5, 0xEF, 0x48, 0xCD, 0xC7, /* 0x90-0x93 */ + 0xF2, 0xA5, 0xEF, 0x49, 0xD3, 0xB1, 0xBF, 0xC5, /* 0x94-0x97 */ + 0xCC, 0xE2, 0xEF, 0x4A, 0xF2, 0xA6, 0xF2, 0xA7, /* 0x98-0x9B */ + 0xD1, 0xD5, 0xB6, 0xEE, 0xF2, 0xA8, 0xF2, 0xA9, /* 0x9C-0x9F */ + 0xB5, 0xDF, 0xF2, 0xAA, 0xF2, 0xAB, 0xEF, 0x4B, /* 0xA0-0xA3 */ + 0xB2, 0xFC, 0xF2, 0xAC, 0xF2, 0xAD, 0xC8, 0xA7, /* 0xA4-0xA7 */ + 0xEF, 0x4C, 0xEF, 0x4D, 0xEF, 0x4E, 0xEF, 0x4F, /* 0xA8-0xAB */ + 0xEF, 0x50, 0xEF, 0x51, 0xEF, 0x52, 0xEF, 0x53, /* 0xAC-0xAF */ + 0xEF, 0x54, 0xEF, 0x55, 0xEF, 0x56, 0xEF, 0x57, /* 0xB0-0xB3 */ + 0xEF, 0x58, 0xEF, 0x59, 0xEF, 0x5A, 0xEF, 0x5B, /* 0xB4-0xB7 */ + 0xEF, 0x5C, 0xEF, 0x5D, 0xEF, 0x5E, 0xEF, 0x5F, /* 0xB8-0xBB */ + 0xEF, 0x60, 0xEF, 0x61, 0xEF, 0x62, 0xEF, 0x63, /* 0xBC-0xBF */ + 0xEF, 0x64, 0xEF, 0x65, 0xEF, 0x66, 0xEF, 0x67, /* 0xC0-0xC3 */ + 0xEF, 0x68, 0xEF, 0x69, 0xEF, 0x6A, 0xEF, 0x6B, /* 0xC4-0xC7 */ + 0xEF, 0x6C, 0xEF, 0x6D, 0xEF, 0x6E, 0xEF, 0x6F, /* 0xC8-0xCB */ + 0xEF, 0x70, 0xEF, 0x71, 0xB7, 0xE7, 0xEF, 0x72, /* 0xCC-0xCF */ + 0xEF, 0x73, 0xEC, 0xA9, 0xEC, 0xAA, 0xEC, 0xAB, /* 0xD0-0xD3 */ + 0xEF, 0x74, 0xEC, 0xAC, 0xEF, 0x75, 0xEF, 0x76, /* 0xD4-0xD7 */ + 0xC6, 0xAE, 0xEC, 0xAD, 0xEC, 0xAE, 0xEF, 0x77, /* 0xD8-0xDB */ + 0xEF, 0x78, 0xEF, 0x79, 0xB7, 0xC9, 0xCA, 0xB3, /* 0xDC-0xDF */ + 0xEF, 0x7A, 0xEF, 0x7B, 0xEF, 0x7C, 0xEF, 0x7D, /* 0xE0-0xE3 */ + 0xEF, 0x7E, 0xEF, 0x80, 0xEF, 0x81, 0xE2, 0xB8, /* 0xE4-0xE7 */ + 0xF7, 0xCF, 0xEF, 0x82, 0xEF, 0x83, 0xEF, 0x84, /* 0xE8-0xEB */ + 0xEF, 0x85, 0xEF, 0x86, 0xEF, 0x87, 0xEF, 0x88, /* 0xEC-0xEF */ + 0xEF, 0x89, 0xEF, 0x8A, 0xEF, 0x8B, 0xEF, 0x8C, /* 0xF0-0xF3 */ + 0xEF, 0x8D, 0xEF, 0x8E, 0xEF, 0x8F, 0xEF, 0x90, /* 0xF4-0xF7 */ + 0xEF, 0x91, 0xEF, 0x92, 0xEF, 0x93, 0xEF, 0x94, /* 0xF8-0xFB */ + 0xEF, 0x95, 0xEF, 0x96, 0xEF, 0x97, 0xEF, 0x98, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_99[512] = { + 0xEF, 0x99, 0xEF, 0x9A, 0xEF, 0x9B, 0xEF, 0x9C, /* 0x00-0x03 */ + 0xEF, 0x9D, 0xEF, 0x9E, 0xEF, 0x9F, 0xEF, 0xA0, /* 0x04-0x07 */ + 0xF0, 0x40, 0xF0, 0x41, 0xF0, 0x42, 0xF0, 0x43, /* 0x08-0x0B */ + 0xF0, 0x44, 0xF7, 0xD0, 0xF0, 0x45, 0xF0, 0x46, /* 0x0C-0x0F */ + 0xB2, 0xCD, 0xF0, 0x47, 0xF0, 0x48, 0xF0, 0x49, /* 0x10-0x13 */ + 0xF0, 0x4A, 0xF0, 0x4B, 0xF0, 0x4C, 0xF0, 0x4D, /* 0x14-0x17 */ + 0xF0, 0x4E, 0xF0, 0x4F, 0xF0, 0x50, 0xF0, 0x51, /* 0x18-0x1B */ + 0xF0, 0x52, 0xF0, 0x53, 0xF0, 0x54, 0xF0, 0x55, /* 0x1C-0x1F */ + 0xF0, 0x56, 0xF0, 0x57, 0xF0, 0x58, 0xF0, 0x59, /* 0x20-0x23 */ + 0xF0, 0x5A, 0xF0, 0x5B, 0xF0, 0x5C, 0xF0, 0x5D, /* 0x24-0x27 */ + 0xF0, 0x5E, 0xF0, 0x5F, 0xF0, 0x60, 0xF0, 0x61, /* 0x28-0x2B */ + 0xF0, 0x62, 0xF0, 0x63, 0xF7, 0xD1, 0xF0, 0x64, /* 0x2C-0x2F */ + 0xF0, 0x65, 0xF0, 0x66, 0xF0, 0x67, 0xF0, 0x68, /* 0x30-0x33 */ + 0xF0, 0x69, 0xF0, 0x6A, 0xF0, 0x6B, 0xF0, 0x6C, /* 0x34-0x37 */ + 0xF0, 0x6D, 0xF0, 0x6E, 0xF0, 0x6F, 0xF0, 0x70, /* 0x38-0x3B */ + 0xF0, 0x71, 0xF0, 0x72, 0xF0, 0x73, 0xF0, 0x74, /* 0x3C-0x3F */ + 0xF0, 0x75, 0xF0, 0x76, 0xF0, 0x77, 0xF0, 0x78, /* 0x40-0x43 */ + 0xF0, 0x79, 0xF0, 0x7A, 0xF0, 0x7B, 0xF0, 0x7C, /* 0x44-0x47 */ + 0xF0, 0x7D, 0xF0, 0x7E, 0xF0, 0x80, 0xF0, 0x81, /* 0x48-0x4B */ + 0xF0, 0x82, 0xF0, 0x83, 0xF0, 0x84, 0xF0, 0x85, /* 0x4C-0x4F */ + 0xF0, 0x86, 0xF0, 0x87, 0xF0, 0x88, 0xF0, 0x89, /* 0x50-0x53 */ + 0xF7, 0xD3, 0xF7, 0xD2, 0xF0, 0x8A, 0xF0, 0x8B, /* 0x54-0x57 */ + 0xF0, 0x8C, 0xF0, 0x8D, 0xF0, 0x8E, 0xF0, 0x8F, /* 0x58-0x5B */ + 0xF0, 0x90, 0xF0, 0x91, 0xF0, 0x92, 0xF0, 0x93, /* 0x5C-0x5F */ + 0xF0, 0x94, 0xF0, 0x95, 0xF0, 0x96, 0xE2, 0xBB, /* 0x60-0x63 */ + 0xF0, 0x97, 0xBC, 0xA2, 0xF0, 0x98, 0xE2, 0xBC, /* 0x64-0x67 */ + 0xE2, 0xBD, 0xE2, 0xBE, 0xE2, 0xBF, 0xE2, 0xC0, /* 0x68-0x6B */ + 0xE2, 0xC1, 0xB7, 0xB9, 0xD2, 0xFB, 0xBD, 0xA4, /* 0x6C-0x6F */ + 0xCA, 0xCE, 0xB1, 0xA5, 0xCB, 0xC7, 0xF0, 0x99, /* 0x70-0x73 */ + 0xE2, 0xC2, 0xB6, 0xFC, 0xC8, 0xC4, 0xE2, 0xC3, /* 0x74-0x77 */ + 0xF0, 0x9A, 0xF0, 0x9B, 0xBD, 0xC8, 0xF0, 0x9C, /* 0x78-0x7B */ + 0xB1, 0xFD, 0xE2, 0xC4, 0xF0, 0x9D, 0xB6, 0xF6, /* 0x7C-0x7F */ + + 0xE2, 0xC5, 0xC4, 0xD9, 0xF0, 0x9E, 0xF0, 0x9F, /* 0x80-0x83 */ + 0xE2, 0xC6, 0xCF, 0xDA, 0xB9, 0xDD, 0xE2, 0xC7, /* 0x84-0x87 */ + 0xC0, 0xA1, 0xF0, 0xA0, 0xE2, 0xC8, 0xB2, 0xF6, /* 0x88-0x8B */ + 0xF1, 0x40, 0xE2, 0xC9, 0xF1, 0x41, 0xC1, 0xF3, /* 0x8C-0x8F */ + 0xE2, 0xCA, 0xE2, 0xCB, 0xC2, 0xF8, 0xE2, 0xCC, /* 0x90-0x93 */ + 0xE2, 0xCD, 0xE2, 0xCE, 0xCA, 0xD7, 0xD8, 0xB8, /* 0x94-0x97 */ + 0xD9, 0xE5, 0xCF, 0xE3, 0xF1, 0x42, 0xF1, 0x43, /* 0x98-0x9B */ + 0xF1, 0x44, 0xF1, 0x45, 0xF1, 0x46, 0xF1, 0x47, /* 0x9C-0x9F */ + 0xF1, 0x48, 0xF1, 0x49, 0xF1, 0x4A, 0xF1, 0x4B, /* 0xA0-0xA3 */ + 0xF1, 0x4C, 0xF0, 0xA5, 0xF1, 0x4D, 0xF1, 0x4E, /* 0xA4-0xA7 */ + 0xDC, 0xB0, 0xF1, 0x4F, 0xF1, 0x50, 0xF1, 0x51, /* 0xA8-0xAB */ + 0xF1, 0x52, 0xF1, 0x53, 0xF1, 0x54, 0xF1, 0x55, /* 0xAC-0xAF */ + 0xF1, 0x56, 0xF1, 0x57, 0xF1, 0x58, 0xF1, 0x59, /* 0xB0-0xB3 */ + 0xF1, 0x5A, 0xF1, 0x5B, 0xF1, 0x5C, 0xF1, 0x5D, /* 0xB4-0xB7 */ + 0xF1, 0x5E, 0xF1, 0x5F, 0xF1, 0x60, 0xF1, 0x61, /* 0xB8-0xBB */ + 0xF1, 0x62, 0xF1, 0x63, 0xF1, 0x64, 0xF1, 0x65, /* 0xBC-0xBF */ + 0xF1, 0x66, 0xF1, 0x67, 0xF1, 0x68, 0xF1, 0x69, /* 0xC0-0xC3 */ + 0xF1, 0x6A, 0xF1, 0x6B, 0xF1, 0x6C, 0xF1, 0x6D, /* 0xC4-0xC7 */ + 0xF1, 0x6E, 0xF1, 0x6F, 0xF1, 0x70, 0xF1, 0x71, /* 0xC8-0xCB */ + 0xF1, 0x72, 0xF1, 0x73, 0xF1, 0x74, 0xF1, 0x75, /* 0xCC-0xCF */ + 0xF1, 0x76, 0xF1, 0x77, 0xF1, 0x78, 0xF1, 0x79, /* 0xD0-0xD3 */ + 0xF1, 0x7A, 0xF1, 0x7B, 0xF1, 0x7C, 0xF1, 0x7D, /* 0xD4-0xD7 */ + 0xF1, 0x7E, 0xF1, 0x80, 0xF1, 0x81, 0xF1, 0x82, /* 0xD8-0xDB */ + 0xF1, 0x83, 0xF1, 0x84, 0xF1, 0x85, 0xF1, 0x86, /* 0xDC-0xDF */ + 0xF1, 0x87, 0xF1, 0x88, 0xF1, 0x89, 0xF1, 0x8A, /* 0xE0-0xE3 */ + 0xF1, 0x8B, 0xF1, 0x8C, 0xF1, 0x8D, 0xF1, 0x8E, /* 0xE4-0xE7 */ + 0xF1, 0x8F, 0xF1, 0x90, 0xF1, 0x91, 0xF1, 0x92, /* 0xE8-0xEB */ + 0xF1, 0x93, 0xF1, 0x94, 0xF1, 0x95, 0xF1, 0x96, /* 0xEC-0xEF */ + 0xF1, 0x97, 0xF1, 0x98, 0xF1, 0x99, 0xF1, 0x9A, /* 0xF0-0xF3 */ + 0xF1, 0x9B, 0xF1, 0x9C, 0xF1, 0x9D, 0xF1, 0x9E, /* 0xF4-0xF7 */ + 0xF1, 0x9F, 0xF1, 0xA0, 0xF2, 0x40, 0xF2, 0x41, /* 0xF8-0xFB */ + 0xF2, 0x42, 0xF2, 0x43, 0xF2, 0x44, 0xF2, 0x45, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_9A[512] = { + 0xF2, 0x46, 0xF2, 0x47, 0xF2, 0x48, 0xF2, 0x49, /* 0x00-0x03 */ + 0xF2, 0x4A, 0xF2, 0x4B, 0xF2, 0x4C, 0xF2, 0x4D, /* 0x04-0x07 */ + 0xF2, 0x4E, 0xF2, 0x4F, 0xF2, 0x50, 0xF2, 0x51, /* 0x08-0x0B */ + 0xF2, 0x52, 0xF2, 0x53, 0xF2, 0x54, 0xF2, 0x55, /* 0x0C-0x0F */ + 0xF2, 0x56, 0xF2, 0x57, 0xF2, 0x58, 0xF2, 0x59, /* 0x10-0x13 */ + 0xF2, 0x5A, 0xF2, 0x5B, 0xF2, 0x5C, 0xF2, 0x5D, /* 0x14-0x17 */ + 0xF2, 0x5E, 0xF2, 0x5F, 0xF2, 0x60, 0xF2, 0x61, /* 0x18-0x1B */ + 0xF2, 0x62, 0xF2, 0x63, 0xF2, 0x64, 0xF2, 0x65, /* 0x1C-0x1F */ + 0xF2, 0x66, 0xF2, 0x67, 0xF2, 0x68, 0xF2, 0x69, /* 0x20-0x23 */ + 0xF2, 0x6A, 0xF2, 0x6B, 0xF2, 0x6C, 0xF2, 0x6D, /* 0x24-0x27 */ + 0xF2, 0x6E, 0xF2, 0x6F, 0xF2, 0x70, 0xF2, 0x71, /* 0x28-0x2B */ + 0xF2, 0x72, 0xF2, 0x73, 0xF2, 0x74, 0xF2, 0x75, /* 0x2C-0x2F */ + 0xF2, 0x76, 0xF2, 0x77, 0xF2, 0x78, 0xF2, 0x79, /* 0x30-0x33 */ + 0xF2, 0x7A, 0xF2, 0x7B, 0xF2, 0x7C, 0xF2, 0x7D, /* 0x34-0x37 */ + 0xF2, 0x7E, 0xF2, 0x80, 0xF2, 0x81, 0xF2, 0x82, /* 0x38-0x3B */ + 0xF2, 0x83, 0xF2, 0x84, 0xF2, 0x85, 0xF2, 0x86, /* 0x3C-0x3F */ + 0xF2, 0x87, 0xF2, 0x88, 0xF2, 0x89, 0xF2, 0x8A, /* 0x40-0x43 */ + 0xF2, 0x8B, 0xF2, 0x8C, 0xF2, 0x8D, 0xF2, 0x8E, /* 0x44-0x47 */ + 0xF2, 0x8F, 0xF2, 0x90, 0xF2, 0x91, 0xF2, 0x92, /* 0x48-0x4B */ + 0xF2, 0x93, 0xF2, 0x94, 0xF2, 0x95, 0xF2, 0x96, /* 0x4C-0x4F */ + 0xF2, 0x97, 0xF2, 0x98, 0xF2, 0x99, 0xF2, 0x9A, /* 0x50-0x53 */ + 0xF2, 0x9B, 0xF2, 0x9C, 0xF2, 0x9D, 0xF2, 0x9E, /* 0x54-0x57 */ + 0xF2, 0x9F, 0xF2, 0xA0, 0xF3, 0x40, 0xF3, 0x41, /* 0x58-0x5B */ + 0xF3, 0x42, 0xF3, 0x43, 0xF3, 0x44, 0xF3, 0x45, /* 0x5C-0x5F */ + 0xF3, 0x46, 0xF3, 0x47, 0xF3, 0x48, 0xF3, 0x49, /* 0x60-0x63 */ + 0xF3, 0x4A, 0xF3, 0x4B, 0xF3, 0x4C, 0xF3, 0x4D, /* 0x64-0x67 */ + 0xF3, 0x4E, 0xF3, 0x4F, 0xF3, 0x50, 0xF3, 0x51, /* 0x68-0x6B */ + 0xC2, 0xED, 0xD4, 0xA6, 0xCD, 0xD4, 0xD1, 0xB1, /* 0x6C-0x6F */ + 0xB3, 0xDB, 0xC7, 0xFD, 0xF3, 0x52, 0xB2, 0xB5, /* 0x70-0x73 */ + 0xC2, 0xBF, 0xE6, 0xE0, 0xCA, 0xBB, 0xE6, 0xE1, /* 0x74-0x77 */ + 0xE6, 0xE2, 0xBE, 0xD4, 0xE6, 0xE3, 0xD7, 0xA4, /* 0x78-0x7B */ + 0xCD, 0xD5, 0xE6, 0xE5, 0xBC, 0xDD, 0xE6, 0xE4, /* 0x7C-0x7F */ + + 0xE6, 0xE6, 0xE6, 0xE7, 0xC2, 0xEE, 0xF3, 0x53, /* 0x80-0x83 */ + 0xBD, 0xBE, 0xE6, 0xE8, 0xC2, 0xE6, 0xBA, 0xA7, /* 0x84-0x87 */ + 0xE6, 0xE9, 0xF3, 0x54, 0xE6, 0xEA, 0xB3, 0xD2, /* 0x88-0x8B */ + 0xD1, 0xE9, 0xF3, 0x55, 0xF3, 0x56, 0xBF, 0xA5, /* 0x8C-0x8F */ + 0xE6, 0xEB, 0xC6, 0xEF, 0xE6, 0xEC, 0xE6, 0xED, /* 0x90-0x93 */ + 0xF3, 0x57, 0xF3, 0x58, 0xE6, 0xEE, 0xC6, 0xAD, /* 0x94-0x97 */ + 0xE6, 0xEF, 0xF3, 0x59, 0xC9, 0xA7, 0xE6, 0xF0, /* 0x98-0x9B */ + 0xE6, 0xF1, 0xE6, 0xF2, 0xE5, 0xB9, 0xE6, 0xF3, /* 0x9C-0x9F */ + 0xE6, 0xF4, 0xC2, 0xE2, 0xE6, 0xF5, 0xE6, 0xF6, /* 0xA0-0xA3 */ + 0xD6, 0xE8, 0xE6, 0xF7, 0xF3, 0x5A, 0xE6, 0xF8, /* 0xA4-0xA7 */ + 0xB9, 0xC7, 0xF3, 0x5B, 0xF3, 0x5C, 0xF3, 0x5D, /* 0xA8-0xAB */ + 0xF3, 0x5E, 0xF3, 0x5F, 0xF3, 0x60, 0xF3, 0x61, /* 0xAC-0xAF */ + 0xF7, 0xBB, 0xF7, 0xBA, 0xF3, 0x62, 0xF3, 0x63, /* 0xB0-0xB3 */ + 0xF3, 0x64, 0xF3, 0x65, 0xF7, 0xBE, 0xF7, 0xBC, /* 0xB4-0xB7 */ + 0xBA, 0xA1, 0xF3, 0x66, 0xF7, 0xBF, 0xF3, 0x67, /* 0xB8-0xBB */ + 0xF7, 0xC0, 0xF3, 0x68, 0xF3, 0x69, 0xF3, 0x6A, /* 0xBC-0xBF */ + 0xF7, 0xC2, 0xF7, 0xC1, 0xF7, 0xC4, 0xF3, 0x6B, /* 0xC0-0xC3 */ + 0xF3, 0x6C, 0xF7, 0xC3, 0xF3, 0x6D, 0xF3, 0x6E, /* 0xC4-0xC7 */ + 0xF3, 0x6F, 0xF3, 0x70, 0xF3, 0x71, 0xF7, 0xC5, /* 0xC8-0xCB */ + 0xF7, 0xC6, 0xF3, 0x72, 0xF3, 0x73, 0xF3, 0x74, /* 0xCC-0xCF */ + 0xF3, 0x75, 0xF7, 0xC7, 0xF3, 0x76, 0xCB, 0xE8, /* 0xD0-0xD3 */ + 0xF3, 0x77, 0xF3, 0x78, 0xF3, 0x79, 0xF3, 0x7A, /* 0xD4-0xD7 */ + 0xB8, 0xDF, 0xF3, 0x7B, 0xF3, 0x7C, 0xF3, 0x7D, /* 0xD8-0xDB */ + 0xF3, 0x7E, 0xF3, 0x80, 0xF3, 0x81, 0xF7, 0xD4, /* 0xDC-0xDF */ + 0xF3, 0x82, 0xF7, 0xD5, 0xF3, 0x83, 0xF3, 0x84, /* 0xE0-0xE3 */ + 0xF3, 0x85, 0xF3, 0x86, 0xF7, 0xD6, 0xF3, 0x87, /* 0xE4-0xE7 */ + 0xF3, 0x88, 0xF3, 0x89, 0xF3, 0x8A, 0xF7, 0xD8, /* 0xE8-0xEB */ + 0xF3, 0x8B, 0xF7, 0xDA, 0xF3, 0x8C, 0xF7, 0xD7, /* 0xEC-0xEF */ + 0xF3, 0x8D, 0xF3, 0x8E, 0xF3, 0x8F, 0xF3, 0x90, /* 0xF0-0xF3 */ + 0xF3, 0x91, 0xF3, 0x92, 0xF3, 0x93, 0xF3, 0x94, /* 0xF4-0xF7 */ + 0xF3, 0x95, 0xF7, 0xDB, 0xF3, 0x96, 0xF7, 0xD9, /* 0xF8-0xFB */ + 0xF3, 0x97, 0xF3, 0x98, 0xF3, 0x99, 0xF3, 0x9A, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_9B[512] = { + 0xF3, 0x9B, 0xF3, 0x9C, 0xF3, 0x9D, 0xD7, 0xD7, /* 0x00-0x03 */ + 0xF3, 0x9E, 0xF3, 0x9F, 0xF3, 0xA0, 0xF4, 0x40, /* 0x04-0x07 */ + 0xF7, 0xDC, 0xF4, 0x41, 0xF4, 0x42, 0xF4, 0x43, /* 0x08-0x0B */ + 0xF4, 0x44, 0xF4, 0x45, 0xF4, 0x46, 0xF7, 0xDD, /* 0x0C-0x0F */ + 0xF4, 0x47, 0xF4, 0x48, 0xF4, 0x49, 0xF7, 0xDE, /* 0x10-0x13 */ + 0xF4, 0x4A, 0xF4, 0x4B, 0xF4, 0x4C, 0xF4, 0x4D, /* 0x14-0x17 */ + 0xF4, 0x4E, 0xF4, 0x4F, 0xF4, 0x50, 0xF4, 0x51, /* 0x18-0x1B */ + 0xF4, 0x52, 0xF4, 0x53, 0xF4, 0x54, 0xF7, 0xDF, /* 0x1C-0x1F */ + 0xF4, 0x55, 0xF4, 0x56, 0xF4, 0x57, 0xF7, 0xE0, /* 0x20-0x23 */ + 0xF4, 0x58, 0xF4, 0x59, 0xF4, 0x5A, 0xF4, 0x5B, /* 0x24-0x27 */ + 0xF4, 0x5C, 0xF4, 0x5D, 0xF4, 0x5E, 0xF4, 0x5F, /* 0x28-0x2B */ + 0xF4, 0x60, 0xF4, 0x61, 0xF4, 0x62, 0xDB, 0xCB, /* 0x2C-0x2F */ + 0xF4, 0x63, 0xF4, 0x64, 0xD8, 0xAA, 0xF4, 0x65, /* 0x30-0x33 */ + 0xF4, 0x66, 0xF4, 0x67, 0xF4, 0x68, 0xF4, 0x69, /* 0x34-0x37 */ + 0xF4, 0x6A, 0xF4, 0x6B, 0xF4, 0x6C, 0xE5, 0xF7, /* 0x38-0x3B */ + 0xB9, 0xED, 0xF4, 0x6D, 0xF4, 0x6E, 0xF4, 0x6F, /* 0x3C-0x3F */ + 0xF4, 0x70, 0xBF, 0xFD, 0xBB, 0xEA, 0xF7, 0xC9, /* 0x40-0x43 */ + 0xC6, 0xC7, 0xF7, 0xC8, 0xF4, 0x71, 0xF7, 0xCA, /* 0x44-0x47 */ + 0xF7, 0xCC, 0xF7, 0xCB, 0xF4, 0x72, 0xF4, 0x73, /* 0x48-0x4B */ + 0xF4, 0x74, 0xF7, 0xCD, 0xF4, 0x75, 0xCE, 0xBA, /* 0x4C-0x4F */ + 0xF4, 0x76, 0xF7, 0xCE, 0xF4, 0x77, 0xF4, 0x78, /* 0x50-0x53 */ + 0xC4, 0xA7, 0xF4, 0x79, 0xF4, 0x7A, 0xF4, 0x7B, /* 0x54-0x57 */ + 0xF4, 0x7C, 0xF4, 0x7D, 0xF4, 0x7E, 0xF4, 0x80, /* 0x58-0x5B */ + 0xF4, 0x81, 0xF4, 0x82, 0xF4, 0x83, 0xF4, 0x84, /* 0x5C-0x5F */ + 0xF4, 0x85, 0xF4, 0x86, 0xF4, 0x87, 0xF4, 0x88, /* 0x60-0x63 */ + 0xF4, 0x89, 0xF4, 0x8A, 0xF4, 0x8B, 0xF4, 0x8C, /* 0x64-0x67 */ + 0xF4, 0x8D, 0xF4, 0x8E, 0xF4, 0x8F, 0xF4, 0x90, /* 0x68-0x6B */ + 0xF4, 0x91, 0xF4, 0x92, 0xF4, 0x93, 0xF4, 0x94, /* 0x6C-0x6F */ + 0xF4, 0x95, 0xF4, 0x96, 0xF4, 0x97, 0xF4, 0x98, /* 0x70-0x73 */ + 0xF4, 0x99, 0xF4, 0x9A, 0xF4, 0x9B, 0xF4, 0x9C, /* 0x74-0x77 */ + 0xF4, 0x9D, 0xF4, 0x9E, 0xF4, 0x9F, 0xF4, 0xA0, /* 0x78-0x7B */ + 0xF5, 0x40, 0xF5, 0x41, 0xF5, 0x42, 0xF5, 0x43, /* 0x7C-0x7F */ + + 0xF5, 0x44, 0xF5, 0x45, 0xF5, 0x46, 0xF5, 0x47, /* 0x80-0x83 */ + 0xF5, 0x48, 0xF5, 0x49, 0xF5, 0x4A, 0xF5, 0x4B, /* 0x84-0x87 */ + 0xF5, 0x4C, 0xF5, 0x4D, 0xF5, 0x4E, 0xF5, 0x4F, /* 0x88-0x8B */ + 0xF5, 0x50, 0xF5, 0x51, 0xF5, 0x52, 0xF5, 0x53, /* 0x8C-0x8F */ + 0xF5, 0x54, 0xF5, 0x55, 0xF5, 0x56, 0xF5, 0x57, /* 0x90-0x93 */ + 0xF5, 0x58, 0xF5, 0x59, 0xF5, 0x5A, 0xF5, 0x5B, /* 0x94-0x97 */ + 0xF5, 0x5C, 0xF5, 0x5D, 0xF5, 0x5E, 0xF5, 0x5F, /* 0x98-0x9B */ + 0xF5, 0x60, 0xF5, 0x61, 0xF5, 0x62, 0xF5, 0x63, /* 0x9C-0x9F */ + 0xF5, 0x64, 0xF5, 0x65, 0xF5, 0x66, 0xF5, 0x67, /* 0xA0-0xA3 */ + 0xF5, 0x68, 0xF5, 0x69, 0xF5, 0x6A, 0xF5, 0x6B, /* 0xA4-0xA7 */ + 0xF5, 0x6C, 0xF5, 0x6D, 0xF5, 0x6E, 0xF5, 0x6F, /* 0xA8-0xAB */ + 0xF5, 0x70, 0xF5, 0x71, 0xF5, 0x72, 0xF5, 0x73, /* 0xAC-0xAF */ + 0xF5, 0x74, 0xF5, 0x75, 0xF5, 0x76, 0xF5, 0x77, /* 0xB0-0xB3 */ + 0xF5, 0x78, 0xF5, 0x79, 0xF5, 0x7A, 0xF5, 0x7B, /* 0xB4-0xB7 */ + 0xF5, 0x7C, 0xF5, 0x7D, 0xF5, 0x7E, 0xF5, 0x80, /* 0xB8-0xBB */ + 0xF5, 0x81, 0xF5, 0x82, 0xF5, 0x83, 0xF5, 0x84, /* 0xBC-0xBF */ + 0xF5, 0x85, 0xF5, 0x86, 0xF5, 0x87, 0xF5, 0x88, /* 0xC0-0xC3 */ + 0xF5, 0x89, 0xF5, 0x8A, 0xF5, 0x8B, 0xF5, 0x8C, /* 0xC4-0xC7 */ + 0xF5, 0x8D, 0xF5, 0x8E, 0xF5, 0x8F, 0xF5, 0x90, /* 0xC8-0xCB */ + 0xF5, 0x91, 0xF5, 0x92, 0xF5, 0x93, 0xF5, 0x94, /* 0xCC-0xCF */ + 0xF5, 0x95, 0xF5, 0x96, 0xF5, 0x97, 0xF5, 0x98, /* 0xD0-0xD3 */ + 0xF5, 0x99, 0xF5, 0x9A, 0xF5, 0x9B, 0xF5, 0x9C, /* 0xD4-0xD7 */ + 0xF5, 0x9D, 0xF5, 0x9E, 0xF5, 0x9F, 0xF5, 0xA0, /* 0xD8-0xDB */ + 0xF6, 0x40, 0xF6, 0x41, 0xF6, 0x42, 0xF6, 0x43, /* 0xDC-0xDF */ + 0xF6, 0x44, 0xF6, 0x45, 0xF6, 0x46, 0xF6, 0x47, /* 0xE0-0xE3 */ + 0xF6, 0x48, 0xF6, 0x49, 0xF6, 0x4A, 0xF6, 0x4B, /* 0xE4-0xE7 */ + 0xF6, 0x4C, 0xF6, 0x4D, 0xF6, 0x4E, 0xF6, 0x4F, /* 0xE8-0xEB */ + 0xF6, 0x50, 0xF6, 0x51, 0xF6, 0x52, 0xF6, 0x53, /* 0xEC-0xEF */ + 0xF6, 0x54, 0xF6, 0x55, 0xF6, 0x56, 0xF6, 0x57, /* 0xF0-0xF3 */ + 0xF6, 0x58, 0xF6, 0x59, 0xF6, 0x5A, 0xF6, 0x5B, /* 0xF4-0xF7 */ + 0xF6, 0x5C, 0xF6, 0x5D, 0xF6, 0x5E, 0xF6, 0x5F, /* 0xF8-0xFB */ + 0xF6, 0x60, 0xF6, 0x61, 0xF6, 0x62, 0xF6, 0x63, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_9C[512] = { + 0xF6, 0x64, 0xF6, 0x65, 0xF6, 0x66, 0xF6, 0x67, /* 0x00-0x03 */ + 0xF6, 0x68, 0xF6, 0x69, 0xF6, 0x6A, 0xF6, 0x6B, /* 0x04-0x07 */ + 0xF6, 0x6C, 0xF6, 0x6D, 0xF6, 0x6E, 0xF6, 0x6F, /* 0x08-0x0B */ + 0xF6, 0x70, 0xF6, 0x71, 0xF6, 0x72, 0xF6, 0x73, /* 0x0C-0x0F */ + 0xF6, 0x74, 0xF6, 0x75, 0xF6, 0x76, 0xF6, 0x77, /* 0x10-0x13 */ + 0xF6, 0x78, 0xF6, 0x79, 0xF6, 0x7A, 0xF6, 0x7B, /* 0x14-0x17 */ + 0xF6, 0x7C, 0xF6, 0x7D, 0xF6, 0x7E, 0xF6, 0x80, /* 0x18-0x1B */ + 0xF6, 0x81, 0xF6, 0x82, 0xF6, 0x83, 0xF6, 0x84, /* 0x1C-0x1F */ + 0xF6, 0x85, 0xF6, 0x86, 0xF6, 0x87, 0xF6, 0x88, /* 0x20-0x23 */ + 0xF6, 0x89, 0xF6, 0x8A, 0xF6, 0x8B, 0xF6, 0x8C, /* 0x24-0x27 */ + 0xF6, 0x8D, 0xF6, 0x8E, 0xF6, 0x8F, 0xF6, 0x90, /* 0x28-0x2B */ + 0xF6, 0x91, 0xF6, 0x92, 0xF6, 0x93, 0xF6, 0x94, /* 0x2C-0x2F */ + 0xF6, 0x95, 0xF6, 0x96, 0xF6, 0x97, 0xF6, 0x98, /* 0x30-0x33 */ + 0xF6, 0x99, 0xF6, 0x9A, 0xF6, 0x9B, 0xF6, 0x9C, /* 0x34-0x37 */ + 0xF6, 0x9D, 0xF6, 0x9E, 0xF6, 0x9F, 0xF6, 0xA0, /* 0x38-0x3B */ + 0xF7, 0x40, 0xF7, 0x41, 0xF7, 0x42, 0xF7, 0x43, /* 0x3C-0x3F */ + 0xF7, 0x44, 0xF7, 0x45, 0xF7, 0x46, 0xF7, 0x47, /* 0x40-0x43 */ + 0xF7, 0x48, 0xF7, 0x49, 0xF7, 0x4A, 0xF7, 0x4B, /* 0x44-0x47 */ + 0xF7, 0x4C, 0xF7, 0x4D, 0xF7, 0x4E, 0xF7, 0x4F, /* 0x48-0x4B */ + 0xF7, 0x50, 0xF7, 0x51, 0xF7, 0x52, 0xF7, 0x53, /* 0x4C-0x4F */ + 0xF7, 0x54, 0xF7, 0x55, 0xF7, 0x56, 0xF7, 0x57, /* 0x50-0x53 */ + 0xF7, 0x58, 0xF7, 0x59, 0xF7, 0x5A, 0xF7, 0x5B, /* 0x54-0x57 */ + 0xF7, 0x5C, 0xF7, 0x5D, 0xF7, 0x5E, 0xF7, 0x5F, /* 0x58-0x5B */ + 0xF7, 0x60, 0xF7, 0x61, 0xF7, 0x62, 0xF7, 0x63, /* 0x5C-0x5F */ + 0xF7, 0x64, 0xF7, 0x65, 0xF7, 0x66, 0xF7, 0x67, /* 0x60-0x63 */ + 0xF7, 0x68, 0xF7, 0x69, 0xF7, 0x6A, 0xF7, 0x6B, /* 0x64-0x67 */ + 0xF7, 0x6C, 0xF7, 0x6D, 0xF7, 0x6E, 0xF7, 0x6F, /* 0x68-0x6B */ + 0xF7, 0x70, 0xF7, 0x71, 0xF7, 0x72, 0xF7, 0x73, /* 0x6C-0x6F */ + 0xF7, 0x74, 0xF7, 0x75, 0xF7, 0x76, 0xF7, 0x77, /* 0x70-0x73 */ + 0xF7, 0x78, 0xF7, 0x79, 0xF7, 0x7A, 0xF7, 0x7B, /* 0x74-0x77 */ + 0xF7, 0x7C, 0xF7, 0x7D, 0xF7, 0x7E, 0xF7, 0x80, /* 0x78-0x7B */ + 0xD3, 0xE3, 0xF7, 0x81, 0xF7, 0x82, 0xF6, 0xCF, /* 0x7C-0x7F */ + + 0xF7, 0x83, 0xC2, 0xB3, 0xF6, 0xD0, 0xF7, 0x84, /* 0x80-0x83 */ + 0xF7, 0x85, 0xF6, 0xD1, 0xF6, 0xD2, 0xF6, 0xD3, /* 0x84-0x87 */ + 0xF6, 0xD4, 0xF7, 0x86, 0xF7, 0x87, 0xF6, 0xD6, /* 0x88-0x8B */ + 0xF7, 0x88, 0xB1, 0xAB, 0xF6, 0xD7, 0xF7, 0x89, /* 0x8C-0x8F */ + 0xF6, 0xD8, 0xF6, 0xD9, 0xF6, 0xDA, 0xF7, 0x8A, /* 0x90-0x93 */ + 0xF6, 0xDB, 0xF6, 0xDC, 0xF7, 0x8B, 0xF7, 0x8C, /* 0x94-0x97 */ + 0xF7, 0x8D, 0xF7, 0x8E, 0xF6, 0xDD, 0xF6, 0xDE, /* 0x98-0x9B */ + 0xCF, 0xCA, 0xF7, 0x8F, 0xF6, 0xDF, 0xF6, 0xE0, /* 0x9C-0x9F */ + 0xF6, 0xE1, 0xF6, 0xE2, 0xF6, 0xE3, 0xF6, 0xE4, /* 0xA0-0xA3 */ + 0xC0, 0xF0, 0xF6, 0xE5, 0xF6, 0xE6, 0xF6, 0xE7, /* 0xA4-0xA7 */ + 0xF6, 0xE8, 0xF6, 0xE9, 0xF7, 0x90, 0xF6, 0xEA, /* 0xA8-0xAB */ + 0xF7, 0x91, 0xF6, 0xEB, 0xF6, 0xEC, 0xF7, 0x92, /* 0xAC-0xAF */ + 0xF6, 0xED, 0xF6, 0xEE, 0xF6, 0xEF, 0xF6, 0xF0, /* 0xB0-0xB3 */ + 0xF6, 0xF1, 0xF6, 0xF2, 0xF6, 0xF3, 0xF6, 0xF4, /* 0xB4-0xB7 */ + 0xBE, 0xA8, 0xF7, 0x93, 0xF6, 0xF5, 0xF6, 0xF6, /* 0xB8-0xBB */ + 0xF6, 0xF7, 0xF6, 0xF8, 0xF7, 0x94, 0xF7, 0x95, /* 0xBC-0xBF */ + 0xF7, 0x96, 0xF7, 0x97, 0xF7, 0x98, 0xC8, 0xFA, /* 0xC0-0xC3 */ + 0xF6, 0xF9, 0xF6, 0xFA, 0xF6, 0xFB, 0xF6, 0xFC, /* 0xC4-0xC7 */ + 0xF7, 0x99, 0xF7, 0x9A, 0xF6, 0xFD, 0xF6, 0xFE, /* 0xC8-0xCB */ + 0xF7, 0xA1, 0xF7, 0xA2, 0xF7, 0xA3, 0xF7, 0xA4, /* 0xCC-0xCF */ + 0xF7, 0xA5, 0xF7, 0x9B, 0xF7, 0x9C, 0xF7, 0xA6, /* 0xD0-0xD3 */ + 0xF7, 0xA7, 0xF7, 0xA8, 0xB1, 0xEE, 0xF7, 0xA9, /* 0xD4-0xD7 */ + 0xF7, 0xAA, 0xF7, 0xAB, 0xF7, 0x9D, 0xF7, 0x9E, /* 0xD8-0xDB */ + 0xF7, 0xAC, 0xF7, 0xAD, 0xC1, 0xDB, 0xF7, 0xAE, /* 0xDC-0xDF */ + 0xF7, 0x9F, 0xF7, 0xA0, 0xF7, 0xAF, 0xF8, 0x40, /* 0xE0-0xE3 */ + 0xF8, 0x41, 0xF8, 0x42, 0xF8, 0x43, 0xF8, 0x44, /* 0xE4-0xE7 */ + 0xF8, 0x45, 0xF8, 0x46, 0xF8, 0x47, 0xF8, 0x48, /* 0xE8-0xEB */ + 0xF8, 0x49, 0xF8, 0x4A, 0xF8, 0x4B, 0xF8, 0x4C, /* 0xEC-0xEF */ + 0xF8, 0x4D, 0xF8, 0x4E, 0xF8, 0x4F, 0xF8, 0x50, /* 0xF0-0xF3 */ + 0xF8, 0x51, 0xF8, 0x52, 0xF8, 0x53, 0xF8, 0x54, /* 0xF4-0xF7 */ + 0xF8, 0x55, 0xF8, 0x56, 0xF8, 0x57, 0xF8, 0x58, /* 0xF8-0xFB */ + 0xF8, 0x59, 0xF8, 0x5A, 0xF8, 0x5B, 0xF8, 0x5C, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_9D[512] = { + 0xF8, 0x5D, 0xF8, 0x5E, 0xF8, 0x5F, 0xF8, 0x60, /* 0x00-0x03 */ + 0xF8, 0x61, 0xF8, 0x62, 0xF8, 0x63, 0xF8, 0x64, /* 0x04-0x07 */ + 0xF8, 0x65, 0xF8, 0x66, 0xF8, 0x67, 0xF8, 0x68, /* 0x08-0x0B */ + 0xF8, 0x69, 0xF8, 0x6A, 0xF8, 0x6B, 0xF8, 0x6C, /* 0x0C-0x0F */ + 0xF8, 0x6D, 0xF8, 0x6E, 0xF8, 0x6F, 0xF8, 0x70, /* 0x10-0x13 */ + 0xF8, 0x71, 0xF8, 0x72, 0xF8, 0x73, 0xF8, 0x74, /* 0x14-0x17 */ + 0xF8, 0x75, 0xF8, 0x76, 0xF8, 0x77, 0xF8, 0x78, /* 0x18-0x1B */ + 0xF8, 0x79, 0xF8, 0x7A, 0xF8, 0x7B, 0xF8, 0x7C, /* 0x1C-0x1F */ + 0xF8, 0x7D, 0xF8, 0x7E, 0xF8, 0x80, 0xF8, 0x81, /* 0x20-0x23 */ + 0xF8, 0x82, 0xF8, 0x83, 0xF8, 0x84, 0xF8, 0x85, /* 0x24-0x27 */ + 0xF8, 0x86, 0xF8, 0x87, 0xF8, 0x88, 0xF8, 0x89, /* 0x28-0x2B */ + 0xF8, 0x8A, 0xF8, 0x8B, 0xF8, 0x8C, 0xF8, 0x8D, /* 0x2C-0x2F */ + 0xF8, 0x8E, 0xF8, 0x8F, 0xF8, 0x90, 0xF8, 0x91, /* 0x30-0x33 */ + 0xF8, 0x92, 0xF8, 0x93, 0xF8, 0x94, 0xF8, 0x95, /* 0x34-0x37 */ + 0xF8, 0x96, 0xF8, 0x97, 0xF8, 0x98, 0xF8, 0x99, /* 0x38-0x3B */ + 0xF8, 0x9A, 0xF8, 0x9B, 0xF8, 0x9C, 0xF8, 0x9D, /* 0x3C-0x3F */ + 0xF8, 0x9E, 0xF8, 0x9F, 0xF8, 0xA0, 0xF9, 0x40, /* 0x40-0x43 */ + 0xF9, 0x41, 0xF9, 0x42, 0xF9, 0x43, 0xF9, 0x44, /* 0x44-0x47 */ + 0xF9, 0x45, 0xF9, 0x46, 0xF9, 0x47, 0xF9, 0x48, /* 0x48-0x4B */ + 0xF9, 0x49, 0xF9, 0x4A, 0xF9, 0x4B, 0xF9, 0x4C, /* 0x4C-0x4F */ + 0xF9, 0x4D, 0xF9, 0x4E, 0xF9, 0x4F, 0xF9, 0x50, /* 0x50-0x53 */ + 0xF9, 0x51, 0xF9, 0x52, 0xF9, 0x53, 0xF9, 0x54, /* 0x54-0x57 */ + 0xF9, 0x55, 0xF9, 0x56, 0xF9, 0x57, 0xF9, 0x58, /* 0x58-0x5B */ + 0xF9, 0x59, 0xF9, 0x5A, 0xF9, 0x5B, 0xF9, 0x5C, /* 0x5C-0x5F */ + 0xF9, 0x5D, 0xF9, 0x5E, 0xF9, 0x5F, 0xF9, 0x60, /* 0x60-0x63 */ + 0xF9, 0x61, 0xF9, 0x62, 0xF9, 0x63, 0xF9, 0x64, /* 0x64-0x67 */ + 0xF9, 0x65, 0xF9, 0x66, 0xF9, 0x67, 0xF9, 0x68, /* 0x68-0x6B */ + 0xF9, 0x69, 0xF9, 0x6A, 0xF9, 0x6B, 0xF9, 0x6C, /* 0x6C-0x6F */ + 0xF9, 0x6D, 0xF9, 0x6E, 0xF9, 0x6F, 0xF9, 0x70, /* 0x70-0x73 */ + 0xF9, 0x71, 0xF9, 0x72, 0xF9, 0x73, 0xF9, 0x74, /* 0x74-0x77 */ + 0xF9, 0x75, 0xF9, 0x76, 0xF9, 0x77, 0xF9, 0x78, /* 0x78-0x7B */ + 0xF9, 0x79, 0xF9, 0x7A, 0xF9, 0x7B, 0xF9, 0x7C, /* 0x7C-0x7F */ + + 0xF9, 0x7D, 0xF9, 0x7E, 0xF9, 0x80, 0xF9, 0x81, /* 0x80-0x83 */ + 0xF9, 0x82, 0xF9, 0x83, 0xF9, 0x84, 0xF9, 0x85, /* 0x84-0x87 */ + 0xF9, 0x86, 0xF9, 0x87, 0xF9, 0x88, 0xF9, 0x89, /* 0x88-0x8B */ + 0xF9, 0x8A, 0xF9, 0x8B, 0xF9, 0x8C, 0xF9, 0x8D, /* 0x8C-0x8F */ + 0xF9, 0x8E, 0xF9, 0x8F, 0xF9, 0x90, 0xF9, 0x91, /* 0x90-0x93 */ + 0xF9, 0x92, 0xF9, 0x93, 0xF9, 0x94, 0xF9, 0x95, /* 0x94-0x97 */ + 0xF9, 0x96, 0xF9, 0x97, 0xF9, 0x98, 0xF9, 0x99, /* 0x98-0x9B */ + 0xF9, 0x9A, 0xF9, 0x9B, 0xF9, 0x9C, 0xF9, 0x9D, /* 0x9C-0x9F */ + 0xF9, 0x9E, 0xF9, 0x9F, 0xF9, 0xA0, 0xFA, 0x40, /* 0xA0-0xA3 */ + 0xFA, 0x41, 0xFA, 0x42, 0xFA, 0x43, 0xFA, 0x44, /* 0xA4-0xA7 */ + 0xFA, 0x45, 0xFA, 0x46, 0xFA, 0x47, 0xFA, 0x48, /* 0xA8-0xAB */ + 0xFA, 0x49, 0xFA, 0x4A, 0xFA, 0x4B, 0xFA, 0x4C, /* 0xAC-0xAF */ + 0xFA, 0x4D, 0xFA, 0x4E, 0xFA, 0x4F, 0xFA, 0x50, /* 0xB0-0xB3 */ + 0xFA, 0x51, 0xFA, 0x52, 0xFA, 0x53, 0xFA, 0x54, /* 0xB4-0xB7 */ + 0xFA, 0x55, 0xFA, 0x56, 0xFA, 0x57, 0xFA, 0x58, /* 0xB8-0xBB */ + 0xFA, 0x59, 0xFA, 0x5A, 0xFA, 0x5B, 0xFA, 0x5C, /* 0xBC-0xBF */ + 0xFA, 0x5D, 0xFA, 0x5E, 0xFA, 0x5F, 0xFA, 0x60, /* 0xC0-0xC3 */ + 0xFA, 0x61, 0xFA, 0x62, 0xFA, 0x63, 0xFA, 0x64, /* 0xC4-0xC7 */ + 0xFA, 0x65, 0xFA, 0x66, 0xFA, 0x67, 0xFA, 0x68, /* 0xC8-0xCB */ + 0xFA, 0x69, 0xFA, 0x6A, 0xFA, 0x6B, 0xFA, 0x6C, /* 0xCC-0xCF */ + 0xFA, 0x6D, 0xFA, 0x6E, 0xFA, 0x6F, 0xFA, 0x70, /* 0xD0-0xD3 */ + 0xFA, 0x71, 0xFA, 0x72, 0xFA, 0x73, 0xFA, 0x74, /* 0xD4-0xD7 */ + 0xFA, 0x75, 0xFA, 0x76, 0xFA, 0x77, 0xFA, 0x78, /* 0xD8-0xDB */ + 0xFA, 0x79, 0xFA, 0x7A, 0xFA, 0x7B, 0xFA, 0x7C, /* 0xDC-0xDF */ + 0xFA, 0x7D, 0xFA, 0x7E, 0xFA, 0x80, 0xFA, 0x81, /* 0xE0-0xE3 */ + 0xFA, 0x82, 0xFA, 0x83, 0xFA, 0x84, 0xFA, 0x85, /* 0xE4-0xE7 */ + 0xFA, 0x86, 0xFA, 0x87, 0xFA, 0x88, 0xFA, 0x89, /* 0xE8-0xEB */ + 0xFA, 0x8A, 0xFA, 0x8B, 0xFA, 0x8C, 0xFA, 0x8D, /* 0xEC-0xEF */ + 0xFA, 0x8E, 0xFA, 0x8F, 0xFA, 0x90, 0xFA, 0x91, /* 0xF0-0xF3 */ + 0xFA, 0x92, 0xFA, 0x93, 0xFA, 0x94, 0xFA, 0x95, /* 0xF4-0xF7 */ + 0xFA, 0x96, 0xFA, 0x97, 0xFA, 0x98, 0xFA, 0x99, /* 0xF8-0xFB */ + 0xFA, 0x9A, 0xFA, 0x9B, 0xFA, 0x9C, 0xFA, 0x9D, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_9E[512] = { + 0xFA, 0x9E, 0xFA, 0x9F, 0xFA, 0xA0, 0xFB, 0x40, /* 0x00-0x03 */ + 0xFB, 0x41, 0xFB, 0x42, 0xFB, 0x43, 0xFB, 0x44, /* 0x04-0x07 */ + 0xFB, 0x45, 0xFB, 0x46, 0xFB, 0x47, 0xFB, 0x48, /* 0x08-0x0B */ + 0xFB, 0x49, 0xFB, 0x4A, 0xFB, 0x4B, 0xFB, 0x4C, /* 0x0C-0x0F */ + 0xFB, 0x4D, 0xFB, 0x4E, 0xFB, 0x4F, 0xFB, 0x50, /* 0x10-0x13 */ + 0xFB, 0x51, 0xFB, 0x52, 0xFB, 0x53, 0xFB, 0x54, /* 0x14-0x17 */ + 0xFB, 0x55, 0xFB, 0x56, 0xFB, 0x57, 0xFB, 0x58, /* 0x18-0x1B */ + 0xFB, 0x59, 0xFB, 0x5A, 0xFB, 0x5B, 0xC4, 0xF1, /* 0x1C-0x1F */ + 0xF0, 0xAF, 0xBC, 0xA6, 0xF0, 0xB0, 0xC3, 0xF9, /* 0x20-0x23 */ + 0xFB, 0x5C, 0xC5, 0xB8, 0xD1, 0xBB, 0xFB, 0x5D, /* 0x24-0x27 */ + 0xF0, 0xB1, 0xF0, 0xB2, 0xF0, 0xB3, 0xF0, 0xB4, /* 0x28-0x2B */ + 0xF0, 0xB5, 0xD1, 0xBC, 0xFB, 0x5E, 0xD1, 0xEC, /* 0x2C-0x2F */ + 0xFB, 0x5F, 0xF0, 0xB7, 0xF0, 0xB6, 0xD4, 0xA7, /* 0x30-0x33 */ + 0xFB, 0x60, 0xCD, 0xD2, 0xF0, 0xB8, 0xF0, 0xBA, /* 0x34-0x37 */ + 0xF0, 0xB9, 0xF0, 0xBB, 0xF0, 0xBC, 0xFB, 0x61, /* 0x38-0x3B */ + 0xFB, 0x62, 0xB8, 0xEB, 0xF0, 0xBD, 0xBA, 0xE8, /* 0x3C-0x3F */ + 0xFB, 0x63, 0xF0, 0xBE, 0xF0, 0xBF, 0xBE, 0xE9, /* 0x40-0x43 */ + 0xF0, 0xC0, 0xB6, 0xEC, 0xF0, 0xC1, 0xF0, 0xC2, /* 0x44-0x47 */ + 0xF0, 0xC3, 0xF0, 0xC4, 0xC8, 0xB5, 0xF0, 0xC5, /* 0x48-0x4B */ + 0xF0, 0xC6, 0xFB, 0x64, 0xF0, 0xC7, 0xC5, 0xF4, /* 0x4C-0x4F */ + 0xFB, 0x65, 0xF0, 0xC8, 0xFB, 0x66, 0xFB, 0x67, /* 0x50-0x53 */ + 0xFB, 0x68, 0xF0, 0xC9, 0xFB, 0x69, 0xF0, 0xCA, /* 0x54-0x57 */ + 0xF7, 0xBD, 0xFB, 0x6A, 0xF0, 0xCB, 0xF0, 0xCC, /* 0x58-0x5B */ + 0xF0, 0xCD, 0xFB, 0x6B, 0xF0, 0xCE, 0xFB, 0x6C, /* 0x5C-0x5F */ + 0xFB, 0x6D, 0xFB, 0x6E, 0xFB, 0x6F, 0xF0, 0xCF, /* 0x60-0x63 */ + 0xBA, 0xD7, 0xFB, 0x70, 0xF0, 0xD0, 0xF0, 0xD1, /* 0x64-0x67 */ + 0xF0, 0xD2, 0xF0, 0xD3, 0xF0, 0xD4, 0xF0, 0xD5, /* 0x68-0x6B */ + 0xF0, 0xD6, 0xF0, 0xD8, 0xFB, 0x71, 0xFB, 0x72, /* 0x6C-0x6F */ + 0xD3, 0xA5, 0xF0, 0xD7, 0xFB, 0x73, 0xF0, 0xD9, /* 0x70-0x73 */ + 0xFB, 0x74, 0xFB, 0x75, 0xFB, 0x76, 0xFB, 0x77, /* 0x74-0x77 */ + 0xFB, 0x78, 0xFB, 0x79, 0xFB, 0x7A, 0xFB, 0x7B, /* 0x78-0x7B */ + 0xFB, 0x7C, 0xFB, 0x7D, 0xF5, 0xBA, 0xC2, 0xB9, /* 0x7C-0x7F */ + + 0xFB, 0x7E, 0xFB, 0x80, 0xF7, 0xE4, 0xFB, 0x81, /* 0x80-0x83 */ + 0xFB, 0x82, 0xFB, 0x83, 0xFB, 0x84, 0xF7, 0xE5, /* 0x84-0x87 */ + 0xF7, 0xE6, 0xFB, 0x85, 0xFB, 0x86, 0xF7, 0xE7, /* 0x88-0x8B */ + 0xFB, 0x87, 0xFB, 0x88, 0xFB, 0x89, 0xFB, 0x8A, /* 0x8C-0x8F */ + 0xFB, 0x8B, 0xFB, 0x8C, 0xF7, 0xE8, 0xC2, 0xB4, /* 0x90-0x93 */ + 0xFB, 0x8D, 0xFB, 0x8E, 0xFB, 0x8F, 0xFB, 0x90, /* 0x94-0x97 */ + 0xFB, 0x91, 0xFB, 0x92, 0xFB, 0x93, 0xFB, 0x94, /* 0x98-0x9B */ + 0xFB, 0x95, 0xF7, 0xEA, 0xFB, 0x96, 0xF7, 0xEB, /* 0x9C-0x9F */ + 0xFB, 0x97, 0xFB, 0x98, 0xFB, 0x99, 0xFB, 0x9A, /* 0xA0-0xA3 */ + 0xFB, 0x9B, 0xFB, 0x9C, 0xC2, 0xF3, 0xFB, 0x9D, /* 0xA4-0xA7 */ + 0xFB, 0x9E, 0xFB, 0x9F, 0xFB, 0xA0, 0xFC, 0x40, /* 0xA8-0xAB */ + 0xFC, 0x41, 0xFC, 0x42, 0xFC, 0x43, 0xFC, 0x44, /* 0xAC-0xAF */ + 0xFC, 0x45, 0xFC, 0x46, 0xFC, 0x47, 0xFC, 0x48, /* 0xB0-0xB3 */ + 0xF4, 0xF0, 0xFC, 0x49, 0xFC, 0x4A, 0xFC, 0x4B, /* 0xB4-0xB7 */ + 0xF4, 0xEF, 0xFC, 0x4C, 0xFC, 0x4D, 0xC2, 0xE9, /* 0xB8-0xBB */ + 0xFC, 0x4E, 0xF7, 0xE1, 0xF7, 0xE2, 0xFC, 0x4F, /* 0xBC-0xBF */ + 0xFC, 0x50, 0xFC, 0x51, 0xFC, 0x52, 0xFC, 0x53, /* 0xC0-0xC3 */ + 0xBB, 0xC6, 0xFC, 0x54, 0xFC, 0x55, 0xFC, 0x56, /* 0xC4-0xC7 */ + 0xFC, 0x57, 0xD9, 0xE4, 0xFC, 0x58, 0xFC, 0x59, /* 0xC8-0xCB */ + 0xFC, 0x5A, 0xCA, 0xF2, 0xC0, 0xE8, 0xF0, 0xA4, /* 0xCC-0xCF */ + 0xFC, 0x5B, 0xBA, 0xDA, 0xFC, 0x5C, 0xFC, 0x5D, /* 0xD0-0xD3 */ + 0xC7, 0xAD, 0xFC, 0x5E, 0xFC, 0x5F, 0xFC, 0x60, /* 0xD4-0xD7 */ + 0xC4, 0xAC, 0xFC, 0x61, 0xFC, 0x62, 0xF7, 0xEC, /* 0xD8-0xDB */ + 0xF7, 0xED, 0xF7, 0xEE, 0xFC, 0x63, 0xF7, 0xF0, /* 0xDC-0xDF */ + 0xF7, 0xEF, 0xFC, 0x64, 0xF7, 0xF1, 0xFC, 0x65, /* 0xE0-0xE3 */ + 0xFC, 0x66, 0xF7, 0xF4, 0xFC, 0x67, 0xF7, 0xF3, /* 0xE4-0xE7 */ + 0xFC, 0x68, 0xF7, 0xF2, 0xF7, 0xF5, 0xFC, 0x69, /* 0xE8-0xEB */ + 0xFC, 0x6A, 0xFC, 0x6B, 0xFC, 0x6C, 0xF7, 0xF6, /* 0xEC-0xEF */ + 0xFC, 0x6D, 0xFC, 0x6E, 0xFC, 0x6F, 0xFC, 0x70, /* 0xF0-0xF3 */ + 0xFC, 0x71, 0xFC, 0x72, 0xFC, 0x73, 0xFC, 0x74, /* 0xF4-0xF7 */ + 0xFC, 0x75, 0xED, 0xE9, 0xFC, 0x76, 0xED, 0xEA, /* 0xF8-0xFB */ + 0xED, 0xEB, 0xFC, 0x77, 0xF6, 0xBC, 0xFC, 0x78, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_9F[512] = { + 0xFC, 0x79, 0xFC, 0x7A, 0xFC, 0x7B, 0xFC, 0x7C, /* 0x00-0x03 */ + 0xFC, 0x7D, 0xFC, 0x7E, 0xFC, 0x80, 0xFC, 0x81, /* 0x04-0x07 */ + 0xFC, 0x82, 0xFC, 0x83, 0xFC, 0x84, 0xF6, 0xBD, /* 0x08-0x0B */ + 0xFC, 0x85, 0xF6, 0xBE, 0xB6, 0xA6, 0xFC, 0x86, /* 0x0C-0x0F */ + 0xD8, 0xBE, 0xFC, 0x87, 0xFC, 0x88, 0xB9, 0xC4, /* 0x10-0x13 */ + 0xFC, 0x89, 0xFC, 0x8A, 0xFC, 0x8B, 0xD8, 0xBB, /* 0x14-0x17 */ + 0xFC, 0x8C, 0xDC, 0xB1, 0xFC, 0x8D, 0xFC, 0x8E, /* 0x18-0x1B */ + 0xFC, 0x8F, 0xFC, 0x90, 0xFC, 0x91, 0xFC, 0x92, /* 0x1C-0x1F */ + 0xCA, 0xF3, 0xFC, 0x93, 0xF7, 0xF7, 0xFC, 0x94, /* 0x20-0x23 */ + 0xFC, 0x95, 0xFC, 0x96, 0xFC, 0x97, 0xFC, 0x98, /* 0x24-0x27 */ + 0xFC, 0x99, 0xFC, 0x9A, 0xFC, 0x9B, 0xFC, 0x9C, /* 0x28-0x2B */ + 0xF7, 0xF8, 0xFC, 0x9D, 0xFC, 0x9E, 0xF7, 0xF9, /* 0x2C-0x2F */ + 0xFC, 0x9F, 0xFC, 0xA0, 0xFD, 0x40, 0xFD, 0x41, /* 0x30-0x33 */ + 0xFD, 0x42, 0xFD, 0x43, 0xFD, 0x44, 0xF7, 0xFB, /* 0x34-0x37 */ + 0xFD, 0x45, 0xF7, 0xFA, 0xFD, 0x46, 0xB1, 0xC7, /* 0x38-0x3B */ + 0xFD, 0x47, 0xF7, 0xFC, 0xF7, 0xFD, 0xFD, 0x48, /* 0x3C-0x3F */ + 0xFD, 0x49, 0xFD, 0x4A, 0xFD, 0x4B, 0xFD, 0x4C, /* 0x40-0x43 */ + 0xF7, 0xFE, 0xFD, 0x4D, 0xFD, 0x4E, 0xFD, 0x4F, /* 0x44-0x47 */ + 0xFD, 0x50, 0xFD, 0x51, 0xFD, 0x52, 0xFD, 0x53, /* 0x48-0x4B */ + 0xFD, 0x54, 0xFD, 0x55, 0xFD, 0x56, 0xFD, 0x57, /* 0x4C-0x4F */ + 0xC6, 0xEB, 0xEC, 0xB4, 0xFD, 0x58, 0xFD, 0x59, /* 0x50-0x53 */ + 0xFD, 0x5A, 0xFD, 0x5B, 0xFD, 0x5C, 0xFD, 0x5D, /* 0x54-0x57 */ + 0xFD, 0x5E, 0xFD, 0x5F, 0xFD, 0x60, 0xFD, 0x61, /* 0x58-0x5B */ + 0xFD, 0x62, 0xFD, 0x63, 0xFD, 0x64, 0xFD, 0x65, /* 0x5C-0x5F */ + 0xFD, 0x66, 0xFD, 0x67, 0xFD, 0x68, 0xFD, 0x69, /* 0x60-0x63 */ + 0xFD, 0x6A, 0xFD, 0x6B, 0xFD, 0x6C, 0xFD, 0x6D, /* 0x64-0x67 */ + 0xFD, 0x6E, 0xFD, 0x6F, 0xFD, 0x70, 0xFD, 0x71, /* 0x68-0x6B */ + 0xFD, 0x72, 0xFD, 0x73, 0xFD, 0x74, 0xFD, 0x75, /* 0x6C-0x6F */ + 0xFD, 0x76, 0xFD, 0x77, 0xFD, 0x78, 0xFD, 0x79, /* 0x70-0x73 */ + 0xFD, 0x7A, 0xFD, 0x7B, 0xFD, 0x7C, 0xFD, 0x7D, /* 0x74-0x77 */ + 0xFD, 0x7E, 0xFD, 0x80, 0xFD, 0x81, 0xFD, 0x82, /* 0x78-0x7B */ + 0xFD, 0x83, 0xFD, 0x84, 0xFD, 0x85, 0xB3, 0xDD, /* 0x7C-0x7F */ + + 0xF6, 0xB3, 0xFD, 0x86, 0xFD, 0x87, 0xF6, 0xB4, /* 0x80-0x83 */ + 0xC1, 0xE4, 0xF6, 0xB5, 0xF6, 0xB6, 0xF6, 0xB7, /* 0x84-0x87 */ + 0xF6, 0xB8, 0xF6, 0xB9, 0xF6, 0xBA, 0xC8, 0xA3, /* 0x88-0x8B */ + 0xF6, 0xBB, 0xFD, 0x88, 0xFD, 0x89, 0xFD, 0x8A, /* 0x8C-0x8F */ + 0xFD, 0x8B, 0xFD, 0x8C, 0xFD, 0x8D, 0xFD, 0x8E, /* 0x90-0x93 */ + 0xFD, 0x8F, 0xFD, 0x90, 0xFD, 0x91, 0xFD, 0x92, /* 0x94-0x97 */ + 0xFD, 0x93, 0xC1, 0xFA, 0xB9, 0xA8, 0xED, 0xE8, /* 0x98-0x9B */ + 0xFD, 0x94, 0xFD, 0x95, 0xFD, 0x96, 0xB9, 0xEA, /* 0x9C-0x9F */ + 0xD9, 0xDF, 0xFD, 0x97, 0xFD, 0x98, 0xFD, 0x99, /* 0xA0-0xA3 */ + 0xFD, 0x9A, 0xFD, 0x9B, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ +}; + +static const unsigned char u2c_DC[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ +}; + +static const unsigned char u2c_F9[512] = { + 0xD8, 0x4D, 0xB8, 0xFC, 0xDC, 0x87, 0xD9, 0x5A, /* 0x00-0x03 */ + 0xBB, 0xAC, 0xB4, 0xAE, 0xBE, 0xE4, 0xFD, 0x94, /* 0x04-0x07 */ + 0xFD, 0x94, 0xC6, 0xF5, 0xBD, 0xF0, 0xC0, 0xAE, /* 0x08-0x0B */ + 0xC4, 0xCE, 0x91, 0xD0, 0xB0, 0x5D, 0xC1, 0x5F, /* 0x0C-0x0F */ + 0xCC, 0x7D, 0xC2, 0xDD, 0xC2, 0xE3, 0xDF, 0x89, /* 0x10-0x13 */ + 0x98, 0xB7, 0xC2, 0xE5, 0xC0, 0xD3, 0xE7, 0xF3, /* 0x14-0x17 */ + 0xC2, 0xE4, 0xC0, 0xD2, 0xF1, 0x98, 0x81, 0x79, /* 0x18-0x1B */ + 0xC2, 0xD1, 0x99, 0xDA, 0xA0, 0x80, 0xCC, 0x6D, /* 0x1C-0x1F */ + 0xFB, 0x5B, 0x8D, 0xB9, 0x9E, 0x45, 0xCB, 0x7B, /* 0x20-0x23 */ + 0xD2, 0x68, 0xC0, 0xAD, 0xC5, 0x44, 0xCF, 0x9E, /* 0x24-0x27 */ + 0xC0, 0xC8, 0xC0, 0xCA, 0xC0, 0xCB, 0xC0, 0xC7, /* 0x28-0x2B */ + 0xFD, 0x9C, 0x81, 0xED, 0xC0, 0xE4, 0x84, 0xDA, /* 0x2C-0x2F */ + 0x93, 0xEF, 0x99, 0xA9, 0xA0, 0x74, 0xB1, 0x52, /* 0x30-0x33 */ + 0xC0, 0xCF, 0xCC, 0x4A, 0xCC, 0x94, 0xC2, 0xB7, /* 0x34-0x37 */ + 0xC2, 0xB6, 0xF4, 0x94, 0xFA, 0x98, 0xC2, 0xB5, /* 0x38-0x3B */ + 0xB5, 0x93, 0xBE, 0x47, 0xC7, 0x8A, 0xE4, 0x9B, /* 0x3C-0x3F */ + 0xC2, 0xB9, 0xD5, 0x93, 0x89, 0xC5, 0xC5, 0xAA, /* 0x40-0x43 */ + 0xBB, 0x5C, 0xC3, 0x40, 0xC0, 0xCE, 0xC0, 0xDA, /* 0x44-0x47 */ + 0xD9, 0x54, 0xC0, 0xD7, 0x89, 0xBE, 0x8C, 0xD2, /* 0x48-0x4B */ + 0x98, 0xC7, 0x9C, 0x49, 0xC2, 0xA9, 0xC0, 0xDB, /* 0x4C-0x4F */ + 0xBF, 0x7C, 0xC2, 0xAA, 0xC0, 0xD5, 0xC0, 0xDF, /* 0x50-0x53 */ + 0x84, 0x43, 0xC1, 0xE8, 0xB6, 0xA0, 0xBE, 0x63, /* 0x54-0x57 */ + 0xC1, 0xE2, 0xC1, 0xEA, 0xD7, 0x78, 0x92, 0x82, /* 0x58-0x5B */ + 0x98, 0xB7, 0xD6, 0x5A, 0xB5, 0xA4, 0x8C, 0x8E, /* 0x5C-0x5F */ + 0xC5, 0xAD, 0xC2, 0xCA, 0xAE, 0x90, 0xB1, 0xB1, /* 0x60-0x63 */ + 0xB4, 0x91, 0xB1, 0xE3, 0x8F, 0xCD, 0xB2, 0xBB, /* 0x64-0x67 */ + 0xC3, 0xDA, 0x94, 0xB5, 0xCB, 0xF7, 0x85, 0xA2, /* 0x68-0x6B */ + 0xC8, 0xFB, 0xCA, 0xA1, 0xC8, 0x7E, 0xD5, 0x66, /* 0x6C-0x6F */ + 0x9A, 0xA2, 0xB3, 0xBD, 0xC9, 0xF2, 0xCA, 0xB0, /* 0x70-0x73 */ + 0xC8, 0xF4, 0xC2, 0xD3, 0xC2, 0xD4, 0xC1, 0xC1, /* 0x74-0x77 */ + 0x83, 0xC9, 0xFD, 0x9D, 0xC1, 0xBA, 0xBC, 0x5A, /* 0x78-0x7B */ + 0xC1, 0xBC, 0xD5, 0x8F, 0xC1, 0xBF, 0x84, 0xEE, /* 0x7C-0x7F */ + + 0x85, 0xCE, 0xC5, 0xAE, 0x8F, 0x5D, 0xC2, 0xC3, /* 0x80-0x83 */ + 0x9E, 0x56, 0xB5, 0x5A, 0xE9, 0x82, 0xF3, 0x50, /* 0x84-0x87 */ + 0xFB, 0x90, 0xC0, 0xE8, 0xC1, 0xA6, 0x95, 0xD1, /* 0x88-0x8B */ + 0x9A, 0x76, 0xDE, 0x5D, 0xC4, 0xEA, 0x91, 0x7A, /* 0x8C-0x8F */ + 0x91, 0xD9, 0x93, 0xD3, 0x9D, 0x69, 0x9F, 0x92, /* 0x90-0x93 */ + 0xAD, 0x49, 0xFD, 0x9E, 0xBE, 0x9A, 0xC2, 0x93, /* 0x94-0x97 */ + 0xDD, 0x82, 0xC9, 0x8F, 0xDF, 0x42, 0xE5, 0x80, /* 0x98-0x9B */ + 0xC1, 0xD0, 0xC1, 0xD3, 0xD1, 0xCA, 0xC1, 0xD2, /* 0x9C-0x9F */ + 0xC1, 0xD1, 0xD5, 0x66, 0xC1, 0xAE, 0xC4, 0xEE, /* 0xA0-0xA3 */ + 0xC4, 0xED, 0x9A, 0x9A, 0xBA, 0x9F, 0xAB, 0x43, /* 0xA4-0xA7 */ + 0xC1, 0xEE, 0xE0, 0xF2, 0x8C, 0x8E, 0x8E, 0x58, /* 0xA8-0xAB */ + 0xC1, 0xAF, 0xC1, 0xE1, 0xAC, 0x93, 0xC1, 0xE7, /* 0xAC-0xAF */ + 0xF1, 0xF6, 0xE2, 0x8F, 0xC1, 0xE3, 0xEC, 0x60, /* 0xB0-0xB3 */ + 0xEE, 0x49, 0xC0, 0xFD, 0xB6, 0x59, 0xF5, 0xB7, /* 0xB4-0xB7 */ + 0xEB, 0x60, 0x90, 0xBA, 0xC1, 0xCB, 0xC1, 0xC5, /* 0xB8-0xBB */ + 0xE5, 0xBC, 0xC4, 0xF2, 0xC1, 0xCF, 0x98, 0xB7, /* 0xBC-0xBF */ + 0xC1, 0xC7, 0xAF, 0x9F, 0xDE, 0xA4, 0xDF, 0x7C, /* 0xC0-0xC3 */ + 0xFD, 0x88, 0x95, 0x9E, 0xC8, 0xEE, 0x84, 0xA2, /* 0xC4-0xC7 */ + 0x96, 0x83, 0xC1, 0xF8, 0xC1, 0xF7, 0xC1, 0xEF, /* 0xC8-0xCB */ + 0xC1, 0xF0, 0xC1, 0xF4, 0xC1, 0xF2, 0xBC, 0x7E, /* 0xCC-0xCF */ + 0xEE, 0x90, 0xC1, 0xF9, 0xC2, 0xBE, 0xEA, 0x91, /* 0xD0-0xD3 */ + 0x82, 0x90, 0x8D, 0x91, 0x9C, 0x53, 0xDD, 0x86, /* 0xD4-0xD7 */ + 0xC2, 0xC9, 0x90, 0xFC, 0xC0, 0xF5, 0xC2, 0xCA, /* 0xD8-0xDB */ + 0xC2, 0xA1, 0xC0, 0xFB, 0xC0, 0xF4, 0xC2, 0xC4, /* 0xDC-0xDF */ + 0xD2, 0xD7, 0xC0, 0xEE, 0xC0, 0xE6, 0xC4, 0xE0, /* 0xE0-0xE3 */ + 0xC0, 0xED, 0xC1, 0xA1, 0xEE, 0xBE, 0xFD, 0x9F, /* 0xE4-0xE7 */ + 0xD1, 0x65, 0xC0, 0xEF, 0xEB, 0x78, 0xC4, 0xE4, /* 0xE8-0xEB */ + 0xC4, 0xE7, 0xC1, 0xDF, 0x9F, 0xFB, 0xAD, 0x55, /* 0xEC-0xEF */ + 0xCC, 0x41, 0xFD, 0xA0, 0xF7, 0x5B, 0xF7, 0xEB, /* 0xF0-0xF3 */ + 0xC1, 0xD6, 0xC1, 0xDC, 0xC5, 0x52, 0xC1, 0xA2, /* 0xF4-0xF7 */ + 0xF3, 0xD2, 0xC1, 0xA3, 0xA0, 0xEE, 0xD6, 0xCB, /* 0xF8-0xFB */ + 0xD7, 0x52, 0xCA, 0xB2, 0xB2, 0xE8, 0xB4, 0xCC, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_FA[512] = { + 0xC7, 0xD0, 0xB6, 0xC8, 0xCD, 0xD8, 0xCC, 0xC7, /* 0x00-0x03 */ + 0xD5, 0xAC, 0xB6, 0xB4, 0xB1, 0xA9, 0xDD, 0x97, /* 0x04-0x07 */ + 0xD0, 0xD0, 0xBD, 0xB5, 0xD2, 0x8A, 0xC0, 0xAA, /* 0x08-0x0B */ + 0xFE, 0x40, 0xFE, 0x41, 0xFE, 0x42, 0xFE, 0x43, /* 0x0C-0x0F */ + 0x89, 0x56, 0xFE, 0x44, 0xC7, 0xE7, 0xFE, 0x45, /* 0x10-0x13 */ + 0xFE, 0x46, 0x84, 0x44, 0xD8, 0x69, 0xD2, 0xE6, /* 0x14-0x17 */ + 0xFE, 0x47, 0xC9, 0xF1, 0xCF, 0xE9, 0xB8, 0xA3, /* 0x18-0x1B */ + 0xBE, 0xB8, 0xBE, 0xAB, 0xD3, 0xF0, 0xFE, 0x48, /* 0x1C-0x1F */ + 0xFE, 0x49, 0xFE, 0x4A, 0xD6, 0x54, 0xFE, 0x4B, /* 0x20-0x23 */ + 0xFE, 0x4C, 0xD2, 0xDD, 0xB6, 0xBC, 0xFE, 0x4D, /* 0x24-0x27 */ + 0xFE, 0x4E, 0xFE, 0x4F, 0xEF, 0x88, 0xEF, 0x95, /* 0x28-0x2B */ + 0xF0, 0x5E, 0xFA, 0x51, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ +}; + +static const unsigned char u2c_FE[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xA9, 0x55, 0xA6, 0xF2, 0x00, 0x00, 0xA6, 0xF4, /* 0x30-0x33 */ + 0xA6, 0xF5, 0xA6, 0xE0, 0xA6, 0xE1, 0xA6, 0xF0, /* 0x34-0x37 */ + 0xA6, 0xF1, 0xA6, 0xE2, 0xA6, 0xE3, 0xA6, 0xEE, /* 0x38-0x3B */ + 0xA6, 0xEF, 0xA6, 0xE6, 0xA6, 0xE7, 0xA6, 0xE4, /* 0x3C-0x3F */ + 0xA6, 0xE5, 0xA6, 0xE8, 0xA6, 0xE9, 0xA6, 0xEA, /* 0x40-0x43 */ + 0xA6, 0xEB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0xA9, 0x68, 0xA9, 0x69, 0xA9, 0x6A, /* 0x48-0x4B */ + 0xA9, 0x6B, 0xA9, 0x6C, 0xA9, 0x6D, 0xA9, 0x6E, /* 0x4C-0x4F */ + 0xA9, 0x6F, 0xA9, 0x70, 0xA9, 0x71, 0x00, 0x00, /* 0x50-0x53 */ + 0xA9, 0x72, 0xA9, 0x73, 0xA9, 0x74, 0xA9, 0x75, /* 0x54-0x57 */ + 0x00, 0x00, 0xA9, 0x76, 0xA9, 0x77, 0xA9, 0x78, /* 0x58-0x5B */ + 0xA9, 0x79, 0xA9, 0x7A, 0xA9, 0x7B, 0xA9, 0x7C, /* 0x5C-0x5F */ + 0xA9, 0x7D, 0xA9, 0x7E, 0xA9, 0x80, 0xA9, 0x81, /* 0x60-0x63 */ + 0xA9, 0x82, 0xA9, 0x83, 0xA9, 0x84, 0x00, 0x00, /* 0x64-0x67 */ + 0xA9, 0x85, 0xA9, 0x86, 0xA9, 0x87, 0xA9, 0x88, /* 0x68-0x6B */ +}; + +static const unsigned char u2c_FF[512] = { + 0x00, 0x00, 0xA3, 0xA1, 0xA3, 0xA2, 0xA3, 0xA3, /* 0x00-0x03 */ + 0xA1, 0xE7, 0xA3, 0xA5, 0xA3, 0xA6, 0xA3, 0xA7, /* 0x04-0x07 */ + 0xA3, 0xA8, 0xA3, 0xA9, 0xA3, 0xAA, 0xA3, 0xAB, /* 0x08-0x0B */ + 0xA3, 0xAC, 0xA3, 0xAD, 0xA3, 0xAE, 0xA3, 0xAF, /* 0x0C-0x0F */ + 0xA3, 0xB0, 0xA3, 0xB1, 0xA3, 0xB2, 0xA3, 0xB3, /* 0x10-0x13 */ + 0xA3, 0xB4, 0xA3, 0xB5, 0xA3, 0xB6, 0xA3, 0xB7, /* 0x14-0x17 */ + 0xA3, 0xB8, 0xA3, 0xB9, 0xA3, 0xBA, 0xA3, 0xBB, /* 0x18-0x1B */ + 0xA3, 0xBC, 0xA3, 0xBD, 0xA3, 0xBE, 0xA3, 0xBF, /* 0x1C-0x1F */ + 0xA3, 0xC0, 0xA3, 0xC1, 0xA3, 0xC2, 0xA3, 0xC3, /* 0x20-0x23 */ + 0xA3, 0xC4, 0xA3, 0xC5, 0xA3, 0xC6, 0xA3, 0xC7, /* 0x24-0x27 */ + 0xA3, 0xC8, 0xA3, 0xC9, 0xA3, 0xCA, 0xA3, 0xCB, /* 0x28-0x2B */ + 0xA3, 0xCC, 0xA3, 0xCD, 0xA3, 0xCE, 0xA3, 0xCF, /* 0x2C-0x2F */ + 0xA3, 0xD0, 0xA3, 0xD1, 0xA3, 0xD2, 0xA3, 0xD3, /* 0x30-0x33 */ + 0xA3, 0xD4, 0xA3, 0xD5, 0xA3, 0xD6, 0xA3, 0xD7, /* 0x34-0x37 */ + 0xA3, 0xD8, 0xA3, 0xD9, 0xA3, 0xDA, 0xA3, 0xDB, /* 0x38-0x3B */ + 0xA3, 0xDC, 0xA3, 0xDD, 0xA3, 0xDE, 0xA3, 0xDF, /* 0x3C-0x3F */ + 0xA3, 0xE0, 0xA3, 0xE1, 0xA3, 0xE2, 0xA3, 0xE3, /* 0x40-0x43 */ + 0xA3, 0xE4, 0xA3, 0xE5, 0xA3, 0xE6, 0xA3, 0xE7, /* 0x44-0x47 */ + 0xA3, 0xE8, 0xA3, 0xE9, 0xA3, 0xEA, 0xA3, 0xEB, /* 0x48-0x4B */ + 0xA3, 0xEC, 0xA3, 0xED, 0xA3, 0xEE, 0xA3, 0xEF, /* 0x4C-0x4F */ + 0xA3, 0xF0, 0xA3, 0xF1, 0xA3, 0xF2, 0xA3, 0xF3, /* 0x50-0x53 */ + 0xA3, 0xF4, 0xA3, 0xF5, 0xA3, 0xF6, 0xA3, 0xF7, /* 0x54-0x57 */ + 0xA3, 0xF8, 0xA3, 0xF9, 0xA3, 0xFA, 0xA3, 0xFB, /* 0x58-0x5B */ + 0xA3, 0xFC, 0xA3, 0xFD, 0xA1, 0xAB, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0xA1, 0xE9, 0xA1, 0xEA, 0xA9, 0x56, 0xA3, 0xFE, /* 0xE0-0xE3 */ + 0xA9, 0x57, 0xA3, 0xA4, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ +}; + +static const unsigned char *const page_uni2charset[256] = { + u2c_00, u2c_01, u2c_02, u2c_03, u2c_04, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + u2c_20, u2c_21, u2c_22, u2c_23, u2c_24, u2c_25, u2c_26, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + u2c_30, u2c_31, u2c_32, u2c_33, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, u2c_4E, u2c_4F, + u2c_50, u2c_51, u2c_52, u2c_53, u2c_54, u2c_55, u2c_56, u2c_57, + u2c_58, u2c_59, u2c_5A, u2c_5B, u2c_5C, u2c_5D, u2c_5E, u2c_5F, + u2c_60, u2c_61, u2c_62, u2c_63, u2c_64, u2c_65, u2c_66, u2c_67, + u2c_68, u2c_69, u2c_6A, u2c_6B, u2c_6C, u2c_6D, u2c_6E, u2c_6F, + u2c_70, u2c_71, u2c_72, u2c_73, u2c_74, u2c_75, u2c_76, u2c_77, + u2c_78, u2c_79, u2c_7A, u2c_7B, u2c_7C, u2c_7D, u2c_7E, u2c_7F, + u2c_80, u2c_81, u2c_82, u2c_83, u2c_84, u2c_85, u2c_86, u2c_87, + u2c_88, u2c_89, u2c_8A, u2c_8B, u2c_8C, u2c_8D, u2c_8E, u2c_8F, + u2c_90, u2c_91, u2c_92, u2c_93, u2c_94, u2c_95, u2c_96, u2c_97, + u2c_98, u2c_99, u2c_9A, u2c_9B, u2c_9C, u2c_9D, u2c_9E, u2c_9F, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, u2c_DC, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, u2c_F9, u2c_FA, NULL, NULL, NULL, u2c_FE, u2c_FF, }; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(const wchar_t uni, + unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni&0xFF; + unsigned char ch = (uni>>8)&0xFF; + unsigned char out0,out1; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + if (uni == 0x20ac) {/* Euro symbol.The only exception with a non-ascii unicode */ + out[0] = 0x80; + return 1; + } + + if (ch == 0) { /* handle the U00 plane*/ + /* if (cl == 0) return -EINVAL;*/ /*U0000 is legal in cp936*/ + out0 = u2c_00[cl*2]; + out1 = u2c_00[cl*2+1]; + if (out0 == 0x00 && out1 == 0x00) { + if (cl<0x80) { + out[0] = cl; + return 1; + } + return -EINVAL; + } else { + if (boundlen <= 1) + return -ENAMETOOLONG; + out[0] = out0; + out[1] = out1; + return 2; + } + } + + uni2charset = page_uni2charset[ch]; + if (uni2charset) { + if (boundlen <= 1) + return -ENAMETOOLONG; + out[0] = uni2charset[cl*2]; + out[1] = uni2charset[cl*2+1]; + if (out[0] == 0x00 && out[1] == 0x00) + return -EINVAL; + return 2; + } + else + return -EINVAL; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, + wchar_t *uni) +{ + unsigned char ch, cl; + const wchar_t *charset2uni; + int n; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + if (boundlen == 1) { + if (rawstring[0]==0x80) { /* Euro symbol.The only exception with a non-ascii unicode */ + *uni = 0x20ac; + } else { + *uni = rawstring[0]; + } + return 1; + } + + ch = rawstring[0]; + cl = rawstring[1]; + + charset2uni = page_charset2uni[ch]; + if (charset2uni && cl) { + *uni = charset2uni[cl]; + if (*uni == 0x0000) + return -EINVAL; + n = 2; + } else{ + if (ch==0x80) {/* Euro symbol.The only exception with a non-ascii unicode */ + *uni = 0x20ac; + } else { + *uni = ch; + } + n = 1; + } + return n; +} + +static struct nls_table table = { + .charset = "cp936", + .alias = "gb2312", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_cp936(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_cp936(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_cp936) +module_exit(exit_nls_cp936) + +MODULE_DESCRIPTION("NLS Simplified Chinese charset (CP936, GB2312)"); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_ALIAS_NLS(gb2312); diff --git a/fs/nls/nls_cp949.c b/fs/nls/nls_cp949.c new file mode 100644 index 000000000..3ae03c76d --- /dev/null +++ b/fs/nls/nls_cp949.c @@ -0,0 +1,13947 @@ +/* + * linux/fs/nls/nls_cp949.c + * + * Charset cp949 translation tables. + * This translation table was generated automatically, the + * original table can be download from the Microsoft website. + * (http://www.microsoft.com/typography/unicode/unicodecp.htm) + */ + +#include +#include +#include +#include +#include + +static const wchar_t c2u_81[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xAC02,0xAC03,0xAC05,0xAC06,0xAC0B,0xAC0C,0xAC0D,/* 0x40-0x47 */ + 0xAC0E,0xAC0F,0xAC18,0xAC1E,0xAC1F,0xAC21,0xAC22,0xAC23,/* 0x48-0x4F */ + 0xAC25,0xAC26,0xAC27,0xAC28,0xAC29,0xAC2A,0xAC2B,0xAC2E,/* 0x50-0x57 */ + 0xAC32,0xAC33,0xAC34,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xAC35,0xAC36,0xAC37,0xAC3A,0xAC3B,0xAC3D,0xAC3E,/* 0x60-0x67 */ + 0xAC3F,0xAC41,0xAC42,0xAC43,0xAC44,0xAC45,0xAC46,0xAC47,/* 0x68-0x6F */ + 0xAC48,0xAC49,0xAC4A,0xAC4C,0xAC4E,0xAC4F,0xAC50,0xAC51,/* 0x70-0x77 */ + 0xAC52,0xAC53,0xAC55,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xAC56,0xAC57,0xAC59,0xAC5A,0xAC5B,0xAC5D,0xAC5E,/* 0x80-0x87 */ + 0xAC5F,0xAC60,0xAC61,0xAC62,0xAC63,0xAC64,0xAC65,0xAC66,/* 0x88-0x8F */ + 0xAC67,0xAC68,0xAC69,0xAC6A,0xAC6B,0xAC6C,0xAC6D,0xAC6E,/* 0x90-0x97 */ + 0xAC6F,0xAC72,0xAC73,0xAC75,0xAC76,0xAC79,0xAC7B,0xAC7C,/* 0x98-0x9F */ + 0xAC7D,0xAC7E,0xAC7F,0xAC82,0xAC87,0xAC88,0xAC8D,0xAC8E,/* 0xA0-0xA7 */ + 0xAC8F,0xAC91,0xAC92,0xAC93,0xAC95,0xAC96,0xAC97,0xAC98,/* 0xA8-0xAF */ + 0xAC99,0xAC9A,0xAC9B,0xAC9E,0xACA2,0xACA3,0xACA4,0xACA5,/* 0xB0-0xB7 */ + 0xACA6,0xACA7,0xACAB,0xACAD,0xACAE,0xACB1,0xACB2,0xACB3,/* 0xB8-0xBF */ + 0xACB4,0xACB5,0xACB6,0xACB7,0xACBA,0xACBE,0xACBF,0xACC0,/* 0xC0-0xC7 */ + 0xACC2,0xACC3,0xACC5,0xACC6,0xACC7,0xACC9,0xACCA,0xACCB,/* 0xC8-0xCF */ + 0xACCD,0xACCE,0xACCF,0xACD0,0xACD1,0xACD2,0xACD3,0xACD4,/* 0xD0-0xD7 */ + 0xACD6,0xACD8,0xACD9,0xACDA,0xACDB,0xACDC,0xACDD,0xACDE,/* 0xD8-0xDF */ + 0xACDF,0xACE2,0xACE3,0xACE5,0xACE6,0xACE9,0xACEB,0xACED,/* 0xE0-0xE7 */ + 0xACEE,0xACF2,0xACF4,0xACF7,0xACF8,0xACF9,0xACFA,0xACFB,/* 0xE8-0xEF */ + 0xACFE,0xACFF,0xAD01,0xAD02,0xAD03,0xAD05,0xAD07,0xAD08,/* 0xF0-0xF7 */ + 0xAD09,0xAD0A,0xAD0B,0xAD0E,0xAD10,0xAD12,0xAD13,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_82[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xAD14,0xAD15,0xAD16,0xAD17,0xAD19,0xAD1A,0xAD1B,/* 0x40-0x47 */ + 0xAD1D,0xAD1E,0xAD1F,0xAD21,0xAD22,0xAD23,0xAD24,0xAD25,/* 0x48-0x4F */ + 0xAD26,0xAD27,0xAD28,0xAD2A,0xAD2B,0xAD2E,0xAD2F,0xAD30,/* 0x50-0x57 */ + 0xAD31,0xAD32,0xAD33,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xAD36,0xAD37,0xAD39,0xAD3A,0xAD3B,0xAD3D,0xAD3E,/* 0x60-0x67 */ + 0xAD3F,0xAD40,0xAD41,0xAD42,0xAD43,0xAD46,0xAD48,0xAD4A,/* 0x68-0x6F */ + 0xAD4B,0xAD4C,0xAD4D,0xAD4E,0xAD4F,0xAD51,0xAD52,0xAD53,/* 0x70-0x77 */ + 0xAD55,0xAD56,0xAD57,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xAD59,0xAD5A,0xAD5B,0xAD5C,0xAD5D,0xAD5E,0xAD5F,/* 0x80-0x87 */ + 0xAD60,0xAD62,0xAD64,0xAD65,0xAD66,0xAD67,0xAD68,0xAD69,/* 0x88-0x8F */ + 0xAD6A,0xAD6B,0xAD6E,0xAD6F,0xAD71,0xAD72,0xAD77,0xAD78,/* 0x90-0x97 */ + 0xAD79,0xAD7A,0xAD7E,0xAD80,0xAD83,0xAD84,0xAD85,0xAD86,/* 0x98-0x9F */ + 0xAD87,0xAD8A,0xAD8B,0xAD8D,0xAD8E,0xAD8F,0xAD91,0xAD92,/* 0xA0-0xA7 */ + 0xAD93,0xAD94,0xAD95,0xAD96,0xAD97,0xAD98,0xAD99,0xAD9A,/* 0xA8-0xAF */ + 0xAD9B,0xAD9E,0xAD9F,0xADA0,0xADA1,0xADA2,0xADA3,0xADA5,/* 0xB0-0xB7 */ + 0xADA6,0xADA7,0xADA8,0xADA9,0xADAA,0xADAB,0xADAC,0xADAD,/* 0xB8-0xBF */ + 0xADAE,0xADAF,0xADB0,0xADB1,0xADB2,0xADB3,0xADB4,0xADB5,/* 0xC0-0xC7 */ + 0xADB6,0xADB8,0xADB9,0xADBA,0xADBB,0xADBC,0xADBD,0xADBE,/* 0xC8-0xCF */ + 0xADBF,0xADC2,0xADC3,0xADC5,0xADC6,0xADC7,0xADC9,0xADCA,/* 0xD0-0xD7 */ + 0xADCB,0xADCC,0xADCD,0xADCE,0xADCF,0xADD2,0xADD4,0xADD5,/* 0xD8-0xDF */ + 0xADD6,0xADD7,0xADD8,0xADD9,0xADDA,0xADDB,0xADDD,0xADDE,/* 0xE0-0xE7 */ + 0xADDF,0xADE1,0xADE2,0xADE3,0xADE5,0xADE6,0xADE7,0xADE8,/* 0xE8-0xEF */ + 0xADE9,0xADEA,0xADEB,0xADEC,0xADED,0xADEE,0xADEF,0xADF0,/* 0xF0-0xF7 */ + 0xADF1,0xADF2,0xADF3,0xADF4,0xADF5,0xADF6,0xADF7,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_83[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xADFA,0xADFB,0xADFD,0xADFE,0xAE02,0xAE03,0xAE04,/* 0x40-0x47 */ + 0xAE05,0xAE06,0xAE07,0xAE0A,0xAE0C,0xAE0E,0xAE0F,0xAE10,/* 0x48-0x4F */ + 0xAE11,0xAE12,0xAE13,0xAE15,0xAE16,0xAE17,0xAE18,0xAE19,/* 0x50-0x57 */ + 0xAE1A,0xAE1B,0xAE1C,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xAE1D,0xAE1E,0xAE1F,0xAE20,0xAE21,0xAE22,0xAE23,/* 0x60-0x67 */ + 0xAE24,0xAE25,0xAE26,0xAE27,0xAE28,0xAE29,0xAE2A,0xAE2B,/* 0x68-0x6F */ + 0xAE2C,0xAE2D,0xAE2E,0xAE2F,0xAE32,0xAE33,0xAE35,0xAE36,/* 0x70-0x77 */ + 0xAE39,0xAE3B,0xAE3C,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xAE3D,0xAE3E,0xAE3F,0xAE42,0xAE44,0xAE47,0xAE48,/* 0x80-0x87 */ + 0xAE49,0xAE4B,0xAE4F,0xAE51,0xAE52,0xAE53,0xAE55,0xAE57,/* 0x88-0x8F */ + 0xAE58,0xAE59,0xAE5A,0xAE5B,0xAE5E,0xAE62,0xAE63,0xAE64,/* 0x90-0x97 */ + 0xAE66,0xAE67,0xAE6A,0xAE6B,0xAE6D,0xAE6E,0xAE6F,0xAE71,/* 0x98-0x9F */ + 0xAE72,0xAE73,0xAE74,0xAE75,0xAE76,0xAE77,0xAE7A,0xAE7E,/* 0xA0-0xA7 */ + 0xAE7F,0xAE80,0xAE81,0xAE82,0xAE83,0xAE86,0xAE87,0xAE88,/* 0xA8-0xAF */ + 0xAE89,0xAE8A,0xAE8B,0xAE8D,0xAE8E,0xAE8F,0xAE90,0xAE91,/* 0xB0-0xB7 */ + 0xAE92,0xAE93,0xAE94,0xAE95,0xAE96,0xAE97,0xAE98,0xAE99,/* 0xB8-0xBF */ + 0xAE9A,0xAE9B,0xAE9C,0xAE9D,0xAE9E,0xAE9F,0xAEA0,0xAEA1,/* 0xC0-0xC7 */ + 0xAEA2,0xAEA3,0xAEA4,0xAEA5,0xAEA6,0xAEA7,0xAEA8,0xAEA9,/* 0xC8-0xCF */ + 0xAEAA,0xAEAB,0xAEAC,0xAEAD,0xAEAE,0xAEAF,0xAEB0,0xAEB1,/* 0xD0-0xD7 */ + 0xAEB2,0xAEB3,0xAEB4,0xAEB5,0xAEB6,0xAEB7,0xAEB8,0xAEB9,/* 0xD8-0xDF */ + 0xAEBA,0xAEBB,0xAEBF,0xAEC1,0xAEC2,0xAEC3,0xAEC5,0xAEC6,/* 0xE0-0xE7 */ + 0xAEC7,0xAEC8,0xAEC9,0xAECA,0xAECB,0xAECE,0xAED2,0xAED3,/* 0xE8-0xEF */ + 0xAED4,0xAED5,0xAED6,0xAED7,0xAEDA,0xAEDB,0xAEDD,0xAEDE,/* 0xF0-0xF7 */ + 0xAEDF,0xAEE0,0xAEE1,0xAEE2,0xAEE3,0xAEE4,0xAEE5,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_84[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xAEE6,0xAEE7,0xAEE9,0xAEEA,0xAEEC,0xAEEE,0xAEEF,/* 0x40-0x47 */ + 0xAEF0,0xAEF1,0xAEF2,0xAEF3,0xAEF5,0xAEF6,0xAEF7,0xAEF9,/* 0x48-0x4F */ + 0xAEFA,0xAEFB,0xAEFD,0xAEFE,0xAEFF,0xAF00,0xAF01,0xAF02,/* 0x50-0x57 */ + 0xAF03,0xAF04,0xAF05,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xAF06,0xAF09,0xAF0A,0xAF0B,0xAF0C,0xAF0E,0xAF0F,/* 0x60-0x67 */ + 0xAF11,0xAF12,0xAF13,0xAF14,0xAF15,0xAF16,0xAF17,0xAF18,/* 0x68-0x6F */ + 0xAF19,0xAF1A,0xAF1B,0xAF1C,0xAF1D,0xAF1E,0xAF1F,0xAF20,/* 0x70-0x77 */ + 0xAF21,0xAF22,0xAF23,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xAF24,0xAF25,0xAF26,0xAF27,0xAF28,0xAF29,0xAF2A,/* 0x80-0x87 */ + 0xAF2B,0xAF2E,0xAF2F,0xAF31,0xAF33,0xAF35,0xAF36,0xAF37,/* 0x88-0x8F */ + 0xAF38,0xAF39,0xAF3A,0xAF3B,0xAF3E,0xAF40,0xAF44,0xAF45,/* 0x90-0x97 */ + 0xAF46,0xAF47,0xAF4A,0xAF4B,0xAF4C,0xAF4D,0xAF4E,0xAF4F,/* 0x98-0x9F */ + 0xAF51,0xAF52,0xAF53,0xAF54,0xAF55,0xAF56,0xAF57,0xAF58,/* 0xA0-0xA7 */ + 0xAF59,0xAF5A,0xAF5B,0xAF5E,0xAF5F,0xAF60,0xAF61,0xAF62,/* 0xA8-0xAF */ + 0xAF63,0xAF66,0xAF67,0xAF68,0xAF69,0xAF6A,0xAF6B,0xAF6C,/* 0xB0-0xB7 */ + 0xAF6D,0xAF6E,0xAF6F,0xAF70,0xAF71,0xAF72,0xAF73,0xAF74,/* 0xB8-0xBF */ + 0xAF75,0xAF76,0xAF77,0xAF78,0xAF7A,0xAF7B,0xAF7C,0xAF7D,/* 0xC0-0xC7 */ + 0xAF7E,0xAF7F,0xAF81,0xAF82,0xAF83,0xAF85,0xAF86,0xAF87,/* 0xC8-0xCF */ + 0xAF89,0xAF8A,0xAF8B,0xAF8C,0xAF8D,0xAF8E,0xAF8F,0xAF92,/* 0xD0-0xD7 */ + 0xAF93,0xAF94,0xAF96,0xAF97,0xAF98,0xAF99,0xAF9A,0xAF9B,/* 0xD8-0xDF */ + 0xAF9D,0xAF9E,0xAF9F,0xAFA0,0xAFA1,0xAFA2,0xAFA3,0xAFA4,/* 0xE0-0xE7 */ + 0xAFA5,0xAFA6,0xAFA7,0xAFA8,0xAFA9,0xAFAA,0xAFAB,0xAFAC,/* 0xE8-0xEF */ + 0xAFAD,0xAFAE,0xAFAF,0xAFB0,0xAFB1,0xAFB2,0xAFB3,0xAFB4,/* 0xF0-0xF7 */ + 0xAFB5,0xAFB6,0xAFB7,0xAFBA,0xAFBB,0xAFBD,0xAFBE,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_85[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xAFBF,0xAFC1,0xAFC2,0xAFC3,0xAFC4,0xAFC5,0xAFC6,/* 0x40-0x47 */ + 0xAFCA,0xAFCC,0xAFCF,0xAFD0,0xAFD1,0xAFD2,0xAFD3,0xAFD5,/* 0x48-0x4F */ + 0xAFD6,0xAFD7,0xAFD8,0xAFD9,0xAFDA,0xAFDB,0xAFDD,0xAFDE,/* 0x50-0x57 */ + 0xAFDF,0xAFE0,0xAFE1,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xAFE2,0xAFE3,0xAFE4,0xAFE5,0xAFE6,0xAFE7,0xAFEA,/* 0x60-0x67 */ + 0xAFEB,0xAFEC,0xAFED,0xAFEE,0xAFEF,0xAFF2,0xAFF3,0xAFF5,/* 0x68-0x6F */ + 0xAFF6,0xAFF7,0xAFF9,0xAFFA,0xAFFB,0xAFFC,0xAFFD,0xAFFE,/* 0x70-0x77 */ + 0xAFFF,0xB002,0xB003,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xB005,0xB006,0xB007,0xB008,0xB009,0xB00A,0xB00B,/* 0x80-0x87 */ + 0xB00D,0xB00E,0xB00F,0xB011,0xB012,0xB013,0xB015,0xB016,/* 0x88-0x8F */ + 0xB017,0xB018,0xB019,0xB01A,0xB01B,0xB01E,0xB01F,0xB020,/* 0x90-0x97 */ + 0xB021,0xB022,0xB023,0xB024,0xB025,0xB026,0xB027,0xB029,/* 0x98-0x9F */ + 0xB02A,0xB02B,0xB02C,0xB02D,0xB02E,0xB02F,0xB030,0xB031,/* 0xA0-0xA7 */ + 0xB032,0xB033,0xB034,0xB035,0xB036,0xB037,0xB038,0xB039,/* 0xA8-0xAF */ + 0xB03A,0xB03B,0xB03C,0xB03D,0xB03E,0xB03F,0xB040,0xB041,/* 0xB0-0xB7 */ + 0xB042,0xB043,0xB046,0xB047,0xB049,0xB04B,0xB04D,0xB04F,/* 0xB8-0xBF */ + 0xB050,0xB051,0xB052,0xB056,0xB058,0xB05A,0xB05B,0xB05C,/* 0xC0-0xC7 */ + 0xB05E,0xB05F,0xB060,0xB061,0xB062,0xB063,0xB064,0xB065,/* 0xC8-0xCF */ + 0xB066,0xB067,0xB068,0xB069,0xB06A,0xB06B,0xB06C,0xB06D,/* 0xD0-0xD7 */ + 0xB06E,0xB06F,0xB070,0xB071,0xB072,0xB073,0xB074,0xB075,/* 0xD8-0xDF */ + 0xB076,0xB077,0xB078,0xB079,0xB07A,0xB07B,0xB07E,0xB07F,/* 0xE0-0xE7 */ + 0xB081,0xB082,0xB083,0xB085,0xB086,0xB087,0xB088,0xB089,/* 0xE8-0xEF */ + 0xB08A,0xB08B,0xB08E,0xB090,0xB092,0xB093,0xB094,0xB095,/* 0xF0-0xF7 */ + 0xB096,0xB097,0xB09B,0xB09D,0xB09E,0xB0A3,0xB0A4,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_86[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xB0A5,0xB0A6,0xB0A7,0xB0AA,0xB0B0,0xB0B2,0xB0B6,/* 0x40-0x47 */ + 0xB0B7,0xB0B9,0xB0BA,0xB0BB,0xB0BD,0xB0BE,0xB0BF,0xB0C0,/* 0x48-0x4F */ + 0xB0C1,0xB0C2,0xB0C3,0xB0C6,0xB0CA,0xB0CB,0xB0CC,0xB0CD,/* 0x50-0x57 */ + 0xB0CE,0xB0CF,0xB0D2,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xB0D3,0xB0D5,0xB0D6,0xB0D7,0xB0D9,0xB0DA,0xB0DB,/* 0x60-0x67 */ + 0xB0DC,0xB0DD,0xB0DE,0xB0DF,0xB0E1,0xB0E2,0xB0E3,0xB0E4,/* 0x68-0x6F */ + 0xB0E6,0xB0E7,0xB0E8,0xB0E9,0xB0EA,0xB0EB,0xB0EC,0xB0ED,/* 0x70-0x77 */ + 0xB0EE,0xB0EF,0xB0F0,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xB0F1,0xB0F2,0xB0F3,0xB0F4,0xB0F5,0xB0F6,0xB0F7,/* 0x80-0x87 */ + 0xB0F8,0xB0F9,0xB0FA,0xB0FB,0xB0FC,0xB0FD,0xB0FE,0xB0FF,/* 0x88-0x8F */ + 0xB100,0xB101,0xB102,0xB103,0xB104,0xB105,0xB106,0xB107,/* 0x90-0x97 */ + 0xB10A,0xB10D,0xB10E,0xB10F,0xB111,0xB114,0xB115,0xB116,/* 0x98-0x9F */ + 0xB117,0xB11A,0xB11E,0xB11F,0xB120,0xB121,0xB122,0xB126,/* 0xA0-0xA7 */ + 0xB127,0xB129,0xB12A,0xB12B,0xB12D,0xB12E,0xB12F,0xB130,/* 0xA8-0xAF */ + 0xB131,0xB132,0xB133,0xB136,0xB13A,0xB13B,0xB13C,0xB13D,/* 0xB0-0xB7 */ + 0xB13E,0xB13F,0xB142,0xB143,0xB145,0xB146,0xB147,0xB149,/* 0xB8-0xBF */ + 0xB14A,0xB14B,0xB14C,0xB14D,0xB14E,0xB14F,0xB152,0xB153,/* 0xC0-0xC7 */ + 0xB156,0xB157,0xB159,0xB15A,0xB15B,0xB15D,0xB15E,0xB15F,/* 0xC8-0xCF */ + 0xB161,0xB162,0xB163,0xB164,0xB165,0xB166,0xB167,0xB168,/* 0xD0-0xD7 */ + 0xB169,0xB16A,0xB16B,0xB16C,0xB16D,0xB16E,0xB16F,0xB170,/* 0xD8-0xDF */ + 0xB171,0xB172,0xB173,0xB174,0xB175,0xB176,0xB177,0xB17A,/* 0xE0-0xE7 */ + 0xB17B,0xB17D,0xB17E,0xB17F,0xB181,0xB183,0xB184,0xB185,/* 0xE8-0xEF */ + 0xB186,0xB187,0xB18A,0xB18C,0xB18E,0xB18F,0xB190,0xB191,/* 0xF0-0xF7 */ + 0xB195,0xB196,0xB197,0xB199,0xB19A,0xB19B,0xB19D,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_87[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xB19E,0xB19F,0xB1A0,0xB1A1,0xB1A2,0xB1A3,0xB1A4,/* 0x40-0x47 */ + 0xB1A5,0xB1A6,0xB1A7,0xB1A9,0xB1AA,0xB1AB,0xB1AC,0xB1AD,/* 0x48-0x4F */ + 0xB1AE,0xB1AF,0xB1B0,0xB1B1,0xB1B2,0xB1B3,0xB1B4,0xB1B5,/* 0x50-0x57 */ + 0xB1B6,0xB1B7,0xB1B8,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xB1B9,0xB1BA,0xB1BB,0xB1BC,0xB1BD,0xB1BE,0xB1BF,/* 0x60-0x67 */ + 0xB1C0,0xB1C1,0xB1C2,0xB1C3,0xB1C4,0xB1C5,0xB1C6,0xB1C7,/* 0x68-0x6F */ + 0xB1C8,0xB1C9,0xB1CA,0xB1CB,0xB1CD,0xB1CE,0xB1CF,0xB1D1,/* 0x70-0x77 */ + 0xB1D2,0xB1D3,0xB1D5,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xB1D6,0xB1D7,0xB1D8,0xB1D9,0xB1DA,0xB1DB,0xB1DE,/* 0x80-0x87 */ + 0xB1E0,0xB1E1,0xB1E2,0xB1E3,0xB1E4,0xB1E5,0xB1E6,0xB1E7,/* 0x88-0x8F */ + 0xB1EA,0xB1EB,0xB1ED,0xB1EE,0xB1EF,0xB1F1,0xB1F2,0xB1F3,/* 0x90-0x97 */ + 0xB1F4,0xB1F5,0xB1F6,0xB1F7,0xB1F8,0xB1FA,0xB1FC,0xB1FE,/* 0x98-0x9F */ + 0xB1FF,0xB200,0xB201,0xB202,0xB203,0xB206,0xB207,0xB209,/* 0xA0-0xA7 */ + 0xB20A,0xB20D,0xB20E,0xB20F,0xB210,0xB211,0xB212,0xB213,/* 0xA8-0xAF */ + 0xB216,0xB218,0xB21A,0xB21B,0xB21C,0xB21D,0xB21E,0xB21F,/* 0xB0-0xB7 */ + 0xB221,0xB222,0xB223,0xB224,0xB225,0xB226,0xB227,0xB228,/* 0xB8-0xBF */ + 0xB229,0xB22A,0xB22B,0xB22C,0xB22D,0xB22E,0xB22F,0xB230,/* 0xC0-0xC7 */ + 0xB231,0xB232,0xB233,0xB235,0xB236,0xB237,0xB238,0xB239,/* 0xC8-0xCF */ + 0xB23A,0xB23B,0xB23D,0xB23E,0xB23F,0xB240,0xB241,0xB242,/* 0xD0-0xD7 */ + 0xB243,0xB244,0xB245,0xB246,0xB247,0xB248,0xB249,0xB24A,/* 0xD8-0xDF */ + 0xB24B,0xB24C,0xB24D,0xB24E,0xB24F,0xB250,0xB251,0xB252,/* 0xE0-0xE7 */ + 0xB253,0xB254,0xB255,0xB256,0xB257,0xB259,0xB25A,0xB25B,/* 0xE8-0xEF */ + 0xB25D,0xB25E,0xB25F,0xB261,0xB262,0xB263,0xB264,0xB265,/* 0xF0-0xF7 */ + 0xB266,0xB267,0xB26A,0xB26B,0xB26C,0xB26D,0xB26E,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_88[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xB26F,0xB270,0xB271,0xB272,0xB273,0xB276,0xB277,/* 0x40-0x47 */ + 0xB278,0xB279,0xB27A,0xB27B,0xB27D,0xB27E,0xB27F,0xB280,/* 0x48-0x4F */ + 0xB281,0xB282,0xB283,0xB286,0xB287,0xB288,0xB28A,0xB28B,/* 0x50-0x57 */ + 0xB28C,0xB28D,0xB28E,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xB28F,0xB292,0xB293,0xB295,0xB296,0xB297,0xB29B,/* 0x60-0x67 */ + 0xB29C,0xB29D,0xB29E,0xB29F,0xB2A2,0xB2A4,0xB2A7,0xB2A8,/* 0x68-0x6F */ + 0xB2A9,0xB2AB,0xB2AD,0xB2AE,0xB2AF,0xB2B1,0xB2B2,0xB2B3,/* 0x70-0x77 */ + 0xB2B5,0xB2B6,0xB2B7,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xB2B8,0xB2B9,0xB2BA,0xB2BB,0xB2BC,0xB2BD,0xB2BE,/* 0x80-0x87 */ + 0xB2BF,0xB2C0,0xB2C1,0xB2C2,0xB2C3,0xB2C4,0xB2C5,0xB2C6,/* 0x88-0x8F */ + 0xB2C7,0xB2CA,0xB2CB,0xB2CD,0xB2CE,0xB2CF,0xB2D1,0xB2D3,/* 0x90-0x97 */ + 0xB2D4,0xB2D5,0xB2D6,0xB2D7,0xB2DA,0xB2DC,0xB2DE,0xB2DF,/* 0x98-0x9F */ + 0xB2E0,0xB2E1,0xB2E3,0xB2E7,0xB2E9,0xB2EA,0xB2F0,0xB2F1,/* 0xA0-0xA7 */ + 0xB2F2,0xB2F6,0xB2FC,0xB2FD,0xB2FE,0xB302,0xB303,0xB305,/* 0xA8-0xAF */ + 0xB306,0xB307,0xB309,0xB30A,0xB30B,0xB30C,0xB30D,0xB30E,/* 0xB0-0xB7 */ + 0xB30F,0xB312,0xB316,0xB317,0xB318,0xB319,0xB31A,0xB31B,/* 0xB8-0xBF */ + 0xB31D,0xB31E,0xB31F,0xB320,0xB321,0xB322,0xB323,0xB324,/* 0xC0-0xC7 */ + 0xB325,0xB326,0xB327,0xB328,0xB329,0xB32A,0xB32B,0xB32C,/* 0xC8-0xCF */ + 0xB32D,0xB32E,0xB32F,0xB330,0xB331,0xB332,0xB333,0xB334,/* 0xD0-0xD7 */ + 0xB335,0xB336,0xB337,0xB338,0xB339,0xB33A,0xB33B,0xB33C,/* 0xD8-0xDF */ + 0xB33D,0xB33E,0xB33F,0xB340,0xB341,0xB342,0xB343,0xB344,/* 0xE0-0xE7 */ + 0xB345,0xB346,0xB347,0xB348,0xB349,0xB34A,0xB34B,0xB34C,/* 0xE8-0xEF */ + 0xB34D,0xB34E,0xB34F,0xB350,0xB351,0xB352,0xB353,0xB357,/* 0xF0-0xF7 */ + 0xB359,0xB35A,0xB35D,0xB360,0xB361,0xB362,0xB363,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_89[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xB366,0xB368,0xB36A,0xB36C,0xB36D,0xB36F,0xB372,/* 0x40-0x47 */ + 0xB373,0xB375,0xB376,0xB377,0xB379,0xB37A,0xB37B,0xB37C,/* 0x48-0x4F */ + 0xB37D,0xB37E,0xB37F,0xB382,0xB386,0xB387,0xB388,0xB389,/* 0x50-0x57 */ + 0xB38A,0xB38B,0xB38D,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xB38E,0xB38F,0xB391,0xB392,0xB393,0xB395,0xB396,/* 0x60-0x67 */ + 0xB397,0xB398,0xB399,0xB39A,0xB39B,0xB39C,0xB39D,0xB39E,/* 0x68-0x6F */ + 0xB39F,0xB3A2,0xB3A3,0xB3A4,0xB3A5,0xB3A6,0xB3A7,0xB3A9,/* 0x70-0x77 */ + 0xB3AA,0xB3AB,0xB3AD,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xB3AE,0xB3AF,0xB3B0,0xB3B1,0xB3B2,0xB3B3,0xB3B4,/* 0x80-0x87 */ + 0xB3B5,0xB3B6,0xB3B7,0xB3B8,0xB3B9,0xB3BA,0xB3BB,0xB3BC,/* 0x88-0x8F */ + 0xB3BD,0xB3BE,0xB3BF,0xB3C0,0xB3C1,0xB3C2,0xB3C3,0xB3C6,/* 0x90-0x97 */ + 0xB3C7,0xB3C9,0xB3CA,0xB3CD,0xB3CF,0xB3D1,0xB3D2,0xB3D3,/* 0x98-0x9F */ + 0xB3D6,0xB3D8,0xB3DA,0xB3DC,0xB3DE,0xB3DF,0xB3E1,0xB3E2,/* 0xA0-0xA7 */ + 0xB3E3,0xB3E5,0xB3E6,0xB3E7,0xB3E9,0xB3EA,0xB3EB,0xB3EC,/* 0xA8-0xAF */ + 0xB3ED,0xB3EE,0xB3EF,0xB3F0,0xB3F1,0xB3F2,0xB3F3,0xB3F4,/* 0xB0-0xB7 */ + 0xB3F5,0xB3F6,0xB3F7,0xB3F8,0xB3F9,0xB3FA,0xB3FB,0xB3FD,/* 0xB8-0xBF */ + 0xB3FE,0xB3FF,0xB400,0xB401,0xB402,0xB403,0xB404,0xB405,/* 0xC0-0xC7 */ + 0xB406,0xB407,0xB408,0xB409,0xB40A,0xB40B,0xB40C,0xB40D,/* 0xC8-0xCF */ + 0xB40E,0xB40F,0xB411,0xB412,0xB413,0xB414,0xB415,0xB416,/* 0xD0-0xD7 */ + 0xB417,0xB419,0xB41A,0xB41B,0xB41D,0xB41E,0xB41F,0xB421,/* 0xD8-0xDF */ + 0xB422,0xB423,0xB424,0xB425,0xB426,0xB427,0xB42A,0xB42C,/* 0xE0-0xE7 */ + 0xB42D,0xB42E,0xB42F,0xB430,0xB431,0xB432,0xB433,0xB435,/* 0xE8-0xEF */ + 0xB436,0xB437,0xB438,0xB439,0xB43A,0xB43B,0xB43C,0xB43D,/* 0xF0-0xF7 */ + 0xB43E,0xB43F,0xB440,0xB441,0xB442,0xB443,0xB444,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_8A[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xB445,0xB446,0xB447,0xB448,0xB449,0xB44A,0xB44B,/* 0x40-0x47 */ + 0xB44C,0xB44D,0xB44E,0xB44F,0xB452,0xB453,0xB455,0xB456,/* 0x48-0x4F */ + 0xB457,0xB459,0xB45A,0xB45B,0xB45C,0xB45D,0xB45E,0xB45F,/* 0x50-0x57 */ + 0xB462,0xB464,0xB466,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xB467,0xB468,0xB469,0xB46A,0xB46B,0xB46D,0xB46E,/* 0x60-0x67 */ + 0xB46F,0xB470,0xB471,0xB472,0xB473,0xB474,0xB475,0xB476,/* 0x68-0x6F */ + 0xB477,0xB478,0xB479,0xB47A,0xB47B,0xB47C,0xB47D,0xB47E,/* 0x70-0x77 */ + 0xB47F,0xB481,0xB482,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xB483,0xB484,0xB485,0xB486,0xB487,0xB489,0xB48A,/* 0x80-0x87 */ + 0xB48B,0xB48C,0xB48D,0xB48E,0xB48F,0xB490,0xB491,0xB492,/* 0x88-0x8F */ + 0xB493,0xB494,0xB495,0xB496,0xB497,0xB498,0xB499,0xB49A,/* 0x90-0x97 */ + 0xB49B,0xB49C,0xB49E,0xB49F,0xB4A0,0xB4A1,0xB4A2,0xB4A3,/* 0x98-0x9F */ + 0xB4A5,0xB4A6,0xB4A7,0xB4A9,0xB4AA,0xB4AB,0xB4AD,0xB4AE,/* 0xA0-0xA7 */ + 0xB4AF,0xB4B0,0xB4B1,0xB4B2,0xB4B3,0xB4B4,0xB4B6,0xB4B8,/* 0xA8-0xAF */ + 0xB4BA,0xB4BB,0xB4BC,0xB4BD,0xB4BE,0xB4BF,0xB4C1,0xB4C2,/* 0xB0-0xB7 */ + 0xB4C3,0xB4C5,0xB4C6,0xB4C7,0xB4C9,0xB4CA,0xB4CB,0xB4CC,/* 0xB8-0xBF */ + 0xB4CD,0xB4CE,0xB4CF,0xB4D1,0xB4D2,0xB4D3,0xB4D4,0xB4D6,/* 0xC0-0xC7 */ + 0xB4D7,0xB4D8,0xB4D9,0xB4DA,0xB4DB,0xB4DE,0xB4DF,0xB4E1,/* 0xC8-0xCF */ + 0xB4E2,0xB4E5,0xB4E7,0xB4E8,0xB4E9,0xB4EA,0xB4EB,0xB4EE,/* 0xD0-0xD7 */ + 0xB4F0,0xB4F2,0xB4F3,0xB4F4,0xB4F5,0xB4F6,0xB4F7,0xB4F9,/* 0xD8-0xDF */ + 0xB4FA,0xB4FB,0xB4FC,0xB4FD,0xB4FE,0xB4FF,0xB500,0xB501,/* 0xE0-0xE7 */ + 0xB502,0xB503,0xB504,0xB505,0xB506,0xB507,0xB508,0xB509,/* 0xE8-0xEF */ + 0xB50A,0xB50B,0xB50C,0xB50D,0xB50E,0xB50F,0xB510,0xB511,/* 0xF0-0xF7 */ + 0xB512,0xB513,0xB516,0xB517,0xB519,0xB51A,0xB51D,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_8B[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xB51E,0xB51F,0xB520,0xB521,0xB522,0xB523,0xB526,/* 0x40-0x47 */ + 0xB52B,0xB52C,0xB52D,0xB52E,0xB52F,0xB532,0xB533,0xB535,/* 0x48-0x4F */ + 0xB536,0xB537,0xB539,0xB53A,0xB53B,0xB53C,0xB53D,0xB53E,/* 0x50-0x57 */ + 0xB53F,0xB542,0xB546,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xB547,0xB548,0xB549,0xB54A,0xB54E,0xB54F,0xB551,/* 0x60-0x67 */ + 0xB552,0xB553,0xB555,0xB556,0xB557,0xB558,0xB559,0xB55A,/* 0x68-0x6F */ + 0xB55B,0xB55E,0xB562,0xB563,0xB564,0xB565,0xB566,0xB567,/* 0x70-0x77 */ + 0xB568,0xB569,0xB56A,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xB56B,0xB56C,0xB56D,0xB56E,0xB56F,0xB570,0xB571,/* 0x80-0x87 */ + 0xB572,0xB573,0xB574,0xB575,0xB576,0xB577,0xB578,0xB579,/* 0x88-0x8F */ + 0xB57A,0xB57B,0xB57C,0xB57D,0xB57E,0xB57F,0xB580,0xB581,/* 0x90-0x97 */ + 0xB582,0xB583,0xB584,0xB585,0xB586,0xB587,0xB588,0xB589,/* 0x98-0x9F */ + 0xB58A,0xB58B,0xB58C,0xB58D,0xB58E,0xB58F,0xB590,0xB591,/* 0xA0-0xA7 */ + 0xB592,0xB593,0xB594,0xB595,0xB596,0xB597,0xB598,0xB599,/* 0xA8-0xAF */ + 0xB59A,0xB59B,0xB59C,0xB59D,0xB59E,0xB59F,0xB5A2,0xB5A3,/* 0xB0-0xB7 */ + 0xB5A5,0xB5A6,0xB5A7,0xB5A9,0xB5AC,0xB5AD,0xB5AE,0xB5AF,/* 0xB8-0xBF */ + 0xB5B2,0xB5B6,0xB5B7,0xB5B8,0xB5B9,0xB5BA,0xB5BE,0xB5BF,/* 0xC0-0xC7 */ + 0xB5C1,0xB5C2,0xB5C3,0xB5C5,0xB5C6,0xB5C7,0xB5C8,0xB5C9,/* 0xC8-0xCF */ + 0xB5CA,0xB5CB,0xB5CE,0xB5D2,0xB5D3,0xB5D4,0xB5D5,0xB5D6,/* 0xD0-0xD7 */ + 0xB5D7,0xB5D9,0xB5DA,0xB5DB,0xB5DC,0xB5DD,0xB5DE,0xB5DF,/* 0xD8-0xDF */ + 0xB5E0,0xB5E1,0xB5E2,0xB5E3,0xB5E4,0xB5E5,0xB5E6,0xB5E7,/* 0xE0-0xE7 */ + 0xB5E8,0xB5E9,0xB5EA,0xB5EB,0xB5ED,0xB5EE,0xB5EF,0xB5F0,/* 0xE8-0xEF */ + 0xB5F1,0xB5F2,0xB5F3,0xB5F4,0xB5F5,0xB5F6,0xB5F7,0xB5F8,/* 0xF0-0xF7 */ + 0xB5F9,0xB5FA,0xB5FB,0xB5FC,0xB5FD,0xB5FE,0xB5FF,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_8C[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xB600,0xB601,0xB602,0xB603,0xB604,0xB605,0xB606,/* 0x40-0x47 */ + 0xB607,0xB608,0xB609,0xB60A,0xB60B,0xB60C,0xB60D,0xB60E,/* 0x48-0x4F */ + 0xB60F,0xB612,0xB613,0xB615,0xB616,0xB617,0xB619,0xB61A,/* 0x50-0x57 */ + 0xB61B,0xB61C,0xB61D,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xB61E,0xB61F,0xB620,0xB621,0xB622,0xB623,0xB624,/* 0x60-0x67 */ + 0xB626,0xB627,0xB628,0xB629,0xB62A,0xB62B,0xB62D,0xB62E,/* 0x68-0x6F */ + 0xB62F,0xB630,0xB631,0xB632,0xB633,0xB635,0xB636,0xB637,/* 0x70-0x77 */ + 0xB638,0xB639,0xB63A,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xB63B,0xB63C,0xB63D,0xB63E,0xB63F,0xB640,0xB641,/* 0x80-0x87 */ + 0xB642,0xB643,0xB644,0xB645,0xB646,0xB647,0xB649,0xB64A,/* 0x88-0x8F */ + 0xB64B,0xB64C,0xB64D,0xB64E,0xB64F,0xB650,0xB651,0xB652,/* 0x90-0x97 */ + 0xB653,0xB654,0xB655,0xB656,0xB657,0xB658,0xB659,0xB65A,/* 0x98-0x9F */ + 0xB65B,0xB65C,0xB65D,0xB65E,0xB65F,0xB660,0xB661,0xB662,/* 0xA0-0xA7 */ + 0xB663,0xB665,0xB666,0xB667,0xB669,0xB66A,0xB66B,0xB66C,/* 0xA8-0xAF */ + 0xB66D,0xB66E,0xB66F,0xB670,0xB671,0xB672,0xB673,0xB674,/* 0xB0-0xB7 */ + 0xB675,0xB676,0xB677,0xB678,0xB679,0xB67A,0xB67B,0xB67C,/* 0xB8-0xBF */ + 0xB67D,0xB67E,0xB67F,0xB680,0xB681,0xB682,0xB683,0xB684,/* 0xC0-0xC7 */ + 0xB685,0xB686,0xB687,0xB688,0xB689,0xB68A,0xB68B,0xB68C,/* 0xC8-0xCF */ + 0xB68D,0xB68E,0xB68F,0xB690,0xB691,0xB692,0xB693,0xB694,/* 0xD0-0xD7 */ + 0xB695,0xB696,0xB697,0xB698,0xB699,0xB69A,0xB69B,0xB69E,/* 0xD8-0xDF */ + 0xB69F,0xB6A1,0xB6A2,0xB6A3,0xB6A5,0xB6A6,0xB6A7,0xB6A8,/* 0xE0-0xE7 */ + 0xB6A9,0xB6AA,0xB6AD,0xB6AE,0xB6AF,0xB6B0,0xB6B2,0xB6B3,/* 0xE8-0xEF */ + 0xB6B4,0xB6B5,0xB6B6,0xB6B7,0xB6B8,0xB6B9,0xB6BA,0xB6BB,/* 0xF0-0xF7 */ + 0xB6BC,0xB6BD,0xB6BE,0xB6BF,0xB6C0,0xB6C1,0xB6C2,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_8D[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xB6C3,0xB6C4,0xB6C5,0xB6C6,0xB6C7,0xB6C8,0xB6C9,/* 0x40-0x47 */ + 0xB6CA,0xB6CB,0xB6CC,0xB6CD,0xB6CE,0xB6CF,0xB6D0,0xB6D1,/* 0x48-0x4F */ + 0xB6D2,0xB6D3,0xB6D5,0xB6D6,0xB6D7,0xB6D8,0xB6D9,0xB6DA,/* 0x50-0x57 */ + 0xB6DB,0xB6DC,0xB6DD,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xB6DE,0xB6DF,0xB6E0,0xB6E1,0xB6E2,0xB6E3,0xB6E4,/* 0x60-0x67 */ + 0xB6E5,0xB6E6,0xB6E7,0xB6E8,0xB6E9,0xB6EA,0xB6EB,0xB6EC,/* 0x68-0x6F */ + 0xB6ED,0xB6EE,0xB6EF,0xB6F1,0xB6F2,0xB6F3,0xB6F5,0xB6F6,/* 0x70-0x77 */ + 0xB6F7,0xB6F9,0xB6FA,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xB6FB,0xB6FC,0xB6FD,0xB6FE,0xB6FF,0xB702,0xB703,/* 0x80-0x87 */ + 0xB704,0xB706,0xB707,0xB708,0xB709,0xB70A,0xB70B,0xB70C,/* 0x88-0x8F */ + 0xB70D,0xB70E,0xB70F,0xB710,0xB711,0xB712,0xB713,0xB714,/* 0x90-0x97 */ + 0xB715,0xB716,0xB717,0xB718,0xB719,0xB71A,0xB71B,0xB71C,/* 0x98-0x9F */ + 0xB71D,0xB71E,0xB71F,0xB720,0xB721,0xB722,0xB723,0xB724,/* 0xA0-0xA7 */ + 0xB725,0xB726,0xB727,0xB72A,0xB72B,0xB72D,0xB72E,0xB731,/* 0xA8-0xAF */ + 0xB732,0xB733,0xB734,0xB735,0xB736,0xB737,0xB73A,0xB73C,/* 0xB0-0xB7 */ + 0xB73D,0xB73E,0xB73F,0xB740,0xB741,0xB742,0xB743,0xB745,/* 0xB8-0xBF */ + 0xB746,0xB747,0xB749,0xB74A,0xB74B,0xB74D,0xB74E,0xB74F,/* 0xC0-0xC7 */ + 0xB750,0xB751,0xB752,0xB753,0xB756,0xB757,0xB758,0xB759,/* 0xC8-0xCF */ + 0xB75A,0xB75B,0xB75C,0xB75D,0xB75E,0xB75F,0xB761,0xB762,/* 0xD0-0xD7 */ + 0xB763,0xB765,0xB766,0xB767,0xB769,0xB76A,0xB76B,0xB76C,/* 0xD8-0xDF */ + 0xB76D,0xB76E,0xB76F,0xB772,0xB774,0xB776,0xB777,0xB778,/* 0xE0-0xE7 */ + 0xB779,0xB77A,0xB77B,0xB77E,0xB77F,0xB781,0xB782,0xB783,/* 0xE8-0xEF */ + 0xB785,0xB786,0xB787,0xB788,0xB789,0xB78A,0xB78B,0xB78E,/* 0xF0-0xF7 */ + 0xB793,0xB794,0xB795,0xB79A,0xB79B,0xB79D,0xB79E,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_8E[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xB79F,0xB7A1,0xB7A2,0xB7A3,0xB7A4,0xB7A5,0xB7A6,/* 0x40-0x47 */ + 0xB7A7,0xB7AA,0xB7AE,0xB7AF,0xB7B0,0xB7B1,0xB7B2,0xB7B3,/* 0x48-0x4F */ + 0xB7B6,0xB7B7,0xB7B9,0xB7BA,0xB7BB,0xB7BC,0xB7BD,0xB7BE,/* 0x50-0x57 */ + 0xB7BF,0xB7C0,0xB7C1,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xB7C2,0xB7C3,0xB7C4,0xB7C5,0xB7C6,0xB7C8,0xB7CA,/* 0x60-0x67 */ + 0xB7CB,0xB7CC,0xB7CD,0xB7CE,0xB7CF,0xB7D0,0xB7D1,0xB7D2,/* 0x68-0x6F */ + 0xB7D3,0xB7D4,0xB7D5,0xB7D6,0xB7D7,0xB7D8,0xB7D9,0xB7DA,/* 0x70-0x77 */ + 0xB7DB,0xB7DC,0xB7DD,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xB7DE,0xB7DF,0xB7E0,0xB7E1,0xB7E2,0xB7E3,0xB7E4,/* 0x80-0x87 */ + 0xB7E5,0xB7E6,0xB7E7,0xB7E8,0xB7E9,0xB7EA,0xB7EB,0xB7EE,/* 0x88-0x8F */ + 0xB7EF,0xB7F1,0xB7F2,0xB7F3,0xB7F5,0xB7F6,0xB7F7,0xB7F8,/* 0x90-0x97 */ + 0xB7F9,0xB7FA,0xB7FB,0xB7FE,0xB802,0xB803,0xB804,0xB805,/* 0x98-0x9F */ + 0xB806,0xB80A,0xB80B,0xB80D,0xB80E,0xB80F,0xB811,0xB812,/* 0xA0-0xA7 */ + 0xB813,0xB814,0xB815,0xB816,0xB817,0xB81A,0xB81C,0xB81E,/* 0xA8-0xAF */ + 0xB81F,0xB820,0xB821,0xB822,0xB823,0xB826,0xB827,0xB829,/* 0xB0-0xB7 */ + 0xB82A,0xB82B,0xB82D,0xB82E,0xB82F,0xB830,0xB831,0xB832,/* 0xB8-0xBF */ + 0xB833,0xB836,0xB83A,0xB83B,0xB83C,0xB83D,0xB83E,0xB83F,/* 0xC0-0xC7 */ + 0xB841,0xB842,0xB843,0xB845,0xB846,0xB847,0xB848,0xB849,/* 0xC8-0xCF */ + 0xB84A,0xB84B,0xB84C,0xB84D,0xB84E,0xB84F,0xB850,0xB852,/* 0xD0-0xD7 */ + 0xB854,0xB855,0xB856,0xB857,0xB858,0xB859,0xB85A,0xB85B,/* 0xD8-0xDF */ + 0xB85E,0xB85F,0xB861,0xB862,0xB863,0xB865,0xB866,0xB867,/* 0xE0-0xE7 */ + 0xB868,0xB869,0xB86A,0xB86B,0xB86E,0xB870,0xB872,0xB873,/* 0xE8-0xEF */ + 0xB874,0xB875,0xB876,0xB877,0xB879,0xB87A,0xB87B,0xB87D,/* 0xF0-0xF7 */ + 0xB87E,0xB87F,0xB880,0xB881,0xB882,0xB883,0xB884,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_8F[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xB885,0xB886,0xB887,0xB888,0xB889,0xB88A,0xB88B,/* 0x40-0x47 */ + 0xB88C,0xB88E,0xB88F,0xB890,0xB891,0xB892,0xB893,0xB894,/* 0x48-0x4F */ + 0xB895,0xB896,0xB897,0xB898,0xB899,0xB89A,0xB89B,0xB89C,/* 0x50-0x57 */ + 0xB89D,0xB89E,0xB89F,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xB8A0,0xB8A1,0xB8A2,0xB8A3,0xB8A4,0xB8A5,0xB8A6,/* 0x60-0x67 */ + 0xB8A7,0xB8A9,0xB8AA,0xB8AB,0xB8AC,0xB8AD,0xB8AE,0xB8AF,/* 0x68-0x6F */ + 0xB8B1,0xB8B2,0xB8B3,0xB8B5,0xB8B6,0xB8B7,0xB8B9,0xB8BA,/* 0x70-0x77 */ + 0xB8BB,0xB8BC,0xB8BD,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xB8BE,0xB8BF,0xB8C2,0xB8C4,0xB8C6,0xB8C7,0xB8C8,/* 0x80-0x87 */ + 0xB8C9,0xB8CA,0xB8CB,0xB8CD,0xB8CE,0xB8CF,0xB8D1,0xB8D2,/* 0x88-0x8F */ + 0xB8D3,0xB8D5,0xB8D6,0xB8D7,0xB8D8,0xB8D9,0xB8DA,0xB8DB,/* 0x90-0x97 */ + 0xB8DC,0xB8DE,0xB8E0,0xB8E2,0xB8E3,0xB8E4,0xB8E5,0xB8E6,/* 0x98-0x9F */ + 0xB8E7,0xB8EA,0xB8EB,0xB8ED,0xB8EE,0xB8EF,0xB8F1,0xB8F2,/* 0xA0-0xA7 */ + 0xB8F3,0xB8F4,0xB8F5,0xB8F6,0xB8F7,0xB8FA,0xB8FC,0xB8FE,/* 0xA8-0xAF */ + 0xB8FF,0xB900,0xB901,0xB902,0xB903,0xB905,0xB906,0xB907,/* 0xB0-0xB7 */ + 0xB908,0xB909,0xB90A,0xB90B,0xB90C,0xB90D,0xB90E,0xB90F,/* 0xB8-0xBF */ + 0xB910,0xB911,0xB912,0xB913,0xB914,0xB915,0xB916,0xB917,/* 0xC0-0xC7 */ + 0xB919,0xB91A,0xB91B,0xB91C,0xB91D,0xB91E,0xB91F,0xB921,/* 0xC8-0xCF */ + 0xB922,0xB923,0xB924,0xB925,0xB926,0xB927,0xB928,0xB929,/* 0xD0-0xD7 */ + 0xB92A,0xB92B,0xB92C,0xB92D,0xB92E,0xB92F,0xB930,0xB931,/* 0xD8-0xDF */ + 0xB932,0xB933,0xB934,0xB935,0xB936,0xB937,0xB938,0xB939,/* 0xE0-0xE7 */ + 0xB93A,0xB93B,0xB93E,0xB93F,0xB941,0xB942,0xB943,0xB945,/* 0xE8-0xEF */ + 0xB946,0xB947,0xB948,0xB949,0xB94A,0xB94B,0xB94D,0xB94E,/* 0xF0-0xF7 */ + 0xB950,0xB952,0xB953,0xB954,0xB955,0xB956,0xB957,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_90[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xB95A,0xB95B,0xB95D,0xB95E,0xB95F,0xB961,0xB962,/* 0x40-0x47 */ + 0xB963,0xB964,0xB965,0xB966,0xB967,0xB96A,0xB96C,0xB96E,/* 0x48-0x4F */ + 0xB96F,0xB970,0xB971,0xB972,0xB973,0xB976,0xB977,0xB979,/* 0x50-0x57 */ + 0xB97A,0xB97B,0xB97D,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xB97E,0xB97F,0xB980,0xB981,0xB982,0xB983,0xB986,/* 0x60-0x67 */ + 0xB988,0xB98B,0xB98C,0xB98F,0xB990,0xB991,0xB992,0xB993,/* 0x68-0x6F */ + 0xB994,0xB995,0xB996,0xB997,0xB998,0xB999,0xB99A,0xB99B,/* 0x70-0x77 */ + 0xB99C,0xB99D,0xB99E,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xB99F,0xB9A0,0xB9A1,0xB9A2,0xB9A3,0xB9A4,0xB9A5,/* 0x80-0x87 */ + 0xB9A6,0xB9A7,0xB9A8,0xB9A9,0xB9AA,0xB9AB,0xB9AE,0xB9AF,/* 0x88-0x8F */ + 0xB9B1,0xB9B2,0xB9B3,0xB9B5,0xB9B6,0xB9B7,0xB9B8,0xB9B9,/* 0x90-0x97 */ + 0xB9BA,0xB9BB,0xB9BE,0xB9C0,0xB9C2,0xB9C3,0xB9C4,0xB9C5,/* 0x98-0x9F */ + 0xB9C6,0xB9C7,0xB9CA,0xB9CB,0xB9CD,0xB9D3,0xB9D4,0xB9D5,/* 0xA0-0xA7 */ + 0xB9D6,0xB9D7,0xB9DA,0xB9DC,0xB9DF,0xB9E0,0xB9E2,0xB9E6,/* 0xA8-0xAF */ + 0xB9E7,0xB9E9,0xB9EA,0xB9EB,0xB9ED,0xB9EE,0xB9EF,0xB9F0,/* 0xB0-0xB7 */ + 0xB9F1,0xB9F2,0xB9F3,0xB9F6,0xB9FB,0xB9FC,0xB9FD,0xB9FE,/* 0xB8-0xBF */ + 0xB9FF,0xBA02,0xBA03,0xBA04,0xBA05,0xBA06,0xBA07,0xBA09,/* 0xC0-0xC7 */ + 0xBA0A,0xBA0B,0xBA0C,0xBA0D,0xBA0E,0xBA0F,0xBA10,0xBA11,/* 0xC8-0xCF */ + 0xBA12,0xBA13,0xBA14,0xBA16,0xBA17,0xBA18,0xBA19,0xBA1A,/* 0xD0-0xD7 */ + 0xBA1B,0xBA1C,0xBA1D,0xBA1E,0xBA1F,0xBA20,0xBA21,0xBA22,/* 0xD8-0xDF */ + 0xBA23,0xBA24,0xBA25,0xBA26,0xBA27,0xBA28,0xBA29,0xBA2A,/* 0xE0-0xE7 */ + 0xBA2B,0xBA2C,0xBA2D,0xBA2E,0xBA2F,0xBA30,0xBA31,0xBA32,/* 0xE8-0xEF */ + 0xBA33,0xBA34,0xBA35,0xBA36,0xBA37,0xBA3A,0xBA3B,0xBA3D,/* 0xF0-0xF7 */ + 0xBA3E,0xBA3F,0xBA41,0xBA43,0xBA44,0xBA45,0xBA46,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_91[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xBA47,0xBA4A,0xBA4C,0xBA4F,0xBA50,0xBA51,0xBA52,/* 0x40-0x47 */ + 0xBA56,0xBA57,0xBA59,0xBA5A,0xBA5B,0xBA5D,0xBA5E,0xBA5F,/* 0x48-0x4F */ + 0xBA60,0xBA61,0xBA62,0xBA63,0xBA66,0xBA6A,0xBA6B,0xBA6C,/* 0x50-0x57 */ + 0xBA6D,0xBA6E,0xBA6F,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xBA72,0xBA73,0xBA75,0xBA76,0xBA77,0xBA79,0xBA7A,/* 0x60-0x67 */ + 0xBA7B,0xBA7C,0xBA7D,0xBA7E,0xBA7F,0xBA80,0xBA81,0xBA82,/* 0x68-0x6F */ + 0xBA86,0xBA88,0xBA89,0xBA8A,0xBA8B,0xBA8D,0xBA8E,0xBA8F,/* 0x70-0x77 */ + 0xBA90,0xBA91,0xBA92,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xBA93,0xBA94,0xBA95,0xBA96,0xBA97,0xBA98,0xBA99,/* 0x80-0x87 */ + 0xBA9A,0xBA9B,0xBA9C,0xBA9D,0xBA9E,0xBA9F,0xBAA0,0xBAA1,/* 0x88-0x8F */ + 0xBAA2,0xBAA3,0xBAA4,0xBAA5,0xBAA6,0xBAA7,0xBAAA,0xBAAD,/* 0x90-0x97 */ + 0xBAAE,0xBAAF,0xBAB1,0xBAB3,0xBAB4,0xBAB5,0xBAB6,0xBAB7,/* 0x98-0x9F */ + 0xBABA,0xBABC,0xBABE,0xBABF,0xBAC0,0xBAC1,0xBAC2,0xBAC3,/* 0xA0-0xA7 */ + 0xBAC5,0xBAC6,0xBAC7,0xBAC9,0xBACA,0xBACB,0xBACC,0xBACD,/* 0xA8-0xAF */ + 0xBACE,0xBACF,0xBAD0,0xBAD1,0xBAD2,0xBAD3,0xBAD4,0xBAD5,/* 0xB0-0xB7 */ + 0xBAD6,0xBAD7,0xBADA,0xBADB,0xBADC,0xBADD,0xBADE,0xBADF,/* 0xB8-0xBF */ + 0xBAE0,0xBAE1,0xBAE2,0xBAE3,0xBAE4,0xBAE5,0xBAE6,0xBAE7,/* 0xC0-0xC7 */ + 0xBAE8,0xBAE9,0xBAEA,0xBAEB,0xBAEC,0xBAED,0xBAEE,0xBAEF,/* 0xC8-0xCF */ + 0xBAF0,0xBAF1,0xBAF2,0xBAF3,0xBAF4,0xBAF5,0xBAF6,0xBAF7,/* 0xD0-0xD7 */ + 0xBAF8,0xBAF9,0xBAFA,0xBAFB,0xBAFD,0xBAFE,0xBAFF,0xBB01,/* 0xD8-0xDF */ + 0xBB02,0xBB03,0xBB05,0xBB06,0xBB07,0xBB08,0xBB09,0xBB0A,/* 0xE0-0xE7 */ + 0xBB0B,0xBB0C,0xBB0E,0xBB10,0xBB12,0xBB13,0xBB14,0xBB15,/* 0xE8-0xEF */ + 0xBB16,0xBB17,0xBB19,0xBB1A,0xBB1B,0xBB1D,0xBB1E,0xBB1F,/* 0xF0-0xF7 */ + 0xBB21,0xBB22,0xBB23,0xBB24,0xBB25,0xBB26,0xBB27,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_92[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xBB28,0xBB2A,0xBB2C,0xBB2D,0xBB2E,0xBB2F,0xBB30,/* 0x40-0x47 */ + 0xBB31,0xBB32,0xBB33,0xBB37,0xBB39,0xBB3A,0xBB3F,0xBB40,/* 0x48-0x4F */ + 0xBB41,0xBB42,0xBB43,0xBB46,0xBB48,0xBB4A,0xBB4B,0xBB4C,/* 0x50-0x57 */ + 0xBB4E,0xBB51,0xBB52,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xBB53,0xBB55,0xBB56,0xBB57,0xBB59,0xBB5A,0xBB5B,/* 0x60-0x67 */ + 0xBB5C,0xBB5D,0xBB5E,0xBB5F,0xBB60,0xBB62,0xBB64,0xBB65,/* 0x68-0x6F */ + 0xBB66,0xBB67,0xBB68,0xBB69,0xBB6A,0xBB6B,0xBB6D,0xBB6E,/* 0x70-0x77 */ + 0xBB6F,0xBB70,0xBB71,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xBB72,0xBB73,0xBB74,0xBB75,0xBB76,0xBB77,0xBB78,/* 0x80-0x87 */ + 0xBB79,0xBB7A,0xBB7B,0xBB7C,0xBB7D,0xBB7E,0xBB7F,0xBB80,/* 0x88-0x8F */ + 0xBB81,0xBB82,0xBB83,0xBB84,0xBB85,0xBB86,0xBB87,0xBB89,/* 0x90-0x97 */ + 0xBB8A,0xBB8B,0xBB8D,0xBB8E,0xBB8F,0xBB91,0xBB92,0xBB93,/* 0x98-0x9F */ + 0xBB94,0xBB95,0xBB96,0xBB97,0xBB98,0xBB99,0xBB9A,0xBB9B,/* 0xA0-0xA7 */ + 0xBB9C,0xBB9D,0xBB9E,0xBB9F,0xBBA0,0xBBA1,0xBBA2,0xBBA3,/* 0xA8-0xAF */ + 0xBBA5,0xBBA6,0xBBA7,0xBBA9,0xBBAA,0xBBAB,0xBBAD,0xBBAE,/* 0xB0-0xB7 */ + 0xBBAF,0xBBB0,0xBBB1,0xBBB2,0xBBB3,0xBBB5,0xBBB6,0xBBB8,/* 0xB8-0xBF */ + 0xBBB9,0xBBBA,0xBBBB,0xBBBC,0xBBBD,0xBBBE,0xBBBF,0xBBC1,/* 0xC0-0xC7 */ + 0xBBC2,0xBBC3,0xBBC5,0xBBC6,0xBBC7,0xBBC9,0xBBCA,0xBBCB,/* 0xC8-0xCF */ + 0xBBCC,0xBBCD,0xBBCE,0xBBCF,0xBBD1,0xBBD2,0xBBD4,0xBBD5,/* 0xD0-0xD7 */ + 0xBBD6,0xBBD7,0xBBD8,0xBBD9,0xBBDA,0xBBDB,0xBBDC,0xBBDD,/* 0xD8-0xDF */ + 0xBBDE,0xBBDF,0xBBE0,0xBBE1,0xBBE2,0xBBE3,0xBBE4,0xBBE5,/* 0xE0-0xE7 */ + 0xBBE6,0xBBE7,0xBBE8,0xBBE9,0xBBEA,0xBBEB,0xBBEC,0xBBED,/* 0xE8-0xEF */ + 0xBBEE,0xBBEF,0xBBF0,0xBBF1,0xBBF2,0xBBF3,0xBBF4,0xBBF5,/* 0xF0-0xF7 */ + 0xBBF6,0xBBF7,0xBBFA,0xBBFB,0xBBFD,0xBBFE,0xBC01,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_93[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xBC03,0xBC04,0xBC05,0xBC06,0xBC07,0xBC0A,0xBC0E,/* 0x40-0x47 */ + 0xBC10,0xBC12,0xBC13,0xBC19,0xBC1A,0xBC20,0xBC21,0xBC22,/* 0x48-0x4F */ + 0xBC23,0xBC26,0xBC28,0xBC2A,0xBC2B,0xBC2C,0xBC2E,0xBC2F,/* 0x50-0x57 */ + 0xBC32,0xBC33,0xBC35,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xBC36,0xBC37,0xBC39,0xBC3A,0xBC3B,0xBC3C,0xBC3D,/* 0x60-0x67 */ + 0xBC3E,0xBC3F,0xBC42,0xBC46,0xBC47,0xBC48,0xBC4A,0xBC4B,/* 0x68-0x6F */ + 0xBC4E,0xBC4F,0xBC51,0xBC52,0xBC53,0xBC54,0xBC55,0xBC56,/* 0x70-0x77 */ + 0xBC57,0xBC58,0xBC59,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xBC5A,0xBC5B,0xBC5C,0xBC5E,0xBC5F,0xBC60,0xBC61,/* 0x80-0x87 */ + 0xBC62,0xBC63,0xBC64,0xBC65,0xBC66,0xBC67,0xBC68,0xBC69,/* 0x88-0x8F */ + 0xBC6A,0xBC6B,0xBC6C,0xBC6D,0xBC6E,0xBC6F,0xBC70,0xBC71,/* 0x90-0x97 */ + 0xBC72,0xBC73,0xBC74,0xBC75,0xBC76,0xBC77,0xBC78,0xBC79,/* 0x98-0x9F */ + 0xBC7A,0xBC7B,0xBC7C,0xBC7D,0xBC7E,0xBC7F,0xBC80,0xBC81,/* 0xA0-0xA7 */ + 0xBC82,0xBC83,0xBC86,0xBC87,0xBC89,0xBC8A,0xBC8D,0xBC8F,/* 0xA8-0xAF */ + 0xBC90,0xBC91,0xBC92,0xBC93,0xBC96,0xBC98,0xBC9B,0xBC9C,/* 0xB0-0xB7 */ + 0xBC9D,0xBC9E,0xBC9F,0xBCA2,0xBCA3,0xBCA5,0xBCA6,0xBCA9,/* 0xB8-0xBF */ + 0xBCAA,0xBCAB,0xBCAC,0xBCAD,0xBCAE,0xBCAF,0xBCB2,0xBCB6,/* 0xC0-0xC7 */ + 0xBCB7,0xBCB8,0xBCB9,0xBCBA,0xBCBB,0xBCBE,0xBCBF,0xBCC1,/* 0xC8-0xCF */ + 0xBCC2,0xBCC3,0xBCC5,0xBCC6,0xBCC7,0xBCC8,0xBCC9,0xBCCA,/* 0xD0-0xD7 */ + 0xBCCB,0xBCCC,0xBCCE,0xBCD2,0xBCD3,0xBCD4,0xBCD6,0xBCD7,/* 0xD8-0xDF */ + 0xBCD9,0xBCDA,0xBCDB,0xBCDD,0xBCDE,0xBCDF,0xBCE0,0xBCE1,/* 0xE0-0xE7 */ + 0xBCE2,0xBCE3,0xBCE4,0xBCE5,0xBCE6,0xBCE7,0xBCE8,0xBCE9,/* 0xE8-0xEF */ + 0xBCEA,0xBCEB,0xBCEC,0xBCED,0xBCEE,0xBCEF,0xBCF0,0xBCF1,/* 0xF0-0xF7 */ + 0xBCF2,0xBCF3,0xBCF7,0xBCF9,0xBCFA,0xBCFB,0xBCFD,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_94[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xBCFE,0xBCFF,0xBD00,0xBD01,0xBD02,0xBD03,0xBD06,/* 0x40-0x47 */ + 0xBD08,0xBD0A,0xBD0B,0xBD0C,0xBD0D,0xBD0E,0xBD0F,0xBD11,/* 0x48-0x4F */ + 0xBD12,0xBD13,0xBD15,0xBD16,0xBD17,0xBD18,0xBD19,0xBD1A,/* 0x50-0x57 */ + 0xBD1B,0xBD1C,0xBD1D,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xBD1E,0xBD1F,0xBD20,0xBD21,0xBD22,0xBD23,0xBD25,/* 0x60-0x67 */ + 0xBD26,0xBD27,0xBD28,0xBD29,0xBD2A,0xBD2B,0xBD2D,0xBD2E,/* 0x68-0x6F */ + 0xBD2F,0xBD30,0xBD31,0xBD32,0xBD33,0xBD34,0xBD35,0xBD36,/* 0x70-0x77 */ + 0xBD37,0xBD38,0xBD39,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xBD3A,0xBD3B,0xBD3C,0xBD3D,0xBD3E,0xBD3F,0xBD41,/* 0x80-0x87 */ + 0xBD42,0xBD43,0xBD44,0xBD45,0xBD46,0xBD47,0xBD4A,0xBD4B,/* 0x88-0x8F */ + 0xBD4D,0xBD4E,0xBD4F,0xBD51,0xBD52,0xBD53,0xBD54,0xBD55,/* 0x90-0x97 */ + 0xBD56,0xBD57,0xBD5A,0xBD5B,0xBD5C,0xBD5D,0xBD5E,0xBD5F,/* 0x98-0x9F */ + 0xBD60,0xBD61,0xBD62,0xBD63,0xBD65,0xBD66,0xBD67,0xBD69,/* 0xA0-0xA7 */ + 0xBD6A,0xBD6B,0xBD6C,0xBD6D,0xBD6E,0xBD6F,0xBD70,0xBD71,/* 0xA8-0xAF */ + 0xBD72,0xBD73,0xBD74,0xBD75,0xBD76,0xBD77,0xBD78,0xBD79,/* 0xB0-0xB7 */ + 0xBD7A,0xBD7B,0xBD7C,0xBD7D,0xBD7E,0xBD7F,0xBD82,0xBD83,/* 0xB8-0xBF */ + 0xBD85,0xBD86,0xBD8B,0xBD8C,0xBD8D,0xBD8E,0xBD8F,0xBD92,/* 0xC0-0xC7 */ + 0xBD94,0xBD96,0xBD97,0xBD98,0xBD9B,0xBD9D,0xBD9E,0xBD9F,/* 0xC8-0xCF */ + 0xBDA0,0xBDA1,0xBDA2,0xBDA3,0xBDA5,0xBDA6,0xBDA7,0xBDA8,/* 0xD0-0xD7 */ + 0xBDA9,0xBDAA,0xBDAB,0xBDAC,0xBDAD,0xBDAE,0xBDAF,0xBDB1,/* 0xD8-0xDF */ + 0xBDB2,0xBDB3,0xBDB4,0xBDB5,0xBDB6,0xBDB7,0xBDB9,0xBDBA,/* 0xE0-0xE7 */ + 0xBDBB,0xBDBC,0xBDBD,0xBDBE,0xBDBF,0xBDC0,0xBDC1,0xBDC2,/* 0xE8-0xEF */ + 0xBDC3,0xBDC4,0xBDC5,0xBDC6,0xBDC7,0xBDC8,0xBDC9,0xBDCA,/* 0xF0-0xF7 */ + 0xBDCB,0xBDCC,0xBDCD,0xBDCE,0xBDCF,0xBDD0,0xBDD1,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_95[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xBDD2,0xBDD3,0xBDD6,0xBDD7,0xBDD9,0xBDDA,0xBDDB,/* 0x40-0x47 */ + 0xBDDD,0xBDDE,0xBDDF,0xBDE0,0xBDE1,0xBDE2,0xBDE3,0xBDE4,/* 0x48-0x4F */ + 0xBDE5,0xBDE6,0xBDE7,0xBDE8,0xBDEA,0xBDEB,0xBDEC,0xBDED,/* 0x50-0x57 */ + 0xBDEE,0xBDEF,0xBDF1,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xBDF2,0xBDF3,0xBDF5,0xBDF6,0xBDF7,0xBDF9,0xBDFA,/* 0x60-0x67 */ + 0xBDFB,0xBDFC,0xBDFD,0xBDFE,0xBDFF,0xBE01,0xBE02,0xBE04,/* 0x68-0x6F */ + 0xBE06,0xBE07,0xBE08,0xBE09,0xBE0A,0xBE0B,0xBE0E,0xBE0F,/* 0x70-0x77 */ + 0xBE11,0xBE12,0xBE13,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xBE15,0xBE16,0xBE17,0xBE18,0xBE19,0xBE1A,0xBE1B,/* 0x80-0x87 */ + 0xBE1E,0xBE20,0xBE21,0xBE22,0xBE23,0xBE24,0xBE25,0xBE26,/* 0x88-0x8F */ + 0xBE27,0xBE28,0xBE29,0xBE2A,0xBE2B,0xBE2C,0xBE2D,0xBE2E,/* 0x90-0x97 */ + 0xBE2F,0xBE30,0xBE31,0xBE32,0xBE33,0xBE34,0xBE35,0xBE36,/* 0x98-0x9F */ + 0xBE37,0xBE38,0xBE39,0xBE3A,0xBE3B,0xBE3C,0xBE3D,0xBE3E,/* 0xA0-0xA7 */ + 0xBE3F,0xBE40,0xBE41,0xBE42,0xBE43,0xBE46,0xBE47,0xBE49,/* 0xA8-0xAF */ + 0xBE4A,0xBE4B,0xBE4D,0xBE4F,0xBE50,0xBE51,0xBE52,0xBE53,/* 0xB0-0xB7 */ + 0xBE56,0xBE58,0xBE5C,0xBE5D,0xBE5E,0xBE5F,0xBE62,0xBE63,/* 0xB8-0xBF */ + 0xBE65,0xBE66,0xBE67,0xBE69,0xBE6B,0xBE6C,0xBE6D,0xBE6E,/* 0xC0-0xC7 */ + 0xBE6F,0xBE72,0xBE76,0xBE77,0xBE78,0xBE79,0xBE7A,0xBE7E,/* 0xC8-0xCF */ + 0xBE7F,0xBE81,0xBE82,0xBE83,0xBE85,0xBE86,0xBE87,0xBE88,/* 0xD0-0xD7 */ + 0xBE89,0xBE8A,0xBE8B,0xBE8E,0xBE92,0xBE93,0xBE94,0xBE95,/* 0xD8-0xDF */ + 0xBE96,0xBE97,0xBE9A,0xBE9B,0xBE9C,0xBE9D,0xBE9E,0xBE9F,/* 0xE0-0xE7 */ + 0xBEA0,0xBEA1,0xBEA2,0xBEA3,0xBEA4,0xBEA5,0xBEA6,0xBEA7,/* 0xE8-0xEF */ + 0xBEA9,0xBEAA,0xBEAB,0xBEAC,0xBEAD,0xBEAE,0xBEAF,0xBEB0,/* 0xF0-0xF7 */ + 0xBEB1,0xBEB2,0xBEB3,0xBEB4,0xBEB5,0xBEB6,0xBEB7,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_96[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xBEB8,0xBEB9,0xBEBA,0xBEBB,0xBEBC,0xBEBD,0xBEBE,/* 0x40-0x47 */ + 0xBEBF,0xBEC0,0xBEC1,0xBEC2,0xBEC3,0xBEC4,0xBEC5,0xBEC6,/* 0x48-0x4F */ + 0xBEC7,0xBEC8,0xBEC9,0xBECA,0xBECB,0xBECC,0xBECD,0xBECE,/* 0x50-0x57 */ + 0xBECF,0xBED2,0xBED3,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xBED5,0xBED6,0xBED9,0xBEDA,0xBEDB,0xBEDC,0xBEDD,/* 0x60-0x67 */ + 0xBEDE,0xBEDF,0xBEE1,0xBEE2,0xBEE6,0xBEE7,0xBEE8,0xBEE9,/* 0x68-0x6F */ + 0xBEEA,0xBEEB,0xBEED,0xBEEE,0xBEEF,0xBEF0,0xBEF1,0xBEF2,/* 0x70-0x77 */ + 0xBEF3,0xBEF4,0xBEF5,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xBEF6,0xBEF7,0xBEF8,0xBEF9,0xBEFA,0xBEFB,0xBEFC,/* 0x80-0x87 */ + 0xBEFD,0xBEFE,0xBEFF,0xBF00,0xBF02,0xBF03,0xBF04,0xBF05,/* 0x88-0x8F */ + 0xBF06,0xBF07,0xBF0A,0xBF0B,0xBF0C,0xBF0D,0xBF0E,0xBF0F,/* 0x90-0x97 */ + 0xBF10,0xBF11,0xBF12,0xBF13,0xBF14,0xBF15,0xBF16,0xBF17,/* 0x98-0x9F */ + 0xBF1A,0xBF1E,0xBF1F,0xBF20,0xBF21,0xBF22,0xBF23,0xBF24,/* 0xA0-0xA7 */ + 0xBF25,0xBF26,0xBF27,0xBF28,0xBF29,0xBF2A,0xBF2B,0xBF2C,/* 0xA8-0xAF */ + 0xBF2D,0xBF2E,0xBF2F,0xBF30,0xBF31,0xBF32,0xBF33,0xBF34,/* 0xB0-0xB7 */ + 0xBF35,0xBF36,0xBF37,0xBF38,0xBF39,0xBF3A,0xBF3B,0xBF3C,/* 0xB8-0xBF */ + 0xBF3D,0xBF3E,0xBF3F,0xBF42,0xBF43,0xBF45,0xBF46,0xBF47,/* 0xC0-0xC7 */ + 0xBF49,0xBF4A,0xBF4B,0xBF4C,0xBF4D,0xBF4E,0xBF4F,0xBF52,/* 0xC8-0xCF */ + 0xBF53,0xBF54,0xBF56,0xBF57,0xBF58,0xBF59,0xBF5A,0xBF5B,/* 0xD0-0xD7 */ + 0xBF5C,0xBF5D,0xBF5E,0xBF5F,0xBF60,0xBF61,0xBF62,0xBF63,/* 0xD8-0xDF */ + 0xBF64,0xBF65,0xBF66,0xBF67,0xBF68,0xBF69,0xBF6A,0xBF6B,/* 0xE0-0xE7 */ + 0xBF6C,0xBF6D,0xBF6E,0xBF6F,0xBF70,0xBF71,0xBF72,0xBF73,/* 0xE8-0xEF */ + 0xBF74,0xBF75,0xBF76,0xBF77,0xBF78,0xBF79,0xBF7A,0xBF7B,/* 0xF0-0xF7 */ + 0xBF7C,0xBF7D,0xBF7E,0xBF7F,0xBF80,0xBF81,0xBF82,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_97[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xBF83,0xBF84,0xBF85,0xBF86,0xBF87,0xBF88,0xBF89,/* 0x40-0x47 */ + 0xBF8A,0xBF8B,0xBF8C,0xBF8D,0xBF8E,0xBF8F,0xBF90,0xBF91,/* 0x48-0x4F */ + 0xBF92,0xBF93,0xBF95,0xBF96,0xBF97,0xBF98,0xBF99,0xBF9A,/* 0x50-0x57 */ + 0xBF9B,0xBF9C,0xBF9D,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xBF9E,0xBF9F,0xBFA0,0xBFA1,0xBFA2,0xBFA3,0xBFA4,/* 0x60-0x67 */ + 0xBFA5,0xBFA6,0xBFA7,0xBFA8,0xBFA9,0xBFAA,0xBFAB,0xBFAC,/* 0x68-0x6F */ + 0xBFAD,0xBFAE,0xBFAF,0xBFB1,0xBFB2,0xBFB3,0xBFB4,0xBFB5,/* 0x70-0x77 */ + 0xBFB6,0xBFB7,0xBFB8,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xBFB9,0xBFBA,0xBFBB,0xBFBC,0xBFBD,0xBFBE,0xBFBF,/* 0x80-0x87 */ + 0xBFC0,0xBFC1,0xBFC2,0xBFC3,0xBFC4,0xBFC6,0xBFC7,0xBFC8,/* 0x88-0x8F */ + 0xBFC9,0xBFCA,0xBFCB,0xBFCE,0xBFCF,0xBFD1,0xBFD2,0xBFD3,/* 0x90-0x97 */ + 0xBFD5,0xBFD6,0xBFD7,0xBFD8,0xBFD9,0xBFDA,0xBFDB,0xBFDD,/* 0x98-0x9F */ + 0xBFDE,0xBFE0,0xBFE2,0xBFE3,0xBFE4,0xBFE5,0xBFE6,0xBFE7,/* 0xA0-0xA7 */ + 0xBFE8,0xBFE9,0xBFEA,0xBFEB,0xBFEC,0xBFED,0xBFEE,0xBFEF,/* 0xA8-0xAF */ + 0xBFF0,0xBFF1,0xBFF2,0xBFF3,0xBFF4,0xBFF5,0xBFF6,0xBFF7,/* 0xB0-0xB7 */ + 0xBFF8,0xBFF9,0xBFFA,0xBFFB,0xBFFC,0xBFFD,0xBFFE,0xBFFF,/* 0xB8-0xBF */ + 0xC000,0xC001,0xC002,0xC003,0xC004,0xC005,0xC006,0xC007,/* 0xC0-0xC7 */ + 0xC008,0xC009,0xC00A,0xC00B,0xC00C,0xC00D,0xC00E,0xC00F,/* 0xC8-0xCF */ + 0xC010,0xC011,0xC012,0xC013,0xC014,0xC015,0xC016,0xC017,/* 0xD0-0xD7 */ + 0xC018,0xC019,0xC01A,0xC01B,0xC01C,0xC01D,0xC01E,0xC01F,/* 0xD8-0xDF */ + 0xC020,0xC021,0xC022,0xC023,0xC024,0xC025,0xC026,0xC027,/* 0xE0-0xE7 */ + 0xC028,0xC029,0xC02A,0xC02B,0xC02C,0xC02D,0xC02E,0xC02F,/* 0xE8-0xEF */ + 0xC030,0xC031,0xC032,0xC033,0xC034,0xC035,0xC036,0xC037,/* 0xF0-0xF7 */ + 0xC038,0xC039,0xC03A,0xC03B,0xC03D,0xC03E,0xC03F,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_98[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xC040,0xC041,0xC042,0xC043,0xC044,0xC045,0xC046,/* 0x40-0x47 */ + 0xC047,0xC048,0xC049,0xC04A,0xC04B,0xC04C,0xC04D,0xC04E,/* 0x48-0x4F */ + 0xC04F,0xC050,0xC052,0xC053,0xC054,0xC055,0xC056,0xC057,/* 0x50-0x57 */ + 0xC059,0xC05A,0xC05B,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xC05D,0xC05E,0xC05F,0xC061,0xC062,0xC063,0xC064,/* 0x60-0x67 */ + 0xC065,0xC066,0xC067,0xC06A,0xC06B,0xC06C,0xC06D,0xC06E,/* 0x68-0x6F */ + 0xC06F,0xC070,0xC071,0xC072,0xC073,0xC074,0xC075,0xC076,/* 0x70-0x77 */ + 0xC077,0xC078,0xC079,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xC07A,0xC07B,0xC07C,0xC07D,0xC07E,0xC07F,0xC080,/* 0x80-0x87 */ + 0xC081,0xC082,0xC083,0xC084,0xC085,0xC086,0xC087,0xC088,/* 0x88-0x8F */ + 0xC089,0xC08A,0xC08B,0xC08C,0xC08D,0xC08E,0xC08F,0xC092,/* 0x90-0x97 */ + 0xC093,0xC095,0xC096,0xC097,0xC099,0xC09A,0xC09B,0xC09C,/* 0x98-0x9F */ + 0xC09D,0xC09E,0xC09F,0xC0A2,0xC0A4,0xC0A6,0xC0A7,0xC0A8,/* 0xA0-0xA7 */ + 0xC0A9,0xC0AA,0xC0AB,0xC0AE,0xC0B1,0xC0B2,0xC0B7,0xC0B8,/* 0xA8-0xAF */ + 0xC0B9,0xC0BA,0xC0BB,0xC0BE,0xC0C2,0xC0C3,0xC0C4,0xC0C6,/* 0xB0-0xB7 */ + 0xC0C7,0xC0CA,0xC0CB,0xC0CD,0xC0CE,0xC0CF,0xC0D1,0xC0D2,/* 0xB8-0xBF */ + 0xC0D3,0xC0D4,0xC0D5,0xC0D6,0xC0D7,0xC0DA,0xC0DE,0xC0DF,/* 0xC0-0xC7 */ + 0xC0E0,0xC0E1,0xC0E2,0xC0E3,0xC0E6,0xC0E7,0xC0E9,0xC0EA,/* 0xC8-0xCF */ + 0xC0EB,0xC0ED,0xC0EE,0xC0EF,0xC0F0,0xC0F1,0xC0F2,0xC0F3,/* 0xD0-0xD7 */ + 0xC0F6,0xC0F8,0xC0FA,0xC0FB,0xC0FC,0xC0FD,0xC0FE,0xC0FF,/* 0xD8-0xDF */ + 0xC101,0xC102,0xC103,0xC105,0xC106,0xC107,0xC109,0xC10A,/* 0xE0-0xE7 */ + 0xC10B,0xC10C,0xC10D,0xC10E,0xC10F,0xC111,0xC112,0xC113,/* 0xE8-0xEF */ + 0xC114,0xC116,0xC117,0xC118,0xC119,0xC11A,0xC11B,0xC121,/* 0xF0-0xF7 */ + 0xC122,0xC125,0xC128,0xC129,0xC12A,0xC12B,0xC12E,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_99[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xC132,0xC133,0xC134,0xC135,0xC137,0xC13A,0xC13B,/* 0x40-0x47 */ + 0xC13D,0xC13E,0xC13F,0xC141,0xC142,0xC143,0xC144,0xC145,/* 0x48-0x4F */ + 0xC146,0xC147,0xC14A,0xC14E,0xC14F,0xC150,0xC151,0xC152,/* 0x50-0x57 */ + 0xC153,0xC156,0xC157,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xC159,0xC15A,0xC15B,0xC15D,0xC15E,0xC15F,0xC160,/* 0x60-0x67 */ + 0xC161,0xC162,0xC163,0xC166,0xC16A,0xC16B,0xC16C,0xC16D,/* 0x68-0x6F */ + 0xC16E,0xC16F,0xC171,0xC172,0xC173,0xC175,0xC176,0xC177,/* 0x70-0x77 */ + 0xC179,0xC17A,0xC17B,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xC17C,0xC17D,0xC17E,0xC17F,0xC180,0xC181,0xC182,/* 0x80-0x87 */ + 0xC183,0xC184,0xC186,0xC187,0xC188,0xC189,0xC18A,0xC18B,/* 0x88-0x8F */ + 0xC18F,0xC191,0xC192,0xC193,0xC195,0xC197,0xC198,0xC199,/* 0x90-0x97 */ + 0xC19A,0xC19B,0xC19E,0xC1A0,0xC1A2,0xC1A3,0xC1A4,0xC1A6,/* 0x98-0x9F */ + 0xC1A7,0xC1AA,0xC1AB,0xC1AD,0xC1AE,0xC1AF,0xC1B1,0xC1B2,/* 0xA0-0xA7 */ + 0xC1B3,0xC1B4,0xC1B5,0xC1B6,0xC1B7,0xC1B8,0xC1B9,0xC1BA,/* 0xA8-0xAF */ + 0xC1BB,0xC1BC,0xC1BE,0xC1BF,0xC1C0,0xC1C1,0xC1C2,0xC1C3,/* 0xB0-0xB7 */ + 0xC1C5,0xC1C6,0xC1C7,0xC1C9,0xC1CA,0xC1CB,0xC1CD,0xC1CE,/* 0xB8-0xBF */ + 0xC1CF,0xC1D0,0xC1D1,0xC1D2,0xC1D3,0xC1D5,0xC1D6,0xC1D9,/* 0xC0-0xC7 */ + 0xC1DA,0xC1DB,0xC1DC,0xC1DD,0xC1DE,0xC1DF,0xC1E1,0xC1E2,/* 0xC8-0xCF */ + 0xC1E3,0xC1E5,0xC1E6,0xC1E7,0xC1E9,0xC1EA,0xC1EB,0xC1EC,/* 0xD0-0xD7 */ + 0xC1ED,0xC1EE,0xC1EF,0xC1F2,0xC1F4,0xC1F5,0xC1F6,0xC1F7,/* 0xD8-0xDF */ + 0xC1F8,0xC1F9,0xC1FA,0xC1FB,0xC1FE,0xC1FF,0xC201,0xC202,/* 0xE0-0xE7 */ + 0xC203,0xC205,0xC206,0xC207,0xC208,0xC209,0xC20A,0xC20B,/* 0xE8-0xEF */ + 0xC20E,0xC210,0xC212,0xC213,0xC214,0xC215,0xC216,0xC217,/* 0xF0-0xF7 */ + 0xC21A,0xC21B,0xC21D,0xC21E,0xC221,0xC222,0xC223,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_9A[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xC224,0xC225,0xC226,0xC227,0xC22A,0xC22C,0xC22E,/* 0x40-0x47 */ + 0xC230,0xC233,0xC235,0xC236,0xC237,0xC238,0xC239,0xC23A,/* 0x48-0x4F */ + 0xC23B,0xC23C,0xC23D,0xC23E,0xC23F,0xC240,0xC241,0xC242,/* 0x50-0x57 */ + 0xC243,0xC244,0xC245,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xC246,0xC247,0xC249,0xC24A,0xC24B,0xC24C,0xC24D,/* 0x60-0x67 */ + 0xC24E,0xC24F,0xC252,0xC253,0xC255,0xC256,0xC257,0xC259,/* 0x68-0x6F */ + 0xC25A,0xC25B,0xC25C,0xC25D,0xC25E,0xC25F,0xC261,0xC262,/* 0x70-0x77 */ + 0xC263,0xC264,0xC266,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xC267,0xC268,0xC269,0xC26A,0xC26B,0xC26E,0xC26F,/* 0x80-0x87 */ + 0xC271,0xC272,0xC273,0xC275,0xC276,0xC277,0xC278,0xC279,/* 0x88-0x8F */ + 0xC27A,0xC27B,0xC27E,0xC280,0xC282,0xC283,0xC284,0xC285,/* 0x90-0x97 */ + 0xC286,0xC287,0xC28A,0xC28B,0xC28C,0xC28D,0xC28E,0xC28F,/* 0x98-0x9F */ + 0xC291,0xC292,0xC293,0xC294,0xC295,0xC296,0xC297,0xC299,/* 0xA0-0xA7 */ + 0xC29A,0xC29C,0xC29E,0xC29F,0xC2A0,0xC2A1,0xC2A2,0xC2A3,/* 0xA8-0xAF */ + 0xC2A6,0xC2A7,0xC2A9,0xC2AA,0xC2AB,0xC2AE,0xC2AF,0xC2B0,/* 0xB0-0xB7 */ + 0xC2B1,0xC2B2,0xC2B3,0xC2B6,0xC2B8,0xC2BA,0xC2BB,0xC2BC,/* 0xB8-0xBF */ + 0xC2BD,0xC2BE,0xC2BF,0xC2C0,0xC2C1,0xC2C2,0xC2C3,0xC2C4,/* 0xC0-0xC7 */ + 0xC2C5,0xC2C6,0xC2C7,0xC2C8,0xC2C9,0xC2CA,0xC2CB,0xC2CC,/* 0xC8-0xCF */ + 0xC2CD,0xC2CE,0xC2CF,0xC2D0,0xC2D1,0xC2D2,0xC2D3,0xC2D4,/* 0xD0-0xD7 */ + 0xC2D5,0xC2D6,0xC2D7,0xC2D8,0xC2D9,0xC2DA,0xC2DB,0xC2DE,/* 0xD8-0xDF */ + 0xC2DF,0xC2E1,0xC2E2,0xC2E5,0xC2E6,0xC2E7,0xC2E8,0xC2E9,/* 0xE0-0xE7 */ + 0xC2EA,0xC2EE,0xC2F0,0xC2F2,0xC2F3,0xC2F4,0xC2F5,0xC2F7,/* 0xE8-0xEF */ + 0xC2FA,0xC2FD,0xC2FE,0xC2FF,0xC301,0xC302,0xC303,0xC304,/* 0xF0-0xF7 */ + 0xC305,0xC306,0xC307,0xC30A,0xC30B,0xC30E,0xC30F,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_9B[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xC310,0xC311,0xC312,0xC316,0xC317,0xC319,0xC31A,/* 0x40-0x47 */ + 0xC31B,0xC31D,0xC31E,0xC31F,0xC320,0xC321,0xC322,0xC323,/* 0x48-0x4F */ + 0xC326,0xC327,0xC32A,0xC32B,0xC32C,0xC32D,0xC32E,0xC32F,/* 0x50-0x57 */ + 0xC330,0xC331,0xC332,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xC333,0xC334,0xC335,0xC336,0xC337,0xC338,0xC339,/* 0x60-0x67 */ + 0xC33A,0xC33B,0xC33C,0xC33D,0xC33E,0xC33F,0xC340,0xC341,/* 0x68-0x6F */ + 0xC342,0xC343,0xC344,0xC346,0xC347,0xC348,0xC349,0xC34A,/* 0x70-0x77 */ + 0xC34B,0xC34C,0xC34D,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xC34E,0xC34F,0xC350,0xC351,0xC352,0xC353,0xC354,/* 0x80-0x87 */ + 0xC355,0xC356,0xC357,0xC358,0xC359,0xC35A,0xC35B,0xC35C,/* 0x88-0x8F */ + 0xC35D,0xC35E,0xC35F,0xC360,0xC361,0xC362,0xC363,0xC364,/* 0x90-0x97 */ + 0xC365,0xC366,0xC367,0xC36A,0xC36B,0xC36D,0xC36E,0xC36F,/* 0x98-0x9F */ + 0xC371,0xC373,0xC374,0xC375,0xC376,0xC377,0xC37A,0xC37B,/* 0xA0-0xA7 */ + 0xC37E,0xC37F,0xC380,0xC381,0xC382,0xC383,0xC385,0xC386,/* 0xA8-0xAF */ + 0xC387,0xC389,0xC38A,0xC38B,0xC38D,0xC38E,0xC38F,0xC390,/* 0xB0-0xB7 */ + 0xC391,0xC392,0xC393,0xC394,0xC395,0xC396,0xC397,0xC398,/* 0xB8-0xBF */ + 0xC399,0xC39A,0xC39B,0xC39C,0xC39D,0xC39E,0xC39F,0xC3A0,/* 0xC0-0xC7 */ + 0xC3A1,0xC3A2,0xC3A3,0xC3A4,0xC3A5,0xC3A6,0xC3A7,0xC3A8,/* 0xC8-0xCF */ + 0xC3A9,0xC3AA,0xC3AB,0xC3AC,0xC3AD,0xC3AE,0xC3AF,0xC3B0,/* 0xD0-0xD7 */ + 0xC3B1,0xC3B2,0xC3B3,0xC3B4,0xC3B5,0xC3B6,0xC3B7,0xC3B8,/* 0xD8-0xDF */ + 0xC3B9,0xC3BA,0xC3BB,0xC3BC,0xC3BD,0xC3BE,0xC3BF,0xC3C1,/* 0xE0-0xE7 */ + 0xC3C2,0xC3C3,0xC3C4,0xC3C5,0xC3C6,0xC3C7,0xC3C8,0xC3C9,/* 0xE8-0xEF */ + 0xC3CA,0xC3CB,0xC3CC,0xC3CD,0xC3CE,0xC3CF,0xC3D0,0xC3D1,/* 0xF0-0xF7 */ + 0xC3D2,0xC3D3,0xC3D4,0xC3D5,0xC3D6,0xC3D7,0xC3DA,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_9C[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xC3DB,0xC3DD,0xC3DE,0xC3E1,0xC3E3,0xC3E4,0xC3E5,/* 0x40-0x47 */ + 0xC3E6,0xC3E7,0xC3EA,0xC3EB,0xC3EC,0xC3EE,0xC3EF,0xC3F0,/* 0x48-0x4F */ + 0xC3F1,0xC3F2,0xC3F3,0xC3F6,0xC3F7,0xC3F9,0xC3FA,0xC3FB,/* 0x50-0x57 */ + 0xC3FC,0xC3FD,0xC3FE,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xC3FF,0xC400,0xC401,0xC402,0xC403,0xC404,0xC405,/* 0x60-0x67 */ + 0xC406,0xC407,0xC409,0xC40A,0xC40B,0xC40C,0xC40D,0xC40E,/* 0x68-0x6F */ + 0xC40F,0xC411,0xC412,0xC413,0xC414,0xC415,0xC416,0xC417,/* 0x70-0x77 */ + 0xC418,0xC419,0xC41A,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xC41B,0xC41C,0xC41D,0xC41E,0xC41F,0xC420,0xC421,/* 0x80-0x87 */ + 0xC422,0xC423,0xC425,0xC426,0xC427,0xC428,0xC429,0xC42A,/* 0x88-0x8F */ + 0xC42B,0xC42D,0xC42E,0xC42F,0xC431,0xC432,0xC433,0xC435,/* 0x90-0x97 */ + 0xC436,0xC437,0xC438,0xC439,0xC43A,0xC43B,0xC43E,0xC43F,/* 0x98-0x9F */ + 0xC440,0xC441,0xC442,0xC443,0xC444,0xC445,0xC446,0xC447,/* 0xA0-0xA7 */ + 0xC449,0xC44A,0xC44B,0xC44C,0xC44D,0xC44E,0xC44F,0xC450,/* 0xA8-0xAF */ + 0xC451,0xC452,0xC453,0xC454,0xC455,0xC456,0xC457,0xC458,/* 0xB0-0xB7 */ + 0xC459,0xC45A,0xC45B,0xC45C,0xC45D,0xC45E,0xC45F,0xC460,/* 0xB8-0xBF */ + 0xC461,0xC462,0xC463,0xC466,0xC467,0xC469,0xC46A,0xC46B,/* 0xC0-0xC7 */ + 0xC46D,0xC46E,0xC46F,0xC470,0xC471,0xC472,0xC473,0xC476,/* 0xC8-0xCF */ + 0xC477,0xC478,0xC47A,0xC47B,0xC47C,0xC47D,0xC47E,0xC47F,/* 0xD0-0xD7 */ + 0xC481,0xC482,0xC483,0xC484,0xC485,0xC486,0xC487,0xC488,/* 0xD8-0xDF */ + 0xC489,0xC48A,0xC48B,0xC48C,0xC48D,0xC48E,0xC48F,0xC490,/* 0xE0-0xE7 */ + 0xC491,0xC492,0xC493,0xC495,0xC496,0xC497,0xC498,0xC499,/* 0xE8-0xEF */ + 0xC49A,0xC49B,0xC49D,0xC49E,0xC49F,0xC4A0,0xC4A1,0xC4A2,/* 0xF0-0xF7 */ + 0xC4A3,0xC4A4,0xC4A5,0xC4A6,0xC4A7,0xC4A8,0xC4A9,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_9D[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xC4AA,0xC4AB,0xC4AC,0xC4AD,0xC4AE,0xC4AF,0xC4B0,/* 0x40-0x47 */ + 0xC4B1,0xC4B2,0xC4B3,0xC4B4,0xC4B5,0xC4B6,0xC4B7,0xC4B9,/* 0x48-0x4F */ + 0xC4BA,0xC4BB,0xC4BD,0xC4BE,0xC4BF,0xC4C0,0xC4C1,0xC4C2,/* 0x50-0x57 */ + 0xC4C3,0xC4C4,0xC4C5,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xC4C6,0xC4C7,0xC4C8,0xC4C9,0xC4CA,0xC4CB,0xC4CC,/* 0x60-0x67 */ + 0xC4CD,0xC4CE,0xC4CF,0xC4D0,0xC4D1,0xC4D2,0xC4D3,0xC4D4,/* 0x68-0x6F */ + 0xC4D5,0xC4D6,0xC4D7,0xC4D8,0xC4D9,0xC4DA,0xC4DB,0xC4DC,/* 0x70-0x77 */ + 0xC4DD,0xC4DE,0xC4DF,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xC4E0,0xC4E1,0xC4E2,0xC4E3,0xC4E4,0xC4E5,0xC4E6,/* 0x80-0x87 */ + 0xC4E7,0xC4E8,0xC4EA,0xC4EB,0xC4EC,0xC4ED,0xC4EE,0xC4EF,/* 0x88-0x8F */ + 0xC4F2,0xC4F3,0xC4F5,0xC4F6,0xC4F7,0xC4F9,0xC4FB,0xC4FC,/* 0x90-0x97 */ + 0xC4FD,0xC4FE,0xC502,0xC503,0xC504,0xC505,0xC506,0xC507,/* 0x98-0x9F */ + 0xC508,0xC509,0xC50A,0xC50B,0xC50D,0xC50E,0xC50F,0xC511,/* 0xA0-0xA7 */ + 0xC512,0xC513,0xC515,0xC516,0xC517,0xC518,0xC519,0xC51A,/* 0xA8-0xAF */ + 0xC51B,0xC51D,0xC51E,0xC51F,0xC520,0xC521,0xC522,0xC523,/* 0xB0-0xB7 */ + 0xC524,0xC525,0xC526,0xC527,0xC52A,0xC52B,0xC52D,0xC52E,/* 0xB8-0xBF */ + 0xC52F,0xC531,0xC532,0xC533,0xC534,0xC535,0xC536,0xC537,/* 0xC0-0xC7 */ + 0xC53A,0xC53C,0xC53E,0xC53F,0xC540,0xC541,0xC542,0xC543,/* 0xC8-0xCF */ + 0xC546,0xC547,0xC54B,0xC54F,0xC550,0xC551,0xC552,0xC556,/* 0xD0-0xD7 */ + 0xC55A,0xC55B,0xC55C,0xC55F,0xC562,0xC563,0xC565,0xC566,/* 0xD8-0xDF */ + 0xC567,0xC569,0xC56A,0xC56B,0xC56C,0xC56D,0xC56E,0xC56F,/* 0xE0-0xE7 */ + 0xC572,0xC576,0xC577,0xC578,0xC579,0xC57A,0xC57B,0xC57E,/* 0xE8-0xEF */ + 0xC57F,0xC581,0xC582,0xC583,0xC585,0xC586,0xC588,0xC589,/* 0xF0-0xF7 */ + 0xC58A,0xC58B,0xC58E,0xC590,0xC592,0xC593,0xC594,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_9E[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xC596,0xC599,0xC59A,0xC59B,0xC59D,0xC59E,0xC59F,/* 0x40-0x47 */ + 0xC5A1,0xC5A2,0xC5A3,0xC5A4,0xC5A5,0xC5A6,0xC5A7,0xC5A8,/* 0x48-0x4F */ + 0xC5AA,0xC5AB,0xC5AC,0xC5AD,0xC5AE,0xC5AF,0xC5B0,0xC5B1,/* 0x50-0x57 */ + 0xC5B2,0xC5B3,0xC5B6,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xC5B7,0xC5BA,0xC5BF,0xC5C0,0xC5C1,0xC5C2,0xC5C3,/* 0x60-0x67 */ + 0xC5CB,0xC5CD,0xC5CF,0xC5D2,0xC5D3,0xC5D5,0xC5D6,0xC5D7,/* 0x68-0x6F */ + 0xC5D9,0xC5DA,0xC5DB,0xC5DC,0xC5DD,0xC5DE,0xC5DF,0xC5E2,/* 0x70-0x77 */ + 0xC5E4,0xC5E6,0xC5E7,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xC5E8,0xC5E9,0xC5EA,0xC5EB,0xC5EF,0xC5F1,0xC5F2,/* 0x80-0x87 */ + 0xC5F3,0xC5F5,0xC5F8,0xC5F9,0xC5FA,0xC5FB,0xC602,0xC603,/* 0x88-0x8F */ + 0xC604,0xC609,0xC60A,0xC60B,0xC60D,0xC60E,0xC60F,0xC611,/* 0x90-0x97 */ + 0xC612,0xC613,0xC614,0xC615,0xC616,0xC617,0xC61A,0xC61D,/* 0x98-0x9F */ + 0xC61E,0xC61F,0xC620,0xC621,0xC622,0xC623,0xC626,0xC627,/* 0xA0-0xA7 */ + 0xC629,0xC62A,0xC62B,0xC62F,0xC631,0xC632,0xC636,0xC638,/* 0xA8-0xAF */ + 0xC63A,0xC63C,0xC63D,0xC63E,0xC63F,0xC642,0xC643,0xC645,/* 0xB0-0xB7 */ + 0xC646,0xC647,0xC649,0xC64A,0xC64B,0xC64C,0xC64D,0xC64E,/* 0xB8-0xBF */ + 0xC64F,0xC652,0xC656,0xC657,0xC658,0xC659,0xC65A,0xC65B,/* 0xC0-0xC7 */ + 0xC65E,0xC65F,0xC661,0xC662,0xC663,0xC664,0xC665,0xC666,/* 0xC8-0xCF */ + 0xC667,0xC668,0xC669,0xC66A,0xC66B,0xC66D,0xC66E,0xC670,/* 0xD0-0xD7 */ + 0xC672,0xC673,0xC674,0xC675,0xC676,0xC677,0xC67A,0xC67B,/* 0xD8-0xDF */ + 0xC67D,0xC67E,0xC67F,0xC681,0xC682,0xC683,0xC684,0xC685,/* 0xE0-0xE7 */ + 0xC686,0xC687,0xC68A,0xC68C,0xC68E,0xC68F,0xC690,0xC691,/* 0xE8-0xEF */ + 0xC692,0xC693,0xC696,0xC697,0xC699,0xC69A,0xC69B,0xC69D,/* 0xF0-0xF7 */ + 0xC69E,0xC69F,0xC6A0,0xC6A1,0xC6A2,0xC6A3,0xC6A6,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_9F[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xC6A8,0xC6AA,0xC6AB,0xC6AC,0xC6AD,0xC6AE,0xC6AF,/* 0x40-0x47 */ + 0xC6B2,0xC6B3,0xC6B5,0xC6B6,0xC6B7,0xC6BB,0xC6BC,0xC6BD,/* 0x48-0x4F */ + 0xC6BE,0xC6BF,0xC6C2,0xC6C4,0xC6C6,0xC6C7,0xC6C8,0xC6C9,/* 0x50-0x57 */ + 0xC6CA,0xC6CB,0xC6CE,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xC6CF,0xC6D1,0xC6D2,0xC6D3,0xC6D5,0xC6D6,0xC6D7,/* 0x60-0x67 */ + 0xC6D8,0xC6D9,0xC6DA,0xC6DB,0xC6DE,0xC6DF,0xC6E2,0xC6E3,/* 0x68-0x6F */ + 0xC6E4,0xC6E5,0xC6E6,0xC6E7,0xC6EA,0xC6EB,0xC6ED,0xC6EE,/* 0x70-0x77 */ + 0xC6EF,0xC6F1,0xC6F2,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xC6F3,0xC6F4,0xC6F5,0xC6F6,0xC6F7,0xC6FA,0xC6FB,/* 0x80-0x87 */ + 0xC6FC,0xC6FE,0xC6FF,0xC700,0xC701,0xC702,0xC703,0xC706,/* 0x88-0x8F */ + 0xC707,0xC709,0xC70A,0xC70B,0xC70D,0xC70E,0xC70F,0xC710,/* 0x90-0x97 */ + 0xC711,0xC712,0xC713,0xC716,0xC718,0xC71A,0xC71B,0xC71C,/* 0x98-0x9F */ + 0xC71D,0xC71E,0xC71F,0xC722,0xC723,0xC725,0xC726,0xC727,/* 0xA0-0xA7 */ + 0xC729,0xC72A,0xC72B,0xC72C,0xC72D,0xC72E,0xC72F,0xC732,/* 0xA8-0xAF */ + 0xC734,0xC736,0xC738,0xC739,0xC73A,0xC73B,0xC73E,0xC73F,/* 0xB0-0xB7 */ + 0xC741,0xC742,0xC743,0xC745,0xC746,0xC747,0xC748,0xC749,/* 0xB8-0xBF */ + 0xC74B,0xC74E,0xC750,0xC759,0xC75A,0xC75B,0xC75D,0xC75E,/* 0xC0-0xC7 */ + 0xC75F,0xC761,0xC762,0xC763,0xC764,0xC765,0xC766,0xC767,/* 0xC8-0xCF */ + 0xC769,0xC76A,0xC76C,0xC76D,0xC76E,0xC76F,0xC770,0xC771,/* 0xD0-0xD7 */ + 0xC772,0xC773,0xC776,0xC777,0xC779,0xC77A,0xC77B,0xC77F,/* 0xD8-0xDF */ + 0xC780,0xC781,0xC782,0xC786,0xC78B,0xC78C,0xC78D,0xC78F,/* 0xE0-0xE7 */ + 0xC792,0xC793,0xC795,0xC799,0xC79B,0xC79C,0xC79D,0xC79E,/* 0xE8-0xEF */ + 0xC79F,0xC7A2,0xC7A7,0xC7A8,0xC7A9,0xC7AA,0xC7AB,0xC7AE,/* 0xF0-0xF7 */ + 0xC7AF,0xC7B1,0xC7B2,0xC7B3,0xC7B5,0xC7B6,0xC7B7,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_A0[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xC7B8,0xC7B9,0xC7BA,0xC7BB,0xC7BE,0xC7C2,0xC7C3,/* 0x40-0x47 */ + 0xC7C4,0xC7C5,0xC7C6,0xC7C7,0xC7CA,0xC7CB,0xC7CD,0xC7CF,/* 0x48-0x4F */ + 0xC7D1,0xC7D2,0xC7D3,0xC7D4,0xC7D5,0xC7D6,0xC7D7,0xC7D9,/* 0x50-0x57 */ + 0xC7DA,0xC7DB,0xC7DC,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xC7DE,0xC7DF,0xC7E0,0xC7E1,0xC7E2,0xC7E3,0xC7E5,/* 0x60-0x67 */ + 0xC7E6,0xC7E7,0xC7E9,0xC7EA,0xC7EB,0xC7ED,0xC7EE,0xC7EF,/* 0x68-0x6F */ + 0xC7F0,0xC7F1,0xC7F2,0xC7F3,0xC7F4,0xC7F5,0xC7F6,0xC7F7,/* 0x70-0x77 */ + 0xC7F8,0xC7F9,0xC7FA,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xC7FB,0xC7FC,0xC7FD,0xC7FE,0xC7FF,0xC802,0xC803,/* 0x80-0x87 */ + 0xC805,0xC806,0xC807,0xC809,0xC80B,0xC80C,0xC80D,0xC80E,/* 0x88-0x8F */ + 0xC80F,0xC812,0xC814,0xC817,0xC818,0xC819,0xC81A,0xC81B,/* 0x90-0x97 */ + 0xC81E,0xC81F,0xC821,0xC822,0xC823,0xC825,0xC826,0xC827,/* 0x98-0x9F */ + 0xC828,0xC829,0xC82A,0xC82B,0xC82E,0xC830,0xC832,0xC833,/* 0xA0-0xA7 */ + 0xC834,0xC835,0xC836,0xC837,0xC839,0xC83A,0xC83B,0xC83D,/* 0xA8-0xAF */ + 0xC83E,0xC83F,0xC841,0xC842,0xC843,0xC844,0xC845,0xC846,/* 0xB0-0xB7 */ + 0xC847,0xC84A,0xC84B,0xC84E,0xC84F,0xC850,0xC851,0xC852,/* 0xB8-0xBF */ + 0xC853,0xC855,0xC856,0xC857,0xC858,0xC859,0xC85A,0xC85B,/* 0xC0-0xC7 */ + 0xC85C,0xC85D,0xC85E,0xC85F,0xC860,0xC861,0xC862,0xC863,/* 0xC8-0xCF */ + 0xC864,0xC865,0xC866,0xC867,0xC868,0xC869,0xC86A,0xC86B,/* 0xD0-0xD7 */ + 0xC86C,0xC86D,0xC86E,0xC86F,0xC872,0xC873,0xC875,0xC876,/* 0xD8-0xDF */ + 0xC877,0xC879,0xC87B,0xC87C,0xC87D,0xC87E,0xC87F,0xC882,/* 0xE0-0xE7 */ + 0xC884,0xC888,0xC889,0xC88A,0xC88E,0xC88F,0xC890,0xC891,/* 0xE8-0xEF */ + 0xC892,0xC893,0xC895,0xC896,0xC897,0xC898,0xC899,0xC89A,/* 0xF0-0xF7 */ + 0xC89B,0xC89C,0xC89E,0xC8A0,0xC8A2,0xC8A3,0xC8A4,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_A1[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xC8A5,0xC8A6,0xC8A7,0xC8A9,0xC8AA,0xC8AB,0xC8AC,/* 0x40-0x47 */ + 0xC8AD,0xC8AE,0xC8AF,0xC8B0,0xC8B1,0xC8B2,0xC8B3,0xC8B4,/* 0x48-0x4F */ + 0xC8B5,0xC8B6,0xC8B7,0xC8B8,0xC8B9,0xC8BA,0xC8BB,0xC8BE,/* 0x50-0x57 */ + 0xC8BF,0xC8C0,0xC8C1,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xC8C2,0xC8C3,0xC8C5,0xC8C6,0xC8C7,0xC8C9,0xC8CA,/* 0x60-0x67 */ + 0xC8CB,0xC8CD,0xC8CE,0xC8CF,0xC8D0,0xC8D1,0xC8D2,0xC8D3,/* 0x68-0x6F */ + 0xC8D6,0xC8D8,0xC8DA,0xC8DB,0xC8DC,0xC8DD,0xC8DE,0xC8DF,/* 0x70-0x77 */ + 0xC8E2,0xC8E3,0xC8E5,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xC8E6,0xC8E7,0xC8E8,0xC8E9,0xC8EA,0xC8EB,0xC8EC,/* 0x80-0x87 */ + 0xC8ED,0xC8EE,0xC8EF,0xC8F0,0xC8F1,0xC8F2,0xC8F3,0xC8F4,/* 0x88-0x8F */ + 0xC8F6,0xC8F7,0xC8F8,0xC8F9,0xC8FA,0xC8FB,0xC8FE,0xC8FF,/* 0x90-0x97 */ + 0xC901,0xC902,0xC903,0xC907,0xC908,0xC909,0xC90A,0xC90B,/* 0x98-0x9F */ + 0xC90E,0x3000,0x3001,0x3002,0x00B7,0x2025,0x2026,0x00A8,/* 0xA0-0xA7 */ + 0x3003,0x00AD,0x2015,0x2225,0xFF3C,0x223C,0x2018,0x2019,/* 0xA8-0xAF */ + 0x201C,0x201D,0x3014,0x3015,0x3008,0x3009,0x300A,0x300B,/* 0xB0-0xB7 */ + 0x300C,0x300D,0x300E,0x300F,0x3010,0x3011,0x00B1,0x00D7,/* 0xB8-0xBF */ + 0x00F7,0x2260,0x2264,0x2265,0x221E,0x2234,0x00B0,0x2032,/* 0xC0-0xC7 */ + 0x2033,0x2103,0x212B,0xFFE0,0xFFE1,0xFFE5,0x2642,0x2640,/* 0xC8-0xCF */ + 0x2220,0x22A5,0x2312,0x2202,0x2207,0x2261,0x2252,0x00A7,/* 0xD0-0xD7 */ + 0x203B,0x2606,0x2605,0x25CB,0x25CF,0x25CE,0x25C7,0x25C6,/* 0xD8-0xDF */ + 0x25A1,0x25A0,0x25B3,0x25B2,0x25BD,0x25BC,0x2192,0x2190,/* 0xE0-0xE7 */ + 0x2191,0x2193,0x2194,0x3013,0x226A,0x226B,0x221A,0x223D,/* 0xE8-0xEF */ + 0x221D,0x2235,0x222B,0x222C,0x2208,0x220B,0x2286,0x2287,/* 0xF0-0xF7 */ + 0x2282,0x2283,0x222A,0x2229,0x2227,0x2228,0xFFE2,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_A2[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xC910,0xC912,0xC913,0xC914,0xC915,0xC916,0xC917,/* 0x40-0x47 */ + 0xC919,0xC91A,0xC91B,0xC91C,0xC91D,0xC91E,0xC91F,0xC920,/* 0x48-0x4F */ + 0xC921,0xC922,0xC923,0xC924,0xC925,0xC926,0xC927,0xC928,/* 0x50-0x57 */ + 0xC929,0xC92A,0xC92B,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xC92D,0xC92E,0xC92F,0xC930,0xC931,0xC932,0xC933,/* 0x60-0x67 */ + 0xC935,0xC936,0xC937,0xC938,0xC939,0xC93A,0xC93B,0xC93C,/* 0x68-0x6F */ + 0xC93D,0xC93E,0xC93F,0xC940,0xC941,0xC942,0xC943,0xC944,/* 0x70-0x77 */ + 0xC945,0xC946,0xC947,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xC948,0xC949,0xC94A,0xC94B,0xC94C,0xC94D,0xC94E,/* 0x80-0x87 */ + 0xC94F,0xC952,0xC953,0xC955,0xC956,0xC957,0xC959,0xC95A,/* 0x88-0x8F */ + 0xC95B,0xC95C,0xC95D,0xC95E,0xC95F,0xC962,0xC964,0xC965,/* 0x90-0x97 */ + 0xC966,0xC967,0xC968,0xC969,0xC96A,0xC96B,0xC96D,0xC96E,/* 0x98-0x9F */ + 0xC96F,0x21D2,0x21D4,0x2200,0x2203,0x00B4,0xFF5E,0x02C7,/* 0xA0-0xA7 */ + 0x02D8,0x02DD,0x02DA,0x02D9,0x00B8,0x02DB,0x00A1,0x00BF,/* 0xA8-0xAF */ + 0x02D0,0x222E,0x2211,0x220F,0x00A4,0x2109,0x2030,0x25C1,/* 0xB0-0xB7 */ + 0x25C0,0x25B7,0x25B6,0x2664,0x2660,0x2661,0x2665,0x2667,/* 0xB8-0xBF */ + 0x2663,0x2299,0x25C8,0x25A3,0x25D0,0x25D1,0x2592,0x25A4,/* 0xC0-0xC7 */ + 0x25A5,0x25A8,0x25A7,0x25A6,0x25A9,0x2668,0x260F,0x260E,/* 0xC8-0xCF */ + 0x261C,0x261E,0x00B6,0x2020,0x2021,0x2195,0x2197,0x2199,/* 0xD0-0xD7 */ + 0x2196,0x2198,0x266D,0x2669,0x266A,0x266C,0x327F,0x321C,/* 0xD8-0xDF */ + 0x2116,0x33C7,0x2122,0x33C2,0x33D8,0x2121,0x20AC,0x00AE,/* 0xE0-0xE7 */ +}; + +static const wchar_t c2u_A3[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xC971,0xC972,0xC973,0xC975,0xC976,0xC977,0xC978,/* 0x40-0x47 */ + 0xC979,0xC97A,0xC97B,0xC97D,0xC97E,0xC97F,0xC980,0xC981,/* 0x48-0x4F */ + 0xC982,0xC983,0xC984,0xC985,0xC986,0xC987,0xC98A,0xC98B,/* 0x50-0x57 */ + 0xC98D,0xC98E,0xC98F,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xC991,0xC992,0xC993,0xC994,0xC995,0xC996,0xC997,/* 0x60-0x67 */ + 0xC99A,0xC99C,0xC99E,0xC99F,0xC9A0,0xC9A1,0xC9A2,0xC9A3,/* 0x68-0x6F */ + 0xC9A4,0xC9A5,0xC9A6,0xC9A7,0xC9A8,0xC9A9,0xC9AA,0xC9AB,/* 0x70-0x77 */ + 0xC9AC,0xC9AD,0xC9AE,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xC9AF,0xC9B0,0xC9B1,0xC9B2,0xC9B3,0xC9B4,0xC9B5,/* 0x80-0x87 */ + 0xC9B6,0xC9B7,0xC9B8,0xC9B9,0xC9BA,0xC9BB,0xC9BC,0xC9BD,/* 0x88-0x8F */ + 0xC9BE,0xC9BF,0xC9C2,0xC9C3,0xC9C5,0xC9C6,0xC9C9,0xC9CB,/* 0x90-0x97 */ + 0xC9CC,0xC9CD,0xC9CE,0xC9CF,0xC9D2,0xC9D4,0xC9D7,0xC9D8,/* 0x98-0x9F */ + 0xC9DB,0xFF01,0xFF02,0xFF03,0xFF04,0xFF05,0xFF06,0xFF07,/* 0xA0-0xA7 */ + 0xFF08,0xFF09,0xFF0A,0xFF0B,0xFF0C,0xFF0D,0xFF0E,0xFF0F,/* 0xA8-0xAF */ + 0xFF10,0xFF11,0xFF12,0xFF13,0xFF14,0xFF15,0xFF16,0xFF17,/* 0xB0-0xB7 */ + 0xFF18,0xFF19,0xFF1A,0xFF1B,0xFF1C,0xFF1D,0xFF1E,0xFF1F,/* 0xB8-0xBF */ + 0xFF20,0xFF21,0xFF22,0xFF23,0xFF24,0xFF25,0xFF26,0xFF27,/* 0xC0-0xC7 */ + 0xFF28,0xFF29,0xFF2A,0xFF2B,0xFF2C,0xFF2D,0xFF2E,0xFF2F,/* 0xC8-0xCF */ + 0xFF30,0xFF31,0xFF32,0xFF33,0xFF34,0xFF35,0xFF36,0xFF37,/* 0xD0-0xD7 */ + 0xFF38,0xFF39,0xFF3A,0xFF3B,0xFFE6,0xFF3D,0xFF3E,0xFF3F,/* 0xD8-0xDF */ + 0xFF40,0xFF41,0xFF42,0xFF43,0xFF44,0xFF45,0xFF46,0xFF47,/* 0xE0-0xE7 */ + 0xFF48,0xFF49,0xFF4A,0xFF4B,0xFF4C,0xFF4D,0xFF4E,0xFF4F,/* 0xE8-0xEF */ + 0xFF50,0xFF51,0xFF52,0xFF53,0xFF54,0xFF55,0xFF56,0xFF57,/* 0xF0-0xF7 */ + 0xFF58,0xFF59,0xFF5A,0xFF5B,0xFF5C,0xFF5D,0xFFE3,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_A4[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xC9DE,0xC9DF,0xC9E1,0xC9E3,0xC9E5,0xC9E6,0xC9E8,/* 0x40-0x47 */ + 0xC9E9,0xC9EA,0xC9EB,0xC9EE,0xC9F2,0xC9F3,0xC9F4,0xC9F5,/* 0x48-0x4F */ + 0xC9F6,0xC9F7,0xC9FA,0xC9FB,0xC9FD,0xC9FE,0xC9FF,0xCA01,/* 0x50-0x57 */ + 0xCA02,0xCA03,0xCA04,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xCA05,0xCA06,0xCA07,0xCA0A,0xCA0E,0xCA0F,0xCA10,/* 0x60-0x67 */ + 0xCA11,0xCA12,0xCA13,0xCA15,0xCA16,0xCA17,0xCA19,0xCA1A,/* 0x68-0x6F */ + 0xCA1B,0xCA1C,0xCA1D,0xCA1E,0xCA1F,0xCA20,0xCA21,0xCA22,/* 0x70-0x77 */ + 0xCA23,0xCA24,0xCA25,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xCA26,0xCA27,0xCA28,0xCA2A,0xCA2B,0xCA2C,0xCA2D,/* 0x80-0x87 */ + 0xCA2E,0xCA2F,0xCA30,0xCA31,0xCA32,0xCA33,0xCA34,0xCA35,/* 0x88-0x8F */ + 0xCA36,0xCA37,0xCA38,0xCA39,0xCA3A,0xCA3B,0xCA3C,0xCA3D,/* 0x90-0x97 */ + 0xCA3E,0xCA3F,0xCA40,0xCA41,0xCA42,0xCA43,0xCA44,0xCA45,/* 0x98-0x9F */ + 0xCA46,0x3131,0x3132,0x3133,0x3134,0x3135,0x3136,0x3137,/* 0xA0-0xA7 */ + 0x3138,0x3139,0x313A,0x313B,0x313C,0x313D,0x313E,0x313F,/* 0xA8-0xAF */ + 0x3140,0x3141,0x3142,0x3143,0x3144,0x3145,0x3146,0x3147,/* 0xB0-0xB7 */ + 0x3148,0x3149,0x314A,0x314B,0x314C,0x314D,0x314E,0x314F,/* 0xB8-0xBF */ + 0x3150,0x3151,0x3152,0x3153,0x3154,0x3155,0x3156,0x3157,/* 0xC0-0xC7 */ + 0x3158,0x3159,0x315A,0x315B,0x315C,0x315D,0x315E,0x315F,/* 0xC8-0xCF */ + 0x3160,0x3161,0x3162,0x3163,0x3164,0x3165,0x3166,0x3167,/* 0xD0-0xD7 */ + 0x3168,0x3169,0x316A,0x316B,0x316C,0x316D,0x316E,0x316F,/* 0xD8-0xDF */ + 0x3170,0x3171,0x3172,0x3173,0x3174,0x3175,0x3176,0x3177,/* 0xE0-0xE7 */ + 0x3178,0x3179,0x317A,0x317B,0x317C,0x317D,0x317E,0x317F,/* 0xE8-0xEF */ + 0x3180,0x3181,0x3182,0x3183,0x3184,0x3185,0x3186,0x3187,/* 0xF0-0xF7 */ + 0x3188,0x3189,0x318A,0x318B,0x318C,0x318D,0x318E,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_A5[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xCA47,0xCA48,0xCA49,0xCA4A,0xCA4B,0xCA4E,0xCA4F,/* 0x40-0x47 */ + 0xCA51,0xCA52,0xCA53,0xCA55,0xCA56,0xCA57,0xCA58,0xCA59,/* 0x48-0x4F */ + 0xCA5A,0xCA5B,0xCA5E,0xCA62,0xCA63,0xCA64,0xCA65,0xCA66,/* 0x50-0x57 */ + 0xCA67,0xCA69,0xCA6A,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xCA6B,0xCA6C,0xCA6D,0xCA6E,0xCA6F,0xCA70,0xCA71,/* 0x60-0x67 */ + 0xCA72,0xCA73,0xCA74,0xCA75,0xCA76,0xCA77,0xCA78,0xCA79,/* 0x68-0x6F */ + 0xCA7A,0xCA7B,0xCA7C,0xCA7E,0xCA7F,0xCA80,0xCA81,0xCA82,/* 0x70-0x77 */ + 0xCA83,0xCA85,0xCA86,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xCA87,0xCA88,0xCA89,0xCA8A,0xCA8B,0xCA8C,0xCA8D,/* 0x80-0x87 */ + 0xCA8E,0xCA8F,0xCA90,0xCA91,0xCA92,0xCA93,0xCA94,0xCA95,/* 0x88-0x8F */ + 0xCA96,0xCA97,0xCA99,0xCA9A,0xCA9B,0xCA9C,0xCA9D,0xCA9E,/* 0x90-0x97 */ + 0xCA9F,0xCAA0,0xCAA1,0xCAA2,0xCAA3,0xCAA4,0xCAA5,0xCAA6,/* 0x98-0x9F */ + 0xCAA7,0x2170,0x2171,0x2172,0x2173,0x2174,0x2175,0x2176,/* 0xA0-0xA7 */ + 0x2177,0x2178,0x2179,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xA8-0xAF */ + 0x2160,0x2161,0x2162,0x2163,0x2164,0x2165,0x2166,0x2167,/* 0xB0-0xB7 */ + 0x2168,0x2169,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xB8-0xBF */ + 0x0000,0x0391,0x0392,0x0393,0x0394,0x0395,0x0396,0x0397,/* 0xC0-0xC7 */ + 0x0398,0x0399,0x039A,0x039B,0x039C,0x039D,0x039E,0x039F,/* 0xC8-0xCF */ + 0x03A0,0x03A1,0x03A3,0x03A4,0x03A5,0x03A6,0x03A7,0x03A8,/* 0xD0-0xD7 */ + 0x03A9,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xD8-0xDF */ + 0x0000,0x03B1,0x03B2,0x03B3,0x03B4,0x03B5,0x03B6,0x03B7,/* 0xE0-0xE7 */ + 0x03B8,0x03B9,0x03BA,0x03BB,0x03BC,0x03BD,0x03BE,0x03BF,/* 0xE8-0xEF */ + 0x03C0,0x03C1,0x03C3,0x03C4,0x03C5,0x03C6,0x03C7,0x03C8,/* 0xF0-0xF7 */ + 0x03C9,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_A6[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xCAA8,0xCAA9,0xCAAA,0xCAAB,0xCAAC,0xCAAD,0xCAAE,/* 0x40-0x47 */ + 0xCAAF,0xCAB0,0xCAB1,0xCAB2,0xCAB3,0xCAB4,0xCAB5,0xCAB6,/* 0x48-0x4F */ + 0xCAB7,0xCAB8,0xCAB9,0xCABA,0xCABB,0xCABE,0xCABF,0xCAC1,/* 0x50-0x57 */ + 0xCAC2,0xCAC3,0xCAC5,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xCAC6,0xCAC7,0xCAC8,0xCAC9,0xCACA,0xCACB,0xCACE,/* 0x60-0x67 */ + 0xCAD0,0xCAD2,0xCAD4,0xCAD5,0xCAD6,0xCAD7,0xCADA,0xCADB,/* 0x68-0x6F */ + 0xCADC,0xCADD,0xCADE,0xCADF,0xCAE1,0xCAE2,0xCAE3,0xCAE4,/* 0x70-0x77 */ + 0xCAE5,0xCAE6,0xCAE7,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xCAE8,0xCAE9,0xCAEA,0xCAEB,0xCAED,0xCAEE,0xCAEF,/* 0x80-0x87 */ + 0xCAF0,0xCAF1,0xCAF2,0xCAF3,0xCAF5,0xCAF6,0xCAF7,0xCAF8,/* 0x88-0x8F */ + 0xCAF9,0xCAFA,0xCAFB,0xCAFC,0xCAFD,0xCAFE,0xCAFF,0xCB00,/* 0x90-0x97 */ + 0xCB01,0xCB02,0xCB03,0xCB04,0xCB05,0xCB06,0xCB07,0xCB09,/* 0x98-0x9F */ + 0xCB0A,0x2500,0x2502,0x250C,0x2510,0x2518,0x2514,0x251C,/* 0xA0-0xA7 */ + 0x252C,0x2524,0x2534,0x253C,0x2501,0x2503,0x250F,0x2513,/* 0xA8-0xAF */ + 0x251B,0x2517,0x2523,0x2533,0x252B,0x253B,0x254B,0x2520,/* 0xB0-0xB7 */ + 0x252F,0x2528,0x2537,0x253F,0x251D,0x2530,0x2525,0x2538,/* 0xB8-0xBF */ + 0x2542,0x2512,0x2511,0x251A,0x2519,0x2516,0x2515,0x250E,/* 0xC0-0xC7 */ + 0x250D,0x251E,0x251F,0x2521,0x2522,0x2526,0x2527,0x2529,/* 0xC8-0xCF */ + 0x252A,0x252D,0x252E,0x2531,0x2532,0x2535,0x2536,0x2539,/* 0xD0-0xD7 */ + 0x253A,0x253D,0x253E,0x2540,0x2541,0x2543,0x2544,0x2545,/* 0xD8-0xDF */ + 0x2546,0x2547,0x2548,0x2549,0x254A,0x0000,0x0000,0x0000,/* 0xE0-0xE7 */ +}; + +static const wchar_t c2u_A7[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xCB0B,0xCB0C,0xCB0D,0xCB0E,0xCB0F,0xCB11,0xCB12,/* 0x40-0x47 */ + 0xCB13,0xCB15,0xCB16,0xCB17,0xCB19,0xCB1A,0xCB1B,0xCB1C,/* 0x48-0x4F */ + 0xCB1D,0xCB1E,0xCB1F,0xCB22,0xCB23,0xCB24,0xCB25,0xCB26,/* 0x50-0x57 */ + 0xCB27,0xCB28,0xCB29,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xCB2A,0xCB2B,0xCB2C,0xCB2D,0xCB2E,0xCB2F,0xCB30,/* 0x60-0x67 */ + 0xCB31,0xCB32,0xCB33,0xCB34,0xCB35,0xCB36,0xCB37,0xCB38,/* 0x68-0x6F */ + 0xCB39,0xCB3A,0xCB3B,0xCB3C,0xCB3D,0xCB3E,0xCB3F,0xCB40,/* 0x70-0x77 */ + 0xCB42,0xCB43,0xCB44,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xCB45,0xCB46,0xCB47,0xCB4A,0xCB4B,0xCB4D,0xCB4E,/* 0x80-0x87 */ + 0xCB4F,0xCB51,0xCB52,0xCB53,0xCB54,0xCB55,0xCB56,0xCB57,/* 0x88-0x8F */ + 0xCB5A,0xCB5B,0xCB5C,0xCB5E,0xCB5F,0xCB60,0xCB61,0xCB62,/* 0x90-0x97 */ + 0xCB63,0xCB65,0xCB66,0xCB67,0xCB68,0xCB69,0xCB6A,0xCB6B,/* 0x98-0x9F */ + 0xCB6C,0x3395,0x3396,0x3397,0x2113,0x3398,0x33C4,0x33A3,/* 0xA0-0xA7 */ + 0x33A4,0x33A5,0x33A6,0x3399,0x339A,0x339B,0x339C,0x339D,/* 0xA8-0xAF */ + 0x339E,0x339F,0x33A0,0x33A1,0x33A2,0x33CA,0x338D,0x338E,/* 0xB0-0xB7 */ + 0x338F,0x33CF,0x3388,0x3389,0x33C8,0x33A7,0x33A8,0x33B0,/* 0xB8-0xBF */ + 0x33B1,0x33B2,0x33B3,0x33B4,0x33B5,0x33B6,0x33B7,0x33B8,/* 0xC0-0xC7 */ + 0x33B9,0x3380,0x3381,0x3382,0x3383,0x3384,0x33BA,0x33BB,/* 0xC8-0xCF */ + 0x33BC,0x33BD,0x33BE,0x33BF,0x3390,0x3391,0x3392,0x3393,/* 0xD0-0xD7 */ + 0x3394,0x2126,0x33C0,0x33C1,0x338A,0x338B,0x338C,0x33D6,/* 0xD8-0xDF */ + 0x33C5,0x33AD,0x33AE,0x33AF,0x33DB,0x33A9,0x33AA,0x33AB,/* 0xE0-0xE7 */ + 0x33AC,0x33DD,0x33D0,0x33D3,0x33C3,0x33C9,0x33DC,0x33C6,/* 0xE8-0xEF */ +}; + +static const wchar_t c2u_A8[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xCB6D,0xCB6E,0xCB6F,0xCB70,0xCB71,0xCB72,0xCB73,/* 0x40-0x47 */ + 0xCB74,0xCB75,0xCB76,0xCB77,0xCB7A,0xCB7B,0xCB7C,0xCB7D,/* 0x48-0x4F */ + 0xCB7E,0xCB7F,0xCB80,0xCB81,0xCB82,0xCB83,0xCB84,0xCB85,/* 0x50-0x57 */ + 0xCB86,0xCB87,0xCB88,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xCB89,0xCB8A,0xCB8B,0xCB8C,0xCB8D,0xCB8E,0xCB8F,/* 0x60-0x67 */ + 0xCB90,0xCB91,0xCB92,0xCB93,0xCB94,0xCB95,0xCB96,0xCB97,/* 0x68-0x6F */ + 0xCB98,0xCB99,0xCB9A,0xCB9B,0xCB9D,0xCB9E,0xCB9F,0xCBA0,/* 0x70-0x77 */ + 0xCBA1,0xCBA2,0xCBA3,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xCBA4,0xCBA5,0xCBA6,0xCBA7,0xCBA8,0xCBA9,0xCBAA,/* 0x80-0x87 */ + 0xCBAB,0xCBAC,0xCBAD,0xCBAE,0xCBAF,0xCBB0,0xCBB1,0xCBB2,/* 0x88-0x8F */ + 0xCBB3,0xCBB4,0xCBB5,0xCBB6,0xCBB7,0xCBB9,0xCBBA,0xCBBB,/* 0x90-0x97 */ + 0xCBBC,0xCBBD,0xCBBE,0xCBBF,0xCBC0,0xCBC1,0xCBC2,0xCBC3,/* 0x98-0x9F */ + 0xCBC4,0x00C6,0x00D0,0x00AA,0x0126,0x0000,0x0132,0x0000,/* 0xA0-0xA7 */ + 0x013F,0x0141,0x00D8,0x0152,0x00BA,0x00DE,0x0166,0x014A,/* 0xA8-0xAF */ + 0x0000,0x3260,0x3261,0x3262,0x3263,0x3264,0x3265,0x3266,/* 0xB0-0xB7 */ + 0x3267,0x3268,0x3269,0x326A,0x326B,0x326C,0x326D,0x326E,/* 0xB8-0xBF */ + 0x326F,0x3270,0x3271,0x3272,0x3273,0x3274,0x3275,0x3276,/* 0xC0-0xC7 */ + 0x3277,0x3278,0x3279,0x327A,0x327B,0x24D0,0x24D1,0x24D2,/* 0xC8-0xCF */ + 0x24D3,0x24D4,0x24D5,0x24D6,0x24D7,0x24D8,0x24D9,0x24DA,/* 0xD0-0xD7 */ + 0x24DB,0x24DC,0x24DD,0x24DE,0x24DF,0x24E0,0x24E1,0x24E2,/* 0xD8-0xDF */ + 0x24E3,0x24E4,0x24E5,0x24E6,0x24E7,0x24E8,0x24E9,0x2460,/* 0xE0-0xE7 */ + 0x2461,0x2462,0x2463,0x2464,0x2465,0x2466,0x2467,0x2468,/* 0xE8-0xEF */ + 0x2469,0x246A,0x246B,0x246C,0x246D,0x246E,0x00BD,0x2153,/* 0xF0-0xF7 */ + 0x2154,0x00BC,0x00BE,0x215B,0x215C,0x215D,0x215E,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_A9[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xCBC5,0xCBC6,0xCBC7,0xCBC8,0xCBC9,0xCBCA,0xCBCB,/* 0x40-0x47 */ + 0xCBCC,0xCBCD,0xCBCE,0xCBCF,0xCBD0,0xCBD1,0xCBD2,0xCBD3,/* 0x48-0x4F */ + 0xCBD5,0xCBD6,0xCBD7,0xCBD8,0xCBD9,0xCBDA,0xCBDB,0xCBDC,/* 0x50-0x57 */ + 0xCBDD,0xCBDE,0xCBDF,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xCBE0,0xCBE1,0xCBE2,0xCBE3,0xCBE5,0xCBE6,0xCBE8,/* 0x60-0x67 */ + 0xCBEA,0xCBEB,0xCBEC,0xCBED,0xCBEE,0xCBEF,0xCBF0,0xCBF1,/* 0x68-0x6F */ + 0xCBF2,0xCBF3,0xCBF4,0xCBF5,0xCBF6,0xCBF7,0xCBF8,0xCBF9,/* 0x70-0x77 */ + 0xCBFA,0xCBFB,0xCBFC,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xCBFD,0xCBFE,0xCBFF,0xCC00,0xCC01,0xCC02,0xCC03,/* 0x80-0x87 */ + 0xCC04,0xCC05,0xCC06,0xCC07,0xCC08,0xCC09,0xCC0A,0xCC0B,/* 0x88-0x8F */ + 0xCC0E,0xCC0F,0xCC11,0xCC12,0xCC13,0xCC15,0xCC16,0xCC17,/* 0x90-0x97 */ + 0xCC18,0xCC19,0xCC1A,0xCC1B,0xCC1E,0xCC1F,0xCC20,0xCC23,/* 0x98-0x9F */ + 0xCC24,0x00E6,0x0111,0x00F0,0x0127,0x0131,0x0133,0x0138,/* 0xA0-0xA7 */ + 0x0140,0x0142,0x00F8,0x0153,0x00DF,0x00FE,0x0167,0x014B,/* 0xA8-0xAF */ + 0x0149,0x3200,0x3201,0x3202,0x3203,0x3204,0x3205,0x3206,/* 0xB0-0xB7 */ + 0x3207,0x3208,0x3209,0x320A,0x320B,0x320C,0x320D,0x320E,/* 0xB8-0xBF */ + 0x320F,0x3210,0x3211,0x3212,0x3213,0x3214,0x3215,0x3216,/* 0xC0-0xC7 */ + 0x3217,0x3218,0x3219,0x321A,0x321B,0x249C,0x249D,0x249E,/* 0xC8-0xCF */ + 0x249F,0x24A0,0x24A1,0x24A2,0x24A3,0x24A4,0x24A5,0x24A6,/* 0xD0-0xD7 */ + 0x24A7,0x24A8,0x24A9,0x24AA,0x24AB,0x24AC,0x24AD,0x24AE,/* 0xD8-0xDF */ + 0x24AF,0x24B0,0x24B1,0x24B2,0x24B3,0x24B4,0x24B5,0x2474,/* 0xE0-0xE7 */ + 0x2475,0x2476,0x2477,0x2478,0x2479,0x247A,0x247B,0x247C,/* 0xE8-0xEF */ + 0x247D,0x247E,0x247F,0x2480,0x2481,0x2482,0x00B9,0x00B2,/* 0xF0-0xF7 */ + 0x00B3,0x2074,0x207F,0x2081,0x2082,0x2083,0x2084,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_AA[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xCC25,0xCC26,0xCC2A,0xCC2B,0xCC2D,0xCC2F,0xCC31,/* 0x40-0x47 */ + 0xCC32,0xCC33,0xCC34,0xCC35,0xCC36,0xCC37,0xCC3A,0xCC3F,/* 0x48-0x4F */ + 0xCC40,0xCC41,0xCC42,0xCC43,0xCC46,0xCC47,0xCC49,0xCC4A,/* 0x50-0x57 */ + 0xCC4B,0xCC4D,0xCC4E,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xCC4F,0xCC50,0xCC51,0xCC52,0xCC53,0xCC56,0xCC5A,/* 0x60-0x67 */ + 0xCC5B,0xCC5C,0xCC5D,0xCC5E,0xCC5F,0xCC61,0xCC62,0xCC63,/* 0x68-0x6F */ + 0xCC65,0xCC67,0xCC69,0xCC6A,0xCC6B,0xCC6C,0xCC6D,0xCC6E,/* 0x70-0x77 */ + 0xCC6F,0xCC71,0xCC72,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xCC73,0xCC74,0xCC76,0xCC77,0xCC78,0xCC79,0xCC7A,/* 0x80-0x87 */ + 0xCC7B,0xCC7C,0xCC7D,0xCC7E,0xCC7F,0xCC80,0xCC81,0xCC82,/* 0x88-0x8F */ + 0xCC83,0xCC84,0xCC85,0xCC86,0xCC87,0xCC88,0xCC89,0xCC8A,/* 0x90-0x97 */ + 0xCC8B,0xCC8C,0xCC8D,0xCC8E,0xCC8F,0xCC90,0xCC91,0xCC92,/* 0x98-0x9F */ + 0xCC93,0x3041,0x3042,0x3043,0x3044,0x3045,0x3046,0x3047,/* 0xA0-0xA7 */ + 0x3048,0x3049,0x304A,0x304B,0x304C,0x304D,0x304E,0x304F,/* 0xA8-0xAF */ + 0x3050,0x3051,0x3052,0x3053,0x3054,0x3055,0x3056,0x3057,/* 0xB0-0xB7 */ + 0x3058,0x3059,0x305A,0x305B,0x305C,0x305D,0x305E,0x305F,/* 0xB8-0xBF */ + 0x3060,0x3061,0x3062,0x3063,0x3064,0x3065,0x3066,0x3067,/* 0xC0-0xC7 */ + 0x3068,0x3069,0x306A,0x306B,0x306C,0x306D,0x306E,0x306F,/* 0xC8-0xCF */ + 0x3070,0x3071,0x3072,0x3073,0x3074,0x3075,0x3076,0x3077,/* 0xD0-0xD7 */ + 0x3078,0x3079,0x307A,0x307B,0x307C,0x307D,0x307E,0x307F,/* 0xD8-0xDF */ + 0x3080,0x3081,0x3082,0x3083,0x3084,0x3085,0x3086,0x3087,/* 0xE0-0xE7 */ + 0x3088,0x3089,0x308A,0x308B,0x308C,0x308D,0x308E,0x308F,/* 0xE8-0xEF */ + 0x3090,0x3091,0x3092,0x3093,0x0000,0x0000,0x0000,0x0000,/* 0xF0-0xF7 */ +}; + +static const wchar_t c2u_AB[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xCC94,0xCC95,0xCC96,0xCC97,0xCC9A,0xCC9B,0xCC9D,/* 0x40-0x47 */ + 0xCC9E,0xCC9F,0xCCA1,0xCCA2,0xCCA3,0xCCA4,0xCCA5,0xCCA6,/* 0x48-0x4F */ + 0xCCA7,0xCCAA,0xCCAE,0xCCAF,0xCCB0,0xCCB1,0xCCB2,0xCCB3,/* 0x50-0x57 */ + 0xCCB6,0xCCB7,0xCCB9,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xCCBA,0xCCBB,0xCCBD,0xCCBE,0xCCBF,0xCCC0,0xCCC1,/* 0x60-0x67 */ + 0xCCC2,0xCCC3,0xCCC6,0xCCC8,0xCCCA,0xCCCB,0xCCCC,0xCCCD,/* 0x68-0x6F */ + 0xCCCE,0xCCCF,0xCCD1,0xCCD2,0xCCD3,0xCCD5,0xCCD6,0xCCD7,/* 0x70-0x77 */ + 0xCCD8,0xCCD9,0xCCDA,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xCCDB,0xCCDC,0xCCDD,0xCCDE,0xCCDF,0xCCE0,0xCCE1,/* 0x80-0x87 */ + 0xCCE2,0xCCE3,0xCCE5,0xCCE6,0xCCE7,0xCCE8,0xCCE9,0xCCEA,/* 0x88-0x8F */ + 0xCCEB,0xCCED,0xCCEE,0xCCEF,0xCCF1,0xCCF2,0xCCF3,0xCCF4,/* 0x90-0x97 */ + 0xCCF5,0xCCF6,0xCCF7,0xCCF8,0xCCF9,0xCCFA,0xCCFB,0xCCFC,/* 0x98-0x9F */ + 0xCCFD,0x30A1,0x30A2,0x30A3,0x30A4,0x30A5,0x30A6,0x30A7,/* 0xA0-0xA7 */ + 0x30A8,0x30A9,0x30AA,0x30AB,0x30AC,0x30AD,0x30AE,0x30AF,/* 0xA8-0xAF */ + 0x30B0,0x30B1,0x30B2,0x30B3,0x30B4,0x30B5,0x30B6,0x30B7,/* 0xB0-0xB7 */ + 0x30B8,0x30B9,0x30BA,0x30BB,0x30BC,0x30BD,0x30BE,0x30BF,/* 0xB8-0xBF */ + 0x30C0,0x30C1,0x30C2,0x30C3,0x30C4,0x30C5,0x30C6,0x30C7,/* 0xC0-0xC7 */ + 0x30C8,0x30C9,0x30CA,0x30CB,0x30CC,0x30CD,0x30CE,0x30CF,/* 0xC8-0xCF */ + 0x30D0,0x30D1,0x30D2,0x30D3,0x30D4,0x30D5,0x30D6,0x30D7,/* 0xD0-0xD7 */ + 0x30D8,0x30D9,0x30DA,0x30DB,0x30DC,0x30DD,0x30DE,0x30DF,/* 0xD8-0xDF */ + 0x30E0,0x30E1,0x30E2,0x30E3,0x30E4,0x30E5,0x30E6,0x30E7,/* 0xE0-0xE7 */ + 0x30E8,0x30E9,0x30EA,0x30EB,0x30EC,0x30ED,0x30EE,0x30EF,/* 0xE8-0xEF */ + 0x30F0,0x30F1,0x30F2,0x30F3,0x30F4,0x30F5,0x30F6,0x0000,/* 0xF0-0xF7 */ +}; + +static const wchar_t c2u_AC[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xCCFE,0xCCFF,0xCD00,0xCD02,0xCD03,0xCD04,0xCD05,/* 0x40-0x47 */ + 0xCD06,0xCD07,0xCD0A,0xCD0B,0xCD0D,0xCD0E,0xCD0F,0xCD11,/* 0x48-0x4F */ + 0xCD12,0xCD13,0xCD14,0xCD15,0xCD16,0xCD17,0xCD1A,0xCD1C,/* 0x50-0x57 */ + 0xCD1E,0xCD1F,0xCD20,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xCD21,0xCD22,0xCD23,0xCD25,0xCD26,0xCD27,0xCD29,/* 0x60-0x67 */ + 0xCD2A,0xCD2B,0xCD2D,0xCD2E,0xCD2F,0xCD30,0xCD31,0xCD32,/* 0x68-0x6F */ + 0xCD33,0xCD34,0xCD35,0xCD36,0xCD37,0xCD38,0xCD3A,0xCD3B,/* 0x70-0x77 */ + 0xCD3C,0xCD3D,0xCD3E,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xCD3F,0xCD40,0xCD41,0xCD42,0xCD43,0xCD44,0xCD45,/* 0x80-0x87 */ + 0xCD46,0xCD47,0xCD48,0xCD49,0xCD4A,0xCD4B,0xCD4C,0xCD4D,/* 0x88-0x8F */ + 0xCD4E,0xCD4F,0xCD50,0xCD51,0xCD52,0xCD53,0xCD54,0xCD55,/* 0x90-0x97 */ + 0xCD56,0xCD57,0xCD58,0xCD59,0xCD5A,0xCD5B,0xCD5D,0xCD5E,/* 0x98-0x9F */ + 0xCD5F,0x0410,0x0411,0x0412,0x0413,0x0414,0x0415,0x0401,/* 0xA0-0xA7 */ + 0x0416,0x0417,0x0418,0x0419,0x041A,0x041B,0x041C,0x041D,/* 0xA8-0xAF */ + 0x041E,0x041F,0x0420,0x0421,0x0422,0x0423,0x0424,0x0425,/* 0xB0-0xB7 */ + 0x0426,0x0427,0x0428,0x0429,0x042A,0x042B,0x042C,0x042D,/* 0xB8-0xBF */ + 0x042E,0x042F,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xC0-0xC7 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xC8-0xCF */ + 0x0000,0x0430,0x0431,0x0432,0x0433,0x0434,0x0435,0x0451,/* 0xD0-0xD7 */ + 0x0436,0x0437,0x0438,0x0439,0x043A,0x043B,0x043C,0x043D,/* 0xD8-0xDF */ + 0x043E,0x043F,0x0440,0x0441,0x0442,0x0443,0x0444,0x0445,/* 0xE0-0xE7 */ + 0x0446,0x0447,0x0448,0x0449,0x044A,0x044B,0x044C,0x044D,/* 0xE8-0xEF */ + 0x044E,0x044F,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xF0-0xF7 */ +}; + +static const wchar_t c2u_AD[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xCD61,0xCD62,0xCD63,0xCD65,0xCD66,0xCD67,0xCD68,/* 0x40-0x47 */ + 0xCD69,0xCD6A,0xCD6B,0xCD6E,0xCD70,0xCD72,0xCD73,0xCD74,/* 0x48-0x4F */ + 0xCD75,0xCD76,0xCD77,0xCD79,0xCD7A,0xCD7B,0xCD7C,0xCD7D,/* 0x50-0x57 */ + 0xCD7E,0xCD7F,0xCD80,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xCD81,0xCD82,0xCD83,0xCD84,0xCD85,0xCD86,0xCD87,/* 0x60-0x67 */ + 0xCD89,0xCD8A,0xCD8B,0xCD8C,0xCD8D,0xCD8E,0xCD8F,0xCD90,/* 0x68-0x6F */ + 0xCD91,0xCD92,0xCD93,0xCD96,0xCD97,0xCD99,0xCD9A,0xCD9B,/* 0x70-0x77 */ + 0xCD9D,0xCD9E,0xCD9F,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xCDA0,0xCDA1,0xCDA2,0xCDA3,0xCDA6,0xCDA8,0xCDAA,/* 0x80-0x87 */ + 0xCDAB,0xCDAC,0xCDAD,0xCDAE,0xCDAF,0xCDB1,0xCDB2,0xCDB3,/* 0x88-0x8F */ + 0xCDB4,0xCDB5,0xCDB6,0xCDB7,0xCDB8,0xCDB9,0xCDBA,0xCDBB,/* 0x90-0x97 */ + 0xCDBC,0xCDBD,0xCDBE,0xCDBF,0xCDC0,0xCDC1,0xCDC2,0xCDC3,/* 0x98-0x9F */ + 0xCDC5,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xA0-0xA7 */ +}; + +static const wchar_t c2u_AE[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xCDC6,0xCDC7,0xCDC8,0xCDC9,0xCDCA,0xCDCB,0xCDCD,/* 0x40-0x47 */ + 0xCDCE,0xCDCF,0xCDD1,0xCDD2,0xCDD3,0xCDD4,0xCDD5,0xCDD6,/* 0x48-0x4F */ + 0xCDD7,0xCDD8,0xCDD9,0xCDDA,0xCDDB,0xCDDC,0xCDDD,0xCDDE,/* 0x50-0x57 */ + 0xCDDF,0xCDE0,0xCDE1,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xCDE2,0xCDE3,0xCDE4,0xCDE5,0xCDE6,0xCDE7,0xCDE9,/* 0x60-0x67 */ + 0xCDEA,0xCDEB,0xCDED,0xCDEE,0xCDEF,0xCDF1,0xCDF2,0xCDF3,/* 0x68-0x6F */ + 0xCDF4,0xCDF5,0xCDF6,0xCDF7,0xCDFA,0xCDFC,0xCDFE,0xCDFF,/* 0x70-0x77 */ + 0xCE00,0xCE01,0xCE02,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xCE03,0xCE05,0xCE06,0xCE07,0xCE09,0xCE0A,0xCE0B,/* 0x80-0x87 */ + 0xCE0D,0xCE0E,0xCE0F,0xCE10,0xCE11,0xCE12,0xCE13,0xCE15,/* 0x88-0x8F */ + 0xCE16,0xCE17,0xCE18,0xCE1A,0xCE1B,0xCE1C,0xCE1D,0xCE1E,/* 0x90-0x97 */ + 0xCE1F,0xCE22,0xCE23,0xCE25,0xCE26,0xCE27,0xCE29,0xCE2A,/* 0x98-0x9F */ + 0xCE2B,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xA0-0xA7 */ +}; + +static const wchar_t c2u_AF[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xCE2C,0xCE2D,0xCE2E,0xCE2F,0xCE32,0xCE34,0xCE36,/* 0x40-0x47 */ + 0xCE37,0xCE38,0xCE39,0xCE3A,0xCE3B,0xCE3C,0xCE3D,0xCE3E,/* 0x48-0x4F */ + 0xCE3F,0xCE40,0xCE41,0xCE42,0xCE43,0xCE44,0xCE45,0xCE46,/* 0x50-0x57 */ + 0xCE47,0xCE48,0xCE49,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xCE4A,0xCE4B,0xCE4C,0xCE4D,0xCE4E,0xCE4F,0xCE50,/* 0x60-0x67 */ + 0xCE51,0xCE52,0xCE53,0xCE54,0xCE55,0xCE56,0xCE57,0xCE5A,/* 0x68-0x6F */ + 0xCE5B,0xCE5D,0xCE5E,0xCE62,0xCE63,0xCE64,0xCE65,0xCE66,/* 0x70-0x77 */ + 0xCE67,0xCE6A,0xCE6C,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xCE6E,0xCE6F,0xCE70,0xCE71,0xCE72,0xCE73,0xCE76,/* 0x80-0x87 */ + 0xCE77,0xCE79,0xCE7A,0xCE7B,0xCE7D,0xCE7E,0xCE7F,0xCE80,/* 0x88-0x8F */ + 0xCE81,0xCE82,0xCE83,0xCE86,0xCE88,0xCE8A,0xCE8B,0xCE8C,/* 0x90-0x97 */ + 0xCE8D,0xCE8E,0xCE8F,0xCE92,0xCE93,0xCE95,0xCE96,0xCE97,/* 0x98-0x9F */ + 0xCE99,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xA0-0xA7 */ +}; + +static const wchar_t c2u_B0[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xCE9A,0xCE9B,0xCE9C,0xCE9D,0xCE9E,0xCE9F,0xCEA2,/* 0x40-0x47 */ + 0xCEA6,0xCEA7,0xCEA8,0xCEA9,0xCEAA,0xCEAB,0xCEAE,0xCEAF,/* 0x48-0x4F */ + 0xCEB0,0xCEB1,0xCEB2,0xCEB3,0xCEB4,0xCEB5,0xCEB6,0xCEB7,/* 0x50-0x57 */ + 0xCEB8,0xCEB9,0xCEBA,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xCEBB,0xCEBC,0xCEBD,0xCEBE,0xCEBF,0xCEC0,0xCEC2,/* 0x60-0x67 */ + 0xCEC3,0xCEC4,0xCEC5,0xCEC6,0xCEC7,0xCEC8,0xCEC9,0xCECA,/* 0x68-0x6F */ + 0xCECB,0xCECC,0xCECD,0xCECE,0xCECF,0xCED0,0xCED1,0xCED2,/* 0x70-0x77 */ + 0xCED3,0xCED4,0xCED5,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xCED6,0xCED7,0xCED8,0xCED9,0xCEDA,0xCEDB,0xCEDC,/* 0x80-0x87 */ + 0xCEDD,0xCEDE,0xCEDF,0xCEE0,0xCEE1,0xCEE2,0xCEE3,0xCEE6,/* 0x88-0x8F */ + 0xCEE7,0xCEE9,0xCEEA,0xCEED,0xCEEE,0xCEEF,0xCEF0,0xCEF1,/* 0x90-0x97 */ + 0xCEF2,0xCEF3,0xCEF6,0xCEFA,0xCEFB,0xCEFC,0xCEFD,0xCEFE,/* 0x98-0x9F */ + 0xCEFF,0xAC00,0xAC01,0xAC04,0xAC07,0xAC08,0xAC09,0xAC0A,/* 0xA0-0xA7 */ + 0xAC10,0xAC11,0xAC12,0xAC13,0xAC14,0xAC15,0xAC16,0xAC17,/* 0xA8-0xAF */ + 0xAC19,0xAC1A,0xAC1B,0xAC1C,0xAC1D,0xAC20,0xAC24,0xAC2C,/* 0xB0-0xB7 */ + 0xAC2D,0xAC2F,0xAC30,0xAC31,0xAC38,0xAC39,0xAC3C,0xAC40,/* 0xB8-0xBF */ + 0xAC4B,0xAC4D,0xAC54,0xAC58,0xAC5C,0xAC70,0xAC71,0xAC74,/* 0xC0-0xC7 */ + 0xAC77,0xAC78,0xAC7A,0xAC80,0xAC81,0xAC83,0xAC84,0xAC85,/* 0xC8-0xCF */ + 0xAC86,0xAC89,0xAC8A,0xAC8B,0xAC8C,0xAC90,0xAC94,0xAC9C,/* 0xD0-0xD7 */ + 0xAC9D,0xAC9F,0xACA0,0xACA1,0xACA8,0xACA9,0xACAA,0xACAC,/* 0xD8-0xDF */ + 0xACAF,0xACB0,0xACB8,0xACB9,0xACBB,0xACBC,0xACBD,0xACC1,/* 0xE0-0xE7 */ + 0xACC4,0xACC8,0xACCC,0xACD5,0xACD7,0xACE0,0xACE1,0xACE4,/* 0xE8-0xEF */ + 0xACE7,0xACE8,0xACEA,0xACEC,0xACEF,0xACF0,0xACF1,0xACF3,/* 0xF0-0xF7 */ + 0xACF5,0xACF6,0xACFC,0xACFD,0xAD00,0xAD04,0xAD06,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B1[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xCF02,0xCF03,0xCF05,0xCF06,0xCF07,0xCF09,0xCF0A,/* 0x40-0x47 */ + 0xCF0B,0xCF0C,0xCF0D,0xCF0E,0xCF0F,0xCF12,0xCF14,0xCF16,/* 0x48-0x4F */ + 0xCF17,0xCF18,0xCF19,0xCF1A,0xCF1B,0xCF1D,0xCF1E,0xCF1F,/* 0x50-0x57 */ + 0xCF21,0xCF22,0xCF23,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xCF25,0xCF26,0xCF27,0xCF28,0xCF29,0xCF2A,0xCF2B,/* 0x60-0x67 */ + 0xCF2E,0xCF32,0xCF33,0xCF34,0xCF35,0xCF36,0xCF37,0xCF39,/* 0x68-0x6F */ + 0xCF3A,0xCF3B,0xCF3C,0xCF3D,0xCF3E,0xCF3F,0xCF40,0xCF41,/* 0x70-0x77 */ + 0xCF42,0xCF43,0xCF44,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xCF45,0xCF46,0xCF47,0xCF48,0xCF49,0xCF4A,0xCF4B,/* 0x80-0x87 */ + 0xCF4C,0xCF4D,0xCF4E,0xCF4F,0xCF50,0xCF51,0xCF52,0xCF53,/* 0x88-0x8F */ + 0xCF56,0xCF57,0xCF59,0xCF5A,0xCF5B,0xCF5D,0xCF5E,0xCF5F,/* 0x90-0x97 */ + 0xCF60,0xCF61,0xCF62,0xCF63,0xCF66,0xCF68,0xCF6A,0xCF6B,/* 0x98-0x9F */ + 0xCF6C,0xAD0C,0xAD0D,0xAD0F,0xAD11,0xAD18,0xAD1C,0xAD20,/* 0xA0-0xA7 */ + 0xAD29,0xAD2C,0xAD2D,0xAD34,0xAD35,0xAD38,0xAD3C,0xAD44,/* 0xA8-0xAF */ + 0xAD45,0xAD47,0xAD49,0xAD50,0xAD54,0xAD58,0xAD61,0xAD63,/* 0xB0-0xB7 */ + 0xAD6C,0xAD6D,0xAD70,0xAD73,0xAD74,0xAD75,0xAD76,0xAD7B,/* 0xB8-0xBF */ + 0xAD7C,0xAD7D,0xAD7F,0xAD81,0xAD82,0xAD88,0xAD89,0xAD8C,/* 0xC0-0xC7 */ + 0xAD90,0xAD9C,0xAD9D,0xADA4,0xADB7,0xADC0,0xADC1,0xADC4,/* 0xC8-0xCF */ + 0xADC8,0xADD0,0xADD1,0xADD3,0xADDC,0xADE0,0xADE4,0xADF8,/* 0xD0-0xD7 */ + 0xADF9,0xADFC,0xADFF,0xAE00,0xAE01,0xAE08,0xAE09,0xAE0B,/* 0xD8-0xDF */ + 0xAE0D,0xAE14,0xAE30,0xAE31,0xAE34,0xAE37,0xAE38,0xAE3A,/* 0xE0-0xE7 */ + 0xAE40,0xAE41,0xAE43,0xAE45,0xAE46,0xAE4A,0xAE4C,0xAE4D,/* 0xE8-0xEF */ + 0xAE4E,0xAE50,0xAE54,0xAE56,0xAE5C,0xAE5D,0xAE5F,0xAE60,/* 0xF0-0xF7 */ + 0xAE61,0xAE65,0xAE68,0xAE69,0xAE6C,0xAE70,0xAE78,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B2[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xCF6D,0xCF6E,0xCF6F,0xCF72,0xCF73,0xCF75,0xCF76,/* 0x40-0x47 */ + 0xCF77,0xCF79,0xCF7A,0xCF7B,0xCF7C,0xCF7D,0xCF7E,0xCF7F,/* 0x48-0x4F */ + 0xCF81,0xCF82,0xCF83,0xCF84,0xCF86,0xCF87,0xCF88,0xCF89,/* 0x50-0x57 */ + 0xCF8A,0xCF8B,0xCF8D,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xCF8E,0xCF8F,0xCF90,0xCF91,0xCF92,0xCF93,0xCF94,/* 0x60-0x67 */ + 0xCF95,0xCF96,0xCF97,0xCF98,0xCF99,0xCF9A,0xCF9B,0xCF9C,/* 0x68-0x6F */ + 0xCF9D,0xCF9E,0xCF9F,0xCFA0,0xCFA2,0xCFA3,0xCFA4,0xCFA5,/* 0x70-0x77 */ + 0xCFA6,0xCFA7,0xCFA9,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xCFAA,0xCFAB,0xCFAC,0xCFAD,0xCFAE,0xCFAF,0xCFB1,/* 0x80-0x87 */ + 0xCFB2,0xCFB3,0xCFB4,0xCFB5,0xCFB6,0xCFB7,0xCFB8,0xCFB9,/* 0x88-0x8F */ + 0xCFBA,0xCFBB,0xCFBC,0xCFBD,0xCFBE,0xCFBF,0xCFC0,0xCFC1,/* 0x90-0x97 */ + 0xCFC2,0xCFC3,0xCFC5,0xCFC6,0xCFC7,0xCFC8,0xCFC9,0xCFCA,/* 0x98-0x9F */ + 0xCFCB,0xAE79,0xAE7B,0xAE7C,0xAE7D,0xAE84,0xAE85,0xAE8C,/* 0xA0-0xA7 */ + 0xAEBC,0xAEBD,0xAEBE,0xAEC0,0xAEC4,0xAECC,0xAECD,0xAECF,/* 0xA8-0xAF */ + 0xAED0,0xAED1,0xAED8,0xAED9,0xAEDC,0xAEE8,0xAEEB,0xAEED,/* 0xB0-0xB7 */ + 0xAEF4,0xAEF8,0xAEFC,0xAF07,0xAF08,0xAF0D,0xAF10,0xAF2C,/* 0xB8-0xBF */ + 0xAF2D,0xAF30,0xAF32,0xAF34,0xAF3C,0xAF3D,0xAF3F,0xAF41,/* 0xC0-0xC7 */ + 0xAF42,0xAF43,0xAF48,0xAF49,0xAF50,0xAF5C,0xAF5D,0xAF64,/* 0xC8-0xCF */ + 0xAF65,0xAF79,0xAF80,0xAF84,0xAF88,0xAF90,0xAF91,0xAF95,/* 0xD0-0xD7 */ + 0xAF9C,0xAFB8,0xAFB9,0xAFBC,0xAFC0,0xAFC7,0xAFC8,0xAFC9,/* 0xD8-0xDF */ + 0xAFCB,0xAFCD,0xAFCE,0xAFD4,0xAFDC,0xAFE8,0xAFE9,0xAFF0,/* 0xE0-0xE7 */ + 0xAFF1,0xAFF4,0xAFF8,0xB000,0xB001,0xB004,0xB00C,0xB010,/* 0xE8-0xEF */ + 0xB014,0xB01C,0xB01D,0xB028,0xB044,0xB045,0xB048,0xB04A,/* 0xF0-0xF7 */ + 0xB04C,0xB04E,0xB053,0xB054,0xB055,0xB057,0xB059,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B3[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xCFCC,0xCFCD,0xCFCE,0xCFCF,0xCFD0,0xCFD1,0xCFD2,/* 0x40-0x47 */ + 0xCFD3,0xCFD4,0xCFD5,0xCFD6,0xCFD7,0xCFD8,0xCFD9,0xCFDA,/* 0x48-0x4F */ + 0xCFDB,0xCFDC,0xCFDD,0xCFDE,0xCFDF,0xCFE2,0xCFE3,0xCFE5,/* 0x50-0x57 */ + 0xCFE6,0xCFE7,0xCFE9,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xCFEA,0xCFEB,0xCFEC,0xCFED,0xCFEE,0xCFEF,0xCFF2,/* 0x60-0x67 */ + 0xCFF4,0xCFF6,0xCFF7,0xCFF8,0xCFF9,0xCFFA,0xCFFB,0xCFFD,/* 0x68-0x6F */ + 0xCFFE,0xCFFF,0xD001,0xD002,0xD003,0xD005,0xD006,0xD007,/* 0x70-0x77 */ + 0xD008,0xD009,0xD00A,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xD00B,0xD00C,0xD00D,0xD00E,0xD00F,0xD010,0xD012,/* 0x80-0x87 */ + 0xD013,0xD014,0xD015,0xD016,0xD017,0xD019,0xD01A,0xD01B,/* 0x88-0x8F */ + 0xD01C,0xD01D,0xD01E,0xD01F,0xD020,0xD021,0xD022,0xD023,/* 0x90-0x97 */ + 0xD024,0xD025,0xD026,0xD027,0xD028,0xD029,0xD02A,0xD02B,/* 0x98-0x9F */ + 0xD02C,0xB05D,0xB07C,0xB07D,0xB080,0xB084,0xB08C,0xB08D,/* 0xA0-0xA7 */ + 0xB08F,0xB091,0xB098,0xB099,0xB09A,0xB09C,0xB09F,0xB0A0,/* 0xA8-0xAF */ + 0xB0A1,0xB0A2,0xB0A8,0xB0A9,0xB0AB,0xB0AC,0xB0AD,0xB0AE,/* 0xB0-0xB7 */ + 0xB0AF,0xB0B1,0xB0B3,0xB0B4,0xB0B5,0xB0B8,0xB0BC,0xB0C4,/* 0xB8-0xBF */ + 0xB0C5,0xB0C7,0xB0C8,0xB0C9,0xB0D0,0xB0D1,0xB0D4,0xB0D8,/* 0xC0-0xC7 */ + 0xB0E0,0xB0E5,0xB108,0xB109,0xB10B,0xB10C,0xB110,0xB112,/* 0xC8-0xCF */ + 0xB113,0xB118,0xB119,0xB11B,0xB11C,0xB11D,0xB123,0xB124,/* 0xD0-0xD7 */ + 0xB125,0xB128,0xB12C,0xB134,0xB135,0xB137,0xB138,0xB139,/* 0xD8-0xDF */ + 0xB140,0xB141,0xB144,0xB148,0xB150,0xB151,0xB154,0xB155,/* 0xE0-0xE7 */ + 0xB158,0xB15C,0xB160,0xB178,0xB179,0xB17C,0xB180,0xB182,/* 0xE8-0xEF */ + 0xB188,0xB189,0xB18B,0xB18D,0xB192,0xB193,0xB194,0xB198,/* 0xF0-0xF7 */ + 0xB19C,0xB1A8,0xB1CC,0xB1D0,0xB1D4,0xB1DC,0xB1DD,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B4[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xD02E,0xD02F,0xD030,0xD031,0xD032,0xD033,0xD036,/* 0x40-0x47 */ + 0xD037,0xD039,0xD03A,0xD03B,0xD03D,0xD03E,0xD03F,0xD040,/* 0x48-0x4F */ + 0xD041,0xD042,0xD043,0xD046,0xD048,0xD04A,0xD04B,0xD04C,/* 0x50-0x57 */ + 0xD04D,0xD04E,0xD04F,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xD051,0xD052,0xD053,0xD055,0xD056,0xD057,0xD059,/* 0x60-0x67 */ + 0xD05A,0xD05B,0xD05C,0xD05D,0xD05E,0xD05F,0xD061,0xD062,/* 0x68-0x6F */ + 0xD063,0xD064,0xD065,0xD066,0xD067,0xD068,0xD069,0xD06A,/* 0x70-0x77 */ + 0xD06B,0xD06E,0xD06F,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xD071,0xD072,0xD073,0xD075,0xD076,0xD077,0xD078,/* 0x80-0x87 */ + 0xD079,0xD07A,0xD07B,0xD07E,0xD07F,0xD080,0xD082,0xD083,/* 0x88-0x8F */ + 0xD084,0xD085,0xD086,0xD087,0xD088,0xD089,0xD08A,0xD08B,/* 0x90-0x97 */ + 0xD08C,0xD08D,0xD08E,0xD08F,0xD090,0xD091,0xD092,0xD093,/* 0x98-0x9F */ + 0xD094,0xB1DF,0xB1E8,0xB1E9,0xB1EC,0xB1F0,0xB1F9,0xB1FB,/* 0xA0-0xA7 */ + 0xB1FD,0xB204,0xB205,0xB208,0xB20B,0xB20C,0xB214,0xB215,/* 0xA8-0xAF */ + 0xB217,0xB219,0xB220,0xB234,0xB23C,0xB258,0xB25C,0xB260,/* 0xB0-0xB7 */ + 0xB268,0xB269,0xB274,0xB275,0xB27C,0xB284,0xB285,0xB289,/* 0xB8-0xBF */ + 0xB290,0xB291,0xB294,0xB298,0xB299,0xB29A,0xB2A0,0xB2A1,/* 0xC0-0xC7 */ + 0xB2A3,0xB2A5,0xB2A6,0xB2AA,0xB2AC,0xB2B0,0xB2B4,0xB2C8,/* 0xC8-0xCF */ + 0xB2C9,0xB2CC,0xB2D0,0xB2D2,0xB2D8,0xB2D9,0xB2DB,0xB2DD,/* 0xD0-0xD7 */ + 0xB2E2,0xB2E4,0xB2E5,0xB2E6,0xB2E8,0xB2EB,0xB2EC,0xB2ED,/* 0xD8-0xDF */ + 0xB2EE,0xB2EF,0xB2F3,0xB2F4,0xB2F5,0xB2F7,0xB2F8,0xB2F9,/* 0xE0-0xE7 */ + 0xB2FA,0xB2FB,0xB2FF,0xB300,0xB301,0xB304,0xB308,0xB310,/* 0xE8-0xEF */ + 0xB311,0xB313,0xB314,0xB315,0xB31C,0xB354,0xB355,0xB356,/* 0xF0-0xF7 */ + 0xB358,0xB35B,0xB35C,0xB35E,0xB35F,0xB364,0xB365,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B5[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xD095,0xD096,0xD097,0xD098,0xD099,0xD09A,0xD09B,/* 0x40-0x47 */ + 0xD09C,0xD09D,0xD09E,0xD09F,0xD0A0,0xD0A1,0xD0A2,0xD0A3,/* 0x48-0x4F */ + 0xD0A6,0xD0A7,0xD0A9,0xD0AA,0xD0AB,0xD0AD,0xD0AE,0xD0AF,/* 0x50-0x57 */ + 0xD0B0,0xD0B1,0xD0B2,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xD0B3,0xD0B6,0xD0B8,0xD0BA,0xD0BB,0xD0BC,0xD0BD,/* 0x60-0x67 */ + 0xD0BE,0xD0BF,0xD0C2,0xD0C3,0xD0C5,0xD0C6,0xD0C7,0xD0CA,/* 0x68-0x6F */ + 0xD0CB,0xD0CC,0xD0CD,0xD0CE,0xD0CF,0xD0D2,0xD0D6,0xD0D7,/* 0x70-0x77 */ + 0xD0D8,0xD0D9,0xD0DA,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xD0DB,0xD0DE,0xD0DF,0xD0E1,0xD0E2,0xD0E3,0xD0E5,/* 0x80-0x87 */ + 0xD0E6,0xD0E7,0xD0E8,0xD0E9,0xD0EA,0xD0EB,0xD0EE,0xD0F2,/* 0x88-0x8F */ + 0xD0F3,0xD0F4,0xD0F5,0xD0F6,0xD0F7,0xD0F9,0xD0FA,0xD0FB,/* 0x90-0x97 */ + 0xD0FC,0xD0FD,0xD0FE,0xD0FF,0xD100,0xD101,0xD102,0xD103,/* 0x98-0x9F */ + 0xD104,0xB367,0xB369,0xB36B,0xB36E,0xB370,0xB371,0xB374,/* 0xA0-0xA7 */ + 0xB378,0xB380,0xB381,0xB383,0xB384,0xB385,0xB38C,0xB390,/* 0xA8-0xAF */ + 0xB394,0xB3A0,0xB3A1,0xB3A8,0xB3AC,0xB3C4,0xB3C5,0xB3C8,/* 0xB0-0xB7 */ + 0xB3CB,0xB3CC,0xB3CE,0xB3D0,0xB3D4,0xB3D5,0xB3D7,0xB3D9,/* 0xB8-0xBF */ + 0xB3DB,0xB3DD,0xB3E0,0xB3E4,0xB3E8,0xB3FC,0xB410,0xB418,/* 0xC0-0xC7 */ + 0xB41C,0xB420,0xB428,0xB429,0xB42B,0xB434,0xB450,0xB451,/* 0xC8-0xCF */ + 0xB454,0xB458,0xB460,0xB461,0xB463,0xB465,0xB46C,0xB480,/* 0xD0-0xD7 */ + 0xB488,0xB49D,0xB4A4,0xB4A8,0xB4AC,0xB4B5,0xB4B7,0xB4B9,/* 0xD8-0xDF */ + 0xB4C0,0xB4C4,0xB4C8,0xB4D0,0xB4D5,0xB4DC,0xB4DD,0xB4E0,/* 0xE0-0xE7 */ + 0xB4E3,0xB4E4,0xB4E6,0xB4EC,0xB4ED,0xB4EF,0xB4F1,0xB4F8,/* 0xE8-0xEF */ + 0xB514,0xB515,0xB518,0xB51B,0xB51C,0xB524,0xB525,0xB527,/* 0xF0-0xF7 */ + 0xB528,0xB529,0xB52A,0xB530,0xB531,0xB534,0xB538,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B6[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xD105,0xD106,0xD107,0xD108,0xD109,0xD10A,0xD10B,/* 0x40-0x47 */ + 0xD10C,0xD10E,0xD10F,0xD110,0xD111,0xD112,0xD113,0xD114,/* 0x48-0x4F */ + 0xD115,0xD116,0xD117,0xD118,0xD119,0xD11A,0xD11B,0xD11C,/* 0x50-0x57 */ + 0xD11D,0xD11E,0xD11F,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xD120,0xD121,0xD122,0xD123,0xD124,0xD125,0xD126,/* 0x60-0x67 */ + 0xD127,0xD128,0xD129,0xD12A,0xD12B,0xD12C,0xD12D,0xD12E,/* 0x68-0x6F */ + 0xD12F,0xD132,0xD133,0xD135,0xD136,0xD137,0xD139,0xD13B,/* 0x70-0x77 */ + 0xD13C,0xD13D,0xD13E,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xD13F,0xD142,0xD146,0xD147,0xD148,0xD149,0xD14A,/* 0x80-0x87 */ + 0xD14B,0xD14E,0xD14F,0xD151,0xD152,0xD153,0xD155,0xD156,/* 0x88-0x8F */ + 0xD157,0xD158,0xD159,0xD15A,0xD15B,0xD15E,0xD160,0xD162,/* 0x90-0x97 */ + 0xD163,0xD164,0xD165,0xD166,0xD167,0xD169,0xD16A,0xD16B,/* 0x98-0x9F */ + 0xD16D,0xB540,0xB541,0xB543,0xB544,0xB545,0xB54B,0xB54C,/* 0xA0-0xA7 */ + 0xB54D,0xB550,0xB554,0xB55C,0xB55D,0xB55F,0xB560,0xB561,/* 0xA8-0xAF */ + 0xB5A0,0xB5A1,0xB5A4,0xB5A8,0xB5AA,0xB5AB,0xB5B0,0xB5B1,/* 0xB0-0xB7 */ + 0xB5B3,0xB5B4,0xB5B5,0xB5BB,0xB5BC,0xB5BD,0xB5C0,0xB5C4,/* 0xB8-0xBF */ + 0xB5CC,0xB5CD,0xB5CF,0xB5D0,0xB5D1,0xB5D8,0xB5EC,0xB610,/* 0xC0-0xC7 */ + 0xB611,0xB614,0xB618,0xB625,0xB62C,0xB634,0xB648,0xB664,/* 0xC8-0xCF */ + 0xB668,0xB69C,0xB69D,0xB6A0,0xB6A4,0xB6AB,0xB6AC,0xB6B1,/* 0xD0-0xD7 */ + 0xB6D4,0xB6F0,0xB6F4,0xB6F8,0xB700,0xB701,0xB705,0xB728,/* 0xD8-0xDF */ + 0xB729,0xB72C,0xB72F,0xB730,0xB738,0xB739,0xB73B,0xB744,/* 0xE0-0xE7 */ + 0xB748,0xB74C,0xB754,0xB755,0xB760,0xB764,0xB768,0xB770,/* 0xE8-0xEF */ + 0xB771,0xB773,0xB775,0xB77C,0xB77D,0xB780,0xB784,0xB78C,/* 0xF0-0xF7 */ + 0xB78D,0xB78F,0xB790,0xB791,0xB792,0xB796,0xB797,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B7[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xD16E,0xD16F,0xD170,0xD171,0xD172,0xD173,0xD174,/* 0x40-0x47 */ + 0xD175,0xD176,0xD177,0xD178,0xD179,0xD17A,0xD17B,0xD17D,/* 0x48-0x4F */ + 0xD17E,0xD17F,0xD180,0xD181,0xD182,0xD183,0xD185,0xD186,/* 0x50-0x57 */ + 0xD187,0xD189,0xD18A,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xD18B,0xD18C,0xD18D,0xD18E,0xD18F,0xD190,0xD191,/* 0x60-0x67 */ + 0xD192,0xD193,0xD194,0xD195,0xD196,0xD197,0xD198,0xD199,/* 0x68-0x6F */ + 0xD19A,0xD19B,0xD19C,0xD19D,0xD19E,0xD19F,0xD1A2,0xD1A3,/* 0x70-0x77 */ + 0xD1A5,0xD1A6,0xD1A7,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xD1A9,0xD1AA,0xD1AB,0xD1AC,0xD1AD,0xD1AE,0xD1AF,/* 0x80-0x87 */ + 0xD1B2,0xD1B4,0xD1B6,0xD1B7,0xD1B8,0xD1B9,0xD1BB,0xD1BD,/* 0x88-0x8F */ + 0xD1BE,0xD1BF,0xD1C1,0xD1C2,0xD1C3,0xD1C4,0xD1C5,0xD1C6,/* 0x90-0x97 */ + 0xD1C7,0xD1C8,0xD1C9,0xD1CA,0xD1CB,0xD1CC,0xD1CD,0xD1CE,/* 0x98-0x9F */ + 0xD1CF,0xB798,0xB799,0xB79C,0xB7A0,0xB7A8,0xB7A9,0xB7AB,/* 0xA0-0xA7 */ + 0xB7AC,0xB7AD,0xB7B4,0xB7B5,0xB7B8,0xB7C7,0xB7C9,0xB7EC,/* 0xA8-0xAF */ + 0xB7ED,0xB7F0,0xB7F4,0xB7FC,0xB7FD,0xB7FF,0xB800,0xB801,/* 0xB0-0xB7 */ + 0xB807,0xB808,0xB809,0xB80C,0xB810,0xB818,0xB819,0xB81B,/* 0xB8-0xBF */ + 0xB81D,0xB824,0xB825,0xB828,0xB82C,0xB834,0xB835,0xB837,/* 0xC0-0xC7 */ + 0xB838,0xB839,0xB840,0xB844,0xB851,0xB853,0xB85C,0xB85D,/* 0xC8-0xCF */ + 0xB860,0xB864,0xB86C,0xB86D,0xB86F,0xB871,0xB878,0xB87C,/* 0xD0-0xD7 */ + 0xB88D,0xB8A8,0xB8B0,0xB8B4,0xB8B8,0xB8C0,0xB8C1,0xB8C3,/* 0xD8-0xDF */ + 0xB8C5,0xB8CC,0xB8D0,0xB8D4,0xB8DD,0xB8DF,0xB8E1,0xB8E8,/* 0xE0-0xE7 */ + 0xB8E9,0xB8EC,0xB8F0,0xB8F8,0xB8F9,0xB8FB,0xB8FD,0xB904,/* 0xE8-0xEF */ + 0xB918,0xB920,0xB93C,0xB93D,0xB940,0xB944,0xB94C,0xB94F,/* 0xF0-0xF7 */ + 0xB951,0xB958,0xB959,0xB95C,0xB960,0xB968,0xB969,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B8[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xD1D0,0xD1D1,0xD1D2,0xD1D3,0xD1D4,0xD1D5,0xD1D6,/* 0x40-0x47 */ + 0xD1D7,0xD1D9,0xD1DA,0xD1DB,0xD1DC,0xD1DD,0xD1DE,0xD1DF,/* 0x48-0x4F */ + 0xD1E0,0xD1E1,0xD1E2,0xD1E3,0xD1E4,0xD1E5,0xD1E6,0xD1E7,/* 0x50-0x57 */ + 0xD1E8,0xD1E9,0xD1EA,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xD1EB,0xD1EC,0xD1ED,0xD1EE,0xD1EF,0xD1F0,0xD1F1,/* 0x60-0x67 */ + 0xD1F2,0xD1F3,0xD1F5,0xD1F6,0xD1F7,0xD1F9,0xD1FA,0xD1FB,/* 0x68-0x6F */ + 0xD1FC,0xD1FD,0xD1FE,0xD1FF,0xD200,0xD201,0xD202,0xD203,/* 0x70-0x77 */ + 0xD204,0xD205,0xD206,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xD208,0xD20A,0xD20B,0xD20C,0xD20D,0xD20E,0xD20F,/* 0x80-0x87 */ + 0xD211,0xD212,0xD213,0xD214,0xD215,0xD216,0xD217,0xD218,/* 0x88-0x8F */ + 0xD219,0xD21A,0xD21B,0xD21C,0xD21D,0xD21E,0xD21F,0xD220,/* 0x90-0x97 */ + 0xD221,0xD222,0xD223,0xD224,0xD225,0xD226,0xD227,0xD228,/* 0x98-0x9F */ + 0xD229,0xB96B,0xB96D,0xB974,0xB975,0xB978,0xB97C,0xB984,/* 0xA0-0xA7 */ + 0xB985,0xB987,0xB989,0xB98A,0xB98D,0xB98E,0xB9AC,0xB9AD,/* 0xA8-0xAF */ + 0xB9B0,0xB9B4,0xB9BC,0xB9BD,0xB9BF,0xB9C1,0xB9C8,0xB9C9,/* 0xB0-0xB7 */ + 0xB9CC,0xB9CE,0xB9CF,0xB9D0,0xB9D1,0xB9D2,0xB9D8,0xB9D9,/* 0xB8-0xBF */ + 0xB9DB,0xB9DD,0xB9DE,0xB9E1,0xB9E3,0xB9E4,0xB9E5,0xB9E8,/* 0xC0-0xC7 */ + 0xB9EC,0xB9F4,0xB9F5,0xB9F7,0xB9F8,0xB9F9,0xB9FA,0xBA00,/* 0xC8-0xCF */ + 0xBA01,0xBA08,0xBA15,0xBA38,0xBA39,0xBA3C,0xBA40,0xBA42,/* 0xD0-0xD7 */ + 0xBA48,0xBA49,0xBA4B,0xBA4D,0xBA4E,0xBA53,0xBA54,0xBA55,/* 0xD8-0xDF */ + 0xBA58,0xBA5C,0xBA64,0xBA65,0xBA67,0xBA68,0xBA69,0xBA70,/* 0xE0-0xE7 */ + 0xBA71,0xBA74,0xBA78,0xBA83,0xBA84,0xBA85,0xBA87,0xBA8C,/* 0xE8-0xEF */ + 0xBAA8,0xBAA9,0xBAAB,0xBAAC,0xBAB0,0xBAB2,0xBAB8,0xBAB9,/* 0xF0-0xF7 */ + 0xBABB,0xBABD,0xBAC4,0xBAC8,0xBAD8,0xBAD9,0xBAFC,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B9[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xD22A,0xD22B,0xD22E,0xD22F,0xD231,0xD232,0xD233,/* 0x40-0x47 */ + 0xD235,0xD236,0xD237,0xD238,0xD239,0xD23A,0xD23B,0xD23E,/* 0x48-0x4F */ + 0xD240,0xD242,0xD243,0xD244,0xD245,0xD246,0xD247,0xD249,/* 0x50-0x57 */ + 0xD24A,0xD24B,0xD24C,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xD24D,0xD24E,0xD24F,0xD250,0xD251,0xD252,0xD253,/* 0x60-0x67 */ + 0xD254,0xD255,0xD256,0xD257,0xD258,0xD259,0xD25A,0xD25B,/* 0x68-0x6F */ + 0xD25D,0xD25E,0xD25F,0xD260,0xD261,0xD262,0xD263,0xD265,/* 0x70-0x77 */ + 0xD266,0xD267,0xD268,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xD269,0xD26A,0xD26B,0xD26C,0xD26D,0xD26E,0xD26F,/* 0x80-0x87 */ + 0xD270,0xD271,0xD272,0xD273,0xD274,0xD275,0xD276,0xD277,/* 0x88-0x8F */ + 0xD278,0xD279,0xD27A,0xD27B,0xD27C,0xD27D,0xD27E,0xD27F,/* 0x90-0x97 */ + 0xD282,0xD283,0xD285,0xD286,0xD287,0xD289,0xD28A,0xD28B,/* 0x98-0x9F */ + 0xD28C,0xBB00,0xBB04,0xBB0D,0xBB0F,0xBB11,0xBB18,0xBB1C,/* 0xA0-0xA7 */ + 0xBB20,0xBB29,0xBB2B,0xBB34,0xBB35,0xBB36,0xBB38,0xBB3B,/* 0xA8-0xAF */ + 0xBB3C,0xBB3D,0xBB3E,0xBB44,0xBB45,0xBB47,0xBB49,0xBB4D,/* 0xB0-0xB7 */ + 0xBB4F,0xBB50,0xBB54,0xBB58,0xBB61,0xBB63,0xBB6C,0xBB88,/* 0xB8-0xBF */ + 0xBB8C,0xBB90,0xBBA4,0xBBA8,0xBBAC,0xBBB4,0xBBB7,0xBBC0,/* 0xC0-0xC7 */ + 0xBBC4,0xBBC8,0xBBD0,0xBBD3,0xBBF8,0xBBF9,0xBBFC,0xBBFF,/* 0xC8-0xCF */ + 0xBC00,0xBC02,0xBC08,0xBC09,0xBC0B,0xBC0C,0xBC0D,0xBC0F,/* 0xD0-0xD7 */ + 0xBC11,0xBC14,0xBC15,0xBC16,0xBC17,0xBC18,0xBC1B,0xBC1C,/* 0xD8-0xDF */ + 0xBC1D,0xBC1E,0xBC1F,0xBC24,0xBC25,0xBC27,0xBC29,0xBC2D,/* 0xE0-0xE7 */ + 0xBC30,0xBC31,0xBC34,0xBC38,0xBC40,0xBC41,0xBC43,0xBC44,/* 0xE8-0xEF */ + 0xBC45,0xBC49,0xBC4C,0xBC4D,0xBC50,0xBC5D,0xBC84,0xBC85,/* 0xF0-0xF7 */ + 0xBC88,0xBC8B,0xBC8C,0xBC8E,0xBC94,0xBC95,0xBC97,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_BA[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xD28D,0xD28E,0xD28F,0xD292,0xD293,0xD294,0xD296,/* 0x40-0x47 */ + 0xD297,0xD298,0xD299,0xD29A,0xD29B,0xD29D,0xD29E,0xD29F,/* 0x48-0x4F */ + 0xD2A1,0xD2A2,0xD2A3,0xD2A5,0xD2A6,0xD2A7,0xD2A8,0xD2A9,/* 0x50-0x57 */ + 0xD2AA,0xD2AB,0xD2AD,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xD2AE,0xD2AF,0xD2B0,0xD2B2,0xD2B3,0xD2B4,0xD2B5,/* 0x60-0x67 */ + 0xD2B6,0xD2B7,0xD2BA,0xD2BB,0xD2BD,0xD2BE,0xD2C1,0xD2C3,/* 0x68-0x6F */ + 0xD2C4,0xD2C5,0xD2C6,0xD2C7,0xD2CA,0xD2CC,0xD2CD,0xD2CE,/* 0x70-0x77 */ + 0xD2CF,0xD2D0,0xD2D1,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xD2D2,0xD2D3,0xD2D5,0xD2D6,0xD2D7,0xD2D9,0xD2DA,/* 0x80-0x87 */ + 0xD2DB,0xD2DD,0xD2DE,0xD2DF,0xD2E0,0xD2E1,0xD2E2,0xD2E3,/* 0x88-0x8F */ + 0xD2E6,0xD2E7,0xD2E8,0xD2E9,0xD2EA,0xD2EB,0xD2EC,0xD2ED,/* 0x90-0x97 */ + 0xD2EE,0xD2EF,0xD2F2,0xD2F3,0xD2F5,0xD2F6,0xD2F7,0xD2F9,/* 0x98-0x9F */ + 0xD2FA,0xBC99,0xBC9A,0xBCA0,0xBCA1,0xBCA4,0xBCA7,0xBCA8,/* 0xA0-0xA7 */ + 0xBCB0,0xBCB1,0xBCB3,0xBCB4,0xBCB5,0xBCBC,0xBCBD,0xBCC0,/* 0xA8-0xAF */ + 0xBCC4,0xBCCD,0xBCCF,0xBCD0,0xBCD1,0xBCD5,0xBCD8,0xBCDC,/* 0xB0-0xB7 */ + 0xBCF4,0xBCF5,0xBCF6,0xBCF8,0xBCFC,0xBD04,0xBD05,0xBD07,/* 0xB8-0xBF */ + 0xBD09,0xBD10,0xBD14,0xBD24,0xBD2C,0xBD40,0xBD48,0xBD49,/* 0xC0-0xC7 */ + 0xBD4C,0xBD50,0xBD58,0xBD59,0xBD64,0xBD68,0xBD80,0xBD81,/* 0xC8-0xCF */ + 0xBD84,0xBD87,0xBD88,0xBD89,0xBD8A,0xBD90,0xBD91,0xBD93,/* 0xD0-0xD7 */ + 0xBD95,0xBD99,0xBD9A,0xBD9C,0xBDA4,0xBDB0,0xBDB8,0xBDD4,/* 0xD8-0xDF */ + 0xBDD5,0xBDD8,0xBDDC,0xBDE9,0xBDF0,0xBDF4,0xBDF8,0xBE00,/* 0xE0-0xE7 */ + 0xBE03,0xBE05,0xBE0C,0xBE0D,0xBE10,0xBE14,0xBE1C,0xBE1D,/* 0xE8-0xEF */ + 0xBE1F,0xBE44,0xBE45,0xBE48,0xBE4C,0xBE4E,0xBE54,0xBE55,/* 0xF0-0xF7 */ + 0xBE57,0xBE59,0xBE5A,0xBE5B,0xBE60,0xBE61,0xBE64,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_BB[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xD2FB,0xD2FC,0xD2FD,0xD2FE,0xD2FF,0xD302,0xD304,/* 0x40-0x47 */ + 0xD306,0xD307,0xD308,0xD309,0xD30A,0xD30B,0xD30F,0xD311,/* 0x48-0x4F */ + 0xD312,0xD313,0xD315,0xD317,0xD318,0xD319,0xD31A,0xD31B,/* 0x50-0x57 */ + 0xD31E,0xD322,0xD323,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xD324,0xD326,0xD327,0xD32A,0xD32B,0xD32D,0xD32E,/* 0x60-0x67 */ + 0xD32F,0xD331,0xD332,0xD333,0xD334,0xD335,0xD336,0xD337,/* 0x68-0x6F */ + 0xD33A,0xD33E,0xD33F,0xD340,0xD341,0xD342,0xD343,0xD346,/* 0x70-0x77 */ + 0xD347,0xD348,0xD349,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xD34A,0xD34B,0xD34C,0xD34D,0xD34E,0xD34F,0xD350,/* 0x80-0x87 */ + 0xD351,0xD352,0xD353,0xD354,0xD355,0xD356,0xD357,0xD358,/* 0x88-0x8F */ + 0xD359,0xD35A,0xD35B,0xD35C,0xD35D,0xD35E,0xD35F,0xD360,/* 0x90-0x97 */ + 0xD361,0xD362,0xD363,0xD364,0xD365,0xD366,0xD367,0xD368,/* 0x98-0x9F */ + 0xD369,0xBE68,0xBE6A,0xBE70,0xBE71,0xBE73,0xBE74,0xBE75,/* 0xA0-0xA7 */ + 0xBE7B,0xBE7C,0xBE7D,0xBE80,0xBE84,0xBE8C,0xBE8D,0xBE8F,/* 0xA8-0xAF */ + 0xBE90,0xBE91,0xBE98,0xBE99,0xBEA8,0xBED0,0xBED1,0xBED4,/* 0xB0-0xB7 */ + 0xBED7,0xBED8,0xBEE0,0xBEE3,0xBEE4,0xBEE5,0xBEEC,0xBF01,/* 0xB8-0xBF */ + 0xBF08,0xBF09,0xBF18,0xBF19,0xBF1B,0xBF1C,0xBF1D,0xBF40,/* 0xC0-0xC7 */ + 0xBF41,0xBF44,0xBF48,0xBF50,0xBF51,0xBF55,0xBF94,0xBFB0,/* 0xC8-0xCF */ + 0xBFC5,0xBFCC,0xBFCD,0xBFD0,0xBFD4,0xBFDC,0xBFDF,0xBFE1,/* 0xD0-0xD7 */ + 0xC03C,0xC051,0xC058,0xC05C,0xC060,0xC068,0xC069,0xC090,/* 0xD8-0xDF */ + 0xC091,0xC094,0xC098,0xC0A0,0xC0A1,0xC0A3,0xC0A5,0xC0AC,/* 0xE0-0xE7 */ + 0xC0AD,0xC0AF,0xC0B0,0xC0B3,0xC0B4,0xC0B5,0xC0B6,0xC0BC,/* 0xE8-0xEF */ + 0xC0BD,0xC0BF,0xC0C0,0xC0C1,0xC0C5,0xC0C8,0xC0C9,0xC0CC,/* 0xF0-0xF7 */ + 0xC0D0,0xC0D8,0xC0D9,0xC0DB,0xC0DC,0xC0DD,0xC0E4,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_BC[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xD36A,0xD36B,0xD36C,0xD36D,0xD36E,0xD36F,0xD370,/* 0x40-0x47 */ + 0xD371,0xD372,0xD373,0xD374,0xD375,0xD376,0xD377,0xD378,/* 0x48-0x4F */ + 0xD379,0xD37A,0xD37B,0xD37E,0xD37F,0xD381,0xD382,0xD383,/* 0x50-0x57 */ + 0xD385,0xD386,0xD387,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xD388,0xD389,0xD38A,0xD38B,0xD38E,0xD392,0xD393,/* 0x60-0x67 */ + 0xD394,0xD395,0xD396,0xD397,0xD39A,0xD39B,0xD39D,0xD39E,/* 0x68-0x6F */ + 0xD39F,0xD3A1,0xD3A2,0xD3A3,0xD3A4,0xD3A5,0xD3A6,0xD3A7,/* 0x70-0x77 */ + 0xD3AA,0xD3AC,0xD3AE,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xD3AF,0xD3B0,0xD3B1,0xD3B2,0xD3B3,0xD3B5,0xD3B6,/* 0x80-0x87 */ + 0xD3B7,0xD3B9,0xD3BA,0xD3BB,0xD3BD,0xD3BE,0xD3BF,0xD3C0,/* 0x88-0x8F */ + 0xD3C1,0xD3C2,0xD3C3,0xD3C6,0xD3C7,0xD3CA,0xD3CB,0xD3CC,/* 0x90-0x97 */ + 0xD3CD,0xD3CE,0xD3CF,0xD3D1,0xD3D2,0xD3D3,0xD3D4,0xD3D5,/* 0x98-0x9F */ + 0xD3D6,0xC0E5,0xC0E8,0xC0EC,0xC0F4,0xC0F5,0xC0F7,0xC0F9,/* 0xA0-0xA7 */ + 0xC100,0xC104,0xC108,0xC110,0xC115,0xC11C,0xC11D,0xC11E,/* 0xA8-0xAF */ + 0xC11F,0xC120,0xC123,0xC124,0xC126,0xC127,0xC12C,0xC12D,/* 0xB0-0xB7 */ + 0xC12F,0xC130,0xC131,0xC136,0xC138,0xC139,0xC13C,0xC140,/* 0xB8-0xBF */ + 0xC148,0xC149,0xC14B,0xC14C,0xC14D,0xC154,0xC155,0xC158,/* 0xC0-0xC7 */ + 0xC15C,0xC164,0xC165,0xC167,0xC168,0xC169,0xC170,0xC174,/* 0xC8-0xCF */ + 0xC178,0xC185,0xC18C,0xC18D,0xC18E,0xC190,0xC194,0xC196,/* 0xD0-0xD7 */ + 0xC19C,0xC19D,0xC19F,0xC1A1,0xC1A5,0xC1A8,0xC1A9,0xC1AC,/* 0xD8-0xDF */ + 0xC1B0,0xC1BD,0xC1C4,0xC1C8,0xC1CC,0xC1D4,0xC1D7,0xC1D8,/* 0xE0-0xE7 */ + 0xC1E0,0xC1E4,0xC1E8,0xC1F0,0xC1F1,0xC1F3,0xC1FC,0xC1FD,/* 0xE8-0xEF */ + 0xC200,0xC204,0xC20C,0xC20D,0xC20F,0xC211,0xC218,0xC219,/* 0xF0-0xF7 */ + 0xC21C,0xC21F,0xC220,0xC228,0xC229,0xC22B,0xC22D,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_BD[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xD3D7,0xD3D9,0xD3DA,0xD3DB,0xD3DC,0xD3DD,0xD3DE,/* 0x40-0x47 */ + 0xD3DF,0xD3E0,0xD3E2,0xD3E4,0xD3E5,0xD3E6,0xD3E7,0xD3E8,/* 0x48-0x4F */ + 0xD3E9,0xD3EA,0xD3EB,0xD3EE,0xD3EF,0xD3F1,0xD3F2,0xD3F3,/* 0x50-0x57 */ + 0xD3F5,0xD3F6,0xD3F7,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xD3F8,0xD3F9,0xD3FA,0xD3FB,0xD3FE,0xD400,0xD402,/* 0x60-0x67 */ + 0xD403,0xD404,0xD405,0xD406,0xD407,0xD409,0xD40A,0xD40B,/* 0x68-0x6F */ + 0xD40C,0xD40D,0xD40E,0xD40F,0xD410,0xD411,0xD412,0xD413,/* 0x70-0x77 */ + 0xD414,0xD415,0xD416,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xD417,0xD418,0xD419,0xD41A,0xD41B,0xD41C,0xD41E,/* 0x80-0x87 */ + 0xD41F,0xD420,0xD421,0xD422,0xD423,0xD424,0xD425,0xD426,/* 0x88-0x8F */ + 0xD427,0xD428,0xD429,0xD42A,0xD42B,0xD42C,0xD42D,0xD42E,/* 0x90-0x97 */ + 0xD42F,0xD430,0xD431,0xD432,0xD433,0xD434,0xD435,0xD436,/* 0x98-0x9F */ + 0xD437,0xC22F,0xC231,0xC232,0xC234,0xC248,0xC250,0xC251,/* 0xA0-0xA7 */ + 0xC254,0xC258,0xC260,0xC265,0xC26C,0xC26D,0xC270,0xC274,/* 0xA8-0xAF */ + 0xC27C,0xC27D,0xC27F,0xC281,0xC288,0xC289,0xC290,0xC298,/* 0xB0-0xB7 */ + 0xC29B,0xC29D,0xC2A4,0xC2A5,0xC2A8,0xC2AC,0xC2AD,0xC2B4,/* 0xB8-0xBF */ + 0xC2B5,0xC2B7,0xC2B9,0xC2DC,0xC2DD,0xC2E0,0xC2E3,0xC2E4,/* 0xC0-0xC7 */ + 0xC2EB,0xC2EC,0xC2ED,0xC2EF,0xC2F1,0xC2F6,0xC2F8,0xC2F9,/* 0xC8-0xCF */ + 0xC2FB,0xC2FC,0xC300,0xC308,0xC309,0xC30C,0xC30D,0xC313,/* 0xD0-0xD7 */ + 0xC314,0xC315,0xC318,0xC31C,0xC324,0xC325,0xC328,0xC329,/* 0xD8-0xDF */ + 0xC345,0xC368,0xC369,0xC36C,0xC370,0xC372,0xC378,0xC379,/* 0xE0-0xE7 */ + 0xC37C,0xC37D,0xC384,0xC388,0xC38C,0xC3C0,0xC3D8,0xC3D9,/* 0xE8-0xEF */ + 0xC3DC,0xC3DF,0xC3E0,0xC3E2,0xC3E8,0xC3E9,0xC3ED,0xC3F4,/* 0xF0-0xF7 */ + 0xC3F5,0xC3F8,0xC408,0xC410,0xC424,0xC42C,0xC430,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_BE[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xD438,0xD439,0xD43A,0xD43B,0xD43C,0xD43D,0xD43E,/* 0x40-0x47 */ + 0xD43F,0xD441,0xD442,0xD443,0xD445,0xD446,0xD447,0xD448,/* 0x48-0x4F */ + 0xD449,0xD44A,0xD44B,0xD44C,0xD44D,0xD44E,0xD44F,0xD450,/* 0x50-0x57 */ + 0xD451,0xD452,0xD453,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xD454,0xD455,0xD456,0xD457,0xD458,0xD459,0xD45A,/* 0x60-0x67 */ + 0xD45B,0xD45D,0xD45E,0xD45F,0xD461,0xD462,0xD463,0xD465,/* 0x68-0x6F */ + 0xD466,0xD467,0xD468,0xD469,0xD46A,0xD46B,0xD46C,0xD46E,/* 0x70-0x77 */ + 0xD470,0xD471,0xD472,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xD473,0xD474,0xD475,0xD476,0xD477,0xD47A,0xD47B,/* 0x80-0x87 */ + 0xD47D,0xD47E,0xD481,0xD483,0xD484,0xD485,0xD486,0xD487,/* 0x88-0x8F */ + 0xD48A,0xD48C,0xD48E,0xD48F,0xD490,0xD491,0xD492,0xD493,/* 0x90-0x97 */ + 0xD495,0xD496,0xD497,0xD498,0xD499,0xD49A,0xD49B,0xD49C,/* 0x98-0x9F */ + 0xD49D,0xC434,0xC43C,0xC43D,0xC448,0xC464,0xC465,0xC468,/* 0xA0-0xA7 */ + 0xC46C,0xC474,0xC475,0xC479,0xC480,0xC494,0xC49C,0xC4B8,/* 0xA8-0xAF */ + 0xC4BC,0xC4E9,0xC4F0,0xC4F1,0xC4F4,0xC4F8,0xC4FA,0xC4FF,/* 0xB0-0xB7 */ + 0xC500,0xC501,0xC50C,0xC510,0xC514,0xC51C,0xC528,0xC529,/* 0xB8-0xBF */ + 0xC52C,0xC530,0xC538,0xC539,0xC53B,0xC53D,0xC544,0xC545,/* 0xC0-0xC7 */ + 0xC548,0xC549,0xC54A,0xC54C,0xC54D,0xC54E,0xC553,0xC554,/* 0xC8-0xCF */ + 0xC555,0xC557,0xC558,0xC559,0xC55D,0xC55E,0xC560,0xC561,/* 0xD0-0xD7 */ + 0xC564,0xC568,0xC570,0xC571,0xC573,0xC574,0xC575,0xC57C,/* 0xD8-0xDF */ + 0xC57D,0xC580,0xC584,0xC587,0xC58C,0xC58D,0xC58F,0xC591,/* 0xE0-0xE7 */ + 0xC595,0xC597,0xC598,0xC59C,0xC5A0,0xC5A9,0xC5B4,0xC5B5,/* 0xE8-0xEF */ + 0xC5B8,0xC5B9,0xC5BB,0xC5BC,0xC5BD,0xC5BE,0xC5C4,0xC5C5,/* 0xF0-0xF7 */ + 0xC5C6,0xC5C7,0xC5C8,0xC5C9,0xC5CA,0xC5CC,0xC5CE,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_BF[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xD49E,0xD49F,0xD4A0,0xD4A1,0xD4A2,0xD4A3,0xD4A4,/* 0x40-0x47 */ + 0xD4A5,0xD4A6,0xD4A7,0xD4A8,0xD4AA,0xD4AB,0xD4AC,0xD4AD,/* 0x48-0x4F */ + 0xD4AE,0xD4AF,0xD4B0,0xD4B1,0xD4B2,0xD4B3,0xD4B4,0xD4B5,/* 0x50-0x57 */ + 0xD4B6,0xD4B7,0xD4B8,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xD4B9,0xD4BA,0xD4BB,0xD4BC,0xD4BD,0xD4BE,0xD4BF,/* 0x60-0x67 */ + 0xD4C0,0xD4C1,0xD4C2,0xD4C3,0xD4C4,0xD4C5,0xD4C6,0xD4C7,/* 0x68-0x6F */ + 0xD4C8,0xD4C9,0xD4CA,0xD4CB,0xD4CD,0xD4CE,0xD4CF,0xD4D1,/* 0x70-0x77 */ + 0xD4D2,0xD4D3,0xD4D5,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xD4D6,0xD4D7,0xD4D8,0xD4D9,0xD4DA,0xD4DB,0xD4DD,/* 0x80-0x87 */ + 0xD4DE,0xD4E0,0xD4E1,0xD4E2,0xD4E3,0xD4E4,0xD4E5,0xD4E6,/* 0x88-0x8F */ + 0xD4E7,0xD4E9,0xD4EA,0xD4EB,0xD4ED,0xD4EE,0xD4EF,0xD4F1,/* 0x90-0x97 */ + 0xD4F2,0xD4F3,0xD4F4,0xD4F5,0xD4F6,0xD4F7,0xD4F9,0xD4FA,/* 0x98-0x9F */ + 0xD4FC,0xC5D0,0xC5D1,0xC5D4,0xC5D8,0xC5E0,0xC5E1,0xC5E3,/* 0xA0-0xA7 */ + 0xC5E5,0xC5EC,0xC5ED,0xC5EE,0xC5F0,0xC5F4,0xC5F6,0xC5F7,/* 0xA8-0xAF */ + 0xC5FC,0xC5FD,0xC5FE,0xC5FF,0xC600,0xC601,0xC605,0xC606,/* 0xB0-0xB7 */ + 0xC607,0xC608,0xC60C,0xC610,0xC618,0xC619,0xC61B,0xC61C,/* 0xB8-0xBF */ + 0xC624,0xC625,0xC628,0xC62C,0xC62D,0xC62E,0xC630,0xC633,/* 0xC0-0xC7 */ + 0xC634,0xC635,0xC637,0xC639,0xC63B,0xC640,0xC641,0xC644,/* 0xC8-0xCF */ + 0xC648,0xC650,0xC651,0xC653,0xC654,0xC655,0xC65C,0xC65D,/* 0xD0-0xD7 */ + 0xC660,0xC66C,0xC66F,0xC671,0xC678,0xC679,0xC67C,0xC680,/* 0xD8-0xDF */ + 0xC688,0xC689,0xC68B,0xC68D,0xC694,0xC695,0xC698,0xC69C,/* 0xE0-0xE7 */ + 0xC6A4,0xC6A5,0xC6A7,0xC6A9,0xC6B0,0xC6B1,0xC6B4,0xC6B8,/* 0xE8-0xEF */ + 0xC6B9,0xC6BA,0xC6C0,0xC6C1,0xC6C3,0xC6C5,0xC6CC,0xC6CD,/* 0xF0-0xF7 */ + 0xC6D0,0xC6D4,0xC6DC,0xC6DD,0xC6E0,0xC6E1,0xC6E8,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_C0[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xD4FE,0xD4FF,0xD500,0xD501,0xD502,0xD503,0xD505,/* 0x40-0x47 */ + 0xD506,0xD507,0xD509,0xD50A,0xD50B,0xD50D,0xD50E,0xD50F,/* 0x48-0x4F */ + 0xD510,0xD511,0xD512,0xD513,0xD516,0xD518,0xD519,0xD51A,/* 0x50-0x57 */ + 0xD51B,0xD51C,0xD51D,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xD51E,0xD51F,0xD520,0xD521,0xD522,0xD523,0xD524,/* 0x60-0x67 */ + 0xD525,0xD526,0xD527,0xD528,0xD529,0xD52A,0xD52B,0xD52C,/* 0x68-0x6F */ + 0xD52D,0xD52E,0xD52F,0xD530,0xD531,0xD532,0xD533,0xD534,/* 0x70-0x77 */ + 0xD535,0xD536,0xD537,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xD538,0xD539,0xD53A,0xD53B,0xD53E,0xD53F,0xD541,/* 0x80-0x87 */ + 0xD542,0xD543,0xD545,0xD546,0xD547,0xD548,0xD549,0xD54A,/* 0x88-0x8F */ + 0xD54B,0xD54E,0xD550,0xD552,0xD553,0xD554,0xD555,0xD556,/* 0x90-0x97 */ + 0xD557,0xD55A,0xD55B,0xD55D,0xD55E,0xD55F,0xD561,0xD562,/* 0x98-0x9F */ + 0xD563,0xC6E9,0xC6EC,0xC6F0,0xC6F8,0xC6F9,0xC6FD,0xC704,/* 0xA0-0xA7 */ + 0xC705,0xC708,0xC70C,0xC714,0xC715,0xC717,0xC719,0xC720,/* 0xA8-0xAF */ + 0xC721,0xC724,0xC728,0xC730,0xC731,0xC733,0xC735,0xC737,/* 0xB0-0xB7 */ + 0xC73C,0xC73D,0xC740,0xC744,0xC74A,0xC74C,0xC74D,0xC74F,/* 0xB8-0xBF */ + 0xC751,0xC752,0xC753,0xC754,0xC755,0xC756,0xC757,0xC758,/* 0xC0-0xC7 */ + 0xC75C,0xC760,0xC768,0xC76B,0xC774,0xC775,0xC778,0xC77C,/* 0xC8-0xCF */ + 0xC77D,0xC77E,0xC783,0xC784,0xC785,0xC787,0xC788,0xC789,/* 0xD0-0xD7 */ + 0xC78A,0xC78E,0xC790,0xC791,0xC794,0xC796,0xC797,0xC798,/* 0xD8-0xDF */ + 0xC79A,0xC7A0,0xC7A1,0xC7A3,0xC7A4,0xC7A5,0xC7A6,0xC7AC,/* 0xE0-0xE7 */ + 0xC7AD,0xC7B0,0xC7B4,0xC7BC,0xC7BD,0xC7BF,0xC7C0,0xC7C1,/* 0xE8-0xEF */ + 0xC7C8,0xC7C9,0xC7CC,0xC7CE,0xC7D0,0xC7D8,0xC7DD,0xC7E4,/* 0xF0-0xF7 */ + 0xC7E8,0xC7EC,0xC800,0xC801,0xC804,0xC808,0xC80A,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_C1[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xD564,0xD566,0xD567,0xD56A,0xD56C,0xD56E,0xD56F,/* 0x40-0x47 */ + 0xD570,0xD571,0xD572,0xD573,0xD576,0xD577,0xD579,0xD57A,/* 0x48-0x4F */ + 0xD57B,0xD57D,0xD57E,0xD57F,0xD580,0xD581,0xD582,0xD583,/* 0x50-0x57 */ + 0xD586,0xD58A,0xD58B,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xD58C,0xD58D,0xD58E,0xD58F,0xD591,0xD592,0xD593,/* 0x60-0x67 */ + 0xD594,0xD595,0xD596,0xD597,0xD598,0xD599,0xD59A,0xD59B,/* 0x68-0x6F */ + 0xD59C,0xD59D,0xD59E,0xD59F,0xD5A0,0xD5A1,0xD5A2,0xD5A3,/* 0x70-0x77 */ + 0xD5A4,0xD5A6,0xD5A7,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xD5A8,0xD5A9,0xD5AA,0xD5AB,0xD5AC,0xD5AD,0xD5AE,/* 0x80-0x87 */ + 0xD5AF,0xD5B0,0xD5B1,0xD5B2,0xD5B3,0xD5B4,0xD5B5,0xD5B6,/* 0x88-0x8F */ + 0xD5B7,0xD5B8,0xD5B9,0xD5BA,0xD5BB,0xD5BC,0xD5BD,0xD5BE,/* 0x90-0x97 */ + 0xD5BF,0xD5C0,0xD5C1,0xD5C2,0xD5C3,0xD5C4,0xD5C5,0xD5C6,/* 0x98-0x9F */ + 0xD5C7,0xC810,0xC811,0xC813,0xC815,0xC816,0xC81C,0xC81D,/* 0xA0-0xA7 */ + 0xC820,0xC824,0xC82C,0xC82D,0xC82F,0xC831,0xC838,0xC83C,/* 0xA8-0xAF */ + 0xC840,0xC848,0xC849,0xC84C,0xC84D,0xC854,0xC870,0xC871,/* 0xB0-0xB7 */ + 0xC874,0xC878,0xC87A,0xC880,0xC881,0xC883,0xC885,0xC886,/* 0xB8-0xBF */ + 0xC887,0xC88B,0xC88C,0xC88D,0xC894,0xC89D,0xC89F,0xC8A1,/* 0xC0-0xC7 */ + 0xC8A8,0xC8BC,0xC8BD,0xC8C4,0xC8C8,0xC8CC,0xC8D4,0xC8D5,/* 0xC8-0xCF */ + 0xC8D7,0xC8D9,0xC8E0,0xC8E1,0xC8E4,0xC8F5,0xC8FC,0xC8FD,/* 0xD0-0xD7 */ + 0xC900,0xC904,0xC905,0xC906,0xC90C,0xC90D,0xC90F,0xC911,/* 0xD8-0xDF */ + 0xC918,0xC92C,0xC934,0xC950,0xC951,0xC954,0xC958,0xC960,/* 0xE0-0xE7 */ + 0xC961,0xC963,0xC96C,0xC970,0xC974,0xC97C,0xC988,0xC989,/* 0xE8-0xEF */ + 0xC98C,0xC990,0xC998,0xC999,0xC99B,0xC99D,0xC9C0,0xC9C1,/* 0xF0-0xF7 */ + 0xC9C4,0xC9C7,0xC9C8,0xC9CA,0xC9D0,0xC9D1,0xC9D3,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_C2[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xD5CA,0xD5CB,0xD5CD,0xD5CE,0xD5CF,0xD5D1,0xD5D3,/* 0x40-0x47 */ + 0xD5D4,0xD5D5,0xD5D6,0xD5D7,0xD5DA,0xD5DC,0xD5DE,0xD5DF,/* 0x48-0x4F */ + 0xD5E0,0xD5E1,0xD5E2,0xD5E3,0xD5E6,0xD5E7,0xD5E9,0xD5EA,/* 0x50-0x57 */ + 0xD5EB,0xD5ED,0xD5EE,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xD5EF,0xD5F0,0xD5F1,0xD5F2,0xD5F3,0xD5F6,0xD5F8,/* 0x60-0x67 */ + 0xD5FA,0xD5FB,0xD5FC,0xD5FD,0xD5FE,0xD5FF,0xD602,0xD603,/* 0x68-0x6F */ + 0xD605,0xD606,0xD607,0xD609,0xD60A,0xD60B,0xD60C,0xD60D,/* 0x70-0x77 */ + 0xD60E,0xD60F,0xD612,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xD616,0xD617,0xD618,0xD619,0xD61A,0xD61B,0xD61D,/* 0x80-0x87 */ + 0xD61E,0xD61F,0xD621,0xD622,0xD623,0xD625,0xD626,0xD627,/* 0x88-0x8F */ + 0xD628,0xD629,0xD62A,0xD62B,0xD62C,0xD62E,0xD62F,0xD630,/* 0x90-0x97 */ + 0xD631,0xD632,0xD633,0xD634,0xD635,0xD636,0xD637,0xD63A,/* 0x98-0x9F */ + 0xD63B,0xC9D5,0xC9D6,0xC9D9,0xC9DA,0xC9DC,0xC9DD,0xC9E0,/* 0xA0-0xA7 */ + 0xC9E2,0xC9E4,0xC9E7,0xC9EC,0xC9ED,0xC9EF,0xC9F0,0xC9F1,/* 0xA8-0xAF */ + 0xC9F8,0xC9F9,0xC9FC,0xCA00,0xCA08,0xCA09,0xCA0B,0xCA0C,/* 0xB0-0xB7 */ + 0xCA0D,0xCA14,0xCA18,0xCA29,0xCA4C,0xCA4D,0xCA50,0xCA54,/* 0xB8-0xBF */ + 0xCA5C,0xCA5D,0xCA5F,0xCA60,0xCA61,0xCA68,0xCA7D,0xCA84,/* 0xC0-0xC7 */ + 0xCA98,0xCABC,0xCABD,0xCAC0,0xCAC4,0xCACC,0xCACD,0xCACF,/* 0xC8-0xCF */ + 0xCAD1,0xCAD3,0xCAD8,0xCAD9,0xCAE0,0xCAEC,0xCAF4,0xCB08,/* 0xD0-0xD7 */ + 0xCB10,0xCB14,0xCB18,0xCB20,0xCB21,0xCB41,0xCB48,0xCB49,/* 0xD8-0xDF */ + 0xCB4C,0xCB50,0xCB58,0xCB59,0xCB5D,0xCB64,0xCB78,0xCB79,/* 0xE0-0xE7 */ + 0xCB9C,0xCBB8,0xCBD4,0xCBE4,0xCBE7,0xCBE9,0xCC0C,0xCC0D,/* 0xE8-0xEF */ + 0xCC10,0xCC14,0xCC1C,0xCC1D,0xCC21,0xCC22,0xCC27,0xCC28,/* 0xF0-0xF7 */ + 0xCC29,0xCC2C,0xCC2E,0xCC30,0xCC38,0xCC39,0xCC3B,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_C3[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xD63D,0xD63E,0xD63F,0xD641,0xD642,0xD643,0xD644,/* 0x40-0x47 */ + 0xD646,0xD647,0xD64A,0xD64C,0xD64E,0xD64F,0xD650,0xD652,/* 0x48-0x4F */ + 0xD653,0xD656,0xD657,0xD659,0xD65A,0xD65B,0xD65D,0xD65E,/* 0x50-0x57 */ + 0xD65F,0xD660,0xD661,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xD662,0xD663,0xD664,0xD665,0xD666,0xD668,0xD66A,/* 0x60-0x67 */ + 0xD66B,0xD66C,0xD66D,0xD66E,0xD66F,0xD672,0xD673,0xD675,/* 0x68-0x6F */ + 0xD676,0xD677,0xD678,0xD679,0xD67A,0xD67B,0xD67C,0xD67D,/* 0x70-0x77 */ + 0xD67E,0xD67F,0xD680,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xD681,0xD682,0xD684,0xD686,0xD687,0xD688,0xD689,/* 0x80-0x87 */ + 0xD68A,0xD68B,0xD68E,0xD68F,0xD691,0xD692,0xD693,0xD695,/* 0x88-0x8F */ + 0xD696,0xD697,0xD698,0xD699,0xD69A,0xD69B,0xD69C,0xD69E,/* 0x90-0x97 */ + 0xD6A0,0xD6A2,0xD6A3,0xD6A4,0xD6A5,0xD6A6,0xD6A7,0xD6A9,/* 0x98-0x9F */ + 0xD6AA,0xCC3C,0xCC3D,0xCC3E,0xCC44,0xCC45,0xCC48,0xCC4C,/* 0xA0-0xA7 */ + 0xCC54,0xCC55,0xCC57,0xCC58,0xCC59,0xCC60,0xCC64,0xCC66,/* 0xA8-0xAF */ + 0xCC68,0xCC70,0xCC75,0xCC98,0xCC99,0xCC9C,0xCCA0,0xCCA8,/* 0xB0-0xB7 */ + 0xCCA9,0xCCAB,0xCCAC,0xCCAD,0xCCB4,0xCCB5,0xCCB8,0xCCBC,/* 0xB8-0xBF */ + 0xCCC4,0xCCC5,0xCCC7,0xCCC9,0xCCD0,0xCCD4,0xCCE4,0xCCEC,/* 0xC0-0xC7 */ + 0xCCF0,0xCD01,0xCD08,0xCD09,0xCD0C,0xCD10,0xCD18,0xCD19,/* 0xC8-0xCF */ + 0xCD1B,0xCD1D,0xCD24,0xCD28,0xCD2C,0xCD39,0xCD5C,0xCD60,/* 0xD0-0xD7 */ + 0xCD64,0xCD6C,0xCD6D,0xCD6F,0xCD71,0xCD78,0xCD88,0xCD94,/* 0xD8-0xDF */ + 0xCD95,0xCD98,0xCD9C,0xCDA4,0xCDA5,0xCDA7,0xCDA9,0xCDB0,/* 0xE0-0xE7 */ + 0xCDC4,0xCDCC,0xCDD0,0xCDE8,0xCDEC,0xCDF0,0xCDF8,0xCDF9,/* 0xE8-0xEF */ + 0xCDFB,0xCDFD,0xCE04,0xCE08,0xCE0C,0xCE14,0xCE19,0xCE20,/* 0xF0-0xF7 */ + 0xCE21,0xCE24,0xCE28,0xCE30,0xCE31,0xCE33,0xCE35,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_C4[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xD6AB,0xD6AD,0xD6AE,0xD6AF,0xD6B1,0xD6B2,0xD6B3,/* 0x40-0x47 */ + 0xD6B4,0xD6B5,0xD6B6,0xD6B7,0xD6B8,0xD6BA,0xD6BC,0xD6BD,/* 0x48-0x4F */ + 0xD6BE,0xD6BF,0xD6C0,0xD6C1,0xD6C2,0xD6C3,0xD6C6,0xD6C7,/* 0x50-0x57 */ + 0xD6C9,0xD6CA,0xD6CB,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xD6CD,0xD6CE,0xD6CF,0xD6D0,0xD6D2,0xD6D3,0xD6D5,/* 0x60-0x67 */ + 0xD6D6,0xD6D8,0xD6DA,0xD6DB,0xD6DC,0xD6DD,0xD6DE,0xD6DF,/* 0x68-0x6F */ + 0xD6E1,0xD6E2,0xD6E3,0xD6E5,0xD6E6,0xD6E7,0xD6E9,0xD6EA,/* 0x70-0x77 */ + 0xD6EB,0xD6EC,0xD6ED,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xD6EE,0xD6EF,0xD6F1,0xD6F2,0xD6F3,0xD6F4,0xD6F6,/* 0x80-0x87 */ + 0xD6F7,0xD6F8,0xD6F9,0xD6FA,0xD6FB,0xD6FE,0xD6FF,0xD701,/* 0x88-0x8F */ + 0xD702,0xD703,0xD705,0xD706,0xD707,0xD708,0xD709,0xD70A,/* 0x90-0x97 */ + 0xD70B,0xD70C,0xD70D,0xD70E,0xD70F,0xD710,0xD712,0xD713,/* 0x98-0x9F */ + 0xD714,0xCE58,0xCE59,0xCE5C,0xCE5F,0xCE60,0xCE61,0xCE68,/* 0xA0-0xA7 */ + 0xCE69,0xCE6B,0xCE6D,0xCE74,0xCE75,0xCE78,0xCE7C,0xCE84,/* 0xA8-0xAF */ + 0xCE85,0xCE87,0xCE89,0xCE90,0xCE91,0xCE94,0xCE98,0xCEA0,/* 0xB0-0xB7 */ + 0xCEA1,0xCEA3,0xCEA4,0xCEA5,0xCEAC,0xCEAD,0xCEC1,0xCEE4,/* 0xB8-0xBF */ + 0xCEE5,0xCEE8,0xCEEB,0xCEEC,0xCEF4,0xCEF5,0xCEF7,0xCEF8,/* 0xC0-0xC7 */ + 0xCEF9,0xCF00,0xCF01,0xCF04,0xCF08,0xCF10,0xCF11,0xCF13,/* 0xC8-0xCF */ + 0xCF15,0xCF1C,0xCF20,0xCF24,0xCF2C,0xCF2D,0xCF2F,0xCF30,/* 0xD0-0xD7 */ + 0xCF31,0xCF38,0xCF54,0xCF55,0xCF58,0xCF5C,0xCF64,0xCF65,/* 0xD8-0xDF */ + 0xCF67,0xCF69,0xCF70,0xCF71,0xCF74,0xCF78,0xCF80,0xCF85,/* 0xE0-0xE7 */ + 0xCF8C,0xCFA1,0xCFA8,0xCFB0,0xCFC4,0xCFE0,0xCFE1,0xCFE4,/* 0xE8-0xEF */ + 0xCFE8,0xCFF0,0xCFF1,0xCFF3,0xCFF5,0xCFFC,0xD000,0xD004,/* 0xF0-0xF7 */ + 0xD011,0xD018,0xD02D,0xD034,0xD035,0xD038,0xD03C,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_C5[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xD715,0xD716,0xD717,0xD71A,0xD71B,0xD71D,0xD71E,/* 0x40-0x47 */ + 0xD71F,0xD721,0xD722,0xD723,0xD724,0xD725,0xD726,0xD727,/* 0x48-0x4F */ + 0xD72A,0xD72C,0xD72E,0xD72F,0xD730,0xD731,0xD732,0xD733,/* 0x50-0x57 */ + 0xD736,0xD737,0xD739,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0xD73A,0xD73B,0xD73D,0xD73E,0xD73F,0xD740,0xD741,/* 0x60-0x67 */ + 0xD742,0xD743,0xD745,0xD746,0xD748,0xD74A,0xD74B,0xD74C,/* 0x68-0x6F */ + 0xD74D,0xD74E,0xD74F,0xD752,0xD753,0xD755,0xD75A,0xD75B,/* 0x70-0x77 */ + 0xD75C,0xD75D,0xD75E,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0xD75F,0xD762,0xD764,0xD766,0xD767,0xD768,0xD76A,/* 0x80-0x87 */ + 0xD76B,0xD76D,0xD76E,0xD76F,0xD771,0xD772,0xD773,0xD775,/* 0x88-0x8F */ + 0xD776,0xD777,0xD778,0xD779,0xD77A,0xD77B,0xD77E,0xD77F,/* 0x90-0x97 */ + 0xD780,0xD782,0xD783,0xD784,0xD785,0xD786,0xD787,0xD78A,/* 0x98-0x9F */ + 0xD78B,0xD044,0xD045,0xD047,0xD049,0xD050,0xD054,0xD058,/* 0xA0-0xA7 */ + 0xD060,0xD06C,0xD06D,0xD070,0xD074,0xD07C,0xD07D,0xD081,/* 0xA8-0xAF */ + 0xD0A4,0xD0A5,0xD0A8,0xD0AC,0xD0B4,0xD0B5,0xD0B7,0xD0B9,/* 0xB0-0xB7 */ + 0xD0C0,0xD0C1,0xD0C4,0xD0C8,0xD0C9,0xD0D0,0xD0D1,0xD0D3,/* 0xB8-0xBF */ + 0xD0D4,0xD0D5,0xD0DC,0xD0DD,0xD0E0,0xD0E4,0xD0EC,0xD0ED,/* 0xC0-0xC7 */ + 0xD0EF,0xD0F0,0xD0F1,0xD0F8,0xD10D,0xD130,0xD131,0xD134,/* 0xC8-0xCF */ + 0xD138,0xD13A,0xD140,0xD141,0xD143,0xD144,0xD145,0xD14C,/* 0xD0-0xD7 */ + 0xD14D,0xD150,0xD154,0xD15C,0xD15D,0xD15F,0xD161,0xD168,/* 0xD8-0xDF */ + 0xD16C,0xD17C,0xD184,0xD188,0xD1A0,0xD1A1,0xD1A4,0xD1A8,/* 0xE0-0xE7 */ + 0xD1B0,0xD1B1,0xD1B3,0xD1B5,0xD1BA,0xD1BC,0xD1C0,0xD1D8,/* 0xE8-0xEF */ + 0xD1F4,0xD1F8,0xD207,0xD209,0xD210,0xD22C,0xD22D,0xD230,/* 0xF0-0xF7 */ + 0xD234,0xD23C,0xD23D,0xD23F,0xD241,0xD248,0xD25C,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_C6[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0xD78D,0xD78E,0xD78F,0xD791,0xD792,0xD793,0xD794,/* 0x40-0x47 */ + 0xD795,0xD796,0xD797,0xD79A,0xD79C,0xD79E,0xD79F,0xD7A0,/* 0x48-0x4F */ + 0xD7A1,0xD7A2,0xD7A3,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0xD264,0xD280,0xD281,0xD284,0xD288,0xD290,0xD291,/* 0xA0-0xA7 */ + 0xD295,0xD29C,0xD2A0,0xD2A4,0xD2AC,0xD2B1,0xD2B8,0xD2B9,/* 0xA8-0xAF */ + 0xD2BC,0xD2BF,0xD2C0,0xD2C2,0xD2C8,0xD2C9,0xD2CB,0xD2D4,/* 0xB0-0xB7 */ + 0xD2D8,0xD2DC,0xD2E4,0xD2E5,0xD2F0,0xD2F1,0xD2F4,0xD2F8,/* 0xB8-0xBF */ + 0xD300,0xD301,0xD303,0xD305,0xD30C,0xD30D,0xD30E,0xD310,/* 0xC0-0xC7 */ + 0xD314,0xD316,0xD31C,0xD31D,0xD31F,0xD320,0xD321,0xD325,/* 0xC8-0xCF */ + 0xD328,0xD329,0xD32C,0xD330,0xD338,0xD339,0xD33B,0xD33C,/* 0xD0-0xD7 */ + 0xD33D,0xD344,0xD345,0xD37C,0xD37D,0xD380,0xD384,0xD38C,/* 0xD8-0xDF */ + 0xD38D,0xD38F,0xD390,0xD391,0xD398,0xD399,0xD39C,0xD3A0,/* 0xE0-0xE7 */ + 0xD3A8,0xD3A9,0xD3AB,0xD3AD,0xD3B4,0xD3B8,0xD3BC,0xD3C4,/* 0xE8-0xEF */ + 0xD3C5,0xD3C8,0xD3C9,0xD3D0,0xD3D8,0xD3E1,0xD3E3,0xD3EC,/* 0xF0-0xF7 */ + 0xD3ED,0xD3F0,0xD3F4,0xD3FC,0xD3FD,0xD3FF,0xD401,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_C7[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0xD408,0xD41D,0xD440,0xD444,0xD45C,0xD460,0xD464,/* 0xA0-0xA7 */ + 0xD46D,0xD46F,0xD478,0xD479,0xD47C,0xD47F,0xD480,0xD482,/* 0xA8-0xAF */ + 0xD488,0xD489,0xD48B,0xD48D,0xD494,0xD4A9,0xD4CC,0xD4D0,/* 0xB0-0xB7 */ + 0xD4D4,0xD4DC,0xD4DF,0xD4E8,0xD4EC,0xD4F0,0xD4F8,0xD4FB,/* 0xB8-0xBF */ + 0xD4FD,0xD504,0xD508,0xD50C,0xD514,0xD515,0xD517,0xD53C,/* 0xC0-0xC7 */ + 0xD53D,0xD540,0xD544,0xD54C,0xD54D,0xD54F,0xD551,0xD558,/* 0xC8-0xCF */ + 0xD559,0xD55C,0xD560,0xD565,0xD568,0xD569,0xD56B,0xD56D,/* 0xD0-0xD7 */ + 0xD574,0xD575,0xD578,0xD57C,0xD584,0xD585,0xD587,0xD588,/* 0xD8-0xDF */ + 0xD589,0xD590,0xD5A5,0xD5C8,0xD5C9,0xD5CC,0xD5D0,0xD5D2,/* 0xE0-0xE7 */ + 0xD5D8,0xD5D9,0xD5DB,0xD5DD,0xD5E4,0xD5E5,0xD5E8,0xD5EC,/* 0xE8-0xEF */ + 0xD5F4,0xD5F5,0xD5F7,0xD5F9,0xD600,0xD601,0xD604,0xD608,/* 0xF0-0xF7 */ + 0xD610,0xD611,0xD613,0xD614,0xD615,0xD61C,0xD620,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_C8[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0xD624,0xD62D,0xD638,0xD639,0xD63C,0xD640,0xD645,/* 0xA0-0xA7 */ + 0xD648,0xD649,0xD64B,0xD64D,0xD651,0xD654,0xD655,0xD658,/* 0xA8-0xAF */ + 0xD65C,0xD667,0xD669,0xD670,0xD671,0xD674,0xD683,0xD685,/* 0xB0-0xB7 */ + 0xD68C,0xD68D,0xD690,0xD694,0xD69D,0xD69F,0xD6A1,0xD6A8,/* 0xB8-0xBF */ + 0xD6AC,0xD6B0,0xD6B9,0xD6BB,0xD6C4,0xD6C5,0xD6C8,0xD6CC,/* 0xC0-0xC7 */ + 0xD6D1,0xD6D4,0xD6D7,0xD6D9,0xD6E0,0xD6E4,0xD6E8,0xD6F0,/* 0xC8-0xCF */ + 0xD6F5,0xD6FC,0xD6FD,0xD700,0xD704,0xD711,0xD718,0xD719,/* 0xD0-0xD7 */ + 0xD71C,0xD720,0xD728,0xD729,0xD72B,0xD72D,0xD734,0xD735,/* 0xD8-0xDF */ + 0xD738,0xD73C,0xD744,0xD747,0xD749,0xD750,0xD751,0xD754,/* 0xE0-0xE7 */ + 0xD756,0xD757,0xD758,0xD759,0xD760,0xD761,0xD763,0xD765,/* 0xE8-0xEF */ + 0xD769,0xD76C,0xD770,0xD774,0xD77C,0xD77D,0xD781,0xD788,/* 0xF0-0xF7 */ + 0xD789,0xD78C,0xD790,0xD798,0xD799,0xD79B,0xD79D,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_CA[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x4F3D,0x4F73,0x5047,0x50F9,0x52A0,0x53EF,0x5475,/* 0xA0-0xA7 */ + 0x54E5,0x5609,0x5AC1,0x5BB6,0x6687,0x67B6,0x67B7,0x67EF,/* 0xA8-0xAF */ + 0x6B4C,0x73C2,0x75C2,0x7A3C,0x82DB,0x8304,0x8857,0x8888,/* 0xB0-0xB7 */ + 0x8A36,0x8CC8,0x8DCF,0x8EFB,0x8FE6,0x99D5,0x523B,0x5374,/* 0xB8-0xBF */ + 0x5404,0x606A,0x6164,0x6BBC,0x73CF,0x811A,0x89BA,0x89D2,/* 0xC0-0xC7 */ + 0x95A3,0x4F83,0x520A,0x58BE,0x5978,0x59E6,0x5E72,0x5E79,/* 0xC8-0xCF */ + 0x61C7,0x63C0,0x6746,0x67EC,0x687F,0x6F97,0x764E,0x770B,/* 0xD0-0xD7 */ + 0x78F5,0x7A08,0x7AFF,0x7C21,0x809D,0x826E,0x8271,0x8AEB,/* 0xD8-0xDF */ + 0x9593,0x4E6B,0x559D,0x66F7,0x6E34,0x78A3,0x7AED,0x845B,/* 0xE0-0xE7 */ + 0x8910,0x874E,0x97A8,0x52D8,0x574E,0x582A,0x5D4C,0x611F,/* 0xE8-0xEF */ + 0x61BE,0x6221,0x6562,0x67D1,0x6A44,0x6E1B,0x7518,0x75B3,/* 0xF0-0xF7 */ + 0x76E3,0x77B0,0x7D3A,0x90AF,0x9451,0x9452,0x9F95,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_CB[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x5323,0x5CAC,0x7532,0x80DB,0x9240,0x9598,0x525B,/* 0xA0-0xA7 */ + 0x5808,0x59DC,0x5CA1,0x5D17,0x5EB7,0x5F3A,0x5F4A,0x6177,/* 0xA8-0xAF */ + 0x6C5F,0x757A,0x7586,0x7CE0,0x7D73,0x7DB1,0x7F8C,0x8154,/* 0xB0-0xB7 */ + 0x8221,0x8591,0x8941,0x8B1B,0x92FC,0x964D,0x9C47,0x4ECB,/* 0xB8-0xBF */ + 0x4EF7,0x500B,0x51F1,0x584F,0x6137,0x613E,0x6168,0x6539,/* 0xC0-0xC7 */ + 0x69EA,0x6F11,0x75A5,0x7686,0x76D6,0x7B87,0x82A5,0x84CB,/* 0xC8-0xCF */ + 0xF900,0x93A7,0x958B,0x5580,0x5BA2,0x5751,0xF901,0x7CB3,/* 0xD0-0xD7 */ + 0x7FB9,0x91B5,0x5028,0x53BB,0x5C45,0x5DE8,0x62D2,0x636E,/* 0xD8-0xDF */ + 0x64DA,0x64E7,0x6E20,0x70AC,0x795B,0x8DDD,0x8E1E,0xF902,/* 0xE0-0xE7 */ + 0x907D,0x9245,0x92F8,0x4E7E,0x4EF6,0x5065,0x5DFE,0x5EFA,/* 0xE8-0xEF */ + 0x6106,0x6957,0x8171,0x8654,0x8E47,0x9375,0x9A2B,0x4E5E,/* 0xF0-0xF7 */ + 0x5091,0x6770,0x6840,0x5109,0x528D,0x5292,0x6AA2,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_CC[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x77BC,0x9210,0x9ED4,0x52AB,0x602F,0x8FF2,0x5048,/* 0xA0-0xA7 */ + 0x61A9,0x63ED,0x64CA,0x683C,0x6A84,0x6FC0,0x8188,0x89A1,/* 0xA8-0xAF */ + 0x9694,0x5805,0x727D,0x72AC,0x7504,0x7D79,0x7E6D,0x80A9,/* 0xB0-0xB7 */ + 0x898B,0x8B74,0x9063,0x9D51,0x6289,0x6C7A,0x6F54,0x7D50,/* 0xB8-0xBF */ + 0x7F3A,0x8A23,0x517C,0x614A,0x7B9D,0x8B19,0x9257,0x938C,/* 0xC0-0xC7 */ + 0x4EAC,0x4FD3,0x501E,0x50BE,0x5106,0x52C1,0x52CD,0x537F,/* 0xC8-0xCF */ + 0x5770,0x5883,0x5E9A,0x5F91,0x6176,0x61AC,0x64CE,0x656C,/* 0xD0-0xD7 */ + 0x666F,0x66BB,0x66F4,0x6897,0x6D87,0x7085,0x70F1,0x749F,/* 0xD8-0xDF */ + 0x74A5,0x74CA,0x75D9,0x786C,0x78EC,0x7ADF,0x7AF6,0x7D45,/* 0xE0-0xE7 */ + 0x7D93,0x8015,0x803F,0x811B,0x8396,0x8B66,0x8F15,0x9015,/* 0xE8-0xEF */ + 0x93E1,0x9803,0x9838,0x9A5A,0x9BE8,0x4FC2,0x5553,0x583A,/* 0xF0-0xF7 */ + 0x5951,0x5B63,0x5C46,0x60B8,0x6212,0x6842,0x68B0,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_CD[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x68E8,0x6EAA,0x754C,0x7678,0x78CE,0x7A3D,0x7CFB,/* 0xA0-0xA7 */ + 0x7E6B,0x7E7C,0x8A08,0x8AA1,0x8C3F,0x968E,0x9DC4,0x53E4,/* 0xA8-0xAF */ + 0x53E9,0x544A,0x5471,0x56FA,0x59D1,0x5B64,0x5C3B,0x5EAB,/* 0xB0-0xB7 */ + 0x62F7,0x6537,0x6545,0x6572,0x66A0,0x67AF,0x69C1,0x6CBD,/* 0xB8-0xBF */ + 0x75FC,0x7690,0x777E,0x7A3F,0x7F94,0x8003,0x80A1,0x818F,/* 0xC0-0xC7 */ + 0x82E6,0x82FD,0x83F0,0x85C1,0x8831,0x88B4,0x8AA5,0xF903,/* 0xC8-0xCF */ + 0x8F9C,0x932E,0x96C7,0x9867,0x9AD8,0x9F13,0x54ED,0x659B,/* 0xD0-0xD7 */ + 0x66F2,0x688F,0x7A40,0x8C37,0x9D60,0x56F0,0x5764,0x5D11,/* 0xD8-0xDF */ + 0x6606,0x68B1,0x68CD,0x6EFE,0x7428,0x889E,0x9BE4,0x6C68,/* 0xE0-0xE7 */ + 0xF904,0x9AA8,0x4F9B,0x516C,0x5171,0x529F,0x5B54,0x5DE5,/* 0xE8-0xEF */ + 0x6050,0x606D,0x62F1,0x63A7,0x653B,0x73D9,0x7A7A,0x86A3,/* 0xF0-0xF7 */ + 0x8CA2,0x978F,0x4E32,0x5BE1,0x6208,0x679C,0x74DC,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_CE[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x79D1,0x83D3,0x8A87,0x8AB2,0x8DE8,0x904E,0x934B,/* 0xA0-0xA7 */ + 0x9846,0x5ED3,0x69E8,0x85FF,0x90ED,0xF905,0x51A0,0x5B98,/* 0xA8-0xAF */ + 0x5BEC,0x6163,0x68FA,0x6B3E,0x704C,0x742F,0x74D8,0x7BA1,/* 0xB0-0xB7 */ + 0x7F50,0x83C5,0x89C0,0x8CAB,0x95DC,0x9928,0x522E,0x605D,/* 0xB8-0xBF */ + 0x62EC,0x9002,0x4F8A,0x5149,0x5321,0x58D9,0x5EE3,0x66E0,/* 0xC0-0xC7 */ + 0x6D38,0x709A,0x72C2,0x73D6,0x7B50,0x80F1,0x945B,0x5366,/* 0xC8-0xCF */ + 0x639B,0x7F6B,0x4E56,0x5080,0x584A,0x58DE,0x602A,0x6127,/* 0xD0-0xD7 */ + 0x62D0,0x69D0,0x9B41,0x5B8F,0x7D18,0x80B1,0x8F5F,0x4EA4,/* 0xD8-0xDF */ + 0x50D1,0x54AC,0x55AC,0x5B0C,0x5DA0,0x5DE7,0x652A,0x654E,/* 0xE0-0xE7 */ + 0x6821,0x6A4B,0x72E1,0x768E,0x77EF,0x7D5E,0x7FF9,0x81A0,/* 0xE8-0xEF */ + 0x854E,0x86DF,0x8F03,0x8F4E,0x90CA,0x9903,0x9A55,0x9BAB,/* 0xF0-0xF7 */ + 0x4E18,0x4E45,0x4E5D,0x4EC7,0x4FF1,0x5177,0x52FE,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_CF[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x5340,0x53E3,0x53E5,0x548E,0x5614,0x5775,0x57A2,/* 0xA0-0xA7 */ + 0x5BC7,0x5D87,0x5ED0,0x61FC,0x62D8,0x6551,0x67B8,0x67E9,/* 0xA8-0xAF */ + 0x69CB,0x6B50,0x6BC6,0x6BEC,0x6C42,0x6E9D,0x7078,0x72D7,/* 0xB0-0xB7 */ + 0x7396,0x7403,0x77BF,0x77E9,0x7A76,0x7D7F,0x8009,0x81FC,/* 0xB8-0xBF */ + 0x8205,0x820A,0x82DF,0x8862,0x8B33,0x8CFC,0x8EC0,0x9011,/* 0xC0-0xC7 */ + 0x90B1,0x9264,0x92B6,0x99D2,0x9A45,0x9CE9,0x9DD7,0x9F9C,/* 0xC8-0xCF */ + 0x570B,0x5C40,0x83CA,0x97A0,0x97AB,0x9EB4,0x541B,0x7A98,/* 0xD0-0xD7 */ + 0x7FA4,0x88D9,0x8ECD,0x90E1,0x5800,0x5C48,0x6398,0x7A9F,/* 0xD8-0xDF */ + 0x5BAE,0x5F13,0x7A79,0x7AAE,0x828E,0x8EAC,0x5026,0x5238,/* 0xE0-0xE7 */ + 0x52F8,0x5377,0x5708,0x62F3,0x6372,0x6B0A,0x6DC3,0x7737,/* 0xE8-0xEF */ + 0x53A5,0x7357,0x8568,0x8E76,0x95D5,0x673A,0x6AC3,0x6F70,/* 0xF0-0xF7 */ + 0x8A6D,0x8ECC,0x994B,0xF906,0x6677,0x6B78,0x8CB4,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D0[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x9B3C,0xF907,0x53EB,0x572D,0x594E,0x63C6,0x69FB,/* 0xA0-0xA7 */ + 0x73EA,0x7845,0x7ABA,0x7AC5,0x7CFE,0x8475,0x898F,0x8D73,/* 0xA8-0xAF */ + 0x9035,0x95A8,0x52FB,0x5747,0x7547,0x7B60,0x83CC,0x921E,/* 0xB0-0xB7 */ + 0xF908,0x6A58,0x514B,0x524B,0x5287,0x621F,0x68D8,0x6975,/* 0xB8-0xBF */ + 0x9699,0x50C5,0x52A4,0x52E4,0x61C3,0x65A4,0x6839,0x69FF,/* 0xC0-0xC7 */ + 0x747E,0x7B4B,0x82B9,0x83EB,0x89B2,0x8B39,0x8FD1,0x9949,/* 0xC8-0xCF */ + 0xF909,0x4ECA,0x5997,0x64D2,0x6611,0x6A8E,0x7434,0x7981,/* 0xD0-0xD7 */ + 0x79BD,0x82A9,0x887E,0x887F,0x895F,0xF90A,0x9326,0x4F0B,/* 0xD8-0xDF */ + 0x53CA,0x6025,0x6271,0x6C72,0x7D1A,0x7D66,0x4E98,0x5162,/* 0xE0-0xE7 */ + 0x77DC,0x80AF,0x4F01,0x4F0E,0x5176,0x5180,0x55DC,0x5668,/* 0xE8-0xEF */ + 0x573B,0x57FA,0x57FC,0x5914,0x5947,0x5993,0x5BC4,0x5C90,/* 0xF0-0xF7 */ + 0x5D0E,0x5DF1,0x5E7E,0x5FCC,0x6280,0x65D7,0x65E3,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D1[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x671E,0x671F,0x675E,0x68CB,0x68C4,0x6A5F,0x6B3A,/* 0xA0-0xA7 */ + 0x6C23,0x6C7D,0x6C82,0x6DC7,0x7398,0x7426,0x742A,0x7482,/* 0xA8-0xAF */ + 0x74A3,0x7578,0x757F,0x7881,0x78EF,0x7941,0x7947,0x7948,/* 0xB0-0xB7 */ + 0x797A,0x7B95,0x7D00,0x7DBA,0x7F88,0x8006,0x802D,0x808C,/* 0xB8-0xBF */ + 0x8A18,0x8B4F,0x8C48,0x8D77,0x9321,0x9324,0x98E2,0x9951,/* 0xC0-0xC7 */ + 0x9A0E,0x9A0F,0x9A65,0x9E92,0x7DCA,0x4F76,0x5409,0x62EE,/* 0xC8-0xCF */ + 0x6854,0x91D1,0x55AB,0x513A,0xF90B,0xF90C,0x5A1C,0x61E6,/* 0xD0-0xD7 */ + 0xF90D,0x62CF,0x62FF,0xF90E,0xF90F,0xF910,0xF911,0xF912,/* 0xD8-0xDF */ + 0xF913,0x90A3,0xF914,0xF915,0xF916,0xF917,0xF918,0x8AFE,/* 0xE0-0xE7 */ + 0xF919,0xF91A,0xF91B,0xF91C,0x6696,0xF91D,0x7156,0xF91E,/* 0xE8-0xEF */ + 0xF91F,0x96E3,0xF920,0x634F,0x637A,0x5357,0xF921,0x678F,/* 0xF0-0xF7 */ + 0x6960,0x6E73,0xF922,0x7537,0xF923,0xF924,0xF925,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D2[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x7D0D,0xF926,0xF927,0x8872,0x56CA,0x5A18,0xF928,/* 0xA0-0xA7 */ + 0xF929,0xF92A,0xF92B,0xF92C,0x4E43,0xF92D,0x5167,0x5948,/* 0xA8-0xAF */ + 0x67F0,0x8010,0xF92E,0x5973,0x5E74,0x649A,0x79CA,0x5FF5,/* 0xB0-0xB7 */ + 0x606C,0x62C8,0x637B,0x5BE7,0x5BD7,0x52AA,0xF92F,0x5974,/* 0xB8-0xBF */ + 0x5F29,0x6012,0xF930,0xF931,0xF932,0x7459,0xF933,0xF934,/* 0xC0-0xC7 */ + 0xF935,0xF936,0xF937,0xF938,0x99D1,0xF939,0xF93A,0xF93B,/* 0xC8-0xCF */ + 0xF93C,0xF93D,0xF93E,0xF93F,0xF940,0xF941,0xF942,0xF943,/* 0xD0-0xD7 */ + 0x6FC3,0xF944,0xF945,0x81BF,0x8FB2,0x60F1,0xF946,0xF947,/* 0xD8-0xDF */ + 0x8166,0xF948,0xF949,0x5C3F,0xF94A,0xF94B,0xF94C,0xF94D,/* 0xE0-0xE7 */ + 0xF94E,0xF94F,0xF950,0xF951,0x5AE9,0x8A25,0x677B,0x7D10,/* 0xE8-0xEF */ + 0xF952,0xF953,0xF954,0xF955,0xF956,0xF957,0x80FD,0xF958,/* 0xF0-0xF7 */ + 0xF959,0x5C3C,0x6CE5,0x533F,0x6EBA,0x591A,0x8336,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D3[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x4E39,0x4EB6,0x4F46,0x55AE,0x5718,0x58C7,0x5F56,/* 0xA0-0xA7 */ + 0x65B7,0x65E6,0x6A80,0x6BB5,0x6E4D,0x77ED,0x7AEF,0x7C1E,/* 0xA8-0xAF */ + 0x7DDE,0x86CB,0x8892,0x9132,0x935B,0x64BB,0x6FBE,0x737A,/* 0xB0-0xB7 */ + 0x75B8,0x9054,0x5556,0x574D,0x61BA,0x64D4,0x66C7,0x6DE1,/* 0xB8-0xBF */ + 0x6E5B,0x6F6D,0x6FB9,0x75F0,0x8043,0x81BD,0x8541,0x8983,/* 0xC0-0xC7 */ + 0x8AC7,0x8B5A,0x931F,0x6C93,0x7553,0x7B54,0x8E0F,0x905D,/* 0xC8-0xCF */ + 0x5510,0x5802,0x5858,0x5E62,0x6207,0x649E,0x68E0,0x7576,/* 0xD0-0xD7 */ + 0x7CD6,0x87B3,0x9EE8,0x4EE3,0x5788,0x576E,0x5927,0x5C0D,/* 0xD8-0xDF */ + 0x5CB1,0x5E36,0x5F85,0x6234,0x64E1,0x73B3,0x81FA,0x888B,/* 0xE0-0xE7 */ + 0x8CB8,0x968A,0x9EDB,0x5B85,0x5FB7,0x60B3,0x5012,0x5200,/* 0xE8-0xEF */ + 0x5230,0x5716,0x5835,0x5857,0x5C0E,0x5C60,0x5CF6,0x5D8B,/* 0xF0-0xF7 */ + 0x5EA6,0x5F92,0x60BC,0x6311,0x6389,0x6417,0x6843,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D4[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x68F9,0x6AC2,0x6DD8,0x6E21,0x6ED4,0x6FE4,0x71FE,/* 0xA0-0xA7 */ + 0x76DC,0x7779,0x79B1,0x7A3B,0x8404,0x89A9,0x8CED,0x8DF3,/* 0xA8-0xAF */ + 0x8E48,0x9003,0x9014,0x9053,0x90FD,0x934D,0x9676,0x97DC,/* 0xB0-0xB7 */ + 0x6BD2,0x7006,0x7258,0x72A2,0x7368,0x7763,0x79BF,0x7BE4,/* 0xB8-0xBF */ + 0x7E9B,0x8B80,0x58A9,0x60C7,0x6566,0x65FD,0x66BE,0x6C8C,/* 0xC0-0xC7 */ + 0x711E,0x71C9,0x8C5A,0x9813,0x4E6D,0x7A81,0x4EDD,0x51AC,/* 0xC8-0xCF */ + 0x51CD,0x52D5,0x540C,0x61A7,0x6771,0x6850,0x68DF,0x6D1E,/* 0xD0-0xD7 */ + 0x6F7C,0x75BC,0x77B3,0x7AE5,0x80F4,0x8463,0x9285,0x515C,/* 0xD8-0xDF */ + 0x6597,0x675C,0x6793,0x75D8,0x7AC7,0x8373,0xF95A,0x8C46,/* 0xE0-0xE7 */ + 0x9017,0x982D,0x5C6F,0x81C0,0x829A,0x9041,0x906F,0x920D,/* 0xE8-0xEF */ + 0x5F97,0x5D9D,0x6A59,0x71C8,0x767B,0x7B49,0x85E4,0x8B04,/* 0xF0-0xF7 */ + 0x9127,0x9A30,0x5587,0x61F6,0xF95B,0x7669,0x7F85,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D5[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x863F,0x87BA,0x88F8,0x908F,0xF95C,0x6D1B,0x70D9,/* 0xA0-0xA7 */ + 0x73DE,0x7D61,0x843D,0xF95D,0x916A,0x99F1,0xF95E,0x4E82,/* 0xA8-0xAF */ + 0x5375,0x6B04,0x6B12,0x703E,0x721B,0x862D,0x9E1E,0x524C,/* 0xB0-0xB7 */ + 0x8FA3,0x5D50,0x64E5,0x652C,0x6B16,0x6FEB,0x7C43,0x7E9C,/* 0xB8-0xBF */ + 0x85CD,0x8964,0x89BD,0x62C9,0x81D8,0x881F,0x5ECA,0x6717,/* 0xC0-0xC7 */ + 0x6D6A,0x72FC,0x7405,0x746F,0x8782,0x90DE,0x4F86,0x5D0D,/* 0xC8-0xCF */ + 0x5FA0,0x840A,0x51B7,0x63A0,0x7565,0x4EAE,0x5006,0x5169,/* 0xD0-0xD7 */ + 0x51C9,0x6881,0x6A11,0x7CAE,0x7CB1,0x7CE7,0x826F,0x8AD2,/* 0xD8-0xDF */ + 0x8F1B,0x91CF,0x4FB6,0x5137,0x52F5,0x5442,0x5EEC,0x616E,/* 0xE0-0xE7 */ + 0x623E,0x65C5,0x6ADA,0x6FFE,0x792A,0x85DC,0x8823,0x95AD,/* 0xE8-0xEF */ + 0x9A62,0x9A6A,0x9E97,0x9ECE,0x529B,0x66C6,0x6B77,0x701D,/* 0xF0-0xF7 */ + 0x792B,0x8F62,0x9742,0x6190,0x6200,0x6523,0x6F23,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D6[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x7149,0x7489,0x7DF4,0x806F,0x84EE,0x8F26,0x9023,/* 0xA0-0xA7 */ + 0x934A,0x51BD,0x5217,0x52A3,0x6D0C,0x70C8,0x88C2,0x5EC9,/* 0xA8-0xAF */ + 0x6582,0x6BAE,0x6FC2,0x7C3E,0x7375,0x4EE4,0x4F36,0x56F9,/* 0xB0-0xB7 */ + 0xF95F,0x5CBA,0x5DBA,0x601C,0x73B2,0x7B2D,0x7F9A,0x7FCE,/* 0xB8-0xBF */ + 0x8046,0x901E,0x9234,0x96F6,0x9748,0x9818,0x9F61,0x4F8B,/* 0xC0-0xC7 */ + 0x6FA7,0x79AE,0x91B4,0x96B7,0x52DE,0xF960,0x6488,0x64C4,/* 0xC8-0xCF */ + 0x6AD3,0x6F5E,0x7018,0x7210,0x76E7,0x8001,0x8606,0x865C,/* 0xD0-0xD7 */ + 0x8DEF,0x8F05,0x9732,0x9B6F,0x9DFA,0x9E75,0x788C,0x797F,/* 0xD8-0xDF */ + 0x7DA0,0x83C9,0x9304,0x9E7F,0x9E93,0x8AD6,0x58DF,0x5F04,/* 0xE0-0xE7 */ + 0x6727,0x7027,0x74CF,0x7C60,0x807E,0x5121,0x7028,0x7262,/* 0xE8-0xEF */ + 0x78CA,0x8CC2,0x8CDA,0x8CF4,0x96F7,0x4E86,0x50DA,0x5BEE,/* 0xF0-0xF7 */ + 0x5ED6,0x6599,0x71CE,0x7642,0x77AD,0x804A,0x84FC,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D7[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x907C,0x9B27,0x9F8D,0x58D8,0x5A41,0x5C62,0x6A13,/* 0xA0-0xA7 */ + 0x6DDA,0x6F0F,0x763B,0x7D2F,0x7E37,0x851E,0x8938,0x93E4,/* 0xA8-0xAF */ + 0x964B,0x5289,0x65D2,0x67F3,0x69B4,0x6D41,0x6E9C,0x700F,/* 0xB0-0xB7 */ + 0x7409,0x7460,0x7559,0x7624,0x786B,0x8B2C,0x985E,0x516D,/* 0xB8-0xBF */ + 0x622E,0x9678,0x4F96,0x502B,0x5D19,0x6DEA,0x7DB8,0x8F2A,/* 0xC0-0xC7 */ + 0x5F8B,0x6144,0x6817,0xF961,0x9686,0x52D2,0x808B,0x51DC,/* 0xC8-0xCF */ + 0x51CC,0x695E,0x7A1C,0x7DBE,0x83F1,0x9675,0x4FDA,0x5229,/* 0xD0-0xD7 */ + 0x5398,0x540F,0x550E,0x5C65,0x60A7,0x674E,0x68A8,0x6D6C,/* 0xD8-0xDF */ + 0x7281,0x72F8,0x7406,0x7483,0xF962,0x75E2,0x7C6C,0x7F79,/* 0xE0-0xE7 */ + 0x7FB8,0x8389,0x88CF,0x88E1,0x91CC,0x91D0,0x96E2,0x9BC9,/* 0xE8-0xEF */ + 0x541D,0x6F7E,0x71D0,0x7498,0x85FA,0x8EAA,0x96A3,0x9C57,/* 0xF0-0xF7 */ + 0x9E9F,0x6797,0x6DCB,0x7433,0x81E8,0x9716,0x782C,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D8[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x7ACB,0x7B20,0x7C92,0x6469,0x746A,0x75F2,0x78BC,/* 0xA0-0xA7 */ + 0x78E8,0x99AC,0x9B54,0x9EBB,0x5BDE,0x5E55,0x6F20,0x819C,/* 0xA8-0xAF */ + 0x83AB,0x9088,0x4E07,0x534D,0x5A29,0x5DD2,0x5F4E,0x6162,/* 0xB0-0xB7 */ + 0x633D,0x6669,0x66FC,0x6EFF,0x6F2B,0x7063,0x779E,0x842C,/* 0xB8-0xBF */ + 0x8513,0x883B,0x8F13,0x9945,0x9C3B,0x551C,0x62B9,0x672B,/* 0xC0-0xC7 */ + 0x6CAB,0x8309,0x896A,0x977A,0x4EA1,0x5984,0x5FD8,0x5FD9,/* 0xC8-0xCF */ + 0x671B,0x7DB2,0x7F54,0x8292,0x832B,0x83BD,0x8F1E,0x9099,/* 0xD0-0xD7 */ + 0x57CB,0x59B9,0x5A92,0x5BD0,0x6627,0x679A,0x6885,0x6BCF,/* 0xD8-0xDF */ + 0x7164,0x7F75,0x8CB7,0x8CE3,0x9081,0x9B45,0x8108,0x8C8A,/* 0xE0-0xE7 */ + 0x964C,0x9A40,0x9EA5,0x5B5F,0x6C13,0x731B,0x76F2,0x76DF,/* 0xE8-0xEF */ + 0x840C,0x51AA,0x8993,0x514D,0x5195,0x52C9,0x68C9,0x6C94,/* 0xF0-0xF7 */ + 0x7704,0x7720,0x7DBF,0x7DEC,0x9762,0x9EB5,0x6EC5,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D9[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x8511,0x51A5,0x540D,0x547D,0x660E,0x669D,0x6927,/* 0xA0-0xA7 */ + 0x6E9F,0x76BF,0x7791,0x8317,0x84C2,0x879F,0x9169,0x9298,/* 0xA8-0xAF */ + 0x9CF4,0x8882,0x4FAE,0x5192,0x52DF,0x59C6,0x5E3D,0x6155,/* 0xB0-0xB7 */ + 0x6478,0x6479,0x66AE,0x67D0,0x6A21,0x6BCD,0x6BDB,0x725F,/* 0xB8-0xBF */ + 0x7261,0x7441,0x7738,0x77DB,0x8017,0x82BC,0x8305,0x8B00,/* 0xC0-0xC7 */ + 0x8B28,0x8C8C,0x6728,0x6C90,0x7267,0x76EE,0x7766,0x7A46,/* 0xC8-0xCF */ + 0x9DA9,0x6B7F,0x6C92,0x5922,0x6726,0x8499,0x536F,0x5893,/* 0xD0-0xD7 */ + 0x5999,0x5EDF,0x63CF,0x6634,0x6773,0x6E3A,0x732B,0x7AD7,/* 0xD8-0xDF */ + 0x82D7,0x9328,0x52D9,0x5DEB,0x61AE,0x61CB,0x620A,0x62C7,/* 0xE0-0xE7 */ + 0x64AB,0x65E0,0x6959,0x6B66,0x6BCB,0x7121,0x73F7,0x755D,/* 0xE8-0xEF */ + 0x7E46,0x821E,0x8302,0x856A,0x8AA3,0x8CBF,0x9727,0x9D61,/* 0xF0-0xF7 */ + 0x58A8,0x9ED8,0x5011,0x520E,0x543B,0x554F,0x6587,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_DA[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x6C76,0x7D0A,0x7D0B,0x805E,0x868A,0x9580,0x96EF,/* 0xA0-0xA7 */ + 0x52FF,0x6C95,0x7269,0x5473,0x5A9A,0x5C3E,0x5D4B,0x5F4C,/* 0xA8-0xAF */ + 0x5FAE,0x672A,0x68B6,0x6963,0x6E3C,0x6E44,0x7709,0x7C73,/* 0xB0-0xB7 */ + 0x7F8E,0x8587,0x8B0E,0x8FF7,0x9761,0x9EF4,0x5CB7,0x60B6,/* 0xB8-0xBF */ + 0x610D,0x61AB,0x654F,0x65FB,0x65FC,0x6C11,0x6CEF,0x739F,/* 0xC0-0xC7 */ + 0x73C9,0x7DE1,0x9594,0x5BC6,0x871C,0x8B10,0x525D,0x535A,/* 0xC8-0xCF */ + 0x62CD,0x640F,0x64B2,0x6734,0x6A38,0x6CCA,0x73C0,0x749E,/* 0xD0-0xD7 */ + 0x7B94,0x7C95,0x7E1B,0x818A,0x8236,0x8584,0x8FEB,0x96F9,/* 0xD8-0xDF */ + 0x99C1,0x4F34,0x534A,0x53CD,0x53DB,0x62CC,0x642C,0x6500,/* 0xE0-0xE7 */ + 0x6591,0x69C3,0x6CEE,0x6F58,0x73ED,0x7554,0x7622,0x76E4,/* 0xE8-0xEF */ + 0x76FC,0x78D0,0x78FB,0x792C,0x7D46,0x822C,0x87E0,0x8FD4,/* 0xF0-0xF7 */ + 0x9812,0x98EF,0x52C3,0x62D4,0x64A5,0x6E24,0x6F51,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_DB[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x767C,0x8DCB,0x91B1,0x9262,0x9AEE,0x9B43,0x5023,/* 0xA0-0xA7 */ + 0x508D,0x574A,0x59A8,0x5C28,0x5E47,0x5F77,0x623F,0x653E,/* 0xA8-0xAF */ + 0x65B9,0x65C1,0x6609,0x678B,0x699C,0x6EC2,0x78C5,0x7D21,/* 0xB0-0xB7 */ + 0x80AA,0x8180,0x822B,0x82B3,0x84A1,0x868C,0x8A2A,0x8B17,/* 0xB8-0xBF */ + 0x90A6,0x9632,0x9F90,0x500D,0x4FF3,0xF963,0x57F9,0x5F98,/* 0xC0-0xC7 */ + 0x62DC,0x6392,0x676F,0x6E43,0x7119,0x76C3,0x80CC,0x80DA,/* 0xC8-0xCF */ + 0x88F4,0x88F5,0x8919,0x8CE0,0x8F29,0x914D,0x966A,0x4F2F,/* 0xD0-0xD7 */ + 0x4F70,0x5E1B,0x67CF,0x6822,0x767D,0x767E,0x9B44,0x5E61,/* 0xD8-0xDF */ + 0x6A0A,0x7169,0x71D4,0x756A,0xF964,0x7E41,0x8543,0x85E9,/* 0xE0-0xE7 */ + 0x98DC,0x4F10,0x7B4F,0x7F70,0x95A5,0x51E1,0x5E06,0x68B5,/* 0xE8-0xEF */ + 0x6C3E,0x6C4E,0x6CDB,0x72AF,0x7BC4,0x8303,0x6CD5,0x743A,/* 0xF0-0xF7 */ + 0x50FB,0x5288,0x58C1,0x64D8,0x6A97,0x74A7,0x7656,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_DC[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x78A7,0x8617,0x95E2,0x9739,0xF965,0x535E,0x5F01,/* 0xA0-0xA7 */ + 0x8B8A,0x8FA8,0x8FAF,0x908A,0x5225,0x77A5,0x9C49,0x9F08,/* 0xA8-0xAF */ + 0x4E19,0x5002,0x5175,0x5C5B,0x5E77,0x661E,0x663A,0x67C4,/* 0xB0-0xB7 */ + 0x68C5,0x70B3,0x7501,0x75C5,0x79C9,0x7ADD,0x8F27,0x9920,/* 0xB8-0xBF */ + 0x9A08,0x4FDD,0x5821,0x5831,0x5BF6,0x666E,0x6B65,0x6D11,/* 0xC0-0xC7 */ + 0x6E7A,0x6F7D,0x73E4,0x752B,0x83E9,0x88DC,0x8913,0x8B5C,/* 0xC8-0xCF */ + 0x8F14,0x4F0F,0x50D5,0x5310,0x535C,0x5B93,0x5FA9,0x670D,/* 0xD0-0xD7 */ + 0x798F,0x8179,0x832F,0x8514,0x8907,0x8986,0x8F39,0x8F3B,/* 0xD8-0xDF */ + 0x99A5,0x9C12,0x672C,0x4E76,0x4FF8,0x5949,0x5C01,0x5CEF,/* 0xE0-0xE7 */ + 0x5CF0,0x6367,0x68D2,0x70FD,0x71A2,0x742B,0x7E2B,0x84EC,/* 0xE8-0xEF */ + 0x8702,0x9022,0x92D2,0x9CF3,0x4E0D,0x4ED8,0x4FEF,0x5085,/* 0xF0-0xF7 */ + 0x5256,0x526F,0x5426,0x5490,0x57E0,0x592B,0x5A66,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_DD[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x5B5A,0x5B75,0x5BCC,0x5E9C,0xF966,0x6276,0x6577,/* 0xA0-0xA7 */ + 0x65A7,0x6D6E,0x6EA5,0x7236,0x7B26,0x7C3F,0x7F36,0x8150,/* 0xA8-0xAF */ + 0x8151,0x819A,0x8240,0x8299,0x83A9,0x8A03,0x8CA0,0x8CE6,/* 0xB0-0xB7 */ + 0x8CFB,0x8D74,0x8DBA,0x90E8,0x91DC,0x961C,0x9644,0x99D9,/* 0xB8-0xBF */ + 0x9CE7,0x5317,0x5206,0x5429,0x5674,0x58B3,0x5954,0x596E,/* 0xC0-0xC7 */ + 0x5FFF,0x61A4,0x626E,0x6610,0x6C7E,0x711A,0x76C6,0x7C89,/* 0xC8-0xCF */ + 0x7CDE,0x7D1B,0x82AC,0x8CC1,0x96F0,0xF967,0x4F5B,0x5F17,/* 0xD0-0xD7 */ + 0x5F7F,0x62C2,0x5D29,0x670B,0x68DA,0x787C,0x7E43,0x9D6C,/* 0xD8-0xDF */ + 0x4E15,0x5099,0x5315,0x532A,0x5351,0x5983,0x5A62,0x5E87,/* 0xE0-0xE7 */ + 0x60B2,0x618A,0x6249,0x6279,0x6590,0x6787,0x69A7,0x6BD4,/* 0xE8-0xEF */ + 0x6BD6,0x6BD7,0x6BD8,0x6CB8,0xF968,0x7435,0x75FA,0x7812,/* 0xF0-0xF7 */ + 0x7891,0x79D5,0x79D8,0x7C83,0x7DCB,0x7FE1,0x80A5,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_DE[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x813E,0x81C2,0x83F2,0x871A,0x88E8,0x8AB9,0x8B6C,/* 0xA0-0xA7 */ + 0x8CBB,0x9119,0x975E,0x98DB,0x9F3B,0x56AC,0x5B2A,0x5F6C,/* 0xA8-0xAF */ + 0x658C,0x6AB3,0x6BAF,0x6D5C,0x6FF1,0x7015,0x725D,0x73AD,/* 0xB0-0xB7 */ + 0x8CA7,0x8CD3,0x983B,0x6191,0x6C37,0x8058,0x9A01,0x4E4D,/* 0xB8-0xBF */ + 0x4E8B,0x4E9B,0x4ED5,0x4F3A,0x4F3C,0x4F7F,0x4FDF,0x50FF,/* 0xC0-0xC7 */ + 0x53F2,0x53F8,0x5506,0x55E3,0x56DB,0x58EB,0x5962,0x5A11,/* 0xC8-0xCF */ + 0x5BEB,0x5BFA,0x5C04,0x5DF3,0x5E2B,0x5F99,0x601D,0x6368,/* 0xD0-0xD7 */ + 0x659C,0x65AF,0x67F6,0x67FB,0x68AD,0x6B7B,0x6C99,0x6CD7,/* 0xD8-0xDF */ + 0x6E23,0x7009,0x7345,0x7802,0x793E,0x7940,0x7960,0x79C1,/* 0xE0-0xE7 */ + 0x7BE9,0x7D17,0x7D72,0x8086,0x820D,0x838E,0x84D1,0x86C7,/* 0xE8-0xEF */ + 0x88DF,0x8A50,0x8A5E,0x8B1D,0x8CDC,0x8D66,0x8FAD,0x90AA,/* 0xF0-0xF7 */ + 0x98FC,0x99DF,0x9E9D,0x524A,0xF969,0x6714,0xF96A,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_DF[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x5098,0x522A,0x5C71,0x6563,0x6C55,0x73CA,0x7523,/* 0xA0-0xA7 */ + 0x759D,0x7B97,0x849C,0x9178,0x9730,0x4E77,0x6492,0x6BBA,/* 0xA8-0xAF */ + 0x715E,0x85A9,0x4E09,0xF96B,0x6749,0x68EE,0x6E17,0x829F,/* 0xB0-0xB7 */ + 0x8518,0x886B,0x63F7,0x6F81,0x9212,0x98AF,0x4E0A,0x50B7,/* 0xB8-0xBF */ + 0x50CF,0x511F,0x5546,0x55AA,0x5617,0x5B40,0x5C19,0x5CE0,/* 0xC0-0xC7 */ + 0x5E38,0x5E8A,0x5EA0,0x5EC2,0x60F3,0x6851,0x6A61,0x6E58,/* 0xC8-0xCF */ + 0x723D,0x7240,0x72C0,0x76F8,0x7965,0x7BB1,0x7FD4,0x88F3,/* 0xD0-0xD7 */ + 0x89F4,0x8A73,0x8C61,0x8CDE,0x971C,0x585E,0x74BD,0x8CFD,/* 0xD8-0xDF */ + 0x55C7,0xF96C,0x7A61,0x7D22,0x8272,0x7272,0x751F,0x7525,/* 0xE0-0xE7 */ + 0xF96D,0x7B19,0x5885,0x58FB,0x5DBC,0x5E8F,0x5EB6,0x5F90,/* 0xE8-0xEF */ + 0x6055,0x6292,0x637F,0x654D,0x6691,0x66D9,0x66F8,0x6816,/* 0xF0-0xF7 */ + 0x68F2,0x7280,0x745E,0x7B6E,0x7D6E,0x7DD6,0x7F72,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E0[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x80E5,0x8212,0x85AF,0x897F,0x8A93,0x901D,0x92E4,/* 0xA0-0xA7 */ + 0x9ECD,0x9F20,0x5915,0x596D,0x5E2D,0x60DC,0x6614,0x6673,/* 0xA8-0xAF */ + 0x6790,0x6C50,0x6DC5,0x6F5F,0x77F3,0x78A9,0x84C6,0x91CB,/* 0xB0-0xB7 */ + 0x932B,0x4ED9,0x50CA,0x5148,0x5584,0x5B0B,0x5BA3,0x6247,/* 0xB8-0xBF */ + 0x657E,0x65CB,0x6E32,0x717D,0x7401,0x7444,0x7487,0x74BF,/* 0xC0-0xC7 */ + 0x766C,0x79AA,0x7DDA,0x7E55,0x7FA8,0x817A,0x81B3,0x8239,/* 0xC8-0xCF */ + 0x861A,0x87EC,0x8A75,0x8DE3,0x9078,0x9291,0x9425,0x994D,/* 0xD0-0xD7 */ + 0x9BAE,0x5368,0x5C51,0x6954,0x6CC4,0x6D29,0x6E2B,0x820C,/* 0xD8-0xDF */ + 0x859B,0x893B,0x8A2D,0x8AAA,0x96EA,0x9F67,0x5261,0x66B9,/* 0xE0-0xE7 */ + 0x6BB2,0x7E96,0x87FE,0x8D0D,0x9583,0x965D,0x651D,0x6D89,/* 0xE8-0xEF */ + 0x71EE,0xF96E,0x57CE,0x59D3,0x5BAC,0x6027,0x60FA,0x6210,/* 0xF0-0xF7 */ + 0x661F,0x665F,0x7329,0x73F9,0x76DB,0x7701,0x7B6C,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E1[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x8056,0x8072,0x8165,0x8AA0,0x9192,0x4E16,0x52E2,/* 0xA0-0xA7 */ + 0x6B72,0x6D17,0x7A05,0x7B39,0x7D30,0xF96F,0x8CB0,0x53EC,/* 0xA8-0xAF */ + 0x562F,0x5851,0x5BB5,0x5C0F,0x5C11,0x5DE2,0x6240,0x6383,/* 0xB0-0xB7 */ + 0x6414,0x662D,0x68B3,0x6CBC,0x6D88,0x6EAF,0x701F,0x70A4,/* 0xB8-0xBF */ + 0x71D2,0x7526,0x758F,0x758E,0x7619,0x7B11,0x7BE0,0x7C2B,/* 0xC0-0xC7 */ + 0x7D20,0x7D39,0x852C,0x856D,0x8607,0x8A34,0x900D,0x9061,/* 0xC8-0xCF */ + 0x90B5,0x92B7,0x97F6,0x9A37,0x4FD7,0x5C6C,0x675F,0x6D91,/* 0xD0-0xD7 */ + 0x7C9F,0x7E8C,0x8B16,0x8D16,0x901F,0x5B6B,0x5DFD,0x640D,/* 0xD8-0xDF */ + 0x84C0,0x905C,0x98E1,0x7387,0x5B8B,0x609A,0x677E,0x6DDE,/* 0xE0-0xE7 */ + 0x8A1F,0x8AA6,0x9001,0x980C,0x5237,0xF970,0x7051,0x788E,/* 0xE8-0xEF */ + 0x9396,0x8870,0x91D7,0x4FEE,0x53D7,0x55FD,0x56DA,0x5782,/* 0xF0-0xF7 */ + 0x58FD,0x5AC2,0x5B88,0x5CAB,0x5CC0,0x5E25,0x6101,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E2[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x620D,0x624B,0x6388,0x641C,0x6536,0x6578,0x6A39,/* 0xA0-0xA7 */ + 0x6B8A,0x6C34,0x6D19,0x6F31,0x71E7,0x72E9,0x7378,0x7407,/* 0xA8-0xAF */ + 0x74B2,0x7626,0x7761,0x79C0,0x7A57,0x7AEA,0x7CB9,0x7D8F,/* 0xB0-0xB7 */ + 0x7DAC,0x7E61,0x7F9E,0x8129,0x8331,0x8490,0x84DA,0x85EA,/* 0xB8-0xBF */ + 0x8896,0x8AB0,0x8B90,0x8F38,0x9042,0x9083,0x916C,0x9296,/* 0xC0-0xC7 */ + 0x92B9,0x968B,0x96A7,0x96A8,0x96D6,0x9700,0x9808,0x9996,/* 0xC8-0xCF */ + 0x9AD3,0x9B1A,0x53D4,0x587E,0x5919,0x5B70,0x5BBF,0x6DD1,/* 0xD0-0xD7 */ + 0x6F5A,0x719F,0x7421,0x74B9,0x8085,0x83FD,0x5DE1,0x5F87,/* 0xD8-0xDF */ + 0x5FAA,0x6042,0x65EC,0x6812,0x696F,0x6A53,0x6B89,0x6D35,/* 0xE0-0xE7 */ + 0x6DF3,0x73E3,0x76FE,0x77AC,0x7B4D,0x7D14,0x8123,0x821C,/* 0xE8-0xEF */ + 0x8340,0x84F4,0x8563,0x8A62,0x8AC4,0x9187,0x931E,0x9806,/* 0xF0-0xF7 */ + 0x99B4,0x620C,0x8853,0x8FF0,0x9265,0x5D07,0x5D27,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E3[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x5D69,0x745F,0x819D,0x8768,0x6FD5,0x62FE,0x7FD2,/* 0xA0-0xA7 */ + 0x8936,0x8972,0x4E1E,0x4E58,0x50E7,0x52DD,0x5347,0x627F,/* 0xA8-0xAF */ + 0x6607,0x7E69,0x8805,0x965E,0x4F8D,0x5319,0x5636,0x59CB,/* 0xB0-0xB7 */ + 0x5AA4,0x5C38,0x5C4E,0x5C4D,0x5E02,0x5F11,0x6043,0x65BD,/* 0xB8-0xBF */ + 0x662F,0x6642,0x67BE,0x67F4,0x731C,0x77E2,0x793A,0x7FC5,/* 0xC0-0xC7 */ + 0x8494,0x84CD,0x8996,0x8A66,0x8A69,0x8AE1,0x8C55,0x8C7A,/* 0xC8-0xCF */ + 0x57F4,0x5BD4,0x5F0F,0x606F,0x62ED,0x690D,0x6B96,0x6E5C,/* 0xD0-0xD7 */ + 0x7184,0x7BD2,0x8755,0x8B58,0x8EFE,0x98DF,0x98FE,0x4F38,/* 0xD8-0xDF */ + 0x4F81,0x4FE1,0x547B,0x5A20,0x5BB8,0x613C,0x65B0,0x6668,/* 0xE0-0xE7 */ + 0x71FC,0x7533,0x795E,0x7D33,0x814E,0x81E3,0x8398,0x85AA,/* 0xE8-0xEF */ + 0x85CE,0x8703,0x8A0A,0x8EAB,0x8F9B,0xF971,0x8FC5,0x5931,/* 0xF0-0xF7 */ + 0x5BA4,0x5BE6,0x6089,0x5BE9,0x5C0B,0x5FC3,0x6C81,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E4[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0xF972,0x6DF1,0x700B,0x751A,0x82AF,0x8AF6,0x4EC0,/* 0xA0-0xA7 */ + 0x5341,0xF973,0x96D9,0x6C0F,0x4E9E,0x4FC4,0x5152,0x555E,/* 0xA8-0xAF */ + 0x5A25,0x5CE8,0x6211,0x7259,0x82BD,0x83AA,0x86FE,0x8859,/* 0xB0-0xB7 */ + 0x8A1D,0x963F,0x96C5,0x9913,0x9D09,0x9D5D,0x580A,0x5CB3,/* 0xB8-0xBF */ + 0x5DBD,0x5E44,0x60E1,0x6115,0x63E1,0x6A02,0x6E25,0x9102,/* 0xC0-0xC7 */ + 0x9354,0x984E,0x9C10,0x9F77,0x5B89,0x5CB8,0x6309,0x664F,/* 0xC8-0xCF */ + 0x6848,0x773C,0x96C1,0x978D,0x9854,0x9B9F,0x65A1,0x8B01,/* 0xD0-0xD7 */ + 0x8ECB,0x95BC,0x5535,0x5CA9,0x5DD6,0x5EB5,0x6697,0x764C,/* 0xD8-0xDF */ + 0x83F4,0x95C7,0x58D3,0x62BC,0x72CE,0x9D28,0x4EF0,0x592E,/* 0xE0-0xE7 */ + 0x600F,0x663B,0x6B83,0x79E7,0x9D26,0x5393,0x54C0,0x57C3,/* 0xE8-0xEF */ + 0x5D16,0x611B,0x66D6,0x6DAF,0x788D,0x827E,0x9698,0x9744,/* 0xF0-0xF7 */ + 0x5384,0x627C,0x6396,0x6DB2,0x7E0A,0x814B,0x984D,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E5[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x6AFB,0x7F4C,0x9DAF,0x9E1A,0x4E5F,0x503B,0x51B6,/* 0xA0-0xA7 */ + 0x591C,0x60F9,0x63F6,0x6930,0x723A,0x8036,0xF974,0x91CE,/* 0xA8-0xAF */ + 0x5F31,0xF975,0xF976,0x7D04,0x82E5,0x846F,0x84BB,0x85E5,/* 0xB0-0xB7 */ + 0x8E8D,0xF977,0x4F6F,0xF978,0xF979,0x58E4,0x5B43,0x6059,/* 0xB8-0xBF */ + 0x63DA,0x6518,0x656D,0x6698,0xF97A,0x694A,0x6A23,0x6D0B,/* 0xC0-0xC7 */ + 0x7001,0x716C,0x75D2,0x760D,0x79B3,0x7A70,0xF97B,0x7F8A,/* 0xC8-0xCF */ + 0xF97C,0x8944,0xF97D,0x8B93,0x91C0,0x967D,0xF97E,0x990A,/* 0xD0-0xD7 */ + 0x5704,0x5FA1,0x65BC,0x6F01,0x7600,0x79A6,0x8A9E,0x99AD,/* 0xD8-0xDF */ + 0x9B5A,0x9F6C,0x5104,0x61B6,0x6291,0x6A8D,0x81C6,0x5043,/* 0xE0-0xE7 */ + 0x5830,0x5F66,0x7109,0x8A00,0x8AFA,0x5B7C,0x8616,0x4FFA,/* 0xE8-0xEF */ + 0x513C,0x56B4,0x5944,0x63A9,0x6DF9,0x5DAA,0x696D,0x5186,/* 0xF0-0xF7 */ + 0x4E88,0x4F59,0xF97F,0xF980,0xF981,0x5982,0xF982,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E6[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0xF983,0x6B5F,0x6C5D,0xF984,0x74B5,0x7916,0xF985,/* 0xA0-0xA7 */ + 0x8207,0x8245,0x8339,0x8F3F,0x8F5D,0xF986,0x9918,0xF987,/* 0xA8-0xAF */ + 0xF988,0xF989,0x4EA6,0xF98A,0x57DF,0x5F79,0x6613,0xF98B,/* 0xB0-0xB7 */ + 0xF98C,0x75AB,0x7E79,0x8B6F,0xF98D,0x9006,0x9A5B,0x56A5,/* 0xB8-0xBF */ + 0x5827,0x59F8,0x5A1F,0x5BB4,0xF98E,0x5EF6,0xF98F,0xF990,/* 0xC0-0xC7 */ + 0x6350,0x633B,0xF991,0x693D,0x6C87,0x6CBF,0x6D8E,0x6D93,/* 0xC8-0xCF */ + 0x6DF5,0x6F14,0xF992,0x70DF,0x7136,0x7159,0xF993,0x71C3,/* 0xD0-0xD7 */ + 0x71D5,0xF994,0x784F,0x786F,0xF995,0x7B75,0x7DE3,0xF996,/* 0xD8-0xDF */ + 0x7E2F,0xF997,0x884D,0x8EDF,0xF998,0xF999,0xF99A,0x925B,/* 0xE0-0xE7 */ + 0xF99B,0x9CF6,0xF99C,0xF99D,0xF99E,0x6085,0x6D85,0xF99F,/* 0xE8-0xEF */ + 0x71B1,0xF9A0,0xF9A1,0x95B1,0x53AD,0xF9A2,0xF9A3,0xF9A4,/* 0xF0-0xF7 */ + 0x67D3,0xF9A5,0x708E,0x7130,0x7430,0x8276,0x82D2,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E7[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0xF9A6,0x95BB,0x9AE5,0x9E7D,0x66C4,0xF9A7,0x71C1,/* 0xA0-0xA7 */ + 0x8449,0xF9A8,0xF9A9,0x584B,0xF9AA,0xF9AB,0x5DB8,0x5F71,/* 0xA8-0xAF */ + 0xF9AC,0x6620,0x668E,0x6979,0x69AE,0x6C38,0x6CF3,0x6E36,/* 0xB0-0xB7 */ + 0x6F41,0x6FDA,0x701B,0x702F,0x7150,0x71DF,0x7370,0xF9AD,/* 0xB8-0xBF */ + 0x745B,0xF9AE,0x74D4,0x76C8,0x7A4E,0x7E93,0xF9AF,0xF9B0,/* 0xC0-0xC7 */ + 0x82F1,0x8A60,0x8FCE,0xF9B1,0x9348,0xF9B2,0x9719,0xF9B3,/* 0xC8-0xCF */ + 0xF9B4,0x4E42,0x502A,0xF9B5,0x5208,0x53E1,0x66F3,0x6C6D,/* 0xD0-0xD7 */ + 0x6FCA,0x730A,0x777F,0x7A62,0x82AE,0x85DD,0x8602,0xF9B6,/* 0xD8-0xDF */ + 0x88D4,0x8A63,0x8B7D,0x8C6B,0xF9B7,0x92B3,0xF9B8,0x9713,/* 0xE0-0xE7 */ + 0x9810,0x4E94,0x4F0D,0x4FC9,0x50B2,0x5348,0x543E,0x5433,/* 0xE8-0xEF */ + 0x55DA,0x5862,0x58BA,0x5967,0x5A1B,0x5BE4,0x609F,0xF9B9,/* 0xF0-0xF7 */ + 0x61CA,0x6556,0x65FF,0x6664,0x68A7,0x6C5A,0x6FB3,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E8[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x70CF,0x71AC,0x7352,0x7B7D,0x8708,0x8AA4,0x9C32,/* 0xA0-0xA7 */ + 0x9F07,0x5C4B,0x6C83,0x7344,0x7389,0x923A,0x6EAB,0x7465,/* 0xA8-0xAF */ + 0x761F,0x7A69,0x7E15,0x860A,0x5140,0x58C5,0x64C1,0x74EE,/* 0xB0-0xB7 */ + 0x7515,0x7670,0x7FC1,0x9095,0x96CD,0x9954,0x6E26,0x74E6,/* 0xB8-0xBF */ + 0x7AA9,0x7AAA,0x81E5,0x86D9,0x8778,0x8A1B,0x5A49,0x5B8C,/* 0xC0-0xC7 */ + 0x5B9B,0x68A1,0x6900,0x6D63,0x73A9,0x7413,0x742C,0x7897,/* 0xC8-0xCF */ + 0x7DE9,0x7FEB,0x8118,0x8155,0x839E,0x8C4C,0x962E,0x9811,/* 0xD0-0xD7 */ + 0x66F0,0x5F80,0x65FA,0x6789,0x6C6A,0x738B,0x502D,0x5A03,/* 0xD8-0xDF */ + 0x6B6A,0x77EE,0x5916,0x5D6C,0x5DCD,0x7325,0x754F,0xF9BA,/* 0xE0-0xE7 */ + 0xF9BB,0x50E5,0x51F9,0x582F,0x592D,0x5996,0x59DA,0x5BE5,/* 0xE8-0xEF */ + 0xF9BC,0xF9BD,0x5DA2,0x62D7,0x6416,0x6493,0x64FE,0xF9BE,/* 0xF0-0xF7 */ + 0x66DC,0xF9BF,0x6A48,0xF9C0,0x71FF,0x7464,0xF9C1,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E9[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x7A88,0x7AAF,0x7E47,0x7E5E,0x8000,0x8170,0xF9C2,/* 0xA0-0xA7 */ + 0x87EF,0x8981,0x8B20,0x9059,0xF9C3,0x9080,0x9952,0x617E,/* 0xA8-0xAF */ + 0x6B32,0x6D74,0x7E1F,0x8925,0x8FB1,0x4FD1,0x50AD,0x5197,/* 0xB0-0xB7 */ + 0x52C7,0x57C7,0x5889,0x5BB9,0x5EB8,0x6142,0x6995,0x6D8C,/* 0xB8-0xBF */ + 0x6E67,0x6EB6,0x7194,0x7462,0x7528,0x752C,0x8073,0x8338,/* 0xC0-0xC7 */ + 0x84C9,0x8E0A,0x9394,0x93DE,0xF9C4,0x4E8E,0x4F51,0x5076,/* 0xC8-0xCF */ + 0x512A,0x53C8,0x53CB,0x53F3,0x5B87,0x5BD3,0x5C24,0x611A,/* 0xD0-0xD7 */ + 0x6182,0x65F4,0x725B,0x7397,0x7440,0x76C2,0x7950,0x7991,/* 0xD8-0xDF */ + 0x79B9,0x7D06,0x7FBD,0x828B,0x85D5,0x865E,0x8FC2,0x9047,/* 0xE0-0xE7 */ + 0x90F5,0x91EA,0x9685,0x96E8,0x96E9,0x52D6,0x5F67,0x65ED,/* 0xE8-0xEF */ + 0x6631,0x682F,0x715C,0x7A36,0x90C1,0x980A,0x4E91,0xF9C5,/* 0xF0-0xF7 */ + 0x6A52,0x6B9E,0x6F90,0x7189,0x8018,0x82B8,0x8553,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_EA[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x904B,0x9695,0x96F2,0x97FB,0x851A,0x9B31,0x4E90,/* 0xA0-0xA7 */ + 0x718A,0x96C4,0x5143,0x539F,0x54E1,0x5713,0x5712,0x57A3,/* 0xA8-0xAF */ + 0x5A9B,0x5AC4,0x5BC3,0x6028,0x613F,0x63F4,0x6C85,0x6D39,/* 0xB0-0xB7 */ + 0x6E72,0x6E90,0x7230,0x733F,0x7457,0x82D1,0x8881,0x8F45,/* 0xB8-0xBF */ + 0x9060,0xF9C6,0x9662,0x9858,0x9D1B,0x6708,0x8D8A,0x925E,/* 0xC0-0xC7 */ + 0x4F4D,0x5049,0x50DE,0x5371,0x570D,0x59D4,0x5A01,0x5C09,/* 0xC8-0xCF */ + 0x6170,0x6690,0x6E2D,0x7232,0x744B,0x7DEF,0x80C3,0x840E,/* 0xD0-0xD7 */ + 0x8466,0x853F,0x875F,0x885B,0x8918,0x8B02,0x9055,0x97CB,/* 0xD8-0xDF */ + 0x9B4F,0x4E73,0x4F91,0x5112,0x516A,0xF9C7,0x552F,0x55A9,/* 0xE0-0xE7 */ + 0x5B7A,0x5BA5,0x5E7C,0x5E7D,0x5EBE,0x60A0,0x60DF,0x6108,/* 0xE8-0xEF */ + 0x6109,0x63C4,0x6538,0x6709,0xF9C8,0x67D4,0x67DA,0xF9C9,/* 0xF0-0xF7 */ + 0x6961,0x6962,0x6CB9,0x6D27,0xF9CA,0x6E38,0xF9CB,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_EB[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x6FE1,0x7336,0x7337,0xF9CC,0x745C,0x7531,0xF9CD,/* 0xA0-0xA7 */ + 0x7652,0xF9CE,0xF9CF,0x7DAD,0x81FE,0x8438,0x88D5,0x8A98,/* 0xA8-0xAF */ + 0x8ADB,0x8AED,0x8E30,0x8E42,0x904A,0x903E,0x907A,0x9149,/* 0xB0-0xB7 */ + 0x91C9,0x936E,0xF9D0,0xF9D1,0x5809,0xF9D2,0x6BD3,0x8089,/* 0xB8-0xBF */ + 0x80B2,0xF9D3,0xF9D4,0x5141,0x596B,0x5C39,0xF9D5,0xF9D6,/* 0xC0-0xC7 */ + 0x6F64,0x73A7,0x80E4,0x8D07,0xF9D7,0x9217,0x958F,0xF9D8,/* 0xC8-0xCF */ + 0xF9D9,0xF9DA,0xF9DB,0x807F,0x620E,0x701C,0x7D68,0x878D,/* 0xD0-0xD7 */ + 0xF9DC,0x57A0,0x6069,0x6147,0x6BB7,0x8ABE,0x9280,0x96B1,/* 0xD8-0xDF */ + 0x4E59,0x541F,0x6DEB,0x852D,0x9670,0x97F3,0x98EE,0x63D6,/* 0xE0-0xE7 */ + 0x6CE3,0x9091,0x51DD,0x61C9,0x81BA,0x9DF9,0x4F9D,0x501A,/* 0xE8-0xEF */ + 0x5100,0x5B9C,0x610F,0x61FF,0x64EC,0x6905,0x6BC5,0x7591,/* 0xF0-0xF7 */ + 0x77E3,0x7FA9,0x8264,0x858F,0x87FB,0x8863,0x8ABC,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_EC[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x8B70,0x91AB,0x4E8C,0x4EE5,0x4F0A,0xF9DD,0xF9DE,/* 0xA0-0xA7 */ + 0x5937,0x59E8,0xF9DF,0x5DF2,0x5F1B,0x5F5B,0x6021,0xF9E0,/* 0xA8-0xAF */ + 0xF9E1,0xF9E2,0xF9E3,0x723E,0x73E5,0xF9E4,0x7570,0x75CD,/* 0xB0-0xB7 */ + 0xF9E5,0x79FB,0xF9E6,0x800C,0x8033,0x8084,0x82E1,0x8351,/* 0xB8-0xBF */ + 0xF9E7,0xF9E8,0x8CBD,0x8CB3,0x9087,0xF9E9,0xF9EA,0x98F4,/* 0xC0-0xC7 */ + 0x990C,0xF9EB,0xF9EC,0x7037,0x76CA,0x7FCA,0x7FCC,0x7FFC,/* 0xC8-0xCF */ + 0x8B1A,0x4EBA,0x4EC1,0x5203,0x5370,0xF9ED,0x54BD,0x56E0,/* 0xD0-0xD7 */ + 0x59FB,0x5BC5,0x5F15,0x5FCD,0x6E6E,0xF9EE,0xF9EF,0x7D6A,/* 0xD8-0xDF */ + 0x8335,0xF9F0,0x8693,0x8A8D,0xF9F1,0x976D,0x9777,0xF9F2,/* 0xE0-0xE7 */ + 0xF9F3,0x4E00,0x4F5A,0x4F7E,0x58F9,0x65E5,0x6EA2,0x9038,/* 0xE8-0xEF */ + 0x93B0,0x99B9,0x4EFB,0x58EC,0x598A,0x59D9,0x6041,0xF9F4,/* 0xF0-0xF7 */ + 0xF9F5,0x7A14,0xF9F6,0x834F,0x8CC3,0x5165,0x5344,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_ED[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0xF9F7,0xF9F8,0xF9F9,0x4ECD,0x5269,0x5B55,0x82BF,/* 0xA0-0xA7 */ + 0x4ED4,0x523A,0x54A8,0x59C9,0x59FF,0x5B50,0x5B57,0x5B5C,/* 0xA8-0xAF */ + 0x6063,0x6148,0x6ECB,0x7099,0x716E,0x7386,0x74F7,0x75B5,/* 0xB0-0xB7 */ + 0x78C1,0x7D2B,0x8005,0x81EA,0x8328,0x8517,0x85C9,0x8AEE,/* 0xB8-0xBF */ + 0x8CC7,0x96CC,0x4F5C,0x52FA,0x56BC,0x65AB,0x6628,0x707C,/* 0xC0-0xC7 */ + 0x70B8,0x7235,0x7DBD,0x828D,0x914C,0x96C0,0x9D72,0x5B71,/* 0xC8-0xCF */ + 0x68E7,0x6B98,0x6F7A,0x76DE,0x5C91,0x66AB,0x6F5B,0x7BB4,/* 0xD0-0xD7 */ + 0x7C2A,0x8836,0x96DC,0x4E08,0x4ED7,0x5320,0x5834,0x58BB,/* 0xD8-0xDF */ + 0x58EF,0x596C,0x5C07,0x5E33,0x5E84,0x5F35,0x638C,0x66B2,/* 0xE0-0xE7 */ + 0x6756,0x6A1F,0x6AA3,0x6B0C,0x6F3F,0x7246,0xF9FA,0x7350,/* 0xE8-0xEF */ + 0x748B,0x7AE0,0x7CA7,0x8178,0x81DF,0x81E7,0x838A,0x846C,/* 0xF0-0xF7 */ + 0x8523,0x8594,0x85CF,0x88DD,0x8D13,0x91AC,0x9577,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_EE[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x969C,0x518D,0x54C9,0x5728,0x5BB0,0x624D,0x6750,/* 0xA0-0xA7 */ + 0x683D,0x6893,0x6E3D,0x6ED3,0x707D,0x7E21,0x88C1,0x8CA1,/* 0xA8-0xAF */ + 0x8F09,0x9F4B,0x9F4E,0x722D,0x7B8F,0x8ACD,0x931A,0x4F47,/* 0xB0-0xB7 */ + 0x4F4E,0x5132,0x5480,0x59D0,0x5E95,0x62B5,0x6775,0x696E,/* 0xB8-0xBF */ + 0x6A17,0x6CAE,0x6E1A,0x72D9,0x732A,0x75BD,0x7BB8,0x7D35,/* 0xC0-0xC7 */ + 0x82E7,0x83F9,0x8457,0x85F7,0x8A5B,0x8CAF,0x8E87,0x9019,/* 0xC8-0xCF */ + 0x90B8,0x96CE,0x9F5F,0x52E3,0x540A,0x5AE1,0x5BC2,0x6458,/* 0xD0-0xD7 */ + 0x6575,0x6EF4,0x72C4,0xF9FB,0x7684,0x7A4D,0x7B1B,0x7C4D,/* 0xD8-0xDF */ + 0x7E3E,0x7FDF,0x837B,0x8B2B,0x8CCA,0x8D64,0x8DE1,0x8E5F,/* 0xE0-0xE7 */ + 0x8FEA,0x8FF9,0x9069,0x93D1,0x4F43,0x4F7A,0x50B3,0x5168,/* 0xE8-0xEF */ + 0x5178,0x524D,0x526A,0x5861,0x587C,0x5960,0x5C08,0x5C55,/* 0xF0-0xF7 */ + 0x5EDB,0x609B,0x6230,0x6813,0x6BBF,0x6C08,0x6FB1,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_EF[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x714E,0x7420,0x7530,0x7538,0x7551,0x7672,0x7B4C,/* 0xA0-0xA7 */ + 0x7B8B,0x7BAD,0x7BC6,0x7E8F,0x8A6E,0x8F3E,0x8F49,0x923F,/* 0xA8-0xAF */ + 0x9293,0x9322,0x942B,0x96FB,0x985A,0x986B,0x991E,0x5207,/* 0xB0-0xB7 */ + 0x622A,0x6298,0x6D59,0x7664,0x7ACA,0x7BC0,0x7D76,0x5360,/* 0xB8-0xBF */ + 0x5CBE,0x5E97,0x6F38,0x70B9,0x7C98,0x9711,0x9B8E,0x9EDE,/* 0xC0-0xC7 */ + 0x63A5,0x647A,0x8776,0x4E01,0x4E95,0x4EAD,0x505C,0x5075,/* 0xC8-0xCF */ + 0x5448,0x59C3,0x5B9A,0x5E40,0x5EAD,0x5EF7,0x5F81,0x60C5,/* 0xD0-0xD7 */ + 0x633A,0x653F,0x6574,0x65CC,0x6676,0x6678,0x67FE,0x6968,/* 0xD8-0xDF */ + 0x6A89,0x6B63,0x6C40,0x6DC0,0x6DE8,0x6E1F,0x6E5E,0x701E,/* 0xE0-0xE7 */ + 0x70A1,0x738E,0x73FD,0x753A,0x775B,0x7887,0x798E,0x7A0B,/* 0xE8-0xEF */ + 0x7A7D,0x7CBE,0x7D8E,0x8247,0x8A02,0x8AEA,0x8C9E,0x912D,/* 0xF0-0xF7 */ + 0x914A,0x91D8,0x9266,0x92CC,0x9320,0x9706,0x9756,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F0[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x975C,0x9802,0x9F0E,0x5236,0x5291,0x557C,0x5824,/* 0xA0-0xA7 */ + 0x5E1D,0x5F1F,0x608C,0x63D0,0x68AF,0x6FDF,0x796D,0x7B2C,/* 0xA8-0xAF */ + 0x81CD,0x85BA,0x88FD,0x8AF8,0x8E44,0x918D,0x9664,0x969B,/* 0xB0-0xB7 */ + 0x973D,0x984C,0x9F4A,0x4FCE,0x5146,0x51CB,0x52A9,0x5632,/* 0xB8-0xBF */ + 0x5F14,0x5F6B,0x63AA,0x64CD,0x65E9,0x6641,0x66FA,0x66F9,/* 0xC0-0xC7 */ + 0x671D,0x689D,0x68D7,0x69FD,0x6F15,0x6F6E,0x7167,0x71E5,/* 0xC8-0xCF */ + 0x722A,0x74AA,0x773A,0x7956,0x795A,0x79DF,0x7A20,0x7A95,/* 0xD0-0xD7 */ + 0x7C97,0x7CDF,0x7D44,0x7E70,0x8087,0x85FB,0x86A4,0x8A54,/* 0xD8-0xDF */ + 0x8ABF,0x8D99,0x8E81,0x9020,0x906D,0x91E3,0x963B,0x96D5,/* 0xE0-0xE7 */ + 0x9CE5,0x65CF,0x7C07,0x8DB3,0x93C3,0x5B58,0x5C0A,0x5352,/* 0xE8-0xEF */ + 0x62D9,0x731D,0x5027,0x5B97,0x5F9E,0x60B0,0x616B,0x68D5,/* 0xF0-0xF7 */ + 0x6DD9,0x742E,0x7A2E,0x7D42,0x7D9C,0x7E31,0x816B,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F1[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x8E2A,0x8E35,0x937E,0x9418,0x4F50,0x5750,0x5DE6,/* 0xA0-0xA7 */ + 0x5EA7,0x632B,0x7F6A,0x4E3B,0x4F4F,0x4F8F,0x505A,0x59DD,/* 0xA8-0xAF */ + 0x80C4,0x546A,0x5468,0x55FE,0x594F,0x5B99,0x5DDE,0x5EDA,/* 0xB0-0xB7 */ + 0x665D,0x6731,0x67F1,0x682A,0x6CE8,0x6D32,0x6E4A,0x6F8D,/* 0xB8-0xBF */ + 0x70B7,0x73E0,0x7587,0x7C4C,0x7D02,0x7D2C,0x7DA2,0x821F,/* 0xC0-0xC7 */ + 0x86DB,0x8A3B,0x8A85,0x8D70,0x8E8A,0x8F33,0x9031,0x914E,/* 0xC8-0xCF */ + 0x9152,0x9444,0x99D0,0x7AF9,0x7CA5,0x4FCA,0x5101,0x51C6,/* 0xD0-0xD7 */ + 0x57C8,0x5BEF,0x5CFB,0x6659,0x6A3D,0x6D5A,0x6E96,0x6FEC,/* 0xD8-0xDF */ + 0x710C,0x756F,0x7AE3,0x8822,0x9021,0x9075,0x96CB,0x99FF,/* 0xE0-0xE7 */ + 0x8301,0x4E2D,0x4EF2,0x8846,0x91CD,0x537D,0x6ADB,0x696B,/* 0xE8-0xEF */ + 0x6C41,0x847A,0x589E,0x618E,0x66FE,0x62EF,0x70DD,0x7511,/* 0xF0-0xF7 */ + 0x75C7,0x7E52,0x84B8,0x8B49,0x8D08,0x4E4B,0x53EA,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F2[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x54AB,0x5730,0x5740,0x5FD7,0x6301,0x6307,0x646F,/* 0xA0-0xA7 */ + 0x652F,0x65E8,0x667A,0x679D,0x67B3,0x6B62,0x6C60,0x6C9A,/* 0xA8-0xAF */ + 0x6F2C,0x77E5,0x7825,0x7949,0x7957,0x7D19,0x80A2,0x8102,/* 0xB0-0xB7 */ + 0x81F3,0x829D,0x82B7,0x8718,0x8A8C,0xF9FC,0x8D04,0x8DBE,/* 0xB8-0xBF */ + 0x9072,0x76F4,0x7A19,0x7A37,0x7E54,0x8077,0x5507,0x55D4,/* 0xC0-0xC7 */ + 0x5875,0x632F,0x6422,0x6649,0x664B,0x686D,0x699B,0x6B84,/* 0xC8-0xCF */ + 0x6D25,0x6EB1,0x73CD,0x7468,0x74A1,0x755B,0x75B9,0x76E1,/* 0xD0-0xD7 */ + 0x771E,0x778B,0x79E6,0x7E09,0x7E1D,0x81FB,0x852F,0x8897,/* 0xD8-0xDF */ + 0x8A3A,0x8CD1,0x8EEB,0x8FB0,0x9032,0x93AD,0x9663,0x9673,/* 0xE0-0xE7 */ + 0x9707,0x4F84,0x53F1,0x59EA,0x5AC9,0x5E19,0x684E,0x74C6,/* 0xE8-0xEF */ + 0x75BE,0x79E9,0x7A92,0x81A3,0x86ED,0x8CEA,0x8DCC,0x8FED,/* 0xF0-0xF7 */ + 0x659F,0x6715,0xF9FD,0x57F7,0x6F57,0x7DDD,0x8F2F,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F3[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x93F6,0x96C6,0x5FB5,0x61F2,0x6F84,0x4E14,0x4F98,/* 0xA0-0xA7 */ + 0x501F,0x53C9,0x55DF,0x5D6F,0x5DEE,0x6B21,0x6B64,0x78CB,/* 0xA8-0xAF */ + 0x7B9A,0xF9FE,0x8E49,0x8ECA,0x906E,0x6349,0x643E,0x7740,/* 0xB0-0xB7 */ + 0x7A84,0x932F,0x947F,0x9F6A,0x64B0,0x6FAF,0x71E6,0x74A8,/* 0xB8-0xBF */ + 0x74DA,0x7AC4,0x7C12,0x7E82,0x7CB2,0x7E98,0x8B9A,0x8D0A,/* 0xC0-0xC7 */ + 0x947D,0x9910,0x994C,0x5239,0x5BDF,0x64E6,0x672D,0x7D2E,/* 0xC8-0xCF */ + 0x50ED,0x53C3,0x5879,0x6158,0x6159,0x61FA,0x65AC,0x7AD9,/* 0xD0-0xD7 */ + 0x8B92,0x8B96,0x5009,0x5021,0x5275,0x5531,0x5A3C,0x5EE0,/* 0xD8-0xDF */ + 0x5F70,0x6134,0x655E,0x660C,0x6636,0x66A2,0x69CD,0x6EC4,/* 0xE0-0xE7 */ + 0x6F32,0x7316,0x7621,0x7A93,0x8139,0x8259,0x83D6,0x84BC,/* 0xE8-0xEF */ + 0x50B5,0x57F0,0x5BC0,0x5BE8,0x5F69,0x63A1,0x7826,0x7DB5,/* 0xF0-0xF7 */ + 0x83DC,0x8521,0x91C7,0x91F5,0x518A,0x67F5,0x7B56,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F4[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x8CAC,0x51C4,0x59BB,0x60BD,0x8655,0x501C,0xF9FF,/* 0xA0-0xA7 */ + 0x5254,0x5C3A,0x617D,0x621A,0x62D3,0x64F2,0x65A5,0x6ECC,/* 0xA8-0xAF */ + 0x7620,0x810A,0x8E60,0x965F,0x96BB,0x4EDF,0x5343,0x5598,/* 0xB0-0xB7 */ + 0x5929,0x5DDD,0x64C5,0x6CC9,0x6DFA,0x7394,0x7A7F,0x821B,/* 0xB8-0xBF */ + 0x85A6,0x8CE4,0x8E10,0x9077,0x91E7,0x95E1,0x9621,0x97C6,/* 0xC0-0xC7 */ + 0x51F8,0x54F2,0x5586,0x5FB9,0x64A4,0x6F88,0x7DB4,0x8F1F,/* 0xC8-0xCF */ + 0x8F4D,0x9435,0x50C9,0x5C16,0x6CBE,0x6DFB,0x751B,0x77BB,/* 0xD0-0xD7 */ + 0x7C3D,0x7C64,0x8A79,0x8AC2,0x581E,0x59BE,0x5E16,0x6377,/* 0xD8-0xDF */ + 0x7252,0x758A,0x776B,0x8ADC,0x8CBC,0x8F12,0x5EF3,0x6674,/* 0xE0-0xE7 */ + 0x6DF8,0x807D,0x83C1,0x8ACB,0x9751,0x9BD6,0xFA00,0x5243,/* 0xE8-0xEF */ + 0x66FF,0x6D95,0x6EEF,0x7DE0,0x8AE6,0x902E,0x905E,0x9AD4,/* 0xF0-0xF7 */ + 0x521D,0x527F,0x54E8,0x6194,0x6284,0x62DB,0x68A2,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F5[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x6912,0x695A,0x6A35,0x7092,0x7126,0x785D,0x7901,/* 0xA0-0xA7 */ + 0x790E,0x79D2,0x7A0D,0x8096,0x8278,0x82D5,0x8349,0x8549,/* 0xA8-0xAF */ + 0x8C82,0x8D85,0x9162,0x918B,0x91AE,0x4FC3,0x56D1,0x71ED,/* 0xB0-0xB7 */ + 0x77D7,0x8700,0x89F8,0x5BF8,0x5FD6,0x6751,0x90A8,0x53E2,/* 0xB8-0xBF */ + 0x585A,0x5BF5,0x60A4,0x6181,0x6460,0x7E3D,0x8070,0x8525,/* 0xC0-0xC7 */ + 0x9283,0x64AE,0x50AC,0x5D14,0x6700,0x589C,0x62BD,0x63A8,/* 0xC8-0xCF */ + 0x690E,0x6978,0x6A1E,0x6E6B,0x76BA,0x79CB,0x82BB,0x8429,/* 0xD0-0xD7 */ + 0x8ACF,0x8DA8,0x8FFD,0x9112,0x914B,0x919C,0x9310,0x9318,/* 0xD8-0xDF */ + 0x939A,0x96DB,0x9A36,0x9C0D,0x4E11,0x755C,0x795D,0x7AFA,/* 0xE0-0xE7 */ + 0x7B51,0x7BC9,0x7E2E,0x84C4,0x8E59,0x8E74,0x8EF8,0x9010,/* 0xE8-0xEF */ + 0x6625,0x693F,0x7443,0x51FA,0x672E,0x9EDC,0x5145,0x5FE0,/* 0xF0-0xF7 */ + 0x6C96,0x87F2,0x885D,0x8877,0x60B4,0x81B5,0x8403,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F6[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x8D05,0x53D6,0x5439,0x5634,0x5A36,0x5C31,0x708A,/* 0xA0-0xA7 */ + 0x7FE0,0x805A,0x8106,0x81ED,0x8DA3,0x9189,0x9A5F,0x9DF2,/* 0xA8-0xAF */ + 0x5074,0x4EC4,0x53A0,0x60FB,0x6E2C,0x5C64,0x4F88,0x5024,/* 0xB0-0xB7 */ + 0x55E4,0x5CD9,0x5E5F,0x6065,0x6894,0x6CBB,0x6DC4,0x71BE,/* 0xB8-0xBF */ + 0x75D4,0x75F4,0x7661,0x7A1A,0x7A49,0x7DC7,0x7DFB,0x7F6E,/* 0xC0-0xC7 */ + 0x81F4,0x86A9,0x8F1C,0x96C9,0x99B3,0x9F52,0x5247,0x52C5,/* 0xC8-0xCF */ + 0x98ED,0x89AA,0x4E03,0x67D2,0x6F06,0x4FB5,0x5BE2,0x6795,/* 0xD0-0xD7 */ + 0x6C88,0x6D78,0x741B,0x7827,0x91DD,0x937C,0x87C4,0x79E4,/* 0xD8-0xDF */ + 0x7A31,0x5FEB,0x4ED6,0x54A4,0x553E,0x58AE,0x59A5,0x60F0,/* 0xE0-0xE7 */ + 0x6253,0x62D6,0x6736,0x6955,0x8235,0x9640,0x99B1,0x99DD,/* 0xE8-0xEF */ + 0x502C,0x5353,0x5544,0x577C,0xFA01,0x6258,0xFA02,0x64E2,/* 0xF0-0xF7 */ + 0x666B,0x67DD,0x6FC1,0x6FEF,0x7422,0x7438,0x8A17,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F7[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x9438,0x5451,0x5606,0x5766,0x5F48,0x619A,0x6B4E,/* 0xA0-0xA7 */ + 0x7058,0x70AD,0x7DBB,0x8A95,0x596A,0x812B,0x63A2,0x7708,/* 0xA8-0xAF */ + 0x803D,0x8CAA,0x5854,0x642D,0x69BB,0x5B95,0x5E11,0x6E6F,/* 0xB0-0xB7 */ + 0xFA03,0x8569,0x514C,0x53F0,0x592A,0x6020,0x614B,0x6B86,/* 0xB8-0xBF */ + 0x6C70,0x6CF0,0x7B1E,0x80CE,0x82D4,0x8DC6,0x90B0,0x98B1,/* 0xC0-0xC7 */ + 0xFA04,0x64C7,0x6FA4,0x6491,0x6504,0x514E,0x5410,0x571F,/* 0xC8-0xCF */ + 0x8A0E,0x615F,0x6876,0xFA05,0x75DB,0x7B52,0x7D71,0x901A,/* 0xD0-0xD7 */ + 0x5806,0x69CC,0x817F,0x892A,0x9000,0x9839,0x5078,0x5957,/* 0xD8-0xDF */ + 0x59AC,0x6295,0x900F,0x9B2A,0x615D,0x7279,0x95D6,0x5761,/* 0xE0-0xE7 */ + 0x5A46,0x5DF4,0x628A,0x64AD,0x64FA,0x6777,0x6CE2,0x6D3E,/* 0xE8-0xEF */ + 0x722C,0x7436,0x7834,0x7F77,0x82AD,0x8DDB,0x9817,0x5224,/* 0xF0-0xF7 */ + 0x5742,0x677F,0x7248,0x74E3,0x8CA9,0x8FA6,0x9211,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F8[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x962A,0x516B,0x53ED,0x634C,0x4F69,0x5504,0x6096,/* 0xA0-0xA7 */ + 0x6557,0x6C9B,0x6D7F,0x724C,0x72FD,0x7A17,0x8987,0x8C9D,/* 0xA8-0xAF */ + 0x5F6D,0x6F8E,0x70F9,0x81A8,0x610E,0x4FBF,0x504F,0x6241,/* 0xB0-0xB7 */ + 0x7247,0x7BC7,0x7DE8,0x7FE9,0x904D,0x97AD,0x9A19,0x8CB6,/* 0xB8-0xBF */ + 0x576A,0x5E73,0x67B0,0x840D,0x8A55,0x5420,0x5B16,0x5E63,/* 0xC0-0xC7 */ + 0x5EE2,0x5F0A,0x6583,0x80BA,0x853D,0x9589,0x965B,0x4F48,/* 0xC8-0xCF */ + 0x5305,0x530D,0x530F,0x5486,0x54FA,0x5703,0x5E03,0x6016,/* 0xD0-0xD7 */ + 0x629B,0x62B1,0x6355,0xFA06,0x6CE1,0x6D66,0x75B1,0x7832,/* 0xD8-0xDF */ + 0x80DE,0x812F,0x82DE,0x8461,0x84B2,0x888D,0x8912,0x900B,/* 0xE0-0xE7 */ + 0x92EA,0x98FD,0x9B91,0x5E45,0x66B4,0x66DD,0x7011,0x7206,/* 0xE8-0xEF */ + 0xFA07,0x4FF5,0x527D,0x5F6A,0x6153,0x6753,0x6A19,0x6F02,/* 0xF0-0xF7 */ + 0x74E2,0x7968,0x8868,0x8C79,0x98C7,0x98C4,0x9A43,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F9[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x54C1,0x7A1F,0x6953,0x8AF7,0x8C4A,0x98A8,0x99AE,/* 0xA0-0xA7 */ + 0x5F7C,0x62AB,0x75B2,0x76AE,0x88AB,0x907F,0x9642,0x5339,/* 0xA8-0xAF */ + 0x5F3C,0x5FC5,0x6CCC,0x73CC,0x7562,0x758B,0x7B46,0x82FE,/* 0xB0-0xB7 */ + 0x999D,0x4E4F,0x903C,0x4E0B,0x4F55,0x53A6,0x590F,0x5EC8,/* 0xB8-0xBF */ + 0x6630,0x6CB3,0x7455,0x8377,0x8766,0x8CC0,0x9050,0x971E,/* 0xC0-0xC7 */ + 0x9C15,0x58D1,0x5B78,0x8650,0x8B14,0x9DB4,0x5BD2,0x6068,/* 0xC8-0xCF */ + 0x608D,0x65F1,0x6C57,0x6F22,0x6FA3,0x701A,0x7F55,0x7FF0,/* 0xD0-0xD7 */ + 0x9591,0x9592,0x9650,0x97D3,0x5272,0x8F44,0x51FD,0x542B,/* 0xD8-0xDF */ + 0x54B8,0x5563,0x558A,0x6ABB,0x6DB5,0x7DD8,0x8266,0x929C,/* 0xE0-0xE7 */ + 0x9677,0x9E79,0x5408,0x54C8,0x76D2,0x86E4,0x95A4,0x95D4,/* 0xE8-0xEF */ + 0x965C,0x4EA2,0x4F09,0x59EE,0x5AE6,0x5DF7,0x6052,0x6297,/* 0xF0-0xF7 */ + 0x676D,0x6841,0x6C86,0x6E2F,0x7F38,0x809B,0x822A,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_FA[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0xFA08,0xFA09,0x9805,0x4EA5,0x5055,0x54B3,0x5793,/* 0xA0-0xA7 */ + 0x595A,0x5B69,0x5BB3,0x61C8,0x6977,0x6D77,0x7023,0x87F9,/* 0xA8-0xAF */ + 0x89E3,0x8A72,0x8AE7,0x9082,0x99ED,0x9AB8,0x52BE,0x6838,/* 0xB0-0xB7 */ + 0x5016,0x5E78,0x674F,0x8347,0x884C,0x4EAB,0x5411,0x56AE,/* 0xB8-0xBF */ + 0x73E6,0x9115,0x97FF,0x9909,0x9957,0x9999,0x5653,0x589F,/* 0xC0-0xC7 */ + 0x865B,0x8A31,0x61B2,0x6AF6,0x737B,0x8ED2,0x6B47,0x96AA,/* 0xC8-0xCF */ + 0x9A57,0x5955,0x7200,0x8D6B,0x9769,0x4FD4,0x5CF4,0x5F26,/* 0xD0-0xD7 */ + 0x61F8,0x665B,0x6CEB,0x70AB,0x7384,0x73B9,0x73FE,0x7729,/* 0xD8-0xDF */ + 0x774D,0x7D43,0x7D62,0x7E23,0x8237,0x8852,0xFA0A,0x8CE2,/* 0xE0-0xE7 */ + 0x9249,0x986F,0x5B51,0x7A74,0x8840,0x9801,0x5ACC,0x4FE0,/* 0xE8-0xEF */ + 0x5354,0x593E,0x5CFD,0x633E,0x6D79,0x72F9,0x8105,0x8107,/* 0xF0-0xF7 */ + 0x83A2,0x92CF,0x9830,0x4EA8,0x5144,0x5211,0x578B,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_FB[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x5F62,0x6CC2,0x6ECE,0x7005,0x7050,0x70AF,0x7192,/* 0xA0-0xA7 */ + 0x73E9,0x7469,0x834A,0x87A2,0x8861,0x9008,0x90A2,0x93A3,/* 0xA8-0xAF */ + 0x99A8,0x516E,0x5F57,0x60E0,0x6167,0x66B3,0x8559,0x8E4A,/* 0xB0-0xB7 */ + 0x91AF,0x978B,0x4E4E,0x4E92,0x547C,0x58D5,0x58FA,0x597D,/* 0xB8-0xBF */ + 0x5CB5,0x5F27,0x6236,0x6248,0x660A,0x6667,0x6BEB,0x6D69,/* 0xC0-0xC7 */ + 0x6DCF,0x6E56,0x6EF8,0x6F94,0x6FE0,0x6FE9,0x705D,0x72D0,/* 0xC8-0xCF */ + 0x7425,0x745A,0x74E0,0x7693,0x795C,0x7CCA,0x7E1E,0x80E1,/* 0xD0-0xD7 */ + 0x82A6,0x846B,0x84BF,0x864E,0x865F,0x8774,0x8B77,0x8C6A,/* 0xD8-0xDF */ + 0x93AC,0x9800,0x9865,0x60D1,0x6216,0x9177,0x5A5A,0x660F,/* 0xE0-0xE7 */ + 0x6DF7,0x6E3E,0x743F,0x9B42,0x5FFD,0x60DA,0x7B0F,0x54C4,/* 0xE8-0xEF */ + 0x5F18,0x6C5E,0x6CD3,0x6D2A,0x70D8,0x7D05,0x8679,0x8A0C,/* 0xF0-0xF7 */ + 0x9D3B,0x5316,0x548C,0x5B05,0x6A3A,0x706B,0x7575,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_FC[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x798D,0x79BE,0x82B1,0x83EF,0x8A71,0x8B41,0x8CA8,/* 0xA0-0xA7 */ + 0x9774,0xFA0B,0x64F4,0x652B,0x78BA,0x78BB,0x7A6B,0x4E38,/* 0xA8-0xAF */ + 0x559A,0x5950,0x5BA6,0x5E7B,0x60A3,0x63DB,0x6B61,0x6665,/* 0xB0-0xB7 */ + 0x6853,0x6E19,0x7165,0x74B0,0x7D08,0x9084,0x9A69,0x9C25,/* 0xB8-0xBF */ + 0x6D3B,0x6ED1,0x733E,0x8C41,0x95CA,0x51F0,0x5E4C,0x5FA8,/* 0xC0-0xC7 */ + 0x604D,0x60F6,0x6130,0x614C,0x6643,0x6644,0x69A5,0x6CC1,/* 0xC8-0xCF */ + 0x6E5F,0x6EC9,0x6F62,0x714C,0x749C,0x7687,0x7BC1,0x7C27,/* 0xD0-0xD7 */ + 0x8352,0x8757,0x9051,0x968D,0x9EC3,0x532F,0x56DE,0x5EFB,/* 0xD8-0xDF */ + 0x5F8A,0x6062,0x6094,0x61F7,0x6666,0x6703,0x6A9C,0x6DEE,/* 0xE0-0xE7 */ + 0x6FAE,0x7070,0x736A,0x7E6A,0x81BE,0x8334,0x86D4,0x8AA8,/* 0xE8-0xEF */ + 0x8CC4,0x5283,0x7372,0x5B96,0x6A6B,0x9404,0x54EE,0x5686,/* 0xF0-0xF7 */ + 0x5B5D,0x6548,0x6585,0x66C9,0x689F,0x6D8D,0x6DC6,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_FD[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x40-0x47 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x48-0x4F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x50-0x57 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x58-0x5F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x60-0x67 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x68-0x6F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x70-0x77 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x723B,0x80B4,0x9175,0x9A4D,0x4FAF,0x5019,0x539A,/* 0xA0-0xA7 */ + 0x540E,0x543C,0x5589,0x55C5,0x5E3F,0x5F8C,0x673D,0x7166,/* 0xA8-0xAF */ + 0x73DD,0x9005,0x52DB,0x52F3,0x5864,0x58CE,0x7104,0x718F,/* 0xB0-0xB7 */ + 0x71FB,0x85B0,0x8A13,0x6688,0x85A8,0x55A7,0x6684,0x714A,/* 0xB8-0xBF */ + 0x8431,0x5349,0x5599,0x6BC1,0x5F59,0x5FBD,0x63EE,0x6689,/* 0xC0-0xC7 */ + 0x7147,0x8AF1,0x8F1D,0x9EBE,0x4F11,0x643A,0x70CB,0x7566,/* 0xC8-0xCF */ + 0x8667,0x6064,0x8B4E,0x9DF8,0x5147,0x51F6,0x5308,0x6D36,/* 0xD0-0xD7 */ + 0x80F8,0x9ED1,0x6615,0x6B23,0x7098,0x75D5,0x5403,0x5C79,/* 0xD8-0xDF */ + 0x7D07,0x8A16,0x6B20,0x6B3D,0x6B46,0x5438,0x6070,0x6D3D,/* 0xE0-0xE7 */ + 0x7FD5,0x8208,0x50D6,0x51DE,0x559C,0x566B,0x56CD,0x59EC,/* 0xE8-0xEF */ + 0x5B09,0x5E0C,0x6199,0x6198,0x6231,0x665E,0x66E6,0x7199,/* 0xF0-0xF7 */ + 0x71B9,0x71BA,0x72A7,0x79A7,0x7A00,0x7FB2,0x8A70,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t *page_charset2uni[256] = { + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, c2u_81, c2u_82, c2u_83, c2u_84, c2u_85, c2u_86, c2u_87, + c2u_88, c2u_89, c2u_8A, c2u_8B, c2u_8C, c2u_8D, c2u_8E, c2u_8F, + c2u_90, c2u_91, c2u_92, c2u_93, c2u_94, c2u_95, c2u_96, c2u_97, + c2u_98, c2u_99, c2u_9A, c2u_9B, c2u_9C, c2u_9D, c2u_9E, c2u_9F, + c2u_A0, c2u_A1, c2u_A2, c2u_A3, c2u_A4, c2u_A5, c2u_A6, c2u_A7, + c2u_A8, c2u_A9, c2u_AA, c2u_AB, c2u_AC, c2u_AD, c2u_AE, c2u_AF, + c2u_B0, c2u_B1, c2u_B2, c2u_B3, c2u_B4, c2u_B5, c2u_B6, c2u_B7, + c2u_B8, c2u_B9, c2u_BA, c2u_BB, c2u_BC, c2u_BD, c2u_BE, c2u_BF, + c2u_C0, c2u_C1, c2u_C2, c2u_C3, c2u_C4, c2u_C5, c2u_C6, c2u_C7, + c2u_C8, NULL, c2u_CA, c2u_CB, c2u_CC, c2u_CD, c2u_CE, c2u_CF, + c2u_D0, c2u_D1, c2u_D2, c2u_D3, c2u_D4, c2u_D5, c2u_D6, c2u_D7, + c2u_D8, c2u_D9, c2u_DA, c2u_DB, c2u_DC, c2u_DD, c2u_DE, c2u_DF, + c2u_E0, c2u_E1, c2u_E2, c2u_E3, c2u_E4, c2u_E5, c2u_E6, c2u_E7, + c2u_E8, c2u_E9, c2u_EA, c2u_EB, c2u_EC, c2u_ED, c2u_EE, c2u_EF, + c2u_F0, c2u_F1, c2u_F2, c2u_F3, c2u_F4, c2u_F5, c2u_F6, c2u_F7, + c2u_F8, c2u_F9, c2u_FA, c2u_FB, c2u_FC, c2u_FD, NULL, NULL, +}; + +static const unsigned char u2c_01[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0xA9, 0xA2, 0xA9, 0xA2, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0xA8, 0xA4, 0xA9, 0xA4, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0xA9, 0xA5, 0xA8, 0xA6, 0xA9, 0xA6, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0xA9, 0xA7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA8, 0xA8, /* 0x3C-0x3F */ + 0xA9, 0xA8, 0xA8, 0xA9, 0xA9, 0xA9, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0xA9, 0xB0, 0xA8, 0xAF, 0xA9, 0xAF, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0xA8, 0xAB, 0xA9, 0xAB, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0xA8, 0xAE, 0xA9, 0xAE, /* 0x64-0x67 */ +}; + +static const unsigned char u2c_02[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA2, 0xA7, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0xA2, 0xB0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xA2, 0xA8, 0xA2, 0xAB, 0xA2, 0xAA, 0xA2, 0xAD, /* 0xD8-0xDB */ + 0x00, 0x00, 0xA2, 0xA9, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ +}; + +static const unsigned char u2c_03[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xA5, 0xC1, 0xA5, 0xC2, 0xA5, 0xC3, /* 0x90-0x93 */ + 0xA5, 0xC4, 0xA5, 0xC5, 0xA5, 0xC6, 0xA5, 0xC7, /* 0x94-0x97 */ + 0xA5, 0xC8, 0xA5, 0xC9, 0xA5, 0xCA, 0xA5, 0xCB, /* 0x98-0x9B */ + 0xA5, 0xCC, 0xA5, 0xCD, 0xA5, 0xCE, 0xA5, 0xCF, /* 0x9C-0x9F */ + 0xA5, 0xD0, 0xA5, 0xD1, 0x00, 0x00, 0xA5, 0xD2, /* 0xA0-0xA3 */ + 0xA5, 0xD3, 0xA5, 0xD4, 0xA5, 0xD5, 0xA5, 0xD6, /* 0xA4-0xA7 */ + 0xA5, 0xD7, 0xA5, 0xD8, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0xA5, 0xE1, 0xA5, 0xE2, 0xA5, 0xE3, /* 0xB0-0xB3 */ + 0xA5, 0xE4, 0xA5, 0xE5, 0xA5, 0xE6, 0xA5, 0xE7, /* 0xB4-0xB7 */ + 0xA5, 0xE8, 0xA5, 0xE9, 0xA5, 0xEA, 0xA5, 0xEB, /* 0xB8-0xBB */ + 0xA5, 0xEC, 0xA5, 0xED, 0xA5, 0xEE, 0xA5, 0xEF, /* 0xBC-0xBF */ + 0xA5, 0xF0, 0xA5, 0xF1, 0x00, 0x00, 0xA5, 0xF2, /* 0xC0-0xC3 */ + 0xA5, 0xF3, 0xA5, 0xF4, 0xA5, 0xF5, 0xA5, 0xF6, /* 0xC4-0xC7 */ + 0xA5, 0xF7, 0xA5, 0xF8, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ +}; + +static const unsigned char u2c_04[512] = { + 0x00, 0x00, 0xAC, 0xA7, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0xAC, 0xA1, 0xAC, 0xA2, 0xAC, 0xA3, 0xAC, 0xA4, /* 0x10-0x13 */ + 0xAC, 0xA5, 0xAC, 0xA6, 0xAC, 0xA8, 0xAC, 0xA9, /* 0x14-0x17 */ + 0xAC, 0xAA, 0xAC, 0xAB, 0xAC, 0xAC, 0xAC, 0xAD, /* 0x18-0x1B */ + 0xAC, 0xAE, 0xAC, 0xAF, 0xAC, 0xB0, 0xAC, 0xB1, /* 0x1C-0x1F */ + 0xAC, 0xB2, 0xAC, 0xB3, 0xAC, 0xB4, 0xAC, 0xB5, /* 0x20-0x23 */ + 0xAC, 0xB6, 0xAC, 0xB7, 0xAC, 0xB8, 0xAC, 0xB9, /* 0x24-0x27 */ + 0xAC, 0xBA, 0xAC, 0xBB, 0xAC, 0xBC, 0xAC, 0xBD, /* 0x28-0x2B */ + 0xAC, 0xBE, 0xAC, 0xBF, 0xAC, 0xC0, 0xAC, 0xC1, /* 0x2C-0x2F */ + 0xAC, 0xD1, 0xAC, 0xD2, 0xAC, 0xD3, 0xAC, 0xD4, /* 0x30-0x33 */ + 0xAC, 0xD5, 0xAC, 0xD6, 0xAC, 0xD8, 0xAC, 0xD9, /* 0x34-0x37 */ + 0xAC, 0xDA, 0xAC, 0xDB, 0xAC, 0xDC, 0xAC, 0xDD, /* 0x38-0x3B */ + 0xAC, 0xDE, 0xAC, 0xDF, 0xAC, 0xE0, 0xAC, 0xE1, /* 0x3C-0x3F */ + 0xAC, 0xE2, 0xAC, 0xE3, 0xAC, 0xE4, 0xAC, 0xE5, /* 0x40-0x43 */ + 0xAC, 0xE6, 0xAC, 0xE7, 0xAC, 0xE8, 0xAC, 0xE9, /* 0x44-0x47 */ + 0xAC, 0xEA, 0xAC, 0xEB, 0xAC, 0xEC, 0xAC, 0xED, /* 0x48-0x4B */ + 0xAC, 0xEE, 0xAC, 0xEF, 0xAC, 0xF0, 0xAC, 0xF1, /* 0x4C-0x4F */ + 0x00, 0x00, 0xAC, 0xD7, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ +}; + +static const unsigned char u2c_11[512] = { + 0xA4, 0xA1, 0xA4, 0xA2, 0xA4, 0xA4, 0xA4, 0xA7, /* 0x00-0x03 */ + 0xA4, 0xA8, 0xA4, 0xA9, 0xA4, 0xB1, 0xA4, 0xB2, /* 0x04-0x07 */ + 0xA4, 0xB3, 0xA4, 0xB5, 0xA4, 0xB6, 0xA4, 0xB7, /* 0x08-0x0B */ + 0xA4, 0xB8, 0xA4, 0xB9, 0xA4, 0xBA, 0xA4, 0xBB, /* 0x0C-0x0F */ + 0xA4, 0xBC, 0xA4, 0xBD, 0xA4, 0xBE, 0x00, 0x00, /* 0x10-0x13 */ + 0xA4, 0xD5, 0xA4, 0xD6, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0xA4, 0xDD, 0x00, 0x00, /* 0x18-0x1B */ + 0xA4, 0xDE, 0xA4, 0xE1, 0xA4, 0xE2, 0x00, 0x00, /* 0x1C-0x1F */ + 0xA4, 0xE3, 0xA4, 0xB4, 0xA4, 0xE4, 0xA4, 0xE5, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA4, 0xE6, /* 0x24-0x27 */ + 0x00, 0x00, 0xA4, 0xE7, 0x00, 0x00, 0xA4, 0xE8, /* 0x28-0x2B */ + 0xA4, 0xE9, 0xA4, 0xEA, 0xA4, 0xEB, 0xA4, 0xEC, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0xA4, 0xED, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0xA4, 0xEE, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0xA4, 0xB5, 0xA4, 0xB6, 0xA4, 0xB5, 0xA4, 0xB6, /* 0x3C-0x3F */ + 0xA4, 0xEF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0xA4, 0xF2, 0xA4, 0xF3, 0xA4, 0xF0, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0xA4, 0xB7, 0x00, 0x00, 0xA4, 0xB8, 0xA4, 0xB9, /* 0x4C-0x4F */ + 0xA4, 0xB8, 0xA4, 0xB9, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0xA4, 0xBA, 0xA4, 0xBA, 0x00, 0x00, 0xA4, 0xF4, /* 0x54-0x57 */ + 0xA4, 0xF5, 0xA4, 0xF6, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA4, 0xD4, /* 0x5C-0x5F */ + 0x00, 0x00, 0xA4, 0xBF, 0xA4, 0xC0, 0xA4, 0xC1, /* 0x60-0x63 */ + 0xA4, 0xC2, 0xA4, 0xC3, 0xA4, 0xC4, 0xA4, 0xC5, /* 0x64-0x67 */ + 0xA4, 0xC6, 0xA4, 0xC7, 0xA4, 0xC8, 0xA4, 0xC9, /* 0x68-0x6B */ + 0xA4, 0xCA, 0xA4, 0xCB, 0xA4, 0xCC, 0xA4, 0xCD, /* 0x6C-0x6F */ + 0xA4, 0xCE, 0xA4, 0xCF, 0xA4, 0xD0, 0xA4, 0xD1, /* 0x70-0x73 */ + 0xA4, 0xD2, 0xA4, 0xD3, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0xA4, 0xF7, 0xA4, 0xF8, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0xA4, 0xF9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xA4, 0xFA, 0xA4, 0xFB, 0x00, 0x00, /* 0x90-0x93 */ + 0xA4, 0xFC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0xA4, 0xFD, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0xA4, 0xFE, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xA4, 0xA1, 0xA4, 0xA2, 0xA4, 0xA3, 0xA4, 0xA4, /* 0xA8-0xAB */ + 0xA4, 0xA5, 0xA4, 0xA6, 0xA4, 0xA7, 0xA4, 0xA9, /* 0xAC-0xAF */ + 0xA4, 0xAA, 0xA4, 0xAB, 0xA4, 0xAC, 0xA4, 0xAD, /* 0xB0-0xB3 */ + 0xA4, 0xAE, 0xA4, 0xAF, 0xA4, 0xB0, 0xA4, 0xB1, /* 0xB4-0xB7 */ + 0xA4, 0xB2, 0xA4, 0xB4, 0xA4, 0xB5, 0xA4, 0xB6, /* 0xB8-0xBB */ + 0xA4, 0xB7, 0xA4, 0xB8, 0xA4, 0xBA, 0xA4, 0xBB, /* 0xBC-0xBF */ + 0xA4, 0xBC, 0xA4, 0xBD, 0xA4, 0xBE, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0xA4, 0xD6, 0xA4, 0xD7, /* 0xC4-0xC7 */ + 0xA4, 0xD8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0xA4, 0xD9, 0x00, 0x00, 0xA4, 0xDA, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA4, 0xDB, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA4, 0xDC, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xA4, 0xDD, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0xA4, 0xDE, 0xA4, 0xDF, 0x00, 0x00, 0xA4, 0xE0, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0xA4, 0xE8, 0xA4, 0xEA, /* 0xE4-0xE7 */ + 0xA4, 0xEC, 0x00, 0x00, 0xA4, 0xED, 0xA4, 0xEF, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0xA4, 0xF0, 0x00, 0x00, /* 0xEC-0xEF */ + 0xA4, 0xB7, 0xA4, 0xF2, 0xA4, 0xF3, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xA4, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xA4, 0xF6, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_20[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0xA1, 0xAA, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0xA1, 0xAE, 0xA1, 0xAF, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0xA1, 0xB0, 0xA1, 0xB1, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0xA2, 0xD3, 0xA2, 0xD4, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0xA1, 0xA5, 0xA1, 0xA6, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xA2, 0xB6, 0x00, 0x00, 0xA1, 0xC7, 0xA1, 0xC8, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA1, 0xD8, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0xA9, 0xF9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA9, 0xFA, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xA9, 0xFB, 0xA9, 0xFC, 0xA9, 0xFD, /* 0x80-0x83 */ + 0xA9, 0xFE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x5C, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ +}; + +static const unsigned char u2c_21[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA1, 0xC9, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xA2, 0xB5, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA7, 0xA4, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xA2, 0xE0, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0xA2, 0xE5, 0xA2, 0xE2, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0xA7, 0xD9, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA1, 0xCA, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA8, 0xF7, /* 0x50-0x53 */ + 0xA8, 0xF8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA8, 0xFB, /* 0x58-0x5B */ + 0xA8, 0xFC, 0xA8, 0xFD, 0xA8, 0xFE, 0x00, 0x00, /* 0x5C-0x5F */ + 0xA5, 0xB0, 0xA5, 0xB1, 0xA5, 0xB2, 0xA5, 0xB3, /* 0x60-0x63 */ + 0xA5, 0xB4, 0xA5, 0xB5, 0xA5, 0xB6, 0xA5, 0xB7, /* 0x64-0x67 */ + 0xA5, 0xB8, 0xA5, 0xB9, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0xA5, 0xA1, 0xA5, 0xA2, 0xA5, 0xA3, 0xA5, 0xA4, /* 0x70-0x73 */ + 0xA5, 0xA5, 0xA5, 0xA6, 0xA5, 0xA7, 0xA5, 0xA8, /* 0x74-0x77 */ + 0xA5, 0xA9, 0xA5, 0xAA, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0xA1, 0xE7, 0xA1, 0xE8, 0xA1, 0xE6, 0xA1, 0xE9, /* 0x90-0x93 */ + 0xA1, 0xEA, 0xA2, 0xD5, 0xA2, 0xD8, 0xA2, 0xD6, /* 0x94-0x97 */ + 0xA2, 0xD9, 0xA2, 0xD7, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xA2, 0xA1, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xA2, 0xA2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ +}; + +static const unsigned char u2c_22[512] = { + 0xA2, 0xA3, 0x00, 0x00, 0xA1, 0xD3, 0xA2, 0xA4, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA1, 0xD4, /* 0x04-0x07 */ + 0xA1, 0xF4, 0x00, 0x00, 0x00, 0x00, 0xA1, 0xF5, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA2, 0xB3, /* 0x0C-0x0F */ + 0x00, 0x00, 0xA2, 0xB2, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xEE, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0xA1, 0xF0, 0xA1, 0xC4, 0x00, 0x00, /* 0x1C-0x1F */ + 0xA1, 0xD0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0xA1, 0xAB, 0x00, 0x00, 0xA1, 0xFC, /* 0x24-0x27 */ + 0xA1, 0xFD, 0xA1, 0xFB, 0xA1, 0xFA, 0xA1, 0xF2, /* 0x28-0x2B */ + 0xA1, 0xF3, 0x00, 0x00, 0xA2, 0xB1, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0xA1, 0xC5, 0xA1, 0xF1, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0xA1, 0xAD, 0xA1, 0xEF, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xD6, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0xA1, 0xC1, 0xA1, 0xD5, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0xA1, 0xC2, 0xA1, 0xC3, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xEC, 0xA1, 0xED, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xF8, 0xA1, 0xF9, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xF6, 0xA1, 0xF7, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0xA2, 0xC1, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xA1, 0xD1, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ +}; + +static const unsigned char u2c_23[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xD2, 0x00, 0x00, /* 0x10-0x13 */ +}; + +static const unsigned char u2c_24[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0xA8, 0xE7, 0xA8, 0xE8, 0xA8, 0xE9, 0xA8, 0xEA, /* 0x60-0x63 */ + 0xA8, 0xEB, 0xA8, 0xEC, 0xA8, 0xED, 0xA8, 0xEE, /* 0x64-0x67 */ + 0xA8, 0xEF, 0xA8, 0xF0, 0xA8, 0xF1, 0xA8, 0xF2, /* 0x68-0x6B */ + 0xA8, 0xF3, 0xA8, 0xF4, 0xA8, 0xF5, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0xA9, 0xE7, 0xA9, 0xE8, 0xA9, 0xE9, 0xA9, 0xEA, /* 0x74-0x77 */ + 0xA9, 0xEB, 0xA9, 0xEC, 0xA9, 0xED, 0xA9, 0xEE, /* 0x78-0x7B */ + 0xA9, 0xEF, 0xA9, 0xF0, 0xA9, 0xF1, 0xA9, 0xF2, /* 0x7C-0x7F */ + + 0xA9, 0xF3, 0xA9, 0xF4, 0xA9, 0xF5, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0xA9, 0xCD, 0xA9, 0xCE, 0xA9, 0xCF, 0xA9, 0xD0, /* 0x9C-0x9F */ + 0xA9, 0xD1, 0xA9, 0xD2, 0xA9, 0xD3, 0xA9, 0xD4, /* 0xA0-0xA3 */ + 0xA9, 0xD5, 0xA9, 0xD6, 0xA9, 0xD7, 0xA9, 0xD8, /* 0xA4-0xA7 */ + 0xA9, 0xD9, 0xA9, 0xDA, 0xA9, 0xDB, 0xA9, 0xDC, /* 0xA8-0xAB */ + 0xA9, 0xDD, 0xA9, 0xDE, 0xA9, 0xDF, 0xA9, 0xE0, /* 0xAC-0xAF */ + 0xA9, 0xE1, 0xA9, 0xE2, 0xA9, 0xE3, 0xA9, 0xE4, /* 0xB0-0xB3 */ + 0xA9, 0xE5, 0xA9, 0xE6, 0xA8, 0xCD, 0xA8, 0xCE, /* 0xB4-0xB7 */ + 0xA8, 0xCF, 0xA8, 0xD0, 0xA8, 0xD1, 0xA8, 0xD2, /* 0xB8-0xBB */ + 0xA8, 0xD3, 0xA8, 0xD4, 0xA8, 0xD5, 0xA8, 0xD6, /* 0xBC-0xBF */ + 0xA8, 0xD7, 0xA8, 0xD8, 0xA8, 0xD9, 0xA8, 0xDA, /* 0xC0-0xC3 */ + 0xA8, 0xDB, 0xA8, 0xDC, 0xA8, 0xDD, 0xA8, 0xDE, /* 0xC4-0xC7 */ + 0xA8, 0xDF, 0xA8, 0xE0, 0xA8, 0xE1, 0xA8, 0xE2, /* 0xC8-0xCB */ + 0xA8, 0xE3, 0xA8, 0xE4, 0xA8, 0xE5, 0xA8, 0xE6, /* 0xCC-0xCF */ + 0xA8, 0xCD, 0xA8, 0xCE, 0xA8, 0xCF, 0xA8, 0xD0, /* 0xD0-0xD3 */ + 0xA8, 0xD1, 0xA8, 0xD2, 0xA8, 0xD3, 0xA8, 0xD4, /* 0xD4-0xD7 */ + 0xA8, 0xD5, 0xA8, 0xD6, 0xA8, 0xD7, 0xA8, 0xD8, /* 0xD8-0xDB */ + 0xA8, 0xD9, 0xA8, 0xDA, 0xA8, 0xDB, 0xA8, 0xDC, /* 0xDC-0xDF */ + 0xA8, 0xDD, 0xA8, 0xDE, 0xA8, 0xDF, 0xA8, 0xE0, /* 0xE0-0xE3 */ + 0xA8, 0xE1, 0xA8, 0xE2, 0xA8, 0xE3, 0xA8, 0xE4, /* 0xE4-0xE7 */ + 0xA8, 0xE5, 0xA8, 0xE6, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ +}; + +static const unsigned char u2c_25[512] = { + 0xA6, 0xA1, 0xA6, 0xAC, 0xA6, 0xA2, 0xA6, 0xAD, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0xA6, 0xA3, 0xA6, 0xC8, 0xA6, 0xC7, 0xA6, 0xAE, /* 0x0C-0x0F */ + 0xA6, 0xA4, 0xA6, 0xC2, 0xA6, 0xC1, 0xA6, 0xAF, /* 0x10-0x13 */ + 0xA6, 0xA6, 0xA6, 0xC6, 0xA6, 0xC5, 0xA6, 0xB1, /* 0x14-0x17 */ + 0xA6, 0xA5, 0xA6, 0xC4, 0xA6, 0xC3, 0xA6, 0xB0, /* 0x18-0x1B */ + 0xA6, 0xA7, 0xA6, 0xBC, 0xA6, 0xC9, 0xA6, 0xCA, /* 0x1C-0x1F */ + 0xA6, 0xB7, 0xA6, 0xCB, 0xA6, 0xCC, 0xA6, 0xB2, /* 0x20-0x23 */ + 0xA6, 0xA9, 0xA6, 0xBE, 0xA6, 0xCD, 0xA6, 0xCE, /* 0x24-0x27 */ + 0xA6, 0xB9, 0xA6, 0xCF, 0xA6, 0xD0, 0xA6, 0xB4, /* 0x28-0x2B */ + 0xA6, 0xA8, 0xA6, 0xD1, 0xA6, 0xD2, 0xA6, 0xB8, /* 0x2C-0x2F */ + 0xA6, 0xBD, 0xA6, 0xD3, 0xA6, 0xD4, 0xA6, 0xB3, /* 0x30-0x33 */ + 0xA6, 0xAA, 0xA6, 0xD5, 0xA6, 0xD6, 0xA6, 0xBA, /* 0x34-0x37 */ + 0xA6, 0xBF, 0xA6, 0xD7, 0xA6, 0xD8, 0xA6, 0xB5, /* 0x38-0x3B */ + 0xA6, 0xAB, 0xA6, 0xD9, 0xA6, 0xDA, 0xA6, 0xBB, /* 0x3C-0x3F */ + 0xA6, 0xDB, 0xA6, 0xDC, 0xA6, 0xC0, 0xA6, 0xDD, /* 0x40-0x43 */ + 0xA6, 0xDE, 0xA6, 0xDF, 0xA6, 0xE0, 0xA6, 0xE1, /* 0x44-0x47 */ + 0xA6, 0xE2, 0xA6, 0xE3, 0xA6, 0xE4, 0xA6, 0xB6, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0xA2, 0xC6, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0xA1, 0xE1, 0xA1, 0xE0, 0x00, 0x00, 0xA2, 0xC3, /* 0xA0-0xA3 */ + 0xA2, 0xC7, 0xA2, 0xC8, 0xA2, 0xCB, 0xA2, 0xCA, /* 0xA4-0xA7 */ + 0xA2, 0xC9, 0xA2, 0xCC, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xE3, 0xA1, 0xE2, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0xA2, 0xBA, 0xA2, 0xB9, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0xA1, 0xE5, 0xA1, 0xE4, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0xA2, 0xB8, 0xA2, 0xB7, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xDF, 0xA1, 0xDE, /* 0xC4-0xC7 */ + 0xA2, 0xC2, 0x00, 0x00, 0x00, 0x00, 0xA1, 0xDB, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xDD, 0xA1, 0xDC, /* 0xCC-0xCF */ + 0xA2, 0xC4, 0xA2, 0xC5, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ +}; + +static const unsigned char u2c_26[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0xA1, 0xDA, 0xA1, 0xD9, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0xA2, 0xCF, 0xA2, 0xCE, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0xA2, 0xD0, 0x00, 0x00, 0xA2, 0xD1, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0xA1, 0xCF, 0x00, 0x00, 0xA1, 0xCE, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0xA2, 0xBC, 0xA2, 0xBD, 0x00, 0x00, 0xA2, 0xC0, /* 0x60-0x63 */ + 0xA2, 0xBB, 0xA2, 0xBE, 0x00, 0x00, 0xA2, 0xBF, /* 0x64-0x67 */ + 0xA2, 0xCD, 0xA2, 0xDB, 0xA2, 0xDC, 0x00, 0x00, /* 0x68-0x6B */ + 0xA2, 0xDD, 0xA2, 0xDA, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ +}; + +static const unsigned char u2c_30[512] = { + 0xA1, 0xA1, 0xA1, 0xA2, 0xA1, 0xA3, 0xA1, 0xA8, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0xA1, 0xB4, 0xA1, 0xB5, 0xA1, 0xB6, 0xA1, 0xB7, /* 0x08-0x0B */ + 0xA1, 0xB8, 0xA1, 0xB9, 0xA1, 0xBA, 0xA1, 0xBB, /* 0x0C-0x0F */ + 0xA1, 0xBC, 0xA1, 0xBD, 0x00, 0x00, 0xA1, 0xEB, /* 0x10-0x13 */ + 0xA1, 0xB2, 0xA1, 0xB3, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0xAA, 0xA1, 0xAA, 0xA2, 0xAA, 0xA3, /* 0x40-0x43 */ + 0xAA, 0xA4, 0xAA, 0xA5, 0xAA, 0xA6, 0xAA, 0xA7, /* 0x44-0x47 */ + 0xAA, 0xA8, 0xAA, 0xA9, 0xAA, 0xAA, 0xAA, 0xAB, /* 0x48-0x4B */ + 0xAA, 0xAC, 0xAA, 0xAD, 0xAA, 0xAE, 0xAA, 0xAF, /* 0x4C-0x4F */ + 0xAA, 0xB0, 0xAA, 0xB1, 0xAA, 0xB2, 0xAA, 0xB3, /* 0x50-0x53 */ + 0xAA, 0xB4, 0xAA, 0xB5, 0xAA, 0xB6, 0xAA, 0xB7, /* 0x54-0x57 */ + 0xAA, 0xB8, 0xAA, 0xB9, 0xAA, 0xBA, 0xAA, 0xBB, /* 0x58-0x5B */ + 0xAA, 0xBC, 0xAA, 0xBD, 0xAA, 0xBE, 0xAA, 0xBF, /* 0x5C-0x5F */ + 0xAA, 0xC0, 0xAA, 0xC1, 0xAA, 0xC2, 0xAA, 0xC3, /* 0x60-0x63 */ + 0xAA, 0xC4, 0xAA, 0xC5, 0xAA, 0xC6, 0xAA, 0xC7, /* 0x64-0x67 */ + 0xAA, 0xC8, 0xAA, 0xC9, 0xAA, 0xCA, 0xAA, 0xCB, /* 0x68-0x6B */ + 0xAA, 0xCC, 0xAA, 0xCD, 0xAA, 0xCE, 0xAA, 0xCF, /* 0x6C-0x6F */ + 0xAA, 0xD0, 0xAA, 0xD1, 0xAA, 0xD2, 0xAA, 0xD3, /* 0x70-0x73 */ + 0xAA, 0xD4, 0xAA, 0xD5, 0xAA, 0xD6, 0xAA, 0xD7, /* 0x74-0x77 */ + 0xAA, 0xD8, 0xAA, 0xD9, 0xAA, 0xDA, 0xAA, 0xDB, /* 0x78-0x7B */ + 0xAA, 0xDC, 0xAA, 0xDD, 0xAA, 0xDE, 0xAA, 0xDF, /* 0x7C-0x7F */ + + 0xAA, 0xE0, 0xAA, 0xE1, 0xAA, 0xE2, 0xAA, 0xE3, /* 0x80-0x83 */ + 0xAA, 0xE4, 0xAA, 0xE5, 0xAA, 0xE6, 0xAA, 0xE7, /* 0x84-0x87 */ + 0xAA, 0xE8, 0xAA, 0xE9, 0xAA, 0xEA, 0xAA, 0xEB, /* 0x88-0x8B */ + 0xAA, 0xEC, 0xAA, 0xED, 0xAA, 0xEE, 0xAA, 0xEF, /* 0x8C-0x8F */ + 0xAA, 0xF0, 0xAA, 0xF1, 0xAA, 0xF2, 0xAA, 0xF3, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0xAB, 0xA1, 0xAB, 0xA2, 0xAB, 0xA3, /* 0xA0-0xA3 */ + 0xAB, 0xA4, 0xAB, 0xA5, 0xAB, 0xA6, 0xAB, 0xA7, /* 0xA4-0xA7 */ + 0xAB, 0xA8, 0xAB, 0xA9, 0xAB, 0xAA, 0xAB, 0xAB, /* 0xA8-0xAB */ + 0xAB, 0xAC, 0xAB, 0xAD, 0xAB, 0xAE, 0xAB, 0xAF, /* 0xAC-0xAF */ + 0xAB, 0xB0, 0xAB, 0xB1, 0xAB, 0xB2, 0xAB, 0xB3, /* 0xB0-0xB3 */ + 0xAB, 0xB4, 0xAB, 0xB5, 0xAB, 0xB6, 0xAB, 0xB7, /* 0xB4-0xB7 */ + 0xAB, 0xB8, 0xAB, 0xB9, 0xAB, 0xBA, 0xAB, 0xBB, /* 0xB8-0xBB */ + 0xAB, 0xBC, 0xAB, 0xBD, 0xAB, 0xBE, 0xAB, 0xBF, /* 0xBC-0xBF */ + 0xAB, 0xC0, 0xAB, 0xC1, 0xAB, 0xC2, 0xAB, 0xC3, /* 0xC0-0xC3 */ + 0xAB, 0xC4, 0xAB, 0xC5, 0xAB, 0xC6, 0xAB, 0xC7, /* 0xC4-0xC7 */ + 0xAB, 0xC8, 0xAB, 0xC9, 0xAB, 0xCA, 0xAB, 0xCB, /* 0xC8-0xCB */ + 0xAB, 0xCC, 0xAB, 0xCD, 0xAB, 0xCE, 0xAB, 0xCF, /* 0xCC-0xCF */ + 0xAB, 0xD0, 0xAB, 0xD1, 0xAB, 0xD2, 0xAB, 0xD3, /* 0xD0-0xD3 */ + 0xAB, 0xD4, 0xAB, 0xD5, 0xAB, 0xD6, 0xAB, 0xD7, /* 0xD4-0xD7 */ + 0xAB, 0xD8, 0xAB, 0xD9, 0xAB, 0xDA, 0xAB, 0xDB, /* 0xD8-0xDB */ + 0xAB, 0xDC, 0xAB, 0xDD, 0xAB, 0xDE, 0xAB, 0xDF, /* 0xDC-0xDF */ + 0xAB, 0xE0, 0xAB, 0xE1, 0xAB, 0xE2, 0xAB, 0xE3, /* 0xE0-0xE3 */ + 0xAB, 0xE4, 0xAB, 0xE5, 0xAB, 0xE6, 0xAB, 0xE7, /* 0xE4-0xE7 */ + 0xAB, 0xE8, 0xAB, 0xE9, 0xAB, 0xEA, 0xAB, 0xEB, /* 0xE8-0xEB */ + 0xAB, 0xEC, 0xAB, 0xED, 0xAB, 0xEE, 0xAB, 0xEF, /* 0xEC-0xEF */ + 0xAB, 0xF0, 0xAB, 0xF1, 0xAB, 0xF2, 0xAB, 0xF3, /* 0xF0-0xF3 */ + 0xAB, 0xF4, 0xAB, 0xF5, 0xAB, 0xF6, 0x00, 0x00, /* 0xF4-0xF7 */ +}; + +static const unsigned char u2c_31[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0xA4, 0xA1, 0xA4, 0xA2, 0xA4, 0xA3, /* 0x30-0x33 */ + 0xA4, 0xA4, 0xA4, 0xA5, 0xA4, 0xA6, 0xA4, 0xA7, /* 0x34-0x37 */ + 0xA4, 0xA8, 0xA4, 0xA9, 0xA4, 0xAA, 0xA4, 0xAB, /* 0x38-0x3B */ + 0xA4, 0xAC, 0xA4, 0xAD, 0xA4, 0xAE, 0xA4, 0xAF, /* 0x3C-0x3F */ + 0xA4, 0xB0, 0xA4, 0xB1, 0xA4, 0xB2, 0xA4, 0xB3, /* 0x40-0x43 */ + 0xA4, 0xB4, 0xA4, 0xB5, 0xA4, 0xB6, 0xA4, 0xB7, /* 0x44-0x47 */ + 0xA4, 0xB8, 0xA4, 0xB9, 0xA4, 0xBA, 0xA4, 0xBB, /* 0x48-0x4B */ + 0xA4, 0xBC, 0xA4, 0xBD, 0xA4, 0xBE, 0xA4, 0xBF, /* 0x4C-0x4F */ + 0xA4, 0xC0, 0xA4, 0xC1, 0xA4, 0xC2, 0xA4, 0xC3, /* 0x50-0x53 */ + 0xA4, 0xC4, 0xA4, 0xC5, 0xA4, 0xC6, 0xA4, 0xC7, /* 0x54-0x57 */ + 0xA4, 0xC8, 0xA4, 0xC9, 0xA4, 0xCA, 0xA4, 0xCB, /* 0x58-0x5B */ + 0xA4, 0xCC, 0xA4, 0xCD, 0xA4, 0xCE, 0xA4, 0xCF, /* 0x5C-0x5F */ + 0xA4, 0xD0, 0xA4, 0xD1, 0xA4, 0xD2, 0xA4, 0xD3, /* 0x60-0x63 */ + 0xA4, 0xD4, 0xA4, 0xD5, 0xA4, 0xD6, 0xA4, 0xD7, /* 0x64-0x67 */ + 0xA4, 0xD8, 0xA4, 0xD9, 0xA4, 0xDA, 0xA4, 0xDB, /* 0x68-0x6B */ + 0xA4, 0xDC, 0xA4, 0xDD, 0xA4, 0xDE, 0xA4, 0xDF, /* 0x6C-0x6F */ + 0xA4, 0xE0, 0xA4, 0xE1, 0xA4, 0xE2, 0xA4, 0xE3, /* 0x70-0x73 */ + 0xA4, 0xE4, 0xA4, 0xE5, 0xA4, 0xE6, 0xA4, 0xE7, /* 0x74-0x77 */ + 0xA4, 0xE8, 0xA4, 0xE9, 0xA4, 0xEA, 0xA4, 0xEB, /* 0x78-0x7B */ + 0xA4, 0xEC, 0xA4, 0xED, 0xA4, 0xEE, 0xA4, 0xEF, /* 0x7C-0x7F */ + + 0xA4, 0xF0, 0xA4, 0xF1, 0xA4, 0xF2, 0xA4, 0xF3, /* 0x80-0x83 */ + 0xA4, 0xF4, 0xA4, 0xF5, 0xA4, 0xF6, 0xA4, 0xF7, /* 0x84-0x87 */ + 0xA4, 0xF8, 0xA4, 0xF9, 0xA4, 0xFA, 0xA4, 0xFB, /* 0x88-0x8B */ + 0xA4, 0xFC, 0xA4, 0xFD, 0xA4, 0xFE, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0xEC, 0xE9, 0xEC, 0xA3, /* 0x90-0x93 */ + 0xDF, 0xB2, 0xDE, 0xCC, 0xDF, 0xBE, 0xF1, 0xE9, /* 0x94-0x97 */ + 0xF9, 0xBB, 0xCB, 0xA3, 0xEB, 0xE0, 0xDC, 0xB0, /* 0x98-0x9B */ + 0xEF, 0xCB, 0xF4, 0xB8, 0xF2, 0xA2, 0xEC, 0xD1, /* 0x9C-0x9F */ +}; + +static const unsigned char u2c_32[512] = { + 0xA9, 0xB1, 0xA9, 0xB2, 0xA9, 0xB3, 0xA9, 0xB4, /* 0x00-0x03 */ + 0xA9, 0xB5, 0xA9, 0xB6, 0xA9, 0xB7, 0xA9, 0xB8, /* 0x04-0x07 */ + 0xA9, 0xB9, 0xA9, 0xBA, 0xA9, 0xBB, 0xA9, 0xBC, /* 0x08-0x0B */ + 0xA9, 0xBD, 0xA9, 0xBE, 0xA9, 0xBF, 0xA9, 0xC0, /* 0x0C-0x0F */ + 0xA9, 0xC1, 0xA9, 0xC2, 0xA9, 0xC3, 0xA9, 0xC4, /* 0x10-0x13 */ + 0xA9, 0xC5, 0xA9, 0xC6, 0xA9, 0xC7, 0xA9, 0xC8, /* 0x14-0x17 */ + 0xA9, 0xC9, 0xA9, 0xCA, 0xA9, 0xCB, 0xA9, 0xCC, /* 0x18-0x1B */ + 0xA2, 0xDF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0xEC, 0xE9, 0xEC, 0xA3, 0xDF, 0xB2, 0xDE, 0xCC, /* 0x20-0x23 */ + 0xE7, 0xE9, 0xD7, 0xBF, 0xF6, 0xD2, 0xF8, 0xA2, /* 0x24-0x27 */ + 0xCE, 0xFA, 0xE4, 0xA8, 0xEA, 0xC5, 0xFB, 0xFD, /* 0x28-0x2B */ + 0xE2, 0xA9, 0xD9, 0xCA, 0xD1, 0xD1, 0xF7, 0xCF, /* 0x2C-0x2F */ + 0xEC, 0xED, 0xF1, 0xBB, 0xEA, 0xF3, 0xDE, 0xE4, /* 0x30-0x33 */ + 0xD9, 0xA3, 0xF7, 0xE5, 0xEE, 0xAF, 0xF5, 0xE6, /* 0x34-0x37 */ + 0xD6, 0xCC, 0xD3, 0xDB, 0xFB, 0xBC, 0xF9, 0xCA, /* 0x38-0x3B */ + 0xCA, 0xF8, 0xD0, 0xEA, 0xED, 0xC0, 0xFA, 0xF0, /* 0x3C-0x3F */ + 0xF0, 0xAE, 0xFD, 0xCC, 0xED, 0xBB, 0xF2, 0xB8, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0xA8, 0xB1, 0xA8, 0xB2, 0xA8, 0xB3, 0xA8, 0xB4, /* 0x60-0x63 */ + 0xA8, 0xB5, 0xA8, 0xB6, 0xA8, 0xB7, 0xA8, 0xB8, /* 0x64-0x67 */ + 0xA8, 0xB9, 0xA8, 0xBA, 0xA8, 0xBB, 0xA8, 0xBC, /* 0x68-0x6B */ + 0xA8, 0xBD, 0xA8, 0xBE, 0xA8, 0xBF, 0xA8, 0xC0, /* 0x6C-0x6F */ + 0xA8, 0xC1, 0xA8, 0xC2, 0xA8, 0xC3, 0xA8, 0xC4, /* 0x70-0x73 */ + 0xA8, 0xC5, 0xA8, 0xC6, 0xA8, 0xC7, 0xA8, 0xC8, /* 0x74-0x77 */ + 0xA8, 0xC9, 0xA8, 0xCA, 0xA8, 0xCB, 0xA8, 0xCC, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA2, 0xDE, /* 0x7C-0x7F */ + + 0xEC, 0xE9, 0xEC, 0xA3, 0xDF, 0xB2, 0xDE, 0xCC, /* 0x80-0x83 */ + 0xE7, 0xE9, 0xD7, 0xBF, 0xF6, 0xD2, 0xF8, 0xA2, /* 0x84-0x87 */ + 0xCE, 0xFA, 0xE4, 0xA8, 0xEA, 0xC5, 0xFB, 0xFD, /* 0x88-0x8B */ + 0xE2, 0xA9, 0xD9, 0xCA, 0xD1, 0xD1, 0xF7, 0xCF, /* 0x8C-0x8F */ + 0xEC, 0xED, 0xF1, 0xBB, 0xEA, 0xF3, 0xDE, 0xE4, /* 0x90-0x93 */ + 0xD9, 0xA3, 0xF7, 0xE5, 0xEE, 0xAF, 0xF5, 0xE6, /* 0x94-0x97 */ + 0xD6, 0xCC, 0xDD, 0xFA, 0xD1, 0xFB, 0xD2, 0xB3, /* 0x98-0x9B */ + 0xEE, 0xEA, 0xE9, 0xD0, 0xEC, 0xD4, 0xF1, 0xBC, /* 0x9C-0x9F */ + 0xFA, 0xA3, 0xFD, 0xCC, 0xDE, 0xD0, 0xEF, 0xE1, /* 0xA0-0xA3 */ + 0xDF, 0xBE, 0xF1, 0xE9, 0xF9, 0xBB, 0xF1, 0xA7, /* 0xA4-0xA7 */ + 0xE9, 0xD3, 0xEC, 0xA2, 0xF0, 0xF3, 0xF9, 0xCA, /* 0xA8-0xAB */ + 0xCA, 0xF8, 0xD0, 0xEA, 0xED, 0xC0, 0xFA, 0xF0, /* 0xAC-0xAF */ + 0xE5, 0xA8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ +}; + +static const unsigned char u2c_33[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xA7, 0xC9, 0xA7, 0xCA, 0xA7, 0xCB, 0xA7, 0xCC, /* 0x80-0x83 */ + 0xA7, 0xCD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0xA7, 0xBA, 0xA7, 0xBB, 0xA7, 0xDC, 0xA7, 0xDD, /* 0x88-0x8B */ + 0xA7, 0xDE, 0xA7, 0xB6, 0xA7, 0xB7, 0xA7, 0xB8, /* 0x8C-0x8F */ + 0xA7, 0xD4, 0xA7, 0xD5, 0xA7, 0xD6, 0xA7, 0xD7, /* 0x90-0x93 */ + 0xA7, 0xD8, 0xA7, 0xA1, 0xA7, 0xA2, 0xA7, 0xA3, /* 0x94-0x97 */ + 0xA7, 0xA5, 0xA7, 0xAB, 0xA7, 0xAC, 0xA7, 0xAD, /* 0x98-0x9B */ + 0xA7, 0xAE, 0xA7, 0xAF, 0xA7, 0xB0, 0xA7, 0xB1, /* 0x9C-0x9F */ + 0xA7, 0xB2, 0xA7, 0xB3, 0xA7, 0xB4, 0xA7, 0xA7, /* 0xA0-0xA3 */ + 0xA7, 0xA8, 0xA7, 0xA9, 0xA7, 0xAA, 0xA7, 0xBD, /* 0xA4-0xA7 */ + 0xA7, 0xBE, 0xA7, 0xE5, 0xA7, 0xE6, 0xA7, 0xE7, /* 0xA8-0xAB */ + 0xA7, 0xE8, 0xA7, 0xE1, 0xA7, 0xE2, 0xA7, 0xE3, /* 0xAC-0xAF */ + 0xA7, 0xBF, 0xA7, 0xC0, 0xA7, 0xC1, 0xA7, 0xC2, /* 0xB0-0xB3 */ + 0xA7, 0xC3, 0xA7, 0xC4, 0xA7, 0xC5, 0xA7, 0xC6, /* 0xB4-0xB7 */ + 0xA7, 0xC7, 0xA7, 0xC8, 0xA7, 0xCE, 0xA7, 0xCF, /* 0xB8-0xBB */ + 0xA7, 0xD0, 0xA7, 0xD1, 0xA7, 0xD2, 0xA7, 0xD3, /* 0xBC-0xBF */ + 0xA7, 0xDA, 0xA7, 0xDB, 0xA2, 0xE3, 0xA7, 0xEC, /* 0xC0-0xC3 */ + 0xA7, 0xA6, 0xA7, 0xE0, 0xA7, 0xEF, 0xA2, 0xE1, /* 0xC4-0xC7 */ + 0xA7, 0xBC, 0xA7, 0xED, 0xA7, 0xB5, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA7, 0xB9, /* 0xCC-0xCF */ + 0xA7, 0xEA, 0x00, 0x00, 0x00, 0x00, 0xA7, 0xEB, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0xA7, 0xDF, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xA2, 0xE4, 0x00, 0x00, 0x00, 0x00, 0xA7, 0xE4, /* 0xD8-0xDB */ + 0xA7, 0xEE, 0xA7, 0xE9, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ +}; + +static const unsigned char u2c_4E[512] = { + 0xEC, 0xE9, 0xEF, 0xCB, 0x00, 0x00, 0xF6, 0xD2, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD8, 0xB2, /* 0x04-0x07 */ + 0xED, 0xDB, 0xDF, 0xB2, 0xDF, 0xBE, 0xF9, 0xBB, /* 0x08-0x0B */ + 0x00, 0x00, 0xDC, 0xF4, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0xF5, 0xE4, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0xF3, 0xA6, 0xDD, 0xE0, 0xE1, 0xA6, 0x00, 0x00, /* 0x14-0x17 */ + 0xCE, 0xF8, 0xDC, 0xB0, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xAA, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0xF1, 0xE9, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0xCD, 0xFA, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0xFC, 0xAF, 0xD3, 0xA1, 0x00, 0x00, 0xF1, 0xAB, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0xD1, 0xD2, 0xAC, /* 0x40-0x43 */ + 0x00, 0x00, 0xCE, 0xF9, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF1, 0xFD, /* 0x48-0x4B */ + 0x00, 0x00, 0xDE, 0xBF, 0xFB, 0xBA, 0xF9, 0xB9, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0xCE, 0xD2, 0x00, 0x00, /* 0x54-0x57 */ + 0xE3, 0xAB, 0xEB, 0xE0, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0xCE, 0xFA, 0xCB, 0xF7, 0xE5, 0xA5, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCA, 0xE1, /* 0x68-0x6B */ + 0x00, 0x00, 0xD4, 0xCC, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEA, 0xE1, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0xDC, 0xE3, 0xDF, 0xAD, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0xCB, 0xEB, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0xD5, 0xAF, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0xD6, 0xF5, 0x00, 0x00, /* 0x84-0x87 */ + 0xE5, 0xF8, 0x00, 0x00, 0x00, 0x00, 0xDE, 0xC0, /* 0x88-0x8B */ + 0xEC, 0xA3, 0x00, 0x00, 0xE9, 0xCD, 0x00, 0x00, /* 0x8C-0x8F */ + 0xEA, 0xA7, 0xE9, 0xF6, 0xFB, 0xBB, 0x00, 0x00, /* 0x90-0x93 */ + 0xE7, 0xE9, 0xEF, 0xCC, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0xD0, 0xE6, 0x00, 0x00, 0x00, 0x00, 0xDE, 0xC1, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0xAC, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0xD8, 0xCC, 0xF9, 0xF1, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xCE, 0xDF, 0xFA, 0xA4, 0xE6, 0xB2, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xFA, 0xFB, 0x00, 0x00, 0x00, 0x00, 0xFA, 0xBD, /* 0xA8-0xAB */ + 0xCC, 0xC8, 0xEF, 0xCD, 0xD5, 0xD5, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0xD3, 0xA2, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0xEC, 0xD1, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0xE4, 0xA7, 0xEC, 0xD2, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xF6, 0xB1, 0x00, 0x00, 0x00, 0x00, 0xCE, 0xFB, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0xD0, 0xD1, 0xCB, 0xBF, /* 0xC8-0xCB */ + 0x00, 0x00, 0xED, 0xA4, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xED, 0xA8, 0xDE, 0xC2, 0xF6, 0xE2, 0xED, 0xDC, /* 0xD4-0xD7 */ + 0xDC, 0xF5, 0xE0, 0xB9, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0xD4, 0xCE, 0x00, 0x00, 0xF4, 0xB5, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD3, 0xDB, /* 0xE0-0xE3 */ + 0xD6, 0xB5, 0xEC, 0xA4, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0xE4, 0xE6, 0x00, 0x00, 0xF1, 0xEA, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0xCB, 0xEC, 0xCB, 0xC0, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEC, 0xF2, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_4F[512] = { + 0x00, 0x00, 0xD0, 0xEA, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xF9, 0xF2, 0xEC, 0xA5, 0xD0, 0xDF, /* 0x08-0x0B */ + 0x00, 0x00, 0xE7, 0xEA, 0xD0, 0xEB, 0xDC, 0xD1, /* 0x0C-0x0F */ + 0xDB, 0xE9, 0xFD, 0xCC, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDB, 0xD7, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0xDA, 0xE1, 0x00, 0x00, 0xD6, 0xB6, 0x00, 0x00, /* 0x34-0x37 */ + 0xE3, 0xDF, 0x00, 0x00, 0xDE, 0xC3, 0x00, 0x00, /* 0x38-0x3B */ + 0xDE, 0xC4, 0xCA, 0xA1, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEE, 0xEC, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xD3, 0xA3, 0xEE, 0xB7, /* 0x44-0x47 */ + 0xF8, 0xCF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0xEA, 0xC8, 0xEE, 0xB8, 0xF1, 0xAC, /* 0x4C-0x4F */ + 0xF1, 0xA5, 0xE9, 0xCE, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0xF9, 0xBC, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0xE5, 0xF9, 0xEC, 0xEA, 0xDD, 0xD6, /* 0x58-0x5B */ + 0xED, 0xC2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0xF8, 0xA5, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE5, 0xBA, /* 0x6C-0x6F */ + 0xDB, 0xD8, 0x00, 0x00, 0x00, 0x00, 0xCA, 0xA2, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0xD1, 0xCD, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0xEE, 0xED, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0xEC, 0xEB, 0xDE, 0xC5, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xE3, 0xE0, 0x00, 0x00, 0xCA, 0xC9, /* 0x80-0x83 */ + 0xF2, 0xE9, 0x00, 0x00, 0xD5, 0xCE, 0x00, 0x00, /* 0x84-0x87 */ + 0xF6, 0xB6, 0x00, 0x00, 0xCE, 0xC2, 0xD6, 0xC7, /* 0x88-0x8B */ + 0x00, 0x00, 0xE3, 0xB4, 0x00, 0x00, 0xF1, 0xAD, /* 0x8C-0x8F */ + 0x00, 0x00, 0xEA, 0xE2, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0xD7, 0xC2, 0x00, 0x00, /* 0x94-0x97 */ + 0xF3, 0xA7, 0x00, 0x00, 0x00, 0x00, 0xCD, 0xEA, /* 0x98-0x9B */ + 0x00, 0x00, 0xEB, 0xEE, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0xD9, 0xB2, 0xFD, 0xA5, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xF6, 0xD5, 0xD5, 0xE2, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF8, 0xB5, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0xCC, 0xF5, 0xF5, 0xB5, /* 0xC0-0xC3 */ + 0xE4, 0xAD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xE7, 0xEB, 0xF1, 0xD5, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0xF0, 0xBB, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0xE9, 0xB5, 0x00, 0x00, 0xCC, 0xC9, /* 0xD0-0xD3 */ + 0xFA, 0xD5, 0x00, 0x00, 0x00, 0x00, 0xE1, 0xD4, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xD7, 0xD6, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0xDC, 0xC1, 0x00, 0x00, 0xDE, 0xC6, /* 0xDC-0xDF */ + 0xFA, 0xEF, 0xE3, 0xE1, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xF3, 0xDC, 0xF6, /* 0xEC-0xEF */ + 0x00, 0x00, 0xCE, 0xFC, 0x00, 0x00, 0xDB, 0xC4, /* 0xF0-0xF3 */ + 0x00, 0x00, 0xF8, 0xF1, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xDC, 0xE4, 0x00, 0x00, 0xE5, 0xEF, 0x00, 0x00, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_50[512] = { + 0x00, 0x00, 0x00, 0x00, 0xDC, 0xB1, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0xD5, 0xD6, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xF3, 0xDA, 0x00, 0x00, 0xCB, 0xC1, /* 0x08-0x0B */ + 0x00, 0x00, 0xDB, 0xC3, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0xD9, 0xFA, 0xD3, 0xEE, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xFA, 0xB8, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0xFD, 0xA6, 0xEB, 0xEF, 0x00, 0x00, /* 0x18-0x1B */ + 0xF4, 0xA6, 0x00, 0x00, 0xCC, 0xCA, 0xF3, 0xA8, /* 0x1C-0x1F */ + 0x00, 0x00, 0xF3, 0xDB, 0x00, 0x00, 0xDB, 0xA7, /* 0x20-0x23 */ + 0xF6, 0xB7, 0x00, 0x00, 0xCF, 0xE6, 0xF0, 0xF2, /* 0x24-0x27 */ + 0xCB, 0xDA, 0x00, 0x00, 0xE7, 0xD2, 0xD7, 0xC3, /* 0x28-0x2B */ + 0xF6, 0xF0, 0xE8, 0xDE, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE5, 0xA6, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE5, 0xE7, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCA, 0xA3, /* 0x44-0x47 */ + 0xCC, 0xA7, 0xEA, 0xC9, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF8, 0xB6, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0xFA, 0xA5, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0xF1, 0xAE, 0x00, 0x00, /* 0x58-0x5B */ + 0xEF, 0xCE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0xCB, 0xED, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0xF6, 0xB0, 0xEF, 0xCF, 0xE9, 0xCF, 0x00, 0x00, /* 0x74-0x77 */ + 0xF7, 0xDE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xCE, 0xD3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0xDC, 0xF7, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0xDB, 0xA8, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xCB, 0xF8, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0xDF, 0xA1, 0xDD, 0xE1, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0xF5, 0xCA, 0xE9, 0xB6, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0xEC, 0xEE, 0xEE, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xF3, 0xF0, 0x00, 0x00, 0xDF, 0xBF, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0xCC, 0xCB, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xD0, 0xC1, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xF4, 0xD2, 0xE0, 0xBA, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDF, 0xC0, /* 0xCC-0xCF */ + 0x00, 0x00, 0xCE, 0xE0, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xDC, 0xD2, 0xFD, 0xEA, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xD6, 0xF6, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0xEA, 0xCA, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xE8, 0xE9, 0x00, 0x00, 0xE3, 0xAC, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0xF3, 0xD0, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xCA, 0xA4, 0x00, 0x00, 0xDB, 0xF8, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDE, 0xC7, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_51[512] = { + 0xEB, 0xF0, 0xF1, 0xD6, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0xE5, 0xE2, 0x00, 0x00, 0xCC, 0xCC, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xCB, 0xFB, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0xEA, 0xE3, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDF, 0xC1, /* 0x1C-0x1F */ + 0x00, 0x00, 0xD6, 0xED, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xD0, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0xEE, 0xB9, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD5, 0xE3, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xD1, 0xD3, 0x00, 0x00, /* 0x38-0x3B */ + 0xE5, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0xE8, 0xB4, 0xEB, 0xC3, 0x00, 0x00, 0xEA, 0xAA, /* 0x40-0x43 */ + 0xFA, 0xFC, 0xF5, 0xF6, 0xF0, 0xBC, 0xFD, 0xD4, /* 0x44-0x47 */ + 0xE0, 0xBB, 0xCE, 0xC3, 0x00, 0x00, 0xD0, 0xBA, /* 0x48-0x4B */ + 0xF7, 0xBA, 0xD8, 0xF3, 0xF7, 0xCD, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0xAE, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0xD4, 0xDF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0xD0, 0xE7, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0xEC, 0xFD, 0x00, 0x00, 0xD2, 0xAE, /* 0x64-0x67 */ + 0xEE, 0xEF, 0xD5, 0xD7, 0xEA, 0xE4, 0xF8, 0xA2, /* 0x68-0x6B */ + 0xCD, 0xEB, 0xD7, 0xBF, 0xFB, 0xB1, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0xCD, 0xEC, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0xDC, 0xB2, 0xD0, 0xEC, 0xCE, 0xFD, /* 0x74-0x77 */ + 0xEE, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0xCC, 0xC2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xD0, 0xED, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xF7, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0xF3, 0xFC, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0xEE, 0xA2, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0xD9, 0xB3, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0xD8, 0xF4, 0x00, 0x00, 0xE9, 0xB7, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0xCE, 0xAE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xD9, 0xA2, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0xD8, 0xF1, 0x00, 0x00, /* 0xA8-0xAB */ + 0xD4, 0xCF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xA7, 0xD5, 0xD2, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0xD6, 0xA9, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xF4, 0xA2, 0x00, 0x00, 0xF1, 0xD7, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xD5, 0xD8, 0x00, 0x00, 0xF0, 0xBD, /* 0xC8-0xCB */ + 0xD7, 0xD0, 0xD4, 0xD0, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0xD7, 0xCF, 0xEB, 0xEA, 0xFD, 0xEB, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0xDB, 0xED, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0xFC, 0xC5, 0xCB, 0xC2, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0xFD, 0xD5, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xF4, 0xC8, 0xE8, 0xEA, 0xF5, 0xF3, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0xF9, 0xDE, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_52[512] = { + 0xD3, 0xEF, 0x00, 0x00, 0x00, 0x00, 0xEC, 0xD3, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0xDD, 0xC2, 0xEF, 0xB7, /* 0x04-0x07 */ + 0xE7, 0xD4, 0x00, 0x00, 0xCA, 0xCA, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0xD9, 0xFB, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0xFA, 0xFD, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD6, 0xAA, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0xF4, 0xF8, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0xF7, 0xF7, 0xDC, 0xAC, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0xD7, 0xD7, 0xDF, 0xA2, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0xCE, 0xBE, 0x00, 0x00, /* 0x2C-0x2F */ + 0xD3, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0xF0, 0xA4, 0xE1, 0xEC, /* 0x34-0x37 */ + 0xCF, 0xE7, 0xF3, 0xCB, 0xED, 0xA9, 0xCA, 0xBE, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xEF, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF6, 0xCE, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0xDE, 0xFB, 0xD0, 0xBB, /* 0x48-0x4B */ + 0xD5, 0xB7, 0xEE, 0xF1, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0xF4, 0xA8, 0x00, 0x00, 0xDC, 0xF8, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCB, 0xA7, /* 0x58-0x5B */ + 0x00, 0x00, 0xDA, 0xCE, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0xE0, 0xE6, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0xED, 0xA5, 0xEE, 0xF2, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDC, 0xF9, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0xF9, 0xDC, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0xF3, 0xDC, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0xF8, 0xF2, 0x00, 0x00, 0xF4, 0xF9, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFC, 0xF1, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD0, 0xBC, /* 0x84-0x87 */ + 0xDB, 0xF9, 0xD7, 0xB1, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0xCB, 0xFC, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xF0, 0xA5, 0xCB, 0xFD, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD5, 0xF4, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCD, 0xED, /* 0x9C-0x9F */ + 0xCA, 0xA5, 0x00, 0x00, 0x00, 0x00, 0xD6, 0xAB, /* 0xA0-0xA3 */ + 0xD0, 0xC2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0xF0, 0xBE, 0xD2, 0xBD, 0xCC, 0xA4, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0xFA, 0xB6, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0xCC, 0xCD, 0x00, 0x00, 0xDA, 0xFA, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xF6, 0xCF, 0x00, 0x00, 0xE9, 0xB8, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xD8, 0xF5, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0xCC, 0xCE, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xD7, 0xCD, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xD4, 0xD1, 0xE9, 0xED, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xCA, 0xEB, 0xD9, 0xE2, 0x00, 0x00, 0xFD, 0xB2, /* 0xD8-0xDB */ + 0x00, 0x00, 0xE3, 0xAD, 0xD6, 0xCC, 0xD9, 0xB4, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xA7, 0xEE, 0xD3, /* 0xE0-0xE3 */ + 0xD0, 0xC3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFD, 0xB3, /* 0xF0-0xF3 */ + 0x00, 0x00, 0xD5, 0xE4, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xCF, 0xE8, 0x00, 0x00, 0xED, 0xC3, 0xD0, 0xB2, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0xCE, 0xFE, 0xDA, 0xA8, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_53[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0xF8, 0xD0, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0xFD, 0xD6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0xF8, 0xD1, 0x00, 0x00, 0xF8, 0xD2, /* 0x0C-0x0F */ + 0xDC, 0xD3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0xDD, 0xE2, 0xFB, 0xF9, 0xDD, 0xC1, /* 0x14-0x17 */ + 0x00, 0x00, 0xE3, 0xB5, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0xED, 0xDD, 0xCE, 0xC4, 0x00, 0x00, 0xCB, 0xA1, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xDD, 0xE3, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFC, 0xDD, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0xF9, 0xAF, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD2, 0xFB, /* 0x3C-0x3F */ + 0xCF, 0xA1, 0xE4, 0xA8, 0x00, 0x00, 0xF4, 0xB6, /* 0x40-0x43 */ + 0xEC, 0xFE, 0x00, 0x00, 0x00, 0x00, 0xE3, 0xAE, /* 0x44-0x47 */ + 0xE7, 0xED, 0xFD, 0xC1, 0xDA, 0xE2, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0xD8, 0xB3, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0xDD, 0xE4, 0xF0, 0xEF, 0xF6, 0xF1, /* 0x50-0x53 */ + 0xFA, 0xF0, 0x00, 0x00, 0x00, 0x00, 0xD1, 0xF5, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0xDA, 0xCF, 0x00, 0x00, /* 0x58-0x5B */ + 0xDC, 0xD4, 0x00, 0x00, 0xDC, 0xA6, 0x00, 0x00, /* 0x5C-0x5F */ + 0xEF, 0xBF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0xCE, 0xCF, 0x00, 0x00, /* 0x64-0x67 */ + 0xE0, 0xD9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD9, 0xD6, /* 0x6C-0x6F */ + 0xEC, 0xD4, 0xEA, 0xCB, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0xCA, 0xBF, 0xD5, 0xB0, 0x00, 0x00, 0xCF, 0xE9, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0xF1, 0xED, 0x00, 0x00, 0xCC, 0xCF, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0xE4, 0xF8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xED, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0xD7, 0xD8, 0x00, 0x00, 0xFD, 0xA7, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEA, 0xAB, /* 0x9C-0x9F */ + 0xF6, 0xB2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xCF, 0xF0, 0xF9, 0xBD, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0xE6, 0xF4, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCB, 0xDB, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF3, 0xD1, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0xE9, 0xD1, 0xF3, 0xA9, 0xD0, 0xE0, 0xE9, 0xD2, /* 0xC8-0xCB */ + 0x00, 0x00, 0xDA, 0xE3, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xE2, 0xD2, 0x00, 0x00, 0xF6, 0xA2, 0xE1, 0xF4, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDA, 0xE4, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0xE7, 0xD5, 0xF5, 0xBF, 0xCF, 0xA2, /* 0xE0-0xE3 */ + 0xCD, 0xAF, 0xCF, 0xA3, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xCD, 0xB0, 0xF1, 0xFE, 0xD0, 0xA3, /* 0xE8-0xEB */ + 0xE1, 0xAF, 0xF8, 0xA3, 0x00, 0x00, 0xCA, 0xA6, /* 0xEC-0xEF */ + 0xF7, 0xBB, 0xF2, 0xEA, 0xDE, 0xC8, 0xE9, 0xD3, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xDE, 0xC9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_54[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFD, 0xDE, /* 0x00-0x03 */ + 0xCA, 0xC0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0xF9, 0xEA, 0xD1, 0xCE, 0xEE, 0xD4, 0x00, 0x00, /* 0x08-0x0B */ + 0xD4, 0xD2, 0xD9, 0xA3, 0xFD, 0xA8, 0xD7, 0xD9, /* 0x0C-0x0F */ + 0xF7, 0xCE, 0xFA, 0xBE, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCF, 0xD6, /* 0x18-0x1B */ + 0x00, 0x00, 0xD7, 0xF0, 0x00, 0x00, 0xEB, 0xE1, /* 0x1C-0x1F */ + 0xF8, 0xC5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0xDC, 0xFA, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0xDD, 0xC3, 0x00, 0x00, 0xF9, 0xDF, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0xEF, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0xFD, 0xE5, 0xF6, 0xA3, 0x00, 0x00, 0xD9, 0xFC, /* 0x38-0x3B */ + 0xFD, 0xA9, 0x00, 0x00, 0xE7, 0xEE, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0xD5, 0xE5, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0xEF, 0xD0, 0x00, 0x00, 0xCD, 0xB1, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0xF7, 0xA2, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0xF1, 0xB2, 0x00, 0x00, 0xF1, 0xB1, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0xCD, 0xB2, 0x00, 0x00, 0xDA, 0xAB, /* 0x70-0x73 */ + 0x00, 0x00, 0xCA, 0xA7, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE3, 0xE2, /* 0x78-0x7B */ + 0xFB, 0xBC, 0xD9, 0xA4, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xEE, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0xF8, 0xD3, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0xFB, 0xFA, 0x00, 0x00, 0xCF, 0xA4, 0x00, 0x00, /* 0x8C-0x8F */ + 0xDC, 0xFB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xF6, 0xE3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xED, 0xAA, 0x00, 0x00, 0x00, 0x00, 0xF2, 0xA1, /* 0xA8-0xAB */ + 0xCE, 0xE1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFA, 0xA6, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xF9, 0xE0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0xEC, 0xD6, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0xE4, 0xEE, 0xF9, 0xA1, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xFB, 0xEF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0xF9, 0xEB, 0xEE, 0xA3, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0xEA, 0xAC, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xCA, 0xA8, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xF4, 0xFA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0xCD, 0xD6, 0xFC, 0xF6, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0xF4, 0xC9, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0xF8, 0xD4, 0x00, 0x00, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_55[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0xF8, 0xA6, 0x00, 0x00, 0xDE, 0xCA, 0xF2, 0xC6, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0xD7, 0xDA, 0x00, 0x00, /* 0x0C-0x0F */ + 0xD3, 0xD0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0xD8, 0xC5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEA, 0xE6, /* 0x2C-0x2F */ + 0x00, 0x00, 0xF3, 0xDD, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0xE4, 0xDA, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xF6, 0xE4, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0xF6, 0xF2, 0x00, 0x00, 0xDF, 0xC2, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD9, 0xFD, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCC, 0xF6, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0xD3, 0xBA, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0xAF, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF9, 0xE1, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0xF0, 0xA6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xCB, 0xD3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0xE0, 0xBC, 0x00, 0x00, 0xF4, 0xCA, 0xD4, 0xFA, /* 0x84-0x87 */ + 0x00, 0x00, 0xFD, 0xAA, 0xF9, 0xE2, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0xF4, 0xB7, 0xFD, 0xC2, 0xFC, 0xB0, 0x00, 0x00, /* 0x98-0x9B */ + 0xFD, 0xEC, 0xCA, 0xE2, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFD, 0xBD, /* 0xA4-0xA7 */ + 0x00, 0x00, 0xEA, 0xE7, 0xDF, 0xC3, 0xD1, 0xD2, /* 0xA8-0xAB */ + 0xCE, 0xE2, 0x00, 0x00, 0xD3, 0xA4, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xFD, 0xAB, 0x00, 0x00, 0xDF, 0xE0, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xF2, 0xC7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0xF0, 0x00, 0x00, /* 0xD8-0xDB */ + 0xD0, 0xEE, 0x00, 0x00, 0x00, 0x00, 0xF3, 0xAA, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDE, 0xCB, /* 0xE0-0xE3 */ + 0xF6, 0xB8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0xE1, 0xF5, 0xF1, 0xB3, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_56[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0xF7, 0xA3, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xCA, 0xA9, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0xCF, 0xA5, 0x00, 0x00, 0x00, 0x00, 0xDF, 0xC4, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE1, 0xB0, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0xF0, 0xBF, 0x00, 0x00, /* 0x30-0x33 */ + 0xF6, 0xA4, 0x00, 0x00, 0xE3, 0xB6, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFA, 0xC6, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0xD0, 0xEF, 0x00, 0x00, 0x00, 0x00, 0xFD, 0xED, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0xDD, 0xC4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0xFC, 0xF7, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xE6, 0xBF, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0xDE, 0xAD, 0x00, 0x00, 0xFA, 0xBF, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xE5, 0xF1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0xED, 0xC4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0xD2, 0xA5, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0xFD, 0xEE, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0xF5, 0xB6, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xF6, 0xDE, 0xCC, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0xFC, 0xDE, 0x00, 0x00, /* 0xDC-0xDF */ + 0xEC, 0xD7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0xCD, 0xDD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xD6, 0xB7, 0xCD, 0xB3, 0x00, 0x00, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_57[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF8, 0xD5, /* 0x00-0x03 */ + 0xE5, 0xD8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0xCF, 0xEA, 0x00, 0x00, 0x00, 0x00, 0xCF, 0xD0, /* 0x08-0x0B */ + 0x00, 0x00, 0xEA, 0xCC, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0xEA, 0xAE, 0xEA, 0xAD, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xD3, 0xF1, 0x00, 0x00, /* 0x14-0x17 */ + 0xD3, 0xA5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF7, 0xCF, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0xEE, 0xA4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0xD0, 0xA4, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xF2, 0xA2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD0, 0xF0, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0xF2, 0xA3, 0x00, 0x00, 0xF7, 0xF8, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD0, 0xB3, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0xDB, 0xA9, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0xD3, 0xBB, 0xCA, 0xEC, 0x00, 0x00, /* 0x4C-0x4F */ + 0xF1, 0xA6, 0xCB, 0xD5, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0xF7, 0xE7, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0xCD, 0xDE, 0x00, 0x00, 0xF7, 0xA4, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0xF8, 0xC0, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0xD3, 0xDD, 0x00, 0x00, /* 0x6C-0x6F */ + 0xCC, 0xD0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0xCF, 0xA6, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0xF6, 0xF3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xF7, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0xD3, 0xDC, 0x00, 0x00, 0x00, 0x00, 0xFA, 0xFE, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFA, 0xA7, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0xEB, 0xD9, 0x00, 0x00, 0xCF, 0xA7, 0xEA, 0xAF, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xEF, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE9, 0xB9, /* 0xC4-0xC7 */ + 0xF1, 0xD8, 0x00, 0x00, 0x00, 0x00, 0xD8, 0xD8, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xF2, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE6, 0xB4, /* 0xDC-0xDF */ + 0xDC, 0xFC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0xF3, 0xF1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xE3, 0xD0, 0x00, 0x00, 0x00, 0x00, 0xF2, 0xFB, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xDB, 0xC6, 0xD0, 0xF1, 0x00, 0x00, /* 0xF8-0xFB */ + 0xD0, 0xF2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_58[512] = { + 0xCF, 0xDC, 0x00, 0x00, 0xD3, 0xD1, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0xCC, 0xB1, 0xF7, 0xD8, 0x00, 0x00, /* 0x04-0x07 */ + 0xCB, 0xA8, 0xEB, 0xBC, 0xE4, 0xBE, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xF4, 0xDC, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0xDC, 0xC2, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0xF0, 0xA7, 0x00, 0x00, 0x00, 0x00, 0xE6, 0xC0, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xCA, 0xED, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE8, 0xEB, /* 0x2C-0x2F */ + 0xE5, 0xE8, 0xDC, 0xC3, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0xED, 0xDE, 0xD3, 0xF2, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xCC, 0xF7, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0xCE, 0xD4, 0xE7, 0xAB, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCB, 0xC3, /* 0x4C-0x4F */ + 0x00, 0x00, 0xE1, 0xB1, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0xF7, 0xB2, 0x00, 0x00, 0x00, 0x00, 0xD3, 0xF3, /* 0x54-0x57 */ + 0xD3, 0xD2, 0x00, 0x00, 0xF5, 0xC0, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0xDF, 0xDD, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0xEE, 0xF3, 0xE7, 0xF1, 0x00, 0x00, /* 0x60-0x63 */ + 0xFD, 0xB4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0xF2, 0xC8, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0xF3, 0xD2, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0xEE, 0xF4, 0x00, 0x00, 0xE2, 0xD3, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCC, 0xD1, /* 0x80-0x83 */ + 0x00, 0x00, 0xDF, 0xEA, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0xE9, 0xBA, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD9, 0xD7, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0xF5, 0xCD, 0x00, 0x00, 0xF1, 0xF2, 0xFA, 0xC7, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xD9, 0xF8, 0xD4, 0xC2, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0xF6, 0xE5, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDD, 0xC5, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0xF2, 0xED, 0xDF, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0xCA, 0xCB, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0xDB, 0xFA, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xE8, 0xB5, 0x00, 0x00, 0xD3, 0xA6, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0xFD, 0xB5, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0xF9, 0xC9, 0x00, 0x00, 0xE4, 0xE2, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xFB, 0xBD, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xD7, 0xA4, 0xCE, 0xC5, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0xCE, 0xD5, 0xD6, 0xE6, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0xE5, 0xBD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDE, 0xCD, /* 0xE8-0xEB */ + 0xEC, 0xF3, 0x00, 0x00, 0x00, 0x00, 0xED, 0xE0, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xEC, 0xEC, 0xFB, 0xBE, 0xDF, 0xEB, /* 0xF8-0xFB */ + 0x00, 0x00, 0xE1, 0xF8, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_59[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF9, 0xBE, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0xD0, 0xF3, 0xE0, 0xAA, 0xE8, 0xE2, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0xE2, 0xD4, 0xD2, 0xFD, 0x00, 0x00, /* 0x18-0x1B */ + 0xE5, 0xA8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0xD9, 0xD3, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD3, 0xDE, /* 0x24-0x27 */ + 0x00, 0x00, 0xF4, 0xB8, 0xF7, 0xBC, 0xDC, 0xFD, /* 0x28-0x2B */ + 0x00, 0x00, 0xE8, 0xEC, 0xE4, 0xE7, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0xE3, 0xF7, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEC, 0xA8, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xFA, 0xF1, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0xE5, 0xF2, 0x00, 0x00, 0x00, 0x00, 0xD0, 0xF4, /* 0x44-0x47 */ + 0xD2, 0xAF, 0xDC, 0xE5, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0xD0, 0xA5, 0xF1, 0xB4, /* 0x4C-0x4F */ + 0xFC, 0xB1, 0xCC, 0xF8, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0xDD, 0xC6, 0xFA, 0xD1, 0x00, 0x00, 0xF7, 0xDF, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0xFA, 0xA8, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0xEE, 0xF5, 0x00, 0x00, 0xDE, 0xCE, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0xF3, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0xF7, 0xAC, 0xEB, 0xC4, /* 0x68-0x6B */ + 0xED, 0xE1, 0xE0, 0xAB, 0xDD, 0xC7, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD2, 0xB3, /* 0x70-0x73 */ + 0xD2, 0xBF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0xCA, 0xCC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0xFB, 0xBF, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xFD, 0xDD, 0xE5, /* 0x80-0x83 */ + 0xD8, 0xCD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0xEC, 0xF4, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD0, 0xF5, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0xE8, 0xED, 0xD0, 0xD2, /* 0x94-0x97 */ + 0x00, 0x00, 0xD9, 0xD8, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xF6, 0xE6, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xDB, 0xAA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0xF7, 0xE0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xD8, 0xD9, 0x00, 0x00, 0xF4, 0xA3, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0xF4, 0xDD, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEF, 0xD1, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0xD9, 0xB5, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xED, 0xAB, 0x00, 0x00, 0xE3, 0xB7, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0xEE, 0xBB, 0xCD, 0xB4, 0x00, 0x00, 0xE0, 0xF3, /* 0xD0-0xD3 */ + 0xEA, 0xCD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xEC, 0xF5, 0xE8, 0xEE, 0x00, 0x00, /* 0xD8-0xDB */ + 0xCB, 0xA9, 0xF1, 0xAF, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0xCA, 0xCD, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xEC, 0xA9, 0x00, 0x00, 0xF2, 0xEB, 0x00, 0x00, /* 0xE8-0xEB */ + 0xFD, 0xEF, 0x00, 0x00, 0xF9, 0xF3, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xE6, 0xC1, 0x00, 0x00, 0x00, 0x00, 0xEC, 0xD8, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xED, 0xAC, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_5A[512] = { + 0x00, 0x00, 0xEA, 0xCE, 0x00, 0x00, 0xE8, 0xDF, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0xDE, 0xCF, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0xD2, 0xA6, 0x00, 0x00, 0x00, 0x00, 0xE7, 0xF4, /* 0x18-0x1B */ + 0xD1, 0xD6, 0x00, 0x00, 0x00, 0x00, 0xE6, 0xC2, /* 0x1C-0x1F */ + 0xE3, 0xE3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0xE4, 0xB0, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0xD8, 0xB4, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0xF6, 0xA5, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0xF3, 0xDE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0xD7, 0xA5, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xF7, 0xE8, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0xE8, 0xC6, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0xFB, 0xE6, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0xDD, 0xE6, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0xDC, 0xFE, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0xD8, 0xDA, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0xDA, 0xAC, 0xEA, 0xB0, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xE3, 0xB8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0xCA, 0xAA, 0xE1, 0xF9, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xEA, 0xB1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xF2, 0xEC, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0xFA, 0xEE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0xEE, 0xD5, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0xF9, 0xF4, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xD2, 0xEC, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ +}; + +static const unsigned char u2c_5B[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0xFB, 0xFB, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xFD, 0xF0, 0x00, 0x00, 0xE0, 0xBD, /* 0x08-0x0B */ + 0xCE, 0xE3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xF8, 0xC6, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xDE, 0xAE, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0xDF, 0xC5, 0x00, 0x00, 0x00, 0x00, 0xE5, 0xBE, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0xED, 0xAD, 0xFA, 0xEA, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0xCD, 0xEE, 0xED, 0xA6, 0x00, 0x00, 0xED, 0xAE, /* 0x54-0x57 */ + 0xF0, 0xED, 0x00, 0x00, 0xDD, 0xA1, 0x00, 0x00, /* 0x58-0x5B */ + 0xED, 0xAF, 0xFC, 0xF8, 0x00, 0x00, 0xD8, 0xEB, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCC, 0xF9, /* 0x60-0x63 */ + 0xCD, 0xB5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0xFA, 0xA9, 0x00, 0x00, 0xE1, 0xDD, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0xE2, 0xD5, 0xED, 0xCF, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0xDD, 0xA2, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0xF9, 0xCA, 0x00, 0x00, 0xEA, 0xE8, 0x00, 0x00, /* 0x78-0x7B */ + 0xE5, 0xED, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0xD3, 0xEB, 0x00, 0x00, 0xE9, 0xD4, /* 0x84-0x87 */ + 0xE1, 0xFA, 0xE4, 0xCC, 0x00, 0x00, 0xE1, 0xE4, /* 0x88-0x8B */ + 0xE8, 0xC7, 0x00, 0x00, 0x00, 0x00, 0xCE, 0xDB, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDC, 0xD5, /* 0x90-0x93 */ + 0x00, 0x00, 0xF7, 0xB5, 0xFC, 0xF3, 0xF0, 0xF3, /* 0x94-0x97 */ + 0xCE, 0xAF, 0xF1, 0xB5, 0xEF, 0xD2, 0xE8, 0xC8, /* 0x98-0x9B */ + 0xEB, 0xF1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0xCB, 0xD4, 0xE0, 0xBE, /* 0xA0-0xA3 */ + 0xE3, 0xF8, 0xEA, 0xE9, 0xFC, 0xB2, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0xE0, 0xF4, 0x00, 0x00, 0xCF, 0xE0, 0x00, 0x00, /* 0xAC-0xAF */ + 0xEE, 0xA5, 0x00, 0x00, 0x00, 0x00, 0xFA, 0xAA, /* 0xB0-0xB3 */ + 0xE6, 0xC3, 0xE1, 0xB2, 0xCA, 0xAB, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xE3, 0xE4, 0xE9, 0xBB, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xD6, /* 0xBC-0xBF */ + 0xF3, 0xF2, 0x00, 0x00, 0xEE, 0xD6, 0xEA, 0xB2, /* 0xC0-0xC3 */ + 0xD0, 0xF6, 0xEC, 0xD9, 0xDA, 0xCB, 0xCF, 0xA8, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0xDD, 0xA3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0xD8, 0xDB, 0x00, 0x00, 0xF9, 0xCE, 0xE9, 0xD5, /* 0xD0-0xD3 */ + 0xE3, 0xD1, 0x00, 0x00, 0x00, 0x00, 0xD2, 0xBC, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0xD8, 0xAC, 0xF3, 0xCC, /* 0xDC-0xDF */ + 0x00, 0x00, 0xCD, 0xFB, 0xF6, 0xD6, 0x00, 0x00, /* 0xE0-0xE3 */ + 0xE7, 0xF5, 0xE8, 0xEF, 0xE3, 0xF9, 0xD2, 0xBB, /* 0xE4-0xE7 */ + 0xF3, 0xF3, 0xE3, 0xFB, 0x00, 0x00, 0xDE, 0xD0, /* 0xE8-0xEB */ + 0xCE, 0xB0, 0x00, 0x00, 0xD6, 0xF7, 0xF1, 0xD9, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0xF5, 0xC1, 0xDC, 0xC4, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xF5, 0xBB, 0x00, 0x00, 0xDE, 0xD1, 0x00, 0x00, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_5C[512] = { + 0x00, 0x00, 0xDC, 0xE6, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0xDE, 0xD2, 0x00, 0x00, 0x00, 0x00, 0xED, 0xE2, /* 0x04-0x07 */ + 0xEE, 0xF6, 0xEA, 0xCF, 0xF0, 0xEE, 0xE3, 0xFC, /* 0x08-0x0B */ + 0x00, 0x00, 0xD3, 0xDF, 0xD3, 0xF4, 0xE1, 0xB3, /* 0x0C-0x0F */ + 0x00, 0x00, 0xE1, 0xB4, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xF4, 0xD3, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0xDF, 0xC6, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0xE9, 0xD6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0xDB, 0xAB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0xF6, 0xA6, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0xE3, 0xB9, 0xEB, 0xC5, 0xF4, 0xA9, 0xCD, 0xB6, /* 0x38-0x3B */ + 0xD2, 0xF9, 0x00, 0x00, 0xDA, 0xAD, 0xD2, 0xE3, /* 0x3C-0x3F */ + 0xCF, 0xD1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0xCB, 0xDC, 0xCC, 0xFA, 0x00, 0x00, /* 0x44-0x47 */ + 0xCF, 0xDD, 0x00, 0x00, 0x00, 0x00, 0xE8, 0xA9, /* 0x48-0x4B */ + 0x00, 0x00, 0xE3, 0xBB, 0xE3, 0xBA, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0xE0, 0xDA, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0xEE, 0xF7, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDC, 0xB3, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0xD3, 0xF5, 0x00, 0x00, 0xD7, 0xA6, 0x00, 0x00, /* 0x60-0x63 */ + 0xF6, 0xB5, 0xD7, 0xDB, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0xE1, 0xD5, 0x00, 0x00, 0x00, 0x00, 0xD4, 0xEA, /* 0x6C-0x6F */ + 0x00, 0x00, 0xDF, 0xA3, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0xFD, 0xDF, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0xD0, 0xF7, 0xED, 0xD4, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0xCB, 0xAA, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0xE4, 0xDB, 0x00, 0x00, 0xE1, 0xFB, /* 0xA8-0xAB */ + 0xCB, 0xA2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0xD3, 0xE0, 0x00, 0x00, 0xE4, 0xBF, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xFB, 0xC0, 0x00, 0x00, 0xDA, 0xBE, /* 0xB4-0xB7 */ + 0xE4, 0xCD, 0x00, 0x00, 0xD6, 0xB9, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0xEF, 0xC0, 0x00, 0x00, /* 0xBC-0xBF */ + 0xE1, 0xFC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xF6, 0xB9, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0xDF, 0xC7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xE4, 0xB1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDC, 0xE7, /* 0xEC-0xEF */ + 0xDC, 0xE8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xFA, 0xD6, 0x00, 0x00, 0xD3, 0xF6, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF1, 0xDA, /* 0xF8-0xFB */ + 0x00, 0x00, 0xFA, 0xF2, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_5D[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xFD, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0xD5, 0xCF, 0xD0, 0xF8, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0xCD, 0xDF, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0xF5, 0xCB, 0x00, 0x00, 0xE4, 0xF0, 0xCB, 0xAB, /* 0x14-0x17 */ + 0x00, 0x00, 0xD7, 0xC4, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xFE, /* 0x24-0x27 */ + 0x00, 0x00, 0xDD, 0xDA, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDA, 0xAE, /* 0x48-0x4B */ + 0xCA, 0xEE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0xD5, 0xB9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0xE3, 0xA1, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0xE8, 0xE3, 0x00, 0x00, 0x00, 0x00, 0xF3, 0xAB, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCF, 0xA9, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD3, 0xF7, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0xD4, 0xF1, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0xCE, 0xE4, 0x00, 0x00, 0xE8, 0xF2, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xF5, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xE7, 0xAE, 0x00, 0x00, 0xD6, 0xBA, 0x00, 0x00, /* 0xB8-0xBB */ + 0xDF, 0xEC, 0xE4, 0xC0, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0xE8, 0xE4, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xD8, 0xB5, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0xDC, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0xF4, 0xB9, 0xF1, 0xB6, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0xE2, 0xDE, 0xE1, 0xB5, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xCD, 0xEF, 0xF1, 0xA7, 0xCE, 0xE5, /* 0xE4-0xE7 */ + 0xCB, 0xDD, 0x00, 0x00, 0x00, 0x00, 0xD9, 0xE3, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0xF3, 0xAC, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0xD0, 0xF9, 0xEC, 0xAB, 0xDE, 0xD3, /* 0xF0-0xF3 */ + 0xF7, 0xE9, 0x00, 0x00, 0x00, 0x00, 0xF9, 0xF5, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0xE1, 0xDE, 0xCB, 0xEE, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_5E[512] = { + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xBC, 0xF8, 0xD6, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0xDB, 0xEE, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0xFD, 0xF1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0xF7, 0xB6, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xF4, 0xDE, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0xF2, 0xED, 0x00, 0x00, 0xDB, 0xD9, /* 0x18-0x1B */ + 0x00, 0x00, 0xF0, 0xA8, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0xE1, 0xFD, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDE, 0xD4, /* 0x28-0x2B */ + 0x00, 0x00, 0xE0, 0xAC, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xED, 0xE3, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0xD3, 0xE1, 0x00, 0x00, /* 0x34-0x37 */ + 0xDF, 0xC8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0xD9, 0xB6, 0x00, 0x00, 0xFD, 0xAC, /* 0x3C-0x3F */ + 0xEF, 0xD3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0xE4, 0xC1, 0xF8, 0xEB, 0x00, 0x00, 0xDB, 0xAC, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0xFC, 0xC6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0xD8, 0xAD, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF6, 0xBA, /* 0x5C-0x5F */ + 0x00, 0x00, 0xDB, 0xDF, 0xD3, 0xD3, 0xF8, 0xC7, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0xCA, 0xCE, 0xF8, 0xC1, /* 0x70-0x73 */ + 0xD2, 0xB4, 0x00, 0x00, 0x00, 0x00, 0xDC, 0xB4, /* 0x74-0x77 */ + 0xFA, 0xB9, 0xCA, 0xCF, 0x00, 0x00, 0xFC, 0xB3, /* 0x78-0x7B */ + 0xEA, 0xEA, 0xEA, 0xEB, 0xD0, 0xFA, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0xED, 0xE4, 0x00, 0x00, 0x00, 0x00, 0xDD, 0xE7, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0xDF, 0xC9, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDF, 0xED, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0xEE, 0xBC, 0x00, 0x00, 0xEF, 0xC1, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0xCC, 0xD2, 0x00, 0x00, /* 0x98-0x9B */ + 0xDD, 0xA4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0xDF, 0xCA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0xD3, 0xF8, 0xF1, 0xA8, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCD, 0xB7, /* 0xA8-0xAB */ + 0x00, 0x00, 0xEF, 0xD4, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xE4, 0xDD, 0xDF, 0xEE, 0xCB, 0xAC, /* 0xB4-0xB7 */ + 0xE9, 0xBC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0xEA, 0xEC, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0xDF, 0xCB, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0xF9, 0xBF, 0xD6, 0xAF, 0xD5, 0xC6, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0xCF, 0xAA, 0x00, 0x00, 0x00, 0x00, 0xCE, 0xA9, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0xD6, 0xF8, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xF1, 0xB7, 0xEE, 0xF8, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD9, 0xD9, /* 0xDC-0xDF */ + 0xF3, 0xDF, 0x00, 0x00, 0xF8, 0xC8, 0xCE, 0xC6, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0xD5, 0xE6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xE6, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0xC5, 0xEF, 0xD5, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0xCB, 0xEF, 0xFC, 0xDF, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_5F[512] = { + 0x00, 0x00, 0xDC, 0xA7, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0xD6, 0xE7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xF8, 0xC9, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE3, 0xD2, /* 0x0C-0x0F */ + 0x00, 0x00, 0xE3, 0xBD, 0x00, 0x00, 0xCF, 0xE1, /* 0x10-0x13 */ + 0xF0, 0xC0, 0xEC, 0xDA, 0x00, 0x00, 0xDD, 0xD7, /* 0x14-0x17 */ + 0xFB, 0xF0, 0x00, 0x00, 0x00, 0x00, 0xEC, 0xAC, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xA9, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0xFA, 0xD7, 0xFB, 0xC1, /* 0x24-0x27 */ + 0x00, 0x00, 0xD2, 0xC0, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0xE5, 0xB0, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0xED, 0xE5, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xCB, 0xAD, 0x00, 0x00, /* 0x38-0x3B */ + 0xF9, 0xB0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0xF7, 0xA5, 0x00, 0x00, 0xCB, 0xAE, 0x00, 0x00, /* 0x48-0x4B */ + 0xDA, 0xAF, 0x00, 0x00, 0xD8, 0xB6, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0xD3, 0xA7, 0xFB, 0xB2, /* 0x54-0x57 */ + 0x00, 0x00, 0xFD, 0xC4, 0x00, 0x00, 0xEC, 0xAD, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0xFB, 0xA1, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xE9, 0xE9, 0xEE, /* 0x64-0x67 */ + 0x00, 0x00, 0xF3, 0xF4, 0xF8, 0xF3, 0xF0, 0xC1, /* 0x68-0x6B */ + 0xDE, 0xAF, 0xF8, 0xB0, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0xF3, 0xE0, 0xE7, 0xAF, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDB, 0xAD, /* 0x74-0x77 */ + 0x00, 0x00, 0xE6, 0xB5, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0xF9, 0xA8, 0x00, 0x00, 0x00, 0x00, 0xDD, 0xD8, /* 0x7C-0x7F */ + + 0xE8, 0xD9, 0xEF, 0xD6, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0xD3, 0xE2, 0x00, 0x00, 0xE2, 0xDF, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0xFC, 0xE0, 0xD7, 0xC8, /* 0x88-0x8B */ + 0xFD, 0xAD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0xDF, 0xEF, 0xCC, 0xD3, 0xD3, 0xF9, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD4, 0xF0, /* 0x94-0x97 */ + 0xDB, 0xC7, 0xDE, 0xD5, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0xF0, 0xF4, 0x00, 0x00, /* 0x9C-0x9F */ + 0xD5, 0xD0, 0xE5, 0xD9, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xFC, 0xC7, 0xDC, 0xD6, 0xE2, 0xE0, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0xDA, 0xB0, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xF3, 0xA3, 0x00, 0x00, 0xD3, 0xEC, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xF4, 0xCB, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0xFD, 0xC5, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE3, 0xFD, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xF9, 0xB1, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0xD0, 0xFB, 0xEC, 0xDB, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0xF5, 0xBC, 0xF2, 0xA4, /* 0xD4-0xD7 */ + 0xD8, 0xCE, 0xD8, 0xCF, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0xF5, 0xF7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF6, 0xE1, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0xD2, 0xB7, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0xFB, 0xEC, 0x00, 0x00, 0xDD, 0xC8, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_60[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xE8, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0xD2, 0xC1, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xF8, 0xD7, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0xD6, 0xBB, 0xDE, 0xD6, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0xF7, 0xBD, 0xEC, 0xAE, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0xD0, 0xE1, 0x00, 0x00, 0xE0, 0xF5, /* 0x24-0x27 */ + 0xEA, 0xB3, 0x00, 0x00, 0xCE, 0xD6, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCC, 0xA5, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0xEC, 0xF6, 0xE2, 0xE1, 0xE3, 0xBE, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0xFC, 0xC8, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0xCD, 0xF0, 0x00, 0x00, 0xF9, 0xF6, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0xDF, 0xF0, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0xE5, 0xBF, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0xCE, 0xBF, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0xFC, 0xE1, 0xED, 0xB0, /* 0x60-0x63 */ + 0xFD, 0xD1, 0xF6, 0xBB, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0xF9, 0xCF, 0xEB, 0xDA, 0xCA, 0xC1, 0x00, 0x00, /* 0x68-0x6B */ + 0xD2, 0xB8, 0xCD, 0xF1, 0x00, 0x00, 0xE3, 0xD3, /* 0x6C-0x6F */ + 0xFD, 0xE6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0xE6, 0xED, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0xE3, 0xFA, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0xF0, 0xAA, 0xF9, 0xD0, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0xFC, 0xE2, 0x00, 0x00, 0xF8, 0xA7, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xE5, 0xEE, 0xF9, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0xF6, /* 0x9C-0x9F */ + 0xEA, 0xED, 0x00, 0x00, 0x00, 0x00, 0xFC, 0xB4, /* 0xA0-0xA3 */ + 0xF5, 0xC2, 0x00, 0x00, 0x00, 0x00, 0xD7, 0xDC, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0xF0, 0xF5, 0x00, 0x00, 0xDD, 0xE8, 0xD3, 0xED, /* 0xB0-0xB3 */ + 0xF5, 0xFC, 0x00, 0x00, 0xDA, 0xBF, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xCC, 0xFB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0xD3, 0xFA, 0xF4, 0xA4, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xEF, 0xD7, 0x00, 0x00, 0xD4, 0xC3, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0xFB, 0xE3, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xFB, 0xED, 0x00, 0x00, /* 0xD8-0xDB */ + 0xE0, 0xAD, 0x00, 0x00, 0x00, 0x00, 0xEA, 0xEE, /* 0xDC-0xDF */ + 0xFB, 0xB3, 0xE4, 0xC2, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0xF6, 0xE7, 0xD2, 0xDD, 0x00, 0x00, 0xDF, 0xCC, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0xFC, 0xC9, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xE5, 0xA9, 0xE0, 0xF6, 0xF6, 0xB3, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_61[512] = { + 0x00, 0x00, 0xE1, 0xFE, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0xCB, 0xF0, 0x00, 0x00, /* 0x04-0x07 */ + 0xEA, 0xEF, 0xEA, 0xF0, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0xDA, 0xC0, 0xF8, 0xB4, 0xEB, 0xF2, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0xE4, 0xC3, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xD7, 0xE4, 0xF1, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCA, 0xEF, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCE, 0xD7, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xFC, 0xCA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0xF3, 0xE1, 0x00, 0x00, 0x00, 0x00, 0xCB, 0xC4, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0xE3, 0xE5, 0x00, 0x00, 0xCB, 0xC5, 0xEA, 0xB4, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xBD, 0x00, 0x00, /* 0x40-0x43 */ + 0xD7, 0xC9, 0x00, 0x00, 0x00, 0x00, 0xEB, 0xDB, /* 0x44-0x47 */ + 0xED, 0xB1, 0x00, 0x00, 0xCC, 0xC3, 0xF7, 0xBE, /* 0x48-0x4B */ + 0xFC, 0xCB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF8, 0xF4, /* 0x50-0x53 */ + 0x00, 0x00, 0xD9, 0xB7, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0xF3, 0xD3, 0xF3, 0xD4, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0xF7, 0xE4, 0x00, 0x00, 0xF7, 0xD1, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0xD8, 0xB7, 0xCE, 0xB1, /* 0x60-0x63 */ + 0xCA, 0xC2, 0x00, 0x00, 0x00, 0x00, 0xFB, 0xB4, /* 0x64-0x67 */ + 0xCB, 0xC6, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xF6, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0xD5, 0xE7, 0x00, 0x00, /* 0x6C-0x6F */ + 0xEA, 0xD0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0xCC, 0xD4, 0xCB, 0xAF, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0xF4, 0xAA, 0xE9, 0xAF, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xF5, 0xC3, 0xE9, 0xD8, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0xDD, 0xE9, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0xF1, 0xF3, 0x00, 0x00, /* 0x8C-0x8F */ + 0xD5, 0xFB, 0xDE, 0xBB, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0xF4, 0xFB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0xFD, 0xF3, 0xFD, 0xF2, 0xF7, 0xA6, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xDD, 0xC9, 0x00, 0x00, 0x00, 0x00, 0xD4, 0xD3, /* 0xA4-0xA7 */ + 0x00, 0x00, 0xCC, 0xA8, 0x00, 0x00, 0xDA, 0xC1, /* 0xA8-0xAB */ + 0xCC, 0xD5, 0x00, 0x00, 0xD9, 0xE4, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xFA, 0xCA, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xE3, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0xD3, 0xBC, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0xCA, 0xF0, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD0, 0xC4, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCA, 0xD0, /* 0xC4-0xC7 */ + 0xFA, 0xAB, 0xEB, 0xEB, 0xE7, 0xF8, 0xD9, 0xE5, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0xD1, 0xD7, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0xF3, 0xA4, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0xD4, 0xFB, 0xFC, 0xE3, /* 0xF4-0xF7 */ + 0xFA, 0xD8, 0x00, 0x00, 0xF3, 0xD5, 0x00, 0x00, /* 0xF8-0xFB */ + 0xCF, 0xAB, 0x00, 0x00, 0x00, 0x00, 0xEB, 0xF3, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_62[512] = { + 0xD5, 0xFC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD3, 0xD4, /* 0x04-0x07 */ + 0xCD, 0xFC, 0x00, 0x00, 0xD9, 0xE6, 0x00, 0x00, /* 0x08-0x0B */ + 0xE2, 0xF9, 0xE2, 0xA1, 0xEB, 0xD4, 0x00, 0x00, /* 0x0C-0x0F */ + 0xE0, 0xF7, 0xE4, 0xB2, 0xCC, 0xFC, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xFB, 0xE4, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0xF4, 0xAB, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD0, 0xBD, /* 0x1C-0x1F */ + 0x00, 0x00, 0xCA, 0xF1, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xEF, 0xB8, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0xD7, 0xC0, 0x00, 0x00, /* 0x2C-0x2F */ + 0xEE, 0xFA, 0xFD, 0xF4, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0xD3, 0xE3, 0x00, 0x00, 0xFB, 0xC2, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xD5, 0xE8, 0xDB, 0xAE, /* 0x3C-0x3F */ + 0xE1, 0xB6, 0xF8, 0xB7, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xBF, /* 0x44-0x47 */ + 0xFB, 0xC3, 0xDD, 0xEA, 0x00, 0x00, 0xE2, 0xA2, /* 0x48-0x4B */ + 0x00, 0x00, 0xEE, 0xA6, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF6, 0xE8, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0xF6, 0xF5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0xDD, 0xCA, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0xD0, 0xE2, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0xDD, 0xA6, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0xDD, 0xEB, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0xE4, 0xF9, 0x00, 0x00, 0x00, 0x00, 0xE3, 0xAF, /* 0x7C-0x7F */ + + 0xD0, 0xFC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0xF4, 0xFC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0xCC, 0xBC, 0xF7, 0xEA, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xE5, 0xE4, 0xDF, 0xF1, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0xF7, 0xE1, 0x00, 0x00, 0xF9, 0xF7, /* 0x94-0x97 */ + 0xEF, 0xB9, 0x00, 0x00, 0x00, 0x00, 0xF8, 0xD8, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF9, 0xA9, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0xF8, 0xD9, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xEE, 0xBD, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xD8, 0xC6, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0xE4, 0xE3, 0xF5, 0xCE, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0xDD, 0xD9, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD9, 0xE7, /* 0xC4-0xC7 */ + 0xD2, 0xB9, 0xD5, 0xC3, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0xDA, 0xE5, 0xDA, 0xD0, 0x00, 0x00, 0xD1, 0xD9, /* 0xCC-0xCF */ + 0xCE, 0xD8, 0x00, 0x00, 0xCB, 0xDE, 0xF4, 0xAC, /* 0xD0-0xD3 */ + 0xDA, 0xFB, 0x00, 0x00, 0xF6, 0xE9, 0xE8, 0xF3, /* 0xD4-0xD7 */ + 0xCF, 0xAC, 0xF0, 0xF0, 0x00, 0x00, 0xF4, 0xFD, /* 0xD8-0xDB */ + 0xDB, 0xC8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0xCE, 0xC0, 0xE3, 0xD4, 0xD1, 0xCF, 0xF1, 0xF5, /* 0xEC-0xEF */ + 0x00, 0x00, 0xCD, 0xF2, 0x00, 0x00, 0xCF, 0xEB, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCD, 0xB8, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xA6, 0xD1, 0xDA, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_63[512] = { + 0x00, 0x00, 0xF2, 0xA5, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF2, 0xA6, /* 0x04-0x07 */ + 0x00, 0x00, 0xE4, 0xCE, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0xD3, 0xFB, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF1, 0xA9, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF2, 0xC9, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xEF, 0xD8, 0xE6, 0xC9, /* 0x38-0x3B */ + 0x00, 0x00, 0xD8, 0xB8, 0xFA, 0xF3, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0xF3, 0xB5, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0xF8, 0xA4, 0x00, 0x00, 0x00, 0x00, 0xD1, 0xF3, /* 0x4C-0x4F */ + 0xE6, 0xC8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0xF8, 0xDA, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDC, 0xE9, /* 0x64-0x67 */ + 0xDE, 0xD7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0xCB, 0xDF, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0xCF, 0xEC, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xDF, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0xD1, 0xF4, 0xD2, 0xBA, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDF, 0xF2, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE1, 0xB7, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0xE2, 0xA3, 0xD3, 0xFC, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0xED, 0xE6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0xDB, 0xC9, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0xFA, 0x00, 0x00, /* 0x94-0x97 */ + 0xCF, 0xDE, 0x00, 0x00, 0x00, 0x00, 0xCE, 0xD0, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0xD5, 0xD3, 0xF3, 0xF5, 0xF7, 0xAE, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xEF, 0xC8, 0x00, 0x00, 0xCD, 0xF3, /* 0xA4-0xA7 */ + 0xF5, 0xCF, 0xE5, 0xF3, 0xF0, 0xC2, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0xCA, 0xD1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xEA, 0xF1, 0x00, 0x00, 0xD0, 0xA6, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD9, 0xDA, /* 0xCC-0xCF */ + 0xF0, 0xAB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0xEB, 0xE7, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xC0, 0xFC, 0xB5, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0xE4, 0xC4, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0xCC, 0xA9, 0xFD, 0xC6, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xEA, 0xB5, 0x00, 0x00, 0xE5, 0xAA, 0xDF, 0xBA, /* 0xF4-0xF7 */ +}; + +static const unsigned char u2c_64[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0xE1, 0xDF, 0x00, 0x00, 0xDA, 0xD1, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0xE1, 0xB8, 0x00, 0x00, 0xE8, 0xF4, 0xD3, 0xFD, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0xE2, 0xA4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0xF2, 0xCA, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0xDA, 0xE6, 0xF7, 0xB3, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xFD, 0xCD, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xF3, 0xB6, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0xEE, 0xD7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0xF5, 0xC4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0xD8, 0xA4, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF2, 0xA7, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0xD9, 0xB8, 0xD9, 0xB9, 0xEF, 0xC9, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0xD6, 0xCE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xF7, 0xCB, 0xDF, 0xAE, 0xE8, 0xF5, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0xD2, 0xB5, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0xD3, 0xD5, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xF4, 0xCC, 0xDA, 0xFC, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD9, 0xE8, /* 0xA8-0xAB */ + 0x00, 0x00, 0xF7, 0xEB, 0xF5, 0xC9, 0x00, 0x00, /* 0xAC-0xAF */ + 0xF3, 0xBC, 0x00, 0x00, 0xDA, 0xD2, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD3, 0xB5, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0xE8, 0xB6, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xD6, 0xCF, 0xF4, 0xBA, 0x00, 0x00, 0xF7, 0xC9, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0xCC, 0xAA, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0xF0, 0xC3, 0xCC, 0xD6, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xD0, 0xD3, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xD3, 0xBD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xDB, 0xFB, 0x00, 0x00, 0xCB, 0xE0, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0xD3, 0xE4, 0xF6, 0xF7, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xD5, 0xBA, 0xF3, 0xCD, 0xCB, 0xE1, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0xEB, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0xF4, 0xAD, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xFC, 0xAA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0xF7, 0xEC, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0xE8, 0xF6, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_65[512] = { + 0xDA, 0xE7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0xF7, 0xCC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0xE5, 0xC1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0xE0, 0xEE, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD5, 0xFD, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xCE, 0xE6, 0xFC, 0xAB, /* 0x28-0x2B */ + 0xD5, 0xBB, 0x00, 0x00, 0x00, 0x00, 0xF2, 0xA8, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0xA5, 0xCD, 0xB9, /* 0x34-0x37 */ + 0xEA, 0xF2, 0xCB, 0xC7, 0x00, 0x00, 0xCD, 0xF4, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xDB, 0xAF, 0xEF, 0xD9, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0xCD, 0xBA, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0xFC, 0xF9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0xDF, 0xF3, 0xCE, 0xE7, 0xDA, 0xC2, /* 0x4C-0x4F */ + 0x00, 0x00, 0xCF, 0xAD, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0xF9, 0xF8, 0xA8, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0xF3, 0xE2, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0xCA, 0xF2, 0xDF, 0xA4, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0xD4, 0xC4, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0xCC, 0xD7, 0xE5, 0xC2, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0xCD, 0xBB, 0x00, 0x00, /* 0x70-0x73 */ + 0xEF, 0xDA, 0xEE, 0xD8, 0x00, 0x00, 0xDD, 0xA7, /* 0x74-0x77 */ + 0xE2, 0xA6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xC0, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0xD6, 0xB0, 0xF8, 0xCA, /* 0x80-0x83 */ + 0x00, 0x00, 0xFC, 0xFA, 0x00, 0x00, 0xD9, 0xFE, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0xDE, 0xB0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0xDD, 0xEC, 0xDA, 0xE8, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD4, 0xE0, /* 0x94-0x97 */ + 0x00, 0x00, 0xD6, 0xF9, 0x00, 0x00, 0xCD, 0xD7, /* 0x98-0x9B */ + 0xDE, 0xD8, 0x00, 0x00, 0x00, 0x00, 0xF2, 0xF8, /* 0x9C-0x9F */ + 0x00, 0x00, 0xE4, 0xD6, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xD0, 0xC5, 0xF4, 0xAE, 0x00, 0x00, 0xDD, 0xA8, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xED, 0xC5, /* 0xA8-0xAB */ + 0xF3, 0xD6, 0x00, 0x00, 0x00, 0x00, 0xDE, 0xD9, /* 0xAC-0xAF */ + 0xE3, 0xE6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD3, 0xA8, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xDB, 0xB0, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0xE5, 0xDA, 0xE3, 0xBF, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0xDB, 0xB1, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xD5, 0xE9, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xC1, /* 0xC8-0xCB */ + 0xEF, 0xDB, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xE9, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xD7, 0xB2, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD0, 0xFD, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0xD9, 0xE9, 0x00, 0x00, 0x00, 0x00, 0xD0, 0xFE, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xEC, 0xED, 0xD3, 0xA9, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xF2, 0xA9, 0xF0, 0xC4, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0xE2, 0xE2, 0xE9, 0xEF, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0xF9, 0xD1, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xE9, 0xD9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0xE8, 0xDA, 0xDA, 0xC3, /* 0xF8-0xFB */ + 0xDA, 0xC4, 0xD4, 0xC5, 0x00, 0x00, 0xE7, 0xFA, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_66[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0xCD, 0xE0, 0xE3, 0xB0, /* 0x04-0x07 */ + 0x00, 0x00, 0xDB, 0xB2, 0xFB, 0xC4, 0x00, 0x00, /* 0x08-0x0B */ + 0xF3, 0xE3, 0x00, 0x00, 0xD9, 0xA5, 0xFB, 0xE7, /* 0x0C-0x0F */ + 0xDD, 0xCB, 0xD0, 0xD4, 0x00, 0x00, 0xE6, 0xB6, /* 0x10-0x13 */ + 0xE0, 0xAE, 0xFD, 0xDA, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xDC, 0xB5, 0xE0, 0xF8, /* 0x1C-0x1F */ + 0xE7, 0xB1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0xF5, 0xF0, 0x00, 0x00, 0xD8, 0xDC, /* 0x24-0x27 */ + 0xED, 0xC6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0xE1, 0xB9, 0x00, 0x00, 0xE3, 0xC0, /* 0x2C-0x2F */ + 0xF9, 0xC0, 0xE9, 0xF0, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0xD9, 0xDB, 0x00, 0x00, 0xF3, 0xE4, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xDC, 0xB6, 0xE4, 0xE9, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0xF0, 0xC5, 0xE3, 0xC1, 0xFC, 0xCC, /* 0x40-0x43 */ + 0xFC, 0xCD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0xF2, 0xCB, 0x00, 0x00, 0xF2, 0xCC, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xCF, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0xF1, 0xDB, 0x00, 0x00, 0xFA, 0xD9, /* 0x58-0x5B */ + 0x00, 0x00, 0xF1, 0xB8, 0xFD, 0xF5, 0xE0, 0xF9, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0xE7, 0xFB, 0xFC, 0xB7, 0xFC, 0xE4, 0xFB, 0xC5, /* 0x64-0x67 */ + 0xE3, 0xE7, 0xD8, 0xB9, 0x00, 0x00, 0xF6, 0xF8, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0xDC, 0xC5, 0xCC, 0xD8, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xAF, /* 0x70-0x73 */ + 0xF4, 0xE7, 0x00, 0x00, 0xEF, 0xDC, 0xCF, 0xFC, /* 0x74-0x77 */ + 0xEF, 0xDD, 0x00, 0x00, 0xF2, 0xAA, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0xFD, 0xBE, 0x00, 0x00, 0x00, 0x00, 0xCA, 0xAC, /* 0x84-0x87 */ + 0xFD, 0xBB, 0xFD, 0xC7, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0xB2, 0x00, 0x00, /* 0x8C-0x8F */ + 0xEA, 0xD1, 0xDF, 0xF4, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0xD1, 0xEC, 0xE4, 0xDE, /* 0x94-0x97 */ + 0xE5, 0xC3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0xD9, 0xA6, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0xCD, 0xBC, 0x00, 0x00, 0xF3, 0xE5, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xED, 0xD5, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0xD9, 0xBA, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xED, 0xE7, 0xFB, 0xB5, /* 0xB0-0xB3 */ + 0xF8, 0xEC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xE0, 0xE7, 0x00, 0x00, 0xCC, 0xD9, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0xD4, 0xC6, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xE7, 0xA5, 0x00, 0x00, 0xD5, 0xF5, 0xD3, 0xBE, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xFC, 0xFB, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0xF2, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xDF, 0xF5, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0xE8, 0xF8, 0xF8, 0xED, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0xCE, 0xC7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0xFD, 0xF6, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0xE8, 0xD8, 0x00, 0x00, 0xCD, 0xD8, 0xE7, 0xD6, /* 0xF0-0xF3 */ + 0xCC, 0xDA, 0x00, 0x00, 0x00, 0x00, 0xCA, 0xE3, /* 0xF4-0xF7 */ + 0xDF, 0xF6, 0xF0, 0xC7, 0xF0, 0xC6, 0x00, 0x00, /* 0xF8-0xFB */ + 0xD8, 0xBA, 0x00, 0x00, 0xF1, 0xF4, 0xF4, 0xF0, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_67[512] = { + 0xF5, 0xCC, 0x00, 0x00, 0x00, 0x00, 0xFC, 0xE5, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0xEA, 0xC5, 0xEA, 0xF3, 0x00, 0x00, 0xDD, 0xDB, /* 0x08-0x0B */ + 0x00, 0x00, 0xDC, 0xD7, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0xDE, 0xFD, 0xF2, 0xF9, 0x00, 0x00, 0xD5, 0xC7, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD8, 0xD0, /* 0x18-0x1B */ + 0x00, 0x00, 0xF0, 0xC8, 0xD1, 0xA1, 0xD1, 0xA2, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0xD9, 0xD4, 0xD6, 0xE8, /* 0x24-0x27 */ + 0xD9, 0xCA, 0x00, 0x00, 0xDA, 0xB1, 0xD8, 0xC7, /* 0x28-0x2B */ + 0xDC, 0xE2, 0xF3, 0xCE, 0xF5, 0xF4, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0xF1, 0xB9, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0xDA, 0xD3, 0x00, 0x00, 0xF6, 0xEA, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xCF, 0xF5, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0xFD, 0xAE, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xCA, 0xD2, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0xDF, 0xB4, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0xD7, 0xDD, 0xFA, 0xBA, /* 0x4C-0x4F */ + 0xEE, 0xA7, 0xF5, 0xBD, 0x00, 0x00, 0xF8, 0xF5, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0xED, 0xE8, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0xD4, 0xE1, 0x00, 0x00, 0xD1, 0xA3, 0xE1, 0xD6, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0xF9, 0xF8, 0x00, 0x00, 0xDB, 0xCA, /* 0x6C-0x6F */ + 0xCB, 0xF9, 0xD4, 0xD4, 0x00, 0x00, 0xD9, 0xDC, /* 0x70-0x73 */ + 0x00, 0x00, 0xEE, 0xBE, 0x00, 0x00, 0xF7, 0xED, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD2, 0xEE, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xE6, 0xF7, 0xF9, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDD, 0xED, /* 0x84-0x87 */ + 0x00, 0x00, 0xE8, 0xDB, 0x00, 0x00, 0xDB, 0xB3, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD1, 0xF7, /* 0x8C-0x8F */ + 0xE0, 0xB0, 0x00, 0x00, 0x00, 0x00, 0xD4, 0xE2, /* 0x90-0x93 */ + 0x00, 0x00, 0xF6, 0xD7, 0x00, 0x00, 0xD7, 0xF9, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0xD8, 0xDD, 0x00, 0x00, /* 0x98-0x9B */ + 0xCD, 0xFD, 0xF2, 0xAB, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCD, 0xBD, /* 0xAC-0xAF */ + 0xF8, 0xC2, 0x00, 0x00, 0x00, 0x00, 0xF2, 0xAC, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0xCA, 0xAD, 0xCA, 0xAE, /* 0xB4-0xB7 */ + 0xCF, 0xAE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xC2, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xDC, 0xB7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDB, 0xDA, /* 0xCC-0xCF */ + 0xD9, 0xBB, 0xCA, 0xF3, 0xF6, 0xD3, 0xE6, 0xF8, /* 0xD0-0xD3 */ + 0xEA, 0xF5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xEA, 0xF6, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0xF6, 0xF9, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xCF, 0xAF, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0xCA, 0xD3, 0x00, 0x00, 0x00, 0x00, 0xCA, 0xAF, /* 0xEC-0xEF */ + 0xD2, 0xB0, 0xF1, 0xBA, 0x00, 0x00, 0xD7, 0xB3, /* 0xF0-0xF3 */ + 0xE3, 0xC3, 0xF3, 0xFD, 0xDE, 0xDA, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDE, 0xDB, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0xEF, 0xDE, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_68[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0xE3, 0xEE, 0xFB, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xDF, 0xF7, 0xD7, 0xCA, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0xCE, 0xE8, 0xDB, 0xDB, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xF1, 0xBB, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE9, 0xF1, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0xFA, 0xB7, 0xD0, 0xC6, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0xCC, 0xAB, 0xEE, 0xA8, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0xCB, 0xFA, 0xF9, 0xF9, 0xCC, 0xFD, 0xD3, 0xFE, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0xE4, 0xD0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0xF2, 0xEE, 0x00, 0x00, /* 0x4C-0x4F */ + 0xD4, 0xD5, 0xDF, 0xCD, 0x00, 0x00, 0xFC, 0xB8, /* 0x50-0x53 */ + 0xD1, 0xD0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0xF2, 0xCD, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0xF7, 0xD2, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCA, 0xD4, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xD5, 0xD9, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0xD8, 0xDE, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCD, 0xD9, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEE, 0xA9, /* 0x90-0x93 */ + 0xF6, 0xBC, 0x00, 0x00, 0x00, 0x00, 0xCC, 0xDB, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0xF0, 0xC9, 0x00, 0x00, 0xFC, 0xFC, /* 0x9C-0x9F */ + 0x00, 0x00, 0xE8, 0xC9, 0xF4, 0xFE, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0xFC, /* 0xA4-0xA7 */ + 0xD7, 0xDE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0xDE, 0xDC, 0x00, 0x00, 0xF0, 0xAC, /* 0xAC-0xAF */ + 0xCC, 0xFE, 0xCD, 0xE1, 0x00, 0x00, 0xE1, 0xBA, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xDB, 0xEF, 0xDA, 0xB2, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xD1, 0xA5, 0xDC, 0xB8, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xD8, 0xF6, 0x00, 0x00, 0xD1, 0xA4, /* 0xC8-0xCB */ + 0x00, 0x00, 0xCD, 0xE2, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xDC, 0xEA, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xF0, 0xF7, 0x00, 0x00, 0xF0, 0xCA, /* 0xD4-0xD7 */ + 0xD0, 0xBE, 0x00, 0x00, 0xDD, 0xDC, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD4, 0xD6, /* 0xDC-0xDF */ + 0xD3, 0xD6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xED, 0xD0, /* 0xE4-0xE7 */ + 0xCD, 0xA1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0xDF, 0xB5, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0xDF, 0xF8, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xD4, 0xA1, 0xCE, 0xB2, 0x00, 0x00, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_69[512] = { + 0xE8, 0xCA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0xEB, 0xF5, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0xE3, 0xD5, 0xF5, 0xD0, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0xF5, 0xA1, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD9, 0xA7, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xE5, 0xAB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0xE6, 0xCB, 0x00, 0x00, 0xF5, 0xF1, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xC5, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF9, 0xA3, /* 0x50-0x53 */ + 0xE0, 0xDB, 0xF6, 0xEB, 0x00, 0x00, 0xCB, 0xF1, /* 0x54-0x57 */ + 0x00, 0x00, 0xD9, 0xEA, 0xF5, 0xA2, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0xD7, 0xD1, 0x00, 0x00, /* 0x5C-0x5F */ + 0xD1, 0xF8, 0xEA, 0xF8, 0xEA, 0xF9, 0xDA, 0xB3, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0xEF, 0xDF, 0x00, 0x00, 0x00, 0x00, 0xF1, 0xEF, /* 0x68-0x6B */ + 0x00, 0x00, 0xE5, 0xF6, 0xEE, 0xBF, 0xE2, 0xE4, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0xD0, 0xBF, 0x00, 0x00, 0xFA, 0xAC, /* 0x74-0x77 */ + 0xF5, 0xD1, 0xE7, 0xB3, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0xE9, 0xBE, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF2, 0xCE, /* 0x98-0x9B */ + 0xDB, 0xB4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xFC, 0xCE, 0x00, 0x00, 0xDD, 0xEE, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0xB4, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xD7, 0xB4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF7, 0xB4, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0xCD, 0xBE, 0x00, 0x00, 0xDA, 0xE9, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCF, 0xB0, /* 0xC8-0xCB */ + 0xF7, 0xD9, 0xF3, 0xE6, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0xCE, 0xD9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xCE, 0xAA, 0x00, 0x00, 0xCB, 0xC8, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD0, 0xA7, /* 0xF8-0xFB */ + 0x00, 0x00, 0xF0, 0xCB, 0x00, 0x00, 0xD0, 0xC7, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_6A[512] = { + 0x00, 0x00, 0x00, 0x00, 0xE4, 0xC5, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xDB, 0xE0, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0xD5, 0xDA, 0x00, 0x00, 0xD7, 0xA7, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEE, 0xC0, /* 0x14-0x17 */ + 0x00, 0x00, 0xF8, 0xF6, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xF5, 0xD2, 0xED, 0xE9, /* 0x1C-0x1F */ + 0x00, 0x00, 0xD9, 0xBC, 0x00, 0x00, 0xE5, 0xC6, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0xF5, 0xA3, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0xDA, 0xD4, 0xE2, 0xA7, 0xFB, 0xFC, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0xF1, 0xDC, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0xCA, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0xE8, 0xFA, 0x00, 0x00, 0x00, 0x00, 0xCE, 0xE9, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xF8, 0xE2, 0xE5, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0xD0, 0xB9, 0xD4, 0xF2, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD1, 0xA6, /* 0x5C-0x5F */ + 0x00, 0x00, 0xDF, 0xCE, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFC, 0xF4, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xD3, 0xAA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0xCC, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0xEF, 0xE0, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0xE5, 0xE5, 0xD0, 0xD5, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDB, 0xFC, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0xFC, 0xE6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0xCB, 0xFE, 0xED, 0xEA, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDE, 0xB1, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF9, 0xE3, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0xD4, 0xA2, 0xCF, 0xF6, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD6, 0xD0, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xD5, 0xEA, 0xF1, 0xEE, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0xFA, 0xCB, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE5, 0xA1, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_6B[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0xD5, 0xB1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xCF, 0xED, 0x00, 0x00, /* 0x08-0x0B */ + 0xED, 0xEB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0xD5, 0xB2, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xD5, 0xBC, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0xFD, 0xE2, 0xF3, 0xAD, 0x00, 0x00, 0xFD, 0xDB, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xB0, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xD1, 0xA7, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0xFD, 0xE3, 0xCE, 0xB3, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xFD, 0xE4, 0xFA, 0xCE, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0xCA, 0xB0, 0x00, 0x00, 0xF7, 0xA7, 0x00, 0x00, /* 0x4C-0x4F */ + 0xCF, 0xB1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE6, 0xA2, /* 0x5C-0x5F */ + 0x00, 0x00, 0xFC, 0xB6, 0xF2, 0xAD, 0xEF, 0xE1, /* 0x60-0x63 */ + 0xF3, 0xAE, 0xDC, 0xC6, 0xD9, 0xEB, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0xE8, 0xE0, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xA8, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD5, 0xF6, /* 0x74-0x77 */ + 0xCF, 0xFD, 0x00, 0x00, 0x00, 0x00, 0xDE, 0xDD, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD9, 0xD1, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xEA, /* 0x80-0x83 */ + 0xF2, 0xCF, 0x00, 0x00, 0xF7, 0xBF, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0xE2, 0xE6, 0xE2, 0xA8, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xD6, 0x00, 0x00, /* 0x94-0x97 */ + 0xED, 0xD1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xF9, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0xD6, 0xB1, 0xDE, 0xB2, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xE8, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xD3, 0xAB, 0x00, 0x00, 0xEB, 0xDC, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0xDF, 0xAF, 0x00, 0x00, /* 0xB8-0xBB */ + 0xCA, 0xC3, 0x00, 0x00, 0x00, 0x00, 0xEE, 0xFC, /* 0xBC-0xBF */ + 0x00, 0x00, 0xFD, 0xC3, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xEB, 0xF6, 0xCF, 0xB2, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD9, 0xEC, /* 0xC8-0xCB */ + 0x00, 0x00, 0xD9, 0xBD, 0x00, 0x00, 0xD8, 0xDF, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xD4, 0xB8, 0xEB, 0xBE, /* 0xD0-0xD3 */ + 0xDD, 0xEF, 0x00, 0x00, 0xDD, 0xF0, 0xDD, 0xF1, /* 0xD4-0xD7 */ + 0xDD, 0xF2, 0x00, 0x00, 0x00, 0x00, 0xD9, 0xBE, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFB, 0xC6, /* 0xE8-0xEB */ + 0xCF, 0xB3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ +}; + +static const unsigned char u2c_6C[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0xEE, 0xFD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xAB, /* 0x0C-0x0F */ + 0x00, 0x00, 0xDA, 0xC5, 0x00, 0x00, 0xD8, 0xEC, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD1, 0xA8, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0xE2, 0xA9, 0x00, 0x00, 0x00, 0x00, 0xDE, 0xBC, /* 0x34-0x37 */ + 0xE7, 0xB5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xDB, 0xF0, 0x00, 0x00, /* 0x3C-0x3F */ + 0xEF, 0xE2, 0xF1, 0xF0, 0xCF, 0xB4, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0xDB, 0xF1, 0x00, 0x00, /* 0x4C-0x4F */ + 0xE0, 0xB1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0xDF, 0xA5, 0x00, 0x00, 0xF9, 0xD2, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0xFD, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0xE6, 0xA3, 0xFB, 0xF1, 0xCB, 0xB0, /* 0x5C-0x5F */ + 0xF2, 0xAE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0xCD, 0xE7, 0x00, 0x00, 0xE8, 0xDC, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0xE7, 0xD7, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0xF7, 0xC0, 0x00, 0x00, 0xD0, 0xE3, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0xDA, 0xA1, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0xCC, 0xBD, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0xD1, 0xA9, 0xDD, 0xCC, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xE3, 0xFE, 0xD1, 0xAA, 0xE8, 0xAA, /* 0x80-0x83 */ + 0x00, 0x00, 0xEA, 0xB6, 0xF9, 0xFA, 0xE6, 0xCC, /* 0x84-0x87 */ + 0xF6, 0xD8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0xD4, 0xC7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0xD9, 0xCB, 0x00, 0x00, 0xD9, 0xD2, 0xD3, 0xCB, /* 0x90-0x93 */ + 0xD8, 0xF7, 0xDA, 0xA9, 0xF5, 0xF8, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0xDE, 0xDE, 0xF2, 0xAF, 0xF8, 0xA9, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD8, 0xC8, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0xEE, 0xC1, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF9, 0xC1, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xDD, 0xF3, 0xEA, 0xFA, 0x00, 0x00, 0xF6, 0xBD, /* 0xB8-0xBB */ + 0xE1, 0xBB, 0xCD, 0xBF, 0xF4, 0xD4, 0xE6, 0xCD, /* 0xBC-0xBF */ + 0x00, 0x00, 0xFC, 0xCF, 0xFB, 0xA2, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xE0, 0xDC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xF4, 0xBB, 0xDA, 0xD5, 0x00, 0x00, /* 0xC8-0xCB */ + 0xF9, 0xB2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFB, 0xF2, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xDB, 0xF6, 0x00, 0x00, 0xDE, 0xDF, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDB, 0xF2, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0xF8, 0xDC, 0xF7, 0xEE, 0xEB, 0xE8, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xD2, 0xFA, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xF1, 0xBC, 0x00, 0x00, 0x00, 0x00, 0xFA, 0xDA, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0xDA, 0xEA, 0xDA, 0xC6, /* 0xEC-0xEF */ + 0xF7, 0xC1, 0x00, 0x00, 0x00, 0x00, 0xE7, 0xB6, /* 0xF0-0xF3 */ +}; + +static const unsigned char u2c_6D[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE5, 0xC7, /* 0x08-0x0B */ + 0xD6, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0xDC, 0xC7, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE1, 0xA9, /* 0x14-0x17 */ + 0x00, 0x00, 0xE2, 0xAA, 0x00, 0x00, 0xD5, 0xA6, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xD4, 0xD7, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0xF2, 0xD0, 0x00, 0x00, 0xEA, 0xFB, /* 0x24-0x27 */ + 0x00, 0x00, 0xE0, 0xDD, 0xFB, 0xF3, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0xF1, 0xBD, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0xE2, 0xE7, 0xFD, 0xD7, 0x00, 0x00, /* 0x34-0x37 */ + 0xCE, 0xC8, 0xEA, 0xB7, 0x00, 0x00, 0xFC, 0xC0, /* 0x38-0x3B */ + 0x00, 0x00, 0xFD, 0xE7, 0xF7, 0xEF, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0xD7, 0xB5, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0xEF, 0xBA, 0xF1, 0xDD, 0x00, 0x00, /* 0x58-0x5B */ + 0xDE, 0xB3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE8, 0xCB, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0xF8, 0xDD, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0xFB, 0xC7, 0xD5, 0xC8, 0x00, 0x00, /* 0x68-0x6B */ + 0xD7, 0xDF, 0x00, 0x00, 0xDD, 0xA9, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0xE9, 0xB1, 0x00, 0x00, 0x00, 0x00, 0xFA, 0xAD, /* 0x74-0x77 */ + 0xF6, 0xD9, 0xFA, 0xF4, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF8, 0xAA, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0xE6, 0xEE, 0x00, 0x00, 0xCC, 0xDC, /* 0x84-0x87 */ + 0xE1, 0xBC, 0xE0, 0xEF, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0xE9, 0xBF, 0xFC, 0xFD, 0xE6, 0xCE, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xE1, 0xD7, 0x00, 0x00, 0xE6, 0xCF, /* 0x90-0x93 */ + 0x00, 0x00, 0xF4, 0xF1, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xF3, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0xFB, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xF9, 0xE4, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0xEF, 0xE3, 0x00, 0x00, 0x00, 0x00, 0xCF, 0xEE, /* 0xC0-0xC3 */ + 0xF6, 0xBE, 0xE0, 0xB2, 0xFC, 0xFE, 0xD1, 0xAB, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD7, 0xFA, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFB, 0xC8, /* 0xCC-0xCF */ + 0x00, 0x00, 0xE2, 0xD7, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xD4, 0xA3, 0xF0, 0xF8, 0xD7, 0xA8, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xE7, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0xD3, 0xBF, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xEF, 0xE4, 0x00, 0x00, 0xD7, 0xC5, 0xEB, 0xE2, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0xFC, 0xE7, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0xE4, 0xA2, 0x00, 0x00, 0xE2, 0xE8, /* 0xF0-0xF3 */ + 0x00, 0x00, 0xE6, 0xD0, 0x00, 0x00, 0xFB, 0xE8, /* 0xF4-0xF7 */ + 0xF4, 0xE8, 0xE5, 0xF4, 0xF4, 0xBC, 0xF4, 0xD5, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_6E[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDF, 0xB6, /* 0x14-0x17 */ + 0x00, 0x00, 0xFC, 0xB9, 0xEE, 0xC2, 0xCA, 0xF5, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEF, 0xE5, /* 0x1C-0x1F */ + 0xCB, 0xE2, 0xD4, 0xA4, 0x00, 0x00, 0xDE, 0xE0, /* 0x20-0x23 */ + 0xDA, 0xFD, 0xE4, 0xC6, 0xE8, 0xBE, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xDE, /* 0x28-0x2B */ + 0xF6, 0xB4, 0xEA, 0xD2, 0x00, 0x00, 0xF9, 0xFB, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xC2, 0x00, 0x00, /* 0x30-0x33 */ + 0xCA, 0xE4, 0x00, 0x00, 0xE7, 0xB7, 0x00, 0x00, /* 0x34-0x37 */ + 0xEA, 0xFD, 0x00, 0x00, 0xD9, 0xDD, 0x00, 0x00, /* 0x38-0x3B */ + 0xDA, 0xB4, 0xEE, 0xAA, 0xFB, 0xE9, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDB, 0xCB, /* 0x40-0x43 */ + 0xDA, 0xB5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0xF1, 0xBE, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0xD3, 0xAC, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0xFB, 0xC9, 0x00, 0x00, /* 0x54-0x57 */ + 0xDF, 0xCF, 0x00, 0x00, 0x00, 0x00, 0xD3, 0xC0, /* 0x58-0x5B */ + 0xE3, 0xD7, 0x00, 0x00, 0xEF, 0xE6, 0xFC, 0xD0, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE9, 0xC0, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF5, 0xD3, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0xEC, 0xDC, 0xF7, 0xB7, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0xEA, 0xB8, 0xD1, 0xF9, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0xDC, 0xC8, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0xEA, 0xB9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0xF1, 0xDE, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0xD7, 0xB6, 0xCF, 0xB5, 0x00, 0x00, 0xD9, 0xA8, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0xEC, 0xEE, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xDD, 0xAA, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0xCD, 0xA2, 0xE8, 0xAE, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE1, 0xBD, /* 0xAC-0xAF */ + 0x00, 0x00, 0xF2, 0xD1, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xC1, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0xD2, 0xFC, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0xDB, 0xB5, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xF3, 0xE7, 0xD8, 0xFE, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xFC, 0xD1, 0x00, 0x00, 0xED, 0xB2, /* 0xC8-0xCB */ + 0xF4, 0xAF, 0x00, 0x00, 0xFB, 0xA3, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0xFC, 0xC1, 0x00, 0x00, 0xEE, 0xAB, /* 0xD0-0xD3 */ + 0xD4, 0xA5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xF2, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xEE, 0xD9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xFB, 0xCA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0xCD, 0xE3, 0xD8, 0xBB, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_6F[512] = { + 0x00, 0x00, 0xE5, 0xDB, 0xF8, 0xF7, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0xF6, 0xD4, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD7, 0xA9, /* 0x0C-0x0F */ + 0x00, 0x00, 0xCB, 0xC9, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0xE6, 0xD1, 0xF0, 0xCC, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0xD8, 0xAE, 0x00, 0x00, 0xF9, 0xD3, 0xD5, 0xFE, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD8, 0xBC, /* 0x28-0x2B */ + 0xF2, 0xB0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0xE2, 0xAB, 0xF3, 0xE8, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0xEF, 0xC2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xED, 0xEC, /* 0x3C-0x3F */ + 0x00, 0x00, 0xE7, 0xB8, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0xDA, 0xFE, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0xCC, 0xBE, 0x00, 0x00, 0x00, 0x00, 0xF2, 0xFC, /* 0x54-0x57 */ + 0xDA, 0xEB, 0x00, 0x00, 0xE2, 0xD8, 0xED, 0xD6, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0xD6, 0xD1, 0xE0, 0xB3, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0xFC, 0xD2, 0x00, 0x00, /* 0x60-0x63 */ + 0xEB, 0xC8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0xD3, 0xC1, 0xF0, 0xCD, 0x00, 0x00, /* 0x6C-0x6F */ + 0xCF, 0xF7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0xED, 0xD2, 0x00, 0x00, /* 0x78-0x7B */ + 0xD4, 0xD8, 0xDC, 0xC9, 0xD7, 0xF1, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xDF, 0xBB, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0xF3, 0xA5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0xF4, 0xCD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0xF1, 0xBF, 0xF8, 0xB1, 0x00, 0x00, /* 0x8C-0x8F */ + 0xE9, 0xFA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0xFB, 0xCB, 0x00, 0x00, 0x00, 0x00, 0xCA, 0xD5, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF9, 0xD4, /* 0xA0-0xA3 */ + 0xF7, 0xCA, 0x00, 0x00, 0x00, 0x00, 0xD6, 0xC8, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0xFC, 0xE8, 0xF3, 0xBD, /* 0xAC-0xAF */ + 0x00, 0x00, 0xEE, 0xFE, 0x00, 0x00, 0xE7, 0xFE, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xD3, 0xC2, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0xD3, 0xB6, 0x00, 0x00, /* 0xBC-0xBF */ + 0xCC, 0xAD, 0xF6, 0xFA, 0xD6, 0xB2, 0xD2, 0xD8, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0xD8, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xE3, 0xA5, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0xB9, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xAD, /* 0xDC-0xDF */ + 0xFB, 0xCC, 0xEB, 0xA1, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0xD4, 0xA6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xFB, 0xCD, 0x00, 0x00, 0xD5, 0xBD, /* 0xE8-0xEB */ + 0xF1, 0xDF, 0x00, 0x00, 0x00, 0x00, 0xF6, 0xFB, /* 0xEC-0xEF */ + 0x00, 0x00, 0xDE, 0xB4, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0xD5, 0xEB, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_70[512] = { + 0x00, 0x00, 0xE5, 0xC8, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0xFB, 0xA4, 0xD4, 0xB9, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xDE, 0xE1, 0x00, 0x00, 0xE4, 0xA3, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD7, 0xB7, /* 0x0C-0x0F */ + 0x00, 0x00, 0xF8, 0xEE, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0xDE, 0xB5, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0xD6, 0xD2, 0x00, 0x00, 0xF9, 0xD5, 0xE7, 0xBA, /* 0x18-0x1B */ + 0xEB, 0xD5, 0xD5, 0xF7, 0xEF, 0xE7, 0xE1, 0xBE, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFA, 0xAE, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD6, 0xE9, /* 0x24-0x27 */ + 0xD6, 0xEE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0xBB, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEC, 0xCB, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xD5, 0xB3, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0xCE, 0xB4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0xFB, 0xA5, 0xE1, 0xEE, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0xF7, 0xA8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0xFB, 0xCE, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD8, 0xBD, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFB, 0xFD, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0xFC, 0xE9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0xCF, 0xB6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0xED, 0xC7, 0xEE, 0xAC, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0xCC, 0xDD, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0xF6, 0xA7, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0xFA, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0xF5, 0xA4, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0xFD, 0xDC, 0xED, 0xB3, 0xCE, 0xC9, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0xEF, 0xE8, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xE1, 0xBF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFA, 0xDB, /* 0xA8-0xAB */ + 0xCB, 0xE3, 0xF7, 0xA9, 0x00, 0x00, 0xFB, 0xA6, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDC, 0xB9, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF1, 0xC0, /* 0xB4-0xB7 */ + 0xED, 0xC8, 0xEF, 0xC3, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0xD6, 0xAD, 0x00, 0x00, 0x00, 0x00, 0xFD, 0xCE, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE8, 0xA1, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xFB, 0xF4, 0xD5, 0xA7, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0xF1, 0xF6, 0x00, 0x00, 0xE6, 0xD3, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0xCC, 0xDE, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xF8, 0xB2, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0xDC, 0xEB, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_71[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0xFD, 0xB6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xE5, 0xEA, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0xF1, 0xE0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0xDB, 0xCC, 0xDD, 0xCD, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xD4, 0xC8, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0xD9, 0xED, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0xF5, 0xA5, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xE6, 0xFB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0xD4, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFD, 0xC8, /* 0x44-0x47 */ + 0x00, 0x00, 0xD6, 0xA1, 0xFD, 0xBF, 0x00, 0x00, /* 0x48-0x4B */ + 0xFC, 0xD3, 0x00, 0x00, 0xEF, 0xA1, 0x00, 0x00, /* 0x4C-0x4F */ + 0xE7, 0xBC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0xD1, 0xEE, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0xE6, 0xD5, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0xE9, 0xF2, 0x00, 0x00, 0xDF, 0xB0, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0xD8, 0xE0, 0xFC, 0xBA, 0xFD, 0xAF, 0xF0, 0xCE, /* 0x64-0x67 */ + 0x00, 0x00, 0xDB, 0xE1, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0xE5, 0xC9, 0x00, 0x00, 0xED, 0xB4, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0xE0, 0xC3, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0xE3, 0xD8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0xE9, 0xFB, 0xEA, 0xA8, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFD, 0xB7, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0xFB, 0xA7, 0x00, 0x00, /* 0x90-0x93 */ + 0xE9, 0xC2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0xFD, 0xF7, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xD9, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0xDC, 0xEC, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0xE8, 0xA2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0xE6, 0xF0, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xFD, 0xF8, 0xFD, 0xF9, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0xF6, 0xBF, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0xE7, 0xA7, 0x00, 0x00, 0xE6, 0xD7, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0xD4, 0xF3, 0xD4, 0xC9, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0xD6, 0xFA, 0x00, 0x00, /* 0xCC-0xCF */ + 0xD7, 0xF2, 0x00, 0x00, 0xE1, 0xC0, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xDB, 0xE2, 0xE6, 0xD8, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0xBD, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xF0, 0xCF, 0xF3, 0xBE, 0xE2, 0xAC, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0xF5, 0xB7, 0xE0, 0xF0, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFD, 0xB8, /* 0xF8-0xFB */ + 0xE3, 0xE8, 0x00, 0x00, 0xD4, 0xA7, 0xE8, 0xFC, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_72[512] = { + 0xFA, 0xD2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0xF8, 0xEF, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0xD6, 0xD3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD5, 0xB4, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xF0, 0xD0, 0x00, 0x00, /* 0x28-0x2B */ + 0xF7, 0xF0, 0xEE, 0xB3, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xEA, 0xBA, 0x00, 0x00, 0xEA, 0xD3, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0xED, 0xC9, 0xDD, 0xAB, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xAC, 0xFD, 0xA1, /* 0x38-0x3B */ + 0x00, 0x00, 0xDF, 0xD0, 0xEC, 0xB3, 0x00, 0x00, /* 0x3C-0x3F */ + 0xDF, 0xD1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xED, 0xED, 0xF8, 0xB8, /* 0x44-0x47 */ + 0xF7, 0xFA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0xF8, 0xAB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0xF4, 0xE0, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0xD4, 0xBA, 0xE4, 0xB3, 0x00, 0x00, 0xE9, 0xDA, /* 0x58-0x5B */ + 0x00, 0x00, 0xDE, 0xB6, 0x00, 0x00, 0xD9, 0xBF, /* 0x5C-0x5F */ + 0x00, 0x00, 0xD9, 0xC0, 0xD6, 0xEF, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD9, 0xCC, /* 0x64-0x67 */ + 0x00, 0x00, 0xDA, 0xAA, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0xDF, 0xE5, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0xF7, 0xE5, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0xCC, 0xB2, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xDF, 0xF9, 0xD7, 0xE0, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0xD4, 0xBB, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFD, 0xFA, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0xCC, 0xB3, 0x00, 0x00, 0x00, 0x00, 0xDB, 0xF3, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0xDF, 0xD2, 0x00, 0x00, 0xCE, 0xCA, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xEE, 0xDA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0xE4, 0x00, 0x00, /* 0xCC-0xCF */ + 0xFB, 0xCF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCF, 0xB7, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xEE, 0xC3, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0xCE, 0xEA, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xE2, 0xAD, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xD7, 0xE1, 0xFA, 0xF5, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0xD5, 0xC9, 0xF8, 0xAC, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_73[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0xD9, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xF3, 0xE9, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD8, 0xED, /* 0x18-0x1B */ + 0xE3, 0xC4, 0xF0, 0xF1, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0xE8, 0xE5, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0xE0, 0xFA, 0xEE, 0xC4, 0xD9, 0xDE, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0xEB, 0xA2, 0xEB, 0xA3, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xFC, 0xC2, 0xEA, 0xBB, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0xE8, 0xAB, 0xDE, 0xE2, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0xED, 0xEF, 0x00, 0x00, 0xE8, 0xA3, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCF, 0xF1, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0xD4, 0xBC, 0x00, 0x00, 0xFC, 0xEA, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0xE7, 0xBE, 0x00, 0x00, 0xFC, 0xF2, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0xD6, 0xB4, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0xE2, 0xAE, 0x00, 0x00, 0xD3, 0xB7, 0xFA, 0xCC, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0xFA, 0xDC, 0x00, 0x00, 0xED, 0xB5, 0xE1, 0xE3, /* 0x84-0x87 */ + 0x00, 0x00, 0xE8, 0xAC, 0x00, 0x00, 0xE8, 0xDD, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0xEF, 0xE9, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0xF4, 0xBD, 0x00, 0x00, 0xCF, 0xB8, 0xE9, 0xDB, /* 0x94-0x97 */ + 0xD1, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDA, 0xC7, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEB, 0xC9, /* 0xA4-0xA7 */ + 0x00, 0x00, 0xE8, 0xCC, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0xDE, 0xB7, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xD6, 0xBC, 0xD3, 0xE5, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xFA, 0xDD, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0xDA, 0xD6, 0x00, 0x00, 0xCA, 0xB1, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xDA, 0xC8, 0xDF, 0xA6, 0x00, 0x00, /* 0xC8-0xCB */ + 0xF9, 0xB3, 0xF2, 0xD2, 0x00, 0x00, 0xCA, 0xC4, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0xCE, 0xCB, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xCD, 0xF5, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0xFD, 0xB0, 0xD5, 0xA8, 0x00, 0x00, /* 0xDC-0xDF */ + 0xF1, 0xC1, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xE9, /* 0xE0-0xE3 */ + 0xDC, 0xCA, 0xEC, 0xB4, 0xFA, 0xC0, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xFB, 0xA8, 0xD0, 0xA8, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0xDA, 0xEC, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD9, 0xEE, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xE0, 0xFB, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0xEF, 0xEA, 0xFA, 0xDE, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_74[512] = { + 0x00, 0x00, 0xE0, 0xC4, 0x00, 0x00, 0xCF, 0xB9, /* 0x00-0x03 */ + 0x00, 0x00, 0xD5, 0xCA, 0xD7, 0xE2, 0xE2, 0xAF, /* 0x04-0x07 */ + 0x00, 0x00, 0xD7, 0xB8, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE8, 0xCD, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF6, 0xDA, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0xEF, 0xA2, 0xE2, 0xDA, 0xF6, 0xFC, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0xFB, 0xD0, 0xD1, 0xAD, 0x00, 0x00, /* 0x24-0x27 */ + 0xCD, 0xE4, 0x00, 0x00, 0xD1, 0xAE, 0xDC, 0xED, /* 0x28-0x2B */ + 0xE8, 0xCE, 0x00, 0x00, 0xF0, 0xF9, 0xCE, 0xB5, /* 0x2C-0x2F */ + 0xE6, 0xFC, 0x00, 0x00, 0x00, 0x00, 0xD7, 0xFB, /* 0x30-0x33 */ + 0xD0, 0xD6, 0xDD, 0xF5, 0xF7, 0xF1, 0x00, 0x00, /* 0x34-0x37 */ + 0xF6, 0xFD, 0x00, 0x00, 0xDB, 0xF7, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFB, 0xEA, /* 0x3C-0x3F */ + 0xE9, 0xDC, 0xD9, 0xC1, 0x00, 0x00, 0xF5, 0xF2, /* 0x40-0x43 */ + 0xE0, 0xC5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEA, 0xD4, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0xF9, 0xC2, 0x00, 0x00, 0xEA, 0xBC, /* 0x54-0x57 */ + 0x00, 0x00, 0xD2, 0xC5, 0xFB, 0xD1, 0xE7, 0xC0, /* 0x58-0x5B */ + 0xEB, 0xA5, 0x00, 0x00, 0xDF, 0xFA, 0xE3, 0xA2, /* 0x5C-0x5F */ + 0xD7, 0xB9, 0x00, 0x00, 0xE9, 0xC3, 0x00, 0x00, /* 0x60-0x63 */ + 0xE8, 0xFD, 0xE8, 0xAF, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0xF2, 0xD3, 0xFB, 0xA9, 0xD8, 0xA5, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD5, 0xCB, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0xD0, 0xC8, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0xD1, 0xAF, 0xD7, 0xE3, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xC6, /* 0x84-0x87 */ + 0x00, 0x00, 0xD6, 0xA2, 0x00, 0x00, 0xED, 0xF0, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0xD7, 0xF3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0xFC, 0xD4, 0x00, 0x00, 0xDA, 0xD7, 0xCC, 0xDF, /* 0x9C-0x9F */ + 0x00, 0x00, 0xF2, 0xD4, 0x00, 0x00, 0xD1, 0xB0, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xCC, 0xE0, 0x00, 0x00, 0xDB, 0xFD, /* 0xA4-0xA7 */ + 0xF3, 0xBF, 0x00, 0x00, 0xF0, 0xD1, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0xFC, 0xBB, 0x00, 0x00, 0xE2, 0xB0, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xE6, 0xA5, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xE2, 0xDB, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0xDF, 0xDE, 0x00, 0x00, 0xE0, 0xC7, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0xF2, 0xEF, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0xCC, 0xE1, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD6, 0xEA, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xE7, 0xC2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xCE, 0xB6, 0x00, 0x00, 0xF3, 0xC0, 0x00, 0x00, /* 0xD8-0xDB */ + 0xCD, 0xFE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0xFB, 0xD2, 0x00, 0x00, 0xF8, 0xF8, 0xF7, 0xFB, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0xE8, 0xBF, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0xE8, 0xB7, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xED, 0xB6, /* 0xF4-0xF7 */ +}; + +static const unsigned char u2c_75[512] = { + 0x00, 0x00, 0xDC, 0xBA, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0xCC, 0xB4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0xF1, 0xF7, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0xE8, 0xB8, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0xCA, 0xF6, 0x00, 0x00, 0xE4, 0xA4, 0xF4, 0xD6, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDF, 0xE6, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDF, 0xA7, /* 0x20-0x23 */ + 0x00, 0x00, 0xDF, 0xE7, 0xE1, 0xC1, 0x00, 0x00, /* 0x24-0x27 */ + 0xE9, 0xC4, 0x00, 0x00, 0x00, 0x00, 0xDC, 0xCB, /* 0x28-0x2B */ + 0xE9, 0xC5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xEF, 0xA3, 0xEB, 0xA6, 0xCB, 0xA3, 0xE3, 0xE9, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD1, 0xFB, /* 0x34-0x37 */ + 0xEF, 0xA4, 0x00, 0x00, 0xEF, 0xEB, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD0, 0xB4, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0xCD, 0xA3, 0x00, 0x00, 0x00, 0x00, 0xE8, 0xE6, /* 0x4C-0x4F */ + 0x00, 0x00, 0xEF, 0xA5, 0x00, 0x00, 0xD3, 0xCC, /* 0x50-0x53 */ + 0xDA, 0xED, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0xD7, 0xBA, 0x00, 0x00, 0xF2, 0xD5, /* 0x58-0x5B */ + 0xF5, 0xE5, 0xD9, 0xEF, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0xF9, 0xB4, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0xD5, 0xD4, 0xFD, 0xCF, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0xDB, 0xE3, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF1, 0xE1, /* 0x6C-0x6F */ + 0xEC, 0xB6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0xFB, 0xFE, 0xD3, 0xD7, 0x00, 0x00, /* 0x74-0x77 */ + 0xD1, 0xB1, 0x00, 0x00, 0xCB, 0xB1, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD1, 0xB2, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0xCB, 0xB2, 0xF1, 0xC2, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0xF4, 0xE1, 0xF9, 0xB5, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xC3, 0xE1, 0xC2, /* 0x8C-0x8F */ + 0x00, 0x00, 0xEB, 0xF7, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0xDF, 0xA8, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xCB, 0xCA, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE6, 0xB9, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0xF8, 0xDE, 0xF9, 0xAA, 0xCA, 0xF7, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xED, 0xB7, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xD3, 0xB8, 0xF2, 0xD6, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0xD4, 0xD9, 0xEE, 0xC5, 0xF2, 0xF0, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0xCA, 0xB2, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xDC, 0xBB, 0x00, 0x00, 0xF1, 0xF8, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0xEC, 0xB7, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xCA, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xF6, 0xC0, 0xFD, 0xDD, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xD4, 0xE3, 0xCC, 0xE2, 0x00, 0x00, 0xF7, 0xD4, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0xD7, 0xE5, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0xD3, 0xC3, 0x00, 0x00, 0xD8, 0xA6, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xF6, 0xC1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0xDD, 0xF6, 0x00, 0x00, /* 0xF8-0xFB */ + 0xCD, 0xC0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_76[512] = { + 0xE5, 0xDC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0xE5, 0xCB, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0xE1, 0xC4, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE8, 0xB0, /* 0x1C-0x1F */ + 0xF4, 0xB0, 0xF3, 0xEA, 0xDA, 0xEE, 0x00, 0x00, /* 0x20-0x23 */ + 0xD7, 0xBB, 0x00, 0x00, 0xE2, 0xB1, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD7, 0xAA, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0xD6, 0xFB, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0xE4, 0xDF, 0x00, 0x00, 0xCA, 0xD6, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0xEB, 0xA8, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0xDB, 0xFE, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0xF6, 0xC2, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0xEF, 0xBB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0xD4, 0xFD, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0xE0, 0xC8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0xE8, 0xB9, 0x00, 0x00, 0xEF, 0xA6, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0xCD, 0xA4, 0x00, 0x00, 0x00, 0x00, 0xD4, 0xF4, /* 0x78-0x7B */ + 0xDB, 0xA1, 0xDB, 0xDC, 0xDB, 0xDD, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0xEE, 0xDC, 0x00, 0x00, 0xCB, 0xCB, 0xFC, 0xD5, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0xCE, 0xEB, 0x00, 0x00, /* 0x8C-0x8F */ + 0xCD, 0xC1, 0x00, 0x00, 0x00, 0x00, 0xFB, 0xD3, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0xF9, 0xAB, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0xF5, 0xD4, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD9, 0xA9, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xDD, 0xDB, 0xCD, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0xDD, 0xCE, 0x00, 0x00, /* 0xC4-0xC7 */ + 0xE7, 0xC3, 0x00, 0x00, 0xEC, 0xCC, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xF9, 0xEC, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0xCB, 0xCC, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xFC, /* 0xD8-0xDB */ + 0xD4, 0xA8, 0x00, 0x00, 0xED, 0xD3, 0xD8, 0xEF, /* 0xDC-0xDF */ + 0x00, 0x00, 0xF2, 0xD7, 0x00, 0x00, 0xCA, 0xF8, /* 0xE0-0xE3 */ + 0xDA, 0xEF, 0x00, 0x00, 0x00, 0x00, 0xD6, 0xD4, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0xD9, 0xCD, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0xD8, 0xEE, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xF2, 0xC1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xDF, 0xD3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0xDA, 0xF0, 0x00, 0x00, 0xE2, 0xEA, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_77[512] = { + 0x00, 0x00, 0xE0, 0xFD, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0xD8, 0xF8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0xF7, 0xAF, 0xDA, 0xB6, 0x00, 0x00, 0xCA, 0xD7, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xF2, 0xD8, 0x00, 0x00, /* 0x1C-0x1F */ + 0xD8, 0xF9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0xFA, 0xDF, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCF, 0xEF, /* 0x34-0x37 */ + 0xD9, 0xC2, 0x00, 0x00, 0xF0, 0xD2, 0x00, 0x00, /* 0x38-0x3B */ + 0xE4, 0xD1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0xF3, 0xB7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0xFA, 0xE0, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEF, 0xEC, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0xE2, 0xB2, 0x00, 0x00, 0xD4, 0xBD, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0xD9, 0xCE, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xE2, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0xD4, 0xA9, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0xCD, 0xC2, 0xE7, 0xDA, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF2, 0xD9, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xD9, 0xAA, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0xD8, 0xBE, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xDC, 0xAD, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0xE2, 0xEB, 0xD6, 0xFC, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0xCA, 0xF9, 0x00, 0x00, 0x00, 0x00, 0xD4, 0xDA, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xD7, /* 0xB8-0xBB */ + 0xCC, 0xA1, 0x00, 0x00, 0x00, 0x00, 0xCF, 0xBA, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF5, 0xB8, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD9, 0xC3, /* 0xD8-0xDB */ + 0xD0, 0xE8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xC5, 0xEB, 0xF8, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xF2, 0xB1, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xCF, 0xBB, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0xD3, 0xAD, 0xE8, 0xE1, 0xCE, 0xEC, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xB4, /* 0xF0-0xF3 */ +}; + +static const unsigned char u2c_78[512] = { + 0x00, 0x00, 0x00, 0x00, 0xDE, 0xE3, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0xDD, 0xF7, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0xF2, 0xB2, 0xF3, 0xF6, 0xF6, 0xDB, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0xD7, 0xFE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0xF8, 0xDF, 0x00, 0x00, /* 0x30-0x33 */ + 0xF7, 0xF2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0xD0, 0xA9, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE6, 0xDA, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0xF5, 0xA6, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD7, 0xBC, /* 0x68-0x6B */ + 0xCC, 0xE3, 0x00, 0x00, 0x00, 0x00, 0xE6, 0xDB, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0xDD, 0xDD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xD1, 0xB3, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEF, 0xED, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0xD6, 0xDE, 0xE4, 0xF4, 0xE1, 0xEF, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xDD, 0xF8, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE8, 0xCF, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCA, 0xE5, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDC, 0xA1, /* 0xA4-0xA7 */ + 0x00, 0x00, 0xE0, 0xB5, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0xFC, 0xAC, 0xFC, 0xAD, /* 0xB8-0xBB */ + 0xD8, 0xA7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0xED, 0xB8, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xDB, 0xB6, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0xD6, 0xF0, 0xF3, 0xAF, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0xCD, 0xA5, 0x00, 0x00, /* 0xCC-0xCF */ + 0xDA, 0xF1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xD8, 0xA8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0xCC, 0xE4, 0x00, 0x00, 0x00, 0x00, 0xD1, 0xB4, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0xCA, 0xD8, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDA, 0xF2, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_79[512] = { + 0x00, 0x00, 0xF5, 0xA7, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0xF5, 0xA8, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0xA6, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xD5, 0xEC, 0xD5, 0xF8, /* 0x28-0x2B */ + 0xDA, 0xF3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xC6, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xDE, 0xE4, 0x00, 0x00, /* 0x3C-0x3F */ + 0xDE, 0xE5, 0xD1, 0xB5, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD1, 0xB6, /* 0x44-0x47 */ + 0xD1, 0xB7, 0xF2, 0xB3, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0xE9, 0xDE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0xF0, 0xD3, 0xF2, 0xB4, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0xF0, 0xD4, 0xCB, 0xE4, /* 0x58-0x5B */ + 0xFB, 0xD4, 0xF5, 0xE6, 0xE3, 0xEA, 0x00, 0x00, /* 0x5C-0x5F */ + 0xDE, 0xE6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0xDF, 0xD4, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0xF8, 0xF9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0xF0, 0xAE, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0xD1, 0xB8, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD6, 0xDF, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xD0, 0xD7, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0xFC, 0xA1, 0xEF, 0xEE, 0xDC, 0xD8, /* 0x8C-0x8F */ + 0x00, 0x00, 0xE9, 0xDF, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xDD, 0xFD, 0xFB, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xC9, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0xD6, 0xC9, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0xD4, 0xAA, 0x00, 0x00, 0xE5, 0xCC, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xE9, 0xE0, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0xD0, 0xD8, 0xFC, 0xA2, 0xD4, 0xBE, /* 0xBC-0xBF */ + 0xE2, 0xB3, 0xDE, 0xE7, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xDC, 0xBC, 0xD2, 0xB6, 0xF5, 0xD5, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0xCE, 0xA1, 0xF5, 0xA9, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xDD, 0xF9, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xDD, 0xFA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xD5, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0xF6, 0xDF, 0x00, 0x00, 0xF2, 0xDA, 0xE4, 0xEB, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xF2, 0xF1, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEC, 0xB9, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_7A[512] = { + 0xFD, 0xFC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0xE1, 0xAA, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0xCA, 0xD9, 0x00, 0x00, 0x00, 0x00, 0xEF, 0xEF, /* 0x08-0x0B */ + 0x00, 0x00, 0xF5, 0xAA, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0xEC, 0xF9, 0x00, 0x00, 0x00, 0x00, 0xF8, 0xAD, /* 0x14-0x17 */ + 0x00, 0x00, 0xF2, 0xC2, 0xF6, 0xC3, 0x00, 0x00, /* 0x18-0x1B */ + 0xD7, 0xD2, 0x00, 0x00, 0x00, 0x00, 0xF9, 0xA2, /* 0x1C-0x1F */ + 0xF0, 0xD6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0xF0, 0xFA, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0xF6, 0xE0, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xF3, 0xF2, 0xC3, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD4, 0xAB, /* 0x38-0x3B */ + 0xCA, 0xB3, 0xCD, 0xA6, 0x00, 0x00, 0xCD, 0xC3, /* 0x3C-0x3F */ + 0xCD, 0xDA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xD9, 0xCF, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0xF6, 0xC4, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0xEE, 0xDD, 0xE7, 0xC4, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xB4, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0xDF, 0xE2, 0xE7, 0xDB, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0xE8, 0xB1, 0x00, 0x00, 0xFC, 0xAE, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0xE5, 0xCD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0xFA, 0xEB, 0x00, 0x00, 0xCF, 0xBC, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0xCF, 0xE2, 0xCD, 0xF6, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0xEF, 0xF0, 0x00, 0x00, 0xF4, 0xBE, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xD4, 0xCD, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0xF3, 0xB8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0xE9, 0xA1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0xF2, 0xF2, 0xF3, 0xEB, /* 0x90-0x93 */ + 0x00, 0x00, 0xF0, 0xD7, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0xCF, 0xD7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCF, 0xDF, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0xE8, 0xC0, 0xE8, 0xC1, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0xCF, 0xE3, 0xE9, 0xA2, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0xD0, 0xAA, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xF3, 0xC1, 0xD0, 0xAB, 0x00, 0x00, 0xD4, 0xE4, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0xEF, 0xBC, 0xD8, 0xA1, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD9, 0xDF, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xF3, 0xD7, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0xDC, 0xBD, 0x00, 0x00, 0xCC, 0xE5, /* 0xDC-0xDF */ + 0xED, 0xF1, 0x00, 0x00, 0x00, 0x00, 0xF1, 0xE2, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xD4, 0xDB, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0xB5, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0xCA, 0xE6, 0x00, 0x00, 0xD3, 0xAE, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0xCC, 0xE6, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xF1, 0xD3, 0xF5, 0xE7, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCA, 0xDA, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_7B[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFB, 0xEE, /* 0x0C-0x0F */ + 0x00, 0x00, 0xE1, 0xC5, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0xDF, 0xE9, 0x00, 0x00, 0xEE, 0xDE, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xF7, 0xC2, 0x00, 0x00, /* 0x1C-0x1F */ + 0xD8, 0xA2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0xDD, 0xAC, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0xF0, 0xAF, 0xD6, 0xBD, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0xE1, 0xAB, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xF9, 0xB6, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0xD4, 0xF5, 0x00, 0x00, 0xD0, 0xC9, /* 0x48-0x4B */ + 0xEF, 0xA7, 0xE2, 0xEC, 0x00, 0x00, 0xDB, 0xEA, /* 0x4C-0x4F */ + 0xCE, 0xCC, 0xF5, 0xE8, 0xF7, 0xD5, 0x00, 0x00, /* 0x50-0x53 */ + 0xD3, 0xCD, 0x00, 0x00, 0xF3, 0xFE, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0xD0, 0xB5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0xE0, 0xFE, 0x00, 0x00, 0xDF, 0xFB, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0xE6, 0xDD, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0xE8, 0xA4, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCB, 0xCD, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEF, 0xA8, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEE, 0xB4, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0xDA, 0xD8, 0xD1, 0xB9, 0x00, 0x00, 0xDF, 0xA9, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0xF3, 0xB0, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0xCC, 0xC4, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0xCE, 0xB7, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0xEF, 0xA9, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0xDF, 0xD5, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xED, 0xD7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xEE, 0xC6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0xEF, 0xBD, 0xFC, 0xD6, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xDB, 0xF4, 0x00, 0x00, 0xEF, 0xAA, 0xF8, 0xB9, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xF5, 0xE9, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xD9, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0xE1, 0xC6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0xD4, 0xBF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xDE, 0xE8, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ +}; + +static const unsigned char u2c_7C[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xEA, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0xF3, 0xC2, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xD3, 0xAF, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0xCA, 0xDB, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFC, 0xD7, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xED, 0xD8, 0xE1, 0xC7, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0xF4, 0xD8, 0xD6, 0xB3, 0xDD, 0xAD, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD5, 0xBE, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0xF1, 0xC3, 0xEE, 0xDF, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0xD6, 0xEB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0xF4, 0xD9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0xD7, 0xE6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDA, 0xB7, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDD, 0xFB, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0xDD, 0xCF, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0xD8, 0xA3, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0xDA, 0xD9, 0x00, 0x00, 0xF0, 0xD8, /* 0x94-0x97 */ + 0xEF, 0xC4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE1, 0xD8, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xF1, 0xD4, 0x00, 0x00, 0xED, 0xF2, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0xD5, 0xDB, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0xD5, 0xDC, 0xF3, 0xC4, 0xCB, 0xD7, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xE2, 0xB6, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0xEF, 0xF1, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0xFB, 0xD5, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0xD3, 0xD8, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0xDD, 0xD0, 0xF0, 0xD9, /* 0xDC-0xDF */ + 0xCB, 0xB3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD5, 0xDD, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCD, 0xA7, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0xD0, 0xAC, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_7D[512] = { + 0xD1, 0xBA, 0x00, 0x00, 0xF1, 0xC4, 0x00, 0x00, /* 0x00-0x03 */ + 0xE5, 0xB3, 0xFB, 0xF5, 0xE9, 0xE1, 0xFD, 0xE0, /* 0x04-0x07 */ + 0xFC, 0xBC, 0x00, 0x00, 0xDA, 0xA2, 0xDA, 0xA3, /* 0x08-0x0B */ + 0x00, 0x00, 0xD2, 0xA1, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0xD2, 0xEF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0xE2, 0xED, 0x00, 0x00, 0x00, 0x00, 0xDE, 0xE9, /* 0x14-0x17 */ + 0xCE, 0xDC, 0xF2, 0xB5, 0xD0, 0xE4, 0xDD, 0xD1, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0xE1, 0xC8, 0xDB, 0xB7, 0xDF, 0xE3, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xED, 0xB9, /* 0x28-0x2B */ + 0xF1, 0xC5, 0x00, 0x00, 0xF3, 0xCF, 0xD7, 0xAB, /* 0x2C-0x2F */ + 0xE1, 0xAC, 0x00, 0x00, 0x00, 0x00, 0xE3, 0xEB, /* 0x30-0x33 */ + 0x00, 0x00, 0xEE, 0xC7, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0xE1, 0xC9, 0xCA, 0xFA, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0xF0, 0xFB, 0xFA, 0xE1, /* 0x40-0x43 */ + 0xF0, 0xDA, 0xCC, 0xE7, 0xDA, 0xF4, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0xCC, 0xBF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0xCE, 0xED, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0xD5, 0xA9, 0xFA, 0xE2, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0xD0, 0xE5, 0x00, 0x00, /* 0x64-0x67 */ + 0xEB, 0xD6, 0x00, 0x00, 0xEC, 0xDF, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0xDF, 0xFC, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0xF7, 0xD6, 0xDE, 0xEA, 0xCB, 0xB4, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0xEF, 0xBE, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0xCC, 0xB5, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCF, 0xBD, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0xEF, 0xF2, 0xE2, 0xB7, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCC, 0xE8, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0xF0, 0xFC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0xD6, 0xE0, 0x00, 0x00, 0xF1, 0xC6, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0xE2, 0xB8, 0xEB, 0xAB, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0xCB, 0xB5, 0xD8, 0xD1, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xF4, 0xCE, 0xF3, 0xF7, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xD7, 0xC6, 0x00, 0x00, 0xD1, 0xBB, 0xF7, 0xAA, /* 0xB8-0xBB */ + 0x00, 0x00, 0xED, 0xCA, 0xD7, 0xD3, 0xD8, 0xFA, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF6, 0xC5, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0xD1, 0xCC, 0xDD, 0xFC, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0xDF, 0xFD, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xF9, 0xE5, 0x00, 0x00, 0xE0, 0xCA, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0xF2, 0xFD, 0xD3, 0xB0, 0x00, 0x00, /* 0xDC-0xDF */ + 0xF4, 0xF3, 0xDA, 0xC9, 0x00, 0x00, 0xE6, 0xDE, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xF8, 0xBA, 0xE8, 0xD0, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0xD8, 0xFB, 0x00, 0x00, 0x00, 0x00, 0xEA, 0xD5, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xD6, 0xA3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF6, 0xC6, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_7E[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xF2, 0xDB, 0xE4, 0xFC, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0xE8, 0xB2, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDA, 0xDA, /* 0x18-0x1B */ + 0x00, 0x00, 0xF2, 0xDC, 0xFB, 0xD6, 0xE9, 0xB2, /* 0x1C-0x1F */ + 0x00, 0x00, 0xEE, 0xAD, 0x00, 0x00, 0xFA, 0xE3, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDC, 0xEE, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0xF5, 0xEA, 0xE6, 0xE0, /* 0x2C-0x2F */ + 0x00, 0x00, 0xF0, 0xFD, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD7, 0xAC, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0xF5, 0xC5, 0xEE, 0xE0, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0xDB, 0xE5, 0x00, 0x00, 0xDD, 0xDE, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xD9, 0xF0, 0xE9, 0xA3, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0xF1, 0xF9, 0x00, 0x00, /* 0x50-0x53 */ + 0xF2, 0xC4, 0xE0, 0xCB, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xA4, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0xE2, 0xB9, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0xE3, 0xB1, 0xFC, 0xEB, 0xCD, 0xA8, /* 0x68-0x6B */ + 0x00, 0x00, 0xCC, 0xB6, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0xF0, 0xDB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0xE6, 0xBA, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0xCD, 0xA9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0xF3, 0xC3, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0xE1, 0xD9, 0x00, 0x00, 0x00, 0x00, 0xEF, 0xAB, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0xC5, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xE9, 0x00, 0x00, /* 0x94-0x97 */ + 0xF3, 0xC5, 0x00, 0x00, 0x00, 0x00, 0xD4, 0xC0, /* 0x98-0x9B */ + 0xD5, 0xBF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ +}; + +static const unsigned char u2c_7F[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0xDD, 0xAE, 0x00, 0x00, /* 0x34-0x37 */ + 0xF9, 0xFC, 0x00, 0x00, 0xCC, 0xC0, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0xE5, 0xA2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0xCE, 0xB8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0xD8, 0xD2, 0xF9, 0xD6, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0xF1, 0xAA, 0xCE, 0xD1, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0xF6, 0xC7, 0x00, 0x00, /* 0x6C-0x6F */ + 0xDB, 0xEB, 0x00, 0x00, 0xDF, 0xFE, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0xD8, 0xE1, 0x00, 0x00, 0xF7, 0xF3, /* 0x74-0x77 */ + 0x00, 0x00, 0xD7, 0xE7, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0xD4, 0xFE, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0xD1, 0xBC, 0x00, 0x00, 0xE5, 0xCF, 0x00, 0x00, /* 0x88-0x8B */ + 0xCB, 0xB6, 0x00, 0x00, 0xDA, 0xB8, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0xCD, 0xC4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0xD6, 0xBE, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0xBA, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xCF, 0xD8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xE0, 0xCC, 0xEB, 0xF9, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xFD, 0xFD, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xD7, 0xE8, 0xCB, 0xD8, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0xE9, 0xE2, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0xE8, 0xBA, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xE3, 0xC7, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0xEC, 0xCD, 0x00, 0x00, /* 0xC8-0xCB */ + 0xEC, 0xCE, 0x00, 0x00, 0xD6, 0xBF, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xA7, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xDF, 0xD6, 0xFD, 0xE8, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEE, 0xE1, /* 0xDC-0xDF */ + 0xF6, 0xA8, 0xDD, 0xFD, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xF8, 0xBB, 0x00, 0x00, 0xE8, 0xD1, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0xF9, 0xD7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xCE, 0xEE, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0xEC, 0xCF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_80[512] = { + 0xE9, 0xA5, 0xD6, 0xD5, 0x00, 0x00, 0xCD, 0xC5, /* 0x00-0x03 */ + 0x00, 0x00, 0xED, 0xBA, 0xD1, 0xBD, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xCF, 0xBE, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0xEC, 0xBB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0xD2, 0xB1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0xCC, 0xE9, 0x00, 0x00, 0xD9, 0xC4, /* 0x14-0x17 */ + 0xE9, 0xFC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0xD1, 0xBE, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEC, 0xBC, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xAD, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0xF7, 0xB0, 0x00, 0x00, 0xCC, 0xEA, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD3, 0xC4, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xD6, 0xC0, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0xD6, 0xFD, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xA1, 0x00, 0x00, /* 0x54-0x57 */ + 0xDE, 0xBD, 0x00, 0x00, 0xF6, 0xA9, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0xDA, 0xA4, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD6, 0xA4, /* 0x6C-0x6F */ + 0xF5, 0xC6, 0x00, 0x00, 0xE1, 0xA2, 0xE9, 0xC6, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF2, 0xC5, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0xF4, 0xE9, 0xD6, 0xEC, 0xEB, 0xD3, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0xEC, 0xBD, 0xE2, 0xDC, 0xDE, 0xEB, 0xF0, 0xDC, /* 0x84-0x87 */ + 0x00, 0x00, 0xEB, 0xBF, 0x00, 0x00, 0xD7, 0xCE, /* 0x88-0x8B */ + 0xD1, 0xBF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0xF5, 0xAB, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF9, 0xFD, /* 0x98-0x9B */ + 0x00, 0x00, 0xCA, 0xDC, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0xCD, 0xC6, 0xF2, 0xB6, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xDD, 0xFE, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0xCC, 0xB7, 0xDB, 0xB8, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD0, 0xE9, /* 0xAC-0xAF */ + 0x00, 0x00, 0xCE, 0xDD, 0xEB, 0xC0, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xFD, 0xA2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0xF8, 0xCB, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEA, 0xD6, /* 0xC0-0xC3 */ + 0xF1, 0xB0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0xDB, 0xCE, 0x00, 0x00, 0xF7, 0xC3, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xDB, 0xCF, 0xCB, 0xA4, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0xF8, 0xE0, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0xFB, 0xD7, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0xEB, 0xCA, 0xE0, 0xA1, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0xCE, 0xCD, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xD4, 0xDC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xFD, 0xD8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0xD2, 0xF6, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_81[512] = { + 0x00, 0x00, 0x00, 0x00, 0xF2, 0xB7, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0xFA, 0xF6, 0xF6, 0xAA, 0xFA, 0xF7, /* 0x04-0x07 */ + 0xD8, 0xE6, 0x00, 0x00, 0xF4, 0xB1, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0xE8, 0xD2, 0x00, 0x00, 0xCA, 0xC5, 0xCC, 0xEB, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xEE, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0xE2, 0xBB, 0x00, 0x00, 0xF7, 0xAD, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF8, 0xE1, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0xF3, 0xEC, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xDE, 0xA1, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xFD, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xEC, 0x00, 0x00, /* 0x4C-0x4F */ + 0xDD, 0xAF, 0xDD, 0xB0, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0xCB, 0xB7, 0xE8, 0xD3, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0xE1, 0xA3, 0xD2, 0xE0, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xFE, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0xE9, 0xA6, 0xCB, 0xF2, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0xED, 0xF3, 0xDC, 0xD9, 0xE0, 0xCD, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF7, 0xDA, /* 0x7C-0x7F */ + + 0xDB, 0xB9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0xCC, 0xAE, 0x00, 0x00, 0xDA, 0xDB, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCD, 0xC7, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0xDD, 0xB1, 0x00, 0x00, /* 0x98-0x9B */ + 0xD8, 0xAF, 0xE3, 0xA3, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0xCE, 0xEF, 0x00, 0x00, 0x00, 0x00, 0xF2, 0xF3, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xF8, 0xB3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xCE, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xF5, 0xFD, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0xEB, 0xEC, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0xD3, 0xC5, 0xFC, 0xEC, 0xD2, 0xDB, /* 0xBC-0xBF */ + 0xD4, 0xEB, 0x00, 0x00, 0xDE, 0xA2, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xE6, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0xF0, 0xB0, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xD5, 0xC4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xED, 0xF4, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE3, 0xED, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xE8, 0xC2, 0x00, 0x00, 0xED, 0xF5, /* 0xE4-0xE7 */ + 0xD7, 0xFC, 0x00, 0x00, 0xED, 0xBB, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0xF6, 0xAB, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF2, 0xB8, /* 0xF0-0xF3 */ + 0xF6, 0xC8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0xD3, 0xE6, 0xF2, 0xDD, /* 0xF8-0xFB */ + 0xCF, 0xBF, 0x00, 0x00, 0xEB, 0xAC, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_82[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0xCF, 0xC0, 0x00, 0x00, 0xE6, 0xA8, /* 0x04-0x07 */ + 0xFD, 0xE9, 0x00, 0x00, 0xCF, 0xC1, 0x00, 0x00, /* 0x08-0x0B */ + 0xE0, 0xDF, 0xDE, 0xEC, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xA2, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xBF, /* 0x18-0x1B */ + 0xE2, 0xEF, 0x00, 0x00, 0xD9, 0xF1, 0xF1, 0xC7, /* 0x1C-0x1F */ + 0x00, 0x00, 0xCB, 0xB8, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xF9, 0xFE, 0xDB, 0xBA, /* 0x28-0x2B */ + 0xDA, 0xF5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0xF6, 0xEC, 0xDA, 0xDC, 0xFA, 0xE4, /* 0x34-0x37 */ + 0x00, 0x00, 0xE0, 0xCF, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0xDD, 0xB2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0xE6, 0xA9, 0x00, 0x00, 0xEF, 0xF3, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0xF3, 0xED, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0xEB, 0xFA, 0x00, 0x00, 0xF9, 0xE6, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0xCA, 0xDD, 0xD5, 0xDE, /* 0x6C-0x6F */ + 0x00, 0x00, 0xCA, 0xDE, 0xDF, 0xE4, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0xFD, 0x00, 0x00, /* 0x74-0x77 */ + 0xF5, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0xF5, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE9, 0xE3, /* 0x88-0x8B */ + 0x00, 0x00, 0xED, 0xCB, 0xCF, 0xE4, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0xD8, 0xD3, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0xDD, 0xB3, 0xD4, 0xEC, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0xF2, 0xB9, 0x00, 0x00, 0xDF, 0xB7, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xCB, 0xCE, 0xFB, 0xD8, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0xD0, 0xD9, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0xDD, 0xD2, 0xF7, 0xF4, 0xE7, 0xDC, 0xE4, 0xA5, /* 0xAC-0xAF */ + 0x00, 0x00, 0xFC, 0xA3, 0x00, 0x00, 0xDB, 0xBB, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF2, 0xBA, /* 0xB4-0xB7 */ + 0xE9, 0xFD, 0xD0, 0xCA, 0x00, 0x00, 0xF5, 0xD6, /* 0xB8-0xBB */ + 0xD9, 0xC5, 0xE4, 0xB4, 0x00, 0x00, 0xED, 0xA7, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0xEA, 0xBD, 0xE6, 0xFE, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xF7, 0xC4, 0xF5, 0xAD, 0x00, 0x00, 0xD9, 0xE0, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCA, 0xB4, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0xF8, 0xE2, 0xCF, 0xC2, /* 0xDC-0xDF */ + 0x00, 0x00, 0xEC, 0xBE, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xE5, 0xB4, 0xCD, 0xC8, 0xEE, 0xC8, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0xE7, 0xC8, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0xCD, 0xC9, 0xF9, 0xB7, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_83[512] = { + 0x00, 0x00, 0xF1, 0xE8, 0xD9, 0xF2, 0xDB, 0xF5, /* 0x00-0x03 */ + 0xCA, 0xB5, 0xD9, 0xC6, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xD8, 0xC9, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD9, 0xAB, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0xED, 0xBC, 0x00, 0x00, 0x00, 0x00, 0xD8, 0xD4, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDC, 0xDA, /* 0x2C-0x2F */ + 0x00, 0x00, 0xE2, 0xBC, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0xFC, 0xED, 0xEC, 0xE0, 0xD2, 0xFE, 0x00, 0x00, /* 0x34-0x37 */ + 0xE9, 0xC7, 0xE6, 0xAA, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0xE2, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFA, 0xBB, /* 0x44-0x47 */ + 0x00, 0x00, 0xF5, 0xAE, 0xFB, 0xAA, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEC, 0xFB, /* 0x4C-0x4F */ + 0x00, 0x00, 0xEC, 0xBF, 0xFC, 0xD8, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD4, 0xE5, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF9, 0xC3, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEE, 0xE2, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0xD7, 0xE9, 0xED, 0xF6, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0xDE, 0xED, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0xCC, 0xEC, 0x00, 0x00, /* 0x94-0x97 */ + 0xE3, 0xEE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0xE8, 0xD4, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0xFA, 0xF8, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0xDD, 0xB4, 0xE4, 0xB5, 0xD8, 0xB0, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0xD8, 0xD5, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0xF4, 0xEA, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xCE, 0xB9, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xD6, 0xE1, 0xCF, 0xD2, 0x00, 0x00, /* 0xC8-0xCB */ + 0xD0, 0xB6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCE, 0xA2, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0xF3, 0xEE, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0xF3, 0xF8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xDC, 0xCC, 0x00, 0x00, 0xD0, 0xCB, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFC, 0xA4, /* 0xEC-0xEF */ + 0xCD, 0xCA, 0xD7, 0xD4, 0xDE, 0xA3, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xE4, 0xE0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xEE, 0xC9, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0xE2, 0xDD, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_84[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF5, 0xFE, /* 0x00-0x03 */ + 0xD4, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xD5, 0xD1, 0x00, 0x00, /* 0x08-0x0B */ + 0xD8, 0xF0, 0xF8, 0xC3, 0xEA, 0xD7, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0xF5, 0xD7, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0xD8, 0xBF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0xFD, 0xC0, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0xEB, 0xAD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0xD5, 0xAA, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0xE7, 0xA8, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEE, 0xCA, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCA, 0xE7, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0xF8, 0xE3, 0x00, 0x00, 0xD4, 0xDD, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0xEA, 0xD8, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFB, 0xD9, /* 0x68-0x6B */ + 0xED, 0xF7, 0x00, 0x00, 0x00, 0x00, 0xE5, 0xB5, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0xD0, 0xAD, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0xF1, 0xF1, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0xE2, 0xBD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0xE3, 0xC8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0xD9, 0xD5, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0xDF, 0xAA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0xDB, 0xBC, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xF8, 0xE4, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xF1, 0xFA, 0x00, 0x00, 0x00, 0x00, 0xE5, 0xB6, /* 0xB8-0xBB */ + 0xF3, 0xEF, 0x00, 0x00, 0x00, 0x00, 0xFB, 0xDA, /* 0xBC-0xBF */ + 0xE1, 0xE0, 0x00, 0x00, 0xD9, 0xAC, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xF5, 0xEB, 0x00, 0x00, 0xE0, 0xB6, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xE9, 0xC8, 0x00, 0x00, 0xCB, 0xCF, /* 0xC8-0xCB */ + 0x00, 0x00, 0xE3, 0xC9, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0xDE, 0xEE, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0xBE, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0xDC, 0xEF, 0x00, 0x00, 0xD6, 0xA5, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xE2, 0xF1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0xD6, 0xFE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_85[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0xD9, 0xA1, 0x00, 0x00, 0xD8, 0xC0, /* 0x10-0x13 */ + 0xDC, 0xDB, 0x00, 0x00, 0x00, 0x00, 0xED, 0xBD, /* 0x14-0x17 */ + 0xDF, 0xB8, 0x00, 0x00, 0xEA, 0xA5, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xD7, 0xAD, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0xF3, 0xF9, 0x00, 0x00, 0xED, 0xF8, /* 0x20-0x23 */ + 0x00, 0x00, 0xF5, 0xC7, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0xE1, 0xCA, 0xEB, 0xE3, 0x00, 0x00, 0xF2, 0xDE, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0xF8, 0xCC, 0x00, 0x00, 0xEA, 0xD9, /* 0x3C-0x3F */ + 0x00, 0x00, 0xD3, 0xC6, 0x00, 0x00, 0xDB, 0xE6, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0xF5, 0xAF, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0xCE, 0xF0, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE9, 0xFE, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0xFB, 0xB6, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xF2, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0xCF, 0xF2, 0xF7, 0xB9, 0xD9, 0xF3, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0xE1, 0xCB, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0xDA, 0xDD, 0x00, 0x00, 0x00, 0x00, 0xDA, 0xB9, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEB, 0xFB, /* 0x8C-0x8F */ + 0x00, 0x00, 0xCB, 0xB9, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0xED, 0xF9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xE0, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0xF4, 0xC0, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xFD, 0xBC, 0xDF, 0xB1, 0xE3, 0xEF, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xA3, /* 0xAC-0xAF */ + 0xFD, 0xB9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0xF0, 0xB1, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0xCD, 0xCB, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xED, 0xBE, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0xD5, 0xC0, 0xE3, 0xF0, 0xED, 0xFA, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xE9, 0xE4, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0xD5, 0xED, 0xE7, 0xDD, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0xD4, 0xF6, 0xE5, 0xB7, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xDB, 0xE7, 0xE2, 0xBF, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEE, 0xCB, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0xD7, 0xF4, 0xF0, 0xDD, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCE, 0xAB, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_86[512] = { + 0x00, 0x00, 0x00, 0x00, 0xE7, 0xDE, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0xD6, 0xD6, 0xE1, 0xCC, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xE8, 0xB3, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xEE, 0xDC, 0xA2, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xD0, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0xD5, 0xB5, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD5, 0xA1, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0xFB, 0xDB, 0x00, 0x00, /* 0x4C-0x4F */ + 0xF9, 0xCB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0xCB, 0xF3, 0xF4, 0xA5, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFA, 0xC8, /* 0x58-0x5B */ + 0xD6, 0xD7, 0x00, 0x00, 0xE9, 0xE5, 0xFB, 0xDC, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFD, 0xD0, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0xFB, 0xF6, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0xDA, 0xA5, 0x00, 0x00, /* 0x88-0x8B */ + 0xDB, 0xBD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEC, 0xE2, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCD, 0xF7, /* 0xA0-0xA3 */ + 0xF0, 0xDE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0xF6, 0xC9, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDE, 0xEF, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD3, 0xB1, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xFC, 0xEE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xE8, 0xC3, 0x00, 0x00, 0xF1, 0xC8, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCE, 0xF1, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0xF9, 0xED, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0xF2, 0xF4, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0xB6, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_87[512] = { + 0xF5, 0xB9, 0x00, 0x00, 0xDC, 0xF0, 0xE3, 0xF1, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0xE8, 0xA5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0xF2, 0xBB, 0x00, 0x00, 0xDE, 0xA4, 0x00, 0x00, /* 0x18-0x1B */ + 0xDA, 0xCC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0xCA, 0xE9, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0xE3, 0xDA, 0x00, 0x00, 0xFC, 0xD9, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEA, 0xDA, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0xF9, 0xC4, 0x00, 0x00, /* 0x64-0x67 */ + 0xE3, 0xA4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0xFB, 0xDD, 0x00, 0x00, 0xEF, 0xCA, 0x00, 0x00, /* 0x74-0x77 */ + 0xE8, 0xC4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0xD5, 0xCC, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0xEB, 0xD7, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD9, 0xAD, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0xFB, 0xAB, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD3, 0xD9, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0xD5, 0xA2, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xF6, 0xDE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0xDA, 0xF6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0xE0, 0xD1, 0x00, 0x00, 0x00, 0x00, 0xE9, 0xA8, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0xF5, 0xF9, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xFA, 0xAF, 0x00, 0x00, 0xEB, 0xFC, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xEA, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_88[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0xE3, 0xB2, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD5, 0xC5, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0xF1, 0xE3, 0xD5, 0xEE, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0xCD, 0xCC, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0xED, 0xD9, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD8, 0xC1, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0xFA, 0xEC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xF1, 0xEB, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0xFA, 0xBC, 0xE6, 0xE2, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0xFA, 0xE5, 0xE2, 0xFA, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCA, 0xB6, /* 0x54-0x57 */ + 0x00, 0x00, 0xE4, 0xB7, 0x00, 0x00, 0xEA, 0xDB, /* 0x58-0x5B */ + 0x00, 0x00, 0xF5, 0xFA, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0xFB, 0xAC, 0xCF, 0xC3, 0xEB, 0xFD, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0xF8, 0xFA, 0x00, 0x00, 0x00, 0x00, 0xDF, 0xB9, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0xE1, 0xF1, 0x00, 0x00, 0xD2, 0xA4, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF5, 0xFB, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0xD0, 0xDA, 0xD0, 0xDB, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xEA, 0xBE, 0xD9, 0xB1, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0xCA, 0xB7, 0x00, 0x00, 0x00, 0x00, 0xD3, 0xE7, /* 0x88-0x8B */ + 0x00, 0x00, 0xF8, 0xE5, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0xD3, 0xB2, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0xC0, 0xF2, 0xDF, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0xCD, 0xE5, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF9, 0xAC, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xCD, 0xCD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0xEE, 0xAE, 0xD6, 0xAE, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD7, 0xEA, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xE7, 0xE0, 0xEB, 0xAE, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xCF, 0xD9, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0xDC, 0xCD, 0xED, 0xFB, 0x00, 0x00, 0xDE, 0xF0, /* 0xDC-0xDF */ + 0x00, 0x00, 0xD7, 0xEB, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xDE, 0xA5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDF, 0xD7, /* 0xF0-0xF3 */ + 0xDB, 0xD0, 0xDB, 0xD1, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xD5, 0xA3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0xF0, 0xB2, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_89[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDC, 0xDC, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0xCA, 0xE8, 0x00, 0x00, 0xF8, 0xE6, 0xDC, 0xCE, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0xEA, 0xDC, 0xDB, 0xD2, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0xE9, 0xB3, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xF7, 0xDB, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xA8, 0x00, 0x00, /* 0x34-0x37 */ + 0xD7, 0xAE, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xE1, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0xCB, 0xBA, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0xE5, 0xD1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD0, 0xDC, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0xD5, 0xC1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0xD8, 0xCA, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xA9, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xA4, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xE9, 0xA9, 0x00, 0x00, 0xD3, 0xC7, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0xDC, 0xDD, 0xF8, 0xAE, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCC, 0xB8, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD0, 0xAE, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD8, 0xF2, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xCA, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0xCC, 0xAF, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0xD4, 0xAD, 0xF6, 0xD1, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xD0, 0xCC, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0xCA, 0xC6, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0xD5, 0xC2, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0xCE, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xCA, 0xC7, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFA, 0xB0, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xDF, 0xD8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xF5, 0xBA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_8A[512] = { + 0xE5, 0xEB, 0x00, 0x00, 0xEF, 0xF4, 0xDD, 0xB5, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0xCD, 0xAA, 0x00, 0x00, 0xE3, 0xF2, 0x00, 0x00, /* 0x08-0x0B */ + 0xFB, 0xF7, 0x00, 0x00, 0xF7, 0xD0, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFD, 0xBA, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xFD, 0xE1, 0xF6, 0xFE, /* 0x14-0x17 */ + 0xD1, 0xC0, 0x00, 0x00, 0x00, 0x00, 0xE8, 0xC5, /* 0x18-0x1B */ + 0x00, 0x00, 0xE4, 0xB8, 0x00, 0x00, 0xE1, 0xE8, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCC, 0xC1, /* 0x20-0x23 */ + 0x00, 0x00, 0xD2, 0xED, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xDB, 0xBE, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0xE0, 0xE2, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0xFA, 0xC9, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0xE1, 0xCD, 0x00, 0x00, 0xCA, 0xB8, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xF2, 0xE0, 0xF1, 0xC9, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0xDE, 0xF1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0xF0, 0xDF, 0xF8, 0xC4, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEE, 0xCC, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0xDE, 0xF2, 0x00, 0x00, /* 0x5C-0x5F */ + 0xE7, 0xC9, 0x00, 0x00, 0xE2, 0xF3, 0xE7, 0xE1, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xCB, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0xE3, 0xCC, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0xCF, 0xF8, 0xEF, 0xAC, 0x00, 0x00, /* 0x6C-0x6F */ + 0xFD, 0xFE, 0xFC, 0xA5, 0xFA, 0xB1, 0xDF, 0xD9, /* 0x70-0x73 */ + 0x00, 0x00, 0xE0, 0xD2, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0xF4, 0xDA, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0xF1, 0xCA, 0x00, 0x00, 0xCE, 0xA3, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0xF2, 0xBC, 0xEC, 0xE3, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xA5, /* 0x90-0x93 */ + 0x00, 0x00, 0xF7, 0xAB, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0xEB, 0xAF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xDE, 0x00, 0x00, /* 0x9C-0x9F */ + 0xE1, 0xA4, 0xCD, 0xAB, 0x00, 0x00, 0xD9, 0xF4, /* 0xA0-0xA3 */ + 0xE8, 0xA6, 0xCD, 0xCE, 0xE1, 0xE9, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xFC, 0xEF, 0x00, 0x00, 0xE0, 0xE3, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0xE2, 0xC1, 0x00, 0x00, 0xCE, 0xA4, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xDE, 0xA6, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0xEB, 0xFE, 0x00, 0x00, 0xEB, 0xDD, 0xF0, 0xE0, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0xF4, 0xDB, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xE2, 0xF4, 0x00, 0x00, 0x00, 0x00, 0xD3, 0xC8, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xEB, /* 0xC8-0xCB */ + 0x00, 0x00, 0xEE, 0xB5, 0x00, 0x00, 0xF5, 0xD8, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xD5, 0xDF, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0xD6, 0xE5, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEB, 0xB0, /* 0xD8-0xDB */ + 0xF4, 0xE3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0xE3, 0xCD, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0xF4, 0xF4, 0xFA, 0xB2, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0xEF, 0xF5, 0xCA, 0xDF, /* 0xE8-0xEB */ + 0x00, 0x00, 0xEB, 0xB1, 0xED, 0xBF, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0xFD, 0xC9, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0xA6, 0xF9, 0xA4, /* 0xF4-0xF7 */ + 0xF0, 0xB3, 0x00, 0x00, 0xE5, 0xEC, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0xD1, 0xE7, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_8B[512] = { + 0xD9, 0xC7, 0xE4, 0xD7, 0xEA, 0xDD, 0x00, 0x00, /* 0x00-0x03 */ + 0xD4, 0xF7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0xDA, 0xBA, 0x00, 0x00, /* 0x0C-0x0F */ + 0xDA, 0xCD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0xF9, 0xCC, 0x00, 0x00, 0xE1, 0xDA, 0xDB, 0xBF, /* 0x14-0x17 */ + 0x00, 0x00, 0xCC, 0xC5, 0xEC, 0xD0, 0xCB, 0xBB, /* 0x18-0x1B */ + 0x00, 0x00, 0xDE, 0xF3, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0xE9, 0xAA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0xD9, 0xC8, 0x00, 0x00, 0x00, 0x00, 0xEE, 0xE3, /* 0x28-0x2B */ + 0xD7, 0xBD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCF, 0xC4, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0xD0, 0xCD, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0xFC, 0xA6, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0xF1, 0xFB, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0xFD, 0xD2, 0xD1, 0xC1, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0xE3, 0xDB, 0x00, 0x00, 0xD3, 0xC9, 0x00, 0x00, /* 0x58-0x5B */ + 0xDC, 0xCF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0xCC, 0xED, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0xDE, 0xA7, 0x00, 0x00, 0x00, 0x00, 0xE6, 0xBB, /* 0x6C-0x6F */ + 0xEC, 0xA1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0xCC, 0xB9, 0x00, 0x00, 0x00, 0x00, 0xFB, 0xDE, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0xE7, 0xE2, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xD4, 0xC1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0xDC, 0xA8, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0xE2, 0xC2, 0x00, 0x00, 0xF3, 0xD8, 0xE5, 0xD3, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0xF3, 0xD9, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0xF3, 0xC6, 0x00, 0x00, /* 0x98-0x9B */ +}; + +static const unsigned char u2c_8C[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCD, 0xDB, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCD, 0xAC, /* 0x3C-0x3F */ + 0x00, 0x00, 0xFC, 0xC3, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xD4, 0xE7, 0x00, 0x00, /* 0x44-0x47 */ + 0xD1, 0xC2, 0x00, 0x00, 0xF9, 0xA5, 0x00, 0x00, /* 0x48-0x4B */ + 0xE8, 0xD5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0xE3, 0xCE, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0xD4, 0xCA, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0xDF, 0xDA, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0xFB, 0xDF, 0xE7, 0xE3, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0xF8, 0xFB, 0xE3, 0xCF, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0xF5, 0xB0, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0xD8, 0xE7, 0x00, 0x00, /* 0x88-0x8B */ + 0xD9, 0xC9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0xF8, 0xAF, 0xEF, 0xF6, 0x00, 0x00, /* 0x9C-0x9F */ + 0xDD, 0xB6, 0xEE, 0xAF, 0xCD, 0xF8, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDE, 0xB8, /* 0xA4-0xA7 */ + 0xFC, 0xA7, 0xF7, 0xFC, 0xF7, 0xB1, 0xCE, 0xBB, /* 0xA8-0xAB */ + 0xF4, 0xA1, 0x00, 0x00, 0x00, 0x00, 0xEE, 0xCD, /* 0xAC-0xAF */ + 0xE1, 0xAE, 0x00, 0x00, 0x00, 0x00, 0xEC, 0xC3, /* 0xB0-0xB3 */ + 0xCF, 0xFE, 0x00, 0x00, 0xF8, 0xBF, 0xD8, 0xE2, /* 0xB4-0xB7 */ + 0xD3, 0xE8, 0x00, 0x00, 0x00, 0x00, 0xDE, 0xA8, /* 0xB8-0xBB */ + 0xF4, 0xE4, 0xEC, 0xC2, 0x00, 0x00, 0xD9, 0xF5, /* 0xBC-0xBF */ + 0xF9, 0xC5, 0xDD, 0xD3, 0xD6, 0xF1, 0xEC, 0xFC, /* 0xC0-0xC3 */ + 0xFC, 0xF0, 0x00, 0x00, 0x00, 0x00, 0xED, 0xC0, /* 0xC4-0xC7 */ + 0xCA, 0xB9, 0x00, 0x00, 0xEE, 0xE4, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0xF2, 0xE1, 0x00, 0x00, 0xDE, 0xB9, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xD6, 0xF2, 0x00, 0x00, /* 0xD8-0xDB */ + 0xDE, 0xF4, 0x00, 0x00, 0xDF, 0xDB, 0x00, 0x00, /* 0xDC-0xDF */ + 0xDB, 0xD3, 0x00, 0x00, 0xFA, 0xE7, 0xD8, 0xE3, /* 0xE0-0xE3 */ + 0xF4, 0xC1, 0x00, 0x00, 0xDD, 0xB7, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0xF2, 0xF5, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0xD4, 0xAE, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xD6, 0xF3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDD, 0xB8, /* 0xF8-0xFB */ + 0xCF, 0xC5, 0xDF, 0xDF, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_8D[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0xF2, 0xBE, 0xF6, 0xA1, 0x00, 0x00, 0xEB, 0xCB, /* 0x04-0x07 */ + 0xF1, 0xFC, 0x00, 0x00, 0xF3, 0xC7, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0xE0, 0xEB, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xED, 0xFC, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xDB, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0xEE, 0xE5, 0x00, 0x00, 0xDE, 0xF5, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFA, 0xD3, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0xF1, 0xCB, 0x00, 0x00, 0x00, 0x00, 0xD0, 0xAF, /* 0x70-0x73 */ + 0xDD, 0xB9, 0x00, 0x00, 0x00, 0x00, 0xD1, 0xC3, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0xF5, 0xB1, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0xEA, 0xC6, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0xF0, 0xE1, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF6, 0xAC, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xF5, 0xD9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xEB, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0xDD, 0xBA, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0xF2, 0xBF, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0xF7, 0xC5, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDB, 0xA2, /* 0xC8-0xCB */ + 0xF2, 0xF6, 0x00, 0x00, 0x00, 0x00, 0xCA, 0xBA, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF7, 0xF5, /* 0xD8-0xDB */ + 0x00, 0x00, 0xCB, 0xE5, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0xEE, 0xE6, 0x00, 0x00, 0xE0, 0xD3, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xCE, 0xA5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD6, 0xD8, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD4, 0xAF, /* 0xF0-0xF3 */ +}; + +static const unsigned char u2c_8E[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xC9, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD3, 0xCE, /* 0x0C-0x0F */ + 0xF4, 0xC2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xCB, 0xE6, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xF1, 0xA1, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xEB, 0xB2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0xF1, 0xA2, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0xEB, 0xB3, 0x00, 0x00, /* 0x40-0x43 */ + 0xF0, 0xB4, 0x00, 0x00, 0x00, 0x00, 0xCB, 0xF4, /* 0x44-0x47 */ + 0xD4, 0xB0, 0xF3, 0xB2, 0xFB, 0xB7, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0xF5, 0xEC, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEE, 0xE7, /* 0x5C-0x5F */ + 0xF4, 0xB2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0xF5, 0xED, 0x00, 0x00, 0xCF, 0xF3, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xF0, 0xE2, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEE, 0xCE, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0xF1, 0xCC, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0xE5, 0xB8, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0xD7, 0xF5, 0xE3, 0xF3, /* 0xA8-0xAB */ + 0xCF, 0xE5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0xCF, 0xC6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0xF3, 0xB3, 0xE4, 0xD8, /* 0xC8-0xCB */ + 0xCF, 0xF9, 0xCF, 0xDA, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xFA, 0xCD, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE6, 0xE3, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF2, 0xE2, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xF5, 0xEE, 0x00, 0x00, 0x00, 0x00, 0xCA, 0xBB, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xDC, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_8F[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCE, 0xF2, /* 0x00-0x03 */ + 0x00, 0x00, 0xD6, 0xD9, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xEE, 0xB0, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0xF4, 0xE5, 0xD8, 0xC2, /* 0x10-0x13 */ + 0xDC, 0xD0, 0xCC, 0xEE, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD5, 0xE0, /* 0x18-0x1B */ + 0xF6, 0xCA, 0xFD, 0xCA, 0xD8, 0xD6, 0xF4, 0xCF, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0xD6, 0xA6, 0xDC, 0xBE, /* 0x24-0x27 */ + 0x00, 0x00, 0xDB, 0xD4, 0xD7, 0xC7, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF2, 0xFE, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF1, 0xCD, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0xE2, 0xC3, 0xDC, 0xDE, 0x00, 0x00, 0xDC, 0xDF, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xEF, 0xAD, 0xE6, 0xAB, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0xF9, 0xDD, 0xEA, 0xBF, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0xEF, 0xAE, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0xF4, 0xD0, 0xCE, 0xF3, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0xE6, 0xAC, 0x00, 0x00, 0xCE, 0xDE, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0xD5, 0xF9, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE3, 0xF4, /* 0x98-0x9B */ + 0xCD, 0xD0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD5, 0xB8, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0xF7, 0xFD, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xDC, 0xA9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0xDE, 0xF6, 0x00, 0x00, 0xDC, 0xAA, /* 0xAC-0xAF */ + 0xF2, 0xE3, 0xE9, 0xB4, 0xD2, 0xDC, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xE6, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xE3, 0xF6, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0xCA, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0xD0, 0xCE, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xDA, 0xF7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0xCA, 0xBC, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0xEE, 0xE8, 0xDA, 0xDE, /* 0xE8-0xEB */ + 0x00, 0x00, 0xF2, 0xF7, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0xE2, 0xFB, 0x00, 0x00, 0xCC, 0xA6, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDA, 0xBB, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xEE, 0xE9, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0xF5, 0xDA, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_90[512] = { + 0xF7, 0xDC, 0xE1, 0xEA, 0xCE, 0xC1, 0xD4, 0xB1, /* 0x00-0x03 */ + 0x00, 0x00, 0xFD, 0xB1, 0xE6, 0xBD, 0x00, 0x00, /* 0x04-0x07 */ + 0xFB, 0xAD, 0x00, 0x00, 0x00, 0x00, 0xF8, 0xE7, /* 0x08-0x0B */ + 0x00, 0x00, 0xE1, 0xCE, 0x00, 0x00, 0xF7, 0xE2, /* 0x0C-0x0F */ + 0xF5, 0xEF, 0xCF, 0xC7, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0xD4, 0xB2, 0xCC, 0xEF, 0x00, 0x00, 0xD4, 0xE8, /* 0x14-0x17 */ + 0x00, 0x00, 0xEE, 0xCF, 0xF7, 0xD7, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0xE0, 0xA6, 0xD6, 0xC1, 0xE1, 0xDC, /* 0x1C-0x1F */ + 0xF0, 0xE3, 0xF1, 0xE4, 0xDC, 0xF1, 0xD6, 0xA7, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0xF4, 0xF5, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0xF1, 0xCE, 0xF2, 0xE4, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0xD0, 0xB0, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0xEC, 0xEF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0xF9, 0xBA, 0x00, 0x00, 0xEB, 0xB5, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0xD4, 0xED, 0xE2, 0xC4, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE9, 0xE7, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0xEB, 0xB4, 0xEA, 0xA1, /* 0x48-0x4B */ + 0x00, 0x00, 0xF8, 0xBC, 0xCE, 0xA6, 0x00, 0x00, /* 0x4C-0x4F */ + 0xF9, 0xC6, 0xFC, 0xDA, 0x00, 0x00, 0xD4, 0xB3, /* 0x50-0x53 */ + 0xD3, 0xB9, 0xEA, 0xDE, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0xE9, 0xAB, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0xE1, 0xE1, 0xD3, 0xCF, 0xF4, 0xF6, 0x00, 0x00, /* 0x5C-0x5F */ + 0xEA, 0xC0, 0xE1, 0xCF, 0x00, 0x00, 0xCC, 0xBA, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0xEE, 0xEA, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0xF0, 0xE4, 0xF3, 0xB4, 0xD4, 0xEE, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0xF2, 0xC0, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0xF1, 0xE5, 0x00, 0x00, 0xF4, 0xC3, /* 0x74-0x77 */ + 0xE0, 0xD4, 0x00, 0x00, 0xEB, 0xB6, 0x00, 0x00, /* 0x78-0x7B */ + 0xD7, 0xA1, 0xCB, 0xE8, 0x00, 0x00, 0xF9, 0xAD, /* 0x7C-0x7F */ + + 0xE9, 0xAD, 0xD8, 0xE4, 0xFA, 0xB3, 0xE2, 0xC5, /* 0x80-0x83 */ + 0xFC, 0xBD, 0x00, 0x00, 0x00, 0x00, 0xEC, 0xC4, /* 0x84-0x87 */ + 0xD8, 0xB1, 0x00, 0x00, 0xDC, 0xAB, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD5, 0xA4, /* 0x8C-0x8F */ + 0x00, 0x00, 0xEB, 0xE9, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0xE8, 0xBB, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0xD8, 0xD7, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0xFB, 0xAE, 0xD1, 0xE1, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0xDB, 0xC0, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xF5, 0xBE, 0x00, 0x00, 0xDE, 0xF7, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCA, 0xFB, /* 0xAC-0xAF */ + 0xF7, 0xC6, 0xCF, 0xC8, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xE1, 0xD0, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xEE, 0xD0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0xE9, 0xF4, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0xCE, 0xF4, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0xD5, 0xCD, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0xCF, 0xDB, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xDD, 0xBB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0xCE, 0xAC, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0xE9, 0xE8, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0xD4, 0xB4, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_91[512] = { + 0x00, 0x00, 0x00, 0x00, 0xE4, 0xC7, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0xF5, 0xDB, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0xFA, 0xC1, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0xDE, 0xA9, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD4, 0xF8, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0xEF, 0xF7, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0xD3, 0xB3, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0xEB, 0xB7, 0xEF, 0xF8, 0xF5, 0xDC, /* 0x48-0x4B */ + 0xED, 0xCC, 0xDB, 0xD5, 0xF1, 0xCF, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0xF1, 0xD0, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0xF5, 0xB2, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0xD9, 0xAE, 0xD5, 0xAC, 0x00, 0x00, /* 0x68-0x6B */ + 0xE2, 0xC6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0xFD, 0xA3, 0x00, 0x00, 0xFB, 0xE5, /* 0x74-0x77 */ + 0xDF, 0xAB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xF5, /* 0x84-0x87 */ + 0x00, 0x00, 0xF6, 0xAD, 0x00, 0x00, 0xF5, 0xB3, /* 0x88-0x8B */ + 0x00, 0x00, 0xF0, 0xB5, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xA5, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0xF5, 0xDD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEC, 0xA2, /* 0xA8-0xAB */ + 0xED, 0xFD, 0x00, 0x00, 0xF5, 0xB4, 0xFB, 0xB8, /* 0xAC-0xAF */ + 0x00, 0x00, 0xDB, 0xA3, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xD6, 0xCA, 0xCB, 0xD9, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0xE5, 0xD4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF3, 0xFA, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xEB, 0xB8, 0x00, 0x00, 0xE0, 0xB7, /* 0xC8-0xCB */ + 0xD7, 0xEC, 0xF1, 0xEC, 0xE5, 0xAF, 0xD5, 0xE1, /* 0xCC-0xCF */ + 0xD7, 0xED, 0xD1, 0xD1, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE1, 0xF2, /* 0xD4-0xD7 */ + 0xEF, 0xF9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0xDD, 0xBC, 0xF6, 0xDC, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xE5, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xC4, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xE9, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0xF3, 0xFB, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ +}; + +static const unsigned char u2c_92[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0xD4, 0xEF, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0xCC, 0xA2, 0xF7, 0xFE, 0xDF, 0xBC, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEB, 0xCD, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xD0, 0xB7, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0xD6, 0xC2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xE8, 0xAD, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEF, 0xAF, /* 0x3C-0x3F */ + 0xCB, 0xA5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0xCB, 0xE9, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0xFA, 0xE8, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCC, 0xC6, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE6, 0xE7, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0xEA, 0xC7, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0xDB, 0xA4, 0x00, 0x00, /* 0x60-0x63 */ + 0xCF, 0xC9, 0xE2, 0xFC, 0xEF, 0xFA, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xEB, 0xDE, 0x00, 0x00, 0x00, 0x00, 0xF5, 0xC8, /* 0x80-0x83 */ + 0x00, 0x00, 0xD4, 0xDE, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xE0, 0xD5, 0x00, 0x00, 0xEF, 0xB0, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0xC7, 0x00, 0x00, /* 0x94-0x97 */ + 0xD9, 0xAF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0xF9, 0xE7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0xE5, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0xCF, 0xCA, 0xE1, 0xD1, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xE2, 0xC8, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0xEF, 0xFB, 0x00, 0x00, 0x00, 0x00, 0xFA, 0xF9, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xDC, 0xF2, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0xE0, 0xA7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0xF8, 0xE8, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xCB, 0xEA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0xCB, 0xBC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_93[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0xD6, 0xE2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0xF5, 0xDE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0xF5, 0xDF, 0x00, 0x00, 0xEE, 0xB6, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0xF6, 0xD3, 0xCA, /* 0x1C-0x1F */ + 0xEF, 0xFC, 0xD1, 0xC4, 0xEF, 0xB1, 0x00, 0x00, /* 0x20-0x23 */ + 0xD1, 0xC5, 0x00, 0x00, 0xD0, 0xDE, 0x00, 0x00, /* 0x24-0x27 */ + 0xD9, 0xE1, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xB8, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0xCD, 0xD1, 0xF3, 0xB9, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0xE7, 0xCC, 0x00, 0x00, 0xD6, 0xA8, 0xCE, 0xA7, /* 0x48-0x4B */ + 0x00, 0x00, 0xD4, 0xB5, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0xE4, 0xC8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD3, 0xB4, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0xEB, 0xB9, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0xCB, 0xF5, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0xF6, 0xDD, 0x00, 0x00, 0xF1, 0xA3, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0xCC, 0xC7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0xE9, 0xCA, 0x00, 0x00, 0xE1, 0xF0, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0xF5, 0xE0, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFB, 0xAF, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCB, 0xD1, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0xFB, 0xE0, 0xF2, 0xE5, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0xEC, 0xF0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xEC, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0xEE, 0xEB, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xCB, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0xCC, 0xF0, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0xD7, 0xAF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0xF3, 0xA1, 0x00, 0x00, /* 0xF4-0xF7 */ +}; + +static const unsigned char u2c_94[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0xFC, 0xF5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0xF1, 0xA4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0xE0, 0xD6, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEF, 0xB2, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0xF4, 0xD1, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0xF7, 0xA1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0xF1, 0xD1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0xCA, 0xFC, 0xCA, 0xFD, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCE, 0xCE, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0xF3, 0xC8, 0x00, 0x00, 0xF3, 0xBA, /* 0x7C-0x7F */ +}; + +static const unsigned char u2c_95[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xED, 0xFE, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xDA, 0xA6, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xEC, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0xF8, 0xCD, 0x00, 0x00, 0xCB, 0xD2, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEB, 0xCE, /* 0x8C-0x8F */ + 0x00, 0x00, 0xF9, 0xD8, 0xF9, 0xD9, 0xCA, 0xE0, /* 0x90-0x93 */ + 0xDA, 0xCA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0xCB, 0xA6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCA, 0xC8, /* 0xA0-0xA3 */ + 0xF9, 0xEE, 0xDB, 0xEC, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xD0, 0xB1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0xD5, 0xEF, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0xE6, 0xF3, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0xA2, /* 0xB8-0xBB */ + 0xE4, 0xD9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xE1, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0xFC, 0xC4, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xF9, 0xEF, 0xCF, 0xF4, 0xF7, 0xE6, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0xCE, 0xBC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0xF4, 0xC5, 0xDC, 0xA3, 0x00, 0x00, /* 0xE0-0xE3 */ +}; + +static const unsigned char u2c_96[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0xDD, 0xBD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0xF4, 0xC6, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xF8, 0xA1, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0xE8, 0xD6, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0xDB, 0xC1, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xE6, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xB9, /* 0x3C-0x3F */ + 0xF6, 0xED, 0x00, 0x00, 0xF9, 0xAE, 0x00, 0x00, /* 0x40-0x43 */ + 0xDD, 0xBE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD7, 0xB0, /* 0x48-0x4B */ + 0xD8, 0xE8, 0xCB, 0xBD, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0xF9, 0xDA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF8, 0xCE, /* 0x58-0x5B */ + 0xF9, 0xF0, 0xE0, 0xED, 0xE3, 0xB3, 0xF4, 0xB3, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0xEA, 0xC2, 0xF2, 0xE6, /* 0x60-0x63 */ + 0xF0, 0xB6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0xDB, 0xD6, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0xEB, 0xE4, 0x00, 0x00, 0x00, 0x00, 0xF2, 0xE7, /* 0x70-0x73 */ + 0x00, 0x00, 0xD7, 0xD5, 0xD4, 0xB6, 0xF9, 0xE8, /* 0x74-0x77 */ + 0xD7, 0xC1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0xE5, 0xD5, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0xE9, 0xEA, 0xD7, 0xCC, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0xD3, 0xE9, 0xE2, 0xC9, /* 0x88-0x8B */ + 0x00, 0x00, 0xFC, 0xDB, 0xCD, 0xAD, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0xCC, 0xB0, 0xEA, 0xA2, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0xE4, 0xF6, 0xD0, 0xC0, 0x00, 0x00, 0xF0, 0xB7, /* 0x98-0x9B */ + 0xEE, 0xA1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD7, 0xF6, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xCA, /* 0xA4-0xA7 */ + 0xE2, 0xCB, 0x00, 0x00, 0xFA, 0xCF, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0xEB, 0xDF, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD6, 0xCB, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xB4, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0xED, 0xCD, 0xE4, 0xD2, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xEA, 0xA9, 0xE4, 0xBA, 0xF3, 0xA2, 0xCD, 0xD2, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xF6, 0xCB, 0x00, 0x00, 0xF1, 0xE6, /* 0xC8-0xCB */ + 0xED, 0xC1, 0xE8, 0xBC, 0xEE, 0xD1, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xF0, 0xE7, 0xE2, 0xCC, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xE4, 0xAA, 0x00, 0x00, 0xF5, 0xE1, /* 0xD8-0xDB */ + 0xED, 0xDA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0xD7, 0xEE, 0xD1, 0xF1, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xE9, 0xEB, 0xE9, 0xEC, 0xE0, 0xE4, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDA, 0xA7, /* 0xEC-0xEF */ + 0xDD, 0xD4, 0x00, 0x00, 0xEA, 0xA3, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0xD6, 0xC3, 0xD6, 0xF4, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xDA, 0xDF, 0x00, 0x00, 0xEF, 0xB3, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_97[512] = { + 0xE2, 0xCD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0xEF, 0xFD, 0xF2, 0xE8, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0xEF, 0xC5, 0x00, 0x00, 0xE7, 0xE7, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xD7, 0xFD, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0xE7, 0xCE, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0xDF, 0xDC, 0x00, 0x00, 0xF9, 0xC7, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD9, 0xF6, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xDF, 0xAC, 0x00, 0x00, 0xD6, 0xDA, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0xDC, 0xA4, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0xF0, 0xB8, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0xD5, 0xFA, 0x00, 0x00, /* 0x40-0x43 */ + 0xE4, 0xF7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0xD6, 0xC4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0xF4, 0xEC, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0xEF, 0xFE, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0xF0, 0xA1, 0x00, 0x00, 0xDE, 0xAA, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0xDA, 0xBC, 0xD8, 0xFC, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0xFA, 0xD4, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0xEC, 0xE5, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0xFC, 0xA8, 0x00, 0x00, 0x00, 0x00, 0xEC, 0xE6, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0xD8, 0xCB, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFB, 0xB9, /* 0x88-0x8B */ + 0x00, 0x00, 0xE4, 0xD3, 0x00, 0x00, 0xCD, 0xF9, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0xCF, 0xD3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xCA, 0xEA, 0x00, 0x00, 0x00, 0x00, 0xCF, 0xD4, /* 0xA8-0xAB */ + 0x00, 0x00, 0xF8, 0xBD, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0xF4, 0xC7, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEA, 0xDF, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF9, 0xDB, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0xD4, 0xB7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEB, 0xE5, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xD2, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEA, 0xA4, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFA, 0xC2, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_98[512] = { + 0xFB, 0xE1, 0xFA, 0xED, 0xF0, 0xA2, 0xCC, 0xF1, /* 0x00-0x03 */ + 0x00, 0x00, 0xFA, 0xA3, 0xE2, 0xF7, 0x00, 0x00, /* 0x04-0x07 */ + 0xE2, 0xCE, 0x00, 0x00, 0xE9, 0xF5, 0x00, 0x00, /* 0x08-0x0B */ + 0xE1, 0xEB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0xE7, 0xE8, 0xE8, 0xD7, 0xDA, 0xF8, 0xD4, 0xCB, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF7, 0xF6, /* 0x14-0x17 */ + 0xD6, 0xC5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0xD4, 0xE9, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xFA, 0xFA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0xCC, 0xF2, 0xF7, 0xDD, 0x00, 0x00, 0xDE, 0xBA, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xCE, 0xA8, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0xF0, 0xB9, 0xE4, 0xFE, 0xE4, 0xC9, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0xE4, 0xD4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0xEA, 0xC3, 0x00, 0x00, 0xEF, 0xB4, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0xD7, 0xBE, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0xFB, 0xE2, 0x00, 0x00, 0xCD, 0xD3, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEF, 0xB5, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFA, 0xE9, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xF9, 0xA6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDF, 0xBD, /* 0xAC-0xAF */ + 0x00, 0x00, 0xF7, 0xC7, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xF8, 0xFD, 0x00, 0x00, 0x00, 0x00, 0xF8, 0xFC, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDE, 0xAB, /* 0xD8-0xDB */ + 0xDB, 0xE8, 0x00, 0x00, 0x00, 0x00, 0xE3, 0xDD, /* 0xDC-0xDF */ + 0x00, 0x00, 0xE1, 0xE2, 0xD1, 0xC6, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0xF6, 0xD0, 0xEB, 0xE6, 0xDA, 0xF9, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xEC, 0xC7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0xDE, 0xF8, 0xF8, 0xE9, 0xE3, 0xDE, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_99[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCE, 0xF5, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xFA, 0xC3, 0xE5, 0xD7, 0x00, 0x00, /* 0x08-0x0B */ + 0xEC, 0xC8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0xF3, 0xC9, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xBB, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0xE6, 0xAE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xEF, 0xB6, 0x00, 0x00, /* 0x1C-0x1F */ + 0xDC, 0xBF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0xCE, 0xBD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0xD8, 0xC3, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0xD0, 0xCF, 0x00, 0x00, 0xCF, 0xFA, /* 0x48-0x4B */ + 0xF3, 0xCA, 0xE0, 0xD7, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0xD1, 0xC7, 0xE9, 0xAE, 0x00, 0x00, /* 0x50-0x53 */ + 0xE8, 0xBD, 0x00, 0x00, 0x00, 0x00, 0xFA, 0xC4, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0xCF, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0xFA, 0xC5, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0xF9, 0xB8, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xDC, 0xE0, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xFB, 0xB0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0xD8, 0xA9, 0xE5, 0xDF, 0xF9, 0xA7, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0xF6, 0xEE, 0x00, 0x00, 0xF6, 0xCC, /* 0xB0-0xB3 */ + 0xE2, 0xF8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xEC, 0xF1, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0xDA, 0xE0, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0xF1, 0xD2, 0xD2, 0xCC, 0xCF, 0xCB, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xCA, 0xBD, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xDD, 0xBF, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0xF6, 0xEF, 0x00, 0x00, 0xDE, 0xF9, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0xFA, 0xB4, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0xD5, 0xAD, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF1, 0xE7, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_9A[512] = { + 0x00, 0x00, 0xDE, 0xBE, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0xDC, 0xC0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0xD1, 0xC8, 0xD1, 0xC9, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0xF8, 0xBE, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCB, 0xF6, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xD4, 0xF9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0xF5, 0xE2, 0xE1, 0xD3, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0xD8, 0xE9, 0x00, 0x00, 0x00, 0x00, 0xF8, 0xFE, /* 0x40-0x43 */ + 0x00, 0x00, 0xCF, 0xCC, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0xFD, 0xA4, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0xCE, 0xF6, 0x00, 0x00, 0xFA, 0xD0, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0xCC, 0xF3, 0xE6, 0xBE, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF6, 0xAE, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0xD5, 0xF0, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0xD1, 0xCA, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0xFC, 0xBE, 0xD5, 0xF1, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xCD, 0xE9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xFA, 0xB5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xD0, /* 0xD0-0xD3 */ + 0xF4, 0xF7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xCD, 0xD4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xE7, 0xA3, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0xDB, 0xA5, 0x00, 0x00, /* 0xEC-0xEF */ +}; + +static const unsigned char u2c_9B[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0xD1, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD7, 0xA2, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xF7, 0xE3, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0xEA, 0xA6, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0xD0, 0xA1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0xCE, 0xDA, 0xFB, 0xEB, 0xDB, 0xA6, /* 0x40-0x43 */ + 0xDB, 0xDE, 0xD8, 0xE5, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEA, 0xE0, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0xD8, 0xAA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xE0, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD6, 0xDB, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0xEF, 0xC6, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xF8, 0xEA, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xD5, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCE, 0xF7, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xD8, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xD7, 0xEF, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0xF4, 0xED, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0xCD, 0xE6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xCC, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ +}; + +static const unsigned char u2c_9C[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0xF5, 0xE3, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0xE4, 0xCA, 0x00, 0x00, 0xDC, 0xE1, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0xF9, 0xC8, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0xFC, 0xBF, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0xE8, 0xA7, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD8, 0xC4, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCB, 0xBE, /* 0x44-0x47 */ + 0x00, 0x00, 0xDC, 0xAE, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD7, 0xF7, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xF0, 0xE8, 0x00, 0x00, 0xDD, 0xC0, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xCF, 0xCD, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDC, 0xF3, /* 0xF0-0xF3 */ + 0xD9, 0xB0, 0x00, 0x00, 0xE6, 0xE9, 0x00, 0x00, /* 0xF4-0xF7 */ +}; + +static const unsigned char u2c_9D[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xE4, 0xBC, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEA, 0xC4, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0xEC, 0x00, 0x00, /* 0x24-0x27 */ + 0xE4, 0xE5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFB, 0xF8, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0xCC, 0xBB, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0xE4, 0xBD, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0xCD, 0xDC, 0xD9, 0xF7, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0xDD, 0xDF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0xED, 0xCE, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0xD9, 0xD0, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE5, 0xA3, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xF9, 0xCD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xCD, 0xAE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCF, 0xCE, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0xF6, 0xAF, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xFD, 0xD3, 0xEB, 0xED, 0xD6, 0xDC, 0x00, 0x00, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_9E[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xA4, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xD5, 0xB6, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0xD6, 0xDD, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0xF9, 0xE9, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0xE7, 0xA4, 0x00, 0x00, 0xD6, 0xE3, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0xD1, 0xCB, 0xD6, 0xE4, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD5, 0xF2, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0xDE, 0xFA, 0x00, 0x00, 0xD7, 0xF8, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xD8, 0xEA, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xCF, 0xD5, 0xD8, 0xFD, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD8, 0xAB, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0xFD, 0xCB, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFC, 0xDC, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0xE0, 0xA8, 0xD5, 0xF3, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0xFD, 0xD9, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xCC, 0xA3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xD9, 0xF9, 0x00, 0x00, 0x00, 0x00, 0xD3, 0xEA, /* 0xD8-0xDB */ + 0xF5, 0xF5, 0x00, 0x00, 0xEF, 0xC7, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xD3, 0xDA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xDA, 0xBD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ +}; + +static const unsigned char u2c_9F[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE8, 0xA8, /* 0x04-0x07 */ + 0xDC, 0xAF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0xF0, 0xA3, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCD, 0xD5, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0xE0, 0xA9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDE, 0xAC, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0xF0, 0xBA, 0xEE, 0xB1, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0xEE, 0xB2, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0xF6, 0xCD, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEE, 0xD2, /* 0x5C-0x5F */ + 0x00, 0x00, 0xD6, 0xC6, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xE5, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0xF3, 0xBB, 0x00, 0x00, /* 0x68-0x6B */ + 0xE5, 0xE1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xCB, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0xD7, 0xA3, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0xDB, 0xC2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0xCA, 0xFE, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0xCF, 0xCF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ +}; + +static const unsigned char u2c_AC[512] = { + 0xB0, 0xA1, 0xB0, 0xA2, 0x81, 0x41, 0x81, 0x42, /* 0x00-0x03 */ + 0xB0, 0xA3, 0x81, 0x43, 0x81, 0x44, 0xB0, 0xA4, /* 0x04-0x07 */ + 0xB0, 0xA5, 0xB0, 0xA6, 0xB0, 0xA7, 0x81, 0x45, /* 0x08-0x0B */ + 0x81, 0x46, 0x81, 0x47, 0x81, 0x48, 0x81, 0x49, /* 0x0C-0x0F */ + 0xB0, 0xA8, 0xB0, 0xA9, 0xB0, 0xAA, 0xB0, 0xAB, /* 0x10-0x13 */ + 0xB0, 0xAC, 0xB0, 0xAD, 0xB0, 0xAE, 0xB0, 0xAF, /* 0x14-0x17 */ + 0x81, 0x4A, 0xB0, 0xB0, 0xB0, 0xB1, 0xB0, 0xB2, /* 0x18-0x1B */ + 0xB0, 0xB3, 0xB0, 0xB4, 0x81, 0x4B, 0x81, 0x4C, /* 0x1C-0x1F */ + 0xB0, 0xB5, 0x81, 0x4D, 0x81, 0x4E, 0x81, 0x4F, /* 0x20-0x23 */ + 0xB0, 0xB6, 0x81, 0x50, 0x81, 0x51, 0x81, 0x52, /* 0x24-0x27 */ + 0x81, 0x53, 0x81, 0x54, 0x81, 0x55, 0x81, 0x56, /* 0x28-0x2B */ + 0xB0, 0xB7, 0xB0, 0xB8, 0x81, 0x57, 0xB0, 0xB9, /* 0x2C-0x2F */ + 0xB0, 0xBA, 0xB0, 0xBB, 0x81, 0x58, 0x81, 0x59, /* 0x30-0x33 */ + 0x81, 0x5A, 0x81, 0x61, 0x81, 0x62, 0x81, 0x63, /* 0x34-0x37 */ + 0xB0, 0xBC, 0xB0, 0xBD, 0x81, 0x64, 0x81, 0x65, /* 0x38-0x3B */ + 0xB0, 0xBE, 0x81, 0x66, 0x81, 0x67, 0x81, 0x68, /* 0x3C-0x3F */ + 0xB0, 0xBF, 0x81, 0x69, 0x81, 0x6A, 0x81, 0x6B, /* 0x40-0x43 */ + 0x81, 0x6C, 0x81, 0x6D, 0x81, 0x6E, 0x81, 0x6F, /* 0x44-0x47 */ + 0x81, 0x70, 0x81, 0x71, 0x81, 0x72, 0xB0, 0xC0, /* 0x48-0x4B */ + 0x81, 0x73, 0xB0, 0xC1, 0x81, 0x74, 0x81, 0x75, /* 0x4C-0x4F */ + 0x81, 0x76, 0x81, 0x77, 0x81, 0x78, 0x81, 0x79, /* 0x50-0x53 */ + 0xB0, 0xC2, 0x81, 0x7A, 0x81, 0x81, 0x81, 0x82, /* 0x54-0x57 */ + 0xB0, 0xC3, 0x81, 0x83, 0x81, 0x84, 0x81, 0x85, /* 0x58-0x5B */ + 0xB0, 0xC4, 0x81, 0x86, 0x81, 0x87, 0x81, 0x88, /* 0x5C-0x5F */ + 0x81, 0x89, 0x81, 0x8A, 0x81, 0x8B, 0x81, 0x8C, /* 0x60-0x63 */ + 0x81, 0x8D, 0x81, 0x8E, 0x81, 0x8F, 0x81, 0x90, /* 0x64-0x67 */ + 0x81, 0x91, 0x81, 0x92, 0x81, 0x93, 0x81, 0x94, /* 0x68-0x6B */ + 0x81, 0x95, 0x81, 0x96, 0x81, 0x97, 0x81, 0x98, /* 0x6C-0x6F */ + 0xB0, 0xC5, 0xB0, 0xC6, 0x81, 0x99, 0x81, 0x9A, /* 0x70-0x73 */ + 0xB0, 0xC7, 0x81, 0x9B, 0x81, 0x9C, 0xB0, 0xC8, /* 0x74-0x77 */ + 0xB0, 0xC9, 0x81, 0x9D, 0xB0, 0xCA, 0x81, 0x9E, /* 0x78-0x7B */ + 0x81, 0x9F, 0x81, 0xA0, 0x81, 0xA1, 0x81, 0xA2, /* 0x7C-0x7F */ + + 0xB0, 0xCB, 0xB0, 0xCC, 0x81, 0xA3, 0xB0, 0xCD, /* 0x80-0x83 */ + 0xB0, 0xCE, 0xB0, 0xCF, 0xB0, 0xD0, 0x81, 0xA4, /* 0x84-0x87 */ + 0x81, 0xA5, 0xB0, 0xD1, 0xB0, 0xD2, 0xB0, 0xD3, /* 0x88-0x8B */ + 0xB0, 0xD4, 0x81, 0xA6, 0x81, 0xA7, 0x81, 0xA8, /* 0x8C-0x8F */ + 0xB0, 0xD5, 0x81, 0xA9, 0x81, 0xAA, 0x81, 0xAB, /* 0x90-0x93 */ + 0xB0, 0xD6, 0x81, 0xAC, 0x81, 0xAD, 0x81, 0xAE, /* 0x94-0x97 */ + 0x81, 0xAF, 0x81, 0xB0, 0x81, 0xB1, 0x81, 0xB2, /* 0x98-0x9B */ + 0xB0, 0xD7, 0xB0, 0xD8, 0x81, 0xB3, 0xB0, 0xD9, /* 0x9C-0x9F */ + 0xB0, 0xDA, 0xB0, 0xDB, 0x81, 0xB4, 0x81, 0xB5, /* 0xA0-0xA3 */ + 0x81, 0xB6, 0x81, 0xB7, 0x81, 0xB8, 0x81, 0xB9, /* 0xA4-0xA7 */ + 0xB0, 0xDC, 0xB0, 0xDD, 0xB0, 0xDE, 0x81, 0xBA, /* 0xA8-0xAB */ + 0xB0, 0xDF, 0x81, 0xBB, 0x81, 0xBC, 0xB0, 0xE0, /* 0xAC-0xAF */ + 0xB0, 0xE1, 0x81, 0xBD, 0x81, 0xBE, 0x81, 0xBF, /* 0xB0-0xB3 */ + 0x81, 0xC0, 0x81, 0xC1, 0x81, 0xC2, 0x81, 0xC3, /* 0xB4-0xB7 */ + 0xB0, 0xE2, 0xB0, 0xE3, 0x81, 0xC4, 0xB0, 0xE4, /* 0xB8-0xBB */ + 0xB0, 0xE5, 0xB0, 0xE6, 0x81, 0xC5, 0x81, 0xC6, /* 0xBC-0xBF */ + 0x81, 0xC7, 0xB0, 0xE7, 0x81, 0xC8, 0x81, 0xC9, /* 0xC0-0xC3 */ + 0xB0, 0xE8, 0x81, 0xCA, 0x81, 0xCB, 0x81, 0xCC, /* 0xC4-0xC7 */ + 0xB0, 0xE9, 0x81, 0xCD, 0x81, 0xCE, 0x81, 0xCF, /* 0xC8-0xCB */ + 0xB0, 0xEA, 0x81, 0xD0, 0x81, 0xD1, 0x81, 0xD2, /* 0xCC-0xCF */ + 0x81, 0xD3, 0x81, 0xD4, 0x81, 0xD5, 0x81, 0xD6, /* 0xD0-0xD3 */ + 0x81, 0xD7, 0xB0, 0xEB, 0x81, 0xD8, 0xB0, 0xEC, /* 0xD4-0xD7 */ + 0x81, 0xD9, 0x81, 0xDA, 0x81, 0xDB, 0x81, 0xDC, /* 0xD8-0xDB */ + 0x81, 0xDD, 0x81, 0xDE, 0x81, 0xDF, 0x81, 0xE0, /* 0xDC-0xDF */ + 0xB0, 0xED, 0xB0, 0xEE, 0x81, 0xE1, 0x81, 0xE2, /* 0xE0-0xE3 */ + 0xB0, 0xEF, 0x81, 0xE3, 0x81, 0xE4, 0xB0, 0xF0, /* 0xE4-0xE7 */ + 0xB0, 0xF1, 0x81, 0xE5, 0xB0, 0xF2, 0x81, 0xE6, /* 0xE8-0xEB */ + 0xB0, 0xF3, 0x81, 0xE7, 0x81, 0xE8, 0xB0, 0xF4, /* 0xEC-0xEF */ + 0xB0, 0xF5, 0xB0, 0xF6, 0x81, 0xE9, 0xB0, 0xF7, /* 0xF0-0xF3 */ + 0x81, 0xEA, 0xB0, 0xF8, 0xB0, 0xF9, 0x81, 0xEB, /* 0xF4-0xF7 */ + 0x81, 0xEC, 0x81, 0xED, 0x81, 0xEE, 0x81, 0xEF, /* 0xF8-0xFB */ + 0xB0, 0xFA, 0xB0, 0xFB, 0x81, 0xF0, 0x81, 0xF1, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_AD[512] = { + 0xB0, 0xFC, 0x81, 0xF2, 0x81, 0xF3, 0x81, 0xF4, /* 0x00-0x03 */ + 0xB0, 0xFD, 0x81, 0xF5, 0xB0, 0xFE, 0x81, 0xF6, /* 0x04-0x07 */ + 0x81, 0xF7, 0x81, 0xF8, 0x81, 0xF9, 0x81, 0xFA, /* 0x08-0x0B */ + 0xB1, 0xA1, 0xB1, 0xA2, 0x81, 0xFB, 0xB1, 0xA3, /* 0x0C-0x0F */ + 0x81, 0xFC, 0xB1, 0xA4, 0x81, 0xFD, 0x81, 0xFE, /* 0x10-0x13 */ + 0x82, 0x41, 0x82, 0x42, 0x82, 0x43, 0x82, 0x44, /* 0x14-0x17 */ + 0xB1, 0xA5, 0x82, 0x45, 0x82, 0x46, 0x82, 0x47, /* 0x18-0x1B */ + 0xB1, 0xA6, 0x82, 0x48, 0x82, 0x49, 0x82, 0x4A, /* 0x1C-0x1F */ + 0xB1, 0xA7, 0x82, 0x4B, 0x82, 0x4C, 0x82, 0x4D, /* 0x20-0x23 */ + 0x82, 0x4E, 0x82, 0x4F, 0x82, 0x50, 0x82, 0x51, /* 0x24-0x27 */ + 0x82, 0x52, 0xB1, 0xA8, 0x82, 0x53, 0x82, 0x54, /* 0x28-0x2B */ + 0xB1, 0xA9, 0xB1, 0xAA, 0x82, 0x55, 0x82, 0x56, /* 0x2C-0x2F */ + 0x82, 0x57, 0x82, 0x58, 0x82, 0x59, 0x82, 0x5A, /* 0x30-0x33 */ + 0xB1, 0xAB, 0xB1, 0xAC, 0x82, 0x61, 0x82, 0x62, /* 0x34-0x37 */ + 0xB1, 0xAD, 0x82, 0x63, 0x82, 0x64, 0x82, 0x65, /* 0x38-0x3B */ + 0xB1, 0xAE, 0x82, 0x66, 0x82, 0x67, 0x82, 0x68, /* 0x3C-0x3F */ + 0x82, 0x69, 0x82, 0x6A, 0x82, 0x6B, 0x82, 0x6C, /* 0x40-0x43 */ + 0xB1, 0xAF, 0xB1, 0xB0, 0x82, 0x6D, 0xB1, 0xB1, /* 0x44-0x47 */ + 0x82, 0x6E, 0xB1, 0xB2, 0x82, 0x6F, 0x82, 0x70, /* 0x48-0x4B */ + 0x82, 0x71, 0x82, 0x72, 0x82, 0x73, 0x82, 0x74, /* 0x4C-0x4F */ + 0xB1, 0xB3, 0x82, 0x75, 0x82, 0x76, 0x82, 0x77, /* 0x50-0x53 */ + 0xB1, 0xB4, 0x82, 0x78, 0x82, 0x79, 0x82, 0x7A, /* 0x54-0x57 */ + 0xB1, 0xB5, 0x82, 0x81, 0x82, 0x82, 0x82, 0x83, /* 0x58-0x5B */ + 0x82, 0x84, 0x82, 0x85, 0x82, 0x86, 0x82, 0x87, /* 0x5C-0x5F */ + 0x82, 0x88, 0xB1, 0xB6, 0x82, 0x89, 0xB1, 0xB7, /* 0x60-0x63 */ + 0x82, 0x8A, 0x82, 0x8B, 0x82, 0x8C, 0x82, 0x8D, /* 0x64-0x67 */ + 0x82, 0x8E, 0x82, 0x8F, 0x82, 0x90, 0x82, 0x91, /* 0x68-0x6B */ + 0xB1, 0xB8, 0xB1, 0xB9, 0x82, 0x92, 0x82, 0x93, /* 0x6C-0x6F */ + 0xB1, 0xBA, 0x82, 0x94, 0x82, 0x95, 0xB1, 0xBB, /* 0x70-0x73 */ + 0xB1, 0xBC, 0xB1, 0xBD, 0xB1, 0xBE, 0x82, 0x96, /* 0x74-0x77 */ + 0x82, 0x97, 0x82, 0x98, 0x82, 0x99, 0xB1, 0xBF, /* 0x78-0x7B */ + 0xB1, 0xC0, 0xB1, 0xC1, 0x82, 0x9A, 0xB1, 0xC2, /* 0x7C-0x7F */ + + 0x82, 0x9B, 0xB1, 0xC3, 0xB1, 0xC4, 0x82, 0x9C, /* 0x80-0x83 */ + 0x82, 0x9D, 0x82, 0x9E, 0x82, 0x9F, 0x82, 0xA0, /* 0x84-0x87 */ + 0xB1, 0xC5, 0xB1, 0xC6, 0x82, 0xA1, 0x82, 0xA2, /* 0x88-0x8B */ + 0xB1, 0xC7, 0x82, 0xA3, 0x82, 0xA4, 0x82, 0xA5, /* 0x8C-0x8F */ + 0xB1, 0xC8, 0x82, 0xA6, 0x82, 0xA7, 0x82, 0xA8, /* 0x90-0x93 */ + 0x82, 0xA9, 0x82, 0xAA, 0x82, 0xAB, 0x82, 0xAC, /* 0x94-0x97 */ + 0x82, 0xAD, 0x82, 0xAE, 0x82, 0xAF, 0x82, 0xB0, /* 0x98-0x9B */ + 0xB1, 0xC9, 0xB1, 0xCA, 0x82, 0xB1, 0x82, 0xB2, /* 0x9C-0x9F */ + 0x82, 0xB3, 0x82, 0xB4, 0x82, 0xB5, 0x82, 0xB6, /* 0xA0-0xA3 */ + 0xB1, 0xCB, 0x82, 0xB7, 0x82, 0xB8, 0x82, 0xB9, /* 0xA4-0xA7 */ + 0x82, 0xBA, 0x82, 0xBB, 0x82, 0xBC, 0x82, 0xBD, /* 0xA8-0xAB */ + 0x82, 0xBE, 0x82, 0xBF, 0x82, 0xC0, 0x82, 0xC1, /* 0xAC-0xAF */ + 0x82, 0xC2, 0x82, 0xC3, 0x82, 0xC4, 0x82, 0xC5, /* 0xB0-0xB3 */ + 0x82, 0xC6, 0x82, 0xC7, 0x82, 0xC8, 0xB1, 0xCC, /* 0xB4-0xB7 */ + 0x82, 0xC9, 0x82, 0xCA, 0x82, 0xCB, 0x82, 0xCC, /* 0xB8-0xBB */ + 0x82, 0xCD, 0x82, 0xCE, 0x82, 0xCF, 0x82, 0xD0, /* 0xBC-0xBF */ + 0xB1, 0xCD, 0xB1, 0xCE, 0x82, 0xD1, 0x82, 0xD2, /* 0xC0-0xC3 */ + 0xB1, 0xCF, 0x82, 0xD3, 0x82, 0xD4, 0x82, 0xD5, /* 0xC4-0xC7 */ + 0xB1, 0xD0, 0x82, 0xD6, 0x82, 0xD7, 0x82, 0xD8, /* 0xC8-0xCB */ + 0x82, 0xD9, 0x82, 0xDA, 0x82, 0xDB, 0x82, 0xDC, /* 0xCC-0xCF */ + 0xB1, 0xD1, 0xB1, 0xD2, 0x82, 0xDD, 0xB1, 0xD3, /* 0xD0-0xD3 */ + 0x82, 0xDE, 0x82, 0xDF, 0x82, 0xE0, 0x82, 0xE1, /* 0xD4-0xD7 */ + 0x82, 0xE2, 0x82, 0xE3, 0x82, 0xE4, 0x82, 0xE5, /* 0xD8-0xDB */ + 0xB1, 0xD4, 0x82, 0xE6, 0x82, 0xE7, 0x82, 0xE8, /* 0xDC-0xDF */ + 0xB1, 0xD5, 0x82, 0xE9, 0x82, 0xEA, 0x82, 0xEB, /* 0xE0-0xE3 */ + 0xB1, 0xD6, 0x82, 0xEC, 0x82, 0xED, 0x82, 0xEE, /* 0xE4-0xE7 */ + 0x82, 0xEF, 0x82, 0xF0, 0x82, 0xF1, 0x82, 0xF2, /* 0xE8-0xEB */ + 0x82, 0xF3, 0x82, 0xF4, 0x82, 0xF5, 0x82, 0xF6, /* 0xEC-0xEF */ + 0x82, 0xF7, 0x82, 0xF8, 0x82, 0xF9, 0x82, 0xFA, /* 0xF0-0xF3 */ + 0x82, 0xFB, 0x82, 0xFC, 0x82, 0xFD, 0x82, 0xFE, /* 0xF4-0xF7 */ + 0xB1, 0xD7, 0xB1, 0xD8, 0x83, 0x41, 0x83, 0x42, /* 0xF8-0xFB */ + 0xB1, 0xD9, 0x83, 0x43, 0x83, 0x44, 0xB1, 0xDA, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_AE[512] = { + 0xB1, 0xDB, 0xB1, 0xDC, 0x83, 0x45, 0x83, 0x46, /* 0x00-0x03 */ + 0x83, 0x47, 0x83, 0x48, 0x83, 0x49, 0x83, 0x4A, /* 0x04-0x07 */ + 0xB1, 0xDD, 0xB1, 0xDE, 0x83, 0x4B, 0xB1, 0xDF, /* 0x08-0x0B */ + 0x83, 0x4C, 0xB1, 0xE0, 0x83, 0x4D, 0x83, 0x4E, /* 0x0C-0x0F */ + 0x83, 0x4F, 0x83, 0x50, 0x83, 0x51, 0x83, 0x52, /* 0x10-0x13 */ + 0xB1, 0xE1, 0x83, 0x53, 0x83, 0x54, 0x83, 0x55, /* 0x14-0x17 */ + 0x83, 0x56, 0x83, 0x57, 0x83, 0x58, 0x83, 0x59, /* 0x18-0x1B */ + 0x83, 0x5A, 0x83, 0x61, 0x83, 0x62, 0x83, 0x63, /* 0x1C-0x1F */ + 0x83, 0x64, 0x83, 0x65, 0x83, 0x66, 0x83, 0x67, /* 0x20-0x23 */ + 0x83, 0x68, 0x83, 0x69, 0x83, 0x6A, 0x83, 0x6B, /* 0x24-0x27 */ + 0x83, 0x6C, 0x83, 0x6D, 0x83, 0x6E, 0x83, 0x6F, /* 0x28-0x2B */ + 0x83, 0x70, 0x83, 0x71, 0x83, 0x72, 0x83, 0x73, /* 0x2C-0x2F */ + 0xB1, 0xE2, 0xB1, 0xE3, 0x83, 0x74, 0x83, 0x75, /* 0x30-0x33 */ + 0xB1, 0xE4, 0x83, 0x76, 0x83, 0x77, 0xB1, 0xE5, /* 0x34-0x37 */ + 0xB1, 0xE6, 0x83, 0x78, 0xB1, 0xE7, 0x83, 0x79, /* 0x38-0x3B */ + 0x83, 0x7A, 0x83, 0x81, 0x83, 0x82, 0x83, 0x83, /* 0x3C-0x3F */ + 0xB1, 0xE8, 0xB1, 0xE9, 0x83, 0x84, 0xB1, 0xEA, /* 0x40-0x43 */ + 0x83, 0x85, 0xB1, 0xEB, 0xB1, 0xEC, 0x83, 0x86, /* 0x44-0x47 */ + 0x83, 0x87, 0x83, 0x88, 0xB1, 0xED, 0x83, 0x89, /* 0x48-0x4B */ + 0xB1, 0xEE, 0xB1, 0xEF, 0xB1, 0xF0, 0x83, 0x8A, /* 0x4C-0x4F */ + 0xB1, 0xF1, 0x83, 0x8B, 0x83, 0x8C, 0x83, 0x8D, /* 0x50-0x53 */ + 0xB1, 0xF2, 0x83, 0x8E, 0xB1, 0xF3, 0x83, 0x8F, /* 0x54-0x57 */ + 0x83, 0x90, 0x83, 0x91, 0x83, 0x92, 0x83, 0x93, /* 0x58-0x5B */ + 0xB1, 0xF4, 0xB1, 0xF5, 0x83, 0x94, 0xB1, 0xF6, /* 0x5C-0x5F */ + 0xB1, 0xF7, 0xB1, 0xF8, 0x83, 0x95, 0x83, 0x96, /* 0x60-0x63 */ + 0x83, 0x97, 0xB1, 0xF9, 0x83, 0x98, 0x83, 0x99, /* 0x64-0x67 */ + 0xB1, 0xFA, 0xB1, 0xFB, 0x83, 0x9A, 0x83, 0x9B, /* 0x68-0x6B */ + 0xB1, 0xFC, 0x83, 0x9C, 0x83, 0x9D, 0x83, 0x9E, /* 0x6C-0x6F */ + 0xB1, 0xFD, 0x83, 0x9F, 0x83, 0xA0, 0x83, 0xA1, /* 0x70-0x73 */ + 0x83, 0xA2, 0x83, 0xA3, 0x83, 0xA4, 0x83, 0xA5, /* 0x74-0x77 */ + 0xB1, 0xFE, 0xB2, 0xA1, 0x83, 0xA6, 0xB2, 0xA2, /* 0x78-0x7B */ + 0xB2, 0xA3, 0xB2, 0xA4, 0x83, 0xA7, 0x83, 0xA8, /* 0x7C-0x7F */ + + 0x83, 0xA9, 0x83, 0xAA, 0x83, 0xAB, 0x83, 0xAC, /* 0x80-0x83 */ + 0xB2, 0xA5, 0xB2, 0xA6, 0x83, 0xAD, 0x83, 0xAE, /* 0x84-0x87 */ + 0x83, 0xAF, 0x83, 0xB0, 0x83, 0xB1, 0x83, 0xB2, /* 0x88-0x8B */ + 0xB2, 0xA7, 0x83, 0xB3, 0x83, 0xB4, 0x83, 0xB5, /* 0x8C-0x8F */ + 0x83, 0xB6, 0x83, 0xB7, 0x83, 0xB8, 0x83, 0xB9, /* 0x90-0x93 */ + 0x83, 0xBA, 0x83, 0xBB, 0x83, 0xBC, 0x83, 0xBD, /* 0x94-0x97 */ + 0x83, 0xBE, 0x83, 0xBF, 0x83, 0xC0, 0x83, 0xC1, /* 0x98-0x9B */ + 0x83, 0xC2, 0x83, 0xC3, 0x83, 0xC4, 0x83, 0xC5, /* 0x9C-0x9F */ + 0x83, 0xC6, 0x83, 0xC7, 0x83, 0xC8, 0x83, 0xC9, /* 0xA0-0xA3 */ + 0x83, 0xCA, 0x83, 0xCB, 0x83, 0xCC, 0x83, 0xCD, /* 0xA4-0xA7 */ + 0x83, 0xCE, 0x83, 0xCF, 0x83, 0xD0, 0x83, 0xD1, /* 0xA8-0xAB */ + 0x83, 0xD2, 0x83, 0xD3, 0x83, 0xD4, 0x83, 0xD5, /* 0xAC-0xAF */ + 0x83, 0xD6, 0x83, 0xD7, 0x83, 0xD8, 0x83, 0xD9, /* 0xB0-0xB3 */ + 0x83, 0xDA, 0x83, 0xDB, 0x83, 0xDC, 0x83, 0xDD, /* 0xB4-0xB7 */ + 0x83, 0xDE, 0x83, 0xDF, 0x83, 0xE0, 0x83, 0xE1, /* 0xB8-0xBB */ + 0xB2, 0xA8, 0xB2, 0xA9, 0xB2, 0xAA, 0x83, 0xE2, /* 0xBC-0xBF */ + 0xB2, 0xAB, 0x83, 0xE3, 0x83, 0xE4, 0x83, 0xE5, /* 0xC0-0xC3 */ + 0xB2, 0xAC, 0x83, 0xE6, 0x83, 0xE7, 0x83, 0xE8, /* 0xC4-0xC7 */ + 0x83, 0xE9, 0x83, 0xEA, 0x83, 0xEB, 0x83, 0xEC, /* 0xC8-0xCB */ + 0xB2, 0xAD, 0xB2, 0xAE, 0x83, 0xED, 0xB2, 0xAF, /* 0xCC-0xCF */ + 0xB2, 0xB0, 0xB2, 0xB1, 0x83, 0xEE, 0x83, 0xEF, /* 0xD0-0xD3 */ + 0x83, 0xF0, 0x83, 0xF1, 0x83, 0xF2, 0x83, 0xF3, /* 0xD4-0xD7 */ + 0xB2, 0xB2, 0xB2, 0xB3, 0x83, 0xF4, 0x83, 0xF5, /* 0xD8-0xDB */ + 0xB2, 0xB4, 0x83, 0xF6, 0x83, 0xF7, 0x83, 0xF8, /* 0xDC-0xDF */ + 0x83, 0xF9, 0x83, 0xFA, 0x83, 0xFB, 0x83, 0xFC, /* 0xE0-0xE3 */ + 0x83, 0xFD, 0x83, 0xFE, 0x84, 0x41, 0x84, 0x42, /* 0xE4-0xE7 */ + 0xB2, 0xB5, 0x84, 0x43, 0x84, 0x44, 0xB2, 0xB6, /* 0xE8-0xEB */ + 0x84, 0x45, 0xB2, 0xB7, 0x84, 0x46, 0x84, 0x47, /* 0xEC-0xEF */ + 0x84, 0x48, 0x84, 0x49, 0x84, 0x4A, 0x84, 0x4B, /* 0xF0-0xF3 */ + 0xB2, 0xB8, 0x84, 0x4C, 0x84, 0x4D, 0x84, 0x4E, /* 0xF4-0xF7 */ + 0xB2, 0xB9, 0x84, 0x4F, 0x84, 0x50, 0x84, 0x51, /* 0xF8-0xFB */ + 0xB2, 0xBA, 0x84, 0x52, 0x84, 0x53, 0x84, 0x54, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_AF[512] = { + 0x84, 0x55, 0x84, 0x56, 0x84, 0x57, 0x84, 0x58, /* 0x00-0x03 */ + 0x84, 0x59, 0x84, 0x5A, 0x84, 0x61, 0xB2, 0xBB, /* 0x04-0x07 */ + 0xB2, 0xBC, 0x84, 0x62, 0x84, 0x63, 0x84, 0x64, /* 0x08-0x0B */ + 0x84, 0x65, 0xB2, 0xBD, 0x84, 0x66, 0x84, 0x67, /* 0x0C-0x0F */ + 0xB2, 0xBE, 0x84, 0x68, 0x84, 0x69, 0x84, 0x6A, /* 0x10-0x13 */ + 0x84, 0x6B, 0x84, 0x6C, 0x84, 0x6D, 0x84, 0x6E, /* 0x14-0x17 */ + 0x84, 0x6F, 0x84, 0x70, 0x84, 0x71, 0x84, 0x72, /* 0x18-0x1B */ + 0x84, 0x73, 0x84, 0x74, 0x84, 0x75, 0x84, 0x76, /* 0x1C-0x1F */ + 0x84, 0x77, 0x84, 0x78, 0x84, 0x79, 0x84, 0x7A, /* 0x20-0x23 */ + 0x84, 0x81, 0x84, 0x82, 0x84, 0x83, 0x84, 0x84, /* 0x24-0x27 */ + 0x84, 0x85, 0x84, 0x86, 0x84, 0x87, 0x84, 0x88, /* 0x28-0x2B */ + 0xB2, 0xBF, 0xB2, 0xC0, 0x84, 0x89, 0x84, 0x8A, /* 0x2C-0x2F */ + 0xB2, 0xC1, 0x84, 0x8B, 0xB2, 0xC2, 0x84, 0x8C, /* 0x30-0x33 */ + 0xB2, 0xC3, 0x84, 0x8D, 0x84, 0x8E, 0x84, 0x8F, /* 0x34-0x37 */ + 0x84, 0x90, 0x84, 0x91, 0x84, 0x92, 0x84, 0x93, /* 0x38-0x3B */ + 0xB2, 0xC4, 0xB2, 0xC5, 0x84, 0x94, 0xB2, 0xC6, /* 0x3C-0x3F */ + 0x84, 0x95, 0xB2, 0xC7, 0xB2, 0xC8, 0xB2, 0xC9, /* 0x40-0x43 */ + 0x84, 0x96, 0x84, 0x97, 0x84, 0x98, 0x84, 0x99, /* 0x44-0x47 */ + 0xB2, 0xCA, 0xB2, 0xCB, 0x84, 0x9A, 0x84, 0x9B, /* 0x48-0x4B */ + 0x84, 0x9C, 0x84, 0x9D, 0x84, 0x9E, 0x84, 0x9F, /* 0x4C-0x4F */ + 0xB2, 0xCC, 0x84, 0xA0, 0x84, 0xA1, 0x84, 0xA2, /* 0x50-0x53 */ + 0x84, 0xA3, 0x84, 0xA4, 0x84, 0xA5, 0x84, 0xA6, /* 0x54-0x57 */ + 0x84, 0xA7, 0x84, 0xA8, 0x84, 0xA9, 0x84, 0xAA, /* 0x58-0x5B */ + 0xB2, 0xCD, 0xB2, 0xCE, 0x84, 0xAB, 0x84, 0xAC, /* 0x5C-0x5F */ + 0x84, 0xAD, 0x84, 0xAE, 0x84, 0xAF, 0x84, 0xB0, /* 0x60-0x63 */ + 0xB2, 0xCF, 0xB2, 0xD0, 0x84, 0xB1, 0x84, 0xB2, /* 0x64-0x67 */ + 0x84, 0xB3, 0x84, 0xB4, 0x84, 0xB5, 0x84, 0xB6, /* 0x68-0x6B */ + 0x84, 0xB7, 0x84, 0xB8, 0x84, 0xB9, 0x84, 0xBA, /* 0x6C-0x6F */ + 0x84, 0xBB, 0x84, 0xBC, 0x84, 0xBD, 0x84, 0xBE, /* 0x70-0x73 */ + 0x84, 0xBF, 0x84, 0xC0, 0x84, 0xC1, 0x84, 0xC2, /* 0x74-0x77 */ + 0x84, 0xC3, 0xB2, 0xD1, 0x84, 0xC4, 0x84, 0xC5, /* 0x78-0x7B */ + 0x84, 0xC6, 0x84, 0xC7, 0x84, 0xC8, 0x84, 0xC9, /* 0x7C-0x7F */ + + 0xB2, 0xD2, 0x84, 0xCA, 0x84, 0xCB, 0x84, 0xCC, /* 0x80-0x83 */ + 0xB2, 0xD3, 0x84, 0xCD, 0x84, 0xCE, 0x84, 0xCF, /* 0x84-0x87 */ + 0xB2, 0xD4, 0x84, 0xD0, 0x84, 0xD1, 0x84, 0xD2, /* 0x88-0x8B */ + 0x84, 0xD3, 0x84, 0xD4, 0x84, 0xD5, 0x84, 0xD6, /* 0x8C-0x8F */ + 0xB2, 0xD5, 0xB2, 0xD6, 0x84, 0xD7, 0x84, 0xD8, /* 0x90-0x93 */ + 0x84, 0xD9, 0xB2, 0xD7, 0x84, 0xDA, 0x84, 0xDB, /* 0x94-0x97 */ + 0x84, 0xDC, 0x84, 0xDD, 0x84, 0xDE, 0x84, 0xDF, /* 0x98-0x9B */ + 0xB2, 0xD8, 0x84, 0xE0, 0x84, 0xE1, 0x84, 0xE2, /* 0x9C-0x9F */ + 0x84, 0xE3, 0x84, 0xE4, 0x84, 0xE5, 0x84, 0xE6, /* 0xA0-0xA3 */ + 0x84, 0xE7, 0x84, 0xE8, 0x84, 0xE9, 0x84, 0xEA, /* 0xA4-0xA7 */ + 0x84, 0xEB, 0x84, 0xEC, 0x84, 0xED, 0x84, 0xEE, /* 0xA8-0xAB */ + 0x84, 0xEF, 0x84, 0xF0, 0x84, 0xF1, 0x84, 0xF2, /* 0xAC-0xAF */ + 0x84, 0xF3, 0x84, 0xF4, 0x84, 0xF5, 0x84, 0xF6, /* 0xB0-0xB3 */ + 0x84, 0xF7, 0x84, 0xF8, 0x84, 0xF9, 0x84, 0xFA, /* 0xB4-0xB7 */ + 0xB2, 0xD9, 0xB2, 0xDA, 0x84, 0xFB, 0x84, 0xFC, /* 0xB8-0xBB */ + 0xB2, 0xDB, 0x84, 0xFD, 0x84, 0xFE, 0x85, 0x41, /* 0xBC-0xBF */ + 0xB2, 0xDC, 0x85, 0x42, 0x85, 0x43, 0x85, 0x44, /* 0xC0-0xC3 */ + 0x85, 0x45, 0x85, 0x46, 0x85, 0x47, 0xB2, 0xDD, /* 0xC4-0xC7 */ + 0xB2, 0xDE, 0xB2, 0xDF, 0x85, 0x48, 0xB2, 0xE0, /* 0xC8-0xCB */ + 0x85, 0x49, 0xB2, 0xE1, 0xB2, 0xE2, 0x85, 0x4A, /* 0xCC-0xCF */ + 0x85, 0x4B, 0x85, 0x4C, 0x85, 0x4D, 0x85, 0x4E, /* 0xD0-0xD3 */ + 0xB2, 0xE3, 0x85, 0x4F, 0x85, 0x50, 0x85, 0x51, /* 0xD4-0xD7 */ + 0x85, 0x52, 0x85, 0x53, 0x85, 0x54, 0x85, 0x55, /* 0xD8-0xDB */ + 0xB2, 0xE4, 0x85, 0x56, 0x85, 0x57, 0x85, 0x58, /* 0xDC-0xDF */ + 0x85, 0x59, 0x85, 0x5A, 0x85, 0x61, 0x85, 0x62, /* 0xE0-0xE3 */ + 0x85, 0x63, 0x85, 0x64, 0x85, 0x65, 0x85, 0x66, /* 0xE4-0xE7 */ + 0xB2, 0xE5, 0xB2, 0xE6, 0x85, 0x67, 0x85, 0x68, /* 0xE8-0xEB */ + 0x85, 0x69, 0x85, 0x6A, 0x85, 0x6B, 0x85, 0x6C, /* 0xEC-0xEF */ + 0xB2, 0xE7, 0xB2, 0xE8, 0x85, 0x6D, 0x85, 0x6E, /* 0xF0-0xF3 */ + 0xB2, 0xE9, 0x85, 0x6F, 0x85, 0x70, 0x85, 0x71, /* 0xF4-0xF7 */ + 0xB2, 0xEA, 0x85, 0x72, 0x85, 0x73, 0x85, 0x74, /* 0xF8-0xFB */ + 0x85, 0x75, 0x85, 0x76, 0x85, 0x77, 0x85, 0x78, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_B0[512] = { + 0xB2, 0xEB, 0xB2, 0xEC, 0x85, 0x79, 0x85, 0x7A, /* 0x00-0x03 */ + 0xB2, 0xED, 0x85, 0x81, 0x85, 0x82, 0x85, 0x83, /* 0x04-0x07 */ + 0x85, 0x84, 0x85, 0x85, 0x85, 0x86, 0x85, 0x87, /* 0x08-0x0B */ + 0xB2, 0xEE, 0x85, 0x88, 0x85, 0x89, 0x85, 0x8A, /* 0x0C-0x0F */ + 0xB2, 0xEF, 0x85, 0x8B, 0x85, 0x8C, 0x85, 0x8D, /* 0x10-0x13 */ + 0xB2, 0xF0, 0x85, 0x8E, 0x85, 0x8F, 0x85, 0x90, /* 0x14-0x17 */ + 0x85, 0x91, 0x85, 0x92, 0x85, 0x93, 0x85, 0x94, /* 0x18-0x1B */ + 0xB2, 0xF1, 0xB2, 0xF2, 0x85, 0x95, 0x85, 0x96, /* 0x1C-0x1F */ + 0x85, 0x97, 0x85, 0x98, 0x85, 0x99, 0x85, 0x9A, /* 0x20-0x23 */ + 0x85, 0x9B, 0x85, 0x9C, 0x85, 0x9D, 0x85, 0x9E, /* 0x24-0x27 */ + 0xB2, 0xF3, 0x85, 0x9F, 0x85, 0xA0, 0x85, 0xA1, /* 0x28-0x2B */ + 0x85, 0xA2, 0x85, 0xA3, 0x85, 0xA4, 0x85, 0xA5, /* 0x2C-0x2F */ + 0x85, 0xA6, 0x85, 0xA7, 0x85, 0xA8, 0x85, 0xA9, /* 0x30-0x33 */ + 0x85, 0xAA, 0x85, 0xAB, 0x85, 0xAC, 0x85, 0xAD, /* 0x34-0x37 */ + 0x85, 0xAE, 0x85, 0xAF, 0x85, 0xB0, 0x85, 0xB1, /* 0x38-0x3B */ + 0x85, 0xB2, 0x85, 0xB3, 0x85, 0xB4, 0x85, 0xB5, /* 0x3C-0x3F */ + 0x85, 0xB6, 0x85, 0xB7, 0x85, 0xB8, 0x85, 0xB9, /* 0x40-0x43 */ + 0xB2, 0xF4, 0xB2, 0xF5, 0x85, 0xBA, 0x85, 0xBB, /* 0x44-0x47 */ + 0xB2, 0xF6, 0x85, 0xBC, 0xB2, 0xF7, 0x85, 0xBD, /* 0x48-0x4B */ + 0xB2, 0xF8, 0x85, 0xBE, 0xB2, 0xF9, 0x85, 0xBF, /* 0x4C-0x4F */ + 0x85, 0xC0, 0x85, 0xC1, 0x85, 0xC2, 0xB2, 0xFA, /* 0x50-0x53 */ + 0xB2, 0xFB, 0xB2, 0xFC, 0x85, 0xC3, 0xB2, 0xFD, /* 0x54-0x57 */ + 0x85, 0xC4, 0xB2, 0xFE, 0x85, 0xC5, 0x85, 0xC6, /* 0x58-0x5B */ + 0x85, 0xC7, 0xB3, 0xA1, 0x85, 0xC8, 0x85, 0xC9, /* 0x5C-0x5F */ + 0x85, 0xCA, 0x85, 0xCB, 0x85, 0xCC, 0x85, 0xCD, /* 0x60-0x63 */ + 0x85, 0xCE, 0x85, 0xCF, 0x85, 0xD0, 0x85, 0xD1, /* 0x64-0x67 */ + 0x85, 0xD2, 0x85, 0xD3, 0x85, 0xD4, 0x85, 0xD5, /* 0x68-0x6B */ + 0x85, 0xD6, 0x85, 0xD7, 0x85, 0xD8, 0x85, 0xD9, /* 0x6C-0x6F */ + 0x85, 0xDA, 0x85, 0xDB, 0x85, 0xDC, 0x85, 0xDD, /* 0x70-0x73 */ + 0x85, 0xDE, 0x85, 0xDF, 0x85, 0xE0, 0x85, 0xE1, /* 0x74-0x77 */ + 0x85, 0xE2, 0x85, 0xE3, 0x85, 0xE4, 0x85, 0xE5, /* 0x78-0x7B */ + 0xB3, 0xA2, 0xB3, 0xA3, 0x85, 0xE6, 0x85, 0xE7, /* 0x7C-0x7F */ + + 0xB3, 0xA4, 0x85, 0xE8, 0x85, 0xE9, 0x85, 0xEA, /* 0x80-0x83 */ + 0xB3, 0xA5, 0x85, 0xEB, 0x85, 0xEC, 0x85, 0xED, /* 0x84-0x87 */ + 0x85, 0xEE, 0x85, 0xEF, 0x85, 0xF0, 0x85, 0xF1, /* 0x88-0x8B */ + 0xB3, 0xA6, 0xB3, 0xA7, 0x85, 0xF2, 0xB3, 0xA8, /* 0x8C-0x8F */ + 0x85, 0xF3, 0xB3, 0xA9, 0x85, 0xF4, 0x85, 0xF5, /* 0x90-0x93 */ + 0x85, 0xF6, 0x85, 0xF7, 0x85, 0xF8, 0x85, 0xF9, /* 0x94-0x97 */ + 0xB3, 0xAA, 0xB3, 0xAB, 0xB3, 0xAC, 0x85, 0xFA, /* 0x98-0x9B */ + 0xB3, 0xAD, 0x85, 0xFB, 0x85, 0xFC, 0xB3, 0xAE, /* 0x9C-0x9F */ + 0xB3, 0xAF, 0xB3, 0xB0, 0xB3, 0xB1, 0x85, 0xFD, /* 0xA0-0xA3 */ + 0x85, 0xFE, 0x86, 0x41, 0x86, 0x42, 0x86, 0x43, /* 0xA4-0xA7 */ + 0xB3, 0xB2, 0xB3, 0xB3, 0x86, 0x44, 0xB3, 0xB4, /* 0xA8-0xAB */ + 0xB3, 0xB5, 0xB3, 0xB6, 0xB3, 0xB7, 0xB3, 0xB8, /* 0xAC-0xAF */ + 0x86, 0x45, 0xB3, 0xB9, 0x86, 0x46, 0xB3, 0xBA, /* 0xB0-0xB3 */ + 0xB3, 0xBB, 0xB3, 0xBC, 0x86, 0x47, 0x86, 0x48, /* 0xB4-0xB7 */ + 0xB3, 0xBD, 0x86, 0x49, 0x86, 0x4A, 0x86, 0x4B, /* 0xB8-0xBB */ + 0xB3, 0xBE, 0x86, 0x4C, 0x86, 0x4D, 0x86, 0x4E, /* 0xBC-0xBF */ + 0x86, 0x4F, 0x86, 0x50, 0x86, 0x51, 0x86, 0x52, /* 0xC0-0xC3 */ + 0xB3, 0xBF, 0xB3, 0xC0, 0x86, 0x53, 0xB3, 0xC1, /* 0xC4-0xC7 */ + 0xB3, 0xC2, 0xB3, 0xC3, 0x86, 0x54, 0x86, 0x55, /* 0xC8-0xCB */ + 0x86, 0x56, 0x86, 0x57, 0x86, 0x58, 0x86, 0x59, /* 0xCC-0xCF */ + 0xB3, 0xC4, 0xB3, 0xC5, 0x86, 0x5A, 0x86, 0x61, /* 0xD0-0xD3 */ + 0xB3, 0xC6, 0x86, 0x62, 0x86, 0x63, 0x86, 0x64, /* 0xD4-0xD7 */ + 0xB3, 0xC7, 0x86, 0x65, 0x86, 0x66, 0x86, 0x67, /* 0xD8-0xDB */ + 0x86, 0x68, 0x86, 0x69, 0x86, 0x6A, 0x86, 0x6B, /* 0xDC-0xDF */ + 0xB3, 0xC8, 0x86, 0x6C, 0x86, 0x6D, 0x86, 0x6E, /* 0xE0-0xE3 */ + 0x86, 0x6F, 0xB3, 0xC9, 0x86, 0x70, 0x86, 0x71, /* 0xE4-0xE7 */ + 0x86, 0x72, 0x86, 0x73, 0x86, 0x74, 0x86, 0x75, /* 0xE8-0xEB */ + 0x86, 0x76, 0x86, 0x77, 0x86, 0x78, 0x86, 0x79, /* 0xEC-0xEF */ + 0x86, 0x7A, 0x86, 0x81, 0x86, 0x82, 0x86, 0x83, /* 0xF0-0xF3 */ + 0x86, 0x84, 0x86, 0x85, 0x86, 0x86, 0x86, 0x87, /* 0xF4-0xF7 */ + 0x86, 0x88, 0x86, 0x89, 0x86, 0x8A, 0x86, 0x8B, /* 0xF8-0xFB */ + 0x86, 0x8C, 0x86, 0x8D, 0x86, 0x8E, 0x86, 0x8F, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_B1[512] = { + 0x86, 0x90, 0x86, 0x91, 0x86, 0x92, 0x86, 0x93, /* 0x00-0x03 */ + 0x86, 0x94, 0x86, 0x95, 0x86, 0x96, 0x86, 0x97, /* 0x04-0x07 */ + 0xB3, 0xCA, 0xB3, 0xCB, 0x86, 0x98, 0xB3, 0xCC, /* 0x08-0x0B */ + 0xB3, 0xCD, 0x86, 0x99, 0x86, 0x9A, 0x86, 0x9B, /* 0x0C-0x0F */ + 0xB3, 0xCE, 0x86, 0x9C, 0xB3, 0xCF, 0xB3, 0xD0, /* 0x10-0x13 */ + 0x86, 0x9D, 0x86, 0x9E, 0x86, 0x9F, 0x86, 0xA0, /* 0x14-0x17 */ + 0xB3, 0xD1, 0xB3, 0xD2, 0x86, 0xA1, 0xB3, 0xD3, /* 0x18-0x1B */ + 0xB3, 0xD4, 0xB3, 0xD5, 0x86, 0xA2, 0x86, 0xA3, /* 0x1C-0x1F */ + 0x86, 0xA4, 0x86, 0xA5, 0x86, 0xA6, 0xB3, 0xD6, /* 0x20-0x23 */ + 0xB3, 0xD7, 0xB3, 0xD8, 0x86, 0xA7, 0x86, 0xA8, /* 0x24-0x27 */ + 0xB3, 0xD9, 0x86, 0xA9, 0x86, 0xAA, 0x86, 0xAB, /* 0x28-0x2B */ + 0xB3, 0xDA, 0x86, 0xAC, 0x86, 0xAD, 0x86, 0xAE, /* 0x2C-0x2F */ + 0x86, 0xAF, 0x86, 0xB0, 0x86, 0xB1, 0x86, 0xB2, /* 0x30-0x33 */ + 0xB3, 0xDB, 0xB3, 0xDC, 0x86, 0xB3, 0xB3, 0xDD, /* 0x34-0x37 */ + 0xB3, 0xDE, 0xB3, 0xDF, 0x86, 0xB4, 0x86, 0xB5, /* 0x38-0x3B */ + 0x86, 0xB6, 0x86, 0xB7, 0x86, 0xB8, 0x86, 0xB9, /* 0x3C-0x3F */ + 0xB3, 0xE0, 0xB3, 0xE1, 0x86, 0xBA, 0x86, 0xBB, /* 0x40-0x43 */ + 0xB3, 0xE2, 0x86, 0xBC, 0x86, 0xBD, 0x86, 0xBE, /* 0x44-0x47 */ + 0xB3, 0xE3, 0x86, 0xBF, 0x86, 0xC0, 0x86, 0xC1, /* 0x48-0x4B */ + 0x86, 0xC2, 0x86, 0xC3, 0x86, 0xC4, 0x86, 0xC5, /* 0x4C-0x4F */ + 0xB3, 0xE4, 0xB3, 0xE5, 0x86, 0xC6, 0x86, 0xC7, /* 0x50-0x53 */ + 0xB3, 0xE6, 0xB3, 0xE7, 0x86, 0xC8, 0x86, 0xC9, /* 0x54-0x57 */ + 0xB3, 0xE8, 0x86, 0xCA, 0x86, 0xCB, 0x86, 0xCC, /* 0x58-0x5B */ + 0xB3, 0xE9, 0x86, 0xCD, 0x86, 0xCE, 0x86, 0xCF, /* 0x5C-0x5F */ + 0xB3, 0xEA, 0x86, 0xD0, 0x86, 0xD1, 0x86, 0xD2, /* 0x60-0x63 */ + 0x86, 0xD3, 0x86, 0xD4, 0x86, 0xD5, 0x86, 0xD6, /* 0x64-0x67 */ + 0x86, 0xD7, 0x86, 0xD8, 0x86, 0xD9, 0x86, 0xDA, /* 0x68-0x6B */ + 0x86, 0xDB, 0x86, 0xDC, 0x86, 0xDD, 0x86, 0xDE, /* 0x6C-0x6F */ + 0x86, 0xDF, 0x86, 0xE0, 0x86, 0xE1, 0x86, 0xE2, /* 0x70-0x73 */ + 0x86, 0xE3, 0x86, 0xE4, 0x86, 0xE5, 0x86, 0xE6, /* 0x74-0x77 */ + 0xB3, 0xEB, 0xB3, 0xEC, 0x86, 0xE7, 0x86, 0xE8, /* 0x78-0x7B */ + 0xB3, 0xED, 0x86, 0xE9, 0x86, 0xEA, 0x86, 0xEB, /* 0x7C-0x7F */ + + 0xB3, 0xEE, 0x86, 0xEC, 0xB3, 0xEF, 0x86, 0xED, /* 0x80-0x83 */ + 0x86, 0xEE, 0x86, 0xEF, 0x86, 0xF0, 0x86, 0xF1, /* 0x84-0x87 */ + 0xB3, 0xF0, 0xB3, 0xF1, 0x86, 0xF2, 0xB3, 0xF2, /* 0x88-0x8B */ + 0x86, 0xF3, 0xB3, 0xF3, 0x86, 0xF4, 0x86, 0xF5, /* 0x8C-0x8F */ + 0x86, 0xF6, 0x86, 0xF7, 0xB3, 0xF4, 0xB3, 0xF5, /* 0x90-0x93 */ + 0xB3, 0xF6, 0x86, 0xF8, 0x86, 0xF9, 0x86, 0xFA, /* 0x94-0x97 */ + 0xB3, 0xF7, 0x86, 0xFB, 0x86, 0xFC, 0x86, 0xFD, /* 0x98-0x9B */ + 0xB3, 0xF8, 0x86, 0xFE, 0x87, 0x41, 0x87, 0x42, /* 0x9C-0x9F */ + 0x87, 0x43, 0x87, 0x44, 0x87, 0x45, 0x87, 0x46, /* 0xA0-0xA3 */ + 0x87, 0x47, 0x87, 0x48, 0x87, 0x49, 0x87, 0x4A, /* 0xA4-0xA7 */ + 0xB3, 0xF9, 0x87, 0x4B, 0x87, 0x4C, 0x87, 0x4D, /* 0xA8-0xAB */ + 0x87, 0x4E, 0x87, 0x4F, 0x87, 0x50, 0x87, 0x51, /* 0xAC-0xAF */ + 0x87, 0x52, 0x87, 0x53, 0x87, 0x54, 0x87, 0x55, /* 0xB0-0xB3 */ + 0x87, 0x56, 0x87, 0x57, 0x87, 0x58, 0x87, 0x59, /* 0xB4-0xB7 */ + 0x87, 0x5A, 0x87, 0x61, 0x87, 0x62, 0x87, 0x63, /* 0xB8-0xBB */ + 0x87, 0x64, 0x87, 0x65, 0x87, 0x66, 0x87, 0x67, /* 0xBC-0xBF */ + 0x87, 0x68, 0x87, 0x69, 0x87, 0x6A, 0x87, 0x6B, /* 0xC0-0xC3 */ + 0x87, 0x6C, 0x87, 0x6D, 0x87, 0x6E, 0x87, 0x6F, /* 0xC4-0xC7 */ + 0x87, 0x70, 0x87, 0x71, 0x87, 0x72, 0x87, 0x73, /* 0xC8-0xCB */ + 0xB3, 0xFA, 0x87, 0x74, 0x87, 0x75, 0x87, 0x76, /* 0xCC-0xCF */ + 0xB3, 0xFB, 0x87, 0x77, 0x87, 0x78, 0x87, 0x79, /* 0xD0-0xD3 */ + 0xB3, 0xFC, 0x87, 0x7A, 0x87, 0x81, 0x87, 0x82, /* 0xD4-0xD7 */ + 0x87, 0x83, 0x87, 0x84, 0x87, 0x85, 0x87, 0x86, /* 0xD8-0xDB */ + 0xB3, 0xFD, 0xB3, 0xFE, 0x87, 0x87, 0xB4, 0xA1, /* 0xDC-0xDF */ + 0x87, 0x88, 0x87, 0x89, 0x87, 0x8A, 0x87, 0x8B, /* 0xE0-0xE3 */ + 0x87, 0x8C, 0x87, 0x8D, 0x87, 0x8E, 0x87, 0x8F, /* 0xE4-0xE7 */ + 0xB4, 0xA2, 0xB4, 0xA3, 0x87, 0x90, 0x87, 0x91, /* 0xE8-0xEB */ + 0xB4, 0xA4, 0x87, 0x92, 0x87, 0x93, 0x87, 0x94, /* 0xEC-0xEF */ + 0xB4, 0xA5, 0x87, 0x95, 0x87, 0x96, 0x87, 0x97, /* 0xF0-0xF3 */ + 0x87, 0x98, 0x87, 0x99, 0x87, 0x9A, 0x87, 0x9B, /* 0xF4-0xF7 */ + 0x87, 0x9C, 0xB4, 0xA6, 0x87, 0x9D, 0xB4, 0xA7, /* 0xF8-0xFB */ + 0x87, 0x9E, 0xB4, 0xA8, 0x87, 0x9F, 0x87, 0xA0, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_B2[512] = { + 0x87, 0xA1, 0x87, 0xA2, 0x87, 0xA3, 0x87, 0xA4, /* 0x00-0x03 */ + 0xB4, 0xA9, 0xB4, 0xAA, 0x87, 0xA5, 0x87, 0xA6, /* 0x04-0x07 */ + 0xB4, 0xAB, 0x87, 0xA7, 0x87, 0xA8, 0xB4, 0xAC, /* 0x08-0x0B */ + 0xB4, 0xAD, 0x87, 0xA9, 0x87, 0xAA, 0x87, 0xAB, /* 0x0C-0x0F */ + 0x87, 0xAC, 0x87, 0xAD, 0x87, 0xAE, 0x87, 0xAF, /* 0x10-0x13 */ + 0xB4, 0xAE, 0xB4, 0xAF, 0x87, 0xB0, 0xB4, 0xB0, /* 0x14-0x17 */ + 0x87, 0xB1, 0xB4, 0xB1, 0x87, 0xB2, 0x87, 0xB3, /* 0x18-0x1B */ + 0x87, 0xB4, 0x87, 0xB5, 0x87, 0xB6, 0x87, 0xB7, /* 0x1C-0x1F */ + 0xB4, 0xB2, 0x87, 0xB8, 0x87, 0xB9, 0x87, 0xBA, /* 0x20-0x23 */ + 0x87, 0xBB, 0x87, 0xBC, 0x87, 0xBD, 0x87, 0xBE, /* 0x24-0x27 */ + 0x87, 0xBF, 0x87, 0xC0, 0x87, 0xC1, 0x87, 0xC2, /* 0x28-0x2B */ + 0x87, 0xC3, 0x87, 0xC4, 0x87, 0xC5, 0x87, 0xC6, /* 0x2C-0x2F */ + 0x87, 0xC7, 0x87, 0xC8, 0x87, 0xC9, 0x87, 0xCA, /* 0x30-0x33 */ + 0xB4, 0xB3, 0x87, 0xCB, 0x87, 0xCC, 0x87, 0xCD, /* 0x34-0x37 */ + 0x87, 0xCE, 0x87, 0xCF, 0x87, 0xD0, 0x87, 0xD1, /* 0x38-0x3B */ + 0xB4, 0xB4, 0x87, 0xD2, 0x87, 0xD3, 0x87, 0xD4, /* 0x3C-0x3F */ + 0x87, 0xD5, 0x87, 0xD6, 0x87, 0xD7, 0x87, 0xD8, /* 0x40-0x43 */ + 0x87, 0xD9, 0x87, 0xDA, 0x87, 0xDB, 0x87, 0xDC, /* 0x44-0x47 */ + 0x87, 0xDD, 0x87, 0xDE, 0x87, 0xDF, 0x87, 0xE0, /* 0x48-0x4B */ + 0x87, 0xE1, 0x87, 0xE2, 0x87, 0xE3, 0x87, 0xE4, /* 0x4C-0x4F */ + 0x87, 0xE5, 0x87, 0xE6, 0x87, 0xE7, 0x87, 0xE8, /* 0x50-0x53 */ + 0x87, 0xE9, 0x87, 0xEA, 0x87, 0xEB, 0x87, 0xEC, /* 0x54-0x57 */ + 0xB4, 0xB5, 0x87, 0xED, 0x87, 0xEE, 0x87, 0xEF, /* 0x58-0x5B */ + 0xB4, 0xB6, 0x87, 0xF0, 0x87, 0xF1, 0x87, 0xF2, /* 0x5C-0x5F */ + 0xB4, 0xB7, 0x87, 0xF3, 0x87, 0xF4, 0x87, 0xF5, /* 0x60-0x63 */ + 0x87, 0xF6, 0x87, 0xF7, 0x87, 0xF8, 0x87, 0xF9, /* 0x64-0x67 */ + 0xB4, 0xB8, 0xB4, 0xB9, 0x87, 0xFA, 0x87, 0xFB, /* 0x68-0x6B */ + 0x87, 0xFC, 0x87, 0xFD, 0x87, 0xFE, 0x88, 0x41, /* 0x6C-0x6F */ + 0x88, 0x42, 0x88, 0x43, 0x88, 0x44, 0x88, 0x45, /* 0x70-0x73 */ + 0xB4, 0xBA, 0xB4, 0xBB, 0x88, 0x46, 0x88, 0x47, /* 0x74-0x77 */ + 0x88, 0x48, 0x88, 0x49, 0x88, 0x4A, 0x88, 0x4B, /* 0x78-0x7B */ + 0xB4, 0xBC, 0x88, 0x4C, 0x88, 0x4D, 0x88, 0x4E, /* 0x7C-0x7F */ + + 0x88, 0x4F, 0x88, 0x50, 0x88, 0x51, 0x88, 0x52, /* 0x80-0x83 */ + 0xB4, 0xBD, 0xB4, 0xBE, 0x88, 0x53, 0x88, 0x54, /* 0x84-0x87 */ + 0x88, 0x55, 0xB4, 0xBF, 0x88, 0x56, 0x88, 0x57, /* 0x88-0x8B */ + 0x88, 0x58, 0x88, 0x59, 0x88, 0x5A, 0x88, 0x61, /* 0x8C-0x8F */ + 0xB4, 0xC0, 0xB4, 0xC1, 0x88, 0x62, 0x88, 0x63, /* 0x90-0x93 */ + 0xB4, 0xC2, 0x88, 0x64, 0x88, 0x65, 0x88, 0x66, /* 0x94-0x97 */ + 0xB4, 0xC3, 0xB4, 0xC4, 0xB4, 0xC5, 0x88, 0x67, /* 0x98-0x9B */ + 0x88, 0x68, 0x88, 0x69, 0x88, 0x6A, 0x88, 0x6B, /* 0x9C-0x9F */ + 0xB4, 0xC6, 0xB4, 0xC7, 0x88, 0x6C, 0xB4, 0xC8, /* 0xA0-0xA3 */ + 0x88, 0x6D, 0xB4, 0xC9, 0xB4, 0xCA, 0x88, 0x6E, /* 0xA4-0xA7 */ + 0x88, 0x6F, 0x88, 0x70, 0xB4, 0xCB, 0x88, 0x71, /* 0xA8-0xAB */ + 0xB4, 0xCC, 0x88, 0x72, 0x88, 0x73, 0x88, 0x74, /* 0xAC-0xAF */ + 0xB4, 0xCD, 0x88, 0x75, 0x88, 0x76, 0x88, 0x77, /* 0xB0-0xB3 */ + 0xB4, 0xCE, 0x88, 0x78, 0x88, 0x79, 0x88, 0x7A, /* 0xB4-0xB7 */ + 0x88, 0x81, 0x88, 0x82, 0x88, 0x83, 0x88, 0x84, /* 0xB8-0xBB */ + 0x88, 0x85, 0x88, 0x86, 0x88, 0x87, 0x88, 0x88, /* 0xBC-0xBF */ + 0x88, 0x89, 0x88, 0x8A, 0x88, 0x8B, 0x88, 0x8C, /* 0xC0-0xC3 */ + 0x88, 0x8D, 0x88, 0x8E, 0x88, 0x8F, 0x88, 0x90, /* 0xC4-0xC7 */ + 0xB4, 0xCF, 0xB4, 0xD0, 0x88, 0x91, 0x88, 0x92, /* 0xC8-0xCB */ + 0xB4, 0xD1, 0x88, 0x93, 0x88, 0x94, 0x88, 0x95, /* 0xCC-0xCF */ + 0xB4, 0xD2, 0x88, 0x96, 0xB4, 0xD3, 0x88, 0x97, /* 0xD0-0xD3 */ + 0x88, 0x98, 0x88, 0x99, 0x88, 0x9A, 0x88, 0x9B, /* 0xD4-0xD7 */ + 0xB4, 0xD4, 0xB4, 0xD5, 0x88, 0x9C, 0xB4, 0xD6, /* 0xD8-0xDB */ + 0x88, 0x9D, 0xB4, 0xD7, 0x88, 0x9E, 0x88, 0x9F, /* 0xDC-0xDF */ + 0x88, 0xA0, 0x88, 0xA1, 0xB4, 0xD8, 0x88, 0xA2, /* 0xE0-0xE3 */ + 0xB4, 0xD9, 0xB4, 0xDA, 0xB4, 0xDB, 0x88, 0xA3, /* 0xE4-0xE7 */ + 0xB4, 0xDC, 0x88, 0xA4, 0x88, 0xA5, 0xB4, 0xDD, /* 0xE8-0xEB */ + 0xB4, 0xDE, 0xB4, 0xDF, 0xB4, 0xE0, 0xB4, 0xE1, /* 0xEC-0xEF */ + 0x88, 0xA6, 0x88, 0xA7, 0x88, 0xA8, 0xB4, 0xE2, /* 0xF0-0xF3 */ + 0xB4, 0xE3, 0xB4, 0xE4, 0x88, 0xA9, 0xB4, 0xE5, /* 0xF4-0xF7 */ + 0xB4, 0xE6, 0xB4, 0xE7, 0xB4, 0xE8, 0xB4, 0xE9, /* 0xF8-0xFB */ + 0x88, 0xAA, 0x88, 0xAB, 0x88, 0xAC, 0xB4, 0xEA, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_B3[512] = { + 0xB4, 0xEB, 0xB4, 0xEC, 0x88, 0xAD, 0x88, 0xAE, /* 0x00-0x03 */ + 0xB4, 0xED, 0x88, 0xAF, 0x88, 0xB0, 0x88, 0xB1, /* 0x04-0x07 */ + 0xB4, 0xEE, 0x88, 0xB2, 0x88, 0xB3, 0x88, 0xB4, /* 0x08-0x0B */ + 0x88, 0xB5, 0x88, 0xB6, 0x88, 0xB7, 0x88, 0xB8, /* 0x0C-0x0F */ + 0xB4, 0xEF, 0xB4, 0xF0, 0x88, 0xB9, 0xB4, 0xF1, /* 0x10-0x13 */ + 0xB4, 0xF2, 0xB4, 0xF3, 0x88, 0xBA, 0x88, 0xBB, /* 0x14-0x17 */ + 0x88, 0xBC, 0x88, 0xBD, 0x88, 0xBE, 0x88, 0xBF, /* 0x18-0x1B */ + 0xB4, 0xF4, 0x88, 0xC0, 0x88, 0xC1, 0x88, 0xC2, /* 0x1C-0x1F */ + 0x88, 0xC3, 0x88, 0xC4, 0x88, 0xC5, 0x88, 0xC6, /* 0x20-0x23 */ + 0x88, 0xC7, 0x88, 0xC8, 0x88, 0xC9, 0x88, 0xCA, /* 0x24-0x27 */ + 0x88, 0xCB, 0x88, 0xCC, 0x88, 0xCD, 0x88, 0xCE, /* 0x28-0x2B */ + 0x88, 0xCF, 0x88, 0xD0, 0x88, 0xD1, 0x88, 0xD2, /* 0x2C-0x2F */ + 0x88, 0xD3, 0x88, 0xD4, 0x88, 0xD5, 0x88, 0xD6, /* 0x30-0x33 */ + 0x88, 0xD7, 0x88, 0xD8, 0x88, 0xD9, 0x88, 0xDA, /* 0x34-0x37 */ + 0x88, 0xDB, 0x88, 0xDC, 0x88, 0xDD, 0x88, 0xDE, /* 0x38-0x3B */ + 0x88, 0xDF, 0x88, 0xE0, 0x88, 0xE1, 0x88, 0xE2, /* 0x3C-0x3F */ + 0x88, 0xE3, 0x88, 0xE4, 0x88, 0xE5, 0x88, 0xE6, /* 0x40-0x43 */ + 0x88, 0xE7, 0x88, 0xE8, 0x88, 0xE9, 0x88, 0xEA, /* 0x44-0x47 */ + 0x88, 0xEB, 0x88, 0xEC, 0x88, 0xED, 0x88, 0xEE, /* 0x48-0x4B */ + 0x88, 0xEF, 0x88, 0xF0, 0x88, 0xF1, 0x88, 0xF2, /* 0x4C-0x4F */ + 0x88, 0xF3, 0x88, 0xF4, 0x88, 0xF5, 0x88, 0xF6, /* 0x50-0x53 */ + 0xB4, 0xF5, 0xB4, 0xF6, 0xB4, 0xF7, 0x88, 0xF7, /* 0x54-0x57 */ + 0xB4, 0xF8, 0x88, 0xF8, 0x88, 0xF9, 0xB4, 0xF9, /* 0x58-0x5B */ + 0xB4, 0xFA, 0x88, 0xFA, 0xB4, 0xFB, 0xB4, 0xFC, /* 0x5C-0x5F */ + 0x88, 0xFB, 0x88, 0xFC, 0x88, 0xFD, 0x88, 0xFE, /* 0x60-0x63 */ + 0xB4, 0xFD, 0xB4, 0xFE, 0x89, 0x41, 0xB5, 0xA1, /* 0x64-0x67 */ + 0x89, 0x42, 0xB5, 0xA2, 0x89, 0x43, 0xB5, 0xA3, /* 0x68-0x6B */ + 0x89, 0x44, 0x89, 0x45, 0xB5, 0xA4, 0x89, 0x46, /* 0x6C-0x6F */ + 0xB5, 0xA5, 0xB5, 0xA6, 0x89, 0x47, 0x89, 0x48, /* 0x70-0x73 */ + 0xB5, 0xA7, 0x89, 0x49, 0x89, 0x4A, 0x89, 0x4B, /* 0x74-0x77 */ + 0xB5, 0xA8, 0x89, 0x4C, 0x89, 0x4D, 0x89, 0x4E, /* 0x78-0x7B */ + 0x89, 0x4F, 0x89, 0x50, 0x89, 0x51, 0x89, 0x52, /* 0x7C-0x7F */ + + 0xB5, 0xA9, 0xB5, 0xAA, 0x89, 0x53, 0xB5, 0xAB, /* 0x80-0x83 */ + 0xB5, 0xAC, 0xB5, 0xAD, 0x89, 0x54, 0x89, 0x55, /* 0x84-0x87 */ + 0x89, 0x56, 0x89, 0x57, 0x89, 0x58, 0x89, 0x59, /* 0x88-0x8B */ + 0xB5, 0xAE, 0x89, 0x5A, 0x89, 0x61, 0x89, 0x62, /* 0x8C-0x8F */ + 0xB5, 0xAF, 0x89, 0x63, 0x89, 0x64, 0x89, 0x65, /* 0x90-0x93 */ + 0xB5, 0xB0, 0x89, 0x66, 0x89, 0x67, 0x89, 0x68, /* 0x94-0x97 */ + 0x89, 0x69, 0x89, 0x6A, 0x89, 0x6B, 0x89, 0x6C, /* 0x98-0x9B */ + 0x89, 0x6D, 0x89, 0x6E, 0x89, 0x6F, 0x89, 0x70, /* 0x9C-0x9F */ + 0xB5, 0xB1, 0xB5, 0xB2, 0x89, 0x71, 0x89, 0x72, /* 0xA0-0xA3 */ + 0x89, 0x73, 0x89, 0x74, 0x89, 0x75, 0x89, 0x76, /* 0xA4-0xA7 */ + 0xB5, 0xB3, 0x89, 0x77, 0x89, 0x78, 0x89, 0x79, /* 0xA8-0xAB */ + 0xB5, 0xB4, 0x89, 0x7A, 0x89, 0x81, 0x89, 0x82, /* 0xAC-0xAF */ + 0x89, 0x83, 0x89, 0x84, 0x89, 0x85, 0x89, 0x86, /* 0xB0-0xB3 */ + 0x89, 0x87, 0x89, 0x88, 0x89, 0x89, 0x89, 0x8A, /* 0xB4-0xB7 */ + 0x89, 0x8B, 0x89, 0x8C, 0x89, 0x8D, 0x89, 0x8E, /* 0xB8-0xBB */ + 0x89, 0x8F, 0x89, 0x90, 0x89, 0x91, 0x89, 0x92, /* 0xBC-0xBF */ + 0x89, 0x93, 0x89, 0x94, 0x89, 0x95, 0x89, 0x96, /* 0xC0-0xC3 */ + 0xB5, 0xB5, 0xB5, 0xB6, 0x89, 0x97, 0x89, 0x98, /* 0xC4-0xC7 */ + 0xB5, 0xB7, 0x89, 0x99, 0x89, 0x9A, 0xB5, 0xB8, /* 0xC8-0xCB */ + 0xB5, 0xB9, 0x89, 0x9B, 0xB5, 0xBA, 0x89, 0x9C, /* 0xCC-0xCF */ + 0xB5, 0xBB, 0x89, 0x9D, 0x89, 0x9E, 0x89, 0x9F, /* 0xD0-0xD3 */ + 0xB5, 0xBC, 0xB5, 0xBD, 0x89, 0xA0, 0xB5, 0xBE, /* 0xD4-0xD7 */ + 0x89, 0xA1, 0xB5, 0xBF, 0x89, 0xA2, 0xB5, 0xC0, /* 0xD8-0xDB */ + 0x89, 0xA3, 0xB5, 0xC1, 0x89, 0xA4, 0x89, 0xA5, /* 0xDC-0xDF */ + 0xB5, 0xC2, 0x89, 0xA6, 0x89, 0xA7, 0x89, 0xA8, /* 0xE0-0xE3 */ + 0xB5, 0xC3, 0x89, 0xA9, 0x89, 0xAA, 0x89, 0xAB, /* 0xE4-0xE7 */ + 0xB5, 0xC4, 0x89, 0xAC, 0x89, 0xAD, 0x89, 0xAE, /* 0xE8-0xEB */ + 0x89, 0xAF, 0x89, 0xB0, 0x89, 0xB1, 0x89, 0xB2, /* 0xEC-0xEF */ + 0x89, 0xB3, 0x89, 0xB4, 0x89, 0xB5, 0x89, 0xB6, /* 0xF0-0xF3 */ + 0x89, 0xB7, 0x89, 0xB8, 0x89, 0xB9, 0x89, 0xBA, /* 0xF4-0xF7 */ + 0x89, 0xBB, 0x89, 0xBC, 0x89, 0xBD, 0x89, 0xBE, /* 0xF8-0xFB */ + 0xB5, 0xC5, 0x89, 0xBF, 0x89, 0xC0, 0x89, 0xC1, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_B4[512] = { + 0x89, 0xC2, 0x89, 0xC3, 0x89, 0xC4, 0x89, 0xC5, /* 0x00-0x03 */ + 0x89, 0xC6, 0x89, 0xC7, 0x89, 0xC8, 0x89, 0xC9, /* 0x04-0x07 */ + 0x89, 0xCA, 0x89, 0xCB, 0x89, 0xCC, 0x89, 0xCD, /* 0x08-0x0B */ + 0x89, 0xCE, 0x89, 0xCF, 0x89, 0xD0, 0x89, 0xD1, /* 0x0C-0x0F */ + 0xB5, 0xC6, 0x89, 0xD2, 0x89, 0xD3, 0x89, 0xD4, /* 0x10-0x13 */ + 0x89, 0xD5, 0x89, 0xD6, 0x89, 0xD7, 0x89, 0xD8, /* 0x14-0x17 */ + 0xB5, 0xC7, 0x89, 0xD9, 0x89, 0xDA, 0x89, 0xDB, /* 0x18-0x1B */ + 0xB5, 0xC8, 0x89, 0xDC, 0x89, 0xDD, 0x89, 0xDE, /* 0x1C-0x1F */ + 0xB5, 0xC9, 0x89, 0xDF, 0x89, 0xE0, 0x89, 0xE1, /* 0x20-0x23 */ + 0x89, 0xE2, 0x89, 0xE3, 0x89, 0xE4, 0x89, 0xE5, /* 0x24-0x27 */ + 0xB5, 0xCA, 0xB5, 0xCB, 0x89, 0xE6, 0xB5, 0xCC, /* 0x28-0x2B */ + 0x89, 0xE7, 0x89, 0xE8, 0x89, 0xE9, 0x89, 0xEA, /* 0x2C-0x2F */ + 0x89, 0xEB, 0x89, 0xEC, 0x89, 0xED, 0x89, 0xEE, /* 0x30-0x33 */ + 0xB5, 0xCD, 0x89, 0xEF, 0x89, 0xF0, 0x89, 0xF1, /* 0x34-0x37 */ + 0x89, 0xF2, 0x89, 0xF3, 0x89, 0xF4, 0x89, 0xF5, /* 0x38-0x3B */ + 0x89, 0xF6, 0x89, 0xF7, 0x89, 0xF8, 0x89, 0xF9, /* 0x3C-0x3F */ + 0x89, 0xFA, 0x89, 0xFB, 0x89, 0xFC, 0x89, 0xFD, /* 0x40-0x43 */ + 0x89, 0xFE, 0x8A, 0x41, 0x8A, 0x42, 0x8A, 0x43, /* 0x44-0x47 */ + 0x8A, 0x44, 0x8A, 0x45, 0x8A, 0x46, 0x8A, 0x47, /* 0x48-0x4B */ + 0x8A, 0x48, 0x8A, 0x49, 0x8A, 0x4A, 0x8A, 0x4B, /* 0x4C-0x4F */ + 0xB5, 0xCE, 0xB5, 0xCF, 0x8A, 0x4C, 0x8A, 0x4D, /* 0x50-0x53 */ + 0xB5, 0xD0, 0x8A, 0x4E, 0x8A, 0x4F, 0x8A, 0x50, /* 0x54-0x57 */ + 0xB5, 0xD1, 0x8A, 0x51, 0x8A, 0x52, 0x8A, 0x53, /* 0x58-0x5B */ + 0x8A, 0x54, 0x8A, 0x55, 0x8A, 0x56, 0x8A, 0x57, /* 0x5C-0x5F */ + 0xB5, 0xD2, 0xB5, 0xD3, 0x8A, 0x58, 0xB5, 0xD4, /* 0x60-0x63 */ + 0x8A, 0x59, 0xB5, 0xD5, 0x8A, 0x5A, 0x8A, 0x61, /* 0x64-0x67 */ + 0x8A, 0x62, 0x8A, 0x63, 0x8A, 0x64, 0x8A, 0x65, /* 0x68-0x6B */ + 0xB5, 0xD6, 0x8A, 0x66, 0x8A, 0x67, 0x8A, 0x68, /* 0x6C-0x6F */ + 0x8A, 0x69, 0x8A, 0x6A, 0x8A, 0x6B, 0x8A, 0x6C, /* 0x70-0x73 */ + 0x8A, 0x6D, 0x8A, 0x6E, 0x8A, 0x6F, 0x8A, 0x70, /* 0x74-0x77 */ + 0x8A, 0x71, 0x8A, 0x72, 0x8A, 0x73, 0x8A, 0x74, /* 0x78-0x7B */ + 0x8A, 0x75, 0x8A, 0x76, 0x8A, 0x77, 0x8A, 0x78, /* 0x7C-0x7F */ + + 0xB5, 0xD7, 0x8A, 0x79, 0x8A, 0x7A, 0x8A, 0x81, /* 0x80-0x83 */ + 0x8A, 0x82, 0x8A, 0x83, 0x8A, 0x84, 0x8A, 0x85, /* 0x84-0x87 */ + 0xB5, 0xD8, 0x8A, 0x86, 0x8A, 0x87, 0x8A, 0x88, /* 0x88-0x8B */ + 0x8A, 0x89, 0x8A, 0x8A, 0x8A, 0x8B, 0x8A, 0x8C, /* 0x8C-0x8F */ + 0x8A, 0x8D, 0x8A, 0x8E, 0x8A, 0x8F, 0x8A, 0x90, /* 0x90-0x93 */ + 0x8A, 0x91, 0x8A, 0x92, 0x8A, 0x93, 0x8A, 0x94, /* 0x94-0x97 */ + 0x8A, 0x95, 0x8A, 0x96, 0x8A, 0x97, 0x8A, 0x98, /* 0x98-0x9B */ + 0x8A, 0x99, 0xB5, 0xD9, 0x8A, 0x9A, 0x8A, 0x9B, /* 0x9C-0x9F */ + 0x8A, 0x9C, 0x8A, 0x9D, 0x8A, 0x9E, 0x8A, 0x9F, /* 0xA0-0xA3 */ + 0xB5, 0xDA, 0x8A, 0xA0, 0x8A, 0xA1, 0x8A, 0xA2, /* 0xA4-0xA7 */ + 0xB5, 0xDB, 0x8A, 0xA3, 0x8A, 0xA4, 0x8A, 0xA5, /* 0xA8-0xAB */ + 0xB5, 0xDC, 0x8A, 0xA6, 0x8A, 0xA7, 0x8A, 0xA8, /* 0xAC-0xAF */ + 0x8A, 0xA9, 0x8A, 0xAA, 0x8A, 0xAB, 0x8A, 0xAC, /* 0xB0-0xB3 */ + 0x8A, 0xAD, 0xB5, 0xDD, 0x8A, 0xAE, 0xB5, 0xDE, /* 0xB4-0xB7 */ + 0x8A, 0xAF, 0xB5, 0xDF, 0x8A, 0xB0, 0x8A, 0xB1, /* 0xB8-0xBB */ + 0x8A, 0xB2, 0x8A, 0xB3, 0x8A, 0xB4, 0x8A, 0xB5, /* 0xBC-0xBF */ + 0xB5, 0xE0, 0x8A, 0xB6, 0x8A, 0xB7, 0x8A, 0xB8, /* 0xC0-0xC3 */ + 0xB5, 0xE1, 0x8A, 0xB9, 0x8A, 0xBA, 0x8A, 0xBB, /* 0xC4-0xC7 */ + 0xB5, 0xE2, 0x8A, 0xBC, 0x8A, 0xBD, 0x8A, 0xBE, /* 0xC8-0xCB */ + 0x8A, 0xBF, 0x8A, 0xC0, 0x8A, 0xC1, 0x8A, 0xC2, /* 0xCC-0xCF */ + 0xB5, 0xE3, 0x8A, 0xC3, 0x8A, 0xC4, 0x8A, 0xC5, /* 0xD0-0xD3 */ + 0x8A, 0xC6, 0xB5, 0xE4, 0x8A, 0xC7, 0x8A, 0xC8, /* 0xD4-0xD7 */ + 0x8A, 0xC9, 0x8A, 0xCA, 0x8A, 0xCB, 0x8A, 0xCC, /* 0xD8-0xDB */ + 0xB5, 0xE5, 0xB5, 0xE6, 0x8A, 0xCD, 0x8A, 0xCE, /* 0xDC-0xDF */ + 0xB5, 0xE7, 0x8A, 0xCF, 0x8A, 0xD0, 0xB5, 0xE8, /* 0xE0-0xE3 */ + 0xB5, 0xE9, 0x8A, 0xD1, 0xB5, 0xEA, 0x8A, 0xD2, /* 0xE4-0xE7 */ + 0x8A, 0xD3, 0x8A, 0xD4, 0x8A, 0xD5, 0x8A, 0xD6, /* 0xE8-0xEB */ + 0xB5, 0xEB, 0xB5, 0xEC, 0x8A, 0xD7, 0xB5, 0xED, /* 0xEC-0xEF */ + 0x8A, 0xD8, 0xB5, 0xEE, 0x8A, 0xD9, 0x8A, 0xDA, /* 0xF0-0xF3 */ + 0x8A, 0xDB, 0x8A, 0xDC, 0x8A, 0xDD, 0x8A, 0xDE, /* 0xF4-0xF7 */ + 0xB5, 0xEF, 0x8A, 0xDF, 0x8A, 0xE0, 0x8A, 0xE1, /* 0xF8-0xFB */ + 0x8A, 0xE2, 0x8A, 0xE3, 0x8A, 0xE4, 0x8A, 0xE5, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_B5[512] = { + 0x8A, 0xE6, 0x8A, 0xE7, 0x8A, 0xE8, 0x8A, 0xE9, /* 0x00-0x03 */ + 0x8A, 0xEA, 0x8A, 0xEB, 0x8A, 0xEC, 0x8A, 0xED, /* 0x04-0x07 */ + 0x8A, 0xEE, 0x8A, 0xEF, 0x8A, 0xF0, 0x8A, 0xF1, /* 0x08-0x0B */ + 0x8A, 0xF2, 0x8A, 0xF3, 0x8A, 0xF4, 0x8A, 0xF5, /* 0x0C-0x0F */ + 0x8A, 0xF6, 0x8A, 0xF7, 0x8A, 0xF8, 0x8A, 0xF9, /* 0x10-0x13 */ + 0xB5, 0xF0, 0xB5, 0xF1, 0x8A, 0xFA, 0x8A, 0xFB, /* 0x14-0x17 */ + 0xB5, 0xF2, 0x8A, 0xFC, 0x8A, 0xFD, 0xB5, 0xF3, /* 0x18-0x1B */ + 0xB5, 0xF4, 0x8A, 0xFE, 0x8B, 0x41, 0x8B, 0x42, /* 0x1C-0x1F */ + 0x8B, 0x43, 0x8B, 0x44, 0x8B, 0x45, 0x8B, 0x46, /* 0x20-0x23 */ + 0xB5, 0xF5, 0xB5, 0xF6, 0x8B, 0x47, 0xB5, 0xF7, /* 0x24-0x27 */ + 0xB5, 0xF8, 0xB5, 0xF9, 0xB5, 0xFA, 0x8B, 0x48, /* 0x28-0x2B */ + 0x8B, 0x49, 0x8B, 0x4A, 0x8B, 0x4B, 0x8B, 0x4C, /* 0x2C-0x2F */ + 0xB5, 0xFB, 0xB5, 0xFC, 0x8B, 0x4D, 0x8B, 0x4E, /* 0x30-0x33 */ + 0xB5, 0xFD, 0x8B, 0x4F, 0x8B, 0x50, 0x8B, 0x51, /* 0x34-0x37 */ + 0xB5, 0xFE, 0x8B, 0x52, 0x8B, 0x53, 0x8B, 0x54, /* 0x38-0x3B */ + 0x8B, 0x55, 0x8B, 0x56, 0x8B, 0x57, 0x8B, 0x58, /* 0x3C-0x3F */ + 0xB6, 0xA1, 0xB6, 0xA2, 0x8B, 0x59, 0xB6, 0xA3, /* 0x40-0x43 */ + 0xB6, 0xA4, 0xB6, 0xA5, 0x8B, 0x5A, 0x8B, 0x61, /* 0x44-0x47 */ + 0x8B, 0x62, 0x8B, 0x63, 0x8B, 0x64, 0xB6, 0xA6, /* 0x48-0x4B */ + 0xB6, 0xA7, 0xB6, 0xA8, 0x8B, 0x65, 0x8B, 0x66, /* 0x4C-0x4F */ + 0xB6, 0xA9, 0x8B, 0x67, 0x8B, 0x68, 0x8B, 0x69, /* 0x50-0x53 */ + 0xB6, 0xAA, 0x8B, 0x6A, 0x8B, 0x6B, 0x8B, 0x6C, /* 0x54-0x57 */ + 0x8B, 0x6D, 0x8B, 0x6E, 0x8B, 0x6F, 0x8B, 0x70, /* 0x58-0x5B */ + 0xB6, 0xAB, 0xB6, 0xAC, 0x8B, 0x71, 0xB6, 0xAD, /* 0x5C-0x5F */ + 0xB6, 0xAE, 0xB6, 0xAF, 0x8B, 0x72, 0x8B, 0x73, /* 0x60-0x63 */ + 0x8B, 0x74, 0x8B, 0x75, 0x8B, 0x76, 0x8B, 0x77, /* 0x64-0x67 */ + 0x8B, 0x78, 0x8B, 0x79, 0x8B, 0x7A, 0x8B, 0x81, /* 0x68-0x6B */ + 0x8B, 0x82, 0x8B, 0x83, 0x8B, 0x84, 0x8B, 0x85, /* 0x6C-0x6F */ + 0x8B, 0x86, 0x8B, 0x87, 0x8B, 0x88, 0x8B, 0x89, /* 0x70-0x73 */ + 0x8B, 0x8A, 0x8B, 0x8B, 0x8B, 0x8C, 0x8B, 0x8D, /* 0x74-0x77 */ + 0x8B, 0x8E, 0x8B, 0x8F, 0x8B, 0x90, 0x8B, 0x91, /* 0x78-0x7B */ + 0x8B, 0x92, 0x8B, 0x93, 0x8B, 0x94, 0x8B, 0x95, /* 0x7C-0x7F */ + + 0x8B, 0x96, 0x8B, 0x97, 0x8B, 0x98, 0x8B, 0x99, /* 0x80-0x83 */ + 0x8B, 0x9A, 0x8B, 0x9B, 0x8B, 0x9C, 0x8B, 0x9D, /* 0x84-0x87 */ + 0x8B, 0x9E, 0x8B, 0x9F, 0x8B, 0xA0, 0x8B, 0xA1, /* 0x88-0x8B */ + 0x8B, 0xA2, 0x8B, 0xA3, 0x8B, 0xA4, 0x8B, 0xA5, /* 0x8C-0x8F */ + 0x8B, 0xA6, 0x8B, 0xA7, 0x8B, 0xA8, 0x8B, 0xA9, /* 0x90-0x93 */ + 0x8B, 0xAA, 0x8B, 0xAB, 0x8B, 0xAC, 0x8B, 0xAD, /* 0x94-0x97 */ + 0x8B, 0xAE, 0x8B, 0xAF, 0x8B, 0xB0, 0x8B, 0xB1, /* 0x98-0x9B */ + 0x8B, 0xB2, 0x8B, 0xB3, 0x8B, 0xB4, 0x8B, 0xB5, /* 0x9C-0x9F */ + 0xB6, 0xB0, 0xB6, 0xB1, 0x8B, 0xB6, 0x8B, 0xB7, /* 0xA0-0xA3 */ + 0xB6, 0xB2, 0x8B, 0xB8, 0x8B, 0xB9, 0x8B, 0xBA, /* 0xA4-0xA7 */ + 0xB6, 0xB3, 0x8B, 0xBB, 0xB6, 0xB4, 0xB6, 0xB5, /* 0xA8-0xAB */ + 0x8B, 0xBC, 0x8B, 0xBD, 0x8B, 0xBE, 0x8B, 0xBF, /* 0xAC-0xAF */ + 0xB6, 0xB6, 0xB6, 0xB7, 0x8B, 0xC0, 0xB6, 0xB8, /* 0xB0-0xB3 */ + 0xB6, 0xB9, 0xB6, 0xBA, 0x8B, 0xC1, 0x8B, 0xC2, /* 0xB4-0xB7 */ + 0x8B, 0xC3, 0x8B, 0xC4, 0x8B, 0xC5, 0xB6, 0xBB, /* 0xB8-0xBB */ + 0xB6, 0xBC, 0xB6, 0xBD, 0x8B, 0xC6, 0x8B, 0xC7, /* 0xBC-0xBF */ + 0xB6, 0xBE, 0x8B, 0xC8, 0x8B, 0xC9, 0x8B, 0xCA, /* 0xC0-0xC3 */ + 0xB6, 0xBF, 0x8B, 0xCB, 0x8B, 0xCC, 0x8B, 0xCD, /* 0xC4-0xC7 */ + 0x8B, 0xCE, 0x8B, 0xCF, 0x8B, 0xD0, 0x8B, 0xD1, /* 0xC8-0xCB */ + 0xB6, 0xC0, 0xB6, 0xC1, 0x8B, 0xD2, 0xB6, 0xC2, /* 0xCC-0xCF */ + 0xB6, 0xC3, 0xB6, 0xC4, 0x8B, 0xD3, 0x8B, 0xD4, /* 0xD0-0xD3 */ + 0x8B, 0xD5, 0x8B, 0xD6, 0x8B, 0xD7, 0x8B, 0xD8, /* 0xD4-0xD7 */ + 0xB6, 0xC5, 0x8B, 0xD9, 0x8B, 0xDA, 0x8B, 0xDB, /* 0xD8-0xDB */ + 0x8B, 0xDC, 0x8B, 0xDD, 0x8B, 0xDE, 0x8B, 0xDF, /* 0xDC-0xDF */ + 0x8B, 0xE0, 0x8B, 0xE1, 0x8B, 0xE2, 0x8B, 0xE3, /* 0xE0-0xE3 */ + 0x8B, 0xE4, 0x8B, 0xE5, 0x8B, 0xE6, 0x8B, 0xE7, /* 0xE4-0xE7 */ + 0x8B, 0xE8, 0x8B, 0xE9, 0x8B, 0xEA, 0x8B, 0xEB, /* 0xE8-0xEB */ + 0xB6, 0xC6, 0x8B, 0xEC, 0x8B, 0xED, 0x8B, 0xEE, /* 0xEC-0xEF */ + 0x8B, 0xEF, 0x8B, 0xF0, 0x8B, 0xF1, 0x8B, 0xF2, /* 0xF0-0xF3 */ + 0x8B, 0xF3, 0x8B, 0xF4, 0x8B, 0xF5, 0x8B, 0xF6, /* 0xF4-0xF7 */ + 0x8B, 0xF7, 0x8B, 0xF8, 0x8B, 0xF9, 0x8B, 0xFA, /* 0xF8-0xFB */ + 0x8B, 0xFB, 0x8B, 0xFC, 0x8B, 0xFD, 0x8B, 0xFE, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_B6[512] = { + 0x8C, 0x41, 0x8C, 0x42, 0x8C, 0x43, 0x8C, 0x44, /* 0x00-0x03 */ + 0x8C, 0x45, 0x8C, 0x46, 0x8C, 0x47, 0x8C, 0x48, /* 0x04-0x07 */ + 0x8C, 0x49, 0x8C, 0x4A, 0x8C, 0x4B, 0x8C, 0x4C, /* 0x08-0x0B */ + 0x8C, 0x4D, 0x8C, 0x4E, 0x8C, 0x4F, 0x8C, 0x50, /* 0x0C-0x0F */ + 0xB6, 0xC7, 0xB6, 0xC8, 0x8C, 0x51, 0x8C, 0x52, /* 0x10-0x13 */ + 0xB6, 0xC9, 0x8C, 0x53, 0x8C, 0x54, 0x8C, 0x55, /* 0x14-0x17 */ + 0xB6, 0xCA, 0x8C, 0x56, 0x8C, 0x57, 0x8C, 0x58, /* 0x18-0x1B */ + 0x8C, 0x59, 0x8C, 0x5A, 0x8C, 0x61, 0x8C, 0x62, /* 0x1C-0x1F */ + 0x8C, 0x63, 0x8C, 0x64, 0x8C, 0x65, 0x8C, 0x66, /* 0x20-0x23 */ + 0x8C, 0x67, 0xB6, 0xCB, 0x8C, 0x68, 0x8C, 0x69, /* 0x24-0x27 */ + 0x8C, 0x6A, 0x8C, 0x6B, 0x8C, 0x6C, 0x8C, 0x6D, /* 0x28-0x2B */ + 0xB6, 0xCC, 0x8C, 0x6E, 0x8C, 0x6F, 0x8C, 0x70, /* 0x2C-0x2F */ + 0x8C, 0x71, 0x8C, 0x72, 0x8C, 0x73, 0x8C, 0x74, /* 0x30-0x33 */ + 0xB6, 0xCD, 0x8C, 0x75, 0x8C, 0x76, 0x8C, 0x77, /* 0x34-0x37 */ + 0x8C, 0x78, 0x8C, 0x79, 0x8C, 0x7A, 0x8C, 0x81, /* 0x38-0x3B */ + 0x8C, 0x82, 0x8C, 0x83, 0x8C, 0x84, 0x8C, 0x85, /* 0x3C-0x3F */ + 0x8C, 0x86, 0x8C, 0x87, 0x8C, 0x88, 0x8C, 0x89, /* 0x40-0x43 */ + 0x8C, 0x8A, 0x8C, 0x8B, 0x8C, 0x8C, 0x8C, 0x8D, /* 0x44-0x47 */ + 0xB6, 0xCE, 0x8C, 0x8E, 0x8C, 0x8F, 0x8C, 0x90, /* 0x48-0x4B */ + 0x8C, 0x91, 0x8C, 0x92, 0x8C, 0x93, 0x8C, 0x94, /* 0x4C-0x4F */ + 0x8C, 0x95, 0x8C, 0x96, 0x8C, 0x97, 0x8C, 0x98, /* 0x50-0x53 */ + 0x8C, 0x99, 0x8C, 0x9A, 0x8C, 0x9B, 0x8C, 0x9C, /* 0x54-0x57 */ + 0x8C, 0x9D, 0x8C, 0x9E, 0x8C, 0x9F, 0x8C, 0xA0, /* 0x58-0x5B */ + 0x8C, 0xA1, 0x8C, 0xA2, 0x8C, 0xA3, 0x8C, 0xA4, /* 0x5C-0x5F */ + 0x8C, 0xA5, 0x8C, 0xA6, 0x8C, 0xA7, 0x8C, 0xA8, /* 0x60-0x63 */ + 0xB6, 0xCF, 0x8C, 0xA9, 0x8C, 0xAA, 0x8C, 0xAB, /* 0x64-0x67 */ + 0xB6, 0xD0, 0x8C, 0xAC, 0x8C, 0xAD, 0x8C, 0xAE, /* 0x68-0x6B */ + 0x8C, 0xAF, 0x8C, 0xB0, 0x8C, 0xB1, 0x8C, 0xB2, /* 0x6C-0x6F */ + 0x8C, 0xB3, 0x8C, 0xB4, 0x8C, 0xB5, 0x8C, 0xB6, /* 0x70-0x73 */ + 0x8C, 0xB7, 0x8C, 0xB8, 0x8C, 0xB9, 0x8C, 0xBA, /* 0x74-0x77 */ + 0x8C, 0xBB, 0x8C, 0xBC, 0x8C, 0xBD, 0x8C, 0xBE, /* 0x78-0x7B */ + 0x8C, 0xBF, 0x8C, 0xC0, 0x8C, 0xC1, 0x8C, 0xC2, /* 0x7C-0x7F */ + + 0x8C, 0xC3, 0x8C, 0xC4, 0x8C, 0xC5, 0x8C, 0xC6, /* 0x80-0x83 */ + 0x8C, 0xC7, 0x8C, 0xC8, 0x8C, 0xC9, 0x8C, 0xCA, /* 0x84-0x87 */ + 0x8C, 0xCB, 0x8C, 0xCC, 0x8C, 0xCD, 0x8C, 0xCE, /* 0x88-0x8B */ + 0x8C, 0xCF, 0x8C, 0xD0, 0x8C, 0xD1, 0x8C, 0xD2, /* 0x8C-0x8F */ + 0x8C, 0xD3, 0x8C, 0xD4, 0x8C, 0xD5, 0x8C, 0xD6, /* 0x90-0x93 */ + 0x8C, 0xD7, 0x8C, 0xD8, 0x8C, 0xD9, 0x8C, 0xDA, /* 0x94-0x97 */ + 0x8C, 0xDB, 0x8C, 0xDC, 0x8C, 0xDD, 0x8C, 0xDE, /* 0x98-0x9B */ + 0xB6, 0xD1, 0xB6, 0xD2, 0x8C, 0xDF, 0x8C, 0xE0, /* 0x9C-0x9F */ + 0xB6, 0xD3, 0x8C, 0xE1, 0x8C, 0xE2, 0x8C, 0xE3, /* 0xA0-0xA3 */ + 0xB6, 0xD4, 0x8C, 0xE4, 0x8C, 0xE5, 0x8C, 0xE6, /* 0xA4-0xA7 */ + 0x8C, 0xE7, 0x8C, 0xE8, 0x8C, 0xE9, 0xB6, 0xD5, /* 0xA8-0xAB */ + 0xB6, 0xD6, 0x8C, 0xEA, 0x8C, 0xEB, 0x8C, 0xEC, /* 0xAC-0xAF */ + 0x8C, 0xED, 0xB6, 0xD7, 0x8C, 0xEE, 0x8C, 0xEF, /* 0xB0-0xB3 */ + 0x8C, 0xF0, 0x8C, 0xF1, 0x8C, 0xF2, 0x8C, 0xF3, /* 0xB4-0xB7 */ + 0x8C, 0xF4, 0x8C, 0xF5, 0x8C, 0xF6, 0x8C, 0xF7, /* 0xB8-0xBB */ + 0x8C, 0xF8, 0x8C, 0xF9, 0x8C, 0xFA, 0x8C, 0xFB, /* 0xBC-0xBF */ + 0x8C, 0xFC, 0x8C, 0xFD, 0x8C, 0xFE, 0x8D, 0x41, /* 0xC0-0xC3 */ + 0x8D, 0x42, 0x8D, 0x43, 0x8D, 0x44, 0x8D, 0x45, /* 0xC4-0xC7 */ + 0x8D, 0x46, 0x8D, 0x47, 0x8D, 0x48, 0x8D, 0x49, /* 0xC8-0xCB */ + 0x8D, 0x4A, 0x8D, 0x4B, 0x8D, 0x4C, 0x8D, 0x4D, /* 0xCC-0xCF */ + 0x8D, 0x4E, 0x8D, 0x4F, 0x8D, 0x50, 0x8D, 0x51, /* 0xD0-0xD3 */ + 0xB6, 0xD8, 0x8D, 0x52, 0x8D, 0x53, 0x8D, 0x54, /* 0xD4-0xD7 */ + 0x8D, 0x55, 0x8D, 0x56, 0x8D, 0x57, 0x8D, 0x58, /* 0xD8-0xDB */ + 0x8D, 0x59, 0x8D, 0x5A, 0x8D, 0x61, 0x8D, 0x62, /* 0xDC-0xDF */ + 0x8D, 0x63, 0x8D, 0x64, 0x8D, 0x65, 0x8D, 0x66, /* 0xE0-0xE3 */ + 0x8D, 0x67, 0x8D, 0x68, 0x8D, 0x69, 0x8D, 0x6A, /* 0xE4-0xE7 */ + 0x8D, 0x6B, 0x8D, 0x6C, 0x8D, 0x6D, 0x8D, 0x6E, /* 0xE8-0xEB */ + 0x8D, 0x6F, 0x8D, 0x70, 0x8D, 0x71, 0x8D, 0x72, /* 0xEC-0xEF */ + 0xB6, 0xD9, 0x8D, 0x73, 0x8D, 0x74, 0x8D, 0x75, /* 0xF0-0xF3 */ + 0xB6, 0xDA, 0x8D, 0x76, 0x8D, 0x77, 0x8D, 0x78, /* 0xF4-0xF7 */ + 0xB6, 0xDB, 0x8D, 0x79, 0x8D, 0x7A, 0x8D, 0x81, /* 0xF8-0xFB */ + 0x8D, 0x82, 0x8D, 0x83, 0x8D, 0x84, 0x8D, 0x85, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_B7[512] = { + 0xB6, 0xDC, 0xB6, 0xDD, 0x8D, 0x86, 0x8D, 0x87, /* 0x00-0x03 */ + 0x8D, 0x88, 0xB6, 0xDE, 0x8D, 0x89, 0x8D, 0x8A, /* 0x04-0x07 */ + 0x8D, 0x8B, 0x8D, 0x8C, 0x8D, 0x8D, 0x8D, 0x8E, /* 0x08-0x0B */ + 0x8D, 0x8F, 0x8D, 0x90, 0x8D, 0x91, 0x8D, 0x92, /* 0x0C-0x0F */ + 0x8D, 0x93, 0x8D, 0x94, 0x8D, 0x95, 0x8D, 0x96, /* 0x10-0x13 */ + 0x8D, 0x97, 0x8D, 0x98, 0x8D, 0x99, 0x8D, 0x9A, /* 0x14-0x17 */ + 0x8D, 0x9B, 0x8D, 0x9C, 0x8D, 0x9D, 0x8D, 0x9E, /* 0x18-0x1B */ + 0x8D, 0x9F, 0x8D, 0xA0, 0x8D, 0xA1, 0x8D, 0xA2, /* 0x1C-0x1F */ + 0x8D, 0xA3, 0x8D, 0xA4, 0x8D, 0xA5, 0x8D, 0xA6, /* 0x20-0x23 */ + 0x8D, 0xA7, 0x8D, 0xA8, 0x8D, 0xA9, 0x8D, 0xAA, /* 0x24-0x27 */ + 0xB6, 0xDF, 0xB6, 0xE0, 0x8D, 0xAB, 0x8D, 0xAC, /* 0x28-0x2B */ + 0xB6, 0xE1, 0x8D, 0xAD, 0x8D, 0xAE, 0xB6, 0xE2, /* 0x2C-0x2F */ + 0xB6, 0xE3, 0x8D, 0xAF, 0x8D, 0xB0, 0x8D, 0xB1, /* 0x30-0x33 */ + 0x8D, 0xB2, 0x8D, 0xB3, 0x8D, 0xB4, 0x8D, 0xB5, /* 0x34-0x37 */ + 0xB6, 0xE4, 0xB6, 0xE5, 0x8D, 0xB6, 0xB6, 0xE6, /* 0x38-0x3B */ + 0x8D, 0xB7, 0x8D, 0xB8, 0x8D, 0xB9, 0x8D, 0xBA, /* 0x3C-0x3F */ + 0x8D, 0xBB, 0x8D, 0xBC, 0x8D, 0xBD, 0x8D, 0xBE, /* 0x40-0x43 */ + 0xB6, 0xE7, 0x8D, 0xBF, 0x8D, 0xC0, 0x8D, 0xC1, /* 0x44-0x47 */ + 0xB6, 0xE8, 0x8D, 0xC2, 0x8D, 0xC3, 0x8D, 0xC4, /* 0x48-0x4B */ + 0xB6, 0xE9, 0x8D, 0xC5, 0x8D, 0xC6, 0x8D, 0xC7, /* 0x4C-0x4F */ + 0x8D, 0xC8, 0x8D, 0xC9, 0x8D, 0xCA, 0x8D, 0xCB, /* 0x50-0x53 */ + 0xB6, 0xEA, 0xB6, 0xEB, 0x8D, 0xCC, 0x8D, 0xCD, /* 0x54-0x57 */ + 0x8D, 0xCE, 0x8D, 0xCF, 0x8D, 0xD0, 0x8D, 0xD1, /* 0x58-0x5B */ + 0x8D, 0xD2, 0x8D, 0xD3, 0x8D, 0xD4, 0x8D, 0xD5, /* 0x5C-0x5F */ + 0xB6, 0xEC, 0x8D, 0xD6, 0x8D, 0xD7, 0x8D, 0xD8, /* 0x60-0x63 */ + 0xB6, 0xED, 0x8D, 0xD9, 0x8D, 0xDA, 0x8D, 0xDB, /* 0x64-0x67 */ + 0xB6, 0xEE, 0x8D, 0xDC, 0x8D, 0xDD, 0x8D, 0xDE, /* 0x68-0x6B */ + 0x8D, 0xDF, 0x8D, 0xE0, 0x8D, 0xE1, 0x8D, 0xE2, /* 0x6C-0x6F */ + 0xB6, 0xEF, 0xB6, 0xF0, 0x8D, 0xE3, 0xB6, 0xF1, /* 0x70-0x73 */ + 0x8D, 0xE4, 0xB6, 0xF2, 0x8D, 0xE5, 0x8D, 0xE6, /* 0x74-0x77 */ + 0x8D, 0xE7, 0x8D, 0xE8, 0x8D, 0xE9, 0x8D, 0xEA, /* 0x78-0x7B */ + 0xB6, 0xF3, 0xB6, 0xF4, 0x8D, 0xEB, 0x8D, 0xEC, /* 0x7C-0x7F */ + + 0xB6, 0xF5, 0x8D, 0xED, 0x8D, 0xEE, 0x8D, 0xEF, /* 0x80-0x83 */ + 0xB6, 0xF6, 0x8D, 0xF0, 0x8D, 0xF1, 0x8D, 0xF2, /* 0x84-0x87 */ + 0x8D, 0xF3, 0x8D, 0xF4, 0x8D, 0xF5, 0x8D, 0xF6, /* 0x88-0x8B */ + 0xB6, 0xF7, 0xB6, 0xF8, 0x8D, 0xF7, 0xB6, 0xF9, /* 0x8C-0x8F */ + 0xB6, 0xFA, 0xB6, 0xFB, 0xB6, 0xFC, 0x8D, 0xF8, /* 0x90-0x93 */ + 0x8D, 0xF9, 0x8D, 0xFA, 0xB6, 0xFD, 0xB6, 0xFE, /* 0x94-0x97 */ + 0xB7, 0xA1, 0xB7, 0xA2, 0x8D, 0xFB, 0x8D, 0xFC, /* 0x98-0x9B */ + 0xB7, 0xA3, 0x8D, 0xFD, 0x8D, 0xFE, 0x8E, 0x41, /* 0x9C-0x9F */ + 0xB7, 0xA4, 0x8E, 0x42, 0x8E, 0x43, 0x8E, 0x44, /* 0xA0-0xA3 */ + 0x8E, 0x45, 0x8E, 0x46, 0x8E, 0x47, 0x8E, 0x48, /* 0xA4-0xA7 */ + 0xB7, 0xA5, 0xB7, 0xA6, 0x8E, 0x49, 0xB7, 0xA7, /* 0xA8-0xAB */ + 0xB7, 0xA8, 0xB7, 0xA9, 0x8E, 0x4A, 0x8E, 0x4B, /* 0xAC-0xAF */ + 0x8E, 0x4C, 0x8E, 0x4D, 0x8E, 0x4E, 0x8E, 0x4F, /* 0xB0-0xB3 */ + 0xB7, 0xAA, 0xB7, 0xAB, 0x8E, 0x50, 0x8E, 0x51, /* 0xB4-0xB7 */ + 0xB7, 0xAC, 0x8E, 0x52, 0x8E, 0x53, 0x8E, 0x54, /* 0xB8-0xBB */ + 0x8E, 0x55, 0x8E, 0x56, 0x8E, 0x57, 0x8E, 0x58, /* 0xBC-0xBF */ + 0x8E, 0x59, 0x8E, 0x5A, 0x8E, 0x61, 0x8E, 0x62, /* 0xC0-0xC3 */ + 0x8E, 0x63, 0x8E, 0x64, 0x8E, 0x65, 0xB7, 0xAD, /* 0xC4-0xC7 */ + 0x8E, 0x66, 0xB7, 0xAE, 0x8E, 0x67, 0x8E, 0x68, /* 0xC8-0xCB */ + 0x8E, 0x69, 0x8E, 0x6A, 0x8E, 0x6B, 0x8E, 0x6C, /* 0xCC-0xCF */ + 0x8E, 0x6D, 0x8E, 0x6E, 0x8E, 0x6F, 0x8E, 0x70, /* 0xD0-0xD3 */ + 0x8E, 0x71, 0x8E, 0x72, 0x8E, 0x73, 0x8E, 0x74, /* 0xD4-0xD7 */ + 0x8E, 0x75, 0x8E, 0x76, 0x8E, 0x77, 0x8E, 0x78, /* 0xD8-0xDB */ + 0x8E, 0x79, 0x8E, 0x7A, 0x8E, 0x81, 0x8E, 0x82, /* 0xDC-0xDF */ + 0x8E, 0x83, 0x8E, 0x84, 0x8E, 0x85, 0x8E, 0x86, /* 0xE0-0xE3 */ + 0x8E, 0x87, 0x8E, 0x88, 0x8E, 0x89, 0x8E, 0x8A, /* 0xE4-0xE7 */ + 0x8E, 0x8B, 0x8E, 0x8C, 0x8E, 0x8D, 0x8E, 0x8E, /* 0xE8-0xEB */ + 0xB7, 0xAF, 0xB7, 0xB0, 0x8E, 0x8F, 0x8E, 0x90, /* 0xEC-0xEF */ + 0xB7, 0xB1, 0x8E, 0x91, 0x8E, 0x92, 0x8E, 0x93, /* 0xF0-0xF3 */ + 0xB7, 0xB2, 0x8E, 0x94, 0x8E, 0x95, 0x8E, 0x96, /* 0xF4-0xF7 */ + 0x8E, 0x97, 0x8E, 0x98, 0x8E, 0x99, 0x8E, 0x9A, /* 0xF8-0xFB */ + 0xB7, 0xB3, 0xB7, 0xB4, 0x8E, 0x9B, 0xB7, 0xB5, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_B8[512] = { + 0xB7, 0xB6, 0xB7, 0xB7, 0x8E, 0x9C, 0x8E, 0x9D, /* 0x00-0x03 */ + 0x8E, 0x9E, 0x8E, 0x9F, 0x8E, 0xA0, 0xB7, 0xB8, /* 0x04-0x07 */ + 0xB7, 0xB9, 0xB7, 0xBA, 0x8E, 0xA1, 0x8E, 0xA2, /* 0x08-0x0B */ + 0xB7, 0xBB, 0x8E, 0xA3, 0x8E, 0xA4, 0x8E, 0xA5, /* 0x0C-0x0F */ + 0xB7, 0xBC, 0x8E, 0xA6, 0x8E, 0xA7, 0x8E, 0xA8, /* 0x10-0x13 */ + 0x8E, 0xA9, 0x8E, 0xAA, 0x8E, 0xAB, 0x8E, 0xAC, /* 0x14-0x17 */ + 0xB7, 0xBD, 0xB7, 0xBE, 0x8E, 0xAD, 0xB7, 0xBF, /* 0x18-0x1B */ + 0x8E, 0xAE, 0xB7, 0xC0, 0x8E, 0xAF, 0x8E, 0xB0, /* 0x1C-0x1F */ + 0x8E, 0xB1, 0x8E, 0xB2, 0x8E, 0xB3, 0x8E, 0xB4, /* 0x20-0x23 */ + 0xB7, 0xC1, 0xB7, 0xC2, 0x8E, 0xB5, 0x8E, 0xB6, /* 0x24-0x27 */ + 0xB7, 0xC3, 0x8E, 0xB7, 0x8E, 0xB8, 0x8E, 0xB9, /* 0x28-0x2B */ + 0xB7, 0xC4, 0x8E, 0xBA, 0x8E, 0xBB, 0x8E, 0xBC, /* 0x2C-0x2F */ + 0x8E, 0xBD, 0x8E, 0xBE, 0x8E, 0xBF, 0x8E, 0xC0, /* 0x30-0x33 */ + 0xB7, 0xC5, 0xB7, 0xC6, 0x8E, 0xC1, 0xB7, 0xC7, /* 0x34-0x37 */ + 0xB7, 0xC8, 0xB7, 0xC9, 0x8E, 0xC2, 0x8E, 0xC3, /* 0x38-0x3B */ + 0x8E, 0xC4, 0x8E, 0xC5, 0x8E, 0xC6, 0x8E, 0xC7, /* 0x3C-0x3F */ + 0xB7, 0xCA, 0x8E, 0xC8, 0x8E, 0xC9, 0x8E, 0xCA, /* 0x40-0x43 */ + 0xB7, 0xCB, 0x8E, 0xCB, 0x8E, 0xCC, 0x8E, 0xCD, /* 0x44-0x47 */ + 0x8E, 0xCE, 0x8E, 0xCF, 0x8E, 0xD0, 0x8E, 0xD1, /* 0x48-0x4B */ + 0x8E, 0xD2, 0x8E, 0xD3, 0x8E, 0xD4, 0x8E, 0xD5, /* 0x4C-0x4F */ + 0x8E, 0xD6, 0xB7, 0xCC, 0x8E, 0xD7, 0xB7, 0xCD, /* 0x50-0x53 */ + 0x8E, 0xD8, 0x8E, 0xD9, 0x8E, 0xDA, 0x8E, 0xDB, /* 0x54-0x57 */ + 0x8E, 0xDC, 0x8E, 0xDD, 0x8E, 0xDE, 0x8E, 0xDF, /* 0x58-0x5B */ + 0xB7, 0xCE, 0xB7, 0xCF, 0x8E, 0xE0, 0x8E, 0xE1, /* 0x5C-0x5F */ + 0xB7, 0xD0, 0x8E, 0xE2, 0x8E, 0xE3, 0x8E, 0xE4, /* 0x60-0x63 */ + 0xB7, 0xD1, 0x8E, 0xE5, 0x8E, 0xE6, 0x8E, 0xE7, /* 0x64-0x67 */ + 0x8E, 0xE8, 0x8E, 0xE9, 0x8E, 0xEA, 0x8E, 0xEB, /* 0x68-0x6B */ + 0xB7, 0xD2, 0xB7, 0xD3, 0x8E, 0xEC, 0xB7, 0xD4, /* 0x6C-0x6F */ + 0x8E, 0xED, 0xB7, 0xD5, 0x8E, 0xEE, 0x8E, 0xEF, /* 0x70-0x73 */ + 0x8E, 0xF0, 0x8E, 0xF1, 0x8E, 0xF2, 0x8E, 0xF3, /* 0x74-0x77 */ + 0xB7, 0xD6, 0x8E, 0xF4, 0x8E, 0xF5, 0x8E, 0xF6, /* 0x78-0x7B */ + 0xB7, 0xD7, 0x8E, 0xF7, 0x8E, 0xF8, 0x8E, 0xF9, /* 0x7C-0x7F */ + + 0x8E, 0xFA, 0x8E, 0xFB, 0x8E, 0xFC, 0x8E, 0xFD, /* 0x80-0x83 */ + 0x8E, 0xFE, 0x8F, 0x41, 0x8F, 0x42, 0x8F, 0x43, /* 0x84-0x87 */ + 0x8F, 0x44, 0x8F, 0x45, 0x8F, 0x46, 0x8F, 0x47, /* 0x88-0x8B */ + 0x8F, 0x48, 0xB7, 0xD8, 0x8F, 0x49, 0x8F, 0x4A, /* 0x8C-0x8F */ + 0x8F, 0x4B, 0x8F, 0x4C, 0x8F, 0x4D, 0x8F, 0x4E, /* 0x90-0x93 */ + 0x8F, 0x4F, 0x8F, 0x50, 0x8F, 0x51, 0x8F, 0x52, /* 0x94-0x97 */ + 0x8F, 0x53, 0x8F, 0x54, 0x8F, 0x55, 0x8F, 0x56, /* 0x98-0x9B */ + 0x8F, 0x57, 0x8F, 0x58, 0x8F, 0x59, 0x8F, 0x5A, /* 0x9C-0x9F */ + 0x8F, 0x61, 0x8F, 0x62, 0x8F, 0x63, 0x8F, 0x64, /* 0xA0-0xA3 */ + 0x8F, 0x65, 0x8F, 0x66, 0x8F, 0x67, 0x8F, 0x68, /* 0xA4-0xA7 */ + 0xB7, 0xD9, 0x8F, 0x69, 0x8F, 0x6A, 0x8F, 0x6B, /* 0xA8-0xAB */ + 0x8F, 0x6C, 0x8F, 0x6D, 0x8F, 0x6E, 0x8F, 0x6F, /* 0xAC-0xAF */ + 0xB7, 0xDA, 0x8F, 0x70, 0x8F, 0x71, 0x8F, 0x72, /* 0xB0-0xB3 */ + 0xB7, 0xDB, 0x8F, 0x73, 0x8F, 0x74, 0x8F, 0x75, /* 0xB4-0xB7 */ + 0xB7, 0xDC, 0x8F, 0x76, 0x8F, 0x77, 0x8F, 0x78, /* 0xB8-0xBB */ + 0x8F, 0x79, 0x8F, 0x7A, 0x8F, 0x81, 0x8F, 0x82, /* 0xBC-0xBF */ + 0xB7, 0xDD, 0xB7, 0xDE, 0x8F, 0x83, 0xB7, 0xDF, /* 0xC0-0xC3 */ + 0x8F, 0x84, 0xB7, 0xE0, 0x8F, 0x85, 0x8F, 0x86, /* 0xC4-0xC7 */ + 0x8F, 0x87, 0x8F, 0x88, 0x8F, 0x89, 0x8F, 0x8A, /* 0xC8-0xCB */ + 0xB7, 0xE1, 0x8F, 0x8B, 0x8F, 0x8C, 0x8F, 0x8D, /* 0xCC-0xCF */ + 0xB7, 0xE2, 0x8F, 0x8E, 0x8F, 0x8F, 0x8F, 0x90, /* 0xD0-0xD3 */ + 0xB7, 0xE3, 0x8F, 0x91, 0x8F, 0x92, 0x8F, 0x93, /* 0xD4-0xD7 */ + 0x8F, 0x94, 0x8F, 0x95, 0x8F, 0x96, 0x8F, 0x97, /* 0xD8-0xDB */ + 0x8F, 0x98, 0xB7, 0xE4, 0x8F, 0x99, 0xB7, 0xE5, /* 0xDC-0xDF */ + 0x8F, 0x9A, 0xB7, 0xE6, 0x8F, 0x9B, 0x8F, 0x9C, /* 0xE0-0xE3 */ + 0x8F, 0x9D, 0x8F, 0x9E, 0x8F, 0x9F, 0x8F, 0xA0, /* 0xE4-0xE7 */ + 0xB7, 0xE7, 0xB7, 0xE8, 0x8F, 0xA1, 0x8F, 0xA2, /* 0xE8-0xEB */ + 0xB7, 0xE9, 0x8F, 0xA3, 0x8F, 0xA4, 0x8F, 0xA5, /* 0xEC-0xEF */ + 0xB7, 0xEA, 0x8F, 0xA6, 0x8F, 0xA7, 0x8F, 0xA8, /* 0xF0-0xF3 */ + 0x8F, 0xA9, 0x8F, 0xAA, 0x8F, 0xAB, 0x8F, 0xAC, /* 0xF4-0xF7 */ + 0xB7, 0xEB, 0xB7, 0xEC, 0x8F, 0xAD, 0xB7, 0xED, /* 0xF8-0xFB */ + 0x8F, 0xAE, 0xB7, 0xEE, 0x8F, 0xAF, 0x8F, 0xB0, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_B9[512] = { + 0x8F, 0xB1, 0x8F, 0xB2, 0x8F, 0xB3, 0x8F, 0xB4, /* 0x00-0x03 */ + 0xB7, 0xEF, 0x8F, 0xB5, 0x8F, 0xB6, 0x8F, 0xB7, /* 0x04-0x07 */ + 0x8F, 0xB8, 0x8F, 0xB9, 0x8F, 0xBA, 0x8F, 0xBB, /* 0x08-0x0B */ + 0x8F, 0xBC, 0x8F, 0xBD, 0x8F, 0xBE, 0x8F, 0xBF, /* 0x0C-0x0F */ + 0x8F, 0xC0, 0x8F, 0xC1, 0x8F, 0xC2, 0x8F, 0xC3, /* 0x10-0x13 */ + 0x8F, 0xC4, 0x8F, 0xC5, 0x8F, 0xC6, 0x8F, 0xC7, /* 0x14-0x17 */ + 0xB7, 0xF0, 0x8F, 0xC8, 0x8F, 0xC9, 0x8F, 0xCA, /* 0x18-0x1B */ + 0x8F, 0xCB, 0x8F, 0xCC, 0x8F, 0xCD, 0x8F, 0xCE, /* 0x1C-0x1F */ + 0xB7, 0xF1, 0x8F, 0xCF, 0x8F, 0xD0, 0x8F, 0xD1, /* 0x20-0x23 */ + 0x8F, 0xD2, 0x8F, 0xD3, 0x8F, 0xD4, 0x8F, 0xD5, /* 0x24-0x27 */ + 0x8F, 0xD6, 0x8F, 0xD7, 0x8F, 0xD8, 0x8F, 0xD9, /* 0x28-0x2B */ + 0x8F, 0xDA, 0x8F, 0xDB, 0x8F, 0xDC, 0x8F, 0xDD, /* 0x2C-0x2F */ + 0x8F, 0xDE, 0x8F, 0xDF, 0x8F, 0xE0, 0x8F, 0xE1, /* 0x30-0x33 */ + 0x8F, 0xE2, 0x8F, 0xE3, 0x8F, 0xE4, 0x8F, 0xE5, /* 0x34-0x37 */ + 0x8F, 0xE6, 0x8F, 0xE7, 0x8F, 0xE8, 0x8F, 0xE9, /* 0x38-0x3B */ + 0xB7, 0xF2, 0xB7, 0xF3, 0x8F, 0xEA, 0x8F, 0xEB, /* 0x3C-0x3F */ + 0xB7, 0xF4, 0x8F, 0xEC, 0x8F, 0xED, 0x8F, 0xEE, /* 0x40-0x43 */ + 0xB7, 0xF5, 0x8F, 0xEF, 0x8F, 0xF0, 0x8F, 0xF1, /* 0x44-0x47 */ + 0x8F, 0xF2, 0x8F, 0xF3, 0x8F, 0xF4, 0x8F, 0xF5, /* 0x48-0x4B */ + 0xB7, 0xF6, 0x8F, 0xF6, 0x8F, 0xF7, 0xB7, 0xF7, /* 0x4C-0x4F */ + 0x8F, 0xF8, 0xB7, 0xF8, 0x8F, 0xF9, 0x8F, 0xFA, /* 0x50-0x53 */ + 0x8F, 0xFB, 0x8F, 0xFC, 0x8F, 0xFD, 0x8F, 0xFE, /* 0x54-0x57 */ + 0xB7, 0xF9, 0xB7, 0xFA, 0x90, 0x41, 0x90, 0x42, /* 0x58-0x5B */ + 0xB7, 0xFB, 0x90, 0x43, 0x90, 0x44, 0x90, 0x45, /* 0x5C-0x5F */ + 0xB7, 0xFC, 0x90, 0x46, 0x90, 0x47, 0x90, 0x48, /* 0x60-0x63 */ + 0x90, 0x49, 0x90, 0x4A, 0x90, 0x4B, 0x90, 0x4C, /* 0x64-0x67 */ + 0xB7, 0xFD, 0xB7, 0xFE, 0x90, 0x4D, 0xB8, 0xA1, /* 0x68-0x6B */ + 0x90, 0x4E, 0xB8, 0xA2, 0x90, 0x4F, 0x90, 0x50, /* 0x6C-0x6F */ + 0x90, 0x51, 0x90, 0x52, 0x90, 0x53, 0x90, 0x54, /* 0x70-0x73 */ + 0xB8, 0xA3, 0xB8, 0xA4, 0x90, 0x55, 0x90, 0x56, /* 0x74-0x77 */ + 0xB8, 0xA5, 0x90, 0x57, 0x90, 0x58, 0x90, 0x59, /* 0x78-0x7B */ + 0xB8, 0xA6, 0x90, 0x5A, 0x90, 0x61, 0x90, 0x62, /* 0x7C-0x7F */ + + 0x90, 0x63, 0x90, 0x64, 0x90, 0x65, 0x90, 0x66, /* 0x80-0x83 */ + 0xB8, 0xA7, 0xB8, 0xA8, 0x90, 0x67, 0xB8, 0xA9, /* 0x84-0x87 */ + 0x90, 0x68, 0xB8, 0xAA, 0xB8, 0xAB, 0x90, 0x69, /* 0x88-0x8B */ + 0x90, 0x6A, 0xB8, 0xAC, 0xB8, 0xAD, 0x90, 0x6B, /* 0x8C-0x8F */ + 0x90, 0x6C, 0x90, 0x6D, 0x90, 0x6E, 0x90, 0x6F, /* 0x90-0x93 */ + 0x90, 0x70, 0x90, 0x71, 0x90, 0x72, 0x90, 0x73, /* 0x94-0x97 */ + 0x90, 0x74, 0x90, 0x75, 0x90, 0x76, 0x90, 0x77, /* 0x98-0x9B */ + 0x90, 0x78, 0x90, 0x79, 0x90, 0x7A, 0x90, 0x81, /* 0x9C-0x9F */ + 0x90, 0x82, 0x90, 0x83, 0x90, 0x84, 0x90, 0x85, /* 0xA0-0xA3 */ + 0x90, 0x86, 0x90, 0x87, 0x90, 0x88, 0x90, 0x89, /* 0xA4-0xA7 */ + 0x90, 0x8A, 0x90, 0x8B, 0x90, 0x8C, 0x90, 0x8D, /* 0xA8-0xAB */ + 0xB8, 0xAE, 0xB8, 0xAF, 0x90, 0x8E, 0x90, 0x8F, /* 0xAC-0xAF */ + 0xB8, 0xB0, 0x90, 0x90, 0x90, 0x91, 0x90, 0x92, /* 0xB0-0xB3 */ + 0xB8, 0xB1, 0x90, 0x93, 0x90, 0x94, 0x90, 0x95, /* 0xB4-0xB7 */ + 0x90, 0x96, 0x90, 0x97, 0x90, 0x98, 0x90, 0x99, /* 0xB8-0xBB */ + 0xB8, 0xB2, 0xB8, 0xB3, 0x90, 0x9A, 0xB8, 0xB4, /* 0xBC-0xBF */ + 0x90, 0x9B, 0xB8, 0xB5, 0x90, 0x9C, 0x90, 0x9D, /* 0xC0-0xC3 */ + 0x90, 0x9E, 0x90, 0x9F, 0x90, 0xA0, 0x90, 0xA1, /* 0xC4-0xC7 */ + 0xB8, 0xB6, 0xB8, 0xB7, 0x90, 0xA2, 0x90, 0xA3, /* 0xC8-0xCB */ + 0xB8, 0xB8, 0x90, 0xA4, 0xB8, 0xB9, 0xB8, 0xBA, /* 0xCC-0xCF */ + 0xB8, 0xBB, 0xB8, 0xBC, 0xB8, 0xBD, 0x90, 0xA5, /* 0xD0-0xD3 */ + 0x90, 0xA6, 0x90, 0xA7, 0x90, 0xA8, 0x90, 0xA9, /* 0xD4-0xD7 */ + 0xB8, 0xBE, 0xB8, 0xBF, 0x90, 0xAA, 0xB8, 0xC0, /* 0xD8-0xDB */ + 0x90, 0xAB, 0xB8, 0xC1, 0xB8, 0xC2, 0x90, 0xAC, /* 0xDC-0xDF */ + 0x90, 0xAD, 0xB8, 0xC3, 0x90, 0xAE, 0xB8, 0xC4, /* 0xE0-0xE3 */ + 0xB8, 0xC5, 0xB8, 0xC6, 0x90, 0xAF, 0x90, 0xB0, /* 0xE4-0xE7 */ + 0xB8, 0xC7, 0x90, 0xB1, 0x90, 0xB2, 0x90, 0xB3, /* 0xE8-0xEB */ + 0xB8, 0xC8, 0x90, 0xB4, 0x90, 0xB5, 0x90, 0xB6, /* 0xEC-0xEF */ + 0x90, 0xB7, 0x90, 0xB8, 0x90, 0xB9, 0x90, 0xBA, /* 0xF0-0xF3 */ + 0xB8, 0xC9, 0xB8, 0xCA, 0x90, 0xBB, 0xB8, 0xCB, /* 0xF4-0xF7 */ + 0xB8, 0xCC, 0xB8, 0xCD, 0xB8, 0xCE, 0x90, 0xBC, /* 0xF8-0xFB */ + 0x90, 0xBD, 0x90, 0xBE, 0x90, 0xBF, 0x90, 0xC0, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_BA[512] = { + 0xB8, 0xCF, 0xB8, 0xD0, 0x90, 0xC1, 0x90, 0xC2, /* 0x00-0x03 */ + 0x90, 0xC3, 0x90, 0xC4, 0x90, 0xC5, 0x90, 0xC6, /* 0x04-0x07 */ + 0xB8, 0xD1, 0x90, 0xC7, 0x90, 0xC8, 0x90, 0xC9, /* 0x08-0x0B */ + 0x90, 0xCA, 0x90, 0xCB, 0x90, 0xCC, 0x90, 0xCD, /* 0x0C-0x0F */ + 0x90, 0xCE, 0x90, 0xCF, 0x90, 0xD0, 0x90, 0xD1, /* 0x10-0x13 */ + 0x90, 0xD2, 0xB8, 0xD2, 0x90, 0xD3, 0x90, 0xD4, /* 0x14-0x17 */ + 0x90, 0xD5, 0x90, 0xD6, 0x90, 0xD7, 0x90, 0xD8, /* 0x18-0x1B */ + 0x90, 0xD9, 0x90, 0xDA, 0x90, 0xDB, 0x90, 0xDC, /* 0x1C-0x1F */ + 0x90, 0xDD, 0x90, 0xDE, 0x90, 0xDF, 0x90, 0xE0, /* 0x20-0x23 */ + 0x90, 0xE1, 0x90, 0xE2, 0x90, 0xE3, 0x90, 0xE4, /* 0x24-0x27 */ + 0x90, 0xE5, 0x90, 0xE6, 0x90, 0xE7, 0x90, 0xE8, /* 0x28-0x2B */ + 0x90, 0xE9, 0x90, 0xEA, 0x90, 0xEB, 0x90, 0xEC, /* 0x2C-0x2F */ + 0x90, 0xED, 0x90, 0xEE, 0x90, 0xEF, 0x90, 0xF0, /* 0x30-0x33 */ + 0x90, 0xF1, 0x90, 0xF2, 0x90, 0xF3, 0x90, 0xF4, /* 0x34-0x37 */ + 0xB8, 0xD3, 0xB8, 0xD4, 0x90, 0xF5, 0x90, 0xF6, /* 0x38-0x3B */ + 0xB8, 0xD5, 0x90, 0xF7, 0x90, 0xF8, 0x90, 0xF9, /* 0x3C-0x3F */ + 0xB8, 0xD6, 0x90, 0xFA, 0xB8, 0xD7, 0x90, 0xFB, /* 0x40-0x43 */ + 0x90, 0xFC, 0x90, 0xFD, 0x90, 0xFE, 0x91, 0x41, /* 0x44-0x47 */ + 0xB8, 0xD8, 0xB8, 0xD9, 0x91, 0x42, 0xB8, 0xDA, /* 0x48-0x4B */ + 0x91, 0x43, 0xB8, 0xDB, 0xB8, 0xDC, 0x91, 0x44, /* 0x4C-0x4F */ + 0x91, 0x45, 0x91, 0x46, 0x91, 0x47, 0xB8, 0xDD, /* 0x50-0x53 */ + 0xB8, 0xDE, 0xB8, 0xDF, 0x91, 0x48, 0x91, 0x49, /* 0x54-0x57 */ + 0xB8, 0xE0, 0x91, 0x4A, 0x91, 0x4B, 0x91, 0x4C, /* 0x58-0x5B */ + 0xB8, 0xE1, 0x91, 0x4D, 0x91, 0x4E, 0x91, 0x4F, /* 0x5C-0x5F */ + 0x91, 0x50, 0x91, 0x51, 0x91, 0x52, 0x91, 0x53, /* 0x60-0x63 */ + 0xB8, 0xE2, 0xB8, 0xE3, 0x91, 0x54, 0xB8, 0xE4, /* 0x64-0x67 */ + 0xB8, 0xE5, 0xB8, 0xE6, 0x91, 0x55, 0x91, 0x56, /* 0x68-0x6B */ + 0x91, 0x57, 0x91, 0x58, 0x91, 0x59, 0x91, 0x5A, /* 0x6C-0x6F */ + 0xB8, 0xE7, 0xB8, 0xE8, 0x91, 0x61, 0x91, 0x62, /* 0x70-0x73 */ + 0xB8, 0xE9, 0x91, 0x63, 0x91, 0x64, 0x91, 0x65, /* 0x74-0x77 */ + 0xB8, 0xEA, 0x91, 0x66, 0x91, 0x67, 0x91, 0x68, /* 0x78-0x7B */ + 0x91, 0x69, 0x91, 0x6A, 0x91, 0x6B, 0x91, 0x6C, /* 0x7C-0x7F */ + + 0x91, 0x6D, 0x91, 0x6E, 0x91, 0x6F, 0xB8, 0xEB, /* 0x80-0x83 */ + 0xB8, 0xEC, 0xB8, 0xED, 0x91, 0x70, 0xB8, 0xEE, /* 0x84-0x87 */ + 0x91, 0x71, 0x91, 0x72, 0x91, 0x73, 0x91, 0x74, /* 0x88-0x8B */ + 0xB8, 0xEF, 0x91, 0x75, 0x91, 0x76, 0x91, 0x77, /* 0x8C-0x8F */ + 0x91, 0x78, 0x91, 0x79, 0x91, 0x7A, 0x91, 0x81, /* 0x90-0x93 */ + 0x91, 0x82, 0x91, 0x83, 0x91, 0x84, 0x91, 0x85, /* 0x94-0x97 */ + 0x91, 0x86, 0x91, 0x87, 0x91, 0x88, 0x91, 0x89, /* 0x98-0x9B */ + 0x91, 0x8A, 0x91, 0x8B, 0x91, 0x8C, 0x91, 0x8D, /* 0x9C-0x9F */ + 0x91, 0x8E, 0x91, 0x8F, 0x91, 0x90, 0x91, 0x91, /* 0xA0-0xA3 */ + 0x91, 0x92, 0x91, 0x93, 0x91, 0x94, 0x91, 0x95, /* 0xA4-0xA7 */ + 0xB8, 0xF0, 0xB8, 0xF1, 0x91, 0x96, 0xB8, 0xF2, /* 0xA8-0xAB */ + 0xB8, 0xF3, 0x91, 0x97, 0x91, 0x98, 0x91, 0x99, /* 0xAC-0xAF */ + 0xB8, 0xF4, 0x91, 0x9A, 0xB8, 0xF5, 0x91, 0x9B, /* 0xB0-0xB3 */ + 0x91, 0x9C, 0x91, 0x9D, 0x91, 0x9E, 0x91, 0x9F, /* 0xB4-0xB7 */ + 0xB8, 0xF6, 0xB8, 0xF7, 0x91, 0xA0, 0xB8, 0xF8, /* 0xB8-0xBB */ + 0x91, 0xA1, 0xB8, 0xF9, 0x91, 0xA2, 0x91, 0xA3, /* 0xBC-0xBF */ + 0x91, 0xA4, 0x91, 0xA5, 0x91, 0xA6, 0x91, 0xA7, /* 0xC0-0xC3 */ + 0xB8, 0xFA, 0x91, 0xA8, 0x91, 0xA9, 0x91, 0xAA, /* 0xC4-0xC7 */ + 0xB8, 0xFB, 0x91, 0xAB, 0x91, 0xAC, 0x91, 0xAD, /* 0xC8-0xCB */ + 0x91, 0xAE, 0x91, 0xAF, 0x91, 0xB0, 0x91, 0xB1, /* 0xCC-0xCF */ + 0x91, 0xB2, 0x91, 0xB3, 0x91, 0xB4, 0x91, 0xB5, /* 0xD0-0xD3 */ + 0x91, 0xB6, 0x91, 0xB7, 0x91, 0xB8, 0x91, 0xB9, /* 0xD4-0xD7 */ + 0xB8, 0xFC, 0xB8, 0xFD, 0x91, 0xBA, 0x91, 0xBB, /* 0xD8-0xDB */ + 0x91, 0xBC, 0x91, 0xBD, 0x91, 0xBE, 0x91, 0xBF, /* 0xDC-0xDF */ + 0x91, 0xC0, 0x91, 0xC1, 0x91, 0xC2, 0x91, 0xC3, /* 0xE0-0xE3 */ + 0x91, 0xC4, 0x91, 0xC5, 0x91, 0xC6, 0x91, 0xC7, /* 0xE4-0xE7 */ + 0x91, 0xC8, 0x91, 0xC9, 0x91, 0xCA, 0x91, 0xCB, /* 0xE8-0xEB */ + 0x91, 0xCC, 0x91, 0xCD, 0x91, 0xCE, 0x91, 0xCF, /* 0xEC-0xEF */ + 0x91, 0xD0, 0x91, 0xD1, 0x91, 0xD2, 0x91, 0xD3, /* 0xF0-0xF3 */ + 0x91, 0xD4, 0x91, 0xD5, 0x91, 0xD6, 0x91, 0xD7, /* 0xF4-0xF7 */ + 0x91, 0xD8, 0x91, 0xD9, 0x91, 0xDA, 0x91, 0xDB, /* 0xF8-0xFB */ + 0xB8, 0xFE, 0x91, 0xDC, 0x91, 0xDD, 0x91, 0xDE, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_BB[512] = { + 0xB9, 0xA1, 0x91, 0xDF, 0x91, 0xE0, 0x91, 0xE1, /* 0x00-0x03 */ + 0xB9, 0xA2, 0x91, 0xE2, 0x91, 0xE3, 0x91, 0xE4, /* 0x04-0x07 */ + 0x91, 0xE5, 0x91, 0xE6, 0x91, 0xE7, 0x91, 0xE8, /* 0x08-0x0B */ + 0x91, 0xE9, 0xB9, 0xA3, 0x91, 0xEA, 0xB9, 0xA4, /* 0x0C-0x0F */ + 0x91, 0xEB, 0xB9, 0xA5, 0x91, 0xEC, 0x91, 0xED, /* 0x10-0x13 */ + 0x91, 0xEE, 0x91, 0xEF, 0x91, 0xF0, 0x91, 0xF1, /* 0x14-0x17 */ + 0xB9, 0xA6, 0x91, 0xF2, 0x91, 0xF3, 0x91, 0xF4, /* 0x18-0x1B */ + 0xB9, 0xA7, 0x91, 0xF5, 0x91, 0xF6, 0x91, 0xF7, /* 0x1C-0x1F */ + 0xB9, 0xA8, 0x91, 0xF8, 0x91, 0xF9, 0x91, 0xFA, /* 0x20-0x23 */ + 0x91, 0xFB, 0x91, 0xFC, 0x91, 0xFD, 0x91, 0xFE, /* 0x24-0x27 */ + 0x92, 0x41, 0xB9, 0xA9, 0x92, 0x42, 0xB9, 0xAA, /* 0x28-0x2B */ + 0x92, 0x43, 0x92, 0x44, 0x92, 0x45, 0x92, 0x46, /* 0x2C-0x2F */ + 0x92, 0x47, 0x92, 0x48, 0x92, 0x49, 0x92, 0x4A, /* 0x30-0x33 */ + 0xB9, 0xAB, 0xB9, 0xAC, 0xB9, 0xAD, 0x92, 0x4B, /* 0x34-0x37 */ + 0xB9, 0xAE, 0x92, 0x4C, 0x92, 0x4D, 0xB9, 0xAF, /* 0x38-0x3B */ + 0xB9, 0xB0, 0xB9, 0xB1, 0xB9, 0xB2, 0x92, 0x4E, /* 0x3C-0x3F */ + 0x92, 0x4F, 0x92, 0x50, 0x92, 0x51, 0x92, 0x52, /* 0x40-0x43 */ + 0xB9, 0xB3, 0xB9, 0xB4, 0x92, 0x53, 0xB9, 0xB5, /* 0x44-0x47 */ + 0x92, 0x54, 0xB9, 0xB6, 0x92, 0x55, 0x92, 0x56, /* 0x48-0x4B */ + 0x92, 0x57, 0xB9, 0xB7, 0x92, 0x58, 0xB9, 0xB8, /* 0x4C-0x4F */ + 0xB9, 0xB9, 0x92, 0x59, 0x92, 0x5A, 0x92, 0x61, /* 0x50-0x53 */ + 0xB9, 0xBA, 0x92, 0x62, 0x92, 0x63, 0x92, 0x64, /* 0x54-0x57 */ + 0xB9, 0xBB, 0x92, 0x65, 0x92, 0x66, 0x92, 0x67, /* 0x58-0x5B */ + 0x92, 0x68, 0x92, 0x69, 0x92, 0x6A, 0x92, 0x6B, /* 0x5C-0x5F */ + 0x92, 0x6C, 0xB9, 0xBC, 0x92, 0x6D, 0xB9, 0xBD, /* 0x60-0x63 */ + 0x92, 0x6E, 0x92, 0x6F, 0x92, 0x70, 0x92, 0x71, /* 0x64-0x67 */ + 0x92, 0x72, 0x92, 0x73, 0x92, 0x74, 0x92, 0x75, /* 0x68-0x6B */ + 0xB9, 0xBE, 0x92, 0x76, 0x92, 0x77, 0x92, 0x78, /* 0x6C-0x6F */ + 0x92, 0x79, 0x92, 0x7A, 0x92, 0x81, 0x92, 0x82, /* 0x70-0x73 */ + 0x92, 0x83, 0x92, 0x84, 0x92, 0x85, 0x92, 0x86, /* 0x74-0x77 */ + 0x92, 0x87, 0x92, 0x88, 0x92, 0x89, 0x92, 0x8A, /* 0x78-0x7B */ + 0x92, 0x8B, 0x92, 0x8C, 0x92, 0x8D, 0x92, 0x8E, /* 0x7C-0x7F */ + + 0x92, 0x8F, 0x92, 0x90, 0x92, 0x91, 0x92, 0x92, /* 0x80-0x83 */ + 0x92, 0x93, 0x92, 0x94, 0x92, 0x95, 0x92, 0x96, /* 0x84-0x87 */ + 0xB9, 0xBF, 0x92, 0x97, 0x92, 0x98, 0x92, 0x99, /* 0x88-0x8B */ + 0xB9, 0xC0, 0x92, 0x9A, 0x92, 0x9B, 0x92, 0x9C, /* 0x8C-0x8F */ + 0xB9, 0xC1, 0x92, 0x9D, 0x92, 0x9E, 0x92, 0x9F, /* 0x90-0x93 */ + 0x92, 0xA0, 0x92, 0xA1, 0x92, 0xA2, 0x92, 0xA3, /* 0x94-0x97 */ + 0x92, 0xA4, 0x92, 0xA5, 0x92, 0xA6, 0x92, 0xA7, /* 0x98-0x9B */ + 0x92, 0xA8, 0x92, 0xA9, 0x92, 0xAA, 0x92, 0xAB, /* 0x9C-0x9F */ + 0x92, 0xAC, 0x92, 0xAD, 0x92, 0xAE, 0x92, 0xAF, /* 0xA0-0xA3 */ + 0xB9, 0xC2, 0x92, 0xB0, 0x92, 0xB1, 0x92, 0xB2, /* 0xA4-0xA7 */ + 0xB9, 0xC3, 0x92, 0xB3, 0x92, 0xB4, 0x92, 0xB5, /* 0xA8-0xAB */ + 0xB9, 0xC4, 0x92, 0xB6, 0x92, 0xB7, 0x92, 0xB8, /* 0xAC-0xAF */ + 0x92, 0xB9, 0x92, 0xBA, 0x92, 0xBB, 0x92, 0xBC, /* 0xB0-0xB3 */ + 0xB9, 0xC5, 0x92, 0xBD, 0x92, 0xBE, 0xB9, 0xC6, /* 0xB4-0xB7 */ + 0x92, 0xBF, 0x92, 0xC0, 0x92, 0xC1, 0x92, 0xC2, /* 0xB8-0xBB */ + 0x92, 0xC3, 0x92, 0xC4, 0x92, 0xC5, 0x92, 0xC6, /* 0xBC-0xBF */ + 0xB9, 0xC7, 0x92, 0xC7, 0x92, 0xC8, 0x92, 0xC9, /* 0xC0-0xC3 */ + 0xB9, 0xC8, 0x92, 0xCA, 0x92, 0xCB, 0x92, 0xCC, /* 0xC4-0xC7 */ + 0xB9, 0xC9, 0x92, 0xCD, 0x92, 0xCE, 0x92, 0xCF, /* 0xC8-0xCB */ + 0x92, 0xD0, 0x92, 0xD1, 0x92, 0xD2, 0x92, 0xD3, /* 0xCC-0xCF */ + 0xB9, 0xCA, 0x92, 0xD4, 0x92, 0xD5, 0xB9, 0xCB, /* 0xD0-0xD3 */ + 0x92, 0xD6, 0x92, 0xD7, 0x92, 0xD8, 0x92, 0xD9, /* 0xD4-0xD7 */ + 0x92, 0xDA, 0x92, 0xDB, 0x92, 0xDC, 0x92, 0xDD, /* 0xD8-0xDB */ + 0x92, 0xDE, 0x92, 0xDF, 0x92, 0xE0, 0x92, 0xE1, /* 0xDC-0xDF */ + 0x92, 0xE2, 0x92, 0xE3, 0x92, 0xE4, 0x92, 0xE5, /* 0xE0-0xE3 */ + 0x92, 0xE6, 0x92, 0xE7, 0x92, 0xE8, 0x92, 0xE9, /* 0xE4-0xE7 */ + 0x92, 0xEA, 0x92, 0xEB, 0x92, 0xEC, 0x92, 0xED, /* 0xE8-0xEB */ + 0x92, 0xEE, 0x92, 0xEF, 0x92, 0xF0, 0x92, 0xF1, /* 0xEC-0xEF */ + 0x92, 0xF2, 0x92, 0xF3, 0x92, 0xF4, 0x92, 0xF5, /* 0xF0-0xF3 */ + 0x92, 0xF6, 0x92, 0xF7, 0x92, 0xF8, 0x92, 0xF9, /* 0xF4-0xF7 */ + 0xB9, 0xCC, 0xB9, 0xCD, 0x92, 0xFA, 0x92, 0xFB, /* 0xF8-0xFB */ + 0xB9, 0xCE, 0x92, 0xFC, 0x92, 0xFD, 0xB9, 0xCF, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_BC[512] = { + 0xB9, 0xD0, 0x92, 0xFE, 0xB9, 0xD1, 0x93, 0x41, /* 0x00-0x03 */ + 0x93, 0x42, 0x93, 0x43, 0x93, 0x44, 0x93, 0x45, /* 0x04-0x07 */ + 0xB9, 0xD2, 0xB9, 0xD3, 0x93, 0x46, 0xB9, 0xD4, /* 0x08-0x0B */ + 0xB9, 0xD5, 0xB9, 0xD6, 0x93, 0x47, 0xB9, 0xD7, /* 0x0C-0x0F */ + 0x93, 0x48, 0xB9, 0xD8, 0x93, 0x49, 0x93, 0x4A, /* 0x10-0x13 */ + 0xB9, 0xD9, 0xB9, 0xDA, 0xB9, 0xDB, 0xB9, 0xDC, /* 0x14-0x17 */ + 0xB9, 0xDD, 0x93, 0x4B, 0x93, 0x4C, 0xB9, 0xDE, /* 0x18-0x1B */ + 0xB9, 0xDF, 0xB9, 0xE0, 0xB9, 0xE1, 0xB9, 0xE2, /* 0x1C-0x1F */ + 0x93, 0x4D, 0x93, 0x4E, 0x93, 0x4F, 0x93, 0x50, /* 0x20-0x23 */ + 0xB9, 0xE3, 0xB9, 0xE4, 0x93, 0x51, 0xB9, 0xE5, /* 0x24-0x27 */ + 0x93, 0x52, 0xB9, 0xE6, 0x93, 0x53, 0x93, 0x54, /* 0x28-0x2B */ + 0x93, 0x55, 0xB9, 0xE7, 0x93, 0x56, 0x93, 0x57, /* 0x2C-0x2F */ + 0xB9, 0xE8, 0xB9, 0xE9, 0x93, 0x58, 0x93, 0x59, /* 0x30-0x33 */ + 0xB9, 0xEA, 0x93, 0x5A, 0x93, 0x61, 0x93, 0x62, /* 0x34-0x37 */ + 0xB9, 0xEB, 0x93, 0x63, 0x93, 0x64, 0x93, 0x65, /* 0x38-0x3B */ + 0x93, 0x66, 0x93, 0x67, 0x93, 0x68, 0x93, 0x69, /* 0x3C-0x3F */ + 0xB9, 0xEC, 0xB9, 0xED, 0x93, 0x6A, 0xB9, 0xEE, /* 0x40-0x43 */ + 0xB9, 0xEF, 0xB9, 0xF0, 0x93, 0x6B, 0x93, 0x6C, /* 0x44-0x47 */ + 0x93, 0x6D, 0xB9, 0xF1, 0x93, 0x6E, 0x93, 0x6F, /* 0x48-0x4B */ + 0xB9, 0xF2, 0xB9, 0xF3, 0x93, 0x70, 0x93, 0x71, /* 0x4C-0x4F */ + 0xB9, 0xF4, 0x93, 0x72, 0x93, 0x73, 0x93, 0x74, /* 0x50-0x53 */ + 0x93, 0x75, 0x93, 0x76, 0x93, 0x77, 0x93, 0x78, /* 0x54-0x57 */ + 0x93, 0x79, 0x93, 0x7A, 0x93, 0x81, 0x93, 0x82, /* 0x58-0x5B */ + 0x93, 0x83, 0xB9, 0xF5, 0x93, 0x84, 0x93, 0x85, /* 0x5C-0x5F */ + 0x93, 0x86, 0x93, 0x87, 0x93, 0x88, 0x93, 0x89, /* 0x60-0x63 */ + 0x93, 0x8A, 0x93, 0x8B, 0x93, 0x8C, 0x93, 0x8D, /* 0x64-0x67 */ + 0x93, 0x8E, 0x93, 0x8F, 0x93, 0x90, 0x93, 0x91, /* 0x68-0x6B */ + 0x93, 0x92, 0x93, 0x93, 0x93, 0x94, 0x93, 0x95, /* 0x6C-0x6F */ + 0x93, 0x96, 0x93, 0x97, 0x93, 0x98, 0x93, 0x99, /* 0x70-0x73 */ + 0x93, 0x9A, 0x93, 0x9B, 0x93, 0x9C, 0x93, 0x9D, /* 0x74-0x77 */ + 0x93, 0x9E, 0x93, 0x9F, 0x93, 0xA0, 0x93, 0xA1, /* 0x78-0x7B */ + 0x93, 0xA2, 0x93, 0xA3, 0x93, 0xA4, 0x93, 0xA5, /* 0x7C-0x7F */ + + 0x93, 0xA6, 0x93, 0xA7, 0x93, 0xA8, 0x93, 0xA9, /* 0x80-0x83 */ + 0xB9, 0xF6, 0xB9, 0xF7, 0x93, 0xAA, 0x93, 0xAB, /* 0x84-0x87 */ + 0xB9, 0xF8, 0x93, 0xAC, 0x93, 0xAD, 0xB9, 0xF9, /* 0x88-0x8B */ + 0xB9, 0xFA, 0x93, 0xAE, 0xB9, 0xFB, 0x93, 0xAF, /* 0x8C-0x8F */ + 0x93, 0xB0, 0x93, 0xB1, 0x93, 0xB2, 0x93, 0xB3, /* 0x90-0x93 */ + 0xB9, 0xFC, 0xB9, 0xFD, 0x93, 0xB4, 0xB9, 0xFE, /* 0x94-0x97 */ + 0x93, 0xB5, 0xBA, 0xA1, 0xBA, 0xA2, 0x93, 0xB6, /* 0x98-0x9B */ + 0x93, 0xB7, 0x93, 0xB8, 0x93, 0xB9, 0x93, 0xBA, /* 0x9C-0x9F */ + 0xBA, 0xA3, 0xBA, 0xA4, 0x93, 0xBB, 0x93, 0xBC, /* 0xA0-0xA3 */ + 0xBA, 0xA5, 0x93, 0xBD, 0x93, 0xBE, 0xBA, 0xA6, /* 0xA4-0xA7 */ + 0xBA, 0xA7, 0x93, 0xBF, 0x93, 0xC0, 0x93, 0xC1, /* 0xA8-0xAB */ + 0x93, 0xC2, 0x93, 0xC3, 0x93, 0xC4, 0x93, 0xC5, /* 0xAC-0xAF */ + 0xBA, 0xA8, 0xBA, 0xA9, 0x93, 0xC6, 0xBA, 0xAA, /* 0xB0-0xB3 */ + 0xBA, 0xAB, 0xBA, 0xAC, 0x93, 0xC7, 0x93, 0xC8, /* 0xB4-0xB7 */ + 0x93, 0xC9, 0x93, 0xCA, 0x93, 0xCB, 0x93, 0xCC, /* 0xB8-0xBB */ + 0xBA, 0xAD, 0xBA, 0xAE, 0x93, 0xCD, 0x93, 0xCE, /* 0xBC-0xBF */ + 0xBA, 0xAF, 0x93, 0xCF, 0x93, 0xD0, 0x93, 0xD1, /* 0xC0-0xC3 */ + 0xBA, 0xB0, 0x93, 0xD2, 0x93, 0xD3, 0x93, 0xD4, /* 0xC4-0xC7 */ + 0x93, 0xD5, 0x93, 0xD6, 0x93, 0xD7, 0x93, 0xD8, /* 0xC8-0xCB */ + 0x93, 0xD9, 0xBA, 0xB1, 0x93, 0xDA, 0xBA, 0xB2, /* 0xCC-0xCF */ + 0xBA, 0xB3, 0xBA, 0xB4, 0x93, 0xDB, 0x93, 0xDC, /* 0xD0-0xD3 */ + 0x93, 0xDD, 0xBA, 0xB5, 0x93, 0xDE, 0x93, 0xDF, /* 0xD4-0xD7 */ + 0xBA, 0xB6, 0x93, 0xE0, 0x93, 0xE1, 0x93, 0xE2, /* 0xD8-0xDB */ + 0xBA, 0xB7, 0x93, 0xE3, 0x93, 0xE4, 0x93, 0xE5, /* 0xDC-0xDF */ + 0x93, 0xE6, 0x93, 0xE7, 0x93, 0xE8, 0x93, 0xE9, /* 0xE0-0xE3 */ + 0x93, 0xEA, 0x93, 0xEB, 0x93, 0xEC, 0x93, 0xED, /* 0xE4-0xE7 */ + 0x93, 0xEE, 0x93, 0xEF, 0x93, 0xF0, 0x93, 0xF1, /* 0xE8-0xEB */ + 0x93, 0xF2, 0x93, 0xF3, 0x93, 0xF4, 0x93, 0xF5, /* 0xEC-0xEF */ + 0x93, 0xF6, 0x93, 0xF7, 0x93, 0xF8, 0x93, 0xF9, /* 0xF0-0xF3 */ + 0xBA, 0xB8, 0xBA, 0xB9, 0xBA, 0xBA, 0x93, 0xFA, /* 0xF4-0xF7 */ + 0xBA, 0xBB, 0x93, 0xFB, 0x93, 0xFC, 0x93, 0xFD, /* 0xF8-0xFB */ + 0xBA, 0xBC, 0x93, 0xFE, 0x94, 0x41, 0x94, 0x42, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_BD[512] = { + 0x94, 0x43, 0x94, 0x44, 0x94, 0x45, 0x94, 0x46, /* 0x00-0x03 */ + 0xBA, 0xBD, 0xBA, 0xBE, 0x94, 0x47, 0xBA, 0xBF, /* 0x04-0x07 */ + 0x94, 0x48, 0xBA, 0xC0, 0x94, 0x49, 0x94, 0x4A, /* 0x08-0x0B */ + 0x94, 0x4B, 0x94, 0x4C, 0x94, 0x4D, 0x94, 0x4E, /* 0x0C-0x0F */ + 0xBA, 0xC1, 0x94, 0x4F, 0x94, 0x50, 0x94, 0x51, /* 0x10-0x13 */ + 0xBA, 0xC2, 0x94, 0x52, 0x94, 0x53, 0x94, 0x54, /* 0x14-0x17 */ + 0x94, 0x55, 0x94, 0x56, 0x94, 0x57, 0x94, 0x58, /* 0x18-0x1B */ + 0x94, 0x59, 0x94, 0x5A, 0x94, 0x61, 0x94, 0x62, /* 0x1C-0x1F */ + 0x94, 0x63, 0x94, 0x64, 0x94, 0x65, 0x94, 0x66, /* 0x20-0x23 */ + 0xBA, 0xC3, 0x94, 0x67, 0x94, 0x68, 0x94, 0x69, /* 0x24-0x27 */ + 0x94, 0x6A, 0x94, 0x6B, 0x94, 0x6C, 0x94, 0x6D, /* 0x28-0x2B */ + 0xBA, 0xC4, 0x94, 0x6E, 0x94, 0x6F, 0x94, 0x70, /* 0x2C-0x2F */ + 0x94, 0x71, 0x94, 0x72, 0x94, 0x73, 0x94, 0x74, /* 0x30-0x33 */ + 0x94, 0x75, 0x94, 0x76, 0x94, 0x77, 0x94, 0x78, /* 0x34-0x37 */ + 0x94, 0x79, 0x94, 0x7A, 0x94, 0x81, 0x94, 0x82, /* 0x38-0x3B */ + 0x94, 0x83, 0x94, 0x84, 0x94, 0x85, 0x94, 0x86, /* 0x3C-0x3F */ + 0xBA, 0xC5, 0x94, 0x87, 0x94, 0x88, 0x94, 0x89, /* 0x40-0x43 */ + 0x94, 0x8A, 0x94, 0x8B, 0x94, 0x8C, 0x94, 0x8D, /* 0x44-0x47 */ + 0xBA, 0xC6, 0xBA, 0xC7, 0x94, 0x8E, 0x94, 0x8F, /* 0x48-0x4B */ + 0xBA, 0xC8, 0x94, 0x90, 0x94, 0x91, 0x94, 0x92, /* 0x4C-0x4F */ + 0xBA, 0xC9, 0x94, 0x93, 0x94, 0x94, 0x94, 0x95, /* 0x50-0x53 */ + 0x94, 0x96, 0x94, 0x97, 0x94, 0x98, 0x94, 0x99, /* 0x54-0x57 */ + 0xBA, 0xCA, 0xBA, 0xCB, 0x94, 0x9A, 0x94, 0x9B, /* 0x58-0x5B */ + 0x94, 0x9C, 0x94, 0x9D, 0x94, 0x9E, 0x94, 0x9F, /* 0x5C-0x5F */ + 0x94, 0xA0, 0x94, 0xA1, 0x94, 0xA2, 0x94, 0xA3, /* 0x60-0x63 */ + 0xBA, 0xCC, 0x94, 0xA4, 0x94, 0xA5, 0x94, 0xA6, /* 0x64-0x67 */ + 0xBA, 0xCD, 0x94, 0xA7, 0x94, 0xA8, 0x94, 0xA9, /* 0x68-0x6B */ + 0x94, 0xAA, 0x94, 0xAB, 0x94, 0xAC, 0x94, 0xAD, /* 0x6C-0x6F */ + 0x94, 0xAE, 0x94, 0xAF, 0x94, 0xB0, 0x94, 0xB1, /* 0x70-0x73 */ + 0x94, 0xB2, 0x94, 0xB3, 0x94, 0xB4, 0x94, 0xB5, /* 0x74-0x77 */ + 0x94, 0xB6, 0x94, 0xB7, 0x94, 0xB8, 0x94, 0xB9, /* 0x78-0x7B */ + 0x94, 0xBA, 0x94, 0xBB, 0x94, 0xBC, 0x94, 0xBD, /* 0x7C-0x7F */ + + 0xBA, 0xCE, 0xBA, 0xCF, 0x94, 0xBE, 0x94, 0xBF, /* 0x80-0x83 */ + 0xBA, 0xD0, 0x94, 0xC0, 0x94, 0xC1, 0xBA, 0xD1, /* 0x84-0x87 */ + 0xBA, 0xD2, 0xBA, 0xD3, 0xBA, 0xD4, 0x94, 0xC2, /* 0x88-0x8B */ + 0x94, 0xC3, 0x94, 0xC4, 0x94, 0xC5, 0x94, 0xC6, /* 0x8C-0x8F */ + 0xBA, 0xD5, 0xBA, 0xD6, 0x94, 0xC7, 0xBA, 0xD7, /* 0x90-0x93 */ + 0x94, 0xC8, 0xBA, 0xD8, 0x94, 0xC9, 0x94, 0xCA, /* 0x94-0x97 */ + 0x94, 0xCB, 0xBA, 0xD9, 0xBA, 0xDA, 0x94, 0xCC, /* 0x98-0x9B */ + 0xBA, 0xDB, 0x94, 0xCD, 0x94, 0xCE, 0x94, 0xCF, /* 0x9C-0x9F */ + 0x94, 0xD0, 0x94, 0xD1, 0x94, 0xD2, 0x94, 0xD3, /* 0xA0-0xA3 */ + 0xBA, 0xDC, 0x94, 0xD4, 0x94, 0xD5, 0x94, 0xD6, /* 0xA4-0xA7 */ + 0x94, 0xD7, 0x94, 0xD8, 0x94, 0xD9, 0x94, 0xDA, /* 0xA8-0xAB */ + 0x94, 0xDB, 0x94, 0xDC, 0x94, 0xDD, 0x94, 0xDE, /* 0xAC-0xAF */ + 0xBA, 0xDD, 0x94, 0xDF, 0x94, 0xE0, 0x94, 0xE1, /* 0xB0-0xB3 */ + 0x94, 0xE2, 0x94, 0xE3, 0x94, 0xE4, 0x94, 0xE5, /* 0xB4-0xB7 */ + 0xBA, 0xDE, 0x94, 0xE6, 0x94, 0xE7, 0x94, 0xE8, /* 0xB8-0xBB */ + 0x94, 0xE9, 0x94, 0xEA, 0x94, 0xEB, 0x94, 0xEC, /* 0xBC-0xBF */ + 0x94, 0xED, 0x94, 0xEE, 0x94, 0xEF, 0x94, 0xF0, /* 0xC0-0xC3 */ + 0x94, 0xF1, 0x94, 0xF2, 0x94, 0xF3, 0x94, 0xF4, /* 0xC4-0xC7 */ + 0x94, 0xF5, 0x94, 0xF6, 0x94, 0xF7, 0x94, 0xF8, /* 0xC8-0xCB */ + 0x94, 0xF9, 0x94, 0xFA, 0x94, 0xFB, 0x94, 0xFC, /* 0xCC-0xCF */ + 0x94, 0xFD, 0x94, 0xFE, 0x95, 0x41, 0x95, 0x42, /* 0xD0-0xD3 */ + 0xBA, 0xDF, 0xBA, 0xE0, 0x95, 0x43, 0x95, 0x44, /* 0xD4-0xD7 */ + 0xBA, 0xE1, 0x95, 0x45, 0x95, 0x46, 0x95, 0x47, /* 0xD8-0xDB */ + 0xBA, 0xE2, 0x95, 0x48, 0x95, 0x49, 0x95, 0x4A, /* 0xDC-0xDF */ + 0x95, 0x4B, 0x95, 0x4C, 0x95, 0x4D, 0x95, 0x4E, /* 0xE0-0xE3 */ + 0x95, 0x4F, 0x95, 0x50, 0x95, 0x51, 0x95, 0x52, /* 0xE4-0xE7 */ + 0x95, 0x53, 0xBA, 0xE3, 0x95, 0x54, 0x95, 0x55, /* 0xE8-0xEB */ + 0x95, 0x56, 0x95, 0x57, 0x95, 0x58, 0x95, 0x59, /* 0xEC-0xEF */ + 0xBA, 0xE4, 0x95, 0x5A, 0x95, 0x61, 0x95, 0x62, /* 0xF0-0xF3 */ + 0xBA, 0xE5, 0x95, 0x63, 0x95, 0x64, 0x95, 0x65, /* 0xF4-0xF7 */ + 0xBA, 0xE6, 0x95, 0x66, 0x95, 0x67, 0x95, 0x68, /* 0xF8-0xFB */ + 0x95, 0x69, 0x95, 0x6A, 0x95, 0x6B, 0x95, 0x6C, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_BE[512] = { + 0xBA, 0xE7, 0x95, 0x6D, 0x95, 0x6E, 0xBA, 0xE8, /* 0x00-0x03 */ + 0x95, 0x6F, 0xBA, 0xE9, 0x95, 0x70, 0x95, 0x71, /* 0x04-0x07 */ + 0x95, 0x72, 0x95, 0x73, 0x95, 0x74, 0x95, 0x75, /* 0x08-0x0B */ + 0xBA, 0xEA, 0xBA, 0xEB, 0x95, 0x76, 0x95, 0x77, /* 0x0C-0x0F */ + 0xBA, 0xEC, 0x95, 0x78, 0x95, 0x79, 0x95, 0x7A, /* 0x10-0x13 */ + 0xBA, 0xED, 0x95, 0x81, 0x95, 0x82, 0x95, 0x83, /* 0x14-0x17 */ + 0x95, 0x84, 0x95, 0x85, 0x95, 0x86, 0x95, 0x87, /* 0x18-0x1B */ + 0xBA, 0xEE, 0xBA, 0xEF, 0x95, 0x88, 0xBA, 0xF0, /* 0x1C-0x1F */ + 0x95, 0x89, 0x95, 0x8A, 0x95, 0x8B, 0x95, 0x8C, /* 0x20-0x23 */ + 0x95, 0x8D, 0x95, 0x8E, 0x95, 0x8F, 0x95, 0x90, /* 0x24-0x27 */ + 0x95, 0x91, 0x95, 0x92, 0x95, 0x93, 0x95, 0x94, /* 0x28-0x2B */ + 0x95, 0x95, 0x95, 0x96, 0x95, 0x97, 0x95, 0x98, /* 0x2C-0x2F */ + 0x95, 0x99, 0x95, 0x9A, 0x95, 0x9B, 0x95, 0x9C, /* 0x30-0x33 */ + 0x95, 0x9D, 0x95, 0x9E, 0x95, 0x9F, 0x95, 0xA0, /* 0x34-0x37 */ + 0x95, 0xA1, 0x95, 0xA2, 0x95, 0xA3, 0x95, 0xA4, /* 0x38-0x3B */ + 0x95, 0xA5, 0x95, 0xA6, 0x95, 0xA7, 0x95, 0xA8, /* 0x3C-0x3F */ + 0x95, 0xA9, 0x95, 0xAA, 0x95, 0xAB, 0x95, 0xAC, /* 0x40-0x43 */ + 0xBA, 0xF1, 0xBA, 0xF2, 0x95, 0xAD, 0x95, 0xAE, /* 0x44-0x47 */ + 0xBA, 0xF3, 0x95, 0xAF, 0x95, 0xB0, 0x95, 0xB1, /* 0x48-0x4B */ + 0xBA, 0xF4, 0x95, 0xB2, 0xBA, 0xF5, 0x95, 0xB3, /* 0x4C-0x4F */ + 0x95, 0xB4, 0x95, 0xB5, 0x95, 0xB6, 0x95, 0xB7, /* 0x50-0x53 */ + 0xBA, 0xF6, 0xBA, 0xF7, 0x95, 0xB8, 0xBA, 0xF8, /* 0x54-0x57 */ + 0x95, 0xB9, 0xBA, 0xF9, 0xBA, 0xFA, 0xBA, 0xFB, /* 0x58-0x5B */ + 0x95, 0xBA, 0x95, 0xBB, 0x95, 0xBC, 0x95, 0xBD, /* 0x5C-0x5F */ + 0xBA, 0xFC, 0xBA, 0xFD, 0x95, 0xBE, 0x95, 0xBF, /* 0x60-0x63 */ + 0xBA, 0xFE, 0x95, 0xC0, 0x95, 0xC1, 0x95, 0xC2, /* 0x64-0x67 */ + 0xBB, 0xA1, 0x95, 0xC3, 0xBB, 0xA2, 0x95, 0xC4, /* 0x68-0x6B */ + 0x95, 0xC5, 0x95, 0xC6, 0x95, 0xC7, 0x95, 0xC8, /* 0x6C-0x6F */ + 0xBB, 0xA3, 0xBB, 0xA4, 0x95, 0xC9, 0xBB, 0xA5, /* 0x70-0x73 */ + 0xBB, 0xA6, 0xBB, 0xA7, 0x95, 0xCA, 0x95, 0xCB, /* 0x74-0x77 */ + 0x95, 0xCC, 0x95, 0xCD, 0x95, 0xCE, 0xBB, 0xA8, /* 0x78-0x7B */ + 0xBB, 0xA9, 0xBB, 0xAA, 0x95, 0xCF, 0x95, 0xD0, /* 0x7C-0x7F */ + + 0xBB, 0xAB, 0x95, 0xD1, 0x95, 0xD2, 0x95, 0xD3, /* 0x80-0x83 */ + 0xBB, 0xAC, 0x95, 0xD4, 0x95, 0xD5, 0x95, 0xD6, /* 0x84-0x87 */ + 0x95, 0xD7, 0x95, 0xD8, 0x95, 0xD9, 0x95, 0xDA, /* 0x88-0x8B */ + 0xBB, 0xAD, 0xBB, 0xAE, 0x95, 0xDB, 0xBB, 0xAF, /* 0x8C-0x8F */ + 0xBB, 0xB0, 0xBB, 0xB1, 0x95, 0xDC, 0x95, 0xDD, /* 0x90-0x93 */ + 0x95, 0xDE, 0x95, 0xDF, 0x95, 0xE0, 0x95, 0xE1, /* 0x94-0x97 */ + 0xBB, 0xB2, 0xBB, 0xB3, 0x95, 0xE2, 0x95, 0xE3, /* 0x98-0x9B */ + 0x95, 0xE4, 0x95, 0xE5, 0x95, 0xE6, 0x95, 0xE7, /* 0x9C-0x9F */ + 0x95, 0xE8, 0x95, 0xE9, 0x95, 0xEA, 0x95, 0xEB, /* 0xA0-0xA3 */ + 0x95, 0xEC, 0x95, 0xED, 0x95, 0xEE, 0x95, 0xEF, /* 0xA4-0xA7 */ + 0xBB, 0xB4, 0x95, 0xF0, 0x95, 0xF1, 0x95, 0xF2, /* 0xA8-0xAB */ + 0x95, 0xF3, 0x95, 0xF4, 0x95, 0xF5, 0x95, 0xF6, /* 0xAC-0xAF */ + 0x95, 0xF7, 0x95, 0xF8, 0x95, 0xF9, 0x95, 0xFA, /* 0xB0-0xB3 */ + 0x95, 0xFB, 0x95, 0xFC, 0x95, 0xFD, 0x95, 0xFE, /* 0xB4-0xB7 */ + 0x96, 0x41, 0x96, 0x42, 0x96, 0x43, 0x96, 0x44, /* 0xB8-0xBB */ + 0x96, 0x45, 0x96, 0x46, 0x96, 0x47, 0x96, 0x48, /* 0xBC-0xBF */ + 0x96, 0x49, 0x96, 0x4A, 0x96, 0x4B, 0x96, 0x4C, /* 0xC0-0xC3 */ + 0x96, 0x4D, 0x96, 0x4E, 0x96, 0x4F, 0x96, 0x50, /* 0xC4-0xC7 */ + 0x96, 0x51, 0x96, 0x52, 0x96, 0x53, 0x96, 0x54, /* 0xC8-0xCB */ + 0x96, 0x55, 0x96, 0x56, 0x96, 0x57, 0x96, 0x58, /* 0xCC-0xCF */ + 0xBB, 0xB5, 0xBB, 0xB6, 0x96, 0x59, 0x96, 0x5A, /* 0xD0-0xD3 */ + 0xBB, 0xB7, 0x96, 0x61, 0x96, 0x62, 0xBB, 0xB8, /* 0xD4-0xD7 */ + 0xBB, 0xB9, 0x96, 0x63, 0x96, 0x64, 0x96, 0x65, /* 0xD8-0xDB */ + 0x96, 0x66, 0x96, 0x67, 0x96, 0x68, 0x96, 0x69, /* 0xDC-0xDF */ + 0xBB, 0xBA, 0x96, 0x6A, 0x96, 0x6B, 0xBB, 0xBB, /* 0xE0-0xE3 */ + 0xBB, 0xBC, 0xBB, 0xBD, 0x96, 0x6C, 0x96, 0x6D, /* 0xE4-0xE7 */ + 0x96, 0x6E, 0x96, 0x6F, 0x96, 0x70, 0x96, 0x71, /* 0xE8-0xEB */ + 0xBB, 0xBE, 0x96, 0x72, 0x96, 0x73, 0x96, 0x74, /* 0xEC-0xEF */ + 0x96, 0x75, 0x96, 0x76, 0x96, 0x77, 0x96, 0x78, /* 0xF0-0xF3 */ + 0x96, 0x79, 0x96, 0x7A, 0x96, 0x81, 0x96, 0x82, /* 0xF4-0xF7 */ + 0x96, 0x83, 0x96, 0x84, 0x96, 0x85, 0x96, 0x86, /* 0xF8-0xFB */ + 0x96, 0x87, 0x96, 0x88, 0x96, 0x89, 0x96, 0x8A, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_BF[512] = { + 0x96, 0x8B, 0xBB, 0xBF, 0x96, 0x8C, 0x96, 0x8D, /* 0x00-0x03 */ + 0x96, 0x8E, 0x96, 0x8F, 0x96, 0x90, 0x96, 0x91, /* 0x04-0x07 */ + 0xBB, 0xC0, 0xBB, 0xC1, 0x96, 0x92, 0x96, 0x93, /* 0x08-0x0B */ + 0x96, 0x94, 0x96, 0x95, 0x96, 0x96, 0x96, 0x97, /* 0x0C-0x0F */ + 0x96, 0x98, 0x96, 0x99, 0x96, 0x9A, 0x96, 0x9B, /* 0x10-0x13 */ + 0x96, 0x9C, 0x96, 0x9D, 0x96, 0x9E, 0x96, 0x9F, /* 0x14-0x17 */ + 0xBB, 0xC2, 0xBB, 0xC3, 0x96, 0xA0, 0xBB, 0xC4, /* 0x18-0x1B */ + 0xBB, 0xC5, 0xBB, 0xC6, 0x96, 0xA1, 0x96, 0xA2, /* 0x1C-0x1F */ + 0x96, 0xA3, 0x96, 0xA4, 0x96, 0xA5, 0x96, 0xA6, /* 0x20-0x23 */ + 0x96, 0xA7, 0x96, 0xA8, 0x96, 0xA9, 0x96, 0xAA, /* 0x24-0x27 */ + 0x96, 0xAB, 0x96, 0xAC, 0x96, 0xAD, 0x96, 0xAE, /* 0x28-0x2B */ + 0x96, 0xAF, 0x96, 0xB0, 0x96, 0xB1, 0x96, 0xB2, /* 0x2C-0x2F */ + 0x96, 0xB3, 0x96, 0xB4, 0x96, 0xB5, 0x96, 0xB6, /* 0x30-0x33 */ + 0x96, 0xB7, 0x96, 0xB8, 0x96, 0xB9, 0x96, 0xBA, /* 0x34-0x37 */ + 0x96, 0xBB, 0x96, 0xBC, 0x96, 0xBD, 0x96, 0xBE, /* 0x38-0x3B */ + 0x96, 0xBF, 0x96, 0xC0, 0x96, 0xC1, 0x96, 0xC2, /* 0x3C-0x3F */ + 0xBB, 0xC7, 0xBB, 0xC8, 0x96, 0xC3, 0x96, 0xC4, /* 0x40-0x43 */ + 0xBB, 0xC9, 0x96, 0xC5, 0x96, 0xC6, 0x96, 0xC7, /* 0x44-0x47 */ + 0xBB, 0xCA, 0x96, 0xC8, 0x96, 0xC9, 0x96, 0xCA, /* 0x48-0x4B */ + 0x96, 0xCB, 0x96, 0xCC, 0x96, 0xCD, 0x96, 0xCE, /* 0x4C-0x4F */ + 0xBB, 0xCB, 0xBB, 0xCC, 0x96, 0xCF, 0x96, 0xD0, /* 0x50-0x53 */ + 0x96, 0xD1, 0xBB, 0xCD, 0x96, 0xD2, 0x96, 0xD3, /* 0x54-0x57 */ + 0x96, 0xD4, 0x96, 0xD5, 0x96, 0xD6, 0x96, 0xD7, /* 0x58-0x5B */ + 0x96, 0xD8, 0x96, 0xD9, 0x96, 0xDA, 0x96, 0xDB, /* 0x5C-0x5F */ + 0x96, 0xDC, 0x96, 0xDD, 0x96, 0xDE, 0x96, 0xDF, /* 0x60-0x63 */ + 0x96, 0xE0, 0x96, 0xE1, 0x96, 0xE2, 0x96, 0xE3, /* 0x64-0x67 */ + 0x96, 0xE4, 0x96, 0xE5, 0x96, 0xE6, 0x96, 0xE7, /* 0x68-0x6B */ + 0x96, 0xE8, 0x96, 0xE9, 0x96, 0xEA, 0x96, 0xEB, /* 0x6C-0x6F */ + 0x96, 0xEC, 0x96, 0xED, 0x96, 0xEE, 0x96, 0xEF, /* 0x70-0x73 */ + 0x96, 0xF0, 0x96, 0xF1, 0x96, 0xF2, 0x96, 0xF3, /* 0x74-0x77 */ + 0x96, 0xF4, 0x96, 0xF5, 0x96, 0xF6, 0x96, 0xF7, /* 0x78-0x7B */ + 0x96, 0xF8, 0x96, 0xF9, 0x96, 0xFA, 0x96, 0xFB, /* 0x7C-0x7F */ + + 0x96, 0xFC, 0x96, 0xFD, 0x96, 0xFE, 0x97, 0x41, /* 0x80-0x83 */ + 0x97, 0x42, 0x97, 0x43, 0x97, 0x44, 0x97, 0x45, /* 0x84-0x87 */ + 0x97, 0x46, 0x97, 0x47, 0x97, 0x48, 0x97, 0x49, /* 0x88-0x8B */ + 0x97, 0x4A, 0x97, 0x4B, 0x97, 0x4C, 0x97, 0x4D, /* 0x8C-0x8F */ + 0x97, 0x4E, 0x97, 0x4F, 0x97, 0x50, 0x97, 0x51, /* 0x90-0x93 */ + 0xBB, 0xCE, 0x97, 0x52, 0x97, 0x53, 0x97, 0x54, /* 0x94-0x97 */ + 0x97, 0x55, 0x97, 0x56, 0x97, 0x57, 0x97, 0x58, /* 0x98-0x9B */ + 0x97, 0x59, 0x97, 0x5A, 0x97, 0x61, 0x97, 0x62, /* 0x9C-0x9F */ + 0x97, 0x63, 0x97, 0x64, 0x97, 0x65, 0x97, 0x66, /* 0xA0-0xA3 */ + 0x97, 0x67, 0x97, 0x68, 0x97, 0x69, 0x97, 0x6A, /* 0xA4-0xA7 */ + 0x97, 0x6B, 0x97, 0x6C, 0x97, 0x6D, 0x97, 0x6E, /* 0xA8-0xAB */ + 0x97, 0x6F, 0x97, 0x70, 0x97, 0x71, 0x97, 0x72, /* 0xAC-0xAF */ + 0xBB, 0xCF, 0x97, 0x73, 0x97, 0x74, 0x97, 0x75, /* 0xB0-0xB3 */ + 0x97, 0x76, 0x97, 0x77, 0x97, 0x78, 0x97, 0x79, /* 0xB4-0xB7 */ + 0x97, 0x7A, 0x97, 0x81, 0x97, 0x82, 0x97, 0x83, /* 0xB8-0xBB */ + 0x97, 0x84, 0x97, 0x85, 0x97, 0x86, 0x97, 0x87, /* 0xBC-0xBF */ + 0x97, 0x88, 0x97, 0x89, 0x97, 0x8A, 0x97, 0x8B, /* 0xC0-0xC3 */ + 0x97, 0x8C, 0xBB, 0xD0, 0x97, 0x8D, 0x97, 0x8E, /* 0xC4-0xC7 */ + 0x97, 0x8F, 0x97, 0x90, 0x97, 0x91, 0x97, 0x92, /* 0xC8-0xCB */ + 0xBB, 0xD1, 0xBB, 0xD2, 0x97, 0x93, 0x97, 0x94, /* 0xCC-0xCF */ + 0xBB, 0xD3, 0x97, 0x95, 0x97, 0x96, 0x97, 0x97, /* 0xD0-0xD3 */ + 0xBB, 0xD4, 0x97, 0x98, 0x97, 0x99, 0x97, 0x9A, /* 0xD4-0xD7 */ + 0x97, 0x9B, 0x97, 0x9C, 0x97, 0x9D, 0x97, 0x9E, /* 0xD8-0xDB */ + 0xBB, 0xD5, 0x97, 0x9F, 0x97, 0xA0, 0xBB, 0xD6, /* 0xDC-0xDF */ + 0x97, 0xA1, 0xBB, 0xD7, 0x97, 0xA2, 0x97, 0xA3, /* 0xE0-0xE3 */ + 0x97, 0xA4, 0x97, 0xA5, 0x97, 0xA6, 0x97, 0xA7, /* 0xE4-0xE7 */ + 0x97, 0xA8, 0x97, 0xA9, 0x97, 0xAA, 0x97, 0xAB, /* 0xE8-0xEB */ + 0x97, 0xAC, 0x97, 0xAD, 0x97, 0xAE, 0x97, 0xAF, /* 0xEC-0xEF */ + 0x97, 0xB0, 0x97, 0xB1, 0x97, 0xB2, 0x97, 0xB3, /* 0xF0-0xF3 */ + 0x97, 0xB4, 0x97, 0xB5, 0x97, 0xB6, 0x97, 0xB7, /* 0xF4-0xF7 */ + 0x97, 0xB8, 0x97, 0xB9, 0x97, 0xBA, 0x97, 0xBB, /* 0xF8-0xFB */ + 0x97, 0xBC, 0x97, 0xBD, 0x97, 0xBE, 0x97, 0xBF, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_C0[512] = { + 0x97, 0xC0, 0x97, 0xC1, 0x97, 0xC2, 0x97, 0xC3, /* 0x00-0x03 */ + 0x97, 0xC4, 0x97, 0xC5, 0x97, 0xC6, 0x97, 0xC7, /* 0x04-0x07 */ + 0x97, 0xC8, 0x97, 0xC9, 0x97, 0xCA, 0x97, 0xCB, /* 0x08-0x0B */ + 0x97, 0xCC, 0x97, 0xCD, 0x97, 0xCE, 0x97, 0xCF, /* 0x0C-0x0F */ + 0x97, 0xD0, 0x97, 0xD1, 0x97, 0xD2, 0x97, 0xD3, /* 0x10-0x13 */ + 0x97, 0xD4, 0x97, 0xD5, 0x97, 0xD6, 0x97, 0xD7, /* 0x14-0x17 */ + 0x97, 0xD8, 0x97, 0xD9, 0x97, 0xDA, 0x97, 0xDB, /* 0x18-0x1B */ + 0x97, 0xDC, 0x97, 0xDD, 0x97, 0xDE, 0x97, 0xDF, /* 0x1C-0x1F */ + 0x97, 0xE0, 0x97, 0xE1, 0x97, 0xE2, 0x97, 0xE3, /* 0x20-0x23 */ + 0x97, 0xE4, 0x97, 0xE5, 0x97, 0xE6, 0x97, 0xE7, /* 0x24-0x27 */ + 0x97, 0xE8, 0x97, 0xE9, 0x97, 0xEA, 0x97, 0xEB, /* 0x28-0x2B */ + 0x97, 0xEC, 0x97, 0xED, 0x97, 0xEE, 0x97, 0xEF, /* 0x2C-0x2F */ + 0x97, 0xF0, 0x97, 0xF1, 0x97, 0xF2, 0x97, 0xF3, /* 0x30-0x33 */ + 0x97, 0xF4, 0x97, 0xF5, 0x97, 0xF6, 0x97, 0xF7, /* 0x34-0x37 */ + 0x97, 0xF8, 0x97, 0xF9, 0x97, 0xFA, 0x97, 0xFB, /* 0x38-0x3B */ + 0xBB, 0xD8, 0x97, 0xFC, 0x97, 0xFD, 0x97, 0xFE, /* 0x3C-0x3F */ + 0x98, 0x41, 0x98, 0x42, 0x98, 0x43, 0x98, 0x44, /* 0x40-0x43 */ + 0x98, 0x45, 0x98, 0x46, 0x98, 0x47, 0x98, 0x48, /* 0x44-0x47 */ + 0x98, 0x49, 0x98, 0x4A, 0x98, 0x4B, 0x98, 0x4C, /* 0x48-0x4B */ + 0x98, 0x4D, 0x98, 0x4E, 0x98, 0x4F, 0x98, 0x50, /* 0x4C-0x4F */ + 0x98, 0x51, 0xBB, 0xD9, 0x98, 0x52, 0x98, 0x53, /* 0x50-0x53 */ + 0x98, 0x54, 0x98, 0x55, 0x98, 0x56, 0x98, 0x57, /* 0x54-0x57 */ + 0xBB, 0xDA, 0x98, 0x58, 0x98, 0x59, 0x98, 0x5A, /* 0x58-0x5B */ + 0xBB, 0xDB, 0x98, 0x61, 0x98, 0x62, 0x98, 0x63, /* 0x5C-0x5F */ + 0xBB, 0xDC, 0x98, 0x64, 0x98, 0x65, 0x98, 0x66, /* 0x60-0x63 */ + 0x98, 0x67, 0x98, 0x68, 0x98, 0x69, 0x98, 0x6A, /* 0x64-0x67 */ + 0xBB, 0xDD, 0xBB, 0xDE, 0x98, 0x6B, 0x98, 0x6C, /* 0x68-0x6B */ + 0x98, 0x6D, 0x98, 0x6E, 0x98, 0x6F, 0x98, 0x70, /* 0x6C-0x6F */ + 0x98, 0x71, 0x98, 0x72, 0x98, 0x73, 0x98, 0x74, /* 0x70-0x73 */ + 0x98, 0x75, 0x98, 0x76, 0x98, 0x77, 0x98, 0x78, /* 0x74-0x77 */ + 0x98, 0x79, 0x98, 0x7A, 0x98, 0x81, 0x98, 0x82, /* 0x78-0x7B */ + 0x98, 0x83, 0x98, 0x84, 0x98, 0x85, 0x98, 0x86, /* 0x7C-0x7F */ + + 0x98, 0x87, 0x98, 0x88, 0x98, 0x89, 0x98, 0x8A, /* 0x80-0x83 */ + 0x98, 0x8B, 0x98, 0x8C, 0x98, 0x8D, 0x98, 0x8E, /* 0x84-0x87 */ + 0x98, 0x8F, 0x98, 0x90, 0x98, 0x91, 0x98, 0x92, /* 0x88-0x8B */ + 0x98, 0x93, 0x98, 0x94, 0x98, 0x95, 0x98, 0x96, /* 0x8C-0x8F */ + 0xBB, 0xDF, 0xBB, 0xE0, 0x98, 0x97, 0x98, 0x98, /* 0x90-0x93 */ + 0xBB, 0xE1, 0x98, 0x99, 0x98, 0x9A, 0x98, 0x9B, /* 0x94-0x97 */ + 0xBB, 0xE2, 0x98, 0x9C, 0x98, 0x9D, 0x98, 0x9E, /* 0x98-0x9B */ + 0x98, 0x9F, 0x98, 0xA0, 0x98, 0xA1, 0x98, 0xA2, /* 0x9C-0x9F */ + 0xBB, 0xE3, 0xBB, 0xE4, 0x98, 0xA3, 0xBB, 0xE5, /* 0xA0-0xA3 */ + 0x98, 0xA4, 0xBB, 0xE6, 0x98, 0xA5, 0x98, 0xA6, /* 0xA4-0xA7 */ + 0x98, 0xA7, 0x98, 0xA8, 0x98, 0xA9, 0x98, 0xAA, /* 0xA8-0xAB */ + 0xBB, 0xE7, 0xBB, 0xE8, 0x98, 0xAB, 0xBB, 0xE9, /* 0xAC-0xAF */ + 0xBB, 0xEA, 0x98, 0xAC, 0x98, 0xAD, 0xBB, 0xEB, /* 0xB0-0xB3 */ + 0xBB, 0xEC, 0xBB, 0xED, 0xBB, 0xEE, 0x98, 0xAE, /* 0xB4-0xB7 */ + 0x98, 0xAF, 0x98, 0xB0, 0x98, 0xB1, 0x98, 0xB2, /* 0xB8-0xBB */ + 0xBB, 0xEF, 0xBB, 0xF0, 0x98, 0xB3, 0xBB, 0xF1, /* 0xBC-0xBF */ + 0xBB, 0xF2, 0xBB, 0xF3, 0x98, 0xB4, 0x98, 0xB5, /* 0xC0-0xC3 */ + 0x98, 0xB6, 0xBB, 0xF4, 0x98, 0xB7, 0x98, 0xB8, /* 0xC4-0xC7 */ + 0xBB, 0xF5, 0xBB, 0xF6, 0x98, 0xB9, 0x98, 0xBA, /* 0xC8-0xCB */ + 0xBB, 0xF7, 0x98, 0xBB, 0x98, 0xBC, 0x98, 0xBD, /* 0xCC-0xCF */ + 0xBB, 0xF8, 0x98, 0xBE, 0x98, 0xBF, 0x98, 0xC0, /* 0xD0-0xD3 */ + 0x98, 0xC1, 0x98, 0xC2, 0x98, 0xC3, 0x98, 0xC4, /* 0xD4-0xD7 */ + 0xBB, 0xF9, 0xBB, 0xFA, 0x98, 0xC5, 0xBB, 0xFB, /* 0xD8-0xDB */ + 0xBB, 0xFC, 0xBB, 0xFD, 0x98, 0xC6, 0x98, 0xC7, /* 0xDC-0xDF */ + 0x98, 0xC8, 0x98, 0xC9, 0x98, 0xCA, 0x98, 0xCB, /* 0xE0-0xE3 */ + 0xBB, 0xFE, 0xBC, 0xA1, 0x98, 0xCC, 0x98, 0xCD, /* 0xE4-0xE7 */ + 0xBC, 0xA2, 0x98, 0xCE, 0x98, 0xCF, 0x98, 0xD0, /* 0xE8-0xEB */ + 0xBC, 0xA3, 0x98, 0xD1, 0x98, 0xD2, 0x98, 0xD3, /* 0xEC-0xEF */ + 0x98, 0xD4, 0x98, 0xD5, 0x98, 0xD6, 0x98, 0xD7, /* 0xF0-0xF3 */ + 0xBC, 0xA4, 0xBC, 0xA5, 0x98, 0xD8, 0xBC, 0xA6, /* 0xF4-0xF7 */ + 0x98, 0xD9, 0xBC, 0xA7, 0x98, 0xDA, 0x98, 0xDB, /* 0xF8-0xFB */ + 0x98, 0xDC, 0x98, 0xDD, 0x98, 0xDE, 0x98, 0xDF, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_C1[512] = { + 0xBC, 0xA8, 0x98, 0xE0, 0x98, 0xE1, 0x98, 0xE2, /* 0x00-0x03 */ + 0xBC, 0xA9, 0x98, 0xE3, 0x98, 0xE4, 0x98, 0xE5, /* 0x04-0x07 */ + 0xBC, 0xAA, 0x98, 0xE6, 0x98, 0xE7, 0x98, 0xE8, /* 0x08-0x0B */ + 0x98, 0xE9, 0x98, 0xEA, 0x98, 0xEB, 0x98, 0xEC, /* 0x0C-0x0F */ + 0xBC, 0xAB, 0x98, 0xED, 0x98, 0xEE, 0x98, 0xEF, /* 0x10-0x13 */ + 0x98, 0xF0, 0xBC, 0xAC, 0x98, 0xF1, 0x98, 0xF2, /* 0x14-0x17 */ + 0x98, 0xF3, 0x98, 0xF4, 0x98, 0xF5, 0x98, 0xF6, /* 0x18-0x1B */ + 0xBC, 0xAD, 0xBC, 0xAE, 0xBC, 0xAF, 0xBC, 0xB0, /* 0x1C-0x1F */ + 0xBC, 0xB1, 0x98, 0xF7, 0x98, 0xF8, 0xBC, 0xB2, /* 0x20-0x23 */ + 0xBC, 0xB3, 0x98, 0xF9, 0xBC, 0xB4, 0xBC, 0xB5, /* 0x24-0x27 */ + 0x98, 0xFA, 0x98, 0xFB, 0x98, 0xFC, 0x98, 0xFD, /* 0x28-0x2B */ + 0xBC, 0xB6, 0xBC, 0xB7, 0x98, 0xFE, 0xBC, 0xB8, /* 0x2C-0x2F */ + 0xBC, 0xB9, 0xBC, 0xBA, 0x99, 0x41, 0x99, 0x42, /* 0x30-0x33 */ + 0x99, 0x43, 0x99, 0x44, 0xBC, 0xBB, 0x99, 0x45, /* 0x34-0x37 */ + 0xBC, 0xBC, 0xBC, 0xBD, 0x99, 0x46, 0x99, 0x47, /* 0x38-0x3B */ + 0xBC, 0xBE, 0x99, 0x48, 0x99, 0x49, 0x99, 0x4A, /* 0x3C-0x3F */ + 0xBC, 0xBF, 0x99, 0x4B, 0x99, 0x4C, 0x99, 0x4D, /* 0x40-0x43 */ + 0x99, 0x4E, 0x99, 0x4F, 0x99, 0x50, 0x99, 0x51, /* 0x44-0x47 */ + 0xBC, 0xC0, 0xBC, 0xC1, 0x99, 0x52, 0xBC, 0xC2, /* 0x48-0x4B */ + 0xBC, 0xC3, 0xBC, 0xC4, 0x99, 0x53, 0x99, 0x54, /* 0x4C-0x4F */ + 0x99, 0x55, 0x99, 0x56, 0x99, 0x57, 0x99, 0x58, /* 0x50-0x53 */ + 0xBC, 0xC5, 0xBC, 0xC6, 0x99, 0x59, 0x99, 0x5A, /* 0x54-0x57 */ + 0xBC, 0xC7, 0x99, 0x61, 0x99, 0x62, 0x99, 0x63, /* 0x58-0x5B */ + 0xBC, 0xC8, 0x99, 0x64, 0x99, 0x65, 0x99, 0x66, /* 0x5C-0x5F */ + 0x99, 0x67, 0x99, 0x68, 0x99, 0x69, 0x99, 0x6A, /* 0x60-0x63 */ + 0xBC, 0xC9, 0xBC, 0xCA, 0x99, 0x6B, 0xBC, 0xCB, /* 0x64-0x67 */ + 0xBC, 0xCC, 0xBC, 0xCD, 0x99, 0x6C, 0x99, 0x6D, /* 0x68-0x6B */ + 0x99, 0x6E, 0x99, 0x6F, 0x99, 0x70, 0x99, 0x71, /* 0x6C-0x6F */ + 0xBC, 0xCE, 0x99, 0x72, 0x99, 0x73, 0x99, 0x74, /* 0x70-0x73 */ + 0xBC, 0xCF, 0x99, 0x75, 0x99, 0x76, 0x99, 0x77, /* 0x74-0x77 */ + 0xBC, 0xD0, 0x99, 0x78, 0x99, 0x79, 0x99, 0x7A, /* 0x78-0x7B */ + 0x99, 0x81, 0x99, 0x82, 0x99, 0x83, 0x99, 0x84, /* 0x7C-0x7F */ + + 0x99, 0x85, 0x99, 0x86, 0x99, 0x87, 0x99, 0x88, /* 0x80-0x83 */ + 0x99, 0x89, 0xBC, 0xD1, 0x99, 0x8A, 0x99, 0x8B, /* 0x84-0x87 */ + 0x99, 0x8C, 0x99, 0x8D, 0x99, 0x8E, 0x99, 0x8F, /* 0x88-0x8B */ + 0xBC, 0xD2, 0xBC, 0xD3, 0xBC, 0xD4, 0x99, 0x90, /* 0x8C-0x8F */ + 0xBC, 0xD5, 0x99, 0x91, 0x99, 0x92, 0x99, 0x93, /* 0x90-0x93 */ + 0xBC, 0xD6, 0x99, 0x94, 0xBC, 0xD7, 0x99, 0x95, /* 0x94-0x97 */ + 0x99, 0x96, 0x99, 0x97, 0x99, 0x98, 0x99, 0x99, /* 0x98-0x9B */ + 0xBC, 0xD8, 0xBC, 0xD9, 0x99, 0x9A, 0xBC, 0xDA, /* 0x9C-0x9F */ + 0x99, 0x9B, 0xBC, 0xDB, 0x99, 0x9C, 0x99, 0x9D, /* 0xA0-0xA3 */ + 0x99, 0x9E, 0xBC, 0xDC, 0x99, 0x9F, 0x99, 0xA0, /* 0xA4-0xA7 */ + 0xBC, 0xDD, 0xBC, 0xDE, 0x99, 0xA1, 0x99, 0xA2, /* 0xA8-0xAB */ + 0xBC, 0xDF, 0x99, 0xA3, 0x99, 0xA4, 0x99, 0xA5, /* 0xAC-0xAF */ + 0xBC, 0xE0, 0x99, 0xA6, 0x99, 0xA7, 0x99, 0xA8, /* 0xB0-0xB3 */ + 0x99, 0xA9, 0x99, 0xAA, 0x99, 0xAB, 0x99, 0xAC, /* 0xB4-0xB7 */ + 0x99, 0xAD, 0x99, 0xAE, 0x99, 0xAF, 0x99, 0xB0, /* 0xB8-0xBB */ + 0x99, 0xB1, 0xBC, 0xE1, 0x99, 0xB2, 0x99, 0xB3, /* 0xBC-0xBF */ + 0x99, 0xB4, 0x99, 0xB5, 0x99, 0xB6, 0x99, 0xB7, /* 0xC0-0xC3 */ + 0xBC, 0xE2, 0x99, 0xB8, 0x99, 0xB9, 0x99, 0xBA, /* 0xC4-0xC7 */ + 0xBC, 0xE3, 0x99, 0xBB, 0x99, 0xBC, 0x99, 0xBD, /* 0xC8-0xCB */ + 0xBC, 0xE4, 0x99, 0xBE, 0x99, 0xBF, 0x99, 0xC0, /* 0xCC-0xCF */ + 0x99, 0xC1, 0x99, 0xC2, 0x99, 0xC3, 0x99, 0xC4, /* 0xD0-0xD3 */ + 0xBC, 0xE5, 0x99, 0xC5, 0x99, 0xC6, 0xBC, 0xE6, /* 0xD4-0xD7 */ + 0xBC, 0xE7, 0x99, 0xC7, 0x99, 0xC8, 0x99, 0xC9, /* 0xD8-0xDB */ + 0x99, 0xCA, 0x99, 0xCB, 0x99, 0xCC, 0x99, 0xCD, /* 0xDC-0xDF */ + 0xBC, 0xE8, 0x99, 0xCE, 0x99, 0xCF, 0x99, 0xD0, /* 0xE0-0xE3 */ + 0xBC, 0xE9, 0x99, 0xD1, 0x99, 0xD2, 0x99, 0xD3, /* 0xE4-0xE7 */ + 0xBC, 0xEA, 0x99, 0xD4, 0x99, 0xD5, 0x99, 0xD6, /* 0xE8-0xEB */ + 0x99, 0xD7, 0x99, 0xD8, 0x99, 0xD9, 0x99, 0xDA, /* 0xEC-0xEF */ + 0xBC, 0xEB, 0xBC, 0xEC, 0x99, 0xDB, 0xBC, 0xED, /* 0xF0-0xF3 */ + 0x99, 0xDC, 0x99, 0xDD, 0x99, 0xDE, 0x99, 0xDF, /* 0xF4-0xF7 */ + 0x99, 0xE0, 0x99, 0xE1, 0x99, 0xE2, 0x99, 0xE3, /* 0xF8-0xFB */ + 0xBC, 0xEE, 0xBC, 0xEF, 0x99, 0xE4, 0x99, 0xE5, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_C2[512] = { + 0xBC, 0xF0, 0x99, 0xE6, 0x99, 0xE7, 0x99, 0xE8, /* 0x00-0x03 */ + 0xBC, 0xF1, 0x99, 0xE9, 0x99, 0xEA, 0x99, 0xEB, /* 0x04-0x07 */ + 0x99, 0xEC, 0x99, 0xED, 0x99, 0xEE, 0x99, 0xEF, /* 0x08-0x0B */ + 0xBC, 0xF2, 0xBC, 0xF3, 0x99, 0xF0, 0xBC, 0xF4, /* 0x0C-0x0F */ + 0x99, 0xF1, 0xBC, 0xF5, 0x99, 0xF2, 0x99, 0xF3, /* 0x10-0x13 */ + 0x99, 0xF4, 0x99, 0xF5, 0x99, 0xF6, 0x99, 0xF7, /* 0x14-0x17 */ + 0xBC, 0xF6, 0xBC, 0xF7, 0x99, 0xF8, 0x99, 0xF9, /* 0x18-0x1B */ + 0xBC, 0xF8, 0x99, 0xFA, 0x99, 0xFB, 0xBC, 0xF9, /* 0x1C-0x1F */ + 0xBC, 0xFA, 0x99, 0xFC, 0x99, 0xFD, 0x99, 0xFE, /* 0x20-0x23 */ + 0x9A, 0x41, 0x9A, 0x42, 0x9A, 0x43, 0x9A, 0x44, /* 0x24-0x27 */ + 0xBC, 0xFB, 0xBC, 0xFC, 0x9A, 0x45, 0xBC, 0xFD, /* 0x28-0x2B */ + 0x9A, 0x46, 0xBC, 0xFE, 0x9A, 0x47, 0xBD, 0xA1, /* 0x2C-0x2F */ + 0x9A, 0x48, 0xBD, 0xA2, 0xBD, 0xA3, 0x9A, 0x49, /* 0x30-0x33 */ + 0xBD, 0xA4, 0x9A, 0x4A, 0x9A, 0x4B, 0x9A, 0x4C, /* 0x34-0x37 */ + 0x9A, 0x4D, 0x9A, 0x4E, 0x9A, 0x4F, 0x9A, 0x50, /* 0x38-0x3B */ + 0x9A, 0x51, 0x9A, 0x52, 0x9A, 0x53, 0x9A, 0x54, /* 0x3C-0x3F */ + 0x9A, 0x55, 0x9A, 0x56, 0x9A, 0x57, 0x9A, 0x58, /* 0x40-0x43 */ + 0x9A, 0x59, 0x9A, 0x5A, 0x9A, 0x61, 0x9A, 0x62, /* 0x44-0x47 */ + 0xBD, 0xA5, 0x9A, 0x63, 0x9A, 0x64, 0x9A, 0x65, /* 0x48-0x4B */ + 0x9A, 0x66, 0x9A, 0x67, 0x9A, 0x68, 0x9A, 0x69, /* 0x4C-0x4F */ + 0xBD, 0xA6, 0xBD, 0xA7, 0x9A, 0x6A, 0x9A, 0x6B, /* 0x50-0x53 */ + 0xBD, 0xA8, 0x9A, 0x6C, 0x9A, 0x6D, 0x9A, 0x6E, /* 0x54-0x57 */ + 0xBD, 0xA9, 0x9A, 0x6F, 0x9A, 0x70, 0x9A, 0x71, /* 0x58-0x5B */ + 0x9A, 0x72, 0x9A, 0x73, 0x9A, 0x74, 0x9A, 0x75, /* 0x5C-0x5F */ + 0xBD, 0xAA, 0x9A, 0x76, 0x9A, 0x77, 0x9A, 0x78, /* 0x60-0x63 */ + 0x9A, 0x79, 0xBD, 0xAB, 0x9A, 0x7A, 0x9A, 0x81, /* 0x64-0x67 */ + 0x9A, 0x82, 0x9A, 0x83, 0x9A, 0x84, 0x9A, 0x85, /* 0x68-0x6B */ + 0xBD, 0xAC, 0xBD, 0xAD, 0x9A, 0x86, 0x9A, 0x87, /* 0x6C-0x6F */ + 0xBD, 0xAE, 0x9A, 0x88, 0x9A, 0x89, 0x9A, 0x8A, /* 0x70-0x73 */ + 0xBD, 0xAF, 0x9A, 0x8B, 0x9A, 0x8C, 0x9A, 0x8D, /* 0x74-0x77 */ + 0x9A, 0x8E, 0x9A, 0x8F, 0x9A, 0x90, 0x9A, 0x91, /* 0x78-0x7B */ + 0xBD, 0xB0, 0xBD, 0xB1, 0x9A, 0x92, 0xBD, 0xB2, /* 0x7C-0x7F */ + + 0x9A, 0x93, 0xBD, 0xB3, 0x9A, 0x94, 0x9A, 0x95, /* 0x80-0x83 */ + 0x9A, 0x96, 0x9A, 0x97, 0x9A, 0x98, 0x9A, 0x99, /* 0x84-0x87 */ + 0xBD, 0xB4, 0xBD, 0xB5, 0x9A, 0x9A, 0x9A, 0x9B, /* 0x88-0x8B */ + 0x9A, 0x9C, 0x9A, 0x9D, 0x9A, 0x9E, 0x9A, 0x9F, /* 0x8C-0x8F */ + 0xBD, 0xB6, 0x9A, 0xA0, 0x9A, 0xA1, 0x9A, 0xA2, /* 0x90-0x93 */ + 0x9A, 0xA3, 0x9A, 0xA4, 0x9A, 0xA5, 0x9A, 0xA6, /* 0x94-0x97 */ + 0xBD, 0xB7, 0x9A, 0xA7, 0x9A, 0xA8, 0xBD, 0xB8, /* 0x98-0x9B */ + 0x9A, 0xA9, 0xBD, 0xB9, 0x9A, 0xAA, 0x9A, 0xAB, /* 0x9C-0x9F */ + 0x9A, 0xAC, 0x9A, 0xAD, 0x9A, 0xAE, 0x9A, 0xAF, /* 0xA0-0xA3 */ + 0xBD, 0xBA, 0xBD, 0xBB, 0x9A, 0xB0, 0x9A, 0xB1, /* 0xA4-0xA7 */ + 0xBD, 0xBC, 0x9A, 0xB2, 0x9A, 0xB3, 0x9A, 0xB4, /* 0xA8-0xAB */ + 0xBD, 0xBD, 0xBD, 0xBE, 0x9A, 0xB5, 0x9A, 0xB6, /* 0xAC-0xAF */ + 0x9A, 0xB7, 0x9A, 0xB8, 0x9A, 0xB9, 0x9A, 0xBA, /* 0xB0-0xB3 */ + 0xBD, 0xBF, 0xBD, 0xC0, 0x9A, 0xBB, 0xBD, 0xC1, /* 0xB4-0xB7 */ + 0x9A, 0xBC, 0xBD, 0xC2, 0x9A, 0xBD, 0x9A, 0xBE, /* 0xB8-0xBB */ + 0x9A, 0xBF, 0x9A, 0xC0, 0x9A, 0xC1, 0x9A, 0xC2, /* 0xBC-0xBF */ + 0x9A, 0xC3, 0x9A, 0xC4, 0x9A, 0xC5, 0x9A, 0xC6, /* 0xC0-0xC3 */ + 0x9A, 0xC7, 0x9A, 0xC8, 0x9A, 0xC9, 0x9A, 0xCA, /* 0xC4-0xC7 */ + 0x9A, 0xCB, 0x9A, 0xCC, 0x9A, 0xCD, 0x9A, 0xCE, /* 0xC8-0xCB */ + 0x9A, 0xCF, 0x9A, 0xD0, 0x9A, 0xD1, 0x9A, 0xD2, /* 0xCC-0xCF */ + 0x9A, 0xD3, 0x9A, 0xD4, 0x9A, 0xD5, 0x9A, 0xD6, /* 0xD0-0xD3 */ + 0x9A, 0xD7, 0x9A, 0xD8, 0x9A, 0xD9, 0x9A, 0xDA, /* 0xD4-0xD7 */ + 0x9A, 0xDB, 0x9A, 0xDC, 0x9A, 0xDD, 0x9A, 0xDE, /* 0xD8-0xDB */ + 0xBD, 0xC3, 0xBD, 0xC4, 0x9A, 0xDF, 0x9A, 0xE0, /* 0xDC-0xDF */ + 0xBD, 0xC5, 0x9A, 0xE1, 0x9A, 0xE2, 0xBD, 0xC6, /* 0xE0-0xE3 */ + 0xBD, 0xC7, 0x9A, 0xE3, 0x9A, 0xE4, 0x9A, 0xE5, /* 0xE4-0xE7 */ + 0x9A, 0xE6, 0x9A, 0xE7, 0x9A, 0xE8, 0xBD, 0xC8, /* 0xE8-0xEB */ + 0xBD, 0xC9, 0xBD, 0xCA, 0x9A, 0xE9, 0xBD, 0xCB, /* 0xEC-0xEF */ + 0x9A, 0xEA, 0xBD, 0xCC, 0x9A, 0xEB, 0x9A, 0xEC, /* 0xF0-0xF3 */ + 0x9A, 0xED, 0x9A, 0xEE, 0xBD, 0xCD, 0x9A, 0xEF, /* 0xF4-0xF7 */ + 0xBD, 0xCE, 0xBD, 0xCF, 0x9A, 0xF0, 0xBD, 0xD0, /* 0xF8-0xFB */ + 0xBD, 0xD1, 0x9A, 0xF1, 0x9A, 0xF2, 0x9A, 0xF3, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_C3[512] = { + 0xBD, 0xD2, 0x9A, 0xF4, 0x9A, 0xF5, 0x9A, 0xF6, /* 0x00-0x03 */ + 0x9A, 0xF7, 0x9A, 0xF8, 0x9A, 0xF9, 0x9A, 0xFA, /* 0x04-0x07 */ + 0xBD, 0xD3, 0xBD, 0xD4, 0x9A, 0xFB, 0x9A, 0xFC, /* 0x08-0x0B */ + 0xBD, 0xD5, 0xBD, 0xD6, 0x9A, 0xFD, 0x9A, 0xFE, /* 0x0C-0x0F */ + 0x9B, 0x41, 0x9B, 0x42, 0x9B, 0x43, 0xBD, 0xD7, /* 0x10-0x13 */ + 0xBD, 0xD8, 0xBD, 0xD9, 0x9B, 0x44, 0x9B, 0x45, /* 0x14-0x17 */ + 0xBD, 0xDA, 0x9B, 0x46, 0x9B, 0x47, 0x9B, 0x48, /* 0x18-0x1B */ + 0xBD, 0xDB, 0x9B, 0x49, 0x9B, 0x4A, 0x9B, 0x4B, /* 0x1C-0x1F */ + 0x9B, 0x4C, 0x9B, 0x4D, 0x9B, 0x4E, 0x9B, 0x4F, /* 0x20-0x23 */ + 0xBD, 0xDC, 0xBD, 0xDD, 0x9B, 0x50, 0x9B, 0x51, /* 0x24-0x27 */ + 0xBD, 0xDE, 0xBD, 0xDF, 0x9B, 0x52, 0x9B, 0x53, /* 0x28-0x2B */ + 0x9B, 0x54, 0x9B, 0x55, 0x9B, 0x56, 0x9B, 0x57, /* 0x2C-0x2F */ + 0x9B, 0x58, 0x9B, 0x59, 0x9B, 0x5A, 0x9B, 0x61, /* 0x30-0x33 */ + 0x9B, 0x62, 0x9B, 0x63, 0x9B, 0x64, 0x9B, 0x65, /* 0x34-0x37 */ + 0x9B, 0x66, 0x9B, 0x67, 0x9B, 0x68, 0x9B, 0x69, /* 0x38-0x3B */ + 0x9B, 0x6A, 0x9B, 0x6B, 0x9B, 0x6C, 0x9B, 0x6D, /* 0x3C-0x3F */ + 0x9B, 0x6E, 0x9B, 0x6F, 0x9B, 0x70, 0x9B, 0x71, /* 0x40-0x43 */ + 0x9B, 0x72, 0xBD, 0xE0, 0x9B, 0x73, 0x9B, 0x74, /* 0x44-0x47 */ + 0x9B, 0x75, 0x9B, 0x76, 0x9B, 0x77, 0x9B, 0x78, /* 0x48-0x4B */ + 0x9B, 0x79, 0x9B, 0x7A, 0x9B, 0x81, 0x9B, 0x82, /* 0x4C-0x4F */ + 0x9B, 0x83, 0x9B, 0x84, 0x9B, 0x85, 0x9B, 0x86, /* 0x50-0x53 */ + 0x9B, 0x87, 0x9B, 0x88, 0x9B, 0x89, 0x9B, 0x8A, /* 0x54-0x57 */ + 0x9B, 0x8B, 0x9B, 0x8C, 0x9B, 0x8D, 0x9B, 0x8E, /* 0x58-0x5B */ + 0x9B, 0x8F, 0x9B, 0x90, 0x9B, 0x91, 0x9B, 0x92, /* 0x5C-0x5F */ + 0x9B, 0x93, 0x9B, 0x94, 0x9B, 0x95, 0x9B, 0x96, /* 0x60-0x63 */ + 0x9B, 0x97, 0x9B, 0x98, 0x9B, 0x99, 0x9B, 0x9A, /* 0x64-0x67 */ + 0xBD, 0xE1, 0xBD, 0xE2, 0x9B, 0x9B, 0x9B, 0x9C, /* 0x68-0x6B */ + 0xBD, 0xE3, 0x9B, 0x9D, 0x9B, 0x9E, 0x9B, 0x9F, /* 0x6C-0x6F */ + 0xBD, 0xE4, 0x9B, 0xA0, 0xBD, 0xE5, 0x9B, 0xA1, /* 0x70-0x73 */ + 0x9B, 0xA2, 0x9B, 0xA3, 0x9B, 0xA4, 0x9B, 0xA5, /* 0x74-0x77 */ + 0xBD, 0xE6, 0xBD, 0xE7, 0x9B, 0xA6, 0x9B, 0xA7, /* 0x78-0x7B */ + 0xBD, 0xE8, 0xBD, 0xE9, 0x9B, 0xA8, 0x9B, 0xA9, /* 0x7C-0x7F */ + + 0x9B, 0xAA, 0x9B, 0xAB, 0x9B, 0xAC, 0x9B, 0xAD, /* 0x80-0x83 */ + 0xBD, 0xEA, 0x9B, 0xAE, 0x9B, 0xAF, 0x9B, 0xB0, /* 0x84-0x87 */ + 0xBD, 0xEB, 0x9B, 0xB1, 0x9B, 0xB2, 0x9B, 0xB3, /* 0x88-0x8B */ + 0xBD, 0xEC, 0x9B, 0xB4, 0x9B, 0xB5, 0x9B, 0xB6, /* 0x8C-0x8F */ + 0x9B, 0xB7, 0x9B, 0xB8, 0x9B, 0xB9, 0x9B, 0xBA, /* 0x90-0x93 */ + 0x9B, 0xBB, 0x9B, 0xBC, 0x9B, 0xBD, 0x9B, 0xBE, /* 0x94-0x97 */ + 0x9B, 0xBF, 0x9B, 0xC0, 0x9B, 0xC1, 0x9B, 0xC2, /* 0x98-0x9B */ + 0x9B, 0xC3, 0x9B, 0xC4, 0x9B, 0xC5, 0x9B, 0xC6, /* 0x9C-0x9F */ + 0x9B, 0xC7, 0x9B, 0xC8, 0x9B, 0xC9, 0x9B, 0xCA, /* 0xA0-0xA3 */ + 0x9B, 0xCB, 0x9B, 0xCC, 0x9B, 0xCD, 0x9B, 0xCE, /* 0xA4-0xA7 */ + 0x9B, 0xCF, 0x9B, 0xD0, 0x9B, 0xD1, 0x9B, 0xD2, /* 0xA8-0xAB */ + 0x9B, 0xD3, 0x9B, 0xD4, 0x9B, 0xD5, 0x9B, 0xD6, /* 0xAC-0xAF */ + 0x9B, 0xD7, 0x9B, 0xD8, 0x9B, 0xD9, 0x9B, 0xDA, /* 0xB0-0xB3 */ + 0x9B, 0xDB, 0x9B, 0xDC, 0x9B, 0xDD, 0x9B, 0xDE, /* 0xB4-0xB7 */ + 0x9B, 0xDF, 0x9B, 0xE0, 0x9B, 0xE1, 0x9B, 0xE2, /* 0xB8-0xBB */ + 0x9B, 0xE3, 0x9B, 0xE4, 0x9B, 0xE5, 0x9B, 0xE6, /* 0xBC-0xBF */ + 0xBD, 0xED, 0x9B, 0xE7, 0x9B, 0xE8, 0x9B, 0xE9, /* 0xC0-0xC3 */ + 0x9B, 0xEA, 0x9B, 0xEB, 0x9B, 0xEC, 0x9B, 0xED, /* 0xC4-0xC7 */ + 0x9B, 0xEE, 0x9B, 0xEF, 0x9B, 0xF0, 0x9B, 0xF1, /* 0xC8-0xCB */ + 0x9B, 0xF2, 0x9B, 0xF3, 0x9B, 0xF4, 0x9B, 0xF5, /* 0xCC-0xCF */ + 0x9B, 0xF6, 0x9B, 0xF7, 0x9B, 0xF8, 0x9B, 0xF9, /* 0xD0-0xD3 */ + 0x9B, 0xFA, 0x9B, 0xFB, 0x9B, 0xFC, 0x9B, 0xFD, /* 0xD4-0xD7 */ + 0xBD, 0xEE, 0xBD, 0xEF, 0x9B, 0xFE, 0x9C, 0x41, /* 0xD8-0xDB */ + 0xBD, 0xF0, 0x9C, 0x42, 0x9C, 0x43, 0xBD, 0xF1, /* 0xDC-0xDF */ + 0xBD, 0xF2, 0x9C, 0x44, 0xBD, 0xF3, 0x9C, 0x45, /* 0xE0-0xE3 */ + 0x9C, 0x46, 0x9C, 0x47, 0x9C, 0x48, 0x9C, 0x49, /* 0xE4-0xE7 */ + 0xBD, 0xF4, 0xBD, 0xF5, 0x9C, 0x4A, 0x9C, 0x4B, /* 0xE8-0xEB */ + 0x9C, 0x4C, 0xBD, 0xF6, 0x9C, 0x4D, 0x9C, 0x4E, /* 0xEC-0xEF */ + 0x9C, 0x4F, 0x9C, 0x50, 0x9C, 0x51, 0x9C, 0x52, /* 0xF0-0xF3 */ + 0xBD, 0xF7, 0xBD, 0xF8, 0x9C, 0x53, 0x9C, 0x54, /* 0xF4-0xF7 */ + 0xBD, 0xF9, 0x9C, 0x55, 0x9C, 0x56, 0x9C, 0x57, /* 0xF8-0xFB */ + 0x9C, 0x58, 0x9C, 0x59, 0x9C, 0x5A, 0x9C, 0x61, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_C4[512] = { + 0x9C, 0x62, 0x9C, 0x63, 0x9C, 0x64, 0x9C, 0x65, /* 0x00-0x03 */ + 0x9C, 0x66, 0x9C, 0x67, 0x9C, 0x68, 0x9C, 0x69, /* 0x04-0x07 */ + 0xBD, 0xFA, 0x9C, 0x6A, 0x9C, 0x6B, 0x9C, 0x6C, /* 0x08-0x0B */ + 0x9C, 0x6D, 0x9C, 0x6E, 0x9C, 0x6F, 0x9C, 0x70, /* 0x0C-0x0F */ + 0xBD, 0xFB, 0x9C, 0x71, 0x9C, 0x72, 0x9C, 0x73, /* 0x10-0x13 */ + 0x9C, 0x74, 0x9C, 0x75, 0x9C, 0x76, 0x9C, 0x77, /* 0x14-0x17 */ + 0x9C, 0x78, 0x9C, 0x79, 0x9C, 0x7A, 0x9C, 0x81, /* 0x18-0x1B */ + 0x9C, 0x82, 0x9C, 0x83, 0x9C, 0x84, 0x9C, 0x85, /* 0x1C-0x1F */ + 0x9C, 0x86, 0x9C, 0x87, 0x9C, 0x88, 0x9C, 0x89, /* 0x20-0x23 */ + 0xBD, 0xFC, 0x9C, 0x8A, 0x9C, 0x8B, 0x9C, 0x8C, /* 0x24-0x27 */ + 0x9C, 0x8D, 0x9C, 0x8E, 0x9C, 0x8F, 0x9C, 0x90, /* 0x28-0x2B */ + 0xBD, 0xFD, 0x9C, 0x91, 0x9C, 0x92, 0x9C, 0x93, /* 0x2C-0x2F */ + 0xBD, 0xFE, 0x9C, 0x94, 0x9C, 0x95, 0x9C, 0x96, /* 0x30-0x33 */ + 0xBE, 0xA1, 0x9C, 0x97, 0x9C, 0x98, 0x9C, 0x99, /* 0x34-0x37 */ + 0x9C, 0x9A, 0x9C, 0x9B, 0x9C, 0x9C, 0x9C, 0x9D, /* 0x38-0x3B */ + 0xBE, 0xA2, 0xBE, 0xA3, 0x9C, 0x9E, 0x9C, 0x9F, /* 0x3C-0x3F */ + 0x9C, 0xA0, 0x9C, 0xA1, 0x9C, 0xA2, 0x9C, 0xA3, /* 0x40-0x43 */ + 0x9C, 0xA4, 0x9C, 0xA5, 0x9C, 0xA6, 0x9C, 0xA7, /* 0x44-0x47 */ + 0xBE, 0xA4, 0x9C, 0xA8, 0x9C, 0xA9, 0x9C, 0xAA, /* 0x48-0x4B */ + 0x9C, 0xAB, 0x9C, 0xAC, 0x9C, 0xAD, 0x9C, 0xAE, /* 0x4C-0x4F */ + 0x9C, 0xAF, 0x9C, 0xB0, 0x9C, 0xB1, 0x9C, 0xB2, /* 0x50-0x53 */ + 0x9C, 0xB3, 0x9C, 0xB4, 0x9C, 0xB5, 0x9C, 0xB6, /* 0x54-0x57 */ + 0x9C, 0xB7, 0x9C, 0xB8, 0x9C, 0xB9, 0x9C, 0xBA, /* 0x58-0x5B */ + 0x9C, 0xBB, 0x9C, 0xBC, 0x9C, 0xBD, 0x9C, 0xBE, /* 0x5C-0x5F */ + 0x9C, 0xBF, 0x9C, 0xC0, 0x9C, 0xC1, 0x9C, 0xC2, /* 0x60-0x63 */ + 0xBE, 0xA5, 0xBE, 0xA6, 0x9C, 0xC3, 0x9C, 0xC4, /* 0x64-0x67 */ + 0xBE, 0xA7, 0x9C, 0xC5, 0x9C, 0xC6, 0x9C, 0xC7, /* 0x68-0x6B */ + 0xBE, 0xA8, 0x9C, 0xC8, 0x9C, 0xC9, 0x9C, 0xCA, /* 0x6C-0x6F */ + 0x9C, 0xCB, 0x9C, 0xCC, 0x9C, 0xCD, 0x9C, 0xCE, /* 0x70-0x73 */ + 0xBE, 0xA9, 0xBE, 0xAA, 0x9C, 0xCF, 0x9C, 0xD0, /* 0x74-0x77 */ + 0x9C, 0xD1, 0xBE, 0xAB, 0x9C, 0xD2, 0x9C, 0xD3, /* 0x78-0x7B */ + 0x9C, 0xD4, 0x9C, 0xD5, 0x9C, 0xD6, 0x9C, 0xD7, /* 0x7C-0x7F */ + + 0xBE, 0xAC, 0x9C, 0xD8, 0x9C, 0xD9, 0x9C, 0xDA, /* 0x80-0x83 */ + 0x9C, 0xDB, 0x9C, 0xDC, 0x9C, 0xDD, 0x9C, 0xDE, /* 0x84-0x87 */ + 0x9C, 0xDF, 0x9C, 0xE0, 0x9C, 0xE1, 0x9C, 0xE2, /* 0x88-0x8B */ + 0x9C, 0xE3, 0x9C, 0xE4, 0x9C, 0xE5, 0x9C, 0xE6, /* 0x8C-0x8F */ + 0x9C, 0xE7, 0x9C, 0xE8, 0x9C, 0xE9, 0x9C, 0xEA, /* 0x90-0x93 */ + 0xBE, 0xAD, 0x9C, 0xEB, 0x9C, 0xEC, 0x9C, 0xED, /* 0x94-0x97 */ + 0x9C, 0xEE, 0x9C, 0xEF, 0x9C, 0xF0, 0x9C, 0xF1, /* 0x98-0x9B */ + 0xBE, 0xAE, 0x9C, 0xF2, 0x9C, 0xF3, 0x9C, 0xF4, /* 0x9C-0x9F */ + 0x9C, 0xF5, 0x9C, 0xF6, 0x9C, 0xF7, 0x9C, 0xF8, /* 0xA0-0xA3 */ + 0x9C, 0xF9, 0x9C, 0xFA, 0x9C, 0xFB, 0x9C, 0xFC, /* 0xA4-0xA7 */ + 0x9C, 0xFD, 0x9C, 0xFE, 0x9D, 0x41, 0x9D, 0x42, /* 0xA8-0xAB */ + 0x9D, 0x43, 0x9D, 0x44, 0x9D, 0x45, 0x9D, 0x46, /* 0xAC-0xAF */ + 0x9D, 0x47, 0x9D, 0x48, 0x9D, 0x49, 0x9D, 0x4A, /* 0xB0-0xB3 */ + 0x9D, 0x4B, 0x9D, 0x4C, 0x9D, 0x4D, 0x9D, 0x4E, /* 0xB4-0xB7 */ + 0xBE, 0xAF, 0x9D, 0x4F, 0x9D, 0x50, 0x9D, 0x51, /* 0xB8-0xBB */ + 0xBE, 0xB0, 0x9D, 0x52, 0x9D, 0x53, 0x9D, 0x54, /* 0xBC-0xBF */ + 0x9D, 0x55, 0x9D, 0x56, 0x9D, 0x57, 0x9D, 0x58, /* 0xC0-0xC3 */ + 0x9D, 0x59, 0x9D, 0x5A, 0x9D, 0x61, 0x9D, 0x62, /* 0xC4-0xC7 */ + 0x9D, 0x63, 0x9D, 0x64, 0x9D, 0x65, 0x9D, 0x66, /* 0xC8-0xCB */ + 0x9D, 0x67, 0x9D, 0x68, 0x9D, 0x69, 0x9D, 0x6A, /* 0xCC-0xCF */ + 0x9D, 0x6B, 0x9D, 0x6C, 0x9D, 0x6D, 0x9D, 0x6E, /* 0xD0-0xD3 */ + 0x9D, 0x6F, 0x9D, 0x70, 0x9D, 0x71, 0x9D, 0x72, /* 0xD4-0xD7 */ + 0x9D, 0x73, 0x9D, 0x74, 0x9D, 0x75, 0x9D, 0x76, /* 0xD8-0xDB */ + 0x9D, 0x77, 0x9D, 0x78, 0x9D, 0x79, 0x9D, 0x7A, /* 0xDC-0xDF */ + 0x9D, 0x81, 0x9D, 0x82, 0x9D, 0x83, 0x9D, 0x84, /* 0xE0-0xE3 */ + 0x9D, 0x85, 0x9D, 0x86, 0x9D, 0x87, 0x9D, 0x88, /* 0xE4-0xE7 */ + 0x9D, 0x89, 0xBE, 0xB1, 0x9D, 0x8A, 0x9D, 0x8B, /* 0xE8-0xEB */ + 0x9D, 0x8C, 0x9D, 0x8D, 0x9D, 0x8E, 0x9D, 0x8F, /* 0xEC-0xEF */ + 0xBE, 0xB2, 0xBE, 0xB3, 0x9D, 0x90, 0x9D, 0x91, /* 0xF0-0xF3 */ + 0xBE, 0xB4, 0x9D, 0x92, 0x9D, 0x93, 0x9D, 0x94, /* 0xF4-0xF7 */ + 0xBE, 0xB5, 0x9D, 0x95, 0xBE, 0xB6, 0x9D, 0x96, /* 0xF8-0xFB */ + 0x9D, 0x97, 0x9D, 0x98, 0x9D, 0x99, 0xBE, 0xB7, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_C5[512] = { + 0xBE, 0xB8, 0xBE, 0xB9, 0x9D, 0x9A, 0x9D, 0x9B, /* 0x00-0x03 */ + 0x9D, 0x9C, 0x9D, 0x9D, 0x9D, 0x9E, 0x9D, 0x9F, /* 0x04-0x07 */ + 0x9D, 0xA0, 0x9D, 0xA1, 0x9D, 0xA2, 0x9D, 0xA3, /* 0x08-0x0B */ + 0xBE, 0xBA, 0x9D, 0xA4, 0x9D, 0xA5, 0x9D, 0xA6, /* 0x0C-0x0F */ + 0xBE, 0xBB, 0x9D, 0xA7, 0x9D, 0xA8, 0x9D, 0xA9, /* 0x10-0x13 */ + 0xBE, 0xBC, 0x9D, 0xAA, 0x9D, 0xAB, 0x9D, 0xAC, /* 0x14-0x17 */ + 0x9D, 0xAD, 0x9D, 0xAE, 0x9D, 0xAF, 0x9D, 0xB0, /* 0x18-0x1B */ + 0xBE, 0xBD, 0x9D, 0xB1, 0x9D, 0xB2, 0x9D, 0xB3, /* 0x1C-0x1F */ + 0x9D, 0xB4, 0x9D, 0xB5, 0x9D, 0xB6, 0x9D, 0xB7, /* 0x20-0x23 */ + 0x9D, 0xB8, 0x9D, 0xB9, 0x9D, 0xBA, 0x9D, 0xBB, /* 0x24-0x27 */ + 0xBE, 0xBE, 0xBE, 0xBF, 0x9D, 0xBC, 0x9D, 0xBD, /* 0x28-0x2B */ + 0xBE, 0xC0, 0x9D, 0xBE, 0x9D, 0xBF, 0x9D, 0xC0, /* 0x2C-0x2F */ + 0xBE, 0xC1, 0x9D, 0xC1, 0x9D, 0xC2, 0x9D, 0xC3, /* 0x30-0x33 */ + 0x9D, 0xC4, 0x9D, 0xC5, 0x9D, 0xC6, 0x9D, 0xC7, /* 0x34-0x37 */ + 0xBE, 0xC2, 0xBE, 0xC3, 0x9D, 0xC8, 0xBE, 0xC4, /* 0x38-0x3B */ + 0x9D, 0xC9, 0xBE, 0xC5, 0x9D, 0xCA, 0x9D, 0xCB, /* 0x3C-0x3F */ + 0x9D, 0xCC, 0x9D, 0xCD, 0x9D, 0xCE, 0x9D, 0xCF, /* 0x40-0x43 */ + 0xBE, 0xC6, 0xBE, 0xC7, 0x9D, 0xD0, 0x9D, 0xD1, /* 0x44-0x47 */ + 0xBE, 0xC8, 0xBE, 0xC9, 0xBE, 0xCA, 0x9D, 0xD2, /* 0x48-0x4B */ + 0xBE, 0xCB, 0xBE, 0xCC, 0xBE, 0xCD, 0x9D, 0xD3, /* 0x4C-0x4F */ + 0x9D, 0xD4, 0x9D, 0xD5, 0x9D, 0xD6, 0xBE, 0xCE, /* 0x50-0x53 */ + 0xBE, 0xCF, 0xBE, 0xD0, 0x9D, 0xD7, 0xBE, 0xD1, /* 0x54-0x57 */ + 0xBE, 0xD2, 0xBE, 0xD3, 0x9D, 0xD8, 0x9D, 0xD9, /* 0x58-0x5B */ + 0x9D, 0xDA, 0xBE, 0xD4, 0xBE, 0xD5, 0x9D, 0xDB, /* 0x5C-0x5F */ + 0xBE, 0xD6, 0xBE, 0xD7, 0x9D, 0xDC, 0x9D, 0xDD, /* 0x60-0x63 */ + 0xBE, 0xD8, 0x9D, 0xDE, 0x9D, 0xDF, 0x9D, 0xE0, /* 0x64-0x67 */ + 0xBE, 0xD9, 0x9D, 0xE1, 0x9D, 0xE2, 0x9D, 0xE3, /* 0x68-0x6B */ + 0x9D, 0xE4, 0x9D, 0xE5, 0x9D, 0xE6, 0x9D, 0xE7, /* 0x6C-0x6F */ + 0xBE, 0xDA, 0xBE, 0xDB, 0x9D, 0xE8, 0xBE, 0xDC, /* 0x70-0x73 */ + 0xBE, 0xDD, 0xBE, 0xDE, 0x9D, 0xE9, 0x9D, 0xEA, /* 0x74-0x77 */ + 0x9D, 0xEB, 0x9D, 0xEC, 0x9D, 0xED, 0x9D, 0xEE, /* 0x78-0x7B */ + 0xBE, 0xDF, 0xBE, 0xE0, 0x9D, 0xEF, 0x9D, 0xF0, /* 0x7C-0x7F */ + + 0xBE, 0xE1, 0x9D, 0xF1, 0x9D, 0xF2, 0x9D, 0xF3, /* 0x80-0x83 */ + 0xBE, 0xE2, 0x9D, 0xF4, 0x9D, 0xF5, 0xBE, 0xE3, /* 0x84-0x87 */ + 0x9D, 0xF6, 0x9D, 0xF7, 0x9D, 0xF8, 0x9D, 0xF9, /* 0x88-0x8B */ + 0xBE, 0xE4, 0xBE, 0xE5, 0x9D, 0xFA, 0xBE, 0xE6, /* 0x8C-0x8F */ + 0x9D, 0xFB, 0xBE, 0xE7, 0x9D, 0xFC, 0x9D, 0xFD, /* 0x90-0x93 */ + 0x9D, 0xFE, 0xBE, 0xE8, 0x9E, 0x41, 0xBE, 0xE9, /* 0x94-0x97 */ + 0xBE, 0xEA, 0x9E, 0x42, 0x9E, 0x43, 0x9E, 0x44, /* 0x98-0x9B */ + 0xBE, 0xEB, 0x9E, 0x45, 0x9E, 0x46, 0x9E, 0x47, /* 0x9C-0x9F */ + 0xBE, 0xEC, 0x9E, 0x48, 0x9E, 0x49, 0x9E, 0x4A, /* 0xA0-0xA3 */ + 0x9E, 0x4B, 0x9E, 0x4C, 0x9E, 0x4D, 0x9E, 0x4E, /* 0xA4-0xA7 */ + 0x9E, 0x4F, 0xBE, 0xED, 0x9E, 0x50, 0x9E, 0x51, /* 0xA8-0xAB */ + 0x9E, 0x52, 0x9E, 0x53, 0x9E, 0x54, 0x9E, 0x55, /* 0xAC-0xAF */ + 0x9E, 0x56, 0x9E, 0x57, 0x9E, 0x58, 0x9E, 0x59, /* 0xB0-0xB3 */ + 0xBE, 0xEE, 0xBE, 0xEF, 0x9E, 0x5A, 0x9E, 0x61, /* 0xB4-0xB7 */ + 0xBE, 0xF0, 0xBE, 0xF1, 0x9E, 0x62, 0xBE, 0xF2, /* 0xB8-0xBB */ + 0xBE, 0xF3, 0xBE, 0xF4, 0xBE, 0xF5, 0x9E, 0x63, /* 0xBC-0xBF */ + 0x9E, 0x64, 0x9E, 0x65, 0x9E, 0x66, 0x9E, 0x67, /* 0xC0-0xC3 */ + 0xBE, 0xF6, 0xBE, 0xF7, 0xBE, 0xF8, 0xBE, 0xF9, /* 0xC4-0xC7 */ + 0xBE, 0xFA, 0xBE, 0xFB, 0xBE, 0xFC, 0x9E, 0x68, /* 0xC8-0xCB */ + 0xBE, 0xFD, 0x9E, 0x69, 0xBE, 0xFE, 0x9E, 0x6A, /* 0xCC-0xCF */ + 0xBF, 0xA1, 0xBF, 0xA2, 0x9E, 0x6B, 0x9E, 0x6C, /* 0xD0-0xD3 */ + 0xBF, 0xA3, 0x9E, 0x6D, 0x9E, 0x6E, 0x9E, 0x6F, /* 0xD4-0xD7 */ + 0xBF, 0xA4, 0x9E, 0x70, 0x9E, 0x71, 0x9E, 0x72, /* 0xD8-0xDB */ + 0x9E, 0x73, 0x9E, 0x74, 0x9E, 0x75, 0x9E, 0x76, /* 0xDC-0xDF */ + 0xBF, 0xA5, 0xBF, 0xA6, 0x9E, 0x77, 0xBF, 0xA7, /* 0xE0-0xE3 */ + 0x9E, 0x78, 0xBF, 0xA8, 0x9E, 0x79, 0x9E, 0x7A, /* 0xE4-0xE7 */ + 0x9E, 0x81, 0x9E, 0x82, 0x9E, 0x83, 0x9E, 0x84, /* 0xE8-0xEB */ + 0xBF, 0xA9, 0xBF, 0xAA, 0xBF, 0xAB, 0x9E, 0x85, /* 0xEC-0xEF */ + 0xBF, 0xAC, 0x9E, 0x86, 0x9E, 0x87, 0x9E, 0x88, /* 0xF0-0xF3 */ + 0xBF, 0xAD, 0x9E, 0x89, 0xBF, 0xAE, 0xBF, 0xAF, /* 0xF4-0xF7 */ + 0x9E, 0x8A, 0x9E, 0x8B, 0x9E, 0x8C, 0x9E, 0x8D, /* 0xF8-0xFB */ + 0xBF, 0xB0, 0xBF, 0xB1, 0xBF, 0xB2, 0xBF, 0xB3, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_C6[512] = { + 0xBF, 0xB4, 0xBF, 0xB5, 0x9E, 0x8E, 0x9E, 0x8F, /* 0x00-0x03 */ + 0x9E, 0x90, 0xBF, 0xB6, 0xBF, 0xB7, 0xBF, 0xB8, /* 0x04-0x07 */ + 0xBF, 0xB9, 0x9E, 0x91, 0x9E, 0x92, 0x9E, 0x93, /* 0x08-0x0B */ + 0xBF, 0xBA, 0x9E, 0x94, 0x9E, 0x95, 0x9E, 0x96, /* 0x0C-0x0F */ + 0xBF, 0xBB, 0x9E, 0x97, 0x9E, 0x98, 0x9E, 0x99, /* 0x10-0x13 */ + 0x9E, 0x9A, 0x9E, 0x9B, 0x9E, 0x9C, 0x9E, 0x9D, /* 0x14-0x17 */ + 0xBF, 0xBC, 0xBF, 0xBD, 0x9E, 0x9E, 0xBF, 0xBE, /* 0x18-0x1B */ + 0xBF, 0xBF, 0x9E, 0x9F, 0x9E, 0xA0, 0x9E, 0xA1, /* 0x1C-0x1F */ + 0x9E, 0xA2, 0x9E, 0xA3, 0x9E, 0xA4, 0x9E, 0xA5, /* 0x20-0x23 */ + 0xBF, 0xC0, 0xBF, 0xC1, 0x9E, 0xA6, 0x9E, 0xA7, /* 0x24-0x27 */ + 0xBF, 0xC2, 0x9E, 0xA8, 0x9E, 0xA9, 0x9E, 0xAA, /* 0x28-0x2B */ + 0xBF, 0xC3, 0xBF, 0xC4, 0xBF, 0xC5, 0x9E, 0xAB, /* 0x2C-0x2F */ + 0xBF, 0xC6, 0x9E, 0xAC, 0x9E, 0xAD, 0xBF, 0xC7, /* 0x30-0x33 */ + 0xBF, 0xC8, 0xBF, 0xC9, 0x9E, 0xAE, 0xBF, 0xCA, /* 0x34-0x37 */ + 0x9E, 0xAF, 0xBF, 0xCB, 0x9E, 0xB0, 0xBF, 0xCC, /* 0x38-0x3B */ + 0x9E, 0xB1, 0x9E, 0xB2, 0x9E, 0xB3, 0x9E, 0xB4, /* 0x3C-0x3F */ + 0xBF, 0xCD, 0xBF, 0xCE, 0x9E, 0xB5, 0x9E, 0xB6, /* 0x40-0x43 */ + 0xBF, 0xCF, 0x9E, 0xB7, 0x9E, 0xB8, 0x9E, 0xB9, /* 0x44-0x47 */ + 0xBF, 0xD0, 0x9E, 0xBA, 0x9E, 0xBB, 0x9E, 0xBC, /* 0x48-0x4B */ + 0x9E, 0xBD, 0x9E, 0xBE, 0x9E, 0xBF, 0x9E, 0xC0, /* 0x4C-0x4F */ + 0xBF, 0xD1, 0xBF, 0xD2, 0x9E, 0xC1, 0xBF, 0xD3, /* 0x50-0x53 */ + 0xBF, 0xD4, 0xBF, 0xD5, 0x9E, 0xC2, 0x9E, 0xC3, /* 0x54-0x57 */ + 0x9E, 0xC4, 0x9E, 0xC5, 0x9E, 0xC6, 0x9E, 0xC7, /* 0x58-0x5B */ + 0xBF, 0xD6, 0xBF, 0xD7, 0x9E, 0xC8, 0x9E, 0xC9, /* 0x5C-0x5F */ + 0xBF, 0xD8, 0x9E, 0xCA, 0x9E, 0xCB, 0x9E, 0xCC, /* 0x60-0x63 */ + 0x9E, 0xCD, 0x9E, 0xCE, 0x9E, 0xCF, 0x9E, 0xD0, /* 0x64-0x67 */ + 0x9E, 0xD1, 0x9E, 0xD2, 0x9E, 0xD3, 0x9E, 0xD4, /* 0x68-0x6B */ + 0xBF, 0xD9, 0x9E, 0xD5, 0x9E, 0xD6, 0xBF, 0xDA, /* 0x6C-0x6F */ + 0x9E, 0xD7, 0xBF, 0xDB, 0x9E, 0xD8, 0x9E, 0xD9, /* 0x70-0x73 */ + 0x9E, 0xDA, 0x9E, 0xDB, 0x9E, 0xDC, 0x9E, 0xDD, /* 0x74-0x77 */ + 0xBF, 0xDC, 0xBF, 0xDD, 0x9E, 0xDE, 0x9E, 0xDF, /* 0x78-0x7B */ + 0xBF, 0xDE, 0x9E, 0xE0, 0x9E, 0xE1, 0x9E, 0xE2, /* 0x7C-0x7F */ + + 0xBF, 0xDF, 0x9E, 0xE3, 0x9E, 0xE4, 0x9E, 0xE5, /* 0x80-0x83 */ + 0x9E, 0xE6, 0x9E, 0xE7, 0x9E, 0xE8, 0x9E, 0xE9, /* 0x84-0x87 */ + 0xBF, 0xE0, 0xBF, 0xE1, 0x9E, 0xEA, 0xBF, 0xE2, /* 0x88-0x8B */ + 0x9E, 0xEB, 0xBF, 0xE3, 0x9E, 0xEC, 0x9E, 0xED, /* 0x8C-0x8F */ + 0x9E, 0xEE, 0x9E, 0xEF, 0x9E, 0xF0, 0x9E, 0xF1, /* 0x90-0x93 */ + 0xBF, 0xE4, 0xBF, 0xE5, 0x9E, 0xF2, 0x9E, 0xF3, /* 0x94-0x97 */ + 0xBF, 0xE6, 0x9E, 0xF4, 0x9E, 0xF5, 0x9E, 0xF6, /* 0x98-0x9B */ + 0xBF, 0xE7, 0x9E, 0xF7, 0x9E, 0xF8, 0x9E, 0xF9, /* 0x9C-0x9F */ + 0x9E, 0xFA, 0x9E, 0xFB, 0x9E, 0xFC, 0x9E, 0xFD, /* 0xA0-0xA3 */ + 0xBF, 0xE8, 0xBF, 0xE9, 0x9E, 0xFE, 0xBF, 0xEA, /* 0xA4-0xA7 */ + 0x9F, 0x41, 0xBF, 0xEB, 0x9F, 0x42, 0x9F, 0x43, /* 0xA8-0xAB */ + 0x9F, 0x44, 0x9F, 0x45, 0x9F, 0x46, 0x9F, 0x47, /* 0xAC-0xAF */ + 0xBF, 0xEC, 0xBF, 0xED, 0x9F, 0x48, 0x9F, 0x49, /* 0xB0-0xB3 */ + 0xBF, 0xEE, 0x9F, 0x4A, 0x9F, 0x4B, 0x9F, 0x4C, /* 0xB4-0xB7 */ + 0xBF, 0xEF, 0xBF, 0xF0, 0xBF, 0xF1, 0x9F, 0x4D, /* 0xB8-0xBB */ + 0x9F, 0x4E, 0x9F, 0x4F, 0x9F, 0x50, 0x9F, 0x51, /* 0xBC-0xBF */ + 0xBF, 0xF2, 0xBF, 0xF3, 0x9F, 0x52, 0xBF, 0xF4, /* 0xC0-0xC3 */ + 0x9F, 0x53, 0xBF, 0xF5, 0x9F, 0x54, 0x9F, 0x55, /* 0xC4-0xC7 */ + 0x9F, 0x56, 0x9F, 0x57, 0x9F, 0x58, 0x9F, 0x59, /* 0xC8-0xCB */ + 0xBF, 0xF6, 0xBF, 0xF7, 0x9F, 0x5A, 0x9F, 0x61, /* 0xCC-0xCF */ + 0xBF, 0xF8, 0x9F, 0x62, 0x9F, 0x63, 0x9F, 0x64, /* 0xD0-0xD3 */ + 0xBF, 0xF9, 0x9F, 0x65, 0x9F, 0x66, 0x9F, 0x67, /* 0xD4-0xD7 */ + 0x9F, 0x68, 0x9F, 0x69, 0x9F, 0x6A, 0x9F, 0x6B, /* 0xD8-0xDB */ + 0xBF, 0xFA, 0xBF, 0xFB, 0x9F, 0x6C, 0x9F, 0x6D, /* 0xDC-0xDF */ + 0xBF, 0xFC, 0xBF, 0xFD, 0x9F, 0x6E, 0x9F, 0x6F, /* 0xE0-0xE3 */ + 0x9F, 0x70, 0x9F, 0x71, 0x9F, 0x72, 0x9F, 0x73, /* 0xE4-0xE7 */ + 0xBF, 0xFE, 0xC0, 0xA1, 0x9F, 0x74, 0x9F, 0x75, /* 0xE8-0xEB */ + 0xC0, 0xA2, 0x9F, 0x76, 0x9F, 0x77, 0x9F, 0x78, /* 0xEC-0xEF */ + 0xC0, 0xA3, 0x9F, 0x79, 0x9F, 0x7A, 0x9F, 0x81, /* 0xF0-0xF3 */ + 0x9F, 0x82, 0x9F, 0x83, 0x9F, 0x84, 0x9F, 0x85, /* 0xF4-0xF7 */ + 0xC0, 0xA4, 0xC0, 0xA5, 0x9F, 0x86, 0x9F, 0x87, /* 0xF8-0xFB */ + 0x9F, 0x88, 0xC0, 0xA6, 0x9F, 0x89, 0x9F, 0x8A, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_C7[512] = { + 0x9F, 0x8B, 0x9F, 0x8C, 0x9F, 0x8D, 0x9F, 0x8E, /* 0x00-0x03 */ + 0xC0, 0xA7, 0xC0, 0xA8, 0x9F, 0x8F, 0x9F, 0x90, /* 0x04-0x07 */ + 0xC0, 0xA9, 0x9F, 0x91, 0x9F, 0x92, 0x9F, 0x93, /* 0x08-0x0B */ + 0xC0, 0xAA, 0x9F, 0x94, 0x9F, 0x95, 0x9F, 0x96, /* 0x0C-0x0F */ + 0x9F, 0x97, 0x9F, 0x98, 0x9F, 0x99, 0x9F, 0x9A, /* 0x10-0x13 */ + 0xC0, 0xAB, 0xC0, 0xAC, 0x9F, 0x9B, 0xC0, 0xAD, /* 0x14-0x17 */ + 0x9F, 0x9C, 0xC0, 0xAE, 0x9F, 0x9D, 0x9F, 0x9E, /* 0x18-0x1B */ + 0x9F, 0x9F, 0x9F, 0xA0, 0x9F, 0xA1, 0x9F, 0xA2, /* 0x1C-0x1F */ + 0xC0, 0xAF, 0xC0, 0xB0, 0x9F, 0xA3, 0x9F, 0xA4, /* 0x20-0x23 */ + 0xC0, 0xB1, 0x9F, 0xA5, 0x9F, 0xA6, 0x9F, 0xA7, /* 0x24-0x27 */ + 0xC0, 0xB2, 0x9F, 0xA8, 0x9F, 0xA9, 0x9F, 0xAA, /* 0x28-0x2B */ + 0x9F, 0xAB, 0x9F, 0xAC, 0x9F, 0xAD, 0x9F, 0xAE, /* 0x2C-0x2F */ + 0xC0, 0xB3, 0xC0, 0xB4, 0x9F, 0xAF, 0xC0, 0xB5, /* 0x30-0x33 */ + 0x9F, 0xB0, 0xC0, 0xB6, 0x9F, 0xB1, 0xC0, 0xB7, /* 0x34-0x37 */ + 0x9F, 0xB2, 0x9F, 0xB3, 0x9F, 0xB4, 0x9F, 0xB5, /* 0x38-0x3B */ + 0xC0, 0xB8, 0xC0, 0xB9, 0x9F, 0xB6, 0x9F, 0xB7, /* 0x3C-0x3F */ + 0xC0, 0xBA, 0x9F, 0xB8, 0x9F, 0xB9, 0x9F, 0xBA, /* 0x40-0x43 */ + 0xC0, 0xBB, 0x9F, 0xBB, 0x9F, 0xBC, 0x9F, 0xBD, /* 0x44-0x47 */ + 0x9F, 0xBE, 0x9F, 0xBF, 0xC0, 0xBC, 0x9F, 0xC0, /* 0x48-0x4B */ + 0xC0, 0xBD, 0xC0, 0xBE, 0x9F, 0xC1, 0xC0, 0xBF, /* 0x4C-0x4F */ + 0x9F, 0xC2, 0xC0, 0xC0, 0xC0, 0xC1, 0xC0, 0xC2, /* 0x50-0x53 */ + 0xC0, 0xC3, 0xC0, 0xC4, 0xC0, 0xC5, 0xC0, 0xC6, /* 0x54-0x57 */ + 0xC0, 0xC7, 0x9F, 0xC3, 0x9F, 0xC4, 0x9F, 0xC5, /* 0x58-0x5B */ + 0xC0, 0xC8, 0x9F, 0xC6, 0x9F, 0xC7, 0x9F, 0xC8, /* 0x5C-0x5F */ + 0xC0, 0xC9, 0x9F, 0xC9, 0x9F, 0xCA, 0x9F, 0xCB, /* 0x60-0x63 */ + 0x9F, 0xCC, 0x9F, 0xCD, 0x9F, 0xCE, 0x9F, 0xCF, /* 0x64-0x67 */ + 0xC0, 0xCA, 0x9F, 0xD0, 0x9F, 0xD1, 0xC0, 0xCB, /* 0x68-0x6B */ + 0x9F, 0xD2, 0x9F, 0xD3, 0x9F, 0xD4, 0x9F, 0xD5, /* 0x6C-0x6F */ + 0x9F, 0xD6, 0x9F, 0xD7, 0x9F, 0xD8, 0x9F, 0xD9, /* 0x70-0x73 */ + 0xC0, 0xCC, 0xC0, 0xCD, 0x9F, 0xDA, 0x9F, 0xDB, /* 0x74-0x77 */ + 0xC0, 0xCE, 0x9F, 0xDC, 0x9F, 0xDD, 0x9F, 0xDE, /* 0x78-0x7B */ + 0xC0, 0xCF, 0xC0, 0xD0, 0xC0, 0xD1, 0x9F, 0xDF, /* 0x7C-0x7F */ + + 0x9F, 0xE0, 0x9F, 0xE1, 0x9F, 0xE2, 0xC0, 0xD2, /* 0x80-0x83 */ + 0xC0, 0xD3, 0xC0, 0xD4, 0x9F, 0xE3, 0xC0, 0xD5, /* 0x84-0x87 */ + 0xC0, 0xD6, 0xC0, 0xD7, 0xC0, 0xD8, 0x9F, 0xE4, /* 0x88-0x8B */ + 0x9F, 0xE5, 0x9F, 0xE6, 0xC0, 0xD9, 0x9F, 0xE7, /* 0x8C-0x8F */ + 0xC0, 0xDA, 0xC0, 0xDB, 0x9F, 0xE8, 0x9F, 0xE9, /* 0x90-0x93 */ + 0xC0, 0xDC, 0x9F, 0xEA, 0xC0, 0xDD, 0xC0, 0xDE, /* 0x94-0x97 */ + 0xC0, 0xDF, 0x9F, 0xEB, 0xC0, 0xE0, 0x9F, 0xEC, /* 0x98-0x9B */ + 0x9F, 0xED, 0x9F, 0xEE, 0x9F, 0xEF, 0x9F, 0xF0, /* 0x9C-0x9F */ + 0xC0, 0xE1, 0xC0, 0xE2, 0x9F, 0xF1, 0xC0, 0xE3, /* 0xA0-0xA3 */ + 0xC0, 0xE4, 0xC0, 0xE5, 0xC0, 0xE6, 0x9F, 0xF2, /* 0xA4-0xA7 */ + 0x9F, 0xF3, 0x9F, 0xF4, 0x9F, 0xF5, 0x9F, 0xF6, /* 0xA8-0xAB */ + 0xC0, 0xE7, 0xC0, 0xE8, 0x9F, 0xF7, 0x9F, 0xF8, /* 0xAC-0xAF */ + 0xC0, 0xE9, 0x9F, 0xF9, 0x9F, 0xFA, 0x9F, 0xFB, /* 0xB0-0xB3 */ + 0xC0, 0xEA, 0x9F, 0xFC, 0x9F, 0xFD, 0x9F, 0xFE, /* 0xB4-0xB7 */ + 0xA0, 0x41, 0xA0, 0x42, 0xA0, 0x43, 0xA0, 0x44, /* 0xB8-0xBB */ + 0xC0, 0xEB, 0xC0, 0xEC, 0xA0, 0x45, 0xC0, 0xED, /* 0xBC-0xBF */ + 0xC0, 0xEE, 0xC0, 0xEF, 0xA0, 0x46, 0xA0, 0x47, /* 0xC0-0xC3 */ + 0xA0, 0x48, 0xA0, 0x49, 0xA0, 0x4A, 0xA0, 0x4B, /* 0xC4-0xC7 */ + 0xC0, 0xF0, 0xC0, 0xF1, 0xA0, 0x4C, 0xA0, 0x4D, /* 0xC8-0xCB */ + 0xC0, 0xF2, 0xA0, 0x4E, 0xC0, 0xF3, 0xA0, 0x4F, /* 0xCC-0xCF */ + 0xC0, 0xF4, 0xA0, 0x50, 0xA0, 0x51, 0xA0, 0x52, /* 0xD0-0xD3 */ + 0xA0, 0x53, 0xA0, 0x54, 0xA0, 0x55, 0xA0, 0x56, /* 0xD4-0xD7 */ + 0xC0, 0xF5, 0xA0, 0x57, 0xA0, 0x58, 0xA0, 0x59, /* 0xD8-0xDB */ + 0xA0, 0x5A, 0xC0, 0xF6, 0xA0, 0x61, 0xA0, 0x62, /* 0xDC-0xDF */ + 0xA0, 0x63, 0xA0, 0x64, 0xA0, 0x65, 0xA0, 0x66, /* 0xE0-0xE3 */ + 0xC0, 0xF7, 0xA0, 0x67, 0xA0, 0x68, 0xA0, 0x69, /* 0xE4-0xE7 */ + 0xC0, 0xF8, 0xA0, 0x6A, 0xA0, 0x6B, 0xA0, 0x6C, /* 0xE8-0xEB */ + 0xC0, 0xF9, 0xA0, 0x6D, 0xA0, 0x6E, 0xA0, 0x6F, /* 0xEC-0xEF */ + 0xA0, 0x70, 0xA0, 0x71, 0xA0, 0x72, 0xA0, 0x73, /* 0xF0-0xF3 */ + 0xA0, 0x74, 0xA0, 0x75, 0xA0, 0x76, 0xA0, 0x77, /* 0xF4-0xF7 */ + 0xA0, 0x78, 0xA0, 0x79, 0xA0, 0x7A, 0xA0, 0x81, /* 0xF8-0xFB */ + 0xA0, 0x82, 0xA0, 0x83, 0xA0, 0x84, 0xA0, 0x85, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_C8[512] = { + 0xC0, 0xFA, 0xC0, 0xFB, 0xA0, 0x86, 0xA0, 0x87, /* 0x00-0x03 */ + 0xC0, 0xFC, 0xA0, 0x88, 0xA0, 0x89, 0xA0, 0x8A, /* 0x04-0x07 */ + 0xC0, 0xFD, 0xA0, 0x8B, 0xC0, 0xFE, 0xA0, 0x8C, /* 0x08-0x0B */ + 0xA0, 0x8D, 0xA0, 0x8E, 0xA0, 0x8F, 0xA0, 0x90, /* 0x0C-0x0F */ + 0xC1, 0xA1, 0xC1, 0xA2, 0xA0, 0x91, 0xC1, 0xA3, /* 0x10-0x13 */ + 0xA0, 0x92, 0xC1, 0xA4, 0xC1, 0xA5, 0xA0, 0x93, /* 0x14-0x17 */ + 0xA0, 0x94, 0xA0, 0x95, 0xA0, 0x96, 0xA0, 0x97, /* 0x18-0x1B */ + 0xC1, 0xA6, 0xC1, 0xA7, 0xA0, 0x98, 0xA0, 0x99, /* 0x1C-0x1F */ + 0xC1, 0xA8, 0xA0, 0x9A, 0xA0, 0x9B, 0xA0, 0x9C, /* 0x20-0x23 */ + 0xC1, 0xA9, 0xA0, 0x9D, 0xA0, 0x9E, 0xA0, 0x9F, /* 0x24-0x27 */ + 0xA0, 0xA0, 0xA0, 0xA1, 0xA0, 0xA2, 0xA0, 0xA3, /* 0x28-0x2B */ + 0xC1, 0xAA, 0xC1, 0xAB, 0xA0, 0xA4, 0xC1, 0xAC, /* 0x2C-0x2F */ + 0xA0, 0xA5, 0xC1, 0xAD, 0xA0, 0xA6, 0xA0, 0xA7, /* 0x30-0x33 */ + 0xA0, 0xA8, 0xA0, 0xA9, 0xA0, 0xAA, 0xA0, 0xAB, /* 0x34-0x37 */ + 0xC1, 0xAE, 0xA0, 0xAC, 0xA0, 0xAD, 0xA0, 0xAE, /* 0x38-0x3B */ + 0xC1, 0xAF, 0xA0, 0xAF, 0xA0, 0xB0, 0xA0, 0xB1, /* 0x3C-0x3F */ + 0xC1, 0xB0, 0xA0, 0xB2, 0xA0, 0xB3, 0xA0, 0xB4, /* 0x40-0x43 */ + 0xA0, 0xB5, 0xA0, 0xB6, 0xA0, 0xB7, 0xA0, 0xB8, /* 0x44-0x47 */ + 0xC1, 0xB1, 0xC1, 0xB2, 0xA0, 0xB9, 0xA0, 0xBA, /* 0x48-0x4B */ + 0xC1, 0xB3, 0xC1, 0xB4, 0xA0, 0xBB, 0xA0, 0xBC, /* 0x4C-0x4F */ + 0xA0, 0xBD, 0xA0, 0xBE, 0xA0, 0xBF, 0xA0, 0xC0, /* 0x50-0x53 */ + 0xC1, 0xB5, 0xA0, 0xC1, 0xA0, 0xC2, 0xA0, 0xC3, /* 0x54-0x57 */ + 0xA0, 0xC4, 0xA0, 0xC5, 0xA0, 0xC6, 0xA0, 0xC7, /* 0x58-0x5B */ + 0xA0, 0xC8, 0xA0, 0xC9, 0xA0, 0xCA, 0xA0, 0xCB, /* 0x5C-0x5F */ + 0xA0, 0xCC, 0xA0, 0xCD, 0xA0, 0xCE, 0xA0, 0xCF, /* 0x60-0x63 */ + 0xA0, 0xD0, 0xA0, 0xD1, 0xA0, 0xD2, 0xA0, 0xD3, /* 0x64-0x67 */ + 0xA0, 0xD4, 0xA0, 0xD5, 0xA0, 0xD6, 0xA0, 0xD7, /* 0x68-0x6B */ + 0xA0, 0xD8, 0xA0, 0xD9, 0xA0, 0xDA, 0xA0, 0xDB, /* 0x6C-0x6F */ + 0xC1, 0xB6, 0xC1, 0xB7, 0xA0, 0xDC, 0xA0, 0xDD, /* 0x70-0x73 */ + 0xC1, 0xB8, 0xA0, 0xDE, 0xA0, 0xDF, 0xA0, 0xE0, /* 0x74-0x77 */ + 0xC1, 0xB9, 0xA0, 0xE1, 0xC1, 0xBA, 0xA0, 0xE2, /* 0x78-0x7B */ + 0xA0, 0xE3, 0xA0, 0xE4, 0xA0, 0xE5, 0xA0, 0xE6, /* 0x7C-0x7F */ + + 0xC1, 0xBB, 0xC1, 0xBC, 0xA0, 0xE7, 0xC1, 0xBD, /* 0x80-0x83 */ + 0xA0, 0xE8, 0xC1, 0xBE, 0xC1, 0xBF, 0xC1, 0xC0, /* 0x84-0x87 */ + 0xA0, 0xE9, 0xA0, 0xEA, 0xA0, 0xEB, 0xC1, 0xC1, /* 0x88-0x8B */ + 0xC1, 0xC2, 0xC1, 0xC3, 0xA0, 0xEC, 0xA0, 0xED, /* 0x8C-0x8F */ + 0xA0, 0xEE, 0xA0, 0xEF, 0xA0, 0xF0, 0xA0, 0xF1, /* 0x90-0x93 */ + 0xC1, 0xC4, 0xA0, 0xF2, 0xA0, 0xF3, 0xA0, 0xF4, /* 0x94-0x97 */ + 0xA0, 0xF5, 0xA0, 0xF6, 0xA0, 0xF7, 0xA0, 0xF8, /* 0x98-0x9B */ + 0xA0, 0xF9, 0xC1, 0xC5, 0xA0, 0xFA, 0xC1, 0xC6, /* 0x9C-0x9F */ + 0xA0, 0xFB, 0xC1, 0xC7, 0xA0, 0xFC, 0xA0, 0xFD, /* 0xA0-0xA3 */ + 0xA0, 0xFE, 0xA1, 0x41, 0xA1, 0x42, 0xA1, 0x43, /* 0xA4-0xA7 */ + 0xC1, 0xC8, 0xA1, 0x44, 0xA1, 0x45, 0xA1, 0x46, /* 0xA8-0xAB */ + 0xA1, 0x47, 0xA1, 0x48, 0xA1, 0x49, 0xA1, 0x4A, /* 0xAC-0xAF */ + 0xA1, 0x4B, 0xA1, 0x4C, 0xA1, 0x4D, 0xA1, 0x4E, /* 0xB0-0xB3 */ + 0xA1, 0x4F, 0xA1, 0x50, 0xA1, 0x51, 0xA1, 0x52, /* 0xB4-0xB7 */ + 0xA1, 0x53, 0xA1, 0x54, 0xA1, 0x55, 0xA1, 0x56, /* 0xB8-0xBB */ + 0xC1, 0xC9, 0xC1, 0xCA, 0xA1, 0x57, 0xA1, 0x58, /* 0xBC-0xBF */ + 0xA1, 0x59, 0xA1, 0x5A, 0xA1, 0x61, 0xA1, 0x62, /* 0xC0-0xC3 */ + 0xC1, 0xCB, 0xA1, 0x63, 0xA1, 0x64, 0xA1, 0x65, /* 0xC4-0xC7 */ + 0xC1, 0xCC, 0xA1, 0x66, 0xA1, 0x67, 0xA1, 0x68, /* 0xC8-0xCB */ + 0xC1, 0xCD, 0xA1, 0x69, 0xA1, 0x6A, 0xA1, 0x6B, /* 0xCC-0xCF */ + 0xA1, 0x6C, 0xA1, 0x6D, 0xA1, 0x6E, 0xA1, 0x6F, /* 0xD0-0xD3 */ + 0xC1, 0xCE, 0xC1, 0xCF, 0xA1, 0x70, 0xC1, 0xD0, /* 0xD4-0xD7 */ + 0xA1, 0x71, 0xC1, 0xD1, 0xA1, 0x72, 0xA1, 0x73, /* 0xD8-0xDB */ + 0xA1, 0x74, 0xA1, 0x75, 0xA1, 0x76, 0xA1, 0x77, /* 0xDC-0xDF */ + 0xC1, 0xD2, 0xC1, 0xD3, 0xA1, 0x78, 0xA1, 0x79, /* 0xE0-0xE3 */ + 0xC1, 0xD4, 0xA1, 0x7A, 0xA1, 0x81, 0xA1, 0x82, /* 0xE4-0xE7 */ + 0xA1, 0x83, 0xA1, 0x84, 0xA1, 0x85, 0xA1, 0x86, /* 0xE8-0xEB */ + 0xA1, 0x87, 0xA1, 0x88, 0xA1, 0x89, 0xA1, 0x8A, /* 0xEC-0xEF */ + 0xA1, 0x8B, 0xA1, 0x8C, 0xA1, 0x8D, 0xA1, 0x8E, /* 0xF0-0xF3 */ + 0xA1, 0x8F, 0xC1, 0xD5, 0xA1, 0x90, 0xA1, 0x91, /* 0xF4-0xF7 */ + 0xA1, 0x92, 0xA1, 0x93, 0xA1, 0x94, 0xA1, 0x95, /* 0xF8-0xFB */ + 0xC1, 0xD6, 0xC1, 0xD7, 0xA1, 0x96, 0xA1, 0x97, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_C9[512] = { + 0xC1, 0xD8, 0xA1, 0x98, 0xA1, 0x99, 0xA1, 0x9A, /* 0x00-0x03 */ + 0xC1, 0xD9, 0xC1, 0xDA, 0xC1, 0xDB, 0xA1, 0x9B, /* 0x04-0x07 */ + 0xA1, 0x9C, 0xA1, 0x9D, 0xA1, 0x9E, 0xA1, 0x9F, /* 0x08-0x0B */ + 0xC1, 0xDC, 0xC1, 0xDD, 0xA1, 0xA0, 0xC1, 0xDE, /* 0x0C-0x0F */ + 0xA2, 0x41, 0xC1, 0xDF, 0xA2, 0x42, 0xA2, 0x43, /* 0x10-0x13 */ + 0xA2, 0x44, 0xA2, 0x45, 0xA2, 0x46, 0xA2, 0x47, /* 0x14-0x17 */ + 0xC1, 0xE0, 0xA2, 0x48, 0xA2, 0x49, 0xA2, 0x4A, /* 0x18-0x1B */ + 0xA2, 0x4B, 0xA2, 0x4C, 0xA2, 0x4D, 0xA2, 0x4E, /* 0x1C-0x1F */ + 0xA2, 0x4F, 0xA2, 0x50, 0xA2, 0x51, 0xA2, 0x52, /* 0x20-0x23 */ + 0xA2, 0x53, 0xA2, 0x54, 0xA2, 0x55, 0xA2, 0x56, /* 0x24-0x27 */ + 0xA2, 0x57, 0xA2, 0x58, 0xA2, 0x59, 0xA2, 0x5A, /* 0x28-0x2B */ + 0xC1, 0xE1, 0xA2, 0x61, 0xA2, 0x62, 0xA2, 0x63, /* 0x2C-0x2F */ + 0xA2, 0x64, 0xA2, 0x65, 0xA2, 0x66, 0xA2, 0x67, /* 0x30-0x33 */ + 0xC1, 0xE2, 0xA2, 0x68, 0xA2, 0x69, 0xA2, 0x6A, /* 0x34-0x37 */ + 0xA2, 0x6B, 0xA2, 0x6C, 0xA2, 0x6D, 0xA2, 0x6E, /* 0x38-0x3B */ + 0xA2, 0x6F, 0xA2, 0x70, 0xA2, 0x71, 0xA2, 0x72, /* 0x3C-0x3F */ + 0xA2, 0x73, 0xA2, 0x74, 0xA2, 0x75, 0xA2, 0x76, /* 0x40-0x43 */ + 0xA2, 0x77, 0xA2, 0x78, 0xA2, 0x79, 0xA2, 0x7A, /* 0x44-0x47 */ + 0xA2, 0x81, 0xA2, 0x82, 0xA2, 0x83, 0xA2, 0x84, /* 0x48-0x4B */ + 0xA2, 0x85, 0xA2, 0x86, 0xA2, 0x87, 0xA2, 0x88, /* 0x4C-0x4F */ + 0xC1, 0xE3, 0xC1, 0xE4, 0xA2, 0x89, 0xA2, 0x8A, /* 0x50-0x53 */ + 0xC1, 0xE5, 0xA2, 0x8B, 0xA2, 0x8C, 0xA2, 0x8D, /* 0x54-0x57 */ + 0xC1, 0xE6, 0xA2, 0x8E, 0xA2, 0x8F, 0xA2, 0x90, /* 0x58-0x5B */ + 0xA2, 0x91, 0xA2, 0x92, 0xA2, 0x93, 0xA2, 0x94, /* 0x5C-0x5F */ + 0xC1, 0xE7, 0xC1, 0xE8, 0xA2, 0x95, 0xC1, 0xE9, /* 0x60-0x63 */ + 0xA2, 0x96, 0xA2, 0x97, 0xA2, 0x98, 0xA2, 0x99, /* 0x64-0x67 */ + 0xA2, 0x9A, 0xA2, 0x9B, 0xA2, 0x9C, 0xA2, 0x9D, /* 0x68-0x6B */ + 0xC1, 0xEA, 0xA2, 0x9E, 0xA2, 0x9F, 0xA2, 0xA0, /* 0x6C-0x6F */ + 0xC1, 0xEB, 0xA3, 0x41, 0xA3, 0x42, 0xA3, 0x43, /* 0x70-0x73 */ + 0xC1, 0xEC, 0xA3, 0x44, 0xA3, 0x45, 0xA3, 0x46, /* 0x74-0x77 */ + 0xA3, 0x47, 0xA3, 0x48, 0xA3, 0x49, 0xA3, 0x4A, /* 0x78-0x7B */ + 0xC1, 0xED, 0xA3, 0x4B, 0xA3, 0x4C, 0xA3, 0x4D, /* 0x7C-0x7F */ + + 0xA3, 0x4E, 0xA3, 0x4F, 0xA3, 0x50, 0xA3, 0x51, /* 0x80-0x83 */ + 0xA3, 0x52, 0xA3, 0x53, 0xA3, 0x54, 0xA3, 0x55, /* 0x84-0x87 */ + 0xC1, 0xEE, 0xC1, 0xEF, 0xA3, 0x56, 0xA3, 0x57, /* 0x88-0x8B */ + 0xC1, 0xF0, 0xA3, 0x58, 0xA3, 0x59, 0xA3, 0x5A, /* 0x8C-0x8F */ + 0xC1, 0xF1, 0xA3, 0x61, 0xA3, 0x62, 0xA3, 0x63, /* 0x90-0x93 */ + 0xA3, 0x64, 0xA3, 0x65, 0xA3, 0x66, 0xA3, 0x67, /* 0x94-0x97 */ + 0xC1, 0xF2, 0xC1, 0xF3, 0xA3, 0x68, 0xC1, 0xF4, /* 0x98-0x9B */ + 0xA3, 0x69, 0xC1, 0xF5, 0xA3, 0x6A, 0xA3, 0x6B, /* 0x9C-0x9F */ + 0xA3, 0x6C, 0xA3, 0x6D, 0xA3, 0x6E, 0xA3, 0x6F, /* 0xA0-0xA3 */ + 0xA3, 0x70, 0xA3, 0x71, 0xA3, 0x72, 0xA3, 0x73, /* 0xA4-0xA7 */ + 0xA3, 0x74, 0xA3, 0x75, 0xA3, 0x76, 0xA3, 0x77, /* 0xA8-0xAB */ + 0xA3, 0x78, 0xA3, 0x79, 0xA3, 0x7A, 0xA3, 0x81, /* 0xAC-0xAF */ + 0xA3, 0x82, 0xA3, 0x83, 0xA3, 0x84, 0xA3, 0x85, /* 0xB0-0xB3 */ + 0xA3, 0x86, 0xA3, 0x87, 0xA3, 0x88, 0xA3, 0x89, /* 0xB4-0xB7 */ + 0xA3, 0x8A, 0xA3, 0x8B, 0xA3, 0x8C, 0xA3, 0x8D, /* 0xB8-0xBB */ + 0xA3, 0x8E, 0xA3, 0x8F, 0xA3, 0x90, 0xA3, 0x91, /* 0xBC-0xBF */ + 0xC1, 0xF6, 0xC1, 0xF7, 0xA3, 0x92, 0xA3, 0x93, /* 0xC0-0xC3 */ + 0xC1, 0xF8, 0xA3, 0x94, 0xA3, 0x95, 0xC1, 0xF9, /* 0xC4-0xC7 */ + 0xC1, 0xFA, 0xA3, 0x96, 0xC1, 0xFB, 0xA3, 0x97, /* 0xC8-0xCB */ + 0xA3, 0x98, 0xA3, 0x99, 0xA3, 0x9A, 0xA3, 0x9B, /* 0xCC-0xCF */ + 0xC1, 0xFC, 0xC1, 0xFD, 0xA3, 0x9C, 0xC1, 0xFE, /* 0xD0-0xD3 */ + 0xA3, 0x9D, 0xC2, 0xA1, 0xC2, 0xA2, 0xA3, 0x9E, /* 0xD4-0xD7 */ + 0xA3, 0x9F, 0xC2, 0xA3, 0xC2, 0xA4, 0xA3, 0xA0, /* 0xD8-0xDB */ + 0xC2, 0xA5, 0xC2, 0xA6, 0xA4, 0x41, 0xA4, 0x42, /* 0xDC-0xDF */ + 0xC2, 0xA7, 0xA4, 0x43, 0xC2, 0xA8, 0xA4, 0x44, /* 0xE0-0xE3 */ + 0xC2, 0xA9, 0xA4, 0x45, 0xA4, 0x46, 0xC2, 0xAA, /* 0xE4-0xE7 */ + 0xA4, 0x47, 0xA4, 0x48, 0xA4, 0x49, 0xA4, 0x4A, /* 0xE8-0xEB */ + 0xC2, 0xAB, 0xC2, 0xAC, 0xA4, 0x4B, 0xC2, 0xAD, /* 0xEC-0xEF */ + 0xC2, 0xAE, 0xC2, 0xAF, 0xA4, 0x4C, 0xA4, 0x4D, /* 0xF0-0xF3 */ + 0xA4, 0x4E, 0xA4, 0x4F, 0xA4, 0x50, 0xA4, 0x51, /* 0xF4-0xF7 */ + 0xC2, 0xB0, 0xC2, 0xB1, 0xA4, 0x52, 0xA4, 0x53, /* 0xF8-0xFB */ + 0xC2, 0xB2, 0xA4, 0x54, 0xA4, 0x55, 0xA4, 0x56, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_CA[512] = { + 0xC2, 0xB3, 0xA4, 0x57, 0xA4, 0x58, 0xA4, 0x59, /* 0x00-0x03 */ + 0xA4, 0x5A, 0xA4, 0x61, 0xA4, 0x62, 0xA4, 0x63, /* 0x04-0x07 */ + 0xC2, 0xB4, 0xC2, 0xB5, 0xA4, 0x64, 0xC2, 0xB6, /* 0x08-0x0B */ + 0xC2, 0xB7, 0xC2, 0xB8, 0xA4, 0x65, 0xA4, 0x66, /* 0x0C-0x0F */ + 0xA4, 0x67, 0xA4, 0x68, 0xA4, 0x69, 0xA4, 0x6A, /* 0x10-0x13 */ + 0xC2, 0xB9, 0xA4, 0x6B, 0xA4, 0x6C, 0xA4, 0x6D, /* 0x14-0x17 */ + 0xC2, 0xBA, 0xA4, 0x6E, 0xA4, 0x6F, 0xA4, 0x70, /* 0x18-0x1B */ + 0xA4, 0x71, 0xA4, 0x72, 0xA4, 0x73, 0xA4, 0x74, /* 0x1C-0x1F */ + 0xA4, 0x75, 0xA4, 0x76, 0xA4, 0x77, 0xA4, 0x78, /* 0x20-0x23 */ + 0xA4, 0x79, 0xA4, 0x7A, 0xA4, 0x81, 0xA4, 0x82, /* 0x24-0x27 */ + 0xA4, 0x83, 0xC2, 0xBB, 0xA4, 0x84, 0xA4, 0x85, /* 0x28-0x2B */ + 0xA4, 0x86, 0xA4, 0x87, 0xA4, 0x88, 0xA4, 0x89, /* 0x2C-0x2F */ + 0xA4, 0x8A, 0xA4, 0x8B, 0xA4, 0x8C, 0xA4, 0x8D, /* 0x30-0x33 */ + 0xA4, 0x8E, 0xA4, 0x8F, 0xA4, 0x90, 0xA4, 0x91, /* 0x34-0x37 */ + 0xA4, 0x92, 0xA4, 0x93, 0xA4, 0x94, 0xA4, 0x95, /* 0x38-0x3B */ + 0xA4, 0x96, 0xA4, 0x97, 0xA4, 0x98, 0xA4, 0x99, /* 0x3C-0x3F */ + 0xA4, 0x9A, 0xA4, 0x9B, 0xA4, 0x9C, 0xA4, 0x9D, /* 0x40-0x43 */ + 0xA4, 0x9E, 0xA4, 0x9F, 0xA4, 0xA0, 0xA5, 0x41, /* 0x44-0x47 */ + 0xA5, 0x42, 0xA5, 0x43, 0xA5, 0x44, 0xA5, 0x45, /* 0x48-0x4B */ + 0xC2, 0xBC, 0xC2, 0xBD, 0xA5, 0x46, 0xA5, 0x47, /* 0x4C-0x4F */ + 0xC2, 0xBE, 0xA5, 0x48, 0xA5, 0x49, 0xA5, 0x4A, /* 0x50-0x53 */ + 0xC2, 0xBF, 0xA5, 0x4B, 0xA5, 0x4C, 0xA5, 0x4D, /* 0x54-0x57 */ + 0xA5, 0x4E, 0xA5, 0x4F, 0xA5, 0x50, 0xA5, 0x51, /* 0x58-0x5B */ + 0xC2, 0xC0, 0xC2, 0xC1, 0xA5, 0x52, 0xC2, 0xC2, /* 0x5C-0x5F */ + 0xC2, 0xC3, 0xC2, 0xC4, 0xA5, 0x53, 0xA5, 0x54, /* 0x60-0x63 */ + 0xA5, 0x55, 0xA5, 0x56, 0xA5, 0x57, 0xA5, 0x58, /* 0x64-0x67 */ + 0xC2, 0xC5, 0xA5, 0x59, 0xA5, 0x5A, 0xA5, 0x61, /* 0x68-0x6B */ + 0xA5, 0x62, 0xA5, 0x63, 0xA5, 0x64, 0xA5, 0x65, /* 0x6C-0x6F */ + 0xA5, 0x66, 0xA5, 0x67, 0xA5, 0x68, 0xA5, 0x69, /* 0x70-0x73 */ + 0xA5, 0x6A, 0xA5, 0x6B, 0xA5, 0x6C, 0xA5, 0x6D, /* 0x74-0x77 */ + 0xA5, 0x6E, 0xA5, 0x6F, 0xA5, 0x70, 0xA5, 0x71, /* 0x78-0x7B */ + 0xA5, 0x72, 0xC2, 0xC6, 0xA5, 0x73, 0xA5, 0x74, /* 0x7C-0x7F */ + + 0xA5, 0x75, 0xA5, 0x76, 0xA5, 0x77, 0xA5, 0x78, /* 0x80-0x83 */ + 0xC2, 0xC7, 0xA5, 0x79, 0xA5, 0x7A, 0xA5, 0x81, /* 0x84-0x87 */ + 0xA5, 0x82, 0xA5, 0x83, 0xA5, 0x84, 0xA5, 0x85, /* 0x88-0x8B */ + 0xA5, 0x86, 0xA5, 0x87, 0xA5, 0x88, 0xA5, 0x89, /* 0x8C-0x8F */ + 0xA5, 0x8A, 0xA5, 0x8B, 0xA5, 0x8C, 0xA5, 0x8D, /* 0x90-0x93 */ + 0xA5, 0x8E, 0xA5, 0x8F, 0xA5, 0x90, 0xA5, 0x91, /* 0x94-0x97 */ + 0xC2, 0xC8, 0xA5, 0x92, 0xA5, 0x93, 0xA5, 0x94, /* 0x98-0x9B */ + 0xA5, 0x95, 0xA5, 0x96, 0xA5, 0x97, 0xA5, 0x98, /* 0x9C-0x9F */ + 0xA5, 0x99, 0xA5, 0x9A, 0xA5, 0x9B, 0xA5, 0x9C, /* 0xA0-0xA3 */ + 0xA5, 0x9D, 0xA5, 0x9E, 0xA5, 0x9F, 0xA5, 0xA0, /* 0xA4-0xA7 */ + 0xA6, 0x41, 0xA6, 0x42, 0xA6, 0x43, 0xA6, 0x44, /* 0xA8-0xAB */ + 0xA6, 0x45, 0xA6, 0x46, 0xA6, 0x47, 0xA6, 0x48, /* 0xAC-0xAF */ + 0xA6, 0x49, 0xA6, 0x4A, 0xA6, 0x4B, 0xA6, 0x4C, /* 0xB0-0xB3 */ + 0xA6, 0x4D, 0xA6, 0x4E, 0xA6, 0x4F, 0xA6, 0x50, /* 0xB4-0xB7 */ + 0xA6, 0x51, 0xA6, 0x52, 0xA6, 0x53, 0xA6, 0x54, /* 0xB8-0xBB */ + 0xC2, 0xC9, 0xC2, 0xCA, 0xA6, 0x55, 0xA6, 0x56, /* 0xBC-0xBF */ + 0xC2, 0xCB, 0xA6, 0x57, 0xA6, 0x58, 0xA6, 0x59, /* 0xC0-0xC3 */ + 0xC2, 0xCC, 0xA6, 0x5A, 0xA6, 0x61, 0xA6, 0x62, /* 0xC4-0xC7 */ + 0xA6, 0x63, 0xA6, 0x64, 0xA6, 0x65, 0xA6, 0x66, /* 0xC8-0xCB */ + 0xC2, 0xCD, 0xC2, 0xCE, 0xA6, 0x67, 0xC2, 0xCF, /* 0xCC-0xCF */ + 0xA6, 0x68, 0xC2, 0xD0, 0xA6, 0x69, 0xC2, 0xD1, /* 0xD0-0xD3 */ + 0xA6, 0x6A, 0xA6, 0x6B, 0xA6, 0x6C, 0xA6, 0x6D, /* 0xD4-0xD7 */ + 0xC2, 0xD2, 0xC2, 0xD3, 0xA6, 0x6E, 0xA6, 0x6F, /* 0xD8-0xDB */ + 0xA6, 0x70, 0xA6, 0x71, 0xA6, 0x72, 0xA6, 0x73, /* 0xDC-0xDF */ + 0xC2, 0xD4, 0xA6, 0x74, 0xA6, 0x75, 0xA6, 0x76, /* 0xE0-0xE3 */ + 0xA6, 0x77, 0xA6, 0x78, 0xA6, 0x79, 0xA6, 0x7A, /* 0xE4-0xE7 */ + 0xA6, 0x81, 0xA6, 0x82, 0xA6, 0x83, 0xA6, 0x84, /* 0xE8-0xEB */ + 0xC2, 0xD5, 0xA6, 0x85, 0xA6, 0x86, 0xA6, 0x87, /* 0xEC-0xEF */ + 0xA6, 0x88, 0xA6, 0x89, 0xA6, 0x8A, 0xA6, 0x8B, /* 0xF0-0xF3 */ + 0xC2, 0xD6, 0xA6, 0x8C, 0xA6, 0x8D, 0xA6, 0x8E, /* 0xF4-0xF7 */ + 0xA6, 0x8F, 0xA6, 0x90, 0xA6, 0x91, 0xA6, 0x92, /* 0xF8-0xFB */ + 0xA6, 0x93, 0xA6, 0x94, 0xA6, 0x95, 0xA6, 0x96, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_CB[512] = { + 0xA6, 0x97, 0xA6, 0x98, 0xA6, 0x99, 0xA6, 0x9A, /* 0x00-0x03 */ + 0xA6, 0x9B, 0xA6, 0x9C, 0xA6, 0x9D, 0xA6, 0x9E, /* 0x04-0x07 */ + 0xC2, 0xD7, 0xA6, 0x9F, 0xA6, 0xA0, 0xA7, 0x41, /* 0x08-0x0B */ + 0xA7, 0x42, 0xA7, 0x43, 0xA7, 0x44, 0xA7, 0x45, /* 0x0C-0x0F */ + 0xC2, 0xD8, 0xA7, 0x46, 0xA7, 0x47, 0xA7, 0x48, /* 0x10-0x13 */ + 0xC2, 0xD9, 0xA7, 0x49, 0xA7, 0x4A, 0xA7, 0x4B, /* 0x14-0x17 */ + 0xC2, 0xDA, 0xA7, 0x4C, 0xA7, 0x4D, 0xA7, 0x4E, /* 0x18-0x1B */ + 0xA7, 0x4F, 0xA7, 0x50, 0xA7, 0x51, 0xA7, 0x52, /* 0x1C-0x1F */ + 0xC2, 0xDB, 0xC2, 0xDC, 0xA7, 0x53, 0xA7, 0x54, /* 0x20-0x23 */ + 0xA7, 0x55, 0xA7, 0x56, 0xA7, 0x57, 0xA7, 0x58, /* 0x24-0x27 */ + 0xA7, 0x59, 0xA7, 0x5A, 0xA7, 0x61, 0xA7, 0x62, /* 0x28-0x2B */ + 0xA7, 0x63, 0xA7, 0x64, 0xA7, 0x65, 0xA7, 0x66, /* 0x2C-0x2F */ + 0xA7, 0x67, 0xA7, 0x68, 0xA7, 0x69, 0xA7, 0x6A, /* 0x30-0x33 */ + 0xA7, 0x6B, 0xA7, 0x6C, 0xA7, 0x6D, 0xA7, 0x6E, /* 0x34-0x37 */ + 0xA7, 0x6F, 0xA7, 0x70, 0xA7, 0x71, 0xA7, 0x72, /* 0x38-0x3B */ + 0xA7, 0x73, 0xA7, 0x74, 0xA7, 0x75, 0xA7, 0x76, /* 0x3C-0x3F */ + 0xA7, 0x77, 0xC2, 0xDD, 0xA7, 0x78, 0xA7, 0x79, /* 0x40-0x43 */ + 0xA7, 0x7A, 0xA7, 0x81, 0xA7, 0x82, 0xA7, 0x83, /* 0x44-0x47 */ + 0xC2, 0xDE, 0xC2, 0xDF, 0xA7, 0x84, 0xA7, 0x85, /* 0x48-0x4B */ + 0xC2, 0xE0, 0xA7, 0x86, 0xA7, 0x87, 0xA7, 0x88, /* 0x4C-0x4F */ + 0xC2, 0xE1, 0xA7, 0x89, 0xA7, 0x8A, 0xA7, 0x8B, /* 0x50-0x53 */ + 0xA7, 0x8C, 0xA7, 0x8D, 0xA7, 0x8E, 0xA7, 0x8F, /* 0x54-0x57 */ + 0xC2, 0xE2, 0xC2, 0xE3, 0xA7, 0x90, 0xA7, 0x91, /* 0x58-0x5B */ + 0xA7, 0x92, 0xC2, 0xE4, 0xA7, 0x93, 0xA7, 0x94, /* 0x5C-0x5F */ + 0xA7, 0x95, 0xA7, 0x96, 0xA7, 0x97, 0xA7, 0x98, /* 0x60-0x63 */ + 0xC2, 0xE5, 0xA7, 0x99, 0xA7, 0x9A, 0xA7, 0x9B, /* 0x64-0x67 */ + 0xA7, 0x9C, 0xA7, 0x9D, 0xA7, 0x9E, 0xA7, 0x9F, /* 0x68-0x6B */ + 0xA7, 0xA0, 0xA8, 0x41, 0xA8, 0x42, 0xA8, 0x43, /* 0x6C-0x6F */ + 0xA8, 0x44, 0xA8, 0x45, 0xA8, 0x46, 0xA8, 0x47, /* 0x70-0x73 */ + 0xA8, 0x48, 0xA8, 0x49, 0xA8, 0x4A, 0xA8, 0x4B, /* 0x74-0x77 */ + 0xC2, 0xE6, 0xC2, 0xE7, 0xA8, 0x4C, 0xA8, 0x4D, /* 0x78-0x7B */ + 0xA8, 0x4E, 0xA8, 0x4F, 0xA8, 0x50, 0xA8, 0x51, /* 0x7C-0x7F */ + + 0xA8, 0x52, 0xA8, 0x53, 0xA8, 0x54, 0xA8, 0x55, /* 0x80-0x83 */ + 0xA8, 0x56, 0xA8, 0x57, 0xA8, 0x58, 0xA8, 0x59, /* 0x84-0x87 */ + 0xA8, 0x5A, 0xA8, 0x61, 0xA8, 0x62, 0xA8, 0x63, /* 0x88-0x8B */ + 0xA8, 0x64, 0xA8, 0x65, 0xA8, 0x66, 0xA8, 0x67, /* 0x8C-0x8F */ + 0xA8, 0x68, 0xA8, 0x69, 0xA8, 0x6A, 0xA8, 0x6B, /* 0x90-0x93 */ + 0xA8, 0x6C, 0xA8, 0x6D, 0xA8, 0x6E, 0xA8, 0x6F, /* 0x94-0x97 */ + 0xA8, 0x70, 0xA8, 0x71, 0xA8, 0x72, 0xA8, 0x73, /* 0x98-0x9B */ + 0xC2, 0xE8, 0xA8, 0x74, 0xA8, 0x75, 0xA8, 0x76, /* 0x9C-0x9F */ + 0xA8, 0x77, 0xA8, 0x78, 0xA8, 0x79, 0xA8, 0x7A, /* 0xA0-0xA3 */ + 0xA8, 0x81, 0xA8, 0x82, 0xA8, 0x83, 0xA8, 0x84, /* 0xA4-0xA7 */ + 0xA8, 0x85, 0xA8, 0x86, 0xA8, 0x87, 0xA8, 0x88, /* 0xA8-0xAB */ + 0xA8, 0x89, 0xA8, 0x8A, 0xA8, 0x8B, 0xA8, 0x8C, /* 0xAC-0xAF */ + 0xA8, 0x8D, 0xA8, 0x8E, 0xA8, 0x8F, 0xA8, 0x90, /* 0xB0-0xB3 */ + 0xA8, 0x91, 0xA8, 0x92, 0xA8, 0x93, 0xA8, 0x94, /* 0xB4-0xB7 */ + 0xC2, 0xE9, 0xA8, 0x95, 0xA8, 0x96, 0xA8, 0x97, /* 0xB8-0xBB */ + 0xA8, 0x98, 0xA8, 0x99, 0xA8, 0x9A, 0xA8, 0x9B, /* 0xBC-0xBF */ + 0xA8, 0x9C, 0xA8, 0x9D, 0xA8, 0x9E, 0xA8, 0x9F, /* 0xC0-0xC3 */ + 0xA8, 0xA0, 0xA9, 0x41, 0xA9, 0x42, 0xA9, 0x43, /* 0xC4-0xC7 */ + 0xA9, 0x44, 0xA9, 0x45, 0xA9, 0x46, 0xA9, 0x47, /* 0xC8-0xCB */ + 0xA9, 0x48, 0xA9, 0x49, 0xA9, 0x4A, 0xA9, 0x4B, /* 0xCC-0xCF */ + 0xA9, 0x4C, 0xA9, 0x4D, 0xA9, 0x4E, 0xA9, 0x4F, /* 0xD0-0xD3 */ + 0xC2, 0xEA, 0xA9, 0x50, 0xA9, 0x51, 0xA9, 0x52, /* 0xD4-0xD7 */ + 0xA9, 0x53, 0xA9, 0x54, 0xA9, 0x55, 0xA9, 0x56, /* 0xD8-0xDB */ + 0xA9, 0x57, 0xA9, 0x58, 0xA9, 0x59, 0xA9, 0x5A, /* 0xDC-0xDF */ + 0xA9, 0x61, 0xA9, 0x62, 0xA9, 0x63, 0xA9, 0x64, /* 0xE0-0xE3 */ + 0xC2, 0xEB, 0xA9, 0x65, 0xA9, 0x66, 0xC2, 0xEC, /* 0xE4-0xE7 */ + 0xA9, 0x67, 0xC2, 0xED, 0xA9, 0x68, 0xA9, 0x69, /* 0xE8-0xEB */ + 0xA9, 0x6A, 0xA9, 0x6B, 0xA9, 0x6C, 0xA9, 0x6D, /* 0xEC-0xEF */ + 0xA9, 0x6E, 0xA9, 0x6F, 0xA9, 0x70, 0xA9, 0x71, /* 0xF0-0xF3 */ + 0xA9, 0x72, 0xA9, 0x73, 0xA9, 0x74, 0xA9, 0x75, /* 0xF4-0xF7 */ + 0xA9, 0x76, 0xA9, 0x77, 0xA9, 0x78, 0xA9, 0x79, /* 0xF8-0xFB */ + 0xA9, 0x7A, 0xA9, 0x81, 0xA9, 0x82, 0xA9, 0x83, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_CC[512] = { + 0xA9, 0x84, 0xA9, 0x85, 0xA9, 0x86, 0xA9, 0x87, /* 0x00-0x03 */ + 0xA9, 0x88, 0xA9, 0x89, 0xA9, 0x8A, 0xA9, 0x8B, /* 0x04-0x07 */ + 0xA9, 0x8C, 0xA9, 0x8D, 0xA9, 0x8E, 0xA9, 0x8F, /* 0x08-0x0B */ + 0xC2, 0xEE, 0xC2, 0xEF, 0xA9, 0x90, 0xA9, 0x91, /* 0x0C-0x0F */ + 0xC2, 0xF0, 0xA9, 0x92, 0xA9, 0x93, 0xA9, 0x94, /* 0x10-0x13 */ + 0xC2, 0xF1, 0xA9, 0x95, 0xA9, 0x96, 0xA9, 0x97, /* 0x14-0x17 */ + 0xA9, 0x98, 0xA9, 0x99, 0xA9, 0x9A, 0xA9, 0x9B, /* 0x18-0x1B */ + 0xC2, 0xF2, 0xC2, 0xF3, 0xA9, 0x9C, 0xA9, 0x9D, /* 0x1C-0x1F */ + 0xA9, 0x9E, 0xC2, 0xF4, 0xC2, 0xF5, 0xA9, 0x9F, /* 0x20-0x23 */ + 0xA9, 0xA0, 0xAA, 0x41, 0xAA, 0x42, 0xC2, 0xF6, /* 0x24-0x27 */ + 0xC2, 0xF7, 0xC2, 0xF8, 0xAA, 0x43, 0xAA, 0x44, /* 0x28-0x2B */ + 0xC2, 0xF9, 0xAA, 0x45, 0xC2, 0xFA, 0xAA, 0x46, /* 0x2C-0x2F */ + 0xC2, 0xFB, 0xAA, 0x47, 0xAA, 0x48, 0xAA, 0x49, /* 0x30-0x33 */ + 0xAA, 0x4A, 0xAA, 0x4B, 0xAA, 0x4C, 0xAA, 0x4D, /* 0x34-0x37 */ + 0xC2, 0xFC, 0xC2, 0xFD, 0xAA, 0x4E, 0xC2, 0xFE, /* 0x38-0x3B */ + 0xC3, 0xA1, 0xC3, 0xA2, 0xC3, 0xA3, 0xAA, 0x4F, /* 0x3C-0x3F */ + 0xAA, 0x50, 0xAA, 0x51, 0xAA, 0x52, 0xAA, 0x53, /* 0x40-0x43 */ + 0xC3, 0xA4, 0xC3, 0xA5, 0xAA, 0x54, 0xAA, 0x55, /* 0x44-0x47 */ + 0xC3, 0xA6, 0xAA, 0x56, 0xAA, 0x57, 0xAA, 0x58, /* 0x48-0x4B */ + 0xC3, 0xA7, 0xAA, 0x59, 0xAA, 0x5A, 0xAA, 0x61, /* 0x4C-0x4F */ + 0xAA, 0x62, 0xAA, 0x63, 0xAA, 0x64, 0xAA, 0x65, /* 0x50-0x53 */ + 0xC3, 0xA8, 0xC3, 0xA9, 0xAA, 0x66, 0xC3, 0xAA, /* 0x54-0x57 */ + 0xC3, 0xAB, 0xC3, 0xAC, 0xAA, 0x67, 0xAA, 0x68, /* 0x58-0x5B */ + 0xAA, 0x69, 0xAA, 0x6A, 0xAA, 0x6B, 0xAA, 0x6C, /* 0x5C-0x5F */ + 0xC3, 0xAD, 0xAA, 0x6D, 0xAA, 0x6E, 0xAA, 0x6F, /* 0x60-0x63 */ + 0xC3, 0xAE, 0xAA, 0x70, 0xC3, 0xAF, 0xAA, 0x71, /* 0x64-0x67 */ + 0xC3, 0xB0, 0xAA, 0x72, 0xAA, 0x73, 0xAA, 0x74, /* 0x68-0x6B */ + 0xAA, 0x75, 0xAA, 0x76, 0xAA, 0x77, 0xAA, 0x78, /* 0x6C-0x6F */ + 0xC3, 0xB1, 0xAA, 0x79, 0xAA, 0x7A, 0xAA, 0x81, /* 0x70-0x73 */ + 0xAA, 0x82, 0xC3, 0xB2, 0xAA, 0x83, 0xAA, 0x84, /* 0x74-0x77 */ + 0xAA, 0x85, 0xAA, 0x86, 0xAA, 0x87, 0xAA, 0x88, /* 0x78-0x7B */ + 0xAA, 0x89, 0xAA, 0x8A, 0xAA, 0x8B, 0xAA, 0x8C, /* 0x7C-0x7F */ + + 0xAA, 0x8D, 0xAA, 0x8E, 0xAA, 0x8F, 0xAA, 0x90, /* 0x80-0x83 */ + 0xAA, 0x91, 0xAA, 0x92, 0xAA, 0x93, 0xAA, 0x94, /* 0x84-0x87 */ + 0xAA, 0x95, 0xAA, 0x96, 0xAA, 0x97, 0xAA, 0x98, /* 0x88-0x8B */ + 0xAA, 0x99, 0xAA, 0x9A, 0xAA, 0x9B, 0xAA, 0x9C, /* 0x8C-0x8F */ + 0xAA, 0x9D, 0xAA, 0x9E, 0xAA, 0x9F, 0xAA, 0xA0, /* 0x90-0x93 */ + 0xAB, 0x41, 0xAB, 0x42, 0xAB, 0x43, 0xAB, 0x44, /* 0x94-0x97 */ + 0xC3, 0xB3, 0xC3, 0xB4, 0xAB, 0x45, 0xAB, 0x46, /* 0x98-0x9B */ + 0xC3, 0xB5, 0xAB, 0x47, 0xAB, 0x48, 0xAB, 0x49, /* 0x9C-0x9F */ + 0xC3, 0xB6, 0xAB, 0x4A, 0xAB, 0x4B, 0xAB, 0x4C, /* 0xA0-0xA3 */ + 0xAB, 0x4D, 0xAB, 0x4E, 0xAB, 0x4F, 0xAB, 0x50, /* 0xA4-0xA7 */ + 0xC3, 0xB7, 0xC3, 0xB8, 0xAB, 0x51, 0xC3, 0xB9, /* 0xA8-0xAB */ + 0xC3, 0xBA, 0xC3, 0xBB, 0xAB, 0x52, 0xAB, 0x53, /* 0xAC-0xAF */ + 0xAB, 0x54, 0xAB, 0x55, 0xAB, 0x56, 0xAB, 0x57, /* 0xB0-0xB3 */ + 0xC3, 0xBC, 0xC3, 0xBD, 0xAB, 0x58, 0xAB, 0x59, /* 0xB4-0xB7 */ + 0xC3, 0xBE, 0xAB, 0x5A, 0xAB, 0x61, 0xAB, 0x62, /* 0xB8-0xBB */ + 0xC3, 0xBF, 0xAB, 0x63, 0xAB, 0x64, 0xAB, 0x65, /* 0xBC-0xBF */ + 0xAB, 0x66, 0xAB, 0x67, 0xAB, 0x68, 0xAB, 0x69, /* 0xC0-0xC3 */ + 0xC3, 0xC0, 0xC3, 0xC1, 0xAB, 0x6A, 0xC3, 0xC2, /* 0xC4-0xC7 */ + 0xAB, 0x6B, 0xC3, 0xC3, 0xAB, 0x6C, 0xAB, 0x6D, /* 0xC8-0xCB */ + 0xAB, 0x6E, 0xAB, 0x6F, 0xAB, 0x70, 0xAB, 0x71, /* 0xCC-0xCF */ + 0xC3, 0xC4, 0xAB, 0x72, 0xAB, 0x73, 0xAB, 0x74, /* 0xD0-0xD3 */ + 0xC3, 0xC5, 0xAB, 0x75, 0xAB, 0x76, 0xAB, 0x77, /* 0xD4-0xD7 */ + 0xAB, 0x78, 0xAB, 0x79, 0xAB, 0x7A, 0xAB, 0x81, /* 0xD8-0xDB */ + 0xAB, 0x82, 0xAB, 0x83, 0xAB, 0x84, 0xAB, 0x85, /* 0xDC-0xDF */ + 0xAB, 0x86, 0xAB, 0x87, 0xAB, 0x88, 0xAB, 0x89, /* 0xE0-0xE3 */ + 0xC3, 0xC6, 0xAB, 0x8A, 0xAB, 0x8B, 0xAB, 0x8C, /* 0xE4-0xE7 */ + 0xAB, 0x8D, 0xAB, 0x8E, 0xAB, 0x8F, 0xAB, 0x90, /* 0xE8-0xEB */ + 0xC3, 0xC7, 0xAB, 0x91, 0xAB, 0x92, 0xAB, 0x93, /* 0xEC-0xEF */ + 0xC3, 0xC8, 0xAB, 0x94, 0xAB, 0x95, 0xAB, 0x96, /* 0xF0-0xF3 */ + 0xAB, 0x97, 0xAB, 0x98, 0xAB, 0x99, 0xAB, 0x9A, /* 0xF4-0xF7 */ + 0xAB, 0x9B, 0xAB, 0x9C, 0xAB, 0x9D, 0xAB, 0x9E, /* 0xF8-0xFB */ + 0xAB, 0x9F, 0xAB, 0xA0, 0xAC, 0x41, 0xAC, 0x42, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_CD[512] = { + 0xAC, 0x43, 0xC3, 0xC9, 0xAC, 0x44, 0xAC, 0x45, /* 0x00-0x03 */ + 0xAC, 0x46, 0xAC, 0x47, 0xAC, 0x48, 0xAC, 0x49, /* 0x04-0x07 */ + 0xC3, 0xCA, 0xC3, 0xCB, 0xAC, 0x4A, 0xAC, 0x4B, /* 0x08-0x0B */ + 0xC3, 0xCC, 0xAC, 0x4C, 0xAC, 0x4D, 0xAC, 0x4E, /* 0x0C-0x0F */ + 0xC3, 0xCD, 0xAC, 0x4F, 0xAC, 0x50, 0xAC, 0x51, /* 0x10-0x13 */ + 0xAC, 0x52, 0xAC, 0x53, 0xAC, 0x54, 0xAC, 0x55, /* 0x14-0x17 */ + 0xC3, 0xCE, 0xC3, 0xCF, 0xAC, 0x56, 0xC3, 0xD0, /* 0x18-0x1B */ + 0xAC, 0x57, 0xC3, 0xD1, 0xAC, 0x58, 0xAC, 0x59, /* 0x1C-0x1F */ + 0xAC, 0x5A, 0xAC, 0x61, 0xAC, 0x62, 0xAC, 0x63, /* 0x20-0x23 */ + 0xC3, 0xD2, 0xAC, 0x64, 0xAC, 0x65, 0xAC, 0x66, /* 0x24-0x27 */ + 0xC3, 0xD3, 0xAC, 0x67, 0xAC, 0x68, 0xAC, 0x69, /* 0x28-0x2B */ + 0xC3, 0xD4, 0xAC, 0x6A, 0xAC, 0x6B, 0xAC, 0x6C, /* 0x2C-0x2F */ + 0xAC, 0x6D, 0xAC, 0x6E, 0xAC, 0x6F, 0xAC, 0x70, /* 0x30-0x33 */ + 0xAC, 0x71, 0xAC, 0x72, 0xAC, 0x73, 0xAC, 0x74, /* 0x34-0x37 */ + 0xAC, 0x75, 0xC3, 0xD5, 0xAC, 0x76, 0xAC, 0x77, /* 0x38-0x3B */ + 0xAC, 0x78, 0xAC, 0x79, 0xAC, 0x7A, 0xAC, 0x81, /* 0x3C-0x3F */ + 0xAC, 0x82, 0xAC, 0x83, 0xAC, 0x84, 0xAC, 0x85, /* 0x40-0x43 */ + 0xAC, 0x86, 0xAC, 0x87, 0xAC, 0x88, 0xAC, 0x89, /* 0x44-0x47 */ + 0xAC, 0x8A, 0xAC, 0x8B, 0xAC, 0x8C, 0xAC, 0x8D, /* 0x48-0x4B */ + 0xAC, 0x8E, 0xAC, 0x8F, 0xAC, 0x90, 0xAC, 0x91, /* 0x4C-0x4F */ + 0xAC, 0x92, 0xAC, 0x93, 0xAC, 0x94, 0xAC, 0x95, /* 0x50-0x53 */ + 0xAC, 0x96, 0xAC, 0x97, 0xAC, 0x98, 0xAC, 0x99, /* 0x54-0x57 */ + 0xAC, 0x9A, 0xAC, 0x9B, 0xAC, 0x9C, 0xAC, 0x9D, /* 0x58-0x5B */ + 0xC3, 0xD6, 0xAC, 0x9E, 0xAC, 0x9F, 0xAC, 0xA0, /* 0x5C-0x5F */ + 0xC3, 0xD7, 0xAD, 0x41, 0xAD, 0x42, 0xAD, 0x43, /* 0x60-0x63 */ + 0xC3, 0xD8, 0xAD, 0x44, 0xAD, 0x45, 0xAD, 0x46, /* 0x64-0x67 */ + 0xAD, 0x47, 0xAD, 0x48, 0xAD, 0x49, 0xAD, 0x4A, /* 0x68-0x6B */ + 0xC3, 0xD9, 0xC3, 0xDA, 0xAD, 0x4B, 0xC3, 0xDB, /* 0x6C-0x6F */ + 0xAD, 0x4C, 0xC3, 0xDC, 0xAD, 0x4D, 0xAD, 0x4E, /* 0x70-0x73 */ + 0xAD, 0x4F, 0xAD, 0x50, 0xAD, 0x51, 0xAD, 0x52, /* 0x74-0x77 */ + 0xC3, 0xDD, 0xAD, 0x53, 0xAD, 0x54, 0xAD, 0x55, /* 0x78-0x7B */ + 0xAD, 0x56, 0xAD, 0x57, 0xAD, 0x58, 0xAD, 0x59, /* 0x7C-0x7F */ + + 0xAD, 0x5A, 0xAD, 0x61, 0xAD, 0x62, 0xAD, 0x63, /* 0x80-0x83 */ + 0xAD, 0x64, 0xAD, 0x65, 0xAD, 0x66, 0xAD, 0x67, /* 0x84-0x87 */ + 0xC3, 0xDE, 0xAD, 0x68, 0xAD, 0x69, 0xAD, 0x6A, /* 0x88-0x8B */ + 0xAD, 0x6B, 0xAD, 0x6C, 0xAD, 0x6D, 0xAD, 0x6E, /* 0x8C-0x8F */ + 0xAD, 0x6F, 0xAD, 0x70, 0xAD, 0x71, 0xAD, 0x72, /* 0x90-0x93 */ + 0xC3, 0xDF, 0xC3, 0xE0, 0xAD, 0x73, 0xAD, 0x74, /* 0x94-0x97 */ + 0xC3, 0xE1, 0xAD, 0x75, 0xAD, 0x76, 0xAD, 0x77, /* 0x98-0x9B */ + 0xC3, 0xE2, 0xAD, 0x78, 0xAD, 0x79, 0xAD, 0x7A, /* 0x9C-0x9F */ + 0xAD, 0x81, 0xAD, 0x82, 0xAD, 0x83, 0xAD, 0x84, /* 0xA0-0xA3 */ + 0xC3, 0xE3, 0xC3, 0xE4, 0xAD, 0x85, 0xC3, 0xE5, /* 0xA4-0xA7 */ + 0xAD, 0x86, 0xC3, 0xE6, 0xAD, 0x87, 0xAD, 0x88, /* 0xA8-0xAB */ + 0xAD, 0x89, 0xAD, 0x8A, 0xAD, 0x8B, 0xAD, 0x8C, /* 0xAC-0xAF */ + 0xC3, 0xE7, 0xAD, 0x8D, 0xAD, 0x8E, 0xAD, 0x8F, /* 0xB0-0xB3 */ + 0xAD, 0x90, 0xAD, 0x91, 0xAD, 0x92, 0xAD, 0x93, /* 0xB4-0xB7 */ + 0xAD, 0x94, 0xAD, 0x95, 0xAD, 0x96, 0xAD, 0x97, /* 0xB8-0xBB */ + 0xAD, 0x98, 0xAD, 0x99, 0xAD, 0x9A, 0xAD, 0x9B, /* 0xBC-0xBF */ + 0xAD, 0x9C, 0xAD, 0x9D, 0xAD, 0x9E, 0xAD, 0x9F, /* 0xC0-0xC3 */ + 0xC3, 0xE8, 0xAD, 0xA0, 0xAE, 0x41, 0xAE, 0x42, /* 0xC4-0xC7 */ + 0xAE, 0x43, 0xAE, 0x44, 0xAE, 0x45, 0xAE, 0x46, /* 0xC8-0xCB */ + 0xC3, 0xE9, 0xAE, 0x47, 0xAE, 0x48, 0xAE, 0x49, /* 0xCC-0xCF */ + 0xC3, 0xEA, 0xAE, 0x4A, 0xAE, 0x4B, 0xAE, 0x4C, /* 0xD0-0xD3 */ + 0xAE, 0x4D, 0xAE, 0x4E, 0xAE, 0x4F, 0xAE, 0x50, /* 0xD4-0xD7 */ + 0xAE, 0x51, 0xAE, 0x52, 0xAE, 0x53, 0xAE, 0x54, /* 0xD8-0xDB */ + 0xAE, 0x55, 0xAE, 0x56, 0xAE, 0x57, 0xAE, 0x58, /* 0xDC-0xDF */ + 0xAE, 0x59, 0xAE, 0x5A, 0xAE, 0x61, 0xAE, 0x62, /* 0xE0-0xE3 */ + 0xAE, 0x63, 0xAE, 0x64, 0xAE, 0x65, 0xAE, 0x66, /* 0xE4-0xE7 */ + 0xC3, 0xEB, 0xAE, 0x67, 0xAE, 0x68, 0xAE, 0x69, /* 0xE8-0xEB */ + 0xC3, 0xEC, 0xAE, 0x6A, 0xAE, 0x6B, 0xAE, 0x6C, /* 0xEC-0xEF */ + 0xC3, 0xED, 0xAE, 0x6D, 0xAE, 0x6E, 0xAE, 0x6F, /* 0xF0-0xF3 */ + 0xAE, 0x70, 0xAE, 0x71, 0xAE, 0x72, 0xAE, 0x73, /* 0xF4-0xF7 */ + 0xC3, 0xEE, 0xC3, 0xEF, 0xAE, 0x74, 0xC3, 0xF0, /* 0xF8-0xFB */ + 0xAE, 0x75, 0xC3, 0xF1, 0xAE, 0x76, 0xAE, 0x77, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_CE[512] = { + 0xAE, 0x78, 0xAE, 0x79, 0xAE, 0x7A, 0xAE, 0x81, /* 0x00-0x03 */ + 0xC3, 0xF2, 0xAE, 0x82, 0xAE, 0x83, 0xAE, 0x84, /* 0x04-0x07 */ + 0xC3, 0xF3, 0xAE, 0x85, 0xAE, 0x86, 0xAE, 0x87, /* 0x08-0x0B */ + 0xC3, 0xF4, 0xAE, 0x88, 0xAE, 0x89, 0xAE, 0x8A, /* 0x0C-0x0F */ + 0xAE, 0x8B, 0xAE, 0x8C, 0xAE, 0x8D, 0xAE, 0x8E, /* 0x10-0x13 */ + 0xC3, 0xF5, 0xAE, 0x8F, 0xAE, 0x90, 0xAE, 0x91, /* 0x14-0x17 */ + 0xAE, 0x92, 0xC3, 0xF6, 0xAE, 0x93, 0xAE, 0x94, /* 0x18-0x1B */ + 0xAE, 0x95, 0xAE, 0x96, 0xAE, 0x97, 0xAE, 0x98, /* 0x1C-0x1F */ + 0xC3, 0xF7, 0xC3, 0xF8, 0xAE, 0x99, 0xAE, 0x9A, /* 0x20-0x23 */ + 0xC3, 0xF9, 0xAE, 0x9B, 0xAE, 0x9C, 0xAE, 0x9D, /* 0x24-0x27 */ + 0xC3, 0xFA, 0xAE, 0x9E, 0xAE, 0x9F, 0xAE, 0xA0, /* 0x28-0x2B */ + 0xAF, 0x41, 0xAF, 0x42, 0xAF, 0x43, 0xAF, 0x44, /* 0x2C-0x2F */ + 0xC3, 0xFB, 0xC3, 0xFC, 0xAF, 0x45, 0xC3, 0xFD, /* 0x30-0x33 */ + 0xAF, 0x46, 0xC3, 0xFE, 0xAF, 0x47, 0xAF, 0x48, /* 0x34-0x37 */ + 0xAF, 0x49, 0xAF, 0x4A, 0xAF, 0x4B, 0xAF, 0x4C, /* 0x38-0x3B */ + 0xAF, 0x4D, 0xAF, 0x4E, 0xAF, 0x4F, 0xAF, 0x50, /* 0x3C-0x3F */ + 0xAF, 0x51, 0xAF, 0x52, 0xAF, 0x53, 0xAF, 0x54, /* 0x40-0x43 */ + 0xAF, 0x55, 0xAF, 0x56, 0xAF, 0x57, 0xAF, 0x58, /* 0x44-0x47 */ + 0xAF, 0x59, 0xAF, 0x5A, 0xAF, 0x61, 0xAF, 0x62, /* 0x48-0x4B */ + 0xAF, 0x63, 0xAF, 0x64, 0xAF, 0x65, 0xAF, 0x66, /* 0x4C-0x4F */ + 0xAF, 0x67, 0xAF, 0x68, 0xAF, 0x69, 0xAF, 0x6A, /* 0x50-0x53 */ + 0xAF, 0x6B, 0xAF, 0x6C, 0xAF, 0x6D, 0xAF, 0x6E, /* 0x54-0x57 */ + 0xC4, 0xA1, 0xC4, 0xA2, 0xAF, 0x6F, 0xAF, 0x70, /* 0x58-0x5B */ + 0xC4, 0xA3, 0xAF, 0x71, 0xAF, 0x72, 0xC4, 0xA4, /* 0x5C-0x5F */ + 0xC4, 0xA5, 0xC4, 0xA6, 0xAF, 0x73, 0xAF, 0x74, /* 0x60-0x63 */ + 0xAF, 0x75, 0xAF, 0x76, 0xAF, 0x77, 0xAF, 0x78, /* 0x64-0x67 */ + 0xC4, 0xA7, 0xC4, 0xA8, 0xAF, 0x79, 0xC4, 0xA9, /* 0x68-0x6B */ + 0xAF, 0x7A, 0xC4, 0xAA, 0xAF, 0x81, 0xAF, 0x82, /* 0x6C-0x6F */ + 0xAF, 0x83, 0xAF, 0x84, 0xAF, 0x85, 0xAF, 0x86, /* 0x70-0x73 */ + 0xC4, 0xAB, 0xC4, 0xAC, 0xAF, 0x87, 0xAF, 0x88, /* 0x74-0x77 */ + 0xC4, 0xAD, 0xAF, 0x89, 0xAF, 0x8A, 0xAF, 0x8B, /* 0x78-0x7B */ + 0xC4, 0xAE, 0xAF, 0x8C, 0xAF, 0x8D, 0xAF, 0x8E, /* 0x7C-0x7F */ + + 0xAF, 0x8F, 0xAF, 0x90, 0xAF, 0x91, 0xAF, 0x92, /* 0x80-0x83 */ + 0xC4, 0xAF, 0xC4, 0xB0, 0xAF, 0x93, 0xC4, 0xB1, /* 0x84-0x87 */ + 0xAF, 0x94, 0xC4, 0xB2, 0xAF, 0x95, 0xAF, 0x96, /* 0x88-0x8B */ + 0xAF, 0x97, 0xAF, 0x98, 0xAF, 0x99, 0xAF, 0x9A, /* 0x8C-0x8F */ + 0xC4, 0xB3, 0xC4, 0xB4, 0xAF, 0x9B, 0xAF, 0x9C, /* 0x90-0x93 */ + 0xC4, 0xB5, 0xAF, 0x9D, 0xAF, 0x9E, 0xAF, 0x9F, /* 0x94-0x97 */ + 0xC4, 0xB6, 0xAF, 0xA0, 0xB0, 0x41, 0xB0, 0x42, /* 0x98-0x9B */ + 0xB0, 0x43, 0xB0, 0x44, 0xB0, 0x45, 0xB0, 0x46, /* 0x9C-0x9F */ + 0xC4, 0xB7, 0xC4, 0xB8, 0xB0, 0x47, 0xC4, 0xB9, /* 0xA0-0xA3 */ + 0xC4, 0xBA, 0xC4, 0xBB, 0xB0, 0x48, 0xB0, 0x49, /* 0xA4-0xA7 */ + 0xB0, 0x4A, 0xB0, 0x4B, 0xB0, 0x4C, 0xB0, 0x4D, /* 0xA8-0xAB */ + 0xC4, 0xBC, 0xC4, 0xBD, 0xB0, 0x4E, 0xB0, 0x4F, /* 0xAC-0xAF */ + 0xB0, 0x50, 0xB0, 0x51, 0xB0, 0x52, 0xB0, 0x53, /* 0xB0-0xB3 */ + 0xB0, 0x54, 0xB0, 0x55, 0xB0, 0x56, 0xB0, 0x57, /* 0xB4-0xB7 */ + 0xB0, 0x58, 0xB0, 0x59, 0xB0, 0x5A, 0xB0, 0x61, /* 0xB8-0xBB */ + 0xB0, 0x62, 0xB0, 0x63, 0xB0, 0x64, 0xB0, 0x65, /* 0xBC-0xBF */ + 0xB0, 0x66, 0xC4, 0xBE, 0xB0, 0x67, 0xB0, 0x68, /* 0xC0-0xC3 */ + 0xB0, 0x69, 0xB0, 0x6A, 0xB0, 0x6B, 0xB0, 0x6C, /* 0xC4-0xC7 */ + 0xB0, 0x6D, 0xB0, 0x6E, 0xB0, 0x6F, 0xB0, 0x70, /* 0xC8-0xCB */ + 0xB0, 0x71, 0xB0, 0x72, 0xB0, 0x73, 0xB0, 0x74, /* 0xCC-0xCF */ + 0xB0, 0x75, 0xB0, 0x76, 0xB0, 0x77, 0xB0, 0x78, /* 0xD0-0xD3 */ + 0xB0, 0x79, 0xB0, 0x7A, 0xB0, 0x81, 0xB0, 0x82, /* 0xD4-0xD7 */ + 0xB0, 0x83, 0xB0, 0x84, 0xB0, 0x85, 0xB0, 0x86, /* 0xD8-0xDB */ + 0xB0, 0x87, 0xB0, 0x88, 0xB0, 0x89, 0xB0, 0x8A, /* 0xDC-0xDF */ + 0xB0, 0x8B, 0xB0, 0x8C, 0xB0, 0x8D, 0xB0, 0x8E, /* 0xE0-0xE3 */ + 0xC4, 0xBF, 0xC4, 0xC0, 0xB0, 0x8F, 0xB0, 0x90, /* 0xE4-0xE7 */ + 0xC4, 0xC1, 0xB0, 0x91, 0xB0, 0x92, 0xC4, 0xC2, /* 0xE8-0xEB */ + 0xC4, 0xC3, 0xB0, 0x93, 0xB0, 0x94, 0xB0, 0x95, /* 0xEC-0xEF */ + 0xB0, 0x96, 0xB0, 0x97, 0xB0, 0x98, 0xB0, 0x99, /* 0xF0-0xF3 */ + 0xC4, 0xC4, 0xC4, 0xC5, 0xB0, 0x9A, 0xC4, 0xC6, /* 0xF4-0xF7 */ + 0xC4, 0xC7, 0xC4, 0xC8, 0xB0, 0x9B, 0xB0, 0x9C, /* 0xF8-0xFB */ + 0xB0, 0x9D, 0xB0, 0x9E, 0xB0, 0x9F, 0xB0, 0xA0, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_CF[512] = { + 0xC4, 0xC9, 0xC4, 0xCA, 0xB1, 0x41, 0xB1, 0x42, /* 0x00-0x03 */ + 0xC4, 0xCB, 0xB1, 0x43, 0xB1, 0x44, 0xB1, 0x45, /* 0x04-0x07 */ + 0xC4, 0xCC, 0xB1, 0x46, 0xB1, 0x47, 0xB1, 0x48, /* 0x08-0x0B */ + 0xB1, 0x49, 0xB1, 0x4A, 0xB1, 0x4B, 0xB1, 0x4C, /* 0x0C-0x0F */ + 0xC4, 0xCD, 0xC4, 0xCE, 0xB1, 0x4D, 0xC4, 0xCF, /* 0x10-0x13 */ + 0xB1, 0x4E, 0xC4, 0xD0, 0xB1, 0x4F, 0xB1, 0x50, /* 0x14-0x17 */ + 0xB1, 0x51, 0xB1, 0x52, 0xB1, 0x53, 0xB1, 0x54, /* 0x18-0x1B */ + 0xC4, 0xD1, 0xB1, 0x55, 0xB1, 0x56, 0xB1, 0x57, /* 0x1C-0x1F */ + 0xC4, 0xD2, 0xB1, 0x58, 0xB1, 0x59, 0xB1, 0x5A, /* 0x20-0x23 */ + 0xC4, 0xD3, 0xB1, 0x61, 0xB1, 0x62, 0xB1, 0x63, /* 0x24-0x27 */ + 0xB1, 0x64, 0xB1, 0x65, 0xB1, 0x66, 0xB1, 0x67, /* 0x28-0x2B */ + 0xC4, 0xD4, 0xC4, 0xD5, 0xB1, 0x68, 0xC4, 0xD6, /* 0x2C-0x2F */ + 0xC4, 0xD7, 0xC4, 0xD8, 0xB1, 0x69, 0xB1, 0x6A, /* 0x30-0x33 */ + 0xB1, 0x6B, 0xB1, 0x6C, 0xB1, 0x6D, 0xB1, 0x6E, /* 0x34-0x37 */ + 0xC4, 0xD9, 0xB1, 0x6F, 0xB1, 0x70, 0xB1, 0x71, /* 0x38-0x3B */ + 0xB1, 0x72, 0xB1, 0x73, 0xB1, 0x74, 0xB1, 0x75, /* 0x3C-0x3F */ + 0xB1, 0x76, 0xB1, 0x77, 0xB1, 0x78, 0xB1, 0x79, /* 0x40-0x43 */ + 0xB1, 0x7A, 0xB1, 0x81, 0xB1, 0x82, 0xB1, 0x83, /* 0x44-0x47 */ + 0xB1, 0x84, 0xB1, 0x85, 0xB1, 0x86, 0xB1, 0x87, /* 0x48-0x4B */ + 0xB1, 0x88, 0xB1, 0x89, 0xB1, 0x8A, 0xB1, 0x8B, /* 0x4C-0x4F */ + 0xB1, 0x8C, 0xB1, 0x8D, 0xB1, 0x8E, 0xB1, 0x8F, /* 0x50-0x53 */ + 0xC4, 0xDA, 0xC4, 0xDB, 0xB1, 0x90, 0xB1, 0x91, /* 0x54-0x57 */ + 0xC4, 0xDC, 0xB1, 0x92, 0xB1, 0x93, 0xB1, 0x94, /* 0x58-0x5B */ + 0xC4, 0xDD, 0xB1, 0x95, 0xB1, 0x96, 0xB1, 0x97, /* 0x5C-0x5F */ + 0xB1, 0x98, 0xB1, 0x99, 0xB1, 0x9A, 0xB1, 0x9B, /* 0x60-0x63 */ + 0xC4, 0xDE, 0xC4, 0xDF, 0xB1, 0x9C, 0xC4, 0xE0, /* 0x64-0x67 */ + 0xB1, 0x9D, 0xC4, 0xE1, 0xB1, 0x9E, 0xB1, 0x9F, /* 0x68-0x6B */ + 0xB1, 0xA0, 0xB2, 0x41, 0xB2, 0x42, 0xB2, 0x43, /* 0x6C-0x6F */ + 0xC4, 0xE2, 0xC4, 0xE3, 0xB2, 0x44, 0xB2, 0x45, /* 0x70-0x73 */ + 0xC4, 0xE4, 0xB2, 0x46, 0xB2, 0x47, 0xB2, 0x48, /* 0x74-0x77 */ + 0xC4, 0xE5, 0xB2, 0x49, 0xB2, 0x4A, 0xB2, 0x4B, /* 0x78-0x7B */ + 0xB2, 0x4C, 0xB2, 0x4D, 0xB2, 0x4E, 0xB2, 0x4F, /* 0x7C-0x7F */ + + 0xC4, 0xE6, 0xB2, 0x50, 0xB2, 0x51, 0xB2, 0x52, /* 0x80-0x83 */ + 0xB2, 0x53, 0xC4, 0xE7, 0xB2, 0x54, 0xB2, 0x55, /* 0x84-0x87 */ + 0xB2, 0x56, 0xB2, 0x57, 0xB2, 0x58, 0xB2, 0x59, /* 0x88-0x8B */ + 0xC4, 0xE8, 0xB2, 0x5A, 0xB2, 0x61, 0xB2, 0x62, /* 0x8C-0x8F */ + 0xB2, 0x63, 0xB2, 0x64, 0xB2, 0x65, 0xB2, 0x66, /* 0x90-0x93 */ + 0xB2, 0x67, 0xB2, 0x68, 0xB2, 0x69, 0xB2, 0x6A, /* 0x94-0x97 */ + 0xB2, 0x6B, 0xB2, 0x6C, 0xB2, 0x6D, 0xB2, 0x6E, /* 0x98-0x9B */ + 0xB2, 0x6F, 0xB2, 0x70, 0xB2, 0x71, 0xB2, 0x72, /* 0x9C-0x9F */ + 0xB2, 0x73, 0xC4, 0xE9, 0xB2, 0x74, 0xB2, 0x75, /* 0xA0-0xA3 */ + 0xB2, 0x76, 0xB2, 0x77, 0xB2, 0x78, 0xB2, 0x79, /* 0xA4-0xA7 */ + 0xC4, 0xEA, 0xB2, 0x7A, 0xB2, 0x81, 0xB2, 0x82, /* 0xA8-0xAB */ + 0xB2, 0x83, 0xB2, 0x84, 0xB2, 0x85, 0xB2, 0x86, /* 0xAC-0xAF */ + 0xC4, 0xEB, 0xB2, 0x87, 0xB2, 0x88, 0xB2, 0x89, /* 0xB0-0xB3 */ + 0xB2, 0x8A, 0xB2, 0x8B, 0xB2, 0x8C, 0xB2, 0x8D, /* 0xB4-0xB7 */ + 0xB2, 0x8E, 0xB2, 0x8F, 0xB2, 0x90, 0xB2, 0x91, /* 0xB8-0xBB */ + 0xB2, 0x92, 0xB2, 0x93, 0xB2, 0x94, 0xB2, 0x95, /* 0xBC-0xBF */ + 0xB2, 0x96, 0xB2, 0x97, 0xB2, 0x98, 0xB2, 0x99, /* 0xC0-0xC3 */ + 0xC4, 0xEC, 0xB2, 0x9A, 0xB2, 0x9B, 0xB2, 0x9C, /* 0xC4-0xC7 */ + 0xB2, 0x9D, 0xB2, 0x9E, 0xB2, 0x9F, 0xB2, 0xA0, /* 0xC8-0xCB */ + 0xB3, 0x41, 0xB3, 0x42, 0xB3, 0x43, 0xB3, 0x44, /* 0xCC-0xCF */ + 0xB3, 0x45, 0xB3, 0x46, 0xB3, 0x47, 0xB3, 0x48, /* 0xD0-0xD3 */ + 0xB3, 0x49, 0xB3, 0x4A, 0xB3, 0x4B, 0xB3, 0x4C, /* 0xD4-0xD7 */ + 0xB3, 0x4D, 0xB3, 0x4E, 0xB3, 0x4F, 0xB3, 0x50, /* 0xD8-0xDB */ + 0xB3, 0x51, 0xB3, 0x52, 0xB3, 0x53, 0xB3, 0x54, /* 0xDC-0xDF */ + 0xC4, 0xED, 0xC4, 0xEE, 0xB3, 0x55, 0xB3, 0x56, /* 0xE0-0xE3 */ + 0xC4, 0xEF, 0xB3, 0x57, 0xB3, 0x58, 0xB3, 0x59, /* 0xE4-0xE7 */ + 0xC4, 0xF0, 0xB3, 0x5A, 0xB3, 0x61, 0xB3, 0x62, /* 0xE8-0xEB */ + 0xB3, 0x63, 0xB3, 0x64, 0xB3, 0x65, 0xB3, 0x66, /* 0xEC-0xEF */ + 0xC4, 0xF1, 0xC4, 0xF2, 0xB3, 0x67, 0xC4, 0xF3, /* 0xF0-0xF3 */ + 0xB3, 0x68, 0xC4, 0xF4, 0xB3, 0x69, 0xB3, 0x6A, /* 0xF4-0xF7 */ + 0xB3, 0x6B, 0xB3, 0x6C, 0xB3, 0x6D, 0xB3, 0x6E, /* 0xF8-0xFB */ + 0xC4, 0xF5, 0xB3, 0x6F, 0xB3, 0x70, 0xB3, 0x71, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_D0[512] = { + 0xC4, 0xF6, 0xB3, 0x72, 0xB3, 0x73, 0xB3, 0x74, /* 0x00-0x03 */ + 0xC4, 0xF7, 0xB3, 0x75, 0xB3, 0x76, 0xB3, 0x77, /* 0x04-0x07 */ + 0xB3, 0x78, 0xB3, 0x79, 0xB3, 0x7A, 0xB3, 0x81, /* 0x08-0x0B */ + 0xB3, 0x82, 0xB3, 0x83, 0xB3, 0x84, 0xB3, 0x85, /* 0x0C-0x0F */ + 0xB3, 0x86, 0xC4, 0xF8, 0xB3, 0x87, 0xB3, 0x88, /* 0x10-0x13 */ + 0xB3, 0x89, 0xB3, 0x8A, 0xB3, 0x8B, 0xB3, 0x8C, /* 0x14-0x17 */ + 0xC4, 0xF9, 0xB3, 0x8D, 0xB3, 0x8E, 0xB3, 0x8F, /* 0x18-0x1B */ + 0xB3, 0x90, 0xB3, 0x91, 0xB3, 0x92, 0xB3, 0x93, /* 0x1C-0x1F */ + 0xB3, 0x94, 0xB3, 0x95, 0xB3, 0x96, 0xB3, 0x97, /* 0x20-0x23 */ + 0xB3, 0x98, 0xB3, 0x99, 0xB3, 0x9A, 0xB3, 0x9B, /* 0x24-0x27 */ + 0xB3, 0x9C, 0xB3, 0x9D, 0xB3, 0x9E, 0xB3, 0x9F, /* 0x28-0x2B */ + 0xB3, 0xA0, 0xC4, 0xFA, 0xB4, 0x41, 0xB4, 0x42, /* 0x2C-0x2F */ + 0xB4, 0x43, 0xB4, 0x44, 0xB4, 0x45, 0xB4, 0x46, /* 0x30-0x33 */ + 0xC4, 0xFB, 0xC4, 0xFC, 0xB4, 0x47, 0xB4, 0x48, /* 0x34-0x37 */ + 0xC4, 0xFD, 0xB4, 0x49, 0xB4, 0x4A, 0xB4, 0x4B, /* 0x38-0x3B */ + 0xC4, 0xFE, 0xB4, 0x4C, 0xB4, 0x4D, 0xB4, 0x4E, /* 0x3C-0x3F */ + 0xB4, 0x4F, 0xB4, 0x50, 0xB4, 0x51, 0xB4, 0x52, /* 0x40-0x43 */ + 0xC5, 0xA1, 0xC5, 0xA2, 0xB4, 0x53, 0xC5, 0xA3, /* 0x44-0x47 */ + 0xB4, 0x54, 0xC5, 0xA4, 0xB4, 0x55, 0xB4, 0x56, /* 0x48-0x4B */ + 0xB4, 0x57, 0xB4, 0x58, 0xB4, 0x59, 0xB4, 0x5A, /* 0x4C-0x4F */ + 0xC5, 0xA5, 0xB4, 0x61, 0xB4, 0x62, 0xB4, 0x63, /* 0x50-0x53 */ + 0xC5, 0xA6, 0xB4, 0x64, 0xB4, 0x65, 0xB4, 0x66, /* 0x54-0x57 */ + 0xC5, 0xA7, 0xB4, 0x67, 0xB4, 0x68, 0xB4, 0x69, /* 0x58-0x5B */ + 0xB4, 0x6A, 0xB4, 0x6B, 0xB4, 0x6C, 0xB4, 0x6D, /* 0x5C-0x5F */ + 0xC5, 0xA8, 0xB4, 0x6E, 0xB4, 0x6F, 0xB4, 0x70, /* 0x60-0x63 */ + 0xB4, 0x71, 0xB4, 0x72, 0xB4, 0x73, 0xB4, 0x74, /* 0x64-0x67 */ + 0xB4, 0x75, 0xB4, 0x76, 0xB4, 0x77, 0xB4, 0x78, /* 0x68-0x6B */ + 0xC5, 0xA9, 0xC5, 0xAA, 0xB4, 0x79, 0xB4, 0x7A, /* 0x6C-0x6F */ + 0xC5, 0xAB, 0xB4, 0x81, 0xB4, 0x82, 0xB4, 0x83, /* 0x70-0x73 */ + 0xC5, 0xAC, 0xB4, 0x84, 0xB4, 0x85, 0xB4, 0x86, /* 0x74-0x77 */ + 0xB4, 0x87, 0xB4, 0x88, 0xB4, 0x89, 0xB4, 0x8A, /* 0x78-0x7B */ + 0xC5, 0xAD, 0xC5, 0xAE, 0xB4, 0x8B, 0xB4, 0x8C, /* 0x7C-0x7F */ + + 0xB4, 0x8D, 0xC5, 0xAF, 0xB4, 0x8E, 0xB4, 0x8F, /* 0x80-0x83 */ + 0xB4, 0x90, 0xB4, 0x91, 0xB4, 0x92, 0xB4, 0x93, /* 0x84-0x87 */ + 0xB4, 0x94, 0xB4, 0x95, 0xB4, 0x96, 0xB4, 0x97, /* 0x88-0x8B */ + 0xB4, 0x98, 0xB4, 0x99, 0xB4, 0x9A, 0xB4, 0x9B, /* 0x8C-0x8F */ + 0xB4, 0x9C, 0xB4, 0x9D, 0xB4, 0x9E, 0xB4, 0x9F, /* 0x90-0x93 */ + 0xB4, 0xA0, 0xB5, 0x41, 0xB5, 0x42, 0xB5, 0x43, /* 0x94-0x97 */ + 0xB5, 0x44, 0xB5, 0x45, 0xB5, 0x46, 0xB5, 0x47, /* 0x98-0x9B */ + 0xB5, 0x48, 0xB5, 0x49, 0xB5, 0x4A, 0xB5, 0x4B, /* 0x9C-0x9F */ + 0xB5, 0x4C, 0xB5, 0x4D, 0xB5, 0x4E, 0xB5, 0x4F, /* 0xA0-0xA3 */ + 0xC5, 0xB0, 0xC5, 0xB1, 0xB5, 0x50, 0xB5, 0x51, /* 0xA4-0xA7 */ + 0xC5, 0xB2, 0xB5, 0x52, 0xB5, 0x53, 0xB5, 0x54, /* 0xA8-0xAB */ + 0xC5, 0xB3, 0xB5, 0x55, 0xB5, 0x56, 0xB5, 0x57, /* 0xAC-0xAF */ + 0xB5, 0x58, 0xB5, 0x59, 0xB5, 0x5A, 0xB5, 0x61, /* 0xB0-0xB3 */ + 0xC5, 0xB4, 0xC5, 0xB5, 0xB5, 0x62, 0xC5, 0xB6, /* 0xB4-0xB7 */ + 0xB5, 0x63, 0xC5, 0xB7, 0xB5, 0x64, 0xB5, 0x65, /* 0xB8-0xBB */ + 0xB5, 0x66, 0xB5, 0x67, 0xB5, 0x68, 0xB5, 0x69, /* 0xBC-0xBF */ + 0xC5, 0xB8, 0xC5, 0xB9, 0xB5, 0x6A, 0xB5, 0x6B, /* 0xC0-0xC3 */ + 0xC5, 0xBA, 0xB5, 0x6C, 0xB5, 0x6D, 0xB5, 0x6E, /* 0xC4-0xC7 */ + 0xC5, 0xBB, 0xC5, 0xBC, 0xB5, 0x6F, 0xB5, 0x70, /* 0xC8-0xCB */ + 0xB5, 0x71, 0xB5, 0x72, 0xB5, 0x73, 0xB5, 0x74, /* 0xCC-0xCF */ + 0xC5, 0xBD, 0xC5, 0xBE, 0xB5, 0x75, 0xC5, 0xBF, /* 0xD0-0xD3 */ + 0xC5, 0xC0, 0xC5, 0xC1, 0xB5, 0x76, 0xB5, 0x77, /* 0xD4-0xD7 */ + 0xB5, 0x78, 0xB5, 0x79, 0xB5, 0x7A, 0xB5, 0x81, /* 0xD8-0xDB */ + 0xC5, 0xC2, 0xC5, 0xC3, 0xB5, 0x82, 0xB5, 0x83, /* 0xDC-0xDF */ + 0xC5, 0xC4, 0xB5, 0x84, 0xB5, 0x85, 0xB5, 0x86, /* 0xE0-0xE3 */ + 0xC5, 0xC5, 0xB5, 0x87, 0xB5, 0x88, 0xB5, 0x89, /* 0xE4-0xE7 */ + 0xB5, 0x8A, 0xB5, 0x8B, 0xB5, 0x8C, 0xB5, 0x8D, /* 0xE8-0xEB */ + 0xC5, 0xC6, 0xC5, 0xC7, 0xB5, 0x8E, 0xC5, 0xC8, /* 0xEC-0xEF */ + 0xC5, 0xC9, 0xC5, 0xCA, 0xB5, 0x8F, 0xB5, 0x90, /* 0xF0-0xF3 */ + 0xB5, 0x91, 0xB5, 0x92, 0xB5, 0x93, 0xB5, 0x94, /* 0xF4-0xF7 */ + 0xC5, 0xCB, 0xB5, 0x95, 0xB5, 0x96, 0xB5, 0x97, /* 0xF8-0xFB */ + 0xB5, 0x98, 0xB5, 0x99, 0xB5, 0x9A, 0xB5, 0x9B, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_D1[512] = { + 0xB5, 0x9C, 0xB5, 0x9D, 0xB5, 0x9E, 0xB5, 0x9F, /* 0x00-0x03 */ + 0xB5, 0xA0, 0xB6, 0x41, 0xB6, 0x42, 0xB6, 0x43, /* 0x04-0x07 */ + 0xB6, 0x44, 0xB6, 0x45, 0xB6, 0x46, 0xB6, 0x47, /* 0x08-0x0B */ + 0xB6, 0x48, 0xC5, 0xCC, 0xB6, 0x49, 0xB6, 0x4A, /* 0x0C-0x0F */ + 0xB6, 0x4B, 0xB6, 0x4C, 0xB6, 0x4D, 0xB6, 0x4E, /* 0x10-0x13 */ + 0xB6, 0x4F, 0xB6, 0x50, 0xB6, 0x51, 0xB6, 0x52, /* 0x14-0x17 */ + 0xB6, 0x53, 0xB6, 0x54, 0xB6, 0x55, 0xB6, 0x56, /* 0x18-0x1B */ + 0xB6, 0x57, 0xB6, 0x58, 0xB6, 0x59, 0xB6, 0x5A, /* 0x1C-0x1F */ + 0xB6, 0x61, 0xB6, 0x62, 0xB6, 0x63, 0xB6, 0x64, /* 0x20-0x23 */ + 0xB6, 0x65, 0xB6, 0x66, 0xB6, 0x67, 0xB6, 0x68, /* 0x24-0x27 */ + 0xB6, 0x69, 0xB6, 0x6A, 0xB6, 0x6B, 0xB6, 0x6C, /* 0x28-0x2B */ + 0xB6, 0x6D, 0xB6, 0x6E, 0xB6, 0x6F, 0xB6, 0x70, /* 0x2C-0x2F */ + 0xC5, 0xCD, 0xC5, 0xCE, 0xB6, 0x71, 0xB6, 0x72, /* 0x30-0x33 */ + 0xC5, 0xCF, 0xB6, 0x73, 0xB6, 0x74, 0xB6, 0x75, /* 0x34-0x37 */ + 0xC5, 0xD0, 0xB6, 0x76, 0xC5, 0xD1, 0xB6, 0x77, /* 0x38-0x3B */ + 0xB6, 0x78, 0xB6, 0x79, 0xB6, 0x7A, 0xB6, 0x81, /* 0x3C-0x3F */ + 0xC5, 0xD2, 0xC5, 0xD3, 0xB6, 0x82, 0xC5, 0xD4, /* 0x40-0x43 */ + 0xC5, 0xD5, 0xC5, 0xD6, 0xB6, 0x83, 0xB6, 0x84, /* 0x44-0x47 */ + 0xB6, 0x85, 0xB6, 0x86, 0xB6, 0x87, 0xB6, 0x88, /* 0x48-0x4B */ + 0xC5, 0xD7, 0xC5, 0xD8, 0xB6, 0x89, 0xB6, 0x8A, /* 0x4C-0x4F */ + 0xC5, 0xD9, 0xB6, 0x8B, 0xB6, 0x8C, 0xB6, 0x8D, /* 0x50-0x53 */ + 0xC5, 0xDA, 0xB6, 0x8E, 0xB6, 0x8F, 0xB6, 0x90, /* 0x54-0x57 */ + 0xB6, 0x91, 0xB6, 0x92, 0xB6, 0x93, 0xB6, 0x94, /* 0x58-0x5B */ + 0xC5, 0xDB, 0xC5, 0xDC, 0xB6, 0x95, 0xC5, 0xDD, /* 0x5C-0x5F */ + 0xB6, 0x96, 0xC5, 0xDE, 0xB6, 0x97, 0xB6, 0x98, /* 0x60-0x63 */ + 0xB6, 0x99, 0xB6, 0x9A, 0xB6, 0x9B, 0xB6, 0x9C, /* 0x64-0x67 */ + 0xC5, 0xDF, 0xB6, 0x9D, 0xB6, 0x9E, 0xB6, 0x9F, /* 0x68-0x6B */ + 0xC5, 0xE0, 0xB6, 0xA0, 0xB7, 0x41, 0xB7, 0x42, /* 0x6C-0x6F */ + 0xB7, 0x43, 0xB7, 0x44, 0xB7, 0x45, 0xB7, 0x46, /* 0x70-0x73 */ + 0xB7, 0x47, 0xB7, 0x48, 0xB7, 0x49, 0xB7, 0x4A, /* 0x74-0x77 */ + 0xB7, 0x4B, 0xB7, 0x4C, 0xB7, 0x4D, 0xB7, 0x4E, /* 0x78-0x7B */ + 0xC5, 0xE1, 0xB7, 0x4F, 0xB7, 0x50, 0xB7, 0x51, /* 0x7C-0x7F */ + + 0xB7, 0x52, 0xB7, 0x53, 0xB7, 0x54, 0xB7, 0x55, /* 0x80-0x83 */ + 0xC5, 0xE2, 0xB7, 0x56, 0xB7, 0x57, 0xB7, 0x58, /* 0x84-0x87 */ + 0xC5, 0xE3, 0xB7, 0x59, 0xB7, 0x5A, 0xB7, 0x61, /* 0x88-0x8B */ + 0xB7, 0x62, 0xB7, 0x63, 0xB7, 0x64, 0xB7, 0x65, /* 0x8C-0x8F */ + 0xB7, 0x66, 0xB7, 0x67, 0xB7, 0x68, 0xB7, 0x69, /* 0x90-0x93 */ + 0xB7, 0x6A, 0xB7, 0x6B, 0xB7, 0x6C, 0xB7, 0x6D, /* 0x94-0x97 */ + 0xB7, 0x6E, 0xB7, 0x6F, 0xB7, 0x70, 0xB7, 0x71, /* 0x98-0x9B */ + 0xB7, 0x72, 0xB7, 0x73, 0xB7, 0x74, 0xB7, 0x75, /* 0x9C-0x9F */ + 0xC5, 0xE4, 0xC5, 0xE5, 0xB7, 0x76, 0xB7, 0x77, /* 0xA0-0xA3 */ + 0xC5, 0xE6, 0xB7, 0x78, 0xB7, 0x79, 0xB7, 0x7A, /* 0xA4-0xA7 */ + 0xC5, 0xE7, 0xB7, 0x81, 0xB7, 0x82, 0xB7, 0x83, /* 0xA8-0xAB */ + 0xB7, 0x84, 0xB7, 0x85, 0xB7, 0x86, 0xB7, 0x87, /* 0xAC-0xAF */ + 0xC5, 0xE8, 0xC5, 0xE9, 0xB7, 0x88, 0xC5, 0xEA, /* 0xB0-0xB3 */ + 0xB7, 0x89, 0xC5, 0xEB, 0xB7, 0x8A, 0xB7, 0x8B, /* 0xB4-0xB7 */ + 0xB7, 0x8C, 0xB7, 0x8D, 0xC5, 0xEC, 0xB7, 0x8E, /* 0xB8-0xBB */ + 0xC5, 0xED, 0xB7, 0x8F, 0xB7, 0x90, 0xB7, 0x91, /* 0xBC-0xBF */ + 0xC5, 0xEE, 0xB7, 0x92, 0xB7, 0x93, 0xB7, 0x94, /* 0xC0-0xC3 */ + 0xB7, 0x95, 0xB7, 0x96, 0xB7, 0x97, 0xB7, 0x98, /* 0xC4-0xC7 */ + 0xB7, 0x99, 0xB7, 0x9A, 0xB7, 0x9B, 0xB7, 0x9C, /* 0xC8-0xCB */ + 0xB7, 0x9D, 0xB7, 0x9E, 0xB7, 0x9F, 0xB7, 0xA0, /* 0xCC-0xCF */ + 0xB8, 0x41, 0xB8, 0x42, 0xB8, 0x43, 0xB8, 0x44, /* 0xD0-0xD3 */ + 0xB8, 0x45, 0xB8, 0x46, 0xB8, 0x47, 0xB8, 0x48, /* 0xD4-0xD7 */ + 0xC5, 0xEF, 0xB8, 0x49, 0xB8, 0x4A, 0xB8, 0x4B, /* 0xD8-0xDB */ + 0xB8, 0x4C, 0xB8, 0x4D, 0xB8, 0x4E, 0xB8, 0x4F, /* 0xDC-0xDF */ + 0xB8, 0x50, 0xB8, 0x51, 0xB8, 0x52, 0xB8, 0x53, /* 0xE0-0xE3 */ + 0xB8, 0x54, 0xB8, 0x55, 0xB8, 0x56, 0xB8, 0x57, /* 0xE4-0xE7 */ + 0xB8, 0x58, 0xB8, 0x59, 0xB8, 0x5A, 0xB8, 0x61, /* 0xE8-0xEB */ + 0xB8, 0x62, 0xB8, 0x63, 0xB8, 0x64, 0xB8, 0x65, /* 0xEC-0xEF */ + 0xB8, 0x66, 0xB8, 0x67, 0xB8, 0x68, 0xB8, 0x69, /* 0xF0-0xF3 */ + 0xC5, 0xF0, 0xB8, 0x6A, 0xB8, 0x6B, 0xB8, 0x6C, /* 0xF4-0xF7 */ + 0xC5, 0xF1, 0xB8, 0x6D, 0xB8, 0x6E, 0xB8, 0x6F, /* 0xF8-0xFB */ + 0xB8, 0x70, 0xB8, 0x71, 0xB8, 0x72, 0xB8, 0x73, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_D2[512] = { + 0xB8, 0x74, 0xB8, 0x75, 0xB8, 0x76, 0xB8, 0x77, /* 0x00-0x03 */ + 0xB8, 0x78, 0xB8, 0x79, 0xB8, 0x7A, 0xC5, 0xF2, /* 0x04-0x07 */ + 0xB8, 0x81, 0xC5, 0xF3, 0xB8, 0x82, 0xB8, 0x83, /* 0x08-0x0B */ + 0xB8, 0x84, 0xB8, 0x85, 0xB8, 0x86, 0xB8, 0x87, /* 0x0C-0x0F */ + 0xC5, 0xF4, 0xB8, 0x88, 0xB8, 0x89, 0xB8, 0x8A, /* 0x10-0x13 */ + 0xB8, 0x8B, 0xB8, 0x8C, 0xB8, 0x8D, 0xB8, 0x8E, /* 0x14-0x17 */ + 0xB8, 0x8F, 0xB8, 0x90, 0xB8, 0x91, 0xB8, 0x92, /* 0x18-0x1B */ + 0xB8, 0x93, 0xB8, 0x94, 0xB8, 0x95, 0xB8, 0x96, /* 0x1C-0x1F */ + 0xB8, 0x97, 0xB8, 0x98, 0xB8, 0x99, 0xB8, 0x9A, /* 0x20-0x23 */ + 0xB8, 0x9B, 0xB8, 0x9C, 0xB8, 0x9D, 0xB8, 0x9E, /* 0x24-0x27 */ + 0xB8, 0x9F, 0xB8, 0xA0, 0xB9, 0x41, 0xB9, 0x42, /* 0x28-0x2B */ + 0xC5, 0xF5, 0xC5, 0xF6, 0xB9, 0x43, 0xB9, 0x44, /* 0x2C-0x2F */ + 0xC5, 0xF7, 0xB9, 0x45, 0xB9, 0x46, 0xB9, 0x47, /* 0x30-0x33 */ + 0xC5, 0xF8, 0xB9, 0x48, 0xB9, 0x49, 0xB9, 0x4A, /* 0x34-0x37 */ + 0xB9, 0x4B, 0xB9, 0x4C, 0xB9, 0x4D, 0xB9, 0x4E, /* 0x38-0x3B */ + 0xC5, 0xF9, 0xC5, 0xFA, 0xB9, 0x4F, 0xC5, 0xFB, /* 0x3C-0x3F */ + 0xB9, 0x50, 0xC5, 0xFC, 0xB9, 0x51, 0xB9, 0x52, /* 0x40-0x43 */ + 0xB9, 0x53, 0xB9, 0x54, 0xB9, 0x55, 0xB9, 0x56, /* 0x44-0x47 */ + 0xC5, 0xFD, 0xB9, 0x57, 0xB9, 0x58, 0xB9, 0x59, /* 0x48-0x4B */ + 0xB9, 0x5A, 0xB9, 0x61, 0xB9, 0x62, 0xB9, 0x63, /* 0x4C-0x4F */ + 0xB9, 0x64, 0xB9, 0x65, 0xB9, 0x66, 0xB9, 0x67, /* 0x50-0x53 */ + 0xB9, 0x68, 0xB9, 0x69, 0xB9, 0x6A, 0xB9, 0x6B, /* 0x54-0x57 */ + 0xB9, 0x6C, 0xB9, 0x6D, 0xB9, 0x6E, 0xB9, 0x6F, /* 0x58-0x5B */ + 0xC5, 0xFE, 0xB9, 0x70, 0xB9, 0x71, 0xB9, 0x72, /* 0x5C-0x5F */ + 0xB9, 0x73, 0xB9, 0x74, 0xB9, 0x75, 0xB9, 0x76, /* 0x60-0x63 */ + 0xC6, 0xA1, 0xB9, 0x77, 0xB9, 0x78, 0xB9, 0x79, /* 0x64-0x67 */ + 0xB9, 0x7A, 0xB9, 0x81, 0xB9, 0x82, 0xB9, 0x83, /* 0x68-0x6B */ + 0xB9, 0x84, 0xB9, 0x85, 0xB9, 0x86, 0xB9, 0x87, /* 0x6C-0x6F */ + 0xB9, 0x88, 0xB9, 0x89, 0xB9, 0x8A, 0xB9, 0x8B, /* 0x70-0x73 */ + 0xB9, 0x8C, 0xB9, 0x8D, 0xB9, 0x8E, 0xB9, 0x8F, /* 0x74-0x77 */ + 0xB9, 0x90, 0xB9, 0x91, 0xB9, 0x92, 0xB9, 0x93, /* 0x78-0x7B */ + 0xB9, 0x94, 0xB9, 0x95, 0xB9, 0x96, 0xB9, 0x97, /* 0x7C-0x7F */ + + 0xC6, 0xA2, 0xC6, 0xA3, 0xB9, 0x98, 0xB9, 0x99, /* 0x80-0x83 */ + 0xC6, 0xA4, 0xB9, 0x9A, 0xB9, 0x9B, 0xB9, 0x9C, /* 0x84-0x87 */ + 0xC6, 0xA5, 0xB9, 0x9D, 0xB9, 0x9E, 0xB9, 0x9F, /* 0x88-0x8B */ + 0xB9, 0xA0, 0xBA, 0x41, 0xBA, 0x42, 0xBA, 0x43, /* 0x8C-0x8F */ + 0xC6, 0xA6, 0xC6, 0xA7, 0xBA, 0x44, 0xBA, 0x45, /* 0x90-0x93 */ + 0xBA, 0x46, 0xC6, 0xA8, 0xBA, 0x47, 0xBA, 0x48, /* 0x94-0x97 */ + 0xBA, 0x49, 0xBA, 0x4A, 0xBA, 0x4B, 0xBA, 0x4C, /* 0x98-0x9B */ + 0xC6, 0xA9, 0xBA, 0x4D, 0xBA, 0x4E, 0xBA, 0x4F, /* 0x9C-0x9F */ + 0xC6, 0xAA, 0xBA, 0x50, 0xBA, 0x51, 0xBA, 0x52, /* 0xA0-0xA3 */ + 0xC6, 0xAB, 0xBA, 0x53, 0xBA, 0x54, 0xBA, 0x55, /* 0xA4-0xA7 */ + 0xBA, 0x56, 0xBA, 0x57, 0xBA, 0x58, 0xBA, 0x59, /* 0xA8-0xAB */ + 0xC6, 0xAC, 0xBA, 0x5A, 0xBA, 0x61, 0xBA, 0x62, /* 0xAC-0xAF */ + 0xBA, 0x63, 0xC6, 0xAD, 0xBA, 0x64, 0xBA, 0x65, /* 0xB0-0xB3 */ + 0xBA, 0x66, 0xBA, 0x67, 0xBA, 0x68, 0xBA, 0x69, /* 0xB4-0xB7 */ + 0xC6, 0xAE, 0xC6, 0xAF, 0xBA, 0x6A, 0xBA, 0x6B, /* 0xB8-0xBB */ + 0xC6, 0xB0, 0xBA, 0x6C, 0xBA, 0x6D, 0xC6, 0xB1, /* 0xBC-0xBF */ + 0xC6, 0xB2, 0xBA, 0x6E, 0xC6, 0xB3, 0xBA, 0x6F, /* 0xC0-0xC3 */ + 0xBA, 0x70, 0xBA, 0x71, 0xBA, 0x72, 0xBA, 0x73, /* 0xC4-0xC7 */ + 0xC6, 0xB4, 0xC6, 0xB5, 0xBA, 0x74, 0xC6, 0xB6, /* 0xC8-0xCB */ + 0xBA, 0x75, 0xBA, 0x76, 0xBA, 0x77, 0xBA, 0x78, /* 0xCC-0xCF */ + 0xBA, 0x79, 0xBA, 0x7A, 0xBA, 0x81, 0xBA, 0x82, /* 0xD0-0xD3 */ + 0xC6, 0xB7, 0xBA, 0x83, 0xBA, 0x84, 0xBA, 0x85, /* 0xD4-0xD7 */ + 0xC6, 0xB8, 0xBA, 0x86, 0xBA, 0x87, 0xBA, 0x88, /* 0xD8-0xDB */ + 0xC6, 0xB9, 0xBA, 0x89, 0xBA, 0x8A, 0xBA, 0x8B, /* 0xDC-0xDF */ + 0xBA, 0x8C, 0xBA, 0x8D, 0xBA, 0x8E, 0xBA, 0x8F, /* 0xE0-0xE3 */ + 0xC6, 0xBA, 0xC6, 0xBB, 0xBA, 0x90, 0xBA, 0x91, /* 0xE4-0xE7 */ + 0xBA, 0x92, 0xBA, 0x93, 0xBA, 0x94, 0xBA, 0x95, /* 0xE8-0xEB */ + 0xBA, 0x96, 0xBA, 0x97, 0xBA, 0x98, 0xBA, 0x99, /* 0xEC-0xEF */ + 0xC6, 0xBC, 0xC6, 0xBD, 0xBA, 0x9A, 0xBA, 0x9B, /* 0xF0-0xF3 */ + 0xC6, 0xBE, 0xBA, 0x9C, 0xBA, 0x9D, 0xBA, 0x9E, /* 0xF4-0xF7 */ + 0xC6, 0xBF, 0xBA, 0x9F, 0xBA, 0xA0, 0xBB, 0x41, /* 0xF8-0xFB */ + 0xBB, 0x42, 0xBB, 0x43, 0xBB, 0x44, 0xBB, 0x45, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_D3[512] = { + 0xC6, 0xC0, 0xC6, 0xC1, 0xBB, 0x46, 0xC6, 0xC2, /* 0x00-0x03 */ + 0xBB, 0x47, 0xC6, 0xC3, 0xBB, 0x48, 0xBB, 0x49, /* 0x04-0x07 */ + 0xBB, 0x4A, 0xBB, 0x4B, 0xBB, 0x4C, 0xBB, 0x4D, /* 0x08-0x0B */ + 0xC6, 0xC4, 0xC6, 0xC5, 0xC6, 0xC6, 0xBB, 0x4E, /* 0x0C-0x0F */ + 0xC6, 0xC7, 0xBB, 0x4F, 0xBB, 0x50, 0xBB, 0x51, /* 0x10-0x13 */ + 0xC6, 0xC8, 0xBB, 0x52, 0xC6, 0xC9, 0xBB, 0x53, /* 0x14-0x17 */ + 0xBB, 0x54, 0xBB, 0x55, 0xBB, 0x56, 0xBB, 0x57, /* 0x18-0x1B */ + 0xC6, 0xCA, 0xC6, 0xCB, 0xBB, 0x58, 0xC6, 0xCC, /* 0x1C-0x1F */ + 0xC6, 0xCD, 0xC6, 0xCE, 0xBB, 0x59, 0xBB, 0x5A, /* 0x20-0x23 */ + 0xBB, 0x61, 0xC6, 0xCF, 0xBB, 0x62, 0xBB, 0x63, /* 0x24-0x27 */ + 0xC6, 0xD0, 0xC6, 0xD1, 0xBB, 0x64, 0xBB, 0x65, /* 0x28-0x2B */ + 0xC6, 0xD2, 0xBB, 0x66, 0xBB, 0x67, 0xBB, 0x68, /* 0x2C-0x2F */ + 0xC6, 0xD3, 0xBB, 0x69, 0xBB, 0x6A, 0xBB, 0x6B, /* 0x30-0x33 */ + 0xBB, 0x6C, 0xBB, 0x6D, 0xBB, 0x6E, 0xBB, 0x6F, /* 0x34-0x37 */ + 0xC6, 0xD4, 0xC6, 0xD5, 0xBB, 0x70, 0xC6, 0xD6, /* 0x38-0x3B */ + 0xC6, 0xD7, 0xC6, 0xD8, 0xBB, 0x71, 0xBB, 0x72, /* 0x3C-0x3F */ + 0xBB, 0x73, 0xBB, 0x74, 0xBB, 0x75, 0xBB, 0x76, /* 0x40-0x43 */ + 0xC6, 0xD9, 0xC6, 0xDA, 0xBB, 0x77, 0xBB, 0x78, /* 0x44-0x47 */ + 0xBB, 0x79, 0xBB, 0x7A, 0xBB, 0x81, 0xBB, 0x82, /* 0x48-0x4B */ + 0xBB, 0x83, 0xBB, 0x84, 0xBB, 0x85, 0xBB, 0x86, /* 0x4C-0x4F */ + 0xBB, 0x87, 0xBB, 0x88, 0xBB, 0x89, 0xBB, 0x8A, /* 0x50-0x53 */ + 0xBB, 0x8B, 0xBB, 0x8C, 0xBB, 0x8D, 0xBB, 0x8E, /* 0x54-0x57 */ + 0xBB, 0x8F, 0xBB, 0x90, 0xBB, 0x91, 0xBB, 0x92, /* 0x58-0x5B */ + 0xBB, 0x93, 0xBB, 0x94, 0xBB, 0x95, 0xBB, 0x96, /* 0x5C-0x5F */ + 0xBB, 0x97, 0xBB, 0x98, 0xBB, 0x99, 0xBB, 0x9A, /* 0x60-0x63 */ + 0xBB, 0x9B, 0xBB, 0x9C, 0xBB, 0x9D, 0xBB, 0x9E, /* 0x64-0x67 */ + 0xBB, 0x9F, 0xBB, 0xA0, 0xBC, 0x41, 0xBC, 0x42, /* 0x68-0x6B */ + 0xBC, 0x43, 0xBC, 0x44, 0xBC, 0x45, 0xBC, 0x46, /* 0x6C-0x6F */ + 0xBC, 0x47, 0xBC, 0x48, 0xBC, 0x49, 0xBC, 0x4A, /* 0x70-0x73 */ + 0xBC, 0x4B, 0xBC, 0x4C, 0xBC, 0x4D, 0xBC, 0x4E, /* 0x74-0x77 */ + 0xBC, 0x4F, 0xBC, 0x50, 0xBC, 0x51, 0xBC, 0x52, /* 0x78-0x7B */ + 0xC6, 0xDB, 0xC6, 0xDC, 0xBC, 0x53, 0xBC, 0x54, /* 0x7C-0x7F */ + + 0xC6, 0xDD, 0xBC, 0x55, 0xBC, 0x56, 0xBC, 0x57, /* 0x80-0x83 */ + 0xC6, 0xDE, 0xBC, 0x58, 0xBC, 0x59, 0xBC, 0x5A, /* 0x84-0x87 */ + 0xBC, 0x61, 0xBC, 0x62, 0xBC, 0x63, 0xBC, 0x64, /* 0x88-0x8B */ + 0xC6, 0xDF, 0xC6, 0xE0, 0xBC, 0x65, 0xC6, 0xE1, /* 0x8C-0x8F */ + 0xC6, 0xE2, 0xC6, 0xE3, 0xBC, 0x66, 0xBC, 0x67, /* 0x90-0x93 */ + 0xBC, 0x68, 0xBC, 0x69, 0xBC, 0x6A, 0xBC, 0x6B, /* 0x94-0x97 */ + 0xC6, 0xE4, 0xC6, 0xE5, 0xBC, 0x6C, 0xBC, 0x6D, /* 0x98-0x9B */ + 0xC6, 0xE6, 0xBC, 0x6E, 0xBC, 0x6F, 0xBC, 0x70, /* 0x9C-0x9F */ + 0xC6, 0xE7, 0xBC, 0x71, 0xBC, 0x72, 0xBC, 0x73, /* 0xA0-0xA3 */ + 0xBC, 0x74, 0xBC, 0x75, 0xBC, 0x76, 0xBC, 0x77, /* 0xA4-0xA7 */ + 0xC6, 0xE8, 0xC6, 0xE9, 0xBC, 0x78, 0xC6, 0xEA, /* 0xA8-0xAB */ + 0xBC, 0x79, 0xC6, 0xEB, 0xBC, 0x7A, 0xBC, 0x81, /* 0xAC-0xAF */ + 0xBC, 0x82, 0xBC, 0x83, 0xBC, 0x84, 0xBC, 0x85, /* 0xB0-0xB3 */ + 0xC6, 0xEC, 0xBC, 0x86, 0xBC, 0x87, 0xBC, 0x88, /* 0xB4-0xB7 */ + 0xC6, 0xED, 0xBC, 0x89, 0xBC, 0x8A, 0xBC, 0x8B, /* 0xB8-0xBB */ + 0xC6, 0xEE, 0xBC, 0x8C, 0xBC, 0x8D, 0xBC, 0x8E, /* 0xBC-0xBF */ + 0xBC, 0x8F, 0xBC, 0x90, 0xBC, 0x91, 0xBC, 0x92, /* 0xC0-0xC3 */ + 0xC6, 0xEF, 0xC6, 0xF0, 0xBC, 0x93, 0xBC, 0x94, /* 0xC4-0xC7 */ + 0xC6, 0xF1, 0xC6, 0xF2, 0xBC, 0x95, 0xBC, 0x96, /* 0xC8-0xCB */ + 0xBC, 0x97, 0xBC, 0x98, 0xBC, 0x99, 0xBC, 0x9A, /* 0xCC-0xCF */ + 0xC6, 0xF3, 0xBC, 0x9B, 0xBC, 0x9C, 0xBC, 0x9D, /* 0xD0-0xD3 */ + 0xBC, 0x9E, 0xBC, 0x9F, 0xBC, 0xA0, 0xBD, 0x41, /* 0xD4-0xD7 */ + 0xC6, 0xF4, 0xBD, 0x42, 0xBD, 0x43, 0xBD, 0x44, /* 0xD8-0xDB */ + 0xBD, 0x45, 0xBD, 0x46, 0xBD, 0x47, 0xBD, 0x48, /* 0xDC-0xDF */ + 0xBD, 0x49, 0xC6, 0xF5, 0xBD, 0x4A, 0xC6, 0xF6, /* 0xE0-0xE3 */ + 0xBD, 0x4B, 0xBD, 0x4C, 0xBD, 0x4D, 0xBD, 0x4E, /* 0xE4-0xE7 */ + 0xBD, 0x4F, 0xBD, 0x50, 0xBD, 0x51, 0xBD, 0x52, /* 0xE8-0xEB */ + 0xC6, 0xF7, 0xC6, 0xF8, 0xBD, 0x53, 0xBD, 0x54, /* 0xEC-0xEF */ + 0xC6, 0xF9, 0xBD, 0x55, 0xBD, 0x56, 0xBD, 0x57, /* 0xF0-0xF3 */ + 0xC6, 0xFA, 0xBD, 0x58, 0xBD, 0x59, 0xBD, 0x5A, /* 0xF4-0xF7 */ + 0xBD, 0x61, 0xBD, 0x62, 0xBD, 0x63, 0xBD, 0x64, /* 0xF8-0xFB */ + 0xC6, 0xFB, 0xC6, 0xFC, 0xBD, 0x65, 0xC6, 0xFD, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_D4[512] = { + 0xBD, 0x66, 0xC6, 0xFE, 0xBD, 0x67, 0xBD, 0x68, /* 0x00-0x03 */ + 0xBD, 0x69, 0xBD, 0x6A, 0xBD, 0x6B, 0xBD, 0x6C, /* 0x04-0x07 */ + 0xC7, 0xA1, 0xBD, 0x6D, 0xBD, 0x6E, 0xBD, 0x6F, /* 0x08-0x0B */ + 0xBD, 0x70, 0xBD, 0x71, 0xBD, 0x72, 0xBD, 0x73, /* 0x0C-0x0F */ + 0xBD, 0x74, 0xBD, 0x75, 0xBD, 0x76, 0xBD, 0x77, /* 0x10-0x13 */ + 0xBD, 0x78, 0xBD, 0x79, 0xBD, 0x7A, 0xBD, 0x81, /* 0x14-0x17 */ + 0xBD, 0x82, 0xBD, 0x83, 0xBD, 0x84, 0xBD, 0x85, /* 0x18-0x1B */ + 0xBD, 0x86, 0xC7, 0xA2, 0xBD, 0x87, 0xBD, 0x88, /* 0x1C-0x1F */ + 0xBD, 0x89, 0xBD, 0x8A, 0xBD, 0x8B, 0xBD, 0x8C, /* 0x20-0x23 */ + 0xBD, 0x8D, 0xBD, 0x8E, 0xBD, 0x8F, 0xBD, 0x90, /* 0x24-0x27 */ + 0xBD, 0x91, 0xBD, 0x92, 0xBD, 0x93, 0xBD, 0x94, /* 0x28-0x2B */ + 0xBD, 0x95, 0xBD, 0x96, 0xBD, 0x97, 0xBD, 0x98, /* 0x2C-0x2F */ + 0xBD, 0x99, 0xBD, 0x9A, 0xBD, 0x9B, 0xBD, 0x9C, /* 0x30-0x33 */ + 0xBD, 0x9D, 0xBD, 0x9E, 0xBD, 0x9F, 0xBD, 0xA0, /* 0x34-0x37 */ + 0xBE, 0x41, 0xBE, 0x42, 0xBE, 0x43, 0xBE, 0x44, /* 0x38-0x3B */ + 0xBE, 0x45, 0xBE, 0x46, 0xBE, 0x47, 0xBE, 0x48, /* 0x3C-0x3F */ + 0xC7, 0xA3, 0xBE, 0x49, 0xBE, 0x4A, 0xBE, 0x4B, /* 0x40-0x43 */ + 0xC7, 0xA4, 0xBE, 0x4C, 0xBE, 0x4D, 0xBE, 0x4E, /* 0x44-0x47 */ + 0xBE, 0x4F, 0xBE, 0x50, 0xBE, 0x51, 0xBE, 0x52, /* 0x48-0x4B */ + 0xBE, 0x53, 0xBE, 0x54, 0xBE, 0x55, 0xBE, 0x56, /* 0x4C-0x4F */ + 0xBE, 0x57, 0xBE, 0x58, 0xBE, 0x59, 0xBE, 0x5A, /* 0x50-0x53 */ + 0xBE, 0x61, 0xBE, 0x62, 0xBE, 0x63, 0xBE, 0x64, /* 0x54-0x57 */ + 0xBE, 0x65, 0xBE, 0x66, 0xBE, 0x67, 0xBE, 0x68, /* 0x58-0x5B */ + 0xC7, 0xA5, 0xBE, 0x69, 0xBE, 0x6A, 0xBE, 0x6B, /* 0x5C-0x5F */ + 0xC7, 0xA6, 0xBE, 0x6C, 0xBE, 0x6D, 0xBE, 0x6E, /* 0x60-0x63 */ + 0xC7, 0xA7, 0xBE, 0x6F, 0xBE, 0x70, 0xBE, 0x71, /* 0x64-0x67 */ + 0xBE, 0x72, 0xBE, 0x73, 0xBE, 0x74, 0xBE, 0x75, /* 0x68-0x6B */ + 0xBE, 0x76, 0xC7, 0xA8, 0xBE, 0x77, 0xC7, 0xA9, /* 0x6C-0x6F */ + 0xBE, 0x78, 0xBE, 0x79, 0xBE, 0x7A, 0xBE, 0x81, /* 0x70-0x73 */ + 0xBE, 0x82, 0xBE, 0x83, 0xBE, 0x84, 0xBE, 0x85, /* 0x74-0x77 */ + 0xC7, 0xAA, 0xC7, 0xAB, 0xBE, 0x86, 0xBE, 0x87, /* 0x78-0x7B */ + 0xC7, 0xAC, 0xBE, 0x88, 0xBE, 0x89, 0xC7, 0xAD, /* 0x7C-0x7F */ + + 0xC7, 0xAE, 0xBE, 0x8A, 0xC7, 0xAF, 0xBE, 0x8B, /* 0x80-0x83 */ + 0xBE, 0x8C, 0xBE, 0x8D, 0xBE, 0x8E, 0xBE, 0x8F, /* 0x84-0x87 */ + 0xC7, 0xB0, 0xC7, 0xB1, 0xBE, 0x90, 0xC7, 0xB2, /* 0x88-0x8B */ + 0xBE, 0x91, 0xC7, 0xB3, 0xBE, 0x92, 0xBE, 0x93, /* 0x8C-0x8F */ + 0xBE, 0x94, 0xBE, 0x95, 0xBE, 0x96, 0xBE, 0x97, /* 0x90-0x93 */ + 0xC7, 0xB4, 0xBE, 0x98, 0xBE, 0x99, 0xBE, 0x9A, /* 0x94-0x97 */ + 0xBE, 0x9B, 0xBE, 0x9C, 0xBE, 0x9D, 0xBE, 0x9E, /* 0x98-0x9B */ + 0xBE, 0x9F, 0xBE, 0xA0, 0xBF, 0x41, 0xBF, 0x42, /* 0x9C-0x9F */ + 0xBF, 0x43, 0xBF, 0x44, 0xBF, 0x45, 0xBF, 0x46, /* 0xA0-0xA3 */ + 0xBF, 0x47, 0xBF, 0x48, 0xBF, 0x49, 0xBF, 0x4A, /* 0xA4-0xA7 */ + 0xBF, 0x4B, 0xC7, 0xB5, 0xBF, 0x4C, 0xBF, 0x4D, /* 0xA8-0xAB */ + 0xBF, 0x4E, 0xBF, 0x4F, 0xBF, 0x50, 0xBF, 0x51, /* 0xAC-0xAF */ + 0xBF, 0x52, 0xBF, 0x53, 0xBF, 0x54, 0xBF, 0x55, /* 0xB0-0xB3 */ + 0xBF, 0x56, 0xBF, 0x57, 0xBF, 0x58, 0xBF, 0x59, /* 0xB4-0xB7 */ + 0xBF, 0x5A, 0xBF, 0x61, 0xBF, 0x62, 0xBF, 0x63, /* 0xB8-0xBB */ + 0xBF, 0x64, 0xBF, 0x65, 0xBF, 0x66, 0xBF, 0x67, /* 0xBC-0xBF */ + 0xBF, 0x68, 0xBF, 0x69, 0xBF, 0x6A, 0xBF, 0x6B, /* 0xC0-0xC3 */ + 0xBF, 0x6C, 0xBF, 0x6D, 0xBF, 0x6E, 0xBF, 0x6F, /* 0xC4-0xC7 */ + 0xBF, 0x70, 0xBF, 0x71, 0xBF, 0x72, 0xBF, 0x73, /* 0xC8-0xCB */ + 0xC7, 0xB6, 0xBF, 0x74, 0xBF, 0x75, 0xBF, 0x76, /* 0xCC-0xCF */ + 0xC7, 0xB7, 0xBF, 0x77, 0xBF, 0x78, 0xBF, 0x79, /* 0xD0-0xD3 */ + 0xC7, 0xB8, 0xBF, 0x7A, 0xBF, 0x81, 0xBF, 0x82, /* 0xD4-0xD7 */ + 0xBF, 0x83, 0xBF, 0x84, 0xBF, 0x85, 0xBF, 0x86, /* 0xD8-0xDB */ + 0xC7, 0xB9, 0xBF, 0x87, 0xBF, 0x88, 0xC7, 0xBA, /* 0xDC-0xDF */ + 0xBF, 0x89, 0xBF, 0x8A, 0xBF, 0x8B, 0xBF, 0x8C, /* 0xE0-0xE3 */ + 0xBF, 0x8D, 0xBF, 0x8E, 0xBF, 0x8F, 0xBF, 0x90, /* 0xE4-0xE7 */ + 0xC7, 0xBB, 0xBF, 0x91, 0xBF, 0x92, 0xBF, 0x93, /* 0xE8-0xEB */ + 0xC7, 0xBC, 0xBF, 0x94, 0xBF, 0x95, 0xBF, 0x96, /* 0xEC-0xEF */ + 0xC7, 0xBD, 0xBF, 0x97, 0xBF, 0x98, 0xBF, 0x99, /* 0xF0-0xF3 */ + 0xBF, 0x9A, 0xBF, 0x9B, 0xBF, 0x9C, 0xBF, 0x9D, /* 0xF4-0xF7 */ + 0xC7, 0xBE, 0xBF, 0x9E, 0xBF, 0x9F, 0xC7, 0xBF, /* 0xF8-0xFB */ + 0xBF, 0xA0, 0xC7, 0xC0, 0xC0, 0x41, 0xC0, 0x42, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_D5[512] = { + 0xC0, 0x43, 0xC0, 0x44, 0xC0, 0x45, 0xC0, 0x46, /* 0x00-0x03 */ + 0xC7, 0xC1, 0xC0, 0x47, 0xC0, 0x48, 0xC0, 0x49, /* 0x04-0x07 */ + 0xC7, 0xC2, 0xC0, 0x4A, 0xC0, 0x4B, 0xC0, 0x4C, /* 0x08-0x0B */ + 0xC7, 0xC3, 0xC0, 0x4D, 0xC0, 0x4E, 0xC0, 0x4F, /* 0x0C-0x0F */ + 0xC0, 0x50, 0xC0, 0x51, 0xC0, 0x52, 0xC0, 0x53, /* 0x10-0x13 */ + 0xC7, 0xC4, 0xC7, 0xC5, 0xC0, 0x54, 0xC7, 0xC6, /* 0x14-0x17 */ + 0xC0, 0x55, 0xC0, 0x56, 0xC0, 0x57, 0xC0, 0x58, /* 0x18-0x1B */ + 0xC0, 0x59, 0xC0, 0x5A, 0xC0, 0x61, 0xC0, 0x62, /* 0x1C-0x1F */ + 0xC0, 0x63, 0xC0, 0x64, 0xC0, 0x65, 0xC0, 0x66, /* 0x20-0x23 */ + 0xC0, 0x67, 0xC0, 0x68, 0xC0, 0x69, 0xC0, 0x6A, /* 0x24-0x27 */ + 0xC0, 0x6B, 0xC0, 0x6C, 0xC0, 0x6D, 0xC0, 0x6E, /* 0x28-0x2B */ + 0xC0, 0x6F, 0xC0, 0x70, 0xC0, 0x71, 0xC0, 0x72, /* 0x2C-0x2F */ + 0xC0, 0x73, 0xC0, 0x74, 0xC0, 0x75, 0xC0, 0x76, /* 0x30-0x33 */ + 0xC0, 0x77, 0xC0, 0x78, 0xC0, 0x79, 0xC0, 0x7A, /* 0x34-0x37 */ + 0xC0, 0x81, 0xC0, 0x82, 0xC0, 0x83, 0xC0, 0x84, /* 0x38-0x3B */ + 0xC7, 0xC7, 0xC7, 0xC8, 0xC0, 0x85, 0xC0, 0x86, /* 0x3C-0x3F */ + 0xC7, 0xC9, 0xC0, 0x87, 0xC0, 0x88, 0xC0, 0x89, /* 0x40-0x43 */ + 0xC7, 0xCA, 0xC0, 0x8A, 0xC0, 0x8B, 0xC0, 0x8C, /* 0x44-0x47 */ + 0xC0, 0x8D, 0xC0, 0x8E, 0xC0, 0x8F, 0xC0, 0x90, /* 0x48-0x4B */ + 0xC7, 0xCB, 0xC7, 0xCC, 0xC0, 0x91, 0xC7, 0xCD, /* 0x4C-0x4F */ + 0xC0, 0x92, 0xC7, 0xCE, 0xC0, 0x93, 0xC0, 0x94, /* 0x50-0x53 */ + 0xC0, 0x95, 0xC0, 0x96, 0xC0, 0x97, 0xC0, 0x98, /* 0x54-0x57 */ + 0xC7, 0xCF, 0xC7, 0xD0, 0xC0, 0x99, 0xC0, 0x9A, /* 0x58-0x5B */ + 0xC7, 0xD1, 0xC0, 0x9B, 0xC0, 0x9C, 0xC0, 0x9D, /* 0x5C-0x5F */ + 0xC7, 0xD2, 0xC0, 0x9E, 0xC0, 0x9F, 0xC0, 0xA0, /* 0x60-0x63 */ + 0xC1, 0x41, 0xC7, 0xD3, 0xC1, 0x42, 0xC1, 0x43, /* 0x64-0x67 */ + 0xC7, 0xD4, 0xC7, 0xD5, 0xC1, 0x44, 0xC7, 0xD6, /* 0x68-0x6B */ + 0xC1, 0x45, 0xC7, 0xD7, 0xC1, 0x46, 0xC1, 0x47, /* 0x6C-0x6F */ + 0xC1, 0x48, 0xC1, 0x49, 0xC1, 0x4A, 0xC1, 0x4B, /* 0x70-0x73 */ + 0xC7, 0xD8, 0xC7, 0xD9, 0xC1, 0x4C, 0xC1, 0x4D, /* 0x74-0x77 */ + 0xC7, 0xDA, 0xC1, 0x4E, 0xC1, 0x4F, 0xC1, 0x50, /* 0x78-0x7B */ + 0xC7, 0xDB, 0xC1, 0x51, 0xC1, 0x52, 0xC1, 0x53, /* 0x7C-0x7F */ + + 0xC1, 0x54, 0xC1, 0x55, 0xC1, 0x56, 0xC1, 0x57, /* 0x80-0x83 */ + 0xC7, 0xDC, 0xC7, 0xDD, 0xC1, 0x58, 0xC7, 0xDE, /* 0x84-0x87 */ + 0xC7, 0xDF, 0xC7, 0xE0, 0xC1, 0x59, 0xC1, 0x5A, /* 0x88-0x8B */ + 0xC1, 0x61, 0xC1, 0x62, 0xC1, 0x63, 0xC1, 0x64, /* 0x8C-0x8F */ + 0xC7, 0xE1, 0xC1, 0x65, 0xC1, 0x66, 0xC1, 0x67, /* 0x90-0x93 */ + 0xC1, 0x68, 0xC1, 0x69, 0xC1, 0x6A, 0xC1, 0x6B, /* 0x94-0x97 */ + 0xC1, 0x6C, 0xC1, 0x6D, 0xC1, 0x6E, 0xC1, 0x6F, /* 0x98-0x9B */ + 0xC1, 0x70, 0xC1, 0x71, 0xC1, 0x72, 0xC1, 0x73, /* 0x9C-0x9F */ + 0xC1, 0x74, 0xC1, 0x75, 0xC1, 0x76, 0xC1, 0x77, /* 0xA0-0xA3 */ + 0xC1, 0x78, 0xC7, 0xE2, 0xC1, 0x79, 0xC1, 0x7A, /* 0xA4-0xA7 */ + 0xC1, 0x81, 0xC1, 0x82, 0xC1, 0x83, 0xC1, 0x84, /* 0xA8-0xAB */ + 0xC1, 0x85, 0xC1, 0x86, 0xC1, 0x87, 0xC1, 0x88, /* 0xAC-0xAF */ + 0xC1, 0x89, 0xC1, 0x8A, 0xC1, 0x8B, 0xC1, 0x8C, /* 0xB0-0xB3 */ + 0xC1, 0x8D, 0xC1, 0x8E, 0xC1, 0x8F, 0xC1, 0x90, /* 0xB4-0xB7 */ + 0xC1, 0x91, 0xC1, 0x92, 0xC1, 0x93, 0xC1, 0x94, /* 0xB8-0xBB */ + 0xC1, 0x95, 0xC1, 0x96, 0xC1, 0x97, 0xC1, 0x98, /* 0xBC-0xBF */ + 0xC1, 0x99, 0xC1, 0x9A, 0xC1, 0x9B, 0xC1, 0x9C, /* 0xC0-0xC3 */ + 0xC1, 0x9D, 0xC1, 0x9E, 0xC1, 0x9F, 0xC1, 0xA0, /* 0xC4-0xC7 */ + 0xC7, 0xE3, 0xC7, 0xE4, 0xC2, 0x41, 0xC2, 0x42, /* 0xC8-0xCB */ + 0xC7, 0xE5, 0xC2, 0x43, 0xC2, 0x44, 0xC2, 0x45, /* 0xCC-0xCF */ + 0xC7, 0xE6, 0xC2, 0x46, 0xC7, 0xE7, 0xC2, 0x47, /* 0xD0-0xD3 */ + 0xC2, 0x48, 0xC2, 0x49, 0xC2, 0x4A, 0xC2, 0x4B, /* 0xD4-0xD7 */ + 0xC7, 0xE8, 0xC7, 0xE9, 0xC2, 0x4C, 0xC7, 0xEA, /* 0xD8-0xDB */ + 0xC2, 0x4D, 0xC7, 0xEB, 0xC2, 0x4E, 0xC2, 0x4F, /* 0xDC-0xDF */ + 0xC2, 0x50, 0xC2, 0x51, 0xC2, 0x52, 0xC2, 0x53, /* 0xE0-0xE3 */ + 0xC7, 0xEC, 0xC7, 0xED, 0xC2, 0x54, 0xC2, 0x55, /* 0xE4-0xE7 */ + 0xC7, 0xEE, 0xC2, 0x56, 0xC2, 0x57, 0xC2, 0x58, /* 0xE8-0xEB */ + 0xC7, 0xEF, 0xC2, 0x59, 0xC2, 0x5A, 0xC2, 0x61, /* 0xEC-0xEF */ + 0xC2, 0x62, 0xC2, 0x63, 0xC2, 0x64, 0xC2, 0x65, /* 0xF0-0xF3 */ + 0xC7, 0xF0, 0xC7, 0xF1, 0xC2, 0x66, 0xC7, 0xF2, /* 0xF4-0xF7 */ + 0xC2, 0x67, 0xC7, 0xF3, 0xC2, 0x68, 0xC2, 0x69, /* 0xF8-0xFB */ + 0xC2, 0x6A, 0xC2, 0x6B, 0xC2, 0x6C, 0xC2, 0x6D, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_D6[512] = { + 0xC7, 0xF4, 0xC7, 0xF5, 0xC2, 0x6E, 0xC2, 0x6F, /* 0x00-0x03 */ + 0xC7, 0xF6, 0xC2, 0x70, 0xC2, 0x71, 0xC2, 0x72, /* 0x04-0x07 */ + 0xC7, 0xF7, 0xC2, 0x73, 0xC2, 0x74, 0xC2, 0x75, /* 0x08-0x0B */ + 0xC2, 0x76, 0xC2, 0x77, 0xC2, 0x78, 0xC2, 0x79, /* 0x0C-0x0F */ + 0xC7, 0xF8, 0xC7, 0xF9, 0xC2, 0x7A, 0xC7, 0xFA, /* 0x10-0x13 */ + 0xC7, 0xFB, 0xC7, 0xFC, 0xC2, 0x81, 0xC2, 0x82, /* 0x14-0x17 */ + 0xC2, 0x83, 0xC2, 0x84, 0xC2, 0x85, 0xC2, 0x86, /* 0x18-0x1B */ + 0xC7, 0xFD, 0xC2, 0x87, 0xC2, 0x88, 0xC2, 0x89, /* 0x1C-0x1F */ + 0xC7, 0xFE, 0xC2, 0x8A, 0xC2, 0x8B, 0xC2, 0x8C, /* 0x20-0x23 */ + 0xC8, 0xA1, 0xC2, 0x8D, 0xC2, 0x8E, 0xC2, 0x8F, /* 0x24-0x27 */ + 0xC2, 0x90, 0xC2, 0x91, 0xC2, 0x92, 0xC2, 0x93, /* 0x28-0x2B */ + 0xC2, 0x94, 0xC8, 0xA2, 0xC2, 0x95, 0xC2, 0x96, /* 0x2C-0x2F */ + 0xC2, 0x97, 0xC2, 0x98, 0xC2, 0x99, 0xC2, 0x9A, /* 0x30-0x33 */ + 0xC2, 0x9B, 0xC2, 0x9C, 0xC2, 0x9D, 0xC2, 0x9E, /* 0x34-0x37 */ + 0xC8, 0xA3, 0xC8, 0xA4, 0xC2, 0x9F, 0xC2, 0xA0, /* 0x38-0x3B */ + 0xC8, 0xA5, 0xC3, 0x41, 0xC3, 0x42, 0xC3, 0x43, /* 0x3C-0x3F */ + 0xC8, 0xA6, 0xC3, 0x44, 0xC3, 0x45, 0xC3, 0x46, /* 0x40-0x43 */ + 0xC3, 0x47, 0xC8, 0xA7, 0xC3, 0x48, 0xC3, 0x49, /* 0x44-0x47 */ + 0xC8, 0xA8, 0xC8, 0xA9, 0xC3, 0x4A, 0xC8, 0xAA, /* 0x48-0x4B */ + 0xC3, 0x4B, 0xC8, 0xAB, 0xC3, 0x4C, 0xC3, 0x4D, /* 0x4C-0x4F */ + 0xC3, 0x4E, 0xC8, 0xAC, 0xC3, 0x4F, 0xC3, 0x50, /* 0x50-0x53 */ + 0xC8, 0xAD, 0xC8, 0xAE, 0xC3, 0x51, 0xC3, 0x52, /* 0x54-0x57 */ + 0xC8, 0xAF, 0xC3, 0x53, 0xC3, 0x54, 0xC3, 0x55, /* 0x58-0x5B */ + 0xC8, 0xB0, 0xC3, 0x56, 0xC3, 0x57, 0xC3, 0x58, /* 0x5C-0x5F */ + 0xC3, 0x59, 0xC3, 0x5A, 0xC3, 0x61, 0xC3, 0x62, /* 0x60-0x63 */ + 0xC3, 0x63, 0xC3, 0x64, 0xC3, 0x65, 0xC8, 0xB1, /* 0x64-0x67 */ + 0xC3, 0x66, 0xC8, 0xB2, 0xC3, 0x67, 0xC3, 0x68, /* 0x68-0x6B */ + 0xC3, 0x69, 0xC3, 0x6A, 0xC3, 0x6B, 0xC3, 0x6C, /* 0x6C-0x6F */ + 0xC8, 0xB3, 0xC8, 0xB4, 0xC3, 0x6D, 0xC3, 0x6E, /* 0x70-0x73 */ + 0xC8, 0xB5, 0xC3, 0x6F, 0xC3, 0x70, 0xC3, 0x71, /* 0x74-0x77 */ + 0xC3, 0x72, 0xC3, 0x73, 0xC3, 0x74, 0xC3, 0x75, /* 0x78-0x7B */ + 0xC3, 0x76, 0xC3, 0x77, 0xC3, 0x78, 0xC3, 0x79, /* 0x7C-0x7F */ + + 0xC3, 0x7A, 0xC3, 0x81, 0xC3, 0x82, 0xC8, 0xB6, /* 0x80-0x83 */ + 0xC3, 0x83, 0xC8, 0xB7, 0xC3, 0x84, 0xC3, 0x85, /* 0x84-0x87 */ + 0xC3, 0x86, 0xC3, 0x87, 0xC3, 0x88, 0xC3, 0x89, /* 0x88-0x8B */ + 0xC8, 0xB8, 0xC8, 0xB9, 0xC3, 0x8A, 0xC3, 0x8B, /* 0x8C-0x8F */ + 0xC8, 0xBA, 0xC3, 0x8C, 0xC3, 0x8D, 0xC3, 0x8E, /* 0x90-0x93 */ + 0xC8, 0xBB, 0xC3, 0x8F, 0xC3, 0x90, 0xC3, 0x91, /* 0x94-0x97 */ + 0xC3, 0x92, 0xC3, 0x93, 0xC3, 0x94, 0xC3, 0x95, /* 0x98-0x9B */ + 0xC3, 0x96, 0xC8, 0xBC, 0xC3, 0x97, 0xC8, 0xBD, /* 0x9C-0x9F */ + 0xC3, 0x98, 0xC8, 0xBE, 0xC3, 0x99, 0xC3, 0x9A, /* 0xA0-0xA3 */ + 0xC3, 0x9B, 0xC3, 0x9C, 0xC3, 0x9D, 0xC3, 0x9E, /* 0xA4-0xA7 */ + 0xC8, 0xBF, 0xC3, 0x9F, 0xC3, 0xA0, 0xC4, 0x41, /* 0xA8-0xAB */ + 0xC8, 0xC0, 0xC4, 0x42, 0xC4, 0x43, 0xC4, 0x44, /* 0xAC-0xAF */ + 0xC8, 0xC1, 0xC4, 0x45, 0xC4, 0x46, 0xC4, 0x47, /* 0xB0-0xB3 */ + 0xC4, 0x48, 0xC4, 0x49, 0xC4, 0x4A, 0xC4, 0x4B, /* 0xB4-0xB7 */ + 0xC4, 0x4C, 0xC8, 0xC2, 0xC4, 0x4D, 0xC8, 0xC3, /* 0xB8-0xBB */ + 0xC4, 0x4E, 0xC4, 0x4F, 0xC4, 0x50, 0xC4, 0x51, /* 0xBC-0xBF */ + 0xC4, 0x52, 0xC4, 0x53, 0xC4, 0x54, 0xC4, 0x55, /* 0xC0-0xC3 */ + 0xC8, 0xC4, 0xC8, 0xC5, 0xC4, 0x56, 0xC4, 0x57, /* 0xC4-0xC7 */ + 0xC8, 0xC6, 0xC4, 0x58, 0xC4, 0x59, 0xC4, 0x5A, /* 0xC8-0xCB */ + 0xC8, 0xC7, 0xC4, 0x61, 0xC4, 0x62, 0xC4, 0x63, /* 0xCC-0xCF */ + 0xC4, 0x64, 0xC8, 0xC8, 0xC4, 0x65, 0xC4, 0x66, /* 0xD0-0xD3 */ + 0xC8, 0xC9, 0xC4, 0x67, 0xC4, 0x68, 0xC8, 0xCA, /* 0xD4-0xD7 */ + 0xC4, 0x69, 0xC8, 0xCB, 0xC4, 0x6A, 0xC4, 0x6B, /* 0xD8-0xDB */ + 0xC4, 0x6C, 0xC4, 0x6D, 0xC4, 0x6E, 0xC4, 0x6F, /* 0xDC-0xDF */ + 0xC8, 0xCC, 0xC4, 0x70, 0xC4, 0x71, 0xC4, 0x72, /* 0xE0-0xE3 */ + 0xC8, 0xCD, 0xC4, 0x73, 0xC4, 0x74, 0xC4, 0x75, /* 0xE4-0xE7 */ + 0xC8, 0xCE, 0xC4, 0x76, 0xC4, 0x77, 0xC4, 0x78, /* 0xE8-0xEB */ + 0xC4, 0x79, 0xC4, 0x7A, 0xC4, 0x81, 0xC4, 0x82, /* 0xEC-0xEF */ + 0xC8, 0xCF, 0xC4, 0x83, 0xC4, 0x84, 0xC4, 0x85, /* 0xF0-0xF3 */ + 0xC4, 0x86, 0xC8, 0xD0, 0xC4, 0x87, 0xC4, 0x88, /* 0xF4-0xF7 */ + 0xC4, 0x89, 0xC4, 0x8A, 0xC4, 0x8B, 0xC4, 0x8C, /* 0xF8-0xFB */ + 0xC8, 0xD1, 0xC8, 0xD2, 0xC4, 0x8D, 0xC4, 0x8E, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_D7[512] = { + 0xC8, 0xD3, 0xC4, 0x8F, 0xC4, 0x90, 0xC4, 0x91, /* 0x00-0x03 */ + 0xC8, 0xD4, 0xC4, 0x92, 0xC4, 0x93, 0xC4, 0x94, /* 0x04-0x07 */ + 0xC4, 0x95, 0xC4, 0x96, 0xC4, 0x97, 0xC4, 0x98, /* 0x08-0x0B */ + 0xC4, 0x99, 0xC4, 0x9A, 0xC4, 0x9B, 0xC4, 0x9C, /* 0x0C-0x0F */ + 0xC4, 0x9D, 0xC8, 0xD5, 0xC4, 0x9E, 0xC4, 0x9F, /* 0x10-0x13 */ + 0xC4, 0xA0, 0xC5, 0x41, 0xC5, 0x42, 0xC5, 0x43, /* 0x14-0x17 */ + 0xC8, 0xD6, 0xC8, 0xD7, 0xC5, 0x44, 0xC5, 0x45, /* 0x18-0x1B */ + 0xC8, 0xD8, 0xC5, 0x46, 0xC5, 0x47, 0xC5, 0x48, /* 0x1C-0x1F */ + 0xC8, 0xD9, 0xC5, 0x49, 0xC5, 0x4A, 0xC5, 0x4B, /* 0x20-0x23 */ + 0xC5, 0x4C, 0xC5, 0x4D, 0xC5, 0x4E, 0xC5, 0x4F, /* 0x24-0x27 */ + 0xC8, 0xDA, 0xC8, 0xDB, 0xC5, 0x50, 0xC8, 0xDC, /* 0x28-0x2B */ + 0xC5, 0x51, 0xC8, 0xDD, 0xC5, 0x52, 0xC5, 0x53, /* 0x2C-0x2F */ + 0xC5, 0x54, 0xC5, 0x55, 0xC5, 0x56, 0xC5, 0x57, /* 0x30-0x33 */ + 0xC8, 0xDE, 0xC8, 0xDF, 0xC5, 0x58, 0xC5, 0x59, /* 0x34-0x37 */ + 0xC8, 0xE0, 0xC5, 0x5A, 0xC5, 0x61, 0xC5, 0x62, /* 0x38-0x3B */ + 0xC8, 0xE1, 0xC5, 0x63, 0xC5, 0x64, 0xC5, 0x65, /* 0x3C-0x3F */ + 0xC5, 0x66, 0xC5, 0x67, 0xC5, 0x68, 0xC5, 0x69, /* 0x40-0x43 */ + 0xC8, 0xE2, 0xC5, 0x6A, 0xC5, 0x6B, 0xC8, 0xE3, /* 0x44-0x47 */ + 0xC5, 0x6C, 0xC8, 0xE4, 0xC5, 0x6D, 0xC5, 0x6E, /* 0x48-0x4B */ + 0xC5, 0x6F, 0xC5, 0x70, 0xC5, 0x71, 0xC5, 0x72, /* 0x4C-0x4F */ + 0xC8, 0xE5, 0xC8, 0xE6, 0xC5, 0x73, 0xC5, 0x74, /* 0x50-0x53 */ + 0xC8, 0xE7, 0xC5, 0x75, 0xC8, 0xE8, 0xC8, 0xE9, /* 0x54-0x57 */ + 0xC8, 0xEA, 0xC8, 0xEB, 0xC5, 0x76, 0xC5, 0x77, /* 0x58-0x5B */ + 0xC5, 0x78, 0xC5, 0x79, 0xC5, 0x7A, 0xC5, 0x81, /* 0x5C-0x5F */ + 0xC8, 0xEC, 0xC8, 0xED, 0xC5, 0x82, 0xC8, 0xEE, /* 0x60-0x63 */ + 0xC5, 0x83, 0xC8, 0xEF, 0xC5, 0x84, 0xC5, 0x85, /* 0x64-0x67 */ + 0xC5, 0x86, 0xC8, 0xF0, 0xC5, 0x87, 0xC5, 0x88, /* 0x68-0x6B */ + 0xC8, 0xF1, 0xC5, 0x89, 0xC5, 0x8A, 0xC5, 0x8B, /* 0x6C-0x6F */ + 0xC8, 0xF2, 0xC5, 0x8C, 0xC5, 0x8D, 0xC5, 0x8E, /* 0x70-0x73 */ + 0xC8, 0xF3, 0xC5, 0x8F, 0xC5, 0x90, 0xC5, 0x91, /* 0x74-0x77 */ + 0xC5, 0x92, 0xC5, 0x93, 0xC5, 0x94, 0xC5, 0x95, /* 0x78-0x7B */ + 0xC8, 0xF4, 0xC8, 0xF5, 0xC5, 0x96, 0xC5, 0x97, /* 0x7C-0x7F */ + + 0xC5, 0x98, 0xC8, 0xF6, 0xC5, 0x99, 0xC5, 0x9A, /* 0x80-0x83 */ + 0xC5, 0x9B, 0xC5, 0x9C, 0xC5, 0x9D, 0xC5, 0x9E, /* 0x84-0x87 */ + 0xC8, 0xF7, 0xC8, 0xF8, 0xC5, 0x9F, 0xC5, 0xA0, /* 0x88-0x8B */ + 0xC8, 0xF9, 0xC6, 0x41, 0xC6, 0x42, 0xC6, 0x43, /* 0x8C-0x8F */ + 0xC8, 0xFA, 0xC6, 0x44, 0xC6, 0x45, 0xC6, 0x46, /* 0x90-0x93 */ + 0xC6, 0x47, 0xC6, 0x48, 0xC6, 0x49, 0xC6, 0x4A, /* 0x94-0x97 */ + 0xC8, 0xFB, 0xC8, 0xFC, 0xC6, 0x4B, 0xC8, 0xFD, /* 0x98-0x9B */ + 0xC6, 0x4C, 0xC8, 0xFE, 0xC6, 0x4D, 0xC6, 0x4E, /* 0x9C-0x9F */ + 0xC6, 0x4F, 0xC6, 0x50, 0xC6, 0x51, 0xC6, 0x52, /* 0xA0-0xA3 */ +}; + +static const unsigned char u2c_DC[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ +}; + +static const unsigned char u2c_F9[512] = { + 0xCB, 0xD0, 0xCB, 0xD6, 0xCB, 0xE7, 0xCD, 0xCF, /* 0x00-0x03 */ + 0xCD, 0xE8, 0xCE, 0xAD, 0xCF, 0xFB, 0xD0, 0xA2, /* 0x04-0x07 */ + 0xD0, 0xB8, 0xD0, 0xD0, 0xD0, 0xDD, 0xD1, 0xD4, /* 0x08-0x0B */ + 0xD1, 0xD5, 0xD1, 0xD8, 0xD1, 0xDB, 0xD1, 0xDC, /* 0x0C-0x0F */ + 0xD1, 0xDD, 0xD1, 0xDE, 0xD1, 0xDF, 0xD1, 0xE0, /* 0x10-0x13 */ + 0xD1, 0xE2, 0xD1, 0xE3, 0xD1, 0xE4, 0xD1, 0xE5, /* 0x14-0x17 */ + 0xD1, 0xE6, 0xD1, 0xE8, 0xD1, 0xE9, 0xD1, 0xEA, /* 0x18-0x1B */ + 0xD1, 0xEB, 0xD1, 0xED, 0xD1, 0xEF, 0xD1, 0xF0, /* 0x1C-0x1F */ + 0xD1, 0xF2, 0xD1, 0xF6, 0xD1, 0xFA, 0xD1, 0xFC, /* 0x20-0x23 */ + 0xD1, 0xFD, 0xD1, 0xFE, 0xD2, 0xA2, 0xD2, 0xA3, /* 0x24-0x27 */ + 0xD2, 0xA7, 0xD2, 0xA8, 0xD2, 0xA9, 0xD2, 0xAA, /* 0x28-0x2B */ + 0xD2, 0xAB, 0xD2, 0xAD, 0xD2, 0xB2, 0xD2, 0xBE, /* 0x2C-0x2F */ + 0xD2, 0xC2, 0xD2, 0xC3, 0xD2, 0xC4, 0xD2, 0xC6, /* 0x30-0x33 */ + 0xD2, 0xC7, 0xD2, 0xC8, 0xD2, 0xC9, 0xD2, 0xCA, /* 0x34-0x37 */ + 0xD2, 0xCB, 0xD2, 0xCD, 0xD2, 0xCE, 0xD2, 0xCF, /* 0x38-0x3B */ + 0xD2, 0xD0, 0xD2, 0xD1, 0xD2, 0xD2, 0xD2, 0xD3, /* 0x3C-0x3F */ + 0xD2, 0xD4, 0xD2, 0xD5, 0xD2, 0xD6, 0xD2, 0xD7, /* 0x40-0x43 */ + 0xD2, 0xD9, 0xD2, 0xDA, 0xD2, 0xDE, 0xD2, 0xDF, /* 0x44-0x47 */ + 0xD2, 0xE1, 0xD2, 0xE2, 0xD2, 0xE4, 0xD2, 0xE5, /* 0x48-0x4B */ + 0xD2, 0xE6, 0xD2, 0xE7, 0xD2, 0xE8, 0xD2, 0xE9, /* 0x4C-0x4F */ + 0xD2, 0xEA, 0xD2, 0xEB, 0xD2, 0xF0, 0xD2, 0xF1, /* 0x50-0x53 */ + 0xD2, 0xF2, 0xD2, 0xF3, 0xD2, 0xF4, 0xD2, 0xF5, /* 0x54-0x57 */ + 0xD2, 0xF7, 0xD2, 0xF8, 0xD4, 0xE6, 0xD4, 0xFC, /* 0x58-0x5B */ + 0xD5, 0xA5, 0xD5, 0xAB, 0xD5, 0xAE, 0xD6, 0xB8, /* 0x5C-0x5F */ + 0xD6, 0xCD, 0xD7, 0xCB, 0xD7, 0xE4, 0xDB, 0xC5, /* 0x60-0x63 */ + 0xDB, 0xE4, 0xDC, 0xA5, 0xDD, 0xA5, 0xDD, 0xD5, /* 0x64-0x67 */ + 0xDD, 0xF4, 0xDE, 0xFC, 0xDE, 0xFE, 0xDF, 0xB3, /* 0x68-0x6B */ + 0xDF, 0xE1, 0xDF, 0xE8, 0xE0, 0xF1, 0xE1, 0xAD, /* 0x6C-0x6F */ + 0xE1, 0xED, 0xE3, 0xF5, 0xE4, 0xA1, 0xE4, 0xA9, /* 0x70-0x73 */ + 0xE5, 0xAE, 0xE5, 0xB1, 0xE5, 0xB2, 0xE5, 0xB9, /* 0x74-0x77 */ + 0xE5, 0xBB, 0xE5, 0xBC, 0xE5, 0xC4, 0xE5, 0xCE, /* 0x78-0x7B */ + 0xE5, 0xD0, 0xE5, 0xD2, 0xE5, 0xD6, 0xE5, 0xFA, /* 0x7C-0x7F */ + + 0xE5, 0xFB, 0xE5, 0xFC, 0xE5, 0xFE, 0xE6, 0xA1, /* 0x80-0x83 */ + 0xE6, 0xA4, 0xE6, 0xA7, 0xE6, 0xAD, 0xE6, 0xAF, /* 0x84-0x87 */ + 0xE6, 0xB0, 0xE6, 0xB1, 0xE6, 0xB3, 0xE6, 0xB7, /* 0x88-0x8B */ + 0xE6, 0xB8, 0xE6, 0xBC, 0xE6, 0xC4, 0xE6, 0xC6, /* 0x8C-0x8F */ + 0xE6, 0xC7, 0xE6, 0xCA, 0xE6, 0xD2, 0xE6, 0xD6, /* 0x90-0x93 */ + 0xE6, 0xD9, 0xE6, 0xDC, 0xE6, 0xDF, 0xE6, 0xE1, /* 0x94-0x97 */ + 0xE6, 0xE4, 0xE6, 0xE5, 0xE6, 0xE6, 0xE6, 0xE8, /* 0x98-0x9B */ + 0xE6, 0xEA, 0xE6, 0xEB, 0xE6, 0xEC, 0xE6, 0xEF, /* 0x9C-0x9F */ + 0xE6, 0xF1, 0xE6, 0xF2, 0xE6, 0xF5, 0xE6, 0xF6, /* 0xA0-0xA3 */ + 0xE6, 0xF7, 0xE6, 0xF9, 0xE7, 0xA1, 0xE7, 0xA6, /* 0xA4-0xA7 */ + 0xE7, 0xA9, 0xE7, 0xAA, 0xE7, 0xAC, 0xE7, 0xAD, /* 0xA8-0xAB */ + 0xE7, 0xB0, 0xE7, 0xBF, 0xE7, 0xC1, 0xE7, 0xC6, /* 0xAC-0xAF */ + 0xE7, 0xC7, 0xE7, 0xCB, 0xE7, 0xCD, 0xE7, 0xCF, /* 0xB0-0xB3 */ + 0xE7, 0xD0, 0xE7, 0xD3, 0xE7, 0xDF, 0xE7, 0xE4, /* 0xB4-0xB7 */ + 0xE7, 0xE6, 0xE7, 0xF7, 0xE8, 0xE7, 0xE8, 0xE8, /* 0xB8-0xBB */ + 0xE8, 0xF0, 0xE8, 0xF1, 0xE8, 0xF7, 0xE8, 0xF9, /* 0xBC-0xBF */ + 0xE8, 0xFB, 0xE8, 0xFE, 0xE9, 0xA7, 0xE9, 0xAC, /* 0xC0-0xC3 */ + 0xE9, 0xCC, 0xE9, 0xF7, 0xEA, 0xC1, 0xEA, 0xE5, /* 0xC4-0xC7 */ + 0xEA, 0xF4, 0xEA, 0xF7, 0xEA, 0xFC, 0xEA, 0xFE, /* 0xC8-0xCB */ + 0xEB, 0xA4, 0xEB, 0xA7, 0xEB, 0xA9, 0xEB, 0xAA, /* 0xCC-0xCF */ + 0xEB, 0xBA, 0xEB, 0xBB, 0xEB, 0xBD, 0xEB, 0xC1, /* 0xD0-0xD3 */ + 0xEB, 0xC2, 0xEB, 0xC6, 0xEB, 0xC7, 0xEB, 0xCC, /* 0xD4-0xD7 */ + 0xEB, 0xCF, 0xEB, 0xD0, 0xEB, 0xD1, 0xEB, 0xD2, /* 0xD8-0xDB */ + 0xEB, 0xD8, 0xEC, 0xA6, 0xEC, 0xA7, 0xEC, 0xAA, /* 0xDC-0xDF */ + 0xEC, 0xAF, 0xEC, 0xB0, 0xEC, 0xB1, 0xEC, 0xB2, /* 0xE0-0xE3 */ + 0xEC, 0xB5, 0xEC, 0xB8, 0xEC, 0xBA, 0xEC, 0xC0, /* 0xE4-0xE7 */ + 0xEC, 0xC1, 0xEC, 0xC5, 0xEC, 0xC6, 0xEC, 0xC9, /* 0xE8-0xEB */ + 0xEC, 0xCA, 0xEC, 0xD5, 0xEC, 0xDD, 0xEC, 0xDE, /* 0xEC-0xEF */ + 0xEC, 0xE1, 0xEC, 0xE4, 0xEC, 0xE7, 0xEC, 0xE8, /* 0xF0-0xF3 */ + 0xEC, 0xF7, 0xEC, 0xF8, 0xEC, 0xFA, 0xED, 0xA1, /* 0xF4-0xF7 */ + 0xED, 0xA2, 0xED, 0xA3, 0xED, 0xEE, 0xEE, 0xDB, /* 0xF8-0xFB */ + 0xF2, 0xBD, 0xF2, 0xFA, 0xF3, 0xB1, 0xF4, 0xA7, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_FA[512] = { + 0xF4, 0xEE, 0xF6, 0xF4, 0xF6, 0xF6, 0xF7, 0xB8, /* 0x00-0x03 */ + 0xF7, 0xC8, 0xF7, 0xD3, 0xF8, 0xDB, 0xF8, 0xF0, /* 0x04-0x07 */ + 0xFA, 0xA1, 0xFA, 0xA2, 0xFA, 0xE6, 0xFC, 0xA9, /* 0x08-0x0B */ + 0xE8, 0xB4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0xF5, 0xC0, 0x00, 0x00, 0xF4, 0xE7, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0xFD, 0xEB, 0x00, 0x00, 0xEC, 0xCC, /* 0x14-0x17 */ + 0x00, 0x00, 0xE3, 0xEA, 0xDF, 0xD4, 0xDC, 0xD8, /* 0x18-0x1B */ + 0xEF, 0xFE, 0xEF, 0xF1, 0xE9, 0xE2, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0xF0, 0xB3, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0xEC, 0xEF, 0xD4, 0xB4, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xDA, 0xF9, 0xDE, 0xF8, /* 0x28-0x2B */ + 0xCE, 0xBD, 0xF9, 0xCD, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ +}; + +static const unsigned char u2c_FF[512] = { + 0x00, 0x00, 0xA3, 0xA1, 0xA3, 0xA2, 0xA3, 0xA3, /* 0x00-0x03 */ + 0xA3, 0xA4, 0xA3, 0xA5, 0xA3, 0xA6, 0xA3, 0xA7, /* 0x04-0x07 */ + 0xA3, 0xA8, 0xA3, 0xA9, 0xA3, 0xAA, 0xA3, 0xAB, /* 0x08-0x0B */ + 0xA3, 0xAC, 0xA3, 0xAD, 0xA3, 0xAE, 0xA3, 0xAF, /* 0x0C-0x0F */ + 0xA3, 0xB0, 0xA3, 0xB1, 0xA3, 0xB2, 0xA3, 0xB3, /* 0x10-0x13 */ + 0xA3, 0xB4, 0xA3, 0xB5, 0xA3, 0xB6, 0xA3, 0xB7, /* 0x14-0x17 */ + 0xA3, 0xB8, 0xA3, 0xB9, 0xA3, 0xBA, 0xA3, 0xBB, /* 0x18-0x1B */ + 0xA3, 0xBC, 0xA3, 0xBD, 0xA3, 0xBE, 0xA3, 0xBF, /* 0x1C-0x1F */ + 0xA3, 0xC0, 0xA3, 0xC1, 0xA3, 0xC2, 0xA3, 0xC3, /* 0x20-0x23 */ + 0xA3, 0xC4, 0xA3, 0xC5, 0xA3, 0xC6, 0xA3, 0xC7, /* 0x24-0x27 */ + 0xA3, 0xC8, 0xA3, 0xC9, 0xA3, 0xCA, 0xA3, 0xCB, /* 0x28-0x2B */ + 0xA3, 0xCC, 0xA3, 0xCD, 0xA3, 0xCE, 0xA3, 0xCF, /* 0x2C-0x2F */ + 0xA3, 0xD0, 0xA3, 0xD1, 0xA3, 0xD2, 0xA3, 0xD3, /* 0x30-0x33 */ + 0xA3, 0xD4, 0xA3, 0xD5, 0xA3, 0xD6, 0xA3, 0xD7, /* 0x34-0x37 */ + 0xA3, 0xD8, 0xA3, 0xD9, 0xA3, 0xDA, 0xA3, 0xDB, /* 0x38-0x3B */ + 0xA1, 0xAC, 0xA3, 0xDD, 0xA3, 0xDE, 0xA3, 0xDF, /* 0x3C-0x3F */ + 0xA3, 0xE0, 0xA3, 0xE1, 0xA3, 0xE2, 0xA3, 0xE3, /* 0x40-0x43 */ + 0xA3, 0xE4, 0xA3, 0xE5, 0xA3, 0xE6, 0xA3, 0xE7, /* 0x44-0x47 */ + 0xA3, 0xE8, 0xA3, 0xE9, 0xA3, 0xEA, 0xA3, 0xEB, /* 0x48-0x4B */ + 0xA3, 0xEC, 0xA3, 0xED, 0xA3, 0xEE, 0xA3, 0xEF, /* 0x4C-0x4F */ + 0xA3, 0xF0, 0xA3, 0xF1, 0xA3, 0xF2, 0xA3, 0xF3, /* 0x50-0x53 */ + 0xA3, 0xF4, 0xA3, 0xF5, 0xA3, 0xF6, 0xA3, 0xF7, /* 0x54-0x57 */ + 0xA3, 0xF8, 0xA3, 0xF9, 0xA3, 0xFA, 0xA3, 0xFB, /* 0x58-0x5B */ + 0xA3, 0xFC, 0xA3, 0xFD, 0xA2, 0xA6, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0xA4, 0xD4, 0xA4, 0xA1, 0xA4, 0xA2, 0xA4, 0xA3, /* 0xA0-0xA3 */ + 0xA4, 0xA4, 0xA4, 0xA5, 0xA4, 0xA6, 0xA4, 0xA7, /* 0xA4-0xA7 */ + 0xA4, 0xA8, 0xA4, 0xA9, 0xA4, 0xAA, 0xA4, 0xAB, /* 0xA8-0xAB */ + 0xA4, 0xAC, 0xA4, 0xAD, 0xA4, 0xAE, 0xA4, 0xAF, /* 0xAC-0xAF */ + 0xA4, 0xB0, 0xA4, 0xB1, 0xA4, 0xB2, 0xA4, 0xB3, /* 0xB0-0xB3 */ + 0xA4, 0xB4, 0xA4, 0xB5, 0xA4, 0xB6, 0xA4, 0xB7, /* 0xB4-0xB7 */ + 0xA4, 0xB8, 0xA4, 0xB9, 0xA4, 0xBA, 0xA4, 0xBB, /* 0xB8-0xBB */ + 0xA4, 0xBC, 0xA4, 0xBD, 0xA4, 0xBE, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0xA4, 0xBF, 0xA4, 0xC0, /* 0xC0-0xC3 */ + 0xA4, 0xC1, 0xA4, 0xC2, 0xA4, 0xC3, 0xA4, 0xC4, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0xA4, 0xC5, 0xA4, 0xC6, /* 0xC8-0xCB */ + 0xA4, 0xC7, 0xA4, 0xC8, 0xA4, 0xC9, 0xA4, 0xCA, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xA4, 0xCB, 0xA4, 0xCC, /* 0xD0-0xD3 */ + 0xA4, 0xCD, 0xA4, 0xCE, 0xA4, 0xCF, 0xA4, 0xD0, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xA4, 0xD1, 0xA4, 0xD2, /* 0xD8-0xDB */ + 0xA4, 0xD3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0xA1, 0xCB, 0xA1, 0xCC, 0xA1, 0xFE, 0xA3, 0xFE, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xA1, 0xCD, 0xA3, 0xDC, 0x00, 0x00, /* 0xE4-0xE7 */ +}; + +static const unsigned char *const page_uni2charset[256] = { + NULL, u2c_01, u2c_02, u2c_03, u2c_04, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, u2c_11, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + u2c_20, u2c_21, u2c_22, u2c_23, u2c_24, u2c_25, u2c_26, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + u2c_30, u2c_31, u2c_32, u2c_33, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, u2c_4E, u2c_4F, + u2c_50, u2c_51, u2c_52, u2c_53, u2c_54, u2c_55, u2c_56, u2c_57, + u2c_58, u2c_59, u2c_5A, u2c_5B, u2c_5C, u2c_5D, u2c_5E, u2c_5F, + u2c_60, u2c_61, u2c_62, u2c_63, u2c_64, u2c_65, u2c_66, u2c_67, + u2c_68, u2c_69, u2c_6A, u2c_6B, u2c_6C, u2c_6D, u2c_6E, u2c_6F, + u2c_70, u2c_71, u2c_72, u2c_73, u2c_74, u2c_75, u2c_76, u2c_77, + u2c_78, u2c_79, u2c_7A, u2c_7B, u2c_7C, u2c_7D, u2c_7E, u2c_7F, + u2c_80, u2c_81, u2c_82, u2c_83, u2c_84, u2c_85, u2c_86, u2c_87, + u2c_88, u2c_89, u2c_8A, u2c_8B, u2c_8C, u2c_8D, u2c_8E, u2c_8F, + u2c_90, u2c_91, u2c_92, u2c_93, u2c_94, u2c_95, u2c_96, u2c_97, + u2c_98, u2c_99, u2c_9A, u2c_9B, u2c_9C, u2c_9D, u2c_9E, u2c_9F, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, u2c_AC, u2c_AD, u2c_AE, u2c_AF, + u2c_B0, u2c_B1, u2c_B2, u2c_B3, u2c_B4, u2c_B5, u2c_B6, u2c_B7, + u2c_B8, u2c_B9, u2c_BA, u2c_BB, u2c_BC, u2c_BD, u2c_BE, u2c_BF, + u2c_C0, u2c_C1, u2c_C2, u2c_C3, u2c_C4, u2c_C5, u2c_C6, u2c_C7, + u2c_C8, u2c_C9, u2c_CA, u2c_CB, u2c_CC, u2c_CD, u2c_CE, u2c_CF, + u2c_D0, u2c_D1, u2c_D2, u2c_D3, u2c_D4, u2c_D5, u2c_D6, u2c_D7, + NULL, NULL, NULL, NULL, u2c_DC, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, u2c_F9, u2c_FA, NULL, NULL, NULL, NULL, u2c_FF, }; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(const wchar_t uni, + unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni&0xFF; + unsigned char ch = (uni>>8)&0xFF; + int n; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + + uni2charset = page_uni2charset[ch]; + if (uni2charset) { + if (boundlen <= 1) + return -ENAMETOOLONG; + out[0] = uni2charset[cl*2]; + out[1] = uni2charset[cl*2+1]; + if (out[0] == 0x00 && out[1] == 0x00) + return -EINVAL; + n = 2; + } else if (ch==0 && cl) { + out[0] = cl; + n = 1; + } + else + return -EINVAL; + + return n; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, + wchar_t *uni) +{ + unsigned char ch, cl; + const wchar_t *charset2uni; + int n; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + if (boundlen == 1) { + *uni = rawstring[0]; + return 1; + } + + ch = rawstring[0]; + cl = rawstring[1]; + + charset2uni = page_charset2uni[ch]; + if (charset2uni && cl) { + *uni = charset2uni[cl]; + if (*uni == 0x0000) + return -EINVAL; + n = 2; + } else{ + *uni = ch; + n = 1; + } + return n; +} + +static struct nls_table table = { + .charset = "cp949", + .alias = "euc-kr", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_cp949(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_cp949(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_cp949) +module_exit(exit_nls_cp949) + +MODULE_DESCRIPTION("NLS Korean charset (CP949, EUC-KR)"); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_ALIAS_NLS(euc-kr); diff --git a/fs/nls/nls_cp950.c b/fs/nls/nls_cp950.c new file mode 100644 index 000000000..e968aa801 --- /dev/null +++ b/fs/nls/nls_cp950.c @@ -0,0 +1,9483 @@ +/* + * linux/fs/nls/nls_cp950.c + * + * Charset cp950 translation tables. + * This translation table was generated automatically, the + * original table can be download from the Microsoft website. + * (http://www.microsoft.com/typography/unicode/unicodecp.htm) + */ + +#include +#include +#include +#include +#include + +static const wchar_t c2u_A1[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x3000,0xFF0C,0x3001,0x3002,0xFF0E,0x2027,0xFF1B,0xFF1A,/* 0x40-0x47 */ + 0xFF1F,0xFF01,0xFE30,0x2026,0x2025,0xFE50,0xFE51,0xFE52,/* 0x48-0x4F */ + 0x00B7,0xFE54,0xFE55,0xFE56,0xFE57,0xFF5C,0x2013,0xFE31,/* 0x50-0x57 */ + 0x2014,0xFE33,0x2574,0xFE34,0xFE4F,0xFF08,0xFF09,0xFE35,/* 0x58-0x5F */ + 0xFE36,0xFF5B,0xFF5D,0xFE37,0xFE38,0x3014,0x3015,0xFE39,/* 0x60-0x67 */ + 0xFE3A,0x3010,0x3011,0xFE3B,0xFE3C,0x300A,0x300B,0xFE3D,/* 0x68-0x6F */ + 0xFE3E,0x3008,0x3009,0xFE3F,0xFE40,0x300C,0x300D,0xFE41,/* 0x70-0x77 */ + 0xFE42,0x300E,0x300F,0xFE43,0xFE44,0xFE59,0xFE5A,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0xFE5B,0xFE5C,0xFE5D,0xFE5E,0x2018,0x2019,0x201C,/* 0xA0-0xA7 */ + 0x201D,0x301D,0x301E,0x2035,0x2032,0xFF03,0xFF06,0xFF0A,/* 0xA8-0xAF */ + 0x203B,0x00A7,0x3003,0x25CB,0x25CF,0x25B3,0x25B2,0x25CE,/* 0xB0-0xB7 */ + 0x2606,0x2605,0x25C7,0x25C6,0x25A1,0x25A0,0x25BD,0x25BC,/* 0xB8-0xBF */ + 0x32A3,0x2105,0x00AF,0xFFE3,0xFF3F,0x02CD,0xFE49,0xFE4A,/* 0xC0-0xC7 */ + 0xFE4D,0xFE4E,0xFE4B,0xFE4C,0xFE5F,0xFE60,0xFE61,0xFF0B,/* 0xC8-0xCF */ + 0xFF0D,0x00D7,0x00F7,0x00B1,0x221A,0xFF1C,0xFF1E,0xFF1D,/* 0xD0-0xD7 */ + 0x2266,0x2267,0x2260,0x221E,0x2252,0x2261,0xFE62,0xFE63,/* 0xD8-0xDF */ + 0xFE64,0xFE65,0xFE66,0xFF5E,0x2229,0x222A,0x22A5,0x2220,/* 0xE0-0xE7 */ + 0x221F,0x22BF,0x33D2,0x33D1,0x222B,0x222E,0x2235,0x2234,/* 0xE8-0xEF */ + 0x2640,0x2642,0x2295,0x2299,0x2191,0x2193,0x2190,0x2192,/* 0xF0-0xF7 */ + 0x2196,0x2197,0x2199,0x2198,0x2225,0x2223,0xFF0F,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_A2[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0xFF3C,0x2215,0xFE68,0xFF04,0xFFE5,0x3012,0xFFE0,0xFFE1,/* 0x40-0x47 */ + 0xFF05,0xFF20,0x2103,0x2109,0xFE69,0xFE6A,0xFE6B,0x33D5,/* 0x48-0x4F */ + 0x339C,0x339D,0x339E,0x33CE,0x33A1,0x338E,0x338F,0x33C4,/* 0x50-0x57 */ + 0x00B0,0x5159,0x515B,0x515E,0x515D,0x5161,0x5163,0x55E7,/* 0x58-0x5F */ + 0x74E9,0x7CCE,0x2581,0x2582,0x2583,0x2584,0x2585,0x2586,/* 0x60-0x67 */ + 0x2587,0x2588,0x258F,0x258E,0x258D,0x258C,0x258B,0x258A,/* 0x68-0x6F */ + 0x2589,0x253C,0x2534,0x252C,0x2524,0x251C,0x2594,0x2500,/* 0x70-0x77 */ + 0x2502,0x2595,0x250C,0x2510,0x2514,0x2518,0x256D,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x256E,0x2570,0x256F,0x2550,0x255E,0x256A,0x2561,/* 0xA0-0xA7 */ + 0x25E2,0x25E3,0x25E5,0x25E4,0x2571,0x2572,0x2573,0xFF10,/* 0xA8-0xAF */ + 0xFF11,0xFF12,0xFF13,0xFF14,0xFF15,0xFF16,0xFF17,0xFF18,/* 0xB0-0xB7 */ + 0xFF19,0x2160,0x2161,0x2162,0x2163,0x2164,0x2165,0x2166,/* 0xB8-0xBF */ + 0x2167,0x2168,0x2169,0x3021,0x3022,0x3023,0x3024,0x3025,/* 0xC0-0xC7 */ + 0x3026,0x3027,0x3028,0x3029,0x5341,0x5344,0x5345,0xFF21,/* 0xC8-0xCF */ + 0xFF22,0xFF23,0xFF24,0xFF25,0xFF26,0xFF27,0xFF28,0xFF29,/* 0xD0-0xD7 */ + 0xFF2A,0xFF2B,0xFF2C,0xFF2D,0xFF2E,0xFF2F,0xFF30,0xFF31,/* 0xD8-0xDF */ + 0xFF32,0xFF33,0xFF34,0xFF35,0xFF36,0xFF37,0xFF38,0xFF39,/* 0xE0-0xE7 */ + 0xFF3A,0xFF41,0xFF42,0xFF43,0xFF44,0xFF45,0xFF46,0xFF47,/* 0xE8-0xEF */ + 0xFF48,0xFF49,0xFF4A,0xFF4B,0xFF4C,0xFF4D,0xFF4E,0xFF4F,/* 0xF0-0xF7 */ + 0xFF50,0xFF51,0xFF52,0xFF53,0xFF54,0xFF55,0xFF56,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_A3[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0xFF57,0xFF58,0xFF59,0xFF5A,0x0391,0x0392,0x0393,0x0394,/* 0x40-0x47 */ + 0x0395,0x0396,0x0397,0x0398,0x0399,0x039A,0x039B,0x039C,/* 0x48-0x4F */ + 0x039D,0x039E,0x039F,0x03A0,0x03A1,0x03A3,0x03A4,0x03A5,/* 0x50-0x57 */ + 0x03A6,0x03A7,0x03A8,0x03A9,0x03B1,0x03B2,0x03B3,0x03B4,/* 0x58-0x5F */ + 0x03B5,0x03B6,0x03B7,0x03B8,0x03B9,0x03BA,0x03BB,0x03BC,/* 0x60-0x67 */ + 0x03BD,0x03BE,0x03BF,0x03C0,0x03C1,0x03C3,0x03C4,0x03C5,/* 0x68-0x6F */ + 0x03C6,0x03C7,0x03C8,0x03C9,0x3105,0x3106,0x3107,0x3108,/* 0x70-0x77 */ + 0x3109,0x310A,0x310B,0x310C,0x310D,0x310E,0x310F,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x3110,0x3111,0x3112,0x3113,0x3114,0x3115,0x3116,/* 0xA0-0xA7 */ + 0x3117,0x3118,0x3119,0x311A,0x311B,0x311C,0x311D,0x311E,/* 0xA8-0xAF */ + 0x311F,0x3120,0x3121,0x3122,0x3123,0x3124,0x3125,0x3126,/* 0xB0-0xB7 */ + 0x3127,0x3128,0x3129,0x02D9,0x02C9,0x02CA,0x02C7,0x02CB,/* 0xB8-0xBF */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xC0-0xC7 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xC8-0xCF */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xD0-0xD7 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xD8-0xDF */ + 0x0000,0x20AC,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0xE0-0xE7 */ +}; + +static const wchar_t c2u_A4[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x4E00,0x4E59,0x4E01,0x4E03,0x4E43,0x4E5D,0x4E86,0x4E8C,/* 0x40-0x47 */ + 0x4EBA,0x513F,0x5165,0x516B,0x51E0,0x5200,0x5201,0x529B,/* 0x48-0x4F */ + 0x5315,0x5341,0x535C,0x53C8,0x4E09,0x4E0B,0x4E08,0x4E0A,/* 0x50-0x57 */ + 0x4E2B,0x4E38,0x51E1,0x4E45,0x4E48,0x4E5F,0x4E5E,0x4E8E,/* 0x58-0x5F */ + 0x4EA1,0x5140,0x5203,0x52FA,0x5343,0x53C9,0x53E3,0x571F,/* 0x60-0x67 */ + 0x58EB,0x5915,0x5927,0x5973,0x5B50,0x5B51,0x5B53,0x5BF8,/* 0x68-0x6F */ + 0x5C0F,0x5C22,0x5C38,0x5C71,0x5DDD,0x5DE5,0x5DF1,0x5DF2,/* 0x70-0x77 */ + 0x5DF3,0x5DFE,0x5E72,0x5EFE,0x5F0B,0x5F13,0x624D,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x4E11,0x4E10,0x4E0D,0x4E2D,0x4E30,0x4E39,0x4E4B,/* 0xA0-0xA7 */ + 0x5C39,0x4E88,0x4E91,0x4E95,0x4E92,0x4E94,0x4EA2,0x4EC1,/* 0xA8-0xAF */ + 0x4EC0,0x4EC3,0x4EC6,0x4EC7,0x4ECD,0x4ECA,0x4ECB,0x4EC4,/* 0xB0-0xB7 */ + 0x5143,0x5141,0x5167,0x516D,0x516E,0x516C,0x5197,0x51F6,/* 0xB8-0xBF */ + 0x5206,0x5207,0x5208,0x52FB,0x52FE,0x52FF,0x5316,0x5339,/* 0xC0-0xC7 */ + 0x5348,0x5347,0x5345,0x535E,0x5384,0x53CB,0x53CA,0x53CD,/* 0xC8-0xCF */ + 0x58EC,0x5929,0x592B,0x592A,0x592D,0x5B54,0x5C11,0x5C24,/* 0xD0-0xD7 */ + 0x5C3A,0x5C6F,0x5DF4,0x5E7B,0x5EFF,0x5F14,0x5F15,0x5FC3,/* 0xD8-0xDF */ + 0x6208,0x6236,0x624B,0x624E,0x652F,0x6587,0x6597,0x65A4,/* 0xE0-0xE7 */ + 0x65B9,0x65E5,0x66F0,0x6708,0x6728,0x6B20,0x6B62,0x6B79,/* 0xE8-0xEF */ + 0x6BCB,0x6BD4,0x6BDB,0x6C0F,0x6C34,0x706B,0x722A,0x7236,/* 0xF0-0xF7 */ + 0x723B,0x7247,0x7259,0x725B,0x72AC,0x738B,0x4E19,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_A5[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x4E16,0x4E15,0x4E14,0x4E18,0x4E3B,0x4E4D,0x4E4F,0x4E4E,/* 0x40-0x47 */ + 0x4EE5,0x4ED8,0x4ED4,0x4ED5,0x4ED6,0x4ED7,0x4EE3,0x4EE4,/* 0x48-0x4F */ + 0x4ED9,0x4EDE,0x5145,0x5144,0x5189,0x518A,0x51AC,0x51F9,/* 0x50-0x57 */ + 0x51FA,0x51F8,0x520A,0x52A0,0x529F,0x5305,0x5306,0x5317,/* 0x58-0x5F */ + 0x531D,0x4EDF,0x534A,0x5349,0x5361,0x5360,0x536F,0x536E,/* 0x60-0x67 */ + 0x53BB,0x53EF,0x53E4,0x53F3,0x53EC,0x53EE,0x53E9,0x53E8,/* 0x68-0x6F */ + 0x53FC,0x53F8,0x53F5,0x53EB,0x53E6,0x53EA,0x53F2,0x53F1,/* 0x70-0x77 */ + 0x53F0,0x53E5,0x53ED,0x53FB,0x56DB,0x56DA,0x5916,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x592E,0x5931,0x5974,0x5976,0x5B55,0x5B83,0x5C3C,/* 0xA0-0xA7 */ + 0x5DE8,0x5DE7,0x5DE6,0x5E02,0x5E03,0x5E73,0x5E7C,0x5F01,/* 0xA8-0xAF */ + 0x5F18,0x5F17,0x5FC5,0x620A,0x6253,0x6254,0x6252,0x6251,/* 0xB0-0xB7 */ + 0x65A5,0x65E6,0x672E,0x672C,0x672A,0x672B,0x672D,0x6B63,/* 0xB8-0xBF */ + 0x6BCD,0x6C11,0x6C10,0x6C38,0x6C41,0x6C40,0x6C3E,0x72AF,/* 0xC0-0xC7 */ + 0x7384,0x7389,0x74DC,0x74E6,0x7518,0x751F,0x7528,0x7529,/* 0xC8-0xCF */ + 0x7530,0x7531,0x7532,0x7533,0x758B,0x767D,0x76AE,0x76BF,/* 0xD0-0xD7 */ + 0x76EE,0x77DB,0x77E2,0x77F3,0x793A,0x79BE,0x7A74,0x7ACB,/* 0xD8-0xDF */ + 0x4E1E,0x4E1F,0x4E52,0x4E53,0x4E69,0x4E99,0x4EA4,0x4EA6,/* 0xE0-0xE7 */ + 0x4EA5,0x4EFF,0x4F09,0x4F19,0x4F0A,0x4F15,0x4F0D,0x4F10,/* 0xE8-0xEF */ + 0x4F11,0x4F0F,0x4EF2,0x4EF6,0x4EFB,0x4EF0,0x4EF3,0x4EFD,/* 0xF0-0xF7 */ + 0x4F01,0x4F0B,0x5149,0x5147,0x5146,0x5148,0x5168,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_A6[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x5171,0x518D,0x51B0,0x5217,0x5211,0x5212,0x520E,0x5216,/* 0x40-0x47 */ + 0x52A3,0x5308,0x5321,0x5320,0x5370,0x5371,0x5409,0x540F,/* 0x48-0x4F */ + 0x540C,0x540A,0x5410,0x5401,0x540B,0x5404,0x5411,0x540D,/* 0x50-0x57 */ + 0x5408,0x5403,0x540E,0x5406,0x5412,0x56E0,0x56DE,0x56DD,/* 0x58-0x5F */ + 0x5733,0x5730,0x5728,0x572D,0x572C,0x572F,0x5729,0x5919,/* 0x60-0x67 */ + 0x591A,0x5937,0x5938,0x5984,0x5978,0x5983,0x597D,0x5979,/* 0x68-0x6F */ + 0x5982,0x5981,0x5B57,0x5B58,0x5B87,0x5B88,0x5B85,0x5B89,/* 0x70-0x77 */ + 0x5BFA,0x5C16,0x5C79,0x5DDE,0x5E06,0x5E76,0x5E74,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x5F0F,0x5F1B,0x5FD9,0x5FD6,0x620E,0x620C,0x620D,/* 0xA0-0xA7 */ + 0x6210,0x6263,0x625B,0x6258,0x6536,0x65E9,0x65E8,0x65EC,/* 0xA8-0xAF */ + 0x65ED,0x66F2,0x66F3,0x6709,0x673D,0x6734,0x6731,0x6735,/* 0xB0-0xB7 */ + 0x6B21,0x6B64,0x6B7B,0x6C16,0x6C5D,0x6C57,0x6C59,0x6C5F,/* 0xB8-0xBF */ + 0x6C60,0x6C50,0x6C55,0x6C61,0x6C5B,0x6C4D,0x6C4E,0x7070,/* 0xC0-0xC7 */ + 0x725F,0x725D,0x767E,0x7AF9,0x7C73,0x7CF8,0x7F36,0x7F8A,/* 0xC8-0xCF */ + 0x7FBD,0x8001,0x8003,0x800C,0x8012,0x8033,0x807F,0x8089,/* 0xD0-0xD7 */ + 0x808B,0x808C,0x81E3,0x81EA,0x81F3,0x81FC,0x820C,0x821B,/* 0xD8-0xDF */ + 0x821F,0x826E,0x8272,0x827E,0x866B,0x8840,0x884C,0x8863,/* 0xE0-0xE7 */ + 0x897F,0x9621,0x4E32,0x4EA8,0x4F4D,0x4F4F,0x4F47,0x4F57,/* 0xE8-0xEF */ + 0x4F5E,0x4F34,0x4F5B,0x4F55,0x4F30,0x4F50,0x4F51,0x4F3D,/* 0xF0-0xF7 */ + 0x4F3A,0x4F38,0x4F43,0x4F54,0x4F3C,0x4F46,0x4F63,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_A7[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x4F5C,0x4F60,0x4F2F,0x4F4E,0x4F36,0x4F59,0x4F5D,0x4F48,/* 0x40-0x47 */ + 0x4F5A,0x514C,0x514B,0x514D,0x5175,0x51B6,0x51B7,0x5225,/* 0x48-0x4F */ + 0x5224,0x5229,0x522A,0x5228,0x52AB,0x52A9,0x52AA,0x52AC,/* 0x50-0x57 */ + 0x5323,0x5373,0x5375,0x541D,0x542D,0x541E,0x543E,0x5426,/* 0x58-0x5F */ + 0x544E,0x5427,0x5446,0x5443,0x5433,0x5448,0x5442,0x541B,/* 0x60-0x67 */ + 0x5429,0x544A,0x5439,0x543B,0x5438,0x542E,0x5435,0x5436,/* 0x68-0x6F */ + 0x5420,0x543C,0x5440,0x5431,0x542B,0x541F,0x542C,0x56EA,/* 0x70-0x77 */ + 0x56F0,0x56E4,0x56EB,0x574A,0x5751,0x5740,0x574D,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x5747,0x574E,0x573E,0x5750,0x574F,0x573B,0x58EF,/* 0xA0-0xA7 */ + 0x593E,0x599D,0x5992,0x59A8,0x599E,0x59A3,0x5999,0x5996,/* 0xA8-0xAF */ + 0x598D,0x59A4,0x5993,0x598A,0x59A5,0x5B5D,0x5B5C,0x5B5A,/* 0xB0-0xB7 */ + 0x5B5B,0x5B8C,0x5B8B,0x5B8F,0x5C2C,0x5C40,0x5C41,0x5C3F,/* 0xB8-0xBF */ + 0x5C3E,0x5C90,0x5C91,0x5C94,0x5C8C,0x5DEB,0x5E0C,0x5E8F,/* 0xC0-0xC7 */ + 0x5E87,0x5E8A,0x5EF7,0x5F04,0x5F1F,0x5F64,0x5F62,0x5F77,/* 0xC8-0xCF */ + 0x5F79,0x5FD8,0x5FCC,0x5FD7,0x5FCD,0x5FF1,0x5FEB,0x5FF8,/* 0xD0-0xD7 */ + 0x5FEA,0x6212,0x6211,0x6284,0x6297,0x6296,0x6280,0x6276,/* 0xD8-0xDF */ + 0x6289,0x626D,0x628A,0x627C,0x627E,0x6279,0x6273,0x6292,/* 0xE0-0xE7 */ + 0x626F,0x6298,0x626E,0x6295,0x6293,0x6291,0x6286,0x6539,/* 0xE8-0xEF */ + 0x653B,0x6538,0x65F1,0x66F4,0x675F,0x674E,0x674F,0x6750,/* 0xF0-0xF7 */ + 0x6751,0x675C,0x6756,0x675E,0x6749,0x6746,0x6760,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_A8[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6753,0x6757,0x6B65,0x6BCF,0x6C42,0x6C5E,0x6C99,0x6C81,/* 0x40-0x47 */ + 0x6C88,0x6C89,0x6C85,0x6C9B,0x6C6A,0x6C7A,0x6C90,0x6C70,/* 0x48-0x4F */ + 0x6C8C,0x6C68,0x6C96,0x6C92,0x6C7D,0x6C83,0x6C72,0x6C7E,/* 0x50-0x57 */ + 0x6C74,0x6C86,0x6C76,0x6C8D,0x6C94,0x6C98,0x6C82,0x7076,/* 0x58-0x5F */ + 0x707C,0x707D,0x7078,0x7262,0x7261,0x7260,0x72C4,0x72C2,/* 0x60-0x67 */ + 0x7396,0x752C,0x752B,0x7537,0x7538,0x7682,0x76EF,0x77E3,/* 0x68-0x6F */ + 0x79C1,0x79C0,0x79BF,0x7A76,0x7CFB,0x7F55,0x8096,0x8093,/* 0x70-0x77 */ + 0x809D,0x8098,0x809B,0x809A,0x80B2,0x826F,0x8292,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x828B,0x828D,0x898B,0x89D2,0x8A00,0x8C37,0x8C46,/* 0xA0-0xA7 */ + 0x8C55,0x8C9D,0x8D64,0x8D70,0x8DB3,0x8EAB,0x8ECA,0x8F9B,/* 0xA8-0xAF */ + 0x8FB0,0x8FC2,0x8FC6,0x8FC5,0x8FC4,0x5DE1,0x9091,0x90A2,/* 0xB0-0xB7 */ + 0x90AA,0x90A6,0x90A3,0x9149,0x91C6,0x91CC,0x9632,0x962E,/* 0xB8-0xBF */ + 0x9631,0x962A,0x962C,0x4E26,0x4E56,0x4E73,0x4E8B,0x4E9B,/* 0xC0-0xC7 */ + 0x4E9E,0x4EAB,0x4EAC,0x4F6F,0x4F9D,0x4F8D,0x4F73,0x4F7F,/* 0xC8-0xCF */ + 0x4F6C,0x4F9B,0x4F8B,0x4F86,0x4F83,0x4F70,0x4F75,0x4F88,/* 0xD0-0xD7 */ + 0x4F69,0x4F7B,0x4F96,0x4F7E,0x4F8F,0x4F91,0x4F7A,0x5154,/* 0xD8-0xDF */ + 0x5152,0x5155,0x5169,0x5177,0x5176,0x5178,0x51BD,0x51FD,/* 0xE0-0xE7 */ + 0x523B,0x5238,0x5237,0x523A,0x5230,0x522E,0x5236,0x5241,/* 0xE8-0xEF */ + 0x52BE,0x52BB,0x5352,0x5354,0x5353,0x5351,0x5366,0x5377,/* 0xF0-0xF7 */ + 0x5378,0x5379,0x53D6,0x53D4,0x53D7,0x5473,0x5475,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_A9[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x5496,0x5478,0x5495,0x5480,0x547B,0x5477,0x5484,0x5492,/* 0x40-0x47 */ + 0x5486,0x547C,0x5490,0x5471,0x5476,0x548C,0x549A,0x5462,/* 0x48-0x4F */ + 0x5468,0x548B,0x547D,0x548E,0x56FA,0x5783,0x5777,0x576A,/* 0x50-0x57 */ + 0x5769,0x5761,0x5766,0x5764,0x577C,0x591C,0x5949,0x5947,/* 0x58-0x5F */ + 0x5948,0x5944,0x5954,0x59BE,0x59BB,0x59D4,0x59B9,0x59AE,/* 0x60-0x67 */ + 0x59D1,0x59C6,0x59D0,0x59CD,0x59CB,0x59D3,0x59CA,0x59AF,/* 0x68-0x6F */ + 0x59B3,0x59D2,0x59C5,0x5B5F,0x5B64,0x5B63,0x5B97,0x5B9A,/* 0x70-0x77 */ + 0x5B98,0x5B9C,0x5B99,0x5B9B,0x5C1A,0x5C48,0x5C45,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x5C46,0x5CB7,0x5CA1,0x5CB8,0x5CA9,0x5CAB,0x5CB1,/* 0xA0-0xA7 */ + 0x5CB3,0x5E18,0x5E1A,0x5E16,0x5E15,0x5E1B,0x5E11,0x5E78,/* 0xA8-0xAF */ + 0x5E9A,0x5E97,0x5E9C,0x5E95,0x5E96,0x5EF6,0x5F26,0x5F27,/* 0xB0-0xB7 */ + 0x5F29,0x5F80,0x5F81,0x5F7F,0x5F7C,0x5FDD,0x5FE0,0x5FFD,/* 0xB8-0xBF */ + 0x5FF5,0x5FFF,0x600F,0x6014,0x602F,0x6035,0x6016,0x602A,/* 0xC0-0xC7 */ + 0x6015,0x6021,0x6027,0x6029,0x602B,0x601B,0x6216,0x6215,/* 0xC8-0xCF */ + 0x623F,0x623E,0x6240,0x627F,0x62C9,0x62CC,0x62C4,0x62BF,/* 0xD0-0xD7 */ + 0x62C2,0x62B9,0x62D2,0x62DB,0x62AB,0x62D3,0x62D4,0x62CB,/* 0xD8-0xDF */ + 0x62C8,0x62A8,0x62BD,0x62BC,0x62D0,0x62D9,0x62C7,0x62CD,/* 0xE0-0xE7 */ + 0x62B5,0x62DA,0x62B1,0x62D8,0x62D6,0x62D7,0x62C6,0x62AC,/* 0xE8-0xEF */ + 0x62CE,0x653E,0x65A7,0x65BC,0x65FA,0x6614,0x6613,0x660C,/* 0xF0-0xF7 */ + 0x6606,0x6602,0x660E,0x6600,0x660F,0x6615,0x660A,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_AA[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6607,0x670D,0x670B,0x676D,0x678B,0x6795,0x6771,0x679C,/* 0x40-0x47 */ + 0x6773,0x6777,0x6787,0x679D,0x6797,0x676F,0x6770,0x677F,/* 0x48-0x4F */ + 0x6789,0x677E,0x6790,0x6775,0x679A,0x6793,0x677C,0x676A,/* 0x50-0x57 */ + 0x6772,0x6B23,0x6B66,0x6B67,0x6B7F,0x6C13,0x6C1B,0x6CE3,/* 0x58-0x5F */ + 0x6CE8,0x6CF3,0x6CB1,0x6CCC,0x6CE5,0x6CB3,0x6CBD,0x6CBE,/* 0x60-0x67 */ + 0x6CBC,0x6CE2,0x6CAB,0x6CD5,0x6CD3,0x6CB8,0x6CC4,0x6CB9,/* 0x68-0x6F */ + 0x6CC1,0x6CAE,0x6CD7,0x6CC5,0x6CF1,0x6CBF,0x6CBB,0x6CE1,/* 0x70-0x77 */ + 0x6CDB,0x6CCA,0x6CAC,0x6CEF,0x6CDC,0x6CD6,0x6CE0,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x7095,0x708E,0x7092,0x708A,0x7099,0x722C,0x722D,/* 0xA0-0xA7 */ + 0x7238,0x7248,0x7267,0x7269,0x72C0,0x72CE,0x72D9,0x72D7,/* 0xA8-0xAF */ + 0x72D0,0x73A9,0x73A8,0x739F,0x73AB,0x73A5,0x753D,0x759D,/* 0xB0-0xB7 */ + 0x7599,0x759A,0x7684,0x76C2,0x76F2,0x76F4,0x77E5,0x77FD,/* 0xB8-0xBF */ + 0x793E,0x7940,0x7941,0x79C9,0x79C8,0x7A7A,0x7A79,0x7AFA,/* 0xC0-0xC7 */ + 0x7CFE,0x7F54,0x7F8C,0x7F8B,0x8005,0x80BA,0x80A5,0x80A2,/* 0xC8-0xCF */ + 0x80B1,0x80A1,0x80AB,0x80A9,0x80B4,0x80AA,0x80AF,0x81E5,/* 0xD0-0xD7 */ + 0x81FE,0x820D,0x82B3,0x829D,0x8299,0x82AD,0x82BD,0x829F,/* 0xD8-0xDF */ + 0x82B9,0x82B1,0x82AC,0x82A5,0x82AF,0x82B8,0x82A3,0x82B0,/* 0xE0-0xE7 */ + 0x82BE,0x82B7,0x864E,0x8671,0x521D,0x8868,0x8ECB,0x8FCE,/* 0xE8-0xEF */ + 0x8FD4,0x8FD1,0x90B5,0x90B8,0x90B1,0x90B6,0x91C7,0x91D1,/* 0xF0-0xF7 */ + 0x9577,0x9580,0x961C,0x9640,0x963F,0x963B,0x9644,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_AB[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9642,0x96B9,0x96E8,0x9752,0x975E,0x4E9F,0x4EAD,0x4EAE,/* 0x40-0x47 */ + 0x4FE1,0x4FB5,0x4FAF,0x4FBF,0x4FE0,0x4FD1,0x4FCF,0x4FDD,/* 0x48-0x4F */ + 0x4FC3,0x4FB6,0x4FD8,0x4FDF,0x4FCA,0x4FD7,0x4FAE,0x4FD0,/* 0x50-0x57 */ + 0x4FC4,0x4FC2,0x4FDA,0x4FCE,0x4FDE,0x4FB7,0x5157,0x5192,/* 0x58-0x5F */ + 0x5191,0x51A0,0x524E,0x5243,0x524A,0x524D,0x524C,0x524B,/* 0x60-0x67 */ + 0x5247,0x52C7,0x52C9,0x52C3,0x52C1,0x530D,0x5357,0x537B,/* 0x68-0x6F */ + 0x539A,0x53DB,0x54AC,0x54C0,0x54A8,0x54CE,0x54C9,0x54B8,/* 0x70-0x77 */ + 0x54A6,0x54B3,0x54C7,0x54C2,0x54BD,0x54AA,0x54C1,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x54C4,0x54C8,0x54AF,0x54AB,0x54B1,0x54BB,0x54A9,/* 0xA0-0xA7 */ + 0x54A7,0x54BF,0x56FF,0x5782,0x578B,0x57A0,0x57A3,0x57A2,/* 0xA8-0xAF */ + 0x57CE,0x57AE,0x5793,0x5955,0x5951,0x594F,0x594E,0x5950,/* 0xB0-0xB7 */ + 0x59DC,0x59D8,0x59FF,0x59E3,0x59E8,0x5A03,0x59E5,0x59EA,/* 0xB8-0xBF */ + 0x59DA,0x59E6,0x5A01,0x59FB,0x5B69,0x5BA3,0x5BA6,0x5BA4,/* 0xC0-0xC7 */ + 0x5BA2,0x5BA5,0x5C01,0x5C4E,0x5C4F,0x5C4D,0x5C4B,0x5CD9,/* 0xC8-0xCF */ + 0x5CD2,0x5DF7,0x5E1D,0x5E25,0x5E1F,0x5E7D,0x5EA0,0x5EA6,/* 0xD0-0xD7 */ + 0x5EFA,0x5F08,0x5F2D,0x5F65,0x5F88,0x5F85,0x5F8A,0x5F8B,/* 0xD8-0xDF */ + 0x5F87,0x5F8C,0x5F89,0x6012,0x601D,0x6020,0x6025,0x600E,/* 0xE0-0xE7 */ + 0x6028,0x604D,0x6070,0x6068,0x6062,0x6046,0x6043,0x606C,/* 0xE8-0xEF */ + 0x606B,0x606A,0x6064,0x6241,0x62DC,0x6316,0x6309,0x62FC,/* 0xF0-0xF7 */ + 0x62ED,0x6301,0x62EE,0x62FD,0x6307,0x62F1,0x62F7,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_AC[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x62EF,0x62EC,0x62FE,0x62F4,0x6311,0x6302,0x653F,0x6545,/* 0x40-0x47 */ + 0x65AB,0x65BD,0x65E2,0x6625,0x662D,0x6620,0x6627,0x662F,/* 0x48-0x4F */ + 0x661F,0x6628,0x6631,0x6624,0x66F7,0x67FF,0x67D3,0x67F1,/* 0x50-0x57 */ + 0x67D4,0x67D0,0x67EC,0x67B6,0x67AF,0x67F5,0x67E9,0x67EF,/* 0x58-0x5F */ + 0x67C4,0x67D1,0x67B4,0x67DA,0x67E5,0x67B8,0x67CF,0x67DE,/* 0x60-0x67 */ + 0x67F3,0x67B0,0x67D9,0x67E2,0x67DD,0x67D2,0x6B6A,0x6B83,/* 0x68-0x6F */ + 0x6B86,0x6BB5,0x6BD2,0x6BD7,0x6C1F,0x6CC9,0x6D0B,0x6D32,/* 0x70-0x77 */ + 0x6D2A,0x6D41,0x6D25,0x6D0C,0x6D31,0x6D1E,0x6D17,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x6D3B,0x6D3D,0x6D3E,0x6D36,0x6D1B,0x6CF5,0x6D39,/* 0xA0-0xA7 */ + 0x6D27,0x6D38,0x6D29,0x6D2E,0x6D35,0x6D0E,0x6D2B,0x70AB,/* 0xA8-0xAF */ + 0x70BA,0x70B3,0x70AC,0x70AF,0x70AD,0x70B8,0x70AE,0x70A4,/* 0xB0-0xB7 */ + 0x7230,0x7272,0x726F,0x7274,0x72E9,0x72E0,0x72E1,0x73B7,/* 0xB8-0xBF */ + 0x73CA,0x73BB,0x73B2,0x73CD,0x73C0,0x73B3,0x751A,0x752D,/* 0xC0-0xC7 */ + 0x754F,0x754C,0x754E,0x754B,0x75AB,0x75A4,0x75A5,0x75A2,/* 0xC8-0xCF */ + 0x75A3,0x7678,0x7686,0x7687,0x7688,0x76C8,0x76C6,0x76C3,/* 0xD0-0xD7 */ + 0x76C5,0x7701,0x76F9,0x76F8,0x7709,0x770B,0x76FE,0x76FC,/* 0xD8-0xDF */ + 0x7707,0x77DC,0x7802,0x7814,0x780C,0x780D,0x7946,0x7949,/* 0xE0-0xE7 */ + 0x7948,0x7947,0x79B9,0x79BA,0x79D1,0x79D2,0x79CB,0x7A7F,/* 0xE8-0xEF */ + 0x7A81,0x7AFF,0x7AFD,0x7C7D,0x7D02,0x7D05,0x7D00,0x7D09,/* 0xF0-0xF7 */ + 0x7D07,0x7D04,0x7D06,0x7F38,0x7F8E,0x7FBF,0x8004,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_AD[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8010,0x800D,0x8011,0x8036,0x80D6,0x80E5,0x80DA,0x80C3,/* 0x40-0x47 */ + 0x80C4,0x80CC,0x80E1,0x80DB,0x80CE,0x80DE,0x80E4,0x80DD,/* 0x48-0x4F */ + 0x81F4,0x8222,0x82E7,0x8303,0x8305,0x82E3,0x82DB,0x82E6,/* 0x50-0x57 */ + 0x8304,0x82E5,0x8302,0x8309,0x82D2,0x82D7,0x82F1,0x8301,/* 0x58-0x5F */ + 0x82DC,0x82D4,0x82D1,0x82DE,0x82D3,0x82DF,0x82EF,0x8306,/* 0x60-0x67 */ + 0x8650,0x8679,0x867B,0x867A,0x884D,0x886B,0x8981,0x89D4,/* 0x68-0x6F */ + 0x8A08,0x8A02,0x8A03,0x8C9E,0x8CA0,0x8D74,0x8D73,0x8DB4,/* 0x70-0x77 */ + 0x8ECD,0x8ECC,0x8FF0,0x8FE6,0x8FE2,0x8FEA,0x8FE5,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x8FED,0x8FEB,0x8FE4,0x8FE8,0x90CA,0x90CE,0x90C1,/* 0xA0-0xA7 */ + 0x90C3,0x914B,0x914A,0x91CD,0x9582,0x9650,0x964B,0x964C,/* 0xA8-0xAF */ + 0x964D,0x9762,0x9769,0x97CB,0x97ED,0x97F3,0x9801,0x98A8,/* 0xB0-0xB7 */ + 0x98DB,0x98DF,0x9996,0x9999,0x4E58,0x4EB3,0x500C,0x500D,/* 0xB8-0xBF */ + 0x5023,0x4FEF,0x5026,0x5025,0x4FF8,0x5029,0x5016,0x5006,/* 0xC0-0xC7 */ + 0x503C,0x501F,0x501A,0x5012,0x5011,0x4FFA,0x5000,0x5014,/* 0xC8-0xCF */ + 0x5028,0x4FF1,0x5021,0x500B,0x5019,0x5018,0x4FF3,0x4FEE,/* 0xD0-0xD7 */ + 0x502D,0x502A,0x4FFE,0x502B,0x5009,0x517C,0x51A4,0x51A5,/* 0xD8-0xDF */ + 0x51A2,0x51CD,0x51CC,0x51C6,0x51CB,0x5256,0x525C,0x5254,/* 0xE0-0xE7 */ + 0x525B,0x525D,0x532A,0x537F,0x539F,0x539D,0x53DF,0x54E8,/* 0xE8-0xEF */ + 0x5510,0x5501,0x5537,0x54FC,0x54E5,0x54F2,0x5506,0x54FA,/* 0xF0-0xF7 */ + 0x5514,0x54E9,0x54ED,0x54E1,0x5509,0x54EE,0x54EA,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_AE[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x54E6,0x5527,0x5507,0x54FD,0x550F,0x5703,0x5704,0x57C2,/* 0x40-0x47 */ + 0x57D4,0x57CB,0x57C3,0x5809,0x590F,0x5957,0x5958,0x595A,/* 0x48-0x4F */ + 0x5A11,0x5A18,0x5A1C,0x5A1F,0x5A1B,0x5A13,0x59EC,0x5A20,/* 0x50-0x57 */ + 0x5A23,0x5A29,0x5A25,0x5A0C,0x5A09,0x5B6B,0x5C58,0x5BB0,/* 0x58-0x5F */ + 0x5BB3,0x5BB6,0x5BB4,0x5BAE,0x5BB5,0x5BB9,0x5BB8,0x5C04,/* 0x60-0x67 */ + 0x5C51,0x5C55,0x5C50,0x5CED,0x5CFD,0x5CFB,0x5CEA,0x5CE8,/* 0x68-0x6F */ + 0x5CF0,0x5CF6,0x5D01,0x5CF4,0x5DEE,0x5E2D,0x5E2B,0x5EAB,/* 0x70-0x77 */ + 0x5EAD,0x5EA7,0x5F31,0x5F92,0x5F91,0x5F90,0x6059,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x6063,0x6065,0x6050,0x6055,0x606D,0x6069,0x606F,/* 0xA0-0xA7 */ + 0x6084,0x609F,0x609A,0x608D,0x6094,0x608C,0x6085,0x6096,/* 0xA8-0xAF */ + 0x6247,0x62F3,0x6308,0x62FF,0x634E,0x633E,0x632F,0x6355,/* 0xB0-0xB7 */ + 0x6342,0x6346,0x634F,0x6349,0x633A,0x6350,0x633D,0x632A,/* 0xB8-0xBF */ + 0x632B,0x6328,0x634D,0x634C,0x6548,0x6549,0x6599,0x65C1,/* 0xC0-0xC7 */ + 0x65C5,0x6642,0x6649,0x664F,0x6643,0x6652,0x664C,0x6645,/* 0xC8-0xCF */ + 0x6641,0x66F8,0x6714,0x6715,0x6717,0x6821,0x6838,0x6848,/* 0xD0-0xD7 */ + 0x6846,0x6853,0x6839,0x6842,0x6854,0x6829,0x68B3,0x6817,/* 0xD8-0xDF */ + 0x684C,0x6851,0x683D,0x67F4,0x6850,0x6840,0x683C,0x6843,/* 0xE0-0xE7 */ + 0x682A,0x6845,0x6813,0x6818,0x6841,0x6B8A,0x6B89,0x6BB7,/* 0xE8-0xEF */ + 0x6C23,0x6C27,0x6C28,0x6C26,0x6C24,0x6CF0,0x6D6A,0x6D95,/* 0xF0-0xF7 */ + 0x6D88,0x6D87,0x6D66,0x6D78,0x6D77,0x6D59,0x6D93,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_AF[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6D6C,0x6D89,0x6D6E,0x6D5A,0x6D74,0x6D69,0x6D8C,0x6D8A,/* 0x40-0x47 */ + 0x6D79,0x6D85,0x6D65,0x6D94,0x70CA,0x70D8,0x70E4,0x70D9,/* 0x48-0x4F */ + 0x70C8,0x70CF,0x7239,0x7279,0x72FC,0x72F9,0x72FD,0x72F8,/* 0x50-0x57 */ + 0x72F7,0x7386,0x73ED,0x7409,0x73EE,0x73E0,0x73EA,0x73DE,/* 0x58-0x5F */ + 0x7554,0x755D,0x755C,0x755A,0x7559,0x75BE,0x75C5,0x75C7,/* 0x60-0x67 */ + 0x75B2,0x75B3,0x75BD,0x75BC,0x75B9,0x75C2,0x75B8,0x768B,/* 0x68-0x6F */ + 0x76B0,0x76CA,0x76CD,0x76CE,0x7729,0x771F,0x7720,0x7728,/* 0x70-0x77 */ + 0x77E9,0x7830,0x7827,0x7838,0x781D,0x7834,0x7837,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x7825,0x782D,0x7820,0x781F,0x7832,0x7955,0x7950,/* 0xA0-0xA7 */ + 0x7960,0x795F,0x7956,0x795E,0x795D,0x7957,0x795A,0x79E4,/* 0xA8-0xAF */ + 0x79E3,0x79E7,0x79DF,0x79E6,0x79E9,0x79D8,0x7A84,0x7A88,/* 0xB0-0xB7 */ + 0x7AD9,0x7B06,0x7B11,0x7C89,0x7D21,0x7D17,0x7D0B,0x7D0A,/* 0xB8-0xBF */ + 0x7D20,0x7D22,0x7D14,0x7D10,0x7D15,0x7D1A,0x7D1C,0x7D0D,/* 0xC0-0xC7 */ + 0x7D19,0x7D1B,0x7F3A,0x7F5F,0x7F94,0x7FC5,0x7FC1,0x8006,/* 0xC8-0xCF */ + 0x8018,0x8015,0x8019,0x8017,0x803D,0x803F,0x80F1,0x8102,/* 0xD0-0xD7 */ + 0x80F0,0x8105,0x80ED,0x80F4,0x8106,0x80F8,0x80F3,0x8108,/* 0xD8-0xDF */ + 0x80FD,0x810A,0x80FC,0x80EF,0x81ED,0x81EC,0x8200,0x8210,/* 0xE0-0xE7 */ + 0x822A,0x822B,0x8228,0x822C,0x82BB,0x832B,0x8352,0x8354,/* 0xE8-0xEF */ + 0x834A,0x8338,0x8350,0x8349,0x8335,0x8334,0x834F,0x8332,/* 0xF0-0xF7 */ + 0x8339,0x8336,0x8317,0x8340,0x8331,0x8328,0x8343,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B0[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8654,0x868A,0x86AA,0x8693,0x86A4,0x86A9,0x868C,0x86A3,/* 0x40-0x47 */ + 0x869C,0x8870,0x8877,0x8881,0x8882,0x887D,0x8879,0x8A18,/* 0x48-0x4F */ + 0x8A10,0x8A0E,0x8A0C,0x8A15,0x8A0A,0x8A17,0x8A13,0x8A16,/* 0x50-0x57 */ + 0x8A0F,0x8A11,0x8C48,0x8C7A,0x8C79,0x8CA1,0x8CA2,0x8D77,/* 0x58-0x5F */ + 0x8EAC,0x8ED2,0x8ED4,0x8ECF,0x8FB1,0x9001,0x9006,0x8FF7,/* 0x60-0x67 */ + 0x9000,0x8FFA,0x8FF4,0x9003,0x8FFD,0x9005,0x8FF8,0x9095,/* 0x68-0x6F */ + 0x90E1,0x90DD,0x90E2,0x9152,0x914D,0x914C,0x91D8,0x91DD,/* 0x70-0x77 */ + 0x91D7,0x91DC,0x91D9,0x9583,0x9662,0x9663,0x9661,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x965B,0x965D,0x9664,0x9658,0x965E,0x96BB,0x98E2,/* 0xA0-0xA7 */ + 0x99AC,0x9AA8,0x9AD8,0x9B25,0x9B32,0x9B3C,0x4E7E,0x507A,/* 0xA8-0xAF */ + 0x507D,0x505C,0x5047,0x5043,0x504C,0x505A,0x5049,0x5065,/* 0xB0-0xB7 */ + 0x5076,0x504E,0x5055,0x5075,0x5074,0x5077,0x504F,0x500F,/* 0xB8-0xBF */ + 0x506F,0x506D,0x515C,0x5195,0x51F0,0x526A,0x526F,0x52D2,/* 0xC0-0xC7 */ + 0x52D9,0x52D8,0x52D5,0x5310,0x530F,0x5319,0x533F,0x5340,/* 0xC8-0xCF */ + 0x533E,0x53C3,0x66FC,0x5546,0x556A,0x5566,0x5544,0x555E,/* 0xD0-0xD7 */ + 0x5561,0x5543,0x554A,0x5531,0x5556,0x554F,0x5555,0x552F,/* 0xD8-0xDF */ + 0x5564,0x5538,0x552E,0x555C,0x552C,0x5563,0x5533,0x5541,/* 0xE0-0xE7 */ + 0x5557,0x5708,0x570B,0x5709,0x57DF,0x5805,0x580A,0x5806,/* 0xE8-0xEF */ + 0x57E0,0x57E4,0x57FA,0x5802,0x5835,0x57F7,0x57F9,0x5920,/* 0xF0-0xF7 */ + 0x5962,0x5A36,0x5A41,0x5A49,0x5A66,0x5A6A,0x5A40,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B1[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x5A3C,0x5A62,0x5A5A,0x5A46,0x5A4A,0x5B70,0x5BC7,0x5BC5,/* 0x40-0x47 */ + 0x5BC4,0x5BC2,0x5BBF,0x5BC6,0x5C09,0x5C08,0x5C07,0x5C60,/* 0x48-0x4F */ + 0x5C5C,0x5C5D,0x5D07,0x5D06,0x5D0E,0x5D1B,0x5D16,0x5D22,/* 0x50-0x57 */ + 0x5D11,0x5D29,0x5D14,0x5D19,0x5D24,0x5D27,0x5D17,0x5DE2,/* 0x58-0x5F */ + 0x5E38,0x5E36,0x5E33,0x5E37,0x5EB7,0x5EB8,0x5EB6,0x5EB5,/* 0x60-0x67 */ + 0x5EBE,0x5F35,0x5F37,0x5F57,0x5F6C,0x5F69,0x5F6B,0x5F97,/* 0x68-0x6F */ + 0x5F99,0x5F9E,0x5F98,0x5FA1,0x5FA0,0x5F9C,0x607F,0x60A3,/* 0x70-0x77 */ + 0x6089,0x60A0,0x60A8,0x60CB,0x60B4,0x60E6,0x60BD,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x60C5,0x60BB,0x60B5,0x60DC,0x60BC,0x60D8,0x60D5,/* 0xA0-0xA7 */ + 0x60C6,0x60DF,0x60B8,0x60DA,0x60C7,0x621A,0x621B,0x6248,/* 0xA8-0xAF */ + 0x63A0,0x63A7,0x6372,0x6396,0x63A2,0x63A5,0x6377,0x6367,/* 0xB0-0xB7 */ + 0x6398,0x63AA,0x6371,0x63A9,0x6389,0x6383,0x639B,0x636B,/* 0xB8-0xBF */ + 0x63A8,0x6384,0x6388,0x6399,0x63A1,0x63AC,0x6392,0x638F,/* 0xC0-0xC7 */ + 0x6380,0x637B,0x6369,0x6368,0x637A,0x655D,0x6556,0x6551,/* 0xC8-0xCF */ + 0x6559,0x6557,0x555F,0x654F,0x6558,0x6555,0x6554,0x659C,/* 0xD0-0xD7 */ + 0x659B,0x65AC,0x65CF,0x65CB,0x65CC,0x65CE,0x665D,0x665A,/* 0xD8-0xDF */ + 0x6664,0x6668,0x6666,0x665E,0x66F9,0x52D7,0x671B,0x6881,/* 0xE0-0xE7 */ + 0x68AF,0x68A2,0x6893,0x68B5,0x687F,0x6876,0x68B1,0x68A7,/* 0xE8-0xEF */ + 0x6897,0x68B0,0x6883,0x68C4,0x68AD,0x6886,0x6885,0x6894,/* 0xF0-0xF7 */ + 0x689D,0x68A8,0x689F,0x68A1,0x6882,0x6B32,0x6BBA,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B2[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6BEB,0x6BEC,0x6C2B,0x6D8E,0x6DBC,0x6DF3,0x6DD9,0x6DB2,/* 0x40-0x47 */ + 0x6DE1,0x6DCC,0x6DE4,0x6DFB,0x6DFA,0x6E05,0x6DC7,0x6DCB,/* 0x48-0x4F */ + 0x6DAF,0x6DD1,0x6DAE,0x6DDE,0x6DF9,0x6DB8,0x6DF7,0x6DF5,/* 0x50-0x57 */ + 0x6DC5,0x6DD2,0x6E1A,0x6DB5,0x6DDA,0x6DEB,0x6DD8,0x6DEA,/* 0x58-0x5F */ + 0x6DF1,0x6DEE,0x6DE8,0x6DC6,0x6DC4,0x6DAA,0x6DEC,0x6DBF,/* 0x60-0x67 */ + 0x6DE6,0x70F9,0x7109,0x710A,0x70FD,0x70EF,0x723D,0x727D,/* 0x68-0x6F */ + 0x7281,0x731C,0x731B,0x7316,0x7313,0x7319,0x7387,0x7405,/* 0x70-0x77 */ + 0x740A,0x7403,0x7406,0x73FE,0x740D,0x74E0,0x74F6,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x74F7,0x751C,0x7522,0x7565,0x7566,0x7562,0x7570,/* 0xA0-0xA7 */ + 0x758F,0x75D4,0x75D5,0x75B5,0x75CA,0x75CD,0x768E,0x76D4,/* 0xA8-0xAF */ + 0x76D2,0x76DB,0x7737,0x773E,0x773C,0x7736,0x7738,0x773A,/* 0xB0-0xB7 */ + 0x786B,0x7843,0x784E,0x7965,0x7968,0x796D,0x79FB,0x7A92,/* 0xB8-0xBF */ + 0x7A95,0x7B20,0x7B28,0x7B1B,0x7B2C,0x7B26,0x7B19,0x7B1E,/* 0xC0-0xC7 */ + 0x7B2E,0x7C92,0x7C97,0x7C95,0x7D46,0x7D43,0x7D71,0x7D2E,/* 0xC8-0xCF */ + 0x7D39,0x7D3C,0x7D40,0x7D30,0x7D33,0x7D44,0x7D2F,0x7D42,/* 0xD0-0xD7 */ + 0x7D32,0x7D31,0x7F3D,0x7F9E,0x7F9A,0x7FCC,0x7FCE,0x7FD2,/* 0xD8-0xDF */ + 0x801C,0x804A,0x8046,0x812F,0x8116,0x8123,0x812B,0x8129,/* 0xE0-0xE7 */ + 0x8130,0x8124,0x8202,0x8235,0x8237,0x8236,0x8239,0x838E,/* 0xE8-0xEF */ + 0x839E,0x8398,0x8378,0x83A2,0x8396,0x83BD,0x83AB,0x8392,/* 0xF0-0xF7 */ + 0x838A,0x8393,0x8389,0x83A0,0x8377,0x837B,0x837C,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B3[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8386,0x83A7,0x8655,0x5F6A,0x86C7,0x86C0,0x86B6,0x86C4,/* 0x40-0x47 */ + 0x86B5,0x86C6,0x86CB,0x86B1,0x86AF,0x86C9,0x8853,0x889E,/* 0x48-0x4F */ + 0x8888,0x88AB,0x8892,0x8896,0x888D,0x888B,0x8993,0x898F,/* 0x50-0x57 */ + 0x8A2A,0x8A1D,0x8A23,0x8A25,0x8A31,0x8A2D,0x8A1F,0x8A1B,/* 0x58-0x5F */ + 0x8A22,0x8C49,0x8C5A,0x8CA9,0x8CAC,0x8CAB,0x8CA8,0x8CAA,/* 0x60-0x67 */ + 0x8CA7,0x8D67,0x8D66,0x8DBE,0x8DBA,0x8EDB,0x8EDF,0x9019,/* 0x68-0x6F */ + 0x900D,0x901A,0x9017,0x9023,0x901F,0x901D,0x9010,0x9015,/* 0x70-0x77 */ + 0x901E,0x9020,0x900F,0x9022,0x9016,0x901B,0x9014,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x90E8,0x90ED,0x90FD,0x9157,0x91CE,0x91F5,0x91E6,/* 0xA0-0xA7 */ + 0x91E3,0x91E7,0x91ED,0x91E9,0x9589,0x966A,0x9675,0x9673,/* 0xA8-0xAF */ + 0x9678,0x9670,0x9674,0x9676,0x9677,0x966C,0x96C0,0x96EA,/* 0xB0-0xB7 */ + 0x96E9,0x7AE0,0x7ADF,0x9802,0x9803,0x9B5A,0x9CE5,0x9E75,/* 0xB8-0xBF */ + 0x9E7F,0x9EA5,0x9EBB,0x50A2,0x508D,0x5085,0x5099,0x5091,/* 0xC0-0xC7 */ + 0x5080,0x5096,0x5098,0x509A,0x6700,0x51F1,0x5272,0x5274,/* 0xC8-0xCF */ + 0x5275,0x5269,0x52DE,0x52DD,0x52DB,0x535A,0x53A5,0x557B,/* 0xD0-0xD7 */ + 0x5580,0x55A7,0x557C,0x558A,0x559D,0x5598,0x5582,0x559C,/* 0xD8-0xDF */ + 0x55AA,0x5594,0x5587,0x558B,0x5583,0x55B3,0x55AE,0x559F,/* 0xE0-0xE7 */ + 0x553E,0x55B2,0x559A,0x55BB,0x55AC,0x55B1,0x557E,0x5589,/* 0xE8-0xEF */ + 0x55AB,0x5599,0x570D,0x582F,0x582A,0x5834,0x5824,0x5830,/* 0xF0-0xF7 */ + 0x5831,0x5821,0x581D,0x5820,0x58F9,0x58FA,0x5960,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B4[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x5A77,0x5A9A,0x5A7F,0x5A92,0x5A9B,0x5AA7,0x5B73,0x5B71,/* 0x40-0x47 */ + 0x5BD2,0x5BCC,0x5BD3,0x5BD0,0x5C0A,0x5C0B,0x5C31,0x5D4C,/* 0x48-0x4F */ + 0x5D50,0x5D34,0x5D47,0x5DFD,0x5E45,0x5E3D,0x5E40,0x5E43,/* 0x50-0x57 */ + 0x5E7E,0x5ECA,0x5EC1,0x5EC2,0x5EC4,0x5F3C,0x5F6D,0x5FA9,/* 0x58-0x5F */ + 0x5FAA,0x5FA8,0x60D1,0x60E1,0x60B2,0x60B6,0x60E0,0x611C,/* 0x60-0x67 */ + 0x6123,0x60FA,0x6115,0x60F0,0x60FB,0x60F4,0x6168,0x60F1,/* 0x68-0x6F */ + 0x610E,0x60F6,0x6109,0x6100,0x6112,0x621F,0x6249,0x63A3,/* 0x70-0x77 */ + 0x638C,0x63CF,0x63C0,0x63E9,0x63C9,0x63C6,0x63CD,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x63D2,0x63E3,0x63D0,0x63E1,0x63D6,0x63ED,0x63EE,/* 0xA0-0xA7 */ + 0x6376,0x63F4,0x63EA,0x63DB,0x6452,0x63DA,0x63F9,0x655E,/* 0xA8-0xAF */ + 0x6566,0x6562,0x6563,0x6591,0x6590,0x65AF,0x666E,0x6670,/* 0xB0-0xB7 */ + 0x6674,0x6676,0x666F,0x6691,0x667A,0x667E,0x6677,0x66FE,/* 0xB8-0xBF */ + 0x66FF,0x671F,0x671D,0x68FA,0x68D5,0x68E0,0x68D8,0x68D7,/* 0xC0-0xC7 */ + 0x6905,0x68DF,0x68F5,0x68EE,0x68E7,0x68F9,0x68D2,0x68F2,/* 0xC8-0xCF */ + 0x68E3,0x68CB,0x68CD,0x690D,0x6912,0x690E,0x68C9,0x68DA,/* 0xD0-0xD7 */ + 0x696E,0x68FB,0x6B3E,0x6B3A,0x6B3D,0x6B98,0x6B96,0x6BBC,/* 0xD8-0xDF */ + 0x6BEF,0x6C2E,0x6C2F,0x6C2C,0x6E2F,0x6E38,0x6E54,0x6E21,/* 0xE0-0xE7 */ + 0x6E32,0x6E67,0x6E4A,0x6E20,0x6E25,0x6E23,0x6E1B,0x6E5B,/* 0xE8-0xEF */ + 0x6E58,0x6E24,0x6E56,0x6E6E,0x6E2D,0x6E26,0x6E6F,0x6E34,/* 0xF0-0xF7 */ + 0x6E4D,0x6E3A,0x6E2C,0x6E43,0x6E1D,0x6E3E,0x6ECB,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B5[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6E89,0x6E19,0x6E4E,0x6E63,0x6E44,0x6E72,0x6E69,0x6E5F,/* 0x40-0x47 */ + 0x7119,0x711A,0x7126,0x7130,0x7121,0x7136,0x716E,0x711C,/* 0x48-0x4F */ + 0x724C,0x7284,0x7280,0x7336,0x7325,0x7334,0x7329,0x743A,/* 0x50-0x57 */ + 0x742A,0x7433,0x7422,0x7425,0x7435,0x7436,0x7434,0x742F,/* 0x58-0x5F */ + 0x741B,0x7426,0x7428,0x7525,0x7526,0x756B,0x756A,0x75E2,/* 0x60-0x67 */ + 0x75DB,0x75E3,0x75D9,0x75D8,0x75DE,0x75E0,0x767B,0x767C,/* 0x68-0x6F */ + 0x7696,0x7693,0x76B4,0x76DC,0x774F,0x77ED,0x785D,0x786C,/* 0x70-0x77 */ + 0x786F,0x7A0D,0x7A08,0x7A0B,0x7A05,0x7A00,0x7A98,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x7A97,0x7A96,0x7AE5,0x7AE3,0x7B49,0x7B56,0x7B46,/* 0xA0-0xA7 */ + 0x7B50,0x7B52,0x7B54,0x7B4D,0x7B4B,0x7B4F,0x7B51,0x7C9F,/* 0xA8-0xAF */ + 0x7CA5,0x7D5E,0x7D50,0x7D68,0x7D55,0x7D2B,0x7D6E,0x7D72,/* 0xB0-0xB7 */ + 0x7D61,0x7D66,0x7D62,0x7D70,0x7D73,0x5584,0x7FD4,0x7FD5,/* 0xB8-0xBF */ + 0x800B,0x8052,0x8085,0x8155,0x8154,0x814B,0x8151,0x814E,/* 0xC0-0xC7 */ + 0x8139,0x8146,0x813E,0x814C,0x8153,0x8174,0x8212,0x821C,/* 0xC8-0xCF */ + 0x83E9,0x8403,0x83F8,0x840D,0x83E0,0x83C5,0x840B,0x83C1,/* 0xD0-0xD7 */ + 0x83EF,0x83F1,0x83F4,0x8457,0x840A,0x83F0,0x840C,0x83CC,/* 0xD8-0xDF */ + 0x83FD,0x83F2,0x83CA,0x8438,0x840E,0x8404,0x83DC,0x8407,/* 0xE0-0xE7 */ + 0x83D4,0x83DF,0x865B,0x86DF,0x86D9,0x86ED,0x86D4,0x86DB,/* 0xE8-0xEF */ + 0x86E4,0x86D0,0x86DE,0x8857,0x88C1,0x88C2,0x88B1,0x8983,/* 0xF0-0xF7 */ + 0x8996,0x8A3B,0x8A60,0x8A55,0x8A5E,0x8A3C,0x8A41,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B6[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8A54,0x8A5B,0x8A50,0x8A46,0x8A34,0x8A3A,0x8A36,0x8A56,/* 0x40-0x47 */ + 0x8C61,0x8C82,0x8CAF,0x8CBC,0x8CB3,0x8CBD,0x8CC1,0x8CBB,/* 0x48-0x4F */ + 0x8CC0,0x8CB4,0x8CB7,0x8CB6,0x8CBF,0x8CB8,0x8D8A,0x8D85,/* 0x50-0x57 */ + 0x8D81,0x8DCE,0x8DDD,0x8DCB,0x8DDA,0x8DD1,0x8DCC,0x8DDB,/* 0x58-0x5F */ + 0x8DC6,0x8EFB,0x8EF8,0x8EFC,0x8F9C,0x902E,0x9035,0x9031,/* 0x60-0x67 */ + 0x9038,0x9032,0x9036,0x9102,0x90F5,0x9109,0x90FE,0x9163,/* 0x68-0x6F */ + 0x9165,0x91CF,0x9214,0x9215,0x9223,0x9209,0x921E,0x920D,/* 0x70-0x77 */ + 0x9210,0x9207,0x9211,0x9594,0x958F,0x958B,0x9591,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x9593,0x9592,0x958E,0x968A,0x968E,0x968B,0x967D,/* 0xA0-0xA7 */ + 0x9685,0x9686,0x968D,0x9672,0x9684,0x96C1,0x96C5,0x96C4,/* 0xA8-0xAF */ + 0x96C6,0x96C7,0x96EF,0x96F2,0x97CC,0x9805,0x9806,0x9808,/* 0xB0-0xB7 */ + 0x98E7,0x98EA,0x98EF,0x98E9,0x98F2,0x98ED,0x99AE,0x99AD,/* 0xB8-0xBF */ + 0x9EC3,0x9ECD,0x9ED1,0x4E82,0x50AD,0x50B5,0x50B2,0x50B3,/* 0xC0-0xC7 */ + 0x50C5,0x50BE,0x50AC,0x50B7,0x50BB,0x50AF,0x50C7,0x527F,/* 0xC8-0xCF */ + 0x5277,0x527D,0x52DF,0x52E6,0x52E4,0x52E2,0x52E3,0x532F,/* 0xD0-0xD7 */ + 0x55DF,0x55E8,0x55D3,0x55E6,0x55CE,0x55DC,0x55C7,0x55D1,/* 0xD8-0xDF */ + 0x55E3,0x55E4,0x55EF,0x55DA,0x55E1,0x55C5,0x55C6,0x55E5,/* 0xE0-0xE7 */ + 0x55C9,0x5712,0x5713,0x585E,0x5851,0x5858,0x5857,0x585A,/* 0xE8-0xEF */ + 0x5854,0x586B,0x584C,0x586D,0x584A,0x5862,0x5852,0x584B,/* 0xF0-0xF7 */ + 0x5967,0x5AC1,0x5AC9,0x5ACC,0x5ABE,0x5ABD,0x5ABC,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B7[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x5AB3,0x5AC2,0x5AB2,0x5D69,0x5D6F,0x5E4C,0x5E79,0x5EC9,/* 0x40-0x47 */ + 0x5EC8,0x5F12,0x5F59,0x5FAC,0x5FAE,0x611A,0x610F,0x6148,/* 0x48-0x4F */ + 0x611F,0x60F3,0x611B,0x60F9,0x6101,0x6108,0x614E,0x614C,/* 0x50-0x57 */ + 0x6144,0x614D,0x613E,0x6134,0x6127,0x610D,0x6106,0x6137,/* 0x58-0x5F */ + 0x6221,0x6222,0x6413,0x643E,0x641E,0x642A,0x642D,0x643D,/* 0x60-0x67 */ + 0x642C,0x640F,0x641C,0x6414,0x640D,0x6436,0x6416,0x6417,/* 0x68-0x6F */ + 0x6406,0x656C,0x659F,0x65B0,0x6697,0x6689,0x6687,0x6688,/* 0x70-0x77 */ + 0x6696,0x6684,0x6698,0x668D,0x6703,0x6994,0x696D,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x695A,0x6977,0x6960,0x6954,0x6975,0x6930,0x6982,/* 0xA0-0xA7 */ + 0x694A,0x6968,0x696B,0x695E,0x6953,0x6979,0x6986,0x695D,/* 0xA8-0xAF */ + 0x6963,0x695B,0x6B47,0x6B72,0x6BC0,0x6BBF,0x6BD3,0x6BFD,/* 0xB0-0xB7 */ + 0x6EA2,0x6EAF,0x6ED3,0x6EB6,0x6EC2,0x6E90,0x6E9D,0x6EC7,/* 0xB8-0xBF */ + 0x6EC5,0x6EA5,0x6E98,0x6EBC,0x6EBA,0x6EAB,0x6ED1,0x6E96,/* 0xC0-0xC7 */ + 0x6E9C,0x6EC4,0x6ED4,0x6EAA,0x6EA7,0x6EB4,0x714E,0x7159,/* 0xC8-0xCF */ + 0x7169,0x7164,0x7149,0x7167,0x715C,0x716C,0x7166,0x714C,/* 0xD0-0xD7 */ + 0x7165,0x715E,0x7146,0x7168,0x7156,0x723A,0x7252,0x7337,/* 0xD8-0xDF */ + 0x7345,0x733F,0x733E,0x746F,0x745A,0x7455,0x745F,0x745E,/* 0xE0-0xE7 */ + 0x7441,0x743F,0x7459,0x745B,0x745C,0x7576,0x7578,0x7600,/* 0xE8-0xEF */ + 0x75F0,0x7601,0x75F2,0x75F1,0x75FA,0x75FF,0x75F4,0x75F3,/* 0xF0-0xF7 */ + 0x76DE,0x76DF,0x775B,0x776B,0x7766,0x775E,0x7763,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B8[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7779,0x776A,0x776C,0x775C,0x7765,0x7768,0x7762,0x77EE,/* 0x40-0x47 */ + 0x788E,0x78B0,0x7897,0x7898,0x788C,0x7889,0x787C,0x7891,/* 0x48-0x4F */ + 0x7893,0x787F,0x797A,0x797F,0x7981,0x842C,0x79BD,0x7A1C,/* 0x50-0x57 */ + 0x7A1A,0x7A20,0x7A14,0x7A1F,0x7A1E,0x7A9F,0x7AA0,0x7B77,/* 0x58-0x5F */ + 0x7BC0,0x7B60,0x7B6E,0x7B67,0x7CB1,0x7CB3,0x7CB5,0x7D93,/* 0x60-0x67 */ + 0x7D79,0x7D91,0x7D81,0x7D8F,0x7D5B,0x7F6E,0x7F69,0x7F6A,/* 0x68-0x6F */ + 0x7F72,0x7FA9,0x7FA8,0x7FA4,0x8056,0x8058,0x8086,0x8084,/* 0x70-0x77 */ + 0x8171,0x8170,0x8178,0x8165,0x816E,0x8173,0x816B,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x8179,0x817A,0x8166,0x8205,0x8247,0x8482,0x8477,/* 0xA0-0xA7 */ + 0x843D,0x8431,0x8475,0x8466,0x846B,0x8449,0x846C,0x845B,/* 0xA8-0xAF */ + 0x843C,0x8435,0x8461,0x8463,0x8469,0x846D,0x8446,0x865E,/* 0xB0-0xB7 */ + 0x865C,0x865F,0x86F9,0x8713,0x8708,0x8707,0x8700,0x86FE,/* 0xB8-0xBF */ + 0x86FB,0x8702,0x8703,0x8706,0x870A,0x8859,0x88DF,0x88D4,/* 0xC0-0xC7 */ + 0x88D9,0x88DC,0x88D8,0x88DD,0x88E1,0x88CA,0x88D5,0x88D2,/* 0xC8-0xCF */ + 0x899C,0x89E3,0x8A6B,0x8A72,0x8A73,0x8A66,0x8A69,0x8A70,/* 0xD0-0xD7 */ + 0x8A87,0x8A7C,0x8A63,0x8AA0,0x8A71,0x8A85,0x8A6D,0x8A62,/* 0xD8-0xDF */ + 0x8A6E,0x8A6C,0x8A79,0x8A7B,0x8A3E,0x8A68,0x8C62,0x8C8A,/* 0xE0-0xE7 */ + 0x8C89,0x8CCA,0x8CC7,0x8CC8,0x8CC4,0x8CB2,0x8CC3,0x8CC2,/* 0xE8-0xEF */ + 0x8CC5,0x8DE1,0x8DDF,0x8DE8,0x8DEF,0x8DF3,0x8DFA,0x8DEA,/* 0xF0-0xF7 */ + 0x8DE4,0x8DE6,0x8EB2,0x8F03,0x8F09,0x8EFE,0x8F0A,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_B9[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8F9F,0x8FB2,0x904B,0x904A,0x9053,0x9042,0x9054,0x903C,/* 0x40-0x47 */ + 0x9055,0x9050,0x9047,0x904F,0x904E,0x904D,0x9051,0x903E,/* 0x48-0x4F */ + 0x9041,0x9112,0x9117,0x916C,0x916A,0x9169,0x91C9,0x9237,/* 0x50-0x57 */ + 0x9257,0x9238,0x923D,0x9240,0x923E,0x925B,0x924B,0x9264,/* 0x58-0x5F */ + 0x9251,0x9234,0x9249,0x924D,0x9245,0x9239,0x923F,0x925A,/* 0x60-0x67 */ + 0x9598,0x9698,0x9694,0x9695,0x96CD,0x96CB,0x96C9,0x96CA,/* 0x68-0x6F */ + 0x96F7,0x96FB,0x96F9,0x96F6,0x9756,0x9774,0x9776,0x9810,/* 0x70-0x77 */ + 0x9811,0x9813,0x980A,0x9812,0x980C,0x98FC,0x98F4,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x98FD,0x98FE,0x99B3,0x99B1,0x99B4,0x9AE1,0x9CE9,/* 0xA0-0xA7 */ + 0x9E82,0x9F0E,0x9F13,0x9F20,0x50E7,0x50EE,0x50E5,0x50D6,/* 0xA8-0xAF */ + 0x50ED,0x50DA,0x50D5,0x50CF,0x50D1,0x50F1,0x50CE,0x50E9,/* 0xB0-0xB7 */ + 0x5162,0x51F3,0x5283,0x5282,0x5331,0x53AD,0x55FE,0x5600,/* 0xB8-0xBF */ + 0x561B,0x5617,0x55FD,0x5614,0x5606,0x5609,0x560D,0x560E,/* 0xC0-0xC7 */ + 0x55F7,0x5616,0x561F,0x5608,0x5610,0x55F6,0x5718,0x5716,/* 0xC8-0xCF */ + 0x5875,0x587E,0x5883,0x5893,0x588A,0x5879,0x5885,0x587D,/* 0xD0-0xD7 */ + 0x58FD,0x5925,0x5922,0x5924,0x596A,0x5969,0x5AE1,0x5AE6,/* 0xD8-0xDF */ + 0x5AE9,0x5AD7,0x5AD6,0x5AD8,0x5AE3,0x5B75,0x5BDE,0x5BE7,/* 0xE0-0xE7 */ + 0x5BE1,0x5BE5,0x5BE6,0x5BE8,0x5BE2,0x5BE4,0x5BDF,0x5C0D,/* 0xE8-0xEF */ + 0x5C62,0x5D84,0x5D87,0x5E5B,0x5E63,0x5E55,0x5E57,0x5E54,/* 0xF0-0xF7 */ + 0x5ED3,0x5ED6,0x5F0A,0x5F46,0x5F70,0x5FB9,0x6147,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_BA[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x613F,0x614B,0x6177,0x6162,0x6163,0x615F,0x615A,0x6158,/* 0x40-0x47 */ + 0x6175,0x622A,0x6487,0x6458,0x6454,0x64A4,0x6478,0x645F,/* 0x48-0x4F */ + 0x647A,0x6451,0x6467,0x6434,0x646D,0x647B,0x6572,0x65A1,/* 0x50-0x57 */ + 0x65D7,0x65D6,0x66A2,0x66A8,0x669D,0x699C,0x69A8,0x6995,/* 0x58-0x5F */ + 0x69C1,0x69AE,0x69D3,0x69CB,0x699B,0x69B7,0x69BB,0x69AB,/* 0x60-0x67 */ + 0x69B4,0x69D0,0x69CD,0x69AD,0x69CC,0x69A6,0x69C3,0x69A3,/* 0x68-0x6F */ + 0x6B49,0x6B4C,0x6C33,0x6F33,0x6F14,0x6EFE,0x6F13,0x6EF4,/* 0x70-0x77 */ + 0x6F29,0x6F3E,0x6F20,0x6F2C,0x6F0F,0x6F02,0x6F22,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x6EFF,0x6EEF,0x6F06,0x6F31,0x6F38,0x6F32,0x6F23,/* 0xA0-0xA7 */ + 0x6F15,0x6F2B,0x6F2F,0x6F88,0x6F2A,0x6EEC,0x6F01,0x6EF2,/* 0xA8-0xAF */ + 0x6ECC,0x6EF7,0x7194,0x7199,0x717D,0x718A,0x7184,0x7192,/* 0xB0-0xB7 */ + 0x723E,0x7292,0x7296,0x7344,0x7350,0x7464,0x7463,0x746A,/* 0xB8-0xBF */ + 0x7470,0x746D,0x7504,0x7591,0x7627,0x760D,0x760B,0x7609,/* 0xC0-0xC7 */ + 0x7613,0x76E1,0x76E3,0x7784,0x777D,0x777F,0x7761,0x78C1,/* 0xC8-0xCF */ + 0x789F,0x78A7,0x78B3,0x78A9,0x78A3,0x798E,0x798F,0x798D,/* 0xD0-0xD7 */ + 0x7A2E,0x7A31,0x7AAA,0x7AA9,0x7AED,0x7AEF,0x7BA1,0x7B95,/* 0xD8-0xDF */ + 0x7B8B,0x7B75,0x7B97,0x7B9D,0x7B94,0x7B8F,0x7BB8,0x7B87,/* 0xE0-0xE7 */ + 0x7B84,0x7CB9,0x7CBD,0x7CBE,0x7DBB,0x7DB0,0x7D9C,0x7DBD,/* 0xE8-0xEF */ + 0x7DBE,0x7DA0,0x7DCA,0x7DB4,0x7DB2,0x7DB1,0x7DBA,0x7DA2,/* 0xF0-0xF7 */ + 0x7DBF,0x7DB5,0x7DB8,0x7DAD,0x7DD2,0x7DC7,0x7DAC,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_BB[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7F70,0x7FE0,0x7FE1,0x7FDF,0x805E,0x805A,0x8087,0x8150,/* 0x40-0x47 */ + 0x8180,0x818F,0x8188,0x818A,0x817F,0x8182,0x81E7,0x81FA,/* 0x48-0x4F */ + 0x8207,0x8214,0x821E,0x824B,0x84C9,0x84BF,0x84C6,0x84C4,/* 0x50-0x57 */ + 0x8499,0x849E,0x84B2,0x849C,0x84CB,0x84B8,0x84C0,0x84D3,/* 0x58-0x5F */ + 0x8490,0x84BC,0x84D1,0x84CA,0x873F,0x871C,0x873B,0x8722,/* 0x60-0x67 */ + 0x8725,0x8734,0x8718,0x8755,0x8737,0x8729,0x88F3,0x8902,/* 0x68-0x6F */ + 0x88F4,0x88F9,0x88F8,0x88FD,0x88E8,0x891A,0x88EF,0x8AA6,/* 0x70-0x77 */ + 0x8A8C,0x8A9E,0x8AA3,0x8A8D,0x8AA1,0x8A93,0x8AA4,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x8AAA,0x8AA5,0x8AA8,0x8A98,0x8A91,0x8A9A,0x8AA7,/* 0xA0-0xA7 */ + 0x8C6A,0x8C8D,0x8C8C,0x8CD3,0x8CD1,0x8CD2,0x8D6B,0x8D99,/* 0xA8-0xAF */ + 0x8D95,0x8DFC,0x8F14,0x8F12,0x8F15,0x8F13,0x8FA3,0x9060,/* 0xB0-0xB7 */ + 0x9058,0x905C,0x9063,0x9059,0x905E,0x9062,0x905D,0x905B,/* 0xB8-0xBF */ + 0x9119,0x9118,0x911E,0x9175,0x9178,0x9177,0x9174,0x9278,/* 0xC0-0xC7 */ + 0x9280,0x9285,0x9298,0x9296,0x927B,0x9293,0x929C,0x92A8,/* 0xC8-0xCF */ + 0x927C,0x9291,0x95A1,0x95A8,0x95A9,0x95A3,0x95A5,0x95A4,/* 0xD0-0xD7 */ + 0x9699,0x969C,0x969B,0x96CC,0x96D2,0x9700,0x977C,0x9785,/* 0xD8-0xDF */ + 0x97F6,0x9817,0x9818,0x98AF,0x98B1,0x9903,0x9905,0x990C,/* 0xE0-0xE7 */ + 0x9909,0x99C1,0x9AAF,0x9AB0,0x9AE6,0x9B41,0x9B42,0x9CF4,/* 0xE8-0xEF */ + 0x9CF6,0x9CF3,0x9EBC,0x9F3B,0x9F4A,0x5104,0x5100,0x50FB,/* 0xF0-0xF7 */ + 0x50F5,0x50F9,0x5102,0x5108,0x5109,0x5105,0x51DC,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_BC[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x5287,0x5288,0x5289,0x528D,0x528A,0x52F0,0x53B2,0x562E,/* 0x40-0x47 */ + 0x563B,0x5639,0x5632,0x563F,0x5634,0x5629,0x5653,0x564E,/* 0x48-0x4F */ + 0x5657,0x5674,0x5636,0x562F,0x5630,0x5880,0x589F,0x589E,/* 0x50-0x57 */ + 0x58B3,0x589C,0x58AE,0x58A9,0x58A6,0x596D,0x5B09,0x5AFB,/* 0x58-0x5F */ + 0x5B0B,0x5AF5,0x5B0C,0x5B08,0x5BEE,0x5BEC,0x5BE9,0x5BEB,/* 0x60-0x67 */ + 0x5C64,0x5C65,0x5D9D,0x5D94,0x5E62,0x5E5F,0x5E61,0x5EE2,/* 0x68-0x6F */ + 0x5EDA,0x5EDF,0x5EDD,0x5EE3,0x5EE0,0x5F48,0x5F71,0x5FB7,/* 0x70-0x77 */ + 0x5FB5,0x6176,0x6167,0x616E,0x615D,0x6155,0x6182,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x617C,0x6170,0x616B,0x617E,0x61A7,0x6190,0x61AB,/* 0xA0-0xA7 */ + 0x618E,0x61AC,0x619A,0x61A4,0x6194,0x61AE,0x622E,0x6469,/* 0xA8-0xAF */ + 0x646F,0x6479,0x649E,0x64B2,0x6488,0x6490,0x64B0,0x64A5,/* 0xB0-0xB7 */ + 0x6493,0x6495,0x64A9,0x6492,0x64AE,0x64AD,0x64AB,0x649A,/* 0xB8-0xBF */ + 0x64AC,0x6499,0x64A2,0x64B3,0x6575,0x6577,0x6578,0x66AE,/* 0xC0-0xC7 */ + 0x66AB,0x66B4,0x66B1,0x6A23,0x6A1F,0x69E8,0x6A01,0x6A1E,/* 0xC8-0xCF */ + 0x6A19,0x69FD,0x6A21,0x6A13,0x6A0A,0x69F3,0x6A02,0x6A05,/* 0xD0-0xD7 */ + 0x69ED,0x6A11,0x6B50,0x6B4E,0x6BA4,0x6BC5,0x6BC6,0x6F3F,/* 0xD8-0xDF */ + 0x6F7C,0x6F84,0x6F51,0x6F66,0x6F54,0x6F86,0x6F6D,0x6F5B,/* 0xE0-0xE7 */ + 0x6F78,0x6F6E,0x6F8E,0x6F7A,0x6F70,0x6F64,0x6F97,0x6F58,/* 0xE8-0xEF */ + 0x6ED5,0x6F6F,0x6F60,0x6F5F,0x719F,0x71AC,0x71B1,0x71A8,/* 0xF0-0xF7 */ + 0x7256,0x729B,0x734E,0x7357,0x7469,0x748B,0x7483,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_BD[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x747E,0x7480,0x757F,0x7620,0x7629,0x761F,0x7624,0x7626,/* 0x40-0x47 */ + 0x7621,0x7622,0x769A,0x76BA,0x76E4,0x778E,0x7787,0x778C,/* 0x48-0x4F */ + 0x7791,0x778B,0x78CB,0x78C5,0x78BA,0x78CA,0x78BE,0x78D5,/* 0x50-0x57 */ + 0x78BC,0x78D0,0x7A3F,0x7A3C,0x7A40,0x7A3D,0x7A37,0x7A3B,/* 0x58-0x5F */ + 0x7AAF,0x7AAE,0x7BAD,0x7BB1,0x7BC4,0x7BB4,0x7BC6,0x7BC7,/* 0x60-0x67 */ + 0x7BC1,0x7BA0,0x7BCC,0x7CCA,0x7DE0,0x7DF4,0x7DEF,0x7DFB,/* 0x68-0x6F */ + 0x7DD8,0x7DEC,0x7DDD,0x7DE8,0x7DE3,0x7DDA,0x7DDE,0x7DE9,/* 0x70-0x77 */ + 0x7D9E,0x7DD9,0x7DF2,0x7DF9,0x7F75,0x7F77,0x7FAF,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x7FE9,0x8026,0x819B,0x819C,0x819D,0x81A0,0x819A,/* 0xA0-0xA7 */ + 0x8198,0x8517,0x853D,0x851A,0x84EE,0x852C,0x852D,0x8513,/* 0xA8-0xAF */ + 0x8511,0x8523,0x8521,0x8514,0x84EC,0x8525,0x84FF,0x8506,/* 0xB0-0xB7 */ + 0x8782,0x8774,0x8776,0x8760,0x8766,0x8778,0x8768,0x8759,/* 0xB8-0xBF */ + 0x8757,0x874C,0x8753,0x885B,0x885D,0x8910,0x8907,0x8912,/* 0xC0-0xC7 */ + 0x8913,0x8915,0x890A,0x8ABC,0x8AD2,0x8AC7,0x8AC4,0x8A95,/* 0xC8-0xCF */ + 0x8ACB,0x8AF8,0x8AB2,0x8AC9,0x8AC2,0x8ABF,0x8AB0,0x8AD6,/* 0xD0-0xD7 */ + 0x8ACD,0x8AB6,0x8AB9,0x8ADB,0x8C4C,0x8C4E,0x8C6C,0x8CE0,/* 0xD8-0xDF */ + 0x8CDE,0x8CE6,0x8CE4,0x8CEC,0x8CED,0x8CE2,0x8CE3,0x8CDC,/* 0xE0-0xE7 */ + 0x8CEA,0x8CE1,0x8D6D,0x8D9F,0x8DA3,0x8E2B,0x8E10,0x8E1D,/* 0xE8-0xEF */ + 0x8E22,0x8E0F,0x8E29,0x8E1F,0x8E21,0x8E1E,0x8EBA,0x8F1D,/* 0xF0-0xF7 */ + 0x8F1B,0x8F1F,0x8F29,0x8F26,0x8F2A,0x8F1C,0x8F1E,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_BE[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8F25,0x9069,0x906E,0x9068,0x906D,0x9077,0x9130,0x912D,/* 0x40-0x47 */ + 0x9127,0x9131,0x9187,0x9189,0x918B,0x9183,0x92C5,0x92BB,/* 0x48-0x4F */ + 0x92B7,0x92EA,0x92AC,0x92E4,0x92C1,0x92B3,0x92BC,0x92D2,/* 0x50-0x57 */ + 0x92C7,0x92F0,0x92B2,0x95AD,0x95B1,0x9704,0x9706,0x9707,/* 0x58-0x5F */ + 0x9709,0x9760,0x978D,0x978B,0x978F,0x9821,0x982B,0x981C,/* 0x60-0x67 */ + 0x98B3,0x990A,0x9913,0x9912,0x9918,0x99DD,0x99D0,0x99DF,/* 0x68-0x6F */ + 0x99DB,0x99D1,0x99D5,0x99D2,0x99D9,0x9AB7,0x9AEE,0x9AEF,/* 0x70-0x77 */ + 0x9B27,0x9B45,0x9B44,0x9B77,0x9B6F,0x9D06,0x9D09,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x9D03,0x9EA9,0x9EBE,0x9ECE,0x58A8,0x9F52,0x5112,/* 0xA0-0xA7 */ + 0x5118,0x5114,0x5110,0x5115,0x5180,0x51AA,0x51DD,0x5291,/* 0xA8-0xAF */ + 0x5293,0x52F3,0x5659,0x566B,0x5679,0x5669,0x5664,0x5678,/* 0xB0-0xB7 */ + 0x566A,0x5668,0x5665,0x5671,0x566F,0x566C,0x5662,0x5676,/* 0xB8-0xBF */ + 0x58C1,0x58BE,0x58C7,0x58C5,0x596E,0x5B1D,0x5B34,0x5B78,/* 0xC0-0xC7 */ + 0x5BF0,0x5C0E,0x5F4A,0x61B2,0x6191,0x61A9,0x618A,0x61CD,/* 0xC8-0xCF */ + 0x61B6,0x61BE,0x61CA,0x61C8,0x6230,0x64C5,0x64C1,0x64CB,/* 0xD0-0xD7 */ + 0x64BB,0x64BC,0x64DA,0x64C4,0x64C7,0x64C2,0x64CD,0x64BF,/* 0xD8-0xDF */ + 0x64D2,0x64D4,0x64BE,0x6574,0x66C6,0x66C9,0x66B9,0x66C4,/* 0xE0-0xE7 */ + 0x66C7,0x66B8,0x6A3D,0x6A38,0x6A3A,0x6A59,0x6A6B,0x6A58,/* 0xE8-0xEF */ + 0x6A39,0x6A44,0x6A62,0x6A61,0x6A4B,0x6A47,0x6A35,0x6A5F,/* 0xF0-0xF7 */ + 0x6A48,0x6B59,0x6B77,0x6C05,0x6FC2,0x6FB1,0x6FA1,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_BF[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6FC3,0x6FA4,0x6FC1,0x6FA7,0x6FB3,0x6FC0,0x6FB9,0x6FB6,/* 0x40-0x47 */ + 0x6FA6,0x6FA0,0x6FB4,0x71BE,0x71C9,0x71D0,0x71D2,0x71C8,/* 0x48-0x4F */ + 0x71D5,0x71B9,0x71CE,0x71D9,0x71DC,0x71C3,0x71C4,0x7368,/* 0x50-0x57 */ + 0x749C,0x74A3,0x7498,0x749F,0x749E,0x74E2,0x750C,0x750D,/* 0x58-0x5F */ + 0x7634,0x7638,0x763A,0x76E7,0x76E5,0x77A0,0x779E,0x779F,/* 0x60-0x67 */ + 0x77A5,0x78E8,0x78DA,0x78EC,0x78E7,0x79A6,0x7A4D,0x7A4E,/* 0x68-0x6F */ + 0x7A46,0x7A4C,0x7A4B,0x7ABA,0x7BD9,0x7C11,0x7BC9,0x7BE4,/* 0x70-0x77 */ + 0x7BDB,0x7BE1,0x7BE9,0x7BE6,0x7CD5,0x7CD6,0x7E0A,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x7E11,0x7E08,0x7E1B,0x7E23,0x7E1E,0x7E1D,0x7E09,/* 0xA0-0xA7 */ + 0x7E10,0x7F79,0x7FB2,0x7FF0,0x7FF1,0x7FEE,0x8028,0x81B3,/* 0xA8-0xAF */ + 0x81A9,0x81A8,0x81FB,0x8208,0x8258,0x8259,0x854A,0x8559,/* 0xB0-0xB7 */ + 0x8548,0x8568,0x8569,0x8543,0x8549,0x856D,0x856A,0x855E,/* 0xB8-0xBF */ + 0x8783,0x879F,0x879E,0x87A2,0x878D,0x8861,0x892A,0x8932,/* 0xC0-0xC7 */ + 0x8925,0x892B,0x8921,0x89AA,0x89A6,0x8AE6,0x8AFA,0x8AEB,/* 0xC8-0xCF */ + 0x8AF1,0x8B00,0x8ADC,0x8AE7,0x8AEE,0x8AFE,0x8B01,0x8B02,/* 0xD0-0xD7 */ + 0x8AF7,0x8AED,0x8AF3,0x8AF6,0x8AFC,0x8C6B,0x8C6D,0x8C93,/* 0xD8-0xDF */ + 0x8CF4,0x8E44,0x8E31,0x8E34,0x8E42,0x8E39,0x8E35,0x8F3B,/* 0xE0-0xE7 */ + 0x8F2F,0x8F38,0x8F33,0x8FA8,0x8FA6,0x9075,0x9074,0x9078,/* 0xE8-0xEF */ + 0x9072,0x907C,0x907A,0x9134,0x9192,0x9320,0x9336,0x92F8,/* 0xF0-0xF7 */ + 0x9333,0x932F,0x9322,0x92FC,0x932B,0x9304,0x931A,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_C0[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9310,0x9326,0x9321,0x9315,0x932E,0x9319,0x95BB,0x96A7,/* 0x40-0x47 */ + 0x96A8,0x96AA,0x96D5,0x970E,0x9711,0x9716,0x970D,0x9713,/* 0x48-0x4F */ + 0x970F,0x975B,0x975C,0x9766,0x9798,0x9830,0x9838,0x983B,/* 0x50-0x57 */ + 0x9837,0x982D,0x9839,0x9824,0x9910,0x9928,0x991E,0x991B,/* 0x58-0x5F */ + 0x9921,0x991A,0x99ED,0x99E2,0x99F1,0x9AB8,0x9ABC,0x9AFB,/* 0x60-0x67 */ + 0x9AED,0x9B28,0x9B91,0x9D15,0x9D23,0x9D26,0x9D28,0x9D12,/* 0x68-0x6F */ + 0x9D1B,0x9ED8,0x9ED4,0x9F8D,0x9F9C,0x512A,0x511F,0x5121,/* 0x70-0x77 */ + 0x5132,0x52F5,0x568E,0x5680,0x5690,0x5685,0x5687,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x568F,0x58D5,0x58D3,0x58D1,0x58CE,0x5B30,0x5B2A,/* 0xA0-0xA7 */ + 0x5B24,0x5B7A,0x5C37,0x5C68,0x5DBC,0x5DBA,0x5DBD,0x5DB8,/* 0xA8-0xAF */ + 0x5E6B,0x5F4C,0x5FBD,0x61C9,0x61C2,0x61C7,0x61E6,0x61CB,/* 0xB0-0xB7 */ + 0x6232,0x6234,0x64CE,0x64CA,0x64D8,0x64E0,0x64F0,0x64E6,/* 0xB8-0xBF */ + 0x64EC,0x64F1,0x64E2,0x64ED,0x6582,0x6583,0x66D9,0x66D6,/* 0xC0-0xC7 */ + 0x6A80,0x6A94,0x6A84,0x6AA2,0x6A9C,0x6ADB,0x6AA3,0x6A7E,/* 0xC8-0xCF */ + 0x6A97,0x6A90,0x6AA0,0x6B5C,0x6BAE,0x6BDA,0x6C08,0x6FD8,/* 0xD0-0xD7 */ + 0x6FF1,0x6FDF,0x6FE0,0x6FDB,0x6FE4,0x6FEB,0x6FEF,0x6F80,/* 0xD8-0xDF */ + 0x6FEC,0x6FE1,0x6FE9,0x6FD5,0x6FEE,0x6FF0,0x71E7,0x71DF,/* 0xE0-0xE7 */ + 0x71EE,0x71E6,0x71E5,0x71ED,0x71EC,0x71F4,0x71E0,0x7235,/* 0xE8-0xEF */ + 0x7246,0x7370,0x7372,0x74A9,0x74B0,0x74A6,0x74A8,0x7646,/* 0xF0-0xF7 */ + 0x7642,0x764C,0x76EA,0x77B3,0x77AA,0x77B0,0x77AC,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_C1[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x77A7,0x77AD,0x77EF,0x78F7,0x78FA,0x78F4,0x78EF,0x7901,/* 0x40-0x47 */ + 0x79A7,0x79AA,0x7A57,0x7ABF,0x7C07,0x7C0D,0x7BFE,0x7BF7,/* 0x48-0x4F */ + 0x7C0C,0x7BE0,0x7CE0,0x7CDC,0x7CDE,0x7CE2,0x7CDF,0x7CD9,/* 0x50-0x57 */ + 0x7CDD,0x7E2E,0x7E3E,0x7E46,0x7E37,0x7E32,0x7E43,0x7E2B,/* 0x58-0x5F */ + 0x7E3D,0x7E31,0x7E45,0x7E41,0x7E34,0x7E39,0x7E48,0x7E35,/* 0x60-0x67 */ + 0x7E3F,0x7E2F,0x7F44,0x7FF3,0x7FFC,0x8071,0x8072,0x8070,/* 0x68-0x6F */ + 0x806F,0x8073,0x81C6,0x81C3,0x81BA,0x81C2,0x81C0,0x81BF,/* 0x70-0x77 */ + 0x81BD,0x81C9,0x81BE,0x81E8,0x8209,0x8271,0x85AA,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x8584,0x857E,0x859C,0x8591,0x8594,0x85AF,0x859B,/* 0xA0-0xA7 */ + 0x8587,0x85A8,0x858A,0x8667,0x87C0,0x87D1,0x87B3,0x87D2,/* 0xA8-0xAF */ + 0x87C6,0x87AB,0x87BB,0x87BA,0x87C8,0x87CB,0x893B,0x8936,/* 0xB0-0xB7 */ + 0x8944,0x8938,0x893D,0x89AC,0x8B0E,0x8B17,0x8B19,0x8B1B,/* 0xB8-0xBF */ + 0x8B0A,0x8B20,0x8B1D,0x8B04,0x8B10,0x8C41,0x8C3F,0x8C73,/* 0xC0-0xC7 */ + 0x8CFA,0x8CFD,0x8CFC,0x8CF8,0x8CFB,0x8DA8,0x8E49,0x8E4B,/* 0xC8-0xCF */ + 0x8E48,0x8E4A,0x8F44,0x8F3E,0x8F42,0x8F45,0x8F3F,0x907F,/* 0xD0-0xD7 */ + 0x907D,0x9084,0x9081,0x9082,0x9080,0x9139,0x91A3,0x919E,/* 0xD8-0xDF */ + 0x919C,0x934D,0x9382,0x9328,0x9375,0x934A,0x9365,0x934B,/* 0xE0-0xE7 */ + 0x9318,0x937E,0x936C,0x935B,0x9370,0x935A,0x9354,0x95CA,/* 0xE8-0xEF */ + 0x95CB,0x95CC,0x95C8,0x95C6,0x96B1,0x96B8,0x96D6,0x971C,/* 0xF0-0xF7 */ + 0x971E,0x97A0,0x97D3,0x9846,0x98B6,0x9935,0x9A01,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_C2[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x99FF,0x9BAE,0x9BAB,0x9BAA,0x9BAD,0x9D3B,0x9D3F,0x9E8B,/* 0x40-0x47 */ + 0x9ECF,0x9EDE,0x9EDC,0x9EDD,0x9EDB,0x9F3E,0x9F4B,0x53E2,/* 0x48-0x4F */ + 0x5695,0x56AE,0x58D9,0x58D8,0x5B38,0x5F5D,0x61E3,0x6233,/* 0x50-0x57 */ + 0x64F4,0x64F2,0x64FE,0x6506,0x64FA,0x64FB,0x64F7,0x65B7,/* 0x58-0x5F */ + 0x66DC,0x6726,0x6AB3,0x6AAC,0x6AC3,0x6ABB,0x6AB8,0x6AC2,/* 0x60-0x67 */ + 0x6AAE,0x6AAF,0x6B5F,0x6B78,0x6BAF,0x7009,0x700B,0x6FFE,/* 0x68-0x6F */ + 0x7006,0x6FFA,0x7011,0x700F,0x71FB,0x71FC,0x71FE,0x71F8,/* 0x70-0x77 */ + 0x7377,0x7375,0x74A7,0x74BF,0x7515,0x7656,0x7658,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x7652,0x77BD,0x77BF,0x77BB,0x77BC,0x790E,0x79AE,/* 0xA0-0xA7 */ + 0x7A61,0x7A62,0x7A60,0x7AC4,0x7AC5,0x7C2B,0x7C27,0x7C2A,/* 0xA8-0xAF */ + 0x7C1E,0x7C23,0x7C21,0x7CE7,0x7E54,0x7E55,0x7E5E,0x7E5A,/* 0xB0-0xB7 */ + 0x7E61,0x7E52,0x7E59,0x7F48,0x7FF9,0x7FFB,0x8077,0x8076,/* 0xB8-0xBF */ + 0x81CD,0x81CF,0x820A,0x85CF,0x85A9,0x85CD,0x85D0,0x85C9,/* 0xC0-0xC7 */ + 0x85B0,0x85BA,0x85B9,0x85A6,0x87EF,0x87EC,0x87F2,0x87E0,/* 0xC8-0xCF */ + 0x8986,0x89B2,0x89F4,0x8B28,0x8B39,0x8B2C,0x8B2B,0x8C50,/* 0xD0-0xD7 */ + 0x8D05,0x8E59,0x8E63,0x8E66,0x8E64,0x8E5F,0x8E55,0x8EC0,/* 0xD8-0xDF */ + 0x8F49,0x8F4D,0x9087,0x9083,0x9088,0x91AB,0x91AC,0x91D0,/* 0xE0-0xE7 */ + 0x9394,0x938A,0x9396,0x93A2,0x93B3,0x93AE,0x93AC,0x93B0,/* 0xE8-0xEF */ + 0x9398,0x939A,0x9397,0x95D4,0x95D6,0x95D0,0x95D5,0x96E2,/* 0xF0-0xF7 */ + 0x96DC,0x96D9,0x96DB,0x96DE,0x9724,0x97A3,0x97A6,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_C3[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x97AD,0x97F9,0x984D,0x984F,0x984C,0x984E,0x9853,0x98BA,/* 0x40-0x47 */ + 0x993E,0x993F,0x993D,0x992E,0x99A5,0x9A0E,0x9AC1,0x9B03,/* 0x48-0x4F */ + 0x9B06,0x9B4F,0x9B4E,0x9B4D,0x9BCA,0x9BC9,0x9BFD,0x9BC8,/* 0x50-0x57 */ + 0x9BC0,0x9D51,0x9D5D,0x9D60,0x9EE0,0x9F15,0x9F2C,0x5133,/* 0x58-0x5F */ + 0x56A5,0x58DE,0x58DF,0x58E2,0x5BF5,0x9F90,0x5EEC,0x61F2,/* 0x60-0x67 */ + 0x61F7,0x61F6,0x61F5,0x6500,0x650F,0x66E0,0x66DD,0x6AE5,/* 0x68-0x6F */ + 0x6ADD,0x6ADA,0x6AD3,0x701B,0x701F,0x7028,0x701A,0x701D,/* 0x70-0x77 */ + 0x7015,0x7018,0x7206,0x720D,0x7258,0x72A2,0x7378,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x737A,0x74BD,0x74CA,0x74E3,0x7587,0x7586,0x765F,/* 0xA0-0xA7 */ + 0x7661,0x77C7,0x7919,0x79B1,0x7A6B,0x7A69,0x7C3E,0x7C3F,/* 0xA8-0xAF */ + 0x7C38,0x7C3D,0x7C37,0x7C40,0x7E6B,0x7E6D,0x7E79,0x7E69,/* 0xB0-0xB7 */ + 0x7E6A,0x7F85,0x7E73,0x7FB6,0x7FB9,0x7FB8,0x81D8,0x85E9,/* 0xB8-0xBF */ + 0x85DD,0x85EA,0x85D5,0x85E4,0x85E5,0x85F7,0x87FB,0x8805,/* 0xC0-0xC7 */ + 0x880D,0x87F9,0x87FE,0x8960,0x895F,0x8956,0x895E,0x8B41,/* 0xC8-0xCF */ + 0x8B5C,0x8B58,0x8B49,0x8B5A,0x8B4E,0x8B4F,0x8B46,0x8B59,/* 0xD0-0xD7 */ + 0x8D08,0x8D0A,0x8E7C,0x8E72,0x8E87,0x8E76,0x8E6C,0x8E7A,/* 0xD8-0xDF */ + 0x8E74,0x8F54,0x8F4E,0x8FAD,0x908A,0x908B,0x91B1,0x91AE,/* 0xE0-0xE7 */ + 0x93E1,0x93D1,0x93DF,0x93C3,0x93C8,0x93DC,0x93DD,0x93D6,/* 0xE8-0xEF */ + 0x93E2,0x93CD,0x93D8,0x93E4,0x93D7,0x93E8,0x95DC,0x96B4,/* 0xF0-0xF7 */ + 0x96E3,0x972A,0x9727,0x9761,0x97DC,0x97FB,0x985E,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_C4[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x9858,0x985B,0x98BC,0x9945,0x9949,0x9A16,0x9A19,0x9B0D,/* 0x40-0x47 */ + 0x9BE8,0x9BE7,0x9BD6,0x9BDB,0x9D89,0x9D61,0x9D72,0x9D6A,/* 0x48-0x4F */ + 0x9D6C,0x9E92,0x9E97,0x9E93,0x9EB4,0x52F8,0x56A8,0x56B7,/* 0x50-0x57 */ + 0x56B6,0x56B4,0x56BC,0x58E4,0x5B40,0x5B43,0x5B7D,0x5BF6,/* 0x58-0x5F */ + 0x5DC9,0x61F8,0x61FA,0x6518,0x6514,0x6519,0x66E6,0x6727,/* 0x60-0x67 */ + 0x6AEC,0x703E,0x7030,0x7032,0x7210,0x737B,0x74CF,0x7662,/* 0x68-0x6F */ + 0x7665,0x7926,0x792A,0x792C,0x792B,0x7AC7,0x7AF6,0x7C4C,/* 0x70-0x77 */ + 0x7C43,0x7C4D,0x7CEF,0x7CF0,0x8FAE,0x7E7D,0x7E7C,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x7E82,0x7F4C,0x8000,0x81DA,0x8266,0x85FB,0x85F9,/* 0xA0-0xA7 */ + 0x8611,0x85FA,0x8606,0x860B,0x8607,0x860A,0x8814,0x8815,/* 0xA8-0xAF */ + 0x8964,0x89BA,0x89F8,0x8B70,0x8B6C,0x8B66,0x8B6F,0x8B5F,/* 0xB0-0xB7 */ + 0x8B6B,0x8D0F,0x8D0D,0x8E89,0x8E81,0x8E85,0x8E82,0x91B4,/* 0xB8-0xBF */ + 0x91CB,0x9418,0x9403,0x93FD,0x95E1,0x9730,0x98C4,0x9952,/* 0xC0-0xC7 */ + 0x9951,0x99A8,0x9A2B,0x9A30,0x9A37,0x9A35,0x9C13,0x9C0D,/* 0xC8-0xCF */ + 0x9E79,0x9EB5,0x9EE8,0x9F2F,0x9F5F,0x9F63,0x9F61,0x5137,/* 0xD0-0xD7 */ + 0x5138,0x56C1,0x56C0,0x56C2,0x5914,0x5C6C,0x5DCD,0x61FC,/* 0xD8-0xDF */ + 0x61FE,0x651D,0x651C,0x6595,0x66E9,0x6AFB,0x6B04,0x6AFA,/* 0xE0-0xE7 */ + 0x6BB2,0x704C,0x721B,0x72A7,0x74D6,0x74D4,0x7669,0x77D3,/* 0xE8-0xEF */ + 0x7C50,0x7E8F,0x7E8C,0x7FBC,0x8617,0x862D,0x861A,0x8823,/* 0xF0-0xF7 */ + 0x8822,0x8821,0x881F,0x896A,0x896C,0x89BD,0x8B74,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_C5[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8B77,0x8B7D,0x8D13,0x8E8A,0x8E8D,0x8E8B,0x8F5F,0x8FAF,/* 0x40-0x47 */ + 0x91BA,0x942E,0x9433,0x9435,0x943A,0x9438,0x9432,0x942B,/* 0x48-0x4F */ + 0x95E2,0x9738,0x9739,0x9732,0x97FF,0x9867,0x9865,0x9957,/* 0x50-0x57 */ + 0x9A45,0x9A43,0x9A40,0x9A3E,0x9ACF,0x9B54,0x9B51,0x9C2D,/* 0x58-0x5F */ + 0x9C25,0x9DAF,0x9DB4,0x9DC2,0x9DB8,0x9E9D,0x9EEF,0x9F19,/* 0x60-0x67 */ + 0x9F5C,0x9F66,0x9F67,0x513C,0x513B,0x56C8,0x56CA,0x56C9,/* 0x68-0x6F */ + 0x5B7F,0x5DD4,0x5DD2,0x5F4E,0x61FF,0x6524,0x6B0A,0x6B61,/* 0x70-0x77 */ + 0x7051,0x7058,0x7380,0x74E4,0x758A,0x766E,0x766C,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x79B3,0x7C60,0x7C5F,0x807E,0x807D,0x81DF,0x8972,/* 0xA0-0xA7 */ + 0x896F,0x89FC,0x8B80,0x8D16,0x8D17,0x8E91,0x8E93,0x8F61,/* 0xA8-0xAF */ + 0x9148,0x9444,0x9451,0x9452,0x973D,0x973E,0x97C3,0x97C1,/* 0xB0-0xB7 */ + 0x986B,0x9955,0x9A55,0x9A4D,0x9AD2,0x9B1A,0x9C49,0x9C31,/* 0xB8-0xBF */ + 0x9C3E,0x9C3B,0x9DD3,0x9DD7,0x9F34,0x9F6C,0x9F6A,0x9F94,/* 0xC0-0xC7 */ + 0x56CC,0x5DD6,0x6200,0x6523,0x652B,0x652A,0x66EC,0x6B10,/* 0xC8-0xCF */ + 0x74DA,0x7ACA,0x7C64,0x7C63,0x7C65,0x7E93,0x7E96,0x7E94,/* 0xD0-0xD7 */ + 0x81E2,0x8638,0x863F,0x8831,0x8B8A,0x9090,0x908F,0x9463,/* 0xD8-0xDF */ + 0x9460,0x9464,0x9768,0x986F,0x995C,0x9A5A,0x9A5B,0x9A57,/* 0xE0-0xE7 */ + 0x9AD3,0x9AD4,0x9AD1,0x9C54,0x9C57,0x9C56,0x9DE5,0x9E9F,/* 0xE8-0xEF */ + 0x9EF4,0x56D1,0x58E9,0x652C,0x705E,0x7671,0x7672,0x77D7,/* 0xF0-0xF7 */ + 0x7F50,0x7F88,0x8836,0x8839,0x8862,0x8B93,0x8B92,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_C6[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8B96,0x8277,0x8D1B,0x91C0,0x946A,0x9742,0x9748,0x9744,/* 0x40-0x47 */ + 0x97C6,0x9870,0x9A5F,0x9B22,0x9B58,0x9C5F,0x9DF9,0x9DFA,/* 0x48-0x4F */ + 0x9E7C,0x9E7D,0x9F07,0x9F77,0x9F72,0x5EF3,0x6B16,0x7063,/* 0x50-0x57 */ + 0x7C6C,0x7C6E,0x883B,0x89C0,0x8EA1,0x91C1,0x9472,0x9470,/* 0x58-0x5F */ + 0x9871,0x995E,0x9AD6,0x9B23,0x9ECC,0x7064,0x77DA,0x8B9A,/* 0x60-0x67 */ + 0x9477,0x97C9,0x9A62,0x9A65,0x7E9C,0x8B9C,0x8EAA,0x91C5,/* 0x68-0x6F */ + 0x947D,0x947E,0x947C,0x9C77,0x9C78,0x9EF7,0x8C54,0x947F,/* 0x70-0x77 */ + 0x9E1A,0x7228,0x9A6A,0x9B31,0x9E1B,0x9E1E,0x7C72,0x0000,/* 0x78-0x7F */ +}; + +static const wchar_t c2u_C9[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x4E42,0x4E5C,0x51F5,0x531A,0x5382,0x4E07,0x4E0C,0x4E47,/* 0x40-0x47 */ + 0x4E8D,0x56D7,0xFA0C,0x5C6E,0x5F73,0x4E0F,0x5187,0x4E0E,/* 0x48-0x4F */ + 0x4E2E,0x4E93,0x4EC2,0x4EC9,0x4EC8,0x5198,0x52FC,0x536C,/* 0x50-0x57 */ + 0x53B9,0x5720,0x5903,0x592C,0x5C10,0x5DFF,0x65E1,0x6BB3,/* 0x58-0x5F */ + 0x6BCC,0x6C14,0x723F,0x4E31,0x4E3C,0x4EE8,0x4EDC,0x4EE9,/* 0x60-0x67 */ + 0x4EE1,0x4EDD,0x4EDA,0x520C,0x531C,0x534C,0x5722,0x5723,/* 0x68-0x6F */ + 0x5917,0x592F,0x5B81,0x5B84,0x5C12,0x5C3B,0x5C74,0x5C73,/* 0x70-0x77 */ + 0x5E04,0x5E80,0x5E82,0x5FC9,0x6209,0x6250,0x6C15,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x6C36,0x6C43,0x6C3F,0x6C3B,0x72AE,0x72B0,0x738A,/* 0xA0-0xA7 */ + 0x79B8,0x808A,0x961E,0x4F0E,0x4F18,0x4F2C,0x4EF5,0x4F14,/* 0xA8-0xAF */ + 0x4EF1,0x4F00,0x4EF7,0x4F08,0x4F1D,0x4F02,0x4F05,0x4F22,/* 0xB0-0xB7 */ + 0x4F13,0x4F04,0x4EF4,0x4F12,0x51B1,0x5213,0x5209,0x5210,/* 0xB8-0xBF */ + 0x52A6,0x5322,0x531F,0x534D,0x538A,0x5407,0x56E1,0x56DF,/* 0xC0-0xC7 */ + 0x572E,0x572A,0x5734,0x593C,0x5980,0x597C,0x5985,0x597B,/* 0xC8-0xCF */ + 0x597E,0x5977,0x597F,0x5B56,0x5C15,0x5C25,0x5C7C,0x5C7A,/* 0xD0-0xD7 */ + 0x5C7B,0x5C7E,0x5DDF,0x5E75,0x5E84,0x5F02,0x5F1A,0x5F74,/* 0xD8-0xDF */ + 0x5FD5,0x5FD4,0x5FCF,0x625C,0x625E,0x6264,0x6261,0x6266,/* 0xE0-0xE7 */ + 0x6262,0x6259,0x6260,0x625A,0x6265,0x65EF,0x65EE,0x673E,/* 0xE8-0xEF */ + 0x6739,0x6738,0x673B,0x673A,0x673F,0x673C,0x6733,0x6C18,/* 0xF0-0xF7 */ + 0x6C46,0x6C52,0x6C5C,0x6C4F,0x6C4A,0x6C54,0x6C4B,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_CA[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6C4C,0x7071,0x725E,0x72B4,0x72B5,0x738E,0x752A,0x767F,/* 0x40-0x47 */ + 0x7A75,0x7F51,0x8278,0x827C,0x8280,0x827D,0x827F,0x864D,/* 0x48-0x4F */ + 0x897E,0x9099,0x9097,0x9098,0x909B,0x9094,0x9622,0x9624,/* 0x50-0x57 */ + 0x9620,0x9623,0x4F56,0x4F3B,0x4F62,0x4F49,0x4F53,0x4F64,/* 0x58-0x5F */ + 0x4F3E,0x4F67,0x4F52,0x4F5F,0x4F41,0x4F58,0x4F2D,0x4F33,/* 0x60-0x67 */ + 0x4F3F,0x4F61,0x518F,0x51B9,0x521C,0x521E,0x5221,0x52AD,/* 0x68-0x6F */ + 0x52AE,0x5309,0x5363,0x5372,0x538E,0x538F,0x5430,0x5437,/* 0x70-0x77 */ + 0x542A,0x5454,0x5445,0x5419,0x541C,0x5425,0x5418,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x543D,0x544F,0x5441,0x5428,0x5424,0x5447,0x56EE,/* 0xA0-0xA7 */ + 0x56E7,0x56E5,0x5741,0x5745,0x574C,0x5749,0x574B,0x5752,/* 0xA8-0xAF */ + 0x5906,0x5940,0x59A6,0x5998,0x59A0,0x5997,0x598E,0x59A2,/* 0xB0-0xB7 */ + 0x5990,0x598F,0x59A7,0x59A1,0x5B8E,0x5B92,0x5C28,0x5C2A,/* 0xB8-0xBF */ + 0x5C8D,0x5C8F,0x5C88,0x5C8B,0x5C89,0x5C92,0x5C8A,0x5C86,/* 0xC0-0xC7 */ + 0x5C93,0x5C95,0x5DE0,0x5E0A,0x5E0E,0x5E8B,0x5E89,0x5E8C,/* 0xC8-0xCF */ + 0x5E88,0x5E8D,0x5F05,0x5F1D,0x5F78,0x5F76,0x5FD2,0x5FD1,/* 0xD0-0xD7 */ + 0x5FD0,0x5FED,0x5FE8,0x5FEE,0x5FF3,0x5FE1,0x5FE4,0x5FE3,/* 0xD8-0xDF */ + 0x5FFA,0x5FEF,0x5FF7,0x5FFB,0x6000,0x5FF4,0x623A,0x6283,/* 0xE0-0xE7 */ + 0x628C,0x628E,0x628F,0x6294,0x6287,0x6271,0x627B,0x627A,/* 0xE8-0xEF */ + 0x6270,0x6281,0x6288,0x6277,0x627D,0x6272,0x6274,0x6537,/* 0xF0-0xF7 */ + 0x65F0,0x65F4,0x65F3,0x65F2,0x65F5,0x6745,0x6747,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_CB[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6759,0x6755,0x674C,0x6748,0x675D,0x674D,0x675A,0x674B,/* 0x40-0x47 */ + 0x6BD0,0x6C19,0x6C1A,0x6C78,0x6C67,0x6C6B,0x6C84,0x6C8B,/* 0x48-0x4F */ + 0x6C8F,0x6C71,0x6C6F,0x6C69,0x6C9A,0x6C6D,0x6C87,0x6C95,/* 0x50-0x57 */ + 0x6C9C,0x6C66,0x6C73,0x6C65,0x6C7B,0x6C8E,0x7074,0x707A,/* 0x58-0x5F */ + 0x7263,0x72BF,0x72BD,0x72C3,0x72C6,0x72C1,0x72BA,0x72C5,/* 0x60-0x67 */ + 0x7395,0x7397,0x7393,0x7394,0x7392,0x753A,0x7539,0x7594,/* 0x68-0x6F */ + 0x7595,0x7681,0x793D,0x8034,0x8095,0x8099,0x8090,0x8092,/* 0x70-0x77 */ + 0x809C,0x8290,0x828F,0x8285,0x828E,0x8291,0x8293,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x828A,0x8283,0x8284,0x8C78,0x8FC9,0x8FBF,0x909F,/* 0xA0-0xA7 */ + 0x90A1,0x90A5,0x909E,0x90A7,0x90A0,0x9630,0x9628,0x962F,/* 0xA8-0xAF */ + 0x962D,0x4E33,0x4F98,0x4F7C,0x4F85,0x4F7D,0x4F80,0x4F87,/* 0xB0-0xB7 */ + 0x4F76,0x4F74,0x4F89,0x4F84,0x4F77,0x4F4C,0x4F97,0x4F6A,/* 0xB8-0xBF */ + 0x4F9A,0x4F79,0x4F81,0x4F78,0x4F90,0x4F9C,0x4F94,0x4F9E,/* 0xC0-0xC7 */ + 0x4F92,0x4F82,0x4F95,0x4F6B,0x4F6E,0x519E,0x51BC,0x51BE,/* 0xC8-0xCF */ + 0x5235,0x5232,0x5233,0x5246,0x5231,0x52BC,0x530A,0x530B,/* 0xD0-0xD7 */ + 0x533C,0x5392,0x5394,0x5487,0x547F,0x5481,0x5491,0x5482,/* 0xD8-0xDF */ + 0x5488,0x546B,0x547A,0x547E,0x5465,0x546C,0x5474,0x5466,/* 0xE0-0xE7 */ + 0x548D,0x546F,0x5461,0x5460,0x5498,0x5463,0x5467,0x5464,/* 0xE8-0xEF */ + 0x56F7,0x56F9,0x576F,0x5772,0x576D,0x576B,0x5771,0x5770,/* 0xF0-0xF7 */ + 0x5776,0x5780,0x5775,0x577B,0x5773,0x5774,0x5762,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_CC[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x5768,0x577D,0x590C,0x5945,0x59B5,0x59BA,0x59CF,0x59CE,/* 0x40-0x47 */ + 0x59B2,0x59CC,0x59C1,0x59B6,0x59BC,0x59C3,0x59D6,0x59B1,/* 0x48-0x4F */ + 0x59BD,0x59C0,0x59C8,0x59B4,0x59C7,0x5B62,0x5B65,0x5B93,/* 0x50-0x57 */ + 0x5B95,0x5C44,0x5C47,0x5CAE,0x5CA4,0x5CA0,0x5CB5,0x5CAF,/* 0x58-0x5F */ + 0x5CA8,0x5CAC,0x5C9F,0x5CA3,0x5CAD,0x5CA2,0x5CAA,0x5CA7,/* 0x60-0x67 */ + 0x5C9D,0x5CA5,0x5CB6,0x5CB0,0x5CA6,0x5E17,0x5E14,0x5E19,/* 0x68-0x6F */ + 0x5F28,0x5F22,0x5F23,0x5F24,0x5F54,0x5F82,0x5F7E,0x5F7D,/* 0x70-0x77 */ + 0x5FDE,0x5FE5,0x602D,0x6026,0x6019,0x6032,0x600B,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x6034,0x600A,0x6017,0x6033,0x601A,0x601E,0x602C,/* 0xA0-0xA7 */ + 0x6022,0x600D,0x6010,0x602E,0x6013,0x6011,0x600C,0x6009,/* 0xA8-0xAF */ + 0x601C,0x6214,0x623D,0x62AD,0x62B4,0x62D1,0x62BE,0x62AA,/* 0xB0-0xB7 */ + 0x62B6,0x62CA,0x62AE,0x62B3,0x62AF,0x62BB,0x62A9,0x62B0,/* 0xB8-0xBF */ + 0x62B8,0x653D,0x65A8,0x65BB,0x6609,0x65FC,0x6604,0x6612,/* 0xC0-0xC7 */ + 0x6608,0x65FB,0x6603,0x660B,0x660D,0x6605,0x65FD,0x6611,/* 0xC8-0xCF */ + 0x6610,0x66F6,0x670A,0x6785,0x676C,0x678E,0x6792,0x6776,/* 0xD0-0xD7 */ + 0x677B,0x6798,0x6786,0x6784,0x6774,0x678D,0x678C,0x677A,/* 0xD8-0xDF */ + 0x679F,0x6791,0x6799,0x6783,0x677D,0x6781,0x6778,0x6779,/* 0xE0-0xE7 */ + 0x6794,0x6B25,0x6B80,0x6B7E,0x6BDE,0x6C1D,0x6C93,0x6CEC,/* 0xE8-0xEF */ + 0x6CEB,0x6CEE,0x6CD9,0x6CB6,0x6CD4,0x6CAD,0x6CE7,0x6CB7,/* 0xF0-0xF7 */ + 0x6CD0,0x6CC2,0x6CBA,0x6CC3,0x6CC6,0x6CED,0x6CF2,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_CD[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6CD2,0x6CDD,0x6CB4,0x6C8A,0x6C9D,0x6C80,0x6CDE,0x6CC0,/* 0x40-0x47 */ + 0x6D30,0x6CCD,0x6CC7,0x6CB0,0x6CF9,0x6CCF,0x6CE9,0x6CD1,/* 0x48-0x4F */ + 0x7094,0x7098,0x7085,0x7093,0x7086,0x7084,0x7091,0x7096,/* 0x50-0x57 */ + 0x7082,0x709A,0x7083,0x726A,0x72D6,0x72CB,0x72D8,0x72C9,/* 0x58-0x5F */ + 0x72DC,0x72D2,0x72D4,0x72DA,0x72CC,0x72D1,0x73A4,0x73A1,/* 0x60-0x67 */ + 0x73AD,0x73A6,0x73A2,0x73A0,0x73AC,0x739D,0x74DD,0x74E8,/* 0x68-0x6F */ + 0x753F,0x7540,0x753E,0x758C,0x7598,0x76AF,0x76F3,0x76F1,/* 0x70-0x77 */ + 0x76F0,0x76F5,0x77F8,0x77FC,0x77F9,0x77FB,0x77FA,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x77F7,0x7942,0x793F,0x79C5,0x7A78,0x7A7B,0x7AFB,/* 0xA0-0xA7 */ + 0x7C75,0x7CFD,0x8035,0x808F,0x80AE,0x80A3,0x80B8,0x80B5,/* 0xA8-0xAF */ + 0x80AD,0x8220,0x82A0,0x82C0,0x82AB,0x829A,0x8298,0x829B,/* 0xB0-0xB7 */ + 0x82B5,0x82A7,0x82AE,0x82BC,0x829E,0x82BA,0x82B4,0x82A8,/* 0xB8-0xBF */ + 0x82A1,0x82A9,0x82C2,0x82A4,0x82C3,0x82B6,0x82A2,0x8670,/* 0xC0-0xC7 */ + 0x866F,0x866D,0x866E,0x8C56,0x8FD2,0x8FCB,0x8FD3,0x8FCD,/* 0xC8-0xCF */ + 0x8FD6,0x8FD5,0x8FD7,0x90B2,0x90B4,0x90AF,0x90B3,0x90B0,/* 0xD0-0xD7 */ + 0x9639,0x963D,0x963C,0x963A,0x9643,0x4FCD,0x4FC5,0x4FD3,/* 0xD8-0xDF */ + 0x4FB2,0x4FC9,0x4FCB,0x4FC1,0x4FD4,0x4FDC,0x4FD9,0x4FBB,/* 0xE0-0xE7 */ + 0x4FB3,0x4FDB,0x4FC7,0x4FD6,0x4FBA,0x4FC0,0x4FB9,0x4FEC,/* 0xE8-0xEF */ + 0x5244,0x5249,0x52C0,0x52C2,0x533D,0x537C,0x5397,0x5396,/* 0xF0-0xF7 */ + 0x5399,0x5398,0x54BA,0x54A1,0x54AD,0x54A5,0x54CF,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_CE[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x54C3,0x830D,0x54B7,0x54AE,0x54D6,0x54B6,0x54C5,0x54C6,/* 0x40-0x47 */ + 0x54A0,0x5470,0x54BC,0x54A2,0x54BE,0x5472,0x54DE,0x54B0,/* 0x48-0x4F */ + 0x57B5,0x579E,0x579F,0x57A4,0x578C,0x5797,0x579D,0x579B,/* 0x50-0x57 */ + 0x5794,0x5798,0x578F,0x5799,0x57A5,0x579A,0x5795,0x58F4,/* 0x58-0x5F */ + 0x590D,0x5953,0x59E1,0x59DE,0x59EE,0x5A00,0x59F1,0x59DD,/* 0x60-0x67 */ + 0x59FA,0x59FD,0x59FC,0x59F6,0x59E4,0x59F2,0x59F7,0x59DB,/* 0x68-0x6F */ + 0x59E9,0x59F3,0x59F5,0x59E0,0x59FE,0x59F4,0x59ED,0x5BA8,/* 0x70-0x77 */ + 0x5C4C,0x5CD0,0x5CD8,0x5CCC,0x5CD7,0x5CCB,0x5CDB,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x5CDE,0x5CDA,0x5CC9,0x5CC7,0x5CCA,0x5CD6,0x5CD3,/* 0xA0-0xA7 */ + 0x5CD4,0x5CCF,0x5CC8,0x5CC6,0x5CCE,0x5CDF,0x5CF8,0x5DF9,/* 0xA8-0xAF */ + 0x5E21,0x5E22,0x5E23,0x5E20,0x5E24,0x5EB0,0x5EA4,0x5EA2,/* 0xB0-0xB7 */ + 0x5E9B,0x5EA3,0x5EA5,0x5F07,0x5F2E,0x5F56,0x5F86,0x6037,/* 0xB8-0xBF */ + 0x6039,0x6054,0x6072,0x605E,0x6045,0x6053,0x6047,0x6049,/* 0xC0-0xC7 */ + 0x605B,0x604C,0x6040,0x6042,0x605F,0x6024,0x6044,0x6058,/* 0xC8-0xCF */ + 0x6066,0x606E,0x6242,0x6243,0x62CF,0x630D,0x630B,0x62F5,/* 0xD0-0xD7 */ + 0x630E,0x6303,0x62EB,0x62F9,0x630F,0x630C,0x62F8,0x62F6,/* 0xD8-0xDF */ + 0x6300,0x6313,0x6314,0x62FA,0x6315,0x62FB,0x62F0,0x6541,/* 0xE0-0xE7 */ + 0x6543,0x65AA,0x65BF,0x6636,0x6621,0x6632,0x6635,0x661C,/* 0xE8-0xEF */ + 0x6626,0x6622,0x6633,0x662B,0x663A,0x661D,0x6634,0x6639,/* 0xF0-0xF7 */ + 0x662E,0x670F,0x6710,0x67C1,0x67F2,0x67C8,0x67BA,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_CF[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x67DC,0x67BB,0x67F8,0x67D8,0x67C0,0x67B7,0x67C5,0x67EB,/* 0x40-0x47 */ + 0x67E4,0x67DF,0x67B5,0x67CD,0x67B3,0x67F7,0x67F6,0x67EE,/* 0x48-0x4F */ + 0x67E3,0x67C2,0x67B9,0x67CE,0x67E7,0x67F0,0x67B2,0x67FC,/* 0x50-0x57 */ + 0x67C6,0x67ED,0x67CC,0x67AE,0x67E6,0x67DB,0x67FA,0x67C9,/* 0x58-0x5F */ + 0x67CA,0x67C3,0x67EA,0x67CB,0x6B28,0x6B82,0x6B84,0x6BB6,/* 0x60-0x67 */ + 0x6BD6,0x6BD8,0x6BE0,0x6C20,0x6C21,0x6D28,0x6D34,0x6D2D,/* 0x68-0x6F */ + 0x6D1F,0x6D3C,0x6D3F,0x6D12,0x6D0A,0x6CDA,0x6D33,0x6D04,/* 0x70-0x77 */ + 0x6D19,0x6D3A,0x6D1A,0x6D11,0x6D00,0x6D1D,0x6D42,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x6D01,0x6D18,0x6D37,0x6D03,0x6D0F,0x6D40,0x6D07,/* 0xA0-0xA7 */ + 0x6D20,0x6D2C,0x6D08,0x6D22,0x6D09,0x6D10,0x70B7,0x709F,/* 0xA8-0xAF */ + 0x70BE,0x70B1,0x70B0,0x70A1,0x70B4,0x70B5,0x70A9,0x7241,/* 0xB0-0xB7 */ + 0x7249,0x724A,0x726C,0x7270,0x7273,0x726E,0x72CA,0x72E4,/* 0xB8-0xBF */ + 0x72E8,0x72EB,0x72DF,0x72EA,0x72E6,0x72E3,0x7385,0x73CC,/* 0xC0-0xC7 */ + 0x73C2,0x73C8,0x73C5,0x73B9,0x73B6,0x73B5,0x73B4,0x73EB,/* 0xC8-0xCF */ + 0x73BF,0x73C7,0x73BE,0x73C3,0x73C6,0x73B8,0x73CB,0x74EC,/* 0xD0-0xD7 */ + 0x74EE,0x752E,0x7547,0x7548,0x75A7,0x75AA,0x7679,0x76C4,/* 0xD8-0xDF */ + 0x7708,0x7703,0x7704,0x7705,0x770A,0x76F7,0x76FB,0x76FA,/* 0xE0-0xE7 */ + 0x77E7,0x77E8,0x7806,0x7811,0x7812,0x7805,0x7810,0x780F,/* 0xE8-0xEF */ + 0x780E,0x7809,0x7803,0x7813,0x794A,0x794C,0x794B,0x7945,/* 0xF0-0xF7 */ + 0x7944,0x79D5,0x79CD,0x79CF,0x79D6,0x79CE,0x7A80,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D0[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7A7E,0x7AD1,0x7B00,0x7B01,0x7C7A,0x7C78,0x7C79,0x7C7F,/* 0x40-0x47 */ + 0x7C80,0x7C81,0x7D03,0x7D08,0x7D01,0x7F58,0x7F91,0x7F8D,/* 0x48-0x4F */ + 0x7FBE,0x8007,0x800E,0x800F,0x8014,0x8037,0x80D8,0x80C7,/* 0x50-0x57 */ + 0x80E0,0x80D1,0x80C8,0x80C2,0x80D0,0x80C5,0x80E3,0x80D9,/* 0x58-0x5F */ + 0x80DC,0x80CA,0x80D5,0x80C9,0x80CF,0x80D7,0x80E6,0x80CD,/* 0x60-0x67 */ + 0x81FF,0x8221,0x8294,0x82D9,0x82FE,0x82F9,0x8307,0x82E8,/* 0x68-0x6F */ + 0x8300,0x82D5,0x833A,0x82EB,0x82D6,0x82F4,0x82EC,0x82E1,/* 0x70-0x77 */ + 0x82F2,0x82F5,0x830C,0x82FB,0x82F6,0x82F0,0x82EA,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x82E4,0x82E0,0x82FA,0x82F3,0x82ED,0x8677,0x8674,/* 0xA0-0xA7 */ + 0x867C,0x8673,0x8841,0x884E,0x8867,0x886A,0x8869,0x89D3,/* 0xA8-0xAF */ + 0x8A04,0x8A07,0x8D72,0x8FE3,0x8FE1,0x8FEE,0x8FE0,0x90F1,/* 0xB0-0xB7 */ + 0x90BD,0x90BF,0x90D5,0x90C5,0x90BE,0x90C7,0x90CB,0x90C8,/* 0xB8-0xBF */ + 0x91D4,0x91D3,0x9654,0x964F,0x9651,0x9653,0x964A,0x964E,/* 0xC0-0xC7 */ + 0x501E,0x5005,0x5007,0x5013,0x5022,0x5030,0x501B,0x4FF5,/* 0xC8-0xCF */ + 0x4FF4,0x5033,0x5037,0x502C,0x4FF6,0x4FF7,0x5017,0x501C,/* 0xD0-0xD7 */ + 0x5020,0x5027,0x5035,0x502F,0x5031,0x500E,0x515A,0x5194,/* 0xD8-0xDF */ + 0x5193,0x51CA,0x51C4,0x51C5,0x51C8,0x51CE,0x5261,0x525A,/* 0xE0-0xE7 */ + 0x5252,0x525E,0x525F,0x5255,0x5262,0x52CD,0x530E,0x539E,/* 0xE8-0xEF */ + 0x5526,0x54E2,0x5517,0x5512,0x54E7,0x54F3,0x54E4,0x551A,/* 0xF0-0xF7 */ + 0x54FF,0x5504,0x5508,0x54EB,0x5511,0x5505,0x54F1,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D1[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x550A,0x54FB,0x54F7,0x54F8,0x54E0,0x550E,0x5503,0x550B,/* 0x40-0x47 */ + 0x5701,0x5702,0x57CC,0x5832,0x57D5,0x57D2,0x57BA,0x57C6,/* 0x48-0x4F */ + 0x57BD,0x57BC,0x57B8,0x57B6,0x57BF,0x57C7,0x57D0,0x57B9,/* 0x50-0x57 */ + 0x57C1,0x590E,0x594A,0x5A19,0x5A16,0x5A2D,0x5A2E,0x5A15,/* 0x58-0x5F */ + 0x5A0F,0x5A17,0x5A0A,0x5A1E,0x5A33,0x5B6C,0x5BA7,0x5BAD,/* 0x60-0x67 */ + 0x5BAC,0x5C03,0x5C56,0x5C54,0x5CEC,0x5CFF,0x5CEE,0x5CF1,/* 0x68-0x6F */ + 0x5CF7,0x5D00,0x5CF9,0x5E29,0x5E28,0x5EA8,0x5EAE,0x5EAA,/* 0x70-0x77 */ + 0x5EAC,0x5F33,0x5F30,0x5F67,0x605D,0x605A,0x6067,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x6041,0x60A2,0x6088,0x6080,0x6092,0x6081,0x609D,/* 0xA0-0xA7 */ + 0x6083,0x6095,0x609B,0x6097,0x6087,0x609C,0x608E,0x6219,/* 0xA8-0xAF */ + 0x6246,0x62F2,0x6310,0x6356,0x632C,0x6344,0x6345,0x6336,/* 0xB0-0xB7 */ + 0x6343,0x63E4,0x6339,0x634B,0x634A,0x633C,0x6329,0x6341,/* 0xB8-0xBF */ + 0x6334,0x6358,0x6354,0x6359,0x632D,0x6347,0x6333,0x635A,/* 0xC0-0xC7 */ + 0x6351,0x6338,0x6357,0x6340,0x6348,0x654A,0x6546,0x65C6,/* 0xC8-0xCF */ + 0x65C3,0x65C4,0x65C2,0x664A,0x665F,0x6647,0x6651,0x6712,/* 0xD0-0xD7 */ + 0x6713,0x681F,0x681A,0x6849,0x6832,0x6833,0x683B,0x684B,/* 0xD8-0xDF */ + 0x684F,0x6816,0x6831,0x681C,0x6835,0x682B,0x682D,0x682F,/* 0xE0-0xE7 */ + 0x684E,0x6844,0x6834,0x681D,0x6812,0x6814,0x6826,0x6828,/* 0xE8-0xEF */ + 0x682E,0x684D,0x683A,0x6825,0x6820,0x6B2C,0x6B2F,0x6B2D,/* 0xF0-0xF7 */ + 0x6B31,0x6B34,0x6B6D,0x8082,0x6B88,0x6BE6,0x6BE4,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D2[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6BE8,0x6BE3,0x6BE2,0x6BE7,0x6C25,0x6D7A,0x6D63,0x6D64,/* 0x40-0x47 */ + 0x6D76,0x6D0D,0x6D61,0x6D92,0x6D58,0x6D62,0x6D6D,0x6D6F,/* 0x48-0x4F */ + 0x6D91,0x6D8D,0x6DEF,0x6D7F,0x6D86,0x6D5E,0x6D67,0x6D60,/* 0x50-0x57 */ + 0x6D97,0x6D70,0x6D7C,0x6D5F,0x6D82,0x6D98,0x6D2F,0x6D68,/* 0x58-0x5F */ + 0x6D8B,0x6D7E,0x6D80,0x6D84,0x6D16,0x6D83,0x6D7B,0x6D7D,/* 0x60-0x67 */ + 0x6D75,0x6D90,0x70DC,0x70D3,0x70D1,0x70DD,0x70CB,0x7F39,/* 0x68-0x6F */ + 0x70E2,0x70D7,0x70D2,0x70DE,0x70E0,0x70D4,0x70CD,0x70C5,/* 0x70-0x77 */ + 0x70C6,0x70C7,0x70DA,0x70CE,0x70E1,0x7242,0x7278,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x7277,0x7276,0x7300,0x72FA,0x72F4,0x72FE,0x72F6,/* 0xA0-0xA7 */ + 0x72F3,0x72FB,0x7301,0x73D3,0x73D9,0x73E5,0x73D6,0x73BC,/* 0xA8-0xAF */ + 0x73E7,0x73E3,0x73E9,0x73DC,0x73D2,0x73DB,0x73D4,0x73DD,/* 0xB0-0xB7 */ + 0x73DA,0x73D7,0x73D8,0x73E8,0x74DE,0x74DF,0x74F4,0x74F5,/* 0xB8-0xBF */ + 0x7521,0x755B,0x755F,0x75B0,0x75C1,0x75BB,0x75C4,0x75C0,/* 0xC0-0xC7 */ + 0x75BF,0x75B6,0x75BA,0x768A,0x76C9,0x771D,0x771B,0x7710,/* 0xC8-0xCF */ + 0x7713,0x7712,0x7723,0x7711,0x7715,0x7719,0x771A,0x7722,/* 0xD0-0xD7 */ + 0x7727,0x7823,0x782C,0x7822,0x7835,0x782F,0x7828,0x782E,/* 0xD8-0xDF */ + 0x782B,0x7821,0x7829,0x7833,0x782A,0x7831,0x7954,0x795B,/* 0xE0-0xE7 */ + 0x794F,0x795C,0x7953,0x7952,0x7951,0x79EB,0x79EC,0x79E0,/* 0xE8-0xEF */ + 0x79EE,0x79ED,0x79EA,0x79DC,0x79DE,0x79DD,0x7A86,0x7A89,/* 0xF0-0xF7 */ + 0x7A85,0x7A8B,0x7A8C,0x7A8A,0x7A87,0x7AD8,0x7B10,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D3[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7B04,0x7B13,0x7B05,0x7B0F,0x7B08,0x7B0A,0x7B0E,0x7B09,/* 0x40-0x47 */ + 0x7B12,0x7C84,0x7C91,0x7C8A,0x7C8C,0x7C88,0x7C8D,0x7C85,/* 0x48-0x4F */ + 0x7D1E,0x7D1D,0x7D11,0x7D0E,0x7D18,0x7D16,0x7D13,0x7D1F,/* 0x50-0x57 */ + 0x7D12,0x7D0F,0x7D0C,0x7F5C,0x7F61,0x7F5E,0x7F60,0x7F5D,/* 0x58-0x5F */ + 0x7F5B,0x7F96,0x7F92,0x7FC3,0x7FC2,0x7FC0,0x8016,0x803E,/* 0x60-0x67 */ + 0x8039,0x80FA,0x80F2,0x80F9,0x80F5,0x8101,0x80FB,0x8100,/* 0x68-0x6F */ + 0x8201,0x822F,0x8225,0x8333,0x832D,0x8344,0x8319,0x8351,/* 0x70-0x77 */ + 0x8325,0x8356,0x833F,0x8341,0x8326,0x831C,0x8322,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x8342,0x834E,0x831B,0x832A,0x8308,0x833C,0x834D,/* 0xA0-0xA7 */ + 0x8316,0x8324,0x8320,0x8337,0x832F,0x8329,0x8347,0x8345,/* 0xA8-0xAF */ + 0x834C,0x8353,0x831E,0x832C,0x834B,0x8327,0x8348,0x8653,/* 0xB0-0xB7 */ + 0x8652,0x86A2,0x86A8,0x8696,0x868D,0x8691,0x869E,0x8687,/* 0xB8-0xBF */ + 0x8697,0x8686,0x868B,0x869A,0x8685,0x86A5,0x8699,0x86A1,/* 0xC0-0xC7 */ + 0x86A7,0x8695,0x8698,0x868E,0x869D,0x8690,0x8694,0x8843,/* 0xC8-0xCF */ + 0x8844,0x886D,0x8875,0x8876,0x8872,0x8880,0x8871,0x887F,/* 0xD0-0xD7 */ + 0x886F,0x8883,0x887E,0x8874,0x887C,0x8A12,0x8C47,0x8C57,/* 0xD8-0xDF */ + 0x8C7B,0x8CA4,0x8CA3,0x8D76,0x8D78,0x8DB5,0x8DB7,0x8DB6,/* 0xE0-0xE7 */ + 0x8ED1,0x8ED3,0x8FFE,0x8FF5,0x9002,0x8FFF,0x8FFB,0x9004,/* 0xE8-0xEF */ + 0x8FFC,0x8FF6,0x90D6,0x90E0,0x90D9,0x90DA,0x90E3,0x90DF,/* 0xF0-0xF7 */ + 0x90E5,0x90D8,0x90DB,0x90D7,0x90DC,0x90E4,0x9150,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D4[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x914E,0x914F,0x91D5,0x91E2,0x91DA,0x965C,0x965F,0x96BC,/* 0x40-0x47 */ + 0x98E3,0x9ADF,0x9B2F,0x4E7F,0x5070,0x506A,0x5061,0x505E,/* 0x48-0x4F */ + 0x5060,0x5053,0x504B,0x505D,0x5072,0x5048,0x504D,0x5041,/* 0x50-0x57 */ + 0x505B,0x504A,0x5062,0x5015,0x5045,0x505F,0x5069,0x506B,/* 0x58-0x5F */ + 0x5063,0x5064,0x5046,0x5040,0x506E,0x5073,0x5057,0x5051,/* 0x60-0x67 */ + 0x51D0,0x526B,0x526D,0x526C,0x526E,0x52D6,0x52D3,0x532D,/* 0x68-0x6F */ + 0x539C,0x5575,0x5576,0x553C,0x554D,0x5550,0x5534,0x552A,/* 0x70-0x77 */ + 0x5551,0x5562,0x5536,0x5535,0x5530,0x5552,0x5545,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x550C,0x5532,0x5565,0x554E,0x5539,0x5548,0x552D,/* 0xA0-0xA7 */ + 0x553B,0x5540,0x554B,0x570A,0x5707,0x57FB,0x5814,0x57E2,/* 0xA8-0xAF */ + 0x57F6,0x57DC,0x57F4,0x5800,0x57ED,0x57FD,0x5808,0x57F8,/* 0xB0-0xB7 */ + 0x580B,0x57F3,0x57CF,0x5807,0x57EE,0x57E3,0x57F2,0x57E5,/* 0xB8-0xBF */ + 0x57EC,0x57E1,0x580E,0x57FC,0x5810,0x57E7,0x5801,0x580C,/* 0xC0-0xC7 */ + 0x57F1,0x57E9,0x57F0,0x580D,0x5804,0x595C,0x5A60,0x5A58,/* 0xC8-0xCF */ + 0x5A55,0x5A67,0x5A5E,0x5A38,0x5A35,0x5A6D,0x5A50,0x5A5F,/* 0xD0-0xD7 */ + 0x5A65,0x5A6C,0x5A53,0x5A64,0x5A57,0x5A43,0x5A5D,0x5A52,/* 0xD8-0xDF */ + 0x5A44,0x5A5B,0x5A48,0x5A8E,0x5A3E,0x5A4D,0x5A39,0x5A4C,/* 0xE0-0xE7 */ + 0x5A70,0x5A69,0x5A47,0x5A51,0x5A56,0x5A42,0x5A5C,0x5B72,/* 0xE8-0xEF */ + 0x5B6E,0x5BC1,0x5BC0,0x5C59,0x5D1E,0x5D0B,0x5D1D,0x5D1A,/* 0xF0-0xF7 */ + 0x5D20,0x5D0C,0x5D28,0x5D0D,0x5D26,0x5D25,0x5D0F,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D5[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x5D30,0x5D12,0x5D23,0x5D1F,0x5D2E,0x5E3E,0x5E34,0x5EB1,/* 0x40-0x47 */ + 0x5EB4,0x5EB9,0x5EB2,0x5EB3,0x5F36,0x5F38,0x5F9B,0x5F96,/* 0x48-0x4F */ + 0x5F9F,0x608A,0x6090,0x6086,0x60BE,0x60B0,0x60BA,0x60D3,/* 0x50-0x57 */ + 0x60D4,0x60CF,0x60E4,0x60D9,0x60DD,0x60C8,0x60B1,0x60DB,/* 0x58-0x5F */ + 0x60B7,0x60CA,0x60BF,0x60C3,0x60CD,0x60C0,0x6332,0x6365,/* 0x60-0x67 */ + 0x638A,0x6382,0x637D,0x63BD,0x639E,0x63AD,0x639D,0x6397,/* 0x68-0x6F */ + 0x63AB,0x638E,0x636F,0x6387,0x6390,0x636E,0x63AF,0x6375,/* 0x70-0x77 */ + 0x639C,0x636D,0x63AE,0x637C,0x63A4,0x633B,0x639F,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x6378,0x6385,0x6381,0x6391,0x638D,0x6370,0x6553,/* 0xA0-0xA7 */ + 0x65CD,0x6665,0x6661,0x665B,0x6659,0x665C,0x6662,0x6718,/* 0xA8-0xAF */ + 0x6879,0x6887,0x6890,0x689C,0x686D,0x686E,0x68AE,0x68AB,/* 0xB0-0xB7 */ + 0x6956,0x686F,0x68A3,0x68AC,0x68A9,0x6875,0x6874,0x68B2,/* 0xB8-0xBF */ + 0x688F,0x6877,0x6892,0x687C,0x686B,0x6872,0x68AA,0x6880,/* 0xC0-0xC7 */ + 0x6871,0x687E,0x689B,0x6896,0x688B,0x68A0,0x6889,0x68A4,/* 0xC8-0xCF */ + 0x6878,0x687B,0x6891,0x688C,0x688A,0x687D,0x6B36,0x6B33,/* 0xD0-0xD7 */ + 0x6B37,0x6B38,0x6B91,0x6B8F,0x6B8D,0x6B8E,0x6B8C,0x6C2A,/* 0xD8-0xDF */ + 0x6DC0,0x6DAB,0x6DB4,0x6DB3,0x6E74,0x6DAC,0x6DE9,0x6DE2,/* 0xE0-0xE7 */ + 0x6DB7,0x6DF6,0x6DD4,0x6E00,0x6DC8,0x6DE0,0x6DDF,0x6DD6,/* 0xE8-0xEF */ + 0x6DBE,0x6DE5,0x6DDC,0x6DDD,0x6DDB,0x6DF4,0x6DCA,0x6DBD,/* 0xF0-0xF7 */ + 0x6DED,0x6DF0,0x6DBA,0x6DD5,0x6DC2,0x6DCF,0x6DC9,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D6[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6DD0,0x6DF2,0x6DD3,0x6DFD,0x6DD7,0x6DCD,0x6DE3,0x6DBB,/* 0x40-0x47 */ + 0x70FA,0x710D,0x70F7,0x7117,0x70F4,0x710C,0x70F0,0x7104,/* 0x48-0x4F */ + 0x70F3,0x7110,0x70FC,0x70FF,0x7106,0x7113,0x7100,0x70F8,/* 0x50-0x57 */ + 0x70F6,0x710B,0x7102,0x710E,0x727E,0x727B,0x727C,0x727F,/* 0x58-0x5F */ + 0x731D,0x7317,0x7307,0x7311,0x7318,0x730A,0x7308,0x72FF,/* 0x60-0x67 */ + 0x730F,0x731E,0x7388,0x73F6,0x73F8,0x73F5,0x7404,0x7401,/* 0x68-0x6F */ + 0x73FD,0x7407,0x7400,0x73FA,0x73FC,0x73FF,0x740C,0x740B,/* 0x70-0x77 */ + 0x73F4,0x7408,0x7564,0x7563,0x75CE,0x75D2,0x75CF,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x75CB,0x75CC,0x75D1,0x75D0,0x768F,0x7689,0x76D3,/* 0xA0-0xA7 */ + 0x7739,0x772F,0x772D,0x7731,0x7732,0x7734,0x7733,0x773D,/* 0xA8-0xAF */ + 0x7725,0x773B,0x7735,0x7848,0x7852,0x7849,0x784D,0x784A,/* 0xB0-0xB7 */ + 0x784C,0x7826,0x7845,0x7850,0x7964,0x7967,0x7969,0x796A,/* 0xB8-0xBF */ + 0x7963,0x796B,0x7961,0x79BB,0x79FA,0x79F8,0x79F6,0x79F7,/* 0xC0-0xC7 */ + 0x7A8F,0x7A94,0x7A90,0x7B35,0x7B47,0x7B34,0x7B25,0x7B30,/* 0xC8-0xCF */ + 0x7B22,0x7B24,0x7B33,0x7B18,0x7B2A,0x7B1D,0x7B31,0x7B2B,/* 0xD0-0xD7 */ + 0x7B2D,0x7B2F,0x7B32,0x7B38,0x7B1A,0x7B23,0x7C94,0x7C98,/* 0xD8-0xDF */ + 0x7C96,0x7CA3,0x7D35,0x7D3D,0x7D38,0x7D36,0x7D3A,0x7D45,/* 0xE0-0xE7 */ + 0x7D2C,0x7D29,0x7D41,0x7D47,0x7D3E,0x7D3F,0x7D4A,0x7D3B,/* 0xE8-0xEF */ + 0x7D28,0x7F63,0x7F95,0x7F9C,0x7F9D,0x7F9B,0x7FCA,0x7FCB,/* 0xF0-0xF7 */ + 0x7FCD,0x7FD0,0x7FD1,0x7FC7,0x7FCF,0x7FC9,0x801F,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D7[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x801E,0x801B,0x8047,0x8043,0x8048,0x8118,0x8125,0x8119,/* 0x40-0x47 */ + 0x811B,0x812D,0x811F,0x812C,0x811E,0x8121,0x8115,0x8127,/* 0x48-0x4F */ + 0x811D,0x8122,0x8211,0x8238,0x8233,0x823A,0x8234,0x8232,/* 0x50-0x57 */ + 0x8274,0x8390,0x83A3,0x83A8,0x838D,0x837A,0x8373,0x83A4,/* 0x58-0x5F */ + 0x8374,0x838F,0x8381,0x8395,0x8399,0x8375,0x8394,0x83A9,/* 0x60-0x67 */ + 0x837D,0x8383,0x838C,0x839D,0x839B,0x83AA,0x838B,0x837E,/* 0x68-0x6F */ + 0x83A5,0x83AF,0x8388,0x8397,0x83B0,0x837F,0x83A6,0x8387,/* 0x70-0x77 */ + 0x83AE,0x8376,0x839A,0x8659,0x8656,0x86BF,0x86B7,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x86C2,0x86C1,0x86C5,0x86BA,0x86B0,0x86C8,0x86B9,/* 0xA0-0xA7 */ + 0x86B3,0x86B8,0x86CC,0x86B4,0x86BB,0x86BC,0x86C3,0x86BD,/* 0xA8-0xAF */ + 0x86BE,0x8852,0x8889,0x8895,0x88A8,0x88A2,0x88AA,0x889A,/* 0xB0-0xB7 */ + 0x8891,0x88A1,0x889F,0x8898,0x88A7,0x8899,0x889B,0x8897,/* 0xB8-0xBF */ + 0x88A4,0x88AC,0x888C,0x8893,0x888E,0x8982,0x89D6,0x89D9,/* 0xC0-0xC7 */ + 0x89D5,0x8A30,0x8A27,0x8A2C,0x8A1E,0x8C39,0x8C3B,0x8C5C,/* 0xC8-0xCF */ + 0x8C5D,0x8C7D,0x8CA5,0x8D7D,0x8D7B,0x8D79,0x8DBC,0x8DC2,/* 0xD0-0xD7 */ + 0x8DB9,0x8DBF,0x8DC1,0x8ED8,0x8EDE,0x8EDD,0x8EDC,0x8ED7,/* 0xD8-0xDF */ + 0x8EE0,0x8EE1,0x9024,0x900B,0x9011,0x901C,0x900C,0x9021,/* 0xE0-0xE7 */ + 0x90EF,0x90EA,0x90F0,0x90F4,0x90F2,0x90F3,0x90D4,0x90EB,/* 0xE8-0xEF */ + 0x90EC,0x90E9,0x9156,0x9158,0x915A,0x9153,0x9155,0x91EC,/* 0xF0-0xF7 */ + 0x91F4,0x91F1,0x91F3,0x91F8,0x91E4,0x91F9,0x91EA,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D8[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x91EB,0x91F7,0x91E8,0x91EE,0x957A,0x9586,0x9588,0x967C,/* 0x40-0x47 */ + 0x966D,0x966B,0x9671,0x966F,0x96BF,0x976A,0x9804,0x98E5,/* 0x48-0x4F */ + 0x9997,0x509B,0x5095,0x5094,0x509E,0x508B,0x50A3,0x5083,/* 0x50-0x57 */ + 0x508C,0x508E,0x509D,0x5068,0x509C,0x5092,0x5082,0x5087,/* 0x58-0x5F */ + 0x515F,0x51D4,0x5312,0x5311,0x53A4,0x53A7,0x5591,0x55A8,/* 0x60-0x67 */ + 0x55A5,0x55AD,0x5577,0x5645,0x55A2,0x5593,0x5588,0x558F,/* 0x68-0x6F */ + 0x55B5,0x5581,0x55A3,0x5592,0x55A4,0x557D,0x558C,0x55A6,/* 0x70-0x77 */ + 0x557F,0x5595,0x55A1,0x558E,0x570C,0x5829,0x5837,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x5819,0x581E,0x5827,0x5823,0x5828,0x57F5,0x5848,/* 0xA0-0xA7 */ + 0x5825,0x581C,0x581B,0x5833,0x583F,0x5836,0x582E,0x5839,/* 0xA8-0xAF */ + 0x5838,0x582D,0x582C,0x583B,0x5961,0x5AAF,0x5A94,0x5A9F,/* 0xB0-0xB7 */ + 0x5A7A,0x5AA2,0x5A9E,0x5A78,0x5AA6,0x5A7C,0x5AA5,0x5AAC,/* 0xB8-0xBF */ + 0x5A95,0x5AAE,0x5A37,0x5A84,0x5A8A,0x5A97,0x5A83,0x5A8B,/* 0xC0-0xC7 */ + 0x5AA9,0x5A7B,0x5A7D,0x5A8C,0x5A9C,0x5A8F,0x5A93,0x5A9D,/* 0xC8-0xCF */ + 0x5BEA,0x5BCD,0x5BCB,0x5BD4,0x5BD1,0x5BCA,0x5BCE,0x5C0C,/* 0xD0-0xD7 */ + 0x5C30,0x5D37,0x5D43,0x5D6B,0x5D41,0x5D4B,0x5D3F,0x5D35,/* 0xD8-0xDF */ + 0x5D51,0x5D4E,0x5D55,0x5D33,0x5D3A,0x5D52,0x5D3D,0x5D31,/* 0xE0-0xE7 */ + 0x5D59,0x5D42,0x5D39,0x5D49,0x5D38,0x5D3C,0x5D32,0x5D36,/* 0xE8-0xEF */ + 0x5D40,0x5D45,0x5E44,0x5E41,0x5F58,0x5FA6,0x5FA5,0x5FAB,/* 0xF0-0xF7 */ + 0x60C9,0x60B9,0x60CC,0x60E2,0x60CE,0x60C4,0x6114,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_D9[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x60F2,0x610A,0x6116,0x6105,0x60F5,0x6113,0x60F8,0x60FC,/* 0x40-0x47 */ + 0x60FE,0x60C1,0x6103,0x6118,0x611D,0x6110,0x60FF,0x6104,/* 0x48-0x4F */ + 0x610B,0x624A,0x6394,0x63B1,0x63B0,0x63CE,0x63E5,0x63E8,/* 0x50-0x57 */ + 0x63EF,0x63C3,0x649D,0x63F3,0x63CA,0x63E0,0x63F6,0x63D5,/* 0x58-0x5F */ + 0x63F2,0x63F5,0x6461,0x63DF,0x63BE,0x63DD,0x63DC,0x63C4,/* 0x60-0x67 */ + 0x63D8,0x63D3,0x63C2,0x63C7,0x63CC,0x63CB,0x63C8,0x63F0,/* 0x68-0x6F */ + 0x63D7,0x63D9,0x6532,0x6567,0x656A,0x6564,0x655C,0x6568,/* 0x70-0x77 */ + 0x6565,0x658C,0x659D,0x659E,0x65AE,0x65D0,0x65D2,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x667C,0x666C,0x667B,0x6680,0x6671,0x6679,0x666A,/* 0xA0-0xA7 */ + 0x6672,0x6701,0x690C,0x68D3,0x6904,0x68DC,0x692A,0x68EC,/* 0xA8-0xAF */ + 0x68EA,0x68F1,0x690F,0x68D6,0x68F7,0x68EB,0x68E4,0x68F6,/* 0xB0-0xB7 */ + 0x6913,0x6910,0x68F3,0x68E1,0x6907,0x68CC,0x6908,0x6970,/* 0xB8-0xBF */ + 0x68B4,0x6911,0x68EF,0x68C6,0x6914,0x68F8,0x68D0,0x68FD,/* 0xC0-0xC7 */ + 0x68FC,0x68E8,0x690B,0x690A,0x6917,0x68CE,0x68C8,0x68DD,/* 0xC8-0xCF */ + 0x68DE,0x68E6,0x68F4,0x68D1,0x6906,0x68D4,0x68E9,0x6915,/* 0xD0-0xD7 */ + 0x6925,0x68C7,0x6B39,0x6B3B,0x6B3F,0x6B3C,0x6B94,0x6B97,/* 0xD8-0xDF */ + 0x6B99,0x6B95,0x6BBD,0x6BF0,0x6BF2,0x6BF3,0x6C30,0x6DFC,/* 0xE0-0xE7 */ + 0x6E46,0x6E47,0x6E1F,0x6E49,0x6E88,0x6E3C,0x6E3D,0x6E45,/* 0xE8-0xEF */ + 0x6E62,0x6E2B,0x6E3F,0x6E41,0x6E5D,0x6E73,0x6E1C,0x6E33,/* 0xF0-0xF7 */ + 0x6E4B,0x6E40,0x6E51,0x6E3B,0x6E03,0x6E2E,0x6E5E,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_DA[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6E68,0x6E5C,0x6E61,0x6E31,0x6E28,0x6E60,0x6E71,0x6E6B,/* 0x40-0x47 */ + 0x6E39,0x6E22,0x6E30,0x6E53,0x6E65,0x6E27,0x6E78,0x6E64,/* 0x48-0x4F */ + 0x6E77,0x6E55,0x6E79,0x6E52,0x6E66,0x6E35,0x6E36,0x6E5A,/* 0x50-0x57 */ + 0x7120,0x711E,0x712F,0x70FB,0x712E,0x7131,0x7123,0x7125,/* 0x58-0x5F */ + 0x7122,0x7132,0x711F,0x7128,0x713A,0x711B,0x724B,0x725A,/* 0x60-0x67 */ + 0x7288,0x7289,0x7286,0x7285,0x728B,0x7312,0x730B,0x7330,/* 0x68-0x6F */ + 0x7322,0x7331,0x7333,0x7327,0x7332,0x732D,0x7326,0x7323,/* 0x70-0x77 */ + 0x7335,0x730C,0x742E,0x742C,0x7430,0x742B,0x7416,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x741A,0x7421,0x742D,0x7431,0x7424,0x7423,0x741D,/* 0xA0-0xA7 */ + 0x7429,0x7420,0x7432,0x74FB,0x752F,0x756F,0x756C,0x75E7,/* 0xA8-0xAF */ + 0x75DA,0x75E1,0x75E6,0x75DD,0x75DF,0x75E4,0x75D7,0x7695,/* 0xB0-0xB7 */ + 0x7692,0x76DA,0x7746,0x7747,0x7744,0x774D,0x7745,0x774A,/* 0xB8-0xBF */ + 0x774E,0x774B,0x774C,0x77DE,0x77EC,0x7860,0x7864,0x7865,/* 0xC0-0xC7 */ + 0x785C,0x786D,0x7871,0x786A,0x786E,0x7870,0x7869,0x7868,/* 0xC8-0xCF */ + 0x785E,0x7862,0x7974,0x7973,0x7972,0x7970,0x7A02,0x7A0A,/* 0xD0-0xD7 */ + 0x7A03,0x7A0C,0x7A04,0x7A99,0x7AE6,0x7AE4,0x7B4A,0x7B3B,/* 0xD8-0xDF */ + 0x7B44,0x7B48,0x7B4C,0x7B4E,0x7B40,0x7B58,0x7B45,0x7CA2,/* 0xE0-0xE7 */ + 0x7C9E,0x7CA8,0x7CA1,0x7D58,0x7D6F,0x7D63,0x7D53,0x7D56,/* 0xE8-0xEF */ + 0x7D67,0x7D6A,0x7D4F,0x7D6D,0x7D5C,0x7D6B,0x7D52,0x7D54,/* 0xF0-0xF7 */ + 0x7D69,0x7D51,0x7D5F,0x7D4E,0x7F3E,0x7F3F,0x7F65,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_DB[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7F66,0x7FA2,0x7FA0,0x7FA1,0x7FD7,0x8051,0x804F,0x8050,/* 0x40-0x47 */ + 0x80FE,0x80D4,0x8143,0x814A,0x8152,0x814F,0x8147,0x813D,/* 0x48-0x4F */ + 0x814D,0x813A,0x81E6,0x81EE,0x81F7,0x81F8,0x81F9,0x8204,/* 0x50-0x57 */ + 0x823C,0x823D,0x823F,0x8275,0x833B,0x83CF,0x83F9,0x8423,/* 0x58-0x5F */ + 0x83C0,0x83E8,0x8412,0x83E7,0x83E4,0x83FC,0x83F6,0x8410,/* 0x60-0x67 */ + 0x83C6,0x83C8,0x83EB,0x83E3,0x83BF,0x8401,0x83DD,0x83E5,/* 0x68-0x6F */ + 0x83D8,0x83FF,0x83E1,0x83CB,0x83CE,0x83D6,0x83F5,0x83C9,/* 0x70-0x77 */ + 0x8409,0x840F,0x83DE,0x8411,0x8406,0x83C2,0x83F3,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x83D5,0x83FA,0x83C7,0x83D1,0x83EA,0x8413,0x83C3,/* 0xA0-0xA7 */ + 0x83EC,0x83EE,0x83C4,0x83FB,0x83D7,0x83E2,0x841B,0x83DB,/* 0xA8-0xAF */ + 0x83FE,0x86D8,0x86E2,0x86E6,0x86D3,0x86E3,0x86DA,0x86EA,/* 0xB0-0xB7 */ + 0x86DD,0x86EB,0x86DC,0x86EC,0x86E9,0x86D7,0x86E8,0x86D1,/* 0xB8-0xBF */ + 0x8848,0x8856,0x8855,0x88BA,0x88D7,0x88B9,0x88B8,0x88C0,/* 0xC0-0xC7 */ + 0x88BE,0x88B6,0x88BC,0x88B7,0x88BD,0x88B2,0x8901,0x88C9,/* 0xC8-0xCF */ + 0x8995,0x8998,0x8997,0x89DD,0x89DA,0x89DB,0x8A4E,0x8A4D,/* 0xD0-0xD7 */ + 0x8A39,0x8A59,0x8A40,0x8A57,0x8A58,0x8A44,0x8A45,0x8A52,/* 0xD8-0xDF */ + 0x8A48,0x8A51,0x8A4A,0x8A4C,0x8A4F,0x8C5F,0x8C81,0x8C80,/* 0xE0-0xE7 */ + 0x8CBA,0x8CBE,0x8CB0,0x8CB9,0x8CB5,0x8D84,0x8D80,0x8D89,/* 0xE8-0xEF */ + 0x8DD8,0x8DD3,0x8DCD,0x8DC7,0x8DD6,0x8DDC,0x8DCF,0x8DD5,/* 0xF0-0xF7 */ + 0x8DD9,0x8DC8,0x8DD7,0x8DC5,0x8EEF,0x8EF7,0x8EFA,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_DC[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8EF9,0x8EE6,0x8EEE,0x8EE5,0x8EF5,0x8EE7,0x8EE8,0x8EF6,/* 0x40-0x47 */ + 0x8EEB,0x8EF1,0x8EEC,0x8EF4,0x8EE9,0x902D,0x9034,0x902F,/* 0x48-0x4F */ + 0x9106,0x912C,0x9104,0x90FF,0x90FC,0x9108,0x90F9,0x90FB,/* 0x50-0x57 */ + 0x9101,0x9100,0x9107,0x9105,0x9103,0x9161,0x9164,0x915F,/* 0x58-0x5F */ + 0x9162,0x9160,0x9201,0x920A,0x9225,0x9203,0x921A,0x9226,/* 0x60-0x67 */ + 0x920F,0x920C,0x9200,0x9212,0x91FF,0x91FD,0x9206,0x9204,/* 0x68-0x6F */ + 0x9227,0x9202,0x921C,0x9224,0x9219,0x9217,0x9205,0x9216,/* 0x70-0x77 */ + 0x957B,0x958D,0x958C,0x9590,0x9687,0x967E,0x9688,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x9689,0x9683,0x9680,0x96C2,0x96C8,0x96C3,0x96F1,/* 0xA0-0xA7 */ + 0x96F0,0x976C,0x9770,0x976E,0x9807,0x98A9,0x98EB,0x9CE6,/* 0xA8-0xAF */ + 0x9EF9,0x4E83,0x4E84,0x4EB6,0x50BD,0x50BF,0x50C6,0x50AE,/* 0xB0-0xB7 */ + 0x50C4,0x50CA,0x50B4,0x50C8,0x50C2,0x50B0,0x50C1,0x50BA,/* 0xB8-0xBF */ + 0x50B1,0x50CB,0x50C9,0x50B6,0x50B8,0x51D7,0x527A,0x5278,/* 0xC0-0xC7 */ + 0x527B,0x527C,0x55C3,0x55DB,0x55CC,0x55D0,0x55CB,0x55CA,/* 0xC8-0xCF */ + 0x55DD,0x55C0,0x55D4,0x55C4,0x55E9,0x55BF,0x55D2,0x558D,/* 0xD0-0xD7 */ + 0x55CF,0x55D5,0x55E2,0x55D6,0x55C8,0x55F2,0x55CD,0x55D9,/* 0xD8-0xDF */ + 0x55C2,0x5714,0x5853,0x5868,0x5864,0x584F,0x584D,0x5849,/* 0xE0-0xE7 */ + 0x586F,0x5855,0x584E,0x585D,0x5859,0x5865,0x585B,0x583D,/* 0xE8-0xEF */ + 0x5863,0x5871,0x58FC,0x5AC7,0x5AC4,0x5ACB,0x5ABA,0x5AB8,/* 0xF0-0xF7 */ + 0x5AB1,0x5AB5,0x5AB0,0x5ABF,0x5AC8,0x5ABB,0x5AC6,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_DD[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x5AB7,0x5AC0,0x5ACA,0x5AB4,0x5AB6,0x5ACD,0x5AB9,0x5A90,/* 0x40-0x47 */ + 0x5BD6,0x5BD8,0x5BD9,0x5C1F,0x5C33,0x5D71,0x5D63,0x5D4A,/* 0x48-0x4F */ + 0x5D65,0x5D72,0x5D6C,0x5D5E,0x5D68,0x5D67,0x5D62,0x5DF0,/* 0x50-0x57 */ + 0x5E4F,0x5E4E,0x5E4A,0x5E4D,0x5E4B,0x5EC5,0x5ECC,0x5EC6,/* 0x58-0x5F */ + 0x5ECB,0x5EC7,0x5F40,0x5FAF,0x5FAD,0x60F7,0x6149,0x614A,/* 0x60-0x67 */ + 0x612B,0x6145,0x6136,0x6132,0x612E,0x6146,0x612F,0x614F,/* 0x68-0x6F */ + 0x6129,0x6140,0x6220,0x9168,0x6223,0x6225,0x6224,0x63C5,/* 0x70-0x77 */ + 0x63F1,0x63EB,0x6410,0x6412,0x6409,0x6420,0x6424,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x6433,0x6443,0x641F,0x6415,0x6418,0x6439,0x6437,/* 0xA0-0xA7 */ + 0x6422,0x6423,0x640C,0x6426,0x6430,0x6428,0x6441,0x6435,/* 0xA8-0xAF */ + 0x642F,0x640A,0x641A,0x6440,0x6425,0x6427,0x640B,0x63E7,/* 0xB0-0xB7 */ + 0x641B,0x642E,0x6421,0x640E,0x656F,0x6592,0x65D3,0x6686,/* 0xB8-0xBF */ + 0x668C,0x6695,0x6690,0x668B,0x668A,0x6699,0x6694,0x6678,/* 0xC0-0xC7 */ + 0x6720,0x6966,0x695F,0x6938,0x694E,0x6962,0x6971,0x693F,/* 0xC8-0xCF */ + 0x6945,0x696A,0x6939,0x6942,0x6957,0x6959,0x697A,0x6948,/* 0xD0-0xD7 */ + 0x6949,0x6935,0x696C,0x6933,0x693D,0x6965,0x68F0,0x6978,/* 0xD8-0xDF */ + 0x6934,0x6969,0x6940,0x696F,0x6944,0x6976,0x6958,0x6941,/* 0xE0-0xE7 */ + 0x6974,0x694C,0x693B,0x694B,0x6937,0x695C,0x694F,0x6951,/* 0xE8-0xEF */ + 0x6932,0x6952,0x692F,0x697B,0x693C,0x6B46,0x6B45,0x6B43,/* 0xF0-0xF7 */ + 0x6B42,0x6B48,0x6B41,0x6B9B,0xFA0D,0x6BFB,0x6BFC,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_DE[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6BF9,0x6BF7,0x6BF8,0x6E9B,0x6ED6,0x6EC8,0x6E8F,0x6EC0,/* 0x40-0x47 */ + 0x6E9F,0x6E93,0x6E94,0x6EA0,0x6EB1,0x6EB9,0x6EC6,0x6ED2,/* 0x48-0x4F */ + 0x6EBD,0x6EC1,0x6E9E,0x6EC9,0x6EB7,0x6EB0,0x6ECD,0x6EA6,/* 0x50-0x57 */ + 0x6ECF,0x6EB2,0x6EBE,0x6EC3,0x6EDC,0x6ED8,0x6E99,0x6E92,/* 0x58-0x5F */ + 0x6E8E,0x6E8D,0x6EA4,0x6EA1,0x6EBF,0x6EB3,0x6ED0,0x6ECA,/* 0x60-0x67 */ + 0x6E97,0x6EAE,0x6EA3,0x7147,0x7154,0x7152,0x7163,0x7160,/* 0x68-0x6F */ + 0x7141,0x715D,0x7162,0x7172,0x7178,0x716A,0x7161,0x7142,/* 0x70-0x77 */ + 0x7158,0x7143,0x714B,0x7170,0x715F,0x7150,0x7153,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x7144,0x714D,0x715A,0x724F,0x728D,0x728C,0x7291,/* 0xA0-0xA7 */ + 0x7290,0x728E,0x733C,0x7342,0x733B,0x733A,0x7340,0x734A,/* 0xA8-0xAF */ + 0x7349,0x7444,0x744A,0x744B,0x7452,0x7451,0x7457,0x7440,/* 0xB0-0xB7 */ + 0x744F,0x7450,0x744E,0x7442,0x7446,0x744D,0x7454,0x74E1,/* 0xB8-0xBF */ + 0x74FF,0x74FE,0x74FD,0x751D,0x7579,0x7577,0x6983,0x75EF,/* 0xC0-0xC7 */ + 0x760F,0x7603,0x75F7,0x75FE,0x75FC,0x75F9,0x75F8,0x7610,/* 0xC8-0xCF */ + 0x75FB,0x75F6,0x75ED,0x75F5,0x75FD,0x7699,0x76B5,0x76DD,/* 0xD0-0xD7 */ + 0x7755,0x775F,0x7760,0x7752,0x7756,0x775A,0x7769,0x7767,/* 0xD8-0xDF */ + 0x7754,0x7759,0x776D,0x77E0,0x7887,0x789A,0x7894,0x788F,/* 0xE0-0xE7 */ + 0x7884,0x7895,0x7885,0x7886,0x78A1,0x7883,0x7879,0x7899,/* 0xE8-0xEF */ + 0x7880,0x7896,0x787B,0x797C,0x7982,0x797D,0x7979,0x7A11,/* 0xF0-0xF7 */ + 0x7A18,0x7A19,0x7A12,0x7A17,0x7A15,0x7A22,0x7A13,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_DF[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7A1B,0x7A10,0x7AA3,0x7AA2,0x7A9E,0x7AEB,0x7B66,0x7B64,/* 0x40-0x47 */ + 0x7B6D,0x7B74,0x7B69,0x7B72,0x7B65,0x7B73,0x7B71,0x7B70,/* 0x48-0x4F */ + 0x7B61,0x7B78,0x7B76,0x7B63,0x7CB2,0x7CB4,0x7CAF,0x7D88,/* 0x50-0x57 */ + 0x7D86,0x7D80,0x7D8D,0x7D7F,0x7D85,0x7D7A,0x7D8E,0x7D7B,/* 0x58-0x5F */ + 0x7D83,0x7D7C,0x7D8C,0x7D94,0x7D84,0x7D7D,0x7D92,0x7F6D,/* 0x60-0x67 */ + 0x7F6B,0x7F67,0x7F68,0x7F6C,0x7FA6,0x7FA5,0x7FA7,0x7FDB,/* 0x68-0x6F */ + 0x7FDC,0x8021,0x8164,0x8160,0x8177,0x815C,0x8169,0x815B,/* 0x70-0x77 */ + 0x8162,0x8172,0x6721,0x815E,0x8176,0x8167,0x816F,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x8144,0x8161,0x821D,0x8249,0x8244,0x8240,0x8242,/* 0xA0-0xA7 */ + 0x8245,0x84F1,0x843F,0x8456,0x8476,0x8479,0x848F,0x848D,/* 0xA8-0xAF */ + 0x8465,0x8451,0x8440,0x8486,0x8467,0x8430,0x844D,0x847D,/* 0xB0-0xB7 */ + 0x845A,0x8459,0x8474,0x8473,0x845D,0x8507,0x845E,0x8437,/* 0xB8-0xBF */ + 0x843A,0x8434,0x847A,0x8443,0x8478,0x8432,0x8445,0x8429,/* 0xC0-0xC7 */ + 0x83D9,0x844B,0x842F,0x8442,0x842D,0x845F,0x8470,0x8439,/* 0xC8-0xCF */ + 0x844E,0x844C,0x8452,0x846F,0x84C5,0x848E,0x843B,0x8447,/* 0xD0-0xD7 */ + 0x8436,0x8433,0x8468,0x847E,0x8444,0x842B,0x8460,0x8454,/* 0xD8-0xDF */ + 0x846E,0x8450,0x870B,0x8704,0x86F7,0x870C,0x86FA,0x86D6,/* 0xE0-0xE7 */ + 0x86F5,0x874D,0x86F8,0x870E,0x8709,0x8701,0x86F6,0x870D,/* 0xE8-0xEF */ + 0x8705,0x88D6,0x88CB,0x88CD,0x88CE,0x88DE,0x88DB,0x88DA,/* 0xF0-0xF7 */ + 0x88CC,0x88D0,0x8985,0x899B,0x89DF,0x89E5,0x89E4,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E0[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x89E1,0x89E0,0x89E2,0x89DC,0x89E6,0x8A76,0x8A86,0x8A7F,/* 0x40-0x47 */ + 0x8A61,0x8A3F,0x8A77,0x8A82,0x8A84,0x8A75,0x8A83,0x8A81,/* 0x48-0x4F */ + 0x8A74,0x8A7A,0x8C3C,0x8C4B,0x8C4A,0x8C65,0x8C64,0x8C66,/* 0x50-0x57 */ + 0x8C86,0x8C84,0x8C85,0x8CCC,0x8D68,0x8D69,0x8D91,0x8D8C,/* 0x58-0x5F */ + 0x8D8E,0x8D8F,0x8D8D,0x8D93,0x8D94,0x8D90,0x8D92,0x8DF0,/* 0x60-0x67 */ + 0x8DE0,0x8DEC,0x8DF1,0x8DEE,0x8DD0,0x8DE9,0x8DE3,0x8DE2,/* 0x68-0x6F */ + 0x8DE7,0x8DF2,0x8DEB,0x8DF4,0x8F06,0x8EFF,0x8F01,0x8F00,/* 0x70-0x77 */ + 0x8F05,0x8F07,0x8F08,0x8F02,0x8F0B,0x9052,0x903F,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x9044,0x9049,0x903D,0x9110,0x910D,0x910F,0x9111,/* 0xA0-0xA7 */ + 0x9116,0x9114,0x910B,0x910E,0x916E,0x916F,0x9248,0x9252,/* 0xA8-0xAF */ + 0x9230,0x923A,0x9266,0x9233,0x9265,0x925E,0x9283,0x922E,/* 0xB0-0xB7 */ + 0x924A,0x9246,0x926D,0x926C,0x924F,0x9260,0x9267,0x926F,/* 0xB8-0xBF */ + 0x9236,0x9261,0x9270,0x9231,0x9254,0x9263,0x9250,0x9272,/* 0xC0-0xC7 */ + 0x924E,0x9253,0x924C,0x9256,0x9232,0x959F,0x959C,0x959E,/* 0xC8-0xCF */ + 0x959B,0x9692,0x9693,0x9691,0x9697,0x96CE,0x96FA,0x96FD,/* 0xD0-0xD7 */ + 0x96F8,0x96F5,0x9773,0x9777,0x9778,0x9772,0x980F,0x980D,/* 0xD8-0xDF */ + 0x980E,0x98AC,0x98F6,0x98F9,0x99AF,0x99B2,0x99B0,0x99B5,/* 0xE0-0xE7 */ + 0x9AAD,0x9AAB,0x9B5B,0x9CEA,0x9CED,0x9CE7,0x9E80,0x9EFD,/* 0xE8-0xEF */ + 0x50E6,0x50D4,0x50D7,0x50E8,0x50F3,0x50DB,0x50EA,0x50DD,/* 0xF0-0xF7 */ + 0x50E4,0x50D3,0x50EC,0x50F0,0x50EF,0x50E3,0x50E0,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E1[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x51D8,0x5280,0x5281,0x52E9,0x52EB,0x5330,0x53AC,0x5627,/* 0x40-0x47 */ + 0x5615,0x560C,0x5612,0x55FC,0x560F,0x561C,0x5601,0x5613,/* 0x48-0x4F */ + 0x5602,0x55FA,0x561D,0x5604,0x55FF,0x55F9,0x5889,0x587C,/* 0x50-0x57 */ + 0x5890,0x5898,0x5886,0x5881,0x587F,0x5874,0x588B,0x587A,/* 0x58-0x5F */ + 0x5887,0x5891,0x588E,0x5876,0x5882,0x5888,0x587B,0x5894,/* 0x60-0x67 */ + 0x588F,0x58FE,0x596B,0x5ADC,0x5AEE,0x5AE5,0x5AD5,0x5AEA,/* 0x68-0x6F */ + 0x5ADA,0x5AED,0x5AEB,0x5AF3,0x5AE2,0x5AE0,0x5ADB,0x5AEC,/* 0x70-0x77 */ + 0x5ADE,0x5ADD,0x5AD9,0x5AE8,0x5ADF,0x5B77,0x5BE0,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x5BE3,0x5C63,0x5D82,0x5D80,0x5D7D,0x5D86,0x5D7A,/* 0xA0-0xA7 */ + 0x5D81,0x5D77,0x5D8A,0x5D89,0x5D88,0x5D7E,0x5D7C,0x5D8D,/* 0xA8-0xAF */ + 0x5D79,0x5D7F,0x5E58,0x5E59,0x5E53,0x5ED8,0x5ED1,0x5ED7,/* 0xB0-0xB7 */ + 0x5ECE,0x5EDC,0x5ED5,0x5ED9,0x5ED2,0x5ED4,0x5F44,0x5F43,/* 0xB8-0xBF */ + 0x5F6F,0x5FB6,0x612C,0x6128,0x6141,0x615E,0x6171,0x6173,/* 0xC0-0xC7 */ + 0x6152,0x6153,0x6172,0x616C,0x6180,0x6174,0x6154,0x617A,/* 0xC8-0xCF */ + 0x615B,0x6165,0x613B,0x616A,0x6161,0x6156,0x6229,0x6227,/* 0xD0-0xD7 */ + 0x622B,0x642B,0x644D,0x645B,0x645D,0x6474,0x6476,0x6472,/* 0xD8-0xDF */ + 0x6473,0x647D,0x6475,0x6466,0x64A6,0x644E,0x6482,0x645E,/* 0xE0-0xE7 */ + 0x645C,0x644B,0x6453,0x6460,0x6450,0x647F,0x643F,0x646C,/* 0xE8-0xEF */ + 0x646B,0x6459,0x6465,0x6477,0x6573,0x65A0,0x66A1,0x66A0,/* 0xF0-0xF7 */ + 0x669F,0x6705,0x6704,0x6722,0x69B1,0x69B6,0x69C9,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E2[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x69A0,0x69CE,0x6996,0x69B0,0x69AC,0x69BC,0x6991,0x6999,/* 0x40-0x47 */ + 0x698E,0x69A7,0x698D,0x69A9,0x69BE,0x69AF,0x69BF,0x69C4,/* 0x48-0x4F */ + 0x69BD,0x69A4,0x69D4,0x69B9,0x69CA,0x699A,0x69CF,0x69B3,/* 0x50-0x57 */ + 0x6993,0x69AA,0x69A1,0x699E,0x69D9,0x6997,0x6990,0x69C2,/* 0x58-0x5F */ + 0x69B5,0x69A5,0x69C6,0x6B4A,0x6B4D,0x6B4B,0x6B9E,0x6B9F,/* 0x60-0x67 */ + 0x6BA0,0x6BC3,0x6BC4,0x6BFE,0x6ECE,0x6EF5,0x6EF1,0x6F03,/* 0x68-0x6F */ + 0x6F25,0x6EF8,0x6F37,0x6EFB,0x6F2E,0x6F09,0x6F4E,0x6F19,/* 0x70-0x77 */ + 0x6F1A,0x6F27,0x6F18,0x6F3B,0x6F12,0x6EED,0x6F0A,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x6F36,0x6F73,0x6EF9,0x6EEE,0x6F2D,0x6F40,0x6F30,/* 0xA0-0xA7 */ + 0x6F3C,0x6F35,0x6EEB,0x6F07,0x6F0E,0x6F43,0x6F05,0x6EFD,/* 0xA8-0xAF */ + 0x6EF6,0x6F39,0x6F1C,0x6EFC,0x6F3A,0x6F1F,0x6F0D,0x6F1E,/* 0xB0-0xB7 */ + 0x6F08,0x6F21,0x7187,0x7190,0x7189,0x7180,0x7185,0x7182,/* 0xB8-0xBF */ + 0x718F,0x717B,0x7186,0x7181,0x7197,0x7244,0x7253,0x7297,/* 0xC0-0xC7 */ + 0x7295,0x7293,0x7343,0x734D,0x7351,0x734C,0x7462,0x7473,/* 0xC8-0xCF */ + 0x7471,0x7475,0x7472,0x7467,0x746E,0x7500,0x7502,0x7503,/* 0xD0-0xD7 */ + 0x757D,0x7590,0x7616,0x7608,0x760C,0x7615,0x7611,0x760A,/* 0xD8-0xDF */ + 0x7614,0x76B8,0x7781,0x777C,0x7785,0x7782,0x776E,0x7780,/* 0xE0-0xE7 */ + 0x776F,0x777E,0x7783,0x78B2,0x78AA,0x78B4,0x78AD,0x78A8,/* 0xE8-0xEF */ + 0x787E,0x78AB,0x789E,0x78A5,0x78A0,0x78AC,0x78A2,0x78A4,/* 0xF0-0xF7 */ + 0x7998,0x798A,0x798B,0x7996,0x7995,0x7994,0x7993,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E3[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7997,0x7988,0x7992,0x7990,0x7A2B,0x7A4A,0x7A30,0x7A2F,/* 0x40-0x47 */ + 0x7A28,0x7A26,0x7AA8,0x7AAB,0x7AAC,0x7AEE,0x7B88,0x7B9C,/* 0x48-0x4F */ + 0x7B8A,0x7B91,0x7B90,0x7B96,0x7B8D,0x7B8C,0x7B9B,0x7B8E,/* 0x50-0x57 */ + 0x7B85,0x7B98,0x5284,0x7B99,0x7BA4,0x7B82,0x7CBB,0x7CBF,/* 0x58-0x5F */ + 0x7CBC,0x7CBA,0x7DA7,0x7DB7,0x7DC2,0x7DA3,0x7DAA,0x7DC1,/* 0x60-0x67 */ + 0x7DC0,0x7DC5,0x7D9D,0x7DCE,0x7DC4,0x7DC6,0x7DCB,0x7DCC,/* 0x68-0x6F */ + 0x7DAF,0x7DB9,0x7D96,0x7DBC,0x7D9F,0x7DA6,0x7DAE,0x7DA9,/* 0x70-0x77 */ + 0x7DA1,0x7DC9,0x7F73,0x7FE2,0x7FE3,0x7FE5,0x7FDE,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x8024,0x805D,0x805C,0x8189,0x8186,0x8183,0x8187,/* 0xA0-0xA7 */ + 0x818D,0x818C,0x818B,0x8215,0x8497,0x84A4,0x84A1,0x849F,/* 0xA8-0xAF */ + 0x84BA,0x84CE,0x84C2,0x84AC,0x84AE,0x84AB,0x84B9,0x84B4,/* 0xB0-0xB7 */ + 0x84C1,0x84CD,0x84AA,0x849A,0x84B1,0x84D0,0x849D,0x84A7,/* 0xB8-0xBF */ + 0x84BB,0x84A2,0x8494,0x84C7,0x84CC,0x849B,0x84A9,0x84AF,/* 0xC0-0xC7 */ + 0x84A8,0x84D6,0x8498,0x84B6,0x84CF,0x84A0,0x84D7,0x84D4,/* 0xC8-0xCF */ + 0x84D2,0x84DB,0x84B0,0x8491,0x8661,0x8733,0x8723,0x8728,/* 0xD0-0xD7 */ + 0x876B,0x8740,0x872E,0x871E,0x8721,0x8719,0x871B,0x8743,/* 0xD8-0xDF */ + 0x872C,0x8741,0x873E,0x8746,0x8720,0x8732,0x872A,0x872D,/* 0xE0-0xE7 */ + 0x873C,0x8712,0x873A,0x8731,0x8735,0x8742,0x8726,0x8727,/* 0xE8-0xEF */ + 0x8738,0x8724,0x871A,0x8730,0x8711,0x88F7,0x88E7,0x88F1,/* 0xF0-0xF7 */ + 0x88F2,0x88FA,0x88FE,0x88EE,0x88FC,0x88F6,0x88FB,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E4[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x88F0,0x88EC,0x88EB,0x899D,0x89A1,0x899F,0x899E,0x89E9,/* 0x40-0x47 */ + 0x89EB,0x89E8,0x8AAB,0x8A99,0x8A8B,0x8A92,0x8A8F,0x8A96,/* 0x48-0x4F */ + 0x8C3D,0x8C68,0x8C69,0x8CD5,0x8CCF,0x8CD7,0x8D96,0x8E09,/* 0x50-0x57 */ + 0x8E02,0x8DFF,0x8E0D,0x8DFD,0x8E0A,0x8E03,0x8E07,0x8E06,/* 0x58-0x5F */ + 0x8E05,0x8DFE,0x8E00,0x8E04,0x8F10,0x8F11,0x8F0E,0x8F0D,/* 0x60-0x67 */ + 0x9123,0x911C,0x9120,0x9122,0x911F,0x911D,0x911A,0x9124,/* 0x68-0x6F */ + 0x9121,0x911B,0x917A,0x9172,0x9179,0x9173,0x92A5,0x92A4,/* 0x70-0x77 */ + 0x9276,0x929B,0x927A,0x92A0,0x9294,0x92AA,0x928D,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x92A6,0x929A,0x92AB,0x9279,0x9297,0x927F,0x92A3,/* 0xA0-0xA7 */ + 0x92EE,0x928E,0x9282,0x9295,0x92A2,0x927D,0x9288,0x92A1,/* 0xA8-0xAF */ + 0x928A,0x9286,0x928C,0x9299,0x92A7,0x927E,0x9287,0x92A9,/* 0xB0-0xB7 */ + 0x929D,0x928B,0x922D,0x969E,0x96A1,0x96FF,0x9758,0x977D,/* 0xB8-0xBF */ + 0x977A,0x977E,0x9783,0x9780,0x9782,0x977B,0x9784,0x9781,/* 0xC0-0xC7 */ + 0x977F,0x97CE,0x97CD,0x9816,0x98AD,0x98AE,0x9902,0x9900,/* 0xC8-0xCF */ + 0x9907,0x999D,0x999C,0x99C3,0x99B9,0x99BB,0x99BA,0x99C2,/* 0xD0-0xD7 */ + 0x99BD,0x99C7,0x9AB1,0x9AE3,0x9AE7,0x9B3E,0x9B3F,0x9B60,/* 0xD8-0xDF */ + 0x9B61,0x9B5F,0x9CF1,0x9CF2,0x9CF5,0x9EA7,0x50FF,0x5103,/* 0xE0-0xE7 */ + 0x5130,0x50F8,0x5106,0x5107,0x50F6,0x50FE,0x510B,0x510C,/* 0xE8-0xEF */ + 0x50FD,0x510A,0x528B,0x528C,0x52F1,0x52EF,0x5648,0x5642,/* 0xF0-0xF7 */ + 0x564C,0x5635,0x5641,0x564A,0x5649,0x5646,0x5658,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E5[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x565A,0x5640,0x5633,0x563D,0x562C,0x563E,0x5638,0x562A,/* 0x40-0x47 */ + 0x563A,0x571A,0x58AB,0x589D,0x58B1,0x58A0,0x58A3,0x58AF,/* 0x48-0x4F */ + 0x58AC,0x58A5,0x58A1,0x58FF,0x5AFF,0x5AF4,0x5AFD,0x5AF7,/* 0x50-0x57 */ + 0x5AF6,0x5B03,0x5AF8,0x5B02,0x5AF9,0x5B01,0x5B07,0x5B05,/* 0x58-0x5F */ + 0x5B0F,0x5C67,0x5D99,0x5D97,0x5D9F,0x5D92,0x5DA2,0x5D93,/* 0x60-0x67 */ + 0x5D95,0x5DA0,0x5D9C,0x5DA1,0x5D9A,0x5D9E,0x5E69,0x5E5D,/* 0x68-0x6F */ + 0x5E60,0x5E5C,0x7DF3,0x5EDB,0x5EDE,0x5EE1,0x5F49,0x5FB2,/* 0x70-0x77 */ + 0x618B,0x6183,0x6179,0x61B1,0x61B0,0x61A2,0x6189,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x619B,0x6193,0x61AF,0x61AD,0x619F,0x6192,0x61AA,/* 0xA0-0xA7 */ + 0x61A1,0x618D,0x6166,0x61B3,0x622D,0x646E,0x6470,0x6496,/* 0xA8-0xAF */ + 0x64A0,0x6485,0x6497,0x649C,0x648F,0x648B,0x648A,0x648C,/* 0xB0-0xB7 */ + 0x64A3,0x649F,0x6468,0x64B1,0x6498,0x6576,0x657A,0x6579,/* 0xB8-0xBF */ + 0x657B,0x65B2,0x65B3,0x66B5,0x66B0,0x66A9,0x66B2,0x66B7,/* 0xC0-0xC7 */ + 0x66AA,0x66AF,0x6A00,0x6A06,0x6A17,0x69E5,0x69F8,0x6A15,/* 0xC8-0xCF */ + 0x69F1,0x69E4,0x6A20,0x69FF,0x69EC,0x69E2,0x6A1B,0x6A1D,/* 0xD0-0xD7 */ + 0x69FE,0x6A27,0x69F2,0x69EE,0x6A14,0x69F7,0x69E7,0x6A40,/* 0xD8-0xDF */ + 0x6A08,0x69E6,0x69FB,0x6A0D,0x69FC,0x69EB,0x6A09,0x6A04,/* 0xE0-0xE7 */ + 0x6A18,0x6A25,0x6A0F,0x69F6,0x6A26,0x6A07,0x69F4,0x6A16,/* 0xE8-0xEF */ + 0x6B51,0x6BA5,0x6BA3,0x6BA2,0x6BA6,0x6C01,0x6C00,0x6BFF,/* 0xF0-0xF7 */ + 0x6C02,0x6F41,0x6F26,0x6F7E,0x6F87,0x6FC6,0x6F92,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E6[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6F8D,0x6F89,0x6F8C,0x6F62,0x6F4F,0x6F85,0x6F5A,0x6F96,/* 0x40-0x47 */ + 0x6F76,0x6F6C,0x6F82,0x6F55,0x6F72,0x6F52,0x6F50,0x6F57,/* 0x48-0x4F */ + 0x6F94,0x6F93,0x6F5D,0x6F00,0x6F61,0x6F6B,0x6F7D,0x6F67,/* 0x50-0x57 */ + 0x6F90,0x6F53,0x6F8B,0x6F69,0x6F7F,0x6F95,0x6F63,0x6F77,/* 0x58-0x5F */ + 0x6F6A,0x6F7B,0x71B2,0x71AF,0x719B,0x71B0,0x71A0,0x719A,/* 0x60-0x67 */ + 0x71A9,0x71B5,0x719D,0x71A5,0x719E,0x71A4,0x71A1,0x71AA,/* 0x68-0x6F */ + 0x719C,0x71A7,0x71B3,0x7298,0x729A,0x7358,0x7352,0x735E,/* 0x70-0x77 */ + 0x735F,0x7360,0x735D,0x735B,0x7361,0x735A,0x7359,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x7362,0x7487,0x7489,0x748A,0x7486,0x7481,0x747D,/* 0xA0-0xA7 */ + 0x7485,0x7488,0x747C,0x7479,0x7508,0x7507,0x757E,0x7625,/* 0xA8-0xAF */ + 0x761E,0x7619,0x761D,0x761C,0x7623,0x761A,0x7628,0x761B,/* 0xB0-0xB7 */ + 0x769C,0x769D,0x769E,0x769B,0x778D,0x778F,0x7789,0x7788,/* 0xB8-0xBF */ + 0x78CD,0x78BB,0x78CF,0x78CC,0x78D1,0x78CE,0x78D4,0x78C8,/* 0xC0-0xC7 */ + 0x78C3,0x78C4,0x78C9,0x799A,0x79A1,0x79A0,0x799C,0x79A2,/* 0xC8-0xCF */ + 0x799B,0x6B76,0x7A39,0x7AB2,0x7AB4,0x7AB3,0x7BB7,0x7BCB,/* 0xD0-0xD7 */ + 0x7BBE,0x7BAC,0x7BCE,0x7BAF,0x7BB9,0x7BCA,0x7BB5,0x7CC5,/* 0xD8-0xDF */ + 0x7CC8,0x7CCC,0x7CCB,0x7DF7,0x7DDB,0x7DEA,0x7DE7,0x7DD7,/* 0xE0-0xE7 */ + 0x7DE1,0x7E03,0x7DFA,0x7DE6,0x7DF6,0x7DF1,0x7DF0,0x7DEE,/* 0xE8-0xEF */ + 0x7DDF,0x7F76,0x7FAC,0x7FB0,0x7FAD,0x7FED,0x7FEB,0x7FEA,/* 0xF0-0xF7 */ + 0x7FEC,0x7FE6,0x7FE8,0x8064,0x8067,0x81A3,0x819F,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E7[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x819E,0x8195,0x81A2,0x8199,0x8197,0x8216,0x824F,0x8253,/* 0x40-0x47 */ + 0x8252,0x8250,0x824E,0x8251,0x8524,0x853B,0x850F,0x8500,/* 0x48-0x4F */ + 0x8529,0x850E,0x8509,0x850D,0x851F,0x850A,0x8527,0x851C,/* 0x50-0x57 */ + 0x84FB,0x852B,0x84FA,0x8508,0x850C,0x84F4,0x852A,0x84F2,/* 0x58-0x5F */ + 0x8515,0x84F7,0x84EB,0x84F3,0x84FC,0x8512,0x84EA,0x84E9,/* 0x60-0x67 */ + 0x8516,0x84FE,0x8528,0x851D,0x852E,0x8502,0x84FD,0x851E,/* 0x68-0x6F */ + 0x84F6,0x8531,0x8526,0x84E7,0x84E8,0x84F0,0x84EF,0x84F9,/* 0x70-0x77 */ + 0x8518,0x8520,0x8530,0x850B,0x8519,0x852F,0x8662,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x8756,0x8763,0x8764,0x8777,0x87E1,0x8773,0x8758,/* 0xA0-0xA7 */ + 0x8754,0x875B,0x8752,0x8761,0x875A,0x8751,0x875E,0x876D,/* 0xA8-0xAF */ + 0x876A,0x8750,0x874E,0x875F,0x875D,0x876F,0x876C,0x877A,/* 0xB0-0xB7 */ + 0x876E,0x875C,0x8765,0x874F,0x877B,0x8775,0x8762,0x8767,/* 0xB8-0xBF */ + 0x8769,0x885A,0x8905,0x890C,0x8914,0x890B,0x8917,0x8918,/* 0xC0-0xC7 */ + 0x8919,0x8906,0x8916,0x8911,0x890E,0x8909,0x89A2,0x89A4,/* 0xC8-0xCF */ + 0x89A3,0x89ED,0x89F0,0x89EC,0x8ACF,0x8AC6,0x8AB8,0x8AD3,/* 0xD0-0xD7 */ + 0x8AD1,0x8AD4,0x8AD5,0x8ABB,0x8AD7,0x8ABE,0x8AC0,0x8AC5,/* 0xD8-0xDF */ + 0x8AD8,0x8AC3,0x8ABA,0x8ABD,0x8AD9,0x8C3E,0x8C4D,0x8C8F,/* 0xE0-0xE7 */ + 0x8CE5,0x8CDF,0x8CD9,0x8CE8,0x8CDA,0x8CDD,0x8CE7,0x8DA0,/* 0xE8-0xEF */ + 0x8D9C,0x8DA1,0x8D9B,0x8E20,0x8E23,0x8E25,0x8E24,0x8E2E,/* 0xF0-0xF7 */ + 0x8E15,0x8E1B,0x8E16,0x8E11,0x8E19,0x8E26,0x8E27,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E8[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8E14,0x8E12,0x8E18,0x8E13,0x8E1C,0x8E17,0x8E1A,0x8F2C,/* 0x40-0x47 */ + 0x8F24,0x8F18,0x8F1A,0x8F20,0x8F23,0x8F16,0x8F17,0x9073,/* 0x48-0x4F */ + 0x9070,0x906F,0x9067,0x906B,0x912F,0x912B,0x9129,0x912A,/* 0x50-0x57 */ + 0x9132,0x9126,0x912E,0x9185,0x9186,0x918A,0x9181,0x9182,/* 0x58-0x5F */ + 0x9184,0x9180,0x92D0,0x92C3,0x92C4,0x92C0,0x92D9,0x92B6,/* 0x60-0x67 */ + 0x92CF,0x92F1,0x92DF,0x92D8,0x92E9,0x92D7,0x92DD,0x92CC,/* 0x68-0x6F */ + 0x92EF,0x92C2,0x92E8,0x92CA,0x92C8,0x92CE,0x92E6,0x92CD,/* 0x70-0x77 */ + 0x92D5,0x92C9,0x92E0,0x92DE,0x92E7,0x92D1,0x92D3,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x92B5,0x92E1,0x92C6,0x92B4,0x957C,0x95AC,0x95AB,/* 0xA0-0xA7 */ + 0x95AE,0x95B0,0x96A4,0x96A2,0x96D3,0x9705,0x9708,0x9702,/* 0xA8-0xAF */ + 0x975A,0x978A,0x978E,0x9788,0x97D0,0x97CF,0x981E,0x981D,/* 0xB0-0xB7 */ + 0x9826,0x9829,0x9828,0x9820,0x981B,0x9827,0x98B2,0x9908,/* 0xB8-0xBF */ + 0x98FA,0x9911,0x9914,0x9916,0x9917,0x9915,0x99DC,0x99CD,/* 0xC0-0xC7 */ + 0x99CF,0x99D3,0x99D4,0x99CE,0x99C9,0x99D6,0x99D8,0x99CB,/* 0xC8-0xCF */ + 0x99D7,0x99CC,0x9AB3,0x9AEC,0x9AEB,0x9AF3,0x9AF2,0x9AF1,/* 0xD0-0xD7 */ + 0x9B46,0x9B43,0x9B67,0x9B74,0x9B71,0x9B66,0x9B76,0x9B75,/* 0xD8-0xDF */ + 0x9B70,0x9B68,0x9B64,0x9B6C,0x9CFC,0x9CFA,0x9CFD,0x9CFF,/* 0xE0-0xE7 */ + 0x9CF7,0x9D07,0x9D00,0x9CF9,0x9CFB,0x9D08,0x9D05,0x9D04,/* 0xE8-0xEF */ + 0x9E83,0x9ED3,0x9F0F,0x9F10,0x511C,0x5113,0x5117,0x511A,/* 0xF0-0xF7 */ + 0x5111,0x51DE,0x5334,0x53E1,0x5670,0x5660,0x566E,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_E9[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x5673,0x5666,0x5663,0x566D,0x5672,0x565E,0x5677,0x571C,/* 0x40-0x47 */ + 0x571B,0x58C8,0x58BD,0x58C9,0x58BF,0x58BA,0x58C2,0x58BC,/* 0x48-0x4F */ + 0x58C6,0x5B17,0x5B19,0x5B1B,0x5B21,0x5B14,0x5B13,0x5B10,/* 0x50-0x57 */ + 0x5B16,0x5B28,0x5B1A,0x5B20,0x5B1E,0x5BEF,0x5DAC,0x5DB1,/* 0x58-0x5F */ + 0x5DA9,0x5DA7,0x5DB5,0x5DB0,0x5DAE,0x5DAA,0x5DA8,0x5DB2,/* 0x60-0x67 */ + 0x5DAD,0x5DAF,0x5DB4,0x5E67,0x5E68,0x5E66,0x5E6F,0x5EE9,/* 0x68-0x6F */ + 0x5EE7,0x5EE6,0x5EE8,0x5EE5,0x5F4B,0x5FBC,0x619D,0x61A8,/* 0x70-0x77 */ + 0x6196,0x61C5,0x61B4,0x61C6,0x61C1,0x61CC,0x61BA,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x61BF,0x61B8,0x618C,0x64D7,0x64D6,0x64D0,0x64CF,/* 0xA0-0xA7 */ + 0x64C9,0x64BD,0x6489,0x64C3,0x64DB,0x64F3,0x64D9,0x6533,/* 0xA8-0xAF */ + 0x657F,0x657C,0x65A2,0x66C8,0x66BE,0x66C0,0x66CA,0x66CB,/* 0xB0-0xB7 */ + 0x66CF,0x66BD,0x66BB,0x66BA,0x66CC,0x6723,0x6A34,0x6A66,/* 0xB8-0xBF */ + 0x6A49,0x6A67,0x6A32,0x6A68,0x6A3E,0x6A5D,0x6A6D,0x6A76,/* 0xC0-0xC7 */ + 0x6A5B,0x6A51,0x6A28,0x6A5A,0x6A3B,0x6A3F,0x6A41,0x6A6A,/* 0xC8-0xCF */ + 0x6A64,0x6A50,0x6A4F,0x6A54,0x6A6F,0x6A69,0x6A60,0x6A3C,/* 0xD0-0xD7 */ + 0x6A5E,0x6A56,0x6A55,0x6A4D,0x6A4E,0x6A46,0x6B55,0x6B54,/* 0xD8-0xDF */ + 0x6B56,0x6BA7,0x6BAA,0x6BAB,0x6BC8,0x6BC7,0x6C04,0x6C03,/* 0xE0-0xE7 */ + 0x6C06,0x6FAD,0x6FCB,0x6FA3,0x6FC7,0x6FBC,0x6FCE,0x6FC8,/* 0xE8-0xEF */ + 0x6F5E,0x6FC4,0x6FBD,0x6F9E,0x6FCA,0x6FA8,0x7004,0x6FA5,/* 0xF0-0xF7 */ + 0x6FAE,0x6FBA,0x6FAC,0x6FAA,0x6FCF,0x6FBF,0x6FB8,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_EA[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6FA2,0x6FC9,0x6FAB,0x6FCD,0x6FAF,0x6FB2,0x6FB0,0x71C5,/* 0x40-0x47 */ + 0x71C2,0x71BF,0x71B8,0x71D6,0x71C0,0x71C1,0x71CB,0x71D4,/* 0x48-0x4F */ + 0x71CA,0x71C7,0x71CF,0x71BD,0x71D8,0x71BC,0x71C6,0x71DA,/* 0x50-0x57 */ + 0x71DB,0x729D,0x729E,0x7369,0x7366,0x7367,0x736C,0x7365,/* 0x58-0x5F */ + 0x736B,0x736A,0x747F,0x749A,0x74A0,0x7494,0x7492,0x7495,/* 0x60-0x67 */ + 0x74A1,0x750B,0x7580,0x762F,0x762D,0x7631,0x763D,0x7633,/* 0x68-0x6F */ + 0x763C,0x7635,0x7632,0x7630,0x76BB,0x76E6,0x779A,0x779D,/* 0x70-0x77 */ + 0x77A1,0x779C,0x779B,0x77A2,0x77A3,0x7795,0x7799,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x7797,0x78DD,0x78E9,0x78E5,0x78EA,0x78DE,0x78E3,/* 0xA0-0xA7 */ + 0x78DB,0x78E1,0x78E2,0x78ED,0x78DF,0x78E0,0x79A4,0x7A44,/* 0xA8-0xAF */ + 0x7A48,0x7A47,0x7AB6,0x7AB8,0x7AB5,0x7AB1,0x7AB7,0x7BDE,/* 0xB0-0xB7 */ + 0x7BE3,0x7BE7,0x7BDD,0x7BD5,0x7BE5,0x7BDA,0x7BE8,0x7BF9,/* 0xB8-0xBF */ + 0x7BD4,0x7BEA,0x7BE2,0x7BDC,0x7BEB,0x7BD8,0x7BDF,0x7CD2,/* 0xC0-0xC7 */ + 0x7CD4,0x7CD7,0x7CD0,0x7CD1,0x7E12,0x7E21,0x7E17,0x7E0C,/* 0xC8-0xCF */ + 0x7E1F,0x7E20,0x7E13,0x7E0E,0x7E1C,0x7E15,0x7E1A,0x7E22,/* 0xD0-0xD7 */ + 0x7E0B,0x7E0F,0x7E16,0x7E0D,0x7E14,0x7E25,0x7E24,0x7F43,/* 0xD8-0xDF */ + 0x7F7B,0x7F7C,0x7F7A,0x7FB1,0x7FEF,0x802A,0x8029,0x806C,/* 0xE0-0xE7 */ + 0x81B1,0x81A6,0x81AE,0x81B9,0x81B5,0x81AB,0x81B0,0x81AC,/* 0xE8-0xEF */ + 0x81B4,0x81B2,0x81B7,0x81A7,0x81F2,0x8255,0x8256,0x8257,/* 0xF0-0xF7 */ + 0x8556,0x8545,0x856B,0x854D,0x8553,0x8561,0x8558,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_EB[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8540,0x8546,0x8564,0x8541,0x8562,0x8544,0x8551,0x8547,/* 0x40-0x47 */ + 0x8563,0x853E,0x855B,0x8571,0x854E,0x856E,0x8575,0x8555,/* 0x48-0x4F */ + 0x8567,0x8560,0x858C,0x8566,0x855D,0x8554,0x8565,0x856C,/* 0x50-0x57 */ + 0x8663,0x8665,0x8664,0x879B,0x878F,0x8797,0x8793,0x8792,/* 0x58-0x5F */ + 0x8788,0x8781,0x8796,0x8798,0x8779,0x8787,0x87A3,0x8785,/* 0x60-0x67 */ + 0x8790,0x8791,0x879D,0x8784,0x8794,0x879C,0x879A,0x8789,/* 0x68-0x6F */ + 0x891E,0x8926,0x8930,0x892D,0x892E,0x8927,0x8931,0x8922,/* 0x70-0x77 */ + 0x8929,0x8923,0x892F,0x892C,0x891F,0x89F1,0x8AE0,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x8AE2,0x8AF2,0x8AF4,0x8AF5,0x8ADD,0x8B14,0x8AE4,/* 0xA0-0xA7 */ + 0x8ADF,0x8AF0,0x8AC8,0x8ADE,0x8AE1,0x8AE8,0x8AFF,0x8AEF,/* 0xA8-0xAF */ + 0x8AFB,0x8C91,0x8C92,0x8C90,0x8CF5,0x8CEE,0x8CF1,0x8CF0,/* 0xB0-0xB7 */ + 0x8CF3,0x8D6C,0x8D6E,0x8DA5,0x8DA7,0x8E33,0x8E3E,0x8E38,/* 0xB8-0xBF */ + 0x8E40,0x8E45,0x8E36,0x8E3C,0x8E3D,0x8E41,0x8E30,0x8E3F,/* 0xC0-0xC7 */ + 0x8EBD,0x8F36,0x8F2E,0x8F35,0x8F32,0x8F39,0x8F37,0x8F34,/* 0xC8-0xCF */ + 0x9076,0x9079,0x907B,0x9086,0x90FA,0x9133,0x9135,0x9136,/* 0xD0-0xD7 */ + 0x9193,0x9190,0x9191,0x918D,0x918F,0x9327,0x931E,0x9308,/* 0xD8-0xDF */ + 0x931F,0x9306,0x930F,0x937A,0x9338,0x933C,0x931B,0x9323,/* 0xE0-0xE7 */ + 0x9312,0x9301,0x9346,0x932D,0x930E,0x930D,0x92CB,0x931D,/* 0xE8-0xEF */ + 0x92FA,0x9325,0x9313,0x92F9,0x92F7,0x9334,0x9302,0x9324,/* 0xF0-0xF7 */ + 0x92FF,0x9329,0x9339,0x9335,0x932A,0x9314,0x930C,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_EC[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x930B,0x92FE,0x9309,0x9300,0x92FB,0x9316,0x95BC,0x95CD,/* 0x40-0x47 */ + 0x95BE,0x95B9,0x95BA,0x95B6,0x95BF,0x95B5,0x95BD,0x96A9,/* 0x48-0x4F */ + 0x96D4,0x970B,0x9712,0x9710,0x9799,0x9797,0x9794,0x97F0,/* 0x50-0x57 */ + 0x97F8,0x9835,0x982F,0x9832,0x9924,0x991F,0x9927,0x9929,/* 0x58-0x5F */ + 0x999E,0x99EE,0x99EC,0x99E5,0x99E4,0x99F0,0x99E3,0x99EA,/* 0x60-0x67 */ + 0x99E9,0x99E7,0x9AB9,0x9ABF,0x9AB4,0x9ABB,0x9AF6,0x9AFA,/* 0x68-0x6F */ + 0x9AF9,0x9AF7,0x9B33,0x9B80,0x9B85,0x9B87,0x9B7C,0x9B7E,/* 0x70-0x77 */ + 0x9B7B,0x9B82,0x9B93,0x9B92,0x9B90,0x9B7A,0x9B95,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x9B7D,0x9B88,0x9D25,0x9D17,0x9D20,0x9D1E,0x9D14,/* 0xA0-0xA7 */ + 0x9D29,0x9D1D,0x9D18,0x9D22,0x9D10,0x9D19,0x9D1F,0x9E88,/* 0xA8-0xAF */ + 0x9E86,0x9E87,0x9EAE,0x9EAD,0x9ED5,0x9ED6,0x9EFA,0x9F12,/* 0xB0-0xB7 */ + 0x9F3D,0x5126,0x5125,0x5122,0x5124,0x5120,0x5129,0x52F4,/* 0xB8-0xBF */ + 0x5693,0x568C,0x568D,0x5686,0x5684,0x5683,0x567E,0x5682,/* 0xC0-0xC7 */ + 0x567F,0x5681,0x58D6,0x58D4,0x58CF,0x58D2,0x5B2D,0x5B25,/* 0xC8-0xCF */ + 0x5B32,0x5B23,0x5B2C,0x5B27,0x5B26,0x5B2F,0x5B2E,0x5B7B,/* 0xD0-0xD7 */ + 0x5BF1,0x5BF2,0x5DB7,0x5E6C,0x5E6A,0x5FBE,0x5FBB,0x61C3,/* 0xD8-0xDF */ + 0x61B5,0x61BC,0x61E7,0x61E0,0x61E5,0x61E4,0x61E8,0x61DE,/* 0xE0-0xE7 */ + 0x64EF,0x64E9,0x64E3,0x64EB,0x64E4,0x64E8,0x6581,0x6580,/* 0xE8-0xEF */ + 0x65B6,0x65DA,0x66D2,0x6A8D,0x6A96,0x6A81,0x6AA5,0x6A89,/* 0xF0-0xF7 */ + 0x6A9F,0x6A9B,0x6AA1,0x6A9E,0x6A87,0x6A93,0x6A8E,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_ED[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x6A95,0x6A83,0x6AA8,0x6AA4,0x6A91,0x6A7F,0x6AA6,0x6A9A,/* 0x40-0x47 */ + 0x6A85,0x6A8C,0x6A92,0x6B5B,0x6BAD,0x6C09,0x6FCC,0x6FA9,/* 0x48-0x4F */ + 0x6FF4,0x6FD4,0x6FE3,0x6FDC,0x6FED,0x6FE7,0x6FE6,0x6FDE,/* 0x50-0x57 */ + 0x6FF2,0x6FDD,0x6FE2,0x6FE8,0x71E1,0x71F1,0x71E8,0x71F2,/* 0x58-0x5F */ + 0x71E4,0x71F0,0x71E2,0x7373,0x736E,0x736F,0x7497,0x74B2,/* 0x60-0x67 */ + 0x74AB,0x7490,0x74AA,0x74AD,0x74B1,0x74A5,0x74AF,0x7510,/* 0x68-0x6F */ + 0x7511,0x7512,0x750F,0x7584,0x7643,0x7648,0x7649,0x7647,/* 0x70-0x77 */ + 0x76A4,0x76E9,0x77B5,0x77AB,0x77B2,0x77B7,0x77B6,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x77B4,0x77B1,0x77A8,0x77F0,0x78F3,0x78FD,0x7902,/* 0xA0-0xA7 */ + 0x78FB,0x78FC,0x78F2,0x7905,0x78F9,0x78FE,0x7904,0x79AB,/* 0xA8-0xAF */ + 0x79A8,0x7A5C,0x7A5B,0x7A56,0x7A58,0x7A54,0x7A5A,0x7ABE,/* 0xB0-0xB7 */ + 0x7AC0,0x7AC1,0x7C05,0x7C0F,0x7BF2,0x7C00,0x7BFF,0x7BFB,/* 0xB8-0xBF */ + 0x7C0E,0x7BF4,0x7C0B,0x7BF3,0x7C02,0x7C09,0x7C03,0x7C01,/* 0xC0-0xC7 */ + 0x7BF8,0x7BFD,0x7C06,0x7BF0,0x7BF1,0x7C10,0x7C0A,0x7CE8,/* 0xC8-0xCF */ + 0x7E2D,0x7E3C,0x7E42,0x7E33,0x9848,0x7E38,0x7E2A,0x7E49,/* 0xD0-0xD7 */ + 0x7E40,0x7E47,0x7E29,0x7E4C,0x7E30,0x7E3B,0x7E36,0x7E44,/* 0xD8-0xDF */ + 0x7E3A,0x7F45,0x7F7F,0x7F7E,0x7F7D,0x7FF4,0x7FF2,0x802C,/* 0xE0-0xE7 */ + 0x81BB,0x81C4,0x81CC,0x81CA,0x81C5,0x81C7,0x81BC,0x81E9,/* 0xE8-0xEF */ + 0x825B,0x825A,0x825C,0x8583,0x8580,0x858F,0x85A7,0x8595,/* 0xF0-0xF7 */ + 0x85A0,0x858B,0x85A3,0x857B,0x85A4,0x859A,0x859E,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_EE[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8577,0x857C,0x8589,0x85A1,0x857A,0x8578,0x8557,0x858E,/* 0x40-0x47 */ + 0x8596,0x8586,0x858D,0x8599,0x859D,0x8581,0x85A2,0x8582,/* 0x48-0x4F */ + 0x8588,0x8585,0x8579,0x8576,0x8598,0x8590,0x859F,0x8668,/* 0x50-0x57 */ + 0x87BE,0x87AA,0x87AD,0x87C5,0x87B0,0x87AC,0x87B9,0x87B5,/* 0x58-0x5F */ + 0x87BC,0x87AE,0x87C9,0x87C3,0x87C2,0x87CC,0x87B7,0x87AF,/* 0x60-0x67 */ + 0x87C4,0x87CA,0x87B4,0x87B6,0x87BF,0x87B8,0x87BD,0x87DE,/* 0x68-0x6F */ + 0x87B2,0x8935,0x8933,0x893C,0x893E,0x8941,0x8952,0x8937,/* 0x70-0x77 */ + 0x8942,0x89AD,0x89AF,0x89AE,0x89F2,0x89F3,0x8B1E,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x8B18,0x8B16,0x8B11,0x8B05,0x8B0B,0x8B22,0x8B0F,/* 0xA0-0xA7 */ + 0x8B12,0x8B15,0x8B07,0x8B0D,0x8B08,0x8B06,0x8B1C,0x8B13,/* 0xA8-0xAF */ + 0x8B1A,0x8C4F,0x8C70,0x8C72,0x8C71,0x8C6F,0x8C95,0x8C94,/* 0xB0-0xB7 */ + 0x8CF9,0x8D6F,0x8E4E,0x8E4D,0x8E53,0x8E50,0x8E4C,0x8E47,/* 0xB8-0xBF */ + 0x8F43,0x8F40,0x9085,0x907E,0x9138,0x919A,0x91A2,0x919B,/* 0xC0-0xC7 */ + 0x9199,0x919F,0x91A1,0x919D,0x91A0,0x93A1,0x9383,0x93AF,/* 0xC8-0xCF */ + 0x9364,0x9356,0x9347,0x937C,0x9358,0x935C,0x9376,0x9349,/* 0xD0-0xD7 */ + 0x9350,0x9351,0x9360,0x936D,0x938F,0x934C,0x936A,0x9379,/* 0xD8-0xDF */ + 0x9357,0x9355,0x9352,0x934F,0x9371,0x9377,0x937B,0x9361,/* 0xE0-0xE7 */ + 0x935E,0x9363,0x9367,0x9380,0x934E,0x9359,0x95C7,0x95C0,/* 0xE8-0xEF */ + 0x95C9,0x95C3,0x95C5,0x95B7,0x96AE,0x96B0,0x96AC,0x9720,/* 0xF0-0xF7 */ + 0x971F,0x9718,0x971D,0x9719,0x979A,0x97A1,0x979C,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_EF[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x979E,0x979D,0x97D5,0x97D4,0x97F1,0x9841,0x9844,0x984A,/* 0x40-0x47 */ + 0x9849,0x9845,0x9843,0x9925,0x992B,0x992C,0x992A,0x9933,/* 0x48-0x4F */ + 0x9932,0x992F,0x992D,0x9931,0x9930,0x9998,0x99A3,0x99A1,/* 0x50-0x57 */ + 0x9A02,0x99FA,0x99F4,0x99F7,0x99F9,0x99F8,0x99F6,0x99FB,/* 0x58-0x5F */ + 0x99FD,0x99FE,0x99FC,0x9A03,0x9ABE,0x9AFE,0x9AFD,0x9B01,/* 0x60-0x67 */ + 0x9AFC,0x9B48,0x9B9A,0x9BA8,0x9B9E,0x9B9B,0x9BA6,0x9BA1,/* 0x68-0x6F */ + 0x9BA5,0x9BA4,0x9B86,0x9BA2,0x9BA0,0x9BAF,0x9D33,0x9D41,/* 0x70-0x77 */ + 0x9D67,0x9D36,0x9D2E,0x9D2F,0x9D31,0x9D38,0x9D30,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x9D45,0x9D42,0x9D43,0x9D3E,0x9D37,0x9D40,0x9D3D,/* 0xA0-0xA7 */ + 0x7FF5,0x9D2D,0x9E8A,0x9E89,0x9E8D,0x9EB0,0x9EC8,0x9EDA,/* 0xA8-0xAF */ + 0x9EFB,0x9EFF,0x9F24,0x9F23,0x9F22,0x9F54,0x9FA0,0x5131,/* 0xB0-0xB7 */ + 0x512D,0x512E,0x5698,0x569C,0x5697,0x569A,0x569D,0x5699,/* 0xB8-0xBF */ + 0x5970,0x5B3C,0x5C69,0x5C6A,0x5DC0,0x5E6D,0x5E6E,0x61D8,/* 0xC0-0xC7 */ + 0x61DF,0x61ED,0x61EE,0x61F1,0x61EA,0x61F0,0x61EB,0x61D6,/* 0xC8-0xCF */ + 0x61E9,0x64FF,0x6504,0x64FD,0x64F8,0x6501,0x6503,0x64FC,/* 0xD0-0xD7 */ + 0x6594,0x65DB,0x66DA,0x66DB,0x66D8,0x6AC5,0x6AB9,0x6ABD,/* 0xD8-0xDF */ + 0x6AE1,0x6AC6,0x6ABA,0x6AB6,0x6AB7,0x6AC7,0x6AB4,0x6AAD,/* 0xE0-0xE7 */ + 0x6B5E,0x6BC9,0x6C0B,0x7007,0x700C,0x700D,0x7001,0x7005,/* 0xE8-0xEF */ + 0x7014,0x700E,0x6FFF,0x7000,0x6FFB,0x7026,0x6FFC,0x6FF7,/* 0xF0-0xF7 */ + 0x700A,0x7201,0x71FF,0x71F9,0x7203,0x71FD,0x7376,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F0[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x74B8,0x74C0,0x74B5,0x74C1,0x74BE,0x74B6,0x74BB,0x74C2,/* 0x40-0x47 */ + 0x7514,0x7513,0x765C,0x7664,0x7659,0x7650,0x7653,0x7657,/* 0x48-0x4F */ + 0x765A,0x76A6,0x76BD,0x76EC,0x77C2,0x77BA,0x78FF,0x790C,/* 0x50-0x57 */ + 0x7913,0x7914,0x7909,0x7910,0x7912,0x7911,0x79AD,0x79AC,/* 0x58-0x5F */ + 0x7A5F,0x7C1C,0x7C29,0x7C19,0x7C20,0x7C1F,0x7C2D,0x7C1D,/* 0x60-0x67 */ + 0x7C26,0x7C28,0x7C22,0x7C25,0x7C30,0x7E5C,0x7E50,0x7E56,/* 0x68-0x6F */ + 0x7E63,0x7E58,0x7E62,0x7E5F,0x7E51,0x7E60,0x7E57,0x7E53,/* 0x70-0x77 */ + 0x7FB5,0x7FB3,0x7FF7,0x7FF8,0x8075,0x81D1,0x81D2,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x81D0,0x825F,0x825E,0x85B4,0x85C6,0x85C0,0x85C3,/* 0xA0-0xA7 */ + 0x85C2,0x85B3,0x85B5,0x85BD,0x85C7,0x85C4,0x85BF,0x85CB,/* 0xA8-0xAF */ + 0x85CE,0x85C8,0x85C5,0x85B1,0x85B6,0x85D2,0x8624,0x85B8,/* 0xB0-0xB7 */ + 0x85B7,0x85BE,0x8669,0x87E7,0x87E6,0x87E2,0x87DB,0x87EB,/* 0xB8-0xBF */ + 0x87EA,0x87E5,0x87DF,0x87F3,0x87E4,0x87D4,0x87DC,0x87D3,/* 0xC0-0xC7 */ + 0x87ED,0x87D8,0x87E3,0x87A4,0x87D7,0x87D9,0x8801,0x87F4,/* 0xC8-0xCF */ + 0x87E8,0x87DD,0x8953,0x894B,0x894F,0x894C,0x8946,0x8950,/* 0xD0-0xD7 */ + 0x8951,0x8949,0x8B2A,0x8B27,0x8B23,0x8B33,0x8B30,0x8B35,/* 0xD8-0xDF */ + 0x8B47,0x8B2F,0x8B3C,0x8B3E,0x8B31,0x8B25,0x8B37,0x8B26,/* 0xE0-0xE7 */ + 0x8B36,0x8B2E,0x8B24,0x8B3B,0x8B3D,0x8B3A,0x8C42,0x8C75,/* 0xE8-0xEF */ + 0x8C99,0x8C98,0x8C97,0x8CFE,0x8D04,0x8D02,0x8D00,0x8E5C,/* 0xF0-0xF7 */ + 0x8E62,0x8E60,0x8E57,0x8E56,0x8E5E,0x8E65,0x8E67,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F1[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8E5B,0x8E5A,0x8E61,0x8E5D,0x8E69,0x8E54,0x8F46,0x8F47,/* 0x40-0x47 */ + 0x8F48,0x8F4B,0x9128,0x913A,0x913B,0x913E,0x91A8,0x91A5,/* 0x48-0x4F */ + 0x91A7,0x91AF,0x91AA,0x93B5,0x938C,0x9392,0x93B7,0x939B,/* 0x50-0x57 */ + 0x939D,0x9389,0x93A7,0x938E,0x93AA,0x939E,0x93A6,0x9395,/* 0x58-0x5F */ + 0x9388,0x9399,0x939F,0x938D,0x93B1,0x9391,0x93B2,0x93A4,/* 0x60-0x67 */ + 0x93A8,0x93B4,0x93A3,0x93A5,0x95D2,0x95D3,0x95D1,0x96B3,/* 0x68-0x6F */ + 0x96D7,0x96DA,0x5DC2,0x96DF,0x96D8,0x96DD,0x9723,0x9722,/* 0x70-0x77 */ + 0x9725,0x97AC,0x97AE,0x97A8,0x97AB,0x97A4,0x97AA,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x97A2,0x97A5,0x97D7,0x97D9,0x97D6,0x97D8,0x97FA,/* 0xA0-0xA7 */ + 0x9850,0x9851,0x9852,0x98B8,0x9941,0x993C,0x993A,0x9A0F,/* 0xA8-0xAF */ + 0x9A0B,0x9A09,0x9A0D,0x9A04,0x9A11,0x9A0A,0x9A05,0x9A07,/* 0xB0-0xB7 */ + 0x9A06,0x9AC0,0x9ADC,0x9B08,0x9B04,0x9B05,0x9B29,0x9B35,/* 0xB8-0xBF */ + 0x9B4A,0x9B4C,0x9B4B,0x9BC7,0x9BC6,0x9BC3,0x9BBF,0x9BC1,/* 0xC0-0xC7 */ + 0x9BB5,0x9BB8,0x9BD3,0x9BB6,0x9BC4,0x9BB9,0x9BBD,0x9D5C,/* 0xC8-0xCF */ + 0x9D53,0x9D4F,0x9D4A,0x9D5B,0x9D4B,0x9D59,0x9D56,0x9D4C,/* 0xD0-0xD7 */ + 0x9D57,0x9D52,0x9D54,0x9D5F,0x9D58,0x9D5A,0x9E8E,0x9E8C,/* 0xD8-0xDF */ + 0x9EDF,0x9F01,0x9F00,0x9F16,0x9F25,0x9F2B,0x9F2A,0x9F29,/* 0xE0-0xE7 */ + 0x9F28,0x9F4C,0x9F55,0x5134,0x5135,0x5296,0x52F7,0x53B4,/* 0xE8-0xEF */ + 0x56AB,0x56AD,0x56A6,0x56A7,0x56AA,0x56AC,0x58DA,0x58DD,/* 0xF0-0xF7 */ + 0x58DB,0x5912,0x5B3D,0x5B3E,0x5B3F,0x5DC3,0x5E70,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F2[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x5FBF,0x61FB,0x6507,0x6510,0x650D,0x6509,0x650C,0x650E,/* 0x40-0x47 */ + 0x6584,0x65DE,0x65DD,0x66DE,0x6AE7,0x6AE0,0x6ACC,0x6AD1,/* 0x48-0x4F */ + 0x6AD9,0x6ACB,0x6ADF,0x6ADC,0x6AD0,0x6AEB,0x6ACF,0x6ACD,/* 0x50-0x57 */ + 0x6ADE,0x6B60,0x6BB0,0x6C0C,0x7019,0x7027,0x7020,0x7016,/* 0x58-0x5F */ + 0x702B,0x7021,0x7022,0x7023,0x7029,0x7017,0x7024,0x701C,/* 0x60-0x67 */ + 0x702A,0x720C,0x720A,0x7207,0x7202,0x7205,0x72A5,0x72A6,/* 0x68-0x6F */ + 0x72A4,0x72A3,0x72A1,0x74CB,0x74C5,0x74B7,0x74C3,0x7516,/* 0x70-0x77 */ + 0x7660,0x77C9,0x77CA,0x77C4,0x77F1,0x791D,0x791B,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x7921,0x791C,0x7917,0x791E,0x79B0,0x7A67,0x7A68,/* 0xA0-0xA7 */ + 0x7C33,0x7C3C,0x7C39,0x7C2C,0x7C3B,0x7CEC,0x7CEA,0x7E76,/* 0xA8-0xAF */ + 0x7E75,0x7E78,0x7E70,0x7E77,0x7E6F,0x7E7A,0x7E72,0x7E74,/* 0xB0-0xB7 */ + 0x7E68,0x7F4B,0x7F4A,0x7F83,0x7F86,0x7FB7,0x7FFD,0x7FFE,/* 0xB8-0xBF */ + 0x8078,0x81D7,0x81D5,0x8264,0x8261,0x8263,0x85EB,0x85F1,/* 0xC0-0xC7 */ + 0x85ED,0x85D9,0x85E1,0x85E8,0x85DA,0x85D7,0x85EC,0x85F2,/* 0xC8-0xCF */ + 0x85F8,0x85D8,0x85DF,0x85E3,0x85DC,0x85D1,0x85F0,0x85E6,/* 0xD0-0xD7 */ + 0x85EF,0x85DE,0x85E2,0x8800,0x87FA,0x8803,0x87F6,0x87F7,/* 0xD8-0xDF */ + 0x8809,0x880C,0x880B,0x8806,0x87FC,0x8808,0x87FF,0x880A,/* 0xE0-0xE7 */ + 0x8802,0x8962,0x895A,0x895B,0x8957,0x8961,0x895C,0x8958,/* 0xE8-0xEF */ + 0x895D,0x8959,0x8988,0x89B7,0x89B6,0x89F6,0x8B50,0x8B48,/* 0xF0-0xF7 */ + 0x8B4A,0x8B40,0x8B53,0x8B56,0x8B54,0x8B4B,0x8B55,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F3[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8B51,0x8B42,0x8B52,0x8B57,0x8C43,0x8C77,0x8C76,0x8C9A,/* 0x40-0x47 */ + 0x8D06,0x8D07,0x8D09,0x8DAC,0x8DAA,0x8DAD,0x8DAB,0x8E6D,/* 0x48-0x4F */ + 0x8E78,0x8E73,0x8E6A,0x8E6F,0x8E7B,0x8EC2,0x8F52,0x8F51,/* 0x50-0x57 */ + 0x8F4F,0x8F50,0x8F53,0x8FB4,0x9140,0x913F,0x91B0,0x91AD,/* 0x58-0x5F */ + 0x93DE,0x93C7,0x93CF,0x93C2,0x93DA,0x93D0,0x93F9,0x93EC,/* 0x60-0x67 */ + 0x93CC,0x93D9,0x93A9,0x93E6,0x93CA,0x93D4,0x93EE,0x93E3,/* 0x68-0x6F */ + 0x93D5,0x93C4,0x93CE,0x93C0,0x93D2,0x93E7,0x957D,0x95DA,/* 0x70-0x77 */ + 0x95DB,0x96E1,0x9729,0x972B,0x972C,0x9728,0x9726,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x97B3,0x97B7,0x97B6,0x97DD,0x97DE,0x97DF,0x985C,/* 0xA0-0xA7 */ + 0x9859,0x985D,0x9857,0x98BF,0x98BD,0x98BB,0x98BE,0x9948,/* 0xA8-0xAF */ + 0x9947,0x9943,0x99A6,0x99A7,0x9A1A,0x9A15,0x9A25,0x9A1D,/* 0xB0-0xB7 */ + 0x9A24,0x9A1B,0x9A22,0x9A20,0x9A27,0x9A23,0x9A1E,0x9A1C,/* 0xB8-0xBF */ + 0x9A14,0x9AC2,0x9B0B,0x9B0A,0x9B0E,0x9B0C,0x9B37,0x9BEA,/* 0xC0-0xC7 */ + 0x9BEB,0x9BE0,0x9BDE,0x9BE4,0x9BE6,0x9BE2,0x9BF0,0x9BD4,/* 0xC8-0xCF */ + 0x9BD7,0x9BEC,0x9BDC,0x9BD9,0x9BE5,0x9BD5,0x9BE1,0x9BDA,/* 0xD0-0xD7 */ + 0x9D77,0x9D81,0x9D8A,0x9D84,0x9D88,0x9D71,0x9D80,0x9D78,/* 0xD8-0xDF */ + 0x9D86,0x9D8B,0x9D8C,0x9D7D,0x9D6B,0x9D74,0x9D75,0x9D70,/* 0xE0-0xE7 */ + 0x9D69,0x9D85,0x9D73,0x9D7B,0x9D82,0x9D6F,0x9D79,0x9D7F,/* 0xE8-0xEF */ + 0x9D87,0x9D68,0x9E94,0x9E91,0x9EC0,0x9EFC,0x9F2D,0x9F40,/* 0xF0-0xF7 */ + 0x9F41,0x9F4D,0x9F56,0x9F57,0x9F58,0x5337,0x56B2,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F4[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x56B5,0x56B3,0x58E3,0x5B45,0x5DC6,0x5DC7,0x5EEE,0x5EEF,/* 0x40-0x47 */ + 0x5FC0,0x5FC1,0x61F9,0x6517,0x6516,0x6515,0x6513,0x65DF,/* 0x48-0x4F */ + 0x66E8,0x66E3,0x66E4,0x6AF3,0x6AF0,0x6AEA,0x6AE8,0x6AF9,/* 0x50-0x57 */ + 0x6AF1,0x6AEE,0x6AEF,0x703C,0x7035,0x702F,0x7037,0x7034,/* 0x58-0x5F */ + 0x7031,0x7042,0x7038,0x703F,0x703A,0x7039,0x7040,0x703B,/* 0x60-0x67 */ + 0x7033,0x7041,0x7213,0x7214,0x72A8,0x737D,0x737C,0x74BA,/* 0x68-0x6F */ + 0x76AB,0x76AA,0x76BE,0x76ED,0x77CC,0x77CE,0x77CF,0x77CD,/* 0x70-0x77 */ + 0x77F2,0x7925,0x7923,0x7927,0x7928,0x7924,0x7929,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x79B2,0x7A6E,0x7A6C,0x7A6D,0x7AF7,0x7C49,0x7C48,/* 0xA0-0xA7 */ + 0x7C4A,0x7C47,0x7C45,0x7CEE,0x7E7B,0x7E7E,0x7E81,0x7E80,/* 0xA8-0xAF */ + 0x7FBA,0x7FFF,0x8079,0x81DB,0x81D9,0x820B,0x8268,0x8269,/* 0xB0-0xB7 */ + 0x8622,0x85FF,0x8601,0x85FE,0x861B,0x8600,0x85F6,0x8604,/* 0xB8-0xBF */ + 0x8609,0x8605,0x860C,0x85FD,0x8819,0x8810,0x8811,0x8817,/* 0xC0-0xC7 */ + 0x8813,0x8816,0x8963,0x8966,0x89B9,0x89F7,0x8B60,0x8B6A,/* 0xC8-0xCF */ + 0x8B5D,0x8B68,0x8B63,0x8B65,0x8B67,0x8B6D,0x8DAE,0x8E86,/* 0xD0-0xD7 */ + 0x8E88,0x8E84,0x8F59,0x8F56,0x8F57,0x8F55,0x8F58,0x8F5A,/* 0xD8-0xDF */ + 0x908D,0x9143,0x9141,0x91B7,0x91B5,0x91B2,0x91B3,0x940B,/* 0xE0-0xE7 */ + 0x9413,0x93FB,0x9420,0x940F,0x9414,0x93FE,0x9415,0x9410,/* 0xE8-0xEF */ + 0x9428,0x9419,0x940D,0x93F5,0x9400,0x93F7,0x9407,0x940E,/* 0xF0-0xF7 */ + 0x9416,0x9412,0x93FA,0x9409,0x93F8,0x940A,0x93FF,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F5[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x93FC,0x940C,0x93F6,0x9411,0x9406,0x95DE,0x95E0,0x95DF,/* 0x40-0x47 */ + 0x972E,0x972F,0x97B9,0x97BB,0x97FD,0x97FE,0x9860,0x9862,/* 0x48-0x4F */ + 0x9863,0x985F,0x98C1,0x98C2,0x9950,0x994E,0x9959,0x994C,/* 0x50-0x57 */ + 0x994B,0x9953,0x9A32,0x9A34,0x9A31,0x9A2C,0x9A2A,0x9A36,/* 0x58-0x5F */ + 0x9A29,0x9A2E,0x9A38,0x9A2D,0x9AC7,0x9ACA,0x9AC6,0x9B10,/* 0x60-0x67 */ + 0x9B12,0x9B11,0x9C0B,0x9C08,0x9BF7,0x9C05,0x9C12,0x9BF8,/* 0x68-0x6F */ + 0x9C40,0x9C07,0x9C0E,0x9C06,0x9C17,0x9C14,0x9C09,0x9D9F,/* 0x70-0x77 */ + 0x9D99,0x9DA4,0x9D9D,0x9D92,0x9D98,0x9D90,0x9D9B,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x9DA0,0x9D94,0x9D9C,0x9DAA,0x9D97,0x9DA1,0x9D9A,/* 0xA0-0xA7 */ + 0x9DA2,0x9DA8,0x9D9E,0x9DA3,0x9DBF,0x9DA9,0x9D96,0x9DA6,/* 0xA8-0xAF */ + 0x9DA7,0x9E99,0x9E9B,0x9E9A,0x9EE5,0x9EE4,0x9EE7,0x9EE6,/* 0xB0-0xB7 */ + 0x9F30,0x9F2E,0x9F5B,0x9F60,0x9F5E,0x9F5D,0x9F59,0x9F91,/* 0xB8-0xBF */ + 0x513A,0x5139,0x5298,0x5297,0x56C3,0x56BD,0x56BE,0x5B48,/* 0xC0-0xC7 */ + 0x5B47,0x5DCB,0x5DCF,0x5EF1,0x61FD,0x651B,0x6B02,0x6AFC,/* 0xC8-0xCF */ + 0x6B03,0x6AF8,0x6B00,0x7043,0x7044,0x704A,0x7048,0x7049,/* 0xD0-0xD7 */ + 0x7045,0x7046,0x721D,0x721A,0x7219,0x737E,0x7517,0x766A,/* 0xD8-0xDF */ + 0x77D0,0x792D,0x7931,0x792F,0x7C54,0x7C53,0x7CF2,0x7E8A,/* 0xE0-0xE7 */ + 0x7E87,0x7E88,0x7E8B,0x7E86,0x7E8D,0x7F4D,0x7FBB,0x8030,/* 0xE8-0xEF */ + 0x81DD,0x8618,0x862A,0x8626,0x861F,0x8623,0x861C,0x8619,/* 0xF0-0xF7 */ + 0x8627,0x862E,0x8621,0x8620,0x8629,0x861E,0x8625,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F6[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8829,0x881D,0x881B,0x8820,0x8824,0x881C,0x882B,0x884A,/* 0x40-0x47 */ + 0x896D,0x8969,0x896E,0x896B,0x89FA,0x8B79,0x8B78,0x8B45,/* 0x48-0x4F */ + 0x8B7A,0x8B7B,0x8D10,0x8D14,0x8DAF,0x8E8E,0x8E8C,0x8F5E,/* 0x50-0x57 */ + 0x8F5B,0x8F5D,0x9146,0x9144,0x9145,0x91B9,0x943F,0x943B,/* 0x58-0x5F */ + 0x9436,0x9429,0x943D,0x943C,0x9430,0x9439,0x942A,0x9437,/* 0x60-0x67 */ + 0x942C,0x9440,0x9431,0x95E5,0x95E4,0x95E3,0x9735,0x973A,/* 0x68-0x6F */ + 0x97BF,0x97E1,0x9864,0x98C9,0x98C6,0x98C0,0x9958,0x9956,/* 0x70-0x77 */ + 0x9A39,0x9A3D,0x9A46,0x9A44,0x9A42,0x9A41,0x9A3A,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x9A3F,0x9ACD,0x9B15,0x9B17,0x9B18,0x9B16,0x9B3A,/* 0xA0-0xA7 */ + 0x9B52,0x9C2B,0x9C1D,0x9C1C,0x9C2C,0x9C23,0x9C28,0x9C29,/* 0xA8-0xAF */ + 0x9C24,0x9C21,0x9DB7,0x9DB6,0x9DBC,0x9DC1,0x9DC7,0x9DCA,/* 0xB0-0xB7 */ + 0x9DCF,0x9DBE,0x9DC5,0x9DC3,0x9DBB,0x9DB5,0x9DCE,0x9DB9,/* 0xB8-0xBF */ + 0x9DBA,0x9DAC,0x9DC8,0x9DB1,0x9DAD,0x9DCC,0x9DB3,0x9DCD,/* 0xC0-0xC7 */ + 0x9DB2,0x9E7A,0x9E9C,0x9EEB,0x9EEE,0x9EED,0x9F1B,0x9F18,/* 0xC8-0xCF */ + 0x9F1A,0x9F31,0x9F4E,0x9F65,0x9F64,0x9F92,0x4EB9,0x56C6,/* 0xD0-0xD7 */ + 0x56C5,0x56CB,0x5971,0x5B4B,0x5B4C,0x5DD5,0x5DD1,0x5EF2,/* 0xD8-0xDF */ + 0x6521,0x6520,0x6526,0x6522,0x6B0B,0x6B08,0x6B09,0x6C0D,/* 0xE0-0xE7 */ + 0x7055,0x7056,0x7057,0x7052,0x721E,0x721F,0x72A9,0x737F,/* 0xE8-0xEF */ + 0x74D8,0x74D5,0x74D9,0x74D7,0x766D,0x76AD,0x7935,0x79B4,/* 0xF0-0xF7 */ + 0x7A70,0x7A71,0x7C57,0x7C5C,0x7C59,0x7C5B,0x7C5A,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F7[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7CF4,0x7CF1,0x7E91,0x7F4F,0x7F87,0x81DE,0x826B,0x8634,/* 0x40-0x47 */ + 0x8635,0x8633,0x862C,0x8632,0x8636,0x882C,0x8828,0x8826,/* 0x48-0x4F */ + 0x882A,0x8825,0x8971,0x89BF,0x89BE,0x89FB,0x8B7E,0x8B84,/* 0x50-0x57 */ + 0x8B82,0x8B86,0x8B85,0x8B7F,0x8D15,0x8E95,0x8E94,0x8E9A,/* 0x58-0x5F */ + 0x8E92,0x8E90,0x8E96,0x8E97,0x8F60,0x8F62,0x9147,0x944C,/* 0x60-0x67 */ + 0x9450,0x944A,0x944B,0x944F,0x9447,0x9445,0x9448,0x9449,/* 0x68-0x6F */ + 0x9446,0x973F,0x97E3,0x986A,0x9869,0x98CB,0x9954,0x995B,/* 0x70-0x77 */ + 0x9A4E,0x9A53,0x9A54,0x9A4C,0x9A4F,0x9A48,0x9A4A,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x9A49,0x9A52,0x9A50,0x9AD0,0x9B19,0x9B2B,0x9B3B,/* 0xA0-0xA7 */ + 0x9B56,0x9B55,0x9C46,0x9C48,0x9C3F,0x9C44,0x9C39,0x9C33,/* 0xA8-0xAF */ + 0x9C41,0x9C3C,0x9C37,0x9C34,0x9C32,0x9C3D,0x9C36,0x9DDB,/* 0xB0-0xB7 */ + 0x9DD2,0x9DDE,0x9DDA,0x9DCB,0x9DD0,0x9DDC,0x9DD1,0x9DDF,/* 0xB8-0xBF */ + 0x9DE9,0x9DD9,0x9DD8,0x9DD6,0x9DF5,0x9DD5,0x9DDD,0x9EB6,/* 0xC0-0xC7 */ + 0x9EF0,0x9F35,0x9F33,0x9F32,0x9F42,0x9F6B,0x9F95,0x9FA2,/* 0xC8-0xCF */ + 0x513D,0x5299,0x58E8,0x58E7,0x5972,0x5B4D,0x5DD8,0x882F,/* 0xD0-0xD7 */ + 0x5F4F,0x6201,0x6203,0x6204,0x6529,0x6525,0x6596,0x66EB,/* 0xD8-0xDF */ + 0x6B11,0x6B12,0x6B0F,0x6BCA,0x705B,0x705A,0x7222,0x7382,/* 0xE0-0xE7 */ + 0x7381,0x7383,0x7670,0x77D4,0x7C67,0x7C66,0x7E95,0x826C,/* 0xE8-0xEF */ + 0x863A,0x8640,0x8639,0x863C,0x8631,0x863B,0x863E,0x8830,/* 0xF0-0xF7 */ + 0x8832,0x882E,0x8833,0x8976,0x8974,0x8973,0x89FE,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F8[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x8B8C,0x8B8E,0x8B8B,0x8B88,0x8C45,0x8D19,0x8E98,0x8F64,/* 0x40-0x47 */ + 0x8F63,0x91BC,0x9462,0x9455,0x945D,0x9457,0x945E,0x97C4,/* 0x48-0x4F */ + 0x97C5,0x9800,0x9A56,0x9A59,0x9B1E,0x9B1F,0x9B20,0x9C52,/* 0x50-0x57 */ + 0x9C58,0x9C50,0x9C4A,0x9C4D,0x9C4B,0x9C55,0x9C59,0x9C4C,/* 0x58-0x5F */ + 0x9C4E,0x9DFB,0x9DF7,0x9DEF,0x9DE3,0x9DEB,0x9DF8,0x9DE4,/* 0x60-0x67 */ + 0x9DF6,0x9DE1,0x9DEE,0x9DE6,0x9DF2,0x9DF0,0x9DE2,0x9DEC,/* 0x68-0x6F */ + 0x9DF4,0x9DF3,0x9DE8,0x9DED,0x9EC2,0x9ED0,0x9EF2,0x9EF3,/* 0x70-0x77 */ + 0x9F06,0x9F1C,0x9F38,0x9F37,0x9F36,0x9F43,0x9F4F,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x9F71,0x9F70,0x9F6E,0x9F6F,0x56D3,0x56CD,0x5B4E,/* 0xA0-0xA7 */ + 0x5C6D,0x652D,0x66ED,0x66EE,0x6B13,0x705F,0x7061,0x705D,/* 0xA8-0xAF */ + 0x7060,0x7223,0x74DB,0x74E5,0x77D5,0x7938,0x79B7,0x79B6,/* 0xB0-0xB7 */ + 0x7C6A,0x7E97,0x7F89,0x826D,0x8643,0x8838,0x8837,0x8835,/* 0xB8-0xBF */ + 0x884B,0x8B94,0x8B95,0x8E9E,0x8E9F,0x8EA0,0x8E9D,0x91BE,/* 0xC0-0xC7 */ + 0x91BD,0x91C2,0x946B,0x9468,0x9469,0x96E5,0x9746,0x9743,/* 0xC8-0xCF */ + 0x9747,0x97C7,0x97E5,0x9A5E,0x9AD5,0x9B59,0x9C63,0x9C67,/* 0xD0-0xD7 */ + 0x9C66,0x9C62,0x9C5E,0x9C60,0x9E02,0x9DFE,0x9E07,0x9E03,/* 0xD8-0xDF */ + 0x9E06,0x9E05,0x9E00,0x9E01,0x9E09,0x9DFF,0x9DFD,0x9E04,/* 0xE0-0xE7 */ + 0x9EA0,0x9F1E,0x9F46,0x9F74,0x9F75,0x9F76,0x56D4,0x652E,/* 0xE8-0xEF */ + 0x65B8,0x6B18,0x6B19,0x6B17,0x6B1A,0x7062,0x7226,0x72AA,/* 0xF0-0xF7 */ + 0x77D8,0x77D9,0x7939,0x7C69,0x7C6B,0x7CF6,0x7E9A,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t c2u_F9[256] = { + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x00-0x07 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x08-0x0F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x10-0x17 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x18-0x1F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x20-0x27 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x28-0x2F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x30-0x37 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x38-0x3F */ + 0x7E98,0x7E9B,0x7E99,0x81E0,0x81E1,0x8646,0x8647,0x8648,/* 0x40-0x47 */ + 0x8979,0x897A,0x897C,0x897B,0x89FF,0x8B98,0x8B99,0x8EA5,/* 0x48-0x4F */ + 0x8EA4,0x8EA3,0x946E,0x946D,0x946F,0x9471,0x9473,0x9749,/* 0x50-0x57 */ + 0x9872,0x995F,0x9C68,0x9C6E,0x9C6D,0x9E0B,0x9E0D,0x9E10,/* 0x58-0x5F */ + 0x9E0F,0x9E12,0x9E11,0x9EA1,0x9EF5,0x9F09,0x9F47,0x9F78,/* 0x60-0x67 */ + 0x9F7B,0x9F7A,0x9F79,0x571E,0x7066,0x7C6F,0x883C,0x8DB2,/* 0x68-0x6F */ + 0x8EA6,0x91C3,0x9474,0x9478,0x9476,0x9475,0x9A60,0x9C74,/* 0x70-0x77 */ + 0x9C73,0x9C71,0x9C75,0x9E14,0x9E13,0x9EF6,0x9F0A,0x0000,/* 0x78-0x7F */ + + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x80-0x87 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x88-0x8F */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x90-0x97 */ + 0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,0x0000,/* 0x98-0x9F */ + 0x0000,0x9FA4,0x7068,0x7065,0x7CF7,0x866A,0x883E,0x883D,/* 0xA0-0xA7 */ + 0x883F,0x8B9E,0x8C9C,0x8EA9,0x8EC9,0x974B,0x9873,0x9874,/* 0xA8-0xAF */ + 0x98CC,0x9961,0x99AB,0x9A64,0x9A66,0x9A67,0x9B24,0x9E15,/* 0xB0-0xB7 */ + 0x9E17,0x9F48,0x6207,0x6B1E,0x7227,0x864C,0x8EA8,0x9482,/* 0xB8-0xBF */ + 0x9480,0x9481,0x9A69,0x9A68,0x9B2E,0x9E19,0x7229,0x864B,/* 0xC0-0xC7 */ + 0x8B9F,0x9483,0x9C79,0x9EB7,0x7675,0x9A6B,0x9C7A,0x9E1D,/* 0xC8-0xCF */ + 0x7069,0x706A,0x9EA4,0x9F7E,0x9F49,0x9F98,0x7881,0x92B9,/* 0xD0-0xD7 */ + 0x88CF,0x58BB,0x6052,0x7CA7,0x5AFA,0x2554,0x2566,0x2557,/* 0xD8-0xDF */ + 0x2560,0x256C,0x2563,0x255A,0x2569,0x255D,0x2552,0x2564,/* 0xE0-0xE7 */ + 0x2555,0x255E,0x256A,0x2561,0x2558,0x2567,0x255B,0x2553,/* 0xE8-0xEF */ + 0x2565,0x2556,0x255F,0x256B,0x2562,0x2559,0x2568,0x255C,/* 0xF0-0xF7 */ + 0x2551,0x2550,0x256D,0x256E,0x2570,0x256F,0x2593,0x0000,/* 0xF8-0xFF */ +}; + +static const wchar_t *page_charset2uni[256] = { + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, c2u_A1, c2u_A2, c2u_A3, c2u_A4, c2u_A5, c2u_A6, c2u_A7, + c2u_A8, c2u_A9, c2u_AA, c2u_AB, c2u_AC, c2u_AD, c2u_AE, c2u_AF, + c2u_B0, c2u_B1, c2u_B2, c2u_B3, c2u_B4, c2u_B5, c2u_B6, c2u_B7, + c2u_B8, c2u_B9, c2u_BA, c2u_BB, c2u_BC, c2u_BD, c2u_BE, c2u_BF, + c2u_C0, c2u_C1, c2u_C2, c2u_C3, c2u_C4, c2u_C5, c2u_C6, NULL, + NULL, c2u_C9, c2u_CA, c2u_CB, c2u_CC, c2u_CD, c2u_CE, c2u_CF, + c2u_D0, c2u_D1, c2u_D2, c2u_D3, c2u_D4, c2u_D5, c2u_D6, c2u_D7, + c2u_D8, c2u_D9, c2u_DA, c2u_DB, c2u_DC, c2u_DD, c2u_DE, c2u_DF, + c2u_E0, c2u_E1, c2u_E2, c2u_E3, c2u_E4, c2u_E5, c2u_E6, c2u_E7, + c2u_E8, c2u_E9, c2u_EA, c2u_EB, c2u_EC, c2u_ED, c2u_EE, c2u_EF, + c2u_F0, c2u_F1, c2u_F2, c2u_F3, c2u_F4, c2u_F5, c2u_F6, c2u_F7, + c2u_F8, c2u_F9, NULL, NULL, NULL, NULL, NULL, NULL, +}; + +static const unsigned char u2c_02[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA3, 0xBE, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xA3, 0xBC, 0xA3, 0xBD, 0xA3, 0xBF, /* 0xC8-0xCB */ + 0x00, 0x00, 0xA1, 0xC5, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xA3, 0xBB, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ +}; + +static const unsigned char u2c_03[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0xA1, 0xC2, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xA3, 0x44, 0xA3, 0x45, 0xA3, 0x46, /* 0x90-0x93 */ + 0xA3, 0x47, 0xA3, 0x48, 0xA3, 0x49, 0xA3, 0x4A, /* 0x94-0x97 */ + 0xA3, 0x4B, 0xA3, 0x4C, 0xA3, 0x4D, 0xA3, 0x4E, /* 0x98-0x9B */ + 0xA3, 0x4F, 0xA3, 0x50, 0xA3, 0x51, 0xA3, 0x52, /* 0x9C-0x9F */ + 0xA3, 0x53, 0xA3, 0x54, 0x00, 0x00, 0xA3, 0x55, /* 0xA0-0xA3 */ + 0xA3, 0x56, 0xA3, 0x57, 0xA3, 0x58, 0xA3, 0x59, /* 0xA4-0xA7 */ + 0xA3, 0x5A, 0xA3, 0x5B, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0xA3, 0x5C, 0xA3, 0x5D, 0xA3, 0x5E, /* 0xB0-0xB3 */ + 0xA3, 0x5F, 0xA3, 0x60, 0xA3, 0x61, 0xA3, 0x62, /* 0xB4-0xB7 */ + 0xA3, 0x63, 0xA3, 0x64, 0xA3, 0x65, 0xA3, 0x66, /* 0xB8-0xBB */ + 0xA3, 0x67, 0xA3, 0x68, 0xA3, 0x69, 0xA3, 0x6A, /* 0xBC-0xBF */ + 0xA3, 0x6B, 0xA3, 0x6C, 0x00, 0x00, 0xA3, 0x6D, /* 0xC0-0xC3 */ + 0xA3, 0x6E, 0xA3, 0x6F, 0xA3, 0x70, 0xA3, 0x71, /* 0xC4-0xC7 */ + 0xA3, 0x72, 0xA3, 0x73, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ +}; + +static const unsigned char u2c_20[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA1, 0x56, /* 0x10-0x13 */ + 0xA1, 0x58, 0xA2, 0x77, 0xA1, 0xFC, 0x00, 0x00, /* 0x14-0x17 */ + 0xA1, 0xA5, 0xA1, 0xA6, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0xA1, 0xA7, 0xA1, 0xA8, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0xA1, 0x45, 0x00, 0x00, /* 0x20-0x23 */ + 0xA3, 0xBB, 0xA1, 0x4C, 0xA1, 0x4B, 0xA1, 0x45, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xAC, 0xA1, 0xB2, /* 0x30-0x33 */ + 0x00, 0x00, 0xA1, 0xAB, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA1, 0xB0, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xC3, 0x00, 0x00, /* 0x3C-0x3F */ +}; + +static const unsigned char u2c_21[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA2, 0x4A, /* 0x00-0x03 */ + 0x00, 0x00, 0xA1, 0xC1, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xA2, 0x4B, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0xA2, 0xB9, 0xA2, 0xBA, 0xA2, 0xBB, 0xA2, 0xBC, /* 0x60-0x63 */ + 0xA2, 0xBD, 0xA2, 0xBE, 0xA2, 0xBF, 0xA2, 0xC0, /* 0x64-0x67 */ + 0xA2, 0xC1, 0xA2, 0xC2, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0xA1, 0xF6, 0xA1, 0xF4, 0xA1, 0xF7, 0xA1, 0xF5, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xF8, 0xA1, 0xF9, /* 0x94-0x97 */ + 0xA1, 0xFB, 0xA1, 0xFA, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ +}; + +static const unsigned char u2c_22[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0xA2, 0x41, 0xA2, 0x42, 0x00, 0x00, /* 0x14-0x17 */ + 0xA2, 0x58, 0x00, 0x00, 0xA1, 0xD4, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xDB, 0xA1, 0xE8, /* 0x1C-0x1F */ + 0xA1, 0xE7, 0x00, 0x00, 0x00, 0x00, 0xA1, 0xFD, /* 0x20-0x23 */ + 0x00, 0x00, 0xA1, 0xFC, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0xA1, 0xE4, 0xA1, 0xE5, 0xA1, 0xEC, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xED, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0xA1, 0xEF, 0xA1, 0xEE, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xDC, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0xA1, 0xDA, 0xA1, 0xDD, 0x00, 0x00, 0xA1, 0xDD, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xD8, 0xA1, 0xD9, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0xA1, 0xF2, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0xA1, 0xF3, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xA1, 0xE6, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA1, 0xE9, /* 0xBC-0xBF */ +}; + +static const unsigned char u2c_23[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA1, 0x5B, /* 0x04-0x07 */ +}; + +static const unsigned char u2c_25[512] = { + 0xA2, 0x77, 0x00, 0x00, 0xA2, 0x78, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0xA2, 0x7A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0xA2, 0x7B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0xA2, 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0xA2, 0x7D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0xA2, 0x75, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0xA2, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0xA2, 0x73, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0xA2, 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0xA2, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0xF9, 0xF9, 0xF9, 0xF8, 0xF9, 0xE6, 0xF9, 0xEF, /* 0x50-0x53 */ + 0xF9, 0xDD, 0xF9, 0xE8, 0xF9, 0xF1, 0xF9, 0xDF, /* 0x54-0x57 */ + 0xF9, 0xEC, 0xF9, 0xF5, 0xF9, 0xE3, 0xF9, 0xEE, /* 0x58-0x5B */ + 0xF9, 0xF7, 0xF9, 0xE5, 0xF9, 0xE9, 0xF9, 0xF2, /* 0x5C-0x5F */ + 0xF9, 0xE0, 0xF9, 0xEB, 0xF9, 0xF4, 0xF9, 0xE2, /* 0x60-0x63 */ + 0xF9, 0xE7, 0xF9, 0xF0, 0xF9, 0xDE, 0xF9, 0xED, /* 0x64-0x67 */ + 0xF9, 0xF6, 0xF9, 0xE4, 0xF9, 0xEA, 0xF9, 0xF3, /* 0x68-0x6B */ + 0xF9, 0xE1, 0xA2, 0x7E, 0xA2, 0xA1, 0xA2, 0xA3, /* 0x6C-0x6F */ + 0xA2, 0xA2, 0xA2, 0xAC, 0xA2, 0xAD, 0xA2, 0xAE, /* 0x70-0x73 */ + 0xA1, 0x5A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xA2, 0x62, 0xA2, 0x63, 0xA2, 0x64, /* 0x80-0x83 */ + 0xA2, 0x65, 0xA2, 0x66, 0xA2, 0x67, 0xA2, 0x68, /* 0x84-0x87 */ + 0xA2, 0x69, 0xA2, 0x70, 0xA2, 0x6F, 0xA2, 0x6E, /* 0x88-0x8B */ + 0xA2, 0x6D, 0xA2, 0x6C, 0xA2, 0x6B, 0xA2, 0x6A, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF9, 0xFE, /* 0x90-0x93 */ + 0xA2, 0x76, 0xA2, 0x79, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0xA1, 0xBD, 0xA1, 0xBC, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xB6, 0xA1, 0xB5, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0xA1, 0xBF, 0xA1, 0xBE, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xBB, 0xA1, 0xBA, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA1, 0xB3, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0xA1, 0xB7, 0xA1, 0xB4, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0xA2, 0xA8, 0xA2, 0xA9, /* 0xE0-0xE3 */ + 0xA2, 0xAB, 0xA2, 0xAA, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ +}; + +static const unsigned char u2c_26[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0xA1, 0xB9, 0xA1, 0xB8, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xA1, 0xF3, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0xA1, 0xF0, 0xA1, 0xF2, 0xA1, 0xF1, 0x00, 0x00, /* 0x40-0x43 */ +}; + +static const unsigned char u2c_30[512] = { + 0xA1, 0x40, 0xA1, 0x42, 0xA1, 0x43, 0xA1, 0xB2, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0xA1, 0x71, 0xA1, 0x72, 0xA1, 0x6D, 0xA1, 0x6E, /* 0x08-0x0B */ + 0xA1, 0x75, 0xA1, 0x76, 0xA1, 0x79, 0xA1, 0x7A, /* 0x0C-0x0F */ + 0xA1, 0x69, 0xA1, 0x6A, 0xA2, 0x45, 0x00, 0x00, /* 0x10-0x13 */ + 0xA1, 0x65, 0xA1, 0x66, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0xA1, 0xE3, 0xA1, 0xA9, 0xA1, 0xAA, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0xA2, 0xC3, 0xA2, 0xC4, 0xA2, 0xC5, /* 0x20-0x23 */ + 0xA2, 0xC6, 0xA2, 0xC7, 0xA2, 0xC8, 0xA2, 0xC9, /* 0x24-0x27 */ + 0xA2, 0xCA, 0xA2, 0xCB, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xA1, 0xCA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ +}; + +static const unsigned char u2c_31[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0xA3, 0x74, 0xA3, 0x75, 0xA3, 0x76, /* 0x04-0x07 */ + 0xA3, 0x77, 0xA3, 0x78, 0xA3, 0x79, 0xA3, 0x7A, /* 0x08-0x0B */ + 0xA3, 0x7B, 0xA3, 0x7C, 0xA3, 0x7D, 0xA3, 0x7E, /* 0x0C-0x0F */ + 0xA3, 0xA1, 0xA3, 0xA2, 0xA3, 0xA3, 0xA3, 0xA4, /* 0x10-0x13 */ + 0xA3, 0xA5, 0xA3, 0xA6, 0xA3, 0xA7, 0xA3, 0xA8, /* 0x14-0x17 */ + 0xA3, 0xA9, 0xA3, 0xAA, 0xA3, 0xAB, 0xA3, 0xAC, /* 0x18-0x1B */ + 0xA3, 0xAD, 0xA3, 0xAE, 0xA3, 0xAF, 0xA3, 0xB0, /* 0x1C-0x1F */ + 0xA3, 0xB1, 0xA3, 0xB2, 0xA3, 0xB3, 0xA3, 0xB4, /* 0x20-0x23 */ + 0xA3, 0xB5, 0xA3, 0xB6, 0xA3, 0xB7, 0xA3, 0xB8, /* 0x24-0x27 */ + 0xA3, 0xB9, 0xA3, 0xBA, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0xA4, 0x40, 0xA4, 0x47, /* 0x90-0x93 */ + 0xA4, 0x54, 0xA5, 0x7C, 0xA4, 0x57, 0xA4, 0xA4, /* 0x94-0x97 */ + 0xA4, 0x55, 0xA5, 0xD2, 0xA4, 0x41, 0xA4, 0xFE, /* 0x98-0x9B */ + 0xA4, 0x42, 0xA4, 0xD1, 0xA6, 0x61, 0xA4, 0x48, /* 0x9C-0x9F */ +}; + +static const unsigned char u2c_32[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0xA4, 0x40, 0xA4, 0x47, 0xA4, 0x54, 0xA5, 0x7C, /* 0x20-0x23 */ + 0xA4, 0xAD, 0xA4, 0xBB, 0xA4, 0x43, 0xA4, 0x4B, /* 0x24-0x27 */ + 0xA4, 0x45, 0xA4, 0x51, 0xA4, 0xEB, 0xA4, 0xF5, /* 0x28-0x2B */ + 0xA4, 0xF4, 0xA4, 0xEC, 0xAA, 0xF7, 0xA4, 0x67, /* 0x2C-0x2F */ + 0xA4, 0xE9, 0xAE, 0xE8, 0xA6, 0xB3, 0xAA, 0xC0, /* 0x30-0x33 */ + 0xA6, 0x57, 0xAF, 0x53, 0xB0, 0x5D, 0xAF, 0xAC, /* 0x34-0x37 */ + 0xB3, 0xD2, 0xA5, 0x4E, 0xA9, 0x49, 0xBE, 0xC7, /* 0x38-0x3B */ + 0xBA, 0xCA, 0xA5, 0xF8, 0xB8, 0xEA, 0xA8, 0xF3, /* 0x3C-0x3F */ + 0xB2, 0xBD, 0xA5, 0xF0, 0xA6, 0xDB, 0xA6, 0xDC, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xA4, 0x40, 0xA4, 0x47, 0xA4, 0x54, 0xA5, 0x7C, /* 0x80-0x83 */ + 0xA4, 0xAD, 0xA4, 0xBB, 0xA4, 0x43, 0xA4, 0x4B, /* 0x84-0x87 */ + 0xA4, 0x45, 0xA4, 0x51, 0xA4, 0xEB, 0xA4, 0xF5, /* 0x88-0x8B */ + 0xA4, 0xF4, 0xA4, 0xEC, 0xAA, 0xF7, 0xA4, 0x67, /* 0x8C-0x8F */ + 0xA4, 0xE9, 0xAE, 0xE8, 0xA6, 0xB3, 0xAA, 0xC0, /* 0x90-0x93 */ + 0xA6, 0x57, 0xAF, 0x53, 0xB0, 0x5D, 0xAF, 0xAC, /* 0x94-0x97 */ + 0xB3, 0xD2, 0xAF, 0xB5, 0xA8, 0x6B, 0xA4, 0x6B, /* 0x98-0x9B */ + 0xBE, 0x41, 0xC0, 0x75, 0xA6, 0x4C, 0xAA, 0x60, /* 0x9C-0x9F */ + 0xB6, 0xB5, 0xA5, 0xF0, 0xBC, 0x67, 0xA1, 0xC0, /* 0xA0-0xA3 */ + 0xA4, 0x57, 0xA4, 0xA4, 0xA4, 0x55, 0xA5, 0xAA, /* 0xA4-0xA7 */ + 0xA5, 0x6B, 0xC2, 0xE5, 0xA9, 0x76, 0xBE, 0xC7, /* 0xA8-0xAB */ + 0xBA, 0xCA, 0xA5, 0xF8, 0xB8, 0xEA, 0xA8, 0xF3, /* 0xAC-0xAF */ + 0xA9, 0x5D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ +}; + +static const unsigned char u2c_33[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0xA2, 0x55, 0xA2, 0x56, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0xA2, 0x50, 0xA2, 0x51, 0xA2, 0x52, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0xA2, 0x54, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xA2, 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0xA2, 0x53, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0xA1, 0xEB, 0xA1, 0xEA, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xA2, 0x4F, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ +}; + +static const unsigned char u2c_4E[512] = { + 0xA4, 0x40, 0xA4, 0x42, 0x00, 0x00, 0xA4, 0x43, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC9, 0x45, /* 0x04-0x07 */ + 0xA4, 0x56, 0xA4, 0x54, 0xA4, 0x57, 0xA4, 0x55, /* 0x08-0x0B */ + 0xC9, 0x46, 0xA4, 0xA3, 0xC9, 0x4F, 0xC9, 0x4D, /* 0x0C-0x0F */ + 0xA4, 0xA2, 0xA4, 0xA1, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0xA5, 0x42, 0xA5, 0x41, 0xA5, 0x40, 0x00, 0x00, /* 0x14-0x17 */ + 0xA5, 0x43, 0xA4, 0xFE, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xA5, 0xE0, 0xA5, 0xE1, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0xA8, 0xC3, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA4, 0x58, /* 0x28-0x2B */ + 0x00, 0x00, 0xA4, 0xA4, 0xC9, 0x50, 0x00, 0x00, /* 0x2C-0x2F */ + 0xA4, 0xA5, 0xC9, 0x63, 0xA6, 0xEA, 0xCB, 0xB1, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0xA4, 0x59, 0xA4, 0xA6, 0x00, 0x00, 0xA5, 0x44, /* 0x38-0x3B */ + 0xC9, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0xC9, 0x40, 0xA4, 0x44, /* 0x40-0x43 */ + 0x00, 0x00, 0xA4, 0x5B, 0x00, 0x00, 0xC9, 0x47, /* 0x44-0x47 */ + 0xA4, 0x5C, 0x00, 0x00, 0x00, 0x00, 0xA4, 0xA7, /* 0x48-0x4B */ + 0x00, 0x00, 0xA5, 0x45, 0xA5, 0x47, 0xA5, 0x46, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0xA5, 0xE2, 0xA5, 0xE3, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0xA8, 0xC4, 0x00, 0x00, /* 0x54-0x57 */ + 0xAD, 0xBC, 0xA4, 0x41, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0xC9, 0x41, 0xA4, 0x45, 0xA4, 0x5E, 0xA4, 0x5D, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0xA5, 0xE4, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA8, 0xC5, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0xB0, 0xAE, 0xD4, 0x4B, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0xB6, 0xC3, 0xDC, 0xB1, /* 0x80-0x83 */ + 0xDC, 0xB2, 0x00, 0x00, 0xA4, 0x46, 0x00, 0x00, /* 0x84-0x87 */ + 0xA4, 0xA9, 0x00, 0x00, 0x00, 0x00, 0xA8, 0xC6, /* 0x88-0x8B */ + 0xA4, 0x47, 0xC9, 0x48, 0xA4, 0x5F, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xA4, 0xAA, 0xA4, 0xAC, 0xC9, 0x51, /* 0x90-0x93 */ + 0xA4, 0xAD, 0xA4, 0xAB, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0xA5, 0xE5, 0x00, 0x00, 0xA8, 0xC7, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0xA8, 0xC8, 0xAB, 0x45, /* 0x9C-0x9F */ + 0x00, 0x00, 0xA4, 0x60, 0xA4, 0xAE, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xA5, 0xE6, 0xA5, 0xE8, 0xA5, 0xE7, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xA6, 0xEB, 0x00, 0x00, 0x00, 0x00, 0xA8, 0xC9, /* 0xA8-0xAB */ + 0xA8, 0xCA, 0xAB, 0x46, 0xAB, 0x47, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAD, 0xBD, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0xDC, 0xB3, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xF6, 0xD6, 0xA4, 0x48, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0xA4, 0xB0, 0xA4, 0xAF, 0xC9, 0x52, 0xA4, 0xB1, /* 0xC0-0xC3 */ + 0xA4, 0xB7, 0x00, 0x00, 0xA4, 0xB2, 0xA4, 0xB3, /* 0xC4-0xC7 */ + 0xC9, 0x54, 0xC9, 0x53, 0xA4, 0xB5, 0xA4, 0xB6, /* 0xC8-0xCB */ + 0x00, 0x00, 0xA4, 0xB4, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xA5, 0x4A, 0xA5, 0x4B, 0xA5, 0x4C, 0xA5, 0x4D, /* 0xD4-0xD7 */ + 0xA5, 0x49, 0xA5, 0x50, 0xC9, 0x6A, 0x00, 0x00, /* 0xD8-0xDB */ + 0xC9, 0x66, 0xC9, 0x69, 0xA5, 0x51, 0xA5, 0x61, /* 0xDC-0xDF */ + 0x00, 0x00, 0xC9, 0x68, 0x00, 0x00, 0xA5, 0x4E, /* 0xE0-0xE3 */ + 0xA5, 0x4F, 0xA5, 0x48, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xC9, 0x65, 0xC9, 0x67, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0xA5, 0xF5, 0xC9, 0xB0, 0xA5, 0xF2, 0xA5, 0xF6, /* 0xF0-0xF3 */ + 0xC9, 0xBA, 0xC9, 0xAE, 0xA5, 0xF3, 0xC9, 0xB2, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA5, 0xF4, /* 0xF8-0xFB */ + 0x00, 0x00, 0xA5, 0xF7, 0x00, 0x00, 0xA5, 0xE9, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_4F[512] = { + 0xC9, 0xB1, 0xA5, 0xF8, 0xC9, 0xB5, 0x00, 0x00, /* 0x00-0x03 */ + 0xC9, 0xB9, 0xC9, 0xB6, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0xC9, 0xB3, 0xA5, 0xEA, 0xA5, 0xEC, 0xA5, 0xF9, /* 0x08-0x0B */ + 0x00, 0x00, 0xA5, 0xEE, 0xC9, 0xAB, 0xA5, 0xF1, /* 0x0C-0x0F */ + 0xA5, 0xEF, 0xA5, 0xF0, 0xC9, 0xBB, 0xC9, 0xB8, /* 0x10-0x13 */ + 0xC9, 0xAF, 0xA5, 0xED, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0xC9, 0xAC, 0xA5, 0xEB, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0xC9, 0xB4, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0xC9, 0xB7, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0xC9, 0xAD, 0xCA, 0x66, 0x00, 0x00, 0xA7, 0x42, /* 0x2C-0x2F */ + 0xA6, 0xF4, 0x00, 0x00, 0x00, 0x00, 0xCA, 0x67, /* 0x30-0x33 */ + 0xA6, 0xF1, 0x00, 0x00, 0xA7, 0x44, 0x00, 0x00, /* 0x34-0x37 */ + 0xA6, 0xF9, 0x00, 0x00, 0xA6, 0xF8, 0xCA, 0x5B, /* 0x38-0x3B */ + 0xA6, 0xFC, 0xA6, 0xF7, 0xCA, 0x60, 0xCA, 0x68, /* 0x3C-0x3F */ + 0x00, 0x00, 0xCA, 0x64, 0x00, 0x00, 0xA6, 0xFA, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xA6, 0xFD, 0xA6, 0xEE, /* 0x44-0x47 */ + 0xA7, 0x47, 0xCA, 0x5D, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0xCB, 0xBD, 0xA6, 0xEC, 0xA7, 0x43, 0xA6, 0xED, /* 0x4C-0x4F */ + 0xA6, 0xF5, 0xA6, 0xF6, 0xCA, 0x62, 0xCA, 0x5E, /* 0x50-0x53 */ + 0xA6, 0xFB, 0xA6, 0xF3, 0xCA, 0x5A, 0xA6, 0xEF, /* 0x54-0x57 */ + 0xCA, 0x65, 0xA7, 0x45, 0xA7, 0x48, 0xA6, 0xF2, /* 0x58-0x5B */ + 0xA7, 0x40, 0xA7, 0x46, 0xA6, 0xF0, 0xCA, 0x63, /* 0x5C-0x5F */ + 0xA7, 0x41, 0xCA, 0x69, 0xCA, 0x5C, 0xA6, 0xFE, /* 0x60-0x63 */ + 0xCA, 0x5F, 0x00, 0x00, 0x00, 0x00, 0xCA, 0x61, /* 0x64-0x67 */ + 0x00, 0x00, 0xA8, 0xD8, 0xCB, 0xBF, 0xCB, 0xCB, /* 0x68-0x6B */ + 0xA8, 0xD0, 0x00, 0x00, 0xCB, 0xCC, 0xA8, 0xCB, /* 0x6C-0x6F */ + 0xA8, 0xD5, 0x00, 0x00, 0x00, 0x00, 0xA8, 0xCE, /* 0x70-0x73 */ + 0xCB, 0xB9, 0xA8, 0xD6, 0xCB, 0xB8, 0xCB, 0xBC, /* 0x74-0x77 */ + 0xCB, 0xC3, 0xCB, 0xC1, 0xA8, 0xDE, 0xA8, 0xD9, /* 0x78-0x7B */ + 0xCB, 0xB3, 0xCB, 0xB5, 0xA8, 0xDB, 0xA8, 0xCF, /* 0x7C-0x7F */ + + 0xCB, 0xB6, 0xCB, 0xC2, 0xCB, 0xC9, 0xA8, 0xD4, /* 0x80-0x83 */ + 0xCB, 0xBB, 0xCB, 0xB4, 0xA8, 0xD3, 0xCB, 0xB7, /* 0x84-0x87 */ + 0xA8, 0xD7, 0xCB, 0xBA, 0x00, 0x00, 0xA8, 0xD2, /* 0x88-0x8B */ + 0x00, 0x00, 0xA8, 0xCD, 0x00, 0x00, 0xA8, 0xDC, /* 0x8C-0x8F */ + 0xCB, 0xC4, 0xA8, 0xDD, 0xCB, 0xC8, 0x00, 0x00, /* 0x90-0x93 */ + 0xCB, 0xC6, 0xCB, 0xCA, 0xA8, 0xDA, 0xCB, 0xBE, /* 0x94-0x97 */ + 0xCB, 0xB2, 0x00, 0x00, 0xCB, 0xC0, 0xA8, 0xD1, /* 0x98-0x9B */ + 0xCB, 0xC5, 0xA8, 0xCC, 0xCB, 0xC7, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0xAB, 0x56, 0xAB, 0x4A, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xCD, 0xE0, 0xCD, 0xE8, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xAB, 0x49, 0xAB, 0x51, 0xAB, 0x5D, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xCD, 0xEE, 0xCD, 0xEC, 0xCD, 0xE7, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAB, 0x4B, /* 0xBC-0xBF */ + 0xCD, 0xED, 0xCD, 0xE3, 0xAB, 0x59, 0xAB, 0x50, /* 0xC0-0xC3 */ + 0xAB, 0x58, 0xCD, 0xDE, 0x00, 0x00, 0xCD, 0xEA, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xCD, 0xE1, 0xAB, 0x54, 0xCD, 0xE2, /* 0xC8-0xCB */ + 0x00, 0x00, 0xCD, 0xDD, 0xAB, 0x5B, 0xAB, 0x4E, /* 0xCC-0xCF */ + 0xAB, 0x57, 0xAB, 0x4D, 0x00, 0x00, 0xCD, 0xDF, /* 0xD0-0xD3 */ + 0xCD, 0xE4, 0x00, 0x00, 0xCD, 0xEB, 0xAB, 0x55, /* 0xD4-0xD7 */ + 0xAB, 0x52, 0xCD, 0xE6, 0xAB, 0x5A, 0xCD, 0xE9, /* 0xD8-0xDB */ + 0xCD, 0xE5, 0xAB, 0x4F, 0xAB, 0x5C, 0xAB, 0x53, /* 0xDC-0xDF */ + 0xAB, 0x4C, 0xAB, 0x48, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0xCD, 0xEF, 0x00, 0x00, 0xAD, 0xD7, 0xAD, 0xC1, /* 0xEC-0xEF */ + 0x00, 0x00, 0xAD, 0xD1, 0x00, 0x00, 0xAD, 0xD6, /* 0xF0-0xF3 */ + 0xD0, 0xD0, 0xD0, 0xCF, 0xD0, 0xD4, 0xD0, 0xD5, /* 0xF4-0xF7 */ + 0xAD, 0xC4, 0x00, 0x00, 0xAD, 0xCD, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0xAD, 0xDA, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_50[512] = { + 0xAD, 0xCE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0xD0, 0xC9, 0xAD, 0xC7, 0xD0, 0xCA, /* 0x04-0x07 */ + 0x00, 0x00, 0xAD, 0xDC, 0x00, 0x00, 0xAD, 0xD3, /* 0x08-0x0B */ + 0xAD, 0xBE, 0xAD, 0xBF, 0xD0, 0xDD, 0xB0, 0xBF, /* 0x0C-0x0F */ + 0x00, 0x00, 0xAD, 0xCC, 0xAD, 0xCB, 0xD0, 0xCB, /* 0x10-0x13 */ + 0xAD, 0xCF, 0xD4, 0x5B, 0xAD, 0xC6, 0xD0, 0xD6, /* 0x14-0x17 */ + 0xAD, 0xD5, 0xAD, 0xD4, 0xAD, 0xCA, 0xD0, 0xCE, /* 0x18-0x1B */ + 0xD0, 0xD7, 0x00, 0x00, 0xD0, 0xC8, 0xAD, 0xC9, /* 0x1C-0x1F */ + 0xD0, 0xD8, 0xAD, 0xD2, 0xD0, 0xCC, 0xAD, 0xC0, /* 0x20-0x23 */ + 0x00, 0x00, 0xAD, 0xC3, 0xAD, 0xC2, 0xD0, 0xD9, /* 0x24-0x27 */ + 0xAD, 0xD0, 0xAD, 0xC5, 0xAD, 0xD9, 0xAD, 0xDB, /* 0x28-0x2B */ + 0xD0, 0xD3, 0xAD, 0xD8, 0x00, 0x00, 0xD0, 0xDB, /* 0x2C-0x2F */ + 0xD0, 0xCD, 0xD0, 0xDC, 0x00, 0x00, 0xD0, 0xD1, /* 0x30-0x33 */ + 0x00, 0x00, 0xD0, 0xDA, 0x00, 0x00, 0xD0, 0xD2, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0xAD, 0xC8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0xD4, 0x63, 0xD4, 0x57, 0x00, 0x00, 0xB0, 0xB3, /* 0x40-0x43 */ + 0x00, 0x00, 0xD4, 0x5C, 0xD4, 0x62, 0xB0, 0xB2, /* 0x44-0x47 */ + 0xD4, 0x55, 0xB0, 0xB6, 0xD4, 0x59, 0xD4, 0x52, /* 0x48-0x4B */ + 0xB0, 0xB4, 0xD4, 0x56, 0xB0, 0xB9, 0xB0, 0xBE, /* 0x4C-0x4F */ + 0x00, 0x00, 0xD4, 0x67, 0x00, 0x00, 0xD4, 0x51, /* 0x50-0x53 */ + 0x00, 0x00, 0xB0, 0xBA, 0x00, 0x00, 0xD4, 0x66, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0xB0, 0xB5, 0xD4, 0x58, /* 0x58-0x5B */ + 0xB0, 0xB1, 0xD4, 0x53, 0xD4, 0x4F, 0xD4, 0x5D, /* 0x5C-0x5F */ + 0xD4, 0x50, 0xD4, 0x4E, 0xD4, 0x5A, 0xD4, 0x60, /* 0x60-0x63 */ + 0xD4, 0x61, 0xB0, 0xB7, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0xD8, 0x5B, 0xD4, 0x5E, 0xD4, 0x4D, 0xD4, 0x5F, /* 0x68-0x6B */ + 0x00, 0x00, 0xB0, 0xC1, 0xD4, 0x64, 0xB0, 0xC0, /* 0x6C-0x6F */ + 0xD4, 0x4C, 0x00, 0x00, 0xD4, 0x54, 0xD4, 0x65, /* 0x70-0x73 */ + 0xB0, 0xBC, 0xB0, 0xBB, 0xB0, 0xB8, 0xB0, 0xBD, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0xB0, 0xAF, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0xB0, 0xB0, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xB3, 0xC8, 0x00, 0x00, 0xD8, 0x5E, 0xD8, 0x57, /* 0x80-0x83 */ + 0x00, 0x00, 0xB3, 0xC5, 0x00, 0x00, 0xD8, 0x5F, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD8, 0x55, /* 0x88-0x8B */ + 0xD8, 0x58, 0xB3, 0xC4, 0xD8, 0x59, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xB3, 0xC7, 0xD8, 0x5D, 0x00, 0x00, /* 0x90-0x93 */ + 0xD8, 0x53, 0xD8, 0x52, 0xB3, 0xC9, 0x00, 0x00, /* 0x94-0x97 */ + 0xB3, 0xCA, 0xB3, 0xC6, 0xB3, 0xCB, 0xD8, 0x51, /* 0x98-0x9B */ + 0xD8, 0x5C, 0xD8, 0x5A, 0xD8, 0x54, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0xB3, 0xC3, 0xD8, 0x56, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0xB6, 0xCA, 0xB6, 0xC4, 0xDC, 0xB7, 0xB6, 0xCD, /* 0xAC-0xAF */ + 0xDC, 0xBD, 0xDC, 0xC0, 0xB6, 0xC6, 0xB6, 0xC7, /* 0xB0-0xB3 */ + 0xDC, 0xBA, 0xB6, 0xC5, 0xDC, 0xC3, 0xB6, 0xCB, /* 0xB4-0xB7 */ + 0xDC, 0xC4, 0x00, 0x00, 0xDC, 0xBF, 0xB6, 0xCC, /* 0xB8-0xBB */ + 0x00, 0x00, 0xDC, 0xB4, 0xB6, 0xC9, 0xDC, 0xB5, /* 0xBC-0xBF */ + 0x00, 0x00, 0xDC, 0xBE, 0xDC, 0xBC, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xDC, 0xB8, 0xB6, 0xC8, 0xDC, 0xB6, 0xB6, 0xCE, /* 0xC4-0xC7 */ + 0xDC, 0xBB, 0xDC, 0xC2, 0xDC, 0xB9, 0xDC, 0xC1, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0xB9, 0xB6, 0xB9, 0xB3, /* 0xCC-0xCF */ + 0x00, 0x00, 0xB9, 0xB4, 0x00, 0x00, 0xE0, 0xF9, /* 0xD0-0xD3 */ + 0xE0, 0xF1, 0xB9, 0xB2, 0xB9, 0xAF, 0xE0, 0xF2, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xB9, 0xB1, 0xE0, 0xF5, /* 0xD8-0xDB */ + 0x00, 0x00, 0xE0, 0xF7, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0xE0, 0xFE, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xFD, /* 0xE0-0xE3 */ + 0xE0, 0xF8, 0xB9, 0xAE, 0xE0, 0xF0, 0xB9, 0xAC, /* 0xE4-0xE7 */ + 0xE0, 0xF3, 0xB9, 0xB7, 0xE0, 0xF6, 0x00, 0x00, /* 0xE8-0xEB */ + 0xE0, 0xFA, 0xB9, 0xB0, 0xB9, 0xAD, 0xE0, 0xFC, /* 0xEC-0xEF */ + 0xE0, 0xFB, 0xB9, 0xB5, 0x00, 0x00, 0xE0, 0xF4, /* 0xF0-0xF3 */ + 0x00, 0x00, 0xBB, 0xF8, 0xE4, 0xEC, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xE4, 0xE9, 0xBB, 0xF9, 0x00, 0x00, 0xBB, 0xF7, /* 0xF8-0xFB */ + 0x00, 0x00, 0xE4, 0xF0, 0xE4, 0xED, 0xE4, 0xE6, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_51[512] = { + 0xBB, 0xF6, 0x00, 0x00, 0xBB, 0xFA, 0xE4, 0xE7, /* 0x00-0x03 */ + 0xBB, 0xF5, 0xBB, 0xFD, 0xE4, 0xEA, 0xE4, 0xEB, /* 0x04-0x07 */ + 0xBB, 0xFB, 0xBB, 0xFC, 0xE4, 0xF1, 0xE4, 0xEE, /* 0x08-0x0B */ + 0xE4, 0xEF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0xBE, 0xAA, 0xE8, 0xF8, 0xBE, 0xA7, 0xE8, 0xF5, /* 0x10-0x13 */ + 0xBE, 0xA9, 0xBE, 0xAB, 0x00, 0x00, 0xE8, 0xF6, /* 0x14-0x17 */ + 0xBE, 0xA8, 0x00, 0x00, 0xE8, 0xF7, 0x00, 0x00, /* 0x18-0x1B */ + 0xE8, 0xF4, 0x00, 0x00, 0x00, 0x00, 0xC0, 0x76, /* 0x1C-0x1F */ + 0xEC, 0xBD, 0xC0, 0x77, 0xEC, 0xBB, 0x00, 0x00, /* 0x20-0x23 */ + 0xEC, 0xBC, 0xEC, 0xBA, 0xEC, 0xB9, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0xEC, 0xBE, 0xC0, 0x75, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0xEF, 0xB8, 0xEF, 0xB9, 0x00, 0x00, /* 0x2C-0x2F */ + 0xE4, 0xE8, 0xEF, 0xB7, 0xC0, 0x78, 0xC3, 0x5F, /* 0x30-0x33 */ + 0xF1, 0xEB, 0xF1, 0xEC, 0x00, 0x00, 0xC4, 0xD7, /* 0x34-0x37 */ + 0xC4, 0xD8, 0xF5, 0xC1, 0xF5, 0xC0, 0xC5, 0x6C, /* 0x38-0x3B */ + 0xC5, 0x6B, 0xF7, 0xD0, 0x00, 0x00, 0xA4, 0x49, /* 0x3C-0x3F */ + 0xA4, 0x61, 0xA4, 0xB9, 0x00, 0x00, 0xA4, 0xB8, /* 0x40-0x43 */ + 0xA5, 0x53, 0xA5, 0x52, 0xA5, 0xFC, 0xA5, 0xFB, /* 0x44-0x47 */ + 0xA5, 0xFD, 0xA5, 0xFA, 0x00, 0x00, 0xA7, 0x4A, /* 0x48-0x4B */ + 0xA7, 0x49, 0xA7, 0x4B, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0xA8, 0xE0, 0x00, 0x00, /* 0x50-0x53 */ + 0xA8, 0xDF, 0xA8, 0xE1, 0x00, 0x00, 0xAB, 0x5E, /* 0x54-0x57 */ + 0x00, 0x00, 0xA2, 0x59, 0xD0, 0xDE, 0xA2, 0x5A, /* 0x58-0x5B */ + 0xB0, 0xC2, 0xA2, 0x5C, 0xA2, 0x5B, 0xD8, 0x60, /* 0x5C-0x5F */ + 0x00, 0x00, 0xA2, 0x5D, 0xB9, 0xB8, 0xA2, 0x5E, /* 0x60-0x63 */ + 0x00, 0x00, 0xA4, 0x4A, 0x00, 0x00, 0xA4, 0xBA, /* 0x64-0x67 */ + 0xA5, 0xFE, 0xA8, 0xE2, 0x00, 0x00, 0xA4, 0x4B, /* 0x68-0x6B */ + 0xA4, 0xBD, 0xA4, 0xBB, 0xA4, 0xBC, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0xA6, 0x40, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0xA7, 0x4C, 0xA8, 0xE4, 0xA8, 0xE3, /* 0x74-0x77 */ + 0xA8, 0xE5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0xAD, 0xDD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xBE, 0xAC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC9, 0x4E, /* 0x84-0x87 */ + 0x00, 0x00, 0xA5, 0x54, 0xA5, 0x55, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0xA6, 0x41, 0x00, 0x00, 0xCA, 0x6A, /* 0x8C-0x8F */ + 0x00, 0x00, 0xAB, 0x60, 0xAB, 0x5F, 0xD0, 0xE0, /* 0x90-0x93 */ + 0xD0, 0xDF, 0xB0, 0xC3, 0x00, 0x00, 0xA4, 0xBE, /* 0x94-0x97 */ + 0xC9, 0x55, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0xCB, 0xCD, 0x00, 0x00, /* 0x9C-0x9F */ + 0xAB, 0x61, 0x00, 0x00, 0xAD, 0xE0, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xAD, 0xDE, 0xAD, 0xDF, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0xBE, 0xAD, 0x00, 0x00, /* 0xA8-0xAB */ + 0xA5, 0x56, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0xA6, 0x42, 0xC9, 0xBC, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0xA7, 0x4D, 0xA7, 0x4E, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xCA, 0x6B, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0xCB, 0xCE, 0xA8, 0xE6, 0xCB, 0xCF, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xD0, 0xE2, 0xD0, 0xE3, 0xAD, 0xE3, 0x00, 0x00, /* 0xC4-0xC7 */ + 0xD0, 0xE4, 0x00, 0x00, 0xD0, 0xE1, 0xAD, 0xE4, /* 0xC8-0xCB */ + 0xAD, 0xE2, 0xAD, 0xE1, 0xD0, 0xE5, 0x00, 0x00, /* 0xCC-0xCF */ + 0xD4, 0x68, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xD8, 0x61, 0x00, 0x00, 0x00, 0x00, 0xDC, 0xC5, /* 0xD4-0xD7 */ + 0xE1, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0xBB, 0xFE, 0xBE, 0xAE, 0xE8, 0xF9, 0x00, 0x00, /* 0xDC-0xDF */ + 0xA4, 0x4C, 0xA4, 0x5A, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0xB0, 0xC4, 0xB3, 0xCD, 0x00, 0x00, 0xB9, 0xB9, /* 0xF0-0xF3 */ + 0x00, 0x00, 0xC9, 0x42, 0xA4, 0xBF, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xA5, 0x59, 0xA5, 0x57, 0xA5, 0x58, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0xA8, 0xE7, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_52[512] = { + 0xA4, 0x4D, 0xA4, 0x4E, 0x00, 0x00, 0xA4, 0x62, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0xA4, 0xC0, 0xA4, 0xC1, /* 0x04-0x07 */ + 0xA4, 0xC2, 0xC9, 0xBE, 0xA5, 0x5A, 0x00, 0x00, /* 0x08-0x0B */ + 0xC9, 0x6B, 0x00, 0x00, 0xA6, 0x46, 0x00, 0x00, /* 0x0C-0x0F */ + 0xC9, 0xBF, 0xA6, 0x44, 0xA6, 0x45, 0xC9, 0xBD, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xA6, 0x47, 0xA6, 0x43, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0xCA, 0x6C, 0xAA, 0xEC, 0xCA, 0x6D, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0xCA, 0x6E, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0xA7, 0x50, 0xA7, 0x4F, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0xA7, 0x53, 0xA7, 0x51, 0xA7, 0x52, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0xA8, 0xED, 0x00, 0x00, /* 0x2C-0x2F */ + 0xA8, 0xEC, 0xCB, 0xD4, 0xCB, 0xD1, 0xCB, 0xD2, /* 0x30-0x33 */ + 0x00, 0x00, 0xCB, 0xD0, 0xA8, 0xEE, 0xA8, 0xEA, /* 0x34-0x37 */ + 0xA8, 0xE9, 0x00, 0x00, 0xA8, 0xEB, 0xA8, 0xE8, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0xA8, 0xEF, 0x00, 0x00, 0xAB, 0x63, /* 0x40-0x43 */ + 0xCD, 0xF0, 0x00, 0x00, 0xCB, 0xD3, 0xAB, 0x68, /* 0x44-0x47 */ + 0x00, 0x00, 0xCD, 0xF1, 0xAB, 0x64, 0xAB, 0x67, /* 0x48-0x4B */ + 0xAB, 0x66, 0xAB, 0x65, 0xAB, 0x62, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0xD0, 0xE8, 0x00, 0x00, /* 0x50-0x53 */ + 0xAD, 0xE7, 0xD0, 0xEB, 0xAD, 0xE5, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0xD0, 0xE7, 0xAD, 0xE8, /* 0x58-0x5B */ + 0xAD, 0xE6, 0xAD, 0xE9, 0xD0, 0xE9, 0xD0, 0xEA, /* 0x5C-0x5F */ + 0x00, 0x00, 0xD0, 0xE6, 0xD0, 0xEC, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0xB3, 0xD1, 0xB0, 0xC5, 0xD4, 0x69, /* 0x68-0x6B */ + 0xD4, 0x6B, 0xD4, 0x6A, 0xD4, 0x6C, 0xB0, 0xC6, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0xB3, 0xCE, 0x00, 0x00, /* 0x70-0x73 */ + 0xB3, 0xCF, 0xB3, 0xD0, 0x00, 0x00, 0xB6, 0xD0, /* 0x74-0x77 */ + 0xDC, 0xC7, 0x00, 0x00, 0xDC, 0xC6, 0xDC, 0xC8, /* 0x78-0x7B */ + 0xDC, 0xC9, 0xB6, 0xD1, 0x00, 0x00, 0xB6, 0xCF, /* 0x7C-0x7F */ + + 0xE1, 0x41, 0xE1, 0x42, 0xB9, 0xBB, 0xB9, 0xBA, /* 0x80-0x83 */ + 0xE3, 0x5A, 0x00, 0x00, 0x00, 0x00, 0xBC, 0x40, /* 0x84-0x87 */ + 0xBC, 0x41, 0xBC, 0x42, 0xBC, 0x44, 0xE4, 0xF2, /* 0x88-0x8B */ + 0xE4, 0xF3, 0xBC, 0x43, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xBE, 0xAF, 0x00, 0x00, 0xBE, 0xB0, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0xF1, 0xED, 0xF5, 0xC3, /* 0x94-0x97 */ + 0xF5, 0xC2, 0xF7, 0xD1, 0x00, 0x00, 0xA4, 0x4F, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA5, 0x5C, /* 0x9C-0x9F */ + 0xA5, 0x5B, 0x00, 0x00, 0x00, 0x00, 0xA6, 0x48, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0xC9, 0xC0, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0xA7, 0x55, 0xA7, 0x56, 0xA7, 0x54, /* 0xA8-0xAB */ + 0xA7, 0x57, 0xCA, 0x6F, 0xCA, 0x70, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA8, 0xF1, /* 0xB8-0xBB */ + 0xCB, 0xD5, 0x00, 0x00, 0xA8, 0xF0, 0x00, 0x00, /* 0xBC-0xBF */ + 0xCD, 0xF2, 0xAB, 0x6C, 0xCD, 0xF3, 0xAB, 0x6B, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAB, 0x69, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xAB, 0x6A, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0xD0, 0xED, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xB0, 0xC7, 0xD4, 0x6E, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xB0, 0xCA, 0xD4, 0x6D, 0xB1, 0xE5, /* 0xD4-0xD7 */ + 0xB0, 0xC9, 0xB0, 0xC8, 0x00, 0x00, 0xB3, 0xD4, /* 0xD8-0xDB */ + 0x00, 0x00, 0xB3, 0xD3, 0xB3, 0xD2, 0xB6, 0xD2, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0xB6, 0xD5, 0xB6, 0xD6, /* 0xE0-0xE3 */ + 0xB6, 0xD4, 0x00, 0x00, 0xB6, 0xD3, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xE1, 0x43, 0x00, 0x00, 0xE1, 0x44, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xF5, /* 0xEC-0xEF */ + 0xBC, 0x45, 0xE4, 0xF4, 0x00, 0x00, 0xBE, 0xB1, /* 0xF0-0xF3 */ + 0xEC, 0xBF, 0xC0, 0x79, 0x00, 0x00, 0xF1, 0xEE, /* 0xF4-0xF7 */ + 0xC4, 0x55, 0x00, 0x00, 0xA4, 0x63, 0xA4, 0xC3, /* 0xF8-0xFB */ + 0xC9, 0x56, 0x00, 0x00, 0xA4, 0xC4, 0xA4, 0xC5, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_53[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0xA5, 0x5D, 0xA5, 0x5E, 0x00, 0x00, /* 0x04-0x07 */ + 0xA6, 0x49, 0xCA, 0x71, 0xCB, 0xD6, 0xCB, 0xD7, /* 0x08-0x0B */ + 0x00, 0x00, 0xAB, 0x6D, 0xD0, 0xEE, 0xB0, 0xCC, /* 0x0C-0x0F */ + 0xB0, 0xCB, 0xD8, 0x63, 0xD8, 0x62, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0xA4, 0x50, 0xA4, 0xC6, 0xA5, 0x5F, /* 0x14-0x17 */ + 0x00, 0x00, 0xB0, 0xCD, 0xC9, 0x43, 0x00, 0x00, /* 0x18-0x1B */ + 0xC9, 0x6C, 0xA5, 0x60, 0x00, 0x00, 0xC9, 0xC2, /* 0x1C-0x1F */ + 0xA6, 0x4B, 0xA6, 0x4A, 0xC9, 0xC1, 0xA7, 0x58, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xAD, 0xEA, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0xD4, 0x6F, 0x00, 0x00, 0xB6, 0xD7, /* 0x2C-0x2F */ + 0xE1, 0x45, 0xB9, 0xBC, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0xE8, 0xFA, 0x00, 0x00, 0x00, 0x00, 0xF3, 0xFD, /* 0x34-0x37 */ + 0x00, 0x00, 0xA4, 0xC7, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0xCB, 0xD8, 0xCD, 0xF4, 0xB0, 0xD0, 0xB0, 0xCE, /* 0x3C-0x3F */ + 0xB0, 0xCF, 0xA4, 0x51, 0x00, 0x00, 0xA4, 0x64, /* 0x40-0x43 */ + 0xA2, 0xCD, 0xA4, 0xCA, 0x00, 0x00, 0xA4, 0xC9, /* 0x44-0x47 */ + 0xA4, 0xC8, 0xA5, 0x63, 0xA5, 0x62, 0x00, 0x00, /* 0x48-0x4B */ + 0xC9, 0x6D, 0xC9, 0xC3, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0xA8, 0xF5, 0xA8, 0xF2, 0xA8, 0xF4, /* 0x50-0x53 */ + 0xA8, 0xF3, 0x00, 0x00, 0x00, 0x00, 0xAB, 0x6E, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0xB3, 0xD5, 0x00, 0x00, /* 0x58-0x5B */ + 0xA4, 0x52, 0x00, 0x00, 0xA4, 0xCB, 0x00, 0x00, /* 0x5C-0x5F */ + 0xA5, 0x65, 0xA5, 0x64, 0x00, 0x00, 0xCA, 0x72, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0xA8, 0xF6, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0xC9, 0x57, 0x00, 0x00, 0xA5, 0x67, 0xA5, 0x66, /* 0x6C-0x6F */ + 0xA6, 0x4C, 0xA6, 0x4D, 0xCA, 0x73, 0xA7, 0x59, /* 0x70-0x73 */ + 0x00, 0x00, 0xA7, 0x5A, 0x00, 0x00, 0xA8, 0xF7, /* 0x74-0x77 */ + 0xA8, 0xF8, 0xA8, 0xF9, 0x00, 0x00, 0xAB, 0x6F, /* 0x78-0x7B */ + 0xCD, 0xF5, 0x00, 0x00, 0x00, 0x00, 0xAD, 0xEB, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0xC9, 0x44, 0x00, 0x00, /* 0x80-0x83 */ + 0xA4, 0xCC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0xC9, 0xC4, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0xCA, 0x74, 0xCA, 0x75, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0xCB, 0xD9, 0x00, 0x00, /* 0x90-0x93 */ + 0xCB, 0xDA, 0x00, 0x00, 0xCD, 0xF7, 0xCD, 0xF6, /* 0x94-0x97 */ + 0xCD, 0xF9, 0xCD, 0xF8, 0xAB, 0x70, 0x00, 0x00, /* 0x98-0x9B */ + 0xD4, 0x70, 0xAD, 0xED, 0xD0, 0xEF, 0xAD, 0xEC, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xD8, 0x64, 0xB3, 0xD6, 0x00, 0x00, 0xD8, 0x65, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0xE1, 0x46, 0xB9, 0xBD, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xBC, 0x46, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xF1, 0xEF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xC9, 0x58, 0x00, 0x00, 0xA5, 0x68, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB0, 0xD1, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0xA4, 0x53, 0xA4, 0x65, 0xA4, 0xCE, 0xA4, 0xCD, /* 0xC8-0xCB */ + 0x00, 0x00, 0xA4, 0xCF, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xA8, 0xFB, 0x00, 0x00, 0xA8, 0xFA, 0xA8, 0xFC, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAB, 0x71, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAD, 0xEE, /* 0xDC-0xDF */ + 0x00, 0x00, 0xE8, 0xFB, 0xC2, 0x4F, 0xA4, 0x66, /* 0xE0-0xE3 */ + 0xA5, 0x6A, 0xA5, 0x79, 0xA5, 0x74, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xA5, 0x6F, 0xA5, 0x6E, 0xA5, 0x75, 0xA5, 0x73, /* 0xE8-0xEB */ + 0xA5, 0x6C, 0xA5, 0x7A, 0xA5, 0x6D, 0xA5, 0x69, /* 0xEC-0xEF */ + 0xA5, 0x78, 0xA5, 0x77, 0xA5, 0x76, 0xA5, 0x6B, /* 0xF0-0xF3 */ + 0x00, 0x00, 0xA5, 0x72, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xA5, 0x71, 0x00, 0x00, 0x00, 0x00, 0xA5, 0x7B, /* 0xF8-0xFB */ + 0xA5, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_54[512] = { + 0x00, 0x00, 0xA6, 0x53, 0x00, 0x00, 0xA6, 0x59, /* 0x00-0x03 */ + 0xA6, 0x55, 0x00, 0x00, 0xA6, 0x5B, 0xC9, 0xC5, /* 0x04-0x07 */ + 0xA6, 0x58, 0xA6, 0x4E, 0xA6, 0x51, 0xA6, 0x54, /* 0x08-0x0B */ + 0xA6, 0x50, 0xA6, 0x57, 0xA6, 0x5A, 0xA6, 0x4F, /* 0x0C-0x0F */ + 0xA6, 0x52, 0xA6, 0x56, 0xA6, 0x5C, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0xCA, 0x7E, 0xCA, 0x7B, 0x00, 0x00, 0xA7, 0x67, /* 0x18-0x1B */ + 0xCA, 0x7C, 0xA7, 0x5B, 0xA7, 0x5D, 0xA7, 0x75, /* 0x1C-0x1F */ + 0xA7, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0xCA, 0xA5, 0xCA, 0x7D, 0xA7, 0x5F, 0xA7, 0x61, /* 0x24-0x27 */ + 0xCA, 0xA4, 0xA7, 0x68, 0xCA, 0x78, 0xA7, 0x74, /* 0x28-0x2B */ + 0xA7, 0x76, 0xA7, 0x5C, 0xA7, 0x6D, 0x00, 0x00, /* 0x2C-0x2F */ + 0xCA, 0x76, 0xA7, 0x73, 0x00, 0x00, 0xA7, 0x64, /* 0x30-0x33 */ + 0x00, 0x00, 0xA7, 0x6E, 0xA7, 0x6F, 0xCA, 0x77, /* 0x34-0x37 */ + 0xA7, 0x6C, 0xA7, 0x6A, 0x00, 0x00, 0xA7, 0x6B, /* 0x38-0x3B */ + 0xA7, 0x71, 0xCA, 0xA1, 0xA7, 0x5E, 0x00, 0x00, /* 0x3C-0x3F */ + 0xA7, 0x72, 0xCA, 0xA3, 0xA7, 0x66, 0xA7, 0x63, /* 0x40-0x43 */ + 0x00, 0x00, 0xCA, 0x7A, 0xA7, 0x62, 0xCA, 0xA6, /* 0x44-0x47 */ + 0xA7, 0x65, 0x00, 0x00, 0xA7, 0x69, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0xA7, 0x60, 0xCA, 0xA2, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0xCA, 0x79, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0xCB, 0xEB, 0xCB, 0xEA, 0xA9, 0x4F, 0xCB, 0xED, /* 0x60-0x63 */ + 0xCB, 0xEF, 0xCB, 0xE4, 0xCB, 0xE7, 0xCB, 0xEE, /* 0x64-0x67 */ + 0xA9, 0x50, 0x00, 0x00, 0x00, 0x00, 0xCB, 0xE1, /* 0x68-0x6B */ + 0xCB, 0xE5, 0x00, 0x00, 0x00, 0x00, 0xCB, 0xE9, /* 0x6C-0x6F */ + 0xCE, 0x49, 0xA9, 0x4B, 0xCE, 0x4D, 0xA8, 0xFD, /* 0x70-0x73 */ + 0xCB, 0xE6, 0xA8, 0xFE, 0xA9, 0x4C, 0xA9, 0x45, /* 0x74-0x77 */ + 0xA9, 0x41, 0x00, 0x00, 0xCB, 0xE2, 0xA9, 0x44, /* 0x78-0x7B */ + 0xA9, 0x49, 0xA9, 0x52, 0xCB, 0xE3, 0xCB, 0xDC, /* 0x7C-0x7F */ + + 0xA9, 0x43, 0xCB, 0xDD, 0xCB, 0xDF, 0x00, 0x00, /* 0x80-0x83 */ + 0xA9, 0x46, 0x00, 0x00, 0xA9, 0x48, 0xCB, 0xDB, /* 0x84-0x87 */ + 0xCB, 0xE0, 0x00, 0x00, 0x00, 0x00, 0xA9, 0x51, /* 0x88-0x8B */ + 0xA9, 0x4D, 0xCB, 0xE8, 0xA9, 0x53, 0x00, 0x00, /* 0x8C-0x8F */ + 0xA9, 0x4A, 0xCB, 0xDE, 0xA9, 0x47, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0xA9, 0x42, 0xA9, 0x40, 0x00, 0x00, /* 0x94-0x97 */ + 0xCB, 0xEC, 0x00, 0x00, 0xA9, 0x4E, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0xCE, 0x48, 0xCD, 0xFB, 0xCE, 0x4B, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xCD, 0xFD, 0xAB, 0x78, 0xAB, 0xA8, /* 0xA4-0xA7 */ + 0xAB, 0x74, 0xAB, 0xA7, 0xAB, 0x7D, 0xAB, 0xA4, /* 0xA8-0xAB */ + 0xAB, 0x72, 0xCD, 0xFC, 0xCE, 0x43, 0xAB, 0xA3, /* 0xAC-0xAF */ + 0xCE, 0x4F, 0xAB, 0xA5, 0x00, 0x00, 0xAB, 0x79, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0xCE, 0x45, 0xCE, 0x42, /* 0xB4-0xB7 */ + 0xAB, 0x77, 0x00, 0x00, 0xCD, 0xFA, 0xAB, 0xA6, /* 0xB8-0xBB */ + 0xCE, 0x4A, 0xAB, 0x7C, 0xCE, 0x4C, 0xAB, 0xA9, /* 0xBC-0xBF */ + 0xAB, 0x73, 0xAB, 0x7E, 0xAB, 0x7B, 0xCE, 0x40, /* 0xC0-0xC3 */ + 0xAB, 0xA1, 0xCE, 0x46, 0xCE, 0x47, 0xAB, 0x7A, /* 0xC4-0xC7 */ + 0xAB, 0xA2, 0xAB, 0x76, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0xAB, 0x75, 0xCD, 0xFE, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0xCE, 0x44, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0xCE, 0x4E, 0x00, 0x00, /* 0xDC-0xDF */ + 0xD1, 0x44, 0xAD, 0xFB, 0xD0, 0xF1, 0x00, 0x00, /* 0xE0-0xE3 */ + 0xD0, 0xF6, 0xAD, 0xF4, 0xAE, 0x40, 0xD0, 0xF4, /* 0xE4-0xE7 */ + 0xAD, 0xEF, 0xAD, 0xF9, 0xAD, 0xFE, 0xD0, 0xFB, /* 0xE8-0xEB */ + 0x00, 0x00, 0xAD, 0xFA, 0xAD, 0xFD, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0xD0, 0xFE, 0xAD, 0xF5, 0xD0, 0xF5, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD1, 0x42, /* 0xF4-0xF7 */ + 0xD1, 0x43, 0x00, 0x00, 0xAD, 0xF7, 0xD1, 0x41, /* 0xF8-0xFB */ + 0xAD, 0xF3, 0xAE, 0x43, 0x00, 0x00, 0xD0, 0xF8, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_55[512] = { + 0x00, 0x00, 0xAD, 0xF1, 0x00, 0x00, 0xD1, 0x46, /* 0x00-0x03 */ + 0xD0, 0xF9, 0xD0, 0xFD, 0xAD, 0xF6, 0xAE, 0x42, /* 0x04-0x07 */ + 0xD0, 0xFA, 0xAD, 0xFC, 0xD1, 0x40, 0xD1, 0x47, /* 0x08-0x0B */ + 0xD4, 0xA1, 0x00, 0x00, 0xD1, 0x45, 0xAE, 0x44, /* 0x0C-0x0F */ + 0xAD, 0xF0, 0xD0, 0xFC, 0xD0, 0xF3, 0x00, 0x00, /* 0x10-0x13 */ + 0xAD, 0xF8, 0x00, 0x00, 0x00, 0x00, 0xD0, 0xF2, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0xD0, 0xF7, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0xD0, 0xF0, 0xAE, 0x41, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xD4, 0x77, 0x00, 0x00, /* 0x28-0x2B */ + 0xB0, 0xE4, 0xD4, 0xA7, 0xB0, 0xE2, 0xB0, 0xDF, /* 0x2C-0x2F */ + 0xD4, 0x7C, 0xB0, 0xDB, 0xD4, 0xA2, 0xB0, 0xE6, /* 0x30-0x33 */ + 0xD4, 0x76, 0xD4, 0x7B, 0xD4, 0x7A, 0xAD, 0xF2, /* 0x34-0x37 */ + 0xB0, 0xE1, 0xD4, 0xA5, 0x00, 0x00, 0xD4, 0xA8, /* 0x38-0x3B */ + 0xD4, 0x73, 0x00, 0x00, 0xB3, 0xE8, 0x00, 0x00, /* 0x3C-0x3F */ + 0xD4, 0xA9, 0xB0, 0xE7, 0x00, 0x00, 0xB0, 0xD9, /* 0x40-0x43 */ + 0xB0, 0xD6, 0xD4, 0x7E, 0xB0, 0xD3, 0x00, 0x00, /* 0x44-0x47 */ + 0xD4, 0xA6, 0x00, 0x00, 0xB0, 0xDA, 0xD4, 0xAA, /* 0x48-0x4B */ + 0x00, 0x00, 0xD4, 0x74, 0xD4, 0xA4, 0xB0, 0xDD, /* 0x4C-0x4F */ + 0xD4, 0x75, 0xD4, 0x78, 0xD4, 0x7D, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0xB0, 0xDE, 0xB0, 0xDC, 0xB0, 0xE8, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0xB0, 0xE3, 0x00, 0x00, 0xB0, 0xD7, 0xB1, 0xD2, /* 0x5C-0x5F */ + 0x00, 0x00, 0xB0, 0xD8, 0xD4, 0x79, 0xB0, 0xE5, /* 0x60-0x63 */ + 0xB0, 0xE0, 0xD4, 0xA3, 0xB0, 0xD5, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0xB0, 0xD4, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0xD4, 0x71, 0xD4, 0x72, 0xD8, 0x6A, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB3, 0xD7, /* 0x78-0x7B */ + 0xB3, 0xDA, 0xD8, 0x75, 0xB3, 0xEE, 0xD8, 0x78, /* 0x7C-0x7F */ + + 0xB3, 0xD8, 0xD8, 0x71, 0xB3, 0xDE, 0xB3, 0xE4, /* 0x80-0x83 */ + 0xB5, 0xBD, 0x00, 0x00, 0x00, 0x00, 0xB3, 0xE2, /* 0x84-0x87 */ + 0xD8, 0x6E, 0xB3, 0xEF, 0xB3, 0xDB, 0xB3, 0xE3, /* 0x88-0x8B */ + 0xD8, 0x76, 0xDC, 0xD7, 0xD8, 0x7B, 0xD8, 0x6F, /* 0x8C-0x8F */ + 0x00, 0x00, 0xD8, 0x66, 0xD8, 0x73, 0xD8, 0x6D, /* 0x90-0x93 */ + 0xB3, 0xE1, 0xD8, 0x79, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0xB3, 0xDD, 0xB3, 0xF1, 0xB3, 0xEA, 0x00, 0x00, /* 0x98-0x9B */ + 0xB3, 0xDF, 0xB3, 0xDC, 0x00, 0x00, 0xB3, 0xE7, /* 0x9C-0x9F */ + 0x00, 0x00, 0xD8, 0x7A, 0xD8, 0x6C, 0xD8, 0x72, /* 0xA0-0xA3 */ + 0xD8, 0x74, 0xD8, 0x68, 0xD8, 0x77, 0xB3, 0xD9, /* 0xA4-0xA7 */ + 0xD8, 0x67, 0x00, 0x00, 0xB3, 0xE0, 0xB3, 0xF0, /* 0xA8-0xAB */ + 0xB3, 0xEC, 0xD8, 0x69, 0xB3, 0xE6, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0xB3, 0xED, 0xB3, 0xE9, 0xB3, 0xE5, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xD8, 0x70, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB3, 0xEB, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDC, 0xD5, /* 0xBC-0xBF */ + 0xDC, 0xD1, 0x00, 0x00, 0xDC, 0xE0, 0xDC, 0xCA, /* 0xC0-0xC3 */ + 0xDC, 0xD3, 0xB6, 0xE5, 0xB6, 0xE6, 0xB6, 0xDE, /* 0xC4-0xC7 */ + 0xDC, 0xDC, 0xB6, 0xE8, 0xDC, 0xCF, 0xDC, 0xCE, /* 0xC8-0xCB */ + 0xDC, 0xCC, 0xDC, 0xDE, 0xB6, 0xDC, 0xDC, 0xD8, /* 0xCC-0xCF */ + 0xDC, 0xCD, 0xB6, 0xDF, 0xDC, 0xD6, 0xB6, 0xDA, /* 0xD0-0xD3 */ + 0xDC, 0xD2, 0xDC, 0xD9, 0xDC, 0xDB, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xDC, 0xDF, 0xB6, 0xE3, 0xDC, 0xCB, /* 0xD8-0xDB */ + 0xB6, 0xDD, 0xDC, 0xD0, 0x00, 0x00, 0xB6, 0xD8, /* 0xDC-0xDF */ + 0x00, 0x00, 0xB6, 0xE4, 0xDC, 0xDA, 0xB6, 0xE0, /* 0xE0-0xE3 */ + 0xB6, 0xE1, 0xB6, 0xE7, 0xB6, 0xDB, 0xA2, 0x5F, /* 0xE4-0xE7 */ + 0xB6, 0xD9, 0xDC, 0xD4, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB6, 0xE2, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0xDC, 0xDD, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0xB9, 0xCD, 0xB9, 0xC8, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xE1, 0x55, 0xE1, 0x51, 0x00, 0x00, /* 0xF8-0xFB */ + 0xE1, 0x4B, 0xB9, 0xC2, 0xB9, 0xBE, 0xE1, 0x54, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_56[512] = { + 0xB9, 0xBF, 0xE1, 0x4E, 0xE1, 0x50, 0x00, 0x00, /* 0x00-0x03 */ + 0xE1, 0x53, 0x00, 0x00, 0xB9, 0xC4, 0x00, 0x00, /* 0x04-0x07 */ + 0xB9, 0xCB, 0xB9, 0xC5, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0xE1, 0x49, 0xB9, 0xC6, 0xB9, 0xC7, 0xE1, 0x4C, /* 0x0C-0x0F */ + 0xB9, 0xCC, 0x00, 0x00, 0xE1, 0x4A, 0xE1, 0x4F, /* 0x10-0x13 */ + 0xB9, 0xC3, 0xE1, 0x48, 0xB9, 0xC9, 0xB9, 0xC1, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB9, 0xC0, /* 0x18-0x1B */ + 0xE1, 0x4D, 0xE1, 0x52, 0x00, 0x00, 0xB9, 0xCA, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE1, 0x47, /* 0x24-0x27 */ + 0x00, 0x00, 0xBC, 0x4D, 0xE5, 0x47, 0x00, 0x00, /* 0x28-0x2B */ + 0xE5, 0x44, 0x00, 0x00, 0xBC, 0x47, 0xBC, 0x53, /* 0x2C-0x2F */ + 0xBC, 0x54, 0x00, 0x00, 0xBC, 0x4A, 0xE5, 0x42, /* 0x30-0x33 */ + 0xBC, 0x4C, 0xE4, 0xF9, 0xBC, 0x52, 0x00, 0x00, /* 0x34-0x37 */ + 0xE5, 0x46, 0xBC, 0x49, 0xE5, 0x48, 0xBC, 0x48, /* 0x38-0x3B */ + 0x00, 0x00, 0xE5, 0x43, 0xE5, 0x45, 0xBC, 0x4B, /* 0x3C-0x3F */ + 0xE5, 0x41, 0xE4, 0xFA, 0xE4, 0xF7, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0xD8, 0x6B, 0xE4, 0xFD, 0x00, 0x00, /* 0x44-0x47 */ + 0xE4, 0xF6, 0xE4, 0xFC, 0xE4, 0xFB, 0x00, 0x00, /* 0x48-0x4B */ + 0xE4, 0xF8, 0x00, 0x00, 0xBC, 0x4F, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBC, 0x4E, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBC, 0x50, /* 0x54-0x57 */ + 0xE4, 0xFE, 0xBE, 0xB2, 0xE5, 0x40, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0x45, 0x00, 0x00, /* 0x5C-0x5F */ + 0xE8, 0xFD, 0x00, 0x00, 0xBE, 0xBE, 0xE9, 0x42, /* 0x60-0x63 */ + 0xBE, 0xB6, 0xBE, 0xBA, 0xE9, 0x41, 0x00, 0x00, /* 0x64-0x67 */ + 0xBE, 0xB9, 0xBE, 0xB5, 0xBE, 0xB8, 0xBE, 0xB3, /* 0x68-0x6B */ + 0xBE, 0xBD, 0xE9, 0x43, 0xE8, 0xFE, 0xBE, 0xBC, /* 0x6C-0x6F */ + 0xE8, 0xFC, 0xBE, 0xBB, 0xE9, 0x44, 0xE9, 0x40, /* 0x70-0x73 */ + 0xBC, 0x51, 0x00, 0x00, 0xBE, 0xBF, 0xE9, 0x46, /* 0x74-0x77 */ + 0xBE, 0xB7, 0xBE, 0xB4, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0xEC, 0xC6, 0xEC, 0xC8, /* 0x7C-0x7F */ + + 0xC0, 0x7B, 0xEC, 0xC9, 0xEC, 0xC7, 0xEC, 0xC5, /* 0x80-0x83 */ + 0xEC, 0xC4, 0xC0, 0x7D, 0xEC, 0xC3, 0xC0, 0x7E, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0xEC, 0xC1, 0xEC, 0xC2, 0xC0, 0x7A, 0xC0, 0xA1, /* 0x8C-0x8F */ + 0xC0, 0x7C, 0x00, 0x00, 0x00, 0x00, 0xEC, 0xC0, /* 0x90-0x93 */ + 0x00, 0x00, 0xC2, 0x50, 0x00, 0x00, 0xEF, 0xBC, /* 0x94-0x97 */ + 0xEF, 0xBA, 0xEF, 0xBF, 0xEF, 0xBD, 0x00, 0x00, /* 0x98-0x9B */ + 0xEF, 0xBB, 0xEF, 0xBE, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xC3, 0x60, 0xF1, 0xF2, 0xF1, 0xF3, /* 0xA4-0xA7 */ + 0xC4, 0x56, 0x00, 0x00, 0xF1, 0xF4, 0xF1, 0xF0, /* 0xA8-0xAB */ + 0xF1, 0xF5, 0xF1, 0xF1, 0xC2, 0x51, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xF3, 0xFE, 0xF4, 0x41, /* 0xB0-0xB3 */ + 0xC4, 0x59, 0xF4, 0x40, 0xC4, 0x58, 0xC4, 0x57, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0xC4, 0x5A, 0xF5, 0xC5, 0xF5, 0xC6, 0x00, 0x00, /* 0xBC-0xBF */ + 0xC4, 0xDA, 0xC4, 0xD9, 0xC4, 0xDB, 0xF5, 0xC4, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xF6, 0xD8, 0xF6, 0xD7, 0x00, 0x00, /* 0xC4-0xC7 */ + 0xC5, 0x6D, 0xC5, 0x6F, 0xC5, 0x6E, 0xF6, 0xD9, /* 0xC8-0xCB */ + 0xC5, 0xC8, 0xF8, 0xA6, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0xC5, 0xF1, 0x00, 0x00, 0xF8, 0xA5, /* 0xD0-0xD3 */ + 0xF8, 0xEE, 0x00, 0x00, 0x00, 0x00, 0xC9, 0x49, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xA5, 0x7D, 0xA5, 0x7C, /* 0xD8-0xDB */ + 0x00, 0x00, 0xA6, 0x5F, 0xA6, 0x5E, 0xC9, 0xC7, /* 0xDC-0xDF */ + 0xA6, 0x5D, 0xC9, 0xC6, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0xA7, 0x79, 0xCA, 0xA9, 0x00, 0x00, 0xCA, 0xA8, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0xA7, 0x77, 0xA7, 0x7A, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0xCA, 0xA7, 0x00, 0x00, /* 0xEC-0xEF */ + 0xA7, 0x78, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCB, 0xF0, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xCB, 0xF1, 0xA9, 0x54, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAB, 0xAA, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_57[512] = { + 0x00, 0x00, 0xD1, 0x48, 0xD1, 0x49, 0xAE, 0x45, /* 0x00-0x03 */ + 0xAE, 0x46, 0x00, 0x00, 0x00, 0x00, 0xD4, 0xAC, /* 0x04-0x07 */ + 0xB0, 0xE9, 0xB0, 0xEB, 0xD4, 0xAB, 0xB0, 0xEA, /* 0x08-0x0B */ + 0xD8, 0x7C, 0xB3, 0xF2, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0xB6, 0xE9, 0xB6, 0xEA, /* 0x10-0x13 */ + 0xDC, 0xE1, 0x00, 0x00, 0xB9, 0xCF, 0x00, 0x00, /* 0x14-0x17 */ + 0xB9, 0xCE, 0x00, 0x00, 0xE5, 0x49, 0xE9, 0x48, /* 0x18-0x1B */ + 0xE9, 0x47, 0x00, 0x00, 0xF9, 0x6B, 0xA4, 0x67, /* 0x1C-0x1F */ + 0xC9, 0x59, 0x00, 0x00, 0xC9, 0x6E, 0xC9, 0x6F, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0xA6, 0x62, 0xA6, 0x66, 0xC9, 0xC9, 0x00, 0x00, /* 0x28-0x2B */ + 0xA6, 0x64, 0xA6, 0x63, 0xC9, 0xC8, 0xA6, 0x65, /* 0x2C-0x2F */ + 0xA6, 0x61, 0x00, 0x00, 0x00, 0x00, 0xA6, 0x60, /* 0x30-0x33 */ + 0xC9, 0xCA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA7, 0xA6, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xA7, 0xA3, 0x00, 0x00, /* 0x3C-0x3F */ + 0xA7, 0x7D, 0xCA, 0xAA, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0xCA, 0xAB, 0x00, 0x00, 0xA7, 0xA1, /* 0x44-0x47 */ + 0x00, 0x00, 0xCA, 0xAD, 0xA7, 0x7B, 0xCA, 0xAE, /* 0x48-0x4B */ + 0xCA, 0xAC, 0xA7, 0x7E, 0xA7, 0xA2, 0xA7, 0xA5, /* 0x4C-0x4F */ + 0xA7, 0xA4, 0xA7, 0x7C, 0xCA, 0xAF, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0xA9, 0x59, 0xCB, 0xFE, 0x00, 0x00, /* 0x60-0x63 */ + 0xA9, 0x5B, 0x00, 0x00, 0xA9, 0x5A, 0x00, 0x00, /* 0x64-0x67 */ + 0xCC, 0x40, 0xA9, 0x58, 0xA9, 0x57, 0xCB, 0xF5, /* 0x68-0x6B */ + 0x00, 0x00, 0xCB, 0xF4, 0x00, 0x00, 0xCB, 0xF2, /* 0x6C-0x6F */ + 0xCB, 0xF7, 0xCB, 0xF6, 0xCB, 0xF3, 0xCB, 0xFC, /* 0x70-0x73 */ + 0xCB, 0xFD, 0xCB, 0xFA, 0xCB, 0xF8, 0xA9, 0x56, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCB, 0xFB, /* 0x78-0x7B */ + 0xA9, 0x5C, 0xCC, 0x41, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xCB, 0xF9, 0x00, 0x00, 0xAB, 0xAB, 0xA9, 0x55, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAB, 0xAC, /* 0x88-0x8B */ + 0xCE, 0x54, 0x00, 0x00, 0x00, 0x00, 0xCE, 0x5A, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAB, 0xB2, /* 0x90-0x93 */ + 0xCE, 0x58, 0xCE, 0x5E, 0x00, 0x00, 0xCE, 0x55, /* 0x94-0x97 */ + 0xCE, 0x59, 0xCE, 0x5B, 0xCE, 0x5D, 0xCE, 0x57, /* 0x98-0x9B */ + 0x00, 0x00, 0xCE, 0x56, 0xCE, 0x51, 0xCE, 0x52, /* 0x9C-0x9F */ + 0xAB, 0xAD, 0x00, 0x00, 0xAB, 0xAF, 0xAB, 0xAE, /* 0xA0-0xA3 */ + 0xCE, 0x53, 0xCE, 0x5C, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0xAB, 0xB1, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xCE, 0x50, 0xD1, 0x53, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xD1, 0x52, 0xD1, 0x57, 0xD1, 0x4E, 0x00, 0x00, /* 0xB8-0xBB */ + 0xD1, 0x51, 0xD1, 0x50, 0x00, 0x00, 0xD1, 0x54, /* 0xBC-0xBF */ + 0x00, 0x00, 0xD1, 0x58, 0xAE, 0x47, 0xAE, 0x4A, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0xD1, 0x4F, 0xD1, 0x55, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAE, 0x49, /* 0xC8-0xCB */ + 0xD1, 0x4A, 0x00, 0x00, 0xAB, 0xB0, 0xD4, 0xBA, /* 0xCC-0xCF */ + 0xD1, 0x56, 0x00, 0x00, 0xD1, 0x4D, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xAE, 0x48, 0xD1, 0x4C, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0xD4, 0xB1, 0x00, 0x00, 0x00, 0x00, 0xB0, 0xEC, /* 0xDC-0xDF */ + 0xB0, 0xF0, 0xD4, 0xC1, 0xD4, 0xAF, 0xD4, 0xBD, /* 0xE0-0xE3 */ + 0xB0, 0xF1, 0xD4, 0xBF, 0x00, 0x00, 0xD4, 0xC5, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xD4, 0xC9, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0xD4, 0xC0, 0xD4, 0xB4, 0xD4, 0xBC, 0x00, 0x00, /* 0xEC-0xEF */ + 0xD4, 0xCA, 0xD4, 0xC8, 0xD4, 0xBE, 0xD4, 0xB9, /* 0xF0-0xF3 */ + 0xD4, 0xB2, 0xD8, 0xA6, 0xD4, 0xB0, 0xB0, 0xF5, /* 0xF4-0xF7 */ + 0xD4, 0xB7, 0xB0, 0xF6, 0xB0, 0xF2, 0xD4, 0xAD, /* 0xF8-0xFB */ + 0xD4, 0xC3, 0xD4, 0xB5, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_58[512] = { + 0xD4, 0xB3, 0xD4, 0xC6, 0xB0, 0xF3, 0x00, 0x00, /* 0x00-0x03 */ + 0xD4, 0xCC, 0xB0, 0xED, 0xB0, 0xEF, 0xD4, 0xBB, /* 0x04-0x07 */ + 0xD4, 0xB6, 0xAE, 0x4B, 0xB0, 0xEE, 0xD4, 0xB8, /* 0x08-0x0B */ + 0xD4, 0xC7, 0xD4, 0xCB, 0xD4, 0xC2, 0x00, 0x00, /* 0x0C-0x0F */ + 0xD4, 0xC4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0xD4, 0xAE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0xD8, 0xA1, 0x00, 0x00, 0xD8, 0xAA, /* 0x18-0x1B */ + 0xD8, 0xA9, 0xB3, 0xFA, 0xD8, 0xA2, 0x00, 0x00, /* 0x1C-0x1F */ + 0xB3, 0xFB, 0xB3, 0xF9, 0x00, 0x00, 0xD8, 0xA4, /* 0x20-0x23 */ + 0xB3, 0xF6, 0xD8, 0xA8, 0x00, 0x00, 0xD8, 0xA3, /* 0x24-0x27 */ + 0xD8, 0xA5, 0xD8, 0x7D, 0xB3, 0xF4, 0x00, 0x00, /* 0x28-0x2B */ + 0xD8, 0xB2, 0xD8, 0xB1, 0xD8, 0xAE, 0xB3, 0xF3, /* 0x2C-0x2F */ + 0xB3, 0xF7, 0xB3, 0xF8, 0xD1, 0x4B, 0xD8, 0xAB, /* 0x30-0x33 */ + 0xB3, 0xF5, 0xB0, 0xF4, 0xD8, 0xAD, 0xD8, 0x7E, /* 0x34-0x37 */ + 0xD8, 0xB0, 0xD8, 0xAF, 0x00, 0x00, 0xD8, 0xB3, /* 0x38-0x3B */ + 0x00, 0x00, 0xDC, 0xEF, 0x00, 0x00, 0xD8, 0xAC, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0xD8, 0xA7, 0xDC, 0xE7, 0xB6, 0xF4, 0xB6, 0xF7, /* 0x48-0x4B */ + 0xB6, 0xF2, 0xDC, 0xE6, 0xDC, 0xEA, 0xDC, 0xE5, /* 0x4C-0x4F */ + 0x00, 0x00, 0xB6, 0xEC, 0xB6, 0xF6, 0xDC, 0xE2, /* 0x50-0x53 */ + 0xB6, 0xF0, 0xDC, 0xE9, 0x00, 0x00, 0xB6, 0xEE, /* 0x54-0x57 */ + 0xB6, 0xED, 0xDC, 0xEC, 0xB6, 0xEF, 0xDC, 0xEE, /* 0x58-0x5B */ + 0x00, 0x00, 0xDC, 0xEB, 0xB6, 0xEB, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0xB6, 0xF5, 0xDC, 0xF0, /* 0x60-0x63 */ + 0xDC, 0xE4, 0xDC, 0xED, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0xDC, 0xE3, 0x00, 0x00, 0x00, 0x00, 0xB6, 0xF1, /* 0x68-0x6B */ + 0x00, 0x00, 0xB6, 0xF3, 0x00, 0x00, 0xDC, 0xE8, /* 0x6C-0x6F */ + 0x00, 0x00, 0xDC, 0xF1, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0xE1, 0x5D, 0xB9, 0xD0, 0xE1, 0x63, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0xB9, 0xD5, 0xE1, 0x5F, 0xE1, 0x66, /* 0x78-0x7B */ + 0xE1, 0x57, 0xB9, 0xD7, 0xB9, 0xD1, 0xE1, 0x5C, /* 0x7C-0x7F */ + + 0xBC, 0x55, 0xE1, 0x5B, 0xE1, 0x64, 0xB9, 0xD2, /* 0x80-0x83 */ + 0x00, 0x00, 0xB9, 0xD6, 0xE1, 0x5A, 0xE1, 0x60, /* 0x84-0x87 */ + 0xE1, 0x65, 0xE1, 0x56, 0xB9, 0xD4, 0xE1, 0x5E, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0x62, 0xE1, 0x68, /* 0x8C-0x8F */ + 0xE1, 0x58, 0xE1, 0x61, 0x00, 0x00, 0xB9, 0xD3, /* 0x90-0x93 */ + 0xE1, 0x67, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0xE1, 0x59, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0xBC, 0x59, 0xE5, 0x4B, 0xBC, 0x57, 0xBC, 0x56, /* 0x9C-0x9F */ + 0xE5, 0x4D, 0xE5, 0x52, 0x00, 0x00, 0xE5, 0x4E, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xE5, 0x51, 0xBC, 0x5C, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xBE, 0xA5, 0xBC, 0x5B, 0x00, 0x00, 0xE5, 0x4A, /* 0xA8-0xAB */ + 0xE5, 0x50, 0x00, 0x00, 0xBC, 0x5A, 0xE5, 0x4F, /* 0xAC-0xAF */ + 0x00, 0x00, 0xE5, 0x4C, 0x00, 0x00, 0xBC, 0x58, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0x4D, 0xF9, 0xD9, /* 0xB8-0xBB */ + 0xE9, 0x4F, 0xE9, 0x4A, 0xBE, 0xC1, 0xE9, 0x4C, /* 0xBC-0xBF */ + 0x00, 0x00, 0xBE, 0xC0, 0xE9, 0x4E, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xBE, 0xC3, 0xE9, 0x50, 0xBE, 0xC2, /* 0xC4-0xC7 */ + 0xE9, 0x49, 0xE9, 0x4B, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0xC0, 0xA5, 0xEC, 0xCC, /* 0xCC-0xCF */ + 0x00, 0x00, 0xC0, 0xA4, 0xEC, 0xCD, 0xC0, 0xA3, /* 0xD0-0xD3 */ + 0xEC, 0xCB, 0xC0, 0xA2, 0xEC, 0xCA, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xC2, 0x53, 0xC2, 0x52, 0xF1, 0xF6, 0xF1, 0xF8, /* 0xD8-0xDB */ + 0x00, 0x00, 0xF1, 0xF7, 0xC3, 0x61, 0xC3, 0x62, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0xC3, 0x63, 0xF4, 0x42, /* 0xE0-0xE3 */ + 0xC4, 0x5B, 0x00, 0x00, 0x00, 0x00, 0xF7, 0xD3, /* 0xE4-0xE7 */ + 0xF7, 0xD2, 0xC5, 0xF2, 0x00, 0x00, 0xA4, 0x68, /* 0xE8-0xEB */ + 0xA4, 0xD0, 0x00, 0x00, 0x00, 0x00, 0xA7, 0xA7, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xCE, 0x5F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xB3, 0xFC, 0xB3, 0xFD, 0x00, 0x00, /* 0xF8-0xFB */ + 0xDC, 0xF2, 0xB9, 0xD8, 0xE1, 0x69, 0xE5, 0x53, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_59[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC9, 0x5A, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0xCA, 0xB0, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0xCC, 0x42, 0xCE, 0x60, 0xD1, 0x59, 0xAE, 0x4C, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0xF1, 0xF9, 0x00, 0x00, /* 0x10-0x13 */ + 0xC4, 0xDC, 0xA4, 0x69, 0xA5, 0x7E, 0xC9, 0x70, /* 0x14-0x17 */ + 0x00, 0x00, 0xA6, 0x67, 0xA6, 0x68, 0x00, 0x00, /* 0x18-0x1B */ + 0xA9, 0x5D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0xB0, 0xF7, 0x00, 0x00, 0xB9, 0xDA, 0x00, 0x00, /* 0x20-0x23 */ + 0xB9, 0xDB, 0xB9, 0xD9, 0x00, 0x00, 0xA4, 0x6A, /* 0x24-0x27 */ + 0x00, 0x00, 0xA4, 0xD1, 0xA4, 0xD3, 0xA4, 0xD2, /* 0x28-0x2B */ + 0xC9, 0x5B, 0xA4, 0xD4, 0xA5, 0xA1, 0xC9, 0x71, /* 0x2C-0x2F */ + 0x00, 0x00, 0xA5, 0xA2, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA6, 0x69, /* 0x34-0x37 */ + 0xA6, 0x6A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0xC9, 0xCB, 0x00, 0x00, 0xA7, 0xA8, 0x00, 0x00, /* 0x3C-0x3F */ + 0xCA, 0xB1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0xA9, 0x61, 0xCC, 0x43, 0x00, 0x00, 0xA9, 0x5F, /* 0x44-0x47 */ + 0xA9, 0x60, 0xA9, 0x5E, 0xD1, 0x5A, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0xAB, 0xB6, 0xAB, 0xB5, /* 0x4C-0x4F */ + 0xAB, 0xB7, 0xAB, 0xB4, 0x00, 0x00, 0xCE, 0x61, /* 0x50-0x53 */ + 0xA9, 0x62, 0xAB, 0xB3, 0x00, 0x00, 0xAE, 0x4D, /* 0x54-0x57 */ + 0xAE, 0x4E, 0x00, 0x00, 0xAE, 0x4F, 0x00, 0x00, /* 0x58-0x5B */ + 0xD4, 0xCD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0xB3, 0xFE, 0xD8, 0xB4, 0xB0, 0xF8, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB6, 0xF8, /* 0x64-0x67 */ + 0x00, 0x00, 0xB9, 0xDD, 0xB9, 0xDC, 0xE1, 0x6A, /* 0x68-0x6B */ + 0x00, 0x00, 0xBC, 0x5D, 0xBE, 0xC4, 0x00, 0x00, /* 0x6C-0x6F */ + 0xEF, 0xC0, 0xF6, 0xDA, 0xF7, 0xD4, 0xA4, 0x6B, /* 0x70-0x73 */ + 0xA5, 0xA3, 0x00, 0x00, 0xA5, 0xA4, 0xC9, 0xD1, /* 0x74-0x77 */ + 0xA6, 0x6C, 0xA6, 0x6F, 0x00, 0x00, 0xC9, 0xCF, /* 0x78-0x7B */ + 0xC9, 0xCD, 0xA6, 0x6E, 0xC9, 0xD0, 0xC9, 0xD2, /* 0x7C-0x7F */ + + 0xC9, 0xCC, 0xA6, 0x71, 0xA6, 0x70, 0xA6, 0x6D, /* 0x80-0x83 */ + 0xA6, 0x6B, 0xC9, 0xCE, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0xA7, 0xB3, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0xA7, 0xB0, 0xCA, 0xB6, 0xCA, 0xB9, /* 0x8C-0x8F */ + 0xCA, 0xB8, 0x00, 0x00, 0xA7, 0xAA, 0xA7, 0xB2, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0xA7, 0xAF, 0xCA, 0xB5, /* 0x94-0x97 */ + 0xCA, 0xB3, 0xA7, 0xAE, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0xA7, 0xA9, 0xA7, 0xAC, 0x00, 0x00, /* 0x9C-0x9F */ + 0xCA, 0xB4, 0xCA, 0xBB, 0xCA, 0xB7, 0xA7, 0xAD, /* 0xA0-0xA3 */ + 0xA7, 0xB1, 0xA7, 0xB4, 0xCA, 0xB2, 0xCA, 0xBA, /* 0xA4-0xA7 */ + 0xA7, 0xAB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0xA9, 0x67, 0xA9, 0x6F, /* 0xAC-0xAF */ + 0x00, 0x00, 0xCC, 0x4F, 0xCC, 0x48, 0xA9, 0x70, /* 0xB0-0xB3 */ + 0xCC, 0x53, 0xCC, 0x44, 0xCC, 0x4B, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xA9, 0x66, 0xCC, 0x45, 0xA9, 0x64, /* 0xB8-0xBB */ + 0xCC, 0x4C, 0xCC, 0x50, 0xA9, 0x63, 0x00, 0x00, /* 0xBC-0xBF */ + 0xCC, 0x51, 0xCC, 0x4A, 0x00, 0x00, 0xCC, 0x4D, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xA9, 0x72, 0xA9, 0x69, 0xCC, 0x54, /* 0xC4-0xC7 */ + 0xCC, 0x52, 0x00, 0x00, 0xA9, 0x6E, 0xA9, 0x6C, /* 0xC8-0xCB */ + 0xCC, 0x49, 0xA9, 0x6B, 0xCC, 0x47, 0xCC, 0x46, /* 0xCC-0xCF */ + 0xA9, 0x6A, 0xA9, 0x68, 0xA9, 0x71, 0xA9, 0x6D, /* 0xD0-0xD3 */ + 0xA9, 0x65, 0x00, 0x00, 0xCC, 0x4E, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xAB, 0xB9, 0x00, 0x00, 0xAB, 0xC0, 0xCE, 0x6F, /* 0xD8-0xDB */ + 0xAB, 0xB8, 0xCE, 0x67, 0xCE, 0x63, 0x00, 0x00, /* 0xDC-0xDF */ + 0xCE, 0x73, 0xCE, 0x62, 0x00, 0x00, 0xAB, 0xBB, /* 0xE0-0xE3 */ + 0xCE, 0x6C, 0xAB, 0xBE, 0xAB, 0xC1, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xAB, 0xBC, 0xCE, 0x70, 0xAB, 0xBF, 0x00, 0x00, /* 0xE8-0xEB */ + 0xAE, 0x56, 0xCE, 0x76, 0xCE, 0x64, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0xCE, 0x66, 0xCE, 0x6D, 0xCE, 0x71, /* 0xF0-0xF3 */ + 0xCE, 0x75, 0xCE, 0x72, 0xCE, 0x6B, 0xCE, 0x6E, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0xCE, 0x68, 0xAB, 0xC3, /* 0xF8-0xFB */ + 0xCE, 0x6A, 0xCE, 0x69, 0xCE, 0x74, 0xAB, 0xBA, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_5A[512] = { + 0xCE, 0x65, 0xAB, 0xC2, 0x00, 0x00, 0xAB, 0xBD, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xAE, 0x5C, 0xD1, 0x62, 0x00, 0x00, /* 0x08-0x0B */ + 0xAE, 0x5B, 0x00, 0x00, 0x00, 0x00, 0xD1, 0x60, /* 0x0C-0x0F */ + 0x00, 0x00, 0xAE, 0x50, 0x00, 0x00, 0xAE, 0x55, /* 0x10-0x13 */ + 0x00, 0x00, 0xD1, 0x5F, 0xD1, 0x5C, 0xD1, 0x61, /* 0x14-0x17 */ + 0xAE, 0x51, 0xD1, 0x5B, 0x00, 0x00, 0xAE, 0x54, /* 0x18-0x1B */ + 0xAE, 0x52, 0x00, 0x00, 0xD1, 0x63, 0xAE, 0x53, /* 0x1C-0x1F */ + 0xAE, 0x57, 0x00, 0x00, 0x00, 0x00, 0xAE, 0x58, /* 0x20-0x23 */ + 0x00, 0x00, 0xAE, 0x5A, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0xAE, 0x59, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0xD1, 0x5D, 0xD1, 0x5E, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD1, 0x64, /* 0x30-0x33 */ + 0x00, 0x00, 0xD4, 0xD4, 0xB0, 0xF9, 0xD8, 0xC2, /* 0x34-0x37 */ + 0xD4, 0xD3, 0xD4, 0xE6, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0xB1, 0x40, 0x00, 0x00, 0xD4, 0xE4, 0x00, 0x00, /* 0x3C-0x3F */ + 0xB0, 0xFE, 0xB0, 0xFA, 0xD4, 0xED, 0xD4, 0xDD, /* 0x40-0x43 */ + 0xD4, 0xE0, 0x00, 0x00, 0xB1, 0x43, 0xD4, 0xEA, /* 0x44-0x47 */ + 0xD4, 0xE2, 0xB0, 0xFB, 0xB1, 0x44, 0x00, 0x00, /* 0x48-0x4B */ + 0xD4, 0xE7, 0xD4, 0xE5, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0xD4, 0xD6, 0xD4, 0xEB, 0xD4, 0xDF, 0xD4, 0xDA, /* 0x50-0x53 */ + 0x00, 0x00, 0xD4, 0xD0, 0xD4, 0xEC, 0xD4, 0xDC, /* 0x54-0x57 */ + 0xD4, 0xCF, 0x00, 0x00, 0xB1, 0x42, 0xD4, 0xE1, /* 0x58-0x5B */ + 0xD4, 0xEE, 0xD4, 0xDE, 0xD4, 0xD2, 0xD4, 0xD7, /* 0x5C-0x5F */ + 0xD4, 0xCE, 0x00, 0x00, 0xB1, 0x41, 0x00, 0x00, /* 0x60-0x63 */ + 0xD4, 0xDB, 0xD4, 0xD8, 0xB0, 0xFC, 0xD4, 0xD1, /* 0x64-0x67 */ + 0x00, 0x00, 0xD4, 0xE9, 0xB0, 0xFD, 0x00, 0x00, /* 0x68-0x6B */ + 0xD4, 0xD9, 0xD4, 0xD5, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0xD4, 0xE8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB4, 0x40, /* 0x74-0x77 */ + 0xD8, 0xBB, 0x00, 0x00, 0xD8, 0xB8, 0xD8, 0xC9, /* 0x78-0x7B */ + 0xD8, 0xBD, 0xD8, 0xCA, 0x00, 0x00, 0xB4, 0x42, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD8, 0xC6, /* 0x80-0x83 */ + 0xD8, 0xC3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0xD8, 0xC4, 0xD8, 0xC7, /* 0x88-0x8B */ + 0xD8, 0xCB, 0x00, 0x00, 0xD4, 0xE3, 0xD8, 0xCD, /* 0x8C-0x8F */ + 0xDD, 0x47, 0x00, 0x00, 0xB4, 0x43, 0xD8, 0xCE, /* 0x90-0x93 */ + 0xD8, 0xB6, 0xD8, 0xC0, 0x00, 0x00, 0xD8, 0xC5, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0xB4, 0x41, 0xB4, 0x44, /* 0x98-0x9B */ + 0xD8, 0xCC, 0xD8, 0xCF, 0xD8, 0xBA, 0xD8, 0xB7, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0xD8, 0xB9, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xD8, 0xBE, 0xD8, 0xBC, 0xB4, 0x45, /* 0xA4-0xA7 */ + 0x00, 0x00, 0xD8, 0xC8, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0xD8, 0xBF, 0x00, 0x00, 0xD8, 0xC1, 0xD8, 0xB5, /* 0xAC-0xAF */ + 0xDC, 0xFA, 0xDC, 0xF8, 0xB7, 0x42, 0xB7, 0x40, /* 0xB0-0xB3 */ + 0xDD, 0x43, 0xDC, 0xF9, 0xDD, 0x44, 0xDD, 0x40, /* 0xB4-0xB7 */ + 0xDC, 0xF7, 0xDD, 0x46, 0xDC, 0xF6, 0xDC, 0xFD, /* 0xB8-0xBB */ + 0xB6, 0xFE, 0xB6, 0xFD, 0xB6, 0xFC, 0xDC, 0xFB, /* 0xBC-0xBF */ + 0xDD, 0x41, 0xB6, 0xF9, 0xB7, 0x41, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xDC, 0xF4, 0x00, 0x00, 0xDC, 0xFE, 0xDC, 0xF3, /* 0xC4-0xC7 */ + 0xDC, 0xFC, 0xB6, 0xFA, 0xDD, 0x42, 0xDC, 0xF5, /* 0xC8-0xCB */ + 0xB6, 0xFB, 0xDD, 0x45, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xE1, 0x6E, 0xB9, 0xE2, 0xB9, 0xE1, /* 0xD4-0xD7 */ + 0xB9, 0xE3, 0xE1, 0x7A, 0xE1, 0x70, 0xE1, 0x76, /* 0xD8-0xDB */ + 0xE1, 0x6B, 0xE1, 0x79, 0xE1, 0x78, 0xE1, 0x7C, /* 0xDC-0xDF */ + 0xE1, 0x75, 0xB9, 0xDE, 0xE1, 0x74, 0xB9, 0xE4, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xE1, 0x6D, 0xB9, 0xDF, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xE1, 0x7B, 0xB9, 0xE0, 0xE1, 0x6F, 0xE1, 0x72, /* 0xE8-0xEB */ + 0xE1, 0x77, 0xE1, 0x71, 0xE1, 0x6C, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE1, 0x73, /* 0xF0-0xF3 */ + 0xE5, 0x55, 0xBC, 0x61, 0xE5, 0x58, 0xE5, 0x57, /* 0xF4-0xF7 */ + 0xE5, 0x5A, 0xE5, 0x5C, 0xF9, 0xDC, 0xBC, 0x5F, /* 0xF8-0xFB */ + 0x00, 0x00, 0xE5, 0x56, 0x00, 0x00, 0xE5, 0x54, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_5B[512] = { + 0x00, 0x00, 0xE5, 0x5D, 0xE5, 0x5B, 0xE5, 0x59, /* 0x00-0x03 */ + 0x00, 0x00, 0xE5, 0x5F, 0x00, 0x00, 0xE5, 0x5E, /* 0x04-0x07 */ + 0xBC, 0x63, 0xBC, 0x5E, 0x00, 0x00, 0xBC, 0x60, /* 0x08-0x0B */ + 0xBC, 0x62, 0x00, 0x00, 0x00, 0x00, 0xE5, 0x60, /* 0x0C-0x0F */ + 0xE9, 0x57, 0x00, 0x00, 0x00, 0x00, 0xE9, 0x56, /* 0x10-0x13 */ + 0xE9, 0x55, 0x00, 0x00, 0xE9, 0x58, 0xE9, 0x51, /* 0x14-0x17 */ + 0x00, 0x00, 0xE9, 0x52, 0xE9, 0x5A, 0xE9, 0x53, /* 0x18-0x1B */ + 0x00, 0x00, 0xBE, 0xC5, 0xE9, 0x5C, 0x00, 0x00, /* 0x1C-0x1F */ + 0xE9, 0x5B, 0xE9, 0x54, 0x00, 0x00, 0xEC, 0xD1, /* 0x20-0x23 */ + 0xC0, 0xA8, 0xEC, 0xCF, 0xEC, 0xD4, 0xEC, 0xD3, /* 0x24-0x27 */ + 0xE9, 0x59, 0x00, 0x00, 0xC0, 0xA7, 0x00, 0x00, /* 0x28-0x2B */ + 0xEC, 0xD2, 0xEC, 0xCE, 0xEC, 0xD6, 0xEC, 0xD5, /* 0x2C-0x2F */ + 0xC0, 0xA6, 0x00, 0x00, 0xEC, 0xD0, 0x00, 0x00, /* 0x30-0x33 */ + 0xBE, 0xC6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0xC2, 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0xEF, 0xC1, 0xF1, 0xFA, 0xF1, 0xFB, 0xF1, 0xFC, /* 0x3C-0x3F */ + 0xC4, 0x5C, 0x00, 0x00, 0x00, 0x00, 0xC4, 0x5D, /* 0x40-0x43 */ + 0x00, 0x00, 0xF4, 0x43, 0x00, 0x00, 0xF5, 0xC8, /* 0x44-0x47 */ + 0xF5, 0xC7, 0x00, 0x00, 0x00, 0x00, 0xF6, 0xDB, /* 0x48-0x4B */ + 0xF6, 0xDC, 0xF7, 0xD5, 0xF8, 0xA7, 0x00, 0x00, /* 0x4C-0x4F */ + 0xA4, 0x6C, 0xA4, 0x6D, 0x00, 0x00, 0xA4, 0x6E, /* 0x50-0x53 */ + 0xA4, 0xD5, 0xA5, 0xA5, 0xC9, 0xD3, 0xA6, 0x72, /* 0x54-0x57 */ + 0xA6, 0x73, 0x00, 0x00, 0xA7, 0xB7, 0xA7, 0xB8, /* 0x58-0x5B */ + 0xA7, 0xB6, 0xA7, 0xB5, 0x00, 0x00, 0xA9, 0x73, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0xCC, 0x55, 0xA9, 0x75, /* 0x60-0x63 */ + 0xA9, 0x74, 0xCC, 0x56, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0xAB, 0xC4, 0x00, 0x00, 0xAE, 0x5D, /* 0x68-0x6B */ + 0xD1, 0x65, 0x00, 0x00, 0xD4, 0xF0, 0x00, 0x00, /* 0x6C-0x6F */ + 0xB1, 0x45, 0xB4, 0x47, 0xD4, 0xEF, 0xB4, 0x46, /* 0x70-0x73 */ + 0x00, 0x00, 0xB9, 0xE5, 0x00, 0x00, 0xE1, 0x7D, /* 0x74-0x77 */ + 0xBE, 0xC7, 0x00, 0x00, 0xC0, 0xA9, 0xEC, 0xD7, /* 0x78-0x7B */ + 0x00, 0x00, 0xC4, 0x5E, 0x00, 0x00, 0xC5, 0x70, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xC9, 0x72, 0x00, 0x00, 0xA5, 0xA6, /* 0x80-0x83 */ + 0xC9, 0x73, 0xA6, 0x76, 0x00, 0x00, 0xA6, 0x74, /* 0x84-0x87 */ + 0xA6, 0x75, 0xA6, 0x77, 0x00, 0x00, 0xA7, 0xBA, /* 0x88-0x8B */ + 0xA7, 0xB9, 0x00, 0x00, 0xCA, 0xBC, 0xA7, 0xBB, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0xCA, 0xBD, 0xCC, 0x57, /* 0x90-0x93 */ + 0x00, 0x00, 0xCC, 0x58, 0x00, 0x00, 0xA9, 0x76, /* 0x94-0x97 */ + 0xA9, 0x78, 0xA9, 0x7A, 0xA9, 0x77, 0xA9, 0x7B, /* 0x98-0x9B */ + 0xA9, 0x79, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0xAB, 0xC8, 0xAB, 0xC5, /* 0xA0-0xA3 */ + 0xAB, 0xC7, 0xAB, 0xC9, 0xAB, 0xC6, 0xD1, 0x66, /* 0xA4-0xA7 */ + 0xCE, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0xD1, 0x68, 0xD1, 0x67, 0xAE, 0x63, 0x00, 0x00, /* 0xAC-0xAF */ + 0xAE, 0x5F, 0x00, 0x00, 0x00, 0x00, 0xAE, 0x60, /* 0xB0-0xB3 */ + 0xAE, 0x62, 0xAE, 0x64, 0xAE, 0x61, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xAE, 0x66, 0xAE, 0x65, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB1, 0x4A, /* 0xBC-0xBF */ + 0xD4, 0xF2, 0xD4, 0xF1, 0xB1, 0x49, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xB1, 0x48, 0xB1, 0x47, 0xB1, 0x4B, 0xB1, 0x46, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0xD8, 0xD5, 0xD8, 0xD2, /* 0xC8-0xCB */ + 0xB4, 0x49, 0xD8, 0xD1, 0xD8, 0xD6, 0x00, 0x00, /* 0xCC-0xCF */ + 0xB4, 0x4B, 0xD8, 0xD4, 0xB4, 0x48, 0xB4, 0x4A, /* 0xD0-0xD3 */ + 0xD8, 0xD3, 0x00, 0x00, 0xDD, 0x48, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xDD, 0x49, 0xDD, 0x4A, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0xB9, 0xE6, 0xB9, 0xEE, /* 0xDC-0xDF */ + 0xE1, 0x7E, 0xB9, 0xE8, 0xB9, 0xEC, 0xE1, 0xA1, /* 0xE0-0xE3 */ + 0xB9, 0xED, 0xB9, 0xE9, 0xB9, 0xEA, 0xB9, 0xE7, /* 0xE4-0xE7 */ + 0xB9, 0xEB, 0xBC, 0x66, 0xD8, 0xD0, 0xBC, 0x67, /* 0xE8-0xEB */ + 0xBC, 0x65, 0x00, 0x00, 0xBC, 0x64, 0xE9, 0x5D, /* 0xEC-0xEF */ + 0xBE, 0xC8, 0xEC, 0xD8, 0xEC, 0xD9, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0xC3, 0x64, 0xC4, 0x5F, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xA4, 0x6F, 0x00, 0x00, 0xA6, 0x78, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_5C[512] = { + 0x00, 0x00, 0xAB, 0xCA, 0x00, 0x00, 0xD1, 0x69, /* 0x00-0x03 */ + 0xAE, 0x67, 0x00, 0x00, 0x00, 0x00, 0xB1, 0x4E, /* 0x04-0x07 */ + 0xB1, 0x4D, 0xB1, 0x4C, 0xB4, 0x4C, 0xB4, 0x4D, /* 0x08-0x0B */ + 0xD8, 0xD7, 0xB9, 0xEF, 0xBE, 0xC9, 0xA4, 0x70, /* 0x0C-0x0F */ + 0xC9, 0x5C, 0xA4, 0xD6, 0xC9, 0x74, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0xC9, 0xD4, 0xA6, 0x79, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0xA9, 0x7C, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDD, 0x4B, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0xA4, 0x71, 0x00, 0x00, /* 0x20-0x23 */ + 0xA4, 0xD7, 0xC9, 0xD5, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0xCA, 0xBE, 0x00, 0x00, 0xCA, 0xBF, 0x00, 0x00, /* 0x28-0x2B */ + 0xA7, 0xBC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xD8, 0xD8, 0xB4, 0x4E, 0x00, 0x00, 0xDD, 0x4C, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC0, 0xAA, /* 0x34-0x37 */ + 0xA4, 0x72, 0xA4, 0xA8, 0xA4, 0xD8, 0xC9, 0x75, /* 0x38-0x3B */ + 0xA5, 0xA7, 0x00, 0x00, 0xA7, 0xC0, 0xA7, 0xBF, /* 0x3C-0x3F */ + 0xA7, 0xBD, 0xA7, 0xBE, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0xCC, 0x59, 0xA9, 0x7E, 0xA9, 0xA1, 0xCC, 0x5A, /* 0x44-0x47 */ + 0xA9, 0x7D, 0x00, 0x00, 0x00, 0x00, 0xAB, 0xCE, /* 0x48-0x4B */ + 0xCE, 0x78, 0xAB, 0xCD, 0xAB, 0xCB, 0xAB, 0xCC, /* 0x4C-0x4F */ + 0xAE, 0x6A, 0xAE, 0x68, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0xD1, 0x6B, 0xAE, 0x69, 0xD1, 0x6A, 0x00, 0x00, /* 0x54-0x57 */ + 0xAE, 0x5E, 0xD4, 0xF3, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0xB1, 0x50, 0xB1, 0x51, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0xB1, 0x4F, 0x00, 0x00, 0xB9, 0xF0, 0xE1, 0xA2, /* 0x60-0x63 */ + 0xBC, 0x68, 0xBC, 0x69, 0x00, 0x00, 0xE5, 0x61, /* 0x64-0x67 */ + 0xC0, 0xAB, 0xEF, 0xC2, 0xEF, 0xC3, 0x00, 0x00, /* 0x68-0x6B */ + 0xC4, 0xDD, 0xF8, 0xA8, 0xC9, 0x4B, 0xA4, 0xD9, /* 0x6C-0x6F */ + 0x00, 0x00, 0xA4, 0x73, 0x00, 0x00, 0xC9, 0x77, /* 0x70-0x73 */ + 0xC9, 0x76, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0xA6, 0x7A, 0xC9, 0xD7, 0xC9, 0xD8, /* 0x78-0x7B */ + 0xC9, 0xD6, 0x00, 0x00, 0xC9, 0xD9, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0xCA, 0xC7, 0x00, 0x00, /* 0x84-0x87 */ + 0xCA, 0xC2, 0xCA, 0xC4, 0xCA, 0xC6, 0xCA, 0xC3, /* 0x88-0x8B */ + 0xA7, 0xC4, 0xCA, 0xC0, 0x00, 0x00, 0xCA, 0xC1, /* 0x8C-0x8F */ + 0xA7, 0xC1, 0xA7, 0xC2, 0xCA, 0xC5, 0xCA, 0xC8, /* 0x90-0x93 */ + 0xA7, 0xC3, 0xCA, 0xC9, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0xCC, 0x68, 0x00, 0x00, 0xCC, 0x62, /* 0x9C-0x9F */ + 0xCC, 0x5D, 0xA9, 0xA3, 0xCC, 0x65, 0xCC, 0x63, /* 0xA0-0xA3 */ + 0xCC, 0x5C, 0xCC, 0x69, 0xCC, 0x6C, 0xCC, 0x67, /* 0xA4-0xA7 */ + 0xCC, 0x60, 0xA9, 0xA5, 0xCC, 0x66, 0xA9, 0xA6, /* 0xA8-0xAB */ + 0xCC, 0x61, 0xCC, 0x64, 0xCC, 0x5B, 0xCC, 0x5F, /* 0xAC-0xAF */ + 0xCC, 0x6B, 0xA9, 0xA7, 0x00, 0x00, 0xA9, 0xA8, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xCC, 0x5E, 0xCC, 0x6A, 0xA9, 0xA2, /* 0xB4-0xB7 */ + 0xA9, 0xA4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0xCE, 0xAB, 0xCE, 0xA4, /* 0xC4-0xC7 */ + 0xCE, 0xAA, 0xCE, 0xA3, 0xCE, 0xA5, 0xCE, 0x7D, /* 0xC8-0xCB */ + 0xCE, 0x7B, 0x00, 0x00, 0xCE, 0xAC, 0xCE, 0xA9, /* 0xCC-0xCF */ + 0xCE, 0x79, 0x00, 0x00, 0xAB, 0xD0, 0xCE, 0xA7, /* 0xD0-0xD3 */ + 0xCE, 0xA8, 0x00, 0x00, 0xCE, 0xA6, 0xCE, 0x7C, /* 0xD4-0xD7 */ + 0xCE, 0x7A, 0xAB, 0xCF, 0xCE, 0xA2, 0xCE, 0x7E, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0xCE, 0xA1, 0xCE, 0xAD, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xAE, 0x6F, 0x00, 0x00, 0xAE, 0x6E, 0x00, 0x00, /* 0xE8-0xEB */ + 0xD1, 0x6C, 0xAE, 0x6B, 0xD1, 0x6E, 0x00, 0x00, /* 0xEC-0xEF */ + 0xAE, 0x70, 0xD1, 0x6F, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xAE, 0x73, 0x00, 0x00, 0xAE, 0x71, 0xD1, 0x70, /* 0xF4-0xF7 */ + 0xCE, 0xAE, 0xD1, 0x72, 0x00, 0x00, 0xAE, 0x6D, /* 0xF8-0xFB */ + 0x00, 0x00, 0xAE, 0x6C, 0x00, 0x00, 0xD1, 0x6D, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_5D[512] = { + 0xD1, 0x71, 0xAE, 0x72, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0xB1, 0x53, 0xB1, 0x52, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD4, 0xF5, /* 0x08-0x0B */ + 0xD4, 0xF9, 0xD4, 0xFB, 0xB1, 0x54, 0xD4, 0xFE, /* 0x0C-0x0F */ + 0x00, 0x00, 0xB1, 0x58, 0xD5, 0x41, 0x00, 0x00, /* 0x10-0x13 */ + 0xB1, 0x5A, 0x00, 0x00, 0xB1, 0x56, 0xB1, 0x5E, /* 0x14-0x17 */ + 0x00, 0x00, 0xB1, 0x5B, 0xD4, 0xF7, 0xB1, 0x55, /* 0x18-0x1B */ + 0x00, 0x00, 0xD4, 0xF6, 0xD4, 0xF4, 0xD5, 0x43, /* 0x1C-0x1F */ + 0xD4, 0xF8, 0x00, 0x00, 0xB1, 0x57, 0xD5, 0x42, /* 0x20-0x23 */ + 0xB1, 0x5C, 0xD4, 0xFD, 0xD4, 0xFC, 0xB1, 0x5D, /* 0x24-0x27 */ + 0xD4, 0xFA, 0xB1, 0x59, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0xD5, 0x44, 0x00, 0x00, /* 0x2C-0x2F */ + 0xD5, 0x40, 0xD8, 0xE7, 0xD8, 0xEE, 0xD8, 0xE3, /* 0x30-0x33 */ + 0xB4, 0x51, 0xD8, 0xDF, 0xD8, 0xEF, 0xD8, 0xD9, /* 0x34-0x37 */ + 0xD8, 0xEC, 0xD8, 0xEA, 0xD8, 0xE4, 0x00, 0x00, /* 0x38-0x3B */ + 0xD8, 0xED, 0xD8, 0xE6, 0x00, 0x00, 0xD8, 0xDE, /* 0x3C-0x3F */ + 0xD8, 0xF0, 0xD8, 0xDC, 0xD8, 0xE9, 0xD8, 0xDA, /* 0x40-0x43 */ + 0x00, 0x00, 0xD8, 0xF1, 0x00, 0x00, 0xB4, 0x52, /* 0x44-0x47 */ + 0x00, 0x00, 0xD8, 0xEB, 0xDD, 0x4F, 0xD8, 0xDD, /* 0x48-0x4B */ + 0xB4, 0x4F, 0x00, 0x00, 0xD8, 0xE1, 0x00, 0x00, /* 0x4C-0x4F */ + 0xB4, 0x50, 0xD8, 0xE0, 0xD8, 0xE5, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0xD8, 0xE2, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0xD8, 0xE8, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0xDD, 0x53, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0xDD, 0x56, 0xDD, 0x4E, /* 0x60-0x63 */ + 0x00, 0x00, 0xDD, 0x50, 0x00, 0x00, 0xDD, 0x55, /* 0x64-0x67 */ + 0xDD, 0x54, 0xB7, 0x43, 0x00, 0x00, 0xD8, 0xDB, /* 0x68-0x6B */ + 0xDD, 0x52, 0x00, 0x00, 0x00, 0x00, 0xB7, 0x44, /* 0x6C-0x6F */ + 0x00, 0x00, 0xDD, 0x4D, 0xDD, 0x51, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE1, 0xA9, /* 0x74-0x77 */ + 0x00, 0x00, 0xE1, 0xB0, 0xE1, 0xA7, 0x00, 0x00, /* 0x78-0x7B */ + 0xE1, 0xAE, 0xE1, 0xA5, 0xE1, 0xAD, 0xE1, 0xB1, /* 0x7C-0x7F */ + + 0xE1, 0xA4, 0xE1, 0xA8, 0xE1, 0xA3, 0x00, 0x00, /* 0x80-0x83 */ + 0xB9, 0xF1, 0x00, 0x00, 0xE1, 0xA6, 0xB9, 0xF2, /* 0x84-0x87 */ + 0xE1, 0xAC, 0xE1, 0xAB, 0xE1, 0xAA, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0xE1, 0xAF, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0x65, 0xE5, 0x67, /* 0x90-0x93 */ + 0xBC, 0x6B, 0xE5, 0x68, 0x00, 0x00, 0xE5, 0x63, /* 0x94-0x97 */ + 0x00, 0x00, 0xE5, 0x62, 0xE5, 0x6C, 0x00, 0x00, /* 0x98-0x9B */ + 0xE5, 0x6A, 0xBC, 0x6A, 0xE5, 0x6D, 0xE5, 0x64, /* 0x9C-0x9F */ + 0xE5, 0x69, 0xE5, 0x6B, 0xE5, 0x66, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE9, 0x61, /* 0xA4-0xA7 */ + 0xE9, 0x66, 0xE9, 0x60, 0xE9, 0x65, 0x00, 0x00, /* 0xA8-0xAB */ + 0xE9, 0x5E, 0xE9, 0x68, 0xE9, 0x64, 0xE9, 0x69, /* 0xAC-0xAF */ + 0xE9, 0x63, 0xE9, 0x5F, 0xE9, 0x67, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xE9, 0x6A, 0xE9, 0x62, 0x00, 0x00, 0xEC, 0xDA, /* 0xB4-0xB7 */ + 0xC0, 0xAF, 0x00, 0x00, 0xC0, 0xAD, 0x00, 0x00, /* 0xB8-0xBB */ + 0xC0, 0xAC, 0xC0, 0xAE, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0xEF, 0xC4, 0x00, 0x00, 0xF1, 0x72, 0xF1, 0xFD, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0xF4, 0x44, 0xF4, 0x45, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xC4, 0x60, 0x00, 0x00, 0xF5, 0xC9, /* 0xC8-0xCB */ + 0x00, 0x00, 0xC4, 0xDE, 0x00, 0x00, 0xF5, 0xCA, /* 0xCC-0xCF */ + 0x00, 0x00, 0xF6, 0xDE, 0xC5, 0x72, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xC5, 0x71, 0xF6, 0xDD, 0xC5, 0xC9, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xF7, 0xD6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0xA4, 0x74, 0xA6, 0x7B, 0xC9, 0xDA, /* 0xDC-0xDF */ + 0xCA, 0xCA, 0xA8, 0xB5, 0xB1, 0x5F, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xA4, 0x75, 0xA5, 0xAA, 0xA5, 0xA9, /* 0xE4-0xE7 */ + 0xA5, 0xA8, 0x00, 0x00, 0x00, 0x00, 0xA7, 0xC5, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0xAE, 0x74, 0x00, 0x00, /* 0xEC-0xEF */ + 0xDD, 0x57, 0xA4, 0x76, 0xA4, 0x77, 0xA4, 0x78, /* 0xF0-0xF3 */ + 0xA4, 0xDA, 0x00, 0x00, 0x00, 0x00, 0xAB, 0xD1, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xCE, 0xAF, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0xB4, 0x53, 0xA4, 0x79, 0xC9, 0x5D, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_5E[512] = { + 0x00, 0x00, 0x00, 0x00, 0xA5, 0xAB, 0xA5, 0xAC, /* 0x00-0x03 */ + 0xC9, 0x78, 0x00, 0x00, 0xA6, 0x7C, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xCA, 0xCB, 0x00, 0x00, /* 0x08-0x0B */ + 0xA7, 0xC6, 0x00, 0x00, 0xCA, 0xCC, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0xA9, 0xAE, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0xCC, 0x6E, 0xA9, 0xAC, 0xA9, 0xAB, 0xCC, 0x6D, /* 0x14-0x17 */ + 0xA9, 0xA9, 0xCC, 0x6F, 0xA9, 0xAA, 0xA9, 0xAD, /* 0x18-0x1B */ + 0x00, 0x00, 0xAB, 0xD2, 0x00, 0x00, 0xAB, 0xD4, /* 0x1C-0x1F */ + 0xCE, 0xB3, 0xCE, 0xB0, 0xCE, 0xB1, 0xCE, 0xB2, /* 0x20-0x23 */ + 0xCE, 0xB4, 0xAB, 0xD3, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0xD1, 0x74, 0xD1, 0x73, 0x00, 0x00, 0xAE, 0x76, /* 0x28-0x2B */ + 0x00, 0x00, 0xAE, 0x75, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB1, 0x62, /* 0x30-0x33 */ + 0xD5, 0x46, 0x00, 0x00, 0xB1, 0x61, 0xB1, 0x63, /* 0x34-0x37 */ + 0xB1, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0xB4, 0x55, 0xD5, 0x45, 0x00, 0x00, /* 0x3C-0x3F */ + 0xB4, 0x56, 0xD8, 0xF3, 0x00, 0x00, 0xB4, 0x57, /* 0x40-0x43 */ + 0xD8, 0xF2, 0xB4, 0x54, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0xDD, 0x5A, 0xDD, 0x5C, /* 0x48-0x4B */ + 0xB7, 0x45, 0xDD, 0x5B, 0xDD, 0x59, 0xDD, 0x58, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE1, 0xB4, /* 0x50-0x53 */ + 0xB9, 0xF7, 0xB9, 0xF5, 0x00, 0x00, 0xB9, 0xF6, /* 0x54-0x57 */ + 0xE1, 0xB2, 0xE1, 0xB3, 0x00, 0x00, 0xB9, 0xF3, /* 0x58-0x5B */ + 0xE5, 0x71, 0xE5, 0x6F, 0x00, 0x00, 0xBC, 0x6D, /* 0x5C-0x5F */ + 0xE5, 0x70, 0xBC, 0x6E, 0xBC, 0x6C, 0xB9, 0xF4, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0x6D, 0xE9, 0x6B, /* 0x64-0x67 */ + 0xE9, 0x6C, 0xE5, 0x6E, 0xEC, 0xDC, 0xC0, 0xB0, /* 0x68-0x6B */ + 0xEC, 0xDB, 0xEF, 0xC5, 0xEF, 0xC6, 0xE9, 0x6E, /* 0x6C-0x6F */ + 0xF1, 0xFE, 0x00, 0x00, 0xA4, 0x7A, 0xA5, 0xAD, /* 0x70-0x73 */ + 0xA6, 0x7E, 0xC9, 0xDB, 0xA6, 0x7D, 0x00, 0x00, /* 0x74-0x77 */ + 0xA9, 0xAF, 0xB7, 0x46, 0x00, 0x00, 0xA4, 0xDB, /* 0x78-0x7B */ + 0xA5, 0xAE, 0xAB, 0xD5, 0xB4, 0x58, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xC9, 0x79, 0x00, 0x00, 0xC9, 0x7A, 0x00, 0x00, /* 0x80-0x83 */ + 0xC9, 0xDC, 0x00, 0x00, 0x00, 0x00, 0xA7, 0xC8, /* 0x84-0x87 */ + 0xCA, 0xD0, 0xCA, 0xCE, 0xA7, 0xC9, 0xCA, 0xCD, /* 0x88-0x8B */ + 0xCA, 0xCF, 0xCA, 0xD1, 0x00, 0x00, 0xA7, 0xC7, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0xA9, 0xB3, 0xA9, 0xB4, 0xA9, 0xB1, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0xA9, 0xB0, 0xCE, 0xB8, /* 0x98-0x9B */ + 0xA9, 0xB2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0xAB, 0xD6, 0x00, 0x00, 0xCE, 0xB7, 0xCE, 0xB9, /* 0xA0-0xA3 */ + 0xCE, 0xB6, 0xCE, 0xBA, 0xAB, 0xD7, 0xAE, 0x79, /* 0xA4-0xA7 */ + 0xD1, 0x75, 0x00, 0x00, 0xD1, 0x77, 0xAE, 0x77, /* 0xA8-0xAB */ + 0xD1, 0x78, 0xAE, 0x78, 0xD1, 0x76, 0x00, 0x00, /* 0xAC-0xAF */ + 0xCE, 0xB5, 0xD5, 0x47, 0xD5, 0x4A, 0xD5, 0x4B, /* 0xB0-0xB3 */ + 0xD5, 0x48, 0xB1, 0x67, 0xB1, 0x66, 0xB1, 0x64, /* 0xB4-0xB7 */ + 0xB1, 0x65, 0xD5, 0x49, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0xB1, 0x68, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0xB4, 0x5A, 0xB4, 0x5B, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xB4, 0x5C, 0xDD, 0x5D, 0xDD, 0x5F, 0xDD, 0x61, /* 0xC4-0xC7 */ + 0xB7, 0x48, 0xB7, 0x47, 0xB4, 0x59, 0xDD, 0x60, /* 0xC8-0xCB */ + 0xDD, 0x5E, 0x00, 0x00, 0xE1, 0xB8, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0xE1, 0xB6, 0xE1, 0xBC, 0xB9, 0xF8, /* 0xD0-0xD3 */ + 0xE1, 0xBD, 0xE1, 0xBA, 0xB9, 0xF9, 0xE1, 0xB7, /* 0xD4-0xD7 */ + 0xE1, 0xB5, 0xE1, 0xBB, 0xBC, 0x70, 0xE5, 0x73, /* 0xD8-0xDB */ + 0xE1, 0xB9, 0xBC, 0x72, 0xE5, 0x74, 0xBC, 0x71, /* 0xDC-0xDF */ + 0xBC, 0x74, 0xE5, 0x75, 0xBC, 0x6F, 0xBC, 0x73, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xE9, 0x73, 0xE9, 0x71, 0xE9, 0x70, /* 0xE4-0xE7 */ + 0xE9, 0x72, 0xE9, 0x6F, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0xC3, 0x66, 0x00, 0x00, 0xF4, 0x46, 0xF4, 0x47, /* 0xEC-0xEF */ + 0x00, 0x00, 0xF5, 0xCB, 0xF6, 0xDF, 0xC6, 0x55, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0xA9, 0xB5, 0xA7, 0xCA, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0xAB, 0xD8, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0xA4, 0x7B, 0xA4, 0xDC, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_5F[512] = { + 0x00, 0x00, 0xA5, 0xAF, 0xC9, 0xDD, 0x00, 0x00, /* 0x00-0x03 */ + 0xA7, 0xCB, 0xCA, 0xD2, 0x00, 0x00, 0xCE, 0xBB, /* 0x04-0x07 */ + 0xAB, 0xD9, 0x00, 0x00, 0xB9, 0xFA, 0xA4, 0x7C, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA6, 0xA1, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0xB7, 0x49, 0xA4, 0x7D, /* 0x10-0x13 */ + 0xA4, 0xDD, 0xA4, 0xDE, 0x00, 0x00, 0xA5, 0xB1, /* 0x14-0x17 */ + 0xA5, 0xB0, 0x00, 0x00, 0xC9, 0xDE, 0xA6, 0xA2, /* 0x18-0x1B */ + 0x00, 0x00, 0xCA, 0xD3, 0x00, 0x00, 0xA7, 0xCC, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0xCC, 0x71, 0xCC, 0x72, /* 0x20-0x23 */ + 0xCC, 0x73, 0x00, 0x00, 0xA9, 0xB6, 0xA9, 0xB7, /* 0x24-0x27 */ + 0xCC, 0x70, 0xA9, 0xB8, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0xAB, 0xDA, 0xCE, 0xBC, 0x00, 0x00, /* 0x2C-0x2F */ + 0xD1, 0x7A, 0xAE, 0x7A, 0x00, 0x00, 0xD1, 0x79, /* 0x30-0x33 */ + 0x00, 0x00, 0xB1, 0x69, 0xD5, 0x4C, 0xB1, 0x6A, /* 0x34-0x37 */ + 0xD5, 0x4D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0xB4, 0x5D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0xDD, 0x62, 0x00, 0x00, 0x00, 0x00, 0xE1, 0xBF, /* 0x40-0x43 */ + 0xE1, 0xBE, 0x00, 0x00, 0xB9, 0xFB, 0x00, 0x00, /* 0x44-0x47 */ + 0xBC, 0x75, 0xE5, 0x76, 0xBE, 0xCA, 0xE9, 0x74, /* 0x48-0x4B */ + 0xC0, 0xB1, 0x00, 0x00, 0xC5, 0x73, 0xF7, 0xD8, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0xCC, 0x74, 0x00, 0x00, 0xCE, 0xBD, 0xB1, 0x6B, /* 0x54-0x57 */ + 0xD8, 0xF4, 0xB7, 0x4A, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0xC2, 0x55, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0xA7, 0xCE, 0x00, 0x00, /* 0x60-0x63 */ + 0xA7, 0xCD, 0xAB, 0xDB, 0x00, 0x00, 0xD1, 0x7B, /* 0x64-0x67 */ + 0x00, 0x00, 0xB1, 0x6D, 0xB3, 0x43, 0xB1, 0x6E, /* 0x68-0x6B */ + 0xB1, 0x6C, 0xB4, 0x5E, 0x00, 0x00, 0xE1, 0xC0, /* 0x6C-0x6F */ + 0xB9, 0xFC, 0xBC, 0x76, 0x00, 0x00, 0xC9, 0x4C, /* 0x70-0x73 */ + 0xC9, 0xDF, 0x00, 0x00, 0xCA, 0xD5, 0xA7, 0xCF, /* 0x74-0x77 */ + 0xCA, 0xD4, 0xA7, 0xD0, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0xA9, 0xBC, 0xCC, 0x77, 0xCC, 0x76, 0xA9, 0xBB, /* 0x7C-0x7F */ + + 0xA9, 0xB9, 0xA9, 0xBA, 0xCC, 0x75, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0xAB, 0xDD, 0xCE, 0xBE, 0xAB, 0xE0, /* 0x84-0x87 */ + 0xAB, 0xDC, 0xAB, 0xE2, 0xAB, 0xDE, 0xAB, 0xDF, /* 0x88-0x8B */ + 0xAB, 0xE1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0xAE, 0x7D, 0xAE, 0x7C, 0xAE, 0x7B, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0xD5, 0x4F, 0xB1, 0x6F, /* 0x94-0x97 */ + 0xB1, 0x72, 0xB1, 0x70, 0x00, 0x00, 0xD5, 0x4E, /* 0x98-0x9B */ + 0xB1, 0x75, 0x00, 0x00, 0xB1, 0x71, 0xD5, 0x50, /* 0x9C-0x9F */ + 0xB1, 0x74, 0xB1, 0x73, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xD8, 0xF6, 0xD8, 0xF5, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xB4, 0x61, 0xB4, 0x5F, 0xB4, 0x60, 0xD8, 0xF7, /* 0xA8-0xAB */ + 0xB7, 0x4B, 0xDD, 0x64, 0xB7, 0x4C, 0xDD, 0x63, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0x77, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xBC, 0x78, 0xE1, 0xC1, 0xBC, 0x77, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xB9, 0xFD, 0x00, 0x00, 0xEC, 0xDE, /* 0xB8-0xBB */ + 0xE9, 0x75, 0xC0, 0xB2, 0xEC, 0xDD, 0xF2, 0x40, /* 0xBC-0xBF */ + 0xF4, 0x48, 0xF4, 0x49, 0x00, 0x00, 0xA4, 0xDF, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xA5, 0xB2, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xC9, 0x7B, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0xA7, 0xD2, 0xA7, 0xD4, 0x00, 0x00, 0xC9, 0xE2, /* 0xCC-0xCF */ + 0xCA, 0xD8, 0xCA, 0xD7, 0xCA, 0xD6, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xC9, 0xE1, 0xC9, 0xE0, 0xA6, 0xA4, 0xA7, 0xD3, /* 0xD4-0xD7 */ + 0xA7, 0xD1, 0xA6, 0xA3, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0xA9, 0xBD, 0xCC, 0x78, 0x00, 0x00, /* 0xDC-0xDF */ + 0xA9, 0xBE, 0xCA, 0xDD, 0x00, 0x00, 0xCA, 0xDF, /* 0xE0-0xE3 */ + 0xCA, 0xDE, 0xCC, 0x79, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xCA, 0xDA, 0x00, 0x00, 0xA7, 0xD8, 0xA7, 0xD6, /* 0xE8-0xEB */ + 0x00, 0x00, 0xCA, 0xD9, 0xCA, 0xDB, 0xCA, 0xE1, /* 0xEC-0xEF */ + 0x00, 0x00, 0xA7, 0xD5, 0x00, 0x00, 0xCA, 0xDC, /* 0xF0-0xF3 */ + 0xCA, 0xE5, 0xA9, 0xC0, 0x00, 0x00, 0xCA, 0xE2, /* 0xF4-0xF7 */ + 0xA7, 0xD7, 0x00, 0x00, 0xCA, 0xE0, 0xCA, 0xE3, /* 0xF8-0xFB */ + 0x00, 0x00, 0xA9, 0xBF, 0x00, 0x00, 0xA9, 0xC1, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_60[512] = { + 0xCA, 0xE4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xCC, 0xAF, 0xCC, 0xA2, 0xCC, 0x7E, /* 0x08-0x0B */ + 0xCC, 0xAE, 0xCC, 0xA9, 0xAB, 0xE7, 0xA9, 0xC2, /* 0x0C-0x0F */ + 0xCC, 0xAA, 0xCC, 0xAD, 0xAB, 0xE3, 0xCC, 0xAC, /* 0x10-0x13 */ + 0xA9, 0xC3, 0xA9, 0xC8, 0xA9, 0xC6, 0xCC, 0xA3, /* 0x14-0x17 */ + 0x00, 0x00, 0xCC, 0x7C, 0xCC, 0xA5, 0xA9, 0xCD, /* 0x18-0x1B */ + 0xCC, 0xB0, 0xAB, 0xE4, 0xCC, 0xA6, 0x00, 0x00, /* 0x1C-0x1F */ + 0xAB, 0xE5, 0xA9, 0xC9, 0xCC, 0xA8, 0x00, 0x00, /* 0x20-0x23 */ + 0xCE, 0xCD, 0xAB, 0xE6, 0xCC, 0x7B, 0xA9, 0xCA, /* 0x24-0x27 */ + 0xAB, 0xE8, 0xA9, 0xCB, 0xA9, 0xC7, 0xA9, 0xCC, /* 0x28-0x2B */ + 0xCC, 0xA7, 0xCC, 0x7A, 0xCC, 0xAB, 0xA9, 0xC4, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0xCC, 0x7D, 0xCC, 0xA4, /* 0x30-0x33 */ + 0xCC, 0xA1, 0xA9, 0xC5, 0x00, 0x00, 0xCE, 0xBF, /* 0x34-0x37 */ + 0x00, 0x00, 0xCE, 0xC0, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0xCE, 0xCA, 0xD1, 0xA1, 0xCE, 0xCB, 0xAB, 0xEE, /* 0x40-0x43 */ + 0xCE, 0xCE, 0xCE, 0xC4, 0xAB, 0xED, 0xCE, 0xC6, /* 0x44-0x47 */ + 0x00, 0x00, 0xCE, 0xC7, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0xCE, 0xC9, 0xAB, 0xE9, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0xAE, 0xA3, 0x00, 0x00, 0xF9, 0xDA, 0xCE, 0xC5, /* 0x50-0x53 */ + 0xCE, 0xC1, 0xAE, 0xA4, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0xCE, 0xCF, 0xAE, 0x7E, 0xD1, 0x7D, 0xCE, 0xC8, /* 0x58-0x5B */ + 0x00, 0x00, 0xD1, 0x7C, 0xCE, 0xC3, 0xCE, 0xCC, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0xAB, 0xEC, 0xAE, 0xA1, /* 0x60-0x63 */ + 0xAB, 0xF2, 0xAE, 0xA2, 0xCE, 0xD0, 0xD1, 0x7E, /* 0x64-0x67 */ + 0xAB, 0xEB, 0xAE, 0xA6, 0xAB, 0xF1, 0xAB, 0xF0, /* 0x68-0x6B */ + 0xAB, 0xEF, 0xAE, 0xA5, 0xCE, 0xD1, 0xAE, 0xA7, /* 0x6C-0x6F */ + 0xAB, 0xEA, 0x00, 0x00, 0xCE, 0xC2, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB1, 0x76, /* 0x7C-0x7F */ + + 0xD1, 0xA4, 0xD1, 0xA6, 0x00, 0x00, 0xD1, 0xA8, /* 0x80-0x83 */ + 0xAE, 0xA8, 0xAE, 0xAE, 0xD5, 0x53, 0xD1, 0xAC, /* 0x84-0x87 */ + 0xD1, 0xA3, 0xB1, 0x78, 0xD5, 0x51, 0x00, 0x00, /* 0x88-0x8B */ + 0xAE, 0xAD, 0xAE, 0xAB, 0xD1, 0xAE, 0x00, 0x00, /* 0x8C-0x8F */ + 0xD5, 0x52, 0x00, 0x00, 0xD1, 0xA5, 0x00, 0x00, /* 0x90-0x93 */ + 0xAE, 0xAC, 0xD1, 0xA9, 0xAE, 0xAF, 0xD1, 0xAB, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0xAE, 0xAA, 0xD1, 0xAA, /* 0x98-0x9B */ + 0xD1, 0xAD, 0xD1, 0xA7, 0x00, 0x00, 0xAE, 0xA9, /* 0x9C-0x9F */ + 0xB1, 0x79, 0x00, 0x00, 0xD1, 0xA2, 0xB1, 0x77, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xB1, 0x7A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0xD5, 0x55, 0xD5, 0x5E, 0xB4, 0x64, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xB1, 0x7C, 0xB1, 0xA3, 0xB4, 0x65, 0xD5, 0x60, /* 0xB4-0xB7 */ + 0xB1, 0xAA, 0xD8, 0xF9, 0xD5, 0x56, 0xB1, 0xA2, /* 0xB8-0xBB */ + 0xB1, 0xA5, 0xB1, 0x7E, 0xD5, 0x54, 0xD5, 0x62, /* 0xBC-0xBF */ + 0xD5, 0x65, 0xD9, 0x49, 0x00, 0x00, 0xD5, 0x63, /* 0xC0-0xC3 */ + 0xD8, 0xFD, 0xB1, 0xA1, 0xB1, 0xA8, 0xB1, 0xAC, /* 0xC4-0xC7 */ + 0xD5, 0x5D, 0xD8, 0xF8, 0xD5, 0x61, 0xB1, 0x7B, /* 0xC8-0xCB */ + 0xD8, 0xFA, 0xD5, 0x64, 0xD8, 0xFC, 0xD5, 0x59, /* 0xCC-0xCF */ + 0x00, 0x00, 0xB4, 0x62, 0x00, 0x00, 0xD5, 0x57, /* 0xD0-0xD3 */ + 0xD5, 0x58, 0xB1, 0xA7, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xB1, 0xA6, 0xD5, 0x5B, 0xB1, 0xAB, 0xD5, 0x5F, /* 0xD8-0xDB */ + 0xB1, 0xA4, 0xD5, 0x5C, 0x00, 0x00, 0xB1, 0xA9, /* 0xDC-0xDF */ + 0xB4, 0x66, 0xB4, 0x63, 0xD8, 0xFB, 0x00, 0x00, /* 0xE0-0xE3 */ + 0xD5, 0x5A, 0x00, 0x00, 0xB1, 0x7D, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0xB4, 0x6B, 0xB4, 0x6F, 0xD9, 0x40, 0xB7, 0x51, /* 0xF0-0xF3 */ + 0xB4, 0x6D, 0xD9, 0x44, 0xB4, 0x71, 0xDD, 0x65, /* 0xF4-0xF7 */ + 0xD9, 0x46, 0xB7, 0x53, 0xB4, 0x69, 0xB4, 0x6C, /* 0xF8-0xFB */ + 0xD9, 0x47, 0x00, 0x00, 0xD9, 0x48, 0xD9, 0x4E, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_61[512] = { + 0xB4, 0x73, 0xB7, 0x54, 0x00, 0x00, 0xD9, 0x4A, /* 0x00-0x03 */ + 0xD9, 0x4F, 0xD9, 0x43, 0xB7, 0x5E, 0x00, 0x00, /* 0x04-0x07 */ + 0xB7, 0x55, 0xB4, 0x72, 0xD9, 0x41, 0xD9, 0x50, /* 0x08-0x0B */ + 0x00, 0x00, 0xB7, 0x5D, 0xB4, 0x70, 0xB7, 0x4E, /* 0x0C-0x0F */ + 0xD9, 0x4D, 0x00, 0x00, 0xB4, 0x74, 0xD9, 0x45, /* 0x10-0x13 */ + 0xD8, 0xFE, 0xB4, 0x6A, 0xD9, 0x42, 0x00, 0x00, /* 0x14-0x17 */ + 0xD9, 0x4B, 0x00, 0x00, 0xB7, 0x4D, 0xB7, 0x52, /* 0x18-0x1B */ + 0xB4, 0x67, 0xD9, 0x4C, 0x00, 0x00, 0xB7, 0x50, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB4, 0x68, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB7, 0x5C, /* 0x24-0x27 */ + 0xE1, 0xC3, 0xDD, 0x70, 0x00, 0x00, 0xDD, 0x68, /* 0x28-0x2B */ + 0xE1, 0xC2, 0x00, 0x00, 0xDD, 0x6C, 0xDD, 0x6E, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0xDD, 0x6B, 0x00, 0x00, /* 0x30-0x33 */ + 0xB7, 0x5B, 0x00, 0x00, 0xDD, 0x6A, 0xB7, 0x5F, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE1, 0xD2, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xB7, 0x5A, 0xBA, 0x40, /* 0x3C-0x3F */ + 0xDD, 0x71, 0xE1, 0xC4, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0xB7, 0x58, 0xDD, 0x69, 0xDD, 0x6D, 0xB9, 0xFE, /* 0x44-0x47 */ + 0xB7, 0x4F, 0xDD, 0x66, 0xDD, 0x67, 0xBA, 0x41, /* 0x48-0x4B */ + 0xB7, 0x57, 0xB7, 0x59, 0xB7, 0x56, 0xDD, 0x6F, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xC8, 0xE1, 0xC9, /* 0x50-0x53 */ + 0xE1, 0xCE, 0xBC, 0x7D, 0xE1, 0xD5, 0x00, 0x00, /* 0x54-0x57 */ + 0xBA, 0x47, 0x00, 0x00, 0xBA, 0x46, 0xE1, 0xD0, /* 0x58-0x5B */ + 0x00, 0x00, 0xBC, 0x7C, 0xE1, 0xC5, 0xBA, 0x45, /* 0x5C-0x5F */ + 0x00, 0x00, 0xE1, 0xD4, 0xBA, 0x43, 0xBA, 0x44, /* 0x60-0x63 */ + 0x00, 0x00, 0xE1, 0xD1, 0xE5, 0xAA, 0xBC, 0x7A, /* 0x64-0x67 */ + 0xB4, 0x6E, 0x00, 0x00, 0xE1, 0xD3, 0xBC, 0xA3, /* 0x68-0x6B */ + 0xE1, 0xCB, 0x00, 0x00, 0xBC, 0x7B, 0x00, 0x00, /* 0x6C-0x6F */ + 0xBC, 0xA2, 0xE1, 0xC6, 0xE1, 0xCA, 0xE1, 0xC7, /* 0x70-0x73 */ + 0xE1, 0xCD, 0xBA, 0x48, 0xBC, 0x79, 0xBA, 0x42, /* 0x74-0x77 */ + 0x00, 0x00, 0xE5, 0x7A, 0xE1, 0xCF, 0x00, 0x00, /* 0x78-0x7B */ + 0xBC, 0xA1, 0x00, 0x00, 0xBC, 0xA4, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xE1, 0xCC, 0x00, 0x00, 0xBC, 0x7E, 0xE5, 0x79, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0xE5, 0x7E, 0xBE, 0xCE, 0xE5, 0x78, /* 0x88-0x8B */ + 0xE9, 0xA3, 0xE5, 0xA9, 0xBC, 0xA8, 0x00, 0x00, /* 0x8C-0x8F */ + 0xBC, 0xA6, 0xBE, 0xCC, 0xE5, 0xA6, 0xE5, 0xA2, /* 0x90-0x93 */ + 0xBC, 0xAC, 0x00, 0x00, 0xE9, 0x78, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0xBC, 0xAA, 0xE5, 0xA1, /* 0x98-0x9B */ + 0x00, 0x00, 0xE9, 0x76, 0x00, 0x00, 0xE5, 0xA5, /* 0x9C-0x9F */ + 0x00, 0x00, 0xE5, 0xA8, 0xE5, 0x7D, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xBC, 0xAB, 0x00, 0x00, 0x00, 0x00, 0xBC, 0xA5, /* 0xA4-0xA7 */ + 0xE9, 0x77, 0xBE, 0xCD, 0xE5, 0xA7, 0xBC, 0xA7, /* 0xA8-0xAB */ + 0xBC, 0xA9, 0xE5, 0xA4, 0xBC, 0xAD, 0xE5, 0xA3, /* 0xAC-0xAF */ + 0xE5, 0x7C, 0xE5, 0x7B, 0xBE, 0xCB, 0xE5, 0xAB, /* 0xB0-0xB3 */ + 0xE9, 0x7A, 0xEC, 0xE0, 0xBE, 0xD0, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xE9, 0xA2, 0x00, 0x00, 0xE9, 0x7E, 0x00, 0x00, /* 0xB8-0xBB */ + 0xEC, 0xE1, 0x00, 0x00, 0xBE, 0xD1, 0xE9, 0xA1, /* 0xBC-0xBF */ + 0x00, 0x00, 0xE9, 0x7C, 0xC0, 0xB4, 0xEC, 0xDF, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xE9, 0x79, 0xE9, 0x7B, 0xC0, 0xB5, /* 0xC4-0xC7 */ + 0xBE, 0xD3, 0xC0, 0xB3, 0xBE, 0xD2, 0xC0, 0xB7, /* 0xC8-0xCB */ + 0xE9, 0x7D, 0xBE, 0xCF, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0xEF, 0xCF, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xEF, 0xC7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0xEC, 0xE7, 0xEF, 0xC8, /* 0xDC-0xDF */ + 0xEC, 0xE3, 0x00, 0x00, 0x00, 0x00, 0xC2, 0x56, /* 0xE0-0xE3 */ + 0xEC, 0xE5, 0xEC, 0xE4, 0xC0, 0xB6, 0xEC, 0xE2, /* 0xE4-0xE7 */ + 0xEC, 0xE6, 0xEF, 0xD0, 0xEF, 0xCC, 0xEF, 0xCE, /* 0xE8-0xEB */ + 0x00, 0x00, 0xEF, 0xC9, 0xEF, 0xCA, 0x00, 0x00, /* 0xEC-0xEF */ + 0xEF, 0xCD, 0xEF, 0xCB, 0xC3, 0x67, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0xC3, 0x6A, 0xC3, 0x69, 0xC3, 0x68, /* 0xF4-0xF7 */ + 0xC4, 0x61, 0xF4, 0x4A, 0xC4, 0x62, 0xF2, 0x41, /* 0xF8-0xFB */ + 0xC4, 0xDF, 0xF5, 0xCC, 0xC4, 0xE0, 0xC5, 0x74, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_62[512] = { + 0xC5, 0xCA, 0xF7, 0xD9, 0x00, 0x00, 0xF7, 0xDA, /* 0x00-0x03 */ + 0xF7, 0xDB, 0x00, 0x00, 0x00, 0x00, 0xF9, 0xBA, /* 0x04-0x07 */ + 0xA4, 0xE0, 0xC9, 0x7C, 0xA5, 0xB3, 0x00, 0x00, /* 0x08-0x0B */ + 0xA6, 0xA6, 0xA6, 0xA7, 0xA6, 0xA5, 0x00, 0x00, /* 0x0C-0x0F */ + 0xA6, 0xA8, 0xA7, 0xDA, 0xA7, 0xD9, 0x00, 0x00, /* 0x10-0x13 */ + 0xCC, 0xB1, 0xA9, 0xCF, 0xA9, 0xCE, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0xD1, 0xAF, 0xB1, 0xAD, 0xB1, 0xAE, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB4, 0x75, /* 0x1C-0x1F */ + 0xDD, 0x72, 0xB7, 0x60, 0xB7, 0x61, 0xDD, 0x74, /* 0x20-0x23 */ + 0xDD, 0x76, 0xDD, 0x75, 0x00, 0x00, 0xE1, 0xD7, /* 0x24-0x27 */ + 0x00, 0x00, 0xE1, 0xD6, 0xBA, 0x49, 0xE1, 0xD8, /* 0x28-0x2B */ + 0x00, 0x00, 0xE5, 0xAC, 0xBC, 0xAE, 0x00, 0x00, /* 0x2C-0x2F */ + 0xBE, 0xD4, 0x00, 0x00, 0xC0, 0xB8, 0xC2, 0x57, /* 0x30-0x33 */ + 0xC0, 0xB9, 0x00, 0x00, 0xA4, 0xE1, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xCA, 0xE6, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0xCC, 0xB2, 0xA9, 0xD1, 0xA9, 0xD0, /* 0x3C-0x3F */ + 0xA9, 0xD2, 0xAB, 0xF3, 0xCE, 0xD2, 0xCE, 0xD3, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xD1, 0xB0, 0xAE, 0xB0, /* 0x44-0x47 */ + 0xB1, 0xAF, 0xB4, 0x76, 0xD9, 0x51, 0xA4, 0xE2, /* 0x48-0x4B */ + 0x00, 0x00, 0xA4, 0x7E, 0xA4, 0xE3, 0x00, 0x00, /* 0x4C-0x4F */ + 0xC9, 0x7D, 0xA5, 0xB7, 0xA5, 0xB6, 0xA5, 0xB4, /* 0x50-0x53 */ + 0xA5, 0xB5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0xA6, 0xAB, 0xC9, 0xE9, 0xC9, 0xEB, 0xA6, 0xAA, /* 0x58-0x5B */ + 0xC9, 0xE3, 0x00, 0x00, 0xC9, 0xE4, 0x00, 0x00, /* 0x5C-0x5F */ + 0xC9, 0xEA, 0xC9, 0xE6, 0xC9, 0xE8, 0xA6, 0xA9, /* 0x60-0x63 */ + 0xC9, 0xE5, 0xC9, 0xEC, 0xC9, 0xE7, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0xA7, 0xE1, 0xA7, 0xEA, 0xA7, 0xE8, /* 0x6C-0x6F */ + 0xCA, 0xF0, 0xCA, 0xED, 0xCA, 0xF5, 0xA7, 0xE6, /* 0x70-0x73 */ + 0xCA, 0xF6, 0x00, 0x00, 0xA7, 0xDF, 0xCA, 0xF3, /* 0x74-0x77 */ + 0x00, 0x00, 0xA7, 0xE5, 0xCA, 0xEF, 0xCA, 0xEE, /* 0x78-0x7B */ + 0xA7, 0xE3, 0xCA, 0xF4, 0xA7, 0xE4, 0xA9, 0xD3, /* 0x7C-0x7F */ + + 0xA7, 0xDE, 0xCA, 0xF1, 0x00, 0x00, 0xCA, 0xE7, /* 0x80-0x83 */ + 0xA7, 0xDB, 0x00, 0x00, 0xA7, 0xEE, 0xCA, 0xEC, /* 0x84-0x87 */ + 0xCA, 0xF2, 0xA7, 0xE0, 0xA7, 0xE2, 0x00, 0x00, /* 0x88-0x8B */ + 0xCA, 0xE8, 0x00, 0x00, 0xCA, 0xE9, 0xCA, 0xEA, /* 0x8C-0x8F */ + 0x00, 0x00, 0xA7, 0xED, 0xA7, 0xE7, 0xA7, 0xEC, /* 0x90-0x93 */ + 0xCA, 0xEB, 0xA7, 0xEB, 0xA7, 0xDD, 0xA7, 0xDC, /* 0x94-0x97 */ + 0xA7, 0xE9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xA9, 0xE1, 0xCC, 0xBE, 0xCC, 0xB7, 0xA9, 0xDC, /* 0xA8-0xAB */ + 0xA9, 0xEF, 0xCC, 0xB3, 0xCC, 0xBA, 0xCC, 0xBC, /* 0xAC-0xAF */ + 0xCC, 0xBF, 0xA9, 0xEA, 0x00, 0x00, 0xCC, 0xBB, /* 0xB0-0xB3 */ + 0xCC, 0xB4, 0xA9, 0xE8, 0xCC, 0xB8, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xCC, 0xC0, 0xA9, 0xD9, 0x00, 0x00, 0xCC, 0xBD, /* 0xB8-0xBB */ + 0xA9, 0xE3, 0xA9, 0xE2, 0xCC, 0xB6, 0xA9, 0xD7, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0xA9, 0xD8, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xA9, 0xD6, 0x00, 0x00, 0xA9, 0xEE, 0xA9, 0xE6, /* 0xC4-0xC7 */ + 0xA9, 0xE0, 0xA9, 0xD4, 0xCC, 0xB9, 0xA9, 0xDF, /* 0xC8-0xCB */ + 0xA9, 0xD5, 0xA9, 0xE7, 0xA9, 0xF0, 0xCE, 0xD4, /* 0xCC-0xCF */ + 0xA9, 0xE4, 0xCC, 0xB5, 0xA9, 0xDA, 0xA9, 0xDD, /* 0xD0-0xD3 */ + 0xA9, 0xDE, 0x00, 0x00, 0xA9, 0xEC, 0xA9, 0xED, /* 0xD4-0xD7 */ + 0xA9, 0xEB, 0xA9, 0xE5, 0xA9, 0xE9, 0xA9, 0xDB, /* 0xD8-0xDB */ + 0xAB, 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCE, 0xDA, /* 0xE8-0xEB */ + 0xAC, 0x41, 0xAB, 0xF8, 0xAB, 0xFA, 0xAC, 0x40, /* 0xEC-0xEF */ + 0xCE, 0xE6, 0xAB, 0xFD, 0xD1, 0xB1, 0xAE, 0xB1, /* 0xF0-0xF3 */ + 0xAC, 0x43, 0xCE, 0xD7, 0xCE, 0xDF, 0xAB, 0xFE, /* 0xF4-0xF7 */ + 0xCE, 0xDE, 0xCE, 0xDB, 0xCE, 0xE3, 0xCE, 0xE5, /* 0xF8-0xFB */ + 0xAB, 0xF7, 0xAB, 0xFB, 0xAC, 0x42, 0xAE, 0xB3, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_63[512] = { + 0xCE, 0xE0, 0xAB, 0xF9, 0xAC, 0x45, 0xCE, 0xD9, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAB, 0xFC, /* 0x04-0x07 */ + 0xAE, 0xB2, 0xAB, 0xF6, 0x00, 0x00, 0xCE, 0xD6, /* 0x08-0x0B */ + 0xCE, 0xDD, 0xCE, 0xD5, 0xCE, 0xD8, 0xCE, 0xDC, /* 0x0C-0x0F */ + 0xD1, 0xB2, 0xAC, 0x44, 0x00, 0x00, 0xCE, 0xE1, /* 0x10-0x13 */ + 0xCE, 0xE2, 0xCE, 0xE4, 0xAB, 0xF5, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0xAE, 0xC1, 0xD1, 0xBE, 0xAE, 0xBF, 0xAE, 0xC0, /* 0x28-0x2B */ + 0xD1, 0xB4, 0xD1, 0xC4, 0x00, 0x00, 0xAE, 0xB6, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0xD5, 0x66, 0xD1, 0xC6, /* 0x30-0x33 */ + 0xD1, 0xC0, 0x00, 0x00, 0xD1, 0xB7, 0x00, 0x00, /* 0x34-0x37 */ + 0xD1, 0xC9, 0xD1, 0xBA, 0xAE, 0xBC, 0xD5, 0x7D, /* 0x38-0x3B */ + 0xD1, 0xBD, 0xAE, 0xBE, 0xAE, 0xB5, 0x00, 0x00, /* 0x3C-0x3F */ + 0xD1, 0xCB, 0xD1, 0xBF, 0xAE, 0xB8, 0xD1, 0xB8, /* 0x40-0x43 */ + 0xD1, 0xB5, 0xD1, 0xB6, 0xAE, 0xB9, 0xD1, 0xC5, /* 0x44-0x47 */ + 0xD1, 0xCC, 0xAE, 0xBB, 0xD1, 0xBC, 0xD1, 0xBB, /* 0x48-0x4B */ + 0xAE, 0xC3, 0xAE, 0xC2, 0xAE, 0xB4, 0xAE, 0xBA, /* 0x4C-0x4F */ + 0xAE, 0xBD, 0xD1, 0xC8, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0xD1, 0xC2, 0xAE, 0xB7, 0xD1, 0xB3, 0xD1, 0xCA, /* 0x54-0x57 */ + 0xD1, 0xC1, 0xD1, 0xC3, 0xD1, 0xC7, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0xD5, 0x67, 0x00, 0x00, 0xB1, 0xB7, /* 0x64-0x67 */ + 0xB1, 0xCB, 0xB1, 0xCA, 0x00, 0x00, 0xB1, 0xBF, /* 0x68-0x6B */ + 0x00, 0x00, 0xD5, 0x79, 0xD5, 0x75, 0xD5, 0x72, /* 0x6C-0x6F */ + 0xD5, 0xA6, 0xB1, 0xBA, 0xB1, 0xB2, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0xD5, 0x77, 0xB4, 0xA8, 0xB1, 0xB6, /* 0x74-0x77 */ + 0xD5, 0xA1, 0x00, 0x00, 0xB1, 0xCC, 0xB1, 0xC9, /* 0x78-0x7B */ + 0xD5, 0x7B, 0xD5, 0x6A, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xB1, 0xC8, 0xD5, 0xA3, 0xD5, 0x69, 0xB1, 0xBD, /* 0x80-0x83 */ + 0xB1, 0xC1, 0xD5, 0xA2, 0x00, 0x00, 0xD5, 0x73, /* 0x84-0x87 */ + 0xB1, 0xC2, 0xB1, 0xBC, 0xD5, 0x68, 0x00, 0x00, /* 0x88-0x8B */ + 0xB4, 0x78, 0xD5, 0xA5, 0xD5, 0x71, 0xB1, 0xC7, /* 0x8C-0x8F */ + 0xD5, 0x74, 0xD5, 0xA4, 0xB1, 0xC6, 0x00, 0x00, /* 0x90-0x93 */ + 0xD9, 0x52, 0x00, 0x00, 0xB1, 0xB3, 0xD5, 0x6F, /* 0x94-0x97 */ + 0xB1, 0xB8, 0xB1, 0xC3, 0x00, 0x00, 0xB1, 0xBE, /* 0x98-0x9B */ + 0xD5, 0x78, 0xD5, 0x6E, 0xD5, 0x6C, 0xD5, 0x7E, /* 0x9C-0x9F */ + 0xB1, 0xB0, 0xB1, 0xC4, 0xB1, 0xB4, 0xB4, 0x77, /* 0xA0-0xA3 */ + 0xD5, 0x7C, 0xB1, 0xB5, 0x00, 0x00, 0xB1, 0xB1, /* 0xA4-0xA7 */ + 0xB1, 0xC0, 0xB1, 0xBB, 0xB1, 0xB9, 0xD5, 0x70, /* 0xA8-0xAB */ + 0xB1, 0xC5, 0xD5, 0x6D, 0xD5, 0x7A, 0xD5, 0x76, /* 0xAC-0xAF */ + 0xD9, 0x54, 0xD9, 0x53, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0xD5, 0x6B, 0xD9, 0x64, 0x00, 0x00, /* 0xBC-0xBF */ + 0xB4, 0x7A, 0x00, 0x00, 0xD9, 0x6A, 0xD9, 0x59, /* 0xC0-0xC3 */ + 0xD9, 0x67, 0xDD, 0x77, 0xB4, 0x7D, 0xD9, 0x6B, /* 0xC4-0xC7 */ + 0xD9, 0x6E, 0xB4, 0x7C, 0xD9, 0x5C, 0xD9, 0x6D, /* 0xC8-0xCB */ + 0xD9, 0x6C, 0xB4, 0x7E, 0xD9, 0x55, 0xB4, 0x79, /* 0xCC-0xCF */ + 0xB4, 0xA3, 0x00, 0x00, 0xB4, 0xA1, 0xD9, 0x69, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xD9, 0x5F, 0xB4, 0xA5, 0xD9, 0x70, /* 0xD4-0xD7 */ + 0xD9, 0x68, 0xD9, 0x71, 0xB4, 0xAD, 0xB4, 0xAB, /* 0xD8-0xDB */ + 0xD9, 0x66, 0xD9, 0x65, 0x00, 0x00, 0xD9, 0x63, /* 0xDC-0xDF */ + 0xD9, 0x5D, 0xB4, 0xA4, 0x00, 0x00, 0xB4, 0xA2, /* 0xE0-0xE3 */ + 0xD1, 0xB9, 0xD9, 0x56, 0x00, 0x00, 0xDD, 0xB7, /* 0xE4-0xE7 */ + 0xD9, 0x57, 0xB4, 0x7B, 0xB4, 0xAA, 0xDD, 0x79, /* 0xE8-0xEB */ + 0x00, 0x00, 0xB4, 0xA6, 0xB4, 0xA7, 0xD9, 0x58, /* 0xEC-0xEF */ + 0xD9, 0x6F, 0xDD, 0x78, 0xD9, 0x60, 0xD9, 0x5B, /* 0xF0-0xF3 */ + 0xB4, 0xA9, 0xD9, 0x61, 0xD9, 0x5E, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xB4, 0xAE, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_64[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0xB7, 0x70, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xDD, 0x7C, 0xDD, 0xB1, 0xDD, 0xB6, /* 0x08-0x0B */ + 0xDD, 0xAA, 0xB7, 0x6C, 0xDD, 0xBB, 0xB7, 0x69, /* 0x0C-0x0F */ + 0xDD, 0x7A, 0x00, 0x00, 0xDD, 0x7B, 0xB7, 0x62, /* 0x10-0x13 */ + 0xB7, 0x6B, 0xDD, 0xA4, 0xB7, 0x6E, 0xB7, 0x6F, /* 0x14-0x17 */ + 0xDD, 0xA5, 0x00, 0x00, 0xDD, 0xB2, 0xDD, 0xB8, /* 0x18-0x1B */ + 0xB7, 0x6A, 0x00, 0x00, 0xB7, 0x64, 0xDD, 0xA3, /* 0x1C-0x1F */ + 0xDD, 0x7D, 0xDD, 0xBA, 0xDD, 0xA8, 0xDD, 0xA9, /* 0x20-0x23 */ + 0xDD, 0x7E, 0xDD, 0xB4, 0xDD, 0xAB, 0xDD, 0xB5, /* 0x24-0x27 */ + 0xDD, 0xAD, 0x00, 0x00, 0xB7, 0x65, 0xE1, 0xD9, /* 0x28-0x2B */ + 0xB7, 0x68, 0xB7, 0x66, 0xDD, 0xB9, 0xDD, 0xB0, /* 0x2C-0x2F */ + 0xDD, 0xAC, 0x00, 0x00, 0x00, 0x00, 0xDD, 0xA1, /* 0x30-0x33 */ + 0xBA, 0x53, 0xDD, 0xAF, 0xB7, 0x6D, 0xDD, 0xA7, /* 0x34-0x37 */ + 0x00, 0x00, 0xDD, 0xA6, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0xB7, 0x67, 0xB7, 0x63, 0xE1, 0xEE, /* 0x3C-0x3F */ + 0xDD, 0xB3, 0xDD, 0xAE, 0x00, 0x00, 0xDD, 0xA2, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE1, 0xE9, /* 0x48-0x4B */ + 0x00, 0x00, 0xE1, 0xDA, 0xE1, 0xE5, 0x00, 0x00, /* 0x4C-0x4F */ + 0xE1, 0xEC, 0xBA, 0x51, 0xB4, 0xAC, 0xE1, 0xEA, /* 0x50-0x53 */ + 0xBA, 0x4C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0xBA, 0x4B, 0xE1, 0xF1, 0x00, 0x00, 0xE1, 0xDB, /* 0x58-0x5B */ + 0xE1, 0xE8, 0xE1, 0xDC, 0xE1, 0xE7, 0xBA, 0x4F, /* 0x5C-0x5F */ + 0xE1, 0xEB, 0xD9, 0x62, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0xE1, 0xF2, 0xE1, 0xE3, 0xBA, 0x52, /* 0x64-0x67 */ + 0xE5, 0xBA, 0xBC, 0xAF, 0x00, 0x00, 0xE1, 0xF0, /* 0x68-0x6B */ + 0xE1, 0xEF, 0xBA, 0x54, 0xE5, 0xAD, 0xBC, 0xB0, /* 0x6C-0x6F */ + 0xE5, 0xAE, 0x00, 0x00, 0xE1, 0xDF, 0xE1, 0xE0, /* 0x70-0x73 */ + 0xE1, 0xDD, 0xE1, 0xE2, 0xE1, 0xDE, 0xE1, 0xF3, /* 0x74-0x77 */ + 0xBA, 0x4E, 0xBC, 0xB1, 0xBA, 0x50, 0xBA, 0x55, /* 0x78-0x7B */ + 0x00, 0x00, 0xE1, 0xE1, 0x00, 0x00, 0xE1, 0xED, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0xE1, 0xE6, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0xE5, 0xB1, 0x00, 0x00, 0xBA, 0x4A, /* 0x84-0x87 */ + 0xBC, 0xB4, 0xE9, 0xAA, 0xE5, 0xB6, 0xE5, 0xB5, /* 0x88-0x8B */ + 0xE5, 0xB7, 0x00, 0x00, 0x00, 0x00, 0xE5, 0xB4, /* 0x8C-0x8F */ + 0xBC, 0xB5, 0x00, 0x00, 0xBC, 0xBB, 0xBC, 0xB8, /* 0x90-0x93 */ + 0x00, 0x00, 0xBC, 0xB9, 0xE5, 0xAF, 0xE5, 0xB2, /* 0x94-0x97 */ + 0xE5, 0xBC, 0xBC, 0xC1, 0xBC, 0xBF, 0x00, 0x00, /* 0x98-0x9B */ + 0xE5, 0xB3, 0xD9, 0x5A, 0xBC, 0xB2, 0xE5, 0xB9, /* 0x9C-0x9F */ + 0xE5, 0xB0, 0x00, 0x00, 0xBC, 0xC2, 0xE5, 0xB8, /* 0xA0-0xA3 */ + 0xBA, 0x4D, 0xBC, 0xB7, 0xE1, 0xE4, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0xBC, 0xBA, 0x00, 0x00, 0xBC, 0xBE, /* 0xA8-0xAB */ + 0xBC, 0xC0, 0xBC, 0xBD, 0xBC, 0xBC, 0x00, 0x00, /* 0xAC-0xAF */ + 0xBC, 0xB6, 0xE5, 0xBB, 0xBC, 0xB3, 0xBC, 0xC3, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBE, 0xD8, /* 0xB8-0xBB */ + 0xBE, 0xD9, 0xE9, 0xA9, 0xBE, 0xE2, 0xBE, 0xDF, /* 0xBC-0xBF */ + 0x00, 0x00, 0xBE, 0xD6, 0xBE, 0xDD, 0xE9, 0xAB, /* 0xC0-0xC3 */ + 0xBE, 0xDB, 0xBE, 0xD5, 0x00, 0x00, 0xBE, 0xDC, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xE9, 0xA8, 0xC0, 0xBB, 0xBE, 0xD7, /* 0xC8-0xCB */ + 0x00, 0x00, 0xBE, 0xDE, 0xC0, 0xBA, 0xE9, 0xA7, /* 0xCC-0xCF */ + 0xE9, 0xA6, 0x00, 0x00, 0xBE, 0xE0, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xBE, 0xE1, 0x00, 0x00, 0xE9, 0xA5, 0xE9, 0xA4, /* 0xD4-0xD7 */ + 0xC0, 0xBC, 0xE9, 0xAE, 0xBE, 0xDA, 0xE9, 0xAC, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0xC0, 0xBD, 0x00, 0x00, 0xC0, 0xC2, 0xEC, 0xEA, /* 0xE0-0xE3 */ + 0xEC, 0xEC, 0x00, 0x00, 0xC0, 0xBF, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xEC, 0xED, 0xEC, 0xE9, 0x00, 0x00, 0xEC, 0xEB, /* 0xE8-0xEB */ + 0xC0, 0xC0, 0xC0, 0xC3, 0x00, 0x00, 0xEC, 0xE8, /* 0xEC-0xEF */ + 0xC0, 0xBE, 0xC0, 0xC1, 0xC2, 0x59, 0xE9, 0xAD, /* 0xF0-0xF3 */ + 0xC2, 0x58, 0x00, 0x00, 0x00, 0x00, 0xC2, 0x5E, /* 0xF4-0xF7 */ + 0xEF, 0xD4, 0x00, 0x00, 0xC2, 0x5C, 0xC2, 0x5D, /* 0xF8-0xFB */ + 0xEF, 0xD7, 0xEF, 0xD3, 0xC2, 0x5A, 0xEF, 0xD1, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_65[512] = { + 0xC3, 0x6B, 0xEF, 0xD5, 0x00, 0x00, 0xEF, 0xD6, /* 0x00-0x03 */ + 0xEF, 0xD2, 0x00, 0x00, 0xC2, 0x5B, 0xF2, 0x42, /* 0x04-0x07 */ + 0x00, 0x00, 0xF2, 0x45, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0xF2, 0x46, 0xF2, 0x44, 0xF2, 0x47, 0xC3, 0x6C, /* 0x0C-0x0F */ + 0xF2, 0x43, 0x00, 0x00, 0x00, 0x00, 0xF4, 0x4E, /* 0x10-0x13 */ + 0xC4, 0x64, 0xF4, 0x4D, 0xF4, 0x4C, 0xF4, 0x4B, /* 0x14-0x17 */ + 0xC4, 0x63, 0xC4, 0x65, 0x00, 0x00, 0xF5, 0xCD, /* 0x18-0x1B */ + 0xC4, 0xE2, 0xC4, 0xE1, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0xF6, 0xE1, 0xF6, 0xE0, 0xF6, 0xE3, 0xC5, 0xCB, /* 0x20-0x23 */ + 0xC5, 0x75, 0xF7, 0xDD, 0xF6, 0xE2, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0xF7, 0xDC, 0xC5, 0xCD, 0xC5, 0xCC, /* 0x28-0x2B */ + 0xC5, 0xF3, 0xF8, 0xA9, 0xF8, 0xEF, 0xA4, 0xE4, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0xD9, 0x72, 0xE9, 0xAF, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0xA6, 0xAC, 0xCA, 0xF7, /* 0x34-0x37 */ + 0xA7, 0xF1, 0xA7, 0xEF, 0x00, 0x00, 0xA7, 0xF0, /* 0x38-0x3B */ + 0x00, 0x00, 0xCC, 0xC1, 0xA9, 0xF1, 0xAC, 0x46, /* 0x3C-0x3F */ + 0x00, 0x00, 0xCE, 0xE7, 0x00, 0x00, 0xCE, 0xE8, /* 0x40-0x43 */ + 0x00, 0x00, 0xAC, 0x47, 0xD1, 0xCE, 0x00, 0x00, /* 0x44-0x47 */ + 0xAE, 0xC4, 0xAE, 0xC5, 0xD1, 0xCD, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB1, 0xD3, /* 0x4C-0x4F */ + 0x00, 0x00, 0xB1, 0xCF, 0x00, 0x00, 0xD5, 0xA7, /* 0x50-0x53 */ + 0xB1, 0xD6, 0xB1, 0xD5, 0xB1, 0xCE, 0xB1, 0xD1, /* 0x54-0x57 */ + 0xB1, 0xD4, 0xB1, 0xD0, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0xD9, 0x76, 0xB1, 0xCD, 0xB4, 0xAF, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0xB4, 0xB1, 0xB4, 0xB2, /* 0x60-0x63 */ + 0xD9, 0x75, 0xD9, 0x78, 0xB4, 0xB0, 0xD9, 0x73, /* 0x64-0x67 */ + 0xD9, 0x77, 0x00, 0x00, 0xD9, 0x74, 0x00, 0x00, /* 0x68-0x6B */ + 0xB7, 0x71, 0x00, 0x00, 0x00, 0x00, 0xDD, 0xBC, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0xBA, 0x56, 0xE1, 0xF4, /* 0x70-0x73 */ + 0xBE, 0xE3, 0xBC, 0xC4, 0xE5, 0xBD, 0xBC, 0xC5, /* 0x74-0x77 */ + 0xBC, 0xC6, 0xE5, 0xBF, 0xE5, 0xBE, 0xE5, 0xC0, /* 0x78-0x7B */ + 0xE9, 0xB1, 0x00, 0x00, 0x00, 0x00, 0xE9, 0xB0, /* 0x7C-0x7F */ + + 0xEC, 0xEF, 0xEC, 0xEE, 0xC0, 0xC4, 0xC0, 0xC5, /* 0x80-0x83 */ + 0xF2, 0x48, 0x00, 0x00, 0x00, 0x00, 0xA4, 0xE5, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0xD9, 0x79, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0xB4, 0xB4, 0xB4, 0xB3, 0xDD, 0xBD, 0x00, 0x00, /* 0x90-0x93 */ + 0xEF, 0xD8, 0xC4, 0xE3, 0xF7, 0xDE, 0xA4, 0xE6, /* 0x94-0x97 */ + 0x00, 0x00, 0xAE, 0xC6, 0x00, 0x00, 0xB1, 0xD8, /* 0x98-0x9B */ + 0xB1, 0xD7, 0xD9, 0x7A, 0xD9, 0x7B, 0xB7, 0x72, /* 0x9C-0x9F */ + 0xE1, 0xF5, 0xBA, 0x57, 0xE9, 0xB2, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xA4, 0xE7, 0xA5, 0xB8, 0x00, 0x00, 0xA9, 0xF2, /* 0xA4-0xA7 */ + 0xCC, 0xC2, 0x00, 0x00, 0xCE, 0xE9, 0xAC, 0x48, /* 0xA8-0xAB */ + 0xB1, 0xD9, 0x00, 0x00, 0xD9, 0x7C, 0xB4, 0xB5, /* 0xAC-0xAF */ + 0xB7, 0x73, 0x00, 0x00, 0xE5, 0xC1, 0xE5, 0xC2, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0xEC, 0xF0, 0xC2, 0x5F, /* 0xB4-0xB7 */ + 0xF8, 0xF0, 0xA4, 0xE8, 0x00, 0x00, 0xCC, 0xC3, /* 0xB8-0xBB */ + 0xA9, 0xF3, 0xAC, 0x49, 0x00, 0x00, 0xCE, 0xEA, /* 0xBC-0xBF */ + 0x00, 0x00, 0xAE, 0xC7, 0xD1, 0xD2, 0xD1, 0xD0, /* 0xC0-0xC3 */ + 0xD1, 0xD1, 0xAE, 0xC8, 0xD1, 0xCF, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB1, 0xDB, /* 0xC8-0xCB */ + 0xB1, 0xDC, 0xD5, 0xA8, 0xB1, 0xDD, 0xB1, 0xDA, /* 0xCC-0xCF */ + 0xD9, 0x7D, 0x00, 0x00, 0xD9, 0x7E, 0xDD, 0xBE, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0xBA, 0x59, 0xBA, 0x58, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xEC, 0xF1, 0xEF, 0xD9, /* 0xD8-0xDB */ + 0x00, 0x00, 0xF2, 0x4A, 0xF2, 0x49, 0xF4, 0x4F, /* 0xDC-0xDF */ + 0x00, 0x00, 0xC9, 0x5E, 0xAC, 0x4A, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xA4, 0xE9, 0xA5, 0xB9, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xA6, 0xAE, 0xA6, 0xAD, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0xA6, 0xAF, 0xA6, 0xB0, 0xC9, 0xEE, 0xC9, 0xED, /* 0xEC-0xEF */ + 0xCA, 0xF8, 0xA7, 0xF2, 0xCA, 0xFB, 0xCA, 0xFA, /* 0xF0-0xF3 */ + 0xCA, 0xF9, 0xCA, 0xFC, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0xA9, 0xF4, 0xCC, 0xC9, /* 0xF8-0xFB */ + 0xCC, 0xC5, 0xCC, 0xCE, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_66[512] = { + 0xA9, 0xFB, 0x00, 0x00, 0xA9, 0xF9, 0xCC, 0xCA, /* 0x00-0x03 */ + 0xCC, 0xC6, 0xCC, 0xCD, 0xA9, 0xF8, 0xAA, 0x40, /* 0x04-0x07 */ + 0xCC, 0xC8, 0xCC, 0xC4, 0xA9, 0xFE, 0xCC, 0xCB, /* 0x08-0x0B */ + 0xA9, 0xF7, 0xCC, 0xCC, 0xA9, 0xFA, 0xA9, 0xFC, /* 0x0C-0x0F */ + 0xCC, 0xD0, 0xCC, 0xCF, 0xCC, 0xC7, 0xA9, 0xF6, /* 0x10-0x13 */ + 0xA9, 0xF5, 0xA9, 0xFD, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0xCE, 0xEF, 0xCE, 0xF5, 0x00, 0x00, 0xAC, 0x50, /* 0x1C-0x1F */ + 0xAC, 0x4D, 0xCE, 0xEC, 0xCE, 0xF1, 0x00, 0x00, /* 0x20-0x23 */ + 0xAC, 0x53, 0xAC, 0x4B, 0xCE, 0xF0, 0xAC, 0x4E, /* 0x24-0x27 */ + 0xAC, 0x51, 0x00, 0x00, 0x00, 0x00, 0xCE, 0xF3, /* 0x28-0x2B */ + 0x00, 0x00, 0xAC, 0x4C, 0xCE, 0xF8, 0xAC, 0x4F, /* 0x2C-0x2F */ + 0x00, 0x00, 0xAC, 0x52, 0xCE, 0xED, 0xCE, 0xF2, /* 0x30-0x33 */ + 0xCE, 0xF6, 0xCE, 0xEE, 0xCE, 0xEB, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0xCE, 0xF7, 0xCE, 0xF4, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0xAE, 0xD0, 0xAE, 0xC9, 0xAE, 0xCC, /* 0x40-0x43 */ + 0x00, 0x00, 0xAE, 0xCF, 0x00, 0x00, 0xD1, 0xD5, /* 0x44-0x47 */ + 0x00, 0x00, 0xAE, 0xCA, 0xD1, 0xD3, 0x00, 0x00, /* 0x48-0x4B */ + 0xAE, 0xCE, 0x00, 0x00, 0x00, 0x00, 0xAE, 0xCB, /* 0x4C-0x4F */ + 0x00, 0x00, 0xD1, 0xD6, 0xAE, 0xCD, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0xD5, 0xAC, 0xB1, 0xDF, 0xD5, 0xAB, /* 0x58-0x5B */ + 0xD5, 0xAD, 0xB1, 0xDE, 0xB1, 0xE3, 0xD1, 0xD4, /* 0x5C-0x5F */ + 0x00, 0x00, 0xD5, 0xAA, 0xD5, 0xAE, 0x00, 0x00, /* 0x60-0x63 */ + 0xB1, 0xE0, 0xD5, 0xA9, 0xB1, 0xE2, 0x00, 0x00, /* 0x64-0x67 */ + 0xB1, 0xE1, 0x00, 0x00, 0xD9, 0xA7, 0x00, 0x00, /* 0x68-0x6B */ + 0xD9, 0xA2, 0x00, 0x00, 0xB4, 0xB6, 0xB4, 0xBA, /* 0x6C-0x6F */ + 0xB4, 0xB7, 0xD9, 0xA5, 0xD9, 0xA8, 0x00, 0x00, /* 0x70-0x73 */ + 0xB4, 0xB8, 0x00, 0x00, 0xB4, 0xB9, 0xB4, 0xBE, /* 0x74-0x77 */ + 0xDD, 0xC7, 0xD9, 0xA6, 0xB4, 0xBC, 0xD9, 0xA3, /* 0x78-0x7B */ + 0xD9, 0xA1, 0x00, 0x00, 0xB4, 0xBD, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xD9, 0xA4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0xB7, 0x79, 0x00, 0x00, 0xDD, 0xBF, 0xB7, 0x76, /* 0x84-0x87 */ + 0xB7, 0x77, 0xB7, 0x75, 0xDD, 0xC4, 0xDD, 0xC3, /* 0x88-0x8B */ + 0xDD, 0xC0, 0xB7, 0x7B, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0xDD, 0xC2, 0xB4, 0xBB, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0xDD, 0xC6, 0xDD, 0xC1, 0xB7, 0x78, 0xB7, 0x74, /* 0x94-0x97 */ + 0xB7, 0x7A, 0xDD, 0xC5, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0xBA, 0x5C, 0x00, 0x00, 0xE1, 0xF8, /* 0x9C-0x9F */ + 0xE1, 0xF7, 0xE1, 0xF6, 0xBA, 0x5A, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xBA, 0x5B, 0xE5, 0xC5, 0xE5, 0xC8, 0xBC, 0xC8, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0xBC, 0xC7, 0xE5, 0xC9, /* 0xAC-0xAF */ + 0xE5, 0xC4, 0xBC, 0xCA, 0xE5, 0xC6, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xBC, 0xC9, 0xE5, 0xC3, 0x00, 0x00, 0xE5, 0xC7, /* 0xB4-0xB7 */ + 0xBE, 0xE9, 0xBE, 0xE6, 0xE9, 0xBB, 0xE9, 0xBA, /* 0xB8-0xBB */ + 0x00, 0x00, 0xE9, 0xB9, 0xE9, 0xB4, 0x00, 0x00, /* 0xBC-0xBF */ + 0xE9, 0xB5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xBE, 0xE7, 0x00, 0x00, 0xBE, 0xE4, 0xBE, 0xE8, /* 0xC4-0xC7 */ + 0xE9, 0xB3, 0xBE, 0xE5, 0xE9, 0xB6, 0xE9, 0xB7, /* 0xC8-0xCB */ + 0xE9, 0xBC, 0x00, 0x00, 0x00, 0x00, 0xE9, 0xB8, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xEC, 0xF2, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0xC0, 0xC7, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xEF, 0xDC, 0xC0, 0xC6, 0xEF, 0xDA, 0xEF, 0xDB, /* 0xD8-0xDB */ + 0xC2, 0x60, 0xC3, 0x6E, 0xF2, 0x4B, 0x00, 0x00, /* 0xDC-0xDF */ + 0xC3, 0x6D, 0x00, 0x00, 0x00, 0x00, 0xF4, 0x51, /* 0xE0-0xE3 */ + 0xF4, 0x52, 0x00, 0x00, 0xC4, 0x66, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xF4, 0x50, 0xC4, 0xE4, 0x00, 0x00, 0xF7, 0xDF, /* 0xE8-0xEB */ + 0xC5, 0xCE, 0xF8, 0xAA, 0xF8, 0xAB, 0x00, 0x00, /* 0xEC-0xEF */ + 0xA4, 0xEA, 0x00, 0x00, 0xA6, 0xB1, 0xA6, 0xB2, /* 0xF0-0xF3 */ + 0xA7, 0xF3, 0x00, 0x00, 0xCC, 0xD1, 0xAC, 0x54, /* 0xF4-0xF7 */ + 0xAE, 0xD1, 0xB1, 0xE4, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0xB0, 0xD2, 0x00, 0x00, 0xB4, 0xBF, 0xB4, 0xC0, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_67[512] = { + 0xB3, 0xCC, 0xD9, 0xA9, 0x00, 0x00, 0xB7, 0x7C, /* 0x00-0x03 */ + 0xE1, 0xFA, 0xE1, 0xF9, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0xA4, 0xEB, 0xA6, 0xB3, 0xCC, 0xD2, 0xAA, 0x42, /* 0x08-0x0B */ + 0x00, 0x00, 0xAA, 0x41, 0x00, 0x00, 0xCE, 0xF9, /* 0x0C-0x0F */ + 0xCE, 0xFA, 0x00, 0x00, 0xD1, 0xD7, 0xD1, 0xD8, /* 0x10-0x13 */ + 0xAE, 0xD2, 0xAE, 0xD3, 0x00, 0x00, 0xAE, 0xD4, /* 0x14-0x17 */ + 0xD5, 0xAF, 0x00, 0x00, 0x00, 0x00, 0xB1, 0xE6, /* 0x18-0x1B */ + 0x00, 0x00, 0xB4, 0xC2, 0x00, 0x00, 0xB4, 0xC1, /* 0x1C-0x1F */ + 0xDD, 0xC8, 0xDF, 0x7A, 0xE1, 0xFB, 0xE9, 0xBD, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0xC2, 0x61, 0xC4, 0x67, /* 0x24-0x27 */ + 0xA4, 0xEC, 0x00, 0x00, 0xA5, 0xBC, 0xA5, 0xBD, /* 0x28-0x2B */ + 0xA5, 0xBB, 0xA5, 0xBE, 0xA5, 0xBA, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0xA6, 0xB6, 0x00, 0x00, 0xC9, 0xF6, /* 0x30-0x33 */ + 0xA6, 0xB5, 0xA6, 0xB7, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0xC9, 0xF1, 0xC9, 0xF0, 0xC9, 0xF3, 0xC9, 0xF2, /* 0x38-0x3B */ + 0xC9, 0xF5, 0xA6, 0xB4, 0xC9, 0xEF, 0xC9, 0xF4, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0xCA, 0xFD, 0xA7, 0xFD, 0xCA, 0xFE, /* 0x44-0x47 */ + 0xCB, 0x43, 0xA7, 0xFC, 0x00, 0x00, 0xCB, 0x47, /* 0x48-0x4B */ + 0xCB, 0x42, 0xCB, 0x45, 0xA7, 0xF5, 0xA7, 0xF6, /* 0x4C-0x4F */ + 0xA7, 0xF7, 0xA7, 0xF8, 0x00, 0x00, 0xA8, 0x40, /* 0x50-0x53 */ + 0x00, 0x00, 0xCB, 0x41, 0xA7, 0xFA, 0xA8, 0x41, /* 0x54-0x57 */ + 0x00, 0x00, 0xCB, 0x40, 0xCB, 0x46, 0x00, 0x00, /* 0x58-0x5B */ + 0xA7, 0xF9, 0xCB, 0x44, 0xA7, 0xFB, 0xA7, 0xF4, /* 0x5C-0x5F */ + 0xA7, 0xFE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0xAA, 0x57, 0x00, 0x00, /* 0x68-0x6B */ + 0xCC, 0xD4, 0xAA, 0x43, 0x00, 0x00, 0xAA, 0x4D, /* 0x6C-0x6F */ + 0xAA, 0x4E, 0xAA, 0x46, 0xAA, 0x58, 0xAA, 0x48, /* 0x70-0x73 */ + 0xCC, 0xDC, 0xAA, 0x53, 0xCC, 0xD7, 0xAA, 0x49, /* 0x74-0x77 */ + 0xCC, 0xE6, 0xCC, 0xE7, 0xCC, 0xDF, 0xCC, 0xD8, /* 0x78-0x7B */ + 0xAA, 0x56, 0xCC, 0xE4, 0xAA, 0x51, 0xAA, 0x4F, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xCC, 0xE5, 0x00, 0x00, 0xCC, 0xE3, /* 0x80-0x83 */ + 0xCC, 0xDB, 0xCC, 0xD3, 0xCC, 0xDA, 0xAA, 0x4A, /* 0x84-0x87 */ + 0x00, 0x00, 0xAA, 0x50, 0x00, 0x00, 0xAA, 0x44, /* 0x88-0x8B */ + 0xCC, 0xDE, 0xCC, 0xDD, 0xCC, 0xD5, 0x00, 0x00, /* 0x8C-0x8F */ + 0xAA, 0x52, 0xCC, 0xE1, 0xCC, 0xD6, 0xAA, 0x55, /* 0x90-0x93 */ + 0xCC, 0xE8, 0xAA, 0x45, 0x00, 0x00, 0xAA, 0x4C, /* 0x94-0x97 */ + 0xCC, 0xD9, 0xCC, 0xE2, 0xAA, 0x54, 0x00, 0x00, /* 0x98-0x9B */ + 0xAA, 0x47, 0xAA, 0x4B, 0x00, 0x00, 0xCC, 0xE0, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0xCF, 0x5B, 0xAC, 0x5C, /* 0xAC-0xAF */ + 0xAC, 0x69, 0x00, 0x00, 0xCF, 0x56, 0xCF, 0x4C, /* 0xB0-0xB3 */ + 0xAC, 0x62, 0xCF, 0x4A, 0xAC, 0x5B, 0xCF, 0x45, /* 0xB4-0xB7 */ + 0xAC, 0x65, 0xCF, 0x52, 0xCE, 0xFE, 0xCF, 0x41, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0xCF, 0x44, 0xCE, 0xFB, 0xCF, 0x51, 0xCF, 0x61, /* 0xC0-0xC3 */ + 0xAC, 0x60, 0xCF, 0x46, 0xCF, 0x58, 0x00, 0x00, /* 0xC4-0xC7 */ + 0xCE, 0xFD, 0xCF, 0x5F, 0xCF, 0x60, 0xCF, 0x63, /* 0xC8-0xCB */ + 0xCF, 0x5A, 0xCF, 0x4B, 0xCF, 0x53, 0xAC, 0x66, /* 0xCC-0xCF */ + 0xAC, 0x59, 0xAC, 0x61, 0xAC, 0x6D, 0xAC, 0x56, /* 0xD0-0xD3 */ + 0xAC, 0x58, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xCF, 0x43, 0xAC, 0x6A, 0xAC, 0x63, 0xCF, 0x5D, /* 0xD8-0xDB */ + 0xCF, 0x40, 0xAC, 0x6C, 0xAC, 0x67, 0xCF, 0x49, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0xAC, 0x6B, 0xCF, 0x50, /* 0xE0-0xE3 */ + 0xCF, 0x48, 0xAC, 0x64, 0xCF, 0x5C, 0xCF, 0x54, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xAC, 0x5E, 0xCF, 0x62, 0xCF, 0x47, /* 0xE8-0xEB */ + 0xAC, 0x5A, 0xCF, 0x59, 0xCF, 0x4F, 0xAC, 0x5F, /* 0xEC-0xEF */ + 0xCF, 0x55, 0xAC, 0x57, 0xCE, 0xFC, 0xAC, 0x68, /* 0xF0-0xF3 */ + 0xAE, 0xE3, 0xAC, 0x5D, 0xCF, 0x4E, 0xCF, 0x4D, /* 0xF4-0xF7 */ + 0xCF, 0x42, 0x00, 0x00, 0xCF, 0x5E, 0x00, 0x00, /* 0xF8-0xFB */ + 0xCF, 0x57, 0x00, 0x00, 0x00, 0x00, 0xAC, 0x55, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_68[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0xD1, 0xEC, 0xAE, 0xEA, /* 0x10-0x13 */ + 0xD1, 0xED, 0x00, 0x00, 0xD1, 0xE1, 0xAE, 0xDF, /* 0x14-0x17 */ + 0xAE, 0xEB, 0x00, 0x00, 0xD1, 0xDA, 0x00, 0x00, /* 0x18-0x1B */ + 0xD1, 0xE3, 0xD1, 0xEB, 0x00, 0x00, 0xD1, 0xD9, /* 0x1C-0x1F */ + 0xD1, 0xF4, 0xAE, 0xD5, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0xD1, 0xF3, 0xD1, 0xEE, 0x00, 0x00, /* 0x24-0x27 */ + 0xD1, 0xEF, 0xAE, 0xDD, 0xAE, 0xE8, 0xD1, 0xE5, /* 0x28-0x2B */ + 0x00, 0x00, 0xD1, 0xE6, 0xD1, 0xF0, 0xD1, 0xE7, /* 0x2C-0x2F */ + 0x00, 0x00, 0xD1, 0xE2, 0xD1, 0xDC, 0xD1, 0xDD, /* 0x30-0x33 */ + 0xD1, 0xEA, 0xD1, 0xE4, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0xAE, 0xD6, 0xAE, 0xDA, 0xD1, 0xF2, 0xD1, 0xDE, /* 0x38-0x3B */ + 0xAE, 0xE6, 0xAE, 0xE2, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0xAE, 0xE5, 0xAE, 0xEC, 0xAE, 0xDB, 0xAE, 0xE7, /* 0x40-0x43 */ + 0xD1, 0xE9, 0xAE, 0xE9, 0xAE, 0xD8, 0x00, 0x00, /* 0x44-0x47 */ + 0xAE, 0xD7, 0xD1, 0xDB, 0x00, 0x00, 0xD1, 0xDF, /* 0x48-0x4B */ + 0xAE, 0xE0, 0xD1, 0xF1, 0xD1, 0xE8, 0xD1, 0xE0, /* 0x4C-0x4F */ + 0xAE, 0xE4, 0xAE, 0xE1, 0x00, 0x00, 0xAE, 0xD9, /* 0x50-0x53 */ + 0xAE, 0xDC, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD5, 0xC4, /* 0x68-0x6B */ + 0x00, 0x00, 0xD5, 0xB4, 0xD5, 0xB5, 0xD5, 0xB9, /* 0x6C-0x6F */ + 0x00, 0x00, 0xD5, 0xC8, 0xD5, 0xC5, 0x00, 0x00, /* 0x70-0x73 */ + 0xD5, 0xBE, 0xD5, 0xBD, 0xB1, 0xED, 0xD5, 0xC1, /* 0x74-0x77 */ + 0xD5, 0xD0, 0xD5, 0xB0, 0x00, 0x00, 0xD5, 0xD1, /* 0x78-0x7B */ + 0xD5, 0xC3, 0xD5, 0xD5, 0xD5, 0xC9, 0xB1, 0xEC, /* 0x7C-0x7F */ + + 0xD5, 0xC7, 0xB1, 0xE7, 0xB1, 0xFC, 0xB1, 0xF2, /* 0x80-0x83 */ + 0x00, 0x00, 0xB1, 0xF6, 0xB1, 0xF5, 0xD5, 0xB1, /* 0x84-0x87 */ + 0x00, 0x00, 0xD5, 0xCE, 0xD5, 0xD4, 0xD5, 0xCC, /* 0x88-0x8B */ + 0xD5, 0xD3, 0x00, 0x00, 0x00, 0x00, 0xD5, 0xC0, /* 0x8C-0x8F */ + 0xD5, 0xB2, 0xD5, 0xD2, 0xD5, 0xC2, 0xB1, 0xEA, /* 0x90-0x93 */ + 0xB1, 0xF7, 0x00, 0x00, 0xD5, 0xCB, 0xB1, 0xF0, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD5, 0xCA, /* 0x98-0x9B */ + 0xD5, 0xB3, 0xB1, 0xF8, 0x00, 0x00, 0xB1, 0xFA, /* 0x9C-0x9F */ + 0xD5, 0xCD, 0xB1, 0xFB, 0xB1, 0xE9, 0xD5, 0xBA, /* 0xA0-0xA3 */ + 0xD5, 0xCF, 0x00, 0x00, 0x00, 0x00, 0xB1, 0xEF, /* 0xA4-0xA7 */ + 0xB1, 0xF9, 0xD5, 0xBC, 0xD5, 0xC6, 0xD5, 0xB7, /* 0xA8-0xAB */ + 0xD5, 0xBB, 0xB1, 0xF4, 0xD5, 0xB6, 0xB1, 0xE8, /* 0xAC-0xAF */ + 0xB1, 0xF1, 0xB1, 0xEE, 0xD5, 0xBF, 0xAE, 0xDE, /* 0xB0-0xB3 */ + 0xD9, 0xC0, 0xB1, 0xEB, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xB1, 0xF3, 0x00, 0x00, 0xD9, 0xC3, 0xD9, 0xD9, /* 0xC4-0xC7 */ + 0xD9, 0xCE, 0xB4, 0xD6, 0x00, 0x00, 0xB4, 0xD1, /* 0xC8-0xCB */ + 0xD9, 0xBD, 0xB4, 0xD2, 0xD9, 0xCD, 0x00, 0x00, /* 0xCC-0xCF */ + 0xD9, 0xC6, 0xD9, 0xD3, 0xB4, 0xCE, 0xD9, 0xAB, /* 0xD0-0xD3 */ + 0xD9, 0xD5, 0xB4, 0xC4, 0xD9, 0xB3, 0xB4, 0xC7, /* 0xD4-0xD7 */ + 0xB4, 0xC6, 0x00, 0x00, 0xB4, 0xD7, 0x00, 0x00, /* 0xD8-0xDB */ + 0xD9, 0xAD, 0xD9, 0xCF, 0xD9, 0xD0, 0xB4, 0xC9, /* 0xDC-0xDF */ + 0xB4, 0xC5, 0xD9, 0xBB, 0x00, 0x00, 0xB4, 0xD0, /* 0xE0-0xE3 */ + 0xD9, 0xB6, 0x00, 0x00, 0xD9, 0xD1, 0xB4, 0xCC, /* 0xE4-0xE7 */ + 0xD9, 0xC9, 0xD9, 0xD6, 0xD9, 0xB0, 0xD9, 0xB5, /* 0xE8-0xEB */ + 0xD9, 0xAF, 0x00, 0x00, 0xB4, 0xCB, 0xD9, 0xC2, /* 0xEC-0xEF */ + 0xDD, 0xDE, 0xD9, 0xB1, 0xB4, 0xCF, 0xD9, 0xBA, /* 0xF0-0xF3 */ + 0xD9, 0xD2, 0xB4, 0xCA, 0xD9, 0xB7, 0xD9, 0xB4, /* 0xF4-0xF7 */ + 0xD9, 0xC5, 0xB4, 0xCD, 0xB4, 0xC3, 0xB4, 0xD9, /* 0xF8-0xFB */ + 0xD9, 0xC8, 0xD9, 0xC7, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_69[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0xD9, 0xAC, 0xB4, 0xC8, 0xD9, 0xD4, 0xD9, 0xBC, /* 0x04-0x07 */ + 0xD9, 0xBE, 0x00, 0x00, 0xD9, 0xCB, 0xD9, 0xCA, /* 0x08-0x0B */ + 0xD9, 0xAA, 0xB4, 0xD3, 0xB4, 0xD5, 0xD9, 0xB2, /* 0x0C-0x0F */ + 0xD9, 0xB9, 0xD9, 0xC1, 0xB4, 0xD4, 0xD9, 0xB8, /* 0x10-0x13 */ + 0xD9, 0xC4, 0xD9, 0xD7, 0x00, 0x00, 0xD9, 0xCC, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0xD9, 0xD8, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xD9, 0xAE, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDD, 0xF2, /* 0x2C-0x2F */ + 0xB7, 0xA6, 0x00, 0x00, 0xDD, 0xF0, 0xDD, 0xDB, /* 0x30-0x33 */ + 0xDD, 0xE0, 0xDD, 0xD9, 0x00, 0x00, 0xDD, 0xEC, /* 0x34-0x37 */ + 0xDD, 0xCB, 0xDD, 0xD2, 0x00, 0x00, 0xDD, 0xEA, /* 0x38-0x3B */ + 0xDD, 0xF4, 0xDD, 0xDC, 0x00, 0x00, 0xDD, 0xCF, /* 0x3C-0x3F */ + 0xDD, 0xE2, 0xDD, 0xE7, 0xDD, 0xD3, 0x00, 0x00, /* 0x40-0x43 */ + 0xDD, 0xE4, 0xDD, 0xD0, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0xDD, 0xD7, 0xDD, 0xD8, 0xB7, 0xA8, 0xDD, 0xEB, /* 0x48-0x4B */ + 0xDD, 0xE9, 0x00, 0x00, 0xDD, 0xCC, 0xDD, 0xEE, /* 0x4C-0x4F */ + 0x00, 0x00, 0xDD, 0xEF, 0xDD, 0xF1, 0xB7, 0xAC, /* 0x50-0x53 */ + 0xB7, 0xA4, 0x00, 0x00, 0xD5, 0xB8, 0xDD, 0xD4, /* 0x54-0x57 */ + 0xDD, 0xE6, 0xDD, 0xD5, 0xB7, 0xA1, 0xB7, 0xB1, /* 0x58-0x5B */ + 0xDD, 0xED, 0xB7, 0xAF, 0xB7, 0xAB, 0xDD, 0xCA, /* 0x5C-0x5F */ + 0xB7, 0xA3, 0x00, 0x00, 0xDD, 0xCD, 0xB7, 0xB0, /* 0x60-0x63 */ + 0x00, 0x00, 0xDD, 0xDD, 0xDD, 0xC9, 0x00, 0x00, /* 0x64-0x67 */ + 0xB7, 0xA9, 0xDD, 0xE1, 0xDD, 0xD1, 0xB7, 0xAA, /* 0x68-0x6B */ + 0xDD, 0xDA, 0xB7, 0x7E, 0xB4, 0xD8, 0xDD, 0xE3, /* 0x6C-0x6F */ + 0xD9, 0xBF, 0xDD, 0xCE, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0xDD, 0xE8, 0xB7, 0xA5, 0xDD, 0xE5, 0xB7, 0xA2, /* 0x74-0x77 */ + 0xDD, 0xDF, 0xB7, 0xAD, 0xDD, 0xD6, 0xDD, 0xF3, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0xB7, 0xA7, 0xDE, 0xC6, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0xB7, 0xAE, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0xE2, 0x4A, 0xE2, 0x48, 0x00, 0x00, /* 0x8C-0x8F */ + 0xE2, 0x5E, 0xE2, 0x46, 0x00, 0x00, 0xE2, 0x58, /* 0x90-0x93 */ + 0xB7, 0x7D, 0xBA, 0x5F, 0xE2, 0x42, 0xE2, 0x5D, /* 0x94-0x97 */ + 0x00, 0x00, 0xE2, 0x47, 0xE2, 0x55, 0xBA, 0x64, /* 0x98-0x9B */ + 0xBA, 0x5D, 0x00, 0x00, 0xE2, 0x5B, 0x00, 0x00, /* 0x9C-0x9F */ + 0xE2, 0x40, 0xE2, 0x5A, 0x00, 0x00, 0xBA, 0x6F, /* 0xA0-0xA3 */ + 0xE2, 0x51, 0xE2, 0x61, 0xBA, 0x6D, 0xE2, 0x49, /* 0xA4-0xA7 */ + 0xBA, 0x5E, 0xE2, 0x4B, 0xE2, 0x59, 0xBA, 0x67, /* 0xA8-0xAB */ + 0xE2, 0x44, 0xBA, 0x6B, 0xBA, 0x61, 0xE2, 0x4D, /* 0xAC-0xAF */ + 0xE2, 0x43, 0xE1, 0xFC, 0x00, 0x00, 0xE2, 0x57, /* 0xB0-0xB3 */ + 0xBA, 0x68, 0xE2, 0x60, 0xE1, 0xFD, 0xBA, 0x65, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xE2, 0x53, 0x00, 0x00, 0xBA, 0x66, /* 0xB8-0xBB */ + 0xE2, 0x45, 0xE2, 0x50, 0xE2, 0x4C, 0xE2, 0x4E, /* 0xBC-0xBF */ + 0x00, 0x00, 0xBA, 0x60, 0xE2, 0x5F, 0xBA, 0x6E, /* 0xC0-0xC3 */ + 0xE2, 0x4F, 0x00, 0x00, 0xE2, 0x62, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xE1, 0xFE, 0xE2, 0x54, 0xBA, 0x63, /* 0xC8-0xCB */ + 0xBA, 0x6C, 0xBA, 0x6A, 0xE2, 0x41, 0xE2, 0x56, /* 0xCC-0xCF */ + 0xBA, 0x69, 0x00, 0x00, 0x00, 0x00, 0xBA, 0x62, /* 0xD0-0xD3 */ + 0xE2, 0x52, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xE2, 0x5C, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0xE5, 0xD5, 0x00, 0x00, /* 0xE0-0xE3 */ + 0xE5, 0xD1, 0xE5, 0xCD, 0xE5, 0xE1, 0xE5, 0xDE, /* 0xE4-0xE7 */ + 0xBC, 0xCD, 0x00, 0x00, 0x00, 0x00, 0xE5, 0xE5, /* 0xE8-0xEB */ + 0xE5, 0xD4, 0xBC, 0xD8, 0xE5, 0xDB, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0xE5, 0xD0, 0xE5, 0xDA, 0xBC, 0xD5, /* 0xF0-0xF3 */ + 0xE5, 0xEE, 0x00, 0x00, 0xE5, 0xEB, 0xE5, 0xDD, /* 0xF4-0xF7 */ + 0xE5, 0xCE, 0x00, 0x00, 0x00, 0x00, 0xE5, 0xE2, /* 0xF8-0xFB */ + 0xE5, 0xE4, 0xBC, 0xD1, 0xE5, 0xD8, 0xE5, 0xD3, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_6A[512] = { + 0xE5, 0xCA, 0xBC, 0xCE, 0xBC, 0xD6, 0x00, 0x00, /* 0x00-0x03 */ + 0xE5, 0xE7, 0xBC, 0xD7, 0xE5, 0xCB, 0xE5, 0xED, /* 0x04-0x07 */ + 0xE5, 0xE0, 0xE5, 0xE6, 0xBC, 0xD4, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0xE5, 0xE3, 0x00, 0x00, 0xE5, 0xEA, /* 0x0C-0x0F */ + 0x00, 0x00, 0xBC, 0xD9, 0x00, 0x00, 0xBC, 0xD3, /* 0x10-0x13 */ + 0xE5, 0xDC, 0xE5, 0xCF, 0xE5, 0xEF, 0xE5, 0xCC, /* 0x14-0x17 */ + 0xE5, 0xE8, 0xBC, 0xD0, 0x00, 0x00, 0xE5, 0xD6, /* 0x18-0x1B */ + 0x00, 0x00, 0xE5, 0xD7, 0xBC, 0xCF, 0xBC, 0xCC, /* 0x1C-0x1F */ + 0xE5, 0xD2, 0xBC, 0xD2, 0x00, 0x00, 0xBC, 0xCB, /* 0x20-0x23 */ + 0x00, 0x00, 0xE5, 0xE9, 0xE5, 0xEC, 0xE5, 0xD9, /* 0x24-0x27 */ + 0xE9, 0xCA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xC2, 0x00, 0x00, /* 0x30-0x33 */ + 0xE9, 0xBE, 0xBE, 0xF6, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0xBE, 0xEB, 0xBE, 0xF0, 0xBE, 0xEC, 0xE9, 0xCC, /* 0x38-0x3B */ + 0xE9, 0xD7, 0xBE, 0xEA, 0xE9, 0xC4, 0xE9, 0xCD, /* 0x3C-0x3F */ + 0xE5, 0xDF, 0xE9, 0xCE, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0xBE, 0xF1, 0x00, 0x00, 0xE9, 0xDD, 0xBE, 0xF5, /* 0x44-0x47 */ + 0xBE, 0xF8, 0xE9, 0xC0, 0x00, 0x00, 0xBE, 0xF4, /* 0x48-0x4B */ + 0x00, 0x00, 0xE9, 0xDB, 0xE9, 0xDC, 0xE9, 0xD2, /* 0x4C-0x4F */ + 0xE9, 0xD1, 0xE9, 0xC9, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0xE9, 0xD3, 0xE9, 0xDA, 0xE9, 0xD9, 0x00, 0x00, /* 0x54-0x57 */ + 0xBE, 0xEF, 0xBE, 0xED, 0xE9, 0xCB, 0xE9, 0xC8, /* 0x58-0x5B */ + 0x00, 0x00, 0xE9, 0xC5, 0xE9, 0xD8, 0xBE, 0xF7, /* 0x5C-0x5F */ + 0xE9, 0xD6, 0xBE, 0xF3, 0xBE, 0xF2, 0x00, 0x00, /* 0x60-0x63 */ + 0xE9, 0xD0, 0x00, 0x00, 0xE9, 0xBF, 0xE9, 0xC1, /* 0x64-0x67 */ + 0xE9, 0xC3, 0xE9, 0xD5, 0xE9, 0xCF, 0xBE, 0xEE, /* 0x68-0x6B */ + 0x00, 0x00, 0xE9, 0xC6, 0x00, 0x00, 0xE9, 0xD4, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xC7, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0xC0, 0xCF, 0xED, 0x45, /* 0x7C-0x7F */ + + 0xC0, 0xC8, 0xEC, 0xF5, 0x00, 0x00, 0xED, 0x41, /* 0x80-0x83 */ + 0xC0, 0xCA, 0xED, 0x48, 0x00, 0x00, 0xEC, 0xFC, /* 0x84-0x87 */ + 0x00, 0x00, 0xEC, 0xF7, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0xED, 0x49, 0xEC, 0xF3, 0xEC, 0xFE, 0x00, 0x00, /* 0x8C-0x8F */ + 0xC0, 0xD1, 0xED, 0x44, 0xED, 0x4A, 0xEC, 0xFD, /* 0x90-0x93 */ + 0xC0, 0xC9, 0xED, 0x40, 0xEC, 0xF4, 0xC0, 0xD0, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0xED, 0x47, 0xEC, 0xF9, /* 0x98-0x9B */ + 0xC0, 0xCC, 0x00, 0x00, 0xEC, 0xFB, 0xEC, 0xF8, /* 0x9C-0x9F */ + 0xC0, 0xD2, 0xEC, 0xFA, 0xC0, 0xCB, 0xC0, 0xCE, /* 0xA0-0xA3 */ + 0xED, 0x43, 0xEC, 0xF6, 0xED, 0x46, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xED, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0xC2, 0x63, 0xEF, 0xE7, 0xC2, 0x68, 0xC2, 0x69, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC2, 0x62, /* 0xB0-0xB3 */ + 0xEF, 0xE6, 0x00, 0x00, 0xEF, 0xE3, 0xEF, 0xE4, /* 0xB4-0xB7 */ + 0xC2, 0x66, 0xEF, 0xDE, 0xEF, 0xE2, 0xC2, 0x65, /* 0xB8-0xBB */ + 0x00, 0x00, 0xEF, 0xDF, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0xC2, 0x67, 0xC2, 0x64, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xEF, 0xDD, 0xEF, 0xE1, 0xEF, 0xE5, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF2, 0x51, /* 0xC8-0xCB */ + 0xF2, 0x4E, 0xF2, 0x57, 0x00, 0x00, 0xF2, 0x56, /* 0xCC-0xCF */ + 0xF2, 0x54, 0xF2, 0x4F, 0x00, 0x00, 0xC3, 0x72, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xF2, 0x50, 0xC3, 0x71, 0xC0, 0xCD, /* 0xD8-0xDB */ + 0xF2, 0x53, 0xC3, 0x70, 0xF2, 0x58, 0xF2, 0x52, /* 0xDC-0xDF */ + 0xF2, 0x4D, 0xEF, 0xE0, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xC3, 0x6F, 0x00, 0x00, 0xF2, 0x4C, /* 0xE4-0xE7 */ + 0xF4, 0x56, 0x00, 0x00, 0xF4, 0x55, 0xF2, 0x55, /* 0xE8-0xEB */ + 0xC4, 0x68, 0x00, 0x00, 0xF4, 0x59, 0xF4, 0x5A, /* 0xEC-0xEF */ + 0xF4, 0x54, 0xF4, 0x58, 0x00, 0x00, 0xF4, 0x53, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xF5, 0xD1, 0xF4, 0x57, 0xC4, 0xE7, 0xC4, 0xE5, /* 0xF8-0xFB */ + 0xF5, 0xCF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_6B[512] = { + 0xF5, 0xD2, 0x00, 0x00, 0xF5, 0xCE, 0xF5, 0xD0, /* 0x00-0x03 */ + 0xC4, 0xE6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0xF6, 0xE5, 0xF6, 0xE6, 0xC5, 0x76, 0xF6, 0xE4, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF7, 0xE2, /* 0x0C-0x0F */ + 0xC5, 0xCF, 0xF7, 0xE0, 0xF7, 0xE1, 0xF8, 0xAC, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xC6, 0x56, 0xF8, 0xF3, /* 0x14-0x17 */ + 0xF8, 0xF1, 0xF8, 0xF2, 0xF8, 0xF4, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xF9, 0xBB, 0x00, 0x00, /* 0x1C-0x1F */ + 0xA4, 0xED, 0xA6, 0xB8, 0x00, 0x00, 0xAA, 0x59, /* 0x20-0x23 */ + 0x00, 0x00, 0xCC, 0xE9, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0xCF, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0xD1, 0xF5, 0xD1, 0xF7, 0x00, 0x00, 0xD1, 0xF6, /* 0x2C-0x2F */ + 0x00, 0x00, 0xD1, 0xF8, 0xB1, 0xFD, 0xD5, 0xD7, /* 0x30-0x33 */ + 0xD1, 0xF9, 0x00, 0x00, 0xD5, 0xD6, 0xD5, 0xD8, /* 0x34-0x37 */ + 0xD5, 0xD9, 0xD9, 0xDA, 0xB4, 0xDB, 0xD9, 0xDB, /* 0x38-0x3B */ + 0xD9, 0xDD, 0xB4, 0xDC, 0xB4, 0xDA, 0xD9, 0xDC, /* 0x3C-0x3F */ + 0x00, 0x00, 0xDD, 0xFA, 0xDD, 0xF8, 0xDD, 0xF7, /* 0x40-0x43 */ + 0x00, 0x00, 0xDD, 0xF6, 0xDD, 0xF5, 0xB7, 0xB2, /* 0x44-0x47 */ + 0xDD, 0xF9, 0xBA, 0x70, 0xE2, 0x63, 0xE2, 0x65, /* 0x48-0x4B */ + 0xBA, 0x71, 0xE2, 0x64, 0xBC, 0xDB, 0x00, 0x00, /* 0x4C-0x4F */ + 0xBC, 0xDA, 0xE5, 0xF0, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0xE9, 0xDF, 0xE9, 0xDE, 0xE9, 0xE0, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0xBE, 0xF9, 0x00, 0x00, 0xED, 0x4B, /* 0x58-0x5B */ + 0xC0, 0xD3, 0x00, 0x00, 0xEF, 0xE8, 0xC2, 0x6A, /* 0x5C-0x5F */ + 0xF2, 0x59, 0xC5, 0x77, 0xA4, 0xEE, 0xA5, 0xBF, /* 0x60-0x63 */ + 0xA6, 0xB9, 0xA8, 0x42, 0xAA, 0x5A, 0xAA, 0x5B, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0xAC, 0x6E, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0xD1, 0xFA, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0xB7, 0xB3, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0xD1, 0xBE, 0xFA, /* 0x74-0x77 */ + 0xC2, 0x6B, 0xA4, 0xEF, 0x00, 0x00, 0xA6, 0xBA, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0xCC, 0xEB, 0xAA, 0x5C, /* 0x7C-0x7F */ + + 0xCC, 0xEA, 0x00, 0x00, 0xCF, 0x65, 0xAC, 0x6F, /* 0x80-0x83 */ + 0xCF, 0x66, 0x00, 0x00, 0xAC, 0x70, 0x00, 0x00, /* 0x84-0x87 */ + 0xD1, 0xFC, 0xAE, 0xEE, 0xAE, 0xED, 0x00, 0x00, /* 0x88-0x8B */ + 0xD5, 0xDE, 0xD5, 0xDC, 0xD5, 0xDD, 0xD5, 0xDB, /* 0x8C-0x8F */ + 0x00, 0x00, 0xD5, 0xDA, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0xD9, 0xDE, 0xD9, 0xE1, 0xB4, 0xDE, 0xD9, 0xDF, /* 0x94-0x97 */ + 0xB4, 0xDD, 0xD9, 0xE0, 0x00, 0x00, 0xDD, 0xFB, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0x66, 0xE2, 0x67, /* 0x9C-0x9F */ + 0xE2, 0x68, 0x00, 0x00, 0xE5, 0xF3, 0xE5, 0xF2, /* 0xA0-0xA3 */ + 0xBC, 0xDC, 0xE5, 0xF1, 0xE5, 0xF4, 0xE9, 0xE1, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xE2, 0xE9, 0xE3, /* 0xA8-0xAB */ + 0x00, 0x00, 0xED, 0x4C, 0xC0, 0xD4, 0xC2, 0x6C, /* 0xAC-0xAF */ + 0xF2, 0x5A, 0x00, 0x00, 0xC4, 0xE8, 0xC9, 0x5F, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xAC, 0x71, 0xCF, 0x67, 0xAE, 0xEF, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0xB1, 0xFE, 0x00, 0x00, /* 0xB8-0xBB */ + 0xB4, 0xDF, 0xD9, 0xE2, 0x00, 0x00, 0xB7, 0xB5, /* 0xBC-0xBF */ + 0xB7, 0xB4, 0x00, 0x00, 0x00, 0x00, 0xE2, 0x69, /* 0xC0-0xC3 */ + 0xE2, 0x6A, 0xBC, 0xDD, 0xBC, 0xDE, 0xE9, 0xE5, /* 0xC4-0xC7 */ + 0xE9, 0xE4, 0xEF, 0xE9, 0xF7, 0xE3, 0xA4, 0xF0, /* 0xC8-0xCB */ + 0xC9, 0x60, 0xA5, 0xC0, 0x00, 0x00, 0xA8, 0x43, /* 0xCC-0xCF */ + 0xCB, 0x48, 0x00, 0x00, 0xAC, 0x72, 0xB7, 0xB6, /* 0xD0-0xD3 */ + 0xA4, 0xF1, 0x00, 0x00, 0xCF, 0x68, 0xAC, 0x73, /* 0xD4-0xD7 */ + 0xCF, 0x69, 0x00, 0x00, 0xC0, 0xD5, 0xA4, 0xF2, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0xCC, 0xEC, 0x00, 0x00, /* 0xDC-0xDF */ + 0xCF, 0x6A, 0x00, 0x00, 0xD2, 0x42, 0xD2, 0x41, /* 0xE0-0xE3 */ + 0xD1, 0xFE, 0x00, 0x00, 0xD1, 0xFD, 0xD2, 0x43, /* 0xE4-0xE7 */ + 0xD2, 0x40, 0x00, 0x00, 0x00, 0x00, 0xB2, 0x40, /* 0xE8-0xEB */ + 0xB2, 0x41, 0x00, 0x00, 0x00, 0x00, 0xB4, 0xE0, /* 0xEC-0xEF */ + 0xD9, 0xE3, 0x00, 0x00, 0xD9, 0xE4, 0xD9, 0xE5, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDE, 0x41, /* 0xF4-0xF7 */ + 0xDE, 0x42, 0xDE, 0x40, 0x00, 0x00, 0xDD, 0xFD, /* 0xF8-0xFB */ + 0xDD, 0xFE, 0xB7, 0xB7, 0xE2, 0x6B, 0xE5, 0xF7, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_6C[512] = { + 0xE5, 0xF6, 0xE5, 0xF5, 0xE5, 0xF8, 0xE9, 0xE7, /* 0x00-0x03 */ + 0xE9, 0xE6, 0xBE, 0xFB, 0xE9, 0xE8, 0x00, 0x00, /* 0x04-0x07 */ + 0xC0, 0xD6, 0xED, 0x4D, 0x00, 0x00, 0xEF, 0xEA, /* 0x08-0x0B */ + 0xF2, 0x5B, 0xF6, 0xE7, 0x00, 0x00, 0xA4, 0xF3, /* 0x0C-0x0F */ + 0xA5, 0xC2, 0xA5, 0xC1, 0x00, 0x00, 0xAA, 0x5D, /* 0x10-0x13 */ + 0xC9, 0x61, 0xC9, 0x7E, 0xA6, 0xBB, 0x00, 0x00, /* 0x14-0x17 */ + 0xC9, 0xF7, 0xCB, 0x49, 0xCB, 0x4A, 0xAA, 0x5E, /* 0x18-0x1B */ + 0x00, 0x00, 0xCC, 0xED, 0x00, 0x00, 0xAC, 0x74, /* 0x1C-0x1F */ + 0xCF, 0x6B, 0xCF, 0x6C, 0x00, 0x00, 0xAE, 0xF0, /* 0x20-0x23 */ + 0xAE, 0xF4, 0xD2, 0x44, 0xAE, 0xF3, 0xAE, 0xF1, /* 0x24-0x27 */ + 0xAE, 0xF2, 0x00, 0x00, 0xD5, 0xDF, 0xB2, 0x42, /* 0x28-0x2B */ + 0xB4, 0xE3, 0x00, 0x00, 0xB4, 0xE1, 0xB4, 0xE2, /* 0x2C-0x2F */ + 0xD9, 0xE6, 0x00, 0x00, 0x00, 0x00, 0xBA, 0x72, /* 0x30-0x33 */ + 0xA4, 0xF4, 0x00, 0x00, 0xC9, 0xA1, 0x00, 0x00, /* 0x34-0x37 */ + 0xA5, 0xC3, 0x00, 0x00, 0x00, 0x00, 0xC9, 0xA4, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xA5, 0xC6, 0xC9, 0xA3, /* 0x3C-0x3F */ + 0xA5, 0xC5, 0xA5, 0xC4, 0xA8, 0x44, 0xC9, 0xA2, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xC9, 0xF8, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0xC9, 0xFC, 0xC9, 0xFE, /* 0x48-0x4B */ + 0xCA, 0x40, 0xA6, 0xC5, 0xA6, 0xC6, 0xC9, 0xFB, /* 0x4C-0x4F */ + 0xA6, 0xC1, 0x00, 0x00, 0xC9, 0xF9, 0x00, 0x00, /* 0x50-0x53 */ + 0xC9, 0xFD, 0xA6, 0xC2, 0x00, 0x00, 0xA6, 0xBD, /* 0x54-0x57 */ + 0x00, 0x00, 0xA6, 0xBE, 0x00, 0x00, 0xA6, 0xC4, /* 0x58-0x5B */ + 0xC9, 0xFA, 0xA6, 0xBC, 0xA8, 0x45, 0xA6, 0xBF, /* 0x5C-0x5F */ + 0xA6, 0xC0, 0xA6, 0xC3, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0xCB, 0x5B, 0xCB, 0x59, 0xCB, 0x4C, /* 0x64-0x67 */ + 0xA8, 0x51, 0xCB, 0x53, 0xA8, 0x4C, 0xCB, 0x4D, /* 0x68-0x6B */ + 0x00, 0x00, 0xCB, 0x55, 0x00, 0x00, 0xCB, 0x52, /* 0x6C-0x6F */ + 0xA8, 0x4F, 0xCB, 0x51, 0xA8, 0x56, 0xCB, 0x5A, /* 0x70-0x73 */ + 0xA8, 0x58, 0x00, 0x00, 0xA8, 0x5A, 0x00, 0x00, /* 0x74-0x77 */ + 0xCB, 0x4B, 0x00, 0x00, 0xA8, 0x4D, 0xCB, 0x5C, /* 0x78-0x7B */ + 0x00, 0x00, 0xA8, 0x54, 0xA8, 0x57, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xCD, 0x45, 0xA8, 0x47, 0xA8, 0x5E, 0xA8, 0x55, /* 0x80-0x83 */ + 0xCB, 0x4E, 0xA8, 0x4A, 0xA8, 0x59, 0xCB, 0x56, /* 0x84-0x87 */ + 0xA8, 0x48, 0xA8, 0x49, 0xCD, 0x43, 0xCB, 0x4F, /* 0x88-0x8B */ + 0xA8, 0x50, 0xA8, 0x5B, 0xCB, 0x5D, 0xCB, 0x50, /* 0x8C-0x8F */ + 0xA8, 0x4E, 0x00, 0x00, 0xA8, 0x53, 0xCC, 0xEE, /* 0x90-0x93 */ + 0xA8, 0x5C, 0xCB, 0x57, 0xA8, 0x52, 0x00, 0x00, /* 0x94-0x97 */ + 0xA8, 0x5D, 0xA8, 0x46, 0xCB, 0x54, 0xA8, 0x4B, /* 0x98-0x9B */ + 0xCB, 0x58, 0xCD, 0x44, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAA, 0x6A, /* 0xA8-0xAB */ + 0xAA, 0x7A, 0xCC, 0xF5, 0xAA, 0x71, 0x00, 0x00, /* 0xAC-0xAF */ + 0xCD, 0x4B, 0xAA, 0x62, 0x00, 0x00, 0xAA, 0x65, /* 0xB0-0xB3 */ + 0xCD, 0x42, 0x00, 0x00, 0xCC, 0xF3, 0xCC, 0xF7, /* 0xB4-0xB7 */ + 0xAA, 0x6D, 0xAA, 0x6F, 0xCC, 0xFA, 0xAA, 0x76, /* 0xB8-0xBB */ + 0xAA, 0x68, 0xAA, 0x66, 0xAA, 0x67, 0xAA, 0x75, /* 0xBC-0xBF */ + 0xCD, 0x47, 0xAA, 0x70, 0xCC, 0xF9, 0xCC, 0xFB, /* 0xC0-0xC3 */ + 0xAA, 0x6E, 0xAA, 0x73, 0xCC, 0xFC, 0xCD, 0x4A, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xAC, 0x75, 0xAA, 0x79, 0x00, 0x00, /* 0xC8-0xCB */ + 0xAA, 0x63, 0xCD, 0x49, 0x00, 0x00, 0xCD, 0x4D, /* 0xCC-0xCF */ + 0xCC, 0xF8, 0xCD, 0x4F, 0xCD, 0x40, 0xAA, 0x6C, /* 0xD0-0xD3 */ + 0xCC, 0xF4, 0xAA, 0x6B, 0xAA, 0x7D, 0xAA, 0x72, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xCC, 0xF2, 0xCF, 0x75, 0xAA, 0x78, /* 0xD8-0xDB */ + 0xAA, 0x7C, 0xCD, 0x41, 0xCD, 0x46, 0x00, 0x00, /* 0xDC-0xDF */ + 0xAA, 0x7E, 0xAA, 0x77, 0xAA, 0x69, 0xAA, 0x5F, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xAA, 0x64, 0x00, 0x00, 0xCC, 0xF6, /* 0xE4-0xE7 */ + 0xAA, 0x60, 0xCD, 0x4E, 0x00, 0x00, 0xCC, 0xF0, /* 0xE8-0xEB */ + 0xCC, 0xEF, 0xCC, 0xFD, 0xCC, 0xF1, 0xAA, 0x7B, /* 0xEC-0xEF */ + 0xAE, 0xF5, 0xAA, 0x74, 0xCC, 0xFE, 0xAA, 0x61, /* 0xF0-0xF3 */ + 0x00, 0x00, 0xAC, 0xA6, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xCD, 0x4C, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_6D[512] = { + 0xCF, 0x7C, 0xCF, 0xA1, 0x00, 0x00, 0xCF, 0xA4, /* 0x00-0x03 */ + 0xCF, 0x77, 0x00, 0x00, 0x00, 0x00, 0xCF, 0xA7, /* 0x04-0x07 */ + 0xCF, 0xAA, 0xCF, 0xAC, 0xCF, 0x74, 0xAC, 0x76, /* 0x08-0x0B */ + 0xAC, 0x7B, 0xD2, 0x49, 0xAC, 0xAD, 0xCF, 0xA5, /* 0x0C-0x0F */ + 0xCF, 0xAD, 0xCF, 0x7B, 0xCF, 0x73, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xD2, 0x64, 0xAC, 0x7E, /* 0x14-0x17 */ + 0xCF, 0xA2, 0xCF, 0x78, 0xCF, 0x7A, 0xAC, 0xA5, /* 0x18-0x1B */ + 0x00, 0x00, 0xCF, 0x7D, 0xAC, 0x7D, 0xCF, 0x70, /* 0x1C-0x1F */ + 0xCF, 0xA8, 0x00, 0x00, 0xCF, 0xAB, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0xAC, 0x7A, 0x00, 0x00, 0xAC, 0xA8, /* 0x24-0x27 */ + 0xCF, 0x6D, 0xAC, 0xAA, 0xAC, 0x78, 0xAC, 0xAE, /* 0x28-0x2B */ + 0xCF, 0xA9, 0xCF, 0x6F, 0xAC, 0xAB, 0xD2, 0x5E, /* 0x2C-0x2F */ + 0xCD, 0x48, 0xAC, 0x7C, 0xAC, 0x77, 0xCF, 0x76, /* 0x30-0x33 */ + 0xCF, 0x6E, 0xAC, 0xAC, 0xAC, 0xA4, 0xCF, 0xA3, /* 0x34-0x37 */ + 0xAC, 0xA9, 0xAC, 0xA7, 0xCF, 0x79, 0xAC, 0xA1, /* 0x38-0x3B */ + 0xCF, 0x71, 0xAC, 0xA2, 0xAC, 0xA3, 0xCF, 0x72, /* 0x3C-0x3F */ + 0xCF, 0xA6, 0xAC, 0x79, 0xCF, 0x7E, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0xD2, 0x4C, 0xAE, 0xFD, 0xAF, 0x43, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0xD2, 0x55, 0xD2, 0x5B, /* 0x5C-0x5F */ + 0xD2, 0x57, 0xD2, 0x4A, 0xD2, 0x4D, 0xD2, 0x46, /* 0x60-0x63 */ + 0xD2, 0x47, 0xAF, 0x4A, 0xAE, 0xFA, 0xD2, 0x56, /* 0x64-0x67 */ + 0xD2, 0x5F, 0xAF, 0x45, 0xAE, 0xF6, 0x00, 0x00, /* 0x68-0x6B */ + 0xAF, 0x40, 0xD2, 0x4E, 0xAF, 0x42, 0xD2, 0x4F, /* 0x6C-0x6F */ + 0xD2, 0x59, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0xAF, 0x44, 0xD2, 0x68, 0xD2, 0x48, 0xAE, 0xFC, /* 0x74-0x77 */ + 0xAE, 0xFB, 0xAF, 0x48, 0xD2, 0x45, 0xD2, 0x66, /* 0x78-0x7B */ + 0xD2, 0x5A, 0xD2, 0x67, 0xD2, 0x61, 0xD2, 0x53, /* 0x7C-0x7F */ + + 0xD2, 0x62, 0x00, 0x00, 0xD2, 0x5C, 0xD2, 0x65, /* 0x80-0x83 */ + 0xD2, 0x63, 0xAF, 0x49, 0xD2, 0x54, 0xAE, 0xF9, /* 0x84-0x87 */ + 0xAE, 0xF8, 0xAF, 0x41, 0xAF, 0x47, 0xD2, 0x60, /* 0x88-0x8B */ + 0xAF, 0x46, 0xD2, 0x51, 0xB2, 0x43, 0x00, 0x00, /* 0x8C-0x8F */ + 0xD2, 0x69, 0xD2, 0x50, 0xD2, 0x4B, 0xAE, 0xFE, /* 0x90-0x93 */ + 0xAF, 0x4B, 0xAE, 0xF7, 0x00, 0x00, 0xD2, 0x58, /* 0x94-0x97 */ + 0xD2, 0x5D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0xB2, 0x65, 0xD5, 0xE1, /* 0xA8-0xAB */ + 0xD5, 0xE5, 0x00, 0x00, 0xB2, 0x52, 0xB2, 0x50, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xB2, 0x47, 0xD5, 0xE3, /* 0xB0-0xB3 */ + 0xD5, 0xE2, 0xB2, 0x5B, 0x00, 0x00, 0xD5, 0xE8, /* 0xB4-0xB7 */ + 0xB2, 0x55, 0x00, 0x00, 0xD5, 0xFA, 0xD6, 0x47, /* 0xB8-0xBB */ + 0xB2, 0x44, 0xD5, 0xF7, 0xD5, 0xF0, 0xB2, 0x67, /* 0xBC-0xBF */ + 0xD5, 0xE0, 0x00, 0x00, 0xD5, 0xFC, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xB2, 0x64, 0xB2, 0x58, 0xB2, 0x63, 0xB2, 0x4E, /* 0xC4-0xC7 */ + 0xD5, 0xEC, 0xD5, 0xFE, 0xD5, 0xF6, 0xB2, 0x4F, /* 0xC8-0xCB */ + 0xB2, 0x49, 0xD6, 0x45, 0x00, 0x00, 0xD5, 0xFD, /* 0xCC-0xCF */ + 0xD6, 0x40, 0xB2, 0x51, 0xB2, 0x59, 0xD6, 0x42, /* 0xD0-0xD3 */ + 0xD5, 0xEA, 0xD5, 0xFB, 0xD5, 0xEF, 0xD6, 0x44, /* 0xD4-0xD7 */ + 0xB2, 0x5E, 0xB2, 0x46, 0xB2, 0x5C, 0xD5, 0xF4, /* 0xD8-0xDB */ + 0xD5, 0xF2, 0xD5, 0xF3, 0xB2, 0x53, 0xD5, 0xEE, /* 0xDC-0xDF */ + 0xD5, 0xED, 0xB2, 0x48, 0xD5, 0xE7, 0xD6, 0x46, /* 0xE0-0xE3 */ + 0xB2, 0x4A, 0xD5, 0xF1, 0xB2, 0x68, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xB2, 0x62, 0xD5, 0xE6, 0xB2, 0x5F, 0xB2, 0x5D, /* 0xE8-0xEB */ + 0xB2, 0x66, 0xD5, 0xF8, 0xB2, 0x61, 0xD2, 0x52, /* 0xEC-0xEF */ + 0xD5, 0xF9, 0xB2, 0x60, 0xD6, 0x41, 0xB2, 0x45, /* 0xF0-0xF3 */ + 0xD5, 0xF5, 0xB2, 0x57, 0xD5, 0xE9, 0xB2, 0x56, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xB2, 0x54, 0xB2, 0x4C, 0xB2, 0x4B, /* 0xF8-0xFB */ + 0xD9, 0xE7, 0xD6, 0x43, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_6E[512] = { + 0xD5, 0xEB, 0x00, 0x00, 0x00, 0x00, 0xD9, 0xFC, /* 0x00-0x03 */ + 0x00, 0x00, 0xB2, 0x4D, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0xB5, 0x41, 0xB2, 0x5A, 0xB4, 0xEE, /* 0x18-0x1B */ + 0xD9, 0xF6, 0xB4, 0xFC, 0x00, 0x00, 0xD9, 0xEA, /* 0x1C-0x1F */ + 0xB4, 0xEB, 0xB4, 0xE7, 0xDA, 0x49, 0xB4, 0xED, /* 0x20-0x23 */ + 0xB4, 0xF1, 0xB4, 0xEC, 0xB4, 0xF5, 0xDA, 0x4D, /* 0x24-0x27 */ + 0xDA, 0x44, 0x00, 0x00, 0x00, 0x00, 0xD9, 0xF1, /* 0x28-0x2B */ + 0xB4, 0xFA, 0xB4, 0xF4, 0xD9, 0xFD, 0xB4, 0xE4, /* 0x2C-0x2F */ + 0xDA, 0x4A, 0xDA, 0x43, 0xB4, 0xE8, 0xD9, 0xF7, /* 0x30-0x33 */ + 0xB4, 0xF7, 0xDA, 0x55, 0xDA, 0x56, 0x00, 0x00, /* 0x34-0x37 */ + 0xB4, 0xE5, 0xDA, 0x48, 0xB4, 0xF9, 0xD9, 0xFB, /* 0x38-0x3B */ + 0xD9, 0xED, 0xD9, 0xEE, 0xB4, 0xFD, 0xD9, 0xF2, /* 0x3C-0x3F */ + 0xD9, 0xF9, 0xD9, 0xF3, 0x00, 0x00, 0xB4, 0xFB, /* 0x40-0x43 */ + 0xB5, 0x44, 0xD9, 0xEF, 0xD9, 0xE8, 0xD9, 0xE9, /* 0x44-0x47 */ + 0x00, 0x00, 0xD9, 0xEB, 0xB4, 0xEA, 0xD9, 0xF8, /* 0x48-0x4B */ + 0x00, 0x00, 0xB4, 0xF8, 0xB5, 0x42, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0xD9, 0xFA, 0xDA, 0x53, 0xDA, 0x4B, /* 0x50-0x53 */ + 0xB4, 0xE6, 0xDA, 0x51, 0xB4, 0xF2, 0x00, 0x00, /* 0x54-0x57 */ + 0xB4, 0xF0, 0x00, 0x00, 0xDA, 0x57, 0xB4, 0xEF, /* 0x58-0x5B */ + 0xDA, 0x41, 0xD9, 0xF4, 0xD9, 0xFE, 0xB5, 0x47, /* 0x5C-0x5F */ + 0xDA, 0x45, 0xDA, 0x42, 0xD9, 0xF0, 0xB5, 0x43, /* 0x60-0x63 */ + 0xDA, 0x4F, 0xDA, 0x4C, 0xDA, 0x54, 0xB4, 0xE9, /* 0x64-0x67 */ + 0xDA, 0x40, 0xB5, 0x46, 0x00, 0x00, 0xDA, 0x47, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0xB4, 0xF3, 0xB4, 0xF6, /* 0x6C-0x6F */ + 0x00, 0x00, 0xDA, 0x46, 0xB5, 0x45, 0xD9, 0xF5, /* 0x70-0x73 */ + 0xD5, 0xE4, 0x00, 0x00, 0x00, 0x00, 0xDA, 0x50, /* 0x74-0x77 */ + 0xDA, 0x4E, 0xDA, 0x52, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0xD9, 0xEC, 0xB5, 0x40, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0xDE, 0x61, 0xDE, 0x60, 0xDE, 0x46, /* 0x8C-0x8F */ + 0xB7, 0xBD, 0x00, 0x00, 0xDE, 0x5F, 0xDE, 0x49, /* 0x90-0x93 */ + 0xDE, 0x4A, 0x00, 0x00, 0xB7, 0xC7, 0xDE, 0x68, /* 0x94-0x97 */ + 0xB7, 0xC2, 0xDE, 0x5E, 0x00, 0x00, 0xDE, 0x43, /* 0x98-0x9B */ + 0xB7, 0xC8, 0xB7, 0xBE, 0xDE, 0x52, 0xDE, 0x48, /* 0x9C-0x9F */ + 0xDE, 0x4B, 0xDE, 0x63, 0xB7, 0xB8, 0xDE, 0x6A, /* 0xA0-0xA3 */ + 0xDE, 0x62, 0xB7, 0xC1, 0xDE, 0x57, 0xB7, 0xCC, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0xB7, 0xCB, 0xB7, 0xC5, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0xDE, 0x69, 0xB7, 0xB9, /* 0xAC-0xAF */ + 0xDE, 0x55, 0xDE, 0x4C, 0xDE, 0x59, 0xDE, 0x65, /* 0xB0-0xB3 */ + 0xB7, 0xCD, 0x00, 0x00, 0xB7, 0xBB, 0xDE, 0x54, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xDE, 0x4D, 0xB7, 0xC4, 0x00, 0x00, /* 0xB8-0xBB */ + 0xB7, 0xC3, 0xDE, 0x50, 0xDE, 0x5A, 0xDE, 0x64, /* 0xBC-0xBF */ + 0xDE, 0x47, 0xDE, 0x51, 0xB7, 0xBC, 0xDE, 0x5B, /* 0xC0-0xC3 */ + 0xB7, 0xC9, 0xB7, 0xC0, 0xDE, 0x4E, 0xB7, 0xBF, /* 0xC4-0xC7 */ + 0xDE, 0x45, 0xDE, 0x53, 0xDE, 0x67, 0xB4, 0xFE, /* 0xC8-0xCB */ + 0xBA, 0xB0, 0xDE, 0x56, 0xE2, 0x6C, 0xDE, 0x58, /* 0xCC-0xCF */ + 0xDE, 0x66, 0xB7, 0xC6, 0xDE, 0x4F, 0xB7, 0xBA, /* 0xD0-0xD3 */ + 0xB7, 0xCA, 0xBC, 0xF0, 0xDE, 0x44, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xDE, 0x5D, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0xDE, 0x5C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xAA, /* 0xE8-0xEB */ + 0xBA, 0xAD, 0xE2, 0x7D, 0xE2, 0xA4, 0xBA, 0xA2, /* 0xEC-0xEF */ + 0x00, 0x00, 0xE2, 0x6E, 0xBA, 0xAF, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xBA, 0x77, 0xE2, 0x6D, 0xE2, 0xB0, 0xBA, 0xB1, /* 0xF4-0xF7 */ + 0xE2, 0x71, 0xE2, 0xA3, 0x00, 0x00, 0xE2, 0x73, /* 0xF8-0xFB */ + 0xE2, 0xB3, 0xE2, 0xAF, 0xBA, 0x75, 0xBA, 0xA1, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_6F[512] = { + 0xE6, 0x53, 0xBA, 0xAE, 0xBA, 0x7D, 0xE2, 0x6F, /* 0x00-0x03 */ + 0x00, 0x00, 0xE2, 0xAE, 0xBA, 0xA3, 0xE2, 0xAB, /* 0x04-0x07 */ + 0xE2, 0xB8, 0xE2, 0x75, 0xE2, 0x7E, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0xE2, 0xB6, 0xE2, 0xAC, 0xBA, 0x7C, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0x7C, 0xBA, 0x76, /* 0x10-0x13 */ + 0xBA, 0x74, 0xBA, 0xA8, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0xE2, 0x7A, 0xE2, 0x77, 0xE2, 0x78, 0x00, 0x00, /* 0x18-0x1B */ + 0xE2, 0xB2, 0x00, 0x00, 0xE2, 0xB7, 0xE2, 0xB5, /* 0x1C-0x1F */ + 0xBA, 0x7A, 0xE2, 0xB9, 0xBA, 0x7E, 0xBA, 0xA7, /* 0x20-0x23 */ + 0x00, 0x00, 0xE2, 0x70, 0xE5, 0xFA, 0xE2, 0x79, /* 0x24-0x27 */ + 0x00, 0x00, 0xBA, 0x78, 0xBA, 0xAC, 0xBA, 0xA9, /* 0x28-0x2B */ + 0xBA, 0x7B, 0xE2, 0xA5, 0xE2, 0x74, 0xBA, 0xAA, /* 0x2C-0x2F */ + 0xE2, 0xA7, 0xBA, 0xA4, 0xBA, 0xA6, 0xBA, 0x73, /* 0x30-0x33 */ + 0x00, 0x00, 0xE2, 0xA9, 0xE2, 0xA1, 0xE2, 0x72, /* 0x34-0x37 */ + 0xBA, 0xA5, 0xE2, 0xB1, 0xE2, 0xB4, 0xE2, 0x7B, /* 0x38-0x3B */ + 0xE2, 0xA8, 0x00, 0x00, 0xBA, 0x79, 0xBC, 0xDF, /* 0x3C-0x3F */ + 0xE2, 0xA6, 0xE5, 0xF9, 0x00, 0x00, 0xE2, 0xAD, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0x76, 0xE6, 0x44, /* 0x4C-0x4F */ + 0xE6, 0x4E, 0xBC, 0xE2, 0xE6, 0x4D, 0xE6, 0x59, /* 0x50-0x53 */ + 0xBC, 0xE4, 0xE6, 0x4B, 0x00, 0x00, 0xE6, 0x4F, /* 0x54-0x57 */ + 0xBC, 0xEF, 0x00, 0x00, 0xE6, 0x46, 0xBC, 0xE7, /* 0x58-0x5B */ + 0x00, 0x00, 0xE6, 0x52, 0xE9, 0xF0, 0xBC, 0xF3, /* 0x5C-0x5F */ + 0xBC, 0xF2, 0xE6, 0x54, 0xE6, 0x43, 0xE6, 0x5E, /* 0x60-0x63 */ + 0xBC, 0xED, 0x00, 0x00, 0xBC, 0xE3, 0xE6, 0x57, /* 0x64-0x67 */ + 0x00, 0x00, 0xE6, 0x5B, 0xE6, 0x60, 0xE6, 0x55, /* 0x68-0x6B */ + 0xE6, 0x49, 0xBC, 0xE6, 0xBC, 0xE9, 0xBC, 0xF1, /* 0x6C-0x6F */ + 0xBC, 0xEC, 0x00, 0x00, 0xE6, 0x4C, 0xE2, 0xA2, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0x48, 0xE6, 0x5F, /* 0x74-0x77 */ + 0xBC, 0xE8, 0x00, 0x00, 0xBC, 0xEB, 0xE6, 0x61, /* 0x78-0x7B */ + 0xBC, 0xE0, 0xE6, 0x56, 0xE5, 0xFB, 0xE6, 0x5C, /* 0x7C-0x7F */ + + 0xC0, 0xDF, 0x00, 0x00, 0xE6, 0x4A, 0x00, 0x00, /* 0x80-0x83 */ + 0xBC, 0xE1, 0xE6, 0x45, 0xBC, 0xE5, 0xE5, 0xFC, /* 0x84-0x87 */ + 0xBA, 0xAB, 0xE6, 0x41, 0x00, 0x00, 0xE6, 0x5A, /* 0x88-0x8B */ + 0xE6, 0x42, 0xE6, 0x40, 0xBC, 0xEA, 0x00, 0x00, /* 0x8C-0x8F */ + 0xE6, 0x58, 0x00, 0x00, 0xE5, 0xFE, 0xE6, 0x51, /* 0x90-0x93 */ + 0xE6, 0x50, 0xE6, 0x5D, 0xE6, 0x47, 0xBC, 0xEE, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0xE9, 0xF3, 0x00, 0x00, /* 0x9C-0x9F */ + 0xBF, 0x49, 0xBE, 0xFE, 0xEA, 0x40, 0xE9, 0xEB, /* 0xA0-0xA3 */ + 0xBF, 0x41, 0xE9, 0xF7, 0xBF, 0x48, 0xBF, 0x43, /* 0xA4-0xA7 */ + 0xE9, 0xF5, 0xED, 0x4F, 0xE9, 0xFB, 0xEA, 0x42, /* 0xA8-0xAB */ + 0xE9, 0xFA, 0xE9, 0xE9, 0xE9, 0xF8, 0xEA, 0x44, /* 0xAC-0xAF */ + 0xEA, 0x46, 0xBE, 0xFD, 0xEA, 0x45, 0xBF, 0x44, /* 0xB0-0xB3 */ + 0xBF, 0x4A, 0x00, 0x00, 0xBF, 0x47, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xE9, 0xFE, 0xBF, 0x46, 0xE9, 0xF9, 0x00, 0x00, /* 0xB8-0xBB */ + 0xE9, 0xED, 0xE9, 0xF2, 0x00, 0x00, 0xE9, 0xFD, /* 0xBC-0xBF */ + 0xBF, 0x45, 0xBF, 0x42, 0xBE, 0xFC, 0xBF, 0x40, /* 0xC0-0xC3 */ + 0xE9, 0xF1, 0x00, 0x00, 0xE5, 0xFD, 0xE9, 0xEC, /* 0xC4-0xC7 */ + 0xE9, 0xEF, 0xEA, 0x41, 0xE9, 0xF4, 0xE9, 0xEA, /* 0xC8-0xCB */ + 0xED, 0x4E, 0xEA, 0x43, 0xE9, 0xEE, 0xE9, 0xFC, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xED, 0x51, 0xC0, 0xE3, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xC0, 0xD7, 0x00, 0x00, 0x00, 0x00, 0xC0, 0xDB, /* 0xD8-0xDB */ + 0xED, 0x53, 0xED, 0x59, 0xED, 0x57, 0xC0, 0xD9, /* 0xDC-0xDF */ + 0xC0, 0xDA, 0xC0, 0xE1, 0xED, 0x5A, 0xED, 0x52, /* 0xE0-0xE3 */ + 0xC0, 0xDC, 0x00, 0x00, 0xED, 0x56, 0xED, 0x55, /* 0xE4-0xE7 */ + 0xED, 0x5B, 0xC0, 0xE2, 0x00, 0x00, 0xC0, 0xDD, /* 0xE8-0xEB */ + 0xC0, 0xE0, 0xED, 0x54, 0xC0, 0xE4, 0xC0, 0xDE, /* 0xEC-0xEF */ + 0xC0, 0xE5, 0xC0, 0xD8, 0xED, 0x58, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xED, 0x50, 0x00, 0x00, 0x00, 0x00, 0xEF, 0xF7, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0xC2, 0x71, 0xEF, 0xF4, /* 0xF8-0xFB */ + 0xEF, 0xF6, 0x00, 0x00, 0xC2, 0x6F, 0xEF, 0xF2, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_70[512] = { + 0xEF, 0xF3, 0xEF, 0xEE, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0xE9, 0xF6, 0xEF, 0xEF, 0xC2, 0x70, 0xEF, 0xEB, /* 0x04-0x07 */ + 0x00, 0x00, 0xC2, 0x6D, 0xEF, 0xF8, 0xC2, 0x6E, /* 0x08-0x0B */ + 0xEF, 0xEC, 0xEF, 0xED, 0xEF, 0xF1, 0xC2, 0x73, /* 0x0C-0x0F */ + 0x00, 0x00, 0xC2, 0x72, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0xEF, 0xF0, 0xC3, 0x78, 0xF2, 0x5F, 0xF2, 0x65, /* 0x14-0x17 */ + 0xC3, 0x79, 0xF2, 0x5C, 0xC3, 0x76, 0xC3, 0x73, /* 0x18-0x1B */ + 0xF2, 0x67, 0xC3, 0x77, 0x00, 0x00, 0xC3, 0x74, /* 0x1C-0x1F */ + 0xF2, 0x5E, 0xF2, 0x61, 0xF2, 0x62, 0xF2, 0x63, /* 0x20-0x23 */ + 0xF2, 0x66, 0x00, 0x00, 0xEF, 0xF5, 0xF2, 0x5D, /* 0x24-0x27 */ + 0xC3, 0x75, 0xF2, 0x64, 0xF2, 0x68, 0xF2, 0x60, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF4, 0x5D, /* 0x2C-0x2F */ + 0xC4, 0x6A, 0xF4, 0x60, 0xC4, 0x6B, 0xF4, 0x68, /* 0x30-0x33 */ + 0xF4, 0x5F, 0xF4, 0x5C, 0x00, 0x00, 0xF4, 0x5E, /* 0x34-0x37 */ + 0xF4, 0x62, 0xF4, 0x65, 0xF4, 0x64, 0xF4, 0x67, /* 0x38-0x3B */ + 0xF4, 0x5B, 0x00, 0x00, 0xC4, 0x69, 0xF4, 0x63, /* 0x3C-0x3F */ + 0xF4, 0x66, 0xF4, 0x69, 0xF4, 0x61, 0xF5, 0xD3, /* 0x40-0x43 */ + 0xF5, 0xD4, 0xF5, 0xD8, 0xF5, 0xD9, 0x00, 0x00, /* 0x44-0x47 */ + 0xF5, 0xD6, 0xF5, 0xD7, 0xF5, 0xD5, 0x00, 0x00, /* 0x48-0x4B */ + 0xC4, 0xE9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0xC5, 0x78, 0xF6, 0xEB, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0xF6, 0xE8, 0xF6, 0xE9, 0xF6, 0xEA, /* 0x54-0x57 */ + 0xC5, 0x79, 0x00, 0x00, 0xF7, 0xE5, 0xF7, 0xE4, /* 0x58-0x5B */ + 0x00, 0x00, 0xF8, 0xAF, 0xC5, 0xF4, 0xF8, 0xAD, /* 0x5C-0x5F */ + 0xF8, 0xB0, 0xF8, 0xAE, 0xF8, 0xF5, 0xC6, 0x57, /* 0x60-0x63 */ + 0xC6, 0x65, 0xF9, 0xA3, 0xF9, 0x6C, 0x00, 0x00, /* 0x64-0x67 */ + 0xF9, 0xA2, 0xF9, 0xD0, 0xF9, 0xD1, 0xA4, 0xF5, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0xA6, 0xC7, 0xCA, 0x41, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0xCB, 0x5E, 0x00, 0x00, 0xA8, 0x5F, 0x00, 0x00, /* 0x74-0x77 */ + 0xA8, 0x62, 0x00, 0x00, 0xCB, 0x5F, 0x00, 0x00, /* 0x78-0x7B */ + 0xA8, 0x60, 0xA8, 0x61, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0xCD, 0x58, 0xCD, 0x5A, /* 0x80-0x83 */ + 0xCD, 0x55, 0xCD, 0x52, 0xCD, 0x54, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0xAA, 0xA4, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0xAA, 0xA2, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xCD, 0x56, 0xAA, 0xA3, 0xCD, 0x53, /* 0x90-0x93 */ + 0xCD, 0x50, 0xAA, 0xA1, 0xCD, 0x57, 0x00, 0x00, /* 0x94-0x97 */ + 0xCD, 0x51, 0xAA, 0xA5, 0xCD, 0x59, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCF, 0xAF, /* 0x9C-0x9F */ + 0x00, 0x00, 0xCF, 0xB3, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xAC, 0xB7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0xCF, 0xB6, 0x00, 0x00, 0xAC, 0xAF, /* 0xA8-0xAB */ + 0xAC, 0xB2, 0xAC, 0xB4, 0xAC, 0xB6, 0xAC, 0xB3, /* 0xAC-0xAF */ + 0xCF, 0xB2, 0xCF, 0xB1, 0x00, 0x00, 0xAC, 0xB1, /* 0xB0-0xB3 */ + 0xCF, 0xB4, 0xCF, 0xB5, 0x00, 0x00, 0xCF, 0xAE, /* 0xB4-0xB7 */ + 0xAC, 0xB5, 0x00, 0x00, 0xAC, 0xB0, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0xCF, 0xB0, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xD2, 0x77, 0xD2, 0x78, 0xD2, 0x79, /* 0xC4-0xC7 */ + 0xAF, 0x50, 0x00, 0x00, 0xAF, 0x4C, 0xD2, 0x6E, /* 0xC8-0xCB */ + 0x00, 0x00, 0xD2, 0x76, 0xD2, 0x7B, 0xAF, 0x51, /* 0xCC-0xCF */ + 0x00, 0x00, 0xD2, 0x6C, 0xD2, 0x72, 0xD2, 0x6B, /* 0xD0-0xD3 */ + 0xD2, 0x75, 0x00, 0x00, 0x00, 0x00, 0xD2, 0x71, /* 0xD4-0xD7 */ + 0xAF, 0x4D, 0xAF, 0x4F, 0xD2, 0x7A, 0x00, 0x00, /* 0xD8-0xDB */ + 0xD2, 0x6A, 0xD2, 0x6D, 0xD2, 0x73, 0x00, 0x00, /* 0xDC-0xDF */ + 0xD2, 0x74, 0xD2, 0x7C, 0xD2, 0x70, 0x00, 0x00, /* 0xE0-0xE3 */ + 0xAF, 0x4E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB2, 0x6D, /* 0xEC-0xEF */ + 0xD6, 0x4E, 0x00, 0x00, 0x00, 0x00, 0xD6, 0x50, /* 0xF0-0xF3 */ + 0xD6, 0x4C, 0x00, 0x00, 0xD6, 0x58, 0xD6, 0x4A, /* 0xF4-0xF7 */ + 0xD6, 0x57, 0xB2, 0x69, 0xD6, 0x48, 0xDA, 0x5B, /* 0xF8-0xFB */ + 0xD6, 0x52, 0xB2, 0x6C, 0x00, 0x00, 0xD6, 0x53, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_71[512] = { + 0xD6, 0x56, 0x00, 0x00, 0xD6, 0x5A, 0x00, 0x00, /* 0x00-0x03 */ + 0xD6, 0x4F, 0x00, 0x00, 0xD6, 0x54, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xB2, 0x6A, 0xB2, 0x6B, 0xD6, 0x59, /* 0x08-0x0B */ + 0xD6, 0x4D, 0xD6, 0x49, 0xD6, 0x5B, 0x00, 0x00, /* 0x0C-0x0F */ + 0xD6, 0x51, 0x00, 0x00, 0x00, 0x00, 0xD6, 0x55, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD6, 0x4B, /* 0x14-0x17 */ + 0x00, 0x00, 0xB5, 0x48, 0xB5, 0x49, 0xDA, 0x65, /* 0x18-0x1B */ + 0xB5, 0x4F, 0x00, 0x00, 0xDA, 0x59, 0xDA, 0x62, /* 0x1C-0x1F */ + 0xDA, 0x58, 0xB5, 0x4C, 0xDA, 0x60, 0xDA, 0x5E, /* 0x20-0x23 */ + 0x00, 0x00, 0xDA, 0x5F, 0xB5, 0x4A, 0x00, 0x00, /* 0x24-0x27 */ + 0xDA, 0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0xDA, 0x5C, 0xDA, 0x5A, /* 0x2C-0x2F */ + 0xB5, 0x4B, 0xDA, 0x5D, 0xDA, 0x61, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0xB5, 0x4D, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xDA, 0x64, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0xDE, 0x70, 0xDE, 0x77, 0xDE, 0x79, /* 0x40-0x43 */ + 0xDE, 0xA1, 0x00, 0x00, 0xB7, 0xDA, 0xDE, 0x6B, /* 0x44-0x47 */ + 0x00, 0x00, 0xB7, 0xD2, 0x00, 0x00, 0xDE, 0x7A, /* 0x48-0x4B */ + 0xB7, 0xD7, 0xDE, 0xA2, 0xB7, 0xCE, 0x00, 0x00, /* 0x4C-0x4F */ + 0xDE, 0x7D, 0x00, 0x00, 0xDE, 0x6D, 0xDE, 0x7E, /* 0x50-0x53 */ + 0xDE, 0x6C, 0x00, 0x00, 0xB7, 0xDC, 0x00, 0x00, /* 0x54-0x57 */ + 0xDE, 0x78, 0xB7, 0xCF, 0xDE, 0xA3, 0x00, 0x00, /* 0x58-0x5B */ + 0xB7, 0xD4, 0xDE, 0x71, 0xB7, 0xD9, 0xDE, 0x7C, /* 0x5C-0x5F */ + 0xDE, 0x6F, 0xDE, 0x76, 0xDE, 0x72, 0xDE, 0x6E, /* 0x60-0x63 */ + 0xB7, 0xD1, 0xB7, 0xD8, 0xB7, 0xD6, 0xB7, 0xD3, /* 0x64-0x67 */ + 0xB7, 0xDB, 0xB7, 0xD0, 0xDE, 0x75, 0x00, 0x00, /* 0x68-0x6B */ + 0xB7, 0xD5, 0x00, 0x00, 0xB5, 0x4E, 0x00, 0x00, /* 0x6C-0x6F */ + 0xDE, 0x7B, 0x00, 0x00, 0xDE, 0x73, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0xDE, 0x74, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xC1, /* 0x78-0x7B */ + 0x00, 0x00, 0xBA, 0xB4, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xE2, 0xBD, 0xE2, 0xC3, 0xE2, 0xBF, 0x00, 0x00, /* 0x80-0x83 */ + 0xBA, 0xB6, 0xE2, 0xBE, 0xE2, 0xC2, 0xE2, 0xBA, /* 0x84-0x87 */ + 0x00, 0x00, 0xE2, 0xBC, 0xBA, 0xB5, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xC0, /* 0x8C-0x8F */ + 0xE2, 0xBB, 0x00, 0x00, 0xBA, 0xB7, 0x00, 0x00, /* 0x90-0x93 */ + 0xBA, 0xB2, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xC4, /* 0x94-0x97 */ + 0x00, 0x00, 0xBA, 0xB3, 0xE6, 0x67, 0xE6, 0x64, /* 0x98-0x9B */ + 0xE6, 0x70, 0xE6, 0x6A, 0xE6, 0x6C, 0xBC, 0xF4, /* 0x9C-0x9F */ + 0xE6, 0x66, 0xE6, 0x6E, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xE6, 0x6D, 0xE6, 0x6B, 0x00, 0x00, 0xE6, 0x71, /* 0xA4-0xA7 */ + 0xBC, 0xF7, 0xE6, 0x68, 0xE6, 0x6F, 0x00, 0x00, /* 0xA8-0xAB */ + 0xBC, 0xF5, 0x00, 0x00, 0x00, 0x00, 0xE6, 0x63, /* 0xAC-0xAF */ + 0xE6, 0x65, 0xBC, 0xF6, 0xE6, 0x62, 0xE6, 0x72, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xE6, 0x69, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xEA, 0x4A, 0xBF, 0x51, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0xEA, 0x55, 0xEA, 0x53, 0xBF, 0x4B, 0xEA, 0x49, /* 0xBC-0xBF */ + 0xEA, 0x4C, 0xEA, 0x4D, 0xEA, 0x48, 0xBF, 0x55, /* 0xC0-0xC3 */ + 0xBF, 0x56, 0xEA, 0x47, 0xEA, 0x56, 0xEA, 0x51, /* 0xC4-0xC7 */ + 0xBF, 0x4F, 0xBF, 0x4C, 0xEA, 0x50, 0xEA, 0x4E, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0xBF, 0x52, 0xEA, 0x52, /* 0xCC-0xCF */ + 0xBF, 0x4D, 0x00, 0x00, 0xBF, 0x4E, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xEA, 0x4F, 0xBF, 0x50, 0xEA, 0x4B, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xEA, 0x54, 0xBF, 0x53, 0xEA, 0x57, 0xEA, 0x58, /* 0xD8-0xDB */ + 0xBF, 0x54, 0x00, 0x00, 0x00, 0x00, 0xC0, 0xE7, /* 0xDC-0xDF */ + 0xC0, 0xEE, 0xED, 0x5C, 0xED, 0x62, 0x00, 0x00, /* 0xE0-0xE3 */ + 0xED, 0x60, 0xC0, 0xEA, 0xC0, 0xE9, 0xC0, 0xE6, /* 0xE4-0xE7 */ + 0xED, 0x5E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0xC0, 0xEC, 0xC0, 0xEB, 0xC0, 0xE8, 0x00, 0x00, /* 0xEC-0xEF */ + 0xED, 0x61, 0xED, 0x5D, 0xED, 0x5F, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xC0, 0xED, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xC2, 0x77, 0xEF, 0xFB, 0x00, 0x00, 0xC2, 0x74, /* 0xF8-0xFB */ + 0xC2, 0x75, 0xEF, 0xFD, 0xC2, 0x76, 0xEF, 0xFA, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_72[512] = { + 0x00, 0x00, 0xEF, 0xF9, 0xF2, 0x6C, 0xEF, 0xFC, /* 0x00-0x03 */ + 0x00, 0x00, 0xF2, 0x6D, 0xC3, 0x7A, 0xF2, 0x6B, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xF2, 0x6A, 0x00, 0x00, /* 0x08-0x0B */ + 0xF2, 0x69, 0xC3, 0x7B, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0xC4, 0x6C, 0x00, 0x00, 0x00, 0x00, 0xF4, 0x6A, /* 0x10-0x13 */ + 0xF4, 0x6B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0xF5, 0xDC, 0xF5, 0xDB, 0xC4, 0xEA, /* 0x18-0x1B */ + 0x00, 0x00, 0xF5, 0xDA, 0xF6, 0xEC, 0xF6, 0xED, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0xF7, 0xE6, 0xF8, 0xB1, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0xF8, 0xF6, 0xF9, 0xBC, /* 0x24-0x27 */ + 0xC6, 0x79, 0xF9, 0xC6, 0xA4, 0xF6, 0x00, 0x00, /* 0x28-0x2B */ + 0xAA, 0xA6, 0xAA, 0xA7, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xAC, 0xB8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0xC0, 0xEF, 0xA4, 0xF7, 0x00, 0x00, /* 0x34-0x37 */ + 0xAA, 0xA8, 0xAF, 0x52, 0xB7, 0xDD, 0xA4, 0xF8, /* 0x38-0x3B */ + 0x00, 0x00, 0xB2, 0x6E, 0xBA, 0xB8, 0xC9, 0x62, /* 0x3C-0x3F */ + 0x00, 0x00, 0xCF, 0xB7, 0xD2, 0x7D, 0x00, 0x00, /* 0x40-0x43 */ + 0xE2, 0xC5, 0x00, 0x00, 0xC0, 0xF0, 0xA4, 0xF9, /* 0x44-0x47 */ + 0xAA, 0xA9, 0xCF, 0xB8, 0xCF, 0xB9, 0xDA, 0x66, /* 0x48-0x4B */ + 0xB5, 0x50, 0x00, 0x00, 0x00, 0x00, 0xDE, 0xA4, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0xB7, 0xDE, 0xE2, 0xC6, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0xBC, 0xF8, 0x00, 0x00, /* 0x54-0x57 */ + 0xC3, 0x7C, 0xA4, 0xFA, 0xDA, 0x67, 0xA4, 0xFB, /* 0x58-0x5B */ + 0x00, 0x00, 0xA6, 0xC9, 0xCA, 0x42, 0xA6, 0xC8, /* 0x5C-0x5F */ + 0xA8, 0x65, 0xA8, 0x64, 0xA8, 0x63, 0xCB, 0x60, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAA, 0xAA, /* 0x64-0x67 */ + 0x00, 0x00, 0xAA, 0xAB, 0xCD, 0x5B, 0x00, 0x00, /* 0x68-0x6B */ + 0xCF, 0xBA, 0x00, 0x00, 0xCF, 0xBD, 0xAC, 0xBA, /* 0x6C-0x6F */ + 0xCF, 0xBB, 0x00, 0x00, 0xAC, 0xB9, 0xCF, 0xBC, /* 0x70-0x73 */ + 0xAC, 0xBB, 0x00, 0x00, 0xD2, 0xA2, 0xD2, 0xA1, /* 0x74-0x77 */ + 0xD2, 0x7E, 0xAF, 0x53, 0x00, 0x00, 0xD6, 0x5D, /* 0x78-0x7B */ + 0xD6, 0x5E, 0xB2, 0x6F, 0xD6, 0x5C, 0xD6, 0x5F, /* 0x7C-0x7F */ + + 0xB5, 0x52, 0xB2, 0x70, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0xB5, 0x51, 0xDA, 0x6B, 0xDA, 0x6A, 0x00, 0x00, /* 0x84-0x87 */ + 0xDA, 0x68, 0xDA, 0x69, 0x00, 0x00, 0xDA, 0x6C, /* 0x88-0x8B */ + 0xDE, 0xA6, 0xDE, 0xA5, 0xDE, 0xA9, 0x00, 0x00, /* 0x8C-0x8F */ + 0xDE, 0xA8, 0xDE, 0xA7, 0xBA, 0xB9, 0xE2, 0xC9, /* 0x90-0x93 */ + 0x00, 0x00, 0xE2, 0xC8, 0xBA, 0xBA, 0xE2, 0xC7, /* 0x94-0x97 */ + 0xE6, 0x73, 0x00, 0x00, 0xE6, 0x74, 0xBC, 0xF9, /* 0x98-0x9B */ + 0x00, 0x00, 0xEA, 0x59, 0xEA, 0x5A, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0xF2, 0x72, 0xC3, 0x7D, 0xF2, 0x71, /* 0xA0-0xA3 */ + 0xF2, 0x70, 0xF2, 0x6E, 0xF2, 0x6F, 0xC4, 0xEB, /* 0xA4-0xA7 */ + 0xF4, 0x6C, 0xF6, 0xEE, 0xF8, 0xF7, 0x00, 0x00, /* 0xA8-0xAB */ + 0xA4, 0xFC, 0x00, 0x00, 0xC9, 0xA5, 0xA5, 0xC7, /* 0xAC-0xAF */ + 0xC9, 0xA6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xCA, 0x43, 0xCA, 0x44, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0xCB, 0x66, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0xCB, 0x62, 0x00, 0x00, 0xCB, 0x61, /* 0xBC-0xBF */ + 0xAA, 0xAC, 0xCB, 0x65, 0xA8, 0x67, 0xCB, 0x63, /* 0xC0-0xC3 */ + 0xA8, 0x66, 0xCB, 0x67, 0xCB, 0x64, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xCD, 0x5F, 0xCF, 0xBE, 0xCD, 0x5D, /* 0xC8-0xCB */ + 0xCD, 0x64, 0x00, 0x00, 0xAA, 0xAD, 0x00, 0x00, /* 0xCC-0xCF */ + 0xAA, 0xB0, 0xCD, 0x65, 0xCD, 0x61, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xCD, 0x62, 0x00, 0x00, 0xCD, 0x5C, 0xAA, 0xAF, /* 0xD4-0xD7 */ + 0xCD, 0x5E, 0xAA, 0xAE, 0xCD, 0x63, 0x00, 0x00, /* 0xD8-0xDB */ + 0xCD, 0x60, 0x00, 0x00, 0x00, 0x00, 0xCF, 0xC2, /* 0xDC-0xDF */ + 0xAC, 0xBD, 0xAC, 0xBE, 0x00, 0x00, 0xCF, 0xC5, /* 0xE0-0xE3 */ + 0xCF, 0xBF, 0x00, 0x00, 0xCF, 0xC4, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xCF, 0xC0, 0xAC, 0xBC, 0xCF, 0xC3, 0xCF, 0xC1, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD2, 0xA8, /* 0xF0-0xF3 */ + 0xD2, 0xA5, 0x00, 0x00, 0xD2, 0xA7, 0xAF, 0x58, /* 0xF4-0xF7 */ + 0xAF, 0x57, 0xAF, 0x55, 0xD2, 0xA4, 0xD2, 0xA9, /* 0xF8-0xFB */ + 0xAF, 0x54, 0xAF, 0x56, 0xD2, 0xA6, 0xD6, 0x67, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_73[512] = { + 0xD2, 0xA3, 0xD2, 0xAA, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD6, 0x62, /* 0x04-0x07 */ + 0xD6, 0x66, 0x00, 0x00, 0xD6, 0x65, 0xDA, 0x6E, /* 0x08-0x0B */ + 0xDA, 0x79, 0x00, 0x00, 0x00, 0x00, 0xD6, 0x68, /* 0x0C-0x0F */ + 0x00, 0x00, 0xD6, 0x63, 0xDA, 0x6D, 0xB2, 0x74, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xB2, 0x73, 0xD6, 0x61, /* 0x14-0x17 */ + 0xD6, 0x64, 0xB2, 0x75, 0x00, 0x00, 0xB2, 0x72, /* 0x18-0x1B */ + 0xB2, 0x71, 0xD6, 0x60, 0xD6, 0x69, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0xDA, 0x70, 0xDA, 0x77, /* 0x20-0x23 */ + 0x00, 0x00, 0xB5, 0x54, 0xDA, 0x76, 0xDA, 0x73, /* 0x24-0x27 */ + 0x00, 0x00, 0xB5, 0x56, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0xDA, 0x75, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xDA, 0x6F, 0xDA, 0x71, 0xDA, 0x74, 0xDA, 0x72, /* 0x30-0x33 */ + 0xB5, 0x55, 0xDA, 0x78, 0xB5, 0x53, 0xB7, 0xDF, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xDE, 0xAD, 0xDE, 0xAC, /* 0x38-0x3B */ + 0xDE, 0xAA, 0x00, 0x00, 0xB7, 0xE2, 0xB7, 0xE1, /* 0x3C-0x3F */ + 0xDE, 0xAE, 0x00, 0x00, 0xDE, 0xAB, 0xE2, 0xCA, /* 0x40-0x43 */ + 0xBA, 0xBB, 0xB7, 0xE0, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0xDE, 0xB0, 0xDE, 0xAF, 0x00, 0x00, /* 0x48-0x4B */ + 0xE2, 0xCD, 0xE2, 0xCB, 0xBC, 0xFA, 0x00, 0x00, /* 0x4C-0x4F */ + 0xBA, 0xBC, 0xE2, 0xCC, 0xE6, 0x76, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBC, 0xFB, /* 0x54-0x57 */ + 0xE6, 0x75, 0xE6, 0x7E, 0xE6, 0x7D, 0xE6, 0x7B, /* 0x58-0x5B */ + 0x00, 0x00, 0xE6, 0x7A, 0xE6, 0x77, 0xE6, 0x78, /* 0x5C-0x5F */ + 0xE6, 0x79, 0xE6, 0x7C, 0xE6, 0xA1, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0xEA, 0x5F, 0xEA, 0x5C, 0xEA, 0x5D, /* 0x64-0x67 */ + 0xBF, 0x57, 0xEA, 0x5B, 0xEA, 0x61, 0xEA, 0x60, /* 0x68-0x6B */ + 0xEA, 0x5E, 0x00, 0x00, 0xED, 0x64, 0xED, 0x65, /* 0x6C-0x6F */ + 0xC0, 0xF1, 0x00, 0x00, 0xC0, 0xF2, 0xED, 0x63, /* 0x70-0x73 */ + 0x00, 0x00, 0xC2, 0x79, 0xEF, 0xFE, 0xC2, 0x78, /* 0x74-0x77 */ + 0xC3, 0x7E, 0x00, 0x00, 0xC3, 0xA1, 0xC4, 0x6D, /* 0x78-0x7B */ + 0xF4, 0x6E, 0xF4, 0x6D, 0xF5, 0xDD, 0xF6, 0xEF, /* 0x7C-0x7F */ + + 0xC5, 0x7A, 0xF7, 0xE8, 0xF7, 0xE7, 0xF7, 0xE9, /* 0x80-0x83 */ + 0xA5, 0xC8, 0xCF, 0xC6, 0xAF, 0x59, 0xB2, 0x76, /* 0x84-0x87 */ + 0xD6, 0x6A, 0xA5, 0xC9, 0xC9, 0xA7, 0xA4, 0xFD, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0xCA, 0x45, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0xCB, 0x6C, 0xCB, 0x6A, /* 0x90-0x93 */ + 0xCB, 0x6B, 0xCB, 0x68, 0xA8, 0x68, 0xCB, 0x69, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0xCD, 0x6D, 0x00, 0x00, 0xAA, 0xB3, /* 0x9C-0x9F */ + 0xCD, 0x6B, 0xCD, 0x67, 0xCD, 0x6A, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xCD, 0x66, 0xAA, 0xB5, 0xCD, 0x69, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xAA, 0xB2, 0xAA, 0xB1, 0x00, 0x00, 0xAA, 0xB4, /* 0xA8-0xAB */ + 0xCD, 0x6C, 0xCD, 0x68, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xAC, 0xC2, 0xAC, 0xC5, /* 0xB0-0xB3 */ + 0xCF, 0xCE, 0xCF, 0xCD, 0xCF, 0xCC, 0xAC, 0xBF, /* 0xB4-0xB7 */ + 0xCF, 0xD5, 0xCF, 0xCB, 0x00, 0x00, 0xAC, 0xC1, /* 0xB8-0xBB */ + 0xD2, 0xAF, 0x00, 0x00, 0xCF, 0xD2, 0xCF, 0xD0, /* 0xBC-0xBF */ + 0xAC, 0xC4, 0x00, 0x00, 0xCF, 0xC8, 0xCF, 0xD3, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xCF, 0xCA, 0xCF, 0xD4, 0xCF, 0xD1, /* 0xC4-0xC7 */ + 0xCF, 0xC9, 0x00, 0x00, 0xAC, 0xC0, 0xCF, 0xD6, /* 0xC8-0xCB */ + 0xCF, 0xC7, 0xAC, 0xC3, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xD2, 0xB4, 0xD2, 0xAB, /* 0xD0-0xD3 */ + 0xD2, 0xB6, 0x00, 0x00, 0xD2, 0xAE, 0xD2, 0xB9, /* 0xD4-0xD7 */ + 0xD2, 0xBA, 0xD2, 0xAC, 0xD2, 0xB8, 0xD2, 0xB5, /* 0xD8-0xDB */ + 0xD2, 0xB3, 0xD2, 0xB7, 0xAF, 0x5F, 0x00, 0x00, /* 0xDC-0xDF */ + 0xAF, 0x5D, 0x00, 0x00, 0x00, 0x00, 0xD2, 0xB1, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xD2, 0xAD, 0x00, 0x00, 0xD2, 0xB0, /* 0xE4-0xE7 */ + 0xD2, 0xBB, 0xD2, 0xB2, 0xAF, 0x5E, 0xCF, 0xCF, /* 0xE8-0xEB */ + 0x00, 0x00, 0xAF, 0x5A, 0xAF, 0x5C, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xD6, 0x78, 0xD6, 0x6D, 0xD6, 0x6B, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xD6, 0x6C, 0x00, 0x00, 0xD6, 0x73, 0x00, 0x00, /* 0xF8-0xFB */ + 0xD6, 0x74, 0xD6, 0x70, 0xB2, 0x7B, 0xD6, 0x75, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_74[512] = { + 0xD6, 0x72, 0xD6, 0x6F, 0x00, 0x00, 0xB2, 0x79, /* 0x00-0x03 */ + 0xD6, 0x6E, 0xB2, 0x77, 0xB2, 0x7A, 0xD6, 0x71, /* 0x04-0x07 */ + 0xD6, 0x79, 0xAF, 0x5B, 0xB2, 0x78, 0xD6, 0x77, /* 0x08-0x0B */ + 0xD6, 0x76, 0xB2, 0x7C, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xDA, 0x7E, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0xDA, 0xA1, 0xB5, 0x60, /* 0x18-0x1B */ + 0x00, 0x00, 0xDA, 0xA7, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0xDA, 0xA9, 0xDA, 0xA2, 0xB5, 0x5A, 0xDA, 0xA6, /* 0x20-0x23 */ + 0xDA, 0xA5, 0xB5, 0x5B, 0xB5, 0x61, 0x00, 0x00, /* 0x24-0x27 */ + 0xB5, 0x62, 0xDA, 0xA8, 0xB5, 0x58, 0xDA, 0x7D, /* 0x28-0x2B */ + 0xDA, 0x7B, 0xDA, 0xA3, 0xDA, 0x7A, 0xB5, 0x5F, /* 0x2C-0x2F */ + 0xDA, 0x7C, 0xDA, 0xA4, 0xDA, 0xAA, 0xB5, 0x59, /* 0x30-0x33 */ + 0xB5, 0x5E, 0xB5, 0x5C, 0xB5, 0x5D, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xB5, 0x57, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB7, 0xE9, /* 0x3C-0x3F */ + 0xDE, 0xB7, 0xB7, 0xE8, 0xDE, 0xBB, 0x00, 0x00, /* 0x40-0x43 */ + 0xDE, 0xB1, 0x00, 0x00, 0xDE, 0xBC, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0xDE, 0xB2, 0xDE, 0xB3, /* 0x48-0x4B */ + 0x00, 0x00, 0xDE, 0xBD, 0xDE, 0xBA, 0xDE, 0xB8, /* 0x4C-0x4F */ + 0xDE, 0xB9, 0xDE, 0xB5, 0xDE, 0xB4, 0x00, 0x00, /* 0x50-0x53 */ + 0xDE, 0xBE, 0xB7, 0xE5, 0x00, 0x00, 0xDE, 0xB6, /* 0x54-0x57 */ + 0x00, 0x00, 0xB7, 0xEA, 0xB7, 0xE4, 0xB7, 0xEB, /* 0x58-0x5B */ + 0xB7, 0xEC, 0x00, 0x00, 0xB7, 0xE7, 0xB7, 0xE6, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0xCE, 0xBA, 0xBE, /* 0x60-0x63 */ + 0xBA, 0xBD, 0x00, 0x00, 0x00, 0x00, 0xE2, 0xD3, /* 0x64-0x67 */ + 0x00, 0x00, 0xBC, 0xFC, 0xBA, 0xBF, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0xBA, 0xC1, 0xE2, 0xD4, 0xB7, 0xE3, /* 0x6C-0x6F */ + 0xBA, 0xC0, 0xE2, 0xD0, 0xE2, 0xD2, 0xE2, 0xCF, /* 0x70-0x73 */ + 0x00, 0x00, 0xE2, 0xD1, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0xE6, 0xAB, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0xE6, 0xAA, 0xE6, 0xA7, 0xBD, 0x40, 0xEA, 0x62, /* 0x7C-0x7F */ + + 0xBD, 0x41, 0xE6, 0xA6, 0x00, 0x00, 0xBC, 0xFE, /* 0x80-0x83 */ + 0x00, 0x00, 0xE6, 0xA8, 0xE6, 0xA5, 0xE6, 0xA2, /* 0x84-0x87 */ + 0xE6, 0xA9, 0xE6, 0xA3, 0xE6, 0xA4, 0xBC, 0xFD, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0xED, 0x69, 0x00, 0x00, 0xEA, 0x66, 0x00, 0x00, /* 0x90-0x93 */ + 0xEA, 0x65, 0xEA, 0x67, 0x00, 0x00, 0xED, 0x66, /* 0x94-0x97 */ + 0xBF, 0x5A, 0x00, 0x00, 0xEA, 0x63, 0x00, 0x00, /* 0x98-0x9B */ + 0xBF, 0x58, 0x00, 0x00, 0xBF, 0x5C, 0xBF, 0x5B, /* 0x9C-0x9F */ + 0xEA, 0x64, 0xEA, 0x68, 0x00, 0x00, 0xBF, 0x59, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xED, 0x6D, 0xC0, 0xF5, 0xC2, 0x7A, /* 0xA4-0xA7 */ + 0xC0, 0xF6, 0xC0, 0xF3, 0xED, 0x6A, 0xED, 0x68, /* 0xA8-0xAB */ + 0x00, 0x00, 0xED, 0x6B, 0x00, 0x00, 0xED, 0x6E, /* 0xAC-0xAF */ + 0xC0, 0xF4, 0xED, 0x6C, 0xED, 0x67, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xF0, 0x42, 0xF0, 0x45, 0xF2, 0x75, /* 0xB4-0xB7 */ + 0xF0, 0x40, 0x00, 0x00, 0xF4, 0x6F, 0xF0, 0x46, /* 0xB8-0xBB */ + 0x00, 0x00, 0xC3, 0xA2, 0xF0, 0x44, 0xC2, 0x7B, /* 0xBC-0xBF */ + 0xF0, 0x41, 0xF0, 0x43, 0xF0, 0x47, 0xF2, 0x76, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xF2, 0x74, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0xC3, 0xA3, 0xF2, 0x73, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC4, 0x6E, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xC4, 0xED, 0xF6, 0xF1, 0xC4, 0xEC, 0xF6, 0xF3, /* 0xD4-0xD7 */ + 0xF6, 0xF0, 0xF6, 0xF2, 0xC5, 0xD0, 0xF8, 0xB2, /* 0xD8-0xDB */ + 0xA5, 0xCA, 0xCD, 0x6E, 0xD2, 0xBC, 0xD2, 0xBD, /* 0xDC-0xDF */ + 0xB2, 0x7D, 0xDE, 0xBF, 0xBF, 0x5D, 0xC3, 0xA4, /* 0xE0-0xE3 */ + 0xC5, 0x7B, 0xF8, 0xB3, 0xA5, 0xCB, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xCD, 0x6F, 0xA2, 0x60, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0xCF, 0xD7, 0x00, 0x00, 0xCF, 0xD8, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xD2, 0xBE, 0xD2, 0xBF, 0xB2, 0x7E, 0xB2, 0xA1, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDA, 0xAB, /* 0xF8-0xFB */ + 0x00, 0x00, 0xDE, 0xC2, 0xDE, 0xC1, 0xDE, 0xC0, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_75[512] = { + 0xE2, 0xD5, 0x00, 0x00, 0xE2, 0xD6, 0xE2, 0xD7, /* 0x00-0x03 */ + 0xBA, 0xC2, 0x00, 0x00, 0x00, 0x00, 0xE6, 0xAD, /* 0x04-0x07 */ + 0xE6, 0xAC, 0x00, 0x00, 0x00, 0x00, 0xEA, 0x69, /* 0x08-0x0B */ + 0xBF, 0x5E, 0xBF, 0x5F, 0x00, 0x00, 0xED, 0x72, /* 0x0C-0x0F */ + 0xED, 0x6F, 0xED, 0x70, 0xED, 0x71, 0xF0, 0x49, /* 0x10-0x13 */ + 0xF0, 0x48, 0xC2, 0x7C, 0xF2, 0x77, 0xF5, 0xDE, /* 0x14-0x17 */ + 0xA5, 0xCC, 0x00, 0x00, 0xAC, 0xC6, 0x00, 0x00, /* 0x18-0x1B */ + 0xB2, 0xA2, 0xDE, 0xC3, 0x00, 0x00, 0xA5, 0xCD, /* 0x1C-0x1F */ + 0x00, 0x00, 0xD2, 0xC0, 0xB2, 0xA3, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0xB5, 0x63, 0xB5, 0x64, 0x00, 0x00, /* 0x24-0x27 */ + 0xA5, 0xCE, 0xA5, 0xCF, 0xCA, 0x46, 0xA8, 0x6A, /* 0x28-0x2B */ + 0xA8, 0x69, 0xAC, 0xC7, 0xCF, 0xD9, 0xDA, 0xAC, /* 0x2C-0x2F */ + 0xA5, 0xD0, 0xA5, 0xD1, 0xA5, 0xD2, 0xA5, 0xD3, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA8, 0x6B, /* 0x34-0x37 */ + 0xA8, 0x6C, 0xCB, 0x6E, 0xCB, 0x6D, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0xAA, 0xB6, 0xCD, 0x72, 0xCD, 0x70, /* 0x3C-0x3F */ + 0xCD, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCF, 0xDA, /* 0x44-0x47 */ + 0xCF, 0xDB, 0x00, 0x00, 0x00, 0x00, 0xAC, 0xCB, /* 0x48-0x4B */ + 0xAC, 0xC9, 0x00, 0x00, 0xAC, 0xCA, 0xAC, 0xC8, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0xAF, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0xAF, 0x64, 0xAF, 0x63, 0xD2, 0xC1, /* 0x58-0x5B */ + 0xAF, 0x62, 0xAF, 0x61, 0x00, 0x00, 0xD2, 0xC2, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0xB2, 0xA6, 0xD6, 0x7B, /* 0x60-0x63 */ + 0xD6, 0x7A, 0xB2, 0xA4, 0xB2, 0xA5, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0xB5, 0x66, 0xB5, 0x65, /* 0x68-0x6B */ + 0xDA, 0xAE, 0x00, 0x00, 0x00, 0x00, 0xDA, 0xAD, /* 0x6C-0x6F */ + 0xB2, 0xA7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0xB7, 0xED, 0xDE, 0xC5, /* 0x74-0x77 */ + 0xB7, 0xEE, 0xDE, 0xC4, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0xE2, 0xD8, 0xE6, 0xAE, 0xBD, 0x42, /* 0x7C-0x7F */ + + 0xEA, 0x6A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0xED, 0x73, 0x00, 0x00, 0xC3, 0xA6, 0xC3, 0xA5, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0xC5, 0x7C, 0xA5, 0xD4, /* 0x88-0x8B */ + 0xCD, 0x73, 0x00, 0x00, 0x00, 0x00, 0xB2, 0xA8, /* 0x8C-0x8F */ + 0xE2, 0xD9, 0xBA, 0xC3, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0xCB, 0x6F, 0xCB, 0x70, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0xCD, 0x74, 0xAA, 0xB8, 0xAA, 0xB9, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0xAA, 0xB7, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0xAC, 0xCF, 0xAC, 0xD0, /* 0xA0-0xA3 */ + 0xAC, 0xCD, 0xAC, 0xCE, 0x00, 0x00, 0xCF, 0xDC, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0xCF, 0xDD, 0xAC, 0xCC, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0xD2, 0xC3, 0x00, 0x00, 0xAF, 0x68, 0xAF, 0x69, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xB2, 0xAB, 0xD2, 0xC9, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xAF, 0x6E, 0xAF, 0x6C, 0xD2, 0xCA, 0xD2, 0xC5, /* 0xB8-0xBB */ + 0xAF, 0x6B, 0xAF, 0x6A, 0xAF, 0x65, 0xD2, 0xC8, /* 0xBC-0xBF */ + 0xD2, 0xC7, 0xD2, 0xC4, 0xAF, 0x6D, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xD2, 0xC6, 0xAF, 0x66, 0x00, 0x00, 0xAF, 0x67, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0xB2, 0xAC, 0xD6, 0xA1, /* 0xC8-0xCB */ + 0xD6, 0xA2, 0xB2, 0xAD, 0xD6, 0x7C, 0xD6, 0x7E, /* 0xCC-0xCF */ + 0xD6, 0xA4, 0xD6, 0xA3, 0xD6, 0x7D, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xB2, 0xA9, 0xB2, 0xAA, 0x00, 0x00, 0xDA, 0xB6, /* 0xD4-0xD7 */ + 0xB5, 0x6B, 0xB5, 0x6A, 0xDA, 0xB0, 0xB5, 0x68, /* 0xD8-0xDB */ + 0x00, 0x00, 0xDA, 0xB3, 0xB5, 0x6C, 0xDA, 0xB4, /* 0xDC-0xDF */ + 0xB5, 0x6D, 0xDA, 0xB1, 0xB5, 0x67, 0xB5, 0x69, /* 0xE0-0xE3 */ + 0xDA, 0xB5, 0x00, 0x00, 0xDA, 0xB2, 0xDA, 0xAF, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0xDE, 0xD2, 0x00, 0x00, 0xDE, 0xC7, /* 0xEC-0xEF */ + 0xB7, 0xF0, 0xB7, 0xF3, 0xB7, 0xF2, 0xB7, 0xF7, /* 0xF0-0xF3 */ + 0xB7, 0xF6, 0xDE, 0xD3, 0xDE, 0xD1, 0xDE, 0xCA, /* 0xF4-0xF7 */ + 0xDE, 0xCE, 0xDE, 0xCD, 0xB7, 0xF4, 0xDE, 0xD0, /* 0xF8-0xFB */ + 0xDE, 0xCC, 0xDE, 0xD4, 0xDE, 0xCB, 0xB7, 0xF5, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_76[512] = { + 0xB7, 0xEF, 0xB7, 0xF1, 0x00, 0x00, 0xDE, 0xC9, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0xE2, 0xDB, 0xBA, 0xC7, 0xE2, 0xDF, 0xBA, 0xC6, /* 0x08-0x0B */ + 0xE2, 0xDC, 0xBA, 0xC5, 0x00, 0x00, 0xDE, 0xC8, /* 0x0C-0x0F */ + 0xDE, 0xCF, 0xE2, 0xDE, 0x00, 0x00, 0xBA, 0xC8, /* 0x10-0x13 */ + 0xE2, 0xE0, 0xE2, 0xDD, 0xE2, 0xDA, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0xE6, 0xB1, 0xE6, 0xB5, 0xE6, 0xB7, /* 0x18-0x1B */ + 0xE6, 0xB3, 0xE6, 0xB2, 0xE6, 0xB0, 0xBD, 0x45, /* 0x1C-0x1F */ + 0xBD, 0x43, 0xBD, 0x48, 0xBD, 0x49, 0xE6, 0xB4, /* 0x20-0x23 */ + 0xBD, 0x46, 0xE6, 0xAF, 0xBD, 0x47, 0xBA, 0xC4, /* 0x24-0x27 */ + 0xE6, 0xB6, 0xBD, 0x44, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0xEA, 0x6C, 0x00, 0x00, 0xEA, 0x6B, /* 0x2C-0x2F */ + 0xEA, 0x73, 0xEA, 0x6D, 0xEA, 0x72, 0xEA, 0x6F, /* 0x30-0x33 */ + 0xBF, 0x60, 0xEA, 0x71, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0xBF, 0x61, 0x00, 0x00, 0xBF, 0x62, 0x00, 0x00, /* 0x38-0x3B */ + 0xEA, 0x70, 0xEA, 0x6E, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0xC0, 0xF8, 0xED, 0x74, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xC0, 0xF7, 0xED, 0x77, /* 0x44-0x47 */ + 0xED, 0x75, 0xED, 0x76, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0xC0, 0xF9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0xF0, 0x4D, 0x00, 0x00, 0xC2, 0xA1, 0xF0, 0x4E, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0xC2, 0x7D, 0xF0, 0x4F, /* 0x54-0x57 */ + 0xC2, 0x7E, 0xF0, 0x4C, 0xF0, 0x50, 0x00, 0x00, /* 0x58-0x5B */ + 0xF0, 0x4A, 0x00, 0x00, 0x00, 0x00, 0xC3, 0xA7, /* 0x5C-0x5F */ + 0xF2, 0x78, 0xC3, 0xA8, 0xC4, 0x6F, 0x00, 0x00, /* 0x60-0x63 */ + 0xF0, 0x4B, 0xC4, 0x70, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0xC4, 0xEE, 0xF5, 0xDF, 0x00, 0x00, /* 0x68-0x6B */ + 0xC5, 0x7E, 0xF6, 0xF4, 0xC5, 0x7D, 0x00, 0x00, /* 0x6C-0x6F */ + 0xF7, 0xEA, 0xC5, 0xF5, 0xC5, 0xF6, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0xF9, 0xCC, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0xAC, 0xD1, 0xCF, 0xDE, 0x00, 0x00, 0xB5, 0x6E, /* 0x78-0x7B */ + 0xB5, 0x6F, 0xA5, 0xD5, 0xA6, 0xCA, 0xCA, 0x47, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xCB, 0x71, 0xA8, 0x6D, 0x00, 0x00, /* 0x80-0x83 */ + 0xAA, 0xBA, 0x00, 0x00, 0xAC, 0xD2, 0xAC, 0xD3, /* 0x84-0x87 */ + 0xAC, 0xD4, 0xD6, 0xA6, 0xD2, 0xCB, 0xAF, 0x6F, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0xB2, 0xAE, 0xD6, 0xA5, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0xDA, 0xB8, 0xB5, 0x71, /* 0x90-0x93 */ + 0x00, 0x00, 0xDA, 0xB7, 0xB5, 0x70, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0xDE, 0xD5, 0xBD, 0x4A, 0xE6, 0xBB, /* 0x98-0x9B */ + 0xE6, 0xB8, 0xE6, 0xB9, 0xE6, 0xBA, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xED, 0x78, 0x00, 0x00, 0xF0, 0x51, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0xF4, 0x71, 0xF4, 0x70, /* 0xA8-0xAB */ + 0x00, 0x00, 0xF6, 0xF5, 0xA5, 0xD6, 0xCD, 0x75, /* 0xAC-0xAF */ + 0xAF, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xB5, 0x72, 0xDE, 0xD6, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xE2, 0xE1, 0x00, 0x00, 0xBD, 0x4B, 0xEA, 0x74, /* 0xB8-0xBB */ + 0x00, 0x00, 0xF0, 0x52, 0xF4, 0x72, 0xA5, 0xD7, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0xAA, 0xBB, 0xAC, 0xD7, /* 0xC0-0xC3 */ + 0xCF, 0xDF, 0xAC, 0xD8, 0xAC, 0xD6, 0x00, 0x00, /* 0xC4-0xC7 */ + 0xAC, 0xD5, 0xD2, 0xCC, 0xAF, 0x71, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0xAF, 0x72, 0xAF, 0x73, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xB2, 0xB0, 0xD6, 0xA7, /* 0xD0-0xD3 */ + 0xB2, 0xAF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xDA, 0xB9, 0xB2, 0xB1, /* 0xD8-0xDB */ + 0xB5, 0x73, 0xDE, 0xD7, 0xB7, 0xF8, 0xB7, 0xF9, /* 0xDC-0xDF */ + 0x00, 0x00, 0xBA, 0xC9, 0x00, 0x00, 0xBA, 0xCA, /* 0xE0-0xE3 */ + 0xBD, 0x4C, 0xBF, 0x64, 0xEA, 0x75, 0xBF, 0x63, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xED, 0x79, 0xC0, 0xFA, 0x00, 0x00, /* 0xE8-0xEB */ + 0xF0, 0x53, 0xF4, 0x73, 0xA5, 0xD8, 0xA8, 0x6E, /* 0xEC-0xEF */ + 0xCD, 0x78, 0xCD, 0x77, 0xAA, 0xBC, 0xCD, 0x76, /* 0xF0-0xF3 */ + 0xAA, 0xBD, 0xCD, 0x79, 0x00, 0x00, 0xCF, 0xE5, /* 0xF4-0xF7 */ + 0xAC, 0xDB, 0xAC, 0xDA, 0xCF, 0xE7, 0xCF, 0xE6, /* 0xF8-0xFB */ + 0xAC, 0xDF, 0x00, 0x00, 0xAC, 0xDE, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_77[512] = { + 0x00, 0x00, 0xAC, 0xD9, 0x00, 0x00, 0xCF, 0xE1, /* 0x00-0x03 */ + 0xCF, 0xE2, 0xCF, 0xE3, 0x00, 0x00, 0xAC, 0xE0, /* 0x04-0x07 */ + 0xCF, 0xE0, 0xAC, 0xDC, 0xCF, 0xE4, 0xAC, 0xDD, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0xD2, 0xCF, 0xD2, 0xD3, 0xD2, 0xD1, 0xD2, 0xD0, /* 0x10-0x13 */ + 0x00, 0x00, 0xD2, 0xD4, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0xD2, 0xD5, 0xD2, 0xD6, 0xD2, 0xCE, /* 0x18-0x1B */ + 0x00, 0x00, 0xD2, 0xCD, 0x00, 0x00, 0xAF, 0x75, /* 0x1C-0x1F */ + 0xAF, 0x76, 0x00, 0x00, 0xD2, 0xD7, 0xD2, 0xD2, /* 0x20-0x23 */ + 0x00, 0x00, 0xD6, 0xB0, 0x00, 0x00, 0xD2, 0xD8, /* 0x24-0x27 */ + 0xAF, 0x77, 0xAF, 0x74, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0xD6, 0xAA, 0x00, 0x00, 0xD6, 0xA9, /* 0x2C-0x2F */ + 0x00, 0x00, 0xD6, 0xAB, 0xD6, 0xAC, 0xD6, 0xAE, /* 0x30-0x33 */ + 0xD6, 0xAD, 0xD6, 0xB2, 0xB2, 0xB5, 0xB2, 0xB2, /* 0x34-0x37 */ + 0xB2, 0xB6, 0xD6, 0xA8, 0xB2, 0xB7, 0xD6, 0xB1, /* 0x38-0x3B */ + 0xB2, 0xB4, 0xD6, 0xAF, 0xB2, 0xB3, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0xDA, 0xBC, 0xDA, 0xBE, 0xDA, 0xBA, 0xDA, 0xBB, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0xDA, 0xBF, 0xDA, 0xC1, /* 0x48-0x4B */ + 0xDA, 0xC2, 0xDA, 0xBD, 0xDA, 0xC0, 0xB5, 0x74, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0xDE, 0xDB, 0x00, 0x00, /* 0x50-0x53 */ + 0xDE, 0xE0, 0xDE, 0xD8, 0xDE, 0xDC, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0xDE, 0xE1, 0xDE, 0xDD, 0xB7, 0xFA, /* 0x58-0x5B */ + 0xB8, 0x43, 0x00, 0x00, 0xB7, 0xFD, 0xDE, 0xD9, /* 0x5C-0x5F */ + 0xDE, 0xDA, 0xBA, 0xCE, 0xB8, 0x46, 0xB7, 0xFE, /* 0x60-0x63 */ + 0x00, 0x00, 0xB8, 0x44, 0xB7, 0xFC, 0xDE, 0xDF, /* 0x64-0x67 */ + 0xB8, 0x45, 0xDE, 0xDE, 0xB8, 0x41, 0xB7, 0xFB, /* 0x68-0x6B */ + 0xB8, 0x42, 0xDE, 0xE2, 0xE2, 0xE6, 0xE2, 0xE8, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0xB8, 0x40, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0xE2, 0xE3, 0xBA, 0xCC, 0xE2, 0xE9, 0xBA, 0xCD, /* 0x7C-0x7F */ + + 0xE2, 0xE7, 0xE2, 0xE2, 0xE2, 0xE5, 0xE2, 0xEA, /* 0x80-0x83 */ + 0xBA, 0xCB, 0xE2, 0xE4, 0x00, 0x00, 0xBD, 0x4E, /* 0x84-0x87 */ + 0xE6, 0xBF, 0xE6, 0xBE, 0x00, 0x00, 0xBD, 0x51, /* 0x88-0x8B */ + 0xBD, 0x4F, 0xE6, 0xBC, 0xBD, 0x4D, 0xE6, 0xBD, /* 0x8C-0x8F */ + 0x00, 0x00, 0xBD, 0x50, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0xEA, 0x7D, 0x00, 0x00, 0xEA, 0xA1, /* 0x94-0x97 */ + 0x00, 0x00, 0xEA, 0x7E, 0xEA, 0x76, 0xEA, 0x7A, /* 0x98-0x9B */ + 0xEA, 0x79, 0xEA, 0x77, 0xBF, 0x66, 0xBF, 0x67, /* 0x9C-0x9F */ + 0xBF, 0x65, 0xEA, 0x78, 0xEA, 0x7B, 0xEA, 0x7C, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xBF, 0x68, 0x00, 0x00, 0xC1, 0x40, /* 0xA4-0xA7 */ + 0xED, 0xA3, 0x00, 0x00, 0xC0, 0xFC, 0xED, 0x7B, /* 0xA8-0xAB */ + 0xC0, 0xFE, 0xC1, 0x41, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0xC0, 0xFD, 0xED, 0xA2, 0xED, 0x7C, 0xC0, 0xFB, /* 0xB0-0xB3 */ + 0xED, 0xA1, 0xED, 0x7A, 0xED, 0x7E, 0xED, 0x7D, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0xF0, 0x55, 0xC2, 0xA4, /* 0xB8-0xBB */ + 0xC2, 0xA5, 0xC2, 0xA2, 0x00, 0x00, 0xC2, 0xA3, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0xF0, 0x54, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xF2, 0x7B, 0x00, 0x00, 0x00, 0x00, 0xC3, 0xA9, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xF2, 0x79, 0xF2, 0x7A, 0x00, 0x00, /* 0xC8-0xCB */ + 0xF4, 0x74, 0xF4, 0x77, 0xF4, 0x75, 0xF4, 0x76, /* 0xCC-0xCF */ + 0xF5, 0xE0, 0x00, 0x00, 0x00, 0x00, 0xC4, 0xEF, /* 0xD0-0xD3 */ + 0xF7, 0xEB, 0xF8, 0xB4, 0x00, 0x00, 0xC5, 0xF7, /* 0xD4-0xD7 */ + 0xF8, 0xF8, 0xF8, 0xF9, 0xC6, 0x66, 0xA5, 0xD9, /* 0xD8-0xDB */ + 0xAC, 0xE1, 0x00, 0x00, 0xDA, 0xC3, 0x00, 0x00, /* 0xDC-0xDF */ + 0xDE, 0xE3, 0x00, 0x00, 0xA5, 0xDA, 0xA8, 0x6F, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xAA, 0xBE, 0x00, 0x00, 0xCF, 0xE8, /* 0xE4-0xE7 */ + 0xCF, 0xE9, 0xAF, 0x78, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0xDA, 0xC4, 0xB5, 0x75, 0xB8, 0x47, 0xC1, 0x42, /* 0xEC-0xEF */ + 0xED, 0xA4, 0xF2, 0x7C, 0xF4, 0x78, 0xA5, 0xDB, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCD, 0xA1, /* 0xF4-0xF7 */ + 0xCD, 0x7A, 0xCD, 0x7C, 0xCD, 0x7E, 0xCD, 0x7D, /* 0xF8-0xFB */ + 0xCD, 0x7B, 0xAA, 0xBF, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_78[512] = { + 0x00, 0x00, 0x00, 0x00, 0xAC, 0xE2, 0xCF, 0xF2, /* 0x00-0x03 */ + 0x00, 0x00, 0xCF, 0xED, 0xCF, 0xEA, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xCF, 0xF1, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0xAC, 0xE4, 0xAC, 0xE5, 0xCF, 0xF0, 0xCF, 0xEF, /* 0x0C-0x0F */ + 0xCF, 0xEE, 0xCF, 0xEB, 0xCF, 0xEC, 0xCF, 0xF3, /* 0x10-0x13 */ + 0xAC, 0xE3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0xAF, 0x7C, 0x00, 0x00, 0xAF, 0xA4, /* 0x1C-0x1F */ + 0xAF, 0xA3, 0xD2, 0xE1, 0xD2, 0xDB, 0xD2, 0xD9, /* 0x20-0x23 */ + 0x00, 0x00, 0xAF, 0xA1, 0xD6, 0xB9, 0xAF, 0x7A, /* 0x24-0x27 */ + 0xD2, 0xDE, 0xD2, 0xE2, 0xD2, 0xE4, 0xD2, 0xE0, /* 0x28-0x2B */ + 0xD2, 0xDA, 0xAF, 0xA2, 0xD2, 0xDF, 0xD2, 0xDD, /* 0x2C-0x2F */ + 0xAF, 0x79, 0xD2, 0xE5, 0xAF, 0xA5, 0xD2, 0xE3, /* 0x30-0x33 */ + 0xAF, 0x7D, 0xD2, 0xDC, 0x00, 0x00, 0xAF, 0x7E, /* 0x34-0x37 */ + 0xAF, 0x7B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB2, 0xB9, /* 0x40-0x43 */ + 0x00, 0x00, 0xD6, 0xBA, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0xD6, 0xB3, 0xD6, 0xB5, 0xD6, 0xB7, 0x00, 0x00, /* 0x48-0x4B */ + 0xD6, 0xB8, 0xD6, 0xB6, 0xB2, 0xBA, 0x00, 0x00, /* 0x4C-0x4F */ + 0xD6, 0xBB, 0x00, 0x00, 0xD6, 0xB4, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0xDA, 0xC8, 0xB5, 0x76, 0xDA, 0xD0, 0x00, 0x00, /* 0x5C-0x5F */ + 0xDA, 0xC5, 0x00, 0x00, 0xDA, 0xD1, 0x00, 0x00, /* 0x60-0x63 */ + 0xDA, 0xC6, 0xDA, 0xC7, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0xDA, 0xCF, 0xDA, 0xCE, 0xDA, 0xCB, 0xB2, 0xB8, /* 0x68-0x6B */ + 0xB5, 0x77, 0xDA, 0xC9, 0xDA, 0xCC, 0xB5, 0x78, /* 0x6C-0x6F */ + 0xDA, 0xCD, 0xDA, 0xCA, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0xDE, 0xEE, 0x00, 0x00, 0xDE, 0xF2, /* 0x78-0x7B */ + 0xB8, 0x4E, 0x00, 0x00, 0xE2, 0xF0, 0xB8, 0x51, /* 0x7C-0x7F */ + + 0xDE, 0xF0, 0xF9, 0xD6, 0x00, 0x00, 0xDE, 0xED, /* 0x80-0x83 */ + 0xDE, 0xE8, 0xDE, 0xEA, 0xDE, 0xEB, 0xDE, 0xE4, /* 0x84-0x87 */ + 0x00, 0x00, 0xB8, 0x4D, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0xB8, 0x4C, 0x00, 0x00, 0xB8, 0x48, 0xDE, 0xE7, /* 0x8C-0x8F */ + 0x00, 0x00, 0xB8, 0x4F, 0x00, 0x00, 0xB8, 0x50, /* 0x90-0x93 */ + 0xDE, 0xE6, 0xDE, 0xE9, 0xDE, 0xF1, 0xB8, 0x4A, /* 0x94-0x97 */ + 0xB8, 0x4B, 0xDE, 0xEF, 0xDE, 0xE5, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0xE2, 0xF2, 0xBA, 0xD0, /* 0x9C-0x9F */ + 0xE2, 0xF4, 0xDE, 0xEC, 0xE2, 0xF6, 0xBA, 0xD4, /* 0xA0-0xA3 */ + 0xE2, 0xF7, 0xE2, 0xF3, 0x00, 0x00, 0xBA, 0xD1, /* 0xA4-0xA7 */ + 0xE2, 0xEF, 0xBA, 0xD3, 0xE2, 0xEC, 0xE2, 0xF1, /* 0xA8-0xAB */ + 0xE2, 0xF5, 0xE2, 0xEE, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0xB8, 0x49, 0x00, 0x00, 0xE2, 0xEB, 0xBA, 0xD2, /* 0xB0-0xB3 */ + 0xE2, 0xED, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0xBD, 0x54, 0xE6, 0xC1, /* 0xB8-0xBB */ + 0xBD, 0x58, 0x00, 0x00, 0xBD, 0x56, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0xBA, 0xCF, 0x00, 0x00, 0xE6, 0xC8, /* 0xC0-0xC3 */ + 0xE6, 0xC9, 0xBD, 0x53, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0xE6, 0xC7, 0xE6, 0xCA, 0xBD, 0x55, 0xBD, 0x52, /* 0xC8-0xCB */ + 0xE6, 0xC3, 0xE6, 0xC0, 0xE6, 0xC5, 0xE6, 0xC2, /* 0xCC-0xCF */ + 0xBD, 0x59, 0xE6, 0xC4, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xE6, 0xC6, 0xBD, 0x57, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xBF, 0x6A, 0xEA, 0xA8, /* 0xD8-0xDB */ + 0x00, 0x00, 0xEA, 0xA2, 0xEA, 0xA6, 0xEA, 0xAC, /* 0xDC-0xDF */ + 0xEA, 0xAD, 0xEA, 0xA9, 0xEA, 0xAA, 0xEA, 0xA7, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xEA, 0xA4, 0x00, 0x00, 0xBF, 0x6C, /* 0xE4-0xE7 */ + 0xBF, 0x69, 0xEA, 0xA3, 0xEA, 0xA5, 0x00, 0x00, /* 0xE8-0xEB */ + 0xBF, 0x6B, 0xEA, 0xAB, 0x00, 0x00, 0xC1, 0x46, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0xED, 0xAA, 0xED, 0xA5, /* 0xF0-0xF3 */ + 0xC1, 0x45, 0x00, 0x00, 0x00, 0x00, 0xC1, 0x43, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xED, 0xAC, 0xC1, 0x44, 0xED, 0xA8, /* 0xF8-0xFB */ + 0xED, 0xA9, 0xED, 0xA6, 0xED, 0xAD, 0xF0, 0x56, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_79[512] = { + 0x00, 0x00, 0xC1, 0x47, 0xED, 0xA7, 0x00, 0x00, /* 0x00-0x03 */ + 0xED, 0xAE, 0xED, 0xAB, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0xF0, 0x5A, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0xF0, 0x57, 0x00, 0x00, 0xC2, 0xA6, 0x00, 0x00, /* 0x0C-0x0F */ + 0xF0, 0x5B, 0xF0, 0x5D, 0xF0, 0x5C, 0xF0, 0x58, /* 0x10-0x13 */ + 0xF0, 0x59, 0x00, 0x00, 0x00, 0x00, 0xF2, 0xA3, /* 0x14-0x17 */ + 0x00, 0x00, 0xC3, 0xAA, 0x00, 0x00, 0xF2, 0x7E, /* 0x18-0x1B */ + 0xF2, 0xA2, 0xF2, 0x7D, 0xF2, 0xA4, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0xF2, 0xA1, 0x00, 0x00, 0xF4, 0x7A, /* 0x20-0x23 */ + 0xF4, 0x7D, 0xF4, 0x79, 0xC4, 0x71, 0xF4, 0x7B, /* 0x24-0x27 */ + 0xF4, 0x7C, 0xF4, 0x7E, 0xC4, 0x72, 0xC4, 0x74, /* 0x28-0x2B */ + 0xC4, 0x73, 0xF5, 0xE1, 0x00, 0x00, 0xF5, 0xE3, /* 0x2C-0x2F */ + 0x00, 0x00, 0xF5, 0xE2, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0xF6, 0xF6, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0xF8, 0xB5, 0xF8, 0xFA, 0xA5, 0xDC, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0xCB, 0x72, 0xAA, 0xC0, 0xCD, 0xA3, /* 0x3C-0x3F */ + 0xAA, 0xC1, 0xAA, 0xC2, 0xCD, 0xA2, 0x00, 0x00, /* 0x40-0x43 */ + 0xCF, 0xF8, 0xCF, 0xF7, 0xAC, 0xE6, 0xAC, 0xE9, /* 0x44-0x47 */ + 0xAC, 0xE8, 0xAC, 0xE7, 0xCF, 0xF4, 0xCF, 0xF6, /* 0x48-0x4B */ + 0xCF, 0xF5, 0x00, 0x00, 0x00, 0x00, 0xD2, 0xE8, /* 0x4C-0x4F */ + 0xAF, 0xA7, 0xD2, 0xEC, 0xD2, 0xEB, 0xD2, 0xEA, /* 0x50-0x53 */ + 0xD2, 0xE6, 0xAF, 0xA6, 0xAF, 0xAA, 0xAF, 0xAD, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0xAF, 0xAE, 0xD2, 0xE7, /* 0x58-0x5B */ + 0xD2, 0xE9, 0xAF, 0xAC, 0xAF, 0xAB, 0xAF, 0xA9, /* 0x5C-0x5F */ + 0xAF, 0xA8, 0xD6, 0xC2, 0x00, 0x00, 0xD6, 0xC0, /* 0x60-0x63 */ + 0xD6, 0xBC, 0xB2, 0xBB, 0x00, 0x00, 0xD6, 0xBD, /* 0x64-0x67 */ + 0xB2, 0xBC, 0xD6, 0xBE, 0xD6, 0xBF, 0xD6, 0xC1, /* 0x68-0x6B */ + 0x00, 0x00, 0xB2, 0xBD, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0xDA, 0xD5, 0x00, 0x00, 0xDA, 0xD4, 0xDA, 0xD3, /* 0x70-0x73 */ + 0xDA, 0xD2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0xDE, 0xF6, 0xB8, 0x52, 0x00, 0x00, /* 0x78-0x7B */ + 0xDE, 0xF3, 0xDE, 0xF5, 0x00, 0x00, 0xB8, 0x53, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xB8, 0x54, 0xDE, 0xF4, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0xE3, 0x41, 0x00, 0x00, 0xE2, 0xF9, 0xE2, 0xFA, /* 0x88-0x8B */ + 0x00, 0x00, 0xBA, 0xD7, 0xBA, 0xD5, 0xBA, 0xD6, /* 0x8C-0x8F */ + 0xE3, 0x43, 0x00, 0x00, 0xE3, 0x42, 0xE2, 0xFE, /* 0x90-0x93 */ + 0xE2, 0xFD, 0xE2, 0xFC, 0xE2, 0xFB, 0xE3, 0x40, /* 0x94-0x97 */ + 0xE2, 0xF8, 0x00, 0x00, 0xE6, 0xCB, 0xE6, 0xD0, /* 0x98-0x9B */ + 0xE6, 0xCE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0xE6, 0xCD, 0xE6, 0xCC, 0xE6, 0xCF, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xEA, 0xAE, 0x00, 0x00, 0xBF, 0x6D, 0xC1, 0x48, /* 0xA4-0xA7 */ + 0xED, 0xB0, 0x00, 0x00, 0xC1, 0x49, 0xED, 0xAF, /* 0xA8-0xAB */ + 0xF0, 0x5F, 0xF0, 0x5E, 0xC2, 0xA7, 0x00, 0x00, /* 0xAC-0xAF */ + 0xF2, 0xA5, 0xC3, 0xAB, 0xF4, 0xA1, 0xC5, 0xA1, /* 0xB0-0xB3 */ + 0xF6, 0xF7, 0x00, 0x00, 0xF8, 0xB7, 0xF8, 0xB6, /* 0xB4-0xB7 */ + 0xC9, 0xA8, 0xAC, 0xEA, 0xAC, 0xEB, 0xD6, 0xC3, /* 0xB8-0xBB */ + 0x00, 0x00, 0xB8, 0x56, 0xA5, 0xDD, 0xA8, 0x72, /* 0xBC-0xBF */ + 0xA8, 0x71, 0xA8, 0x70, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xCD, 0xA4, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0xAA, 0xC4, 0xAA, 0xC3, 0x00, 0x00, 0xAC, 0xEE, /* 0xC8-0xCB */ + 0x00, 0x00, 0xCF, 0xFA, 0xCF, 0xFD, 0xCF, 0xFB, /* 0xCC-0xCF */ + 0x00, 0x00, 0xAC, 0xEC, 0xAC, 0xED, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xCF, 0xF9, 0xCF, 0xFC, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xAF, 0xB5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0xD2, 0xF3, 0xD2, 0xF5, 0xD2, 0xF4, 0xAF, 0xB2, /* 0xDC-0xDF */ + 0xD2, 0xEF, 0x00, 0x00, 0x00, 0x00, 0xAF, 0xB0, /* 0xE0-0xE3 */ + 0xAF, 0xAF, 0x00, 0x00, 0xAF, 0xB3, 0xAF, 0xB1, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xAF, 0xB4, 0xD2, 0xF2, 0xD2, 0xED, /* 0xE8-0xEB */ + 0xD2, 0xEE, 0xD2, 0xF1, 0xD2, 0xF0, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0xD6, 0xC6, 0xD6, 0xC7, /* 0xF4-0xF7 */ + 0xD6, 0xC5, 0x00, 0x00, 0xD6, 0xC4, 0xB2, 0xBE, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_7A[512] = { + 0xB5, 0x7D, 0x00, 0x00, 0xDA, 0xD6, 0xDA, 0xD8, /* 0x00-0x03 */ + 0xDA, 0xDA, 0xB5, 0x7C, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0xB5, 0x7A, 0x00, 0x00, 0xDA, 0xD7, 0xB5, 0x7B, /* 0x08-0x0B */ + 0xDA, 0xD9, 0xB5, 0x79, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0xDF, 0x41, 0xDE, 0xF7, 0xDE, 0xFA, 0xDE, 0xFE, /* 0x10-0x13 */ + 0xB8, 0x5A, 0xDE, 0xFC, 0x00, 0x00, 0xDE, 0xFB, /* 0x14-0x17 */ + 0xDE, 0xF8, 0xDE, 0xF9, 0xB8, 0x58, 0xDF, 0x40, /* 0x18-0x1B */ + 0xB8, 0x57, 0x00, 0x00, 0xB8, 0x5C, 0xB8, 0x5B, /* 0x1C-0x1F */ + 0xB8, 0x59, 0x00, 0x00, 0xDE, 0xFD, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0x49, 0x00, 0x00, /* 0x24-0x27 */ + 0xE3, 0x48, 0x00, 0x00, 0x00, 0x00, 0xE3, 0x44, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0xBA, 0xD8, 0xE3, 0x47, /* 0x2C-0x2F */ + 0xE3, 0x46, 0xBA, 0xD9, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBD, 0x5E, /* 0x34-0x37 */ + 0x00, 0x00, 0xE6, 0xD2, 0x00, 0x00, 0xBD, 0x5F, /* 0x38-0x3B */ + 0xBD, 0x5B, 0xBD, 0x5D, 0x00, 0x00, 0xBD, 0x5A, /* 0x3C-0x3F */ + 0xBD, 0x5C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0xEA, 0xAF, 0x00, 0x00, 0xBF, 0x70, 0xEA, 0xB1, /* 0x44-0x47 */ + 0xEA, 0xB0, 0x00, 0x00, 0xE3, 0x45, 0xBF, 0x72, /* 0x48-0x4B */ + 0xBF, 0x71, 0xBF, 0x6E, 0xBF, 0x6F, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0xED, 0xB5, 0x00, 0x00, 0xED, 0xB3, 0xC1, 0x4A, /* 0x54-0x57 */ + 0xED, 0xB4, 0x00, 0x00, 0xED, 0xB6, 0xED, 0xB2, /* 0x58-0x5B */ + 0xED, 0xB1, 0x00, 0x00, 0x00, 0x00, 0xF0, 0x60, /* 0x5C-0x5F */ + 0xC2, 0xAA, 0xC2, 0xA8, 0xC2, 0xA9, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF2, 0xA6, /* 0x64-0x67 */ + 0xF2, 0xA7, 0xC3, 0xAD, 0x00, 0x00, 0xC3, 0xAC, /* 0x68-0x6B */ + 0xF4, 0xA3, 0xF4, 0xA4, 0xF4, 0xA2, 0x00, 0x00, /* 0x6C-0x6F */ + 0xF6, 0xF8, 0xF6, 0xF9, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0xA5, 0xDE, 0xCA, 0x48, 0xA8, 0x73, 0x00, 0x00, /* 0x74-0x77 */ + 0xCD, 0xA5, 0xAA, 0xC6, 0xAA, 0xC5, 0xCD, 0xA6, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0xD0, 0x40, 0xAC, 0xEF, /* 0x7C-0x7F */ + + 0xCF, 0xFE, 0xAC, 0xF0, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0xAF, 0xB6, 0xD2, 0xF8, 0xD2, 0xF6, 0xD2, 0xFC, /* 0x84-0x87 */ + 0xAF, 0xB7, 0xD2, 0xF7, 0xD2, 0xFB, 0xD2, 0xF9, /* 0x88-0x8B */ + 0xD2, 0xFA, 0x00, 0x00, 0x00, 0x00, 0xD6, 0xC8, /* 0x8C-0x8F */ + 0xD6, 0xCA, 0x00, 0x00, 0xB2, 0xBF, 0x00, 0x00, /* 0x90-0x93 */ + 0xD6, 0xC9, 0xB2, 0xC0, 0xB5, 0xA2, 0xB5, 0xA1, /* 0x94-0x97 */ + 0xB5, 0x7E, 0xDA, 0xDB, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0xDF, 0x44, 0xB8, 0x5D, /* 0x9C-0x9F */ + 0xB8, 0x5E, 0x00, 0x00, 0xDF, 0x43, 0xDF, 0x42, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xE3, 0x4A, 0xBA, 0xDB, 0xBA, 0xDA, 0xE3, 0x4B, /* 0xA8-0xAB */ + 0xE3, 0x4C, 0x00, 0x00, 0xBD, 0x61, 0xBD, 0x60, /* 0xAC-0xAF */ + 0x00, 0x00, 0xEA, 0xB5, 0xE6, 0xD3, 0xE6, 0xD5, /* 0xB0-0xB3 */ + 0xE6, 0xD4, 0xEA, 0xB4, 0xEA, 0xB2, 0xEA, 0xB6, /* 0xB4-0xB7 */ + 0xEA, 0xB3, 0x00, 0x00, 0xBF, 0x73, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0xED, 0xB7, 0xC1, 0x4B, /* 0xBC-0xBF */ + 0xED, 0xB8, 0xED, 0xB9, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xC2, 0xAB, 0xC2, 0xAC, 0x00, 0x00, 0xC4, 0x75, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0xC5, 0xD1, 0xA5, 0xDF, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0xD0, 0x41, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xD2, 0xFD, 0xAF, 0xB8, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB3, 0xBA, /* 0xDC-0xDF */ + 0xB3, 0xB9, 0x00, 0x00, 0x00, 0x00, 0xB5, 0xA4, /* 0xE0-0xE3 */ + 0xDA, 0xDD, 0xB5, 0xA3, 0xDA, 0xDC, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDF, 0x45, /* 0xE8-0xEB */ + 0x00, 0x00, 0xBA, 0xDC, 0xE3, 0x4D, 0xBA, 0xDD, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0xC4, 0x76, 0xF4, 0xA5, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xA6, 0xCB, 0xAA, 0xC7, 0xCD, 0xA7, /* 0xF8-0xFB */ + 0x00, 0x00, 0xAC, 0xF2, 0x00, 0x00, 0xAC, 0xF1, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_7B[512] = { + 0xD0, 0x42, 0xD0, 0x43, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0xD3, 0x40, 0xD3, 0x42, 0xAF, 0xB9, 0x00, 0x00, /* 0x04-0x07 */ + 0xD3, 0x44, 0xD3, 0x47, 0xD3, 0x45, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0xD3, 0x46, 0xD3, 0x43, /* 0x0C-0x0F */ + 0xD2, 0xFE, 0xAF, 0xBA, 0xD3, 0x48, 0xD3, 0x41, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0xD6, 0xD3, 0xB2, 0xC6, 0xD6, 0xDC, 0xB2, 0xC3, /* 0x18-0x1B */ + 0x00, 0x00, 0xD6, 0xD5, 0xB2, 0xC7, 0x00, 0x00, /* 0x1C-0x1F */ + 0xB2, 0xC1, 0x00, 0x00, 0xD6, 0xD0, 0xD6, 0xDD, /* 0x20-0x23 */ + 0xD6, 0xD1, 0xD6, 0xCE, 0xB2, 0xC5, 0x00, 0x00, /* 0x24-0x27 */ + 0xB2, 0xC2, 0x00, 0x00, 0xD6, 0xD4, 0xD6, 0xD7, /* 0x28-0x2B */ + 0xB2, 0xC4, 0xD6, 0xD8, 0xB2, 0xC8, 0xD6, 0xD9, /* 0x2C-0x2F */ + 0xD6, 0xCF, 0xD6, 0xD6, 0xD6, 0xDA, 0xD6, 0xD2, /* 0x30-0x33 */ + 0xD6, 0xCD, 0xD6, 0xCB, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0xD6, 0xDB, 0x00, 0x00, 0x00, 0x00, 0xDA, 0xDF, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0xDA, 0xE4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0xDA, 0xE0, 0xDA, 0xE6, 0xB5, 0xA7, 0xD6, 0xCC, /* 0x44-0x47 */ + 0xDA, 0xE1, 0xB5, 0xA5, 0xDA, 0xDE, 0xB5, 0xAC, /* 0x48-0x4B */ + 0xDA, 0xE2, 0xB5, 0xAB, 0xDA, 0xE3, 0xB5, 0xAD, /* 0x4C-0x4F */ + 0xB5, 0xA8, 0xB5, 0xAE, 0xB5, 0xA9, 0x00, 0x00, /* 0x50-0x53 */ + 0xB5, 0xAA, 0x00, 0x00, 0xB5, 0xA6, 0x00, 0x00, /* 0x54-0x57 */ + 0xDA, 0xE5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0xB8, 0x61, 0xDF, 0x50, 0x00, 0x00, 0xDF, 0x53, /* 0x60-0x63 */ + 0xDF, 0x47, 0xDF, 0x4C, 0xDF, 0x46, 0xB8, 0x63, /* 0x64-0x67 */ + 0x00, 0x00, 0xDF, 0x4A, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0xDF, 0x48, 0xB8, 0x62, 0x00, 0x00, /* 0x6C-0x6F */ + 0xDF, 0x4F, 0xDF, 0x4E, 0xDF, 0x4B, 0xDF, 0x4D, /* 0x70-0x73 */ + 0xDF, 0x49, 0xBA, 0xE1, 0xDF, 0x52, 0xB8, 0x5F, /* 0x74-0x77 */ + 0xDF, 0x51, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0xE3, 0x5D, 0x00, 0x00, /* 0x80-0x83 */ + 0xBA, 0xE8, 0xE3, 0x58, 0x00, 0x00, 0xBA, 0xE7, /* 0x84-0x87 */ + 0xE3, 0x4E, 0x00, 0x00, 0xE3, 0x50, 0xBA, 0xE0, /* 0x88-0x8B */ + 0xE3, 0x55, 0xE3, 0x54, 0xE3, 0x57, 0xBA, 0xE5, /* 0x8C-0x8F */ + 0xE3, 0x52, 0xE3, 0x51, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0xBA, 0xE4, 0xBA, 0xDF, 0xE3, 0x53, 0xBA, 0xE2, /* 0x94-0x97 */ + 0xE3, 0x59, 0xE3, 0x5B, 0x00, 0x00, 0xE3, 0x56, /* 0x98-0x9B */ + 0xE3, 0x4F, 0xBA, 0xE3, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0xBD, 0x69, 0xBA, 0xDE, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xE3, 0x5C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0xE6, 0xD9, 0xBD, 0x62, 0x00, 0x00, 0xE6, 0xDB, /* 0xAC-0xAF */ + 0x00, 0x00, 0xBD, 0x63, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xBD, 0x65, 0xE6, 0xDE, 0x00, 0x00, 0xE6, 0xD6, /* 0xB4-0xB7 */ + 0xBA, 0xE6, 0xE6, 0xDC, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0xD8, 0x00, 0x00, /* 0xBC-0xBF */ + 0xB8, 0x60, 0xBD, 0x68, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xBD, 0x64, 0x00, 0x00, 0xBD, 0x66, 0xBD, 0x67, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xBF, 0x76, 0xE6, 0xDD, 0xE6, 0xD7, /* 0xC8-0xCB */ + 0xBD, 0x6A, 0x00, 0x00, 0xE6, 0xDA, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xEA, 0xC0, 0xEA, 0xBB, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xEA, 0xC5, 0xBF, 0x74, 0xEA, 0xBD, 0xBF, 0x78, /* 0xD8-0xDB */ + 0xEA, 0xC3, 0xEA, 0xBA, 0xEA, 0xB7, 0xEA, 0xC6, /* 0xDC-0xDF */ + 0xC1, 0x51, 0xBF, 0x79, 0xEA, 0xC2, 0xEA, 0xB8, /* 0xE0-0xE3 */ + 0xBF, 0x77, 0xEA, 0xBC, 0xBF, 0x7B, 0xEA, 0xB9, /* 0xE4-0xE7 */ + 0xEA, 0xBE, 0xBF, 0x7A, 0xEA, 0xC1, 0xEA, 0xC4, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0xED, 0xCB, 0xED, 0xCC, 0xED, 0xBC, 0xED, 0xC3, /* 0xF0-0xF3 */ + 0xED, 0xC1, 0x00, 0x00, 0x00, 0x00, 0xC1, 0x4F, /* 0xF4-0xF7 */ + 0xED, 0xC8, 0xEA, 0xBF, 0x00, 0x00, 0xED, 0xBF, /* 0xF8-0xFB */ + 0x00, 0x00, 0xED, 0xC9, 0xC1, 0x4E, 0xED, 0xBE, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_7C[512] = { + 0xED, 0xBD, 0xED, 0xC7, 0xED, 0xC4, 0xED, 0xC6, /* 0x00-0x03 */ + 0x00, 0x00, 0xED, 0xBA, 0xED, 0xCA, 0xC1, 0x4C, /* 0x04-0x07 */ + 0x00, 0x00, 0xED, 0xC5, 0xED, 0xCE, 0xED, 0xC2, /* 0x08-0x0B */ + 0xC1, 0x50, 0xC1, 0x4D, 0xED, 0xC0, 0xED, 0xBB, /* 0x0C-0x0F */ + 0xED, 0xCD, 0xBF, 0x75, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0xF0, 0x63, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0xF0, 0x61, 0xF0, 0x67, 0xC2, 0xB0, 0xF0, 0x65, /* 0x1C-0x1F */ + 0xF0, 0x64, 0xC2, 0xB2, 0xF0, 0x6A, 0xC2, 0xB1, /* 0x20-0x23 */ + 0x00, 0x00, 0xF0, 0x6B, 0xF0, 0x68, 0xC2, 0xAE, /* 0x24-0x27 */ + 0xF0, 0x69, 0xF0, 0x62, 0xC2, 0xAF, 0xC2, 0xAD, /* 0x28-0x2B */ + 0xF2, 0xAB, 0xF0, 0x66, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xF0, 0x6C, 0x00, 0x00, 0x00, 0x00, 0xF2, 0xA8, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC3, 0xB2, /* 0x34-0x37 */ + 0xC3, 0xB0, 0xF2, 0xAA, 0x00, 0x00, 0xF2, 0xAC, /* 0x38-0x3B */ + 0xF2, 0xA9, 0xC3, 0xB1, 0xC3, 0xAE, 0xC3, 0xAF, /* 0x3C-0x3F */ + 0xC3, 0xB3, 0x00, 0x00, 0x00, 0x00, 0xC4, 0x78, /* 0x40-0x43 */ + 0x00, 0x00, 0xF4, 0xAA, 0x00, 0x00, 0xF4, 0xA9, /* 0x44-0x47 */ + 0xF4, 0xA7, 0xF4, 0xA6, 0xF4, 0xA8, 0x00, 0x00, /* 0x48-0x4B */ + 0xC4, 0x77, 0xC4, 0x79, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0xC4, 0xF0, 0x00, 0x00, 0x00, 0x00, 0xF5, 0xE5, /* 0x50-0x53 */ + 0xF5, 0xE4, 0x00, 0x00, 0x00, 0x00, 0xF6, 0xFA, /* 0x54-0x57 */ + 0x00, 0x00, 0xF6, 0xFC, 0xF6, 0xFE, 0xF6, 0xFD, /* 0x58-0x5B */ + 0xF6, 0xFB, 0x00, 0x00, 0x00, 0x00, 0xC5, 0xA3, /* 0x5C-0x5F */ + 0xC5, 0xA2, 0x00, 0x00, 0x00, 0x00, 0xC5, 0xD3, /* 0x60-0x63 */ + 0xC5, 0xD2, 0xC5, 0xD4, 0xF7, 0xED, 0xF7, 0xEC, /* 0x64-0x67 */ + 0x00, 0x00, 0xF8, 0xFB, 0xF8, 0xB8, 0xF8, 0xFC, /* 0x68-0x6B */ + 0xC6, 0x58, 0x00, 0x00, 0xC6, 0x59, 0xF9, 0x6D, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0xC6, 0x7E, 0xA6, 0xCC, /* 0x70-0x73 */ + 0x00, 0x00, 0xCD, 0xA8, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0xD0, 0x45, 0xD0, 0x46, 0xD0, 0x44, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0xAC, 0xF3, 0x00, 0x00, 0xD0, 0x47, /* 0x7C-0x7F */ + + 0xD0, 0x48, 0xD0, 0x49, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0xD3, 0x49, 0xD3, 0x4F, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0xD3, 0x4D, 0xAF, 0xBB, 0xD3, 0x4B, 0x00, 0x00, /* 0x88-0x8B */ + 0xD3, 0x4C, 0xD3, 0x4E, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xD3, 0x4A, 0xB2, 0xC9, 0x00, 0x00, /* 0x90-0x93 */ + 0xD6, 0xDE, 0xB2, 0xCB, 0xD6, 0xE0, 0xB2, 0xCA, /* 0x94-0x97 */ + 0xD6, 0xDF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0xDA, 0xE8, 0xB5, 0xAF, /* 0x9C-0x9F */ + 0x00, 0x00, 0xDA, 0xEA, 0xDA, 0xE7, 0xD6, 0xE1, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xB5, 0xB0, 0x00, 0x00, 0xF9, 0xDB, /* 0xA4-0xA7 */ + 0xDA, 0xE9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDF, 0x56, /* 0xAC-0xAF */ + 0x00, 0x00, 0xB8, 0x64, 0xDF, 0x54, 0xB8, 0x65, /* 0xB0-0xB3 */ + 0xDF, 0x55, 0xB8, 0x66, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xBA, 0xE9, 0xE3, 0x61, 0xE3, 0x5E, /* 0xB8-0xBB */ + 0xE3, 0x60, 0xBA, 0xEA, 0xBA, 0xEB, 0xE3, 0x5F, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xE6, 0xDF, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0xE6, 0xE0, 0x00, 0x00, 0xBD, 0x6B, 0xE6, 0xE2, /* 0xC8-0xCB */ + 0xE6, 0xE1, 0x00, 0x00, 0xA2, 0x61, 0x00, 0x00, /* 0xCC-0xCF */ + 0xEA, 0xCA, 0xEA, 0xCB, 0xEA, 0xC7, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xEA, 0xC8, 0xBF, 0x7C, 0xBF, 0x7D, 0xEA, 0xC9, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xC1, 0x57, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0xC1, 0x53, 0xC1, 0x58, 0xC1, 0x54, 0xC1, 0x56, /* 0xDC-0xDF */ + 0xC1, 0x52, 0x00, 0x00, 0xC1, 0x55, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC2, 0xB3, /* 0xE4-0xE7 */ + 0xED, 0xCF, 0x00, 0x00, 0xF2, 0xAE, 0x00, 0x00, /* 0xE8-0xEB */ + 0xF2, 0xAD, 0x00, 0x00, 0xF4, 0xAB, 0xC4, 0x7A, /* 0xEC-0xEF */ + 0xC4, 0x7B, 0xF7, 0x41, 0xF5, 0xE6, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xF7, 0x40, 0x00, 0x00, 0xF8, 0xFD, 0xF9, 0xA4, /* 0xF4-0xF7 */ + 0xA6, 0xCD, 0x00, 0x00, 0x00, 0x00, 0xA8, 0x74, /* 0xF8-0xFB */ + 0x00, 0x00, 0xCD, 0xA9, 0xAA, 0xC8, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_7D[512] = { + 0xAC, 0xF6, 0xD0, 0x4C, 0xAC, 0xF4, 0xD0, 0x4A, /* 0x00-0x03 */ + 0xAC, 0xF9, 0xAC, 0xF5, 0xAC, 0xFA, 0xAC, 0xF8, /* 0x04-0x07 */ + 0xD0, 0x4B, 0xAC, 0xF7, 0xAF, 0xBF, 0xAF, 0xBE, /* 0x08-0x0B */ + 0xD3, 0x5A, 0xAF, 0xC7, 0xD3, 0x53, 0xD3, 0x59, /* 0x0C-0x0F */ + 0xAF, 0xC3, 0xD3, 0x52, 0xD3, 0x58, 0xD3, 0x56, /* 0x10-0x13 */ + 0xAF, 0xC2, 0xAF, 0xC4, 0xD3, 0x55, 0xAF, 0xBD, /* 0x14-0x17 */ + 0xD3, 0x54, 0xAF, 0xC8, 0xAF, 0xC5, 0xAF, 0xC9, /* 0x18-0x1B */ + 0xAF, 0xC6, 0xD3, 0x51, 0xD3, 0x50, 0xD3, 0x57, /* 0x1C-0x1F */ + 0xAF, 0xC0, 0xAF, 0xBC, 0xAF, 0xC1, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0xD6, 0xF0, 0xD6, 0xE9, 0x00, 0x00, 0xB5, 0xB5, /* 0x28-0x2B */ + 0xD6, 0xE8, 0x00, 0x00, 0xB2, 0xCF, 0xB2, 0xD6, /* 0x2C-0x2F */ + 0xB2, 0xD3, 0xB2, 0xD9, 0xB2, 0xD8, 0xB2, 0xD4, /* 0x30-0x33 */ + 0x00, 0x00, 0xD6, 0xE2, 0xD6, 0xE5, 0x00, 0x00, /* 0x34-0x37 */ + 0xD6, 0xE4, 0xB2, 0xD0, 0xD6, 0xE6, 0xD6, 0xEF, /* 0x38-0x3B */ + 0xB2, 0xD1, 0xD6, 0xE3, 0xD6, 0xEC, 0xD6, 0xED, /* 0x3C-0x3F */ + 0xB2, 0xD2, 0xD6, 0xEA, 0xB2, 0xD7, 0xB2, 0xCD, /* 0x40-0x43 */ + 0xB2, 0xD5, 0xD6, 0xE7, 0xB2, 0xCC, 0xD6, 0xEB, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0xD6, 0xEE, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0xDA, 0xFB, 0xDA, 0xF2, /* 0x4C-0x4F */ + 0xB5, 0xB2, 0xDA, 0xF9, 0xDA, 0xF6, 0xDA, 0xEE, /* 0x50-0x53 */ + 0xDA, 0xF7, 0xB5, 0xB4, 0xDA, 0xEF, 0x00, 0x00, /* 0x54-0x57 */ + 0xDA, 0xEB, 0x00, 0x00, 0x00, 0x00, 0xB8, 0x6C, /* 0x58-0x5B */ + 0xDA, 0xF4, 0x00, 0x00, 0xB5, 0xB1, 0xDA, 0xFA, /* 0x5C-0x5F */ + 0x00, 0x00, 0xB5, 0xB8, 0xB5, 0xBA, 0xDA, 0xED, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0xB5, 0xB9, 0xDA, 0xF0, /* 0x64-0x67 */ + 0xB5, 0xB3, 0xDA, 0xF8, 0xDA, 0xF1, 0xDA, 0xF5, /* 0x68-0x6B */ + 0x00, 0x00, 0xDA, 0xF3, 0xB5, 0xB6, 0xDA, 0xEC, /* 0x6C-0x6F */ + 0xB5, 0xBB, 0xB2, 0xCE, 0xB5, 0xB7, 0xB5, 0xBC, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0xB8, 0x68, 0xDF, 0x5D, 0xDF, 0x5F, /* 0x78-0x7B */ + 0xDF, 0x61, 0xDF, 0x65, 0x00, 0x00, 0xDF, 0x5B, /* 0x7C-0x7F */ + + 0xDF, 0x59, 0xB8, 0x6A, 0x00, 0x00, 0xDF, 0x60, /* 0x80-0x83 */ + 0xDF, 0x64, 0xDF, 0x5C, 0xDF, 0x58, 0x00, 0x00, /* 0x84-0x87 */ + 0xDF, 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0xDF, 0x62, 0xDF, 0x5A, 0xDF, 0x5E, 0xB8, 0x6B, /* 0x8C-0x8F */ + 0x00, 0x00, 0xB8, 0x69, 0xDF, 0x66, 0xB8, 0x67, /* 0x90-0x93 */ + 0xDF, 0x63, 0x00, 0x00, 0xE3, 0x72, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0xBA, 0xEE, 0xE3, 0x6A, 0xBD, 0x78, 0xE3, 0x74, /* 0x9C-0x9F */ + 0xBA, 0xF1, 0xE3, 0x78, 0xBA, 0xF7, 0xE3, 0x65, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0x75, 0xE3, 0x62, /* 0xA4-0xA7 */ + 0x00, 0x00, 0xE3, 0x77, 0xE3, 0x66, 0x00, 0x00, /* 0xA8-0xAB */ + 0xBA, 0xFE, 0xBA, 0xFB, 0xE3, 0x76, 0xE3, 0x70, /* 0xAC-0xAF */ + 0xBA, 0xED, 0xBA, 0xF5, 0xBA, 0xF4, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xBA, 0xF3, 0xBA, 0xF9, 0x00, 0x00, 0xE3, 0x63, /* 0xB4-0xB7 */ + 0xBA, 0xFA, 0xE3, 0x71, 0xBA, 0xF6, 0xBA, 0xEC, /* 0xB8-0xBB */ + 0xE3, 0x73, 0xBA, 0xEF, 0xBA, 0xF0, 0xBA, 0xF8, /* 0xBC-0xBF */ + 0xE3, 0x68, 0xE3, 0x67, 0xE3, 0x64, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xE3, 0x6C, 0xE3, 0x69, 0xE3, 0x6D, 0xBA, 0xFD, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xE3, 0x79, 0xBA, 0xF2, 0xE3, 0x6E, /* 0xC8-0xCB */ + 0xE3, 0x6F, 0x00, 0x00, 0xE3, 0x6B, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xBA, 0xFC, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE6, 0xE7, /* 0xD4-0xD7 */ + 0xBD, 0x70, 0xBD, 0x79, 0xBD, 0x75, 0xE6, 0xE4, /* 0xD8-0xDB */ + 0x00, 0x00, 0xBD, 0x72, 0xBD, 0x76, 0xE6, 0xF0, /* 0xDC-0xDF */ + 0xBD, 0x6C, 0xE6, 0xE8, 0x00, 0x00, 0xBD, 0x74, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0xE6, 0xEB, 0xE6, 0xE6, /* 0xE4-0xE7 */ + 0xBD, 0x73, 0xBD, 0x77, 0xE6, 0xE5, 0x00, 0x00, /* 0xE8-0xEB */ + 0xBD, 0x71, 0x00, 0x00, 0xE6, 0xEF, 0xBD, 0x6E, /* 0xEC-0xEF */ + 0xE6, 0xEE, 0xE6, 0xED, 0xBD, 0x7A, 0xE5, 0x72, /* 0xF0-0xF3 */ + 0xBD, 0x6D, 0x00, 0x00, 0xE6, 0xEC, 0xE6, 0xE3, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xBD, 0x7B, 0xE6, 0xEA, 0xBD, 0x6F, /* 0xF8-0xFB */ +}; + +static const unsigned char u2c_7E[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE6, 0xE9, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0xBF, 0xA2, 0xBF, 0xA7, 0xBF, 0x7E, 0xEA, 0xD8, /* 0x08-0x0B */ + 0xEA, 0xCF, 0xEA, 0xDB, 0xEA, 0xD3, 0xEA, 0xD9, /* 0x0C-0x0F */ + 0xBF, 0xA8, 0xBF, 0xA1, 0xEA, 0xCC, 0xEA, 0xD2, /* 0x10-0x13 */ + 0xEA, 0xDC, 0xEA, 0xD5, 0xEA, 0xDA, 0xEA, 0xCE, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0xEA, 0xD6, 0xBF, 0xA3, /* 0x18-0x1B */ + 0xEA, 0xD4, 0xBF, 0xA6, 0xBF, 0xA5, 0xEA, 0xD0, /* 0x1C-0x1F */ + 0xEA, 0xD1, 0xEA, 0xCD, 0xEA, 0xD7, 0xBF, 0xA4, /* 0x20-0x23 */ + 0xEA, 0xDE, 0xEA, 0xDD, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0xED, 0xDA, 0xED, 0xD6, 0xC1, 0x5F, /* 0x28-0x2B */ + 0x00, 0x00, 0xED, 0xD0, 0xC1, 0x59, 0xC1, 0x69, /* 0x2C-0x2F */ + 0xED, 0xDC, 0xC1, 0x61, 0xC1, 0x5D, 0xED, 0xD3, /* 0x30-0x33 */ + 0xC1, 0x64, 0xC1, 0x67, 0xED, 0xDE, 0xC1, 0x5C, /* 0x34-0x37 */ + 0xED, 0xD5, 0xC1, 0x65, 0xED, 0xE0, 0xED, 0xDD, /* 0x38-0x3B */ + 0xED, 0xD1, 0xC1, 0x60, 0xC1, 0x5A, 0xC1, 0x68, /* 0x3C-0x3F */ + 0xED, 0xD8, 0xC1, 0x63, 0xED, 0xD2, 0xC1, 0x5E, /* 0x40-0x43 */ + 0xED, 0xDF, 0xC1, 0x62, 0xC1, 0x5B, 0xED, 0xD9, /* 0x44-0x47 */ + 0xC1, 0x66, 0xED, 0xD7, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0xED, 0xDB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0xF0, 0x6E, 0xF0, 0x74, 0xC2, 0xB9, 0xF0, 0x77, /* 0x50-0x53 */ + 0xC2, 0xB4, 0xC2, 0xB5, 0xF0, 0x6F, 0xF0, 0x76, /* 0x54-0x57 */ + 0xF0, 0x71, 0xC2, 0xBA, 0xC2, 0xB7, 0x00, 0x00, /* 0x58-0x5B */ + 0xF0, 0x6D, 0x00, 0x00, 0xC2, 0xB6, 0xF0, 0x73, /* 0x5C-0x5F */ + 0xF0, 0x75, 0xC2, 0xB8, 0xF0, 0x72, 0xF0, 0x70, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0xF2, 0xB8, 0xC3, 0xB7, 0xC3, 0xB8, 0xC3, 0xB4, /* 0x68-0x6B */ + 0x00, 0x00, 0xC3, 0xB5, 0x00, 0x00, 0xF2, 0xB4, /* 0x6C-0x6F */ + 0xF2, 0xB2, 0x00, 0x00, 0xF2, 0xB6, 0xC3, 0xBA, /* 0x70-0x73 */ + 0xF2, 0xB7, 0xF2, 0xB0, 0xF2, 0xAF, 0xF2, 0xB3, /* 0x74-0x77 */ + 0xF2, 0xB1, 0xC3, 0xB6, 0xF2, 0xB5, 0xF4, 0xAC, /* 0x78-0x7B */ + 0xC4, 0x7E, 0xC4, 0x7D, 0xF4, 0xAD, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xF4, 0xAF, 0xF4, 0xAE, 0xC4, 0xA1, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0xF5, 0xEB, 0xF5, 0xE8, /* 0x84-0x87 */ + 0xF5, 0xE9, 0x00, 0x00, 0xF5, 0xE7, 0xF5, 0xEA, /* 0x88-0x8B */ + 0xC4, 0xF2, 0xF5, 0xEC, 0x00, 0x00, 0xC4, 0xF1, /* 0x8C-0x8F */ + 0x00, 0x00, 0xF7, 0x42, 0x00, 0x00, 0xC5, 0xD5, /* 0x90-0x93 */ + 0xC5, 0xD7, 0xF7, 0xEE, 0xC5, 0xD6, 0xF8, 0xB9, /* 0x94-0x97 */ + 0xF9, 0x40, 0xF9, 0x42, 0xF8, 0xFE, 0xF9, 0x41, /* 0x98-0x9B */ + 0xC6, 0x6C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ +}; + +static const unsigned char u2c_7F[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0xA6, 0xCE, 0x00, 0x00, /* 0x34-0x37 */ + 0xAC, 0xFB, 0xD2, 0x6F, 0xAF, 0xCA, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0xB2, 0xDA, 0xDA, 0xFC, 0xDA, 0xFD, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEA, 0xDF, /* 0x40-0x43 */ + 0xC1, 0x6A, 0xED, 0xE1, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0xC2, 0xBB, 0x00, 0x00, 0xF2, 0xBA, 0xF2, 0xB9, /* 0x48-0x4B */ + 0xC4, 0xA2, 0xF5, 0xED, 0x00, 0x00, 0xF7, 0x43, /* 0x4C-0x4F */ + 0xC5, 0xF8, 0xCA, 0x49, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0xAA, 0xC9, 0xA8, 0x75, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0xD0, 0x4D, 0x00, 0x00, 0x00, 0x00, 0xD3, 0x60, /* 0x58-0x5B */ + 0xD3, 0x5B, 0xD3, 0x5F, 0xD3, 0x5D, 0xAF, 0xCB, /* 0x5C-0x5F */ + 0xD3, 0x5E, 0xD3, 0x5C, 0x00, 0x00, 0xD6, 0xF1, /* 0x60-0x63 */ + 0x00, 0x00, 0xDA, 0xFE, 0xDB, 0x40, 0xDF, 0x69, /* 0x64-0x67 */ + 0xDF, 0x6A, 0xB8, 0x6E, 0xB8, 0x6F, 0xDF, 0x68, /* 0x68-0x6B */ + 0xDF, 0x6B, 0xDF, 0x67, 0xB8, 0x6D, 0x00, 0x00, /* 0x6C-0x6F */ + 0xBB, 0x40, 0x00, 0x00, 0xB8, 0x70, 0xE3, 0x7A, /* 0x70-0x73 */ + 0x00, 0x00, 0xBD, 0x7C, 0xE6, 0xF1, 0xBD, 0x7D, /* 0x74-0x77 */ + 0x00, 0x00, 0xBF, 0xA9, 0xEA, 0xE2, 0xEA, 0xE0, /* 0x78-0x7B */ + 0xEA, 0xE1, 0xED, 0xE4, 0xED, 0xE3, 0xED, 0xE2, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF2, 0xBB, /* 0x80-0x83 */ + 0x00, 0x00, 0xC3, 0xB9, 0xF2, 0xBC, 0xF7, 0x44, /* 0x84-0x87 */ + 0xC5, 0xF9, 0xF8, 0xBA, 0xA6, 0xCF, 0xAA, 0xCB, /* 0x88-0x8B */ + 0xAA, 0xCA, 0xD0, 0x4F, 0xAC, 0xFC, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xD0, 0x4E, 0xD3, 0x62, 0x00, 0x00, /* 0x90-0x93 */ + 0xAF, 0xCC, 0xD6, 0xF2, 0xD3, 0x61, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0xB2, 0xDC, 0xD6, 0xF5, /* 0x98-0x9B */ + 0xD6, 0xF3, 0xD6, 0xF4, 0xB2, 0xDB, 0x00, 0x00, /* 0x9C-0x9F */ + 0xDB, 0x42, 0xDB, 0x43, 0xDB, 0x41, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xB8, 0x73, 0xDF, 0x6D, 0xDF, 0x6C, 0xDF, 0x6E, /* 0xA4-0xA7 */ + 0xB8, 0x72, 0xB8, 0x71, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0xE6, 0xF2, 0xE6, 0xF4, 0x00, 0x00, 0xBD, 0x7E, /* 0xAC-0xAF */ + 0xE6, 0xF3, 0xEA, 0xE3, 0xBF, 0xAA, 0xF0, 0x79, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xF0, 0x78, 0xC3, 0xBB, 0xF2, 0xBD, /* 0xB4-0xB7 */ + 0xC3, 0xBD, 0xC3, 0xBC, 0xF4, 0xB0, 0xF5, 0xEE, /* 0xB8-0xBB */ + 0xC4, 0xF3, 0xA6, 0xD0, 0xD0, 0x50, 0xAC, 0xFD, /* 0xBC-0xBF */ + 0xD3, 0x65, 0xAF, 0xCE, 0xD3, 0x64, 0xD3, 0x63, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xAF, 0xCD, 0x00, 0x00, 0xD6, 0xFB, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xD6, 0xFD, 0xD6, 0xF6, 0xD6, 0xF7, /* 0xC8-0xCB */ + 0xB2, 0xDD, 0xD6, 0xF8, 0xB2, 0xDE, 0xD6, 0xFC, /* 0xCC-0xCF */ + 0xD6, 0xF9, 0xD6, 0xFA, 0xB2, 0xDF, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xB5, 0xBE, 0xB5, 0xBF, 0x00, 0x00, 0xDB, 0x44, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDF, 0x6F, /* 0xD8-0xDB */ + 0xDF, 0x70, 0x00, 0x00, 0xE3, 0x7E, 0xBB, 0x43, /* 0xDC-0xDF */ + 0xBB, 0x41, 0xBB, 0x42, 0xE3, 0x7B, 0xE3, 0x7C, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xE3, 0x7D, 0xE6, 0xF9, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xE6, 0xFA, 0xBD, 0xA1, 0xE6, 0xF7, 0xE6, 0xF6, /* 0xE8-0xEB */ + 0xE6, 0xF8, 0xE6, 0xF5, 0xBF, 0xAD, 0xEA, 0xE4, /* 0xEC-0xEF */ + 0xBF, 0xAB, 0xBF, 0xAC, 0xED, 0xE6, 0xC1, 0x6B, /* 0xF0-0xF3 */ + 0xED, 0xE5, 0xEF, 0xA8, 0x00, 0x00, 0xF0, 0x7A, /* 0xF4-0xF7 */ + 0xF0, 0x7B, 0xC2, 0xBC, 0x00, 0x00, 0xC2, 0xBD, /* 0xF8-0xFB */ + 0xC1, 0x6C, 0xF2, 0xBE, 0xF2, 0xBF, 0xF4, 0xB1, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_80[512] = { + 0xC4, 0xA3, 0xA6, 0xD1, 0x00, 0x00, 0xA6, 0xD2, /* 0x00-0x03 */ + 0xAC, 0xFE, 0xAA, 0xCC, 0xAF, 0xCF, 0xD0, 0x51, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB5, 0xC0, /* 0x08-0x0B */ + 0xA6, 0xD3, 0xAD, 0x41, 0xD0, 0x52, 0xD0, 0x53, /* 0x0C-0x0F */ + 0xAD, 0x40, 0xAD, 0x42, 0xA6, 0xD4, 0x00, 0x00, /* 0x10-0x13 */ + 0xD0, 0x54, 0xAF, 0xD1, 0xD3, 0x66, 0xAF, 0xD3, /* 0x14-0x17 */ + 0xAF, 0xD0, 0xAF, 0xD2, 0x00, 0x00, 0xD7, 0x41, /* 0x18-0x1B */ + 0xB2, 0xE0, 0x00, 0x00, 0xD7, 0x40, 0xD6, 0xFE, /* 0x1C-0x1F */ + 0x00, 0x00, 0xDF, 0x71, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0xE3, 0xA1, 0x00, 0x00, 0xBD, 0xA2, 0x00, 0x00, /* 0x24-0x27 */ + 0xBF, 0xAE, 0xEA, 0xE6, 0xEA, 0xE5, 0x00, 0x00, /* 0x28-0x2B */ + 0xED, 0xE7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xF5, 0xEF, 0x00, 0x00, 0x00, 0x00, 0xA6, 0xD5, /* 0x30-0x33 */ + 0xCB, 0x73, 0xCD, 0xAA, 0xAD, 0x43, 0xD0, 0x55, /* 0x34-0x37 */ + 0x00, 0x00, 0xD3, 0x68, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0xAF, 0xD4, 0xD3, 0x67, 0xAF, 0xD5, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD7, 0x43, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xB2, 0xE2, 0xD7, 0x42, /* 0x44-0x47 */ + 0xD7, 0x44, 0x00, 0x00, 0xB2, 0xE1, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDB, 0x46, /* 0x4C-0x4F */ + 0xDB, 0x47, 0xDB, 0x45, 0xB5, 0xC1, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0xB8, 0x74, 0x00, 0x00, /* 0x54-0x57 */ + 0xB8, 0x75, 0x00, 0x00, 0xBB, 0x45, 0x00, 0x00, /* 0x58-0x5B */ + 0xE3, 0xA3, 0xE3, 0xA2, 0xBB, 0x44, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0xE6, 0xFB, 0x00, 0x00, 0x00, 0x00, 0xE6, 0xFC, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0xEA, 0xE7, 0x00, 0x00, 0x00, 0x00, 0xC1, 0x70, /* 0x6C-0x6F */ + 0xC1, 0x6F, 0xC1, 0x6D, 0xC1, 0x6E, 0xC1, 0x71, /* 0x70-0x73 */ + 0x00, 0x00, 0xF0, 0x7C, 0xC2, 0xBF, 0xC2, 0xBE, /* 0x74-0x77 */ + 0xF2, 0xC0, 0xF4, 0xB2, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0xC5, 0xA5, 0xC5, 0xA4, 0xA6, 0xD6, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0xD1, 0xFB, 0x00, 0x00, /* 0x80-0x83 */ + 0xB8, 0x77, 0xB5, 0xC2, 0xB8, 0x76, 0xBB, 0x46, /* 0x84-0x87 */ + 0x00, 0x00, 0xA6, 0xD7, 0xC9, 0xA9, 0xA6, 0xD8, /* 0x88-0x8B */ + 0xA6, 0xD9, 0x00, 0x00, 0x00, 0x00, 0xCD, 0xAB, /* 0x8C-0x8F */ + 0xCB, 0x76, 0x00, 0x00, 0xCB, 0x77, 0xA8, 0x77, /* 0x90-0x93 */ + 0x00, 0x00, 0xCB, 0x74, 0xA8, 0x76, 0x00, 0x00, /* 0x94-0x97 */ + 0xA8, 0x79, 0xCB, 0x75, 0xA8, 0x7B, 0xA8, 0x7A, /* 0x98-0x9B */ + 0xCB, 0x78, 0xA8, 0x78, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0xAA, 0xD1, 0xAA, 0xCF, 0xCD, 0xAD, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xAA, 0xCE, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0xAA, 0xD3, 0xAA, 0xD5, 0xAA, 0xD2, /* 0xA8-0xAB */ + 0x00, 0x00, 0xCD, 0xB0, 0xCD, 0xAC, 0xAA, 0xD6, /* 0xAC-0xAF */ + 0x00, 0x00, 0xAA, 0xD0, 0xA8, 0x7C, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xAA, 0xD4, 0xCD, 0xAF, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xCD, 0xAE, 0x00, 0x00, 0xAA, 0xCD, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0xD0, 0x5B, 0xAD, 0x47, /* 0xC0-0xC3 */ + 0xAD, 0x48, 0xD0, 0x5D, 0x00, 0x00, 0xD0, 0x57, /* 0xC4-0xC7 */ + 0xD0, 0x5A, 0xD0, 0x63, 0xD0, 0x61, 0x00, 0x00, /* 0xC8-0xCB */ + 0xAD, 0x49, 0xD0, 0x67, 0xAD, 0x4C, 0xD0, 0x64, /* 0xCC-0xCF */ + 0xD0, 0x5C, 0xD0, 0x59, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xDB, 0x49, 0xD0, 0x62, 0xAD, 0x44, 0xD0, 0x65, /* 0xD4-0xD7 */ + 0xD0, 0x56, 0xD0, 0x5F, 0xAD, 0x46, 0xAD, 0x4B, /* 0xD8-0xDB */ + 0xD0, 0x60, 0xAD, 0x4F, 0xAD, 0x4D, 0x00, 0x00, /* 0xDC-0xDF */ + 0xD0, 0x58, 0xAD, 0x4A, 0x00, 0x00, 0xD0, 0x5E, /* 0xE0-0xE3 */ + 0xAD, 0x4E, 0xAD, 0x45, 0xD0, 0x66, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0xAF, 0xDA, 0x00, 0x00, 0xAF, 0xE3, /* 0xEC-0xEF */ + 0xAF, 0xD8, 0xAF, 0xD6, 0xD3, 0x6A, 0xAF, 0xDE, /* 0xF0-0xF3 */ + 0xAF, 0xDB, 0xD3, 0x6C, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xAF, 0xDD, 0xD3, 0x6B, 0xD3, 0x69, 0xD3, 0x6E, /* 0xF8-0xFB */ + 0xAF, 0xE2, 0xAF, 0xE0, 0xDB, 0x48, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_81[512] = { + 0xD3, 0x6F, 0xD3, 0x6D, 0xAF, 0xD7, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0xAF, 0xD9, 0xAF, 0xDC, 0x00, 0x00, /* 0x04-0x07 */ + 0xAF, 0xDF, 0x00, 0x00, 0xAF, 0xE1, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0xD7, 0x4E, 0xB2, 0xE4, 0x00, 0x00, /* 0x14-0x17 */ + 0xD7, 0x45, 0xD7, 0x47, 0x00, 0x00, 0xD7, 0x48, /* 0x18-0x1B */ + 0x00, 0x00, 0xD7, 0x50, 0xD7, 0x4C, 0xD7, 0x4A, /* 0x1C-0x1F */ + 0x00, 0x00, 0xD7, 0x4D, 0xD7, 0x51, 0xB2, 0xE5, /* 0x20-0x23 */ + 0xB2, 0xE9, 0xD7, 0x46, 0x00, 0x00, 0xD7, 0x4F, /* 0x24-0x27 */ + 0x00, 0x00, 0xB2, 0xE7, 0x00, 0x00, 0xB2, 0xE6, /* 0x28-0x2B */ + 0xD7, 0x4B, 0xD7, 0x49, 0x00, 0x00, 0xB2, 0xE3, /* 0x2C-0x2F */ + 0xB2, 0xE8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0xB5, 0xC8, 0xDB, 0x51, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0xDB, 0x4F, 0xB5, 0xCA, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDB, 0x4A, /* 0x40-0x43 */ + 0xDF, 0xA1, 0x00, 0x00, 0xB5, 0xC9, 0xDB, 0x4E, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0xDB, 0x4B, 0xB5, 0xC5, /* 0x48-0x4B */ + 0xB5, 0xCB, 0xDB, 0x50, 0xB5, 0xC7, 0xDB, 0x4D, /* 0x4C-0x4F */ + 0xBB, 0x47, 0xB5, 0xC6, 0xDB, 0x4C, 0xB5, 0xCC, /* 0x50-0x53 */ + 0xB5, 0xC4, 0xB5, 0xC3, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDF, 0x77, /* 0x58-0x5B */ + 0xDF, 0x75, 0x00, 0x00, 0xDF, 0x7B, 0x00, 0x00, /* 0x5C-0x5F */ + 0xDF, 0x73, 0xDF, 0xA2, 0xDF, 0x78, 0x00, 0x00, /* 0x60-0x63 */ + 0xDF, 0x72, 0xB8, 0x7B, 0xB8, 0xA3, 0xDF, 0x7D, /* 0x64-0x67 */ + 0x00, 0x00, 0xDF, 0x76, 0x00, 0x00, 0xB8, 0x7E, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0xB8, 0x7C, 0xDF, 0x7E, /* 0x6C-0x6F */ + 0xB8, 0x79, 0xB8, 0x78, 0xDF, 0x79, 0xB8, 0x7D, /* 0x70-0x73 */ + 0xB5, 0xCD, 0x00, 0x00, 0xDF, 0x7C, 0xDF, 0x74, /* 0x74-0x77 */ + 0xB8, 0x7A, 0xB8, 0xA1, 0xB8, 0xA2, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBB, 0x4C, /* 0x7C-0x7F */ + + 0xBB, 0x48, 0x00, 0x00, 0xBB, 0x4D, 0xE3, 0xA6, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xA5, 0xE3, 0xA7, /* 0x84-0x87 */ + 0xBB, 0x4A, 0xE3, 0xA4, 0xBB, 0x4B, 0xE3, 0xAA, /* 0x88-0x8B */ + 0xE3, 0xA9, 0xE3, 0xA8, 0x00, 0x00, 0xBB, 0x49, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0xE7, 0x41, 0x00, 0x00, 0xE7, 0x44, /* 0x94-0x97 */ + 0xBD, 0xA8, 0xE7, 0x43, 0xBD, 0xA7, 0xBD, 0xA3, /* 0x98-0x9B */ + 0xBD, 0xA4, 0xBD, 0xA5, 0xE7, 0x40, 0xE6, 0xFE, /* 0x9C-0x9F */ + 0xBD, 0xA6, 0x00, 0x00, 0xE7, 0x42, 0xE6, 0xFD, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0xEA, 0xE9, 0xEA, 0xF3, /* 0xA4-0xA7 */ + 0xBF, 0xB1, 0xBF, 0xB0, 0x00, 0x00, 0xEA, 0xED, /* 0xA8-0xAB */ + 0xEA, 0xEF, 0x00, 0x00, 0xEA, 0xEA, 0x00, 0x00, /* 0xAC-0xAF */ + 0xEA, 0xEE, 0xEA, 0xE8, 0xEA, 0xF1, 0xBF, 0xAF, /* 0xB0-0xB3 */ + 0xEA, 0xF0, 0xEA, 0xEC, 0x00, 0x00, 0xEA, 0xF2, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xEA, 0xEB, 0xC1, 0x74, 0xED, 0xE8, /* 0xB8-0xBB */ + 0xED, 0xEE, 0xC1, 0x78, 0xC1, 0x7A, 0xC1, 0x77, /* 0xBC-0xBF */ + 0xC1, 0x76, 0x00, 0x00, 0xC1, 0x75, 0xC1, 0x73, /* 0xC0-0xC3 */ + 0xED, 0xE9, 0xED, 0xEC, 0xC1, 0x72, 0xED, 0xED, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xC1, 0x79, 0xED, 0xEB, 0x00, 0x00, /* 0xC8-0xCB */ + 0xED, 0xEA, 0xC2, 0xC0, 0x00, 0x00, 0xC2, 0xC1, /* 0xCC-0xCF */ + 0xF0, 0xA1, 0xF0, 0x7D, 0xF0, 0x7E, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xF2, 0xC2, 0x00, 0x00, 0xF2, 0xC1, /* 0xD4-0xD7 */ + 0xC3, 0xBE, 0xF4, 0xB4, 0xC4, 0xA4, 0xF4, 0xB3, /* 0xD8-0xDB */ + 0x00, 0x00, 0xF5, 0xF0, 0xF7, 0x45, 0xC5, 0xA6, /* 0xDC-0xDF */ + 0xF9, 0x43, 0xF9, 0x44, 0xC5, 0xD8, 0xA6, 0xDA, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xAA, 0xD7, 0xDB, 0x52, 0xBB, 0x4E, /* 0xE4-0xE7 */ + 0xC1, 0x7B, 0xED, 0xEF, 0xA6, 0xDB, 0x00, 0x00, /* 0xE8-0xEB */ + 0xAF, 0xE5, 0xAF, 0xE4, 0xDB, 0x53, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0xEA, 0xF4, 0xA6, 0xDC, /* 0xF0-0xF3 */ + 0xAD, 0x50, 0x00, 0x00, 0x00, 0x00, 0xDB, 0x54, /* 0xF4-0xF7 */ + 0xDB, 0x55, 0xDB, 0x56, 0xBB, 0x4F, 0xBF, 0xB2, /* 0xF8-0xFB */ + 0xA6, 0xDD, 0x00, 0x00, 0xAA, 0xD8, 0xD0, 0x68, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_82[512] = { + 0xAF, 0xE6, 0xD3, 0x70, 0xB2, 0xEA, 0x00, 0x00, /* 0x00-0x03 */ + 0xDB, 0x57, 0xB8, 0xA4, 0x00, 0x00, 0xBB, 0x50, /* 0x04-0x07 */ + 0xBF, 0xB3, 0xC1, 0x7C, 0xC2, 0xC2, 0xF4, 0xB5, /* 0x08-0x0B */ + 0xA6, 0xDE, 0xAA, 0xD9, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0xAF, 0xE7, 0xD7, 0x52, 0xB5, 0xCE, 0x00, 0x00, /* 0x10-0x13 */ + 0xBB, 0x51, 0xE3, 0xAB, 0xE7, 0x45, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA6, 0xDF, /* 0x18-0x1B */ + 0xB5, 0xCF, 0xDF, 0xA3, 0xBB, 0x52, 0xA6, 0xE0, /* 0x1C-0x1F */ + 0xCD, 0xB1, 0xD0, 0x69, 0xAD, 0x51, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0xD3, 0x72, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0xAF, 0xEA, 0x00, 0x00, 0xAF, 0xE8, 0xAF, 0xE9, /* 0x28-0x2B */ + 0xAF, 0xEB, 0x00, 0x00, 0x00, 0x00, 0xD3, 0x71, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0xD7, 0x57, 0xD7, 0x54, /* 0x30-0x33 */ + 0xD7, 0x56, 0xB2, 0xEB, 0xB2, 0xED, 0xB2, 0xEC, /* 0x34-0x37 */ + 0xD7, 0x53, 0xB2, 0xEE, 0xD7, 0x55, 0x00, 0x00, /* 0x38-0x3B */ + 0xDB, 0x58, 0xDB, 0x59, 0x00, 0x00, 0xDB, 0x5A, /* 0x3C-0x3F */ + 0xDF, 0xA6, 0x00, 0x00, 0xDF, 0xA7, 0x00, 0x00, /* 0x40-0x43 */ + 0xDF, 0xA5, 0xDF, 0xA8, 0x00, 0x00, 0xB8, 0xA5, /* 0x44-0x47 */ + 0x00, 0x00, 0xDF, 0xA4, 0x00, 0x00, 0xBB, 0x53, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0x4A, 0xE7, 0x46, /* 0x4C-0x4F */ + 0xE7, 0x49, 0xE7, 0x4B, 0xE7, 0x48, 0xE7, 0x47, /* 0x50-0x53 */ + 0x00, 0x00, 0xEA, 0xF5, 0xEA, 0xF6, 0xEA, 0xF7, /* 0x54-0x57 */ + 0xBF, 0xB4, 0xBF, 0xB5, 0xED, 0xF1, 0xED, 0xF0, /* 0x58-0x5B */ + 0xED, 0xF2, 0x00, 0x00, 0xF0, 0xA3, 0xF0, 0xA2, /* 0x5C-0x5F */ + 0x00, 0x00, 0xF2, 0xC4, 0x00, 0x00, 0xF2, 0xC5, /* 0x60-0x63 */ + 0xF2, 0xC3, 0x00, 0x00, 0xC4, 0xA5, 0x00, 0x00, /* 0x64-0x67 */ + 0xF4, 0xB6, 0xF4, 0xB7, 0x00, 0x00, 0xF7, 0x46, /* 0x68-0x6B */ + 0xF7, 0xEF, 0xF8, 0xBB, 0xA6, 0xE1, 0xA8, 0x7D, /* 0x6C-0x6F */ + 0x00, 0x00, 0xC1, 0x7D, 0xA6, 0xE2, 0x00, 0x00, /* 0x70-0x73 */ + 0xD7, 0x58, 0xDB, 0x5B, 0x00, 0x00, 0xC6, 0x41, /* 0x74-0x77 */ + 0xCA, 0x4A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0xCA, 0x4B, 0xCA, 0x4D, 0xA6, 0xE3, 0xCA, 0x4E, /* 0x7C-0x7F */ + + 0xCA, 0x4C, 0x00, 0x00, 0x00, 0x00, 0xCB, 0xA2, /* 0x80-0x83 */ + 0xCB, 0xA3, 0xCB, 0x7B, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0xCB, 0xA1, 0xA8, 0xA1, /* 0x88-0x8B */ + 0x00, 0x00, 0xA8, 0xA2, 0xCB, 0x7C, 0xCB, 0x7A, /* 0x8C-0x8F */ + 0xCB, 0x79, 0xCB, 0x7D, 0xA8, 0x7E, 0xCB, 0x7E, /* 0x90-0x93 */ + 0xD0, 0x6A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0xCD, 0xB6, 0xAA, 0xDC, 0xCD, 0xB5, 0xCD, 0xB7, /* 0x98-0x9B */ + 0x00, 0x00, 0xAA, 0xDB, 0xCD, 0xBC, 0xAA, 0xDF, /* 0x9C-0x9F */ + 0xCD, 0xB2, 0xCD, 0xC0, 0xCD, 0xC6, 0xAA, 0xE6, /* 0xA0-0xA3 */ + 0xCD, 0xC3, 0xAA, 0xE3, 0x00, 0x00, 0xCD, 0xB9, /* 0xA4-0xA7 */ + 0xCD, 0xBF, 0xCD, 0xC1, 0x00, 0x00, 0xCD, 0xB4, /* 0xA8-0xAB */ + 0xAA, 0xE2, 0xAA, 0xDD, 0xCD, 0xBA, 0xAA, 0xE4, /* 0xAC-0xAF */ + 0xAA, 0xE7, 0xAA, 0xE1, 0x00, 0x00, 0xAA, 0xDA, /* 0xB0-0xB3 */ + 0xCD, 0xBE, 0xCD, 0xB8, 0xCD, 0xC5, 0xAA, 0xE9, /* 0xB4-0xB7 */ + 0xAA, 0xE5, 0xAA, 0xE0, 0xCD, 0xBD, 0xAF, 0xEC, /* 0xB8-0xBB */ + 0xCD, 0xBB, 0xAA, 0xDE, 0xAA, 0xE8, 0x00, 0x00, /* 0xBC-0xBF */ + 0xCD, 0xB3, 0x00, 0x00, 0xCD, 0xC2, 0xCD, 0xC4, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0xAD, 0x62, 0xAD, 0x5C, 0xAD, 0x64, /* 0xD0-0xD3 */ + 0xAD, 0x61, 0xD0, 0x71, 0xD0, 0x74, 0xAD, 0x5D, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xD0, 0x6B, 0x00, 0x00, 0xAD, 0x56, /* 0xD8-0xDB */ + 0xAD, 0x60, 0x00, 0x00, 0xAD, 0x63, 0xAD, 0x65, /* 0xDC-0xDF */ + 0xD0, 0xA2, 0xD0, 0x77, 0x00, 0x00, 0xAD, 0x55, /* 0xE0-0xE3 */ + 0xD0, 0xA1, 0xAD, 0x59, 0xAD, 0x57, 0xAD, 0x52, /* 0xE4-0xE7 */ + 0xD0, 0x6F, 0x00, 0x00, 0xD0, 0x7E, 0xD0, 0x73, /* 0xE8-0xEB */ + 0xD0, 0x76, 0xD0, 0xA5, 0x00, 0x00, 0xAD, 0x66, /* 0xEC-0xEF */ + 0xD0, 0x7D, 0xAD, 0x5E, 0xD0, 0x78, 0xD0, 0xA4, /* 0xF0-0xF3 */ + 0xD0, 0x75, 0xD0, 0x79, 0xD0, 0x7C, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xD0, 0x6D, 0xD0, 0xA3, 0xD0, 0x7B, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0xD0, 0x6C, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_83[512] = { + 0xD0, 0x70, 0xAD, 0x5F, 0xAD, 0x5A, 0xAD, 0x53, /* 0x00-0x03 */ + 0xAD, 0x58, 0xAD, 0x54, 0xAD, 0x67, 0xD0, 0x6E, /* 0x04-0x07 */ + 0xD3, 0xA5, 0xAD, 0x5B, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0xD0, 0x7A, 0xCE, 0x41, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xD3, 0xA8, 0xAF, 0xFA, /* 0x14-0x17 */ + 0x00, 0x00, 0xD3, 0x76, 0x00, 0x00, 0xD3, 0xA3, /* 0x18-0x1B */ + 0xD3, 0x7D, 0x00, 0x00, 0xD3, 0xB2, 0x00, 0x00, /* 0x1C-0x1F */ + 0xD3, 0xAA, 0x00, 0x00, 0xD3, 0x7E, 0x00, 0x00, /* 0x20-0x23 */ + 0xD3, 0xA9, 0xD3, 0x78, 0xD3, 0x7C, 0xD3, 0xB5, /* 0x24-0x27 */ + 0xAF, 0xFD, 0xD3, 0xAD, 0xD3, 0xA4, 0xAF, 0xED, /* 0x28-0x2B */ + 0xD3, 0xB3, 0xD3, 0x74, 0x00, 0x00, 0xD3, 0xAC, /* 0x2C-0x2F */ + 0x00, 0x00, 0xAF, 0xFC, 0xAF, 0xF7, 0xD3, 0x73, /* 0x30-0x33 */ + 0xAF, 0xF5, 0xAF, 0xF4, 0xAF, 0xF9, 0xD3, 0xAB, /* 0x34-0x37 */ + 0xAF, 0xF1, 0xAF, 0xF8, 0xD0, 0x72, 0xDB, 0x5C, /* 0x38-0x3B */ + 0xD3, 0xA6, 0x00, 0x00, 0x00, 0x00, 0xD3, 0x7A, /* 0x3C-0x3F */ + 0xAF, 0xFB, 0xD3, 0x7B, 0xD3, 0xA1, 0xAF, 0xFE, /* 0x40-0x43 */ + 0xD3, 0x75, 0xD3, 0xAF, 0x00, 0x00, 0xD3, 0xAE, /* 0x44-0x47 */ + 0xD3, 0xB6, 0xAF, 0xF3, 0xAF, 0xF0, 0xD3, 0xB4, /* 0x48-0x4B */ + 0xD3, 0xB0, 0xD3, 0xA7, 0xD3, 0xA2, 0xAF, 0xF6, /* 0x4C-0x4F */ + 0xAF, 0xF2, 0xD3, 0x77, 0xAF, 0xEE, 0xD3, 0xB1, /* 0x50-0x53 */ + 0xAF, 0xEF, 0x00, 0x00, 0xD3, 0x79, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD7, 0x5E, /* 0x70-0x73 */ + 0xD7, 0x60, 0xD7, 0x65, 0xD7, 0x79, 0xB2, 0xFC, /* 0x74-0x77 */ + 0xB2, 0xF2, 0x00, 0x00, 0xD7, 0x5D, 0xB2, 0xFD, /* 0x78-0x7B */ + 0xB2, 0xFE, 0xD7, 0x68, 0xD7, 0x6F, 0xD7, 0x75, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xD7, 0x62, 0x00, 0x00, 0xD7, 0x69, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0xB3, 0x40, 0xD7, 0x77, /* 0x84-0x87 */ + 0xD7, 0x72, 0xB2, 0xFA, 0xB2, 0xF8, 0xD7, 0x6E, /* 0x88-0x8B */ + 0xD7, 0x6A, 0xD7, 0x5C, 0xB2, 0xEF, 0xD7, 0x61, /* 0x8C-0x8F */ + 0xD7, 0x59, 0x00, 0x00, 0xB2, 0xF7, 0xB2, 0xF9, /* 0x90-0x93 */ + 0xD7, 0x66, 0xD7, 0x63, 0xB2, 0xF4, 0xD7, 0x73, /* 0x94-0x97 */ + 0xB2, 0xF1, 0xD7, 0x64, 0xD7, 0x7A, 0xD7, 0x6C, /* 0x98-0x9B */ + 0x00, 0x00, 0xD7, 0x6B, 0xB2, 0xF0, 0x00, 0x00, /* 0x9C-0x9F */ + 0xB2, 0xFB, 0x00, 0x00, 0xB2, 0xF3, 0xD7, 0x5A, /* 0xA0-0xA3 */ + 0xD7, 0x5F, 0xD7, 0x70, 0xD7, 0x76, 0xB3, 0x41, /* 0xA4-0xA7 */ + 0xD7, 0x5B, 0xD7, 0x67, 0xD7, 0x6D, 0xB2, 0xF6, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0xD7, 0x78, 0xD7, 0x71, /* 0xAC-0xAF */ + 0xD7, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0xB2, 0xF5, 0x00, 0x00, 0xDB, 0x6C, /* 0xBC-0xBF */ + 0xDB, 0x60, 0xB5, 0xD7, 0xDB, 0x7D, 0xDB, 0xA7, /* 0xC0-0xC3 */ + 0xDB, 0xAA, 0xB5, 0xD5, 0xDB, 0x68, 0xDB, 0xA3, /* 0xC4-0xC7 */ + 0xDB, 0x69, 0xDB, 0x77, 0xB5, 0xE2, 0xDB, 0x73, /* 0xC8-0xCB */ + 0xB5, 0xDF, 0x00, 0x00, 0xDB, 0x74, 0xDB, 0x5D, /* 0xCC-0xCF */ + 0x00, 0x00, 0xDB, 0xA4, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xB5, 0xE8, 0xDB, 0xA1, 0xDB, 0x75, 0xDB, 0xAC, /* 0xD4-0xD7 */ + 0xDB, 0x70, 0xDF, 0xC8, 0x00, 0x00, 0xDB, 0xAF, /* 0xD8-0xDB */ + 0xB5, 0xE6, 0xDB, 0x6E, 0xDB, 0x7A, 0xB5, 0xE9, /* 0xDC-0xDF */ + 0xB5, 0xD4, 0xDB, 0x72, 0xDB, 0xAD, 0xDB, 0x6B, /* 0xE0-0xE3 */ + 0xDB, 0x64, 0xDB, 0x6F, 0x00, 0x00, 0xDB, 0x63, /* 0xE4-0xE7 */ + 0xDB, 0x61, 0xB5, 0xD0, 0xDB, 0xA5, 0xDB, 0x6A, /* 0xE8-0xEB */ + 0xDB, 0xA8, 0x00, 0x00, 0xDB, 0xA9, 0xB5, 0xD8, /* 0xEC-0xEF */ + 0xB5, 0xDD, 0xB5, 0xD9, 0xB5, 0xE1, 0xDB, 0x7E, /* 0xF0-0xF3 */ + 0xB5, 0xDA, 0xDB, 0x76, 0xDB, 0x66, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xB5, 0xD2, 0xDB, 0x5E, 0xDB, 0xA2, 0xDB, 0xAB, /* 0xF8-0xFB */ + 0xDB, 0x65, 0xB5, 0xE0, 0xDB, 0xB0, 0xDB, 0x71, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_84[512] = { + 0x00, 0x00, 0xDB, 0x6D, 0x00, 0x00, 0xB5, 0xD1, /* 0x00-0x03 */ + 0xB5, 0xE5, 0x00, 0x00, 0xDB, 0x7C, 0xB5, 0xE7, /* 0x04-0x07 */ + 0x00, 0x00, 0xDB, 0x78, 0xB5, 0xDC, 0xB5, 0xD6, /* 0x08-0x0B */ + 0xB5, 0xDE, 0xB5, 0xD3, 0xB5, 0xE4, 0xDB, 0x79, /* 0x0C-0x0F */ + 0xDB, 0x67, 0xDB, 0x7B, 0xDB, 0x62, 0xDB, 0xA6, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDB, 0xAE, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDB, 0x5F, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0xDF, 0xC7, 0x00, 0x00, 0xDF, 0xDD, /* 0x28-0x2B */ + 0xB8, 0x55, 0xDF, 0xCC, 0x00, 0x00, 0xDF, 0xCA, /* 0x2C-0x2F */ + 0xDF, 0xB5, 0xB8, 0xA9, 0xDF, 0xC5, 0xDF, 0xD9, /* 0x30-0x33 */ + 0xDF, 0xC1, 0xB8, 0xB1, 0xDF, 0xD8, 0xDF, 0xBF, /* 0x34-0x37 */ + 0xB5, 0xE3, 0xDF, 0xCF, 0xDF, 0xC0, 0xDF, 0xD6, /* 0x38-0x3B */ + 0xB8, 0xB0, 0xB8, 0xA8, 0x00, 0x00, 0xDF, 0xAA, /* 0x3C-0x3F */ + 0xDF, 0xB2, 0x00, 0x00, 0xDF, 0xCB, 0xDF, 0xC3, /* 0x40-0x43 */ + 0xDF, 0xDC, 0xDF, 0xC6, 0xB8, 0xB6, 0xDF, 0xD7, /* 0x44-0x47 */ + 0x00, 0x00, 0xB8, 0xAD, 0x00, 0x00, 0xDF, 0xC9, /* 0x48-0x4B */ + 0xDF, 0xD1, 0xDF, 0xB6, 0xDF, 0xD0, 0x00, 0x00, /* 0x4C-0x4F */ + 0xDF, 0xE1, 0xDF, 0xB1, 0xDF, 0xD2, 0x00, 0x00, /* 0x50-0x53 */ + 0xDF, 0xDF, 0x00, 0x00, 0xDF, 0xAB, 0xB5, 0xDB, /* 0x54-0x57 */ + 0x00, 0x00, 0xDF, 0xB9, 0xDF, 0xB8, 0xB8, 0xAF, /* 0x58-0x5B */ + 0x00, 0x00, 0xDF, 0xBC, 0xDF, 0xBE, 0xDF, 0xCD, /* 0x5C-0x5F */ + 0xDF, 0xDE, 0xB8, 0xB2, 0x00, 0x00, 0xB8, 0xB3, /* 0x60-0x63 */ + 0x00, 0x00, 0xDF, 0xB0, 0xB8, 0xAB, 0xDF, 0xB4, /* 0x64-0x67 */ + 0xDF, 0xDA, 0xB8, 0xB4, 0x00, 0x00, 0xB8, 0xAC, /* 0x68-0x6B */ + 0xB8, 0xAE, 0xB8, 0xB5, 0xDF, 0xE0, 0xDF, 0xD3, /* 0x6C-0x6F */ + 0xDF, 0xCE, 0x00, 0x00, 0x00, 0x00, 0xDF, 0xBB, /* 0x70-0x73 */ + 0xDF, 0xBA, 0xB8, 0xAA, 0xDF, 0xAC, 0xB8, 0xA7, /* 0x74-0x77 */ + 0xDF, 0xC4, 0xDF, 0xAD, 0xDF, 0xC2, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0xDF, 0xB7, 0xDF, 0xDB, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0xB8, 0xA6, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0xDF, 0xB3, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0xDF, 0xAF, 0xDF, 0xD5, 0xDF, 0xAE, /* 0x8C-0x8F */ + 0xBB, 0x60, 0xE3, 0xD3, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0xE3, 0xC2, 0x00, 0x00, 0x00, 0x00, 0xE3, 0xAC, /* 0x94-0x97 */ + 0xE3, 0xCA, 0xBB, 0x58, 0xE3, 0xBB, 0xE3, 0xC5, /* 0x98-0x9B */ + 0xBB, 0x5B, 0xE3, 0xBE, 0xBB, 0x59, 0xE3, 0xAF, /* 0x9C-0x9F */ + 0xE3, 0xCD, 0xE3, 0xAE, 0xE3, 0xC1, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xE3, 0xAD, 0x00, 0x00, 0x00, 0x00, 0xE3, 0xBF, /* 0xA4-0xA7 */ + 0xE3, 0xC8, 0xE3, 0xC6, 0xE3, 0xBA, 0xE3, 0xB5, /* 0xA8-0xAB */ + 0xE3, 0xB3, 0x00, 0x00, 0xE3, 0xB4, 0xE3, 0xC7, /* 0xAC-0xAF */ + 0xE3, 0xD2, 0xE3, 0xBC, 0xBB, 0x5A, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xE3, 0xB7, 0x00, 0x00, 0xE3, 0xCB, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xBB, 0x5D, 0xE3, 0xB6, 0xE3, 0xB0, 0xE3, 0xC0, /* 0xB8-0xBB */ + 0xBB, 0x61, 0x00, 0x00, 0x00, 0x00, 0xBB, 0x55, /* 0xBC-0xBF */ + 0xBB, 0x5E, 0xE3, 0xB8, 0xE3, 0xB2, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xBB, 0x57, 0xDF, 0xD4, 0xBB, 0x56, 0xE3, 0xC3, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xBB, 0x54, 0xBB, 0x63, 0xBB, 0x5C, /* 0xC8-0xCB */ + 0xE3, 0xC4, 0xE3, 0xB9, 0xE3, 0xB1, 0xE3, 0xCC, /* 0xCC-0xCF */ + 0xE3, 0xBD, 0xBB, 0x62, 0xE3, 0xD0, 0xBB, 0x5F, /* 0xD0-0xD3 */ + 0xE3, 0xCF, 0x00, 0x00, 0xE3, 0xC9, 0xE3, 0xCE, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE3, 0xD1, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0x73, /* 0xE4-0xE7 */ + 0xE7, 0x74, 0xE7, 0x67, 0xE7, 0x66, 0xE7, 0x62, /* 0xE8-0xEB */ + 0xBD, 0xB4, 0x00, 0x00, 0xBD, 0xAC, 0xE7, 0x76, /* 0xEC-0xEF */ + 0xE7, 0x75, 0xDF, 0xA9, 0xE7, 0x5F, 0xE7, 0x63, /* 0xF0-0xF3 */ + 0xE7, 0x5D, 0x00, 0x00, 0xE7, 0x70, 0xE7, 0x61, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xE7, 0x77, 0xE7, 0x5A, 0xE7, 0x58, /* 0xF8-0xFB */ + 0xE7, 0x64, 0xE7, 0x6E, 0xE7, 0x69, 0xBD, 0xB6, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_85[512] = { + 0xE7, 0x4F, 0x00, 0x00, 0xE7, 0x6D, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0xBD, 0xB7, 0xDF, 0xBD, /* 0x04-0x07 */ + 0xE7, 0x5B, 0xE7, 0x52, 0xE7, 0x55, 0xE7, 0x7B, /* 0x08-0x0B */ + 0xE7, 0x5C, 0xE7, 0x53, 0xE7, 0x51, 0xE7, 0x4E, /* 0x0C-0x0F */ + 0x00, 0x00, 0xBD, 0xB0, 0xE7, 0x65, 0xBD, 0xAF, /* 0x10-0x13 */ + 0xBD, 0xB3, 0xE7, 0x60, 0xE7, 0x68, 0xBD, 0xA9, /* 0x14-0x17 */ + 0xE7, 0x78, 0xE7, 0x7C, 0xBD, 0xAB, 0x00, 0x00, /* 0x18-0x1B */ + 0xE7, 0x57, 0xE7, 0x6B, 0xE7, 0x6F, 0xE7, 0x54, /* 0x1C-0x1F */ + 0xE7, 0x79, 0xBD, 0xB2, 0x00, 0x00, 0xBD, 0xB1, /* 0x20-0x23 */ + 0xE7, 0x4C, 0xBD, 0xB5, 0xE7, 0x72, 0xE7, 0x56, /* 0x24-0x27 */ + 0xE7, 0x6A, 0xE7, 0x50, 0xE7, 0x5E, 0xE7, 0x59, /* 0x28-0x2B */ + 0xBD, 0xAD, 0xBD, 0xAE, 0xE7, 0x6C, 0xE7, 0x7D, /* 0x2C-0x2F */ + 0xE7, 0x7A, 0xE7, 0x71, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0x4D, /* 0x38-0x3B */ + 0x00, 0x00, 0xBD, 0xAA, 0xEB, 0x49, 0x00, 0x00, /* 0x3C-0x3F */ + 0xEB, 0x40, 0xEB, 0x43, 0x00, 0x00, 0xBF, 0xBB, /* 0x40-0x43 */ + 0xEB, 0x45, 0xEA, 0xF9, 0xEB, 0x41, 0xEB, 0x47, /* 0x44-0x47 */ + 0xBF, 0xB8, 0xBF, 0xBC, 0xBF, 0xB6, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0xEA, 0xFB, 0xEB, 0x4C, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0xEB, 0x46, 0x00, 0x00, 0xEA, 0xFC, /* 0x50-0x53 */ + 0xEB, 0x55, 0xEB, 0x4F, 0xEA, 0xF8, 0xEE, 0x46, /* 0x54-0x57 */ + 0xEA, 0xFE, 0xBF, 0xB7, 0x00, 0x00, 0xEB, 0x4A, /* 0x58-0x5B */ + 0x00, 0x00, 0xEB, 0x54, 0xBF, 0xBF, 0x00, 0x00, /* 0x5C-0x5F */ + 0xEB, 0x51, 0xEA, 0xFD, 0xEB, 0x44, 0xEB, 0x48, /* 0x60-0x63 */ + 0xEB, 0x42, 0xEB, 0x56, 0xEB, 0x53, 0xEB, 0x50, /* 0x64-0x67 */ + 0xBF, 0xB9, 0xBF, 0xBA, 0xBF, 0xBE, 0xEA, 0xFA, /* 0x68-0x6B */ + 0xEB, 0x57, 0xBF, 0xBD, 0xEB, 0x4D, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0xEB, 0x4B, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0xEB, 0x4E, 0xEE, 0x53, 0xEE, 0x40, /* 0x74-0x77 */ + 0xEE, 0x45, 0xEE, 0x52, 0xEE, 0x44, 0xED, 0xFB, /* 0x78-0x7B */ + 0xEE, 0x41, 0x00, 0x00, 0xC1, 0xA2, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xED, 0xF4, 0xEE, 0x4D, 0xEE, 0x4F, 0xED, 0xF3, /* 0x80-0x83 */ + 0xC1, 0xA1, 0xEE, 0x51, 0xEE, 0x49, 0xC1, 0xA8, /* 0x84-0x87 */ + 0xEE, 0x50, 0xEE, 0x42, 0xC1, 0xAA, 0xED, 0xF9, /* 0x88-0x8B */ + 0xEB, 0x52, 0xEE, 0x4A, 0xEE, 0x47, 0xED, 0xF5, /* 0x8C-0x8F */ + 0xEE, 0x55, 0xC1, 0xA4, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0xC1, 0xA5, 0xED, 0xF7, 0xEE, 0x48, 0x00, 0x00, /* 0x94-0x97 */ + 0xEE, 0x54, 0xEE, 0x4B, 0xED, 0xFD, 0xC1, 0xA7, /* 0x98-0x9B */ + 0xC1, 0xA3, 0xEE, 0x4C, 0xED, 0xFE, 0xEE, 0x56, /* 0x9C-0x9F */ + 0xED, 0xF8, 0xEE, 0x43, 0xEE, 0x4E, 0xED, 0xFA, /* 0xA0-0xA3 */ + 0xED, 0xFC, 0x00, 0x00, 0xC2, 0xCB, 0xED, 0xF6, /* 0xA4-0xA7 */ + 0xC1, 0xA9, 0xC2, 0xC4, 0xC1, 0x7E, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC1, 0xA6, /* 0xAC-0xAF */ + 0xC2, 0xC8, 0xF0, 0xB3, 0x00, 0x00, 0xF0, 0xA9, /* 0xB0-0xB3 */ + 0xF0, 0xA4, 0xF0, 0xAA, 0xF0, 0xB4, 0xF0, 0xB8, /* 0xB4-0xB7 */ + 0xF0, 0xB7, 0xC2, 0xCA, 0xC2, 0xC9, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0xF0, 0xAB, 0xF0, 0xB9, 0xF0, 0xAE, /* 0xBC-0xBF */ + 0xF0, 0xA6, 0x00, 0x00, 0xF0, 0xA8, 0xF0, 0xA7, /* 0xC0-0xC3 */ + 0xF0, 0xAD, 0xF0, 0xB2, 0xF0, 0xA5, 0xF0, 0xAC, /* 0xC4-0xC7 */ + 0xF0, 0xB1, 0xC2, 0xC7, 0x00, 0x00, 0xF0, 0xAF, /* 0xC8-0xCB */ + 0x00, 0x00, 0xC2, 0xC5, 0xF0, 0xB0, 0xC2, 0xC3, /* 0xCC-0xCF */ + 0xC2, 0xC6, 0xF2, 0xD5, 0xF0, 0xB5, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xC3, 0xC2, 0x00, 0x00, 0xF2, 0xCD, /* 0xD4-0xD7 */ + 0xF2, 0xD1, 0xF2, 0xC9, 0xF2, 0xCC, 0x00, 0x00, /* 0xD8-0xDB */ + 0xF2, 0xD4, 0xC3, 0xC0, 0xF2, 0xD9, 0xF2, 0xD2, /* 0xDC-0xDF */ + 0x00, 0x00, 0xF2, 0xCA, 0xF2, 0xDA, 0xF2, 0xD3, /* 0xE0-0xE3 */ + 0xC3, 0xC3, 0xC3, 0xC4, 0xF2, 0xD7, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xF2, 0xCB, 0xC3, 0xBF, 0xC3, 0xC1, 0xF2, 0xC6, /* 0xE8-0xEB */ + 0xF2, 0xCE, 0xF2, 0xC8, 0x00, 0x00, 0xF2, 0xD8, /* 0xEC-0xEF */ + 0xF2, 0xD6, 0xF2, 0xC7, 0xF2, 0xCF, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0xF4, 0xBE, 0xC3, 0xC5, /* 0xF4-0xF7 */ + 0xF2, 0xD0, 0xC4, 0xA7, 0xC4, 0xA9, 0xC4, 0xA6, /* 0xF8-0xFB */ + 0x00, 0x00, 0xF4, 0xC3, 0xF4, 0xBB, 0xF4, 0xB9, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_86[512] = { + 0xF4, 0xBD, 0xF4, 0xBA, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0xF4, 0xBF, 0xF4, 0xC1, 0xC4, 0xAA, 0xC4, 0xAC, /* 0x04-0x07 */ + 0x00, 0x00, 0xF4, 0xC0, 0xC4, 0xAD, 0xC4, 0xAB, /* 0x08-0x0B */ + 0xF4, 0xC2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0xC4, 0xA8, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC4, 0xF4, /* 0x14-0x17 */ + 0xF5, 0xF1, 0xF5, 0xF7, 0xC4, 0xF6, 0xF4, 0xBC, /* 0x18-0x1B */ + 0xF5, 0xF6, 0x00, 0x00, 0xF5, 0xFD, 0xF5, 0xF4, /* 0x1C-0x1F */ + 0xF5, 0xFB, 0xF5, 0xFA, 0xF4, 0xB8, 0xF5, 0xF5, /* 0x20-0x23 */ + 0xF0, 0xB6, 0xF5, 0xFE, 0xF5, 0xF3, 0xF5, 0xF8, /* 0x24-0x27 */ + 0x00, 0x00, 0xF5, 0xFC, 0xF5, 0xF2, 0x00, 0x00, /* 0x28-0x2B */ + 0xF7, 0x4A, 0xC4, 0xF5, 0xF5, 0xF9, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0xF7, 0xF4, 0xF7, 0x4B, 0xF7, 0x49, /* 0x30-0x33 */ + 0xF7, 0x47, 0xF7, 0x48, 0xF7, 0x4C, 0x00, 0x00, /* 0x34-0x37 */ + 0xC5, 0xD9, 0xF7, 0xF2, 0xF7, 0xF0, 0xF7, 0xF5, /* 0x38-0x3B */ + 0xF7, 0xF3, 0x00, 0x00, 0xF7, 0xF6, 0xC5, 0xDA, /* 0x3C-0x3F */ + 0xF7, 0xF1, 0x00, 0x00, 0x00, 0x00, 0xF8, 0xBC, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xF9, 0x45, 0xF9, 0x46, /* 0x44-0x47 */ + 0xF9, 0x47, 0x00, 0x00, 0x00, 0x00, 0xF9, 0xC7, /* 0x48-0x4B */ + 0xF9, 0xBD, 0xCA, 0x4F, 0xAA, 0xEA, 0x00, 0x00, /* 0x4C-0x4F */ + 0xAD, 0x68, 0x00, 0x00, 0xD3, 0xB8, 0xD3, 0xB7, /* 0x50-0x53 */ + 0xB0, 0x40, 0xB3, 0x42, 0xD7, 0x7C, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0xD7, 0x7B, 0x00, 0x00, 0xB5, 0xEA, /* 0x58-0x5B */ + 0xB8, 0xB8, 0x00, 0x00, 0xB8, 0xB7, 0xB8, 0xB9, /* 0x5C-0x5F */ + 0x00, 0x00, 0xE3, 0xD4, 0xE7, 0x7E, 0xEB, 0x58, /* 0x60-0x63 */ + 0xEB, 0x5A, 0xEB, 0x59, 0x00, 0x00, 0xC1, 0xAB, /* 0x64-0x67 */ + 0xEE, 0x57, 0xF0, 0xBA, 0xF9, 0xA5, 0xA6, 0xE4, /* 0x68-0x6B */ + 0x00, 0x00, 0xCD, 0xC9, 0xCD, 0xCA, 0xCD, 0xC8, /* 0x6C-0x6F */ + 0xCD, 0xC7, 0xAA, 0xEB, 0x00, 0x00, 0xD0, 0xA9, /* 0x70-0x73 */ + 0xD0, 0xA7, 0x00, 0x00, 0x00, 0x00, 0xD0, 0xA6, /* 0x74-0x77 */ + 0x00, 0x00, 0xAD, 0x69, 0xAD, 0x6B, 0xAD, 0x6A, /* 0x78-0x7B */ + 0xD0, 0xA8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0xD3, 0xC4, 0xD3, 0xC1, 0xD3, 0xBF, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0xB0, 0x41, 0xD3, 0xC2, /* 0x88-0x8B */ + 0xB0, 0x46, 0xD3, 0xBC, 0xD3, 0xCB, 0x00, 0x00, /* 0x8C-0x8F */ + 0xD3, 0xCD, 0xD3, 0xBD, 0x00, 0x00, 0xB0, 0x43, /* 0x90-0x93 */ + 0xD3, 0xCE, 0xD3, 0xC9, 0xD3, 0xBB, 0xD3, 0xC0, /* 0x94-0x97 */ + 0xD3, 0xCA, 0xD3, 0xC6, 0xD3, 0xC3, 0x00, 0x00, /* 0x98-0x9B */ + 0xB0, 0x48, 0xD3, 0xCC, 0xD3, 0xBE, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0xD3, 0xC7, 0xD3, 0xB9, 0xB0, 0x47, /* 0xA0-0xA3 */ + 0xB0, 0x44, 0xD3, 0xC5, 0x00, 0x00, 0xD3, 0xC8, /* 0xA4-0xA7 */ + 0xD3, 0xBA, 0xB0, 0x45, 0xB0, 0x42, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB3, 0x4C, /* 0xAC-0xAF */ + 0xD7, 0xA5, 0xB3, 0x4B, 0x00, 0x00, 0xD7, 0xA8, /* 0xB0-0xB3 */ + 0xD7, 0xAB, 0xB3, 0x48, 0xB3, 0x46, 0xD7, 0x7E, /* 0xB4-0xB7 */ + 0xD7, 0xA9, 0xD7, 0xA7, 0xD7, 0xA4, 0xD7, 0xAC, /* 0xB8-0xBB */ + 0xD7, 0xAD, 0xD7, 0xAF, 0xD7, 0xB0, 0xD7, 0x7D, /* 0xBC-0xBF */ + 0xB3, 0x45, 0xD7, 0xA2, 0xD7, 0xA1, 0xD7, 0xAE, /* 0xC0-0xC3 */ + 0xB3, 0x47, 0xD7, 0xA3, 0xB3, 0x49, 0xB3, 0x44, /* 0xC4-0xC7 */ + 0xD7, 0xA6, 0xB3, 0x4D, 0x00, 0x00, 0xB3, 0x4A, /* 0xC8-0xCB */ + 0xD7, 0xAA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0xB5, 0xF1, 0xDB, 0xBF, 0x00, 0x00, 0xDB, 0xB4, /* 0xD0-0xD3 */ + 0xB5, 0xEE, 0x00, 0x00, 0xDF, 0xE7, 0xDB, 0xBD, /* 0xD4-0xD7 */ + 0xDB, 0xB1, 0xB5, 0xEC, 0xDB, 0xB6, 0xB5, 0xEF, /* 0xD8-0xDB */ + 0xDB, 0xBA, 0xDB, 0xB8, 0xB5, 0xF2, 0xB5, 0xEB, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0xDB, 0xB2, 0xDB, 0xB5, /* 0xE0-0xE3 */ + 0xB5, 0xF0, 0x00, 0x00, 0xDB, 0xB3, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xDB, 0xBE, 0xDB, 0xBC, 0xDB, 0xB7, 0xDB, 0xB9, /* 0xE8-0xEB */ + 0xDB, 0xBB, 0xB5, 0xED, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0xDF, 0xE8, 0xDF, 0xEE, 0xDF, 0xE4, /* 0xF4-0xF7 */ + 0xDF, 0xEA, 0xB8, 0xBA, 0xDF, 0xE6, 0xB8, 0xC0, /* 0xF8-0xFB */ + 0x00, 0x00, 0x00, 0x00, 0xB8, 0xBF, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_87[512] = { + 0xB8, 0xBE, 0xDF, 0xED, 0xB8, 0xC1, 0xB8, 0xC2, /* 0x00-0x03 */ + 0xDF, 0xE3, 0xDF, 0xF0, 0xB8, 0xC3, 0xB8, 0xBD, /* 0x04-0x07 */ + 0xB8, 0xBC, 0xDF, 0xEC, 0xB8, 0xC4, 0xDF, 0xE2, /* 0x08-0x0B */ + 0xDF, 0xE5, 0xDF, 0xEF, 0xDF, 0xEB, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0xE3, 0xF4, 0xE3, 0xE9, 0xB8, 0xBB, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0xBB, 0x6A, 0xE3, 0xDD, 0xE3, 0xF2, 0xE3, 0xDE, /* 0x18-0x1B */ + 0xBB, 0x65, 0x00, 0x00, 0xE3, 0xDB, 0x00, 0x00, /* 0x1C-0x1F */ + 0xE3, 0xE4, 0xE3, 0xDC, 0xBB, 0x67, 0xE3, 0xD6, /* 0x20-0x23 */ + 0xE3, 0xF1, 0xBB, 0x68, 0xE3, 0xEE, 0xE3, 0xEF, /* 0x24-0x27 */ + 0xE3, 0xD7, 0xBB, 0x6D, 0xE3, 0xE6, 0x00, 0x00, /* 0x28-0x2B */ + 0xE3, 0xE0, 0xE3, 0xE7, 0xE3, 0xDA, 0x00, 0x00, /* 0x2C-0x2F */ + 0xE3, 0xF3, 0xE3, 0xEB, 0xE3, 0xE5, 0xE3, 0xD5, /* 0x30-0x33 */ + 0xBB, 0x69, 0xE3, 0xEC, 0x00, 0x00, 0xBB, 0x6C, /* 0x34-0x37 */ + 0xE3, 0xF0, 0x00, 0x00, 0xE3, 0xEA, 0xBB, 0x66, /* 0x38-0x3B */ + 0xE3, 0xE8, 0x00, 0x00, 0xE3, 0xE2, 0xBB, 0x64, /* 0x3C-0x3F */ + 0xE3, 0xD9, 0xE3, 0xE1, 0xE3, 0xED, 0xE3, 0xDF, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xE3, 0xE3, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0xBD, 0xC1, 0xDF, 0xE9, 0xE7, 0xB2, 0xE7, 0xBB, /* 0x4C-0x4F */ + 0xE7, 0xB1, 0xE7, 0xAD, 0xE7, 0xAA, 0xBD, 0xC2, /* 0x50-0x53 */ + 0xE7, 0xA8, 0xBB, 0x6B, 0xE7, 0xA1, 0xBD, 0xC0, /* 0x54-0x57 */ + 0xE7, 0xA7, 0xBD, 0xBF, 0xE7, 0xAC, 0xE7, 0xA9, /* 0x58-0x5B */ + 0xE7, 0xB9, 0xE7, 0xB4, 0xE7, 0xAE, 0xE7, 0xB3, /* 0x5C-0x5F */ + 0xBD, 0xBB, 0xE7, 0xAB, 0xE7, 0xBE, 0xE7, 0xA2, /* 0x60-0x63 */ + 0xE7, 0xA3, 0xE7, 0xBA, 0xBD, 0xBC, 0xE7, 0xBF, /* 0x64-0x67 */ + 0xBD, 0xBE, 0xE7, 0xC0, 0xE7, 0xB0, 0xE3, 0xD8, /* 0x68-0x6B */ + 0xE7, 0xB6, 0xE7, 0xAF, 0xE7, 0xB8, 0xE7, 0xB5, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE7, 0xA6, /* 0x70-0x73 */ + 0xBD, 0xB9, 0xE7, 0xBD, 0xBD, 0xBA, 0xE7, 0xA4, /* 0x74-0x77 */ + 0xBD, 0xBD, 0xEB, 0x64, 0xE7, 0xB7, 0xE7, 0xBC, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xEB, 0x61, 0xBD, 0xB8, 0xBF, 0xC0, /* 0x80-0x83 */ + 0xEB, 0x6B, 0xEB, 0x67, 0x00, 0x00, 0xEB, 0x65, /* 0x84-0x87 */ + 0xEB, 0x60, 0xEB, 0x6F, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0xBF, 0xC4, 0x00, 0x00, 0xEB, 0x5C, /* 0x8C-0x8F */ + 0xEB, 0x68, 0xEB, 0x69, 0xEB, 0x5F, 0xEB, 0x5E, /* 0x90-0x93 */ + 0xEB, 0x6C, 0x00, 0x00, 0xEB, 0x62, 0xEB, 0x5D, /* 0x94-0x97 */ + 0xEB, 0x63, 0x00, 0x00, 0xEB, 0x6E, 0xEB, 0x5B, /* 0x98-0x9B */ + 0xEB, 0x6D, 0xEB, 0x6A, 0xBF, 0xC2, 0xBF, 0xC1, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0xBF, 0xC3, 0xEB, 0x66, /* 0xA0-0xA3 */ + 0xF0, 0xCB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0xEE, 0x59, 0xC1, 0xB1, /* 0xA8-0xAB */ + 0xEE, 0x5D, 0xEE, 0x5A, 0xEE, 0x61, 0xEE, 0x67, /* 0xAC-0xAF */ + 0xEE, 0x5C, 0x00, 0x00, 0xEE, 0x70, 0xC1, 0xAE, /* 0xB0-0xB3 */ + 0xEE, 0x6A, 0xEE, 0x5F, 0xEE, 0x6B, 0xEE, 0x66, /* 0xB4-0xB7 */ + 0xEE, 0x6D, 0xEE, 0x5E, 0xC1, 0xB3, 0xC1, 0xB2, /* 0xB8-0xBB */ + 0xEE, 0x60, 0xEE, 0x6E, 0xEE, 0x58, 0xEE, 0x6C, /* 0xBC-0xBF */ + 0xC1, 0xAC, 0x00, 0x00, 0xEE, 0x64, 0xEE, 0x63, /* 0xC0-0xC3 */ + 0xEE, 0x68, 0xEE, 0x5B, 0xC1, 0xB0, 0x00, 0x00, /* 0xC4-0xC7 */ + 0xC1, 0xB4, 0xEE, 0x62, 0xEE, 0x69, 0xC1, 0xB5, /* 0xC8-0xCB */ + 0xEE, 0x65, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0xC1, 0xAD, 0xC1, 0xAF, 0xF0, 0xC7, /* 0xD0-0xD3 */ + 0xF0, 0xC5, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xCC, /* 0xD4-0xD7 */ + 0xF0, 0xC9, 0xF0, 0xCD, 0x00, 0x00, 0xF0, 0xBE, /* 0xD8-0xDB */ + 0xF0, 0xC6, 0xF0, 0xD1, 0xEE, 0x6F, 0xF0, 0xC2, /* 0xDC-0xDF */ + 0xC2, 0xCF, 0xE7, 0xA5, 0xF0, 0xBD, 0xF0, 0xCA, /* 0xE0-0xE3 */ + 0xF0, 0xC4, 0xF0, 0xC1, 0xF0, 0xBC, 0xF0, 0xBB, /* 0xE4-0xE7 */ + 0xF0, 0xD0, 0x00, 0x00, 0xF0, 0xC0, 0xF0, 0xBF, /* 0xE8-0xEB */ + 0xC2, 0xCD, 0xF0, 0xC8, 0x00, 0x00, 0xC2, 0xCC, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0xC2, 0xCE, 0xF0, 0xC3, /* 0xF0-0xF3 */ + 0xF0, 0xCF, 0x00, 0x00, 0xF2, 0xDE, 0xF2, 0xDF, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xC3, 0xC9, 0xF2, 0xDC, 0xC3, 0xC6, /* 0xF8-0xFB */ + 0xF2, 0xE4, 0x00, 0x00, 0xC3, 0xCA, 0xF2, 0xE6, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_88[512] = { + 0xF2, 0xDB, 0xF0, 0xCE, 0xF2, 0xE8, 0xF2, 0xDD, /* 0x00-0x03 */ + 0x00, 0x00, 0xC3, 0xC7, 0xF2, 0xE3, 0x00, 0x00, /* 0x04-0x07 */ + 0xF2, 0xE5, 0xF2, 0xE0, 0xF2, 0xE7, 0xF2, 0xE2, /* 0x08-0x0B */ + 0xF2, 0xE1, 0xC3, 0xC8, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0xF4, 0xC5, 0xF4, 0xC6, 0x00, 0x00, 0xF4, 0xC8, /* 0x10-0x13 */ + 0xC4, 0xAE, 0xC4, 0xAF, 0xF4, 0xC9, 0xF4, 0xC7, /* 0x14-0x17 */ + 0x00, 0x00, 0xF4, 0xC4, 0x00, 0x00, 0xF6, 0x42, /* 0x18-0x1B */ + 0xF6, 0x45, 0xF6, 0x41, 0x00, 0x00, 0xC4, 0xFA, /* 0x1C-0x1F */ + 0xF6, 0x43, 0xC4, 0xF9, 0xC4, 0xF8, 0xC4, 0xF7, /* 0x20-0x23 */ + 0xF6, 0x44, 0xF7, 0x51, 0xF7, 0x4F, 0x00, 0x00, /* 0x24-0x27 */ + 0xF7, 0x4E, 0xF6, 0x40, 0xF7, 0x50, 0xF6, 0x46, /* 0x28-0x2B */ + 0xF7, 0x4D, 0x00, 0x00, 0xF7, 0xF9, 0xF7, 0xD7, /* 0x2C-0x2F */ + 0xF7, 0xF7, 0xC5, 0xDB, 0xF7, 0xF8, 0xF7, 0xFA, /* 0x30-0x33 */ + 0x00, 0x00, 0xF8, 0xBF, 0xC5, 0xFA, 0xF8, 0xBE, /* 0x34-0x37 */ + 0xF8, 0xBD, 0xC5, 0xFB, 0x00, 0x00, 0xC6, 0x5A, /* 0x38-0x3B */ + 0xF9, 0x6E, 0xF9, 0xA7, 0xF9, 0xA6, 0xF9, 0xA8, /* 0x3C-0x3F */ + 0xA6, 0xE5, 0xD0, 0xAA, 0x00, 0x00, 0xD3, 0xCF, /* 0x40-0x43 */ + 0xD3, 0xD0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0xDB, 0xC0, 0x00, 0x00, 0xF6, 0x47, 0xF8, 0xC0, /* 0x48-0x4B */ + 0xA6, 0xE6, 0xAD, 0x6C, 0xD0, 0xAB, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0xD7, 0xB1, 0xB3, 0x4E, /* 0x50-0x53 */ + 0x00, 0x00, 0xDB, 0xC2, 0xDB, 0xC1, 0xB5, 0xF3, /* 0x54-0x57 */ + 0x00, 0x00, 0xB8, 0xC5, 0xE7, 0xC1, 0xBD, 0xC3, /* 0x58-0x5B */ + 0x00, 0x00, 0xBD, 0xC4, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0xBF, 0xC5, 0xC5, 0xFC, 0xA6, 0xE7, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD0, 0xAC, /* 0x64-0x67 */ + 0xAA, 0xED, 0xD0, 0xAE, 0xD0, 0xAD, 0xAD, 0x6D, /* 0x68-0x6B */ + 0x00, 0x00, 0xD3, 0xD1, 0x00, 0x00, 0xD3, 0xD8, /* 0x6C-0x6F */ + 0xB0, 0x49, 0xD3, 0xD6, 0xD3, 0xD4, 0x00, 0x00, /* 0x70-0x73 */ + 0xD3, 0xDB, 0xD3, 0xD2, 0xD3, 0xD3, 0xB0, 0x4A, /* 0x74-0x77 */ + 0x00, 0x00, 0xB0, 0x4E, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0xD3, 0xDC, 0xB0, 0x4D, 0xD3, 0xDA, 0xD3, 0xD7, /* 0x7C-0x7F */ + + 0xD3, 0xD5, 0xB0, 0x4B, 0xB0, 0x4C, 0xD3, 0xD9, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0xB3, 0x50, 0xD7, 0xB2, 0x00, 0x00, 0xB3, 0x55, /* 0x88-0x8B */ + 0xD7, 0xC2, 0xB3, 0x54, 0xD7, 0xC4, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xD7, 0xB8, 0xB3, 0x52, 0xD7, 0xC3, /* 0x90-0x93 */ + 0x00, 0x00, 0xD7, 0xB3, 0xB3, 0x53, 0xD7, 0xBF, /* 0x94-0x97 */ + 0xD7, 0xBB, 0xD7, 0xBD, 0xD7, 0xB7, 0xD7, 0xBE, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0xB3, 0x4F, 0xD7, 0xBA, /* 0x9C-0x9F */ + 0x00, 0x00, 0xD7, 0xB9, 0xD7, 0xB5, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xD7, 0xC0, 0x00, 0x00, 0x00, 0x00, 0xD7, 0xBC, /* 0xA4-0xA7 */ + 0xD7, 0xB4, 0x00, 0x00, 0xD7, 0xB6, 0xB3, 0x51, /* 0xA8-0xAB */ + 0xD7, 0xC1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0xB5, 0xF6, 0xDB, 0xCD, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0xDB, 0xC9, 0xDB, 0xCB, /* 0xB4-0xB7 */ + 0xDB, 0xC6, 0xDB, 0xC5, 0xDB, 0xC3, 0x00, 0x00, /* 0xB8-0xBB */ + 0xDB, 0xCA, 0xDB, 0xCC, 0xDB, 0xC8, 0x00, 0x00, /* 0xBC-0xBF */ + 0xDB, 0xC7, 0xB5, 0xF4, 0xB5, 0xF5, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xDB, 0xCF, 0xB8, 0xCD, 0xDF, 0xF2, /* 0xC8-0xCB */ + 0xDF, 0xF8, 0xDF, 0xF3, 0xDF, 0xF4, 0xF9, 0xD8, /* 0xCC-0xCF */ + 0xDF, 0xF9, 0x00, 0x00, 0xB8, 0xCF, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xB8, 0xC7, 0xB8, 0xCE, 0xDF, 0xF1, 0xDB, 0xC4, /* 0xD4-0xD7 */ + 0xB8, 0xCA, 0xB8, 0xC8, 0xDF, 0xF7, 0xDF, 0xF6, /* 0xD8-0xDB */ + 0xB8, 0xC9, 0xB8, 0xCB, 0xDF, 0xF5, 0xB8, 0xC6, /* 0xDC-0xDF */ + 0x00, 0x00, 0xB8, 0xCC, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE3, 0xF6, /* 0xE4-0xE7 */ + 0xBB, 0x74, 0x00, 0x00, 0x00, 0x00, 0xE4, 0x42, /* 0xE8-0xEB */ + 0xE4, 0x41, 0x00, 0x00, 0xE3, 0xFB, 0xBB, 0x76, /* 0xEC-0xEF */ + 0xE4, 0x40, 0xE3, 0xF7, 0xE3, 0xF8, 0xBB, 0x6E, /* 0xF0-0xF3 */ + 0xBB, 0x70, 0x00, 0x00, 0xE3, 0xFD, 0xE3, 0xF5, /* 0xF4-0xF7 */ + 0xBB, 0x72, 0xBB, 0x71, 0xE3, 0xF9, 0xE3, 0xFE, /* 0xF8-0xFB */ + 0xE3, 0xFC, 0xBB, 0x73, 0xE3, 0xFA, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_89[512] = { + 0x00, 0x00, 0xDB, 0xCE, 0xBB, 0x6F, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0xE7, 0xC2, 0xE7, 0xC9, 0xBD, 0xC6, /* 0x04-0x07 */ + 0x00, 0x00, 0xE7, 0xCD, 0xBD, 0xCA, 0xE7, 0xC5, /* 0x08-0x0B */ + 0xE7, 0xC3, 0x00, 0x00, 0xE7, 0xCC, 0x00, 0x00, /* 0x0C-0x0F */ + 0xBD, 0xC5, 0xE7, 0xCB, 0xBD, 0xC7, 0xBD, 0xC8, /* 0x10-0x13 */ + 0xE7, 0xC4, 0xBD, 0xC9, 0xE7, 0xCA, 0xE7, 0xC6, /* 0x14-0x17 */ + 0xE7, 0xC7, 0xE7, 0xC8, 0xBB, 0x75, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xEB, 0x70, 0xEB, 0x7C, /* 0x1C-0x1F */ + 0x00, 0x00, 0xBF, 0xCA, 0xEB, 0x77, 0xEB, 0x79, /* 0x20-0x23 */ + 0x00, 0x00, 0xBF, 0xC8, 0xEB, 0x71, 0xEB, 0x75, /* 0x24-0x27 */ + 0x00, 0x00, 0xEB, 0x78, 0xBF, 0xC6, 0xBF, 0xC9, /* 0x28-0x2B */ + 0xEB, 0x7B, 0xEB, 0x73, 0xEB, 0x74, 0xEB, 0x7A, /* 0x2C-0x2F */ + 0xEB, 0x72, 0xEB, 0x76, 0xBF, 0xC7, 0xEE, 0x72, /* 0x30-0x33 */ + 0x00, 0x00, 0xEE, 0x71, 0xC1, 0xB7, 0xEE, 0x77, /* 0x34-0x37 */ + 0xC1, 0xB9, 0x00, 0x00, 0x00, 0x00, 0xC1, 0xB6, /* 0x38-0x3B */ + 0xEE, 0x73, 0xC1, 0xBA, 0xEE, 0x74, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0xEE, 0x75, 0xEE, 0x78, 0x00, 0x00, /* 0x40-0x43 */ + 0xC1, 0xB8, 0x00, 0x00, 0xF0, 0xD6, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0xF0, 0xD9, 0x00, 0x00, 0xF0, 0xD3, /* 0x48-0x4B */ + 0xF0, 0xD5, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xD4, /* 0x4C-0x4F */ + 0xF0, 0xD7, 0xF0, 0xD8, 0xEE, 0x76, 0xF0, 0xD2, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0xC3, 0xCD, 0xF2, 0xEC, /* 0x54-0x57 */ + 0xF2, 0xEF, 0xF2, 0xF1, 0xF2, 0xEA, 0xF2, 0xEB, /* 0x58-0x5B */ + 0xF2, 0xEE, 0xF2, 0xF0, 0xC3, 0xCE, 0xC3, 0xCC, /* 0x5C-0x5F */ + 0xC3, 0xCB, 0xF2, 0xED, 0xF2, 0xE9, 0xF4, 0xCA, /* 0x60-0x63 */ + 0xC4, 0xB0, 0x00, 0x00, 0xF4, 0xCB, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0xF6, 0x49, 0xC4, 0xFB, 0xF6, 0x4B, /* 0x68-0x6B */ + 0xC4, 0xFC, 0xF6, 0x48, 0xF6, 0x4A, 0xC5, 0xA8, /* 0x6C-0x6F */ + 0x00, 0x00, 0xF7, 0x52, 0xC5, 0xA7, 0xF7, 0xFD, /* 0x70-0x73 */ + 0xF7, 0xFC, 0x00, 0x00, 0xF7, 0xFB, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0xF9, 0x48, 0xF9, 0x49, 0xF9, 0x4B, /* 0x78-0x7B */ + 0xF9, 0x4A, 0x00, 0x00, 0xCA, 0x50, 0xA6, 0xE8, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xAD, 0x6E, 0xD7, 0xC5, 0xB5, 0xF7, /* 0x80-0x83 */ + 0x00, 0x00, 0xDF, 0xFA, 0xC2, 0xD0, 0x00, 0x00, /* 0x84-0x87 */ + 0xF2, 0xF2, 0x00, 0x00, 0x00, 0x00, 0xA8, 0xA3, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB3, 0x57, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB3, 0x56, /* 0x90-0x93 */ + 0x00, 0x00, 0xDB, 0xD0, 0xB5, 0xF8, 0xDB, 0xD2, /* 0x94-0x97 */ + 0xDB, 0xD1, 0x00, 0x00, 0x00, 0x00, 0xDF, 0xFB, /* 0x98-0x9B */ + 0xB8, 0xD0, 0xE4, 0x43, 0xE4, 0x46, 0xE4, 0x45, /* 0x9C-0x9F */ + 0x00, 0x00, 0xE4, 0x44, 0xE7, 0xCE, 0xE7, 0xD0, /* 0xA0-0xA3 */ + 0xE7, 0xCF, 0x00, 0x00, 0xBF, 0xCC, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0xBF, 0xCB, 0x00, 0x00, /* 0xA8-0xAB */ + 0xC1, 0xBB, 0xEE, 0x79, 0xEE, 0x7B, 0xEE, 0x7A, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xC2, 0xD1, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0xF2, 0xF4, 0xF2, 0xF3, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xF4, 0xCC, 0xC4, 0xB1, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0xC4, 0xFD, 0xF7, 0x54, 0xF7, 0x53, /* 0xBC-0xBF */ + 0xC6, 0x5B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xA8, 0xA4, 0xD0, 0xAF, /* 0xD0-0xD3 */ + 0xAD, 0x6F, 0xD7, 0xC8, 0xD7, 0xC6, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xD7, 0xC7, 0xDB, 0xD4, 0xDB, 0xD5, /* 0xD8-0xDB */ + 0xE0, 0x43, 0xDB, 0xD3, 0x00, 0x00, 0xDF, 0xFC, /* 0xDC-0xDF */ + 0xE0, 0x41, 0xE0, 0x40, 0xE0, 0x42, 0xB8, 0xD1, /* 0xE0-0xE3 */ + 0xDF, 0xFE, 0xDF, 0xFD, 0xE0, 0x44, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xE4, 0x49, 0xE4, 0x47, 0x00, 0x00, 0xE4, 0x48, /* 0xE8-0xEB */ + 0xE7, 0xD3, 0xE7, 0xD1, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0xE7, 0xD2, 0xEB, 0x7D, 0xEE, 0x7C, 0xEE, 0x7D, /* 0xF0-0xF3 */ + 0xC2, 0xD2, 0x00, 0x00, 0xF2, 0xF5, 0xF4, 0xCD, /* 0xF4-0xF7 */ + 0xC4, 0xB2, 0x00, 0x00, 0xF6, 0x4C, 0xF7, 0x55, /* 0xF8-0xFB */ + 0xC5, 0xA9, 0x00, 0x00, 0xF7, 0xFE, 0xF9, 0x4C, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_8A[512] = { + 0xA8, 0xA5, 0x00, 0x00, 0xAD, 0x71, 0xAD, 0x72, /* 0x00-0x03 */ + 0xD0, 0xB0, 0x00, 0x00, 0x00, 0x00, 0xD0, 0xB1, /* 0x04-0x07 */ + 0xAD, 0x70, 0x00, 0x00, 0xB0, 0x54, 0x00, 0x00, /* 0x08-0x0B */ + 0xB0, 0x52, 0x00, 0x00, 0xB0, 0x51, 0xB0, 0x58, /* 0x0C-0x0F */ + 0xB0, 0x50, 0xB0, 0x59, 0xD3, 0xDD, 0xB0, 0x56, /* 0x10-0x13 */ + 0x00, 0x00, 0xB0, 0x53, 0xB0, 0x57, 0xB0, 0x55, /* 0x14-0x17 */ + 0xB0, 0x4F, 0x00, 0x00, 0x00, 0x00, 0xB3, 0x5F, /* 0x18-0x1B */ + 0x00, 0x00, 0xB3, 0x59, 0xD7, 0xCC, 0xB3, 0x5E, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0xB3, 0x60, 0xB3, 0x5A, /* 0x20-0x23 */ + 0x00, 0x00, 0xB3, 0x5B, 0x00, 0x00, 0xD7, 0xCA, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xB3, 0x58, 0x00, 0x00, /* 0x28-0x2B */ + 0xD7, 0xCB, 0xB3, 0x5D, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xD7, 0xC9, 0xB3, 0x5C, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0xB6, 0x44, 0x00, 0x00, 0xB6, 0x46, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0xDB, 0xD8, 0xB6, 0x45, 0xB5, 0xF9, /* 0x38-0x3B */ + 0xB5, 0xFD, 0x00, 0x00, 0xB8, 0xE4, 0xE0, 0x49, /* 0x3C-0x3F */ + 0xDB, 0xDA, 0xB5, 0xFE, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0xDB, 0xDD, 0xDB, 0xDE, 0xB6, 0x43, 0x00, 0x00, /* 0x44-0x47 */ + 0xDB, 0xE0, 0x00, 0x00, 0xDB, 0xE2, 0x00, 0x00, /* 0x48-0x4B */ + 0xDB, 0xE3, 0xDB, 0xD7, 0xDB, 0xD6, 0xDB, 0xE4, /* 0x4C-0x4F */ + 0xB6, 0x42, 0xDB, 0xE1, 0xDB, 0xDF, 0x00, 0x00, /* 0x50-0x53 */ + 0xB6, 0x40, 0xB5, 0xFB, 0xB6, 0x47, 0xDB, 0xDB, /* 0x54-0x57 */ + 0xDB, 0xDC, 0xDB, 0xD9, 0x00, 0x00, 0xB6, 0x41, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0xB5, 0xFC, 0x00, 0x00, /* 0x5C-0x5F */ + 0xB5, 0xFA, 0xE0, 0x48, 0xB8, 0xDF, 0xB8, 0xDA, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0xB8, 0xD5, 0x00, 0x00, /* 0x64-0x67 */ + 0xB8, 0xE5, 0xB8, 0xD6, 0x00, 0x00, 0xB8, 0xD2, /* 0x68-0x6B */ + 0xB8, 0xE1, 0xB8, 0xDE, 0xB8, 0xE0, 0x00, 0x00, /* 0x6C-0x6F */ + 0xB8, 0xD7, 0xB8, 0xDC, 0xB8, 0xD3, 0xB8, 0xD4, /* 0x70-0x73 */ + 0xE0, 0x50, 0xE0, 0x4D, 0xE0, 0x45, 0xE0, 0x4A, /* 0x74-0x77 */ + 0x00, 0x00, 0xB8, 0xE2, 0xE0, 0x51, 0xB8, 0xE3, /* 0x78-0x7B */ + 0xB8, 0xD9, 0x00, 0x00, 0x00, 0x00, 0xE0, 0x47, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xE0, 0x4F, 0xE0, 0x4B, 0xE0, 0x4E, /* 0x80-0x83 */ + 0xE0, 0x4C, 0xB8, 0xDD, 0xE0, 0x46, 0xB8, 0xD8, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0x4C, /* 0x88-0x8B */ + 0xBB, 0x78, 0xBB, 0x7B, 0x00, 0x00, 0xE4, 0x4E, /* 0x8C-0x8F */ + 0x00, 0x00, 0xBB, 0xA5, 0xE4, 0x4D, 0xBB, 0x7D, /* 0x90-0x93 */ + 0x00, 0x00, 0xBD, 0xCF, 0xE4, 0x4F, 0x00, 0x00, /* 0x94-0x97 */ + 0xBB, 0xA4, 0xE4, 0x4B, 0xBB, 0xA6, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0xBB, 0x79, 0x00, 0x00, /* 0x9C-0x9F */ + 0xB8, 0xDB, 0xBB, 0x7C, 0x00, 0x00, 0xBB, 0x7A, /* 0xA0-0xA3 */ + 0xBB, 0x7E, 0xBB, 0xA2, 0xBB, 0x77, 0xBB, 0xA7, /* 0xA4-0xA7 */ + 0xBB, 0xA3, 0x00, 0x00, 0xBB, 0xA1, 0xE4, 0x4A, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0xBD, 0xD6, 0x00, 0x00, 0xBD, 0xD2, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0xBD, 0xD9, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xE7, 0xD6, 0xBD, 0xDA, 0xE7, 0xE2, 0xE7, 0xDB, /* 0xB8-0xBB */ + 0xBD, 0xCB, 0xE7, 0xE3, 0xE7, 0xDD, 0xBD, 0xD5, /* 0xBC-0xBF */ + 0xE7, 0xDE, 0x00, 0x00, 0xBD, 0xD4, 0xE7, 0xE1, /* 0xC0-0xC3 */ + 0xBD, 0xCE, 0xE7, 0xDF, 0xE7, 0xD5, 0xBD, 0xCD, /* 0xC4-0xC7 */ + 0xEB, 0xAA, 0xBD, 0xD3, 0x00, 0x00, 0xBD, 0xD0, /* 0xC8-0xCB */ + 0x00, 0x00, 0xBD, 0xD8, 0x00, 0x00, 0xE7, 0xD4, /* 0xCC-0xCF */ + 0x00, 0x00, 0xE7, 0xD8, 0xBD, 0xCC, 0xE7, 0xD7, /* 0xD0-0xD3 */ + 0xE7, 0xD9, 0xE7, 0xDA, 0xBD, 0xD7, 0xE7, 0xDC, /* 0xD4-0xD7 */ + 0xE7, 0xE0, 0xE7, 0xE4, 0x00, 0x00, 0xBD, 0xDB, /* 0xD8-0xDB */ + 0xBF, 0xD2, 0xEB, 0xA5, 0xEB, 0xAB, 0xEB, 0xA8, /* 0xDC-0xDF */ + 0xEB, 0x7E, 0xEB, 0xAC, 0xEB, 0xA1, 0x00, 0x00, /* 0xE0-0xE3 */ + 0xEB, 0xA7, 0x00, 0x00, 0xBF, 0xCD, 0xBF, 0xD3, /* 0xE4-0xE7 */ + 0xEB, 0xAD, 0x00, 0x00, 0x00, 0x00, 0xBF, 0xCF, /* 0xE8-0xEB */ + 0x00, 0x00, 0xBF, 0xD9, 0xBF, 0xD4, 0xEB, 0xAF, /* 0xEC-0xEF */ + 0xEB, 0xA9, 0xBF, 0xD0, 0xEB, 0xA2, 0xBF, 0xDA, /* 0xF0-0xF3 */ + 0xEB, 0xA3, 0xEB, 0xA4, 0xBF, 0xDB, 0xBF, 0xD8, /* 0xF4-0xF7 */ + 0xBD, 0xD1, 0x00, 0x00, 0xBF, 0xCE, 0xEB, 0xB0, /* 0xF8-0xFB */ + 0xBF, 0xDC, 0x00, 0x00, 0xBF, 0xD5, 0xEB, 0xAE, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_8B[512] = { + 0xBF, 0xD1, 0xBF, 0xD6, 0xBF, 0xD7, 0x00, 0x00, /* 0x00-0x03 */ + 0xC1, 0xC3, 0xEE, 0xA4, 0xEE, 0xAD, 0xEE, 0xAA, /* 0x04-0x07 */ + 0xEE, 0xAC, 0x00, 0x00, 0xC1, 0xC0, 0xEE, 0xA5, /* 0x08-0x0B */ + 0x00, 0x00, 0xEE, 0xAB, 0xC1, 0xBC, 0xEE, 0xA7, /* 0x0C-0x0F */ + 0xC1, 0xC4, 0xEE, 0xA3, 0xEE, 0xA8, 0xEE, 0xAF, /* 0x10-0x13 */ + 0xEB, 0xA6, 0xEE, 0xA9, 0xEE, 0xA2, 0xC1, 0xBD, /* 0x14-0x17 */ + 0xEE, 0xA1, 0xC1, 0xBE, 0xEE, 0xB0, 0xC1, 0xBF, /* 0x18-0x1B */ + 0xEE, 0xAE, 0xC1, 0xC2, 0xEE, 0x7E, 0x00, 0x00, /* 0x1C-0x1F */ + 0xC1, 0xC1, 0x00, 0x00, 0xEE, 0xA6, 0xF0, 0xDC, /* 0x20-0x23 */ + 0xF0, 0xEA, 0xF0, 0xE5, 0xF0, 0xE7, 0xF0, 0xDB, /* 0x24-0x27 */ + 0xC2, 0xD3, 0x00, 0x00, 0xF0, 0xDA, 0xC2, 0xD6, /* 0x28-0x2B */ + 0xC2, 0xD5, 0x00, 0x00, 0xF0, 0xE9, 0xF0, 0xE1, /* 0x2C-0x2F */ + 0xF0, 0xDE, 0xF0, 0xE4, 0x00, 0x00, 0xF0, 0xDD, /* 0x30-0x33 */ + 0x00, 0x00, 0xF0, 0xDF, 0xF0, 0xE8, 0xF0, 0xE6, /* 0x34-0x37 */ + 0x00, 0x00, 0xC2, 0xD4, 0xF0, 0xED, 0xF0, 0xEB, /* 0x38-0x3B */ + 0xF0, 0xE2, 0xF0, 0xEC, 0xF0, 0xE3, 0x00, 0x00, /* 0x3C-0x3F */ + 0xF2, 0xF9, 0xC3, 0xCF, 0xF3, 0x41, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0xF6, 0x4F, 0xC3, 0xD6, 0xF0, 0xE0, /* 0x44-0x47 */ + 0xF2, 0xF7, 0xC3, 0xD2, 0xF2, 0xF8, 0xF2, 0xFD, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0xC3, 0xD4, 0xC3, 0xD5, /* 0x4C-0x4F */ + 0xF2, 0xF6, 0xF3, 0x40, 0xF3, 0x42, 0xF2, 0xFA, /* 0x50-0x53 */ + 0xF2, 0xFC, 0xF2, 0xFE, 0xF2, 0xFB, 0xF3, 0x43, /* 0x54-0x57 */ + 0xC3, 0xD1, 0xC3, 0xD7, 0xC3, 0xD3, 0x00, 0x00, /* 0x58-0x5B */ + 0xC3, 0xD0, 0xF4, 0xD0, 0x00, 0x00, 0xC4, 0xB7, /* 0x5C-0x5F */ + 0xF4, 0xCE, 0x00, 0x00, 0x00, 0x00, 0xF4, 0xD2, /* 0x60-0x63 */ + 0x00, 0x00, 0xF4, 0xD3, 0xC4, 0xB5, 0xF4, 0xD4, /* 0x64-0x67 */ + 0xF4, 0xD1, 0x00, 0x00, 0xF4, 0xCF, 0xC4, 0xB8, /* 0x68-0x6B */ + 0xC4, 0xB4, 0xF4, 0xD5, 0x00, 0x00, 0xC4, 0xB6, /* 0x6C-0x6F */ + 0xC4, 0xB3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0xC4, 0xFE, 0x00, 0x00, 0x00, 0x00, 0xC5, 0x40, /* 0x74-0x77 */ + 0xF6, 0x4E, 0xF6, 0x4D, 0xF6, 0x50, 0xF6, 0x51, /* 0x78-0x7B */ + 0x00, 0x00, 0xC5, 0x41, 0xF7, 0x56, 0xF7, 0x5B, /* 0x7C-0x7F */ + + 0xC5, 0xAA, 0x00, 0x00, 0xF7, 0x58, 0x00, 0x00, /* 0x80-0x83 */ + 0xF7, 0x57, 0xF7, 0x5A, 0xF7, 0x59, 0x00, 0x00, /* 0x84-0x87 */ + 0xF8, 0x43, 0x00, 0x00, 0xC5, 0xDC, 0xF8, 0x42, /* 0x88-0x8B */ + 0xF8, 0x40, 0x00, 0x00, 0xF8, 0x41, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0xC5, 0xFE, 0xC5, 0xFD, /* 0x90-0x93 */ + 0xF8, 0xC1, 0xF8, 0xC2, 0xC6, 0x40, 0x00, 0x00, /* 0x94-0x97 */ + 0xF9, 0x4D, 0xF9, 0x4E, 0xC6, 0x67, 0x00, 0x00, /* 0x98-0x9B */ + 0xC6, 0x6D, 0x00, 0x00, 0xF9, 0xA9, 0xF9, 0xC8, /* 0x9C-0x9F */ +}; + +static const unsigned char u2c_8C[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA8, 0xA6, /* 0x34-0x37 */ + 0x00, 0x00, 0xD7, 0xCD, 0x00, 0x00, 0xD7, 0xCE, /* 0x38-0x3B */ + 0xE0, 0x52, 0xE4, 0x50, 0xE7, 0xE5, 0xC1, 0xC6, /* 0x3C-0x3F */ + 0x00, 0x00, 0xC1, 0xC5, 0xF0, 0xEE, 0xF3, 0x44, /* 0x40-0x43 */ + 0x00, 0x00, 0xF8, 0x44, 0xA8, 0xA7, 0xD3, 0xDE, /* 0x44-0x47 */ + 0xB0, 0x5A, 0xB3, 0x61, 0xE0, 0x54, 0xE0, 0x53, /* 0x48-0x4B */ + 0xBD, 0xDC, 0xE7, 0xE6, 0xBD, 0xDD, 0xEE, 0xB1, /* 0x4C-0x4F */ + 0xC2, 0xD7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0xC6, 0x76, 0xA8, 0xA8, 0xCD, 0xCB, 0xD3, 0xDF, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0xB3, 0x62, 0x00, 0x00, /* 0x58-0x5B */ + 0xD7, 0xCF, 0xD7, 0xD0, 0x00, 0x00, 0xDB, 0xE5, /* 0x5C-0x5F */ + 0x00, 0x00, 0xB6, 0x48, 0xB8, 0xE6, 0x00, 0x00, /* 0x60-0x63 */ + 0xE0, 0x56, 0xE0, 0x55, 0xE0, 0x57, 0x00, 0x00, /* 0x64-0x67 */ + 0xE4, 0x51, 0xE4, 0x52, 0xBB, 0xA8, 0xBF, 0xDD, /* 0x68-0x6B */ + 0xBD, 0xDE, 0xBF, 0xDE, 0x00, 0x00, 0xEE, 0xB5, /* 0x6C-0x6F */ + 0xEE, 0xB2, 0xEE, 0xB4, 0xEE, 0xB3, 0xC1, 0xC7, /* 0x70-0x73 */ + 0x00, 0x00, 0xF0, 0xEF, 0xF3, 0x46, 0xF3, 0x45, /* 0x74-0x77 */ + 0xCB, 0xA4, 0xB0, 0x5C, 0xB0, 0x5B, 0xD3, 0xE0, /* 0x78-0x7B */ + 0x00, 0x00, 0xD7, 0xD1, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xDB, 0xE7, 0xDB, 0xE6, 0xB6, 0x49, 0x00, 0x00, /* 0x80-0x83 */ + 0xE0, 0x59, 0xE0, 0x5A, 0xE0, 0x58, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0xB8, 0xE8, 0xB8, 0xE7, 0x00, 0x00, /* 0x88-0x8B */ + 0xBB, 0xAA, 0xBB, 0xA9, 0x00, 0x00, 0xE7, 0xE7, /* 0x8C-0x8F */ + 0xEB, 0xB3, 0xEB, 0xB1, 0xEB, 0xB2, 0xBF, 0xDF, /* 0x90-0x93 */ + 0xEE, 0xB7, 0xEE, 0xB6, 0x00, 0x00, 0xF0, 0xF2, /* 0x94-0x97 */ + 0xF0, 0xF1, 0xF0, 0xF0, 0xF3, 0x47, 0x00, 0x00, /* 0x98-0x9B */ + 0xF9, 0xAA, 0xA8, 0xA9, 0xAD, 0x73, 0x00, 0x00, /* 0x9C-0x9F */ + 0xAD, 0x74, 0xB0, 0x5D, 0xB0, 0x5E, 0xD3, 0xE2, /* 0xA0-0xA3 */ + 0xD3, 0xE1, 0xD7, 0xD2, 0x00, 0x00, 0xB3, 0x68, /* 0xA4-0xA7 */ + 0xB3, 0x66, 0xB3, 0x63, 0xB3, 0x67, 0xB3, 0x65, /* 0xA8-0xAB */ + 0xB3, 0x64, 0x00, 0x00, 0x00, 0x00, 0xB6, 0x4A, /* 0xAC-0xAF */ + 0xDB, 0xEA, 0x00, 0x00, 0xB8, 0xED, 0xB6, 0x4C, /* 0xB0-0xB3 */ + 0xB6, 0x51, 0xDB, 0xEC, 0xB6, 0x53, 0xB6, 0x52, /* 0xB4-0xB7 */ + 0xB6, 0x55, 0xDB, 0xEB, 0xDB, 0xE8, 0xB6, 0x4F, /* 0xB8-0xBB */ + 0xB6, 0x4B, 0xB6, 0x4D, 0xDB, 0xE9, 0xB6, 0x54, /* 0xBC-0xBF */ + 0xB6, 0x50, 0xB6, 0x4E, 0xB8, 0xEF, 0xB8, 0xEE, /* 0xC0-0xC3 */ + 0xB8, 0xEC, 0xB8, 0xF0, 0x00, 0x00, 0xB8, 0xEA, /* 0xC4-0xC7 */ + 0xB8, 0xEB, 0x00, 0x00, 0xB8, 0xE9, 0x00, 0x00, /* 0xC8-0xCB */ + 0xE0, 0x5B, 0x00, 0x00, 0x00, 0x00, 0xE4, 0x54, /* 0xCC-0xCF */ + 0x00, 0x00, 0xBB, 0xAC, 0xBB, 0xAD, 0xBB, 0xAB, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xE4, 0x53, 0x00, 0x00, 0xE4, 0x55, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xE7, 0xEA, 0xE7, 0xEC, 0x00, 0x00, /* 0xD8-0xDB */ + 0xBD, 0xE7, 0xE7, 0xED, 0xBD, 0xE0, 0xE7, 0xE9, /* 0xDC-0xDF */ + 0xBD, 0xDF, 0xBD, 0xE9, 0xBD, 0xE5, 0xBD, 0xE6, /* 0xE0-0xE3 */ + 0xBD, 0xE2, 0xE7, 0xE8, 0xBD, 0xE1, 0xE7, 0xEE, /* 0xE4-0xE7 */ + 0xE7, 0xEB, 0x00, 0x00, 0xBD, 0xE8, 0x00, 0x00, /* 0xE8-0xEB */ + 0xBD, 0xE3, 0xBD, 0xE4, 0xEB, 0xB5, 0x00, 0x00, /* 0xEC-0xEF */ + 0xEB, 0xB7, 0xEB, 0xB6, 0x00, 0x00, 0xEB, 0xB8, /* 0xF0-0xF3 */ + 0xBF, 0xE0, 0xEB, 0xB4, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xC1, 0xCB, 0xEE, 0xB8, 0xC1, 0xC8, 0xC1, 0xCC, /* 0xF8-0xFB */ + 0xC1, 0xCA, 0xC1, 0xC9, 0xF0, 0xF3, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_8D[512] = { + 0xF0, 0xF6, 0x00, 0x00, 0xF0, 0xF5, 0x00, 0x00, /* 0x00-0x03 */ + 0xF0, 0xF4, 0xC2, 0xD8, 0xF3, 0x48, 0xF3, 0x49, /* 0x04-0x07 */ + 0xC3, 0xD8, 0xF3, 0x4A, 0xC3, 0xD9, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0xC4, 0xBA, 0x00, 0x00, 0xC4, 0xB9, /* 0x0C-0x0F */ + 0xF6, 0x52, 0x00, 0x00, 0x00, 0x00, 0xC5, 0x42, /* 0x10-0x13 */ + 0xF6, 0x53, 0xF7, 0x5C, 0xC5, 0xAB, 0xC5, 0xAC, /* 0x14-0x17 */ + 0x00, 0x00, 0xF8, 0x45, 0x00, 0x00, 0xC6, 0x42, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0xA8, 0xAA, 0x00, 0x00, 0xB3, 0x6A, 0xB3, 0x69, /* 0x64-0x67 */ + 0xE0, 0x5C, 0xE0, 0x5D, 0x00, 0x00, 0xBB, 0xAE, /* 0x68-0x6B */ + 0xEB, 0xB9, 0xBD, 0xEA, 0xEB, 0xBA, 0xEE, 0xB9, /* 0x6C-0x6F */ + 0xA8, 0xAB, 0x00, 0x00, 0xD0, 0xB2, 0xAD, 0x76, /* 0x70-0x73 */ + 0xAD, 0x75, 0x00, 0x00, 0xD3, 0xE3, 0xB0, 0x5F, /* 0x74-0x77 */ + 0xD3, 0xE4, 0xD7, 0xD5, 0x00, 0x00, 0xD7, 0xD4, /* 0x78-0x7B */ + 0x00, 0x00, 0xD7, 0xD3, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xDB, 0xEE, 0xB6, 0x58, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0xDB, 0xED, 0xB6, 0x57, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0xDB, 0xEF, 0xB6, 0x56, 0x00, 0x00, /* 0x88-0x8B */ + 0xE0, 0x5F, 0xE0, 0x62, 0xE0, 0x60, 0xE0, 0x61, /* 0x8C-0x8F */ + 0xE0, 0x65, 0xE0, 0x5E, 0xE0, 0x66, 0xE0, 0x63, /* 0x90-0x93 */ + 0xE0, 0x64, 0xBB, 0xB0, 0xE4, 0x56, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0xBB, 0xAF, 0x00, 0x00, 0xE7, 0xF2, /* 0x98-0x9B */ + 0xE7, 0xF0, 0x00, 0x00, 0x00, 0x00, 0xBD, 0xEB, /* 0x9C-0x9F */ + 0xE7, 0xEF, 0xE7, 0xF1, 0x00, 0x00, 0xBD, 0xEC, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xEB, 0xBB, 0x00, 0x00, 0xEB, 0xBC, /* 0xA4-0xA7 */ + 0xC1, 0xCD, 0x00, 0x00, 0xF3, 0x4C, 0xF3, 0x4E, /* 0xA8-0xAB */ + 0xF3, 0x4B, 0xF3, 0x4D, 0xF4, 0xD6, 0xF6, 0x54, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xF9, 0x6F, 0xA8, 0xAC, /* 0xB0-0xB3 */ + 0xAD, 0x77, 0xD3, 0xE5, 0xD3, 0xE7, 0xD3, 0xE6, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xD7, 0xD8, 0xB3, 0x6C, 0x00, 0x00, /* 0xB8-0xBB */ + 0xD7, 0xD6, 0x00, 0x00, 0xB3, 0x6B, 0xD7, 0xD9, /* 0xBC-0xBF */ + 0x00, 0x00, 0xD7, 0xDA, 0xD7, 0xD7, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xDB, 0xFB, 0xB6, 0x60, 0xDB, 0xF3, /* 0xC4-0xC7 */ + 0xDB, 0xF9, 0x00, 0x00, 0x00, 0x00, 0xB6, 0x5B, /* 0xC8-0xCB */ + 0xB6, 0x5E, 0xDB, 0xF2, 0xB6, 0x59, 0xDB, 0xF6, /* 0xCC-0xCF */ + 0xE0, 0x6C, 0xB6, 0x5D, 0x00, 0x00, 0xDB, 0xF1, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xDB, 0xF7, 0xDB, 0xF4, 0xDB, 0xFA, /* 0xD4-0xD7 */ + 0xDB, 0xF0, 0xDB, 0xF8, 0xB6, 0x5C, 0xB6, 0x5F, /* 0xD8-0xDB */ + 0xDB, 0xF5, 0xB6, 0x5A, 0x00, 0x00, 0xB8, 0xF2, /* 0xDC-0xDF */ + 0xE0, 0x68, 0xB8, 0xF1, 0xE0, 0x6F, 0xE0, 0x6E, /* 0xE0-0xE3 */ + 0xB8, 0xF8, 0x00, 0x00, 0xB8, 0xF9, 0xE0, 0x70, /* 0xE4-0xE7 */ + 0xB8, 0xF3, 0xE0, 0x6D, 0xB8, 0xF7, 0xE0, 0x72, /* 0xE8-0xEB */ + 0xE0, 0x69, 0x00, 0x00, 0xE0, 0x6B, 0xB8, 0xF4, /* 0xEC-0xEF */ + 0xE0, 0x67, 0xE0, 0x6A, 0xE0, 0x71, 0xB8, 0xF5, /* 0xF0-0xF3 */ + 0xE0, 0x73, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0x00, 0x00, 0xB8, 0xF6, 0x00, 0x00, /* 0xF8-0xFB */ + 0xBB, 0xB1, 0xE4, 0x5B, 0xE4, 0x61, 0xE4, 0x59, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_8E[512] = { + 0xE4, 0x62, 0x00, 0x00, 0xE4, 0x58, 0xE4, 0x5D, /* 0x00-0x03 */ + 0xE4, 0x63, 0xE4, 0x60, 0xE4, 0x5F, 0xE4, 0x5E, /* 0x04-0x07 */ + 0x00, 0x00, 0xE4, 0x57, 0xE4, 0x5C, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0xE4, 0x5A, 0x00, 0x00, 0xBD, 0xF1, /* 0x0C-0x0F */ + 0xBD, 0xEE, 0xE7, 0xFB, 0xE8, 0x41, 0xE8, 0x43, /* 0x10-0x13 */ + 0xE8, 0x40, 0xE7, 0xF8, 0xE7, 0xFA, 0xE8, 0x45, /* 0x14-0x17 */ + 0xE8, 0x42, 0xE7, 0xFC, 0xE8, 0x46, 0xE7, 0xF9, /* 0x18-0x1B */ + 0xE8, 0x44, 0xBD, 0xEF, 0xBD, 0xF5, 0xBD, 0xF3, /* 0x1C-0x1F */ + 0xE7, 0xF3, 0xBD, 0xF4, 0xBD, 0xF0, 0xE7, 0xF4, /* 0x20-0x23 */ + 0xE7, 0xF6, 0xE7, 0xF5, 0xE7, 0xFD, 0xE7, 0xFE, /* 0x24-0x27 */ + 0x00, 0x00, 0xBD, 0xF2, 0x00, 0x00, 0xBD, 0xED, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0xE7, 0xF7, 0x00, 0x00, /* 0x2C-0x2F */ + 0xEB, 0xC6, 0xBF, 0xE2, 0x00, 0x00, 0xEB, 0xBD, /* 0x30-0x33 */ + 0xBF, 0xE3, 0xBF, 0xE6, 0xEB, 0xC2, 0x00, 0x00, /* 0x34-0x37 */ + 0xEB, 0xBF, 0xBF, 0xE5, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0xEB, 0xC3, 0xEB, 0xC4, 0xEB, 0xBE, 0xEB, 0xC7, /* 0x3C-0x3F */ + 0xEB, 0xC0, 0xEB, 0xC5, 0xBF, 0xE4, 0x00, 0x00, /* 0x40-0x43 */ + 0xBF, 0xE1, 0xEB, 0xC1, 0x00, 0x00, 0xEE, 0xBF, /* 0x44-0x47 */ + 0xC1, 0xD0, 0xC1, 0xCE, 0xC1, 0xD1, 0xC1, 0xCF, /* 0x48-0x4B */ + 0xEE, 0xBE, 0xEE, 0xBB, 0xEE, 0xBA, 0x00, 0x00, /* 0x4C-0x4F */ + 0xEE, 0xBD, 0x00, 0x00, 0x00, 0x00, 0xEE, 0xBC, /* 0x50-0x53 */ + 0xF1, 0x45, 0xC2, 0xDE, 0xF0, 0xFB, 0xF0, 0xFA, /* 0x54-0x57 */ + 0x00, 0x00, 0xC2, 0xD9, 0xF1, 0x41, 0xF1, 0x40, /* 0x58-0x5B */ + 0xF0, 0xF7, 0xF1, 0x43, 0xF0, 0xFC, 0xC2, 0xDD, /* 0x5C-0x5F */ + 0xF0, 0xF9, 0xF1, 0x42, 0xF0, 0xF8, 0xC2, 0xDA, /* 0x60-0x63 */ + 0xC2, 0xDC, 0xF0, 0xFD, 0xC2, 0xDB, 0xF0, 0xFE, /* 0x64-0x67 */ + 0x00, 0x00, 0xF1, 0x44, 0xF3, 0x52, 0x00, 0x00, /* 0x68-0x6B */ + 0xC3, 0xDE, 0xF3, 0x4F, 0x00, 0x00, 0xF3, 0x53, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0xC3, 0xDB, 0xF3, 0x51, /* 0x70-0x73 */ + 0xC3, 0xE0, 0x00, 0x00, 0xC3, 0xDD, 0x00, 0x00, /* 0x74-0x77 */ + 0xF3, 0x50, 0x00, 0x00, 0xC3, 0xDF, 0xF3, 0x54, /* 0x78-0x7B */ + 0xC3, 0xDA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0xC4, 0xBC, 0xC4, 0xBE, 0x00, 0x00, /* 0x80-0x83 */ + 0xF4, 0xD9, 0xC4, 0xBD, 0xF4, 0xD7, 0xC3, 0xDC, /* 0x84-0x87 */ + 0xF4, 0xD8, 0xC4, 0xBB, 0xC5, 0x43, 0xC5, 0x45, /* 0x88-0x8B */ + 0xF6, 0x56, 0xC5, 0x44, 0xF6, 0x55, 0x00, 0x00, /* 0x8C-0x8F */ + 0xF7, 0x61, 0xC5, 0xAD, 0xF7, 0x60, 0xC5, 0xAE, /* 0x90-0x93 */ + 0xF7, 0x5E, 0xF7, 0x5D, 0xF7, 0x62, 0xF7, 0x63, /* 0x94-0x97 */ + 0xF8, 0x46, 0x00, 0x00, 0xF7, 0x5F, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0xF8, 0xC6, 0xF8, 0xC3, 0xF8, 0xC4, /* 0x9C-0x9F */ + 0xF8, 0xC5, 0xC6, 0x5C, 0x00, 0x00, 0xF9, 0x51, /* 0xA0-0xA3 */ + 0xF9, 0x50, 0xF9, 0x4F, 0xF9, 0x70, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xF9, 0xBE, 0xF9, 0xAB, 0xC6, 0x6E, 0xA8, 0xAD, /* 0xA8-0xAB */ + 0xB0, 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xB8, 0xFA, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0xBD, 0xF6, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0xEB, 0xC8, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0xC2, 0xDF, 0x00, 0x00, 0xF3, 0x55, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xF9, 0xAC, 0xA8, 0xAE, 0xAA, 0xEE, /* 0xC8-0xCB */ + 0xAD, 0x79, 0xAD, 0x78, 0x00, 0x00, 0xB0, 0x63, /* 0xCC-0xCF */ + 0x00, 0x00, 0xD3, 0xE8, 0xB0, 0x61, 0xD3, 0xE9, /* 0xD0-0xD3 */ + 0xB0, 0x62, 0x00, 0x00, 0x00, 0x00, 0xD7, 0xDF, /* 0xD4-0xD7 */ + 0xD7, 0xDB, 0x00, 0x00, 0x00, 0x00, 0xB3, 0x6D, /* 0xD8-0xDB */ + 0xD7, 0xDE, 0xD7, 0xDD, 0xD7, 0xDC, 0xB3, 0x6E, /* 0xDC-0xDF */ + 0xD7, 0xE0, 0xD7, 0xE1, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xDC, 0x43, 0xDC, 0x41, 0xDC, 0x45, /* 0xE4-0xE7 */ + 0xDC, 0x46, 0xDC, 0x4C, 0x00, 0x00, 0xDC, 0x48, /* 0xE8-0xEB */ + 0xDC, 0x4A, 0x00, 0x00, 0xDC, 0x42, 0xDB, 0xFC, /* 0xEC-0xEF */ + 0x00, 0x00, 0xDC, 0x49, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xDC, 0x4B, 0xDC, 0x44, 0xDC, 0x47, 0xDB, 0xFD, /* 0xF4-0xF7 */ + 0xB6, 0x62, 0xDC, 0x40, 0xDB, 0xFE, 0xB6, 0x61, /* 0xF8-0xFB */ + 0xB6, 0x63, 0x00, 0x00, 0xB8, 0xFD, 0xE0, 0x75, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_8F[512] = { + 0xE0, 0x77, 0xE0, 0x76, 0xE0, 0x7B, 0xB8, 0xFB, /* 0x00-0x03 */ + 0x00, 0x00, 0xE0, 0x78, 0xE0, 0x74, 0xE0, 0x79, /* 0x04-0x07 */ + 0xE0, 0x7A, 0xB8, 0xFC, 0xB8, 0xFE, 0xE0, 0x7C, /* 0x08-0x0B */ + 0x00, 0x00, 0xE4, 0x67, 0xE4, 0x66, 0x00, 0x00, /* 0x0C-0x0F */ + 0xE4, 0x64, 0xE4, 0x65, 0xBB, 0xB3, 0xBB, 0xB5, /* 0x10-0x13 */ + 0xBB, 0xB2, 0xBB, 0xB4, 0xE8, 0x4D, 0xE8, 0x4E, /* 0x14-0x17 */ + 0xE8, 0x49, 0x00, 0x00, 0xE8, 0x4A, 0xBD, 0xF8, /* 0x18-0x1B */ + 0xBD, 0xFD, 0xBD, 0xF7, 0xBD, 0xFE, 0xBD, 0xF9, /* 0x1C-0x1F */ + 0xE8, 0x4B, 0x00, 0x00, 0x00, 0x00, 0xE8, 0x4C, /* 0x20-0x23 */ + 0xE8, 0x48, 0xBE, 0x40, 0xBD, 0xFB, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0xBD, 0xFA, 0xBD, 0xFC, 0x00, 0x00, /* 0x28-0x2B */ + 0xE8, 0x47, 0x00, 0x00, 0xEB, 0xCA, 0xBF, 0xE8, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0xEB, 0xCC, 0xBF, 0xEA, /* 0x30-0x33 */ + 0xEB, 0xCF, 0xEB, 0xCB, 0xEB, 0xC9, 0xEB, 0xCE, /* 0x34-0x37 */ + 0xBF, 0xE9, 0xEB, 0xCD, 0x00, 0x00, 0xBF, 0xE7, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xC1, 0xD3, 0xC1, 0xD6, /* 0x3C-0x3F */ + 0xEE, 0xC1, 0x00, 0x00, 0xC1, 0xD4, 0xEE, 0xC0, /* 0x40-0x43 */ + 0xC1, 0xD2, 0xC1, 0xD5, 0xF1, 0x46, 0xF1, 0x47, /* 0x44-0x47 */ + 0xF1, 0x48, 0xC2, 0xE0, 0x00, 0x00, 0xF1, 0x49, /* 0x48-0x4B */ + 0x00, 0x00, 0xC2, 0xE1, 0xC3, 0xE2, 0xF3, 0x58, /* 0x4C-0x4F */ + 0xF3, 0x59, 0xF3, 0x57, 0xF3, 0x56, 0xF3, 0x5A, /* 0x50-0x53 */ + 0xC3, 0xE1, 0xF4, 0xDD, 0xF4, 0xDB, 0xF4, 0xDC, /* 0x54-0x57 */ + 0xF4, 0xDE, 0xF4, 0xDA, 0xF4, 0xDF, 0xF6, 0x58, /* 0x58-0x5B */ + 0x00, 0x00, 0xF6, 0x59, 0xF6, 0x57, 0xC5, 0x46, /* 0x5C-0x5F */ + 0xF7, 0x64, 0xC5, 0xAF, 0xF7, 0x65, 0xF8, 0x48, /* 0x60-0x63 */ + 0xF8, 0x47, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xA8, 0xAF, /* 0x98-0x9B */ + 0xB6, 0x64, 0x00, 0x00, 0x00, 0x00, 0xB9, 0x40, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBB, 0xB6, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0xBF, 0xEC, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xBF, 0xEB, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0xC3, 0xE3, 0xC4, 0x7C, 0xC5, 0x47, /* 0xAC-0xAF */ + 0xA8, 0xB0, 0xB0, 0x64, 0xB9, 0x41, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xF3, 0x5B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCB, 0xA6, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0xA8, 0xB1, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xA8, 0xB4, 0xA8, 0xB3, 0xA8, 0xB2, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xCB, 0xA5, 0x00, 0x00, 0xCD, 0xCD, /* 0xC8-0xCB */ + 0x00, 0x00, 0xCD, 0xCF, 0xAA, 0xEF, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0xAA, 0xF1, 0xCD, 0xCC, 0xCD, 0xCE, /* 0xD0-0xD3 */ + 0xAA, 0xF0, 0xCD, 0xD1, 0xCD, 0xD0, 0xCD, 0xD2, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0xD0, 0xB6, 0xD0, 0xB4, 0xAD, 0x7C, 0xD0, 0xB3, /* 0xE0-0xE3 */ + 0xAD, 0xA3, 0xAD, 0x7E, 0xAD, 0x7B, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xAD, 0xA4, 0x00, 0x00, 0xAD, 0x7D, 0xAD, 0xA2, /* 0xE8-0xEB */ + 0x00, 0x00, 0xAD, 0xA1, 0xD0, 0xB5, 0x00, 0x00, /* 0xEC-0xEF */ + 0xAD, 0x7A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xB0, 0x6A, 0xD3, 0xEB, 0xD3, 0xF1, 0xB0, 0x67, /* 0xF4-0xF7 */ + 0xB0, 0x6E, 0x00, 0x00, 0xB0, 0x69, 0xD3, 0xEE, /* 0xF8-0xFB */ + 0xD3, 0xF0, 0xB0, 0x6C, 0xD3, 0xEA, 0xD3, 0xED, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_90[512] = { + 0xB0, 0x68, 0xB0, 0x65, 0xD3, 0xEC, 0xB0, 0x6B, /* 0x00-0x03 */ + 0xD3, 0xEF, 0xB0, 0x6D, 0xB0, 0x66, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD7, 0xE3, /* 0x08-0x0B */ + 0xD7, 0xE6, 0xB3, 0x70, 0x00, 0x00, 0xB3, 0x7A, /* 0x0C-0x0F */ + 0xB3, 0x76, 0xD7, 0xE4, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0xB3, 0x7E, 0xB3, 0x77, 0xB3, 0x7C, 0xB3, 0x72, /* 0x14-0x17 */ + 0x00, 0x00, 0xB3, 0x6F, 0xB3, 0x71, 0xB3, 0x7D, /* 0x18-0x1B */ + 0xD7, 0xE5, 0xB3, 0x75, 0xB3, 0x78, 0xB3, 0x74, /* 0x1C-0x1F */ + 0xB3, 0x79, 0xD7, 0xE7, 0xB3, 0x7B, 0xB3, 0x73, /* 0x20-0x23 */ + 0xD7, 0xE2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0xDC, 0x4D, 0xB6, 0x65, 0xDC, 0x4F, /* 0x2C-0x2F */ + 0x00, 0x00, 0xB6, 0x67, 0xB6, 0x69, 0x00, 0x00, /* 0x30-0x33 */ + 0xDC, 0x4E, 0xB6, 0x66, 0xB6, 0x6A, 0x00, 0x00, /* 0x34-0x37 */ + 0xB6, 0x68, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0xB9, 0x47, 0xE0, 0xA3, 0xB9, 0x4F, 0xE0, 0x7E, /* 0x3C-0x3F */ + 0x00, 0x00, 0xB9, 0x50, 0xB9, 0x45, 0x00, 0x00, /* 0x40-0x43 */ + 0xE0, 0xA1, 0x00, 0x00, 0x00, 0x00, 0xB9, 0x4A, /* 0x44-0x47 */ + 0x00, 0x00, 0xE0, 0xA2, 0xB9, 0x43, 0xB9, 0x42, /* 0x48-0x4B */ + 0x00, 0x00, 0xB9, 0x4D, 0xB9, 0x4C, 0xB9, 0x4B, /* 0x4C-0x4F */ + 0xB9, 0x49, 0xB9, 0x4E, 0xE0, 0x7D, 0xB9, 0x44, /* 0x50-0x53 */ + 0xB9, 0x46, 0xB9, 0x48, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0xBB, 0xB8, 0xBB, 0xBB, 0x00, 0x00, 0xBB, 0xBF, /* 0x58-0x5B */ + 0xBB, 0xB9, 0xBB, 0xBE, 0xBB, 0xBC, 0x00, 0x00, /* 0x5C-0x5F */ + 0xBB, 0xB7, 0x00, 0x00, 0xBB, 0xBD, 0xBB, 0xBA, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE8, 0x52, /* 0x64-0x67 */ + 0xBE, 0x43, 0xBE, 0x41, 0x00, 0x00, 0xE8, 0x53, /* 0x68-0x6B */ + 0x00, 0x00, 0xBE, 0x44, 0xBE, 0x42, 0xE8, 0x51, /* 0x6C-0x6F */ + 0xE8, 0x50, 0x00, 0x00, 0xBF, 0xF0, 0xE8, 0x4F, /* 0x70-0x73 */ + 0xBF, 0xEE, 0xBF, 0xED, 0xEB, 0xD0, 0xBE, 0x45, /* 0x74-0x77 */ + 0xBF, 0xEF, 0xEB, 0xD1, 0xBF, 0xF2, 0xEB, 0xD2, /* 0x78-0x7B */ + 0xBF, 0xF1, 0xC1, 0xD8, 0xEE, 0xC3, 0xC1, 0xD7, /* 0x7C-0x7F */ + + 0xC1, 0xDC, 0xC1, 0xDA, 0xC1, 0xDB, 0xC2, 0xE3, /* 0x80-0x83 */ + 0xC1, 0xD9, 0xEE, 0xC2, 0xEB, 0xD3, 0xC2, 0xE2, /* 0x84-0x87 */ + 0xC2, 0xE4, 0x00, 0x00, 0xC3, 0xE4, 0xC3, 0xE5, /* 0x88-0x8B */ + 0x00, 0x00, 0xF4, 0xE0, 0x00, 0x00, 0xC5, 0xDE, /* 0x8C-0x8F */ + 0xC5, 0xDD, 0xA8, 0xB6, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0xCA, 0x55, 0xB0, 0x6F, 0x00, 0x00, 0xCA, 0x52, /* 0x94-0x97 */ + 0xCA, 0x53, 0xCA, 0x51, 0x00, 0x00, 0xCA, 0x54, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0xCB, 0xAA, 0xCB, 0xA7, /* 0x9C-0x9F */ + 0xCB, 0xAC, 0xCB, 0xA8, 0xA8, 0xB7, 0xA8, 0xBA, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xCB, 0xA9, 0xA8, 0xB9, 0xCB, 0xAB, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0xA8, 0xB8, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xCD, 0xD5, /* 0xAC-0xAF */ + 0xCD, 0xD7, 0xAA, 0xF4, 0xCD, 0xD3, 0xCD, 0xD6, /* 0xB0-0xB3 */ + 0xCD, 0xD4, 0xAA, 0xF2, 0xAA, 0xF5, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xAA, 0xF3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0xD0, 0xB8, 0xD0, 0xBC, 0xD0, 0xB9, /* 0xBC-0xBF */ + 0x00, 0x00, 0xAD, 0xA7, 0x00, 0x00, 0xAD, 0xA8, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xD0, 0xBB, 0x00, 0x00, 0xD0, 0xBD, /* 0xC4-0xC7 */ + 0xD0, 0xBF, 0x00, 0x00, 0xAD, 0xA5, 0xD0, 0xBE, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0xAD, 0xA6, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xD7, 0xEE, 0xD0, 0xBA, 0xD3, 0xF2, 0xD3, 0xFB, /* 0xD4-0xD7 */ + 0xD3, 0xF9, 0xD3, 0xF4, 0xD3, 0xF5, 0xD3, 0xFA, /* 0xD8-0xDB */ + 0xD3, 0xFC, 0xB0, 0x71, 0x00, 0x00, 0xD3, 0xF7, /* 0xDC-0xDF */ + 0xD3, 0xF3, 0xB0, 0x70, 0xB0, 0x72, 0xD3, 0xF6, /* 0xE0-0xE3 */ + 0xD3, 0xFD, 0xD3, 0xF8, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xB3, 0xA1, 0xD7, 0xF1, 0xD7, 0xE9, 0xD7, 0xEF, /* 0xE8-0xEB */ + 0xD7, 0xF0, 0xB3, 0xA2, 0x00, 0x00, 0xD7, 0xE8, /* 0xEC-0xEF */ + 0xD7, 0xEA, 0xD0, 0xB7, 0xD7, 0xEC, 0xD7, 0xED, /* 0xF0-0xF3 */ + 0xD7, 0xEB, 0xB6, 0x6C, 0x00, 0x00, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xDC, 0x56, 0xEB, 0xD4, 0xDC, 0x57, /* 0xF8-0xFB */ + 0xDC, 0x54, 0xB3, 0xA3, 0xB6, 0x6E, 0xDC, 0x53, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_91[512] = { + 0xDC, 0x59, 0xDC, 0x58, 0xB6, 0x6B, 0xDC, 0x5C, /* 0x00-0x03 */ + 0xDC, 0x52, 0xDC, 0x5B, 0xDC, 0x50, 0xDC, 0x5A, /* 0x04-0x07 */ + 0xDC, 0x55, 0xB6, 0x6D, 0x00, 0x00, 0xE0, 0xAA, /* 0x08-0x0B */ + 0x00, 0x00, 0xE0, 0xA5, 0xE0, 0xAB, 0xE0, 0xA6, /* 0x0C-0x0F */ + 0xE0, 0xA4, 0xE0, 0xA7, 0xB9, 0x51, 0x00, 0x00, /* 0x10-0x13 */ + 0xE0, 0xA9, 0x00, 0x00, 0xE0, 0xA8, 0xB9, 0x52, /* 0x14-0x17 */ + 0xBB, 0xC1, 0xBB, 0xC0, 0xE4, 0x6E, 0xE4, 0x71, /* 0x18-0x1B */ + 0xE4, 0x69, 0xE4, 0x6D, 0xBB, 0xC2, 0xE4, 0x6C, /* 0x1C-0x1F */ + 0xE4, 0x6A, 0xE4, 0x70, 0xE4, 0x6B, 0xE4, 0x68, /* 0x20-0x23 */ + 0xE4, 0x6F, 0x00, 0x00, 0xE8, 0x59, 0xBE, 0x48, /* 0x24-0x27 */ + 0xF1, 0x4A, 0xE8, 0x56, 0xE8, 0x57, 0xE8, 0x55, /* 0x28-0x2B */ + 0xDC, 0x51, 0xBE, 0x47, 0xE8, 0x5A, 0xE8, 0x54, /* 0x2C-0x2F */ + 0xBE, 0x46, 0xBE, 0x49, 0xE8, 0x58, 0xEB, 0xD5, /* 0x30-0x33 */ + 0xBF, 0xF3, 0xEB, 0xD6, 0xEB, 0xD7, 0x00, 0x00, /* 0x34-0x37 */ + 0xEE, 0xC4, 0xC1, 0xDD, 0xF1, 0x4B, 0xF1, 0x4C, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0xF1, 0x4D, 0xF3, 0x5D, /* 0x3C-0x3F */ + 0xF3, 0x5C, 0xF4, 0xE2, 0x00, 0x00, 0xF4, 0xE1, /* 0x40-0x43 */ + 0xF6, 0x5B, 0xF6, 0x5C, 0xF6, 0x5A, 0xF7, 0x66, /* 0x44-0x47 */ + 0xC5, 0xB0, 0xA8, 0xBB, 0xAD, 0xAA, 0xAD, 0xA9, /* 0x48-0x4B */ + 0xB0, 0x75, 0xB0, 0x74, 0xD4, 0x40, 0xD4, 0x41, /* 0x4C-0x4F */ + 0xD3, 0xFE, 0x00, 0x00, 0xB0, 0x73, 0xD7, 0xF5, /* 0x50-0x53 */ + 0x00, 0x00, 0xD7, 0xF6, 0xD7, 0xF2, 0xB3, 0xA4, /* 0x54-0x57 */ + 0xD7, 0xF3, 0x00, 0x00, 0xD7, 0xF4, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xDC, 0x5F, /* 0x5C-0x5F */ + 0xDC, 0x61, 0xDC, 0x5D, 0xDC, 0x60, 0xB6, 0x6F, /* 0x60-0x63 */ + 0xDC, 0x5E, 0xB6, 0x70, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0xDD, 0x73, 0xB9, 0x55, 0xB9, 0x54, 0x00, 0x00, /* 0x68-0x6B */ + 0xB9, 0x53, 0x00, 0x00, 0xE0, 0xAC, 0xE0, 0xAD, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0x73, 0xE4, 0x75, /* 0x70-0x73 */ + 0xBB, 0xC6, 0xBB, 0xC3, 0x00, 0x00, 0xBB, 0xC5, /* 0x74-0x77 */ + 0xBB, 0xC4, 0xE4, 0x74, 0xE4, 0x72, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xE8, 0x61, 0xE8, 0x5E, 0xE8, 0x5F, 0xBE, 0x4D, /* 0x80-0x83 */ + 0xE8, 0x60, 0xE8, 0x5B, 0xE8, 0x5C, 0xBE, 0x4A, /* 0x84-0x87 */ + 0x00, 0x00, 0xBE, 0x4B, 0xE8, 0x5D, 0xBE, 0x4C, /* 0x88-0x8B */ + 0x00, 0x00, 0xEB, 0xDB, 0x00, 0x00, 0xEB, 0xDC, /* 0x8C-0x8F */ + 0xEB, 0xD9, 0xEB, 0xDA, 0xBF, 0xF4, 0xEB, 0xD8, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0xEE, 0xC8, 0xEE, 0xC5, 0xEE, 0xC7, /* 0x98-0x9B */ + 0xC1, 0xE0, 0xEE, 0xCB, 0xC1, 0xDF, 0xEE, 0xC9, /* 0x9C-0x9F */ + 0xEE, 0xCC, 0xEE, 0xCA, 0xEE, 0xC6, 0xC1, 0xDE, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xF1, 0x4F, 0x00, 0x00, 0xF1, 0x50, /* 0xA4-0xA7 */ + 0xF1, 0x4E, 0x00, 0x00, 0xF1, 0x52, 0xC2, 0xE5, /* 0xA8-0xAB */ + 0xC2, 0xE6, 0xF3, 0x5F, 0xC3, 0xE7, 0xF1, 0x51, /* 0xAC-0xAF */ + 0xF3, 0x5E, 0xC3, 0xE6, 0xF4, 0xE5, 0xF4, 0xE6, /* 0xB0-0xB3 */ + 0xC4, 0xBF, 0xF4, 0xE4, 0x00, 0x00, 0xF4, 0xE3, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xF6, 0x5D, 0xC5, 0x48, 0x00, 0x00, /* 0xB8-0xBB */ + 0xF8, 0x49, 0xF8, 0xC8, 0xF8, 0xC7, 0x00, 0x00, /* 0xBC-0xBF */ + 0xC6, 0x43, 0xC6, 0x5D, 0xF8, 0xC9, 0xF9, 0x71, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xC6, 0x6F, 0xA8, 0xBC, 0xAA, 0xF6, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xB9, 0x56, 0x00, 0x00, 0xC4, 0xC0, /* 0xC8-0xCB */ + 0xA8, 0xBD, 0xAD, 0xAB, 0xB3, 0xA5, 0xB6, 0x71, /* 0xCC-0xCF */ + 0xC2, 0xE7, 0xAA, 0xF7, 0x00, 0x00, 0xD0, 0xC1, /* 0xD0-0xD3 */ + 0xD0, 0xC0, 0xD4, 0x42, 0x00, 0x00, 0xB0, 0x78, /* 0xD4-0xD7 */ + 0xB0, 0x76, 0xB0, 0x7A, 0xD4, 0x44, 0x00, 0x00, /* 0xD8-0xDB */ + 0xB0, 0x79, 0xB0, 0x77, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0xD4, 0x43, 0xB3, 0xA8, /* 0xE0-0xE3 */ + 0xD7, 0xFC, 0x00, 0x00, 0xB3, 0xA7, 0xB3, 0xA9, /* 0xE4-0xE7 */ + 0xD8, 0x42, 0xB3, 0xAB, 0xD7, 0xFE, 0xD8, 0x40, /* 0xE8-0xEB */ + 0xD7, 0xF7, 0xB3, 0xAA, 0xD8, 0x43, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0xD7, 0xF9, 0x00, 0x00, 0xD7, 0xFA, /* 0xF0-0xF3 */ + 0xD7, 0xF8, 0xB3, 0xA6, 0x00, 0x00, 0xD8, 0x41, /* 0xF4-0xF7 */ + 0xD7, 0xFB, 0xD7, 0xFD, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0xDC, 0x6D, 0x00, 0x00, 0xDC, 0x6C, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_92[512] = { + 0xDC, 0x6A, 0xDC, 0x62, 0xDC, 0x71, 0xDC, 0x65, /* 0x00-0x03 */ + 0xDC, 0x6F, 0xDC, 0x76, 0xDC, 0x6E, 0xB6, 0x79, /* 0x04-0x07 */ + 0x00, 0x00, 0xB6, 0x75, 0xDC, 0x63, 0x00, 0x00, /* 0x08-0x0B */ + 0xDC, 0x69, 0xB6, 0x77, 0x00, 0x00, 0xDC, 0x68, /* 0x0C-0x0F */ + 0xB6, 0x78, 0xB6, 0x7A, 0xDC, 0x6B, 0x00, 0x00, /* 0x10-0x13 */ + 0xB6, 0x72, 0xB6, 0x73, 0xDC, 0x77, 0xDC, 0x75, /* 0x14-0x17 */ + 0x00, 0x00, 0xDC, 0x74, 0xDC, 0x66, 0x00, 0x00, /* 0x18-0x1B */ + 0xDC, 0x72, 0x00, 0x00, 0xB6, 0x76, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB6, 0x74, /* 0x20-0x23 */ + 0xDC, 0x73, 0xDC, 0x64, 0xDC, 0x67, 0xDC, 0x70, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0xE4, 0xBA, 0xE0, 0xB7, 0x00, 0x00, /* 0x2C-0x2F */ + 0xE0, 0xB0, 0xE0, 0xC3, 0xE0, 0xCC, 0xE0, 0xB3, /* 0x30-0x33 */ + 0xB9, 0x61, 0x00, 0x00, 0xE0, 0xC0, 0xB9, 0x57, /* 0x34-0x37 */ + 0xB9, 0x59, 0xB9, 0x65, 0xE0, 0xB1, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0xB9, 0x5A, 0xB9, 0x5C, 0xB9, 0x66, /* 0x3C-0x3F */ + 0xB9, 0x5B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0xB9, 0x64, 0xE0, 0xB9, 0x00, 0x00, /* 0x44-0x47 */ + 0xE0, 0xAE, 0xB9, 0x62, 0xE0, 0xB8, 0xB9, 0x5E, /* 0x48-0x4B */ + 0xE0, 0xCA, 0xB9, 0x63, 0xE0, 0xC8, 0xE0, 0xBC, /* 0x4C-0x4F */ + 0xE0, 0xC6, 0xB9, 0x60, 0xE0, 0xAF, 0xE0, 0xC9, /* 0x50-0x53 */ + 0xE0, 0xC4, 0x00, 0x00, 0xE0, 0xCB, 0xB9, 0x58, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0xB9, 0x67, 0xB9, 0x5D, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0xE0, 0xB5, 0x00, 0x00, /* 0x5C-0x5F */ + 0xE0, 0xBD, 0xE0, 0xC1, 0x00, 0x00, 0xE0, 0xC5, /* 0x60-0x63 */ + 0xB9, 0x5F, 0xE0, 0xB4, 0xE0, 0xB2, 0xE0, 0xBE, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0xE0, 0xBB, 0xE0, 0xBA, 0x00, 0x00, 0xE0, 0xBF, /* 0x6C-0x6F */ + 0xE0, 0xC2, 0x00, 0x00, 0xE0, 0xC7, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0x78, 0x00, 0x00, /* 0x74-0x77 */ + 0xBB, 0xC7, 0xE4, 0xA4, 0xE4, 0x7A, 0xBB, 0xCC, /* 0x78-0x7B */ + 0xBB, 0xD0, 0xE4, 0xAD, 0xE4, 0xB5, 0xE4, 0xA6, /* 0x7C-0x7F */ + + 0xBB, 0xC8, 0x00, 0x00, 0xE4, 0xAA, 0xE0, 0xB6, /* 0x80-0x83 */ + 0x00, 0x00, 0xBB, 0xC9, 0xE4, 0xB1, 0xE4, 0xB6, /* 0x84-0x87 */ + 0xE4, 0xAE, 0x00, 0x00, 0xE4, 0xB0, 0xE4, 0xB9, /* 0x88-0x8B */ + 0xE4, 0xB2, 0xE4, 0x7E, 0xE4, 0xA9, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xBB, 0xD1, 0x00, 0x00, 0xBB, 0xCD, /* 0x90-0x93 */ + 0xE4, 0x7C, 0xE4, 0xAB, 0xBB, 0xCB, 0xE4, 0xA5, /* 0x94-0x97 */ + 0xBB, 0xCA, 0xE4, 0xB3, 0xE4, 0xA2, 0xE4, 0x79, /* 0x98-0x9B */ + 0xBB, 0xCE, 0xE4, 0xB8, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0xE4, 0x7B, 0xE4, 0xAF, 0xE4, 0xAC, 0xE4, 0xA7, /* 0xA0-0xA3 */ + 0xE4, 0x77, 0xE4, 0x76, 0xE4, 0xA1, 0xE4, 0xB4, /* 0xA4-0xA7 */ + 0xBB, 0xCF, 0xE4, 0xB7, 0xE4, 0x7D, 0xE4, 0xA3, /* 0xA8-0xAB */ + 0xBE, 0x52, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0xBE, 0x5A, 0xBE, 0x55, /* 0xB0-0xB3 */ + 0xE8, 0xA4, 0xE8, 0xA1, 0xE8, 0x67, 0xBE, 0x50, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xF9, 0xD7, 0x00, 0x00, 0xBE, 0x4F, /* 0xB8-0xBB */ + 0xBE, 0x56, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0xE8, 0x65, 0xBE, 0x54, 0xE8, 0x71, 0xE8, 0x63, /* 0xC0-0xC3 */ + 0xE8, 0x64, 0xBE, 0x4E, 0xE8, 0xA3, 0xBE, 0x58, /* 0xC4-0xC7 */ + 0xE8, 0x74, 0xE8, 0x79, 0xE8, 0x73, 0xEB, 0xEE, /* 0xC8-0xCB */ + 0xE8, 0x6F, 0xE8, 0x77, 0xE8, 0x75, 0xE8, 0x68, /* 0xCC-0xCF */ + 0xE8, 0x62, 0xE8, 0x7D, 0xBE, 0x57, 0xE8, 0x7E, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xE8, 0x78, 0x00, 0x00, 0xE8, 0x6D, /* 0xD4-0xD7 */ + 0xE8, 0x6B, 0xE8, 0x66, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0xE8, 0x6E, 0xE8, 0x7B, 0xE8, 0x6A, /* 0xDC-0xDF */ + 0xE8, 0x7A, 0xE8, 0xA2, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0xBE, 0x53, 0x00, 0x00, 0xE8, 0x76, 0xE8, 0x7C, /* 0xE4-0xE7 */ + 0xE8, 0x72, 0xE8, 0x6C, 0xBE, 0x51, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0xA8, 0xE8, 0x70, /* 0xEC-0xEF */ + 0xBE, 0x59, 0xE8, 0x69, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEB, 0xF4, /* 0xF4-0xF7 */ + 0xBF, 0xF7, 0xEB, 0xF3, 0xEB, 0xF0, 0xEC, 0x44, /* 0xF8-0xFB */ + 0xBF, 0xFB, 0x00, 0x00, 0xEC, 0x41, 0xEB, 0xF8, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_93[512] = { + 0xEC, 0x43, 0xEB, 0xE9, 0xEB, 0xF6, 0x00, 0x00, /* 0x00-0x03 */ + 0xBF, 0xFD, 0x00, 0x00, 0xEB, 0xE1, 0x00, 0x00, /* 0x04-0x07 */ + 0xEB, 0xDF, 0xEC, 0x42, 0x00, 0x00, 0xEC, 0x40, /* 0x08-0x0B */ + 0xEB, 0xFE, 0xEB, 0xED, 0xEB, 0xEC, 0xEB, 0xE2, /* 0x0C-0x0F */ + 0xC0, 0x40, 0x00, 0x00, 0xEB, 0xE8, 0xEB, 0xF2, /* 0x10-0x13 */ + 0xEB, 0xFD, 0xC0, 0x43, 0xEC, 0x45, 0x00, 0x00, /* 0x14-0x17 */ + 0xC1, 0xE8, 0xC0, 0x45, 0xBF, 0xFE, 0xEB, 0xE6, /* 0x18-0x1B */ + 0x00, 0x00, 0xEB, 0xEF, 0xEB, 0xDE, 0xEB, 0xE0, /* 0x1C-0x1F */ + 0xBF, 0xF5, 0xC0, 0x42, 0xBF, 0xFA, 0xEB, 0xE7, /* 0x20-0x23 */ + 0xEB, 0xF7, 0xEB, 0xF1, 0xC0, 0x41, 0xEB, 0xDD, /* 0x24-0x27 */ + 0xC1, 0xE3, 0xEB, 0xF9, 0xEB, 0xFC, 0xBF, 0xFC, /* 0x28-0x2B */ + 0x00, 0x00, 0xEB, 0xEB, 0xC0, 0x44, 0xBF, 0xF9, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xBF, 0xF8, /* 0x30-0x33 */ + 0xEB, 0xF5, 0xEB, 0xFB, 0xBF, 0xF6, 0x00, 0x00, /* 0x34-0x37 */ + 0xEB, 0xE4, 0xEB, 0xFA, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0xEB, 0xE5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xEB, 0xEA, 0xEE, 0xD2, /* 0x44-0x47 */ + 0x00, 0x00, 0xEE, 0xD7, 0xC1, 0xE5, 0xC1, 0xE7, /* 0x48-0x4B */ + 0xEE, 0xDD, 0xC1, 0xE1, 0xEE, 0xEC, 0xEE, 0xE3, /* 0x4C-0x4F */ + 0xEE, 0xD8, 0xEE, 0xD9, 0xEE, 0xE2, 0x00, 0x00, /* 0x50-0x53 */ + 0xC1, 0xEE, 0xEE, 0xE1, 0xEE, 0xD1, 0xEE, 0xE0, /* 0x54-0x57 */ + 0xEE, 0xD4, 0xEE, 0xED, 0xC1, 0xED, 0xC1, 0xEB, /* 0x58-0x5B */ + 0xEE, 0xD5, 0x00, 0x00, 0xEE, 0xE8, 0x00, 0x00, /* 0x5C-0x5F */ + 0xEE, 0xDA, 0xEE, 0xE7, 0x00, 0x00, 0xEE, 0xE9, /* 0x60-0x63 */ + 0xEE, 0xD0, 0xC1, 0xE6, 0x00, 0x00, 0xEE, 0xEA, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0xEE, 0xDE, 0x00, 0x00, /* 0x68-0x6B */ + 0xC1, 0xEA, 0xEE, 0xDB, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0xC1, 0xEC, 0xEE, 0xE4, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0xC1, 0xE4, 0xEE, 0xD6, 0xEE, 0xE5, /* 0x74-0x77 */ + 0x00, 0x00, 0xEE, 0xDF, 0xEB, 0xE3, 0xEE, 0xE6, /* 0x78-0x7B */ + 0xEE, 0xD3, 0x00, 0x00, 0xC1, 0xE9, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xEE, 0xEB, 0x00, 0x00, 0xC1, 0xE2, 0xEE, 0xCE, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0xF1, 0x60, 0xF1, 0x59, 0xC2, 0xE9, 0x00, 0x00, /* 0x88-0x8B */ + 0xF1, 0x54, 0xF1, 0x63, 0xF1, 0x5B, 0xEE, 0xDC, /* 0x8C-0x8F */ + 0x00, 0x00, 0xF1, 0x65, 0xF1, 0x55, 0x00, 0x00, /* 0x90-0x93 */ + 0xC2, 0xE8, 0xF1, 0x5F, 0xC2, 0xEA, 0xC2, 0xF2, /* 0x94-0x97 */ + 0xC2, 0xF0, 0xF1, 0x61, 0xC2, 0xF1, 0xF1, 0x57, /* 0x98-0x9B */ + 0x00, 0x00, 0xF1, 0x58, 0xF1, 0x5D, 0xF1, 0x62, /* 0x9C-0x9F */ + 0x00, 0x00, 0xEE, 0xCD, 0xC2, 0xEB, 0xF1, 0x6A, /* 0xA0-0xA3 */ + 0xF1, 0x67, 0xF1, 0x6B, 0xF1, 0x5E, 0xF1, 0x5A, /* 0xA4-0xA7 */ + 0xF1, 0x68, 0xF3, 0x6A, 0xF1, 0x5C, 0x00, 0x00, /* 0xA8-0xAB */ + 0xC2, 0xEE, 0x00, 0x00, 0xC2, 0xED, 0xEE, 0xCF, /* 0xAC-0xAF */ + 0xC2, 0xEF, 0xF1, 0x64, 0xF1, 0x66, 0xC2, 0xEC, /* 0xB0-0xB3 */ + 0xF1, 0x69, 0xF1, 0x53, 0x00, 0x00, 0xF1, 0x56, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0xF3, 0x73, 0x00, 0x00, 0xF3, 0x63, 0xC3, 0xEB, /* 0xC0-0xC3 */ + 0xF3, 0x71, 0x00, 0x00, 0x00, 0x00, 0xF3, 0x61, /* 0xC4-0xC7 */ + 0xC3, 0xEC, 0x00, 0x00, 0xF3, 0x6C, 0x00, 0x00, /* 0xC8-0xCB */ + 0xF3, 0x68, 0xC3, 0xF1, 0xF3, 0x72, 0xF3, 0x62, /* 0xCC-0xCF */ + 0xF3, 0x65, 0xC3, 0xE9, 0xF3, 0x74, 0x00, 0x00, /* 0xD0-0xD3 */ + 0xF3, 0x6D, 0xF3, 0x70, 0xC3, 0xEF, 0xC3, 0xF4, /* 0xD4-0xD7 */ + 0xC3, 0xF2, 0xF3, 0x69, 0xF3, 0x64, 0x00, 0x00, /* 0xD8-0xDB */ + 0xC3, 0xED, 0xC3, 0xEE, 0xF3, 0x60, 0xC3, 0xEA, /* 0xDC-0xDF */ + 0x00, 0x00, 0xC3, 0xE8, 0xC3, 0xF0, 0xF3, 0x6F, /* 0xE0-0xE3 */ + 0xC3, 0xF3, 0x00, 0x00, 0xF3, 0x6B, 0xF3, 0x75, /* 0xE4-0xE7 */ + 0xC3, 0xF5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0xF3, 0x67, 0x00, 0x00, 0xF3, 0x6E, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0xF4, 0xF3, 0xF5, 0x42, 0xF4, 0xF5, /* 0xF4-0xF7 */ + 0xF4, 0xFC, 0xF3, 0x66, 0xF4, 0xFA, 0xF4, 0xE9, /* 0xF8-0xFB */ + 0xF5, 0x40, 0xC4, 0xC3, 0xF4, 0xED, 0xF4, 0xFE, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_94[512] = { + 0xF4, 0xF4, 0x00, 0x00, 0x00, 0x00, 0xC4, 0xC2, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0xF5, 0x44, 0xF4, 0xF6, /* 0x04-0x07 */ + 0x00, 0x00, 0xF4, 0xFB, 0xF4, 0xFD, 0xF4, 0xE7, /* 0x08-0x0B */ + 0xF5, 0x41, 0xF4, 0xF2, 0xF4, 0xF7, 0xF4, 0xEB, /* 0x0C-0x0F */ + 0xF4, 0xEF, 0xF5, 0x43, 0xF4, 0xF9, 0xF4, 0xE8, /* 0x10-0x13 */ + 0xF4, 0xEC, 0xF4, 0xEE, 0xF4, 0xF8, 0x00, 0x00, /* 0x14-0x17 */ + 0xC4, 0xC1, 0xF4, 0xF1, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0xF4, 0xEA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0xF4, 0xF0, 0xF6, 0x61, 0xF6, 0x66, 0xC5, 0x4F, /* 0x28-0x2B */ + 0xF6, 0x68, 0x00, 0x00, 0xC5, 0x49, 0x00, 0x00, /* 0x2C-0x2F */ + 0xF6, 0x64, 0xF6, 0x6A, 0xC5, 0x4E, 0xC5, 0x4A, /* 0x30-0x33 */ + 0x00, 0x00, 0xC5, 0x4B, 0xF6, 0x60, 0xF6, 0x67, /* 0x34-0x37 */ + 0xC5, 0x4D, 0xF6, 0x65, 0xC5, 0x4C, 0xF6, 0x5F, /* 0x38-0x3B */ + 0xF6, 0x63, 0xF6, 0x62, 0x00, 0x00, 0xF6, 0x5E, /* 0x3C-0x3F */ + 0xF6, 0x69, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0xC5, 0xB1, 0xF7, 0x6D, 0xF7, 0x70, 0xF7, 0x6C, /* 0x44-0x47 */ + 0xF7, 0x6E, 0xF7, 0x6F, 0xF7, 0x69, 0xF7, 0x6A, /* 0x48-0x4B */ + 0xF7, 0x67, 0x00, 0x00, 0x00, 0x00, 0xF7, 0x6B, /* 0x4C-0x4F */ + 0xF7, 0x68, 0xC5, 0xB2, 0xC5, 0xB3, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0xF8, 0x4B, 0x00, 0x00, 0xF8, 0x4D, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0xF8, 0x4C, 0xF8, 0x4E, 0x00, 0x00, /* 0x5C-0x5F */ + 0xC5, 0xE0, 0x00, 0x00, 0xF8, 0x4A, 0xC5, 0xDF, /* 0x60-0x63 */ + 0xC5, 0xE1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0xF8, 0xCB, 0xF8, 0xCC, 0xC6, 0x44, 0xF8, 0xCA, /* 0x68-0x6B */ + 0x00, 0x00, 0xF9, 0x53, 0xF9, 0x52, 0xF9, 0x54, /* 0x6C-0x6F */ + 0xC6, 0x5F, 0xF9, 0x55, 0xC6, 0x5E, 0xF9, 0x56, /* 0x70-0x73 */ + 0xF9, 0x72, 0xF9, 0x75, 0xF9, 0x74, 0xC6, 0x68, /* 0x74-0x77 */ + 0xF9, 0x73, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0xC6, 0x72, 0xC6, 0x70, 0xC6, 0x71, 0xC6, 0x77, /* 0x7C-0x7F */ + + 0xF9, 0xC0, 0xF9, 0xC1, 0xF9, 0xBF, 0xF9, 0xC9, /* 0x80-0x83 */ +}; + +static const unsigned char u2c_95[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAA, 0xF8, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0xD8, 0x44, 0xDC, 0x78, /* 0x78-0x7B */ + 0xE8, 0xA5, 0xF3, 0x76, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xAA, 0xF9, 0x00, 0x00, 0xAD, 0xAC, 0xB0, 0x7B, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0xD8, 0x45, 0x00, 0x00, /* 0x84-0x87 */ + 0xD8, 0x46, 0xB3, 0xAC, 0x00, 0x00, 0xB6, 0x7D, /* 0x88-0x8B */ + 0xDC, 0x7A, 0xDC, 0x79, 0xB6, 0xA3, 0xB6, 0x7C, /* 0x8C-0x8F */ + 0xDC, 0x7B, 0xB6, 0x7E, 0xB6, 0xA2, 0xB6, 0xA1, /* 0x90-0x93 */ + 0xB6, 0x7B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0xB9, 0x68, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xD0, /* 0x98-0x9B */ + 0xE0, 0xCE, 0x00, 0x00, 0xE0, 0xCF, 0xE0, 0xCD, /* 0x9C-0x9F */ + 0x00, 0x00, 0xBB, 0xD2, 0x00, 0x00, 0xBB, 0xD5, /* 0xA0-0xA3 */ + 0xBB, 0xD7, 0xBB, 0xD6, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xBB, 0xD3, 0xBB, 0xD4, 0x00, 0x00, 0xE8, 0xA7, /* 0xA8-0xAB */ + 0xE8, 0xA6, 0xBE, 0x5B, 0xE8, 0xA8, 0x00, 0x00, /* 0xAC-0xAF */ + 0xE8, 0xA9, 0xBE, 0x5C, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xEC, 0x4D, 0xEC, 0x4B, 0xEE, 0xF3, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xEC, 0x49, 0xEC, 0x4A, 0xC0, 0x46, /* 0xB8-0xBB */ + 0xEC, 0x46, 0xEC, 0x4E, 0xEC, 0x48, 0xEC, 0x4C, /* 0xBC-0xBF */ + 0xEE, 0xEF, 0x00, 0x00, 0x00, 0x00, 0xEE, 0xF1, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xEE, 0xF2, 0xC1, 0xF3, 0xEE, 0xEE, /* 0xC4-0xC7 */ + 0xC1, 0xF2, 0xEE, 0xF0, 0xC1, 0xEF, 0xC1, 0xF0, /* 0xC8-0xCB */ + 0xC1, 0xF1, 0xEC, 0x47, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0xC2, 0xF5, 0xF1, 0x6E, 0xF1, 0x6C, 0xF1, 0x6D, /* 0xD0-0xD3 */ + 0xC2, 0xF3, 0xC2, 0xF6, 0xC2, 0xF4, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0xF3, 0x77, 0xF3, 0x78, /* 0xD8-0xDB */ + 0xC3, 0xF6, 0x00, 0x00, 0xF5, 0x45, 0xF5, 0x47, /* 0xDC-0xDF */ + 0xF5, 0x46, 0xC4, 0xC4, 0xC5, 0x50, 0xF6, 0x6D, /* 0xE0-0xE3 */ + 0xF6, 0x6C, 0xF6, 0x6B, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ +}; + +static const unsigned char u2c_96[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0xAA, 0xFA, 0x00, 0x00, 0xC9, 0xAA, 0x00, 0x00, /* 0x1C-0x1F */ + 0xCA, 0x58, 0xA6, 0xE9, 0xCA, 0x56, 0xCA, 0x59, /* 0x20-0x23 */ + 0xCA, 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0xCB, 0xAE, 0x00, 0x00, 0xA8, 0xC1, 0x00, 0x00, /* 0x28-0x2B */ + 0xA8, 0xC2, 0xCB, 0xB0, 0xA8, 0xBF, 0xCB, 0xAF, /* 0x2C-0x2F */ + 0xCB, 0xAD, 0xA8, 0xC0, 0xA8, 0xBE, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0xCD, 0xD8, 0xCD, 0xDB, 0xAA, 0xFD, /* 0x38-0x3B */ + 0xCD, 0xDA, 0xCD, 0xD9, 0x00, 0x00, 0xAA, 0xFC, /* 0x3C-0x3F */ + 0xAA, 0xFB, 0x00, 0x00, 0xAB, 0x40, 0xCD, 0xDC, /* 0x40-0x43 */ + 0xAA, 0xFE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0xD0, 0xC6, 0xAD, 0xAE, /* 0x48-0x4B */ + 0xAD, 0xAF, 0xAD, 0xB0, 0xD0, 0xC7, 0xD0, 0xC3, /* 0x4C-0x4F */ + 0xAD, 0xAD, 0xD0, 0xC4, 0x00, 0x00, 0xD0, 0xC5, /* 0x50-0x53 */ + 0xD0, 0xC2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0xB0, 0xA4, 0x00, 0x00, 0x00, 0x00, 0xB0, 0xA1, /* 0x58-0x5B */ + 0xD4, 0x45, 0xB0, 0xA2, 0xB0, 0xA5, 0xD4, 0x46, /* 0x5C-0x5F */ + 0x00, 0x00, 0xB0, 0x7E, 0xB0, 0x7C, 0xB0, 0x7D, /* 0x60-0x63 */ + 0xB0, 0xA3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0xB3, 0xAD, 0xD8, 0x49, /* 0x68-0x6B */ + 0xB3, 0xB5, 0xD8, 0x48, 0x00, 0x00, 0xD8, 0x4B, /* 0x6C-0x6F */ + 0xB3, 0xB1, 0xD8, 0x4A, 0xB6, 0xAB, 0xB3, 0xAF, /* 0x70-0x73 */ + 0xB3, 0xB2, 0xB3, 0xAE, 0xB3, 0xB3, 0xB3, 0xB4, /* 0x74-0x77 */ + 0xB3, 0xB0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0xD8, 0x47, 0xB6, 0xA7, 0xDC, 0x7D, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xDC, 0xA3, 0x00, 0x00, 0x00, 0x00, 0xDC, 0xA2, /* 0x80-0x83 */ + 0xB6, 0xAC, 0xB6, 0xA8, 0xB6, 0xA9, 0xDC, 0x7C, /* 0x84-0x87 */ + 0xDC, 0x7E, 0xDC, 0xA1, 0xB6, 0xA4, 0xB6, 0xA6, /* 0x88-0x8B */ + 0x00, 0x00, 0xB6, 0xAA, 0xB6, 0xA5, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xE0, 0xD3, 0xE0, 0xD1, 0xE0, 0xD2, /* 0x90-0x93 */ + 0xB9, 0x6A, 0xB9, 0x6B, 0x00, 0x00, 0xE0, 0xD4, /* 0x94-0x97 */ + 0xB9, 0x69, 0xBB, 0xD8, 0x00, 0x00, 0xBB, 0xDA, /* 0x98-0x9B */ + 0xBB, 0xD9, 0x00, 0x00, 0xE4, 0xBB, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0xE4, 0xBC, 0xE8, 0xAB, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xE8, 0xAA, 0x00, 0x00, 0x00, 0x00, 0xC0, 0x47, /* 0xA4-0xA7 */ + 0xC0, 0x48, 0xEC, 0x4F, 0xC0, 0x49, 0x00, 0x00, /* 0xA8-0xAB */ + 0xEE, 0xF6, 0x00, 0x00, 0xEE, 0xF4, 0x00, 0x00, /* 0xAC-0xAF */ + 0xEE, 0xF5, 0xC1, 0xF4, 0x00, 0x00, 0xF1, 0x6F, /* 0xB0-0xB3 */ + 0xC3, 0xF7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xC1, 0xF5, 0xAB, 0x41, 0x00, 0x00, 0xB0, 0xA6, /* 0xB8-0xBB */ + 0xD4, 0x47, 0x00, 0x00, 0x00, 0x00, 0xD8, 0x4C, /* 0xBC-0xBF */ + 0xB3, 0xB6, 0xB6, 0xAD, 0xDC, 0xA4, 0xDC, 0xA6, /* 0xC0-0xC3 */ + 0xB6, 0xAF, 0xB6, 0xAE, 0xB6, 0xB0, 0xB6, 0xB1, /* 0xC4-0xC7 */ + 0xDC, 0xA5, 0xB9, 0x6E, 0xB9, 0x6F, 0xB9, 0x6D, /* 0xC8-0xCB */ + 0xBB, 0xDB, 0xB9, 0x6C, 0xE0, 0xD5, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0xBB, 0xDC, 0xE8, 0xAC, /* 0xD0-0xD3 */ + 0xEC, 0x50, 0xC0, 0x4A, 0xC1, 0xF6, 0xF1, 0x70, /* 0xD4-0xD7 */ + 0xF1, 0x74, 0xC2, 0xF9, 0xF1, 0x71, 0xC2, 0xFA, /* 0xD8-0xDB */ + 0xC2, 0xF8, 0xF1, 0x75, 0xC2, 0xFB, 0xF1, 0x73, /* 0xDC-0xDF */ + 0x00, 0x00, 0xF3, 0x79, 0xC2, 0xF7, 0xC3, 0xF8, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xF8, 0xCD, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xAB, 0x42, 0xB3, 0xB8, 0xB3, 0xB7, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB6, 0xB2, /* 0xEC-0xEF */ + 0xDC, 0xA8, 0xDC, 0xA7, 0xB6, 0xB3, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0xE0, 0xD9, 0xB9, 0x73, 0xB9, 0x70, /* 0xF4-0xF7 */ + 0xE0, 0xD8, 0xB9, 0x72, 0xE0, 0xD6, 0xB9, 0x71, /* 0xF8-0xFB */ + 0x00, 0x00, 0xE0, 0xD7, 0x00, 0x00, 0xE4, 0xBD, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_97[512] = { + 0xBB, 0xDD, 0x00, 0x00, 0xE8, 0xAF, 0x00, 0x00, /* 0x00-0x03 */ + 0xBE, 0x5D, 0xE8, 0xAD, 0xBE, 0x5E, 0xBE, 0x5F, /* 0x04-0x07 */ + 0xE8, 0xAE, 0xBE, 0x60, 0x00, 0x00, 0xEC, 0x51, /* 0x08-0x0B */ + 0x00, 0x00, 0xC0, 0x4E, 0xC0, 0x4B, 0xC0, 0x50, /* 0x0C-0x0F */ + 0xEC, 0x53, 0xC0, 0x4C, 0xEC, 0x52, 0xC0, 0x4F, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xC0, 0x4D, 0x00, 0x00, /* 0x14-0x17 */ + 0xEE, 0xF9, 0xEE, 0xFB, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0xC1, 0xF7, 0xEE, 0xFA, 0xC1, 0xF8, 0xEE, 0xF8, /* 0x1C-0x1F */ + 0xEE, 0xF7, 0x00, 0x00, 0xF1, 0x77, 0xF1, 0x76, /* 0x20-0x23 */ + 0xC2, 0xFC, 0xF1, 0x78, 0xF3, 0x7E, 0xC3, 0xFA, /* 0x24-0x27 */ + 0xF3, 0x7D, 0xF3, 0x7A, 0xC3, 0xF9, 0xF3, 0x7B, /* 0x28-0x2B */ + 0xF3, 0x7C, 0x00, 0x00, 0xF5, 0x48, 0xF5, 0x49, /* 0x2C-0x2F */ + 0xC4, 0xC5, 0x00, 0x00, 0xC5, 0x53, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0xF6, 0x6E, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0xC5, 0x51, 0xC5, 0x52, 0xF6, 0x6F, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0xC5, 0xB4, 0xC5, 0xB5, 0xF7, 0x71, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0xC6, 0x45, 0xF8, 0xCF, /* 0x40-0x43 */ + 0xC6, 0x47, 0x00, 0x00, 0xF8, 0xCE, 0xF8, 0xD0, /* 0x44-0x47 */ + 0xC6, 0x46, 0xF9, 0x57, 0x00, 0x00, 0xF9, 0xAD, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0xAB, 0x43, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0xB9, 0x74, 0x00, 0x00, /* 0x54-0x57 */ + 0xE4, 0xBE, 0x00, 0x00, 0xE8, 0xB0, 0xC0, 0x51, /* 0x58-0x5B */ + 0xC0, 0x52, 0x00, 0x00, 0xAB, 0x44, 0x00, 0x00, /* 0x5C-0x5F */ + 0xBE, 0x61, 0xC3, 0xFB, 0xAD, 0xB1, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0xC0, 0x53, 0x00, 0x00, /* 0x64-0x67 */ + 0xC5, 0xE2, 0xAD, 0xB2, 0xD8, 0x4D, 0x00, 0x00, /* 0x68-0x6B */ + 0xDC, 0xA9, 0x00, 0x00, 0xDC, 0xAB, 0x00, 0x00, /* 0x6C-0x6F */ + 0xDC, 0xAA, 0x00, 0x00, 0xE0, 0xDD, 0xE0, 0xDA, /* 0x70-0x73 */ + 0xB9, 0x75, 0x00, 0x00, 0xB9, 0x76, 0xE0, 0xDB, /* 0x74-0x77 */ + 0xE0, 0xDC, 0x00, 0x00, 0xE4, 0xC0, 0xE4, 0xC5, /* 0x78-0x7B */ + 0xBB, 0xDE, 0xE4, 0xBF, 0xE4, 0xC1, 0xE4, 0xC8, /* 0x7C-0x7F */ + + 0xE4, 0xC3, 0xE4, 0xC7, 0xE4, 0xC4, 0xE4, 0xC2, /* 0x80-0x83 */ + 0xE4, 0xC6, 0xBB, 0xDF, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0xE8, 0xB3, 0x00, 0x00, 0xE8, 0xB1, 0xBE, 0x63, /* 0x88-0x8B */ + 0x00, 0x00, 0xBE, 0x62, 0xE8, 0xB2, 0xBE, 0x64, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0xEC, 0x56, 0x00, 0x00, 0x00, 0x00, 0xEC, 0x55, /* 0x94-0x97 */ + 0xC0, 0x54, 0xEC, 0x54, 0xEE, 0xFC, 0x00, 0x00, /* 0x98-0x9B */ + 0xEE, 0xFE, 0xEF, 0x41, 0xEF, 0x40, 0x00, 0x00, /* 0x9C-0x9F */ + 0xC1, 0xF9, 0xEE, 0xFD, 0xF1, 0xA1, 0xC2, 0xFD, /* 0xA0-0xA3 */ + 0xF1, 0x7D, 0xF1, 0xA2, 0xC2, 0xFE, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xF1, 0x7B, 0x00, 0x00, 0xF1, 0x7E, 0xF1, 0x7C, /* 0xA8-0xAB */ + 0xF1, 0x79, 0xC3, 0x40, 0xF1, 0x7A, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF3, 0xA1, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0xF3, 0xA3, 0xF3, 0xA2, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xF5, 0x4A, 0x00, 0x00, 0xF5, 0x4B, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF6, 0x70, /* 0xBC-0xBF */ + 0x00, 0x00, 0xC5, 0xB7, 0x00, 0x00, 0xC5, 0xB6, /* 0xC0-0xC3 */ + 0xF8, 0x4F, 0xF8, 0x50, 0xC6, 0x48, 0xF8, 0xD1, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xC6, 0x69, 0x00, 0x00, 0xAD, 0xB3, /* 0xC8-0xCB */ + 0xB6, 0xB4, 0xE4, 0xCA, 0xE4, 0xC9, 0xE8, 0xB5, /* 0xCC-0xCF */ + 0xE8, 0xB4, 0x00, 0x00, 0x00, 0x00, 0xC1, 0xFA, /* 0xD0-0xD3 */ + 0xEF, 0x43, 0xEF, 0x42, 0xF1, 0xA5, 0xF1, 0xA3, /* 0xD4-0xD7 */ + 0xF1, 0xA6, 0xF1, 0xA4, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0xC3, 0xFC, 0xF3, 0xA4, 0xF3, 0xA5, 0xF3, 0xA6, /* 0xDC-0xDF */ + 0x00, 0x00, 0xF6, 0x71, 0x00, 0x00, 0xF7, 0x72, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xF8, 0xD2, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0xAD, 0xB4, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0xEC, 0x57, 0xEF, 0x44, 0x00, 0x00, 0xAD, 0xB5, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0xBB, 0xE0, 0x00, 0x00, /* 0xF4-0xF7 */ + 0xEC, 0x58, 0xC3, 0x41, 0xF1, 0xA7, 0xC3, 0xFD, /* 0xF8-0xFB */ + 0x00, 0x00, 0xF5, 0x4C, 0xF5, 0x4D, 0xC5, 0x54, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_98[512] = { + 0xF8, 0x51, 0xAD, 0xB6, 0xB3, 0xBB, 0xB3, 0xBC, /* 0x00-0x03 */ + 0xD8, 0x4E, 0xB6, 0xB5, 0xB6, 0xB6, 0xDC, 0xAC, /* 0x04-0x07 */ + 0xB6, 0xB7, 0x00, 0x00, 0xB9, 0x7A, 0x00, 0x00, /* 0x08-0x0B */ + 0xB9, 0x7C, 0xE0, 0xDF, 0xE0, 0xE0, 0xE0, 0xDE, /* 0x0C-0x0F */ + 0xB9, 0x77, 0xB9, 0x78, 0xB9, 0x7B, 0xB9, 0x79, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0xE4, 0xCB, 0xBB, 0xE1, /* 0x14-0x17 */ + 0xBB, 0xE2, 0x00, 0x00, 0x00, 0x00, 0xE8, 0xBC, /* 0x18-0x1B */ + 0xBE, 0x67, 0xE8, 0xB7, 0xE8, 0xB6, 0x00, 0x00, /* 0x1C-0x1F */ + 0xE8, 0xBB, 0xBE, 0x65, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0xC0, 0x5B, 0x00, 0x00, 0xE8, 0xB8, 0xE8, 0xBD, /* 0x24-0x27 */ + 0xE8, 0xBA, 0xE8, 0xB9, 0x00, 0x00, 0xBE, 0x66, /* 0x28-0x2B */ + 0x00, 0x00, 0xC0, 0x59, 0x00, 0x00, 0xEC, 0x5A, /* 0x2C-0x2F */ + 0xC0, 0x55, 0x00, 0x00, 0xEC, 0x5B, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0xEC, 0x59, 0x00, 0x00, 0xC0, 0x58, /* 0x34-0x37 */ + 0xC0, 0x56, 0xC0, 0x5A, 0x00, 0x00, 0xC0, 0x57, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0xEF, 0x45, 0x00, 0x00, 0xEF, 0x4A, /* 0x40-0x43 */ + 0xEF, 0x46, 0xEF, 0x49, 0xC1, 0xFB, 0x00, 0x00, /* 0x44-0x47 */ + 0xED, 0xD4, 0xEF, 0x48, 0xEF, 0x47, 0x00, 0x00, /* 0x48-0x4B */ + 0xC3, 0x44, 0xC3, 0x42, 0xC3, 0x45, 0xC3, 0x43, /* 0x4C-0x4F */ + 0xF1, 0xA8, 0xF1, 0xA9, 0xF1, 0xAA, 0xC3, 0x46, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF3, 0xAA, /* 0x54-0x57 */ + 0xC4, 0x40, 0xF3, 0xA8, 0x00, 0x00, 0xC4, 0x41, /* 0x58-0x5B */ + 0xF3, 0xA7, 0xF3, 0xA9, 0xC3, 0xFE, 0xF5, 0x51, /* 0x5C-0x5F */ + 0xF5, 0x4E, 0x00, 0x00, 0xF5, 0x4F, 0xF5, 0x50, /* 0x60-0x63 */ + 0xF6, 0x72, 0xC5, 0x56, 0x00, 0x00, 0xC5, 0x55, /* 0x64-0x67 */ + 0x00, 0x00, 0xF7, 0x74, 0xF7, 0x73, 0xC5, 0xB8, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC5, 0xE3, /* 0x6C-0x6F */ + 0xC6, 0x49, 0xC6, 0x60, 0xF9, 0x58, 0xF9, 0xAE, /* 0x70-0x73 */ + 0xF9, 0xAF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xAD, 0xB7, 0xDC, 0xAD, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0xE0, 0xE1, 0xE4, 0xCC, 0xE4, 0xCD, 0xBB, 0xE3, /* 0xAC-0xAF */ + 0x00, 0x00, 0xBB, 0xE4, 0xE8, 0xBE, 0xBE, 0x68, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0xC1, 0xFC, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xF1, 0xAB, 0x00, 0x00, 0xC3, 0x47, 0xF3, 0xAD, /* 0xB8-0xBB */ + 0xC4, 0x42, 0xF3, 0xAC, 0xF3, 0xAE, 0xF3, 0xAB, /* 0xBC-0xBF */ + 0xF6, 0x75, 0xF5, 0x52, 0xF5, 0x53, 0x00, 0x00, /* 0xC0-0xC3 */ + 0xC4, 0xC6, 0x00, 0x00, 0xF6, 0x74, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xF6, 0x73, 0x00, 0x00, 0xF7, 0x75, /* 0xC8-0xCB */ + 0xF9, 0xB0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAD, 0xB8, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAD, 0xB9, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0xB0, 0xA7, 0xD4, 0x48, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xD8, 0x4F, 0x00, 0x00, 0xB6, 0xB8, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xB6, 0xBB, 0xB6, 0xB9, 0xDC, 0xAE, /* 0xE8-0xEB */ + 0x00, 0x00, 0xB6, 0xBD, 0x00, 0x00, 0xB6, 0xBA, /* 0xEC-0xEF */ + 0x00, 0x00, 0x00, 0x00, 0xB6, 0xBC, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xB9, 0x7E, 0x00, 0x00, 0xE0, 0xE2, 0x00, 0x00, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xE0, 0xE3, 0xE8, 0xC0, 0x00, 0x00, /* 0xF8-0xFB */ + 0xB9, 0x7D, 0xB9, 0xA1, 0xB9, 0xA2, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_99[512] = { + 0xE4, 0xCF, 0x00, 0x00, 0xE4, 0xCE, 0xBB, 0xE5, /* 0x00-0x03 */ + 0x00, 0x00, 0xBB, 0xE6, 0x00, 0x00, 0xE4, 0xD0, /* 0x04-0x07 */ + 0xE8, 0xBF, 0xBB, 0xE8, 0xBE, 0x69, 0x00, 0x00, /* 0x08-0x0B */ + 0xBB, 0xE7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0xC0, 0x5C, 0xE8, 0xC1, 0xBE, 0x6B, 0xBE, 0x6A, /* 0x10-0x13 */ + 0xE8, 0xC2, 0xE8, 0xC5, 0xE8, 0xC3, 0xE8, 0xC4, /* 0x14-0x17 */ + 0xBE, 0x6C, 0x00, 0x00, 0xC0, 0x61, 0xC0, 0x5F, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xC0, 0x5E, 0xEC, 0x5D, /* 0x1C-0x1F */ + 0x00, 0x00, 0xC0, 0x60, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0xEC, 0x5C, 0xEF, 0x4B, 0x00, 0x00, 0xEC, 0x5E, /* 0x24-0x27 */ + 0xC0, 0x5D, 0xEC, 0x5F, 0xEF, 0x4E, 0xEF, 0x4C, /* 0x28-0x2B */ + 0xEF, 0x4D, 0xEF, 0x52, 0xC3, 0x4B, 0xEF, 0x51, /* 0x2C-0x2F */ + 0xEF, 0x54, 0xEF, 0x53, 0xEF, 0x50, 0xEF, 0x4F, /* 0x30-0x33 */ + 0x00, 0x00, 0xC1, 0xFD, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xF1, 0xAE, 0x00, 0x00, /* 0x38-0x3B */ + 0xF1, 0xAD, 0xC3, 0x4A, 0xC3, 0x48, 0xC3, 0x49, /* 0x3C-0x3F */ + 0x00, 0x00, 0xF1, 0xAC, 0x00, 0x00, 0xF3, 0xB1, /* 0x40-0x43 */ + 0x00, 0x00, 0xC4, 0x43, 0x00, 0x00, 0xF3, 0xB0, /* 0x44-0x47 */ + 0xF3, 0xAF, 0xC4, 0x44, 0x00, 0x00, 0xF5, 0x58, /* 0x48-0x4B */ + 0xF5, 0x57, 0x00, 0x00, 0xF5, 0x55, 0x00, 0x00, /* 0x4C-0x4F */ + 0xF5, 0x54, 0xC4, 0xC8, 0xC4, 0xC7, 0xF5, 0x59, /* 0x50-0x53 */ + 0xF7, 0x76, 0xC5, 0xB9, 0xF6, 0x77, 0xC5, 0x57, /* 0x54-0x57 */ + 0xF6, 0x76, 0xF5, 0x56, 0x00, 0x00, 0xF7, 0x77, /* 0x58-0x5B */ + 0xC5, 0xE4, 0x00, 0x00, 0xC6, 0x61, 0xF9, 0x59, /* 0x5C-0x5F */ + 0x00, 0x00, 0xF9, 0xB1, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0xAD, 0xBA, 0xD8, 0x50, /* 0x94-0x97 */ + 0xEF, 0x55, 0xAD, 0xBB, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0xE4, 0xD2, 0xE4, 0xD1, 0xEC, 0x60, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0xEF, 0x57, 0x00, 0x00, 0xEF, 0x56, /* 0xA0-0xA3 */ + 0x00, 0x00, 0xC3, 0x4C, 0xF3, 0xB2, 0xF3, 0xB3, /* 0xA4-0xA7 */ + 0xC4, 0xC9, 0x00, 0x00, 0x00, 0x00, 0xF9, 0xB2, /* 0xA8-0xAB */ + 0xB0, 0xA8, 0xB6, 0xBF, 0xB6, 0xBE, 0xE0, 0xE4, /* 0xAC-0xAF */ + 0xE0, 0xE6, 0xB9, 0xA4, 0xE0, 0xE5, 0xB9, 0xA3, /* 0xB0-0xB3 */ + 0xB9, 0xA5, 0xE0, 0xE7, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0xE4, 0xD4, 0xE4, 0xD6, 0xE4, 0xD5, /* 0xB8-0xBB */ + 0x00, 0x00, 0xE4, 0xD8, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0xBB, 0xE9, 0xE4, 0xD7, 0xE4, 0xD3, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xD9, /* 0xC4-0xC7 */ + 0x00, 0x00, 0xE8, 0xCC, 0x00, 0x00, 0xE8, 0xCF, /* 0xC8-0xCB */ + 0xE8, 0xD1, 0xE8, 0xC7, 0xE8, 0xCB, 0xE8, 0xC8, /* 0xCC-0xCF */ + 0xBE, 0x6E, 0xBE, 0x71, 0xBE, 0x73, 0xE8, 0xC9, /* 0xD0-0xD3 */ + 0xE8, 0xCA, 0xBE, 0x72, 0xE8, 0xCD, 0xE8, 0xD0, /* 0xD4-0xD7 */ + 0xE8, 0xCE, 0xBE, 0x74, 0x00, 0x00, 0xBE, 0x70, /* 0xD8-0xDB */ + 0xE8, 0xC6, 0xBE, 0x6D, 0x00, 0x00, 0xBE, 0x6F, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0xC0, 0x63, 0xEC, 0x66, /* 0xE0-0xE3 */ + 0xEC, 0x64, 0xEC, 0x63, 0x00, 0x00, 0xEC, 0x69, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xEC, 0x68, 0xEC, 0x67, 0x00, 0x00, /* 0xE8-0xEB */ + 0xEC, 0x62, 0xC0, 0x62, 0xEC, 0x61, 0x00, 0x00, /* 0xEC-0xEF */ + 0xEC, 0x65, 0xC0, 0x64, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0xEF, 0x5A, 0x00, 0x00, 0xEF, 0x5E, 0xEF, 0x5B, /* 0xF4-0xF7 */ + 0xEF, 0x5D, 0xEF, 0x5C, 0xEF, 0x59, 0xEF, 0x5F, /* 0xF8-0xFB */ + 0xEF, 0x62, 0xEF, 0x60, 0xEF, 0x61, 0xC2, 0x40, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_9A[512] = { + 0x00, 0x00, 0xC1, 0xFE, 0xEF, 0x58, 0xEF, 0x63, /* 0x00-0x03 */ + 0xF1, 0xB3, 0xF1, 0xB6, 0xF1, 0xB8, 0xF1, 0xB7, /* 0x04-0x07 */ + 0x00, 0x00, 0xF1, 0xB1, 0xF1, 0xB5, 0xF1, 0xB0, /* 0x08-0x0B */ + 0x00, 0x00, 0xF1, 0xB2, 0xC3, 0x4D, 0xF1, 0xAF, /* 0x0C-0x0F */ + 0x00, 0x00, 0xF1, 0xB4, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0xF3, 0xC0, 0xF3, 0xB5, 0xC4, 0x45, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0xC4, 0x46, 0xF3, 0xB4, 0xF3, 0xB9, /* 0x18-0x1B */ + 0xF3, 0xBF, 0xF3, 0xB7, 0xF3, 0xBE, 0x00, 0x00, /* 0x1C-0x1F */ + 0xF3, 0xBB, 0x00, 0x00, 0xF3, 0xBA, 0xF3, 0xBD, /* 0x20-0x23 */ + 0xF3, 0xB8, 0xF3, 0xB6, 0x00, 0x00, 0xF3, 0xBC, /* 0x24-0x27 */ + 0x00, 0x00, 0xF5, 0x60, 0xF5, 0x5E, 0xC4, 0xCA, /* 0x28-0x2B */ + 0xF5, 0x5D, 0xF5, 0x63, 0xF5, 0x61, 0x00, 0x00, /* 0x2C-0x2F */ + 0xC4, 0xCB, 0xF5, 0x5C, 0xF5, 0x5A, 0x00, 0x00, /* 0x30-0x33 */ + 0xF5, 0x5B, 0xC4, 0xCD, 0xF5, 0x5F, 0xC4, 0xCC, /* 0x34-0x37 */ + 0xF5, 0x62, 0xF6, 0x78, 0xF6, 0x7E, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0xF6, 0x79, 0xC5, 0x5B, 0xF6, 0xA1, /* 0x3C-0x3F */ + 0xC5, 0x5A, 0xF6, 0x7D, 0xF6, 0x7C, 0xC5, 0x59, /* 0x40-0x43 */ + 0xF6, 0x7B, 0xC5, 0x58, 0xF6, 0x7A, 0x00, 0x00, /* 0x44-0x47 */ + 0xF7, 0x7D, 0xF7, 0xA1, 0xF7, 0x7E, 0x00, 0x00, /* 0x48-0x4B */ + 0xF7, 0x7B, 0xC5, 0xBB, 0xF7, 0x78, 0xF7, 0x7C, /* 0x4C-0x4F */ + 0xF7, 0xA3, 0x00, 0x00, 0xF7, 0xA2, 0xF7, 0x79, /* 0x50-0x53 */ + 0xF7, 0x7A, 0xC5, 0xBA, 0xF8, 0x52, 0xC5, 0xE7, /* 0x54-0x57 */ + 0x00, 0x00, 0xF8, 0x53, 0xC5, 0xE5, 0xC5, 0xE6, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0xF8, 0xD3, 0xC6, 0x4A, /* 0x5C-0x5F */ + 0xF9, 0x76, 0x00, 0x00, 0xC6, 0x6A, 0x00, 0x00, /* 0x60-0x63 */ + 0xF9, 0xB3, 0xC6, 0x6B, 0xF9, 0xB4, 0xF9, 0xB5, /* 0x64-0x67 */ + 0xF9, 0xC3, 0xF9, 0xC2, 0xC6, 0x7A, 0xF9, 0xCD, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xB0, 0xA9, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xE9, /* 0xA8-0xAB */ + 0x00, 0x00, 0xE0, 0xE8, 0x00, 0x00, 0xBB, 0xEA, /* 0xAC-0xAF */ + 0xBB, 0xEB, 0xE4, 0xDA, 0x00, 0x00, 0xE8, 0xD2, /* 0xB0-0xB3 */ + 0xEC, 0x6C, 0x00, 0x00, 0x00, 0x00, 0xBE, 0x75, /* 0xB4-0xB7 */ + 0xC0, 0x65, 0xEC, 0x6A, 0x00, 0x00, 0xEC, 0x6D, /* 0xB8-0xBB */ + 0xC0, 0x66, 0x00, 0x00, 0xEF, 0x64, 0xEC, 0x6B, /* 0xBC-0xBF */ + 0xF1, 0xB9, 0xC3, 0x4E, 0xF3, 0xC1, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0xF5, 0x66, 0xF5, 0x64, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0xF5, 0x65, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0xF6, 0xA2, 0x00, 0x00, 0xC5, 0x5C, /* 0xCC-0xCF */ + 0xF7, 0xA4, 0xC5, 0xEA, 0xC5, 0xBC, 0xC5, 0xE8, /* 0xD0-0xD3 */ + 0xC5, 0xE9, 0xF8, 0xD4, 0xC6, 0x62, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xB0, 0xAA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0xF1, 0xBA, 0x00, 0x00, 0x00, 0x00, 0xD4, 0x49, /* 0xDC-0xDF */ + 0x00, 0x00, 0xB9, 0xA6, 0x00, 0x00, 0xE4, 0xDB, /* 0xE0-0xE3 */ + 0x00, 0x00, 0x00, 0x00, 0xBB, 0xEC, 0xE4, 0xDC, /* 0xE4-0xE7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE8, 0xD4, /* 0xE8-0xEB */ + 0xE8, 0xD3, 0xC0, 0x68, 0xBE, 0x76, 0xBE, 0x77, /* 0xEC-0xEF */ + 0x00, 0x00, 0xE8, 0xD7, 0xE8, 0xD6, 0xE8, 0xD5, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0xEC, 0x6E, 0xEC, 0x71, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xEC, 0x70, 0xEC, 0x6F, 0xC0, 0x67, /* 0xF8-0xFB */ + 0xEF, 0x68, 0xEF, 0x66, 0xEF, 0x65, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_9B[512] = { + 0x00, 0x00, 0xEF, 0x67, 0x00, 0x00, 0xC3, 0x4F, /* 0x00-0x03 */ + 0xF1, 0xBC, 0xF1, 0xBD, 0xC3, 0x50, 0x00, 0x00, /* 0x04-0x07 */ + 0xF1, 0xBB, 0x00, 0x00, 0xF3, 0xC3, 0xF3, 0xC2, /* 0x08-0x0B */ + 0xF3, 0xC5, 0xC4, 0x47, 0xF3, 0xC4, 0x00, 0x00, /* 0x0C-0x0F */ + 0xF5, 0x67, 0xF5, 0x69, 0xF5, 0x68, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0xF6, 0xA3, 0xF6, 0xA6, 0xF6, 0xA4, /* 0x14-0x17 */ + 0xF6, 0xA5, 0xF7, 0xA5, 0xC5, 0xBD, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0xF8, 0x54, 0xF8, 0x55, /* 0x1C-0x1F */ + 0xF8, 0x56, 0x00, 0x00, 0xC6, 0x4B, 0xC6, 0x63, /* 0x20-0x23 */ + 0xF9, 0xB6, 0xB0, 0xAB, 0x00, 0x00, 0xBE, 0x78, /* 0x24-0x27 */ + 0xC0, 0x69, 0xF1, 0xBE, 0x00, 0x00, 0xF7, 0xA6, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0xF9, 0xC4, 0xD4, 0x4A, /* 0x2C-0x2F */ + 0x00, 0x00, 0xC6, 0x7B, 0xB0, 0xAC, 0xEC, 0x72, /* 0x30-0x33 */ + 0x00, 0x00, 0xF1, 0xBF, 0x00, 0x00, 0xF3, 0xC6, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0xF6, 0xA7, 0xF7, 0xA7, /* 0x38-0x3B */ + 0xB0, 0xAD, 0x00, 0x00, 0xE4, 0xDD, 0xE4, 0xDE, /* 0x3C-0x3F */ + 0x00, 0x00, 0xBB, 0xED, 0xBB, 0xEE, 0xE8, 0xD9, /* 0x40-0x43 */ + 0xBE, 0x7A, 0xBE, 0x79, 0xE8, 0xD8, 0x00, 0x00, /* 0x44-0x47 */ + 0xEF, 0x69, 0x00, 0x00, 0xF1, 0xC0, 0xF1, 0xC2, /* 0x48-0x4B */ + 0xF1, 0xC1, 0xC3, 0x53, 0xC3, 0x52, 0xC3, 0x51, /* 0x4C-0x4F */ + 0x00, 0x00, 0xC5, 0x5E, 0xF6, 0xA8, 0x00, 0x00, /* 0x50-0x53 */ + 0xC5, 0x5D, 0xF7, 0xA9, 0xF7, 0xA8, 0x00, 0x00, /* 0x54-0x57 */ + 0xC6, 0x4C, 0xF8, 0xD5, 0xB3, 0xBD, 0xE0, 0xEA, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE4, 0xE1, /* 0x5C-0x5F */ + 0xE4, 0xDF, 0xE4, 0xE0, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0xE8, 0xE2, 0x00, 0x00, 0xE8, 0xDD, 0xE8, 0xDA, /* 0x64-0x67 */ + 0xE8, 0xE1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0xE8, 0xE3, 0x00, 0x00, 0x00, 0x00, 0xBE, 0x7C, /* 0x6C-0x6F */ + 0xE8, 0xE0, 0xE8, 0xDC, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0xE8, 0xDB, 0xE8, 0xDF, 0xE8, 0xDE, 0xBE, 0x7B, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0xEC, 0x7D, 0xEC, 0x78, /* 0x78-0x7B */ + 0xEC, 0x76, 0xEC, 0xA1, 0xEC, 0x77, 0x00, 0x00, /* 0x7C-0x7F */ + + 0xEC, 0x73, 0x00, 0x00, 0xEC, 0x79, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0xEC, 0x74, 0xEF, 0x72, 0xEC, 0x75, /* 0x84-0x87 */ + 0xEC, 0xA2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0xEC, 0x7C, 0xC0, 0x6A, 0xEC, 0x7B, 0xEC, 0x7A, /* 0x90-0x93 */ + 0x00, 0x00, 0xEC, 0x7E, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0xEF, 0x6A, 0xEF, 0x6D, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0xEF, 0x6C, 0x00, 0x00, /* 0x9C-0x9F */ + 0xEF, 0x74, 0xEF, 0x6F, 0xEF, 0x73, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xEF, 0x71, 0xEF, 0x70, 0xEF, 0x6E, 0x00, 0x00, /* 0xA4-0xA7 */ + 0xEF, 0x6B, 0x00, 0x00, 0xC2, 0x43, 0xC2, 0x42, /* 0xA8-0xAB */ + 0x00, 0x00, 0xC2, 0x44, 0xC2, 0x41, 0xEF, 0x75, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0xF1, 0xC8, 0xF1, 0xCB, 0x00, 0x00, /* 0xB4-0xB7 */ + 0xF1, 0xC9, 0xF1, 0xCD, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0xF1, 0xCE, 0x00, 0x00, 0xF1, 0xC6, /* 0xBC-0xBF */ + 0xC3, 0x58, 0xF1, 0xC7, 0x00, 0x00, 0xF1, 0xC5, /* 0xC0-0xC3 */ + 0xF1, 0xCC, 0x00, 0x00, 0xF1, 0xC4, 0xF1, 0xC3, /* 0xC4-0xC7 */ + 0xC3, 0x57, 0xC3, 0x55, 0xC3, 0x54, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF1, 0xCA, /* 0xD0-0xD3 */ + 0xF3, 0xCF, 0xF3, 0xD5, 0xC4, 0x4A, 0xF3, 0xD0, /* 0xD4-0xD7 */ + 0x00, 0x00, 0xF3, 0xD3, 0xF3, 0xD7, 0xC4, 0x4B, /* 0xD8-0xDB */ + 0xF3, 0xD2, 0x00, 0x00, 0xF3, 0xCA, 0x00, 0x00, /* 0xDC-0xDF */ + 0xF3, 0xC9, 0xF3, 0xD6, 0xF3, 0xCD, 0x00, 0x00, /* 0xE0-0xE3 */ + 0xF3, 0xCB, 0xF3, 0xD4, 0xF3, 0xCC, 0xC4, 0x49, /* 0xE4-0xE7 */ + 0xC4, 0x48, 0x00, 0x00, 0xF3, 0xC7, 0xF3, 0xC8, /* 0xE8-0xEB */ + 0xF3, 0xD1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0xF3, 0xCE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF0-0xF3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF5, 0x6C, /* 0xF4-0xF7 */ + 0xF5, 0x6F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xF8-0xFB */ + 0x00, 0x00, 0xC3, 0x56, 0x00, 0x00, 0x00, 0x00, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_9C[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0xF5, 0x6D, 0xF5, 0x73, 0xF5, 0x71, /* 0x04-0x07 */ + 0xF5, 0x6B, 0xF5, 0x76, 0x00, 0x00, 0xF5, 0x6A, /* 0x08-0x0B */ + 0x00, 0x00, 0xC4, 0xCF, 0xF5, 0x72, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0xF5, 0x6E, 0xC4, 0xCE, /* 0x10-0x13 */ + 0xF5, 0x75, 0x00, 0x00, 0x00, 0x00, 0xF5, 0x74, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0xF6, 0xAB, 0xF6, 0xAA, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0xF6, 0xB1, 0x00, 0x00, 0xF6, 0xAD, /* 0x20-0x23 */ + 0xF6, 0xB0, 0xC5, 0x60, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0xF6, 0xAE, 0xF6, 0xAF, 0x00, 0x00, 0xF6, 0xA9, /* 0x28-0x2B */ + 0xF6, 0xAC, 0xC5, 0x5F, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0xC5, 0xBF, 0xF7, 0xB4, 0xF7, 0xAF, /* 0x30-0x33 */ + 0xF7, 0xB3, 0x00, 0x00, 0xF7, 0xB6, 0xF7, 0xB2, /* 0x34-0x37 */ + 0x00, 0x00, 0xF7, 0xAE, 0x00, 0x00, 0xC5, 0xC1, /* 0x38-0x3B */ + 0xF7, 0xB1, 0xF7, 0xB5, 0xC5, 0xC0, 0xF7, 0xAC, /* 0x3C-0x3F */ + 0xF5, 0x70, 0xF7, 0xB0, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0xF7, 0xAD, 0x00, 0x00, 0xF7, 0xAA, 0x00, 0x00, /* 0x44-0x47 */ + 0xF7, 0xAB, 0xC5, 0xBE, 0xF8, 0x5A, 0xF8, 0x5C, /* 0x48-0x4B */ + 0xF8, 0x5F, 0xF8, 0x5B, 0xF8, 0x60, 0x00, 0x00, /* 0x4C-0x4F */ + 0xF8, 0x59, 0x00, 0x00, 0xF8, 0x57, 0x00, 0x00, /* 0x50-0x53 */ + 0xC5, 0xEB, 0xF8, 0x5D, 0xC5, 0xED, 0xC5, 0xEC, /* 0x54-0x57 */ + 0xF8, 0x58, 0xF8, 0x5E, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0xF8, 0xDA, 0xC6, 0x4D, /* 0x5C-0x5F */ + 0xF8, 0xDB, 0x00, 0x00, 0xF8, 0xD9, 0xF8, 0xD6, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0xF8, 0xD8, 0xF8, 0xD7, /* 0x64-0x67 */ + 0xF9, 0x5A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0xF9, 0x5C, 0xF9, 0x5B, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0xF9, 0x79, 0x00, 0x00, 0xF9, 0x78, /* 0x70-0x73 */ + 0xF9, 0x77, 0xF9, 0x7A, 0x00, 0x00, 0xC6, 0x73, /* 0x74-0x77 */ + 0xC6, 0x74, 0xF9, 0xCA, 0xF9, 0xCE, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xB3, 0xBE, 0xDC, 0xAF, 0xE0, 0xED, /* 0xE4-0xE7 */ + 0x00, 0x00, 0xB9, 0xA7, 0xE0, 0xEB, 0x00, 0x00, /* 0xE8-0xEB */ + 0x00, 0x00, 0xE0, 0xEC, 0x00, 0x00, 0x00, 0x00, /* 0xEC-0xEF */ + 0x00, 0x00, 0xE4, 0xE2, 0xE4, 0xE3, 0xBB, 0xF1, /* 0xF0-0xF3 */ + 0xBB, 0xEF, 0xE4, 0xE4, 0xBB, 0xF0, 0xE8, 0xE8, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xE8, 0xEB, 0xE8, 0xE5, 0xE8, 0xEC, /* 0xF8-0xFB */ + 0xE8, 0xE4, 0xE8, 0xE6, 0x00, 0x00, 0xE8, 0xE7, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_9D[512] = { + 0xE8, 0xEA, 0x00, 0x00, 0x00, 0x00, 0xBE, 0xA1, /* 0x00-0x03 */ + 0xE8, 0xEF, 0xE8, 0xEE, 0xBE, 0x7D, 0xE8, 0xE9, /* 0x04-0x07 */ + 0xE8, 0xED, 0xBE, 0x7E, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0xEC, 0xAC, 0x00, 0x00, 0xC0, 0x6F, 0x00, 0x00, /* 0x10-0x13 */ + 0xEC, 0xA7, 0xC0, 0x6B, 0x00, 0x00, 0xEC, 0xA4, /* 0x14-0x17 */ + 0xEC, 0xAA, 0xEC, 0xAD, 0x00, 0x00, 0xC0, 0x70, /* 0x18-0x1B */ + 0x00, 0x00, 0xEC, 0xA9, 0xEC, 0xA6, 0xEC, 0xAE, /* 0x1C-0x1F */ + 0xEC, 0xA5, 0x00, 0x00, 0xEC, 0xAB, 0xC0, 0x6C, /* 0x20-0x23 */ + 0x00, 0x00, 0xEC, 0xA3, 0xC0, 0x6D, 0x00, 0x00, /* 0x24-0x27 */ + 0xC0, 0x6E, 0xEC, 0xA8, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0xEF, 0xA9, 0xEF, 0x7A, 0xEF, 0x7B, /* 0x2C-0x2F */ + 0xEF, 0x7E, 0xEF, 0x7C, 0x00, 0x00, 0xEF, 0x76, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0xEF, 0x79, 0xEF, 0xA5, /* 0x34-0x37 */ + 0xEF, 0x7D, 0x00, 0x00, 0x00, 0x00, 0xC2, 0x45, /* 0x38-0x3B */ + 0x00, 0x00, 0xEF, 0xA7, 0xEF, 0xA4, 0xC2, 0x46, /* 0x3C-0x3F */ + 0xEF, 0xA6, 0xEF, 0x77, 0xEF, 0xA2, 0xEF, 0xA3, /* 0x40-0x43 */ + 0x00, 0x00, 0xEF, 0xA1, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0xF1, 0xD2, 0xF1, 0xD4, /* 0x48-0x4B */ + 0xF1, 0xD7, 0x00, 0x00, 0x00, 0x00, 0xF1, 0xD1, /* 0x4C-0x4F */ + 0x00, 0x00, 0xC3, 0x59, 0xF1, 0xD9, 0xF1, 0xD0, /* 0x50-0x53 */ + 0xF1, 0xDA, 0x00, 0x00, 0xF1, 0xD6, 0xF1, 0xD8, /* 0x54-0x57 */ + 0xF1, 0xDC, 0xF1, 0xD5, 0xF1, 0xDD, 0xF1, 0xD3, /* 0x58-0x5B */ + 0xF1, 0xCF, 0xC3, 0x5A, 0x00, 0x00, 0xF1, 0xDB, /* 0x5C-0x5F */ + 0xC3, 0x5B, 0xC4, 0x4D, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xEF, 0x78, /* 0x64-0x67 */ + 0xF3, 0xF1, 0xF3, 0xE8, 0xC4, 0x4F, 0xF3, 0xE4, /* 0x68-0x6B */ + 0xC4, 0x50, 0x00, 0x00, 0x00, 0x00, 0xF3, 0xED, /* 0x6C-0x6F */ + 0xF3, 0xE7, 0xF3, 0xDD, 0xC4, 0x4E, 0xF3, 0xEA, /* 0x70-0x73 */ + 0xF3, 0xE5, 0xF3, 0xE6, 0x00, 0x00, 0xF3, 0xD8, /* 0x74-0x77 */ + 0xF3, 0xDF, 0xF3, 0xEE, 0x00, 0x00, 0xF3, 0xEB, /* 0x78-0x7B */ + 0x00, 0x00, 0xF3, 0xE3, 0x00, 0x00, 0xF3, 0xEF, /* 0x7C-0x7F */ + + 0xF3, 0xDE, 0xF3, 0xD9, 0xF3, 0xEC, 0x00, 0x00, /* 0x80-0x83 */ + 0xF3, 0xDB, 0xF3, 0xE9, 0xF3, 0xE0, 0xF3, 0xF0, /* 0x84-0x87 */ + 0xF3, 0xDC, 0xC4, 0x4C, 0xF3, 0xDA, 0xF3, 0xE1, /* 0x88-0x8B */ + 0xF3, 0xE2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0xF5, 0x7D, 0x00, 0x00, 0xF5, 0x7B, 0x00, 0x00, /* 0x90-0x93 */ + 0xF5, 0xA2, 0x00, 0x00, 0xF5, 0xAE, 0xF5, 0xA5, /* 0x94-0x97 */ + 0xF5, 0x7C, 0xF5, 0x78, 0xF5, 0xA7, 0xF5, 0x7E, /* 0x98-0x9B */ + 0xF5, 0xA3, 0xF5, 0x7A, 0xF5, 0xAA, 0xF5, 0x77, /* 0x9C-0x9F */ + 0xF5, 0xA1, 0xF5, 0xA6, 0xF5, 0xA8, 0xF5, 0xAB, /* 0xA0-0xA3 */ + 0xF5, 0x79, 0x00, 0x00, 0xF5, 0xAF, 0xF5, 0xB0, /* 0xA4-0xA7 */ + 0xF5, 0xA9, 0xF5, 0xAD, 0xF5, 0xA4, 0x00, 0x00, /* 0xA8-0xAB */ + 0xF6, 0xC1, 0xF6, 0xC4, 0x00, 0x00, 0xC5, 0x61, /* 0xAC-0xAF */ + 0x00, 0x00, 0xF6, 0xC3, 0xF6, 0xC8, 0xF6, 0xC6, /* 0xB0-0xB3 */ + 0xC5, 0x62, 0xF6, 0xBD, 0xF6, 0xB3, 0xF6, 0xB2, /* 0xB4-0xB7 */ + 0xC5, 0x64, 0xF6, 0xBF, 0xF6, 0xC0, 0xF6, 0xBC, /* 0xB8-0xBB */ + 0xF6, 0xB4, 0x00, 0x00, 0xF6, 0xB9, 0xF5, 0xAC, /* 0xBC-0xBF */ + 0x00, 0x00, 0xF6, 0xB5, 0xC5, 0x63, 0xF6, 0xBB, /* 0xC0-0xC3 */ + 0x00, 0x00, 0xF6, 0xBA, 0x00, 0x00, 0xF6, 0xB6, /* 0xC4-0xC7 */ + 0xF6, 0xC2, 0x00, 0x00, 0xF6, 0xB7, 0xF7, 0xBB, /* 0xC8-0xCB */ + 0xF6, 0xC5, 0xF6, 0xC7, 0xF6, 0xBE, 0xF6, 0xB8, /* 0xCC-0xCF */ + 0xF7, 0xBC, 0xF7, 0xBE, 0xF7, 0xB8, 0xC5, 0xC2, /* 0xD0-0xD3 */ + 0x00, 0x00, 0xF7, 0xC5, 0xF7, 0xC3, 0xC5, 0xC3, /* 0xD4-0xD7 */ + 0xF7, 0xC2, 0xF7, 0xC1, 0xF7, 0xBA, 0xF7, 0xB7, /* 0xD8-0xDB */ + 0xF7, 0xBD, 0xF7, 0xC6, 0xF7, 0xB9, 0xF7, 0xBF, /* 0xDC-0xDF */ + 0x00, 0x00, 0xF8, 0x69, 0xF8, 0x6E, 0xF8, 0x64, /* 0xE0-0xE3 */ + 0xF8, 0x67, 0xC5, 0xEE, 0xF8, 0x6B, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xF8, 0x72, 0xF7, 0xC0, 0x00, 0x00, 0xF8, 0x65, /* 0xE8-0xEB */ + 0xF8, 0x6F, 0xF8, 0x73, 0xF8, 0x6A, 0xF8, 0x63, /* 0xEC-0xEF */ + 0xF8, 0x6D, 0x00, 0x00, 0xF8, 0x6C, 0xF8, 0x71, /* 0xF0-0xF3 */ + 0xF8, 0x70, 0xF7, 0xC4, 0xF8, 0x68, 0xF8, 0x62, /* 0xF4-0xF7 */ + 0xF8, 0x66, 0xC6, 0x4E, 0xC6, 0x4F, 0xF8, 0x61, /* 0xF8-0xFB */ + 0x00, 0x00, 0xF8, 0xE6, 0xF8, 0xDD, 0xF8, 0xE5, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_9E[512] = { + 0xF8, 0xE2, 0xF8, 0xE3, 0xF8, 0xDC, 0xF8, 0xDF, /* 0x00-0x03 */ + 0xF8, 0xE7, 0xF8, 0xE1, 0xF8, 0xE0, 0xF8, 0xDE, /* 0x04-0x07 */ + 0x00, 0x00, 0xF8, 0xE4, 0x00, 0x00, 0xF9, 0x5D, /* 0x08-0x0B */ + 0x00, 0x00, 0xF9, 0x5E, 0x00, 0x00, 0xF9, 0x60, /* 0x0C-0x0F */ + 0xF9, 0x5F, 0xF9, 0x62, 0xF9, 0x61, 0xF9, 0x7C, /* 0x10-0x13 */ + 0xF9, 0x7B, 0xF9, 0xB7, 0x00, 0x00, 0xF9, 0xB8, /* 0x14-0x17 */ + 0x00, 0x00, 0xF9, 0xC5, 0xC6, 0x78, 0xC6, 0x7C, /* 0x18-0x1B */ + 0x00, 0x00, 0xF9, 0xCF, 0xC6, 0x7D, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x33 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x34-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x3C-0x3F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x53 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x54-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0xB3, 0xBF, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0xC4, 0xD0, 0xF6, 0xC9, 0x00, 0x00, /* 0x78-0x7B */ + 0xC6, 0x50, 0xC6, 0x51, 0x00, 0x00, 0xB3, 0xC0, /* 0x7C-0x7F */ + + 0xE0, 0xEE, 0x00, 0x00, 0xB9, 0xA8, 0xE8, 0xF0, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0xEC, 0xB0, 0xEC, 0xB1, /* 0x84-0x87 */ + 0xEC, 0xAF, 0xEF, 0xAB, 0xEF, 0xAA, 0xC2, 0x47, /* 0x88-0x8B */ + 0xF1, 0xDF, 0xEF, 0xAC, 0xF1, 0xDE, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0xF3, 0xF3, 0xC4, 0x51, 0xC4, 0x53, /* 0x90-0x93 */ + 0xF3, 0xF2, 0x00, 0x00, 0x00, 0x00, 0xC4, 0x52, /* 0x94-0x97 */ + 0x00, 0x00, 0xF5, 0xB1, 0xF5, 0xB3, 0xF5, 0xB2, /* 0x98-0x9B */ + 0xF6, 0xCA, 0xC5, 0x65, 0x00, 0x00, 0xC5, 0xEF, /* 0x9C-0x9F */ + 0xF8, 0xE8, 0xF9, 0x63, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xF9, 0xD2, 0xB3, 0xC1, 0x00, 0x00, 0xE4, 0xE5, /* 0xA4-0xA7 */ + 0x00, 0x00, 0xBE, 0xA2, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0xEC, 0xB3, 0xEC, 0xB2, 0x00, 0x00, /* 0xAC-0xAF */ + 0xEF, 0xAD, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0xC4, 0x54, 0xC4, 0xD1, 0xF7, 0xC7, 0xF9, 0xCB, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xB3, 0xC2, /* 0xB8-0xBB */ + 0xBB, 0xF2, 0x00, 0x00, 0xBE, 0xA3, 0x00, 0x00, /* 0xBC-0xBF */ + 0xF3, 0xF4, 0x00, 0x00, 0xF8, 0x74, 0xB6, 0xC0, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0xEF, 0xAE, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0xC6, 0x64, 0xB6, 0xC1, 0xBE, 0xA4, 0xC2, 0x48, /* 0xCC-0xCF */ + 0xF8, 0x75, 0xB6, 0xC2, 0x00, 0x00, 0xE8, 0xF1, /* 0xD0-0xD3 */ + 0xC0, 0x72, 0xEC, 0xB4, 0xEC, 0xB5, 0x00, 0x00, /* 0xD4-0xD7 */ + 0xC0, 0x71, 0x00, 0x00, 0xEF, 0xAF, 0xC2, 0x4C, /* 0xD8-0xDB */ + 0xC2, 0x4A, 0xC2, 0x4B, 0xC2, 0x49, 0xF1, 0xE0, /* 0xDC-0xDF */ + 0xC3, 0x5C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xE0-0xE3 */ + 0xF5, 0xB5, 0xF5, 0xB4, 0xF5, 0xB7, 0xF5, 0xB6, /* 0xE4-0xE7 */ + 0xC4, 0xD2, 0x00, 0x00, 0x00, 0x00, 0xF6, 0xCB, /* 0xE8-0xEB */ + 0x00, 0x00, 0xF6, 0xCD, 0xF6, 0xCC, 0xC5, 0x66, /* 0xEC-0xEF */ + 0xF7, 0xC8, 0x00, 0x00, 0xF8, 0x76, 0xF8, 0x77, /* 0xF0-0xF3 */ + 0xC5, 0xF0, 0xF9, 0x64, 0xF9, 0x7D, 0xC6, 0x75, /* 0xF4-0xF7 */ + 0x00, 0x00, 0xDC, 0xB0, 0xEC, 0xB6, 0xEF, 0xB0, /* 0xF8-0xFB */ + 0xF3, 0xF5, 0xE0, 0xEF, 0x00, 0x00, 0xEF, 0xB1, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_9F[512] = { + 0xF1, 0xE2, 0xF1, 0xE1, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0xF8, 0x78, 0xC6, 0x52, /* 0x04-0x07 */ + 0x00, 0x00, 0xF9, 0x65, 0xF9, 0x7E, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0xB9, 0xA9, 0xE8, 0xF2, /* 0x0C-0x0F */ + 0xE8, 0xF3, 0x00, 0x00, 0xEC, 0xB7, 0xB9, 0xAA, /* 0x10-0x13 */ + 0x00, 0x00, 0xC3, 0x5D, 0xF1, 0xE3, 0x00, 0x00, /* 0x14-0x17 */ + 0xF6, 0xCF, 0xC5, 0x67, 0xF6, 0xD0, 0xF6, 0xCE, /* 0x18-0x1B */ + 0xF8, 0x79, 0x00, 0x00, 0xF8, 0xE9, 0x00, 0x00, /* 0x1C-0x1F */ + 0xB9, 0xAB, 0x00, 0x00, 0xEF, 0xB4, 0xEF, 0xB3, /* 0x20-0x23 */ + 0xEF, 0xB2, 0xF1, 0xE4, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0xF1, 0xE8, 0xF1, 0xE7, 0xF1, 0xE6, 0xF1, 0xE5, /* 0x28-0x2B */ + 0xC3, 0x5E, 0xF3, 0xF6, 0xF5, 0xB9, 0xC4, 0xD3, /* 0x2C-0x2F */ + 0xF5, 0xB8, 0xF6, 0xD1, 0xF7, 0xCB, 0xF7, 0xCA, /* 0x30-0x33 */ + 0xC5, 0xC4, 0xF7, 0xC9, 0xF8, 0x7C, 0xF8, 0x7B, /* 0x34-0x37 */ + 0xF8, 0x7A, 0x00, 0x00, 0x00, 0x00, 0xBB, 0xF3, /* 0x38-0x3B */ + 0x00, 0x00, 0xEC, 0xB8, 0xC2, 0x4D, 0x00, 0x00, /* 0x3C-0x3F */ + 0xF3, 0xF7, 0xF3, 0xF8, 0xF7, 0xCC, 0xF8, 0x7D, /* 0x40-0x43 */ + 0x00, 0x00, 0x00, 0x00, 0xF8, 0xEA, 0xF9, 0x66, /* 0x44-0x47 */ + 0xF9, 0xB9, 0xF9, 0xD4, 0xBB, 0xF4, 0xC2, 0x4E, /* 0x48-0x4B */ + 0xF1, 0xE9, 0xF3, 0xF9, 0xF6, 0xD2, 0xF8, 0x7E, /* 0x4C-0x4F */ + 0x00, 0x00, 0x00, 0x00, 0xBE, 0xA6, 0x00, 0x00, /* 0x50-0x53 */ + 0xEF, 0xB5, 0xF1, 0xEA, 0xF3, 0xFA, 0xF3, 0xFB, /* 0x54-0x57 */ + 0xF3, 0xFC, 0xF5, 0xBE, 0x00, 0x00, 0xF5, 0xBA, /* 0x58-0x5B */ + 0xC5, 0x68, 0xF5, 0xBD, 0xF5, 0xBC, 0xC4, 0xD4, /* 0x5C-0x5F */ + 0xF5, 0xBB, 0xC4, 0xD6, 0x00, 0x00, 0xC4, 0xD5, /* 0x60-0x63 */ + 0xF6, 0xD4, 0xF6, 0xD3, 0xC5, 0x69, 0xC5, 0x6A, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0xC5, 0xC6, 0xF7, 0xCD, /* 0x68-0x6B */ + 0xC5, 0xC5, 0x00, 0x00, 0xF8, 0xA3, 0xF8, 0xA4, /* 0x6C-0x6F */ + 0xF8, 0xA2, 0xF8, 0xA1, 0xC6, 0x54, 0x00, 0x00, /* 0x70-0x73 */ + 0xF8, 0xEB, 0xF8, 0xEC, 0xF8, 0xED, 0xC6, 0x53, /* 0x74-0x77 */ + 0xF9, 0x67, 0xF9, 0x6A, 0xF9, 0x69, 0xF9, 0x68, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0xF9, 0xD3, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0xC0, 0x73, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0xC3, 0x65, 0xF5, 0xBF, 0xF6, 0xD5, 0x00, 0x00, /* 0x90-0x93 */ + 0xC5, 0xC7, 0xF7, 0xCE, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0xF9, 0xD5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0xC0, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0xEF, 0xB6, 0x00, 0x00, 0xF7, 0xCF, 0x00, 0x00, /* 0xA0-0xA3 */ + 0xF9, 0xA1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ +}; + +static const unsigned char u2c_DC[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ +}; + +static const unsigned char u2c_F9[512] = { + 0xB0, 0x5A, 0xA7, 0xF3, 0xA8, 0xAE, 0xB8, 0xEB, /* 0x00-0x03 */ + 0xB7, 0xC6, 0xA6, 0xEA, 0xA5, 0x79, 0xC0, 0x74, /* 0x04-0x07 */ + 0xC0, 0x74, 0xAB, 0xB4, 0xAA, 0xF7, 0xB3, 0xE2, /* 0x08-0x0B */ + 0xA9, 0x60, 0xC3, 0x69, 0xC4, 0xEE, 0xC3, 0xB9, /* 0x0C-0x0F */ + 0xC5, 0xDA, 0xC1, 0xB3, 0xBB, 0x72, 0xC5, 0xDE, /* 0x10-0x13 */ + 0xBC, 0xD6, 0xAC, 0xA5, 0xAF, 0x4F, 0xAF, 0x5F, /* 0x14-0x17 */ + 0xB8, 0xA8, 0xB9, 0x54, 0xC0, 0x64, 0xB6, 0xC3, /* 0x18-0x1B */ + 0xA7, 0x5A, 0xC4, 0xE6, 0xC4, 0xEA, 0xC4, 0xF5, /* 0x1C-0x1F */ + 0xC6, 0x7D, 0xB4, 0x50, 0xC0, 0xDD, 0xC2, 0xC5, /* 0x20-0x23 */ + 0xC4, 0xB0, 0xA9, 0xD4, 0xC3, 0xBE, 0xC4, 0xFA, /* 0x24-0x27 */ + 0xB4, 0x59, 0xAE, 0xD4, 0xAE, 0xF6, 0xAF, 0x54, /* 0x28-0x2B */ + 0x00, 0x00, 0xA8, 0xD3, 0xA7, 0x4E, 0xB3, 0xD2, /* 0x2C-0x2F */ + 0xBE, 0xDB, 0xC3, 0x72, 0xC4, 0x6C, 0xBF, 0x63, /* 0x30-0x33 */ + 0xA6, 0xD1, 0xC4, 0xAA, 0xB8, 0xB8, 0xB8, 0xF4, /* 0x34-0x37 */ + 0xC5, 0x53, 0xBE, 0x7C, 0xC6, 0x4F, 0xB8, 0x4C, /* 0x38-0x3B */ + 0xB8, 0x53, 0xBA, 0xF1, 0xDB, 0x77, 0xBF, 0xFD, /* 0x3C-0x3F */ + 0xB3, 0xC0, 0xBD, 0xD7, 0xC3, 0x62, 0xA7, 0xCB, /* 0x40-0x43 */ + 0xC5, 0xA2, 0xC5, 0xA4, 0xA8, 0x63, 0xBD, 0x55, /* 0x44-0x47 */ + 0xB8, 0xEF, 0xB9, 0x70, 0xC2, 0x53, 0xB9, 0xF0, /* 0x48-0x4B */ + 0xBC, 0xD3, 0xB2, 0x5C, 0xBA, 0x7C, 0xB2, 0xD6, /* 0x4C-0x4F */ + 0xC1, 0x5C, 0xAD, 0xAE, 0xB0, 0xC7, 0xA6, 0xD8, /* 0x50-0x53 */ + 0xBB, 0xFE, 0xAD, 0xE2, 0xB8, 0x57, 0xBA, 0xF0, /* 0x54-0x57 */ + 0xB5, 0xD9, 0xB3, 0xAE, 0xC5, 0xAA, 0xCE, 0xD4, /* 0x58-0x5B */ + 0xBC, 0xD6, 0xBF, 0xD5, 0xA4, 0xA6, 0xB9, 0xE7, /* 0x5C-0x5F */ + 0xAB, 0xE3, 0xB2, 0x76, 0xB2, 0xA7, 0xA5, 0x5F, /* 0x60-0x63 */ + 0xED, 0xA8, 0xAB, 0x4B, 0xB4, 0x5F, 0xA4, 0xA3, /* 0x64-0x67 */ + 0xAA, 0x63, 0xBC, 0xC6, 0xAF, 0xC1, 0xB0, 0xD1, /* 0x68-0x6B */ + 0xB6, 0xEB, 0xAC, 0xD9, 0xB8, 0xAD, 0xBB, 0xA1, /* 0x6C-0x6F */ + 0xB1, 0xFE, 0xA8, 0xB0, 0xA8, 0x48, 0xAC, 0x42, /* 0x70-0x73 */ + 0xAD, 0x59, 0xB1, 0xB0, 0xB2, 0xA4, 0xAB, 0x47, /* 0x74-0x77 */ + 0xA8, 0xE2, 0x00, 0x00, 0xB1, 0xE7, 0xC2, 0xB3, /* 0x78-0x7B */ + 0xA8, 0x7D, 0xBD, 0xCC, 0xB6, 0x71, 0xC0, 0x79, /* 0x7C-0x7F */ + + 0xA7, 0x66, 0xA4, 0x6B, 0xC3, 0x66, 0xAE, 0xC8, /* 0x80-0x83 */ + 0xC2, 0x6F, 0xC4, 0x72, 0xBE, 0x5B, 0xC6, 0x7A, /* 0x84-0x87 */ + 0xC4, 0x52, 0xBE, 0xA4, 0xA4, 0x4F, 0xBE, 0xE4, /* 0x88-0x8B */ + 0xBE, 0xFA, 0xF7, 0x65, 0xA6, 0x7E, 0xBC, 0xA6, /* 0x8C-0x8F */ + 0xC5, 0xCA, 0xBC, 0xBF, 0xBA, 0xA7, 0xB7, 0xD2, /* 0x90-0x93 */ + 0xE6, 0xA3, 0x00, 0x00, 0xBD, 0x6D, 0xC1, 0x70, /* 0x94-0x97 */ + 0xBD, 0xFB, 0xBD, 0xAC, 0xB3, 0x73, 0xC1, 0xE5, /* 0x98-0x9B */ + 0xA6, 0x43, 0xA6, 0x48, 0xAB, 0x7C, 0xAF, 0x50, /* 0x9C-0x9F */ + 0xB5, 0xF5, 0xBB, 0xA1, 0xB7, 0x47, 0xA9, 0xC0, /* 0xA0-0xA3 */ + 0xB1, 0xC9, 0xC0, 0xD4, 0xC3, 0xAE, 0xC2, 0x79, /* 0xA4-0xA7 */ + 0xA5, 0x4F, 0xCB, 0xF1, 0xB9, 0xE7, 0xC0, 0xAD, /* 0xA8-0xAB */ + 0xCC, 0xB0, 0xAC, 0xC2, 0xBC, 0xFC, 0xB2, 0xDC, /* 0xAC-0xAF */ + 0xB2, 0xE2, 0xB9, 0x61, 0xB9, 0x73, 0xC6, 0x46, /* 0xB0-0xB3 */ + 0xBB, 0xE2, 0xA8, 0xD2, 0xC2, 0xA7, 0xC4, 0xBF, /* 0xB4-0xB7 */ + 0xC1, 0xF5, 0xB4, 0x63, 0xA4, 0x46, 0xB9, 0xB1, /* 0xB8-0xBB */ + 0xBC, 0x64, 0xA7, 0xBF, 0xAE, 0xC6, 0xBC, 0xD6, /* 0xBC-0xBF */ + 0xBF, 0x52, 0xC0, 0xF8, 0xE7, 0x64, 0xBF, 0xF1, /* 0xC0-0xC3 */ + 0xC0, 0x73, 0xB7, 0x77, 0xA8, 0xBF, 0xBC, 0x42, /* 0xC4-0xC7 */ + 0xCC, 0xD8, 0xAC, 0x68, 0xAC, 0x79, 0xB7, 0xC8, /* 0xC8-0xCB */ + 0xAF, 0x5B, 0xAF, 0x64, 0xB2, 0xB8, 0xAF, 0xC3, /* 0xCC-0xCF */ + 0xC3, 0xFE, 0xA4, 0xBB, 0xBC, 0xAE, 0xB3, 0xB0, /* 0xD0-0xD3 */ + 0xAD, 0xDB, 0xB1, 0x5B, 0xB2, 0x5F, 0xBD, 0xFC, /* 0xD4-0xD7 */ + 0xAB, 0xDF, 0xB7, 0x58, 0xAE, 0xDF, 0xB2, 0x76, /* 0xD8-0xDB */ + 0xB6, 0xA9, 0xA7, 0x51, 0xA6, 0x4F, 0xBC, 0x69, /* 0xDC-0xDF */ + 0xA9, 0xF6, 0xA7, 0xF5, 0xB1, 0xF9, 0xAA, 0x64, /* 0xE0-0xE3 */ + 0xB2, 0x7A, 0xB5, 0x67, 0xBF, 0xA9, 0x00, 0x00, /* 0xE4-0xE7 */ + 0xB8, 0xCC, 0xA8, 0xBD, 0xC2, 0xF7, 0xB0, 0xCE, /* 0xE8-0xEB */ + 0xB7, 0xC4, 0xA7, 0x5B, 0xBF, 0x4D, 0xBF, 0x5A, /* 0xEC-0xEF */ + 0xC4, 0xA9, 0x00, 0x00, 0xC5, 0xEC, 0xC5, 0xEF, /* 0xF0-0xF3 */ + 0xAA, 0x4C, 0xB2, 0x4F, 0xC1, 0x7B, 0xA5, 0xDF, /* 0xF4-0xF7 */ + 0xB2, 0xC1, 0xB2, 0xC9, 0xAA, 0xAC, 0xAA, 0xA5, /* 0xF8-0xFB */ + 0xC3, 0xD1, 0xA4, 0xB0, 0xAF, 0xF9, 0xA8, 0xEB, /* 0xFC-0xFF */ +}; + +static const unsigned char u2c_FA[512] = { + 0xA4, 0xC1, 0xAB, 0xD7, 0xA9, 0xDD, 0xBF, 0x7D, /* 0x00-0x03 */ + 0xA6, 0x76, 0xAC, 0x7D, 0xBC, 0xC9, 0xBF, 0xE7, /* 0x04-0x07 */ + 0xA6, 0xE6, 0xAD, 0xB0, 0xA8, 0xA3, 0xB9, 0xF8, /* 0x08-0x0B */ + 0xC9, 0x4A, 0xDD, 0xFC, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0xB6, 0xEF, 0x00, 0x00, 0xB4, 0xB8, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0xE8, 0xF9, 0xBD, 0xDE, 0xAF, 0x71, /* 0x14-0x17 */ + 0x00, 0x00, 0xAF, 0xAB, 0xB2, 0xBB, 0xBA, 0xD6, /* 0x18-0x1B */ + 0xB9, 0x74, 0xBA, 0xEB, 0xA6, 0xD0, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0xBD, 0xD1, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0xB6, 0x68, 0xB3, 0xA3, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0xB6, 0xBA, 0xB9, 0x7D, /* 0x28-0x2B */ + 0xC0, 0x5D, 0xC5, 0x62, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ +}; + +static const unsigned char u2c_FE[512] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x03 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x04-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x0C-0x0F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x13 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x14-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x1C-0x1F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x23 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x24-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x2C-0x2F */ + 0xA1, 0x4A, 0xA1, 0x57, 0x00, 0x00, 0xA1, 0x59, /* 0x30-0x33 */ + 0xA1, 0x5B, 0xA1, 0x5F, 0xA1, 0x60, 0xA1, 0x63, /* 0x34-0x37 */ + 0xA1, 0x64, 0xA1, 0x67, 0xA1, 0x68, 0xA1, 0x6B, /* 0x38-0x3B */ + 0xA1, 0x6C, 0xA1, 0x6F, 0xA1, 0x70, 0xA1, 0x73, /* 0x3C-0x3F */ + 0xA1, 0x74, 0xA1, 0x77, 0xA1, 0x78, 0xA1, 0x7B, /* 0x40-0x43 */ + 0xA1, 0x7C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x44-0x47 */ + 0x00, 0x00, 0xA1, 0xC6, 0xA1, 0xC7, 0xA1, 0xCA, /* 0x48-0x4B */ + 0xA1, 0xCB, 0xA1, 0xC8, 0xA1, 0xC9, 0xA1, 0x5C, /* 0x4C-0x4F */ + 0xA1, 0x4D, 0xA1, 0x4E, 0xA1, 0x4F, 0x00, 0x00, /* 0x50-0x53 */ + 0xA1, 0x51, 0xA1, 0x52, 0xA1, 0x53, 0xA1, 0x54, /* 0x54-0x57 */ + 0x00, 0x00, 0xA1, 0x7D, 0xA1, 0x7E, 0xA1, 0xA1, /* 0x58-0x5B */ + 0xA1, 0xA2, 0xA1, 0xA3, 0xA1, 0xA4, 0xA1, 0xCC, /* 0x5C-0x5F */ + 0xA1, 0xCD, 0xA1, 0xCE, 0xA1, 0xDE, 0xA1, 0xDF, /* 0x60-0x63 */ + 0xA1, 0xE0, 0xA1, 0xE1, 0xA1, 0xE2, 0x00, 0x00, /* 0x64-0x67 */ + 0xA2, 0x42, 0xA2, 0x4C, 0xA2, 0x4D, 0xA2, 0x4E, /* 0x68-0x6B */ +}; + +static const unsigned char u2c_FF[512] = { + 0x00, 0x00, 0xA1, 0x49, 0xA1, 0xA8, 0xA1, 0xAD, /* 0x00-0x03 */ + 0xA2, 0x43, 0xA2, 0x48, 0xA1, 0xAE, 0xA1, 0xA6, /* 0x04-0x07 */ + 0xA1, 0x5D, 0xA1, 0x5E, 0xA1, 0xAF, 0xA1, 0xCF, /* 0x08-0x0B */ + 0xA1, 0x41, 0xA1, 0xD0, 0xA1, 0x44, 0xA1, 0xFE, /* 0x0C-0x0F */ + 0xA2, 0xAF, 0xA2, 0xB0, 0xA2, 0xB1, 0xA2, 0xB2, /* 0x10-0x13 */ + 0xA2, 0xB3, 0xA2, 0xB4, 0xA2, 0xB5, 0xA2, 0xB6, /* 0x14-0x17 */ + 0xA2, 0xB7, 0xA2, 0xB8, 0xA1, 0x47, 0xA1, 0x46, /* 0x18-0x1B */ + 0xA1, 0xD5, 0xA1, 0xD7, 0xA1, 0xD6, 0xA1, 0x48, /* 0x1C-0x1F */ + 0xA2, 0x49, 0xA2, 0xCF, 0xA2, 0xD0, 0xA2, 0xD1, /* 0x20-0x23 */ + 0xA2, 0xD2, 0xA2, 0xD3, 0xA2, 0xD4, 0xA2, 0xD5, /* 0x24-0x27 */ + 0xA2, 0xD6, 0xA2, 0xD7, 0xA2, 0xD8, 0xA2, 0xD9, /* 0x28-0x2B */ + 0xA2, 0xDA, 0xA2, 0xDB, 0xA2, 0xDC, 0xA2, 0xDD, /* 0x2C-0x2F */ + 0xA2, 0xDE, 0xA2, 0xDF, 0xA2, 0xE0, 0xA2, 0xE1, /* 0x30-0x33 */ + 0xA2, 0xE2, 0xA2, 0xE3, 0xA2, 0xE4, 0xA2, 0xE5, /* 0x34-0x37 */ + 0xA2, 0xE6, 0xA2, 0xE7, 0xA2, 0xE8, 0xA1, 0x65, /* 0x38-0x3B */ + 0xA2, 0x40, 0xA1, 0x66, 0xA1, 0x73, 0xA1, 0xC4, /* 0x3C-0x3F */ + 0xA1, 0xA5, 0xA2, 0xE9, 0xA2, 0xEA, 0xA2, 0xEB, /* 0x40-0x43 */ + 0xA2, 0xEC, 0xA2, 0xED, 0xA2, 0xEE, 0xA2, 0xEF, /* 0x44-0x47 */ + 0xA2, 0xF0, 0xA2, 0xF1, 0xA2, 0xF2, 0xA2, 0xF3, /* 0x48-0x4B */ + 0xA2, 0xF4, 0xA2, 0xF5, 0xA2, 0xF6, 0xA2, 0xF7, /* 0x4C-0x4F */ + 0xA2, 0xF8, 0xA2, 0xF9, 0xA2, 0xFA, 0xA2, 0xFB, /* 0x50-0x53 */ + 0xA2, 0xFC, 0xA2, 0xFD, 0xA2, 0xFE, 0xA3, 0x40, /* 0x54-0x57 */ + 0xA3, 0x41, 0xA3, 0x42, 0xA3, 0x43, 0xA1, 0x61, /* 0x58-0x5B */ + 0xA1, 0x55, 0xA1, 0x62, 0xA1, 0xE3, 0x00, 0x00, /* 0x5C-0x5F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x63 */ + 0xA1, 0x4E, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x64-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x6C-0x6F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x73 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x74-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x7C-0x7F */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x83 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x84-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x8C-0x8F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x93 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x94-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9B */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x9C-0x9F */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA0-0xA3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA4-0xA7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xA8-0xAB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xAC-0xAF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB0-0xB3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB4-0xB7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xB8-0xBB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xBC-0xBF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC0-0xC3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC4-0xC7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xC8-0xCB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xCC-0xCF */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD0-0xD3 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD4-0xD7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xD8-0xDB */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xDC-0xDF */ + 0xA2, 0x46, 0xA2, 0x47, 0x00, 0x00, 0xA1, 0xC3, /* 0xE0-0xE3 */ + 0x00, 0x00, 0xA2, 0x44, 0x00, 0x00, 0x00, 0x00, /* 0xE4-0xE7 */ +}; + +static const unsigned char *const page_uni2charset[256] = { + NULL, NULL, u2c_02, u2c_03, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + u2c_20, u2c_21, u2c_22, u2c_23, NULL, u2c_25, u2c_26, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + u2c_30, u2c_31, u2c_32, u2c_33, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, u2c_4E, u2c_4F, + u2c_50, u2c_51, u2c_52, u2c_53, u2c_54, u2c_55, u2c_56, u2c_57, + u2c_58, u2c_59, u2c_5A, u2c_5B, u2c_5C, u2c_5D, u2c_5E, u2c_5F, + u2c_60, u2c_61, u2c_62, u2c_63, u2c_64, u2c_65, u2c_66, u2c_67, + u2c_68, u2c_69, u2c_6A, u2c_6B, u2c_6C, u2c_6D, u2c_6E, u2c_6F, + u2c_70, u2c_71, u2c_72, u2c_73, u2c_74, u2c_75, u2c_76, u2c_77, + u2c_78, u2c_79, u2c_7A, u2c_7B, u2c_7C, u2c_7D, u2c_7E, u2c_7F, + u2c_80, u2c_81, u2c_82, u2c_83, u2c_84, u2c_85, u2c_86, u2c_87, + u2c_88, u2c_89, u2c_8A, u2c_8B, u2c_8C, u2c_8D, u2c_8E, u2c_8F, + u2c_90, u2c_91, u2c_92, u2c_93, u2c_94, u2c_95, u2c_96, u2c_97, + u2c_98, u2c_99, u2c_9A, u2c_9B, u2c_9C, u2c_9D, u2c_9E, u2c_9F, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, u2c_DC, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, u2c_F9, u2c_FA, NULL, NULL, NULL, u2c_FE, u2c_FF, }; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(const wchar_t uni, + unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni&0xFF; + unsigned char ch = (uni>>8)&0xFF; + int n; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + + uni2charset = page_uni2charset[ch]; + if (uni2charset) { + if (boundlen <= 1) + return -ENAMETOOLONG; + out[0] = uni2charset[cl*2]; + out[1] = uni2charset[cl*2+1]; + if (out[0] == 0x00 && out[1] == 0x00) + return -EINVAL; + n = 2; + } else if (ch==0 && cl) { + out[0] = cl; + n = 1; + } + else + return -EINVAL; + + return n; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, + wchar_t *uni) +{ + unsigned char ch, cl; + const wchar_t *charset2uni; + int n; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + if (boundlen == 1) { + *uni = rawstring[0]; + return 1; + } + + ch = rawstring[0]; + cl = rawstring[1]; + + charset2uni = page_charset2uni[ch]; + if (charset2uni && cl) { + *uni = charset2uni[cl]; + if (*uni == 0x0000) + return -EINVAL; + n = 2; + } else{ + *uni = ch; + n = 1; + } + return n; +} + +static struct nls_table table = { + .charset = "cp950", + .alias = "big5", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_cp950(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_cp950(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_cp950) +module_exit(exit_nls_cp950) + +MODULE_DESCRIPTION("NLS Traditional Chinese charset (Big5)"); +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_ALIAS_NLS(big5); diff --git a/fs/nls/nls_euc-jp.c b/fs/nls/nls_euc-jp.c new file mode 100644 index 000000000..0191cc9d9 --- /dev/null +++ b/fs/nls/nls_euc-jp.c @@ -0,0 +1,581 @@ +/* + * linux/fs/nls/nls_euc-jp.c + * + * Added `OSF/JVC Recommended Code Set Conversion Specification + * between Japanese EUC and Shift-JIS' support: + * (http://www.opengroup.or.jp/jvc/cde/sjis-euc-e.html) + */ + +#include +#include +#include +#include +#include + +static struct nls_table *p_nls; + +#define IS_SJIS_LOW_BYTE(l) ((0x40 <= (l)) && ((l) <= 0xFC) && ((l) != 0x7F)) +/* JIS X 0208 (include NEC spesial characters) */ +#define IS_SJIS_JISX0208(h, l) ((((0x81 <= (h)) && ((h) <= 0x9F)) \ + || ((0xE0 <= (h)) && ((h) <= 0xEA))) \ + && IS_SJIS_LOW_BYTE(l)) +#define IS_SJIS_JISX0201KANA(c) ((0xA1 <= (c)) && ((c) <= 0xDF)) +#define IS_SJIS_UDC_LOW(h, l) (((0xF0 <= (h)) && ((h) <= 0xF4)) \ + && IS_SJIS_LOW_BYTE(l)) +#define IS_SJIS_UDC_HI(h, l) (((0xF5 <= (h)) && ((h) <= 0xF9)) \ + && IS_SJIS_LOW_BYTE(l)) +#define IS_SJIS_IBM(h, l) (((0xFA <= (h)) && ((h) <= 0xFC)) \ + && IS_SJIS_LOW_BYTE(l)) +#define IS_SJIS_NECIBM(h, l) (((0xED <= (h)) && ((h) <= 0xEE)) \ + && IS_SJIS_LOW_BYTE(l)) +#define MAP_SJIS2EUC(sjis_hi, sjis_lo, sjis_p, euc_hi, euc_lo, euc_p) { \ + if ((sjis_lo) >= 0x9F) { \ + (euc_hi) = (sjis_hi) * 2 - (((sjis_p) * 2 - (euc_p)) - 1); \ + (euc_lo) = (sjis_lo) + 2; \ + } else { \ + (euc_hi) = (sjis_hi) * 2 - ((sjis_p) * 2 - (euc_p)); \ + (euc_lo) = (sjis_lo) + ((sjis_lo) >= 0x7F ? 0x60 : 0x61); \ + } \ +} while(0) + +#define SS2 (0x8E) /* Single Shift 2 */ +#define SS3 (0x8F) /* Single Shift 3 */ +#define IS_EUC_BYTE(c) ((0xA1 <= (c)) && ((c) <= 0xFE)) +#define IS_EUC_JISX0208(h, l) (IS_EUC_BYTE(h) && IS_EUC_BYTE(l)) +#define IS_EUC_JISX0201KANA(h, l) (((h) == SS2) && (0xA1 <= (l) && (l) <= 0xDF)) +#define IS_EUC_UDC_LOW(h, l) (((0xF5 <= (h)) && ((h) <= 0xFE)) \ + && IS_EUC_BYTE(l)) +#define IS_EUC_UDC_HI(h, l) IS_EUC_UDC_LOW(h, l) /* G3 block */ +#define MAP_EUC2SJIS(euc_hi, euc_lo, euc_p, sjis_hi, sjis_lo, sjis_p) { \ + if ((euc_hi) & 1) { \ + (sjis_hi) = (euc_hi) / 2 + ((sjis_p) - (euc_p) / 2); \ + (sjis_lo) = (euc_lo) - ((euc_lo) >= 0xE0 ? 0x60 : 0x61); \ + } else { \ + (sjis_hi) = (euc_hi) / 2 + (((sjis_p) - (euc_p) / 2) - 1); \ + (sjis_lo) = (euc_lo) - 2; \ + } \ +} while(0) + +/* SJIS IBM extended characters to EUC map */ +static const unsigned char sjisibm2euc_map[][2] = { + {0xF3, 0xF3}, {0xF3, 0xF4}, {0xF3, 0xF5}, {0xF3, 0xF6}, {0xF3, 0xF7}, + {0xF3, 0xF8}, {0xF3, 0xF9}, {0xF3, 0xFA}, {0xF3, 0xFB}, {0xF3, 0xFC}, + {0xF3, 0xFD}, {0xF3, 0xFE}, {0xF4, 0xA1}, {0xF4, 0xA2}, {0xF4, 0xA3}, + {0xF4, 0xA4}, {0xF4, 0xA5}, {0xF4, 0xA6}, {0xF4, 0xA7}, {0xF4, 0xA8}, + {0xA2, 0xCC}, {0xA2, 0xC3}, {0xF4, 0xA9}, {0xF4, 0xAA}, {0xF4, 0xAB}, + {0xF4, 0xAC}, {0xF4, 0xAD}, {0xA2, 0xE8}, {0xD4, 0xE3}, {0xDC, 0xDF}, + {0xE4, 0xE9}, {0xE3, 0xF8}, {0xD9, 0xA1}, {0xB1, 0xBB}, {0xF4, 0xAE}, + {0xC2, 0xAD}, {0xC3, 0xFC}, {0xE4, 0xD0}, {0xC2, 0xBF}, {0xBC, 0xF4}, + {0xB0, 0xA9}, {0xB0, 0xC8}, {0xF4, 0xAF}, {0xB0, 0xD2}, {0xB0, 0xD4}, + {0xB0, 0xE3}, {0xB0, 0xEE}, {0xB1, 0xA7}, {0xB1, 0xA3}, {0xB1, 0xAC}, + {0xB1, 0xA9}, {0xB1, 0xBE}, {0xB1, 0xDF}, {0xB1, 0xD8}, {0xB1, 0xC8}, + {0xB1, 0xD7}, {0xB1, 0xE3}, {0xB1, 0xF4}, {0xB1, 0xE1}, {0xB2, 0xA3}, + {0xF4, 0xB0}, {0xB2, 0xBB}, {0xB2, 0xE6}, {0x00, 0x00}, {0xB2, 0xED}, + {0xB2, 0xF5}, {0xB2, 0xFC}, {0xF4, 0xB1}, {0xB3, 0xB5}, {0xB3, 0xD8}, + {0xB3, 0xDB}, {0xB3, 0xE5}, {0xB3, 0xEE}, {0xB3, 0xFB}, {0xF4, 0xB2}, + {0xF4, 0xB3}, {0xB4, 0xC0}, {0xB4, 0xC7}, {0xB4, 0xD0}, {0xB4, 0xDE}, + {0xF4, 0xB4}, {0xB5, 0xAA}, {0xF4, 0xB5}, {0xB5, 0xAF}, {0xB5, 0xC4}, + {0xB5, 0xE8}, {0xF4, 0xB6}, {0xB7, 0xC2}, {0xB7, 0xE4}, {0xB7, 0xE8}, + {0xB7, 0xE7}, {0xF4, 0xB7}, {0xF4, 0xB8}, {0xF4, 0xB9}, {0xB8, 0xCE}, + {0xB8, 0xE1}, {0xB8, 0xF5}, {0xB8, 0xF7}, {0xB8, 0xF8}, {0xB8, 0xFC}, + {0xB9, 0xAF}, {0xB9, 0xB7}, {0xBA, 0xBE}, {0xBA, 0xDB}, {0xCD, 0xAA}, + {0xBA, 0xE1}, {0xF4, 0xBA}, {0xBA, 0xEB}, {0xBB, 0xB3}, {0xBB, 0xB8}, + {0xF4, 0xBB}, {0xBB, 0xCA}, {0xF4, 0xBC}, {0xF4, 0xBD}, {0xBB, 0xD0}, + {0xBB, 0xDE}, {0xBB, 0xF4}, {0xBB, 0xF5}, {0xBB, 0xF9}, {0xBC, 0xE4}, + {0xBC, 0xED}, {0xBC, 0xFE}, {0xF4, 0xBE}, {0xBD, 0xC2}, {0xBD, 0xE7}, + {0xF4, 0xBF}, {0xBD, 0xF0}, {0xBE, 0xB0}, {0xBE, 0xAC}, {0xF4, 0xC0}, + {0xBE, 0xB3}, {0xBE, 0xBD}, {0xBE, 0xCD}, {0xBE, 0xC9}, {0xBE, 0xE4}, + {0xBF, 0xA8}, {0xBF, 0xC9}, {0xC0, 0xC4}, {0xC0, 0xE4}, {0xC0, 0xF4}, + {0xC1, 0xA6}, {0xF4, 0xC1}, {0xC1, 0xF5}, {0xC1, 0xFC}, {0xF4, 0xC2}, + {0xC1, 0xF8}, {0xC2, 0xAB}, {0xC2, 0xA1}, {0xC2, 0xA5}, {0xF4, 0xC3}, + {0xC2, 0xB8}, {0xC2, 0xBA}, {0xF4, 0xC4}, {0xC2, 0xC4}, {0xC2, 0xD2}, + {0xC2, 0xD7}, {0xC2, 0xDB}, {0xC2, 0xDE}, {0xC2, 0xED}, {0xC2, 0xF0}, + {0xF4, 0xC5}, {0xC3, 0xA1}, {0xC3, 0xB5}, {0xC3, 0xC9}, {0xC3, 0xB9}, + {0xF4, 0xC6}, {0xC3, 0xD8}, {0xC3, 0xFE}, {0xF4, 0xC7}, {0xC4, 0xCC}, + {0xF4, 0xC8}, {0xC4, 0xD9}, {0xC4, 0xEA}, {0xC4, 0xFD}, {0xF4, 0xC9}, + {0xC5, 0xA7}, {0xC5, 0xB5}, {0xC5, 0xB6}, {0xF4, 0xCA}, {0xC5, 0xD5}, + {0xC6, 0xB8}, {0xC6, 0xD7}, {0xC6, 0xE0}, {0xC6, 0xEA}, {0xC6, 0xE3}, + {0xC7, 0xA1}, {0xC7, 0xAB}, {0xC7, 0xC7}, {0xC7, 0xC3}, {0xC7, 0xCB}, + {0xC7, 0xCF}, {0xC7, 0xD9}, {0xF4, 0xCB}, {0xF4, 0xCC}, {0xC7, 0xE6}, + {0xC7, 0xEE}, {0xC7, 0xFC}, {0xC7, 0xEB}, {0xC7, 0xF0}, {0xC8, 0xB1}, + {0xC8, 0xE5}, {0xC8, 0xF8}, {0xC9, 0xA6}, {0xC9, 0xAB}, {0xC9, 0xAD}, + {0xF4, 0xCD}, {0xC9, 0xCA}, {0xC9, 0xD3}, {0xC9, 0xE9}, {0xC9, 0xE3}, + {0xC9, 0xFC}, {0xC9, 0xF4}, {0xC9, 0xF5}, {0xF4, 0xCE}, {0xCA, 0xB3}, + {0xCA, 0xBD}, {0xCA, 0xEF}, {0xCA, 0xF1}, {0xCB, 0xAE}, {0xF4, 0xCF}, + {0xCB, 0xCA}, {0xCB, 0xE6}, {0xCB, 0xEA}, {0xCB, 0xF0}, {0xCB, 0xF4}, + {0xCB, 0xEE}, {0xCC, 0xA5}, {0xCB, 0xF9}, {0xCC, 0xAB}, {0xCC, 0xAE}, + {0xCC, 0xAD}, {0xCC, 0xB2}, {0xCC, 0xC2}, {0xCC, 0xD0}, {0xCC, 0xD9}, + {0xF4, 0xD0}, {0xCD, 0xBB}, {0xF4, 0xD1}, {0xCE, 0xBB}, {0xF4, 0xD2}, + {0xCE, 0xBA}, {0xCE, 0xC3}, {0xF4, 0xD3}, {0xCE, 0xF2}, {0xB3, 0xDD}, + {0xCF, 0xD5}, {0xCF, 0xE2}, {0xCF, 0xE9}, {0xCF, 0xED}, {0xF4, 0xD4}, + {0xF4, 0xD5}, {0xF4, 0xD6}, {0x00, 0x00}, {0xF4, 0xD7}, {0xD0, 0xE5}, + {0xF4, 0xD8}, {0xD0, 0xE9}, {0xD1, 0xE8}, {0xF4, 0xD9}, {0xF4, 0xDA}, + {0xD1, 0xEC}, {0xD2, 0xBB}, {0xF4, 0xDB}, {0xD3, 0xE1}, {0xD3, 0xE8}, + {0xD4, 0xA7}, {0xF4, 0xDC}, {0xF4, 0xDD}, {0xD4, 0xD4}, {0xD4, 0xF2}, + {0xD5, 0xAE}, {0xF4, 0xDE}, {0xD7, 0xDE}, {0xF4, 0xDF}, {0xD8, 0xA2}, + {0xD8, 0xB7}, {0xD8, 0xC1}, {0xD8, 0xD1}, {0xD8, 0xF4}, {0xD9, 0xC6}, + {0xD9, 0xC8}, {0xD9, 0xD1}, {0xF4, 0xE0}, {0xF4, 0xE1}, {0xF4, 0xE2}, + {0xF4, 0xE3}, {0xF4, 0xE4}, {0xDC, 0xD3}, {0xDD, 0xC8}, {0xDD, 0xD4}, + {0xDD, 0xEA}, {0xDD, 0xFA}, {0xDE, 0xA4}, {0xDE, 0xB0}, {0xF4, 0xE5}, + {0xDE, 0xB5}, {0xDE, 0xCB}, {0xF4, 0xE6}, {0xDF, 0xB9}, {0xF4, 0xE7}, + {0xDF, 0xC3}, {0xF4, 0xE8}, {0xF4, 0xE9}, {0xE0, 0xD9}, {0xF4, 0xEA}, + {0xF4, 0xEB}, {0xE1, 0xE2}, {0xF4, 0xEC}, {0xF4, 0xED}, {0xF4, 0xEE}, + {0xE2, 0xC7}, {0xE3, 0xA8}, {0xE3, 0xA6}, {0xE3, 0xA9}, {0xE3, 0xAF}, + {0xE3, 0xB0}, {0xE3, 0xAA}, {0xE3, 0xAB}, {0xE3, 0xBC}, {0xE3, 0xC1}, + {0xE3, 0xBF}, {0xE3, 0xD5}, {0xE3, 0xD8}, {0xE3, 0xD6}, {0xE3, 0xDF}, + {0xE3, 0xE3}, {0xE3, 0xE1}, {0xE3, 0xD4}, {0xE3, 0xE9}, {0xE4, 0xA6}, + {0xE3, 0xF1}, {0xE3, 0xF2}, {0xE4, 0xCB}, {0xE4, 0xC1}, {0xE4, 0xC3}, + {0xE4, 0xBE}, {0xF4, 0xEF}, {0xE4, 0xC0}, {0xE4, 0xC7}, {0xE4, 0xBF}, + {0xE4, 0xE0}, {0xE4, 0xDE}, {0xE4, 0xD1}, {0xF4, 0xF0}, {0xE4, 0xDC}, + {0xE4, 0xD2}, {0xE4, 0xDB}, {0xE4, 0xD4}, {0xE4, 0xFA}, {0xE4, 0xEF}, + {0xE5, 0xB3}, {0xE5, 0xBF}, {0xE5, 0xC9}, {0xE5, 0xD0}, {0xE5, 0xE2}, + {0xE5, 0xEA}, {0xE5, 0xEB}, {0xF4, 0xF1}, {0xF4, 0xF2}, {0xF4, 0xF3}, + {0xE6, 0xE8}, {0xE6, 0xEF}, {0xE7, 0xAC}, {0xF4, 0xF4}, {0xE7, 0xAE}, + {0xF4, 0xF5}, {0xE7, 0xB1}, {0xF4, 0xF6}, {0xE7, 0xB2}, {0xE8, 0xB1}, + {0xE8, 0xB6}, {0xF4, 0xF7}, {0xF4, 0xF8}, {0xE8, 0xDD}, {0xF4, 0xF9}, + {0xF4, 0xFA}, {0xE9, 0xD1}, {0xF4, 0xFB}, {0xE9, 0xED}, {0xEA, 0xCD}, + {0xF4, 0xFC}, {0xEA, 0xDB}, {0xEA, 0xE6}, {0xEA, 0xEA}, {0xEB, 0xA5}, + {0xEB, 0xFB}, {0xEB, 0xFA}, {0xF4, 0xFD}, {0xEC, 0xD6}, {0xF4, 0xFE}, +}; + +#define IS_EUC_IBM2JISX0208(h, l) \ + (((h) == 0xA2 && (l) == 0xCC) || ((h) == 0xA2 && (l) == 0xE8)) + +/* EUC to SJIS IBM extended characters map (G3 JIS X 0212 block) */ +static struct { + unsigned short euc; + unsigned char sjis[2]; +} euc2sjisibm_jisx0212_map[] = { + {0xA2C3, {0xFA, 0x55}}, {0xB0A9, {0xFA, 0x68}}, {0xB0C8, {0xFA, 0x69}}, + {0xB0D2, {0xFA, 0x6B}}, {0xB0D4, {0xFA, 0x6C}}, {0xB0E3, {0xFA, 0x6D}}, + {0xB0EE, {0xFA, 0x6E}}, {0xB1A3, {0xFA, 0x70}}, {0xB1A7, {0xFA, 0x6F}}, + {0xB1A9, {0xFA, 0x72}}, {0xB1AC, {0xFA, 0x71}}, {0xB1BB, {0xFA, 0x61}}, + {0xB1BE, {0xFA, 0x73}}, {0xB1C8, {0xFA, 0x76}}, {0xB1D7, {0xFA, 0x77}}, + {0xB1D8, {0xFA, 0x75}}, {0xB1DF, {0xFA, 0x74}}, {0xB1E1, {0xFA, 0x7A}}, + {0xB1E3, {0xFA, 0x78}}, {0xB1F4, {0xFA, 0x79}}, {0xB2A3, {0xFA, 0x7B}}, + {0xB2BB, {0xFA, 0x7D}}, {0xB2E6, {0xFA, 0x7E}}, {0xB2ED, {0xFA, 0x80}}, + {0xB2F5, {0xFA, 0x81}}, {0xB2FC, {0xFA, 0x82}}, {0xB3B5, {0xFA, 0x84}}, + {0xB3D8, {0xFA, 0x85}}, {0xB3DB, {0xFA, 0x86}}, {0xB3DD, {0xFB, 0x77}}, + {0xB3E5, {0xFA, 0x87}}, {0xB3EE, {0xFA, 0x88}}, {0xB3FB, {0xFA, 0x89}}, + {0xB4C0, {0xFA, 0x8C}}, {0xB4C7, {0xFA, 0x8D}}, {0xB4D0, {0xFA, 0x8E}}, + {0xB4DE, {0xFA, 0x8F}}, {0xB5AA, {0xFA, 0x91}}, {0xB5AF, {0xFA, 0x93}}, + {0xB5C4, {0xFA, 0x94}}, {0xB5E8, {0xFA, 0x95}}, {0xB7C2, {0xFA, 0x97}}, + {0xB7E4, {0xFA, 0x98}}, {0xB7E7, {0xFA, 0x9A}}, {0xB7E8, {0xFA, 0x99}}, + {0xB8CE, {0xFA, 0x9E}}, {0xB8E1, {0xFA, 0x9F}}, {0xB8F5, {0xFA, 0xA0}}, + {0xB8F7, {0xFA, 0xA1}}, {0xB8F8, {0xFA, 0xA2}}, {0xB8FC, {0xFA, 0xA3}}, + {0xB9AF, {0xFA, 0xA4}}, {0xB9B7, {0xFA, 0xA5}}, {0xBABE, {0xFA, 0xA6}}, + {0xBADB, {0xFA, 0xA7}}, {0xBAE1, {0xFA, 0xA9}}, {0xBAEB, {0xFA, 0xAB}}, + {0xBBB3, {0xFA, 0xAC}}, {0xBBB8, {0xFA, 0xAD}}, {0xBBCA, {0xFA, 0xAF}}, + {0xBBD0, {0xFA, 0xB2}}, {0xBBDE, {0xFA, 0xB3}}, {0xBBF4, {0xFA, 0xB4}}, + {0xBBF5, {0xFA, 0xB5}}, {0xBBF9, {0xFA, 0xB6}}, {0xBCE4, {0xFA, 0xB7}}, + {0xBCED, {0xFA, 0xB8}}, {0xBCF4, {0xFA, 0x67}}, {0xBCFE, {0xFA, 0xB9}}, + {0xBDC2, {0xFA, 0xBB}}, {0xBDE7, {0xFA, 0xBC}}, {0xBDF0, {0xFA, 0xBE}}, + {0xBEAC, {0xFA, 0xC0}}, {0xBEB0, {0xFA, 0xBF}}, {0xBEB3, {0xFA, 0xC2}}, + {0xBEBD, {0xFA, 0xC3}}, {0xBEC9, {0xFA, 0xC5}}, {0xBECD, {0xFA, 0xC4}}, + {0xBEE4, {0xFA, 0xC6}}, {0xBFA8, {0xFA, 0xC7}}, {0xBFC9, {0xFA, 0xC8}}, + {0xC0C4, {0xFA, 0xC9}}, {0xC0E4, {0xFA, 0xCA}}, {0xC0F4, {0xFA, 0xCB}}, + {0xC1A6, {0xFA, 0xCC}}, {0xC1F5, {0xFA, 0xCE}}, {0xC1F8, {0xFA, 0xD1}}, + {0xC1FC, {0xFA, 0xCF}}, {0xC2A1, {0xFA, 0xD3}}, {0xC2A5, {0xFA, 0xD4}}, + {0xC2AB, {0xFA, 0xD2}}, {0xC2AD, {0xFA, 0x63}}, {0xC2B8, {0xFA, 0xD6}}, + {0xC2BA, {0xFA, 0xD7}}, {0xC2BF, {0xFA, 0x66}}, {0xC2C4, {0xFA, 0xD9}}, + {0xC2D2, {0xFA, 0xDA}}, {0xC2D7, {0xFA, 0xDB}}, {0xC2DB, {0xFA, 0xDC}}, + {0xC2DE, {0xFA, 0xDD}}, {0xC2ED, {0xFA, 0xDE}}, {0xC2F0, {0xFA, 0xDF}}, + {0xC3A1, {0xFA, 0xE1}}, {0xC3B5, {0xFA, 0xE2}}, {0xC3B9, {0xFA, 0xE4}}, + {0xC3C9, {0xFA, 0xE3}}, {0xC3D8, {0xFA, 0xE6}}, {0xC3FC, {0xFA, 0x64}}, + {0xC3FE, {0xFA, 0xE7}}, {0xC4CC, {0xFA, 0xE9}}, {0xC4D9, {0xFA, 0xEB}}, + {0xC4EA, {0xFA, 0xEC}}, {0xC4FD, {0xFA, 0xED}}, {0xC5A7, {0xFA, 0xEF}}, + {0xC5B5, {0xFA, 0xF0}}, {0xC5B6, {0xFA, 0xF1}}, {0xC5D5, {0xFA, 0xF3}}, + {0xC6B8, {0xFA, 0xF4}}, {0xC6D7, {0xFA, 0xF5}}, {0xC6E0, {0xFA, 0xF6}}, + {0xC6E3, {0xFA, 0xF8}}, {0xC6EA, {0xFA, 0xF7}}, {0xC7A1, {0xFA, 0xF9}}, + {0xC7AB, {0xFA, 0xFA}}, {0xC7C3, {0xFA, 0xFC}}, {0xC7C7, {0xFA, 0xFB}}, + {0xC7CB, {0xFB, 0x40}}, {0xC7CF, {0xFB, 0x41}}, {0xC7D9, {0xFB, 0x42}}, + {0xC7E6, {0xFB, 0x45}}, {0xC7EB, {0xFB, 0x48}}, {0xC7EE, {0xFB, 0x46}}, + {0xC7F0, {0xFB, 0x49}}, {0xC7FC, {0xFB, 0x47}}, {0xC8B1, {0xFB, 0x4A}}, + {0xC8E5, {0xFB, 0x4B}}, {0xC8F8, {0xFB, 0x4C}}, {0xC9A6, {0xFB, 0x4D}}, + {0xC9AB, {0xFB, 0x4E}}, {0xC9AD, {0xFB, 0x4F}}, {0xC9CA, {0xFB, 0x51}}, + {0xC9D3, {0xFB, 0x52}}, {0xC9E3, {0xFB, 0x54}}, {0xC9E9, {0xFB, 0x53}}, + {0xC9F4, {0xFB, 0x56}}, {0xC9F5, {0xFB, 0x57}}, {0xC9FC, {0xFB, 0x55}}, + {0xCAB3, {0xFB, 0x59}}, {0xCABD, {0xFB, 0x5A}}, {0xCAEF, {0xFB, 0x5B}}, + {0xCAF1, {0xFB, 0x5C}}, {0xCBAE, {0xFB, 0x5D}}, {0xCBCA, {0xFB, 0x5F}}, + {0xCBE6, {0xFB, 0x60}}, {0xCBEA, {0xFB, 0x61}}, {0xCBEE, {0xFB, 0x64}}, + {0xCBF0, {0xFB, 0x62}}, {0xCBF4, {0xFB, 0x63}}, {0xCBF9, {0xFB, 0x66}}, + {0xCCA5, {0xFB, 0x65}}, {0xCCAB, {0xFB, 0x67}}, {0xCCAD, {0xFB, 0x69}}, + {0xCCAE, {0xFB, 0x68}}, {0xCCB2, {0xFB, 0x6A}}, {0xCCC2, {0xFB, 0x6B}}, + {0xCCD0, {0xFB, 0x6C}}, {0xCCD9, {0xFB, 0x6D}}, {0xCDAA, {0xFA, 0xA8}}, + {0xCDBB, {0xFB, 0x6F}}, {0xCEBA, {0xFB, 0x73}}, {0xCEBB, {0xFB, 0x71}}, + {0xCEC3, {0xFB, 0x74}}, {0xCEF2, {0xFB, 0x76}}, {0xCFD5, {0xFB, 0x78}}, + {0xCFE2, {0xFB, 0x79}}, {0xCFE9, {0xFB, 0x7A}}, {0xCFED, {0xFB, 0x7B}}, + {0xD0E5, {0xFB, 0x81}}, {0xD0E9, {0xFB, 0x83}}, {0xD1E8, {0xFB, 0x84}}, + {0xD1EC, {0xFB, 0x87}}, {0xD2BB, {0xFB, 0x88}}, {0xD3E1, {0xFB, 0x8A}}, + {0xD3E8, {0xFB, 0x8B}}, {0xD4A7, {0xFB, 0x8C}}, {0xD4D4, {0xFB, 0x8F}}, + {0xD4E3, {0xFA, 0x5C}}, {0xD4F2, {0xFB, 0x90}}, {0xD5AE, {0xFB, 0x91}}, + {0xD7DE, {0xFB, 0x93}}, {0xD8A2, {0xFB, 0x95}}, {0xD8B7, {0xFB, 0x96}}, + {0xD8C1, {0xFB, 0x97}}, {0xD8D1, {0xFB, 0x98}}, {0xD8F4, {0xFB, 0x99}}, + {0xD9A1, {0xFA, 0x60}}, {0xD9C6, {0xFB, 0x9A}}, {0xD9C8, {0xFB, 0x9B}}, + {0xD9D1, {0xFB, 0x9C}}, {0xDCD3, {0xFB, 0xA2}}, {0xDCDF, {0xFA, 0x5D}}, + {0xDDC8, {0xFB, 0xA3}}, {0xDDD4, {0xFB, 0xA4}}, {0xDDEA, {0xFB, 0xA5}}, + {0xDDFA, {0xFB, 0xA6}}, {0xDEA4, {0xFB, 0xA7}}, {0xDEB0, {0xFB, 0xA8}}, + {0xDEB5, {0xFB, 0xAA}}, {0xDECB, {0xFB, 0xAB}}, {0xDFB9, {0xFB, 0xAD}}, + {0xDFC3, {0xFB, 0xAF}}, {0xE0D9, {0xFB, 0xB2}}, {0xE1E2, {0xFB, 0xB5}}, + {0xE2C7, {0xFB, 0xB9}}, {0xE3A6, {0xFB, 0xBB}}, {0xE3A8, {0xFB, 0xBA}}, + {0xE3A9, {0xFB, 0xBC}}, {0xE3AA, {0xFB, 0xBF}}, {0xE3AB, {0xFB, 0xC0}}, + {0xE3AF, {0xFB, 0xBD}}, {0xE3B0, {0xFB, 0xBE}}, {0xE3BC, {0xFB, 0xC1}}, + {0xE3BF, {0xFB, 0xC3}}, {0xE3C1, {0xFB, 0xC2}}, {0xE3D4, {0xFB, 0xCA}}, + {0xE3D5, {0xFB, 0xC4}}, {0xE3D6, {0xFB, 0xC6}}, {0xE3D8, {0xFB, 0xC5}}, + {0xE3DF, {0xFB, 0xC7}}, {0xE3E1, {0xFB, 0xC9}}, {0xE3E3, {0xFB, 0xC8}}, + {0xE3E9, {0xFB, 0xCB}}, {0xE3F1, {0xFB, 0xCD}}, {0xE3F2, {0xFB, 0xCE}}, + {0xE3F8, {0xFA, 0x5F}}, {0xE4A6, {0xFB, 0xCC}}, {0xE4BE, {0xFB, 0xD2}}, + {0xE4BF, {0xFB, 0xD6}}, {0xE4C0, {0xFB, 0xD4}}, {0xE4C1, {0xFB, 0xD0}}, + {0xE4C3, {0xFB, 0xD1}}, {0xE4C7, {0xFB, 0xD5}}, {0xE4CB, {0xFB, 0xCF}}, + {0xE4D0, {0xFA, 0x65}}, {0xE4D1, {0xFB, 0xD9}}, {0xE4D2, {0xFB, 0xDC}}, + {0xE4D4, {0xFB, 0xDE}}, {0xE4DB, {0xFB, 0xDD}}, {0xE4DC, {0xFB, 0xDB}}, + {0xE4DE, {0xFB, 0xD8}}, {0xE4E0, {0xFB, 0xD7}}, {0xE4E9, {0xFA, 0x5E}}, + {0xE4EF, {0xFB, 0xE0}}, {0xE4FA, {0xFB, 0xDF}}, {0xE5B3, {0xFB, 0xE1}}, + {0xE5BF, {0xFB, 0xE2}}, {0xE5C9, {0xFB, 0xE3}}, {0xE5D0, {0xFB, 0xE4}}, + {0xE5E2, {0xFB, 0xE5}}, {0xE5EA, {0xFB, 0xE6}}, {0xE5EB, {0xFB, 0xE7}}, + {0xE6E8, {0xFB, 0xEB}}, {0xE6EF, {0xFB, 0xEC}}, {0xE7AC, {0xFB, 0xED}}, + {0xE7AE, {0xFB, 0xEF}}, {0xE7B1, {0xFB, 0xF1}}, {0xE7B2, {0xFB, 0xF3}}, + {0xE8B1, {0xFB, 0xF4}}, {0xE8B6, {0xFB, 0xF5}}, {0xE8DD, {0xFB, 0xF8}}, + {0xE9D1, {0xFB, 0xFB}}, {0xE9ED, {0xFC, 0x40}}, {0xEACD, {0xFC, 0x41}}, + {0xEADB, {0xFC, 0x43}}, {0xEAE6, {0xFC, 0x44}}, {0xEAEA, {0xFC, 0x45}}, + {0xEBA5, {0xFC, 0x46}}, {0xEBFA, {0xFC, 0x48}}, {0xEBFB, {0xFC, 0x47}}, + {0xECD6, {0xFC, 0x4A}}, +}; + +/* EUC to SJIS IBM extended characters map (G3 Upper block) */ +static const unsigned char euc2sjisibm_g3upper_map[][2] = { + {0xFA, 0x40}, {0xFA, 0x41}, {0xFA, 0x42}, {0xFA, 0x43}, {0xFA, 0x44}, + {0xFA, 0x45}, {0xFA, 0x46}, {0xFA, 0x47}, {0xFA, 0x48}, {0xFA, 0x49}, + {0xFA, 0x4A}, {0xFA, 0x4B}, {0xFA, 0x4C}, {0xFA, 0x4D}, {0xFA, 0x4E}, + {0xFA, 0x4F}, {0xFA, 0x50}, {0xFA, 0x51}, {0xFA, 0x52}, {0xFA, 0x53}, + {0xFA, 0x56}, {0xFA, 0x57}, {0xFA, 0x58}, {0xFA, 0x59}, {0xFA, 0x5A}, + {0xFA, 0x62}, {0xFA, 0x6A}, {0xFA, 0x7C}, {0xFA, 0x83}, {0xFA, 0x8A}, + {0xFA, 0x8B}, {0xFA, 0x90}, {0xFA, 0x92}, {0xFA, 0x96}, {0xFA, 0x9B}, + {0xFA, 0x9C}, {0xFA, 0x9D}, {0xFA, 0xAA}, {0xFA, 0xAE}, {0xFA, 0xB0}, + {0xFA, 0xB1}, {0xFA, 0xBA}, {0xFA, 0xBD}, {0xFA, 0xC1}, {0xFA, 0xCD}, + {0xFA, 0xD0}, {0xFA, 0xD5}, {0xFA, 0xD8}, {0xFA, 0xE0}, {0xFA, 0xE5}, + {0xFA, 0xE8}, {0xFA, 0xEA}, {0xFA, 0xEE}, {0xFA, 0xF2}, {0xFB, 0x43}, + {0xFB, 0x44}, {0xFB, 0x50}, {0xFB, 0x58}, {0xFB, 0x5E}, {0xFB, 0x6E}, + {0xFB, 0x70}, {0xFB, 0x72}, {0xFB, 0x75}, {0xFB, 0x7C}, {0xFB, 0x7D}, + {0xFB, 0x7E}, {0xFB, 0x80}, {0xFB, 0x82}, {0xFB, 0x85}, {0xFB, 0x86}, + {0xFB, 0x89}, {0xFB, 0x8D}, {0xFB, 0x8E}, {0xFB, 0x92}, {0xFB, 0x94}, + {0xFB, 0x9D}, {0xFB, 0x9E}, {0xFB, 0x9F}, {0xFB, 0xA0}, {0xFB, 0xA1}, + {0xFB, 0xA9}, {0xFB, 0xAC}, {0xFB, 0xAE}, {0xFB, 0xB0}, {0xFB, 0xB1}, + {0xFB, 0xB3}, {0xFB, 0xB4}, {0xFB, 0xB6}, {0xFB, 0xB7}, {0xFB, 0xB8}, + {0xFB, 0xD3}, {0xFB, 0xDA}, {0xFB, 0xE8}, {0xFB, 0xE9}, {0xFB, 0xEA}, + {0xFB, 0xEE}, {0xFB, 0xF0}, {0xFB, 0xF2}, {0xFB, 0xF6}, {0xFB, 0xF7}, + {0xFB, 0xF9}, {0xFB, 0xFA}, {0xFB, 0xFC}, {0xFC, 0x42}, {0xFC, 0x49}, + {0xFC, 0x4B}, +}; + +static inline int sjisibm2euc(unsigned char *euc, const unsigned char sjis_hi, + const unsigned char sjis_lo); +static inline int euc2sjisibm_jisx0212(unsigned char *sjis, const unsigned char euc_hi, + const unsigned char euc_lo); +static inline int euc2sjisibm_g3upper(unsigned char *sjis, const unsigned char euc_hi, + const unsigned char euc_lo); +static inline int euc2sjisibm(unsigned char *sjis, const unsigned char euc_hi, + const unsigned char euc_lo); +static inline int sjisnec2sjisibm(unsigned char *sjisibm, + const unsigned char sjisnec_hi, + const unsigned char sjisnec_lo); + +/* SJIS IBM extended characters to EUC */ +static inline int sjisibm2euc(unsigned char *euc, const unsigned char sjis_hi, + const unsigned char sjis_lo) +{ + int index; + + index = ((sjis_hi - 0xFA) * (0xFD - 0x40)) + (sjis_lo - 0x40); + if (IS_EUC_IBM2JISX0208(sjisibm2euc_map[index][0], + sjisibm2euc_map[index][1])) { + euc[0] = sjisibm2euc_map[index][0]; + euc[1] = sjisibm2euc_map[index][1]; + return 2; + } else { + euc[0] = SS3; + euc[1] = sjisibm2euc_map[index][0]; + euc[2] = sjisibm2euc_map[index][1]; + return 3; + } +} + +/* EUC to SJIS IBM extended characters (G3 JIS X 0212 block) */ +static inline int euc2sjisibm_jisx0212(unsigned char *sjis, const unsigned char euc_hi, + const unsigned char euc_lo) +{ + int index, min_index, max_index; + unsigned short euc; + + min_index = 0; + max_index = ARRAY_SIZE(euc2sjisibm_jisx0212_map) - 1; + euc = (euc_hi << 8) | euc_lo; + + while (min_index <= max_index) { + index = (min_index + max_index) / 2; + if (euc < euc2sjisibm_jisx0212_map[index].euc) + max_index = index - 1; + else + min_index = index + 1; + if (euc == euc2sjisibm_jisx0212_map[index].euc) { + sjis[0] = euc2sjisibm_jisx0212_map[index].sjis[0]; + sjis[1] = euc2sjisibm_jisx0212_map[index].sjis[1]; + return 3; + } + } + return 0; +} + +/* EUC to SJIS IBM extended characters (G3 Upper block) */ +static inline int euc2sjisibm_g3upper(unsigned char *sjis, const unsigned char euc_hi, + const unsigned char euc_lo) +{ + int index; + + if (euc_hi == 0xF3) + index = ((euc_hi << 8) | euc_lo) - 0xF3F3; + else + index = ((euc_hi << 8) | euc_lo) - 0xF4A1 + 12; + + if ((index < 0) || (index >= ARRAY_SIZE(euc2sjisibm_g3upper_map))) + return 0; + + sjis[0] = euc2sjisibm_g3upper_map[index][0]; + sjis[1] = euc2sjisibm_g3upper_map[index][1]; + + return 3; +} + +/* EUC to SJIS IBM extended characters (G3 block) */ +static inline int euc2sjisibm(unsigned char *sjis, const unsigned char euc_hi, + const unsigned char euc_lo) +{ + int n; + +#if 0 + if ((euc_hi == 0xA2) && (euc_lo == 0xCC)) { + sjis[0] = 0xFA; + sjis[1] = 0x54; + return 2; + } else if ((euc_hi == 0xA2) && (euc_lo == 0xE8)) { + sjis[0] = 0xFA; + sjis[1] = 0x5B; + return 2; + } +#endif + if ((n = euc2sjisibm_g3upper(sjis, euc_hi, euc_lo))) { + return n; + } else if ((n = euc2sjisibm_jisx0212(sjis, euc_hi, euc_lo))) { + return n; + } + + return 0; +} + +/* NEC/IBM extended characters to IBM extended characters */ +static inline int sjisnec2sjisibm(unsigned char *sjisibm, + const unsigned char sjisnec_hi, + const unsigned char sjisnec_lo) +{ + int count; + + if (! IS_SJIS_NECIBM(sjisnec_hi, sjisnec_lo)) + return 0; + + if ((sjisnec_hi == 0xEE) && (sjisnec_lo == 0xF9)) { + sjisibm[0] = 0x81; + sjisibm[1] = 0xCA; + return 2; + } + + if ((sjisnec_hi == 0xEE) && (sjisnec_lo >= 0xEF)) { + count = (sjisnec_hi << 8 | sjisnec_lo) + - (sjisnec_lo <= 0xF9 ? 0xEEEF : (0xEEEF - 10)); + } else { + count = (sjisnec_hi - 0xED) * (0xFC - 0x40) + + (sjisnec_lo - 0x40) + (0x5C - 0x40); + if (sjisnec_lo >= 0x7F) + count--; + } + + sjisibm[0] = 0xFA + (count / (0xFC - 0x40)); + sjisibm[1] = 0x40 + (count % (0xFC - 0x40)); + if (sjisibm[1] >= 0x7F) + sjisibm[1]++; + + return 2; +} + +static int uni2char(const wchar_t uni, + unsigned char *out, int boundlen) +{ + int n; + + if (!p_nls) + return -EINVAL; + if ((n = p_nls->uni2char(uni, out, boundlen)) < 0) + return n; + + /* translate SJIS into EUC-JP */ + if (n == 1) { + if (IS_SJIS_JISX0201KANA(out[0])) { + /* JIS X 0201 KANA */ + if (boundlen < 2) + return -ENAMETOOLONG; + + out[1] = out[0]; + out[0] = SS2; + return 2; + } + } else if (n == 2) { + /* NEC/IBM extended characters to IBM extended characters */ + sjisnec2sjisibm(out, out[0], out[1]); + + if (IS_SJIS_UDC_LOW(out[0], out[1])) { + /* User defined characters half low */ + MAP_SJIS2EUC(out[0], out[1], 0xF0, out[0], out[1], 0xF5); + } else if (IS_SJIS_UDC_HI(out[0], out[1])) { + /* User defined characters half high */ + unsigned char ch, cl; + + if (boundlen < 3) + return -ENAMETOOLONG; + + n = 3; ch = out[0]; cl = out[1]; + out[0] = SS3; + MAP_SJIS2EUC(ch, cl, 0xF5, out[1], out[2], 0xF5); + } else if (IS_SJIS_IBM(out[0], out[1])) { + /* IBM extended characters */ + unsigned char euc[3], i; + + n = sjisibm2euc(euc, out[0], out[1]); + if (boundlen < n) + return -ENAMETOOLONG; + for (i = 0; i < n; i++) + out[i] = euc[i]; + } else if (IS_SJIS_JISX0208(out[0], out[1])) { + /* JIS X 0208 (include NEC special characters) */ + out[0] = (out[0]^0xA0)*2 + 0x5F; + if (out[1] > 0x9E) + out[0]++; + + if (out[1] < 0x7F) + out[1] = out[1] + 0x61; + else if (out[1] < 0x9F) + out[1] = out[1] + 0x60; + else + out[1] = out[1] + 0x02; + } else { + /* Invalid characters */ + return -EINVAL; + } + } + else + return -EINVAL; + + return n; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, + wchar_t *uni) +{ + unsigned char sjis_temp[2]; + int euc_offset, n; + + if ( !p_nls ) + return -EINVAL; + if (boundlen <= 0) + return -ENAMETOOLONG; + + /* translate EUC-JP into SJIS */ + if (rawstring[0] > 0x7F) { + if (rawstring[0] == SS3) { + if (boundlen < 3) + return -EINVAL; + euc_offset = 3; + + if (IS_EUC_UDC_HI(rawstring[1], rawstring[2])) { + /* User defined characters half high */ + MAP_EUC2SJIS(rawstring[1], rawstring[2], 0xF5, + sjis_temp[0], sjis_temp[1], 0xF5); + } else if (euc2sjisibm(sjis_temp,rawstring[1],rawstring[2])) { + /* IBM extended characters */ + } else { + /* JIS X 0212 and Invalid characters*/ + return -EINVAL; + + /* 'GETA' with SJIS coding */ + /* sjis_temp[0] = 0x81; */ + /* sjis_temp[1] = 0xAC; */ + } + } else { + if (boundlen < 2) + return -EINVAL; + euc_offset = 2; + + if (IS_EUC_JISX0201KANA(rawstring[0], rawstring[1])) { + /* JIS X 0201 KANA */ + sjis_temp[0] = rawstring[1]; + sjis_temp[1] = 0x00; + } else if (IS_EUC_UDC_LOW(rawstring[0], rawstring[1])) { + /* User defined characters half low */ + MAP_EUC2SJIS(rawstring[0], rawstring[1], 0xF5, + sjis_temp[0], sjis_temp[1], 0xF0); + } else if (IS_EUC_JISX0208(rawstring[0], rawstring[1])) { + /* JIS X 0208 (include NEC spesial characters) */ + sjis_temp[0] = ((rawstring[0]-0x5f)/2) ^ 0xA0; + if (!(rawstring[0] & 1)) + sjis_temp[1] = rawstring[1] - 0x02; + else if (rawstring[1] < 0xE0) + sjis_temp[1] = rawstring[1] - 0x61; + else + sjis_temp[1] = rawstring[1] - 0x60; + } else { + /* Invalid characters */ + return -EINVAL; + } + } + } else { + euc_offset = 1; + + /* JIS X 0201 ROMAJI */ + sjis_temp[0] = rawstring[0]; + sjis_temp[1] = 0x00; + } + + if ( (n = p_nls->char2uni(sjis_temp, sizeof(sjis_temp), uni)) < 0) + return n; + + return euc_offset; +} + +static struct nls_table table = { + .charset = "euc-jp", + .uni2char = uni2char, + .char2uni = char2uni, +}; + +static int __init init_nls_euc_jp(void) +{ + p_nls = load_nls("cp932"); + + if (p_nls) { + table.charset2upper = p_nls->charset2upper; + table.charset2lower = p_nls->charset2lower; + return register_nls(&table); + } + + return -EINVAL; +} + +static void __exit exit_nls_euc_jp(void) +{ + unregister_nls(&table); + unload_nls(p_nls); +} + +module_init(init_nls_euc_jp) +module_exit(exit_nls_euc_jp) + +MODULE_DESCRIPTION("NLS Japanese charset (EUC-JP)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_iso8859-1.c b/fs/nls/nls_iso8859-1.c new file mode 100644 index 000000000..a181be488 --- /dev/null +++ b/fs/nls/nls_iso8859-1.c @@ -0,0 +1,258 @@ +/* + * linux/fs/nls/nls_iso8859-1.c + * + * Charset iso8859-1 translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x0080, 0x0081, 0x0082, 0x0083, + 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, + 0x008c, 0x008d, 0x008e, 0x008f, + /* 0x90*/ + 0x0090, 0x0091, 0x0092, 0x0093, + 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, + 0x009c, 0x009d, 0x009e, 0x009f, + /* 0xa0*/ + 0x00a0, 0x00a1, 0x00a2, 0x00a3, + 0x00a4, 0x00a5, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0x00aa, 0x00ab, + 0x00ac, 0x00ad, 0x00ae, 0x00af, + /* 0xb0*/ + 0x00b0, 0x00b1, 0x00b2, 0x00b3, + 0x00b4, 0x00b5, 0x00b6, 0x00b7, + 0x00b8, 0x00b9, 0x00ba, 0x00bb, + 0x00bc, 0x00bd, 0x00be, 0x00bf, + /* 0xc0*/ + 0x00c0, 0x00c1, 0x00c2, 0x00c3, + 0x00c4, 0x00c5, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, + 0x00cc, 0x00cd, 0x00ce, 0x00cf, + /* 0xd0*/ + 0x00d0, 0x00d1, 0x00d2, 0x00d3, + 0x00d4, 0x00d5, 0x00d6, 0x00d7, + 0x00d8, 0x00d9, 0x00da, 0x00db, + 0x00dc, 0x00dd, 0x00de, 0x00df, + /* 0xe0*/ + 0x00e0, 0x00e1, 0x00e2, 0x00e3, + 0x00e4, 0x00e5, 0x00e6, 0x00e7, + 0x00e8, 0x00e9, 0x00ea, 0x00eb, + 0x00ec, 0x00ed, 0x00ee, 0x00ef, + /* 0xf0*/ + 0x00f0, 0x00f1, 0x00f2, 0x00f3, + 0x00f4, 0x00f5, 0x00f6, 0x00f7, + 0x00f8, 0x00f9, 0x00fa, 0x00fb, + 0x00fc, 0x00fd, 0x00fe, 0x00ff, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, NULL, NULL, NULL, NULL, NULL, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xc0-0xc7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xc8-0xcf */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xd7, /* 0xd0-0xd7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0x00, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xe0-0xe7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xe8-0xef */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xf7, /* 0xf0-0xf7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0x00, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "iso8859-1", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_iso8859_1(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_iso8859_1(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_iso8859_1) +module_exit(exit_nls_iso8859_1) + +MODULE_DESCRIPTION("NLS ISO 8859-1 (Latin 1; Western European Languages)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_iso8859-13.c b/fs/nls/nls_iso8859-13.c new file mode 100644 index 000000000..8e2be5bfe --- /dev/null +++ b/fs/nls/nls_iso8859-13.c @@ -0,0 +1,286 @@ +/* + * linux/fs/nls/nls_iso8859-13.c + * + * Charset iso8859-13 translation tables. + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x0080, 0x0081, 0x0082, 0x0083, + 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, + 0x008c, 0x008d, 0x008e, 0x008f, + /* 0x90*/ + 0x0090, 0x0091, 0x0092, 0x0093, + 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, + 0x009c, 0x009d, 0x009e, 0x009f, + /* 0xa0*/ + 0x00a0, 0x201d, 0x00a2, 0x00a3, + 0x00a4, 0x201e, 0x00a6, 0x00a7, + 0x00d8, 0x00a9, 0x0156, 0x00ab, + 0x00ac, 0x00ad, 0x00ae, 0x00c6, + /* 0xb0*/ + 0x00b0, 0x00b1, 0x00b2, 0x00b3, + 0x201c, 0x00b5, 0x00b6, 0x00b7, + 0x00f8, 0x00b9, 0x0157, 0x00bb, + 0x00bc, 0x00bd, 0x00be, 0x00e6, + /* 0xc0*/ + 0x0104, 0x012e, 0x0100, 0x0106, + 0x00c4, 0x00c5, 0x0118, 0x0112, + 0x010c, 0x00c9, 0x0179, 0x0116, + 0x0122, 0x0136, 0x012a, 0x013b, + /* 0xd0*/ + 0x0160, 0x0143, 0x0145, 0x00d3, + 0x014c, 0x00d5, 0x00d6, 0x00d7, + 0x0172, 0x0141, 0x015a, 0x016a, + 0x00dc, 0x017b, 0x017d, 0x00df, + /* 0xe0*/ + 0x0105, 0x012f, 0x0101, 0x0107, + 0x00e4, 0x00e5, 0x0119, 0x0113, + 0x010d, 0x00e9, 0x017a, 0x0117, + 0x0123, 0x0137, 0x012b, 0x013c, + /* 0xf0*/ + 0x0161, 0x0144, 0x0146, 0x00f3, + 0x014d, 0x00f5, 0x00f6, 0x00f7, + 0x0173, 0x0142, 0x015b, 0x016b, + 0x00fc, 0x017c, 0x017e, 0x2019, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0x00, 0xa2, 0xa3, 0xa4, 0x00, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0x00, 0xa9, 0x00, 0xab, 0xac, 0xad, 0xae, 0x00, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0x00, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0x00, 0xb9, 0x00, 0xbb, 0xbc, 0xbd, 0xbe, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0xc4, 0xc5, 0xaf, 0x00, /* 0xc0-0xc7 */ + 0x00, 0xc9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0xd3, 0x00, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xa8, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0xdf, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0xe4, 0xe5, 0xbf, 0x00, /* 0xe0-0xe7 */ + 0x00, 0xe9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0xf3, 0x00, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xb8, 0x00, 0x00, 0x00, 0xfc, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0xc2, 0xe2, 0x00, 0x00, 0xc0, 0xe0, 0xc3, 0xe3, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xc8, 0xe8, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0xc7, 0xe7, 0x00, 0x00, 0xcb, 0xeb, /* 0x10-0x17 */ + 0xc6, 0xe6, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0xcc, 0xec, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0xce, 0xee, 0x00, 0x00, 0xc1, 0xe1, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xcd, 0xed, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0xcf, 0xef, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0xd9, 0xf9, 0xd1, 0xf1, 0xd2, 0xf2, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0xd4, 0xf4, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xaa, 0xba, /* 0x50-0x57 */ + 0x00, 0x00, 0xda, 0xfa, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0xd0, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0xdb, 0xfb, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0xd8, 0xf8, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0xca, 0xea, 0xdd, 0xfd, 0xde, 0xfe, 0x00, /* 0x78-0x7f */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0xff, 0x00, 0x00, 0xb4, 0xa1, 0xa5, 0x00, /* 0x18-0x1f */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + page20, NULL, NULL, NULL, NULL, NULL, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xb1, 0xa2, 0xb3, 0xa4, 0xb5, 0xb6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xb9, 0xba, 0xbb, 0xbc, 0xad, 0xbe, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbf, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xc0-0xc7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xc8-0xcf */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xd7, /* 0xd0-0xd7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xa1, 0xb2, 0xa3, 0xb4, 0xa5, 0xa6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xa9, 0xaa, 0xab, 0xac, 0xbd, 0xae, 0xbd, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xe0-0xe7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xe8-0xef */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xf7, /* 0xf0-0xf7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "iso8859-13", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_iso8859_13(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_iso8859_13(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_iso8859_13) +module_exit(exit_nls_iso8859_13) + +MODULE_DESCRIPTION("NLS ISO 8859-13 (Latin 7; Baltic)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_iso8859-14.c b/fs/nls/nls_iso8859-14.c new file mode 100644 index 000000000..c789eccb8 --- /dev/null +++ b/fs/nls/nls_iso8859-14.c @@ -0,0 +1,342 @@ +/* + * linux/fs/nls/nls_iso8859-14.c + * + * Charset iso8859-14 translation tables. + * + * Generated automatically from the Unicode and charset table + * provided by the Unicode Organisation at + * http://www.unicode.org/ + * The Unicode to charset table has only exact mappings. + * + * Rhys Jones, Swansea University Computer Society + * rhys@sucs.swan.ac.uk + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x0080, 0x0081, 0x0082, 0x0083, + 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, + 0x008c, 0x008d, 0x008e, 0x008f, + /* 0x90*/ + 0x0090, 0x0091, 0x0092, 0x0093, + 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, + 0x009c, 0x009d, 0x009e, 0x009f, + /* 0xa0*/ + 0x00a0, 0x1e02, 0x1e03, 0x00a3, + 0x010a, 0x010b, 0x1e0a, 0x00a7, + 0x1e80, 0x00a9, 0x1e82, 0x1e0b, + 0x1ef2, 0x00ad, 0x00ae, 0x0178, + /* 0xb0*/ + 0x1e1e, 0x1e1f, 0x0120, 0x0121, + 0x1e40, 0x1e41, 0x00b6, 0x1e56, + 0x1e81, 0x1e57, 0x1e83, 0x1e60, + 0x1ef3, 0x1e84, 0x1e85, 0x1e61, + /* 0xc0*/ + 0x00c0, 0x00c1, 0x00c2, 0x00c3, + 0x00c4, 0x00c5, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, + 0x00cc, 0x00cd, 0x00ce, 0x00cf, + /* 0xd0*/ + 0x0174, 0x00d1, 0x00d2, 0x00d3, + 0x00d4, 0x00d5, 0x00d6, 0x1e6a, + 0x00d8, 0x00d9, 0x00da, 0x00db, + 0x00dc, 0x00dd, 0x0176, 0x00df, + /* 0xe0*/ + 0x00e0, 0x00e1, 0x00e2, 0x00e3, + 0x00e4, 0x00e5, 0x00e6, 0x00e7, + 0x00e8, 0x00e9, 0x00ea, 0x00eb, + 0x00ec, 0x00ed, 0x00ee, 0x00ef, + /* 0xf0*/ + 0x0175, 0x00f1, 0x00f2, 0x00f3, + 0x00f4, 0x00f5, 0x00f6, 0x1e6b, + 0x00f8, 0x00f9, 0x00fa, 0x00fb, + 0x00fc, 0x00fd, 0x0177, 0x00ff, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0x00, 0x00, 0xa3, 0x00, 0x00, 0x00, 0xa7, /* 0xa0-0xa7 */ + 0x00, 0xa9, 0x00, 0x00, 0x00, 0xad, 0xae, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb6, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0x00, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0x00, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0x00, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0x00, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0x00, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0x00, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0x00, 0x00, 0xa1, 0xa2, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0xa6, 0xab, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0xb1, /* 0x18-0x1f */ + 0xb2, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0xb4, 0xb5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb7, 0xb9, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0xbb, 0xbf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0xd7, 0xf7, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0xd0, 0xf0, 0xde, 0xfe, /* 0x70-0x77 */ + 0xaf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0xa8, 0xb8, 0xaa, 0xba, 0xbd, 0xbe, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0xac, 0xbc, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page1e[256] = { + 0x00, 0x00, 0xa1, 0xa2, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0xa6, 0xab, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb0, 0xb1, /* 0x18-0x1f */ + 0xb2, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0xb4, 0xb5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb7, 0xb9, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0xbb, 0xbf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0xd7, 0xf7, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0xd0, 0xf0, 0xde, 0xfe, /* 0x70-0x77 */ + 0xaf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0xa8, 0xb8, 0xaa, 0xba, 0xbd, 0xbe, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0xac, 0xbc, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, page1e, NULL, + + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa2, 0xa2, 0xa3, 0xab, 0xab, 0xab, 0xa7, /* 0xa0-0xa7 */ + 0xb8, 0xa9, 0xba, 0xab, 0xbc, 0xad, 0xae, 0xff, /* 0xa8-0xaf */ + 0xb1, 0xb1, 0xb3, 0xb3, 0xb5, 0xb5, 0xb6, 0xb9, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbf, 0xbc, 0xbe, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xc0-0xc7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xc8-0xcf */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xd0-0xd7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa1, 0xa3, 0xa6, 0xa6, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xa6, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb0, 0xb2, 0xb2, 0xb4, 0xb4, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xa8, 0xb7, 0xaa, 0xbb, 0xac, 0xbd, 0xbd, 0xbb, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xe0-0xe7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xe8-0xef */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xf0-0xf7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xaf, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "iso8859-14", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_iso8859_14(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_iso8859_14(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_iso8859_14) +module_exit(exit_nls_iso8859_14) + +MODULE_DESCRIPTION("NLS ISO 8859-14 (Latin 8; Celtic)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_iso8859-15.c b/fs/nls/nls_iso8859-15.c new file mode 100644 index 000000000..ffec64917 --- /dev/null +++ b/fs/nls/nls_iso8859-15.c @@ -0,0 +1,308 @@ +/* + * linux/fs/nls/nls_iso8859-15.c + * + * Charset iso8859-15 translation tables. + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x0080, 0x0081, 0x0082, 0x0083, + 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, + 0x008c, 0x008d, 0x008e, 0x008f, + /* 0x90*/ + 0x0090, 0x0091, 0x0092, 0x0093, + 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, + 0x009c, 0x009d, 0x009e, 0x009f, + /* 0xa0*/ + 0x00a0, 0x00a1, 0x00a2, 0x00a3, + 0x20ac, 0x00a5, 0x0160, 0x00a7, + 0x0161, 0x00a9, 0x00aa, 0x00ab, + 0x00ac, 0x00ad, 0x00ae, 0x00af, + /* 0xb0*/ + 0x00b0, 0x00b1, 0x00b2, 0x00b3, + 0x017d, 0x00b5, 0x00b6, 0x00b7, + 0x017e, 0x00b9, 0x00ba, 0x00bb, + 0x0152, 0x0153, 0x0178, 0x00bf, + /* 0xc0*/ + 0x00c0, 0x00c1, 0x00c2, 0x00c3, + 0x00c4, 0x00c5, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, + 0x00cc, 0x00cd, 0x00ce, 0x00cf, + /* 0xd0*/ + 0x00d0, 0x00d1, 0x00d2, 0x00d3, + 0x00d4, 0x00d5, 0x00d6, 0x00d7, + 0x00d8, 0x00d9, 0x00da, 0x00db, + 0x00dc, 0x00dd, 0x00de, 0x00df, + /* 0xe0*/ + 0x00e0, 0x00e1, 0x00e2, 0x00e3, + 0x00e4, 0x00e5, 0x00e6, 0x00e7, + 0x00e8, 0x00e9, 0x00ea, 0x00eb, + 0x00ec, 0x00ed, 0x00ee, 0x00ef, + /* 0xf0*/ + 0x00f0, 0x00f1, 0x00f2, 0x00f3, + 0x00f4, 0x00f5, 0x00f6, 0x00f7, + 0x00f8, 0x00f9, 0x00fa, 0x00fb, + 0x00fc, 0x00fd, 0x00fe, 0x00ff, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0x00, 0xa5, 0x00, 0xa7, /* 0xa0-0xa7 */ + 0x00, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0x00, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0x00, 0xb9, 0xba, 0xbb, 0x00, 0x00, 0x00, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0xbc, 0xbd, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0xa6, 0xa8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0xbe, 0x00, 0x00, 0x00, 0x00, 0xb4, 0xb8, 0x00, /* 0x78-0x7f */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0xa4, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + + page20, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa8, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb8, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbd, 0xbd, 0xff, 0xbf, /* 0xb8-0xbf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xc0-0xc7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xc8-0xcf */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xd7, /* 0xd0-0xd7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa6, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0x00, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb4, 0xb9, 0xba, 0xbb, 0xbc, 0xbc, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xe0-0xe7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xe8-0xef */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xf7, /* 0xf0-0xf7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xbe, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "iso8859-15", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_iso8859_15(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_iso8859_15(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_iso8859_15) +module_exit(exit_nls_iso8859_15) + +MODULE_DESCRIPTION("NLS ISO 8859-15 (Latin 9; Western European Languages with Euro)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_iso8859-2.c b/fs/nls/nls_iso8859-2.c new file mode 100644 index 000000000..d352334d0 --- /dev/null +++ b/fs/nls/nls_iso8859-2.c @@ -0,0 +1,309 @@ +/* + * linux/fs/nls/nls_iso8859-2.c + * + * Charset iso8859-2 translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x0080, 0x0081, 0x0082, 0x0083, + 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, + 0x008c, 0x008d, 0x008e, 0x008f, + /* 0x90*/ + 0x0090, 0x0091, 0x0092, 0x0093, + 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, + 0x009c, 0x009d, 0x009e, 0x009f, + /* 0xa0*/ + 0x00a0, 0x0104, 0x02d8, 0x0141, + 0x00a4, 0x013d, 0x015a, 0x00a7, + 0x00a8, 0x0160, 0x015e, 0x0164, + 0x0179, 0x00ad, 0x017d, 0x017b, + /* 0xb0*/ + 0x00b0, 0x0105, 0x02db, 0x0142, + 0x00b4, 0x013e, 0x015b, 0x02c7, + 0x00b8, 0x0161, 0x015f, 0x0165, + 0x017a, 0x02dd, 0x017e, 0x017c, + /* 0xc0*/ + 0x0154, 0x00c1, 0x00c2, 0x0102, + 0x00c4, 0x0139, 0x0106, 0x00c7, + 0x010c, 0x00c9, 0x0118, 0x00cb, + 0x011a, 0x00cd, 0x00ce, 0x010e, + /* 0xd0*/ + 0x0110, 0x0143, 0x0147, 0x00d3, + 0x00d4, 0x0150, 0x00d6, 0x00d7, + 0x0158, 0x016e, 0x00da, 0x0170, + 0x00dc, 0x00dd, 0x0162, 0x00df, + /* 0xe0*/ + 0x0155, 0x00e1, 0x00e2, 0x0103, + 0x00e4, 0x013a, 0x0107, 0x00e7, + 0x010d, 0x00e9, 0x0119, 0x00eb, + 0x011b, 0x00ed, 0x00ee, 0x010f, + /* 0xf0*/ + 0x0111, 0x0144, 0x0148, 0x00f3, + 0x00f4, 0x0151, 0x00f6, 0x00f7, + 0x0159, 0x016f, 0x00fa, 0x0171, + 0x00fc, 0x00fd, 0x0163, 0x02d9, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0x00, 0x00, 0x00, 0xa4, 0x00, 0x00, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0x00, 0x00, 0x00, 0x00, 0xad, 0x00, 0x00, /* 0xa8-0xaf */ + 0xb0, 0x00, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0xc1, 0xc2, 0x00, 0xc4, 0x00, 0x00, 0xc7, /* 0xc0-0xc7 */ + 0x00, 0xc9, 0x00, 0xcb, 0x00, 0xcd, 0xce, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0xd3, 0xd4, 0x00, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0x00, 0x00, 0xda, 0x00, 0xdc, 0xdd, 0x00, 0xdf, /* 0xd8-0xdf */ + 0x00, 0xe1, 0xe2, 0x00, 0xe4, 0x00, 0x00, 0xe7, /* 0xe0-0xe7 */ + 0x00, 0xe9, 0x00, 0xeb, 0x00, 0xed, 0xee, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0xf3, 0xf4, 0x00, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0x00, 0x00, 0xfa, 0x00, 0xfc, 0xfd, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0x00, 0x00, 0xc3, 0xe3, 0xa1, 0xb1, 0xc6, 0xe6, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xc8, 0xe8, 0xcf, 0xef, /* 0x08-0x0f */ + 0xd0, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0xca, 0xea, 0xcc, 0xec, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0xc5, 0xe5, 0x00, 0x00, 0xa5, 0xb5, 0x00, /* 0x38-0x3f */ + 0x00, 0xa3, 0xb3, 0xd1, 0xf1, 0x00, 0x00, 0xd2, /* 0x40-0x47 */ + 0xf2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0xd5, 0xf5, 0x00, 0x00, 0xc0, 0xe0, 0x00, 0x00, /* 0x50-0x57 */ + 0xd8, 0xf8, 0xa6, 0xb6, 0x00, 0x00, 0xaa, 0xba, /* 0x58-0x5f */ + 0xa9, 0xb9, 0xde, 0xfe, 0xab, 0xbb, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd9, 0xf9, /* 0x68-0x6f */ + 0xdb, 0xfb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0xac, 0xbc, 0xaf, 0xbf, 0xae, 0xbe, 0x00, /* 0x78-0x7f */ +}; + +static const unsigned char page02[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb7, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0xa2, 0xff, 0x00, 0xb2, 0x00, 0xbd, 0x00, 0x00, /* 0xd8-0xdf */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, page02, NULL, NULL, NULL, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xb1, 0xa2, 0xb3, 0xa4, 0xb5, 0xb6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xb9, 0xba, 0xbb, 0xbc, 0xad, 0xbe, 0xbf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xc0-0xc7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xc8-0xcf */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xd7, /* 0xd0-0xd7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xa1, 0xb2, 0xa3, 0xb4, 0xa5, 0xa6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xa9, 0xaa, 0xab, 0xac, 0xbd, 0xae, 0xaf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xe0-0xe7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xe8-0xef */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xf7, /* 0xf0-0xf7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "iso8859-2", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_iso8859_2(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_iso8859_2(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_iso8859_2) +module_exit(exit_nls_iso8859_2) + +MODULE_DESCRIPTION("NLS ISO 8859-2 (Latin 2; Slavic/Central European Languages)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_iso8859-3.c b/fs/nls/nls_iso8859-3.c new file mode 100644 index 000000000..09990e663 --- /dev/null +++ b/fs/nls/nls_iso8859-3.c @@ -0,0 +1,309 @@ +/* + * linux/fs/nls/nls_iso8859-3.c + * + * Charset iso8859-3 translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x0080, 0x0081, 0x0082, 0x0083, + 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, + 0x008c, 0x008d, 0x008e, 0x008f, + /* 0x90*/ + 0x0090, 0x0091, 0x0092, 0x0093, + 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, + 0x009c, 0x009d, 0x009e, 0x009f, + /* 0xa0*/ + 0x00a0, 0x0126, 0x02d8, 0x00a3, + 0x00a4, 0x0000, 0x0124, 0x00a7, + 0x00a8, 0x0130, 0x015e, 0x011e, + 0x0134, 0x00ad, 0x0000, 0x017b, + /* 0xb0*/ + 0x00b0, 0x0127, 0x00b2, 0x00b3, + 0x00b4, 0x00b5, 0x0125, 0x00b7, + 0x00b8, 0x0131, 0x015f, 0x011f, + 0x0135, 0x00bd, 0x0000, 0x017c, + /* 0xc0*/ + 0x00c0, 0x00c1, 0x00c2, 0x0000, + 0x00c4, 0x010a, 0x0108, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, + 0x00cc, 0x00cd, 0x00ce, 0x00cf, + /* 0xd0*/ + 0x0000, 0x00d1, 0x00d2, 0x00d3, + 0x00d4, 0x0120, 0x00d6, 0x00d7, + 0x011c, 0x00d9, 0x00da, 0x00db, + 0x00dc, 0x016c, 0x015c, 0x00df, + /* 0xe0*/ + 0x00e0, 0x00e1, 0x00e2, 0x0000, + 0x00e4, 0x010b, 0x0109, 0x00e7, + 0x00e8, 0x00e9, 0x00ea, 0x00eb, + 0x00ec, 0x00ed, 0x00ee, 0x00ef, + /* 0xf0*/ + 0x0000, 0x00f1, 0x00f2, 0x00f3, + 0x00f4, 0x0121, 0x00f6, 0x00f7, + 0x011d, 0x00f9, 0x00fa, 0x00fb, + 0x00fc, 0x016d, 0x015d, 0x02d9, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0x00, 0x00, 0xa3, 0xa4, 0x00, 0x00, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0x00, 0x00, 0x00, 0x00, 0xad, 0x00, 0x00, /* 0xa8-0xaf */ + 0xb0, 0x00, 0xb2, 0xb3, 0xb4, 0xb5, 0x00, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0x00, 0x00, 0x00, 0x00, 0xbd, 0x00, 0x00, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0x00, 0xc4, 0x00, 0x00, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0x00, 0xd1, 0xd2, 0xd3, 0xd4, 0x00, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0x00, 0xd9, 0xda, 0xdb, 0xdc, 0x00, 0x00, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0x00, 0xe4, 0x00, 0x00, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0x00, 0xf1, 0xf2, 0xf3, 0xf4, 0x00, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0x00, 0xf9, 0xfa, 0xfb, 0xfc, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0xc6, 0xe6, 0xc5, 0xe5, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0xd8, 0xf8, 0xab, 0xbb, /* 0x18-0x1f */ + 0xd5, 0xf5, 0x00, 0x00, 0xa6, 0xb6, 0xa1, 0xb1, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0xa9, 0xb9, 0x00, 0x00, 0xac, 0xbc, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0xde, 0xfe, 0xaa, 0xba, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0xdd, 0xfd, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0xaf, 0xbf, 0x00, 0x00, 0x00, /* 0x78-0x7f */ +}; + +static const unsigned char page02[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0xa2, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, page02, NULL, NULL, NULL, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xb1, 0xa2, 0xa3, 0xa4, 0x00, 0xb6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0x69, 0xba, 0xbb, 0xbc, 0xad, 0x00, 0xbf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0x00, 0xbf, /* 0xb8-0xbf */ + 0xe0, 0xe1, 0xe2, 0x00, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xc0-0xc7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xc8-0xcf */ + 0x00, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xd7, /* 0xd0-0xd7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0x00, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0x00, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0x00, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0x00, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xa1, 0xb2, 0xb3, 0xb4, 0x00, 0xa6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0x49, 0xaa, 0xab, 0xac, 0xbd, 0x00, 0xaf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0x00, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0x00, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xc0, 0xc1, 0xc2, 0x00, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xe0-0xe7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xe8-0xef */ + 0x00, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xf7, /* 0xf0-0xf7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "iso8859-3", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_iso8859_3(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_iso8859_3(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_iso8859_3) +module_exit(exit_nls_iso8859_3) + +MODULE_DESCRIPTION("NLS ISO 8859-3 (Latin 3; Esperanto, Galician, Maltese, Turkish)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_iso8859-4.c b/fs/nls/nls_iso8859-4.c new file mode 100644 index 000000000..927952249 --- /dev/null +++ b/fs/nls/nls_iso8859-4.c @@ -0,0 +1,309 @@ +/* + * linux/fs/nls/nls_iso8859-4.c + * + * Charset iso8859-4 translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x0080, 0x0081, 0x0082, 0x0083, + 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, + 0x008c, 0x008d, 0x008e, 0x008f, + /* 0x90*/ + 0x0090, 0x0091, 0x0092, 0x0093, + 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, + 0x009c, 0x009d, 0x009e, 0x009f, + /* 0xa0*/ + 0x00a0, 0x0104, 0x0138, 0x0156, + 0x00a4, 0x0128, 0x013b, 0x00a7, + 0x00a8, 0x0160, 0x0112, 0x0122, + 0x0166, 0x00ad, 0x017d, 0x00af, + /* 0xb0*/ + 0x00b0, 0x0105, 0x02db, 0x0157, + 0x00b4, 0x0129, 0x013c, 0x02c7, + 0x00b8, 0x0161, 0x0113, 0x0123, + 0x0167, 0x014a, 0x017e, 0x014b, + /* 0xc0*/ + 0x0100, 0x00c1, 0x00c2, 0x00c3, + 0x00c4, 0x00c5, 0x00c6, 0x012e, + 0x010c, 0x00c9, 0x0118, 0x00cb, + 0x0116, 0x00cd, 0x00ce, 0x012a, + /* 0xd0*/ + 0x0110, 0x0145, 0x014c, 0x0136, + 0x00d4, 0x00d5, 0x00d6, 0x00d7, + 0x00d8, 0x0172, 0x00da, 0x00db, + 0x00dc, 0x0168, 0x016a, 0x00df, + /* 0xe0*/ + 0x0101, 0x00e1, 0x00e2, 0x00e3, + 0x00e4, 0x00e5, 0x00e6, 0x012f, + 0x010d, 0x00e9, 0x0119, 0x00eb, + 0x0117, 0x00ed, 0x00ee, 0x012b, + /* 0xf0*/ + 0x0111, 0x0146, 0x014d, 0x0137, + 0x00f4, 0x00f5, 0x00f6, 0x00f7, + 0x00f8, 0x0173, 0x00fa, 0x00fb, + 0x00fc, 0x0169, 0x016b, 0x02d9, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0x00, 0x00, 0x00, 0xa4, 0x00, 0x00, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0x00, 0x00, 0x00, 0x00, 0xad, 0x00, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0x00, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0x00, /* 0xc0-0xc7 */ + 0x00, 0xc9, 0x00, 0xcb, 0x00, 0xcd, 0xce, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0x00, 0xda, 0xdb, 0xdc, 0x00, 0x00, 0xdf, /* 0xd8-0xdf */ + 0x00, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0x00, /* 0xe0-0xe7 */ + 0x00, 0xe9, 0x00, 0xeb, 0x00, 0xed, 0xee, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0x00, 0xfa, 0xfb, 0xfc, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0xc0, 0xe0, 0x00, 0x00, 0xa1, 0xb1, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xc8, 0xe8, 0x00, 0x00, /* 0x08-0x0f */ + 0xd0, 0xf0, 0xaa, 0xba, 0x00, 0x00, 0xcc, 0xec, /* 0x10-0x17 */ + 0xca, 0xea, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0xab, 0xbb, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0xa5, 0xb5, 0xcf, 0xef, 0x00, 0x00, 0xc7, 0xe7, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd3, 0xf3, /* 0x30-0x37 */ + 0xa2, 0x00, 0x00, 0xa6, 0xb6, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0xd1, 0xf1, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0xbd, 0xbf, 0xd2, 0xf2, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xa3, 0xb3, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0xa9, 0xb9, 0x00, 0x00, 0x00, 0x00, 0xac, 0xbc, /* 0x60-0x67 */ + 0xdd, 0xfd, 0xde, 0xfe, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0xd9, 0xf9, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0xae, 0xbe, 0x00, /* 0x78-0x7f */ +}; + +static const unsigned char page02[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xb7, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0xff, 0x00, 0xb2, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, page02, NULL, NULL, NULL, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xb1, 0xa2, 0xb3, 0xa4, 0xb5, 0xb6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xb9, 0xba, 0xbb, 0xbc, 0xad, 0xbe, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbf, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xc0-0xc7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xc8-0xcf */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xd7, /* 0xd0-0xd7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xa1, 0xb2, 0xa3, 0xb4, 0xa5, 0xa6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xa9, 0xaa, 0xab, 0xac, 0xbd, 0xae, 0xbd, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xe0-0xe7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xe8-0xef */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xf7, /* 0xf0-0xf7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "iso8859-4", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_iso8859_4(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_iso8859_4(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_iso8859_4) +module_exit(exit_nls_iso8859_4) + +MODULE_DESCRIPTION("NLS ISO 8859-4 (Latin 4; old Baltic charset)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_iso8859-5.c b/fs/nls/nls_iso8859-5.c new file mode 100644 index 000000000..323093153 --- /dev/null +++ b/fs/nls/nls_iso8859-5.c @@ -0,0 +1,273 @@ +/* + * linux/fs/nls/nls_iso8859-5.c + * + * Charset iso8859-5 translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x0080, 0x0081, 0x0082, 0x0083, + 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, + 0x008c, 0x008d, 0x008e, 0x008f, + /* 0x90*/ + 0x0090, 0x0091, 0x0092, 0x0093, + 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, + 0x009c, 0x009d, 0x009e, 0x009f, + /* 0xa0*/ + 0x00a0, 0x0401, 0x0402, 0x0403, + 0x0404, 0x0405, 0x0406, 0x0407, + 0x0408, 0x0409, 0x040a, 0x040b, + 0x040c, 0x00ad, 0x040e, 0x040f, + /* 0xb0*/ + 0x0410, 0x0411, 0x0412, 0x0413, + 0x0414, 0x0415, 0x0416, 0x0417, + 0x0418, 0x0419, 0x041a, 0x041b, + 0x041c, 0x041d, 0x041e, 0x041f, + /* 0xc0*/ + 0x0420, 0x0421, 0x0422, 0x0423, + 0x0424, 0x0425, 0x0426, 0x0427, + 0x0428, 0x0429, 0x042a, 0x042b, + 0x042c, 0x042d, 0x042e, 0x042f, + /* 0xd0*/ + 0x0430, 0x0431, 0x0432, 0x0433, + 0x0434, 0x0435, 0x0436, 0x0437, + 0x0438, 0x0439, 0x043a, 0x043b, + 0x043c, 0x043d, 0x043e, 0x043f, + /* 0xe0*/ + 0x0440, 0x0441, 0x0442, 0x0443, + 0x0444, 0x0445, 0x0446, 0x0447, + 0x0448, 0x0449, 0x044a, 0x044b, + 0x044c, 0x044d, 0x044e, 0x044f, + /* 0xf0*/ + 0x2116, 0x0451, 0x0452, 0x0453, + 0x0454, 0x0455, 0x0456, 0x0457, + 0x0458, 0x0459, 0x045a, 0x045b, + 0x045c, 0x00a7, 0x045e, 0x045f, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfd, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0xad, 0x00, 0x00, /* 0xa8-0xaf */ +}; + +static const unsigned char page04[256] = { + 0x00, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0x00-0x07 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0x00, 0xae, 0xaf, /* 0x08-0x0f */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0x10-0x17 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0x18-0x1f */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0x20-0x27 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0x28-0x2f */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0x30-0x37 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0x38-0x3f */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0x40-0x47 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0x48-0x4f */ + 0x00, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0x50-0x57 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0x00, 0xfe, 0xff, /* 0x58-0x5f */ +}; + +static const unsigned char page21[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0, 0x00, /* 0x10-0x17 */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, NULL, NULL, NULL, page04, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, page21, NULL, NULL, NULL, NULL, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xa0-0xa7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xad, 0xfe, 0xff, /* 0xa8-0xaf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xb0-0xb7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xb8-0xbf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xc0-0xc7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xd0-0xd7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xd8-0xdf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xe0-0xe7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xe8-0xef */ + 0xf0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xf0-0xf7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xfd, 0xae, 0xaf, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "iso8859-5", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_iso8859_5(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_iso8859_5(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_iso8859_5) +module_exit(exit_nls_iso8859_5) + +MODULE_DESCRIPTION("NLS ISO 8859-5 (Cyrillic)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_iso8859-6.c b/fs/nls/nls_iso8859-6.c new file mode 100644 index 000000000..c18183469 --- /dev/null +++ b/fs/nls/nls_iso8859-6.c @@ -0,0 +1,264 @@ +/* + * linux/fs/nls/nls_iso8859-6.c + * + * Charset iso8859-6 translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0660, 0x0661, 0x0662, 0x0663, + 0x0664, 0x0665, 0x0666, 0x0667, + 0x0668, 0x0669, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x0080, 0x0081, 0x0082, 0x0083, + 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, + 0x008c, 0x008d, 0x008e, 0x008f, + /* 0x90*/ + 0x0090, 0x0091, 0x0092, 0x0093, + 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, + 0x009c, 0x009d, 0x009e, 0x009f, + /* 0xa0*/ + 0x00a0, 0x0000, 0x0000, 0x0000, + 0x00a4, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, + 0x060c, 0x00ad, 0x0000, 0x0000, + /* 0xb0*/ + 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x061b, + 0x0000, 0x0000, 0x0000, 0x061f, + /* 0xc0*/ + 0x0000, 0x0621, 0x0622, 0x0623, + 0x0624, 0x0625, 0x0626, 0x0627, + 0x0628, 0x0629, 0x062a, 0x062b, + 0x062c, 0x062d, 0x062e, 0x062f, + /* 0xd0*/ + 0x0630, 0x0631, 0x0632, 0x0633, + 0x0634, 0x0635, 0x0636, 0x0637, + 0x0638, 0x0639, 0x063a, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, + /* 0xe0*/ + 0x0640, 0x0641, 0x0642, 0x0643, + 0x0644, 0x0645, 0x0646, 0x0647, + 0x0648, 0x0649, 0x064a, 0x064b, + 0x064c, 0x064d, 0x064e, 0x064f, + /* 0xf0*/ + 0x0650, 0x0651, 0x0652, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0x00, 0x00, 0x00, 0xa4, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0xad, 0x00, 0x00, /* 0xa8-0xaf */ +}; + +static const unsigned char page06[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0xac, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0xbb, 0x00, 0x00, 0x00, 0xbf, /* 0x18-0x1f */ + 0x00, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0x20-0x27 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0x28-0x2f */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0x30-0x37 */ + 0xd8, 0xd9, 0xda, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0x40-0x47 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0x48-0x4f */ + 0xf0, 0xf1, 0xf2, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x60-0x67 */ + 0x38, 0x39, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, NULL, NULL, NULL, NULL, NULL, page06, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0x00, 0x00, 0x00, 0xa4, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0xac, 0xad, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0xbb, 0x00, 0x00, 0x00, 0xbf, /* 0xb8-0xbf */ + 0x00, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0x00, 0x00, 0x00, 0xa4, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0xac, 0xad, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0xbb, 0x00, 0x00, 0x00, 0xbf, /* 0xb8-0xbf */ + 0x00, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xf0-0xf7 */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "iso8859-6", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_iso8859_6(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_iso8859_6(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_iso8859_6) +module_exit(exit_nls_iso8859_6) + +MODULE_DESCRIPTION("NLS ISO 8859-6 (Arabic)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_iso8859-7.c b/fs/nls/nls_iso8859-7.c new file mode 100644 index 000000000..3652d6832 --- /dev/null +++ b/fs/nls/nls_iso8859-7.c @@ -0,0 +1,318 @@ +/* + * linux/fs/nls/nls_iso8859-7.c + * + * Charset iso8859-7 translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x0080, 0x0081, 0x0082, 0x0083, + 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, + 0x008c, 0x008d, 0x008e, 0x008f, + /* 0x90*/ + 0x0090, 0x0091, 0x0092, 0x0093, + 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, + 0x009c, 0x009d, 0x009e, 0x009f, + /* 0xa0*/ + 0x00a0, 0x02bd, 0x02bc, 0x00a3, + 0x0000, 0x0000, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0x0000, 0x00ab, + 0x00ac, 0x00ad, 0x0000, 0x2015, + /* 0xb0*/ + 0x00b0, 0x00b1, 0x00b2, 0x00b3, + 0x0384, 0x0385, 0x0386, 0x00b7, + 0x0388, 0x0389, 0x038a, 0x00bb, + 0x038c, 0x00bd, 0x038e, 0x038f, + /* 0xc0*/ + 0x0390, 0x0391, 0x0392, 0x0393, + 0x0394, 0x0395, 0x0396, 0x0397, + 0x0398, 0x0399, 0x039a, 0x039b, + 0x039c, 0x039d, 0x039e, 0x039f, + /* 0xd0*/ + 0x03a0, 0x03a1, 0x0000, 0x03a3, + 0x03a4, 0x03a5, 0x03a6, 0x03a7, + 0x03a8, 0x03a9, 0x03aa, 0x03ab, + 0x03ac, 0x03ad, 0x03ae, 0x03af, + /* 0xe0*/ + 0x03b0, 0x03b1, 0x03b2, 0x03b3, + 0x03b4, 0x03b5, 0x03b6, 0x03b7, + 0x03b8, 0x03b9, 0x03ba, 0x03bb, + 0x03bc, 0x03bd, 0x03be, 0x03bf, + /* 0xf0*/ + 0x03c0, 0x03c1, 0x03c2, 0x03c3, + 0x03c4, 0x03c5, 0x03c6, 0x03c7, + 0x03c8, 0x03c9, 0x03ca, 0x03cb, + 0x03cc, 0x03cd, 0x03ce, 0x0000, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0x00, 0x00, 0xa3, 0x00, 0x00, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0x00, 0xab, 0xac, 0xad, 0x00, 0x00, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0x00, 0x00, 0x00, 0xb7, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0xbb, 0x00, 0xbd, 0x00, 0x00, /* 0xb8-0xbf */ +}; + +static const unsigned char page02[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0xa2, 0xa1, 0x00, 0x00, /* 0xb8-0xbf */ +}; + +static const unsigned char page03[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0xb4, 0xb5, 0xb6, 0x00, /* 0x80-0x87 */ + 0xb8, 0xb9, 0xba, 0x00, 0xbc, 0x00, 0xbe, 0xbf, /* 0x88-0x8f */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0x90-0x97 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0x98-0x9f */ + 0xd0, 0xd1, 0x00, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xa0-0xa7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xa8-0xaf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xb0-0xb7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xb8-0xbf */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xc0-0xc7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0x00, /* 0xc8-0xcf */ +}; + +static const unsigned char page20[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0xaf, 0x00, 0x00, /* 0x10-0x17 */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, NULL, page02, page03, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + page20, NULL, NULL, NULL, NULL, NULL, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0x00, 0x00, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0x00, 0xab, 0xac, 0xad, 0x00, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xdc, 0xb7, /* 0xb0-0xb7 */ + 0xdd, 0xde, 0xdf, 0xbb, 0xfc, 0xbd, 0xfd, 0xfe, /* 0xb8-0xbf */ + 0xc0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xc0-0xc7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xc8-0xcf */ + 0xf0, 0xf1, 0x00, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xd0-0xd7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0x00, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0x00, 0x00, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0x00, 0xab, 0xac, 0xad, 0x00, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0x00, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xb6, 0xb8, 0xb9, 0xba, /* 0xd8-0xdf */ + 0xe0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xe0-0xe7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xe8-0xef */ + 0xd0, 0xd1, 0xd3, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xf0-0xf7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xbc, 0xbe, 0xbf, 0x00, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "iso8859-7", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_iso8859_7(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_iso8859_7(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_iso8859_7) +module_exit(exit_nls_iso8859_7) + +MODULE_DESCRIPTION("NLS ISO 8859-7 (Modern Greek)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_iso8859-9.c b/fs/nls/nls_iso8859-9.c new file mode 100644 index 000000000..11a67834b --- /dev/null +++ b/fs/nls/nls_iso8859-9.c @@ -0,0 +1,273 @@ +/* + * linux/fs/nls/nls_iso8859-9.c + * + * Charset iso8859-9 translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x0080, 0x0081, 0x0082, 0x0083, + 0x0084, 0x0085, 0x0086, 0x0087, + 0x0088, 0x0089, 0x008a, 0x008b, + 0x008c, 0x008d, 0x008e, 0x008f, + /* 0x90*/ + 0x0090, 0x0091, 0x0092, 0x0093, + 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, + 0x009c, 0x009d, 0x009e, 0x009f, + /* 0xa0*/ + 0x00a0, 0x00a1, 0x00a2, 0x00a3, + 0x00a4, 0x00a5, 0x00a6, 0x00a7, + 0x00a8, 0x00a9, 0x00aa, 0x00ab, + 0x00ac, 0x00ad, 0x00ae, 0x00af, + /* 0xb0*/ + 0x00b0, 0x00b1, 0x00b2, 0x00b3, + 0x00b4, 0x00b5, 0x00b6, 0x00b7, + 0x00b8, 0x00b9, 0x00ba, 0x00bb, + 0x00bc, 0x00bd, 0x00be, 0x00bf, + /* 0xc0*/ + 0x00c0, 0x00c1, 0x00c2, 0x00c3, + 0x00c4, 0x00c5, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, + 0x00cc, 0x00cd, 0x00ce, 0x00cf, + /* 0xd0*/ + 0x011e, 0x00d1, 0x00d2, 0x00d3, + 0x00d4, 0x00d5, 0x00d6, 0x00d7, + 0x00d8, 0x00d9, 0x00da, 0x00db, + 0x00dc, 0x0130, 0x015e, 0x00df, + /* 0xe0*/ + 0x00e0, 0x00e1, 0x00e2, 0x00e3, + 0x00e4, 0x00e5, 0x00e6, 0x00e7, + 0x00e8, 0x00e9, 0x00ea, 0x00eb, + 0x00ec, 0x00ed, 0x00ee, 0x00ef, + /* 0xf0*/ + 0x011f, 0x00f1, 0x00f2, 0x00f3, + 0x00f4, 0x00f5, 0x00f6, 0x00f7, + 0x00f8, 0x00f9, 0x00fa, 0x00fb, + 0x00fc, 0x0131, 0x015f, 0x00ff, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0x00, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0x00, 0x00, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0x00, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0x00, 0x00, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char page01[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd0, 0xf0, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0xdd, 0xfd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xde, 0xfe, /* 0x58-0x5f */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, page01, NULL, NULL, NULL, NULL, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xc0-0xc7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xc8-0xcf */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xd7, /* 0xd0-0xd7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0x69, 0xfe, 0xdf, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0x00, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xe0-0xe7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xe8-0xef */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xf7, /* 0xf0-0xf7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0x49, 0xde, 0x00, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "iso8859-9", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_iso8859_9(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_iso8859_9(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_iso8859_9) +module_exit(exit_nls_iso8859_9) + +MODULE_DESCRIPTION("NLS ISO 8859-9 (Latin 5; Turkish)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_koi8-r.c b/fs/nls/nls_koi8-r.c new file mode 100644 index 000000000..e3dca27a3 --- /dev/null +++ b/fs/nls/nls_koi8-r.c @@ -0,0 +1,324 @@ +/* + * linux/fs/nls/nls_koi8-r.c + * + * Charset koi8-r translation tables. + * Generated automatically from the Unicode and charset + * tables from the Unicode Organization (www.unicode.org). + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x2500, 0x2502, 0x250c, 0x2510, + 0x2514, 0x2518, 0x251c, 0x2524, + 0x252c, 0x2534, 0x253c, 0x2580, + 0x2584, 0x2588, 0x258c, 0x2590, + /* 0x90*/ + 0x2591, 0x2592, 0x2593, 0x2320, + 0x25a0, 0x2219, 0x221a, 0x2248, + 0x2264, 0x2265, 0x00a0, 0x2321, + 0x00b0, 0x00b2, 0x00b7, 0x00f7, + /* 0xa0*/ + 0x2550, 0x2551, 0x2552, 0x0451, + 0x2553, 0x2554, 0x2555, 0x2556, + 0x2557, 0x2558, 0x2559, 0x255a, + 0x255b, 0x255c, 0x255d, 0x255e, + /* 0xb0*/ + 0x255f, 0x2560, 0x2561, 0x0401, + 0x2562, 0x2563, 0x2564, 0x2565, + 0x2566, 0x2567, 0x2568, 0x2569, + 0x256a, 0x256b, 0x256c, 0x00a9, + /* 0xc0*/ + 0x044e, 0x0430, 0x0431, 0x0446, + 0x0434, 0x0435, 0x0444, 0x0433, + 0x0445, 0x0438, 0x0439, 0x043a, + 0x043b, 0x043c, 0x043d, 0x043e, + /* 0xd0*/ + 0x043f, 0x044f, 0x0440, 0x0441, + 0x0442, 0x0443, 0x0436, 0x0432, + 0x044c, 0x044b, 0x0437, 0x0448, + 0x044d, 0x0449, 0x0447, 0x044a, + /* 0xe0*/ + 0x042e, 0x0410, 0x0411, 0x0426, + 0x0414, 0x0415, 0x0424, 0x0413, + 0x0425, 0x0418, 0x0419, 0x041a, + 0x041b, 0x041c, 0x041d, 0x041e, + /* 0xf0*/ + 0x041f, 0x042f, 0x0420, 0x0421, + 0x0422, 0x0423, 0x0416, 0x0412, + 0x042c, 0x042b, 0x0417, 0x0428, + 0x042d, 0x0429, 0x0427, 0x042a, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x9a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0xbf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x9c, 0x00, 0x9d, 0x00, 0x00, 0x00, 0x00, 0x9e, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9f, /* 0xf0-0xf7 */ +}; + +static const unsigned char page04[256] = { + 0x00, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0xe1, 0xe2, 0xf7, 0xe7, 0xe4, 0xe5, 0xf6, 0xfa, /* 0x10-0x17 */ + 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, /* 0x18-0x1f */ + 0xf2, 0xf3, 0xf4, 0xf5, 0xe6, 0xe8, 0xe3, 0xfe, /* 0x20-0x27 */ + 0xfb, 0xfd, 0xff, 0xf9, 0xf8, 0xfc, 0xe0, 0xf1, /* 0x28-0x2f */ + 0xc1, 0xc2, 0xd7, 0xc7, 0xc4, 0xc5, 0xd6, 0xda, /* 0x30-0x37 */ + 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, /* 0x38-0x3f */ + 0xd2, 0xd3, 0xd4, 0xd5, 0xc6, 0xc8, 0xc3, 0xde, /* 0x40-0x47 */ + 0xdb, 0xdd, 0xdf, 0xd9, 0xd8, 0xdc, 0xc0, 0xd1, /* 0x48-0x4f */ + 0x00, 0xa3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ +}; + +static const unsigned char page22[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x95, 0x96, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x97, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x98, 0x99, 0x00, 0x00, /* 0x60-0x67 */ +}; + +static const unsigned char page23[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x93, 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ +}; + +static const unsigned char page25[256] = { + 0x80, 0x00, 0x81, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x82, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x83, 0x00, 0x00, 0x00, 0x84, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x85, 0x00, 0x00, 0x00, 0x86, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x87, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x88, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x8a, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0xa0, 0xa1, 0xa2, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, /* 0x50-0x57 */ + 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, /* 0x58-0x5f */ + 0xb1, 0xb2, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, /* 0x60-0x67 */ + 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x8b, 0x00, 0x00, 0x00, 0x8c, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x8d, 0x00, 0x00, 0x00, 0x8e, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x8f, 0x90, 0x91, 0x92, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x94, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, NULL, NULL, NULL, page04, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, page22, page23, NULL, page25, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xa3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xe0-0xe7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xe8-0xef */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xf0-0xf7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xb3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xc0-0xc7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xc8-0xcf */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xd0-0xd7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "koi8-r", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_koi8_r(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_koi8_r(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_koi8_r) +module_exit(exit_nls_koi8_r) + +MODULE_DESCRIPTION("NLS KOI8-R (Russian)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_koi8-ru.c b/fs/nls/nls_koi8-ru.c new file mode 100644 index 000000000..07afcd9e5 --- /dev/null +++ b/fs/nls/nls_koi8-ru.c @@ -0,0 +1,83 @@ +/* + * linux/fs/nls/nls_koi8-ru.c + * + * Charset koi8-ru translation based on charset koi8-u. + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static struct nls_table *p_nls; + +static int uni2char(const wchar_t uni, + unsigned char *out, int boundlen) +{ + if (boundlen <= 0) + return -ENAMETOOLONG; + + if ((uni & 0xffaf) == 0x040e || (uni & 0xffce) == 0x254c) { + /* koi8-ru and koi8-u differ only on two characters */ + if (uni == 0x040e) + out[0] = 0xbe; + else if (uni == 0x045e) + out[0] = 0xae; + else if (uni == 0x255d || uni == 0x256c) + return 0; + else + return p_nls->uni2char(uni, out, boundlen); + return 1; + } + else + /* fast path */ + return p_nls->uni2char(uni, out, boundlen); +} + +static int char2uni(const unsigned char *rawstring, int boundlen, + wchar_t *uni) +{ + int n; + + if ((*rawstring & 0xef) != 0xae) { + /* koi8-ru and koi8-u differ only on two characters */ + *uni = (*rawstring & 0x10) ? 0x040e : 0x045e; + return 1; + } + + n = p_nls->char2uni(rawstring, boundlen, uni); + return n; +} + +static struct nls_table table = { + .charset = "koi8-ru", + .uni2char = uni2char, + .char2uni = char2uni, +}; + +static int __init init_nls_koi8_ru(void) +{ + p_nls = load_nls("koi8-u"); + + if (p_nls) { + table.charset2upper = p_nls->charset2upper; + table.charset2lower = p_nls->charset2lower; + return register_nls(&table); + } + + return -EINVAL; +} + +static void __exit exit_nls_koi8_ru(void) +{ + unregister_nls(&table); + unload_nls(p_nls); +} + +module_init(init_nls_koi8_ru) +module_exit(exit_nls_koi8_ru) + +MODULE_DESCRIPTION("NLS KOI8-RU (Belarusian)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_koi8-u.c b/fs/nls/nls_koi8-u.c new file mode 100644 index 000000000..f60645758 --- /dev/null +++ b/fs/nls/nls_koi8-u.c @@ -0,0 +1,331 @@ +/* + * linux/fs/nls/nls_koi8-u.c + * + * Charset koi8-u translation tables. + * The Unicode to charset table has only exact mappings. + */ + +#include +#include +#include +#include +#include + +static const wchar_t charset2uni[256] = { + /* 0x00*/ + 0x0000, 0x0001, 0x0002, 0x0003, + 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, + 0x000c, 0x000d, 0x000e, 0x000f, + /* 0x10*/ + 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x001a, 0x001b, + 0x001c, 0x001d, 0x001e, 0x001f, + /* 0x20*/ + 0x0020, 0x0021, 0x0022, 0x0023, + 0x0024, 0x0025, 0x0026, 0x0027, + 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, + /* 0x30*/ + 0x0030, 0x0031, 0x0032, 0x0033, + 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, + 0x003c, 0x003d, 0x003e, 0x003f, + /* 0x40*/ + 0x0040, 0x0041, 0x0042, 0x0043, + 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, + 0x004c, 0x004d, 0x004e, 0x004f, + /* 0x50*/ + 0x0050, 0x0051, 0x0052, 0x0053, + 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, + /* 0x60*/ + 0x0060, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, + 0x006c, 0x006d, 0x006e, 0x006f, + /* 0x70*/ + 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, + 0x0078, 0x0079, 0x007a, 0x007b, + 0x007c, 0x007d, 0x007e, 0x007f, + /* 0x80*/ + 0x2500, 0x2502, 0x250c, 0x2510, + 0x2514, 0x2518, 0x251c, 0x2524, + 0x252c, 0x2534, 0x253c, 0x2580, + 0x2584, 0x2588, 0x258c, 0x2590, + /* 0x90*/ + 0x2591, 0x2592, 0x2593, 0x2320, + 0x25a0, 0x2219, 0x221a, 0x2248, + 0x2264, 0x2265, 0x00a0, 0x2321, + 0x00b0, 0x00b2, 0x00b7, 0x00f7, + /* 0xa0*/ + 0x2550, 0x2551, 0x2552, 0x0451, + 0x0454, 0x2554, 0x0456, 0x0457, + 0x2557, 0x2558, 0x2559, 0x255a, + 0x255b, 0x0491, 0x255d, 0x255e, + /* 0xb0*/ + 0x255f, 0x2560, 0x2561, 0x0401, + 0x0404, 0x2563, 0x0406, 0x0407, + 0x2566, 0x2567, 0x2568, 0x2569, + 0x256a, 0x0490, 0x256c, 0x00a9, + /* 0xc0*/ + 0x044e, 0x0430, 0x0431, 0x0446, + 0x0434, 0x0435, 0x0444, 0x0433, + 0x0445, 0x0438, 0x0439, 0x043a, + 0x043b, 0x043c, 0x043d, 0x043e, + /* 0xd0*/ + 0x043f, 0x044f, 0x0440, 0x0441, + 0x0442, 0x0443, 0x0436, 0x0432, + 0x044c, 0x044b, 0x0437, 0x0448, + 0x044d, 0x0449, 0x0447, 0x044a, + /* 0xe0*/ + 0x042e, 0x0410, 0x0411, 0x0426, + 0x0414, 0x0415, 0x0424, 0x0413, + 0x0425, 0x0418, 0x0419, 0x041a, + 0x041b, 0x041c, 0x041d, 0x041e, + /* 0xf0*/ + 0x041f, 0x042f, 0x0420, 0x0421, + 0x0422, 0x0423, 0x0416, 0x0412, + 0x042c, 0x042b, 0x0417, 0x0428, + 0x042d, 0x0429, 0x0427, 0x042a, +}; + +static const unsigned char page00[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x9a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ + 0x00, 0xbf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */ + 0x9c, 0x00, 0x9d, 0x00, 0x00, 0x00, 0x00, 0x9e, /* 0xb0-0xb7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xb8-0xbf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc0-0xc7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xc8-0xcf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd0-0xd7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xd8-0xdf */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe0-0xe7 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xe8-0xef */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x9f, /* 0xf0-0xf7 */ +}; + +static const unsigned char page04[256] = { + 0x00, 0xb3, 0x00, 0x00, 0xb4, 0x00, 0xb6, 0xb7, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0xe1, 0xe2, 0xf7, 0xe7, 0xe4, 0xe5, 0xf6, 0xfa, /* 0x10-0x17 */ + 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, /* 0x18-0x1f */ + 0xf2, 0xf3, 0xf4, 0xf5, 0xe6, 0xe8, 0xe3, 0xfe, /* 0x20-0x27 */ + 0xfb, 0xfd, 0xff, 0xf9, 0xf8, 0xfc, 0xe0, 0xf1, /* 0x28-0x2f */ + 0xc1, 0xc2, 0xd7, 0xc7, 0xc4, 0xc5, 0xd6, 0xda, /* 0x30-0x37 */ + 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, /* 0x38-0x3f */ + 0xd2, 0xd3, 0xd4, 0xd5, 0xc6, 0xc8, 0xc3, 0xde, /* 0x40-0x47 */ + 0xdb, 0xdd, 0xdf, 0xd9, 0xd8, 0xdc, 0xc0, 0xd1, /* 0x48-0x4f */ + 0x00, 0xa3, 0x00, 0x00, 0xa4, 0x00, 0xa6, 0xa7, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x60-0x67 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0xbd, 0xad, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ +}; + +static const unsigned char page22[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x95, 0x96, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x97, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x50-0x57 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x58-0x5f */ + 0x00, 0x00, 0x00, 0x00, 0x98, 0x99, 0x00, 0x00, /* 0x60-0x67 */ +}; + +static const unsigned char page23[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x93, 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x20-0x27 */ +}; + +static const unsigned char page25[256] = { + 0x80, 0x00, 0x81, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */ + 0x00, 0x00, 0x00, 0x00, 0x82, 0x00, 0x00, 0x00, /* 0x08-0x0f */ + 0x83, 0x00, 0x00, 0x00, 0x84, 0x00, 0x00, 0x00, /* 0x10-0x17 */ + 0x85, 0x00, 0x00, 0x00, 0x86, 0x00, 0x00, 0x00, /* 0x18-0x1f */ + 0x00, 0x00, 0x00, 0x00, 0x87, 0x00, 0x00, 0x00, /* 0x20-0x27 */ + 0x00, 0x00, 0x00, 0x00, 0x88, 0x00, 0x00, 0x00, /* 0x28-0x2f */ + 0x00, 0x00, 0x00, 0x00, 0x89, 0x00, 0x00, 0x00, /* 0x30-0x37 */ + 0x00, 0x00, 0x00, 0x00, 0x8a, 0x00, 0x00, 0x00, /* 0x38-0x3f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */ + 0xa0, 0xa1, 0xa2, 0x00, 0xa5, 0x00, 0x00, 0xa8, /* 0x50-0x57 */ + 0xa9, 0xaa, 0xab, 0xac, 0x00, 0xae, 0xaf, 0xb0, /* 0x58-0x5f */ + 0xb1, 0xb2, 0x00, 0xb5, 0x00, 0x00, 0xb8, 0xb9, /* 0x60-0x67 */ + 0xba, 0xbb, 0xbc, 0x00, 0xbe, 0x00, 0x00, 0x00, /* 0x68-0x6f */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */ + + 0x8b, 0x00, 0x00, 0x00, 0x8c, 0x00, 0x00, 0x00, /* 0x80-0x87 */ + 0x8d, 0x00, 0x00, 0x00, 0x8e, 0x00, 0x00, 0x00, /* 0x88-0x8f */ + 0x8f, 0x90, 0x91, 0x92, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */ + 0x94, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */ +}; + +static const unsigned char *const page_uni2charset[256] = { + page00, NULL, NULL, NULL, page04, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, page22, page23, NULL, page25, NULL, NULL, +}; + +static const unsigned char charset2lower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */ + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */ + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */ + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */ + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xa3, 0xa4, 0xb5, 0xa6, 0xa7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xad, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */ + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xe0-0xe7 */ + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xe8-0xef */ + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xf0-0xf7 */ + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xf8-0xff */ +}; + +static const unsigned char charset2upper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */ + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */ + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */ + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */ + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */ + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */ + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */ + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */ + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */ + + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */ + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */ + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */ + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */ + 0xa0, 0xa1, 0xa2, 0xb3, 0xb4, 0xa5, 0xb6, 0xb7, /* 0xa0-0xa7 */ + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xbd, 0xae, 0xaf, /* 0xa8-0xaf */ + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */ + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xc0-0xc7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xc8-0xcf */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xd0-0xd7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xd8-0xdf */ + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */ + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */ + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, /* 0xf0-0xf7 */ + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */ +}; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + const unsigned char *uni2charset; + unsigned char cl = uni & 0x00ff; + unsigned char ch = (uni & 0xff00) >> 8; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + uni2charset = page_uni2charset[ch]; + if (uni2charset && uni2charset[cl]) + out[0] = uni2charset[cl]; + else + return -EINVAL; + return 1; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + *uni = charset2uni[*rawstring]; + if (*uni == 0x0000) + return -EINVAL; + return 1; +} + +static struct nls_table table = { + .charset = "koi8-u", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = charset2lower, + .charset2upper = charset2upper, +}; + +static int __init init_nls_koi8_u(void) +{ + return register_nls(&table); +} + +static void __exit exit_nls_koi8_u(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_koi8_u) +module_exit(exit_nls_koi8_u) + +MODULE_DESCRIPTION("NLS KOI8-U (Ukrainian)"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/nls/nls_ucs2_data.h b/fs/nls/nls_ucs2_data.h new file mode 100644 index 000000000..1f454dc0f --- /dev/null +++ b/fs/nls/nls_ucs2_data.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#ifndef _NLS_UCS2_DATA_H +#define _NLS_UCS2_DATA_H + +struct UniCaseRange { + wchar_t start; + wchar_t end; + signed char *table; +}; + +extern signed char NlsUniUpperTable[512]; +extern const struct UniCaseRange NlsUniUpperRange[]; + +#endif /* _NLS_UCS2_DATA_H */ diff --git a/fs/nls/nls_ucs2_utils.c b/fs/nls/nls_ucs2_utils.c new file mode 100644 index 000000000..b81c298e4 --- /dev/null +++ b/fs/nls/nls_ucs2_utils.c @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Some of the source code in this file came from fs/cifs/uniupr.h + * Copyright (c) International Business Machines Corp., 2000,2002 + * + * Some of the source code in this file came from fs/cifs/cifs_unicode.c + * + * Copyright (c) International Business Machines Corp., 2000,2009 + * Modified by Steve French (sfrench@us.ibm.com) + * Modified by Namjae Jeon (linkinjeon@kernel.org) + * + */ +#include +#include +#include +#include +#include "nls_ucs2_utils.h" + +MODULE_DESCRIPTION("NLS UCS-2"); +MODULE_LICENSE("GPL"); + +/* + * Latin upper case + */ +signed char NlsUniUpperTable[512] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 000-00f */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 010-01f */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 020-02f */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 030-03f */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 040-04f */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 050-05f */ + 0, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, + -32, -32, -32, -32, -32, /* 060-06f */ + -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, + -32, 0, 0, 0, 0, 0, /* 070-07f */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 080-08f */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 090-09f */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0a0-0af */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0b0-0bf */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0c0-0cf */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0d0-0df */ + -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, + -32, -32, -32, -32, -32, -32, /* 0e0-0ef */ + -32, -32, -32, -32, -32, -32, -32, 0, -32, -32, + -32, -32, -32, -32, -32, 121, /* 0f0-0ff */ + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, /* 100-10f */ + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, /* 110-11f */ + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, /* 120-12f */ + 0, 0, 0, -1, 0, -1, 0, -1, 0, 0, -1, 0, -1, 0, -1, 0, /* 130-13f */ + -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, 0, -1, 0, -1, 0, -1, /* 140-14f */ + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, /* 150-15f */ + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, /* 160-16f */ + 0, -1, 0, -1, 0, -1, 0, -1, 0, 0, -1, 0, -1, 0, -1, 0, /* 170-17f */ + 0, 0, 0, -1, 0, -1, 0, 0, -1, 0, 0, 0, -1, 0, 0, 0, /* 180-18f */ + 0, 0, -1, 0, 0, 0, 0, 0, 0, -1, 0, 0, 0, 0, 0, 0, /* 190-19f */ + 0, -1, 0, -1, 0, -1, 0, 0, -1, 0, 0, 0, 0, -1, 0, 0, /* 1a0-1af */ + -1, 0, 0, 0, -1, 0, -1, 0, 0, -1, 0, 0, 0, -1, 0, 0, /* 1b0-1bf */ + 0, 0, 0, 0, 0, -1, -2, 0, -1, -2, 0, -1, -2, 0, -1, 0, /* 1c0-1cf */ + -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, -79, 0, -1, /* 1d0-1df */ + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, /* 1e0-1ef */ + 0, 0, -1, -2, 0, -1, 0, 0, 0, -1, 0, -1, 0, -1, 0, -1, /* 1f0-1ff */ +}; +EXPORT_SYMBOL_GPL(NlsUniUpperTable); + +/* Upper case range - Greek */ +static signed char UniCaseRangeU03a0[47] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -38, -37, -37, -37, /* 3a0-3af */ + 0, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, + -32, -32, -32, -32, /* 3b0-3bf */ + -32, -32, -31, -32, -32, -32, -32, -32, -32, -32, -32, -32, -64, + -63, -63, +}; + +/* Upper case range - Cyrillic */ +static signed char UniCaseRangeU0430[48] = { + -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, + -32, -32, -32, -32, /* 430-43f */ + -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, + -32, -32, -32, -32, /* 440-44f */ + 0, -80, -80, -80, -80, -80, -80, -80, -80, -80, -80, + -80, -80, 0, -80, -80, /* 450-45f */ +}; + +/* Upper case range - Extended cyrillic */ +static signed char UniCaseRangeU0490[61] = { + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, /* 490-49f */ + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, /* 4a0-4af */ + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, /* 4b0-4bf */ + 0, 0, -1, 0, -1, 0, 0, 0, -1, 0, 0, 0, -1, +}; + +/* Upper case range - Extended latin and greek */ +static signed char UniCaseRangeU1e00[509] = { + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, /* 1e00-1e0f */ + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, /* 1e10-1e1f */ + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, /* 1e20-1e2f */ + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, /* 1e30-1e3f */ + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, /* 1e40-1e4f */ + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, /* 1e50-1e5f */ + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, /* 1e60-1e6f */ + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, /* 1e70-1e7f */ + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, /* 1e80-1e8f */ + 0, -1, 0, -1, 0, -1, 0, 0, 0, 0, 0, -59, 0, -1, 0, -1, /* 1e90-1e9f */ + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, /* 1ea0-1eaf */ + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, /* 1eb0-1ebf */ + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, /* 1ec0-1ecf */ + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, /* 1ed0-1edf */ + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, /* 1ee0-1eef */ + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, 0, 0, 0, 0, 0, /* 1ef0-1eff */ + 8, 8, 8, 8, 8, 8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, /* 1f00-1f0f */ + 8, 8, 8, 8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1f10-1f1f */ + 8, 8, 8, 8, 8, 8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, /* 1f20-1f2f */ + 8, 8, 8, 8, 8, 8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, /* 1f30-1f3f */ + 8, 8, 8, 8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1f40-1f4f */ + 0, 8, 0, 8, 0, 8, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, /* 1f50-1f5f */ + 8, 8, 8, 8, 8, 8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, /* 1f60-1f6f */ + 74, 74, 86, 86, 86, 86, 100, 100, 0, 0, 112, 112, + 126, 126, 0, 0, /* 1f70-1f7f */ + 8, 8, 8, 8, 8, 8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, /* 1f80-1f8f */ + 8, 8, 8, 8, 8, 8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, /* 1f90-1f9f */ + 8, 8, 8, 8, 8, 8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, /* 1fa0-1faf */ + 8, 8, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1fb0-1fbf */ + 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1fc0-1fcf */ + 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1fd0-1fdf */ + 8, 8, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1fe0-1fef */ + 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; + +/* Upper case range - Wide latin */ +static signed char UniCaseRangeUff40[27] = { + 0, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, + -32, -32, -32, -32, -32, /* ff40-ff4f */ + -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, -32, +}; + +/* + * Upper Case Range + */ +const struct UniCaseRange NlsUniUpperRange[] = { + {0x03a0, 0x03ce, UniCaseRangeU03a0}, + {0x0430, 0x045f, UniCaseRangeU0430}, + {0x0490, 0x04cc, UniCaseRangeU0490}, + {0x1e00, 0x1ffc, UniCaseRangeU1e00}, + {0xff40, 0xff5a, UniCaseRangeUff40}, + {0} +}; +EXPORT_SYMBOL_GPL(NlsUniUpperRange); diff --git a/fs/nls/nls_ucs2_utils.h b/fs/nls/nls_ucs2_utils.h new file mode 100644 index 000000000..ef18d30db --- /dev/null +++ b/fs/nls/nls_ucs2_utils.h @@ -0,0 +1,285 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Some of the source code in this file came from fs/cifs/cifs_unicode.c + * and then via server/unicode.c + * cifs_unicode: Unicode kernel case support + * + * Function: + * Convert a unicode character to upper or lower case using + * compressed tables. + * + * Copyright (c) International Business Machines Corp., 2000,2009 + * + * + * Notes: + * These APIs are based on the C library functions. The semantics + * should match the C functions but with expanded size operands. + * + * The upper/lower functions are based on a table created by mkupr. + * This is a compressed table of upper and lower case conversion. + * + */ +#ifndef _NLS_UCS2_UTILS_H +#define _NLS_UCS2_UTILS_H + +#include +#include +#include +#include +#include "nls_ucs2_data.h" + +/* + * Windows maps these to the user defined 16 bit Unicode range since they are + * reserved symbols (along with \ and /), otherwise illegal to store + * in filenames in NTFS + */ +#define UNI_ASTERISK ((__u16)('*' + 0xF000)) +#define UNI_QUESTION ((__u16)('?' + 0xF000)) +#define UNI_COLON ((__u16)(':' + 0xF000)) +#define UNI_GRTRTHAN ((__u16)('>' + 0xF000)) +#define UNI_LESSTHAN ((__u16)('<' + 0xF000)) +#define UNI_PIPE ((__u16)('|' + 0xF000)) +#define UNI_SLASH ((__u16)('\\' + 0xF000)) + +/* + * UniStrcat: Concatenate the second string to the first + * + * Returns: + * Address of the first string + */ +static inline wchar_t *UniStrcat(wchar_t *ucs1, const wchar_t *ucs2) +{ + wchar_t *anchor = ucs1; /* save a pointer to start of ucs1 */ + + while (*ucs1++) + /*NULL*/; /* To end of first string */ + ucs1--; /* Return to the null */ + while ((*ucs1++ = *ucs2++)) + /*NULL*/; /* copy string 2 over */ + return anchor; +} + +/* + * UniStrchr: Find a character in a string + * + * Returns: + * Address of first occurrence of character in string + * or NULL if the character is not in the string + */ +static inline wchar_t *UniStrchr(const wchar_t *ucs, wchar_t uc) +{ + while ((*ucs != uc) && *ucs) + ucs++; + + if (*ucs == uc) + return (wchar_t *)ucs; + return NULL; +} + +/* + * UniStrcmp: Compare two strings + * + * Returns: + * < 0: First string is less than second + * = 0: Strings are equal + * > 0: First string is greater than second + */ +static inline int UniStrcmp(const wchar_t *ucs1, const wchar_t *ucs2) +{ + while ((*ucs1 == *ucs2) && *ucs1) { + ucs1++; + ucs2++; + } + return (int)*ucs1 - (int)*ucs2; +} + +/* + * UniStrcpy: Copy a string + */ +static inline wchar_t *UniStrcpy(wchar_t *ucs1, const wchar_t *ucs2) +{ + wchar_t *anchor = ucs1; /* save the start of result string */ + + while ((*ucs1++ = *ucs2++)) + /*NULL*/; + return anchor; +} + +/* + * UniStrlen: Return the length of a string (in 16 bit Unicode chars not bytes) + */ +static inline size_t UniStrlen(const wchar_t *ucs1) +{ + int i = 0; + + while (*ucs1++) + i++; + return i; +} + +/* + * UniStrnlen: Return the length (in 16 bit Unicode chars not bytes) of a + * string (length limited) + */ +static inline size_t UniStrnlen(const wchar_t *ucs1, int maxlen) +{ + int i = 0; + + while (*ucs1++) { + i++; + if (i >= maxlen) + break; + } + return i; +} + +/* + * UniStrncat: Concatenate length limited string + */ +static inline wchar_t *UniStrncat(wchar_t *ucs1, const wchar_t *ucs2, size_t n) +{ + wchar_t *anchor = ucs1; /* save pointer to string 1 */ + + while (*ucs1++) + /*NULL*/; + ucs1--; /* point to null terminator of s1 */ + while (n-- && (*ucs1 = *ucs2)) { /* copy s2 after s1 */ + ucs1++; + ucs2++; + } + *ucs1 = 0; /* Null terminate the result */ + return anchor; +} + +/* + * UniStrncmp: Compare length limited string + */ +static inline int UniStrncmp(const wchar_t *ucs1, const wchar_t *ucs2, size_t n) +{ + if (!n) + return 0; /* Null strings are equal */ + while ((*ucs1 == *ucs2) && *ucs1 && --n) { + ucs1++; + ucs2++; + } + return (int)*ucs1 - (int)*ucs2; +} + +/* + * UniStrncmp_le: Compare length limited string - native to little-endian + */ +static inline int +UniStrncmp_le(const wchar_t *ucs1, const wchar_t *ucs2, size_t n) +{ + if (!n) + return 0; /* Null strings are equal */ + while ((*ucs1 == __le16_to_cpu(*ucs2)) && *ucs1 && --n) { + ucs1++; + ucs2++; + } + return (int)*ucs1 - (int)__le16_to_cpu(*ucs2); +} + +/* + * UniStrncpy: Copy length limited string with pad + */ +static inline wchar_t *UniStrncpy(wchar_t *ucs1, const wchar_t *ucs2, size_t n) +{ + wchar_t *anchor = ucs1; + + while (n-- && *ucs2) /* Copy the strings */ + *ucs1++ = *ucs2++; + + n++; + while (n--) /* Pad with nulls */ + *ucs1++ = 0; + return anchor; +} + +/* + * UniStrncpy_le: Copy length limited string with pad to little-endian + */ +static inline wchar_t *UniStrncpy_le(wchar_t *ucs1, const wchar_t *ucs2, size_t n) +{ + wchar_t *anchor = ucs1; + + while (n-- && *ucs2) /* Copy the strings */ + *ucs1++ = __le16_to_cpu(*ucs2++); + + n++; + while (n--) /* Pad with nulls */ + *ucs1++ = 0; + return anchor; +} + +/* + * UniStrstr: Find a string in a string + * + * Returns: + * Address of first match found + * NULL if no matching string is found + */ +static inline wchar_t *UniStrstr(const wchar_t *ucs1, const wchar_t *ucs2) +{ + const wchar_t *anchor1 = ucs1; + const wchar_t *anchor2 = ucs2; + + while (*ucs1) { + if (*ucs1 == *ucs2) { + /* Partial match found */ + ucs1++; + ucs2++; + } else { + if (!*ucs2) /* Match found */ + return (wchar_t *)anchor1; + ucs1 = ++anchor1; /* No match */ + ucs2 = anchor2; + } + } + + if (!*ucs2) /* Both end together */ + return (wchar_t *)anchor1; /* Match found */ + return NULL; /* No match */ +} + +#ifndef UNIUPR_NOUPPER +/* + * UniToupper: Convert a unicode character to upper case + */ +static inline wchar_t UniToupper(register wchar_t uc) +{ + register const struct UniCaseRange *rp; + + if (uc < sizeof(NlsUniUpperTable)) { + /* Latin characters */ + return uc + NlsUniUpperTable[uc]; /* Use base tables */ + } + + rp = NlsUniUpperRange; /* Use range tables */ + while (rp->start) { + if (uc < rp->start) /* Before start of range */ + return uc; /* Uppercase = input */ + if (uc <= rp->end) /* In range */ + return uc + rp->table[uc - rp->start]; + rp++; /* Try next range */ + } + return uc; /* Past last range */ +} + +/* + * UniStrupr: Upper case a unicode string + */ +static inline __le16 *UniStrupr(register __le16 *upin) +{ + register __le16 *up; + + up = upin; + while (*up) { /* For all characters */ + *up = cpu_to_le16(UniToupper(le16_to_cpu(*up))); + up++; + } + return upin; /* Return input pointer */ +} +#endif /* UNIUPR_NOUPPER */ + +#endif /* _NLS_UCS2_UTILS_H */ diff --git a/fs/nls/nls_utf8.c b/fs/nls/nls_utf8.c new file mode 100644 index 000000000..a0fa0610e --- /dev/null +++ b/fs/nls/nls_utf8.c @@ -0,0 +1,68 @@ +/* + * Module for handling utf8 just like any other charset. + * By Urban Widmark 2000 + */ + +#include +#include +#include +#include +#include + +static unsigned char identity[256]; + +static int uni2char(wchar_t uni, unsigned char *out, int boundlen) +{ + int n; + + if (boundlen <= 0) + return -ENAMETOOLONG; + + n = utf32_to_utf8(uni, out, boundlen); + if (n < 0) { + *out = '?'; + return -EINVAL; + } + return n; +} + +static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) +{ + int n; + unicode_t u; + + n = utf8_to_utf32(rawstring, boundlen, &u); + if (n < 0 || u > MAX_WCHAR_T) { + *uni = 0x003f; /* ? */ + return -EINVAL; + } + *uni = (wchar_t) u; + return n; +} + +static struct nls_table table = { + .charset = "utf8", + .uni2char = uni2char, + .char2uni = char2uni, + .charset2lower = identity, /* no conversion */ + .charset2upper = identity, +}; + +static int __init init_nls_utf8(void) +{ + int i; + for (i=0; i<256; i++) + identity[i] = i; + + return register_nls(&table); +} + +static void __exit exit_nls_utf8(void) +{ + unregister_nls(&table); +} + +module_init(init_nls_utf8) +module_exit(exit_nls_utf8) +MODULE_DESCRIPTION("NLS UTF-8"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/fs/notify/Kconfig b/fs/notify/Kconfig new file mode 100644 index 000000000..c6c72c90f --- /dev/null +++ b/fs/notify/Kconfig @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only +config FSNOTIFY + def_bool n + +source "fs/notify/dnotify/Kconfig" +source "fs/notify/inotify/Kconfig" +source "fs/notify/fanotify/Kconfig" diff --git a/fs/notify/Makefile b/fs/notify/Makefile new file mode 100644 index 000000000..63a4b8828 --- /dev/null +++ b/fs/notify/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_FSNOTIFY) += fsnotify.o notification.o group.o mark.o \ + fdinfo.o + +obj-y += dnotify/ +obj-y += inotify/ +obj-y += fanotify/ diff --git a/fs/notify/dnotify/Kconfig b/fs/notify/dnotify/Kconfig new file mode 100644 index 000000000..3df7def17 --- /dev/null +++ b/fs/notify/dnotify/Kconfig @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0-only +config DNOTIFY + bool "Dnotify support" + select FSNOTIFY + default y + help + Dnotify is a directory-based per-fd file change notification system + that uses signals to communicate events to user-space. There exist + superior alternatives, but some applications may still rely on + dnotify. + + If unsure, say Y. diff --git a/fs/notify/dnotify/Makefile b/fs/notify/dnotify/Makefile new file mode 100644 index 000000000..121b4dd6b --- /dev/null +++ b/fs/notify/dnotify/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_DNOTIFY) += dnotify.o diff --git a/fs/notify/dnotify/dnotify.c b/fs/notify/dnotify/dnotify.c new file mode 100644 index 000000000..9fb73bafd --- /dev/null +++ b/fs/notify/dnotify/dnotify.c @@ -0,0 +1,413 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Directory notifications for Linux. + * + * Copyright (C) 2000,2001,2002 Stephen Rothwell + * + * Copyright (C) 2009 Eric Paris + * dnotify was largly rewritten to use the new fsnotify infrastructure + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static int dir_notify_enable __read_mostly = 1; +#ifdef CONFIG_SYSCTL +static const struct ctl_table dnotify_sysctls[] = { + { + .procname = "dir-notify-enable", + .data = &dir_notify_enable, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +}; +static void __init dnotify_sysctl_init(void) +{ + register_sysctl_init("fs", dnotify_sysctls); +} +#else +#define dnotify_sysctl_init() do { } while (0) +#endif + +static struct kmem_cache *dnotify_struct_cache __ro_after_init; +static struct kmem_cache *dnotify_mark_cache __ro_after_init; +static struct fsnotify_group *dnotify_group __ro_after_init; + +/* + * dnotify will attach one of these to each inode (i_fsnotify_marks) which + * is being watched by dnotify. If multiple userspace applications are watching + * the same directory with dnotify their information is chained in dn + */ +struct dnotify_mark { + struct fsnotify_mark fsn_mark; + struct dnotify_struct *dn; +}; + +/* + * When a process starts or stops watching an inode the set of events which + * dnotify cares about for that inode may change. This function runs the + * list of everything receiving dnotify events about this directory and calculates + * the set of all those events. After it updates what dnotify is interested in + * it calls the fsnotify function so it can update the set of all events relevant + * to this inode. + */ +static void dnotify_recalc_inode_mask(struct fsnotify_mark *fsn_mark) +{ + __u32 new_mask = 0; + struct dnotify_struct *dn; + struct dnotify_mark *dn_mark = container_of(fsn_mark, + struct dnotify_mark, + fsn_mark); + + assert_spin_locked(&fsn_mark->lock); + + for (dn = dn_mark->dn; dn != NULL; dn = dn->dn_next) + new_mask |= (dn->dn_mask & ~FS_DN_MULTISHOT); + if (fsn_mark->mask == new_mask) + return; + fsn_mark->mask = new_mask; + + fsnotify_recalc_mask(fsn_mark->connector); +} + +/* + * Mains fsnotify call where events are delivered to dnotify. + * Find the dnotify mark on the relevant inode, run the list of dnotify structs + * on that mark and determine which of them has expressed interest in receiving + * events of this type. When found send the correct process and signal and + * destroy the dnotify struct if it was not registered to receive multiple + * events. + */ +static int dnotify_handle_event(struct fsnotify_mark *inode_mark, u32 mask, + struct inode *inode, struct inode *dir, + const struct qstr *name, u32 cookie) +{ + struct dnotify_mark *dn_mark; + struct dnotify_struct *dn; + struct dnotify_struct **prev; + struct fown_struct *fown; + __u32 test_mask = mask & ~FS_EVENT_ON_CHILD; + + /* not a dir, dnotify doesn't care */ + if (!dir && !(mask & FS_ISDIR)) + return 0; + + dn_mark = container_of(inode_mark, struct dnotify_mark, fsn_mark); + + spin_lock(&inode_mark->lock); + prev = &dn_mark->dn; + while ((dn = *prev) != NULL) { + if ((dn->dn_mask & test_mask) == 0) { + prev = &dn->dn_next; + continue; + } + fown = file_f_owner(dn->dn_filp); + send_sigio(fown, dn->dn_fd, POLL_MSG); + if (dn->dn_mask & FS_DN_MULTISHOT) + prev = &dn->dn_next; + else { + *prev = dn->dn_next; + kmem_cache_free(dnotify_struct_cache, dn); + dnotify_recalc_inode_mask(inode_mark); + } + } + + spin_unlock(&inode_mark->lock); + + return 0; +} + +static void dnotify_free_mark(struct fsnotify_mark *fsn_mark) +{ + struct dnotify_mark *dn_mark = container_of(fsn_mark, + struct dnotify_mark, + fsn_mark); + + BUG_ON(dn_mark->dn); + + kmem_cache_free(dnotify_mark_cache, dn_mark); +} + +static const struct fsnotify_ops dnotify_fsnotify_ops = { + .handle_inode_event = dnotify_handle_event, + .free_mark = dnotify_free_mark, +}; + +/* + * Called every time a file is closed. Looks first for a dnotify mark on the + * inode. If one is found run all of the ->dn structures attached to that + * mark for one relevant to this process closing the file and remove that + * dnotify_struct. If that was the last dnotify_struct also remove the + * fsnotify_mark. + */ +void dnotify_flush(struct file *filp, fl_owner_t id) +{ + struct fsnotify_mark *fsn_mark; + struct dnotify_mark *dn_mark; + struct dnotify_struct *dn; + struct dnotify_struct **prev; + struct inode *inode; + bool free = false; + + inode = file_inode(filp); + if (!S_ISDIR(inode->i_mode)) + return; + + fsn_mark = fsnotify_find_inode_mark(inode, dnotify_group); + if (!fsn_mark) + return; + dn_mark = container_of(fsn_mark, struct dnotify_mark, fsn_mark); + + fsnotify_group_lock(dnotify_group); + + spin_lock(&fsn_mark->lock); + prev = &dn_mark->dn; + while ((dn = *prev) != NULL) { + if ((dn->dn_owner == id) && (dn->dn_filp == filp)) { + *prev = dn->dn_next; + kmem_cache_free(dnotify_struct_cache, dn); + dnotify_recalc_inode_mask(fsn_mark); + break; + } + prev = &dn->dn_next; + } + + spin_unlock(&fsn_mark->lock); + + /* nothing else could have found us thanks to the dnotify_groups + mark_mutex */ + if (dn_mark->dn == NULL) { + fsnotify_detach_mark(fsn_mark); + free = true; + } + + fsnotify_group_unlock(dnotify_group); + + if (free) + fsnotify_free_mark(fsn_mark); + fsnotify_put_mark(fsn_mark); +} + +/* this conversion is done only at watch creation */ +static __u32 convert_arg(unsigned int arg) +{ + __u32 new_mask = FS_EVENT_ON_CHILD; + + if (arg & DN_MULTISHOT) + new_mask |= FS_DN_MULTISHOT; + if (arg & DN_DELETE) + new_mask |= (FS_DELETE | FS_MOVED_FROM); + if (arg & DN_MODIFY) + new_mask |= FS_MODIFY; + if (arg & DN_ACCESS) + new_mask |= FS_ACCESS; + if (arg & DN_ATTRIB) + new_mask |= FS_ATTRIB; + if (arg & DN_RENAME) + new_mask |= FS_RENAME; + if (arg & DN_CREATE) + new_mask |= (FS_CREATE | FS_MOVED_TO); + + return new_mask; +} + +/* + * If multiple processes watch the same inode with dnotify there is only one + * dnotify mark in inode->i_fsnotify_marks but we chain a dnotify_struct + * onto that mark. This function either attaches the new dnotify_struct onto + * that list, or it |= the mask onto an existing dnofiy_struct. + */ +static int attach_dn(struct dnotify_struct *dn, struct dnotify_mark *dn_mark, + fl_owner_t id, int fd, struct file *filp, __u32 mask) +{ + struct dnotify_struct *odn; + + odn = dn_mark->dn; + while (odn != NULL) { + /* adding more events to existing dnofiy_struct? */ + if ((odn->dn_owner == id) && (odn->dn_filp == filp)) { + odn->dn_fd = fd; + odn->dn_mask |= mask; + return -EEXIST; + } + odn = odn->dn_next; + } + + dn->dn_mask = mask; + dn->dn_fd = fd; + dn->dn_filp = filp; + dn->dn_owner = id; + dn->dn_next = dn_mark->dn; + dn_mark->dn = dn; + + return 0; +} + +/* + * When a process calls fcntl to attach a dnotify watch to a directory it ends + * up here. Allocate both a mark for fsnotify to add and a dnotify_struct to be + * attached to the fsnotify_mark. + */ +int fcntl_dirnotify(int fd, struct file *filp, unsigned int arg) +{ + struct dnotify_mark *new_dn_mark, *dn_mark; + struct fsnotify_mark *new_fsn_mark, *fsn_mark; + struct dnotify_struct *dn; + struct inode *inode; + fl_owner_t id = current->files; + struct file *f = NULL; + int destroy = 0, error = 0; + __u32 mask; + + /* we use these to tell if we need to kfree */ + new_fsn_mark = NULL; + dn = NULL; + + if (!dir_notify_enable) { + error = -EINVAL; + goto out_err; + } + + /* a 0 mask means we are explicitly removing the watch */ + if ((arg & ~DN_MULTISHOT) == 0) { + dnotify_flush(filp, id); + error = 0; + goto out_err; + } + + /* dnotify only works on directories */ + inode = file_inode(filp); + if (!S_ISDIR(inode->i_mode)) { + error = -ENOTDIR; + goto out_err; + } + + /* + * convert the userspace DN_* "arg" to the internal FS_* + * defined in fsnotify + */ + mask = convert_arg(arg); + + error = security_path_notify(&filp->f_path, mask, + FSNOTIFY_OBJ_TYPE_INODE); + if (error) + goto out_err; + + /* expect most fcntl to add new rather than augment old */ + dn = kmem_cache_alloc(dnotify_struct_cache, GFP_KERNEL); + if (!dn) { + error = -ENOMEM; + goto out_err; + } + + error = file_f_owner_allocate(filp); + if (error) + goto out_err; + + /* new fsnotify mark, we expect most fcntl calls to add a new mark */ + new_dn_mark = kmem_cache_alloc(dnotify_mark_cache, GFP_KERNEL); + if (!new_dn_mark) { + error = -ENOMEM; + goto out_err; + } + + /* set up the new_fsn_mark and new_dn_mark */ + new_fsn_mark = &new_dn_mark->fsn_mark; + fsnotify_init_mark(new_fsn_mark, dnotify_group); + new_fsn_mark->mask = mask; + new_dn_mark->dn = NULL; + + /* this is needed to prevent the fcntl/close race described below */ + fsnotify_group_lock(dnotify_group); + + /* add the new_fsn_mark or find an old one. */ + fsn_mark = fsnotify_find_inode_mark(inode, dnotify_group); + if (fsn_mark) { + dn_mark = container_of(fsn_mark, struct dnotify_mark, fsn_mark); + spin_lock(&fsn_mark->lock); + } else { + error = fsnotify_add_inode_mark_locked(new_fsn_mark, inode, 0); + if (error) { + fsnotify_group_unlock(dnotify_group); + goto out_err; + } + spin_lock(&new_fsn_mark->lock); + fsn_mark = new_fsn_mark; + dn_mark = new_dn_mark; + /* we used new_fsn_mark, so don't free it */ + new_fsn_mark = NULL; + } + + f = fget_raw(fd); + + /* if (f != filp) means that we lost a race and another task/thread + * actually closed the fd we are still playing with before we grabbed + * the dnotify_groups mark_mutex and fsn_mark->lock. Since closing the + * fd is the only time we clean up the marks we need to get our mark + * off the list. */ + if (f != filp) { + /* if we added ourselves, shoot ourselves, it's possible that + * the flush actually did shoot this fsn_mark. That's fine too + * since multiple calls to destroy_mark is perfectly safe, if + * we found a dn_mark already attached to the inode, just sod + * off silently as the flush at close time dealt with it. + */ + if (dn_mark == new_dn_mark) + destroy = 1; + error = 0; + goto out; + } + + __f_setown(filp, task_pid(current), PIDTYPE_TGID, 0); + + error = attach_dn(dn, dn_mark, id, fd, filp, mask); + /* !error means that we attached the dn to the dn_mark, so don't free it */ + if (!error) + dn = NULL; + /* -EEXIST means that we didn't add this new dn and used an old one. + * that isn't an error (and the unused dn should be freed) */ + else if (error == -EEXIST) + error = 0; + + dnotify_recalc_inode_mask(fsn_mark); +out: + spin_unlock(&fsn_mark->lock); + + if (destroy) + fsnotify_detach_mark(fsn_mark); + fsnotify_group_unlock(dnotify_group); + if (destroy) + fsnotify_free_mark(fsn_mark); + fsnotify_put_mark(fsn_mark); +out_err: + if (new_fsn_mark) + fsnotify_put_mark(new_fsn_mark); + if (dn) + kmem_cache_free(dnotify_struct_cache, dn); + if (f) + fput(f); + return error; +} + +static int __init dnotify_init(void) +{ + dnotify_struct_cache = KMEM_CACHE(dnotify_struct, + SLAB_PANIC|SLAB_ACCOUNT); + dnotify_mark_cache = KMEM_CACHE(dnotify_mark, SLAB_PANIC|SLAB_ACCOUNT); + + dnotify_group = fsnotify_alloc_group(&dnotify_fsnotify_ops, 0); + if (IS_ERR(dnotify_group)) + panic("unable to allocate fsnotify group for dnotify\n"); + dnotify_sysctl_init(); + return 0; +} + +module_init(dnotify_init) diff --git a/fs/notify/fanotify/Kconfig b/fs/notify/fanotify/Kconfig new file mode 100644 index 000000000..0e36aaf37 --- /dev/null +++ b/fs/notify/fanotify/Kconfig @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: GPL-2.0-only +config FANOTIFY + bool "Filesystem wide access notification" + select FSNOTIFY + select EXPORTFS + default n + help + Say Y here to enable fanotify support. fanotify is a file access + notification system which differs from inotify in that it sends + an open file descriptor to the userspace listener along with + the event. + + If unsure, say Y. + +config FANOTIFY_ACCESS_PERMISSIONS + bool "fanotify permissions checking" + depends on FANOTIFY + default n + help + Say Y here is you want fanotify listeners to be able to make permissions + decisions concerning filesystem events. This is used by some fanotify + listeners which need to scan files before allowing the system access to + use those files. This is used by some anti-malware vendors and by some + hierarchical storage management systems. + + If unsure, say N. diff --git a/fs/notify/fanotify/Makefile b/fs/notify/fanotify/Makefile new file mode 100644 index 000000000..25ef22291 --- /dev/null +++ b/fs/notify/fanotify/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_FANOTIFY) += fanotify.o fanotify_user.o diff --git a/fs/notify/fanotify/fanotify.c b/fs/notify/fanotify/fanotify.c new file mode 100644 index 000000000..a208a7ec1 --- /dev/null +++ b/fs/notify/fanotify/fanotify.c @@ -0,0 +1,1121 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include /* UINT_MAX */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fanotify.h" + +static bool fanotify_path_equal(const struct path *p1, const struct path *p2) +{ + return p1->mnt == p2->mnt && p1->dentry == p2->dentry; +} + +static unsigned int fanotify_hash_path(const struct path *path) +{ + return hash_ptr(path->dentry, FANOTIFY_EVENT_HASH_BITS) ^ + hash_ptr(path->mnt, FANOTIFY_EVENT_HASH_BITS); +} + +static unsigned int fanotify_hash_fsid(__kernel_fsid_t *fsid) +{ + return hash_32(fsid->val[0], FANOTIFY_EVENT_HASH_BITS) ^ + hash_32(fsid->val[1], FANOTIFY_EVENT_HASH_BITS); +} + +static bool fanotify_fh_equal(struct fanotify_fh *fh1, + struct fanotify_fh *fh2) +{ + if (fh1->type != fh2->type || fh1->len != fh2->len) + return false; + + return !fh1->len || + !memcmp(fanotify_fh_buf(fh1), fanotify_fh_buf(fh2), fh1->len); +} + +static unsigned int fanotify_hash_fh(struct fanotify_fh *fh) +{ + long salt = (long)fh->type | (long)fh->len << 8; + + /* + * full_name_hash() works long by long, so it handles fh buf optimally. + */ + return full_name_hash((void *)salt, fanotify_fh_buf(fh), fh->len); +} + +static bool fanotify_fid_event_equal(struct fanotify_fid_event *ffe1, + struct fanotify_fid_event *ffe2) +{ + /* Do not merge fid events without object fh */ + if (!ffe1->object_fh.len) + return false; + + return fanotify_fsid_equal(&ffe1->fsid, &ffe2->fsid) && + fanotify_fh_equal(&ffe1->object_fh, &ffe2->object_fh); +} + +static bool fanotify_info_equal(struct fanotify_info *info1, + struct fanotify_info *info2) +{ + if (info1->dir_fh_totlen != info2->dir_fh_totlen || + info1->dir2_fh_totlen != info2->dir2_fh_totlen || + info1->file_fh_totlen != info2->file_fh_totlen || + info1->name_len != info2->name_len || + info1->name2_len != info2->name2_len) + return false; + + if (info1->dir_fh_totlen && + !fanotify_fh_equal(fanotify_info_dir_fh(info1), + fanotify_info_dir_fh(info2))) + return false; + + if (info1->dir2_fh_totlen && + !fanotify_fh_equal(fanotify_info_dir2_fh(info1), + fanotify_info_dir2_fh(info2))) + return false; + + if (info1->file_fh_totlen && + !fanotify_fh_equal(fanotify_info_file_fh(info1), + fanotify_info_file_fh(info2))) + return false; + + if (info1->name_len && + memcmp(fanotify_info_name(info1), fanotify_info_name(info2), + info1->name_len)) + return false; + + return !info1->name2_len || + !memcmp(fanotify_info_name2(info1), fanotify_info_name2(info2), + info1->name2_len); +} + +static bool fanotify_name_event_equal(struct fanotify_name_event *fne1, + struct fanotify_name_event *fne2) +{ + struct fanotify_info *info1 = &fne1->info; + struct fanotify_info *info2 = &fne2->info; + + /* Do not merge name events without dir fh */ + if (!info1->dir_fh_totlen) + return false; + + if (!fanotify_fsid_equal(&fne1->fsid, &fne2->fsid)) + return false; + + return fanotify_info_equal(info1, info2); +} + +static bool fanotify_error_event_equal(struct fanotify_error_event *fee1, + struct fanotify_error_event *fee2) +{ + /* Error events against the same file system are always merged. */ + return fanotify_fsid_equal(&fee1->fsid, &fee2->fsid); +} + +static bool fanotify_should_merge(struct fanotify_event *old, + struct fanotify_event *new) +{ + pr_debug("%s: old=%p new=%p\n", __func__, old, new); + + if (old->hash != new->hash || + old->type != new->type || old->pid != new->pid) + return false; + + /* + * We want to merge many dirent events in the same dir (i.e. + * creates/unlinks/renames), but we do not want to merge dirent + * events referring to subdirs with dirent events referring to + * non subdirs, otherwise, user won't be able to tell from a + * mask FAN_CREATE|FAN_DELETE|FAN_ONDIR if it describes mkdir+ + * unlink pair or rmdir+create pair of events. + */ + if ((old->mask & FS_ISDIR) != (new->mask & FS_ISDIR)) + return false; + + /* + * FAN_RENAME event is reported with special info record types, + * so we cannot merge it with other events. + */ + if ((old->mask & FAN_RENAME) != (new->mask & FAN_RENAME)) + return false; + + switch (old->type) { + case FANOTIFY_EVENT_TYPE_PATH: + return fanotify_path_equal(fanotify_event_path(old), + fanotify_event_path(new)); + case FANOTIFY_EVENT_TYPE_FID: + return fanotify_fid_event_equal(FANOTIFY_FE(old), + FANOTIFY_FE(new)); + case FANOTIFY_EVENT_TYPE_FID_NAME: + return fanotify_name_event_equal(FANOTIFY_NE(old), + FANOTIFY_NE(new)); + case FANOTIFY_EVENT_TYPE_FS_ERROR: + return fanotify_error_event_equal(FANOTIFY_EE(old), + FANOTIFY_EE(new)); + case FANOTIFY_EVENT_TYPE_MNT: + return false; + default: + WARN_ON_ONCE(1); + } + + return false; +} + +/* Limit event merges to limit CPU overhead per event */ +#define FANOTIFY_MAX_MERGE_EVENTS 128 + +/* and the list better be locked by something too! */ +static int fanotify_merge(struct fsnotify_group *group, + struct fsnotify_event *event) +{ + struct fanotify_event *old, *new = FANOTIFY_E(event); + unsigned int bucket = fanotify_event_hash_bucket(group, new); + struct hlist_head *hlist = &group->fanotify_data.merge_hash[bucket]; + int i = 0; + + pr_debug("%s: group=%p event=%p bucket=%u\n", __func__, + group, event, bucket); + + /* + * Don't merge a permission event with any other event so that we know + * the event structure we have created in fanotify_handle_event() is the + * one we should check for permission response. + */ + if (fanotify_is_perm_event(new->mask)) + return 0; + + hlist_for_each_entry(old, hlist, merge_list) { + if (++i > FANOTIFY_MAX_MERGE_EVENTS) + break; + if (fanotify_should_merge(old, new)) { + old->mask |= new->mask; + + if (fanotify_is_error_event(old->mask)) + FANOTIFY_EE(old)->err_count++; + + return 1; + } + } + + return 0; +} + +/* + * Wait for response to permission event. The function also takes care of + * freeing the permission event (or offloads that in case the wait is canceled + * by a signal). The function returns 0 in case access got allowed by userspace, + * -EPERM in case userspace disallowed the access, and -ERESTARTSYS in case + * the wait got interrupted by a signal. + */ +static int fanotify_get_response(struct fsnotify_group *group, + struct fanotify_perm_event *event, + struct fsnotify_iter_info *iter_info) +{ + int ret, errno; + + pr_debug("%s: group=%p event=%p\n", __func__, group, event); + + ret = wait_event_state(group->fanotify_data.access_waitq, + event->state == FAN_EVENT_ANSWERED, + (TASK_KILLABLE|TASK_FREEZABLE)); + + /* Signal pending? */ + if (ret < 0) { + spin_lock(&group->notification_lock); + /* Event reported to userspace and no answer yet? */ + if (event->state == FAN_EVENT_REPORTED) { + /* Event will get freed once userspace answers to it */ + event->state = FAN_EVENT_CANCELED; + spin_unlock(&group->notification_lock); + return ret; + } + /* Event not yet reported? Just remove it. */ + if (event->state == FAN_EVENT_INIT) { + fsnotify_remove_queued_event(group, &event->fae.fse); + /* Permission events are not supposed to be hashed */ + WARN_ON_ONCE(!hlist_unhashed(&event->fae.merge_list)); + } + /* + * Event may be also answered in case signal delivery raced + * with wakeup. In that case we have nothing to do besides + * freeing the event and reporting error. + */ + spin_unlock(&group->notification_lock); + goto out; + } + + /* userspace responded, convert to something usable */ + switch (event->response & FANOTIFY_RESPONSE_ACCESS) { + case FAN_ALLOW: + ret = 0; + break; + case FAN_DENY: + /* Check custom errno from pre-content events */ + errno = fanotify_get_response_errno(event->response); + if (errno) { + ret = -errno; + break; + } + fallthrough; + default: + ret = -EPERM; + } + + /* Check if the response should be audited */ + if (event->response & FAN_AUDIT) { + u32 response = event->response & + (FANOTIFY_RESPONSE_ACCESS | FANOTIFY_RESPONSE_FLAGS); + audit_fanotify(response & ~FAN_AUDIT, &event->audit_rule); + } + + pr_debug("%s: group=%p event=%p about to return ret=%d\n", __func__, + group, event, ret); +out: + fsnotify_destroy_event(group, &event->fae.fse); + + return ret; +} + +/* + * This function returns a mask for an event that only contains the flags + * that have been specifically requested by the user. Flags that may have + * been included within the event mask, but have not been explicitly + * requested by the user, will not be present in the returned mask. + */ +static u32 fanotify_group_event_mask(struct fsnotify_group *group, + struct fsnotify_iter_info *iter_info, + u32 *match_mask, u32 event_mask, + const void *data, int data_type, + struct inode *dir) +{ + __u32 marks_mask = 0, marks_ignore_mask = 0; + __u32 test_mask, user_mask = FANOTIFY_OUTGOING_EVENTS | + FANOTIFY_EVENT_FLAGS; + const struct path *path = fsnotify_data_path(data, data_type); + unsigned int fid_mode = FAN_GROUP_FLAG(group, FANOTIFY_FID_BITS); + struct fsnotify_mark *mark; + bool ondir = event_mask & FAN_ONDIR; + int type; + + pr_debug("%s: report_mask=%x mask=%x data=%p data_type=%d\n", + __func__, iter_info->report_mask, event_mask, data, data_type); + + if (FAN_GROUP_FLAG(group, FAN_REPORT_MNT)) { + if (data_type != FSNOTIFY_EVENT_MNT) + return 0; + } else if (!fid_mode) { + /* Do we have path to open a file descriptor? */ + if (!path) + return 0; + /* Path type events are only relevant for files and dirs */ + if (!d_is_reg(path->dentry) && !d_can_lookup(path->dentry)) + return 0; + } else if (!(fid_mode & FAN_REPORT_FID)) { + /* Do we have a directory inode to report? */ + if (!dir && !ondir) + return 0; + } + + fsnotify_foreach_iter_mark_type(iter_info, mark, type) { + /* + * Apply ignore mask depending on event flags in ignore mask. + */ + marks_ignore_mask |= + fsnotify_effective_ignore_mask(mark, ondir, type); + + /* + * Send the event depending on event flags in mark mask. + */ + if (!fsnotify_mask_applicable(mark->mask, ondir, type)) + continue; + + marks_mask |= mark->mask; + + /* Record the mark types of this group that matched the event */ + *match_mask |= 1U << type; + } + + test_mask = event_mask & marks_mask & ~marks_ignore_mask; + + /* + * For dirent modification events (create/delete/move) that do not carry + * the child entry name information, we report FAN_ONDIR for mkdir/rmdir + * so user can differentiate them from creat/unlink. + * + * For backward compatibility and consistency, do not report FAN_ONDIR + * to user in legacy fanotify mode (reporting fd) and report FAN_ONDIR + * to user in fid mode for all event types. + * + * We never report FAN_EVENT_ON_CHILD to user, but we do pass it in to + * fanotify_alloc_event() when group is reporting fid as indication + * that event happened on child. + */ + if (fid_mode) { + /* Do not report event flags without any event */ + if (!(test_mask & ~FANOTIFY_EVENT_FLAGS)) + return 0; + } else { + user_mask &= ~FANOTIFY_EVENT_FLAGS; + } + + return test_mask & user_mask; +} + +/* + * Check size needed to encode fanotify_fh. + * + * Return size of encoded fh without fanotify_fh header. + * Return 0 on failure to encode. + */ +static int fanotify_encode_fh_len(struct inode *inode) +{ + int dwords = 0; + int fh_len; + + if (!inode) + return 0; + + exportfs_encode_fid(inode, NULL, &dwords); + fh_len = dwords << 2; + + /* + * struct fanotify_error_event might be preallocated and is + * limited to MAX_HANDLE_SZ. This should never happen, but + * safeguard by forcing an invalid file handle. + */ + if (WARN_ON_ONCE(fh_len > MAX_HANDLE_SZ)) + return 0; + + return fh_len; +} + +/* + * Encode fanotify_fh. + * + * Return total size of encoded fh including fanotify_fh header. + * Return 0 on failure to encode. + */ +static int fanotify_encode_fh(struct fanotify_fh *fh, struct inode *inode, + unsigned int fh_len, unsigned int *hash, + gfp_t gfp) +{ + int dwords, type = 0; + char *ext_buf = NULL; + void *buf = fh + 1; + int err; + + fh->type = FILEID_ROOT; + fh->len = 0; + fh->flags = 0; + + /* + * Invalid FHs are used by FAN_FS_ERROR for errors not + * linked to any inode. The f_handle won't be reported + * back to userspace. + */ + if (!inode) + goto out; + + /* + * !gpf means preallocated variable size fh, but fh_len could + * be zero in that case if encoding fh len failed. + */ + err = -ENOENT; + if (fh_len < 4 || WARN_ON_ONCE(fh_len % 4) || fh_len > MAX_HANDLE_SZ) + goto out_err; + + /* No external buffer in a variable size allocated fh */ + if (gfp && fh_len > FANOTIFY_INLINE_FH_LEN) { + /* Treat failure to allocate fh as failure to encode fh */ + err = -ENOMEM; + ext_buf = kmalloc(fh_len, gfp); + if (!ext_buf) + goto out_err; + + *fanotify_fh_ext_buf_ptr(fh) = ext_buf; + buf = ext_buf; + fh->flags |= FANOTIFY_FH_FLAG_EXT_BUF; + } + + dwords = fh_len >> 2; + type = exportfs_encode_fid(inode, buf, &dwords); + err = -EINVAL; + /* + * Unlike file_handle, type and len of struct fanotify_fh are u8. + * Traditionally, filesystem return handle_type < 0xff, but there + * is no enforcement for that in vfs. + */ + BUILD_BUG_ON(MAX_HANDLE_SZ > 0xff || FILEID_INVALID > 0xff); + if (type <= 0 || type >= FILEID_INVALID || fh_len != dwords << 2) + goto out_err; + + fh->type = type; + fh->len = fh_len; + +out: + /* + * Mix fh into event merge key. Hash might be NULL in case of + * unhashed FID events (i.e. FAN_FS_ERROR). + */ + if (hash) + *hash ^= fanotify_hash_fh(fh); + + return FANOTIFY_FH_HDR_LEN + fh_len; + +out_err: + pr_warn_ratelimited("fanotify: failed to encode fid (type=%d, len=%d, err=%i)\n", + type, fh_len, err); + kfree(ext_buf); + *fanotify_fh_ext_buf_ptr(fh) = NULL; + /* Report the event without a file identifier on encode error */ + fh->type = FILEID_INVALID; + fh->len = 0; + return 0; +} + +/* + * FAN_REPORT_FID is ambiguous in that it reports the fid of the child for + * some events and the fid of the parent for create/delete/move events. + * + * With the FAN_REPORT_TARGET_FID flag, the fid of the child is reported + * also in create/delete/move events in addition to the fid of the parent + * and the name of the child. + */ +static inline bool fanotify_report_child_fid(unsigned int fid_mode, u32 mask) +{ + if (mask & ALL_FSNOTIFY_DIRENT_EVENTS) + return (fid_mode & FAN_REPORT_TARGET_FID); + + return (fid_mode & FAN_REPORT_FID) && !(mask & FAN_ONDIR); +} + +/* + * The inode to use as identifier when reporting fid depends on the event + * and the group flags. + * + * With the group flag FAN_REPORT_TARGET_FID, always report the child fid. + * + * Without the group flag FAN_REPORT_TARGET_FID, report the modified directory + * fid on dirent events and the child fid otherwise. + * + * For example: + * FS_ATTRIB reports the child fid even if reported on a watched parent. + * FS_CREATE reports the modified dir fid without FAN_REPORT_TARGET_FID. + * and reports the created child fid with FAN_REPORT_TARGET_FID. + */ +static struct inode *fanotify_fid_inode(u32 event_mask, const void *data, + int data_type, struct inode *dir, + unsigned int fid_mode) +{ + if ((event_mask & ALL_FSNOTIFY_DIRENT_EVENTS) && + !(fid_mode & FAN_REPORT_TARGET_FID)) + return dir; + + return fsnotify_data_inode(data, data_type); +} + +/* + * The inode to use as identifier when reporting dir fid depends on the event. + * Report the modified directory inode on dirent modification events. + * Report the "victim" inode if "victim" is a directory. + * Report the parent inode if "victim" is not a directory and event is + * reported to parent. + * Otherwise, do not report dir fid. + */ +static struct inode *fanotify_dfid_inode(u32 event_mask, const void *data, + int data_type, struct inode *dir) +{ + struct inode *inode = fsnotify_data_inode(data, data_type); + + if (event_mask & ALL_FSNOTIFY_DIRENT_EVENTS) + return dir; + + if (inode && S_ISDIR(inode->i_mode)) + return inode; + + return dir; +} + +static struct fanotify_event *fanotify_alloc_path_event(const struct path *path, + unsigned int *hash, + gfp_t gfp) +{ + struct fanotify_path_event *pevent; + + pevent = kmem_cache_alloc(fanotify_path_event_cachep, gfp); + if (!pevent) + return NULL; + + pevent->fae.type = FANOTIFY_EVENT_TYPE_PATH; + pevent->path = *path; + *hash ^= fanotify_hash_path(path); + path_get(path); + + return &pevent->fae; +} + +static struct fanotify_event *fanotify_alloc_mnt_event(u64 mnt_id, gfp_t gfp) +{ + struct fanotify_mnt_event *pevent; + + pevent = kmem_cache_alloc(fanotify_mnt_event_cachep, gfp); + if (!pevent) + return NULL; + + pevent->fae.type = FANOTIFY_EVENT_TYPE_MNT; + pevent->mnt_id = mnt_id; + + return &pevent->fae; +} + +static struct fanotify_event *fanotify_alloc_perm_event(const void *data, + int data_type, + gfp_t gfp) +{ + const struct path *path = fsnotify_data_path(data, data_type); + const struct file_range *range = + fsnotify_data_file_range(data, data_type); + struct fanotify_perm_event *pevent; + + pevent = kmem_cache_alloc(fanotify_perm_event_cachep, gfp); + if (!pevent) + return NULL; + + pevent->fae.type = FANOTIFY_EVENT_TYPE_PATH_PERM; + pevent->response = 0; + pevent->hdr.type = FAN_RESPONSE_INFO_NONE; + pevent->hdr.pad = 0; + pevent->hdr.len = 0; + pevent->state = FAN_EVENT_INIT; + pevent->watchdog_cnt = 0; + pevent->path = *path; + pevent->pos = range ? range->pos : FANOTIFY_NO_RANGE; + pevent->count = range ? range->count : 0; + path_get(path); + + return &pevent->fae; +} + +static struct fanotify_event *fanotify_alloc_fid_event(struct inode *id, + __kernel_fsid_t *fsid, + unsigned int *hash, + gfp_t gfp) +{ + struct fanotify_fid_event *ffe; + + ffe = kmem_cache_alloc(fanotify_fid_event_cachep, gfp); + if (!ffe) + return NULL; + + ffe->fae.type = FANOTIFY_EVENT_TYPE_FID; + ffe->fsid = *fsid; + *hash ^= fanotify_hash_fsid(fsid); + fanotify_encode_fh(&ffe->object_fh, id, fanotify_encode_fh_len(id), + hash, gfp); + + return &ffe->fae; +} + +static struct fanotify_event *fanotify_alloc_name_event(struct inode *dir, + __kernel_fsid_t *fsid, + const struct qstr *name, + struct inode *child, + struct dentry *moved, + unsigned int *hash, + gfp_t gfp) +{ + struct fanotify_name_event *fne; + struct fanotify_info *info; + struct fanotify_fh *dfh, *ffh; + struct inode *dir2 = moved ? d_inode(moved->d_parent) : NULL; + const struct qstr *name2 = moved ? &moved->d_name : NULL; + unsigned int dir_fh_len = fanotify_encode_fh_len(dir); + unsigned int dir2_fh_len = fanotify_encode_fh_len(dir2); + unsigned int child_fh_len = fanotify_encode_fh_len(child); + unsigned long name_len = name ? name->len : 0; + unsigned long name2_len = name2 ? name2->len : 0; + unsigned int len, size; + + /* Reserve terminating null byte even for empty name */ + size = sizeof(*fne) + name_len + name2_len + 2; + if (dir_fh_len) + size += FANOTIFY_FH_HDR_LEN + dir_fh_len; + if (dir2_fh_len) + size += FANOTIFY_FH_HDR_LEN + dir2_fh_len; + if (child_fh_len) + size += FANOTIFY_FH_HDR_LEN + child_fh_len; + fne = kmalloc(size, gfp); + if (!fne) + return NULL; + + fne->fae.type = FANOTIFY_EVENT_TYPE_FID_NAME; + fne->fsid = *fsid; + *hash ^= fanotify_hash_fsid(fsid); + info = &fne->info; + fanotify_info_init(info); + if (dir_fh_len) { + dfh = fanotify_info_dir_fh(info); + len = fanotify_encode_fh(dfh, dir, dir_fh_len, hash, 0); + fanotify_info_set_dir_fh(info, len); + } + if (dir2_fh_len) { + dfh = fanotify_info_dir2_fh(info); + len = fanotify_encode_fh(dfh, dir2, dir2_fh_len, hash, 0); + fanotify_info_set_dir2_fh(info, len); + } + if (child_fh_len) { + ffh = fanotify_info_file_fh(info); + len = fanotify_encode_fh(ffh, child, child_fh_len, hash, 0); + fanotify_info_set_file_fh(info, len); + } + if (name_len) { + fanotify_info_copy_name(info, name); + *hash ^= full_name_hash((void *)name_len, name->name, name_len); + } + if (name2_len) { + fanotify_info_copy_name2(info, name2); + *hash ^= full_name_hash((void *)name2_len, name2->name, + name2_len); + } + + pr_debug("%s: size=%u dir_fh_len=%u child_fh_len=%u name_len=%u name='%.*s'\n", + __func__, size, dir_fh_len, child_fh_len, + info->name_len, info->name_len, fanotify_info_name(info)); + + if (dir2_fh_len) { + pr_debug("%s: dir2_fh_len=%u name2_len=%u name2='%.*s'\n", + __func__, dir2_fh_len, info->name2_len, + info->name2_len, fanotify_info_name2(info)); + } + + return &fne->fae; +} + +static struct fanotify_event *fanotify_alloc_error_event( + struct fsnotify_group *group, + __kernel_fsid_t *fsid, + const void *data, int data_type, + unsigned int *hash) +{ + struct fs_error_report *report = + fsnotify_data_error_report(data, data_type); + struct inode *inode; + struct fanotify_error_event *fee; + int fh_len; + + if (WARN_ON_ONCE(!report)) + return NULL; + + fee = mempool_alloc(&group->fanotify_data.error_events_pool, GFP_NOFS); + if (!fee) + return NULL; + + fee->fae.type = FANOTIFY_EVENT_TYPE_FS_ERROR; + fee->error = report->error; + fee->err_count = 1; + fee->fsid = *fsid; + + inode = report->inode; + fh_len = fanotify_encode_fh_len(inode); + + /* Bad fh_len. Fallback to using an invalid fh. Should never happen. */ + if (!fh_len && inode) + inode = NULL; + + fanotify_encode_fh(&fee->object_fh, inode, fh_len, NULL, 0); + + *hash ^= fanotify_hash_fsid(fsid); + + return &fee->fae; +} + +static struct fanotify_event *fanotify_alloc_event( + struct fsnotify_group *group, + u32 mask, const void *data, int data_type, + struct inode *dir, const struct qstr *file_name, + __kernel_fsid_t *fsid, u32 match_mask) +{ + struct fanotify_event *event = NULL; + gfp_t gfp = GFP_KERNEL_ACCOUNT; + unsigned int fid_mode = FAN_GROUP_FLAG(group, FANOTIFY_FID_BITS); + struct inode *id = fanotify_fid_inode(mask, data, data_type, dir, + fid_mode); + struct inode *dirid = fanotify_dfid_inode(mask, data, data_type, dir); + const struct path *path = fsnotify_data_path(data, data_type); + u64 mnt_id = fsnotify_data_mnt_id(data, data_type); + struct mem_cgroup *old_memcg; + struct dentry *moved = NULL; + struct inode *child = NULL; + bool name_event = false; + unsigned int hash = 0; + bool ondir = mask & FAN_ONDIR; + struct pid *pid; + + if ((fid_mode & FAN_REPORT_DIR_FID) && dirid) { + /* + * For certain events and group flags, report the child fid + * in addition to reporting the parent fid and maybe child name. + */ + if (fanotify_report_child_fid(fid_mode, mask) && id != dirid) + child = id; + + id = dirid; + + /* + * We record file name only in a group with FAN_REPORT_NAME + * and when we have a directory inode to report. + * + * For directory entry modification event, we record the fid of + * the directory and the name of the modified entry. + * + * For event on non-directory that is reported to parent, we + * record the fid of the parent and the name of the child. + * + * Even if not reporting name, we need a variable length + * fanotify_name_event if reporting both parent and child fids. + */ + if (!(fid_mode & FAN_REPORT_NAME)) { + name_event = !!child; + file_name = NULL; + } else if ((mask & ALL_FSNOTIFY_DIRENT_EVENTS) || !ondir) { + name_event = true; + } + + /* + * In the special case of FAN_RENAME event, use the match_mask + * to determine if we need to report only the old parent+name, + * only the new parent+name or both. + * 'dirid' and 'file_name' are the old parent+name and + * 'moved' has the new parent+name. + */ + if (mask & FAN_RENAME) { + bool report_old, report_new; + + if (WARN_ON_ONCE(!match_mask)) + return NULL; + + /* Report both old and new parent+name if sb watching */ + report_old = report_new = + match_mask & (1U << FSNOTIFY_ITER_TYPE_SB); + report_old |= + match_mask & (1U << FSNOTIFY_ITER_TYPE_INODE); + report_new |= + match_mask & (1U << FSNOTIFY_ITER_TYPE_INODE2); + + if (!report_old) { + /* Do not report old parent+name */ + dirid = NULL; + file_name = NULL; + } + if (report_new) { + /* Report new parent+name */ + moved = fsnotify_data_dentry(data, data_type); + } + } + } + + /* + * For queues with unlimited length lost events are not expected and + * can possibly have security implications. Avoid losing events when + * memory is short. For the limited size queues, avoid OOM killer in the + * target monitoring memcg as it may have security repercussion. + */ + if (group->max_events == UINT_MAX) + gfp |= __GFP_NOFAIL; + else + gfp |= __GFP_RETRY_MAYFAIL; + + /* Whoever is interested in the event, pays for the allocation. */ + old_memcg = set_active_memcg(group->memcg); + + if (FAN_GROUP_FLAG(group, FAN_REPORT_TID)) + pid = task_pid(current); + else + pid = task_tgid(current); + + if (FAN_GROUP_FLAG(group, FAN_REPORT_PIDFD) && + pidfs_register_pid_gfp(pid, gfp)) + goto out; + + if (fanotify_is_perm_event(mask)) { + event = fanotify_alloc_perm_event(data, data_type, gfp); + } else if (fanotify_is_error_event(mask)) { + event = fanotify_alloc_error_event(group, fsid, data, + data_type, &hash); + } else if (name_event && (file_name || moved || child)) { + event = fanotify_alloc_name_event(dirid, fsid, file_name, child, + moved, &hash, gfp); + } else if (fid_mode) { + event = fanotify_alloc_fid_event(id, fsid, &hash, gfp); + } else if (path) { + event = fanotify_alloc_path_event(path, &hash, gfp); + } else if (mnt_id) { + event = fanotify_alloc_mnt_event(mnt_id, gfp); + } else { + WARN_ON_ONCE(1); + } + + if (!event) + goto out; + + /* Mix event info, FAN_ONDIR flag and pid into event merge key */ + hash ^= hash_long((unsigned long)pid | ondir, FANOTIFY_EVENT_HASH_BITS); + fanotify_init_event(event, hash, mask); + event->pid = get_pid(pid); + +out: + set_active_memcg(old_memcg); + return event; +} + +/* + * Get cached fsid of the filesystem containing the object from any mark. + * All marks are supposed to have the same fsid, but we do not verify that here. + */ +static __kernel_fsid_t fanotify_get_fsid(struct fsnotify_iter_info *iter_info) +{ + struct fsnotify_mark *mark; + int type; + __kernel_fsid_t fsid = {}; + + fsnotify_foreach_iter_mark_type(iter_info, mark, type) { + if (!(mark->flags & FSNOTIFY_MARK_FLAG_HAS_FSID)) + continue; + fsid = FANOTIFY_MARK(mark)->fsid; + if (!(mark->flags & FSNOTIFY_MARK_FLAG_WEAK_FSID) && + WARN_ON_ONCE(!fsid.val[0] && !fsid.val[1])) + continue; + return fsid; + } + + return fsid; +} + +/* + * Add an event to hash table for faster merge. + */ +static void fanotify_insert_event(struct fsnotify_group *group, + struct fsnotify_event *fsn_event) +{ + struct fanotify_event *event = FANOTIFY_E(fsn_event); + unsigned int bucket = fanotify_event_hash_bucket(group, event); + struct hlist_head *hlist = &group->fanotify_data.merge_hash[bucket]; + + assert_spin_locked(&group->notification_lock); + + if (!fanotify_is_hashed_event(event->mask)) + return; + + pr_debug("%s: group=%p event=%p bucket=%u\n", __func__, + group, event, bucket); + + hlist_add_head(&event->merge_list, hlist); +} + +static int fanotify_handle_event(struct fsnotify_group *group, u32 mask, + const void *data, int data_type, + struct inode *dir, + const struct qstr *file_name, u32 cookie, + struct fsnotify_iter_info *iter_info) +{ + int ret = 0; + struct fanotify_event *event; + struct fsnotify_event *fsn_event; + __kernel_fsid_t fsid = {}; + u32 match_mask = 0; + + BUILD_BUG_ON(FAN_ACCESS != FS_ACCESS); + BUILD_BUG_ON(FAN_MODIFY != FS_MODIFY); + BUILD_BUG_ON(FAN_ATTRIB != FS_ATTRIB); + BUILD_BUG_ON(FAN_CLOSE_NOWRITE != FS_CLOSE_NOWRITE); + BUILD_BUG_ON(FAN_CLOSE_WRITE != FS_CLOSE_WRITE); + BUILD_BUG_ON(FAN_OPEN != FS_OPEN); + BUILD_BUG_ON(FAN_MOVED_TO != FS_MOVED_TO); + BUILD_BUG_ON(FAN_MOVED_FROM != FS_MOVED_FROM); + BUILD_BUG_ON(FAN_CREATE != FS_CREATE); + BUILD_BUG_ON(FAN_DELETE != FS_DELETE); + BUILD_BUG_ON(FAN_DELETE_SELF != FS_DELETE_SELF); + BUILD_BUG_ON(FAN_MOVE_SELF != FS_MOVE_SELF); + BUILD_BUG_ON(FAN_EVENT_ON_CHILD != FS_EVENT_ON_CHILD); + BUILD_BUG_ON(FAN_Q_OVERFLOW != FS_Q_OVERFLOW); + BUILD_BUG_ON(FAN_OPEN_PERM != FS_OPEN_PERM); + BUILD_BUG_ON(FAN_ACCESS_PERM != FS_ACCESS_PERM); + BUILD_BUG_ON(FAN_ONDIR != FS_ISDIR); + BUILD_BUG_ON(FAN_OPEN_EXEC != FS_OPEN_EXEC); + BUILD_BUG_ON(FAN_OPEN_EXEC_PERM != FS_OPEN_EXEC_PERM); + BUILD_BUG_ON(FAN_FS_ERROR != FS_ERROR); + BUILD_BUG_ON(FAN_RENAME != FS_RENAME); + BUILD_BUG_ON(FAN_PRE_ACCESS != FS_PRE_ACCESS); + + BUILD_BUG_ON(HWEIGHT32(ALL_FANOTIFY_EVENT_BITS) != 24); + + mask = fanotify_group_event_mask(group, iter_info, &match_mask, + mask, data, data_type, dir); + if (!mask) + return 0; + + pr_debug("%s: group=%p mask=%x report_mask=%x\n", __func__, + group, mask, match_mask); + + if (fanotify_is_perm_event(mask)) { + /* + * fsnotify_prepare_user_wait() fails if we race with mark + * deletion. Just let the operation pass in that case. + */ + if (!fsnotify_prepare_user_wait(iter_info)) + return 0; + } + + if (FAN_GROUP_FLAG(group, FANOTIFY_FID_BITS)) + fsid = fanotify_get_fsid(iter_info); + + event = fanotify_alloc_event(group, mask, data, data_type, dir, + file_name, &fsid, match_mask); + ret = -ENOMEM; + if (unlikely(!event)) { + /* + * We don't queue overflow events for permission events as + * there the access is denied and so no event is in fact lost. + */ + if (!fanotify_is_perm_event(mask)) + fsnotify_queue_overflow(group); + goto finish; + } + + fsn_event = &event->fse; + ret = fsnotify_insert_event(group, fsn_event, fanotify_merge, + fanotify_insert_event); + if (ret) { + /* Permission events shouldn't be merged */ + BUG_ON(ret == 1 && mask & FANOTIFY_PERM_EVENTS); + /* Our event wasn't used in the end. Free it. */ + fsnotify_destroy_event(group, fsn_event); + + ret = 0; + } else if (fanotify_is_perm_event(mask)) { + ret = fanotify_get_response(group, FANOTIFY_PERM(event), + iter_info); + } +finish: + if (fanotify_is_perm_event(mask)) + fsnotify_finish_user_wait(iter_info); + + return ret; +} + +static void fanotify_free_group_priv(struct fsnotify_group *group) +{ + put_user_ns(group->user_ns); + kfree(group->fanotify_data.merge_hash); + if (group->fanotify_data.ucounts) + dec_ucount(group->fanotify_data.ucounts, + UCOUNT_FANOTIFY_GROUPS); + + if (mempool_initialized(&group->fanotify_data.error_events_pool)) + mempool_exit(&group->fanotify_data.error_events_pool); +} + +static void fanotify_free_path_event(struct fanotify_event *event) +{ + path_put(fanotify_event_path(event)); + kmem_cache_free(fanotify_path_event_cachep, FANOTIFY_PE(event)); +} + +static void fanotify_free_perm_event(struct fanotify_event *event) +{ + path_put(fanotify_event_path(event)); + kmem_cache_free(fanotify_perm_event_cachep, FANOTIFY_PERM(event)); +} + +static void fanotify_free_fid_event(struct fanotify_event *event) +{ + struct fanotify_fid_event *ffe = FANOTIFY_FE(event); + + if (fanotify_fh_has_ext_buf(&ffe->object_fh)) + kfree(fanotify_fh_ext_buf(&ffe->object_fh)); + kmem_cache_free(fanotify_fid_event_cachep, ffe); +} + +static void fanotify_free_name_event(struct fanotify_event *event) +{ + kfree(FANOTIFY_NE(event)); +} + +static void fanotify_free_error_event(struct fsnotify_group *group, + struct fanotify_event *event) +{ + struct fanotify_error_event *fee = FANOTIFY_EE(event); + + mempool_free(fee, &group->fanotify_data.error_events_pool); +} + +static void fanotify_free_mnt_event(struct fanotify_event *event) +{ + kmem_cache_free(fanotify_mnt_event_cachep, FANOTIFY_ME(event)); +} + +static void fanotify_free_event(struct fsnotify_group *group, + struct fsnotify_event *fsn_event) +{ + struct fanotify_event *event; + + event = FANOTIFY_E(fsn_event); + put_pid(event->pid); + switch (event->type) { + case FANOTIFY_EVENT_TYPE_PATH: + fanotify_free_path_event(event); + break; + case FANOTIFY_EVENT_TYPE_PATH_PERM: + fanotify_free_perm_event(event); + break; + case FANOTIFY_EVENT_TYPE_FID: + fanotify_free_fid_event(event); + break; + case FANOTIFY_EVENT_TYPE_FID_NAME: + fanotify_free_name_event(event); + break; + case FANOTIFY_EVENT_TYPE_OVERFLOW: + kfree(event); + break; + case FANOTIFY_EVENT_TYPE_FS_ERROR: + fanotify_free_error_event(group, event); + break; + case FANOTIFY_EVENT_TYPE_MNT: + fanotify_free_mnt_event(event); + break; + default: + WARN_ON_ONCE(1); + } +} + +static void fanotify_freeing_mark(struct fsnotify_mark *mark, + struct fsnotify_group *group) +{ + if (!FAN_GROUP_FLAG(group, FAN_UNLIMITED_MARKS)) + dec_ucount(group->fanotify_data.ucounts, UCOUNT_FANOTIFY_MARKS); +} + +static void fanotify_free_mark(struct fsnotify_mark *fsn_mark) +{ + kmem_cache_free(fanotify_mark_cache, FANOTIFY_MARK(fsn_mark)); +} + +const struct fsnotify_ops fanotify_fsnotify_ops = { + .handle_event = fanotify_handle_event, + .free_group_priv = fanotify_free_group_priv, + .free_event = fanotify_free_event, + .freeing_mark = fanotify_freeing_mark, + .free_mark = fanotify_free_mark, +}; diff --git a/fs/notify/fanotify/fanotify.h b/fs/notify/fanotify/fanotify.h new file mode 100644 index 000000000..3710543db --- /dev/null +++ b/fs/notify/fanotify/fanotify.h @@ -0,0 +1,557 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include +#include +#include +#include +#include + +extern struct kmem_cache *fanotify_mark_cache; +extern struct kmem_cache *fanotify_fid_event_cachep; +extern struct kmem_cache *fanotify_path_event_cachep; +extern struct kmem_cache *fanotify_perm_event_cachep; +extern struct kmem_cache *fanotify_mnt_event_cachep; + +/* Possible states of the permission event */ +enum { + FAN_EVENT_INIT, + FAN_EVENT_REPORTED, + FAN_EVENT_ANSWERED, + FAN_EVENT_CANCELED, +}; + +/* + * 3 dwords are sufficient for most local fs (64bit ino, 32bit generation). + * fh buf should be dword aligned. On 64bit arch, the ext_buf pointer is + * stored in either the first or last 2 dwords. + */ +#define FANOTIFY_INLINE_FH_LEN (3 << 2) +#define FANOTIFY_FH_HDR_LEN sizeof(struct fanotify_fh) + +/* Fixed size struct for file handle */ +struct fanotify_fh { + u8 type; + u8 len; +#define FANOTIFY_FH_FLAG_EXT_BUF 1 + u8 flags; + u8 pad; +} __aligned(4); + +/* Variable size struct for dir file handle + child file handle + name */ +struct fanotify_info { + /* size of dir_fh/file_fh including fanotify_fh hdr size */ + u8 dir_fh_totlen; + u8 dir2_fh_totlen; + u8 file_fh_totlen; + u8 name_len; + u8 name2_len; + u8 pad[3]; + unsigned char buf[]; + /* + * (struct fanotify_fh) dir_fh starts at buf[0] + * (optional) dir2_fh starts at buf[dir_fh_totlen] + * (optional) file_fh starts at buf[dir_fh_totlen + dir2_fh_totlen] + * name starts at buf[dir_fh_totlen + dir2_fh_totlen + file_fh_totlen] + * ... + */ +#define FANOTIFY_DIR_FH_SIZE(info) ((info)->dir_fh_totlen) +#define FANOTIFY_DIR2_FH_SIZE(info) ((info)->dir2_fh_totlen) +#define FANOTIFY_FILE_FH_SIZE(info) ((info)->file_fh_totlen) +#define FANOTIFY_NAME_SIZE(info) ((info)->name_len + 1) +#define FANOTIFY_NAME2_SIZE(info) ((info)->name2_len + 1) + +#define FANOTIFY_DIR_FH_OFFSET(info) 0 +#define FANOTIFY_DIR2_FH_OFFSET(info) \ + (FANOTIFY_DIR_FH_OFFSET(info) + FANOTIFY_DIR_FH_SIZE(info)) +#define FANOTIFY_FILE_FH_OFFSET(info) \ + (FANOTIFY_DIR2_FH_OFFSET(info) + FANOTIFY_DIR2_FH_SIZE(info)) +#define FANOTIFY_NAME_OFFSET(info) \ + (FANOTIFY_FILE_FH_OFFSET(info) + FANOTIFY_FILE_FH_SIZE(info)) +#define FANOTIFY_NAME2_OFFSET(info) \ + (FANOTIFY_NAME_OFFSET(info) + FANOTIFY_NAME_SIZE(info)) + +#define FANOTIFY_DIR_FH_BUF(info) \ + ((info)->buf + FANOTIFY_DIR_FH_OFFSET(info)) +#define FANOTIFY_DIR2_FH_BUF(info) \ + ((info)->buf + FANOTIFY_DIR2_FH_OFFSET(info)) +#define FANOTIFY_FILE_FH_BUF(info) \ + ((info)->buf + FANOTIFY_FILE_FH_OFFSET(info)) +#define FANOTIFY_NAME_BUF(info) \ + ((info)->buf + FANOTIFY_NAME_OFFSET(info)) +#define FANOTIFY_NAME2_BUF(info) \ + ((info)->buf + FANOTIFY_NAME2_OFFSET(info)) +} __aligned(4); + +static inline bool fanotify_fh_has_ext_buf(struct fanotify_fh *fh) +{ + return (fh->flags & FANOTIFY_FH_FLAG_EXT_BUF); +} + +static inline char **fanotify_fh_ext_buf_ptr(struct fanotify_fh *fh) +{ + BUILD_BUG_ON(FANOTIFY_FH_HDR_LEN % 4); + BUILD_BUG_ON(__alignof__(char *) - 4 + sizeof(char *) > + FANOTIFY_INLINE_FH_LEN); + return (char **)ALIGN((unsigned long)(fh + 1), __alignof__(char *)); +} + +static inline void *fanotify_fh_ext_buf(struct fanotify_fh *fh) +{ + return *fanotify_fh_ext_buf_ptr(fh); +} + +static inline void *fanotify_fh_buf(struct fanotify_fh *fh) +{ + return fanotify_fh_has_ext_buf(fh) ? fanotify_fh_ext_buf(fh) : fh + 1; +} + +static inline int fanotify_info_dir_fh_len(struct fanotify_info *info) +{ + if (!info->dir_fh_totlen || + WARN_ON_ONCE(info->dir_fh_totlen < FANOTIFY_FH_HDR_LEN)) + return 0; + + return info->dir_fh_totlen - FANOTIFY_FH_HDR_LEN; +} + +static inline struct fanotify_fh *fanotify_info_dir_fh(struct fanotify_info *info) +{ + BUILD_BUG_ON(offsetof(struct fanotify_info, buf) % 4); + + return (struct fanotify_fh *)FANOTIFY_DIR_FH_BUF(info); +} + +static inline int fanotify_info_dir2_fh_len(struct fanotify_info *info) +{ + if (!info->dir2_fh_totlen || + WARN_ON_ONCE(info->dir2_fh_totlen < FANOTIFY_FH_HDR_LEN)) + return 0; + + return info->dir2_fh_totlen - FANOTIFY_FH_HDR_LEN; +} + +static inline struct fanotify_fh *fanotify_info_dir2_fh(struct fanotify_info *info) +{ + return (struct fanotify_fh *)FANOTIFY_DIR2_FH_BUF(info); +} + +static inline int fanotify_info_file_fh_len(struct fanotify_info *info) +{ + if (!info->file_fh_totlen || + WARN_ON_ONCE(info->file_fh_totlen < FANOTIFY_FH_HDR_LEN)) + return 0; + + return info->file_fh_totlen - FANOTIFY_FH_HDR_LEN; +} + +static inline struct fanotify_fh *fanotify_info_file_fh(struct fanotify_info *info) +{ + return (struct fanotify_fh *)FANOTIFY_FILE_FH_BUF(info); +} + +static inline char *fanotify_info_name(struct fanotify_info *info) +{ + if (!info->name_len) + return NULL; + + return FANOTIFY_NAME_BUF(info); +} + +static inline char *fanotify_info_name2(struct fanotify_info *info) +{ + if (!info->name2_len) + return NULL; + + return FANOTIFY_NAME2_BUF(info); +} + +static inline void fanotify_info_init(struct fanotify_info *info) +{ + BUILD_BUG_ON(FANOTIFY_FH_HDR_LEN + MAX_HANDLE_SZ > U8_MAX); + BUILD_BUG_ON(NAME_MAX > U8_MAX); + + info->dir_fh_totlen = 0; + info->dir2_fh_totlen = 0; + info->file_fh_totlen = 0; + info->name_len = 0; + info->name2_len = 0; +} + +/* These set/copy helpers MUST be called by order */ +static inline void fanotify_info_set_dir_fh(struct fanotify_info *info, + unsigned int totlen) +{ + if (WARN_ON_ONCE(info->dir2_fh_totlen > 0) || + WARN_ON_ONCE(info->file_fh_totlen > 0) || + WARN_ON_ONCE(info->name_len > 0) || + WARN_ON_ONCE(info->name2_len > 0)) + return; + + info->dir_fh_totlen = totlen; +} + +static inline void fanotify_info_set_dir2_fh(struct fanotify_info *info, + unsigned int totlen) +{ + if (WARN_ON_ONCE(info->file_fh_totlen > 0) || + WARN_ON_ONCE(info->name_len > 0) || + WARN_ON_ONCE(info->name2_len > 0)) + return; + + info->dir2_fh_totlen = totlen; +} + +static inline void fanotify_info_set_file_fh(struct fanotify_info *info, + unsigned int totlen) +{ + if (WARN_ON_ONCE(info->name_len > 0) || + WARN_ON_ONCE(info->name2_len > 0)) + return; + + info->file_fh_totlen = totlen; +} + +static inline void fanotify_info_copy_name(struct fanotify_info *info, + const struct qstr *name) +{ + if (WARN_ON_ONCE(name->len > NAME_MAX) || + WARN_ON_ONCE(info->name2_len > 0)) + return; + + info->name_len = name->len; + strscpy(fanotify_info_name(info), name->name, name->len + 1); +} + +static inline void fanotify_info_copy_name2(struct fanotify_info *info, + const struct qstr *name) +{ + if (WARN_ON_ONCE(name->len > NAME_MAX)) + return; + + info->name2_len = name->len; + strscpy(fanotify_info_name2(info), name->name, name->len + 1); +} + +/* + * Common structure for fanotify events. Concrete structs are allocated in + * fanotify_handle_event() and freed when the information is retrieved by + * userspace. The type of event determines how it was allocated, how it will + * be freed and which concrete struct it may be cast to. + */ +enum fanotify_event_type { + FANOTIFY_EVENT_TYPE_FID, /* fixed length */ + FANOTIFY_EVENT_TYPE_FID_NAME, /* variable length */ + FANOTIFY_EVENT_TYPE_PATH, + FANOTIFY_EVENT_TYPE_PATH_PERM, + FANOTIFY_EVENT_TYPE_OVERFLOW, /* struct fanotify_event */ + FANOTIFY_EVENT_TYPE_FS_ERROR, /* struct fanotify_error_event */ + FANOTIFY_EVENT_TYPE_MNT, + __FANOTIFY_EVENT_TYPE_NUM +}; + +#define FANOTIFY_EVENT_TYPE_BITS \ + (ilog2(__FANOTIFY_EVENT_TYPE_NUM - 1) + 1) +#define FANOTIFY_EVENT_HASH_BITS \ + (32 - FANOTIFY_EVENT_TYPE_BITS) + +struct fanotify_event { + struct fsnotify_event fse; + struct hlist_node merge_list; /* List for hashed merge */ + u32 mask; + struct { + unsigned int type : FANOTIFY_EVENT_TYPE_BITS; + unsigned int hash : FANOTIFY_EVENT_HASH_BITS; + }; + struct pid *pid; +}; + +static inline void fanotify_init_event(struct fanotify_event *event, + unsigned int hash, u32 mask) +{ + fsnotify_init_event(&event->fse); + INIT_HLIST_NODE(&event->merge_list); + event->hash = hash; + event->mask = mask; + event->pid = NULL; +} + +#define FANOTIFY_INLINE_FH(name, size) \ +struct { \ + struct fanotify_fh name; \ + /* Space for filehandle - access with fanotify_fh_buf() */ \ + unsigned char _inline_fh_buf[size]; \ +} + +struct fanotify_fid_event { + struct fanotify_event fae; + __kernel_fsid_t fsid; + + FANOTIFY_INLINE_FH(object_fh, FANOTIFY_INLINE_FH_LEN); +}; + +static inline struct fanotify_fid_event * +FANOTIFY_FE(struct fanotify_event *event) +{ + return container_of(event, struct fanotify_fid_event, fae); +} + +struct fanotify_name_event { + struct fanotify_event fae; + __kernel_fsid_t fsid; + struct fanotify_info info; +}; + +static inline struct fanotify_name_event * +FANOTIFY_NE(struct fanotify_event *event) +{ + return container_of(event, struct fanotify_name_event, fae); +} + +struct fanotify_error_event { + struct fanotify_event fae; + s32 error; /* Error reported by the Filesystem. */ + u32 err_count; /* Suppressed errors count */ + + __kernel_fsid_t fsid; /* FSID this error refers to. */ + + FANOTIFY_INLINE_FH(object_fh, MAX_HANDLE_SZ); +}; + +static inline struct fanotify_error_event * +FANOTIFY_EE(struct fanotify_event *event) +{ + return container_of(event, struct fanotify_error_event, fae); +} + +static inline __kernel_fsid_t *fanotify_event_fsid(struct fanotify_event *event) +{ + if (event->type == FANOTIFY_EVENT_TYPE_FID) + return &FANOTIFY_FE(event)->fsid; + else if (event->type == FANOTIFY_EVENT_TYPE_FID_NAME) + return &FANOTIFY_NE(event)->fsid; + else if (event->type == FANOTIFY_EVENT_TYPE_FS_ERROR) + return &FANOTIFY_EE(event)->fsid; + else + return NULL; +} + +static inline struct fanotify_fh *fanotify_event_object_fh( + struct fanotify_event *event) +{ + if (event->type == FANOTIFY_EVENT_TYPE_FID) + return &FANOTIFY_FE(event)->object_fh; + else if (event->type == FANOTIFY_EVENT_TYPE_FID_NAME) + return fanotify_info_file_fh(&FANOTIFY_NE(event)->info); + else if (event->type == FANOTIFY_EVENT_TYPE_FS_ERROR) + return &FANOTIFY_EE(event)->object_fh; + else + return NULL; +} + +static inline struct fanotify_info *fanotify_event_info( + struct fanotify_event *event) +{ + if (event->type == FANOTIFY_EVENT_TYPE_FID_NAME) + return &FANOTIFY_NE(event)->info; + else + return NULL; +} + +static inline int fanotify_event_object_fh_len(struct fanotify_event *event) +{ + struct fanotify_info *info = fanotify_event_info(event); + struct fanotify_fh *fh = fanotify_event_object_fh(event); + + if (info) + return info->file_fh_totlen ? fh->len : 0; + else + return fh ? fh->len : 0; +} + +static inline int fanotify_event_dir_fh_len(struct fanotify_event *event) +{ + struct fanotify_info *info = fanotify_event_info(event); + + return info ? fanotify_info_dir_fh_len(info) : 0; +} + +static inline int fanotify_event_dir2_fh_len(struct fanotify_event *event) +{ + struct fanotify_info *info = fanotify_event_info(event); + + return info ? fanotify_info_dir2_fh_len(info) : 0; +} + +static inline bool fanotify_event_has_object_fh(struct fanotify_event *event) +{ + /* For error events, even zeroed fh are reported. */ + if (event->type == FANOTIFY_EVENT_TYPE_FS_ERROR) + return true; + return fanotify_event_object_fh_len(event) > 0; +} + +static inline bool fanotify_event_has_dir_fh(struct fanotify_event *event) +{ + return fanotify_event_dir_fh_len(event) > 0; +} + +static inline bool fanotify_event_has_dir2_fh(struct fanotify_event *event) +{ + return fanotify_event_dir2_fh_len(event) > 0; +} + +static inline bool fanotify_event_has_any_dir_fh(struct fanotify_event *event) +{ + return fanotify_event_has_dir_fh(event) || + fanotify_event_has_dir2_fh(event); +} + +struct fanotify_path_event { + struct fanotify_event fae; + struct path path; +}; + +struct fanotify_mnt_event { + struct fanotify_event fae; + u64 mnt_id; +}; + +static inline struct fanotify_path_event * +FANOTIFY_PE(struct fanotify_event *event) +{ + return container_of(event, struct fanotify_path_event, fae); +} + +static inline struct fanotify_mnt_event * +FANOTIFY_ME(struct fanotify_event *event) +{ + return container_of(event, struct fanotify_mnt_event, fae); +} + +#define FANOTIFY_NO_RANGE ((loff_t)-1) + +/* + * Structure for permission fanotify events. It gets allocated and freed in + * fanotify_handle_event() since we wait there for user response. When the + * information is retrieved by userspace the structure is moved from + * group->notification_list to group->fanotify_data.access_list to wait for + * user response. + */ +struct fanotify_perm_event { + struct fanotify_event fae; + struct path path; + loff_t pos; /* FANOTIFY_NO_RANGE if unavailable */ + size_t count; + u32 response; /* userspace answer to the event */ + unsigned short state; /* state of the event */ + unsigned short watchdog_cnt; /* already scanned by watchdog? */ + int fd; /* fd we passed to userspace for this event */ + pid_t recv_pid; /* pid of task receiving the event */ + union { + struct fanotify_response_info_header hdr; + struct fanotify_response_info_audit_rule audit_rule; + }; +}; + +static inline struct fanotify_perm_event * +FANOTIFY_PERM(struct fanotify_event *event) +{ + return container_of(event, struct fanotify_perm_event, fae); +} + +static inline bool fanotify_is_perm_event(u32 mask) +{ + return IS_ENABLED(CONFIG_FANOTIFY_ACCESS_PERMISSIONS) && + mask & FANOTIFY_PERM_EVENTS; +} + +static inline bool fanotify_event_has_access_range(struct fanotify_event *event) +{ + if (!(event->mask & FANOTIFY_PRE_CONTENT_EVENTS)) + return false; + + return FANOTIFY_PERM(event)->pos != FANOTIFY_NO_RANGE; +} + +static inline struct fanotify_event *FANOTIFY_E(struct fsnotify_event *fse) +{ + return container_of(fse, struct fanotify_event, fse); +} + +static inline bool fanotify_is_error_event(u32 mask) +{ + return mask & FAN_FS_ERROR; +} + +static inline bool fanotify_is_mnt_event(u32 mask) +{ + return mask & (FAN_MNT_ATTACH | FAN_MNT_DETACH); +} + +static inline const struct path *fanotify_event_path(struct fanotify_event *event) +{ + if (event->type == FANOTIFY_EVENT_TYPE_PATH) + return &FANOTIFY_PE(event)->path; + else if (event->type == FANOTIFY_EVENT_TYPE_PATH_PERM) + return &FANOTIFY_PERM(event)->path; + else + return NULL; +} + +/* + * Use 128 size hash table to speed up events merge. + */ +#define FANOTIFY_HTABLE_BITS (7) +#define FANOTIFY_HTABLE_SIZE (1 << FANOTIFY_HTABLE_BITS) +#define FANOTIFY_HTABLE_MASK (FANOTIFY_HTABLE_SIZE - 1) + +/* + * Permission events and overflow event do not get merged - don't hash them. + */ +static inline bool fanotify_is_hashed_event(u32 mask) +{ + return !(fanotify_is_perm_event(mask) || + fsnotify_is_overflow_event(mask)); +} + +static inline unsigned int fanotify_event_hash_bucket( + struct fsnotify_group *group, + struct fanotify_event *event) +{ + return event->hash & FANOTIFY_HTABLE_MASK; +} + +struct fanotify_mark { + struct fsnotify_mark fsn_mark; + __kernel_fsid_t fsid; +}; + +static inline struct fanotify_mark *FANOTIFY_MARK(struct fsnotify_mark *mark) +{ + return container_of(mark, struct fanotify_mark, fsn_mark); +} + +static inline bool fanotify_fsid_equal(__kernel_fsid_t *fsid1, + __kernel_fsid_t *fsid2) +{ + return fsid1->val[0] == fsid2->val[0] && fsid1->val[1] == fsid2->val[1]; +} + +static inline unsigned int fanotify_mark_user_flags(struct fsnotify_mark *mark) +{ + unsigned int mflags = 0; + + if (mark->flags & FSNOTIFY_MARK_FLAG_IGNORED_SURV_MODIFY) + mflags |= FAN_MARK_IGNORED_SURV_MODIFY; + if (mark->flags & FSNOTIFY_MARK_FLAG_NO_IREF) + mflags |= FAN_MARK_EVICTABLE; + if (mark->flags & FSNOTIFY_MARK_FLAG_HAS_IGNORE_FLAGS) + mflags |= FAN_MARK_IGNORE; + + return mflags; +} + +static inline u32 fanotify_get_response_errno(int res) +{ + return (res >> FAN_ERRNO_SHIFT) & FAN_ERRNO_MASK; +} diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c new file mode 100644 index 000000000..a32c6634d --- /dev/null +++ b/fs/notify/fanotify/fanotify_user.c @@ -0,0 +1,2204 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "../fsnotify.h" +#include "../fdinfo.h" +#include "fanotify.h" + +#define FANOTIFY_DEFAULT_MAX_EVENTS 16384 +#define FANOTIFY_OLD_DEFAULT_MAX_MARKS 8192 +#define FANOTIFY_DEFAULT_MAX_GROUPS 128 +#define FANOTIFY_DEFAULT_FEE_POOL_SIZE 32 + +/* + * Legacy fanotify marks limits (8192) is per group and we introduced a tunable + * limit of marks per user, similar to inotify. Effectively, the legacy limit + * of fanotify marks per user is * . + * This default limit (1M) also happens to match the increased limit of inotify + * max_user_watches since v5.10. + */ +#define FANOTIFY_DEFAULT_MAX_USER_MARKS \ + (FANOTIFY_OLD_DEFAULT_MAX_MARKS * FANOTIFY_DEFAULT_MAX_GROUPS) + +/* + * Most of the memory cost of adding an inode mark is pinning the marked inode. + * The size of the filesystem inode struct is not uniform across filesystems, + * so double the size of a VFS inode is used as a conservative approximation. + */ +#define INODE_MARK_COST (2 * sizeof(struct inode)) + +/* configurable via /proc/sys/fs/fanotify/ */ +static int fanotify_max_queued_events __read_mostly; +static int perm_group_timeout __read_mostly; + +#ifdef CONFIG_SYSCTL + +#include + +static long ft_zero = 0; +static long ft_int_max = INT_MAX; + +static const struct ctl_table fanotify_table[] = { + { + .procname = "max_user_groups", + .data = &init_user_ns.ucount_max[UCOUNT_FANOTIFY_GROUPS], + .maxlen = sizeof(long), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + .extra1 = &ft_zero, + .extra2 = &ft_int_max, + }, + { + .procname = "max_user_marks", + .data = &init_user_ns.ucount_max[UCOUNT_FANOTIFY_MARKS], + .maxlen = sizeof(long), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + .extra1 = &ft_zero, + .extra2 = &ft_int_max, + }, + { + .procname = "max_queued_events", + .data = &fanotify_max_queued_events, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO + }, + { + .procname = "watchdog_timeout", + .data = &perm_group_timeout, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + }, +}; + +static void __init fanotify_sysctls_init(void) +{ + register_sysctl("fs/fanotify", fanotify_table); +} +#else +#define fanotify_sysctls_init() do { } while (0) +#endif /* CONFIG_SYSCTL */ + +static LIST_HEAD(perm_group_list); +static DEFINE_SPINLOCK(perm_group_lock); +static void perm_group_watchdog(struct work_struct *work); +static DECLARE_DELAYED_WORK(perm_group_work, perm_group_watchdog); + +static void perm_group_watchdog_schedule(void) +{ + int timeout = READ_ONCE(perm_group_timeout); + + if (!timeout) + return; + + schedule_delayed_work(&perm_group_work, secs_to_jiffies(timeout)); +} + +static void perm_group_watchdog(struct work_struct *work) +{ + struct fsnotify_group *group; + struct fanotify_perm_event *event; + struct task_struct *task; + pid_t failed_pid = 0; + + guard(spinlock)(&perm_group_lock); + if (list_empty(&perm_group_list)) + return; + + list_for_each_entry(group, &perm_group_list, + fanotify_data.perm_grp_list) { + /* + * Ok to test without lock, racing with an addition is + * fine, will deal with it next round + */ + if (list_empty(&group->fanotify_data.access_list)) + continue; + + spin_lock(&group->notification_lock); + list_for_each_entry(event, &group->fanotify_data.access_list, + fae.fse.list) { + if (likely(event->watchdog_cnt == 0)) { + event->watchdog_cnt = 1; + } else if (event->watchdog_cnt == 1) { + /* Report on event only once */ + event->watchdog_cnt = 2; + + /* Do not report same pid repeatedly */ + if (event->recv_pid == failed_pid) + continue; + + failed_pid = event->recv_pid; + rcu_read_lock(); + task = find_task_by_pid_ns(event->recv_pid, + &init_pid_ns); + pr_warn_ratelimited( + "PID %u (%s) failed to respond to fanotify queue for more than %d seconds\n", + event->recv_pid, + task ? task->comm : NULL, + perm_group_timeout); + rcu_read_unlock(); + } + } + spin_unlock(&group->notification_lock); + } + perm_group_watchdog_schedule(); +} + +static void fanotify_perm_watchdog_group_remove(struct fsnotify_group *group) +{ + if (!list_empty(&group->fanotify_data.perm_grp_list)) { + /* Perm event watchdog can no longer scan this group. */ + spin_lock(&perm_group_lock); + list_del_init(&group->fanotify_data.perm_grp_list); + spin_unlock(&perm_group_lock); + } +} + +static void fanotify_perm_watchdog_group_add(struct fsnotify_group *group) +{ + if (!perm_group_timeout) + return; + + spin_lock(&perm_group_lock); + if (list_empty(&group->fanotify_data.perm_grp_list)) { + /* Add to perm_group_list for monitoring by watchdog. */ + if (list_empty(&perm_group_list)) + perm_group_watchdog_schedule(); + list_add_tail(&group->fanotify_data.perm_grp_list, &perm_group_list); + } + spin_unlock(&perm_group_lock); +} + +/* + * All flags that may be specified in parameter event_f_flags of fanotify_init. + * + * Internal and external open flags are stored together in field f_flags of + * struct file. Only external open flags shall be allowed in event_f_flags. + * Internal flags like FMODE_EXEC shall be excluded. + */ +#define FANOTIFY_INIT_ALL_EVENT_F_BITS ( \ + O_ACCMODE | O_APPEND | O_NONBLOCK | \ + __O_SYNC | O_DSYNC | O_CLOEXEC | \ + O_LARGEFILE | O_NOATIME ) + +extern const struct fsnotify_ops fanotify_fsnotify_ops; + +struct kmem_cache *fanotify_mark_cache __ro_after_init; +struct kmem_cache *fanotify_fid_event_cachep __ro_after_init; +struct kmem_cache *fanotify_path_event_cachep __ro_after_init; +struct kmem_cache *fanotify_perm_event_cachep __ro_after_init; +struct kmem_cache *fanotify_mnt_event_cachep __ro_after_init; + +#define FANOTIFY_EVENT_ALIGN 4 +#define FANOTIFY_FID_INFO_HDR_LEN \ + (sizeof(struct fanotify_event_info_fid) + sizeof(struct file_handle)) +#define FANOTIFY_PIDFD_INFO_LEN \ + sizeof(struct fanotify_event_info_pidfd) +#define FANOTIFY_ERROR_INFO_LEN \ + (sizeof(struct fanotify_event_info_error)) +#define FANOTIFY_RANGE_INFO_LEN \ + (sizeof(struct fanotify_event_info_range)) +#define FANOTIFY_MNT_INFO_LEN \ + (sizeof(struct fanotify_event_info_mnt)) + +static int fanotify_fid_info_len(int fh_len, int name_len) +{ + int info_len = fh_len; + + if (name_len) + info_len += name_len + 1; + + return roundup(FANOTIFY_FID_INFO_HDR_LEN + info_len, + FANOTIFY_EVENT_ALIGN); +} + +/* FAN_RENAME may have one or two dir+name info records */ +static int fanotify_dir_name_info_len(struct fanotify_event *event) +{ + struct fanotify_info *info = fanotify_event_info(event); + int dir_fh_len = fanotify_event_dir_fh_len(event); + int dir2_fh_len = fanotify_event_dir2_fh_len(event); + int info_len = 0; + + if (dir_fh_len) + info_len += fanotify_fid_info_len(dir_fh_len, + info->name_len); + if (dir2_fh_len) + info_len += fanotify_fid_info_len(dir2_fh_len, + info->name2_len); + + return info_len; +} + +static size_t fanotify_event_len(unsigned int info_mode, + struct fanotify_event *event) +{ + size_t event_len = FAN_EVENT_METADATA_LEN; + int fh_len; + int dot_len = 0; + + if (fanotify_is_error_event(event->mask)) + event_len += FANOTIFY_ERROR_INFO_LEN; + + if (fanotify_event_has_any_dir_fh(event)) { + event_len += fanotify_dir_name_info_len(event); + } else if ((info_mode & FAN_REPORT_NAME) && + (event->mask & FAN_ONDIR)) { + /* + * With group flag FAN_REPORT_NAME, if name was not recorded in + * event on a directory, we will report the name ".". + */ + dot_len = 1; + } + + if (fanotify_event_has_object_fh(event)) { + fh_len = fanotify_event_object_fh_len(event); + event_len += fanotify_fid_info_len(fh_len, dot_len); + } + if (fanotify_is_mnt_event(event->mask)) + event_len += FANOTIFY_MNT_INFO_LEN; + + if (info_mode & FAN_REPORT_PIDFD) + event_len += FANOTIFY_PIDFD_INFO_LEN; + + if (fanotify_event_has_access_range(event)) + event_len += FANOTIFY_RANGE_INFO_LEN; + + return event_len; +} + +/* + * Remove an hashed event from merge hash table. + */ +static void fanotify_unhash_event(struct fsnotify_group *group, + struct fanotify_event *event) +{ + assert_spin_locked(&group->notification_lock); + + pr_debug("%s: group=%p event=%p bucket=%u\n", __func__, + group, event, fanotify_event_hash_bucket(group, event)); + + if (WARN_ON_ONCE(hlist_unhashed(&event->merge_list))) + return; + + hlist_del_init(&event->merge_list); +} + +/* + * Get an fanotify notification event if one exists and is small + * enough to fit in "count". Return an error pointer if the count + * is not large enough. When permission event is dequeued, its state is + * updated accordingly. + */ +static struct fanotify_event *get_one_event(struct fsnotify_group *group, + size_t count) +{ + size_t event_size; + struct fanotify_event *event = NULL; + struct fsnotify_event *fsn_event; + unsigned int info_mode = FAN_GROUP_FLAG(group, FANOTIFY_INFO_MODES); + + pr_debug("%s: group=%p count=%zd\n", __func__, group, count); + + spin_lock(&group->notification_lock); + fsn_event = fsnotify_peek_first_event(group); + if (!fsn_event) + goto out; + + event = FANOTIFY_E(fsn_event); + event_size = fanotify_event_len(info_mode, event); + + if (event_size > count) { + event = ERR_PTR(-EINVAL); + goto out; + } + + /* + * Held the notification_lock the whole time, so this is the + * same event we peeked above. + */ + fsnotify_remove_first_event(group); + if (fanotify_is_perm_event(event->mask)) + FANOTIFY_PERM(event)->state = FAN_EVENT_REPORTED; + if (fanotify_is_hashed_event(event->mask)) + fanotify_unhash_event(group, event); +out: + spin_unlock(&group->notification_lock); + return event; +} + +static int create_fd(struct fsnotify_group *group, const struct path *path, + struct file **file) +{ + int client_fd; + struct file *new_file; + + client_fd = get_unused_fd_flags(group->fanotify_data.f_flags); + if (client_fd < 0) + return client_fd; + + /* + * We provide an fd for the userspace program, so it could access the + * file without generating fanotify events itself. + */ + new_file = dentry_open_nonotify(path, group->fanotify_data.f_flags, + current_cred()); + if (IS_ERR(new_file)) { + put_unused_fd(client_fd); + client_fd = PTR_ERR(new_file); + } else { + *file = new_file; + } + + return client_fd; +} + +static int process_access_response_info(const char __user *info, + size_t info_len, + struct fanotify_response_info_audit_rule *friar) +{ + if (info_len != sizeof(*friar)) + return -EINVAL; + + if (copy_from_user(friar, info, sizeof(*friar))) + return -EFAULT; + + if (friar->hdr.type != FAN_RESPONSE_INFO_AUDIT_RULE) + return -EINVAL; + if (friar->hdr.pad != 0) + return -EINVAL; + if (friar->hdr.len != sizeof(*friar)) + return -EINVAL; + + return info_len; +} + +/* + * Finish processing of permission event by setting it to ANSWERED state and + * drop group->notification_lock. + */ +static void finish_permission_event(struct fsnotify_group *group, + struct fanotify_perm_event *event, u32 response, + struct fanotify_response_info_audit_rule *friar) + __releases(&group->notification_lock) +{ + bool destroy = false; + + assert_spin_locked(&group->notification_lock); + event->response = response & ~FAN_INFO; + if (response & FAN_INFO) + memcpy(&event->audit_rule, friar, sizeof(*friar)); + + if (event->state == FAN_EVENT_CANCELED) + destroy = true; + else + event->state = FAN_EVENT_ANSWERED; + spin_unlock(&group->notification_lock); + if (destroy) + fsnotify_destroy_event(group, &event->fae.fse); +} + +static int process_access_response(struct fsnotify_group *group, + struct fanotify_response *response_struct, + const char __user *info, + size_t info_len) +{ + struct fanotify_perm_event *event; + int fd = response_struct->fd; + u32 response = response_struct->response; + int errno = fanotify_get_response_errno(response); + int ret = info_len; + struct fanotify_response_info_audit_rule friar; + + pr_debug("%s: group=%p fd=%d response=%x errno=%d buf=%p size=%zu\n", + __func__, group, fd, response, errno, info, info_len); + /* + * make sure the response is valid, if invalid we do nothing and either + * userspace can send a valid response or we will clean it up after the + * timeout + */ + if (response & ~FANOTIFY_RESPONSE_VALID_MASK) + return -EINVAL; + + switch (response & FANOTIFY_RESPONSE_ACCESS) { + case FAN_ALLOW: + if (errno) + return -EINVAL; + break; + case FAN_DENY: + /* Custom errno is supported only for pre-content groups */ + if (errno && group->priority != FSNOTIFY_PRIO_PRE_CONTENT) + return -EINVAL; + + /* + * Limit errno to values expected on open(2)/read(2)/write(2) + * of regular files. + */ + switch (errno) { + case 0: + case EIO: + case EPERM: + case EBUSY: + case ETXTBSY: + case EAGAIN: + case ENOSPC: + case EDQUOT: + break; + default: + return -EINVAL; + } + break; + default: + return -EINVAL; + } + + if ((response & FAN_AUDIT) && !FAN_GROUP_FLAG(group, FAN_ENABLE_AUDIT)) + return -EINVAL; + + if (response & FAN_INFO) { + ret = process_access_response_info(info, info_len, &friar); + if (ret < 0) + return ret; + if (fd == FAN_NOFD) + return ret; + } else { + ret = 0; + } + + if (fd < 0) + return -EINVAL; + + spin_lock(&group->notification_lock); + list_for_each_entry(event, &group->fanotify_data.access_list, + fae.fse.list) { + if (event->fd != fd) + continue; + + list_del_init(&event->fae.fse.list); + finish_permission_event(group, event, response, &friar); + wake_up(&group->fanotify_data.access_waitq); + return ret; + } + spin_unlock(&group->notification_lock); + + return -ENOENT; +} + +static size_t copy_mnt_info_to_user(struct fanotify_event *event, + char __user *buf, int count) +{ + struct fanotify_event_info_mnt info = { }; + + info.hdr.info_type = FAN_EVENT_INFO_TYPE_MNT; + info.hdr.len = FANOTIFY_MNT_INFO_LEN; + + if (WARN_ON(count < info.hdr.len)) + return -EFAULT; + + info.mnt_id = FANOTIFY_ME(event)->mnt_id; + + if (copy_to_user(buf, &info, sizeof(info))) + return -EFAULT; + + return info.hdr.len; +} + +static size_t copy_error_info_to_user(struct fanotify_event *event, + char __user *buf, int count) +{ + struct fanotify_event_info_error info = { }; + struct fanotify_error_event *fee = FANOTIFY_EE(event); + + info.hdr.info_type = FAN_EVENT_INFO_TYPE_ERROR; + info.hdr.len = FANOTIFY_ERROR_INFO_LEN; + + if (WARN_ON(count < info.hdr.len)) + return -EFAULT; + + info.error = fee->error; + info.error_count = fee->err_count; + + if (copy_to_user(buf, &info, sizeof(info))) + return -EFAULT; + + return info.hdr.len; +} + +static int copy_fid_info_to_user(__kernel_fsid_t *fsid, struct fanotify_fh *fh, + int info_type, const char *name, + size_t name_len, + char __user *buf, size_t count) +{ + struct fanotify_event_info_fid info = { }; + struct file_handle handle = { }; + unsigned char bounce[FANOTIFY_INLINE_FH_LEN], *fh_buf; + size_t fh_len = fh ? fh->len : 0; + size_t info_len = fanotify_fid_info_len(fh_len, name_len); + size_t len = info_len; + + pr_debug("%s: fh_len=%zu name_len=%zu, info_len=%zu, count=%zu\n", + __func__, fh_len, name_len, info_len, count); + + if (WARN_ON_ONCE(len < sizeof(info) || len > count)) + return -EFAULT; + + /* + * Copy event info fid header followed by variable sized file handle + * and optionally followed by variable sized filename. + */ + switch (info_type) { + case FAN_EVENT_INFO_TYPE_FID: + case FAN_EVENT_INFO_TYPE_DFID: + if (WARN_ON_ONCE(name_len)) + return -EFAULT; + break; + case FAN_EVENT_INFO_TYPE_DFID_NAME: + case FAN_EVENT_INFO_TYPE_OLD_DFID_NAME: + case FAN_EVENT_INFO_TYPE_NEW_DFID_NAME: + if (WARN_ON_ONCE(!name || !name_len)) + return -EFAULT; + break; + default: + return -EFAULT; + } + + info.hdr.info_type = info_type; + info.hdr.len = len; + info.fsid = *fsid; + if (copy_to_user(buf, &info, sizeof(info))) + return -EFAULT; + + buf += sizeof(info); + len -= sizeof(info); + if (WARN_ON_ONCE(len < sizeof(handle))) + return -EFAULT; + + handle.handle_type = fh->type; + handle.handle_bytes = fh_len; + + /* Mangle handle_type for bad file_handle */ + if (!fh_len) + handle.handle_type = FILEID_INVALID; + + if (copy_to_user(buf, &handle, sizeof(handle))) + return -EFAULT; + + buf += sizeof(handle); + len -= sizeof(handle); + if (WARN_ON_ONCE(len < fh_len)) + return -EFAULT; + + /* + * For an inline fh and inline file name, copy through stack to exclude + * the copy from usercopy hardening protections. + */ + fh_buf = fanotify_fh_buf(fh); + if (fh_len <= FANOTIFY_INLINE_FH_LEN) { + memcpy(bounce, fh_buf, fh_len); + fh_buf = bounce; + } + if (copy_to_user(buf, fh_buf, fh_len)) + return -EFAULT; + + buf += fh_len; + len -= fh_len; + + if (name_len) { + /* Copy the filename with terminating null */ + name_len++; + if (WARN_ON_ONCE(len < name_len)) + return -EFAULT; + + if (copy_to_user(buf, name, name_len)) + return -EFAULT; + + buf += name_len; + len -= name_len; + } + + /* Pad with 0's */ + WARN_ON_ONCE(len < 0 || len >= FANOTIFY_EVENT_ALIGN); + if (len > 0 && clear_user(buf, len)) + return -EFAULT; + + return info_len; +} + +static int copy_pidfd_info_to_user(int pidfd, + char __user *buf, + size_t count) +{ + struct fanotify_event_info_pidfd info = { }; + size_t info_len = FANOTIFY_PIDFD_INFO_LEN; + + if (WARN_ON_ONCE(info_len > count)) + return -EFAULT; + + info.hdr.info_type = FAN_EVENT_INFO_TYPE_PIDFD; + info.hdr.len = info_len; + info.pidfd = pidfd; + + if (copy_to_user(buf, &info, info_len)) + return -EFAULT; + + return info_len; +} + +static size_t copy_range_info_to_user(struct fanotify_event *event, + char __user *buf, int count) +{ + struct fanotify_perm_event *pevent = FANOTIFY_PERM(event); + struct fanotify_event_info_range info = { }; + size_t info_len = FANOTIFY_RANGE_INFO_LEN; + + if (WARN_ON_ONCE(info_len > count)) + return -EFAULT; + + info.hdr.info_type = FAN_EVENT_INFO_TYPE_RANGE; + info.hdr.len = info_len; + info.offset = pevent->pos; + info.count = pevent->count; + + if (copy_to_user(buf, &info, info_len)) + return -EFAULT; + + return info_len; +} + +static int copy_info_records_to_user(struct fanotify_event *event, + struct fanotify_info *info, + unsigned int info_mode, int pidfd, + char __user *buf, size_t count) +{ + int ret, total_bytes = 0, info_type = 0; + unsigned int fid_mode = info_mode & FANOTIFY_FID_BITS; + unsigned int pidfd_mode = info_mode & FAN_REPORT_PIDFD; + + /* + * Event info records order is as follows: + * 1. dir fid + name + * 2. (optional) new dir fid + new name + * 3. (optional) child fid + */ + if (fanotify_event_has_dir_fh(event)) { + info_type = info->name_len ? FAN_EVENT_INFO_TYPE_DFID_NAME : + FAN_EVENT_INFO_TYPE_DFID; + + /* FAN_RENAME uses special info types */ + if (event->mask & FAN_RENAME) + info_type = FAN_EVENT_INFO_TYPE_OLD_DFID_NAME; + + ret = copy_fid_info_to_user(fanotify_event_fsid(event), + fanotify_info_dir_fh(info), + info_type, + fanotify_info_name(info), + info->name_len, buf, count); + if (ret < 0) + return ret; + + buf += ret; + count -= ret; + total_bytes += ret; + } + + /* New dir fid+name may be reported in addition to old dir fid+name */ + if (fanotify_event_has_dir2_fh(event)) { + info_type = FAN_EVENT_INFO_TYPE_NEW_DFID_NAME; + ret = copy_fid_info_to_user(fanotify_event_fsid(event), + fanotify_info_dir2_fh(info), + info_type, + fanotify_info_name2(info), + info->name2_len, buf, count); + if (ret < 0) + return ret; + + buf += ret; + count -= ret; + total_bytes += ret; + } + + if (fanotify_event_has_object_fh(event)) { + const char *dot = NULL; + int dot_len = 0; + + if (fid_mode == FAN_REPORT_FID || info_type) { + /* + * With only group flag FAN_REPORT_FID only type FID is + * reported. Second info record type is always FID. + */ + info_type = FAN_EVENT_INFO_TYPE_FID; + } else if ((fid_mode & FAN_REPORT_NAME) && + (event->mask & FAN_ONDIR)) { + /* + * With group flag FAN_REPORT_NAME, if name was not + * recorded in an event on a directory, report the name + * "." with info type DFID_NAME. + */ + info_type = FAN_EVENT_INFO_TYPE_DFID_NAME; + dot = "."; + dot_len = 1; + } else if ((event->mask & ALL_FSNOTIFY_DIRENT_EVENTS) || + (event->mask & FAN_ONDIR)) { + /* + * With group flag FAN_REPORT_DIR_FID, a single info + * record has type DFID for directory entry modification + * event and for event on a directory. + */ + info_type = FAN_EVENT_INFO_TYPE_DFID; + } else { + /* + * With group flags FAN_REPORT_DIR_FID|FAN_REPORT_FID, + * a single info record has type FID for event on a + * non-directory, when there is no directory to report. + * For example, on FAN_DELETE_SELF event. + */ + info_type = FAN_EVENT_INFO_TYPE_FID; + } + + ret = copy_fid_info_to_user(fanotify_event_fsid(event), + fanotify_event_object_fh(event), + info_type, dot, dot_len, + buf, count); + if (ret < 0) + return ret; + + buf += ret; + count -= ret; + total_bytes += ret; + } + + if (pidfd_mode) { + ret = copy_pidfd_info_to_user(pidfd, buf, count); + if (ret < 0) + return ret; + + buf += ret; + count -= ret; + total_bytes += ret; + } + + if (fanotify_is_error_event(event->mask)) { + ret = copy_error_info_to_user(event, buf, count); + if (ret < 0) + return ret; + buf += ret; + count -= ret; + total_bytes += ret; + } + + if (fanotify_event_has_access_range(event)) { + ret = copy_range_info_to_user(event, buf, count); + if (ret < 0) + return ret; + buf += ret; + count -= ret; + total_bytes += ret; + } + + if (fanotify_is_mnt_event(event->mask)) { + ret = copy_mnt_info_to_user(event, buf, count); + if (ret < 0) + return ret; + buf += ret; + count -= ret; + total_bytes += ret; + } + + return total_bytes; +} + +static ssize_t copy_event_to_user(struct fsnotify_group *group, + struct fanotify_event *event, + char __user *buf, size_t count) +{ + struct fanotify_event_metadata metadata; + const struct path *path = fanotify_event_path(event); + struct fanotify_info *info = fanotify_event_info(event); + unsigned int info_mode = FAN_GROUP_FLAG(group, FANOTIFY_INFO_MODES); + unsigned int pidfd_mode = info_mode & FAN_REPORT_PIDFD; + struct file *f = NULL, *pidfd_file = NULL; + int ret, pidfd = -ESRCH, fd = -EBADF; + + pr_debug("%s: group=%p event=%p\n", __func__, group, event); + + metadata.event_len = fanotify_event_len(info_mode, event); + metadata.metadata_len = FAN_EVENT_METADATA_LEN; + metadata.vers = FANOTIFY_METADATA_VERSION; + metadata.reserved = 0; + metadata.mask = event->mask & FANOTIFY_OUTGOING_EVENTS; + metadata.pid = pid_vnr(event->pid); + /* + * For an unprivileged listener, event->pid can be used to identify the + * events generated by the listener process itself, without disclosing + * the pids of other processes. + */ + if (FAN_GROUP_FLAG(group, FANOTIFY_UNPRIV) && + task_tgid(current) != event->pid) + metadata.pid = 0; + + /* + * For now, fid mode is required for an unprivileged listener and + * fid mode does not report fd in events. Keep this check anyway + * for safety in case fid mode requirement is relaxed in the future + * to allow unprivileged listener to get events with no fd and no fid. + */ + if (!FAN_GROUP_FLAG(group, FANOTIFY_UNPRIV) && + path && path->mnt && path->dentry) { + fd = create_fd(group, path, &f); + /* + * Opening an fd from dentry can fail for several reasons. + * For example, when tasks are gone and we try to open their + * /proc files or we try to open a WRONLY file like in sysfs + * or when trying to open a file that was deleted on the + * remote network server. + * + * For a group with FAN_REPORT_FD_ERROR, we will send the + * event with the error instead of the open fd, otherwise + * Userspace may not get the error at all. + * In any case, userspace will not know which file failed to + * open, so add a debug print for further investigation. + */ + if (fd < 0) { + pr_debug("fanotify: create_fd(%pd2) failed err=%d\n", + path->dentry, fd); + if (!FAN_GROUP_FLAG(group, FAN_REPORT_FD_ERROR)) { + /* + * Historically, we've handled EOPENSTALE in a + * special way and silently dropped such + * events. Now we have to keep it to maintain + * backward compatibility... + */ + if (fd == -EOPENSTALE) + fd = 0; + return fd; + } + } + } + if (FAN_GROUP_FLAG(group, FAN_REPORT_FD_ERROR)) + metadata.fd = fd; + else + metadata.fd = fd >= 0 ? fd : FAN_NOFD; + + if (pidfd_mode) { + unsigned int pidfd_flags = PIDFD_STALE; + + if (FAN_GROUP_FLAG(group, FAN_REPORT_TID)) + pidfd_flags |= PIDFD_THREAD; + + if (metadata.pid) + pidfd = pidfd_prepare(event->pid, pidfd_flags, &pidfd_file); + + if (!FAN_GROUP_FLAG(group, FAN_REPORT_FD_ERROR) && pidfd < 0) + pidfd = pidfd == -ESRCH ? FAN_NOPIDFD : FAN_EPIDFD; + } + + ret = -EFAULT; + /* + * Sanity check copy size in case get_one_event() and + * event_len sizes ever get out of sync. + */ + if (WARN_ON_ONCE(metadata.event_len > count)) + goto out_close_fd; + + if (copy_to_user(buf, &metadata, FAN_EVENT_METADATA_LEN)) + goto out_close_fd; + + buf += FAN_EVENT_METADATA_LEN; + count -= FAN_EVENT_METADATA_LEN; + + ret = copy_info_records_to_user(event, info, info_mode, pidfd, + buf, count); + if (ret < 0) + goto out_close_fd; + + if (f) + fd_install(fd, f); + + if (pidfd_file) + fd_install(pidfd, pidfd_file); + + if (fanotify_is_perm_event(event->mask)) + FANOTIFY_PERM(event)->fd = fd; + + return metadata.event_len; + +out_close_fd: + if (f) { + put_unused_fd(fd); + fput(f); + } + + if (pidfd_file) { + put_unused_fd(pidfd); + fput(pidfd_file); + } + + return ret; +} + +/* intofiy userspace file descriptor functions */ +static __poll_t fanotify_poll(struct file *file, poll_table *wait) +{ + struct fsnotify_group *group = file->private_data; + __poll_t ret = 0; + + poll_wait(file, &group->notification_waitq, wait); + spin_lock(&group->notification_lock); + if (!fsnotify_notify_queue_is_empty(group)) + ret = EPOLLIN | EPOLLRDNORM; + spin_unlock(&group->notification_lock); + + return ret; +} + +static ssize_t fanotify_read(struct file *file, char __user *buf, + size_t count, loff_t *pos) +{ + struct fsnotify_group *group; + struct fanotify_event *event; + char __user *start; + int ret; + DEFINE_WAIT_FUNC(wait, woken_wake_function); + + start = buf; + group = file->private_data; + + pr_debug("%s: group=%p\n", __func__, group); + + add_wait_queue(&group->notification_waitq, &wait); + while (1) { + /* + * User can supply arbitrarily large buffer. Avoid softlockups + * in case there are lots of available events. + */ + cond_resched(); + event = get_one_event(group, count); + if (IS_ERR(event)) { + ret = PTR_ERR(event); + break; + } + + if (!event) { + ret = -EAGAIN; + if (file->f_flags & O_NONBLOCK) + break; + + ret = -ERESTARTSYS; + if (signal_pending(current)) + break; + + if (start != buf) + break; + + wait_woken(&wait, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); + continue; + } + + ret = copy_event_to_user(group, event, buf, count); + + /* + * Permission events get queued to wait for response. Other + * events can be destroyed now. + */ + if (!fanotify_is_perm_event(event->mask)) { + fsnotify_destroy_event(group, &event->fse); + } else { + if (ret <= 0 || FANOTIFY_PERM(event)->fd < 0) { + spin_lock(&group->notification_lock); + finish_permission_event(group, + FANOTIFY_PERM(event), FAN_DENY, NULL); + wake_up(&group->fanotify_data.access_waitq); + } else { + spin_lock(&group->notification_lock); + list_add_tail(&event->fse.list, + &group->fanotify_data.access_list); + FANOTIFY_PERM(event)->recv_pid = current->pid; + spin_unlock(&group->notification_lock); + } + } + if (ret < 0) + break; + buf += ret; + count -= ret; + } + remove_wait_queue(&group->notification_waitq, &wait); + + if (start != buf && ret != -EFAULT) + ret = buf - start; + return ret; +} + +static ssize_t fanotify_write(struct file *file, const char __user *buf, size_t count, loff_t *pos) +{ + struct fanotify_response response; + struct fsnotify_group *group; + int ret; + const char __user *info_buf = buf + sizeof(struct fanotify_response); + size_t info_len; + + if (!IS_ENABLED(CONFIG_FANOTIFY_ACCESS_PERMISSIONS)) + return -EINVAL; + + group = file->private_data; + + pr_debug("%s: group=%p count=%zu\n", __func__, group, count); + + if (count < sizeof(response)) + return -EINVAL; + + if (copy_from_user(&response, buf, sizeof(response))) + return -EFAULT; + + info_len = count - sizeof(response); + + ret = process_access_response(group, &response, info_buf, info_len); + if (ret < 0) + count = ret; + else + count = sizeof(response) + ret; + + return count; +} + +static int fanotify_release(struct inode *ignored, struct file *file) +{ + struct fsnotify_group *group = file->private_data; + struct fsnotify_event *fsn_event; + + /* + * Stop new events from arriving in the notification queue. since + * userspace cannot use fanotify fd anymore, no event can enter or + * leave access_list by now either. + */ + fsnotify_group_stop_queueing(group); + + fanotify_perm_watchdog_group_remove(group); + + /* + * Process all permission events on access_list and notification queue + * and simulate reply from userspace. + */ + spin_lock(&group->notification_lock); + while (!list_empty(&group->fanotify_data.access_list)) { + struct fanotify_perm_event *event; + + event = list_first_entry(&group->fanotify_data.access_list, + struct fanotify_perm_event, fae.fse.list); + list_del_init(&event->fae.fse.list); + finish_permission_event(group, event, FAN_ALLOW, NULL); + spin_lock(&group->notification_lock); + } + + /* + * Destroy all non-permission events. For permission events just + * dequeue them and set the response. They will be freed once the + * response is consumed and fanotify_get_response() returns. + */ + while ((fsn_event = fsnotify_remove_first_event(group))) { + struct fanotify_event *event = FANOTIFY_E(fsn_event); + + if (!(event->mask & FANOTIFY_PERM_EVENTS)) { + spin_unlock(&group->notification_lock); + fsnotify_destroy_event(group, fsn_event); + } else { + finish_permission_event(group, FANOTIFY_PERM(event), + FAN_ALLOW, NULL); + } + spin_lock(&group->notification_lock); + } + spin_unlock(&group->notification_lock); + + /* Response for all permission events it set, wakeup waiters */ + wake_up(&group->fanotify_data.access_waitq); + + /* matches the fanotify_init->fsnotify_alloc_group */ + fsnotify_destroy_group(group); + + return 0; +} + +static long fanotify_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + struct fsnotify_group *group; + struct fsnotify_event *fsn_event; + unsigned int info_mode; + void __user *p; + int ret = -ENOTTY; + size_t send_len = 0; + + group = file->private_data; + info_mode = FAN_GROUP_FLAG(group, FANOTIFY_INFO_MODES); + + p = (void __user *) arg; + + switch (cmd) { + case FIONREAD: + spin_lock(&group->notification_lock); + list_for_each_entry(fsn_event, &group->notification_list, list) + send_len += fanotify_event_len(info_mode, + FANOTIFY_E(fsn_event)); + spin_unlock(&group->notification_lock); + ret = put_user(send_len, (int __user *) p); + break; + } + + return ret; +} + +static const struct file_operations fanotify_fops = { + .show_fdinfo = fanotify_show_fdinfo, + .poll = fanotify_poll, + .read = fanotify_read, + .write = fanotify_write, + .fasync = NULL, + .release = fanotify_release, + .unlocked_ioctl = fanotify_ioctl, + .compat_ioctl = compat_ptr_ioctl, + .llseek = noop_llseek, +}; + +static int fanotify_find_path(int dfd, const char __user *filename, + struct path *path, unsigned int flags, __u64 mask, + unsigned int obj_type) +{ + int ret; + + pr_debug("%s: dfd=%d filename=%p flags=%x\n", __func__, + dfd, filename, flags); + + if (filename == NULL) { + CLASS(fd, f)(dfd); + + if (fd_empty(f)) + return -EBADF; + + if ((flags & FAN_MARK_ONLYDIR) && + !(S_ISDIR(file_inode(fd_file(f))->i_mode))) + return -ENOTDIR; + + *path = fd_file(f)->f_path; + path_get(path); + ret = 0; + } else { + unsigned int lookup_flags = 0; + + if (!(flags & FAN_MARK_DONT_FOLLOW)) + lookup_flags |= LOOKUP_FOLLOW; + if (flags & FAN_MARK_ONLYDIR) + lookup_flags |= LOOKUP_DIRECTORY; + + ret = user_path_at(dfd, filename, lookup_flags, path); + } + return ret; +} + +static __u32 fanotify_mark_remove_from_mask(struct fsnotify_mark *fsn_mark, + __u32 mask, unsigned int flags, + __u32 umask, int *destroy) +{ + __u32 oldmask, newmask; + + /* umask bits cannot be removed by user */ + mask &= ~umask; + spin_lock(&fsn_mark->lock); + oldmask = fsnotify_calc_mask(fsn_mark); + if (!(flags & FANOTIFY_MARK_IGNORE_BITS)) { + fsn_mark->mask &= ~mask; + } else { + fsn_mark->ignore_mask &= ~mask; + } + newmask = fsnotify_calc_mask(fsn_mark); + /* + * We need to keep the mark around even if remaining mask cannot + * result in any events (e.g. mask == FAN_ONDIR) to support incremenal + * changes to the mask. + * Destroy mark when only umask bits remain. + */ + *destroy = !((fsn_mark->mask | fsn_mark->ignore_mask) & ~umask); + spin_unlock(&fsn_mark->lock); + + return oldmask & ~newmask; +} + +static int fanotify_remove_mark(struct fsnotify_group *group, + void *obj, unsigned int obj_type, __u32 mask, + unsigned int flags, __u32 umask) +{ + struct fsnotify_mark *fsn_mark = NULL; + __u32 removed; + int destroy_mark; + + fsnotify_group_lock(group); + fsn_mark = fsnotify_find_mark(obj, obj_type, group); + if (!fsn_mark) { + fsnotify_group_unlock(group); + return -ENOENT; + } + + removed = fanotify_mark_remove_from_mask(fsn_mark, mask, flags, + umask, &destroy_mark); + if (removed & fsnotify_conn_mask(fsn_mark->connector)) + fsnotify_recalc_mask(fsn_mark->connector); + if (destroy_mark) + fsnotify_detach_mark(fsn_mark); + fsnotify_group_unlock(group); + if (destroy_mark) + fsnotify_free_mark(fsn_mark); + + /* matches the fsnotify_find_mark() */ + fsnotify_put_mark(fsn_mark); + return 0; +} + +static bool fanotify_mark_update_flags(struct fsnotify_mark *fsn_mark, + unsigned int fan_flags) +{ + bool want_iref = !(fan_flags & FAN_MARK_EVICTABLE); + unsigned int ignore = fan_flags & FANOTIFY_MARK_IGNORE_BITS; + bool recalc = false; + + /* + * When using FAN_MARK_IGNORE for the first time, mark starts using + * independent event flags in ignore mask. After that, trying to + * update the ignore mask with the old FAN_MARK_IGNORED_MASK API + * will result in EEXIST error. + */ + if (ignore == FAN_MARK_IGNORE) + fsn_mark->flags |= FSNOTIFY_MARK_FLAG_HAS_IGNORE_FLAGS; + + /* + * Setting FAN_MARK_IGNORED_SURV_MODIFY for the first time may lead to + * the removal of the FS_MODIFY bit in calculated mask if it was set + * because of an ignore mask that is now going to survive FS_MODIFY. + */ + if (ignore && (fan_flags & FAN_MARK_IGNORED_SURV_MODIFY) && + !(fsn_mark->flags & FSNOTIFY_MARK_FLAG_IGNORED_SURV_MODIFY)) { + fsn_mark->flags |= FSNOTIFY_MARK_FLAG_IGNORED_SURV_MODIFY; + if (!(fsn_mark->mask & FS_MODIFY)) + recalc = true; + } + + if (fsn_mark->connector->type != FSNOTIFY_OBJ_TYPE_INODE || + want_iref == !(fsn_mark->flags & FSNOTIFY_MARK_FLAG_NO_IREF)) + return recalc; + + /* + * NO_IREF may be removed from a mark, but not added. + * When removed, fsnotify_recalc_mask() will take the inode ref. + */ + WARN_ON_ONCE(!want_iref); + fsn_mark->flags &= ~FSNOTIFY_MARK_FLAG_NO_IREF; + + return true; +} + +static bool fanotify_mark_add_to_mask(struct fsnotify_mark *fsn_mark, + __u32 mask, unsigned int fan_flags) +{ + __u32 old_mask; + bool recalc; + + spin_lock(&fsn_mark->lock); + if (!(fan_flags & FANOTIFY_MARK_IGNORE_BITS)) { + old_mask = fsn_mark->mask; + fsn_mark->mask |= mask; + recalc = old_mask != fsn_mark->mask; + } else { + fsn_mark->ignore_mask |= mask; + recalc = true; + } + + recalc |= fanotify_mark_update_flags(fsn_mark, fan_flags); + spin_unlock(&fsn_mark->lock); + + return recalc; +} + +struct fan_fsid { + struct super_block *sb; + __kernel_fsid_t id; + bool weak; +}; + +static int fanotify_set_mark_fsid(struct fsnotify_group *group, + struct fsnotify_mark *mark, + struct fan_fsid *fsid) +{ + struct fsnotify_mark_connector *conn; + struct fsnotify_mark *old; + struct super_block *old_sb = NULL; + + FANOTIFY_MARK(mark)->fsid = fsid->id; + mark->flags |= FSNOTIFY_MARK_FLAG_HAS_FSID; + if (fsid->weak) + mark->flags |= FSNOTIFY_MARK_FLAG_WEAK_FSID; + + /* First mark added will determine if group is single or multi fsid */ + if (list_empty(&group->marks_list)) + return 0; + + /* Find sb of an existing mark */ + list_for_each_entry(old, &group->marks_list, g_list) { + conn = READ_ONCE(old->connector); + if (!conn) + continue; + old_sb = fsnotify_connector_sb(conn); + if (old_sb) + break; + } + + /* Only detached marks left? */ + if (!old_sb) + return 0; + + /* Do not allow mixing of marks with weak and strong fsid */ + if ((mark->flags ^ old->flags) & FSNOTIFY_MARK_FLAG_WEAK_FSID) + return -EXDEV; + + /* Allow mixing of marks with strong fsid from different fs */ + if (!fsid->weak) + return 0; + + /* Do not allow mixing marks with weak fsid from different fs */ + if (old_sb != fsid->sb) + return -EXDEV; + + /* Do not allow mixing marks from different btrfs sub-volumes */ + if (!fanotify_fsid_equal(&FANOTIFY_MARK(old)->fsid, + &FANOTIFY_MARK(mark)->fsid)) + return -EXDEV; + + return 0; +} + +static struct fsnotify_mark *fanotify_add_new_mark(struct fsnotify_group *group, + void *obj, + unsigned int obj_type, + unsigned int fan_flags, + struct fan_fsid *fsid) +{ + struct ucounts *ucounts = group->fanotify_data.ucounts; + struct fanotify_mark *fan_mark; + struct fsnotify_mark *mark; + int ret; + + /* + * Enforce per user marks limits per user in all containing user ns. + * A group with FAN_UNLIMITED_MARKS does not contribute to mark count + * in the limited groups account. + */ + BUILD_BUG_ON(!(FANOTIFY_ADMIN_INIT_FLAGS & FAN_UNLIMITED_MARKS)); + if (!FAN_GROUP_FLAG(group, FAN_UNLIMITED_MARKS) && + !inc_ucount(ucounts->ns, ucounts->uid, UCOUNT_FANOTIFY_MARKS)) + return ERR_PTR(-ENOSPC); + + fan_mark = kmem_cache_alloc(fanotify_mark_cache, GFP_KERNEL); + if (!fan_mark) { + ret = -ENOMEM; + goto out_dec_ucounts; + } + + mark = &fan_mark->fsn_mark; + fsnotify_init_mark(mark, group); + if (fan_flags & FAN_MARK_EVICTABLE) + mark->flags |= FSNOTIFY_MARK_FLAG_NO_IREF; + + /* Cache fsid of filesystem containing the marked object */ + if (fsid) { + ret = fanotify_set_mark_fsid(group, mark, fsid); + if (ret) + goto out_put_mark; + } else { + fan_mark->fsid.val[0] = fan_mark->fsid.val[1] = 0; + } + + ret = fsnotify_add_mark_locked(mark, obj, obj_type, 0); + if (ret) + goto out_put_mark; + + return mark; + +out_put_mark: + fsnotify_put_mark(mark); +out_dec_ucounts: + if (!FAN_GROUP_FLAG(group, FAN_UNLIMITED_MARKS)) + dec_ucount(ucounts, UCOUNT_FANOTIFY_MARKS); + return ERR_PTR(ret); +} + +static int fanotify_group_init_error_pool(struct fsnotify_group *group) +{ + if (mempool_initialized(&group->fanotify_data.error_events_pool)) + return 0; + + return mempool_init_kmalloc_pool(&group->fanotify_data.error_events_pool, + FANOTIFY_DEFAULT_FEE_POOL_SIZE, + sizeof(struct fanotify_error_event)); +} + +static int fanotify_may_update_existing_mark(struct fsnotify_mark *fsn_mark, + __u32 mask, unsigned int fan_flags) +{ + /* + * Non evictable mark cannot be downgraded to evictable mark. + */ + if (fan_flags & FAN_MARK_EVICTABLE && + !(fsn_mark->flags & FSNOTIFY_MARK_FLAG_NO_IREF)) + return -EEXIST; + + /* + * New ignore mask semantics cannot be downgraded to old semantics. + */ + if (fan_flags & FAN_MARK_IGNORED_MASK && + fsn_mark->flags & FSNOTIFY_MARK_FLAG_HAS_IGNORE_FLAGS) + return -EEXIST; + + /* + * An ignore mask that survives modify could never be downgraded to not + * survive modify. With new FAN_MARK_IGNORE semantics we make that rule + * explicit and return an error when trying to update the ignore mask + * without the original FAN_MARK_IGNORED_SURV_MODIFY value. + */ + if (fan_flags & FAN_MARK_IGNORE && + !(fan_flags & FAN_MARK_IGNORED_SURV_MODIFY) && + fsn_mark->flags & FSNOTIFY_MARK_FLAG_IGNORED_SURV_MODIFY) + return -EEXIST; + + /* For now pre-content events are not generated for directories */ + mask |= fsn_mark->mask; + if (mask & FANOTIFY_PRE_CONTENT_EVENTS && mask & FAN_ONDIR) + return -EEXIST; + + return 0; +} + +static int fanotify_add_mark(struct fsnotify_group *group, + void *obj, unsigned int obj_type, + __u32 mask, unsigned int fan_flags, + struct fan_fsid *fsid) +{ + struct fsnotify_mark *fsn_mark; + bool recalc; + int ret = 0; + + fsnotify_group_lock(group); + fsn_mark = fsnotify_find_mark(obj, obj_type, group); + if (!fsn_mark) { + fsn_mark = fanotify_add_new_mark(group, obj, obj_type, + fan_flags, fsid); + if (IS_ERR(fsn_mark)) { + fsnotify_group_unlock(group); + return PTR_ERR(fsn_mark); + } + } + + /* + * Check if requested mark flags conflict with an existing mark flags. + */ + ret = fanotify_may_update_existing_mark(fsn_mark, mask, fan_flags); + if (ret) + goto out; + + /* + * Error events are pre-allocated per group, only if strictly + * needed (i.e. FAN_FS_ERROR was requested). + */ + if (!(fan_flags & FANOTIFY_MARK_IGNORE_BITS) && + (mask & FAN_FS_ERROR)) { + ret = fanotify_group_init_error_pool(group); + if (ret) + goto out; + } + + recalc = fanotify_mark_add_to_mask(fsn_mark, mask, fan_flags); + if (recalc) + fsnotify_recalc_mask(fsn_mark->connector); + +out: + fsnotify_group_unlock(group); + + fsnotify_put_mark(fsn_mark); + + if (!ret && (mask & FANOTIFY_PERM_EVENTS)) + fanotify_perm_watchdog_group_add(group); + + return ret; +} + +static struct fsnotify_event *fanotify_alloc_overflow_event(void) +{ + struct fanotify_event *oevent; + + oevent = kmalloc_obj(*oevent, GFP_KERNEL_ACCOUNT); + if (!oevent) + return NULL; + + fanotify_init_event(oevent, 0, FS_Q_OVERFLOW); + oevent->type = FANOTIFY_EVENT_TYPE_OVERFLOW; + + return &oevent->fse; +} + +static struct hlist_head *fanotify_alloc_merge_hash(void) +{ + struct hlist_head *hash; + + hash = kmalloc(sizeof(struct hlist_head) << FANOTIFY_HTABLE_BITS, + GFP_KERNEL_ACCOUNT); + if (!hash) + return NULL; + + __hash_init(hash, FANOTIFY_HTABLE_SIZE); + + return hash; +} + +DEFINE_CLASS(fsnotify_group, + struct fsnotify_group *, + if (!IS_ERR_OR_NULL(_T)) fsnotify_destroy_group(_T), + fsnotify_alloc_group(ops, flags), + const struct fsnotify_ops *ops, int flags) + +/* fanotify syscalls */ +SYSCALL_DEFINE2(fanotify_init, unsigned int, flags, unsigned int, event_f_flags) +{ + struct user_namespace *user_ns = current_user_ns(); + int f_flags, fd; + unsigned int fid_mode = flags & FANOTIFY_FID_BITS; + unsigned int class = flags & FANOTIFY_CLASS_BITS; + unsigned int internal_flags = 0; + + pr_debug("%s: flags=%x event_f_flags=%x\n", + __func__, flags, event_f_flags); + + /* + * An unprivileged user can setup an fanotify group with limited + * functionality - an unprivileged group is limited to notification + * events with file handles or mount ids and it cannot use unlimited + * queue/marks. + */ + if (((flags & FANOTIFY_ADMIN_INIT_FLAGS) || + !(flags & (FANOTIFY_FID_BITS | FAN_REPORT_MNT))) && + !capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (!ns_capable_noaudit(&init_user_ns, CAP_SYS_ADMIN)) { + /* + * Setting the internal flag FANOTIFY_UNPRIV on the group + * prevents setting mount/filesystem marks on this group and + * prevents reporting pid and open fd in events. + */ + internal_flags |= FANOTIFY_UNPRIV; + } + +#ifdef CONFIG_AUDITSYSCALL + if (flags & ~(FANOTIFY_INIT_FLAGS | FAN_ENABLE_AUDIT)) +#else + if (flags & ~FANOTIFY_INIT_FLAGS) +#endif + return -EINVAL; + + /* Don't allow mixing mnt events with inode events for now */ + if (flags & FAN_REPORT_MNT) { + if (class != FAN_CLASS_NOTIF) + return -EINVAL; + if (flags & (FANOTIFY_FID_BITS | FAN_REPORT_FD_ERROR)) + return -EINVAL; + } + + if (event_f_flags & ~FANOTIFY_INIT_ALL_EVENT_F_BITS) + return -EINVAL; + + switch (event_f_flags & O_ACCMODE) { + case O_RDONLY: + case O_RDWR: + case O_WRONLY: + break; + default: + return -EINVAL; + } + + if (fid_mode && class != FAN_CLASS_NOTIF) + return -EINVAL; + + /* + * Child name is reported with parent fid so requires dir fid. + * We can report both child fid and dir fid with or without name. + */ + if ((fid_mode & FAN_REPORT_NAME) && !(fid_mode & FAN_REPORT_DIR_FID)) + return -EINVAL; + + /* + * FAN_REPORT_TARGET_FID requires FAN_REPORT_NAME and FAN_REPORT_FID + * and is used as an indication to report both dir and child fid on all + * dirent events. + */ + if ((fid_mode & FAN_REPORT_TARGET_FID) && + (!(fid_mode & FAN_REPORT_NAME) || !(fid_mode & FAN_REPORT_FID))) + return -EINVAL; + + f_flags = O_RDWR; + if (flags & FAN_CLOEXEC) + f_flags |= O_CLOEXEC; + if (flags & FAN_NONBLOCK) + f_flags |= O_NONBLOCK; + + CLASS(fsnotify_group, group)(&fanotify_fsnotify_ops, + FSNOTIFY_GROUP_USER); + /* fsnotify_alloc_group takes a ref. Dropped in fanotify_release */ + if (IS_ERR(group)) + return PTR_ERR(group); + + /* Enforce groups limits per user in all containing user ns */ + group->fanotify_data.ucounts = inc_ucount(user_ns, current_euid(), + UCOUNT_FANOTIFY_GROUPS); + if (!group->fanotify_data.ucounts) + return -EMFILE; + + group->fanotify_data.flags = flags | internal_flags; + group->memcg = get_mem_cgroup_from_mm(current->mm); + group->user_ns = get_user_ns(user_ns); + + group->fanotify_data.merge_hash = fanotify_alloc_merge_hash(); + if (!group->fanotify_data.merge_hash) + return -ENOMEM; + + group->overflow_event = fanotify_alloc_overflow_event(); + if (unlikely(!group->overflow_event)) + return -ENOMEM; + + if (force_o_largefile()) + event_f_flags |= O_LARGEFILE; + group->fanotify_data.f_flags = event_f_flags; + init_waitqueue_head(&group->fanotify_data.access_waitq); + INIT_LIST_HEAD(&group->fanotify_data.access_list); + INIT_LIST_HEAD(&group->fanotify_data.perm_grp_list); + switch (class) { + case FAN_CLASS_NOTIF: + group->priority = FSNOTIFY_PRIO_NORMAL; + break; + case FAN_CLASS_CONTENT: + group->priority = FSNOTIFY_PRIO_CONTENT; + break; + case FAN_CLASS_PRE_CONTENT: + group->priority = FSNOTIFY_PRIO_PRE_CONTENT; + break; + default: + return -EINVAL; + } + + BUILD_BUG_ON(!(FANOTIFY_ADMIN_INIT_FLAGS & FAN_UNLIMITED_QUEUE)); + if (flags & FAN_UNLIMITED_QUEUE) { + group->max_events = UINT_MAX; + } else { + group->max_events = fanotify_max_queued_events; + } + + if (flags & FAN_ENABLE_AUDIT) { + if (!capable(CAP_AUDIT_WRITE)) + return -EPERM; + } + + fd = FD_ADD(f_flags, + anon_inode_getfile_fmode("[fanotify]", &fanotify_fops, + group, f_flags, FMODE_NONOTIFY)); + if (fd >= 0) + retain_and_null_ptr(group); + return fd; +} + +static int fanotify_test_fsid(struct dentry *dentry, unsigned int flags, + struct fan_fsid *fsid) +{ + unsigned int mark_type = flags & FANOTIFY_MARK_TYPE_BITS; + __kernel_fsid_t root_fsid; + int err; + + /* + * Make sure dentry is not of a filesystem with zero fsid (e.g. fuse). + */ + err = vfs_get_fsid(dentry, &fsid->id); + if (err) + return err; + + fsid->sb = dentry->d_sb; + if (!fsid->id.val[0] && !fsid->id.val[1]) { + err = -ENODEV; + goto weak; + } + + /* + * Make sure dentry is not of a filesystem subvolume (e.g. btrfs) + * which uses a different fsid than sb root. + */ + err = vfs_get_fsid(dentry->d_sb->s_root, &root_fsid); + if (err) + return err; + + if (!fanotify_fsid_equal(&root_fsid, &fsid->id)) { + err = -EXDEV; + goto weak; + } + + fsid->weak = false; + return 0; + +weak: + /* Allow weak fsid when marking inodes */ + fsid->weak = true; + return (mark_type == FAN_MARK_INODE) ? 0 : err; +} + +/* Check if filesystem can encode a unique fid */ +static int fanotify_test_fid(struct dentry *dentry, unsigned int flags) +{ + unsigned int mark_type = flags & FANOTIFY_MARK_TYPE_BITS; + const struct export_operations *nop = dentry->d_sb->s_export_op; + + /* + * We need to make sure that the filesystem supports encoding of + * file handles so user can use name_to_handle_at() to compare fids + * reported with events to the file handle of watched objects. + */ + if (!exportfs_can_encode_fid(nop)) + return -EOPNOTSUPP; + + /* + * For sb/mount mark, we also need to make sure that the filesystem + * supports decoding file handles, so user has a way to map back the + * reported fids to filesystem objects. + */ + if (mark_type != FAN_MARK_INODE && !exportfs_can_decode_fh(nop)) + return -EOPNOTSUPP; + + return 0; +} + +static int fanotify_events_supported(struct fsnotify_group *group, + const struct path *path, __u64 mask, + unsigned int flags) +{ + unsigned int mark_type = flags & FANOTIFY_MARK_TYPE_BITS; + bool is_dir = d_is_dir(path->dentry); + /* Strict validation of events in non-dir inode mask with v5.17+ APIs */ + bool strict_dir_events = FAN_GROUP_FLAG(group, FAN_REPORT_TARGET_FID) || + (mask & FAN_RENAME) || + (flags & FAN_MARK_IGNORE); + + /* + * Filesystems need to opt-into pre-content evnets (a.k.a HSM) + * and they are only supported on regular files and directories. + */ + if (mask & FANOTIFY_PRE_CONTENT_EVENTS) { + if (!(path->mnt->mnt_sb->s_iflags & SB_I_ALLOW_HSM)) + return -EOPNOTSUPP; + if (!is_dir && !d_is_reg(path->dentry)) + return -EINVAL; + } + + /* + * Some filesystems such as 'proc' acquire unusual locks when opening + * files. For them fanotify permission events have high chances of + * deadlocking the system - open done when reporting fanotify event + * blocks on this "unusual" lock while another process holding the lock + * waits for fanotify permission event to be answered. Just disallow + * permission events for such filesystems. + */ + if (mask & FANOTIFY_PERM_EVENTS && + path->mnt->mnt_sb->s_type->fs_flags & FS_DISALLOW_NOTIFY_PERM) + return -EINVAL; + + /* + * mount and sb marks are not allowed on kernel internal pseudo fs, + * like pipe_mnt, because that would subscribe to events on all the + * anonynous pipes in the system. + * + * SB_NOUSER covers all of the internal pseudo fs whose objects are not + * exposed to user's mount namespace, but there are other SB_KERNMOUNT + * fs, like nsfs, debugfs, for which the value of allowing sb and mount + * mark is questionable. For now we leave them alone. + */ + if (mark_type != FAN_MARK_INODE && + path->mnt->mnt_sb->s_flags & SB_NOUSER) + return -EINVAL; + + /* + * We shouldn't have allowed setting dirent events and the directory + * flags FAN_ONDIR and FAN_EVENT_ON_CHILD in mask of non-dir inode, + * but because we always allowed it, error only when using new APIs. + */ + if (strict_dir_events && mark_type == FAN_MARK_INODE && + !is_dir && (mask & FANOTIFY_DIRONLY_EVENT_BITS)) + return -ENOTDIR; + + return 0; +} + +static int do_fanotify_mark(int fanotify_fd, unsigned int flags, __u64 mask, + int dfd, const char __user *pathname) +{ + struct inode *inode = NULL; + struct fsnotify_group *group; + struct path path; + struct fan_fsid __fsid, *fsid = NULL; + struct user_namespace *user_ns = NULL; + struct mnt_namespace *mntns; + u32 valid_mask = FANOTIFY_EVENTS | FANOTIFY_EVENT_FLAGS; + unsigned int mark_type = flags & FANOTIFY_MARK_TYPE_BITS; + unsigned int mark_cmd = flags & FANOTIFY_MARK_CMD_BITS; + unsigned int ignore = flags & FANOTIFY_MARK_IGNORE_BITS; + unsigned int obj_type, fid_mode; + void *obj = NULL; + u32 umask = 0; + int ret; + + pr_debug("%s: fanotify_fd=%d flags=%x dfd=%d pathname=%p mask=%llx\n", + __func__, fanotify_fd, flags, dfd, pathname, mask); + + /* we only use the lower 32 bits as of right now. */ + if (upper_32_bits(mask)) + return -EINVAL; + + if (flags & ~FANOTIFY_MARK_FLAGS) + return -EINVAL; + + switch (mark_type) { + case FAN_MARK_INODE: + obj_type = FSNOTIFY_OBJ_TYPE_INODE; + break; + case FAN_MARK_MOUNT: + obj_type = FSNOTIFY_OBJ_TYPE_VFSMOUNT; + break; + case FAN_MARK_FILESYSTEM: + obj_type = FSNOTIFY_OBJ_TYPE_SB; + break; + case FAN_MARK_MNTNS: + obj_type = FSNOTIFY_OBJ_TYPE_MNTNS; + break; + default: + return -EINVAL; + } + + switch (mark_cmd) { + case FAN_MARK_ADD: + case FAN_MARK_REMOVE: + if (!mask) + return -EINVAL; + break; + case FAN_MARK_FLUSH: + if (flags & ~(FANOTIFY_MARK_TYPE_BITS | FAN_MARK_FLUSH)) + return -EINVAL; + break; + default: + return -EINVAL; + } + + if (IS_ENABLED(CONFIG_FANOTIFY_ACCESS_PERMISSIONS)) + valid_mask |= FANOTIFY_PERM_EVENTS; + + if (mask & ~valid_mask) + return -EINVAL; + + + /* We don't allow FAN_MARK_IGNORE & FAN_MARK_IGNORED_MASK together */ + if (ignore == (FAN_MARK_IGNORE | FAN_MARK_IGNORED_MASK)) + return -EINVAL; + + /* + * Event flags (FAN_ONDIR, FAN_EVENT_ON_CHILD) have no effect with + * FAN_MARK_IGNORED_MASK. + */ + if (ignore == FAN_MARK_IGNORED_MASK) { + mask &= ~FANOTIFY_EVENT_FLAGS; + umask = FANOTIFY_EVENT_FLAGS; + } + + CLASS(fd, f)(fanotify_fd); + if (fd_empty(f)) + return -EBADF; + + /* verify that this is indeed an fanotify instance */ + if (unlikely(fd_file(f)->f_op != &fanotify_fops)) + return -EINVAL; + group = fd_file(f)->private_data; + + /* Only report mount events on mnt namespace */ + if (FAN_GROUP_FLAG(group, FAN_REPORT_MNT)) { + if (mask & ~FANOTIFY_MOUNT_EVENTS) + return -EINVAL; + if (mark_type != FAN_MARK_MNTNS) + return -EINVAL; + } else { + if (mask & FANOTIFY_MOUNT_EVENTS) + return -EINVAL; + if (mark_type == FAN_MARK_MNTNS) + return -EINVAL; + } + + /* + * A user is allowed to setup sb/mount/mntns marks only if it is + * capable in the user ns where the group was created. + */ + if (mark_type != FAN_MARK_INODE && + !ns_capable(group->user_ns, CAP_SYS_ADMIN)) + return -EPERM; + + /* + * Permission events are not allowed for FAN_CLASS_NOTIF. + * Pre-content permission events are not allowed for FAN_CLASS_CONTENT. + */ + if (mask & FANOTIFY_PERM_EVENTS && + group->priority == FSNOTIFY_PRIO_NORMAL) + return -EINVAL; + else if (mask & FANOTIFY_PRE_CONTENT_EVENTS && + group->priority == FSNOTIFY_PRIO_CONTENT) + return -EINVAL; + + if (mask & FAN_FS_ERROR && + mark_type != FAN_MARK_FILESYSTEM) + return -EINVAL; + + /* + * Evictable is only relevant for inode marks, because only inode object + * can be evicted on memory pressure. + */ + if (flags & FAN_MARK_EVICTABLE && + mark_type != FAN_MARK_INODE) + return -EINVAL; + + /* + * Events that do not carry enough information to report + * event->fd require a group that supports reporting fid. Those + * events are not supported on a mount mark, because they do not + * carry enough information (i.e. path) to be filtered by mount + * point. + */ + fid_mode = FAN_GROUP_FLAG(group, FANOTIFY_FID_BITS); + if (mask & ~(FANOTIFY_FD_EVENTS|FANOTIFY_MOUNT_EVENTS|FANOTIFY_EVENT_FLAGS) && + (!fid_mode || mark_type == FAN_MARK_MOUNT)) + return -EINVAL; + + /* + * FAN_RENAME uses special info type records to report the old and + * new parent+name. Reporting only old and new parent id is less + * useful and was not implemented. + */ + if (mask & FAN_RENAME && !(fid_mode & FAN_REPORT_NAME)) + return -EINVAL; + + /* Pre-content events are not currently generated for directories. */ + if (mask & FANOTIFY_PRE_CONTENT_EVENTS && mask & FAN_ONDIR) + return -EINVAL; + + if (mark_cmd == FAN_MARK_FLUSH) { + fsnotify_clear_marks_by_group(group, obj_type); + return 0; + } + + ret = fanotify_find_path(dfd, pathname, &path, flags, + (mask & ALL_FSNOTIFY_EVENTS), obj_type); + if (ret) + return ret; + + if (mark_cmd == FAN_MARK_ADD) { + ret = fanotify_events_supported(group, &path, mask, flags); + if (ret) + goto path_put_and_out; + } + + /* you can only watch an inode if you have read permissions on it */ + ret = path_permission(&path, MAY_READ); + if (ret) + goto path_put_and_out; + + ret = security_path_notify(&path, mask, obj_type); + if (ret) + goto path_put_and_out; + + if (fid_mode) { + ret = fanotify_test_fsid(path.dentry, flags, &__fsid); + if (ret) + goto path_put_and_out; + + ret = fanotify_test_fid(path.dentry, flags); + if (ret) + goto path_put_and_out; + + fsid = &__fsid; + } + + /* + * In addition to being capable in the user ns where group was created, + * the user also needs to be capable in the user ns associated with + * the filesystem or in the user ns associated with the mntns + * (when marking mntns). + */ + if (obj_type == FSNOTIFY_OBJ_TYPE_INODE) { + inode = path.dentry->d_inode; + obj = inode; + } else if (obj_type == FSNOTIFY_OBJ_TYPE_VFSMOUNT) { + user_ns = path.mnt->mnt_sb->s_user_ns; + obj = path.mnt; + } else if (obj_type == FSNOTIFY_OBJ_TYPE_SB) { + user_ns = path.mnt->mnt_sb->s_user_ns; + obj = path.mnt->mnt_sb; + } else if (obj_type == FSNOTIFY_OBJ_TYPE_MNTNS) { + ret = -EINVAL; + mntns = mnt_ns_from_dentry(path.dentry); + if (!mntns) + goto path_put_and_out; + user_ns = mntns->user_ns; + obj = mntns; + } + + ret = -EPERM; + if (user_ns && !ns_capable(user_ns, CAP_SYS_ADMIN)) + goto path_put_and_out; + + ret = -EINVAL; + if (!obj) + goto path_put_and_out; + + /* + * If some other task has this inode open for write we should not add + * an ignore mask, unless that ignore mask is supposed to survive + * modification changes anyway. + */ + if (mark_cmd == FAN_MARK_ADD && (flags & FANOTIFY_MARK_IGNORE_BITS) && + !(flags & FAN_MARK_IGNORED_SURV_MODIFY)) { + ret = !inode ? -EINVAL : -EISDIR; + /* FAN_MARK_IGNORE requires SURV_MODIFY for sb/mount/dir marks */ + if (ignore == FAN_MARK_IGNORE && + (!inode || S_ISDIR(inode->i_mode))) + goto path_put_and_out; + + ret = 0; + if (inode && inode_is_open_for_write(inode)) + goto path_put_and_out; + } + + /* Mask out FAN_EVENT_ON_CHILD flag for sb/mount/non-dir marks */ + if (!inode || !S_ISDIR(inode->i_mode)) { + mask &= ~FAN_EVENT_ON_CHILD; + umask = FAN_EVENT_ON_CHILD; + /* + * If group needs to report parent fid, register for getting + * events with parent/name info for non-directory. + */ + if ((fid_mode & FAN_REPORT_DIR_FID) && + (flags & FAN_MARK_ADD) && !ignore) + mask |= FAN_EVENT_ON_CHILD; + } + + /* create/update an inode mark */ + switch (mark_cmd) { + case FAN_MARK_ADD: + ret = fanotify_add_mark(group, obj, obj_type, mask, flags, + fsid); + break; + case FAN_MARK_REMOVE: + ret = fanotify_remove_mark(group, obj, obj_type, mask, flags, + umask); + break; + default: + ret = -EINVAL; + } + +path_put_and_out: + path_put(&path); + return ret; +} + +#ifndef CONFIG_ARCH_SPLIT_ARG64 +SYSCALL_DEFINE5(fanotify_mark, int, fanotify_fd, unsigned int, flags, + __u64, mask, int, dfd, + const char __user *, pathname) +{ + return do_fanotify_mark(fanotify_fd, flags, mask, dfd, pathname); +} +#endif + +#if defined(CONFIG_ARCH_SPLIT_ARG64) || defined(CONFIG_COMPAT) +SYSCALL32_DEFINE6(fanotify_mark, + int, fanotify_fd, unsigned int, flags, + SC_ARG64(mask), int, dfd, + const char __user *, pathname) +{ + return do_fanotify_mark(fanotify_fd, flags, SC_VAL64(__u64, mask), + dfd, pathname); +} +#endif + +/* + * fanotify_user_setup - Our initialization function. Note that we cannot return + * error because we have compiled-in VFS hooks. So an (unlikely) failure here + * must result in panic(). + */ +static int __init fanotify_user_setup(void) +{ + struct sysinfo si; + int max_marks; + + si_meminfo(&si); + /* + * Allow up to 1% of addressable memory to be accounted for per user + * marks limited to the range [8192, 1048576]. mount and sb marks are + * a lot cheaper than inode marks, but there is no reason for a user + * to have many of those, so calculate by the cost of inode marks. + */ + max_marks = (((si.totalram - si.totalhigh) / 100) << PAGE_SHIFT) / + INODE_MARK_COST; + max_marks = clamp(max_marks, FANOTIFY_OLD_DEFAULT_MAX_MARKS, + FANOTIFY_DEFAULT_MAX_USER_MARKS); + + BUILD_BUG_ON(FANOTIFY_INIT_FLAGS & FANOTIFY_INTERNAL_GROUP_FLAGS); + BUILD_BUG_ON(HWEIGHT32(FANOTIFY_INIT_FLAGS) != 14); + BUILD_BUG_ON(HWEIGHT32(FANOTIFY_MARK_FLAGS) != 11); + + fanotify_mark_cache = KMEM_CACHE(fanotify_mark, + SLAB_PANIC|SLAB_ACCOUNT); + fanotify_fid_event_cachep = KMEM_CACHE(fanotify_fid_event, + SLAB_PANIC); + fanotify_path_event_cachep = KMEM_CACHE(fanotify_path_event, + SLAB_PANIC); + if (IS_ENABLED(CONFIG_FANOTIFY_ACCESS_PERMISSIONS)) { + fanotify_perm_event_cachep = + KMEM_CACHE(fanotify_perm_event, SLAB_PANIC); + } + fanotify_mnt_event_cachep = KMEM_CACHE(fanotify_mnt_event, SLAB_PANIC); + + fanotify_max_queued_events = FANOTIFY_DEFAULT_MAX_EVENTS; + init_user_ns.ucount_max[UCOUNT_FANOTIFY_GROUPS] = + FANOTIFY_DEFAULT_MAX_GROUPS; + init_user_ns.ucount_max[UCOUNT_FANOTIFY_MARKS] = max_marks; + fanotify_sysctls_init(); + + return 0; +} +device_initcall(fanotify_user_setup); diff --git a/fs/notify/fdinfo.c b/fs/notify/fdinfo.c new file mode 100644 index 000000000..0f731edde --- /dev/null +++ b/fs/notify/fdinfo.c @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "inotify/inotify.h" +#include "fanotify/fanotify.h" +#include "fdinfo.h" +#include "fsnotify.h" +#include "../internal.h" + +#if defined(CONFIG_PROC_FS) + +#if defined(CONFIG_INOTIFY_USER) || defined(CONFIG_FANOTIFY) + +static void show_fdinfo(struct seq_file *m, struct file *f, + void (*show)(struct seq_file *m, + struct fsnotify_mark *mark)) +{ + struct fsnotify_group *group = f->private_data; + struct fsnotify_mark *mark; + + fsnotify_group_lock(group); + list_for_each_entry(mark, &group->marks_list, g_list) { + show(m, mark); + if (seq_has_overflowed(m)) + break; + } + fsnotify_group_unlock(group); +} + +#if defined(CONFIG_EXPORTFS) +static void show_mark_fhandle(struct seq_file *m, struct inode *inode) +{ + DEFINE_FLEX(struct file_handle, f, f_handle, handle_bytes, MAX_HANDLE_SZ); + int size, ret, i; + + size = f->handle_bytes >> 2; + + if (!super_trylock_shared(inode->i_sb)) + return; + + ret = exportfs_encode_fid(inode, (struct fid *)f->f_handle, &size); + up_read(&inode->i_sb->s_umount); + + if ((ret == FILEID_INVALID) || (ret < 0)) + return; + + f->handle_type = ret; + f->handle_bytes = size * sizeof(u32); + + seq_printf(m, "fhandle-bytes:%x fhandle-type:%x f_handle:", + f->handle_bytes, f->handle_type); + + for (i = 0; i < f->handle_bytes; i++) + seq_printf(m, "%02x", (int)f->f_handle[i]); +} +#else +static void show_mark_fhandle(struct seq_file *m, struct inode *inode) +{ +} +#endif + +#ifdef CONFIG_INOTIFY_USER + +static void inotify_fdinfo(struct seq_file *m, struct fsnotify_mark *mark) +{ + struct inotify_inode_mark *inode_mark; + struct inode *inode; + + if (mark->connector->type != FSNOTIFY_OBJ_TYPE_INODE) + return; + + inode_mark = container_of(mark, struct inotify_inode_mark, fsn_mark); + inode = igrab(fsnotify_conn_inode(mark->connector)); + if (inode) { + seq_printf(m, "inotify wd:%x ino:%llx sdev:%x mask:%x ignored_mask:0 ", + inode_mark->wd, inode->i_ino, inode->i_sb->s_dev, + inotify_mark_user_mask(mark)); + show_mark_fhandle(m, inode); + seq_putc(m, '\n'); + iput(inode); + } +} + +void inotify_show_fdinfo(struct seq_file *m, struct file *f) +{ + show_fdinfo(m, f, inotify_fdinfo); +} + +#endif /* CONFIG_INOTIFY_USER */ + +#ifdef CONFIG_FANOTIFY + +static void fanotify_fdinfo(struct seq_file *m, struct fsnotify_mark *mark) +{ + unsigned int mflags = fanotify_mark_user_flags(mark); + struct inode *inode; + + if (mark->connector->type == FSNOTIFY_OBJ_TYPE_INODE) { + inode = igrab(fsnotify_conn_inode(mark->connector)); + if (!inode) + return; + seq_printf(m, "fanotify ino:%llx sdev:%x mflags:%x mask:%x ignored_mask:%x ", + inode->i_ino, inode->i_sb->s_dev, + mflags, mark->mask, mark->ignore_mask); + show_mark_fhandle(m, inode); + seq_putc(m, '\n'); + iput(inode); + } else if (mark->connector->type == FSNOTIFY_OBJ_TYPE_VFSMOUNT) { + struct mount *mnt = fsnotify_conn_mount(mark->connector); + + seq_printf(m, "fanotify mnt_id:%x mflags:%x mask:%x ignored_mask:%x\n", + mnt->mnt_id, mflags, mark->mask, mark->ignore_mask); + } else if (mark->connector->type == FSNOTIFY_OBJ_TYPE_SB) { + struct super_block *sb = fsnotify_conn_sb(mark->connector); + + seq_printf(m, "fanotify sdev:%x mflags:%x mask:%x ignored_mask:%x\n", + sb->s_dev, mflags, mark->mask, mark->ignore_mask); + } else if (mark->connector->type == FSNOTIFY_OBJ_TYPE_MNTNS) { + struct mnt_namespace *mnt_ns = fsnotify_conn_mntns(mark->connector); + + seq_printf(m, "fanotify mnt_ns:%u mflags:%x mask:%x ignored_mask:%x\n", + mnt_ns->ns.inum, mflags, mark->mask, mark->ignore_mask); + } +} + +void fanotify_show_fdinfo(struct seq_file *m, struct file *f) +{ + struct fsnotify_group *group = f->private_data; + + seq_printf(m, "fanotify flags:%x event-flags:%x\n", + group->fanotify_data.flags & FANOTIFY_INIT_FLAGS, + group->fanotify_data.f_flags); + + show_fdinfo(m, f, fanotify_fdinfo); +} + +#endif /* CONFIG_FANOTIFY */ + +#endif /* CONFIG_INOTIFY_USER || CONFIG_FANOTIFY */ + +#endif /* CONFIG_PROC_FS */ diff --git a/fs/notify/fdinfo.h b/fs/notify/fdinfo.h new file mode 100644 index 000000000..5c9937e02 --- /dev/null +++ b/fs/notify/fdinfo.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __FSNOTIFY_FDINFO_H__ +#define __FSNOTIFY_FDINFO_H__ + +#include +#include + +struct seq_file; +struct file; + +#ifdef CONFIG_PROC_FS + +#ifdef CONFIG_INOTIFY_USER +void inotify_show_fdinfo(struct seq_file *m, struct file *f); +#endif + +#ifdef CONFIG_FANOTIFY +void fanotify_show_fdinfo(struct seq_file *m, struct file *f); +#endif + +#else /* CONFIG_PROC_FS */ + +#define inotify_show_fdinfo NULL +#define fanotify_show_fdinfo NULL + +#endif /* CONFIG_PROC_FS */ + +#endif /* __FSNOTIFY_FDINFO_H__ */ diff --git a/fs/notify/fsnotify.c b/fs/notify/fsnotify.c new file mode 100644 index 000000000..7e2f330fd --- /dev/null +++ b/fs/notify/fsnotify.c @@ -0,0 +1,733 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2008 Red Hat, Inc., Eric Paris + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include "fsnotify.h" + +#define CREATE_TRACE_POINTS +#include + +/* + * Clear all of the marks on an inode when it is being evicted from core + */ +void __fsnotify_inode_delete(struct inode *inode) +{ + fsnotify_clear_marks_by_inode(inode); +} +EXPORT_SYMBOL_GPL(__fsnotify_inode_delete); + +void __fsnotify_vfsmount_delete(struct vfsmount *mnt) +{ + fsnotify_clear_marks_by_mount(mnt); +} + +void __fsnotify_mntns_delete(struct mnt_namespace *mntns) +{ + fsnotify_clear_marks_by_mntns(mntns); +} + +void fsnotify_sb_delete(struct super_block *sb) +{ + struct fsnotify_sb_info *sbinfo = fsnotify_sb_info(sb); + + /* Were any marks ever added to any object on this sb? */ + if (!sbinfo) + return; + + fsnotify_unmount_inodes(sbinfo); + fsnotify_clear_marks_by_sb(sb); + /* Wait for outstanding object references from connectors */ + wait_var_event(fsnotify_sb_watched_objects(sb), + !atomic_long_read(fsnotify_sb_watched_objects(sb))); + WARN_ON(fsnotify_sb_has_priority_watchers(sb, FSNOTIFY_PRIO_CONTENT)); + WARN_ON(fsnotify_sb_has_priority_watchers(sb, + FSNOTIFY_PRIO_PRE_CONTENT)); +} + +void fsnotify_sb_free(struct super_block *sb) +{ + if (sb->s_fsnotify_info) { + WARN_ON_ONCE(!list_empty(&sb->s_fsnotify_info->inode_conn_list)); + kfree(sb->s_fsnotify_info); + } +} + +/* + * Given an inode, first check if we care what happens to our children. Inotify + * and dnotify both tell their parents about events. If we care about any event + * on a child we run all of our children and set a dentry flag saying that the + * parent cares. Thus when an event happens on a child it can quickly tell + * if there is a need to find a parent and send the event to the parent. + */ +void fsnotify_set_children_dentry_flags(struct inode *inode) +{ + struct dentry *alias; + + if (!S_ISDIR(inode->i_mode)) + return; + + spin_lock(&inode->i_lock); + /* run all of the dentries associated with this inode. Since this is a + * directory, there damn well better only be one item on this list */ + for_each_alias(alias, inode) { + struct dentry *child; + + /* run all of the children of the original inode and fix their + * d_flags to indicate parental interest (their parent is the + * original inode) */ + spin_lock(&alias->d_lock); + hlist_for_each_entry(child, &alias->d_children, d_sib) { + if (!child->d_inode) + continue; + + spin_lock_nested(&child->d_lock, DENTRY_D_LOCK_NESTED); + child->d_flags |= DCACHE_FSNOTIFY_PARENT_WATCHED; + spin_unlock(&child->d_lock); + } + spin_unlock(&alias->d_lock); + } + spin_unlock(&inode->i_lock); +} + +/* + * Lazily clear false positive PARENT_WATCHED flag for child whose parent had + * stopped watching children. + */ +static void fsnotify_clear_child_dentry_flag(struct inode *pinode, + struct dentry *dentry) +{ + spin_lock(&dentry->d_lock); + /* + * d_lock is a sufficient barrier to prevent observing a non-watched + * parent state from before the fsnotify_set_children_dentry_flags() + * or fsnotify_update_flags() call that had set PARENT_WATCHED. + */ + if (!fsnotify_inode_watches_children(pinode)) + dentry->d_flags &= ~DCACHE_FSNOTIFY_PARENT_WATCHED; + spin_unlock(&dentry->d_lock); +} + +/* Are inode/sb/mount interested in parent and name info with this event? */ +static bool fsnotify_event_needs_parent(struct inode *inode, __u32 mnt_mask, + __u32 mask) +{ + __u32 marks_mask = 0; + + /* We only send parent/name to inode/sb/mount for events on non-dir */ + if (mask & FS_ISDIR) + return false; + + /* + * All events that are possible on child can also may be reported with + * parent/name info to inode/sb/mount. Otherwise, a watching parent + * could result in events reported with unexpected name info to sb/mount. + */ + BUILD_BUG_ON(FS_EVENTS_POSS_ON_CHILD & ~FS_EVENTS_POSS_TO_PARENT); + + /* Did either inode/sb/mount subscribe for events with parent/name? */ + marks_mask |= fsnotify_parent_needed_mask( + READ_ONCE(inode->i_fsnotify_mask)); + marks_mask |= fsnotify_parent_needed_mask( + READ_ONCE(inode->i_sb->s_fsnotify_mask)); + marks_mask |= fsnotify_parent_needed_mask(mnt_mask); + + /* Did they subscribe for this event with parent/name info? */ + return mask & marks_mask; +} + +/* Are there any inode/mount/sb objects that watch for these events? */ +static inline __u32 fsnotify_object_watched(struct inode *inode, __u32 mnt_mask, + __u32 mask) +{ + __u32 marks_mask = READ_ONCE(inode->i_fsnotify_mask) | mnt_mask | + READ_ONCE(inode->i_sb->s_fsnotify_mask); + + return mask & marks_mask & ALL_FSNOTIFY_EVENTS; +} + +/* Report pre-content event with optional range info */ +int fsnotify_pre_content(const struct path *path, const loff_t *ppos, + size_t count) +{ + struct file_range range; + + /* Report page aligned range only when pos is known */ + if (!ppos) + return fsnotify_path(path, FS_PRE_ACCESS); + + range.path = path; + range.pos = PAGE_ALIGN_DOWN(*ppos); + range.count = PAGE_ALIGN(*ppos + count) - range.pos; + + return fsnotify_parent(path->dentry, FS_PRE_ACCESS, &range, + FSNOTIFY_EVENT_FILE_RANGE); +} + +/* + * Notify this dentry's parent about a child's events with child name info + * if parent is watching or if inode/sb/mount are interested in events with + * parent and name info. + * + * Notify only the child without name info if parent is not watching and + * inode/sb/mount are not interested in events with parent and name info. + */ +int __fsnotify_parent(struct dentry *dentry, __u32 mask, const void *data, + int data_type) +{ + const struct path *path = fsnotify_data_path(data, data_type); + __u32 mnt_mask = path ? + READ_ONCE(real_mount(path->mnt)->mnt_fsnotify_mask) : 0; + struct inode *inode = d_inode(dentry); + struct dentry *parent; + bool parent_watched = dentry->d_flags & DCACHE_FSNOTIFY_PARENT_WATCHED; + bool parent_needed, parent_interested; + __u32 p_mask; + struct inode *p_inode = NULL; + struct name_snapshot name; + struct qstr *file_name = NULL; + int ret = 0; + + /* Optimize the likely case of nobody watching this path */ + if (likely(!parent_watched && + !fsnotify_object_watched(inode, mnt_mask, mask))) + return 0; + + parent = NULL; + parent_needed = fsnotify_event_needs_parent(inode, mnt_mask, mask); + if (!parent_watched && !parent_needed) + goto notify; + + /* Does parent inode care about events on children? */ + parent = dget_parent(dentry); + p_inode = parent->d_inode; + p_mask = fsnotify_inode_watches_children(p_inode); + if (unlikely(parent_watched && !p_mask)) + fsnotify_clear_child_dentry_flag(p_inode, dentry); + + /* + * Include parent/name in notification either if some notification + * groups require parent info or the parent is interested in this event. + * The parent interest in ACCESS/MODIFY events does not apply to special + * files, where read/write are not on the filesystem of the parent and + * events can provide an undesirable side-channel for information + * exfiltration. + */ + parent_interested = mask & p_mask & ALL_FSNOTIFY_EVENTS && + !(data_type == FSNOTIFY_EVENT_PATH && + d_is_special(dentry) && + (mask & (FS_ACCESS | FS_MODIFY))); + if (parent_needed || parent_interested) { + /* When notifying parent, child should be passed as data */ + WARN_ON_ONCE(inode != fsnotify_data_inode(data, data_type)); + + /* Notify both parent and child with child name info */ + take_dentry_name_snapshot(&name, dentry); + file_name = &name.name; + if (parent_interested) + mask |= FS_EVENT_ON_CHILD; + } + +notify: + ret = fsnotify(mask, data, data_type, p_inode, file_name, inode, 0); + + if (file_name) + release_dentry_name_snapshot(&name); + dput(parent); + + return ret; +} +EXPORT_SYMBOL_GPL(__fsnotify_parent); + +static int fsnotify_handle_inode_event(struct fsnotify_group *group, + struct fsnotify_mark *inode_mark, + u32 mask, const void *data, int data_type, + struct inode *dir, const struct qstr *name, + u32 cookie) +{ + const struct path *path = fsnotify_data_path(data, data_type); + struct inode *inode = fsnotify_data_inode(data, data_type); + const struct fsnotify_ops *ops = group->ops; + + if (WARN_ON_ONCE(!ops->handle_inode_event)) + return 0; + + if (WARN_ON_ONCE(!inode && !dir)) + return 0; + + if ((inode_mark->flags & FSNOTIFY_MARK_FLAG_EXCL_UNLINK) && + path && d_unlinked(path->dentry)) + return 0; + + /* Check interest of this mark in case event was sent with two marks */ + if (!(mask & inode_mark->mask & ALL_FSNOTIFY_EVENTS)) + return 0; + + return ops->handle_inode_event(inode_mark, mask, inode, dir, name, cookie); +} + +static int fsnotify_handle_event(struct fsnotify_group *group, __u32 mask, + const void *data, int data_type, + struct inode *dir, const struct qstr *name, + u32 cookie, struct fsnotify_iter_info *iter_info) +{ + struct fsnotify_mark *inode_mark = fsnotify_iter_inode_mark(iter_info); + struct fsnotify_mark *parent_mark = fsnotify_iter_parent_mark(iter_info); + int ret; + + if (WARN_ON_ONCE(fsnotify_iter_sb_mark(iter_info)) || + WARN_ON_ONCE(fsnotify_iter_vfsmount_mark(iter_info))) + return 0; + + /* + * For FS_RENAME, 'dir' is old dir and 'data' is new dentry. + * The only ->handle_inode_event() backend that supports FS_RENAME is + * dnotify, where it means file was renamed within same parent. + */ + if (mask & FS_RENAME) { + struct dentry *moved = fsnotify_data_dentry(data, data_type); + + if (dir != moved->d_parent->d_inode) + return 0; + } + + if (parent_mark) { + ret = fsnotify_handle_inode_event(group, parent_mark, mask, + data, data_type, dir, name, 0); + if (ret) + return ret; + } + + if (!inode_mark) + return 0; + + /* + * Some events can be sent on both parent dir and child marks (e.g. + * FS_ATTRIB). If both parent dir and child are watching, report the + * event once to parent dir with name (if interested) and once to child + * without name (if interested). + * + * In any case regardless whether the parent is watching or not, the + * child watcher is expecting an event without the FS_EVENT_ON_CHILD + * flag. The file name is expected if and only if this is a directory + * event. + */ + mask &= ~FS_EVENT_ON_CHILD; + if (!(mask & ALL_FSNOTIFY_DIRENT_EVENTS)) { + dir = NULL; + name = NULL; + } + + return fsnotify_handle_inode_event(group, inode_mark, mask, data, data_type, + dir, name, cookie); +} + +static int send_to_group(__u32 mask, const void *data, int data_type, + struct inode *dir, const struct qstr *file_name, + u32 cookie, struct fsnotify_iter_info *iter_info) +{ + struct fsnotify_group *group = NULL; + __u32 test_mask = (mask & ALL_FSNOTIFY_EVENTS); + __u32 marks_mask = 0; + __u32 marks_ignore_mask = 0; + bool is_dir = mask & FS_ISDIR; + struct fsnotify_mark *mark; + int type; + + if (!iter_info->report_mask) + return 0; + + /* clear ignored on inode modification */ + if (mask & FS_MODIFY) { + fsnotify_foreach_iter_mark_type(iter_info, mark, type) { + if (!(mark->flags & + FSNOTIFY_MARK_FLAG_IGNORED_SURV_MODIFY)) + mark->ignore_mask = 0; + } + } + + /* Are any of the group marks interested in this event? */ + fsnotify_foreach_iter_mark_type(iter_info, mark, type) { + group = mark->group; + marks_mask |= mark->mask; + marks_ignore_mask |= + fsnotify_effective_ignore_mask(mark, is_dir, type); + } + + pr_debug("%s: group=%p mask=%x marks_mask=%x marks_ignore_mask=%x data=%p data_type=%d dir=%p cookie=%d\n", + __func__, group, mask, marks_mask, marks_ignore_mask, + data, data_type, dir, cookie); + + if (!(test_mask & marks_mask & ~marks_ignore_mask)) + return 0; + + if (group->ops->handle_event) { + return group->ops->handle_event(group, mask, data, data_type, dir, + file_name, cookie, iter_info); + } + + return fsnotify_handle_event(group, mask, data, data_type, dir, + file_name, cookie, iter_info); +} + +static struct fsnotify_mark *fsnotify_first_mark(struct fsnotify_mark_connector *const *connp) +{ + struct fsnotify_mark_connector *conn; + struct hlist_node *node = NULL; + + conn = srcu_dereference(*connp, &fsnotify_mark_srcu); + if (conn) + node = srcu_dereference(conn->list.first, &fsnotify_mark_srcu); + + return hlist_entry_safe(node, struct fsnotify_mark, obj_list); +} + +struct fsnotify_mark *fsnotify_next_mark(struct fsnotify_mark *mark) +{ + struct hlist_node *node = NULL; + + if (mark) + node = srcu_dereference(mark->obj_list.next, + &fsnotify_mark_srcu); + + return hlist_entry_safe(node, struct fsnotify_mark, obj_list); +} + +/* + * iter_info is a multi head priority queue of marks. + * Pick a subset of marks from queue heads, all with the same group + * and set the report_mask to a subset of the selected marks. + * Returns false if there are no more groups to iterate. + */ +static bool fsnotify_iter_select_report_types( + struct fsnotify_iter_info *iter_info) +{ + struct fsnotify_group *max_prio_group = NULL; + struct fsnotify_mark *mark; + int type; + + /* Choose max prio group among groups of all queue heads */ + fsnotify_foreach_iter_type(type) { + mark = iter_info->marks[type]; + if (mark && + fsnotify_compare_groups(max_prio_group, mark->group) > 0) + max_prio_group = mark->group; + } + + if (!max_prio_group) + return false; + + /* Set the report mask for marks from same group as max prio group */ + iter_info->current_group = max_prio_group; + iter_info->report_mask = 0; + fsnotify_foreach_iter_type(type) { + mark = iter_info->marks[type]; + if (mark && mark->group == iter_info->current_group) { + /* + * FSNOTIFY_ITER_TYPE_PARENT indicates that this inode + * is watching children and interested in this event, + * which is an event possible on child. + * But is *this mark* watching children? + */ + if (type == FSNOTIFY_ITER_TYPE_PARENT && + !(mark->mask & FS_EVENT_ON_CHILD) && + !(fsnotify_ignore_mask(mark) & FS_EVENT_ON_CHILD)) + continue; + + fsnotify_iter_set_report_type(iter_info, type); + } + } + + return true; +} + +/* + * Pop from iter_info multi head queue, the marks that belong to the group of + * current iteration step. + */ +static void fsnotify_iter_next(struct fsnotify_iter_info *iter_info) +{ + struct fsnotify_mark *mark; + int type; + + /* + * We cannot use fsnotify_foreach_iter_mark_type() here because we + * may need to advance a mark of type X that belongs to current_group + * but was not selected for reporting. + */ + fsnotify_foreach_iter_type(type) { + mark = iter_info->marks[type]; + if (mark && mark->group == iter_info->current_group) + iter_info->marks[type] = + fsnotify_next_mark(iter_info->marks[type]); + } +} + +/* + * fsnotify - This is the main call to fsnotify. + * + * The VFS calls into hook specific functions in linux/fsnotify.h. + * Those functions then in turn call here. Here will call out to all of the + * registered fsnotify_group. Those groups can then use the notification event + * in whatever means they feel necessary. + * + * @mask: event type and flags + * @data: object that event happened on + * @data_type: type of object for fanotify_data_XXX() accessors + * @dir: optional directory associated with event - + * if @file_name is not NULL, this is the directory that + * @file_name is relative to + * @file_name: optional file name associated with event + * @inode: optional inode associated with event - + * If @dir and @inode are both non-NULL, event may be + * reported to both. + * @cookie: inotify rename cookie + */ +int fsnotify(__u32 mask, const void *data, int data_type, struct inode *dir, + const struct qstr *file_name, struct inode *inode, u32 cookie) +{ + const struct path *path = fsnotify_data_path(data, data_type); + struct super_block *sb = fsnotify_data_sb(data, data_type); + const struct fsnotify_mnt *mnt_data = fsnotify_data_mnt(data, data_type); + struct fsnotify_sb_info *sbinfo = sb ? fsnotify_sb_info(sb) : NULL; + struct fsnotify_iter_info iter_info = {}; + struct mount *mnt = NULL; + struct inode *inode2 = NULL; + struct dentry *moved; + int inode2_type; + int ret = 0; + __u32 test_mask, marks_mask = 0; + + trace_fsnotify(mask, data, data_type, dir, file_name, inode, cookie); + + if (path) + mnt = real_mount(path->mnt); + + if (!inode) { + /* Dirent event - report on TYPE_INODE to dir */ + inode = dir; + /* For FS_RENAME, inode is old_dir and inode2 is new_dir */ + if (mask & FS_RENAME) { + moved = fsnotify_data_dentry(data, data_type); + inode2 = moved->d_parent->d_inode; + inode2_type = FSNOTIFY_ITER_TYPE_INODE2; + } + } else if (mask & FS_EVENT_ON_CHILD) { + /* + * Event on child - report on TYPE_PARENT to dir if it is + * watching children and on TYPE_INODE to child. + */ + inode2 = dir; + inode2_type = FSNOTIFY_ITER_TYPE_PARENT; + } + + /* + * Optimization: srcu_read_lock() has a memory barrier which can + * be expensive. It protects walking the *_fsnotify_marks lists. + * However, if we do not walk the lists, we do not have to do + * SRCU because we have no references to any objects and do not + * need SRCU to keep them "alive". + */ + if ((!sbinfo || !sbinfo->sb_marks) && + (!mnt || !mnt->mnt_fsnotify_marks) && + (!inode || !inode->i_fsnotify_marks) && + (!inode2 || !inode2->i_fsnotify_marks) && + (!mnt_data || !mnt_data->ns->n_fsnotify_marks)) + return 0; + + if (sb) + marks_mask |= READ_ONCE(sb->s_fsnotify_mask); + if (mnt) + marks_mask |= READ_ONCE(mnt->mnt_fsnotify_mask); + if (inode) + marks_mask |= READ_ONCE(inode->i_fsnotify_mask); + if (inode2) + marks_mask |= READ_ONCE(inode2->i_fsnotify_mask); + if (mnt_data) + marks_mask |= READ_ONCE(mnt_data->ns->n_fsnotify_mask); + + /* + * If this is a modify event we may need to clear some ignore masks. + * In that case, the object with ignore masks will have the FS_MODIFY + * event in its mask. + * Otherwise, return if none of the marks care about this type of event. + */ + test_mask = (mask & ALL_FSNOTIFY_EVENTS); + if (!(test_mask & marks_mask)) + return 0; + + iter_info.srcu_idx = srcu_read_lock(&fsnotify_mark_srcu); + + if (sbinfo) { + iter_info.marks[FSNOTIFY_ITER_TYPE_SB] = + fsnotify_first_mark(&sbinfo->sb_marks); + } + if (mnt) { + iter_info.marks[FSNOTIFY_ITER_TYPE_VFSMOUNT] = + fsnotify_first_mark(&mnt->mnt_fsnotify_marks); + } + if (inode) { + iter_info.marks[FSNOTIFY_ITER_TYPE_INODE] = + fsnotify_first_mark(&inode->i_fsnotify_marks); + } + if (inode2) { + iter_info.marks[inode2_type] = + fsnotify_first_mark(&inode2->i_fsnotify_marks); + } + if (mnt_data) { + iter_info.marks[FSNOTIFY_ITER_TYPE_MNTNS] = + fsnotify_first_mark(&mnt_data->ns->n_fsnotify_marks); + } + + /* + * We need to merge inode/vfsmount/sb mark lists so that e.g. inode mark + * ignore masks are properly reflected for mount/sb mark notifications. + * That's why this traversal is so complicated... + */ + while (fsnotify_iter_select_report_types(&iter_info)) { + ret = send_to_group(mask, data, data_type, dir, file_name, + cookie, &iter_info); + + if (ret && (mask & ALL_FSNOTIFY_PERM_EVENTS)) + goto out; + + fsnotify_iter_next(&iter_info); + } + ret = 0; +out: + srcu_read_unlock(&fsnotify_mark_srcu, iter_info.srcu_idx); + + return ret; +} +EXPORT_SYMBOL_GPL(fsnotify); + +#ifdef CONFIG_FANOTIFY_ACCESS_PERMISSIONS +/* + * At open time we check fsnotify_sb_has_priority_watchers(), call the open perm + * hook and set the FMODE_NONOTIFY_ mode bits accordignly. + * Later, fsnotify permission hooks do not check if there are permission event + * watches, but that there were permission event watches at open time. + */ +int fsnotify_open_perm_and_set_mode(struct file *file) +{ + struct dentry *dentry = file->f_path.dentry, *parent; + struct super_block *sb = dentry->d_sb; + __u32 mnt_mask, p_mask = 0; + + /* Is it a file opened by fanotify? */ + if (FMODE_FSNOTIFY_NONE(file->f_mode)) + return 0; + + /* + * Permission events is a super set of pre-content events, so if there + * are no permission event watchers, there are also no pre-content event + * watchers and this is implied from the single FMODE_NONOTIFY_PERM bit. + */ + if (likely(!fsnotify_sb_has_priority_watchers(sb, + FSNOTIFY_PRIO_CONTENT))) { + file_set_fsnotify_mode(file, FMODE_NONOTIFY_PERM); + return 0; + } + + /* + * OK, there are some permission event watchers. Check if anybody is + * watching for permission events on *this* file. + */ + mnt_mask = READ_ONCE(real_mount(file->f_path.mnt)->mnt_fsnotify_mask); + p_mask = fsnotify_object_watched(d_inode(dentry), mnt_mask, + ALL_FSNOTIFY_PERM_EVENTS); + if (dentry->d_flags & DCACHE_FSNOTIFY_PARENT_WATCHED) { + parent = dget_parent(dentry); + p_mask |= fsnotify_inode_watches_children(d_inode(parent)); + dput(parent); + } + + /* + * Legacy FAN_ACCESS_PERM events have very high performance overhead, + * so unlikely to be used in the wild. If they are used there will be + * no optimizations at all. + */ + if (unlikely(p_mask & FS_ACCESS_PERM)) { + /* Enable all permission and pre-content events */ + file_set_fsnotify_mode(file, 0); + goto open_perm; + } + + /* + * Pre-content events are only supported on regular files. + * If there are pre-content event watchers and no permission access + * watchers, set FMODE_NONOTIFY | FMODE_NONOTIFY_PERM to indicate that. + * That is the common case with HSM service. + */ + if (d_is_reg(dentry) && (p_mask & FSNOTIFY_PRE_CONTENT_EVENTS)) { + file_set_fsnotify_mode(file, FMODE_NONOTIFY | + FMODE_NONOTIFY_PERM); + goto open_perm; + } + + /* Nobody watching permission and pre-content events on this file */ + file_set_fsnotify_mode(file, FMODE_NONOTIFY_PERM); + +open_perm: + /* + * Send open perm events depending on object masks and regardless of + * FMODE_NONOTIFY_PERM. + */ + if (file->f_flags & __FMODE_EXEC && p_mask & FS_OPEN_EXEC_PERM) { + int ret = fsnotify_path(&file->f_path, FS_OPEN_EXEC_PERM); + + if (ret) + return ret; + } + + if (p_mask & FS_OPEN_PERM) + return fsnotify_path(&file->f_path, FS_OPEN_PERM); + + return 0; +} +#endif + +void fsnotify_mnt(__u32 mask, struct mnt_namespace *ns, struct vfsmount *mnt) +{ + struct fsnotify_mnt data = { + .ns = ns, + .mnt_id = real_mount(mnt)->mnt_id_unique, + }; + + if (WARN_ON_ONCE(!ns)) + return; + + /* + * This is an optimization as well as making sure fsnotify_init() has + * been called. + */ + if (!ns->n_fsnotify_marks) + return; + + fsnotify(mask, &data, FSNOTIFY_EVENT_MNT, NULL, NULL, NULL, 0); +} + +static __init int fsnotify_init(void) +{ + int ret; + + BUILD_BUG_ON(HWEIGHT32(ALL_FSNOTIFY_BITS) != 26); + + ret = init_srcu_struct(&fsnotify_mark_srcu); + if (ret) + panic("initializing fsnotify_mark_srcu"); + + fsnotify_init_connector_caches(); + + return 0; +} +core_initcall(fsnotify_init); diff --git a/fs/notify/fsnotify.h b/fs/notify/fsnotify.h new file mode 100644 index 000000000..58c7bb25e --- /dev/null +++ b/fs/notify/fsnotify.h @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __FS_NOTIFY_FSNOTIFY_H_ +#define __FS_NOTIFY_FSNOTIFY_H_ + +#include +#include +#include +#include + +#include "../mount.h" + +/* + * fsnotify_connp_t is what we embed in objects which connector can be attached + * to. + */ +typedef struct fsnotify_mark_connector __rcu *fsnotify_connp_t; + +static inline struct inode *fsnotify_conn_inode( + struct fsnotify_mark_connector *conn) +{ + return conn->obj; +} + +static inline struct mount *fsnotify_conn_mount( + struct fsnotify_mark_connector *conn) +{ + return real_mount(conn->obj); +} + +static inline struct super_block *fsnotify_conn_sb( + struct fsnotify_mark_connector *conn) +{ + return conn->obj; +} + +static inline struct mnt_namespace *fsnotify_conn_mntns( + struct fsnotify_mark_connector *conn) +{ + return conn->obj; +} + +static inline struct super_block *fsnotify_object_sb(void *obj, + enum fsnotify_obj_type obj_type) +{ + switch (obj_type) { + case FSNOTIFY_OBJ_TYPE_INODE: + return ((struct inode *)obj)->i_sb; + case FSNOTIFY_OBJ_TYPE_VFSMOUNT: + return ((struct vfsmount *)obj)->mnt_sb; + case FSNOTIFY_OBJ_TYPE_SB: + return (struct super_block *)obj; + default: + return NULL; + } +} + +static inline struct super_block *fsnotify_connector_sb( + struct fsnotify_mark_connector *conn) +{ + return fsnotify_object_sb(conn->obj, conn->type); +} + +static inline fsnotify_connp_t *fsnotify_sb_marks(struct super_block *sb) +{ + struct fsnotify_sb_info *sbinfo = fsnotify_sb_info(sb); + + return sbinfo ? &sbinfo->sb_marks : NULL; +} + +/* destroy all events sitting in this groups notification queue */ +extern void fsnotify_flush_notify(struct fsnotify_group *group); + +/* protects reads of inode and vfsmount marks list */ +extern struct srcu_struct fsnotify_mark_srcu; + +/* compare two groups for sorting of marks lists */ +extern int fsnotify_compare_groups(struct fsnotify_group *a, + struct fsnotify_group *b); + +/* Destroy all inode marks for given superblock */ +void fsnotify_unmount_inodes(struct fsnotify_sb_info *sbinfo); + +/* Destroy all marks attached to an object via connector */ +extern void fsnotify_destroy_marks(fsnotify_connp_t *connp); +/* run the list of all marks associated with inode and destroy them */ +static inline void fsnotify_clear_marks_by_inode(struct inode *inode) +{ + fsnotify_destroy_marks(&inode->i_fsnotify_marks); +} +/* run the list of all marks associated with vfsmount and destroy them */ +static inline void fsnotify_clear_marks_by_mount(struct vfsmount *mnt) +{ + fsnotify_destroy_marks(&real_mount(mnt)->mnt_fsnotify_marks); +} +/* run the list of all marks associated with sb and destroy them */ +static inline void fsnotify_clear_marks_by_sb(struct super_block *sb) +{ + fsnotify_destroy_marks(fsnotify_sb_marks(sb)); +} + +static inline void fsnotify_clear_marks_by_mntns(struct mnt_namespace *mntns) +{ + fsnotify_destroy_marks(&mntns->n_fsnotify_marks); +} + +/* + * update the dentry->d_flags of all of inode's children to indicate if inode cares + * about events that happen to its children. + */ +extern void fsnotify_set_children_dentry_flags(struct inode *inode); + +void fsnotify_init_connector_caches(void); + +#endif /* __FS_NOTIFY_FSNOTIFY_H_ */ diff --git a/fs/notify/group.c b/fs/notify/group.c new file mode 100644 index 000000000..b56d1c1d9 --- /dev/null +++ b/fs/notify/group.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2008 Red Hat, Inc., Eric Paris + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include "fsnotify.h" + +#include + +/* + * Final freeing of a group + */ +static void fsnotify_final_destroy_group(struct fsnotify_group *group) +{ + if (group->ops->free_group_priv) + group->ops->free_group_priv(group); + + mem_cgroup_put(group->memcg); + mutex_destroy(&group->mark_mutex); + + kfree(group); +} + +/* + * Stop queueing new events for this group. Once this function returns + * fsnotify_add_event() will not add any new events to the group's queue. + */ +void fsnotify_group_stop_queueing(struct fsnotify_group *group) +{ + spin_lock(&group->notification_lock); + group->shutdown = true; + spin_unlock(&group->notification_lock); +} + +/* + * Trying to get rid of a group. Remove all marks, flush all events and release + * the group reference. + * Note that another thread calling fsnotify_clear_marks_by_group() may still + * hold a ref to the group. + */ +void fsnotify_destroy_group(struct fsnotify_group *group) +{ + /* + * Stop queueing new events. The code below is careful enough to not + * require this but fanotify needs to stop queuing events even before + * fsnotify_destroy_group() is called and this makes the other callers + * of fsnotify_destroy_group() to see the same behavior. + */ + fsnotify_group_stop_queueing(group); + + /* Clear all marks for this group and queue them for destruction */ + fsnotify_clear_marks_by_group(group, FSNOTIFY_OBJ_TYPE_ANY); + + /* + * Some marks can still be pinned when waiting for response from + * userspace. Wait for those now. fsnotify_prepare_user_wait() will + * not succeed now so this wait is race-free. + */ + wait_event(group->notification_waitq, !atomic_read(&group->user_waits)); + + /* + * Wait until all marks get really destroyed. We could actually destroy + * them ourselves instead of waiting for worker to do it, however that + * would be racy as worker can already be processing some marks before + * we even entered fsnotify_destroy_group(). + */ + fsnotify_wait_marks_destroyed(); + + /* + * Since we have waited for fsnotify_mark_srcu in + * fsnotify_mark_destroy_list() there can be no outstanding event + * notification against this group. So clearing the notification queue + * of all events is reliable now. + */ + fsnotify_flush_notify(group); + + /* + * Destroy overflow event (we cannot use fsnotify_destroy_event() as + * that deliberately ignores overflow events. + */ + if (group->overflow_event) + group->ops->free_event(group, group->overflow_event); + + fsnotify_put_group(group); +} + +/* + * Get reference to a group. + */ +void fsnotify_get_group(struct fsnotify_group *group) +{ + refcount_inc(&group->refcnt); +} + +/* + * Drop a reference to a group. Free it if it's through. + */ +void fsnotify_put_group(struct fsnotify_group *group) +{ + if (refcount_dec_and_test(&group->refcnt)) + fsnotify_final_destroy_group(group); +} +EXPORT_SYMBOL_GPL(fsnotify_put_group); + +static struct fsnotify_group *__fsnotify_alloc_group( + const struct fsnotify_ops *ops, + int flags, gfp_t gfp) +{ + struct fsnotify_group *group; + + group = kzalloc_obj(struct fsnotify_group, gfp); + if (!group) + return ERR_PTR(-ENOMEM); + + /* set to 0 when there a no external references to this group */ + refcount_set(&group->refcnt, 1); + atomic_set(&group->user_waits, 0); + + spin_lock_init(&group->notification_lock); + INIT_LIST_HEAD(&group->notification_list); + init_waitqueue_head(&group->notification_waitq); + group->max_events = UINT_MAX; + + mutex_init(&group->mark_mutex); + INIT_LIST_HEAD(&group->marks_list); + + group->ops = ops; + group->flags = flags; + + return group; +} + +/* + * Create a new fsnotify_group and hold a reference for the group returned. + */ +struct fsnotify_group *fsnotify_alloc_group(const struct fsnotify_ops *ops, + int flags) +{ + gfp_t gfp = (flags & FSNOTIFY_GROUP_USER) ? GFP_KERNEL_ACCOUNT : + GFP_KERNEL; + + return __fsnotify_alloc_group(ops, flags, gfp); +} +EXPORT_SYMBOL_GPL(fsnotify_alloc_group); + +int fsnotify_fasync(int fd, struct file *file, int on) +{ + struct fsnotify_group *group = file->private_data; + + return fasync_helper(fd, file, on, &group->fsn_fa) >= 0 ? 0 : -EIO; +} diff --git a/fs/notify/inotify/Kconfig b/fs/notify/inotify/Kconfig new file mode 100644 index 000000000..1cc8be25d --- /dev/null +++ b/fs/notify/inotify/Kconfig @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: GPL-2.0-only +config INOTIFY_USER + bool "Inotify support for userspace" + select FSNOTIFY + default y + help + Say Y here to enable inotify support for userspace, including the + associated system calls. Inotify allows monitoring of both files and + directories via a single open fd. Events are read from the file + descriptor, which is also select()- and poll()-able. + Inotify fixes numerous shortcomings in dnotify and introduces several + new features including multiple file events, one-shot support, and + unmount notification. + + For more information, see + + If unsure, say Y. diff --git a/fs/notify/inotify/Makefile b/fs/notify/inotify/Makefile new file mode 100644 index 000000000..812237eec --- /dev/null +++ b/fs/notify/inotify/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_INOTIFY_USER) += inotify_fsnotify.o inotify_user.o diff --git a/fs/notify/inotify/inotify.h b/fs/notify/inotify/inotify.h new file mode 100644 index 000000000..7d5df7a21 --- /dev/null +++ b/fs/notify/inotify/inotify.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include +#include /* struct kmem_cache */ + +struct inotify_event_info { + struct fsnotify_event fse; + u32 mask; + int wd; + u32 sync_cookie; + int name_len; + char name[]; +}; + +struct inotify_inode_mark { + struct fsnotify_mark fsn_mark; + int wd; +}; + +static inline struct inotify_event_info *INOTIFY_E(struct fsnotify_event *fse) +{ + return container_of(fse, struct inotify_event_info, fse); +} + +/* + * INOTIFY_USER_FLAGS represents all of the mask bits that we expose to + * userspace. There is at least one bit (FS_EVENT_ON_CHILD) which is + * used only internally to the kernel. + */ +#define INOTIFY_USER_MASK (IN_ALL_EVENTS) + +static inline __u32 inotify_mark_user_mask(struct fsnotify_mark *fsn_mark) +{ + __u32 mask = fsn_mark->mask & INOTIFY_USER_MASK; + + if (fsn_mark->flags & FSNOTIFY_MARK_FLAG_EXCL_UNLINK) + mask |= IN_EXCL_UNLINK; + if (fsn_mark->flags & FSNOTIFY_MARK_FLAG_IN_ONESHOT) + mask |= IN_ONESHOT; + + return mask; +} + +extern void inotify_ignored_and_remove_idr(struct fsnotify_mark *fsn_mark, + struct fsnotify_group *group); +extern int inotify_handle_inode_event(struct fsnotify_mark *inode_mark, + u32 mask, struct inode *inode, + struct inode *dir, + const struct qstr *name, u32 cookie); + +extern const struct fsnotify_ops inotify_fsnotify_ops; +extern struct kmem_cache *inotify_inode_mark_cachep; + +#ifdef CONFIG_INOTIFY_USER +static inline void dec_inotify_instances(struct ucounts *ucounts) +{ + dec_ucount(ucounts, UCOUNT_INOTIFY_INSTANCES); +} + +static inline struct ucounts *inc_inotify_watches(struct ucounts *ucounts) +{ + return inc_ucount(ucounts->ns, ucounts->uid, UCOUNT_INOTIFY_WATCHES); +} + +static inline void dec_inotify_watches(struct ucounts *ucounts) +{ + dec_ucount(ucounts, UCOUNT_INOTIFY_WATCHES); +} +#endif diff --git a/fs/notify/inotify/inotify_fsnotify.c b/fs/notify/inotify/inotify_fsnotify.c new file mode 100644 index 000000000..7c326ec2e --- /dev/null +++ b/fs/notify/inotify/inotify_fsnotify.c @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * fs/inotify_user.c - inotify support for userspace + * + * Authors: + * John McCutchan + * Robert Love + * + * Copyright (C) 2005 John McCutchan + * Copyright 2006 Hewlett-Packard Development Company, L.P. + * + * Copyright (C) 2009 Eric Paris + * inotify was largely rewritten to make use of the fsnotify infrastructure + */ + +#include /* d_unlinked */ +#include /* struct inode */ +#include +#include +#include /* struct path */ +#include /* kmem_* */ +#include +#include +#include +#include + +#include "inotify.h" + +/* + * Check if 2 events contain the same information. + */ +static bool event_compare(struct fsnotify_event *old_fsn, + struct fsnotify_event *new_fsn) +{ + struct inotify_event_info *old, *new; + + old = INOTIFY_E(old_fsn); + new = INOTIFY_E(new_fsn); + if (old->mask & FS_IN_IGNORED) + return false; + if ((old->mask == new->mask) && + (old->wd == new->wd) && + (old->name_len == new->name_len) && + (!old->name_len || !strcmp(old->name, new->name))) + return true; + return false; +} + +static int inotify_merge(struct fsnotify_group *group, + struct fsnotify_event *event) +{ + struct list_head *list = &group->notification_list; + struct fsnotify_event *last_event; + + last_event = list_entry(list->prev, struct fsnotify_event, list); + return event_compare(last_event, event); +} + +int inotify_handle_inode_event(struct fsnotify_mark *inode_mark, u32 mask, + struct inode *inode, struct inode *dir, + const struct qstr *name, u32 cookie) +{ + struct inotify_inode_mark *i_mark; + struct inotify_event_info *event; + struct fsnotify_event *fsn_event; + struct fsnotify_group *group = inode_mark->group; + int ret; + int len = 0, wd; + int alloc_len = sizeof(struct inotify_event_info); + struct mem_cgroup *old_memcg; + + if (name) { + len = name->len; + alloc_len += len + 1; + } + + pr_debug("%s: group=%p mark=%p mask=%x\n", __func__, group, inode_mark, + mask); + + i_mark = container_of(inode_mark, struct inotify_inode_mark, + fsn_mark); + + /* + * We can be racing with mark being detached. Don't report event with + * invalid wd. + */ + wd = READ_ONCE(i_mark->wd); + if (wd == -1) + return 0; + /* + * Whoever is interested in the event, pays for the allocation. Do not + * trigger OOM killer in the target monitoring memcg as it may have + * security repercussion. + */ + old_memcg = set_active_memcg(group->memcg); + event = kmalloc(alloc_len, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL); + set_active_memcg(old_memcg); + + if (unlikely(!event)) { + /* + * Treat lost event due to ENOMEM the same way as queue + * overflow to let userspace know event was lost. + */ + fsnotify_queue_overflow(group); + return -ENOMEM; + } + + /* + * We now report FS_ISDIR flag with MOVE_SELF and DELETE_SELF events + * for fanotify. inotify never reported IN_ISDIR with those events. + * It looks like an oversight, but to avoid the risk of breaking + * existing inotify programs, mask the flag out from those events. + */ + if (mask & (IN_MOVE_SELF | IN_DELETE_SELF)) + mask &= ~IN_ISDIR; + + fsn_event = &event->fse; + fsnotify_init_event(fsn_event); + event->mask = mask; + event->wd = wd; + event->sync_cookie = cookie; + event->name_len = len; + if (len) + strscpy(event->name, name->name, event->name_len + 1); + + ret = fsnotify_add_event(group, fsn_event, inotify_merge); + if (ret) { + /* Our event wasn't used in the end. Free it. */ + fsnotify_destroy_event(group, fsn_event); + } + + if (inode_mark->flags & FSNOTIFY_MARK_FLAG_IN_ONESHOT) + fsnotify_destroy_mark(inode_mark, group); + + return 0; +} + +static void inotify_freeing_mark(struct fsnotify_mark *fsn_mark, struct fsnotify_group *group) +{ + inotify_ignored_and_remove_idr(fsn_mark, group); +} + +/* + * This is NEVER supposed to be called. Inotify marks should either have been + * removed from the idr when the watch was removed or in the + * fsnotify_destroy_mark_by_group() call when the inotify instance was being + * torn down. This is only called if the idr is about to be freed but there + * are still marks in it. + */ +static int idr_callback(int id, void *p, void *data) +{ + struct fsnotify_mark *fsn_mark; + struct inotify_inode_mark *i_mark; + static bool warned = false; + + if (warned) + return 0; + + warned = true; + fsn_mark = p; + i_mark = container_of(fsn_mark, struct inotify_inode_mark, fsn_mark); + + WARN(1, "inotify closing but id=%d for fsn_mark=%p in group=%p still in " + "idr. Probably leaking memory\n", id, p, data); + + /* + * I'm taking the liberty of assuming that the mark in question is a + * valid address and I'm dereferencing it. This might help to figure + * out why we got here and the panic is no worse than the original + * BUG() that was here. + */ + if (fsn_mark) + printk(KERN_WARNING "fsn_mark->group=%p wd=%d\n", + fsn_mark->group, i_mark->wd); + return 0; +} + +static void inotify_free_group_priv(struct fsnotify_group *group) +{ + /* ideally the idr is empty and we won't hit the BUG in the callback */ + idr_for_each(&group->inotify_data.idr, idr_callback, group); + idr_destroy(&group->inotify_data.idr); + if (group->inotify_data.ucounts) + dec_inotify_instances(group->inotify_data.ucounts); +} + +static void inotify_free_event(struct fsnotify_group *group, + struct fsnotify_event *fsn_event) +{ + kfree(INOTIFY_E(fsn_event)); +} + +/* ding dong the mark is dead */ +static void inotify_free_mark(struct fsnotify_mark *fsn_mark) +{ + struct inotify_inode_mark *i_mark; + + i_mark = container_of(fsn_mark, struct inotify_inode_mark, fsn_mark); + + kmem_cache_free(inotify_inode_mark_cachep, i_mark); +} + +const struct fsnotify_ops inotify_fsnotify_ops = { + .handle_inode_event = inotify_handle_inode_event, + .free_group_priv = inotify_free_group_priv, + .free_event = inotify_free_event, + .freeing_mark = inotify_freeing_mark, + .free_mark = inotify_free_mark, +}; diff --git a/fs/notify/inotify/inotify_user.c b/fs/notify/inotify/inotify_user.c new file mode 100644 index 000000000..5f19c24ec --- /dev/null +++ b/fs/notify/inotify/inotify_user.c @@ -0,0 +1,848 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * fs/inotify_user.c - inotify support for userspace + * + * Authors: + * John McCutchan + * Robert Love + * + * Copyright (C) 2005 John McCutchan + * Copyright 2006 Hewlett-Packard Development Company, L.P. + * + * Copyright (C) 2009 Eric Paris + * inotify was largely rewriten to make use of the fsnotify infrastructure + */ + +#include +#include /* struct inode */ +#include +#include +#include /* fs_initcall */ +#include +#include /* roundup() */ +#include /* LOOKUP_FOLLOW */ +#include +#include /* struct kmem_cache */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "inotify.h" +#include "../fdinfo.h" + +#include + +/* + * An inotify watch requires allocating an inotify_inode_mark structure as + * well as pinning the watched inode. Doubling the size of a VFS inode + * should be more than enough to cover the additional filesystem inode + * size increase. + */ +#define INOTIFY_WATCH_COST (sizeof(struct inotify_inode_mark) + \ + 2 * sizeof(struct inode)) + +/* configurable via /proc/sys/fs/inotify/ */ +static int inotify_max_queued_events __read_mostly; + +struct kmem_cache *inotify_inode_mark_cachep __ro_after_init; + +#ifdef CONFIG_SYSCTL + +#include + +static long it_zero = 0; +static long it_int_max = INT_MAX; + +static const struct ctl_table inotify_table[] = { + { + .procname = "max_user_instances", + .data = &init_user_ns.ucount_max[UCOUNT_INOTIFY_INSTANCES], + .maxlen = sizeof(long), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + .extra1 = &it_zero, + .extra2 = &it_int_max, + }, + { + .procname = "max_user_watches", + .data = &init_user_ns.ucount_max[UCOUNT_INOTIFY_WATCHES], + .maxlen = sizeof(long), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + .extra1 = &it_zero, + .extra2 = &it_int_max, + }, + { + .procname = "max_queued_events", + .data = &inotify_max_queued_events, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO + }, +}; + +static void __init inotify_sysctls_init(void) +{ + register_sysctl("fs/inotify", inotify_table); +} + +#else +#define inotify_sysctls_init() do { } while (0) +#endif /* CONFIG_SYSCTL */ + +static inline __u32 inotify_arg_to_mask(struct inode *inode, u32 arg) +{ + __u32 mask; + + /* + * Everything should receive events when the inode is unmounted. + * All directories care about children. + */ + mask = (FS_UNMOUNT); + if (S_ISDIR(inode->i_mode)) + mask |= FS_EVENT_ON_CHILD; + + /* mask off the flags used to open the fd */ + mask |= (arg & INOTIFY_USER_MASK); + + return mask; +} + +#define INOTIFY_MARK_FLAGS \ + (FSNOTIFY_MARK_FLAG_EXCL_UNLINK | FSNOTIFY_MARK_FLAG_IN_ONESHOT) + +static inline unsigned int inotify_arg_to_flags(u32 arg) +{ + unsigned int flags = 0; + + if (arg & IN_EXCL_UNLINK) + flags |= FSNOTIFY_MARK_FLAG_EXCL_UNLINK; + if (arg & IN_ONESHOT) + flags |= FSNOTIFY_MARK_FLAG_IN_ONESHOT; + + return flags; +} + +static inline u32 inotify_mask_to_arg(__u32 mask) +{ + return mask & (IN_ALL_EVENTS | IN_ISDIR | IN_UNMOUNT | IN_IGNORED | + IN_Q_OVERFLOW); +} + +/* inotify userspace file descriptor functions */ +static __poll_t inotify_poll(struct file *file, poll_table *wait) +{ + struct fsnotify_group *group = file->private_data; + __poll_t ret = 0; + + poll_wait(file, &group->notification_waitq, wait); + spin_lock(&group->notification_lock); + if (!fsnotify_notify_queue_is_empty(group)) + ret = EPOLLIN | EPOLLRDNORM; + spin_unlock(&group->notification_lock); + + return ret; +} + +static int round_event_name_len(struct fsnotify_event *fsn_event) +{ + struct inotify_event_info *event; + + event = INOTIFY_E(fsn_event); + if (!event->name_len) + return 0; + return roundup(event->name_len + 1, sizeof(struct inotify_event)); +} + +/* + * Get an inotify_kernel_event if one exists and is small + * enough to fit in "count". Return an error pointer if + * not large enough. + * + * Called with the group->notification_lock held. + */ +static struct fsnotify_event *get_one_event(struct fsnotify_group *group, + size_t count) +{ + size_t event_size = sizeof(struct inotify_event); + struct fsnotify_event *event; + + event = fsnotify_peek_first_event(group); + if (!event) + return NULL; + + pr_debug("%s: group=%p event=%p\n", __func__, group, event); + + event_size += round_event_name_len(event); + if (event_size > count) + return ERR_PTR(-EINVAL); + + /* held the notification_lock the whole time, so this is the + * same event we peeked above */ + fsnotify_remove_first_event(group); + + return event; +} + +/* + * Copy an event to user space, returning how much we copied. + * + * We already checked that the event size is smaller than the + * buffer we had in "get_one_event()" above. + */ +static ssize_t copy_event_to_user(struct fsnotify_group *group, + struct fsnotify_event *fsn_event, + char __user *buf) +{ + struct inotify_event inotify_event; + struct inotify_event_info *event; + size_t event_size = sizeof(struct inotify_event); + size_t name_len; + size_t pad_name_len; + + pr_debug("%s: group=%p event=%p\n", __func__, group, fsn_event); + + event = INOTIFY_E(fsn_event); + name_len = event->name_len; + /* + * round up name length so it is a multiple of event_size + * plus an extra byte for the terminating '\0'. + */ + pad_name_len = round_event_name_len(fsn_event); + inotify_event.len = pad_name_len; + inotify_event.mask = inotify_mask_to_arg(event->mask); + inotify_event.wd = event->wd; + inotify_event.cookie = event->sync_cookie; + + /* send the main event */ + if (copy_to_user(buf, &inotify_event, event_size)) + return -EFAULT; + + buf += event_size; + + /* + * fsnotify only stores the pathname, so here we have to send the pathname + * and then pad that pathname out to a multiple of sizeof(inotify_event) + * with zeros. + */ + if (pad_name_len) { + /* copy the path name */ + if (copy_to_user(buf, event->name, name_len)) + return -EFAULT; + buf += name_len; + + /* fill userspace with 0's */ + if (clear_user(buf, pad_name_len - name_len)) + return -EFAULT; + event_size += pad_name_len; + } + + return event_size; +} + +static ssize_t inotify_read(struct file *file, char __user *buf, + size_t count, loff_t *pos) +{ + struct fsnotify_group *group; + struct fsnotify_event *kevent; + char __user *start; + int ret; + DEFINE_WAIT_FUNC(wait, woken_wake_function); + + start = buf; + group = file->private_data; + + add_wait_queue(&group->notification_waitq, &wait); + while (1) { + spin_lock(&group->notification_lock); + kevent = get_one_event(group, count); + spin_unlock(&group->notification_lock); + + pr_debug("%s: group=%p kevent=%p\n", __func__, group, kevent); + + if (kevent) { + ret = PTR_ERR(kevent); + if (IS_ERR(kevent)) + break; + ret = copy_event_to_user(group, kevent, buf); + fsnotify_destroy_event(group, kevent); + if (ret < 0) + break; + buf += ret; + count -= ret; + continue; + } + + ret = -EAGAIN; + if (file->f_flags & O_NONBLOCK) + break; + ret = -ERESTARTSYS; + if (signal_pending(current)) + break; + + if (start != buf) + break; + + wait_woken(&wait, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); + } + remove_wait_queue(&group->notification_waitq, &wait); + + if (start != buf && ret != -EFAULT) + ret = buf - start; + return ret; +} + +static int inotify_release(struct inode *ignored, struct file *file) +{ + struct fsnotify_group *group = file->private_data; + + pr_debug("%s: group=%p\n", __func__, group); + + /* free this group, matching get was inotify_init->fsnotify_obtain_group */ + fsnotify_destroy_group(group); + + return 0; +} + +static long inotify_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct fsnotify_group *group; + struct fsnotify_event *fsn_event; + void __user *p; + int ret = -ENOTTY; + size_t send_len = 0; + + group = file->private_data; + p = (void __user *) arg; + + pr_debug("%s: group=%p cmd=%u\n", __func__, group, cmd); + + switch (cmd) { + case FIONREAD: + spin_lock(&group->notification_lock); + list_for_each_entry(fsn_event, &group->notification_list, + list) { + send_len += sizeof(struct inotify_event); + send_len += round_event_name_len(fsn_event); + } + spin_unlock(&group->notification_lock); + ret = put_user(send_len, (int __user *) p); + break; +#ifdef CONFIG_CHECKPOINT_RESTORE + case INOTIFY_IOC_SETNEXTWD: + ret = -EINVAL; + if (arg >= 1 && arg <= INT_MAX) { + struct inotify_group_private_data *data; + + data = &group->inotify_data; + spin_lock(&data->idr_lock); + idr_set_cursor(&data->idr, (unsigned int)arg); + spin_unlock(&data->idr_lock); + ret = 0; + } + break; +#endif /* CONFIG_CHECKPOINT_RESTORE */ + } + + return ret; +} + +static const struct file_operations inotify_fops = { + .show_fdinfo = inotify_show_fdinfo, + .poll = inotify_poll, + .read = inotify_read, + .fasync = fsnotify_fasync, + .release = inotify_release, + .unlocked_ioctl = inotify_ioctl, + .compat_ioctl = inotify_ioctl, + .llseek = noop_llseek, +}; + + +/* + * find_inode - resolve a user-given path to a specific inode + */ +static int inotify_find_inode(const char __user *dirname, struct path *path, + unsigned int flags, __u64 mask) +{ + int error; + + error = user_path_at(AT_FDCWD, dirname, flags, path); + if (error) + return error; + /* you can only watch an inode if you have read permissions on it */ + error = path_permission(path, MAY_READ); + if (error) { + path_put(path); + return error; + } + error = security_path_notify(path, mask, + FSNOTIFY_OBJ_TYPE_INODE); + if (error) + path_put(path); + + return error; +} + +static int inotify_add_to_idr(struct idr *idr, spinlock_t *idr_lock, + struct inotify_inode_mark *i_mark) +{ + int ret; + + idr_preload(GFP_KERNEL); + spin_lock(idr_lock); + + ret = idr_alloc_cyclic(idr, i_mark, 1, 0, GFP_NOWAIT); + if (ret >= 0) { + /* we added the mark to the idr, take a reference */ + i_mark->wd = ret; + fsnotify_get_mark(&i_mark->fsn_mark); + } + + spin_unlock(idr_lock); + idr_preload_end(); + return ret < 0 ? ret : 0; +} + +static struct inotify_inode_mark *inotify_idr_find_locked(struct fsnotify_group *group, + int wd) +{ + struct idr *idr = &group->inotify_data.idr; + spinlock_t *idr_lock = &group->inotify_data.idr_lock; + struct inotify_inode_mark *i_mark; + + assert_spin_locked(idr_lock); + + i_mark = idr_find(idr, wd); + if (i_mark) { + struct fsnotify_mark *fsn_mark = &i_mark->fsn_mark; + + fsnotify_get_mark(fsn_mark); + /* One ref for being in the idr, one ref we just took */ + BUG_ON(refcount_read(&fsn_mark->refcnt) < 2); + } + + return i_mark; +} + +static struct inotify_inode_mark *inotify_idr_find(struct fsnotify_group *group, + int wd) +{ + struct inotify_inode_mark *i_mark; + spinlock_t *idr_lock = &group->inotify_data.idr_lock; + + spin_lock(idr_lock); + i_mark = inotify_idr_find_locked(group, wd); + spin_unlock(idr_lock); + + return i_mark; +} + +/* + * Remove the mark from the idr (if present) and drop the reference + * on the mark because it was in the idr. + */ +static void inotify_remove_from_idr(struct fsnotify_group *group, + struct inotify_inode_mark *i_mark) +{ + struct idr *idr = &group->inotify_data.idr; + spinlock_t *idr_lock = &group->inotify_data.idr_lock; + struct inotify_inode_mark *found_i_mark = NULL; + int wd; + + spin_lock(idr_lock); + wd = i_mark->wd; + + /* + * does this i_mark think it is in the idr? we shouldn't get called + * if it wasn't.... + */ + if (wd == -1) { + WARN_ONCE(1, "%s: i_mark=%p i_mark->wd=%d i_mark->group=%p\n", + __func__, i_mark, i_mark->wd, i_mark->fsn_mark.group); + goto out; + } + + /* Lets look in the idr to see if we find it */ + found_i_mark = inotify_idr_find_locked(group, wd); + if (unlikely(!found_i_mark)) { + WARN_ONCE(1, "%s: i_mark=%p i_mark->wd=%d i_mark->group=%p\n", + __func__, i_mark, i_mark->wd, i_mark->fsn_mark.group); + goto out; + } + + /* + * We found an mark in the idr at the right wd, but it's + * not the mark we were told to remove. eparis seriously + * fucked up somewhere. + */ + if (unlikely(found_i_mark != i_mark)) { + WARN_ONCE(1, "%s: i_mark=%p i_mark->wd=%d i_mark->group=%p " + "found_i_mark=%p found_i_mark->wd=%d " + "found_i_mark->group=%p\n", __func__, i_mark, + i_mark->wd, i_mark->fsn_mark.group, found_i_mark, + found_i_mark->wd, found_i_mark->fsn_mark.group); + goto out; + } + + /* + * One ref for being in the idr + * one ref grabbed by inotify_idr_find + */ + if (unlikely(refcount_read(&i_mark->fsn_mark.refcnt) < 2)) { + printk(KERN_ERR "%s: i_mark=%p i_mark->wd=%d i_mark->group=%p\n", + __func__, i_mark, i_mark->wd, i_mark->fsn_mark.group); + /* we can't really recover with bad ref cnting.. */ + BUG(); + } + + idr_remove(idr, wd); + /* Removed from the idr, drop that ref. */ + fsnotify_put_mark(&i_mark->fsn_mark); +out: + i_mark->wd = -1; + spin_unlock(idr_lock); + /* match the ref taken by inotify_idr_find_locked() */ + if (found_i_mark) + fsnotify_put_mark(&found_i_mark->fsn_mark); +} + +/* + * Send IN_IGNORED for this wd, remove this wd from the idr. + */ +void inotify_ignored_and_remove_idr(struct fsnotify_mark *fsn_mark, + struct fsnotify_group *group) +{ + struct inotify_inode_mark *i_mark; + + /* Queue ignore event for the watch */ + inotify_handle_inode_event(fsn_mark, FS_IN_IGNORED, NULL, NULL, NULL, + 0); + + i_mark = container_of(fsn_mark, struct inotify_inode_mark, fsn_mark); + /* remove this mark from the idr */ + inotify_remove_from_idr(group, i_mark); + + dec_inotify_watches(group->inotify_data.ucounts); +} + +static int inotify_update_existing_watch(struct fsnotify_group *group, + struct inode *inode, + u32 arg) +{ + struct fsnotify_mark *fsn_mark; + struct inotify_inode_mark *i_mark; + int replace = !(arg & IN_MASK_ADD); + int create = (arg & IN_MASK_CREATE); + int ret; + + fsn_mark = fsnotify_find_inode_mark(inode, group); + if (!fsn_mark) + return -ENOENT; + else if (create) { + ret = -EEXIST; + goto out; + } + + i_mark = container_of(fsn_mark, struct inotify_inode_mark, fsn_mark); + + spin_lock(&fsn_mark->lock); + if (replace) { + fsn_mark->mask = 0; + fsn_mark->flags &= ~INOTIFY_MARK_FLAGS; + } + fsn_mark->mask |= inotify_arg_to_mask(inode, arg); + fsn_mark->flags |= inotify_arg_to_flags(arg); + spin_unlock(&fsn_mark->lock); + + fsnotify_recalc_mask(fsn_mark->connector); + + /* return the wd */ + ret = i_mark->wd; + +out: + /* match the get from fsnotify_find_mark() */ + fsnotify_put_mark(fsn_mark); + + return ret; +} + +static int inotify_new_watch(struct fsnotify_group *group, + struct inode *inode, + u32 arg) +{ + struct inotify_inode_mark *tmp_i_mark; + int ret; + struct idr *idr = &group->inotify_data.idr; + spinlock_t *idr_lock = &group->inotify_data.idr_lock; + + tmp_i_mark = kmem_cache_alloc(inotify_inode_mark_cachep, GFP_KERNEL); + if (unlikely(!tmp_i_mark)) + return -ENOMEM; + + fsnotify_init_mark(&tmp_i_mark->fsn_mark, group); + tmp_i_mark->fsn_mark.mask = inotify_arg_to_mask(inode, arg); + tmp_i_mark->fsn_mark.flags = inotify_arg_to_flags(arg); + tmp_i_mark->wd = -1; + + ret = inotify_add_to_idr(idr, idr_lock, tmp_i_mark); + if (ret) + goto out_err; + + /* increment the number of watches the user has */ + if (!inc_inotify_watches(group->inotify_data.ucounts)) { + inotify_remove_from_idr(group, tmp_i_mark); + ret = -ENOSPC; + goto out_err; + } + + /* we are on the idr, now get on the inode */ + ret = fsnotify_add_inode_mark_locked(&tmp_i_mark->fsn_mark, inode, 0); + if (ret) { + /* we failed to get on the inode, get off the idr */ + inotify_remove_from_idr(group, tmp_i_mark); + dec_inotify_watches(group->inotify_data.ucounts); + goto out_err; + } + + + /* return the watch descriptor for this new mark */ + ret = tmp_i_mark->wd; + +out_err: + /* match the ref from fsnotify_init_mark() */ + fsnotify_put_mark(&tmp_i_mark->fsn_mark); + + return ret; +} + +static int inotify_update_watch(struct fsnotify_group *group, struct inode *inode, u32 arg) +{ + int ret = 0; + + fsnotify_group_lock(group); + /* try to update and existing watch with the new arg */ + ret = inotify_update_existing_watch(group, inode, arg); + /* no mark present, try to add a new one */ + if (ret == -ENOENT) + ret = inotify_new_watch(group, inode, arg); + fsnotify_group_unlock(group); + + return ret; +} + +static struct fsnotify_group *inotify_new_group(unsigned int max_events) +{ + struct fsnotify_group *group; + struct inotify_event_info *oevent; + + group = fsnotify_alloc_group(&inotify_fsnotify_ops, + FSNOTIFY_GROUP_USER); + if (IS_ERR(group)) + return group; + + oevent = kmalloc_obj(struct inotify_event_info, GFP_KERNEL_ACCOUNT); + if (unlikely(!oevent)) { + fsnotify_destroy_group(group); + return ERR_PTR(-ENOMEM); + } + group->overflow_event = &oevent->fse; + fsnotify_init_event(group->overflow_event); + oevent->mask = FS_Q_OVERFLOW; + oevent->wd = -1; + oevent->sync_cookie = 0; + oevent->name_len = 0; + + group->max_events = max_events; + group->memcg = get_mem_cgroup_from_mm(current->mm); + + spin_lock_init(&group->inotify_data.idr_lock); + idr_init(&group->inotify_data.idr); + group->inotify_data.ucounts = inc_ucount(current_user_ns(), + current_euid(), + UCOUNT_INOTIFY_INSTANCES); + + if (!group->inotify_data.ucounts) { + fsnotify_destroy_group(group); + return ERR_PTR(-EMFILE); + } + + return group; +} + + +/* inotify syscalls */ +static int do_inotify_init(int flags) +{ + struct fsnotify_group *group; + int ret; + + /* Check the IN_* constants for consistency. */ + BUILD_BUG_ON(IN_CLOEXEC != O_CLOEXEC); + BUILD_BUG_ON(IN_NONBLOCK != O_NONBLOCK); + + if (flags & ~(IN_CLOEXEC | IN_NONBLOCK)) + return -EINVAL; + + /* fsnotify_obtain_group took a reference to group, we put this when we kill the file in the end */ + group = inotify_new_group(inotify_max_queued_events); + if (IS_ERR(group)) + return PTR_ERR(group); + + ret = anon_inode_getfd("inotify", &inotify_fops, group, + O_RDONLY | flags); + if (ret < 0) + fsnotify_destroy_group(group); + + return ret; +} + +SYSCALL_DEFINE1(inotify_init1, int, flags) +{ + return do_inotify_init(flags); +} + +SYSCALL_DEFINE0(inotify_init) +{ + return do_inotify_init(0); +} + +SYSCALL_DEFINE3(inotify_add_watch, int, fd, const char __user *, pathname, + u32, mask) +{ + struct fsnotify_group *group; + struct inode *inode; + struct path path; + int ret; + unsigned flags = 0; + + /* + * We share a lot of code with fs/dnotify. We also share + * the bit layout between inotify's IN_* and the fsnotify + * FS_*. This check ensures that only the inotify IN_* + * bits get passed in and set in watches/events. + */ + if (unlikely(mask & ~ALL_INOTIFY_BITS)) + return -EINVAL; + /* + * Require at least one valid bit set in the mask. + * Without _something_ set, we would have no events to + * watch for. + */ + if (unlikely(!(mask & ALL_INOTIFY_BITS))) + return -EINVAL; + + CLASS(fd, f)(fd); + if (fd_empty(f)) + return -EBADF; + + /* IN_MASK_ADD and IN_MASK_CREATE don't make sense together */ + if (unlikely((mask & IN_MASK_ADD) && (mask & IN_MASK_CREATE))) + return -EINVAL; + + /* verify that this is indeed an inotify instance */ + if (unlikely(fd_file(f)->f_op != &inotify_fops)) + return -EINVAL; + + if (!(mask & IN_DONT_FOLLOW)) + flags |= LOOKUP_FOLLOW; + if (mask & IN_ONLYDIR) + flags |= LOOKUP_DIRECTORY; + + ret = inotify_find_inode(pathname, &path, flags, + (mask & IN_ALL_EVENTS)); + if (ret) + return ret; + + /* inode held in place by reference to path; group by fget on fd */ + inode = path.dentry->d_inode; + group = fd_file(f)->private_data; + + /* create/update an inode mark */ + ret = inotify_update_watch(group, inode, mask); + path_put(&path); + return ret; +} + +SYSCALL_DEFINE2(inotify_rm_watch, int, fd, __s32, wd) +{ + struct fsnotify_group *group; + struct inotify_inode_mark *i_mark; + CLASS(fd, f)(fd); + + if (fd_empty(f)) + return -EBADF; + + /* verify that this is indeed an inotify instance */ + if (unlikely(fd_file(f)->f_op != &inotify_fops)) + return -EINVAL; + + group = fd_file(f)->private_data; + + i_mark = inotify_idr_find(group, wd); + if (unlikely(!i_mark)) + return -EINVAL; + + fsnotify_destroy_mark(&i_mark->fsn_mark, group); + + /* match ref taken by inotify_idr_find */ + fsnotify_put_mark(&i_mark->fsn_mark); + return 0; +} + +/* + * inotify_user_setup - Our initialization function. Note that we cannot return + * error because we have compiled-in VFS hooks. So an (unlikely) failure here + * must result in panic(). + */ +static int __init inotify_user_setup(void) +{ + unsigned long watches_max; + struct sysinfo si; + + si_meminfo(&si); + /* + * Allow up to 1% of addressable memory to be allocated for inotify + * watches (per user) limited to the range [8192, 1048576]. + */ + watches_max = (((si.totalram - si.totalhigh) / 100) << PAGE_SHIFT) / + INOTIFY_WATCH_COST; + watches_max = clamp(watches_max, 8192UL, 1048576UL); + + BUILD_BUG_ON(IN_ACCESS != FS_ACCESS); + BUILD_BUG_ON(IN_MODIFY != FS_MODIFY); + BUILD_BUG_ON(IN_ATTRIB != FS_ATTRIB); + BUILD_BUG_ON(IN_CLOSE_WRITE != FS_CLOSE_WRITE); + BUILD_BUG_ON(IN_CLOSE_NOWRITE != FS_CLOSE_NOWRITE); + BUILD_BUG_ON(IN_OPEN != FS_OPEN); + BUILD_BUG_ON(IN_MOVED_FROM != FS_MOVED_FROM); + BUILD_BUG_ON(IN_MOVED_TO != FS_MOVED_TO); + BUILD_BUG_ON(IN_CREATE != FS_CREATE); + BUILD_BUG_ON(IN_DELETE != FS_DELETE); + BUILD_BUG_ON(IN_DELETE_SELF != FS_DELETE_SELF); + BUILD_BUG_ON(IN_MOVE_SELF != FS_MOVE_SELF); + BUILD_BUG_ON(IN_UNMOUNT != FS_UNMOUNT); + BUILD_BUG_ON(IN_Q_OVERFLOW != FS_Q_OVERFLOW); + BUILD_BUG_ON(IN_IGNORED != FS_IN_IGNORED); + BUILD_BUG_ON(IN_ISDIR != FS_ISDIR); + + BUILD_BUG_ON(HWEIGHT32(ALL_INOTIFY_BITS) != 22); + + inotify_inode_mark_cachep = KMEM_CACHE(inotify_inode_mark, + SLAB_PANIC|SLAB_ACCOUNT); + + inotify_max_queued_events = 16384; + init_user_ns.ucount_max[UCOUNT_INOTIFY_INSTANCES] = 128; + init_user_ns.ucount_max[UCOUNT_INOTIFY_WATCHES] = watches_max; + inotify_sysctls_init(); + + return 0; +} +fs_initcall(inotify_user_setup); diff --git a/fs/notify/mark.c b/fs/notify/mark.c new file mode 100644 index 000000000..b2640d836 --- /dev/null +++ b/fs/notify/mark.c @@ -0,0 +1,1194 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2008 Red Hat, Inc., Eric Paris + */ + +/* + * fsnotify inode mark locking/lifetime/and refcnting + * + * REFCNT: + * The group->recnt and mark->refcnt tell how many "things" in the kernel + * currently are referencing the objects. Both kind of objects typically will + * live inside the kernel with a refcnt of 2, one for its creation and one for + * the reference a group and a mark hold to each other. + * If you are holding the appropriate locks, you can take a reference and the + * object itself is guaranteed to survive until the reference is dropped. + * + * LOCKING: + * There are 3 locks involved with fsnotify inode marks and they MUST be taken + * in order as follows: + * + * group->mark_mutex + * mark->lock + * mark->connector->lock + * + * group->mark_mutex protects the marks_list anchored inside a given group and + * each mark is hooked via the g_list. It also protects the groups private + * data (i.e group limits). + + * mark->lock protects the marks attributes like its masks and flags. + * Furthermore it protects the access to a reference of the group that the mark + * is assigned to as well as the access to a reference of the inode/vfsmount + * that is being watched by the mark. + * + * mark->connector->lock protects the list of marks anchored inside an + * inode / vfsmount and each mark is hooked via the i_list. + * + * A list of notification marks relating to inode / mnt is contained in + * fsnotify_mark_connector. That structure is alive as long as there are any + * marks in the list and is also protected by fsnotify_mark_srcu. A mark gets + * detached from fsnotify_mark_connector when last reference to the mark is + * dropped. Thus having mark reference is enough to protect mark->connector + * pointer and to make sure fsnotify_mark_connector cannot disappear. Also + * because we remove mark from g_list before dropping mark reference associated + * with that, any mark found through g_list is guaranteed to have + * mark->connector set until we drop group->mark_mutex. + * + * LIFETIME: + * Inode marks survive between when they are added to an inode and when their + * refcnt==0. Marks are also protected by fsnotify_mark_srcu. + * + * The inode mark can be cleared for a number of different reasons including: + * - The inode is unlinked for the last time. (fsnotify_inode_remove) + * - The inode is being evicted from cache. (fsnotify_inode_delete) + * - The fs the inode is on is unmounted. (fsnotify_inode_delete/fsnotify_unmount_inodes) + * - Something explicitly requests that it be removed. (fsnotify_destroy_mark) + * - The fsnotify_group associated with the mark is going away and all such marks + * need to be cleaned up. (fsnotify_clear_marks_by_group) + * + * This has the very interesting property of being able to run concurrently with + * any (or all) other directions. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include "fsnotify.h" + +#define FSNOTIFY_REAPER_DELAY (1) /* 1 jiffy */ + +struct srcu_struct fsnotify_mark_srcu; +static struct kmem_cache *fsnotify_mark_connector_cachep; +static struct kmem_cache *fsnotify_inode_mark_connector_cachep; + +static DEFINE_SPINLOCK(destroy_lock); +static LIST_HEAD(destroy_list); +static struct fsnotify_mark_connector *connector_destroy_list; + +static void fsnotify_mark_destroy_workfn(struct work_struct *work); +static DECLARE_DELAYED_WORK(reaper_work, fsnotify_mark_destroy_workfn); + +static void fsnotify_connector_destroy_workfn(struct work_struct *work); +static DECLARE_WORK(connector_reaper_work, fsnotify_connector_destroy_workfn); + +void fsnotify_get_mark(struct fsnotify_mark *mark) +{ + WARN_ON_ONCE(!refcount_read(&mark->refcnt)); + refcount_inc(&mark->refcnt); +} + +static fsnotify_connp_t *fsnotify_object_connp(void *obj, + enum fsnotify_obj_type obj_type) +{ + switch (obj_type) { + case FSNOTIFY_OBJ_TYPE_INODE: + return &((struct inode *)obj)->i_fsnotify_marks; + case FSNOTIFY_OBJ_TYPE_VFSMOUNT: + return &real_mount(obj)->mnt_fsnotify_marks; + case FSNOTIFY_OBJ_TYPE_SB: + return fsnotify_sb_marks(obj); + case FSNOTIFY_OBJ_TYPE_MNTNS: + return &((struct mnt_namespace *)obj)->n_fsnotify_marks; + default: + return NULL; + } +} + +static __u32 *fsnotify_conn_mask_p(struct fsnotify_mark_connector *conn) +{ + if (conn->type == FSNOTIFY_OBJ_TYPE_INODE) + return &fsnotify_conn_inode(conn)->i_fsnotify_mask; + else if (conn->type == FSNOTIFY_OBJ_TYPE_VFSMOUNT) + return &fsnotify_conn_mount(conn)->mnt_fsnotify_mask; + else if (conn->type == FSNOTIFY_OBJ_TYPE_SB) + return &fsnotify_conn_sb(conn)->s_fsnotify_mask; + else if (conn->type == FSNOTIFY_OBJ_TYPE_MNTNS) + return &fsnotify_conn_mntns(conn)->n_fsnotify_mask; + return NULL; +} + +__u32 fsnotify_conn_mask(struct fsnotify_mark_connector *conn) +{ + if (WARN_ON(!fsnotify_valid_obj_type(conn->type))) + return 0; + + return READ_ONCE(*fsnotify_conn_mask_p(conn)); +} + +static void fsnotify_get_sb_watched_objects(struct super_block *sb) +{ + atomic_long_inc(fsnotify_sb_watched_objects(sb)); +} + +static void fsnotify_put_sb_watched_objects(struct super_block *sb) +{ + atomic_long_t *watched_objects = fsnotify_sb_watched_objects(sb); + + /* the superblock can go away after this decrement */ + if (atomic_long_dec_and_test(watched_objects)) + wake_up_var(watched_objects); +} + +static void fsnotify_get_inode_ref(struct inode *inode) +{ + ihold(inode); + fsnotify_get_sb_watched_objects(inode->i_sb); +} + +static void fsnotify_put_inode_ref(struct inode *inode) +{ + /* read ->i_sb before the inode can go away */ + struct super_block *sb = inode->i_sb; + + iput(inode); + fsnotify_put_sb_watched_objects(sb); +} + +/* + * Grab or drop watched objects reference depending on whether the connector + * is attached and has any marks attached. + */ +static void fsnotify_update_sb_watchers(struct super_block *sb, + struct fsnotify_mark_connector *conn) +{ + struct fsnotify_sb_info *sbinfo = fsnotify_sb_info(sb); + bool is_watched = conn->flags & FSNOTIFY_CONN_FLAG_IS_WATCHED; + struct fsnotify_mark *first_mark = NULL; + unsigned int highest_prio = 0; + + if (conn->obj) + first_mark = hlist_entry_safe(conn->list.first, + struct fsnotify_mark, obj_list); + if (first_mark) + highest_prio = first_mark->group->priority; + if (WARN_ON(highest_prio >= __FSNOTIFY_PRIO_NUM)) + highest_prio = 0; + + /* + * If the highest priority of group watching this object is prio, + * then watched object has a reference on counters [0..prio]. + * Update priority >= 1 watched objects counters. + */ + for (unsigned int p = conn->prio + 1; p <= highest_prio; p++) + atomic_long_inc(&sbinfo->watched_objects[p]); + for (unsigned int p = conn->prio; p > highest_prio; p--) + atomic_long_dec(&sbinfo->watched_objects[p]); + conn->prio = highest_prio; + + /* Update priority >= 0 (a.k.a total) watched objects counter */ + BUILD_BUG_ON(FSNOTIFY_PRIO_NORMAL != 0); + if (first_mark && !is_watched) { + conn->flags |= FSNOTIFY_CONN_FLAG_IS_WATCHED; + fsnotify_get_sb_watched_objects(sb); + } else if (!first_mark && is_watched) { + conn->flags &= ~FSNOTIFY_CONN_FLAG_IS_WATCHED; + fsnotify_put_sb_watched_objects(sb); + } +} + +/* + * Grab or drop inode reference for the connector if needed. + * + * When it's time to drop the reference, we only clear the HAS_IREF flag and + * return the inode object. fsnotify_drop_object() will be resonsible for doing + * iput() outside of spinlocks. This happens when last mark that wanted iref is + * detached. + */ +static struct inode *fsnotify_update_iref(struct fsnotify_mark_connector *conn, + bool want_iref) +{ + bool has_iref = conn->flags & FSNOTIFY_CONN_FLAG_HAS_IREF; + struct inode *inode = NULL; + + if (conn->type != FSNOTIFY_OBJ_TYPE_INODE || + want_iref == has_iref) + return NULL; + + if (want_iref) { + /* Pin inode if any mark wants inode refcount held */ + fsnotify_get_inode_ref(fsnotify_conn_inode(conn)); + conn->flags |= FSNOTIFY_CONN_FLAG_HAS_IREF; + } else { + /* Unpin inode after detach of last mark that wanted iref */ + inode = fsnotify_conn_inode(conn); + conn->flags &= ~FSNOTIFY_CONN_FLAG_HAS_IREF; + } + + return inode; +} + +/* + * Calculate mask of events for a list of marks. + * + * Return true if any of the attached marks want to hold an inode reference. + */ +static bool __fsnotify_recalc_mask(struct fsnotify_mark_connector *conn) +{ + u32 new_mask = 0; + bool want_iref = false; + struct fsnotify_mark *mark; + + assert_spin_locked(&conn->lock); + /* We can get detached connector here when inode is getting unlinked. */ + if (!fsnotify_valid_obj_type(conn->type)) + return NULL; + hlist_for_each_entry(mark, &conn->list, obj_list) { + if (!(mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED)) + continue; + new_mask |= fsnotify_calc_mask(mark); + if (conn->type == FSNOTIFY_OBJ_TYPE_INODE && + !(mark->flags & FSNOTIFY_MARK_FLAG_NO_IREF)) + want_iref = true; + } + /* + * We use WRITE_ONCE() to prevent silly compiler optimizations from + * confusing readers not holding conn->lock with partial updates. + */ + WRITE_ONCE(*fsnotify_conn_mask_p(conn), new_mask); + + return want_iref; +} + +/* + * Calculate mask of events for a list of marks after attach/modify mark + * and get an inode reference for the connector if needed. + * + * A concurrent add of evictable mark and detach of non-evictable mark can + * lead to __fsnotify_recalc_mask() returning false want_iref, but in this + * case we defer clearing iref to fsnotify_recalc_mask_clear_iref() called + * from fsnotify_put_mark(). + */ +static void fsnotify_recalc_mask_set_iref(struct fsnotify_mark_connector *conn) +{ + bool has_iref = conn->flags & FSNOTIFY_CONN_FLAG_HAS_IREF; + bool want_iref = __fsnotify_recalc_mask(conn) || has_iref; + + (void) fsnotify_update_iref(conn, want_iref); +} + +/* + * Calculate mask of events for a list of marks after detach mark + * and return the inode object if its reference is no longer needed. + */ +static void *fsnotify_recalc_mask_clear_iref(struct fsnotify_mark_connector *conn) +{ + bool want_iref = __fsnotify_recalc_mask(conn); + + return fsnotify_update_iref(conn, want_iref); +} + +static bool fsnotify_conn_watches_children( + struct fsnotify_mark_connector *conn) +{ + if (conn->type != FSNOTIFY_OBJ_TYPE_INODE) + return false; + + return fsnotify_inode_watches_children(fsnotify_conn_inode(conn)); +} + +static void fsnotify_conn_set_children_dentry_flags( + struct fsnotify_mark_connector *conn) +{ + if (conn->type != FSNOTIFY_OBJ_TYPE_INODE) + return; + + fsnotify_set_children_dentry_flags(fsnotify_conn_inode(conn)); +} + +/* + * Calculate mask of events for a list of marks. The caller must make sure + * connector and connector->obj cannot disappear under us. Callers achieve + * this by holding a mark->lock or mark->group->mark_mutex for a mark on this + * list. + */ +void fsnotify_recalc_mask(struct fsnotify_mark_connector *conn) +{ + bool update_children; + + if (!conn) + return; + + spin_lock(&conn->lock); + update_children = !fsnotify_conn_watches_children(conn); + fsnotify_recalc_mask_set_iref(conn); + update_children &= fsnotify_conn_watches_children(conn); + spin_unlock(&conn->lock); + /* + * Set children's PARENT_WATCHED flags only if parent started watching. + * When parent stops watching, we clear false positive PARENT_WATCHED + * flags lazily in __fsnotify_parent(). + */ + if (update_children) + fsnotify_conn_set_children_dentry_flags(conn); +} + +/** + * fsnotify_modify_mark_mask - set and/or clear flags in a mark's mask + * @mark: mark to be modified + * @set: bits to be set in mask + * @clear: bits to be cleared in mask + * + * Modify a fsnotify_mark mask as directed, and update its associated conn. + * The caller is expected to hold a reference to the mark. + */ +void fsnotify_modify_mark_mask(struct fsnotify_mark *mark, u32 set, u32 clear) +{ + bool recalc = false; + u32 mask; + + WARN_ON_ONCE(clear & set); + + spin_lock(&mark->lock); + mask = mark->mask; + mark->mask |= set; + mark->mask &= ~clear; + if (mark->mask != mask) + recalc = true; + spin_unlock(&mark->lock); + + if (recalc) + fsnotify_recalc_mask(mark->connector); +} +EXPORT_SYMBOL_GPL(fsnotify_modify_mark_mask); + +/* Free all connectors queued for freeing once SRCU period ends */ +static void fsnotify_connector_destroy_workfn(struct work_struct *work) +{ + struct fsnotify_mark_connector *conn, *free; + + spin_lock(&destroy_lock); + conn = connector_destroy_list; + connector_destroy_list = NULL; + spin_unlock(&destroy_lock); + + synchronize_srcu(&fsnotify_mark_srcu); + while (conn) { + free = conn; + conn = conn->destroy_next; + kfree(free); + } +} + +static void fsnotify_untrack_connector(struct fsnotify_mark_connector *conn); + +static void *fsnotify_detach_connector_from_object( + struct fsnotify_mark_connector *conn, + unsigned int *type) +{ + fsnotify_connp_t *connp = fsnotify_object_connp(conn->obj, conn->type); + struct super_block *sb = fsnotify_connector_sb(conn); + struct inode *inode = NULL; + + *type = conn->type; + if (conn->type == FSNOTIFY_OBJ_TYPE_DETACHED) + return NULL; + + if (conn->type == FSNOTIFY_OBJ_TYPE_INODE) { + inode = fsnotify_conn_inode(conn); + inode->i_fsnotify_mask = 0; + fsnotify_untrack_connector(conn); + + /* Unpin inode when detaching from connector */ + if (!(conn->flags & FSNOTIFY_CONN_FLAG_HAS_IREF)) + inode = NULL; + } else if (conn->type == FSNOTIFY_OBJ_TYPE_VFSMOUNT) { + fsnotify_conn_mount(conn)->mnt_fsnotify_mask = 0; + } else if (conn->type == FSNOTIFY_OBJ_TYPE_SB) { + fsnotify_conn_sb(conn)->s_fsnotify_mask = 0; + } else if (conn->type == FSNOTIFY_OBJ_TYPE_MNTNS) { + fsnotify_conn_mntns(conn)->n_fsnotify_mask = 0; + } + + rcu_assign_pointer(*connp, NULL); + conn->obj = NULL; + conn->type = FSNOTIFY_OBJ_TYPE_DETACHED; + if (sb) + fsnotify_update_sb_watchers(sb, conn); + + return inode; +} + +static void fsnotify_final_mark_destroy(struct fsnotify_mark *mark) +{ + struct fsnotify_group *group = mark->group; + + if (WARN_ON_ONCE(!group)) + return; + group->ops->free_mark(mark); + fsnotify_put_group(group); +} + +/* Drop object reference originally held by a connector */ +static void fsnotify_drop_object(unsigned int type, void *objp) +{ + if (!objp) + return; + /* Currently only inode references are passed to be dropped */ + if (WARN_ON_ONCE(type != FSNOTIFY_OBJ_TYPE_INODE)) + return; + fsnotify_put_inode_ref(objp); +} + +void fsnotify_put_mark(struct fsnotify_mark *mark) +{ + struct fsnotify_mark_connector *conn = READ_ONCE(mark->connector); + void *objp = NULL; + unsigned int type = FSNOTIFY_OBJ_TYPE_DETACHED; + bool free_conn = false; + + /* Catch marks that were actually never attached to object */ + if (!conn) { + if (refcount_dec_and_test(&mark->refcnt)) + fsnotify_final_mark_destroy(mark); + return; + } + + /* + * We have to be careful so that traversals of obj_list under lock can + * safely grab mark reference. + */ + if (!refcount_dec_and_lock(&mark->refcnt, &conn->lock)) + return; + + hlist_del_init_rcu(&mark->obj_list); + if (hlist_empty(&conn->list)) { + objp = fsnotify_detach_connector_from_object(conn, &type); + free_conn = true; + } else { + struct super_block *sb = fsnotify_connector_sb(conn); + + /* Update watched objects after detaching mark */ + if (sb) + fsnotify_update_sb_watchers(sb, conn); + objp = fsnotify_recalc_mask_clear_iref(conn); + type = conn->type; + } + WRITE_ONCE(mark->connector, NULL); + spin_unlock(&conn->lock); + + fsnotify_drop_object(type, objp); + + if (free_conn) { + spin_lock(&destroy_lock); + conn->destroy_next = connector_destroy_list; + connector_destroy_list = conn; + spin_unlock(&destroy_lock); + queue_work(system_dfl_wq, &connector_reaper_work); + } + /* + * Note that we didn't update flags telling whether inode cares about + * what's happening with children. We update these flags from + * __fsnotify_parent() lazily when next event happens on one of our + * children. + */ + spin_lock(&destroy_lock); + list_add(&mark->g_list, &destroy_list); + spin_unlock(&destroy_lock); + queue_delayed_work(system_dfl_wq, &reaper_work, + FSNOTIFY_REAPER_DELAY); +} +EXPORT_SYMBOL_GPL(fsnotify_put_mark); + +/* + * Get mark reference when we found the mark via lockless traversal of object + * list. Mark can be already removed from the list by now and on its way to be + * destroyed once SRCU period ends. + * + * Also pin the group so it doesn't disappear under us. + */ +static bool fsnotify_get_mark_safe(struct fsnotify_mark *mark) +{ + if (refcount_inc_not_zero(&mark->refcnt)) { + spin_lock(&mark->lock); + if (mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED) { + /* mark is attached, group is still alive then */ + atomic_inc(&mark->group->user_waits); + spin_unlock(&mark->lock); + return true; + } + spin_unlock(&mark->lock); + fsnotify_put_mark(mark); + } + return false; +} + +/* + * Puts marks and wakes up group destruction if necessary. + * + * Pairs with fsnotify_get_mark_safe() + */ +static void fsnotify_put_mark_wake(struct fsnotify_mark *mark) +{ + if (mark) { + struct fsnotify_group *group = mark->group; + + fsnotify_put_mark(mark); + /* + * We abuse notification_waitq on group shutdown for waiting for + * all marks pinned when waiting for userspace. + */ + if (atomic_dec_and_test(&group->user_waits) && group->shutdown) + wake_up(&group->notification_waitq); + } +} + +bool fsnotify_prepare_user_wait(struct fsnotify_iter_info *iter_info) + __releases(&fsnotify_mark_srcu) +{ + int type; + + fsnotify_foreach_iter_type(type) { + struct fsnotify_mark *mark = iter_info->marks[type]; + + /* This can fail if mark is being removed */ + while (mark && !fsnotify_get_mark_safe(mark)) { + if (mark->group == iter_info->current_group) { + __release(&fsnotify_mark_srcu); + goto fail; + } + /* This is a mark in an unrelated group, skip */ + mark = fsnotify_next_mark(mark); + iter_info->marks[type] = mark; + } + } + + /* + * Now that all marks are pinned by refcount in the inode / vfsmount / etc + * lists, we can drop SRCU lock, and safely resume the list iteration + * once userspace returns. + */ + srcu_read_unlock(&fsnotify_mark_srcu, iter_info->srcu_idx); + + return true; + +fail: + for (type--; type >= 0; type--) + fsnotify_put_mark_wake(iter_info->marks[type]); + return false; +} + +void fsnotify_finish_user_wait(struct fsnotify_iter_info *iter_info) + __acquires(&fsnotify_mark_srcu) +{ + int type; + + iter_info->srcu_idx = srcu_read_lock(&fsnotify_mark_srcu); + fsnotify_foreach_iter_type(type) + fsnotify_put_mark_wake(iter_info->marks[type]); +} + +/* + * Mark mark as detached, remove it from group list. Mark still stays in object + * list until its last reference is dropped. Note that we rely on mark being + * removed from group list before corresponding reference to it is dropped. In + * particular we rely on mark->connector being valid while we hold + * group->mark_mutex if we found the mark through g_list. + * + * Must be called with group->mark_mutex held. The caller must either hold + * reference to the mark or be protected by fsnotify_mark_srcu. + */ +void fsnotify_detach_mark(struct fsnotify_mark *mark) +{ + fsnotify_group_assert_locked(mark->group); + WARN_ON_ONCE(!srcu_read_lock_held(&fsnotify_mark_srcu) && + refcount_read(&mark->refcnt) < 1 + + !!(mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED)); + + spin_lock(&mark->lock); + /* something else already called this function on this mark */ + if (!(mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED)) { + spin_unlock(&mark->lock); + return; + } + mark->flags &= ~FSNOTIFY_MARK_FLAG_ATTACHED; + list_del_init(&mark->g_list); + spin_unlock(&mark->lock); + + /* Drop mark reference acquired in fsnotify_add_mark_locked() */ + fsnotify_put_mark(mark); +} + +/* + * Free fsnotify mark. The mark is actually only marked as being freed. The + * freeing is actually happening only once last reference to the mark is + * dropped from a workqueue which first waits for srcu period end. + * + * Caller must have a reference to the mark or be protected by + * fsnotify_mark_srcu. + */ +void fsnotify_free_mark(struct fsnotify_mark *mark) +{ + struct fsnotify_group *group = mark->group; + + spin_lock(&mark->lock); + /* something else already called this function on this mark */ + if (!(mark->flags & FSNOTIFY_MARK_FLAG_ALIVE)) { + spin_unlock(&mark->lock); + return; + } + mark->flags &= ~FSNOTIFY_MARK_FLAG_ALIVE; + spin_unlock(&mark->lock); + + /* + * Some groups like to know that marks are being freed. This is a + * callback to the group function to let it know that this mark + * is being freed. + */ + if (group->ops->freeing_mark) + group->ops->freeing_mark(mark, group); +} + +void fsnotify_destroy_mark(struct fsnotify_mark *mark, + struct fsnotify_group *group) +{ + fsnotify_group_lock(group); + fsnotify_detach_mark(mark); + fsnotify_group_unlock(group); + fsnotify_free_mark(mark); +} +EXPORT_SYMBOL_GPL(fsnotify_destroy_mark); + +/* + * Sorting function for lists of fsnotify marks. + * + * Fanotify supports different notification classes (reflected as priority of + * notification group). Events shall be passed to notification groups in + * decreasing priority order. To achieve this marks in notification lists for + * inodes and vfsmounts are sorted so that priorities of corresponding groups + * are descending. + * + * Furthermore correct handling of the ignore mask requires processing inode + * and vfsmount marks of each group together. Using the group address as + * further sort criterion provides a unique sorting order and thus we can + * merge inode and vfsmount lists of marks in linear time and find groups + * present in both lists. + * + * A return value of 1 signifies that b has priority over a. + * A return value of 0 signifies that the two marks have to be handled together. + * A return value of -1 signifies that a has priority over b. + */ +int fsnotify_compare_groups(struct fsnotify_group *a, struct fsnotify_group *b) +{ + if (a == b) + return 0; + if (!a) + return 1; + if (!b) + return -1; + if (a->priority < b->priority) + return 1; + if (a->priority > b->priority) + return -1; + if (a < b) + return 1; + return -1; +} + +static int fsnotify_attach_info_to_sb(struct super_block *sb) +{ + struct fsnotify_sb_info *sbinfo; + + /* sb info is freed on fsnotify_sb_delete() */ + sbinfo = kzalloc_obj(*sbinfo); + if (!sbinfo) + return -ENOMEM; + + INIT_LIST_HEAD(&sbinfo->inode_conn_list); + spin_lock_init(&sbinfo->list_lock); + /* + * cmpxchg() provides the barrier so that callers of fsnotify_sb_info() + * will observe an initialized structure + */ + if (cmpxchg(&sb->s_fsnotify_info, NULL, sbinfo)) { + /* Someone else created sbinfo for us */ + kfree(sbinfo); + } + return 0; +} + +struct fsnotify_inode_mark_connector { + struct fsnotify_mark_connector common; + struct list_head conns_list; +}; + +static struct inode *fsnotify_get_living_inode(struct fsnotify_sb_info *sbinfo) +{ + struct fsnotify_inode_mark_connector *iconn; + struct inode *inode; + + spin_lock(&sbinfo->list_lock); + /* Find the first non-evicting inode */ + list_for_each_entry(iconn, &sbinfo->inode_conn_list, conns_list) { + /* All connectors on the list are still attached to an inode */ + inode = iconn->common.obj; + /* + * For connectors without FSNOTIFY_CONN_FLAG_HAS_IREF + * (evictable marks) corresponding inode may well have 0 + * refcount and can be undergoing eviction. OTOH list_lock + * protects us from the connector getting detached and inode + * freed. So we can poke around the inode safely. + */ + spin_lock(&inode->i_lock); + if (likely( + !(inode_state_read(inode) & (I_FREEING | I_WILL_FREE)))) { + __iget(inode); + spin_unlock(&inode->i_lock); + spin_unlock(&sbinfo->list_lock); + return inode; + } + spin_unlock(&inode->i_lock); + } + spin_unlock(&sbinfo->list_lock); + + return NULL; +} + +/** + * fsnotify_unmount_inodes - an sb is unmounting. Handle any watched inodes. + * @sbinfo: fsnotify info for superblock being unmounted. + * + * Walk all inode connectors for the superblock and free all associated marks. + */ +void fsnotify_unmount_inodes(struct fsnotify_sb_info *sbinfo) +{ + struct inode *inode; + + while ((inode = fsnotify_get_living_inode(sbinfo))) { + fsnotify_inode(inode, FS_UNMOUNT); + fsnotify_clear_marks_by_inode(inode); + iput(inode); + cond_resched(); + } +} + +static void fsnotify_init_connector(struct fsnotify_mark_connector *conn, + void *obj, unsigned int obj_type) +{ + spin_lock_init(&conn->lock); + INIT_HLIST_HEAD(&conn->list); + conn->flags = 0; + conn->prio = 0; + conn->type = obj_type; + conn->obj = obj; +} + +static struct fsnotify_mark_connector * +fsnotify_alloc_inode_connector(struct inode *inode) +{ + struct fsnotify_inode_mark_connector *iconn; + struct fsnotify_sb_info *sbinfo = fsnotify_sb_info(inode->i_sb); + + iconn = kmem_cache_alloc(fsnotify_inode_mark_connector_cachep, + GFP_KERNEL); + if (!iconn) + return NULL; + + fsnotify_init_connector(&iconn->common, inode, FSNOTIFY_OBJ_TYPE_INODE); + spin_lock(&sbinfo->list_lock); + list_add(&iconn->conns_list, &sbinfo->inode_conn_list); + spin_unlock(&sbinfo->list_lock); + + return &iconn->common; +} + +static void fsnotify_untrack_connector(struct fsnotify_mark_connector *conn) +{ + struct fsnotify_inode_mark_connector *iconn; + struct fsnotify_sb_info *sbinfo; + + if (conn->type != FSNOTIFY_OBJ_TYPE_INODE) + return; + + iconn = container_of(conn, struct fsnotify_inode_mark_connector, common); + sbinfo = fsnotify_sb_info(fsnotify_conn_inode(conn)->i_sb); + spin_lock(&sbinfo->list_lock); + list_del(&iconn->conns_list); + spin_unlock(&sbinfo->list_lock); +} + +static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp, + void *obj, unsigned int obj_type) +{ + struct fsnotify_mark_connector *conn; + + if (obj_type == FSNOTIFY_OBJ_TYPE_INODE) { + struct inode *inode = obj; + + conn = fsnotify_alloc_inode_connector(inode); + } else { + conn = kmem_cache_alloc(fsnotify_mark_connector_cachep, + GFP_KERNEL); + if (conn) + fsnotify_init_connector(conn, obj, obj_type); + } + if (!conn) + return -ENOMEM; + + /* + * cmpxchg() provides the barrier so that readers of *connp can see + * only initialized structure + */ + if (cmpxchg(connp, NULL, conn)) { + /* Someone else created list structure for us */ + fsnotify_untrack_connector(conn); + kfree(conn); + } + return 0; +} + +/* + * Get mark connector, make sure it is alive and return with its lock held. + * This is for users that get connector pointer from inode or mount. Users that + * hold reference to a mark on the list may directly lock connector->lock as + * they are sure list cannot go away under them. + */ +static struct fsnotify_mark_connector *fsnotify_grab_connector( + fsnotify_connp_t *connp) +{ + struct fsnotify_mark_connector *conn; + int idx; + + idx = srcu_read_lock(&fsnotify_mark_srcu); + conn = srcu_dereference(*connp, &fsnotify_mark_srcu); + if (!conn) + goto out; + spin_lock(&conn->lock); + if (conn->type == FSNOTIFY_OBJ_TYPE_DETACHED) { + spin_unlock(&conn->lock); + srcu_read_unlock(&fsnotify_mark_srcu, idx); + return NULL; + } +out: + srcu_read_unlock(&fsnotify_mark_srcu, idx); + return conn; +} + +/* + * Add mark into proper place in given list of marks. These marks may be used + * for the fsnotify backend to determine which event types should be delivered + * to which group and for which inodes. These marks are ordered according to + * priority, highest number first, and then by the group's location in memory. + */ +static int fsnotify_add_mark_list(struct fsnotify_mark *mark, void *obj, + unsigned int obj_type, int add_flags) +{ + struct super_block *sb = fsnotify_object_sb(obj, obj_type); + struct fsnotify_mark *lmark, *last = NULL; + struct fsnotify_mark_connector *conn; + fsnotify_connp_t *connp; + int cmp; + int err = 0; + + if (WARN_ON(!fsnotify_valid_obj_type(obj_type))) + return -EINVAL; + + /* + * Attach the sb info before attaching a connector to any object on sb. + * The sb info will remain attached as long as sb lives. + */ + if (sb && !fsnotify_sb_info(sb)) { + err = fsnotify_attach_info_to_sb(sb); + if (err) + return err; + } + + connp = fsnotify_object_connp(obj, obj_type); +restart: + spin_lock(&mark->lock); + conn = fsnotify_grab_connector(connp); + if (!conn) { + spin_unlock(&mark->lock); + err = fsnotify_attach_connector_to_object(connp, obj, obj_type); + if (err) + return err; + goto restart; + } + + /* is mark the first mark? */ + if (hlist_empty(&conn->list)) { + hlist_add_head_rcu(&mark->obj_list, &conn->list); + goto added; + } + + /* should mark be in the middle of the current list? */ + hlist_for_each_entry(lmark, &conn->list, obj_list) { + last = lmark; + + if ((lmark->group == mark->group) && + (lmark->flags & FSNOTIFY_MARK_FLAG_ATTACHED) && + !(mark->group->flags & FSNOTIFY_GROUP_DUPS)) { + err = -EEXIST; + goto out_err; + } + + cmp = fsnotify_compare_groups(lmark->group, mark->group); + if (cmp >= 0) { + hlist_add_before_rcu(&mark->obj_list, &lmark->obj_list); + goto added; + } + } + + BUG_ON(last == NULL); + /* mark should be the last entry. last is the current last entry */ + hlist_add_behind_rcu(&mark->obj_list, &last->obj_list); +added: + if (sb) + fsnotify_update_sb_watchers(sb, conn); + /* + * Since connector is attached to object using cmpxchg() we are + * guaranteed that connector initialization is fully visible by anyone + * seeing mark->connector set. + */ + WRITE_ONCE(mark->connector, conn); +out_err: + spin_unlock(&conn->lock); + spin_unlock(&mark->lock); + return err; +} + +/* + * Attach an initialized mark to a given group and fs object. + * These marks may be used for the fsnotify backend to determine which + * event types should be delivered to which group. + */ +int fsnotify_add_mark_locked(struct fsnotify_mark *mark, + void *obj, unsigned int obj_type, + int add_flags) +{ + struct fsnotify_group *group = mark->group; + int ret = 0; + + fsnotify_group_assert_locked(group); + + /* + * LOCKING ORDER!!!! + * group->mark_mutex + * mark->lock + * mark->connector->lock + */ + spin_lock(&mark->lock); + mark->flags |= FSNOTIFY_MARK_FLAG_ALIVE | FSNOTIFY_MARK_FLAG_ATTACHED; + + list_add(&mark->g_list, &group->marks_list); + fsnotify_get_mark(mark); /* for g_list */ + spin_unlock(&mark->lock); + + ret = fsnotify_add_mark_list(mark, obj, obj_type, add_flags); + if (ret) + goto err; + + fsnotify_recalc_mask(mark->connector); + + return ret; +err: + spin_lock(&mark->lock); + mark->flags &= ~(FSNOTIFY_MARK_FLAG_ALIVE | + FSNOTIFY_MARK_FLAG_ATTACHED); + list_del_init(&mark->g_list); + spin_unlock(&mark->lock); + + fsnotify_put_mark(mark); + return ret; +} + +int fsnotify_add_mark(struct fsnotify_mark *mark, void *obj, + unsigned int obj_type, int add_flags) +{ + int ret; + struct fsnotify_group *group = mark->group; + + fsnotify_group_lock(group); + ret = fsnotify_add_mark_locked(mark, obj, obj_type, add_flags); + fsnotify_group_unlock(group); + return ret; +} +EXPORT_SYMBOL_GPL(fsnotify_add_mark); + +/* + * Given a list of marks, find the mark associated with given group. If found + * take a reference to that mark and return it, else return NULL. + */ +struct fsnotify_mark *fsnotify_find_mark(void *obj, unsigned int obj_type, + struct fsnotify_group *group) +{ + fsnotify_connp_t *connp = fsnotify_object_connp(obj, obj_type); + struct fsnotify_mark_connector *conn; + struct fsnotify_mark *mark; + + if (!connp) + return NULL; + + conn = fsnotify_grab_connector(connp); + if (!conn) + return NULL; + + hlist_for_each_entry(mark, &conn->list, obj_list) { + if (mark->group == group && + (mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED)) { + fsnotify_get_mark(mark); + spin_unlock(&conn->lock); + return mark; + } + } + spin_unlock(&conn->lock); + return NULL; +} +EXPORT_SYMBOL_GPL(fsnotify_find_mark); + +/* Clear any marks in a group with given type mask */ +void fsnotify_clear_marks_by_group(struct fsnotify_group *group, + unsigned int obj_type) +{ + struct fsnotify_mark *lmark, *mark; + LIST_HEAD(to_free); + struct list_head *head = &to_free; + + /* Skip selection step if we want to clear all marks. */ + if (obj_type == FSNOTIFY_OBJ_TYPE_ANY) { + head = &group->marks_list; + goto clear; + } + /* + * We have to be really careful here. Anytime we drop mark_mutex, e.g. + * fsnotify_clear_marks_by_inode() can come and free marks. Even in our + * to_free list so we have to use mark_mutex even when accessing that + * list. And freeing mark requires us to drop mark_mutex. So we can + * reliably free only the first mark in the list. That's why we first + * move marks to free to to_free list in one go and then free marks in + * to_free list one by one. + */ + fsnotify_group_lock(group); + list_for_each_entry_safe(mark, lmark, &group->marks_list, g_list) { + if (mark->connector->type == obj_type) + list_move(&mark->g_list, &to_free); + } + fsnotify_group_unlock(group); + +clear: + while (1) { + fsnotify_group_lock(group); + if (list_empty(head)) { + fsnotify_group_unlock(group); + break; + } + mark = list_first_entry(head, struct fsnotify_mark, g_list); + fsnotify_get_mark(mark); + fsnotify_detach_mark(mark); + fsnotify_group_unlock(group); + fsnotify_free_mark(mark); + fsnotify_put_mark(mark); + } +} + +/* Destroy all marks attached to an object via connector */ +void fsnotify_destroy_marks(fsnotify_connp_t *connp) +{ + struct fsnotify_mark_connector *conn; + struct fsnotify_mark *mark, *old_mark = NULL; + void *objp; + unsigned int type; + + conn = fsnotify_grab_connector(connp); + if (!conn) + return; + /* + * We have to be careful since we can race with e.g. + * fsnotify_clear_marks_by_group() and once we drop the conn->lock, the + * list can get modified. However we are holding mark reference and + * thus our mark cannot be removed from obj_list so we can continue + * iteration after regaining conn->lock. + */ + hlist_for_each_entry(mark, &conn->list, obj_list) { + fsnotify_get_mark(mark); + spin_unlock(&conn->lock); + if (old_mark) + fsnotify_put_mark(old_mark); + old_mark = mark; + fsnotify_destroy_mark(mark, mark->group); + spin_lock(&conn->lock); + } + /* + * Detach list from object now so that we don't pin inode until all + * mark references get dropped. It would lead to strange results such + * as delaying inode deletion or blocking unmount. + */ + objp = fsnotify_detach_connector_from_object(conn, &type); + spin_unlock(&conn->lock); + if (old_mark) + fsnotify_put_mark(old_mark); + fsnotify_drop_object(type, objp); +} + +/* + * Nothing fancy, just initialize lists and locks and counters. + */ +void fsnotify_init_mark(struct fsnotify_mark *mark, + struct fsnotify_group *group) +{ + memset(mark, 0, sizeof(*mark)); + spin_lock_init(&mark->lock); + refcount_set(&mark->refcnt, 1); + fsnotify_get_group(group); + mark->group = group; + WRITE_ONCE(mark->connector, NULL); +} +EXPORT_SYMBOL_GPL(fsnotify_init_mark); + +/* + * Destroy all marks in destroy_list, waits for SRCU period to finish before + * actually freeing marks. + */ +static void fsnotify_mark_destroy_workfn(struct work_struct *work) +{ + struct fsnotify_mark *mark, *next; + struct list_head private_destroy_list; + + spin_lock(&destroy_lock); + /* exchange the list head */ + list_replace_init(&destroy_list, &private_destroy_list); + spin_unlock(&destroy_lock); + + synchronize_srcu(&fsnotify_mark_srcu); + + list_for_each_entry_safe(mark, next, &private_destroy_list, g_list) { + list_del_init(&mark->g_list); + fsnotify_final_mark_destroy(mark); + } +} + +/* Wait for all marks queued for destruction to be actually destroyed */ +void fsnotify_wait_marks_destroyed(void) +{ + flush_delayed_work(&reaper_work); +} +EXPORT_SYMBOL_GPL(fsnotify_wait_marks_destroyed); + +__init void fsnotify_init_connector_caches(void) +{ + fsnotify_mark_connector_cachep = KMEM_CACHE(fsnotify_mark_connector, + SLAB_PANIC); + fsnotify_inode_mark_connector_cachep = KMEM_CACHE( + fsnotify_inode_mark_connector, + SLAB_PANIC); +} diff --git a/fs/notify/notification.c b/fs/notify/notification.c new file mode 100644 index 000000000..9022ae650 --- /dev/null +++ b/fs/notify/notification.c @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2008 Red Hat, Inc., Eric Paris + */ + +/* + * Basic idea behind the notification queue: An fsnotify group (like inotify) + * sends the userspace notification about events asynchronously some time after + * the event happened. When inotify gets an event it will need to add that + * event to the group notify queue. Since a single event might need to be on + * multiple group's notification queues we can't add the event directly to each + * queue and instead add a small "event_holder" to each queue. This event_holder + * has a pointer back to the original event. Since the majority of events are + * going to end up on one, and only one, notification queue we embed one + * event_holder into each event. This means we have a single allocation instead + * of always needing two. If the embedded event_holder is already in use by + * another group a new event_holder (from fsnotify_event_holder_cachep) will be + * allocated and used. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include "fsnotify.h" + +static atomic_t fsnotify_sync_cookie = ATOMIC_INIT(0); + +/** + * fsnotify_get_cookie - return a unique cookie for use in synchronizing events. + * Called from fsnotify_move, which is inlined into filesystem modules. + */ +u32 fsnotify_get_cookie(void) +{ + return atomic_inc_return(&fsnotify_sync_cookie); +} +EXPORT_SYMBOL_GPL(fsnotify_get_cookie); + +void fsnotify_destroy_event(struct fsnotify_group *group, + struct fsnotify_event *event) +{ + /* Overflow events are per-group and we don't want to free them */ + if (!event || event == group->overflow_event) + return; + /* + * If the event is still queued, we have a problem... Do an unreliable + * lockless check first to avoid locking in the common case. The + * locking may be necessary for permission events which got removed + * from the list by a different CPU than the one freeing the event. + */ + if (!list_empty(&event->list)) { + spin_lock(&group->notification_lock); + WARN_ON(!list_empty(&event->list)); + spin_unlock(&group->notification_lock); + } + group->ops->free_event(group, event); +} + +/* + * Try to add an event to the notification queue. + * The group can later pull this event off the queue to deal with. + * The group can use the @merge hook to merge the event with a queued event. + * The group can use the @insert hook to insert the event into hash table. + * The function returns: + * 0 if the event was added to a queue + * 1 if the event was merged with some other queued event + * 2 if the event was not queued - either the queue of events has overflown + * or the group is shutting down. + */ +int fsnotify_insert_event(struct fsnotify_group *group, + struct fsnotify_event *event, + int (*merge)(struct fsnotify_group *, + struct fsnotify_event *), + void (*insert)(struct fsnotify_group *, + struct fsnotify_event *)) +{ + int ret = 0; + struct list_head *list = &group->notification_list; + + pr_debug("%s: group=%p event=%p\n", __func__, group, event); + + spin_lock(&group->notification_lock); + + if (group->shutdown) { + spin_unlock(&group->notification_lock); + return 2; + } + + if (event == group->overflow_event || + group->q_len >= group->max_events) { + ret = 2; + /* Queue overflow event only if it isn't already queued */ + if (!list_empty(&group->overflow_event->list)) { + spin_unlock(&group->notification_lock); + return ret; + } + event = group->overflow_event; + goto queue; + } + + if (!list_empty(list) && merge) { + ret = merge(group, event); + if (ret) { + spin_unlock(&group->notification_lock); + return ret; + } + } + +queue: + group->q_len++; + list_add_tail(&event->list, list); + if (insert) + insert(group, event); + spin_unlock(&group->notification_lock); + + wake_up(&group->notification_waitq); + kill_fasync(&group->fsn_fa, SIGIO, POLL_IN); + return ret; +} + +void fsnotify_remove_queued_event(struct fsnotify_group *group, + struct fsnotify_event *event) +{ + assert_spin_locked(&group->notification_lock); + /* + * We need to init list head for the case of overflow event so that + * check in fsnotify_add_event() works + */ + list_del_init(&event->list); + group->q_len--; +} + +/* + * Return the first event on the notification list without removing it. + * Returns NULL if the list is empty. + */ +struct fsnotify_event *fsnotify_peek_first_event(struct fsnotify_group *group) +{ + assert_spin_locked(&group->notification_lock); + + if (fsnotify_notify_queue_is_empty(group)) + return NULL; + + return list_first_entry(&group->notification_list, + struct fsnotify_event, list); +} + +/* + * Remove and return the first event from the notification list. It is the + * responsibility of the caller to destroy the obtained event + */ +struct fsnotify_event *fsnotify_remove_first_event(struct fsnotify_group *group) +{ + struct fsnotify_event *event = fsnotify_peek_first_event(group); + + if (!event) + return NULL; + + pr_debug("%s: group=%p event=%p\n", __func__, group, event); + + fsnotify_remove_queued_event(group, event); + + return event; +} + +/* + * Called when a group is being torn down to clean up any outstanding + * event notifications. + */ +void fsnotify_flush_notify(struct fsnotify_group *group) +{ + struct fsnotify_event *event; + + spin_lock(&group->notification_lock); + while (!fsnotify_notify_queue_is_empty(group)) { + event = fsnotify_remove_first_event(group); + spin_unlock(&group->notification_lock); + fsnotify_destroy_event(group, event); + spin_lock(&group->notification_lock); + } + spin_unlock(&group->notification_lock); +} diff --git a/fs/nsfs.c b/fs/nsfs.c new file mode 100644 index 000000000..c3b6ae765 --- /dev/null +++ b/fs/nsfs.c @@ -0,0 +1,713 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mount.h" +#include "internal.h" + +static struct vfsmount *nsfs_mnt; + +static struct path nsfs_root_path = {}; + +void nsfs_get_root(struct path *path) +{ + *path = nsfs_root_path; + path_get(path); +} + +static long ns_ioctl(struct file *filp, unsigned int ioctl, + unsigned long arg); +static const struct file_operations ns_file_operations = { + .unlocked_ioctl = ns_ioctl, + .compat_ioctl = compat_ptr_ioctl, +}; + +static char *ns_dname(struct dentry *dentry, char *buffer, int buflen) +{ + struct inode *inode = d_inode(dentry); + struct ns_common *ns = inode->i_private; + const struct proc_ns_operations *ns_ops = ns->ops; + + return dynamic_dname(buffer, buflen, "%s:[%llu]", + ns_ops->name, inode->i_ino); +} + +const struct dentry_operations ns_dentry_operations = { + .d_dname = ns_dname, + .d_prune = stashed_dentry_prune, +}; + +static void nsfs_evict(struct inode *inode) +{ + struct ns_common *ns = inode->i_private; + + __ns_ref_active_put(ns); + clear_inode(inode); + ns->ops->put(ns); +} + +int ns_get_path_cb(struct path *path, ns_get_path_helper_t *ns_get_cb, + void *private_data) +{ + struct ns_common *ns; + + ns = ns_get_cb(private_data); + if (!ns) + return -ENOENT; + + return path_from_stashed(&ns->stashed, nsfs_mnt, ns, path); +} + +struct ns_get_path_task_args { + const struct proc_ns_operations *ns_ops; + struct task_struct *task; +}; + +static struct ns_common *ns_get_path_task(void *private_data) +{ + struct ns_get_path_task_args *args = private_data; + + return args->ns_ops->get(args->task); +} + +int ns_get_path(struct path *path, struct task_struct *task, + const struct proc_ns_operations *ns_ops) +{ + struct ns_get_path_task_args args = { + .ns_ops = ns_ops, + .task = task, + }; + + return ns_get_path_cb(path, ns_get_path_task, &args); +} + +struct file *open_namespace_file(struct ns_common *ns) +{ + struct path path __free(path_put) = {}; + int err; + + /* call first to consume reference */ + err = path_from_stashed(&ns->stashed, nsfs_mnt, ns, &path); + if (err < 0) + return ERR_PTR(err); + + return dentry_open(&path, O_RDONLY, current_cred()); +} + +/** + * open_namespace - open a namespace + * @ns: the namespace to open + * + * This will consume a reference to @ns indendent of success or failure. + * + * Return: A file descriptor on success or a negative error code on failure. + */ +int open_namespace(struct ns_common *ns) +{ + struct path path __free(path_put) = {}; + int err; + + /* call first to consume reference */ + err = path_from_stashed(&ns->stashed, nsfs_mnt, ns, &path); + if (err < 0) + return err; + + return FD_ADD(O_CLOEXEC, dentry_open(&path, O_RDONLY, current_cred())); +} + +int open_related_ns(struct ns_common *ns, + struct ns_common *(*get_ns)(struct ns_common *ns)) +{ + struct ns_common *relative; + + relative = get_ns(ns); + if (IS_ERR(relative)) + return PTR_ERR(relative); + + return open_namespace(relative); +} +EXPORT_SYMBOL_GPL(open_related_ns); + +static int copy_ns_info_to_user(const struct mnt_namespace *mnt_ns, + struct mnt_ns_info __user *uinfo, size_t usize, + struct mnt_ns_info *kinfo) +{ + /* + * If userspace and the kernel have the same struct size it can just + * be copied. If userspace provides an older struct, only the bits that + * userspace knows about will be copied. If userspace provides a new + * struct, only the bits that the kernel knows aobut will be copied and + * the size value will be set to the size the kernel knows about. + */ + kinfo->size = min(usize, sizeof(*kinfo)); + kinfo->mnt_ns_id = mnt_ns->ns.ns_id; + kinfo->nr_mounts = READ_ONCE(mnt_ns->nr_mounts); + /* Subtract the root mount of the mount namespace. */ + if (kinfo->nr_mounts) + kinfo->nr_mounts--; + + if (copy_to_user(uinfo, kinfo, kinfo->size)) + return -EFAULT; + + return 0; +} + +static bool nsfs_ioctl_valid(unsigned int cmd) +{ + switch (cmd) { + case NS_GET_USERNS: + case NS_GET_PARENT: + case NS_GET_NSTYPE: + case NS_GET_OWNER_UID: + case NS_GET_MNTNS_ID: + case NS_GET_PID_FROM_PIDNS: + case NS_GET_TGID_FROM_PIDNS: + case NS_GET_PID_IN_PIDNS: + case NS_GET_TGID_IN_PIDNS: + case NS_GET_ID: + return true; + } + + /* Extensible ioctls require some extra handling. */ + switch (_IOC_NR(cmd)) { + case _IOC_NR(NS_MNT_GET_INFO): + return extensible_ioctl_valid(cmd, NS_MNT_GET_INFO, MNT_NS_INFO_SIZE_VER0); + case _IOC_NR(NS_MNT_GET_NEXT): + return extensible_ioctl_valid(cmd, NS_MNT_GET_NEXT, MNT_NS_INFO_SIZE_VER0); + case _IOC_NR(NS_MNT_GET_PREV): + return extensible_ioctl_valid(cmd, NS_MNT_GET_PREV, MNT_NS_INFO_SIZE_VER0); + } + + return false; +} + +static bool may_use_nsfs_ioctl(unsigned int cmd) +{ + switch (_IOC_NR(cmd)) { + case _IOC_NR(NS_MNT_GET_NEXT): + fallthrough; + case _IOC_NR(NS_MNT_GET_PREV): + return may_see_all_namespaces(); + } + return true; +} + +static long ns_ioctl(struct file *filp, unsigned int ioctl, + unsigned long arg) +{ + struct user_namespace *user_ns; + struct pid_namespace *pid_ns; + struct task_struct *tsk; + struct ns_common *ns; + struct mnt_namespace *mnt_ns; + bool previous = false; + uid_t __user *argp; + uid_t uid; + int ret; + + if (!nsfs_ioctl_valid(ioctl)) + return -ENOIOCTLCMD; + if (!may_use_nsfs_ioctl(ioctl)) + return -EPERM; + + ns = get_proc_ns(file_inode(filp)); + switch (ioctl) { + case NS_GET_USERNS: + return open_related_ns(ns, ns_get_owner); + case NS_GET_PARENT: + if (!ns->ops->get_parent) + return -EINVAL; + return open_related_ns(ns, ns->ops->get_parent); + case NS_GET_NSTYPE: + return ns->ns_type; + case NS_GET_OWNER_UID: + if (ns->ns_type != CLONE_NEWUSER) + return -EINVAL; + user_ns = container_of(ns, struct user_namespace, ns); + argp = (uid_t __user *) arg; + uid = from_kuid_munged(current_user_ns(), user_ns->owner); + return put_user(uid, argp); + case NS_GET_PID_FROM_PIDNS: + fallthrough; + case NS_GET_TGID_FROM_PIDNS: + fallthrough; + case NS_GET_PID_IN_PIDNS: + fallthrough; + case NS_GET_TGID_IN_PIDNS: { + if (ns->ns_type != CLONE_NEWPID) + return -EINVAL; + + ret = -ESRCH; + pid_ns = container_of(ns, struct pid_namespace, ns); + + guard(rcu)(); + + if (ioctl == NS_GET_PID_IN_PIDNS || + ioctl == NS_GET_TGID_IN_PIDNS) + tsk = find_task_by_vpid(arg); + else + tsk = find_task_by_pid_ns(arg, pid_ns); + if (!tsk) + return ret; + + switch (ioctl) { + case NS_GET_PID_FROM_PIDNS: + ret = task_pid_vnr(tsk); + break; + case NS_GET_TGID_FROM_PIDNS: + ret = task_tgid_vnr(tsk); + break; + case NS_GET_PID_IN_PIDNS: + ret = task_pid_nr_ns(tsk, pid_ns); + break; + case NS_GET_TGID_IN_PIDNS: + ret = task_tgid_nr_ns(tsk, pid_ns); + break; + default: + ret = 0; + break; + } + + if (!ret) + ret = -ESRCH; + return ret; + } + case NS_GET_MNTNS_ID: + if (ns->ns_type != CLONE_NEWNS) + return -EINVAL; + fallthrough; + case NS_GET_ID: { + __u64 __user *idp; + __u64 id; + + idp = (__u64 __user *)arg; + id = ns->ns_id; + return put_user(id, idp); + } + } + + /* extensible ioctls */ + switch (_IOC_NR(ioctl)) { + case _IOC_NR(NS_MNT_GET_INFO): { + struct mnt_ns_info kinfo = {}; + struct mnt_ns_info __user *uinfo = (struct mnt_ns_info __user *)arg; + size_t usize = _IOC_SIZE(ioctl); + + if (ns->ns_type != CLONE_NEWNS) + return -EINVAL; + + if (!uinfo) + return -EINVAL; + + if (usize < MNT_NS_INFO_SIZE_VER0) + return -EINVAL; + + return copy_ns_info_to_user(to_mnt_ns(ns), uinfo, usize, &kinfo); + } + case _IOC_NR(NS_MNT_GET_PREV): + previous = true; + fallthrough; + case _IOC_NR(NS_MNT_GET_NEXT): { + struct mnt_ns_info kinfo = {}; + struct mnt_ns_info __user *uinfo = (struct mnt_ns_info __user *)arg; + struct path path __free(path_put) = {}; + size_t usize = _IOC_SIZE(ioctl); + + if (ns->ns_type != CLONE_NEWNS) + return -EINVAL; + + if (usize < MNT_NS_INFO_SIZE_VER0) + return -EINVAL; + + mnt_ns = get_sequential_mnt_ns(to_mnt_ns(ns), previous); + if (IS_ERR(mnt_ns)) + return PTR_ERR(mnt_ns); + + ns = to_ns_common(mnt_ns); + /* Transfer ownership of @mnt_ns reference to @path. */ + ret = path_from_stashed(&ns->stashed, nsfs_mnt, ns, &path); + if (ret) + return ret; + + FD_PREPARE(fdf, O_CLOEXEC, dentry_open(&path, O_RDONLY, current_cred())); + if (fdf.err) + return fdf.err; + /* + * If @uinfo is passed return all information about the + * mount namespace as well. + */ + ret = copy_ns_info_to_user(to_mnt_ns(ns), uinfo, usize, &kinfo); + if (ret) + return ret; + ret = fd_publish(fdf); + break; + } + default: + ret = -ENOTTY; + } + + return ret; +} + +int ns_get_name(char *buf, size_t size, struct task_struct *task, + const struct proc_ns_operations *ns_ops) +{ + struct ns_common *ns; + int res = -ENOENT; + const char *name; + ns = ns_ops->get(task); + if (ns) { + name = ns_ops->real_ns_name ? : ns_ops->name; + res = snprintf(buf, size, "%s:[%u]", name, ns->inum); + ns_ops->put(ns); + } + return res; +} + +bool proc_ns_file(const struct file *file) +{ + return file->f_op == &ns_file_operations; +} + +/** + * ns_match() - Returns true if current namespace matches dev/ino provided. + * @ns: current namespace + * @dev: dev_t from nsfs that will be matched against current nsfs + * @ino: ino_t from nsfs that will be matched against current nsfs + * + * Return: true if dev and ino matches the current nsfs. + */ +bool ns_match(const struct ns_common *ns, dev_t dev, ino_t ino) +{ + return (ns->inum == ino) && (nsfs_mnt->mnt_sb->s_dev == dev); +} + + +static int nsfs_show_path(struct seq_file *seq, struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + const struct ns_common *ns = inode->i_private; + const struct proc_ns_operations *ns_ops = ns->ops; + + seq_printf(seq, "%s:[%llu]", ns_ops->name, inode->i_ino); + return 0; +} + +static const struct super_operations nsfs_ops = { + .statfs = simple_statfs, + .evict_inode = nsfs_evict, + .show_path = nsfs_show_path, + .drop_inode = inode_just_drop, +}; + +static int nsfs_init_inode(struct inode *inode, void *data) +{ + struct ns_common *ns = data; + + inode->i_private = data; + inode->i_mode |= S_IRUGO; + inode->i_fop = &ns_file_operations; + inode->i_ino = ns->inum; + + /* + * Bring the namespace subtree back to life if we have to. This + * can happen when e.g., all processes using a network namespace + * and all namespace files or namespace file bind-mounts have + * died but there are still sockets pinning it. The SIOCGSKNS + * ioctl on such a socket will resurrect the relevant namespace + * subtree. + */ + __ns_ref_active_get(ns); + return 0; +} + +static void nsfs_put_data(void *data) +{ + struct ns_common *ns = data; + ns->ops->put(ns); +} + +static const struct stashed_operations nsfs_stashed_ops = { + .init_inode = nsfs_init_inode, + .put_data = nsfs_put_data, +}; + +#define NSFS_FID_SIZE_U32_VER0 (NSFS_FILE_HANDLE_SIZE_VER0 / sizeof(u32)) +#define NSFS_FID_SIZE_U32_LATEST (NSFS_FILE_HANDLE_SIZE_LATEST / sizeof(u32)) + +static int nsfs_encode_fh(struct inode *inode, u32 *fh, int *max_len, + struct inode *parent) +{ + struct nsfs_file_handle *fid = (struct nsfs_file_handle *)fh; + struct ns_common *ns = inode->i_private; + int len = *max_len; + + if (parent) + return FILEID_INVALID; + + if (len < NSFS_FID_SIZE_U32_VER0) { + *max_len = NSFS_FID_SIZE_U32_LATEST; + return FILEID_INVALID; + } else if (len > NSFS_FID_SIZE_U32_LATEST) { + *max_len = NSFS_FID_SIZE_U32_LATEST; + } + + fid->ns_id = ns->ns_id; + fid->ns_type = ns->ns_type; + fid->ns_inum = inode->i_ino; + return FILEID_NSFS; +} + +bool is_current_namespace(struct ns_common *ns) +{ + switch (ns->ns_type) { +#ifdef CONFIG_CGROUPS + case CLONE_NEWCGROUP: + return current_in_namespace(to_cg_ns(ns)); +#endif +#ifdef CONFIG_IPC_NS + case CLONE_NEWIPC: + return current_in_namespace(to_ipc_ns(ns)); +#endif + case CLONE_NEWNS: + return current_in_namespace(to_mnt_ns(ns)); +#ifdef CONFIG_NET_NS + case CLONE_NEWNET: + return current_in_namespace(to_net_ns(ns)); +#endif +#ifdef CONFIG_PID_NS + case CLONE_NEWPID: + return current_in_namespace(to_pid_ns(ns)); +#endif +#ifdef CONFIG_TIME_NS + case CLONE_NEWTIME: + return current_in_namespace(to_time_ns(ns)); +#endif +#ifdef CONFIG_USER_NS + case CLONE_NEWUSER: + return current_in_namespace(to_user_ns(ns)); +#endif +#ifdef CONFIG_UTS_NS + case CLONE_NEWUTS: + return current_in_namespace(to_uts_ns(ns)); +#endif + default: + VFS_WARN_ON_ONCE(true); + return false; + } +} + +static struct dentry *nsfs_fh_to_dentry(struct super_block *sb, struct fid *fh, + int fh_len, int fh_type) +{ + struct path path __free(path_put) = {}; + struct nsfs_file_handle *fid = (struct nsfs_file_handle *)fh; + struct user_namespace *owning_ns = NULL; + struct ns_common *ns; + int ret; + + if (fh_len < NSFS_FID_SIZE_U32_VER0) + return NULL; + + /* Check that any trailing bytes are zero. */ + if ((fh_len > NSFS_FID_SIZE_U32_LATEST) && + memchr_inv((void *)fid + NSFS_FID_SIZE_U32_LATEST, 0, + fh_len - NSFS_FID_SIZE_U32_LATEST)) + return NULL; + + switch (fh_type) { + case FILEID_NSFS: + break; + default: + return NULL; + } + + if (!fid->ns_id) + return NULL; + /* Either both are set or both are unset. */ + if (!fid->ns_inum != !fid->ns_type) + return NULL; + + scoped_guard(rcu) { + ns = ns_tree_lookup_rcu(fid->ns_id, fid->ns_type); + if (!ns) + return NULL; + + VFS_WARN_ON_ONCE(ns->ns_id != fid->ns_id); + + if (fid->ns_inum && (fid->ns_inum != ns->inum)) + return NULL; + if (fid->ns_type && (fid->ns_type != ns->ns_type)) + return NULL; + + /* + * This is racy because we're not actually taking an + * active reference. IOW, it could happen that the + * namespace becomes inactive after this check. + * We don't care because nsfs_init_inode() will just + * resurrect the relevant namespace tree for us. If it + * has been active here we just allow it's resurrection. + * We could try to take an active reference here and + * then drop it again. But really, why bother. + */ + if (!ns_get_unless_inactive(ns)) + return NULL; + } + + switch (ns->ns_type) { +#ifdef CONFIG_CGROUPS + case CLONE_NEWCGROUP: + if (!current_in_namespace(to_cg_ns(ns))) + owning_ns = to_cg_ns(ns)->user_ns; + break; +#endif +#ifdef CONFIG_IPC_NS + case CLONE_NEWIPC: + if (!current_in_namespace(to_ipc_ns(ns))) + owning_ns = to_ipc_ns(ns)->user_ns; + break; +#endif + case CLONE_NEWNS: + if (!current_in_namespace(to_mnt_ns(ns))) + owning_ns = to_mnt_ns(ns)->user_ns; + break; +#ifdef CONFIG_NET_NS + case CLONE_NEWNET: + if (!current_in_namespace(to_net_ns(ns))) + owning_ns = to_net_ns(ns)->user_ns; + break; +#endif +#ifdef CONFIG_PID_NS + case CLONE_NEWPID: + if (!current_in_namespace(to_pid_ns(ns))) { + owning_ns = to_pid_ns(ns)->user_ns; + } else if (!READ_ONCE(to_pid_ns(ns)->child_reaper)) { + ns->ops->put(ns); + return ERR_PTR(-EPERM); + } + break; +#endif +#ifdef CONFIG_TIME_NS + case CLONE_NEWTIME: + if (!current_in_namespace(to_time_ns(ns))) + owning_ns = to_time_ns(ns)->user_ns; + break; +#endif +#ifdef CONFIG_USER_NS + case CLONE_NEWUSER: + if (!current_in_namespace(to_user_ns(ns))) + owning_ns = to_user_ns(ns); + break; +#endif +#ifdef CONFIG_UTS_NS + case CLONE_NEWUTS: + if (!current_in_namespace(to_uts_ns(ns))) + owning_ns = to_uts_ns(ns)->user_ns; + break; +#endif + default: + return ERR_PTR(-EOPNOTSUPP); + } + + if (owning_ns && !may_see_all_namespaces()) { + ns->ops->put(ns); + return ERR_PTR(-EPERM); + } + + /* path_from_stashed() unconditionally consumes the reference. */ + ret = path_from_stashed(&ns->stashed, nsfs_mnt, ns, &path); + if (ret) + return ERR_PTR(ret); + + return no_free_ptr(path.dentry); +} + +static int nsfs_export_permission(struct handle_to_path_ctx *ctx, + unsigned int oflags) +{ + /* nsfs_fh_to_dentry() performs all permission checks. */ + return 0; +} + +static struct file *nsfs_export_open(const struct path *path, unsigned int oflags) +{ + return file_open_root(path, "", oflags, 0); +} + +static const struct export_operations nsfs_export_operations = { + .encode_fh = nsfs_encode_fh, + .fh_to_dentry = nsfs_fh_to_dentry, + .open = nsfs_export_open, + .permission = nsfs_export_permission, +}; + +static int nsfs_init_fs_context(struct fs_context *fc) +{ + struct pseudo_fs_context *ctx = init_pseudo(fc, NSFS_MAGIC); + if (!ctx) + return -ENOMEM; + ctx->s_d_flags |= DCACHE_DONTCACHE; + ctx->ops = &nsfs_ops; + ctx->eops = &nsfs_export_operations; + ctx->dops = &ns_dentry_operations; + fc->s_fs_info = (void *)&nsfs_stashed_ops; + return 0; +} + +static struct file_system_type nsfs = { + .name = "nsfs", + .init_fs_context = nsfs_init_fs_context, + .kill_sb = kill_anon_super, +}; + +void __init nsfs_init(void) +{ + nsfs_mnt = kern_mount(&nsfs); + if (IS_ERR(nsfs_mnt)) + panic("can't set nsfs up\n"); + nsfs_mnt->mnt_sb->s_flags &= ~SB_NOUSER; + nsfs_root_path.mnt = nsfs_mnt; + nsfs_root_path.dentry = nsfs_mnt->mnt_root; +} + +void nsproxy_ns_active_get(struct nsproxy *ns) +{ + ns_ref_active_get(ns->mnt_ns); + ns_ref_active_get(ns->uts_ns); + ns_ref_active_get(ns->ipc_ns); + ns_ref_active_get(ns->pid_ns_for_children); + ns_ref_active_get(ns->cgroup_ns); + ns_ref_active_get(ns->net_ns); + ns_ref_active_get(ns->time_ns); + ns_ref_active_get(ns->time_ns_for_children); +} + +void nsproxy_ns_active_put(struct nsproxy *ns) +{ + ns_ref_active_put(ns->mnt_ns); + ns_ref_active_put(ns->uts_ns); + ns_ref_active_put(ns->ipc_ns); + ns_ref_active_put(ns->pid_ns_for_children); + ns_ref_active_put(ns->cgroup_ns); + ns_ref_active_put(ns->net_ns); + ns_ref_active_put(ns->time_ns); + ns_ref_active_put(ns->time_ns_for_children); +} diff --git a/fs/ntfs/Kconfig b/fs/ntfs/Kconfig new file mode 100644 index 000000000..6a6acde9b --- /dev/null +++ b/fs/ntfs/Kconfig @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: GPL-2.0-only +config NTFS_FS + tristate "NTFS file system support" + select NLS + select FS_IOMAP + help + NTFS is the file system of Microsoft Windows NT, 2000, XP and 2003. + This allows you to mount devices formatted with the ntfs file system. + + To compile this as a module, choose M here: the module will be called + ntfs. + +config NTFS_DEBUG + bool "NTFS debugging support" + depends on NTFS_FS + help + If you are experiencing any problems with the NTFS file system, say + Y here. This will result in additional consistency checks to be + performed by the driver as well as additional debugging messages to + be written to the system log. Note that debugging messages are + disabled by default. To enable them, supply the option debug_msgs=1 + at the kernel command line when booting the kernel or as an option + to insmod when loading the ntfs module. Once the driver is active, + you can enable debugging messages by doing (as root): + echo 1 > /proc/sys/fs/ntfs-debug + Replacing the "1" with "0" would disable debug messages. + + If you leave debugging messages disabled, this results in little + overhead, but enabling debug messages results in very significant + slowdown of the system. + + When reporting bugs, please try to have available a full dump of + debugging messages while the misbehaviour was occurring. + +config NTFS_FS_POSIX_ACL + bool "NTFS POSIX Access Control Lists" + depends on NTFS_FS + select FS_POSIX_ACL + help + POSIX Access Control Lists (ACLs) support additional access rights + for users and groups beyond the standard owner/group/world scheme. + + This option enables ACL support for ntfs, providing functional parity + with ntfs3 drivier. + + NOTE: this is linux only feature. Windows will ignore these ACLs. + + If you don't know what Access Control Lists are, say N. diff --git a/fs/ntfs/Makefile b/fs/ntfs/Makefile new file mode 100644 index 000000000..e120c2e69 --- /dev/null +++ b/fs/ntfs/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_NTFS_FS) += ntfs.o + +ntfs-y := aops.o attrib.o collate.o dir.o file.o index.o inode.o \ + mft.o mst.o namei.o runlist.o super.o unistr.o attrlist.o ea.o \ + upcase.o bitmap.o lcnalloc.o logfile.o reparse.o compress.o \ + iomap.o debug.o sysctl.o object_id.o bdev-io.o + +ccflags-$(CONFIG_NTFS_DEBUG) += -DDEBUG diff --git a/fs/ntfs/aops.c b/fs/ntfs/aops.c new file mode 100644 index 000000000..173de4cbe --- /dev/null +++ b/fs/ntfs/aops.c @@ -0,0 +1,318 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * NTFS kernel address space operations and page cache handling. + * + * Copyright (c) 2001-2014 Anton Altaparmakov and Tuxera Inc. + * Copyright (c) 2002 Richard Russon + * Copyright (c) 2025 LG Electronics Co., Ltd. + */ + +#include + +#include "attrib.h" +#include "mft.h" +#include "ntfs.h" +#include "debug.h" +#include "iomap.h" + +static void ntfs_iomap_read_end_io(struct bio *bio) +{ + int error = blk_status_to_errno(bio->bi_status); + struct folio_iter iter; + + bio_for_each_folio_all(iter, bio) { + struct folio *folio = iter.folio; + struct ntfs_inode *ni = NTFS_I(folio->mapping->host); + s64 init_size; + loff_t pos = folio_pos(folio); + + init_size = ni->initialized_size; + if (pos + iter.offset < init_size && + pos + iter.offset + iter.length > init_size) + folio_zero_segment(folio, offset_in_folio(folio, init_size), + iter.offset + iter.length); + + iomap_finish_folio_read(folio, iter.offset, iter.length, error); + } + bio_put(bio); +} + +static void ntfs_iomap_bio_submit_read(const struct iomap_iter *iter, + struct iomap_read_folio_ctx *ctx) +{ + iomap_bio_submit_read_endio(iter, ctx, ntfs_iomap_read_end_io); +} + +static const struct iomap_read_ops ntfs_iomap_bio_read_ops = { + .read_folio_range = iomap_bio_read_folio_range, + .submit_read = ntfs_iomap_bio_submit_read, +}; + +/* + * ntfs_read_folio - Read data for a folio from the device + * @file: open file to which the folio @folio belongs or NULL + * @folio: page cache folio to fill with data + * + * This function handles reading data into the page cache. It first checks + * for specific ntfs attribute type like encryption and compression. + * + * - If the attribute is encrypted, access is denied (-EACCES) because + * decryption is not supported in this path. + * - If the attribute is non-resident and compressed, the read operation is + * delegated to ntfs_read_compressed_block(). + * - For normal resident or non-resident attribute, it utilizes the generic + * iomap infrastructure via iomap_bio_read_folio() to perform the I/O. + * + * Return: 0 on success, or -errno on error. + */ +static int ntfs_read_folio(struct file *file, struct folio *folio) +{ + struct ntfs_inode *ni = NTFS_I(folio->mapping->host); + struct iomap_read_folio_ctx ctx = { + .cur_folio = folio, + .ops = &ntfs_iomap_bio_read_ops, + }; + + /* + * Only $DATA attributes can be encrypted and only unnamed $DATA + * attributes can be compressed. Index root can have the flags set but + * this means to create compressed/encrypted files, not that the + * attribute is compressed/encrypted. Note we need to check for + * AT_INDEX_ALLOCATION since this is the type of both directory and + * index inodes. + */ + if (ni->type != AT_INDEX_ALLOCATION) { + /* + * EFS-encrypted files are not supported. + * (decryption/encryption is not implemented yet) + */ + if (NInoEncrypted(ni)) { + folio_unlock(folio); + return -EOPNOTSUPP; + } + /* Compressed data streams are handled in compress.c. */ + if (NInoNonResident(ni) && NInoCompressed(ni)) + return ntfs_read_compressed_block(folio); + } + + iomap_read_folio(&ntfs_read_iomap_ops, &ctx, NULL); + return 0; +} + +/* + * ntfs_bmap - map logical file block to physical device block + * @mapping: address space mapping to which the block to be mapped belongs + * @block: logical block to map to its physical device block + * + * For regular, non-resident files (i.e. not compressed and not encrypted), map + * the logical @block belonging to the file described by the address space + * mapping @mapping to its physical device block. + * + * The size of the block is equal to the @s_blocksize field of the super block + * of the mounted file system which is guaranteed to be smaller than or equal + * to the cluster size thus the block is guaranteed to fit entirely inside the + * cluster which means we do not need to care how many contiguous bytes are + * available after the beginning of the block. + * + * Return the physical device block if the mapping succeeded or 0 if the block + * is sparse or there was an error. + * + * Note: This is a problem if someone tries to run bmap() on $Boot system file + * as that really is in block zero but there is nothing we can do. bmap() is + * just broken in that respect (just like it cannot distinguish sparse from + * not available or error). + */ +static sector_t ntfs_bmap(struct address_space *mapping, sector_t block) +{ + s64 ofs, size; + loff_t i_size; + s64 lcn; + unsigned long blocksize, flags; + struct ntfs_inode *ni = NTFS_I(mapping->host); + struct ntfs_volume *vol = ni->vol; + unsigned int delta; + unsigned char blocksize_bits; + + ntfs_debug("Entering for mft_no 0x%llx, logical block 0x%llx.", + ni->mft_no, (unsigned long long)block); + if (ni->type != AT_DATA || !NInoNonResident(ni) || NInoEncrypted(ni) || + NInoMstProtected(ni)) { + ntfs_error(vol->sb, "BMAP does not make sense for %s attributes, returning 0.", + (ni->type != AT_DATA) ? "non-data" : + (!NInoNonResident(ni) ? "resident" : + "encrypted")); + return 0; + } + /* None of these can happen. */ + blocksize = vol->sb->s_blocksize; + blocksize_bits = vol->sb->s_blocksize_bits; + ofs = (s64)block << blocksize_bits; + read_lock_irqsave(&ni->size_lock, flags); + size = ni->initialized_size; + i_size = i_size_read(VFS_I(ni)); + read_unlock_irqrestore(&ni->size_lock, flags); + /* + * If the offset is outside the initialized size or the block straddles + * the initialized size then pretend it is a hole unless the + * initialized size equals the file size. + */ + if (unlikely(ofs >= size || (ofs + blocksize > size && size < i_size))) + goto hole; + down_read(&ni->runlist.lock); + lcn = ntfs_attr_vcn_to_lcn_nolock(ni, ntfs_bytes_to_cluster(vol, ofs), + false); + up_read(&ni->runlist.lock); + if (unlikely(lcn < LCN_HOLE)) { + /* + * Step down to an integer to avoid gcc doing a long long + * comparision in the switch when we know @lcn is between + * LCN_HOLE and LCN_EIO (i.e. -1 to -5). + * + * Otherwise older gcc (at least on some architectures) will + * try to use __cmpdi2() which is of course not available in + * the kernel. + */ + switch ((int)lcn) { + case LCN_ENOENT: + /* + * If the offset is out of bounds then pretend it is a + * hole. + */ + goto hole; + case LCN_ENOMEM: + ntfs_error(vol->sb, + "Not enough memory to complete mapping for inode 0x%llx. Returning 0.", + ni->mft_no); + break; + default: + ntfs_error(vol->sb, + "Failed to complete mapping for inode 0x%llx. Run chkdsk. Returning 0.", + ni->mft_no); + break; + } + return 0; + } + if (lcn < 0) { + /* It is a hole. */ +hole: + ntfs_debug("Done (returning hole)."); + return 0; + } + /* + * The block is really allocated and fullfils all our criteria. + * Convert the cluster to units of block size and return the result. + */ + delta = ofs & vol->cluster_size_mask; + if (unlikely(sizeof(block) < sizeof(lcn))) { + block = lcn = (ntfs_cluster_to_bytes(vol, lcn) + delta) >> + blocksize_bits; + /* If the block number was truncated return 0. */ + if (unlikely(block != lcn)) { + ntfs_error(vol->sb, + "Physical block 0x%llx is too large to be returned, returning 0.", + (long long)lcn); + return 0; + } + } else + block = (ntfs_cluster_to_bytes(vol, lcn) + delta) >> + blocksize_bits; + ntfs_debug("Done (returning block 0x%llx).", (unsigned long long)lcn); + return block; +} + +static void ntfs_readahead(struct readahead_control *rac) +{ + struct address_space *mapping = rac->mapping; + struct inode *inode = mapping->host; + struct ntfs_inode *ni = NTFS_I(inode); + struct iomap_read_folio_ctx ctx = { + .ops = &ntfs_iomap_bio_read_ops, + .rac = rac, + }; + + /* + * Resident files are not cached in the page cache, + * and readahead is not implemented for compressed files. + */ + if (!NInoNonResident(ni) || NInoCompressed(ni)) + return; + iomap_readahead(&ntfs_read_iomap_ops, &ctx, NULL); +} + +static int ntfs_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct inode *inode = mapping->host; + struct ntfs_inode *ni = NTFS_I(inode); + struct iomap_writepage_ctx wpc = { + .inode = mapping->host, + .wbc = wbc, + .ops = &ntfs_writeback_ops, + }; + bool need_iput = false; + int ret; + + if (NVolShutdown(ni->vol)) + return -EIO; + + if (!NInoNonResident(ni)) + return 0; + + /* + * EFS-encrypted files are not supported. + * (decryption/encryption is not implemented yet) + */ + if (NInoEncrypted(ni)) { + ntfs_debug("Encrypted I/O not supported"); + return -EOPNOTSUPP; + } + + /* + * Prevent eviction in writeback to avoid deadlock in + * ntfs_drop_big_inode(). + */ + if ((ni->type == AT_DATA || ni->type == AT_INDEX_ALLOCATION) && + igrab(inode)) + need_iput = true; + + ret = iomap_writepages(&wpc); + + if (need_iput) + iput(inode); + + return ret; +} + +static int ntfs_swap_activate(struct swap_info_struct *sis, + struct file *swap_file, sector_t *span) +{ + return iomap_swapfile_activate(sis, swap_file, span, + &ntfs_read_iomap_ops); +} + +const struct address_space_operations ntfs_aops = { + .read_folio = ntfs_read_folio, + .readahead = ntfs_readahead, + .writepages = ntfs_writepages, + .dirty_folio = iomap_dirty_folio, + .bmap = ntfs_bmap, + .migrate_folio = filemap_migrate_folio, + .is_partially_uptodate = iomap_is_partially_uptodate, + .error_remove_folio = generic_error_remove_folio, + .release_folio = iomap_release_folio, + .invalidate_folio = iomap_invalidate_folio, + .swap_activate = ntfs_swap_activate, +}; + +const struct address_space_operations ntfs_mft_aops = { + .read_folio = ntfs_read_folio, + .readahead = ntfs_readahead, + .writepages = ntfs_mft_writepages, + .dirty_folio = iomap_dirty_folio, + .bmap = ntfs_bmap, + .migrate_folio = filemap_migrate_folio, + .is_partially_uptodate = iomap_is_partially_uptodate, + .error_remove_folio = generic_error_remove_folio, + .release_folio = iomap_release_folio, + .invalidate_folio = iomap_invalidate_folio, +}; diff --git a/fs/ntfs/attrib.c b/fs/ntfs/attrib.c new file mode 100644 index 000000000..58f32aac5 --- /dev/null +++ b/fs/ntfs/attrib.c @@ -0,0 +1,5721 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * NTFS attribute operations. + * + * Copyright (c) 2001-2012 Anton Altaparmakov and Tuxera Inc. + * Copyright (c) 2002 Richard Russon + * Copyright (c) 2025 LG Electronics Co., Ltd. + * + * Part of this file is based on code from the NTFS-3G. + * and is copyrighted by the respective authors below: + * Copyright (c) 2000-2010 Anton Altaparmakov + * Copyright (c) 2002-2005 Richard Russon + * Copyright (c) 2002-2008 Szabolcs Szakacsits + * Copyright (c) 2004-2007 Yura Pakhuchiy + * Copyright (c) 2007-2021 Jean-Pierre Andre + * Copyright (c) 2010 Erik Larsson + */ + +#include +#include +#include + +#include "attrib.h" +#include "attrlist.h" +#include "lcnalloc.h" +#include "debug.h" +#include "mft.h" +#include "ntfs.h" +#include "iomap.h" + +__le16 AT_UNNAMED[] = { cpu_to_le16('\0') }; + +/* + * Maximum size allowed for reading attributes by ntfs_attr_readall(). + * Extended attribute, reparse point are not expected to be larger than this size. + */ + +#define NTFS_ATTR_READALL_MAX_SIZE (64 * 1024) + +/* + * ntfs_map_runlist_nolock - map (a part of) a runlist of an ntfs inode + * @ni: ntfs inode for which to map (part of) a runlist + * @vcn: map runlist part containing this vcn + * @ctx: active attribute search context if present or NULL if not + * + * Map the part of a runlist containing the @vcn of the ntfs inode @ni. + * + * If @ctx is specified, it is an active search context of @ni and its base mft + * record. This is needed when ntfs_map_runlist_nolock() encounters unmapped + * runlist fragments and allows their mapping. If you do not have the mft + * record mapped, you can specify @ctx as NULL and ntfs_map_runlist_nolock() + * will perform the necessary mapping and unmapping. + * + * Note, ntfs_map_runlist_nolock() saves the state of @ctx on entry and + * restores it before returning. Thus, @ctx will be left pointing to the same + * attribute on return as on entry. However, the actual pointers in @ctx may + * point to different memory locations on return, so you must remember to reset + * any cached pointers from the @ctx, i.e. after the call to + * ntfs_map_runlist_nolock(), you will probably want to do: + * m = ctx->mrec; + * a = ctx->attr; + * Assuming you cache ctx->attr in a variable @a of type struct attr_record * + * and that you cache ctx->mrec in a variable @m of type struct mft_record *. + * + * Return 0 on success and -errno on error. There is one special error code + * which is not an error as such. This is -ENOENT. It means that @vcn is out + * of bounds of the runlist. + * + * Note the runlist can be NULL after this function returns if @vcn is zero and + * the attribute has zero allocated size, i.e. there simply is no runlist. + * + * WARNING: If @ctx is supplied, regardless of whether success or failure is + * returned, you need to check IS_ERR(@ctx->mrec) and if 'true' the @ctx + * is no longer valid, i.e. you need to either call + * ntfs_attr_reinit_search_ctx() or ntfs_attr_put_search_ctx() on it. + * In that case PTR_ERR(@ctx->mrec) will give you the error code for + * why the mapping of the old inode failed. + * + * Locking: - The runlist described by @ni must be locked for writing on entry + * and is locked on return. Note the runlist will be modified. + * - If @ctx is NULL, the base mft record of @ni must not be mapped on + * entry and it will be left unmapped on return. + * - If @ctx is not NULL, the base mft record must be mapped on entry + * and it will be left mapped on return. + */ +int ntfs_map_runlist_nolock(struct ntfs_inode *ni, s64 vcn, struct ntfs_attr_search_ctx *ctx) +{ + s64 end_vcn; + unsigned long flags; + struct ntfs_inode *base_ni; + struct mft_record *m; + struct attr_record *a; + struct runlist_element *rl; + struct folio *put_this_folio = NULL; + int err = 0; + bool ctx_is_temporary = false, ctx_needs_reset = false; + struct ntfs_attr_search_ctx old_ctx = { NULL, }; + size_t new_rl_count; + + ntfs_debug("Mapping runlist part containing vcn 0x%llx.", + (unsigned long long)vcn); + if (!NInoAttr(ni)) + base_ni = ni; + else + base_ni = ni->ext.base_ntfs_ino; + if (!ctx) { + ctx_is_temporary = ctx_needs_reset = true; + m = map_mft_record(base_ni); + if (IS_ERR(m)) + return PTR_ERR(m); + ctx = ntfs_attr_get_search_ctx(base_ni, m); + if (unlikely(!ctx)) { + err = -ENOMEM; + goto err_out; + } + } else { + s64 allocated_size_vcn; + + WARN_ON(IS_ERR(ctx->mrec)); + a = ctx->attr; + if (!a->non_resident) { + err = -EIO; + goto err_out; + } + end_vcn = le64_to_cpu(a->data.non_resident.highest_vcn); + read_lock_irqsave(&ni->size_lock, flags); + allocated_size_vcn = + ntfs_bytes_to_cluster(ni->vol, ni->allocated_size); + read_unlock_irqrestore(&ni->size_lock, flags); + if (!a->data.non_resident.lowest_vcn && end_vcn <= 0) + end_vcn = allocated_size_vcn - 1; + /* + * If we already have the attribute extent containing @vcn in + * @ctx, no need to look it up again. We slightly cheat in + * that if vcn exceeds the allocated size, we will refuse to + * map the runlist below, so there is definitely no need to get + * the right attribute extent. + */ + if (vcn >= allocated_size_vcn || (a->type == ni->type && + a->name_length == ni->name_len && + !memcmp((u8 *)a + le16_to_cpu(a->name_offset), + ni->name, ni->name_len) && + le64_to_cpu(a->data.non_resident.lowest_vcn) + <= vcn && end_vcn >= vcn)) + ctx_needs_reset = false; + else { + /* Save the old search context. */ + old_ctx = *ctx; + /* + * If the currently mapped (extent) inode is not the + * base inode we will unmap it when we reinitialize the + * search context which means we need to get a + * reference to the page containing the mapped mft + * record so we do not accidentally drop changes to the + * mft record when it has not been marked dirty yet. + */ + if (old_ctx.base_ntfs_ino && old_ctx.ntfs_ino != + old_ctx.base_ntfs_ino) { + put_this_folio = old_ctx.ntfs_ino->folio; + folio_get(put_this_folio); + } + /* + * Reinitialize the search context so we can lookup the + * needed attribute extent. + */ + ntfs_attr_reinit_search_ctx(ctx); + ctx_needs_reset = true; + } + } + if (ctx_needs_reset) { + err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len, + CASE_SENSITIVE, vcn, NULL, 0, ctx); + if (unlikely(err)) { + if (err == -ENOENT) + err = -EIO; + goto err_out; + } + if (unlikely(!ctx->attr->non_resident)) { + err = -EIO; + goto err_out; + } + } + a = ctx->attr; + /* + * Only decompress the mapping pairs if @vcn is inside it. Otherwise + * we get into problems when we try to map an out of bounds vcn because + * we then try to map the already mapped runlist fragment and + * ntfs_mapping_pairs_decompress() fails. + */ + end_vcn = le64_to_cpu(a->data.non_resident.highest_vcn) + 1; + if (unlikely(vcn && vcn >= end_vcn)) { + err = -ENOENT; + goto err_out; + } + rl = ntfs_mapping_pairs_decompress(ni->vol, a, &ni->runlist, &new_rl_count); + if (IS_ERR(rl)) + err = PTR_ERR(rl); + else { + ni->runlist.rl = rl; + ni->runlist.count = new_rl_count; + } +err_out: + if (ctx_is_temporary) { + if (likely(ctx)) + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(base_ni); + } else if (ctx_needs_reset) { + /* + * If there is no attribute list, restoring the search context + * is accomplished simply by copying the saved context back over + * the caller supplied context. If there is an attribute list, + * things are more complicated as we need to deal with mapping + * of mft records and resulting potential changes in pointers. + */ + if (NInoAttrList(base_ni)) { + /* + * If the currently mapped (extent) inode is not the + * one we had before, we need to unmap it and map the + * old one. + */ + if (ctx->ntfs_ino != old_ctx.ntfs_ino) { + /* + * If the currently mapped inode is not the + * base inode, unmap it. + */ + if (ctx->base_ntfs_ino && ctx->ntfs_ino != + ctx->base_ntfs_ino) { + unmap_extent_mft_record(ctx->ntfs_ino); + ctx->mrec = ctx->base_mrec; + WARN_ON(!ctx->mrec); + } + /* + * If the old mapped inode is not the base + * inode, map it. + */ + if (old_ctx.base_ntfs_ino && + old_ctx.ntfs_ino != old_ctx.base_ntfs_ino) { +retry_map: + ctx->mrec = map_mft_record(old_ctx.ntfs_ino); + /* + * Something bad has happened. If out + * of memory retry till it succeeds. + * Any other errors are fatal and we + * return the error code in ctx->mrec. + * Let the caller deal with it... We + * just need to fudge things so the + * caller can reinit and/or put the + * search context safely. + */ + if (IS_ERR(ctx->mrec)) { + if (PTR_ERR(ctx->mrec) == -ENOMEM) { + schedule(); + goto retry_map; + } else + old_ctx.ntfs_ino = + old_ctx.base_ntfs_ino; + } + } + } + /* Update the changed pointers in the saved context. */ + if (ctx->mrec != old_ctx.mrec) { + if (!IS_ERR(ctx->mrec)) + old_ctx.attr = (struct attr_record *)( + (u8 *)ctx->mrec + + ((u8 *)old_ctx.attr - + (u8 *)old_ctx.mrec)); + old_ctx.mrec = ctx->mrec; + } + } + /* Restore the search context to the saved one. */ + *ctx = old_ctx; + /* + * We drop the reference on the page we took earlier. In the + * case that IS_ERR(ctx->mrec) is true this means we might lose + * some changes to the mft record that had been made between + * the last time it was marked dirty/written out and now. This + * at this stage is not a problem as the mapping error is fatal + * enough that the mft record cannot be written out anyway and + * the caller is very likely to shutdown the whole inode + * immediately and mark the volume dirty for chkdsk to pick up + * the pieces anyway. + */ + if (put_this_folio) + folio_put(put_this_folio); + } + return err; +} + +/* + * ntfs_map_runlist - map (a part of) a runlist of an ntfs inode + * @ni: ntfs inode for which to map (part of) a runlist + * @vcn: map runlist part containing this vcn + * + * Map the part of a runlist containing the @vcn of the ntfs inode @ni. + * + * Return 0 on success and -errno on error. There is one special error code + * which is not an error as such. This is -ENOENT. It means that @vcn is out + * of bounds of the runlist. + * + * Locking: - The runlist must be unlocked on entry and is unlocked on return. + * - This function takes the runlist lock for writing and may modify + * the runlist. + */ +int ntfs_map_runlist(struct ntfs_inode *ni, s64 vcn) +{ + int err = 0; + + down_write(&ni->runlist.lock); + /* Make sure someone else didn't do the work while we were sleeping. */ + if (likely(ntfs_rl_vcn_to_lcn(ni->runlist.rl, vcn) <= + LCN_RL_NOT_MAPPED)) + err = ntfs_map_runlist_nolock(ni, vcn, NULL); + up_write(&ni->runlist.lock); + return err; +} + +struct runlist_element *ntfs_attr_vcn_to_rl(struct ntfs_inode *ni, s64 vcn, s64 *lcn) +{ + struct runlist_element *rl = ni->runlist.rl; + int err; + bool is_retry = false; + + if (!rl) { + err = ntfs_attr_map_whole_runlist(ni); + if (err) + return ERR_PTR(-ENOENT); + rl = ni->runlist.rl; + } + +remap_rl: + /* Seek to element containing target vcn. */ + while (rl->length && rl[1].vcn <= vcn) + rl++; + *lcn = ntfs_rl_vcn_to_lcn(rl, vcn); + + if (*lcn <= LCN_RL_NOT_MAPPED && is_retry == false) { + is_retry = true; + if (!ntfs_map_runlist_nolock(ni, vcn, NULL)) { + rl = ni->runlist.rl; + goto remap_rl; + } + } + + return rl; +} + +/* + * ntfs_attr_vcn_to_lcn_nolock - convert a vcn into a lcn given an ntfs inode + * @ni: ntfs inode of the attribute whose runlist to search + * @vcn: vcn to convert + * @write_locked: true if the runlist is locked for writing + * + * Find the virtual cluster number @vcn in the runlist of the ntfs attribute + * described by the ntfs inode @ni and return the corresponding logical cluster + * number (lcn). + * + * If the @vcn is not mapped yet, the attempt is made to map the attribute + * extent containing the @vcn and the vcn to lcn conversion is retried. + * + * If @write_locked is true the caller has locked the runlist for writing and + * if false for reading. + * + * Since lcns must be >= 0, we use negative return codes with special meaning: + * + * Return code Meaning / Description + * ========================================== + * LCN_HOLE Hole / not allocated on disk. + * LCN_ENOENT There is no such vcn in the runlist, i.e. @vcn is out of bounds. + * LCN_ENOMEM Not enough memory to map runlist. + * LCN_EIO Critical error (runlist/file is corrupt, i/o error, etc). + * + * Locking: - The runlist must be locked on entry and is left locked on return. + * - If @write_locked is 'false', i.e. the runlist is locked for reading, + * the lock may be dropped inside the function so you cannot rely on + * the runlist still being the same when this function returns. + */ +s64 ntfs_attr_vcn_to_lcn_nolock(struct ntfs_inode *ni, const s64 vcn, + const bool write_locked) +{ + s64 lcn; + unsigned long flags; + bool is_retry = false; + + ntfs_debug("Entering for i_ino 0x%llx, vcn 0x%llx, %s_locked.", + ni->mft_no, (unsigned long long)vcn, + write_locked ? "write" : "read"); + if (!ni->runlist.rl) { + read_lock_irqsave(&ni->size_lock, flags); + if (!ni->allocated_size) { + read_unlock_irqrestore(&ni->size_lock, flags); + return LCN_ENOENT; + } + read_unlock_irqrestore(&ni->size_lock, flags); + } +retry_remap: + /* Convert vcn to lcn. If that fails map the runlist and retry once. */ + lcn = ntfs_rl_vcn_to_lcn(ni->runlist.rl, vcn); + if (likely(lcn >= LCN_HOLE)) { + ntfs_debug("Done, lcn 0x%llx.", (long long)lcn); + return lcn; + } + if (lcn != LCN_RL_NOT_MAPPED) { + if (lcn != LCN_ENOENT) + lcn = LCN_EIO; + } else if (!is_retry) { + int err; + + if (!write_locked) { + up_read(&ni->runlist.lock); + down_write(&ni->runlist.lock); + if (unlikely(ntfs_rl_vcn_to_lcn(ni->runlist.rl, vcn) != + LCN_RL_NOT_MAPPED)) { + up_write(&ni->runlist.lock); + down_read(&ni->runlist.lock); + goto retry_remap; + } + } + err = ntfs_map_runlist_nolock(ni, vcn, NULL); + if (!write_locked) { + up_write(&ni->runlist.lock); + down_read(&ni->runlist.lock); + } + if (likely(!err)) { + is_retry = true; + goto retry_remap; + } + if (err == -ENOENT) + lcn = LCN_ENOENT; + else if (err == -ENOMEM) + lcn = LCN_ENOMEM; + else + lcn = LCN_EIO; + } + if (lcn != LCN_ENOENT) + ntfs_error(ni->vol->sb, "Failed with error code %lli.", + (long long)lcn); + return lcn; +} + +struct runlist_element *__ntfs_attr_find_vcn_nolock(struct runlist *runlist, const s64 vcn) +{ + size_t lower_idx, upper_idx, idx; + struct runlist_element *run; + int rh = runlist->rl_hint; + + if (runlist->count <= 1) + return ERR_PTR(-ENOENT); + + if (runlist->count - 1 > rh && runlist->rl[rh].vcn <= vcn) { + if (vcn < runlist->rl[rh].vcn + runlist->rl[rh].length) + return &runlist->rl[rh]; + if (runlist->count - 2 == rh) + return ERR_PTR(-ENOENT); + + lower_idx = rh + 1; + } else { + run = &runlist->rl[0]; + if (vcn < run->vcn) + return ERR_PTR(-ENOENT); + else if (vcn < run->vcn + run->length) { + runlist->rl_hint = 0; + return run; + } + + lower_idx = 1; + } + + run = &runlist->rl[runlist->count - 2]; + if (vcn >= run->vcn && vcn < run->vcn + run->length) { + runlist->rl_hint = runlist->count - 2; + return run; + } + if (vcn >= run->vcn + run->length) + return ERR_PTR(-ENOENT); + + upper_idx = runlist->count - 2; + + while (lower_idx <= upper_idx) { + idx = (lower_idx + upper_idx) >> 1; + run = &runlist->rl[idx]; + + if (vcn < run->vcn) + upper_idx = idx - 1; + else if (vcn >= run->vcn + run->length) + lower_idx = idx + 1; + else { + runlist->rl_hint = idx; + return run; + } + } + + return ERR_PTR(-ENOENT); +} + +/* + * ntfs_attr_find_vcn_nolock - find a vcn in the runlist of an ntfs inode + * @ni: ntfs inode describing the runlist to search + * @vcn: vcn to find + * @ctx: active attribute search context if present or NULL if not + * + * Find the virtual cluster number @vcn in the runlist described by the ntfs + * inode @ni and return the address of the runlist element containing the @vcn. + * + * If the @vcn is not mapped yet, the attempt is made to map the attribute + * extent containing the @vcn and the vcn to lcn conversion is retried. + * + * If @ctx is specified, it is an active search context of @ni and its base mft + * record. This is needed when ntfs_attr_find_vcn_nolock() encounters unmapped + * runlist fragments and allows their mapping. If you do not have the mft + * record mapped, you can specify @ctx as NULL and ntfs_attr_find_vcn_nolock() + * will perform the necessary mapping and unmapping. + * + * Note, ntfs_attr_find_vcn_nolock() saves the state of @ctx on entry and + * restores it before returning. Thus, @ctx will be left pointing to the same + * attribute on return as on entry. However, the actual pointers in @ctx may + * point to different memory locations on return, so you must remember to reset + * any cached pointers from the @ctx, i.e. after the call to + * ntfs_attr_find_vcn_nolock(), you will probably want to do: + * m = ctx->mrec; + * a = ctx->attr; + * Assuming you cache ctx->attr in a variable @a of type attr_record * and that + * you cache ctx->mrec in a variable @m of type struct mft_record *. + * Note you need to distinguish between the lcn of the returned runlist element + * being >= 0 and LCN_HOLE. In the later case you have to return zeroes on + * read and allocate clusters on write. + */ +struct runlist_element *ntfs_attr_find_vcn_nolock(struct ntfs_inode *ni, const s64 vcn, + struct ntfs_attr_search_ctx *ctx) +{ + unsigned long flags; + struct runlist_element *rl; + int err = 0; + bool is_retry = false; + + ntfs_debug("Entering for i_ino 0x%llx, vcn 0x%llx, with%s ctx.", + ni->mft_no, (unsigned long long)vcn, ctx ? "" : "out"); + if (!ni->runlist.rl) { + read_lock_irqsave(&ni->size_lock, flags); + if (!ni->allocated_size) { + read_unlock_irqrestore(&ni->size_lock, flags); + return ERR_PTR(-ENOENT); + } + read_unlock_irqrestore(&ni->size_lock, flags); + } + +retry_remap: + rl = ni->runlist.rl; + if (likely(rl && vcn >= rl[0].vcn)) { + rl = __ntfs_attr_find_vcn_nolock(&ni->runlist, vcn); + if (IS_ERR(rl)) + err = PTR_ERR(rl); + else if (rl->lcn >= LCN_HOLE) + return rl; + else if (rl->lcn <= LCN_ENOENT) + err = -EIO; + } + if (!err && !is_retry) { + /* + * If the search context is invalid we cannot map the unmapped + * region. + */ + if (ctx && IS_ERR(ctx->mrec)) + err = PTR_ERR(ctx->mrec); + else { + /* + * The @vcn is in an unmapped region, map the runlist + * and retry. + */ + err = ntfs_map_runlist_nolock(ni, vcn, ctx); + if (likely(!err)) { + is_retry = true; + goto retry_remap; + } + } + if (err == -EINVAL) + err = -EIO; + } else if (!err) + err = -EIO; + if (err != -ENOENT) + ntfs_error(ni->vol->sb, "Failed with error code %i.", err); + return ERR_PTR(err); +} + +static u32 ntfs_resident_attr_min_value_length(const __le32 type) +{ + switch (type) { + case AT_STANDARD_INFORMATION: + return offsetof(struct standard_information, ver) + + sizeof(((struct standard_information *)0)->ver.v1.reserved12); + case AT_FILE_NAME: + return offsetof(struct file_name_attr, file_name) + + sizeof(__le16) * 1; + case AT_VOLUME_INFORMATION: + return sizeof(struct volume_information); + case AT_INDEX_ROOT: + return sizeof(struct index_root); + case AT_EA_INFORMATION: + return sizeof(struct ea_information); + default: + return 0; + } +} + +static bool ntfs_attr_type_is_resident_only(const __le32 type) +{ + switch (type) { + case AT_STANDARD_INFORMATION: + case AT_FILE_NAME: + case AT_OBJECT_ID: + case AT_VOLUME_NAME: + case AT_VOLUME_INFORMATION: + case AT_INDEX_ROOT: + case AT_EA_INFORMATION: + return true; + default: + return false; + } +} + +static bool ntfs_file_name_attr_value_is_valid(const u8 *value, const u32 value_length) +{ + const struct file_name_attr *fn; + u32 file_name_size; + + fn = (const struct file_name_attr *)value; + file_name_size = fn->file_name_length * sizeof(__le16); + + return file_name_size <= + value_length - offsetof(struct file_name_attr, file_name); +} + +static bool ntfs_volume_name_attr_value_is_valid(const u32 value_length) +{ + if (value_length & 1) + return false; + + return value_length <= NTFS_MAX_LABEL_LEN * sizeof(__le16); +} + +static bool ntfs_index_root_attr_value_is_valid(const u8 *value, const u32 value_length) +{ + const struct index_root *ir; + u32 index_size; + u32 entries_offset; + u32 index_length; + u32 allocated_size; + + ir = (const struct index_root *)value; + index_size = value_length - offsetof(struct index_root, index); + entries_offset = le32_to_cpu(ir->index.entries_offset); + index_length = le32_to_cpu(ir->index.index_length); + allocated_size = le32_to_cpu(ir->index.allocated_size); + + if ((entries_offset | index_length | allocated_size) & 7 || + entries_offset < sizeof(struct index_header) || + entries_offset > index_length || + index_length > allocated_size || + allocated_size > index_size || + index_length - entries_offset < sizeof(struct index_entry_header)) + return false; + + return true; +} + +struct ntfs_resident_attr_value { + const u8 *data; + u32 len; +}; + +static bool ntfs_resident_attr_value_get(const struct attr_record *a, + struct ntfs_resident_attr_value *value) +{ + u32 attr_len; + u16 value_offset; + + attr_len = le32_to_cpu(a->length); + if (attr_len < offsetof(struct attr_record, data.resident.reserved) + + sizeof(a->data.resident.reserved)) + return false; + + value->len = le32_to_cpu(a->data.resident.value_length); + value_offset = le16_to_cpu(a->data.resident.value_offset); + + if (value->len > attr_len || value_offset > attr_len - value->len) + return false; + + value->data = (const u8 *)a + value_offset; + return true; +} + +static bool ntfs_non_resident_attr_value_is_valid(const struct attr_record *a) +{ + u32 attr_len; + u32 min_len; + u16 mp_offset; + + attr_len = le32_to_cpu(a->length); + min_len = offsetof(struct attr_record, data.non_resident.initialized_size) + + sizeof(a->data.non_resident.initialized_size); + if (attr_len < min_len) + return false; + + mp_offset = le16_to_cpu(a->data.non_resident.mapping_pairs_offset); + return mp_offset >= min_len && mp_offset <= attr_len; +} + +static bool ntfs_attr_value_is_valid(struct ntfs_volume *vol, + const struct attr_record *a, + const u64 mft_no) +{ + struct ntfs_resident_attr_value value; + u32 min_len; + + if (a->non_resident) { + if (ntfs_attr_type_is_resident_only(a->type)) + goto corrupt; + if (!ntfs_non_resident_attr_value_is_valid(a)) + goto corrupt; + return true; + } + + if (!ntfs_resident_attr_value_get(a, &value)) + goto corrupt; + + min_len = ntfs_resident_attr_min_value_length(a->type); + if (min_len && value.len < min_len) + goto corrupt; + + switch (a->type) { + case AT_FILE_NAME: + if (!ntfs_file_name_attr_value_is_valid(value.data, value.len)) + goto corrupt; + break; + case AT_VOLUME_NAME: + if (!ntfs_volume_name_attr_value_is_valid(value.len)) + goto corrupt; + break; + case AT_INDEX_ROOT: + if (!ntfs_index_root_attr_value_is_valid(value.data, value.len)) + goto corrupt; + break; + } + return true; + +corrupt: + ntfs_error(vol->sb, + "Corrupt %#x attribute in MFT record %llu\n", + le32_to_cpu(a->type), mft_no); + return false; +} + +/* + * ntfs_attr_find - find (next) attribute in mft record + * @type: attribute type to find + * @name: attribute name to find (optional, i.e. NULL means don't care) + * @name_len: attribute name length (only needed if @name present) + * @ic: IGNORE_CASE or CASE_SENSITIVE (ignored if @name not present) + * @val: attribute value to find (optional, resident attributes only) + * @val_len: attribute value length + * @ctx: search context with mft record and attribute to search from + * + * You should not need to call this function directly. Use ntfs_attr_lookup() + * instead. + * + * ntfs_attr_find() takes a search context @ctx as parameter and searches the + * mft record specified by @ctx->mrec, beginning at @ctx->attr, for an + * attribute of @type, optionally @name and @val. + * + * If the attribute is found, ntfs_attr_find() returns 0 and @ctx->attr will + * point to the found attribute. + * + * If the attribute is not found, ntfs_attr_find() returns -ENOENT and + * @ctx->attr will point to the attribute before which the attribute being + * searched for would need to be inserted if such an action were to be desired. + * + * On actual error, ntfs_attr_find() returns -EIO. In this case @ctx->attr is + * undefined and in particular do not rely on it not changing. + * + * If @ctx->is_first is 'true', the search begins with @ctx->attr itself. If it + * is 'false', the search begins after @ctx->attr. + * + * If @ic is IGNORE_CASE, the @name comparisson is not case sensitive and + * @ctx->ntfs_ino must be set to the ntfs inode to which the mft record + * @ctx->mrec belongs. This is so we can get at the ntfs volume and hence at + * the upcase table. If @ic is CASE_SENSITIVE, the comparison is case + * sensitive. When @name is present, @name_len is the @name length in Unicode + * characters. + * + * If @name is not present (NULL), we assume that the unnamed attribute is + * being searched for. + * + * Finally, the resident attribute value @val is looked for, if present. If + * @val is not present (NULL), @val_len is ignored. + * + * ntfs_attr_find() only searches the specified mft record and it ignores the + * presence of an attribute list attribute (unless it is the one being searched + * for, obviously). If you need to take attribute lists into consideration, + * use ntfs_attr_lookup() instead (see below). This also means that you cannot + * use ntfs_attr_find() to search for extent records of non-resident + * attributes, as extents with lowest_vcn != 0 are usually described by the + * attribute list attribute only. - Note that it is possible that the first + * extent is only in the attribute list while the last extent is in the base + * mft record, so do not rely on being able to find the first extent in the + * base mft record. + * + * Warning: Never use @val when looking for attribute types which can be + * non-resident as this most likely will result in a crash! + */ +static int ntfs_attr_find(const __le32 type, const __le16 *name, + const u32 name_len, const u32 ic, + const u8 *val, const u32 val_len, struct ntfs_attr_search_ctx *ctx) +{ + struct attr_record *a; + struct ntfs_volume *vol = ctx->ntfs_ino->vol; + __le16 *upcase = vol->upcase; + u32 upcase_len = vol->upcase_len; + unsigned int space; + u16 name_offset; + u32 attr_len; + u32 name_size; + + /* + * Iterate over attributes in mft record starting at @ctx->attr, or the + * attribute following that, if @ctx->is_first is 'true'. + */ + if (ctx->is_first) { + a = ctx->attr; + ctx->is_first = false; + } else + a = (struct attr_record *)((u8 *)ctx->attr + + le32_to_cpu(ctx->attr->length)); + for (;; a = (struct attr_record *)((u8 *)a + le32_to_cpu(a->length))) { + if ((u8 *)a < (u8 *)ctx->mrec || (u8 *)a > (u8 *)ctx->mrec + + le32_to_cpu(ctx->mrec->bytes_allocated)) + break; + + space = le32_to_cpu(ctx->mrec->bytes_in_use) - ((u8 *)a - (u8 *)ctx->mrec); + if ((space < offsetof(struct attr_record, data.resident.reserved) + 1 || + space < le32_to_cpu(a->length)) && (space < 4 || a->type != AT_END)) + break; + + ctx->attr = a; + if (((type != AT_UNUSED) && (le32_to_cpu(a->type) > le32_to_cpu(type))) || + a->type == AT_END) + return -ENOENT; + if (unlikely(!a->length)) + break; + if (a->name_length) { + name_offset = le16_to_cpu(a->name_offset); + attr_len = le32_to_cpu(a->length); + name_size = a->name_length * sizeof(__le16); + + if (name_offset > attr_len || + attr_len - name_offset < name_size) { + ntfs_error(vol->sb, + "Corrupt attribute name in MFT record %llu\n", + ctx->ntfs_ino->mft_no); + break; + } + } + + if (type == AT_UNUSED) { + if (!ntfs_attr_value_is_valid(vol, a, ctx->ntfs_ino->mft_no)) + break; + return 0; + } + if (a->type != type) + continue; + /* + * If @name is present, compare the two names. If @name is + * missing, assume we want an unnamed attribute. + */ + if (!name || name == AT_UNNAMED) { + /* The search failed if the found attribute is named. */ + if (a->name_length) + return -ENOENT; + } else { + if (!ntfs_are_names_equal(name, name_len, + (__le16 *)((u8 *)a + le16_to_cpu(a->name_offset)), + a->name_length, ic, upcase, upcase_len)) { + register int rc; + + rc = ntfs_collate_names(name, name_len, + (__le16 *)((u8 *)a + le16_to_cpu(a->name_offset)), + a->name_length, 1, IGNORE_CASE, + upcase, upcase_len); + /* + * If @name collates before a->name, there is no + * matching attribute. + */ + if (rc == -1) + return -ENOENT; + /* If the strings are not equal, continue search. */ + if (rc) + continue; + rc = ntfs_collate_names(name, name_len, + (__le16 *)((u8 *)a + le16_to_cpu(a->name_offset)), + a->name_length, 1, CASE_SENSITIVE, + upcase, upcase_len); + if (rc == -1) + return -ENOENT; + if (rc) + continue; + } + } + + if (!ntfs_attr_value_is_valid(vol, a, ctx->ntfs_ino->mft_no)) + break; + + /* + * The names match or @name not present and attribute is + * unnamed. If no @val specified, we have found the attribute + * and are done. + */ + if (!val || a->non_resident) + return 0; + /* @val is present; compare values. */ + else { + u32 value_length = le32_to_cpu(a->data.resident.value_length); + int rc; + + rc = memcmp(val, (u8 *)a + le16_to_cpu( + a->data.resident.value_offset), + min_t(u32, val_len, value_length)); + /* + * If @val collates before the current attribute's + * value, there is no matching attribute. + */ + if (!rc) { + if (val_len == value_length) + return 0; + if (val_len < value_length) + return -ENOENT; + } else if (rc < 0) + return -ENOENT; + } + } + ntfs_error(vol->sb, "mft %#llx, type %#x is corrupt. Run chkdsk.", + (long long)ctx->ntfs_ino->mft_no, le32_to_cpu(type)); + NVolSetErrors(vol); + return -EIO; +} + +void ntfs_attr_name_free(unsigned char **name) +{ + if (*name) { + kfree(*name); + *name = NULL; + } +} + +char *ntfs_attr_name_get(const struct ntfs_volume *vol, const __le16 *uname, + const int uname_len) +{ + unsigned char *name = NULL; + int name_len; + + name_len = ntfs_ucstonls(vol, uname, uname_len, &name, 0); + if (name_len < 0) { + ntfs_error(vol->sb, "ntfs_ucstonls error"); + /* This function when returns -1, memory for name might + * be allocated. So lets free this memory. + */ + ntfs_attr_name_free(&name); + return NULL; + + } else if (name_len > 0) + return name; + + ntfs_attr_name_free(&name); + return NULL; +} + +/* + * ntfs_attr_list_entry_is_valid - sanity check one $ATTRIBUTE_LIST entry + * @ale: the attribute-list entry to check + * @al_end: end of the attribute-list buffer @ale lives in + * + * Verify that @ale is a well-formed attr_list_entry wholly contained in + * [.., @al_end): its fixed header must lie in range before any field is + * dereferenced, its length must be a multiple of 8 that covers the fixed + * header plus the name, the name must lie within the buffer, the entry must + * be in use and carry a live MFT reference. Return true if valid. + */ +bool ntfs_attr_list_entry_is_valid(const struct attr_list_entry *ale, + const u8 *al_end) +{ + const u8 *al = (const u8 *)ale; + u16 ale_len; + + /* The fixed header must be in bounds before it is parsed. */ + if (al + offsetof(struct attr_list_entry, name) > al_end) + return false; + ale_len = le16_to_cpu(ale->length); + /* On-disk entries are 8-byte aligned (see struct attr_list_entry). */ + if (ale_len & 7) + return false; + if (ale->name_offset != sizeof(struct attr_list_entry)) + return false; + if ((u32)ale->name_offset + + (u32)ale->name_length * sizeof(__le16) > ale_len || + al + ale_len > al_end) + return false; + if (ale->type == AT_UNUSED) + return false; + if (MSEQNO_LE(ale->mft_reference) == 0) + return false; + return true; +} + +/* + * ntfs_attr_list_is_valid - sanity check an in-memory $ATTRIBUTE_LIST + * @al_start: start of the attribute list buffer + * @size: length of the attribute list in bytes + * + * Verify that [@al_start, @al_start + @size) is a sequence of valid + * attr_list_entry records (see ntfs_attr_list_entry_is_valid()) that tile the + * buffer exactly. Return true if valid, false otherwise. + */ +bool ntfs_attr_list_is_valid(const u8 *al_start, s64 size) +{ + const u8 *al = al_start; + const u8 *al_end = al_start + size; + + while (al < al_end) { + const struct attr_list_entry *ale = + (const struct attr_list_entry *)al; + + if (!ntfs_attr_list_entry_is_valid(ale, al_end)) + return false; + al += le16_to_cpu(ale->length); + } + return al == al_end; +} + +int load_attribute_list(struct ntfs_inode *base_ni, u8 *al_start, const s64 size) +{ + struct inode *attr_vi = NULL; + + if (!al_start || size <= 0) + return -EINVAL; + + attr_vi = ntfs_attr_iget(VFS_I(base_ni), AT_ATTRIBUTE_LIST, AT_UNNAMED, 0); + if (IS_ERR(attr_vi)) { + ntfs_error(base_ni->vol->sb, + "Failed to open an inode for Attribute list, mft = %llu", + base_ni->mft_no); + return PTR_ERR(attr_vi); + } + + if (ntfs_inode_attr_pread(attr_vi, 0, size, al_start) != size) { + iput(attr_vi); + ntfs_error(base_ni->vol->sb, + "Failed to read attribute list, mft = %llu", + base_ni->mft_no); + return -EIO; + } + iput(attr_vi); + + if (!ntfs_attr_list_is_valid(al_start, size)) { + ntfs_error(base_ni->vol->sb, "Corrupt attribute list, mft = %llu", + base_ni->mft_no); + return -EIO; + } + return 0; +} + +/* + * ntfs_external_attr_find - find an attribute in the attribute list of an inode + * @type: attribute type to find + * @name: attribute name to find (optional, i.e. NULL means don't care) + * @name_len: attribute name length (only needed if @name present) + * @ic: IGNORE_CASE or CASE_SENSITIVE (ignored if @name not present) + * @lowest_vcn: lowest vcn to find (optional, non-resident attributes only) + * @val: attribute value to find (optional, resident attributes only) + * @val_len: attribute value length + * @ctx: search context with mft record and attribute to search from + * + * You should not need to call this function directly. Use ntfs_attr_lookup() + * instead. + * + * Find an attribute by searching the attribute list for the corresponding + * attribute list entry. Having found the entry, map the mft record if the + * attribute is in a different mft record/inode, ntfs_attr_find() the attribute + * in there and return it. + * + * On first search @ctx->ntfs_ino must be the base mft record and @ctx must + * have been obtained from a call to ntfs_attr_get_search_ctx(). On subsequent + * calls @ctx->ntfs_ino can be any extent inode, too (@ctx->base_ntfs_ino is + * then the base inode). + * + * After finishing with the attribute/mft record you need to call + * ntfs_attr_put_search_ctx() to cleanup the search context (unmapping any + * mapped inodes, etc). + * + * If the attribute is found, ntfs_external_attr_find() returns 0 and + * @ctx->attr will point to the found attribute. @ctx->mrec will point to the + * mft record in which @ctx->attr is located and @ctx->al_entry will point to + * the attribute list entry for the attribute. + * + * If the attribute is not found, ntfs_external_attr_find() returns -ENOENT and + * @ctx->attr will point to the attribute in the base mft record before which + * the attribute being searched for would need to be inserted if such an action + * were to be desired. @ctx->mrec will point to the mft record in which + * @ctx->attr is located and @ctx->al_entry will point to the attribute list + * entry of the attribute before which the attribute being searched for would + * need to be inserted if such an action were to be desired. + * + * Thus to insert the not found attribute, one wants to add the attribute to + * @ctx->mrec (the base mft record) and if there is not enough space, the + * attribute should be placed in a newly allocated extent mft record. The + * attribute list entry for the inserted attribute should be inserted in the + * attribute list attribute at @ctx->al_entry. + * + * On actual error, ntfs_external_attr_find() returns -EIO. In this case + * @ctx->attr is undefined and in particular do not rely on it not changing. + */ +static int ntfs_external_attr_find(const __le32 type, + const __le16 *name, const u32 name_len, + const u32 ic, const s64 lowest_vcn, + const u8 *val, const u32 val_len, struct ntfs_attr_search_ctx *ctx) +{ + struct ntfs_inode *base_ni = ctx->base_ntfs_ino, *ni = ctx->ntfs_ino; + struct ntfs_volume *vol; + struct attr_list_entry *al_entry, *next_al_entry; + u8 *al_start, *al_end; + struct attr_record *a; + __le16 *al_name; + u32 al_name_len; + u32 attr_len, mft_free_len; + bool is_first_search = false; + int err = 0; + static const char *es = " Unmount and run chkdsk."; + + ntfs_debug("Entering for inode 0x%llx, type 0x%x.", ni->mft_no, type); + if (!base_ni) { + /* First call happens with the base mft record. */ + base_ni = ctx->base_ntfs_ino = ctx->ntfs_ino; + ctx->base_mrec = ctx->mrec; + ctx->mapped_base_mrec = ctx->mapped_mrec; + } + if (ni == base_ni) + ctx->base_attr = ctx->attr; + if (type == AT_END) + goto not_found; + vol = base_ni->vol; + al_start = base_ni->attr_list; + al_end = al_start + base_ni->attr_list_size; + if (!ctx->al_entry) { + ctx->al_entry = (struct attr_list_entry *)al_start; + is_first_search = true; + } + /* + * Iterate over entries in attribute list starting at @ctx->al_entry, + * or the entry following that, if @ctx->is_first is 'true'. + */ + if (ctx->is_first) { + al_entry = ctx->al_entry; + ctx->is_first = false; + /* + * If an enumeration and the first attribute is higher than + * the attribute list itself, need to return the attribute list + * attribute. + */ + if ((type == AT_UNUSED) && is_first_search && + le32_to_cpu(al_entry->type) > + le32_to_cpu(AT_ATTRIBUTE_LIST)) + goto find_attr_list_attr; + } else { + /* Check for small entry */ + if (((al_end - (u8 *)ctx->al_entry) < + (long)offsetof(struct attr_list_entry, name)) || + (le16_to_cpu(ctx->al_entry->length) & 7) || + (le16_to_cpu(ctx->al_entry->length) < offsetof(struct attr_list_entry, name))) + goto corrupt; + + al_entry = (struct attr_list_entry *)((u8 *)ctx->al_entry + + le16_to_cpu(ctx->al_entry->length)); + + if ((u8 *)al_entry == al_end) + goto not_found; + + /* Preliminary check for small entry */ + if ((al_end - (u8 *)al_entry) < + (long)offsetof(struct attr_list_entry, name)) + goto corrupt; + + /* + * If this is an enumeration and the attribute list attribute + * is the next one in the enumeration sequence, just return the + * attribute list attribute from the base mft record as it is + * not listed in the attribute list itself. + */ + if ((type == AT_UNUSED) && le32_to_cpu(ctx->al_entry->type) < + le32_to_cpu(AT_ATTRIBUTE_LIST) && + le32_to_cpu(al_entry->type) > + le32_to_cpu(AT_ATTRIBUTE_LIST)) { +find_attr_list_attr: + + /* Check for bogus calls. */ + if (name || name_len || val || val_len || lowest_vcn) + return -EINVAL; + + /* We want the base record. */ + if (ctx->ntfs_ino != base_ni) + unmap_mft_record(ctx->ntfs_ino); + ctx->ntfs_ino = base_ni; + ctx->mapped_mrec = ctx->mapped_base_mrec; + ctx->mrec = ctx->base_mrec; + ctx->is_first = true; + + /* Sanity checks are performed elsewhere. */ + ctx->attr = (struct attr_record *)((u8 *)ctx->mrec + + le16_to_cpu(ctx->mrec->attrs_offset)); + + /* Find the attribute list attribute. */ + err = ntfs_attr_find(AT_ATTRIBUTE_LIST, NULL, 0, + IGNORE_CASE, NULL, 0, ctx); + + /* + * Setup the search context so the correct + * attribute is returned next time round. + */ + ctx->al_entry = al_entry; + ctx->is_first = true; + + /* Got it. Done. */ + if (!err) + return 0; + + /* Error! If other than not found return it. */ + if (err != -ENOENT) + return err; + + /* Not found?!? Absurd! */ + ntfs_error(ctx->ntfs_ino->vol->sb, "Attribute list wasn't found"); + return -EIO; + } + } + for (;; al_entry = next_al_entry) { + /* Out of bounds check. */ + if ((u8 *)al_entry < base_ni->attr_list || + (u8 *)al_entry > al_end) + break; /* Inode is corrupt. */ + ctx->al_entry = al_entry; + /* Catch the end of the attribute list. */ + if ((u8 *)al_entry == al_end) + goto not_found; + + if ((((u8 *)al_entry + offsetof(struct attr_list_entry, name)) > al_end) || + ((u8 *)al_entry + le16_to_cpu(al_entry->length) > al_end) || + (le16_to_cpu(al_entry->length) & 7) || + (le16_to_cpu(al_entry->length) < + offsetof(struct attr_list_entry, name_length)) || + (al_entry->name_length && ((u8 *)al_entry + al_entry->name_offset + + al_entry->name_length * sizeof(__le16)) > al_end)) + break; /* corrupt */ + + next_al_entry = (struct attr_list_entry *)((u8 *)al_entry + + le16_to_cpu(al_entry->length)); + if (type != AT_UNUSED) { + if (le32_to_cpu(al_entry->type) > le32_to_cpu(type)) + goto not_found; + if (type != al_entry->type) + continue; + } + /* + * If @name is present, compare the two names. If @name is + * missing, assume we want an unnamed attribute. + */ + al_name_len = al_entry->name_length; + al_name = (__le16 *)((u8 *)al_entry + al_entry->name_offset); + + /* + * If !@type we want the attribute represented by this + * attribute list entry. + */ + if (type == AT_UNUSED) + goto is_enumeration; + + if (!name || name == AT_UNNAMED) { + if (al_name_len) + goto not_found; + } else if (!ntfs_are_names_equal(al_name, al_name_len, name, + name_len, ic, vol->upcase, vol->upcase_len)) { + register int rc; + + rc = ntfs_collate_names(name, name_len, al_name, + al_name_len, 1, IGNORE_CASE, + vol->upcase, vol->upcase_len); + /* + * If @name collates before al_name, there is no + * matching attribute. + */ + if (rc == -1) + goto not_found; + /* If the strings are not equal, continue search. */ + if (rc) + continue; + + rc = ntfs_collate_names(name, name_len, al_name, + al_name_len, 1, CASE_SENSITIVE, + vol->upcase, vol->upcase_len); + if (rc == -1) + goto not_found; + if (rc) + continue; + } + /* + * The names match or @name not present and attribute is + * unnamed. Now check @lowest_vcn. Continue search if the + * next attribute list entry still fits @lowest_vcn. Otherwise + * we have reached the right one or the search has failed. + */ + if (lowest_vcn && (u8 *)next_al_entry >= al_start && + ntfs_attr_list_entry_is_valid(next_al_entry, + al_end) && + le64_to_cpu(next_al_entry->lowest_vcn) <= + lowest_vcn && + next_al_entry->type == al_entry->type && + next_al_entry->name_length == al_name_len && + ntfs_are_names_equal((__le16 *)((u8 *) + next_al_entry + + next_al_entry->name_offset), + next_al_entry->name_length, + al_name, al_name_len, CASE_SENSITIVE, + vol->upcase, vol->upcase_len)) + continue; + +is_enumeration: + if (MREF_LE(al_entry->mft_reference) == ni->mft_no) { + if (MSEQNO_LE(al_entry->mft_reference) != ni->seq_no) { + ntfs_error(vol->sb, + "Found stale mft reference in attribute list of base inode 0x%llx.%s", + base_ni->mft_no, es); + err = -EIO; + break; + } + } else { /* Mft references do not match. */ + /* If there is a mapped record unmap it first. */ + if (ni != base_ni) + unmap_extent_mft_record(ni); + /* Do we want the base record back? */ + if (MREF_LE(al_entry->mft_reference) == + base_ni->mft_no) { + ni = ctx->ntfs_ino = base_ni; + ctx->mrec = ctx->base_mrec; + ctx->mapped_mrec = ctx->mapped_base_mrec; + } else { + /* We want an extent record. */ + ctx->mrec = map_extent_mft_record(base_ni, + le64_to_cpu( + al_entry->mft_reference), &ni); + if (IS_ERR(ctx->mrec)) { + ntfs_error(vol->sb, + "Failed to map extent mft record 0x%lx of base inode 0x%llx.%s", + MREF_LE(al_entry->mft_reference), + base_ni->mft_no, es); + err = PTR_ERR(ctx->mrec); + if (err == -ENOENT) + err = -EIO; + /* Cause @ctx to be sanitized below. */ + ni = NULL; + break; + } + ctx->ntfs_ino = ni; + ctx->mapped_mrec = true; + + } + } + a = ctx->attr = (struct attr_record *)((u8 *)ctx->mrec + + le16_to_cpu(ctx->mrec->attrs_offset)); + /* + * ctx->vfs_ino, ctx->mrec, and ctx->attr now point to the + * mft record containing the attribute represented by the + * current al_entry. + */ + /* + * We could call into ntfs_attr_find() to find the right + * attribute in this mft record but this would be less + * efficient and not quite accurate as ntfs_attr_find() ignores + * the attribute instance numbers for example which become + * important when one plays with attribute lists. Also, + * because a proper match has been found in the attribute list + * entry above, the comparison can now be optimized. So it is + * worth re-implementing a simplified ntfs_attr_find() here. + */ + /* + * Use a manual loop so we can still use break and continue + * with the same meanings as above. + */ +do_next_attr_loop: + if ((u8 *)a < (u8 *)ctx->mrec || + (u8 *)a >= (u8 *)ctx->mrec + le32_to_cpu(ctx->mrec->bytes_allocated) || + (u8 *)a >= (u8 *)ctx->mrec + le32_to_cpu(ctx->mrec->bytes_in_use)) + break; + + mft_free_len = le32_to_cpu(ctx->mrec->bytes_in_use) - + ((u8 *)a - (u8 *)ctx->mrec); + if (mft_free_len >= sizeof(a->type) && a->type == AT_END) + continue; + + attr_len = le32_to_cpu(a->length); + if (!attr_len || + attr_len < offsetof(struct attr_record, data.resident.reserved) + + sizeof(a->data.resident.reserved) || + attr_len > mft_free_len) + break; + + if (al_entry->instance != a->instance) + goto do_next_attr; + /* + * If the type and/or the name are mismatched between the + * attribute list entry and the attribute record, there is + * corruption so we break and return error EIO. + */ + if (al_entry->type != a->type) + break; + if (a->name_length && ((le16_to_cpu(a->name_offset) + + a->name_length * sizeof(__le16)) > attr_len)) + break; + if (!ntfs_are_names_equal((__le16 *)((u8 *)a + + le16_to_cpu(a->name_offset)), a->name_length, + al_name, al_name_len, CASE_SENSITIVE, + vol->upcase, vol->upcase_len)) + break; + + ctx->attr = a; + + if (!ntfs_attr_value_is_valid(vol, a, ctx->ntfs_ino->mft_no)) + break; + + /* + * If no @val specified or @val specified and it matches, we + * have found it! + */ + if ((type == AT_UNUSED) || !val) + goto attr_found; + if (!a->non_resident) { + u32 value_length = le32_to_cpu(a->data.resident.value_length); + u16 value_offset = le16_to_cpu(a->data.resident.value_offset); + + if (value_length == val_len && + !memcmp((u8 *)a + value_offset, val, val_len)) { +attr_found: + ntfs_debug("Done, found."); + return 0; + } + } +do_next_attr: + /* Proceed to the next attribute in the current mft record. */ + a = (struct attr_record *)((u8 *)a + attr_len); + goto do_next_attr_loop; + } + +corrupt: + if (ni != base_ni) { + if (ni) + unmap_extent_mft_record(ni); + ctx->ntfs_ino = base_ni; + ctx->mrec = ctx->base_mrec; + ctx->attr = ctx->base_attr; + ctx->mapped_mrec = ctx->mapped_base_mrec; + } + + if (!err) { + u64 mft_no = ctx->al_entry ? MREF_LE(ctx->al_entry->mft_reference) : 0; + u32 type = ctx->al_entry ? le32_to_cpu(ctx->al_entry->type) : 0; + + ntfs_error(vol->sb, + "Base inode 0x%llx contains corrupt attribute, mft %#llx, type %#x. %s", + (long long)base_ni->mft_no, (long long)mft_no, type, + "Unmount and run chkdsk."); + err = -EIO; + } + + if (err != -ENOMEM) + NVolSetErrors(vol); + return err; +not_found: + /* + * If we were looking for AT_END, we reset the search context @ctx and + * use ntfs_attr_find() to seek to the end of the base mft record. + */ + if (type == AT_UNUSED || type == AT_END) { + ntfs_attr_reinit_search_ctx(ctx); + return ntfs_attr_find(AT_END, name, name_len, ic, val, val_len, + ctx); + } + /* + * The attribute was not found. Before we return, we want to ensure + * @ctx->mrec and @ctx->attr indicate the position at which the + * attribute should be inserted in the base mft record. Since we also + * want to preserve @ctx->al_entry we cannot reinitialize the search + * context using ntfs_attr_reinit_search_ctx() as this would set + * @ctx->al_entry to NULL. Thus we do the necessary bits manually (see + * ntfs_attr_init_search_ctx() below). Note, we _only_ preserve + * @ctx->al_entry as the remaining fields (base_*) are identical to + * their non base_ counterparts and we cannot set @ctx->base_attr + * correctly yet as we do not know what @ctx->attr will be set to by + * the call to ntfs_attr_find() below. + */ + if (ni != base_ni) + unmap_extent_mft_record(ni); + ctx->mrec = ctx->base_mrec; + ctx->attr = (struct attr_record *)((u8 *)ctx->mrec + + le16_to_cpu(ctx->mrec->attrs_offset)); + ctx->is_first = true; + ctx->ntfs_ino = base_ni; + ctx->base_ntfs_ino = NULL; + ctx->base_mrec = NULL; + ctx->base_attr = NULL; + ctx->mapped_mrec = ctx->mapped_base_mrec; + /* + * In case there are multiple matches in the base mft record, need to + * keep enumerating until we get an attribute not found response (or + * another error), otherwise we would keep returning the same attribute + * over and over again and all programs using us for enumeration would + * lock up in a tight loop. + */ + do { + err = ntfs_attr_find(type, name, name_len, ic, val, val_len, + ctx); + } while (!err); + ntfs_debug("Done, not found."); + return err; +} + +/* + * ntfs_attr_lookup - find an attribute in an ntfs inode + * @type: attribute type to find + * @name: attribute name to find (optional, i.e. NULL means don't care) + * @name_len: attribute name length (only needed if @name present) + * @ic: IGNORE_CASE or CASE_SENSITIVE (ignored if @name not present) + * @lowest_vcn: lowest vcn to find (optional, non-resident attributes only) + * @val: attribute value to find (optional, resident attributes only) + * @val_len: attribute value length + * @ctx: search context with mft record and attribute to search from + * + * Find an attribute in an ntfs inode. On first search @ctx->ntfs_ino must + * be the base mft record and @ctx must have been obtained from a call to + * ntfs_attr_get_search_ctx(). + * + * This function transparently handles attribute lists and @ctx is used to + * continue searches where they were left off at. + * + * After finishing with the attribute/mft record you need to call + * ntfs_attr_put_search_ctx() to cleanup the search context (unmapping any + * mapped inodes, etc). + * + * Return 0 if the search was successful and -errno if not. + * + * When 0, @ctx->attr is the found attribute and it is in mft record + * @ctx->mrec. If an attribute list attribute is present, @ctx->al_entry is + * the attribute list entry of the found attribute. + * + * When -ENOENT, @ctx->attr is the attribute which collates just after the + * attribute being searched for, i.e. if one wants to add the attribute to the + * mft record this is the correct place to insert it into. If an attribute + * list attribute is present, @ctx->al_entry is the attribute list entry which + * collates just after the attribute list entry of the attribute being searched + * for, i.e. if one wants to add the attribute to the mft record this is the + * correct place to insert its attribute list entry into. + */ +int ntfs_attr_lookup(const __le32 type, const __le16 *name, + const u32 name_len, const u32 ic, + const s64 lowest_vcn, const u8 *val, const u32 val_len, + struct ntfs_attr_search_ctx *ctx) +{ + struct ntfs_inode *base_ni; + + ntfs_debug("Entering."); + if (ctx->base_ntfs_ino) + base_ni = ctx->base_ntfs_ino; + else + base_ni = ctx->ntfs_ino; + /* Sanity check, just for debugging really. */ + if (!base_ni || !NInoAttrList(base_ni) || type == AT_ATTRIBUTE_LIST) + return ntfs_attr_find(type, name, name_len, ic, val, val_len, + ctx); + return ntfs_external_attr_find(type, name, name_len, ic, lowest_vcn, + val, val_len, ctx); +} + +/** + * ntfs_attr_init_search_ctx - initialize an attribute search context + * @ctx: attribute search context to initialize + * @ni: ntfs inode with which to initialize the search context + * @mrec: mft record with which to initialize the search context + * + * Initialize the attribute search context @ctx with @ni and @mrec. + */ +static bool ntfs_attr_init_search_ctx(struct ntfs_attr_search_ctx *ctx, + struct ntfs_inode *ni, struct mft_record *mrec) +{ + if (!mrec) { + mrec = map_mft_record(ni); + if (IS_ERR(mrec)) + return false; + ctx->mapped_mrec = true; + } else { + ctx->mapped_mrec = false; + } + + ctx->mrec = mrec; + /* Sanity checks are performed elsewhere. */ + ctx->attr = (struct attr_record *)((u8 *)mrec + le16_to_cpu(mrec->attrs_offset)); + ctx->is_first = true; + ctx->ntfs_ino = ni; + ctx->al_entry = NULL; + ctx->base_ntfs_ino = NULL; + ctx->base_mrec = NULL; + ctx->base_attr = NULL; + ctx->mapped_base_mrec = false; + return true; +} + +/* + * ntfs_attr_reinit_search_ctx - reinitialize an attribute search context + * @ctx: attribute search context to reinitialize + * + * Reinitialize the attribute search context @ctx, unmapping an associated + * extent mft record if present, and initialize the search context again. + * + * This is used when a search for a new attribute is being started to reset + * the search context to the beginning. + */ +void ntfs_attr_reinit_search_ctx(struct ntfs_attr_search_ctx *ctx) +{ + bool mapped_mrec; + + if (likely(!ctx->base_ntfs_ino)) { + /* No attribute list. */ + ctx->is_first = true; + /* Sanity checks are performed elsewhere. */ + ctx->attr = (struct attr_record *)((u8 *)ctx->mrec + + le16_to_cpu(ctx->mrec->attrs_offset)); + /* + * This needs resetting due to ntfs_external_attr_find() which + * can leave it set despite having zeroed ctx->base_ntfs_ino. + */ + ctx->al_entry = NULL; + return; + } /* Attribute list. */ + if (ctx->ntfs_ino != ctx->base_ntfs_ino && ctx->ntfs_ino) + unmap_extent_mft_record(ctx->ntfs_ino); + + mapped_mrec = ctx->mapped_base_mrec; + ntfs_attr_init_search_ctx(ctx, ctx->base_ntfs_ino, ctx->base_mrec); + ctx->mapped_mrec = mapped_mrec; +} + +/* + * ntfs_attr_get_search_ctx - allocate/initialize a new attribute search context + * @ni: ntfs inode with which to initialize the search context + * @mrec: mft record with which to initialize the search context + * + * Allocate a new attribute search context, initialize it with @ni and @mrec, + * and return it. Return NULL if allocation failed. + */ +struct ntfs_attr_search_ctx *ntfs_attr_get_search_ctx(struct ntfs_inode *ni, + struct mft_record *mrec) +{ + struct ntfs_attr_search_ctx *ctx; + bool init; + + ctx = kmem_cache_alloc(ntfs_attr_ctx_cache, GFP_NOFS); + if (ctx) { + init = ntfs_attr_init_search_ctx(ctx, ni, mrec); + if (init == false) { + kmem_cache_free(ntfs_attr_ctx_cache, ctx); + ctx = NULL; + } + } + + return ctx; +} + +/* + * ntfs_attr_put_search_ctx - release an attribute search context + * @ctx: attribute search context to free + * + * Release the attribute search context @ctx, unmapping an associated extent + * mft record if present. + */ +void ntfs_attr_put_search_ctx(struct ntfs_attr_search_ctx *ctx) +{ + if (ctx->mapped_mrec) + unmap_mft_record(ctx->ntfs_ino); + + if (ctx->mapped_base_mrec && ctx->base_ntfs_ino && + ctx->ntfs_ino != ctx->base_ntfs_ino) + unmap_extent_mft_record(ctx->base_ntfs_ino); + kmem_cache_free(ntfs_attr_ctx_cache, ctx); +} + +/* + * ntfs_attr_find_in_attrdef - find an attribute in the $AttrDef system file + * @vol: ntfs volume to which the attribute belongs + * @type: attribute type which to find + * + * Search for the attribute definition record corresponding to the attribute + * @type in the $AttrDef system file. + * + * Return the attribute type definition record if found and NULL if not found. + */ +static struct attr_def *ntfs_attr_find_in_attrdef(const struct ntfs_volume *vol, + const __le32 type) +{ + struct attr_def *ad; + + WARN_ON(!type); + for (ad = vol->attrdef; (u8 *)ad - (u8 *)vol->attrdef < + vol->attrdef_size && ad->type; ++ad) { + /* We have not found it yet, carry on searching. */ + if (likely(le32_to_cpu(ad->type) < le32_to_cpu(type))) + continue; + /* We found the attribute; return it. */ + if (likely(ad->type == type)) + return ad; + /* We have gone too far already. No point in continuing. */ + break; + } + /* Attribute not found. */ + ntfs_debug("Attribute type 0x%x not found in $AttrDef.", + le32_to_cpu(type)); + return NULL; +} + +/* + * ntfs_attr_size_bounds_check - check a size of an attribute type for validity + * @vol: ntfs volume to which the attribute belongs + * @type: attribute type which to check + * @size: size which to check + * + * Check whether the @size in bytes is valid for an attribute of @type on the + * ntfs volume @vol. This information is obtained from $AttrDef system file. + */ +int ntfs_attr_size_bounds_check(const struct ntfs_volume *vol, const __le32 type, + const s64 size) +{ + struct attr_def *ad; + + if (size < 0) + return -EINVAL; + + /* + * $ATTRIBUTE_LIST has a maximum size of 256kiB, but this is not + * listed in $AttrDef. + */ + if (unlikely(type == AT_ATTRIBUTE_LIST && size > 256 * 1024)) + return -ERANGE; + /* Get the $AttrDef entry for the attribute @type. */ + ad = ntfs_attr_find_in_attrdef(vol, type); + if (unlikely(!ad)) + return -ENOENT; + /* Do the bounds check. */ + if (((le64_to_cpu(ad->min_size) > 0) && + size < le64_to_cpu(ad->min_size)) || + ((le64_to_cpu(ad->max_size) > 0) && size > + le64_to_cpu(ad->max_size))) + return -ERANGE; + return 0; +} + +/* + * ntfs_attr_can_be_non_resident - check if an attribute can be non-resident + * @vol: ntfs volume to which the attribute belongs + * @type: attribute type which to check + * + * Check whether the attribute of @type on the ntfs volume @vol is allowed to + * be non-resident. This information is obtained from $AttrDef system file. + */ +static int ntfs_attr_can_be_non_resident(const struct ntfs_volume *vol, + const __le32 type) +{ + struct attr_def *ad; + + /* Find the attribute definition record in $AttrDef. */ + ad = ntfs_attr_find_in_attrdef(vol, type); + if (unlikely(!ad)) + return -ENOENT; + /* Check the flags and return the result. */ + if (ad->flags & ATTR_DEF_RESIDENT) + return -EPERM; + return 0; +} + +/* + * ntfs_attr_can_be_resident - check if an attribute can be resident + * @vol: ntfs volume to which the attribute belongs + * @type: attribute type which to check + * + * Check whether the attribute of @type on the ntfs volume @vol is allowed to + * be resident. This information is derived from our ntfs knowledge and may + * not be completely accurate, especially when user defined attributes are + * present. Basically we allow everything to be resident except for index + * allocation and $EA attributes. + * + * Return 0 if the attribute is allowed to be non-resident and -EPERM if not. + * + * Warning: In the system file $MFT the attribute $Bitmap must be non-resident + * otherwise windows will not boot (blue screen of death)! We cannot + * check for this here as we do not know which inode's $Bitmap is + * being asked about so the caller needs to special case this. + */ +int ntfs_attr_can_be_resident(const struct ntfs_volume *vol, const __le32 type) +{ + if (type == AT_INDEX_ALLOCATION) + return -EPERM; + return 0; +} + +/* + * ntfs_attr_record_resize - resize an attribute record + * @m: mft record containing attribute record + * @a: attribute record to resize + * @new_size: new size in bytes to which to resize the attribute record @a + * + * Resize the attribute record @a, i.e. the resident part of the attribute, in + * the mft record @m to @new_size bytes. + */ +int ntfs_attr_record_resize(struct mft_record *m, struct attr_record *a, u32 new_size) +{ + u32 old_size, alloc_size, attr_size; + + old_size = le32_to_cpu(m->bytes_in_use); + alloc_size = le32_to_cpu(m->bytes_allocated); + attr_size = le32_to_cpu(a->length); + + ntfs_debug("Sizes: old=%u alloc=%u attr=%u new=%u\n", + (unsigned int)old_size, (unsigned int)alloc_size, + (unsigned int)attr_size, (unsigned int)new_size); + + /* Align to 8 bytes if it is not already done. */ + if (new_size & 7) + new_size = (new_size + 7) & ~7; + /* If the actual attribute length has changed, move things around. */ + if (new_size != attr_size) { + u32 new_muse = le32_to_cpu(m->bytes_in_use) - + attr_size + new_size; + /* Not enough space in this mft record. */ + if (new_muse > le32_to_cpu(m->bytes_allocated)) + return -ENOSPC; + + if (a->type == AT_INDEX_ROOT && new_size > attr_size && + new_muse + 120 > alloc_size && old_size + 120 <= alloc_size) { + ntfs_debug("Too big struct index_root (%u > %u)\n", + new_muse, alloc_size); + return -ENOSPC; + } + + /* Move attributes following @a to their new location. */ + memmove((u8 *)a + new_size, (u8 *)a + le32_to_cpu(a->length), + le32_to_cpu(m->bytes_in_use) - ((u8 *)a - + (u8 *)m) - attr_size); + /* Adjust @m to reflect the change in used space. */ + m->bytes_in_use = cpu_to_le32(new_muse); + /* Adjust @a to reflect the new size. */ + if (new_size >= offsetof(struct attr_record, length) + sizeof(a->length)) + a->length = cpu_to_le32(new_size); + } + return 0; +} + +/* + * ntfs_resident_attr_value_resize - resize the value of a resident attribute + * @m: mft record containing attribute record + * @a: attribute record whose value to resize + * @new_size: new size in bytes to which to resize the attribute value of @a + * + * Resize the value of the attribute @a in the mft record @m to @new_size bytes. + * If the value is made bigger, the newly allocated space is cleared. + */ +int ntfs_resident_attr_value_resize(struct mft_record *m, struct attr_record *a, + const u32 new_size) +{ + u32 old_size; + + /* Resize the resident part of the attribute record. */ + if (ntfs_attr_record_resize(m, a, + le16_to_cpu(a->data.resident.value_offset) + new_size)) + return -ENOSPC; + /* + * The resize succeeded! If we made the attribute value bigger, clear + * the area between the old size and @new_size. + */ + old_size = le32_to_cpu(a->data.resident.value_length); + if (new_size > old_size) + memset((u8 *)a + le16_to_cpu(a->data.resident.value_offset) + + old_size, 0, new_size - old_size); + /* Finally update the length of the attribute value. */ + a->data.resident.value_length = cpu_to_le32(new_size); + return 0; +} + +/* + * ntfs_attr_make_non_resident - convert a resident to a non-resident attribute + * @ni: ntfs inode describing the attribute to convert + * @data_size: size of the resident data to copy to the non-resident attribute + * + * Convert the resident ntfs attribute described by the ntfs inode @ni to a + * non-resident one. + * + * @data_size must be equal to the attribute value size. This is needed since + * we need to know the size before we can map the mft record and our callers + * always know it. The reason we cannot simply read the size from the vfs + * inode i_size is that this is not necessarily uptodate. This happens when + * ntfs_attr_make_non_resident() is called in the ->truncate call path(s). + */ +int ntfs_attr_make_non_resident(struct ntfs_inode *ni, const u32 data_size) +{ + s64 new_size; + struct inode *vi = VFS_I(ni); + struct ntfs_volume *vol = ni->vol; + struct ntfs_inode *base_ni; + struct mft_record *m; + struct attr_record *a; + struct ntfs_attr_search_ctx *ctx; + struct folio *folio; + struct runlist_element *rl; + unsigned long flags; + int mp_size, mp_ofs, name_ofs, arec_size, err, err2; + u32 attr_size; + u8 old_res_attr_flags; + + if (NInoNonResident(ni)) { + ntfs_warning(vol->sb, + "Trying to make non-resident attribute non-resident. Aborting...\n"); + return -EINVAL; + } + + /* Check that the attribute is allowed to be non-resident. */ + err = ntfs_attr_can_be_non_resident(vol, ni->type); + if (unlikely(err)) { + if (err == -EPERM) + ntfs_debug("Attribute is not allowed to be non-resident."); + else + ntfs_debug("Attribute not defined on the NTFS volume!"); + return err; + } + + if (NInoEncrypted(ni)) + return -EIO; + + if (!NInoAttr(ni)) + base_ni = ni; + else + base_ni = ni->ext.base_ntfs_ino; + m = map_mft_record(base_ni); + if (IS_ERR(m)) { + err = PTR_ERR(m); + m = NULL; + ctx = NULL; + goto err_out; + } + ctx = ntfs_attr_get_search_ctx(base_ni, m); + if (unlikely(!ctx)) { + err = -ENOMEM; + goto err_out; + } + err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len, + CASE_SENSITIVE, 0, NULL, 0, ctx); + if (unlikely(err)) { + if (err == -ENOENT) + err = -EIO; + goto err_out; + } + m = ctx->mrec; + a = ctx->attr; + + /* + * The size needs to be aligned to a cluster boundary for allocation + * purposes. + */ + new_size = (data_size + vol->cluster_size - 1) & + ~(vol->cluster_size - 1); + if (new_size > 0) { + if ((a->flags & ATTR_COMPRESSION_MASK) == ATTR_IS_COMPRESSED) { + /* must allocate full compression blocks */ + new_size = + ((new_size - 1) | + ((1L << (STANDARD_COMPRESSION_UNIT + + vol->cluster_size_bits)) - 1)) + 1; + } + + /* + * Will need folio later and since folio lock nests + * outside all ntfs locks, we need to get the folio now. + */ + folio = __filemap_get_folio(vi->i_mapping, 0, + FGP_CREAT | FGP_LOCK, + mapping_gfp_mask(vi->i_mapping)); + if (IS_ERR(folio)) { + err = -ENOMEM; + goto err_out; + } + + /* Start by allocating clusters to hold the attribute value. */ + rl = ntfs_cluster_alloc(vol, 0, + ntfs_bytes_to_cluster(vol, new_size), + -1, DATA_ZONE, true, false, false); + if (IS_ERR(rl)) { + err = PTR_ERR(rl); + ntfs_debug("Failed to allocate cluster%s, error code %i.", + str_plural(ntfs_bytes_to_cluster(vol, new_size)), + err); + goto folio_err_out; + } + } else { + rl = NULL; + folio = NULL; + } + + down_write(&ni->runlist.lock); + /* Determine the size of the mapping pairs array. */ + mp_size = ntfs_get_size_for_mapping_pairs(vol, rl, 0, -1, -1); + if (unlikely(mp_size < 0)) { + err = mp_size; + ntfs_debug("Failed to get size for mapping pairs array, error code %i.\n", err); + goto rl_err_out; + } + + if (NInoNonResident(ni) || a->non_resident) { + err = -EIO; + goto rl_err_out; + } + + /* + * Calculate new offsets for the name and the mapping pairs array. + */ + if (NInoSparse(ni) || NInoCompressed(ni)) + name_ofs = (offsetof(struct attr_record, + data.non_resident.compressed_size) + + sizeof(a->data.non_resident.compressed_size) + + 7) & ~7; + else + name_ofs = (offsetof(struct attr_record, + data.non_resident.compressed_size) + 7) & ~7; + mp_ofs = (name_ofs + a->name_length * sizeof(__le16) + 7) & ~7; + /* + * Determine the size of the resident part of the now non-resident + * attribute record. + */ + arec_size = (mp_ofs + mp_size + 7) & ~7; + /* + * If the folio is not uptodate bring it uptodate by copying from the + * attribute value. + */ + attr_size = le32_to_cpu(a->data.resident.value_length); + WARN_ON(attr_size != data_size); + if (folio && !folio_test_uptodate(folio)) { + folio_fill_tail(folio, 0, (u8 *)a + + le16_to_cpu(a->data.resident.value_offset), + attr_size); + folio_mark_uptodate(folio); + } + + /* Backup the attribute flag. */ + old_res_attr_flags = a->data.resident.flags; + /* Resize the resident part of the attribute record. */ + err = ntfs_attr_record_resize(m, a, arec_size); + if (unlikely(err)) + goto rl_err_out; + + /* + * Convert the resident part of the attribute record to describe a + * non-resident attribute. + */ + a->non_resident = 1; + /* Move the attribute name if it exists and update the offset. */ + if (a->name_length) + memmove((u8 *)a + name_ofs, (u8 *)a + le16_to_cpu(a->name_offset), + a->name_length * sizeof(__le16)); + a->name_offset = cpu_to_le16(name_ofs); + /* Setup the fields specific to non-resident attributes. */ + a->data.non_resident.lowest_vcn = 0; + a->data.non_resident.highest_vcn = + cpu_to_le64(ntfs_bytes_to_cluster(vol, new_size - 1)); + a->data.non_resident.mapping_pairs_offset = cpu_to_le16(mp_ofs); + memset(&a->data.non_resident.reserved, 0, + sizeof(a->data.non_resident.reserved)); + a->data.non_resident.allocated_size = cpu_to_le64(new_size); + a->data.non_resident.data_size = + a->data.non_resident.initialized_size = + cpu_to_le64(attr_size); + if (NInoSparse(ni) || NInoCompressed(ni)) { + a->data.non_resident.compression_unit = 0; + if (NInoCompressed(ni) || vol->major_ver < 3) + a->data.non_resident.compression_unit = 4; + a->data.non_resident.compressed_size = + a->data.non_resident.allocated_size; + } else + a->data.non_resident.compression_unit = 0; + /* Generate the mapping pairs array into the attribute record. */ + err = ntfs_mapping_pairs_build(vol, (u8 *)a + mp_ofs, + arec_size - mp_ofs, rl, 0, -1, NULL, NULL, NULL); + if (unlikely(err)) { + ntfs_error(vol->sb, "Failed to build mapping pairs, error code %i.", + err); + goto undo_err_out; + } + + /* Setup the in-memory attribute structure to be non-resident. */ + ni->runlist.rl = rl; + if (rl) { + for (ni->runlist.count = 1; rl->length != 0; rl++) + ni->runlist.count++; + } else + ni->runlist.count = 0; + write_lock_irqsave(&ni->size_lock, flags); + ni->allocated_size = new_size; + if (NInoSparse(ni) || NInoCompressed(ni)) { + ni->itype.compressed.size = ni->allocated_size; + if (a->data.non_resident.compression_unit) { + ni->itype.compressed.block_size = 1U << + (a->data.non_resident.compression_unit + + vol->cluster_size_bits); + ni->itype.compressed.block_size_bits = + ffs(ni->itype.compressed.block_size) - + 1; + ni->itype.compressed.block_clusters = 1U << + a->data.non_resident.compression_unit; + } else { + ni->itype.compressed.block_size = 0; + ni->itype.compressed.block_size_bits = 0; + ni->itype.compressed.block_clusters = 0; + } + vi->i_blocks = ni->itype.compressed.size >> 9; + } else + vi->i_blocks = ni->allocated_size >> 9; + write_unlock_irqrestore(&ni->size_lock, flags); + /* + * This needs to be last since the address space operations ->read_folio + * and ->writepage can run concurrently with us as they are not + * serialized on i_mutex. Note, we are not allowed to fail once we flip + * this switch, which is another reason to do this last. + */ + NInoSetNonResident(ni); + NInoSetFullyMapped(ni); + /* Mark the mft record dirty, so it gets written back. */ + mark_mft_record_dirty(ctx->ntfs_ino); + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(base_ni); + up_write(&ni->runlist.lock); + if (folio) { + iomap_dirty_folio(vi->i_mapping, folio); + folio_unlock(folio); + folio_put(folio); + } + ntfs_debug("Done."); + return 0; +undo_err_out: + /* Convert the attribute back into a resident attribute. */ + a->non_resident = 0; + /* Move the attribute name if it exists and update the offset. */ + name_ofs = (offsetof(struct attr_record, data.resident.reserved) + + sizeof(a->data.resident.reserved) + 7) & ~7; + if (a->name_length) + memmove((u8 *)a + name_ofs, (u8 *)a + le16_to_cpu(a->name_offset), + a->name_length * sizeof(__le16)); + mp_ofs = (name_ofs + a->name_length * sizeof(__le16) + 7) & ~7; + a->name_offset = cpu_to_le16(name_ofs); + arec_size = (mp_ofs + attr_size + 7) & ~7; + /* Resize the resident part of the attribute record. */ + err2 = ntfs_attr_record_resize(m, a, arec_size); + if (unlikely(err2)) { + /* + * This cannot happen (well if memory corruption is at work it + * could happen in theory), but deal with it as well as we can. + * If the old size is too small, truncate the attribute, + * otherwise simply give it a larger allocated size. + */ + arec_size = le32_to_cpu(a->length); + if ((mp_ofs + attr_size) > arec_size) { + err2 = attr_size; + attr_size = arec_size - mp_ofs; + ntfs_error(vol->sb, + "Failed to undo partial resident to non-resident attribute conversion. Truncating inode 0x%llx, attribute type 0x%x from %i bytes to %i bytes to maintain metadata consistency. THIS MEANS YOU ARE LOSING %i BYTES DATA FROM THIS %s.", + ni->mft_no, + (unsigned int)le32_to_cpu(ni->type), + err2, attr_size, err2 - attr_size, + ((ni->type == AT_DATA) && + !ni->name_len) ? "FILE" : "ATTRIBUTE"); + write_lock_irqsave(&ni->size_lock, flags); + ni->initialized_size = attr_size; + i_size_write(vi, attr_size); + write_unlock_irqrestore(&ni->size_lock, flags); + } + } + /* Setup the fields specific to resident attributes. */ + a->data.resident.value_length = cpu_to_le32(attr_size); + a->data.resident.value_offset = cpu_to_le16(mp_ofs); + a->data.resident.flags = old_res_attr_flags; + memset(&a->data.resident.reserved, 0, + sizeof(a->data.resident.reserved)); + /* Copy the data from folio back to the attribute value. */ + if (folio) + memcpy_from_folio((u8 *)a + mp_ofs, folio, 0, attr_size); + /* Setup the allocated size in the ntfs inode in case it changed. */ + write_lock_irqsave(&ni->size_lock, flags); + ni->allocated_size = arec_size - mp_ofs; + write_unlock_irqrestore(&ni->size_lock, flags); + /* Mark the mft record dirty, so it gets written back. */ + mark_mft_record_dirty(ctx->ntfs_ino); +rl_err_out: + up_write(&ni->runlist.lock); + if (rl) { + if (ntfs_cluster_free_from_rl(vol, rl) < 0) { + ntfs_error(vol->sb, + "Failed to release allocated cluster(s) in error code path. Run chkdsk to recover the lost cluster(s)."); + NVolSetErrors(vol); + } + kvfree(rl); +folio_err_out: + folio_unlock(folio); + folio_put(folio); + } +err_out: + if (ctx) + ntfs_attr_put_search_ctx(ctx); + if (m) + unmap_mft_record(base_ni); + ni->runlist.rl = NULL; + + if (err == -EINVAL) + err = -EIO; + return err; +} + +/* + * ntfs_attr_set - fill (a part of) an attribute with a byte + * @ni: ntfs inode describing the attribute to fill + * @ofs: offset inside the attribute at which to start to fill + * @cnt: number of bytes to fill + * @val: the unsigned 8-bit value with which to fill the attribute + * + * Fill @cnt bytes of the attribute described by the ntfs inode @ni starting at + * byte offset @ofs inside the attribute with the constant byte @val. + * + * This function is effectively like memset() applied to an ntfs attribute. + * Note thie function actually only operates on the page cache pages belonging + * to the ntfs attribute and it marks them dirty after doing the memset(). + * Thus it relies on the vm dirty page write code paths to cause the modified + * pages to be written to the mft record/disk. + */ +int ntfs_attr_set(struct ntfs_inode *ni, s64 ofs, s64 cnt, const u8 val) +{ + struct address_space *mapping = VFS_I(ni)->i_mapping; + struct folio *folio; + pgoff_t index; + u8 *addr; + unsigned long offset; + size_t attr_len; + int ret = 0; + + index = ofs >> PAGE_SHIFT; + while (cnt) { + folio = read_mapping_folio(mapping, index, NULL); + if (IS_ERR(folio)) { + ret = PTR_ERR(folio); + ntfs_error(VFS_I(ni)->i_sb, "Failed to read a page %lu for attr %#x: %ld", + index, ni->type, PTR_ERR(folio)); + break; + } + + offset = offset_in_folio(folio, ofs); + attr_len = min_t(size_t, (size_t)cnt, folio_size(folio) - offset); + + folio_lock(folio); + addr = kmap_local_folio(folio, offset); + memset(addr, val, attr_len); + kunmap_local(addr); + + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + + ofs += attr_len; + cnt -= attr_len; + index++; + cond_resched(); + } + + return ret; +} + +int ntfs_attr_set_initialized_size(struct ntfs_inode *ni, loff_t new_size) +{ + struct ntfs_attr_search_ctx *ctx; + int err = 0; + + if (!NInoNonResident(ni)) + return -EINVAL; + + ctx = ntfs_attr_get_search_ctx(ni, NULL); + if (!ctx) + return -ENOMEM; + + err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len, + CASE_SENSITIVE, 0, NULL, 0, ctx); + if (err) + goto out_ctx; + + ctx->attr->data.non_resident.initialized_size = cpu_to_le64(new_size); + ni->initialized_size = new_size; + mark_mft_record_dirty(ctx->ntfs_ino); +out_ctx: + ntfs_attr_put_search_ctx(ctx); + return err; +} + +/* + * ntfs_make_room_for_attr - make room for an attribute inside an mft record + * @m: mft record + * @pos: position at which to make space + * @size: byte size to make available at this position + * + * @pos points to the attribute in front of which we want to make space. + */ +static int ntfs_make_room_for_attr(struct mft_record *m, u8 *pos, u32 size) +{ + u32 biu; + + ntfs_debug("Entering for pos 0x%x, size %u.\n", + (int)(pos - (u8 *)m), (unsigned int) size); + + /* Make size 8-byte alignment. */ + size = (size + 7) & ~7; + + /* Rigorous consistency checks. */ + if (!m || !pos || pos < (u8 *)m) { + pr_err("%s: pos=%p m=%p\n", __func__, pos, m); + return -EINVAL; + } + + /* The -8 is for the attribute terminator. */ + if (pos - (u8 *)m > (int)le32_to_cpu(m->bytes_in_use) - 8) + return -EINVAL; + /* Nothing to do. */ + if (!size) + return 0; + + biu = le32_to_cpu(m->bytes_in_use); + /* Do we have enough space? */ + if (biu + size > le32_to_cpu(m->bytes_allocated) || + pos + size > (u8 *)m + le32_to_cpu(m->bytes_allocated)) { + ntfs_debug("No enough space in the MFT record\n"); + return -ENOSPC; + } + /* Move everything after pos to pos + size. */ + memmove(pos + size, pos, biu - (pos - (u8 *)m)); + /* Update mft record. */ + m->bytes_in_use = cpu_to_le32(biu + size); + return 0; +} + +/* + * ntfs_resident_attr_record_add - add resident attribute to inode + * @ni: opened ntfs inode to which MFT record add attribute + * @type: type of the new attribute + * @name: name of the new attribute + * @name_len: name length of the new attribute + * @val: value of the new attribute + * @size: size of new attribute (length of @val, if @val != NULL) + * @flags: flags of the new attribute + */ +int ntfs_resident_attr_record_add(struct ntfs_inode *ni, __le32 type, + __le16 *name, u8 name_len, u8 *val, u32 size, + __le16 flags) +{ + struct ntfs_attr_search_ctx *ctx; + u32 length; + struct attr_record *a; + struct mft_record *m; + int err, offset; + struct ntfs_inode *base_ni; + + if (!ni || (!name && name_len)) + return -EINVAL; + + ntfs_debug("Entering for inode 0x%llx, attr 0x%x, flags 0x%x.\n", + (long long) ni->mft_no, (unsigned int) le32_to_cpu(type), + (unsigned int) le16_to_cpu(flags)); + + err = ntfs_attr_can_be_resident(ni->vol, type); + if (err) { + if (err == -EPERM) + ntfs_debug("Attribute can't be resident.\n"); + else + ntfs_debug("ntfs_attr_can_be_resident failed.\n"); + return err; + } + + /* Locate place where record should be. */ + ctx = ntfs_attr_get_search_ctx(ni, NULL); + if (!ctx) { + ntfs_error(ni->vol->sb, "%s: Failed to get search context", + __func__); + return -ENOMEM; + } + /* + * Use ntfs_attr_find instead of ntfs_attr_lookup to find place for + * attribute in @ni->mrec, not any extent inode in case if @ni is base + * file record. + */ + err = ntfs_attr_find(type, name, name_len, CASE_SENSITIVE, val, size, ctx); + if (!err) { + err = -EEXIST; + ntfs_debug("Attribute already present.\n"); + goto put_err_out; + } + if (err != -ENOENT) { + err = -EIO; + goto put_err_out; + } + a = ctx->attr; + m = ctx->mrec; + + /* Make room for attribute. */ + length = offsetof(struct attr_record, data.resident.reserved) + + sizeof(a->data.resident.reserved) + + ((name_len * sizeof(__le16) + 7) & ~7) + + ((size + 7) & ~7); + err = ntfs_make_room_for_attr(ctx->mrec, (u8 *) ctx->attr, length); + if (err) { + ntfs_debug("Failed to make room for attribute.\n"); + goto put_err_out; + } + + /* Setup record fields. */ + offset = ((u8 *)a - (u8 *)m); + a->type = type; + a->length = cpu_to_le32(length); + a->non_resident = 0; + a->name_length = name_len; + a->name_offset = + name_len ? cpu_to_le16((offsetof(struct attr_record, data.resident.reserved) + + sizeof(a->data.resident.reserved))) : cpu_to_le16(0); + + a->flags = flags; + a->instance = m->next_attr_instance; + a->data.resident.value_length = cpu_to_le32(size); + a->data.resident.value_offset = cpu_to_le16(length - ((size + 7) & ~7)); + if (val) + memcpy((u8 *)a + le16_to_cpu(a->data.resident.value_offset), val, size); + else + memset((u8 *)a + le16_to_cpu(a->data.resident.value_offset), 0, size); + if (type == AT_FILE_NAME) + a->data.resident.flags = RESIDENT_ATTR_IS_INDEXED; + else + a->data.resident.flags = 0; + if (name_len) + memcpy((u8 *)a + le16_to_cpu(a->name_offset), + name, sizeof(__le16) * name_len); + m->next_attr_instance = + cpu_to_le16((le16_to_cpu(m->next_attr_instance) + 1) & 0xffff); + if (ni->nr_extents == -1) + base_ni = ni->ext.base_ntfs_ino; + else + base_ni = ni; + if (type != AT_ATTRIBUTE_LIST && NInoAttrList(base_ni)) { + err = ntfs_attrlist_entry_add(ni, a); + if (err) { + ntfs_attr_record_resize(m, a, 0); + mark_mft_record_dirty(ctx->ntfs_ino); + ntfs_debug("Failed add attribute entry to ATTRIBUTE_LIST.\n"); + goto put_err_out; + } + } + mark_mft_record_dirty(ni); + ntfs_attr_put_search_ctx(ctx); + return offset; +put_err_out: + ntfs_attr_put_search_ctx(ctx); + return -EIO; +} + +/* + * ntfs_non_resident_attr_record_add - add extent of non-resident attribute + * @ni: opened ntfs inode to which MFT record add attribute + * @type: type of the new attribute extent + * @name: name of the new attribute extent + * @name_len: name length of the new attribute extent + * @lowest_vcn: lowest vcn of the new attribute extent + * @dataruns_size: dataruns size of the new attribute extent + * @flags: flags of the new attribute extent + */ +static int ntfs_non_resident_attr_record_add(struct ntfs_inode *ni, __le32 type, + __le16 *name, u8 name_len, s64 lowest_vcn, int dataruns_size, + __le16 flags) +{ + struct ntfs_attr_search_ctx *ctx; + u32 length; + struct attr_record *a; + struct mft_record *m; + struct ntfs_inode *base_ni; + int err, offset; + + if (!ni || dataruns_size <= 0 || (!name && name_len)) + return -EINVAL; + + ntfs_debug("Entering for inode 0x%llx, attr 0x%x, lowest_vcn %lld, dataruns_size %d, flags 0x%x.\n", + (long long) ni->mft_no, (unsigned int) le32_to_cpu(type), + (long long) lowest_vcn, dataruns_size, + (unsigned int) le16_to_cpu(flags)); + + err = ntfs_attr_can_be_non_resident(ni->vol, type); + if (err) { + if (err == -EPERM) + pr_err("Attribute can't be non resident\n"); + else + pr_err("ntfs_attr_can_be_non_resident failed\n"); + return err; + } + + /* Locate place where record should be. */ + ctx = ntfs_attr_get_search_ctx(ni, NULL); + if (!ctx) { + pr_err("%s: Failed to get search context\n", __func__); + return -ENOMEM; + } + /* + * Use ntfs_attr_find instead of ntfs_attr_lookup to find place for + * attribute in @ni->mrec, not any extent inode in case if @ni is base + * file record. + */ + err = ntfs_attr_find(type, name, name_len, CASE_SENSITIVE, NULL, 0, ctx); + if (!err) { + err = -EEXIST; + pr_err("Attribute 0x%x already present\n", type); + goto put_err_out; + } + if (err != -ENOENT) { + pr_err("ntfs_attr_find failed\n"); + err = -EIO; + goto put_err_out; + } + a = ctx->attr; + m = ctx->mrec; + + /* Make room for attribute. */ + dataruns_size = (dataruns_size + 7) & ~7; + length = offsetof(struct attr_record, data.non_resident.compressed_size) + + ((sizeof(__le16) * name_len + 7) & ~7) + dataruns_size + + ((flags & (ATTR_IS_COMPRESSED | ATTR_IS_SPARSE)) ? + sizeof(a->data.non_resident.compressed_size) : 0); + err = ntfs_make_room_for_attr(ctx->mrec, (u8 *) ctx->attr, length); + if (err) { + pr_err("Failed to make room for attribute\n"); + goto put_err_out; + } + + /* Setup record fields. */ + a->type = type; + a->length = cpu_to_le32(length); + a->non_resident = 1; + a->name_length = name_len; + a->name_offset = cpu_to_le16(offsetof(struct attr_record, + data.non_resident.compressed_size) + + ((flags & (ATTR_IS_COMPRESSED | ATTR_IS_SPARSE)) ? + sizeof(a->data.non_resident.compressed_size) : 0)); + a->flags = flags; + a->instance = m->next_attr_instance; + a->data.non_resident.lowest_vcn = cpu_to_le64(lowest_vcn); + a->data.non_resident.mapping_pairs_offset = cpu_to_le16(length - dataruns_size); + a->data.non_resident.compression_unit = + (flags & ATTR_IS_COMPRESSED) ? STANDARD_COMPRESSION_UNIT : 0; + /* If @lowest_vcn == 0, than setup empty attribute. */ + if (!lowest_vcn) { + a->data.non_resident.highest_vcn = cpu_to_le64(-1); + a->data.non_resident.allocated_size = 0; + a->data.non_resident.data_size = 0; + a->data.non_resident.initialized_size = 0; + /* Set empty mapping pairs. */ + *((u8 *)a + le16_to_cpu(a->data.non_resident.mapping_pairs_offset)) = 0; + } + if (name_len) + memcpy((u8 *)a + le16_to_cpu(a->name_offset), + name, sizeof(__le16) * name_len); + m->next_attr_instance = + cpu_to_le16((le16_to_cpu(m->next_attr_instance) + 1) & 0xffff); + if (ni->nr_extents == -1) + base_ni = ni->ext.base_ntfs_ino; + else + base_ni = ni; + if (type != AT_ATTRIBUTE_LIST && NInoAttrList(base_ni)) { + err = ntfs_attrlist_entry_add(ni, a); + if (err) { + pr_err("Failed add attr entry to attrlist\n"); + ntfs_attr_record_resize(m, a, 0); + goto put_err_out; + } + } + mark_mft_record_dirty(ni); + /* + * Locate offset from start of the MFT record where new attribute is + * placed. We need relookup it, because record maybe moved during + * update of attribute list. + */ + ntfs_attr_reinit_search_ctx(ctx); + err = ntfs_attr_lookup(type, name, name_len, CASE_SENSITIVE, + lowest_vcn, NULL, 0, ctx); + if (err) { + pr_err("%s: attribute lookup failed\n", __func__); + ntfs_attr_put_search_ctx(ctx); + return err; + + } + offset = (u8 *)ctx->attr - (u8 *)ctx->mrec; + ntfs_attr_put_search_ctx(ctx); + return offset; +put_err_out: + ntfs_attr_put_search_ctx(ctx); + return -1; +} + +/* + * ntfs_attr_record_rm - remove attribute extent + * @ctx: search context describing the attribute which should be removed + * + * If this function succeed, user should reinit search context if he/she wants + * use it anymore. + */ +int ntfs_attr_record_rm(struct ntfs_attr_search_ctx *ctx) +{ + struct ntfs_inode *base_ni, *ni; + __le32 type; + int err; + + if (!ctx || !ctx->ntfs_ino || !ctx->mrec || !ctx->attr) + return -EINVAL; + + ntfs_debug("Entering for inode 0x%llx, attr 0x%x.\n", + (long long) ctx->ntfs_ino->mft_no, + (unsigned int) le32_to_cpu(ctx->attr->type)); + type = ctx->attr->type; + ni = ctx->ntfs_ino; + if (ctx->base_ntfs_ino) + base_ni = ctx->base_ntfs_ino; + else + base_ni = ctx->ntfs_ino; + + /* Remove attribute itself. */ + if (ntfs_attr_record_resize(ctx->mrec, ctx->attr, 0)) { + ntfs_debug("Couldn't remove attribute record. Bug or damaged MFT record.\n"); + return -EIO; + } + mark_mft_record_dirty(ni); + + /* + * Remove record from $ATTRIBUTE_LIST if present and we don't want + * delete $ATTRIBUTE_LIST itself. + */ + if (NInoAttrList(base_ni) && type != AT_ATTRIBUTE_LIST) { + err = ntfs_attrlist_entry_rm(ctx); + if (err) { + ntfs_debug("Couldn't delete record from $ATTRIBUTE_LIST.\n"); + return err; + } + } + + /* Post $ATTRIBUTE_LIST delete setup. */ + if (type == AT_ATTRIBUTE_LIST) { + if (NInoAttrList(base_ni) && base_ni->attr_list) + kvfree(base_ni->attr_list); + base_ni->attr_list = NULL; + NInoClearAttrList(base_ni); + } + + /* Free MFT record, if it doesn't contain attributes. */ + if (le32_to_cpu(ctx->mrec->bytes_in_use) - + le16_to_cpu(ctx->mrec->attrs_offset) == 8) { + if (ntfs_mft_record_free(ni->vol, ni)) { + ntfs_debug("Couldn't free MFT record.\n"); + return -EIO; + } + /* Remove done if we freed base inode. */ + if (ni == base_ni) + return 0; + ntfs_inode_close(ni); + ctx->ntfs_ino = ni = NULL; + } + + if (type == AT_ATTRIBUTE_LIST || !NInoAttrList(base_ni)) + return 0; + + /* Remove attribute list if we don't need it any more. */ + if (!ntfs_attrlist_need(base_ni)) { + struct ntfs_attr na; + struct inode *attr_vi; + + ntfs_attr_reinit_search_ctx(ctx); + if (ntfs_attr_lookup(AT_ATTRIBUTE_LIST, NULL, 0, CASE_SENSITIVE, + 0, NULL, 0, ctx)) { + ntfs_debug("Couldn't find attribute list. Succeed anyway.\n"); + return 0; + } + /* Deallocate clusters. */ + if (ctx->attr->non_resident) { + struct runlist_element *al_rl; + size_t new_rl_count; + + al_rl = ntfs_mapping_pairs_decompress(base_ni->vol, + ctx->attr, NULL, &new_rl_count); + if (IS_ERR(al_rl)) { + ntfs_debug("Couldn't decompress attribute list runlist. Succeed anyway.\n"); + return 0; + } + if (ntfs_cluster_free_from_rl(base_ni->vol, al_rl)) + ntfs_debug("Leaking clusters! Run chkdsk. Couldn't free clusters from attribute list runlist.\n"); + kvfree(al_rl); + } + /* Remove attribute record itself. */ + if (ntfs_attr_record_rm(ctx)) { + ntfs_debug("Couldn't remove attribute list. Succeed anyway.\n"); + return 0; + } + + na.mft_no = VFS_I(base_ni)->i_ino; + na.type = AT_ATTRIBUTE_LIST; + na.name = NULL; + na.name_len = 0; + + attr_vi = ilookup5(VFS_I(base_ni)->i_sb, VFS_I(base_ni)->i_ino, + ntfs_test_inode, &na); + if (attr_vi) { + clear_nlink(attr_vi); + iput(attr_vi); + } + + } + return 0; +} + +/* + * ntfs_attr_add - add attribute to inode + * @ni: opened ntfs inode to which add attribute + * @type: type of the new attribute + * @name: name in unicode of the new attribute + * @name_len: name length in unicode characters of the new attribute + * @val: value of new attribute + * @size: size of the new attribute / length of @val (if specified) + * + * @val should always be specified for always resident attributes (eg. FILE_NAME + * attribute), for attributes that can become non-resident @val can be NULL + * (eg. DATA attribute). @size can be specified even if @val is NULL, in this + * case data size will be equal to @size and initialized size will be equal + * to 0. + * + * If inode haven't got enough space to add attribute, add attribute to one of + * it extents, if no extents present or no one of them have enough space, than + * allocate new extent and add attribute to it. + * + * If on one of this steps attribute list is needed but not present, than it is + * added transparently to caller. So, this function should not be called with + * @type == AT_ATTRIBUTE_LIST, if you really need to add attribute list call + * ntfs_inode_add_attrlist instead. + * + * On success return 0. On error return -1 with errno set to the error code. + */ +int ntfs_attr_add(struct ntfs_inode *ni, __le32 type, + __le16 *name, u8 name_len, u8 *val, s64 size) +{ + struct super_block *sb; + u32 attr_rec_size; + int err, i, offset; + bool is_resident; + bool can_be_non_resident = false; + struct ntfs_inode *attr_ni; + struct inode *attr_vi; + struct mft_record *ni_mrec; + + if (!ni || size < 0 || type == AT_ATTRIBUTE_LIST) + return -EINVAL; + + ntfs_debug("Entering for inode 0x%llx, attr %x, size %lld.\n", + (long long) ni->mft_no, type, size); + + if (ni->nr_extents == -1) + ni = ni->ext.base_ntfs_ino; + + /* Check the attribute type and the size. */ + err = ntfs_attr_size_bounds_check(ni->vol, type, size); + if (err) { + if (err == -ENOENT) + err = -EIO; + return err; + } + + sb = ni->vol->sb; + /* Sanity checks for always resident attributes. */ + err = ntfs_attr_can_be_non_resident(ni->vol, type); + if (err) { + if (err != -EPERM) { + ntfs_error(sb, "ntfs_attr_can_be_non_resident failed"); + goto err_out; + } + /* @val is mandatory. */ + if (!val) { + ntfs_error(sb, + "val is mandatory for always resident attributes"); + return -EINVAL; + } + if (size > ni->vol->mft_record_size) { + ntfs_error(sb, "Attribute is too big"); + return -ERANGE; + } + } else + can_be_non_resident = true; + + /* + * Determine resident or not will be new attribute. We add 8 to size in + * non resident case for mapping pairs. + */ + err = ntfs_attr_can_be_resident(ni->vol, type); + if (!err) { + is_resident = true; + } else { + if (err != -EPERM) { + ntfs_error(sb, "ntfs_attr_can_be_resident failed"); + goto err_out; + } + is_resident = false; + } + + /* Calculate attribute record size. */ + if (is_resident) + attr_rec_size = offsetof(struct attr_record, data.resident.reserved) + + 1 + + ((name_len * sizeof(__le16) + 7) & ~7) + + ((size + 7) & ~7); + else + attr_rec_size = offsetof(struct attr_record, data.non_resident.compressed_size) + + ((name_len * sizeof(__le16) + 7) & ~7) + 8; + + /* + * If we have enough free space for the new attribute in the base MFT + * record, then add attribute to it. + */ +retry: + ni_mrec = map_mft_record(ni); + if (IS_ERR(ni_mrec)) { + err = -EIO; + goto err_out; + } + + if (le32_to_cpu(ni_mrec->bytes_allocated) - + le32_to_cpu(ni_mrec->bytes_in_use) >= attr_rec_size) { + attr_ni = ni; + unmap_mft_record(ni); + goto add_attr_record; + } + unmap_mft_record(ni); + + /* Try to add to extent inodes. */ + err = ntfs_inode_attach_all_extents(ni); + if (err) { + ntfs_error(sb, "Failed to attach all extents to inode"); + goto err_out; + } + + for (i = 0; i < ni->nr_extents; i++) { + attr_ni = ni->ext.extent_ntfs_inos[i]; + ni_mrec = map_mft_record(attr_ni); + if (IS_ERR(ni_mrec)) { + err = -EIO; + goto err_out; + } + + if (le32_to_cpu(ni_mrec->bytes_allocated) - + le32_to_cpu(ni_mrec->bytes_in_use) >= + attr_rec_size) { + unmap_mft_record(attr_ni); + goto add_attr_record; + } + unmap_mft_record(attr_ni); + } + + /* There is no extent that contain enough space for new attribute. */ + if (!NInoAttrList(ni)) { + /* Add attribute list not present, add it and retry. */ + err = ntfs_inode_add_attrlist(ni); + if (err) { + ntfs_error(sb, "Failed to add attribute list"); + goto err_out; + } + goto retry; + } + + attr_ni = NULL; + /* Allocate new extent. */ + err = ntfs_mft_record_alloc(ni->vol, 0, &attr_ni, ni, NULL); + if (err) { + ntfs_error(sb, "Failed to allocate extent record"); + goto err_out; + } + unmap_mft_record(attr_ni); + +add_attr_record: + if (is_resident) { + /* Add resident attribute. */ + offset = ntfs_resident_attr_record_add(attr_ni, type, name, + name_len, val, size, 0); + if (offset < 0) { + if (offset == -ENOSPC && can_be_non_resident) + goto add_non_resident; + err = offset; + ntfs_error(sb, "Failed to add resident attribute"); + goto free_err_out; + } + return 0; + } + +add_non_resident: + /* Add non resident attribute. */ + offset = ntfs_non_resident_attr_record_add(attr_ni, type, name, + name_len, 0, 8, 0); + if (offset < 0) { + err = offset; + ntfs_error(sb, "Failed to add non resident attribute"); + goto free_err_out; + } + + /* If @size == 0, we are done. */ + if (!size) + return 0; + + /* Open new attribute and resize it. */ + attr_vi = ntfs_attr_iget(VFS_I(ni), type, name, name_len); + if (IS_ERR(attr_vi)) { + err = PTR_ERR(attr_vi); + ntfs_error(sb, "Failed to open just added attribute"); + goto rm_attr_err_out; + } + attr_ni = NTFS_I(attr_vi); + + /* Resize and set attribute value. */ + if (ntfs_attr_truncate(attr_ni, size) || + (val && (ntfs_inode_attr_pwrite(attr_vi, 0, size, val, false) != size))) { + err = -EIO; + ntfs_error(sb, "Failed to initialize just added attribute"); + if (ntfs_attr_rm(attr_ni)) + ntfs_error(sb, "Failed to remove just added attribute"); + iput(attr_vi); + goto err_out; + } + iput(attr_vi); + return 0; + +rm_attr_err_out: + /* Remove just added attribute. */ + ni_mrec = map_mft_record(attr_ni); + if (!IS_ERR(ni_mrec)) { + if (ntfs_attr_record_resize(ni_mrec, + (struct attr_record *)((u8 *)ni_mrec + offset), 0)) + ntfs_error(sb, "Failed to remove just added attribute #2"); + unmap_mft_record(attr_ni); + } else + pr_err("EIO when try to remove new added attr\n"); + +free_err_out: + /* Free MFT record, if it doesn't contain attributes. */ + ni_mrec = map_mft_record(attr_ni); + if (!IS_ERR(ni_mrec)) { + int attr_size; + + attr_size = le32_to_cpu(ni_mrec->bytes_in_use) - + le16_to_cpu(ni_mrec->attrs_offset); + unmap_mft_record(attr_ni); + if (attr_size == 8) { + if (ntfs_mft_record_free(attr_ni->vol, attr_ni)) + ntfs_error(sb, "Failed to free MFT record"); + if (attr_ni->nr_extents < 0) + ntfs_inode_close(attr_ni); + } + } else + pr_err("EIO when testing mft record is free-able\n"); + +err_out: + return err; +} + +/* + * __ntfs_attr_init - primary initialization of an ntfs attribute structure + * @ni: ntfs attribute inode to initialize + * @ni: ntfs inode with which to initialize the ntfs attribute + * @type: attribute type + * @name: attribute name in little endian Unicode or NULL + * @name_len: length of attribute @name in Unicode characters (if @name given) + * + * Initialize the ntfs attribute @na with @ni, @type, @name, and @name_len. + */ +static void __ntfs_attr_init(struct ntfs_inode *ni, + const __le32 type, __le16 *name, const u32 name_len) +{ + ni->runlist.rl = NULL; + ni->type = type; + ni->name = name; + if (name) + ni->name_len = name_len; + else + ni->name_len = 0; +} + +/* + * ntfs_attr_init - initialize an ntfs_attr with data sizes and status + * @ni: ntfs inode to initialize + * @non_resident: true if attribute is non-resident + * @compressed: true if attribute is compressed + * @encrypted: true if attribute is encrypted + * @sparse: true if attribute is sparse + * @allocated_size: allocated size of the attribute + * @data_size: actual data size of the attribute + * @initialized_size: initialized size of the attribute + * @compressed_size: compressed size (if compressed or sparse) + * @compression_unit: compression unit size (log2 of clusters) + * + * Final initialization for an ntfs attribute. + */ +static void ntfs_attr_init(struct ntfs_inode *ni, const bool non_resident, + const bool compressed, const bool encrypted, const bool sparse, + const s64 allocated_size, const s64 data_size, + const s64 initialized_size, const s64 compressed_size, + const u8 compression_unit) +{ + if (non_resident) + NInoSetNonResident(ni); + if (compressed) { + NInoSetCompressed(ni); + ni->flags |= FILE_ATTR_COMPRESSED; + } + if (encrypted) { + NInoSetEncrypted(ni); + ni->flags |= FILE_ATTR_ENCRYPTED; + } + if (sparse) { + NInoSetSparse(ni); + ni->flags |= FILE_ATTR_SPARSE_FILE; + } + ni->allocated_size = allocated_size; + ni->data_size = data_size; + ni->initialized_size = initialized_size; + if (compressed || sparse) { + struct ntfs_volume *vol = ni->vol; + + ni->itype.compressed.size = compressed_size; + ni->itype.compressed.block_clusters = 1 << compression_unit; + ni->itype.compressed.block_size = 1 << (compression_unit + + vol->cluster_size_bits); + ni->itype.compressed.block_size_bits = ffs( + ni->itype.compressed.block_size) - 1; + } +} + +/* + * ntfs_attr_open - open an ntfs attribute for access + * @ni: open ntfs inode in which the ntfs attribute resides + * @type: attribute type + * @name: attribute name in little endian Unicode or AT_UNNAMED or NULL + * @name_len: length of attribute @name in Unicode characters (if @name given) + */ +int ntfs_attr_open(struct ntfs_inode *ni, const __le32 type, + __le16 *name, u32 name_len) +{ + struct ntfs_attr_search_ctx *ctx; + __le16 *newname = NULL; + struct attr_record *a; + bool cs; + struct ntfs_inode *base_ni; + int err; + + if (!ni || !ni->vol) + return -EINVAL; + + ntfs_debug("Entering for inode %lld, attr 0x%x.\n", + ni->mft_no, type); + + if (NInoAttr(ni)) + base_ni = ni->ext.base_ntfs_ino; + else + base_ni = ni; + + if (name && name != AT_UNNAMED && name != I30) { + name = ntfs_ucsndup(name, name_len); + if (!name) { + err = -ENOMEM; + goto err_out; + } + newname = name; + } + + ctx = ntfs_attr_get_search_ctx(base_ni, NULL); + if (!ctx) { + err = -ENOMEM; + pr_err("%s: Failed to get search context\n", __func__); + goto err_out; + } + + err = ntfs_attr_lookup(type, name, name_len, 0, 0, NULL, 0, ctx); + if (err) + goto put_err_out; + + a = ctx->attr; + + if (!name) { + if (a->name_length) { + name = ntfs_ucsndup((__le16 *)((u8 *)a + le16_to_cpu(a->name_offset)), + a->name_length); + if (!name) + goto put_err_out; + newname = name; + name_len = a->name_length; + } else { + name = AT_UNNAMED; + name_len = 0; + } + } + + __ntfs_attr_init(ni, type, name, name_len); + + /* + * Wipe the flags in case they are not zero for an attribute list + * attribute. Windows does not complain about invalid flags and chkdsk + * does not detect or fix them so we need to cope with it, too. + */ + if (type == AT_ATTRIBUTE_LIST) + a->flags = 0; + + if ((type == AT_DATA) && + (a->non_resident ? !a->data.non_resident.initialized_size : + !a->data.resident.value_length)) { + /* + * Define/redefine the compression state if stream is + * empty, based on the compression mark on parent + * directory (for unnamed data streams) or on current + * inode (for named data streams). The compression mark + * may change any time, the compression state can only + * change when stream is wiped out. + * + * Also prevent compression on NTFS version < 3.0 + * or cluster size > 4K or compression is disabled + */ + a->flags &= ~ATTR_COMPRESSION_MASK; + if (NInoCompressed(ni) + && (ni->vol->major_ver >= 3) + && NVolCompression(ni->vol) + && (ni->vol->cluster_size <= MAX_COMPRESSION_CLUSTER_SIZE)) + a->flags |= ATTR_IS_COMPRESSED; + } + + cs = a->flags & (ATTR_IS_COMPRESSED | ATTR_IS_SPARSE); + + if (ni->type == AT_DATA && ni->name == AT_UNNAMED && + ((!(a->flags & ATTR_IS_COMPRESSED) != !NInoCompressed(ni)) || + (!(a->flags & ATTR_IS_SPARSE) != !NInoSparse(ni)) || + (!(a->flags & ATTR_IS_ENCRYPTED) != !NInoEncrypted(ni)))) { + err = -EIO; + pr_err("Inode %lld has corrupt attribute flags (0x%x <> 0x%x)\n", + (unsigned long long)ni->mft_no, + a->flags, ni->flags); + goto put_err_out; + } + + if (a->non_resident) { + if (((a->flags & ATTR_COMPRESSION_MASK) || a->data.non_resident.compression_unit) && + (ni->vol->major_ver < 3)) { + err = -EIO; + pr_err("Compressed inode %lld not allowed on NTFS %d.%d\n", + (unsigned long long)ni->mft_no, + ni->vol->major_ver, + ni->vol->major_ver); + goto put_err_out; + } + + if ((a->flags & ATTR_IS_COMPRESSED) && !a->data.non_resident.compression_unit) { + err = -EIO; + pr_err("Compressed inode %lld attr 0x%x has no compression unit\n", + (unsigned long long)ni->mft_no, type); + goto put_err_out; + } + if ((a->flags & ATTR_COMPRESSION_MASK) && + (a->data.non_resident.compression_unit != STANDARD_COMPRESSION_UNIT)) { + err = -EIO; + pr_err("Compressed inode %lld attr 0x%lx has an unsupported compression unit %d\n", + (unsigned long long)ni->mft_no, + (long)le32_to_cpu(type), + (int)a->data.non_resident.compression_unit); + goto put_err_out; + } + ntfs_attr_init(ni, true, a->flags & ATTR_IS_COMPRESSED, + a->flags & ATTR_IS_ENCRYPTED, + a->flags & ATTR_IS_SPARSE, + le64_to_cpu(a->data.non_resident.allocated_size), + le64_to_cpu(a->data.non_resident.data_size), + le64_to_cpu(a->data.non_resident.initialized_size), + cs ? le64_to_cpu(a->data.non_resident.compressed_size) : 0, + cs ? a->data.non_resident.compression_unit : 0); + } else { + s64 l = le32_to_cpu(a->data.resident.value_length); + + ntfs_attr_init(ni, false, a->flags & ATTR_IS_COMPRESSED, + a->flags & ATTR_IS_ENCRYPTED, + a->flags & ATTR_IS_SPARSE, (l + 7) & ~7, l, l, + cs ? (l + 7) & ~7 : 0, 0); + } + ntfs_attr_put_search_ctx(ctx); +out: + ntfs_debug("\n"); + return err; + +put_err_out: + ntfs_attr_put_search_ctx(ctx); +err_out: + kfree(newname); + goto out; +} + +/* + * ntfs_attr_close - free an ntfs attribute structure + * @ni: ntfs inode to free + * + * Release all memory associated with the ntfs attribute @na and then release + * @na itself. + */ +void ntfs_attr_close(struct ntfs_inode *ni) +{ + if (NInoNonResident(ni) && ni->runlist.rl) + kvfree(ni->runlist.rl); + /* Don't release if using an internal constant. */ + if (ni->name != AT_UNNAMED && ni->name != I30) + kfree(ni->name); +} + +/* + * ntfs_attr_map_whole_runlist - map the whole runlist of an ntfs attribute + * @ni: ntfs inode for which to map the runlist + * + * Map the whole runlist of the ntfs attribute @na. For an attribute made up + * of only one attribute extent this is the same as calling + * ntfs_map_runlist(ni, 0) but for an attribute with multiple extents this + * will map the runlist fragments from each of the extents thus giving access + * to the entirety of the disk allocation of an attribute. + */ +int ntfs_attr_map_whole_runlist(struct ntfs_inode *ni) +{ + s64 next_vcn, last_vcn, highest_vcn; + struct ntfs_attr_search_ctx *ctx; + struct ntfs_volume *vol = ni->vol; + struct super_block *sb = vol->sb; + struct attr_record *a; + int err; + struct ntfs_inode *base_ni; + int not_mapped; + size_t new_rl_count; + + ntfs_debug("Entering for inode 0x%llx, attr 0x%x.\n", + (unsigned long long)ni->mft_no, ni->type); + + if (NInoFullyMapped(ni) && ni->runlist.rl) + return 0; + + if (NInoAttr(ni)) + base_ni = ni->ext.base_ntfs_ino; + else + base_ni = ni; + + ctx = ntfs_attr_get_search_ctx(base_ni, NULL); + if (!ctx) { + ntfs_error(sb, "%s: Failed to get search context", __func__); + return -ENOMEM; + } + + /* Map all attribute extents one by one. */ + next_vcn = last_vcn = highest_vcn = 0; + a = NULL; + while (1) { + struct runlist_element *rl; + + not_mapped = 0; + if (ntfs_rl_vcn_to_lcn(ni->runlist.rl, next_vcn) == LCN_RL_NOT_MAPPED) + not_mapped = 1; + + err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len, + CASE_SENSITIVE, next_vcn, NULL, 0, ctx); + if (err) + break; + + a = ctx->attr; + + if (not_mapped) { + /* Decode the runlist. */ + rl = ntfs_mapping_pairs_decompress(ni->vol, a, &ni->runlist, + &new_rl_count); + if (IS_ERR(rl)) { + err = PTR_ERR(rl); + goto err_out; + } + ni->runlist.rl = rl; + ni->runlist.count = new_rl_count; + } + + /* Are we in the first extent? */ + if (!next_vcn) { + if (a->data.non_resident.lowest_vcn) { + err = -EIO; + ntfs_error(sb, + "First extent of inode %llu attribute has non-zero lowest_vcn", + (unsigned long long)ni->mft_no); + goto err_out; + } + /* Get the last vcn in the attribute. */ + last_vcn = ntfs_bytes_to_cluster(vol, + le64_to_cpu(a->data.non_resident.allocated_size)); + } + + /* Get the lowest vcn for the next extent. */ + highest_vcn = le64_to_cpu(a->data.non_resident.highest_vcn); + next_vcn = highest_vcn + 1; + + /* Only one extent or error, which we catch below. */ + if (next_vcn <= 0) { + err = -ENOENT; + break; + } + + /* Avoid endless loops due to corruption. */ + if (next_vcn < le64_to_cpu(a->data.non_resident.lowest_vcn)) { + err = -EIO; + ntfs_error(sb, "Inode %llu has corrupt attribute list", + (unsigned long long)ni->mft_no); + goto err_out; + } + } + if (!a) { + ntfs_error(sb, "Couldn't find attribute for runlist mapping"); + goto err_out; + } + if (not_mapped && highest_vcn && highest_vcn != last_vcn - 1) { + err = -EIO; + ntfs_error(sb, + "Failed to load full runlist: inode: %llu highest_vcn: 0x%llx last_vcn: 0x%llx", + (unsigned long long)ni->mft_no, + (long long)highest_vcn, (long long)last_vcn); + goto err_out; + } + ntfs_attr_put_search_ctx(ctx); + if (err == -ENOENT) { + NInoSetFullyMapped(ni); + return 0; + } + + return err; + +err_out: + ntfs_attr_put_search_ctx(ctx); + return err; +} + +/* + * ntfs_attr_record_move_to - move attribute record to target inode + * @ctx: attribute search context describing the attribute record + * @ni: opened ntfs inode to which move attribute record + */ +int ntfs_attr_record_move_to(struct ntfs_attr_search_ctx *ctx, struct ntfs_inode *ni) +{ + struct ntfs_attr_search_ctx *nctx; + struct attr_record *a; + int err; + struct mft_record *ni_mrec; + struct super_block *sb; + + if (!ctx || !ctx->attr || !ctx->ntfs_ino || !ni) { + ntfs_debug("Invalid arguments passed.\n"); + return -EINVAL; + } + + sb = ni->vol->sb; + ntfs_debug("Entering for ctx->attr->type 0x%x, ctx->ntfs_ino->mft_no 0x%llx, ni->mft_no 0x%llx.\n", + (unsigned int) le32_to_cpu(ctx->attr->type), + (long long) ctx->ntfs_ino->mft_no, + (long long) ni->mft_no); + + if (ctx->ntfs_ino == ni) + return 0; + + if (!ctx->al_entry) { + ntfs_debug("Inode should contain attribute list to use this function.\n"); + return -EINVAL; + } + + /* Find place in MFT record where attribute will be moved. */ + a = ctx->attr; + nctx = ntfs_attr_get_search_ctx(ni, NULL); + if (!nctx) { + ntfs_error(sb, "%s: Failed to get search context", __func__); + return -ENOMEM; + } + + /* + * Use ntfs_attr_find instead of ntfs_attr_lookup to find place for + * attribute in @ni->mrec, not any extent inode in case if @ni is base + * file record. + */ + err = ntfs_attr_find(a->type, (__le16 *)((u8 *)a + le16_to_cpu(a->name_offset)), + a->name_length, CASE_SENSITIVE, NULL, + 0, nctx); + if (!err) { + ntfs_debug("Attribute of such type, with same name already present in this MFT record.\n"); + err = -EEXIST; + goto put_err_out; + } + if (err != -ENOENT) { + ntfs_debug("Attribute lookup failed.\n"); + goto put_err_out; + } + + /* Make space and move attribute. */ + ni_mrec = map_mft_record(ni); + if (IS_ERR(ni_mrec)) { + err = -EIO; + goto put_err_out; + } + + err = ntfs_make_room_for_attr(ni_mrec, (u8 *) nctx->attr, + le32_to_cpu(a->length)); + if (err) { + ntfs_debug("Couldn't make space for attribute.\n"); + unmap_mft_record(ni); + goto put_err_out; + } + memcpy(nctx->attr, a, le32_to_cpu(a->length)); + nctx->attr->instance = nctx->mrec->next_attr_instance; + nctx->mrec->next_attr_instance = + cpu_to_le16((le16_to_cpu(nctx->mrec->next_attr_instance) + 1) & 0xffff); + ntfs_attr_record_resize(ctx->mrec, a, 0); + mark_mft_record_dirty(ctx->ntfs_ino); + mark_mft_record_dirty(ni); + + /* Update attribute list. */ + ctx->al_entry->mft_reference = + MK_LE_MREF(ni->mft_no, le16_to_cpu(ni_mrec->sequence_number)); + ctx->al_entry->instance = nctx->attr->instance; + unmap_mft_record(ni); +put_err_out: + ntfs_attr_put_search_ctx(nctx); + return err; +} + +/* + * ntfs_attr_record_move_away - move away attribute record from it's mft record + * @ctx: attribute search context describing the attribute record + * @extra: minimum amount of free space in the new holder of record + */ +int ntfs_attr_record_move_away(struct ntfs_attr_search_ctx *ctx, int extra) +{ + struct ntfs_inode *base_ni, *ni = NULL; + struct mft_record *m; + int i, err; + struct super_block *sb; + + if (!ctx || !ctx->attr || !ctx->ntfs_ino || extra < 0) + return -EINVAL; + + ntfs_debug("Entering for attr 0x%x, inode %llu\n", + (unsigned int) le32_to_cpu(ctx->attr->type), + (unsigned long long)ctx->ntfs_ino->mft_no); + + if (ctx->ntfs_ino->nr_extents == -1) + base_ni = ctx->base_ntfs_ino; + else + base_ni = ctx->ntfs_ino; + + sb = ctx->ntfs_ino->vol->sb; + if (!NInoAttrList(base_ni)) { + ntfs_error(sb, "Inode %llu has no attrlist", + (unsigned long long)base_ni->mft_no); + return -EINVAL; + } + + err = ntfs_inode_attach_all_extents(ctx->ntfs_ino); + if (err) { + ntfs_error(sb, "Couldn't attach extents, inode=%llu", + (unsigned long long)base_ni->mft_no); + return err; + } + + mutex_lock(&base_ni->extent_lock); + /* Walk through all extents and try to move attribute to them. */ + for (i = 0; i < base_ni->nr_extents; i++) { + ni = base_ni->ext.extent_ntfs_inos[i]; + + if (ctx->ntfs_ino->mft_no == ni->mft_no) + continue; + m = map_mft_record(ni); + if (IS_ERR(m)) { + ntfs_error(sb, "Can not map mft record for mft_no %lld", + (unsigned long long)ni->mft_no); + mutex_unlock(&base_ni->extent_lock); + return -EIO; + } + if (le32_to_cpu(m->bytes_allocated) - + le32_to_cpu(m->bytes_in_use) < le32_to_cpu(ctx->attr->length) + extra) { + unmap_mft_record(ni); + continue; + } + unmap_mft_record(ni); + + /* + * ntfs_attr_record_move_to can fail if extent with other lowest + * s64 already present in inode we trying move record to. So, + * do not return error. + */ + if (!ntfs_attr_record_move_to(ctx, ni)) { + mutex_unlock(&base_ni->extent_lock); + return 0; + } + } + mutex_unlock(&base_ni->extent_lock); + + /* + * Failed to move attribute to one of the current extents, so allocate + * new extent and move attribute to it. + */ + ni = NULL; + err = ntfs_mft_record_alloc(base_ni->vol, 0, &ni, base_ni, NULL); + if (err) { + ntfs_error(sb, "Couldn't allocate MFT record, err : %d", err); + return err; + } + unmap_mft_record(ni); + + err = ntfs_attr_record_move_to(ctx, ni); + if (err) + ntfs_error(sb, "Couldn't move attribute to MFT record"); + + return err; +} + +/* + * If we are in the first extent, then set/clean sparse bit, + * update allocated and compressed size. + */ +static int ntfs_attr_update_meta(struct attr_record *a, struct ntfs_inode *ni, + struct mft_record *m, struct ntfs_attr_search_ctx *ctx) +{ + int sparse, err = 0; + struct ntfs_inode *base_ni; + struct super_block *sb = ni->vol->sb; + + ntfs_debug("Entering for inode 0x%llx, attr 0x%x\n", + (unsigned long long)ni->mft_no, ni->type); + + if (NInoAttr(ni)) + base_ni = ni->ext.base_ntfs_ino; + else + base_ni = ni; + + if (a->data.non_resident.lowest_vcn) + goto out; + + a->data.non_resident.allocated_size = cpu_to_le64(ni->allocated_size); + + sparse = ntfs_rl_sparse(ni->runlist.rl); + if (sparse < 0) { + err = -EIO; + goto out; + } + + /* Attribute become sparse. */ + if (sparse && !(a->flags & (ATTR_IS_SPARSE | ATTR_IS_COMPRESSED))) { + /* + * Move attribute to another mft record, if attribute is too + * small to add compressed_size field to it and we have no + * free space in the current mft record. + */ + if ((le32_to_cpu(a->length) - + le16_to_cpu(a->data.non_resident.mapping_pairs_offset) == 8) && + !(le32_to_cpu(m->bytes_allocated) - le32_to_cpu(m->bytes_in_use))) { + + if (!NInoAttrList(base_ni)) { + err = ntfs_inode_add_attrlist(base_ni); + if (err) + goto out; + err = -EAGAIN; + goto out; + } + err = ntfs_attr_record_move_away(ctx, 8); + if (err) { + ntfs_error(sb, "Failed to move attribute"); + goto out; + } + + err = ntfs_attrlist_update(base_ni); + if (err) + goto out; + err = -EAGAIN; + goto out; + } + if (!(le32_to_cpu(a->length) - + le16_to_cpu(a->data.non_resident.mapping_pairs_offset))) { + err = -EIO; + ntfs_error(sb, "Mapping pairs space is 0"); + goto out; + } + + NInoSetSparse(ni); + ni->flags |= FILE_ATTR_SPARSE_FILE; + a->flags |= ATTR_IS_SPARSE; + a->data.non_resident.compression_unit = 0; + + memmove((u8 *)a + le16_to_cpu(a->name_offset) + 8, + (u8 *)a + le16_to_cpu(a->name_offset), + a->name_length * sizeof(__le16)); + + a->name_offset = cpu_to_le16(le16_to_cpu(a->name_offset) + 8); + + a->data.non_resident.mapping_pairs_offset = + cpu_to_le16(le16_to_cpu(a->data.non_resident.mapping_pairs_offset) + 8); + } + + /* Attribute no longer sparse. */ + if (!sparse && (a->flags & ATTR_IS_SPARSE) && + !(a->flags & ATTR_IS_COMPRESSED)) { + NInoClearSparse(ni); + ni->flags &= ~FILE_ATTR_SPARSE_FILE; + a->flags &= ~ATTR_IS_SPARSE; + a->data.non_resident.compression_unit = 0; + + memmove((u8 *)a + le16_to_cpu(a->name_offset) - 8, + (u8 *)a + le16_to_cpu(a->name_offset), + a->name_length * sizeof(__le16)); + + if (le16_to_cpu(a->name_offset) >= 8) + a->name_offset = cpu_to_le16(le16_to_cpu(a->name_offset) - 8); + + a->data.non_resident.mapping_pairs_offset = + cpu_to_le16(le16_to_cpu(a->data.non_resident.mapping_pairs_offset) - 8); + } + + /* Update compressed size if required. */ + if (NInoFullyMapped(ni) && (sparse || NInoCompressed(ni))) { + s64 new_compr_size; + + new_compr_size = ntfs_rl_get_compressed_size(ni->vol, ni->runlist.rl); + if (new_compr_size < 0) { + err = new_compr_size; + goto out; + } + + ni->itype.compressed.size = new_compr_size; + a->data.non_resident.compressed_size = cpu_to_le64(new_compr_size); + } + + if (NInoSparse(ni) || NInoCompressed(ni)) + VFS_I(base_ni)->i_blocks = ni->itype.compressed.size >> 9; + else + VFS_I(base_ni)->i_blocks = ni->allocated_size >> 9; + /* + * Set FILE_NAME dirty flag, to update sparse bit and + * allocated size in the index. + */ + if (ni->type == AT_DATA && ni->name == AT_UNNAMED) + NInoSetFileNameDirty(ni); +out: + return err; +} + +#define NTFS_VCN_DELETE_MARK -2 +/* + * ntfs_attr_update_mapping_pairs - update mapping pairs for ntfs attribute + * @ni: non-resident ntfs inode for which we need update + * @from_vcn: update runlist starting this VCN + * + * Build mapping pairs from @na->rl and write them to the disk. Also, this + * function updates sparse bit, allocated and compressed size (allocates/frees + * space for this field if required). + * + * @na->allocated_size should be set to correct value for the new runlist before + * call to this function. Vice-versa @na->compressed_size will be calculated and + * set to correct value during this function. + */ +int ntfs_attr_update_mapping_pairs(struct ntfs_inode *ni, s64 from_vcn) +{ + struct ntfs_attr_search_ctx *ctx; + struct ntfs_inode *base_ni; + struct mft_record *m; + struct attr_record *a; + s64 stop_vcn; + int err = 0, mp_size, cur_max_mp_size, exp_max_mp_size; + bool finished_build; + bool first_updated = false; + struct super_block *sb; + struct runlist_element *start_rl; + unsigned int de_cluster_count = 0; + +retry: + if (!ni || !ni->runlist.rl) + return -EINVAL; + + ntfs_debug("Entering for inode %llu, attr 0x%x\n", + (unsigned long long)ni->mft_no, ni->type); + + sb = ni->vol->sb; + if (!NInoNonResident(ni)) { + ntfs_error(sb, "%s: resident attribute", __func__); + return -EINVAL; + } + + if (ni->nr_extents == -1) + base_ni = ni->ext.base_ntfs_ino; + else + base_ni = ni; + + ctx = ntfs_attr_get_search_ctx(base_ni, NULL); + if (!ctx) { + ntfs_error(sb, "%s: Failed to get search context", __func__); + return -ENOMEM; + } + + /* Fill attribute records with new mapping pairs. */ + stop_vcn = 0; + finished_build = false; + start_rl = ni->runlist.rl; + while (!(err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len, + CASE_SENSITIVE, from_vcn, NULL, 0, ctx))) { + unsigned int de_cnt = 0; + + a = ctx->attr; + m = ctx->mrec; + if (!a->data.non_resident.lowest_vcn) + first_updated = true; + + /* + * If runlist is updating not from the beginning, then set + * @stop_vcn properly, i.e. to the lowest vcn of record that + * contain @from_vcn. Also we do not need @from_vcn anymore, + * set it to 0 to make ntfs_attr_lookup enumerate attributes. + */ + if (from_vcn) { + s64 first_lcn; + + stop_vcn = le64_to_cpu(a->data.non_resident.lowest_vcn); + from_vcn = 0; + /* + * Check whether the first run we need to update is + * the last run in runlist, if so, then deallocate + * all attrubute extents starting this one. + */ + first_lcn = ntfs_rl_vcn_to_lcn(ni->runlist.rl, stop_vcn); + if (first_lcn == LCN_EINVAL) { + err = -EIO; + ntfs_error(sb, "Bad runlist"); + goto put_err_out; + } + if (first_lcn == LCN_ENOENT || + first_lcn == LCN_RL_NOT_MAPPED) + finished_build = true; + } + + /* + * Check whether we finished mapping pairs build, if so mark + * extent as need to delete (by setting highest vcn to + * NTFS_VCN_DELETE_MARK (-2), we shall check it later and + * delete extent) and continue search. + */ + if (finished_build) { + ntfs_debug("Mark attr 0x%x for delete in inode 0x%llx.\n", + (unsigned int)le32_to_cpu(a->type), ctx->ntfs_ino->mft_no); + a->data.non_resident.highest_vcn = cpu_to_le64(NTFS_VCN_DELETE_MARK); + mark_mft_record_dirty(ctx->ntfs_ino); + continue; + } + + err = ntfs_attr_update_meta(a, ni, m, ctx); + if (err < 0) { + if (err == -EAGAIN) { + ntfs_attr_put_search_ctx(ctx); + goto retry; + } + goto put_err_out; + } + + /* + * Determine maximum possible length of mapping pairs, + * if we shall *not* expand space for mapping pairs. + */ + cur_max_mp_size = le32_to_cpu(a->length) - + le16_to_cpu(a->data.non_resident.mapping_pairs_offset); + /* + * Determine maximum possible length of mapping pairs in the + * current mft record, if we shall expand space for mapping + * pairs. + */ + exp_max_mp_size = le32_to_cpu(m->bytes_allocated) - + le32_to_cpu(m->bytes_in_use) + cur_max_mp_size; + + /* Get the size for the rest of mapping pairs array. */ + mp_size = ntfs_get_size_for_mapping_pairs(ni->vol, start_rl, + stop_vcn, -1, exp_max_mp_size); + if (mp_size <= 0) { + err = mp_size; + ntfs_error(sb, "%s: get MP size failed", __func__); + goto put_err_out; + } + /* Test mapping pairs for fitting in the current mft record. */ + if (mp_size > exp_max_mp_size) { + /* + * Mapping pairs of $ATTRIBUTE_LIST attribute must fit + * in the base mft record. Try to move out other + * attributes and try again. + */ + if (ni->type == AT_ATTRIBUTE_LIST) { + ntfs_attr_put_search_ctx(ctx); + if (ntfs_inode_free_space(base_ni, mp_size - + cur_max_mp_size)) { + ntfs_debug("Attribute list is too big. Defragment the volume\n"); + return -ENOSPC; + } + if (ntfs_attrlist_update(base_ni)) + return -EIO; + goto retry; + } + + /* Add attribute list if it isn't present, and retry. */ + if (!NInoAttrList(base_ni)) { + ntfs_attr_put_search_ctx(ctx); + if (ntfs_inode_add_attrlist(base_ni)) { + ntfs_error(sb, "Can not add attrlist"); + return -EIO; + } + goto retry; + } + + /* + * Set mapping pairs size to maximum possible for this + * mft record. We shall write the rest of mapping pairs + * to another MFT records. + */ + mp_size = exp_max_mp_size; + } + + /* Change space for mapping pairs if we need it. */ + if (((mp_size + 7) & ~7) != cur_max_mp_size) { + if (ntfs_attr_record_resize(m, a, + le16_to_cpu(a->data.non_resident.mapping_pairs_offset) + + mp_size)) { + err = -EIO; + ntfs_error(sb, "Failed to resize attribute"); + goto put_err_out; + } + } + + /* Update lowest vcn. */ + a->data.non_resident.lowest_vcn = cpu_to_le64(stop_vcn); + mark_mft_record_dirty(ctx->ntfs_ino); + if ((ctx->ntfs_ino->nr_extents == -1 || NInoAttrList(ctx->ntfs_ino)) && + ctx->attr->type != AT_ATTRIBUTE_LIST) { + ctx->al_entry->lowest_vcn = cpu_to_le64(stop_vcn); + err = ntfs_attrlist_update(base_ni); + if (err) + goto put_err_out; + } + + /* + * Generate the new mapping pairs array directly into the + * correct destination, i.e. the attribute record itself. + */ + err = ntfs_mapping_pairs_build(ni->vol, + (u8 *)a + le16_to_cpu(a->data.non_resident.mapping_pairs_offset), + mp_size, start_rl, stop_vcn, -1, &stop_vcn, &start_rl, &de_cnt); + if (!err) + finished_build = true; + if (!finished_build && err != -ENOSPC) { + ntfs_error(sb, "Failed to build mapping pairs"); + goto put_err_out; + } + a->data.non_resident.highest_vcn = cpu_to_le64(stop_vcn - 1); + mark_mft_record_dirty(ctx->ntfs_ino); + de_cluster_count += de_cnt; + } + + /* Check whether error occurred. */ + if (err && err != -ENOENT) { + ntfs_error(sb, "%s: Attribute lookup failed", __func__); + goto put_err_out; + } + + /* + * If the base extent was skipped in the above process, + * we still may have to update the sizes. + */ + if (!first_updated) { + ntfs_attr_reinit_search_ctx(ctx); + err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len, + CASE_SENSITIVE, 0, NULL, 0, ctx); + if (!err) { + a = ctx->attr; + a->data.non_resident.allocated_size = cpu_to_le64(ni->allocated_size); + if (NInoCompressed(ni) || NInoSparse(ni)) + a->data.non_resident.compressed_size = + cpu_to_le64(ni->itype.compressed.size); + /* Updating sizes taints the extent holding the attr */ + if (ni->type == AT_DATA && ni->name == AT_UNNAMED) + NInoSetFileNameDirty(ni); + mark_mft_record_dirty(ctx->ntfs_ino); + } else { + ntfs_error(sb, "Failed to update sizes in base extent\n"); + goto put_err_out; + } + } + + /* Deallocate not used attribute extents and return with success. */ + if (finished_build) { + ntfs_attr_reinit_search_ctx(ctx); + ntfs_debug("Deallocate marked extents.\n"); + while (!(err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len, + CASE_SENSITIVE, 0, NULL, 0, ctx))) { + if (le64_to_cpu(ctx->attr->data.non_resident.highest_vcn) != + NTFS_VCN_DELETE_MARK) + continue; + /* Remove unused attribute record. */ + err = ntfs_attr_record_rm(ctx); + if (err) { + ntfs_error(sb, "Could not remove unused attr"); + goto put_err_out; + } + ntfs_attr_reinit_search_ctx(ctx); + } + if (err && err != -ENOENT) { + ntfs_error(sb, "%s: Attr lookup failed", __func__); + goto put_err_out; + } + ntfs_debug("Deallocate done.\n"); + ntfs_attr_put_search_ctx(ctx); + goto out; + } + ntfs_attr_put_search_ctx(ctx); + ctx = NULL; + + /* Allocate new MFT records for the rest of mapping pairs. */ + while (1) { + struct ntfs_inode *ext_ni = NULL; + unsigned int de_cnt = 0; + + /* Allocate new mft record. */ + err = ntfs_mft_record_alloc(ni->vol, 0, &ext_ni, base_ni, NULL); + if (err) { + ntfs_error(sb, "Failed to allocate extent record"); + goto put_err_out; + } + unmap_mft_record(ext_ni); + + m = map_mft_record(ext_ni); + if (IS_ERR(m)) { + ntfs_error(sb, "Could not map new MFT record"); + if (ntfs_mft_record_free(ni->vol, ext_ni)) + ntfs_error(sb, "Could not free MFT record"); + ntfs_inode_close(ext_ni); + err = -ENOMEM; + ext_ni = NULL; + goto put_err_out; + } + /* + * If mapping size exceed available space, set them to + * possible maximum. + */ + cur_max_mp_size = le32_to_cpu(m->bytes_allocated) - + le32_to_cpu(m->bytes_in_use) - + (sizeof(struct attr_record) + + ((NInoCompressed(ni) || NInoSparse(ni)) ? + sizeof(a->data.non_resident.compressed_size) : 0)) - + ((sizeof(__le16) * ni->name_len + 7) & ~7); + + /* Calculate size of rest mapping pairs. */ + mp_size = ntfs_get_size_for_mapping_pairs(ni->vol, + start_rl, stop_vcn, -1, cur_max_mp_size); + if (mp_size <= 0) { + unmap_mft_record(ext_ni); + ntfs_inode_close(ext_ni); + err = mp_size; + ntfs_error(sb, "%s: get mp size failed", __func__); + goto put_err_out; + } + + if (mp_size > cur_max_mp_size) + mp_size = cur_max_mp_size; + /* Add attribute extent to new record. */ + err = ntfs_non_resident_attr_record_add(ext_ni, ni->type, + ni->name, ni->name_len, stop_vcn, mp_size, 0); + if (err < 0) { + ntfs_error(sb, "Could not add attribute extent"); + unmap_mft_record(ext_ni); + if (ntfs_mft_record_free(ni->vol, ext_ni)) + ntfs_error(sb, "Could not free MFT record"); + ntfs_inode_close(ext_ni); + goto put_err_out; + } + a = (struct attr_record *)((u8 *)m + err); + + err = ntfs_mapping_pairs_build(ni->vol, (u8 *)a + + le16_to_cpu(a->data.non_resident.mapping_pairs_offset), + mp_size, start_rl, stop_vcn, -1, &stop_vcn, &start_rl, + &de_cnt); + if (err < 0 && err != -ENOSPC) { + ntfs_error(sb, "Failed to build MP"); + unmap_mft_record(ext_ni); + if (ntfs_mft_record_free(ni->vol, ext_ni)) + ntfs_error(sb, "Couldn't free MFT record"); + goto put_err_out; + } + a->data.non_resident.highest_vcn = cpu_to_le64(stop_vcn - 1); + mark_mft_record_dirty(ext_ni); + unmap_mft_record(ext_ni); + + de_cluster_count += de_cnt; + /* All mapping pairs has been written. */ + if (!err) + break; + } +out: + if (from_vcn == 0) + ni->i_dealloc_clusters = de_cluster_count; + return 0; + +put_err_out: + if (ctx) + ntfs_attr_put_search_ctx(ctx); + return err; +} + +/* + * ntfs_attr_make_resident - convert a non-resident to a resident attribute + * @ni: open ntfs attribute to make resident + * @ctx: ntfs search context describing the attribute + * + * Convert a non-resident ntfs attribute to a resident one. + */ +static int ntfs_attr_make_resident(struct ntfs_inode *ni, struct ntfs_attr_search_ctx *ctx) +{ + struct ntfs_volume *vol = ni->vol; + struct super_block *sb = vol->sb; + struct attr_record *a = ctx->attr; + int name_ofs, val_ofs, err; + s64 arec_size; + + ntfs_debug("Entering for inode 0x%llx, attr 0x%x.\n", + (unsigned long long)ni->mft_no, ni->type); + + /* Should be called for the first extent of the attribute. */ + if (le64_to_cpu(a->data.non_resident.lowest_vcn)) { + ntfs_debug("Eeek! Should be called for the first extent of the attribute. Aborting...\n"); + return -EINVAL; + } + + /* Some preliminary sanity checking. */ + if (!NInoNonResident(ni)) { + ntfs_debug("Eeek! Trying to make resident attribute resident. Aborting...\n"); + return -EINVAL; + } + + /* Make sure this is not $MFT/$BITMAP or Windows will not boot! */ + if (ni->type == AT_BITMAP && ni->mft_no == FILE_MFT) + return -EPERM; + + /* Check that the attribute is allowed to be resident. */ + err = ntfs_attr_can_be_resident(vol, ni->type); + if (err) + return err; + + if (NInoCompressed(ni) || NInoEncrypted(ni)) { + ntfs_debug("Making compressed or encrypted files resident is not implemented yet.\n"); + return -EOPNOTSUPP; + } + + /* Work out offsets into and size of the resident attribute. */ + name_ofs = 24; /* = sizeof(resident_struct attr_record); */ + val_ofs = (name_ofs + a->name_length * sizeof(__le16) + 7) & ~7; + arec_size = (val_ofs + ni->data_size + 7) & ~7; + + /* Sanity check the size before we start modifying the attribute. */ + if (le32_to_cpu(ctx->mrec->bytes_in_use) - le32_to_cpu(a->length) + + arec_size > le32_to_cpu(ctx->mrec->bytes_allocated)) { + ntfs_debug("Not enough space to make attribute resident\n"); + return -ENOSPC; + } + + /* Read and cache the whole runlist if not already done. */ + err = ntfs_attr_map_whole_runlist(ni); + if (err) + return err; + + /* Move the attribute name if it exists and update the offset. */ + if (a->name_length) { + memmove((u8 *)a + name_ofs, (u8 *)a + le16_to_cpu(a->name_offset), + a->name_length * sizeof(__le16)); + } + a->name_offset = cpu_to_le16(name_ofs); + + /* Resize the resident part of the attribute record. */ + if (ntfs_attr_record_resize(ctx->mrec, a, arec_size) < 0) { + /* + * Bug, because ntfs_attr_record_resize should not fail (we + * already checked that attribute fits MFT record). + */ + ntfs_error(ctx->ntfs_ino->vol->sb, "BUG! Failed to resize attribute record. "); + return -EIO; + } + + /* Convert the attribute record to describe a resident attribute. */ + a->non_resident = 0; + a->flags = 0; + a->data.resident.value_length = cpu_to_le32(ni->data_size); + a->data.resident.value_offset = cpu_to_le16(val_ofs); + /* + * File names cannot be non-resident so we would never see this here + * but at least it serves as a reminder that there may be attributes + * for which we do need to set this flag. (AIA) + */ + if (a->type == AT_FILE_NAME) + a->data.resident.flags = RESIDENT_ATTR_IS_INDEXED; + else + a->data.resident.flags = 0; + a->data.resident.reserved = 0; + + /* + * Deallocate clusters from the runlist. + * + * NOTE: We can use ntfs_cluster_free() because we have already mapped + * the whole run list and thus it doesn't matter that the attribute + * record is in a transiently corrupted state at this moment in time. + */ + err = ntfs_cluster_free(ni, 0, -1, ctx); + if (err) { + ntfs_error(sb, "Eeek! Failed to release allocated clusters"); + ntfs_debug("Ignoring error and leaving behind wasted clusters.\n"); + } + + /* Throw away the now unused runlist. */ + kvfree(ni->runlist.rl); + ni->runlist.rl = NULL; + ni->runlist.count = 0; + /* Update in-memory struct ntfs_attr. */ + NInoClearNonResident(ni); + NInoClearCompressed(ni); + ni->flags &= ~FILE_ATTR_COMPRESSED; + NInoClearSparse(ni); + ni->flags &= ~FILE_ATTR_SPARSE_FILE; + NInoClearEncrypted(ni); + ni->flags &= ~FILE_ATTR_ENCRYPTED; + ni->initialized_size = ni->data_size; + ni->allocated_size = ni->itype.compressed.size = (ni->data_size + 7) & ~7; + ni->itype.compressed.block_size = 0; + ni->itype.compressed.block_size_bits = ni->itype.compressed.block_clusters = 0; + return 0; +} + +/* + * ntfs_non_resident_attr_shrink - shrink a non-resident, open ntfs attribute + * @ni: non-resident ntfs attribute to shrink + * @newsize: new size (in bytes) to which to shrink the attribute + * + * Reduce the size of a non-resident, open ntfs attribute @na to @newsize bytes. + */ +static int ntfs_non_resident_attr_shrink(struct ntfs_inode *ni, const s64 newsize) +{ + struct ntfs_volume *vol; + struct ntfs_attr_search_ctx *ctx; + s64 first_free_vcn; + s64 nr_freed_clusters; + int err; + struct ntfs_inode *base_ni; + + ntfs_debug("Inode 0x%llx attr 0x%x new size %lld\n", + (unsigned long long)ni->mft_no, ni->type, (long long)newsize); + + vol = ni->vol; + + if (NInoAttr(ni)) + base_ni = ni->ext.base_ntfs_ino; + else + base_ni = ni; + + /* + * Check the attribute type and the corresponding minimum size + * against @newsize and fail if @newsize is too small. + */ + err = ntfs_attr_size_bounds_check(vol, ni->type, newsize); + if (err) { + if (err == -ERANGE) + ntfs_debug("Eeek! Size bounds check failed. Aborting...\n"); + else if (err == -ENOENT) + err = -EIO; + return err; + } + + /* The first cluster outside the new allocation. */ + if (NInoCompressed(ni)) + /* + * For compressed files we must keep full compressions blocks, + * but currently we do not decompress/recompress the last + * block to truncate the data, so we may leave more allocated + * clusters than really needed. + */ + first_free_vcn = ntfs_bytes_to_cluster(vol, + ((newsize - 1) | (ni->itype.compressed.block_size - 1)) + 1); + else + first_free_vcn = + ntfs_bytes_to_cluster(vol, newsize + vol->cluster_size - 1); + + if (first_free_vcn < 0) + return -EINVAL; + /* + * Compare the new allocation with the old one and only deallocate + * clusters if there is a change. + */ + if (ntfs_bytes_to_cluster(vol, ni->allocated_size) != first_free_vcn) { + struct ntfs_attr_search_ctx *ctx; + + err = ntfs_attr_map_whole_runlist(ni); + if (err) { + ntfs_debug("Eeek! ntfs_attr_map_whole_runlist failed.\n"); + return err; + } + + ctx = ntfs_attr_get_search_ctx(ni, NULL); + if (!ctx) { + ntfs_error(vol->sb, "%s: Failed to get search context", __func__); + return -ENOMEM; + } + + /* Deallocate all clusters starting with the first free one. */ + nr_freed_clusters = ntfs_cluster_free(ni, first_free_vcn, -1, ctx); + if (nr_freed_clusters < 0) { + ntfs_debug("Eeek! Freeing of clusters failed. Aborting...\n"); + ntfs_attr_put_search_ctx(ctx); + return (int)nr_freed_clusters; + } + ntfs_attr_put_search_ctx(ctx); + + /* Truncate the runlist itself. */ + if (ntfs_rl_truncate_nolock(vol, &ni->runlist, first_free_vcn)) { + /* + * Failed to truncate the runlist, so just throw it + * away, it will be mapped afresh on next use. + */ + kvfree(ni->runlist.rl); + ni->runlist.rl = NULL; + ntfs_error(vol->sb, "Eeek! Run list truncation failed.\n"); + return -EIO; + } + + /* Prepare to mapping pairs update. */ + ni->allocated_size = ntfs_cluster_to_bytes(vol, first_free_vcn); + + if (NInoSparse(ni) || NInoCompressed(ni)) { + if (nr_freed_clusters) { + ni->itype.compressed.size -= + ntfs_cluster_to_bytes(vol, nr_freed_clusters); + VFS_I(base_ni)->i_blocks = ni->itype.compressed.size >> 9; + } + } else + VFS_I(base_ni)->i_blocks = ni->allocated_size >> 9; + + /* Write mapping pairs for new runlist. */ + err = ntfs_attr_update_mapping_pairs(ni, 0 /*first_free_vcn*/); + if (err) { + ntfs_debug("Eeek! Mapping pairs update failed. Leaving inconstant metadata. Run chkdsk.\n"); + return err; + } + } + + /* Get the first attribute record. */ + ctx = ntfs_attr_get_search_ctx(base_ni, NULL); + if (!ctx) { + ntfs_error(vol->sb, "%s: Failed to get search context", __func__); + return -ENOMEM; + } + + err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len, CASE_SENSITIVE, + 0, NULL, 0, ctx); + if (err) { + if (err == -ENOENT) + err = -EIO; + ntfs_debug("Eeek! Lookup of first attribute extent failed. Leaving inconstant metadata.\n"); + goto put_err_out; + } + + /* Update data and initialized size. */ + ni->data_size = newsize; + ctx->attr->data.non_resident.data_size = cpu_to_le64(newsize); + if (newsize < ni->initialized_size) { + ni->initialized_size = newsize; + ctx->attr->data.non_resident.initialized_size = cpu_to_le64(newsize); + } + + /* + * Drop any page-cache folios that now lie beyond the shrunk + * attribute. The clusters backing them have just been freed and the + * runlist truncated, so leaving stale dirty folios around makes a + * later writeback map a vcn past the new allocation, which fails with + * -ENOENT and loses the write. + */ + truncate_inode_pages(VFS_I(ni)->i_mapping, newsize); + + /* Update data size in the index. */ + if (ni->type == AT_DATA && ni->name == AT_UNNAMED) + NInoSetFileNameDirty(ni); + + /* If the attribute now has zero size, make it resident. */ + if (!newsize && !NInoEncrypted(ni) && !NInoCompressed(ni)) { + err = ntfs_attr_make_resident(ni, ctx); + if (err) { + /* If couldn't make resident, just continue. */ + if (err != -EPERM) + ntfs_error(ni->vol->sb, + "Failed to make attribute resident. Leaving as is...\n"); + } + } + + /* Set the inode dirty so it is written out later. */ + mark_mft_record_dirty(ctx->ntfs_ino); + /* Done! */ + ntfs_attr_put_search_ctx(ctx); + return 0; +put_err_out: + ntfs_attr_put_search_ctx(ctx); + return err; +} + +/* + * ntfs_non_resident_attr_expand - expand a non-resident, open ntfs attribute + * @ni: non-resident ntfs attribute to expand + * @prealloc_size: preallocation size (in bytes) to which to expand the attribute + * @newsize: new size (in bytes) to which to expand the attribute + * @holes: how to create a hole if expanding + * @need_lock: whether mrec lock is needed or not + * + * Expand the size of a non-resident, open ntfs attribute @na to @newsize bytes, + * by allocating new clusters. + */ +static int ntfs_non_resident_attr_expand(struct ntfs_inode *ni, const s64 newsize, + const s64 prealloc_size, unsigned int holes, bool need_lock) +{ + s64 lcn_seek_from; + s64 first_free_vcn; + struct ntfs_volume *vol; + struct ntfs_attr_search_ctx *ctx = NULL; + struct runlist_element *rl, *rln; + s64 org_alloc_size, org_compressed_size; + int err, err2; + struct ntfs_inode *base_ni; + struct super_block *sb = ni->vol->sb; + size_t new_rl_count; + + ntfs_debug("Inode 0x%llx, attr 0x%x, new size %lld old size %lld\n", + (unsigned long long)ni->mft_no, ni->type, + (long long)newsize, (long long)ni->data_size); + + vol = ni->vol; + + if (NInoAttr(ni)) + base_ni = ni->ext.base_ntfs_ino; + else + base_ni = ni; + + /* + * Check the attribute type and the corresponding maximum size + * against @newsize and fail if @newsize is too big. + */ + err = ntfs_attr_size_bounds_check(vol, ni->type, newsize); + if (err < 0) { + ntfs_error(sb, "%s: bounds check failed", __func__); + return err; + } + + /* Save for future use. */ + org_alloc_size = ni->allocated_size; + org_compressed_size = ni->itype.compressed.size; + + /* The first cluster outside the new allocation. */ + if (prealloc_size) + first_free_vcn = + ntfs_bytes_to_cluster(vol, prealloc_size + vol->cluster_size - 1); + else + first_free_vcn = + ntfs_bytes_to_cluster(vol, newsize + vol->cluster_size - 1); + if (first_free_vcn < 0) + return -EFBIG; + + /* + * Compare the new allocation with the old one and only allocate + * clusters if there is a change. + */ + if (ntfs_bytes_to_cluster(vol, ni->allocated_size) < first_free_vcn) { + err = ntfs_attr_map_whole_runlist(ni); + if (err) { + ntfs_error(sb, "ntfs_attr_map_whole_runlist failed"); + return err; + } + + /* + * If we extend $DATA attribute on NTFS 3+ volume, we can add + * sparse runs instead of real allocation of clusters. + */ + if ((ni->type == AT_DATA && (vol->major_ver >= 3 || !NInoSparseDisabled(ni))) && + (holes != HOLES_NO)) { + if (NInoCompressed(ni)) { + int last = 0, i = 0; + s64 alloc_size; + u64 more_entries = round_up(first_free_vcn - + ntfs_bytes_to_cluster(vol, ni->allocated_size), + ni->itype.compressed.block_clusters); + + do_div(more_entries, ni->itype.compressed.block_clusters); + + while (ni->runlist.rl[last].length) + last++; + + rl = ntfs_rl_realloc(ni->runlist.rl, last + 1, + last + more_entries + 1); + if (IS_ERR(rl)) { + err = -ENOMEM; + goto put_err_out; + } + + alloc_size = ni->allocated_size; + while (i++ < more_entries) { + rl[last].vcn = ntfs_bytes_to_cluster(vol, + round_up(alloc_size, vol->cluster_size)); + rl[last].length = ni->itype.compressed.block_clusters - + (rl[last].vcn & + (ni->itype.compressed.block_clusters - 1)); + rl[last].lcn = LCN_HOLE; + last++; + alloc_size += ni->itype.compressed.block_size; + } + + rl[last].vcn = first_free_vcn; + rl[last].lcn = LCN_ENOENT; + rl[last].length = 0; + + ni->runlist.rl = rl; + ni->runlist.count += more_entries; + } else { + rl = kmalloc(sizeof(struct runlist_element) * 2, GFP_NOFS); + if (!rl) { + err = -ENOMEM; + goto put_err_out; + } + + rl[0].vcn = ntfs_bytes_to_cluster(vol, ni->allocated_size); + rl[0].lcn = LCN_HOLE; + rl[0].length = first_free_vcn - + ntfs_bytes_to_cluster(vol, ni->allocated_size); + rl[1].vcn = first_free_vcn; + rl[1].lcn = LCN_ENOENT; + rl[1].length = 0; + } + } else { + /* + * Determine first after last LCN of attribute. + * We will start seek clusters from this LCN to avoid + * fragmentation. If there are no valid LCNs in the + * attribute let the cluster allocator choose the + * starting LCN. + */ + lcn_seek_from = -1; + if (ni->runlist.rl->length) { + /* Seek to the last run list element. */ + for (rl = ni->runlist.rl; (rl + 1)->length; rl++) + ; + /* + * If the last LCN is a hole or similar seek + * back to last valid LCN. + */ + while (rl->lcn < 0 && rl != ni->runlist.rl) + rl--; + /* + * Only set lcn_seek_from it the LCN is valid. + */ + if (rl->lcn >= 0) + lcn_seek_from = rl->lcn + rl->length; + } + + rl = ntfs_cluster_alloc(vol, + ntfs_bytes_to_cluster(vol, ni->allocated_size), + first_free_vcn - + ntfs_bytes_to_cluster(vol, ni->allocated_size), + lcn_seek_from, DATA_ZONE, false, false, false); + if (IS_ERR(rl)) { + ntfs_debug("Cluster allocation failed (%lld)", + (long long)first_free_vcn - + ntfs_bytes_to_cluster(vol, ni->allocated_size)); + return PTR_ERR(rl); + } + } + + if (!NInoCompressed(ni)) { + /* Append new clusters to attribute runlist. */ + rln = ntfs_runlists_merge(&ni->runlist, rl, 0, &new_rl_count); + if (IS_ERR(rln)) { + /* Failed, free just allocated clusters. */ + ntfs_error(sb, "Run list merge failed"); + ntfs_cluster_free_from_rl(vol, rl); + kvfree(rl); + return -EIO; + } + ni->runlist.rl = rln; + ni->runlist.count = new_rl_count; + } + + /* Prepare to mapping pairs update. */ + ni->allocated_size = ntfs_cluster_to_bytes(vol, first_free_vcn); + err = ntfs_attr_update_mapping_pairs(ni, 0); + if (err) { + ntfs_debug("Mapping pairs update failed"); + goto rollback; + } + } + + ctx = ntfs_attr_get_search_ctx(base_ni, NULL); + if (!ctx) { + err = -ENOMEM; + if (ni->allocated_size == org_alloc_size) + return err; + goto rollback; + } + + err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len, CASE_SENSITIVE, + 0, NULL, 0, ctx); + if (err) { + if (err == -ENOENT) + err = -EIO; + if (ni->allocated_size != org_alloc_size) + goto rollback; + goto put_err_out; + } + + /* Update data size. */ + ni->data_size = newsize; + ctx->attr->data.non_resident.data_size = cpu_to_le64(newsize); + /* Update data size in the index. */ + if (ni->type == AT_DATA && ni->name == AT_UNNAMED) + NInoSetFileNameDirty(ni); + /* Set the inode dirty so it is written out later. */ + mark_mft_record_dirty(ctx->ntfs_ino); + /* Done! */ + ntfs_attr_put_search_ctx(ctx); + return 0; +rollback: + /* Free allocated clusters. */ + err2 = ntfs_cluster_free(ni, ntfs_bytes_to_cluster(vol, org_alloc_size), + -1, ctx); + if (err2) + ntfs_debug("Leaking clusters"); + + /* Now, truncate the runlist itself. */ + if (need_lock) + down_write(&ni->runlist.lock); + err2 = ntfs_rl_truncate_nolock(vol, &ni->runlist, + ntfs_bytes_to_cluster(vol, org_alloc_size)); + if (need_lock) + up_write(&ni->runlist.lock); + if (err2) { + /* + * Failed to truncate the runlist, so just throw it away, it + * will be mapped afresh on next use. + */ + kvfree(ni->runlist.rl); + ni->runlist.rl = NULL; + ntfs_error(sb, "Couldn't truncate runlist. Rollback failed"); + } else { + /* Prepare to mapping pairs update. */ + ni->allocated_size = org_alloc_size; + /* Restore mapping pairs. */ + if (need_lock) + down_read(&ni->runlist.lock); + if (ntfs_attr_update_mapping_pairs(ni, 0)) + ntfs_error(sb, "Failed to restore old mapping pairs"); + if (need_lock) + up_read(&ni->runlist.lock); + + if (NInoSparse(ni) || NInoCompressed(ni)) { + ni->itype.compressed.size = org_compressed_size; + VFS_I(base_ni)->i_blocks = ni->itype.compressed.size >> 9; + } else + VFS_I(base_ni)->i_blocks = ni->allocated_size >> 9; + } + if (ctx) + ntfs_attr_put_search_ctx(ctx); + return err; +put_err_out: + if (ctx) + ntfs_attr_put_search_ctx(ctx); + return err; +} + +/* + * ntfs_resident_attr_resize - resize a resident, open ntfs attribute + * @attr_ni: resident ntfs inode to resize + * @newsize: new size (in bytes) to which to resize the attribute + * @prealloc_size: preallocation size (in bytes) to which to resize the attribute + * @holes: flags indicating how to handle holes + * + * Change the size of a resident, open ntfs attribute @na to @newsize bytes. + */ +static int ntfs_resident_attr_resize(struct ntfs_inode *attr_ni, const s64 newsize, + const s64 prealloc_size, unsigned int holes) +{ + struct ntfs_attr_search_ctx *ctx; + struct ntfs_volume *vol = attr_ni->vol; + struct super_block *sb = vol->sb; + int err = -EIO; + struct ntfs_inode *base_ni, *ext_ni = NULL; + +attr_resize_again: + ntfs_debug("Inode 0x%llx attr 0x%x new size %lld\n", + (unsigned long long)attr_ni->mft_no, attr_ni->type, + (long long)newsize); + + if (NInoAttr(attr_ni)) + base_ni = attr_ni->ext.base_ntfs_ino; + else + base_ni = attr_ni; + + /* Get the attribute record that needs modification. */ + ctx = ntfs_attr_get_search_ctx(base_ni, NULL); + if (!ctx) { + ntfs_error(sb, "%s: Failed to get search context", __func__); + return -ENOMEM; + } + + err = ntfs_attr_lookup(attr_ni->type, attr_ni->name, attr_ni->name_len, + 0, 0, NULL, 0, ctx); + if (err) { + ntfs_error(sb, "ntfs_attr_lookup failed"); + goto put_err_out; + } + + /* + * Check the attribute type and the corresponding minimum and maximum + * sizes against @newsize and fail if @newsize is out of bounds. + */ + err = ntfs_attr_size_bounds_check(vol, attr_ni->type, newsize); + if (err) { + if (err == -ENOENT) + err = -EIO; + ntfs_debug("%s: bounds check failed", __func__); + goto put_err_out; + } + /* + * If @newsize is bigger than the mft record we need to make the + * attribute non-resident if the attribute type supports it. If it is + * smaller we can go ahead and attempt the resize. + */ + if (newsize < vol->mft_record_size) { + /* Perform the resize of the attribute record. */ + err = ntfs_resident_attr_value_resize(ctx->mrec, ctx->attr, + newsize); + if (!err) { + /* Update attribute size everywhere. */ + attr_ni->data_size = attr_ni->initialized_size = newsize; + attr_ni->allocated_size = (newsize + 7) & ~7; + if (NInoCompressed(attr_ni) || NInoSparse(attr_ni)) + attr_ni->itype.compressed.size = attr_ni->allocated_size; + if (attr_ni->type == AT_DATA && attr_ni->name == AT_UNNAMED) + NInoSetFileNameDirty(attr_ni); + goto resize_done; + } + + /* Prefer AT_INDEX_ALLOCATION instead of AT_ATTRIBUTE_LIST */ + if (err == -ENOSPC && ctx->attr->type == AT_INDEX_ROOT) + goto put_err_out; + + } + /* There is not enough space in the mft record to perform the resize. */ + + /* Make the attribute non-resident if possible. */ + err = ntfs_attr_make_non_resident(attr_ni, + le32_to_cpu(ctx->attr->data.resident.value_length)); + if (!err) { + mark_mft_record_dirty(ctx->ntfs_ino); + ntfs_attr_put_search_ctx(ctx); + /* Resize non-resident attribute */ + return ntfs_non_resident_attr_expand(attr_ni, newsize, prealloc_size, holes, true); + } else if (err != -ENOSPC && err != -EPERM) { + ntfs_error(sb, "Failed to make attribute non-resident"); + goto put_err_out; + } + + /* Try to make other attributes non-resident and retry each time. */ + ntfs_attr_reinit_search_ctx(ctx); + while (!(err = ntfs_attr_lookup(AT_UNUSED, NULL, 0, 0, 0, NULL, 0, ctx))) { + struct inode *tvi; + struct attr_record *a; + u32 value_len; + + a = ctx->attr; + if (a->non_resident || a->type == AT_ATTRIBUTE_LIST) + continue; + value_len = le32_to_cpu(a->data.resident.value_length); + + if (ntfs_attr_can_be_non_resident(vol, a->type)) + continue; + + /* + * Check out whether convert is reasonable. Assume that mapping + * pairs will take 8 bytes. + */ + if (le32_to_cpu(a->length) <= (sizeof(struct attr_record) - sizeof(s64)) + + ((a->name_length * sizeof(__le16) + 7) & ~7) + 8) + continue; + if (a->type == AT_DATA && !value_len) + continue; + + if (a->type == AT_DATA) + tvi = ntfs_iget(sb, base_ni->mft_no); + else + tvi = ntfs_attr_iget(VFS_I(base_ni), a->type, + (__le16 *)((u8 *)a + le16_to_cpu(a->name_offset)), + a->name_length); + if (IS_ERR(tvi)) { + ntfs_error(sb, "Couldn't open attribute"); + continue; + } + + if (ntfs_attr_make_non_resident(NTFS_I(tvi), value_len)) { + iput(tvi); + continue; + } + + mark_mft_record_dirty(ctx->ntfs_ino); + iput(tvi); + ntfs_attr_put_search_ctx(ctx); + goto attr_resize_again; + } + + /* Check whether error occurred. */ + if (err != -ENOENT) { + ntfs_error(sb, "%s: Attribute lookup failed 1", __func__); + goto put_err_out; + } + + /* + * The standard information and attribute list attributes can't be + * moved out from the base MFT record, so try to move out others. + */ + if (attr_ni->type == AT_STANDARD_INFORMATION || + attr_ni->type == AT_ATTRIBUTE_LIST) { + ntfs_attr_put_search_ctx(ctx); + + if (!NInoAttrList(base_ni)) { + err = ntfs_inode_add_attrlist(base_ni); + if (err) + return err; + } + + err = ntfs_inode_free_space(base_ni, sizeof(struct attr_record)); + if (err) { + err = -ENOSPC; + ntfs_error(sb, + "Couldn't free space in the MFT record to make attribute list non resident"); + return err; + } + err = ntfs_attrlist_update(base_ni); + if (err) + return err; + goto attr_resize_again; + } + + /* + * Move the attribute to a new mft record, creating an attribute list + * attribute or modifying it if it is already present. + */ + + /* Point search context back to attribute which we need resize. */ + ntfs_attr_reinit_search_ctx(ctx); + err = ntfs_attr_lookup(attr_ni->type, attr_ni->name, attr_ni->name_len, + CASE_SENSITIVE, 0, NULL, 0, ctx); + if (err) { + ntfs_error(sb, "%s: Attribute lookup failed 2", __func__); + goto put_err_out; + } + + /* + * Check whether attribute is already single in this MFT record. + * 8 added for the attribute terminator. + */ + if (le32_to_cpu(ctx->mrec->bytes_in_use) == + le16_to_cpu(ctx->mrec->attrs_offset) + le32_to_cpu(ctx->attr->length) + 8) { + err = -ENOSPC; + ntfs_debug("MFT record is filled with one attribute\n"); + goto put_err_out; + } + + /* Add attribute list if not present. */ + if (!NInoAttrList(base_ni)) { + ntfs_attr_put_search_ctx(ctx); + err = ntfs_inode_add_attrlist(base_ni); + if (err) + return err; + goto attr_resize_again; + } + + /* Allocate new mft record. */ + err = ntfs_mft_record_alloc(base_ni->vol, 0, &ext_ni, base_ni, NULL); + if (err) { + ntfs_error(sb, "Couldn't allocate MFT record"); + goto put_err_out; + } + unmap_mft_record(ext_ni); + + /* Move attribute to it. */ + err = ntfs_attr_record_move_to(ctx, ext_ni); + if (err) { + ntfs_error(sb, "Couldn't move attribute to new MFT record"); + err = -ENOMEM; + goto put_err_out; + } + + err = ntfs_attrlist_update(base_ni); + if (err < 0) + goto put_err_out; + + ntfs_attr_put_search_ctx(ctx); + /* Try to perform resize once again. */ + goto attr_resize_again; + +resize_done: + /* + * Set the inode (and its base inode if it exists) dirty so it is + * written out later. + */ + mark_mft_record_dirty(ctx->ntfs_ino); + ntfs_attr_put_search_ctx(ctx); + return 0; + +put_err_out: + ntfs_attr_put_search_ctx(ctx); + return err; +} + +int __ntfs_attr_truncate_vfs(struct ntfs_inode *ni, const s64 newsize, + const s64 i_size) +{ + int err = 0; + + if (newsize < 0 || + (ni->mft_no == FILE_MFT && ni->type == AT_DATA)) { + ntfs_debug("Invalid arguments passed.\n"); + return -EINVAL; + } + + ntfs_debug("Entering for inode 0x%llx, attr 0x%x, size %lld\n", + (unsigned long long)ni->mft_no, ni->type, newsize); + + if (NInoNonResident(ni)) { + if (newsize > i_size) { + down_write(&ni->runlist.lock); + err = ntfs_non_resident_attr_expand(ni, newsize, 0, + NVolDisableSparse(ni->vol) ? + HOLES_NO : HOLES_OK, + false); + up_write(&ni->runlist.lock); + } else + err = ntfs_non_resident_attr_shrink(ni, newsize); + } else + err = ntfs_resident_attr_resize(ni, newsize, 0, + NVolDisableSparse(ni->vol) ? + HOLES_NO : HOLES_OK); + ntfs_debug("Return status %d\n", err); + return err; +} + +int ntfs_attr_expand(struct ntfs_inode *ni, const s64 newsize, const s64 prealloc_size) +{ + int err = 0; + + if (newsize < 0 || + (ni->mft_no == FILE_MFT && ni->type == AT_DATA)) { + ntfs_debug("Invalid arguments passed.\n"); + return -EINVAL; + } + + ntfs_debug("Entering for inode 0x%llx, attr 0x%x, size %lld\n", + (unsigned long long)ni->mft_no, ni->type, newsize); + + if (ni->data_size == newsize) { + ntfs_debug("Size is already ok\n"); + return 0; + } + + /* + * Encrypted attributes are not supported. We return access denied, + * which is what Windows NT4 does, too. + */ + if (NInoEncrypted(ni)) { + pr_err("Failed to truncate encrypted attribute\n"); + return -EACCES; + } + + if (NInoNonResident(ni)) { + if (newsize > ni->data_size) + err = ntfs_non_resident_attr_expand(ni, newsize, prealloc_size, + NVolDisableSparse(ni->vol) ? + HOLES_NO : HOLES_OK, true); + } else + err = ntfs_resident_attr_resize(ni, newsize, prealloc_size, + NVolDisableSparse(ni->vol) ? + HOLES_NO : HOLES_OK); + if (!err) + i_size_write(VFS_I(ni), newsize); + ntfs_debug("Return status %d\n", err); + return err; +} + +/* + * ntfs_attr_truncate_i - resize an ntfs attribute + * @ni: open ntfs inode to resize + * @newsize: new size (in bytes) to which to resize the attribute + * @holes: how to create a hole if expanding + * + * Change the size of an open ntfs attribute @na to @newsize bytes. If the + * attribute is made bigger and the attribute is resident the newly + * "allocated" space is cleared and if the attribute is non-resident the + * newly allocated space is marked as not initialised and no real allocation + * on disk is performed. + */ +int ntfs_attr_truncate_i(struct ntfs_inode *ni, const s64 newsize, unsigned int holes) +{ + int err; + + if (newsize < 0 || + (ni->mft_no == FILE_MFT && ni->type == AT_DATA)) { + ntfs_debug("Invalid arguments passed.\n"); + return -EINVAL; + } + + ntfs_debug("Entering for inode 0x%llx, attr 0x%x, size %lld\n", + (unsigned long long)ni->mft_no, ni->type, newsize); + + if (ni->data_size == newsize) { + ntfs_debug("Size is already ok\n"); + return 0; + } + + /* + * Encrypted attributes are not supported. We return access denied, + * which is what Windows NT4 does, too. + */ + if (NInoEncrypted(ni)) { + pr_err("Failed to truncate encrypted attribute\n"); + return -EACCES; + } + + if (NInoCompressed(ni)) { + pr_err("Failed to truncate compressed attribute\n"); + return -EOPNOTSUPP; + } + + if (NInoNonResident(ni)) { + if (newsize > ni->data_size) + err = ntfs_non_resident_attr_expand(ni, newsize, 0, holes, true); + else + err = ntfs_non_resident_attr_shrink(ni, newsize); + } else + err = ntfs_resident_attr_resize(ni, newsize, 0, holes); + ntfs_debug("Return status %d\n", err); + return err; +} + +/* + * Resize an attribute, creating a hole if relevant + */ +int ntfs_attr_truncate(struct ntfs_inode *ni, const s64 newsize) +{ + return ntfs_attr_truncate_i(ni, newsize, + NVolDisableSparse(ni->vol) ? + HOLES_NO : HOLES_OK); +} + +int ntfs_attr_map_cluster(struct ntfs_inode *ni, s64 vcn_start, s64 *lcn_start, + s64 *lcn_count, s64 max_clu_count, bool *balloc, bool update_mp, + bool skip_holes) +{ + struct ntfs_volume *vol = ni->vol; + struct ntfs_attr_search_ctx *ctx; + struct runlist_element *rl, *rlc; + s64 vcn = vcn_start, lcn, clu_count; + s64 lcn_seek_from = -1; + int err = 0; + size_t new_rl_count; + + err = ntfs_attr_map_whole_runlist(ni); + if (err) + return err; + + if (NInoAttr(ni)) + ctx = ntfs_attr_get_search_ctx(ni->ext.base_ntfs_ino, NULL); + else + ctx = ntfs_attr_get_search_ctx(ni, NULL); + if (!ctx) { + ntfs_error(vol->sb, "%s: Failed to get search context", __func__); + return -ENOMEM; + } + + err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len, + CASE_SENSITIVE, vcn, NULL, 0, ctx); + if (err) { + ntfs_error(vol->sb, + "ntfs_attr_lookup failed, ntfs inode(mft_no : %llu) type : 0x%x, err : %d", + ni->mft_no, ni->type, err); + goto out; + } + + rl = ntfs_attr_find_vcn_nolock(ni, vcn, ctx); + if (IS_ERR(rl)) { + ntfs_error(vol->sb, "Failed to find run after mapping runlist."); + err = PTR_ERR(rl); + goto out; + } + + lcn = ntfs_rl_vcn_to_lcn(rl, vcn); + clu_count = min(max_clu_count, rl->length - (vcn - rl->vcn)); + if (lcn >= LCN_HOLE) { + if (lcn > LCN_DELALLOC || + (lcn == LCN_HOLE && skip_holes)) { + *lcn_start = lcn; + *lcn_count = clu_count; + *balloc = false; + goto out; + } + } else { + WARN_ON(lcn == LCN_RL_NOT_MAPPED); + if (lcn == LCN_ENOENT) + err = -ENOENT; + else + err = -EIO; + goto out; + } + + /* Search backwards to find the best lcn to start seek from. */ + rlc = rl; + while (rlc->vcn) { + rlc--; + if (rlc->lcn >= 0) { + /* + * avoid fragmenting a compressed file + * Windows does not do that, and that may + * not be desirable for files which can + * be updated + */ + if (NInoCompressed(ni)) + lcn_seek_from = rlc->lcn + rlc->length; + else + lcn_seek_from = rlc->lcn + (vcn - rlc->vcn); + break; + } + } + + if (lcn_seek_from == -1) { + /* Backwards search failed, search forwards. */ + rlc = rl; + while (rlc->length) { + rlc++; + if (rlc->lcn >= 0) { + lcn_seek_from = rlc->lcn - (rlc->vcn - vcn); + if (lcn_seek_from < -1) + lcn_seek_from = -1; + break; + } + } + } + + rlc = ntfs_cluster_alloc(vol, vcn, clu_count, lcn_seek_from, DATA_ZONE, + false, true, true); + if (IS_ERR(rlc)) { + err = PTR_ERR(rlc); + goto out; + } + + WARN_ON(rlc->vcn != vcn); + lcn = rlc->lcn; + clu_count = rlc->length; + + rl = ntfs_runlists_merge(&ni->runlist, rlc, 0, &new_rl_count); + if (IS_ERR(rl)) { + ntfs_error(vol->sb, "Failed to merge runlists"); + err = PTR_ERR(rl); + if (ntfs_cluster_free_from_rl(vol, rlc)) + ntfs_error(vol->sb, "Failed to free hot clusters."); + kvfree(rlc); + goto out; + } + ni->runlist.rl = rl; + ni->runlist.count = new_rl_count; + + if (!update_mp) { + u64 free = atomic64_read(&vol->free_clusters) * 100; + + do_div(free, vol->nr_clusters); + if (free <= 5) + update_mp = true; + } + + if (update_mp) { + ntfs_attr_reinit_search_ctx(ctx); + err = ntfs_attr_update_mapping_pairs(ni, 0); + if (err) { + int err2; + + err2 = ntfs_cluster_free(ni, vcn, clu_count, ctx); + if (err2 < 0) + ntfs_error(vol->sb, + "Failed to free cluster allocation. Leaving inconstant metadata.\n"); + goto out; + } + } else { + VFS_I(ni)->i_blocks += clu_count << (vol->cluster_size_bits - 9); + NInoSetRunlistDirty(ni); + mark_mft_record_dirty(ni); + } + + *lcn_start = lcn; + *lcn_count = clu_count; + *balloc = true; +out: + ntfs_attr_put_search_ctx(ctx); + return err; +} + +/* + * ntfs_attr_rm - remove attribute from ntfs inode + * @ni: opened ntfs attribute to delete + * + * Remove attribute and all it's extents from ntfs inode. If attribute was non + * resident also free all clusters allocated by attribute. + */ +int ntfs_attr_rm(struct ntfs_inode *ni) +{ + struct ntfs_attr_search_ctx *ctx; + int err = 0, ret = 0; + struct ntfs_inode *base_ni; + struct super_block *sb = ni->vol->sb; + + if (NInoAttr(ni)) + base_ni = ni->ext.base_ntfs_ino; + else + base_ni = ni; + + ntfs_debug("Entering for inode 0x%llx, attr 0x%x.\n", + (long long) ni->mft_no, ni->type); + + /* Free cluster allocation. */ + if (NInoNonResident(ni)) { + struct ntfs_attr_search_ctx *ctx; + + err = ntfs_attr_map_whole_runlist(ni); + if (err) + return err; + ctx = ntfs_attr_get_search_ctx(ni, NULL); + if (!ctx) { + ntfs_error(sb, "%s: Failed to get search context", __func__); + return -ENOMEM; + } + + ret = ntfs_cluster_free(ni, 0, -1, ctx); + if (ret < 0) + ntfs_error(sb, + "Failed to free cluster allocation. Leaving inconstant metadata.\n"); + ntfs_attr_put_search_ctx(ctx); + } + + /* Search for attribute extents and remove them all. */ + ctx = ntfs_attr_get_search_ctx(base_ni, NULL); + if (!ctx) { + ntfs_error(sb, "%s: Failed to get search context", __func__); + return -ENOMEM; + } + while (!(err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len, + CASE_SENSITIVE, 0, NULL, 0, ctx))) { + err = ntfs_attr_record_rm(ctx); + if (err) { + ntfs_error(sb, + "Failed to remove attribute extent. Leaving inconstant metadata.\n"); + ret = err; + } + ntfs_attr_reinit_search_ctx(ctx); + } + ntfs_attr_put_search_ctx(ctx); + if (err != -ENOENT) { + ntfs_error(sb, "Attribute lookup failed. Probably leaving inconstant metadata.\n"); + ret = err; + } + + return ret; +} + +int ntfs_attr_exist(struct ntfs_inode *ni, const __le32 type, __le16 *name, + u32 name_len) +{ + struct ntfs_attr_search_ctx *ctx; + int ret; + + ntfs_debug("Entering\n"); + + ctx = ntfs_attr_get_search_ctx(ni, NULL); + if (!ctx) { + ntfs_error(ni->vol->sb, "%s: Failed to get search context", + __func__); + return 0; + } + + ret = ntfs_attr_lookup(type, name, name_len, CASE_SENSITIVE, + 0, NULL, 0, ctx); + ntfs_attr_put_search_ctx(ctx); + + return !ret; +} + +int ntfs_attr_remove(struct ntfs_inode *ni, const __le32 type, __le16 *name, + u32 name_len) +{ + int err; + struct inode *attr_vi; + struct ntfs_inode *attr_ni; + + ntfs_debug("Entering\n"); + + if (!ni) + return -EINVAL; + + attr_vi = ntfs_attr_iget(VFS_I(ni), type, name, name_len); + if (IS_ERR(attr_vi)) { + err = PTR_ERR(attr_vi); + ntfs_error(ni->vol->sb, "Failed to open attribute 0x%02x of inode 0x%llx", + type, (unsigned long long)ni->mft_no); + return err; + } + attr_ni = NTFS_I(attr_vi); + + err = ntfs_attr_rm(attr_ni); + if (err) + ntfs_error(ni->vol->sb, "Failed to remove attribute 0x%02x of inode 0x%llx", + type, (unsigned long long)ni->mft_no); + iput(attr_vi); + return err; +} + +/* + * ntfs_attr_readall - read the entire data from an ntfs attribute + * @ni: open ntfs inode in which the ntfs attribute resides + * @type: attribute type + * @name: attribute name in little endian Unicode or AT_UNNAMED or NULL + * @name_len: length of attribute @name in Unicode characters (if @name given) + * @data_size: if non-NULL then store here the data size + * + * This function will read the entire content of an ntfs attribute. + * If @name is AT_UNNAMED then look specifically for an unnamed attribute. + * If @name is NULL then the attribute could be either named or not. + * In both those cases @name_len is not used at all. + * + * On success a buffer is allocated with the content of the attribute + * and which needs to be freed when it's not needed anymore. If the + * @data_size parameter is non-NULL then the data size is set there. + */ +void *ntfs_attr_readall(struct ntfs_inode *ni, const __le32 type, + __le16 *name, u32 name_len, s64 *data_size) +{ + struct ntfs_inode *bmp_ni; + struct inode *bmp_vi; + void *data, *ret = NULL; + s64 size; + struct super_block *sb = ni->vol->sb; + + ntfs_debug("Entering\n"); + + bmp_vi = ntfs_attr_iget(VFS_I(ni), type, name, name_len); + if (IS_ERR(bmp_vi)) { + ntfs_debug("ntfs_attr_iget failed"); + goto err_exit; + } + bmp_ni = NTFS_I(bmp_vi); + + if (bmp_ni->data_size > NTFS_ATTR_READALL_MAX_SIZE && + (bmp_ni->type != AT_BITMAP || + bmp_ni->data_size > ((ni->vol->nr_clusters + 7) >> 3))) { + ntfs_error(sb, "Invalid attribute data size"); + goto out; + } + + data = kvmalloc(bmp_ni->data_size, GFP_NOFS); + if (!data) + goto out; + + size = ntfs_inode_attr_pread(VFS_I(bmp_ni), 0, bmp_ni->data_size, + (u8 *)data); + if (size != bmp_ni->data_size) { + ntfs_error(sb, "ntfs_attr_pread failed"); + kvfree(data); + goto out; + } + ret = data; + if (data_size) + *data_size = size; +out: + iput(bmp_vi); +err_exit: + ntfs_debug("\n"); + return ret; +} + +int ntfs_non_resident_attr_insert_range(struct ntfs_inode *ni, s64 start_vcn, s64 len) +{ + struct ntfs_volume *vol = ni->vol; + struct runlist_element *hole_rl, *rl; + struct ntfs_attr_search_ctx *ctx; + int ret; + size_t new_rl_count; + + if (NInoAttr(ni) || ni->type != AT_DATA) + return -EOPNOTSUPP; + if (start_vcn > ntfs_bytes_to_cluster(vol, ni->allocated_size)) + return -EINVAL; + + hole_rl = kmalloc(sizeof(*hole_rl) * 2, GFP_NOFS); + if (!hole_rl) + return -ENOMEM; + hole_rl[0].vcn = start_vcn; + hole_rl[0].lcn = LCN_HOLE; + hole_rl[0].length = len; + hole_rl[1].vcn = start_vcn + len; + hole_rl[1].lcn = LCN_ENOENT; + hole_rl[1].length = 0; + + down_write(&ni->runlist.lock); + ret = ntfs_attr_map_whole_runlist(ni); + if (ret) { + up_write(&ni->runlist.lock); + kfree(hole_rl); + return ret; + } + + rl = ntfs_rl_find_vcn_nolock(ni->runlist.rl, start_vcn); + if (!rl) { + up_write(&ni->runlist.lock); + kfree(hole_rl); + return -EIO; + } + + rl = ntfs_rl_insert_range(ni->runlist.rl, (int)ni->runlist.count, + hole_rl, 1, &new_rl_count); + if (IS_ERR(rl)) { + up_write(&ni->runlist.lock); + kfree(hole_rl); + return PTR_ERR(rl); + } + ni->runlist.rl = rl; + ni->runlist.count = new_rl_count; + + ni->allocated_size += ntfs_cluster_to_bytes(vol, len); + ni->data_size += ntfs_cluster_to_bytes(vol, len); + if (ntfs_cluster_to_bytes(vol, start_vcn) < ni->initialized_size) + ni->initialized_size += ntfs_cluster_to_bytes(vol, len); + ret = ntfs_attr_update_mapping_pairs(ni, 0); + up_write(&ni->runlist.lock); + if (ret) + return ret; + + ctx = ntfs_attr_get_search_ctx(ni, NULL); + if (!ctx) { + ret = -ENOMEM; + return ret; + } + + ret = ntfs_attr_lookup(ni->type, ni->name, ni->name_len, CASE_SENSITIVE, + 0, NULL, 0, ctx); + if (ret) { + ntfs_attr_put_search_ctx(ctx); + return ret; + } + + ctx->attr->data.non_resident.data_size = cpu_to_le64(ni->data_size); + ctx->attr->data.non_resident.initialized_size = cpu_to_le64(ni->initialized_size); + if (ni->type == AT_DATA && ni->name == AT_UNNAMED) + NInoSetFileNameDirty(ni); + mark_mft_record_dirty(ctx->ntfs_ino); + ntfs_attr_put_search_ctx(ctx); + return ret; +} + +int ntfs_non_resident_attr_collapse_range(struct ntfs_inode *ni, s64 start_vcn, s64 len) +{ + struct ntfs_volume *vol = ni->vol; + struct runlist_element *punch_rl, *rl; + struct ntfs_attr_search_ctx *ctx = NULL; + s64 end_vcn; + int dst_cnt; + int ret; + size_t new_rl_cnt; + + if (NInoAttr(ni) || ni->type != AT_DATA) + return -EOPNOTSUPP; + + end_vcn = ntfs_bytes_to_cluster(vol, ni->allocated_size); + if (start_vcn >= end_vcn) + return -EINVAL; + + down_write(&ni->runlist.lock); + ret = ntfs_attr_map_whole_runlist(ni); + if (ret) { + up_write(&ni->runlist.lock); + return ret; + } + + len = min(len, end_vcn - start_vcn); + for (rl = ni->runlist.rl, dst_cnt = 0; rl && rl->length; rl++) + dst_cnt++; + rl = ntfs_rl_find_vcn_nolock(ni->runlist.rl, start_vcn); + if (!rl) { + up_write(&ni->runlist.lock); + return -EIO; + } + + rl = ntfs_rl_collapse_range(ni->runlist.rl, dst_cnt + 1, + start_vcn, len, &punch_rl, &new_rl_cnt); + if (IS_ERR(rl)) { + up_write(&ni->runlist.lock); + return PTR_ERR(rl); + } + ni->runlist.rl = rl; + ni->runlist.count = new_rl_cnt; + + ni->allocated_size -= ntfs_cluster_to_bytes(vol, len); + if (ni->data_size > ntfs_cluster_to_bytes(vol, start_vcn)) { + if (ni->data_size > ntfs_cluster_to_bytes(vol, (start_vcn + len))) + ni->data_size -= ntfs_cluster_to_bytes(vol, len); + else + ni->data_size = ntfs_cluster_to_bytes(vol, start_vcn); + } + if (ni->initialized_size > ntfs_cluster_to_bytes(vol, start_vcn)) { + if (ni->initialized_size > + ntfs_cluster_to_bytes(vol, start_vcn + len)) + ni->initialized_size -= ntfs_cluster_to_bytes(vol, len); + else + ni->initialized_size = ntfs_cluster_to_bytes(vol, start_vcn); + } + + if (ni->allocated_size > 0) { + ret = ntfs_attr_update_mapping_pairs(ni, 0); + if (ret) { + up_write(&ni->runlist.lock); + goto out_rl; + } + } + up_write(&ni->runlist.lock); + + ctx = ntfs_attr_get_search_ctx(ni, NULL); + if (!ctx) { + ret = -ENOMEM; + goto out_rl; + } + + ret = ntfs_attr_lookup(ni->type, ni->name, ni->name_len, CASE_SENSITIVE, + 0, NULL, 0, ctx); + if (ret) + goto out_ctx; + + ctx->attr->data.non_resident.data_size = cpu_to_le64(ni->data_size); + ctx->attr->data.non_resident.initialized_size = cpu_to_le64(ni->initialized_size); + if (ni->allocated_size == 0) + ntfs_attr_make_resident(ni, ctx); + mark_mft_record_dirty(ctx->ntfs_ino); + + ret = ntfs_cluster_free_from_rl(vol, punch_rl); + if (ret) + ntfs_error(vol->sb, "Freeing of clusters failed"); +out_ctx: + if (ctx) + ntfs_attr_put_search_ctx(ctx); +out_rl: + kvfree(punch_rl); + mark_mft_record_dirty(ni); + return ret; +} + +int ntfs_non_resident_attr_punch_hole(struct ntfs_inode *ni, s64 start_vcn, s64 len) +{ + struct ntfs_volume *vol = ni->vol; + struct runlist_element *punch_rl, *rl; + s64 end_vcn; + int dst_cnt; + int ret; + size_t new_rl_count; + + if (NInoAttr(ni) || ni->type != AT_DATA) + return -EOPNOTSUPP; + + end_vcn = ntfs_bytes_to_cluster(vol, ni->allocated_size); + if (start_vcn >= end_vcn) + return -EINVAL; + + down_write(&ni->runlist.lock); + ret = ntfs_attr_map_whole_runlist(ni); + if (ret) { + up_write(&ni->runlist.lock); + return ret; + } + + len = min(len, end_vcn - start_vcn + 1); + for (rl = ni->runlist.rl, dst_cnt = 0; rl && rl->length; rl++) + dst_cnt++; + rl = ntfs_rl_find_vcn_nolock(ni->runlist.rl, start_vcn); + if (!rl) { + up_write(&ni->runlist.lock); + return -EIO; + } + + rl = ntfs_rl_punch_hole(ni->runlist.rl, dst_cnt + 1, + start_vcn, len, &punch_rl, &new_rl_count); + if (IS_ERR(rl)) { + up_write(&ni->runlist.lock); + return PTR_ERR(rl); + } + ni->runlist.rl = rl; + ni->runlist.count = new_rl_count; + + ret = ntfs_attr_update_mapping_pairs(ni, 0); + up_write(&ni->runlist.lock); + if (ret) { + kvfree(punch_rl); + return ret; + } + + ret = ntfs_cluster_free_from_rl(vol, punch_rl); + if (ret) + ntfs_error(vol->sb, "Freeing of clusters failed"); + + kvfree(punch_rl); + mark_mft_record_dirty(ni); + return ret; +} + +int ntfs_attr_fallocate(struct ntfs_inode *ni, loff_t start, loff_t byte_len, bool keep_size) +{ + struct ntfs_volume *vol = ni->vol; + struct mft_record *mrec; + struct ntfs_attr_search_ctx *ctx; + s64 old_data_size; + s64 vcn_start, vcn_end, vcn_uninit, vcn, try_alloc_cnt; + s64 lcn, alloc_cnt; + s64 rl_lcn, rl_length, rl_vcn; + int err = 0; + struct runlist_element *rl; + bool balloc; + + if (NInoAttr(ni) || ni->type != AT_DATA) + return -EINVAL; + + if (NInoNonResident(ni) && !NInoFullyMapped(ni)) { + down_write(&ni->runlist.lock); + err = ntfs_attr_map_whole_runlist(ni); + up_write(&ni->runlist.lock); + if (err) + return err; + } + + mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL); + mrec = map_mft_record(ni); + if (IS_ERR(mrec)) { + mutex_unlock(&ni->mrec_lock); + return PTR_ERR(mrec); + } + + ctx = ntfs_attr_get_search_ctx(ni, mrec); + if (!ctx) { + err = -ENOMEM; + goto out_unmap; + } + + err = ntfs_attr_lookup(AT_DATA, AT_UNNAMED, 0, 0, 0, NULL, 0, ctx); + if (err) { + err = -EIO; + goto out_unmap; + } + + old_data_size = ni->data_size; + if (start + byte_len > ni->data_size) { + err = ntfs_attr_truncate(ni, start + byte_len); + if (err) + goto out_unmap; + if (keep_size) { + ntfs_attr_reinit_search_ctx(ctx); + err = ntfs_attr_lookup(AT_DATA, AT_UNNAMED, 0, 0, 0, NULL, 0, ctx); + if (err) { + err = -EIO; + goto out_unmap; + } + ni->data_size = old_data_size; + if (NInoNonResident(ni)) + ctx->attr->data.non_resident.data_size = + cpu_to_le64(old_data_size); + else + ctx->attr->data.resident.value_length = + cpu_to_le32((u32)old_data_size); + mark_mft_record_dirty(ni); + } + } + + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(ni); + mutex_unlock(&ni->mrec_lock); + + if (!NInoNonResident(ni)) + goto out; + + vcn_start = (s64)ntfs_bytes_to_cluster(vol, start); + vcn_end = (s64)ntfs_bytes_to_cluster(vol, + round_up(start + byte_len, vol->cluster_size)); + vcn_uninit = (s64)ntfs_bytes_to_cluster(vol, + round_up(ni->initialized_size, vol->cluster_size)); + vcn_uninit = min_t(s64, vcn_uninit, vcn_end); + + /* + * we have to allocate clusters for holes and delayed within initialized_size, + * and zero out the clusters only for the holes. + */ + vcn = vcn_start; + while (vcn < vcn_uninit) { + down_read(&ni->runlist.lock); + rl = ntfs_attr_find_vcn_nolock(ni, vcn, NULL); + if (IS_ERR(rl)) { + up_read(&ni->runlist.lock); + err = PTR_ERR(rl); + goto out; + } + rl_lcn = rl->lcn; + rl_length = rl->length; + rl_vcn = rl->vcn; + up_read(&ni->runlist.lock); + + if (rl_lcn > 0) { + vcn += rl_length - (vcn - rl_vcn); + } else if (rl_lcn == LCN_DELALLOC || rl_lcn == LCN_HOLE) { + try_alloc_cnt = min(rl_length - (vcn - rl_vcn), + vcn_uninit - vcn); + + if (rl_lcn == LCN_DELALLOC) { + vcn += try_alloc_cnt; + continue; + } + + while (try_alloc_cnt > 0) { + mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL); + down_write(&ni->runlist.lock); + err = ntfs_attr_map_cluster(ni, vcn, &lcn, &alloc_cnt, + try_alloc_cnt, &balloc, false, false); + up_write(&ni->runlist.lock); + mutex_unlock(&ni->mrec_lock); + if (err) + goto out; + + if (balloc) { + err = ntfs_dio_zero_range(VFS_I(ni), + lcn << vol->cluster_size_bits, + alloc_cnt << + vol->cluster_size_bits); + if (err > 0) + goto out; + } + + if (signal_pending(current)) + goto out; + + vcn += alloc_cnt; + try_alloc_cnt -= alloc_cnt; + } + } else { + err = -EIO; + goto out; + } + } + + /* allocate clusters outside of initialized_size */ + try_alloc_cnt = vcn_end - vcn; + while (try_alloc_cnt > 0) { + mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL); + down_write(&ni->runlist.lock); + err = ntfs_attr_map_cluster(ni, vcn, &lcn, &alloc_cnt, + try_alloc_cnt, &balloc, false, false); + up_write(&ni->runlist.lock); + mutex_unlock(&ni->mrec_lock); + if (err || signal_pending(current)) + goto out; + + vcn += alloc_cnt; + try_alloc_cnt -= alloc_cnt; + cond_resched(); + } + + if (NInoRunlistDirty(ni)) { + mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL); + down_write(&ni->runlist.lock); + err = ntfs_attr_update_mapping_pairs(ni, 0); + if (err) + ntfs_error(ni->vol->sb, "Updating mapping pairs failed"); + else + NInoClearRunlistDirty(ni); + up_write(&ni->runlist.lock); + mutex_unlock(&ni->mrec_lock); + } + return err; +out_unmap: + if (ctx) + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(ni); + mutex_unlock(&ni->mrec_lock); +out: + return err >= 0 ? 0 : err; +} diff --git a/fs/ntfs/attrib.h b/fs/ntfs/attrib.h new file mode 100644 index 000000000..e2224fbfa --- /dev/null +++ b/fs/ntfs/attrib.h @@ -0,0 +1,168 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Defines for attribute handling in NTFS Linux kernel driver. + * + * Copyright (c) 2001-2005 Anton Altaparmakov + * Copyright (c) 2002 Richard Russon + * Copyright (c) 2025 LG Electronics Co., Ltd. + */ + +#ifndef _LINUX_NTFS_ATTRIB_H +#define _LINUX_NTFS_ATTRIB_H + +#include "ntfs.h" +#include "dir.h" + +extern __le16 AT_UNNAMED[]; + +/* + * ntfs_attr_search_ctx - used in attribute search functions + * @mrec: buffer containing mft record to search + * @mapped_mrec: true if @mrec was mapped by the search functions + * @attr: attribute record in @mrec where to begin/continue search + * @is_first: if true ntfs_attr_lookup() begins search with @attr, else after + * @ntfs_ino: Inode owning this attribute search + * @al_entry: Current attribute list entry + * @base_ntfs_ino: Base inode + * @mapped_base_mrec: true if @base_mrec was mapped by the search + * @base_attr: Base attribute record pointer + * + * Structure must be initialized to zero before the first call to one of the + * attribute search functions. Initialize @mrec to point to the mft record to + * search, and @attr to point to the first attribute within @mrec (not necessary + * if calling the _first() functions), and set @is_first to 'true' (not necessary + * if calling the _first() functions). + * + * If @is_first is 'true', the search begins with @attr. If @is_first is 'false', + * the search begins after @attr. This is so that, after the first call to one + * of the search attribute functions, we can call the function again, without + * any modification of the search context, to automagically get the next + * matching attribute. + */ +struct ntfs_attr_search_ctx { + struct mft_record *mrec; + bool mapped_mrec; + struct attr_record *attr; + bool is_first; + struct ntfs_inode *ntfs_ino; + struct attr_list_entry *al_entry; + struct ntfs_inode *base_ntfs_ino; + struct mft_record *base_mrec; + bool mapped_base_mrec; + struct attr_record *base_attr; +}; + +enum { /* ways of processing holes when expanding */ + HOLES_NO, + HOLES_OK, +}; + +int ntfs_map_runlist_nolock(struct ntfs_inode *ni, s64 vcn, + struct ntfs_attr_search_ctx *ctx); +int ntfs_map_runlist(struct ntfs_inode *ni, s64 vcn); +s64 ntfs_attr_vcn_to_lcn_nolock(struct ntfs_inode *ni, const s64 vcn, + const bool write_locked); +struct runlist_element *ntfs_attr_find_vcn_nolock(struct ntfs_inode *ni, + const s64 vcn, struct ntfs_attr_search_ctx *ctx); +struct runlist_element *__ntfs_attr_find_vcn_nolock(struct runlist *runlist, + const s64 vcn); +int ntfs_attr_map_whole_runlist(struct ntfs_inode *ni); +int ntfs_attr_lookup(const __le32 type, const __le16 *name, + const u32 name_len, const u32 ic, + const s64 lowest_vcn, const u8 *val, const u32 val_len, + struct ntfs_attr_search_ctx *ctx); +bool ntfs_attr_list_entry_is_valid(const struct attr_list_entry *ale, + const u8 *al_end); +bool ntfs_attr_list_is_valid(const u8 *al_start, s64 size); + +int load_attribute_list(struct ntfs_inode *base_ni, + u8 *al_start, const s64 size); + +static inline s64 ntfs_attr_size(const struct attr_record *a) +{ + if (!a->non_resident) + return (s64)le32_to_cpu(a->data.resident.value_length); + return le64_to_cpu(a->data.non_resident.data_size); +} + +void ntfs_attr_reinit_search_ctx(struct ntfs_attr_search_ctx *ctx); +struct ntfs_attr_search_ctx *ntfs_attr_get_search_ctx(struct ntfs_inode *ni, + struct mft_record *mrec); +void ntfs_attr_put_search_ctx(struct ntfs_attr_search_ctx *ctx); +int ntfs_attr_size_bounds_check(const struct ntfs_volume *vol, + const __le32 type, const s64 size); +int ntfs_attr_can_be_resident(const struct ntfs_volume *vol, + const __le32 type); +int ntfs_attr_map_cluster(struct ntfs_inode *ni, s64 vcn_start, s64 *lcn_start, + s64 *lcn_count, s64 max_clu_count, bool *balloc, bool update_mp, bool skip_holes); +int ntfs_attr_record_resize(struct mft_record *m, struct attr_record *a, u32 new_size); +int ntfs_resident_attr_value_resize(struct mft_record *m, struct attr_record *a, + const u32 new_size); +int ntfs_attr_make_non_resident(struct ntfs_inode *ni, const u32 data_size); +int ntfs_attr_set(struct ntfs_inode *ni, const s64 ofs, const s64 cnt, + const u8 val); +int ntfs_attr_set_initialized_size(struct ntfs_inode *ni, loff_t new_size); +int ntfs_attr_open(struct ntfs_inode *ni, const __le32 type, + __le16 *name, u32 name_len); +void ntfs_attr_close(struct ntfs_inode *n); +int ntfs_attr_fallocate(struct ntfs_inode *ni, loff_t start, loff_t byte_len, bool keep_size); +int ntfs_non_resident_attr_insert_range(struct ntfs_inode *ni, s64 start_vcn, s64 len); +int ntfs_non_resident_attr_collapse_range(struct ntfs_inode *ni, s64 start_vcn, s64 len); +int ntfs_non_resident_attr_punch_hole(struct ntfs_inode *ni, s64 start_vcn, s64 len); +int __ntfs_attr_truncate_vfs(struct ntfs_inode *ni, const s64 newsize, + const s64 i_size); +int ntfs_attr_expand(struct ntfs_inode *ni, const s64 newsize, const s64 prealloc_size); +int ntfs_attr_truncate_i(struct ntfs_inode *ni, const s64 newsize, unsigned int holes); +int ntfs_attr_truncate(struct ntfs_inode *ni, const s64 newsize); +int ntfs_attr_rm(struct ntfs_inode *ni); +int ntfs_attr_exist(struct ntfs_inode *ni, const __le32 type, __le16 *name, + u32 name_len); +int ntfs_attr_remove(struct ntfs_inode *ni, const __le32 type, __le16 *name, + u32 name_len); +int ntfs_attr_record_rm(struct ntfs_attr_search_ctx *ctx); +int ntfs_attr_record_move_to(struct ntfs_attr_search_ctx *ctx, struct ntfs_inode *ni); +int ntfs_attr_add(struct ntfs_inode *ni, __le32 type, + __le16 *name, u8 name_len, u8 *val, s64 size); +int ntfs_attr_record_move_away(struct ntfs_attr_search_ctx *ctx, int extra); +char *ntfs_attr_name_get(const struct ntfs_volume *vol, const __le16 *uname, + const int uname_len); +void ntfs_attr_name_free(unsigned char **name); +void *ntfs_attr_readall(struct ntfs_inode *ni, const __le32 type, + __le16 *name, u32 name_len, s64 *data_size); +int ntfs_resident_attr_record_add(struct ntfs_inode *ni, __le32 type, + __le16 *name, u8 name_len, u8 *val, u32 size, + __le16 flags); +int ntfs_attr_update_mapping_pairs(struct ntfs_inode *ni, s64 from_vcn); +struct runlist_element *ntfs_attr_vcn_to_rl(struct ntfs_inode *ni, s64 vcn, s64 *lcn); + +/* + * ntfs_attrs_walk - syntactic sugar for walking all attributes in an inode + * @ctx: initialised attribute search context + * + * Syntactic sugar for walking attributes in an inode. + * + * Return 0 on success and -1 on error with errno set to the error code from + * ntfs_attr_lookup(). + * + * Example: When you want to enumerate all attributes in an open ntfs inode + * @ni, you can simply do: + * + * int err; + * struct ntfs_attr_search_ctx *ctx = ntfs_attr_get_search_ctx(ni, NULL); + * if (!ctx) + * // Error code is in errno. Handle this case. + * while (!(err = ntfs_attrs_walk(ctx))) { + * struct attr_record *attr = ctx->attr; + * // attr now contains the next attribute. Do whatever you want + * // with it and then just continue with the while loop. + * } + * if (err && errno != ENOENT) + * // Ooops. An error occurred! You should handle this case. + * // Now finished with all attributes in the inode. + */ +static inline int ntfs_attrs_walk(struct ntfs_attr_search_ctx *ctx) +{ + return ntfs_attr_lookup(AT_UNUSED, NULL, 0, CASE_SENSITIVE, 0, + NULL, 0, ctx); +} +#endif /* _LINUX_NTFS_ATTRIB_H */ diff --git a/fs/ntfs/attrlist.c b/fs/ntfs/attrlist.c new file mode 100644 index 000000000..be3086d34 --- /dev/null +++ b/fs/ntfs/attrlist.c @@ -0,0 +1,302 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Attribute list attribute handling code. + * Part of this file is based on code from the NTFS-3G. + * + * Copyright (c) 2004-2005 Anton Altaparmakov + * Copyright (c) 2004-2005 Yura Pakhuchiy + * Copyright (c) 2006 Szabolcs Szakacsits + * Copyright (c) 2025 LG Electronics Co., Ltd. + */ + +#include "mft.h" +#include "attrib.h" +#include "attrlist.h" + +/* + * ntfs_attrlist_need - check whether inode need attribute list + * @ni: opened ntfs inode for which perform check + * + * Check whether all are attributes belong to one MFT record, in that case + * attribute list is not needed. + * + * Return 1 if inode need attribute list, 0 if not, or -errno on error. + */ +int ntfs_attrlist_need(struct ntfs_inode *ni) +{ + struct attr_list_entry *ale; + + if (!ni) { + ntfs_debug("Invalid arguments.\n"); + return -EINVAL; + } + ntfs_debug("Entering for inode 0x%llx.\n", (long long) ni->mft_no); + + if (!NInoAttrList(ni)) { + ntfs_debug("Inode haven't got attribute list.\n"); + return -EINVAL; + } + + if (!ni->attr_list) { + ntfs_debug("Corrupt in-memory struct.\n"); + return -EINVAL; + } + + ale = (struct attr_list_entry *)ni->attr_list; + while ((u8 *)ale < ni->attr_list + ni->attr_list_size) { + if (MREF_LE(ale->mft_reference) != ni->mft_no) + return 1; + ale = (struct attr_list_entry *)((u8 *)ale + le16_to_cpu(ale->length)); + } + return 0; +} + +int ntfs_attrlist_update(struct ntfs_inode *base_ni) +{ + struct inode *attr_vi; + struct ntfs_inode *attr_ni; + int err; + + /* + * generic_shutdown_super() clears SB_ACTIVE before evicting cached + * inodes. Do not look up the attribute-list inode after SB_ACTIVE has + * been cleared; it may already be I_FREEING, and waiting on it can + * self-deadlock. + */ + if (!(VFS_I(base_ni)->i_sb->s_flags & SB_ACTIVE)) + return -EIO; + + attr_vi = ntfs_attr_iget(VFS_I(base_ni), AT_ATTRIBUTE_LIST, AT_UNNAMED, 0); + if (IS_ERR(attr_vi)) { + err = PTR_ERR(attr_vi); + return err; + } + attr_ni = NTFS_I(attr_vi); + + err = ntfs_attr_truncate_i(attr_ni, base_ni->attr_list_size, HOLES_NO); + if (err == -ENOSPC && attr_ni->mft_no == FILE_MFT) { + err = ntfs_attr_truncate(attr_ni, 0); + if (err || ntfs_attr_truncate_i(attr_ni, base_ni->attr_list_size, HOLES_NO) != 0) { + iput(attr_vi); + ntfs_error(base_ni->vol->sb, + "Failed to truncate attribute list of inode %#llx", + (long long)base_ni->mft_no); + return -EIO; + } + } else if (err) { + iput(attr_vi); + ntfs_error(base_ni->vol->sb, + "Failed to truncate attribute list of inode %#llx", + (long long)base_ni->mft_no); + return -EIO; + } + + i_size_write(attr_vi, base_ni->attr_list_size); + + if (NInoNonResident(attr_ni) && !NInoAttrListNonResident(base_ni)) + NInoSetAttrListNonResident(base_ni); + + if (ntfs_inode_attr_pwrite(attr_vi, 0, base_ni->attr_list_size, + base_ni->attr_list, false) != + base_ni->attr_list_size) { + iput(attr_vi); + ntfs_error(base_ni->vol->sb, + "Failed to write attribute list of inode %#llx", + (long long)base_ni->mft_no); + return -EIO; + } + + NInoSetAttrListDirty(base_ni); + iput(attr_vi); + return 0; +} + +/* + * ntfs_attrlist_entry_add - add an attribute list attribute entry + * @ni: opened ntfs inode, which contains that attribute + * @attr: attribute record to add to attribute list + * + * Return 0 on success and -errno on error. + */ +int ntfs_attrlist_entry_add(struct ntfs_inode *ni, struct attr_record *attr) +{ + struct attr_list_entry *ale; + __le64 mref; + struct ntfs_attr_search_ctx *ctx; + u8 *new_al; + int entry_len, entry_offset, err; + struct mft_record *ni_mrec; + u8 *old_al; + __le64 lowest_vcn; + + if (!ni || !attr) { + ntfs_debug("Invalid arguments.\n"); + return -EINVAL; + } + + ntfs_debug("Entering for inode 0x%llx, attr 0x%x.\n", + ni->mft_no, (unsigned int) le32_to_cpu(attr->type)); + + ni_mrec = map_mft_record(ni); + if (IS_ERR(ni_mrec)) { + ntfs_debug("Invalid arguments.\n"); + return -EIO; + } + + mref = MK_LE_MREF(ni->mft_no, le16_to_cpu(ni_mrec->sequence_number)); + unmap_mft_record(ni); + + if (ni->nr_extents == -1) + ni = ni->ext.base_ntfs_ino; + + if (!NInoAttrList(ni)) { + ntfs_debug("Attribute list isn't present.\n"); + return -ENOENT; + } + + /* Determine size and allocate memory for new attribute list. */ + entry_len = (sizeof(struct attr_list_entry) + sizeof(__le16) * + attr->name_length + 7) & ~7; + new_al = kvzalloc(ni->attr_list_size + entry_len, GFP_NOFS); + if (!new_al) + return -ENOMEM; + + /* Find place for the new entry. */ + ctx = ntfs_attr_get_search_ctx(ni, NULL); + if (!ctx) { + err = -ENOMEM; + ntfs_error(ni->vol->sb, "Failed to get search context"); + goto err_out; + } + if (attr->non_resident) + lowest_vcn = attr->data.non_resident.lowest_vcn; + else + lowest_vcn = 0; + + err = ntfs_attr_lookup(attr->type, (attr->name_length) ? (__le16 *) + ((u8 *)attr + le16_to_cpu(attr->name_offset)) : + AT_UNNAMED, attr->name_length, CASE_SENSITIVE, + le64_to_cpu(lowest_vcn), + (attr->non_resident) ? NULL : ((u8 *)attr + + le16_to_cpu(attr->data.resident.value_offset)), (attr->non_resident) ? + 0 : le32_to_cpu(attr->data.resident.value_length), ctx); + if (!err) { + /* Found some extent, check it to be before new extent. */ + if (ctx->al_entry->lowest_vcn == lowest_vcn) { + err = -EEXIST; + ntfs_debug("Such attribute already present in the attribute list.\n"); + ntfs_attr_put_search_ctx(ctx); + goto err_out; + } + /* Add new entry after this extent. */ + ale = (struct attr_list_entry *)((u8 *)ctx->al_entry + + le16_to_cpu(ctx->al_entry->length)); + } else { + /* Check for real errors. */ + if (err != -ENOENT) { + ntfs_debug("Attribute lookup failed.\n"); + ntfs_attr_put_search_ctx(ctx); + goto err_out; + } + /* No previous extents found. */ + ale = ctx->al_entry; + } + /* Don't need it anymore, @ctx->al_entry points to @ni->attr_list. */ + ntfs_attr_put_search_ctx(ctx); + + /* Determine new entry offset. */ + entry_offset = ((u8 *)ale - ni->attr_list); + /* Set pointer to new entry. */ + ale = (struct attr_list_entry *)(new_al + entry_offset); + memset(ale, 0, entry_len); + /* Form new entry. */ + ale->type = attr->type; + ale->length = cpu_to_le16(entry_len); + ale->name_length = attr->name_length; + ale->name_offset = offsetof(struct attr_list_entry, name); + if (attr->non_resident) + ale->lowest_vcn = attr->data.non_resident.lowest_vcn; + else + ale->lowest_vcn = 0; + ale->mft_reference = mref; + ale->instance = attr->instance; + memcpy(ale->name, (u8 *)attr + le16_to_cpu(attr->name_offset), + attr->name_length * sizeof(__le16)); + + /* Copy entries from old attribute list to new. */ + memcpy(new_al, ni->attr_list, entry_offset); + memcpy(new_al + entry_offset + entry_len, ni->attr_list + + entry_offset, ni->attr_list_size - entry_offset); + + /* Set new runlist. */ + old_al = ni->attr_list; + ni->attr_list = new_al; + ni->attr_list_size = ni->attr_list_size + entry_len; + + err = ntfs_attrlist_update(ni); + if (err) { + ni->attr_list = old_al; + ni->attr_list_size -= entry_len; + goto err_out; + } + kvfree(old_al); + return 0; +err_out: + kvfree(new_al); + return err; +} + +/* + * ntfs_attrlist_entry_rm - remove an attribute list attribute entry + * @ctx: attribute search context describing the attribute list entry + * + * Remove the attribute list entry @ctx->al_entry from the attribute list. + * + * Return 0 on success and -errno on error. + */ +int ntfs_attrlist_entry_rm(struct ntfs_attr_search_ctx *ctx) +{ + u8 *new_al; + int new_al_len; + struct ntfs_inode *base_ni; + struct attr_list_entry *ale; + + if (!ctx || !ctx->ntfs_ino || !ctx->al_entry) { + ntfs_debug("Invalid arguments.\n"); + return -EINVAL; + } + + if (ctx->base_ntfs_ino) + base_ni = ctx->base_ntfs_ino; + else + base_ni = ctx->ntfs_ino; + ale = ctx->al_entry; + + ntfs_debug("Entering for inode 0x%llx, attr 0x%x, lowest_vcn %lld.\n", + (long long)ctx->ntfs_ino->mft_no, + (unsigned int)le32_to_cpu(ctx->al_entry->type), + (long long)le64_to_cpu(ctx->al_entry->lowest_vcn)); + + if (!NInoAttrList(base_ni)) { + ntfs_debug("Attribute list isn't present.\n"); + return -ENOENT; + } + + /* Allocate memory for new attribute list. */ + new_al_len = base_ni->attr_list_size - le16_to_cpu(ale->length); + new_al = kvzalloc(new_al_len, GFP_NOFS); + if (!new_al) + return -ENOMEM; + + /* Copy entries from old attribute list to new. */ + memcpy(new_al, base_ni->attr_list, (u8 *)ale - base_ni->attr_list); + memcpy(new_al + ((u8 *)ale - base_ni->attr_list), (u8 *)ale + le16_to_cpu( + ale->length), new_al_len - ((u8 *)ale - base_ni->attr_list)); + + /* Set new runlist. */ + kvfree(base_ni->attr_list); + base_ni->attr_list = new_al; + base_ni->attr_list_size = new_al_len; + + return ntfs_attrlist_update(base_ni); +} diff --git a/fs/ntfs/attrlist.h b/fs/ntfs/attrlist.h new file mode 100644 index 000000000..1892a3934 --- /dev/null +++ b/fs/ntfs/attrlist.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Exports for attribute list attribute handling. + * + * Copyright (c) 2004 Anton Altaparmakov + * Copyright (c) 2004 Yura Pakhuchiy + * Copyright (c) 2025 LG Electronics Co., Ltd. + */ + +#ifndef _NTFS_ATTRLIST_H +#define _NTFS_ATTRLIST_H + +#include "attrib.h" + +int ntfs_attrlist_need(struct ntfs_inode *ni); +int ntfs_attrlist_entry_add(struct ntfs_inode *ni, struct attr_record *attr); +int ntfs_attrlist_entry_rm(struct ntfs_attr_search_ctx *ctx); +int ntfs_attrlist_update(struct ntfs_inode *base_ni); + +#endif /* defined _NTFS_ATTRLIST_H */ diff --git a/fs/ntfs/bdev-io.c b/fs/ntfs/bdev-io.c new file mode 100644 index 000000000..27d7c2767 --- /dev/null +++ b/fs/ntfs/bdev-io.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * NTFS block device I/O. + * + * Copyright (c) 2026 LG Electronics Co., Ltd. + */ + +#include + +#include "ntfs.h" + +/* + * ntfs_bdev_read - Read data directly from block device using bio + * @bdev: block device to read from + * @data: destination buffer + * @start: starting byte offset on the block device + * @size: number of bytes to read + * + * Reads @size bytes starting from byte offset @start directly from the block + * device using one or more BIOs. This function bypasses the page cache + * completely and performs synchronous I/O with REQ_META | REQ_SYNC flags set. + * + * The @start offset must be sector-aligned (512 bytes). If it is not aligned, + * the function will return -EINVAL. + * + * If the destination buffer @data is not a vmalloc address, it falls back + * to the more efficient bdev_rw_virt() helper. + * + * Return: 0 on success, negative error code on failure. + */ +int ntfs_bdev_read(struct block_device *bdev, char *data, loff_t start, size_t size) +{ + unsigned int done = 0, added; + int error; + struct bio *bio; + enum req_op op; + sector_t sector = start >> SECTOR_SHIFT; + + if (start & (SECTOR_SIZE - 1)) + return -EINVAL; + + op = REQ_OP_READ | REQ_META | REQ_SYNC; + if (!is_vmalloc_addr(data)) + return bdev_rw_virt(bdev, sector, data, size, op); + + bio = bio_alloc(bdev, + bio_max_segs(DIV_ROUND_UP(size, PAGE_SIZE)), + op, GFP_KERNEL); + bio->bi_iter.bi_sector = sector; + + do { + added = bio_add_vmalloc_chunk(bio, data + done, size - done); + if (!added) { + struct bio *prev = bio; + + bio = bio_alloc(prev->bi_bdev, + bio_max_segs(DIV_ROUND_UP(size - done, PAGE_SIZE)), + prev->bi_opf, GFP_KERNEL); + bio->bi_iter.bi_sector = bio_end_sector(prev); + bio_chain(prev, bio); + submit_bio(prev); + } + done += added; + } while (done < size); + + error = submit_bio_wait(bio); + bio_put(bio); + + if (op == REQ_OP_READ) + invalidate_kernel_vmap_range(data, size); + return error; +} + +/* + * ntfs_bdev_write - Update block device contents via page cache + * @sb: super block of the mounted NTFS filesystem + * @buf: source buffer containing data to write + * @start: starting byte offset on the block device + * @size: number of bytes to write + * + * Writes @size bytes from @buf to the block device (sb->s_bdev) starting + * at byte offset @start. The write is performed entirely through the page + * cache of the block device's address space. + */ +int ntfs_bdev_write(struct super_block *sb, void *buf, loff_t start, size_t size) +{ + pgoff_t idx, idx_end; + loff_t offset, end = start + size; + u32 from, to, buf_off = 0; + struct folio *folio; + + idx = start >> PAGE_SHIFT; + idx_end = end >> PAGE_SHIFT; + from = start & ~PAGE_MASK; + + if (idx == idx_end) + idx_end++; + + for (; idx < idx_end; idx++, from = 0) { + u32 len; + + folio = read_mapping_folio(sb->s_bdev->bd_mapping, idx, NULL); + if (IS_ERR(folio)) { + ntfs_error(sb, "Unable to read %ld page", idx); + return PTR_ERR(folio); + } + + offset = (loff_t)idx << PAGE_SHIFT; + to = min_t(u32, end - offset, PAGE_SIZE); + len = to - from; + + memcpy_to_folio(folio, from, buf + buf_off, len); + buf_off += len; + folio_mark_uptodate(folio); + folio_mark_dirty(folio); + folio_put(folio); + } + + return 0; +} diff --git a/fs/ntfs/bitmap.c b/fs/ntfs/bitmap.c new file mode 100644 index 000000000..b1436b315 --- /dev/null +++ b/fs/ntfs/bitmap.c @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * NTFS kernel bitmap handling. + * + * Copyright (c) 2004-2005 Anton Altaparmakov + * Copyright (c) 2025 LG Electronics Co., Ltd. + */ + +#include +#include + +#include "bitmap.h" +#include "ntfs.h" + +int ntfs_trim_fs(struct ntfs_volume *vol, struct fstrim_range *range) +{ + size_t buf_clusters; + pgoff_t index, start_index, end_index; + struct file_ra_state *ra; + struct folio *folio; + unsigned long *bitmap; + char *kaddr; + u64 end, trimmed = 0, start_buf, end_buf, end_cluster; + u64 start_cluster = ntfs_bytes_to_cluster(vol, range->start); + u32 dq = bdev_discard_granularity(vol->sb->s_bdev); + int ret = 0; + + if (!dq) + dq = vol->cluster_size; + + if (start_cluster >= vol->nr_clusters) + return -EINVAL; + + if (range->len == (u64)-1) + end_cluster = vol->nr_clusters; + else { + end_cluster = ntfs_bytes_to_cluster(vol, + (range->start + range->len + vol->cluster_size - 1)); + if (end_cluster > vol->nr_clusters) + end_cluster = vol->nr_clusters; + } + + ra = kzalloc(sizeof(*ra), GFP_NOFS); + if (!ra) + return -ENOMEM; + + buf_clusters = PAGE_SIZE * 8; + start_index = start_cluster >> 15; + end_index = (end_cluster + buf_clusters - 1) >> 15; + + for (index = start_index; index < end_index; index++) { + folio = ntfs_get_locked_folio(vol->lcnbmp_ino->i_mapping, + index, end_index, ra); + if (IS_ERR(folio)) { + ret = PTR_ERR(folio); + goto out_free; + } + + kaddr = kmap_local_folio(folio, 0); + bitmap = (unsigned long *)kaddr; + + start_buf = max_t(u64, index * buf_clusters, start_cluster); + end_buf = min_t(u64, (index + 1) * buf_clusters, end_cluster); + + end = start_buf; + while (end < end_buf) { + u64 aligned_start, aligned_count; + u64 start = find_next_zero_bit(bitmap, end_buf - start_buf, + end - start_buf) + start_buf; + if (start >= end_buf) + break; + + end = find_next_bit(bitmap, end_buf - start_buf, + start - start_buf) + start_buf; + + aligned_start = ALIGN(ntfs_cluster_to_bytes(vol, start), dq); + aligned_count = + ALIGN_DOWN(ntfs_cluster_to_bytes(vol, end - start), dq); + if (aligned_count >= range->minlen) { + ret = blkdev_issue_discard(vol->sb->s_bdev, aligned_start >> 9, + aligned_count >> 9, GFP_NOFS); + if (ret) + goto out_unmap; + trimmed += aligned_count; + } + } + +out_unmap: + kunmap_local(kaddr); + folio_unlock(folio); + folio_put(folio); + + if (ret) + goto out_free; + } + + range->len = trimmed; + +out_free: + kfree(ra); + return ret; +} + +/* + * __ntfs_bitmap_set_bits_in_run - set a run of bits in a bitmap to a value + * @vi: vfs inode describing the bitmap + * @start_bit: first bit to set + * @count: number of bits to set + * @value: value to set the bits to (i.e. 0 or 1) + * @is_rollback: if 'true' this is a rollback operation + * + * Set @count bits starting at bit @start_bit in the bitmap described by the + * vfs inode @vi to @value, where @value is either 0 or 1. + * + * @is_rollback should always be 'false', it is for internal use to rollback + * errors. You probably want to use ntfs_bitmap_set_bits_in_run() instead. + * + * Return 0 on success and -errno on error. + */ +int __ntfs_bitmap_set_bits_in_run(struct inode *vi, const s64 start_bit, + const s64 count, const u8 value, const bool is_rollback) +{ + s64 cnt = count; + pgoff_t index, end_index; + struct address_space *mapping; + struct folio *folio; + u8 *kaddr; + int pos, len, err; + u8 bit; + struct ntfs_inode *ni = NTFS_I(vi); + struct ntfs_volume *vol = ni->vol; + + ntfs_debug("Entering for i_ino 0x%llx, start_bit 0x%llx, count 0x%llx, value %u.%s", + ni->mft_no, (unsigned long long)start_bit, + (unsigned long long)cnt, (unsigned int)value, + is_rollback ? " (rollback)" : ""); + + if (start_bit < 0 || cnt < 0 || value > 1) + return -EINVAL; + + /* + * Calculate the indices for the pages containing the first and last + * bits, i.e. @start_bit and @start_bit + @cnt - 1, respectively. + */ + index = start_bit >> (3 + PAGE_SHIFT); + end_index = (start_bit + cnt - 1) >> (3 + PAGE_SHIFT); + + /* Get the page containing the first bit (@start_bit). */ + mapping = vi->i_mapping; + folio = read_mapping_folio(mapping, index, NULL); + if (IS_ERR(folio)) { + if (!is_rollback) + ntfs_error(vi->i_sb, + "Failed to map first page (error %li), aborting.", + PTR_ERR(folio)); + return PTR_ERR(folio); + } + + folio_lock(folio); + kaddr = kmap_local_folio(folio, 0); + + /* Set @pos to the position of the byte containing @start_bit. */ + pos = (start_bit >> 3) & ~PAGE_MASK; + + /* Calculate the position of @start_bit in the first byte. */ + bit = start_bit & 7; + + /* If the first byte is partial, modify the appropriate bits in it. */ + if (bit) { + u8 *byte = kaddr + pos; + + if (ni->mft_no == FILE_Bitmap) + ntfs_set_lcn_empty_bits(vol, index, value, min_t(s64, 8 - bit, cnt)); + while ((bit & 7) && cnt) { + cnt--; + if (value) + *byte |= 1 << bit++; + else + *byte &= ~(1 << bit++); + } + /* If we are done, unmap the page and return success. */ + if (!cnt) + goto done; + + /* Update @pos to the new position. */ + pos++; + } + /* + * Depending on @value, modify all remaining whole bytes in the page up + * to @cnt. + */ + len = min_t(s64, cnt >> 3, PAGE_SIZE - pos); + memset(kaddr + pos, value ? 0xff : 0, len); + cnt -= len << 3; + if (ni->mft_no == FILE_Bitmap) + ntfs_set_lcn_empty_bits(vol, index, value, len << 3); + + /* Update @len to point to the first not-done byte in the page. */ + if (cnt < 8) + len += pos; + + /* If we are not in the last page, deal with all subsequent pages. */ + while (index < end_index) { + if (cnt <= 0) { + err = -EIO; + goto rollback; + } + + /* Update @index and get the next folio. */ + folio_mark_dirty(folio); + folio_unlock(folio); + kunmap_local(kaddr); + folio_put(folio); + folio = read_mapping_folio(mapping, ++index, NULL); + if (IS_ERR(folio)) { + ntfs_error(vi->i_sb, + "Failed to map subsequent page (error %li), aborting.", + PTR_ERR(folio)); + err = PTR_ERR(folio); + goto rollback; + } + + folio_lock(folio); + kaddr = kmap_local_folio(folio, 0); + /* + * Depending on @value, modify all remaining whole bytes in the + * page up to @cnt. + */ + len = min_t(s64, cnt >> 3, PAGE_SIZE); + memset(kaddr, value ? 0xff : 0, len); + cnt -= len << 3; + if (ni->mft_no == FILE_Bitmap) + ntfs_set_lcn_empty_bits(vol, index, value, len << 3); + } + /* + * The currently mapped page is the last one. If the last byte is + * partial, modify the appropriate bits in it. Note, @len is the + * position of the last byte inside the page. + */ + if (cnt) { + u8 *byte; + + WARN_ON(cnt > 7); + + bit = cnt; + byte = kaddr + len; + if (ni->mft_no == FILE_Bitmap) + ntfs_set_lcn_empty_bits(vol, index, value, bit); + while (bit--) { + if (value) + *byte |= 1 << bit; + else + *byte &= ~(1 << bit); + } + } +done: + /* We are done. Unmap the folio and return success. */ + folio_mark_dirty(folio); + folio_unlock(folio); + kunmap_local(kaddr); + folio_put(folio); + ntfs_debug("Done."); + return 0; +rollback: + /* + * Current state: + * - no pages are mapped + * - @count - @cnt is the number of bits that have been modified + */ + if (is_rollback) + return err; + if (count != cnt) + pos = __ntfs_bitmap_set_bits_in_run(vi, start_bit, count - cnt, + value ? 0 : 1, true); + else + pos = 0; + if (!pos) { + /* Rollback was successful. */ + ntfs_error(vi->i_sb, + "Failed to map subsequent page (error %i), aborting.", + err); + } else { + /* Rollback failed. */ + ntfs_error(vi->i_sb, + "Failed to map subsequent page (error %i) and rollback failed (error %i). Aborting and leaving inconsistent metadata. Unmount and run chkdsk.", + err, pos); + NVolSetErrors(NTFS_SB(vi->i_sb)); + } + return err; +} diff --git a/fs/ntfs/bitmap.h b/fs/ntfs/bitmap.h new file mode 100644 index 000000000..31177fe61 --- /dev/null +++ b/fs/ntfs/bitmap.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Defines for NTFS kernel bitmap handling. + * + * Copyright (c) 2004 Anton Altaparmakov + */ + +#ifndef _LINUX_NTFS_BITMAP_H +#define _LINUX_NTFS_BITMAP_H + +#include + +#include "volume.h" + +int ntfs_trim_fs(struct ntfs_volume *vol, struct fstrim_range *range); +int __ntfs_bitmap_set_bits_in_run(struct inode *vi, const s64 start_bit, + const s64 count, const u8 value, const bool is_rollback); + +/* + * ntfs_bitmap_set_bits_in_run - set a run of bits in a bitmap to a value + * @vi: vfs inode describing the bitmap + * @start_bit: first bit to set + * @count: number of bits to set + * @value: value to set the bits to (i.e. 0 or 1) + * + * Set @count bits starting at bit @start_bit in the bitmap described by the + * vfs inode @vi to @value, where @value is either 0 or 1. + * + * Return 0 on success and -errno on error. + */ +static inline int ntfs_bitmap_set_bits_in_run(struct inode *vi, + const s64 start_bit, const s64 count, const u8 value) +{ + return __ntfs_bitmap_set_bits_in_run(vi, start_bit, count, value, + false); +} + +/* + * ntfs_bitmap_set_run - set a run of bits in a bitmap + * @vi: vfs inode describing the bitmap + * @start_bit: first bit to set + * @count: number of bits to set + * + * Set @count bits starting at bit @start_bit in the bitmap described by the + * vfs inode @vi. + * + * Return 0 on success and -errno on error. + */ +static inline int ntfs_bitmap_set_run(struct inode *vi, const s64 start_bit, + const s64 count) +{ + return ntfs_bitmap_set_bits_in_run(vi, start_bit, count, 1); +} + +/* + * ntfs_bitmap_clear_run - clear a run of bits in a bitmap + * @vi: vfs inode describing the bitmap + * @start_bit: first bit to clear + * @count: number of bits to clear + * + * Clear @count bits starting at bit @start_bit in the bitmap described by the + * vfs inode @vi. + * + * Return 0 on success and -errno on error. + */ +static inline int ntfs_bitmap_clear_run(struct inode *vi, const s64 start_bit, + const s64 count) +{ + return ntfs_bitmap_set_bits_in_run(vi, start_bit, count, 0); +} + +/* + * ntfs_bitmap_set_bit - set a bit in a bitmap + * @vi: vfs inode describing the bitmap + * @bit: bit to set + * + * Set bit @bit in the bitmap described by the vfs inode @vi. + * + * Return 0 on success and -errno on error. + */ +static inline int ntfs_bitmap_set_bit(struct inode *vi, const s64 bit) +{ + return ntfs_bitmap_set_run(vi, bit, 1); +} + +/* + * ntfs_bitmap_clear_bit - clear a bit in a bitmap + * @vi: vfs inode describing the bitmap + * @bit: bit to clear + * + * Clear bit @bit in the bitmap described by the vfs inode @vi. + * + * Return 0 on success and -errno on error. + */ +static inline int ntfs_bitmap_clear_bit(struct inode *vi, const s64 bit) +{ + return ntfs_bitmap_clear_run(vi, bit, 1); +} + +#endif /* defined _LINUX_NTFS_BITMAP_H */ diff --git a/fs/ntfs/collate.c b/fs/ntfs/collate.c new file mode 100644 index 000000000..77e340389 --- /dev/null +++ b/fs/ntfs/collate.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * NTFS kernel collation handling. + * + * Copyright (c) 2004 Anton Altaparmakov + * + * Part of this file is based on code from the NTFS-3G. + * and is copyrighted by the respective authors below: + * Copyright (c) 2004 Anton Altaparmakov + * Copyright (c) 2005 Yura Pakhuchiy + */ + +#include "collate.h" +#include "debug.h" +#include "ntfs.h" + +#include + +static int ntfs_collate_binary(struct ntfs_volume *vol, + const void *data1, const u32 data1_len, + const void *data2, const u32 data2_len) +{ + int rc; + + rc = memcmp(data1, data2, min(data1_len, data2_len)); + if (!rc && (data1_len != data2_len)) { + if (data1_len < data2_len) + rc = -1; + else + rc = 1; + } + return rc; +} + +static int ntfs_collate_ntofs_ulong(struct ntfs_volume *vol, + const void *data1, const u32 data1_len, + const void *data2, const u32 data2_len) +{ + int rc; + u32 d1 = le32_to_cpup(data1), d2 = le32_to_cpup(data2); + + if (data1_len != data2_len || data1_len != 4) + return -EINVAL; + + if (d1 < d2) + rc = -1; + else { + if (d1 == d2) + rc = 0; + else + rc = 1; + } + return rc; +} + +/* + * ntfs_collate_ntofs_ulongs - Which of two le32 arrays should be listed first + * @vol: ntfs volume + * @data1: first ulong array to collate + * @data1_len: length in bytes of @data1 + * @data2: second ulong array to collate + * @data2_len: length in bytes of @data2 + * + * Returns: -1, 0 or 1 depending of how the arrays compare + */ +static int ntfs_collate_ntofs_ulongs(struct ntfs_volume *vol, + const void *data1, const u32 data1_len, + const void *data2, const u32 data2_len) +{ + int len; + const __le32 *p1 = data1, *p2 = data2; + u32 d1, d2; + + if (data1_len != data2_len || data1_len & 3) { + ntfs_error(vol->sb, "data1_len or data2_len not valid\n"); + return -1; + } + + len = data1_len; + do { + d1 = le32_to_cpup(p1); + p1++; + d2 = le32_to_cpup(p2); + p2++; + } while (d1 == d2 && (len -= 4) > 0); + return cmp_int(d1, d2); +} + +/* + * ntfs_collate_file_name - Which of two filenames should be listed first + * @vol: ntfs volume + * @data1: first filename to collate + * @data1_len: length in bytes of @data1(unused) + * @data2: second filename to collate + * @data2_len: length in bytes of @data2(unused) + */ +static int ntfs_collate_file_name(struct ntfs_volume *vol, + const void *data1, const u32 data1_len, + const void *data2, const u32 data2_len) +{ + int rc; + + rc = ntfs_file_compare_values(data1, data2, -EINVAL, + IGNORE_CASE, vol->upcase, vol->upcase_len); + if (!rc) + rc = ntfs_file_compare_values(data1, data2, + -EINVAL, CASE_SENSITIVE, vol->upcase, vol->upcase_len); + return rc; +} + +/* + * ntfs_collate - collate two data items using a specified collation rule + * @vol: ntfs volume to which the data items belong + * @cr: collation rule to use when comparing the items + * @data1: first data item to collate + * @data1_len: length in bytes of @data1 + * @data2: second data item to collate + * @data2_len: length in bytes of @data2 + * + * Collate the two data items @data1 and @data2 using the collation rule @cr + * and return -1, 0, ir 1 if @data1 is found, respectively, to collate before, + * to match, or to collate after @data2. return -EINVAL if an error occurred. + */ +int ntfs_collate(struct ntfs_volume *vol, __le32 cr, + const void *data1, const u32 data1_len, + const void *data2, const u32 data2_len) +{ + switch (le32_to_cpu(cr)) { + case le32_to_cpu(COLLATION_BINARY): + return ntfs_collate_binary(vol, data1, data1_len, + data2, data2_len); + case le32_to_cpu(COLLATION_FILE_NAME): + return ntfs_collate_file_name(vol, data1, data1_len, + data2, data2_len); + case le32_to_cpu(COLLATION_NTOFS_ULONG): + return ntfs_collate_ntofs_ulong(vol, data1, data1_len, + data2, data2_len); + case le32_to_cpu(COLLATION_NTOFS_ULONGS): + return ntfs_collate_ntofs_ulongs(vol, data1, data1_len, + data2, data2_len); + default: + ntfs_error(vol->sb, "Unknown collation rule 0x%x", + le32_to_cpu(cr)); + return -EINVAL; + } +} diff --git a/fs/ntfs/collate.h b/fs/ntfs/collate.h new file mode 100644 index 000000000..8ad1f04be --- /dev/null +++ b/fs/ntfs/collate.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Defines for NTFS kernel collation handling. + * + * Copyright (c) 2004 Anton Altaparmakov + * + * Part of this file is based on code from the NTFS-3G. + * and is copyrighted by the respective authors below: + * Copyright (c) 2004 Anton Altaparmakov + * Copyright (c) 2005 Yura Pakhuchiy + */ + +#ifndef _LINUX_NTFS_COLLATE_H +#define _LINUX_NTFS_COLLATE_H + +#include "volume.h" + +static inline bool ntfs_is_collation_rule_supported(__le32 cr) +{ + int i; + + if (unlikely(cr != COLLATION_BINARY && cr != COLLATION_NTOFS_ULONG && + cr != COLLATION_FILE_NAME) && cr != COLLATION_NTOFS_ULONGS) + return false; + i = le32_to_cpu(cr); + if (likely(((i >= 0) && (i <= 0x02)) || + ((i >= 0x10) && (i <= 0x13)))) + return true; + return false; +} + +int ntfs_collate(struct ntfs_volume *vol, __le32 cr, + const void *data1, const u32 data1_len, + const void *data2, const u32 data2_len); + +#endif /* _LINUX_NTFS_COLLATE_H */ diff --git a/fs/ntfs/compress.c b/fs/ntfs/compress.c new file mode 100644 index 000000000..76bd806b4 --- /dev/null +++ b/fs/ntfs/compress.c @@ -0,0 +1,1578 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * NTFS kernel compressed attributes handling. + * + * Copyright (c) 2001-2004 Anton Altaparmakov + * Copyright (c) 2002 Richard Russon + * Copyright (c) 2025 LG Electronics Co., Ltd. + * + * Part of this file is based on code from the NTFS-3G. + * and is copyrighted by the respective authors below: + * Copyright (c) 2004-2005 Anton Altaparmakov + * Copyright (c) 2004-2006 Szabolcs Szakacsits + * Copyright (c) 2005 Yura Pakhuchiy + * Copyright (c) 2009-2014 Jean-Pierre Andre + * Copyright (c) 2014 Eric Biggers + */ + +#include +#include +#include +#include + +#include "attrib.h" +#include "inode.h" +#include "debug.h" +#include "ntfs.h" +#include "lcnalloc.h" +#include "mft.h" + +/* + * Constants used in the compression code + */ +enum { + /* Token types and access mask. */ + NTFS_SYMBOL_TOKEN = 0, + NTFS_PHRASE_TOKEN = 1, + NTFS_TOKEN_MASK = 1, + + /* Compression sub-block constants. */ + NTFS_SB_SIZE_MASK = 0x0fff, + NTFS_SB_SIZE = 0x1000, + NTFS_SB_IS_COMPRESSED = 0x8000, + + /* + * The maximum compression block size is by definition 16 * the cluster + * size, with the maximum supported cluster size being 4kiB. Thus the + * maximum compression buffer size is 64kiB, so we use this when + * initializing the compression buffer. + */ + NTFS_MAX_CB_SIZE = 64 * 1024, +}; + +/* + * ntfs_compression_buffer - one buffer for the decompression engine + */ +static u8 *ntfs_compression_buffer; + +/* + * ntfs_cb_lock - mutex lock which protects ntfs_compression_buffer + */ +static DEFINE_MUTEX(ntfs_cb_lock); + +/* + * allocate_compression_buffers - allocate the decompression buffers + * + * Caller has to hold the ntfs_lock mutex. + * + * Return 0 on success or -ENOMEM if the allocations failed. + */ +int allocate_compression_buffers(void) +{ + if (ntfs_compression_buffer) + return 0; + + ntfs_compression_buffer = vmalloc(NTFS_MAX_CB_SIZE); + if (!ntfs_compression_buffer) + return -ENOMEM; + return 0; +} + +/* + * free_compression_buffers - free the decompression buffers + * + * Caller has to hold the ntfs_lock mutex. + */ +void free_compression_buffers(void) +{ + mutex_lock(&ntfs_cb_lock); + if (!ntfs_compression_buffer) { + mutex_unlock(&ntfs_cb_lock); + return; + } + + vfree(ntfs_compression_buffer); + ntfs_compression_buffer = NULL; + mutex_unlock(&ntfs_cb_lock); +} + +/* + * zero_partial_compressed_page - zero out of bounds compressed page region + * @page: page to zero + * @initialized_size: initialized size of the attribute + */ +static void zero_partial_compressed_page(struct page *page, + const s64 initialized_size) +{ + u8 *kp = page_address(page); + unsigned int kp_ofs; + + ntfs_debug("Zeroing page region outside initialized size."); + if (((s64)page->__folio_index << PAGE_SHIFT) >= initialized_size) { + clear_page(kp); + return; + } + kp_ofs = initialized_size & ~PAGE_MASK; + memset(kp + kp_ofs, 0, PAGE_SIZE - kp_ofs); +} + +/* + * handle_bounds_compressed_page - test for&handle out of bounds compressed page + * @page: page to check and handle + * @i_size: file size + * @initialized_size: initialized size of the attribute + */ +static inline void handle_bounds_compressed_page(struct page *page, + const loff_t i_size, const s64 initialized_size) +{ + if ((page->__folio_index >= (initialized_size >> PAGE_SHIFT)) && + (initialized_size < i_size)) + zero_partial_compressed_page(page, initialized_size); +} + +/* + * ntfs_decompress - decompress a compression block into an array of pages + * @dest_pages: destination array of pages + * @completed_pages: scratch space to track completed pages + * @dest_index: current index into @dest_pages (IN/OUT) + * @dest_ofs: current offset within @dest_pages[@dest_index] (IN/OUT) + * @dest_max_index: maximum index into @dest_pages (IN) + * @dest_max_ofs: maximum offset within @dest_pages[@dest_max_index] (IN) + * @xpage: the target page (-1 if none) (IN) + * @xpage_done: set to 1 if xpage was completed successfully (IN/OUT) + * @cb_start: compression block to decompress (IN) + * @cb_size: size of compression block @cb_start in bytes (IN) + * @i_size: file size when we started the read (IN) + * @initialized_size: initialized file size when we started the read (IN) + * + * The caller must have disabled preemption. ntfs_decompress() reenables it when + * the critical section is finished. + * + * This decompresses the compression block @cb_start into the array of + * destination pages @dest_pages starting at index @dest_index into @dest_pages + * and at offset @dest_pos into the page @dest_pages[@dest_index]. + * + * When the page @dest_pages[@xpage] is completed, @xpage_done is set to 1. + * If xpage is -1 or @xpage has not been completed, @xpage_done is not modified. + * + * @cb_start is a pointer to the compression block which needs decompressing + * and @cb_size is the size of @cb_start in bytes (8-64kiB). + * + * Return 0 if success or -EOVERFLOW on error in the compressed stream. + * @xpage_done indicates whether the target page (@dest_pages[@xpage]) was + * completed during the decompression of the compression block (@cb_start). + * + * Warning: This function *REQUIRES* PAGE_SIZE >= 4096 or it will blow up + * unpredicatbly! You have been warned! + * + * Note to hackers: This function may not sleep until it has finished accessing + * the compression block @cb_start as it is a per-CPU buffer. + */ +static int ntfs_decompress(struct page *dest_pages[], int completed_pages[], + int *dest_index, int *dest_ofs, const int dest_max_index, + const int dest_max_ofs, const int xpage, char *xpage_done, + u8 *const cb_start, const u32 cb_size, const loff_t i_size, + const s64 initialized_size) +{ + /* + * Pointers into the compressed data, i.e. the compression block (cb), + * and the therein contained sub-blocks (sb). + */ + u8 *cb_end = cb_start + cb_size; /* End of cb. */ + u8 *cb = cb_start; /* Current position in cb. */ + u8 *cb_sb_start = cb; /* Beginning of the current sb in the cb. */ + u8 *cb_sb_end; /* End of current sb / beginning of next sb. */ + + /* Variables for uncompressed data / destination. */ + struct page *dp; /* Current destination page being worked on. */ + u8 *dp_addr; /* Current pointer into dp. */ + u8 *dp_sb_start; /* Start of current sub-block in dp. */ + u8 *dp_sb_end; /* End of current sb in dp (dp_sb_start + NTFS_SB_SIZE). */ + u16 do_sb_start; /* @dest_ofs when starting this sub-block. */ + u16 do_sb_end; /* @dest_ofs of end of this sb (do_sb_start + NTFS_SB_SIZE). */ + + /* Variables for tag and token parsing. */ + u8 tag; /* Current tag. */ + int token; /* Loop counter for the eight tokens in tag. */ + int nr_completed_pages = 0; + + /* Default error code. */ + int err = -EOVERFLOW; + + ntfs_debug("Entering, cb_size = 0x%x.", cb_size); +do_next_sb: + ntfs_debug("Beginning sub-block at offset = 0x%zx in the cb.", + cb - cb_start); + /* + * Have we reached the end of the compression block or the end of the + * decompressed data? The latter can happen for example if the current + * position in the compression block is one byte before its end so the + * first two checks do not detect it. + */ + if (cb == cb_end || !le16_to_cpup((__le16 *)cb) || + (*dest_index == dest_max_index && + *dest_ofs == dest_max_ofs)) { + int i; + + ntfs_debug("Completed. Returning success (0)."); + err = 0; +return_error: + /* We can sleep from now on, so we drop lock. */ + mutex_unlock(&ntfs_cb_lock); + /* Second stage: finalize completed pages. */ + if (nr_completed_pages > 0) { + for (i = 0; i < nr_completed_pages; i++) { + int di = completed_pages[i]; + + dp = dest_pages[di]; + /* + * If we are outside the initialized size, zero + * the out of bounds page range. + */ + handle_bounds_compressed_page(dp, i_size, + initialized_size); + flush_dcache_page(dp); + kunmap_local(page_address(dp)); + SetPageUptodate(dp); + unlock_page(dp); + if (di == xpage) + *xpage_done = 1; + else + put_page(dp); + dest_pages[di] = NULL; + } + } + return err; + } + + /* Setup offsets for the current sub-block destination. */ + do_sb_start = *dest_ofs; + do_sb_end = do_sb_start + NTFS_SB_SIZE; + + /* Check that we are still within allowed boundaries. */ + if (*dest_index == dest_max_index && do_sb_end > dest_max_ofs) + goto return_overflow; + + /* Does the minimum size of a compressed sb overflow valid range? */ + if (cb + 6 > cb_end) + goto return_overflow; + + /* Setup the current sub-block source pointers and validate range. */ + cb_sb_start = cb; + cb_sb_end = cb_sb_start + (le16_to_cpup((__le16 *)cb) & NTFS_SB_SIZE_MASK) + + 3; + if (cb_sb_end > cb_end) + goto return_overflow; + + /* Get the current destination page. */ + dp = dest_pages[*dest_index]; + if (!dp) { + /* No page present. Skip decompression of this sub-block. */ + cb = cb_sb_end; + + /* Advance destination position to next sub-block. */ + *dest_ofs = (*dest_ofs + NTFS_SB_SIZE) & ~PAGE_MASK; + if (!*dest_ofs && (++*dest_index > dest_max_index)) + goto return_overflow; + goto do_next_sb; + } + + /* We have a valid destination page. Setup the destination pointers. */ + dp_addr = (u8 *)page_address(dp) + do_sb_start; + + /* Now, we are ready to process the current sub-block (sb). */ + if (!(le16_to_cpup((__le16 *)cb) & NTFS_SB_IS_COMPRESSED)) { + ntfs_debug("Found uncompressed sub-block."); + /* This sb is not compressed, just copy it into destination. */ + + /* Advance source position to first data byte. */ + cb += 2; + + /* An uncompressed sb must be full size. */ + if (cb_sb_end - cb != NTFS_SB_SIZE) + goto return_overflow; + + /* Copy the block and advance the source position. */ + memcpy(dp_addr, cb, NTFS_SB_SIZE); + cb += NTFS_SB_SIZE; + + /* Advance destination position to next sub-block. */ + *dest_ofs += NTFS_SB_SIZE; + *dest_ofs &= ~PAGE_MASK; + if (!(*dest_ofs)) { +finalize_page: + /* + * First stage: add current page index to array of + * completed pages. + */ + completed_pages[nr_completed_pages++] = *dest_index; + if (++*dest_index > dest_max_index) + goto return_overflow; + } + goto do_next_sb; + } + ntfs_debug("Found compressed sub-block."); + /* This sb is compressed, decompress it into destination. */ + + /* Setup destination pointers. */ + dp_sb_start = dp_addr; + dp_sb_end = dp_sb_start + NTFS_SB_SIZE; + + /* Forward to the first tag in the sub-block. */ + cb += 2; +do_next_tag: + if (cb == cb_sb_end) { + /* Check if the decompressed sub-block was not full-length. */ + if (dp_addr < dp_sb_end) { + int nr_bytes = do_sb_end - *dest_ofs; + + ntfs_debug("Filling incomplete sub-block with zeroes."); + /* Zero remainder and update destination position. */ + memset(dp_addr, 0, nr_bytes); + *dest_ofs += nr_bytes; + } + /* We have finished the current sub-block. */ + *dest_ofs &= ~PAGE_MASK; + if (!(*dest_ofs)) + goto finalize_page; + goto do_next_sb; + } + + /* Check we are still in range. */ + if (cb > cb_sb_end || dp_addr > dp_sb_end) + goto return_overflow; + + /* Get the next tag and advance to first token. */ + tag = *cb++; + + /* Parse the eight tokens described by the tag. */ + for (token = 0; token < 8; token++, tag >>= 1) { + register u16 i; + u16 lg, pt, length, max_non_overlap; + u8 *dp_back_addr; + + /* Check if we are done / still in range. */ + if (cb >= cb_sb_end || dp_addr > dp_sb_end) + break; + + /* Determine token type and parse appropriately.*/ + if ((tag & NTFS_TOKEN_MASK) == NTFS_SYMBOL_TOKEN) { + /* + * We have a symbol token, copy the symbol across, and + * advance the source and destination positions. + */ + *dp_addr++ = *cb++; + ++*dest_ofs; + + /* Continue with the next token. */ + continue; + } + + /* + * We have a phrase token. Make sure it is not the first tag in + * the sb as this is illegal and would confuse the code below. + */ + if (dp_addr == dp_sb_start) + goto return_overflow; + + /* + * Determine the number of bytes to go back (p) and the number + * of bytes to copy (l). We use an optimized algorithm in which + * we first calculate log2(current destination position in sb), + * which allows determination of l and p in O(1) rather than + * O(n). We just need an arch-optimized log2() function now. + */ + lg = 0; + for (i = *dest_ofs - do_sb_start - 1; i >= 0x10; i >>= 1) + lg++; + + /* Get the phrase token into i. */ + pt = le16_to_cpup((__le16 *)cb); + + /* + * Calculate starting position of the byte sequence in + * the destination using the fact that p = (pt >> (12 - lg)) + 1 + * and make sure we don't go too far back. + */ + dp_back_addr = dp_addr - (pt >> (12 - lg)) - 1; + if (dp_back_addr < dp_sb_start) + goto return_overflow; + + /* Now calculate the length of the byte sequence. */ + length = (pt & (0xfff >> lg)) + 3; + + /* Advance destination position and verify it is in range. */ + *dest_ofs += length; + if (*dest_ofs > do_sb_end) + goto return_overflow; + + /* The number of non-overlapping bytes. */ + max_non_overlap = dp_addr - dp_back_addr; + + if (length <= max_non_overlap) { + /* The byte sequence doesn't overlap, just copy it. */ + memcpy(dp_addr, dp_back_addr, length); + + /* Advance destination pointer. */ + dp_addr += length; + } else { + /* + * The byte sequence does overlap, copy non-overlapping + * part and then do a slow byte by byte copy for the + * overlapping part. Also, advance the destination + * pointer. + */ + memcpy(dp_addr, dp_back_addr, max_non_overlap); + dp_addr += max_non_overlap; + dp_back_addr += max_non_overlap; + length -= max_non_overlap; + while (length--) + *dp_addr++ = *dp_back_addr++; + } + + /* Advance source position and continue with the next token. */ + cb += 2; + } + + /* No tokens left in the current tag. Continue with the next tag. */ + goto do_next_tag; + +return_overflow: + ntfs_error(NULL, "Failed. Returning -EOVERFLOW."); + goto return_error; +} + +/* + * ntfs_read_compressed_block - read a compressed block into the page cache + * @folio: locked folio in the compression block(s) we need to read + * + * When we are called the page has already been verified to be locked and the + * attribute is known to be non-resident, not encrypted, but compressed. + * + * 1. Determine which compression block(s) @page is in. + * 2. Get hold of all pages corresponding to this/these compression block(s). + * 3. Read the (first) compression block. + * 4. Decompress it into the corresponding pages. + * 5. Throw the compressed data away and proceed to 3. for the next compression + * block or return success if no more compression blocks left. + * + * Warning: We have to be careful what we do about existing pages. They might + * have been written to so that we would lose data if we were to just overwrite + * them with the out-of-date uncompressed data. + */ +int ntfs_read_compressed_block(struct folio *folio) +{ + struct page *page = &folio->page; + loff_t i_size; + s64 initialized_size; + struct address_space *mapping = page->mapping; + struct ntfs_inode *ni = NTFS_I(mapping->host); + struct ntfs_volume *vol = ni->vol; + struct super_block *sb = vol->sb; + struct runlist_element *rl; + unsigned long flags; + u8 *cb, *cb_pos, *cb_end; + unsigned long offset, index = page->__folio_index; + u32 cb_size = ni->itype.compressed.block_size; + u64 cb_size_mask = cb_size - 1UL; + s64 vcn; + s64 lcn; + /* The first wanted vcn (minimum alignment is PAGE_SIZE). */ + s64 start_vcn = (((s64)index << PAGE_SHIFT) & ~cb_size_mask) >> + vol->cluster_size_bits; + /* + * The first vcn after the last wanted vcn (minimum alignment is again + * PAGE_SIZE. + */ + s64 end_vcn = ((((s64)(index + 1UL) << PAGE_SHIFT) + cb_size - 1) + & ~cb_size_mask) >> vol->cluster_size_bits; + /* Number of compression blocks (cbs) in the wanted vcn range. */ + unsigned int nr_cbs = ntfs_cluster_to_bytes(vol, end_vcn - start_vcn) >> + ni->itype.compressed.block_size_bits; + /* + * Number of pages required to store the uncompressed data from all + * compression blocks (cbs) overlapping @page. Due to alignment + * guarantees of start_vcn and end_vcn, no need to round up here. + */ + unsigned int nr_pages = ntfs_cluster_to_pidx(vol, end_vcn - start_vcn); + unsigned int xpage, max_page, cur_page, cur_ofs, i, page_ofs, page_index; + unsigned int cb_clusters, cb_max_ofs; + int cb_max_page, err = 0; + struct page **pages; + int *completed_pages; + unsigned char xpage_done = 0; + struct page *lpage; + + ntfs_debug("Entering, page->index = 0x%lx, cb_size = 0x%x, nr_pages = %i.", + index, cb_size, nr_pages); + /* + * Bad things happen if we get here for anything that is not an + * unnamed $DATA attribute. + */ + if (ni->type != AT_DATA || ni->name_len) { + unlock_page(page); + return -EIO; + } + + pages = kmalloc_array(nr_pages, sizeof(struct page *), GFP_NOFS); + completed_pages = kmalloc_array(nr_pages + 1, sizeof(int), GFP_NOFS); + + if (unlikely(!pages || !completed_pages)) { + kfree(pages); + kfree(completed_pages); + unlock_page(page); + ntfs_error(vol->sb, "Failed to allocate internal buffers."); + return -ENOMEM; + } + + /* + * We have already been given one page, this is the one we must do. + * Once again, the alignment guarantees keep it simple. + */ + offset = ntfs_cluster_to_pidx(vol, start_vcn); + xpage = index - offset; + pages[xpage] = page; + /* + * The remaining pages need to be allocated and inserted into the page + * cache, alignment guarantees keep all the below much simpler. (-8 + */ + read_lock_irqsave(&ni->size_lock, flags); + i_size = i_size_read(VFS_I(ni)); + initialized_size = ni->initialized_size; + read_unlock_irqrestore(&ni->size_lock, flags); + max_page = ((i_size + PAGE_SIZE - 1) >> PAGE_SHIFT) - + offset; + /* Is the page fully outside i_size? (truncate in progress) */ + if (xpage >= max_page) { + kfree(pages); + kfree(completed_pages); + zero_user_segments(page, 0, PAGE_SIZE, 0, 0); + ntfs_debug("Compressed read outside i_size - truncated?"); + SetPageUptodate(page); + unlock_page(page); + return 0; + } + if (nr_pages < max_page) + max_page = nr_pages; + + for (i = 0; i < max_page; i++, offset++) { + if (i != xpage) + pages[i] = grab_cache_page_nowait(mapping, offset); + page = pages[i]; + if (page) { + /* + * We only (re)read the page if it isn't already read + * in and/or dirty or we would be losing data or at + * least wasting our time. + */ + if (!PageDirty(page) && (!PageUptodate(page))) { + kmap_local_page(page); + continue; + } + unlock_page(page); + put_page(page); + pages[i] = NULL; + } + } + + /* + * We have the runlist, and all the destination pages we need to fill. + * Now read the first compression block. + */ + cur_page = 0; + cur_ofs = 0; + cb_clusters = ni->itype.compressed.block_clusters; +do_next_cb: + nr_cbs--; + + mutex_lock(&ntfs_cb_lock); + if (!ntfs_compression_buffer) + if (allocate_compression_buffers()) { + mutex_unlock(&ntfs_cb_lock); + goto err_out; + } + + + cb = ntfs_compression_buffer; + cb_pos = cb; + cb_end = cb + cb_size; + + rl = NULL; + for (vcn = start_vcn, start_vcn += cb_clusters; vcn < start_vcn; + vcn++) { + bool is_retry = false; + + if (!rl) { +lock_retry_remap: + down_read(&ni->runlist.lock); + rl = ni->runlist.rl; + } + if (likely(rl != NULL)) { + /* Seek to element containing target vcn. */ + while (rl->length && rl[1].vcn <= vcn) + rl++; + lcn = ntfs_rl_vcn_to_lcn(rl, vcn); + } else + lcn = LCN_RL_NOT_MAPPED; + ntfs_debug("Reading vcn = 0x%llx, lcn = 0x%llx.", + (unsigned long long)vcn, + (unsigned long long)lcn); + if (lcn < 0) { + /* + * When we reach the first sparse cluster we have + * finished with the cb. + */ + if (lcn == LCN_HOLE) + break; + if (is_retry || lcn != LCN_RL_NOT_MAPPED) { + mutex_unlock(&ntfs_cb_lock); + goto rl_err; + } + is_retry = true; + /* + * Attempt to map runlist, dropping lock for the + * duration. + */ + up_read(&ni->runlist.lock); + if (!ntfs_map_runlist(ni, vcn)) + goto lock_retry_remap; + mutex_unlock(&ntfs_cb_lock); + goto map_rl_err; + } + + page_ofs = ntfs_cluster_to_poff(vol, lcn); + page_index = ntfs_cluster_to_pidx(vol, lcn); + + lpage = read_mapping_page(sb->s_bdev->bd_mapping, + page_index, NULL); + if (IS_ERR(lpage)) { + err = PTR_ERR(lpage); + mutex_unlock(&ntfs_cb_lock); + goto read_err; + } + + lock_page(lpage); + memcpy(cb_pos, page_address(lpage) + page_ofs, + vol->cluster_size); + unlock_page(lpage); + put_page(lpage); + cb_pos += vol->cluster_size; + } + + /* Release the lock if we took it. */ + if (rl) + up_read(&ni->runlist.lock); + + /* Just a precaution. */ + if (cb_pos + 2 <= cb + cb_size) + *(u16 *)cb_pos = 0; + + /* Reset cb_pos back to the beginning. */ + cb_pos = cb; + + /* We now have both source (if present) and destination. */ + ntfs_debug("Successfully read the compression block."); + + /* The last page and maximum offset within it for the current cb. */ + cb_max_page = (cur_page << PAGE_SHIFT) + cur_ofs + cb_size; + cb_max_ofs = cb_max_page & ~PAGE_MASK; + cb_max_page >>= PAGE_SHIFT; + + /* Catch end of file inside a compression block. */ + if (cb_max_page > max_page) + cb_max_page = max_page; + + if (vcn == start_vcn - cb_clusters) { + /* Sparse cb, zero out page range overlapping the cb. */ + ntfs_debug("Found sparse compression block."); + /* We can sleep from now on, so we drop lock. */ + mutex_unlock(&ntfs_cb_lock); + if (cb_max_ofs) + cb_max_page--; + for (; cur_page < cb_max_page; cur_page++) { + page = pages[cur_page]; + if (page) { + if (likely(!cur_ofs)) + clear_page(page_address(page)); + else + memset(page_address(page) + cur_ofs, 0, + PAGE_SIZE - + cur_ofs); + flush_dcache_page(page); + kunmap_local(page_address(page)); + SetPageUptodate(page); + unlock_page(page); + if (cur_page == xpage) + xpage_done = 1; + else + put_page(page); + pages[cur_page] = NULL; + } + cb_pos += PAGE_SIZE - cur_ofs; + cur_ofs = 0; + if (cb_pos >= cb_end) + break; + } + /* If we have a partial final page, deal with it now. */ + if (cb_max_ofs && cb_pos < cb_end) { + page = pages[cur_page]; + if (page) + memset(page_address(page) + cur_ofs, 0, + cb_max_ofs - cur_ofs); + /* + * No need to update cb_pos at this stage: + * cb_pos += cb_max_ofs - cur_ofs; + */ + cur_ofs = cb_max_ofs; + } + } else if (vcn == start_vcn) { + /* We can't sleep so we need two stages. */ + unsigned int cur2_page = cur_page; + unsigned int cur_ofs2 = cur_ofs; + u8 *cb_pos2 = cb_pos; + + ntfs_debug("Found uncompressed compression block."); + /* Uncompressed cb, copy it to the destination pages. */ + if (cb_max_ofs) + cb_max_page--; + /* First stage: copy data into destination pages. */ + for (; cur_page < cb_max_page; cur_page++) { + page = pages[cur_page]; + if (page) + memcpy(page_address(page) + cur_ofs, cb_pos, + PAGE_SIZE - cur_ofs); + cb_pos += PAGE_SIZE - cur_ofs; + cur_ofs = 0; + if (cb_pos >= cb_end) + break; + } + /* If we have a partial final page, deal with it now. */ + if (cb_max_ofs && cb_pos < cb_end) { + page = pages[cur_page]; + if (page) + memcpy(page_address(page) + cur_ofs, cb_pos, + cb_max_ofs - cur_ofs); + cb_pos += cb_max_ofs - cur_ofs; + cur_ofs = cb_max_ofs; + } + /* We can sleep from now on, so drop lock. */ + mutex_unlock(&ntfs_cb_lock); + /* Second stage: finalize pages. */ + for (; cur2_page < cb_max_page; cur2_page++) { + page = pages[cur2_page]; + if (page) { + /* + * If we are outside the initialized size, zero + * the out of bounds page range. + */ + handle_bounds_compressed_page(page, i_size, + initialized_size); + flush_dcache_page(page); + kunmap_local(page_address(page)); + SetPageUptodate(page); + unlock_page(page); + if (cur2_page == xpage) + xpage_done = 1; + else + put_page(page); + pages[cur2_page] = NULL; + } + cb_pos2 += PAGE_SIZE - cur_ofs2; + cur_ofs2 = 0; + if (cb_pos2 >= cb_end) + break; + } + } else { + /* Compressed cb, decompress it into the destination page(s). */ + unsigned int prev_cur_page = cur_page; + + ntfs_debug("Found compressed compression block."); + err = ntfs_decompress(pages, completed_pages, &cur_page, + &cur_ofs, cb_max_page, cb_max_ofs, xpage, + &xpage_done, cb_pos, cb_size - (cb_pos - cb), + i_size, initialized_size); + /* + * We can sleep from now on, lock already dropped by + * ntfs_decompress(). + */ + if (err) { + ntfs_error(vol->sb, + "ntfs_decompress() failed in inode 0x%llx with error code %i. Skipping this compression block.", + ni->mft_no, -err); + /* Release the unfinished pages. */ + for (; prev_cur_page < cur_page; prev_cur_page++) { + page = pages[prev_cur_page]; + if (page) { + flush_dcache_page(page); + kunmap_local(page_address(page)); + unlock_page(page); + if (prev_cur_page != xpage) + put_page(page); + pages[prev_cur_page] = NULL; + } + } + } + } + + /* Do we have more work to do? */ + if (nr_cbs) + goto do_next_cb; + + /* Clean up if we have any pages left. Should never happen. */ + for (cur_page = 0; cur_page < max_page; cur_page++) { + page = pages[cur_page]; + if (page) { + ntfs_error(vol->sb, + "Still have pages left! Terminating them with extreme prejudice. Inode 0x%llx, page index 0x%lx.", + ni->mft_no, page->__folio_index); + flush_dcache_page(page); + kunmap_local(page_address(page)); + unlock_page(page); + if (cur_page != xpage) + put_page(page); + pages[cur_page] = NULL; + } + } + + /* We no longer need the list of pages. */ + kfree(pages); + kfree(completed_pages); + + /* If we have completed the requested page, we return success. */ + if (likely(xpage_done)) + return 0; + + ntfs_debug("Failed. Returning error code %s.", err == -EOVERFLOW ? + "EOVERFLOW" : (!err ? "EIO" : "unknown error")); + return err < 0 ? err : -EIO; + +map_rl_err: + ntfs_error(vol->sb, "ntfs_map_runlist() failed. Cannot read compression block."); + goto err_out; + +rl_err: + up_read(&ni->runlist.lock); + ntfs_error(vol->sb, "ntfs_rl_vcn_to_lcn() failed. Cannot read compression block."); + goto err_out; + +read_err: + up_read(&ni->runlist.lock); + ntfs_error(vol->sb, "IO error while reading compressed data."); + +err_out: + for (i = cur_page; i < max_page; i++) { + page = pages[i]; + if (page) { + flush_dcache_page(page); + kunmap_local(page_address(page)); + unlock_page(page); + if (i != xpage) + put_page(page); + } + } + kfree(pages); + kfree(completed_pages); + return -EIO; +} + +/* + * Match length at or above which ntfs_best_match() will stop searching for + * longer matches. + */ +#define NICE_MATCH_LEN 18 + +/* + * Maximum number of potential matches that ntfs_best_match() will consider at + * each position. + */ +#define MAX_SEARCH_DEPTH 24 + +/* log base 2 of the number of entries in the hash table for match-finding. */ +#define HASH_SHIFT 14 + +/* + * Constant for the multiplicative hash function. These hashing constants + * are used solely for the match-finding algorithm during compression. + * They are NOT part of the on-disk format. The decompressor does not + * utilize this hash. + */ +#define HASH_MULTIPLIER 0x1E35A7BD + +struct compress_context { + const unsigned char *inbuf; + int bufsize; + int size; + int rel; + int mxsz; + s16 head[1 << HASH_SHIFT]; + s16 prev[NTFS_SB_SIZE]; +}; + +/* + * Hash the next 3-byte sequence in the input buffer + */ +static inline unsigned int ntfs_hash(const u8 *p) +{ + u32 str; + u32 hash; + + /* + * Unaligned access allowed, and little endian CPU. + * Callers ensure that at least 4 (not 3) bytes are remaining. + */ + str = *(const u32 *)p & 0xFFFFFF; + hash = str * HASH_MULTIPLIER; + + /* High bits are more random than the low bits. */ + return hash >> (32 - HASH_SHIFT); +} + +/* + * Search for the longest sequence matching current position + * + * A hash table, each entry of which points to a chain of sequence + * positions sharing the corresponding hash code, is maintained to speed up + * searching for matches. To maintain the hash table, either + * ntfs_best_match() or ntfs_skip_position() has to be called for each + * consecutive position. + * + * This function is heavily used; it has to be optimized carefully. + * + * This function sets pctx->size and pctx->rel to the length and offset, + * respectively, of the longest match found. + * + * The minimum match length is assumed to be 3, and the maximum match + * length is assumed to be pctx->mxsz. If this function produces + * pctx->size < 3, then no match was found. + * + * Note: for the following reasons, this function is not guaranteed to find + * *the* longest match up to pctx->mxsz: + * + * (1) If this function finds a match of NICE_MATCH_LEN bytes or greater, + * it ends early because a match this long is good enough and it's not + * worth spending more time searching. + * + * (2) If this function considers MAX_SEARCH_DEPTH matches with a single + * position, it ends early and returns the longest match found so far. + * This saves a lot of time on degenerate inputs. + */ +static void ntfs_best_match(struct compress_context *pctx, const int i, + int best_len) +{ + const u8 * const inbuf = pctx->inbuf; + const u8 * const strptr = &inbuf[i]; /* String we're matching against */ + s16 * const prev = pctx->prev; + const int max_len = min(pctx->bufsize - i, pctx->mxsz); + const int nice_len = min(NICE_MATCH_LEN, max_len); + int depth_remaining = MAX_SEARCH_DEPTH; + const u8 *best_matchptr = strptr; + unsigned int hash; + s16 cur_match; + const u8 *matchptr; + int len; + + if (max_len < 4) + goto out; + + /* Insert the current sequence into the appropriate hash chain. */ + hash = ntfs_hash(strptr); + cur_match = pctx->head[hash]; + prev[i] = cur_match; + pctx->head[hash] = i; + + if (best_len >= max_len) { + /* + * Lazy match is being attempted, but there aren't enough length + * bits remaining to code a longer match. + */ + goto out; + } + + /* Search the appropriate hash chain for matches. */ + + for (; cur_match >= 0 && depth_remaining--; cur_match = prev[cur_match]) { + matchptr = &inbuf[cur_match]; + + /* + * Considering the potential match at 'matchptr': is it longer + * than 'best_len'? + * + * The bytes at index 'best_len' are the most likely to differ, + * so check them first. + * + * The bytes at indices 'best_len - 1' and '0' are less + * important to check separately. But doing so still gives a + * slight performance improvement, at least on x86_64, probably + * because they create separate branches for the CPU to predict + * independently of the branches in the main comparison loops. + */ + if (matchptr[best_len] != strptr[best_len] || + matchptr[best_len - 1] != strptr[best_len - 1] || + matchptr[0] != strptr[0]) + goto next_match; + + for (len = 1; len < best_len - 1; len++) + if (matchptr[len] != strptr[len]) + goto next_match; + + /* + * The match is the longest found so far --- + * at least 'best_len' + 1 bytes. Continue extending it. + */ + + best_matchptr = matchptr; + + do { + if (++best_len >= nice_len) { + /* + * 'nice_len' reached; don't waste time + * searching for longer matches. Extend the + * match as far as possible and terminate the + * search. + */ + while (best_len < max_len && + (best_matchptr[best_len] == + strptr[best_len])) + best_len++; + goto out; + } + } while (best_matchptr[best_len] == strptr[best_len]); + + /* Found a longer match, but 'nice_len' not yet reached. */ + +next_match: + /* Continue to next match in the chain. */ + ; + } + + /* + * Reached end of chain, or ended early due to reaching the maximum + * search depth. + */ + +out: + /* Return the longest match we were able to find. */ + pctx->size = best_len; + pctx->rel = best_matchptr - strptr; /* given as a negative number! */ +} + +/* + * Advance the match-finder, but don't search for matches. + */ +static void ntfs_skip_position(struct compress_context *pctx, const int i) +{ + unsigned int hash; + + if (pctx->bufsize - i < 4) + return; + + /* Insert the current sequence into the appropriate hash chain. */ + hash = ntfs_hash(pctx->inbuf + i); + pctx->prev[i] = pctx->head[hash]; + pctx->head[hash] = i; +} + +/* + * Compress a 4096-byte block + * + * Returns a header of two bytes followed by the compressed data. + * If compression is not effective, the header and an uncompressed + * block is returned. + * + * Note : two bytes may be output before output buffer overflow + * is detected, so a 4100-bytes output buffer must be reserved. + * + * Returns the size of the compressed block, including the + * header (minimal size is 2, maximum size is 4098) + * 0 if an error has been met. + */ +static unsigned int ntfs_compress_block(const char *inbuf, const int bufsize, + char *outbuf) +{ + struct compress_context *pctx; + int i; /* current position */ + int j; /* end of best match from current position */ + int k; /* end of best match from next position */ + int offs; /* offset to best match */ + int bp; /* bits to store offset */ + int bp_cur; /* saved bits to store offset at current position */ + int mxoff; /* max match offset : 1 << bp */ + unsigned int xout; + unsigned int q; /* aggregated offset and size */ + int have_match; /* do we have a match at the current position? */ + char *ptag; /* location reserved for a tag */ + int tag; /* current value of tag */ + int ntag; /* count of bits still undefined in tag */ + + pctx = kvzalloc(sizeof(struct compress_context), GFP_NOFS); + if (!pctx) + return -ENOMEM; + + /* + * All hash chains start as empty. The special value '-1' indicates the + * end of each hash chain. + */ + memset(pctx->head, 0xFF, sizeof(pctx->head)); + + pctx->inbuf = (const unsigned char *)inbuf; + pctx->bufsize = bufsize; + xout = 2; + i = 0; + bp = 4; + mxoff = 1 << bp; + pctx->mxsz = (1 << (16 - bp)) + 2; + have_match = 0; + tag = 0; + ntag = 8; + ptag = &outbuf[xout++]; + + while ((i < bufsize) && (xout < (NTFS_SB_SIZE + 2))) { + + /* + * This implementation uses "lazy" parsing: it always chooses + * the longest match, unless the match at the next position is + * longer. This is the same strategy used by the high + * compression modes of zlib. + */ + if (!have_match) { + /* + * Find the longest match at the current position. But + * first adjust the maximum match length if needed. + * (This loop might need to run more than one time in + * the case that we just output a long match.) + */ + while (mxoff < i) { + bp++; + mxoff <<= 1; + pctx->mxsz = (pctx->mxsz + 2) >> 1; + } + ntfs_best_match(pctx, i, 2); + } + + if (pctx->size >= 3) { + /* Found a match at the current position. */ + j = i + pctx->size; + bp_cur = bp; + offs = pctx->rel; + + if (pctx->size >= NICE_MATCH_LEN) { + /* Choose long matches immediately. */ + q = (~offs << (16 - bp_cur)) + (j - i - 3); + outbuf[xout++] = q & 255; + outbuf[xout++] = (q >> 8) & 255; + tag |= (1 << (8 - ntag)); + + if (j == bufsize) { + /* + * Shortcut if the match extends to the + * end of the buffer. + */ + i = j; + --ntag; + break; + } + i += 1; + do { + ntfs_skip_position(pctx, i); + } while (++i != j); + have_match = 0; + } else { + /* + * Check for a longer match at the next + * position. + */ + + /* + * Doesn't need to be while() since we just + * adjusted the maximum match length at the + * previous position. + */ + if (mxoff < i + 1) { + bp++; + mxoff <<= 1; + pctx->mxsz = (pctx->mxsz + 2) >> 1; + } + ntfs_best_match(pctx, i + 1, pctx->size); + k = i + 1 + pctx->size; + + if (k > (j + 1)) { + /* + * Next match is longer. + * Output a literal. + */ + outbuf[xout++] = inbuf[i++]; + have_match = 1; + } else { + /* + * Next match isn't longer. + * Output the current match. + */ + q = (~offs << (16 - bp_cur)) + + (j - i - 3); + outbuf[xout++] = q & 255; + outbuf[xout++] = (q >> 8) & 255; + tag |= (1 << (8 - ntag)); + + /* + * The minimum match length is 3, and + * we've run two bytes through the + * matchfinder already. So the minimum + * number of positions we need to skip + * is 1. + */ + i += 2; + do { + ntfs_skip_position(pctx, i); + } while (++i != j); + have_match = 0; + } + } + } else { + /* No match at current position. Output a literal. */ + outbuf[xout++] = inbuf[i++]; + have_match = 0; + } + + /* Store the tag if fully used. */ + if (!--ntag) { + *ptag = tag; + ntag = 8; + ptag = &outbuf[xout++]; + tag = 0; + } + } + + /* Store the last tag if partially used. */ + if (ntag == 8) + xout--; + else + *ptag = tag; + + /* Determine whether to store the data compressed or uncompressed. */ + if ((i >= bufsize) && (xout < (NTFS_SB_SIZE + 2))) { + /* Compressed. */ + outbuf[0] = (xout - 3) & 255; + outbuf[1] = 0xb0 + (((xout - 3) >> 8) & 15); + } else { + /* Uncompressed. */ + memcpy(&outbuf[2], inbuf, bufsize); + if (bufsize < NTFS_SB_SIZE) + memset(&outbuf[bufsize + 2], 0, NTFS_SB_SIZE - bufsize); + outbuf[0] = 0xff; + outbuf[1] = 0x3f; + xout = NTFS_SB_SIZE + 2; + } + + /* + * Free the compression context and return the total number of bytes + * written to 'outbuf'. + */ + kvfree(pctx); + return xout; +} + +static int ntfs_write_cb(struct ntfs_inode *ni, loff_t pos, struct page **pages, + int pages_per_cb) +{ + struct ntfs_volume *vol = ni->vol; + char *outbuf = NULL, *pbuf, *inbuf; + u32 compsz, p, insz = pages_per_cb << PAGE_SHIFT; + s32 rounded, bio_size; + unsigned int sz, bsz; + bool fail = false, allzeroes; + /* a single compressed zero */ + static char onezero[] = {0x01, 0xb0, 0x00, 0x00}; + /* a couple of compressed zeroes */ + static char twozeroes[] = {0x02, 0xb0, 0x00, 0x00, 0x00}; + /* more compressed zeroes, to be followed by some count */ + static char morezeroes[] = {0x03, 0xb0, 0x02, 0x00}; + struct page **pages_disk = NULL, *pg; + s64 bio_lcn; + struct runlist_element *rlc, *rl; + int i, err; + int pages_count = (round_up(ni->itype.compressed.block_size + 2 * + (ni->itype.compressed.block_size / NTFS_SB_SIZE) + 2, PAGE_SIZE)) / PAGE_SIZE; + size_t new_rl_count; + struct bio *bio = NULL; + loff_t new_length; + s64 new_vcn; + + inbuf = vmap(pages, pages_per_cb, VM_MAP, PAGE_KERNEL_RO); + if (!inbuf) + return -ENOMEM; + + /* may need 2 extra bytes per block and 2 more bytes */ + pages_disk = kcalloc(pages_count, sizeof(struct page *), GFP_NOFS); + if (!pages_disk) { + vunmap(inbuf); + return -ENOMEM; + } + + for (i = 0; i < pages_count; i++) { + pg = alloc_page(GFP_KERNEL); + if (!pg) { + err = -ENOMEM; + goto out; + } + pages_disk[i] = pg; + lock_page(pg); + kmap_local_page(pg); + } + + outbuf = vmap(pages_disk, pages_count, VM_MAP, PAGE_KERNEL); + if (!outbuf) { + err = -ENOMEM; + goto out; + } + + compsz = 0; + allzeroes = true; + for (p = 0; (p < insz) && !fail; p += NTFS_SB_SIZE) { + if ((p + NTFS_SB_SIZE) < insz) + bsz = NTFS_SB_SIZE; + else + bsz = insz - p; + pbuf = &outbuf[compsz]; + sz = ntfs_compress_block(&inbuf[p], bsz, pbuf); + /* fail if all the clusters (or more) are needed */ + if (!sz || ((compsz + sz + vol->cluster_size + 2) > + ni->itype.compressed.block_size)) + fail = true; + else { + if (allzeroes) { + /* check whether this is all zeroes */ + switch (sz) { + case 4: + allzeroes = !memcmp(pbuf, onezero, 4); + break; + case 5: + allzeroes = !memcmp(pbuf, twozeroes, 5); + break; + case 6: + allzeroes = !memcmp(pbuf, morezeroes, 4); + break; + default: + allzeroes = false; + break; + } + } + compsz += sz; + } + } + + if (!fail && !allzeroes) { + outbuf[compsz++] = 0; + outbuf[compsz++] = 0; + rounded = ((compsz - 1) | (vol->cluster_size - 1)) + 1; + memset(&outbuf[compsz], 0, rounded - compsz); + bio_size = rounded; + pages = pages_disk; + } else if (allzeroes) { + err = 0; + goto out; + } else { + bio_size = insz; + } + + new_vcn = ntfs_bytes_to_cluster(vol, + pos & ~((loff_t)ni->itype.compressed.block_size - 1)); + new_length = ntfs_bytes_to_cluster(vol, round_up(bio_size, vol->cluster_size)); + + err = ntfs_non_resident_attr_punch_hole(ni, new_vcn, ni->itype.compressed.block_clusters); + if (err < 0) + goto out; + + rlc = ntfs_cluster_alloc(vol, new_vcn, new_length, -1, DATA_ZONE, + false, true, true); + if (IS_ERR(rlc)) { + err = PTR_ERR(rlc); + goto out; + } + + bio_lcn = rlc->lcn; + down_write(&ni->runlist.lock); + rl = ntfs_runlists_merge(&ni->runlist, rlc, 0, &new_rl_count); + if (IS_ERR(rl)) { + up_write(&ni->runlist.lock); + ntfs_error(vol->sb, "Failed to merge runlists"); + err = PTR_ERR(rl); + if (ntfs_cluster_free_from_rl(vol, rlc)) + ntfs_error(vol->sb, "Failed to free hot clusters."); + kvfree(rlc); + goto out; + } + + ni->runlist.count = new_rl_count; + ni->runlist.rl = rl; + + err = ntfs_attr_update_mapping_pairs(ni, 0); + up_write(&ni->runlist.lock); + if (err) { + err = -EIO; + goto out; + } + + i = 0; + while (bio_size > 0) { + int page_size; + + if (bio_size >= PAGE_SIZE) { + page_size = PAGE_SIZE; + bio_size -= PAGE_SIZE; + } else { + page_size = bio_size; + bio_size = 0; + } + +setup_bio: + if (!bio) { + bio = bio_alloc(vol->sb->s_bdev, 1, REQ_OP_WRITE, + GFP_NOIO); + bio->bi_iter.bi_sector = + ntfs_bytes_to_sector(vol, + ntfs_cluster_to_bytes(vol, bio_lcn + i)); + } + + if (!bio_add_page(bio, pages[i], page_size, 0)) { + err = submit_bio_wait(bio); + bio_put(bio); + if (err) + goto out; + bio = NULL; + goto setup_bio; + } + i++; + } + + err = submit_bio_wait(bio); + bio_put(bio); +out: + vunmap(outbuf); + for (i = 0; i < pages_count; i++) { + pg = pages_disk[i]; + if (pg) { + kunmap_local(page_address(pg)); + unlock_page(pg); + put_page(pg); + } + } + kfree(pages_disk); + vunmap(inbuf); + NInoSetFileNameDirty(ni); + mark_mft_record_dirty(ni); + + return err; +} + +int ntfs_compress_write(struct ntfs_inode *ni, loff_t pos, size_t count, + struct iov_iter *from) +{ + struct folio *folio; + struct page **pages = NULL, *page; + int pages_per_cb = ni->itype.compressed.block_size >> PAGE_SHIFT; + int cb_size = ni->itype.compressed.block_size, cb_off, err = 0; + int i, ip; + size_t written = 0; + struct address_space *mapping = VFS_I(ni)->i_mapping; + + if (NInoCompressed(ni) && pos + count > ni->allocated_size) { + int err; + loff_t end = pos + count; + + err = ntfs_attr_expand(ni, end, + round_up(end, ni->itype.compressed.block_size)); + if (err) + return err; + } + + pages = kmalloc_array(pages_per_cb, sizeof(struct page *), GFP_NOFS); + if (!pages) + return -ENOMEM; + + while (count) { + pgoff_t index; + size_t copied, bytes; + int off; + + off = pos & (cb_size - 1); + bytes = cb_size - off; + if (bytes > count) + bytes = count; + + cb_off = pos & ~(cb_size - 1); + index = cb_off >> PAGE_SHIFT; + + if (unlikely(fault_in_iov_iter_readable(from, bytes))) { + err = -EFAULT; + goto out; + } + + for (i = 0; i < pages_per_cb; i++) { + folio = read_mapping_folio(mapping, index + i, NULL); + if (IS_ERR(folio)) { + for (ip = 0; ip < i; ip++) { + folio_unlock(page_folio(pages[ip])); + folio_put(page_folio(pages[ip])); + } + err = PTR_ERR(folio); + goto out; + } + + folio_lock(folio); + pages[i] = folio_page(folio, 0); + } + + WARN_ON(!bytes); + copied = 0; + ip = off >> PAGE_SHIFT; + off = offset_in_page(pos); + + for (;;) { + size_t cp, tail = PAGE_SIZE - off; + + page = pages[ip]; + cp = copy_folio_from_iter_atomic(page_folio(page), off, + min(tail, bytes), from); + flush_dcache_page(page); + + copied += cp; + bytes -= cp; + if (!bytes || !cp) + break; + + if (cp < tail) { + off += cp; + } else { + ip++; + off = 0; + } + } + + err = ntfs_write_cb(ni, pos, pages, pages_per_cb); + + for (i = 0; i < pages_per_cb; i++) { + folio = page_folio(pages[i]); + if (i < ip) { + folio_clear_dirty(folio); + folio_mark_uptodate(folio); + } + folio_unlock(folio); + folio_put(folio); + } + + if (err) + goto out; + + cond_resched(); + pos += copied; + written += copied; + count = iov_iter_count(from); + } + +out: + kfree(pages); + if (err < 0) + written = err; + + return written; +} diff --git a/fs/ntfs/debug.c b/fs/ntfs/debug.c new file mode 100644 index 000000000..f241ebc97 --- /dev/null +++ b/fs/ntfs/debug.c @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * NTFS kernel debug support. + * + * Copyright (c) 2001-2004 Anton Altaparmakov + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include "debug.h" + +/* + * __ntfs_warning - output a warning to the syslog + * @function: name of function outputting the warning + * @sb: super block of mounted ntfs filesystem + * @fmt: warning string containing format specifications + * @...: a variable number of arguments specified in @fmt + * + * Outputs a warning to the syslog for the mounted ntfs filesystem described + * by @sb. + * + * @fmt and the corresponding @... is printf style format string containing + * the warning string and the corresponding format arguments, respectively. + * + * @function is the name of the function from which __ntfs_warning is being + * called. + * + * Note, you should be using debug.h::ntfs_warning(@sb, @fmt, @...) instead + * as this provides the @function parameter automatically. + */ +void __ntfs_warning(const char *function, const struct super_block *sb, + const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + int flen = 0; + + if (function) + flen = strlen(function); + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; +#ifdef DEBUG + if (sb) + pr_warn("(device %s): %s(): %pV\n", + sb->s_id, flen ? function : "", &vaf); + else + pr_warn("%s(): %pV\n", flen ? function : "", &vaf); +#else + if (sb) + pr_warn_ratelimited("(device %s): %s(): %pV\n", + sb->s_id, flen ? function : "", &vaf); + else + pr_warn_ratelimited("%s(): %pV\n", flen ? function : "", &vaf); +#endif + va_end(args); +} + +/* + * __ntfs_error - output an error to the syslog + * @function: name of function outputting the error + * @sb: super block of mounted ntfs filesystem + * @fmt: error string containing format specifications + * @...: a variable number of arguments specified in @fmt + * + * Outputs an error to the syslog for the mounted ntfs filesystem described + * by @sb. + * + * @fmt and the corresponding @... is printf style format string containing + * the error string and the corresponding format arguments, respectively. + * + * @function is the name of the function from which __ntfs_error is being + * called. + * + * Note, you should be using debug.h::ntfs_error(@sb, @fmt, @...) instead + * as this provides the @function parameter automatically. + */ +void __ntfs_error(const char *function, struct super_block *sb, + const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + int flen = 0; + + if (function) + flen = strlen(function); + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; +#ifdef DEBUG + if (sb) + pr_err("(device %s): %s(): %pV\n", + sb->s_id, flen ? function : "", &vaf); + else + pr_err("%s(): %pV\n", flen ? function : "", &vaf); +#else + if (sb) + pr_err_ratelimited("(device %s): %s(): %pV\n", + sb->s_id, flen ? function : "", &vaf); + else + pr_err_ratelimited("%s(): %pV\n", flen ? function : "", &vaf); +#endif + va_end(args); + + if (sb) + ntfs_handle_error(sb); +} + +#ifdef DEBUG + +/* If 1, output debug messages, and if 0, don't. */ +int debug_msgs; + +void __ntfs_debug(const char *file, int line, const char *function, + const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + int flen = 0; + + if (!debug_msgs) + return; + if (function) + flen = strlen(function); + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + pr_debug("(%s, %d): %s(): %pV", file, line, flen ? function : "", &vaf); + va_end(args); +} + +/* Dump a runlist. Caller has to provide synchronisation for @rl. */ +void ntfs_debug_dump_runlist(const struct runlist_element *rl) +{ + int i; + const char *lcn_str[5] = { "LCN_DELALLOC ", "LCN_HOLE ", + "LCN_RL_NOT_MAPPED", "LCN_ENOENT ", + "LCN_unknown " }; + + if (!debug_msgs) + return; + pr_debug("Dumping runlist (values in hex):\n"); + if (!rl) { + pr_debug("Run list not present.\n"); + return; + } + pr_debug("VCN LCN Run length\n"); + for (i = 0; ; i++) { + s64 lcn = (rl + i)->lcn; + + if (lcn < 0) { + int index = -lcn - 1; + + if (index > -LCN_ENOENT - 1) + index = 3; + pr_debug("%-16Lx %s %-16Lx%s\n", + (long long)(rl + i)->vcn, lcn_str[index], + (long long)(rl + i)->length, + (rl + i)->length ? "" : + " (runlist end)"); + } else + pr_debug("%-16Lx %-16Lx %-16Lx%s\n", + (long long)(rl + i)->vcn, + (long long)(rl + i)->lcn, + (long long)(rl + i)->length, + (rl + i)->length ? "" : + " (runlist end)"); + if (!(rl + i)->length) + break; + } +} + +#endif diff --git a/fs/ntfs/debug.h b/fs/ntfs/debug.h new file mode 100644 index 000000000..d8831d492 --- /dev/null +++ b/fs/ntfs/debug.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * NTFS kernel debug support. + * + * Copyright (c) 2001-2004 Anton Altaparmakov + */ + +#ifndef _LINUX_NTFS_DEBUG_H +#define _LINUX_NTFS_DEBUG_H + +#include + +#include "runlist.h" + +#ifdef DEBUG + +extern int debug_msgs; + +extern __printf(4, 5) +void __ntfs_debug(const char *file, int line, const char *function, + const char *format, ...); +/* + * ntfs_debug - write a debug level message to syslog + * @f: a printf format string containing the message + * @...: the variables to substitute into @f + * + * ntfs_debug() writes a DEBUG level message to the syslog but only if the + * driver was compiled with -DDEBUG. Otherwise, the call turns into a NOP. + */ +#define ntfs_debug(f, a...) \ + __ntfs_debug(__FILE__, __LINE__, __func__, f, ##a) + +void ntfs_debug_dump_runlist(const struct runlist_element *rl); + +#else /* !DEBUG */ + +#define ntfs_debug(fmt, ...) \ +do { \ + if (0) \ + no_printk(fmt, ##__VA_ARGS__); \ +} while (0) + +#define ntfs_debug_dump_runlist(rl) \ +do { \ + if (0) \ + (void)rl; \ +} while (0) + +#endif /* !DEBUG */ + +extern __printf(3, 4) +void __ntfs_warning(const char *function, const struct super_block *sb, + const char *fmt, ...); +#define ntfs_warning(sb, f, a...) __ntfs_warning(__func__, sb, f, ##a) + +extern __printf(3, 4) +void __ntfs_error(const char *function, struct super_block *sb, + const char *fmt, ...); +#define ntfs_error(sb, f, a...) __ntfs_error(__func__, sb, f, ##a) + +void ntfs_handle_error(struct super_block *sb); + +#endif /* _LINUX_NTFS_DEBUG_H */ diff --git a/fs/ntfs/dir.c b/fs/ntfs/dir.c new file mode 100644 index 000000000..6fa9ae337 --- /dev/null +++ b/fs/ntfs/dir.c @@ -0,0 +1,1206 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * NTFS kernel directory operations. + * + * Copyright (c) 2001-2007 Anton Altaparmakov + * Copyright (c) 2002 Richard Russon + * Copyright (c) 2025 LG Electronics Co., Ltd. + */ + +#include + +#include "dir.h" +#include "mft.h" +#include "ntfs.h" +#include "index.h" +#include "reparse.h" + +#include + +/* + * The little endian Unicode string $I30 as a global constant. + */ +__le16 I30[5] = { cpu_to_le16('$'), cpu_to_le16('I'), + cpu_to_le16('3'), cpu_to_le16('0'), 0 }; + +static inline u64 ntfs_check_mref(u64 mref) +{ + if (IS_ERR_MREF(mref)) + return ERR_MREF(-EIO); + return mref; +} + +/* + * ntfs_lookup_inode_by_name - find an inode in a directory given its name + * @dir_ni: ntfs inode of the directory in which to search for the name + * @uname: Unicode name for which to search in the directory + * @uname_len: length of the name @uname in Unicode characters + * @res: return the found file name if necessary (see below) + * + * Look for an inode with name @uname in the directory with inode @dir_ni. + * ntfs_lookup_inode_by_name() walks the contents of the directory looking for + * the Unicode name. If the name is found in the directory, the corresponding + * inode number (>= 0) is returned as a mft reference in cpu format, i.e. it + * is a 64-bit number containing the sequence number. + * + * On error, a negative value is returned corresponding to the error code. In + * particular if the inode is not found -ENOENT is returned. Note that you + * can't just check the return value for being negative, you have to check the + * inode number for being negative which you can extract using MREC(return + * value). + * + * Note, @uname_len does not include the (optional) terminating NULL character. + * + * Note, we look for a case sensitive match first but we also look for a case + * insensitive match at the same time. If we find a case insensitive match, we + * save that for the case that we don't find an exact match, where we return + * the case insensitive match and setup @res (which we allocate!) with the mft + * reference, the file name type, length and with a copy of the little endian + * Unicode file name itself. If we match a file name which is in the DOS name + * space, we only return the mft reference and file name type in @res. + * ntfs_lookup() then uses this to find the long file name in the inode itself. + * This is to avoid polluting the dcache with short file names. We want them to + * work but we don't care for how quickly one can access them. This also fixes + * the dcache aliasing issues. + * + * Locking: - Caller must hold i_mutex on the directory. + * - Each page cache page in the index allocation mapping must be + * locked whilst being accessed otherwise we may find a corrupt + * page due to it being under ->writepage at the moment which + * applies the mst protection fixups before writing out and then + * removes them again after the write is complete after which it + * unlocks the page. + */ +u64 ntfs_lookup_inode_by_name(struct ntfs_inode *dir_ni, const __le16 *uname, + const int uname_len, struct ntfs_name **res) +{ + struct ntfs_volume *vol = dir_ni->vol; + struct super_block *sb = vol->sb; + struct inode *ia_vi = NULL; + struct mft_record *m; + struct index_root *ir; + struct index_entry *ie; + struct index_block *ia; + u8 *index_end; + u64 mref; + struct ntfs_attr_search_ctx *ctx; + int err, rc; + s64 vcn, old_vcn; + struct address_space *ia_mapping; + struct folio *folio; + u8 *kaddr = NULL; + struct ntfs_name *name = NULL; + + /* Get hold of the mft record for the directory. */ + m = map_mft_record(dir_ni); + if (IS_ERR(m)) { + ntfs_error(sb, "map_mft_record() failed with error code %ld.", + -PTR_ERR(m)); + return ERR_MREF(PTR_ERR(m)); + } + ctx = ntfs_attr_get_search_ctx(dir_ni, m); + if (unlikely(!ctx)) { + err = -ENOMEM; + goto err_out; + } + /* Find the index root attribute in the mft record. */ + err = ntfs_attr_lookup(AT_INDEX_ROOT, I30, 4, CASE_SENSITIVE, 0, NULL, + 0, ctx); + if (unlikely(err)) { + if (err == -ENOENT) { + ntfs_error(sb, + "Index root attribute missing in directory inode 0x%llx.", + dir_ni->mft_no); + err = -EIO; + } + goto err_out; + } + /* Get to the index root value (it's been verified in read_inode). */ + ir = (struct index_root *)((u8 *)ctx->attr + + le16_to_cpu(ctx->attr->data.resident.value_offset)); + index_end = (u8 *)&ir->index + le32_to_cpu(ir->index.index_length); + /* The first index entry. */ + ie = (struct index_entry *)((u8 *)&ir->index + + le32_to_cpu(ir->index.entries_offset)); + /* + * Loop until we exceed valid memory (corruption case) or until we + * reach the last entry. + */ + for (;; ie = (struct index_entry *)((u8 *)ie + le16_to_cpu(ie->length))) { + /* Bounds checks. */ + if ((u8 *)ie < (u8 *)ctx->mrec || + (u8 *)ie + sizeof(struct index_entry_header) > index_end || + (u8 *)ie + sizeof(struct index_entry_header) + le16_to_cpu(ie->key_length) > + index_end || (u8 *)ie + le16_to_cpu(ie->length) > index_end) + goto dir_err_out; + /* + * The last entry cannot contain a name. It can however contain + * a pointer to a child node in the B+tree so we just break out. + */ + if (ie->flags & INDEX_ENTRY_END) + break; + /* Key length should not be zero if it is not last entry. */ + if (!ie->key_length) + goto dir_err_out; + /* + * We perform a case sensitive comparison and if that matches + * we are done and return the mft reference of the inode (i.e. + * the inode number together with the sequence number for + * consistency checking). We convert it to cpu format before + * returning. + */ + if (ntfs_are_names_equal(uname, uname_len, + (__le16 *)&ie->key.file_name.file_name, + ie->key.file_name.file_name_length, + CASE_SENSITIVE, vol->upcase, vol->upcase_len)) { +found_it: + /* + * We have a perfect match, so we don't need to care + * about having matched imperfectly before, so we can + * free name and set *res to NULL. + * However, if the perfect match is a short file name, + * we need to signal this through *res, so that + * ntfs_lookup() can fix dcache aliasing issues. + * As an optimization we just reuse an existing + * allocation of *res. + */ + if (ie->key.file_name.file_name_type == FILE_NAME_DOS) { + if (!name) { + name = kmalloc(sizeof(struct ntfs_name), + GFP_NOFS); + if (!name) { + err = -ENOMEM; + goto err_out; + } + } + name->mref = le64_to_cpu( + ie->data.dir.indexed_file); + name->type = FILE_NAME_DOS; + name->len = 0; + *res = name; + } else { + kfree(name); + *res = NULL; + } + mref = le64_to_cpu(ie->data.dir.indexed_file); + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(dir_ni); + return ntfs_check_mref(mref); + } + /* + * For a case insensitive mount, we also perform a case + * insensitive comparison (provided the file name is not in the + * POSIX namespace). If the comparison matches, and the name is + * in the WIN32 namespace, we cache the filename in *res so + * that the caller, ntfs_lookup(), can work on it. If the + * comparison matches, and the name is in the DOS namespace, we + * only cache the mft reference and the file name type (we set + * the name length to zero for simplicity). + */ + if ((!NVolCaseSensitive(vol) || + ie->key.file_name.file_name_type == FILE_NAME_DOS) && + ntfs_are_names_equal(uname, uname_len, + (__le16 *)&ie->key.file_name.file_name, + ie->key.file_name.file_name_length, + IGNORE_CASE, vol->upcase, + vol->upcase_len)) { + int name_size = sizeof(struct ntfs_name); + u8 type = ie->key.file_name.file_name_type; + u8 len = ie->key.file_name.file_name_length; + + /* Only one case insensitive matching name allowed. */ + if (name) { + ntfs_error(sb, + "Found already allocated name in phase 1. Please run chkdsk"); + goto dir_err_out; + } + + if (type != FILE_NAME_DOS) + name_size += len * sizeof(__le16); + name = kmalloc(name_size, GFP_NOFS); + if (!name) { + err = -ENOMEM; + goto err_out; + } + name->mref = le64_to_cpu(ie->data.dir.indexed_file); + name->type = type; + if (type != FILE_NAME_DOS) { + name->len = len; + memcpy(name->name, ie->key.file_name.file_name, + len * sizeof(__le16)); + } else + name->len = 0; + *res = name; + } + /* + * Not a perfect match, need to do full blown collation so we + * know which way in the B+tree we have to go. + */ + rc = ntfs_collate_names(uname, uname_len, + (__le16 *)&ie->key.file_name.file_name, + ie->key.file_name.file_name_length, 1, + IGNORE_CASE, vol->upcase, vol->upcase_len); + /* + * If uname collates before the name of the current entry, there + * is definitely no such name in this index but we might need to + * descend into the B+tree so we just break out of the loop. + */ + if (rc == -1) + break; + /* The names are not equal, continue the search. */ + if (rc) + continue; + /* + * Names match with case insensitive comparison, now try the + * case sensitive comparison, which is required for proper + * collation. + */ + rc = ntfs_collate_names(uname, uname_len, + (__le16 *)&ie->key.file_name.file_name, + ie->key.file_name.file_name_length, 1, + CASE_SENSITIVE, vol->upcase, vol->upcase_len); + if (rc == -1) + break; + if (rc) + continue; + /* + * Perfect match, this will never happen as the + * ntfs_are_names_equal() call will have gotten a match but we + * still treat it correctly. + */ + goto found_it; + } + /* + * We have finished with this index without success. Check for the + * presence of a child node and if not present return -ENOENT, unless + * we have got a matching name cached in name in which case return the + * mft reference associated with it. + */ + if (!(ie->flags & INDEX_ENTRY_NODE)) { + if (name) { + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(dir_ni); + return ntfs_check_mref(name->mref); + } + ntfs_debug("Entry not found."); + err = -ENOENT; + goto err_out; + } /* Child node present, descend into it. */ + + /* Get the starting vcn of the index_block holding the child node. */ + vcn = le64_to_cpup((__le64 *)((u8 *)ie + le16_to_cpu(ie->length) - 8)); + + /* + * We are done with the index root and the mft record. Release them, + * otherwise we deadlock with read_mapping_folio(). + */ + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(dir_ni); + m = NULL; + ctx = NULL; + + ia_vi = ntfs_index_iget(VFS_I(dir_ni), I30, 4); + if (IS_ERR(ia_vi)) { + err = PTR_ERR(ia_vi); + goto err_out; + } + + ia_mapping = ia_vi->i_mapping; +descend_into_child_node: + /* + * Convert vcn to index into the index allocation attribute in units + * of PAGE_SIZE and map the page cache page, reading it from + * disk if necessary. + */ + folio = read_mapping_folio(ia_mapping, vcn << + dir_ni->itype.index.vcn_size_bits >> PAGE_SHIFT, NULL); + if (IS_ERR(folio)) { + ntfs_error(sb, "Failed to map directory index page, error %ld.", + -PTR_ERR(folio)); + err = PTR_ERR(folio); + goto err_out; + } + + folio_lock(folio); + kaddr = kmalloc(PAGE_SIZE, GFP_NOFS); + if (!kaddr) { + err = -ENOMEM; + folio_unlock(folio); + folio_put(folio); + goto unm_err_out; + } + + memcpy_from_folio(kaddr, folio, 0, PAGE_SIZE); + post_read_mst_fixup((struct ntfs_record *)kaddr, PAGE_SIZE); + folio_unlock(folio); + folio_put(folio); +fast_descend_into_child_node: + /* Get to the index allocation block. */ + ia = (struct index_block *)(kaddr + ((vcn << + dir_ni->itype.index.vcn_size_bits) & ~PAGE_MASK)); + /* Bounds checks. */ + if ((u8 *)ia < kaddr || (u8 *)ia > kaddr + PAGE_SIZE) { + ntfs_error(sb, + "Out of bounds check failed. Corrupt directory inode 0x%llx or driver bug.", + dir_ni->mft_no); + goto unm_err_out; + } + index_end = (u8 *)ia + dir_ni->itype.index.block_size; + if (index_end > kaddr + PAGE_SIZE) { + ntfs_error(sb, + "Index buffer (VCN 0x%llx) of directory inode 0x%llx crosses page boundary. Impossible! Cannot access! This is probably a bug in the driver.", + vcn, dir_ni->mft_no); + goto unm_err_out; + } + err = ntfs_index_block_inconsistent(vol, ia, + dir_ni->itype.index.block_size, + vcn, COLLATION_FILE_NAME, + dir_ni->mft_no); + if (err) + goto unm_err_out; + index_end = (u8 *)&ia->index + le32_to_cpu(ia->index.index_length); + /* The first index entry. */ + ie = (struct index_entry *)((u8 *)&ia->index + + le32_to_cpu(ia->index.entries_offset)); + /* + * Iterate similar to above big loop but applied to index buffer, thus + * loop until we exceed valid memory (corruption case) or until we + * reach the last entry. + */ + for (;; ie = (struct index_entry *)((u8 *)ie + le16_to_cpu(ie->length))) { + /* + * The last entry cannot contain a name. It can however contain + * a pointer to a child node in the B+tree so we just break out. + */ + if (ie->flags & INDEX_ENTRY_END) + break; + /* Key length should not be zero if it is not last entry. */ + if (!ie->key_length) + goto unm_err_out; + /* + * We perform a case sensitive comparison and if that matches + * we are done and return the mft reference of the inode (i.e. + * the inode number together with the sequence number for + * consistency checking). We convert it to cpu format before + * returning. + */ + if (ntfs_are_names_equal(uname, uname_len, + (__le16 *)&ie->key.file_name.file_name, + ie->key.file_name.file_name_length, + CASE_SENSITIVE, vol->upcase, vol->upcase_len)) { +found_it2: + /* + * We have a perfect match, so we don't need to care + * about having matched imperfectly before, so we can + * free name and set *res to NULL. + * However, if the perfect match is a short file name, + * we need to signal this through *res, so that + * ntfs_lookup() can fix dcache aliasing issues. + * As an optimization we just reuse an existing + * allocation of *res. + */ + if (ie->key.file_name.file_name_type == FILE_NAME_DOS) { + if (!name) { + name = kmalloc(sizeof(struct ntfs_name), + GFP_NOFS); + if (!name) { + err = -ENOMEM; + goto unm_err_out; + } + } + name->mref = le64_to_cpu( + ie->data.dir.indexed_file); + name->type = FILE_NAME_DOS; + name->len = 0; + *res = name; + } else { + kfree(name); + *res = NULL; + } + mref = le64_to_cpu(ie->data.dir.indexed_file); + kfree(kaddr); + iput(ia_vi); + return ntfs_check_mref(mref); + } + /* + * For a case insensitive mount, we also perform a case + * insensitive comparison (provided the file name is not in the + * POSIX namespace). If the comparison matches, and the name is + * in the WIN32 namespace, we cache the filename in *res so + * that the caller, ntfs_lookup(), can work on it. If the + * comparison matches, and the name is in the DOS namespace, we + * only cache the mft reference and the file name type (we set + * the name length to zero for simplicity). + */ + if ((!NVolCaseSensitive(vol) || + ie->key.file_name.file_name_type == FILE_NAME_DOS) && + ntfs_are_names_equal(uname, uname_len, + (__le16 *)&ie->key.file_name.file_name, + ie->key.file_name.file_name_length, + IGNORE_CASE, vol->upcase, + vol->upcase_len)) { + int name_size = sizeof(struct ntfs_name); + u8 type = ie->key.file_name.file_name_type; + u8 len = ie->key.file_name.file_name_length; + + /* Only one case insensitive matching name allowed. */ + if (name) { + ntfs_error(sb, + "Found already allocated name in phase 2. Please run chkdsk"); + kfree(kaddr); + goto dir_err_out; + } + + if (type != FILE_NAME_DOS) + name_size += len * sizeof(__le16); + name = kmalloc(name_size, GFP_NOFS); + if (!name) { + err = -ENOMEM; + goto unm_err_out; + } + name->mref = le64_to_cpu(ie->data.dir.indexed_file); + name->type = type; + if (type != FILE_NAME_DOS) { + name->len = len; + memcpy(name->name, ie->key.file_name.file_name, + len * sizeof(__le16)); + } else + name->len = 0; + *res = name; + } + /* + * Not a perfect match, need to do full blown collation so we + * know which way in the B+tree we have to go. + */ + rc = ntfs_collate_names(uname, uname_len, + (__le16 *)&ie->key.file_name.file_name, + ie->key.file_name.file_name_length, 1, + IGNORE_CASE, vol->upcase, vol->upcase_len); + /* + * If uname collates before the name of the current entry, there + * is definitely no such name in this index but we might need to + * descend into the B+tree so we just break out of the loop. + */ + if (rc == -1) + break; + /* The names are not equal, continue the search. */ + if (rc) + continue; + /* + * Names match with case insensitive comparison, now try the + * case sensitive comparison, which is required for proper + * collation. + */ + rc = ntfs_collate_names(uname, uname_len, + (__le16 *)&ie->key.file_name.file_name, + ie->key.file_name.file_name_length, 1, + CASE_SENSITIVE, vol->upcase, vol->upcase_len); + if (rc == -1) + break; + if (rc) + continue; + /* + * Perfect match, this will never happen as the + * ntfs_are_names_equal() call will have gotten a match but we + * still treat it correctly. + */ + goto found_it2; + } + /* + * We have finished with this index buffer without success. Check for + * the presence of a child node. + */ + if (ie->flags & INDEX_ENTRY_NODE) { + if ((ia->index.flags & NODE_MASK) == LEAF_NODE) { + ntfs_error(sb, + "Index entry with child node found in a leaf node in directory inode 0x%llx.", + dir_ni->mft_no); + goto unm_err_out; + } + /* Child node present, descend into it. */ + old_vcn = vcn; + vcn = le64_to_cpup((__le64 *)((u8 *)ie + + le16_to_cpu(ie->length) - 8)); + if (vcn >= 0) { + /* + * If vcn is in the same page cache page as old_vcn we + * recycle the mapped page. + */ + if (ntfs_cluster_to_pidx(vol, old_vcn) == + ntfs_cluster_to_pidx(vol, vcn)) + goto fast_descend_into_child_node; + kfree(kaddr); + kaddr = NULL; + goto descend_into_child_node; + } + ntfs_error(sb, "Negative child node vcn in directory inode 0x%llx.", + dir_ni->mft_no); + goto unm_err_out; + } + /* + * No child node present, return -ENOENT, unless we have got a matching + * name cached in name in which case return the mft reference + * associated with it. + */ + if (name) { + kfree(kaddr); + iput(ia_vi); + return ntfs_check_mref(name->mref); + } + ntfs_debug("Entry not found."); + err = -ENOENT; +unm_err_out: + kfree(kaddr); +err_out: + if (!err) + err = -EIO; + if (ctx) + ntfs_attr_put_search_ctx(ctx); + if (m) + unmap_mft_record(dir_ni); + kfree(name); + *res = NULL; + if (!IS_ERR_OR_NULL(ia_vi)) + iput(ia_vi); + return ERR_MREF(err); +dir_err_out: + ntfs_error(sb, "Corrupt directory. Aborting lookup."); + goto err_out; +} + +/* + * ntfs_filldir - ntfs specific filldir method + * @vol: current ntfs volume + * @ndir: ntfs inode of current directory + * @ia_page: page in which the index allocation buffer @ie is in resides + * @ie: current index entry + * @name: buffer to use for the converted name + * @actor: what to feed the entries to + * + * Convert the Unicode @name to the loaded NLS and pass it to the @filldir + * callback. + * + * If @ia_page is not NULL it is the locked page containing the index + * allocation block containing the index entry @ie. + * + * Note, we drop (and then reacquire) the page lock on @ia_page across the + * @filldir() call otherwise we would deadlock with NFSd when it calls ->lookup + * since ntfs_lookup() will lock the same page. As an optimization, we do not + * retake the lock if we are returning a non-zero value as ntfs_readdir() + * would need to drop the lock immediately anyway. + */ +static inline int ntfs_filldir(struct ntfs_volume *vol, + struct ntfs_inode *ndir, struct page *ia_page, struct index_entry *ie, + u8 *name, struct dir_context *actor) +{ + unsigned long mref; + int name_len; + unsigned int dt_type; + u8 name_type; + + name_type = ie->key.file_name.file_name_type; + if (name_type == FILE_NAME_DOS) { + ntfs_debug("Skipping DOS name space entry."); + return 0; + } + if (MREF_LE(ie->data.dir.indexed_file) == FILE_root) { + ntfs_debug("Skipping root directory self reference entry."); + return 0; + } + if (MREF_LE(ie->data.dir.indexed_file) < FILE_first_user && + !NVolShowSystemFiles(vol)) { + ntfs_debug("Skipping system file."); + return 0; + } + if (!NVolShowHiddenFiles(vol) && + (ie->key.file_name.file_attributes & FILE_ATTR_HIDDEN)) { + ntfs_debug("Skipping hidden file."); + return 0; + } + + name_len = ntfs_ucstonls(vol, (__le16 *)&ie->key.file_name.file_name, + ie->key.file_name.file_name_length, &name, + NTFS_MAX_NAME_LEN * NLS_MAX_CHARSET_SIZE + 1); + if (name_len <= 0) { + ntfs_warning(vol->sb, "Skipping unrepresentable inode 0x%llx.", + (long long)MREF_LE(ie->data.dir.indexed_file)); + return 0; + } + + mref = MREF_LE(ie->data.dir.indexed_file); + if (ie->key.file_name.file_attributes & + FILE_ATTR_DUP_FILE_NAME_INDEX_PRESENT) + dt_type = DT_DIR; + else if (ie->key.file_name.file_attributes & FILE_ATTR_REPARSE_POINT) + dt_type = ntfs_reparse_tag_dt_types(vol, mref); + else + dt_type = DT_REG; + + /* + * Drop the page lock otherwise we deadlock with NFS when it calls + * ->lookup since ntfs_lookup() will lock the same page. + */ + if (ia_page) + unlock_page(ia_page); + ntfs_debug("Calling filldir for %s with len %i, fpos 0x%llx, inode 0x%lx, DT_%s.", + name, name_len, actor->pos, mref, dt_type == DT_DIR ? "DIR" : "REG"); + if (!dir_emit(actor, name, name_len, mref, dt_type)) + return 1; + /* Relock the page but not if we are aborting ->readdir. */ + if (ia_page) + lock_page(ia_page); + return 0; +} + +struct ntfs_file_private { + void *key; + __le16 key_length; + bool end_in_iterate; + loff_t curr_pos; +}; + +struct ntfs_index_ra { + unsigned long start_index; + unsigned int count; + struct rb_node rb_node; +}; + +static void ntfs_insert_rb(struct ntfs_index_ra *nir, struct rb_root *root) +{ + struct rb_node **new = &root->rb_node, *parent = NULL; + struct ntfs_index_ra *cnir; + + while (*new) { + parent = *new; + cnir = rb_entry(parent, struct ntfs_index_ra, rb_node); + if (nir->start_index < cnir->start_index) + new = &parent->rb_left; + else if (nir->start_index >= cnir->start_index + cnir->count) + new = &parent->rb_right; + else { + pr_err("nir start index : %ld, count : %d, cnir start_index : %ld, count : %d\n", + nir->start_index, nir->count, cnir->start_index, cnir->count); + return; + } + } + + rb_link_node(&nir->rb_node, parent, new); + rb_insert_color(&nir->rb_node, root); +} + +static int ntfs_ia_blocks_readahead(struct ntfs_inode *ia_ni, loff_t pos) +{ + unsigned long dir_start_index, dir_end_index; + struct inode *ia_vi = VFS_I(ia_ni); + struct file_ra_state *dir_ra; + + dir_end_index = (i_size_read(ia_vi) + PAGE_SIZE - 1) >> PAGE_SHIFT; + dir_start_index = (pos + PAGE_SIZE - 1) >> PAGE_SHIFT; + + if (dir_start_index >= dir_end_index) + return 0; + + dir_ra = kzalloc(sizeof(*dir_ra), GFP_NOFS); + if (!dir_ra) + return -ENOMEM; + + file_ra_state_init(dir_ra, ia_vi->i_mapping); + dir_end_index = (i_size_read(ia_vi) + PAGE_SIZE - 1) >> PAGE_SHIFT; + dir_start_index = (pos + PAGE_SIZE - 1) >> PAGE_SHIFT; + dir_ra->ra_pages = dir_end_index - dir_start_index; + page_cache_sync_readahead(ia_vi->i_mapping, dir_ra, NULL, + dir_start_index, dir_end_index - dir_start_index); + kfree(dir_ra); + + return 0; +} + +static int ntfs_readdir(struct file *file, struct dir_context *actor) +{ + struct inode *vdir = file_inode(file); + struct super_block *sb = vdir->i_sb; + struct ntfs_inode *ndir = NTFS_I(vdir); + struct ntfs_volume *vol = NTFS_SB(sb); + struct ntfs_attr_search_ctx *ctx = NULL; + struct ntfs_index_context *ictx = NULL; + u8 *name; + struct index_root *ir; + struct index_entry *next = NULL; + struct ntfs_file_private *private = NULL; + int err = 0; + loff_t ie_pos = 2; /* initialize it with dot and dotdot size */ + struct ntfs_index_ra *nir = NULL; + unsigned long index; + struct rb_root ra_root = RB_ROOT; + struct file_ra_state *ra; + + ntfs_debug("Entering for inode 0x%llx, fpos 0x%llx.", + ndir->mft_no, actor->pos); + + if (file->private_data) { + private = file->private_data; + + if (actor->pos != private->curr_pos) { + /* + * If actor->pos is different from the previous passed + * one, Discard the private->key and fill dirent buffer + * with linear lookup. + */ + kfree(private->key); + private->key = NULL; + private->end_in_iterate = false; + } else if (private->end_in_iterate) { + kfree(private->key); + kfree(file->private_data); + file->private_data = NULL; + return 0; + } + } + + /* Emulate . and .. for all directories. */ + if (!dir_emit_dots(file, actor)) + return 0; + + /* + * Allocate a buffer to store the current name being processed + * converted to format determined by current NLS. + */ + name = kmalloc(NTFS_MAX_NAME_LEN * NLS_MAX_CHARSET_SIZE + 1, GFP_NOFS); + if (unlikely(!name)) + return -ENOMEM; + + mutex_lock_nested(&ndir->mrec_lock, NTFS_INODE_MUTEX_PARENT); + ictx = ntfs_index_ctx_get(ndir, I30, 4); + if (!ictx) { + kfree(name); + mutex_unlock(&ndir->mrec_lock); + return -ENOMEM; + } + + ra = kzalloc(sizeof(struct file_ra_state), GFP_NOFS); + if (!ra) { + kfree(name); + ntfs_index_ctx_put(ictx); + mutex_unlock(&ndir->mrec_lock); + return -ENOMEM; + } + file_ra_state_init(ra, vol->mft_ino->i_mapping); + + if (private && private->key) { + /* + * Find index witk private->key using ntfs_index_lookup() + * instead of linear index lookup. + */ + err = ntfs_index_lookup(private->key, + le16_to_cpu(private->key_length), + ictx); + if (!err) { + next = ictx->entry; + /* + * Update ie_pos with private->curr_pos + * to make next d_off of dirent correct. + */ + ie_pos = private->curr_pos; + + if (actor->pos > vol->mft_record_size && ictx->ia_ni) { + err = ntfs_ia_blocks_readahead(ictx->ia_ni, actor->pos); + if (err) + goto out; + } + + goto nextdir; + } else { + goto out; + } + } else if (!private) { + private = kzalloc(sizeof(struct ntfs_file_private), GFP_KERNEL); + if (!private) { + err = -ENOMEM; + goto out; + } + file->private_data = private; + } + + ctx = ntfs_attr_get_search_ctx(ndir, NULL); + if (!ctx) { + err = -ENOMEM; + goto out; + } + + /* Find the index root attribute in the mft record. */ + if (ntfs_attr_lookup(AT_INDEX_ROOT, I30, 4, CASE_SENSITIVE, 0, NULL, 0, + ctx)) { + ntfs_error(sb, "Index root attribute missing in directory inode %llu", + ndir->mft_no); + ntfs_attr_put_search_ctx(ctx); + err = -ENOMEM; + goto out; + } + + /* Get to the index root value. */ + ir = (struct index_root *)((u8 *)ctx->attr + + le16_to_cpu(ctx->attr->data.resident.value_offset)); + + ictx->ir = ir; + ictx->actx = ctx; + ictx->parent_vcn[ictx->pindex] = VCN_INDEX_ROOT_PARENT; + ictx->is_in_root = true; + ictx->parent_pos[ictx->pindex] = 0; + + ictx->block_size = le32_to_cpu(ir->index_block_size); + if (ictx->block_size < NTFS_BLOCK_SIZE) { + ntfs_error(sb, "Index block size (%d) is smaller than the sector size (%d)", + ictx->block_size, NTFS_BLOCK_SIZE); + err = -EIO; + goto out; + } + + if (vol->cluster_size <= ictx->block_size) + ictx->vcn_size_bits = vol->cluster_size_bits; + else + ictx->vcn_size_bits = NTFS_BLOCK_SIZE_BITS; + ictx->cr = ir->collation_rule; + + /* The first index entry. */ + next = (struct index_entry *)((u8 *)&ir->index + + le32_to_cpu(ir->index.entries_offset)); + + if (next->flags & INDEX_ENTRY_NODE) { + ictx->ia_ni = ntfs_ia_open(ictx, ictx->idx_ni); + if (!ictx->ia_ni) { + err = -EINVAL; + goto out; + } + + err = ntfs_ia_blocks_readahead(ictx->ia_ni, actor->pos); + if (err) + goto out; + } + + if (next->flags & INDEX_ENTRY_NODE) { + next = ntfs_index_walk_down(next, ictx); + if (IS_ERR(next)) { + err = PTR_ERR(next); + goto out; + } + } + + if (next && !(next->flags & INDEX_ENTRY_END)) + goto nextdir; + + while (1) { + next = ntfs_index_next(next, ictx); + if (IS_ERR(next)) { + err = PTR_ERR(next); + goto out; + } + if (!next) + break; +nextdir: + if (ie_pos < actor->pos) { + ie_pos += le16_to_cpu(next->length); + continue; + } + + actor->pos = ie_pos; + + index = ntfs_mft_no_to_pidx(vol, + MREF_LE(next->data.dir.indexed_file)); + if (nir) { + struct ntfs_index_ra *cnir; + struct rb_node *node = ra_root.rb_node; + + if (nir->start_index <= index && + index < nir->start_index + nir->count) { + /* No behavior */ + goto filldir; + } + + while (node) { + cnir = rb_entry(node, struct ntfs_index_ra, rb_node); + if (cnir->start_index <= index && + index < cnir->start_index + cnir->count) { + goto filldir; + } else if (cnir->start_index + cnir->count == index) { + cnir->count++; + goto filldir; + } else if (!cnir->start_index && cnir->start_index - 1 == index) { + cnir->start_index = index; + goto filldir; + } + + if (index < cnir->start_index) + node = node->rb_left; + else if (index >= cnir->start_index + cnir->count) + node = node->rb_right; + } + + if (nir->start_index + nir->count == index) { + nir->count++; + } else if (!nir->start_index && nir->start_index - 1 == index) { + nir->start_index = index; + } else if (nir->count > 2) { + ntfs_insert_rb(nir, &ra_root); + nir = NULL; + } else { + nir->start_index = index; + nir->count = 1; + } + } + + if (!nir) { + nir = kzalloc(sizeof(struct ntfs_index_ra), GFP_KERNEL); + if (nir) { + nir->start_index = index; + nir->count = 1; + } + } + +filldir: + /* Submit the name to the filldir callback. */ + err = ntfs_filldir(vol, ndir, NULL, next, name, actor); + if (err) { + /* + * Store index key value to file private_data to start + * from current index offset on next round. + */ + private = file->private_data; + kfree(private->key); + private->key = kmalloc(le16_to_cpu(next->key_length), GFP_KERNEL); + if (!private->key) { + err = -ENOMEM; + goto out; + } + + memcpy(private->key, &next->key.file_name, le16_to_cpu(next->key_length)); + private->key_length = next->key_length; + break; + } + ie_pos += le16_to_cpu(next->length); + } + + if (!err) + private->end_in_iterate = true; + else + err = 0; + + private->curr_pos = actor->pos = ie_pos; +out: + while (!RB_EMPTY_ROOT(&ra_root)) { + struct ntfs_index_ra *cnir; + struct rb_node *node; + + node = rb_first(&ra_root); + cnir = rb_entry(node, struct ntfs_index_ra, rb_node); + ra->ra_pages = cnir->count; + page_cache_sync_readahead(vol->mft_ino->i_mapping, ra, NULL, + cnir->start_index, cnir->count); + rb_erase(node, &ra_root); + kfree(cnir); + } + + if (err) { + if (private) { + private->curr_pos = actor->pos; + private->end_in_iterate = true; + } + err = 0; + } + ntfs_index_ctx_put(ictx); + kfree(name); + kfree(nir); + kfree(ra); + mutex_unlock(&ndir->mrec_lock); + return err; +} + +int ntfs_check_empty_dir(struct ntfs_inode *ni, struct mft_record *ni_mrec) +{ + struct ntfs_attr_search_ctx *ctx; + int ret = 0; + + if (!(ni_mrec->flags & MFT_RECORD_IS_DIRECTORY)) + return 0; + + ctx = ntfs_attr_get_search_ctx(ni, NULL); + if (!ctx) { + ntfs_error(ni->vol->sb, "Failed to get search context"); + return -ENOMEM; + } + + /* Find the index root attribute in the mft record. */ + ret = ntfs_attr_lookup(AT_INDEX_ROOT, I30, 4, CASE_SENSITIVE, 0, NULL, + 0, ctx); + if (ret) { + ntfs_error(ni->vol->sb, "Index root attribute missing in directory inode %llu", + ni->mft_no); + ntfs_attr_put_search_ctx(ctx); + return ret; + } + + /* Non-empty directory? */ + if (le32_to_cpu(ctx->attr->data.resident.value_length) != + sizeof(struct index_root) + sizeof(struct index_entry_header)) { + /* Both ENOTEMPTY and EEXIST are ok. We use the more common. */ + ret = -ENOTEMPTY; + ntfs_debug("Directory is not empty\n"); + } + + ntfs_attr_put_search_ctx(ctx); + + return ret; +} + +/* + * ntfs_dir_open - called when an inode is about to be opened + * @vi: inode to be opened + * @filp: file structure describing the inode + * + * Limit directory size to the page cache limit on architectures where unsigned + * long is 32-bits. This is the most we can do for now without overflowing the + * page cache page index. Doing it this way means we don't run into problems + * because of existing too large directories. It would be better to allow the + * user to read the accessible part of the directory but I doubt very much + * anyone is going to hit this check on a 32-bit architecture, so there is no + * point in adding the extra complexity required to support this. + * + * On 64-bit architectures, the check is hopefully optimized away by the + * compiler. + */ +static int ntfs_dir_open(struct inode *vi, struct file *filp) +{ + if (sizeof(unsigned long) < 8) { + if (i_size_read(vi) > MAX_LFS_FILESIZE) + return -EFBIG; + } + return 0; +} + +static int ntfs_dir_release(struct inode *vi, struct file *filp) +{ + if (filp->private_data) { + kfree(((struct ntfs_file_private *)filp->private_data)->key); + kfree(filp->private_data); + filp->private_data = NULL; + } + return 0; +} + +/* + * ntfs_dir_fsync - sync a directory to disk + * @filp: file describing the directory to be synced + * @start: start offset to be synced + * @end: end offset to be synced + * @datasync: if non-zero only flush user data and not metadata + * + * Data integrity sync of a directory to disk. Used for fsync, fdatasync, and + * msync system calls. This function is based on file.c::ntfs_file_fsync(). + * + * Write the mft record and all associated extent mft records as well as the + * $INDEX_ALLOCATION and $BITMAP attributes and then sync the block device. + * + * If @datasync is true, we do not wait on the inode(s) to be written out + * but we always wait on the page cache pages to be written out. + * + * Note: In the past @filp could be NULL so we ignore it as we don't need it + * anyway. + * + * Locking: Caller must hold i_mutex on the inode. + */ +static int ntfs_dir_fsync(struct file *filp, loff_t start, loff_t end, + int datasync) +{ + struct inode *bmp_vi, *vi = filp->f_mapping->host; + struct ntfs_volume *vol = NTFS_I(vi)->vol; + struct ntfs_inode *ni = NTFS_I(vi); + struct ntfs_attr_search_ctx *ctx; + struct inode *parent_vi, *ia_vi; + int err, ret; + struct ntfs_attr na; + + ntfs_debug("Entering for inode 0x%llx.", ni->mft_no); + + if (NVolShutdown(vol)) + return -EIO; + + ctx = ntfs_attr_get_search_ctx(ni, NULL); + if (!ctx) + return -ENOMEM; + + mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL_CHILD); + while (!(err = ntfs_attr_lookup(AT_FILE_NAME, NULL, 0, 0, 0, NULL, 0, ctx))) { + struct file_name_attr *fn = (struct file_name_attr *)((u8 *)ctx->attr + + le16_to_cpu(ctx->attr->data.resident.value_offset)); + + if (MREF_LE(fn->parent_directory) == ni->mft_no) + continue; + + parent_vi = ntfs_iget(vi->i_sb, MREF_LE(fn->parent_directory)); + if (IS_ERR(parent_vi)) + continue; + mutex_lock_nested(&NTFS_I(parent_vi)->mrec_lock, NTFS_INODE_MUTEX_NORMAL); + ia_vi = ntfs_index_iget(parent_vi, I30, 4); + mutex_unlock(&NTFS_I(parent_vi)->mrec_lock); + if (IS_ERR(ia_vi)) { + iput(parent_vi); + continue; + } + write_inode_now(ia_vi, 1); + iput(ia_vi); + write_inode_now(parent_vi, 1); + iput(parent_vi); + } + mutex_unlock(&ni->mrec_lock); + ntfs_attr_put_search_ctx(ctx); + + err = file_write_and_wait_range(filp, start, end); + if (err) + return err; + inode_lock(vi); + + /* If the bitmap attribute inode is in memory sync it, too. */ + na.mft_no = vi->i_ino; + na.type = AT_BITMAP; + na.name = I30; + na.name_len = 4; + bmp_vi = ilookup5(vi->i_sb, vi->i_ino, ntfs_test_inode, &na); + if (bmp_vi) { + write_inode_now(bmp_vi, !datasync); + iput(bmp_vi); + } + ret = __ntfs_write_inode(vi, 1); + + write_inode_now(vi, !datasync); + + write_inode_now(vol->mftbmp_ino, 1); + down_write(&vol->lcnbmp_lock); + write_inode_now(vol->lcnbmp_ino, 1); + up_write(&vol->lcnbmp_lock); + write_inode_now(vol->mft_ino, 1); + + err = sync_blockdev(vi->i_sb->s_bdev); + if (unlikely(err && !ret)) + ret = err; + if (likely(!ret)) + ntfs_debug("Done."); + else + ntfs_warning(vi->i_sb, + "Failed to f%ssync inode 0x%llx. Error %u.", + datasync ? "data" : "", ni->mft_no, -ret); + inode_unlock(vi); + return ret; +} + +const struct file_operations ntfs_dir_ops = { + .llseek = generic_file_llseek, /* Seek inside directory. */ + .read = generic_read_dir, /* Return -EISDIR. */ + .iterate_shared = ntfs_readdir, /* Read directory contents. */ + .fsync = ntfs_dir_fsync, /* Sync a directory to disk. */ + .open = ntfs_dir_open, /* Open directory. */ + .release = ntfs_dir_release, + .unlocked_ioctl = ntfs_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = ntfs_compat_ioctl, +#endif + .setlease = generic_setlease, +}; diff --git a/fs/ntfs/dir.h b/fs/ntfs/dir.h new file mode 100644 index 000000000..a38c6b071 --- /dev/null +++ b/fs/ntfs/dir.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Defines for directory handling in NTFS Linux kernel driver. + * + * Copyright (c) 2002-2004 Anton Altaparmakov + */ + +#ifndef _LINUX_NTFS_DIR_H +#define _LINUX_NTFS_DIR_H + +#include "inode.h" + +/* + * ntfs_name is used to return the file name to the caller of + * ntfs_lookup_inode_by_name() in order for the caller (namei.c::ntfs_lookup()) + * to be able to deal with dcache aliasing issues. + */ +struct ntfs_name { + u64 mref; + u8 type; + u8 len; + __le16 name[]; +} __packed; + +/* The little endian Unicode string $I30 as a global constant. */ +extern __le16 I30[5]; + +u64 ntfs_lookup_inode_by_name(struct ntfs_inode *dir_ni, + const __le16 *uname, const int uname_len, struct ntfs_name **res); +int ntfs_check_empty_dir(struct ntfs_inode *ni, struct mft_record *ni_mrec); + +#endif /* _LINUX_NTFS_FS_DIR_H */ diff --git a/fs/ntfs/ea.c b/fs/ntfs/ea.c new file mode 100644 index 000000000..0cd192752 --- /dev/null +++ b/fs/ntfs/ea.c @@ -0,0 +1,957 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Pocessing of EA's + * + * Part of this file is based on code from the NTFS-3G. + * + * Copyright (c) 2014-2021 Jean-Pierre Andre + * Copyright (c) 2025 LG Electronics Co., Ltd. + */ + +#include +#include +#include +#include + +#include "layout.h" +#include "attrib.h" +#include "index.h" +#include "dir.h" +#include "ea.h" + +static int ntfs_write_ea(struct ntfs_inode *ni, __le32 type, char *value, s64 ea_off, + s64 ea_size, bool need_truncate) +{ + struct inode *ea_vi; + int err = 0; + s64 written; + + ea_vi = ntfs_attr_iget(VFS_I(ni), type, AT_UNNAMED, 0); + if (IS_ERR(ea_vi)) + return PTR_ERR(ea_vi); + + written = ntfs_inode_attr_pwrite(ea_vi, ea_off, ea_size, value, false); + if (written != ea_size) + err = -EIO; + else { + struct ntfs_inode *ea_ni = NTFS_I(ea_vi); + + if (need_truncate && ea_ni->data_size > ea_off + ea_size) + ntfs_attr_truncate(ea_ni, ea_off + ea_size); + mark_mft_record_dirty(ni); + } + + iput(ea_vi); + return err; +} + +static int ntfs_ea_lookup(char *ea_buf, s64 ea_buf_size, const char *name, + int name_len, s64 *ea_offset, s64 *ea_size) +{ + const struct ea_attr *p_ea; + size_t actual_size; + loff_t offset, p_ea_size; + unsigned int next; + + offset = 0; + do { + if (ea_buf_size - offset < sizeof(struct ea_attr)) + break; + + p_ea = (const struct ea_attr *)&ea_buf[offset]; + next = le32_to_cpu(p_ea->next_entry_offset); + p_ea_size = next ? next : (ea_buf_size - offset); + + if (p_ea_size < sizeof(struct ea_attr) || + offset + p_ea_size > ea_buf_size) + break; + + if ((s64)p_ea->ea_name_length + 1 > + p_ea_size - offsetof(struct ea_attr, ea_name)) + break; + + actual_size = ALIGN(struct_size(p_ea, ea_name, 1 + p_ea->ea_name_length + + le16_to_cpu(p_ea->ea_value_length)), 4); + if (actual_size > p_ea_size) + break; + + if (p_ea->ea_name_length == name_len && + !memcmp(p_ea->ea_name, name, name_len)) { + *ea_offset = offset; + *ea_size = next ? next : actual_size; + + if (ea_buf_size < *ea_offset + *ea_size) + goto out; + + return 0; + } + offset += next; + } while (next > 0 && offset < ea_buf_size); + +out: + return -ENOENT; +} + +/* + * Return the existing EA + * + * The EA_INFORMATION is not examined and the consistency of the + * existing EA is not checked. + * + * If successful, the full attribute is returned unchanged + * and its size is returned. + * If the designated buffer is too small, the needed size is + * returned, and the buffer is left unchanged. + * If there is an error, a negative value is returned and errno + * is set according to the error. + */ +static int ntfs_get_ea(struct inode *inode, const char *name, size_t name_len, + void *buffer, size_t size) +{ + struct ntfs_inode *ni = NTFS_I(inode); + const struct ea_attr *p_ea; + char *ea_buf; + s64 ea_off, ea_size, all_ea_size, ea_info_size; + int err; + u32 ea_info_qlen; + u16 ea_value_len; + struct ea_information *p_ea_info; + + if (!NInoHasEA(ni)) + return -ENODATA; + + p_ea_info = ntfs_attr_readall(ni, AT_EA_INFORMATION, NULL, 0, + &ea_info_size); + if (!p_ea_info || ea_info_size != sizeof(struct ea_information)) { + kvfree(p_ea_info); + return -ENODATA; + } + + ea_info_qlen = le32_to_cpu(p_ea_info->ea_query_length); + kvfree(p_ea_info); + + ea_buf = ntfs_attr_readall(ni, AT_EA, NULL, 0, &all_ea_size); + if (!ea_buf) + return -ENODATA; + + if (ea_info_qlen > all_ea_size) { + err = -EIO; + goto free_ea_buf; + } + + err = ntfs_ea_lookup(ea_buf, ea_info_qlen, name, name_len, &ea_off, + &ea_size); + if (!err) { + p_ea = (struct ea_attr *)&ea_buf[ea_off]; + ea_value_len = le16_to_cpu(p_ea->ea_value_length); + if (!buffer) { + kvfree(ea_buf); + return ea_value_len; + } + + if (ea_value_len > size) { + err = -ERANGE; + goto free_ea_buf; + } + + memcpy(buffer, &p_ea->ea_name[p_ea->ea_name_length + 1], + ea_value_len); + kvfree(ea_buf); + return ea_value_len; + } + + err = -ENODATA; +free_ea_buf: + kvfree(ea_buf); + return err; +} + +static inline int ea_packed_size(const struct ea_attr *p_ea) +{ + /* + * 4 bytes for header (flags and lengths) + name length + 1 + + * value length. + */ + return 5 + p_ea->ea_name_length + le16_to_cpu(p_ea->ea_value_length); +} + +/* + * Set a new EA, and set EA_INFORMATION accordingly + * + * This is roughly the same as ZwSetEaFile() on Windows, however + * the "offset to next" of the last EA should not be cleared. + * + * Consistency of the new EA is first checked. + * + * EA_INFORMATION is set first, and it is restored to its former + * state if setting EA fails. + */ +static int ntfs_set_ea(struct inode *inode, const char *name, size_t name_len, + const void *value, size_t val_size, int flags, + __le16 *packed_ea_size) +{ + struct ntfs_inode *ni = NTFS_I(inode); + struct ea_information *p_ea_info = NULL; + int ea_packed, err = 0; + struct ea_attr *p_ea; + u32 ea_info_qsize = 0; + char *ea_buf = NULL; + size_t new_ea_size = ALIGN(struct_size(p_ea, ea_name, 1 + name_len + val_size), 4); + s64 ea_off, ea_info_size, all_ea_size, ea_size; + + if (name_len > 255) + return -ENAMETOOLONG; + + if (ntfs_attr_exist(ni, AT_EA_INFORMATION, AT_UNNAMED, 0)) { + p_ea_info = ntfs_attr_readall(ni, AT_EA_INFORMATION, NULL, 0, + &ea_info_size); + if (!p_ea_info || ea_info_size != sizeof(struct ea_information)) + goto out; + + ea_buf = ntfs_attr_readall(ni, AT_EA, NULL, 0, &all_ea_size); + if (!ea_buf) { + ea_info_qsize = 0; + kvfree(p_ea_info); + goto create_ea_info; + } + + ea_info_qsize = le32_to_cpu(p_ea_info->ea_query_length); + } else { +create_ea_info: + p_ea_info = kzalloc(sizeof(struct ea_information), GFP_NOFS); + if (!p_ea_info) + return -ENOMEM; + + ea_info_qsize = 0; + err = ntfs_attr_add(ni, AT_EA_INFORMATION, AT_UNNAMED, 0, + (char *)p_ea_info, sizeof(struct ea_information)); + if (err) + goto out; + + if (ntfs_attr_exist(ni, AT_EA, AT_UNNAMED, 0)) { + err = ntfs_attr_remove(ni, AT_EA, AT_UNNAMED, 0); + if (err) + goto out; + } + + goto alloc_new_ea; + } + + if (ea_info_qsize > all_ea_size) { + err = -EIO; + goto out; + } + + err = ntfs_ea_lookup(ea_buf, ea_info_qsize, name, name_len, &ea_off, + &ea_size); + if (ea_info_qsize && !err) { + if (flags & XATTR_CREATE) { + err = -EEXIST; + goto out; + } + + p_ea = (struct ea_attr *)(ea_buf + ea_off); + + if (val_size && + le16_to_cpu(p_ea->ea_value_length) == val_size && + !memcmp(p_ea->ea_name + p_ea->ea_name_length + 1, value, + val_size)) + goto out; + + le16_add_cpu(&p_ea_info->ea_length, 0 - ea_packed_size(p_ea)); + + if (p_ea->flags & NEED_EA) + le16_add_cpu(&p_ea_info->need_ea_count, -1); + + memmove((char *)p_ea, (char *)p_ea + ea_size, ea_info_qsize - (ea_off + ea_size)); + ea_info_qsize -= ea_size; + p_ea_info->ea_query_length = cpu_to_le32(ea_info_qsize); + + err = ntfs_write_ea(ni, AT_EA_INFORMATION, (char *)p_ea_info, 0, + sizeof(struct ea_information), false); + if (err) + goto out; + + err = ntfs_write_ea(ni, AT_EA, ea_buf, 0, ea_info_qsize, true); + if (err) + goto out; + + if ((flags & XATTR_REPLACE) && !val_size) { + /* Remove xattr. */ + goto out; + } + } else { + if (flags & XATTR_REPLACE) { + err = -ENODATA; + goto out; + } + } + kvfree(ea_buf); + +alloc_new_ea: + ea_buf = kzalloc(new_ea_size, GFP_NOFS); + if (!ea_buf) { + err = -ENOMEM; + goto out; + } + + /* + * EA and REPARSE_POINT compatibility not checked any more, + * required by Windows 10, but having both may lead to + * problems with earlier versions. + */ + p_ea = (struct ea_attr *)ea_buf; + memcpy(p_ea->ea_name, name, name_len); + p_ea->ea_name_length = name_len; + p_ea->ea_name[name_len] = 0; + memcpy(p_ea->ea_name + name_len + 1, value, val_size); + p_ea->ea_value_length = cpu_to_le16(val_size); + p_ea->next_entry_offset = cpu_to_le32(new_ea_size); + + ea_packed = le16_to_cpu(p_ea_info->ea_length) + ea_packed_size(p_ea); + p_ea_info->ea_length = cpu_to_le16(ea_packed); + p_ea_info->ea_query_length = cpu_to_le32(ea_info_qsize + new_ea_size); + + if (ea_packed > 0xffff || + ntfs_attr_size_bounds_check(ni->vol, AT_EA, new_ea_size)) { + err = -EFBIG; + goto out; + } + + /* + * no EA or EA_INFORMATION : add them + */ + if (!ntfs_attr_exist(ni, AT_EA, AT_UNNAMED, 0)) { + err = ntfs_attr_add(ni, AT_EA, AT_UNNAMED, 0, (char *)p_ea, + new_ea_size); + if (err) + goto out; + } else { + err = ntfs_write_ea(ni, AT_EA, (char *)p_ea, ea_info_qsize, + new_ea_size, false); + if (err) + goto out; + } + + err = ntfs_write_ea(ni, AT_EA_INFORMATION, (char *)p_ea_info, 0, + sizeof(struct ea_information), false); + if (err) + goto out; + + if (packed_ea_size) + *packed_ea_size = p_ea_info->ea_length; + mark_mft_record_dirty(ni); +out: + if (ea_info_qsize > 0) + NInoSetHasEA(ni); + else + NInoClearHasEA(ni); + + kvfree(ea_buf); + kvfree(p_ea_info); + + return err; +} + +/* + * Check for the presence of an EA "$LXDEV" (used by WSL) + * and return its value as a device address + */ +int ntfs_ea_get_wsl_inode(struct inode *inode, dev_t *rdevp, unsigned int flags) +{ + int err; + __le32 v; + + if (!(flags & NTFS_VOL_UID)) { + /* Load uid to lxuid EA */ + err = ntfs_get_ea(inode, "$LXUID", sizeof("$LXUID") - 1, &v, + sizeof(v)); + if (err < 0) + return err; + if (err != sizeof(v)) + return -EIO; + i_uid_write(inode, le32_to_cpu(v)); + } + + if (!(flags & NTFS_VOL_GID)) { + /* Load gid to lxgid EA */ + err = ntfs_get_ea(inode, "$LXGID", sizeof("$LXGID") - 1, &v, + sizeof(v)); + if (err < 0) + return err; + if (err != sizeof(v)) + return -EIO; + i_gid_write(inode, le32_to_cpu(v)); + } + + /* Load mode to lxmod EA */ + err = ntfs_get_ea(inode, "$LXMOD", sizeof("$LXMOD") - 1, &v, sizeof(v)); + if (err == sizeof(v)) { + inode->i_mode = le32_to_cpu(v); + } else { + /* Everyone gets all permissions. */ + inode->i_mode |= 0777; + } + + /* Load mode to lxdev EA */ + err = ntfs_get_ea(inode, "$LXDEV", sizeof("$LXDEV") - 1, &v, sizeof(v)); + if (err == sizeof(v)) + *rdevp = le32_to_cpu(v); + err = 0; + + return err; +} + +int ntfs_ea_set_wsl_inode(struct inode *inode, dev_t rdev, __le16 *ea_size, + unsigned int flags) +{ + __le32 v; + int err = 0; + + if (ea_size) + *ea_size = 0; + + if (flags & NTFS_EA_UID) { + /* Store uid to lxuid EA */ + v = cpu_to_le32(i_uid_read(inode)); + err = ntfs_set_ea(inode, "$LXUID", sizeof("$LXUID") - 1, &v, + sizeof(v), 0, ea_size); + if (err) + return err; + } + + if (flags & NTFS_EA_GID) { + /* Store gid to lxgid EA */ + v = cpu_to_le32(i_gid_read(inode)); + err = ntfs_set_ea(inode, "$LXGID", sizeof("$LXGID") - 1, &v, + sizeof(v), 0, ea_size); + if (err) + return err; + } + + if (flags & NTFS_EA_MODE) { + /* Store mode to lxmod EA */ + v = cpu_to_le32(inode->i_mode); + err = ntfs_set_ea(inode, "$LXMOD", sizeof("$LXMOD") - 1, &v, + sizeof(v), 0, ea_size); + if (err) + return err; + } + + if (rdev) { + v = cpu_to_le32(rdev); + err = ntfs_set_ea(inode, "$LXDEV", sizeof("$LXDEV") - 1, &v, sizeof(v), + 0, ea_size); + } + + return err; +} + +ssize_t ntfs_listxattr(struct dentry *dentry, char *buffer, size_t size) +{ + struct inode *inode = d_inode(dentry); + struct ntfs_inode *ni = NTFS_I(inode); + const struct ea_attr *p_ea; + s64 offset, ea_buf_size, ea_info_size; + s64 ea_size; + u32 next; + int err = 0; + u32 ea_info_qsize; + char *ea_buf = NULL; + ssize_t ret = 0; + struct ea_information *ea_info; + + if (!NInoHasEA(ni)) + return 0; + + mutex_lock(&NTFS_I(inode)->mrec_lock); + ea_info = ntfs_attr_readall(ni, AT_EA_INFORMATION, NULL, 0, + &ea_info_size); + if (!ea_info || ea_info_size != sizeof(struct ea_information)) + goto out; + + ea_info_qsize = le32_to_cpu(ea_info->ea_query_length); + + ea_buf = ntfs_attr_readall(ni, AT_EA, NULL, 0, &ea_buf_size); + if (!ea_buf) + goto out; + + if (ea_info_qsize > ea_buf_size || ea_info_qsize == 0) + goto out; + + offset = 0; + do { + if (ea_info_qsize - offset < sizeof(struct ea_attr)) { + err = -EIO; + goto out; + } + + p_ea = (const struct ea_attr *)&ea_buf[offset]; + next = le32_to_cpu(p_ea->next_entry_offset); + ea_size = next ? next : (ea_info_qsize - offset); + + if (ea_size < sizeof(struct ea_attr) || + offset + ea_size > ea_info_qsize) { + err = -EIO; + goto out; + } + + if ((int)p_ea->ea_name_length + 1 > + ea_size - offsetof(struct ea_attr, ea_name)) { + err = -EIO; + goto out; + } + + if (buffer) { + if (ret + p_ea->ea_name_length + 1 > size) { + err = -ERANGE; + goto out; + } + + memcpy(buffer + ret, p_ea->ea_name, p_ea->ea_name_length); + buffer[ret + p_ea->ea_name_length] = 0; + } + + ret += p_ea->ea_name_length + 1; + offset += ea_size; + } while (next > 0 && offset < ea_info_qsize); + +out: + mutex_unlock(&NTFS_I(inode)->mrec_lock); + kvfree(ea_info); + kvfree(ea_buf); + + return err ? err : ret; +} + +// clang-format off +#define SYSTEM_DOS_ATTRIB "system.dos_attrib" +#define SYSTEM_NTFS_ATTRIB "system.ntfs_attrib" +#define SYSTEM_NTFS_ATTRIB_BE "system.ntfs_attrib_be" +// clang-format on + +static int ntfs_getxattr(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, const char *name, + void *buffer, size_t size) +{ + struct ntfs_inode *ni = NTFS_I(inode); + int err; + + if (NVolShutdown(ni->vol)) + return -EIO; + + if (!strcmp(name, SYSTEM_DOS_ATTRIB)) { + if (!buffer) { + err = sizeof(u8); + } else if (size < sizeof(u8)) { + err = -ENODATA; + } else { + err = sizeof(u8); + *(u8 *)buffer = (u8)(le32_to_cpu(ni->flags) & 0x3F); + } + goto out; + } + + if (!strcmp(name, SYSTEM_NTFS_ATTRIB) || + !strcmp(name, SYSTEM_NTFS_ATTRIB_BE)) { + if (!buffer) { + err = sizeof(u32); + } else if (size < sizeof(u32)) { + err = -ENODATA; + } else { + err = sizeof(u32); + *(u32 *)buffer = le32_to_cpu(ni->flags); + if (!strcmp(name, SYSTEM_NTFS_ATTRIB_BE)) + *(__be32 *)buffer = cpu_to_be32(*(u32 *)buffer); + } + goto out; + } + + mutex_lock(&ni->mrec_lock); + err = ntfs_get_ea(inode, name, strlen(name), buffer, size); + mutex_unlock(&ni->mrec_lock); + +out: + return err; +} + +static int ntfs_new_attr_flags(struct ntfs_inode *ni, __le32 fattr) +{ + struct ntfs_attr_search_ctx *ctx; + struct mft_record *m; + struct attr_record *a; + __le16 new_aflags; + int mp_size, mp_ofs, name_ofs, arec_size, err; + + m = map_mft_record(ni); + if (IS_ERR(m)) + return PTR_ERR(m); + + ctx = ntfs_attr_get_search_ctx(ni, m); + if (!ctx) { + err = -ENOMEM; + goto err_out; + } + + err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len, + CASE_SENSITIVE, 0, NULL, 0, ctx); + if (err) { + err = -EINVAL; + goto err_out; + } + + a = ctx->attr; + new_aflags = ctx->attr->flags; + + if (fattr & FILE_ATTR_SPARSE_FILE) + new_aflags |= ATTR_IS_SPARSE; + else + new_aflags &= ~ATTR_IS_SPARSE; + + if (fattr & FILE_ATTR_COMPRESSED) + new_aflags |= ATTR_IS_COMPRESSED; + else + new_aflags &= ~ATTR_IS_COMPRESSED; + + if (new_aflags == a->flags) + return 0; + + if ((new_aflags & (ATTR_IS_SPARSE | ATTR_IS_COMPRESSED)) == + (ATTR_IS_SPARSE | ATTR_IS_COMPRESSED)) { + pr_err("file can't be sparsed and compressed\n"); + err = -EOPNOTSUPP; + goto err_out; + } + + if (!a->non_resident) + goto out; + + if (a->data.non_resident.data_size) { + pr_err("Can't change sparsed/compressed for non-empty file\n"); + err = -EOPNOTSUPP; + goto err_out; + } + + if (new_aflags & (ATTR_IS_SPARSE | ATTR_IS_COMPRESSED)) + name_ofs = (offsetof(struct attr_record, + data.non_resident.compressed_size) + + sizeof(a->data.non_resident.compressed_size) + 7) & ~7; + else + name_ofs = (offsetof(struct attr_record, + data.non_resident.compressed_size) + 7) & ~7; + + mp_size = ntfs_get_size_for_mapping_pairs(ni->vol, ni->runlist.rl, 0, -1, -1); + if (unlikely(mp_size < 0)) { + err = mp_size; + ntfs_debug("Failed to get size for mapping pairs array, error code %i.\n", err); + goto err_out; + } + + mp_ofs = (name_ofs + a->name_length * sizeof(__le16) + 7) & ~7; + arec_size = (mp_ofs + mp_size + 7) & ~7; + + err = ntfs_attr_record_resize(m, a, arec_size); + if (unlikely(err)) + goto err_out; + + if (new_aflags & (ATTR_IS_SPARSE | ATTR_IS_COMPRESSED)) { + a->data.non_resident.compression_unit = 0; + if (new_aflags & ATTR_IS_COMPRESSED || ni->vol->major_ver < 3) + a->data.non_resident.compression_unit = 4; + a->data.non_resident.compressed_size = 0; + ni->itype.compressed.size = 0; + if (a->data.non_resident.compression_unit) { + ni->itype.compressed.block_size = 1U << + (a->data.non_resident.compression_unit + + ni->vol->cluster_size_bits); + ni->itype.compressed.block_size_bits = + ffs(ni->itype.compressed.block_size) - + 1; + ni->itype.compressed.block_clusters = 1U << + a->data.non_resident.compression_unit; + } else { + ni->itype.compressed.block_size = 0; + ni->itype.compressed.block_size_bits = 0; + ni->itype.compressed.block_clusters = 0; + } + + if (new_aflags & ATTR_IS_SPARSE) { + NInoSetSparse(ni); + ni->flags |= FILE_ATTR_SPARSE_FILE; + } + + if (new_aflags & ATTR_IS_COMPRESSED) { + NInoSetCompressed(ni); + ni->flags |= FILE_ATTR_COMPRESSED; + } + } else { + ni->flags &= ~(FILE_ATTR_SPARSE_FILE | FILE_ATTR_COMPRESSED); + a->data.non_resident.compression_unit = 0; + NInoClearSparse(ni); + NInoClearCompressed(ni); + } + + a->name_offset = cpu_to_le16(name_ofs); + a->data.non_resident.mapping_pairs_offset = cpu_to_le16(mp_ofs); + +out: + a->flags = new_aflags; + mark_mft_record_dirty(ctx->ntfs_ino); +err_out: + if (ctx) + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(ni); + return err; +} + +static int ntfs_setxattr(const struct xattr_handler *handler, + struct mnt_idmap *idmap, struct dentry *unused, + struct inode *inode, const char *name, const void *value, + size_t size, int flags) +{ + struct ntfs_inode *ni = NTFS_I(inode); + int err; + __le32 fattr; + + if (NVolShutdown(ni->vol)) + return -EIO; + + if (!strcmp(name, SYSTEM_DOS_ATTRIB)) { + if (sizeof(u8) != size) { + err = -EINVAL; + goto out; + } + fattr = cpu_to_le32(*(u8 *)value); + goto set_fattr; + } + + if (!strcmp(name, SYSTEM_NTFS_ATTRIB) || + !strcmp(name, SYSTEM_NTFS_ATTRIB_BE)) { + if (size != sizeof(u32)) { + err = -EINVAL; + goto out; + } + if (!strcmp(name, SYSTEM_NTFS_ATTRIB_BE)) + fattr = cpu_to_le32(be32_to_cpu(*(__be32 *)value)); + else + fattr = cpu_to_le32(*(u32 *)value); + + if (S_ISREG(inode->i_mode)) { + mutex_lock(&ni->mrec_lock); + err = ntfs_new_attr_flags(ni, fattr); + mutex_unlock(&ni->mrec_lock); + if (err) + goto out; + } + +set_fattr: + if (S_ISDIR(inode->i_mode)) + fattr |= FILE_ATTR_DIRECTORY; + else + fattr &= ~FILE_ATTR_DIRECTORY; + + if (ni->flags != fattr) { + ni->flags = fattr; + if (fattr & FILE_ATTR_READONLY) + inode->i_mode &= ~0222; + else + inode->i_mode |= 0222; + NInoSetFileNameDirty(ni); + mark_inode_dirty(inode); + } + err = 0; + goto out; + } + + mutex_lock(&ni->mrec_lock); + err = ntfs_set_ea(inode, name, strlen(name), value, size, flags, NULL); + mutex_unlock(&ni->mrec_lock); + +out: + inode_set_ctime_current(inode); + mark_inode_dirty(inode); + return err; +} + +static bool ntfs_xattr_user_list(struct dentry *dentry) +{ + return true; +} + +// clang-format off +static const struct xattr_handler ntfs_other_xattr_handler = { + .prefix = "", + .get = ntfs_getxattr, + .set = ntfs_setxattr, + .list = ntfs_xattr_user_list, +}; + +const struct xattr_handler * const ntfs_xattr_handlers[] = { + &ntfs_other_xattr_handler, + NULL, +}; +// clang-format on + +#ifdef CONFIG_NTFS_FS_POSIX_ACL +struct posix_acl *ntfs_get_acl(struct mnt_idmap *idmap, struct dentry *dentry, + int type) +{ + struct inode *inode = d_inode(dentry); + struct ntfs_inode *ni = NTFS_I(inode); + const char *name; + size_t name_len; + struct posix_acl *acl; + int err; + void *buf; + + buf = kmalloc(PATH_MAX, GFP_KERNEL); + if (!buf) + return ERR_PTR(-ENOMEM); + + /* Possible values of 'type' was already checked above. */ + if (type == ACL_TYPE_ACCESS) { + name = XATTR_NAME_POSIX_ACL_ACCESS; + name_len = sizeof(XATTR_NAME_POSIX_ACL_ACCESS) - 1; + } else { + name = XATTR_NAME_POSIX_ACL_DEFAULT; + name_len = sizeof(XATTR_NAME_POSIX_ACL_DEFAULT) - 1; + } + + mutex_lock(&ni->mrec_lock); + err = ntfs_get_ea(inode, name, name_len, buf, PATH_MAX); + mutex_unlock(&ni->mrec_lock); + + /* Translate extended attribute to acl. */ + if (err >= 0) + acl = posix_acl_from_xattr(&init_user_ns, buf, err); + else if (err == -ENODATA) + acl = NULL; + else + acl = ERR_PTR(err); + + if (!IS_ERR(acl)) + set_cached_acl(inode, type, acl); + + kfree(buf); + + return acl; +} + +static noinline int ntfs_set_acl_ex(struct mnt_idmap *idmap, + struct inode *inode, struct posix_acl *acl, + int type, bool init_acl) +{ + const char *name; + size_t size, name_len; + void *value; + int err; + int flags; + umode_t mode; + + if (S_ISLNK(inode->i_mode)) + return -EOPNOTSUPP; + + mode = inode->i_mode; + switch (type) { + case ACL_TYPE_ACCESS: + /* Do not change i_mode if we are in init_acl */ + if (acl && !init_acl) { + err = posix_acl_update_mode(idmap, inode, &mode, &acl); + if (err) + return err; + } + name = XATTR_NAME_POSIX_ACL_ACCESS; + name_len = sizeof(XATTR_NAME_POSIX_ACL_ACCESS) - 1; + break; + + case ACL_TYPE_DEFAULT: + if (!S_ISDIR(inode->i_mode)) + return acl ? -EACCES : 0; + name = XATTR_NAME_POSIX_ACL_DEFAULT; + name_len = sizeof(XATTR_NAME_POSIX_ACL_DEFAULT) - 1; + break; + + default: + return -EINVAL; + } + + if (!acl) { + /* Remove xattr if it can be presented via mode. */ + size = 0; + value = NULL; + flags = XATTR_REPLACE; + } else { + value = posix_acl_to_xattr(&init_user_ns, acl, &size, GFP_NOFS); + if (!value) + return -ENOMEM; + flags = 0; + } + + mutex_lock(&NTFS_I(inode)->mrec_lock); + err = ntfs_set_ea(inode, name, name_len, value, size, flags, NULL); + mutex_unlock(&NTFS_I(inode)->mrec_lock); + if (err == -ENODATA && !size) + err = 0; /* Removing non existed xattr. */ + if (!err) { + __le16 ea_size = 0; + umode_t old_mode = inode->i_mode; + + inode->i_mode = mode; + mutex_lock(&NTFS_I(inode)->mrec_lock); + err = ntfs_ea_set_wsl_inode(inode, 0, &ea_size, NTFS_EA_MODE); + if (err) { + ntfs_set_ea(inode, name, name_len, NULL, 0, + XATTR_REPLACE, NULL); + mutex_unlock(&NTFS_I(inode)->mrec_lock); + inode->i_mode = old_mode; + goto out; + } + mutex_unlock(&NTFS_I(inode)->mrec_lock); + + set_cached_acl(inode, type, acl); + inode_set_ctime_current(inode); + mark_inode_dirty(inode); + } + +out: + kfree(value); + + return err; +} + +int ntfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type) +{ + return ntfs_set_acl_ex(idmap, d_inode(dentry), acl, type, false); +} + +int ntfs_init_acl(struct mnt_idmap *idmap, struct inode *inode, + struct inode *dir) +{ + struct posix_acl *default_acl, *acl; + int err; + + err = posix_acl_create(dir, &inode->i_mode, &default_acl, &acl); + if (err) + return err; + + if (default_acl) { + err = ntfs_set_acl_ex(idmap, inode, default_acl, + ACL_TYPE_DEFAULT, true); + posix_acl_release(default_acl); + } else { + inode->i_default_acl = NULL; + } + + if (acl) { + if (!err) + err = ntfs_set_acl_ex(idmap, inode, acl, + ACL_TYPE_ACCESS, true); + posix_acl_release(acl); + } else { + inode->i_acl = NULL; + } + + return err; +} +#endif diff --git a/fs/ntfs/ea.h b/fs/ntfs/ea.h new file mode 100644 index 000000000..1f63bd55e --- /dev/null +++ b/fs/ntfs/ea.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#ifndef _LINUX_NTFS_EA_H +#define _LINUX_NTFS_EA_H + +#define NTFS_EA_UID BIT(1) +#define NTFS_EA_GID BIT(2) +#define NTFS_EA_MODE BIT(3) + +extern const struct xattr_handler *const ntfs_xattr_handlers[]; + +int ntfs_ea_set_wsl_not_symlink(struct ntfs_inode *ni, mode_t mode, dev_t dev); +int ntfs_ea_get_wsl_inode(struct inode *inode, dev_t *rdevp, unsigned int flags); +int ntfs_ea_set_wsl_inode(struct inode *inode, dev_t rdev, __le16 *ea_size, + unsigned int flags); +ssize_t ntfs_listxattr(struct dentry *dentry, char *buffer, size_t size); + +#ifdef CONFIG_NTFS_FS_POSIX_ACL +struct posix_acl *ntfs_get_acl(struct mnt_idmap *idmap, struct dentry *dentry, + int type); +int ntfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type); +int ntfs_init_acl(struct mnt_idmap *idmap, struct inode *inode, + struct inode *dir); +#else +#define ntfs_get_acl NULL +#define ntfs_set_acl NULL +#endif + +#endif /* _LINUX_NTFS_EA_H */ diff --git a/fs/ntfs/file.c b/fs/ntfs/file.c new file mode 100644 index 000000000..6a7b638e5 --- /dev/null +++ b/fs/ntfs/file.c @@ -0,0 +1,1189 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * NTFS kernel file operations. + * + * Copyright (c) 2001-2015 Anton Altaparmakov and Tuxera Inc. + * Copyright (c) 2025 LG Electronics Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "lcnalloc.h" +#include "ntfs.h" +#include "reparse.h" +#include "ea.h" +#include "iomap.h" +#include "bitmap.h" +#include "volume.h" + +#include + +/* + * ntfs_file_open - called when an inode is about to be opened + * @vi: inode to be opened + * @filp: file structure describing the inode + * + * Limit file size to the page cache limit on architectures where unsigned long + * is 32-bits. This is the most we can do for now without overflowing the page + * cache page index. Doing it this way means we don't run into problems because + * of existing too large files. It would be better to allow the user to read + * the beginning of the file but I doubt very much anyone is going to hit this + * check on a 32-bit architecture, so there is no point in adding the extra + * complexity required to support this. + * + * On 64-bit architectures, the check is hopefully optimized away by the + * compiler. + * + * After the check passes, just call generic_file_open() to do its work. + */ +static int ntfs_file_open(struct inode *vi, struct file *filp) +{ + struct ntfs_inode *ni = NTFS_I(vi); + + if (NVolShutdown(ni->vol)) + return -EIO; + + if (sizeof(unsigned long) < 8) { + if (i_size_read(vi) > MAX_LFS_FILESIZE) + return -EOVERFLOW; + } + + filp->f_mode |= FMODE_NOWAIT | FMODE_CAN_ODIRECT; + + return generic_file_open(vi, filp); +} + +/* + * Trim preallocated space on file release. + * + * When the preallo_size mount option is set (default 64KB), writes extend + * allocated_size and runlist in units of preallocated size to reduce + * runlist merge overhead for small writes. This can leave + * allocated_size > data_size if not all preallocated space is used. + * + * We perform the trim here because ->release() is called only when + * the file is no longer open. At this point, no further writes can occur, + * so it is safe to reclaim the unused preallocated space. + * + * Returns 0 on success, or negative error on failure. + */ +static int ntfs_trim_prealloc(struct inode *vi) +{ + struct ntfs_inode *ni = NTFS_I(vi); + struct ntfs_volume *vol = ni->vol; + struct runlist_element *rl; + s64 aligned_data_size; + s64 vcn_ds, vcn_tr; + ssize_t rc; + int err = 0; + + inode_lock(vi); + mutex_lock(&ni->mrec_lock); + down_write(&ni->runlist.lock); + + aligned_data_size = round_up(ni->data_size, vol->cluster_size); + if (aligned_data_size >= ni->allocated_size) + goto out_unlock; + + vcn_ds = ntfs_bytes_to_cluster(vol, aligned_data_size); + vcn_tr = -1; + rc = ni->runlist.count - 2; + rl = ni->runlist.rl; + + while (rc >= 0 && rl[rc].lcn == LCN_HOLE && vcn_ds <= rl[rc].vcn) { + vcn_tr = rl[rc].vcn; + rc--; + } + + if (vcn_tr >= 0) { + err = ntfs_rl_truncate_nolock(vol, &ni->runlist, vcn_tr); + if (err) { + kvfree(ni->runlist.rl); + ni->runlist.rl = NULL; + ntfs_error(vol->sb, "Preallocated block rollback failed"); + } else { + ni->allocated_size = ntfs_cluster_to_bytes(vol, vcn_tr); + err = ntfs_attr_update_mapping_pairs(ni, 0); + if (err) + ntfs_error(vol->sb, + "Failed to rollback mapping pairs for prealloc"); + } + } + +out_unlock: + up_write(&ni->runlist.lock); + mutex_unlock(&ni->mrec_lock); + inode_unlock(vi); + + return err; +} + +static int ntfs_file_release(struct inode *vi, struct file *filp) +{ + if (!NInoCompressed(NTFS_I(vi))) + return ntfs_trim_prealloc(vi); + + return 0; +} + +/* + * ntfs_file_fsync - sync a file to disk + * @filp: file to be synced + * @start: start offset to be synced + * @end: end offset to be synced + * @datasync: if non-zero only flush user data and not metadata + * + * Data integrity sync of a file to disk. Used for fsync, fdatasync, and msync + * system calls. This function is inspired by fs/buffer.c::file_fsync(). + * + * If @datasync is false, write the mft record and all associated extent mft + * records as well as the $DATA attribute and then sync the block device. + * + * If @datasync is true and the attribute is non-resident, we skip the writing + * of the mft record and all associated extent mft records (this might still + * happen due to the write_inode_now() call). + * + * Also, if @datasync is true, we do not wait on the inode to be written out + * but we always wait on the page cache pages to be written out. + */ +static int ntfs_file_fsync(struct file *filp, loff_t start, loff_t end, + int datasync) +{ + struct inode *vi = filp->f_mapping->host; + struct ntfs_inode *ni = NTFS_I(vi); + struct ntfs_volume *vol = ni->vol; + int err, ret = 0; + struct inode *parent_vi, *ia_vi; + struct ntfs_attr_search_ctx *ctx; + + ntfs_debug("Entering for inode 0x%llx.", ni->mft_no); + + if (NVolShutdown(vol)) + return -EIO; + + err = file_write_and_wait_range(filp, start, end); + if (err) + return err; + + if (!datasync || !NInoNonResident(NTFS_I(vi))) + ret = __ntfs_write_inode(vi, 1); + write_inode_now(vi, !datasync); + + ctx = ntfs_attr_get_search_ctx(ni, NULL); + if (!ctx) + return -ENOMEM; + + mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL_CHILD); + while (!(err = ntfs_attr_lookup(AT_UNUSED, NULL, 0, 0, 0, NULL, 0, ctx))) { + if (ctx->attr->type == AT_FILE_NAME) { + struct file_name_attr *fn = (struct file_name_attr *)((u8 *)ctx->attr + + le16_to_cpu(ctx->attr->data.resident.value_offset)); + + parent_vi = ntfs_iget(vi->i_sb, MREF_LE(fn->parent_directory)); + if (IS_ERR(parent_vi)) + continue; + mutex_lock_nested(&NTFS_I(parent_vi)->mrec_lock, NTFS_INODE_MUTEX_NORMAL); + ia_vi = ntfs_index_iget(parent_vi, I30, 4); + mutex_unlock(&NTFS_I(parent_vi)->mrec_lock); + if (IS_ERR(ia_vi)) { + iput(parent_vi); + continue; + } + write_inode_now(ia_vi, 1); + iput(ia_vi); + write_inode_now(parent_vi, 1); + iput(parent_vi); + } else if (ctx->attr->non_resident) { + struct inode *attr_vi; + __le16 *name; + + name = (__le16 *)((u8 *)ctx->attr + le16_to_cpu(ctx->attr->name_offset)); + if (ctx->attr->type == AT_DATA && ctx->attr->name_length == 0) + continue; + + attr_vi = ntfs_attr_iget(vi, ctx->attr->type, + name, ctx->attr->name_length); + if (IS_ERR(attr_vi)) + continue; + spin_lock(&attr_vi->i_lock); + if (inode_state_read_once(attr_vi) & I_DIRTY_PAGES) { + spin_unlock(&attr_vi->i_lock); + filemap_write_and_wait(attr_vi->i_mapping); + } else + spin_unlock(&attr_vi->i_lock); + iput(attr_vi); + } + } + mutex_unlock(&ni->mrec_lock); + ntfs_attr_put_search_ctx(ctx); + + write_inode_now(vol->mftbmp_ino, 1); + down_write(&vol->lcnbmp_lock); + write_inode_now(vol->lcnbmp_ino, 1); + up_write(&vol->lcnbmp_lock); + write_inode_now(vol->mft_ino, 1); + + /* + * NOTE: If we were to use mapping->private_list (see ext2 and + * fs/buffer.c) for dirty blocks then we could optimize the below to be + * sync_mapping_buffers(vi->i_mapping). + */ + err = sync_blockdev(vi->i_sb->s_bdev); + if (unlikely(err && !ret)) + ret = err; + if (likely(!ret)) + ntfs_debug("Done."); + else + ntfs_warning(vi->i_sb, + "Failed to f%ssync inode 0x%llx. Error %u.", + datasync ? "data" : "", ni->mft_no, -ret); + if (!ret) + blkdev_issue_flush(vi->i_sb->s_bdev); + return ret; +} + +static int ntfs_setattr_size(struct inode *vi, struct iattr *attr) +{ + struct ntfs_inode *ni = NTFS_I(vi); + int err; + loff_t old_size = vi->i_size; + + if (NInoCompressed(ni) || NInoEncrypted(ni)) { + ntfs_warning(vi->i_sb, + "Changes in inode size are not supported yet for %s files, ignoring.", + NInoCompressed(ni) ? "compressed" : "encrypted"); + return -EOPNOTSUPP; + } + + err = inode_newsize_ok(vi, attr->ia_size); + if (err) + return err; + + inode_dio_wait(vi); + truncate_setsize(vi, attr->ia_size); + err = ntfs_truncate_vfs(vi, attr->ia_size, old_size); + if (err) { + i_size_write(vi, old_size); + return err; + } + + if (NInoNonResident(ni) && attr->ia_size > old_size && + old_size % PAGE_SIZE != 0) { + loff_t len = min_t(loff_t, + round_up(old_size, PAGE_SIZE) - old_size, + attr->ia_size - old_size); + err = iomap_zero_range(vi, old_size, len, + NULL, &ntfs_seek_iomap_ops, + &ntfs_iomap_folio_ops, NULL); + } + + return err; +} + +/* + * ntfs_setattr + * + * Called from notify_change() when an attribute is being changed. + * + * NOTE: Changes in inode size are not supported yet for compressed or + * encrypted files. + */ +int ntfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +{ + struct inode *vi = d_inode(dentry); + int err; + unsigned int ia_valid = attr->ia_valid; + struct ntfs_inode *ni = NTFS_I(vi); + struct ntfs_volume *vol = ni->vol; + + if (NVolShutdown(vol)) + return -EIO; + + err = setattr_prepare(idmap, dentry, attr); + if (err) + goto out; + + if (!(vol->vol_flags & VOLUME_IS_DIRTY)) + ntfs_set_volume_flags(vol, VOLUME_IS_DIRTY); + + if (ia_valid & ATTR_SIZE) { + err = ntfs_setattr_size(vi, attr); + if (err) + goto out; + + ia_valid |= ATTR_MTIME | ATTR_CTIME; + } + + setattr_copy(idmap, vi, attr); + + if (vol->sb->s_flags & SB_POSIXACL && !S_ISLNK(vi->i_mode)) { + err = posix_acl_chmod(idmap, dentry, vi->i_mode); + if (err) + goto out; + } + + if (0222 & vi->i_mode) + ni->flags &= ~FILE_ATTR_READONLY; + else + ni->flags |= FILE_ATTR_READONLY; + + if (ia_valid & (ATTR_UID | ATTR_GID | ATTR_MODE)) { + unsigned int flags = 0; + + if (ia_valid & ATTR_UID) + flags |= NTFS_EA_UID; + if (ia_valid & ATTR_GID) + flags |= NTFS_EA_GID; + if (ia_valid & ATTR_MODE) + flags |= NTFS_EA_MODE; + + if (S_ISDIR(vi->i_mode)) + vi->i_mode &= ~vol->dmask; + else + vi->i_mode &= ~vol->fmask; + + mutex_lock(&ni->mrec_lock); + ntfs_ea_set_wsl_inode(vi, 0, NULL, flags); + mutex_unlock(&ni->mrec_lock); + } + + mark_inode_dirty(vi); +out: + return err; +} + +int ntfs_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, unsigned int request_mask, + unsigned int query_flags) +{ + struct inode *inode = d_backing_inode(path->dentry); + struct ntfs_inode *ni = NTFS_I(inode); + + generic_fillattr(idmap, request_mask, inode, stat); + + stat->blksize = NTFS_SB(inode->i_sb)->cluster_size; + stat->blocks = (((u64)NTFS_I(inode)->i_dealloc_clusters << + NTFS_SB(inode->i_sb)->cluster_size_bits) >> 9) + inode->i_blocks; + stat->result_mask |= STATX_BTIME; + stat->btime = NTFS_I(inode)->i_crtime; + + if (NInoCompressed(ni)) + stat->attributes |= STATX_ATTR_COMPRESSED; + + if (NInoEncrypted(ni)) + stat->attributes |= STATX_ATTR_ENCRYPTED; + + if (inode->i_flags & S_IMMUTABLE) + stat->attributes |= STATX_ATTR_IMMUTABLE; + + if (inode->i_flags & S_APPEND) + stat->attributes |= STATX_ATTR_APPEND; + + stat->attributes_mask |= STATX_ATTR_COMPRESSED | STATX_ATTR_ENCRYPTED | + STATX_ATTR_IMMUTABLE | STATX_ATTR_APPEND; + + /* + * If it's a compressed or encrypted file, NTFS currently + * does not support DIO. For normal files, we report the bdev + * logical block size. + */ + if (request_mask & STATX_DIOALIGN && S_ISREG(inode->i_mode)) { + unsigned int align = + bdev_logical_block_size(inode->i_sb->s_bdev); + + stat->result_mask |= STATX_DIOALIGN; + if (!NInoCompressed(ni) && !NInoEncrypted(ni)) { + stat->dio_mem_align = align; + stat->dio_offset_align = align; + } + } + + return 0; +} + +static loff_t ntfs_file_llseek(struct file *file, loff_t offset, int whence) +{ + struct inode *inode = file->f_mapping->host; + + switch (whence) { + case SEEK_HOLE: + inode_lock_shared(inode); + offset = iomap_seek_hole(inode, offset, &ntfs_seek_iomap_ops); + inode_unlock_shared(inode); + break; + case SEEK_DATA: + inode_lock_shared(inode); + offset = iomap_seek_data(inode, offset, &ntfs_seek_iomap_ops); + inode_unlock_shared(inode); + break; + default: + return generic_file_llseek_size(file, offset, whence, + inode->i_sb->s_maxbytes, + i_size_read(inode)); + } + if (offset < 0) + return offset; + return vfs_setpos(file, offset, inode->i_sb->s_maxbytes); +} + +static ssize_t ntfs_file_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct inode *vi = file_inode(iocb->ki_filp); + struct super_block *sb = vi->i_sb; + ssize_t ret; + + if (NVolShutdown(NTFS_SB(sb))) + return -EIO; + + if (NInoCompressed(NTFS_I(vi)) && iocb->ki_flags & IOCB_DIRECT) + return -EOPNOTSUPP; + + inode_lock_shared(vi); + + if (iocb->ki_flags & IOCB_DIRECT) { + size_t count = iov_iter_count(to); + + if ((iocb->ki_pos | count) & (sb->s_blocksize - 1)) { + ret = -EINVAL; + goto inode_unlock; + } + + file_accessed(iocb->ki_filp); + ret = iomap_dio_rw(iocb, to, &ntfs_read_iomap_ops, NULL, 0, + NULL, 0); + } else { + ret = generic_file_read_iter(iocb, to); + } + +inode_unlock: + inode_unlock_shared(vi); + + return ret; +} + +static int ntfs_file_write_dio_end_io(struct kiocb *iocb, ssize_t size, + int error, unsigned int flags) +{ + struct inode *inode = file_inode(iocb->ki_filp); + + if (error) + return error; + + if (size) { + if (i_size_read(inode) < iocb->ki_pos + size) { + i_size_write(inode, iocb->ki_pos + size); + mark_inode_dirty(inode); + } + } + + return 0; +} + +static const struct iomap_dio_ops ntfs_write_dio_ops = { + .end_io = ntfs_file_write_dio_end_io, +}; + +static ssize_t ntfs_dio_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + ssize_t ret; + + ret = iomap_dio_rw(iocb, from, &ntfs_dio_iomap_ops, + &ntfs_write_dio_ops, 0, NULL, 0); + if (ret == -ENOTBLK) + ret = 0; + else if (ret < 0) + goto out; + + if (iov_iter_count(from)) { + loff_t offset, end; + ssize_t written; + int ret2; + + offset = iocb->ki_pos; + iocb->ki_flags &= ~IOCB_DIRECT; + written = iomap_file_buffered_write(iocb, from, + &ntfs_write_iomap_ops, &ntfs_iomap_folio_ops, + NULL); + if (written < 0) { + ret = written; + goto out; + } + + ret += written; + end = iocb->ki_pos + written - 1; + ret2 = filemap_write_and_wait_range(iocb->ki_filp->f_mapping, + offset, end); + if (ret2) { + ret = -EIO; + goto out; + } + invalidate_mapping_pages(iocb->ki_filp->f_mapping, + offset >> PAGE_SHIFT, + end >> PAGE_SHIFT); + } + +out: + return ret; +} + +static ssize_t ntfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct inode *vi = file->f_mapping->host; + struct ntfs_inode *ni = NTFS_I(vi); + struct ntfs_volume *vol = ni->vol; + ssize_t ret; + ssize_t count; + loff_t pos; + int err; + loff_t old_data_size, old_init_size; + + if (NVolShutdown(vol)) + return -EIO; + + if (NInoEncrypted(ni)) { + ntfs_error(vi->i_sb, "Writing for %s files is not supported yet", + NInoCompressed(ni) ? "Compressed" : "Encrypted"); + return -EOPNOTSUPP; + } + + if (NInoCompressed(ni) && iocb->ki_flags & IOCB_DIRECT) + return -EOPNOTSUPP; + + if (iocb->ki_flags & IOCB_NOWAIT) { + if (!inode_trylock(vi)) + return -EAGAIN; + } else + inode_lock(vi); + + ret = generic_write_checks(iocb, from); + if (ret <= 0) + goto out_lock; + + err = file_modified(iocb->ki_filp); + if (err) { + ret = err; + goto out_lock; + } + + if (!(vol->vol_flags & VOLUME_IS_DIRTY)) + ntfs_set_volume_flags(vol, VOLUME_IS_DIRTY); + + pos = iocb->ki_pos; + count = ret; + + old_data_size = ni->data_size; + old_init_size = ni->initialized_size; + + if (NInoNonResident(ni) && NInoCompressed(ni)) { + ret = ntfs_compress_write(ni, pos, count, from); + if (ret > 0) + iocb->ki_pos += ret; + goto out; + } + + if (NInoNonResident(ni) && iocb->ki_flags & IOCB_DIRECT) + ret = ntfs_dio_write_iter(iocb, from); + else + ret = iomap_file_buffered_write(iocb, from, &ntfs_write_iomap_ops, + &ntfs_iomap_folio_ops, NULL); +out: + if (ret < 0 && ret != -EIOCBQUEUED) { + if (ni->initialized_size != old_init_size) { + mutex_lock(&ni->mrec_lock); + ntfs_attr_set_initialized_size(ni, old_init_size); + mutex_unlock(&ni->mrec_lock); + } + if (ni->data_size != old_data_size) { + truncate_setsize(vi, old_data_size); + ntfs_attr_truncate(ni, old_data_size); + } + } +out_lock: + inode_unlock(vi); + if (ret > 0) + ret = generic_write_sync(iocb, ret); + return ret; +} + +static vm_fault_t ntfs_filemap_page_mkwrite(struct vm_fault *vmf) +{ + struct inode *inode = file_inode(vmf->vma->vm_file); + vm_fault_t ret; + + sb_start_pagefault(inode->i_sb); + file_update_time(vmf->vma->vm_file); + + ret = iomap_page_mkwrite(vmf, &ntfs_page_mkwrite_iomap_ops, NULL); + sb_end_pagefault(inode->i_sb); + return ret; +} + +static const struct vm_operations_struct ntfs_file_vm_ops = { + .fault = filemap_fault, + .map_pages = filemap_map_pages, + .page_mkwrite = ntfs_filemap_page_mkwrite, +}; + +static int ntfs_file_mmap_prepare(struct vm_area_desc *desc) +{ + struct file *file = desc->file; + struct inode *inode = file_inode(file); + + if (NVolShutdown(NTFS_SB(file->f_mapping->host->i_sb))) + return -EIO; + + if (NInoCompressed(NTFS_I(inode))) + return -EOPNOTSUPP; + + if (vma_desc_test_all(desc, VMA_SHARED_BIT, VMA_MAYWRITE_BIT)) { + struct inode *inode = file_inode(file); + loff_t from, to; + int err; + + from = ((loff_t)desc->pgoff << PAGE_SHIFT); + to = min_t(loff_t, i_size_read(inode), + from + desc->end - desc->start); + + if (NTFS_I(inode)->initialized_size < to) { + err = ntfs_extend_initialized_size(inode, to, to, false); + if (err) + return err; + } + } + + + file_accessed(file); + desc->vm_ops = &ntfs_file_vm_ops; + return 0; +} + +static int ntfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + u64 start, u64 len) +{ + return iomap_fiemap(inode, fieinfo, start, len, &ntfs_read_iomap_ops); +} + +static const char *ntfs_get_link(struct dentry *dentry, struct inode *inode, + struct delayed_call *done) +{ + struct ntfs_inode *ni = NTFS_I(inode); + char *target; + int err; + + if (!dentry) + return ERR_PTR(-ECHILD); + + if (!ni->target) + return ERR_PTR(-EINVAL); + + if (ni->reparse_tag == IO_REPARSE_TAG_MOUNT_POINT || + (ni->reparse_tag == IO_REPARSE_TAG_SYMLINK && + !(ni->reparse_flags & cpu_to_le32(SYMLINK_FLAG_RELATIVE)))) { + if (NVolNativeSymlinkRel(ni->vol)) { + err = ntfs_translate_symlink_path(dentry, ni->target, &target); + if (err < 0) + return ERR_PTR(err); + set_delayed_call(done, kfree_link, target); + return target; + } + } + + return ni->target; +} + +static ssize_t ntfs_file_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, unsigned int flags) +{ + if (NVolShutdown(NTFS_SB(in->f_mapping->host->i_sb))) + return -EIO; + + return filemap_splice_read(in, ppos, pipe, len, flags); +} + +static int ntfs_ioctl_shutdown(struct super_block *sb, unsigned long arg) +{ + u32 flags; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (get_user(flags, (__u32 __user *)arg)) + return -EFAULT; + + return ntfs_force_shutdown(sb, flags); +} + +static int ntfs_ioctl_get_volume_label(struct file *filp, unsigned long arg) +{ + struct ntfs_volume *vol = NTFS_SB(file_inode(filp)->i_sb); + char __user *buf = (char __user *)arg; + char label[FSLABEL_MAX]; + ssize_t len; + + mutex_lock(&vol->volume_label_lock); + if (!vol->volume_label) { + label[0] = '\0'; + len = 0; + } else { + len = strscpy(label, vol->volume_label, sizeof(label)); + if (len == -E2BIG) + len = FSLABEL_MAX - 1; + } + mutex_unlock(&vol->volume_label_lock); + + if (copy_to_user(buf, label, len + 1)) + return -EFAULT; + return 0; +} + +static int ntfs_ioctl_set_volume_label(struct file *filp, unsigned long arg) +{ + struct ntfs_volume *vol = NTFS_SB(file_inode(filp)->i_sb); + char *label; + int ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + label = strndup_user((const char __user *)arg, FSLABEL_MAX); + if (IS_ERR(label)) + return PTR_ERR(label); + + ret = mnt_want_write_file(filp); + if (ret) + goto out; + + ret = ntfs_write_volume_label(vol, label); + mnt_drop_write_file(filp); +out: + kfree(label); + return ret; +} + +static int ntfs_ioctl_fitrim(struct ntfs_volume *vol, unsigned long arg) +{ + struct fstrim_range __user *user_range; + struct fstrim_range range; + struct block_device *dev; + int err; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + dev = vol->sb->s_bdev; + if (!bdev_max_discard_sectors(dev)) + return -EOPNOTSUPP; + + user_range = (struct fstrim_range __user *)arg; + if (copy_from_user(&range, user_range, sizeof(range))) + return -EFAULT; + + if (range.len == 0) + return -EINVAL; + + if (range.len < vol->cluster_size) + return -EINVAL; + + range.minlen = max_t(u32, range.minlen, bdev_discard_granularity(dev)); + + err = ntfs_trim_fs(vol, &range); + if (err < 0) + return err; + + if (copy_to_user(user_range, &range, sizeof(range))) + return -EFAULT; + + return 0; +} + +long ntfs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + switch (cmd) { + case FS_IOC_SHUTDOWN: + return ntfs_ioctl_shutdown(file_inode(filp)->i_sb, arg); + case FS_IOC_GETFSLABEL: + return ntfs_ioctl_get_volume_label(filp, arg); + case FS_IOC_SETFSLABEL: + return ntfs_ioctl_set_volume_label(filp, arg); + case FITRIM: + return ntfs_ioctl_fitrim(NTFS_SB(file_inode(filp)->i_sb), arg); + default: + return -ENOTTY; + } +} + +#ifdef CONFIG_COMPAT +long ntfs_compat_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg) +{ + return ntfs_ioctl(filp, cmd, (unsigned long)compat_ptr(arg)); +} +#endif + +static int ntfs_allocate_range(struct ntfs_inode *ni, int mode, loff_t offset, + loff_t len) +{ + struct inode *vi = VFS_I(ni); + struct ntfs_volume *vol = ni->vol; + s64 need_space; + loff_t old_size, new_size; + s64 start_vcn, end_vcn; + int err; + + old_size = i_size_read(vi); + new_size = max_t(loff_t, old_size, offset + len); + start_vcn = ntfs_bytes_to_cluster(vol, offset); + end_vcn = ntfs_bytes_to_cluster(vol, offset + len - 1) + 1; + + err = inode_newsize_ok(vi, new_size); + if (err) + goto out; + + need_space = ntfs_bytes_to_cluster(vol, ni->allocated_size); + if (need_space > start_vcn) + need_space = end_vcn - need_space; + else + need_space = end_vcn - start_vcn; + if (need_space > 0 && + need_space > (atomic64_read(&vol->free_clusters) - + atomic64_read(&vol->dirty_clusters))) { + err = -ENOSPC; + goto out; + } + + err = ntfs_attr_fallocate(ni, offset, len, + mode & FALLOC_FL_KEEP_SIZE ? true : false); + + if (!(mode & FALLOC_FL_KEEP_SIZE) && new_size != old_size) + i_size_write(vi, ni->data_size); +out: + return err; +} + +static int ntfs_punch_hole(struct ntfs_inode *ni, int mode, loff_t offset, + loff_t len) +{ + struct ntfs_volume *vol = ni->vol; + struct inode *vi = VFS_I(ni); + loff_t end_offset; + s64 start_vcn, end_vcn; + int err = 0; + + loff_t offset_down = round_down(offset, max_t(unsigned int, + vol->cluster_size, PAGE_SIZE)); + + if (NVolDisableSparse(vol)) { + err = -EOPNOTSUPP; + goto out; + } + + if (offset >= ni->data_size) + goto out; + + if (offset + len > ni->data_size) + end_offset = ni->data_size; + else + end_offset = offset + len; + + err = filemap_write_and_wait_range(vi->i_mapping, offset_down, LLONG_MAX); + if (err) + goto out; + truncate_pagecache(vi, offset_down); + + start_vcn = ntfs_bytes_to_cluster(vol, offset); + end_vcn = ntfs_bytes_to_cluster(vol, end_offset - 1) + 1; + + if (offset & vol->cluster_size_mask) { + if (offset < ni->initialized_size) { + loff_t to; + + to = min_t(loff_t, + ntfs_cluster_to_bytes(vol, start_vcn + 1), + end_offset); + err = iomap_zero_range(vi, offset, to - offset, + NULL, &ntfs_seek_iomap_ops, + &ntfs_iomap_folio_ops, NULL); + if (err < 0) + goto out; + } + if (end_vcn - start_vcn == 1) + goto out; + start_vcn++; + } + + if (end_offset & vol->cluster_size_mask) { + loff_t from; + + from = ntfs_cluster_to_bytes(vol, end_vcn - 1); + if (from < ni->initialized_size) { + err = iomap_zero_range(vi, from, end_offset - from, + NULL, &ntfs_seek_iomap_ops, + &ntfs_iomap_folio_ops, NULL); + if (err < 0) + goto out; + } + if (end_vcn - start_vcn == 1) + goto out; + end_vcn--; + } + + mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL); + err = ntfs_non_resident_attr_punch_hole(ni, start_vcn, + end_vcn - start_vcn); + mutex_unlock(&ni->mrec_lock); +out: + return err; +} + +static int ntfs_collapse_range(struct ntfs_inode *ni, loff_t offset, loff_t len) +{ + struct ntfs_volume *vol = ni->vol; + struct inode *vi = VFS_I(ni); + loff_t old_size, new_size; + s64 start_vcn, end_vcn; + int err; + + loff_t offset_down = round_down(offset, + max_t(unsigned long, vol->cluster_size, PAGE_SIZE)); + + if ((offset & vol->cluster_size_mask) || + (len & vol->cluster_size_mask) || + offset >= ni->allocated_size) { + err = -EINVAL; + goto out; + } + + old_size = i_size_read(vi); + start_vcn = ntfs_bytes_to_cluster(vol, offset); + end_vcn = ntfs_bytes_to_cluster(vol, offset + len - 1) + 1; + + if (ntfs_cluster_to_bytes(vol, end_vcn) > ni->allocated_size) + end_vcn = (round_up(ni->allocated_size - 1, + vol->cluster_size) >> vol->cluster_size_bits) + 1; + new_size = old_size - ntfs_cluster_to_bytes(vol, end_vcn - start_vcn); + if (new_size < 0) + new_size = 0; + err = filemap_write_and_wait_range(vi->i_mapping, + offset_down, LLONG_MAX); + if (err) + goto out; + + truncate_pagecache(vi, offset_down); + + mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL); + err = ntfs_non_resident_attr_collapse_range(ni, start_vcn, + end_vcn - start_vcn); + mutex_unlock(&ni->mrec_lock); + + if (new_size != old_size) + i_size_write(vi, ni->data_size); +out: + return err; +} + +static int ntfs_insert_range(struct ntfs_inode *ni, loff_t offset, loff_t len) +{ + struct ntfs_volume *vol = ni->vol; + struct inode *vi = VFS_I(ni); + loff_t offset_down = round_down(offset, + max_t(unsigned long, vol->cluster_size, PAGE_SIZE)); + loff_t alloc_size, end_offset = offset + len; + loff_t old_size, new_size; + s64 start_vcn, end_vcn; + int err; + + if (NVolDisableSparse(vol)) { + err = -EOPNOTSUPP; + goto out; + } + + if ((offset & vol->cluster_size_mask) || + (len & vol->cluster_size_mask) || + offset >= ni->allocated_size) { + err = -EINVAL; + goto out; + } + + old_size = i_size_read(vi); + start_vcn = ntfs_bytes_to_cluster(vol, offset); + end_vcn = ntfs_bytes_to_cluster(vol, end_offset - 1) + 1; + + new_size = old_size + ntfs_cluster_to_bytes(vol, end_vcn - start_vcn); + alloc_size = ni->allocated_size + + ntfs_cluster_to_bytes(vol, end_vcn - start_vcn); + if (alloc_size < 0) { + err = -EFBIG; + goto out; + } + err = inode_newsize_ok(vi, alloc_size); + if (err) + goto out; + + err = filemap_write_and_wait_range(vi->i_mapping, + offset_down, LLONG_MAX); + if (err) + goto out; + + truncate_pagecache(vi, offset_down); + + mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL); + err = ntfs_non_resident_attr_insert_range(ni, start_vcn, + end_vcn - start_vcn); + mutex_unlock(&ni->mrec_lock); + + if (new_size != old_size) + i_size_write(vi, ni->data_size); +out: + return err; +} + +#define NTFS_FALLOC_FL_SUPPORTED \ + (FALLOC_FL_ALLOCATE_RANGE | FALLOC_FL_KEEP_SIZE | \ + FALLOC_FL_INSERT_RANGE | FALLOC_FL_PUNCH_HOLE | \ + FALLOC_FL_COLLAPSE_RANGE) + +static long ntfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len) +{ + struct inode *vi = file_inode(file); + struct ntfs_inode *ni = NTFS_I(vi); + struct ntfs_volume *vol = ni->vol; + int err = 0; + loff_t old_size; + bool map_locked = false; + + if (mode & ~(NTFS_FALLOC_FL_SUPPORTED)) + return -EOPNOTSUPP; + + if (!NVolFreeClusterKnown(vol)) + wait_event(vol->free_waitq, NVolFreeClusterKnown(vol)); + + if ((ni->vol->mft_zone_end - ni->vol->mft_zone_start) == 0) + return -ENOSPC; + + if (NInoNonResident(ni) && !NInoFullyMapped(ni)) { + down_write(&ni->runlist.lock); + err = ntfs_attr_map_whole_runlist(ni); + up_write(&ni->runlist.lock); + if (err) + return err; + } + + if (!(vol->vol_flags & VOLUME_IS_DIRTY)) { + err = ntfs_set_volume_flags(vol, VOLUME_IS_DIRTY); + if (err) + return err; + } + + old_size = i_size_read(vi); + + inode_lock(vi); + if (NInoCompressed(ni) || NInoEncrypted(ni)) { + err = -EOPNOTSUPP; + goto out; + } + + inode_dio_wait(vi); + if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_COLLAPSE_RANGE | + FALLOC_FL_INSERT_RANGE)) { + filemap_invalidate_lock(vi->i_mapping); + map_locked = true; + } + + switch (mode & FALLOC_FL_MODE_MASK) { + case FALLOC_FL_ALLOCATE_RANGE: + case FALLOC_FL_KEEP_SIZE: + err = ntfs_allocate_range(ni, mode, offset, len); + break; + case FALLOC_FL_PUNCH_HOLE: + err = ntfs_punch_hole(ni, mode, offset, len); + break; + case FALLOC_FL_COLLAPSE_RANGE: + err = ntfs_collapse_range(ni, offset, len); + break; + case FALLOC_FL_INSERT_RANGE: + err = ntfs_insert_range(ni, offset, len); + break; + default: + err = -EOPNOTSUPP; + } + + if (err) + goto out; + + err = file_modified(file); +out: + if (map_locked) + filemap_invalidate_unlock(vi->i_mapping); + if (!err) { + if (mode == 0 && NInoNonResident(ni) && + offset > old_size && old_size % PAGE_SIZE != 0) { + loff_t len = min_t(loff_t, + round_up(old_size, PAGE_SIZE) - old_size, + offset - old_size); + err = iomap_zero_range(vi, old_size, len, NULL, + &ntfs_seek_iomap_ops, + &ntfs_iomap_folio_ops, NULL); + } + NInoSetFileNameDirty(ni); + inode_set_mtime_to_ts(vi, inode_set_ctime_current(vi)); + mark_inode_dirty(vi); + } + + inode_unlock(vi); + return err; +} + +const struct file_operations ntfs_file_ops = { + .llseek = ntfs_file_llseek, + .read_iter = ntfs_file_read_iter, + .write_iter = ntfs_file_write_iter, + .fsync = ntfs_file_fsync, + .mmap_prepare = ntfs_file_mmap_prepare, + .open = ntfs_file_open, + .release = ntfs_file_release, + .splice_read = ntfs_file_splice_read, + .splice_write = iter_file_splice_write, + .unlocked_ioctl = ntfs_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = ntfs_compat_ioctl, +#endif + .fallocate = ntfs_fallocate, + .setlease = generic_setlease, +}; + +const struct inode_operations ntfs_file_inode_ops = { + .setattr = ntfs_setattr, + .getattr = ntfs_getattr, + .listxattr = ntfs_listxattr, + .get_acl = ntfs_get_acl, + .set_acl = ntfs_set_acl, + .fiemap = ntfs_fiemap, +}; + +const struct inode_operations ntfs_symlink_inode_operations = { + .get_link = ntfs_get_link, + .setattr = ntfs_setattr, + .listxattr = ntfs_listxattr, +}; + +const struct inode_operations ntfs_special_inode_operations = { + .setattr = ntfs_setattr, + .getattr = ntfs_getattr, + .listxattr = ntfs_listxattr, + .get_acl = ntfs_get_acl, + .set_acl = ntfs_set_acl, +}; + +const struct file_operations ntfs_empty_file_ops = {}; + +const struct inode_operations ntfs_empty_inode_ops = {}; diff --git a/fs/ntfs/index.c b/fs/ntfs/index.c new file mode 100644 index 000000000..faa7ee920 --- /dev/null +++ b/fs/ntfs/index.c @@ -0,0 +1,2233 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * NTFS kernel index handling. + * + * Copyright (c) 2004-2005 Anton Altaparmakov + * Copyright (c) 2025 LG Electronics Co., Ltd. + * + * Part of this file is based on code from the NTFS-3G. + * and is copyrighted by the respective authors below: + * Copyright (c) 2004-2005 Anton Altaparmakov + * Copyright (c) 2004-2005 Richard Russon + * Copyright (c) 2005-2006 Yura Pakhuchiy + * Copyright (c) 2005-2008 Szabolcs Szakacsits + * Copyright (c) 2007-2021 Jean-Pierre Andre + */ + +#include "collate.h" +#include "index.h" +#include "ntfs.h" +#include "attrlist.h" + +/* + * ntfs_index_entry_inconsistent - Check the consistency of an index entry + * + * Make sure data and key do not overflow from entry. + * As a side effect, an entry with zero length is rejected. + * This entry must be a full one (no INDEX_ENTRY_END flag), and its + * length must have been checked beforehand to not overflow from the + * index record. + */ +static int ntfs_index_entry_inconsistent(const struct ntfs_volume *vol, + const struct index_entry *ie, + __le32 collation_rule, u64 inum) +{ + if (ie->key_length && + ((le16_to_cpu(ie->key_length) + offsetof(struct index_entry, key)) > + le16_to_cpu(ie->length))) { + ntfs_error(vol->sb, "Overflow from index entry in inode %lld\n", + (long long)inum); + return -EIO; + + } else { + if (collation_rule == COLLATION_FILE_NAME) { + if ((offsetof(struct index_entry, key.file_name.file_name) + + ie->key.file_name.file_name_length * sizeof(__le16)) > + le16_to_cpu(ie->length)) { + ntfs_error(vol->sb, + "File name overflow from index entry in inode %lld\n", + (long long)inum); + return -EIO; + } + } else { + if (ie->data.vi.data_length && + ((le16_to_cpu(ie->data.vi.data_offset) + + le16_to_cpu(ie->data.vi.data_length)) > + le16_to_cpu(ie->length))) { + ntfs_error(vol->sb, + "Data overflow from index entry in inode %lld\n", + (long long)inum); + return -EIO; + } + } + } + + return 0; +} + +/* + * ntfs_index_entry_mark_dirty - mark an index entry dirty + * @ictx: ntfs index context describing the index entry + * + * Mark the index entry described by the index entry context @ictx dirty. + * + * If the index entry is in the index root attribute, simply mark the inode + * containing the index root attribute dirty. This ensures the mftrecord, and + * hence the index root attribute, will be written out to disk later. + * + * If the index entry is in an index block belonging to the index allocation + * attribute, set ib_dirty to true, thus index block will be updated during + * ntfs_index_ctx_put. + */ +void ntfs_index_entry_mark_dirty(struct ntfs_index_context *ictx) +{ + if (ictx->is_in_root) + mark_mft_record_dirty(ictx->actx->ntfs_ino); + else if (ictx->ib) + ictx->ib_dirty = true; +} + +static s64 ntfs_ib_vcn_to_pos(struct ntfs_index_context *icx, s64 vcn) +{ + return vcn << icx->vcn_size_bits; +} + +static s64 ntfs_ib_pos_to_vcn(struct ntfs_index_context *icx, s64 pos) +{ + return pos >> icx->vcn_size_bits; +} + +static int ntfs_ib_write(struct ntfs_index_context *icx, struct index_block *ib) +{ + s64 ret, vcn = le64_to_cpu(ib->index_block_vcn); + + ntfs_debug("vcn: %lld\n", vcn); + + ret = pre_write_mst_fixup((struct ntfs_record *)ib, icx->block_size); + if (ret) + return -EIO; + + ret = ntfs_inode_attr_pwrite(VFS_I(icx->ia_ni), + ntfs_ib_vcn_to_pos(icx, vcn), icx->block_size, + (u8 *)ib, icx->sync_write); + + /* Perform data restoration before returning */ + post_write_mst_fixup((struct ntfs_record *)ib); + + if (ret != icx->block_size) { + ntfs_debug("Failed to write index block %lld, inode %llu", + vcn, (unsigned long long)icx->idx_ni->mft_no); + return ret; + } + + return 0; +} + +static int ntfs_icx_ib_write(struct ntfs_index_context *icx) +{ + int err; + + err = ntfs_ib_write(icx, icx->ib); + if (err) + return err; + + icx->ib_dirty = false; + + return 0; +} + +int ntfs_icx_ib_sync_write(struct ntfs_index_context *icx) +{ + int ret; + + if (icx->ib_dirty == false) + return 0; + + icx->sync_write = true; + + ret = ntfs_ib_write(icx, icx->ib); + if (!ret) { + kvfree(icx->ib); + icx->ib = NULL; + icx->ib_dirty = false; + } else { + icx->sync_write = false; + } + + return ret; +} + +/* + * ntfs_index_ctx_get - allocate and initialize a new index context + * @ni: ntfs inode with which to initialize the context + * @name: name of the which context describes + * @name_len: length of the index name + * + * Allocate a new index context, initialize it with @ni and return it. + * Return NULL if allocation failed. + */ +struct ntfs_index_context *ntfs_index_ctx_get(struct ntfs_inode *ni, + __le16 *name, u32 name_len) +{ + struct ntfs_index_context *icx; + + ntfs_debug("Entering\n"); + + if (!ni) + return NULL; + + if (ni->nr_extents == -1) + ni = ni->ext.base_ntfs_ino; + + icx = kmem_cache_alloc(ntfs_index_ctx_cache, GFP_NOFS); + if (icx) + *icx = (struct ntfs_index_context) { + .idx_ni = ni, + .name = name, + .name_len = name_len, + }; + return icx; +} + +static void ntfs_index_ctx_free(struct ntfs_index_context *icx) +{ + ntfs_debug("Entering\n"); + + if (icx->actx) { + ntfs_attr_put_search_ctx(icx->actx); + icx->actx = NULL; + } + + if (!icx->is_in_root) { + if (icx->ib_dirty) + ntfs_ib_write(icx, icx->ib); + kvfree(icx->ib); + icx->ib = NULL; + } + + if (icx->ia_ni) { + iput(VFS_I(icx->ia_ni)); + icx->ia_ni = NULL; + } +} + +/* + * ntfs_index_ctx_put - release an index context + * @icx: index context to free + * + * Release the index context @icx, releasing all associated resources. + */ +void ntfs_index_ctx_put(struct ntfs_index_context *icx) +{ + ntfs_index_ctx_free(icx); + kmem_cache_free(ntfs_index_ctx_cache, icx); +} + +/* + * ntfs_index_ctx_reinit - reinitialize an index context + * @icx: index context to reinitialize + * + * Reinitialize the index context @icx so it can be used for ntfs_index_lookup. + */ +void ntfs_index_ctx_reinit(struct ntfs_index_context *icx) +{ + ntfs_debug("Entering\n"); + + ntfs_index_ctx_free(icx); + + *icx = (struct ntfs_index_context) { + .idx_ni = icx->idx_ni, + .name = icx->name, + .name_len = icx->name_len, + }; +} + +static __le64 *ntfs_ie_get_vcn_addr(struct index_entry *ie) +{ + return (__le64 *)((u8 *)ie + le16_to_cpu(ie->length) - sizeof(s64)); +} + +/* + * Get the subnode vcn to which the index entry refers. + */ +static s64 ntfs_ie_get_vcn(struct index_entry *ie) +{ + return le64_to_cpup(ntfs_ie_get_vcn_addr(ie)); +} + +static struct index_entry *ntfs_ie_get_first(struct index_header *ih) +{ + return (struct index_entry *)((u8 *)ih + le32_to_cpu(ih->entries_offset)); +} + +static struct index_entry *ntfs_ie_get_next(struct index_entry *ie) +{ + return (struct index_entry *)((char *)ie + le16_to_cpu(ie->length)); +} + +static u8 *ntfs_ie_get_end(struct index_header *ih) +{ + return (u8 *)ih + le32_to_cpu(ih->index_length); +} + +static int ntfs_ie_end(struct index_entry *ie) +{ + return ie->flags & INDEX_ENTRY_END || !ie->length; +} + +static int ntfs_index_header_inconsistent(struct ntfs_volume *vol, + const struct index_header *ih, + u32 bytes_available, u64 inum) +{ + u32 entries_offset, index_length, allocated_size; + + if (bytes_available < sizeof(struct index_header)) { + ntfs_error(vol->sb, + "index block in inode %llu is smaller than an index header.", + (unsigned long long)inum); + return -EIO; + } + + entries_offset = le32_to_cpu(ih->entries_offset); + index_length = le32_to_cpu(ih->index_length); + allocated_size = le32_to_cpu(ih->allocated_size); + + if (entries_offset < sizeof(struct index_header) || + entries_offset > bytes_available) { + ntfs_error(vol->sb, + "Invalid index entry offset in inode %llu.", + (unsigned long long)inum); + return -EIO; + } + + if (index_length <= entries_offset) { + ntfs_error(vol->sb, + "No space for index entries in inode %llu.", + (unsigned long long)inum); + return -EIO; + } + + if (allocated_size < index_length) { + ntfs_error(vol->sb, + "Index entries overflow in inode %llu.", + (unsigned long long)inum); + return -EIO; + } + + if (allocated_size > bytes_available || index_length > bytes_available) { + ntfs_error(vol->sb, + "Index entries in inode %llu exceed the available buffer.", + (unsigned long long)inum); + return -EIO; + } + + return 0; +} + +int ntfs_index_entries_inconsistent(const struct ntfs_volume *vol, + const struct index_header *ih, + __le32 collation_rule, u64 inum) +{ + struct index_entry *ie; + u8 *index_end = (u8 *)ih + le32_to_cpu(ih->index_length); + + for (ie = ntfs_ie_get_first((struct index_header *)ih); + ; ie = ntfs_ie_get_next(ie)) { + if ((u8 *)ie + sizeof(struct index_entry_header) > index_end || + (u8 *)ie + le16_to_cpu(ie->length) > index_end) { + ntfs_error(vol->sb, + "Index entry out of bounds in inode %llu.", + (unsigned long long)inum); + return -EIO; + } + + if (le16_to_cpu(ie->length) < sizeof(struct index_entry_header)) { + ntfs_error(vol->sb, + "Index entry too small in inode %llu.", + inum); + return -EIO; + } + + if (ntfs_ie_end(ie)) + break; + + if (!ie->key_length) + return -EIO; + + if (ntfs_index_entry_inconsistent(vol, ie, + collation_rule, inum)) + return -EIO; + } + + return 0; +} + +/* + * Find the last entry in the index block + */ +static struct index_entry *ntfs_ie_get_last(struct index_entry *ie, char *ies_end) +{ + ntfs_debug("Entering\n"); + + while ((char *)ie < ies_end && !ntfs_ie_end(ie)) + ie = ntfs_ie_get_next(ie); + + return ie; +} + +static struct index_entry *ntfs_ie_get_by_pos(struct index_header *ih, int pos) +{ + struct index_entry *ie; + + ntfs_debug("pos: %d\n", pos); + + ie = ntfs_ie_get_first(ih); + + while (pos-- > 0) + ie = ntfs_ie_get_next(ie); + + return ie; +} + +static struct index_entry *ntfs_ie_prev(struct index_header *ih, struct index_entry *ie) +{ + struct index_entry *ie_prev = NULL; + struct index_entry *tmp; + + ntfs_debug("Entering\n"); + + tmp = ntfs_ie_get_first(ih); + + while (tmp != ie) { + ie_prev = tmp; + tmp = ntfs_ie_get_next(tmp); + } + + return ie_prev; +} + +static int ntfs_ih_numof_entries(struct index_header *ih) +{ + int n; + struct index_entry *ie; + u8 *end; + + ntfs_debug("Entering\n"); + + end = ntfs_ie_get_end(ih); + ie = ntfs_ie_get_first(ih); + for (n = 0; !ntfs_ie_end(ie) && (u8 *)ie < end; n++) + ie = ntfs_ie_get_next(ie); + return n; +} + +static int ntfs_ih_one_entry(struct index_header *ih) +{ + return (ntfs_ih_numof_entries(ih) == 1); +} + +static int ntfs_ih_zero_entry(struct index_header *ih) +{ + return (ntfs_ih_numof_entries(ih) == 0); +} + +static void ntfs_ie_delete(struct index_header *ih, struct index_entry *ie) +{ + u32 new_size; + + ntfs_debug("Entering\n"); + + new_size = le32_to_cpu(ih->index_length) - le16_to_cpu(ie->length); + ih->index_length = cpu_to_le32(new_size); + memmove(ie, (u8 *)ie + le16_to_cpu(ie->length), + new_size - ((u8 *)ie - (u8 *)ih)); +} + +static void ntfs_ie_set_vcn(struct index_entry *ie, s64 vcn) +{ + *ntfs_ie_get_vcn_addr(ie) = cpu_to_le64(vcn); +} + +/* + * Insert @ie index entry at @pos entry. Used @ih values should be ok already. + */ +static void ntfs_ie_insert(struct index_header *ih, struct index_entry *ie, + struct index_entry *pos) +{ + int ie_size = le16_to_cpu(ie->length); + + ntfs_debug("Entering\n"); + + ih->index_length = cpu_to_le32(le32_to_cpu(ih->index_length) + ie_size); + memmove((u8 *)pos + ie_size, pos, + le32_to_cpu(ih->index_length) - ((u8 *)pos - (u8 *)ih) - ie_size); + memcpy(pos, ie, ie_size); +} + +static struct index_entry *ntfs_ie_dup(struct index_entry *ie) +{ + ntfs_debug("Entering\n"); + + return kmemdup(ie, le16_to_cpu(ie->length), GFP_NOFS); +} + +static struct index_entry *ntfs_ie_dup_novcn(struct index_entry *ie) +{ + struct index_entry *dup; + int size = le16_to_cpu(ie->length); + + ntfs_debug("Entering\n"); + + if (ie->flags & INDEX_ENTRY_NODE) + size -= sizeof(s64); + + dup = kmemdup(ie, size, GFP_NOFS); + if (dup) { + dup->flags &= ~INDEX_ENTRY_NODE; + dup->length = cpu_to_le16(size); + } + return dup; +} + +/* + * Check the consistency of an index block + * + * Make sure the index block does not overflow from the index record. + * The size of block is assumed to have been checked to be what is + * defined in the index root. + * + * Returns 0 if no error was found, -EIO otherwise + * + * |<--->| offsetof(struct index_block, index) + * | |<--->| sizeof(struct index_header) + * | | | + * | | | seq index entries unused + * |=====|=====|=====|===========================|==============| + * | | | | | + * | |<--------->| entries_offset | | + * | |<---------------- index_length ------->| | + * | |<--------------------- allocated_size --------------->| + * |<--------------------------- block_size ------------------->| + * + * size(struct index_header) <= ent_offset < ind_length <= alloc_size < bk_size + */ +int ntfs_index_block_inconsistent(struct ntfs_volume *vol, + const struct index_block *ib, + u32 block_size, s64 vcn, __le32 cr, + u64 inum) +{ + u32 ib_size = (unsigned int)le32_to_cpu(ib->index.allocated_size) + + offsetof(struct index_block, index); + struct super_block *sb = vol->sb; + + ntfs_debug("Entering\n"); + + if (!ntfs_is_indx_record(ib->magic)) { + ntfs_error(sb, "Corrupt index block signature: vcn %lld inode %llu\n", + vcn, (unsigned long long)inum); + return -EIO; + } + + if (le64_to_cpu(ib->index_block_vcn) != vcn) { + ntfs_error(sb, + "Corrupt index block: s64 (%lld) is different from expected s64 (%lld) in inode %llu\n", + (long long)le64_to_cpu(ib->index_block_vcn), + vcn, inum); + return -EIO; + } + + if (ib_size != block_size) { + ntfs_error(sb, + "Corrupt index block : s64 (%lld) of inode %llu has a size (%u) differing from the index specified size (%u)\n", + vcn, inum, ib_size, block_size); + return -EIO; + } + + if (ntfs_index_header_inconsistent(vol, &ib->index, + block_size - + offsetof(struct index_block, index), + inum)) + return -EIO; + if (ntfs_index_entries_inconsistent(vol, &ib->index, cr, inum)) + return -EIO; + return 0; +} + +int ntfs_index_root_inconsistent(struct ntfs_volume *vol, + const struct attr_record *a, + const struct index_root *ir, u64 inum) +{ + u32 value_length = le32_to_cpu(a->data.resident.value_length); + + if (value_length < offsetof(struct index_root, index)) { + ntfs_error(vol->sb, "$INDEX_ROOT in inode %llu is too small.", + (unsigned long long)inum); + return -EIO; + } + + return ntfs_index_header_inconsistent(vol, &ir->index, + value_length - + offsetof(struct index_root, index), + inum); +} + +static struct index_root *ntfs_ir_lookup(struct ntfs_inode *ni, __le16 *name, + u32 name_len, struct ntfs_attr_search_ctx **ctx) +{ + struct attr_record *a; + struct index_root *ir = NULL; + + ntfs_debug("Entering\n"); + *ctx = ntfs_attr_get_search_ctx(ni, NULL); + if (!*ctx) { + ntfs_error(ni->vol->sb, "%s, Failed to get search context", __func__); + return NULL; + } + + if (ntfs_attr_lookup(AT_INDEX_ROOT, name, name_len, CASE_SENSITIVE, + 0, NULL, 0, *ctx)) { + ntfs_error(ni->vol->sb, "Failed to lookup $INDEX_ROOT"); + goto err_out; + } + + a = (*ctx)->attr; + if (a->non_resident) { + ntfs_error(ni->vol->sb, "Non-resident $INDEX_ROOT detected"); + goto err_out; + } + + ir = (struct index_root *)((char *)a + le16_to_cpu(a->data.resident.value_offset)); +err_out: + if (!ir) { + ntfs_attr_put_search_ctx(*ctx); + *ctx = NULL; + } + return ir; +} + +static struct index_root *ntfs_ir_lookup2(struct ntfs_inode *ni, __le16 *name, u32 len) +{ + struct ntfs_attr_search_ctx *ctx; + struct index_root *ir; + + ir = ntfs_ir_lookup(ni, name, len, &ctx); + if (ir) + ntfs_attr_put_search_ctx(ctx); + return ir; +} + +/* + * Find a key in the index block. + */ +static int ntfs_ie_lookup(const void *key, const u32 key_len, + struct ntfs_index_context *icx, struct index_header *ih, + s64 *vcn, struct index_entry **ie_out) +{ + struct index_entry *ie; + u8 *index_end; + int rc, item = 0; + + ntfs_debug("Entering\n"); + + index_end = ntfs_ie_get_end(ih); + + /* + * Loop until we exceed valid memory (corruption case) or until we + * reach the last entry. + */ + for (ie = ntfs_ie_get_first(ih); ; ie = ntfs_ie_get_next(ie)) { + /* Bounds checks. */ + if ((u8 *)ie + sizeof(struct index_entry_header) > index_end || + (u8 *)ie + le16_to_cpu(ie->length) > index_end) { + ntfs_error(icx->idx_ni->vol->sb, + "Index entry out of bounds in inode %llu.\n", + (unsigned long long)icx->idx_ni->mft_no); + return -ERANGE; + } + + /* + * The last entry cannot contain a key. It can however contain + * a pointer to a child node in the B+tree so we just break out. + */ + if (ntfs_ie_end(ie)) + break; + + /* + * Not a perfect match, need to do full blown collation so we + * know which way in the B+tree we have to go. + */ + rc = ntfs_collate(icx->idx_ni->vol, icx->cr, key, key_len, &ie->key, + le16_to_cpu(ie->key_length)); + if (rc == -EINVAL) { + ntfs_error(icx->idx_ni->vol->sb, + "Collation error. Perhaps a filename contains invalid characters?\n"); + return -ERANGE; + } + /* + * If @key collates before the key of the current entry, there + * is definitely no such key in this index but we might need to + * descend into the B+tree so we just break out of the loop. + */ + if (rc == -1) + break; + + if (!rc) { + *ie_out = ie; + icx->parent_pos[icx->pindex] = item; + return 0; + } + + item++; + } + /* + * We have finished with this index block without success. Check for the + * presence of a child node and if not present return with errno ENOENT, + * otherwise we will keep searching in another index block. + */ + if (!(ie->flags & INDEX_ENTRY_NODE)) { + ntfs_debug("Index entry wasn't found.\n"); + *ie_out = ie; + return -ENOENT; + } + + /* Get the starting vcn of the index_block holding the child node. */ + *vcn = ntfs_ie_get_vcn(ie); + if (*vcn < 0) { + ntfs_error(icx->idx_ni->vol->sb, "Negative vcn in inode %llu\n", + (unsigned long long)icx->idx_ni->mft_no); + return -EINVAL; + } + + ntfs_debug("Parent entry number %d\n", item); + icx->parent_pos[icx->pindex] = item; + + return -EAGAIN; +} + +struct ntfs_inode *ntfs_ia_open(struct ntfs_index_context *icx, struct ntfs_inode *ni) +{ + struct inode *ia_vi; + + ia_vi = ntfs_index_iget(VFS_I(ni), icx->name, icx->name_len); + if (IS_ERR(ia_vi)) { + ntfs_error(icx->idx_ni->vol->sb, + "Failed to open index allocation of inode %llu", + (unsigned long long)ni->mft_no); + return NULL; + } + + return NTFS_I(ia_vi); +} + +static int ntfs_ib_read(struct ntfs_index_context *icx, s64 vcn, struct index_block *dst) +{ + s64 pos, ret; + + ntfs_debug("vcn: %lld\n", vcn); + + pos = ntfs_ib_vcn_to_pos(icx, vcn); + + ret = ntfs_inode_attr_pread(VFS_I(icx->ia_ni), pos, icx->block_size, (u8 *)dst); + if (ret != icx->block_size) { + if (ret == -1) + ntfs_error(icx->idx_ni->vol->sb, "Failed to read index block"); + else + ntfs_error(icx->idx_ni->vol->sb, + "Failed to read full index block at %lld\n", pos); + return -EIO; + } + + post_read_mst_fixup((struct ntfs_record *)((u8 *)dst), icx->block_size); + if (ntfs_index_block_inconsistent(icx->idx_ni->vol, dst, + icx->block_size, vcn, icx->cr, + icx->idx_ni->mft_no)) + return -EIO; + return 0; +} + +static int ntfs_icx_parent_inc(struct ntfs_index_context *icx) +{ + if (icx->pindex >= MAX_PARENT_VCN - 1) { + ntfs_error(icx->idx_ni->vol->sb, "Index is over %d level deep", MAX_PARENT_VCN); + return -EOPNOTSUPP; + } + icx->pindex++; + return 0; +} + +static int ntfs_icx_parent_dec(struct ntfs_index_context *icx) +{ + icx->pindex--; + if (icx->pindex < 0) { + ntfs_error(icx->idx_ni->vol->sb, "Corrupt index pointer (%d)", icx->pindex); + return -EINVAL; + } + return 0; +} + +/* + * ntfs_index_lookup - find a key in an index and return its index entry + * @key: key for which to search in the index + * @key_len: length of @key in bytes + * @icx: context describing the index and the returned entry + * + * Before calling ntfs_index_lookup(), @icx must have been obtained from a + * call to ntfs_index_ctx_get(). + * + * Look for the @key in the index specified by the index lookup context @icx. + * ntfs_index_lookup() walks the contents of the index looking for the @key. + * + * If the @key is found in the index, 0 is returned and @icx is setup to + * describe the index entry containing the matching @key. @icx->entry is the + * index entry and @icx->data and @icx->data_len are the index entry data and + * its length in bytes, respectively. + * + * If the @key is not found in the index, -ENOENT is returned and + * @icx is setup to describe the index entry whose key collates immediately + * after the search @key, i.e. this is the position in the index at which + * an index entry with a key of @key would need to be inserted. + * + * When finished with the entry and its data, call ntfs_index_ctx_put() to free + * the context and other associated resources. + * + * If the index entry was modified, call ntfs_index_entry_mark_dirty() before + * the call to ntfs_index_ctx_put() to ensure that the changes are written + * to disk. + */ +int ntfs_index_lookup(const void *key, const u32 key_len, struct ntfs_index_context *icx) +{ + s64 old_vcn, vcn; + struct ntfs_inode *ni = icx->idx_ni; + struct super_block *sb = ni->vol->sb; + struct index_root *ir; + struct index_entry *ie; + struct index_block *ib = NULL; + int err = 0; + + ntfs_debug("Entering\n"); + + if (!key) { + ntfs_error(sb, "key: %p key_len: %d", key, key_len); + return -EINVAL; + } + + ir = ntfs_ir_lookup(ni, icx->name, icx->name_len, &icx->actx); + if (!ir) + return -EIO; + + icx->block_size = le32_to_cpu(ir->index_block_size); + if (icx->block_size < NTFS_BLOCK_SIZE) { + err = -EINVAL; + ntfs_error(sb, + "Index block size (%d) is smaller than the sector size (%d)", + icx->block_size, NTFS_BLOCK_SIZE); + goto err_out; + } + + if (ni->vol->cluster_size <= icx->block_size) + icx->vcn_size_bits = ni->vol->cluster_size_bits; + else + icx->vcn_size_bits = ni->vol->sector_size_bits; + + icx->cr = ir->collation_rule; + if (!ntfs_is_collation_rule_supported(icx->cr)) { + err = -EOPNOTSUPP; + ntfs_error(sb, "Unknown collation rule 0x%x", + (unsigned int)le32_to_cpu(icx->cr)); + goto err_out; + } + + old_vcn = VCN_INDEX_ROOT_PARENT; + err = ntfs_ie_lookup(key, key_len, icx, &ir->index, &vcn, &ie); + if (err == -ERANGE || err == -EINVAL) + goto err_out; + + icx->ir = ir; + if (err != -EAGAIN) { + icx->is_in_root = true; + icx->parent_vcn[icx->pindex] = old_vcn; + goto done; + } + + /* Child node present, descend into it. */ + icx->ia_ni = ntfs_ia_open(icx, ni); + if (!icx->ia_ni) { + err = -ENOENT; + goto err_out; + } + + ib = kvzalloc(icx->block_size, GFP_NOFS); + if (!ib) { + err = -ENOMEM; + goto err_out; + } + +descend_into_child_node: + icx->parent_vcn[icx->pindex] = old_vcn; + if (ntfs_icx_parent_inc(icx)) { + err = -EIO; + goto err_out; + } + old_vcn = vcn; + + ntfs_debug("Descend into node with s64 %lld.\n", vcn); + + if (ntfs_ib_read(icx, vcn, ib)) { + err = -EIO; + goto err_out; + } + err = ntfs_ie_lookup(key, key_len, icx, &ib->index, &vcn, &ie); + if (err != -EAGAIN) { + if (err == -EINVAL || err == -ERANGE) + goto err_out; + + icx->is_in_root = false; + icx->ib = ib; + icx->parent_vcn[icx->pindex] = vcn; + goto done; + } + + if ((ib->index.flags & NODE_MASK) == LEAF_NODE) { + ntfs_error(icx->idx_ni->vol->sb, + "Index entry with child node found in a leaf node in inode 0x%llx.\n", + (unsigned long long)ni->mft_no); + goto err_out; + } + + goto descend_into_child_node; +err_out: + if (icx->actx) { + ntfs_attr_put_search_ctx(icx->actx); + icx->actx = NULL; + } + kvfree(ib); + if (!err) + err = -EIO; + return err; +done: + icx->entry = ie; + icx->data = (u8 *)ie + offsetof(struct index_entry, key); + icx->data_len = le16_to_cpu(ie->key_length); + ntfs_debug("Done.\n"); + return err; + +} + +static struct index_block *ntfs_ib_alloc(s64 ib_vcn, u32 ib_size, + u8 node_type) +{ + struct index_block *ib; + int ih_size = sizeof(struct index_header); + + ntfs_debug("Entering ib_vcn = %lld ib_size = %u\n", ib_vcn, ib_size); + + ib = kvzalloc(ib_size, GFP_NOFS); + if (!ib) + return NULL; + + ib->magic = magic_INDX; + ib->usa_ofs = cpu_to_le16(sizeof(struct index_block)); + ib->usa_count = cpu_to_le16(ib_size / NTFS_BLOCK_SIZE + 1); + /* Set USN to 1 */ + *(__le16 *)((char *)ib + le16_to_cpu(ib->usa_ofs)) = cpu_to_le16(1); + ib->lsn = 0; + ib->index_block_vcn = cpu_to_le64(ib_vcn); + ib->index.entries_offset = cpu_to_le32((ih_size + + le16_to_cpu(ib->usa_count) * 2 + 7) & ~7); + ib->index.index_length = 0; + ib->index.allocated_size = cpu_to_le32(ib_size - + (sizeof(struct index_block) - ih_size)); + ib->index.flags = node_type; + + return ib; +} + +/* + * Find the median by going through all the entries + */ +static struct index_entry *ntfs_ie_get_median(struct index_header *ih) +{ + struct index_entry *ie, *ie_start; + u8 *ie_end; + int i = 0, median; + + ntfs_debug("Entering\n"); + + ie = ie_start = ntfs_ie_get_first(ih); + ie_end = (u8 *)ntfs_ie_get_end(ih); + + while ((u8 *)ie < ie_end && !ntfs_ie_end(ie)) { + ie = ntfs_ie_get_next(ie); + i++; + } + /* + * NOTE: this could be also the entry at the half of the index block. + */ + median = i / 2 - 1; + + ntfs_debug("Entries: %d median: %d\n", i, median); + + for (i = 0, ie = ie_start; i <= median; i++) + ie = ntfs_ie_get_next(ie); + + return ie; +} + +static u64 ntfs_ibm_vcn_to_pos(struct ntfs_index_context *icx, s64 vcn) +{ + u64 pos = ntfs_ib_vcn_to_pos(icx, vcn); + + do_div(pos, icx->block_size); + return pos; +} + +static s64 ntfs_ibm_pos_to_vcn(struct ntfs_index_context *icx, s64 pos) +{ + return ntfs_ib_pos_to_vcn(icx, pos * icx->block_size); +} + +static int ntfs_ibm_add(struct ntfs_index_context *icx) +{ + u8 bmp[8]; + + ntfs_debug("Entering\n"); + + if (ntfs_attr_exist(icx->idx_ni, AT_BITMAP, icx->name, icx->name_len)) + return 0; + /* + * AT_BITMAP must be at least 8 bytes. + */ + memset(bmp, 0, sizeof(bmp)); + if (ntfs_attr_add(icx->idx_ni, AT_BITMAP, icx->name, icx->name_len, + bmp, sizeof(bmp))) { + ntfs_error(icx->idx_ni->vol->sb, "Failed to add AT_BITMAP"); + return -EINVAL; + } + + return 0; +} + +static int ntfs_ibm_modify(struct ntfs_index_context *icx, s64 vcn, int set) +{ + u8 byte; + u64 pos = ntfs_ibm_vcn_to_pos(icx, vcn); + u32 bpos = pos / 8; + u32 bit = 1 << (pos % 8); + struct ntfs_inode *bmp_ni; + struct inode *bmp_vi; + int ret = 0; + + ntfs_debug("%s vcn: %lld\n", set ? "set" : "clear", vcn); + + bmp_vi = ntfs_attr_iget(VFS_I(icx->idx_ni), AT_BITMAP, icx->name, icx->name_len); + if (IS_ERR(bmp_vi)) { + ntfs_error(icx->idx_ni->vol->sb, "Failed to open $BITMAP attribute"); + return PTR_ERR(bmp_vi); + } + + bmp_ni = NTFS_I(bmp_vi); + + if (set) { + if (bmp_ni->data_size < bpos + 1) { + ret = ntfs_attr_truncate(bmp_ni, (bmp_ni->data_size + 8) & ~7); + if (ret) { + ntfs_error(icx->idx_ni->vol->sb, "Failed to truncate AT_BITMAP"); + goto err; + } + i_size_write(bmp_vi, (loff_t)bmp_ni->data_size); + } + } + + if (ntfs_inode_attr_pread(bmp_vi, bpos, 1, &byte) != 1) { + ret = -EIO; + ntfs_error(icx->idx_ni->vol->sb, "Failed to read $BITMAP"); + goto err; + } + + if (set) + byte |= bit; + else + byte &= ~bit; + + if (ntfs_inode_attr_pwrite(bmp_vi, bpos, 1, &byte, false) != 1) { + ret = -EIO; + ntfs_error(icx->idx_ni->vol->sb, "Failed to write $Bitmap"); + goto err; + } + +err: + iput(bmp_vi); + return ret; +} + +static int ntfs_ibm_set(struct ntfs_index_context *icx, s64 vcn) +{ + return ntfs_ibm_modify(icx, vcn, 1); +} + +static int ntfs_ibm_clear(struct ntfs_index_context *icx, s64 vcn) +{ + return ntfs_ibm_modify(icx, vcn, 0); +} + +static s64 ntfs_ibm_get_free(struct ntfs_index_context *icx) +{ + u8 *bm; + int bit; + s64 vcn, byte, size; + + ntfs_debug("Entering\n"); + + bm = ntfs_attr_readall(icx->idx_ni, AT_BITMAP, icx->name, icx->name_len, + &size); + if (!bm) + return (s64)-1; + + for (byte = 0; byte < size; byte++) { + if (bm[byte] == 255) + continue; + + for (bit = 0; bit < 8; bit++) { + if (!(bm[byte] & (1 << bit))) { + vcn = ntfs_ibm_pos_to_vcn(icx, byte * 8 + bit); + goto out; + } + } + } + + vcn = ntfs_ibm_pos_to_vcn(icx, size * 8); +out: + ntfs_debug("allocated vcn: %lld\n", vcn); + + if (ntfs_ibm_set(icx, vcn)) + vcn = (s64)-1; + + kvfree(bm); + return vcn; +} + +static struct index_block *ntfs_ir_to_ib(struct index_root *ir, s64 ib_vcn) +{ + struct index_block *ib; + struct index_entry *ie_last; + char *ies_start, *ies_end; + int i; + + ntfs_debug("Entering\n"); + + ib = ntfs_ib_alloc(ib_vcn, le32_to_cpu(ir->index_block_size), LEAF_NODE); + if (!ib) + return NULL; + + ies_start = (char *)ntfs_ie_get_first(&ir->index); + ies_end = (char *)ntfs_ie_get_end(&ir->index); + ie_last = ntfs_ie_get_last((struct index_entry *)ies_start, ies_end); + /* + * Copy all entries, including the termination entry + * as well, which can never have any data. + */ + i = (char *)ie_last - ies_start + le16_to_cpu(ie_last->length); + memcpy(ntfs_ie_get_first(&ib->index), ies_start, i); + + ib->index.flags = ir->index.flags; + ib->index.index_length = cpu_to_le32(i + + le32_to_cpu(ib->index.entries_offset)); + return ib; +} + +static void ntfs_ir_nill(struct index_root *ir) +{ + struct index_entry *ie_last; + char *ies_start, *ies_end; + + ntfs_debug("Entering\n"); + + ies_start = (char *)ntfs_ie_get_first(&ir->index); + ies_end = (char *)ntfs_ie_get_end(&ir->index); + ie_last = ntfs_ie_get_last((struct index_entry *)ies_start, ies_end); + /* + * Move the index root termination entry forward + */ + if ((char *)ie_last > ies_start) { + memmove((char *)ntfs_ie_get_first(&ir->index), + (char *)ie_last, le16_to_cpu(ie_last->length)); + ie_last = (struct index_entry *)ies_start; + } +} + +static int ntfs_ib_copy_tail(struct ntfs_index_context *icx, struct index_block *src, + struct index_entry *median, s64 new_vcn) +{ + u8 *ies_end; + struct index_entry *ie_head; /* first entry after the median */ + int tail_size, ret; + struct index_block *dst; + + ntfs_debug("Entering\n"); + + dst = ntfs_ib_alloc(new_vcn, icx->block_size, + src->index.flags & NODE_MASK); + if (!dst) + return -ENOMEM; + + ie_head = ntfs_ie_get_next(median); + + ies_end = (u8 *)ntfs_ie_get_end(&src->index); + tail_size = ies_end - (u8 *)ie_head; + memcpy(ntfs_ie_get_first(&dst->index), ie_head, tail_size); + + dst->index.index_length = cpu_to_le32(tail_size + + le32_to_cpu(dst->index.entries_offset)); + ret = ntfs_ib_write(icx, dst); + + kvfree(dst); + return ret; +} + +static int ntfs_ib_cut_tail(struct ntfs_index_context *icx, struct index_block *ib, + struct index_entry *ie) +{ + char *ies_start, *ies_end; + struct index_entry *ie_last; + int ret; + + ntfs_debug("Entering\n"); + + ies_start = (char *)ntfs_ie_get_first(&ib->index); + ies_end = (char *)ntfs_ie_get_end(&ib->index); + + ie_last = ntfs_ie_get_last((struct index_entry *)ies_start, ies_end); + if (ie_last->flags & INDEX_ENTRY_NODE) + ntfs_ie_set_vcn(ie_last, ntfs_ie_get_vcn(ie)); + + unsafe_memcpy(ie, ie_last, le16_to_cpu(ie_last->length), + /* alloc is larger than ie_last->length, see ntfs_ie_get_last() */); + + ib->index.index_length = cpu_to_le32(((char *)ie - ies_start) + + le16_to_cpu(ie->length) + le32_to_cpu(ib->index.entries_offset)); + + ret = ntfs_ib_write(icx, ib); + return ret; +} + +static int ntfs_ia_add(struct ntfs_index_context *icx) +{ + int ret; + + ntfs_debug("Entering\n"); + + ret = ntfs_ibm_add(icx); + if (ret) + return ret; + + if (!ntfs_attr_exist(icx->idx_ni, AT_INDEX_ALLOCATION, icx->name, icx->name_len)) { + ret = ntfs_attr_add(icx->idx_ni, AT_INDEX_ALLOCATION, icx->name, + icx->name_len, NULL, 0); + if (ret) { + ntfs_error(icx->idx_ni->vol->sb, "Failed to add AT_INDEX_ALLOCATION"); + return ret; + } + } + + icx->ia_ni = ntfs_ia_open(icx, icx->idx_ni); + if (!icx->ia_ni) + return -ENOENT; + + return 0; +} + +static int ntfs_ir_reparent(struct ntfs_index_context *icx) +{ + struct ntfs_attr_search_ctx *ctx = NULL; + struct index_root *ir; + struct index_entry *ie; + struct index_block *ib = NULL; + s64 new_ib_vcn; + u32 index_length; + u32 old_value_length; + int ix_root_size; + int ret = 0; + + ntfs_debug("Entering\n"); + + ir = ntfs_ir_lookup2(icx->idx_ni, icx->name, icx->name_len); + if (!ir) { + ret = -ENOENT; + goto out; + } + + if ((ir->index.flags & NODE_MASK) == SMALL_INDEX) { + ret = ntfs_ia_add(icx); + if (ret) + goto out; + } + + new_ib_vcn = ntfs_ibm_get_free(icx); + if (new_ib_vcn < 0) { + ret = -EINVAL; + goto out; + } + + ir = ntfs_ir_lookup2(icx->idx_ni, icx->name, icx->name_len); + if (!ir) { + ret = -ENOENT; + goto clear_bmp; + } + + ib = ntfs_ir_to_ib(ir, new_ib_vcn); + if (ib == NULL) { + ret = -EIO; + ntfs_error(icx->idx_ni->vol->sb, "Failed to move index root to index block"); + goto clear_bmp; + } + + ret = ntfs_ib_write(icx, ib); + if (ret) + goto clear_bmp; + +retry: + ir = ntfs_ir_lookup(icx->idx_ni, icx->name, icx->name_len, &ctx); + if (!ir) { + ret = -ENOENT; + goto clear_bmp; + } + + old_value_length = le32_to_cpu(ctx->attr->data.resident.value_length); + index_length = le32_to_cpu(ir->index.entries_offset) + + sizeof(struct index_entry_header) + sizeof(s64); + ix_root_size = offsetof(struct index_root, index) + index_length; + /* Grow the resident value before publishing the larger root header. */ + if (ix_root_size > old_value_length) { + ret = ntfs_resident_attr_value_resize(ctx->mrec, ctx->attr, ix_root_size); + if (ret) + goto resize_failed; + + icx->idx_ni->data_size = ix_root_size; + icx->idx_ni->initialized_size = ix_root_size; + icx->idx_ni->allocated_size = (ix_root_size + 7) & ~7; + } + + ntfs_ir_nill(ir); + + ie = ntfs_ie_get_first(&ir->index); + ie->flags |= INDEX_ENTRY_NODE; + ie->length = cpu_to_le16(sizeof(struct index_entry_header) + sizeof(s64)); + + ir->index.flags = LARGE_INDEX; + NInoSetIndexAllocPresent(icx->idx_ni); + ir->index.index_length = cpu_to_le32(index_length); + ir->index.allocated_size = ir->index.index_length; + + if (ix_root_size <= old_value_length) { + ret = ntfs_resident_attr_value_resize(ctx->mrec, ctx->attr, ix_root_size); + if (ret) + goto resize_failed; + + icx->idx_ni->data_size = ix_root_size; + icx->idx_ni->initialized_size = ix_root_size; + icx->idx_ni->allocated_size = (ix_root_size + 7) & ~7; + } + ntfs_ie_set_vcn(ie, new_ib_vcn); + goto err_out; + +resize_failed: + /* + * When there is no space to build a non-resident + * index, we may have to move the root to an extent + */ + if ((ret == -ENOSPC) && (ctx->al_entry || !ntfs_inode_add_attrlist(icx->idx_ni))) { + ntfs_attr_put_search_ctx(ctx); + ctx = NULL; + ir = ntfs_ir_lookup(icx->idx_ni, icx->name, icx->name_len, &ctx); + if (ir && !ntfs_attr_record_move_away(ctx, ix_root_size - + le32_to_cpu(ctx->attr->data.resident.value_length))) { + if (ntfs_attrlist_update(ctx->base_ntfs_ino ? + ctx->base_ntfs_ino : ctx->ntfs_ino)) + goto clear_bmp; + ntfs_attr_put_search_ctx(ctx); + ctx = NULL; + goto retry; + } + } +clear_bmp: + ntfs_ibm_clear(icx, new_ib_vcn); + goto err_out; +err_out: + kvfree(ib); + if (ctx) + ntfs_attr_put_search_ctx(ctx); +out: + return ret; +} + +/* + * ntfs_ir_truncate - Truncate index root attribute + * @icx: index context + * @data_size: new data size for the index root + */ +static int ntfs_ir_truncate(struct ntfs_index_context *icx, int data_size) +{ + int ret; + u32 old_allocated_size; + bool shrink; + + ntfs_debug("Entering\n"); + + old_allocated_size = le32_to_cpu(icx->ir->index.allocated_size); + shrink = data_size < old_allocated_size; + if (shrink) + icx->ir->index.allocated_size = cpu_to_le32(data_size); + + /* + * INDEX_ROOT must be resident and its entries can be moved to + * struct index_block, so ENOSPC isn't a real error. + */ + ret = ntfs_attr_truncate(icx->idx_ni, data_size + offsetof(struct index_root, index)); + if (!ret) { + i_size_write(VFS_I(icx->idx_ni), icx->idx_ni->initialized_size); + icx->ir = ntfs_ir_lookup2(icx->idx_ni, icx->name, icx->name_len); + if (!icx->ir) + return -ENOENT; + + if (!shrink) + icx->ir->index.allocated_size = cpu_to_le32(data_size); + } else { + if (shrink) + icx->ir->index.allocated_size = cpu_to_le32(old_allocated_size); + if (ret != -ENOSPC) + ntfs_error(icx->idx_ni->vol->sb, "Failed to truncate INDEX_ROOT"); + } + + return ret; +} + +/* + * ntfs_ir_make_space - Make more space for the index root attribute + * @icx: index context + * @data_size: required data size for the index root + */ +static int ntfs_ir_make_space(struct ntfs_index_context *icx, int data_size) +{ + int ret; + + ntfs_debug("Entering\n"); + + ret = ntfs_ir_truncate(icx, data_size); + if (ret == -ENOSPC) { + ret = ntfs_ir_reparent(icx); + if (!ret) + ret = -EAGAIN; + else + ntfs_error(icx->idx_ni->vol->sb, "Failed to modify INDEX_ROOT"); + } + + return ret; +} + +/* + * NOTE: 'ie' must be a copy of a real index entry. + */ +static int ntfs_ie_add_vcn(struct index_entry **ie) +{ + struct index_entry *p, *old = *ie; + + old->length = cpu_to_le16(le16_to_cpu(old->length) + sizeof(s64)); + p = krealloc(old, le16_to_cpu(old->length), GFP_NOFS); + if (!p) + return -ENOMEM; + + p->flags |= INDEX_ENTRY_NODE; + *ie = p; + return 0; +} + +static int ntfs_ih_insert(struct index_header *ih, struct index_entry *orig_ie, s64 new_vcn, + int pos) +{ + struct index_entry *ie_node, *ie; + int ret = 0; + s64 old_vcn; + + ntfs_debug("Entering\n"); + ie = ntfs_ie_dup(orig_ie); + if (!ie) + return -ENOMEM; + + if (!(ie->flags & INDEX_ENTRY_NODE)) { + ret = ntfs_ie_add_vcn(&ie); + if (ret) + goto out; + } + + ie_node = ntfs_ie_get_by_pos(ih, pos); + old_vcn = ntfs_ie_get_vcn(ie_node); + ntfs_ie_set_vcn(ie_node, new_vcn); + + ntfs_ie_insert(ih, ie, ie_node); + ntfs_ie_set_vcn(ie_node, old_vcn); +out: + kfree(ie); + return ret; +} + +static s64 ntfs_icx_parent_vcn(struct ntfs_index_context *icx) +{ + return icx->parent_vcn[icx->pindex]; +} + +static s64 ntfs_icx_parent_pos(struct ntfs_index_context *icx) +{ + return icx->parent_pos[icx->pindex]; +} + +static int ntfs_ir_insert_median(struct ntfs_index_context *icx, struct index_entry *median, + s64 new_vcn) +{ + u32 new_size; + int ret; + + ntfs_debug("Entering\n"); + + icx->ir = ntfs_ir_lookup2(icx->idx_ni, icx->name, icx->name_len); + if (!icx->ir) + return -ENOENT; + + new_size = le32_to_cpu(icx->ir->index.index_length) + + le16_to_cpu(median->length); + if (!(median->flags & INDEX_ENTRY_NODE)) + new_size += sizeof(s64); + + ret = ntfs_ir_make_space(icx, new_size); + if (ret) + return ret; + + icx->ir = ntfs_ir_lookup2(icx->idx_ni, icx->name, icx->name_len); + if (!icx->ir) + return -ENOENT; + + return ntfs_ih_insert(&icx->ir->index, median, new_vcn, + ntfs_icx_parent_pos(icx)); +} + +static int ntfs_ib_split(struct ntfs_index_context *icx, struct index_block *ib); + +struct split_info { + struct list_head entry; + s64 new_vcn; + struct index_block *ib; +}; + +static int ntfs_ib_insert(struct ntfs_index_context *icx, struct index_entry *ie, s64 new_vcn, + struct split_info *si) +{ + struct index_block *ib; + u32 idx_size, allocated_size; + int err; + s64 old_vcn; + + ntfs_debug("Entering\n"); + + ib = kvzalloc(icx->block_size, GFP_NOFS); + if (!ib) + return -ENOMEM; + + old_vcn = ntfs_icx_parent_vcn(icx); + + err = ntfs_ib_read(icx, old_vcn, ib); + if (err) + goto err_out; + + idx_size = le32_to_cpu(ib->index.index_length); + allocated_size = le32_to_cpu(ib->index.allocated_size); + if (idx_size + le16_to_cpu(ie->length) + sizeof(s64) > allocated_size) { + si->ib = ib; + si->new_vcn = new_vcn; + return -EAGAIN; + } + + err = ntfs_ih_insert(&ib->index, ie, new_vcn, ntfs_icx_parent_pos(icx)); + if (err) + goto err_out; + + err = ntfs_ib_write(icx, ib); + +err_out: + kvfree(ib); + return err; +} + +/* + * ntfs_ib_split - Split an index block + * @icx: index context + * @ib: index block to split + */ +static int ntfs_ib_split(struct ntfs_index_context *icx, struct index_block *ib) +{ + struct index_entry *median; + s64 new_vcn; + int ret; + struct split_info *si; + LIST_HEAD(ntfs_cut_tail_list); + + ntfs_debug("Entering\n"); + +resplit: + ret = ntfs_icx_parent_dec(icx); + if (ret) + goto out; + + median = ntfs_ie_get_median(&ib->index); + new_vcn = ntfs_ibm_get_free(icx); + if (new_vcn < 0) { + ret = -EINVAL; + goto out; + } + + ret = ntfs_ib_copy_tail(icx, ib, median, new_vcn); + if (ret) { + ntfs_ibm_clear(icx, new_vcn); + goto out; + } + + if (ntfs_icx_parent_vcn(icx) == VCN_INDEX_ROOT_PARENT) { + ret = ntfs_ir_insert_median(icx, median, new_vcn); + if (ret) { + ntfs_ibm_clear(icx, new_vcn); + goto out; + } + } else { + si = kzalloc(sizeof(struct split_info), GFP_NOFS); + if (!si) { + ntfs_ibm_clear(icx, new_vcn); + ret = -ENOMEM; + goto out; + } + + ret = ntfs_ib_insert(icx, median, new_vcn, si); + if (ret == -EAGAIN) { + list_add_tail(&si->entry, &ntfs_cut_tail_list); + ib = si->ib; + goto resplit; + } else if (ret) { + kvfree(si->ib); + kfree(si); + ntfs_ibm_clear(icx, new_vcn); + goto out; + } + kfree(si); + } + + ret = ntfs_ib_cut_tail(icx, ib, median); + +out: + while (!list_empty(&ntfs_cut_tail_list)) { + si = list_last_entry(&ntfs_cut_tail_list, struct split_info, entry); + ntfs_ibm_clear(icx, si->new_vcn); + kvfree(si->ib); + list_del(&si->entry); + kfree(si); + if (!ret) + ret = -EAGAIN; + } + + return ret; +} + +int ntfs_ie_add(struct ntfs_index_context *icx, struct index_entry *ie) +{ + struct index_header *ih; + int allocated_size, new_size; + int ret; + + while (1) { + ret = ntfs_index_lookup(&ie->key, le16_to_cpu(ie->key_length), icx); + if (!ret) { + ret = -EEXIST; + ntfs_error(icx->idx_ni->vol->sb, "Index already have such entry"); + goto err_out; + } + if (ret != -ENOENT) { + ntfs_error(icx->idx_ni->vol->sb, "Failed to find place for new entry"); + goto err_out; + } + ret = 0; + + if (icx->is_in_root) + ih = &icx->ir->index; + else + ih = &icx->ib->index; + + allocated_size = le32_to_cpu(ih->allocated_size); + new_size = le32_to_cpu(ih->index_length) + le16_to_cpu(ie->length); + + if (new_size <= allocated_size) + break; + + ntfs_debug("index block sizes: allocated: %d needed: %d\n", + allocated_size, new_size); + + if (icx->is_in_root) + ret = ntfs_ir_make_space(icx, new_size); + else + ret = ntfs_ib_split(icx, icx->ib); + if (ret && ret != -EAGAIN) + goto err_out; + + mark_mft_record_dirty(icx->actx->ntfs_ino); + ntfs_index_ctx_reinit(icx); + } + + ntfs_ie_insert(ih, ie, icx->entry); + ntfs_index_entry_mark_dirty(icx); + +err_out: + ntfs_debug("%s\n", ret ? "Failed" : "Done"); + return ret; +} + +/* + * ntfs_index_add_filename - add filename to directory index + * @ni: ntfs inode describing directory to which index add filename + * @fn: FILE_NAME attribute to add + * @mref: reference of the inode which @fn describes + */ +int ntfs_index_add_filename(struct ntfs_inode *ni, struct file_name_attr *fn, u64 mref) +{ + struct index_entry *ie; + struct ntfs_index_context *icx; + int fn_size, ie_size, err; + + ntfs_debug("Entering\n"); + + if (!ni || !fn) + return -EINVAL; + + fn_size = (fn->file_name_length * sizeof(__le16)) + + sizeof(struct file_name_attr); + ie_size = (sizeof(struct index_entry_header) + fn_size + 7) & ~7; + + ie = kzalloc(ie_size, GFP_NOFS); + if (!ie) + return -ENOMEM; + + ie->data.dir.indexed_file = cpu_to_le64(mref); + ie->length = cpu_to_le16(ie_size); + ie->key_length = cpu_to_le16(fn_size); + + unsafe_memcpy(&ie->key, fn, fn_size, + /* "fn_size" was correctly calculated above */); + + icx = ntfs_index_ctx_get(ni, I30, 4); + if (!icx) { + err = -ENOMEM; + goto out; + } + + err = ntfs_ie_add(icx, ie); + ntfs_index_ctx_put(icx); +out: + kfree(ie); + return err; +} + +static int ntfs_ih_takeout(struct ntfs_index_context *icx, struct index_header *ih, + struct index_entry *ie, struct index_block *ib) +{ + struct index_entry *ie_roam; + int freed_space; + bool full; + int ret = 0; + + ntfs_debug("Entering\n"); + + full = ih->index_length == ih->allocated_size; + ie_roam = ntfs_ie_dup_novcn(ie); + if (!ie_roam) + return -ENOMEM; + + ntfs_ie_delete(ih, ie); + + if (ntfs_icx_parent_vcn(icx) == VCN_INDEX_ROOT_PARENT) { + /* + * Recover the space which may have been freed + * while deleting an entry from root index + */ + freed_space = le32_to_cpu(ih->allocated_size) - + le32_to_cpu(ih->index_length); + if (full && (freed_space > 0) && !(freed_space & 7)) { + ntfs_ir_truncate(icx, le32_to_cpu(ih->index_length)); + /* do nothing if truncation fails */ + } + + mark_mft_record_dirty(icx->actx->ntfs_ino); + } else { + ret = ntfs_ib_write(icx, ib); + if (ret) + goto out; + } + + ntfs_index_ctx_reinit(icx); + + ret = ntfs_ie_add(icx, ie_roam); +out: + kfree(ie_roam); + return ret; +} + +/* + * Used if an empty index block to be deleted has END entry as the parent + * in the INDEX_ROOT which is the only one there. + */ +static void ntfs_ir_leafify(struct ntfs_index_context *icx, struct index_header *ih) +{ + struct index_entry *ie; + + ntfs_debug("Entering\n"); + + ie = ntfs_ie_get_first(ih); + ie->flags &= ~INDEX_ENTRY_NODE; + ie->length = cpu_to_le16(le16_to_cpu(ie->length) - sizeof(s64)); + + ih->index_length = cpu_to_le32(le32_to_cpu(ih->index_length) - sizeof(s64)); + ih->flags &= ~LARGE_INDEX; + NInoClearIndexAllocPresent(icx->idx_ni); + + /* Not fatal error */ + ntfs_ir_truncate(icx, le32_to_cpu(ih->index_length)); +} + +/* + * Used if an empty index block to be deleted has END entry as the parent + * in the INDEX_ROOT which is not the only one there. + */ +static int ntfs_ih_reparent_end(struct ntfs_index_context *icx, struct index_header *ih, + struct index_block *ib) +{ + struct index_entry *ie, *ie_prev; + + ntfs_debug("Entering\n"); + + ie = ntfs_ie_get_by_pos(ih, ntfs_icx_parent_pos(icx)); + ie_prev = ntfs_ie_prev(ih, ie); + if (!ie_prev) + return -EIO; + ntfs_ie_set_vcn(ie, ntfs_ie_get_vcn(ie_prev)); + + return ntfs_ih_takeout(icx, ih, ie_prev, ib); +} + +static int ntfs_index_rm_leaf(struct ntfs_index_context *icx) +{ + struct index_block *ib = NULL; + struct index_header *parent_ih; + struct index_entry *ie; + int ret; + + ntfs_debug("pindex: %d\n", icx->pindex); + + ret = ntfs_icx_parent_dec(icx); + if (ret) + return ret; + + ret = ntfs_ibm_clear(icx, icx->parent_vcn[icx->pindex + 1]); + if (ret) + return ret; + + if (ntfs_icx_parent_vcn(icx) == VCN_INDEX_ROOT_PARENT) + parent_ih = &icx->ir->index; + else { + ib = kvzalloc(icx->block_size, GFP_NOFS); + if (!ib) + return -ENOMEM; + + ret = ntfs_ib_read(icx, ntfs_icx_parent_vcn(icx), ib); + if (ret) + goto out; + + parent_ih = &ib->index; + } + + ie = ntfs_ie_get_by_pos(parent_ih, ntfs_icx_parent_pos(icx)); + if (!ntfs_ie_end(ie)) { + ret = ntfs_ih_takeout(icx, parent_ih, ie, ib); + goto out; + } + + if (ntfs_ih_zero_entry(parent_ih)) { + if (ntfs_icx_parent_vcn(icx) == VCN_INDEX_ROOT_PARENT) { + ntfs_ir_leafify(icx, parent_ih); + goto out; + } + + ret = ntfs_index_rm_leaf(icx); + goto out; + } + + ret = ntfs_ih_reparent_end(icx, parent_ih, ib); +out: + kvfree(ib); + return ret; +} + +static int ntfs_index_rm_node(struct ntfs_index_context *icx) +{ + int entry_pos, pindex; + s64 vcn; + struct index_block *ib = NULL; + struct index_entry *ie_succ, *ie, *entry = icx->entry; + struct index_header *ih; + u32 new_size; + int delta, ret; + + ntfs_debug("Entering\n"); + + if (!icx->ia_ni) { + icx->ia_ni = ntfs_ia_open(icx, icx->idx_ni); + if (!icx->ia_ni) + return -EINVAL; + } + + ib = kvzalloc(icx->block_size, GFP_NOFS); + if (!ib) + return -ENOMEM; + + ie_succ = ntfs_ie_get_next(icx->entry); + entry_pos = icx->parent_pos[icx->pindex]++; + pindex = icx->pindex; +descend: + vcn = ntfs_ie_get_vcn(ie_succ); + ret = ntfs_ib_read(icx, vcn, ib); + if (ret) + goto out; + + ie_succ = ntfs_ie_get_first(&ib->index); + + ret = ntfs_icx_parent_inc(icx); + if (ret) + goto out; + + icx->parent_vcn[icx->pindex] = vcn; + icx->parent_pos[icx->pindex] = 0; + + if ((ib->index.flags & NODE_MASK) == INDEX_NODE) + goto descend; + + if (ntfs_ih_zero_entry(&ib->index)) { + ret = -EIO; + ntfs_error(icx->idx_ni->vol->sb, "Empty index block"); + goto out; + } + + ie = ntfs_ie_dup(ie_succ); + if (!ie) { + ret = -ENOMEM; + goto out; + } + + ret = ntfs_ie_add_vcn(&ie); + if (ret) + goto out2; + + ntfs_ie_set_vcn(ie, ntfs_ie_get_vcn(icx->entry)); + + if (icx->is_in_root) + ih = &icx->ir->index; + else + ih = &icx->ib->index; + + delta = le16_to_cpu(ie->length) - le16_to_cpu(icx->entry->length); + new_size = le32_to_cpu(ih->index_length) + delta; + if (delta > 0) { + if (icx->is_in_root) { + ret = ntfs_ir_make_space(icx, new_size); + if (ret != 0) + goto out2; + + ih = &icx->ir->index; + entry = ntfs_ie_get_by_pos(ih, entry_pos); + + } else if (new_size > le32_to_cpu(ih->allocated_size)) { + icx->pindex = pindex; + ret = ntfs_ib_split(icx, icx->ib); + if (!ret) + ret = -EAGAIN; + goto out2; + } + } + + ntfs_ie_delete(ih, entry); + ntfs_ie_insert(ih, ie, entry); + + if (icx->is_in_root) + ret = ntfs_ir_truncate(icx, new_size); + else + ret = ntfs_icx_ib_write(icx); + if (ret) + goto out2; + + ntfs_ie_delete(&ib->index, ie_succ); + + if (ntfs_ih_zero_entry(&ib->index)) + ret = ntfs_index_rm_leaf(icx); + else + ret = ntfs_ib_write(icx, ib); + +out2: + kfree(ie); +out: + kvfree(ib); + return ret; +} + +/* + * ntfs_index_rm - remove entry from the index + * @icx: index context describing entry to delete + * + * Delete entry described by @icx from the index. Index context is always + * reinitialized after use of this function, so it can be used for index + * lookup once again. + */ +int ntfs_index_rm(struct ntfs_index_context *icx) +{ + struct index_header *ih; + int ret = 0; + + ntfs_debug("Entering\n"); + + if (!icx || (!icx->ib && !icx->ir) || ntfs_ie_end(icx->entry)) { + ret = -EINVAL; + goto err_out; + } + if (icx->is_in_root) + ih = &icx->ir->index; + else + ih = &icx->ib->index; + + if (icx->entry->flags & INDEX_ENTRY_NODE) { + ret = ntfs_index_rm_node(icx); + if (ret) + goto err_out; + } else if (icx->is_in_root || !ntfs_ih_one_entry(ih)) { + ntfs_ie_delete(ih, icx->entry); + + if (icx->is_in_root) + ret = ntfs_ir_truncate(icx, le32_to_cpu(ih->index_length)); + else + ret = ntfs_icx_ib_write(icx); + if (ret) + goto err_out; + } else { + ret = ntfs_index_rm_leaf(icx); + if (ret) + goto err_out; + } + + return 0; +err_out: + return ret; +} + +int ntfs_index_remove(struct ntfs_inode *dir_ni, const void *key, const u32 keylen) +{ + int ret = 0; + struct ntfs_index_context *icx; + + icx = ntfs_index_ctx_get(dir_ni, I30, 4); + if (!icx) + return -EINVAL; + + while (1) { + ret = ntfs_index_lookup(key, keylen, icx); + if (ret) + goto err_out; + + ret = ntfs_index_rm(icx); + if (ret && ret != -EAGAIN) + goto err_out; + else if (!ret) + break; + + mark_mft_record_dirty(icx->actx->ntfs_ino); + ntfs_index_ctx_reinit(icx); + } + + mark_mft_record_dirty(icx->actx->ntfs_ino); + + ntfs_index_ctx_put(icx); + return 0; +err_out: + ntfs_index_ctx_put(icx); + ntfs_error(dir_ni->vol->sb, "Delete failed"); + return ret; +} + +/* + * ntfs_index_walk_down - walk down the index tree (leaf bound) + * until there are no subnode in the first index entry returns + * the entry at the bottom left in subnode + */ +struct index_entry *ntfs_index_walk_down(struct index_entry *ie, struct ntfs_index_context *ictx) +{ + struct index_entry *entry; + struct index_block *ib; + int err; + s64 vcn; + + entry = ie; + do { + vcn = ntfs_ie_get_vcn(entry); + if (ictx->is_in_root) { + ib = kvzalloc(ictx->block_size, GFP_NOFS); + if (!ib) + return ERR_PTR(-ENOMEM); + /* + * Descending from root index (level 0) to the first + * child level. is_in_root == true implies pindex == 0, + * so advance to level 1. + */ + ictx->pindex = 1; + ictx->ir = NULL; + ictx->ib = ib; + ictx->is_in_root = false; + } else { + /* down from non-zero level */ + err = ntfs_icx_parent_inc(ictx); + if (err) + return ERR_PTR(err); + } + + ictx->parent_pos[ictx->pindex] = 0; + ictx->parent_vcn[ictx->pindex] = vcn; + if (!ntfs_ib_read(ictx, vcn, ictx->ib)) { + ictx->entry = ntfs_ie_get_first(&ictx->ib->index); + entry = ictx->entry; + } else + entry = ERR_PTR(-EIO); + } while (!IS_ERR(entry) && (entry->flags & INDEX_ENTRY_NODE)); + + return entry; +} + +/* + * ntfs_index_walk_up - walk up the index tree (root bound) until + * there is a valid data entry in parent returns the parent entry + * or NULL if no more parent. + * @ie: current index entry + * @ictx: index context + */ +static struct index_entry *ntfs_index_walk_up(struct index_entry *ie, + struct ntfs_index_context *ictx) +{ + struct index_entry *entry = ie; + s64 vcn; + + if (ictx->pindex <= 0) + return NULL; + + do { + ictx->pindex--; + if (!ictx->pindex) { + /* we have reached the root */ + kfree(ictx->ib); + ictx->ib = NULL; + ictx->is_in_root = true; + /* a new search context is to be allocated */ + if (ictx->actx) + ntfs_attr_put_search_ctx(ictx->actx); + ictx->ir = ntfs_ir_lookup(ictx->idx_ni, ictx->name, + ictx->name_len, &ictx->actx); + if (ictx->ir) + entry = ntfs_ie_get_by_pos( + &ictx->ir->index, + ictx->parent_pos[ictx->pindex]); + else + entry = NULL; + } else { + /* up into non-root node */ + vcn = ictx->parent_vcn[ictx->pindex]; + if (!ntfs_ib_read(ictx, vcn, ictx->ib)) { + entry = ntfs_ie_get_by_pos( + &ictx->ib->index, + ictx->parent_pos[ictx->pindex]); + } else + entry = NULL; + } + ictx->entry = entry; + } while (entry && (ictx->pindex > 0) && + (entry->flags & INDEX_ENTRY_END)); + return entry; +} + +/* + * ntfs_index_next - get next entry in an index according to collating sequence. + * Returns next entry or NULL if none. + * + * Sample layout : + * + * +---+---+---+---+---+---+---+---+ n ptrs to subnodes + * | | | 10| 25| 33| | | | n-1 keys in between + * +---+---+---+---+---+---+---+---+ no key in last entry + * | A | A + * | | | +-------------------------------+ + * +--------------------------+ | +-----+ | + * | +--+ | | + * V | V | + * +---+---+---+---+---+---+---+---+ | +---+---+---+---+---+---+---+---+ + * | 11| 12| 13| 14| 15| 16| 17| | | | 26| 27| 28| 29| 30| 31| 32| | + * +---+---+---+---+---+---+---+---+ | +---+---+---+---+---+---+---+---+ + * | | + * +-----------------------+ | + * | | + * +---+---+---+---+---+---+---+---+ + * | 18| 19| 20| 21| 22| 23| 24| | + * +---+---+---+---+---+---+---+---+ + * + * @ie: current index entry + * @ictx: index context + */ +struct index_entry *ntfs_index_next(struct index_entry *ie, struct ntfs_index_context *ictx) +{ + struct index_entry *next; + __le16 flags; + + /* + * lookup() may have returned an invalid node + * when searching for a partial key + * if this happens, walk up + */ + if (ie->flags & INDEX_ENTRY_END) + next = ntfs_index_walk_up(ie, ictx); + else { + /* + * get next entry in same node + * there is always one after any entry with data + */ + next = (struct index_entry *)((char *)ie + le16_to_cpu(ie->length)); + ++ictx->parent_pos[ictx->pindex]; + flags = next->flags; + + /* walk down if it has a subnode */ + if (flags & INDEX_ENTRY_NODE) { + if (!ictx->ia_ni) { + ictx->ia_ni = ntfs_ia_open(ictx, ictx->idx_ni); + if (!ictx->ia_ni) + return ERR_PTR(-EIO); + } + + next = ntfs_index_walk_down(next, ictx); + if (IS_ERR(next)) + return next; + } else { + + /* walk up it has no subnode, nor data */ + if (flags & INDEX_ENTRY_END) + next = ntfs_index_walk_up(next, ictx); + } + } + + /* return NULL if stuck at end of a block */ + if (next && (next->flags & INDEX_ENTRY_END)) + next = NULL; + + return next; +} diff --git a/fs/ntfs/index.h b/fs/ntfs/index.h new file mode 100644 index 000000000..9a03f53bb --- /dev/null +++ b/fs/ntfs/index.h @@ -0,0 +1,119 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Defines for NTFS kernel index handling. + * + * Copyright (c) 2004 Anton Altaparmakov + */ + +#ifndef _LINUX_NTFS_INDEX_H +#define _LINUX_NTFS_INDEX_H + +#include + +#include "attrib.h" +#include "mft.h" + +#define VCN_INDEX_ROOT_PARENT ((s64)-2) + +#define MAX_PARENT_VCN 32 + +/* + * @idx_ni: index inode containing the @entry described by this context + * @name: Unicode name of the indexed attribute + * (usually $I30 for directories) + * @name_len: length of @name in Unicode characters + * @entry: index entry (points into @ir or @ia) + * @cr: creation time of the entry (for sorting/validation) + * @data: index entry data (points into @entry) + * @data_len: length in bytes of @data + * @is_in_root: 'true' if @entry is in @ir and 'false' if it is in @ia + * @ir: index root if @is_in_root and NULL otherwise + * @actx: attribute search context if @is_in_root and NULL otherwise + * @ib: index block header (valid when @is_in_root is 'false') + * @ia_ni: index allocation inode (extent inode) for @ia + * @parent_pos: array of parent entry positions in the B-tree nodes + * @parent_vcn: VCNs of parent index blocks in the B-tree traversal + * @pindex: current depth (number of parent nodes) in the traversal + * (maximum is MAX_PARENT_VCN) + * @ib_dirty: true if the current index block (@ia/@ib) was modified + * @block_size: size of index blocks in bytes (from $INDEX_ROOT or $Boot) + * @vcn_size_bits: log2(cluster size) + * @sync_write: true if synchronous writeback is requested for this context + * + * @idx_ni is the index inode this context belongs to. + * + * @entry is the index entry described by this context. @data and @data_len + * are the index entry data and its length in bytes, respectively. @data + * simply points into @entry. This is probably what the user is interested in. + * + * If @is_in_root is 'true', @entry is in the index root attribute @ir described + * by the attribute search context @actx and the base inode @base_ni. @ia and + * @page are NULL in this case. + * + * If @is_in_root is 'false', @entry is in the index allocation attribute and @ia + * and @page point to the index allocation block and the mapped, locked page it + * is in, respectively. @ir, @actx and @base_ni are NULL in this case. + * + * To obtain a context call ntfs_index_ctx_get(). + * + * We use this context to allow ntfs_index_lookup() to return the found index + * @entry and its @data without having to allocate a buffer and copy the @entry + * and/or its @data into it. + * + * When finished with the @entry and its @data, call ntfs_index_ctx_put() to + * free the context and other associated resources. + * + * If the index entry was modified, ntfs_index_entry_mark_dirty() + * or ntfs_index_entry_write() before the call to ntfs_index_ctx_put() to + * ensure that the changes are written to disk. + */ +struct ntfs_index_context { + struct ntfs_inode *idx_ni; + __le16 *name; + u32 name_len; + struct index_entry *entry; + __le32 cr; + void *data; + u16 data_len; + bool is_in_root; + struct index_root *ir; + struct ntfs_attr_search_ctx *actx; + struct index_block *ib; + struct ntfs_inode *ia_ni; + int parent_pos[MAX_PARENT_VCN]; + s64 parent_vcn[MAX_PARENT_VCN]; + int pindex; + bool ib_dirty; + u32 block_size; + u8 vcn_size_bits; + bool sync_write; +}; + +int ntfs_index_root_inconsistent(struct ntfs_volume *vol, + const struct attr_record *a, + const struct index_root *ir, u64 inum); +int ntfs_index_block_inconsistent(struct ntfs_volume *vol, + const struct index_block *ib, + u32 block_size, s64 vcn, + __le32 cr, u64 inum); +int ntfs_index_entries_inconsistent(const struct ntfs_volume *vol, + const struct index_header *ih, + __le32 collation_rule, u64 inum); +struct ntfs_index_context *ntfs_index_ctx_get(struct ntfs_inode *ni, __le16 *name, + u32 name_len); +void ntfs_index_ctx_put(struct ntfs_index_context *ictx); +int ntfs_index_lookup(const void *key, const u32 key_len, + struct ntfs_index_context *ictx); + +void ntfs_index_entry_mark_dirty(struct ntfs_index_context *ictx); +int ntfs_index_add_filename(struct ntfs_inode *ni, struct file_name_attr *fn, u64 mref); +int ntfs_index_remove(struct ntfs_inode *ni, const void *key, const u32 keylen); +struct ntfs_inode *ntfs_ia_open(struct ntfs_index_context *icx, struct ntfs_inode *ni); +struct index_entry *ntfs_index_walk_down(struct index_entry *ie, struct ntfs_index_context *ictx); +struct index_entry *ntfs_index_next(struct index_entry *ie, struct ntfs_index_context *ictx); +int ntfs_index_rm(struct ntfs_index_context *icx); +void ntfs_index_ctx_reinit(struct ntfs_index_context *icx); +int ntfs_ie_add(struct ntfs_index_context *icx, struct index_entry *ie); +int ntfs_icx_ib_sync_write(struct ntfs_index_context *icx); + +#endif /* _LINUX_NTFS_INDEX_H */ diff --git a/fs/ntfs/inode.c b/fs/ntfs/inode.c new file mode 100644 index 000000000..7381a18cf --- /dev/null +++ b/fs/ntfs/inode.c @@ -0,0 +1,3815 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * NTFS kernel inode handling. + * + * Copyright (c) 2001-2014 Anton Altaparmakov and Tuxera Inc. + * Copyright (c) 2025 LG Electronics Co., Ltd. + */ + +#include +#include + +#include "lcnalloc.h" +#include "time.h" +#include "ntfs.h" +#include "index.h" +#include "attrlist.h" +#include "reparse.h" +#include "ea.h" +#include "attrib.h" +#include "iomap.h" +#include "object_id.h" + +/* + * ntfs_test_inode - compare two (possibly fake) inodes for equality + * @vi: vfs inode which to test + * @data: data which is being tested with + * + * Compare the ntfs attribute embedded in the ntfs specific part of the vfs + * inode @vi for equality with the ntfs attribute @data. + * + * If searching for the normal file/directory inode, set @na->type to AT_UNUSED. + * @na->name and @na->name_len are then ignored. + * + * Return 1 if the attributes match and 0 if not. + * + * NOTE: This function runs with the inode_hash_lock spin lock held so it is not + * allowed to sleep. + */ +int ntfs_test_inode(struct inode *vi, void *data) +{ + struct ntfs_attr *na = data; + struct ntfs_inode *ni = NTFS_I(vi); + + if (vi->i_ino != na->mft_no) + return 0; + + /* If !NInoAttr(ni), @vi is a normal file or directory inode. */ + if (likely(!NInoAttr(ni))) { + /* If not looking for a normal inode this is a mismatch. */ + if (unlikely(na->type != AT_UNUSED)) + return 0; + } else { + /* A fake inode describing an attribute. */ + if (ni->type != na->type) + return 0; + if (ni->name_len != na->name_len) + return 0; + if (na->name_len && memcmp(ni->name, na->name, + na->name_len * sizeof(__le16))) + return 0; + if (!ni->ext.base_ntfs_ino) + return 0; + } + + /* Match! */ + return 1; +} + +/* + * ntfs_init_locked_inode - initialize an inode + * @vi: vfs inode to initialize + * @data: data which to initialize @vi to + * + * Initialize the vfs inode @vi with the values from the ntfs attribute @data in + * order to enable ntfs_test_inode() to do its work. + * + * If initializing the normal file/directory inode, set @na->type to AT_UNUSED. + * In that case, @na->name and @na->name_len should be set to NULL and 0, + * respectively. Although that is not strictly necessary as + * ntfs_read_locked_inode() will fill them in later. + * + * Return 0 on success and error. + * + * NOTE: This function runs with the inode->i_lock spin lock held so it is not + * allowed to sleep. (Hence the GFP_ATOMIC allocation.) + */ +static int ntfs_init_locked_inode(struct inode *vi, void *data) +{ + struct ntfs_attr *na = data; + struct ntfs_inode *ni = NTFS_I(vi); + + vi->i_ino = (unsigned long)na->mft_no; + + if (na->type == AT_INDEX_ALLOCATION) + NInoSetMstProtected(ni); + else + ni->type = na->type; + + ni->name = na->name; + ni->name_len = na->name_len; + ni->folio = NULL; + atomic_set(&ni->count, 1); + + /* If initializing a normal inode, we are done. */ + if (likely(na->type == AT_UNUSED)) + return 0; + + /* It is a fake inode. */ + NInoSetAttr(ni); + + /* + * We have I30 global constant as an optimization as it is the name + * in >99.9% of named attributes! The other <0.1% incur a GFP_ATOMIC + * allocation but that is ok. And most attributes are unnamed anyway, + * thus the fraction of named attributes with name != I30 is actually + * absolutely tiny. + */ + if (na->name_len && na->name != I30) { + unsigned int i; + + i = na->name_len * sizeof(__le16); + ni->name = kmalloc(i + sizeof(__le16), GFP_ATOMIC); + if (!ni->name) + return -ENOMEM; + memcpy(ni->name, na->name, i); + ni->name[na->name_len] = 0; + } + return 0; +} + +static int ntfs_read_locked_inode(struct inode *vi); +static int ntfs_read_locked_attr_inode(struct inode *base_vi, struct inode *vi); +static int ntfs_read_locked_index_inode(struct inode *base_vi, + struct inode *vi); + +/* + * ntfs_iget - obtain a struct inode corresponding to a specific normal inode + * @sb: super block of mounted volume + * @mft_no: mft record number / inode number to obtain + * + * Obtain the struct inode corresponding to a specific normal inode (i.e. a + * file or directory). + * + * If the inode is in the cache, it is just returned with an increased + * reference count. Otherwise, a new struct inode is allocated and initialized, + * and finally ntfs_read_locked_inode() is called to read in the inode and + * fill in the remainder of the inode structure. + * + * Return the struct inode on success. Check the return value with IS_ERR() and + * if true, the function failed and the error code is obtained from PTR_ERR(). + */ +struct inode *ntfs_iget(struct super_block *sb, u64 mft_no) +{ + struct inode *vi; + int err; + struct ntfs_attr na; + + na.mft_no = mft_no; + na.type = AT_UNUSED; + na.name = NULL; + na.name_len = 0; + + vi = iget5_locked(sb, mft_no, ntfs_test_inode, + ntfs_init_locked_inode, &na); + if (unlikely(!vi)) + return ERR_PTR(-ENOMEM); + + err = 0; + + /* If this is a freshly allocated inode, need to read it now. */ + if (inode_state_read_once(vi) & I_NEW) { + err = ntfs_read_locked_inode(vi); + unlock_new_inode(vi); + } + /* + * There is no point in keeping bad inodes around if the failure was + * due to ENOMEM. We want to be able to retry again later. + */ + if (unlikely(err == -ENOMEM)) { + iput(vi); + vi = ERR_PTR(err); + } + return vi; +} + +/* + * ntfs_attr_iget - obtain a struct inode corresponding to an attribute + * @base_vi: vfs base inode containing the attribute + * @type: attribute type + * @name: Unicode name of the attribute (NULL if unnamed) + * @name_len: length of @name in Unicode characters (0 if unnamed) + * + * Obtain the (fake) struct inode corresponding to the attribute specified by + * @type, @name, and @name_len, which is present in the base mft record + * specified by the vfs inode @base_vi. + * + * If the attribute inode is in the cache, it is just returned with an + * increased reference count. Otherwise, a new struct inode is allocated and + * initialized, and finally ntfs_read_locked_attr_inode() is called to read the + * attribute and fill in the inode structure. + * + * Note, for index allocation attributes, you need to use ntfs_index_iget() + * instead of ntfs_attr_iget() as working with indices is a lot more complex. + * + * Return the struct inode of the attribute inode on success. Check the return + * value with IS_ERR() and if true, the function failed and the error code is + * obtained from PTR_ERR(). + */ +struct inode *ntfs_attr_iget(struct inode *base_vi, __le32 type, + __le16 *name, u32 name_len) +{ + struct inode *vi; + int err; + struct ntfs_attr na; + + /* Make sure no one calls ntfs_attr_iget() for indices. */ + WARN_ON(type == AT_INDEX_ALLOCATION); + + na.mft_no = base_vi->i_ino; + na.type = type; + na.name = name; + na.name_len = name_len; + + vi = iget5_locked(base_vi->i_sb, na.mft_no, ntfs_test_inode, + ntfs_init_locked_inode, &na); + if (unlikely(!vi)) + return ERR_PTR(-ENOMEM); + err = 0; + + /* If this is a freshly allocated inode, need to read it now. */ + if (inode_state_read_once(vi) & I_NEW) { + err = ntfs_read_locked_attr_inode(base_vi, vi); + unlock_new_inode(vi); + } + /* + * There is no point in keeping bad attribute inodes around. This also + * simplifies things in that we never need to check for bad attribute + * inodes elsewhere. + */ + if (unlikely(err)) { + iput(vi); + vi = ERR_PTR(err); + } + return vi; +} + +/* + * ntfs_index_iget - obtain a struct inode corresponding to an index + * @base_vi: vfs base inode containing the index related attributes + * @name: Unicode name of the index + * @name_len: length of @name in Unicode characters + * + * Obtain the (fake) struct inode corresponding to the index specified by @name + * and @name_len, which is present in the base mft record specified by the vfs + * inode @base_vi. + * + * If the index inode is in the cache, it is just returned with an increased + * reference count. Otherwise, a new struct inode is allocated and + * initialized, and finally ntfs_read_locked_index_inode() is called to read + * the index related attributes and fill in the inode structure. + * + * Return the struct inode of the index inode on success. Check the return + * value with IS_ERR() and if true, the function failed and the error code is + * obtained from PTR_ERR(). + */ +struct inode *ntfs_index_iget(struct inode *base_vi, __le16 *name, + u32 name_len) +{ + struct inode *vi; + int err; + struct ntfs_attr na; + + na.mft_no = base_vi->i_ino; + na.type = AT_INDEX_ALLOCATION; + na.name = name; + na.name_len = name_len; + + vi = iget5_locked(base_vi->i_sb, na.mft_no, ntfs_test_inode, + ntfs_init_locked_inode, &na); + if (unlikely(!vi)) + return ERR_PTR(-ENOMEM); + + err = 0; + + /* If this is a freshly allocated inode, need to read it now. */ + if (inode_state_read_once(vi) & I_NEW) { + err = ntfs_read_locked_index_inode(base_vi, vi); + unlock_new_inode(vi); + } + /* + * There is no point in keeping bad index inodes around. This also + * simplifies things in that we never need to check for bad index + * inodes elsewhere. + */ + if (unlikely(err)) { + iput(vi); + vi = ERR_PTR(err); + } + return vi; +} + +struct inode *ntfs_alloc_big_inode(struct super_block *sb) +{ + struct ntfs_inode *ni; + + ntfs_debug("Entering."); + ni = alloc_inode_sb(sb, ntfs_big_inode_cache, GFP_NOFS); + if (likely(ni != NULL)) { + ni->state = 0; + ni->type = 0; + ni->mft_no = 0; + return VFS_I(ni); + } + ntfs_error(sb, "Allocation of NTFS big inode structure failed."); + return NULL; +} + +void ntfs_free_big_inode(struct inode *inode) +{ + kmem_cache_free(ntfs_big_inode_cache, NTFS_I(inode)); +} + +static int ntfs_non_resident_dealloc_clusters(struct ntfs_inode *ni) +{ + struct super_block *sb = ni->vol->sb; + struct ntfs_attr_search_ctx *actx; + int err = 0; + + actx = ntfs_attr_get_search_ctx(ni, NULL); + if (!actx) + return -ENOMEM; + WARN_ON(actx->mrec->link_count != 0); + + /** + * ntfs_truncate_vfs cannot be called in evict() context due + * to some limitations, which are the @ni vfs inode is marked + * with I_FREEING, and etc. + */ + if (NInoRunlistDirty(ni)) { + err = ntfs_cluster_free_from_rl(ni->vol, ni->runlist.rl); + if (err) + ntfs_error(sb, + "Failed to free clusters. Leaving inconsistent metadata.\n"); + } + + while ((err = ntfs_attrs_walk(actx)) == 0) { + if (actx->attr->non_resident && + (!NInoRunlistDirty(ni) || actx->attr->type != AT_DATA)) { + struct runlist_element *rl; + size_t new_rl_count; + + rl = ntfs_mapping_pairs_decompress(ni->vol, actx->attr, NULL, + &new_rl_count); + if (IS_ERR(rl)) { + err = PTR_ERR(rl); + ntfs_error(sb, + "Failed to decompress runlist. Leaving inconsistent metadata.\n"); + continue; + } + + err = ntfs_cluster_free_from_rl(ni->vol, rl); + if (err) + ntfs_error(sb, + "Failed to free attribute clusters. Leaving inconsistent metadata.\n"); + kvfree(rl); + } + } + + ntfs_release_dirty_clusters(ni->vol, ni->i_dealloc_clusters); + ntfs_attr_put_search_ctx(actx); + return err; +} + +int ntfs_drop_big_inode(struct inode *inode) +{ + struct ntfs_inode *ni = NTFS_I(inode); + + if (!inode_unhashed(inode) && inode_state_read_once(inode) & I_SYNC) { + if (ni->type == AT_DATA || ni->type == AT_INDEX_ALLOCATION) { + if (!inode->i_nlink) { + struct ntfs_inode *ni = NTFS_I(inode); + + if (ni->data_size == 0) + return 0; + + /* To avoid evict_inode call simultaneously */ + atomic_inc(&inode->i_count); + spin_unlock(&inode->i_lock); + + truncate_setsize(VFS_I(ni), 0); + ntfs_truncate_vfs(VFS_I(ni), 0, 1); + + sb_start_intwrite(inode->i_sb); + i_size_write(inode, 0); + ni->allocated_size = ni->initialized_size = ni->data_size = 0; + + truncate_inode_pages_final(inode->i_mapping); + sb_end_intwrite(inode->i_sb); + + spin_lock(&inode->i_lock); + atomic_dec(&inode->i_count); + } + } + return 0; + } + + return inode_generic_drop(inode); +} + +static inline struct ntfs_inode *ntfs_alloc_extent_inode(void) +{ + struct ntfs_inode *ni; + + ntfs_debug("Entering."); + ni = kmem_cache_alloc(ntfs_inode_cache, GFP_NOFS); + if (likely(ni != NULL)) { + ni->state = 0; + return ni; + } + ntfs_error(NULL, "Allocation of NTFS inode structure failed."); + return NULL; +} + +static void ntfs_destroy_extent_inode(struct ntfs_inode *ni) +{ + ntfs_debug("Entering."); + + if (!atomic_dec_and_test(&ni->count)) + WARN_ON(1); + if (ni->folio) + folio_put(ni->folio); + kfree(ni->mrec); + kmem_cache_free(ntfs_inode_cache, ni); +} + +static struct lock_class_key attr_inode_mrec_lock_class; +static struct lock_class_key attr_list_inode_mrec_lock_class; + +/* + * The attribute runlist lock has separate locking rules from the + * normal runlist lock, so split the two lock-classes: + */ +static struct lock_class_key attr_list_rl_lock_class; + +/* + * __ntfs_init_inode - initialize ntfs specific part of an inode + * @sb: super block of mounted volume + * @ni: freshly allocated ntfs inode which to initialize + * + * Initialize an ntfs inode to defaults. + * + * NOTE: ni->mft_no, ni->state, ni->type, ni->name, and ni->name_len are left + * untouched. Make sure to initialize them elsewhere. + */ +void __ntfs_init_inode(struct super_block *sb, struct ntfs_inode *ni) +{ + ntfs_debug("Entering."); + rwlock_init(&ni->size_lock); + ni->initialized_size = ni->allocated_size = 0; + ni->seq_no = 0; + atomic_set(&ni->count, 1); + ni->vol = NTFS_SB(sb); + ntfs_init_runlist(&ni->runlist); + mutex_init(&ni->mrec_lock); + if (ni->type == AT_ATTRIBUTE_LIST) { + lockdep_set_class(&ni->mrec_lock, + &attr_list_inode_mrec_lock_class); + lockdep_set_class(&ni->runlist.lock, + &attr_list_rl_lock_class); + } else if (NInoAttr(ni)) { + lockdep_set_class(&ni->mrec_lock, + &attr_inode_mrec_lock_class); + } + + ni->folio = NULL; + ni->folio_ofs = 0; + ni->mrec = NULL; + ni->attr_list_size = 0; + ni->attr_list = NULL; + ni->itype.index.block_size = 0; + ni->itype.index.vcn_size = 0; + ni->itype.index.collation_rule = 0; + ni->itype.index.block_size_bits = 0; + ni->itype.index.vcn_size_bits = 0; + mutex_init(&ni->extent_lock); + ni->nr_extents = 0; + ni->ext.base_ntfs_ino = NULL; + ni->flags = 0; + ni->mft_lcn[0] = LCN_RL_NOT_MAPPED; + ni->mft_lcn_count = 0; + ni->reparse_tag = 0; + ni->reparse_flags = 0; + ni->target = NULL; + ni->i_dealloc_clusters = 0; +} + +/* + * Extent inodes get MFT-mapped in a nested way, while the base inode + * is still mapped. Teach this nesting to the lock validator by creating + * a separate class for nested inode's mrec_lock's: + */ +static struct lock_class_key extent_inode_mrec_lock_key; + +inline struct ntfs_inode *ntfs_new_extent_inode(struct super_block *sb, + u64 mft_no) +{ + struct ntfs_inode *ni = ntfs_alloc_extent_inode(); + + ntfs_debug("Entering."); + if (likely(ni != NULL)) { + __ntfs_init_inode(sb, ni); + lockdep_set_class(&ni->mrec_lock, &extent_inode_mrec_lock_key); + ni->mft_no = mft_no; + ni->type = AT_UNUSED; + ni->name = NULL; + ni->name_len = 0; + } + return ni; +} + +/* + * ntfs_is_extended_system_file - check if a file is in the $Extend directory + * @ctx: initialized attribute search context + * + * Search all file name attributes in the inode described by the attribute + * search context @ctx and check if any of the names are in the $Extend system + * directory. + * + * Return values: + * 3: file is $ObjId in $Extend directory + * 2: file is $Reparse in $Extend directory + * 1: file is in $Extend directory + * 0: file is not in $Extend directory + * -errno: failed to determine if the file is in the $Extend directory + */ +static int ntfs_is_extended_system_file(struct ntfs_attr_search_ctx *ctx) +{ + int nr_links, err; + + /* Restart search. */ + ntfs_attr_reinit_search_ctx(ctx); + + /* Get number of hard links. */ + nr_links = le16_to_cpu(ctx->mrec->link_count); + + /* Loop through all hard links. */ + while (!(err = ntfs_attr_lookup(AT_FILE_NAME, NULL, 0, 0, 0, NULL, 0, + ctx))) { + struct file_name_attr *file_name_attr; + struct attr_record *attr = ctx->attr; + u8 *p, *p2; + + nr_links--; + /* + * Maximum sanity checking as we are called on an inode that + * we suspect might be corrupt. + */ + p = (u8 *)attr + le32_to_cpu(attr->length); + if (p < (u8 *)ctx->mrec || (u8 *)p > (u8 *)ctx->mrec + + le32_to_cpu(ctx->mrec->bytes_in_use)) { +err_corrupt_attr: + ntfs_error(ctx->ntfs_ino->vol->sb, + "Corrupt file name attribute. You should run chkdsk."); + return -EIO; + } + if (attr->non_resident) { + ntfs_error(ctx->ntfs_ino->vol->sb, + "Non-resident file name. You should run chkdsk."); + return -EIO; + } + if (attr->flags) { + ntfs_error(ctx->ntfs_ino->vol->sb, + "File name with invalid flags. You should run chkdsk."); + return -EIO; + } + if (!(attr->data.resident.flags & RESIDENT_ATTR_IS_INDEXED)) { + ntfs_error(ctx->ntfs_ino->vol->sb, + "Unindexed file name. You should run chkdsk."); + return -EIO; + } + file_name_attr = (struct file_name_attr *)((u8 *)attr + + le16_to_cpu(attr->data.resident.value_offset)); + p2 = (u8 *)file_name_attr + le32_to_cpu(attr->data.resident.value_length); + if (p2 < (u8 *)attr || p2 > p) + goto err_corrupt_attr; + /* This attribute is ok, but is it in the $Extend directory? */ + if (MREF_LE(file_name_attr->parent_directory) == FILE_Extend) { + unsigned char *s; + + s = ntfs_attr_name_get(ctx->ntfs_ino->vol, + file_name_attr->file_name, + file_name_attr->file_name_length); + if (!s) + return 1; + if (!strcmp("$Reparse", s)) { + ntfs_attr_name_free(&s); + return 2; /* it's reparse point file */ + } + if (!strcmp("$ObjId", s)) { + ntfs_attr_name_free(&s); + return 3; /* it's object id file */ + } + ntfs_attr_name_free(&s); + return 1; /* YES, it's an extended system file. */ + } + } + if (unlikely(err != -ENOENT)) + return err; + if (unlikely(nr_links)) { + ntfs_error(ctx->ntfs_ino->vol->sb, + "Inode hard link count doesn't match number of name attributes. You should run chkdsk."); + return -EIO; + } + return 0; /* NO, it is not an extended system file. */ +} + +static struct lock_class_key ntfs_dir_inval_lock_key; + +void ntfs_set_vfs_operations(struct inode *inode, mode_t mode, dev_t dev) +{ + if (S_ISDIR(mode)) { + if (!NInoAttr(NTFS_I(inode))) { + inode->i_op = &ntfs_dir_inode_ops; + inode->i_fop = &ntfs_dir_ops; + } + inode->i_mapping->a_ops = &ntfs_aops; + lockdep_set_class(&inode->i_mapping->invalidate_lock, + &ntfs_dir_inval_lock_key); + } else if (S_ISLNK(mode)) { + inode->i_op = &ntfs_symlink_inode_operations; + inode->i_mapping->a_ops = &ntfs_aops; + } else if (S_ISCHR(mode) || S_ISBLK(mode) || S_ISFIFO(mode) || S_ISSOCK(mode)) { + inode->i_op = &ntfs_special_inode_operations; + init_special_inode(inode, inode->i_mode, dev); + } else { + if (!NInoAttr(NTFS_I(inode))) { + inode->i_op = &ntfs_file_inode_ops; + inode->i_fop = &ntfs_file_ops; + } + if (inode->i_ino == FILE_MFT) + inode->i_mapping->a_ops = &ntfs_mft_aops; + else + inode->i_mapping->a_ops = &ntfs_aops; + } +} + +/* + * ntfs_read_locked_inode - read an inode from its device + * @vi: inode to read + * + * ntfs_read_locked_inode() is called from ntfs_iget() to read the inode + * described by @vi into memory from the device. + * + * The only fields in @vi that we need to/can look at when the function is + * called are i_sb, pointing to the mounted device's super block, and i_ino, + * the number of the inode to load. + * + * ntfs_read_locked_inode() maps, pins and locks the mft record number i_ino + * for reading and sets up the necessary @vi fields as well as initializing + * the ntfs inode. + * + * Q: What locks are held when the function is called? + * A: i_state has I_NEW set, hence the inode is locked, also + * i_count is set to 1, so it is not going to go away + * i_flags is set to 0 and we have no business touching it. Only an ioctl() + * is allowed to write to them. We should of course be honouring them but + * we need to do that using the IS_* macros defined in include/linux/fs.h. + * In any case ntfs_read_locked_inode() has nothing to do with i_flags. + * + * Return 0 on success and -errno on error. + */ +static int ntfs_read_locked_inode(struct inode *vi) +{ + struct ntfs_volume *vol = NTFS_SB(vi->i_sb); + struct ntfs_inode *ni = NTFS_I(vi); + struct mft_record *m; + struct attr_record *a; + struct standard_information *si; + struct ntfs_attr_search_ctx *ctx; + int err = 0; + __le16 *name = I30; + unsigned int name_len = 4, flags = 0; + int extend_sys = 0; + dev_t dev = 0; + bool vol_err = true; + + ntfs_debug("Entering for i_ino 0x%llx.", ni->mft_no); + + if (uid_valid(vol->uid)) { + vi->i_uid = vol->uid; + flags |= NTFS_VOL_UID; + } else + vi->i_uid = GLOBAL_ROOT_UID; + + if (gid_valid(vol->gid)) { + vi->i_gid = vol->gid; + flags |= NTFS_VOL_GID; + } else + vi->i_gid = GLOBAL_ROOT_GID; + + vi->i_mode = 0777; + + /* + * Initialize the ntfs specific part of @vi special casing + * FILE_MFT which we need to do at mount time. + */ + if (vi->i_ino != FILE_MFT) + ntfs_init_big_inode(vi); + + m = map_mft_record(ni); + if (IS_ERR(m)) { + err = PTR_ERR(m); + goto err_out; + } + + ctx = ntfs_attr_get_search_ctx(ni, m); + if (!ctx) { + err = -ENOMEM; + goto unm_err_out; + } + + if (!(m->flags & MFT_RECORD_IN_USE)) { + err = -ENOENT; + vol_err = false; + goto unm_err_out; + } + + if (m->base_mft_record) { + ntfs_error(vi->i_sb, "Inode is an extent inode!"); + goto unm_err_out; + } + + /* Transfer information from mft record into vfs and ntfs inodes. */ + vi->i_generation = ni->seq_no = le16_to_cpu(m->sequence_number); + + if (le16_to_cpu(m->link_count) < 1) { + ntfs_error(vi->i_sb, "Inode link count is 0!"); + goto unm_err_out; + } + set_nlink(vi, le16_to_cpu(m->link_count)); + + /* If read-only, no one gets write permissions. */ + if (IS_RDONLY(vi)) + vi->i_mode &= ~0222; + + /* + * Find the standard information attribute in the mft record. At this + * stage we haven't setup the attribute list stuff yet, so this could + * in fact fail if the standard information is in an extent record, but + * I don't think this actually ever happens. + */ + ntfs_attr_reinit_search_ctx(ctx); + err = ntfs_attr_lookup(AT_STANDARD_INFORMATION, NULL, 0, 0, 0, NULL, 0, + ctx); + if (unlikely(err)) { + if (err == -ENOENT) + ntfs_error(vi->i_sb, "$STANDARD_INFORMATION attribute is missing."); + goto unm_err_out; + } + a = ctx->attr; + /* Get the standard information attribute value. */ + si = (struct standard_information *)((u8 *)a + + le16_to_cpu(a->data.resident.value_offset)); + + /* Transfer information from the standard information into vi. */ + /* + * Note: The i_?times do not quite map perfectly onto the NTFS times, + * but they are close enough, and in the end it doesn't really matter + * that much... + */ + /* + * mtime is the last change of the data within the file. Not changed + * when only metadata is changed, e.g. a rename doesn't affect mtime. + */ + ni->i_crtime = ntfs2utc(si->creation_time); + + inode_set_mtime_to_ts(vi, ntfs2utc(si->last_data_change_time)); + /* + * ctime is the last change of the metadata of the file. This obviously + * always changes, when mtime is changed. ctime can be changed on its + * own, mtime is then not changed, e.g. when a file is renamed. + */ + inode_set_ctime_to_ts(vi, ntfs2utc(si->last_mft_change_time)); + /* + * Last access to the data within the file. Not changed during a rename + * for example but changed whenever the file is written to. + */ + inode_set_atime_to_ts(vi, ntfs2utc(si->last_access_time)); + ni->flags = si->file_attributes; + + /* Find the attribute list attribute if present. */ + ntfs_attr_reinit_search_ctx(ctx); + err = ntfs_attr_lookup(AT_ATTRIBUTE_LIST, NULL, 0, 0, 0, NULL, 0, ctx); + if (err) { + if (unlikely(err != -ENOENT)) { + ntfs_error(vi->i_sb, "Failed to lookup attribute list attribute."); + goto unm_err_out; + } + } else { + if (vi->i_ino == FILE_MFT) + goto skip_attr_list_load; + ntfs_debug("Attribute list found in inode 0x%llx.", ni->mft_no); + NInoSetAttrList(ni); + a = ctx->attr; + if (a->flags & ATTR_COMPRESSION_MASK) { + ntfs_error(vi->i_sb, + "Attribute list attribute is compressed."); + goto unm_err_out; + } + if (a->flags & ATTR_IS_ENCRYPTED || + a->flags & ATTR_IS_SPARSE) { + if (a->non_resident) { + ntfs_error(vi->i_sb, + "Non-resident attribute list attribute is encrypted/sparse."); + goto unm_err_out; + } + ntfs_warning(vi->i_sb, + "Resident attribute list attribute in inode 0x%llx is marked encrypted/sparse which is not true. However, Windows allows this and chkdsk does not detect or correct it so we will just ignore the invalid flags and pretend they are not set.", + ni->mft_no); + } + /* Now allocate memory for the attribute list. */ + ni->attr_list_size = (u32)ntfs_attr_size(a); + if (!ni->attr_list_size) { + ntfs_error(vi->i_sb, "Attr_list_size is zero"); + goto unm_err_out; + } + ni->attr_list = kvzalloc(ni->attr_list_size, GFP_NOFS); + if (!ni->attr_list) { + ntfs_error(vi->i_sb, + "Not enough memory to allocate buffer for attribute list."); + err = -ENOMEM; + goto unm_err_out; + } + if (a->non_resident) { + NInoSetAttrListNonResident(ni); + if (a->data.non_resident.lowest_vcn) { + ntfs_error(vi->i_sb, "Attribute list has non zero lowest_vcn."); + goto unm_err_out; + } + + /* Now load the attribute list. */ + err = load_attribute_list(ni, ni->attr_list, ni->attr_list_size); + if (err) { + ntfs_error(vi->i_sb, "Failed to load attribute list attribute."); + goto unm_err_out; + } + } else /* if (!a->non_resident) */ { + /* Now copy the attribute list. */ + memcpy(ni->attr_list, (u8 *)a + le16_to_cpu( + a->data.resident.value_offset), + le32_to_cpu( + a->data.resident.value_length)); + /* A resident list is not validated on load; check it now. */ + if (!ntfs_attr_list_is_valid(ni->attr_list, + ni->attr_list_size)) { + ntfs_error(vi->i_sb, "Corrupt attribute list."); + goto unm_err_out; + } + } + } +skip_attr_list_load: + err = ntfs_attr_lookup(AT_EA_INFORMATION, NULL, 0, 0, 0, NULL, 0, ctx); + if (!err) { + NInoSetHasEA(ni); + ntfs_ea_get_wsl_inode(vi, &dev, flags); + } + + if (ni->flags & FILE_ATTR_REPARSE_POINT) { + unsigned int mode; + + mode = ntfs_make_symlink(ni); + if (mode) + vi->i_mode |= mode; + else { + vi->i_mode &= ~S_IFLNK; + if (m->flags & MFT_RECORD_IS_DIRECTORY) + vi->i_mode |= S_IFDIR; + else + vi->i_mode |= S_IFREG; + } + } else if (m->flags & MFT_RECORD_IS_DIRECTORY) { + vi->i_mode |= S_IFDIR; + } else { + vi->i_mode |= S_IFREG; + } + + if (S_ISDIR(vi->i_mode)) { + /* + * Apply the directory permissions mask set in the mount + * options. + */ + vi->i_mode &= ~vol->dmask; + /* Things break without this kludge! */ + if (vi->i_nlink > 1) + set_nlink(vi, 1); + } else { + /* Apply the file permissions mask set in the mount options. */ + vi->i_mode &= ~vol->fmask; + } + + /* + * If an attribute list is present we now have the attribute list value + * in ntfs_ino->attr_list and it is ntfs_ino->attr_list_size bytes. + */ + if (m->flags & MFT_RECORD_IS_DIRECTORY) { + struct index_root *ir; + +view_index_meta: + /* It is a directory, find index root attribute. */ + ntfs_attr_reinit_search_ctx(ctx); + err = ntfs_attr_lookup(AT_INDEX_ROOT, name, name_len, CASE_SENSITIVE, + 0, NULL, 0, ctx); + if (unlikely(err)) { + if (err == -ENOENT) + ntfs_error(vi->i_sb, "$INDEX_ROOT attribute is missing."); + goto unm_err_out; + } + a = ctx->attr; + /* Set up the state. */ + if (unlikely(a->non_resident)) { + ntfs_error(vol->sb, + "$INDEX_ROOT attribute is not resident."); + goto unm_err_out; + } + /* Ensure the attribute name is placed before the value. */ + if (unlikely(a->name_length && (le16_to_cpu(a->name_offset) >= + le16_to_cpu(a->data.resident.value_offset)))) { + ntfs_error(vol->sb, + "$INDEX_ROOT attribute name is placed after the attribute value."); + goto unm_err_out; + } + /* + * Compressed/encrypted index root just means that the newly + * created files in that directory should be created compressed/ + * encrypted. However index root cannot be both compressed and + * encrypted. + */ + if (a->flags & ATTR_COMPRESSION_MASK) { + NInoSetCompressed(ni); + ni->flags |= FILE_ATTR_COMPRESSED; + } + if (a->flags & ATTR_IS_ENCRYPTED) { + if (a->flags & ATTR_COMPRESSION_MASK) { + ntfs_error(vi->i_sb, "Found encrypted and compressed attribute."); + goto unm_err_out; + } + NInoSetEncrypted(ni); + ni->flags |= FILE_ATTR_ENCRYPTED; + } + if (a->flags & ATTR_IS_SPARSE) { + NInoSetSparse(ni); + ni->flags |= FILE_ATTR_SPARSE_FILE; + } + ir = (struct index_root *)((u8 *)a + + le16_to_cpu(a->data.resident.value_offset)); + if (ntfs_index_root_inconsistent(ni->vol, a, ir, ni->mft_no) || + ntfs_index_entries_inconsistent(ni->vol, &ir->index, + ir->collation_rule, ni->mft_no)) { + ntfs_error(vi->i_sb, "Directory index is corrupt."); + goto unm_err_out; + } + + if (extend_sys) { + if (ir->type) { + ntfs_error(vi->i_sb, "Indexed attribute is not zero."); + goto unm_err_out; + } + } else { + if (ir->type != AT_FILE_NAME) { + ntfs_error(vi->i_sb, "Indexed attribute is not $FILE_NAME."); + goto unm_err_out; + } + + if (ir->collation_rule != COLLATION_FILE_NAME) { + ntfs_error(vi->i_sb, + "Index collation rule is not COLLATION_FILE_NAME."); + goto unm_err_out; + } + } + + ni->itype.index.collation_rule = ir->collation_rule; + ni->itype.index.block_size = le32_to_cpu(ir->index_block_size); + if (ni->itype.index.block_size & + (ni->itype.index.block_size - 1)) { + ntfs_error(vi->i_sb, "Index block size (%u) is not a power of two.", + ni->itype.index.block_size); + goto unm_err_out; + } + if (ni->itype.index.block_size > PAGE_SIZE) { + ntfs_error(vi->i_sb, + "Index block size (%u) > PAGE_SIZE (%ld) is not supported.", + ni->itype.index.block_size, + PAGE_SIZE); + err = -EOPNOTSUPP; + goto unm_err_out; + } + if (ni->itype.index.block_size < NTFS_BLOCK_SIZE) { + ntfs_error(vi->i_sb, + "Index block size (%u) < NTFS_BLOCK_SIZE (%i) is not supported.", + ni->itype.index.block_size, + NTFS_BLOCK_SIZE); + err = -EOPNOTSUPP; + goto unm_err_out; + } + ni->itype.index.block_size_bits = + ffs(ni->itype.index.block_size) - 1; + /* Determine the size of a vcn in the directory index. */ + if (vol->cluster_size <= ni->itype.index.block_size) { + ni->itype.index.vcn_size = vol->cluster_size; + ni->itype.index.vcn_size_bits = vol->cluster_size_bits; + } else { + ni->itype.index.vcn_size = vol->sector_size; + ni->itype.index.vcn_size_bits = vol->sector_size_bits; + } + + /* Setup the index allocation attribute, even if not present. */ + ni->type = AT_INDEX_ROOT; + ni->name = name; + ni->name_len = name_len; + vi->i_size = ni->initialized_size = ni->data_size = + le32_to_cpu(a->data.resident.value_length); + ni->allocated_size = (ni->data_size + 7) & ~7; + /* We are done with the mft record, so we release it. */ + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(ni); + m = NULL; + ctx = NULL; + /* Setup the operations for this inode. */ + ntfs_set_vfs_operations(vi, vi->i_mode, 0); + if (ir->index.flags & LARGE_INDEX) + NInoSetIndexAllocPresent(ni); + } else { + /* It is a file. */ + ntfs_attr_reinit_search_ctx(ctx); + + /* Setup the data attribute, even if not present. */ + ni->type = AT_DATA; + ni->name = AT_UNNAMED; + ni->name_len = 0; + + /* Find first extent of the unnamed data attribute. */ + err = ntfs_attr_lookup(AT_DATA, NULL, 0, 0, 0, NULL, 0, ctx); + if (unlikely(err)) { + vi->i_size = ni->initialized_size = + ni->allocated_size = 0; + if (err != -ENOENT) { + ntfs_error(vi->i_sb, "Failed to lookup $DATA attribute."); + goto unm_err_out; + } + /* + * FILE_Secure does not have an unnamed $DATA + * attribute, so we special case it here. + */ + if (vi->i_ino == FILE_Secure) + goto no_data_attr_special_case; + /* + * Most if not all the system files in the $Extend + * system directory do not have unnamed data + * attributes so we need to check if the parent + * directory of the file is FILE_Extend and if it is + * ignore this error. To do this we need to get the + * name of this inode from the mft record as the name + * contains the back reference to the parent directory. + */ + extend_sys = ntfs_is_extended_system_file(ctx); + if (extend_sys > 0) { + if (m->flags & MFT_RECORD_IS_VIEW_INDEX) { + if (extend_sys == 2) { + name = reparse_index_name; + name_len = 2; + goto view_index_meta; + } else if (extend_sys == 3) { + name = objid_index_name; + name_len = 2; + goto view_index_meta; + } + } + goto no_data_attr_special_case; + } + + err = extend_sys; + ntfs_error(vi->i_sb, "$DATA attribute is missing, err : %d", err); + goto unm_err_out; + } + a = ctx->attr; + /* Setup the state. */ + if (a->flags & (ATTR_COMPRESSION_MASK | ATTR_IS_SPARSE)) { + if (a->flags & ATTR_COMPRESSION_MASK) { + NInoSetCompressed(ni); + ni->flags |= FILE_ATTR_COMPRESSED; + if (vol->cluster_size > 4096) { + ntfs_error(vi->i_sb, + "Found compressed data but compression is disabled due to cluster size (%i) > 4kiB.", + vol->cluster_size); + goto unm_err_out; + } + if ((a->flags & ATTR_COMPRESSION_MASK) + != ATTR_IS_COMPRESSED) { + ntfs_error(vi->i_sb, + "Found unknown compression method or corrupt file."); + goto unm_err_out; + } + } + if (a->flags & ATTR_IS_SPARSE) { + NInoSetSparse(ni); + ni->flags |= FILE_ATTR_SPARSE_FILE; + } + } + if (a->flags & ATTR_IS_ENCRYPTED) { + if (NInoCompressed(ni)) { + ntfs_error(vi->i_sb, "Found encrypted and compressed data."); + goto unm_err_out; + } + NInoSetEncrypted(ni); + ni->flags |= FILE_ATTR_ENCRYPTED; + } + if (a->non_resident) { + NInoSetNonResident(ni); + if (NInoCompressed(ni) || NInoSparse(ni)) { + if (NInoCompressed(ni) && + a->data.non_resident.compression_unit != 4) { + ntfs_error(vi->i_sb, + "Found non-standard compression unit (%u instead of 4). Cannot handle this.", + a->data.non_resident.compression_unit); + err = -EOPNOTSUPP; + goto unm_err_out; + } + + if (NInoSparse(ni) && + a->data.non_resident.compression_unit && + a->data.non_resident.compression_unit != + vol->sparse_compression_unit) { + ntfs_error(vi->i_sb, + "Found non-standard compression unit (%u instead of 0 or %d). Cannot handle this.", + a->data.non_resident.compression_unit, + vol->sparse_compression_unit); + err = -EOPNOTSUPP; + goto unm_err_out; + } + + + if (a->data.non_resident.compression_unit) { + ni->itype.compressed.block_size = 1U << + (a->data.non_resident.compression_unit + + vol->cluster_size_bits); + ni->itype.compressed.block_size_bits = + ffs(ni->itype.compressed.block_size) - 1; + ni->itype.compressed.block_clusters = + 1U << a->data.non_resident.compression_unit; + } else { + ni->itype.compressed.block_size = 0; + ni->itype.compressed.block_size_bits = + 0; + ni->itype.compressed.block_clusters = + 0; + } + ni->itype.compressed.size = le64_to_cpu( + a->data.non_resident.compressed_size); + } + if (a->data.non_resident.lowest_vcn) { + ntfs_error(vi->i_sb, + "First extent of $DATA attribute has non zero lowest_vcn."); + goto unm_err_out; + } + vi->i_size = ni->data_size = le64_to_cpu(a->data.non_resident.data_size); + ni->initialized_size = le64_to_cpu(a->data.non_resident.initialized_size); + ni->allocated_size = le64_to_cpu(a->data.non_resident.allocated_size); + } else { /* Resident attribute. */ + vi->i_size = ni->data_size = ni->initialized_size = le32_to_cpu( + a->data.resident.value_length); + ni->allocated_size = le32_to_cpu(a->length) - + le16_to_cpu( + a->data.resident.value_offset); + if (vi->i_size > ni->allocated_size) { + ntfs_error(vi->i_sb, + "Resident data attribute is corrupt (size exceeds allocation)."); + goto unm_err_out; + } + } +no_data_attr_special_case: + /* We are done with the mft record, so we release it. */ + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(ni); + m = NULL; + ctx = NULL; + /* Setup the operations for this inode. */ + ntfs_set_vfs_operations(vi, vi->i_mode, dev); + } + + if (NVolSysImmutable(vol) && (ni->flags & FILE_ATTR_SYSTEM) && + !S_ISFIFO(vi->i_mode) && !S_ISSOCK(vi->i_mode) && !S_ISLNK(vi->i_mode)) + vi->i_flags |= S_IMMUTABLE; + + /* + * System files such as $Bitmap and $MFT are maintained by the driver + * itself, and writing them from userspace corrupts the volume. + * Always make them immutable regardless of the sys_immutable option. + * Directories are skipped so the root and $Extend stay usable. + */ + if (ni->mft_no < FILE_first_user && S_ISREG(vi->i_mode)) + vi->i_flags |= S_IMMUTABLE; + + /* + * The number of 512-byte blocks used on disk (for stat). This is in so + * far inaccurate as it doesn't account for any named streams or other + * special non-resident attributes, but that is how Windows works, too, + * so we are at least consistent with Windows, if not entirely + * consistent with the Linux Way. Doing it the Linux Way would cause a + * significant slowdown as it would involve iterating over all + * attributes in the mft record and adding the allocated/compressed + * sizes of all non-resident attributes present to give us the Linux + * correct size that should go into i_blocks (after division by 512). + */ + if (S_ISREG(vi->i_mode) && (NInoCompressed(ni) || NInoSparse(ni))) + vi->i_blocks = ni->itype.compressed.size >> 9; + else + vi->i_blocks = ni->allocated_size >> 9; + + if (S_ISLNK(vi->i_mode) && ni->target) + vi->i_size = strlen(ni->target); + + ntfs_debug("Done."); + return 0; +unm_err_out: + if (!err) + err = -EIO; + if (ctx) + ntfs_attr_put_search_ctx(ctx); + if (m) + unmap_mft_record(ni); +err_out: + if (err != -EOPNOTSUPP && err != -ENOMEM && vol_err == true) { + ntfs_error(vol->sb, + "Failed with error code %i. Marking corrupt inode 0x%llx as bad. Run chkdsk.", + err, ni->mft_no); + NVolSetErrors(vol); + } + return err; +} + +/* + * ntfs_read_locked_attr_inode - read an attribute inode from its base inode + * @base_vi: base inode + * @vi: attribute inode to read + * + * ntfs_read_locked_attr_inode() is called from ntfs_attr_iget() to read the + * attribute inode described by @vi into memory from the base mft record + * described by @base_ni. + * + * ntfs_read_locked_attr_inode() maps, pins and locks the base inode for + * reading and looks up the attribute described by @vi before setting up the + * necessary fields in @vi as well as initializing the ntfs inode. + * + * Q: What locks are held when the function is called? + * A: i_state has I_NEW set, hence the inode is locked, also + * i_count is set to 1, so it is not going to go away + * + * Return 0 on success and -errno on error. + * + * Note this cannot be called for AT_INDEX_ALLOCATION. + */ +static int ntfs_read_locked_attr_inode(struct inode *base_vi, struct inode *vi) +{ + struct ntfs_volume *vol = NTFS_SB(vi->i_sb); + struct ntfs_inode *ni = NTFS_I(vi), *base_ni = NTFS_I(base_vi); + struct mft_record *m; + struct attr_record *a; + struct ntfs_attr_search_ctx *ctx; + int err = 0; + + ntfs_debug("Entering for i_ino 0x%llx.", ni->mft_no); + + ntfs_init_big_inode(vi); + + /* Just mirror the values from the base inode. */ + vi->i_uid = base_vi->i_uid; + vi->i_gid = base_vi->i_gid; + set_nlink(vi, base_vi->i_nlink); + inode_set_mtime_to_ts(vi, inode_get_mtime(base_vi)); + inode_set_ctime_to_ts(vi, inode_get_ctime(base_vi)); + inode_set_atime_to_ts(vi, inode_get_atime(base_vi)); + vi->i_generation = ni->seq_no = base_ni->seq_no; + + /* Set inode type to zero but preserve permissions. */ + vi->i_mode = base_vi->i_mode & ~S_IFMT; + + m = map_mft_record(base_ni); + if (IS_ERR(m)) { + err = PTR_ERR(m); + goto err_out; + } + ctx = ntfs_attr_get_search_ctx(base_ni, m); + if (!ctx) { + err = -ENOMEM; + goto unm_err_out; + } + /* Find the attribute. */ + err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len, + CASE_SENSITIVE, 0, NULL, 0, ctx); + if (unlikely(err)) + goto unm_err_out; + a = ctx->attr; + if (a->flags & (ATTR_COMPRESSION_MASK | ATTR_IS_SPARSE)) { + if (a->flags & ATTR_COMPRESSION_MASK) { + NInoSetCompressed(ni); + ni->flags |= FILE_ATTR_COMPRESSED; + if ((ni->type != AT_DATA) || (ni->type == AT_DATA && + ni->name_len)) { + ntfs_error(vi->i_sb, + "Found compressed non-data or named data attribute."); + goto unm_err_out; + } + if (vol->cluster_size > 4096) { + ntfs_error(vi->i_sb, + "Found compressed attribute but compression is disabled due to cluster size (%i) > 4kiB.", + vol->cluster_size); + goto unm_err_out; + } + if ((a->flags & ATTR_COMPRESSION_MASK) != + ATTR_IS_COMPRESSED) { + ntfs_error(vi->i_sb, "Found unknown compression method."); + goto unm_err_out; + } + } + /* + * The compressed/sparse flag set in an index root just means + * to compress all files. + */ + if (NInoMstProtected(ni) && ni->type != AT_INDEX_ROOT) { + ntfs_error(vi->i_sb, + "Found mst protected attribute but the attribute is %s.", + NInoCompressed(ni) ? "compressed" : "sparse"); + goto unm_err_out; + } + if (a->flags & ATTR_IS_SPARSE) { + NInoSetSparse(ni); + ni->flags |= FILE_ATTR_SPARSE_FILE; + } + } + if (a->flags & ATTR_IS_ENCRYPTED) { + if (NInoCompressed(ni)) { + ntfs_error(vi->i_sb, "Found encrypted and compressed data."); + goto unm_err_out; + } + /* + * The encryption flag set in an index root just means to + * encrypt all files. + */ + if (NInoMstProtected(ni) && ni->type != AT_INDEX_ROOT) { + ntfs_error(vi->i_sb, + "Found mst protected attribute but the attribute is encrypted."); + goto unm_err_out; + } + if (ni->type != AT_DATA) { + ntfs_error(vi->i_sb, + "Found encrypted non-data attribute."); + goto unm_err_out; + } + NInoSetEncrypted(ni); + ni->flags |= FILE_ATTR_ENCRYPTED; + } + if (!a->non_resident) { + /* Ensure the attribute name is placed before the value. */ + if (unlikely(a->name_length && (le16_to_cpu(a->name_offset) >= + le16_to_cpu(a->data.resident.value_offset)))) { + ntfs_error(vol->sb, + "Attribute name is placed after the attribute value."); + goto unm_err_out; + } + if (NInoMstProtected(ni)) { + ntfs_error(vi->i_sb, + "Found mst protected attribute but the attribute is resident."); + goto unm_err_out; + } + vi->i_size = ni->initialized_size = ni->data_size = le32_to_cpu( + a->data.resident.value_length); + ni->allocated_size = le32_to_cpu(a->length) - + le16_to_cpu(a->data.resident.value_offset); + if (vi->i_size > ni->allocated_size) { + ntfs_error(vi->i_sb, + "Resident attribute is corrupt (size exceeds allocation)."); + goto unm_err_out; + } + } else { + NInoSetNonResident(ni); + /* + * Ensure the attribute name is placed before the mapping pairs + * array. + */ + if (unlikely(a->name_length && (le16_to_cpu(a->name_offset) >= + le16_to_cpu( + a->data.non_resident.mapping_pairs_offset)))) { + ntfs_error(vol->sb, + "Attribute name is placed after the mapping pairs array."); + goto unm_err_out; + } + if (NInoCompressed(ni) || NInoSparse(ni)) { + if (NInoCompressed(ni) && a->data.non_resident.compression_unit != 4) { + ntfs_error(vi->i_sb, + "Found non-standard compression unit (%u instead of 4). Cannot handle this.", + a->data.non_resident.compression_unit); + err = -EOPNOTSUPP; + goto unm_err_out; + } + if (a->data.non_resident.compression_unit) { + ni->itype.compressed.block_size = 1U << + (a->data.non_resident.compression_unit + + vol->cluster_size_bits); + ni->itype.compressed.block_size_bits = + ffs(ni->itype.compressed.block_size) - 1; + ni->itype.compressed.block_clusters = 1U << + a->data.non_resident.compression_unit; + } else { + ni->itype.compressed.block_size = 0; + ni->itype.compressed.block_size_bits = 0; + ni->itype.compressed.block_clusters = 0; + } + ni->itype.compressed.size = le64_to_cpu( + a->data.non_resident.compressed_size); + } + if (a->data.non_resident.lowest_vcn) { + ntfs_error(vi->i_sb, "First extent of attribute has non-zero lowest_vcn."); + goto unm_err_out; + } + vi->i_size = ni->data_size = le64_to_cpu(a->data.non_resident.data_size); + ni->initialized_size = le64_to_cpu(a->data.non_resident.initialized_size); + ni->allocated_size = le64_to_cpu(a->data.non_resident.allocated_size); + } + vi->i_mapping->a_ops = &ntfs_aops; + if ((NInoCompressed(ni) || NInoSparse(ni)) && ni->type != AT_INDEX_ROOT) + vi->i_blocks = ni->itype.compressed.size >> 9; + else + vi->i_blocks = ni->allocated_size >> 9; + /* + * Make sure the base inode does not go away and attach it to the + * attribute inode. + */ + if (!igrab(base_vi)) { + err = -ENOENT; + goto unm_err_out; + } + ni->ext.base_ntfs_ino = base_ni; + ni->nr_extents = -1; + + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(base_ni); + + ntfs_debug("Done."); + return 0; + +unm_err_out: + if (!err) + err = -EIO; + if (ctx) + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(base_ni); +err_out: + if (err != -ENOENT) + ntfs_error(vol->sb, + "Failed with error code %i while reading attribute inode (mft_no 0x%llx, type 0x%x, name_len %i). Marking corrupt inode and base inode 0x%llx as bad. Run chkdsk.", + err, ni->mft_no, ni->type, ni->name_len, + base_ni->mft_no); + if (err != -ENOENT && err != -ENOMEM) + NVolSetErrors(vol); + return err; +} + +/* + * ntfs_read_locked_index_inode - read an index inode from its base inode + * @base_vi: base inode + * @vi: index inode to read + * + * ntfs_read_locked_index_inode() is called from ntfs_index_iget() to read the + * index inode described by @vi into memory from the base mft record described + * by @base_ni. + * + * ntfs_read_locked_index_inode() maps, pins and locks the base inode for + * reading and looks up the attributes relating to the index described by @vi + * before setting up the necessary fields in @vi as well as initializing the + * ntfs inode. + * + * Note, index inodes are essentially attribute inodes (NInoAttr() is true) + * with the attribute type set to AT_INDEX_ALLOCATION. Apart from that, they + * are setup like directory inodes since directories are a special case of + * indices ao they need to be treated in much the same way. Most importantly, + * for small indices the index allocation attribute might not actually exist. + * However, the index root attribute always exists but this does not need to + * have an inode associated with it and this is why we define a new inode type + * index. Also, like for directories, we need to have an attribute inode for + * the bitmap attribute corresponding to the index allocation attribute and we + * can store this in the appropriate field of the inode, just like we do for + * normal directory inodes. + * + * Q: What locks are held when the function is called? + * A: i_state has I_NEW set, hence the inode is locked, also + * i_count is set to 1, so it is not going to go away + * + * Return 0 on success and -errno on error. + */ +static int ntfs_read_locked_index_inode(struct inode *base_vi, struct inode *vi) +{ + loff_t bvi_size; + struct ntfs_volume *vol = NTFS_SB(vi->i_sb); + struct ntfs_inode *ni = NTFS_I(vi), *base_ni = NTFS_I(base_vi), *bni; + struct inode *bvi; + struct mft_record *m; + struct attr_record *a; + struct ntfs_attr_search_ctx *ctx; + struct index_root *ir; + int err = 0; + + ntfs_debug("Entering for i_ino 0x%llx.", ni->mft_no); + lockdep_assert_held(&base_ni->mrec_lock); + + ntfs_init_big_inode(vi); + /* Just mirror the values from the base inode. */ + vi->i_uid = base_vi->i_uid; + vi->i_gid = base_vi->i_gid; + set_nlink(vi, base_vi->i_nlink); + inode_set_mtime_to_ts(vi, inode_get_mtime(base_vi)); + inode_set_ctime_to_ts(vi, inode_get_ctime(base_vi)); + inode_set_atime_to_ts(vi, inode_get_atime(base_vi)); + vi->i_generation = ni->seq_no = base_ni->seq_no; + /* Set inode type to zero but preserve permissions. */ + vi->i_mode = base_vi->i_mode & ~S_IFMT; + /* Map the mft record for the base inode. */ + m = map_mft_record(base_ni); + if (IS_ERR(m)) { + err = PTR_ERR(m); + goto err_out; + } + ctx = ntfs_attr_get_search_ctx(base_ni, m); + if (!ctx) { + err = -ENOMEM; + goto unm_err_out; + } + /* Find the index root attribute. */ + err = ntfs_attr_lookup(AT_INDEX_ROOT, ni->name, ni->name_len, + CASE_SENSITIVE, 0, NULL, 0, ctx); + if (unlikely(err)) { + if (err == -ENOENT) + ntfs_error(vi->i_sb, "$INDEX_ROOT attribute is missing."); + goto unm_err_out; + } + a = ctx->attr; + /* Set up the state. */ + if (unlikely(a->non_resident)) { + ntfs_error(vol->sb, "$INDEX_ROOT attribute is not resident."); + goto unm_err_out; + } + /* Ensure the attribute name is placed before the value. */ + if (unlikely(a->name_length && (le16_to_cpu(a->name_offset) >= + le16_to_cpu(a->data.resident.value_offset)))) { + ntfs_error(vol->sb, + "$INDEX_ROOT attribute name is placed after the attribute value."); + goto unm_err_out; + } + + ir = (struct index_root *)((u8 *)a + le16_to_cpu(a->data.resident.value_offset)); + if (ntfs_index_root_inconsistent(vol, a, ir, ni->mft_no) || + ntfs_index_entries_inconsistent(vol, &ir->index, + ir->collation_rule, ni->mft_no)) { + ntfs_error(vi->i_sb, "Index is corrupt."); + goto unm_err_out; + } + + ni->itype.index.collation_rule = ir->collation_rule; + ntfs_debug("Index collation rule is 0x%x.", + le32_to_cpu(ir->collation_rule)); + ni->itype.index.block_size = le32_to_cpu(ir->index_block_size); + if (!is_power_of_2(ni->itype.index.block_size)) { + ntfs_error(vi->i_sb, "Index block size (%u) is not a power of two.", + ni->itype.index.block_size); + goto unm_err_out; + } + if (ni->itype.index.block_size > PAGE_SIZE) { + ntfs_error(vi->i_sb, "Index block size (%u) > PAGE_SIZE (%ld) is not supported.", + ni->itype.index.block_size, PAGE_SIZE); + err = -EOPNOTSUPP; + goto unm_err_out; + } + if (ni->itype.index.block_size < NTFS_BLOCK_SIZE) { + ntfs_error(vi->i_sb, + "Index block size (%u) < NTFS_BLOCK_SIZE (%i) is not supported.", + ni->itype.index.block_size, NTFS_BLOCK_SIZE); + err = -EOPNOTSUPP; + goto unm_err_out; + } + ni->itype.index.block_size_bits = ffs(ni->itype.index.block_size) - 1; + /* Determine the size of a vcn in the index. */ + if (vol->cluster_size <= ni->itype.index.block_size) { + ni->itype.index.vcn_size = vol->cluster_size; + ni->itype.index.vcn_size_bits = vol->cluster_size_bits; + } else { + ni->itype.index.vcn_size = vol->sector_size; + ni->itype.index.vcn_size_bits = vol->sector_size_bits; + } + + /* Find index allocation attribute. */ + ntfs_attr_reinit_search_ctx(ctx); + err = ntfs_attr_lookup(AT_INDEX_ALLOCATION, ni->name, ni->name_len, + CASE_SENSITIVE, 0, NULL, 0, ctx); + if (unlikely(err)) { + if (err == -ENOENT) { + /* No index allocation. */ + vi->i_size = ni->initialized_size = ni->allocated_size = 0; + /* We are done with the mft record, so we release it. */ + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(base_ni); + m = NULL; + ctx = NULL; + goto skip_large_index_stuff; + } else + ntfs_error(vi->i_sb, "Failed to lookup $INDEX_ALLOCATION attribute."); + goto unm_err_out; + } + NInoSetIndexAllocPresent(ni); + NInoSetNonResident(ni); + ni->type = AT_INDEX_ALLOCATION; + + a = ctx->attr; + if (!a->non_resident) { + ntfs_error(vi->i_sb, "$INDEX_ALLOCATION attribute is resident."); + goto unm_err_out; + } + /* + * Ensure the attribute name is placed before the mapping pairs array. + */ + if (unlikely(a->name_length && (le16_to_cpu(a->name_offset) >= + le16_to_cpu(a->data.non_resident.mapping_pairs_offset)))) { + ntfs_error(vol->sb, + "$INDEX_ALLOCATION attribute name is placed after the mapping pairs array."); + goto unm_err_out; + } + if (a->flags & ATTR_IS_ENCRYPTED) { + ntfs_error(vi->i_sb, "$INDEX_ALLOCATION attribute is encrypted."); + goto unm_err_out; + } + if (a->flags & ATTR_IS_SPARSE) { + ntfs_error(vi->i_sb, "$INDEX_ALLOCATION attribute is sparse."); + goto unm_err_out; + } + if (a->flags & ATTR_COMPRESSION_MASK) { + ntfs_error(vi->i_sb, + "$INDEX_ALLOCATION attribute is compressed."); + goto unm_err_out; + } + if (a->data.non_resident.lowest_vcn) { + ntfs_error(vi->i_sb, + "First extent of $INDEX_ALLOCATION attribute has non zero lowest_vcn."); + goto unm_err_out; + } + vi->i_size = ni->data_size = le64_to_cpu(a->data.non_resident.data_size); + ni->initialized_size = le64_to_cpu(a->data.non_resident.initialized_size); + ni->allocated_size = le64_to_cpu(a->data.non_resident.allocated_size); + /* + * We are done with the mft record, so we release it. Otherwise + * we would deadlock in ntfs_attr_iget(). + */ + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(base_ni); + m = NULL; + ctx = NULL; + /* Get the index bitmap attribute inode. */ + bvi = ntfs_attr_iget(base_vi, AT_BITMAP, ni->name, ni->name_len); + if (IS_ERR(bvi)) { + ntfs_error(vi->i_sb, "Failed to get bitmap attribute."); + err = PTR_ERR(bvi); + goto unm_err_out; + } + bni = NTFS_I(bvi); + if (NInoCompressed(bni) || NInoEncrypted(bni) || + NInoSparse(bni)) { + ntfs_error(vi->i_sb, + "$BITMAP attribute is compressed and/or encrypted and/or sparse."); + goto iput_unm_err_out; + } + /* Consistency check bitmap size vs. index allocation size. */ + bvi_size = i_size_read(bvi); + if ((bvi_size << 3) < (vi->i_size >> ni->itype.index.block_size_bits)) { + ntfs_error(vi->i_sb, + "Index bitmap too small (0x%llx) for index allocation (0x%llx).", + bvi_size << 3, vi->i_size); + goto iput_unm_err_out; + } + iput(bvi); +skip_large_index_stuff: + /* Setup the operations for this index inode. */ + ntfs_set_vfs_operations(vi, S_IFDIR, 0); + vi->i_blocks = ni->allocated_size >> 9; + /* + * Make sure the base inode doesn't go away and attach it to the + * index inode. + */ + if (!igrab(base_vi)) + goto unm_err_out; + ni->ext.base_ntfs_ino = base_ni; + ni->nr_extents = -1; + + ntfs_debug("Done."); + return 0; +iput_unm_err_out: + iput(bvi); +unm_err_out: + if (!err) + err = -EIO; + if (ctx) + ntfs_attr_put_search_ctx(ctx); + if (m) + unmap_mft_record(base_ni); +err_out: + ntfs_error(vi->i_sb, + "Failed with error code %i while reading index inode (mft_no 0x%llx, name_len %i.", + err, ni->mft_no, ni->name_len); + if (err != -EOPNOTSUPP && err != -ENOMEM) + NVolSetErrors(vol); + return err; +} + +/* + * load_attribute_list_mount - load an attribute list into memory + * @vol: ntfs volume from which to read + * @rl: runlist of the attribute list + * @al_start: destination buffer + * @size: size of the destination buffer in bytes + * @initialized_size: initialized size of the attribute list + * + * Walk the runlist @rl and load all clusters from it copying them into + * the linear buffer @al. The maximum number of bytes copied to @al is @size + * bytes. Note, @size does not need to be a multiple of the cluster size. If + * @initialized_size is less than @size, the region in @al between + * @initialized_size and @size will be zeroed and not read from disk. + * + * Return 0 on success or -errno on error. + */ +static int load_attribute_list_mount(struct ntfs_volume *vol, + struct runlist_element *rl, u8 *al_start, const s64 size, + const s64 initialized_size) +{ + s64 lcn; + u8 *al = al_start; + u8 *al_end = al + initialized_size; + struct super_block *sb; + int err = 0; + loff_t rl_byte_off, rl_byte_len; + + ntfs_debug("Entering."); + if (!vol || !rl || !al || size <= 0 || initialized_size < 0 || + initialized_size > size) + return -EINVAL; + if (!initialized_size) { + memset(al, 0, size); + return 0; + } + sb = vol->sb; + + /* Read all clusters specified by the runlist one run at a time. */ + while (rl->length) { + lcn = ntfs_rl_vcn_to_lcn(rl, rl->vcn); + ntfs_debug("Reading vcn = 0x%llx, lcn = 0x%llx.", + (unsigned long long)rl->vcn, + (unsigned long long)lcn); + /* The attribute list cannot be sparse. */ + if (lcn < 0) { + ntfs_error(sb, "ntfs_rl_vcn_to_lcn() failed. Cannot read attribute list."); + return -EIO; + } + + rl_byte_off = ntfs_cluster_to_bytes(vol, lcn); + rl_byte_len = ntfs_cluster_to_bytes(vol, rl->length); + + if (al + rl_byte_len > al_end) + rl_byte_len = al_end - al; + + err = ntfs_bdev_read(sb->s_bdev, al, rl_byte_off, + round_up(rl_byte_len, SECTOR_SIZE)); + if (err) { + ntfs_error(sb, "Cannot read attribute list."); + return -EIO; + } + + if (al + rl_byte_len >= al_end) { + if (initialized_size < size) + goto initialize; + goto done; + } + + al += rl_byte_len; + rl++; + } + if (initialized_size < size) { +initialize: + memset(al_start + initialized_size, 0, size - initialized_size); + } +done: + return err; +} + +/* + * The MFT inode has special locking, so teach the lock validator + * about this by splitting off the locking rules of the MFT from + * the locking rules of other inodes. The MFT inode can never be + * accessed from the VFS side (or even internally), only by the + * map_mft functions. + */ +static struct lock_class_key mft_ni_runlist_lock_key, mft_ni_mrec_lock_key; + +/* + * ntfs_read_inode_mount - special read_inode for mount time use only + * @vi: inode to read + * + * Read inode FILE_MFT at mount time, only called with super_block lock + * held from within the read_super() code path. + * + * This function exists because when it is called the page cache for $MFT/$DATA + * is not initialized and hence we cannot get at the contents of mft records + * by calling map_mft_record*(). + * + * Further it needs to cope with the circular references problem, i.e. cannot + * load any attributes other than $ATTRIBUTE_LIST until $DATA is loaded, because + * we do not know where the other extent mft records are yet and again, because + * we cannot call map_mft_record*() yet. Obviously this applies only when an + * attribute list is actually present in $MFT inode. + * + * We solve these problems by starting with the $DATA attribute before anything + * else and iterating using ntfs_attr_lookup($DATA) over all extents. As each + * extent is found, we ntfs_mapping_pairs_decompress() including the implied + * ntfs_runlists_merge(). Each step of the iteration necessarily provides + * sufficient information for the next step to complete. + * + * This should work but there are two possible pit falls (see inline comments + * below), but only time will tell if they are real pits or just smoke... + */ +int ntfs_read_inode_mount(struct inode *vi) +{ + s64 next_vcn, last_vcn, highest_vcn; + struct super_block *sb = vi->i_sb; + struct ntfs_volume *vol = NTFS_SB(sb); + struct ntfs_inode *ni = NTFS_I(vi); + struct mft_record *m = NULL; + struct attr_record *a; + struct ntfs_attr_search_ctx *ctx; + unsigned int i, nr_blocks; + int err; + size_t new_rl_count; + + ntfs_debug("Entering."); + + /* Initialize the ntfs specific part of @vi. */ + ntfs_init_big_inode(vi); + + + /* Setup the data attribute. It is special as it is mst protected. */ + NInoSetNonResident(ni); + NInoSetMstProtected(ni); + NInoSetSparseDisabled(ni); + ni->type = AT_DATA; + ni->name = AT_UNNAMED; + ni->name_len = 0; + /* + * This sets up our little cheat allowing us to reuse the async read io + * completion handler for directories. + */ + ni->itype.index.block_size = vol->mft_record_size; + ni->itype.index.block_size_bits = vol->mft_record_size_bits; + + /* Very important! Needed to be able to call map_mft_record*(). */ + vol->mft_ino = vi; + + /* Allocate enough memory to read the first mft record. */ + if (vol->mft_record_size > 64 * 1024) { + ntfs_error(sb, "Unsupported mft record size %i (max 64kiB).", + vol->mft_record_size); + goto err_out; + } + + i = vol->mft_record_size; + if (i < sb->s_blocksize) + i = sb->s_blocksize; + + m = kzalloc(i, GFP_NOFS); + if (!m) { + ntfs_error(sb, "Failed to allocate buffer for $MFT record 0."); + goto err_out; + } + + /* Determine the first block of the $MFT/$DATA attribute. */ + nr_blocks = ntfs_bytes_to_sector(vol, vol->mft_record_size); + if (!nr_blocks) + nr_blocks = 1; + + /* Load $MFT/$DATA's first mft record. */ + err = ntfs_bdev_read(sb->s_bdev, (char *)m, + ntfs_cluster_to_bytes(vol, vol->mft_lcn), i); + if (err) { + ntfs_error(sb, "Device read failed."); + goto err_out; + } + + if (le32_to_cpu(m->bytes_allocated) != vol->mft_record_size) { + ntfs_error(sb, "Incorrect mft record size %u in superblock, should be %u.", + le32_to_cpu(m->bytes_allocated), vol->mft_record_size); + goto err_out; + } + + /* Apply the mst fixups. */ + if (post_read_mst_fixup((struct ntfs_record *)m, vol->mft_record_size)) { + ntfs_error(sb, "MST fixup failed. $MFT is corrupt."); + goto err_out; + } + + if (ntfs_mft_record_check(vol, m, FILE_MFT)) { + ntfs_error(sb, "ntfs_mft_record_check failed. $MFT is corrupt."); + goto err_out; + } + + /* Need this to sanity check attribute list references to $MFT. */ + vi->i_generation = ni->seq_no = le16_to_cpu(m->sequence_number); + + /* Provides read_folio() for map_mft_record(). */ + vi->i_mapping->a_ops = &ntfs_mft_aops; + + ctx = ntfs_attr_get_search_ctx(ni, m); + if (!ctx) { + err = -ENOMEM; + goto err_out; + } + + /* Find the attribute list attribute if present. */ + err = ntfs_attr_lookup(AT_ATTRIBUTE_LIST, NULL, 0, 0, 0, NULL, 0, ctx); + if (err) { + if (unlikely(err != -ENOENT)) { + ntfs_error(sb, + "Failed to lookup attribute list attribute. You should run chkdsk."); + goto put_err_out; + } + } else /* if (!err) */ { + struct attr_list_entry *al_entry, *next_al_entry; + u8 *al_end; + static const char *es = " Not allowed. $MFT is corrupt. You should run chkdsk."; + + ntfs_debug("Attribute list attribute found in $MFT."); + NInoSetAttrList(ni); + a = ctx->attr; + if (a->flags & ATTR_COMPRESSION_MASK) { + ntfs_error(sb, + "Attribute list attribute is compressed.%s", + es); + goto put_err_out; + } + if (a->flags & ATTR_IS_ENCRYPTED || + a->flags & ATTR_IS_SPARSE) { + if (a->non_resident) { + ntfs_error(sb, + "Non-resident attribute list attribute is encrypted/sparse.%s", + es); + goto put_err_out; + } + ntfs_warning(sb, + "Resident attribute list attribute in $MFT system file is marked encrypted/sparse which is not true. However, Windows allows this and chkdsk does not detect or correct it so we will just ignore the invalid flags and pretend they are not set."); + } + /* Now allocate memory for the attribute list. */ + ni->attr_list_size = (u32)ntfs_attr_size(a); + if (!ni->attr_list_size) { + ntfs_error(sb, "Attr_list_size is zero"); + goto put_err_out; + } + ni->attr_list = kvzalloc(round_up(ni->attr_list_size, SECTOR_SIZE), + GFP_NOFS); + if (!ni->attr_list) { + ntfs_error(sb, "Not enough memory to allocate buffer for attribute list."); + goto put_err_out; + } + if (a->non_resident) { + struct runlist_element *rl; + size_t new_rl_count; + + NInoSetAttrListNonResident(ni); + if (a->data.non_resident.lowest_vcn) { + ntfs_error(sb, + "Attribute list has non zero lowest_vcn. $MFT is corrupt. You should run chkdsk."); + goto put_err_out; + } + + rl = ntfs_mapping_pairs_decompress(vol, a, NULL, &new_rl_count); + if (IS_ERR(rl)) { + err = PTR_ERR(rl); + ntfs_error(sb, + "Mapping pairs decompression failed with error code %i.", + -err); + goto put_err_out; + } + + err = load_attribute_list_mount(vol, rl, ni->attr_list, ni->attr_list_size, + le64_to_cpu(a->data.non_resident.initialized_size)); + kvfree(rl); + if (err) { + ntfs_error(sb, + "Failed to load attribute list with error code %i.", + -err); + goto put_err_out; + } + } else /* if (!ctx.attr->non_resident) */ { + /* Now copy the attribute list. */ + memcpy(ni->attr_list, (u8 *)a + le16_to_cpu( + a->data.resident.value_offset), + le32_to_cpu(a->data.resident.value_length)); + } + /* The attribute list is now setup in memory. */ + al_entry = (struct attr_list_entry *)ni->attr_list; + al_end = (u8 *)al_entry + ni->attr_list_size; + for (;; al_entry = next_al_entry) { + /* Out of bounds check. */ + if ((u8 *)al_entry < ni->attr_list || + (u8 *)al_entry > al_end) + goto em_put_err_out; + /* Catch the end of the attribute list. */ + if ((u8 *)al_entry == al_end) + goto em_put_err_out; + if (!ntfs_attr_list_entry_is_valid(al_entry, al_end)) + goto em_put_err_out; + next_al_entry = (struct attr_list_entry *)((u8 *)al_entry + + le16_to_cpu(al_entry->length)); + if (le32_to_cpu(al_entry->type) > le32_to_cpu(AT_DATA)) + goto em_put_err_out; + if (al_entry->type != AT_DATA) + continue; + /* We want an unnamed attribute. */ + if (al_entry->name_length) + goto em_put_err_out; + /* Want the first entry, i.e. lowest_vcn == 0. */ + if (al_entry->lowest_vcn) + goto em_put_err_out; + /* First entry has to be in the base mft record. */ + if (MREF_LE(al_entry->mft_reference) != vi->i_ino) { + /* MFT references do not match, logic fails. */ + ntfs_error(sb, + "BUG: The first $DATA extent of $MFT is not in the base mft record."); + goto put_err_out; + } else { + /* Sequence numbers must match. */ + if (MSEQNO_LE(al_entry->mft_reference) != + ni->seq_no) + goto em_put_err_out; + /* Got it. All is ok. We can stop now. */ + break; + } + } + } + + ntfs_attr_reinit_search_ctx(ctx); + + /* Now load all attribute extents. */ + a = NULL; + next_vcn = last_vcn = highest_vcn = 0; + while (!(err = ntfs_attr_lookup(AT_DATA, NULL, 0, 0, next_vcn, NULL, 0, + ctx))) { + struct runlist_element *nrl; + + /* Cache the current attribute. */ + a = ctx->attr; + /* $MFT must be non-resident. */ + if (!a->non_resident) { + ntfs_error(sb, + "$MFT must be non-resident but a resident extent was found. $MFT is corrupt. Run chkdsk."); + goto put_err_out; + } + /* $MFT must be uncompressed and unencrypted. */ + if (a->flags & ATTR_COMPRESSION_MASK || + a->flags & ATTR_IS_ENCRYPTED || + a->flags & ATTR_IS_SPARSE) { + ntfs_error(sb, + "$MFT must be uncompressed, non-sparse, and unencrypted but a compressed/sparse/encrypted extent was found. $MFT is corrupt. Run chkdsk."); + goto put_err_out; + } + /* + * Decompress the mapping pairs array of this extent and merge + * the result into the existing runlist. No need for locking + * as we have exclusive access to the inode at this time and we + * are a mount in progress task, too. + */ + nrl = ntfs_mapping_pairs_decompress(vol, a, &ni->runlist, + &new_rl_count); + if (IS_ERR(nrl)) { + ntfs_error(sb, + "ntfs_mapping_pairs_decompress() failed with error code %ld.", + PTR_ERR(nrl)); + goto put_err_out; + } + ni->runlist.rl = nrl; + ni->runlist.count = new_rl_count; + + /* Are we in the first extent? */ + if (!next_vcn) { + if (a->data.non_resident.lowest_vcn) { + ntfs_error(sb, + "First extent of $DATA attribute has non zero lowest_vcn. $MFT is corrupt. You should run chkdsk."); + goto put_err_out; + } + /* Get the last vcn in the $DATA attribute. */ + last_vcn = ntfs_bytes_to_cluster(vol, + le64_to_cpu(a->data.non_resident.allocated_size)); + /* Fill in the inode size. */ + vi->i_size = le64_to_cpu(a->data.non_resident.data_size); + ni->initialized_size = le64_to_cpu(a->data.non_resident.initialized_size); + ni->allocated_size = le64_to_cpu(a->data.non_resident.allocated_size); + /* + * Verify the number of mft records does not exceed + * 2^32 - 1. + */ + if ((vi->i_size >> vol->mft_record_size_bits) >= + (1ULL << 32)) { + ntfs_error(sb, "$MFT is too big! Aborting."); + goto put_err_out; + } + /* + * We have got the first extent of the runlist for + * $MFT which means it is now relatively safe to call + * the normal ntfs_read_inode() function. + * Complete reading the inode, this will actually + * re-read the mft record for $MFT, this time entering + * it into the page cache with which we complete the + * kick start of the volume. It should be safe to do + * this now as the first extent of $MFT/$DATA is + * already known and we would hope that we don't need + * further extents in order to find the other + * attributes belonging to $MFT. Only time will tell if + * this is really the case. If not we will have to play + * magic at this point, possibly duplicating a lot of + * ntfs_read_inode() at this point. We will need to + * ensure we do enough of its work to be able to call + * ntfs_read_inode() on extents of $MFT/$DATA. But lets + * hope this never happens... + */ + err = ntfs_read_locked_inode(vi); + if (err) { + ntfs_error(sb, "ntfs_read_inode() of $MFT failed.\n"); + ntfs_attr_put_search_ctx(ctx); + /* Revert to the safe super operations. */ + kfree(m); + return -1; + } + /* + * Re-initialize some specifics about $MFT's inode as + * ntfs_read_inode() will have set up the default ones. + */ + /* Set uid and gid to root. */ + vi->i_uid = GLOBAL_ROOT_UID; + vi->i_gid = GLOBAL_ROOT_GID; + /* Regular file. No access for anyone. */ + vi->i_mode = S_IFREG; + /* No VFS initiated operations allowed for $MFT. */ + vi->i_op = &ntfs_empty_inode_ops; + vi->i_fop = &ntfs_empty_file_ops; + } + + /* Get the lowest vcn for the next extent. */ + highest_vcn = le64_to_cpu(a->data.non_resident.highest_vcn); + next_vcn = highest_vcn + 1; + + /* Only one extent or error, which we catch below. */ + if (next_vcn <= 0) + break; + + /* Avoid endless loops due to corruption. */ + if (next_vcn < le64_to_cpu(a->data.non_resident.lowest_vcn)) { + ntfs_error(sb, "$MFT has corrupt attribute list attribute. Run chkdsk."); + goto put_err_out; + } + } + if (err != -ENOENT) { + ntfs_error(sb, "Failed to lookup $MFT/$DATA attribute extent. Run chkdsk.\n"); + goto put_err_out; + } + if (!a) { + ntfs_error(sb, "$MFT/$DATA attribute not found. $MFT is corrupt. Run chkdsk."); + goto put_err_out; + } + if (highest_vcn && highest_vcn != last_vcn - 1) { + ntfs_error(sb, "Failed to load the complete runlist for $MFT/$DATA. Run chkdsk."); + ntfs_debug("highest_vcn = 0x%llx, last_vcn - 1 = 0x%llx", + (unsigned long long)highest_vcn, + (unsigned long long)last_vcn - 1); + goto put_err_out; + } + ntfs_attr_put_search_ctx(ctx); + ntfs_debug("Done."); + kfree(m); + + /* + * Split the locking rules of the MFT inode from the + * locking rules of other inodes: + */ + lockdep_set_class(&ni->runlist.lock, &mft_ni_runlist_lock_key); + lockdep_set_class(&ni->mrec_lock, &mft_ni_mrec_lock_key); + + return 0; + +em_put_err_out: + ntfs_error(sb, + "Couldn't find first extent of $DATA attribute in attribute list. $MFT is corrupt. Run chkdsk."); +put_err_out: + ntfs_attr_put_search_ctx(ctx); +err_out: + ntfs_error(sb, "Failed. Marking inode as bad."); + kfree(m); + return -1; +} + +static void __ntfs_clear_inode(struct ntfs_inode *ni) +{ + /* Free all alocated memory. */ + if (NInoNonResident(ni) && ni->runlist.rl) { + kvfree(ni->runlist.rl); + ni->runlist.rl = NULL; + } + + if (ni->attr_list) { + kvfree(ni->attr_list); + ni->attr_list = NULL; + } + + if (ni->name_len && ni->name != I30 && + ni->name != reparse_index_name && + ni->name != objid_index_name) { + WARN_ON(!ni->name); + kfree(ni->name); + } +} + +void ntfs_clear_extent_inode(struct ntfs_inode *ni) +{ + ntfs_debug("Entering for inode 0x%llx.", ni->mft_no); + + WARN_ON(NInoAttr(ni)); + WARN_ON(ni->nr_extents != -1); + + __ntfs_clear_inode(ni); + ntfs_destroy_extent_inode(ni); +} + +static int ntfs_delete_base_inode(struct ntfs_inode *ni) +{ + struct super_block *sb = ni->vol->sb; + int err; + + if (NInoAttr(ni) || ni->nr_extents == -1) + return 0; + + err = ntfs_non_resident_dealloc_clusters(ni); + + /* + * Deallocate extent mft records and free extent inodes. + * No need to lock as no one else has a reference. + */ + while (ni->nr_extents) { + err = ntfs_mft_record_free(ni->vol, *(ni->ext.extent_ntfs_inos)); + if (err) + ntfs_error(sb, + "Failed to free extent MFT record. Leaving inconsistent metadata.\n"); + ntfs_inode_close(*(ni->ext.extent_ntfs_inos)); + } + + /* Deallocate base mft record */ + err = ntfs_mft_record_free(ni->vol, ni); + if (err) + ntfs_error(sb, "Failed to free base MFT record. Leaving inconsistent metadata.\n"); + return err; +} + +/* + * ntfs_evict_big_inode - clean up the ntfs specific part of an inode + * @vi: vfs inode pending annihilation + * + * When the VFS is going to remove an inode from memory, ntfs_clear_big_inode() + * is called, which deallocates all memory belonging to the NTFS specific part + * of the inode and returns. + * + * If the MFT record is dirty, we commit it before doing anything else. + */ +void ntfs_evict_big_inode(struct inode *vi) +{ + struct ntfs_inode *ni = NTFS_I(vi); + + truncate_inode_pages_final(&vi->i_data); + + if (!vi->i_nlink) { + if (!NInoAttr(ni)) { + /* Never called with extent inodes */ + WARN_ON(ni->nr_extents == -1); + ntfs_delete_base_inode(ni); + } + goto release; + } + + if (NInoDirty(ni)) { + /* Committing the inode also commits all extent inodes. */ + ntfs_commit_inode(vi); + + if (NInoDirty(ni)) { + ntfs_debug("Failed to commit dirty inode 0x%llx. Losing data!", + ni->mft_no); + NInoClearAttrListDirty(ni); + NInoClearDirty(ni); + } + } + + /* No need to lock at this stage as no one else has a reference. */ + if (ni->nr_extents > 0) { + int i; + + for (i = 0; i < ni->nr_extents; i++) { + if (ni->ext.extent_ntfs_inos[i]) + ntfs_clear_extent_inode(ni->ext.extent_ntfs_inos[i]); + } + ni->nr_extents = 0; + kvfree(ni->ext.extent_ntfs_inos); + } + +release: + clear_inode(vi); + __ntfs_clear_inode(ni); + + if (NInoAttr(ni)) { + /* Release the base inode if we are holding it. */ + if (ni->nr_extents == -1) { + iput(VFS_I(ni->ext.base_ntfs_ino)); + ni->nr_extents = 0; + ni->ext.base_ntfs_ino = NULL; + } + } + + if (!atomic_dec_and_test(&ni->count)) + WARN_ON(1); + if (ni->folio) + folio_put(ni->folio); + kfree(ni->mrec); + kvfree(ni->target); +} + +/* + * ntfs_show_options - show mount options in /proc/mounts + * @sf: seq_file in which to write our mount options + * @root: root of the mounted tree whose mount options to display + * + * Called by the VFS once for each mounted ntfs volume when someone reads + * /proc/mounts in order to display the NTFS specific mount options of each + * mount. The mount options of fs specified by @root are written to the seq file + * @sf and success is returned. + */ +int ntfs_show_options(struct seq_file *sf, struct dentry *root) +{ + struct ntfs_volume *vol = NTFS_SB(root->d_sb); + int i; + + if (uid_valid(vol->uid)) + seq_printf(sf, ",uid=%i", from_kuid_munged(&init_user_ns, vol->uid)); + if (gid_valid(vol->gid)) + seq_printf(sf, ",gid=%i", from_kgid_munged(&init_user_ns, vol->gid)); + if (vol->fmask == vol->dmask) + seq_printf(sf, ",umask=0%o", vol->fmask); + else { + seq_printf(sf, ",fmask=0%o", vol->fmask); + seq_printf(sf, ",dmask=0%o", vol->dmask); + } + seq_printf(sf, ",iocharset=%s", vol->nls_map->charset); + if (NVolCaseSensitive(vol)) + seq_puts(sf, ",case_sensitive"); + else + seq_puts(sf, ",nocase"); + if (NVolShowSystemFiles(vol)) + seq_puts(sf, ",show_sys_files,showmeta"); + for (i = 0; on_errors_arr[i].val; i++) { + if (on_errors_arr[i].val == vol->on_errors) + seq_printf(sf, ",errors=%s", on_errors_arr[i].str); + } + seq_printf(sf, ",mft_zone_multiplier=%i", vol->mft_zone_multiplier); + if (NVolSysImmutable(vol)) + seq_puts(sf, ",sys_immutable"); + if (!NVolShowHiddenFiles(vol)) + seq_puts(sf, ",nohidden"); + if (NVolHideDotFiles(vol)) + seq_puts(sf, ",hide_dot_files"); + if (NVolCheckWindowsNames(vol)) + seq_puts(sf, ",windows_names"); + if (NVolDiscard(vol)) + seq_puts(sf, ",discard"); + if (NVolDisableSparse(vol)) + seq_puts(sf, ",disable_sparse"); + if (NVolNativeSymlinkRel(vol)) + seq_puts(sf, ",native_symlink=rel"); + else + seq_puts(sf, ",native_symlink=raw"); + if (NVolSymlinkNative(vol)) + seq_puts(sf, ",symlink=native"); + else + seq_puts(sf, ",symlink=wsl"); + if (vol->sb->s_flags & SB_POSIXACL) + seq_puts(sf, ",acl"); + return 0; +} + +int ntfs_extend_initialized_size(struct inode *vi, const loff_t offset, + const loff_t new_size, bool bsync) +{ + struct ntfs_inode *ni = NTFS_I(vi); + loff_t old_init_size; + unsigned long flags; + int err; + + read_lock_irqsave(&ni->size_lock, flags); + old_init_size = ni->initialized_size; + read_unlock_irqrestore(&ni->size_lock, flags); + + if (!NInoNonResident(ni)) + return -EINVAL; + if (old_init_size >= new_size) + return 0; + + err = ntfs_attr_map_whole_runlist(ni); + if (err) + return err; + + if (!NInoCompressed(ni) && old_init_size < offset) { + err = iomap_zero_range(vi, old_init_size, + offset - old_init_size, + NULL, &ntfs_seek_iomap_ops, + &ntfs_iomap_folio_ops, NULL); + if (err) + return err; + if (bsync) + err = filemap_write_and_wait_range(vi->i_mapping, + old_init_size, + offset - 1); + } + + + mutex_lock(&ni->mrec_lock); + err = ntfs_attr_set_initialized_size(ni, new_size); + mutex_unlock(&ni->mrec_lock); + if (err) + truncate_setsize(vi, old_init_size); + return err; +} + +int ntfs_truncate_vfs(struct inode *vi, loff_t new_size, loff_t i_size) +{ + struct ntfs_inode *ni = NTFS_I(vi); + int err; + + mutex_lock(&ni->mrec_lock); + err = __ntfs_attr_truncate_vfs(ni, new_size, i_size); + mutex_unlock(&ni->mrec_lock); + if (err < 0) + return err; + + inode_set_mtime_to_ts(vi, inode_set_ctime_current(vi)); + return 0; +} + +/* + * ntfs_inode_sync_standard_information - update standard information attribute + * @vi: inode to update standard information + * @m: mft record + * + * Return 0 on success or -errno on error. + */ +static int ntfs_inode_sync_standard_information(struct inode *vi, struct mft_record *m) +{ + struct ntfs_inode *ni = NTFS_I(vi); + struct ntfs_attr_search_ctx *ctx; + struct standard_information *si; + __le64 nt; + int err = 0; + bool modified = false; + + /* Update the access times in the standard information attribute. */ + ctx = ntfs_attr_get_search_ctx(ni, m); + if (unlikely(!ctx)) + return -ENOMEM; + err = ntfs_attr_lookup(AT_STANDARD_INFORMATION, NULL, 0, + CASE_SENSITIVE, 0, NULL, 0, ctx); + if (unlikely(err)) { + ntfs_attr_put_search_ctx(ctx); + return err; + } + si = (struct standard_information *)((u8 *)ctx->attr + + le16_to_cpu(ctx->attr->data.resident.value_offset)); + if (si->file_attributes != ni->flags) { + si->file_attributes = ni->flags; + modified = true; + } + + /* Update the creation times if they have changed. */ + nt = utc2ntfs(ni->i_crtime); + if (si->creation_time != nt) { + ntfs_debug("Updating creation time for inode 0x%llx: old = 0x%llx, new = 0x%llx", + ni->mft_no, le64_to_cpu(si->creation_time), + le64_to_cpu(nt)); + si->creation_time = nt; + modified = true; + } + + /* Update the access times if they have changed. */ + nt = utc2ntfs(inode_get_mtime(vi)); + if (si->last_data_change_time != nt) { + ntfs_debug("Updating mtime for inode 0x%llx: old = 0x%llx, new = 0x%llx", + ni->mft_no, le64_to_cpu(si->last_data_change_time), + le64_to_cpu(nt)); + si->last_data_change_time = nt; + modified = true; + } + + nt = utc2ntfs(inode_get_ctime(vi)); + if (si->last_mft_change_time != nt) { + ntfs_debug("Updating ctime for inode 0x%llx: old = 0x%llx, new = 0x%llx", + ni->mft_no, le64_to_cpu(si->last_mft_change_time), + le64_to_cpu(nt)); + si->last_mft_change_time = nt; + modified = true; + } + nt = utc2ntfs(inode_get_atime(vi)); + if (si->last_access_time != nt) { + ntfs_debug("Updating atime for inode 0x%llx: old = 0x%llx, new = 0x%llx", + ni->mft_no, + le64_to_cpu(si->last_access_time), + le64_to_cpu(nt)); + si->last_access_time = nt; + modified = true; + } + + /* + * If we just modified the standard information attribute we need to + * mark the mft record it is in dirty. We do this manually so that + * mark_inode_dirty() is not called which would redirty the inode and + * hence result in an infinite loop of trying to write the inode. + * There is no need to mark the base inode nor the base mft record + * dirty, since we are going to write this mft record below in any case + * and the base mft record may actually not have been modified so it + * might not need to be written out. + * NOTE: It is not a problem when the inode for $MFT itself is being + * written out as ntfs_mft_mark_dirty() will only set I_DIRTY_PAGES + * on the $MFT inode and hence ntfs_write_inode() will not be + * re-invoked because of it which in turn is ok since the dirtied mft + * record will be cleaned and written out to disk below, i.e. before + * this function returns. + */ + if (modified) + NInoSetDirty(ctx->ntfs_ino); + ntfs_attr_put_search_ctx(ctx); + + return err; +} + +/* + * ntfs_inode_sync_filename - update FILE_NAME attributes + * @ni: ntfs inode to update FILE_NAME attributes + * + * Update all FILE_NAME attributes for inode @ni in the index. + * + * Return 0 on success or error. + */ +int ntfs_inode_sync_filename(struct ntfs_inode *ni) +{ + struct inode *index_vi; + struct super_block *sb = VFS_I(ni)->i_sb; + struct ntfs_attr_search_ctx *ctx = NULL; + struct ntfs_index_context *ictx; + struct ntfs_inode *index_ni; + struct file_name_attr *fn; + struct file_name_attr *fnx; + struct reparse_point *rpp; + __le32 reparse_tag; + int err = 0; + unsigned long flags; + + ntfs_debug("Entering for inode %llu\n", ni->mft_no); + + ctx = ntfs_attr_get_search_ctx(ni, NULL); + if (!ctx) + return -ENOMEM; + + /* Collect the reparse tag, if any */ + reparse_tag = cpu_to_le32(0); + if (ni->flags & FILE_ATTR_REPARSE_POINT) { + if (!ntfs_attr_lookup(AT_REPARSE_POINT, NULL, + 0, CASE_SENSITIVE, 0, NULL, 0, ctx)) { + rpp = (struct reparse_point *)((u8 *)ctx->attr + + le16_to_cpu(ctx->attr->data.resident.value_offset)); + reparse_tag = rpp->reparse_tag; + } + ntfs_attr_reinit_search_ctx(ctx); + } + + /* Walk through all FILE_NAME attributes and update them. */ + while (!(err = ntfs_attr_lookup(AT_FILE_NAME, NULL, 0, 0, 0, NULL, 0, ctx))) { + fn = (struct file_name_attr *)((u8 *)ctx->attr + + le16_to_cpu(ctx->attr->data.resident.value_offset)); + if (MREF_LE(fn->parent_directory) == ni->mft_no) + continue; + + index_vi = ntfs_iget(sb, MREF_LE(fn->parent_directory)); + if (IS_ERR(index_vi)) { + ntfs_error(sb, "Failed to open inode %lld with index", + (long long)MREF_LE(fn->parent_directory)); + continue; + } + + index_ni = NTFS_I(index_vi); + + mutex_lock_nested(&index_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT); + if (NInoBeingDeleted(ni)) { + mutex_unlock(&index_ni->mrec_lock); + iput(index_vi); + continue; + } + + ictx = ntfs_index_ctx_get(index_ni, I30, 4); + if (!ictx) { + ntfs_error(sb, "Failed to get index ctx, inode %llu", + index_ni->mft_no); + mutex_unlock(&index_ni->mrec_lock); + iput(index_vi); + continue; + } + + err = ntfs_index_lookup(fn, sizeof(struct file_name_attr), ictx); + if (err) { + ntfs_debug("Index lookup failed, inode %llu", + index_ni->mft_no); + ntfs_index_ctx_put(ictx); + mutex_unlock(&index_ni->mrec_lock); + iput(index_vi); + continue; + } + /* Update flags and file size. */ + fnx = (struct file_name_attr *)ictx->data; + fnx->file_attributes = + (fnx->file_attributes & ~FILE_ATTR_VALID_FLAGS) | + (ni->flags & FILE_ATTR_VALID_FLAGS); + if (ctx->mrec->flags & MFT_RECORD_IS_DIRECTORY) + fnx->data_size = fnx->allocated_size = 0; + else { + read_lock_irqsave(&ni->size_lock, flags); + if (NInoSparse(ni) || NInoCompressed(ni)) + fnx->allocated_size = cpu_to_le64(ni->itype.compressed.size); + else + fnx->allocated_size = cpu_to_le64(ni->allocated_size); + fnx->data_size = cpu_to_le64(ni->data_size); + + /* + * The file name record has also to be fixed if some + * attribute update implied the unnamed data to be + * made non-resident + */ + fn->allocated_size = fnx->allocated_size; + fn->data_size = fnx->data_size; + read_unlock_irqrestore(&ni->size_lock, flags); + } + + /* update or clear the reparse tag in the index */ + fnx->type.rp.reparse_point_tag = reparse_tag; + fnx->creation_time = fn->creation_time; + fnx->last_data_change_time = fn->last_data_change_time; + fnx->last_mft_change_time = fn->last_mft_change_time; + fnx->last_access_time = fn->last_access_time; + ntfs_index_entry_mark_dirty(ictx); + ntfs_icx_ib_sync_write(ictx); + NInoSetDirty(ctx->ntfs_ino); + ntfs_index_ctx_put(ictx); + mutex_unlock(&index_ni->mrec_lock); + iput(index_vi); + } + /* Check for real error occurred. */ + if (err != -ENOENT) { + ntfs_error(sb, "Attribute lookup failed, err : %d, inode %llu", err, + ni->mft_no); + } else + err = 0; + + ntfs_attr_put_search_ctx(ctx); + return err; +} + +int ntfs_get_block_mft_record(struct ntfs_inode *mft_ni, struct ntfs_inode *ni) +{ + s64 vcn; + struct runlist_element *rl; + + if (ni->mft_lcn[0] != LCN_RL_NOT_MAPPED) + return 0; + + vcn = (s64)ni->mft_no << mft_ni->vol->mft_record_size_bits >> + mft_ni->vol->cluster_size_bits; + + rl = mft_ni->runlist.rl; + if (!rl) { + ntfs_error(mft_ni->vol->sb, "$MFT runlist is not present"); + return -EIO; + } + + /* Seek to element containing target vcn. */ + while (rl->length && rl[1].vcn <= vcn) + rl++; + ni->mft_lcn[0] = ntfs_rl_vcn_to_lcn(rl, vcn); + ni->mft_lcn_count = 1; + + if (mft_ni->vol->cluster_size < mft_ni->vol->mft_record_size && + (rl->length - (vcn - rl->vcn)) <= 1) { + rl++; + ni->mft_lcn[1] = ntfs_rl_vcn_to_lcn(rl, vcn + 1); + ni->mft_lcn_count++; + } + return 0; +} + +/* + * __ntfs_write_inode - write out a dirty inode + * @vi: inode to write out + * @sync: if true, write out synchronously + * + * Write out a dirty inode to disk including any extent inodes if present. + * + * If @sync is true, commit the inode to disk and wait for io completion. This + * is done using write_mft_record(). + * + * If @sync is false, just schedule the write to happen but do not wait for i/o + * completion. + * + * Return 0 on success and -errno on error. + */ +int __ntfs_write_inode(struct inode *vi, int sync) +{ + struct ntfs_inode *ni = NTFS_I(vi); + struct ntfs_inode *mft_ni = NTFS_I(ni->vol->mft_ino); + struct mft_record *m; + int err = 0; + bool need_iput = false; + + ntfs_debug("Entering for %sinode 0x%llx.", NInoAttr(ni) ? "attr " : "", + ni->mft_no); + + if (NVolShutdown(ni->vol)) + return -EIO; + + /* + * Dirty attribute inodes are written via their real inodes so just + * clean them here. Access time updates are taken care off when the + * real inode is written. + */ + if (NInoAttr(ni) || ni->nr_extents == -1) { + NInoClearDirty(ni); + ntfs_debug("Done."); + return 0; + } + + /* igrab prevents vi from being evicted while mrec_lock is hold. */ + if (igrab(vi) != NULL) + need_iput = true; + + mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL); + /* Map, pin, and lock the mft record belonging to the inode. */ + m = map_mft_record(ni); + if (IS_ERR(m)) { + mutex_unlock(&ni->mrec_lock); + err = PTR_ERR(m); + goto err_out; + } + + if (NInoNonResident(ni) && NInoRunlistDirty(ni)) { + down_write(&ni->runlist.lock); + err = ntfs_attr_update_mapping_pairs(ni, 0); + if (!err) + NInoClearRunlistDirty(ni); + up_write(&ni->runlist.lock); + } + + err = ntfs_inode_sync_standard_information(vi, m); + if (err) + goto unm_err_out; + + /* + * when being umounted and inodes are evicted, write_inode() + * is called with all inodes being marked with I_FREEING. + * then ntfs_inode_sync_filename() waits infinitly because + * of ntfs_iget. This situation happens only where sync_filesysem() + * from umount fails because of a disk unplug and etc. + * the absent of SB_ACTIVE means umounting. + */ + if ((vi->i_sb->s_flags & SB_ACTIVE) && NInoTestClearFileNameDirty(ni)) + ntfs_inode_sync_filename(ni); + + /* Now the access times are updated, write the base mft record. */ + if (NInoDirty(ni)) { + down_read(&mft_ni->runlist.lock); + err = ntfs_get_block_mft_record(mft_ni, ni); + up_read(&mft_ni->runlist.lock); + if (err) + goto unm_err_out; + + err = write_mft_record(ni, m, sync); + if (err) + ntfs_error(vi->i_sb, "write_mft_record failed, err : %d\n", err); + } + unmap_mft_record(ni); + + /* Map any unmapped extent mft records with LCNs. */ + down_read(&mft_ni->runlist.lock); + mutex_lock(&ni->extent_lock); + if (ni->nr_extents > 0) { + int i; + + for (i = 0; i < ni->nr_extents; i++) { + err = ntfs_get_block_mft_record(mft_ni, + ni->ext.extent_ntfs_inos[i]); + if (err) { + mutex_unlock(&ni->extent_lock); + up_read(&mft_ni->runlist.lock); + mutex_unlock(&ni->mrec_lock); + goto err_out; + } + } + } + mutex_unlock(&ni->extent_lock); + up_read(&mft_ni->runlist.lock); + + /* Write all attached extent mft records. */ + mutex_lock(&ni->extent_lock); + if (ni->nr_extents > 0) { + struct ntfs_inode **extent_nis = ni->ext.extent_ntfs_inos; + int i; + + ntfs_debug("Writing %i extent inodes.", ni->nr_extents); + for (i = 0; i < ni->nr_extents; i++) { + struct ntfs_inode *tni = extent_nis[i]; + + if (NInoDirty(tni)) { + struct mft_record *tm; + int ret; + + mutex_lock(&tni->mrec_lock); + tm = map_mft_record(tni); + if (IS_ERR(tm)) { + mutex_unlock(&tni->mrec_lock); + if (!err || err == -ENOMEM) + err = PTR_ERR(tm); + continue; + } + + ret = write_mft_record(tni, tm, sync); + unmap_mft_record(tni); + mutex_unlock(&tni->mrec_lock); + + if (unlikely(ret)) { + if (!err || err == -ENOMEM) + err = ret; + } + } + } + } + mutex_unlock(&ni->extent_lock); + mutex_unlock(&ni->mrec_lock); + + if (unlikely(err)) + goto err_out; + if (need_iput) + iput(vi); + ntfs_debug("Done."); + return 0; +unm_err_out: + unmap_mft_record(ni); + mutex_unlock(&ni->mrec_lock); +err_out: + if (err == -ENOMEM) + mark_inode_dirty(vi); + else { + ntfs_error(vi->i_sb, "Failed (error %i): Run chkdsk.", -err); + NVolSetErrors(ni->vol); + } + if (need_iput) + iput(vi); + return err; +} + +/* + * ntfs_extent_inode_open - load an extent inode and attach it to its base + * @base_ni: base ntfs inode + * @mref: mft reference of the extent inode to load (in little endian) + * + * First check if the extent inode @mref is already attached to the base ntfs + * inode @base_ni, and if so, return a pointer to the attached extent inode. + * + * If the extent inode is not already attached to the base inode, allocate an + * ntfs_inode structure and initialize it for the given inode @mref. @mref + * specifies the inode number / mft record to read, including the sequence + * number, which can be 0 if no sequence number checking is to be performed. + * + * Then, allocate a buffer for the mft record, read the mft record from the + * volume @base_ni->vol, and attach it to the ntfs_inode structure (->mrec). + * The mft record is mst deprotected and sanity checked for validity and we + * abort if deprotection or checks fail. + * + * Finally attach the ntfs inode to its base inode @base_ni and return a + * pointer to the ntfs_inode structure on success or NULL on error, with errno + * set to the error code. + * + * Note, extent inodes are never closed directly. They are automatically + * disposed off by the closing of the base inode. + */ +static struct ntfs_inode *ntfs_extent_inode_open(struct ntfs_inode *base_ni, + const __le64 mref) +{ + u64 mft_no = MREF_LE(mref); + struct ntfs_inode *ni = NULL; + struct ntfs_inode **extent_nis; + int i; + struct mft_record *ni_mrec; + struct super_block *sb; + + if (!base_ni) + return NULL; + + sb = base_ni->vol->sb; + ntfs_debug("Opening extent inode %llu (base mft record %llu).\n", + mft_no, base_ni->mft_no); + + /* Is the extent inode already open and attached to the base inode? */ + if (base_ni->nr_extents > 0) { + extent_nis = base_ni->ext.extent_ntfs_inos; + for (i = 0; i < base_ni->nr_extents; i++) { + u16 seq_no; + + ni = extent_nis[i]; + if (mft_no != ni->mft_no) + continue; + ni_mrec = map_mft_record(ni); + if (IS_ERR(ni_mrec)) { + ntfs_error(sb, "failed to map mft record for %llu", + ni->mft_no); + goto out; + } + /* Verify the sequence number if given. */ + seq_no = MSEQNO_LE(mref); + if (seq_no && + seq_no != le16_to_cpu(ni_mrec->sequence_number)) { + ntfs_error(sb, "Found stale extent mft reference mft=%llu", + ni->mft_no); + unmap_mft_record(ni); + goto out; + } + unmap_mft_record(ni); + goto out; + } + } + /* Wasn't there, we need to load the extent inode. */ + ni = ntfs_new_extent_inode(base_ni->vol->sb, mft_no); + if (!ni) + goto out; + + ni->seq_no = (u16)MSEQNO_LE(mref); + ni->nr_extents = -1; + ni->ext.base_ntfs_ino = base_ni; + /* Attach extent inode to base inode, reallocating memory if needed. */ + if (!(base_ni->nr_extents & 3)) { + i = (base_ni->nr_extents + 4) * sizeof(struct ntfs_inode *); + + extent_nis = kvzalloc(i, GFP_NOFS); + if (!extent_nis) + goto err_out; + if (base_ni->nr_extents) { + memcpy(extent_nis, base_ni->ext.extent_ntfs_inos, + i - 4 * sizeof(struct ntfs_inode *)); + kvfree(base_ni->ext.extent_ntfs_inos); + } + base_ni->ext.extent_ntfs_inos = extent_nis; + } + base_ni->ext.extent_ntfs_inos[base_ni->nr_extents++] = ni; + +out: + ntfs_debug("\n"); + return ni; +err_out: + ntfs_destroy_ext_inode(ni); + ni = NULL; + goto out; +} + +/* + * ntfs_inode_attach_all_extents - attach all extents for target inode + * @ni: opened ntfs inode for which perform attach + * + * Return 0 on success and error. + */ +int ntfs_inode_attach_all_extents(struct ntfs_inode *ni) +{ + struct attr_list_entry *ale; + u64 prev_attached = 0; + + if (!ni) { + ntfs_debug("Invalid arguments.\n"); + return -EINVAL; + } + + if (NInoAttr(ni)) + ni = ni->ext.base_ntfs_ino; + + ntfs_debug("Entering for inode 0x%llx.\n", ni->mft_no); + + /* Inode haven't got attribute list, thus nothing to attach. */ + if (!NInoAttrList(ni)) + return 0; + + if (!ni->attr_list) { + ntfs_debug("Corrupt in-memory struct.\n"); + return -EINVAL; + } + + /* Walk through attribute list and attach all extents. */ + ale = (struct attr_list_entry *)ni->attr_list; + while ((u8 *)ale < ni->attr_list + ni->attr_list_size) { + if (ni->mft_no != MREF_LE(ale->mft_reference) && + prev_attached != MREF_LE(ale->mft_reference)) { + if (!ntfs_extent_inode_open(ni, ale->mft_reference)) { + ntfs_debug("Couldn't attach extent inode.\n"); + return -1; + } + prev_attached = MREF_LE(ale->mft_reference); + } + ale = (struct attr_list_entry *)((u8 *)ale + le16_to_cpu(ale->length)); + } + return 0; +} + +/* + * ntfs_inode_add_attrlist - add attribute list to inode and fill it + * @ni: opened ntfs inode to which add attribute list + * + * Return 0 on success or error. + */ +int ntfs_inode_add_attrlist(struct ntfs_inode *ni) +{ + int err; + struct ntfs_attr_search_ctx *ctx; + u8 *al = NULL, *aln; + int al_len = 0; + struct attr_list_entry *ale = NULL; + struct mft_record *ni_mrec; + u32 attr_al_len; + + if (!ni) + return -EINVAL; + + ntfs_debug("inode %llu\n", ni->mft_no); + + if (NInoAttrList(ni) || ni->nr_extents) { + ntfs_error(ni->vol->sb, "Inode already has attribute list"); + return -EEXIST; + } + + ni_mrec = map_mft_record(ni); + if (IS_ERR(ni_mrec)) + return -EIO; + + /* Form attribute list. */ + ctx = ntfs_attr_get_search_ctx(ni, ni_mrec); + if (!ctx) { + err = -ENOMEM; + goto err_out; + } + + /* Walk through all attributes. */ + while (!(err = ntfs_attr_lookup(AT_UNUSED, NULL, 0, 0, 0, NULL, 0, ctx))) { + int ale_size; + + if (ctx->attr->type == AT_ATTRIBUTE_LIST) { + err = -EIO; + ntfs_error(ni->vol->sb, "Attribute list already present"); + goto put_err_out; + } + + ale_size = (sizeof(struct attr_list_entry) + sizeof(__le16) * + ctx->attr->name_length + 7) & ~7; + al_len += ale_size; + + aln = kvrealloc(al, al_len, GFP_NOFS); + if (!aln) { + err = -ENOMEM; + ntfs_error(ni->vol->sb, "Failed to realloc %d bytes", al_len); + goto put_err_out; + } + ale = (struct attr_list_entry *)(aln + ((u8 *)ale - al)); + al = aln; + + memset(ale, 0, ale_size); + + /* Add attribute to attribute list. */ + ale->type = ctx->attr->type; + ale->length = cpu_to_le16((sizeof(struct attr_list_entry) + + sizeof(__le16) * ctx->attr->name_length + 7) & ~7); + ale->name_length = ctx->attr->name_length; + ale->name_offset = (u8 *)ale->name - (u8 *)ale; + if (ctx->attr->non_resident) + ale->lowest_vcn = + ctx->attr->data.non_resident.lowest_vcn; + else + ale->lowest_vcn = 0; + ale->mft_reference = MK_LE_MREF(ni->mft_no, + le16_to_cpu(ni_mrec->sequence_number)); + ale->instance = ctx->attr->instance; + memcpy(ale->name, (u8 *)ctx->attr + + le16_to_cpu(ctx->attr->name_offset), + ctx->attr->name_length * sizeof(__le16)); + ale = (struct attr_list_entry *)(al + al_len); + } + + /* Check for real error occurred. */ + if (err != -ENOENT) { + ntfs_error(ni->vol->sb, "%s: Attribute lookup failed, inode %llu", + __func__, ni->mft_no); + goto put_err_out; + } + + /* Set in-memory attribute list. */ + ni->attr_list = al; + ni->attr_list_size = al_len; + NInoSetAttrList(ni); + + attr_al_len = offsetof(struct attr_record, data.resident.reserved) + 1 + + ((al_len + 7) & ~7); + /* Free space if there is not enough it for $ATTRIBUTE_LIST. */ + if (le32_to_cpu(ni_mrec->bytes_allocated) - + le32_to_cpu(ni_mrec->bytes_in_use) < attr_al_len) { + if (ntfs_inode_free_space(ni, (int)attr_al_len)) { + /* Failed to free space. */ + err = -ENOSPC; + ntfs_error(ni->vol->sb, "Failed to free space for attrlist"); + goto rollback; + } + } + + /* Add $ATTRIBUTE_LIST to mft record. */ + err = ntfs_resident_attr_record_add(ni, AT_ATTRIBUTE_LIST, AT_UNNAMED, 0, + NULL, al_len, 0); + if (err < 0) { + ntfs_error(ni->vol->sb, "Couldn't add $ATTRIBUTE_LIST to MFT"); + goto rollback; + } + + err = ntfs_attrlist_update(ni); + if (err < 0) + goto remove_attrlist_record; + + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(ni); + return 0; + +remove_attrlist_record: + /* Prevent ntfs_attr_recorm_rm from freeing attribute list. */ + ni->attr_list = NULL; + NInoClearAttrList(ni); + /* Remove $ATTRIBUTE_LIST record. */ + ntfs_attr_reinit_search_ctx(ctx); + if (!ntfs_attr_lookup(AT_ATTRIBUTE_LIST, NULL, 0, + CASE_SENSITIVE, 0, NULL, 0, ctx)) { + if (ntfs_attr_record_rm(ctx)) + ntfs_error(ni->vol->sb, "Rollback failed to remove attrlist"); + } else { + ntfs_error(ni->vol->sb, "Rollback failed to find attrlist"); + } + + /* Setup back in-memory runlist. */ + ni->attr_list = al; + ni->attr_list_size = al_len; + NInoSetAttrList(ni); +rollback: + /* + * Scan attribute list for attributes that placed not in the base MFT + * record and move them to it. + */ + ntfs_attr_reinit_search_ctx(ctx); + ale = (struct attr_list_entry *)al; + while ((u8 *)ale < al + al_len) { + if (MREF_LE(ale->mft_reference) != ni->mft_no) { + if (!ntfs_attr_lookup(ale->type, ale->name, + ale->name_length, + CASE_SENSITIVE, + le64_to_cpu(ale->lowest_vcn), + NULL, 0, ctx)) { + if (ntfs_attr_record_move_to(ctx, ni)) + ntfs_error(ni->vol->sb, + "Rollback failed to move attribute"); + } else { + ntfs_error(ni->vol->sb, "Rollback failed to find attr"); + } + ntfs_attr_reinit_search_ctx(ctx); + } + ale = (struct attr_list_entry *)((u8 *)ale + le16_to_cpu(ale->length)); + } + + /* Remove in-memory attribute list. */ + ni->attr_list = NULL; + ni->attr_list_size = 0; + NInoClearAttrList(ni); + NInoClearAttrListDirty(ni); +put_err_out: + ntfs_attr_put_search_ctx(ctx); +err_out: + kvfree(al); + unmap_mft_record(ni); + return err; +} + +/* + * ntfs_inode_close - close an ntfs inode and free all associated memory + * @ni: ntfs inode to close + * + * Make sure the ntfs inode @ni is clean. + * + * If the ntfs inode @ni is a base inode, close all associated extent inodes, + * then deallocate all memory attached to it, and finally free the ntfs inode + * structure itself. + * + * If it is an extent inode, we disconnect it from its base inode before we + * destroy it. + * + * It is OK to pass NULL to this function, it is just noop in this case. + * + * Return 0 on success or error. + */ +int ntfs_inode_close(struct ntfs_inode *ni) +{ + int err = -1; + struct ntfs_inode **tmp_nis; + struct ntfs_inode *base_ni; + s32 i; + + if (!ni) + return 0; + + ntfs_debug("Entering for inode %llu\n", ni->mft_no); + + /* Is this a base inode with mapped extent inodes? */ + /* + * If the inode is an extent inode, disconnect it from the + * base inode before destroying it. + */ + base_ni = ni->ext.base_ntfs_ino; + tmp_nis = base_ni->ext.extent_ntfs_inos; + if (!tmp_nis) + goto out; + for (i = 0; i < base_ni->nr_extents; ++i) { + if (tmp_nis[i] != ni) + continue; + /* Found it. Disconnect. */ + memmove(tmp_nis + i, tmp_nis + i + 1, + (base_ni->nr_extents - i - 1) * + sizeof(struct ntfs_inode *)); + /* Buffer should be for multiple of four extents. */ + if ((--base_ni->nr_extents) & 3) + break; + /* + * ElectricFence is unhappy with realloc(x,0) as free(x) + * thus we explicitly separate these two cases. + */ + if (base_ni->nr_extents) { + /* Resize the memory buffer. */ + tmp_nis = kvrealloc(tmp_nis, base_ni->nr_extents * + sizeof(struct ntfs_inode *), GFP_NOFS); + /* Ignore errors, they don't really matter. */ + if (tmp_nis) + base_ni->ext.extent_ntfs_inos = tmp_nis; + } else if (tmp_nis) { + kvfree(tmp_nis); + base_ni->ext.extent_ntfs_inos = NULL; + } + break; + } + +out: + if (NInoDirty(ni)) + ntfs_error(ni->vol->sb, "Releasing dirty inode %llu!\n", + ni->mft_no); + if (NInoAttrList(ni) && ni->attr_list) + kvfree(ni->attr_list); + ntfs_destroy_ext_inode(ni); + err = 0; + ntfs_debug("\n"); + return err; +} + +void ntfs_destroy_ext_inode(struct ntfs_inode *ni) +{ + ntfs_debug("Entering."); + if (ni == NULL) + return; + + ntfs_attr_close(ni); + + if (NInoDirty(ni)) + ntfs_error(ni->vol->sb, "Releasing dirty ext inode %llu!\n", + ni->mft_no); + if (NInoAttrList(ni) && ni->attr_list) + kvfree(ni->attr_list); + kfree(ni->mrec); + kmem_cache_free(ntfs_inode_cache, ni); +} + +static struct ntfs_inode *ntfs_inode_base(struct ntfs_inode *ni) +{ + if (ni->nr_extents == -1) + return ni->ext.base_ntfs_ino; + return ni; +} + +static int ntfs_attr_position(__le32 type, struct ntfs_attr_search_ctx *ctx) +{ + int err; + + err = ntfs_attr_lookup(type, NULL, 0, CASE_SENSITIVE, 0, NULL, + 0, ctx); + if (err) { + __le32 atype; + + if (err != -ENOENT) + return err; + + atype = ctx->attr->type; + if (atype == AT_END) + return -ENOSPC; + + /* + * if ntfs_external_attr_lookup return -ENOENT, ctx->al_entry + * could point to an attribute in an extent mft record, but + * ctx->attr and ctx->ntfs_ino always points to an attibute in + * a base mft record. + */ + if (ctx->al_entry && + MREF_LE(ctx->al_entry->mft_reference) != ctx->ntfs_ino->mft_no) { + ntfs_attr_reinit_search_ctx(ctx); + err = ntfs_attr_lookup(atype, NULL, 0, CASE_SENSITIVE, 0, NULL, + 0, ctx); + if (err) + return err; + } + } + return 0; +} + +/* + * ntfs_inode_free_space - free space in the MFT record of inode + * @ni: ntfs inode in which MFT record free space + * @size: amount of space needed to free + * + * Return 0 on success or error. + */ +int ntfs_inode_free_space(struct ntfs_inode *ni, int size) +{ + struct ntfs_attr_search_ctx *ctx; + int freed, err; + struct mft_record *ni_mrec; + struct super_block *sb; + + if (!ni || size < 0) + return -EINVAL; + ntfs_debug("Entering for inode %llu, size %d\n", ni->mft_no, size); + + sb = ni->vol->sb; + ni_mrec = map_mft_record(ni); + if (IS_ERR(ni_mrec)) + return -EIO; + + freed = (le32_to_cpu(ni_mrec->bytes_allocated) - + le32_to_cpu(ni_mrec->bytes_in_use)); + + unmap_mft_record(ni); + + if (size <= freed) + return 0; + + ctx = ntfs_attr_get_search_ctx(ni, NULL); + if (!ctx) { + ntfs_error(sb, "%s, Failed to get search context", __func__); + return -ENOMEM; + } + + /* + * Chkdsk complain if $STANDARD_INFORMATION is not in the base MFT + * record. + * + * Also we can't move $ATTRIBUTE_LIST from base MFT_RECORD, so position + * search context on first attribute after $STANDARD_INFORMATION and + * $ATTRIBUTE_LIST. + * + * Why we reposition instead of simply skip this attributes during + * enumeration? Because in case we have got only in-memory attribute + * list ntfs_attr_lookup will fail when it will try to find + * $ATTRIBUTE_LIST. + */ + err = ntfs_attr_position(AT_FILE_NAME, ctx); + if (err) + goto put_err_out; + + while (1) { + int record_size; + + /* + * Check whether attribute is from different MFT record. If so, + * find next, because we don't need such. + */ + while (ctx->ntfs_ino->mft_no != ni->mft_no) { +retry: + err = ntfs_attr_lookup(AT_UNUSED, NULL, 0, CASE_SENSITIVE, + 0, NULL, 0, ctx); + if (err) { + if (err != -ENOENT) + ntfs_error(sb, "Attr lookup failed #2"); + else if (ctx->attr->type == AT_END) + err = -ENOSPC; + else + err = 0; + + if (err) + goto put_err_out; + } + } + + if (ntfs_inode_base(ctx->ntfs_ino)->mft_no == FILE_MFT && + ctx->attr->type == AT_DATA) + goto retry; + + if (ctx->attr->type == AT_INDEX_ROOT) + goto retry; + + record_size = le32_to_cpu(ctx->attr->length); + + /* Move away attribute. */ + err = ntfs_attr_record_move_away(ctx, 0); + if (err) { + ntfs_error(sb, "Failed to move out attribute #2"); + break; + } + freed += record_size; + + /* Check whether we done. */ + if (size <= freed) { + ntfs_attr_put_search_ctx(ctx); + return 0; + } + + /* + * Reposition to first attribute after $STANDARD_INFORMATION and + * $ATTRIBUTE_LIST (see comments upwards). + */ + ntfs_attr_reinit_search_ctx(ctx); + err = ntfs_attr_position(AT_FILE_NAME, ctx); + if (err) + break; + } +put_err_out: + ntfs_attr_put_search_ctx(ctx); + if (err == -ENOSPC) + ntfs_debug("No attributes left that can be moved out.\n"); + return err; +} + +s64 ntfs_inode_attr_pread(struct inode *vi, s64 pos, s64 count, u8 *buf) +{ + struct address_space *mapping = vi->i_mapping; + struct folio *folio; + struct ntfs_inode *ni = NTFS_I(vi); + s64 isize; + u32 attr_len, total = 0, offset; + pgoff_t index; + int err = 0; + + WARN_ON(!NInoAttr(ni)); + if (!count) + return 0; + + mutex_lock(&ni->mrec_lock); + isize = i_size_read(vi); + if (pos > isize) { + mutex_unlock(&ni->mrec_lock); + return -EINVAL; + } + if (pos + count > isize) + count = isize - pos; + + if (!NInoNonResident(ni)) { + struct ntfs_attr_search_ctx *ctx; + u8 *attr; + + ctx = ntfs_attr_get_search_ctx(ni->ext.base_ntfs_ino, NULL); + if (!ctx) { + ntfs_error(vi->i_sb, "Failed to get attr search ctx"); + err = -ENOMEM; + mutex_unlock(&ni->mrec_lock); + goto out; + } + + err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len, CASE_SENSITIVE, + 0, NULL, 0, ctx); + if (err) { + ntfs_error(vi->i_sb, "Failed to look up attr %#x", ni->type); + ntfs_attr_put_search_ctx(ctx); + mutex_unlock(&ni->mrec_lock); + goto out; + } + + attr = (u8 *)ctx->attr + le16_to_cpu(ctx->attr->data.resident.value_offset); + memcpy(buf, (u8 *)attr + pos, count); + ntfs_attr_put_search_ctx(ctx); + mutex_unlock(&ni->mrec_lock); + return count; + } + mutex_unlock(&ni->mrec_lock); + + index = pos >> PAGE_SHIFT; + do { + /* Update @index and get the next folio. */ + folio = read_mapping_folio(mapping, index, NULL); + if (IS_ERR(folio)) + break; + + offset = offset_in_folio(folio, pos); + attr_len = min_t(size_t, (size_t)count, folio_size(folio) - offset); + + folio_lock(folio); + memcpy_from_folio(buf, folio, offset, attr_len); + folio_unlock(folio); + folio_put(folio); + + total += attr_len; + buf += attr_len; + pos += attr_len; + count -= attr_len; + index++; + } while (count); +out: + return err ? (s64)err : total; +} + +static inline int ntfs_enlarge_attribute(struct inode *vi, s64 pos, s64 count, + struct ntfs_attr_search_ctx *ctx) +{ + struct ntfs_inode *ni = NTFS_I(vi); + struct super_block *sb = vi->i_sb; + int ret; + + if (pos + count <= ni->initialized_size) + return 0; + + if (NInoEncrypted(ni) && NInoNonResident(ni)) + return -EACCES; + + if (NInoCompressed(ni)) + return -EOPNOTSUPP; + + if (pos + count > ni->data_size) { + if (ntfs_attr_truncate(ni, pos + count)) { + ntfs_debug("Failed to truncate attribute"); + return -1; + } + + ntfs_attr_reinit_search_ctx(ctx); + ret = ntfs_attr_lookup(ni->type, + ni->name, ni->name_len, CASE_SENSITIVE, + 0, NULL, 0, ctx); + if (ret) { + ntfs_error(sb, "Failed to look up attr %#x", ni->type); + return ret; + } + } + + if (!NInoNonResident(ni)) { + if (likely(i_size_read(vi) < ni->data_size)) + i_size_write(vi, ni->data_size); + return 0; + } + + if (pos + count > ni->initialized_size) { + ctx->attr->data.non_resident.initialized_size = cpu_to_le64(pos + count); + mark_mft_record_dirty(ctx->ntfs_ino); + ni->initialized_size = pos + count; + if (i_size_read(vi) < ni->initialized_size) + i_size_write(vi, ni->initialized_size); + } + return 0; +} + +static s64 __ntfs_inode_resident_attr_pwrite(struct inode *vi, + s64 pos, s64 count, u8 *buf, + struct ntfs_attr_search_ctx *ctx) +{ + struct ntfs_inode *ni = NTFS_I(vi); + struct folio *folio; + struct address_space *mapping = vi->i_mapping; + u8 *addr; + int err = 0; + + WARN_ON(NInoNonResident(ni)); + if (pos + count > PAGE_SIZE) { + ntfs_error(vi->i_sb, "Out of write into resident attr %#x", ni->type); + return -EINVAL; + } + + /* Copy to mft record page */ + addr = (u8 *)ctx->attr + le16_to_cpu(ctx->attr->data.resident.value_offset); + memcpy(addr + pos, buf, count); + mark_mft_record_dirty(ctx->ntfs_ino); + + /* Keep the first page clean and uptodate */ + folio = __filemap_get_folio(mapping, 0, FGP_WRITEBEGIN | FGP_NOFS, + mapping_gfp_mask(mapping)); + if (IS_ERR(folio)) { + err = PTR_ERR(folio); + ntfs_error(vi->i_sb, "Failed to read a page 0 for attr %#x: %d", + ni->type, err); + goto out; + } + if (!folio_test_uptodate(folio)) + folio_fill_tail(folio, 0, addr, + le32_to_cpu(ctx->attr->data.resident.value_length)); + else + memcpy_to_folio(folio, offset_in_folio(folio, pos), buf, count); + folio_mark_uptodate(folio); + folio_unlock(folio); + folio_put(folio); +out: + return err ? err : count; +} + +static s64 __ntfs_inode_non_resident_attr_pwrite(struct inode *vi, + s64 pos, s64 count, u8 *buf, + struct ntfs_attr_search_ctx *ctx, + bool sync) +{ + struct ntfs_inode *ni = NTFS_I(vi); + struct address_space *mapping = vi->i_mapping; + struct folio *folio; + pgoff_t index; + unsigned long offset, length; + size_t attr_len; + s64 ret = 0, written = 0; + + WARN_ON(!NInoNonResident(ni)); + + index = pos >> PAGE_SHIFT; + while (count) { + if (count == PAGE_SIZE) { + folio = __filemap_get_folio(vi->i_mapping, index, + FGP_CREAT | FGP_LOCK, + mapping_gfp_mask(mapping)); + if (IS_ERR(folio)) { + ret = -ENOMEM; + break; + } + } else { + folio = read_mapping_folio(mapping, index, NULL); + if (IS_ERR(folio)) { + ret = PTR_ERR(folio); + ntfs_error(vi->i_sb, "Failed to read a page %lu for attr %#x: %ld", + index, ni->type, PTR_ERR(folio)); + break; + } + + folio_lock(folio); + } + + if (count == PAGE_SIZE) { + offset = 0; + attr_len = count; + } else { + offset = offset_in_folio(folio, pos); + attr_len = min_t(size_t, (size_t)count, folio_size(folio) - offset); + } + memcpy_to_folio(folio, offset, buf, attr_len); + + if (sync) { + struct ntfs_volume *vol = ni->vol; + s64 lcn, lcn_count; + unsigned int lcn_folio_off = 0; + struct bio *bio; + u64 rl_length = 0; + s64 vcn; + struct runlist_element *rl; + + lcn_count = max_t(s64, 1, ntfs_bytes_to_cluster(vol, attr_len)); + vcn = ntfs_pidx_to_cluster(vol, folio->index); + + do { + down_write(&ni->runlist.lock); + rl = ntfs_attr_vcn_to_rl(ni, vcn, &lcn); + if (IS_ERR(rl)) { + ret = PTR_ERR(rl); + up_write(&ni->runlist.lock); + goto err_unlock_folio; + } + + rl_length = rl->length - (vcn - rl->vcn); + if (rl_length < lcn_count) { + lcn_count -= rl_length; + } else { + rl_length = lcn_count; + lcn_count = 0; + } + up_write(&ni->runlist.lock); + + if (vol->cluster_size_bits > PAGE_SHIFT) { + lcn_folio_off = folio->index << PAGE_SHIFT; + lcn_folio_off &= vol->cluster_size_mask; + } + + bio = bio_alloc(vol->sb->s_bdev, 1, REQ_OP_WRITE, + GFP_NOIO); + bio->bi_iter.bi_sector = + ntfs_bytes_to_sector(vol, + ntfs_cluster_to_bytes(vol, lcn) + + lcn_folio_off); + + length = min_t(unsigned long, + ntfs_cluster_to_bytes(vol, rl_length), + folio_size(folio)); + if (!bio_add_folio(bio, folio, length, offset)) { + ret = -EIO; + bio_put(bio); + goto err_unlock_folio; + } + + submit_bio_wait(bio); + bio_put(bio); + vcn += rl_length; + offset += length; + } while (lcn_count != 0); + + folio_mark_uptodate(folio); + } else { + folio_mark_uptodate(folio); + folio_mark_dirty(folio); + } +err_unlock_folio: + folio_unlock(folio); + folio_put(folio); + + if (ret) + break; + + written += attr_len; + buf += attr_len; + pos += attr_len; + count -= attr_len; + index++; + + cond_resched(); + } + + return ret ? ret : written; +} + +s64 ntfs_inode_attr_pwrite(struct inode *vi, s64 pos, s64 count, u8 *buf, bool sync) +{ + struct ntfs_inode *ni = NTFS_I(vi); + struct ntfs_attr_search_ctx *ctx; + s64 ret; + + WARN_ON(!NInoAttr(ni)); + + ctx = ntfs_attr_get_search_ctx(ni->ext.base_ntfs_ino, NULL); + if (!ctx) { + ntfs_error(vi->i_sb, "Failed to get attr search ctx"); + return -ENOMEM; + } + + ret = ntfs_attr_lookup(ni->type, ni->name, ni->name_len, CASE_SENSITIVE, + 0, NULL, 0, ctx); + if (ret) { + ntfs_attr_put_search_ctx(ctx); + ntfs_error(vi->i_sb, "Failed to look up attr %#x", ni->type); + return ret; + } + + mutex_lock(&ni->mrec_lock); + ret = ntfs_enlarge_attribute(vi, pos, count, ctx); + mutex_unlock(&ni->mrec_lock); + if (ret) + goto out; + + if (NInoNonResident(ni)) + ret = __ntfs_inode_non_resident_attr_pwrite(vi, pos, count, buf, ctx, sync); + else + ret = __ntfs_inode_resident_attr_pwrite(vi, pos, count, buf, ctx); +out: + ntfs_attr_put_search_ctx(ctx); + return ret; +} + +struct folio *ntfs_get_locked_folio(struct address_space *mapping, + pgoff_t index, pgoff_t end_index, struct file_ra_state *ra) +{ + struct folio *folio; + + folio = filemap_lock_folio(mapping, index); + if (IS_ERR(folio)) { + if (PTR_ERR(folio) != -ENOENT) + return folio; + + page_cache_sync_readahead(mapping, ra, NULL, index, + end_index - index); + folio = read_mapping_folio(mapping, index, NULL); + if (!IS_ERR(folio)) + folio_lock(folio); + } + + return folio; +} diff --git a/fs/ntfs/inode.h b/fs/ntfs/inode.h new file mode 100644 index 000000000..9aacd5787 --- /dev/null +++ b/fs/ntfs/inode.h @@ -0,0 +1,360 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Defines for inode structures NTFS Linux kernel driver. + * + * Copyright (c) 2001-2007 Anton Altaparmakov + * Copyright (c) 2002 Richard Russon + * Copyright (c) 2025 LG Electronics Co., Ltd. + */ + +#ifndef _LINUX_NTFS_INODE_H +#define _LINUX_NTFS_INODE_H + +#include "debug.h" + +enum ntfs_inode_mutex_lock_class { + NTFS_INODE_MUTEX_PARENT, + NTFS_INODE_MUTEX_NORMAL, + NTFS_INODE_MUTEX_NORMAL_CHILD, + NTFS_INODE_MUTEX_PARENT_2, + NTFS_INODE_MUTEX_NORMAL_2, + NTFS_EXTEND_MUTEX_PARENT, + NTFS_EA_MUTEX_NORMAL +}; + +/* + * The NTFS in-memory inode structure. It is just used as an extension to the + * fields already provided in the VFS inode. + * @size_lock: Lock serializing access to inode sizes. + * @state: NTFS specific flags describing this inode. + * @flags: Flags describing the file. (Copy from STANDARD_INFORMATION). + * @mft_no: Number of the mft record / inode. + * @seq_no: Sequence number of the mft record. + * @count: Inode reference count for book keeping. + * @vol: Pointer to the ntfs volume of this inode. + * + * If NInoAttr() is true, the below fields describe the attribute which + * this fake inode belongs to. The actual inode of this attribute is + * pointed to by base_ntfs_ino and nr_extents is always set to -1 (see + * below). For real inodes, we also set the type (AT_DATA for files and + * AT_INDEX_ALLOCATION for directories), with the name = NULL and + * name_len = 0 for files and name = I30 (global constant) and + * name_len = 4 for directories. + * @type: Attribute type of this fake inode. + * @name: Attribute name of this fake inode. + * @name_len: Attribute name length of this fake inode. + * @runlist: If state has the NI_NonResident bit set, the runlist of + * the unnamed data attribute (if a file) or of the index allocation + * attribute (directory) or of the attribute described by the fake inode + * (if NInoAttr()). If runlist.rl is NULL, the runlist has not been read + * in yet or has been unmapped. If NI_NonResident is clear, the attribute + * is resident (file and fake inode) or there is no $I30 index allocation + * attribute (small directory). In the latter case runlist.rl is always + * NULL. + * @data_size: Copy from the attribute record. + * @initialized_size: Copy from the attribute record. + * @allocated_size: Copy from the attribute record. + * @i_crtime: File Creation time. + * @mrec: MFT record + * @mrec_lock: Lock for serializing access to the mft record belonging to + * this inode. + * @folio: The folio containing the mft record of the inode. + * @folio_ofs: Offset into the folio at which the mft record begins. + * @mft_lcn: Number containing the mft record. + * @mft_lcn_count: Number of clusters per mft record. + * + * Attribute list support (only for use by the attribute lookup + * functions). Setup during read_inode for all inodes with attribute + * lists. Only valid if NI_AttrList is set in state. + * @attr_list_size: Length of attribute list value in bytes. + * @attr_list: Attribute list value itself. + * + * It is a directory, $MFT, or an index inode. + * @block_size: Size of an index block. + * @vcn_size: Size of a vcn in this index. + * @collation_rule: The collation rule for the index. + * @block_size_bits: Log2 of the above. + * @vcn_size_bits: Log2 of the above. + * + * It is a compressed/sparse file/attribute inode. + * @size: Copy of compressed_size from $DATA. + * @block_size: Size of a compression block (cb). + * @block_size_bits: Log2 of the size of a cb. + * @block_clusters: Number of clusters per cb. + * @extent_lock: Lock for accessing/modifying the below. + * @nr_extents: For a base mft record, the number of attached extent inodes + * (0 if none), for extent records and for fake inodes describing an + * attribute this is -1. + * + * This union is only used if nr_extents != 0. + * @extent_ntfs_inos: For nr_extents > 0, array of the ntfs inodes of + * the extent mft records belonging to this base inode which have been + * loaded. + * @base_ntfs_ino: For nr_extents == -1, the ntfs inode of the base mft + * record. For fake inodes, the real (base) inode to which the attribute + * belongs. + * @i_dealloc_clusters: delayed allocated clusters. + * @target: symlink buffer. + */ +struct ntfs_inode { + rwlock_t size_lock; + unsigned long state; + __le32 flags; + u64 mft_no; + u16 seq_no; + atomic_t count; + struct ntfs_volume *vol; + __le32 type; + __le16 *name; + u32 name_len; + struct runlist runlist; + s64 data_size; + s64 initialized_size; + s64 allocated_size; + struct timespec64 i_crtime; + void *mrec; + struct mutex mrec_lock; + struct folio *folio; + int folio_ofs; + s64 mft_lcn[2]; + unsigned int mft_lcn_count; + u32 attr_list_size; + u8 *attr_list; + union { + struct { + u32 block_size; + u32 vcn_size; + __le32 collation_rule; + u8 block_size_bits; + u8 vcn_size_bits; + } index; + struct { + s64 size; + u32 block_size; + u8 block_size_bits; + u8 block_clusters; + } compressed; + } itype; + struct mutex extent_lock; + s32 nr_extents; + union { + struct ntfs_inode **extent_ntfs_inos; + struct ntfs_inode *base_ntfs_ino; + } ext; + unsigned int i_dealloc_clusters; + __le32 reparse_tag; + __le32 reparse_flags; + char *target; +}; + +/* + * Defined bits for the state field in the ntfs_inode structure. + * (f) = files only, (d) = directories only, (a) = attributes/fake inodes only + * + * NI_Dirty Mft record needs to be written to disk. + * NI_AttrListDirty Mft record contains an attribute list. + * NI_AttrList Mft record contains an attribute list. + * NI_AttrListNonResident Attribute list is non-resident. Implies + * NI_AttrList is set. + * NI_Attr 1: Fake inode for attribute i/o. + * 0: Real inode or extent inode. + * NI_MstProtected Attribute is protected by MST fixups. + * NI_NonResident Unnamed data attr is non-resident (f) + * Attribute is non-resident (a). + * NI_IndexAllocPresent $I30 index alloc attr is present (d). + * NI_Compressed Unnamed data attr is compressed (f). + * Create compressed files by default (d). + * Attribute is compressed (a). + * NI_Encrypted Unnamed data attr is encrypted (f). + * Create encrypted files by default (d). + * Attribute is encrypted (a). + * NI_Sparse Unnamed data attr is sparse (f). + * Create sparse files by default (d). + * Attribute is sparse (a). + * NI_SparseDisabled May not create sparse regions. + * NI_FullyMapped Runlist is fully mapped. + * NI_FileNameDirty FILE_NAME attributes need to be updated. + * NI_BeingDeleted ntfs inode is being delated. + * NI_BeingCreated ntfs inode is being created. + * NI_HasEA ntfs inode has EA attribute. + * NI_RunlistDirty runlist need to be updated. + */ +enum { + NI_Dirty, + NI_AttrListDirty, + NI_AttrList, + NI_AttrListNonResident, + NI_Attr, + NI_MstProtected, + NI_NonResident, + NI_IndexAllocPresent, + NI_Compressed, + NI_Encrypted, + NI_Sparse, + NI_SparseDisabled, + NI_FullyMapped, + NI_FileNameDirty, + NI_BeingDeleted, + NI_BeingCreated, + NI_HasEA, + NI_RunlistDirty, +}; + +/* + * NOTE: We should be adding dirty mft records to a list somewhere and they + * should be independent of the (ntfs/vfs) inode structure so that an inode can + * be removed but the record can be left dirty for syncing later. + */ + +/* + * Macro tricks to expand the NInoFoo(), NInoSetFoo(), and NInoClearFoo() + * functions. + */ +#define NINO_FNS(flag) \ +static inline int NIno##flag(struct ntfs_inode *ni) \ +{ \ + return test_bit(NI_##flag, &(ni)->state); \ +} \ +static inline void NInoSet##flag(struct ntfs_inode *ni) \ +{ \ + set_bit(NI_##flag, &(ni)->state); \ +} \ +static inline void NInoClear##flag(struct ntfs_inode *ni) \ +{ \ + clear_bit(NI_##flag, &(ni)->state); \ +} + +/* + * As above for NInoTestSetFoo() and NInoTestClearFoo(). + */ +#define TAS_NINO_FNS(flag) \ +static inline int NInoTestSet##flag(struct ntfs_inode *ni) \ +{ \ + return test_and_set_bit(NI_##flag, &(ni)->state); \ +} \ +static inline int NInoTestClear##flag(struct ntfs_inode *ni) \ +{ \ + return test_and_clear_bit(NI_##flag, &(ni)->state); \ +} + +/* Emit the ntfs inode bitops functions. */ +NINO_FNS(Dirty) +TAS_NINO_FNS(Dirty) +NINO_FNS(AttrList) +NINO_FNS(AttrListDirty) +NINO_FNS(AttrListNonResident) +NINO_FNS(Attr) +NINO_FNS(MstProtected) +NINO_FNS(NonResident) +NINO_FNS(IndexAllocPresent) +NINO_FNS(Compressed) +NINO_FNS(Encrypted) +NINO_FNS(Sparse) +NINO_FNS(SparseDisabled) +NINO_FNS(FullyMapped) +NINO_FNS(FileNameDirty) +TAS_NINO_FNS(FileNameDirty) +NINO_FNS(BeingDeleted) +NINO_FNS(HasEA) +NINO_FNS(RunlistDirty) + +/* + * The full structure containing a ntfs_inode and a vfs struct inode. Used for + * all real and fake inodes but not for extent inodes which lack the vfs struct + * inode. + */ +struct big_ntfs_inode { + struct ntfs_inode ntfs_inode; + struct inode vfs_inode; /* The vfs inode structure. */ +}; + +/* + * NTFS_I - return the ntfs inode given a vfs inode + * @inode: VFS inode + * + * NTFS_I() returns the ntfs inode associated with the VFS @inode. + */ +static inline struct ntfs_inode *NTFS_I(struct inode *inode) +{ + return &container_of(inode, struct big_ntfs_inode, vfs_inode)->ntfs_inode; +} + +static inline struct inode *VFS_I(struct ntfs_inode *ni) +{ + return &container_of(ni, struct big_ntfs_inode, ntfs_inode)->vfs_inode; +} + +/* + * ntfs_attr - ntfs in memory attribute structure + * + * This structure exists only to provide a small structure for the + * ntfs_{attr_}iget()/ntfs_test_inode()/ntfs_init_locked_inode() mechanism. + * + * NOTE: Elements are ordered by size to make the structure as compact as + * possible on all architectures. + */ +struct ntfs_attr { + u64 mft_no; + __le16 *name; + u32 name_len; + __le32 type; + unsigned long state; +}; + +int ntfs_test_inode(struct inode *vi, void *data); +struct inode *ntfs_iget(struct super_block *sb, u64 mft_no); +struct inode *ntfs_attr_iget(struct inode *base_vi, __le32 type, + __le16 *name, u32 name_len); +struct inode *ntfs_index_iget(struct inode *base_vi, __le16 *name, + u32 name_len); +struct inode *ntfs_alloc_big_inode(struct super_block *sb); +void ntfs_free_big_inode(struct inode *inode); +int ntfs_drop_big_inode(struct inode *inode); +void ntfs_evict_big_inode(struct inode *vi); +void __ntfs_init_inode(struct super_block *sb, struct ntfs_inode *ni); + +static inline void ntfs_init_big_inode(struct inode *vi) +{ + struct ntfs_inode *ni = NTFS_I(vi); + + ntfs_debug("Entering."); + __ntfs_init_inode(vi->i_sb, ni); + ni->mft_no = vi->i_ino; +} + +struct ntfs_inode *ntfs_new_extent_inode(struct super_block *sb, u64 mft_no); +void ntfs_clear_extent_inode(struct ntfs_inode *ni); +int ntfs_read_inode_mount(struct inode *vi); +int ntfs_show_options(struct seq_file *sf, struct dentry *root); +int ntfs_truncate_vfs(struct inode *vi, loff_t new_size, loff_t i_size); + +int ntfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr); +int ntfs_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, unsigned int request_mask, + unsigned int query_flags); + +int ntfs_get_block_mft_record(struct ntfs_inode *mft_ni, struct ntfs_inode *ni); +int __ntfs_write_inode(struct inode *vi, int sync); +int ntfs_inode_attach_all_extents(struct ntfs_inode *ni); +int ntfs_inode_add_attrlist(struct ntfs_inode *ni); +void ntfs_destroy_ext_inode(struct ntfs_inode *ni); +int ntfs_inode_free_space(struct ntfs_inode *ni, int size); +s64 ntfs_inode_attr_pread(struct inode *vi, s64 pos, s64 count, u8 *buf); +s64 ntfs_inode_attr_pwrite(struct inode *vi, s64 pos, s64 count, u8 *buf, + bool sync); +int ntfs_inode_close(struct ntfs_inode *ni); + +static inline void ntfs_commit_inode(struct inode *vi) +{ + __ntfs_write_inode(vi, 1); +} + +int ntfs_inode_sync_filename(struct ntfs_inode *ni); +int ntfs_extend_initialized_size(struct inode *vi, const loff_t offset, + const loff_t new_size, bool bsync); +void ntfs_set_vfs_operations(struct inode *inode, mode_t mode, dev_t dev); +struct folio *ntfs_get_locked_folio(struct address_space *mapping, + pgoff_t index, pgoff_t end_index, struct file_ra_state *ra); + +#endif /* _LINUX_NTFS_INODE_H */ diff --git a/fs/ntfs/iomap.c b/fs/ntfs/iomap.c new file mode 100644 index 000000000..d0964ac84 --- /dev/null +++ b/fs/ntfs/iomap.c @@ -0,0 +1,831 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * iomap callack functions + * + * Copyright (c) 2025 LG Electronics Co., Ltd. + */ + +#include + +#include "attrib.h" +#include "mft.h" +#include "ntfs.h" +#include "iomap.h" + +static void ntfs_iomap_put_folio_non_resident(struct inode *inode, loff_t pos, + unsigned int len, struct folio *folio) +{ + struct ntfs_inode *ni = NTFS_I(inode); + unsigned long sector_size = 1UL << inode->i_blkbits; + loff_t start_down, end_up, init; + + start_down = round_down(pos, sector_size); + end_up = (pos + len - 1) | (sector_size - 1); + init = ni->initialized_size; + + if (init >= start_down && init <= end_up) { + if (init < pos) { + loff_t offset = offset_in_folio(folio, pos + len); + + if (offset == 0) + offset = folio_size(folio); + folio_zero_segments(folio, + offset_in_folio(folio, init), + offset_in_folio(folio, pos), + offset, + folio_size(folio)); + + } else { + loff_t offset = max_t(loff_t, pos + len, init); + + offset = offset_in_folio(folio, offset); + if (offset == 0) + offset = folio_size(folio); + folio_zero_segment(folio, + offset, + folio_size(folio)); + } + } else if (init <= pos) { + loff_t offset = 0, offset2 = offset_in_folio(folio, pos + len); + + if ((init >> folio_shift(folio)) == (pos >> folio_shift(folio))) + offset = offset_in_folio(folio, init); + if (offset2 == 0) + offset2 = folio_size(folio); + folio_zero_segments(folio, + offset, + offset_in_folio(folio, pos), + offset2, + folio_size(folio)); + } + folio_unlock(folio); + folio_put(folio); +} + +/* + * iomap_zero_range is called for an area beyond the initialized size, + * garbage values can be read, so zeroing out is needed. + */ +static void ntfs_iomap_put_folio(struct inode *inode, loff_t pos, + unsigned int len, struct folio *folio) +{ + if (NInoNonResident(NTFS_I(inode))) + return ntfs_iomap_put_folio_non_resident(inode, pos, + len, folio); + folio_unlock(folio); + folio_put(folio); +} + +const struct iomap_write_ops ntfs_iomap_folio_ops = { + .put_folio = ntfs_iomap_put_folio, +}; + +static int ntfs_read_iomap_begin_resident(struct inode *inode, loff_t offset, loff_t length, + unsigned int flags, struct iomap *iomap) +{ + struct ntfs_inode *base_ni, *ni = NTFS_I(inode); + struct ntfs_attr_search_ctx *ctx; + loff_t i_size; + u32 attr_len; + int err = 0; + char *kattr; + + if (NInoAttr(ni)) + base_ni = ni->ext.base_ntfs_ino; + else + base_ni = ni; + + ctx = ntfs_attr_get_search_ctx(base_ni, NULL); + if (!ctx) { + err = -ENOMEM; + goto out; + } + + err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len, + CASE_SENSITIVE, 0, NULL, 0, ctx); + if (unlikely(err)) + goto out; + + attr_len = le32_to_cpu(ctx->attr->data.resident.value_length); + if (unlikely(attr_len > ni->initialized_size)) + attr_len = ni->initialized_size; + i_size = i_size_read(inode); + + if (unlikely(attr_len > i_size)) { + /* Race with shrinking truncate. */ + attr_len = i_size; + } + + if (offset >= attr_len) { + if (flags & IOMAP_REPORT) + err = -ENOENT; + else { + iomap->type = IOMAP_HOLE; + iomap->offset = offset; + iomap->length = length; + } + goto out; + } + + kattr = (u8 *)ctx->attr + le16_to_cpu(ctx->attr->data.resident.value_offset); + + iomap->type = IOMAP_INLINE; + iomap->inline_data = kattr; + iomap->offset = 0; + iomap->length = attr_len; + +out: + if (ctx) + ntfs_attr_put_search_ctx(ctx); + + return err; +} + +/* + * ntfs_read_iomap_begin_non_resident - map non-resident NTFS file data + * @inode: inode to map + * @offset: file offset to map + * @length: length of mapping + * @flags: IOMAP flags + * @iomap: iomap structure to fill + * @need_unwritten: true if UNWRITTEN extent type is needed + * + * Map a range of a non-resident NTFS file to an iomap extent. + * + * NTFS UNWRITTEN extent handling: + * ================================ + * The concept of an unwritten extent in NTFS is slightly different from + * that of other filesystems. NTFS conceptually manages only a single + * continuous unwritten region, which is strictly defined based on + * initialized_size. + * + * File offset layout: + * 0 initialized_size i_size(EOF) + * |----------#0----------|----------#1----------|----------#2----------| + * | Actual data | Pre-allocated | Pre-allocated | + * | (user written) | (within initialized) | (initialized ~ EOF) | + * |----------------------|----------------------|----------------------| + * MAPPED MAPPED UNWRITTEN (conditionally) + * + * Region #0: User-written data, initialized and valid. + * Region #1: Pre-allocated within initialized_size, must be zero-initialized + * by the filesystem before exposure to userspace. + * Region #2: Pre-allocated beyond initialized_size, does not need initialization. + * + * The @need_unwritten parameter controls whether region #2 is mapped as + * IOMAP_UNWRITTEN or IOMAP_MAPPED: + * - For seek operations (SEEK_DATA/SEEK_HOLE): IOMAP_MAPPED is needed to + * prevent iomap_seek_data from incorrectly interpreting pre-allocated + * space as a hole. Since NTFS does not support multiple unwritten extents, + * all pre-allocated regions should be treated as data, not holes. + * - For zero_range operations: IOMAP_MAPPED is needed to be zeroed out. + * + * Return: 0 on success, negative error code on failure. + */ +static int ntfs_read_iomap_begin_non_resident(struct inode *inode, loff_t offset, + loff_t length, unsigned int flags, struct iomap *iomap, + bool need_unwritten) +{ + struct ntfs_inode *ni = NTFS_I(inode); + s64 vcn; + s64 lcn; + struct runlist_element *rl; + struct ntfs_volume *vol = ni->vol; + loff_t vcn_ofs; + loff_t rl_length; + + vcn = ntfs_bytes_to_cluster(vol, offset); + vcn_ofs = ntfs_bytes_to_cluster_off(vol, offset); + + down_write(&ni->runlist.lock); + rl = ntfs_attr_vcn_to_rl(ni, vcn, &lcn); + if (IS_ERR(rl)) { + up_write(&ni->runlist.lock); + return PTR_ERR(rl); + } + + if (flags & IOMAP_REPORT) { + if (lcn < LCN_HOLE) { + up_write(&ni->runlist.lock); + return -ENOENT; + } + } else if (lcn < LCN_ENOENT) { + up_write(&ni->runlist.lock); + return -EINVAL; + } + + iomap->bdev = inode->i_sb->s_bdev; + iomap->offset = offset; + + if (lcn <= LCN_DELALLOC) { + if (lcn == LCN_DELALLOC) + iomap->type = IOMAP_DELALLOC; + else + iomap->type = IOMAP_HOLE; + iomap->addr = IOMAP_NULL_ADDR; + } else { + if (need_unwritten && offset >= ni->initialized_size) + iomap->type = IOMAP_UNWRITTEN; + else + iomap->type = IOMAP_MAPPED; + iomap->addr = ntfs_cluster_to_bytes(vol, lcn) + vcn_ofs; + } + + rl_length = ntfs_cluster_to_bytes(vol, rl->length - (vcn - rl->vcn)); + + if (rl_length == 0 && rl->lcn > LCN_DELALLOC) { + ntfs_error(inode->i_sb, + "runlist(vcn : %lld, length : %lld, lcn : %lld) is corrupted\n", + rl->vcn, rl->length, rl->lcn); + up_write(&ni->runlist.lock); + return -EIO; + } + + if (rl_length && length > rl_length - vcn_ofs) + iomap->length = rl_length - vcn_ofs; + else + iomap->length = length; + up_write(&ni->runlist.lock); + + if (!(flags & IOMAP_ZERO) && + iomap->type == IOMAP_MAPPED && + iomap->offset < ni->initialized_size && + iomap->offset + iomap->length > ni->initialized_size) { + iomap->length = round_up(ni->initialized_size, 1 << inode->i_blkbits) - + iomap->offset; + } + iomap->flags |= IOMAP_F_MERGED; + + return 0; +} + +static int __ntfs_read_iomap_begin(struct inode *inode, loff_t offset, loff_t length, + unsigned int flags, struct iomap *iomap, struct iomap *srcmap, + bool need_unwritten) +{ + if (NInoNonResident(NTFS_I(inode))) + return ntfs_read_iomap_begin_non_resident(inode, offset, length, + flags, iomap, need_unwritten); + return ntfs_read_iomap_begin_resident(inode, offset, length, + flags, iomap); +} + +static int ntfs_read_iomap_begin(struct inode *inode, loff_t offset, loff_t length, + unsigned int flags, struct iomap *iomap, struct iomap *srcmap) +{ + return __ntfs_read_iomap_begin(inode, offset, length, flags, iomap, + srcmap, true); +} + +static DEFINE_IOMAP_ITER_NEXT(ntfs_read_iomap_next, ntfs_read_iomap_begin); + +const struct iomap_ops ntfs_read_iomap_ops = { + .iomap_next = ntfs_read_iomap_next, +}; + +/* + * Check that the cached iomap still matches the NTFS runlist before + * iomap_zero_range() is called. if the runlist changes while iomap is + * iterating a cached iomap, iomap_zero_range() may overwrite folios + * that have been already written with valid data. + */ +static bool ntfs_iomap_valid(struct inode *inode, const struct iomap *iomap) +{ + struct ntfs_inode *ni = NTFS_I(inode); + struct runlist_element *rl; + s64 vcn, lcn; + + if (!NInoNonResident(ni)) + return false; + + vcn = iomap->offset >> ni->vol->cluster_size_bits; + + down_read(&ni->runlist.lock); + rl = __ntfs_attr_find_vcn_nolock(&ni->runlist, vcn); + if (IS_ERR(rl)) { + up_read(&ni->runlist.lock); + return false; + } + lcn = ntfs_rl_vcn_to_lcn(rl, vcn); + up_read(&ni->runlist.lock); + return lcn == LCN_DELALLOC; +} + +static const struct iomap_write_ops ntfs_zero_iomap_folio_ops = { + .put_folio = ntfs_iomap_put_folio, + .iomap_valid = ntfs_iomap_valid, +}; + +static int ntfs_seek_iomap_begin(struct inode *inode, loff_t offset, loff_t length, + unsigned int flags, struct iomap *iomap, struct iomap *srcmap) +{ + return __ntfs_read_iomap_begin(inode, offset, length, flags, iomap, + srcmap, false); +} + +static int ntfs_zero_read_iomap_end(struct inode *inode, loff_t pos, loff_t length, + ssize_t written, unsigned int flags, struct iomap *iomap) +{ + if ((flags & IOMAP_ZERO) && (iomap->flags & IOMAP_F_STALE)) + return -EPERM; + return written; +} + +static DEFINE_IOMAP_ITER_NEXT_END(ntfs_zero_read_iomap_next, + ntfs_seek_iomap_begin, ntfs_zero_read_iomap_end); + +static const struct iomap_ops ntfs_zero_read_iomap_ops = { + .iomap_next = ntfs_zero_read_iomap_next, +}; + +static DEFINE_IOMAP_ITER_NEXT(ntfs_seek_iomap_next, ntfs_seek_iomap_begin); + +const struct iomap_ops ntfs_seek_iomap_ops = { + .iomap_next = ntfs_seek_iomap_next, +}; + +int ntfs_dio_zero_range(struct inode *inode, loff_t offset, loff_t length) +{ + if ((offset | length) & (SECTOR_SIZE - 1)) + return -EINVAL; + + return blkdev_issue_zeroout(inode->i_sb->s_bdev, + offset >> SECTOR_SHIFT, + length >> SECTOR_SHIFT, + GFP_NOFS, + BLKDEV_ZERO_NOUNMAP); +} + +static int ntfs_zero_range(struct inode *inode, loff_t offset, loff_t length) +{ + return iomap_zero_range(inode, + offset, length, + NULL, + &ntfs_zero_read_iomap_ops, + &ntfs_zero_iomap_folio_ops, + NULL); +} + +static int ntfs_write_simple_iomap_begin_non_resident(struct inode *inode, loff_t offset, + loff_t length, struct iomap *iomap) +{ + struct ntfs_inode *ni = NTFS_I(inode); + struct ntfs_volume *vol = ni->vol; + loff_t vcn_ofs, rl_length; + struct runlist_element *rl, *rlc; + bool is_retry = false; + int err = 0; + s64 vcn, lcn; + s64 max_clu_count = + ntfs_bytes_to_cluster(vol, round_up(length, vol->cluster_size)); + + vcn = ntfs_bytes_to_cluster(vol, offset); + vcn_ofs = ntfs_bytes_to_cluster_off(vol, offset); + + down_read(&ni->runlist.lock); + rl = ni->runlist.rl; + if (!rl) { + up_read(&ni->runlist.lock); + err = ntfs_map_runlist(ni, vcn); + if (err) { + mutex_unlock(&ni->mrec_lock); + return -ENOENT; + } + down_read(&ni->runlist.lock); + rl = ni->runlist.rl; + } + up_read(&ni->runlist.lock); + + down_write(&ni->runlist.lock); +remap_rl: + /* Seek to element containing target vcn. */ + rl = __ntfs_attr_find_vcn_nolock(&ni->runlist, vcn); + if (IS_ERR(rl)) { + up_write(&ni->runlist.lock); + mutex_unlock(&ni->mrec_lock); + return -EIO; + } + lcn = ntfs_rl_vcn_to_lcn(rl, vcn); + + if (lcn <= LCN_RL_NOT_MAPPED && is_retry == false) { + is_retry = true; + if (!ntfs_map_runlist_nolock(ni, vcn, NULL)) { + rl = ni->runlist.rl; + goto remap_rl; + } + } + + max_clu_count = min(max_clu_count, rl->length - (vcn - rl->vcn)); + if (max_clu_count == 0) { + ntfs_error(inode->i_sb, + "runlist(vcn : %lld, length : %lld) is corrupted\n", + rl->vcn, rl->length); + up_write(&ni->runlist.lock); + mutex_unlock(&ni->mrec_lock); + return -EIO; + } + + iomap->bdev = inode->i_sb->s_bdev; + iomap->offset = offset; + + if (lcn <= LCN_DELALLOC) { + if (lcn < LCN_DELALLOC) { + max_clu_count = + ntfs_available_clusters_count(vol, max_clu_count); + if (max_clu_count < 0) { + err = max_clu_count; + up_write(&ni->runlist.lock); + mutex_unlock(&ni->mrec_lock); + return err; + } + } + + iomap->type = IOMAP_DELALLOC; + iomap->addr = IOMAP_NULL_ADDR; + + if (lcn <= LCN_HOLE) { + size_t new_rl_count; + + rlc = kmalloc(sizeof(struct runlist_element) * 2, + GFP_NOFS); + if (!rlc) { + up_write(&ni->runlist.lock); + mutex_unlock(&ni->mrec_lock); + return -ENOMEM; + } + + rlc->vcn = vcn; + rlc->lcn = LCN_DELALLOC; + rlc->length = max_clu_count; + + rlc[1].vcn = vcn + max_clu_count; + rlc[1].lcn = LCN_RL_NOT_MAPPED; + rlc[1].length = 0; + + rl = ntfs_runlists_merge(&ni->runlist, rlc, 0, + &new_rl_count); + if (IS_ERR(rl)) { + ntfs_error(vol->sb, "Failed to merge runlists"); + up_write(&ni->runlist.lock); + mutex_unlock(&ni->mrec_lock); + kvfree(rlc); + return PTR_ERR(rl); + } + + ni->runlist.rl = rl; + ni->runlist.count = new_rl_count; + ni->i_dealloc_clusters += max_clu_count; + } + up_write(&ni->runlist.lock); + mutex_unlock(&ni->mrec_lock); + + if (lcn < LCN_DELALLOC) + ntfs_hold_dirty_clusters(vol, max_clu_count); + + rl_length = ntfs_cluster_to_bytes(vol, max_clu_count); + if (length > rl_length - vcn_ofs) + iomap->length = rl_length - vcn_ofs; + else + iomap->length = length; + + iomap->flags = IOMAP_F_NEW; + if (lcn <= LCN_HOLE) { + loff_t end = offset + length; + + if (vcn_ofs || ((vol->cluster_size > iomap->length) && + end < ni->initialized_size)) { + loff_t z_start, z_end; + + z_start = vcn << vol->cluster_size_bits; + z_end = min_t(loff_t, z_start + vol->cluster_size, + i_size_read(inode)); + if (z_end > z_start) + err = ntfs_zero_range(inode, + z_start, + z_end - z_start); + } + if ((!err || err == -EPERM) && + max_clu_count > 1 && + (iomap->length & vol->cluster_size_mask && + end < ni->initialized_size)) { + loff_t z_start, z_end; + + z_start = (vcn + max_clu_count - 1) << + vol->cluster_size_bits; + z_end = min_t(loff_t, z_start + vol->cluster_size, + i_size_read(inode)); + if (z_end > z_start) + err = ntfs_zero_range(inode, + z_start, + z_end - z_start); + } + + if (err == -EPERM) + err = 0; + if (err) { + ntfs_release_dirty_clusters(vol, max_clu_count); + return err; + } + } + } else { + up_write(&ni->runlist.lock); + mutex_unlock(&ni->mrec_lock); + + iomap->type = IOMAP_MAPPED; + iomap->addr = ntfs_cluster_to_bytes(vol, lcn) + vcn_ofs; + + rl_length = ntfs_cluster_to_bytes(vol, max_clu_count); + if (length > rl_length - vcn_ofs) + iomap->length = rl_length - vcn_ofs; + else + iomap->length = length; + } + + return 0; +} + +#define NTFS_IOMAP_FLAGS_BEGIN BIT(1) +#define NTFS_IOMAP_FLAGS_DIO BIT(2) +#define NTFS_IOMAP_FLAGS_MKWRITE BIT(3) +#define NTFS_IOMAP_FLAGS_WRITEBACK BIT(4) + +static int ntfs_write_da_iomap_begin_non_resident(struct inode *inode, + loff_t offset, loff_t length, unsigned int flags, + struct iomap *iomap, int ntfs_iomap_flags) +{ + struct ntfs_inode *ni = NTFS_I(inode); + struct ntfs_volume *vol = ni->vol; + loff_t vcn_ofs, rl_length; + s64 vcn, start_lcn, lcn_count; + bool balloc = false, update_mp; + int err; + s64 max_clu_count = + ntfs_bytes_to_cluster(vol, round_up(length, vol->cluster_size)); + + vcn = ntfs_bytes_to_cluster(vol, offset); + vcn_ofs = ntfs_bytes_to_cluster_off(vol, offset); + + update_mp = ntfs_iomap_flags & (NTFS_IOMAP_FLAGS_DIO | NTFS_IOMAP_FLAGS_MKWRITE) || + NInoAttr(ni) || ni->mft_no < FILE_first_user; + down_write(&ni->runlist.lock); + err = ntfs_attr_map_cluster(ni, vcn, &start_lcn, &lcn_count, + max_clu_count, &balloc, update_mp, + ntfs_iomap_flags & NTFS_IOMAP_FLAGS_WRITEBACK); + up_write(&ni->runlist.lock); + mutex_unlock(&ni->mrec_lock); + if (err) { + ni->i_dealloc_clusters = 0; + return err; + } + + iomap->bdev = inode->i_sb->s_bdev; + iomap->offset = offset; + + rl_length = ntfs_cluster_to_bytes(vol, lcn_count); + if (length > rl_length - vcn_ofs) + iomap->length = rl_length - vcn_ofs; + else + iomap->length = length; + + if (start_lcn == LCN_HOLE) + iomap->type = IOMAP_HOLE; + else + iomap->type = IOMAP_MAPPED; + if (balloc == true) + iomap->flags = IOMAP_F_NEW; + + iomap->addr = ntfs_cluster_to_bytes(vol, start_lcn) + vcn_ofs; + + if (balloc == true) { + if (flags & IOMAP_DIRECT || + ntfs_iomap_flags & NTFS_IOMAP_FLAGS_MKWRITE) { + loff_t end = offset + length; + + if (vcn_ofs || ((vol->cluster_size > iomap->length) && + end < ni->initialized_size)) + err = ntfs_dio_zero_range(inode, + start_lcn << + vol->cluster_size_bits, + vol->cluster_size); + if (!err && lcn_count > 1 && + (iomap->length & vol->cluster_size_mask && + end < ni->initialized_size)) + err = ntfs_dio_zero_range(inode, + (start_lcn + lcn_count - 1) << + vol->cluster_size_bits, + vol->cluster_size); + } else { + if (lcn_count > ni->i_dealloc_clusters) + ni->i_dealloc_clusters = 0; + else + ni->i_dealloc_clusters -= lcn_count; + } + if (err < 0) + return err; + } + + if (ntfs_iomap_flags & NTFS_IOMAP_FLAGS_MKWRITE && + iomap->offset + iomap->length > ni->initialized_size) { + err = ntfs_attr_set_initialized_size(ni, iomap->offset + + iomap->length); + } + + return err; +} + +static int ntfs_write_iomap_begin_resident(struct inode *inode, loff_t offset, + struct iomap *iomap) +{ + struct ntfs_inode *ni = NTFS_I(inode); + struct attr_record *a; + struct ntfs_attr_search_ctx *ctx; + u32 attr_len; + int err = 0; + char *kattr; + + ctx = ntfs_attr_get_search_ctx(ni, NULL); + if (!ctx) { + err = -ENOMEM; + goto out; + } + + err = ntfs_attr_lookup(ni->type, ni->name, ni->name_len, + CASE_SENSITIVE, 0, NULL, 0, ctx); + if (err) { + if (err == -ENOENT) + err = -EIO; + goto out; + } + + a = ctx->attr; + /* The total length of the attribute value. */ + attr_len = le32_to_cpu(a->data.resident.value_length); + kattr = (u8 *)a + le16_to_cpu(a->data.resident.value_offset); + + iomap->type = IOMAP_INLINE; + iomap->inline_data = kattr; + iomap->offset = 0; + iomap->length = attr_len; + +out: + if (ctx) + ntfs_attr_put_search_ctx(ctx); + + if (err) + mutex_unlock(&ni->mrec_lock); + + return err; +} + +static int ntfs_write_iomap_begin_non_resident(struct inode *inode, loff_t offset, + loff_t length, unsigned int flags, + struct iomap *iomap, int ntfs_iomap_flags) +{ + struct ntfs_inode *ni = NTFS_I(inode); + + if (ntfs_iomap_flags & (NTFS_IOMAP_FLAGS_BEGIN | NTFS_IOMAP_FLAGS_DIO) && + offset + length > ni->initialized_size) { + int ret; + + ret = ntfs_extend_initialized_size(inode, offset, + offset + length, + ntfs_iomap_flags & + NTFS_IOMAP_FLAGS_DIO); + if (ret < 0) + return ret; + } + + mutex_lock(&ni->mrec_lock); + if (ntfs_iomap_flags & NTFS_IOMAP_FLAGS_BEGIN) + return ntfs_write_simple_iomap_begin_non_resident(inode, offset, + length, iomap); + else + return ntfs_write_da_iomap_begin_non_resident(inode, + offset, length, + flags, iomap, + ntfs_iomap_flags); +} + +static int __ntfs_write_iomap_begin(struct inode *inode, loff_t offset, + loff_t length, unsigned int flags, + struct iomap *iomap, int ntfs_iomap_flags) +{ + struct ntfs_inode *ni = NTFS_I(inode); + loff_t end = offset + length; + + if (NVolShutdown(ni->vol)) + return -EIO; + + if (ntfs_iomap_flags & (NTFS_IOMAP_FLAGS_BEGIN | NTFS_IOMAP_FLAGS_DIO) && + end > ni->data_size) { + struct ntfs_volume *vol = ni->vol; + int ret; + + mutex_lock(&ni->mrec_lock); + if (end > ni->allocated_size && + end < ni->allocated_size + vol->preallocated_size) + ret = ntfs_attr_expand(ni, end, + ni->allocated_size + vol->preallocated_size); + else + ret = ntfs_attr_expand(ni, end, 0); + mutex_unlock(&ni->mrec_lock); + if (ret) + return ret; + } + + if (!NInoNonResident(ni)) { + mutex_lock(&ni->mrec_lock); + return ntfs_write_iomap_begin_resident(inode, offset, iomap); + } + return ntfs_write_iomap_begin_non_resident(inode, offset, length, flags, + iomap, ntfs_iomap_flags); +} + +static int ntfs_write_iomap_begin(struct inode *inode, loff_t offset, + loff_t length, unsigned int flags, + struct iomap *iomap, struct iomap *srcmap) +{ + return __ntfs_write_iomap_begin(inode, offset, length, flags, iomap, + NTFS_IOMAP_FLAGS_BEGIN); +} + +static int ntfs_write_iomap_end_resident(struct inode *inode, loff_t pos, + loff_t length, ssize_t written, + unsigned int flags, struct iomap *iomap) +{ + struct ntfs_inode *ni = NTFS_I(inode); + + mark_mft_record_dirty(ni); + mutex_unlock(&ni->mrec_lock); + return written; +} + +static int ntfs_write_iomap_end(struct inode *inode, loff_t pos, loff_t length, + ssize_t written, unsigned int flags, + struct iomap *iomap) +{ + if (iomap->type == IOMAP_INLINE) + return ntfs_write_iomap_end_resident(inode, pos, length, + written, flags, iomap); + return written; +} + +static DEFINE_IOMAP_ITER_NEXT_END(ntfs_write_iomap_next, + ntfs_write_iomap_begin, ntfs_write_iomap_end); + +const struct iomap_ops ntfs_write_iomap_ops = { + .iomap_next = ntfs_write_iomap_next, +}; + +static int ntfs_page_mkwrite_iomap_begin(struct inode *inode, loff_t offset, + loff_t length, unsigned int flags, + struct iomap *iomap, struct iomap *srcmap) +{ + return __ntfs_write_iomap_begin(inode, offset, length, flags, iomap, + NTFS_IOMAP_FLAGS_MKWRITE); +} + +static DEFINE_IOMAP_ITER_NEXT_END(ntfs_page_mkwrite_iomap_next, + ntfs_page_mkwrite_iomap_begin, ntfs_write_iomap_end); + +const struct iomap_ops ntfs_page_mkwrite_iomap_ops = { + .iomap_next = ntfs_page_mkwrite_iomap_next, +}; + +static int ntfs_dio_iomap_begin(struct inode *inode, loff_t offset, + loff_t length, unsigned int flags, + struct iomap *iomap, struct iomap *srcmap) +{ + return __ntfs_write_iomap_begin(inode, offset, length, flags, iomap, + NTFS_IOMAP_FLAGS_DIO); +} + +static DEFINE_IOMAP_ITER_NEXT_END(ntfs_dio_iomap_next, + ntfs_dio_iomap_begin, ntfs_write_iomap_end); + +const struct iomap_ops ntfs_dio_iomap_ops = { + .iomap_next = ntfs_dio_iomap_next, +}; + +static ssize_t ntfs_writeback_range(struct iomap_writepage_ctx *wpc, + struct folio *folio, u64 offset, unsigned int len, u64 end_pos) +{ + if (offset < wpc->iomap.offset || + offset >= wpc->iomap.offset + wpc->iomap.length) { + int error; + + error = __ntfs_write_iomap_begin(wpc->inode, offset, + NTFS_I(wpc->inode)->allocated_size - offset, + IOMAP_WRITE, &wpc->iomap, + NTFS_IOMAP_FLAGS_WRITEBACK); + if (error) + return error; + } + + return iomap_add_to_ioend(wpc, folio, offset, end_pos, len); +} + +const struct iomap_writeback_ops ntfs_writeback_ops = { + .writeback_range = ntfs_writeback_range, + .writeback_submit = iomap_ioend_writeback_submit, +}; diff --git a/fs/ntfs/iomap.h b/fs/ntfs/iomap.h new file mode 100644 index 000000000..3abc1d493 --- /dev/null +++ b/fs/ntfs/iomap.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2025 LG Electronics Co., Ltd. + */ + +#ifndef _LINUX_NTFS_IOMAP_H +#define _LINUX_NTFS_IOMAP_H + +#include +#include + +#include "volume.h" +#include "inode.h" + +extern const struct iomap_ops ntfs_write_iomap_ops; +extern const struct iomap_ops ntfs_read_iomap_ops; +extern const struct iomap_ops ntfs_seek_iomap_ops; +extern const struct iomap_ops ntfs_page_mkwrite_iomap_ops; +extern const struct iomap_ops ntfs_dio_iomap_ops; +extern const struct iomap_writeback_ops ntfs_writeback_ops; +extern const struct iomap_write_ops ntfs_iomap_folio_ops; +extern int ntfs_dio_zero_range(struct inode *inode, loff_t offset, loff_t length); +#endif /* _LINUX_NTFS_IOMAP_H */ diff --git a/fs/ntfs/layout.h b/fs/ntfs/layout.h new file mode 100644 index 000000000..9438fd9b6 --- /dev/null +++ b/fs/ntfs/layout.h @@ -0,0 +1,2365 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * All NTFS associated on-disk structures. + * + * Copyright (c) 2001-2005 Anton Altaparmakov + * Copyright (c) 2002 Richard Russon + */ + +#ifndef _LINUX_NTFS_LAYOUT_H +#define _LINUX_NTFS_LAYOUT_H + +#include +#include +#include +#include + +/* The NTFS oem_id "NTFS " */ +#define magicNTFS cpu_to_le64(0x202020205346544eULL) + +/* + * Location of bootsector on partition: + * The standard NTFS_BOOT_SECTOR is on sector 0 of the partition. + * On NT4 and above there is one backup copy of the boot sector to + * be found on the last sector of the partition (not normally accessible + * from within Windows as the bootsector contained number of sectors + * value is one less than the actual value!). + * On versions of NT 3.51 and earlier, the backup copy was located at + * number of sectors/2 (integer divide), i.e. in the middle of the volume. + */ + +/* + * BIOS parameter block (bpb) structure. + * + * @bytes_per_sector: Size of a sector in bytes (usually 512). + * Matches the logical sector size of the underlying device. + * @sectors_per_cluster: Size of a cluster in sectors (NTFS cluster size / sector size). + * @reserved_sectors: Number of reserved sectors at the beginning of the volume. + * Always set to 0 in NTFS. + * @fats: Number of FAT tables. + * Always 0 in NTFS (no FAT tables exist). + * @root_entries: Number of entries in the root directory. + * Always 0 in NTFS. + * @sectors: Total number of sectors on the volume. + * Always 0 in NTFS (use @large_sectors instead). + * @media_type: Media descriptor byte. + * 0xF8 for hard disk (fixed media) in NTFS. + * @sectors_per_fat: Number of sectors per FAT table. + * Always 0 in NTFS. + * @sectors_per_track: Number of sectors per track. + * Irrelevant for NTFS. + * @heads: Number of heads (CHS geometry). + * Irrelevant for NTFS. + * @hidden_sectors: Number of hidden sectors before the start of the partition. + * Always 0 in NTFS boot sector. + * @large_sectors: Total number of sectors on the volume. + */ +struct bios_parameter_block { + __le16 bytes_per_sector; + u8 sectors_per_cluster; + __le16 reserved_sectors; + u8 fats; + __le16 root_entries; + __le16 sectors; + u8 media_type; + __le16 sectors_per_fat; + __le16 sectors_per_track; + __le16 heads; + __le32 hidden_sectors; + __le32 large_sectors; +} __packed; + +/* + * NTFS boot sector structure. + * + * @jump: 3-byte jump instruction to boot code (irrelevant for NTFS). + * Typically 0xEB 0x52 0x90 or similar. + * @oem_id: OEM identifier string (8 bytes). + * Always "NTFS " (with trailing spaces) in NTFS volumes. + * @bpb: Legacy BIOS Parameter Block (see struct bios_parameter_block). + * Mostly zeroed or set to fixed values for NTFS compatibility. + * @unused: 4 bytes, reserved/unused. + * NTFS disk editors show it as: + * - physical_drive (0x80 for fixed disk) + * - current_head (0) + * - extended_boot_signature (0x80 or 0x28) + * - unused (0) + * Always zero in practice for NTFS. + * @number_of_sectors: Number of sectors in volume. Gives maximum volume + * size of 2^63 sectors. Assuming standard sector + * size of 512 bytes, the maximum byte size is + * approx. 4.7x10^21 bytes. (-; + * @mft_lcn: Logical cluster number (LCN) of the $MFT data attribute. + * Location of the Master File Table. + * @mftmirr_lcn: LCN of the $MFTMirr (first 3-4 MFT records copy). + * Mirror for boot-time recovery. + * @clusters_per_mft_record: + * Size of each MFT record in clusters. + * @reserved0: 3 bytes, reserved/zero. + * @clusters_per_index_record: + * Size of each index block/record in clusters. + * @reserved1: 3 bytes, reserved/zero. + * @volume_serial_number: + * 64-bit volume serial number. + * Used for identification (irrelevant for NTFS operation). + * @checksum: 32-bit checksum of the boot sector (excluding this field). + * Used to detect boot sector corruption. + * @bootstrap: 426 bytes of bootstrap code. + * Irrelevant for NTFS (contains x86 boot loader stub). + * @end_of_sector_marker: + * 2-byte end-of-sector signature. + * Always 0xAA55 (little-endian magic number). + */ +struct ntfs_boot_sector { + u8 jump[3]; + __le64 oem_id; + struct bios_parameter_block bpb; + u8 unused[4]; + __le64 number_of_sectors; + __le64 mft_lcn; + __le64 mftmirr_lcn; + s8 clusters_per_mft_record; + u8 reserved0[3]; + s8 clusters_per_index_record; + u8 reserved1[3]; + __le64 volume_serial_number; + __le32 checksum; + u8 bootstrap[426]; + __le16 end_of_sector_marker; +} __packed; + +static_assert(sizeof(struct ntfs_boot_sector) == 512); + +/* + * Magic identifiers present at the beginning of all ntfs record containing + * records (like mft records for example). + * + * magic_FILE: MFT entry header ("FILE" in ASCII). + * Used in $MFT/$DATA for all master file table records. + * magic_INDX: Index buffer header ("INDX" in ASCII). + * Used in $INDEX_ALLOCATION attributes (directories, $I30 indexes). + * magic_HOLE: Hole marker ("HOLE" in ASCII). + * Introduced in NTFS 3.0+, used for sparse/hole regions in some contexts. + * magic_RSTR: Restart page header ("RSTR" in ASCII). + * Used in LogFile for restart pages (transaction log recovery). + * magic_RCRD: Log record page header ("RCRD" in ASCII). + * Used in LogFile for individual log record pages. + * magic_CHKD: Chkdsk modified marker ("CHKD" in ASCII). + * Set by chkdsk when it modifies a record; indicates repair was done. + * magic_BAAD: Bad record marker ("BAAD" in ASCII). + * Indicates a multi-sector transfer failure was detected. + * The record is corrupted/unusable; often set during I/O errors. + * magic_empty: Empty/uninitialized page marker (0xffffffff). + * Used in LogFile when a page is filled with 0xff bytes + * and has not yet been initialized. Must be formatted before use. + */ +enum { + magic_FILE = cpu_to_le32(0x454c4946), + magic_INDX = cpu_to_le32(0x58444e49), + magic_HOLE = cpu_to_le32(0x454c4f48), + magic_RSTR = cpu_to_le32(0x52545352), + magic_RCRD = cpu_to_le32(0x44524352), + magic_CHKD = cpu_to_le32(0x444b4843), + magic_BAAD = cpu_to_le32(0x44414142), + magic_empty = cpu_to_le32(0xffffffff) +}; + +/* + * Generic magic comparison macros. Finally found a use for the ## preprocessor + * operator! (-8 + */ + +static inline bool __ntfs_is_magic(__le32 x, __le32 r) +{ + return (x == r); +} +#define ntfs_is_magic(x, m) __ntfs_is_magic(x, magic_##m) + +static inline bool __ntfs_is_magicp(__le32 *p, __le32 r) +{ + return (*p == r); +} +#define ntfs_is_magicp(p, m) __ntfs_is_magicp(p, magic_##m) + +/* + * Specialised magic comparison macros for the NTFS_RECORD_TYPEs defined above. + */ +#define ntfs_is_file_record(x) (ntfs_is_magic(x, FILE)) +#define ntfs_is_file_recordp(p) (ntfs_is_magicp(p, FILE)) +#define ntfs_is_mft_record(x) (ntfs_is_file_record(x)) +#define ntfs_is_mft_recordp(p) (ntfs_is_file_recordp(p)) +#define ntfs_is_indx_record(x) (ntfs_is_magic(x, INDX)) +#define ntfs_is_indx_recordp(p) (ntfs_is_magicp(p, INDX)) +#define ntfs_is_hole_record(x) (ntfs_is_magic(x, HOLE)) +#define ntfs_is_hole_recordp(p) (ntfs_is_magicp(p, HOLE)) + +#define ntfs_is_rstr_record(x) (ntfs_is_magic(x, RSTR)) +#define ntfs_is_rstr_recordp(p) (ntfs_is_magicp(p, RSTR)) +#define ntfs_is_rcrd_record(x) (ntfs_is_magic(x, RCRD)) +#define ntfs_is_rcrd_recordp(p) (ntfs_is_magicp(p, RCRD)) + +#define ntfs_is_chkd_record(x) (ntfs_is_magic(x, CHKD)) +#define ntfs_is_chkd_recordp(p) (ntfs_is_magicp(p, CHKD)) + +#define ntfs_is_baad_record(x) (ntfs_is_magic(x, BAAD)) +#define ntfs_is_baad_recordp(p) (ntfs_is_magicp(p, BAAD)) + +#define ntfs_is_empty_record(x) (ntfs_is_magic(x, empty)) +#define ntfs_is_empty_recordp(p) (ntfs_is_magicp(p, empty)) + +/* + * struct ntfs_record - Common header for all multi-sector protected NTFS records + * + * @magic: 4-byte magic identifier for the record type and/or status. + * Common values are defined in the magic_* enum (FILE, INDX, RSTR, + * RCRD, CHKD, BAAD, HOLE, empty). + * - "FILE" = MFT record + * - "INDX" = Index allocation block + * - "BAAD" = Record corrupted (multi-sector fixup failed) + * - 0xffffffff = Uninitialized/empty page + * @usa_ofs: Offset (in bytes) from the start of this record to the Update + * Sequence Array (USA). + * The USA is located at record + usa_ofs. + * @usa_count: Number of 16-bit entries in the USA array (including the Update + * Sequence Number itself). + * - Number of fixup locations = usa_count - 1 + * - Each fixup location is a 16-bit value in the record that needs + * protection against torn writes. + * + * The Update Sequence Array (usa) is an array of the __le16 values which belong + * to the end of each sector protected by the update sequence record in which + * this array is contained. Note that the first entry is the Update Sequence + * Number (usn), a cyclic counter of how many times the protected record has + * been written to disk. The values 0 and -1 (ie. 0xffff) are not used. All + * last le16's of each sector have to be equal to the usn (during reading) or + * are set to it (during writing). If they are not, an incomplete multi sector + * transfer has occurred when the data was written. + * The maximum size for the update sequence array is fixed to: + * maximum size = usa_ofs + (usa_count * 2) = 510 bytes + * The 510 bytes comes from the fact that the last __le16 in the array has to + * (obviously) finish before the last __le16 of the first 512-byte sector. + * This formula can be used as a consistency check in that usa_ofs + + * (usa_count * 2) has to be less than or equal to 510. + */ +struct ntfs_record { + __le32 magic; + __le16 usa_ofs; + __le16 usa_count; +} __packed; + +/* + * System files mft record numbers. All these files are always marked as used + * in the bitmap attribute of the mft; presumably in order to avoid accidental + * allocation for random other mft records. Also, the sequence number for each + * of the system files is always equal to their mft record number and it is + * never modified. + * + * FILE_MFT: Master File Table (MFT) itself. + * Data attribute contains all MFT entries; + * Bitmap attribute tracks which records are in use (bit==1). + * FILE_MFTMirr: MFT mirror: copy of the first four (or more) MFT records + * in its data attribute. + * If cluster size > 4 KiB, copies first N records where + * N = cluster_size / mft_record_size. + * FILE_LogFile: Journaling log (LogFile) in data attribute. + * Used for transaction logging and recovery. + * FILE_Volume: Volume information and name. + * Contains $VolumeName (label) and $VolumeInformation + * (flags, NTFS version). Windows calls this the volume DASD. + * FILE_AttrDef: Attribute definitions array in data attribute. + * Defines all possible attribute types and their properties. + * FILE_root: Root directory ($Root). + * The top-level directory of the filesystem. + * FILE_Bitmap: Cluster allocation bitmap ($Bitmap) in data attribute. + * Tracks free/used clusters (LCNs) on the volume. + * FILE_Boot: Boot sector ($Boot) in data attribute. + * Always located at cluster 0; contains BPB and NTFS parameters. + * FILE_BadClus: Bad cluster list ($BadClus) in non-resident data attribute. + * Marks all known bad clusters. + * FILE_Secure: Security descriptors ($Secure). + * Contains shared $SDS (security descriptors) and two indexes + * ($SDH, $SII). Introduced in Windows 2000. + * Before that, it was called $Quota but was unused. + * FILE_UpCase: Uppercase table ($UpCase) in data attribute. + * Maps all 65536 Unicode characters to their uppercase forms. + * FILE_Extend: System directory ($Extend). + * Contains additional system files ($ObjId, $Quota, $Reparse, + * $UsnJrnl, etc.). Introduced in NTFS 3.0 (Windows 2000). + * FILE_reserved12: Reserved for future use (MFT records 12–15). + * FILE_reserved13: Reserved. + * FILE_reserved14: Reserved. + * FILE_reserved15: Reserved. + * FILE_first_user: First possible user-created file MFT record number. + * Used as a boundary to distinguish system files from user files. + */ +enum { + FILE_MFT = 0, + FILE_MFTMirr = 1, + FILE_LogFile = 2, + FILE_Volume = 3, + FILE_AttrDef = 4, + FILE_root = 5, + FILE_Bitmap = 6, + FILE_Boot = 7, + FILE_BadClus = 8, + FILE_Secure = 9, + FILE_UpCase = 10, + FILE_Extend = 11, + FILE_reserved12 = 12, + FILE_reserved13 = 13, + FILE_reserved14 = 14, + FILE_reserved15 = 15, + FILE_first_user = 16, +}; + +/* + * enum - Flags for MFT record header + * + * These are the so far known MFT_RECORD_* flags (16-bit) which contain + * information about the mft record in which they are present. + * + * MFT_RECORD_IN_USE: This MFT record is allocated and in use. + * (bit set = record is valid/used; clear = free) + * MFT_RECORD_IS_DIRECTORY: This MFT record represents a directory. + * (Used to quickly distinguish files from directories) + * MFT_RECORD_IS_4: Indicates the record is a special "record 4" type. + * (Rarely used; related to NTFS internal special cases, + * often for $AttrDef or early system files) + * MFT_RECORD_IS_VIEW_INDEX: This MFT record is used as a view index. + * (Specific to NTFS indexed views or object ID indexes) + * MFT_REC_SPACE_FILLER: Dummy value to force the enum to be 16-bit wide. + * (Not a real flag; just a sentinel to ensure the type + * is __le16 and no higher bits are accidentally used) + */ +enum { + MFT_RECORD_IN_USE = cpu_to_le16(0x0001), + MFT_RECORD_IS_DIRECTORY = cpu_to_le16(0x0002), + MFT_RECORD_IS_4 = cpu_to_le16(0x0004), + MFT_RECORD_IS_VIEW_INDEX = cpu_to_le16(0x0008), + MFT_REC_SPACE_FILLER = cpu_to_le16(0xffff), /*Just to make flags 16-bit.*/ +} __packed; + +/* + * mft references (aka file references or file record segment references) are + * used whenever a structure needs to refer to a record in the mft. + * + * A reference consists of a 48-bit index into the mft and a 16-bit sequence + * number used to detect stale references. + * + * For error reporting purposes we treat the 48-bit index as a signed quantity. + * + * The sequence number is a circular counter (skipping 0) describing how many + * times the referenced mft record has been (re)used. This has to match the + * sequence number of the mft record being referenced, otherwise the reference + * is considered stale and removed. + * + * If the sequence number is zero it is assumed that no sequence number + * consistency checking should be performed. + */ + +/* + * Define two unpacking macros to get to the reference (MREF) and + * sequence number (MSEQNO) respectively. + * The _LE versions are to be applied on little endian MFT_REFs. + * Note: The _LE versions will return a CPU endian formatted value! + */ +#define MFT_REF_MASK_CPU 0x0000ffffffffffffULL +#define MFT_REF_MASK_LE cpu_to_le64(MFT_REF_MASK_CPU) + +#define MK_MREF(m, s) ((u64)(((u64)(s) << 48) | \ + ((u64)(m) & MFT_REF_MASK_CPU))) +#define MK_LE_MREF(m, s) cpu_to_le64(MK_MREF(m, s)) + +#define MREF(x) ((unsigned long)((x) & MFT_REF_MASK_CPU)) +#define MSEQNO(x) ((u16)(((x) >> 48) & 0xffff)) +#define MREF_LE(x) ((unsigned long)(le64_to_cpu(x) & MFT_REF_MASK_CPU)) +#define MREF_INO(x) ((unsigned long)MREF_LE(x)) +#define MSEQNO_LE(x) ((u16)((le64_to_cpu(x) >> 48) & 0xffff)) + +#define IS_ERR_MREF(x) (((x) & 0x0000800000000000ULL) ? true : false) +#define ERR_MREF(x) ((u64)((s64)(x))) +#define MREF_ERR(x) ((int)((s64)(x))) + +/* + * struct mft_record - NTFS Master File Table (MFT) record header + * + * The mft record header present at the beginning of every record in the mft. + * This is followed by a sequence of variable length attribute records which + * is terminated by an attribute of type AT_END which is a truncated attribute + * in that it only consists of the attribute type code AT_END and none of the + * other members of the attribute structure are present. + * + * magic: Record magic ("FILE" for valid MFT entries). + * See ntfs_record magic enum for other values. + * usa_ofs: Offset to Update Sequence Array (see ntfs_record). + * usa_count: Number of entries in USA (see ntfs_record). + * lsn: Log sequence number (LSN) from LogFile. + * Incremented on every modification to this record. + * sequence_number: Reuse count of this MFT record slot. + * Incremented (skipping zero) when the file is deleted. + * Zero means never reused or special case. + * Part of MFT reference (together with record number). + * link_count: Number of hard links (directory entries) to this file. + * Only meaningful in base MFT records. + * When deleting a directory entry: + * - If link_count == 1, delete the whole file + * - Else remove only the $FILE_NAME attribute and decrement + * attrs_offset: Byte offset from start of MFT record to first attribute. + * Must be 8-byte aligned. + * flags: Bit array of MFT_RECORD_* flags (see MFT_RECORD_IN_USE enum). + * MFT_RECORD_IN_USE cleared when record is freed/deleted. + * bytes_in_use: Number of bytes actually used in this MFT record. + * Must be 8-byte aligned. + * Includes header + all attributes + padding. + * bytes_allocated: Total allocated size of this MFT record. + * Usually equal to MFT record size (1024 bytes or cluster size). + * base_mft_record: MFT reference to the base record. + * 0 for base records. + * Non-zero for extension records → points to base record + * containing the $ATTRIBUTE_LIST that describes this extension. + * next_attr_instance: Next attribute instance number to assign. + * Incremented after each use. + * Reset to 0 when MFT record is reused. + * First instance is always 0. + * reserved: Reserved for alignment (NTFS 3.1+). + * mft_record_number: This MFT record's number (index in $MFT). + * Only present in NTFS 3.1+ (Windows XP and above). + */ +struct mft_record { + __le32 magic; + __le16 usa_ofs; + __le16 usa_count; + + __le64 lsn; + __le16 sequence_number; + __le16 link_count; + __le16 attrs_offset; + __le16 flags; + __le32 bytes_in_use; + __le32 bytes_allocated; + __le64 base_mft_record; + __le16 next_attr_instance; + __le16 reserved; + __le32 mft_record_number; +} __packed; + +static_assert(sizeof(struct mft_record) == 48); + +/**x + * struct mft_record_old - Old NTFS MFT record header (pre-NTFS 3.1 / Windows XP) + * + * This is the older version of the MFT record header used in NTFS versions + * prior to 3.1 (Windows XP and later). It lacks the additional fields + * @reserved and @mft_record_number that were added in NTFS 3.1+. + * + * @magic: Record magic ("FILE" for valid MFT entries). + * See ntfs_record magic enum for other values. + * @usa_ofs: Offset to Update Sequence Array (see ntfs_record). + * @usa_count: Number of entries in USA (see ntfs_record). + * @lsn: Log sequence number (LSN) from LogFile. + * Incremented on every modification to this record. + * @sequence_number: Reuse count of this MFT record slot. + * Incremented (skipping zero) when the file is deleted. + * Zero means never reused or special case. + * Part of MFT reference (together with record number). + * @link_count: Number of hard links (directory entries) to this file. + * Only meaningful in base MFT records. + * When deleting a directory entry: + * - If link_count == 1, delete the whole file + * - Else remove only the $FILE_NAME attribute and decrement + * @attrs_offset: Byte offset from start of MFT record to first attribute. + * Must be 8-byte aligned. + * @flags: Bit array of MFT_RECORD_* flags (see MFT_RECORD_IN_USE enum). + * MFT_RECORD_IN_USE cleared when record is freed/deleted. + * @bytes_in_use: Number of bytes actually used in this MFT record. + * Must be 8-byte aligned. + * Includes header + all attributes + padding. + * @bytes_allocated: Total allocated size of this MFT record. + * Usually equal to MFT record size (1024 bytes or cluster size). + * @base_mft_record: MFT reference to the base record. + * 0 for base records. + * Non-zero for extension records → points to base record + * containing the $ATTRIBUTE_LIST that describes this extension. + * @next_attr_instance: Next attribute instance number to assign. + * Incremented after each use. + * Reset to 0 when MFT record is reused. + * First instance is always 0. + */ +struct mft_record_old { + __le32 magic; + __le16 usa_ofs; + __le16 usa_count; + + __le64 lsn; + __le16 sequence_number; + __le16 link_count; + __le16 attrs_offset; + __le16 flags; + __le32 bytes_in_use; + __le32 bytes_allocated; + __le64 base_mft_record; + __le16 next_attr_instance; +} __packed; + +static_assert(sizeof(struct mft_record_old) == 42); + +/* + * System defined attributes (32-bit). Each attribute type has a corresponding + * attribute name (Unicode string of maximum 64 character length) as described + * by the attribute definitions present in the data attribute of the $AttrDef + * system file. On NTFS 3.0 volumes the names are just as the types are named + * in the below defines exchanging AT_ for the dollar sign ($). If that is not + * a revealing choice of symbol I do not know what is... (-; + */ +enum { + AT_UNUSED = cpu_to_le32(0), + AT_STANDARD_INFORMATION = cpu_to_le32(0x10), + AT_ATTRIBUTE_LIST = cpu_to_le32(0x20), + AT_FILE_NAME = cpu_to_le32(0x30), + AT_OBJECT_ID = cpu_to_le32(0x40), + AT_SECURITY_DESCRIPTOR = cpu_to_le32(0x50), + AT_VOLUME_NAME = cpu_to_le32(0x60), + AT_VOLUME_INFORMATION = cpu_to_le32(0x70), + AT_DATA = cpu_to_le32(0x80), + AT_INDEX_ROOT = cpu_to_le32(0x90), + AT_INDEX_ALLOCATION = cpu_to_le32(0xa0), + AT_BITMAP = cpu_to_le32(0xb0), + AT_REPARSE_POINT = cpu_to_le32(0xc0), + AT_EA_INFORMATION = cpu_to_le32(0xd0), + AT_EA = cpu_to_le32(0xe0), + AT_PROPERTY_SET = cpu_to_le32(0xf0), + AT_LOGGED_UTILITY_STREAM = cpu_to_le32(0x100), + AT_FIRST_USER_DEFINED_ATTRIBUTE = cpu_to_le32(0x1000), + AT_END = cpu_to_le32(0xffffffff) +}; + +/* + * The collation rules for sorting views/indexes/etc (32-bit). + * + * COLLATION_BINARY - Collate by binary compare where the first byte is most + * significant. + * COLLATION_UNICODE_STRING - Collate Unicode strings by comparing their binary + * Unicode values, except that when a character can be uppercased, the + * upper case value collates before the lower case one. + * COLLATION_FILE_NAME - Collate file names as Unicode strings. The collation + * is done very much like COLLATION_UNICODE_STRING. In fact I have no idea + * what the difference is. Perhaps the difference is that file names + * would treat some special characters in an odd way (see + * unistr.c::ntfs_collate_names() and unistr.c::legal_ansi_char_array[] + * for what I mean but COLLATION_UNICODE_STRING would not give any special + * treatment to any characters at all, but this is speculation. + * COLLATION_NTOFS_ULONG - Sorting is done according to ascending __le32 key + * values. E.g. used for $SII index in FILE_Secure, which sorts by + * security_id (le32). + * COLLATION_NTOFS_SID - Sorting is done according to ascending SID values. + * E.g. used for $O index in FILE_Extend/$Quota. + * COLLATION_NTOFS_SECURITY_HASH - Sorting is done first by ascending hash + * values and second by ascending security_id values. E.g. used for $SDH + * index in FILE_Secure. + * COLLATION_NTOFS_ULONGS - Sorting is done according to a sequence of ascending + * __le32 key values. E.g. used for $O index in FILE_Extend/$ObjId, which + * sorts by object_id (16-byte), by splitting up the object_id in four + * __le32 values and using them as individual keys. E.g. take the following + * two security_ids, stored as follows on disk: + * 1st: a1 61 65 b7 65 7b d4 11 9e 3d 00 e0 81 10 42 59 + * 2nd: 38 14 37 d2 d2 f3 d4 11 a5 21 c8 6b 79 b1 97 45 + * To compare them, they are split into four __le32 values each, like so: + * 1st: 0xb76561a1 0x11d47b65 0xe0003d9e 0x59421081 + * 2nd: 0xd2371438 0x11d4f3d2 0x6bc821a5 0x4597b179 + * Now, it is apparent why the 2nd object_id collates after the 1st: the + * first __le32 value of the 1st object_id is less than the first __le32 of + * the 2nd object_id. If the first __le32 values of both object_ids were + * equal then the second __le32 values would be compared, etc. + */ +enum { + COLLATION_BINARY = cpu_to_le32(0x00), + COLLATION_FILE_NAME = cpu_to_le32(0x01), + COLLATION_UNICODE_STRING = cpu_to_le32(0x02), + COLLATION_NTOFS_ULONG = cpu_to_le32(0x10), + COLLATION_NTOFS_SID = cpu_to_le32(0x11), + COLLATION_NTOFS_SECURITY_HASH = cpu_to_le32(0x12), + COLLATION_NTOFS_ULONGS = cpu_to_le32(0x13), +}; + +/* + * enum - Attribute definition flags + * + * The flags (32-bit) describing attribute properties in the attribute + * definition structure. + * The INDEXABLE flag is fairly certainly correct as only the file + * name attribute has this flag set and this is the only attribute indexed in + * NT4. + * + * ATTR_DEF_INDEXABLE: Attribute can be indexed. + * (Used for creating indexes like $I30, $SDH, etc.) + * ATTR_DEF_MULTIPLE: Attribute type can be present multiple times + * in the MFT record of an inode. + * (e.g., multiple $FILE_NAME, $DATA streams) + * ATTR_DEF_NOT_ZERO: Attribute value must contain at least one non-zero byte. + * (Prevents empty or all-zero values) + * ATTR_DEF_INDEXED_UNIQUE: Attribute must be indexed and the value must be unique + * for this attribute type across all MFT records of an inode. + * (e.g., security descriptor IDs in $Secure) + * ATTR_DEF_NAMED_UNIQUE: Attribute must be named and the name must be unique + * for this attribute type across all MFT records of an inode. + * (e.g., named $DATA streams or alternate data streams) + * ATTR_DEF_RESIDENT: Attribute must be resident (stored in MFT record). + * (Cannot be non-resident/sparse/compressed) + * ATTR_DEF_ALWAYS_LOG: Always log modifications to this attribute in LogFile, + * regardless of whether it is resident or non-resident. + * Without this flag, modifications are logged only if resident. + * (Used for critical metadata attributes) + */ +enum { + ATTR_DEF_INDEXABLE = cpu_to_le32(0x02), + ATTR_DEF_MULTIPLE = cpu_to_le32(0x04), + ATTR_DEF_NOT_ZERO = cpu_to_le32(0x08), + ATTR_DEF_INDEXED_UNIQUE = cpu_to_le32(0x10), + ATTR_DEF_NAMED_UNIQUE = cpu_to_le32(0x20), + ATTR_DEF_RESIDENT = cpu_to_le32(0x40), + ATTR_DEF_ALWAYS_LOG = cpu_to_le32(0x80), +}; + +/* + * struct attr_def - Attribute definition entry ($AttrDef array) + * + * The data attribute of FILE_AttrDef contains a sequence of attribute + * definitions for the NTFS volume. With this, it is supposed to be safe for an + * older NTFS driver to mount a volume containing a newer NTFS version without + * damaging it (that's the theory. In practice it's: not damaging it too much). + * Entries are sorted by attribute type. The flags describe whether the + * attribute can be resident/non-resident and possibly other things, but the + * actual bits are unknown. + * + * @name: Unicode (UTF-16LE) name of the attribute (e.g. "$DATA", "$FILE_NAME"). + * Zero-terminated string, maximum 0x40 characters (128 bytes). + * Used for human-readable display and debugging. + * @type: Attribute type code (ATTR_TYPE_* constants). + * Defines which attribute this entry describes. + * @display_rule: Default display rule (usually 0; rarely used in modern NTFS). + * Controls how the attribute is displayed in tools (legacy). + * @collation_rule: Default collation rule for indexing this attribute. + * Determines sort order when indexed (e.g. CASE_SENSITIVE, UNICODE). + * Used in $I30, $SDH, $SII, etc. + * @flags: Bit array of attribute constraints (ATTR_DEF_* flags). + * See ATTR_DEF_INDEXABLE, ATTR_DEF_MULTIPLE, etc. + * Defines whether the attribute can be indexed, multiple, resident-only, etc. + * @min_size: Optional minimum size of the attribute value (in bytes). + * 0 means no minimum enforced. + * @max_size: Maximum allowed size of the attribute value (in bytes). + */ +struct attr_def { + __le16 name[0x40]; + __le32 type; + __le32 display_rule; + __le32 collation_rule; + __le32 flags; + __le64 min_size; + __le64 max_size; +} __packed; + +static_assert(sizeof(struct attr_def) == 160); + +/* + * enum - Attribute flags (16-bit) for non-resident attributes + * + * ATTR_IS_COMPRESSED: Attribute is compressed. + * If set, data is compressed using the method in + * ATTR_COMPRESSION_MASK. + * ATTR_COMPRESSION_MASK: Mask for compression method. + * Valid values are defined in NTFS compression types + * (e.g., 0x02 = LZNT1, etc.). + * Also serves as the first illegal value for method. + * ATTR_IS_ENCRYPTED: Attribute is encrypted. + * Data is encrypted using EFS (Encrypting File System). + * ATTR_IS_SPARSE: Attribute is sparse. + * Contains holes (unallocated regions) that read as zeros. + */ +enum { + ATTR_IS_COMPRESSED = cpu_to_le16(0x0001), + ATTR_COMPRESSION_MASK = cpu_to_le16(0x00ff), + ATTR_IS_ENCRYPTED = cpu_to_le16(0x4000), + ATTR_IS_SPARSE = cpu_to_le16(0x8000), +} __packed; + +/* + * Attribute compression. + * + * Only the data attribute is ever compressed in the current ntfs driver in + * Windows. Further, compression is only applied when the data attribute is + * non-resident. Finally, to use compression, the maximum allowed cluster size + * on a volume is 4kib. + * + * The compression method is based on independently compressing blocks of X + * clusters, where X is determined from the compression_unit value found in the + * non-resident attribute record header (more precisely: X = 2^compression_unit + * clusters). On Windows NT/2k, X always is 16 clusters (compression_unit = 4). + * + * There are three different cases of how a compression block of X clusters + * can be stored: + * + * 1) The data in the block is all zero (a sparse block): + * This is stored as a sparse block in the runlist, i.e. the runlist + * entry has length = X and lcn = -1. The mapping pairs array actually + * uses a delta_lcn value length of 0, i.e. delta_lcn is not present at + * all, which is then interpreted by the driver as lcn = -1. + * NOTE: Even uncompressed files can be sparse on NTFS 3.0 volumes, then + * the same principles apply as above, except that the length is not + * restricted to being any particular value. + * + * 2) The data in the block is not compressed: + * This happens when compression doesn't reduce the size of the block + * in clusters. I.e. if compression has a small effect so that the + * compressed data still occupies X clusters, then the uncompressed data + * is stored in the block. + * This case is recognised by the fact that the runlist entry has + * length = X and lcn >= 0. The mapping pairs array stores this as + * normal with a run length of X and some specific delta_lcn, i.e. + * delta_lcn has to be present. + * + * 3) The data in the block is compressed: + * The common case. This case is recognised by the fact that the run + * list entry has length L < X and lcn >= 0. The mapping pairs array + * stores this as normal with a run length of X and some specific + * delta_lcn, i.e. delta_lcn has to be present. This runlist entry is + * immediately followed by a sparse entry with length = X - L and + * lcn = -1. The latter entry is to make up the vcn counting to the + * full compression block size X. + * + * In fact, life is more complicated because adjacent entries of the same type + * can be coalesced. This means that one has to keep track of the number of + * clusters handled and work on a basis of X clusters at a time being one + * block. An example: if length L > X this means that this particular runlist + * entry contains a block of length X and part of one or more blocks of length + * L - X. Another example: if length L < X, this does not necessarily mean that + * the block is compressed as it might be that the lcn changes inside the block + * and hence the following runlist entry describes the continuation of the + * potentially compressed block. The block would be compressed if the + * following runlist entry describes at least X - L sparse clusters, thus + * making up the compression block length as described in point 3 above. (Of + * course, there can be several runlist entries with small lengths so that the + * sparse entry does not follow the first data containing entry with + * length < X.) + * + * NOTE: At the end of the compressed attribute value, there most likely is not + * just the right amount of data to make up a compression block, thus this data + * is not even attempted to be compressed. It is just stored as is, unless + * the number of clusters it occupies is reduced when compressed in which case + * it is stored as a compressed compression block, complete with sparse + * clusters at the end. + */ + +/* + * enum - Flags for resident attributes (8-bit) + * + * RESIDENT_ATTR_IS_INDEXED: Attribute is referenced in an index. + * (e.g., part of an index key or entry) + * Has implications for deletion and modification: + * - Cannot be freely removed if indexed + * - Index must be updated when value changes + * - Used for attributes like $FILE_NAME in directories + */ +enum { + RESIDENT_ATTR_IS_INDEXED = 0x01, +} __packed; + +/* + * struct attr_record - NTFS attribute record header + * + * Common header for both resident and non-resident attributes. + * Always aligned to an 8-byte boundary on disk. + * Located at attrs_offset in the MFT record (see struct mft_record). + * + * @type: 32-bit attribute type (ATTR_TYPE_* constants). + * Identifies the attribute + * (e.g. 0x10 = $STANDARD_INFORMATION). + * @length: Total byte size of this attribute record (resident). + * 8-byte aligned; used to locate the next attribute. + * @non_resident: 0 = resident attribute + * 1 = non-resident attribute + * @name_length: Number of Unicode characters in the attribute name. + * 0 if unnamed (most system attributes are unnamed). + * @name_offset: Byte offset from start of attribute record to the name. + * 8-byte aligned; when creating, place at end of header. + * @flags: Attribute flags (see ATTR_IS_COMPRESSED, + * ATTR_IS_ENCRYPTED, etc.). + * For resident: see RESIDENT_ATTR_* flags. + * @instance: Unique instance number within this MFT record. + * Incremented via next_attr_instance; unique per record. + * + * Resident attributes (when @non_resident == 0): + * @data.resident.value_length: Byte size of the attribute value. + * @data.resident.value_offset: Byte offset from start of attribute + * record to the value data. + * 8-byte aligned if name present. + * @data.resident.flags: Resident-specific flags + * @data.resident.reserved: Reserved/alignment to 8 bytes. + * + * Non-resident attributes (when @non_resident == 1): + * @data.non_resident.lowest_vcn: Lowest valid VCN in this extent. + * Usually 0 unless attribute list is used. + * @data.non_resident.highest_vcn: Highest valid VCN in this extent. + * -1 for zero-length, 0 for single extent. + * @data.non_resident.mapping_pairs_offset: + * Byte offset to mapping pairs array + * (VCN → LCN mappings). + * 8-byte aligned when creating. + * @data.non_resident.compression_unit: + * Log2 of clusters per compression unit. + * 0 = not compressed. + * WinNT4 used 4; sparse files use 0 + * on XP SP2+. + * @data.non_resident.reserved: 5 bytes for 8-byte alignment. + * @data.non_resident.allocated_size: + * Allocated disk space in bytes. + * For compressed: logical allocated size. + * @data.non_resident.data_size: Logical attribute value size in bytes. + * Can be larger than allocated_size if + * compressed/sparse. + * @data.non_resident.initialized_size: + * Initialized portion size in bytes. + * Usually equals data_size. + * @data.non_resident.compressed_size: + * Compressed on-disk size in bytes. + * Only present when compressed or sparse. + * Actual disk usage. + */ +struct attr_record { + __le32 type; + __le32 length; + u8 non_resident; + u8 name_length; + __le16 name_offset; + __le16 flags; + __le16 instance; + union { + struct { + __le32 value_length; + __le16 value_offset; + u8 flags; + s8 reserved; + } __packed resident; + struct { + __le64 lowest_vcn; + __le64 highest_vcn; + __le16 mapping_pairs_offset; + u8 compression_unit; + u8 reserved[5]; + __le64 allocated_size; + __le64 data_size; + __le64 initialized_size; + __le64 compressed_size; + } __packed non_resident; + } __packed data; +} __packed; + +/* + * enum - NTFS file attribute flags (32-bit) + * + * File attribute flags (32-bit) appearing in the file_attributes fields of the + * STANDARD_INFORMATION attribute of MFT_RECORDs and the FILENAME_ATTR + * attributes of MFT_RECORDs and directory index entries. + * + * All of the below flags appear in the directory index entries but only some + * appear in the STANDARD_INFORMATION attribute whilst only some others appear + * in the FILENAME_ATTR attribute of MFT_RECORDs. Unless otherwise stated the + * flags appear in all of the above. + * + * FILE_ATTR_READONLY: File is read-only. + * FILE_ATTR_HIDDEN: File is hidden (not shown by default). + * FILE_ATTR_SYSTEM: System file (protected by OS). + * FILE_ATTR_DIRECTORY: Directory flag (reserved in NT; use MFT flag instead). + * FILE_ATTR_ARCHIVE: File needs archiving (backup flag). + * FILE_ATTR_DEVICE: Device file (rarely used). + * FILE_ATTR_NORMAL: Normal file (no special attributes). + * FILE_ATTR_TEMPORARY: Temporary file (delete on close). + * FILE_ATTR_SPARSE_FILE: Sparse file (contains holes). + * FILE_ATTR_REPARSE_POINT: Reparse point (junction, symlink, mount point). + * FILE_ATTR_COMPRESSED: File is compressed. + * FILE_ATTR_OFFLINE: File data is offline (not locally available). + * FILE_ATTR_NOT_CONTENT_INDEXED: + * File is excluded from content indexing. + * FILE_ATTR_ENCRYPTED: File is encrypted (EFS). + * FILE_ATTR_VALID_FLAGS: Mask of all valid flags for reading. + * FILE_ATTR_VALID_SET_FLAGS: Mask of flags that can be set by user. + * FILE_ATTRIBUTE_RECALL_ON_OPEN: + * Recall data on open (cloud/HSM related). + * FILE_ATTR_DUP_FILE_NAME_INDEX_PRESENT: + * $FILE_NAME has duplicate index entry. + * FILE_ATTR_DUP_VIEW_INDEX_PRESENT: + * Duplicate view index present (object ID, quota, etc.). + */ +enum { + FILE_ATTR_READONLY = cpu_to_le32(0x00000001), + FILE_ATTR_HIDDEN = cpu_to_le32(0x00000002), + FILE_ATTR_SYSTEM = cpu_to_le32(0x00000004), + /* Old DOS volid. Unused in NT. = cpu_to_le32(0x00000008), */ + FILE_ATTR_DIRECTORY = cpu_to_le32(0x00000010), + FILE_ATTR_ARCHIVE = cpu_to_le32(0x00000020), + FILE_ATTR_DEVICE = cpu_to_le32(0x00000040), + FILE_ATTR_NORMAL = cpu_to_le32(0x00000080), + + FILE_ATTR_TEMPORARY = cpu_to_le32(0x00000100), + FILE_ATTR_SPARSE_FILE = cpu_to_le32(0x00000200), + FILE_ATTR_REPARSE_POINT = cpu_to_le32(0x00000400), + FILE_ATTR_COMPRESSED = cpu_to_le32(0x00000800), + + FILE_ATTR_OFFLINE = cpu_to_le32(0x00001000), + FILE_ATTR_NOT_CONTENT_INDEXED = cpu_to_le32(0x00002000), + FILE_ATTR_ENCRYPTED = cpu_to_le32(0x00004000), + + FILE_ATTR_VALID_FLAGS = cpu_to_le32(0x00007fb7), + FILE_ATTR_VALID_SET_FLAGS = cpu_to_le32(0x000031a7), + FILE_ATTRIBUTE_RECALL_ON_OPEN = cpu_to_le32(0x00040000), + FILE_ATTR_DUP_FILE_NAME_INDEX_PRESENT = cpu_to_le32(0x10000000), + FILE_ATTR_DUP_VIEW_INDEX_PRESENT = cpu_to_le32(0x20000000), +}; + +/* + * NOTE on times in NTFS: All times are in MS standard time format, i.e. they + * are the number of 100-nanosecond intervals since 1st January 1601, 00:00:00 + * universal coordinated time (UTC). (In Linux time starts 1st January 1970, + * 00:00:00 UTC and is stored as the number of 1-second intervals since then.) + */ + +/* + * struct standard_information - $STANDARD_INFORMATION attribute content + * + * NOTE: Always resident. + * NOTE: Present in all base file records on a volume. + * NOTE: There is conflicting information about the meaning of each of the time + * fields but the meaning as defined below has been verified to be + * correct by practical experimentation on Windows NT4 SP6a and is hence + * assumed to be the one and only correct interpretation. + * + * @creation_time: File creation time (NTFS timestamp). + * Updated on filename change(?). + * @last_data_change_time: Last modification time of data streams. + * @last_mft_change_time: Last modification time of this MFT record. + * @last_access_time: Last access time (approximate). + * Not updated on read-only volumes; can be disabled. + * @file_attributes: File attribute flags (FILE_ATTR_* bits). + * + * Union (version-specific fields): + * @ver.v1.reserved12: 12 bytes reserved/alignment (NTFS 1.2 only). + * + * @ver.v3 (NTFS 3.x / Windows 2000+): + * @maximum_versions: Max allowed file versions (0 = disabled). + * @version_number: Current version number (0 if disabled). + * @class_id: Class ID (from bidirectional index?). + * @owner_id: Owner ID (maps to $Quota via $Q index). + * @security_id: Security ID (maps to $Secure $SII/$SDS). + * @quota_charged: Quota charge in bytes (0 if quotas disabled). + * @usn: Last USN from $UsnJrnl (0 if disabled). + */ +struct standard_information { + __le64 creation_time; + __le64 last_data_change_time; + __le64 last_mft_change_time; + __le64 last_access_time; + __le32 file_attributes; + union { + struct { + u8 reserved12[12]; + } __packed v1; + struct { + __le32 maximum_versions; + __le32 version_number; + __le32 class_id; + __le32 owner_id; + __le32 security_id; + __le64 quota_charged; + __le64 usn; + } __packed v3; + } __packed ver; +} __packed; + +/* + * struct attr_list_entry - Entry in $ATTRIBUTE_LIST attribute. + * + * @type: Attribute type code (ATTR_TYPE_*). + * @length: Byte size of this entry (8-byte aligned). + * @name_length: Unicode char count of attribute name (0 if unnamed). + * @name_offset: Byte offset from start of entry to name (always set). + * @lowest_vcn: Lowest VCN of this attribute extent (usually 0). + * Signed value; non-zero when attribute spans extents. + * @mft_reference: MFT record reference holding this attribute extent. + * @instance: Attribute instance number (if lowest_vcn == 0); else 0. + * @name: Variable Unicode name (use @name_offset when reading). + * + * - Can be either resident or non-resident. + * - Value consists of a sequence of variable length, 8-byte aligned, + * ATTR_LIST_ENTRY records. + * - The list is not terminated by anything at all! The only way to know when + * the end is reached is to keep track of the current offset and compare it to + * the attribute value size. + * - The attribute list attribute contains one entry for each attribute of + * the file in which the list is located, except for the list attribute + * itself. The list is sorted: first by attribute type, second by attribute + * name (if present), third by instance number. The extents of one + * non-resident attribute (if present) immediately follow after the initial + * extent. They are ordered by lowest_vcn and have their instance set to zero. + * It is not allowed to have two attributes with all sorting keys equal. + * - Further restrictions: + * - If not resident, the vcn to lcn mapping array has to fit inside the + * base mft record. + * - The attribute list attribute value has a maximum size of 256kb. This + * is imposed by the Windows cache manager. + * - Attribute lists are only used when the attributes of mft record do not + * fit inside the mft record despite all attributes (that can be made + * non-resident) having been made non-resident. This can happen e.g. when: + * - File has a large number of hard links (lots of file name + * attributes present). + * - The mapping pairs array of some non-resident attribute becomes so + * large due to fragmentation that it overflows the mft record. + * - The security descriptor is very complex (not applicable to + * NTFS 3.0 volumes). + * - There are many named streams. + */ +struct attr_list_entry { + __le32 type; + __le16 length; + u8 name_length; + u8 name_offset; + __le64 lowest_vcn; + __le64 mft_reference; + __le16 instance; + __le16 name[]; +} __packed; + +/* + * The maximum allowed length for a file name. + */ +#define MAXIMUM_FILE_NAME_LENGTH 255 + +/* + * enum - Possible namespaces for filenames in ntfs (8-bit). + * + * FILE_NAME_POSIX POSIX namespace (case sensitive, most permissive). + * Allows all Unicode except '\0' and '/'. + * WinNT/2k/2003 default utilities ignore case + * differences. SFU (Services For Unix) enables true + * case sensitivity. + * SFU restricts some chars: '"', '/', '<', '>', '\'. + * FILE_NAME_WIN32 Standard WinNT/2k long filename namespace + * (case insensitive). + * Disallows '\0', '"', '*', '/', ':', '<', '>', '?', + * '\', '|'. Names cannot end with '.' or space. + * FILE_NAME_DOS DOS 8.3 namespace (uppercase only). + * Allows 8-bit chars > space except '"', '*', '+', + * ',', '/', ':', ';', '<', '=', '>', '?', '\'. + * FILE_NAME_WIN32_AND_DOS + * Win32 and DOS names are identical (single record). + * Value 0x03 indicates both are stored in one entry. + */ +enum { + FILE_NAME_POSIX = 0x00, + FILE_NAME_WIN32 = 0x01, + FILE_NAME_DOS = 0x02, + FILE_NAME_WIN32_AND_DOS = 0x03, +} __packed; + +/* + * struct file_name_attr - $FILE_NAME attribute content + * + * NOTE: Always resident. + * NOTE: All fields, except the parent_directory, are only updated when the + * filename is changed. Until then, they just become out of sync with + * reality and the more up to date values are present in the standard + * information attribute. + * NOTE: There is conflicting information about the meaning of each of the time + * fields but the meaning as defined below has been verified to be + * correct by practical experimentation on Windows NT4 SP6a and is hence + * assumed to be the one and only correct interpretation. + * + * @parent_directory: MFT reference to parent directory. + * @creation_time: File creation time (NTFS timestamp). + * @last_data_change_time: + * Last data modification time. + * @last_mft_change_time: + * Last MFT record modification time. + * @last_access_time: Last access time (approximate; may not + * update always). + * @allocated_size: On-disk allocated size for unnamed $DATA. + * Equals compressed_size if compressed/sparse. + * 0 for directories or no $DATA. + * Multiple of cluster size. + * @data_size: Logical size of unnamed $DATA. + * 0 for directories or no $DATA. + * @file_attributes: File attribute flags (FILE_ATTR_* bits). + * @type.ea.packed_ea_size: + * Size needed to pack EAs (if present). + * @type.ea.reserved: Alignment padding. + * @type.rp.reparse_point_tag: + * Reparse point type (if reparse point, no EAs). + * @file_name_length: Length of filename in Unicode characters. + * @file_name_type: Namespace (FILE_NAME_POSIX, WIN32, DOS, etc.). + * @file_name: Variable-length Unicode filename. + */ +struct file_name_attr { + __le64 parent_directory; + __le64 creation_time; + __le64 last_data_change_time; + __le64 last_mft_change_time; + __le64 last_access_time; + __le64 allocated_size; + __le64 data_size; + __le32 file_attributes; + union { + struct { + __le16 packed_ea_size; + __le16 reserved; + } __packed ea; + struct { + __le32 reparse_point_tag; + } __packed rp; + } __packed type; + u8 file_name_length; + u8 file_name_type; + __le16 file_name[]; +} __packed; + +/* + * struct guid - Globally Unique Identifier (GUID) structure + * + * GUID structures store globally unique identifiers (GUID). A GUID is a + * 128-bit value consisting of one group of eight hexadecimal digits, followed + * by three groups of four hexadecimal digits each, followed by one group of + * twelve hexadecimal digits. GUIDs are Microsoft's implementation of the + * distributed computing environment (DCE) universally unique identifier (UUID). + * Example of a GUID: + * 1F010768-5A73-BC91-0010A52216A7 + * + * @data1: First 32 bits (first 8 hex digits). + * @data2: Next 16 bits (first group of 4 hex digits). + * @data3: Next 16 bits (second group of 4 hex digits). + * @data4: Final 64 bits (third group of 4 + last 12 hex digits). + * data4[0-1]: third group; data4[2-7]: remaining part. + */ +struct guid { + __le32 data1; + __le16 data2; + __le16 data3; + u8 data4[8]; +} __packed; + +/* + * struct object_id_attr - $OBJECT_ID attribute content (NTFS 3.0+) + * + * NOTE: Always resident. + * + * @object_id: Unique 128-bit GUID assigned to the file. + * Core identifier; always present. + * + * Optional extended info (union; total value size 16–64 bytes): + * @extended_info.birth_volume_id: + * Birth volume GUID (where file was first created). + * @extended_info.birth_object_id: + * Birth object GUID (original ID before copy/move). + * @extended_info.domain_id: + * Domain GUID (usually zero; reserved). + */ +struct object_id_attr { + struct guid object_id; + union { + struct { + struct guid birth_volume_id; + struct guid birth_object_id; + struct guid domain_id; + } __packed; + u8 extended_info[48]; + } __packed; +} __packed; + +/* + * enum - RIDs (Relative Identifiers) in Windows/NTFS security + * + * These relative identifiers (RIDs) are used with the above identifier + * authorities to make up universal well-known SIDs. + * + * SECURITY_NULL_RID S-1-0 (Null authority) + * SECURITY_WORLD_RID S-1-1 (World/Everyone) + * SECURITY_LOCAL_RID S-1-2 (Local) + * SECURITY_CREATOR_OWNER_RID S-1-3-0 (Creator Owner) + * SECURITY_CREATOR_GROUP_RID S-1-3-1 (Creator Group) + * SECURITY_CREATOR_OWNER_SERVER_RID S-1-3-2 (Server Creator Owner) + * SECURITY_CREATOR_GROUP_SERVER_RID S-1-3-3 (Server Creator Group) + * SECURITY_DIALUP_RID S-1-5-1 (Dialup) + * SECURITY_NETWORK_RID S-1-5-2 (Network) + * SECURITY_BATCH_RID S-1-5-3 (Batch) + * SECURITY_INTERACTIVE_RID S-1-5-4 (Interactive) + * SECURITY_SERVICE_RID S-1-5-6 (Service) + * SECURITY_ANONYMOUS_LOGON_RID S-1-5-7 (Anonymous Logon) + * SECURITY_PROXY_RID S-1-5-8 (Proxy) + * SECURITY_ENTERPRISE_CONTROLLERS_RID S-1-5-9 (Enterprise DCs) + * SECURITY_SERVER_LOGON_RID S-1-5-9 (Server Logon alias) + * SECURITY_PRINCIPAL_SELF_RID S-1-5-10 (Self/PrincipalSelf) + * SECURITY_AUTHENTICATED_USER_RID S-1-5-11 (Authenticated Users) + * SECURITY_RESTRICTED_CODE_RID S-1-5-12 (Restricted Code) + * SECURITY_TERMINAL_SERVER_RID S-1-5-13 (Terminal Server) + * SECURITY_LOGON_IDS_RID S-1-5-5 (Logon session IDs base) + * SECURITY_LOCAL_SYSTEM_RID S-1-5-18 (Local System) + * SECURITY_NT_NON_UNIQUE S-1-5-21 (NT non-unique authority) + * SECURITY_BUILTIN_DOMAIN_RID S-1-5-32 (Built-in domain) + * + * Built-in domain relative RIDs (S-1-5-32-...): + * Users: + * DOMAIN_USER_RID_ADMIN Administrator + * DOMAIN_USER_RID_GUEST Guest + * DOMAIN_USER_RID_KRBTGT krbtgt (Kerberos ticket-granting) + * + * Groups: + * DOMAIN_GROUP_RID_ADMINS Administrators + * DOMAIN_GROUP_RID_USERS Users + * DOMAIN_GROUP_RID_GUESTS Guests + * DOMAIN_GROUP_RID_COMPUTERS Computers + * DOMAIN_GROUP_RID_CONTROLLERS Domain Controllers + * DOMAIN_GROUP_RID_CERT_ADMINS Cert Publishers + * DOMAIN_GROUP_RID_SCHEMA_ADMINS Schema Admins + * DOMAIN_GROUP_RID_ENTERPRISE_ADMINS Enterprise Admins + * DOMAIN_GROUP_RID_POLICY_ADMINS Policy Admins (if present) + * + * Aliases: + * DOMAIN_ALIAS_RID_ADMINS Administrators alias + * DOMAIN_ALIAS_RID_USERS Users alias + * DOMAIN_ALIAS_RID_GUESTS Guests alias + * DOMAIN_ALIAS_RID_POWER_USERS Power Users + * DOMAIN_ALIAS_RID_ACCOUNT_OPS Account Operators + * DOMAIN_ALIAS_RID_SYSTEM_OPS Server Operators + * DOMAIN_ALIAS_RID_PRINT_OPS Print Operators + * DOMAIN_ALIAS_RID_BACKUP_OPS Backup Operators + * DOMAIN_ALIAS_RID_REPLICATOR Replicator + * DOMAIN_ALIAS_RID_RAS_SERVERS RAS Servers + * DOMAIN_ALIAS_RID_PREW2KCOMPACCESS Pre-Windows 2000 Compatible Access + * + * Note: The relative identifier (RID) refers to the portion of a SID, which + * identifies a user or group in relation to the authority that issued the SID. + * For example, the universal well-known SID Creator Owner ID (S-1-3-0) is + * made up of the identifier authority SECURITY_CREATOR_SID_AUTHORITY (3) and + * the relative identifier SECURITY_CREATOR_OWNER_RID (0). + */ +enum { /* Identifier authority. */ + SECURITY_NULL_RID = 0, /* S-1-0 */ + SECURITY_WORLD_RID = 0, /* S-1-1 */ + SECURITY_LOCAL_RID = 0, /* S-1-2 */ + + SECURITY_CREATOR_OWNER_RID = 0, /* S-1-3 */ + SECURITY_CREATOR_GROUP_RID = 1, /* S-1-3 */ + + SECURITY_CREATOR_OWNER_SERVER_RID = 2, /* S-1-3 */ + SECURITY_CREATOR_GROUP_SERVER_RID = 3, /* S-1-3 */ + + SECURITY_DIALUP_RID = 1, + SECURITY_NETWORK_RID = 2, + SECURITY_BATCH_RID = 3, + SECURITY_INTERACTIVE_RID = 4, + SECURITY_SERVICE_RID = 6, + SECURITY_ANONYMOUS_LOGON_RID = 7, + SECURITY_PROXY_RID = 8, + SECURITY_ENTERPRISE_CONTROLLERS_RID = 9, + SECURITY_SERVER_LOGON_RID = 9, + SECURITY_PRINCIPAL_SELF_RID = 0xa, + SECURITY_AUTHENTICATED_USER_RID = 0xb, + SECURITY_RESTRICTED_CODE_RID = 0xc, + SECURITY_TERMINAL_SERVER_RID = 0xd, + + SECURITY_LOGON_IDS_RID = 5, + SECURITY_LOGON_IDS_RID_COUNT = 3, + + SECURITY_LOCAL_SYSTEM_RID = 0x12, + + SECURITY_NT_NON_UNIQUE = 0x15, + + SECURITY_BUILTIN_DOMAIN_RID = 0x20, + + /* + * Well-known domain relative sub-authority values (RIDs). + */ + + /* Users. */ + DOMAIN_USER_RID_ADMIN = 0x1f4, + DOMAIN_USER_RID_GUEST = 0x1f5, + DOMAIN_USER_RID_KRBTGT = 0x1f6, + + /* Groups. */ + DOMAIN_GROUP_RID_ADMINS = 0x200, + DOMAIN_GROUP_RID_USERS = 0x201, + DOMAIN_GROUP_RID_GUESTS = 0x202, + DOMAIN_GROUP_RID_COMPUTERS = 0x203, + DOMAIN_GROUP_RID_CONTROLLERS = 0x204, + DOMAIN_GROUP_RID_CERT_ADMINS = 0x205, + DOMAIN_GROUP_RID_SCHEMA_ADMINS = 0x206, + DOMAIN_GROUP_RID_ENTERPRISE_ADMINS = 0x207, + DOMAIN_GROUP_RID_POLICY_ADMINS = 0x208, + + /* Aliases. */ + DOMAIN_ALIAS_RID_ADMINS = 0x220, + DOMAIN_ALIAS_RID_USERS = 0x221, + DOMAIN_ALIAS_RID_GUESTS = 0x222, + DOMAIN_ALIAS_RID_POWER_USERS = 0x223, + + DOMAIN_ALIAS_RID_ACCOUNT_OPS = 0x224, + DOMAIN_ALIAS_RID_SYSTEM_OPS = 0x225, + DOMAIN_ALIAS_RID_PRINT_OPS = 0x226, + DOMAIN_ALIAS_RID_BACKUP_OPS = 0x227, + + DOMAIN_ALIAS_RID_REPLICATOR = 0x228, + DOMAIN_ALIAS_RID_RAS_SERVERS = 0x229, + DOMAIN_ALIAS_RID_PREW2KCOMPACCESS = 0x22a, +}; + +/* + * The universal well-known SIDs: + * + * NULL_SID S-1-0-0 + * WORLD_SID S-1-1-0 + * LOCAL_SID S-1-2-0 + * CREATOR_OWNER_SID S-1-3-0 + * CREATOR_GROUP_SID S-1-3-1 + * CREATOR_OWNER_SERVER_SID S-1-3-2 + * CREATOR_GROUP_SERVER_SID S-1-3-3 + * + * (Non-unique IDs) S-1-4 + * + * NT well-known SIDs: + * + * NT_AUTHORITY_SID S-1-5 + * DIALUP_SID S-1-5-1 + * + * NETWORD_SID S-1-5-2 + * BATCH_SID S-1-5-3 + * INTERACTIVE_SID S-1-5-4 + * SERVICE_SID S-1-5-6 + * ANONYMOUS_LOGON_SID S-1-5-7 (aka null logon session) + * PROXY_SID S-1-5-8 + * SERVER_LOGON_SID S-1-5-9 (aka domain controller account) + * SELF_SID S-1-5-10 (self RID) + * AUTHENTICATED_USER_SID S-1-5-11 + * RESTRICTED_CODE_SID S-1-5-12 (running restricted code) + * TERMINAL_SERVER_SID S-1-5-13 (running on terminal server) + * + * (Logon IDs) S-1-5-5-X-Y + * + * (NT non-unique IDs) S-1-5-0x15-... + * + * (Built-in domain) S-1-5-0x20 + */ + +/* + * struct ntfs_sid - Security Identifier (SID) structure + * + * @revision: SID revision level (usually 1). + * @sub_authority_count: Number of sub-authorities (1 or more). + * @identifier_authority: + * 48-bit identifier authority (S-1-x-...). + * @parts.high_part: high 16 bits. + * @parts.low_part: low 32 bits. + * @value: raw 6-byte array. + * @sub_authority: Variable array of 32-bit RIDs. + * At least one; defines the SID relative to authority. + * + * The SID structure is a variable-length structure used to uniquely identify + * users or groups. SID stands for security identifier. + * + * The standard textual representation of the SID is of the form: + * S-R-I-S-S... + * Where: + * - The first "S" is the literal character 'S' identifying the following + * digits as a SID. + * - R is the revision level of the SID expressed as a sequence of digits + * either in decimal or hexadecimal (if the later, prefixed by "0x"). + * - I is the 48-bit identifier_authority, expressed as digits as R above. + * - S... is one or more sub_authority values, expressed as digits as above. + * + * Example SID; the domain-relative SID of the local Administrators group on + * Windows NT/2k: + * S-1-5-32-544 + * This translates to a SID with: + * revision = 1, + * sub_authority_count = 2, + * identifier_authority = {0,0,0,0,0,5}, // SECURITY_NT_AUTHORITY + * sub_authority[0] = 32, // SECURITY_BUILTIN_DOMAIN_RID + * sub_authority[1] = 544 // DOMAIN_ALIAS_RID_ADMINS + */ +struct ntfs_sid { + u8 revision; + u8 sub_authority_count; + union { + struct { + u16 high_part; + u32 low_part; + } __packed parts; + u8 value[6]; + } identifier_authority; + __le32 sub_authority[]; +} __packed; + +/* + * enum - Predefined ACE types (8-bit) for NTFS security descriptors + * + * ACCESS_MIN_MS_ACE_TYPE: Minimum MS ACE type (0). + * ACCESS_ALLOWED_ACE_TYPE: Allow access (standard ACE). + * ACCESS_DENIED_ACE_TYPE: Deny access (standard ACE). + * SYSTEM_AUDIT_ACE_TYPE: Audit successful/failed access. + * SYSTEM_ALARM_ACE_TYPE: Alarm on access (not in Win2k+). + * ACCESS_MAX_MS_V2_ACE_TYPE: Max for V2 ACE types. + * ACCESS_ALLOWED_COMPOUND_ACE_TYPE: + * Compound ACE (legacy). + * ACCESS_MAX_MS_V3_ACE_TYPE: Max for V3 ACE types. + * ACCESS_MIN_MS_OBJECT_ACE_TYPE: Min for object ACE types (Win2k+). + * ACCESS_ALLOWED_OBJECT_ACE_TYPE: Allow with object-specific rights. + * ACCESS_DENIED_OBJECT_ACE_TYPE: Deny with object-specific rights. + * SYSTEM_AUDIT_OBJECT_ACE_TYPE: Audit with object-specific rights. + * SYSTEM_ALARM_OBJECT_ACE_TYPE: Alarm with object-specific rights. + * ACCESS_MAX_MS_OBJECT_ACE_TYPE: Max for object ACE types. + * ACCESS_MAX_MS_V4_ACE_TYPE: Max for V4 ACE types. + * ACCESS_MAX_MS_ACE_TYPE: Overall max ACE type (WinNT/2k). + */ +enum { + ACCESS_MIN_MS_ACE_TYPE = 0, + ACCESS_ALLOWED_ACE_TYPE = 0, + ACCESS_DENIED_ACE_TYPE = 1, + SYSTEM_AUDIT_ACE_TYPE = 2, + SYSTEM_ALARM_ACE_TYPE = 3, + ACCESS_MAX_MS_V2_ACE_TYPE = 3, + + ACCESS_ALLOWED_COMPOUND_ACE_TYPE = 4, + ACCESS_MAX_MS_V3_ACE_TYPE = 4, + ACCESS_MIN_MS_OBJECT_ACE_TYPE = 5, + ACCESS_ALLOWED_OBJECT_ACE_TYPE = 5, + ACCESS_DENIED_OBJECT_ACE_TYPE = 6, + SYSTEM_AUDIT_OBJECT_ACE_TYPE = 7, + SYSTEM_ALARM_OBJECT_ACE_TYPE = 8, + ACCESS_MAX_MS_OBJECT_ACE_TYPE = 8, + + ACCESS_MAX_MS_V4_ACE_TYPE = 8, + ACCESS_MAX_MS_ACE_TYPE = 8, +} __packed; + +/* + * enum - ACE inheritance and audit flags (8-bit) + * + * OBJECT_INHERIT_ACE: Object inherit (files inherit this ACE). + * CONTAINER_INHERIT_ACE: Container inherit (subdirectories inherit). + * NO_PROPAGATE_INHERIT_ACE: No propagation (stop inheritance after this level). + * INHERIT_ONLY_ACE: Inherit only (not applied to current object). + * INHERITED_ACE: ACE was inherited (Win2k+ only). + * VALID_INHERIT_FLAGS: Mask of all valid inheritance flags (0x1f). + * SUCCESSFUL_ACCESS_ACE_FLAG: Audit successful access (system audit ACE). + * FAILED_ACCESS_ACE_FLAG: Audit failed access (system audit ACE). + * + * SUCCESSFUL_ACCESS_ACE_FLAG is only used with system audit and alarm ACE + * types to indicate that a message is generated (in Windows!) for successful + * accesses. + * + * FAILED_ACCESS_ACE_FLAG is only used with system audit and alarm ACE types + * to indicate that a message is generated (in Windows!) for failed accesses. + */ +enum { + OBJECT_INHERIT_ACE = 0x01, + CONTAINER_INHERIT_ACE = 0x02, + NO_PROPAGATE_INHERIT_ACE = 0x04, + INHERIT_ONLY_ACE = 0x08, + INHERITED_ACE = 0x10, + VALID_INHERIT_FLAGS = 0x1f, + SUCCESSFUL_ACCESS_ACE_FLAG = 0x40, + FAILED_ACCESS_ACE_FLAG = 0x80, +} __packed; + +/* + * enum - NTFS access rights masks (32-bit) + * + * FILE_READ_DATA / FILE_LIST_DIRECTORY: Read file data / list dir contents. + * FILE_WRITE_DATA / FILE_ADD_FILE: Write file data / create file in dir. + * FILE_APPEND_DATA / FILE_ADD_SUBDIRECTORY: Append data / create subdir. + * FILE_READ_EA: Read extended attributes. + * FILE_WRITE_EA: Write extended attributes. + * FILE_EXECUTE / FILE_TRAVERSE: Execute file / traverse dir. + * FILE_DELETE_CHILD: Delete children in dir. + * FILE_READ_ATTRIBUTES: Read attributes. + * FILE_WRITE_ATTRIBUTES: Write attributes. + * + * Standard rights (object-independent): + * DELETE: Delete object. + * READ_CONTROL: Read security descriptor/owner. + * WRITE_DAC: Modify DACL. + * WRITE_OWNER: Change owner. + * SYNCHRONIZE: Wait on object signal state. + * + * Combinations: + * STANDARD_RIGHTS_READ / WRITE / EXECUTE: Aliases for READ_CONTROL. + * STANDARD_RIGHTS_REQUIRED: DELETE + READ_CONTROL + + * WRITE_DAC + WRITE_OWNER. + * STANDARD_RIGHTS_ALL: Above + SYNCHRONIZE. + * + * System/access types: + * ACCESS_SYSTEM_SECURITY: Access system ACL. + * MAXIMUM_ALLOWED: Maximum allowed access. + * + * Generic rights (high bits, map to specific/standard): + * GENERIC_ALL: Full access. + * GENERIC_EXECUTE: Execute/traverse. + * GENERIC_WRITE: Write (append, attrs, data, EA, etc.). + * GENERIC_READ: Read (attrs, data, EA, etc.). + * + * The specific rights (bits 0 to 15). These depend on the type of the object + * being secured by the ACE. + */ +enum { + FILE_READ_DATA = cpu_to_le32(0x00000001), + FILE_LIST_DIRECTORY = cpu_to_le32(0x00000001), + FILE_WRITE_DATA = cpu_to_le32(0x00000002), + FILE_ADD_FILE = cpu_to_le32(0x00000002), + FILE_APPEND_DATA = cpu_to_le32(0x00000004), + FILE_ADD_SUBDIRECTORY = cpu_to_le32(0x00000004), + FILE_READ_EA = cpu_to_le32(0x00000008), + FILE_WRITE_EA = cpu_to_le32(0x00000010), + FILE_EXECUTE = cpu_to_le32(0x00000020), + FILE_TRAVERSE = cpu_to_le32(0x00000020), + FILE_DELETE_CHILD = cpu_to_le32(0x00000040), + FILE_READ_ATTRIBUTES = cpu_to_le32(0x00000080), + FILE_WRITE_ATTRIBUTES = cpu_to_le32(0x00000100), + DELETE = cpu_to_le32(0x00010000), + READ_CONTROL = cpu_to_le32(0x00020000), + WRITE_DAC = cpu_to_le32(0x00040000), + WRITE_OWNER = cpu_to_le32(0x00080000), + SYNCHRONIZE = cpu_to_le32(0x00100000), + STANDARD_RIGHTS_READ = cpu_to_le32(0x00020000), + STANDARD_RIGHTS_WRITE = cpu_to_le32(0x00020000), + STANDARD_RIGHTS_EXECUTE = cpu_to_le32(0x00020000), + STANDARD_RIGHTS_REQUIRED = cpu_to_le32(0x000f0000), + STANDARD_RIGHTS_ALL = cpu_to_le32(0x001f0000), + ACCESS_SYSTEM_SECURITY = cpu_to_le32(0x01000000), + MAXIMUM_ALLOWED = cpu_to_le32(0x02000000), + GENERIC_ALL = cpu_to_le32(0x10000000), + GENERIC_EXECUTE = cpu_to_le32(0x20000000), + GENERIC_WRITE = cpu_to_le32(0x40000000), + GENERIC_READ = cpu_to_le32(0x80000000), +}; + +/* + * struct ntfs_ace - Access Control Entry (ACE) structure + * + * @type: ACE type (ACCESS_ALLOWED_ACE_TYPE, ACCESS_DENIED_ACE_TYPE, etc.). + * @flags: Inheritance and audit flags (OBJECT_INHERIT_ACE, etc.). + * @size: Total byte size of this ACE (header + SID + variable data). + * @mask: Access rights mask (FILE_READ_DATA, DELETE, GENERIC_ALL, etc.). + * @sid: Security Identifier (SID) this ACE applies to. + */ +struct ntfs_ace { + u8 type; + u8 flags; + __le16 size; + __le32 mask; + struct ntfs_sid sid; +} __packed; + +/* + * The object ACE flags (32-bit). + */ +enum { + ACE_OBJECT_TYPE_PRESENT = cpu_to_le32(1), + ACE_INHERITED_OBJECT_TYPE_PRESENT = cpu_to_le32(2), +}; + +/* + * struct ntfs_acl - NTFS Access Control List (ACL) header + * + * An ACL is an access-control list (ACL). + * An ACL starts with an ACL header structure, which specifies the size of + * the ACL and the number of ACEs it contains. The ACL header is followed by + * zero or more access control entries (ACEs). The ACL as well as each ACE + * are aligned on 4-byte boundaries. + * + * @revision: ACL revision level (usually 2 or 4). + * @alignment1: Padding/alignment byte (zero). + * @size: Total allocated size in bytes (header + all ACEs + + * free space). + * @ace_count: Number of ACE entries following the header. + * @alignment2: Padding/alignment (zero). + */ +struct ntfs_acl { + u8 revision; + u8 alignment1; + __le16 size; + __le16 ace_count; + __le16 alignment2; +} __packed; + +static_assert(sizeof(struct ntfs_acl) == 8); + +/* + * The security descriptor control flags (16-bit). + * + * SE_OWNER_DEFAULTED - This boolean flag, when set, indicates that the SID + * pointed to by the Owner field was provided by a defaulting mechanism + * rather than explicitly provided by the original provider of the + * security descriptor. This may affect the treatment of the SID with + * respect to inheritance of an owner. + * + * SE_GROUP_DEFAULTED - This boolean flag, when set, indicates that the SID in + * the Group field was provided by a defaulting mechanism rather than + * explicitly provided by the original provider of the security + * descriptor. This may affect the treatment of the SID with respect to + * inheritance of a primary group. + * + * SE_DACL_PRESENT - This boolean flag, when set, indicates that the security + * descriptor contains a discretionary ACL. If this flag is set and the + * Dacl field of the SECURITY_DESCRIPTOR is null, then a null ACL is + * explicitly being specified. + * + * SE_DACL_DEFAULTED - This boolean flag, when set, indicates that the ACL + * pointed to by the Dacl field was provided by a defaulting mechanism + * rather than explicitly provided by the original provider of the + * security descriptor. This may affect the treatment of the ACL with + * respect to inheritance of an ACL. This flag is ignored if the + * DaclPresent flag is not set. + * + * SE_SACL_PRESENT - This boolean flag, when set, indicates that the security + * descriptor contains a system ACL pointed to by the Sacl field. If this + * flag is set and the Sacl field of the SECURITY_DESCRIPTOR is null, then + * an empty (but present) ACL is being specified. + * + * SE_SACL_DEFAULTED - This boolean flag, when set, indicates that the ACL + * pointed to by the Sacl field was provided by a defaulting mechanism + * rather than explicitly provided by the original provider of the + * security descriptor. This may affect the treatment of the ACL with + * respect to inheritance of an ACL. This flag is ignored if the + * SaclPresent flag is not set. + * + * SE_SELF_RELATIVE - This boolean flag, when set, indicates that the security + * descriptor is in self-relative form. In this form, all fields of the + * security descriptor are contiguous in memory and all pointer fields are + * expressed as offsets from the beginning of the security descriptor. + */ +enum { + SE_OWNER_DEFAULTED = cpu_to_le16(0x0001), + SE_GROUP_DEFAULTED = cpu_to_le16(0x0002), + SE_DACL_PRESENT = cpu_to_le16(0x0004), + SE_DACL_DEFAULTED = cpu_to_le16(0x0008), + + SE_SACL_PRESENT = cpu_to_le16(0x0010), + SE_SACL_DEFAULTED = cpu_to_le16(0x0020), + + SE_DACL_AUTO_INHERIT_REQ = cpu_to_le16(0x0100), + SE_SACL_AUTO_INHERIT_REQ = cpu_to_le16(0x0200), + SE_DACL_AUTO_INHERITED = cpu_to_le16(0x0400), + SE_SACL_AUTO_INHERITED = cpu_to_le16(0x0800), + + SE_DACL_PROTECTED = cpu_to_le16(0x1000), + SE_SACL_PROTECTED = cpu_to_le16(0x2000), + SE_RM_CONTROL_VALID = cpu_to_le16(0x4000), + SE_SELF_RELATIVE = cpu_to_le16(0x8000) +} __packed; + +/* + * struct security_descriptor_relative - Relative security descriptor + * + * Self-relative security descriptor. Contains the owner and group SIDs as well + * as the sacl and dacl ACLs inside the security descriptor itself. + * + * @revision: Security descriptor revision (usually 1). + * @alignment: Padding/alignment byte (zero). + * @control: Control flags (SE_OWNER_DEFAULTED, SE_DACL_PRESENT, + * SE_SACL_PRESENT, SE_SACL_AUTO_INHERITED, etc.). + * @owner: Byte offset to owner SID (from start of descriptor). + * 0 if no owner SID present. + * @group: Byte offset to primary group SID. + * 0 if no group SID present. + * @sacl: Byte offset to System ACL (SACL). + * Valid only if SE_SACL_PRESENT in @control. + * 0 means NULL SACL. + * @dacl: Byte offset to Discretionary ACL (DACL). + * Valid only if SE_DACL_PRESENT in @control. + * 0 means NULL DACL (full access granted). + */ +struct security_descriptor_relative { + u8 revision; + u8 alignment; + __le16 control; + __le32 owner; + __le32 group; + __le32 sacl; + __le32 dacl; +} __packed; + +static_assert(sizeof(struct security_descriptor_relative) == 20); + +/* + * On NTFS 3.0+, all security descriptors are stored in FILE_Secure. Only one + * referenced instance of each unique security descriptor is stored. + * + * FILE_Secure contains no unnamed data attribute, i.e. it has zero length. It + * does, however, contain two indexes ($SDH and $SII) as well as a named data + * stream ($SDS). + * + * Every unique security descriptor is assigned a unique security identifier + * (security_id, not to be confused with a SID). The security_id is unique for + * the NTFS volume and is used as an index into the $SII index, which maps + * security_ids to the security descriptor's storage location within the $SDS + * data attribute. The $SII index is sorted by ascending security_id. + * + * A simple hash is computed from each security descriptor. This hash is used + * as an index into the $SDH index, which maps security descriptor hashes to + * the security descriptor's storage location within the $SDS data attribute. + * The $SDH index is sorted by security descriptor hash and is stored in a B+ + * tree. When searching $SDH (with the intent of determining whether or not a + * new security descriptor is already present in the $SDS data stream), if a + * matching hash is found, but the security descriptors do not match, the + * search in the $SDH index is continued, searching for a next matching hash. + * + * When a precise match is found, the security_id coresponding to the security + * descriptor in the $SDS attribute is read from the found $SDH index entry and + * is stored in the $STANDARD_INFORMATION attribute of the file/directory to + * which the security descriptor is being applied. The $STANDARD_INFORMATION + * attribute is present in all base mft records (i.e. in all files and + * directories). + * + * If a match is not found, the security descriptor is assigned a new unique + * security_id and is added to the $SDS data attribute. Then, entries + * referencing the this security descriptor in the $SDS data attribute are + * added to the $SDH and $SII indexes. + * + * Note: Entries are never deleted from FILE_Secure, even if nothing + * references an entry any more. + */ + +/* + * struct sii_index_key - Key for $SII index in $Secure file + * + * The index entry key used in the $SII index. The collation type is + * COLLATION_NTOFS_ULONG. + * + * @security_id: 32-bit security identifier. + * Unique ID assigned to a security descriptor. + */ +struct sii_index_key { + __le32 security_id; +} __packed; + +/* + * struct sdh_index_key - Key for $SDH index in $Secure file + * + * The index entry key used in the $SDH index. The keys are sorted first by + * hash and then by security_id. The collation rule is + * COLLATION_NTOFS_SECURITY_HASH. + * + * @hash: 32-bit hash of the security descriptor. + * Used for quick collision checks and indexing. + * @security_id: 32-bit security identifier. + * Unique ID assigned to the descriptor. + */ +struct sdh_index_key { + __le32 hash; + __le32 security_id; +} __packed; + +/* + * enum - NTFS volume flags (16-bit) + * + * These flags are stored in $VolumeInformation attribute. + * They indicate volume state and required actions. + * + * VOLUME_IS_DIRTY: Volume is dirty (needs chkdsk). + * VOLUME_RESIZE_LOG_FILE: Resize LogFile on next mount. + * VOLUME_UPGRADE_ON_MOUNT: Upgrade volume on mount (old NTFS). + * VOLUME_MOUNTED_ON_NT4: Mounted on NT4 (compatibility flag). + * VOLUME_DELETE_USN_UNDERWAY: USN journal deletion in progress. + * VOLUME_REPAIR_OBJECT_ID: Repair $ObjId on next mount. + * VOLUME_CHKDSK_UNDERWAY: Chkdsk is running. + * VOLUME_MODIFIED_BY_CHKDSK: Modified by chkdsk. + * VOLUME_FLAGS_MASK: Mask of all valid flags (0xc03f). + * VOLUME_MUST_MOUNT_RO_MASK: Flags forcing read-only mount (0xc027). + * If any set, mount read-only. + */ +enum { + VOLUME_IS_DIRTY = cpu_to_le16(0x0001), + VOLUME_RESIZE_LOG_FILE = cpu_to_le16(0x0002), + VOLUME_UPGRADE_ON_MOUNT = cpu_to_le16(0x0004), + VOLUME_MOUNTED_ON_NT4 = cpu_to_le16(0x0008), + + VOLUME_DELETE_USN_UNDERWAY = cpu_to_le16(0x0010), + VOLUME_REPAIR_OBJECT_ID = cpu_to_le16(0x0020), + + VOLUME_CHKDSK_UNDERWAY = cpu_to_le16(0x4000), + VOLUME_MODIFIED_BY_CHKDSK = cpu_to_le16(0x8000), + + VOLUME_FLAGS_MASK = cpu_to_le16(0xc03f), + + VOLUME_MUST_MOUNT_RO_MASK = cpu_to_le16(0xc027), +} __packed; + +/* + * struct volume_information - $VOLUME_INFORMATION (0x70) + * + * @reserved: Reserved 64-bit field (currently unused). + * @major_ver: Major NTFS version number (e.g., 3 for NTFS 3.1). + * @minor_ver: Minor NTFS version number (e.g., 1 for NTFS 3.1). + * @flags: Volume flags (VOLUME_IS_DIRTY, VOLUME_CHKDSK_UNDERWAY, etc.). + * See volume flags enum for details. + * + * NOTE: Always resident. + * NOTE: Present only in FILE_Volume. + * NOTE: Windows 2000 uses NTFS 3.0 while Windows NT4 service pack 6a uses + * NTFS 1.2. I haven't personally seen other values yet. + */ +struct volume_information { + __le64 reserved; + u8 major_ver; + u8 minor_ver; + __le16 flags; +} __packed; + +/* + * enum - Index header flags + * + * These flags are stored in the index header (INDEX_HEADER.flags) for both + * index root ($INDEX_ROOT) and index allocation blocks ($INDEX_ALLOCATION). + * + * For index root ($INDEX_ROOT attribute): + * SMALL_INDEX: Index fits entirely in root attribute (no $INDEX_ALLOCATION). + * LARGE_INDEX: Index too large for root; $INDEX_ALLOCATION present. + * + * For index blocks ($INDEX_ALLOCATION): + * LEAF_NODE: Leaf node (no child nodes; contains actual entries). + * INDEX_NODE: Internal node (indexes other nodes; contains keys/pointers). + * + * NODE_MASK: Mask to extract node type bits (0x01). + */ +enum { + SMALL_INDEX = 0, + LARGE_INDEX = 1, + LEAF_NODE = 0, + INDEX_NODE = 1, + NODE_MASK = 1, +} __packed; + +/* + * struct index_header - Common header for index root and index blocks + * + * entries_offset: Byte offset to first INDEX_ENTRY (8-byte aligned). + * index_length: Bytes used by index entries (8-byte aligned). + * From entries_offset to end of used data. + * allocated_size: Total allocated bytes for this index block. + * Fixed size in index allocation; dynamic in root. + * flags: Index flags (SMALL_INDEX, LARGE_INDEX, LEAF_NODE, etc.). + * See INDEX_HEADER_FLAGS enum. + * reserved: 3 bytes reserved/padding (zero, 8-byte aligned). + * + * This is the header for indexes, describing the INDEX_ENTRY records, which + * follow the index_header. Together the index header and the index entries + * make up a complete index. + * + * IMPORTANT NOTE: The offset, length and size structure members are counted + * relative to the start of the index header structure and not relative to the + * start of the index root or index allocation structures themselves. + * + * For the index root attribute, the above two numbers are always + * equal, as the attribute is resident and it is resized as needed. In + * the case of the index allocation attribute the attribute is not + * resident and hence the allocated_size is a fixed value and must + * equal the index_block_size specified by the INDEX_ROOT attribute + * corresponding to the INDEX_ALLOCATION attribute this INDEX_BLOCK + * belongs to. + */ +struct index_header { + __le32 entries_offset; + __le32 index_length; + __le32 allocated_size; + u8 flags; + u8 reserved[3]; +} __packed; + +/* + * struct index_root - $INDEX_ROOT attribute (0x90). + * + * @type: Indexed attribute type ($FILE_NAME for dirs, + * 0 for view indexes). + * @collation_rule: Collation rule for sorting entries + * (COLLATION_FILE_NAME for $FILE_NAME). + * @index_block_size: Size of each index block in bytes + * (in $INDEX_ALLOCATION). + * @clusters_per_index_block: + * Clusters per index block (or log2(bytes) + * if < cluster). + * Power of 2; used for encoding block size. + * @reserved: 3 bytes reserved/alignment (zero). + * @index: Index header for root entries (entries follow + * immediately). + * + * NOTE: Always resident. + * + * This is followed by a sequence of index entries (INDEX_ENTRY structures) + * as described by the index header. + * + * When a directory is small enough to fit inside the index root then this + * is the only attribute describing the directory. When the directory is too + * large to fit in the index root, on the other hand, two additional attributes + * are present: an index allocation attribute, containing sub-nodes of the B+ + * directory tree (see below), and a bitmap attribute, describing which virtual + * cluster numbers (vcns) in the index allocation attribute are in use by an + * index block. + * + * NOTE: The root directory (FILE_root) contains an entry for itself. Other + * directories do not contain entries for themselves, though. + */ +struct index_root { + __le32 type; + __le32 collation_rule; + __le32 index_block_size; + u8 clusters_per_index_block; + u8 reserved[3]; + struct index_header index; +} __packed; + +/* + * struct index_block - Index allocation (0xa0). + * + * @magic: Magic value "INDX" (see magic_INDX). + * @usa_ofs: Offset to Update Sequence Array (see ntfs_record). + * @usa_count: Number of USA entries (see ntfs_record). + * @lsn: Log sequence number of last modification. + * @index_block_vcn: VCN of this index block. + * Units: clusters if cluster_size <= index_block_size; + * sectors otherwise. + * @index: Index header describing entries in this block. + * + * When creating the index block, we place the update sequence array at this + * offset, i.e. before we start with the index entries. This also makes sense, + * otherwise we could run into problems with the update sequence array + * containing in itself the last two bytes of a sector which would mean that + * multi sector transfer protection wouldn't work. As you can't protect data + * by overwriting it since you then can't get it back... + * When reading use the data from the ntfs record header. + * + * NOTE: Always non-resident (doesn't make sense to be resident anyway!). + * + * This is an array of index blocks. Each index block starts with an + * index_block structure containing an index header, followed by a sequence of + * index entries (INDEX_ENTRY structures), as described by the struct index_header. + */ +struct index_block { + __le32 magic; + __le16 usa_ofs; + __le16 usa_count; + __le64 lsn; + __le64 index_block_vcn; + struct index_header index; +} __packed; + +static_assert(sizeof(struct index_block) == 40); + +/* + * struct reparse_index_key - Key for $R reparse index in $Extend/$Reparse + * + * @reparse_tag: Reparse point type (including flags, REPARSE_TAG_*). + * @file_id: MFT record number of the file with $REPARSE_POINT + * attribute. + * + * The system file FILE_Extend/$Reparse contains an index named $R listing + * all reparse points on the volume. The index entry keys are as defined + * below. Note, that there is no index data associated with the index entries. + * + * The index entries are sorted by the index key file_id. The collation rule is + * COLLATION_NTOFS_ULONGS. + */ +struct reparse_index_key { + __le32 reparse_tag; + __le64 file_id; +} __packed; + +/* + * enum - Quota entry flags (32-bit) in $Quota/$Q + * + * These flags are stored in quota control entries ($Quota file). + * They control quota tracking, limits, and state. + * + * User quota flags (mask 0x00000007): + * @QUOTA_FLAG_DEFAULT_LIMITS: Use default limits. + * @QUOTA_FLAG_LIMIT_REACHED: Quota limit reached. + * @QUOTA_FLAG_ID_DELETED: Quota ID deleted. + * @QUOTA_FLAG_USER_MASK: Mask for user quota flags (0x00000007). + * + * Default entry flags (owner_id = QUOTA_DEFAULTS_ID): + * @QUOTA_FLAG_TRACKING_ENABLED: Quota tracking enabled. + * @QUOTA_FLAG_ENFORCEMENT_ENABLED: Quota enforcement enabled. + * @QUOTA_FLAG_TRACKING_REQUESTED: Tracking requested (pending). + * @QUOTA_FLAG_LOG_THRESHOLD: Log when threshold reached. + * @QUOTA_FLAG_LOG_LIMIT: Log when limit reached. + * @QUOTA_FLAG_OUT_OF_DATE: Quota data out of date. + * @QUOTA_FLAG_CORRUPT: Quota entry corrupt. + * @QUOTA_FLAG_PENDING_DELETES: Pending quota deletes. + * + */ +enum { + QUOTA_FLAG_DEFAULT_LIMITS = cpu_to_le32(0x00000001), + QUOTA_FLAG_LIMIT_REACHED = cpu_to_le32(0x00000002), + QUOTA_FLAG_ID_DELETED = cpu_to_le32(0x00000004), + + QUOTA_FLAG_USER_MASK = cpu_to_le32(0x00000007), + QUOTA_FLAG_TRACKING_ENABLED = cpu_to_le32(0x00000010), + QUOTA_FLAG_ENFORCEMENT_ENABLED = cpu_to_le32(0x00000020), + QUOTA_FLAG_TRACKING_REQUESTED = cpu_to_le32(0x00000040), + QUOTA_FLAG_LOG_THRESHOLD = cpu_to_le32(0x00000080), + + QUOTA_FLAG_LOG_LIMIT = cpu_to_le32(0x00000100), + QUOTA_FLAG_OUT_OF_DATE = cpu_to_le32(0x00000200), + QUOTA_FLAG_CORRUPT = cpu_to_le32(0x00000400), + QUOTA_FLAG_PENDING_DELETES = cpu_to_le32(0x00000800), +}; + +/* + * struct quota_control_entry - Quota entry in $Quota/$Q + * + * @version: Currently 2. + * @flags: Quota flags (QUOTA_FLAG_* bits). + * @bytes_used: Current quota usage in bytes. + * @change_time: Last modification time (NTFS timestamp). + * @threshold: Soft quota limit (-1 = unlimited). + * @limit: Hard quota limit (-1 = unlimited). + * @exceeded_time: Time soft quota has been exceeded. + * @sid: SID of user/object (zero for defaults entry). + * + * The system file FILE_Extend/$Quota contains two indexes $O and $Q. Quotas + * are on a per volume and per user basis. + * + * The $Q index contains one entry for each existing user_id on the volume. The + * index key is the user_id of the user/group owning this quota control entry, + * i.e. the key is the owner_id. The user_id of the owner of a file, i.e. the + * owner_id, is found in the standard information attribute. The collation rule + * for $Q is COLLATION_NTOFS_ULONG. + * + * The $O index contains one entry for each user/group who has been assigned + * a quota on that volume. The index key holds the SID of the user_id the + * entry belongs to, i.e. the owner_id. The collation rule for $O is + * COLLATION_NTOFS_SID. + * + * The $O index entry data is the user_id of the user corresponding to the SID. + * This user_id is used as an index into $Q to find the quota control entry + * associated with the SID. + * + * The $Q index entry data is the quota control entry and is defined below. + */ +struct quota_control_entry { + __le32 version; + __le32 flags; + __le64 bytes_used; + __le64 change_time; + __le64 threshold; + __le64 limit; + __le64 exceeded_time; + struct ntfs_sid sid; +} __packed; + +/* + * Predefined owner_id values (32-bit). + */ +enum { + QUOTA_INVALID_ID = cpu_to_le32(0x00000000), + QUOTA_DEFAULTS_ID = cpu_to_le32(0x00000001), + QUOTA_FIRST_USER_ID = cpu_to_le32(0x00000100), +}; + +/* + * Current constants for quota control entries. + */ +enum { + /* Current version. */ + QUOTA_VERSION = 2, +}; + +/* + * enum - Index entry flags (16-bit) + * + * These flags are in INDEX_ENTRY.flags (after key data). + * They describe entry type and status in index blocks/root. + * + * @INDEX_ENTRY_NODE: Entry points to a sub-node (index block VCN). + * (Not a leaf entry; internal node reference.) + * i.e. a reference to an index block in form of + * a virtual cluster number + * @INDEX_ENTRY_END: Last entry in index block/root. + * Does not represent a real file; can point to sub-node. + * @INDEX_ENTRY_SPACE_FILLER: + * Dummy value to force enum to 16-bit width. + */ +enum { + INDEX_ENTRY_NODE = cpu_to_le16(1), + INDEX_ENTRY_END = cpu_to_le16(2), + INDEX_ENTRY_SPACE_FILLER = cpu_to_le16(0xffff), +} __packed; + +/* + * struct index_entry_header - Common header for all NTFS index entries + * + * This is the fixed header at the start of every INDEX_ENTRY in index + * blocks or index root. It is followed by the variable key, data, and + * sub-node VCN. + * + * Union @data: + * - When INDEX_ENTRY_END is not set: + * @data.dir.indexed_file: MFT reference of the file described by + * this entry. Used in directory indexes ($I30). + * - When INDEX_ENTRY_END is set or for view indexes: + * @data.vi.data_offset: Byte offset from end of this header to + * entry data. + * @data.vi.data_length: Length of data in bytes. + * @data.vi.reservedV: Reserved (zero). + * + * @length: Total byte size of this index entry + * (multiple of 8 bytes). + * @key_length: Byte size of the key (not multiple of 8 bytes). + * Key follows the header immediately. + * @flags: Bit field of INDEX_ENTRY_* flags (INDEX_ENTRY_NODE, etc.). + * @reserved: Reserved/padding (zero; align to 8 bytes). + */ +struct index_entry_header { + union { + struct { + __le64 indexed_file; + } __packed dir; + struct { + __le16 data_offset; + __le16 data_length; + __le32 reservedV; + } __packed vi; + } __packed data; + __le16 length; + __le16 key_length; + __le16 flags; + __le16 reserved; +} __packed; + +static_assert(sizeof(struct index_entry_header) == 16); + +/* + * struct index_entry - NTFS index entry structure + * + * This is an index entry. A sequence of such entries follows each index_header + * structure. Together they make up a complete index. The index follows either + * an index root attribute or an index allocation attribute. + * + * Union @data (valid when INDEX_ENTRY_END not set): + * @data.dir.indexed_file: MFT ref of file (for directory indexes). + * @data.vi.data_offset: Offset to data after key. + * @data.vi.data_length: Length of data in bytes. + * @data.vi.reservedV: Reserved (zero). + * + * Fields: + * @length: Total byte size of entry (multiple of 8 bytes). + * @key_length: Byte size of key (not multiple of 8). + * @flags: INDEX_ENTRY_* flags (NODE, END, etc.). + * @reserved: Reserved/padding (zero). + * + * Union @key (valid when INDEX_ENTRY_END not set) + * The key of the indexed attribute. NOTE: Only present + * if INDEX_ENTRY_END bit in flags is not set. NOTE: On + * NTFS versions before 3.0 the only valid key is the + * struct file_name_attr. On NTFS 3.0+ the following + * additional index keys are defined: + * @key.file_name: $FILE_NAME attr (for $I30 directory indexes). + * @key.sii: $SII key (for $Secure $SII index). + * @key.sdh: $SDH key (for $Secure $SDH index). + * @key.object_id: GUID (for $ObjId $O index). + * @key.reparse: Reparse tag + file ID (for $Reparse $R). + * @key.sid: SID (for $Quota $O index). + * @key.owner_id: User ID (for $Quota $Q index). + * + * The (optional) index data is inserted here when creating. + * __le64 vcn; If INDEX_ENTRY_NODE bit in flags is set, the last + * eight bytes of this index entry contain the virtual + * cluster number of the index block that holds the + * entries immediately preceding the current entry (the + * vcn references the corresponding cluster in the data + * of the non-resident index allocation attribute). If + * the key_length is zero, then the vcn immediately + * follows the INDEX_ENTRY_HEADER. Regardless of + * key_length, the address of the 8-byte boundary + * aligned vcn of INDEX_ENTRY{_HEADER} *ie is given by + * (char*)ie + le16_to_cpu(ie*)->length) - sizeof(VCN), + * where sizeof(VCN) can be hardcoded as 8 if wanted. + * + * NOTE: Before NTFS 3.0 only filename attributes were indexed. + */ +struct index_entry { + union { + struct { + __le64 indexed_file; + } __packed dir; + struct { + __le16 data_offset; + __le16 data_length; + __le32 reservedV; + } __packed vi; + } __packed data; + __le16 length; + __le16 key_length; + __le16 flags; + __le16 reserved; + union { + struct file_name_attr file_name; + struct sii_index_key sii; + struct sdh_index_key sdh; + struct guid object_id; + struct reparse_index_key reparse; + struct ntfs_sid sid; + __le32 owner_id; + } __packed key; +} __packed; + +/* + * The reparse point tag defines the type of the reparse point. It also + * includes several flags, which further describe the reparse point. + * + * The reparse point tag is an unsigned 32-bit value divided in three parts: + * + * 1. The least significant 16 bits (i.e. bits 0 to 15) specify the type of + * the reparse point. + * 2. The 12 bits after this (i.e. bits 16 to 27) are reserved for future use. + * 3. The most significant four bits are flags describing the reparse point. + * They are defined as follows: + * bit 28: Directory bit. If set, the directory is not a surrogate + * and can be used the usual way. + * bit 29: Name surrogate bit. If set, the filename is an alias for + * another object in the system. + * bit 30: High-latency bit. If set, accessing the first byte of data will + * be slow. (E.g. the data is stored on a tape drive.) + * bit 31: Microsoft bit. If set, the tag is owned by Microsoft. User + * defined tags have to use zero here. + * 4. Moreover, on Windows 10 : + * Some flags may be used in bits 12 to 15 to further describe the + * reparse point. + */ +enum { + IO_REPARSE_TAG_DIRECTORY = cpu_to_le32(0x10000000), + IO_REPARSE_TAG_IS_ALIAS = cpu_to_le32(0x20000000), + IO_REPARSE_TAG_IS_HIGH_LATENCY = cpu_to_le32(0x40000000), + IO_REPARSE_TAG_IS_MICROSOFT = cpu_to_le32(0x80000000), + + IO_REPARSE_TAG_RESERVED_ZERO = cpu_to_le32(0x00000000), + IO_REPARSE_TAG_RESERVED_ONE = cpu_to_le32(0x00000001), + IO_REPARSE_TAG_RESERVED_RANGE = cpu_to_le32(0x00000001), + + IO_REPARSE_TAG_CSV = cpu_to_le32(0x80000009), + IO_REPARSE_TAG_DEDUP = cpu_to_le32(0x80000013), + IO_REPARSE_TAG_DFS = cpu_to_le32(0x8000000A), + IO_REPARSE_TAG_DFSR = cpu_to_le32(0x80000012), + IO_REPARSE_TAG_HSM = cpu_to_le32(0xC0000004), + IO_REPARSE_TAG_HSM2 = cpu_to_le32(0x80000006), + IO_REPARSE_TAG_MOUNT_POINT = cpu_to_le32(0xA0000003), + IO_REPARSE_TAG_NFS = cpu_to_le32(0x80000014), + IO_REPARSE_TAG_SIS = cpu_to_le32(0x80000007), + IO_REPARSE_TAG_SYMLINK = cpu_to_le32(0xA000000C), + IO_REPARSE_TAG_WIM = cpu_to_le32(0x80000008), + IO_REPARSE_TAG_DFM = cpu_to_le32(0x80000016), + IO_REPARSE_TAG_WOF = cpu_to_le32(0x80000017), + IO_REPARSE_TAG_WCI = cpu_to_le32(0x80000018), + IO_REPARSE_TAG_CLOUD = cpu_to_le32(0x9000001A), + IO_REPARSE_TAG_APPEXECLINK = cpu_to_le32(0x8000001B), + IO_REPARSE_TAG_GVFS = cpu_to_le32(0x9000001C), + IO_REPARSE_TAG_LX_SYMLINK = cpu_to_le32(0xA000001D), + IO_REPARSE_TAG_AF_UNIX = cpu_to_le32(0x80000023), + IO_REPARSE_TAG_LX_FIFO = cpu_to_le32(0x80000024), + IO_REPARSE_TAG_LX_CHR = cpu_to_le32(0x80000025), + IO_REPARSE_TAG_LX_BLK = cpu_to_le32(0x80000026), + + IO_REPARSE_TAG_VALID_VALUES = cpu_to_le32(0xf000ffff), + IO_REPARSE_PLUGIN_SELECT = cpu_to_le32(0xffff0fff), +}; + +#define SYMLINK_FLAG_RELATIVE 1 + +/* + * struct reparse_point - $REPARSE_POINT attribute content (0xc0)\ + * + * @reparse_tag: Reparse point type (with flags; REPARSE_TAG_*). + * @reparse_data_length: Byte size of @reparse_data. + * @reserved: Reserved/padding (zero; 8-byte alignment). + * @reparse_data: Variable reparse data (meaning depends on @reparse_tag). + * - Symbolic link/junction: struct reparse_symlink + * - Mount point: similar symlink structure + * - Other tags: vendor-specific or extended data + * + * NOTE: Can be resident or non-resident. + */ +struct reparse_point { + __le32 reparse_tag; + __le16 reparse_data_length; + __le16 reserved; + u8 reparse_data[]; +} __packed; + +struct mount_point_reparse_data { + __le16 substitute_name_offset; + __le16 substitute_name_length; + __le16 print_name_offset; + __le16 print_name_length; + __le16 path_buffer[]; +} __packed; + +struct symlink_reparse_data { + __le16 substitute_name_offset; + __le16 substitute_name_length; + __le16 print_name_offset; + __le16 print_name_length; + __le32 flags; + __le16 path_buffer[]; +} __packed; + +/* + * struct ea_information - $EA_INFORMATION attribute content (0xd0) + * + * @ea_length: Byte size of packed EAs. + * @need_ea_count: Number of EAs with NEED_EA bit set. + * @ea_query_length: Byte size needed to unpack/query EAs via ZwQueryEaFile(). + * (Unpacked format size.) + * + * NOTE: Always resident. (Is this true???) + */ +struct ea_information { + __le16 ea_length; + __le16 need_ea_count; + __le32 ea_query_length; +} __packed; + +/* + * enum - Extended attribute flags (8-bit) + * + * These flags are stored in the EA header of each extended attribute + * (in $EA attribute, type 0xe0). + * + * @NEED_EA: If set, the file cannot be properly interpreted + * without understanding its associated EAs. + * (Critical EA; applications must process it.) + */ +enum { + NEED_EA = 0x80 +} __packed; + +/* + * struct ea_attr - Extended attribute (EA) entry (0xe0) + * + * @next_entry_offset: Byte offset to the next EA_ATTR entry. + * (From start of current entry.) + * @flags: EA flags (NEED_EA = 0x80 if critical). + * @ea_name_length: Length of @ea_name in bytes (excluding '\0'). + * @ea_value_length: Byte size of the EA value. + * @ea_name: ASCII name of the EA (zero-terminated). + * Value immediately follows the name. + * u8 ea_value[]; The value of the EA. Immediately follows the name. + * + * This is one variable-length record in the $EA attribute value. + * The attribute can be resident or non-resident. + * Sequence of these entries forms the packed EA list. + * + * NOTE: Can be resident or non-resident. + */ +struct ea_attr { + __le32 next_entry_offset; + u8 flags; + u8 ea_name_length; + __le16 ea_value_length; + u8 ea_name[]; +} __packed; + +#endif /* _LINUX_NTFS_LAYOUT_H */ diff --git a/fs/ntfs/lcnalloc.c b/fs/ntfs/lcnalloc.c new file mode 100644 index 000000000..835a04102 --- /dev/null +++ b/fs/ntfs/lcnalloc.c @@ -0,0 +1,1049 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Cluster (de)allocation code. + * + * Copyright (c) 2004-2005 Anton Altaparmakov + * Copyright (c) 2025 LG Electronics Co., Ltd. + * + * Part of this file is based on code from the NTFS-3G. + * and is copyrighted by the respective authors below: + * Copyright (c) 2002-2004 Anton Altaparmakov + * Copyright (c) 2004 Yura Pakhuchiy + * Copyright (c) 2004-2008 Szabolcs Szakacsits + * Copyright (c) 2008-2009 Jean-Pierre Andre + */ + +#include + +#include "lcnalloc.h" +#include "bitmap.h" +#include "ntfs.h" + +/* + * ntfs_cluster_free_from_rl_nolock - free clusters from runlist + * @vol: mounted ntfs volume on which to free the clusters + * @rl: runlist describing the clusters to free + * + * Free all the clusters described by the runlist @rl on the volume @vol. In + * the case of an error being returned, at least some of the clusters were not + * freed. + * + * Return 0 on success and -errno on error. + * + * Locking: - The volume lcn bitmap must be locked for writing on entry and is + * left locked on return. + */ +int ntfs_cluster_free_from_rl_nolock(struct ntfs_volume *vol, + const struct runlist_element *rl) +{ + struct inode *lcnbmp_vi = vol->lcnbmp_ino; + int ret = 0; + s64 nr_freed = 0; + + ntfs_debug("Entering."); + if (!rl) + return 0; + + if (!NVolFreeClusterKnown(vol)) + wait_event(vol->free_waitq, NVolFreeClusterKnown(vol)); + + for (; rl->length; rl++) { + int err; + + if (rl->lcn < 0) + continue; + err = ntfs_bitmap_clear_run(lcnbmp_vi, rl->lcn, rl->length); + if (unlikely(err && (!ret || ret == -ENOMEM) && ret != err)) + ret = err; + else + nr_freed += rl->length; + } + ntfs_inc_free_clusters(vol, nr_freed); + ntfs_debug("Done."); + return ret; +} + +static s64 max_empty_bit_range(unsigned char *buf, int size) +{ + int i, j, run = 0; + int max_range = 0; + s64 start_pos = -1; + + ntfs_debug("Entering\n"); + + i = 0; + while (i < size) { + switch (*buf) { + case 0: + do { + buf++; + run += 8; + i++; + } while ((i < size) && !*buf); + break; + case 255: + if (run > max_range) { + max_range = run; + start_pos = (s64)i * 8 - run; + } + run = 0; + do { + buf++; + i++; + } while ((i < size) && (*buf == 255)); + break; + default: + for (j = 0; j < 8; j++) { + int bit = *buf & (1 << j); + + if (bit) { + if (run > max_range) { + max_range = run; + start_pos = (s64)i * 8 + (j - run); + } + run = 0; + } else + run++; + } + i++; + buf++; + } + } + + if (run > max_range) + start_pos = (s64)i * 8 - run; + + return start_pos; +} + +/* + * ntfs_cluster_alloc - allocate clusters on an ntfs volume + * @vol: mounted ntfs volume on which to allocate clusters + * @start_vcn: vcn of the first allocated cluster + * @count: number of clusters to allocate + * @start_lcn: starting lcn at which to allocate the clusters or -1 if none + * @zone: zone from which to allocate (MFT_ZONE or DATA_ZONE) + * @is_extension: if true, the caller is extending an attribute + * @is_contig: if true, require contiguous allocation + * @is_dealloc: if true, the allocation is for deallocation purposes + * + * Allocate @count clusters preferably starting at cluster @start_lcn or at the + * current allocator position if @start_lcn is -1, on the mounted ntfs volume + * @vol. @zone is either DATA_ZONE for allocation of normal clusters or + * MFT_ZONE for allocation of clusters for the master file table, i.e. the + * $MFT/$DATA attribute. + * + * @start_vcn specifies the vcn of the first allocated cluster. This makes + * merging the resulting runlist with the old runlist easier. + * + * If @is_extension is 'true', the caller is allocating clusters to extend an + * attribute and if it is 'false', the caller is allocating clusters to fill a + * hole in an attribute. Practically the difference is that if @is_extension + * is 'true' the returned runlist will be terminated with LCN_ENOENT and if + * @is_extension is 'false' the runlist will be terminated with + * LCN_RL_NOT_MAPPED. + * + * You need to check the return value with IS_ERR(). If this is false, the + * function was successful and the return value is a runlist describing the + * allocated cluster(s). If IS_ERR() is true, the function failed and + * PTR_ERR() gives you the error code. + * + * Notes on the allocation algorithm + * ================================= + * + * There are two data zones. First is the area between the end of the mft zone + * and the end of the volume, and second is the area between the start of the + * volume and the start of the mft zone. On unmodified/standard NTFS 1.x + * volumes, the second data zone does not exist due to the mft zone being + * expanded to cover the start of the volume in order to reserve space for the + * mft bitmap attribute. + * + * This is not the prettiest function but the complexity stems from the need of + * implementing the mft vs data zoned approach and from the fact that we have + * access to the lcn bitmap in portions of up to 8192 bytes at a time, so we + * need to cope with crossing over boundaries of two buffers. Further, the + * fact that the allocator allows for caller supplied hints as to the location + * of where allocation should begin and the fact that the allocator keeps track + * of where in the data zones the next natural allocation should occur, + * contribute to the complexity of the function. But it should all be + * worthwhile, because this allocator should: 1) be a full implementation of + * the MFT zone approach used by Windows NT, 2) cause reduction in + * fragmentation, and 3) be speedy in allocations (the code is not optimized + * for speed, but the algorithm is, so further speed improvements are probably + * possible). + * + * Locking: - The volume lcn bitmap must be unlocked on entry and is unlocked + * on return. + * - This function takes the volume lcn bitmap lock for writing and + * modifies the bitmap contents. + * + * Return: Runlist describing the allocated cluster(s) on success, error pointer + * on failure. + */ +struct runlist_element *ntfs_cluster_alloc(struct ntfs_volume *vol, const s64 start_vcn, + const s64 count, const s64 start_lcn, + const int zone, + const bool is_extension, + const bool is_contig, + const bool is_dealloc) +{ + s64 zone_start, zone_end, bmp_pos, bmp_initial_pos, last_read_pos, lcn; + s64 prev_lcn = 0, prev_run_len = 0, mft_zone_size; + s64 clusters, free_clusters; + loff_t i_size; + struct inode *lcnbmp_vi; + struct runlist_element *rl = NULL; + struct address_space *mapping; + struct folio *folio = NULL; + u8 *buf = NULL, *byte; + int err = 0, rlpos, rlsize, buf_size, pg_off; + u8 pass, done_zones, search_zone, need_writeback = 0, bit; + unsigned int memalloc_flags; + u8 has_guess, used_zone_pos; + pgoff_t index; + + ntfs_debug("Entering for start_vcn 0x%llx, count 0x%llx, start_lcn 0x%llx, zone %s_ZONE.", + start_vcn, count, start_lcn, + zone == MFT_ZONE ? "MFT" : "DATA"); + + lcnbmp_vi = vol->lcnbmp_ino; + if (start_vcn < 0 || start_lcn < LCN_HOLE || + zone < FIRST_ZONE || zone > LAST_ZONE) + return ERR_PTR(-EINVAL); + + /* Return NULL if @count is zero. */ + if (count < 0 || !count) + return ERR_PTR(-EINVAL); + + memalloc_flags = memalloc_nofs_save(); + + if (!NVolFreeClusterKnown(vol)) + wait_event(vol->free_waitq, NVolFreeClusterKnown(vol)); + free_clusters = atomic64_read(&vol->free_clusters); + + /* Take the lcnbmp lock for writing. */ + down_write(&vol->lcnbmp_lock); + if (is_dealloc == false) + free_clusters -= atomic64_read(&vol->dirty_clusters); + + if (free_clusters < count) { + err = -ENOSPC; + goto out_restore; + } + + /* + * If no specific @start_lcn was requested, use the current data zone + * position, otherwise use the requested @start_lcn but make sure it + * lies outside the mft zone. Also set done_zones to 0 (no zones done) + * and pass depending on whether we are starting inside a zone (1) or + * at the beginning of a zone (2). If requesting from the MFT_ZONE, + * we either start at the current position within the mft zone or at + * the specified position. If the latter is out of bounds then we start + * at the beginning of the MFT_ZONE. + */ + done_zones = 0; + pass = 1; + /* + * zone_start and zone_end are the current search range. search_zone + * is 1 for mft zone, 2 for data zone 1 (end of mft zone till end of + * volume) and 4 for data zone 2 (start of volume till start of mft + * zone). + */ + has_guess = 1; + zone_start = start_lcn; + + if (zone_start < 0) { + if (zone == DATA_ZONE) + zone_start = vol->data1_zone_pos; + else + zone_start = vol->mft_zone_pos; + if (!zone_start) { + /* + * Zone starts at beginning of volume which means a + * single pass is sufficient. + */ + pass = 2; + } + has_guess = 0; + } + + used_zone_pos = has_guess ? 0 : 1; + + if (!zone_start || zone_start == vol->mft_zone_start || + zone_start == vol->mft_zone_end) + pass = 2; + + if (zone_start < vol->mft_zone_start) { + zone_end = vol->mft_zone_start; + search_zone = 4; + /* Skip searching the mft zone. */ + done_zones |= 1; + } else if (zone_start < vol->mft_zone_end) { + zone_end = vol->mft_zone_end; + search_zone = 1; + } else { + zone_end = vol->nr_clusters; + search_zone = 2; + /* Skip searching the mft zone. */ + done_zones |= 1; + } + + /* + * bmp_pos is the current bit position inside the bitmap. We use + * bmp_initial_pos to determine whether or not to do a zone switch. + */ + bmp_pos = bmp_initial_pos = zone_start; + + /* Loop until all clusters are allocated, i.e. clusters == 0. */ + clusters = count; + rlpos = rlsize = 0; + mapping = lcnbmp_vi->i_mapping; + i_size = i_size_read(lcnbmp_vi); + while (1) { + ntfs_debug("Start of outer while loop: done_zones 0x%x, search_zone %i, pass %i, zone_start 0x%llx, zone_end 0x%llx, bmp_initial_pos 0x%llx, bmp_pos 0x%llx, rlpos %i, rlsize %i.", + done_zones, search_zone, pass, + zone_start, zone_end, bmp_initial_pos, + bmp_pos, rlpos, rlsize); + /* Loop until we run out of free clusters. */ + last_read_pos = bmp_pos >> 3; + ntfs_debug("last_read_pos 0x%llx.", last_read_pos); + if (last_read_pos >= i_size) { + ntfs_debug("End of attribute reached. Skipping to zone_pass_done."); + goto zone_pass_done; + } + if (likely(folio)) { + if (need_writeback) { + ntfs_debug("Marking page dirty."); + folio_mark_dirty(folio); + need_writeback = 0; + } + folio_unlock(folio); + kunmap_local(buf); + folio_put(folio); + folio = NULL; + } + + index = last_read_pos >> PAGE_SHIFT; + pg_off = last_read_pos & ~PAGE_MASK; + buf_size = PAGE_SIZE - pg_off; + if (unlikely(last_read_pos + buf_size > i_size)) + buf_size = i_size - last_read_pos; + buf_size <<= 3; + lcn = bmp_pos & 7; + bmp_pos &= ~(s64)7; + + if (vol->lcn_empty_bits_per_page[index] == 0) + goto next_bmp_pos; + + folio = read_mapping_folio(mapping, index, NULL); + if (IS_ERR(folio)) { + err = PTR_ERR(folio); + ntfs_error(vol->sb, "Failed to map page."); + goto out; + } + + folio_lock(folio); + buf = kmap_local_folio(folio, 0) + pg_off; + ntfs_debug("Before inner while loop: buf_size %i, lcn 0x%llx, bmp_pos 0x%llx, need_writeback %i.", + buf_size, lcn, bmp_pos, need_writeback); + while (lcn < buf_size && lcn + bmp_pos < zone_end) { + byte = buf + (lcn >> 3); + ntfs_debug("In inner while loop: buf_size %i, lcn 0x%llx, bmp_pos 0x%llx, need_writeback %i, byte ofs 0x%x, *byte 0x%x.", + buf_size, lcn, bmp_pos, need_writeback, + (unsigned int)(lcn >> 3), + (unsigned int)*byte); + bit = 1 << (lcn & 7); + ntfs_debug("bit 0x%x.", bit); + + if (has_guess) { + if (*byte & bit) { + if (is_contig == true && prev_run_len > 0) + goto done; + + has_guess = 0; + break; + } + } else { + lcn = max_empty_bit_range(buf, buf_size >> 3); + if (lcn < 0) + break; + has_guess = 1; + continue; + } + /* + * Allocate more memory if needed, including space for + * the terminator element. + * kvzalloc() operates on whole pages only. + */ + if ((rlpos + 2) * sizeof(*rl) > rlsize) { + struct runlist_element *rl2; + + ntfs_debug("Reallocating memory."); + if (!rl) + ntfs_debug("First free bit is at s64 0x%llx.", + lcn + bmp_pos); + rl2 = kvzalloc(rlsize + PAGE_SIZE, GFP_NOFS); + if (unlikely(!rl2)) { + err = -ENOMEM; + ntfs_error(vol->sb, "Failed to allocate memory."); + goto out; + } + memcpy(rl2, rl, rlsize); + kvfree(rl); + rl = rl2; + rlsize += PAGE_SIZE; + ntfs_debug("Reallocated memory, rlsize 0x%x.", + rlsize); + } + /* Allocate the bitmap bit. */ + *byte |= bit; + /* We need to write this bitmap page to disk. */ + need_writeback = 1; + ntfs_debug("*byte 0x%x, need_writeback is set.", + (unsigned int)*byte); + ntfs_dec_free_clusters(vol, 1); + ntfs_set_lcn_empty_bits(vol, index, 1, 1); + + /* + * Coalesce with previous run if adjacent LCNs. + * Otherwise, append a new run. + */ + ntfs_debug("Adding run (lcn 0x%llx, len 0x%llx), prev_lcn 0x%llx, lcn 0x%llx, bmp_pos 0x%llx, prev_run_len 0x%llx, rlpos %i.", + lcn + bmp_pos, 1ULL, prev_lcn, + lcn, bmp_pos, prev_run_len, rlpos); + if (prev_lcn == lcn + bmp_pos - prev_run_len && rlpos) { + ntfs_debug("Coalescing to run (lcn 0x%llx, len 0x%llx).", + rl[rlpos - 1].lcn, + rl[rlpos - 1].length); + rl[rlpos - 1].length = ++prev_run_len; + ntfs_debug("Run now (lcn 0x%llx, len 0x%llx), prev_run_len 0x%llx.", + rl[rlpos - 1].lcn, + rl[rlpos - 1].length, + prev_run_len); + } else { + if (likely(rlpos)) { + ntfs_debug("Adding new run, (previous run lcn 0x%llx, len 0x%llx).", + rl[rlpos - 1].lcn, rl[rlpos - 1].length); + rl[rlpos].vcn = rl[rlpos - 1].vcn + + prev_run_len; + } else { + ntfs_debug("Adding new run, is first run."); + rl[rlpos].vcn = start_vcn; + } + rl[rlpos].lcn = prev_lcn = lcn + bmp_pos; + rl[rlpos].length = prev_run_len = 1; + rlpos++; + } + /* Done? */ + if (!--clusters) { + s64 tc; +done: + if (!used_zone_pos) + goto out; + /* + * Update the current zone position. Positions + * of already scanned zones have been updated + * during the respective zone switches. + */ + tc = lcn + bmp_pos + 1; + ntfs_debug("Done. Updating current zone position, tc 0x%llx, search_zone %i.", + tc, search_zone); + switch (search_zone) { + case 1: + ntfs_debug("Before checks, vol->mft_zone_pos 0x%llx.", + vol->mft_zone_pos); + if (tc >= vol->mft_zone_end) { + vol->mft_zone_pos = + vol->mft_lcn; + if (!vol->mft_zone_end) + vol->mft_zone_pos = 0; + } else if ((bmp_initial_pos >= + vol->mft_zone_pos || + tc > vol->mft_zone_pos) + && tc >= vol->mft_lcn) + vol->mft_zone_pos = tc; + ntfs_debug("After checks, vol->mft_zone_pos 0x%llx.", + vol->mft_zone_pos); + break; + case 2: + ntfs_debug("Before checks, vol->data1_zone_pos 0x%llx.", + vol->data1_zone_pos); + if (tc >= vol->nr_clusters) + vol->data1_zone_pos = + vol->mft_zone_end; + else if ((bmp_initial_pos >= + vol->data1_zone_pos || + tc > vol->data1_zone_pos) + && tc >= vol->mft_zone_end) + vol->data1_zone_pos = tc; + ntfs_debug("After checks, vol->data1_zone_pos 0x%llx.", + vol->data1_zone_pos); + break; + case 4: + ntfs_debug("Before checks, vol->data2_zone_pos 0x%llx.", + vol->data2_zone_pos); + if (tc >= vol->mft_zone_start) + vol->data2_zone_pos = 0; + else if (bmp_initial_pos >= + vol->data2_zone_pos || + tc > vol->data2_zone_pos) + vol->data2_zone_pos = tc; + ntfs_debug("After checks, vol->data2_zone_pos 0x%llx.", + vol->data2_zone_pos); + break; + default: + WARN_ON(1); + } + ntfs_debug("Finished. Going to out."); + goto out; + } + lcn++; + } + + if (!used_zone_pos) { + used_zone_pos = 1; + if (search_zone == 1) + zone_start = vol->mft_zone_pos; + else if (search_zone == 2) + zone_start = vol->data1_zone_pos; + else + zone_start = vol->data2_zone_pos; + + if (!zone_start || zone_start == vol->mft_zone_start || + zone_start == vol->mft_zone_end) + pass = 2; + bmp_pos = zone_start; + } else { +next_bmp_pos: + bmp_pos += buf_size; + } + + ntfs_debug("After inner while loop: buf_size 0x%x, lcn 0x%llx, bmp_pos 0x%llx, need_writeback %i.", + buf_size, lcn, bmp_pos, need_writeback); + if (bmp_pos < zone_end) { + ntfs_debug("Continuing outer while loop, bmp_pos 0x%llx, zone_end 0x%llx.", + bmp_pos, zone_end); + continue; + } +zone_pass_done: /* Finished with the current zone pass. */ + ntfs_debug("At zone_pass_done, pass %i.", pass); + if (pass == 1) { + /* + * Now do pass 2, scanning the first part of the zone + * we omitted in pass 1. + */ + pass = 2; + zone_end = zone_start; + switch (search_zone) { + case 1: /* mft_zone */ + zone_start = vol->mft_zone_start; + break; + case 2: /* data1_zone */ + zone_start = vol->mft_zone_end; + break; + case 4: /* data2_zone */ + zone_start = 0; + break; + default: + WARN_ON(1); + } + /* Sanity check. */ + if (zone_end < zone_start) + zone_end = zone_start; + bmp_pos = zone_start; + ntfs_debug("Continuing outer while loop, pass 2, zone_start 0x%llx, zone_end 0x%llx, bmp_pos 0x%llx.", + zone_start, zone_end, bmp_pos); + continue; + } /* pass == 2 */ +done_zones_check: + ntfs_debug("At done_zones_check, search_zone %i, done_zones before 0x%x, done_zones after 0x%x.", + search_zone, done_zones, + done_zones | search_zone); + done_zones |= search_zone; + if (done_zones < 7) { + ntfs_debug("Switching zone."); + /* Now switch to the next zone we haven't done yet. */ + pass = 1; + switch (search_zone) { + case 1: + ntfs_debug("Switching from mft zone to data1 zone."); + /* Update mft zone position. */ + if (rlpos && used_zone_pos) { + s64 tc; + + ntfs_debug("Before checks, vol->mft_zone_pos 0x%llx.", + vol->mft_zone_pos); + tc = rl[rlpos - 1].lcn + + rl[rlpos - 1].length; + if (tc >= vol->mft_zone_end) { + vol->mft_zone_pos = + vol->mft_lcn; + if (!vol->mft_zone_end) + vol->mft_zone_pos = 0; + } else if ((bmp_initial_pos >= + vol->mft_zone_pos || + tc > vol->mft_zone_pos) + && tc >= vol->mft_lcn) + vol->mft_zone_pos = tc; + ntfs_debug("After checks, vol->mft_zone_pos 0x%llx.", + vol->mft_zone_pos); + } + /* Switch from mft zone to data1 zone. */ +switch_to_data1_zone: search_zone = 2; + zone_start = bmp_initial_pos = + vol->data1_zone_pos; + zone_end = vol->nr_clusters; + if (zone_start == vol->mft_zone_end) + pass = 2; + if (zone_start >= zone_end) { + vol->data1_zone_pos = zone_start = + vol->mft_zone_end; + pass = 2; + } + break; + case 2: + ntfs_debug("Switching from data1 zone to data2 zone."); + /* Update data1 zone position. */ + if (rlpos && used_zone_pos) { + s64 tc; + + ntfs_debug("Before checks, vol->data1_zone_pos 0x%llx.", + vol->data1_zone_pos); + tc = rl[rlpos - 1].lcn + + rl[rlpos - 1].length; + if (tc >= vol->nr_clusters) + vol->data1_zone_pos = + vol->mft_zone_end; + else if ((bmp_initial_pos >= + vol->data1_zone_pos || + tc > vol->data1_zone_pos) + && tc >= vol->mft_zone_end) + vol->data1_zone_pos = tc; + ntfs_debug("After checks, vol->data1_zone_pos 0x%llx.", + vol->data1_zone_pos); + } + /* Switch from data1 zone to data2 zone. */ + search_zone = 4; + zone_start = bmp_initial_pos = + vol->data2_zone_pos; + zone_end = vol->mft_zone_start; + if (!zone_start) + pass = 2; + if (zone_start >= zone_end) { + vol->data2_zone_pos = zone_start = + bmp_initial_pos = 0; + pass = 2; + } + break; + case 4: + ntfs_debug("Switching from data2 zone to data1 zone."); + /* Update data2 zone position. */ + if (rlpos && used_zone_pos) { + s64 tc; + + ntfs_debug("Before checks, vol->data2_zone_pos 0x%llx.", + vol->data2_zone_pos); + tc = rl[rlpos - 1].lcn + + rl[rlpos - 1].length; + if (tc >= vol->mft_zone_start) + vol->data2_zone_pos = 0; + else if (bmp_initial_pos >= + vol->data2_zone_pos || + tc > vol->data2_zone_pos) + vol->data2_zone_pos = tc; + ntfs_debug("After checks, vol->data2_zone_pos 0x%llx.", + vol->data2_zone_pos); + } + /* Switch from data2 zone to data1 zone. */ + goto switch_to_data1_zone; + default: + WARN_ON(1); + } + ntfs_debug("After zone switch, search_zone %i, pass %i, bmp_initial_pos 0x%llx, zone_start 0x%llx, zone_end 0x%llx.", + search_zone, pass, + bmp_initial_pos, + zone_start, + zone_end); + bmp_pos = zone_start; + if (zone_start == zone_end) { + ntfs_debug("Empty zone, going to done_zones_check."); + /* Empty zone. Don't bother searching it. */ + goto done_zones_check; + } + ntfs_debug("Continuing outer while loop."); + continue; + } /* done_zones == 7 */ + ntfs_debug("All zones are finished."); + /* + * All zones are finished! If DATA_ZONE, shrink mft zone. If + * MFT_ZONE, we have really run out of space. + */ + mft_zone_size = vol->mft_zone_end - vol->mft_zone_start; + ntfs_debug("vol->mft_zone_start 0x%llx, vol->mft_zone_end 0x%llx, mft_zone_size 0x%llx.", + vol->mft_zone_start, vol->mft_zone_end, + mft_zone_size); + if (zone == MFT_ZONE || mft_zone_size <= 0) { + ntfs_debug("No free clusters left, going to out."); + /* Really no more space left on device. */ + err = -ENOSPC; + goto out; + } /* zone == DATA_ZONE && mft_zone_size > 0 */ + ntfs_debug("Shrinking mft zone."); + zone_end = vol->mft_zone_end; + mft_zone_size >>= 1; + if (mft_zone_size > 0) + vol->mft_zone_end = vol->mft_zone_start + mft_zone_size; + else /* mft zone and data2 zone no longer exist. */ + vol->data2_zone_pos = vol->mft_zone_start = + vol->mft_zone_end = 0; + if (vol->mft_zone_pos >= vol->mft_zone_end) { + vol->mft_zone_pos = vol->mft_lcn; + if (!vol->mft_zone_end) + vol->mft_zone_pos = 0; + } + bmp_pos = zone_start = bmp_initial_pos = + vol->data1_zone_pos = vol->mft_zone_end; + search_zone = 2; + pass = 2; + done_zones &= ~2; + ntfs_debug("After shrinking mft zone, mft_zone_size 0x%llx, vol->mft_zone_start 0x%llx, vol->mft_zone_end 0x%llx, vol->mft_zone_pos 0x%llx, search_zone 2, pass 2, dones_zones 0x%x, zone_start 0x%llx, zone_end 0x%llx, vol->data1_zone_pos 0x%llx, continuing outer while loop.", + mft_zone_size, vol->mft_zone_start, + vol->mft_zone_end, vol->mft_zone_pos, + done_zones, zone_start, zone_end, + vol->data1_zone_pos); + } + ntfs_debug("After outer while loop."); +out: + ntfs_debug("At out."); + /* Add runlist terminator element. */ + if (likely(rl)) { + rl[rlpos].vcn = rl[rlpos - 1].vcn + rl[rlpos - 1].length; + rl[rlpos].lcn = is_extension ? LCN_ENOENT : LCN_RL_NOT_MAPPED; + rl[rlpos].length = 0; + } + if (!IS_ERR_OR_NULL(folio)) { + if (need_writeback) { + ntfs_debug("Marking page dirty."); + folio_mark_dirty(folio); + need_writeback = 0; + } + folio_unlock(folio); + kunmap_local(buf); + folio_put(folio); + } + if (likely(!err)) { + if (!rl) { + err = -EIO; + goto out_restore; + } + if (is_dealloc == true) + ntfs_release_dirty_clusters(vol, rl->length); + ntfs_debug("Done."); + goto out_restore; + } + if (err != -ENOSPC) + ntfs_error(vol->sb, + "Failed to allocate clusters, aborting (error %i).", + err); + if (rl) { + int err2; + + if (err == -ENOSPC) + ntfs_debug("Not enough space to complete allocation, err -ENOSPC, first free lcn 0x%llx, could allocate up to 0x%llx clusters.", + rl[0].lcn, count - clusters); + /* Deallocate all allocated clusters. */ + ntfs_debug("Attempting rollback..."); + err2 = ntfs_cluster_free_from_rl_nolock(vol, rl); + if (err2) { + ntfs_error(vol->sb, + "Failed to rollback (error %i). Leaving inconsistent metadata! Unmount and run chkdsk.", + err2); + NVolSetErrors(vol); + } + /* Free the runlist. */ + kvfree(rl); + } else if (err == -ENOSPC) + ntfs_debug("No space left at all, err = -ENOSPC, first free lcn = 0x%llx.", + vol->data1_zone_pos); + atomic64_set(&vol->dirty_clusters, 0); + +out_restore: + up_write(&vol->lcnbmp_lock); + memalloc_nofs_restore(memalloc_flags); + + return err < 0 ? ERR_PTR(err) : rl; +} + +/* + * __ntfs_cluster_free - free clusters on an ntfs volume + * @ni: ntfs inode whose runlist describes the clusters to free + * @start_vcn: vcn in the runlist of @ni at which to start freeing clusters + * @count: number of clusters to free or -1 for all clusters + * @ctx: active attribute search context if present or NULL if not + * @is_rollback: true if this is a rollback operation + * + * Free @count clusters starting at the cluster @start_vcn in the runlist + * described by the vfs inode @ni. + * + * If @count is -1, all clusters from @start_vcn to the end of the runlist are + * deallocated. Thus, to completely free all clusters in a runlist, use + * @start_vcn = 0 and @count = -1. + * + * If @ctx is specified, it is an active search context of @ni and its base mft + * record. This is needed when __ntfs_cluster_free() encounters unmapped + * runlist fragments and allows their mapping. If you do not have the mft + * record mapped, you can specify @ctx as NULL and __ntfs_cluster_free() will + * perform the necessary mapping and unmapping. + * + * Note, __ntfs_cluster_free() saves the state of @ctx on entry and restores it + * before returning. Thus, @ctx will be left pointing to the same attribute on + * return as on entry. However, the actual pointers in @ctx may point to + * different memory locations on return, so you must remember to reset any + * cached pointers from the @ctx, i.e. after the call to __ntfs_cluster_free(), + * you will probably want to do: + * m = ctx->mrec; + * a = ctx->attr; + * Assuming you cache ctx->attr in a variable @a of type attr_record * and that + * you cache ctx->mrec in a variable @m of type struct mft_record *. + * + * @is_rollback should always be 'false', it is for internal use to rollback + * errors. You probably want to use ntfs_cluster_free() instead. + * + * Note, __ntfs_cluster_free() does not modify the runlist, so you have to + * remove from the runlist or mark sparse the freed runs later. + * + * Return the number of deallocated clusters (not counting sparse ones) on + * success and -errno on error. + * + * WARNING: If @ctx is supplied, regardless of whether success or failure is + * returned, you need to check IS_ERR(@ctx->mrec) and if 'true' the @ctx + * is no longer valid, i.e. you need to either call + * ntfs_attr_reinit_search_ctx() or ntfs_attr_put_search_ctx() on it. + * In that case PTR_ERR(@ctx->mrec) will give you the error code for + * why the mapping of the old inode failed. + * + * Locking: - The runlist described by @ni must be locked for writing on entry + * and is locked on return. Note the runlist may be modified when + * needed runlist fragments need to be mapped. + * - The volume lcn bitmap must be unlocked on entry and is unlocked + * on return. + * - This function takes the volume lcn bitmap lock for writing and + * modifies the bitmap contents. + * - If @ctx is NULL, the base mft record of @ni must not be mapped on + * entry and it will be left unmapped on return. + * - If @ctx is not NULL, the base mft record must be mapped on entry + * and it will be left mapped on return. + */ +s64 __ntfs_cluster_free(struct ntfs_inode *ni, const s64 start_vcn, s64 count, + struct ntfs_attr_search_ctx *ctx, const bool is_rollback) +{ + s64 delta, to_free, total_freed, real_freed; + struct ntfs_volume *vol; + struct inode *lcnbmp_vi; + struct runlist_element *rl; + int err; + unsigned int memalloc_flags; + + ntfs_debug("Entering for i_ino 0x%llx, start_vcn 0x%llx, count 0x%llx.%s", + ni->mft_no, start_vcn, count, + is_rollback ? " (rollback)" : ""); + vol = ni->vol; + lcnbmp_vi = vol->lcnbmp_ino; + if (start_vcn < 0 || count < -1) + return -EINVAL; + + if (!NVolFreeClusterKnown(vol)) + wait_event(vol->free_waitq, NVolFreeClusterKnown(vol)); + + /* + * Lock the lcn bitmap for writing but only if not rolling back. We + * must hold the lock all the way including through rollback otherwise + * rollback is not possible because once we have cleared a bit and + * dropped the lock, anyone could have set the bit again, thus + * allocating the cluster for another use. + */ + if (likely(!is_rollback)) { + memalloc_flags = memalloc_nofs_save(); + down_write(&vol->lcnbmp_lock); + } + + total_freed = real_freed = 0; + + rl = ntfs_attr_find_vcn_nolock(ni, start_vcn, ctx); + if (IS_ERR(rl)) { + err = PTR_ERR(rl); + if (err == -ENOENT) { + if (likely(!is_rollback)) { + up_write(&vol->lcnbmp_lock); + memalloc_nofs_restore(memalloc_flags); + } + return 0; + } + + if (!is_rollback) + ntfs_error(vol->sb, + "Failed to find first runlist element (error %d), aborting.", + err); + goto err_out; + } + if (unlikely(rl->lcn < LCN_HOLE)) { + if (!is_rollback) + ntfs_error(vol->sb, "First runlist element has invalid lcn, aborting."); + err = -EIO; + goto err_out; + } + /* Find the starting cluster inside the run that needs freeing. */ + delta = start_vcn - rl->vcn; + + /* The number of clusters in this run that need freeing. */ + to_free = rl->length - delta; + if (count >= 0 && to_free > count) + to_free = count; + + if (likely(rl->lcn >= 0)) { + /* Do the actual freeing of the clusters in this run. */ + err = ntfs_bitmap_set_bits_in_run(lcnbmp_vi, rl->lcn + delta, + to_free, likely(!is_rollback) ? 0 : 1); + if (unlikely(err)) { + if (!is_rollback) + ntfs_error(vol->sb, + "Failed to clear first run (error %i), aborting.", + err); + goto err_out; + } + /* We have freed @to_free real clusters. */ + real_freed = to_free; + } + /* Go to the next run and adjust the number of clusters left to free. */ + ++rl; + if (count >= 0) + count -= to_free; + + /* Keep track of the total "freed" clusters, including sparse ones. */ + total_freed = to_free; + /* + * Loop over the remaining runs, using @count as a capping value, and + * free them. + */ + for (; rl->length && count != 0; ++rl) { + if (unlikely(rl->lcn < LCN_HOLE)) { + s64 vcn; + + /* Attempt to map runlist. */ + vcn = rl->vcn; + rl = ntfs_attr_find_vcn_nolock(ni, vcn, ctx); + if (IS_ERR(rl)) { + err = PTR_ERR(rl); + if (!is_rollback) + ntfs_error(vol->sb, + "Failed to map runlist fragment or failed to find subsequent runlist element."); + goto err_out; + } + if (unlikely(rl->lcn < LCN_HOLE)) { + if (!is_rollback) + ntfs_error(vol->sb, + "Runlist element has invalid lcn (0x%llx).", + rl->lcn); + err = -EIO; + goto err_out; + } + } + /* The number of clusters in this run that need freeing. */ + to_free = rl->length; + if (count >= 0 && to_free > count) + to_free = count; + + if (likely(rl->lcn >= 0)) { + /* Do the actual freeing of the clusters in the run. */ + err = ntfs_bitmap_set_bits_in_run(lcnbmp_vi, rl->lcn, + to_free, likely(!is_rollback) ? 0 : 1); + if (unlikely(err)) { + if (!is_rollback) + ntfs_error(vol->sb, "Failed to clear subsequent run."); + goto err_out; + } + /* We have freed @to_free real clusters. */ + real_freed += to_free; + } + /* Adjust the number of clusters left to free. */ + if (count >= 0) + count -= to_free; + + /* Update the total done clusters. */ + total_freed += to_free; + } + ntfs_inc_free_clusters(vol, real_freed); + if (likely(!is_rollback)) { + up_write(&vol->lcnbmp_lock); + memalloc_nofs_restore(memalloc_flags); + } + + WARN_ON(count > 0); + + if (NVolDiscard(vol) && !is_rollback) { + s64 total_discarded = 0, rl_off; + u32 gran = bdev_discard_granularity(vol->sb->s_bdev); + + rl = ntfs_attr_find_vcn_nolock(ni, start_vcn, ctx); + if (IS_ERR(rl)) + return real_freed; + rl_off = start_vcn - rl->vcn; + while (rl->length && total_discarded < total_freed) { + s64 to_discard = rl->length - rl_off; + + if (to_discard + total_discarded > total_freed) + to_discard = total_freed - total_discarded; + if (rl->lcn >= 0) { + sector_t start_sector, end_sector; + int ret; + + start_sector = ALIGN(NTFS_CLU_TO_B(vol, rl->lcn + rl_off), + gran) >> SECTOR_SHIFT; + end_sector = ALIGN_DOWN(NTFS_CLU_TO_B(vol, + rl->lcn + rl_off + to_discard), + gran) >> SECTOR_SHIFT; + if (start_sector < end_sector) { + ret = blkdev_issue_discard(vol->sb->s_bdev, start_sector, + end_sector - start_sector, + GFP_NOFS); + if (ret) + break; + } + } + + total_discarded += to_discard; + ++rl; + rl_off = 0; + } + } + + /* We are done. Return the number of actually freed clusters. */ + ntfs_debug("Done."); + return real_freed; +err_out: + if (is_rollback) + return err; + /* If no real clusters were freed, no need to rollback. */ + if (!real_freed) { + up_write(&vol->lcnbmp_lock); + memalloc_nofs_restore(memalloc_flags); + return err; + } + /* + * Attempt to rollback and if that succeeds just return the error code. + * If rollback fails, set the volume errors flag, emit an error + * message, and return the error code. + */ + delta = __ntfs_cluster_free(ni, start_vcn, total_freed, ctx, true); + if (delta < 0) { + ntfs_error(vol->sb, + "Failed to rollback (error %i). Leaving inconsistent metadata! Unmount and run chkdsk.", + (int)delta); + NVolSetErrors(vol); + } + ntfs_dec_free_clusters(vol, delta); + up_write(&vol->lcnbmp_lock); + memalloc_nofs_restore(memalloc_flags); + ntfs_error(vol->sb, "Aborting (error %i).", err); + return err; +} diff --git a/fs/ntfs/lcnalloc.h b/fs/ntfs/lcnalloc.h new file mode 100644 index 000000000..d9ac622e4 --- /dev/null +++ b/fs/ntfs/lcnalloc.h @@ -0,0 +1,134 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Exports for NTFS kernel cluster (de)allocation. + * + * Copyright (c) 2004-2005 Anton Altaparmakov + */ + +#ifndef _LINUX_NTFS_LCNALLOC_H +#define _LINUX_NTFS_LCNALLOC_H + +#include + +#include "attrib.h" + +/* + * enum zone_type - Zone identifiers for cluster allocation policy + * + * FIRST_ZONE For sanity checking. + * MFT_ZONE Allocate from $MFT zone. + * DATA_ZONE Allocate from $DATA zone. + * LAST_ZONE For sanity checking. + */ +enum { + FIRST_ZONE = 0, + MFT_ZONE = 0, + DATA_ZONE = 1, + LAST_ZONE = 1, +}; + +struct runlist_element *ntfs_cluster_alloc(struct ntfs_volume *vol, + const s64 start_vcn, const s64 count, const s64 start_lcn, + const int zone, + const bool is_extension, + const bool is_contig, + const bool is_dealloc); +s64 __ntfs_cluster_free(struct ntfs_inode *ni, const s64 start_vcn, + s64 count, struct ntfs_attr_search_ctx *ctx, const bool is_rollback); + +/* + * ntfs_cluster_free - free clusters on an ntfs volume + * @ni: ntfs inode whose runlist describes the clusters to free + * @start_vcn: vcn in the runlist of @ni at which to start freeing clusters + * @count: number of clusters to free or -1 for all clusters + * @ctx: active attribute search context if present or NULL if not + * + * Free @count clusters starting at the cluster @start_vcn in the runlist + * described by the ntfs inode @ni. + * + * If @count is -1, all clusters from @start_vcn to the end of the runlist are + * deallocated. Thus, to completely free all clusters in a runlist, use + * @start_vcn = 0 and @count = -1. + * + * If @ctx is specified, it is an active search context of @ni and its base mft + * record. This is needed when ntfs_cluster_free() encounters unmapped runlist + * fragments and allows their mapping. If you do not have the mft record + * mapped, you can specify @ctx as NULL and ntfs_cluster_free() will perform + * the necessary mapping and unmapping. + * + * Note, ntfs_cluster_free() saves the state of @ctx on entry and restores it + * before returning. Thus, @ctx will be left pointing to the same attribute on + * return as on entry. However, the actual pointers in @ctx may point to + * different memory locations on return, so you must remember to reset any + * cached pointers from the @ctx, i.e. after the call to ntfs_cluster_free(), + * you will probably want to do: + * m = ctx->mrec; + * a = ctx->attr; + * Assuming you cache ctx->attr in a variable @a of type ATTR_RECORD * and that + * you cache ctx->mrec in a variable @m of type MFT_RECORD *. + * + * Note, ntfs_cluster_free() does not modify the runlist, so you have to remove + * from the runlist or mark sparse the freed runs later. + * + * Return the number of deallocated clusters (not counting sparse ones) on + * success and -errno on error. + * + * WARNING: If @ctx is supplied, regardless of whether success or failure is + * returned, you need to check IS_ERR(@ctx->mrec) and if 'true' the @ctx + * is no longer valid, i.e. you need to either call + * ntfs_attr_reinit_search_ctx() or ntfs_attr_put_search_ctx() on it. + * In that case PTR_ERR(@ctx->mrec) will give you the error code for + * why the mapping of the old inode failed. + * + * Locking: - The runlist described by @ni must be locked for writing on entry + * and is locked on return. Note the runlist may be modified when + * needed runlist fragments need to be mapped. + * - The volume lcn bitmap must be unlocked on entry and is unlocked + * on return. + * - This function takes the volume lcn bitmap lock for writing and + * modifies the bitmap contents. + * - If @ctx is NULL, the base mft record of @ni must not be mapped on + * entry and it will be left unmapped on return. + * - If @ctx is not NULL, the base mft record must be mapped on entry + * and it will be left mapped on return. + */ +static inline s64 ntfs_cluster_free(struct ntfs_inode *ni, const s64 start_vcn, + s64 count, struct ntfs_attr_search_ctx *ctx) +{ + return __ntfs_cluster_free(ni, start_vcn, count, ctx, false); +} + +int ntfs_cluster_free_from_rl_nolock(struct ntfs_volume *vol, + const struct runlist_element *rl); + +/* + * ntfs_cluster_free_from_rl - free clusters from runlist + * @vol: mounted ntfs volume on which to free the clusters + * @rl: runlist describing the clusters to free + * + * Free all the clusters described by the runlist @rl on the volume @vol. In + * the case of an error being returned, at least some of the clusters were not + * freed. + * + * Return 0 on success and -errno on error. + * + * Locking: - This function takes the volume lcn bitmap lock for writing and + * modifies the bitmap contents. + * - The caller must have locked the runlist @rl for reading or + * writing. + */ +static inline int ntfs_cluster_free_from_rl(struct ntfs_volume *vol, + const struct runlist_element *rl) +{ + int ret; + unsigned int memalloc_flags; + + memalloc_flags = memalloc_nofs_save(); + down_write(&vol->lcnbmp_lock); + ret = ntfs_cluster_free_from_rl_nolock(vol, rl); + up_write(&vol->lcnbmp_lock); + memalloc_nofs_restore(memalloc_flags); + return ret; +} + +#endif /* defined _LINUX_NTFS_LCNALLOC_H */ diff --git a/fs/ntfs/logfile.c b/fs/ntfs/logfile.c new file mode 100644 index 000000000..024ddee42 --- /dev/null +++ b/fs/ntfs/logfile.c @@ -0,0 +1,780 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * NTFS kernel journal handling. + * + * Copyright (c) 2002-2007 Anton Altaparmakov + */ + +#include + +#include "attrib.h" +#include "logfile.h" +#include "ntfs.h" + +/* + * ntfs_check_restart_page_header - check the page header for consistency + * @vi: LogFile inode to which the restart page header belongs + * @rp: restart page header to check + * @pos: position in @vi at which the restart page header resides + * + * Check the restart page header @rp for consistency and return 'true' if it is + * consistent and 'false' otherwise. + * + * This function only needs NTFS_BLOCK_SIZE bytes in @rp, i.e. it does not + * require the full restart page. + */ +static bool ntfs_check_restart_page_header(struct inode *vi, + struct restart_page_header *rp, s64 pos) +{ + u32 logfile_system_page_size, logfile_log_page_size; + u16 ra_ofs, usa_count, usa_ofs, usa_end = 0; + bool have_usa = true; + + ntfs_debug("Entering."); + /* + * If the system or log page sizes are smaller than the ntfs block size + * or either is not a power of 2 we cannot handle this log file. + */ + logfile_system_page_size = le32_to_cpu(rp->system_page_size); + logfile_log_page_size = le32_to_cpu(rp->log_page_size); + if (logfile_system_page_size < NTFS_BLOCK_SIZE || + logfile_log_page_size < NTFS_BLOCK_SIZE || + logfile_system_page_size & + (logfile_system_page_size - 1) || + !is_power_of_2(logfile_log_page_size)) { + ntfs_error(vi->i_sb, "LogFile uses unsupported page size."); + return false; + } + /* + * We must be either at !pos (1st restart page) or at pos = system page + * size (2nd restart page). + */ + if (pos && pos != logfile_system_page_size) { + ntfs_error(vi->i_sb, "Found restart area in incorrect position in LogFile."); + return false; + } + /* We only know how to handle version 1.1. */ + if (le16_to_cpu(rp->major_ver) != 1 || + le16_to_cpu(rp->minor_ver) != 1) { + ntfs_error(vi->i_sb, + "LogFile version %i.%i is not supported. (This driver supports version 1.1 only.)", + (int)le16_to_cpu(rp->major_ver), + (int)le16_to_cpu(rp->minor_ver)); + return false; + } + /* + * If chkdsk has been run the restart page may not be protected by an + * update sequence array. + */ + if (ntfs_is_chkd_record(rp->magic) && !le16_to_cpu(rp->usa_count)) { + have_usa = false; + goto skip_usa_checks; + } + /* Verify the size of the update sequence array. */ + usa_count = 1 + (logfile_system_page_size >> NTFS_BLOCK_SIZE_BITS); + if (usa_count != le16_to_cpu(rp->usa_count)) { + ntfs_error(vi->i_sb, + "LogFile restart page specifies inconsistent update sequence array count."); + return false; + } + /* Verify the position of the update sequence array. */ + usa_ofs = le16_to_cpu(rp->usa_ofs); + usa_end = usa_ofs + usa_count * sizeof(u16); + if (usa_ofs < sizeof(struct restart_page_header) || + usa_end > NTFS_BLOCK_SIZE - sizeof(u16)) { + ntfs_error(vi->i_sb, + "LogFile restart page specifies inconsistent update sequence array offset."); + return false; + } +skip_usa_checks: + /* + * Verify the position of the restart area. It must be: + * - aligned to 8-byte boundary, + * - after the update sequence array, and + * - within the system page size. + */ + ra_ofs = le16_to_cpu(rp->restart_area_offset); + if (ra_ofs & 7 || (have_usa ? ra_ofs < usa_end : + ra_ofs < sizeof(struct restart_page_header)) || + ra_ofs > logfile_system_page_size) { + ntfs_error(vi->i_sb, + "LogFile restart page specifies inconsistent restart area offset."); + return false; + } + /* + * Only restart pages modified by chkdsk are allowed to have chkdsk_lsn + * set. + */ + if (!ntfs_is_chkd_record(rp->magic) && le64_to_cpu(rp->chkdsk_lsn)) { + ntfs_error(vi->i_sb, + "LogFile restart page is not modified by chkdsk but a chkdsk LSN is specified."); + return false; + } + ntfs_debug("Done."); + return true; +} + +/* + * ntfs_check_restart_area - check the restart area for consistency + * @vi: LogFile inode to which the restart page belongs + * @rp: restart page whose restart area to check + * + * Check the restart area of the restart page @rp for consistency and return + * 'true' if it is consistent and 'false' otherwise. + * + * This function assumes that the restart page header has already been + * consistency checked. + * + * This function only needs NTFS_BLOCK_SIZE bytes in @rp, i.e. it does not + * require the full restart page. + */ +static bool ntfs_check_restart_area(struct inode *vi, struct restart_page_header *rp) +{ + u64 file_size; + struct restart_area *ra; + u32 ra_ofs, ra_len, ca_ofs; + u8 fs_bits; + + ntfs_debug("Entering."); + ra_ofs = le16_to_cpu(rp->restart_area_offset); + ra = (struct restart_area *)((u8 *)rp + ra_ofs); + /* + * Everything before ra->file_size must be before the first word + * protected by an update sequence number. This ensures that it is + * safe to access ra->client_array_offset. + */ + if (ra_ofs + offsetof(struct restart_area, file_size) > + NTFS_BLOCK_SIZE - sizeof(u16)) { + ntfs_error(vi->i_sb, + "LogFile restart area specifies inconsistent file offset."); + return false; + } + /* + * Now that we can access ra->client_array_offset, make sure everything + * up to the log client array is before the first word protected by an + * update sequence number. This ensures we can access all of the + * restart area elements safely. Also, the client array offset must be + * aligned to an 8-byte boundary. + */ + ca_ofs = le16_to_cpu(ra->client_array_offset); + if (((ca_ofs + 7) & ~7) != ca_ofs || + ra_ofs + ca_ofs > NTFS_BLOCK_SIZE - sizeof(u16)) { + ntfs_error(vi->i_sb, + "LogFile restart area specifies inconsistent client array offset."); + return false; + } + /* + * The restart area must end within the system page size both when + * calculated manually and as specified by ra->restart_area_length. + * Also, the calculated length must not exceed the specified length. + */ + ra_len = ca_ofs + le16_to_cpu(ra->log_clients) * + sizeof(struct log_client_record); + if (ra_ofs + ra_len > le32_to_cpu(rp->system_page_size) || + ra_ofs + le16_to_cpu(ra->restart_area_length) > + le32_to_cpu(rp->system_page_size) || + ra_len > le16_to_cpu(ra->restart_area_length)) { + ntfs_error(vi->i_sb, + "LogFile restart area is out of bounds of the system page size specified by the restart page header and/or the specified restart area length is inconsistent."); + return false; + } + /* + * The ra->client_free_list and ra->client_in_use_list must be either + * LOGFILE_NO_CLIENT or less than ra->log_clients or they are + * overflowing the client array. + */ + if ((ra->client_free_list != LOGFILE_NO_CLIENT && + le16_to_cpu(ra->client_free_list) >= + le16_to_cpu(ra->log_clients)) || + (ra->client_in_use_list != LOGFILE_NO_CLIENT && + le16_to_cpu(ra->client_in_use_list) >= + le16_to_cpu(ra->log_clients))) { + ntfs_error(vi->i_sb, + "LogFile restart area specifies overflowing client free and/or in use lists."); + return false; + } + /* + * Check ra->seq_number_bits against ra->file_size for consistency. + * We cannot just use ffs() because the file size is not a power of 2. + */ + file_size = le64_to_cpu(ra->file_size); + fs_bits = 0; + while (file_size) { + file_size >>= 1; + fs_bits++; + } + if (le32_to_cpu(ra->seq_number_bits) != 67 - fs_bits) { + ntfs_error(vi->i_sb, + "LogFile restart area specifies inconsistent sequence number bits."); + return false; + } + /* The log record header length must be a multiple of 8. */ + if (((le16_to_cpu(ra->log_record_header_length) + 7) & ~7) != + le16_to_cpu(ra->log_record_header_length)) { + ntfs_error(vi->i_sb, + "LogFile restart area specifies inconsistent log record header length."); + return false; + } + /* Dito for the log page data offset. */ + if (((le16_to_cpu(ra->log_page_data_offset) + 7) & ~7) != + le16_to_cpu(ra->log_page_data_offset)) { + ntfs_error(vi->i_sb, + "LogFile restart area specifies inconsistent log page data offset."); + return false; + } + ntfs_debug("Done."); + return true; +} + +/* + * ntfs_check_log_client_array - check the log client array for consistency + * @vi: LogFile inode to which the restart page belongs + * @rp: restart page whose log client array to check + * + * Check the log client array of the restart page @rp for consistency and + * return 'true' if it is consistent and 'false' otherwise. + * + * This function assumes that the restart page header and the restart area have + * already been consistency checked. + * + * Unlike ntfs_check_restart_page_header() and ntfs_check_restart_area(), this + * function needs @rp->system_page_size bytes in @rp, i.e. it requires the full + * restart page and the page must be multi sector transfer deprotected. + */ +static bool ntfs_check_log_client_array(struct inode *vi, + struct restart_page_header *rp) +{ + struct restart_area *ra; + struct log_client_record *ca, *cr; + u16 nr_clients, idx; + bool in_free_list, idx_is_first; + + ntfs_debug("Entering."); + ra = (struct restart_area *)((u8 *)rp + le16_to_cpu(rp->restart_area_offset)); + ca = (struct log_client_record *)((u8 *)ra + + le16_to_cpu(ra->client_array_offset)); + /* + * Check the ra->client_free_list first and then check the + * ra->client_in_use_list. Check each of the log client records in + * each of the lists and check that the array does not overflow the + * ra->log_clients value. Also keep track of the number of records + * visited as there cannot be more than ra->log_clients records and + * that way we detect eventual loops in within a list. + */ + nr_clients = le16_to_cpu(ra->log_clients); + idx = le16_to_cpu(ra->client_free_list); + in_free_list = true; +check_list: + for (idx_is_first = true; idx != LOGFILE_NO_CLIENT_CPU; nr_clients--, + idx = le16_to_cpu(cr->next_client)) { + if (!nr_clients || idx >= le16_to_cpu(ra->log_clients)) + goto err_out; + /* Set @cr to the current log client record. */ + cr = ca + idx; + /* The first log client record must not have a prev_client. */ + if (idx_is_first) { + if (cr->prev_client != LOGFILE_NO_CLIENT) + goto err_out; + idx_is_first = false; + } + } + /* Switch to and check the in use list if we just did the free list. */ + if (in_free_list) { + in_free_list = false; + idx = le16_to_cpu(ra->client_in_use_list); + goto check_list; + } + ntfs_debug("Done."); + return true; +err_out: + ntfs_error(vi->i_sb, "LogFile log client array is corrupt."); + return false; +} + +/* + * ntfs_check_and_load_restart_page - check the restart page for consistency + * @vi: LogFile inode to which the restart page belongs + * @rp: restart page to check + * @pos: position in @vi at which the restart page resides + * @wrp: [OUT] copy of the multi sector transfer deprotected restart page + * @lsn: [OUT] set to the current logfile lsn on success + * + * Check the restart page @rp for consistency and return 0 if it is consistent + * and -errno otherwise. The restart page may have been modified by chkdsk in + * which case its magic is CHKD instead of RSTR. + * + * This function only needs NTFS_BLOCK_SIZE bytes in @rp, i.e. it does not + * require the full restart page. + * + * If @wrp is not NULL, on success, *@wrp will point to a buffer containing a + * copy of the complete multi sector transfer deprotected page. On failure, + * *@wrp is undefined. + * + * Simillarly, if @lsn is not NULL, on success *@lsn will be set to the current + * logfile lsn according to this restart page. On failure, *@lsn is undefined. + * + * The following error codes are defined: + * -EINVAL - The restart page is inconsistent. + * -ENOMEM - Not enough memory to load the restart page. + * -EIO - Failed to reading from LogFile. + */ +static int ntfs_check_and_load_restart_page(struct inode *vi, + struct restart_page_header *rp, s64 pos, struct restart_page_header **wrp, + s64 *lsn) +{ + struct restart_area *ra; + struct restart_page_header *trp; + int size, err; + + ntfs_debug("Entering."); + /* Check the restart page header for consistency. */ + if (!ntfs_check_restart_page_header(vi, rp, pos)) { + /* Error output already done inside the function. */ + return -EINVAL; + } + /* Check the restart area for consistency. */ + if (!ntfs_check_restart_area(vi, rp)) { + /* Error output already done inside the function. */ + return -EINVAL; + } + ra = (struct restart_area *)((u8 *)rp + le16_to_cpu(rp->restart_area_offset)); + /* + * Allocate a buffer to store the whole restart page so we can multi + * sector transfer deprotect it. + */ + trp = kvzalloc(le32_to_cpu(rp->system_page_size), GFP_NOFS); + if (!trp) { + ntfs_error(vi->i_sb, "Failed to allocate memory for LogFile restart page buffer."); + return -ENOMEM; + } + /* + * Read the whole of the restart page into the buffer. If it fits + * completely inside @rp, just copy it from there. Otherwise map all + * the required pages and copy the data from them. + */ + size = PAGE_SIZE - (pos & ~PAGE_MASK); + if (size >= le32_to_cpu(rp->system_page_size)) { + memcpy(trp, rp, le32_to_cpu(rp->system_page_size)); + } else { + pgoff_t idx; + struct folio *folio; + int have_read, to_read; + + /* First copy what we already have in @rp. */ + memcpy(trp, rp, size); + /* Copy the remaining data one page at a time. */ + have_read = size; + to_read = le32_to_cpu(rp->system_page_size) - size; + idx = (pos + size) >> PAGE_SHIFT; + do { + folio = read_mapping_folio(vi->i_mapping, idx, NULL); + if (IS_ERR(folio)) { + ntfs_error(vi->i_sb, "Error mapping LogFile page (index %lu).", + idx); + err = PTR_ERR(folio); + if (err != -EIO && err != -ENOMEM) + err = -EIO; + goto err_out; + } + size = min_t(int, to_read, PAGE_SIZE); + memcpy((u8 *)trp + have_read, folio_address(folio), size); + folio_put(folio); + have_read += size; + to_read -= size; + idx++; + } while (to_read > 0); + } + /* + * Perform the multi sector transfer deprotection on the buffer if the + * restart page is protected. + */ + if ((!ntfs_is_chkd_record(trp->magic) || le16_to_cpu(trp->usa_count)) && + post_read_mst_fixup((struct ntfs_record *)trp, le32_to_cpu(rp->system_page_size))) { + /* + * A multi sector transfer error was detected. We only need to + * abort if the restart page contents exceed the multi sector + * transfer fixup of the first sector. + */ + if (le16_to_cpu(rp->restart_area_offset) + + le16_to_cpu(ra->restart_area_length) > + NTFS_BLOCK_SIZE - sizeof(u16)) { + ntfs_error(vi->i_sb, + "Multi sector transfer error detected in LogFile restart page."); + err = -EINVAL; + goto err_out; + } + } + /* + * If the restart page is modified by chkdsk or there are no active + * logfile clients, the logfile is consistent. Otherwise, need to + * check the log client records for consistency, too. + */ + err = 0; + if (ntfs_is_rstr_record(rp->magic) && + ra->client_in_use_list != LOGFILE_NO_CLIENT) { + if (!ntfs_check_log_client_array(vi, trp)) { + err = -EINVAL; + goto err_out; + } + } + if (lsn) { + if (ntfs_is_rstr_record(rp->magic)) + *lsn = le64_to_cpu(ra->current_lsn); + else /* if (ntfs_is_chkd_record(rp->magic)) */ + *lsn = le64_to_cpu(rp->chkdsk_lsn); + } + ntfs_debug("Done."); + if (wrp) + *wrp = trp; + else { +err_out: + kvfree(trp); + } + return err; +} + +/* + * ntfs_check_logfile - check the journal for consistency + * @log_vi: struct inode of loaded journal LogFile to check + * @rp: [OUT] on success this is a copy of the current restart page + * + * Check the LogFile journal for consistency and return 'true' if it is + * consistent and 'false' if not. On success, the current restart page is + * returned in *@rp. Caller must call kvfree(*@rp) when finished with it. + * + * At present we only check the two restart pages and ignore the log record + * pages. + * + * Note that the MstProtected flag is not set on the LogFile inode and hence + * when reading pages they are not deprotected. This is because we do not know + * if the LogFile was created on a system with a different page size to ours + * yet and mst deprotection would fail if our page size is smaller. + */ +bool ntfs_check_logfile(struct inode *log_vi, struct restart_page_header **rp) +{ + s64 size, pos; + s64 rstr1_lsn, rstr2_lsn; + struct ntfs_volume *vol = NTFS_SB(log_vi->i_sb); + struct address_space *mapping = log_vi->i_mapping; + struct folio *folio = NULL; + u8 *kaddr = NULL; + struct restart_page_header *rstr1_ph = NULL; + struct restart_page_header *rstr2_ph = NULL; + int log_page_size, err; + bool logfile_is_empty = true; + u8 log_page_bits; + + ntfs_debug("Entering."); + /* An empty LogFile must have been clean before it got emptied. */ + if (NVolLogFileEmpty(vol)) + goto is_empty; + size = i_size_read(log_vi); + /* Make sure the file doesn't exceed the maximum allowed size. */ + if (size > MaxLogFileSize) + size = MaxLogFileSize; + /* + * Truncate size to a multiple of the page cache size or the default + * log page size if the page cache size is between the default log page + * log page size if the page cache size is between the default log page + * size and twice that. + */ + if (DefaultLogPageSize <= PAGE_SIZE && + DefaultLogPageSize * 2 <= PAGE_SIZE) + log_page_size = DefaultLogPageSize; + else + log_page_size = PAGE_SIZE; + /* + * Use ntfs_ffs() instead of ffs() to enable the compiler to + * optimize log_page_size and log_page_bits into constants. + */ + log_page_bits = ntfs_ffs(log_page_size) - 1; + size &= ~(s64)(log_page_size - 1); + /* + * Ensure the log file is big enough to store at least the two restart + * pages and the minimum number of log record pages. + */ + if (size < log_page_size * 2 || (size - log_page_size * 2) >> + log_page_bits < MinLogRecordPages) { + ntfs_error(vol->sb, "LogFile is too small."); + return false; + } + /* + * Read through the file looking for a restart page. Since the restart + * page header is at the beginning of a page we only need to search at + * what could be the beginning of a page (for each page size) rather + * than scanning the whole file byte by byte. If all potential places + * contain empty and uninitialzed records, the log file can be assumed + * to be empty. + */ + for (pos = 0; pos < size; pos <<= 1) { + pgoff_t idx = pos >> PAGE_SHIFT; + + if (!folio || folio->index != idx) { + if (folio) { + kunmap_local(kaddr); + folio_put(folio); + } + folio = read_mapping_folio(mapping, idx, NULL); + if (IS_ERR(folio)) { + ntfs_error(vol->sb, "Error mapping LogFile page (index %lu).", + idx); + goto err_out; + } + } + kaddr = (u8 *)kmap_local_folio(folio, 0) + (pos & ~PAGE_MASK); + /* + * A non-empty block means the logfile is not empty while an + * empty block after a non-empty block has been encountered + * means we are done. + */ + if (!ntfs_is_empty_recordp((__le32 *)kaddr)) + logfile_is_empty = false; + else if (!logfile_is_empty) + break; + /* + * A log record page means there cannot be a restart page after + * this so no need to continue searching. + */ + if (ntfs_is_rcrd_recordp((__le32 *)kaddr)) + break; + /* If not a (modified by chkdsk) restart page, continue. */ + if (!ntfs_is_rstr_recordp((__le32 *)kaddr) && + !ntfs_is_chkd_recordp((__le32 *)kaddr)) { + if (!pos) + pos = NTFS_BLOCK_SIZE >> 1; + continue; + } + /* + * Check the (modified by chkdsk) restart page for consistency + * and get a copy of the complete multi sector transfer + * deprotected restart page. + */ + err = ntfs_check_and_load_restart_page(log_vi, + (struct restart_page_header *)kaddr, pos, + !rstr1_ph ? &rstr1_ph : &rstr2_ph, + !rstr1_ph ? &rstr1_lsn : &rstr2_lsn); + if (!err) { + /* + * If we have now found the first (modified by chkdsk) + * restart page, continue looking for the second one. + */ + if (!pos) { + pos = NTFS_BLOCK_SIZE >> 1; + continue; + } + /* + * We have now found the second (modified by chkdsk) + * restart page, so we can stop looking. + */ + break; + } + /* + * Error output already done inside the function. Note, we do + * not abort if the restart page was invalid as we might still + * find a valid one further in the file. + */ + if (err != -EINVAL) { + kunmap_local(kaddr); + folio_put(folio); + goto err_out; + } + /* Continue looking. */ + if (!pos) + pos = NTFS_BLOCK_SIZE >> 1; + } + if (folio) { + kunmap_local(kaddr); + folio_put(folio); + } + if (logfile_is_empty) { + NVolSetLogFileEmpty(vol); +is_empty: + ntfs_debug("Done. (LogFile is empty.)"); + return true; + } + if (!rstr1_ph) { + ntfs_error(vol->sb, + "Did not find any restart pages in LogFile and it was not empty."); + return false; + } + /* If both restart pages were found, use the more recent one. */ + if (rstr2_ph) { + /* + * If the second restart area is more recent, switch to it. + * Otherwise just throw it away. + */ + if (rstr2_lsn > rstr1_lsn) { + ntfs_debug("Using second restart page as it is more recent."); + kvfree(rstr1_ph); + rstr1_ph = rstr2_ph; + /* rstr1_lsn = rstr2_lsn; */ + } else { + ntfs_debug("Using first restart page as it is more recent."); + kvfree(rstr2_ph); + } + rstr2_ph = NULL; + } + /* All consistency checks passed. */ + if (rp) + *rp = rstr1_ph; + else + kvfree(rstr1_ph); + ntfs_debug("Done."); + return true; +err_out: + kvfree(rstr1_ph); + return false; +} + +/* + * ntfs_empty_logfile - empty the contents of the LogFile journal + * @log_vi: struct inode of loaded journal LogFile to empty + * + * Empty the contents of the LogFile journal @log_vi and return 'true' on + * success and 'false' on error. + * + * This function assumes that the LogFile journal has already been consistency + * checked by a call to ntfs_check_logfile() and that ntfs_is_logfile_clean() + * has been used to ensure that the LogFile is clean. + */ +bool ntfs_empty_logfile(struct inode *log_vi) +{ + s64 vcn, end_vcn; + struct ntfs_inode *log_ni = NTFS_I(log_vi); + struct ntfs_volume *vol = log_ni->vol; + struct super_block *sb = vol->sb; + struct runlist_element *rl; + unsigned long flags; + int err; + bool should_wait = true; + char *empty_buf = NULL; + struct file_ra_state *ra = NULL; + + ntfs_debug("Entering."); + if (NVolLogFileEmpty(vol)) { + ntfs_debug("Done."); + return true; + } + + /* + * We cannot use ntfs_attr_set() because we may be still in the middle + * of a mount operation. Thus we do the emptying by hand by first + * zapping the page cache pages for the LogFile/DATA attribute and + * then emptying each of the buffers in each of the clusters specified + * by the runlist by hand. + */ + vcn = 0; + read_lock_irqsave(&log_ni->size_lock, flags); + end_vcn = (log_ni->initialized_size + vol->cluster_size_mask) >> + vol->cluster_size_bits; + read_unlock_irqrestore(&log_ni->size_lock, flags); + truncate_inode_pages(log_vi->i_mapping, 0); + down_write(&log_ni->runlist.lock); + rl = log_ni->runlist.rl; + if (unlikely(!rl || vcn < rl->vcn || !rl->length)) { +map_vcn: + err = ntfs_map_runlist_nolock(log_ni, vcn, NULL); + if (err) { + ntfs_error(sb, "Failed to map runlist fragment (error %d).", -err); + goto err; + } + rl = log_ni->runlist.rl; + } + /* Seek to the runlist element containing @vcn. */ + while (rl->length && vcn >= rl[1].vcn) + rl++; + + err = -ENOMEM; + empty_buf = kvzalloc(vol->cluster_size, GFP_NOFS); + if (!empty_buf) + goto err; + + memset(empty_buf, 0xff, vol->cluster_size); + + ra = kzalloc(sizeof(*ra), GFP_NOFS); + if (!ra) + goto err; + + file_ra_state_init(ra, sb->s_bdev->bd_mapping); + do { + s64 lcn; + loff_t start, end; + s64 len; + + /* + * If this run is not mapped map it now and start again as the + * runlist will have been updated. + */ + lcn = rl->lcn; + if (unlikely(lcn == LCN_RL_NOT_MAPPED)) { + vcn = rl->vcn; + kvfree(empty_buf); + empty_buf = NULL; + kfree(ra); + ra = NULL; + goto map_vcn; + } + /* If this run is not valid abort with an error. */ + if (unlikely(!rl->length || lcn < LCN_HOLE)) + goto rl_err; + /* Skip holes. */ + if (lcn == LCN_HOLE) + continue; + start = NTFS_CLU_TO_B(vol, lcn); + len = rl->length; + if (rl[1].vcn > end_vcn) + len = end_vcn - rl->vcn; + end = NTFS_CLU_TO_B(vol, lcn + len); + + page_cache_sync_readahead(sb->s_bdev->bd_mapping, ra, NULL, + start >> PAGE_SHIFT, (end - start) >> PAGE_SHIFT); + + do { + err = ntfs_bdev_write(sb, empty_buf, start, + vol->cluster_size); + if (err) { + ntfs_error(sb, "ntfs_dev_write failed, err : %d\n", err); + goto io_err; + } + + /* + * Submit the buffer and wait for i/o to complete but + * only for the first buffer so we do not miss really + * serious i/o errors. Once the first buffer has + * completed ignore errors afterwards as we can assume + * that if one buffer worked all of them will work. + */ + if (should_wait) { + should_wait = false; + err = filemap_write_and_wait_range(sb->s_bdev->bd_mapping, + start, start + vol->cluster_size); + if (err) + goto io_err; + } + start += vol->cluster_size; + } while (start < end); + } while ((++rl)->vcn < end_vcn); + up_write(&log_ni->runlist.lock); + kvfree(empty_buf); + kfree(ra); + truncate_inode_pages(log_vi->i_mapping, 0); + /* Set the flag so we do not have to do it again on remount. */ + NVolSetLogFileEmpty(vol); + ntfs_debug("Done."); + return true; +io_err: + ntfs_error(sb, "Failed to write buffer. Unmount and run chkdsk."); + goto dirty_err; +rl_err: + ntfs_error(sb, "Runlist is corrupt. Unmount and run chkdsk."); +dirty_err: + NVolSetErrors(vol); + err = -EIO; +err: + kvfree(empty_buf); + kfree(ra); + up_write(&log_ni->runlist.lock); + ntfs_error(sb, "Failed to fill LogFile with 0xff bytes (error %d).", + -err); + return false; +} diff --git a/fs/ntfs/logfile.h b/fs/ntfs/logfile.h new file mode 100644 index 000000000..dadb56fd0 --- /dev/null +++ b/fs/ntfs/logfile.h @@ -0,0 +1,245 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Defines for NTFS kernel journal (LogFile) handling. + * + * Copyright (c) 2000-2005 Anton Altaparmakov + */ + +#ifndef _LINUX_NTFS_LOGFILE_H +#define _LINUX_NTFS_LOGFILE_H + +#include "layout.h" + +/* + * Journal (LogFile) organization: + * + * Two restart areas present in the first two pages (restart pages, one restart + * area in each page). When the volume is dismounted they should be identical, + * except for the update sequence array which usually has a different update + * sequence number. + * + * These are followed by log records organized in pages headed by a log record + * header going up to log file size. Not all pages contain log records when a + * volume is first formatted, but as the volume ages, all records will be used. + * When the log file fills up, the records at the beginning are purged (by + * modifying the oldest_lsn to a higher value presumably) and writing begins + * at the beginning of the file. Effectively, the log file is viewed as a + * circular entity. + * + * NOTE: Windows NT, 2000, and XP all use log file version 1.1 but they accept + * versions <= 1.x, including 0.-1. (Yes, that is a minus one in there!) We + * probably only want to support 1.1 as this seems to be the current version + * and we don't know how that differs from the older versions. The only + * exception is if the journal is clean as marked by the two restart pages + * then it doesn't matter whether we are on an earlier version. We can just + * reinitialize the logfile and start again with version 1.1. + */ + +/* Some LogFile related constants. */ +#define MaxLogFileSize 0x100000000ULL +#define DefaultLogPageSize 4096 +#define MinLogRecordPages 48 + +/* + * Log file restart page header (begins the restart area). + * + * @magic: The magic is "RSTR". + * @usa_ofs: See ntfs_record struct definition in layout.h. When creating, + * set this to be immediately after this header structure (without any + * alignment). + * @usa_count: See ntfs_record struct definition in layout.h. + * @chkdsk_lsn: The last log file sequence number found by chkdsk. Only + * used when the magic is changed to "CHKD". Otherwise this is zero. + * @system_page_size: Byte size of system pages when the log file was + * created, has to be >= 512 and a power of 2. Use this to calculate + * the required size of the usa (usa_count) and add it to usa_ofs. Then + * verify that the result is less than the value of + * the restart_area_offset. + * @log_page_size: Byte size of log file pages, has to be >= 512 and + * a power of 2. The default is 4096 and is used when the system page + * size is between 4096 and 8192. Otherwise this is set to the system + * page size instead. + * @restart_area_offset: Byte offset from the start of this header to + * the RESTART_AREA. Value has to be aligned to 8-byte boundary. When + * creating, set this to be after the usa. + * @minor_ver: Log file minor version. Only check if major version is 1. + * @major_ver: Log file major version. We only support version 1.1. + */ +struct restart_page_header { + __le32 magic; + __le16 usa_ofs; + __le16 usa_count; + __le64 chkdsk_lsn; + __le32 system_page_size; + __le32 log_page_size; + __le16 restart_area_offset; + __le16 minor_ver; + __le16 major_ver; +} __packed; + +/* + * Constant for the log client indices meaning that there are no client records + * in this particular client array. Also inside the client records themselves, + * this means that there are no client records preceding or following this one. + */ +#define LOGFILE_NO_CLIENT cpu_to_le16(0xffff) +#define LOGFILE_NO_CLIENT_CPU 0xffff + +/* + * These are the so far known RESTART_AREA_* flags (16-bit) which contain + * information about the log file in which they are present. + * gcc: Force enum bit width to 16. + */ +enum { + RESTART_VOLUME_IS_CLEAN = cpu_to_le16(0x0002), + RESTART_SPACE_FILLER = cpu_to_le16(0xffff), +} __packed; + +/* + * Log file restart area record. The offset of this record is found by adding + * the offset of the RESTART_PAGE_HEADER to the restart_area_offset value found + * in it. See notes at restart_area_offset above. + * + * @current_lsn: The current, i.e. last LSN inside the log when + * the restart area was last written. This happens often but what is + * the interval? Is it just fixed time or is it every time a check point + * is written or somethine else? On create set to 0. + * @log_clients: Number of log client records in the array of log client + * records which follows this restart area. Must be 1. + * @client_free_list: The index of the first free log client record in + * the array of log client records. LOGFILE_NO_CLIENT means that there + * are no free log client records in the array. If != LOGFILE_NO_CLIENT, + * check that log_clients > client_free_list. On Win2k and presumably + * earlier, on a clean volume this is != LOGFILE_NO_CLIENT, and it should + * be 0, i.e. the first (and only) client record is free and thus + * the logfile is closed and hence clean. A dirty volume would have left + * the logfile open and hence this would be LOGFILE_NO_CLIENT. On WinXP + * and presumably later, the logfile is always open, even on clean + * shutdown so this should always be LOGFILE_NO_CLIENT. + * @client_in_use_list: The index of the first in-use log client record in + * the array of log client records. LOGFILE_NO_CLIENT means that there + * are no in-use log client records in the array. + * If != LOGFILE_NO_CLIENT check that log_clients > client_in_use_list. + * On Win2k and presumably earlier, on a clean volume this is + * LOGFILE_NO_CLIENT, i.e. there are no client records in use and thus + * the logfile is closed and hence clean. A dirty volume would have left + * the logfile open and hence this would be != LOGFILE_NO_CLIENT, and it + * should be 0, i.e. the first (and only) client record is in use. On + * WinXP and presumably later, the logfile is always open, even on clean + * shutdown so this should always be 0. + * @flags: Flags modifying LFS behaviour. On Win2k and presumably earlier + * this is always 0. On WinXP and presumably later, if the logfile was + * shutdown cleanly, the second bit, RESTART_VOLUME_IS_CLEAN, is set. + * This bit is cleared when the volume is mounted by WinXP and set when + * the volume is dismounted, thus if the logfile is dirty, this bit is + * clear. Thus we don't need to check the Windows version to determine + * if the logfile is clean. Instead if the logfile is closed, we know + * it must be clean. If it is open and this bit is set, we also know + * it must be clean. If on the other hand the logfile is open and this + * bit is clear, we can be almost certain that the logfile is dirty. + * @seq_number_bits: How many bits to use for the sequence number. This + * is calculated as 67 - the number of bits required to store the logfile + * size in bytes and this can be used in with the specified file_size as + * a consistency check. + * @restart_area_length: Length of the restart area including the client + * array. Following checks required if version matches. Otherwise, + * skip them. restart_area_offset + restart_area_length has to be + * <= system_page_size. Also, restart_area_length has to be >= + * client_array_offset + (log_clients * sizeof(log client record)). + * @client_array_offset: Offset from the start of this record to the first + * log client record if versions are matched. When creating, set this + * to be after this restart area structure, aligned to 8-bytes boundary. + * If the versions do not match, this is ignored and the offset is + * assumed to be (sizeof(RESTART_AREA) + 7) & ~7, i.e. rounded up to + * first 8-byte boundary. Either way, client_array_offset has to be + * aligned to an 8-byte boundary. Also, restart_area_offset + + * client_array_offset has to be <= 510. Finally, client_array_offset + + * (log_clients * sizeof(log client record)) has to be <= system_page_size. + * On Win2k and presumably earlier, this is 0x30, i.e. immediately + * following this record. On WinXP and presumably later, this is 0x40, + * i.e. there are 16 extra bytes between this record and the client + * array. This probably means that the RESTART_AREA record is actually + * bigger in WinXP and later. + * @file_size: Usable byte size of the log file. + * If the restart_area_offset + the offset of the file_size are > 510 + * then corruption has occurred. This is the very first check when + * starting with the restart_area as if it fails it means that some of + * the above values will be corrupted by the multi sector transfer + * protection. The file_size has to be rounded down to be a multiple + * of the log_page_size in the RESTART_PAGE_HEADER and then it has to be + * at least big enough to store the two restart pages and 48 (0x30) log + * record pages. + * @last_lsn_data_length: Length of data of last LSN, not including the log + * record header. On create set to 0. + * @log_record_header_length: Byte size of the log record header. If the + * version matches then check that the value of log_record_header_length + * is a multiple of 8, i.e. (log_record_header_length + 7) & ~7 == + * log_record_header_length. When creating set it to + * sizeof(LOG_RECORD_HEADER), aligned to 8 bytes. + * @log_page_data_offset: Offset to the start of data in a log record page. + * Must be a multiple of 8. On create set it to immediately after + * the update sequence array of the log record page. + * @restart_log_open_count: A counter that gets incremented every time + * the logfile is restarted which happens at mount time when the logfile + * is opened. When creating set to a random value. Win2k sets it to + * the low 32 bits of the current system time in NTFS format (see time.h). + * @reserved: Reserved/alignment to 8-byte boundary. + */ +struct restart_area { + __le64 current_lsn; + __le16 log_clients; + __le16 client_free_list; + __le16 client_in_use_list; + __le16 flags; + __le32 seq_number_bits; + __le16 restart_area_length; + __le16 client_array_offset; + __le64 file_size; + __le32 last_lsn_data_length; + __le16 log_record_header_length; + __le16 log_page_data_offset; + __le32 restart_log_open_count; + __le32 reserved; +} __packed; + +/* + * Log client record. The offset of this record is found by adding the offset + * of the RESTART_AREA to the client_array_offset value found in it. + * + * @oldest_lsn: Oldest LSN needed by this client. On create set to 0. + * @client_restart_lsn: LSN at which this client needs to restart + * the volume, i.e. the current position within the log file. + * At present, if clean this should = current_lsn in restart area but it + * probably also = current_lsn when dirty most of the time. + * At create set to 0. + * @prev_client: The offset to the previous log client record in the array + * of log client records. LOGFILE_NO_CLIENT means there is no previous + * client record, i.e. this is the first one. This is always + * LOGFILE_NO_CLIENT. + * @next_client: The offset to the next log client record in the array of + * log client records. LOGFILE_NO_CLIENT means there are no next client + * records, i.e. this is the last one. This is always LOGFILE_NO_CLIENT. + * @seq_number: On Win2k and presumably earlier, this is set to zero every + * time the logfile is restarted and it is incremented when the logfile + * is closed at dismount time. Thus it is 0 when dirty and 1 when clean. + * On WinXP and presumably later, this is always 0. + * @reserved[6]: Reserved/alignment. + * @client_name_length: Length of client name in bytes. Should always be 8. + * @client_name[64]: Name of the client in Unicode. Should always be "NTFS" + * with the remaining bytes set to 0. + */ +struct log_client_record { + __le64 oldest_lsn; + __le64 client_restart_lsn; + __le16 prev_client; + __le16 next_client; + __le16 seq_number; + u8 reserved[6]; + __le32 client_name_length; + __le16 client_name[64]; +} __packed; + +bool ntfs_check_logfile(struct inode *log_vi, + struct restart_page_header **rp); +bool ntfs_empty_logfile(struct inode *log_vi); +#endif /* _LINUX_NTFS_LOGFILE_H */ diff --git a/fs/ntfs/malloc.h b/fs/ntfs/malloc.h new file mode 100644 index 000000000..706842573 --- /dev/null +++ b/fs/ntfs/malloc.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * malloc.h - NTFS kernel memory handling. Part of the Linux-NTFS project. + * + * Copyright (c) 2001-2005 Anton Altaparmakov + */ + +#ifndef _LINUX_NTFS_MALLOC_H +#define _LINUX_NTFS_MALLOC_H + +#include +#include +#include + +/** + * __ntfs_malloc - allocate memory in multiples of pages + * @size: number of bytes to allocate + * @gfp_mask: extra flags for the allocator + * + * Internal function. You probably want ntfs_malloc_nofs()... + * + * Allocates @size bytes of memory, rounded up to multiples of PAGE_SIZE and + * returns a pointer to the allocated memory. + * + * If there was insufficient memory to complete the request, return NULL. + * Depending on @gfp_mask the allocation may be guaranteed to succeed. + */ +static inline void *__ntfs_malloc(unsigned long size, gfp_t gfp_mask) +{ + if (likely(size <= PAGE_SIZE)) { + BUG_ON(!size); + /* kmalloc() has per-CPU caches so is faster for now. */ + return kmalloc(PAGE_SIZE, gfp_mask & ~__GFP_HIGHMEM); + /* return (void *)__get_free_page(gfp_mask); */ + } + if (likely((size >> PAGE_SHIFT) < totalram_pages())) + return __vmalloc(size, gfp_mask); + return NULL; +} + +/** + * ntfs_malloc_nofs - allocate memory in multiples of pages + * @size: number of bytes to allocate + * + * Allocates @size bytes of memory, rounded up to multiples of PAGE_SIZE and + * returns a pointer to the allocated memory. + * + * If there was insufficient memory to complete the request, return NULL. + */ +static inline void *ntfs_malloc_nofs(unsigned long size) +{ + return __ntfs_malloc(size, GFP_NOFS | __GFP_HIGHMEM); +} + +/** + * ntfs_malloc_nofs_nofail - allocate memory in multiples of pages + * @size: number of bytes to allocate + * + * Allocates @size bytes of memory, rounded up to multiples of PAGE_SIZE and + * returns a pointer to the allocated memory. + * + * This function guarantees that the allocation will succeed. It will sleep + * for as long as it takes to complete the allocation. + * + * If there was insufficient memory to complete the request, return NULL. + */ +static inline void *ntfs_malloc_nofs_nofail(unsigned long size) +{ + return __ntfs_malloc(size, GFP_NOFS | __GFP_HIGHMEM | __GFP_NOFAIL); +} + +static inline void ntfs_free(void *addr) +{ + kvfree(addr); +} + +#endif /* _LINUX_NTFS_MALLOC_H */ diff --git a/fs/ntfs/mft.c b/fs/ntfs/mft.c new file mode 100644 index 000000000..fd20d7abd --- /dev/null +++ b/fs/ntfs/mft.c @@ -0,0 +1,2845 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * NTFS kernel mft record operations. + * Part of this file is based on code from the NTFS-3G. + * + * Copyright (c) 2001-2012 Anton Altaparmakov and Tuxera Inc. + * Copyright (c) 2002 Richard Russon + * Copyright (c) 2025 LG Electronics Co., Ltd. + */ + +#include +#include +#include + +#include "bitmap.h" +#include "lcnalloc.h" +#include "mft.h" +#include "ntfs.h" + +/* + * ntfs_mft_record_check - Check the consistency of an MFT record + * + * Make sure its general fields are safe, then examine all its + * attributes and apply generic checks to them. + * + * Returns 0 if the checks are successful. If not, return -EIO. + */ +int ntfs_mft_record_check(const struct ntfs_volume *vol, struct mft_record *m, + u64 mft_no) +{ + struct attr_record *a; + struct super_block *sb = vol->sb; + u16 attrs_offset; + u32 bytes_in_use; + + if (!ntfs_is_file_record(m->magic)) { + ntfs_error(sb, "Record %llu has no FILE magic (0x%x)\n", + mft_no, le32_to_cpu(*(__le32 *)m)); + goto err_out; + } + + if (le16_to_cpu(m->usa_ofs) & 0x1 || + (vol->mft_record_size >> NTFS_BLOCK_SIZE_BITS) + 1 != le16_to_cpu(m->usa_count) || + le16_to_cpu(m->usa_ofs) + le16_to_cpu(m->usa_count) * 2 > vol->mft_record_size) { + ntfs_error(sb, "Record %llu has corrupt fix-up values fields\n", + mft_no); + goto err_out; + } + + if (le32_to_cpu(m->bytes_allocated) != vol->mft_record_size) { + ntfs_error(sb, "Record %llu has corrupt allocation size (%u <> %u)\n", + mft_no, vol->mft_record_size, + le32_to_cpu(m->bytes_allocated)); + goto err_out; + } + + if (le32_to_cpu(m->bytes_in_use) > vol->mft_record_size) { + ntfs_error(sb, "Record %llu has corrupt in-use size (%u > %u)\n", + mft_no, le32_to_cpu(m->bytes_in_use), + vol->mft_record_size); + goto err_out; + } + + if (le16_to_cpu(m->attrs_offset) & 7) { + ntfs_error(sb, "Attributes badly aligned in record %llu\n", + mft_no); + goto err_out; + } + + attrs_offset = le16_to_cpu(m->attrs_offset); + bytes_in_use = le32_to_cpu(m->bytes_in_use); + + if (attrs_offset > bytes_in_use || + bytes_in_use - attrs_offset < sizeof_field(struct attr_record, type)) { + ntfs_error(sb, "Record %llu has corrupt attribute offset\n", mft_no); + goto err_out; + } + + a = (struct attr_record *)((char *)m + attrs_offset); + if ((char *)a < (char *)m || (char *)a > (char *)m + vol->mft_record_size) { + ntfs_error(sb, "Record %llu is corrupt\n", mft_no); + goto err_out; + } + + return 0; + +err_out: + return -EIO; +} + +/* + * map_mft_record_folio - map the folio in which a specific mft record resides + * @ni: ntfs inode whose mft record page to map + * + * This maps the folio in which the mft record of the ntfs inode @ni is + * situated. + * + * This allocates a new buffer (@ni->mrec), copies the MFT record data from + * the mapped folio into this buffer, and applies the MST (Multi Sector + * Transfer) fixups on the copy. + * + * The folio is pinned (referenced) in @ni->folio to ensure the data remains + * valid in the page cache, but the returned pointer is the allocated copy. + * + * Return: A pointer to the allocated and fixed-up mft record (@ni->mrec). + * The return value needs to be checked with IS_ERR(). If it is true, + * PTR_ERR() contains the negative error code. + */ +static inline struct mft_record *map_mft_record_folio(struct ntfs_inode *ni) +{ + loff_t i_size; + struct ntfs_volume *vol = ni->vol; + struct inode *mft_vi = vol->mft_ino; + struct folio *folio; + unsigned long index, end_index; + unsigned int ofs; + + WARN_ON(ni->folio); + /* + * The index into the page cache and the offset within the page cache + * page of the wanted mft record. + */ + index = NTFS_MFT_NR_TO_PIDX(vol, ni->mft_no); + ofs = NTFS_MFT_NR_TO_POFS(vol, ni->mft_no); + + i_size = i_size_read(mft_vi); + /* The maximum valid index into the page cache for $MFT's data. */ + end_index = i_size >> PAGE_SHIFT; + + /* If the wanted index is out of bounds the mft record doesn't exist. */ + if (unlikely(index >= end_index)) { + if (index > end_index || (i_size & ~PAGE_MASK) < ofs + + vol->mft_record_size) { + folio = ERR_PTR(-ENOENT); + ntfs_error(vol->sb, + "Attempt to read mft record 0x%llx, which is beyond the end of the mft. This is probably a bug in the ntfs driver.", + ni->mft_no); + goto err_out; + } + } + + /* Read, map, and pin the folio. */ + folio = read_mapping_folio(mft_vi->i_mapping, index, NULL); + if (!IS_ERR(folio)) { + u8 *addr; + + ni->mrec = kmalloc(vol->mft_record_size, GFP_NOFS); + if (!ni->mrec) { + folio_put(folio); + folio = ERR_PTR(-ENOMEM); + goto err_out; + } + + addr = kmap_local_folio(folio, 0); + memcpy(ni->mrec, addr + ofs, vol->mft_record_size); + post_read_mst_fixup((struct ntfs_record *)ni->mrec, vol->mft_record_size); + + /* Catch multi sector transfer fixup errors. */ + if (!ntfs_mft_record_check(vol, (struct mft_record *)ni->mrec, ni->mft_no)) { + kunmap_local(addr); + ni->folio = folio; + ni->folio_ofs = ofs; + return ni->mrec; + } + kunmap_local(addr); + folio_put(folio); + kfree(ni->mrec); + ni->mrec = NULL; + folio = ERR_PTR(-EIO); + NVolSetErrors(vol); + } +err_out: + ni->folio = NULL; + ni->folio_ofs = 0; + return (struct mft_record *)folio; +} + +/* + * map_mft_record - map and pin an mft record + * @ni: ntfs inode whose MFT record to map + * + * This function ensures the MFT record for the given inode is mapped and + * accessible. + * + * It increments the reference count of the ntfs inode. If the record is + * already mapped (@ni->folio is set), it returns the cached record + * immediately. + * + * Otherwise, it calls map_mft_record_folio() to read the folio from disk + * (if necessary via read_mapping_folio), allocate a buffer, and copy the + * record data. + * + * Return: A pointer to the mft record. You need to check the returned + * pointer with IS_ERR(). + */ +struct mft_record *map_mft_record(struct ntfs_inode *ni) +{ + struct mft_record *m; + + if (!ni) + return ERR_PTR(-EINVAL); + + ntfs_debug("Entering for mft_no 0x%llx.", ni->mft_no); + + /* Make sure the ntfs inode doesn't go away. */ + atomic_inc(&ni->count); + + if (ni->folio) + return (struct mft_record *)ni->mrec; + + m = map_mft_record_folio(ni); + if (!IS_ERR(m)) + return m; + + atomic_dec(&ni->count); + ntfs_error(ni->vol->sb, "Failed with error code %lu.", -PTR_ERR(m)); + return m; +} + +/* + * unmap_mft_record - release a reference to a mapped mft record + * @ni: ntfs inode whose MFT record to unmap + * + * This decrements the reference count of the ntfs inode. + * + * It releases the caller's hold on the inode. If the reference count indicates + * that there are still other users (count > 1), the function returns + * immediately, keeping the resources (folio and mrec buffer) pinned for + * those users. + * + * NOTE: If caller has modified the mft record, it is imperative to set the mft + * record dirty BEFORE calling unmap_mft_record(). + */ +void unmap_mft_record(struct ntfs_inode *ni) +{ + struct folio *folio; + + if (!ni) + return; + + ntfs_debug("Entering for mft_no 0x%llx.", ni->mft_no); + + folio = ni->folio; + if (atomic_dec_return(&ni->count) > 1) + return; + WARN_ON(!folio); +} + +/* + * map_extent_mft_record - load an extent inode and attach it to its base + * @base_ni: base ntfs inode + * @mref: mft reference of the extent inode to load + * @ntfs_ino: on successful return, pointer to the struct ntfs_inode structure + * + * Load the extent mft record @mref and attach it to its base inode @base_ni. + * Return the mapped extent mft record if IS_ERR(result) is false. Otherwise + * PTR_ERR(result) gives the negative error code. + * + * On successful return, @ntfs_ino contains a pointer to the ntfs_inode + * structure of the mapped extent inode. + */ +struct mft_record *map_extent_mft_record(struct ntfs_inode *base_ni, u64 mref, + struct ntfs_inode **ntfs_ino) +{ + struct mft_record *m; + struct ntfs_inode *ni = NULL; + struct ntfs_inode **extent_nis = NULL; + int i; + u64 mft_no = MREF(mref); + u16 seq_no = MSEQNO(mref); + bool destroy_ni = false; + + ntfs_debug("Mapping extent mft record 0x%llx (base mft record 0x%llx).", + mft_no, base_ni->mft_no); + /* Make sure the base ntfs inode doesn't go away. */ + atomic_inc(&base_ni->count); + /* + * Check if this extent inode has already been added to the base inode, + * in which case just return it. If not found, add it to the base + * inode before returning it. + */ +retry: + mutex_lock(&base_ni->extent_lock); + if (base_ni->nr_extents > 0) { + extent_nis = base_ni->ext.extent_ntfs_inos; + for (i = 0; i < base_ni->nr_extents; i++) { + if (mft_no != extent_nis[i]->mft_no) + continue; + ni = extent_nis[i]; + /* Make sure the ntfs inode doesn't go away. */ + atomic_inc(&ni->count); + break; + } + } + if (likely(ni != NULL)) { + mutex_unlock(&base_ni->extent_lock); + atomic_dec(&base_ni->count); + /* We found the record; just have to map and return it. */ + m = map_mft_record(ni); + /* map_mft_record() has incremented this on success. */ + atomic_dec(&ni->count); + if (!IS_ERR(m)) { + /* Verify the sequence number. */ + if (likely(le16_to_cpu(m->sequence_number) == seq_no)) { + ntfs_debug("Done 1."); + *ntfs_ino = ni; + return m; + } + unmap_mft_record(ni); + ntfs_error(base_ni->vol->sb, + "Found stale extent mft reference! Corrupt filesystem. Run chkdsk."); + return ERR_PTR(-EIO); + } +map_err_out: + ntfs_error(base_ni->vol->sb, + "Failed to map extent mft record, error code %ld.", + -PTR_ERR(m)); + return m; + } + mutex_unlock(&base_ni->extent_lock); + + /* Record wasn't there. Get a new ntfs inode and initialize it. */ + ni = ntfs_new_extent_inode(base_ni->vol->sb, mft_no); + if (unlikely(!ni)) { + atomic_dec(&base_ni->count); + return ERR_PTR(-ENOMEM); + } + ni->vol = base_ni->vol; + ni->seq_no = seq_no; + ni->nr_extents = -1; + ni->ext.base_ntfs_ino = base_ni; + /* Now map the record. */ + m = map_mft_record(ni); + if (IS_ERR(m)) { + atomic_dec(&base_ni->count); + ntfs_clear_extent_inode(ni); + goto map_err_out; + } + /* Verify the sequence number if it is present. */ + if (seq_no && (le16_to_cpu(m->sequence_number) != seq_no)) { + ntfs_error(base_ni->vol->sb, + "Found stale extent mft reference! Corrupt filesystem. Run chkdsk."); + destroy_ni = true; + m = ERR_PTR(-EIO); + goto unm_nolock_err_out; + } + + mutex_lock(&base_ni->extent_lock); + for (i = 0; i < base_ni->nr_extents; i++) { + if (mft_no == extent_nis[i]->mft_no) { + mutex_unlock(&base_ni->extent_lock); + ntfs_clear_extent_inode(ni); + goto retry; + } + } + /* Attach extent inode to base inode, reallocating memory if needed. */ + if (!(base_ni->nr_extents & 3)) { + struct ntfs_inode **tmp; + int new_size = (base_ni->nr_extents + 4) * sizeof(struct ntfs_inode *); + + tmp = kvzalloc(new_size, GFP_NOFS); + if (unlikely(!tmp)) { + ntfs_error(base_ni->vol->sb, "Failed to allocate internal buffer."); + destroy_ni = true; + m = ERR_PTR(-ENOMEM); + goto unm_err_out; + } + if (base_ni->nr_extents) { + WARN_ON(!base_ni->ext.extent_ntfs_inos); + memcpy(tmp, base_ni->ext.extent_ntfs_inos, new_size - + 4 * sizeof(struct ntfs_inode *)); + kvfree(base_ni->ext.extent_ntfs_inos); + } + base_ni->ext.extent_ntfs_inos = tmp; + } + base_ni->ext.extent_ntfs_inos[base_ni->nr_extents++] = ni; + mutex_unlock(&base_ni->extent_lock); + atomic_dec(&base_ni->count); + ntfs_debug("Done 2."); + *ntfs_ino = ni; + return m; +unm_err_out: + mutex_unlock(&base_ni->extent_lock); +unm_nolock_err_out: + unmap_mft_record(ni); + atomic_dec(&base_ni->count); + /* + * If the extent inode was not attached to the base inode we need to + * release it or we will leak memory. + */ + if (destroy_ni) + ntfs_clear_extent_inode(ni); + return m; +} + +/* + * __mark_mft_record_dirty - mark the base vfs inode dirty + * @ni: ntfs inode describing the mapped mft record + * + * Internal function. Users should call mark_mft_record_dirty() instead. + * + * This function determines the base ntfs inode (in case @ni is an extent + * inode) and marks the corresponding VFS inode dirty. + * + * NOTE: We only set I_DIRTY_DATASYNC (and not I_DIRTY_PAGES) + * on the base vfs inode, because even though file data may have been modified, + * it is dirty in the inode meta data rather than the data page cache of the + * inode, and thus there are no data pages that need writing out. Therefore, a + * full mark_inode_dirty() is overkill. A mark_inode_dirty_sync(), on the + * other hand, is not sufficient, because ->write_inode needs to be called even + * in case of fdatasync. This needs to happen or the file data would not + * necessarily hit the device synchronously, even though the vfs inode has the + * O_SYNC flag set. Also, I_DIRTY_DATASYNC simply "feels" better than just + * I_DIRTY_SYNC, since the file data has not actually hit the block device yet, + * which is not what I_DIRTY_SYNC on its own would suggest. + */ +void __mark_mft_record_dirty(struct ntfs_inode *ni) +{ + struct ntfs_inode *base_ni; + + ntfs_debug("Entering for inode 0x%llx.", ni->mft_no); + WARN_ON(NInoAttr(ni)); + /* Determine the base vfs inode and mark it dirty, too. */ + if (likely(ni->nr_extents >= 0)) + base_ni = ni; + else + base_ni = ni->ext.base_ntfs_ino; + __mark_inode_dirty(VFS_I(base_ni), I_DIRTY_DATASYNC); +} + +/* + * ntfs_bio_end_io - bio completion callback for MFT record writes + * + * Decrements the folio reference count that was incremented before + * submit_bio(). This prevents a race condition where umount could + * evict the inode and release the folio while I/O is still in flight, + * potentially causing data corruption or use-after-free. + */ +static void ntfs_bio_end_io(struct bio *bio) +{ + if (bio->bi_private) + folio_put((struct folio *)bio->bi_private); + bio_put(bio); +} + +/* + * ntfs_sync_mft_mirror - synchronize an mft record to the mft mirror + * @vol: ntfs volume on which the mft record to synchronize resides + * @mft_no: mft record number of mft record to synchronize + * @m: mapped, mst protected (extent) mft record to synchronize + * + * Write the mapped, mst protected (extent) mft record @m with mft record + * number @mft_no to the mft mirror ($MFTMirr) of the ntfs volume @vol. + * + * On success return 0. On error return -errno and set the volume errors flag + * in the ntfs volume @vol. + * + * NOTE: We always perform synchronous i/o. + */ +int ntfs_sync_mft_mirror(struct ntfs_volume *vol, const u64 mft_no, + struct mft_record *m) +{ + u8 *kmirr; + struct folio *folio; + unsigned int folio_ofs, lcn_folio_off = 0; + int err = 0; + struct bio *bio; + + ntfs_debug("Entering for inode 0x%llx.", mft_no); + + if (unlikely(!vol->mftmirr_ino)) { + /* This could happen during umount... */ + err = -EIO; + goto err_out; + } + /* Get the page containing the mirror copy of the mft record @m. */ + folio = read_mapping_folio(vol->mftmirr_ino->i_mapping, + NTFS_MFT_NR_TO_PIDX(vol, mft_no), NULL); + if (IS_ERR(folio)) { + ntfs_error(vol->sb, "Failed to map mft mirror page."); + err = PTR_ERR(folio); + goto err_out; + } + + folio_lock(folio); + folio_clear_uptodate(folio); + /* Offset of the mft mirror record inside the page. */ + folio_ofs = NTFS_MFT_NR_TO_POFS(vol, mft_no); + /* The address in the page of the mirror copy of the mft record @m. */ + kmirr = kmap_local_folio(folio, 0) + folio_ofs; + /* Copy the mst protected mft record to the mirror. */ + memcpy(kmirr, m, vol->mft_record_size); + kunmap_local(kmirr); + + if (vol->cluster_size_bits > PAGE_SHIFT) { + lcn_folio_off = folio->index << PAGE_SHIFT; + lcn_folio_off &= vol->cluster_size_mask; + } + + bio = bio_alloc(vol->sb->s_bdev, 1, REQ_OP_WRITE, GFP_NOIO); + bio->bi_iter.bi_sector = + NTFS_B_TO_SECTOR(vol, NTFS_CLU_TO_B(vol, vol->mftmirr_lcn) + + lcn_folio_off + folio_ofs); + + if (bio_add_folio(bio, folio, vol->mft_record_size, folio_ofs)) + err = submit_bio_wait(bio); + else + err = -EIO; + bio_put(bio); + + /* + * The in-memory mirror is now valid because we just memcpy()'d the + * mst-protected mft record into it. Mark the folio uptodate even on + * write error so a subsequent read_mapping_folio() does not refetch + * the stale on-disk mirror and overwrite this copy. The error is + * propagated to the caller via @err. + */ + folio_mark_uptodate(folio); + + folio_unlock(folio); + folio_put(folio); + if (likely(!err)) { + ntfs_debug("Done."); + } else { + ntfs_error(vol->sb, "I/O error while writing mft mirror record 0x%llx!", mft_no); +err_out: + ntfs_error(vol->sb, + "Failed to synchronize $MFTMirr (error code %i). Volume will be left marked dirty on umount. Run chkdsk on the partition after umounting to correct this.", + err); + NVolSetErrors(vol); + } + return err; +} + +/* + * write_mft_record_nolock - write out a mapped (extent) mft record + * @ni: ntfs inode describing the mapped (extent) mft record + * @m: mapped (extent) mft record to write + * @sync: if true, wait for i/o completion + * + * Write the mapped (extent) mft record @m described by the (regular or extent) + * ntfs inode @ni to backing store. If the mft record @m has a counterpart in + * the mft mirror, that is also updated. + * + * We only write the mft record if the ntfs inode @ni is dirty. + * + * On success, clean the mft record and return 0. + * On error (specifically ENOMEM), we redirty the record so it can be retried. + * For other errors, we mark the volume with errors. + */ +int write_mft_record_nolock(struct ntfs_inode *ni, struct mft_record *m, int sync) +{ + struct ntfs_volume *vol = ni->vol; + struct folio *folio = ni->folio; + int err = 0, i = 0; + u8 *kaddr; + struct mft_record *fixup_m; + struct bio *bio; + unsigned int offset = 0, folio_size; + + ntfs_debug("Entering for inode 0x%llx.", ni->mft_no); + + WARN_ON(NInoAttr(ni)); + WARN_ON(!folio_test_locked(folio)); + + /* + * If the struct ntfs_inode is clean no need to do anything. If it is dirty, + * mark it as clean now so that it can be redirtied later on if needed. + * There is no danger of races since the caller is holding the locks + * for the mft record @m and the page it is in. + */ + if (!NInoTestClearDirty(ni)) + goto done; + + kaddr = kmap_local_folio(folio, 0); + fixup_m = (struct mft_record *)(kaddr + ni->folio_ofs); + memcpy(fixup_m, m, vol->mft_record_size); + + /* Apply the mst protection fixups. */ + err = pre_write_mst_fixup((struct ntfs_record *)fixup_m, vol->mft_record_size); + if (err) { + ntfs_error(vol->sb, "Failed to apply mst fixups!"); + goto err_out; + } + + folio_size = vol->mft_record_size / ni->mft_lcn_count; + while (i < ni->mft_lcn_count) { + unsigned int clu_off; + + clu_off = (unsigned int)((s64)ni->mft_no * vol->mft_record_size + offset) & + vol->cluster_size_mask; + + bio = bio_alloc(vol->sb->s_bdev, 1, REQ_OP_WRITE, GFP_NOIO); + bio->bi_iter.bi_sector = + NTFS_B_TO_SECTOR(vol, NTFS_CLU_TO_B(vol, ni->mft_lcn[i]) + + clu_off); + + if (!bio_add_folio(bio, folio, folio_size, + ni->folio_ofs + offset)) { + err = -EIO; + goto put_bio_out; + } + + /* Synchronize the mft mirror now if not @sync. */ + if (!sync && ni->mft_no < vol->mftmirr_size) { + int sub_err = ntfs_sync_mft_mirror(vol, ni->mft_no, + fixup_m); + if (unlikely(sub_err) && !err) + err = sub_err; + } + + if (sync) { + int sub_err = submit_bio_wait(bio); + + bio_put(bio); + if (unlikely(sub_err) && !err) + err = sub_err; + } else { + folio_get(folio); + bio->bi_private = folio; + bio->bi_end_io = ntfs_bio_end_io; + submit_bio(bio); + } + offset += vol->cluster_size; + i++; + } + + /* If @sync, now synchronize the mft mirror. */ + if (sync && ni->mft_no < vol->mftmirr_size) { + int sub_err = ntfs_sync_mft_mirror(vol, ni->mft_no, fixup_m); + + if (unlikely(sub_err) && !err) + err = sub_err; + } + kunmap_local(kaddr); + if (unlikely(err)) { + /* I/O error during writing. This is really bad! */ + ntfs_error(vol->sb, + "I/O error while writing mft record 0x%llx! Marking base inode as bad. You should unmount the volume and run chkdsk.", + ni->mft_no); + goto err_out; + } +done: + ntfs_debug("Done."); + return 0; +put_bio_out: + bio_put(bio); +err_out: + /* + * The caller should mark the base inode as bad so no more I/O + * happens. ->drop_inode() will still be invoked so all extent inodes + * and other allocated memory will be freed. ENOMEM is retried by + * redirtying the mft record below. + */ + if (err == -ENOMEM) { + ntfs_error(vol->sb, + "Not enough memory to write mft record. Redirtying so the write is retried later."); + mark_mft_record_dirty(ni); + err = 0; + } else + NVolSetErrors(vol); + return err; +} + +static int ntfs_test_inode_wb(struct inode *vi, u64 ino, void *data) +{ + struct ntfs_attr *na = data; + + if (!ntfs_test_inode(vi, na)) + return 0; + + /* + * Without this, ntfs_write_mst_block() could call iput_final() + * , and ntfs_evict_big_inode() could try to unlink this inode + * and the contex could be blocked infinitly in map_mft_record(). + */ + if (NInoBeingDeleted(NTFS_I(vi))) { + na->state = NI_BeingDeleted; + return -1; + } + + /* + * This condition can prevent ntfs_write_mst_block() + * from applying/undo fixups while ntfs_create() being + * called + */ + spin_lock(&vi->i_lock); + if (inode_state_read_once(vi) & I_CREATING) { + spin_unlock(&vi->i_lock); + na->state = NI_BeingCreated; + return -1; + } + spin_unlock(&vi->i_lock); + + return igrab(vi) ? 1 : -1; +} + +/* + * ntfs_may_write_mft_record - check if an mft record may be written out + * @vol: [IN] ntfs volume on which the mft record to check resides + * @mft_no: [IN] mft record number of the mft record to check + * @m: [IN] mapped mft record to check + * @locked_ni: [OUT] caller has to unlock this ntfs inode if one is returned + * @ref_vi: [OUT] caller has to drop this vfs inode if one is returned + * + * Check if the mapped (base or extent) mft record @m with mft record number + * @mft_no belonging to the ntfs volume @vol may be written out. If necessary + * and possible the ntfs inode of the mft record is locked and the base vfs + * inode is pinned. The locked ntfs inode is then returned in @locked_ni. The + * caller is responsible for unlocking the ntfs inode and unpinning the base + * vfs inode. + * + * To avoid deadlock when the caller holds a folio lock, if the function + * returns @ref_vi it defers dropping the vfs inode reference by returning + * it in @ref_vi instead of calling iput() directly. The caller must call + * iput() on @ref_vi after releasing the folio lock. + * + * Return 'true' if the mft record may be written out and 'false' if not. + * + * The caller has locked the page and cleared the uptodate flag on it which + * means that we can safely write out any dirty mft records that do not have + * their inodes in icache as determined by find_inode_nowait(). + * + * Here is a description of the tests we perform: + * + * If the inode is found in icache we know the mft record must be a base mft + * record. If it is dirty, we do not write it and return 'false' as the vfs + * inode write paths will result in the access times being updated which would + * cause the base mft record to be redirtied and written out again. + * + * If the inode is in icache and not dirty, we attempt to lock the mft record + * and if we find the lock was already taken, it is not safe to write the mft + * record and we return 'false'. + * + * If we manage to obtain the lock we have exclusive access to the mft record, + * which also allows us safe writeout of the mft record. We then set + * @locked_ni to the locked ntfs inode and return 'true'. + * + * Note we cannot just lock the mft record and sleep while waiting for the lock + * because this would deadlock due to lock reversal. + * + * If the inode is not in icache we need to perform further checks. + * + * If the mft record is not a FILE record or it is a base mft record, we can + * safely write it and return 'true'. + */ +static bool ntfs_may_write_mft_record(struct ntfs_volume *vol, const u64 mft_no, + const struct mft_record *m, struct ntfs_inode **locked_ni, + struct inode **ref_vi) +{ + struct super_block *sb = vol->sb; + struct inode *mft_vi = vol->mft_ino; + struct inode *vi; + struct ntfs_inode *ni; + struct ntfs_attr na = {0}; + + ntfs_debug("Entering for inode 0x%llx.", mft_no); + /* + * Normally we do not return a locked inode so set @locked_ni to NULL. + */ + *locked_ni = NULL; + *ref_vi = NULL; + + /* + * Check if the inode corresponding to this mft record is in the VFS + * inode cache and obtain a reference to it if it is. + */ + ntfs_debug("Looking for inode 0x%llx in icache.", mft_no); + na.mft_no = mft_no; + na.type = AT_UNUSED; + /* + * Optimize inode 0, i.e. $MFT itself, since we have it in memory and + * we get here for it rather often. + */ + if (!mft_no) { + /* Balance the below iput(). */ + vi = igrab(mft_vi); + WARN_ON(vi != mft_vi); + } else { + /* + * Have to use find_inode_nowait() since ilookup5_nowait() + * waits for inode with I_FREEING, which causes ntfs to deadlock + * when inodes are unlinked concurrently + */ + vi = find_inode_nowait(sb, mft_no, ntfs_test_inode_wb, &na); + if (na.state == NI_BeingDeleted || na.state == NI_BeingCreated) + return false; + } + if (vi) { + ntfs_debug("Base inode 0x%llx is in icache.", mft_no); + /* The inode is in icache. */ + ni = NTFS_I(vi); + /* Take a reference to the ntfs inode. */ + atomic_inc(&ni->count); + /* If the inode is dirty, do not write this record. */ + if (NInoDirty(ni)) { + ntfs_debug("Inode 0x%llx is dirty, do not write it.", + mft_no); + atomic_dec(&ni->count); + *ref_vi = vi; + return false; + } + ntfs_debug("Inode 0x%llx is not dirty.", mft_no); + /* The inode is not dirty, try to take the mft record lock. */ + if (unlikely(!mutex_trylock(&ni->mrec_lock))) { + ntfs_debug("Mft record 0x%llx is already locked, do not write it.", mft_no); + atomic_dec(&ni->count); + *ref_vi = vi; + return false; + } + ntfs_debug("Managed to lock mft record 0x%llx, write it.", + mft_no); + /* + * The write has to occur while we hold the mft record lock so + * return the locked ntfs inode. + */ + *locked_ni = ni; + return true; + } + ntfs_debug("Inode 0x%llx is not in icache.", mft_no); + /* The inode is not in icache. */ + /* Write the record if it is not a mft record (type "FILE"). */ + if (!ntfs_is_mft_record(m->magic)) { + ntfs_debug("Mft record 0x%llx is not a FILE record, write it.", + mft_no); + return true; + } + /* Write the mft record if it is a base inode. */ + if (!m->base_mft_record) { + ntfs_debug("Mft record 0x%llx is a base record, write it.", + mft_no); + return true; + } + + ntfs_debug("Mft record 0x%llx is an extent record, skip it.", + mft_no); + return false; +} + +static const char *es = " Leaving inconsistent metadata. Unmount and run chkdsk."; + +#define RESERVED_MFT_RECORDS 64 + +/* + * ntfs_mft_bitmap_find_and_alloc_free_rec_nolock - see name + * @vol: volume on which to search for a free mft record + * @base_ni: open base inode if allocating an extent mft record or NULL + * + * Search for a free mft record in the mft bitmap attribute on the ntfs volume + * @vol. + * + * If @base_ni is NULL start the search at the default allocator position. + * + * If @base_ni is not NULL start the search at the mft record after the base + * mft record @base_ni. + * + * Return the free mft record on success and -errno on error. An error code of + * -ENOSPC means that there are no free mft records in the currently + * initialized mft bitmap. + * + * Locking: Caller must hold vol->mftbmp_lock for writing. + */ +static s64 ntfs_mft_bitmap_find_and_alloc_free_rec_nolock(struct ntfs_volume *vol, + struct ntfs_inode *base_ni) +{ + s64 pass_end, ll, data_pos, pass_start, ofs, bit; + unsigned long flags; + struct address_space *mftbmp_mapping; + u8 *buf = NULL, *byte; + struct folio *folio; + unsigned int folio_ofs, size; + u8 pass, b; + + ntfs_debug("Searching for free mft record in the currently initialized mft bitmap."); + mftbmp_mapping = vol->mftbmp_ino->i_mapping; + /* + * Set the end of the pass making sure we do not overflow the mft + * bitmap. + */ + read_lock_irqsave(&NTFS_I(vol->mft_ino)->size_lock, flags); + pass_end = NTFS_I(vol->mft_ino)->allocated_size >> + vol->mft_record_size_bits; + read_unlock_irqrestore(&NTFS_I(vol->mft_ino)->size_lock, flags); + read_lock_irqsave(&NTFS_I(vol->mftbmp_ino)->size_lock, flags); + ll = NTFS_I(vol->mftbmp_ino)->initialized_size << 3; + read_unlock_irqrestore(&NTFS_I(vol->mftbmp_ino)->size_lock, flags); + if (pass_end > ll) + pass_end = ll; + pass = 1; + if (!base_ni) + data_pos = vol->mft_data_pos; + else + data_pos = base_ni->mft_no + 1; + if (data_pos < RESERVED_MFT_RECORDS) + data_pos = RESERVED_MFT_RECORDS; + if (data_pos >= pass_end) { + data_pos = RESERVED_MFT_RECORDS; + pass = 2; + /* This happens on a freshly formatted volume. */ + if (data_pos >= pass_end) + return -ENOSPC; + } + + if (base_ni && base_ni->mft_no == FILE_MFT) { + data_pos = 0; + pass = 2; + } + + pass_start = data_pos; + ntfs_debug("Starting bitmap search: pass %u, pass_start 0x%llx, pass_end 0x%llx, data_pos 0x%llx.", + pass, pass_start, pass_end, data_pos); + /* Loop until a free mft record is found. */ + for (; pass <= 2;) { + /* Cap size to pass_end. */ + ofs = data_pos >> 3; + folio_ofs = ofs & ~PAGE_MASK; + size = PAGE_SIZE - folio_ofs; + ll = ((pass_end + 7) >> 3) - ofs; + if (size > ll) + size = ll; + size <<= 3; + /* + * If we are still within the active pass, search the next page + * for a zero bit. + */ + if (size) { + folio = read_mapping_folio(mftbmp_mapping, + ofs >> PAGE_SHIFT, NULL); + if (IS_ERR(folio)) { + ntfs_error(vol->sb, "Failed to read mft bitmap, aborting."); + return PTR_ERR(folio); + } + folio_lock(folio); + buf = (u8 *)kmap_local_folio(folio, 0) + folio_ofs; + bit = data_pos & 7; + data_pos &= ~7ull; + ntfs_debug("Before inner for loop: size 0x%x, data_pos 0x%llx, bit 0x%llx", + size, data_pos, bit); + for (; bit < size && data_pos + bit < pass_end; + bit &= ~7ull, bit += 8) { + /* + * If we're extending $MFT and running out of the first + * mft record (base record) then give up searching since + * no guarantee that the found record will be accessible. + */ + if (base_ni && base_ni->mft_no == FILE_MFT && bit > 400) { + folio_unlock(folio); + kunmap_local(buf); + folio_put(folio); + return -ENOSPC; + } + + byte = buf + (bit >> 3); + if (*byte == 0xff) + continue; + b = ffz((unsigned long)*byte); + if (b < 8 && b >= (bit & 7)) { + ll = data_pos + (bit & ~7ull) + b; + if (unlikely(ll >= (1ll << 32))) { + folio_unlock(folio); + kunmap_local(buf); + folio_put(folio); + return -ENOSPC; + } + *byte |= 1 << b; + folio_mark_dirty(folio); + folio_unlock(folio); + kunmap_local(buf); + folio_put(folio); + ntfs_debug("Done. (Found and allocated mft record 0x%llx.)", + ll); + return ll; + } + } + ntfs_debug("After inner for loop: size 0x%x, data_pos 0x%llx, bit 0x%llx", + size, data_pos, bit); + data_pos += size; + folio_unlock(folio); + kunmap_local(buf); + folio_put(folio); + /* + * If the end of the pass has not been reached yet, + * continue searching the mft bitmap for a zero bit. + */ + if (data_pos < pass_end) + continue; + } + /* Do the next pass. */ + if (++pass == 2) { + /* + * Starting the second pass, in which we scan the first + * part of the zone which we omitted earlier. + */ + pass_end = pass_start; + data_pos = pass_start = RESERVED_MFT_RECORDS; + ntfs_debug("pass %i, pass_start 0x%llx, pass_end 0x%llx.", + pass, pass_start, pass_end); + if (data_pos >= pass_end) + break; + } + } + /* No free mft records in currently initialized mft bitmap. */ + ntfs_debug("Done. (No free mft records left in currently initialized mft bitmap.)"); + return -ENOSPC; +} + +static int ntfs_mft_attr_extend(struct ntfs_inode *ni) +{ + int ret = 0; + struct ntfs_inode *base_ni; + + if (NInoAttr(ni)) + base_ni = ni->ext.base_ntfs_ino; + else + base_ni = ni; + + if (!NInoAttrList(base_ni)) { + ret = ntfs_inode_add_attrlist(base_ni); + if (ret) { + pr_err("Can not add attrlist\n"); + goto out; + } else { + ret = -EAGAIN; + goto out; + } + } + + ret = ntfs_attr_update_mapping_pairs(ni, 0); + if (ret) + pr_err("MP update failed\n"); + +out: + return ret; +} + +/* + * ntfs_mft_bitmap_extend_allocation_nolock - extend mft bitmap by a cluster + * @vol: volume on which to extend the mft bitmap attribute + * + * Extend the mft bitmap attribute on the ntfs volume @vol by one cluster. + * + * Note: Only changes allocated_size, i.e. does not touch initialized_size or + * data_size. + * + * Return 0 on success and -errno on error. + * + * Locking: - Caller must hold vol->mftbmp_lock for writing. + * - This function takes NTFS_I(vol->mftbmp_ino)->runlist.lock for + * writing and releases it before returning. + * - This function takes vol->lcnbmp_lock for writing and releases it + * before returning. + */ +static int ntfs_mft_bitmap_extend_allocation_nolock(struct ntfs_volume *vol) +{ + s64 lcn; + s64 ll; + unsigned long flags; + struct folio *folio; + struct ntfs_inode *mft_ni, *mftbmp_ni; + struct runlist_element *rl, *rl2 = NULL; + struct ntfs_attr_search_ctx *ctx = NULL; + struct mft_record *mrec; + struct attr_record *a = NULL; + int ret, mp_size; + u32 old_alen = 0; + u8 *b, tb; + struct { + u8 added_cluster:1; + u8 added_run:1; + u8 mp_rebuilt:1; + u8 mp_extended:1; + } status = { 0, 0, 0, 0 }; + size_t new_rl_count; + + ntfs_debug("Extending mft bitmap allocation."); + mft_ni = NTFS_I(vol->mft_ino); + mftbmp_ni = NTFS_I(vol->mftbmp_ino); + /* + * Determine the last lcn of the mft bitmap. The allocated size of the + * mft bitmap cannot be zero so we are ok to do this. + */ + down_write(&mftbmp_ni->runlist.lock); + read_lock_irqsave(&mftbmp_ni->size_lock, flags); + ll = mftbmp_ni->allocated_size; + read_unlock_irqrestore(&mftbmp_ni->size_lock, flags); + rl = ntfs_attr_find_vcn_nolock(mftbmp_ni, + NTFS_B_TO_CLU(vol, ll - 1), NULL); + if (IS_ERR(rl) || unlikely(!rl->length || rl->lcn < 0)) { + up_write(&mftbmp_ni->runlist.lock); + ntfs_error(vol->sb, + "Failed to determine last allocated cluster of mft bitmap attribute."); + if (!IS_ERR(rl)) + ret = -EIO; + else + ret = PTR_ERR(rl); + return ret; + } + lcn = rl->lcn + rl->length; + ntfs_debug("Last lcn of mft bitmap attribute is 0x%llx.", + (long long)lcn); + /* + * Attempt to get the cluster following the last allocated cluster by + * hand as it may be in the MFT zone so the allocator would not give it + * to us. + */ + ll = lcn >> 3; + folio = read_mapping_folio(vol->lcnbmp_ino->i_mapping, + ll >> PAGE_SHIFT, NULL); + if (IS_ERR(folio)) { + up_write(&mftbmp_ni->runlist.lock); + ntfs_error(vol->sb, "Failed to read from lcn bitmap."); + return PTR_ERR(folio); + } + + down_write(&vol->lcnbmp_lock); + folio_lock(folio); + b = (u8 *)kmap_local_folio(folio, 0) + (ll & ~PAGE_MASK); + tb = 1 << (lcn & 7ull); + if (*b != 0xff && !(*b & tb)) { + /* Next cluster is free, allocate it. */ + *b |= tb; + folio_mark_dirty(folio); + folio_unlock(folio); + kunmap_local(b); + folio_put(folio); + up_write(&vol->lcnbmp_lock); + /* Update the mft bitmap runlist. */ + rl->length++; + rl[1].vcn++; + status.added_cluster = 1; + ntfs_debug("Appending one cluster to mft bitmap."); + } else { + folio_unlock(folio); + kunmap_local(b); + folio_put(folio); + up_write(&vol->lcnbmp_lock); + /* Allocate a cluster from the DATA_ZONE. */ + rl2 = ntfs_cluster_alloc(vol, rl[1].vcn, 1, lcn, DATA_ZONE, + true, false, false); + if (IS_ERR(rl2)) { + up_write(&mftbmp_ni->runlist.lock); + ntfs_error(vol->sb, + "Failed to allocate a cluster for the mft bitmap."); + return PTR_ERR(rl2); + } + rl = ntfs_runlists_merge(&mftbmp_ni->runlist, rl2, 0, &new_rl_count); + if (IS_ERR(rl)) { + up_write(&mftbmp_ni->runlist.lock); + ntfs_error(vol->sb, "Failed to merge runlists for mft bitmap."); + if (ntfs_cluster_free_from_rl(vol, rl2)) { + ntfs_error(vol->sb, "Failed to deallocate allocated cluster.%s", + es); + NVolSetErrors(vol); + } + kvfree(rl2); + return PTR_ERR(rl); + } + mftbmp_ni->runlist.rl = rl; + mftbmp_ni->runlist.count = new_rl_count; + status.added_run = 1; + ntfs_debug("Adding one run to mft bitmap."); + /* Find the last run in the new runlist. */ + for (; rl[1].length; rl++) + ; + } + /* + * Update the attribute record as well. Note: @rl is the last + * (non-terminator) runlist element of mft bitmap. + */ + mrec = map_mft_record(mft_ni); + if (IS_ERR(mrec)) { + ntfs_error(vol->sb, "Failed to map mft record."); + ret = PTR_ERR(mrec); + goto undo_alloc; + } + ctx = ntfs_attr_get_search_ctx(mft_ni, mrec); + if (unlikely(!ctx)) { + ntfs_error(vol->sb, "Failed to get search context."); + ret = -ENOMEM; + goto undo_alloc; + } + ret = ntfs_attr_lookup(mftbmp_ni->type, mftbmp_ni->name, + mftbmp_ni->name_len, CASE_SENSITIVE, rl[1].vcn, NULL, + 0, ctx); + if (unlikely(ret)) { + ntfs_error(vol->sb, + "Failed to find last attribute extent of mft bitmap attribute."); + if (ret == -ENOENT) + ret = -EIO; + goto undo_alloc; + } + a = ctx->attr; + ll = le64_to_cpu(a->data.non_resident.lowest_vcn); + /* Search back for the previous last allocated cluster of mft bitmap. */ + for (rl2 = rl; rl2 > mftbmp_ni->runlist.rl; rl2--) { + if (ll >= rl2->vcn) + break; + } + WARN_ON(ll < rl2->vcn); + WARN_ON(ll >= rl2->vcn + rl2->length); + /* Get the size for the new mapping pairs array for this extent. */ + mp_size = ntfs_get_size_for_mapping_pairs(vol, rl2, ll, -1, -1); + if (unlikely(mp_size <= 0)) { + ntfs_error(vol->sb, + "Get size for mapping pairs failed for mft bitmap attribute extent."); + ret = mp_size; + if (!ret) + ret = -EIO; + goto undo_alloc; + } + /* Expand the attribute record if necessary. */ + old_alen = le32_to_cpu(a->length); + ret = ntfs_attr_record_resize(ctx->mrec, a, mp_size + + le16_to_cpu(a->data.non_resident.mapping_pairs_offset)); + if (unlikely(ret)) { + ret = ntfs_mft_attr_extend(mftbmp_ni); + if (!ret) + goto extended_ok; + if (ret != -EAGAIN) + status.mp_extended = 1; + goto undo_alloc; + } + status.mp_rebuilt = 1; + /* Generate the mapping pairs array directly into the attr record. */ + ret = ntfs_mapping_pairs_build(vol, (u8 *)a + + le16_to_cpu(a->data.non_resident.mapping_pairs_offset), + mp_size, rl2, ll, -1, NULL, NULL, NULL); + if (unlikely(ret)) { + ntfs_error(vol->sb, + "Failed to build mapping pairs array for mft bitmap attribute."); + goto undo_alloc; + } + /* Update the highest_vcn. */ + a->data.non_resident.highest_vcn = cpu_to_le64(rl[1].vcn - 1); + /* + * We now have extended the mft bitmap allocated_size by one cluster. + * Reflect this in the struct ntfs_inode structure and the attribute record. + */ + if (a->data.non_resident.lowest_vcn) { + /* + * We are not in the first attribute extent, switch to it, but + * first ensure the changes will make it to disk later. + */ + mark_mft_record_dirty(ctx->ntfs_ino); +extended_ok: + ntfs_attr_reinit_search_ctx(ctx); + ret = ntfs_attr_lookup(mftbmp_ni->type, mftbmp_ni->name, + mftbmp_ni->name_len, CASE_SENSITIVE, 0, NULL, + 0, ctx); + if (unlikely(ret)) { + ntfs_error(vol->sb, + "Failed to find first attribute extent of mft bitmap attribute."); + goto restore_undo_alloc; + } + a = ctx->attr; + } + + write_lock_irqsave(&mftbmp_ni->size_lock, flags); + mftbmp_ni->allocated_size += vol->cluster_size; + a->data.non_resident.allocated_size = + cpu_to_le64(mftbmp_ni->allocated_size); + write_unlock_irqrestore(&mftbmp_ni->size_lock, flags); + /* Ensure the changes make it to disk. */ + mark_mft_record_dirty(ctx->ntfs_ino); + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(mft_ni); + up_write(&mftbmp_ni->runlist.lock); + ntfs_debug("Done."); + return 0; + +restore_undo_alloc: + ntfs_attr_reinit_search_ctx(ctx); + if (ntfs_attr_lookup(mftbmp_ni->type, mftbmp_ni->name, + mftbmp_ni->name_len, CASE_SENSITIVE, rl[1].vcn, NULL, + 0, ctx)) { + ntfs_error(vol->sb, + "Failed to find last attribute extent of mft bitmap attribute.%s", es); + write_lock_irqsave(&mftbmp_ni->size_lock, flags); + mftbmp_ni->allocated_size += vol->cluster_size; + write_unlock_irqrestore(&mftbmp_ni->size_lock, flags); + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(mft_ni); + up_write(&mftbmp_ni->runlist.lock); + /* + * The only thing that is now wrong is ->allocated_size of the + * base attribute extent which chkdsk should be able to fix. + */ + NVolSetErrors(vol); + return ret; + } + a = ctx->attr; + a->data.non_resident.highest_vcn = cpu_to_le64(rl[1].vcn - 2); +undo_alloc: + if (status.added_cluster) { + /* Truncate the last run in the runlist by one cluster. */ + rl->length--; + rl[1].vcn--; + } else if (status.added_run) { + lcn = rl->lcn; + /* Remove the last run from the runlist. */ + rl->lcn = rl[1].lcn; + rl->length = 0; + mftbmp_ni->runlist.count--; + } + /* Deallocate the cluster. */ + down_write(&vol->lcnbmp_lock); + if (ntfs_bitmap_clear_bit(vol->lcnbmp_ino, lcn)) { + ntfs_error(vol->sb, "Failed to free allocated cluster.%s", es); + NVolSetErrors(vol); + } else + ntfs_inc_free_clusters(vol, 1); + up_write(&vol->lcnbmp_lock); + if (status.mp_rebuilt) { + if (ntfs_mapping_pairs_build(vol, (u8 *)a + le16_to_cpu( + a->data.non_resident.mapping_pairs_offset), + old_alen - le16_to_cpu( + a->data.non_resident.mapping_pairs_offset), + rl2, ll, -1, NULL, NULL, NULL)) { + ntfs_error(vol->sb, "Failed to restore mapping pairs array.%s", es); + NVolSetErrors(vol); + } + if (ntfs_attr_record_resize(ctx->mrec, a, old_alen)) { + ntfs_error(vol->sb, "Failed to restore attribute record.%s", es); + NVolSetErrors(vol); + } + mark_mft_record_dirty(ctx->ntfs_ino); + } else if (status.mp_extended && ntfs_attr_update_mapping_pairs(mftbmp_ni, 0)) { + ntfs_error(vol->sb, "Failed to restore mapping pairs.%s", es); + NVolSetErrors(vol); + } + if (ctx) + ntfs_attr_put_search_ctx(ctx); + if (!IS_ERR(mrec)) + unmap_mft_record(mft_ni); + up_write(&mftbmp_ni->runlist.lock); + return ret; +} + +/* + * ntfs_mft_bitmap_extend_initialized_nolock - extend mftbmp initialized data + * @vol: volume on which to extend the mft bitmap attribute + * + * Extend the initialized portion of the mft bitmap attribute on the ntfs + * volume @vol by 8 bytes. + * + * Note: Only changes initialized_size and data_size, i.e. requires that + * allocated_size is big enough to fit the new initialized_size. + * + * Return 0 on success and -error on error. + * + * Locking: Caller must hold vol->mftbmp_lock for writing. + */ +static int ntfs_mft_bitmap_extend_initialized_nolock(struct ntfs_volume *vol) +{ + s64 old_data_size, old_initialized_size; + unsigned long flags; + struct inode *mftbmp_vi; + struct ntfs_inode *mft_ni, *mftbmp_ni; + struct ntfs_attr_search_ctx *ctx; + struct mft_record *mrec; + struct attr_record *a; + int ret; + + ntfs_debug("Extending mft bitmap initialized (and data) size."); + mft_ni = NTFS_I(vol->mft_ino); + mftbmp_vi = vol->mftbmp_ino; + mftbmp_ni = NTFS_I(mftbmp_vi); + /* Get the attribute record. */ + mrec = map_mft_record(mft_ni); + if (IS_ERR(mrec)) { + ntfs_error(vol->sb, "Failed to map mft record."); + return PTR_ERR(mrec); + } + ctx = ntfs_attr_get_search_ctx(mft_ni, mrec); + if (unlikely(!ctx)) { + ntfs_error(vol->sb, "Failed to get search context."); + ret = -ENOMEM; + goto unm_err_out; + } + ret = ntfs_attr_lookup(mftbmp_ni->type, mftbmp_ni->name, + mftbmp_ni->name_len, CASE_SENSITIVE, 0, NULL, 0, ctx); + if (unlikely(ret)) { + ntfs_error(vol->sb, + "Failed to find first attribute extent of mft bitmap attribute."); + if (ret == -ENOENT) + ret = -EIO; + goto put_err_out; + } + a = ctx->attr; + write_lock_irqsave(&mftbmp_ni->size_lock, flags); + old_data_size = i_size_read(mftbmp_vi); + old_initialized_size = mftbmp_ni->initialized_size; + /* + * We can simply update the initialized_size before filling the space + * with zeroes because the caller is holding the mft bitmap lock for + * writing which ensures that no one else is trying to access the data. + */ + mftbmp_ni->initialized_size += 8; + a->data.non_resident.initialized_size = + cpu_to_le64(mftbmp_ni->initialized_size); + if (mftbmp_ni->initialized_size > old_data_size) { + i_size_write(mftbmp_vi, mftbmp_ni->initialized_size); + a->data.non_resident.data_size = + cpu_to_le64(mftbmp_ni->initialized_size); + } + write_unlock_irqrestore(&mftbmp_ni->size_lock, flags); + /* Ensure the changes make it to disk. */ + mark_mft_record_dirty(ctx->ntfs_ino); + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(mft_ni); + /* Initialize the mft bitmap attribute value with zeroes. */ + ret = ntfs_attr_set(mftbmp_ni, old_initialized_size, 8, 0); + if (likely(!ret)) { + ntfs_debug("Done. (Wrote eight initialized bytes to mft bitmap."); + ntfs_inc_free_mft_records(vol, 8 * 8); + return 0; + } + ntfs_error(vol->sb, "Failed to write to mft bitmap."); + /* Try to recover from the error. */ + mrec = map_mft_record(mft_ni); + if (IS_ERR(mrec)) { + ntfs_error(vol->sb, "Failed to map mft record.%s", es); + NVolSetErrors(vol); + return ret; + } + ctx = ntfs_attr_get_search_ctx(mft_ni, mrec); + if (unlikely(!ctx)) { + ntfs_error(vol->sb, "Failed to get search context.%s", es); + NVolSetErrors(vol); + goto unm_err_out; + } + if (ntfs_attr_lookup(mftbmp_ni->type, mftbmp_ni->name, + mftbmp_ni->name_len, CASE_SENSITIVE, 0, NULL, 0, ctx)) { + ntfs_error(vol->sb, + "Failed to find first attribute extent of mft bitmap attribute.%s", es); + NVolSetErrors(vol); +put_err_out: + ntfs_attr_put_search_ctx(ctx); +unm_err_out: + unmap_mft_record(mft_ni); + goto err_out; + } + a = ctx->attr; + write_lock_irqsave(&mftbmp_ni->size_lock, flags); + mftbmp_ni->initialized_size = old_initialized_size; + a->data.non_resident.initialized_size = + cpu_to_le64(old_initialized_size); + if (i_size_read(mftbmp_vi) != old_data_size) { + i_size_write(mftbmp_vi, old_data_size); + a->data.non_resident.data_size = cpu_to_le64(old_data_size); + } + write_unlock_irqrestore(&mftbmp_ni->size_lock, flags); + mark_mft_record_dirty(ctx->ntfs_ino); + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(mft_ni); +#ifdef DEBUG + read_lock_irqsave(&mftbmp_ni->size_lock, flags); + ntfs_debug("Restored status of mftbmp: allocated_size 0x%llx, data_size 0x%llx, initialized_size 0x%llx.", + mftbmp_ni->allocated_size, i_size_read(mftbmp_vi), + mftbmp_ni->initialized_size); + read_unlock_irqrestore(&mftbmp_ni->size_lock, flags); +#endif /* DEBUG */ +err_out: + return ret; +} + +/* + * ntfs_mft_data_extend_allocation_nolock - extend mft data attribute + * @vol: volume on which to extend the mft data attribute + * + * Extend the mft data attribute on the ntfs volume @vol by 16 mft records + * worth of clusters or if not enough space for this by one mft record worth + * of clusters. + * + * Note: Only changes allocated_size, i.e. does not touch initialized_size or + * data_size. + * + * Return 0 on success and -errno on error. + * + * Locking: - Caller must hold vol->mftbmp_lock for writing. + * - This function takes NTFS_I(vol->mft_ino)->runlist.lock for + * writing and releases it before returning. + * - This function calls functions which take vol->lcnbmp_lock for + * writing and release it before returning. + */ +static int ntfs_mft_data_extend_allocation_nolock(struct ntfs_volume *vol) +{ + s64 lcn; + s64 old_last_vcn; + s64 min_nr, nr, ll; + unsigned long flags; + struct ntfs_inode *mft_ni; + struct runlist_element *rl, *rl2; + struct ntfs_attr_search_ctx *ctx = NULL; + struct mft_record *mrec; + struct attr_record *a = NULL; + int ret, mp_size; + u32 old_alen = 0; + bool mp_rebuilt = false, mp_extended = false; + size_t new_rl_count; + + ntfs_debug("Extending mft data allocation."); + mft_ni = NTFS_I(vol->mft_ino); + /* + * Determine the preferred allocation location, i.e. the last lcn of + * the mft data attribute. The allocated size of the mft data + * attribute cannot be zero so we are ok to do this. + */ + down_write(&mft_ni->runlist.lock); + read_lock_irqsave(&mft_ni->size_lock, flags); + ll = mft_ni->allocated_size; + read_unlock_irqrestore(&mft_ni->size_lock, flags); + rl = ntfs_attr_find_vcn_nolock(mft_ni, + NTFS_B_TO_CLU(vol, ll - 1), NULL); + if (IS_ERR(rl) || unlikely(!rl->length || rl->lcn < 0)) { + up_write(&mft_ni->runlist.lock); + ntfs_error(vol->sb, + "Failed to determine last allocated cluster of mft data attribute."); + if (!IS_ERR(rl)) + ret = -EIO; + else + ret = PTR_ERR(rl); + return ret; + } + lcn = rl->lcn + rl->length; + ntfs_debug("Last lcn of mft data attribute is 0x%llx.", lcn); + /* Minimum allocation is one mft record worth of clusters. */ + min_nr = NTFS_B_TO_CLU(vol, vol->mft_record_size); + if (!min_nr) + min_nr = 1; + /* Want to allocate 16 mft records worth of clusters. */ + nr = vol->mft_record_size << 4 >> vol->cluster_size_bits; + if (!nr) + nr = min_nr; + /* Ensure we do not go above 2^32-1 mft records. */ + read_lock_irqsave(&mft_ni->size_lock, flags); + ll = mft_ni->allocated_size; + read_unlock_irqrestore(&mft_ni->size_lock, flags); + if (unlikely((ll + NTFS_CLU_TO_B(vol, nr)) >> + vol->mft_record_size_bits >= (1ll << 32))) { + nr = min_nr; + if (unlikely((ll + NTFS_CLU_TO_B(vol, nr)) >> + vol->mft_record_size_bits >= (1ll << 32))) { + ntfs_warning(vol->sb, + "Cannot allocate mft record because the maximum number of inodes (2^32) has already been reached."); + up_write(&mft_ni->runlist.lock); + return -ENOSPC; + } + } + ntfs_debug("Trying mft data allocation with %s cluster count %lli.", + nr > min_nr ? "default" : "minimal", (long long)nr); + old_last_vcn = rl[1].vcn; + /* + * We can release the mft_ni runlist lock, Because this function is + * the only one that expends $MFT data attribute and is called with + * mft_ni->mrec_lock. + * This is required for the lock order, vol->lcnbmp_lock => + * mft_ni->runlist.lock. + */ + up_write(&mft_ni->runlist.lock); + + do { + rl2 = ntfs_cluster_alloc(vol, old_last_vcn, nr, lcn, MFT_ZONE, + true, false, false); + if (!IS_ERR(rl2)) + break; + if (PTR_ERR(rl2) != -ENOSPC || nr == min_nr) { + ntfs_error(vol->sb, + "Failed to allocate the minimal number of clusters (%lli) for the mft data attribute.", + nr); + return PTR_ERR(rl2); + } + /* + * There is not enough space to do the allocation, but there + * might be enough space to do a minimal allocation so try that + * before failing. + */ + nr = min_nr; + ntfs_debug("Retrying mft data allocation with minimal cluster count %lli.", nr); + } while (1); + + down_write(&mft_ni->runlist.lock); + rl = ntfs_runlists_merge(&mft_ni->runlist, rl2, 0, &new_rl_count); + if (IS_ERR(rl)) { + up_write(&mft_ni->runlist.lock); + ntfs_error(vol->sb, "Failed to merge runlists for mft data attribute."); + if (ntfs_cluster_free_from_rl(vol, rl2)) { + ntfs_error(vol->sb, + "Failed to deallocate clusters from the mft data attribute.%s", es); + NVolSetErrors(vol); + } + kvfree(rl2); + return PTR_ERR(rl); + } + mft_ni->runlist.rl = rl; + mft_ni->runlist.count = new_rl_count; + ntfs_debug("Allocated %lli clusters.", (long long)nr); + /* Find the last run in the new runlist. */ + for (; rl[1].length; rl++) + ; + up_write(&mft_ni->runlist.lock); + + /* Update the attribute record as well. */ + mrec = map_mft_record(mft_ni); + if (IS_ERR(mrec)) { + ntfs_error(vol->sb, "Failed to map mft record."); + ret = PTR_ERR(mrec); + down_write(&mft_ni->runlist.lock); + goto undo_alloc; + } + ctx = ntfs_attr_get_search_ctx(mft_ni, mrec); + if (unlikely(!ctx)) { + ntfs_error(vol->sb, "Failed to get search context."); + ret = -ENOMEM; + goto undo_alloc; + } + ret = ntfs_attr_lookup(mft_ni->type, mft_ni->name, mft_ni->name_len, + CASE_SENSITIVE, rl[1].vcn, NULL, 0, ctx); + if (unlikely(ret)) { + ntfs_error(vol->sb, "Failed to find last attribute extent of mft data attribute."); + if (ret == -ENOENT) + ret = -EIO; + goto undo_alloc; + } + a = ctx->attr; + ll = le64_to_cpu(a->data.non_resident.lowest_vcn); + + down_write(&mft_ni->runlist.lock); + /* Search back for the previous last allocated cluster of mft bitmap. */ + for (rl2 = rl; rl2 > mft_ni->runlist.rl; rl2--) { + if (ll >= rl2->vcn) + break; + } + WARN_ON(ll < rl2->vcn); + WARN_ON(ll >= rl2->vcn + rl2->length); + /* Get the size for the new mapping pairs array for this extent. */ + mp_size = ntfs_get_size_for_mapping_pairs(vol, rl2, ll, -1, -1); + if (unlikely(mp_size <= 0)) { + ntfs_error(vol->sb, + "Get size for mapping pairs failed for mft data attribute extent."); + ret = mp_size; + if (!ret) + ret = -EIO; + up_write(&mft_ni->runlist.lock); + goto undo_alloc; + } + up_write(&mft_ni->runlist.lock); + + /* Expand the attribute record if necessary. */ + old_alen = le32_to_cpu(a->length); + ret = ntfs_attr_record_resize(ctx->mrec, a, mp_size + + le16_to_cpu(a->data.non_resident.mapping_pairs_offset)); + if (unlikely(ret)) { + ret = ntfs_mft_attr_extend(mft_ni); + if (!ret) + goto extended_ok; + if (ret != -EAGAIN) + mp_extended = true; + goto undo_alloc; + } + mp_rebuilt = true; + /* Generate the mapping pairs array directly into the attr record. */ + ret = ntfs_mapping_pairs_build(vol, (u8 *)a + + le16_to_cpu(a->data.non_resident.mapping_pairs_offset), + mp_size, rl2, ll, -1, NULL, NULL, NULL); + if (unlikely(ret)) { + ntfs_error(vol->sb, "Failed to build mapping pairs array of mft data attribute."); + goto undo_alloc; + } + /* Update the highest_vcn. */ + a->data.non_resident.highest_vcn = cpu_to_le64(rl[1].vcn - 1); + /* + * We now have extended the mft data allocated_size by nr clusters. + * Reflect this in the struct ntfs_inode structure and the attribute record. + * @rl is the last (non-terminator) runlist element of mft data + * attribute. + */ + if (a->data.non_resident.lowest_vcn) { + /* + * We are not in the first attribute extent, switch to it, but + * first ensure the changes will make it to disk later. + */ + mark_mft_record_dirty(ctx->ntfs_ino); +extended_ok: + ntfs_attr_reinit_search_ctx(ctx); + ret = ntfs_attr_lookup(mft_ni->type, mft_ni->name, + mft_ni->name_len, CASE_SENSITIVE, 0, NULL, 0, + ctx); + if (unlikely(ret)) { + ntfs_error(vol->sb, + "Failed to find first attribute extent of mft data attribute."); + goto restore_undo_alloc; + } + a = ctx->attr; + } + + write_lock_irqsave(&mft_ni->size_lock, flags); + mft_ni->allocated_size += NTFS_CLU_TO_B(vol, nr); + a->data.non_resident.allocated_size = + cpu_to_le64(mft_ni->allocated_size); + write_unlock_irqrestore(&mft_ni->size_lock, flags); + /* Ensure the changes make it to disk. */ + mark_mft_record_dirty(ctx->ntfs_ino); + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(mft_ni); + ntfs_debug("Done."); + return 0; +restore_undo_alloc: + ntfs_attr_reinit_search_ctx(ctx); + if (ntfs_attr_lookup(mft_ni->type, mft_ni->name, mft_ni->name_len, + CASE_SENSITIVE, rl[1].vcn, NULL, 0, ctx)) { + ntfs_error(vol->sb, + "Failed to find last attribute extent of mft data attribute.%s", es); + write_lock_irqsave(&mft_ni->size_lock, flags); + mft_ni->allocated_size += NTFS_CLU_TO_B(vol, nr); + write_unlock_irqrestore(&mft_ni->size_lock, flags); + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(mft_ni); + up_write(&mft_ni->runlist.lock); + /* + * The only thing that is now wrong is ->allocated_size of the + * base attribute extent which chkdsk should be able to fix. + */ + NVolSetErrors(vol); + return ret; + } + ctx->attr->data.non_resident.highest_vcn = + cpu_to_le64(old_last_vcn - 1); +undo_alloc: + if (ntfs_cluster_free(mft_ni, old_last_vcn, -1, ctx) < 0) { + ntfs_error(vol->sb, "Failed to free clusters from mft data attribute.%s", es); + NVolSetErrors(vol); + } + + if (ntfs_rl_truncate_nolock(vol, &mft_ni->runlist, old_last_vcn)) { + ntfs_error(vol->sb, "Failed to truncate mft data attribute runlist.%s", es); + NVolSetErrors(vol); + } + if (mp_extended && ntfs_attr_update_mapping_pairs(mft_ni, 0)) { + ntfs_error(vol->sb, "Failed to restore mapping pairs.%s", + es); + NVolSetErrors(vol); + } + if (ctx) { + a = ctx->attr; + if (mp_rebuilt && !IS_ERR(ctx->mrec)) { + if (ntfs_mapping_pairs_build(vol, (u8 *)a + le16_to_cpu( + a->data.non_resident.mapping_pairs_offset), + old_alen - le16_to_cpu( + a->data.non_resident.mapping_pairs_offset), + rl2, ll, -1, NULL, NULL, NULL)) { + ntfs_error(vol->sb, "Failed to restore mapping pairs array.%s", es); + NVolSetErrors(vol); + } + if (ntfs_attr_record_resize(ctx->mrec, a, old_alen)) { + ntfs_error(vol->sb, "Failed to restore attribute record.%s", es); + NVolSetErrors(vol); + } + mark_mft_record_dirty(ctx->ntfs_ino); + } else if (IS_ERR(ctx->mrec)) { + ntfs_error(vol->sb, "Failed to restore attribute search context.%s", es); + NVolSetErrors(vol); + } + ntfs_attr_put_search_ctx(ctx); + } + if (!IS_ERR(mrec)) + unmap_mft_record(mft_ni); + return ret; +} + +/* + * ntfs_mft_record_layout - layout an mft record into a memory buffer + * @vol: volume to which the mft record will belong + * @mft_no: mft reference specifying the mft record number + * @m: destination buffer of size >= @vol->mft_record_size bytes + * + * Layout an empty, unused mft record with the mft record number @mft_no into + * the buffer @m. The volume @vol is needed because the mft record structure + * was modified in NTFS 3.1 so we need to know which volume version this mft + * record will be used on. + * + * Return 0 on success and -errno on error. + */ +static int ntfs_mft_record_layout(const struct ntfs_volume *vol, const s64 mft_no, + struct mft_record *m) +{ + struct attr_record *a; + + ntfs_debug("Entering for mft record 0x%llx.", (long long)mft_no); + if (mft_no >= (1ll << 32)) { + ntfs_error(vol->sb, "Mft record number 0x%llx exceeds maximum of 2^32.", + (long long)mft_no); + return -ERANGE; + } + /* Start by clearing the whole mft record to gives us a clean slate. */ + memset(m, 0, vol->mft_record_size); + /* Aligned to 2-byte boundary. */ + if (vol->major_ver < 3 || (vol->major_ver == 3 && !vol->minor_ver)) + m->usa_ofs = cpu_to_le16((sizeof(struct mft_record_old) + 1) & ~1); + else { + m->usa_ofs = cpu_to_le16((sizeof(struct mft_record) + 1) & ~1); + /* + * Set the NTFS 3.1+ specific fields while we know that the + * volume version is 3.1+. + */ + m->reserved = 0; + m->mft_record_number = cpu_to_le32((u32)mft_no); + } + m->magic = magic_FILE; + if (vol->mft_record_size >= NTFS_BLOCK_SIZE) + m->usa_count = cpu_to_le16(vol->mft_record_size / + NTFS_BLOCK_SIZE + 1); + else { + m->usa_count = cpu_to_le16(1); + ntfs_warning(vol->sb, + "Sector size is bigger than mft record size. Setting usa_count to 1. If chkdsk reports this as corruption"); + } + /* Set the update sequence number to 1. */ + *(__le16 *)((u8 *)m + le16_to_cpu(m->usa_ofs)) = cpu_to_le16(1); + m->lsn = 0; + m->sequence_number = cpu_to_le16(1); + m->link_count = 0; + /* + * Place the attributes straight after the update sequence array, + * aligned to 8-byte boundary. + */ + m->attrs_offset = cpu_to_le16((le16_to_cpu(m->usa_ofs) + + (le16_to_cpu(m->usa_count) << 1) + 7) & ~7); + m->flags = 0; + /* + * Using attrs_offset plus eight bytes (for the termination attribute). + * attrs_offset is already aligned to 8-byte boundary, so no need to + * align again. + */ + m->bytes_in_use = cpu_to_le32(le16_to_cpu(m->attrs_offset) + 8); + m->bytes_allocated = cpu_to_le32(vol->mft_record_size); + m->base_mft_record = 0; + m->next_attr_instance = 0; + /* Add the termination attribute. */ + a = (struct attr_record *)((u8 *)m + le16_to_cpu(m->attrs_offset)); + a->type = AT_END; + a->length = 0; + ntfs_debug("Done."); + return 0; +} + +/* + * ntfs_mft_record_format - format an mft record on an ntfs volume + * @vol: volume on which to format the mft record + * @mft_no: mft record number to format + * + * Format the mft record @mft_no in $MFT/$DATA, i.e. lay out an empty, unused + * mft record into the appropriate place of the mft data attribute. This is + * used when extending the mft data attribute. + * + * Return 0 on success and -errno on error. + */ +static int ntfs_mft_record_format(const struct ntfs_volume *vol, const s64 mft_no) +{ + loff_t i_size; + struct inode *mft_vi = vol->mft_ino; + struct folio *folio; + struct mft_record *m; + pgoff_t index, end_index; + unsigned int ofs; + int err; + + ntfs_debug("Entering for mft record 0x%llx.", (long long)mft_no); + /* + * The index into the page cache and the offset within the page cache + * page of the wanted mft record. + */ + index = NTFS_MFT_NR_TO_PIDX(vol, mft_no); + ofs = NTFS_MFT_NR_TO_POFS(vol, mft_no); + /* The maximum valid index into the page cache for $MFT's data. */ + i_size = i_size_read(mft_vi); + end_index = i_size >> PAGE_SHIFT; + if (unlikely(index >= end_index)) { + if (unlikely(index > end_index || + ofs + vol->mft_record_size > (i_size & ~PAGE_MASK))) { + ntfs_error(vol->sb, "Tried to format non-existing mft record 0x%llx.", + (long long)mft_no); + return -ENOENT; + } + } + + /* Read, map, and pin the folio containing the mft record. */ + folio = read_mapping_folio(mft_vi->i_mapping, index, NULL); + if (IS_ERR(folio)) { + ntfs_error(vol->sb, "Failed to map page containing mft record to format 0x%llx.", + (long long)mft_no); + return PTR_ERR(folio); + } + folio_lock(folio); + folio_clear_uptodate(folio); + m = (struct mft_record *)((u8 *)kmap_local_folio(folio, 0) + ofs); + err = ntfs_mft_record_layout(vol, mft_no, m); + if (unlikely(err)) { + ntfs_error(vol->sb, "Failed to layout mft record 0x%llx.", + (long long)mft_no); + folio_mark_uptodate(folio); + folio_unlock(folio); + kunmap_local(m); + folio_put(folio); + return err; + } + pre_write_mst_fixup((struct ntfs_record *)m, vol->mft_record_size); + folio_mark_uptodate(folio); + /* + * Make sure the mft record is written out to disk. We could use + * ilookup5() to check if an inode is in icache and so on but this is + * unnecessary as ntfs_writepage() will write the dirty record anyway. + */ + ntfs_mft_mark_dirty(folio); + folio_unlock(folio); + kunmap_local(m); + folio_put(folio); + ntfs_debug("Done."); + return 0; +} + +/* + * ntfs_mft_record_alloc - allocate an mft record on an ntfs volume + * @vol: [IN] volume on which to allocate the mft record + * @mode: [IN] mode if want a file or directory, i.e. base inode or 0 + * @ni: [OUT] on success, set to the allocated ntfs inode + * @base_ni: [IN] open base inode if allocating an extent mft record or NULL + * @ni_mrec: [OUT] on successful return this is the mapped mft record + * + * Allocate an mft record in $MFT/$DATA of an open ntfs volume @vol. + * + * If @base_ni is NULL make the mft record a base mft record, i.e. a file or + * direvctory inode, and allocate it at the default allocator position. In + * this case @mode is the file mode as given to us by the caller. We in + * particular use @mode to distinguish whether a file or a directory is being + * created (S_IFDIR(mode) and S_IFREG(mode), respectively). + * + * If @base_ni is not NULL make the allocated mft record an extent record, + * allocate it starting at the mft record after the base mft record and attach + * the allocated and opened ntfs inode to the base inode @base_ni. In this + * case @mode must be 0 as it is meaningless for extent inodes. + * + * You need to check the return value with IS_ERR(). If false, the function + * was successful and the return value is the now opened ntfs inode of the + * allocated mft record. *@mrec is then set to the allocated, mapped, pinned, + * and locked mft record. If IS_ERR() is true, the function failed and the + * error code is obtained from PTR_ERR(return value). *@mrec is undefined in + * this case. + * + * Allocation strategy: + * + * To find a free mft record, we scan the mft bitmap for a zero bit. To + * optimize this we start scanning at the place specified by @base_ni or if + * @base_ni is NULL we start where we last stopped and we perform wrap around + * when we reach the end. Note, we do not try to allocate mft records below + * number 64 because numbers 0 to 15 are the defined system files anyway and 16 + * to 64 are special in that they are used for storing extension mft records + * for the $DATA attribute of $MFT. This is required to avoid the possibility + * of creating a runlist with a circular dependency which once written to disk + * can never be read in again. Windows will only use records 16 to 24 for + * normal files if the volume is completely out of space. We never use them + * which means that when the volume is really out of space we cannot create any + * more files while Windows can still create up to 8 small files. We can start + * doing this at some later time, it does not matter much for now. + * + * When scanning the mft bitmap, we only search up to the last allocated mft + * record. If there are no free records left in the range 64 to number of + * allocated mft records, then we extend the $MFT/$DATA attribute in order to + * create free mft records. We extend the allocated size of $MFT/$DATA by 16 + * records at a time or one cluster, if cluster size is above 16kiB. If there + * is not sufficient space to do this, we try to extend by a single mft record + * or one cluster, if cluster size is above the mft record size. + * + * No matter how many mft records we allocate, we initialize only the first + * allocated mft record, incrementing mft data size and initialized size + * accordingly, open an struct ntfs_inode for it and return it to the caller, unless + * there are less than 64 mft records, in which case we allocate and initialize + * mft records until we reach record 64 which we consider as the first free mft + * record for use by normal files. + * + * If during any stage we overflow the initialized data in the mft bitmap, we + * extend the initialized size (and data size) by 8 bytes, allocating another + * cluster if required. The bitmap data size has to be at least equal to the + * number of mft records in the mft, but it can be bigger, in which case the + * superfluous bits are padded with zeroes. + * + * Thus, when we return successfully (IS_ERR() is false), we will have: + * - initialized / extended the mft bitmap if necessary, + * - initialized / extended the mft data if necessary, + * - set the bit corresponding to the mft record being allocated in the + * mft bitmap, + * - opened an struct ntfs_inode for the allocated mft record, and we will have + * - returned the struct ntfs_inode as well as the allocated mapped, pinned, and + * locked mft record. + * + * On error, the volume will be left in a consistent state and no record will + * be allocated. If rolling back a partial operation fails, we may leave some + * inconsistent metadata in which case we set NVolErrors() so the volume is + * left dirty when unmounted. + * + * Note, this function cannot make use of most of the normal functions, like + * for example for attribute resizing, etc, because when the run list overflows + * the base mft record and an attribute list is used, it is very important that + * the extension mft records used to store the $DATA attribute of $MFT can be + * reached without having to read the information contained inside them, as + * this would make it impossible to find them in the first place after the + * volume is unmounted. $MFT/$BITMAP probably does not need to follow this + * rule because the bitmap is not essential for finding the mft records, but on + * the other hand, handling the bitmap in this special way would make life + * easier because otherwise there might be circular invocations of functions + * when reading the bitmap. + */ +int ntfs_mft_record_alloc(struct ntfs_volume *vol, const int mode, + struct ntfs_inode **ni, struct ntfs_inode *base_ni, + struct mft_record **ni_mrec) +{ + s64 ll, bit, old_data_initialized, old_data_size; + unsigned long flags; + struct folio *folio; + struct ntfs_inode *mft_ni, *mftbmp_ni; + struct ntfs_attr_search_ctx *ctx; + struct mft_record *m = NULL; + struct attr_record *a; + pgoff_t index; + unsigned int ofs; + int err; + __le16 seq_no, usn; + bool record_formatted = false; + unsigned int memalloc_flags; + + if (base_ni && *ni) + return -EINVAL; + + /* @mode and @base_ni are mutually exclusive. */ + if (mode && base_ni) + return -EINVAL; + + if (base_ni) + ntfs_debug("Entering (allocating an extent mft record for base mft record 0x%llx).", + (long long)base_ni->mft_no); + else + ntfs_debug("Entering (allocating a base mft record)."); + + memalloc_flags = memalloc_nofs_save(); + + mft_ni = NTFS_I(vol->mft_ino); + if (!base_ni || base_ni->mft_no != FILE_MFT) + mutex_lock(&mft_ni->mrec_lock); + mftbmp_ni = NTFS_I(vol->mftbmp_ino); +search_free_rec: + if (!base_ni || base_ni->mft_no != FILE_MFT) + down_write(&vol->mftbmp_lock); + bit = ntfs_mft_bitmap_find_and_alloc_free_rec_nolock(vol, base_ni); + if (bit >= 0) { + ntfs_debug("Found and allocated free record (#1), bit 0x%llx.", + (long long)bit); + goto have_alloc_rec; + } + if (bit != -ENOSPC) { + if (!base_ni || base_ni->mft_no != FILE_MFT) { + up_write(&vol->mftbmp_lock); + mutex_unlock(&mft_ni->mrec_lock); + } + memalloc_nofs_restore(memalloc_flags); + return bit; + } + + if (base_ni && base_ni->mft_no == FILE_MFT) { + memalloc_nofs_restore(memalloc_flags); + return bit; + } + + /* + * No free mft records left. If the mft bitmap already covers more + * than the currently used mft records, the next records are all free, + * so we can simply allocate the first unused mft record. + * Note: We also have to make sure that the mft bitmap at least covers + * the first 24 mft records as they are special and whilst they may not + * be in use, we do not allocate from them. + */ + read_lock_irqsave(&mft_ni->size_lock, flags); + ll = mft_ni->initialized_size >> vol->mft_record_size_bits; + read_unlock_irqrestore(&mft_ni->size_lock, flags); + read_lock_irqsave(&mftbmp_ni->size_lock, flags); + old_data_initialized = mftbmp_ni->initialized_size; + read_unlock_irqrestore(&mftbmp_ni->size_lock, flags); + if (old_data_initialized << 3 > ll && + old_data_initialized > RESERVED_MFT_RECORDS / 8) { + bit = ll; + if (bit < RESERVED_MFT_RECORDS) + bit = RESERVED_MFT_RECORDS; + if (unlikely(bit >= (1ll << 32))) + goto max_err_out; + ntfs_debug("Found free record (#2), bit 0x%llx.", + (long long)bit); + goto found_free_rec; + } + /* + * The mft bitmap needs to be expanded until it covers the first unused + * mft record that we can allocate. + * Note: The smallest mft record we allocate is mft record 24. + */ + bit = old_data_initialized << 3; + if (unlikely(bit >= (1ll << 32))) + goto max_err_out; + read_lock_irqsave(&mftbmp_ni->size_lock, flags); + old_data_size = mftbmp_ni->allocated_size; + ntfs_debug("Status of mftbmp before extension: allocated_size 0x%llx, data_size 0x%llx, initialized_size 0x%llx.", + old_data_size, i_size_read(vol->mftbmp_ino), + old_data_initialized); + read_unlock_irqrestore(&mftbmp_ni->size_lock, flags); + if (old_data_initialized + 8 > old_data_size) { + /* Need to extend bitmap by one more cluster. */ + ntfs_debug("mftbmp: initialized_size + 8 > allocated_size."); + err = ntfs_mft_bitmap_extend_allocation_nolock(vol); + if (err == -EAGAIN) + err = ntfs_mft_bitmap_extend_allocation_nolock(vol); + + if (unlikely(err)) { + if (!base_ni || base_ni->mft_no != FILE_MFT) + up_write(&vol->mftbmp_lock); + goto err_out; + } +#ifdef DEBUG + read_lock_irqsave(&mftbmp_ni->size_lock, flags); + ntfs_debug("Status of mftbmp after allocation extension: allocated_size 0x%llx, data_size 0x%llx, initialized_size 0x%llx.", + mftbmp_ni->allocated_size, + i_size_read(vol->mftbmp_ino), + mftbmp_ni->initialized_size); + read_unlock_irqrestore(&mftbmp_ni->size_lock, flags); +#endif /* DEBUG */ + } + /* + * We now have sufficient allocated space, extend the initialized_size + * as well as the data_size if necessary and fill the new space with + * zeroes. + */ + err = ntfs_mft_bitmap_extend_initialized_nolock(vol); + if (unlikely(err)) { + if (!base_ni || base_ni->mft_no != FILE_MFT) + up_write(&vol->mftbmp_lock); + goto err_out; + } +#ifdef DEBUG + read_lock_irqsave(&mftbmp_ni->size_lock, flags); + ntfs_debug("Status of mftbmp after initialized extension: allocated_size 0x%llx, data_size 0x%llx, initialized_size 0x%llx.", + mftbmp_ni->allocated_size, + i_size_read(vol->mftbmp_ino), + mftbmp_ni->initialized_size); + read_unlock_irqrestore(&mftbmp_ni->size_lock, flags); +#endif /* DEBUG */ + ntfs_debug("Found free record (#3), bit 0x%llx.", (long long)bit); +found_free_rec: + /* @bit is the found free mft record, allocate it in the mft bitmap. */ + ntfs_debug("At found_free_rec."); + err = ntfs_bitmap_set_bit(vol->mftbmp_ino, bit); + if (unlikely(err)) { + ntfs_error(vol->sb, "Failed to allocate bit in mft bitmap."); + if (!base_ni || base_ni->mft_no != FILE_MFT) + up_write(&vol->mftbmp_lock); + goto err_out; + } + ntfs_debug("Set bit 0x%llx in mft bitmap.", (long long)bit); +have_alloc_rec: + /* + * The mft bitmap is now uptodate. Deal with mft data attribute now. + * Note, we keep hold of the mft bitmap lock for writing until all + * modifications to the mft data attribute are complete, too, as they + * will impact decisions for mft bitmap and mft record allocation done + * by a parallel allocation and if the lock is not maintained a + * parallel allocation could allocate the same mft record as this one. + */ + ll = (bit + 1) << vol->mft_record_size_bits; + read_lock_irqsave(&mft_ni->size_lock, flags); + old_data_initialized = mft_ni->initialized_size; + read_unlock_irqrestore(&mft_ni->size_lock, flags); + if (ll <= old_data_initialized) { + ntfs_debug("Allocated mft record already initialized."); + goto mft_rec_already_initialized; + } + ntfs_debug("Initializing allocated mft record."); + /* + * The mft record is outside the initialized data. Extend the mft data + * attribute until it covers the allocated record. The loop is only + * actually traversed more than once when a freshly formatted volume is + * first written to so it optimizes away nicely in the common case. + */ + if (!base_ni || base_ni->mft_no != FILE_MFT) { + read_lock_irqsave(&mft_ni->size_lock, flags); + ntfs_debug("Status of mft data before extension: allocated_size 0x%llx, data_size 0x%llx, initialized_size 0x%llx.", + mft_ni->allocated_size, i_size_read(vol->mft_ino), + mft_ni->initialized_size); + while (ll > mft_ni->allocated_size) { + read_unlock_irqrestore(&mft_ni->size_lock, flags); + err = ntfs_mft_data_extend_allocation_nolock(vol); + if (err == -EAGAIN) + err = ntfs_mft_data_extend_allocation_nolock(vol); + + if (unlikely(err)) { + ntfs_error(vol->sb, "Failed to extend mft data allocation."); + goto undo_mftbmp_alloc_nolock; + } + read_lock_irqsave(&mft_ni->size_lock, flags); + ntfs_debug("Status of mft data after allocation extension: allocated_size 0x%llx, data_size 0x%llx, initialized_size 0x%llx.", + mft_ni->allocated_size, i_size_read(vol->mft_ino), + mft_ni->initialized_size); + } + read_unlock_irqrestore(&mft_ni->size_lock, flags); + } else if (ll > mft_ni->allocated_size) { + err = -ENOSPC; + goto undo_mftbmp_alloc_nolock; + } + /* + * Extend mft data initialized size (and data size of course) to reach + * the allocated mft record, formatting the mft records allong the way. + * Note: We only modify the struct ntfs_inode structure as that is all that is + * needed by ntfs_mft_record_format(). We will update the attribute + * record itself in one fell swoop later on. + */ + write_lock_irqsave(&mft_ni->size_lock, flags); + old_data_initialized = mft_ni->initialized_size; + old_data_size = vol->mft_ino->i_size; + while (ll > mft_ni->initialized_size) { + s64 new_initialized_size, mft_no; + + new_initialized_size = mft_ni->initialized_size + + vol->mft_record_size; + mft_no = mft_ni->initialized_size >> vol->mft_record_size_bits; + if (new_initialized_size > i_size_read(vol->mft_ino)) + i_size_write(vol->mft_ino, new_initialized_size); + write_unlock_irqrestore(&mft_ni->size_lock, flags); + ntfs_debug("Initializing mft record 0x%llx.", + (long long)mft_no); + err = ntfs_mft_record_format(vol, mft_no); + if (unlikely(err)) { + ntfs_error(vol->sb, "Failed to format mft record."); + goto undo_data_init; + } + write_lock_irqsave(&mft_ni->size_lock, flags); + mft_ni->initialized_size = new_initialized_size; + } + write_unlock_irqrestore(&mft_ni->size_lock, flags); + record_formatted = true; + /* Update the mft data attribute record to reflect the new sizes. */ + m = map_mft_record(mft_ni); + if (IS_ERR(m)) { + ntfs_error(vol->sb, "Failed to map mft record."); + err = PTR_ERR(m); + goto undo_data_init; + } + ctx = ntfs_attr_get_search_ctx(mft_ni, m); + if (unlikely(!ctx)) { + ntfs_error(vol->sb, "Failed to get search context."); + err = -ENOMEM; + unmap_mft_record(mft_ni); + goto undo_data_init; + } + err = ntfs_attr_lookup(mft_ni->type, mft_ni->name, mft_ni->name_len, + CASE_SENSITIVE, 0, NULL, 0, ctx); + if (unlikely(err)) { + ntfs_error(vol->sb, "Failed to find first attribute extent of mft data attribute."); + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(mft_ni); + goto undo_data_init; + } + a = ctx->attr; + read_lock_irqsave(&mft_ni->size_lock, flags); + a->data.non_resident.initialized_size = + cpu_to_le64(mft_ni->initialized_size); + a->data.non_resident.data_size = + cpu_to_le64(i_size_read(vol->mft_ino)); + read_unlock_irqrestore(&mft_ni->size_lock, flags); + /* Ensure the changes make it to disk. */ + mark_mft_record_dirty(ctx->ntfs_ino); + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(mft_ni); + read_lock_irqsave(&mft_ni->size_lock, flags); + ntfs_debug("Status of mft data after mft record initialization: allocated_size 0x%llx, data_size 0x%llx, initialized_size 0x%llx.", + mft_ni->allocated_size, i_size_read(vol->mft_ino), + mft_ni->initialized_size); + WARN_ON(i_size_read(vol->mft_ino) > mft_ni->allocated_size); + WARN_ON(mft_ni->initialized_size > i_size_read(vol->mft_ino)); + read_unlock_irqrestore(&mft_ni->size_lock, flags); +mft_rec_already_initialized: + /* + * We can finally drop the mft bitmap lock as the mft data attribute + * has been fully updated. The only disparity left is that the + * allocated mft record still needs to be marked as in use to match the + * set bit in the mft bitmap but this is actually not a problem since + * this mft record is not referenced from anywhere yet and the fact + * that it is allocated in the mft bitmap means that no-one will try to + * allocate it either. + */ + if (!base_ni || base_ni->mft_no != FILE_MFT) + up_write(&vol->mftbmp_lock); + /* + * We now have allocated and initialized the mft record. Calculate the + * index of and the offset within the page cache page the record is in. + */ + index = NTFS_MFT_NR_TO_PIDX(vol, bit); + ofs = NTFS_MFT_NR_TO_POFS(vol, bit); + /* Read, map, and pin the folio containing the mft record. */ + folio = read_mapping_folio(vol->mft_ino->i_mapping, index, NULL); + if (IS_ERR(folio)) { + ntfs_error(vol->sb, "Failed to map page containing allocated mft record 0x%llx.", + bit); + err = PTR_ERR(folio); + goto undo_mftbmp_alloc; + } + folio_lock(folio); + folio_clear_uptodate(folio); + m = (struct mft_record *)((u8 *)kmap_local_folio(folio, 0) + ofs); + /* If we just formatted the mft record no need to do it again. */ + if (!record_formatted) { + /* Sanity check that the mft record is really not in use. */ + if (ntfs_is_file_record(m->magic) && + (m->flags & MFT_RECORD_IN_USE)) { + ntfs_warning(vol->sb, + "Mft record 0x%llx was marked free in mft bitmap but is marked used itself. Unmount and run chkdsk.", + bit); + folio_mark_uptodate(folio); + folio_unlock(folio); + kunmap_local(m); + folio_put(folio); + NVolSetErrors(vol); + goto search_free_rec; + } + /* + * We need to (re-)format the mft record, preserving the + * sequence number if it is not zero as well as the update + * sequence number if it is not zero or -1 (0xffff). This + * means we do not need to care whether or not something went + * wrong with the previous mft record. + */ + seq_no = m->sequence_number; + usn = *(__le16 *)((u8 *)m + le16_to_cpu(m->usa_ofs)); + err = ntfs_mft_record_layout(vol, bit, m); + if (unlikely(err)) { + ntfs_error(vol->sb, "Failed to layout allocated mft record 0x%llx.", + bit); + folio_mark_uptodate(folio); + folio_unlock(folio); + kunmap_local(m); + folio_put(folio); + goto undo_mftbmp_alloc; + } + if (seq_no) + m->sequence_number = seq_no; + if (usn && le16_to_cpu(usn) != 0xffff) + *(__le16 *)((u8 *)m + le16_to_cpu(m->usa_ofs)) = usn; + pre_write_mst_fixup((struct ntfs_record *)m, vol->mft_record_size); + } + /* Set the mft record itself in use. */ + m->flags |= MFT_RECORD_IN_USE; + if (S_ISDIR(mode)) + m->flags |= MFT_RECORD_IS_DIRECTORY; + folio_mark_uptodate(folio); + if (base_ni) { + struct mft_record *m_tmp; + + /* + * Setup the base mft record in the extent mft record. This + * completes initialization of the allocated extent mft record + * and we can simply use it with map_extent_mft_record(). + */ + m->base_mft_record = MK_LE_MREF(base_ni->mft_no, + base_ni->seq_no); + /* + * Allocate an extent inode structure for the new mft record, + * attach it to the base inode @base_ni and map, pin, and lock + * its, i.e. the allocated, mft record. + */ + m_tmp = map_extent_mft_record(base_ni, + MK_MREF(bit, le16_to_cpu(m->sequence_number)), + ni); + if (IS_ERR(m_tmp)) { + ntfs_error(vol->sb, "Failed to map allocated extent mft record 0x%llx.", + bit); + err = PTR_ERR(m_tmp); + /* Set the mft record itself not in use. */ + m->flags &= cpu_to_le16( + ~le16_to_cpu(MFT_RECORD_IN_USE)); + /* Make sure the mft record is written out to disk. */ + ntfs_mft_mark_dirty(folio); + folio_unlock(folio); + kunmap_local(m); + folio_put(folio); + goto undo_mftbmp_alloc; + } + + /* + * Make sure the allocated mft record is written out to disk. + * No need to set the inode dirty because the caller is going + * to do that anyway after finishing with the new extent mft + * record (e.g. at a minimum a new attribute will be added to + * the mft record. + */ + ntfs_mft_mark_dirty(folio); + folio_unlock(folio); + /* + * Need to unmap the page since map_extent_mft_record() mapped + * it as well so we have it mapped twice at the moment. + */ + kunmap_local(m); + folio_put(folio); + } else { + /* + * Manually map, pin, and lock the mft record as we already + * have its page mapped and it is very easy to do. + */ + (*ni)->seq_no = le16_to_cpu(m->sequence_number); + /* + * Make sure the allocated mft record is written out to disk. + * NOTE: We do not set the ntfs inode dirty because this would + * fail in ntfs_write_inode() because the inode does not have a + * standard information attribute yet. Also, there is no need + * to set the inode dirty because the caller is going to do + * that anyway after finishing with the new mft record (e.g. at + * a minimum some new attributes will be added to the mft + * record. + */ + + (*ni)->mrec = kmalloc(vol->mft_record_size, GFP_NOFS); + if (!(*ni)->mrec) { + folio_unlock(folio); + kunmap_local(m); + folio_put(folio); + err = -ENOMEM; + goto undo_mftbmp_alloc; + } + + memcpy((*ni)->mrec, m, vol->mft_record_size); + post_read_mst_fixup((struct ntfs_record *)(*ni)->mrec, vol->mft_record_size); + ntfs_mft_mark_dirty(folio); + folio_unlock(folio); + (*ni)->folio = folio; + (*ni)->folio_ofs = ofs; + atomic_inc(&(*ni)->count); + /* Update the default mft allocation position. */ + vol->mft_data_pos = bit + 1; + } + if (!base_ni || base_ni->mft_no != FILE_MFT) + mutex_unlock(&mft_ni->mrec_lock); + memalloc_nofs_restore(memalloc_flags); + + /* + * Return the opened, allocated inode of the allocated mft record as + * well as the mapped, pinned, and locked mft record. + */ + ntfs_debug("Returning opened, allocated %sinode 0x%llx.", + base_ni ? "extent " : "", bit); + (*ni)->mft_no = bit; + if (ni_mrec) + *ni_mrec = (*ni)->mrec; + ntfs_dec_free_mft_records(vol, 1); + return 0; +undo_data_init: + write_lock_irqsave(&mft_ni->size_lock, flags); + mft_ni->initialized_size = old_data_initialized; + i_size_write(vol->mft_ino, old_data_size); + write_unlock_irqrestore(&mft_ni->size_lock, flags); + goto undo_mftbmp_alloc_nolock; +undo_mftbmp_alloc: + if (!base_ni || base_ni->mft_no != FILE_MFT) + down_write(&vol->mftbmp_lock); +undo_mftbmp_alloc_nolock: + if (ntfs_bitmap_clear_bit(vol->mftbmp_ino, bit)) { + ntfs_error(vol->sb, "Failed to clear bit in mft bitmap.%s", es); + NVolSetErrors(vol); + } + if (!base_ni || base_ni->mft_no != FILE_MFT) + up_write(&vol->mftbmp_lock); +err_out: + if (!base_ni || base_ni->mft_no != FILE_MFT) + mutex_unlock(&mft_ni->mrec_lock); + memalloc_nofs_restore(memalloc_flags); + return err; +max_err_out: + ntfs_warning(vol->sb, + "Cannot allocate mft record because the maximum number of inodes (2^32) has already been reached."); + if (!base_ni || base_ni->mft_no != FILE_MFT) { + up_write(&vol->mftbmp_lock); + mutex_unlock(&mft_ni->mrec_lock); + } + memalloc_nofs_restore(memalloc_flags); + return -ENOSPC; +} + +/* + * ntfs_mft_record_free - free an mft record on an ntfs volume + * @vol: volume on which to free the mft record + * @ni: open ntfs inode of the mft record to free + * + * Free the mft record of the open inode @ni on the mounted ntfs volume @vol. + * Note that this function calls ntfs_inode_close() internally and hence you + * cannot use the pointer @ni any more after this function returns success. + * + * On success return 0 and on error return -1 with errno set to the error code. + */ +int ntfs_mft_record_free(struct ntfs_volume *vol, struct ntfs_inode *ni) +{ + u64 mft_no; + int err; + u16 seq_no; + __le16 old_seq_no; + struct mft_record *ni_mrec; + unsigned int memalloc_flags; + struct ntfs_inode *base_ni; + + if (!vol || !ni) + return -EINVAL; + + ntfs_debug("Entering for inode 0x%llx.\n", (long long)ni->mft_no); + + ni_mrec = map_mft_record(ni); + if (IS_ERR(ni_mrec)) + return -EIO; + + /* Cache the mft reference for later. */ + mft_no = ni->mft_no; + + /* Mark the mft record as not in use. */ + ni_mrec->flags &= ~MFT_RECORD_IN_USE; + + /* Increment the sequence number, skipping zero, if it is not zero. */ + old_seq_no = ni_mrec->sequence_number; + seq_no = le16_to_cpu(old_seq_no); + if (seq_no == 0xffff) + seq_no = 1; + else if (seq_no) + seq_no++; + ni_mrec->sequence_number = cpu_to_le16(seq_no); + + down_read(&NTFS_I(vol->mft_ino)->runlist.lock); + err = ntfs_get_block_mft_record(NTFS_I(vol->mft_ino), ni); + up_read(&NTFS_I(vol->mft_ino)->runlist.lock); + if (err) { + unmap_mft_record(ni); + return err; + } + + /* + * Set the ntfs inode dirty and write it out. We do not need to worry + * about the base inode here since whatever caused the extent mft + * record to be freed is guaranteed to do it already. + */ + NInoSetDirty(ni); + err = write_mft_record(ni, ni_mrec, 0); + if (err) + goto sync_rollback; + + if (likely(ni->nr_extents >= 0)) + base_ni = ni; + else + base_ni = ni->ext.base_ntfs_ino; + + /* Clear the bit in the $MFT/$BITMAP corresponding to this record. */ + memalloc_flags = memalloc_nofs_save(); + if (base_ni->mft_no != FILE_MFT) + down_write(&vol->mftbmp_lock); + err = ntfs_bitmap_clear_bit(vol->mftbmp_ino, mft_no); + if (base_ni->mft_no != FILE_MFT) + up_write(&vol->mftbmp_lock); + memalloc_nofs_restore(memalloc_flags); + if (err) + goto bitmap_rollback; + + unmap_mft_record(ni); + ntfs_inc_free_mft_records(vol, 1); + return 0; + + /* Rollback what we did... */ +bitmap_rollback: + memalloc_flags = memalloc_nofs_save(); + if (base_ni->mft_no != FILE_MFT) + down_write(&vol->mftbmp_lock); + if (ntfs_bitmap_set_bit(vol->mftbmp_ino, mft_no)) + ntfs_error(vol->sb, "ntfs_bitmap_set_bit failed in bitmap_rollback\n"); + if (base_ni->mft_no != FILE_MFT) + up_write(&vol->mftbmp_lock); + memalloc_nofs_restore(memalloc_flags); +sync_rollback: + ntfs_error(vol->sb, + "Eeek! Rollback failed in %s. Leaving inconsistent metadata!\n", __func__); + ni_mrec->flags |= MFT_RECORD_IN_USE; + ni_mrec->sequence_number = old_seq_no; + NInoSetDirty(ni); + write_mft_record(ni, ni_mrec, 0); + unmap_mft_record(ni); + return err; +} + +static s64 lcn_from_index(struct ntfs_volume *vol, struct ntfs_inode *ni, + unsigned long index) +{ + s64 vcn; + s64 lcn; + + vcn = ntfs_pidx_to_cluster(vol, index); + + down_read(&ni->runlist.lock); + lcn = ntfs_attr_vcn_to_lcn_nolock(ni, vcn, false); + up_read(&ni->runlist.lock); + + return lcn; +} + +/* + * ntfs_write_mft_block - Write back a folio containing MFT records + * @folio: The folio to write back (contains one or more MFT records) + * @wbc: Writeback control structure + * + * This function is called as part of the address_space_operations + * .writepages implementation for the $MFT inode (or $MFTMirr). + * It handles writing one folio (normally 4KiB page) worth of MFT records + * to the underlying block device. + * + * Return: 0 on success, or -errno on error. + */ +static int ntfs_write_mft_block(struct folio *folio, struct writeback_control *wbc) +{ + struct address_space *mapping = folio->mapping; + struct inode *vi = mapping->host; + struct ntfs_inode *ni = NTFS_I(vi); + struct ntfs_volume *vol = ni->vol; + u8 *kaddr; + struct ntfs_inode **locked_nis __free(kfree) = kmalloc_array(PAGE_SIZE / NTFS_BLOCK_SIZE, + sizeof(struct ntfs_inode *), GFP_NOFS); + int nr_locked_nis = 0, err = 0, mft_ofs, prev_mft_ofs; + struct inode **ref_inos __free(kfree) = kmalloc_array(PAGE_SIZE / NTFS_BLOCK_SIZE, + sizeof(struct inode *), GFP_NOFS); + int nr_ref_inos = 0; + struct bio *bio = NULL; + u64 mft_no; + struct ntfs_inode *tni; + s64 lcn; + s64 vcn = ntfs_pidx_to_cluster(vol, folio->index); + s64 end_vcn = ntfs_bytes_to_cluster(vol, ni->allocated_size); + unsigned int folio_sz; + loff_t i_size = i_size_read(vi); + + ntfs_debug("Entering for inode 0x%llx, attribute type 0x%x, folio index 0x%lx.", + ni->mft_no, ni->type, folio->index); + + if (!locked_nis || !ref_inos) { + folio_redirty_for_writepage(wbc, folio); + folio_unlock(folio); + return -ENOMEM; + } + + /* We have to zero every time due to mmap-at-end-of-file. */ + if (folio->index >= (i_size >> folio_shift(folio))) + /* The page straddles i_size. */ + folio_zero_segment(folio, + offset_in_folio(folio, i_size), + folio_size(folio)); + + lcn = lcn_from_index(vol, ni, folio->index); + if (lcn <= LCN_HOLE) { + folio_start_writeback(folio); + folio_unlock(folio); + folio_end_writeback(folio); + return -EIO; + } + + /* Map folio so we can access its contents. */ + kaddr = kmap_local_folio(folio, 0); + /* Clear the page uptodate flag whilst the mst fixups are applied. */ + folio_clear_uptodate(folio); + + for (mft_ofs = 0; mft_ofs < PAGE_SIZE && vcn < end_vcn; + mft_ofs += vol->mft_record_size) { + /* Get the mft record number. */ + mft_no = (((s64)folio->index << PAGE_SHIFT) + mft_ofs) >> + vol->mft_record_size_bits; + vcn = ntfs_mft_no_to_cluster(vol, mft_no); + /* Check whether to write this mft record. */ + tni = NULL; + if (ntfs_may_write_mft_record(vol, mft_no, + (struct mft_record *)(kaddr + mft_ofs), + &tni, &ref_inos[nr_ref_inos])) { + unsigned int mft_record_off = 0; + s64 vcn_off = vcn; + s64 rl_len = 0; + + /* + * The record should be written. If a locked ntfs + * inode was returned, add it to the array of locked + * ntfs inodes. + */ + if (tni) + locked_nis[nr_locked_nis++] = tni; + else if (ref_inos[nr_ref_inos]) + nr_ref_inos++; + + if (bio && (mft_ofs != prev_mft_ofs + vol->mft_record_size)) { +flush_bio: + bio->bi_end_io = ntfs_bio_end_io; + submit_bio(bio); + bio = NULL; + } + + if (vol->cluster_size < folio_size(folio)) { + struct runlist_element *rl; + + down_write(&ni->runlist.lock); + rl = ntfs_attr_vcn_to_rl(ni, vcn_off, &lcn); + if (!IS_ERR(rl)) + rl_len = rl->length - (vcn_off - rl->vcn); + up_write(&ni->runlist.lock); + if (IS_ERR(rl) || lcn < 0) { + err = -EIO; + goto unm_done; + } + + if (bio && + (bio_end_sector(bio) >> (vol->cluster_size_bits - 9)) != + lcn) { + bio->bi_end_io = ntfs_bio_end_io; + submit_bio(bio); + bio = NULL; + } + } + + if (!bio) { + unsigned int off; + + off = ((mft_no << vol->mft_record_size_bits) + + mft_record_off) & vol->cluster_size_mask; + + bio = bio_alloc(vol->sb->s_bdev, 1, REQ_OP_WRITE, + GFP_NOIO); + bio->bi_iter.bi_sector = + ntfs_bytes_to_sector(vol, + ntfs_cluster_to_bytes(vol, lcn) + off); + } + + if (vol->cluster_size == NTFS_BLOCK_SIZE && + (mft_record_off || + rl_len == 1 || + mft_ofs + NTFS_BLOCK_SIZE >= PAGE_SIZE)) + folio_sz = NTFS_BLOCK_SIZE; + else + folio_sz = vol->mft_record_size; + if (!bio_add_folio(bio, folio, folio_sz, + mft_ofs + mft_record_off)) { + err = -EIO; + bio_put(bio); + goto unm_done; + } + mft_record_off += folio_sz; + + if (mft_record_off != vol->mft_record_size) { + vcn_off++; + goto flush_bio; + } + prev_mft_ofs = mft_ofs; + + if (mft_no < vol->mftmirr_size) { + int sub_err = ntfs_sync_mft_mirror(vol, mft_no, + (struct mft_record *)(kaddr + mft_ofs)); + + if (unlikely(sub_err) && !err) + err = sub_err; + } + } else if (ref_inos[nr_ref_inos]) + nr_ref_inos++; + } + + if (bio) { + bio->bi_end_io = ntfs_bio_end_io; + submit_bio(bio); + } +unm_done: + folio_mark_uptodate(folio); + kunmap_local(kaddr); + + folio_start_writeback(folio); + folio_unlock(folio); + folio_end_writeback(folio); + + /* Unlock any locked inodes. */ + while (nr_locked_nis-- > 0) { + struct ntfs_inode *base_tni; + + tni = locked_nis[nr_locked_nis]; + mutex_unlock(&tni->mrec_lock); + + /* Get the base inode. */ + mutex_lock(&tni->extent_lock); + if (tni->nr_extents >= 0) + base_tni = tni; + else + base_tni = tni->ext.base_ntfs_ino; + mutex_unlock(&tni->extent_lock); + ntfs_debug("Unlocking %s inode 0x%llx.", + tni == base_tni ? "base" : "extent", + tni->mft_no); + atomic_dec(&tni->count); + iput(VFS_I(base_tni)); + } + + /* Dropping deferred references */ + while (nr_ref_inos-- > 0) { + if (ref_inos[nr_ref_inos]) + iput(ref_inos[nr_ref_inos]); + } + + if (unlikely(err && err != -ENOMEM)) + NVolSetErrors(vol); + if (likely(!err)) + ntfs_debug("Done."); + return err; +} + +/* + * ntfs_mft_writepages - Write back dirty folios for the $MFT inode + * @mapping: address space of the $MFT inode + * @wbc: writeback control + * + * Writeback iterator for MFT records. Iterates over dirty folios and + * delegates actual writing to ntfs_write_mft_block() for each folio. + * Called from the address_space_operations .writepages vector of the + * $MFT inode. + * + * Returns 0 on success, or the first error encountered. + */ +int ntfs_mft_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct folio *folio = NULL; + int error; + + if (NVolShutdown(NTFS_I(mapping->host)->vol)) + return -EIO; + + while ((folio = writeback_iter(mapping, wbc, folio, &error))) + error = ntfs_write_mft_block(folio, wbc); + return error; +} + +void ntfs_mft_mark_dirty(struct folio *folio) +{ + iomap_dirty_folio(folio->mapping, folio); +} diff --git a/fs/ntfs/mft.h b/fs/ntfs/mft.h new file mode 100644 index 000000000..75a51a98d --- /dev/null +++ b/fs/ntfs/mft.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Defines for mft record handling in NTFS Linux kernel driver. + * + * Copyright (c) 2001-2004 Anton Altaparmakov + */ + +#ifndef _LINUX_NTFS_MFT_H +#define _LINUX_NTFS_MFT_H + +#include +#include + +#include "inode.h" + +struct mft_record *map_mft_record(struct ntfs_inode *ni); +void unmap_mft_record(struct ntfs_inode *ni); +struct mft_record *map_extent_mft_record(struct ntfs_inode *base_ni, u64 mref, + struct ntfs_inode **ntfs_ino); + +static inline void unmap_extent_mft_record(struct ntfs_inode *ni) +{ + unmap_mft_record(ni); +} + +void __mark_mft_record_dirty(struct ntfs_inode *ni); + +/* + * mark_mft_record_dirty - set the mft record and the page containing it dirty + * @ni: ntfs inode describing the mapped mft record + * + * Set the mapped (extent) mft record of the (base or extent) ntfs inode @ni, + * as well as the page containing the mft record, dirty. Also, mark the base + * vfs inode dirty. This ensures that any changes to the mft record are + * written out to disk. + * + * NOTE: Do not do anything if the mft record is already marked dirty. + */ +static inline void mark_mft_record_dirty(struct ntfs_inode *ni) +{ + if (!NInoTestSetDirty(ni)) + __mark_mft_record_dirty(ni); +} + +int ntfs_sync_mft_mirror(struct ntfs_volume *vol, const u64 mft_no, + struct mft_record *m); +int write_mft_record_nolock(struct ntfs_inode *ni, struct mft_record *m, int sync); + +/* + * write_mft_record - write out a mapped (extent) mft record + * @ni: ntfs inode describing the mapped (extent) mft record + * @m: mapped (extent) mft record to write + * @sync: if true, wait for i/o completion + * + * This is just a wrapper for write_mft_record_nolock() (see mft.c), which + * locks the page for the duration of the write. This ensures that there are + * no race conditions between writing the mft record via the dirty inode code + * paths and via the page cache write back code paths or between writing + * neighbouring mft records residing in the same page. + * + * Locking the page also serializes us against ->read_folio() if the page is not + * uptodate. + * + * On success, clean the mft record and return 0. On error, leave the mft + * record dirty and return -errno. + */ +static inline int write_mft_record(struct ntfs_inode *ni, struct mft_record *m, int sync) +{ + struct folio *folio = ni->folio; + int err; + + folio_lock(folio); + err = write_mft_record_nolock(ni, m, sync); + folio_unlock(folio); + + return err; +} + +int ntfs_mft_record_alloc(struct ntfs_volume *vol, const int mode, + struct ntfs_inode **ni, struct ntfs_inode *base_ni, + struct mft_record **ni_mrec); +int ntfs_mft_record_free(struct ntfs_volume *vol, struct ntfs_inode *ni); +int ntfs_mft_records_write(const struct ntfs_volume *vol, const u64 mref, + const s64 count, struct mft_record *b); +int ntfs_mft_record_check(const struct ntfs_volume *vol, struct mft_record *m, + u64 mft_no); +int ntfs_mft_writepages(struct address_space *mapping, + struct writeback_control *wbc); +void ntfs_mft_mark_dirty(struct folio *folio); + +#endif /* _LINUX_NTFS_MFT_H */ diff --git a/fs/ntfs/mst.c b/fs/ntfs/mst.c new file mode 100644 index 000000000..7f9faad92 --- /dev/null +++ b/fs/ntfs/mst.c @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * NTFS multi sector transfer protection handling code. + * + * Copyright (c) 2001-2004 Anton Altaparmakov + */ + +#include + +#include "ntfs.h" + +/* + * post_read_mst_fixup - deprotect multi sector transfer protected data + * @b: pointer to the data to deprotect + * @size: size in bytes of @b + * + * Perform the necessary post read multi sector transfer fixup and detect the + * presence of incomplete multi sector transfers. - In that case, overwrite the + * magic of the ntfs record header being processed with "BAAD" (in memory only!) + * and abort processing. + * + * Return 0 on success and -EINVAL on error ("BAAD" magic will be present). + * + * NOTE: We consider the absence / invalidity of an update sequence array to + * mean that the structure is not protected at all and hence doesn't need to + * be fixed up. Thus, we return success and not failure in this case. This is + * in contrast to pre_write_mst_fixup(), see below. + */ +int post_read_mst_fixup(struct ntfs_record *b, const u32 size) +{ + u16 usa_ofs, usa_count, usn; + u16 *usa_pos, *data_pos; + + /* Setup the variables. */ + usa_ofs = le16_to_cpu(b->usa_ofs); + /* Decrement usa_count to get number of fixups. */ + usa_count = le16_to_cpu(b->usa_count) - 1; + /* Size and alignment checks. */ + if (size & (NTFS_BLOCK_SIZE - 1) || usa_ofs & 1 || + usa_ofs + (usa_count * 2) > size || + (size >> NTFS_BLOCK_SIZE_BITS) != usa_count) + return 0; + /* Position of usn in update sequence array. */ + usa_pos = (u16 *)b + usa_ofs/sizeof(u16); + /* + * The update sequence number which has to be equal to each of the + * u16 values before they are fixed up. Note no need to care for + * endianness since we are comparing and moving data for on disk + * structures which means the data is consistent. - If it is + * consistenty the wrong endianness it doesn't make any difference. + */ + usn = *usa_pos; + /* + * Position in protected data of first u16 that needs fixing up. + */ + data_pos = (u16 *)b + NTFS_BLOCK_SIZE / sizeof(u16) - 1; + /* + * Check for incomplete multi sector transfer(s). + */ + while (usa_count--) { + if (*data_pos != usn) { + struct mft_record *m = (struct mft_record *)b; + + pr_err_ratelimited("ntfs: Incomplete multi sector transfer detected! (Record magic : 0x%x, mft number : 0x%x, base mft number : 0x%lx, mft in use : %d, data : 0x%x, usn 0x%x)\n", + le32_to_cpu(m->magic), le32_to_cpu(m->mft_record_number), + MREF_LE(m->base_mft_record), m->flags & MFT_RECORD_IN_USE, + *data_pos, usn); + /* + * Incomplete multi sector transfer detected! )-: + * Set the magic to "BAAD" and return failure. + * Note that magic_BAAD is already converted to le32. + */ + b->magic = magic_BAAD; + return -EINVAL; + } + data_pos += NTFS_BLOCK_SIZE / sizeof(u16); + } + /* Re-setup the variables. */ + usa_count = le16_to_cpu(b->usa_count) - 1; + data_pos = (u16 *)b + NTFS_BLOCK_SIZE / sizeof(u16) - 1; + /* Fixup all sectors. */ + while (usa_count--) { + /* + * Increment position in usa and restore original data from + * the usa into the data buffer. + */ + *data_pos = *(++usa_pos); + /* Increment position in data as well. */ + data_pos += NTFS_BLOCK_SIZE/sizeof(u16); + } + return 0; +} + +/* + * pre_write_mst_fixup - apply multi sector transfer protection + * @b: pointer to the data to protect + * @size: size in bytes of @b + * + * Perform the necessary pre write multi sector transfer fixup on the data + * pointer to by @b of @size. + * + * Return 0 if fixup applied (success) or -EINVAL if no fixup was performed + * (assumed not needed). This is in contrast to post_read_mst_fixup() above. + * + * NOTE: We consider the absence / invalidity of an update sequence array to + * mean that the structure is not subject to protection and hence doesn't need + * to be fixed up. This means that you have to create a valid update sequence + * array header in the ntfs record before calling this function, otherwise it + * will fail (the header needs to contain the position of the update sequence + * array together with the number of elements in the array). You also need to + * initialise the update sequence number before calling this function + * otherwise a random word will be used (whatever was in the record at that + * position at that time). + */ +int pre_write_mst_fixup(struct ntfs_record *b, const u32 size) +{ + __le16 *usa_pos, *data_pos; + u16 usa_ofs, usa_count, usn; + __le16 le_usn; + + /* Sanity check + only fixup if it makes sense. */ + if (!b || ntfs_is_baad_record(b->magic) || + ntfs_is_hole_record(b->magic)) + return -EINVAL; + /* Setup the variables. */ + usa_ofs = le16_to_cpu(b->usa_ofs); + /* Decrement usa_count to get number of fixups. */ + usa_count = le16_to_cpu(b->usa_count) - 1; + /* Size and alignment checks. */ + if (size & (NTFS_BLOCK_SIZE - 1) || usa_ofs & 1 || + usa_ofs + (usa_count * 2) > size || + (size >> NTFS_BLOCK_SIZE_BITS) != usa_count) + return -EINVAL; + /* Position of usn in update sequence array. */ + usa_pos = (__le16 *)((u8 *)b + usa_ofs); + /* + * Cyclically increment the update sequence number + * (skipping 0 and -1, i.e. 0xffff). + */ + usn = le16_to_cpup(usa_pos) + 1; + if (usn == 0xffff || !usn) + usn = 1; + le_usn = cpu_to_le16(usn); + *usa_pos = le_usn; + /* Position in data of first u16 that needs fixing up. */ + data_pos = (__le16 *)b + NTFS_BLOCK_SIZE/sizeof(__le16) - 1; + /* Fixup all sectors. */ + while (usa_count--) { + /* + * Increment the position in the usa and save the + * original data from the data buffer into the usa. + */ + *(++usa_pos) = *data_pos; + /* Apply fixup to data. */ + *data_pos = le_usn; + /* Increment position in data as well. */ + data_pos += NTFS_BLOCK_SIZE / sizeof(__le16); + } + return 0; +} + +/* + * post_write_mst_fixup - fast deprotect multi sector transfer protected data + * @b: pointer to the data to deprotect + * + * Perform the necessary post write multi sector transfer fixup, not checking + * for any errors, because we assume we have just used pre_write_mst_fixup(), + * thus the data will be fine or we would never have gotten here. + */ +void post_write_mst_fixup(struct ntfs_record *b) +{ + __le16 *usa_pos, *data_pos; + + u16 usa_ofs = le16_to_cpu(b->usa_ofs); + u16 usa_count = le16_to_cpu(b->usa_count) - 1; + + /* Position of usn in update sequence array. */ + usa_pos = (__le16 *)b + usa_ofs/sizeof(__le16); + + /* Position in protected data of first u16 that needs fixing up. */ + data_pos = (__le16 *)b + NTFS_BLOCK_SIZE/sizeof(__le16) - 1; + + /* Fixup all sectors. */ + while (usa_count--) { + /* + * Increment position in usa and restore original data from + * the usa into the data buffer. + */ + *data_pos = *(++usa_pos); + + /* Increment position in data as well. */ + data_pos += NTFS_BLOCK_SIZE/sizeof(__le16); + } +} diff --git a/fs/ntfs/namei.c b/fs/ntfs/namei.c new file mode 100644 index 000000000..8364d7bb5 --- /dev/null +++ b/fs/ntfs/namei.c @@ -0,0 +1,1682 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * NTFS kernel directory inode operations. + * + * Copyright (c) 2001-2006 Anton Altaparmakov + * Copyright (c) 2025 LG Electronics Co., Ltd. + */ + +#include +#include + +#include "ntfs.h" +#include "time.h" +#include "index.h" +#include "reparse.h" +#include "object_id.h" +#include "ea.h" + +static const __le16 aux_name_le[3] = { + cpu_to_le16('A'), cpu_to_le16('U'), cpu_to_le16('X') +}; + +static const __le16 con_name_le[3] = { + cpu_to_le16('C'), cpu_to_le16('O'), cpu_to_le16('N') +}; + +static const __le16 com_name_le[3] = { + cpu_to_le16('C'), cpu_to_le16('O'), cpu_to_le16('M') +}; + +static const __le16 lpt_name_le[3] = { + cpu_to_le16('L'), cpu_to_le16('P'), cpu_to_le16('T') +}; + +static const __le16 nul_name_le[3] = { + cpu_to_le16('N'), cpu_to_le16('U'), cpu_to_le16('L') +}; + +static const __le16 prn_name_le[3] = { + cpu_to_le16('P'), cpu_to_le16('R'), cpu_to_le16('N') +}; + +static inline int ntfs_check_bad_char(const __le16 *wc, unsigned int wc_len) +{ + int i; + + for (i = 0; i < wc_len; i++) { + u16 c = le16_to_cpu(wc[i]); + + if (c < 0x0020 || + c == 0x0022 || c == 0x002A || c == 0x002F || + c == 0x003A || c == 0x003C || c == 0x003E || + c == 0x003F || c == 0x005C || c == 0x007C) + return -EINVAL; + } + + return 0; +} + +static int ntfs_check_bad_windows_name(struct ntfs_volume *vol, + const __le16 *wc, + unsigned int wc_len) +{ + if (ntfs_check_bad_char(wc, wc_len)) + return -EINVAL; + + if (!NVolCheckWindowsNames(vol)) + return 0; + + /* Check for trailing space or dot. */ + if (wc_len > 0 && + (wc[wc_len - 1] == cpu_to_le16(' ') || + wc[wc_len - 1] == cpu_to_le16('.'))) + return -EINVAL; + + if (wc_len == 3 || (wc_len > 3 && wc[3] == cpu_to_le16('.'))) { + __le16 *upcase = vol->upcase; + u32 size = vol->upcase_len; + + if (ntfs_are_names_equal(wc, 3, aux_name_le, 3, IGNORE_CASE, upcase, size) || + ntfs_are_names_equal(wc, 3, con_name_le, 3, IGNORE_CASE, upcase, size) || + ntfs_are_names_equal(wc, 3, nul_name_le, 3, IGNORE_CASE, upcase, size) || + ntfs_are_names_equal(wc, 3, prn_name_le, 3, IGNORE_CASE, upcase, size)) + return -EINVAL; + } + + if (wc_len == 4 || (wc_len > 4 && wc[4] == cpu_to_le16('.'))) { + __le16 *upcase = vol->upcase; + u32 size = vol->upcase_len, port; + + if (ntfs_are_names_equal(wc, 3, com_name_le, 3, IGNORE_CASE, upcase, size) || + ntfs_are_names_equal(wc, 3, lpt_name_le, 3, IGNORE_CASE, upcase, size)) { + port = le16_to_cpu(wc[3]); + if (port >= '1' && port <= '9') + return -EINVAL; + } + } + return 0; +} + +/* + * ntfs_lookup - find the inode represented by a dentry in a directory inode + * @dir_ino: directory inode in which to look for the inode + * @dent: dentry representing the inode to look for + * @flags: lookup flags + * + * In short, ntfs_lookup() looks for the inode represented by the dentry @dent + * in the directory inode @dir_ino and if found attaches the inode to the + * dentry @dent. + * + * In more detail, the dentry @dent specifies which inode to look for by + * supplying the name of the inode in @dent->d_name.name. ntfs_lookup() + * converts the name to Unicode and walks the contents of the directory inode + * @dir_ino looking for the converted Unicode name. If the name is found in the + * directory, the corresponding inode is loaded by calling ntfs_iget() on its + * inode number and the inode is associated with the dentry @dent via a call to + * d_splice_alias(). + * + * If the name is not found in the directory, a NULL inode is inserted into the + * dentry @dent via a call to d_add(). The dentry is then termed a negative + * dentry. + * + * Only if an actual error occurs, do we return an error via ERR_PTR(). + * + * In order to handle the case insensitivity issues of NTFS with regards to the + * dcache and the dcache requiring only one dentry per directory, we deal with + * dentry aliases that only differ in case in ->ntfs_lookup() while maintaining + * a case sensitive dcache. This means that we get the full benefit of dcache + * speed when the file/directory is looked up with the same case as returned by + * ->ntfs_readdir() but that a lookup for any other case (or for the short file + * name) will not find anything in dcache and will enter ->ntfs_lookup() + * instead, where we search the directory for a fully matching file name + * (including case) and if that is not found, we search for a file name that + * matches with different case and if that has non-POSIX semantics we return + * that. We actually do only one search (case sensitive) and keep tabs on + * whether we have found a case insensitive match in the process. + * + * To simplify matters for us, we do not treat the short vs long filenames as + * two hard links but instead if the lookup matches a short filename, we + * return the dentry for the corresponding long filename instead. + * + * There are three cases we need to distinguish here: + * + * 1) @dent perfectly matches (i.e. including case) a directory entry with a + * file name in the WIN32 or POSIX namespaces. In this case + * ntfs_lookup_inode_by_name() will return with name set to NULL and we + * just d_splice_alias() @dent. + * 2) @dent matches (not including case) a directory entry with a file name in + * the WIN32 namespace. In this case ntfs_lookup_inode_by_name() will return + * with name set to point to a kmalloc()ed ntfs_name structure containing + * the properly cased little endian Unicode name. We convert the name to the + * current NLS code page, search if a dentry with this name already exists + * and if so return that instead of @dent. At this point things are + * complicated by the possibility of 'disconnected' dentries due to NFS + * which we deal with appropriately (see the code comments). The VFS will + * then destroy the old @dent and use the one we returned. If a dentry is + * not found, we allocate a new one, d_splice_alias() it, and return it as + * above. + * 3) @dent matches either perfectly or not (i.e. we don't care about case) a + * directory entry with a file name in the DOS namespace. In this case + * ntfs_lookup_inode_by_name() will return with name set to point to a + * kmalloc()ed ntfs_name structure containing the mft reference (cpu endian) + * of the inode. We use the mft reference to read the inode and to find the + * file name in the WIN32 namespace corresponding to the matched short file + * name. We then convert the name to the current NLS code page, and proceed + * searching for a dentry with this name, etc, as in case 2), above. + * + * Locking: Caller must hold i_mutex on the directory. + */ +static struct dentry *ntfs_lookup(struct inode *dir_ino, struct dentry *dent, + unsigned int flags) +{ + struct ntfs_volume *vol = NTFS_SB(dir_ino->i_sb); + struct inode *dent_inode; + __le16 *uname; + struct ntfs_name *name = NULL; + u64 mref; + unsigned long dent_ino; + int uname_len; + + ntfs_debug("Looking up %pd in directory inode 0x%llx.", + dent, NTFS_I(dir_ino)->mft_no); + /* Convert the name of the dentry to Unicode. */ + uname_len = ntfs_nlstoucs(vol, dent->d_name.name, dent->d_name.len, + &uname, NTFS_MAX_NAME_LEN); + if (uname_len < 0) { + if (uname_len != -ENAMETOOLONG) + ntfs_debug("Failed to convert name to Unicode."); + return ERR_PTR(uname_len); + } + mutex_lock(&NTFS_I(dir_ino)->mrec_lock); + mref = ntfs_lookup_inode_by_name(NTFS_I(dir_ino), uname, uname_len, + &name); + mutex_unlock(&NTFS_I(dir_ino)->mrec_lock); + kmem_cache_free(ntfs_name_cache, uname); + if (!IS_ERR_MREF(mref)) { + dent_ino = MREF(mref); + ntfs_debug("Found inode 0x%lx. Calling ntfs_iget.", dent_ino); + dent_inode = ntfs_iget(vol->sb, dent_ino); + if (!IS_ERR(dent_inode)) { + /* Consistency check. */ + if (MSEQNO(mref) == NTFS_I(dent_inode)->seq_no || + dent_ino == FILE_MFT) { + /* Perfect WIN32/POSIX match. -- Case 1. */ + if (!name) { + ntfs_debug("Done. (Case 1.)"); + return d_splice_alias(dent_inode, dent); + } + /* + * We are too indented. Handle imperfect + * matches and short file names further below. + */ + goto handle_name; + } + ntfs_error(vol->sb, + "Found stale reference to inode 0x%lx (reference sequence number = 0x%x, inode sequence number = 0x%x), returning -EIO. Run chkdsk.", + dent_ino, MSEQNO(mref), + NTFS_I(dent_inode)->seq_no); + iput(dent_inode); + dent_inode = ERR_PTR(-EIO); + } else + ntfs_error(vol->sb, "ntfs_iget(0x%lx) failed with error code %li.", + dent_ino, PTR_ERR(dent_inode)); + kfree(name); + /* Return the error code. */ + return ERR_CAST(dent_inode); + } + kfree(name); + /* It is guaranteed that @name is no longer allocated at this point. */ + if (MREF_ERR(mref) == -ENOENT) { + ntfs_debug("Entry was not found, adding negative dentry."); + /* The dcache will handle negative entries. */ + ntfs_debug("Done."); + return d_splice_alias(NULL, dent); + } + ntfs_error(vol->sb, "ntfs_lookup_ino_by_name() failed with error code %i.", + -MREF_ERR(mref)); + return ERR_PTR(MREF_ERR(mref)); +handle_name: + { + struct mft_record *m; + struct ntfs_attr_search_ctx *ctx; + struct ntfs_inode *ni = NTFS_I(dent_inode); + int err; + struct qstr nls_name; + + nls_name.name = NULL; + if (name->type != FILE_NAME_DOS) { /* Case 2. */ + ntfs_debug("Case 2."); + nls_name.len = (unsigned int)ntfs_ucstonls(vol, + (__le16 *)&name->name, name->len, + (unsigned char **)&nls_name.name, 0); + kfree(name); + } else /* if (name->type == FILE_NAME_DOS) */ { /* Case 3. */ + struct file_name_attr *fn; + + ntfs_debug("Case 3."); + kfree(name); + + /* Find the WIN32 name corresponding to the matched DOS name. */ + ni = NTFS_I(dent_inode); + m = map_mft_record(ni); + if (IS_ERR(m)) { + err = PTR_ERR(m); + m = NULL; + ctx = NULL; + goto err_out; + } + ctx = ntfs_attr_get_search_ctx(ni, m); + if (unlikely(!ctx)) { + err = -ENOMEM; + goto err_out; + } + do { + struct attr_record *a; + + err = ntfs_attr_lookup(AT_FILE_NAME, NULL, 0, 0, 0, + NULL, 0, ctx); + if (unlikely(err)) { + ntfs_error(vol->sb, + "Inode corrupt: No WIN32 namespace counterpart to DOS file name. Run chkdsk."); + if (err == -ENOENT) + err = -EIO; + goto err_out; + } + /* Consistency checks. */ + a = ctx->attr; + if (a->non_resident || a->flags) + goto eio_err_out; + fn = (struct file_name_attr *)((u8 *)ctx->attr + le16_to_cpu( + ctx->attr->data.resident.value_offset)); + } while (fn->file_name_type != FILE_NAME_WIN32); + + /* Convert the found WIN32 name to current NLS code page. */ + nls_name.len = (unsigned int)ntfs_ucstonls(vol, + (__le16 *)&fn->file_name, fn->file_name_length, + (unsigned char **)&nls_name.name, 0); + + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(ni); + } + m = NULL; + ctx = NULL; + + /* Check if a conversion error occurred. */ + if ((int)nls_name.len < 0) { + err = (int)nls_name.len; + goto err_out; + } + nls_name.hash = full_name_hash(dent, nls_name.name, nls_name.len); + + dent = d_add_ci(dent, dent_inode, &nls_name); + kfree(nls_name.name); + return dent; + +eio_err_out: + ntfs_error(vol->sb, "Illegal file name attribute. Run chkdsk."); + err = -EIO; +err_out: + if (ctx) + ntfs_attr_put_search_ctx(ctx); + if (m) + unmap_mft_record(ni); + iput(dent_inode); + ntfs_error(vol->sb, "Failed, returning error code %i.", err); + return ERR_PTR(err); + } +} + +static int ntfs_sd_add_everyone(struct ntfs_inode *ni) +{ + struct security_descriptor_relative *sd; + struct ntfs_acl *acl; + struct ntfs_ace *ace; + struct ntfs_sid *sid; + int ret, sd_len; + + /* Create SECURITY_DESCRIPTOR attribute (everyone has full access). */ + /* + * Calculate security descriptor length. We have 2 sub-authorities in + * owner and group SIDs, So add 8 bytes to every SID. + */ + sd_len = sizeof(struct security_descriptor_relative) + 2 * + (sizeof(struct ntfs_sid) + 8) + sizeof(struct ntfs_acl) + + sizeof(struct ntfs_ace) + 4; + sd = kzalloc(sd_len, GFP_NOFS); + if (!sd) + return -ENOMEM; + + sd->revision = 1; + sd->control = SE_DACL_PRESENT | SE_SELF_RELATIVE; + + sid = (struct ntfs_sid *)((u8 *)sd + sizeof(struct security_descriptor_relative)); + sid->revision = 1; + sid->sub_authority_count = 2; + sid->sub_authority[0] = cpu_to_le32(SECURITY_BUILTIN_DOMAIN_RID); + sid->sub_authority[1] = cpu_to_le32(DOMAIN_ALIAS_RID_ADMINS); + sid->identifier_authority.value[5] = 5; + sd->owner = cpu_to_le32((u8 *)sid - (u8 *)sd); + + sid = (struct ntfs_sid *)((u8 *)sid + sizeof(struct ntfs_sid) + 8); + sid->revision = 1; + sid->sub_authority_count = 2; + sid->sub_authority[0] = cpu_to_le32(SECURITY_BUILTIN_DOMAIN_RID); + sid->sub_authority[1] = cpu_to_le32(DOMAIN_ALIAS_RID_ADMINS); + sid->identifier_authority.value[5] = 5; + sd->group = cpu_to_le32((u8 *)sid - (u8 *)sd); + + acl = (struct ntfs_acl *)((u8 *)sid + sizeof(struct ntfs_sid) + 8); + acl->revision = 2; + acl->size = cpu_to_le16(sizeof(struct ntfs_acl) + sizeof(struct ntfs_ace) + 4); + acl->ace_count = cpu_to_le16(1); + sd->dacl = cpu_to_le32((u8 *)acl - (u8 *)sd); + + ace = (struct ntfs_ace *)((u8 *)acl + sizeof(struct ntfs_acl)); + ace->type = ACCESS_ALLOWED_ACE_TYPE; + ace->flags = OBJECT_INHERIT_ACE | CONTAINER_INHERIT_ACE; + ace->size = cpu_to_le16(sizeof(struct ntfs_ace) + 4); + ace->mask = cpu_to_le32(0x1f01ff); + ace->sid.revision = 1; + ace->sid.sub_authority_count = 1; + ace->sid.sub_authority[0] = 0; + ace->sid.identifier_authority.value[5] = 1; + + ret = ntfs_attr_add(ni, AT_SECURITY_DESCRIPTOR, AT_UNNAMED, 0, (u8 *)sd, + sd_len); + if (ret) + ntfs_error(ni->vol->sb, "Failed to add SECURITY_DESCRIPTOR\n"); + + kfree(sd); + return ret; +} + +static struct ntfs_inode *__ntfs_create(struct mnt_idmap *idmap, struct inode *dir, + __le16 *name, u8 name_len, mode_t mode, dev_t dev, + const char *target, int target_len) +{ + struct ntfs_inode *dir_ni = NTFS_I(dir); + struct ntfs_volume *vol = dir_ni->vol; + struct ntfs_inode *ni; + bool rollback_data = false, rollback_sd = false, rollback_reparse = false; + struct file_name_attr *fn = NULL; + struct standard_information *si = NULL; + int err = 0, fn_len, si_len; + struct inode *vi; + struct mft_record *ni_mrec, *dni_mrec; + struct super_block *sb = dir_ni->vol->sb; + __le64 parent_mft_ref; + u64 child_mft_ref; + __le16 ea_size; + + vi = new_inode(vol->sb); + if (!vi) + return ERR_PTR(-ENOMEM); + + ntfs_init_big_inode(vi); + ni = NTFS_I(vi); + ni->vol = dir_ni->vol; + ni->name_len = 0; + ni->name = NULL; + + /* + * Set the appropriate mode, attribute type, and name. For + * directories, also setup the index values to the defaults. + */ + if (S_ISDIR(mode)) { + mode &= ~vol->dmask; + + NInoSetMstProtected(ni); + ni->itype.index.block_size = 4096; + ni->itype.index.block_size_bits = ntfs_ffs(4096) - 1; + ni->itype.index.collation_rule = COLLATION_FILE_NAME; + if (vol->cluster_size <= ni->itype.index.block_size) { + ni->itype.index.vcn_size = vol->cluster_size; + ni->itype.index.vcn_size_bits = + vol->cluster_size_bits; + } else { + ni->itype.index.vcn_size = vol->sector_size; + ni->itype.index.vcn_size_bits = + vol->sector_size_bits; + } + } else { + mode &= ~vol->fmask; + } + + if (IS_RDONLY(vi)) + mode &= ~0222; + + inode_init_owner(idmap, vi, dir, mode); + + mode = vi->i_mode; + +#ifdef CONFIG_NTFS_FS_POSIX_ACL + if (!S_ISLNK(mode) && (sb->s_flags & SB_POSIXACL)) { + err = ntfs_init_acl(idmap, vi, dir); + if (err) + goto err_out; + } else +#endif + { + vi->i_flags |= S_NOSEC; + } + + if (uid_valid(vol->uid)) + vi->i_uid = vol->uid; + + if (gid_valid(vol->gid)) + vi->i_gid = vol->gid; + + /* + * Set the file size to 0, the ntfs inode sizes are set to 0 by + * the call to ntfs_init_big_inode() below. + */ + vi->i_size = 0; + vi->i_blocks = 0; + + inode_inc_iversion(vi); + + simple_inode_init_ts(vi); + ni->i_crtime = inode_get_ctime(vi); + + inode_set_mtime_to_ts(dir, ni->i_crtime); + inode_set_ctime_to_ts(dir, ni->i_crtime); + mark_inode_dirty(dir); + + err = ntfs_mft_record_alloc(dir_ni->vol, mode, &ni, NULL, + &ni_mrec); + if (err) { + iput(vi); + return ERR_PTR(err); + } + + /* + * Prevent iget and writeback from finding this inode. + * Caller must call d_instantiate_new instead of d_instantiate. + */ + spin_lock(&vi->i_lock); + inode_state_set(vi, I_NEW | I_CREATING); + spin_unlock(&vi->i_lock); + + /* Add the inode to the inode hash for the superblock. */ + vi->i_ino = (unsigned long)ni->mft_no; + inode_set_iversion(vi, 1); + insert_inode_hash(vi); + + mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL); + mutex_lock_nested(&dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT); + if (NInoBeingDeleted(dir_ni)) { + err = -ENOENT; + goto err_out; + } + + dni_mrec = map_mft_record(dir_ni); + if (IS_ERR(dni_mrec)) { + ntfs_error(dir_ni->vol->sb, "failed to map mft record for file 0x%llx.\n", + dir_ni->mft_no); + err = -EIO; + goto err_out; + } + parent_mft_ref = MK_LE_MREF(dir_ni->mft_no, + le16_to_cpu(dni_mrec->sequence_number)); + unmap_mft_record(dir_ni); + + /* + * Create STANDARD_INFORMATION attribute. Write STANDARD_INFORMATION + * version 1.2, windows will upgrade it to version 3 if needed. + */ + si_len = offsetof(struct standard_information, file_attributes) + + sizeof(__le32) + 12; + si = kzalloc(si_len, GFP_NOFS); + if (!si) { + err = -ENOMEM; + goto err_out; + } + + si->creation_time = si->last_data_change_time = utc2ntfs(ni->i_crtime); + si->last_mft_change_time = si->last_access_time = si->creation_time; + + if (!S_ISREG(mode) && !S_ISDIR(mode)) + si->file_attributes = FILE_ATTR_SYSTEM; + + /* Add STANDARD_INFORMATION to inode. */ + err = ntfs_attr_add(ni, AT_STANDARD_INFORMATION, AT_UNNAMED, 0, (u8 *)si, + si_len); + if (err) { + ntfs_error(sb, "Failed to add STANDARD_INFORMATION attribute.\n"); + goto err_out; + } + + err = ntfs_sd_add_everyone(ni); + if (err) + goto err_out; + rollback_sd = true; + + if (S_ISDIR(mode)) { + struct index_root *ir = NULL; + struct index_entry *ie; + int ir_len, index_len; + + /* Create struct index_root attribute. */ + index_len = sizeof(struct index_header) + sizeof(struct index_entry_header); + ir_len = offsetof(struct index_root, index) + index_len; + ir = kzalloc(ir_len, GFP_NOFS); + if (!ir) { + err = -ENOMEM; + goto err_out; + } + ir->type = AT_FILE_NAME; + ir->collation_rule = COLLATION_FILE_NAME; + ir->index_block_size = cpu_to_le32(ni->vol->index_record_size); + if (ni->vol->cluster_size <= ni->vol->index_record_size) + ir->clusters_per_index_block = + NTFS_B_TO_CLU(vol, ni->vol->index_record_size); + else + ir->clusters_per_index_block = + ni->vol->index_record_size >> ni->vol->sector_size_bits; + ir->index.entries_offset = cpu_to_le32(sizeof(struct index_header)); + ir->index.index_length = cpu_to_le32(index_len); + ir->index.allocated_size = cpu_to_le32(index_len); + ie = (struct index_entry *)((u8 *)ir + sizeof(struct index_root)); + ie->length = cpu_to_le16(sizeof(struct index_entry_header)); + ie->key_length = 0; + ie->flags = INDEX_ENTRY_END; + + /* Add struct index_root attribute to inode. */ + err = ntfs_attr_add(ni, AT_INDEX_ROOT, I30, 4, (u8 *)ir, ir_len); + if (err) { + kfree(ir); + ntfs_error(vi->i_sb, "Failed to add struct index_root attribute.\n"); + goto err_out; + } + kfree(ir); + err = ntfs_attr_open(ni, AT_INDEX_ROOT, I30, 4); + if (err) + goto err_out; + } else { + /* Add DATA attribute to inode. */ + err = ntfs_attr_add(ni, AT_DATA, AT_UNNAMED, 0, NULL, 0); + if (err) { + ntfs_error(dir_ni->vol->sb, "Failed to add DATA attribute.\n"); + goto err_out; + } + rollback_data = true; + + err = ntfs_attr_open(ni, AT_DATA, AT_UNNAMED, 0); + if (err) + goto err_out; + + if (S_ISLNK(mode)) { + if (NVolSymlinkNative(vol)) + err = ntfs_reparse_set_native_symlink(ni, target, target_len); + else + err = ntfs_reparse_set_wsl_symlink(ni, target, target_len); + if (!err) + rollback_reparse = true; + } else if (S_ISBLK(mode) || S_ISCHR(mode) || S_ISSOCK(mode) || + S_ISFIFO(mode)) { + si->file_attributes = FILE_ATTRIBUTE_RECALL_ON_OPEN; + ni->flags = FILE_ATTRIBUTE_RECALL_ON_OPEN; + err = ntfs_reparse_set_wsl_not_symlink(ni, mode); + if (!err) + rollback_reparse = true; + } + if (err) + goto err_out; + } + + err = ntfs_ea_set_wsl_inode(vi, dev, &ea_size, + NTFS_EA_UID | NTFS_EA_GID | NTFS_EA_MODE); + if (err) + goto err_out; + + /* Create FILE_NAME attribute. */ + fn_len = sizeof(struct file_name_attr) + name_len * sizeof(__le16); + fn = kzalloc(fn_len, GFP_NOFS); + if (!fn) { + err = -ENOMEM; + goto err_out; + } + + fn->file_attributes |= ni->flags; + fn->parent_directory = parent_mft_ref; + fn->file_name_length = name_len; + fn->file_name_type = FILE_NAME_POSIX; + fn->type.ea.packed_ea_size = ea_size; + if (S_ISDIR(mode)) { + fn->file_attributes = FILE_ATTR_DUP_FILE_NAME_INDEX_PRESENT; + fn->allocated_size = fn->data_size = 0; + } else { + fn->data_size = cpu_to_le64(ni->data_size); + fn->allocated_size = cpu_to_le64(ni->allocated_size); + } + if (!S_ISREG(mode) && !S_ISDIR(mode)) { + fn->file_attributes = FILE_ATTR_SYSTEM; + if (rollback_reparse) + fn->file_attributes |= FILE_ATTR_REPARSE_POINT; + } + if (NVolHideDotFiles(vol) && name_len > 0 && name[0] == cpu_to_le16('.')) + fn->file_attributes |= FILE_ATTR_HIDDEN; + fn->creation_time = fn->last_data_change_time = utc2ntfs(ni->i_crtime); + fn->last_mft_change_time = fn->last_access_time = fn->creation_time; + memcpy(fn->file_name, name, name_len * sizeof(__le16)); + + /* Add FILE_NAME attribute to inode. */ + err = ntfs_attr_add(ni, AT_FILE_NAME, AT_UNNAMED, 0, (u8 *)fn, fn_len); + if (err) { + ntfs_error(sb, "Failed to add FILE_NAME attribute.\n"); + goto err_out; + } + + child_mft_ref = MK_MREF(ni->mft_no, + le16_to_cpu(ni_mrec->sequence_number)); + /* Set hard links count and directory flag. */ + ni_mrec->link_count = cpu_to_le16(1); + mark_mft_record_dirty(ni); + + /* Add FILE_NAME attribute to index. */ + err = ntfs_index_add_filename(dir_ni, fn, child_mft_ref); + if (err) { + ntfs_debug("Failed to add entry to the index"); + goto err_out; + } + + unmap_mft_record(ni); + mutex_unlock(&dir_ni->mrec_lock); + mutex_unlock(&ni->mrec_lock); + + ni->flags = fn->file_attributes | + (ni->flags & FILE_ATTRIBUTE_RECALL_ON_OPEN); + /* Set the sequence number. */ + vi->i_generation = ni->seq_no; + set_nlink(vi, 1); + ntfs_set_vfs_operations(vi, mode, dev); + + /* Done! */ + kfree(fn); + kfree(si); + ntfs_debug("Done.\n"); + return ni; + +err_out: + if (rollback_sd) + ntfs_attr_remove(ni, AT_SECURITY_DESCRIPTOR, AT_UNNAMED, 0); + + if (rollback_data) + ntfs_attr_remove(ni, AT_DATA, AT_UNNAMED, 0); + + if (rollback_reparse) + ntfs_delete_reparse_index(ni); + /* + * Free extent MFT records (should not exist any with current + * ntfs_create implementation, but for any case if something will be + * changed in the future). + */ + while (ni->nr_extents != 0) { + int err2; + + err2 = ntfs_mft_record_free(ni->vol, *(ni->ext.extent_ntfs_inos)); + if (err2) + ntfs_error(sb, + "Failed to free extent MFT record. Leaving inconsistent metadata.\n"); + ntfs_inode_close(*(ni->ext.extent_ntfs_inos)); + } + if (ntfs_mft_record_free(ni->vol, ni)) + ntfs_error(sb, + "Failed to free MFT record. Leaving inconsistent metadata. Run chkdsk.\n"); + unmap_mft_record(ni); + kfree(fn); + kfree(si); + + mutex_unlock(&dir_ni->mrec_lock); + mutex_unlock(&ni->mrec_lock); + + remove_inode_hash(vi); + discard_new_inode(vi); + return ERR_PTR(err); +} + +static int ntfs_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct ntfs_volume *vol = NTFS_SB(dir->i_sb); + struct ntfs_inode *ni; + __le16 *uname; + int uname_len, err; + + if (NVolShutdown(vol)) + return -EIO; + + uname_len = ntfs_nlstoucs(vol, dentry->d_name.name, dentry->d_name.len, + &uname, NTFS_MAX_NAME_LEN); + if (uname_len < 0) { + if (uname_len != -ENAMETOOLONG) + ntfs_error(vol->sb, "Failed to convert name to unicode."); + return uname_len; + } + + err = ntfs_check_bad_windows_name(vol, uname, uname_len); + if (err) { + kmem_cache_free(ntfs_name_cache, uname); + return err; + } + + if (!(vol->vol_flags & VOLUME_IS_DIRTY)) + ntfs_set_volume_flags(vol, VOLUME_IS_DIRTY); + + ni = __ntfs_create(idmap, dir, uname, uname_len, S_IFREG | mode, 0, NULL, 0); + kmem_cache_free(ntfs_name_cache, uname); + if (IS_ERR(ni)) + return PTR_ERR(ni); + + d_instantiate_new(dentry, VFS_I(ni)); + + return 0; +} + +static int ntfs_check_unlinkable_dir(struct ntfs_attr_search_ctx *ctx, struct file_name_attr *fn) +{ + int link_count; + int ret; + struct ntfs_inode *ni = ctx->base_ntfs_ino ? ctx->base_ntfs_ino : ctx->ntfs_ino; + struct mft_record *ni_mrec = ctx->base_mrec ? ctx->base_mrec : ctx->mrec; + + ret = ntfs_check_empty_dir(ni, ni_mrec); + if (!ret || ret != -ENOTEMPTY) + return ret; + + link_count = le16_to_cpu(ni_mrec->link_count); + /* + * Directory is non-empty, so we can unlink only if there is more than + * one "real" hard link, i.e. links aren't different DOS and WIN32 names + */ + if ((link_count == 1) || + (link_count == 2 && fn->file_name_type == FILE_NAME_DOS)) { + ret = -ENOTEMPTY; + ntfs_debug("Non-empty directory without hard links\n"); + goto no_hardlink; + } + + ret = 0; +no_hardlink: + return ret; +} + +static int ntfs_test_inode_attr(struct inode *vi, void *data) +{ + struct ntfs_inode *ni = NTFS_I(vi); + u64 mft_no = (u64)(uintptr_t)data; + + if (ni->mft_no != mft_no) + return 0; + if (NInoAttr(ni) || ni->nr_extents == -1) + return 1; + else + return 0; +} + +/* + * ntfs_delete - delete file or directory from ntfs volume + * @ni: ntfs inode for object to delte + * @dir_ni: ntfs inode for directory in which delete object + * @name: unicode name of the object to delete + * @name_len: length of the name in unicode characters + * @need_lock: whether mrec lock is needed or not + * + * Delete the specified name from the directory index @dir_ni and decrement + * the link count of the target inode @ni. + * + * Return 0 on success and -errno on error. + */ +static int ntfs_delete(struct ntfs_inode *ni, struct ntfs_inode *dir_ni, + __le16 *name, u8 name_len, bool need_lock) +{ + struct ntfs_attr_search_ctx *actx = NULL; + struct file_name_attr *fn = NULL; + bool looking_for_dos_name = false, looking_for_win32_name = false; + bool case_sensitive_match = true; + int err = 0; + struct mft_record *ni_mrec; + struct super_block *sb; + bool link_count_zero = false; + + ntfs_debug("Entering.\n"); + + if (need_lock == true) { + mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL); + mutex_lock_nested(&dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT); + } + + sb = dir_ni->vol->sb; + + if (ni->nr_extents == -1) + ni = ni->ext.base_ntfs_ino; + if (dir_ni->nr_extents == -1) + dir_ni = dir_ni->ext.base_ntfs_ino; + /* + * Search for FILE_NAME attribute with such name. If it's in POSIX or + * WIN32_AND_DOS namespace, then simply remove it from index and inode. + * If filename in DOS or in WIN32 namespace, then remove DOS name first, + * only then remove WIN32 name. + */ + actx = ntfs_attr_get_search_ctx(ni, NULL); + if (!actx) { + ntfs_error(sb, "%s, Failed to get search context", __func__); + if (need_lock) { + mutex_unlock(&dir_ni->mrec_lock); + mutex_unlock(&ni->mrec_lock); + } + return -ENOMEM; + } +search: + while ((err = ntfs_attr_lookup(AT_FILE_NAME, AT_UNNAMED, 0, CASE_SENSITIVE, + 0, NULL, 0, actx)) == 0) { +#ifdef DEBUG + unsigned char *s; +#endif + bool case_sensitive = IGNORE_CASE; + + fn = (struct file_name_attr *)((u8 *)actx->attr + + le16_to_cpu(actx->attr->data.resident.value_offset)); +#ifdef DEBUG + s = ntfs_attr_name_get(ni->vol, fn->file_name, fn->file_name_length); + ntfs_debug("name: '%s' type: %d dos: %d win32: %d case: %d\n", + s, fn->file_name_type, + looking_for_dos_name, looking_for_win32_name, + case_sensitive_match); + ntfs_attr_name_free(&s); +#endif + if (looking_for_dos_name) { + if (fn->file_name_type == FILE_NAME_DOS) + break; + continue; + } + if (looking_for_win32_name) { + if (fn->file_name_type == FILE_NAME_WIN32) + break; + continue; + } + + /* Ignore hard links from other directories */ + if (dir_ni->mft_no != MREF_LE(fn->parent_directory)) { + ntfs_debug("MFT record numbers don't match (%llu != %lu)\n", + dir_ni->mft_no, + MREF_LE(fn->parent_directory)); + continue; + } + + if (fn->file_name_type == FILE_NAME_POSIX || case_sensitive_match) + case_sensitive = CASE_SENSITIVE; + + if (ntfs_names_are_equal(fn->file_name, fn->file_name_length, + name, name_len, case_sensitive, + ni->vol->upcase, ni->vol->upcase_len)) { + if (fn->file_name_type == FILE_NAME_WIN32) { + looking_for_dos_name = true; + ntfs_attr_reinit_search_ctx(actx); + continue; + } + if (fn->file_name_type == FILE_NAME_DOS) + looking_for_dos_name = true; + break; + } + } + if (err) { + /* + * If case sensitive search failed, then try once again + * ignoring case. + */ + if (err == -ENOENT && case_sensitive_match) { + case_sensitive_match = false; + ntfs_attr_reinit_search_ctx(actx); + goto search; + } + goto err_out; + } + + err = ntfs_check_unlinkable_dir(actx, fn); + if (err) + goto err_out; + + err = ntfs_index_remove(dir_ni, fn, le32_to_cpu(actx->attr->data.resident.value_length)); + if (err) + goto err_out; + + err = ntfs_attr_record_rm(actx); + if (err) + goto err_out; + + ni_mrec = actx->base_mrec ? actx->base_mrec : actx->mrec; + ni_mrec->link_count = cpu_to_le16(le16_to_cpu(ni_mrec->link_count) - 1); + if (!S_ISDIR(VFS_I(ni)->i_mode)) + drop_nlink(VFS_I(ni)); + + mark_mft_record_dirty(ni); + if (looking_for_dos_name) { + looking_for_dos_name = false; + looking_for_win32_name = true; + ntfs_attr_reinit_search_ctx(actx); + goto search; + } + + /* + * For directories, Drop VFS nlink only when mft record link count + * becomes zero. Because we fixes VFS nlink to 1 for directories. + */ + if (S_ISDIR(VFS_I(ni)->i_mode) && !le16_to_cpu(ni_mrec->link_count)) + drop_nlink(VFS_I(ni)); + + /* + * If hard link count is not equal to zero then we are done. In other + * case there are no reference to this inode left, so we should free all + * non-resident attributes and mark all MFT record as not in use. + */ + if (ni_mrec->link_count == 0) { + NInoSetBeingDeleted(ni); + ntfs_delete_reparse_index(ni); + ntfs_delete_object_id_index(ni); + link_count_zero = true; + } + + ntfs_attr_put_search_ctx(actx); + if (need_lock == true) { + mutex_unlock(&dir_ni->mrec_lock); + mutex_unlock(&ni->mrec_lock); + } + + /* + * If hard link count is not equal to zero then we are done. In other + * case there are no reference to this inode left, so we should free all + * non-resident attributes and mark all MFT record as not in use. + */ + if (link_count_zero == true) { + struct inode *attr_vi; + + while ((attr_vi = ilookup5(sb, ni->mft_no, ntfs_test_inode_attr, + (void *)(uintptr_t)ni->mft_no)) != NULL) { + clear_nlink(attr_vi); + iput(attr_vi); + } + } + ntfs_debug("Done.\n"); + return 0; +err_out: + ntfs_attr_put_search_ctx(actx); + if (need_lock) { + mutex_unlock(&dir_ni->mrec_lock); + mutex_unlock(&ni->mrec_lock); + } + return err; +} + +static int ntfs_unlink(struct inode *dir, struct dentry *dentry) +{ + struct inode *vi = dentry->d_inode; + struct super_block *sb = dir->i_sb; + struct ntfs_volume *vol = NTFS_SB(sb); + int err = 0; + struct ntfs_inode *ni = NTFS_I(vi); + __le16 *uname = NULL; + int uname_len; + + if (NVolShutdown(vol)) + return -EIO; + + uname_len = ntfs_nlstoucs(vol, dentry->d_name.name, dentry->d_name.len, + &uname, NTFS_MAX_NAME_LEN); + if (uname_len < 0) { + if (uname_len != -ENAMETOOLONG) + ntfs_error(sb, "Failed to convert name to Unicode."); + return -ENOMEM; + } + + err = ntfs_check_bad_windows_name(vol, uname, uname_len); + if (err) { + kmem_cache_free(ntfs_name_cache, uname); + return err; + } + + if (!(vol->vol_flags & VOLUME_IS_DIRTY)) + ntfs_set_volume_flags(vol, VOLUME_IS_DIRTY); + + err = ntfs_delete(ni, NTFS_I(dir), uname, uname_len, true); + if (err) + goto out; + + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + mark_inode_dirty(dir); + inode_set_ctime_to_ts(vi, inode_get_ctime(dir)); + if (vi->i_nlink) + mark_inode_dirty(vi); +out: + kmem_cache_free(ntfs_name_cache, uname); + return err; +} + +static struct dentry *ntfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct super_block *sb = dir->i_sb; + struct ntfs_volume *vol = NTFS_SB(sb); + int err = 0; + struct ntfs_inode *ni; + __le16 *uname; + int uname_len; + + if (NVolShutdown(vol)) + return ERR_PTR(-EIO); + + uname_len = ntfs_nlstoucs(vol, dentry->d_name.name, dentry->d_name.len, + &uname, NTFS_MAX_NAME_LEN); + if (uname_len < 0) { + if (uname_len != -ENAMETOOLONG) + ntfs_error(sb, "Failed to convert name to unicode."); + return ERR_PTR(-ENOMEM); + } + + err = ntfs_check_bad_windows_name(vol, uname, uname_len); + if (err) { + kmem_cache_free(ntfs_name_cache, uname); + return ERR_PTR(err); + } + + if (!(vol->vol_flags & VOLUME_IS_DIRTY)) + ntfs_set_volume_flags(vol, VOLUME_IS_DIRTY); + + ni = __ntfs_create(idmap, dir, uname, uname_len, mode, 0, NULL, 0); + kmem_cache_free(ntfs_name_cache, uname); + if (IS_ERR(ni)) { + err = PTR_ERR(ni); + return ERR_PTR(err); + } + + d_instantiate_new(dentry, VFS_I(ni)); + return NULL; +} + +static int ntfs_rmdir(struct inode *dir, struct dentry *dentry) +{ + struct inode *vi = dentry->d_inode; + struct super_block *sb = dir->i_sb; + struct ntfs_volume *vol = NTFS_SB(sb); + int err = 0; + struct ntfs_inode *ni; + __le16 *uname = NULL; + int uname_len; + + if (NVolShutdown(vol)) + return -EIO; + + ni = NTFS_I(vi); + uname_len = ntfs_nlstoucs(vol, dentry->d_name.name, dentry->d_name.len, + &uname, NTFS_MAX_NAME_LEN); + if (uname_len < 0) { + if (uname_len != -ENAMETOOLONG) + ntfs_error(sb, "Failed to convert name to unicode."); + return -ENOMEM; + } + + err = ntfs_check_bad_windows_name(vol, uname, uname_len); + if (err) { + kmem_cache_free(ntfs_name_cache, uname); + return err; + } + + if (!(vol->vol_flags & VOLUME_IS_DIRTY)) + ntfs_set_volume_flags(vol, VOLUME_IS_DIRTY); + + err = ntfs_delete(ni, NTFS_I(dir), uname, uname_len, true); + if (err) + goto out; + + inode_set_mtime_to_ts(vi, inode_set_atime_to_ts(vi, current_time(vi))); +out: + kmem_cache_free(ntfs_name_cache, uname); + return err; +} + +/* + * __ntfs_link - create hard link for file or directory + * @ni: ntfs inode for object to create hard link + * @dir_ni: ntfs inode for directory in which new link should be placed + * @name: unicode name of the new link + * @name_len: length of the name in unicode characters + * + * Create a new hard link. This involves adding an entry to the directory + * index and adding a new FILE_NAME attribute to the target inode. + * + * Return 0 on success and -errno on error. + */ +static int __ntfs_link(struct ntfs_inode *ni, struct ntfs_inode *dir_ni, + __le16 *name, u8 name_len) +{ + struct super_block *sb; + struct inode *vi = VFS_I(ni); + struct file_name_attr *fn = NULL; + int fn_len, err = 0; + struct mft_record *dir_mrec = NULL, *ni_mrec = NULL; + + ntfs_debug("Entering.\n"); + + sb = dir_ni->vol->sb; + if (NInoBeingDeleted(dir_ni) || NInoBeingDeleted(ni)) + return -ENOENT; + + ni_mrec = map_mft_record(ni); + if (IS_ERR(ni_mrec)) { + err = -EIO; + goto err_out; + } + + if (le16_to_cpu(ni_mrec->link_count) == 0) { + err = -ENOENT; + goto err_out; + } + + /* Create FILE_NAME attribute. */ + fn_len = sizeof(struct file_name_attr) + name_len * sizeof(__le16); + + fn = kzalloc(fn_len, GFP_NOFS); + if (!fn) { + err = -ENOMEM; + goto err_out; + } + + dir_mrec = map_mft_record(dir_ni); + if (IS_ERR(dir_mrec)) { + err = -EIO; + goto err_out; + } + + fn->parent_directory = MK_LE_MREF(dir_ni->mft_no, + le16_to_cpu(dir_mrec->sequence_number)); + unmap_mft_record(dir_ni); + fn->file_name_length = name_len; + fn->file_name_type = FILE_NAME_POSIX; + fn->file_attributes = ni->flags; + if (ni_mrec->flags & MFT_RECORD_IS_DIRECTORY) { + fn->file_attributes |= FILE_ATTR_DUP_FILE_NAME_INDEX_PRESENT; + fn->allocated_size = fn->data_size = 0; + } else { + if (NInoSparse(ni) || NInoCompressed(ni)) + fn->allocated_size = + cpu_to_le64(ni->itype.compressed.size); + else + fn->allocated_size = cpu_to_le64(ni->allocated_size); + fn->data_size = cpu_to_le64(ni->data_size); + } + if (NVolHideDotFiles(dir_ni->vol) && name_len > 0 && name[0] == cpu_to_le16('.')) + fn->file_attributes |= FILE_ATTR_HIDDEN; + + fn->creation_time = utc2ntfs(ni->i_crtime); + fn->last_data_change_time = utc2ntfs(inode_get_mtime(vi)); + fn->last_mft_change_time = utc2ntfs(inode_get_ctime(vi)); + fn->last_access_time = utc2ntfs(inode_get_atime(vi)); + memcpy(fn->file_name, name, name_len * sizeof(__le16)); + + /* Add FILE_NAME attribute to index. */ + err = ntfs_index_add_filename(dir_ni, fn, MK_MREF(ni->mft_no, + le16_to_cpu(ni_mrec->sequence_number))); + if (err) { + ntfs_error(sb, "Failed to add filename to the index"); + goto err_out; + } + /* Add FILE_NAME attribute to inode. */ + err = ntfs_attr_add(ni, AT_FILE_NAME, AT_UNNAMED, 0, (u8 *)fn, fn_len); + if (err) { + ntfs_error(sb, "Failed to add FILE_NAME attribute.\n"); + /* Try to remove just added attribute from index. */ + if (ntfs_index_remove(dir_ni, fn, fn_len)) + goto rollback_failed; + goto err_out; + } + /* Increment hard links count. */ + ni_mrec->link_count = cpu_to_le16(le16_to_cpu(ni_mrec->link_count) + 1); + if (!S_ISDIR(vi->i_mode)) + inc_nlink(VFS_I(ni)); + + /* Done! */ + mark_mft_record_dirty(ni); + kfree(fn); + unmap_mft_record(ni); + + ntfs_debug("Done.\n"); + + return 0; +rollback_failed: + ntfs_error(sb, "Rollback failed. Leaving inconsistent metadata.\n"); +err_out: + kfree(fn); + if (!IS_ERR_OR_NULL(ni_mrec)) + unmap_mft_record(ni); + return err; +} + +static int ntfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) +{ + struct inode *old_inode, *new_inode = NULL; + int err = 0; + int is_dir; + struct super_block *sb = old_dir->i_sb; + __le16 *uname_new = NULL; + __le16 *uname_old = NULL; + int new_name_len; + int old_name_len; + struct ntfs_volume *vol = NTFS_SB(sb); + struct ntfs_inode *old_ni, *new_ni = NULL; + struct ntfs_inode *old_dir_ni = NTFS_I(old_dir), *new_dir_ni = NTFS_I(new_dir); + bool new_dir_first = false; + + if (NVolShutdown(old_dir_ni->vol)) + return -EIO; + + if (flags & (RENAME_EXCHANGE | RENAME_WHITEOUT)) + return -EINVAL; + + new_name_len = ntfs_nlstoucs(NTFS_I(new_dir)->vol, new_dentry->d_name.name, + new_dentry->d_name.len, &uname_new, + NTFS_MAX_NAME_LEN); + if (new_name_len < 0) { + if (new_name_len != -ENAMETOOLONG) + ntfs_error(sb, "Failed to convert name to unicode."); + return -ENOMEM; + } + + err = ntfs_check_bad_windows_name(vol, uname_new, new_name_len); + if (err) { + kmem_cache_free(ntfs_name_cache, uname_new); + return err; + } + + old_name_len = ntfs_nlstoucs(NTFS_I(old_dir)->vol, old_dentry->d_name.name, + old_dentry->d_name.len, &uname_old, + NTFS_MAX_NAME_LEN); + if (old_name_len < 0) { + kmem_cache_free(ntfs_name_cache, uname_new); + if (old_name_len != -ENAMETOOLONG) + ntfs_error(sb, "Failed to convert name to unicode."); + return -ENOMEM; + } + + old_inode = old_dentry->d_inode; + new_inode = new_dentry->d_inode; + old_ni = NTFS_I(old_inode); + if (new_inode) + new_ni = NTFS_I(new_inode); + if (old_dir != new_dir) + new_dir_first = is_subdir(new_dentry->d_parent, + old_dentry->d_parent); + + if (!(vol->vol_flags & VOLUME_IS_DIRTY)) + ntfs_set_volume_flags(vol, VOLUME_IS_DIRTY); + + mutex_lock_nested(&old_ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL); + if (new_ni) + mutex_lock_nested(&new_ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL_2); + + if (old_dir == new_dir) { + mutex_lock_nested(&old_dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT); + } else if (new_dir_first) { + mutex_lock_nested(&new_dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT); + mutex_lock_nested(&old_dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT_2); + } else { + mutex_lock_nested(&old_dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT); + mutex_lock_nested(&new_dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT_2); + } + + if (NInoBeingDeleted(old_ni) || NInoBeingDeleted(old_dir_ni) || + (new_ni && NInoBeingDeleted(new_ni)) || + (old_dir != new_dir && NInoBeingDeleted(new_dir_ni))) { + err = -ENOENT; + goto err_out; + } + + is_dir = S_ISDIR(old_inode->i_mode); + + if (new_inode) { + if (is_dir) { + struct mft_record *ni_mrec; + + ni_mrec = map_mft_record(NTFS_I(new_inode)); + if (IS_ERR(ni_mrec)) { + err = -EIO; + goto err_out; + } + err = ntfs_check_empty_dir(NTFS_I(new_inode), ni_mrec); + unmap_mft_record(NTFS_I(new_inode)); + if (err) + goto err_out; + } + + err = ntfs_delete(new_ni, new_dir_ni, uname_new, new_name_len, false); + if (err) + goto err_out; + } + + err = __ntfs_link(old_ni, new_dir_ni, uname_new, new_name_len); + if (err) + goto err_out; + + err = ntfs_delete(old_ni, old_dir_ni, uname_old, old_name_len, false); + if (err) { + int err2; + + ntfs_error(sb, "Failed to delete old ntfs inode(%llu) in old dir, err : %d\n", + old_ni->mft_no, err); + err2 = ntfs_delete(old_ni, new_dir_ni, uname_new, new_name_len, false); + if (err2) + ntfs_error(sb, "Failed to delete old ntfs inode in new dir, err : %d\n", + err2); + goto err_out; + } + + simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry); + mark_inode_dirty(old_inode); + mark_inode_dirty(old_dir); + if (old_dir != new_dir) + mark_inode_dirty(new_dir); + if (new_inode) + mark_inode_dirty(old_inode); + + inode_inc_iversion(new_dir); + +err_out: + if (old_dir == new_dir) { + mutex_unlock(&old_dir_ni->mrec_lock); + } else if (new_dir_first) { + mutex_unlock(&old_dir_ni->mrec_lock); + mutex_unlock(&new_dir_ni->mrec_lock); + } else { + mutex_unlock(&new_dir_ni->mrec_lock); + mutex_unlock(&old_dir_ni->mrec_lock); + } + if (new_ni) + mutex_unlock(&new_ni->mrec_lock); + mutex_unlock(&old_ni->mrec_lock); + if (uname_new) + kmem_cache_free(ntfs_name_cache, uname_new); + if (uname_old) + kmem_cache_free(ntfs_name_cache, uname_old); + + return err; +} + +static int ntfs_symlink(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const char *symname) +{ + struct super_block *sb = dir->i_sb; + struct ntfs_volume *vol = NTFS_SB(sb); + struct inode *vi; + int err = 0; + struct ntfs_inode *ni; + __le16 *usrc; + int usrc_len; + int symlen = strlen(symname); + + if (NVolShutdown(vol)) + return -EIO; + + usrc_len = ntfs_nlstoucs(vol, dentry->d_name.name, + dentry->d_name.len, &usrc, NTFS_MAX_NAME_LEN); + if (usrc_len < 0) { + if (usrc_len != -ENAMETOOLONG) + ntfs_error(sb, "Failed to convert name to Unicode."); + err = -ENOMEM; + goto out; + } + + err = ntfs_check_bad_windows_name(vol, usrc, usrc_len); + if (err) { + kmem_cache_free(ntfs_name_cache, usrc); + goto out; + } + + if (!(vol->vol_flags & VOLUME_IS_DIRTY)) + ntfs_set_volume_flags(vol, VOLUME_IS_DIRTY); + + ni = __ntfs_create(idmap, dir, usrc, usrc_len, S_IFLNK | 0777, 0, + symname, symlen); + kmem_cache_free(ntfs_name_cache, usrc); + if (IS_ERR(ni)) { + err = PTR_ERR(ni); + goto out; + } + + vi = VFS_I(ni); + vi->i_size = symlen; + d_instantiate_new(dentry, vi); +out: + return err; +} + +static int ntfs_mknod(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) +{ + struct super_block *sb = dir->i_sb; + struct ntfs_volume *vol = NTFS_SB(sb); + int err = 0; + struct ntfs_inode *ni; + __le16 *uname = NULL; + int uname_len; + + if (NVolShutdown(vol)) + return -EIO; + + uname_len = ntfs_nlstoucs(vol, dentry->d_name.name, + dentry->d_name.len, &uname, NTFS_MAX_NAME_LEN); + if (uname_len < 0) { + if (uname_len != -ENAMETOOLONG) + ntfs_error(sb, "Failed to convert name to Unicode."); + return -ENOMEM; + } + + err = ntfs_check_bad_windows_name(vol, uname, uname_len); + if (err) { + kmem_cache_free(ntfs_name_cache, uname); + return err; + } + + if (!(vol->vol_flags & VOLUME_IS_DIRTY)) + ntfs_set_volume_flags(vol, VOLUME_IS_DIRTY); + + switch (mode & S_IFMT) { + case S_IFCHR: + case S_IFBLK: + ni = __ntfs_create(idmap, dir, uname, uname_len, + mode, rdev, NULL, 0); + break; + default: + ni = __ntfs_create(idmap, dir, uname, uname_len, + mode, 0, NULL, 0); + } + + kmem_cache_free(ntfs_name_cache, uname); + if (IS_ERR(ni)) { + err = PTR_ERR(ni); + goto out; + } + + d_instantiate_new(dentry, VFS_I(ni)); +out: + return err; +} + +static int ntfs_link(struct dentry *old_dentry, struct inode *dir, + struct dentry *dentry) +{ + struct inode *vi = old_dentry->d_inode; + struct super_block *sb = vi->i_sb; + struct ntfs_volume *vol = NTFS_SB(sb); + __le16 *uname = NULL; + int uname_len; + int err; + struct ntfs_inode *ni = NTFS_I(vi), *dir_ni = NTFS_I(dir); + + if (NVolShutdown(vol)) + return -EIO; + + uname_len = ntfs_nlstoucs(vol, dentry->d_name.name, + dentry->d_name.len, &uname, NTFS_MAX_NAME_LEN); + if (uname_len < 0) { + if (uname_len != -ENAMETOOLONG) + ntfs_error(sb, "Failed to convert name to unicode."); + return -ENOMEM; + } + + if (!(vol->vol_flags & VOLUME_IS_DIRTY)) + ntfs_set_volume_flags(vol, VOLUME_IS_DIRTY); + + ihold(vi); + mutex_lock_nested(&ni->mrec_lock, NTFS_INODE_MUTEX_NORMAL); + mutex_lock_nested(&dir_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT); + err = __ntfs_link(NTFS_I(vi), NTFS_I(dir), uname, uname_len); + if (err) { + mutex_unlock(&dir_ni->mrec_lock); + mutex_unlock(&ni->mrec_lock); + iput(vi); + pr_err("failed to create link, err = %d\n", err); + goto out; + } + + inode_inc_iversion(dir); + simple_inode_init_ts(dir); + + inode_inc_iversion(vi); + simple_inode_init_ts(vi); + + /* timestamp is already written, so mark_inode_dirty() is unneeded. */ + d_instantiate(dentry, vi); + mutex_unlock(&dir_ni->mrec_lock); + mutex_unlock(&ni->mrec_lock); + +out: + kmem_cache_free(ntfs_name_cache, uname); + return err; +} + +/* + * Inode operations for directories. + */ +const struct inode_operations ntfs_dir_inode_ops = { + .lookup = ntfs_lookup, /* VFS: Lookup directory. */ + .create = ntfs_create, + .unlink = ntfs_unlink, + .mkdir = ntfs_mkdir, + .rmdir = ntfs_rmdir, + .rename = ntfs_rename, + .get_acl = ntfs_get_acl, + .set_acl = ntfs_set_acl, + .listxattr = ntfs_listxattr, + .setattr = ntfs_setattr, + .getattr = ntfs_getattr, + .symlink = ntfs_symlink, + .mknod = ntfs_mknod, + .link = ntfs_link, +}; + +/* + * ntfs_get_parent - find the dentry of the parent of a given directory dentry + * @child_dent: dentry of the directory whose parent directory to find + * + * Find the dentry for the parent directory of the directory specified by the + * dentry @child_dent. This function is called from + * fs/exportfs/expfs.c::find_exported_dentry() which in turn is called from the + * default ->decode_fh() which is export_decode_fh() in the same file. + * + * Note: ntfs_get_parent() is called with @d_inode(child_dent)->i_mutex down. + * + * Return the dentry of the parent directory on success or the error code on + * error (IS_ERR() is true). + */ +static struct dentry *ntfs_get_parent(struct dentry *child_dent) +{ + struct inode *vi = d_inode(child_dent); + struct ntfs_inode *ni = NTFS_I(vi); + struct mft_record *mrec; + struct ntfs_attr_search_ctx *ctx; + struct attr_record *attr; + struct file_name_attr *fn; + unsigned long parent_ino; + int err; + + ntfs_debug("Entering for inode 0x%llx.", ni->mft_no); + /* Get the mft record of the inode belonging to the child dentry. */ + mrec = map_mft_record(ni); + if (IS_ERR(mrec)) + return ERR_CAST(mrec); + /* Find the first file name attribute in the mft record. */ + ctx = ntfs_attr_get_search_ctx(ni, mrec); + if (unlikely(!ctx)) { + unmap_mft_record(ni); + return ERR_PTR(-ENOMEM); + } +try_next: + err = ntfs_attr_lookup(AT_FILE_NAME, NULL, 0, CASE_SENSITIVE, 0, NULL, + 0, ctx); + if (unlikely(err)) { + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(ni); + if (err == -ENOENT) + ntfs_error(vi->i_sb, + "Inode 0x%llx does not have a file name attribute. Run chkdsk.", + ni->mft_no); + return ERR_PTR(err); + } + attr = ctx->attr; + if (unlikely(attr->non_resident)) + goto try_next; + fn = (struct file_name_attr *)((u8 *)attr + + le16_to_cpu(attr->data.resident.value_offset)); + if (unlikely((u8 *)fn + le32_to_cpu(attr->data.resident.value_length) > + (u8 *)attr + le32_to_cpu(attr->length))) + goto try_next; + /* Get the inode number of the parent directory. */ + parent_ino = MREF_LE(fn->parent_directory); + /* Release the search context and the mft record of the child. */ + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(ni); + + return d_obtain_alias(ntfs_iget(vi->i_sb, parent_ino)); +} + +static struct inode *ntfs_nfs_get_inode(struct super_block *sb, + u64 ino, u32 generation) +{ + struct inode *inode; + + inode = ntfs_iget(sb, ino); + if (!IS_ERR(inode)) { + if (inode->i_generation != generation) { + iput(inode); + inode = ERR_PTR(-ESTALE); + } + } + + return inode; +} + +static struct dentry *ntfs_fh_to_dentry(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + return generic_fh_to_dentry(sb, fid, fh_len, fh_type, + ntfs_nfs_get_inode); +} + +static struct dentry *ntfs_fh_to_parent(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + return generic_fh_to_parent(sb, fid, fh_len, fh_type, + ntfs_nfs_get_inode); +} + +/* + * Export operations allowing NFS exporting of mounted NTFS partitions. + */ +const struct export_operations ntfs_export_ops = { + .encode_fh = generic_encode_ino32_fh, + .get_parent = ntfs_get_parent, /* Find the parent of a given directory. */ + .fh_to_dentry = ntfs_fh_to_dentry, + .fh_to_parent = ntfs_fh_to_parent, +}; diff --git a/fs/ntfs/ntfs.h b/fs/ntfs/ntfs.h new file mode 100644 index 000000000..45064dbcc --- /dev/null +++ b/fs/ntfs/ntfs.h @@ -0,0 +1,294 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Defines for NTFS Linux kernel driver. + * + * Copyright (c) 2001-2014 Anton Altaparmakov and Tuxera Inc. + * Copyright (C) 2002 Richard Russon + * Copyright (c) 2025 LG Electronics Co., Ltd. + */ + +#ifndef _LINUX_NTFS_H +#define _LINUX_NTFS_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "volume.h" +#include "layout.h" +#include "inode.h" + +#ifdef pr_fmt +#undef pr_fmt +#endif + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +/* + * Default pre-allocation size is optimize runlist merge overhead + * with small chunk size. + */ +#define NTFS_DEF_PREALLOC_SIZE 65536 + +/* + * The log2 of the standard number of clusters per compression block. + * A value of 4 corresponds to 16 clusters (1 << 4), which is the + * default chunk size used by NTFS LZNT1 compression. + */ +#define STANDARD_COMPRESSION_UNIT 4 + +/* + * The maximum cluster size (4KB) allowed for compression to be enabled. + * By design, NTFS does not support compression on volumes where the + * cluster size exceeds 4096 bytes. + */ +#define MAX_COMPRESSION_CLUSTER_SIZE 4096 + +#define NTFS_B_TO_CLU(vol, b) ((b) >> (vol)->cluster_size_bits) +#define NTFS_CLU_TO_B(vol, clu) ((u64)(clu) << (vol)->cluster_size_bits) +#define NTFS_B_TO_CLU_OFS(vol, clu) ((u64)(clu) & (vol)->cluster_size_mask) + +#define NTFS_MFT_NR_TO_CLU(vol, mft_no) (((u64)mft_no << (vol)->mft_record_size_bits) >> \ + (vol)->cluster_size_bits) +#define NTFS_MFT_NR_TO_PIDX(vol, mft_no) (mft_no >> (PAGE_SHIFT - \ + (vol)->mft_record_size_bits)) +#define NTFS_MFT_NR_TO_POFS(vol, mft_no) (((u64)mft_no << (vol)->mft_record_size_bits) & \ + ~PAGE_MASK) + +#define NTFS_PIDX_TO_BLK(vol, idx) (((u64)idx << PAGE_SHIFT) >> \ + ((vol)->sb)->s_blocksize_bits) +#define NTFS_PIDX_TO_CLU(vol, idx) (((u64)idx << PAGE_SHIFT) >> \ + (vol)->cluster_size_bits) +#define NTFS_CLU_TO_PIDX(vol, clu) (((u64)(clu) << (vol)->cluster_size_bits) >> \ + PAGE_SHIFT) +#define NTFS_CLU_TO_POFS(vol, clu) (((u64)(clu) << (vol)->cluster_size_bits) & \ + ~PAGE_MASK) + +#define NTFS_B_TO_SECTOR(vol, b) ((b) >> ((vol)->sb)->s_blocksize_bits) + +enum { + NTFS_BLOCK_SIZE = 512, + NTFS_BLOCK_SIZE_BITS = 9, + NTFS_SB_MAGIC = 0x5346544e, /* 'NTFS' */ + NTFS_MAX_NAME_LEN = 255, + NTFS_MAX_LABEL_LEN = 128, +}; + +enum { + CASE_SENSITIVE = 0, + IGNORE_CASE = 1, +}; + +/* + * Conversion helpers for NTFS units. + */ + +/* Convert bytes to cluster count */ +static inline u64 ntfs_bytes_to_cluster(const struct ntfs_volume *vol, + s64 bytes) +{ + return bytes >> vol->cluster_size_bits; +} + +/* Convert cluster count to bytes */ +static inline u64 ntfs_cluster_to_bytes(const struct ntfs_volume *vol, + u64 clusters) +{ + return clusters << vol->cluster_size_bits; +} + +/* Get the byte offset within a cluster from a linear byte address */ +static inline u64 ntfs_bytes_to_cluster_off(const struct ntfs_volume *vol, + u64 bytes) +{ + return bytes & vol->cluster_size_mask; +} + +/* Calculate the physical cluster number containing a specific MFT record. */ +static inline u64 ntfs_mft_no_to_cluster(const struct ntfs_volume *vol, + unsigned long mft_no) +{ + return ((u64)mft_no << vol->mft_record_size_bits) >> + vol->cluster_size_bits; +} + +/* Calculate the folio index where the MFT record resides. */ +static inline pgoff_t ntfs_mft_no_to_pidx(const struct ntfs_volume *vol, + unsigned long mft_no) +{ + return mft_no >> (PAGE_SHIFT - vol->mft_record_size_bits); +} + +/* Calculate the byte offset within a folio for an MFT record. */ +static inline u64 ntfs_mft_no_to_poff(const struct ntfs_volume *vol, + unsigned long mft_no) +{ + return ((u64)mft_no << vol->mft_record_size_bits) & ~PAGE_MASK; +} + +/* Convert folio index to cluster number. */ +static inline u64 ntfs_pidx_to_cluster(const struct ntfs_volume *vol, + pgoff_t idx) +{ + return ((u64)idx << PAGE_SHIFT) >> vol->cluster_size_bits; +} + +/* Convert cluster number to folio index. */ +static inline pgoff_t ntfs_cluster_to_pidx(const struct ntfs_volume *vol, + u64 clu) +{ + return (clu << vol->cluster_size_bits) >> PAGE_SHIFT; +} + +/* Get the byte offset within a folio from a cluster number */ +static inline u64 ntfs_cluster_to_poff(const struct ntfs_volume *vol, + u64 clu) +{ + return (clu << vol->cluster_size_bits) & ~PAGE_MASK; +} + +/* Convert byte offset to sector (block) number. */ +static inline sector_t ntfs_bytes_to_sector(const struct ntfs_volume *vol, + u64 bytes) +{ + return bytes >> vol->sb->s_blocksize_bits; +} + +/* Global variables. */ + +/* Slab caches (from super.c). */ +extern struct kmem_cache *ntfs_name_cache; +extern struct kmem_cache *ntfs_inode_cache; +extern struct kmem_cache *ntfs_big_inode_cache; +extern struct kmem_cache *ntfs_attr_ctx_cache; +extern struct kmem_cache *ntfs_index_ctx_cache; + +/* The various operations structs defined throughout the driver files. */ +extern const struct address_space_operations ntfs_aops; +extern const struct address_space_operations ntfs_mft_aops; + +extern const struct file_operations ntfs_file_ops; +extern const struct inode_operations ntfs_file_inode_ops; +extern const struct inode_operations ntfs_symlink_inode_operations; +extern const struct inode_operations ntfs_special_inode_operations; + +extern const struct file_operations ntfs_dir_ops; +extern const struct inode_operations ntfs_dir_inode_ops; + +extern const struct file_operations ntfs_empty_file_ops; +extern const struct inode_operations ntfs_empty_inode_ops; + +extern const struct export_operations ntfs_export_ops; + +/* + * NTFS_SB - return the ntfs volume given a vfs super block + * @sb: VFS super block + * + * NTFS_SB() returns the ntfs volume associated with the VFS super block @sb. + */ +static inline struct ntfs_volume *NTFS_SB(struct super_block *sb) +{ + return sb->s_fs_info; +} + +/* Declarations of functions and global variables. */ + +/* From fs/ntfs/compress.c */ +int ntfs_read_compressed_block(struct folio *folio); +int allocate_compression_buffers(void); +void free_compression_buffers(void); +int ntfs_compress_write(struct ntfs_inode *ni, loff_t pos, size_t count, + struct iov_iter *from); + +/* From fs/ntfs/super.c */ +#define default_upcase_len 0x10000 +extern struct mutex ntfs_lock; + +struct option_t { + int val; + char *str; +}; +extern const struct option_t on_errors_arr[]; +int ntfs_set_volume_flags(struct ntfs_volume *vol, __le16 flags); +int ntfs_clear_volume_flags(struct ntfs_volume *vol, __le16 flags); +int ntfs_write_volume_label(struct ntfs_volume *vol, char *label); + +/* From fs/ntfs/mst.c */ +int post_read_mst_fixup(struct ntfs_record *b, const u32 size); +int pre_write_mst_fixup(struct ntfs_record *b, const u32 size); +void post_write_mst_fixup(struct ntfs_record *b); + +/* From fs/ntfs/unistr.c */ +bool ntfs_are_names_equal(const __le16 *s1, size_t s1_len, + const __le16 *s2, size_t s2_len, + const u32 ic, + const __le16 *upcase, const u32 upcase_size); +int ntfs_collate_names(const __le16 *name1, const u32 name1_len, + const __le16 *name2, const u32 name2_len, + const int err_val, const u32 ic, + const __le16 *upcase, const u32 upcase_len); +int ntfs_ucsncmp(const __le16 *s1, const __le16 *s2, size_t n); +int ntfs_ucsncasecmp(const __le16 *s1, const __le16 *s2, size_t n, + const __le16 *upcase, const u32 upcase_size); +int ntfs_file_compare_values(const struct file_name_attr *file_name_attr1, + const struct file_name_attr *file_name_attr2, + const int err_val, const u32 ic, + const __le16 *upcase, const u32 upcase_len); +int ntfs_nlstoucs(const struct ntfs_volume *vol, const char *ins, + const int ins_len, __le16 **outs, int max_name_len); +int ntfs_ucstonls(const struct ntfs_volume *vol, const __le16 *ins, + const int ins_len, unsigned char **outs, int outs_len); +__le16 *ntfs_ucsndup(const __le16 *s, u32 maxlen); +bool ntfs_names_are_equal(const __le16 *s1, size_t s1_len, + const __le16 *s2, size_t s2_len, + const u32 ic, + const __le16 *upcase, const u32 upcase_size); +int ntfs_force_shutdown(struct super_block *sb, u32 flags); +long ntfs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); +#ifdef CONFIG_COMPAT +long ntfs_compat_ioctl(struct file *filp, unsigned int cmd, + unsigned long arg); +#endif + +/* From fs/ntfs/upcase.c */ +__le16 *generate_default_upcase(void); + +static inline int ntfs_ffs(int x) +{ + int r = 1; + + if (!x) + return 0; + if (!(x & 0xffff)) { + x >>= 16; + r += 16; + } + if (!(x & 0xff)) { + x >>= 8; + r += 8; + } + if (!(x & 0xf)) { + x >>= 4; + r += 4; + } + if (!(x & 3)) { + x >>= 2; + r += 2; + } + if (!(x & 1)) + r += 1; + return r; +} + +/* From fs/ntfs/bdev-io.c */ +int ntfs_bdev_read(struct block_device *bdev, char *data, loff_t start, size_t size); +int ntfs_bdev_write(struct super_block *sb, void *buf, loff_t start, size_t size); + +#endif /* _LINUX_NTFS_H */ diff --git a/fs/ntfs/object_id.c b/fs/ntfs/object_id.c new file mode 100644 index 000000000..89002c375 --- /dev/null +++ b/fs/ntfs/object_id.c @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Pocessing of object ids + * + * Part of this file is based on code from the NTFS-3G. + * + * Copyright (c) 2009-2019 Jean-Pierre Andre + * Copyright (c) 2026 LG Electronics Co., Ltd. + */ + +#include "ntfs.h" +#include "index.h" +#include "object_id.h" + +struct object_id_index_key { + union { + u32 alignment; + struct guid guid; + } object_id; +} __packed; + +struct object_id_index_data { + __le64 file_id; + struct guid birth_volume_id; + struct guid birth_object_id; + struct guid domain_id; +} __packed; + +/* Index entry in $Extend/$ObjId */ +struct object_id_index { + struct index_entry_header header; + struct object_id_index_key key; + struct object_id_index_data data; +} __packed; + +__le16 objid_index_name[] = {cpu_to_le16('$'), cpu_to_le16('O'), 0}; + +/* + * open_object_id_index - Open the $Extend/$ObjId file and its index + * @vol: NTFS volume structure + * + * Opens the $ObjId system file and retrieves its index context. + * + * Return: The index context if opened successfully, or NULL if an error + * occurred. + */ +static struct ntfs_index_context *open_object_id_index(struct ntfs_volume *vol) +{ + struct inode *dir_vi, *vi; + struct ntfs_inode *dir_ni; + struct ntfs_index_context *xo = NULL; + struct ntfs_name *name = NULL; + u64 mref; + int uname_len; + __le16 *uname; + + uname_len = ntfs_nlstoucs(vol, "$ObjId", 6, &uname, + NTFS_MAX_NAME_LEN); + if (uname_len < 0) + return NULL; + + /* do not use path_name_to inode - could reopen root */ + dir_vi = ntfs_iget(vol->sb, FILE_Extend); + if (IS_ERR(dir_vi)) { + kmem_cache_free(ntfs_name_cache, uname); + return NULL; + } + dir_ni = NTFS_I(dir_vi); + + mutex_lock_nested(&dir_ni->mrec_lock, NTFS_EXTEND_MUTEX_PARENT); + mref = ntfs_lookup_inode_by_name(dir_ni, uname, uname_len, &name); + mutex_unlock(&dir_ni->mrec_lock); + kfree(name); + kmem_cache_free(ntfs_name_cache, uname); + if (IS_ERR_MREF(mref)) + goto put_dir_vi; + + vi = ntfs_iget(vol->sb, MREF(mref)); + if (IS_ERR(vi)) + goto put_dir_vi; + + xo = ntfs_index_ctx_get(NTFS_I(vi), objid_index_name, 2); + if (!xo) + iput(vi); +put_dir_vi: + iput(dir_vi); + return xo; +} + + +/* + * remove_object_id_index - Remove an object id index entry if attribute present + * @ni: NTFS inode structure containing the attribute + * @xo: Index context for the object id index + * + * Reads the existing object ID attribute and removes it from the index. + * + * Return: 0 on success, or a negative error code on failure. + */ +static int remove_object_id_index(struct ntfs_inode *ni, struct ntfs_index_context *xo) +{ + struct object_id_index_key key = {0}; + s64 size; + + if (ni->data_size == 0) + return -ENODATA; + + /* read the existing object id attribute */ + size = ntfs_inode_attr_pread(VFS_I(ni), 0, sizeof(struct guid), + (char *)&key); + if (size != sizeof(struct guid)) + return -ENODATA; + + if (!ntfs_index_lookup(&key, sizeof(struct object_id_index_key), xo)) + return ntfs_index_rm(xo); + + return 0; +} + +/* + * ntfs_delete_object_id_index - Delete an object_id index entry + * @ni: NTFS inode structure + * + * Opens the object ID index and removes the entry corresponding to the inode. + * + * Return: 0 on success, or a negative error code on failure. + */ +int ntfs_delete_object_id_index(struct ntfs_inode *ni) +{ + struct ntfs_index_context *xo; + struct ntfs_inode *xoni; + struct inode *attr_vi; + int ret = 0; + + attr_vi = ntfs_attr_iget(VFS_I(ni), AT_OBJECT_ID, AT_UNNAMED, 0); + if (IS_ERR(attr_vi)) + return PTR_ERR(attr_vi); + + /* + * read the existing object id and un-index it + */ + xo = open_object_id_index(ni->vol); + if (xo) { + xoni = xo->idx_ni; + mutex_lock_nested(&xoni->mrec_lock, NTFS_EXTEND_MUTEX_PARENT); + ret = remove_object_id_index(NTFS_I(attr_vi), xo); + if (!ret) { + ntfs_index_entry_mark_dirty(xo); + mark_mft_record_dirty(xoni); + } + ntfs_index_ctx_put(xo); + mutex_unlock(&xoni->mrec_lock); + iput(VFS_I(xoni)); + } + + iput(attr_vi); + return ret; +} diff --git a/fs/ntfs/object_id.h b/fs/ntfs/object_id.h new file mode 100644 index 000000000..e4b8e529d --- /dev/null +++ b/fs/ntfs/object_id.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2008-2021 Jean-Pierre Andre + * Copyright (c) 2025 LG Electronics Co., Ltd. + */ + +#ifndef _LINUX_NTFS_OBJECT_ID_H +#define _LINUX_NTFS_OBJECT_ID_H + +extern __le16 objid_index_name[]; + +int ntfs_delete_object_id_index(struct ntfs_inode *ni); + +#endif /* _LINUX_NTFS_OBJECT_ID_H */ diff --git a/fs/ntfs/reparse.c b/fs/ntfs/reparse.c new file mode 100644 index 000000000..fa523dc36 --- /dev/null +++ b/fs/ntfs/reparse.c @@ -0,0 +1,964 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Processing of reparse points + * + * Part of this file is based on code from the NTFS-3G. + * + * Copyright (c) 2008-2021 Jean-Pierre Andre + * Copyright (c) 2025 LG Electronics Co., Ltd. + */ + +#include "ntfs.h" +#include "layout.h" +#include "attrib.h" +#include "inode.h" +#include "dir.h" +#include "volume.h" +#include "mft.h" +#include "index.h" +#include "lcnalloc.h" +#include "reparse.h" + +struct wsl_link_reparse_data { + __le32 type; + char link[]; +}; + +static bool reparse_name_is_valid(size_t size, size_t name_off, u16 len) +{ + if ((name_off | len) & 1) + return false; + + return name_off + len <= size; +} + +/* + * Windows-native reparse payloads store pathnames as UTF-16 strings with '\\' + * separators. Convert the on-disk UTF-16 target into the mount's NLS and + * normalize path separators. + */ +static int ntfs_reparse_target_to_nls(struct ntfs_volume *vol, + const __le16 *uname, u16 ulen, + char **target) +{ + int err, i; + + *target = NULL; + ulen >>= 1; + if (!ulen) + return -EINVAL; + + if (!uname[ulen - 1]) + ulen--; + + err = ntfs_ucstonls(vol, uname, ulen, (unsigned char **)target, 0); + if (err < 0) { + ntfs_attr_name_free((unsigned char **)target); + return err; + } + + for (i = 0; i < err; i++) { + if ((*target)[i] == '\\') + (*target)[i] = '/'; + } + + return 0; +} + +/* Index entry in $Extend/$Reparse */ +struct reparse_index { + struct index_entry_header header; + struct reparse_index_key key; + __le32 filling; +}; + +__le16 reparse_index_name[] = {cpu_to_le16('$'), cpu_to_le16('R'), 0}; + + +/* + * Check if the reparse point attribute buffer is valid. + * Returns true if valid, false otherwise. + */ +static bool valid_reparse_buffer(struct ntfs_inode *ni, + const struct reparse_point *reparse_attr, + size_t size, + size_t payload_min_len) +{ + size_t expected; + + if (!ni || !reparse_attr) + return false; + + /* Minimum size must cover reparse_point header */ + if (size < sizeof(struct reparse_point)) + return false; + + /* The payload must contain the fixed fields for the current tag. */ + if (payload_min_len && + le16_to_cpu(reparse_attr->reparse_data_length) < payload_min_len) + return false; + + /* Reserved zero tag is invalid */ + if (reparse_attr->reparse_tag == IO_REPARSE_TAG_RESERVED_ZERO) + return false; + + /* Calculate expected total size */ + expected = sizeof(struct reparse_point) + + le16_to_cpu(reparse_attr->reparse_data_length); + + /* Add GUID size for non-Microsoft tags */ + if (!(reparse_attr->reparse_tag & IO_REPARSE_TAG_IS_MICROSOFT)) + expected += sizeof(struct guid); + + /* Buffer must exactly match the expected size */ + return expected == size; +} + +/* + * Do some sanity checks on reparse data + * + * Microsoft reparse points have an 8-byte header whereas + * non-Microsoft reparse points have a 24-byte header. In each case, + * 'reparse_data_length' must equal the number of non-header bytes. + * + * If the reparse data looks like a junction point or symbolic + * link, more checks can be done. + */ +static bool valid_reparse_data(struct ntfs_inode *ni, + const struct reparse_point *reparse_attr, size_t size) +{ + if (size < sizeof(*reparse_attr)) + return false; + + switch (reparse_attr->reparse_tag) { + case IO_REPARSE_TAG_MOUNT_POINT: + { + struct mount_point_reparse_data *data; + size_t data_offs; + + if (!valid_reparse_buffer(ni, reparse_attr, size, sizeof(*data))) + return false; + + data = (struct mount_point_reparse_data *)reparse_attr->reparse_data; + data_offs = offsetof(struct reparse_point, reparse_data) + + offsetof(struct mount_point_reparse_data, path_buffer); + + if (!reparse_name_is_valid(size, + data_offs + + le16_to_cpu(data->substitute_name_offset), + le16_to_cpu(data->substitute_name_length)) || + !reparse_name_is_valid(size, + data_offs + + le16_to_cpu(data->print_name_offset), + le16_to_cpu(data->print_name_length))) + return false; + break; + } + case IO_REPARSE_TAG_SYMLINK: + { + struct symlink_reparse_data *data; + size_t data_offs; + + if (!valid_reparse_buffer(ni, reparse_attr, size, + sizeof(*data))) + return false; + + data = (struct symlink_reparse_data *)reparse_attr->reparse_data; + data_offs = offsetof(struct reparse_point, reparse_data) + + offsetof(struct symlink_reparse_data, path_buffer); + + if (!reparse_name_is_valid(size, + data_offs + + le16_to_cpu(data->substitute_name_offset), + le16_to_cpu(data->substitute_name_length)) || + !reparse_name_is_valid(size, + data_offs + + le16_to_cpu(data->print_name_offset), + le16_to_cpu(data->print_name_length))) + return false; + break; + } + case IO_REPARSE_TAG_LX_SYMLINK: + { + struct wsl_link_reparse_data *data; + + if (!valid_reparse_buffer(ni, reparse_attr, size, + sizeof(*data))) + return false; + + data = (struct wsl_link_reparse_data *)reparse_attr->reparse_data; + + if (le16_to_cpu(reparse_attr->reparse_data_length) <= sizeof(data->type) || + data->type != cpu_to_le32(2)) + return false; + break; + } + case IO_REPARSE_TAG_AF_UNIX: + case IO_REPARSE_TAG_LX_FIFO: + case IO_REPARSE_TAG_LX_CHR: + case IO_REPARSE_TAG_LX_BLK: + if (!valid_reparse_buffer(ni, reparse_attr, size, 0)) + return false; + if (le16_to_cpu(reparse_attr->reparse_data_length) || + !(ni->flags & FILE_ATTRIBUTE_RECALL_ON_OPEN)) + return false; + break; + default: + if (!valid_reparse_buffer(ni, reparse_attr, size, 0)) + return false; + break; + } + + return true; +} + +static unsigned int ntfs_reparse_tag_mode(__le32 reparse_tag) +{ + unsigned int mode = 0; + + switch (reparse_tag) { + case IO_REPARSE_TAG_MOUNT_POINT: + case IO_REPARSE_TAG_SYMLINK: + case IO_REPARSE_TAG_LX_SYMLINK: + mode = S_IFLNK; + break; + case IO_REPARSE_TAG_AF_UNIX: + mode = S_IFSOCK; + break; + case IO_REPARSE_TAG_LX_FIFO: + mode = S_IFIFO; + break; + case IO_REPARSE_TAG_LX_CHR: + mode = S_IFCHR; + break; + case IO_REPARSE_TAG_LX_BLK: + mode = S_IFBLK; + } + + return mode; +} + +/* + * Get the target for symbolic link + */ +unsigned int ntfs_make_symlink(struct ntfs_inode *ni) +{ + s64 attr_size = 0; + int err; + unsigned int lth; + struct reparse_point *reparse_attr; + unsigned int mode = 0; + + kvfree(ni->target); + ni->target = NULL; + ni->reparse_tag = 0; + ni->reparse_flags = 0; + + reparse_attr = ntfs_attr_readall(ni, AT_REPARSE_POINT, NULL, 0, + &attr_size); + if (reparse_attr && + valid_reparse_data(ni, reparse_attr, attr_size)) { + err = -EINVAL; + + switch (reparse_attr->reparse_tag) { + case IO_REPARSE_TAG_MOUNT_POINT: + { + struct mount_point_reparse_data *data = + (struct mount_point_reparse_data *)reparse_attr->reparse_data; + const __le16 *name = (const __le16 *)((u8 *)data->path_buffer + + le16_to_cpu(data->substitute_name_offset)); + + err = ntfs_reparse_target_to_nls(ni->vol, + name, + le16_to_cpu(data->substitute_name_length), + &ni->target); + break; + } + case IO_REPARSE_TAG_SYMLINK: + { + struct symlink_reparse_data *data = + (struct symlink_reparse_data *)reparse_attr->reparse_data; + const __le16 *name = (const __le16 *)((u8 *)data->path_buffer + + le16_to_cpu(data->substitute_name_offset)); + + err = ntfs_reparse_target_to_nls(ni->vol, + name, + le16_to_cpu(data->substitute_name_length), + &ni->target); + if (!err) + ni->reparse_flags = data->flags; + break; + } + case IO_REPARSE_TAG_LX_SYMLINK: + { + struct wsl_link_reparse_data *wsl_link_data = + (struct wsl_link_reparse_data *)reparse_attr->reparse_data; + + if (wsl_link_data->type == cpu_to_le32(2)) { + lth = le16_to_cpu(reparse_attr->reparse_data_length) - + sizeof(wsl_link_data->type); + ni->target = kvzalloc(lth + 1, GFP_NOFS); + if (ni->target) { + memcpy(ni->target, wsl_link_data->link, lth); + ni->target[lth] = 0; + err = 0; + } + } + break; + } + default: + err = 0; + } + + if (!err) { + mode = ntfs_reparse_tag_mode(reparse_attr->reparse_tag); + ni->reparse_tag = reparse_attr->reparse_tag; + } + } else + ni->flags &= ~FILE_ATTR_REPARSE_POINT; + + if (reparse_attr) + kvfree(reparse_attr); + + return mode; +} + +unsigned int ntfs_reparse_tag_dt_types(struct ntfs_volume *vol, unsigned long mref) +{ + s64 attr_size = 0; + struct reparse_point *reparse_attr; + unsigned int dt_type = DT_UNKNOWN; + struct inode *vi; + + vi = ntfs_iget(vol->sb, mref); + if (IS_ERR(vi)) + return PTR_ERR(vi); + + reparse_attr = (struct reparse_point *)ntfs_attr_readall(NTFS_I(vi), + AT_REPARSE_POINT, NULL, 0, &attr_size); + + if (reparse_attr && attr_size >= sizeof(*reparse_attr)) { + switch (reparse_attr->reparse_tag) { + case IO_REPARSE_TAG_MOUNT_POINT: + case IO_REPARSE_TAG_SYMLINK: + case IO_REPARSE_TAG_LX_SYMLINK: + dt_type = DT_LNK; + break; + case IO_REPARSE_TAG_AF_UNIX: + dt_type = DT_SOCK; + break; + case IO_REPARSE_TAG_LX_FIFO: + dt_type = DT_FIFO; + break; + case IO_REPARSE_TAG_LX_CHR: + dt_type = DT_CHR; + break; + case IO_REPARSE_TAG_LX_BLK: + dt_type = DT_BLK; + } + } + + if (reparse_attr) + kvfree(reparse_attr); + + iput(vi); + return dt_type; +} + +static bool ntfs_is_drive_letter(const char *target) +{ + return ((target[0] >= 'A' && target[0] <= 'Z') || + (target[0] >= 'a' && target[0] <= 'z')) && + target[1] == ':'; +} + +/* + * ntfs_translate_symlink_path + * + * @dentry: dentry of the symlink/junction being resolved + * @target: NUL-terminated NLS target string with '\\' already normalized to '/' + * @translated: out parameter, set to a newly kmalloc'd relative path on success + * + * Windows junctions (IO_REPARSE_TAG_MOUNT_POINT) and non-relative symlinks + * (IO_REPARSE_TAG_SYMLINK without SYMLINK_FLAG_RELATIVE) store substitute + * names such as "/??/C:/foo", "//?/C:/foo", "/foo", or "C:/foo". Linux + * cannot continue pathname lookup from those syntaxes, so rewrite them as a + * path relative to the symlink's containing directory on this NTFS volume, + * anchored at the volume root via "../". + * + * Note: bind-mounted subtrees of the volume may resolve to unexpected + * locations because the computed "../" depth is relative to the NTFS volume + * root, not the bind-mounted subtree root. + * + * Return: 0 on success with *translated set to a newly allocated string the + * caller must kfree(); negative errno on failure. + */ +int ntfs_translate_symlink_path(struct dentry *dentry, const char *target, + char **translated) +{ + char *buf, *link_path, *out, *p; + const char *path, *tail; + unsigned int up_levels = 0; + size_t tail_len, out_len; + int err; + + if (!dentry || !target || !translated) + return -EINVAL; + + path = target; + /* reject UNC path. */ + if (path[0] == '/' && path[1] == '/' && + !(path[2] == '?' && path[3] == '/')) + return -EOPNOTSUPP; + + /* target starts with "/??/" or "//?/"? */ + if ((path[0] == '/' && path[1] == '?' && path[2] == '?' && path[3] == '/') || + (path[0] == '/' && path[1] == '/' && path[2] == '?' && path[3] == '/')) + path += 4; + + /* target must start with a drive character or '/'. */ + if (ntfs_is_drive_letter(path)) { + if (path[2] && path[2] != '/') + return -EOPNOTSUPP; + tail = path + 2; + if (*tail == '/') + tail++; + } else if (*path == '/') { + tail = path + 1; + } else { + return -EOPNOTSUPP; + } + + tail_len = strlen(tail); + + buf = kmalloc(PATH_MAX, GFP_NOFS); + if (!buf) + return -ENOMEM; + + link_path = dentry_path_raw(dentry, buf, PATH_MAX); + if (IS_ERR(link_path)) { + err = PTR_ERR(link_path); + goto out; + } + + /* count '/' after the leading slash. */ + for (p = link_path + 1; *p; p++) + if (*p == '/') + up_levels++; + + /* build "./" + ("../" * up_levels) + tail. */ + out_len = 2 + up_levels * 3 + tail_len; + if (out_len >= PATH_MAX) { + err = -ENAMETOOLONG; + goto out; + } + + out = kmalloc(out_len + 1, GFP_NOFS); + if (!out) { + err = -ENOMEM; + goto out; + } + + memcpy(out, "./", 2); + p = out + 2; + while (up_levels--) { + memcpy(p, "../", 3); + p += 3; + } + memcpy(p, tail, tail_len + 1); + + *translated = out; + err = 0; +out: + kfree(buf); + return err; +} + +/* + * Set the index for new reparse data + */ +static int set_reparse_index(struct ntfs_inode *ni, struct ntfs_index_context *xr, + __le32 reparse_tag) +{ + struct reparse_index indx; + u64 file_id_cpu; + __le64 file_id; + + file_id_cpu = MK_MREF(ni->mft_no, ni->seq_no); + file_id = cpu_to_le64(file_id_cpu); + indx.header.data.vi.data_offset = + cpu_to_le16(sizeof(struct index_entry_header) + sizeof(struct reparse_index_key)); + indx.header.data.vi.data_length = 0; + indx.header.data.vi.reservedV = 0; + indx.header.length = cpu_to_le16(sizeof(struct reparse_index)); + indx.header.key_length = cpu_to_le16(sizeof(struct reparse_index_key)); + indx.header.flags = 0; + indx.header.reserved = 0; + indx.key.reparse_tag = reparse_tag; + /* danger on processors which require proper alignment! */ + memcpy(&indx.key.file_id, &file_id, 8); + indx.filling = 0; + ntfs_index_ctx_reinit(xr); + + return ntfs_ie_add(xr, (struct index_entry *)&indx); +} + +/* + * Remove a reparse data index entry if attribute present + */ +static int remove_reparse_index(struct inode *rp, struct ntfs_index_context *xr, + __le32 *preparse_tag) +{ + struct reparse_index_key key; + u64 file_id_cpu; + __le64 file_id; + s64 size; + struct ntfs_inode *ni = NTFS_I(rp); + int err = 0, ret = ni->data_size; + + if (ni->data_size == 0) + return 0; + + /* read the existing reparse_tag */ + size = ntfs_inode_attr_pread(rp, 0, 4, (char *)preparse_tag); + if (size != 4) + return -ENODATA; + + file_id_cpu = MK_MREF(ni->mft_no, ni->seq_no); + file_id = cpu_to_le64(file_id_cpu); + key.reparse_tag = *preparse_tag; + /* danger on processors which require proper alignment! */ + memcpy(&key.file_id, &file_id, 8); + if (!ntfs_index_lookup(&key, sizeof(struct reparse_index_key), xr)) { + err = ntfs_index_rm(xr); + if (err) + ret = err; + } + return ret; +} + +/* + * Open the $Extend/$Reparse file and its index + */ +static struct ntfs_index_context *open_reparse_index(struct ntfs_volume *vol) +{ + struct ntfs_index_context *xr = NULL; + u64 mref; + __le16 *uname; + struct ntfs_name *name = NULL; + int uname_len; + struct inode *vi, *dir_vi; + + /* do not use path_name_to inode - could reopen root */ + dir_vi = ntfs_iget(vol->sb, FILE_Extend); + if (IS_ERR(dir_vi)) + return NULL; + + uname_len = ntfs_nlstoucs(vol, "$Reparse", 8, &uname, + NTFS_MAX_NAME_LEN); + if (uname_len < 0) { + iput(dir_vi); + return NULL; + } + + mutex_lock_nested(&NTFS_I(dir_vi)->mrec_lock, NTFS_EXTEND_MUTEX_PARENT); + mref = ntfs_lookup_inode_by_name(NTFS_I(dir_vi), uname, uname_len, + &name); + mutex_unlock(&NTFS_I(dir_vi)->mrec_lock); + kfree(name); + kmem_cache_free(ntfs_name_cache, uname); + if (IS_ERR_MREF(mref)) + goto put_dir_vi; + + vi = ntfs_iget(vol->sb, MREF(mref)); + if (IS_ERR(vi)) + goto put_dir_vi; + + xr = ntfs_index_ctx_get(NTFS_I(vi), reparse_index_name, 2); + if (!xr) + iput(vi); +put_dir_vi: + iput(dir_vi); + return xr; +} + + +/* + * Update the reparse data and index + * + * The reparse data attribute should have been created, and + * an existing index is expected if there is an existing value. + * + */ +static int update_reparse_data(struct ntfs_inode *ni, struct ntfs_index_context *xr, + char *value, size_t size) +{ + struct inode *rp_inode; + int err = 0; + s64 written; + int oldsize; + __le32 reparse_tag; + struct ntfs_inode *rp_ni; + + rp_inode = ntfs_attr_iget(VFS_I(ni), AT_REPARSE_POINT, AT_UNNAMED, 0); + if (IS_ERR(rp_inode)) + return -EINVAL; + rp_ni = NTFS_I(rp_inode); + + /* remove the existing reparse data */ + oldsize = remove_reparse_index(rp_inode, xr, &reparse_tag); + if (oldsize < 0) { + err = oldsize; + goto put_rp_inode; + } + + /* overwrite value if any */ + written = ntfs_inode_attr_pwrite(rp_inode, 0, size, value, false); + if (written != size) { + ntfs_error(ni->vol->sb, "Failed to update reparse data\n"); + err = -EIO; + goto put_rp_inode; + } + + if (set_reparse_index(ni, xr, ((const struct reparse_point *)value)->reparse_tag) && + oldsize > 0) { + /* + * If cannot index, try to remove the reparse + * data and log the error. There will be an + * inconsistency if removal fails. + */ + ntfs_attr_rm(rp_ni); + ntfs_error(ni->vol->sb, + "Failed to index reparse data. Possible corruption.\n"); + } + + mark_mft_record_dirty(ni); +put_rp_inode: + iput(rp_inode); + + return err; +} + +/* + * Delete a reparse index entry + */ +int ntfs_delete_reparse_index(struct ntfs_inode *ni) +{ + struct inode *vi; + struct ntfs_index_context *xr; + struct ntfs_inode *xrni; + __le32 reparse_tag; + int err = 0; + + if (!(ni->flags & FILE_ATTR_REPARSE_POINT)) + return 0; + + vi = ntfs_attr_iget(VFS_I(ni), AT_REPARSE_POINT, AT_UNNAMED, 0); + if (IS_ERR(vi)) + return PTR_ERR(vi); + + /* + * read the existing reparse data (the tag is enough) + * and un-index it + */ + xr = open_reparse_index(ni->vol); + if (xr) { + xrni = xr->idx_ni; + mutex_lock_nested(&xrni->mrec_lock, NTFS_EXTEND_MUTEX_PARENT); + err = remove_reparse_index(vi, xr, &reparse_tag); + if (err < 0) { + ntfs_index_ctx_put(xr); + mutex_unlock(&xrni->mrec_lock); + iput(VFS_I(xrni)); + goto out; + } + mark_mft_record_dirty(xrni); + ntfs_index_ctx_put(xr); + mutex_unlock(&xrni->mrec_lock); + iput(VFS_I(xrni)); + } + + ni->flags &= ~FILE_ATTR_REPARSE_POINT; + NInoSetFileNameDirty(ni); + mark_mft_record_dirty(ni); + +out: + iput(vi); + return err; +} + +/* + * Set the reparse data from an extended attribute + */ +static int ntfs_set_ntfs_reparse_data(struct ntfs_inode *ni, char *value, size_t size) +{ + int err = 0; + struct ntfs_inode *xrni; + struct ntfs_index_context *xr; + + if (!ni) + return -EINVAL; + + /* + * reparse data compatibily with EA is not checked + * any more, it is required by Windows 10, but may + * lead to problems with earlier versions. + */ + if (valid_reparse_data(ni, (const struct reparse_point *)value, size) == false) + return -EINVAL; + + xr = open_reparse_index(ni->vol); + if (!xr) + return -EINVAL; + xrni = xr->idx_ni; + + if (!ntfs_attr_exist(ni, AT_REPARSE_POINT, AT_UNNAMED, 0)) { + struct reparse_point rp = {0, }; + + /* + * no reparse data attribute : add one, + * apparently, this does not feed the new value in + * Note : NTFS version must be >= 3 + */ + if (ni->vol->major_ver < 3) { + err = -EOPNOTSUPP; + ntfs_index_ctx_put(xr); + goto out; + } + + err = ntfs_attr_add(ni, AT_REPARSE_POINT, AT_UNNAMED, 0, (u8 *)&rp, sizeof(rp)); + if (err) { + ntfs_index_ctx_put(xr); + goto out; + } + ni->flags |= FILE_ATTR_REPARSE_POINT; + NInoSetFileNameDirty(ni); + mark_mft_record_dirty(ni); + } + + /* update value and index */ + mutex_lock_nested(&xrni->mrec_lock, NTFS_EXTEND_MUTEX_PARENT); + err = update_reparse_data(ni, xr, value, size); + if (err) { + ni->flags &= ~FILE_ATTR_REPARSE_POINT; + NInoSetFileNameDirty(ni); + mark_mft_record_dirty(ni); + } + ntfs_index_ctx_put(xr); + mutex_unlock(&xrni->mrec_lock); + +out: + if (!err) + mark_mft_record_dirty(xrni); + iput(VFS_I(xrni)); + + return err; +} + +/* + * Set reparse data for a WSL type symlink + */ +int ntfs_reparse_set_wsl_symlink(struct ntfs_inode *ni, + const char *target, int target_len) +{ + int err = 0; + int reparse_len; + struct reparse_point *reparse; + struct wsl_link_reparse_data *data; + + reparse_len = sizeof(struct reparse_point) + sizeof(data->type) + + target_len; + reparse = kvzalloc(reparse_len, GFP_NOFS); + if (!reparse) + return -ENOMEM; + + ni->target = kstrdup(target, GFP_NOFS); + if (!ni->target) { + kvfree(reparse); + return -ENOMEM; + } + + data = (struct wsl_link_reparse_data *)reparse->reparse_data; + reparse->reparse_tag = IO_REPARSE_TAG_LX_SYMLINK; + reparse->reparse_data_length = + cpu_to_le16(sizeof(data->type) + target_len); + reparse->reserved = 0; + data->type = cpu_to_le32(2); + memcpy(data->link, target, target_len); + err = ntfs_set_ntfs_reparse_data(ni, + (char *)reparse, reparse_len); + kvfree(reparse); + if (err) { + kfree(ni->target); + ni->target = NULL; + } else { + ni->reparse_tag = IO_REPARSE_TAG_LX_SYMLINK; + ni->reparse_flags = 0; + } + return err; +} + +int ntfs_reparse_set_native_symlink(struct ntfs_inode *ni, + const char *target, int target_len) +{ + int err = 0; + bool is_absolute, prt_sub_shared = true; + char *sub_name = NULL; + char *prt_name = NULL; + __le16 *sub_name_utf16 = NULL; + __le16 *prt_name_utf16 = NULL; + int sub_len, prt_len; + int total_data_len, total_reparse_len; + struct reparse_point *reparse = NULL; + struct symlink_reparse_data *data; + int i; + + /* Determine if target is absolute (starts with drive letter like C:/ or C:\) */ + is_absolute = target_len > 2 && + ntfs_is_drive_letter(target) && + (target[2] == '/' || target[2] == '\\'); + + + /* Normalize and prepare NLS paths */ + prt_name = kstrdup(target, GFP_NOFS); + if (!prt_name) + return -ENOMEM; + + /* Replace '/' with '\' */ + for (i = 0; i < target_len; i++) { + if (prt_name[i] == '/') + prt_name[i] = '\\'; + } + + if (is_absolute) { + /* Prepend '\??\' to Substitutename */ + sub_name = kmalloc(target_len + 5, GFP_NOFS); + if (!sub_name) { + err = -ENOMEM; + goto out; + } + snprintf(sub_name, target_len + 5, "\\??\\%s", prt_name); + prt_sub_shared = false; + } else { + /* For relative symlinks (including absolute paths without drive letters), + * SubstituteName and PrintName are identical. + */ + sub_name = prt_name; + } + + /* Convert NLS paths to UTF-16 */ + sub_len = ntfs_nlstoucs(ni->vol, sub_name, strlen(sub_name), + &sub_name_utf16, PATH_MAX); + if (sub_len < 0) { + err = sub_len; + goto out; + } + + prt_len = ntfs_nlstoucs(ni->vol, prt_name, strlen(prt_name), + &prt_name_utf16, PATH_MAX); + if (prt_len < 0) { + err = prt_len; + goto out; + } + + /* Check for buffer size limits */ + total_data_len = sizeof(struct symlink_reparse_data) + + (sub_len + prt_len) * sizeof(__le16); + if (total_data_len > 16384) { /* 16KB max reparse tag size */ + err = -EFBIG; + goto out; + } + + total_reparse_len = sizeof(struct reparse_point) + total_data_len; + reparse = kvzalloc(total_reparse_len, GFP_NOFS); + if (!reparse) { + err = -ENOMEM; + goto out; + } + + /* Pack fields in reparse buffer */ + reparse->reparse_tag = IO_REPARSE_TAG_SYMLINK; + reparse->reparse_data_length = cpu_to_le16(total_data_len); + reparse->reserved = 0; + + data = (struct symlink_reparse_data *)reparse->reparse_data; + data->substitute_name_offset = 0; + data->substitute_name_length = cpu_to_le16(sub_len * sizeof(__le16)); + data->print_name_offset = data->substitute_name_length; + data->print_name_length = cpu_to_le16(prt_len * sizeof(__le16)); + data->flags = is_absolute ? 0 : cpu_to_le32(SYMLINK_FLAG_RELATIVE); + + /* Copy names to path_buffer */ + memcpy(data->path_buffer, sub_name_utf16, sub_len * sizeof(__le16)); + memcpy(data->path_buffer + sub_len, prt_name_utf16, prt_len * sizeof(__le16)); + + err = ntfs_set_ntfs_reparse_data(ni, (char *)reparse, total_reparse_len); + if (!err) { + int len = strlen(sub_name); + + for (i = 0; i < len; i++) { + if (sub_name[i] == '\\') + sub_name[i] = '/'; + } + ni->target = sub_name; + sub_name = NULL; + if (prt_sub_shared) + prt_name = NULL; + ni->reparse_tag = IO_REPARSE_TAG_SYMLINK; + ni->reparse_flags = is_absolute ? 0 : + cpu_to_le32(SYMLINK_FLAG_RELATIVE); + } + +out: + kfree(prt_name); + if (!prt_sub_shared) + kfree(sub_name); + kvfree(sub_name_utf16); + kvfree(prt_name_utf16); + kvfree(reparse); + return err; +} + +/* + * Set reparse data for a WSL special file other than a symlink + * (socket, fifo, character or block device) + */ +int ntfs_reparse_set_wsl_not_symlink(struct ntfs_inode *ni, mode_t mode) +{ + int err; + int len; + int reparse_len; + __le32 reparse_tag; + struct reparse_point *reparse; + + len = 0; + if (S_ISSOCK(mode)) + reparse_tag = IO_REPARSE_TAG_AF_UNIX; + else if (S_ISFIFO(mode)) + reparse_tag = IO_REPARSE_TAG_LX_FIFO; + else if (S_ISCHR(mode)) + reparse_tag = IO_REPARSE_TAG_LX_CHR; + else if (S_ISBLK(mode)) + reparse_tag = IO_REPARSE_TAG_LX_BLK; + else + return -EOPNOTSUPP; + + reparse_len = sizeof(struct reparse_point) + len; + reparse = kvzalloc(reparse_len, GFP_NOFS); + if (!reparse) + err = -ENOMEM; + else { + reparse->reparse_tag = reparse_tag; + reparse->reparse_data_length = cpu_to_le16(len); + reparse->reserved = cpu_to_le16(0); + err = ntfs_set_ntfs_reparse_data(ni, (char *)reparse, + reparse_len); + kvfree(reparse); + if (!err) { + ni->reparse_tag = reparse_tag; + ni->reparse_flags = 0; + } + } + + return err; +} diff --git a/fs/ntfs/reparse.h b/fs/ntfs/reparse.h new file mode 100644 index 000000000..c11a5bb7e --- /dev/null +++ b/fs/ntfs/reparse.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2008-2021 Jean-Pierre Andre + * Copyright (c) 2025 LG Electronics Co., Ltd. + */ + +#ifndef _LINUX_NTFS_REPARSE_H +#define _LINUX_NTFS_REPARSE_H + +extern __le16 reparse_index_name[]; + +unsigned int ntfs_make_symlink(struct ntfs_inode *ni); +unsigned int ntfs_reparse_tag_dt_types(struct ntfs_volume *vol, unsigned long mref); +int ntfs_translate_symlink_path(struct dentry *dentry, const char *target, + char **translated); +int ntfs_reparse_set_wsl_symlink(struct ntfs_inode *ni, + const char *target, int target_len); +int ntfs_reparse_set_native_symlink(struct ntfs_inode *ni, + const char *symname, int symlen); +int ntfs_reparse_set_wsl_not_symlink(struct ntfs_inode *ni, mode_t mode); +int ntfs_delete_reparse_index(struct ntfs_inode *ni); +int ntfs_remove_ntfs_reparse_data(struct ntfs_inode *ni); + +#endif /* _LINUX_NTFS_REPARSE_H */ diff --git a/fs/ntfs/runlist.c b/fs/ntfs/runlist.c new file mode 100644 index 000000000..8e0fd400e --- /dev/null +++ b/fs/ntfs/runlist.c @@ -0,0 +1,2093 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * NTFS runlist handling code. + * + * Copyright (c) 2001-2007 Anton Altaparmakov + * Copyright (c) 2002-2005 Richard Russon + * Copyright (c) 2025 LG Electronics Co., Ltd. + * + * Part of this file is based on code from the NTFS-3G. + * and is copyrighted by the respective authors below: + * Copyright (c) 2002-2005 Anton Altaparmakov + * Copyright (c) 2002-2005 Richard Russon + * Copyright (c) 2002-2008 Szabolcs Szakacsits + * Copyright (c) 2004 Yura Pakhuchiy + * Copyright (c) 2007-2022 Jean-Pierre Andre + */ + +#include + +#include "ntfs.h" +#include "attrib.h" + +/* + * ntfs_rl_mm - runlist memmove + * @base: base runlist array + * @dst: destination index in @base + * @src: source index in @base + * @size: number of elements to move + * + * It is up to the caller to serialize access to the runlist @base. + */ +static inline void ntfs_rl_mm(struct runlist_element *base, int dst, int src, int size) +{ + if (likely((dst != src) && (size > 0))) + memmove(base + dst, base + src, size * sizeof(*base)); +} + +/* + * ntfs_rl_mc - runlist memory copy + * @dstbase: destination runlist array + * @dst: destination index in @dstbase + * @srcbase: source runlist array + * @src: source index in @srcbase + * @size: number of elements to copy + * + * It is up to the caller to serialize access to the runlists @dstbase and + * @srcbase. + */ +static inline void ntfs_rl_mc(struct runlist_element *dstbase, int dst, + struct runlist_element *srcbase, int src, int size) +{ + if (likely(size > 0)) + memcpy(dstbase + dst, srcbase + src, size * sizeof(*dstbase)); +} + +/* + * ntfs_rl_realloc - Reallocate memory for runlists + * @rl: original runlist + * @old_size: number of runlist elements in the original runlist @rl + * @new_size: number of runlist elements we need space for + * + * As the runlists grow, more memory will be required. To prevent the + * kernel having to allocate and reallocate large numbers of small bits of + * memory, this function returns an entire page of memory. + * + * It is up to the caller to serialize access to the runlist @rl. + * + * N.B. If the new allocation doesn't require a different number of pages in + * memory, the function will return the original pointer. + * + * On success, return a pointer to the newly allocated, or recycled, memory. + * On error, return -errno. + */ +static inline struct runlist_element *ntfs_rl_realloc_gfp(struct runlist_element *rl, + int old_size, int new_size, gfp_t gfp) +{ + struct runlist_element *new_rl; + size_t new_bytes; + + if (old_size < 0 || new_size < 0) + return ERR_PTR(-EINVAL); + + if (old_size == new_size) + return rl; + + if (check_mul_overflow(new_size, sizeof(*rl), &new_bytes)) + return ERR_PTR(-EINVAL); + + new_rl = kvzalloc(new_bytes, gfp); + if (unlikely(!new_rl)) + return ERR_PTR(-ENOMEM); + + if (likely(rl != NULL)) { + size_t old_bytes; + + if (check_mul_overflow(old_size, sizeof(*rl), &old_bytes)) { + kvfree(new_rl); + return ERR_PTR(-EINVAL); + } + if (unlikely(old_bytes > new_bytes)) + old_bytes = new_bytes; + memcpy(new_rl, rl, old_bytes); + kvfree(rl); + } + return new_rl; +} + +struct runlist_element *ntfs_rl_realloc(struct runlist_element *rl, + int old_size, int new_size) +{ + return ntfs_rl_realloc_gfp(rl, old_size, new_size, GFP_NOFS); +} + +/* + * ntfs_rl_realloc_nofail - Reallocate memory for runlists + * @rl: original runlist + * @old_size: number of runlist elements in the original runlist @rl + * @new_size: number of runlist elements we need space for + * + * As the runlists grow, more memory will be required. To prevent the + * kernel having to allocate and reallocate large numbers of small bits of + * memory, this function returns an entire page of memory. + * + * This function guarantees that the allocation will succeed. It will sleep + * for as long as it takes to complete the allocation. + * + * It is up to the caller to serialize access to the runlist @rl. + * + * N.B. If the new allocation doesn't require a different number of pages in + * memory, the function will return the original pointer. + * + * On success, return a pointer to the newly allocated, or recycled, memory. + * On error, return -errno. + */ +static inline struct runlist_element *ntfs_rl_realloc_nofail(struct runlist_element *rl, + int old_size, int new_size) +{ + return ntfs_rl_realloc_gfp(rl, old_size, new_size, + GFP_NOFS | __GFP_NOFAIL); +} + +/* + * ntfs_are_rl_mergeable - test if two runlists can be joined together + * @dst: original runlist + * @src: new runlist to test for mergeability with @dst + * + * Test if two runlists can be joined together. For this, their VCNs and LCNs + * must be adjacent. + * + * It is up to the caller to serialize access to the runlists @dst and @src. + * + * Return: true Success, the runlists can be merged. + * false Failure, the runlists cannot be merged. + */ +static inline bool ntfs_are_rl_mergeable(struct runlist_element *dst, + struct runlist_element *src) +{ + /* We can merge unmapped regions even if they are misaligned. */ + if ((dst->lcn == LCN_RL_NOT_MAPPED) && (src->lcn == LCN_RL_NOT_MAPPED)) + return true; + /* If the runs are misaligned, we cannot merge them. */ + if ((dst->vcn + dst->length) != src->vcn) + return false; + /* If both runs are non-sparse and contiguous, we can merge them. */ + if ((dst->lcn >= 0) && (src->lcn >= 0) && + ((dst->lcn + dst->length) == src->lcn)) + return true; + /* If we are merging two holes, we can merge them. */ + if ((dst->lcn == LCN_HOLE) && (src->lcn == LCN_HOLE)) + return true; + /* If we are merging two dealloc, we can merge them. */ + if ((dst->lcn == LCN_DELALLOC) && (src->lcn == LCN_DELALLOC)) + return true; + /* Cannot merge. */ + return false; +} + +/* + * __ntfs_rl_merge - merge two runlists without testing if they can be merged + * @dst: original, destination runlist + * @src: new runlist to merge with @dst + * + * Merge the two runlists, writing into the destination runlist @dst. The + * caller must make sure the runlists can be merged or this will corrupt the + * destination runlist. + * + * It is up to the caller to serialize access to the runlists @dst and @src. + */ +static inline void __ntfs_rl_merge(struct runlist_element *dst, struct runlist_element *src) +{ + dst->length += src->length; +} + +/* + * ntfs_rl_append - append a runlist after a given element + * @dst: destination runlist to append to + * @dsize: number of elements in @dst + * @src: source runlist to append from + * @ssize: number of elements in @src + * @loc: index in @dst after which to append @src + * @new_size: on success, set to the new combined size + * + * Append the runlist @src after element @loc in @dst. Merge the right end of + * the new runlist, if necessary. Adjust the size of the hole before the + * appended runlist. + * + * It is up to the caller to serialize access to the runlists @dst and @src. + * + * On success, return a pointer to the new, combined, runlist. Note, both + * runlists @dst and @src are deallocated before returning so you cannot use + * the pointers for anything any more. (Strictly speaking the returned runlist + * may be the same as @dst but this is irrelevant.) + * + * On error, return -errno. Both runlists are left unmodified. + */ +static inline struct runlist_element *ntfs_rl_append(struct runlist_element *dst, + int dsize, struct runlist_element *src, int ssize, int loc, + size_t *new_size) +{ + bool right = false; /* Right end of @src needs merging. */ + int marker; /* End of the inserted runs. */ + + /* First, check if the right hand end needs merging. */ + if ((loc + 1) < dsize) + right = ntfs_are_rl_mergeable(src + ssize - 1, dst + loc + 1); + + /* Space required: @dst size + @src size, less one if we merged. */ + dst = ntfs_rl_realloc(dst, dsize, dsize + ssize - right); + if (IS_ERR(dst)) + return dst; + + *new_size = dsize + ssize - right; + /* + * We are guaranteed to succeed from here so can start modifying the + * original runlists. + */ + + /* First, merge the right hand end, if necessary. */ + if (right) + __ntfs_rl_merge(src + ssize - 1, dst + loc + 1); + + /* First run after the @src runs that have been inserted. */ + marker = loc + ssize + 1; + + /* Move the tail of @dst out of the way, then copy in @src. */ + ntfs_rl_mm(dst, marker, loc + 1 + right, dsize - (loc + 1 + right)); + ntfs_rl_mc(dst, loc + 1, src, 0, ssize); + + /* Adjust the size of the preceding hole. */ + dst[loc].length = dst[loc + 1].vcn - dst[loc].vcn; + + /* We may have changed the length of the file, so fix the end marker */ + if (dst[marker].lcn == LCN_ENOENT) + dst[marker].vcn = dst[marker - 1].vcn + dst[marker - 1].length; + + return dst; +} + +/* + * ntfs_rl_insert - insert a runlist into another + * @dst: destination runlist to insert into + * @dsize: number of elements in @dst + * @src: source runlist to insert from + * @ssize: number of elements in @src + * @loc: index in @dst at which to insert @src + * @new_size: on success, set to the new combined size + * + * Insert the runlist @src before element @loc in the runlist @dst. Merge the + * left end of the new runlist, if necessary. Adjust the size of the hole + * after the inserted runlist. + * + * It is up to the caller to serialize access to the runlists @dst and @src. + * + * On success, return a pointer to the new, combined, runlist. Note, both + * runlists @dst and @src are deallocated before returning so you cannot use + * the pointers for anything any more. (Strictly speaking the returned runlist + * may be the same as @dst but this is irrelevant.) + * + * On error, return -errno. Both runlists are left unmodified. + */ +static inline struct runlist_element *ntfs_rl_insert(struct runlist_element *dst, + int dsize, struct runlist_element *src, int ssize, int loc, + size_t *new_size) +{ + bool left = false; /* Left end of @src needs merging. */ + bool disc = false; /* Discontinuity between @dst and @src. */ + int marker; /* End of the inserted runs. */ + + /* + * disc => Discontinuity between the end of @dst and the start of @src. + * This means we might need to insert a "not mapped" run. + */ + if (loc == 0) + disc = (src[0].vcn > 0); + else { + s64 merged_length; + + left = ntfs_are_rl_mergeable(dst + loc - 1, src); + + merged_length = dst[loc - 1].length; + if (left) + merged_length += src->length; + + disc = (src[0].vcn > dst[loc - 1].vcn + merged_length); + } + /* + * Space required: @dst size + @src size, less one if we merged, plus + * one if there was a discontinuity. + */ + dst = ntfs_rl_realloc(dst, dsize, dsize + ssize - left + disc); + if (IS_ERR(dst)) + return dst; + + *new_size = dsize + ssize - left + disc; + /* + * We are guaranteed to succeed from here so can start modifying the + * original runlist. + */ + if (left) + __ntfs_rl_merge(dst + loc - 1, src); + /* + * First run after the @src runs that have been inserted. + * Nominally, @marker equals @loc + @ssize, i.e. location + number of + * runs in @src. However, if @left, then the first run in @src has + * been merged with one in @dst. And if @disc, then @dst and @src do + * not meet and we need an extra run to fill the gap. + */ + marker = loc + ssize - left + disc; + + /* Move the tail of @dst out of the way, then copy in @src. */ + ntfs_rl_mm(dst, marker, loc, dsize - loc); + ntfs_rl_mc(dst, loc + disc, src, left, ssize - left); + + /* Adjust the VCN of the first run after the insertion... */ + dst[marker].vcn = dst[marker - 1].vcn + dst[marker - 1].length; + /* ... and the length. */ + if (dst[marker].lcn == LCN_HOLE || dst[marker].lcn == LCN_RL_NOT_MAPPED || + dst[marker].lcn == LCN_DELALLOC) + dst[marker].length = dst[marker + 1].vcn - dst[marker].vcn; + + /* Writing beyond the end of the file and there is a discontinuity. */ + if (disc) { + if (loc > 0) { + dst[loc].vcn = dst[loc - 1].vcn + dst[loc - 1].length; + dst[loc].length = dst[loc + 1].vcn - dst[loc].vcn; + } else { + dst[loc].vcn = 0; + dst[loc].length = dst[loc + 1].vcn; + } + dst[loc].lcn = LCN_RL_NOT_MAPPED; + } + return dst; +} + +/* + * ntfs_rl_replace - overwrite a runlist element with another runlist + * @dst: destination runlist to replace in + * @dsize: number of elements in @dst + * @src: source runlist to replace with + * @ssize: number of elements in @src + * @loc: index in @dst to replace + * @new_size: on success, set to the new combined size + * + * Replace the runlist element @dst at @loc with @src. Merge the left and + * right ends of the inserted runlist, if necessary. + * + * It is up to the caller to serialize access to the runlists @dst and @src. + * + * On success, return a pointer to the new, combined, runlist. Note, both + * runlists @dst and @src are deallocated before returning so you cannot use + * the pointers for anything any more. (Strictly speaking the returned runlist + * may be the same as @dst but this is irrelevant.) + * + * On error, return -errno. Both runlists are left unmodified. + */ +static inline struct runlist_element *ntfs_rl_replace(struct runlist_element *dst, + int dsize, struct runlist_element *src, int ssize, int loc, + size_t *new_size) +{ + int delta; + bool left = false; /* Left end of @src needs merging. */ + bool right = false; /* Right end of @src needs merging. */ + int tail; /* Start of tail of @dst. */ + int marker; /* End of the inserted runs. */ + + /* First, see if the left and right ends need merging. */ + if ((loc + 1) < dsize) + right = ntfs_are_rl_mergeable(src + ssize - 1, dst + loc + 1); + if (loc > 0) + left = ntfs_are_rl_mergeable(dst + loc - 1, src); + /* + * Allocate some space. We will need less if the left, right, or both + * ends get merged. The -1 accounts for the run being replaced. + */ + delta = ssize - 1 - left - right; + if (delta > 0) { + dst = ntfs_rl_realloc(dst, dsize, dsize + delta); + if (IS_ERR(dst)) + return dst; + } + + *new_size = dsize + delta; + /* + * We are guaranteed to succeed from here so can start modifying the + * original runlists. + */ + + /* First, merge the left and right ends, if necessary. */ + if (right) + __ntfs_rl_merge(src + ssize - 1, dst + loc + 1); + if (left) + __ntfs_rl_merge(dst + loc - 1, src); + /* + * Offset of the tail of @dst. This needs to be moved out of the way + * to make space for the runs to be copied from @src, i.e. the first + * run of the tail of @dst. + * Nominally, @tail equals @loc + 1, i.e. location, skipping the + * replaced run. However, if @right, then one of @dst's runs is + * already merged into @src. + */ + tail = loc + right + 1; + /* + * First run after the @src runs that have been inserted, i.e. where + * the tail of @dst needs to be moved to. + * Nominally, @marker equals @loc + @ssize, i.e. location + number of + * runs in @src. However, if @left, then the first run in @src has + * been merged with one in @dst. + */ + marker = loc + ssize - left; + + /* Move the tail of @dst out of the way, then copy in @src. */ + ntfs_rl_mm(dst, marker, tail, dsize - tail); + ntfs_rl_mc(dst, loc, src, left, ssize - left); + + /* We may have changed the length of the file, so fix the end marker. */ + if (dsize - tail > 0 && dst[marker].lcn == LCN_ENOENT) + dst[marker].vcn = dst[marker - 1].vcn + dst[marker - 1].length; + return dst; +} + +/* + * ntfs_rl_split - insert a runlist into the centre of a hole + * @dst: destination runlist with a hole + * @dsize: number of elements in @dst + * @src: source runlist to insert + * @ssize: number of elements in @src + * @loc: index in @dst of the hole to split + * @new_size: on success, set to the new combined size + * + * Split the runlist @dst at @loc into two and insert @new in between the two + * fragments. No merging of runlists is necessary. Adjust the size of the + * holes either side. + * + * It is up to the caller to serialize access to the runlists @dst and @src. + * + * On success, return a pointer to the new, combined, runlist. Note, both + * runlists @dst and @src are deallocated before returning so you cannot use + * the pointers for anything any more. (Strictly speaking the returned runlist + * may be the same as @dst but this is irrelevant.) + * + * On error, return -errno. Both runlists are left unmodified. + */ +static inline struct runlist_element *ntfs_rl_split(struct runlist_element *dst, int dsize, + struct runlist_element *src, int ssize, int loc, + size_t *new_size) +{ + /* Space required: @dst size + @src size + one new hole. */ + dst = ntfs_rl_realloc(dst, dsize, dsize + ssize + 1); + if (IS_ERR(dst)) + return dst; + + *new_size = dsize + ssize + 1; + /* + * We are guaranteed to succeed from here so can start modifying the + * original runlists. + */ + + /* Move the tail of @dst out of the way, then copy in @src. */ + ntfs_rl_mm(dst, loc + 1 + ssize, loc, dsize - loc); + ntfs_rl_mc(dst, loc + 1, src, 0, ssize); + + /* Adjust the size of the holes either size of @src. */ + dst[loc].length = dst[loc+1].vcn - dst[loc].vcn; + dst[loc+ssize+1].vcn = dst[loc+ssize].vcn + dst[loc+ssize].length; + dst[loc+ssize+1].length = dst[loc+ssize+2].vcn - dst[loc+ssize+1].vcn; + + return dst; +} + +/* + * ntfs_runlists_merge - merge two runlists into one + * @d_runlist: destination runlist structure to merge into + * @srl: source runlist to merge from + * @s_rl_count: number of elements in @srl (0 to auto-detect) + * @new_rl_count: on success, set to the new combined runlist size + * + * First we sanity check the two runlists @srl and @drl to make sure that they + * are sensible and can be merged. The runlist @srl must be either after the + * runlist @drl or completely within a hole (or unmapped region) in @drl. + * + * It is up to the caller to serialize access to the runlists @drl and @srl. + * + * Merging of runlists is necessary in two cases: + * 1. When attribute lists are used and a further extent is being mapped. + * 2. When new clusters are allocated to fill a hole or extend a file. + * + * There are four possible ways @srl can be merged. It can: + * - be inserted at the beginning of a hole, + * - split the hole in two and be inserted between the two fragments, + * - be appended at the end of a hole, or it can + * - replace the whole hole. + * It can also be appended to the end of the runlist, which is just a variant + * of the insert case. + * + * On success, return a pointer to the new, combined, runlist. Note, both + * runlists @drl and @srl are deallocated before returning so you cannot use + * the pointers for anything any more. (Strictly speaking the returned runlist + * may be the same as @dst but this is irrelevant.) + * + * On error, return -errno. Both runlists are left unmodified. + */ +struct runlist_element *ntfs_runlists_merge(struct runlist *d_runlist, + struct runlist_element *srl, size_t s_rl_count, + size_t *new_rl_count) +{ + int di, si; /* Current index into @[ds]rl. */ + int sstart; /* First index with lcn > LCN_RL_NOT_MAPPED. */ + int dins; /* Index into @drl at which to insert @srl. */ + int dend, send; /* Last index into @[ds]rl. */ + int dfinal, sfinal; /* The last index into @[ds]rl with lcn >= LCN_HOLE. */ + int marker = 0; + s64 marker_vcn = 0; + struct runlist_element *drl = d_runlist->rl, *rl; + +#ifdef DEBUG + ntfs_debug("dst:"); + ntfs_debug_dump_runlist(drl); + ntfs_debug("src:"); + ntfs_debug_dump_runlist(srl); +#endif + + /* Check for silly calling... */ + if (unlikely(!srl)) + return drl; + if (IS_ERR(srl) || IS_ERR(drl)) + return ERR_PTR(-EINVAL); + + if (s_rl_count == 0) { + for (; srl[s_rl_count].length; s_rl_count++) + ; + s_rl_count++; + } + + /* Check for the case where the first mapping is being done now. */ + if (unlikely(!drl)) { + drl = srl; + /* Complete the source runlist if necessary. */ + if (unlikely(drl[0].vcn)) { + /* Scan to the end of the source runlist. */ + drl = ntfs_rl_realloc(drl, s_rl_count, s_rl_count + 1); + if (IS_ERR(drl)) + return drl; + /* Insert start element at the front of the runlist. */ + ntfs_rl_mm(drl, 1, 0, s_rl_count); + drl[0].vcn = 0; + drl[0].lcn = LCN_RL_NOT_MAPPED; + drl[0].length = drl[1].vcn; + s_rl_count++; + } + + *new_rl_count = s_rl_count; + goto finished; + } + + if (d_runlist->count < 1 || s_rl_count < 2) + return ERR_PTR(-EINVAL); + + si = di = 0; + + /* Skip any unmapped start element(s) in the source runlist. */ + while (srl[si].length && srl[si].lcn < LCN_HOLE) + si++; + + /* Can't have an entirely unmapped source runlist. */ + WARN_ON(!srl[si].length); + + /* Record the starting points. */ + sstart = si; + + /* + * Skip forward in @drl until we reach the position where @srl needs to + * be inserted. If we reach the end of @drl, @srl just needs to be + * appended to @drl. + */ + rl = __ntfs_attr_find_vcn_nolock(d_runlist, srl[sstart].vcn); + if (IS_ERR(rl)) + di = (int)d_runlist->count - 1; + else + di = (int)(rl - d_runlist->rl); + dins = di; + + /* Sanity check for illegal overlaps. */ + if ((drl[di].vcn == srl[si].vcn) && (drl[di].lcn >= 0) && + (srl[si].lcn >= 0)) { + ntfs_error(NULL, "Run lists overlap. Cannot merge!"); + return ERR_PTR(-ERANGE); + } + + /* Scan to the end of both runlists in order to know their sizes. */ + send = (int)s_rl_count - 1; + dend = (int)d_runlist->count - 1; + + if (srl[send].lcn == LCN_ENOENT) + marker_vcn = srl[marker = send].vcn; + + /* Scan to the last element with lcn >= LCN_HOLE. */ + for (sfinal = send; sfinal >= 0 && srl[sfinal].lcn < LCN_HOLE; sfinal--) + ; + for (dfinal = dend; dfinal >= 0 && drl[dfinal].lcn < LCN_HOLE; dfinal--) + ; + + { + bool start; + bool finish; + int ds = dend + 1; /* Number of elements in drl & srl */ + int ss = sfinal - sstart + 1; + + start = ((drl[dins].lcn < LCN_RL_NOT_MAPPED) || /* End of file */ + (drl[dins].vcn == srl[sstart].vcn)); /* Start of hole */ + finish = ((drl[dins].lcn >= LCN_RL_NOT_MAPPED) && /* End of file */ + ((drl[dins].vcn + drl[dins].length) <= /* End of hole */ + (srl[send - 1].vcn + srl[send - 1].length))); + + /* Or we will lose an end marker. */ + if (finish && !drl[dins].length) + ss++; + if (marker && (drl[dins].vcn + drl[dins].length > srl[send - 1].vcn)) + finish = false; + + if (start) { + if (finish) + drl = ntfs_rl_replace(drl, ds, srl + sstart, ss, dins, new_rl_count); + else + drl = ntfs_rl_insert(drl, ds, srl + sstart, ss, dins, new_rl_count); + } else { + if (finish) + drl = ntfs_rl_append(drl, ds, srl + sstart, ss, dins, new_rl_count); + else + drl = ntfs_rl_split(drl, ds, srl + sstart, ss, dins, new_rl_count); + } + if (IS_ERR(drl)) { + ntfs_error(NULL, "Merge failed."); + return drl; + } + kvfree(srl); + if (marker) { + ntfs_debug("Triggering marker code."); + for (ds = dend; drl[ds].length; ds++) + ; + /* We only need to care if @srl ended after @drl. */ + if (drl[ds].vcn <= marker_vcn) { + int slots = 0; + + if (drl[ds].vcn == marker_vcn) { + ntfs_debug("Old marker = 0x%llx, replacing with LCN_ENOENT.", + drl[ds].lcn); + drl[ds].lcn = LCN_ENOENT; + goto finished; + } + /* + * We need to create an unmapped runlist element in + * @drl or extend an existing one before adding the + * ENOENT terminator. + */ + if (drl[ds].lcn == LCN_ENOENT) { + ds--; + slots = 1; + } + if (drl[ds].lcn != LCN_RL_NOT_MAPPED) { + /* Add an unmapped runlist element. */ + if (!slots) { + drl = ntfs_rl_realloc_nofail(drl, ds, + ds + 2); + slots = 2; + *new_rl_count += 2; + } + ds++; + /* Need to set vcn if it isn't set already. */ + if (slots != 1) + drl[ds].vcn = drl[ds - 1].vcn + + drl[ds - 1].length; + drl[ds].lcn = LCN_RL_NOT_MAPPED; + /* We now used up a slot. */ + slots--; + } + drl[ds].length = marker_vcn - drl[ds].vcn; + /* Finally add the ENOENT terminator. */ + ds++; + if (!slots) { + drl = ntfs_rl_realloc_nofail(drl, ds, ds + 1); + *new_rl_count += 1; + } + drl[ds].vcn = marker_vcn; + drl[ds].lcn = LCN_ENOENT; + drl[ds].length = (s64)0; + } + } + } + +finished: + /* The merge was completed successfully. */ + ntfs_debug("Merged runlist:"); + ntfs_debug_dump_runlist(drl); + return drl; +} + +/* + * ntfs_mapping_pairs_decompress - convert mapping pairs array to runlist + * @vol: ntfs volume + * @attr: attribute record whose mapping pairs to decompress + * @old_runlist: optional runlist to merge the decompressed runlist into + * @new_rl_count: on success, set to the new runlist size + * + * It is up to the caller to serialize access to the runlist @old_rl. + * + * Decompress the attribute @attr's mapping pairs array into a runlist. On + * success, return the decompressed runlist. + * + * If @old_rl is not NULL, decompressed runlist is inserted into the + * appropriate place in @old_rl and the resultant, combined runlist is + * returned. The original @old_rl is deallocated. + * + * On error, return -errno. @old_rl is left unmodified in that case. + */ +struct runlist_element *ntfs_mapping_pairs_decompress(const struct ntfs_volume *vol, + const struct attr_record *attr, struct runlist *old_runlist, + size_t *new_rl_count) +{ + s64 vcn; /* Current vcn. */ + s64 lcn; /* Current lcn. */ + s64 deltaxcn; /* Change in [vl]cn. */ + struct runlist_element *rl, *new_rl; /* The output runlist. */ + u8 *buf; /* Current position in mapping pairs array. */ + u8 *attr_end; /* End of attribute. */ + int rlsize; /* Size of runlist buffer. */ + u16 rlpos; /* Current runlist position in units of struct runlist_elements. */ + u8 b; /* Current byte offset in buf. */ + u64 lowest_vcn; /* Raw on-disk lowest_vcn. */ + +#ifdef DEBUG + /* Make sure attr exists and is non-resident. */ + if (!attr || !attr->non_resident) { + ntfs_error(vol->sb, "Invalid arguments."); + return ERR_PTR(-EINVAL); + } +#endif + lowest_vcn = le64_to_cpu(attr->data.non_resident.lowest_vcn); + /* Validate lowest_vcn from on-disk metadata to ensure it is sane. */ + if (overflows_type(lowest_vcn, vcn)) { + ntfs_error(vol->sb, "Invalid lowest_vcn in mapping pairs."); + return ERR_PTR(-EIO); + } + /* Start at vcn = lowest_vcn and lcn 0. */ + vcn = lowest_vcn; + lcn = 0; + /* Get start of the mapping pairs array. */ + buf = (u8 *)attr + + le16_to_cpu(attr->data.non_resident.mapping_pairs_offset); + attr_end = (u8 *)attr + le32_to_cpu(attr->length); + if (unlikely(buf < (u8 *)attr || buf >= attr_end)) { + ntfs_error(vol->sb, "Corrupt attribute."); + return ERR_PTR(-EIO); + } + + /* Current position in runlist array. */ + rlpos = 0; + /* Allocate first page and set current runlist size to one page. */ + rl = kvzalloc(rlsize = PAGE_SIZE, GFP_NOFS); + if (unlikely(!rl)) + return ERR_PTR(-ENOMEM); + /* Insert unmapped starting element if necessary. */ + if (vcn) { + rl->vcn = 0; + rl->lcn = LCN_RL_NOT_MAPPED; + rl->length = vcn; + rlpos++; + } + while (buf < attr_end && *buf) { + /* + * Allocate more memory if needed, including space for the + * not-mapped and terminator elements. kvzalloc() + * operates on whole pages only. + */ + if (((rlpos + 3) * sizeof(*rl)) > rlsize) { + struct runlist_element *rl2; + + rl2 = kvzalloc(rlsize + PAGE_SIZE, GFP_NOFS); + if (unlikely(!rl2)) { + kvfree(rl); + return ERR_PTR(-ENOMEM); + } + memcpy(rl2, rl, rlsize); + kvfree(rl); + rl = rl2; + rlsize += PAGE_SIZE; + } + /* Enter the current vcn into the current runlist element. */ + rl[rlpos].vcn = vcn; + /* + * Get the change in vcn, i.e. the run length in clusters. + * Doing it this way ensures that we signextend negative values. + * A negative run length doesn't make any sense, but hey, I + * didn't make up the NTFS specs and Windows NT4 treats the run + * length as a signed value so that's how it is... + */ + b = *buf & 0xf; + if (b) { + if (unlikely(buf + b >= attr_end)) + goto io_error; + for (deltaxcn = (s8)buf[b--]; b; b--) + deltaxcn = (deltaxcn << 8) + buf[b]; + } else { /* The length entry is compulsory. */ + ntfs_error(vol->sb, "Missing length entry in mapping pairs array."); + deltaxcn = (s64)-1; + } + /* + * Assume a negative length to indicate data corruption and + * hence clean-up and return NULL. + */ + if (unlikely(deltaxcn < 0)) { + ntfs_error(vol->sb, "Invalid length in mapping pairs array."); + goto err_out; + } + /* + * Enter the current run length into the current runlist + * element. + */ + rl[rlpos].length = deltaxcn; + /* + * Increment the current vcn by the current run length. + * Guard against s64 overflow from a crafted mapping + * pairs array to preserve the monotonically-increasing + * vcn invariant. + */ + if (unlikely(check_add_overflow(vcn, deltaxcn, &vcn))) { + ntfs_error(vol->sb, "VCN overflow in mapping pairs array."); + goto err_out; + } + + /* + * There might be no lcn change at all, as is the case for + * sparse clusters on NTFS 3.0+, in which case we set the lcn + * to LCN_HOLE. + */ + if (!(*buf & 0xf0)) + rl[rlpos].lcn = LCN_HOLE; + else { + /* Get the lcn change which really can be negative. */ + u8 b2 = *buf & 0xf; + + b = b2 + ((*buf >> 4) & 0xf); + if (buf + b >= attr_end) + goto io_error; + for (deltaxcn = (s8)buf[b--]; b > b2; b--) + deltaxcn = (deltaxcn << 8) + buf[b]; + /* Change the current lcn to its new value. */ + if (unlikely(check_add_overflow(lcn, deltaxcn, &lcn))) { + ntfs_error(vol->sb, + "LCN overflow in mapping pairs array."); + goto err_out; + } +#ifdef DEBUG + /* + * On NTFS 1.2-, apparently can have lcn == -1 to + * indicate a hole. But we haven't verified ourselves + * whether it is really the lcn or the deltaxcn that is + * -1. So if either is found give us a message so we + * can investigate it further! + */ + if (vol->major_ver < 3) { + if (unlikely(deltaxcn == -1)) + ntfs_error(vol->sb, "lcn delta == -1"); + if (unlikely(lcn == -1)) + ntfs_error(vol->sb, "lcn == -1"); + } +#endif + /* Check lcn is not below -1. */ + if (unlikely(lcn < -1)) { + ntfs_error(vol->sb, "Invalid s64 < -1 in mapping pairs array."); + goto err_out; + } + + /* chkdsk accepts zero-sized runs only for holes */ + if ((lcn != -1) && !rl[rlpos].length) { + ntfs_error(vol->sb, + "Invalid zero-sized data run(lcn : %lld).\n", + lcn); + goto err_out; + } + + /* Enter the current lcn into the runlist element. */ + rl[rlpos].lcn = lcn; + } + /* Get to the next runlist element, skipping zero-sized holes */ + if (rl[rlpos].length) + rlpos++; + /* Increment the buffer position to the next mapping pair. */ + buf += (*buf & 0xf) + ((*buf >> 4) & 0xf) + 1; + } + if (unlikely(buf >= attr_end)) + goto io_error; + /* + * If there is a highest_vcn specified, it must be equal to the final + * vcn in the runlist - 1, or something has gone badly wrong. + */ + deltaxcn = le64_to_cpu(attr->data.non_resident.highest_vcn); + if (unlikely(deltaxcn && vcn - 1 != deltaxcn)) { +mpa_err: + ntfs_error(vol->sb, "Corrupt mapping pairs array in non-resident attribute."); + goto err_out; + } + /* Setup not mapped runlist element if this is the base extent. */ + if (!attr->data.non_resident.lowest_vcn) { + s64 max_cluster; + + max_cluster = ((le64_to_cpu(attr->data.non_resident.allocated_size) + + vol->cluster_size - 1) >> + vol->cluster_size_bits) - 1; + /* + * A highest_vcn of zero means this is a single extent + * attribute so simply terminate the runlist with LCN_ENOENT). + */ + if (deltaxcn) { + /* + * If there is a difference between the highest_vcn and + * the highest cluster, the runlist is either corrupt + * or, more likely, there are more extents following + * this one. + */ + if (deltaxcn < max_cluster) { + ntfs_debug("More extents to follow; deltaxcn = 0x%llx, max_cluster = 0x%llx", + deltaxcn, max_cluster); + rl[rlpos].vcn = vcn; + vcn += rl[rlpos].length = max_cluster - + deltaxcn; + rl[rlpos].lcn = LCN_RL_NOT_MAPPED; + rlpos++; + } else if (unlikely(deltaxcn > max_cluster)) { + ntfs_error(vol->sb, + "Corrupt attribute. deltaxcn = 0x%llx, max_cluster = 0x%llx", + deltaxcn, max_cluster); + goto mpa_err; + } + } + rl[rlpos].lcn = LCN_ENOENT; + } else /* Not the base extent. There may be more extents to follow. */ + rl[rlpos].lcn = LCN_RL_NOT_MAPPED; + + /* Setup terminating runlist element. */ + rl[rlpos].vcn = vcn; + rl[rlpos].length = (s64)0; + /* If no existing runlist was specified, we are done. */ + if (!old_runlist || !old_runlist->rl) { + *new_rl_count = rlpos + 1; + ntfs_debug("Mapping pairs array successfully decompressed:"); + ntfs_debug_dump_runlist(rl); + return rl; + } + /* Now combine the new and old runlists checking for overlaps. */ + new_rl = ntfs_runlists_merge(old_runlist, rl, rlpos + 1, new_rl_count); + if (!IS_ERR(new_rl)) + return new_rl; + kvfree(rl); + ntfs_error(vol->sb, "Failed to merge runlists."); + return new_rl; +io_error: + ntfs_error(vol->sb, "Corrupt attribute."); +err_out: + kvfree(rl); + return ERR_PTR(-EIO); +} + +/* + * ntfs_rl_vcn_to_lcn - convert a vcn into a lcn given a runlist + * @rl: runlist to use for conversion + * @vcn: vcn to convert + * + * Convert the virtual cluster number @vcn of an attribute into a logical + * cluster number (lcn) of a device using the runlist @rl to map vcns to their + * corresponding lcns. + * + * It is up to the caller to serialize access to the runlist @rl. + * + * Since lcns must be >= 0, we use negative return codes with special meaning: + * + * Return code Meaning / Description + * ================================================== + * LCN_HOLE Hole / not allocated on disk. + * LCN_RL_NOT_MAPPED This is part of the runlist which has not been + * inserted into the runlist yet. + * LCN_ENOENT There is no such vcn in the attribute. + * + * Locking: - The caller must have locked the runlist (for reading or writing). + * - This function does not touch the lock, nor does it modify the + * runlist. + */ +s64 ntfs_rl_vcn_to_lcn(const struct runlist_element *rl, const s64 vcn) +{ + int i; + + /* + * If rl is NULL, assume that we have found an unmapped runlist. The + * caller can then attempt to map it and fail appropriately if + * necessary. + */ + if (unlikely(!rl)) + return LCN_RL_NOT_MAPPED; + + /* Catch out of lower bounds vcn. */ + if (unlikely(vcn < rl[0].vcn)) + return LCN_ENOENT; + + for (i = 0; likely(rl[i].length); i++) { + if (vcn < rl[i+1].vcn) { + if (likely(rl[i].lcn >= 0)) + return rl[i].lcn + (vcn - rl[i].vcn); + return rl[i].lcn; + } + } + /* + * The terminator element is setup to the correct value, i.e. one of + * LCN_HOLE, LCN_RL_NOT_MAPPED, or LCN_ENOENT. + */ + if (likely(rl[i].lcn < 0)) + return rl[i].lcn; + /* Just in case... We could replace this with BUG() some day. */ + return LCN_ENOENT; +} + +/* + * ntfs_rl_find_vcn_nolock - find a vcn in a runlist + * @rl: runlist to search + * @vcn: vcn to find + * + * Find the virtual cluster number @vcn in the runlist @rl and return the + * address of the runlist element containing the @vcn on success. + * + * Return NULL if @rl is NULL or @vcn is in an unmapped part/out of bounds of + * the runlist. + * + * Locking: The runlist must be locked on entry. + */ +struct runlist_element *ntfs_rl_find_vcn_nolock(struct runlist_element *rl, const s64 vcn) +{ + if (unlikely(!rl || vcn < rl[0].vcn)) + return NULL; + while (likely(rl->length)) { + if (unlikely(vcn < rl[1].vcn)) { + if (likely(rl->lcn >= LCN_HOLE)) + return rl; + return NULL; + } + rl++; + } + if (likely(rl->lcn == LCN_ENOENT)) + return rl; + return NULL; +} + +/* + * ntfs_get_nr_significant_bytes - get number of bytes needed to store a number + * @n: number for which to get the number of bytes for + * + * Return the number of bytes required to store @n unambiguously as + * a signed number. + * + * This is used in the context of the mapping pairs array to determine how + * many bytes will be needed in the array to store a given logical cluster + * number (lcn) or a specific run length. + * + * Return the number of bytes written. This function cannot fail. + */ +static inline int ntfs_get_nr_significant_bytes(const s64 n) +{ + s64 l = n; + int i; + s8 j; + + i = 0; + do { + l >>= 8; + i++; + } while (l != 0 && l != -1); + j = (n >> 8 * (i - 1)) & 0xff; + /* If the sign bit is wrong, we need an extra byte. */ + if ((n < 0 && j >= 0) || (n > 0 && j < 0)) + i++; + return i; +} + +/* + * ntfs_get_size_for_mapping_pairs - get bytes needed for mapping pairs array + * @vol: ntfs volume + * @rl: runlist to calculate the mapping pairs array size for + * @first_vcn: first vcn which to include in the mapping pairs array + * @last_vcn: last vcn which to include in the mapping pairs array + * @max_mp_size: maximum size to return (0 or less means unlimited) + * + * Walk the locked runlist @rl and calculate the size in bytes of the mapping + * pairs array corresponding to the runlist @rl, starting at vcn @first_vcn and + * finishing with vcn @last_vcn. + * + * A @last_vcn of -1 means end of runlist and in that case the size of the + * mapping pairs array corresponding to the runlist starting at vcn @first_vcn + * and finishing at the end of the runlist is determined. + * + * This for example allows us to allocate a buffer of the right size when + * building the mapping pairs array. + * + * If @rl is NULL, just return 1 (for the single terminator byte). + * + * Return the calculated size in bytes on success. On error, return -errno. + */ +int ntfs_get_size_for_mapping_pairs(const struct ntfs_volume *vol, + const struct runlist_element *rl, const s64 first_vcn, + const s64 last_vcn, int max_mp_size) +{ + s64 prev_lcn; + int rls; + bool the_end = false; + + if (first_vcn < 0 || last_vcn < -1) + return -EINVAL; + + if (last_vcn >= 0 && first_vcn > last_vcn) + return -EINVAL; + + if (!rl) { + WARN_ON(first_vcn); + WARN_ON(last_vcn > 0); + return 1; + } + if (max_mp_size <= 0) + max_mp_size = INT_MAX; + /* Skip to runlist element containing @first_vcn. */ + while (rl->length && first_vcn >= rl[1].vcn) + rl++; + if (unlikely((!rl->length && first_vcn > rl->vcn) || + first_vcn < rl->vcn)) + return -EINVAL; + prev_lcn = 0; + /* Always need the termining zero byte. */ + rls = 1; + /* Do the first partial run if present. */ + if (first_vcn > rl->vcn) { + s64 delta, length = rl->length; + + /* We know rl->length != 0 already. */ + if (unlikely(length < 0 || rl->lcn < LCN_HOLE)) + goto err_out; + /* + * If @stop_vcn is given and finishes inside this run, cap the + * run length. + */ + if (unlikely(last_vcn >= 0 && rl[1].vcn > last_vcn)) { + s64 s1 = last_vcn + 1; + + if (unlikely(rl[1].vcn > s1)) + length = s1 - rl->vcn; + the_end = true; + } + delta = first_vcn - rl->vcn; + /* Header byte + length. */ + rls += 1 + ntfs_get_nr_significant_bytes(length - delta); + /* + * If the logical cluster number (lcn) denotes a hole and we + * are on NTFS 3.0+, we don't store it at all, i.e. we need + * zero space. On earlier NTFS versions we just store the lcn. + * Note: this assumes that on NTFS 1.2-, holes are stored with + * an lcn of -1 and not a delta_lcn of -1 (unless both are -1). + */ + if (likely(rl->lcn >= 0 || vol->major_ver < 3)) { + prev_lcn = rl->lcn; + if (likely(rl->lcn >= 0)) + prev_lcn += delta; + /* Change in lcn. */ + rls += ntfs_get_nr_significant_bytes(prev_lcn); + } + /* Go to next runlist element. */ + rl++; + } + /* Do the full runs. */ + for (; rl->length && !the_end; rl++) { + s64 length = rl->length; + + if (unlikely(length < 0 || rl->lcn < LCN_HOLE)) + goto err_out; + /* + * If @stop_vcn is given and finishes inside this run, cap the + * run length. + */ + if (unlikely(last_vcn >= 0 && rl[1].vcn > last_vcn)) { + s64 s1 = last_vcn + 1; + + if (unlikely(rl[1].vcn > s1)) + length = s1 - rl->vcn; + the_end = true; + } + /* Header byte + length. */ + rls += 1 + ntfs_get_nr_significant_bytes(length); + /* + * If the logical cluster number (lcn) denotes a hole and we + * are on NTFS 3.0+, we don't store it at all, i.e. we need + * zero space. On earlier NTFS versions we just store the lcn. + * Note: this assumes that on NTFS 1.2-, holes are stored with + * an lcn of -1 and not a delta_lcn of -1 (unless both are -1). + */ + if (likely(rl->lcn >= 0 || vol->major_ver < 3)) { + /* Change in lcn. */ + rls += ntfs_get_nr_significant_bytes(rl->lcn - + prev_lcn); + prev_lcn = rl->lcn; + } + + if (rls > max_mp_size) + break; + } + return rls; +err_out: + if (rl->lcn == LCN_RL_NOT_MAPPED) + rls = -EINVAL; + else + rls = -EIO; + return rls; +} + +/* + * ntfs_write_significant_bytes - write the significant bytes of a number + * @dst: destination buffer to write to + * @dst_max: pointer to last byte of destination buffer for bounds checking + * @n: number whose significant bytes to write + * + * Store in @dst, the minimum bytes of the number @n which are required to + * identify @n unambiguously as a signed number, taking care not to exceed + * @dest_max, the maximum position within @dst to which we are allowed to + * write. + * + * This is used when building the mapping pairs array of a runlist to compress + * a given logical cluster number (lcn) or a specific run length to the minimum + * size possible. + * + * Return the number of bytes written on success. On error, i.e. the + * destination buffer @dst is too small, return -ENOSPC. + */ +static inline int ntfs_write_significant_bytes(s8 *dst, const s8 *dst_max, + const s64 n) +{ + s64 l = n; + int i; + s8 j; + + i = 0; + do { + if (unlikely(dst > dst_max)) + goto err_out; + *dst++ = l & 0xffll; + l >>= 8; + i++; + } while (l != 0 && l != -1); + j = (n >> 8 * (i - 1)) & 0xff; + /* If the sign bit is wrong, we need an extra byte. */ + if (n < 0 && j >= 0) { + if (unlikely(dst > dst_max)) + goto err_out; + i++; + *dst = (s8)-1; + } else if (n > 0 && j < 0) { + if (unlikely(dst > dst_max)) + goto err_out; + i++; + *dst = (s8)0; + } + return i; +err_out: + return -ENOSPC; +} + +/* + * ntfs_mapping_pairs_build - build the mapping pairs array from a runlist + * @vol: ntfs volume + * @dst: destination buffer to build mapping pairs array into + * @dst_len: size of @dst in bytes + * @rl: runlist to build the mapping pairs array from + * @first_vcn: first vcn which to include in the mapping pairs array + * @last_vcn: last vcn which to include in the mapping pairs array + * @stop_vcn: on return, set to the first vcn outside the destination buffer + * @stop_rl: on return, set to the runlist element where encoding stopped + * @de_cluster_count: on return, set to the number of clusters encoded + * + * Create the mapping pairs array from the locked runlist @rl, starting at vcn + * @first_vcn and finishing with vcn @last_vcn and save the array in @dst. + * @dst_len is the size of @dst in bytes and it should be at least equal to the + * value obtained by calling ntfs_get_size_for_mapping_pairs(). + * + * A @last_vcn of -1 means end of runlist and in that case the mapping pairs + * array corresponding to the runlist starting at vcn @first_vcn and finishing + * at the end of the runlist is created. + * + * If @rl is NULL, just write a single terminator byte to @dst. + * + * On success or -ENOSPC error, if @stop_vcn is not NULL, *@stop_vcn is set to + * the first vcn outside the destination buffer. Note that on error, @dst has + * been filled with all the mapping pairs that will fit, thus it can be treated + * as partial success, in that a new attribute extent needs to be created or + * the next extent has to be used and the mapping pairs build has to be + * continued with @first_vcn set to *@stop_vcn. + * + * Return 0 on success and -errno on error. The following error codes are + * defined: + * -EINVAL - Run list contains unmapped elements. Make sure to only pass + * fully mapped runlists to this function. + * -EIO - The runlist is corrupt. + * -ENOSPC - The destination buffer is too small. + * + * Locking: @rl must be locked on entry (either for reading or writing), it + * remains locked throughout, and is left locked upon return. + */ +int ntfs_mapping_pairs_build(const struct ntfs_volume *vol, s8 *dst, + const int dst_len, const struct runlist_element *rl, + const s64 first_vcn, const s64 last_vcn, s64 *const stop_vcn, + struct runlist_element **stop_rl, unsigned int *de_cluster_count) +{ + s64 prev_lcn; + s8 *dst_max, *dst_next; + int err = -ENOSPC; + bool the_end = false; + s8 len_len, lcn_len; + unsigned int de_cnt = 0; + + if (first_vcn < 0 || last_vcn < -1 || dst_len < 1) + return -EINVAL; + if (last_vcn >= 0 && first_vcn > last_vcn) + return -EINVAL; + + if (!rl) { + WARN_ON(first_vcn || last_vcn > 0); + if (stop_vcn) + *stop_vcn = 0; + /* Terminator byte. */ + *dst = 0; + return 0; + } + /* Skip to runlist element containing @first_vcn. */ + while (rl->length && first_vcn >= rl[1].vcn) + rl++; + if (unlikely((!rl->length && first_vcn > rl->vcn) || + first_vcn < rl->vcn)) + return -EINVAL; + /* + * @dst_max is used for bounds checking in + * ntfs_write_significant_bytes(). + */ + dst_max = dst + dst_len - 1; + prev_lcn = 0; + /* Do the first partial run if present. */ + if (first_vcn > rl->vcn) { + s64 delta, length = rl->length; + + /* We know rl->length != 0 already. */ + if (unlikely(length < 0 || rl->lcn < LCN_HOLE)) + goto err_out; + /* + * If @stop_vcn is given and finishes inside this run, cap the + * run length. + */ + if (unlikely(last_vcn >= 0 && rl[1].vcn > last_vcn)) { + s64 s1 = last_vcn + 1; + + if (unlikely(rl[1].vcn > s1)) + length = s1 - rl->vcn; + the_end = true; + } + delta = first_vcn - rl->vcn; + /* Write length. */ + len_len = ntfs_write_significant_bytes(dst + 1, dst_max, + length - delta); + if (unlikely(len_len < 0)) + goto size_err; + /* + * If the logical cluster number (lcn) denotes a hole and we + * are on NTFS 3.0+, we don't store it at all, i.e. we need + * zero space. On earlier NTFS versions we just write the lcn + * change. + */ + if (likely(rl->lcn >= 0 || vol->major_ver < 3)) { + prev_lcn = rl->lcn; + if (likely(rl->lcn >= 0)) + prev_lcn += delta; + /* Write change in lcn. */ + lcn_len = ntfs_write_significant_bytes(dst + 1 + + len_len, dst_max, prev_lcn); + if (unlikely(lcn_len < 0)) + goto size_err; + } else + lcn_len = 0; + dst_next = dst + len_len + lcn_len + 1; + if (unlikely(dst_next > dst_max)) + goto size_err; + /* Update header byte. */ + *dst = lcn_len << 4 | len_len; + /* Position at next mapping pairs array element. */ + dst = dst_next; + /* Go to next runlist element. */ + rl++; + } + /* Do the full runs. */ + for (; rl->length && !the_end; rl++) { + s64 length = rl->length; + + if (unlikely(length < 0 || rl->lcn < LCN_HOLE)) + goto err_out; + /* + * If @stop_vcn is given and finishes inside this run, cap the + * run length. + */ + if (unlikely(last_vcn >= 0 && rl[1].vcn > last_vcn)) { + s64 s1 = last_vcn + 1; + + if (unlikely(rl[1].vcn > s1)) + length = s1 - rl->vcn; + the_end = true; + } + /* Write length. */ + len_len = ntfs_write_significant_bytes(dst + 1, dst_max, + length); + if (unlikely(len_len < 0)) + goto size_err; + /* + * If the logical cluster number (lcn) denotes a hole and we + * are on NTFS 3.0+, we don't store it at all, i.e. we need + * zero space. On earlier NTFS versions we just write the lcn + * change. + */ + if (likely(rl->lcn >= 0 || vol->major_ver < 3)) { + /* Write change in lcn. */ + lcn_len = ntfs_write_significant_bytes(dst + 1 + + len_len, dst_max, rl->lcn - prev_lcn); + if (unlikely(lcn_len < 0)) + goto size_err; + prev_lcn = rl->lcn; + } else { + if (rl->lcn == LCN_DELALLOC) + de_cnt += rl->length; + lcn_len = 0; + } + dst_next = dst + len_len + lcn_len + 1; + if (unlikely(dst_next > dst_max)) + goto size_err; + /* Update header byte. */ + *dst = lcn_len << 4 | len_len; + /* Position at next mapping pairs array element. */ + dst = dst_next; + } + /* Success. */ + if (de_cluster_count) + *de_cluster_count = de_cnt; + err = 0; +size_err: + /* Set stop vcn. */ + if (stop_vcn) + *stop_vcn = rl->vcn; + if (stop_rl) + *stop_rl = (struct runlist_element *)rl; + /* Add terminator byte. */ + *dst = 0; + return err; +err_out: + if (rl->lcn == LCN_RL_NOT_MAPPED) + err = -EINVAL; + else + err = -EIO; + return err; +} + +/* + * ntfs_rl_truncate_nolock - truncate a runlist starting at a specified vcn + * @vol: ntfs volume (needed for error output) + * @runlist: runlist to truncate + * @new_length: the new length of the runlist in VCNs + * + * Truncate the runlist described by @runlist as well as the memory buffer + * holding the runlist elements to a length of @new_length VCNs. + * + * If @new_length lies within the runlist, the runlist elements with VCNs of + * @new_length and above are discarded. As a special case if @new_length is + * zero, the runlist is discarded and set to NULL. + * + * If @new_length lies beyond the runlist, a sparse runlist element is added to + * the end of the runlist @runlist or if the last runlist element is a sparse + * one already, this is extended. + * + * Note, no checking is done for unmapped runlist elements. It is assumed that + * the caller has mapped any elements that need to be mapped already. + * + * Return 0 on success and -errno on error. + * + * Locking: The caller must hold @runlist->lock for writing. + */ +int ntfs_rl_truncate_nolock(const struct ntfs_volume *vol, struct runlist *const runlist, + const s64 new_length) +{ + struct runlist_element *rl; + int old_size; + + ntfs_debug("Entering for new_length 0x%llx.", (long long)new_length); + + if (!runlist || new_length < 0) + return -EINVAL; + + rl = runlist->rl; + if (new_length < rl->vcn) + return -EINVAL; + + /* Find @new_length in the runlist. */ + while (likely(rl->length && new_length >= rl[1].vcn)) + rl++; + /* + * If not at the end of the runlist we need to shrink it. + * If at the end of the runlist we need to expand it. + */ + if (rl->length) { + struct runlist_element *trl; + bool is_end; + + ntfs_debug("Shrinking runlist."); + /* Determine the runlist size. */ + trl = rl + 1; + while (likely(trl->length)) + trl++; + old_size = trl - runlist->rl + 1; + /* Truncate the run. */ + rl->length = new_length - rl->vcn; + /* + * If a run was partially truncated, make the following runlist + * element a terminator. + */ + is_end = false; + if (rl->length) { + rl++; + if (!rl->length) + is_end = true; + rl->vcn = new_length; + rl->length = 0; + } + rl->lcn = LCN_ENOENT; + runlist->count = rl - runlist->rl + 1; + /* Reallocate memory if necessary. */ + if (!is_end) { + int new_size = rl - runlist->rl + 1; + + rl = ntfs_rl_realloc(runlist->rl, old_size, new_size); + if (IS_ERR(rl)) + ntfs_warning(vol->sb, + "Failed to shrink runlist buffer. This just wastes a bit of memory temporarily so we ignore it and return success."); + else + runlist->rl = rl; + } + } else if (likely(/* !rl->length && */ new_length > rl->vcn)) { + ntfs_debug("Expanding runlist."); + /* + * If there is a previous runlist element and it is a sparse + * one, extend it. Otherwise need to add a new, sparse runlist + * element. + */ + if ((rl > runlist->rl) && ((rl - 1)->lcn == LCN_HOLE)) + (rl - 1)->length = new_length - (rl - 1)->vcn; + else { + /* Determine the runlist size. */ + old_size = rl - runlist->rl + 1; + /* Reallocate memory if necessary. */ + rl = ntfs_rl_realloc(runlist->rl, old_size, + old_size + 1); + if (IS_ERR(rl)) { + ntfs_error(vol->sb, "Failed to expand runlist buffer, aborting."); + return PTR_ERR(rl); + } + runlist->rl = rl; + /* + * Set @rl to the same runlist element in the new + * runlist as before in the old runlist. + */ + rl += old_size - 1; + /* Add a new, sparse runlist element. */ + rl->lcn = LCN_HOLE; + rl->length = new_length - rl->vcn; + /* Add a new terminator runlist element. */ + rl++; + rl->length = 0; + runlist->count = old_size + 1; + } + rl->vcn = new_length; + rl->lcn = LCN_ENOENT; + } else /* if (unlikely(!rl->length && new_length == rl->vcn)) */ { + /* Runlist already has same size as requested. */ + rl->lcn = LCN_ENOENT; + } + ntfs_debug("Done."); + return 0; +} + +/* + * ntfs_rl_sparse - check whether runlist have sparse regions or not. + * @rl: runlist to check + * + * Return 1 if have, 0 if not, -errno on error. + */ +int ntfs_rl_sparse(struct runlist_element *rl) +{ + struct runlist_element *rlc; + + if (!rl) + return -EINVAL; + + for (rlc = rl; rlc->length; rlc++) + if (rlc->lcn < 0) { + if (rlc->lcn != LCN_HOLE && rlc->lcn != LCN_DELALLOC) { + pr_err("%s: bad runlist\n", __func__); + return -EINVAL; + } + return 1; + } + return 0; +} + +/* + * ntfs_rl_get_compressed_size - calculate length of non sparse regions + * @vol: ntfs volume (need for cluster size) + * @rl: runlist to calculate for + * + * Return compressed size or -errno on error. + */ +s64 ntfs_rl_get_compressed_size(struct ntfs_volume *vol, struct runlist_element *rl) +{ + struct runlist_element *rlc; + s64 ret = 0; + + if (!rl) + return -EINVAL; + + for (rlc = rl; rlc->length; rlc++) { + if (rlc->lcn < 0) { + if (rlc->lcn != LCN_HOLE && rlc->lcn != LCN_DELALLOC) { + ntfs_error(vol->sb, "%s: bad runlist, rlc->lcn : %lld", + __func__, rlc->lcn); + return -EINVAL; + } + } else + ret += rlc->length; + } + return NTFS_CLU_TO_B(vol, ret); +} + +static inline bool ntfs_rle_lcn_contiguous(struct runlist_element *left_rle, + struct runlist_element *right_rle) +{ + if (left_rle->lcn > LCN_HOLE && + left_rle->lcn + left_rle->length == right_rle->lcn) + return true; + else if (left_rle->lcn == LCN_HOLE && right_rle->lcn == LCN_HOLE) + return true; + else + return false; +} + +static inline bool ntfs_rle_contain(struct runlist_element *rle, s64 vcn) +{ + if (rle->length > 0 && + vcn >= rle->vcn && vcn < rle->vcn + rle->length) + return true; + else + return false; +} + +struct runlist_element *ntfs_rl_insert_range(struct runlist_element *dst_rl, int dst_cnt, + struct runlist_element *src_rl, int src_cnt, + size_t *new_rl_cnt) +{ + struct runlist_element *i_rl, *new_rl, *src_rl_origin = src_rl; + struct runlist_element dst_rl_split; + s64 start_vcn; + int new_1st_cnt, new_2nd_cnt, new_3rd_cnt, new_cnt; + + if (!dst_rl || !src_rl || !new_rl_cnt) + return ERR_PTR(-EINVAL); + if (dst_cnt <= 0 || src_cnt <= 0) + return ERR_PTR(-EINVAL); + if (!(dst_rl[dst_cnt - 1].lcn == LCN_ENOENT && + dst_rl[dst_cnt - 1].length == 0) || + src_rl[src_cnt - 1].lcn < LCN_HOLE) + return ERR_PTR(-EINVAL); + + start_vcn = src_rl[0].vcn; + + i_rl = ntfs_rl_find_vcn_nolock(dst_rl, start_vcn); + if (!i_rl || + (i_rl->lcn == LCN_ENOENT && i_rl->vcn != start_vcn) || + (i_rl->lcn != LCN_ENOENT && !ntfs_rle_contain(i_rl, start_vcn))) + return ERR_PTR(-EINVAL); + + new_1st_cnt = (int)(i_rl - dst_rl); + if (new_1st_cnt > dst_cnt) + return ERR_PTR(-EINVAL); + new_3rd_cnt = dst_cnt - new_1st_cnt; + if (new_3rd_cnt < 1) + return ERR_PTR(-EINVAL); + + if (i_rl[0].vcn != start_vcn) { + if (i_rl[0].lcn == LCN_HOLE && src_rl[0].lcn == LCN_HOLE) + goto merge_src_rle; + + /* split @i_rl[0] and create @dst_rl_split */ + dst_rl_split.vcn = i_rl[0].vcn; + dst_rl_split.length = start_vcn - i_rl[0].vcn; + dst_rl_split.lcn = i_rl[0].lcn; + + i_rl[0].vcn = start_vcn; + i_rl[0].length -= dst_rl_split.length; + i_rl[0].lcn += dst_rl_split.length; + } else { + struct runlist_element *dst_rle, *src_rle; +merge_src_rle: + + /* not split @i_rl[0] */ + dst_rl_split.lcn = LCN_ENOENT; + + /* merge @src_rl's first run and @i_rl[0]'s left run if possible */ + dst_rle = &dst_rl[new_1st_cnt - 1]; + src_rle = &src_rl[0]; + if (new_1st_cnt > 0 && ntfs_rle_lcn_contiguous(dst_rle, src_rle)) { + WARN_ON(dst_rle->vcn + dst_rle->length != src_rle->vcn); + dst_rle->length += src_rle->length; + src_rl++; + src_cnt--; + } else { + /* merge @src_rl's last run and @i_rl[0]'s right if possible */ + dst_rle = &dst_rl[new_1st_cnt]; + src_rle = &src_rl[src_cnt - 1]; + + if (ntfs_rle_lcn_contiguous(dst_rle, src_rle)) { + dst_rle->length += src_rle->length; + src_cnt--; + } + } + } + + new_2nd_cnt = src_cnt; + new_cnt = new_1st_cnt + new_2nd_cnt + new_3rd_cnt; + new_cnt += dst_rl_split.lcn >= LCN_HOLE ? 1 : 0; + new_rl = kvcalloc(new_cnt, sizeof(*new_rl), GFP_NOFS); + if (!new_rl) + return ERR_PTR(-ENOMEM); + + /* Copy the @dst_rl's first half to @new_rl */ + ntfs_rl_mc(new_rl, 0, dst_rl, 0, new_1st_cnt); + if (dst_rl_split.lcn >= LCN_HOLE) { + ntfs_rl_mc(new_rl, new_1st_cnt, &dst_rl_split, 0, 1); + new_1st_cnt++; + } + /* Copy the @src_rl to @new_rl */ + ntfs_rl_mc(new_rl, new_1st_cnt, src_rl, 0, new_2nd_cnt); + /* Copy the @dst_rl's second half to @new_rl */ + if (new_3rd_cnt >= 1) { + struct runlist_element *rl, *rl_3rd; + int dst_1st_cnt = dst_rl_split.lcn >= LCN_HOLE ? + new_1st_cnt - 1 : new_1st_cnt; + + ntfs_rl_mc(new_rl, new_1st_cnt + new_2nd_cnt, + dst_rl, dst_1st_cnt, new_3rd_cnt); + /* Update vcn of the @dst_rl's second half runs to reflect + * appended @src_rl. + */ + if (new_1st_cnt + new_2nd_cnt == 0) { + rl_3rd = &new_rl[new_1st_cnt + new_2nd_cnt + 1]; + rl = &new_rl[new_1st_cnt + new_2nd_cnt]; + } else { + rl_3rd = &new_rl[new_1st_cnt + new_2nd_cnt]; + rl = &new_rl[new_1st_cnt + new_2nd_cnt - 1]; + } + do { + rl_3rd->vcn = rl->vcn + rl->length; + if (rl_3rd->length <= 0) + break; + rl = rl_3rd; + rl_3rd++; + } while (1); + } + *new_rl_cnt = new_1st_cnt + new_2nd_cnt + new_3rd_cnt; + + kvfree(dst_rl); + kvfree(src_rl_origin); + return new_rl; +} + +struct runlist_element *ntfs_rl_punch_hole(struct runlist_element *dst_rl, int dst_cnt, + s64 start_vcn, s64 len, + struct runlist_element **punch_rl, + size_t *new_rl_cnt) +{ + struct runlist_element *s_rl, *e_rl, *new_rl, *dst_3rd_rl, hole_rl[1]; + s64 end_vcn; + int new_1st_cnt, dst_3rd_cnt, new_cnt, punch_cnt, merge_cnt; + bool begin_split, end_split, one_split_3; + + if (dst_cnt < 2 || + !(dst_rl[dst_cnt - 1].lcn == LCN_ENOENT && + dst_rl[dst_cnt - 1].length == 0)) + return ERR_PTR(-EINVAL); + + end_vcn = min(start_vcn + len - 1, + dst_rl[dst_cnt - 2].vcn + dst_rl[dst_cnt - 2].length - 1); + + s_rl = ntfs_rl_find_vcn_nolock(dst_rl, start_vcn); + if (!s_rl || + s_rl->lcn <= LCN_ENOENT || + !ntfs_rle_contain(s_rl, start_vcn)) + return ERR_PTR(-EINVAL); + + begin_split = s_rl->vcn != start_vcn; + + e_rl = ntfs_rl_find_vcn_nolock(dst_rl, end_vcn); + if (!e_rl || + e_rl->lcn <= LCN_ENOENT || + !ntfs_rle_contain(e_rl, end_vcn)) + return ERR_PTR(-EINVAL); + + end_split = e_rl->vcn + e_rl->length - 1 != end_vcn; + + /* @s_rl has to be split into left, punched hole, and right */ + one_split_3 = e_rl == s_rl && begin_split && end_split; + + punch_cnt = (int)(e_rl - s_rl) + 1; + + *punch_rl = kvcalloc(punch_cnt + 1, sizeof(struct runlist_element), + GFP_NOFS); + if (!*punch_rl) + return ERR_PTR(-ENOMEM); + + new_cnt = dst_cnt - (int)(e_rl - s_rl + 1) + 3; + new_rl = kvcalloc(new_cnt, sizeof(struct runlist_element), GFP_NOFS); + if (!new_rl) { + kvfree(*punch_rl); + *punch_rl = NULL; + return ERR_PTR(-ENOMEM); + } + + new_1st_cnt = (int)(s_rl - dst_rl) + 1; + ntfs_rl_mc(*punch_rl, 0, dst_rl, new_1st_cnt - 1, punch_cnt); + + (*punch_rl)[punch_cnt].lcn = LCN_ENOENT; + (*punch_rl)[punch_cnt].length = 0; + + if (!begin_split) + new_1st_cnt--; + dst_3rd_rl = e_rl; + dst_3rd_cnt = (int)(&dst_rl[dst_cnt - 1] - e_rl) + 1; + if (!end_split) { + dst_3rd_rl++; + dst_3rd_cnt--; + } + + /* Copy the 1st part of @dst_rl into @new_rl */ + ntfs_rl_mc(new_rl, 0, dst_rl, 0, new_1st_cnt); + if (begin_split) { + /* the @e_rl has to be splited and copied into the last of @new_rl + * and the first of @punch_rl + */ + s64 first_cnt = start_vcn - dst_rl[new_1st_cnt - 1].vcn; + + if (new_1st_cnt) + new_rl[new_1st_cnt - 1].length = first_cnt; + + (*punch_rl)[0].vcn = start_vcn; + (*punch_rl)[0].length -= first_cnt; + if ((*punch_rl)[0].lcn > LCN_HOLE) + (*punch_rl)[0].lcn += first_cnt; + } + + /* Copy a hole into @new_rl */ + hole_rl[0].vcn = start_vcn; + hole_rl[0].length = (s64)len; + hole_rl[0].lcn = LCN_HOLE; + ntfs_rl_mc(new_rl, new_1st_cnt, hole_rl, 0, 1); + + /* Copy the 3rd part of @dst_rl into @new_rl */ + ntfs_rl_mc(new_rl, new_1st_cnt + 1, dst_3rd_rl, 0, dst_3rd_cnt); + if (end_split) { + /* the @e_rl has to be splited and copied into the first of + * @new_rl and the last of @punch_rl + */ + s64 first_cnt = end_vcn - dst_3rd_rl[0].vcn + 1; + + new_rl[new_1st_cnt + 1].vcn = end_vcn + 1; + new_rl[new_1st_cnt + 1].length -= first_cnt; + if (new_rl[new_1st_cnt + 1].lcn > LCN_HOLE) + new_rl[new_1st_cnt + 1].lcn += first_cnt; + + if (one_split_3) + (*punch_rl)[punch_cnt - 1].length -= + new_rl[new_1st_cnt + 1].length; + else + (*punch_rl)[punch_cnt - 1].length = first_cnt; + } + + /* Merge left and hole, or hole and right in @new_rl, if left or right + * consists of holes. + */ + merge_cnt = 0; + if (new_1st_cnt > 0 && new_rl[new_1st_cnt - 1].lcn == LCN_HOLE) { + /* Merge right and hole */ + s_rl = &new_rl[new_1st_cnt - 1]; + s_rl->length += s_rl[1].length; + merge_cnt = 1; + /* Merge left and right */ + if (new_1st_cnt + 1 < new_cnt && + new_rl[new_1st_cnt + 1].lcn == LCN_HOLE) { + s_rl->length += s_rl[2].length; + merge_cnt++; + } + } else if (new_1st_cnt + 1 < new_cnt && + new_rl[new_1st_cnt + 1].lcn == LCN_HOLE) { + /* Merge left and hole */ + s_rl = &new_rl[new_1st_cnt]; + s_rl->length += s_rl[1].length; + merge_cnt = 1; + } + if (merge_cnt) { + struct runlist_element *d_rl, *src_rl; + + d_rl = s_rl + 1; + src_rl = s_rl + 1 + merge_cnt; + ntfs_rl_mm(new_rl, (int)(d_rl - new_rl), (int)(src_rl - new_rl), + (int)(&new_rl[new_cnt - 1] - src_rl) + 1); + } + + (*punch_rl)[punch_cnt].vcn = (*punch_rl)[punch_cnt - 1].vcn + + (*punch_rl)[punch_cnt - 1].length; + + /* punch_cnt elements of dst are replaced with one hole */ + *new_rl_cnt = dst_cnt - (punch_cnt - (int)begin_split - (int)end_split) + + 1 - merge_cnt; + kvfree(dst_rl); + return new_rl; +} + +struct runlist_element *ntfs_rl_collapse_range(struct runlist_element *dst_rl, int dst_cnt, + s64 start_vcn, s64 len, + struct runlist_element **punch_rl, + size_t *new_rl_cnt) +{ + struct runlist_element *s_rl, *e_rl, *new_rl, *dst_3rd_rl; + s64 end_vcn; + int new_1st_cnt, dst_3rd_cnt, new_cnt, punch_cnt, merge_cnt, i; + bool begin_split, end_split, one_split_3; + + if (dst_cnt < 2 || + !(dst_rl[dst_cnt - 1].lcn == LCN_ENOENT && + dst_rl[dst_cnt - 1].length == 0)) + return ERR_PTR(-EINVAL); + + end_vcn = min(start_vcn + len - 1, + dst_rl[dst_cnt - 1].vcn - 1); + + s_rl = ntfs_rl_find_vcn_nolock(dst_rl, start_vcn); + if (!s_rl || + s_rl->lcn <= LCN_ENOENT || + !ntfs_rle_contain(s_rl, start_vcn)) + return ERR_PTR(-EINVAL); + + begin_split = s_rl->vcn != start_vcn; + + e_rl = ntfs_rl_find_vcn_nolock(dst_rl, end_vcn); + if (!e_rl || + e_rl->lcn <= LCN_ENOENT || + !ntfs_rle_contain(e_rl, end_vcn)) + return ERR_PTR(-EINVAL); + + end_split = e_rl->vcn + e_rl->length - 1 != end_vcn; + + /* @s_rl has to be split into left, collapsed, and right */ + one_split_3 = e_rl == s_rl && begin_split && end_split; + + punch_cnt = (int)(e_rl - s_rl) + 1; + *punch_rl = kvcalloc(punch_cnt + 1, sizeof(struct runlist_element), + GFP_NOFS); + if (!*punch_rl) + return ERR_PTR(-ENOMEM); + + new_cnt = dst_cnt - (int)(e_rl - s_rl + 1) + 3; + new_rl = kvcalloc(new_cnt, sizeof(struct runlist_element), GFP_NOFS); + if (!new_rl) { + kvfree(*punch_rl); + *punch_rl = NULL; + return ERR_PTR(-ENOMEM); + } + + new_1st_cnt = (int)(s_rl - dst_rl) + 1; + ntfs_rl_mc(*punch_rl, 0, dst_rl, new_1st_cnt - 1, punch_cnt); + (*punch_rl)[punch_cnt].lcn = LCN_ENOENT; + (*punch_rl)[punch_cnt].length = 0; + + if (!begin_split) + new_1st_cnt--; + dst_3rd_rl = e_rl; + dst_3rd_cnt = (int)(&dst_rl[dst_cnt - 1] - e_rl) + 1; + if (!end_split) { + dst_3rd_rl++; + dst_3rd_cnt--; + } + + /* Copy the 1st part of @dst_rl into @new_rl */ + ntfs_rl_mc(new_rl, 0, dst_rl, 0, new_1st_cnt); + if (begin_split) { + /* the @e_rl has to be splited and copied into the last of @new_rl + * and the first of @punch_rl + */ + s64 first_cnt = start_vcn - dst_rl[new_1st_cnt - 1].vcn; + + new_rl[new_1st_cnt - 1].length = first_cnt; + + (*punch_rl)[0].vcn = start_vcn; + (*punch_rl)[0].length -= first_cnt; + if ((*punch_rl)[0].lcn > LCN_HOLE) + (*punch_rl)[0].lcn += first_cnt; + } + + /* Copy the 3rd part of @dst_rl into @new_rl */ + ntfs_rl_mc(new_rl, new_1st_cnt, dst_3rd_rl, 0, dst_3rd_cnt); + if (end_split) { + /* the @e_rl has to be splited and copied into the first of + * @new_rl and the last of @punch_rl + */ + s64 first_cnt = end_vcn - dst_3rd_rl[0].vcn + 1; + + new_rl[new_1st_cnt].vcn = end_vcn + 1; + new_rl[new_1st_cnt].length -= first_cnt; + if (new_rl[new_1st_cnt].lcn > LCN_HOLE) + new_rl[new_1st_cnt].lcn += first_cnt; + + if (one_split_3) + (*punch_rl)[punch_cnt - 1].length -= + new_rl[new_1st_cnt].length; + else + (*punch_rl)[punch_cnt - 1].length = first_cnt; + } + + /* Adjust vcn */ + if (new_1st_cnt == 0) + new_rl[new_1st_cnt].vcn = 0; + for (i = new_1st_cnt == 0 ? 1 : new_1st_cnt; new_rl[i].length; i++) + new_rl[i].vcn = new_rl[i - 1].vcn + new_rl[i - 1].length; + new_rl[i].vcn = new_rl[i - 1].vcn + new_rl[i - 1].length; + + /* Merge left and hole, or hole and right in @new_rl, if left or right + * consists of holes. + */ + merge_cnt = 0; + if (new_1st_cnt > 0 && + ntfs_rle_lcn_contiguous(&new_rl[new_1st_cnt - 1], + &new_rl[new_1st_cnt])) { + /* Merge right and left. */ + s_rl = &new_rl[new_1st_cnt - 1]; + s_rl->length += s_rl[1].length; + merge_cnt = 1; + } + if (merge_cnt) { + struct runlist_element *d_rl, *src_rl; + + d_rl = s_rl + 1; + src_rl = s_rl + 1 + merge_cnt; + ntfs_rl_mm(new_rl, (int)(d_rl - new_rl), (int)(src_rl - new_rl), + (int)(&new_rl[new_cnt - 1] - src_rl) + 1); + } + + (*punch_rl)[punch_cnt].vcn = (*punch_rl)[punch_cnt - 1].vcn + + (*punch_rl)[punch_cnt - 1].length; + + /* punch_cnt elements of dst are extracted */ + *new_rl_cnt = dst_cnt - (punch_cnt - (int)begin_split - (int)end_split) - + merge_cnt; + + kvfree(dst_rl); + return new_rl; +} diff --git a/fs/ntfs/runlist.h b/fs/ntfs/runlist.h new file mode 100644 index 000000000..cc92e09fb --- /dev/null +++ b/fs/ntfs/runlist.h @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Defines for runlist handling in NTFS Linux kernel driver. + * + * Copyright (c) 2001-2005 Anton Altaparmakov + * Copyright (c) 2002 Richard Russon + * Copyright (c) 2025 LG Electronics Co., Ltd. + */ + +#ifndef _LINUX_NTFS_RUNLIST_H +#define _LINUX_NTFS_RUNLIST_H + +#include "volume.h" + +/* + * runlist_element - in memory vcn to lcn mapping array element + * @vcn: starting vcn of the current array element + * @lcn: starting lcn of the current array element + * @length: length in clusters of the current array element + * + * The last vcn (in fact the last vcn + 1) is reached when length == 0. + * + * When lcn == -1 this means that the count vcns starting at vcn are not + * physically allocated (i.e. this is a hole / data is sparse). + * + * In memory vcn to lcn mapping structure element. + * @vcn: vcn = Starting virtual cluster number. + * @lcn: lcn = Starting logical cluster number. + * @length: Run length in clusters. + */ +struct runlist_element { + s64 vcn; + s64 lcn; + s64 length; +}; + +/* + * runlist - in memory vcn to lcn mapping array including a read/write lock + * @rl: pointer to an array of runlist elements + * @lock: read/write spinlock for serializing access to @rl + * @rl_hint: hint/cache pointing to the last accessed runlist element + */ +struct runlist { + struct runlist_element *rl; + struct rw_semaphore lock; + size_t count; + int rl_hint; +}; + +static inline void ntfs_init_runlist(struct runlist *rl) +{ + rl->rl = NULL; + init_rwsem(&rl->lock); + rl->count = 0; + rl->rl_hint = -1; +} + +enum { + LCN_DELALLOC = -1, + LCN_HOLE = -2, + LCN_RL_NOT_MAPPED = -3, + LCN_ENOENT = -4, + LCN_ENOMEM = -5, + LCN_EIO = -6, + LCN_EINVAL = -7, +}; + +struct runlist_element *ntfs_runlists_merge(struct runlist *d_runlist, + struct runlist_element *srl, size_t s_rl_count, + size_t *new_rl_count); +struct runlist_element *ntfs_mapping_pairs_decompress(const struct ntfs_volume *vol, + const struct attr_record *attr, struct runlist *old_runlist, + size_t *new_rl_count); +s64 ntfs_rl_vcn_to_lcn(const struct runlist_element *rl, const s64 vcn); +struct runlist_element *ntfs_rl_find_vcn_nolock(struct runlist_element *rl, const s64 vcn); +int ntfs_get_size_for_mapping_pairs(const struct ntfs_volume *vol, + const struct runlist_element *rl, const s64 first_vcn, + const s64 last_vcn, int max_mp_size); +int ntfs_mapping_pairs_build(const struct ntfs_volume *vol, s8 *dst, + const int dst_len, const struct runlist_element *rl, + const s64 first_vcn, const s64 last_vcn, s64 *const stop_vcn, + struct runlist_element **stop_rl, unsigned int *de_cluster_count); +int ntfs_rl_truncate_nolock(const struct ntfs_volume *vol, + struct runlist *const runlist, const s64 new_length); +int ntfs_rl_sparse(struct runlist_element *rl); +s64 ntfs_rl_get_compressed_size(struct ntfs_volume *vol, struct runlist_element *rl); +struct runlist_element *ntfs_rl_insert_range(struct runlist_element *dst_rl, int dst_cnt, + struct runlist_element *src_rl, int src_cnt, size_t *new_cnt); +struct runlist_element *ntfs_rl_punch_hole(struct runlist_element *dst_rl, int dst_cnt, + s64 start_vcn, s64 len, struct runlist_element **punch_rl, + size_t *new_rl_cnt); +struct runlist_element *ntfs_rl_collapse_range(struct runlist_element *dst_rl, int dst_cnt, + s64 start_vcn, s64 len, struct runlist_element **punch_rl, + size_t *new_rl_cnt); +struct runlist_element *ntfs_rl_realloc(struct runlist_element *rl, int old_size, + int new_size); +#endif /* _LINUX_NTFS_RUNLIST_H */ diff --git a/fs/ntfs/super.c b/fs/ntfs/super.c new file mode 100644 index 000000000..8abe7bee4 --- /dev/null +++ b/fs/ntfs/super.c @@ -0,0 +1,2714 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * NTFS kernel super block handling. + * + * Copyright (c) 2001-2012 Anton Altaparmakov and Tuxera Inc. + * Copyright (c) 2001,2002 Richard Russon + * Copyright (c) 2025 LG Electronics Co., Ltd. + */ + +#include /* For bdev_logical_block_size(). */ +#include +#include +#include +#include +#include +#include + +#include "sysctl.h" +#include "logfile.h" +#include "index.h" +#include "ntfs.h" +#include "ea.h" +#include "volume.h" + +/* A global default upcase table and a corresponding reference count. */ +static __le16 *default_upcase; +static unsigned long ntfs_nr_upcase_users; + +static struct workqueue_struct *ntfs_wq; + +/* Error constants/strings used in inode.c::ntfs_show_options(). */ +enum { + /* One of these must be present, default is ON_ERRORS_CONTINUE. */ + ON_ERRORS_PANIC = 0x01, + ON_ERRORS_REMOUNT_RO = 0x02, + ON_ERRORS_CONTINUE = 0x04, +}; + +static const struct constant_table ntfs_param_enums[] = { + { "panic", ON_ERRORS_PANIC }, + { "remount-ro", ON_ERRORS_REMOUNT_RO }, + { "continue", ON_ERRORS_CONTINUE }, + {} +}; + +enum { + NATIVE_SYMLINK_RAW, + NATIVE_SYMLINK_REL, +}; + +static const struct constant_table ntfs_native_symlink_enums[] = { + { "raw", NATIVE_SYMLINK_RAW }, + { "rel", NATIVE_SYMLINK_REL }, + {} +}; + +enum { + SYMLINK_WSL, + SYMLINK_NATIVE, +}; + +static const struct constant_table ntfs_symlink_enums[] = { + { "wsl", SYMLINK_WSL }, + { "native", SYMLINK_NATIVE }, + {} +}; + +enum { + Opt_uid, + Opt_gid, + Opt_umask, + Opt_dmask, + Opt_fmask, + Opt_errors, + Opt_nls, + Opt_charset, + Opt_show_sys_files, + Opt_show_meta, + Opt_case_sensitive, + Opt_disable_sparse, + Opt_sparse, + Opt_mft_zone_multiplier, + Opt_preallocated_size, + Opt_sys_immutable, + Opt_nohidden, + Opt_hide_dot_files, + Opt_check_windows_names, + Opt_acl, + Opt_discard, + Opt_nocase, + Opt_native_symlink, + Opt_symlink, +}; + +static const struct fs_parameter_spec ntfs_parameters[] = { + fsparam_u32("uid", Opt_uid), + fsparam_u32("gid", Opt_gid), + fsparam_u32oct("umask", Opt_umask), + fsparam_u32oct("dmask", Opt_dmask), + fsparam_u32oct("fmask", Opt_fmask), + fsparam_string("nls", Opt_nls), + fsparam_string("iocharset", Opt_charset), + fsparam_enum("errors", Opt_errors, ntfs_param_enums), + fsparam_flag("show_sys_files", Opt_show_sys_files), + fsparam_flag("showmeta", Opt_show_meta), + fsparam_flag("case_sensitive", Opt_case_sensitive), + fsparam_flag("disable_sparse", Opt_disable_sparse), + fsparam_s32("mft_zone_multiplier", Opt_mft_zone_multiplier), + fsparam_u64("preallocated_size", Opt_preallocated_size), + fsparam_flag("sys_immutable", Opt_sys_immutable), + fsparam_flag("nohidden", Opt_nohidden), + fsparam_flag("hide_dot_files", Opt_hide_dot_files), + fsparam_flag("windows_names", Opt_check_windows_names), + fsparam_flag("acl", Opt_acl), + fsparam_flag("discard", Opt_discard), + fsparam_flag("sparse", Opt_sparse), + fsparam_flag("nocase", Opt_nocase), + fsparam_enum("native_symlink", Opt_native_symlink, ntfs_native_symlink_enums), + fsparam_enum("symlink", Opt_symlink, ntfs_symlink_enums), + {} +}; + +static int ntfs_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct ntfs_volume *vol = fc->s_fs_info; + struct fs_parse_result result; + int opt; + + opt = fs_parse(fc, ntfs_parameters, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_uid: + vol->uid = make_kuid(current_user_ns(), result.uint_32); + break; + case Opt_gid: + vol->gid = make_kgid(current_user_ns(), result.uint_32); + break; + case Opt_umask: + vol->fmask = vol->dmask = result.uint_32; + break; + case Opt_dmask: + vol->dmask = result.uint_32; + break; + case Opt_fmask: + vol->fmask = result.uint_32; + break; + case Opt_errors: + vol->on_errors = result.uint_32; + break; + case Opt_nls: + case Opt_charset: + if (vol->nls_map) + unload_nls(vol->nls_map); + vol->nls_map = load_nls(param->string); + if (!vol->nls_map) { + ntfs_error(vol->sb, "Failed to load NLS table '%s'.", + param->string); + return -EINVAL; + } + break; + case Opt_mft_zone_multiplier: + if (vol->mft_zone_multiplier && vol->mft_zone_multiplier != + result.int_32) { + ntfs_error(vol->sb, "Cannot change mft_zone_multiplier on remount."); + return -EINVAL; + } + if (result.int_32 < 1 || result.int_32 > 4) { + ntfs_error(vol->sb, + "Invalid mft_zone_multiplier. Using default value, i.e. 1."); + vol->mft_zone_multiplier = 1; + } else + vol->mft_zone_multiplier = result.int_32; + break; + case Opt_show_sys_files: + case Opt_show_meta: + if (result.boolean) + NVolSetShowSystemFiles(vol); + else + NVolClearShowSystemFiles(vol); + break; + case Opt_case_sensitive: + if (result.boolean) + NVolSetCaseSensitive(vol); + else + NVolClearCaseSensitive(vol); + break; + case Opt_nocase: + if (result.boolean) + NVolClearCaseSensitive(vol); + else + NVolSetCaseSensitive(vol); + break; + case Opt_preallocated_size: + vol->preallocated_size = (loff_t)result.uint_64; + break; + case Opt_sys_immutable: + if (result.boolean) + NVolSetSysImmutable(vol); + else + NVolClearSysImmutable(vol); + break; + case Opt_nohidden: + if (result.boolean) + NVolClearShowHiddenFiles(vol); + else + NVolSetShowHiddenFiles(vol); + break; + case Opt_hide_dot_files: + if (result.boolean) + NVolSetHideDotFiles(vol); + else + NVolClearHideDotFiles(vol); + break; + case Opt_check_windows_names: + if (result.boolean) + NVolSetCheckWindowsNames(vol); + else + NVolClearCheckWindowsNames(vol); + break; + case Opt_acl: +#ifdef CONFIG_NTFS_FS_POSIX_ACL + if (result.boolean) + fc->sb_flags |= SB_POSIXACL; + else + fc->sb_flags &= ~SB_POSIXACL; + break; +#else + return -EINVAL; +#endif + case Opt_discard: + if (result.boolean) + NVolSetDiscard(vol); + else + NVolClearDiscard(vol); + break; + case Opt_disable_sparse: + if (result.boolean) + NVolSetDisableSparse(vol); + else + NVolClearDisableSparse(vol); + break; + case Opt_native_symlink: + if (result.uint_32 == NATIVE_SYMLINK_REL) + NVolSetNativeSymlinkRel(vol); + else + NVolClearNativeSymlinkRel(vol); + break; + case Opt_symlink: + if (result.uint_32 == SYMLINK_NATIVE) + NVolSetSymlinkNative(vol); + else + NVolClearSymlinkNative(vol); + break; + case Opt_sparse: + break; + default: + return -EINVAL; + } + + return 0; +} + +static int ntfs_reconfigure(struct fs_context *fc) +{ + struct super_block *sb = fc->root->d_sb; + struct ntfs_volume *vol = NTFS_SB(sb); + + ntfs_debug("Entering with remount"); + + sync_filesystem(sb); + + /* + * For the read-write compiled driver, if we are remounting read-write, + * make sure there are no volume errors and that no unsupported volume + * flags are set. Also, empty the logfile journal as it would become + * stale as soon as something is written to the volume and mark the + * volume dirty so that chkdsk is run if the volume is not umounted + * cleanly. + * + * When remounting read-only, mark the volume clean if no volume errors + * have occurred. + */ + if (sb_rdonly(sb) && !(fc->sb_flags & SB_RDONLY)) { + static const char *es = ". Cannot remount read-write."; + + /* Remounting read-write. */ + if (NVolErrors(vol)) { + ntfs_error(sb, "Volume has errors and is read-only%s", + es); + return -EROFS; + } + if (vol->vol_flags & VOLUME_IS_DIRTY) { + ntfs_error(sb, "Volume is dirty and read-only%s", es); + return -EROFS; + } + if (vol->vol_flags & VOLUME_MODIFIED_BY_CHKDSK) { + ntfs_error(sb, "Volume has been modified by chkdsk and is read-only%s", es); + return -EROFS; + } + if (vol->vol_flags & VOLUME_MUST_MOUNT_RO_MASK) { + ntfs_error(sb, "Volume has unsupported flags set (0x%x) and is read-only%s", + le16_to_cpu(vol->vol_flags), es); + return -EROFS; + } + if (vol->logfile_ino && !ntfs_empty_logfile(vol->logfile_ino)) { + ntfs_error(sb, "Failed to empty journal LogFile%s", + es); + NVolSetErrors(vol); + return -EROFS; + } + } else if (!sb_rdonly(sb) && (fc->sb_flags & SB_RDONLY)) { + /* Remounting read-only. */ + if (!NVolErrors(vol)) { + if (ntfs_clear_volume_flags(vol, VOLUME_IS_DIRTY)) + ntfs_warning(sb, + "Failed to clear dirty bit in volume information flags. Run chkdsk."); + } + } + + ntfs_debug("Done."); + return 0; +} + +const struct option_t on_errors_arr[] = { + { ON_ERRORS_PANIC, "panic" }, + { ON_ERRORS_REMOUNT_RO, "remount-ro", }, + { ON_ERRORS_CONTINUE, "continue", }, + { 0, NULL } +}; + +void ntfs_handle_error(struct super_block *sb) +{ + struct ntfs_volume *vol = NTFS_SB(sb); + + if (sb_rdonly(sb)) + return; + + if (vol->on_errors == ON_ERRORS_REMOUNT_RO) { + sb->s_flags |= SB_RDONLY; + pr_crit("(device %s): Filesystem has been set read-only\n", + sb->s_id); + } else if (vol->on_errors == ON_ERRORS_PANIC) { + panic("ntfs: (device %s): panic from previous error\n", + sb->s_id); + } else if (vol->on_errors == ON_ERRORS_CONTINUE) { + if (errseq_check(&sb->s_wb_err, vol->wb_err) == -ENODEV) { + NVolSetShutdown(vol); + vol->wb_err = sb->s_wb_err; + } + } +} + +/* + * ntfs_write_volume_flags - write new flags to the volume information flags + * @vol: ntfs volume on which to modify the flags + * @flags: new flags value for the volume information flags + * + * Internal function. You probably want to use ntfs_{set,clear}_volume_flags() + * instead (see below). + * + * Replace the volume information flags on the volume @vol with the value + * supplied in @flags. Note, this overwrites the volume information flags, so + * make sure to combine the flags you want to modify with the old flags and use + * the result when calling ntfs_write_volume_flags(). + * + * Return 0 on success and -errno on error. + */ +static int ntfs_write_volume_flags(struct ntfs_volume *vol, const __le16 flags) +{ + struct ntfs_inode *ni = NTFS_I(vol->vol_ino); + struct volume_information *vi; + struct ntfs_attr_search_ctx *ctx; + int err; + + ntfs_debug("Entering, old flags = 0x%x, new flags = 0x%x.", + le16_to_cpu(vol->vol_flags), le16_to_cpu(flags)); + mutex_lock(&ni->mrec_lock); + if (vol->vol_flags == flags) + goto done; + + ctx = ntfs_attr_get_search_ctx(ni, NULL); + if (!ctx) { + err = -ENOMEM; + goto put_unm_err_out; + } + + err = ntfs_attr_lookup(AT_VOLUME_INFORMATION, NULL, 0, 0, 0, NULL, 0, + ctx); + if (err) + goto put_unm_err_out; + + vi = (struct volume_information *)((u8 *)ctx->attr + + le16_to_cpu(ctx->attr->data.resident.value_offset)); + vol->vol_flags = vi->flags = flags; + mark_mft_record_dirty(ctx->ntfs_ino); + ntfs_attr_put_search_ctx(ctx); +done: + mutex_unlock(&ni->mrec_lock); + ntfs_debug("Done."); + return 0; +put_unm_err_out: + if (ctx) + ntfs_attr_put_search_ctx(ctx); + mutex_unlock(&ni->mrec_lock); + ntfs_error(vol->sb, "Failed with error code %i.", -err); + return err; +} + +/* + * ntfs_set_volume_flags - set bits in the volume information flags + * @vol: ntfs volume on which to modify the flags + * @flags: flags to set on the volume + * + * Set the bits in @flags in the volume information flags on the volume @vol. + * + * Return 0 on success and -errno on error. + */ +int ntfs_set_volume_flags(struct ntfs_volume *vol, __le16 flags) +{ + flags &= VOLUME_FLAGS_MASK; + return ntfs_write_volume_flags(vol, vol->vol_flags | flags); +} + +/* + * ntfs_clear_volume_flags - clear bits in the volume information flags + * @vol: ntfs volume on which to modify the flags + * @flags: flags to clear on the volume + * + * Clear the bits in @flags in the volume information flags on the volume @vol. + * + * Return 0 on success and -errno on error. + */ +int ntfs_clear_volume_flags(struct ntfs_volume *vol, __le16 flags) +{ + flags &= VOLUME_FLAGS_MASK; + flags = vol->vol_flags & cpu_to_le16(~le16_to_cpu(flags)); + return ntfs_write_volume_flags(vol, flags); +} + +int ntfs_write_volume_label(struct ntfs_volume *vol, char *label) +{ + struct ntfs_inode *vol_ni = NTFS_I(vol->vol_ino); + struct ntfs_attr_search_ctx *ctx; + char *new_label; + __le16 *uname; + int uname_len, ret; + + uname_len = ntfs_nlstoucs(vol, label, strlen(label), + &uname, FSLABEL_MAX); + if (uname_len < 0) { + ntfs_error(vol->sb, + "Failed to convert volume label '%s' to Unicode.", + label); + return uname_len; + } + + if (uname_len > NTFS_MAX_LABEL_LEN) { + ntfs_error(vol->sb, + "Volume label is too long (max %d characters).", + NTFS_MAX_LABEL_LEN); + kvfree(uname); + return -EINVAL; + } + + /* + * Allocate the in-memory label copy up front. If kstrdup() fails we + * bail out before touching on-disk metadata, so the in-memory label + * and the on-disk label stay in sync. + */ + new_label = kstrdup(label, GFP_KERNEL); + if (!new_label) { + kvfree(uname); + return -ENOMEM; + } + + mutex_lock(&vol_ni->mrec_lock); + ctx = ntfs_attr_get_search_ctx(vol_ni, NULL); + if (!ctx) { + ret = -ENOMEM; + goto out; + } + + ret = ntfs_attr_lookup(AT_VOLUME_NAME, NULL, 0, 0, 0, NULL, 0, + ctx); + if (!ret) + ret = ntfs_attr_record_rm(ctx); + else if (ret == -ENOENT) + ret = 0; + ntfs_attr_put_search_ctx(ctx); + if (ret) + goto out; + + ret = ntfs_resident_attr_record_add(vol_ni, AT_VOLUME_NAME, AT_UNNAMED, 0, + (u8 *)uname, uname_len * sizeof(__le16), 0); +out: + if (ret >= 0) { + char *old_label; + + mutex_lock(&vol->volume_label_lock); + old_label = vol->volume_label; + vol->volume_label = new_label; + mutex_unlock(&vol->volume_label_lock); + + kfree(old_label); + mark_inode_dirty_sync(vol->vol_ino); + ret = 0; + } + mutex_unlock(&vol_ni->mrec_lock); + kvfree(uname); + + if (ret < 0) + kfree(new_label); + return ret; +} + +/* + * is_boot_sector_ntfs - check whether a boot sector is a valid NTFS boot sector + * @sb: Super block of the device to which @b belongs. + * @b: Boot sector of device @sb to check. + * @silent: If 'true', all output will be silenced. + * + * is_boot_sector_ntfs() checks whether the boot sector @b is a valid NTFS boot + * sector. Returns 'true' if it is valid and 'false' if not. + * + * @sb is only needed for warning/error output, i.e. it can be NULL when silent + * is 'true'. + */ +static bool is_boot_sector_ntfs(const struct super_block *sb, + const struct ntfs_boot_sector *b, const bool silent) +{ + /* + * Check that checksum == sum of u32 values from b to the checksum + * field. If checksum is zero, no checking is done. We will work when + * the checksum test fails, since some utilities update the boot sector + * ignoring the checksum which leaves the checksum out-of-date. We + * report a warning if this is the case. + */ + if ((void *)b < (void *)&b->checksum && b->checksum && !silent) { + __le32 *u; + u32 i; + + for (i = 0, u = (__le32 *)b; u < (__le32 *)(&b->checksum); ++u) + i += le32_to_cpup(u); + if (le32_to_cpu(b->checksum) != i) + ntfs_warning(sb, "Invalid boot sector checksum."); + } + /* Check OEMidentifier is "NTFS " */ + if (b->oem_id != magicNTFS) + goto not_ntfs; + /* Check bytes per sector value is between 256 and 4096. */ + if (le16_to_cpu(b->bpb.bytes_per_sector) < 0x100 || + le16_to_cpu(b->bpb.bytes_per_sector) > 0x1000) + goto not_ntfs; + /* + * Check sectors per cluster value is valid and the cluster size + * is not above the maximum (2MB). + */ + if (b->bpb.sectors_per_cluster > 0x80 && + b->bpb.sectors_per_cluster < 0xf4) + goto not_ntfs; + + /* Check reserved/unused fields are really zero. */ + if (le16_to_cpu(b->bpb.reserved_sectors) || + le16_to_cpu(b->bpb.root_entries) || + le16_to_cpu(b->bpb.sectors) || + le16_to_cpu(b->bpb.sectors_per_fat) || + le32_to_cpu(b->bpb.large_sectors) || b->bpb.fats) + goto not_ntfs; + /* Check clusters per file mft record value is valid. */ + if ((u8)b->clusters_per_mft_record < 0xe1 || + (u8)b->clusters_per_mft_record > 0xf7) + switch (b->clusters_per_mft_record) { + case 1: case 2: case 4: case 8: case 16: case 32: case 64: + break; + default: + goto not_ntfs; + } + /* Check clusters per index block value is valid. */ + if ((u8)b->clusters_per_index_record < 0xe1 || + (u8)b->clusters_per_index_record > 0xf7) + switch (b->clusters_per_index_record) { + case 1: case 2: case 4: case 8: case 16: case 32: case 64: + break; + default: + goto not_ntfs; + } + /* + * Check for valid end of sector marker. We will work without it, but + * many BIOSes will refuse to boot from a bootsector if the magic is + * incorrect, so we emit a warning. + */ + if (!silent && b->end_of_sector_marker != cpu_to_le16(0xaa55)) + ntfs_warning(sb, "Invalid end of sector marker."); + return true; +not_ntfs: + return false; +} + +/* + * read_ntfs_boot_sector - read the NTFS boot sector of a device + * @sb: super block of device to read the boot sector from + * @silent: if true, suppress all output + * + * Reads the boot sector from the device and validates it. + */ +static char *read_ntfs_boot_sector(struct super_block *sb, + const int silent) +{ + char *boot_sector; + + boot_sector = kzalloc(PAGE_SIZE, GFP_NOFS); + if (!boot_sector) + return NULL; + + if (ntfs_bdev_read(sb->s_bdev, boot_sector, 0, PAGE_SIZE)) { + if (!silent) + ntfs_error(sb, "Unable to read primary boot sector."); + kfree(boot_sector); + return NULL; + } + + if (!is_boot_sector_ntfs(sb, (struct ntfs_boot_sector *)boot_sector, + silent)) { + if (!silent) + ntfs_error(sb, "Primary boot sector is invalid."); + kfree(boot_sector); + return NULL; + } + + return boot_sector; +} + +/* + * parse_ntfs_boot_sector - parse the boot sector and store the data in @vol + * @vol: volume structure to initialise with data from boot sector + * @b: boot sector to parse + * + * Parse the ntfs boot sector @b and store all imporant information therein in + * the ntfs super block @vol. Return 'true' on success and 'false' on error. + */ +static bool parse_ntfs_boot_sector(struct ntfs_volume *vol, + const struct ntfs_boot_sector *b) +{ + unsigned int sectors_per_cluster, sectors_per_cluster_bits, nr_hidden_sects; + int clusters_per_mft_record, clusters_per_index_record; + s64 ll; + + vol->sector_size = le16_to_cpu(b->bpb.bytes_per_sector); + vol->sector_size_bits = ffs(vol->sector_size) - 1; + ntfs_debug("vol->sector_size = %i (0x%x)", vol->sector_size, + vol->sector_size); + ntfs_debug("vol->sector_size_bits = %i (0x%x)", vol->sector_size_bits, + vol->sector_size_bits); + if (vol->sector_size < vol->sb->s_blocksize) { + ntfs_error(vol->sb, + "Sector size (%i) is smaller than the device block size (%lu). This is not supported.", + vol->sector_size, vol->sb->s_blocksize); + return false; + } + + if (b->bpb.sectors_per_cluster >= 0xf4) + sectors_per_cluster = 1U << -(s8)b->bpb.sectors_per_cluster; + else + sectors_per_cluster = b->bpb.sectors_per_cluster; + ntfs_debug("sectors_per_cluster = 0x%x", b->bpb.sectors_per_cluster); + sectors_per_cluster_bits = ffs(sectors_per_cluster) - 1; + ntfs_debug("sectors_per_cluster_bits = 0x%x", + sectors_per_cluster_bits); + nr_hidden_sects = le32_to_cpu(b->bpb.hidden_sectors); + ntfs_debug("number of hidden sectors = 0x%x", nr_hidden_sects); + vol->cluster_size = vol->sector_size << sectors_per_cluster_bits; + vol->cluster_size_mask = vol->cluster_size - 1; + vol->cluster_size_bits = ffs(vol->cluster_size) - 1; + ntfs_debug("vol->cluster_size = %i (0x%x)", vol->cluster_size, + vol->cluster_size); + ntfs_debug("vol->cluster_size_mask = 0x%x", vol->cluster_size_mask); + ntfs_debug("vol->cluster_size_bits = %i", vol->cluster_size_bits); + if (vol->cluster_size < vol->sector_size) { + ntfs_error(vol->sb, + "Cluster size (%i) is smaller than the sector size (%i). This is not supported.", + vol->cluster_size, vol->sector_size); + return false; + } + clusters_per_mft_record = b->clusters_per_mft_record; + ntfs_debug("clusters_per_mft_record = %i (0x%x)", + clusters_per_mft_record, clusters_per_mft_record); + if (clusters_per_mft_record > 0) + vol->mft_record_size = vol->cluster_size << + (ffs(clusters_per_mft_record) - 1); + else + /* + * When mft_record_size < cluster_size, clusters_per_mft_record + * = -log2(mft_record_size) bytes. mft_record_size normaly is + * 1024 bytes, which is encoded as 0xF6 (-10 in decimal). + */ + vol->mft_record_size = 1 << -clusters_per_mft_record; + vol->mft_record_size_mask = vol->mft_record_size - 1; + vol->mft_record_size_bits = ffs(vol->mft_record_size) - 1; + ntfs_debug("vol->mft_record_size = %i (0x%x)", vol->mft_record_size, + vol->mft_record_size); + ntfs_debug("vol->mft_record_size_mask = 0x%x", + vol->mft_record_size_mask); + ntfs_debug("vol->mft_record_size_bits = %i (0x%x)", + vol->mft_record_size_bits, vol->mft_record_size_bits); + /* + * We cannot support mft record sizes above the PAGE_SIZE since + * we store $MFT/$DATA, the table of mft records in the page cache. + */ + if (vol->mft_record_size > PAGE_SIZE) { + ntfs_error(vol->sb, + "Mft record size (%i) exceeds the PAGE_SIZE on your system (%lu). This is not supported.", + vol->mft_record_size, PAGE_SIZE); + return false; + } + /* We cannot support mft record sizes below the sector size. */ + if (vol->mft_record_size < vol->sector_size) { + ntfs_warning(vol->sb, "Mft record size (%i) is smaller than the sector size (%i).", + vol->mft_record_size, vol->sector_size); + } + clusters_per_index_record = b->clusters_per_index_record; + ntfs_debug("clusters_per_index_record = %i (0x%x)", + clusters_per_index_record, clusters_per_index_record); + if (clusters_per_index_record > 0) + vol->index_record_size = vol->cluster_size << + (ffs(clusters_per_index_record) - 1); + else + /* + * When index_record_size < cluster_size, + * clusters_per_index_record = -log2(index_record_size) bytes. + * index_record_size normaly equals 4096 bytes, which is + * encoded as 0xF4 (-12 in decimal). + */ + vol->index_record_size = 1 << -clusters_per_index_record; + vol->index_record_size_mask = vol->index_record_size - 1; + vol->index_record_size_bits = ffs(vol->index_record_size) - 1; + ntfs_debug("vol->index_record_size = %i (0x%x)", + vol->index_record_size, vol->index_record_size); + ntfs_debug("vol->index_record_size_mask = 0x%x", + vol->index_record_size_mask); + ntfs_debug("vol->index_record_size_bits = %i (0x%x)", + vol->index_record_size_bits, + vol->index_record_size_bits); + /* We cannot support index record sizes below the sector size. */ + if (vol->index_record_size < vol->sector_size) { + ntfs_error(vol->sb, + "Index record size (%i) is smaller than the sector size (%i). This is not supported.", + vol->index_record_size, vol->sector_size); + return false; + } + /* + * Get the size of the volume in clusters and check for 64-bit-ness. + * Windows currently only uses 32 bits to save the clusters so we do + * the same as it is much faster on 32-bit CPUs. + */ + ll = le64_to_cpu(b->number_of_sectors) >> sectors_per_cluster_bits; + if ((u64)ll >= 1ULL << 32) { + ntfs_error(vol->sb, "Cannot handle 64-bit clusters."); + return false; + } + vol->nr_clusters = ll; + ntfs_debug("vol->nr_clusters = 0x%llx", vol->nr_clusters); + ll = le64_to_cpu(b->mft_lcn); + if (ll >= vol->nr_clusters) { + ntfs_error(vol->sb, "MFT LCN (%lli, 0x%llx) is beyond end of volume. Weird.", + ll, ll); + return false; + } + vol->mft_lcn = ll; + ntfs_debug("vol->mft_lcn = 0x%llx", vol->mft_lcn); + ll = le64_to_cpu(b->mftmirr_lcn); + if (ll >= vol->nr_clusters) { + ntfs_error(vol->sb, "MFTMirr LCN (%lli, 0x%llx) is beyond end of volume. Weird.", + ll, ll); + return false; + } + vol->mftmirr_lcn = ll; + ntfs_debug("vol->mftmirr_lcn = 0x%llx", vol->mftmirr_lcn); + /* + * Work out the size of the mft mirror in number of mft records. If the + * cluster size is less than or equal to the size taken by four mft + * records, the mft mirror stores the first four mft records. If the + * cluster size is bigger than the size taken by four mft records, the + * mft mirror contains as many mft records as will fit into one + * cluster. + */ + if (vol->cluster_size <= (4 << vol->mft_record_size_bits)) + vol->mftmirr_size = 4; + else + vol->mftmirr_size = vol->cluster_size >> + vol->mft_record_size_bits; + ntfs_debug("vol->mftmirr_size = %i", vol->mftmirr_size); + vol->serial_no = le64_to_cpu(b->volume_serial_number); + ntfs_debug("vol->serial_no = 0x%llx", vol->serial_no); + + vol->sparse_compression_unit = 4; + if (vol->cluster_size > 4096) { + switch (vol->cluster_size) { + case 65536: + vol->sparse_compression_unit = 0; + break; + case 32768: + vol->sparse_compression_unit = 1; + break; + case 16384: + vol->sparse_compression_unit = 2; + break; + case 8192: + vol->sparse_compression_unit = 3; + break; + } + } + + return true; +} + +/* + * ntfs_setup_allocators - initialize the cluster and mft allocators + * @vol: volume structure for which to setup the allocators + * + * Setup the cluster (lcn) and mft allocators to the starting values. + */ +static void ntfs_setup_allocators(struct ntfs_volume *vol) +{ + s64 mft_zone_size, mft_lcn; + + ntfs_debug("vol->mft_zone_multiplier = 0x%x", + vol->mft_zone_multiplier); + /* Determine the size of the MFT zone. */ + mft_zone_size = vol->nr_clusters; + switch (vol->mft_zone_multiplier) { /* % of volume size in clusters */ + case 4: + mft_zone_size >>= 1; /* 50% */ + break; + case 3: + mft_zone_size = (mft_zone_size + + (mft_zone_size >> 1)) >> 2; /* 37.5% */ + break; + case 2: + mft_zone_size >>= 2; /* 25% */ + break; + /* case 1: */ + default: + mft_zone_size >>= 3; /* 12.5% */ + break; + } + /* Setup the mft zone. */ + vol->mft_zone_start = vol->mft_zone_pos = vol->mft_lcn; + ntfs_debug("vol->mft_zone_pos = 0x%llx", vol->mft_zone_pos); + /* + * Calculate the mft_lcn for an unmodified NTFS volume (see mkntfs + * source) and if the actual mft_lcn is in the expected place or even + * further to the front of the volume, extend the mft_zone to cover the + * beginning of the volume as well. This is in order to protect the + * area reserved for the mft bitmap as well within the mft_zone itself. + * On non-standard volumes we do not protect it as the overhead would + * be higher than the speed increase we would get by doing it. + */ + mft_lcn = NTFS_B_TO_CLU(vol, 8192 + 2 * vol->cluster_size - 1); + if (mft_lcn * vol->cluster_size < 16 * 1024) + mft_lcn = (16 * 1024 + vol->cluster_size - 1) >> + vol->cluster_size_bits; + if (vol->mft_zone_start <= mft_lcn) + vol->mft_zone_start = 0; + ntfs_debug("vol->mft_zone_start = 0x%llx", vol->mft_zone_start); + /* + * Need to cap the mft zone on non-standard volumes so that it does + * not point outside the boundaries of the volume. We do this by + * halving the zone size until we are inside the volume. + */ + vol->mft_zone_end = vol->mft_lcn + mft_zone_size; + while (vol->mft_zone_end >= vol->nr_clusters) { + mft_zone_size >>= 1; + vol->mft_zone_end = vol->mft_lcn + mft_zone_size; + } + ntfs_debug("vol->mft_zone_end = 0x%llx", vol->mft_zone_end); + /* + * Set the current position within each data zone to the start of the + * respective zone. + */ + vol->data1_zone_pos = vol->mft_zone_end; + ntfs_debug("vol->data1_zone_pos = 0x%llx", vol->data1_zone_pos); + vol->data2_zone_pos = 0; + ntfs_debug("vol->data2_zone_pos = 0x%llx", vol->data2_zone_pos); + + /* Set the mft data allocation position to mft record 24. */ + vol->mft_data_pos = 24; + ntfs_debug("vol->mft_data_pos = 0x%llx", vol->mft_data_pos); +} + +static struct lock_class_key mftmirr_runlist_lock_key, + mftmirr_mrec_lock_key; +/* + * load_and_init_mft_mirror - load and setup the mft mirror inode for a volume + * @vol: ntfs super block describing device whose mft mirror to load + * + * Return 'true' on success or 'false' on error. + */ +static bool load_and_init_mft_mirror(struct ntfs_volume *vol) +{ + struct inode *tmp_ino; + struct ntfs_inode *tmp_ni; + + ntfs_debug("Entering."); + /* Get mft mirror inode. */ + tmp_ino = ntfs_iget(vol->sb, FILE_MFTMirr); + if (IS_ERR(tmp_ino)) { + if (!IS_ERR(tmp_ino)) + iput(tmp_ino); + /* Caller will display error message. */ + return false; + } + lockdep_set_class(&NTFS_I(tmp_ino)->runlist.lock, + &mftmirr_runlist_lock_key); + lockdep_set_class(&NTFS_I(tmp_ino)->mrec_lock, + &mftmirr_mrec_lock_key); + /* + * Re-initialize some specifics about $MFTMirr's inode as + * ntfs_read_inode() will have set up the default ones. + */ + /* Set uid and gid to root. */ + tmp_ino->i_uid = GLOBAL_ROOT_UID; + tmp_ino->i_gid = GLOBAL_ROOT_GID; + /* Regular file. No access for anyone. */ + tmp_ino->i_mode = S_IFREG; + /* No VFS initiated operations allowed for $MFTMirr. */ + tmp_ino->i_op = &ntfs_empty_inode_ops; + tmp_ino->i_fop = &ntfs_empty_file_ops; + /* Put in our special address space operations. */ + tmp_ino->i_mapping->a_ops = &ntfs_aops; + tmp_ni = NTFS_I(tmp_ino); + /* The $MFTMirr, like the $MFT is multi sector transfer protected. */ + NInoSetMstProtected(tmp_ni); + NInoSetSparseDisabled(tmp_ni); + /* + * Set up our little cheat allowing us to reuse the async read io + * completion handler for directories. + */ + tmp_ni->itype.index.block_size = vol->mft_record_size; + tmp_ni->itype.index.block_size_bits = vol->mft_record_size_bits; + vol->mftmirr_ino = tmp_ino; + ntfs_debug("Done."); + return true; +} + +/* + * check_mft_mirror - compare contents of the mft mirror with the mft + * @vol: ntfs super block describing device whose mft mirror to check + * + * Return 'true' on success or 'false' on error. + * + * Note, this function also results in the mft mirror runlist being completely + * mapped into memory. The mft mirror write code requires this and will BUG() + * should it find an unmapped runlist element. + */ +static bool check_mft_mirror(struct ntfs_volume *vol) +{ + struct super_block *sb = vol->sb; + struct ntfs_inode *mirr_ni; + struct folio *mft_folio = NULL, *mirr_folio = NULL; + u8 *kmft = NULL, *kmirr = NULL; + struct runlist_element *rl, rl2[2]; + pgoff_t index; + int mrecs_per_page, i; + + ntfs_debug("Entering."); + /* Compare contents of $MFT and $MFTMirr. */ + mrecs_per_page = PAGE_SIZE / vol->mft_record_size; + index = i = 0; + do { + u32 bytes; + + /* Switch pages if necessary. */ + if (!(i % mrecs_per_page)) { + if (index) { + kunmap_local(kmirr); + folio_put(mirr_folio); + kunmap_local(kmft); + folio_put(mft_folio); + } + /* Get the $MFT page. */ + mft_folio = read_mapping_folio(vol->mft_ino->i_mapping, + index, NULL); + if (IS_ERR(mft_folio)) { + ntfs_error(sb, "Failed to read $MFT."); + return false; + } + kmft = kmap_local_folio(mft_folio, 0); + /* Get the $MFTMirr page. */ + mirr_folio = read_mapping_folio(vol->mftmirr_ino->i_mapping, + index, NULL); + if (IS_ERR(mirr_folio)) { + ntfs_error(sb, "Failed to read $MFTMirr."); + goto mft_unmap_out; + } + kmirr = kmap_local_folio(mirr_folio, 0); + ++index; + } + + /* Do not check the record if it is not in use. */ + if (((struct mft_record *)kmft)->flags & MFT_RECORD_IN_USE) { + /* Make sure the record is ok. */ + if (ntfs_is_baad_recordp((__le32 *)kmft)) { + ntfs_error(sb, + "Incomplete multi sector transfer detected in mft record %i.", + i); +mm_unmap_out: + kunmap_local(kmirr); + folio_put(mirr_folio); +mft_unmap_out: + kunmap_local(kmft); + folio_put(mft_folio); + return false; + } + } + /* Do not check the mirror record if it is not in use. */ + if (((struct mft_record *)kmirr)->flags & MFT_RECORD_IN_USE) { + if (ntfs_is_baad_recordp((__le32 *)kmirr)) { + ntfs_error(sb, + "Incomplete multi sector transfer detected in mft mirror record %i.", + i); + goto mm_unmap_out; + } + } + /* Get the amount of data in the current record. */ + bytes = le32_to_cpu(((struct mft_record *)kmft)->bytes_in_use); + if (bytes < sizeof(struct mft_record_old) || + bytes > vol->mft_record_size || + ntfs_is_baad_recordp((__le32 *)kmft)) { + bytes = le32_to_cpu(((struct mft_record *)kmirr)->bytes_in_use); + if (bytes < sizeof(struct mft_record_old) || + bytes > vol->mft_record_size || + ntfs_is_baad_recordp((__le32 *)kmirr)) + bytes = vol->mft_record_size; + } + /* Compare the two records. */ + if (memcmp(kmft, kmirr, bytes)) { + ntfs_error(sb, + "$MFT and $MFTMirr record %i do not match. Run chkdsk.", + i); + goto mm_unmap_out; + } + kmft += vol->mft_record_size; + kmirr += vol->mft_record_size; + } while (++i < vol->mftmirr_size); + /* Release the last folios. */ + kunmap_local(kmirr); + folio_put(mirr_folio); + kunmap_local(kmft); + folio_put(mft_folio); + + /* Construct the mft mirror runlist by hand. */ + rl2[0].vcn = 0; + rl2[0].lcn = vol->mftmirr_lcn; + rl2[0].length = NTFS_B_TO_CLU(vol, vol->mftmirr_size * vol->mft_record_size + + vol->cluster_size - 1); + rl2[1].vcn = rl2[0].length; + rl2[1].lcn = LCN_ENOENT; + rl2[1].length = 0; + /* + * Because we have just read all of the mft mirror, we know we have + * mapped the full runlist for it. + */ + mirr_ni = NTFS_I(vol->mftmirr_ino); + down_read(&mirr_ni->runlist.lock); + rl = mirr_ni->runlist.rl; + /* Compare the two runlists. They must be identical. */ + i = 0; + do { + if (rl2[i].vcn != rl[i].vcn || rl2[i].lcn != rl[i].lcn || + rl2[i].length != rl[i].length) { + ntfs_error(sb, "$MFTMirr location mismatch. Run chkdsk."); + up_read(&mirr_ni->runlist.lock); + return false; + } + } while (rl2[i++].length); + up_read(&mirr_ni->runlist.lock); + ntfs_debug("Done."); + return true; +} + +/* + * load_and_check_logfile - load and check the logfile inode for a volume + * @vol: ntfs volume to load the logfile for + * @rp: on success, set to the restart page header + * + * Return 0 on success or errno on error. + */ +static int load_and_check_logfile(struct ntfs_volume *vol, + struct restart_page_header **rp) +{ + struct inode *tmp_ino; + int err = 0; + + ntfs_debug("Entering."); + tmp_ino = ntfs_iget(vol->sb, FILE_LogFile); + if (IS_ERR(tmp_ino)) { + if (!IS_ERR(tmp_ino)) + iput(tmp_ino); + /* Caller will display error message. */ + return -ENOENT; + } + if (!ntfs_check_logfile(tmp_ino, rp)) + err = -EINVAL; + NInoSetSparseDisabled(NTFS_I(tmp_ino)); + vol->logfile_ino = tmp_ino; + ntfs_debug("Done."); + return err; +} + +#define NTFS_HIBERFIL_HEADER_SIZE 4096 + +/* + * check_windows_hibernation_status - check if Windows is suspended on a volume + * @vol: ntfs super block of device to check + * + * Check if Windows is hibernated on the ntfs volume @vol. This is done by + * looking for the file hiberfil.sys in the root directory of the volume. If + * the file is not present Windows is definitely not suspended. + * + * If hiberfil.sys exists and is less than 4kiB in size it means Windows is + * definitely suspended (this volume is not the system volume). Caveat: on a + * system with many volumes it is possible that the < 4kiB check is bogus but + * for now this should do fine. + * + * If hiberfil.sys exists and is larger than 4kiB in size, we need to read the + * hiberfil header (which is the first 4kiB). If this begins with "hibr", + * Windows is definitely suspended. If it is completely full of zeroes, + * Windows is definitely not hibernated. Any other case is treated as if + * Windows is suspended. This caters for the above mentioned caveat of a + * system with many volumes where no "hibr" magic would be present and there is + * no zero header. + * + * Return 0 if Windows is not hibernated on the volume, >0 if Windows is + * hibernated on the volume, and -errno on error. + */ +static int check_windows_hibernation_status(struct ntfs_volume *vol) +{ + static const __le16 hiberfil[13] = { cpu_to_le16('h'), + cpu_to_le16('i'), cpu_to_le16('b'), + cpu_to_le16('e'), cpu_to_le16('r'), + cpu_to_le16('f'), cpu_to_le16('i'), + cpu_to_le16('l'), cpu_to_le16('.'), + cpu_to_le16('s'), cpu_to_le16('y'), + cpu_to_le16('s'), 0 }; + u64 mref; + struct inode *vi; + struct folio *folio; + u32 *kaddr, *kend, *start_addr = NULL; + struct ntfs_name *name = NULL; + int ret = 1; + + ntfs_debug("Entering."); + /* + * Find the inode number for the hibernation file by looking up the + * filename hiberfil.sys in the root directory. + */ + inode_lock(vol->root_ino); + mref = ntfs_lookup_inode_by_name(NTFS_I(vol->root_ino), hiberfil, 12, + &name); + inode_unlock(vol->root_ino); + kfree(name); + if (IS_ERR_MREF(mref)) { + ret = MREF_ERR(mref); + /* If the file does not exist, Windows is not hibernated. */ + if (ret == -ENOENT) { + ntfs_debug("hiberfil.sys not present. Windows is not hibernated on the volume."); + return 0; + } + /* A real error occurred. */ + ntfs_error(vol->sb, "Failed to find inode number for hiberfil.sys."); + return ret; + } + /* Get the inode. */ + vi = ntfs_iget(vol->sb, MREF(mref)); + if (IS_ERR(vi)) { + if (!IS_ERR(vi)) + iput(vi); + ntfs_error(vol->sb, "Failed to load hiberfil.sys."); + return IS_ERR(vi) ? PTR_ERR(vi) : -EIO; + } + if (unlikely(i_size_read(vi) < NTFS_HIBERFIL_HEADER_SIZE)) { + ntfs_debug("hiberfil.sys is smaller than 4kiB (0x%llx). Windows is hibernated on the volume. This is not the system volume.", + i_size_read(vi)); + goto iput_out; + } + + folio = read_mapping_folio(vi->i_mapping, 0, NULL); + if (IS_ERR(folio)) { + ntfs_error(vol->sb, "Failed to read from hiberfil.sys."); + ret = PTR_ERR(folio); + goto iput_out; + } + start_addr = (u32 *)kmap_local_folio(folio, 0); + kaddr = start_addr; + if (*(__le32 *)kaddr == cpu_to_le32(0x72626968)/*'hibr'*/) { + ntfs_debug("Magic \"hibr\" found in hiberfil.sys. Windows is hibernated on the volume. This is the system volume."); + goto unm_iput_out; + } + kend = kaddr + NTFS_HIBERFIL_HEADER_SIZE/sizeof(*kaddr); + do { + if (unlikely(*kaddr)) { + ntfs_debug("hiberfil.sys is larger than 4kiB (0x%llx), does not contain the \"hibr\" magic, and does not have a zero header. Windows is hibernated on the volume. This is not the system volume.", + i_size_read(vi)); + goto unm_iput_out; + } + } while (++kaddr < kend); + ntfs_debug("hiberfil.sys contains a zero header. Windows is not hibernated on the volume. This is the system volume."); + ret = 0; +unm_iput_out: + kunmap_local(start_addr); + folio_put(folio); +iput_out: + iput(vi); + return ret; +} + +/* + * load_and_init_attrdef - load the attribute definitions table for a volume + * @vol: ntfs super block describing device whose attrdef to load + * + * Return 'true' on success or 'false' on error. + */ +static bool load_and_init_attrdef(struct ntfs_volume *vol) +{ + loff_t i_size; + struct super_block *sb = vol->sb; + struct inode *ino; + struct folio *folio; + u8 *addr; + pgoff_t index, max_index; + unsigned int size; + + ntfs_debug("Entering."); + /* Read attrdef table and setup vol->attrdef and vol->attrdef_size. */ + ino = ntfs_iget(sb, FILE_AttrDef); + if (IS_ERR(ino)) { + if (!IS_ERR(ino)) + iput(ino); + goto failed; + } + NInoSetSparseDisabled(NTFS_I(ino)); + /* The size of FILE_AttrDef must be above 0 and fit inside 31 bits. */ + i_size = i_size_read(ino); + if (i_size <= 0 || i_size > 0x7fffffff) + goto iput_failed; + vol->attrdef = kvzalloc(i_size, GFP_NOFS); + if (!vol->attrdef) + goto iput_failed; + index = 0; + max_index = i_size >> PAGE_SHIFT; + size = PAGE_SIZE; + while (index < max_index) { + /* Read the attrdef table and copy it into the linear buffer. */ +read_partial_attrdef_page: + folio = read_mapping_folio(ino->i_mapping, index, NULL); + if (IS_ERR(folio)) + goto free_iput_failed; + addr = kmap_local_folio(folio, 0); + memcpy((u8 *)vol->attrdef + (index++ << PAGE_SHIFT), + addr, size); + kunmap_local(addr); + folio_put(folio); + } + if (size == PAGE_SIZE) { + size = i_size & ~PAGE_MASK; + if (size) + goto read_partial_attrdef_page; + } + vol->attrdef_size = i_size; + ntfs_debug("Read %llu bytes from $AttrDef.", i_size); + iput(ino); + return true; +free_iput_failed: + kvfree(vol->attrdef); + vol->attrdef = NULL; +iput_failed: + iput(ino); +failed: + ntfs_error(sb, "Failed to initialize attribute definition table."); + return false; +} + +/* + * load_and_init_upcase - load the upcase table for an ntfs volume + * @vol: ntfs super block describing device whose upcase to load + * + * Return 'true' on success or 'false' on error. + */ +static bool load_and_init_upcase(struct ntfs_volume *vol) +{ + loff_t i_size; + struct super_block *sb = vol->sb; + struct inode *ino; + struct folio *folio; + u8 *addr; + pgoff_t index, max_index; + unsigned int size; + + ntfs_debug("Entering."); + /* Read upcase table and setup vol->upcase and vol->upcase_len. */ + ino = ntfs_iget(sb, FILE_UpCase); + if (IS_ERR(ino)) { + if (!IS_ERR(ino)) + iput(ino); + goto upcase_failed; + } + /* + * The upcase size must not be above 64k Unicode characters, must not + * be zero and must be a multiple of sizeof(__le16). + */ + i_size = i_size_read(ino); + if (!i_size || i_size & (sizeof(__le16) - 1) || + i_size > 64ULL * 1024 * sizeof(__le16)) + goto iput_upcase_failed; + vol->upcase = kvzalloc(i_size, GFP_NOFS); + if (!vol->upcase) + goto iput_upcase_failed; + index = 0; + max_index = i_size >> PAGE_SHIFT; + size = PAGE_SIZE; + while (index < max_index) { + /* Read the upcase table and copy it into the linear buffer. */ +read_partial_upcase_page: + folio = read_mapping_folio(ino->i_mapping, index, NULL); + if (IS_ERR(folio)) + goto iput_upcase_failed; + addr = kmap_local_folio(folio, 0); + memcpy((char *)vol->upcase + (index++ << PAGE_SHIFT), + addr, size); + kunmap_local(addr); + folio_put(folio); + } + if (size == PAGE_SIZE) { + size = i_size & ~PAGE_MASK; + if (size) + goto read_partial_upcase_page; + } + vol->upcase_len = i_size >> sizeof(unsigned char); + ntfs_debug("Read %llu bytes from $UpCase (expected %zu bytes).", + i_size, 64 * 1024 * sizeof(__le16)); + iput(ino); + mutex_lock(&ntfs_lock); + if (!default_upcase) { + ntfs_debug("Using volume specified $UpCase since default is not present."); + mutex_unlock(&ntfs_lock); + return true; + } + if (default_upcase_len == vol->upcase_len && + !memcmp(vol->upcase, default_upcase, + default_upcase_len * sizeof(*default_upcase))) { + kvfree(vol->upcase); + vol->upcase = default_upcase; + ntfs_nr_upcase_users++; + mutex_unlock(&ntfs_lock); + ntfs_debug("Volume specified $UpCase matches default. Using default."); + return true; + } + mutex_unlock(&ntfs_lock); + ntfs_debug("Using volume specified $UpCase since it does not match the default."); + return true; +iput_upcase_failed: + iput(ino); + kvfree(vol->upcase); + vol->upcase = NULL; +upcase_failed: + mutex_lock(&ntfs_lock); + if (default_upcase) { + vol->upcase = default_upcase; + vol->upcase_len = default_upcase_len; + ntfs_nr_upcase_users++; + mutex_unlock(&ntfs_lock); + ntfs_error(sb, "Failed to load $UpCase from the volume. Using default."); + return true; + } + mutex_unlock(&ntfs_lock); + ntfs_error(sb, "Failed to initialize upcase table."); + return false; +} + +/* + * The lcn and mft bitmap inodes are NTFS-internal inodes with + * their own special locking rules: + */ +static struct lock_class_key + lcnbmp_runlist_lock_key, lcnbmp_mrec_lock_key, + mftbmp_runlist_lock_key, mftbmp_mrec_lock_key; + +/* + * load_system_files - open the system files using normal functions + * @vol: ntfs super block describing device whose system files to load + * + * Open the system files with normal access functions and complete setting up + * the ntfs super block @vol. + * + * Return 'true' on success or 'false' on error. + */ +static bool load_system_files(struct ntfs_volume *vol) +{ + struct super_block *sb = vol->sb; + struct mft_record *m; + struct volume_information *vi; + struct ntfs_attr_search_ctx *ctx; + struct restart_page_header *rp; + int err; + + ntfs_debug("Entering."); + /* Get mft mirror inode compare the contents of $MFT and $MFTMirr. */ + if (!load_and_init_mft_mirror(vol) || !check_mft_mirror(vol)) { + /* If a read-write mount, convert it to a read-only mount. */ + if (!sb_rdonly(sb) && vol->on_errors == ON_ERRORS_REMOUNT_RO) { + static const char *es1 = "Failed to load $MFTMirr"; + static const char *es2 = "$MFTMirr does not match $MFT"; + static const char *es3 = ". Run ntfsck and/or chkdsk."; + + sb->s_flags |= SB_RDONLY; + ntfs_error(sb, "%s. Mounting read-only%s", + !vol->mftmirr_ino ? es1 : es2, es3); + } + NVolSetErrors(vol); + } + /* Get mft bitmap attribute inode. */ + vol->mftbmp_ino = ntfs_attr_iget(vol->mft_ino, AT_BITMAP, NULL, 0); + if (IS_ERR(vol->mftbmp_ino)) { + ntfs_error(sb, "Failed to load $MFT/$BITMAP attribute."); + goto iput_mirr_err_out; + } + lockdep_set_class(&NTFS_I(vol->mftbmp_ino)->runlist.lock, + &mftbmp_runlist_lock_key); + lockdep_set_class(&NTFS_I(vol->mftbmp_ino)->mrec_lock, + &mftbmp_mrec_lock_key); + /* Read upcase table and setup @vol->upcase and @vol->upcase_len. */ + if (!load_and_init_upcase(vol)) + goto iput_mftbmp_err_out; + /* + * Read attribute definitions table and setup @vol->attrdef and + * @vol->attrdef_size. + */ + if (!load_and_init_attrdef(vol)) + goto iput_upcase_err_out; + /* + * Get the cluster allocation bitmap inode and verify the size, no + * need for any locking at this stage as we are already running + * exclusively as we are mount in progress task. + */ + vol->lcnbmp_ino = ntfs_iget(sb, FILE_Bitmap); + if (IS_ERR(vol->lcnbmp_ino)) { + if (!IS_ERR(vol->lcnbmp_ino)) + iput(vol->lcnbmp_ino); + goto bitmap_failed; + } + lockdep_set_class(&NTFS_I(vol->lcnbmp_ino)->runlist.lock, + &lcnbmp_runlist_lock_key); + lockdep_set_class(&NTFS_I(vol->lcnbmp_ino)->mrec_lock, + &lcnbmp_mrec_lock_key); + + NInoSetSparseDisabled(NTFS_I(vol->lcnbmp_ino)); + if ((vol->nr_clusters + 7) >> 3 > i_size_read(vol->lcnbmp_ino)) { + iput(vol->lcnbmp_ino); +bitmap_failed: + ntfs_error(sb, "Failed to load $Bitmap."); + goto iput_attrdef_err_out; + } + /* + * Get the volume inode and setup our cache of the volume flags and + * version. + */ + vol->vol_ino = ntfs_iget(sb, FILE_Volume); + if (IS_ERR(vol->vol_ino)) { + if (!IS_ERR(vol->vol_ino)) + iput(vol->vol_ino); +volume_failed: + ntfs_error(sb, "Failed to load $Volume."); + goto iput_lcnbmp_err_out; + } + m = map_mft_record(NTFS_I(vol->vol_ino)); + if (IS_ERR(m)) { +iput_volume_failed: + iput(vol->vol_ino); + goto volume_failed; + } + + ctx = ntfs_attr_get_search_ctx(NTFS_I(vol->vol_ino), m); + if (!ctx) { + ntfs_error(sb, "Failed to get attribute search context."); + goto get_ctx_vol_failed; + } + + if (!ntfs_attr_lookup(AT_VOLUME_NAME, NULL, 0, 0, 0, NULL, 0, ctx) && + !ctx->attr->non_resident && + !(ctx->attr->flags & (ATTR_IS_SPARSE | ATTR_IS_COMPRESSED)) && + le32_to_cpu(ctx->attr->data.resident.value_length) > 0) { + err = ntfs_ucstonls(vol, (__le16 *)((u8 *)ctx->attr + + le16_to_cpu(ctx->attr->data.resident.value_offset)), + le32_to_cpu(ctx->attr->data.resident.value_length) / 2, + &vol->volume_label, NTFS_MAX_LABEL_LEN); + if (err < 0) + vol->volume_label = NULL; + } + + ntfs_attr_reinit_search_ctx(ctx); + if (ntfs_attr_lookup(AT_VOLUME_INFORMATION, NULL, 0, 0, 0, NULL, 0, + ctx) || ctx->attr->non_resident || ctx->attr->flags) { + ntfs_attr_put_search_ctx(ctx); +get_ctx_vol_failed: + unmap_mft_record(NTFS_I(vol->vol_ino)); + goto iput_volume_failed; + } + vi = (struct volume_information *)((char *)ctx->attr + + le16_to_cpu(ctx->attr->data.resident.value_offset)); + /* Copy the volume flags and version to the struct ntfs_volume structure. */ + vol->vol_flags = vi->flags; + vol->major_ver = vi->major_ver; + vol->minor_ver = vi->minor_ver; + ntfs_attr_put_search_ctx(ctx); + unmap_mft_record(NTFS_I(vol->vol_ino)); + pr_info("volume version %i.%i, dev %s, cluster size %d\n", + vol->major_ver, vol->minor_ver, sb->s_id, vol->cluster_size); + + /* Make sure that no unsupported volume flags are set. */ + if (vol->vol_flags & VOLUME_MUST_MOUNT_RO_MASK) { + static const char *es1a = "Volume is dirty"; + static const char *es1b = "Volume has been modified by chkdsk"; + static const char *es1c = "Volume has unsupported flags set"; + static const char *es2a = ". Run chkdsk and mount in Windows."; + static const char *es2b = ". Mount in Windows."; + const char *es1, *es2; + + es2 = es2a; + if (vol->vol_flags & VOLUME_IS_DIRTY) + es1 = es1a; + else if (vol->vol_flags & VOLUME_MODIFIED_BY_CHKDSK) { + es1 = es1b; + es2 = es2b; + } else { + es1 = es1c; + ntfs_warning(sb, "Unsupported volume flags 0x%x encountered.", + (unsigned int)le16_to_cpu(vol->vol_flags)); + } + /* If a read-write mount, convert it to a read-only mount. */ + if (!sb_rdonly(sb) && vol->on_errors == ON_ERRORS_REMOUNT_RO) { + sb->s_flags |= SB_RDONLY; + ntfs_error(sb, "%s. Mounting read-only%s", es1, es2); + } + /* + * Do not set NVolErrors() because ntfs_remount() re-checks the + * flags which we need to do in case any flags have changed. + */ + } + /* + * Get the inode for the logfile, check it and determine if the volume + * was shutdown cleanly. + */ + rp = NULL; + err = load_and_check_logfile(vol, &rp); + if (err) { + /* If a read-write mount, convert it to a read-only mount. */ + if (!sb_rdonly(sb) && vol->on_errors == ON_ERRORS_REMOUNT_RO) { + sb->s_flags |= SB_RDONLY; + ntfs_error(sb, "Failed to load LogFile. Mounting read-only."); + } + NVolSetErrors(vol); + } + + kvfree(rp); + /* Get the root directory inode so we can do path lookups. */ + vol->root_ino = ntfs_iget(sb, FILE_root); + if (IS_ERR(vol->root_ino)) { + if (!IS_ERR(vol->root_ino)) + iput(vol->root_ino); + ntfs_error(sb, "Failed to load root directory."); + goto iput_logfile_err_out; + } + /* + * Check if Windows is suspended to disk on the target volume. If it + * is hibernated, we must not write *anything* to the disk so set + * NVolErrors() without setting the dirty volume flag and mount + * read-only. This will prevent read-write remounting and it will also + * prevent all writes. + */ + err = check_windows_hibernation_status(vol); + if (unlikely(err)) { + static const char *es1a = "Failed to determine if Windows is hibernated"; + static const char *es1b = "Windows is hibernated"; + static const char *es2 = ". Run chkdsk."; + const char *es1; + + es1 = err < 0 ? es1a : es1b; + /* If a read-write mount, convert it to a read-only mount. */ + if (!sb_rdonly(sb) && vol->on_errors == ON_ERRORS_REMOUNT_RO) { + sb->s_flags |= SB_RDONLY; + ntfs_error(sb, "%s. Mounting read-only%s", es1, es2); + } + NVolSetErrors(vol); + } + + /* If (still) a read-write mount, empty the logfile. */ + if (!sb_rdonly(sb) && + vol->logfile_ino && !ntfs_empty_logfile(vol->logfile_ino) && + vol->on_errors == ON_ERRORS_REMOUNT_RO) { + static const char *es1 = "Failed to empty LogFile"; + static const char *es2 = ". Mount in Windows."; + + /* Convert to a read-only mount. */ + ntfs_error(sb, "%s. Mounting read-only%s", es1, es2); + sb->s_flags |= SB_RDONLY; + NVolSetErrors(vol); + } + /* If on NTFS versions before 3.0, we are done. */ + if (unlikely(vol->major_ver < 3)) + return true; + /* NTFS 3.0+ specific initialization. */ + /* Get the security descriptors inode. */ + vol->secure_ino = ntfs_iget(sb, FILE_Secure); + if (IS_ERR(vol->secure_ino)) { + if (!IS_ERR(vol->secure_ino)) + iput(vol->secure_ino); + ntfs_error(sb, "Failed to load $Secure."); + goto iput_root_err_out; + } + /* Get the extended system files' directory inode. */ + vol->extend_ino = ntfs_iget(sb, FILE_Extend); + if (IS_ERR(vol->extend_ino) || + !S_ISDIR(vol->extend_ino->i_mode)) { + if (!IS_ERR(vol->extend_ino)) + iput(vol->extend_ino); + ntfs_error(sb, "Failed to load $Extend."); + goto iput_sec_err_out; + } + return true; + +iput_sec_err_out: + iput(vol->secure_ino); +iput_root_err_out: + iput(vol->root_ino); +iput_logfile_err_out: + if (vol->logfile_ino) + iput(vol->logfile_ino); + iput(vol->vol_ino); +iput_lcnbmp_err_out: + iput(vol->lcnbmp_ino); +iput_attrdef_err_out: + vol->attrdef_size = 0; + if (vol->attrdef) { + kvfree(vol->attrdef); + vol->attrdef = NULL; + } +iput_upcase_err_out: + vol->upcase_len = 0; + mutex_lock(&ntfs_lock); + if (vol->upcase && vol->upcase == default_upcase) { + ntfs_nr_upcase_users--; + vol->upcase = NULL; + } + mutex_unlock(&ntfs_lock); + if (vol->upcase) { + kvfree(vol->upcase); + vol->upcase = NULL; + } +iput_mftbmp_err_out: + iput(vol->mftbmp_ino); +iput_mirr_err_out: + iput(vol->mftmirr_ino); + return false; +} + +static void ntfs_volume_free(struct ntfs_volume *vol) +{ + /* Throw away the table of attribute definitions. */ + vol->attrdef_size = 0; + if (vol->attrdef) { + kvfree(vol->attrdef); + vol->attrdef = NULL; + } + vol->upcase_len = 0; + /* + * Destroy the global default upcase table if necessary. Also decrease + * the number of upcase users if we are a user. + */ + mutex_lock(&ntfs_lock); + if (vol->upcase && vol->upcase == default_upcase) { + ntfs_nr_upcase_users--; + vol->upcase = NULL; + } + + if (!ntfs_nr_upcase_users) { + kvfree(default_upcase); + default_upcase = NULL; + } + + free_compression_buffers(); + + mutex_unlock(&ntfs_lock); + if (vol->upcase) { + kvfree(vol->upcase); + vol->upcase = NULL; + } + + unload_nls(vol->nls_map); + + kvfree(vol->lcn_empty_bits_per_page); + kfree(vol->volume_label); + kfree(vol); +} + +/* + * ntfs_put_super - called by the vfs to unmount a volume + * @sb: vfs superblock of volume to unmount + */ +static void ntfs_put_super(struct super_block *sb) +{ + struct ntfs_volume *vol = NTFS_SB(sb); + + pr_info("Entering %s, dev %s\n", __func__, sb->s_id); + + cancel_work_sync(&vol->precalc_work); + + /* + * Commit all inodes while they are still open in case some of them + * cause others to be dirtied. + */ + ntfs_commit_inode(vol->vol_ino); + + /* NTFS 3.0+ specific. */ + if (vol->major_ver >= 3) { + if (vol->extend_ino) + ntfs_commit_inode(vol->extend_ino); + if (vol->secure_ino) + ntfs_commit_inode(vol->secure_ino); + } + + ntfs_commit_inode(vol->root_ino); + + ntfs_commit_inode(vol->lcnbmp_ino); + + /* + * the GFP_NOFS scope is not needed because ntfs_commit_inode + * does nothing + */ + ntfs_commit_inode(vol->mftbmp_ino); + + if (vol->logfile_ino) + ntfs_commit_inode(vol->logfile_ino); + + if (vol->mftmirr_ino) + ntfs_commit_inode(vol->mftmirr_ino); + ntfs_commit_inode(vol->mft_ino); + + /* + * If a read-write mount and no volume errors have occurred, mark the + * volume clean. Also, re-commit all affected inodes. + */ + if (!sb_rdonly(sb)) { + if (!NVolErrors(vol)) { + if (ntfs_clear_volume_flags(vol, VOLUME_IS_DIRTY)) + ntfs_warning(sb, + "Failed to clear dirty bit in volume information flags. Run chkdsk."); + ntfs_commit_inode(vol->vol_ino); + ntfs_commit_inode(vol->root_ino); + if (vol->mftmirr_ino) + ntfs_commit_inode(vol->mftmirr_ino); + ntfs_commit_inode(vol->mft_ino); + } else { + ntfs_warning(sb, + "Volume has errors. Leaving volume marked dirty. Run chkdsk."); + } + } + + iput(vol->vol_ino); + vol->vol_ino = NULL; + + /* NTFS 3.0+ specific clean up. */ + if (vol->major_ver >= 3) { + if (vol->extend_ino) { + iput(vol->extend_ino); + vol->extend_ino = NULL; + } + if (vol->secure_ino) { + iput(vol->secure_ino); + vol->secure_ino = NULL; + } + } + + iput(vol->root_ino); + vol->root_ino = NULL; + + iput(vol->lcnbmp_ino); + vol->lcnbmp_ino = NULL; + + iput(vol->mftbmp_ino); + vol->mftbmp_ino = NULL; + + if (vol->logfile_ino) { + iput(vol->logfile_ino); + vol->logfile_ino = NULL; + } + if (vol->mftmirr_ino) { + /* Re-commit the mft mirror and mft just in case. */ + ntfs_commit_inode(vol->mftmirr_ino); + ntfs_commit_inode(vol->mft_ino); + iput(vol->mftmirr_ino); + vol->mftmirr_ino = NULL; + } + /* + * We should have no dirty inodes left, due to + * mft.c::ntfs_mft_writepage() cleaning all the dirty pages as + * the underlying mft records are written out and cleaned. + */ + ntfs_commit_inode(vol->mft_ino); + write_inode_now(vol->mft_ino, 1); + + iput(vol->mft_ino); + vol->mft_ino = NULL; + blkdev_issue_flush(sb->s_bdev); + + ntfs_volume_free(vol); +} + +int ntfs_force_shutdown(struct super_block *sb, u32 flags) +{ + struct ntfs_volume *vol = NTFS_SB(sb); + int ret; + + if (NVolShutdown(vol)) + return 0; + + switch (flags) { + case FS_SHUTDOWN_FLAGS_DEFAULT: + case FS_SHUTDOWN_FLAGS_LOGFLUSH: + ret = bdev_freeze(sb->s_bdev); + if (ret) + return ret; + bdev_thaw(sb->s_bdev); + NVolSetShutdown(vol); + break; + case FS_SHUTDOWN_FLAGS_NOLOGFLUSH: + NVolSetShutdown(vol); + break; + default: + return -EINVAL; + } + + return 0; +} + +static void ntfs_shutdown(struct super_block *sb) +{ + ntfs_force_shutdown(sb, FS_SHUTDOWN_FLAGS_NOLOGFLUSH); + +} + +static int ntfs_sync_fs(struct super_block *sb, int wait) +{ + struct ntfs_volume *vol = NTFS_SB(sb); + int err = 0; + + if (NVolShutdown(vol)) + return -EIO; + + if (!wait) + return 0; + + /* If there are some dirty buffers in the bdev inode */ + if (ntfs_clear_volume_flags(vol, VOLUME_IS_DIRTY)) { + ntfs_warning(sb, "Failed to clear dirty bit in volume information flags. Run chkdsk."); + err = -EIO; + } + sync_inodes_sb(sb); + sync_blockdev(sb->s_bdev); + blkdev_issue_flush(sb->s_bdev); + return err; +} + +/* + * get_nr_free_clusters - return the number of free clusters on a volume + * @vol: ntfs volume for which to obtain free cluster count + * + * Calculate the number of free clusters on the mounted NTFS volume @vol. We + * actually calculate the number of clusters in use instead because this + * allows us to not care about partial pages as these will be just zero filled + * and hence not be counted as allocated clusters. + * + * The only particularity is that clusters beyond the end of the logical ntfs + * volume will be marked as allocated to prevent errors which means we have to + * discount those at the end. This is important as the cluster bitmap always + * has a size in multiples of 8 bytes, i.e. up to 63 clusters could be outside + * the logical volume and marked in use when they are not as they do not exist. + * + * If any pages cannot be read we assume all clusters in the erroring pages are + * in use. This means we return an underestimate on errors which is better than + * an overestimate. + */ +s64 get_nr_free_clusters(struct ntfs_volume *vol) +{ + s64 nr_free = vol->nr_clusters; + u32 nr_used; + struct address_space *mapping = vol->lcnbmp_ino->i_mapping; + struct folio *folio; + pgoff_t index, max_index; + struct file_ra_state ra = { 0 }; + + ntfs_debug("Entering."); + /* Serialize accesses to the cluster bitmap. */ + + if (NVolFreeClusterKnown(vol)) + return atomic64_read(&vol->free_clusters); + + file_ra_state_init(&ra, mapping); + + /* + * Convert the number of bits into bytes rounded up, then convert into + * multiples of PAGE_SIZE, rounding up so that if we have one + * full and one partial page max_index = 2. + */ + max_index = (((vol->nr_clusters + 7) >> 3) + PAGE_SIZE - 1) >> + PAGE_SHIFT; + /* Use multiples of 4 bytes, thus max_size is PAGE_SIZE / 4. */ + ntfs_debug("Reading $Bitmap, max_index = 0x%lx, max_size = 0x%lx.", + max_index, PAGE_SIZE / 4); + for (index = 0; index < max_index; index++) { + unsigned long *kaddr; + + /* + * Get folio from page cache, getting it from backing store + * if necessary, and increment the use count. + */ + folio = ntfs_get_locked_folio(mapping, index, max_index, &ra); + + /* Ignore pages which errored synchronously. */ + if (IS_ERR(folio)) { + ntfs_debug("Skipping page (index 0x%lx).", index); + nr_free -= PAGE_SIZE * 8; + vol->lcn_empty_bits_per_page[index] = 0; + continue; + } + + kaddr = kmap_local_folio(folio, 0); + /* + * Subtract the number of set bits. If this + * is the last page and it is partial we don't really care as + * it just means we do a little extra work but it won't affect + * the result as all out of range bytes are set to zero by + * ntfs_readpage(). + */ + nr_used = bitmap_weight(kaddr, PAGE_SIZE * BITS_PER_BYTE); + nr_free -= nr_used; + vol->lcn_empty_bits_per_page[index] = PAGE_SIZE * BITS_PER_BYTE - nr_used; + kunmap_local(kaddr); + folio_unlock(folio); + folio_put(folio); + } + ntfs_debug("Finished reading $Bitmap, last index = 0x%lx.", index - 1); + /* + * Fixup for eventual bits outside logical ntfs volume (see function + * description above). + */ + if (vol->nr_clusters & 63) + nr_free += 64 - (vol->nr_clusters & 63); + + /* If errors occurred we may well have gone below zero, fix this. */ + if (nr_free < 0) + nr_free = 0; + else + atomic64_set(&vol->free_clusters, nr_free); + + NVolSetFreeClusterKnown(vol); + wake_up_all(&vol->free_waitq); + ntfs_debug("Exiting."); + return nr_free; +} + +/* + * @nr_clusters is the number of clusters requested for allocation. + * + * Return the number of clusters available for allocation within + * the range of @nr_clusters, which is counts that considered + * for delayed allocation. + */ +s64 ntfs_available_clusters_count(struct ntfs_volume *vol, s64 nr_clusters) +{ + s64 free_clusters; + + /* wait event */ + if (!NVolFreeClusterKnown(vol)) + wait_event(vol->free_waitq, NVolFreeClusterKnown(vol)); + + free_clusters = atomic64_read(&vol->free_clusters) - + atomic64_read(&vol->dirty_clusters); + if (free_clusters <= 0) + return -ENOSPC; + else if (free_clusters < nr_clusters) + nr_clusters = free_clusters; + + return nr_clusters; +} + +/* + * __get_nr_free_mft_records - return the number of free inodes on a volume + * @vol: ntfs volume for which to obtain free inode count + * @nr_free: number of mft records in filesystem + * @max_index: maximum number of pages containing set bits + * + * Calculate the number of free mft records (inodes) on the mounted NTFS + * volume @vol. We actually calculate the number of mft records in use instead + * because this allows us to not care about partial pages as these will be just + * zero filled and hence not be counted as allocated mft record. + * + * If any pages cannot be read we assume all mft records in the erroring pages + * are in use. This means we return an underestimate on errors which is better + * than an overestimate. + * + * NOTE: Caller must hold mftbmp_lock rw_semaphore for reading or writing. + */ +static unsigned long __get_nr_free_mft_records(struct ntfs_volume *vol, + s64 nr_free, const pgoff_t max_index) +{ + struct address_space *mapping = vol->mftbmp_ino->i_mapping; + struct folio *folio; + pgoff_t index; + struct file_ra_state ra = { 0 }; + + ntfs_debug("Entering."); + + file_ra_state_init(&ra, mapping); + + /* Use multiples of 4 bytes, thus max_size is PAGE_SIZE / 4. */ + ntfs_debug("Reading $MFT/$BITMAP, max_index = 0x%lx, max_size = 0x%lx.", + max_index, PAGE_SIZE / 4); + for (index = 0; index < max_index; index++) { + unsigned long *kaddr; + + /* + * Get folio from page cache, getting it from backing store + * if necessary, and increment the use count. + */ + folio = ntfs_get_locked_folio(mapping, index, max_index, &ra); + + /* Ignore pages which errored synchronously. */ + if (IS_ERR(folio)) { + ntfs_debug("read_mapping_page() error. Skipping page (index 0x%lx).", + index); + nr_free -= PAGE_SIZE * 8; + continue; + } + + kaddr = kmap_local_folio(folio, 0); + /* + * Subtract the number of set bits. If this + * is the last page and it is partial we don't really care as + * it just means we do a little extra work but it won't affect + * the result as all out of range bytes are set to zero by + * ntfs_readpage(). + */ + nr_free -= bitmap_weight(kaddr, + PAGE_SIZE * BITS_PER_BYTE); + kunmap_local(kaddr); + folio_unlock(folio); + folio_put(folio); + } + ntfs_debug("Finished reading $MFT/$BITMAP, last index = 0x%lx.", + index - 1); + /* If errors occurred we may well have gone below zero, fix this. */ + if (nr_free < 0) + nr_free = 0; + else + atomic64_set(&vol->free_mft_records, nr_free); + + ntfs_debug("Exiting."); + return nr_free; +} + +/* + * ntfs_statfs - return information about mounted NTFS volume + * @dentry: dentry from mounted volume + * @sfs: statfs structure in which to return the information + * + * Return information about the mounted NTFS volume @dentry in the statfs structure + * pointed to by @sfs (this is initialized with zeros before ntfs_statfs is + * called). We interpret the values to be correct of the moment in time at + * which we are called. Most values are variable otherwise and this isn't just + * the free values but the totals as well. For example we can increase the + * total number of file nodes if we run out and we can keep doing this until + * there is no more space on the volume left at all. + * + * Called from vfs_statfs which is used to handle the statfs, fstatfs, and + * ustat system calls. + * + * Return 0 on success or -errno on error. + */ +static int ntfs_statfs(struct dentry *dentry, struct kstatfs *sfs) +{ + struct super_block *sb = dentry->d_sb; + s64 size; + struct ntfs_volume *vol = NTFS_SB(sb); + struct ntfs_inode *mft_ni = NTFS_I(vol->mft_ino); + unsigned long flags; + + ntfs_debug("Entering."); + /* Type of filesystem. */ + sfs->f_type = NTFS_SB_MAGIC; + /* Optimal transfer block size. */ + sfs->f_bsize = vol->cluster_size; + /* Fundamental file system block size, used as the unit. */ + sfs->f_frsize = vol->cluster_size; + + /* + * Total data blocks in filesystem in units of f_bsize and since + * inodes are also stored in data blocs ($MFT is a file) this is just + * the total clusters. + */ + sfs->f_blocks = vol->nr_clusters; + + /* wait event */ + if (!NVolFreeClusterKnown(vol)) + wait_event(vol->free_waitq, NVolFreeClusterKnown(vol)); + + /* Free data blocks in filesystem in units of f_bsize. */ + size = atomic64_read(&vol->free_clusters) - + atomic64_read(&vol->dirty_clusters); + if (size < 0LL) + size = 0LL; + + /* Free blocks avail to non-superuser, same as above on NTFS. */ + sfs->f_bavail = sfs->f_bfree = size; + + /* Number of inodes in filesystem (at this point in time). */ + read_lock_irqsave(&mft_ni->size_lock, flags); + sfs->f_files = i_size_read(vol->mft_ino) >> vol->mft_record_size_bits; + read_unlock_irqrestore(&mft_ni->size_lock, flags); + + /* Free inodes in fs (based on current total count). */ + sfs->f_ffree = atomic64_read(&vol->free_mft_records); + + /* + * File system id. This is extremely *nix flavour dependent and even + * within Linux itself all fs do their own thing. I interpret this to + * mean a unique id associated with the mounted fs and not the id + * associated with the filesystem driver, the latter is already given + * by the filesystem type in sfs->f_type. Thus we use the 64-bit + * volume serial number splitting it into two 32-bit parts. We enter + * the least significant 32-bits in f_fsid[0] and the most significant + * 32-bits in f_fsid[1]. + */ + sfs->f_fsid = u64_to_fsid(vol->serial_no); + /* Maximum length of filenames. */ + sfs->f_namelen = NTFS_MAX_NAME_LEN; + + return 0; +} + +static int ntfs_write_inode(struct inode *vi, struct writeback_control *wbc) +{ + return __ntfs_write_inode(vi, wbc->sync_mode == WB_SYNC_ALL); +} + +/* + * The complete super operations. + */ +static const struct super_operations ntfs_sops = { + .alloc_inode = ntfs_alloc_big_inode, /* VFS: Allocate new inode. */ + .free_inode = ntfs_free_big_inode, /* VFS: Deallocate inode. */ + .drop_inode = ntfs_drop_big_inode, + .write_inode = ntfs_write_inode, /* VFS: Write dirty inode to disk. */ + .put_super = ntfs_put_super, /* Syscall: umount. */ + .shutdown = ntfs_shutdown, + .sync_fs = ntfs_sync_fs, /* Syscall: sync. */ + .statfs = ntfs_statfs, /* Syscall: statfs */ + .evict_inode = ntfs_evict_big_inode, + .show_options = ntfs_show_options, /* Show mount options in proc. */ +}; + +static void precalc_free_clusters(struct work_struct *work) +{ + struct ntfs_volume *vol = container_of(work, struct ntfs_volume, precalc_work); + s64 nr_free; + + nr_free = get_nr_free_clusters(vol); + + ntfs_debug("pre-calculate free clusters(%lld) using workqueue", + nr_free); +} + +static struct lock_class_key ntfs_mft_inval_lock_key; + +/* + * ntfs_fill_super - mount an ntfs filesystem + * @sb: super block of the device to mount + * @fc: filesystem context containing mount options + * + * ntfs_fill_super() is called by the VFS to mount the device described by @sb + * with the mount otions in @data with the NTFS filesystem. + * + * If @silent is true, remain silent even if errors are detected. This is used + * during bootup, when the kernel tries to mount the root filesystem with all + * registered filesystems one after the other until one succeeds. This implies + * that all filesystems except the correct one will quite correctly and + * expectedly return an error, but nobody wants to see error messages when in + * fact this is what is supposed to happen. + */ +static int ntfs_fill_super(struct super_block *sb, struct fs_context *fc) +{ + char *boot; + struct inode *tmp_ino; + int blocksize, result; + pgoff_t lcn_bit_pages; + struct ntfs_volume *vol = NTFS_SB(sb); + int silent = fc->sb_flags & SB_SILENT; + + vol->sb = sb; + + /* + * We do a pretty difficult piece of bootstrap by reading the + * MFT (and other metadata) from disk into memory. We'll only + * release this metadata during umount, so the locking patterns + * observed during bootstrap do not count. So turn off the + * observation of locking patterns (strictly for this context + * only) while mounting NTFS. [The validator is still active + * otherwise, even for this context: it will for example record + * lock class registrations.] + */ + lockdep_off(); + ntfs_debug("Entering."); + + if (vol->nls_map && !strcmp(vol->nls_map->charset, "utf8")) + vol->nls_utf8 = true; + if (NVolDisableSparse(vol)) + vol->preallocated_size = 0; + + if (NVolDiscard(vol) && !bdev_max_discard_sectors(sb->s_bdev)) { + ntfs_warning( + sb, + "Discard requested but device does not support discard. Discard disabled."); + NVolClearDiscard(vol); + } + + /* We support sector sizes up to the PAGE_SIZE. */ + if (bdev_logical_block_size(sb->s_bdev) > PAGE_SIZE) { + if (!silent) + ntfs_error(sb, + "Device has unsupported sector size (%i). The maximum supported sector size on this architecture is %lu bytes.", + bdev_logical_block_size(sb->s_bdev), + PAGE_SIZE); + goto err_out_now; + } + + /* + * Setup the device access block size to NTFS_BLOCK_SIZE or the hard + * sector size, whichever is bigger. + */ + blocksize = sb_min_blocksize(sb, NTFS_BLOCK_SIZE); + if (blocksize < NTFS_BLOCK_SIZE) { + if (!silent) + ntfs_error(sb, "Unable to set device block size."); + goto err_out_now; + } + + ntfs_debug("Set device block size to %i bytes (block size bits %i).", + blocksize, sb->s_blocksize_bits); + /* Determine the size of the device in units of block_size bytes. */ + if (!bdev_nr_bytes(sb->s_bdev)) { + if (!silent) + ntfs_error(sb, "Unable to determine device size."); + goto err_out_now; + } + vol->nr_blocks = bdev_nr_bytes(sb->s_bdev) >> + sb->s_blocksize_bits; + /* Read the boot sector and return unlocked buffer head to it. */ + boot = read_ntfs_boot_sector(sb, silent); + if (!boot) { + if (!silent) + ntfs_error(sb, "Not an NTFS volume."); + goto err_out_now; + } + /* + * Extract the data from the boot sector and setup the ntfs volume + * using it. + */ + result = parse_ntfs_boot_sector(vol, (struct ntfs_boot_sector *)boot); + kfree(boot); + if (!result) { + if (!silent) + ntfs_error(sb, "Unsupported NTFS filesystem."); + goto err_out_now; + } + + if (vol->sector_size > blocksize) { + blocksize = sb_set_blocksize(sb, vol->sector_size); + if (blocksize != vol->sector_size) { + if (!silent) + ntfs_error(sb, + "Unable to set device block size to sector size (%i).", + vol->sector_size); + goto err_out_now; + } + vol->nr_blocks = bdev_nr_bytes(sb->s_bdev) >> + sb->s_blocksize_bits; + ntfs_debug("Changed device block size to %i bytes (block size bits %i) to match volume sector size.", + blocksize, sb->s_blocksize_bits); + } + /* Initialize the cluster and mft allocators. */ + ntfs_setup_allocators(vol); + /* Setup remaining fields in the super block. */ + sb->s_magic = NTFS_SB_MAGIC; + /* + * Ntfs allows 63 bits for the file size, i.e. correct would be: + * sb->s_maxbytes = ~0ULL >> 1; + * But the kernel uses a long as the page cache page index which on + * 32-bit architectures is only 32-bits. MAX_LFS_FILESIZE is kernel + * defined to the maximum the page cache page index can cope with + * without overflowing the index or to 2^63 - 1, whichever is smaller. + */ + sb->s_maxbytes = MAX_LFS_FILESIZE; + /* Ntfs measures time in 100ns intervals. */ + sb->s_time_gran = 100; + + sb->s_xattr = ntfs_xattr_handlers; + /* + * Now load the metadata required for the page cache and our address + * space operations to function. We do this by setting up a specialised + * read_inode method and then just calling the normal iget() to obtain + * the inode for $MFT which is sufficient to allow our normal inode + * operations and associated address space operations to function. + */ + sb->s_op = &ntfs_sops; + tmp_ino = new_inode(sb); + if (!tmp_ino) { + if (!silent) + ntfs_error(sb, "Failed to load essential metadata."); + goto err_out_now; + } + + tmp_ino->i_ino = FILE_MFT; + insert_inode_hash(tmp_ino); + if (ntfs_read_inode_mount(tmp_ino) < 0) { + if (!silent) + ntfs_error(sb, "Failed to load essential metadata."); + goto iput_tmp_ino_err_out_now; + } + lockdep_set_class(&tmp_ino->i_mapping->invalidate_lock, + &ntfs_mft_inval_lock_key); + + mutex_lock(&ntfs_lock); + + /* + * Generate the global default upcase table if necessary. Also + * temporarily increment the number of upcase users to avoid race + * conditions with concurrent (u)mounts. + */ + if (!default_upcase) + default_upcase = generate_default_upcase(); + ntfs_nr_upcase_users++; + mutex_unlock(&ntfs_lock); + + lcn_bit_pages = (((vol->nr_clusters + 7) >> 3) + PAGE_SIZE - 1) >> PAGE_SHIFT; + vol->lcn_empty_bits_per_page = kvmalloc_array(lcn_bit_pages, sizeof(unsigned int), + GFP_KERNEL); + if (!vol->lcn_empty_bits_per_page) { + ntfs_error(sb, + "Unable to allocate pages for storing LCN empty bit counts\n"); + goto unl_upcase_iput_tmp_ino_err_out_now; + } + + /* + * From now on, ignore @silent parameter. If we fail below this line, + * it will be due to a corrupt fs or a system error, so we report it. + */ + /* + * Open the system files with normal access functions and complete + * setting up the ntfs super block. + */ + if (!load_system_files(vol)) { + ntfs_error(sb, "Failed to load system files."); + goto unl_upcase_iput_tmp_ino_err_out_now; + } + + /* We grab a reference, simulating an ntfs_iget(). */ + ihold(vol->root_ino); + sb->s_root = d_make_root(vol->root_ino); + if (sb->s_root) { + s64 nr_records; + + ntfs_debug("Exiting, status successful."); + + /* Release the default upcase if it has no users. */ + mutex_lock(&ntfs_lock); + if (!--ntfs_nr_upcase_users && default_upcase) { + kvfree(default_upcase); + default_upcase = NULL; + } + mutex_unlock(&ntfs_lock); + sb->s_export_op = &ntfs_export_ops; + lockdep_on(); + + nr_records = __get_nr_free_mft_records(vol, + i_size_read(vol->mft_ino) >> vol->mft_record_size_bits, + ((((NTFS_I(vol->mft_ino)->initialized_size >> + vol->mft_record_size_bits) + + 7) >> 3) + PAGE_SIZE - 1) >> PAGE_SHIFT); + ntfs_debug("Free mft records(%lld)", nr_records); + + init_waitqueue_head(&vol->free_waitq); + INIT_WORK(&vol->precalc_work, precalc_free_clusters); + queue_work(ntfs_wq, &vol->precalc_work); + return 0; + } + ntfs_error(sb, "Failed to allocate root directory."); + /* Clean up after the successful load_system_files() call from above. */ + iput(vol->vol_ino); + vol->vol_ino = NULL; + /* NTFS 3.0+ specific clean up. */ + if (vol->major_ver >= 3) { + if (vol->extend_ino) { + iput(vol->extend_ino); + vol->extend_ino = NULL; + } + if (vol->secure_ino) { + iput(vol->secure_ino); + vol->secure_ino = NULL; + } + } + iput(vol->root_ino); + vol->root_ino = NULL; + iput(vol->lcnbmp_ino); + vol->lcnbmp_ino = NULL; + iput(vol->mftbmp_ino); + vol->mftbmp_ino = NULL; + if (vol->logfile_ino) { + iput(vol->logfile_ino); + vol->logfile_ino = NULL; + } + if (vol->mftmirr_ino) { + iput(vol->mftmirr_ino); + vol->mftmirr_ino = NULL; + } + /* Throw away the table of attribute definitions. */ + vol->attrdef_size = 0; + if (vol->attrdef) { + kvfree(vol->attrdef); + vol->attrdef = NULL; + } + vol->upcase_len = 0; + mutex_lock(&ntfs_lock); + if (vol->upcase && vol->upcase == default_upcase) { + ntfs_nr_upcase_users--; + vol->upcase = NULL; + } + mutex_unlock(&ntfs_lock); + if (vol->upcase) { + kvfree(vol->upcase); + vol->upcase = NULL; + } + if (vol->nls_map) { + unload_nls(vol->nls_map); + vol->nls_map = NULL; + } + /* Error exit code path. */ +unl_upcase_iput_tmp_ino_err_out_now: + /* + * Decrease the number of upcase users and destroy the global default + * upcase table if necessary. + */ + mutex_lock(&ntfs_lock); + if (!--ntfs_nr_upcase_users && default_upcase) { + kvfree(default_upcase); + default_upcase = NULL; + } + + mutex_unlock(&ntfs_lock); +iput_tmp_ino_err_out_now: + iput(tmp_ino); + if (vol->mft_ino && vol->mft_ino != tmp_ino) + iput(vol->mft_ino); + vol->mft_ino = NULL; + /* Errors at this stage are irrelevant. */ +err_out_now: + sb->s_fs_info = NULL; + kvfree(vol->lcn_empty_bits_per_page); + kfree(vol->volume_label); + unload_nls(vol->nls_map); + kfree(vol); + ntfs_debug("Failed, returning -EINVAL."); + lockdep_on(); + return -EINVAL; +} + +/* + * This is a slab cache to optimize allocations and deallocations of Unicode + * strings of the maximum length allowed by NTFS, which is NTFS_MAX_NAME_LEN + * (255) Unicode characters + a terminating NULL Unicode character. + */ +struct kmem_cache *ntfs_name_cache; + +/* Slab caches for efficient allocation/deallocation of inodes. */ +struct kmem_cache *ntfs_inode_cache; +struct kmem_cache *ntfs_big_inode_cache; + +/* Init once constructor for the inode slab cache. */ +static void ntfs_big_inode_init_once(void *foo) +{ + struct ntfs_inode *ni = foo; + + inode_init_once(VFS_I(ni)); +} + +/* + * Slab caches to optimize allocations and deallocations of attribute search + * contexts and index contexts, respectively. + */ +struct kmem_cache *ntfs_attr_ctx_cache; +struct kmem_cache *ntfs_index_ctx_cache; + +/* Driver wide mutex. */ +DEFINE_MUTEX(ntfs_lock); + +static int ntfs_get_tree(struct fs_context *fc) +{ + return get_tree_bdev(fc, ntfs_fill_super); +} + +static void ntfs_free_fs_context(struct fs_context *fc) +{ + struct ntfs_volume *vol = fc->s_fs_info; + + if (vol) + ntfs_volume_free(vol); +} + +static const struct fs_context_operations ntfs_context_ops = { + .parse_param = ntfs_parse_param, + .get_tree = ntfs_get_tree, + .free = ntfs_free_fs_context, + .reconfigure = ntfs_reconfigure, +}; + +static int ntfs_init_fs_context(struct fs_context *fc) +{ + struct ntfs_volume *vol; + + /* Allocate a new struct ntfs_volume and place it in sb->s_fs_info. */ + vol = kmalloc(sizeof(struct ntfs_volume), GFP_NOFS); + if (!vol) + return -ENOMEM; + + /* Initialize struct ntfs_volume structure. */ + *vol = (struct ntfs_volume) { + .uid = INVALID_UID, + .gid = INVALID_GID, + .fmask = 0, + .dmask = 0, + .mft_zone_multiplier = 1, + .on_errors = ON_ERRORS_CONTINUE, + .nls_map = load_nls_default(), + .preallocated_size = NTFS_DEF_PREALLOC_SIZE, + }; + + NVolSetShowHiddenFiles(vol); + NVolSetCaseSensitive(vol); + init_rwsem(&vol->mftbmp_lock); + init_rwsem(&vol->lcnbmp_lock); + mutex_init(&vol->volume_label_lock); + + fc->s_fs_info = vol; + fc->ops = &ntfs_context_ops; + return 0; +} + +static struct file_system_type ntfs_fs_type = { + .owner = THIS_MODULE, + .name = "ntfs", + .init_fs_context = ntfs_init_fs_context, + .parameters = ntfs_parameters, + .kill_sb = kill_block_super, + .fs_flags = FS_REQUIRES_DEV | FS_ALLOW_IDMAP, +}; +MODULE_ALIAS_FS("ntfs"); + +static int ntfs_workqueue_init(void) +{ + ntfs_wq = alloc_workqueue("ntfs-bg-io", WQ_PERCPU, 0); + if (!ntfs_wq) + return -ENOMEM; + return 0; +} + +static void ntfs_workqueue_destroy(void) +{ + destroy_workqueue(ntfs_wq); + ntfs_wq = NULL; +} + +/* Stable names for the slab caches. */ +static const char ntfs_index_ctx_cache_name[] = "ntfs_index_ctx_cache"; +static const char ntfs_attr_ctx_cache_name[] = "ntfs_attr_ctx_cache"; +static const char ntfs_name_cache_name[] = "ntfs_name_cache"; +static const char ntfs_inode_cache_name[] = "ntfs_inode_cache"; +static const char ntfs_big_inode_cache_name[] = "ntfs_big_inode_cache"; + +static int __init init_ntfs_fs(void) +{ + int err = 0; + + err = ntfs_workqueue_init(); + if (err) { + pr_crit("Failed to register workqueue!\n"); + return err; + } + + ntfs_index_ctx_cache = kmem_cache_create(ntfs_index_ctx_cache_name, + sizeof(struct ntfs_index_context), 0 /* offset */, + SLAB_HWCACHE_ALIGN, NULL /* ctor */); + if (!ntfs_index_ctx_cache) { + pr_crit("Failed to create %s!\n", ntfs_index_ctx_cache_name); + goto ictx_err_out; + } + ntfs_attr_ctx_cache = kmem_cache_create(ntfs_attr_ctx_cache_name, + sizeof(struct ntfs_attr_search_ctx), 0 /* offset */, + SLAB_HWCACHE_ALIGN, NULL /* ctor */); + if (!ntfs_attr_ctx_cache) { + pr_crit("NTFS: Failed to create %s!\n", + ntfs_attr_ctx_cache_name); + goto actx_err_out; + } + + ntfs_name_cache = kmem_cache_create(ntfs_name_cache_name, + (NTFS_MAX_NAME_LEN+2) * sizeof(__le16), 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!ntfs_name_cache) { + pr_crit("Failed to create %s!\n", ntfs_name_cache_name); + goto name_err_out; + } + + ntfs_inode_cache = kmem_cache_create(ntfs_inode_cache_name, + sizeof(struct ntfs_inode), 0, SLAB_RECLAIM_ACCOUNT, NULL); + if (!ntfs_inode_cache) { + pr_crit("Failed to create %s!\n", ntfs_inode_cache_name); + goto inode_err_out; + } + + ntfs_big_inode_cache = kmem_cache_create(ntfs_big_inode_cache_name, + sizeof(struct big_ntfs_inode), 0, SLAB_HWCACHE_ALIGN | + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, + ntfs_big_inode_init_once); + if (!ntfs_big_inode_cache) { + pr_crit("Failed to create %s!\n", ntfs_big_inode_cache_name); + goto big_inode_err_out; + } + + /* Register the ntfs sysctls. */ + err = ntfs_sysctl(1); + if (err) { + pr_crit("Failed to register NTFS sysctls!\n"); + goto sysctl_err_out; + } + + err = register_filesystem(&ntfs_fs_type); + if (!err) { + ntfs_debug("NTFS driver registered successfully."); + return 0; /* Success! */ + } + pr_crit("Failed to register NTFS filesystem driver!\n"); + + /* Unregister the ntfs sysctls. */ + ntfs_sysctl(0); +sysctl_err_out: + kmem_cache_destroy(ntfs_big_inode_cache); +big_inode_err_out: + kmem_cache_destroy(ntfs_inode_cache); +inode_err_out: + kmem_cache_destroy(ntfs_name_cache); +name_err_out: + kmem_cache_destroy(ntfs_attr_ctx_cache); +actx_err_out: + kmem_cache_destroy(ntfs_index_ctx_cache); +ictx_err_out: + if (!err) { + pr_crit("Aborting NTFS filesystem driver registration...\n"); + err = -ENOMEM; + } + return err; +} + +static void __exit exit_ntfs_fs(void) +{ + ntfs_debug("Unregistering NTFS driver."); + + unregister_filesystem(&ntfs_fs_type); + + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(ntfs_big_inode_cache); + kmem_cache_destroy(ntfs_inode_cache); + kmem_cache_destroy(ntfs_name_cache); + kmem_cache_destroy(ntfs_attr_ctx_cache); + kmem_cache_destroy(ntfs_index_ctx_cache); + ntfs_workqueue_destroy(); + /* Unregister the ntfs sysctls. */ + ntfs_sysctl(0); +} + +module_init(init_ntfs_fs); +module_exit(exit_ntfs_fs); + +MODULE_AUTHOR("Anton Altaparmakov "); /* Original read-only NTFS driver */ +MODULE_AUTHOR("Namjae Jeon "); /* Add write, iomap and various features */ +MODULE_DESCRIPTION("NTFS read-write filesystem driver"); +MODULE_LICENSE("GPL"); +#ifdef DEBUG +module_param(debug_msgs, uint, 0); +MODULE_PARM_DESC(debug_msgs, "Enable debug messages."); +#endif diff --git a/fs/ntfs/sysctl.c b/fs/ntfs/sysctl.c new file mode 100644 index 000000000..8c5c1b7cd --- /dev/null +++ b/fs/ntfs/sysctl.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Code for sysctl handling in NTFS Linux kernel driver. + * + * Copyright (C) 1997 Martin von Löwis, Régis Duchesne + * Copyright (c) 2002-2005 Anton Altaparmakov + */ + +#ifdef DEBUG + +#include + +#ifdef CONFIG_SYSCTL + +#include +#include + +#include "sysctl.h" +#include "debug.h" + +/* Definition of the ntfs sysctl. */ +static const struct ctl_table ntfs_sysctls[] = { + { + .procname = "ntfs-debug", + .data = &debug_msgs, /* Data pointer and size. */ + .maxlen = sizeof(debug_msgs), + .mode = 0644, /* Mode, proc handler. */ + .proc_handler = proc_dointvec + }, +}; + +/* Storage for the sysctls header. */ +static struct ctl_table_header *sysctls_root_table; + +/* + * ntfs_sysctl - add or remove the debug sysctl + * @add: add (1) or remove (0) the sysctl + * + * Add or remove the debug sysctl. Return 0 on success or -errno on error. + */ +int ntfs_sysctl(int add) +{ + if (add) { + sysctls_root_table = register_sysctl("fs/ntfs", ntfs_sysctls); + if (!sysctls_root_table) + return -ENOMEM; + } else { + unregister_sysctl_table(sysctls_root_table); + sysctls_root_table = NULL; + } + return 0; +} +#endif /* CONFIG_SYSCTL */ +#endif /* DEBUG */ diff --git a/fs/ntfs/sysctl.h b/fs/ntfs/sysctl.h new file mode 100644 index 000000000..037127c99 --- /dev/null +++ b/fs/ntfs/sysctl.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Defines for sysctl handling in NTFS Linux kernel driver. + * + * Copyright (C) 1997 Martin von Löwis, Régis Duchesne + * Copyright (c) 2002-2004 Anton Altaparmakov + */ + +#ifndef _LINUX_NTFS_SYSCTL_H +#define _LINUX_NTFS_SYSCTL_H + + +#if defined(DEBUG) && defined(CONFIG_SYSCTL) + +int ntfs_sysctl(int add); + +#else + +/* Just return success. */ +static inline int ntfs_sysctl(int add) +{ + return 0; +} + +#endif /* DEBUG && CONFIG_SYSCTL */ +#endif /* _LINUX_NTFS_SYSCTL_H */ diff --git a/fs/ntfs/time.h b/fs/ntfs/time.h new file mode 100644 index 000000000..aeed195bd --- /dev/null +++ b/fs/ntfs/time.h @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * NTFS time conversion functions. + * + * Copyright (c) 2001-2005 Anton Altaparmakov + */ + +#ifndef _LINUX_NTFS_TIME_H +#define _LINUX_NTFS_TIME_H + +#include +#include /* For do_div(). */ + +#define NTFS_TIME_OFFSET ((s64)(369 * 365 + 89) * 24 * 3600) + +/* + * utc2ntfs - convert Linux UTC time to NTFS time + * @ts: Linux UTC time to convert to NTFS time + * + * Convert the Linux UTC time @ts to its corresponding NTFS time and return + * that in little endian format. + * + * Linux stores time in a struct timespec64 consisting of a time64_t tv_sec + * and a long tv_nsec where tv_sec is the number of 1-second intervals since + * 1st January 1970, 00:00:00 UTC and tv_nsec is the number of 1-nano-second + * intervals since the value of tv_sec. + * + * NTFS uses Microsoft's standard time format which is stored in a s64 and is + * measured as the number of 100-nano-second intervals since 1st January 1601, + * 00:00:00 UTC. + */ +static inline __le64 utc2ntfs(const struct timespec64 ts) +{ + /* + * Convert the seconds to 100ns intervals, add the nano-seconds + * converted to 100ns intervals, and then add the NTFS time offset. + */ + return cpu_to_le64((u64)(ts.tv_sec + NTFS_TIME_OFFSET) * 10000000 + + ts.tv_nsec / 100); +} + +/* + * get_current_ntfs_time - get the current time in little endian NTFS format + * + * Get the current time from the Linux kernel, convert it to its corresponding + * NTFS time and return that in little endian format. + */ +static inline __le64 get_current_ntfs_time(void) +{ + struct timespec64 ts; + + ktime_get_coarse_real_ts64(&ts); + return utc2ntfs(ts); +} + +/* + * ntfs2utc - convert NTFS time to Linux time + * @time: NTFS time (little endian) to convert to Linux UTC + * + * Convert the little endian NTFS time @time to its corresponding Linux UTC + * time and return that in cpu format. + * + * Linux stores time in a struct timespec64 consisting of a time64_t tv_sec + * and a long tv_nsec where tv_sec is the number of 1-second intervals since + * 1st January 1970, 00:00:00 UTC and tv_nsec is the number of 1-nano-second + * intervals since the value of tv_sec. + * + * NTFS uses Microsoft's standard time format which is stored in a s64 and is + * measured as the number of 100 nano-second intervals since 1st January 1601, + * 00:00:00 UTC. + */ +static inline struct timespec64 ntfs2utc(const __le64 time) +{ + struct timespec64 ts; + s32 t32; + + /* Subtract the NTFS time offset. */ + s64 t = le64_to_cpu(time) - NTFS_TIME_OFFSET * 10000000; + /* + * Convert the time to 1-second intervals and the remainder to + * 1-nano-second intervals. + */ + ts.tv_sec = div_s64_rem(t, 10000000, &t32); + ts.tv_nsec = t32 * 100; + return ts; +} +#endif /* _LINUX_NTFS_TIME_H */ diff --git a/fs/ntfs/unistr.c b/fs/ntfs/unistr.c new file mode 100644 index 000000000..7f11a2825 --- /dev/null +++ b/fs/ntfs/unistr.c @@ -0,0 +1,477 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * NTFS Unicode string handling. + * + * Copyright (c) 2001-2006 Anton Altaparmakov + */ + +#include "ntfs.h" + +/* + * IMPORTANT + * ========= + * + * All these routines assume that the Unicode characters are in little endian + * encoding inside the strings!!! + */ + +/* + * This is used by the name collation functions to quickly determine what + * characters are (in)valid. + */ +static const u8 legal_ansi_char_array[0x40] = { + 0x00, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, + + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, + + 0x17, 0x07, 0x18, 0x17, 0x17, 0x17, 0x17, 0x17, + 0x17, 0x17, 0x18, 0x16, 0x16, 0x17, 0x07, 0x00, + + 0x17, 0x17, 0x17, 0x17, 0x17, 0x17, 0x17, 0x17, + 0x17, 0x17, 0x04, 0x16, 0x18, 0x16, 0x18, 0x18, +}; + +/* + * ntfs_are_names_equal - compare two Unicode names for equality + * @s1: name to compare to @s2 + * @s1_len: length in Unicode characters of @s1 + * @s2: name to compare to @s1 + * @s2_len: length in Unicode characters of @s2 + * @ic: ignore case bool + * @upcase: upcase table (only if @ic == IGNORE_CASE) + * @upcase_size: length in Unicode characters of @upcase (if present) + * + * Compare the names @s1 and @s2 and return 'true' (1) if the names are + * identical, or 'false' (0) if they are not identical. If @ic is IGNORE_CASE, + * the @upcase table is used to performa a case insensitive comparison. + */ +bool ntfs_are_names_equal(const __le16 *s1, size_t s1_len, + const __le16 *s2, size_t s2_len, const u32 ic, + const __le16 *upcase, const u32 upcase_size) +{ + if (s1_len != s2_len) + return false; + if (ic == CASE_SENSITIVE) + return !ntfs_ucsncmp(s1, s2, s1_len); + return !ntfs_ucsncasecmp(s1, s2, s1_len, upcase, upcase_size); +} + +/* + * ntfs_collate_names - collate two Unicode names + * @name1: first Unicode name to compare + * @name1_len: first Unicode name length + * @name2: second Unicode name to compare + * @name2_len: second Unicode name length + * @err_val: if @name1 contains an invalid character return this value + * @ic: either CASE_SENSITIVE or IGNORE_CASE + * @upcase: upcase table (ignored if @ic is CASE_SENSITIVE) + * @upcase_len: upcase table size (ignored if @ic is CASE_SENSITIVE) + * + * ntfs_collate_names collates two Unicode names and returns: + * + * -1 if the first name collates before the second one, + * 0 if the names match, + * 1 if the second name collates before the first one, or + * @err_val if an invalid character is found in @name1 during the comparison. + * + * The following characters are considered invalid: '"', '*', '<', '>' and '?'. + */ +int ntfs_collate_names(const __le16 *name1, const u32 name1_len, + const __le16 *name2, const u32 name2_len, + const int err_val, const u32 ic, + const __le16 *upcase, const u32 upcase_len) +{ + u32 cnt, min_len; + u16 c1, c2; + + min_len = name1_len; + if (name1_len > name2_len) + min_len = name2_len; + for (cnt = 0; cnt < min_len; ++cnt) { + c1 = le16_to_cpu(*name1++); + c2 = le16_to_cpu(*name2++); + if (ic) { + if (c1 < upcase_len) + c1 = le16_to_cpu(upcase[c1]); + if (c2 < upcase_len) + c2 = le16_to_cpu(upcase[c2]); + } + if (c1 < 64 && legal_ansi_char_array[c1] & 8) + return err_val; + if (c1 < c2) + return -1; + if (c1 > c2) + return 1; + } + if (name1_len < name2_len) + return -1; + if (name1_len == name2_len) + return 0; + /* name1_len > name2_len */ + c1 = le16_to_cpu(*name1); + if (c1 < 64 && legal_ansi_char_array[c1] & 8) + return err_val; + return 1; +} + +/* + * ntfs_ucsncmp - compare two little endian Unicode strings + * @s1: first string + * @s2: second string + * @n: maximum unicode characters to compare + * + * Compare the first @n characters of the Unicode strings @s1 and @s2, + * The strings in little endian format and appropriate le16_to_cpu() + * conversion is performed on non-little endian machines. + * + * The function returns an integer less than, equal to, or greater than zero + * if @s1 (or the first @n Unicode characters thereof) is found, respectively, + * to be less than, to match, or be greater than @s2. + */ +int ntfs_ucsncmp(const __le16 *s1, const __le16 *s2, size_t n) +{ + u16 c1, c2; + size_t i; + + for (i = 0; i < n; ++i) { + c1 = le16_to_cpu(s1[i]); + c2 = le16_to_cpu(s2[i]); + if (c1 < c2) + return -1; + if (c1 > c2) + return 1; + if (!c1) + break; + } + return 0; +} + +/* + * ntfs_ucsncasecmp - compare two little endian Unicode strings, ignoring case + * @s1: first string + * @s2: second string + * @n: maximum unicode characters to compare + * @upcase: upcase table + * @upcase_size: upcase table size in Unicode characters + * + * Compare the first @n characters of the Unicode strings @s1 and @s2, + * ignoring case. The strings in little endian format and appropriate + * le16_to_cpu() conversion is performed on non-little endian machines. + * + * Each character is uppercased using the @upcase table before the comparison. + * + * The function returns an integer less than, equal to, or greater than zero + * if @s1 (or the first @n Unicode characters thereof) is found, respectively, + * to be less than, to match, or be greater than @s2. + */ +int ntfs_ucsncasecmp(const __le16 *s1, const __le16 *s2, size_t n, + const __le16 *upcase, const u32 upcase_size) +{ + size_t i; + u16 c1, c2; + + for (i = 0; i < n; ++i) { + c1 = le16_to_cpu(s1[i]); + if (c1 < upcase_size) + c1 = le16_to_cpu(upcase[c1]); + c2 = le16_to_cpu(s2[i]); + if (c2 < upcase_size) + c2 = le16_to_cpu(upcase[c2]); + if (c1 < c2) + return -1; + if (c1 > c2) + return 1; + if (!c1) + break; + } + return 0; +} + +int ntfs_file_compare_values(const struct file_name_attr *file_name_attr1, + const struct file_name_attr *file_name_attr2, + const int err_val, const u32 ic, + const __le16 *upcase, const u32 upcase_len) +{ + return ntfs_collate_names((__le16 *)&file_name_attr1->file_name, + file_name_attr1->file_name_length, + (__le16 *)&file_name_attr2->file_name, + file_name_attr2->file_name_length, + err_val, ic, upcase, upcase_len); +} + +/* + * ntfs_nlstoucs - convert NLS string to little endian Unicode string + * @vol: ntfs volume which we are working with + * @ins: input NLS string buffer + * @ins_len: length of input string in bytes + * @outs: on return contains the allocated output Unicode string buffer + * @max_name_len: maximum number of Unicode characters allowed for the output name + * + * Convert the input string @ins, which is in whatever format the loaded NLS + * map dictates, into a little endian, 2-byte Unicode string. + * + * This function allocates the string and the caller is responsible for + * calling kmem_cache_free(ntfs_name_cache, *@outs); when finished with it. + * + * On success the function returns the number of Unicode characters written to + * the output string *@outs (>= 0), not counting the terminating Unicode NULL + * character. *@outs is set to the allocated output string buffer. + * + * On error, a negative number corresponding to the error code is returned. In + * that case the output string is not allocated. Both *@outs and *@outs_len + * are then undefined. + * + * This might look a bit odd due to fast path optimization... + */ +int ntfs_nlstoucs(const struct ntfs_volume *vol, const char *ins, + const int ins_len, __le16 **outs, int max_name_len) +{ + struct nls_table *nls = vol->nls_map; + __le16 *ucs; + wchar_t wc; + int i, o, wc_len; + + /* We do not trust outside sources. */ + if (likely(ins)) { + if (max_name_len > NTFS_MAX_NAME_LEN) + ucs = kvmalloc((max_name_len + 2) * sizeof(__le16), + GFP_NOFS | __GFP_ZERO); + else + ucs = kmem_cache_alloc(ntfs_name_cache, GFP_NOFS); + if (likely(ucs)) { + if (vol->nls_utf8) { + o = utf8s_to_utf16s(ins, ins_len, + UTF16_LITTLE_ENDIAN, + (wchar_t *)ucs, + max_name_len + 2); + if (o < 0 || o > max_name_len) { + wc_len = o; + goto name_err; + } + } else { + for (i = o = 0; i < ins_len; i += wc_len) { + wc_len = nls->char2uni(ins + i, ins_len - i, + &wc); + if (likely(wc_len >= 0 && + o < max_name_len)) { + if (likely(wc)) { + ucs[o++] = cpu_to_le16(wc); + continue; + } /* else if (!wc) */ + break; + } + + goto name_err; + } + } + ucs[o] = 0; + *outs = ucs; + return o; + } /* else if (!ucs) */ + ntfs_debug("Failed to allocate buffer for converted name from ntfs_name_cache."); + return -ENOMEM; + } /* else if (!ins) */ + ntfs_error(vol->sb, "Received NULL pointer."); + return -EINVAL; +name_err: + if (max_name_len > NTFS_MAX_NAME_LEN) + kvfree(ucs); + else + kmem_cache_free(ntfs_name_cache, ucs); + if (wc_len < 0) { + ntfs_debug("Name using character set %s contains characters that cannot be converted to Unicode.", + nls->charset); + i = -EILSEQ; + } else { + ntfs_debug("Name is too long (maximum length for a name on NTFS is %d Unicode characters.", + max_name_len); + i = -ENAMETOOLONG; + } + return i; +} + +/* + * ntfs_ucstonls - convert little endian Unicode string to NLS string + * @vol: ntfs volume which we are working with + * @ins: input Unicode string buffer + * @ins_len: length of input string in Unicode characters + * @outs: on return contains the (allocated) output NLS string buffer + * @outs_len: length of output string buffer in bytes + * + * Convert the input little endian, 2-byte Unicode string @ins, of length + * @ins_len into the string format dictated by the loaded NLS. + * + * If *@outs is NULL, this function allocates the string and the caller is + * responsible for calling kfree(*@outs); when finished with it. In this case + * @outs_len is ignored and can be 0. + * + * On success the function returns the number of bytes written to the output + * string *@outs (>= 0), not counting the terminating NULL byte. If the output + * string buffer was allocated, *@outs is set to it. + * + * On error, a negative number corresponding to the error code is returned. In + * that case the output string is not allocated. The contents of *@outs are + * then undefined. + * + * This might look a bit odd due to fast path optimization... + */ +int ntfs_ucstonls(const struct ntfs_volume *vol, const __le16 *ins, + const int ins_len, unsigned char **outs, int outs_len) +{ + struct nls_table *nls = vol->nls_map; + unsigned char *ns; + int i, o, ns_len, wc; + + /* We don't trust outside sources. */ + if (ins) { + ns = *outs; + ns_len = outs_len; + if (ns && !ns_len) { + wc = -ENAMETOOLONG; + goto conversion_err; + } + if (!ns) { + ns_len = ins_len * NLS_MAX_CHARSET_SIZE; + ns = kmalloc(ns_len + 1, GFP_NOFS); + if (!ns) + goto mem_err_out; + } + + if (vol->nls_utf8) { + o = utf16s_to_utf8s((const wchar_t *)ins, ins_len, + UTF16_LITTLE_ENDIAN, ns, ns_len); + if (o >= ns_len) { + wc = -ENAMETOOLONG; + goto conversion_err; + } + goto done; + } + + for (i = o = 0; i < ins_len; i++) { +retry: + wc = nls->uni2char(le16_to_cpu(ins[i]), ns + o, + ns_len - o); + if (wc > 0) { + o += wc; + continue; + } else if (!wc) + break; + else if (wc == -ENAMETOOLONG && ns != *outs) { + unsigned char *tc; + /* Grow in multiples of 64 bytes. */ + tc = kmalloc((ns_len + 64) & + ~63, GFP_NOFS); + if (tc) { + memcpy(tc, ns, ns_len); + ns_len = ((ns_len + 64) & ~63) - 1; + kfree(ns); + ns = tc; + goto retry; + } /* No memory so goto conversion_error; */ + } /* wc < 0, real error. */ + goto conversion_err; + } +done: + ns[o] = 0; + *outs = ns; + return o; + } /* else (!ins) */ + ntfs_error(vol->sb, "Received NULL pointer."); + return -EINVAL; +conversion_err: + ntfs_error(vol->sb, + "Unicode name contains characters that cannot be converted to character set %s. You might want to try to use the mount option nls=utf8.", + nls->charset); + if (ns != *outs) + kfree(ns); + if (wc != -ENAMETOOLONG) + wc = -EILSEQ; + return wc; +mem_err_out: + ntfs_error(vol->sb, "Failed to allocate name!"); + return -ENOMEM; +} + +/* + * ntfs_ucsnlen - determine the length of a little endian Unicode string + * @s: pointer to Unicode string + * @maxlen: maximum length of string @s + * + * Return the number of Unicode characters in the little endian Unicode + * string @s up to a maximum of maxlen Unicode characters, not including + * the terminating (__le16)'\0'. If there is no (__le16)'\0' between @s + * and @s + @maxlen, @maxlen is returned. + * + * This function never looks beyond @s + @maxlen. + */ +static u32 ntfs_ucsnlen(const __le16 *s, u32 maxlen) +{ + u32 i; + + for (i = 0; i < maxlen; i++) { + if (!le16_to_cpu(s[i])) + break; + } + return i; +} + +/* + * ntfs_ucsndup - duplicate little endian Unicode string + * @s: pointer to Unicode string + * @maxlen: maximum length of string @s + * + * Return a pointer to a new little endian Unicode string which is a duplicate + * of the string s. Memory for the new string is obtained with kmalloc, + * and can be freed with kfree. + * + * A maximum of @maxlen Unicode characters are copied and a terminating + * (__le16)'\0' little endian Unicode character is added. + * + * This function never looks beyond @s + @maxlen. + * + * Return a pointer to the new little endian Unicode string on success and NULL + * on failure with errno set to the error code. + */ +__le16 *ntfs_ucsndup(const __le16 *s, u32 maxlen) +{ + __le16 *dst; + u32 len; + + len = ntfs_ucsnlen(s, maxlen); + dst = kmalloc((len + 1) * sizeof(__le16), GFP_NOFS); + if (dst) { + memcpy(dst, s, len * sizeof(__le16)); + dst[len] = cpu_to_le16(L'\0'); + } + return dst; +} + +/* + * ntfs_names_are_equal - compare two Unicode names for equality + * @s1: name to compare to @s2 + * @s1_len: length in Unicode characters of @s1 + * @s2: name to compare to @s1 + * @s2_len: length in Unicode characters of @s2 + * @ic: ignore case bool + * @upcase: upcase table (only if @ic == IGNORE_CASE) + * @upcase_size: length in Unicode characters of @upcase (if present) + * + * Compare the names @s1 and @s2 and return TRUE (1) if the names are + * identical, or FALSE (0) if they are not identical. If @ic is IGNORE_CASE, + * the @upcase table is used to perform a case insensitive comparison. + */ +bool ntfs_names_are_equal(const __le16 *s1, size_t s1_len, + const __le16 *s2, size_t s2_len, + const u32 ic, + const __le16 *upcase, const u32 upcase_size) +{ + if (s1_len != s2_len) + return false; + if (!s1_len) + return true; + if (ic == CASE_SENSITIVE) + return ntfs_ucsncmp(s1, s2, s1_len) ? false : true; + return ntfs_ucsncasecmp(s1, s2, s1_len, upcase, upcase_size) ? false : true; +} diff --git a/fs/ntfs/upcase.c b/fs/ntfs/upcase.c new file mode 100644 index 000000000..4b9544708 --- /dev/null +++ b/fs/ntfs/upcase.c @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Generate the full NTFS Unicode upcase table in little endian. + * + * Copyright (c) 2001 Richard Russon + * Copyright (c) 2001-2006 Anton Altaparmakov + */ + +#include "ntfs.h" + +__le16 *generate_default_upcase(void) +{ + static const int uc_run_table[][3] = { /* Start, End, Add */ + {0x0061, 0x007B, -32}, {0x0451, 0x045D, -80}, {0x1F70, 0x1F72, 74}, + {0x00E0, 0x00F7, -32}, {0x045E, 0x0460, -80}, {0x1F72, 0x1F76, 86}, + {0x00F8, 0x00FF, -32}, {0x0561, 0x0587, -48}, {0x1F76, 0x1F78, 100}, + {0x0256, 0x0258, -205}, {0x1F00, 0x1F08, 8}, {0x1F78, 0x1F7A, 128}, + {0x028A, 0x028C, -217}, {0x1F10, 0x1F16, 8}, {0x1F7A, 0x1F7C, 112}, + {0x03AC, 0x03AD, -38}, {0x1F20, 0x1F28, 8}, {0x1F7C, 0x1F7E, 126}, + {0x03AD, 0x03B0, -37}, {0x1F30, 0x1F38, 8}, {0x1FB0, 0x1FB2, 8}, + {0x03B1, 0x03C2, -32}, {0x1F40, 0x1F46, 8}, {0x1FD0, 0x1FD2, 8}, + {0x03C2, 0x03C3, -31}, {0x1F51, 0x1F52, 8}, {0x1FE0, 0x1FE2, 8}, + {0x03C3, 0x03CC, -32}, {0x1F53, 0x1F54, 8}, {0x1FE5, 0x1FE6, 7}, + {0x03CC, 0x03CD, -64}, {0x1F55, 0x1F56, 8}, {0x2170, 0x2180, -16}, + {0x03CD, 0x03CF, -63}, {0x1F57, 0x1F58, 8}, {0x24D0, 0x24EA, -26}, + {0x0430, 0x0450, -32}, {0x1F60, 0x1F68, 8}, {0xFF41, 0xFF5B, -32}, + {0} + }; + + static const int uc_dup_table[][2] = { /* Start, End */ + {0x0100, 0x012F}, {0x01A0, 0x01A6}, {0x03E2, 0x03EF}, {0x04CB, 0x04CC}, + {0x0132, 0x0137}, {0x01B3, 0x01B7}, {0x0460, 0x0481}, {0x04D0, 0x04EB}, + {0x0139, 0x0149}, {0x01CD, 0x01DD}, {0x0490, 0x04BF}, {0x04EE, 0x04F5}, + {0x014A, 0x0178}, {0x01DE, 0x01EF}, {0x04BF, 0x04BF}, {0x04F8, 0x04F9}, + {0x0179, 0x017E}, {0x01F4, 0x01F5}, {0x04C1, 0x04C4}, {0x1E00, 0x1E95}, + {0x018B, 0x018B}, {0x01FA, 0x0218}, {0x04C7, 0x04C8}, {0x1EA0, 0x1EF9}, + {0} + }; + + static const int uc_word_table[][2] = { /* Offset, Value */ + {0x00FF, 0x0178}, {0x01AD, 0x01AC}, {0x01F3, 0x01F1}, {0x0269, 0x0196}, + {0x0183, 0x0182}, {0x01B0, 0x01AF}, {0x0253, 0x0181}, {0x026F, 0x019C}, + {0x0185, 0x0184}, {0x01B9, 0x01B8}, {0x0254, 0x0186}, {0x0272, 0x019D}, + {0x0188, 0x0187}, {0x01BD, 0x01BC}, {0x0259, 0x018F}, {0x0275, 0x019F}, + {0x018C, 0x018B}, {0x01C6, 0x01C4}, {0x025B, 0x0190}, {0x0283, 0x01A9}, + {0x0192, 0x0191}, {0x01C9, 0x01C7}, {0x0260, 0x0193}, {0x0288, 0x01AE}, + {0x0199, 0x0198}, {0x01CC, 0x01CA}, {0x0263, 0x0194}, {0x0292, 0x01B7}, + {0x01A8, 0x01A7}, {0x01DD, 0x018E}, {0x0268, 0x0197}, + {0} + }; + + int i, r; + __le16 *uc; + + uc = kvcalloc(default_upcase_len, sizeof(__le16), GFP_NOFS); + if (!uc) + return uc; + /* Generate the little endian Unicode upcase table used by ntfs. */ + for (i = 0; i < default_upcase_len; i++) + uc[i] = cpu_to_le16(i); + for (r = 0; uc_run_table[r][0]; r++) + for (i = uc_run_table[r][0]; i < uc_run_table[r][1]; i++) + le16_add_cpu(&uc[i], uc_run_table[r][2]); + for (r = 0; uc_dup_table[r][0]; r++) + for (i = uc_dup_table[r][0]; i < uc_dup_table[r][1]; i += 2) + le16_add_cpu(&uc[i + 1], -1); + for (r = 0; uc_word_table[r][0]; r++) + uc[uc_word_table[r][0]] = cpu_to_le16(uc_word_table[r][1]); + return uc; +} diff --git a/fs/ntfs/volume.h b/fs/ntfs/volume.h new file mode 100644 index 000000000..65fd3908a --- /dev/null +++ b/fs/ntfs/volume.h @@ -0,0 +1,296 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Defines for volume structures in NTFS Linux kernel driver. + * + * Copyright (c) 2001-2006 Anton Altaparmakov + * Copyright (c) 2002 Richard Russon + * Copyright (c) 2025 LG Electronics Co., Ltd. + */ + +#ifndef _LINUX_NTFS_VOLUME_H +#define _LINUX_NTFS_VOLUME_H + +#include +#include +#include +#include +#include +#include + +#include "layout.h" + +#define NTFS_VOL_UID BIT(1) +#define NTFS_VOL_GID BIT(2) + +/* + * The NTFS in memory super block structure. + * + * @sb: Pointer back to the super_block. + * @nr_blocks: Number of sb->s_blocksize bytes sized blocks on the device. + * @flags: Miscellaneous flags, see below. + * @uid: uid that files will be mounted as. + * @gid: gid that files will be mounted as. + * @fmask: The mask for file permissions. + * @dmask: The mask for directory permissions. + * @mft_zone_multiplier: Initial mft zone multiplier. + * @on_errors: What to do on filesystem errors. + * @wb_err: Writeback error tracking. + * @sector_size: in bytes + * @sector_size_bits: log2(sector_size) + * @cluster_size: in bytes + * @cluster_size_mask: cluster_size - 1 + * @cluster_size_bits: log2(cluster_size) + * @mft_record_size: in bytes + * @mft_record_size_mask: mft_record_size - 1 + * @mft_record_size_bits: log2(mft_record_size) + * @index_record_size: in bytes + * @index_record_size_mask: index_record_size - 1 + * @index_record_size_bits: log2(index_record_size) + * @nr_clusters: Volume size in clusters == number of bits in lcn bitmap. + * @mft_lcn: Cluster location of mft data. + * @mftmirr_lcn: Cluster location of copy of mft. + * @serial_no: The volume serial number. + * @upcase_len: Number of entries in upcase[]. + * @upcase: The upcase table. + * @attrdef_size: Size of the attribute definition table in bytes. + * @attrdef: Table of attribute definitions. Obtained from FILE_AttrDef. + * @mft_data_pos: Mft record number at which to allocate the next mft record. + * @mft_zone_start: First cluster of the mft zone. + * @mft_zone_end: First cluster beyond the mft zone. + * @mft_zone_pos: Current position in the mft zone. + * @data1_zone_pos: Current position in the first data zone. + * @data2_zone_pos: Current position in the second data zone. + * @mft_ino: The VFS inode of $MFT. + * @mftbmp_ino: Attribute inode for $MFT/$BITMAP. + * @mftbmp_lock: Lock for serializing accesses to the mft record bitmap. + * @mftmirr_ino: The VFS inode of $MFTMirr. + * @mftmirr_size: Size of mft mirror in mft records. + * @logfile_ino: The VFS inode of LogFile. + * @lcnbmp_ino: The VFS inode of $Bitmap. + * @lcnbmp_lock: Lock for serializing accesses to the cluster bitmap + * @vol_ino: The VFS inode of $Volume. + * @vol_flags: Volume flags. + * @major_ver: Ntfs major version of volume. + * @minor_ver: Ntfs minor version of volume. + * @volume_label_lock: protects @volume_label. + * @volume_label: volume label. + * @root_ino: The VFS inode of the root directory. + * @secure_ino: The VFS inode of $Secure (NTFS3.0+ only, otherwise NULL). + * @extend_ino: The VFS inode of $Extend (NTFS3.0+ only, otherwise NULL). + * @nls_map: NLS (National Language Support) table. + * @nls_utf8: NLS table for UTF-8. + * @free_waitq: Wait queue for threads waiting for free clusters or MFT records. + * @free_clusters: Track the number of free clusters. + * @free_mft_records: Track the free mft records. + * @dirty_clusters: Number of clusters that are dirty. + * @sparse_compression_unit: Size of compression/sparse unit in clusters. + * @lcn_empty_bits_per_page: Number of empty bits per page in the LCN bitmap. + * @precalc_work: Work structure for background pre-calculation tasks. + * @preallocated_size: reallocation size (in bytes). + */ +struct ntfs_volume { + struct super_block *sb; + s64 nr_blocks; + unsigned long flags; + kuid_t uid; + kgid_t gid; + umode_t fmask; + umode_t dmask; + u8 mft_zone_multiplier; + u8 on_errors; + errseq_t wb_err; + u16 sector_size; + u8 sector_size_bits; + u32 cluster_size; + u32 cluster_size_mask; + u8 cluster_size_bits; + u32 mft_record_size; + u32 mft_record_size_mask; + u8 mft_record_size_bits; + u32 index_record_size; + u32 index_record_size_mask; + u8 index_record_size_bits; + s64 nr_clusters; + s64 mft_lcn; + s64 mftmirr_lcn; + u64 serial_no; + u32 upcase_len; + __le16 *upcase; + s32 attrdef_size; + struct attr_def *attrdef; + s64 mft_data_pos; + s64 mft_zone_start; + s64 mft_zone_end; + s64 mft_zone_pos; + s64 data1_zone_pos; + s64 data2_zone_pos; + struct inode *mft_ino; + struct inode *mftbmp_ino; + struct rw_semaphore mftbmp_lock; + struct inode *mftmirr_ino; + int mftmirr_size; + struct inode *logfile_ino; + struct inode *lcnbmp_ino; + struct rw_semaphore lcnbmp_lock; + struct mutex volume_label_lock; + struct inode *vol_ino; + __le16 vol_flags; + u8 major_ver; + u8 minor_ver; + unsigned char *volume_label; + struct inode *root_ino; + struct inode *secure_ino; + struct inode *extend_ino; + struct nls_table *nls_map; + bool nls_utf8; + wait_queue_head_t free_waitq; + atomic64_t free_clusters; + atomic64_t free_mft_records; + atomic64_t dirty_clusters; + u8 sparse_compression_unit; + unsigned int *lcn_empty_bits_per_page; + struct work_struct precalc_work; + loff_t preallocated_size; +}; + +/* + * Defined bits for the flags field in the ntfs_volume structure. + * + * NV_Errors Volume has errors, prevent remount rw. + * NV_ShowSystemFiles Return system files in ntfs_readdir(). + * NV_CaseSensitive Treat file names as case sensitive and + * create filenames in the POSIX namespace. + * Otherwise be case insensitive but still + * create file names in POSIX namespace. + * NV_LogFileEmpty LogFile journal is empty. + * NV_UsnJrnlStamped UsnJrnl has been stamped. + * NV_ReadOnly Volume is mounted read-only. + * NV_Compression Volume supports compression. + * NV_FreeClusterKnown Free cluster count is known and up-to-date. + * NV_Shutdown Volume is in shutdown state + * NV_SysImmutable Protect system files from deletion. + * NV_ShowHiddenFiles Return hidden files in ntfs_readdir(). + * NV_HideDotFiles Hide names beginning with a dot ("."). + * NV_CheckWindowsNames Refuse creation/rename of files with + * Windows-reserved names (CON, AUX, NUL, COM1, + * LPT1, etc.) or invalid characters. + * + * NV_Discard Issue discard/TRIM commands for freed clusters. + * NV_DisableSparse Disable creation of sparse regions. + * NV_NativeSymlinkRel Translate absolute Windows reparse targets (native_symlink=rel). + */ +enum { + NV_Errors, + NV_ShowSystemFiles, + NV_CaseSensitive, + NV_LogFileEmpty, + NV_UsnJrnlStamped, + NV_ReadOnly, + NV_Compression, + NV_FreeClusterKnown, + NV_Shutdown, + NV_SysImmutable, + NV_ShowHiddenFiles, + NV_HideDotFiles, + NV_CheckWindowsNames, + NV_Discard, + NV_DisableSparse, + NV_NativeSymlinkRel, + NV_SymlinkNative, +}; + +/* + * Macro tricks to expand the NVolFoo(), NVolSetFoo(), and NVolClearFoo() + * functions. + */ +#define DEFINE_NVOL_BIT_OPS(flag) \ +static inline int NVol##flag(struct ntfs_volume *vol) \ +{ \ + return test_bit(NV_##flag, &(vol)->flags); \ +} \ +static inline void NVolSet##flag(struct ntfs_volume *vol) \ +{ \ + set_bit(NV_##flag, &(vol)->flags); \ +} \ +static inline void NVolClear##flag(struct ntfs_volume *vol) \ +{ \ + clear_bit(NV_##flag, &(vol)->flags); \ +} + +/* Emit the ntfs volume bitops functions. */ +DEFINE_NVOL_BIT_OPS(Errors) +DEFINE_NVOL_BIT_OPS(ShowSystemFiles) +DEFINE_NVOL_BIT_OPS(CaseSensitive) +DEFINE_NVOL_BIT_OPS(LogFileEmpty) +DEFINE_NVOL_BIT_OPS(UsnJrnlStamped) +DEFINE_NVOL_BIT_OPS(ReadOnly) +DEFINE_NVOL_BIT_OPS(Compression) +DEFINE_NVOL_BIT_OPS(FreeClusterKnown) +DEFINE_NVOL_BIT_OPS(Shutdown) +DEFINE_NVOL_BIT_OPS(SysImmutable) +DEFINE_NVOL_BIT_OPS(ShowHiddenFiles) +DEFINE_NVOL_BIT_OPS(HideDotFiles) +DEFINE_NVOL_BIT_OPS(CheckWindowsNames) +DEFINE_NVOL_BIT_OPS(Discard) +DEFINE_NVOL_BIT_OPS(DisableSparse) +DEFINE_NVOL_BIT_OPS(NativeSymlinkRel) +DEFINE_NVOL_BIT_OPS(SymlinkNative) + +static inline void ntfs_inc_free_clusters(struct ntfs_volume *vol, s64 nr) +{ + if (!NVolFreeClusterKnown(vol)) + wait_event(vol->free_waitq, NVolFreeClusterKnown(vol)); + atomic64_add(nr, &vol->free_clusters); +} + +static inline void ntfs_dec_free_clusters(struct ntfs_volume *vol, s64 nr) +{ + if (!NVolFreeClusterKnown(vol)) + wait_event(vol->free_waitq, NVolFreeClusterKnown(vol)); + atomic64_sub(nr, &vol->free_clusters); +} + +static inline void ntfs_inc_free_mft_records(struct ntfs_volume *vol, s64 nr) +{ + if (!NVolFreeClusterKnown(vol)) + return; + + atomic64_add(nr, &vol->free_mft_records); +} + +static inline void ntfs_dec_free_mft_records(struct ntfs_volume *vol, s64 nr) +{ + if (!NVolFreeClusterKnown(vol)) + return; + + atomic64_sub(nr, &vol->free_mft_records); +} + +static inline void ntfs_set_lcn_empty_bits(struct ntfs_volume *vol, unsigned long index, + u8 val, unsigned int count) +{ + if (!NVolFreeClusterKnown(vol)) + wait_event(vol->free_waitq, NVolFreeClusterKnown(vol)); + + if (val) + vol->lcn_empty_bits_per_page[index] -= count; + else + vol->lcn_empty_bits_per_page[index] += count; +} + +static __always_inline void ntfs_hold_dirty_clusters(struct ntfs_volume *vol, s64 nr_clusters) +{ + atomic64_add(nr_clusters, &vol->dirty_clusters); +} + +static __always_inline void ntfs_release_dirty_clusters(struct ntfs_volume *vol, s64 nr_clusters) +{ + if (atomic64_read(&vol->dirty_clusters) < nr_clusters) + atomic64_set(&vol->dirty_clusters, 0); + else + atomic64_sub(nr_clusters, &vol->dirty_clusters); +} + +s64 ntfs_available_clusters_count(struct ntfs_volume *vol, s64 nr_clusters); +s64 get_nr_free_clusters(struct ntfs_volume *vol); +#endif /* _LINUX_NTFS_VOLUME_H */ diff --git a/fs/ntfs3/Kconfig b/fs/ntfs3/Kconfig new file mode 100644 index 000000000..876dbc613 --- /dev/null +++ b/fs/ntfs3/Kconfig @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: GPL-2.0-only +config NTFS3_FS + tristate "NTFS Read-Write file system support" + depends on !NTFS_FS || m + select BUFFER_HEAD + select NLS + select LEGACY_DIRECT_IO + help + Windows OS native file system (NTFS) support up to NTFS version 3.1. + + Y or M enables the NTFS3 driver with full features enabled (read, + write, journal replaying, sparse/compressed files support). + File system type to use on mount is "ntfs3". Module name (M option) + is also "ntfs3". + + Documentation: + +config NTFS3_64BIT_CLUSTER + bool "64 bits per NTFS clusters" + depends on NTFS3_FS && 64BIT + help + Windows implementation of ntfs.sys uses 32 bits per clusters. + If activated 64 bits per clusters you will be able to use 4k cluster + for 16T+ volumes. Windows will not be able to mount such volumes. + + It is recommended to say N here. + +config NTFS3_LZX_XPRESS + bool "activate support of external compressions lzx/xpress" + depends on NTFS3_FS + help + In Windows 10 one can use command "compact" to compress any files. + 4 possible variants of compression are: xpress4k, xpress8k, xpress16k and lzx. + If activated you will be able to read such files correctly. + + It is recommended to say Y here. + +config NTFS3_FS_POSIX_ACL + bool "NTFS POSIX Access Control Lists" + depends on NTFS3_FS + select FS_POSIX_ACL + help + POSIX Access Control Lists (ACLs) support additional access rights + for users and groups beyond the standard owner/group/world scheme, + and this option selects support for ACLs specifically for ntfs + filesystems. + NOTE: this is linux only feature. Windows will ignore these ACLs. + + If you don't know what Access Control Lists are, say N. diff --git a/fs/ntfs3/Makefile b/fs/ntfs3/Makefile new file mode 100644 index 000000000..53bf2c17a --- /dev/null +++ b/fs/ntfs3/Makefile @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the ntfs3 filesystem support. +# + +# Subset of W=1 warnings +subdir-ccflags-y += -Wextra -Wunused -Wno-unused-parameter +subdir-ccflags-y += -Wmissing-declarations +subdir-ccflags-y += -Wmissing-format-attribute +subdir-ccflags-y += -Wmissing-prototypes +subdir-ccflags-y += -Wold-style-definition +subdir-ccflags-y += -Wmissing-include-dirs +condflags := \ + $(call cc-option, -Wunused-but-set-variable) \ + $(call cc-option, -Wunused-const-variable) \ + $(call cc-option, -Wpacked-not-aligned) \ + $(call cc-option, -Wstringop-truncation) \ + $(call cc-option, -Wmaybe-uninitialized) +subdir-ccflags-y += $(condflags) +# The following turn off the warnings enabled by -Wextra +subdir-ccflags-y += -Wno-missing-field-initializers +subdir-ccflags-y += -Wno-sign-compare +subdir-ccflags-y += -Wno-type-limits +subdir-ccflags-y += -Wno-shift-negative-value + +# to check robot warnings +ccflags-y += -Wint-to-pointer-cast \ + $(call cc-option,-Wunused-but-set-variable,-Wunused-const-variable) \ + $(call cc-option,-Wold-style-declaration,-Wout-of-line-declaration) + +obj-$(CONFIG_NTFS3_FS) += ntfs3.o + +ntfs3-y := attrib.o \ + attrlist.o \ + bitfunc.o \ + bitmap.o \ + dir.o \ + fsntfs.o \ + frecord.o \ + file.o \ + fslog.o \ + inode.o \ + index.o \ + lznt.o \ + namei.o \ + record.o \ + run.o \ + super.o \ + upcase.o \ + xattr.o + +ntfs3-$(CONFIG_NTFS3_LZX_XPRESS) += $(addprefix lib/,\ + decompress_common.o \ + lzx_decompress.o \ + xpress_decompress.o \ + ) \ No newline at end of file diff --git a/fs/ntfs3/attrib.c b/fs/ntfs3/attrib.c new file mode 100644 index 000000000..b1c315206 --- /dev/null +++ b/fs/ntfs3/attrib.c @@ -0,0 +1,2812 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + * TODO: Merge attr_set_size/attr_data_get_block/attr_allocate_frame? + */ + +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +/* + * You can set external NTFS_MIN_LOG2_OF_CLUMP/NTFS_MAX_LOG2_OF_CLUMP to manage + * preallocate algorithm. + */ +#ifndef NTFS_MIN_LOG2_OF_CLUMP +#define NTFS_MIN_LOG2_OF_CLUMP 16 +#endif + +#ifndef NTFS_MAX_LOG2_OF_CLUMP +#define NTFS_MAX_LOG2_OF_CLUMP 26 +#endif + +// 16M +#define NTFS_CLUMP_MIN (1 << (NTFS_MIN_LOG2_OF_CLUMP + 8)) +// 16G +#define NTFS_CLUMP_MAX (1ull << (NTFS_MAX_LOG2_OF_CLUMP + 8)) + +static inline u64 get_pre_allocated(u64 size) +{ + u32 clump; + u8 align_shift; + u64 ret; + + if (size <= NTFS_CLUMP_MIN) { + clump = 1 << NTFS_MIN_LOG2_OF_CLUMP; + align_shift = NTFS_MIN_LOG2_OF_CLUMP; + } else if (size >= NTFS_CLUMP_MAX) { + clump = 1 << NTFS_MAX_LOG2_OF_CLUMP; + align_shift = NTFS_MAX_LOG2_OF_CLUMP; + } else { + align_shift = NTFS_MIN_LOG2_OF_CLUMP - 1 + + __ffs(size >> (8 + NTFS_MIN_LOG2_OF_CLUMP)); + clump = 1u << align_shift; + } + + ret = (((size + clump - 1) >> align_shift)) << align_shift; + + return ret; +} + +/* + * attr_load_runs - Load all runs stored in @attr. + */ +static int attr_load_runs(struct ATTRIB *attr, struct ntfs_inode *ni, + struct runs_tree *run, const CLST *vcn) +{ + int err; + CLST svcn = le64_to_cpu(attr->nres.svcn); + CLST evcn = le64_to_cpu(attr->nres.evcn); + u32 asize; + u16 run_off; + + if (svcn >= evcn + 1 || run_is_mapped_full(run, svcn, evcn)) + return 0; + + if (vcn && (evcn < *vcn || *vcn < svcn)) + return -EINVAL; + + asize = le32_to_cpu(attr->size); + run_off = le16_to_cpu(attr->nres.run_off); + + if (run_off > asize) + return -EINVAL; + + err = run_unpack_ex(run, ni->mi.sbi, ni->mi.rno, svcn, evcn, + vcn ? *vcn : svcn, Add2Ptr(attr, run_off), + asize - run_off); + if (err < 0) + return err; + + return 0; +} + +/* + * run_deallocate_ex - Deallocate clusters. + */ +static int run_deallocate_ex(struct ntfs_sb_info *sbi, struct runs_tree *run, + CLST vcn, CLST len, CLST *done, bool trim, + struct runs_tree *run_da) +{ + int err = 0; + CLST vcn_next, vcn0 = vcn, lcn, clen, dn = 0; + size_t idx; + + if (!len) + goto out; + + if (!run_lookup_entry(run, vcn, &lcn, &clen, &idx)) { +failed: + run_truncate(run, vcn0); + err = -EINVAL; + goto out; + } + + for (;;) { + if (clen > len) + clen = len; + + if (!clen) { + err = -EINVAL; + goto out; + } + + if (lcn != SPARSE_LCN) { + if (sbi) { + /* mark bitmap range [lcn + clen) as free and trim clusters. */ + mark_as_free_ex(sbi, lcn, clen, trim); + + if (run_da) { + CLST da_len; + if (!run_remove_range(run_da, vcn, clen, + &da_len)) { + err = -ENOMEM; + goto failed; + } + ntfs_sub_da(sbi, da_len); + } + } + dn += clen; + } + + len -= clen; + if (!len) + break; + + vcn_next = vcn + clen; + if (!run_get_entry(run, ++idx, &vcn, &lcn, &clen) || + vcn != vcn_next) { + /* Save memory - don't load entire run. */ + goto failed; + } + } + +out: + if (done) + *done += dn; + + return err; +} + +/* + * attr_allocate_clusters - Find free space, mark it as used and store in @run. + */ +int attr_allocate_clusters(struct ntfs_sb_info *sbi, struct runs_tree *run, + struct runs_tree *run_da, CLST vcn, CLST lcn, + CLST len, CLST *pre_alloc, enum ALLOCATE_OPT opt, + CLST *alen, const size_t fr, CLST *new_lcn, + CLST *new_len) +{ + int err; + CLST flen, vcn0 = vcn, pre = pre_alloc ? *pre_alloc : 0; + size_t cnt = run->count; + + for (;;) { + err = ntfs_look_for_free_space(sbi, lcn, len + pre, &lcn, &flen, + opt); + + if (err == -ENOSPC && pre) { + pre = 0; + if (pre_alloc) + *pre_alloc = 0; + continue; + } + + if (err == -ENOSPC && new_len && vcn - vcn0) { + /* Keep already allocated clusters. */ + *alen = vcn - vcn0; + return 0; + } + + if (err) + goto out; + + if (vcn == vcn0) { + /* Return the first fragment. */ + if (new_lcn) + *new_lcn = lcn; + if (new_len) + *new_len = flen; + } + + /* Add new fragment into run storage. */ + if (!run_add_entry(run, vcn, lcn, flen, opt & ALLOCATE_MFT)) { +undo_alloc: + /* Undo last 'ntfs_look_for_free_space' */ + mark_as_free_ex(sbi, lcn, len, false); + err = -ENOMEM; + goto out; + } + + if (run_da) { + CLST da_len; + if (!run_remove_range(run_da, vcn, flen, &da_len)) { + goto undo_alloc; + } + ntfs_sub_da(sbi, da_len); + } + + if (opt & ALLOCATE_ZERO) { + u8 shift = sbi->cluster_bits - SECTOR_SHIFT; + + err = blkdev_issue_zeroout(sbi->sb->s_bdev, + (sector_t)lcn << shift, + (sector_t)flen << shift, + GFP_NOFS, 0); + if (err) + goto out; + } + + vcn += flen; + + if (flen >= len || (opt & ALLOCATE_MFT) || + (opt & ALLOCATE_ONE_FR) || (fr && run->count - cnt >= fr)) { + *alen = vcn - vcn0; + return 0; + } + + len -= flen; + } + +out: + /* Undo 'ntfs_look_for_free_space' */ + if (vcn - vcn0) { + run_deallocate_ex(sbi, run, vcn0, vcn - vcn0, NULL, false, + run_da); + run_truncate(run, vcn0); + } + + return err; +} + +/* + * attr_make_nonresident + * + * If page is not NULL - it is already contains resident data + * and locked (called from ni_write_frame()). + */ +int attr_make_nonresident(struct ntfs_inode *ni, struct ATTRIB *attr, + struct ATTR_LIST_ENTRY *le, struct mft_inode *mi, + u64 new_size, struct runs_tree *run, + struct ATTRIB **ins_attr, struct page *page) +{ + struct ntfs_sb_info *sbi; + struct ATTRIB *attr_s; + struct MFT_REC *rec; + u32 used, asize, rsize, aoff; + bool is_data; + CLST len, alen; + char *next; + int err; + + if (attr->non_res) { + *ins_attr = attr; + return 0; + } + + sbi = mi->sbi; + rec = mi->mrec; + attr_s = NULL; + used = le32_to_cpu(rec->used); + asize = le32_to_cpu(attr->size); + next = Add2Ptr(attr, asize); + aoff = PtrOffset(rec, attr); + rsize = le32_to_cpu(attr->res.data_size); + is_data = attr->type == ATTR_DATA; + + /* len - how many clusters required to store 'rsize' bytes */ + if (is_attr_compressed(attr)) { + u8 shift = sbi->cluster_bits + NTFS_LZNT_CUNIT; + len = ((rsize + (1u << shift) - 1) >> shift) << NTFS_LZNT_CUNIT; + } else { + len = bytes_to_cluster(sbi, rsize); + } + + run_init(run); + + /* Make a copy of original attribute. */ + attr_s = kmemdup(attr, asize, GFP_NOFS); + if (!attr_s) { + err = -ENOMEM; + goto out; + } + + if (!len) { + /* Empty resident -> Empty nonresident. */ + alen = 0; + } else { + const char *data = resident_data(attr); + + err = attr_allocate_clusters(sbi, run, NULL, 0, 0, len, NULL, + ALLOCATE_DEF, &alen, 0, NULL, + NULL); + if (err) + goto out1; + + if (!rsize) { + /* Empty resident -> Non empty nonresident. */ + } else if (!is_data) { + err = ntfs_sb_write_run(sbi, run, 0, data, rsize, 0); + if (err) + goto out2; + } else if (!page) { + struct address_space *mapping = ni->vfs_inode.i_mapping; + struct folio *folio; + + folio = __filemap_get_folio( + mapping, 0, FGP_LOCK | FGP_ACCESSED | FGP_CREAT, + mapping_gfp_mask(mapping)); + if (IS_ERR(folio)) { + err = PTR_ERR(folio); + goto out2; + } + folio_fill_tail(folio, 0, data, rsize); + folio_mark_uptodate(folio); + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + } + } + + /* Remove original attribute. */ + used -= asize; + memmove(attr, Add2Ptr(attr, asize), used - aoff); + rec->used = cpu_to_le32(used); + mi->dirty = true; + if (le) + al_remove_le(ni, le); + + err = ni_insert_nonresident(ni, attr_s->type, attr_name(attr_s), + attr_s->name_len, run, 0, alen, + attr_s->flags, &attr, NULL, NULL); + if (err) + goto out3; + + kfree(attr_s); + attr->nres.data_size = cpu_to_le64(rsize); + attr->nres.valid_size = attr->nres.data_size; + + *ins_attr = attr; + + if (is_data) + ni->ni_flags &= ~NI_FLAG_RESIDENT; + + /* Resident attribute becomes non resident. */ + return 0; + +out3: + attr = Add2Ptr(rec, aoff); + memmove(next, attr, used - aoff); + memcpy(attr, attr_s, asize); + rec->used = cpu_to_le32(used + asize); + mi->dirty = true; +out2: + /* Undo: do not trim new allocated clusters. */ + run_deallocate(sbi, run, false); + run_close(run); +out1: + kfree(attr_s); +out: + return err; +} + +/* + * attr_set_size_res - Helper for attr_set_size(). + */ +static int attr_set_size_res(struct ntfs_inode *ni, struct ATTRIB *attr, + struct ATTR_LIST_ENTRY *le, struct mft_inode *mi, + u64 new_size, struct runs_tree *run, + struct ATTRIB **ins_attr) +{ + struct ntfs_sb_info *sbi = mi->sbi; + struct MFT_REC *rec = mi->mrec; + u32 used = le32_to_cpu(rec->used); + u32 asize = le32_to_cpu(attr->size); + u32 aoff = PtrOffset(rec, attr); + u32 rsize = le32_to_cpu(attr->res.data_size); + u32 tail = used - aoff - asize; + char *next = Add2Ptr(attr, asize); + s64 dsize = ALIGN(new_size, 8) - ALIGN(rsize, 8); + + if (dsize < 0) { + memmove(next + dsize, next, tail); + } else if (dsize > 0) { + if (used + dsize > sbi->max_bytes_per_attr) + return attr_make_nonresident(ni, attr, le, mi, new_size, + run, ins_attr, NULL); + + memmove(next + dsize, next, tail); + memset(next, 0, dsize); + } + + if (new_size > rsize) + memset(Add2Ptr(resident_data(attr), rsize), 0, + new_size - rsize); + + rec->used = cpu_to_le32(used + dsize); + attr->size = cpu_to_le32(asize + dsize); + attr->res.data_size = cpu_to_le32(new_size); + mi->dirty = true; + *ins_attr = attr; + + return 0; +} + +/* + * attr_set_size_ex - Change the size of attribute. + * + * Extend: + * - Sparse/compressed: No allocated clusters. + * - Normal: Append allocated and preallocated new clusters. + * Shrink: + * - No deallocate if @keep_prealloc is set. + */ +int attr_set_size_ex(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, struct runs_tree *run, + u64 new_size, const u64 *new_valid, bool keep_prealloc, + struct ATTRIB **ret, bool no_da) +{ + int err = 0; + struct ntfs_inode *nb = ni->base; + struct ntfs_sb_info *sbi = ni->mi.sbi; + u8 cluster_bits = sbi->cluster_bits; + bool is_mft = ni->mi.rno == MFT_REC_MFT && type == ATTR_DATA && + !name_len; + u64 old_valid, old_size, old_alloc, new_alloc_tmp; + u64 new_alloc = 0; + struct ATTRIB *attr = NULL, *attr_b; + struct ATTR_LIST_ENTRY *le, *le_b; + struct mft_inode *mi, *mi_b; + CLST alen, vcn, lcn, new_alen, old_alen, svcn, evcn; + CLST next_svcn, pre_alloc = -1, done = 0; + bool is_ext = false, is_bad = false; + bool dirty = false; + struct runs_tree *run_da = run == &ni->file.run ? &ni->file.run_da : + NULL; + bool da = !is_mft && sbi->options->delalloc && run_da && !no_da; + u32 align; + struct MFT_REC *rec; + +again: + alen = 0; + le_b = NULL; + attr_b = ni_find_attr(ni, NULL, &le_b, type, name, name_len, NULL, + &mi_b); + if (!attr_b) { + err = -ENOENT; + goto bad_inode; + } + + if (!attr_b->non_res) { + err = attr_set_size_res(ni, attr_b, le_b, mi_b, new_size, run, + &attr_b); + if (err) + return err; + + /* Return if file is still resident. */ + if (!attr_b->non_res) { + dirty = true; + goto ok1; + } + + /* Layout of records may be changed, so do a full search. */ + goto again; + } + + is_ext = is_attr_ext(attr_b); + align = sbi->cluster_size; + if (is_ext) { + align <<= attr_b->nres.c_unit; + keep_prealloc = false; + da = false; + } + + old_valid = le64_to_cpu(attr_b->nres.valid_size); + old_size = le64_to_cpu(attr_b->nres.data_size); + old_alloc = le64_to_cpu(attr_b->nres.alloc_size); + +again_1: + old_alen = old_alloc >> cluster_bits; + + new_alloc = (new_size + align - 1) & ~(u64)(align - 1); + new_alen = new_alloc >> cluster_bits; + + if (keep_prealloc && new_size < old_size) { + attr_b->nres.data_size = cpu_to_le64(new_size); + mi_b->dirty = dirty = true; + goto ok; + } + + if (da && + (vcn = old_alen + run_len(&ni->file.run_da), new_alen > vcn)) { + /* Resize up normal file. Delay new clusters allocation. */ + alen = new_alen - vcn; + + if (ntfs_check_free_space(sbi, alen, 0, true)) { + if (!run_add_entry(&ni->file.run_da, vcn, SPARSE_LCN, + alen, false)) { + err = -ENOMEM; + goto out; + } + + ntfs_add_da(sbi, alen); + goto ok1; + } + } + + if (!keep_prealloc && run_da && run_da->count && + (vcn = run_get_max_vcn(run_da), new_alen < vcn)) { + /* Shrink delayed clusters. */ + + /* Try to remove fragment from delay allocated run. */ + if (!run_remove_range(run_da, new_alen, vcn - new_alen, + &alen)) { + err = -ENOMEM; + goto out; + } + + ntfs_sub_da(sbi, alen); + } + + vcn = old_alen - 1; + + svcn = le64_to_cpu(attr_b->nres.svcn); + evcn = le64_to_cpu(attr_b->nres.evcn); + + if (svcn <= vcn && vcn <= evcn) { + attr = attr_b; + le = le_b; + mi = mi_b; + } else if (!le_b) { + err = -EINVAL; + goto bad_inode; + } else { + le = le_b; + attr = ni_find_attr(ni, attr_b, &le, type, name, name_len, &vcn, + &mi); + if (!attr) { + err = -EINVAL; + goto bad_inode; + } + +next_le_1: + svcn = le64_to_cpu(attr->nres.svcn); + evcn = le64_to_cpu(attr->nres.evcn); + } + /* + * Here we have: + * attr,mi,le - last attribute segment (containing 'vcn'). + * attr_b,mi_b,le_b - base (primary) attribute segment. + */ +next_le: + rec = mi->mrec; + err = attr_load_runs(attr, ni, run, NULL); + if (err) + goto out; + + if (new_size > old_size) { + CLST to_allocate; + size_t free; + + if (new_alloc <= old_alloc) { + attr_b->nres.data_size = cpu_to_le64(new_size); + mi_b->dirty = dirty = true; + goto ok; + } + + /* + * Add clusters. In simple case we have to: + * - allocate space (vcn, lcn, len) + * - update packed run in 'mi' + * - update attr->nres.evcn + * - update attr_b->nres.data_size/attr_b->nres.alloc_size + */ + to_allocate = new_alen - old_alen; +add_alloc_in_same_attr_seg: + lcn = 0; + if (is_mft) { + /* MFT allocates clusters from MFT zone. */ + pre_alloc = 0; + } else if (is_ext) { + /* No preallocate for sparse/compress. */ + pre_alloc = 0; + } else if (pre_alloc == -1) { + pre_alloc = 0; + if (type == ATTR_DATA && !name_len && + sbi->options->prealloc) { + pre_alloc = bytes_to_cluster( + sbi, get_pre_allocated( + new_size)) - + new_alen; + } + + /* Get the last LCN to allocate from. */ + if (old_alen && + !run_lookup_entry(run, vcn, &lcn, NULL, NULL)) { + lcn = SPARSE_LCN; + } + + if (lcn == SPARSE_LCN) + lcn = 0; + else if (lcn) + lcn += 1; + + free = wnd_zeroes(&sbi->used.bitmap); + if (to_allocate > free) { + err = -ENOSPC; + goto out; + } + + if (pre_alloc && to_allocate + pre_alloc > free) + pre_alloc = 0; + } + + vcn = old_alen; + + if (is_ext) { + if (!run_add_entry(run, vcn, SPARSE_LCN, to_allocate, + false)) { + err = -ENOMEM; + goto out; + } + alen = to_allocate; + } else { + /* ~3 bytes per fragment. */ + err = attr_allocate_clusters( + sbi, run, run_da, vcn, lcn, to_allocate, + &pre_alloc, + is_mft ? ALLOCATE_MFT : ALLOCATE_DEF, &alen, + is_mft ? 0 : + (sbi->record_size - + le32_to_cpu(rec->used) + 8) / + 3 + + 1, + NULL, NULL); + if (err) + goto out; + } + + done += alen; + vcn += alen; + if (to_allocate > alen) + to_allocate -= alen; + else + to_allocate = 0; + +pack_runs: + err = mi_pack_runs(mi, attr, run, vcn - svcn); + if (err) + goto undo_1; + + next_svcn = le64_to_cpu(attr->nres.evcn) + 1; + new_alloc_tmp = (u64)next_svcn << cluster_bits; + attr_b->nres.alloc_size = cpu_to_le64(new_alloc_tmp); + mi_b->dirty = dirty = true; + + if (next_svcn >= vcn && !to_allocate) { + /* Normal way. Update attribute and exit. */ + attr_b->nres.data_size = cpu_to_le64(new_size); + goto ok; + } + + /* At least two MFT to avoid recursive loop. */ + if (is_mft && next_svcn == vcn && + ((u64)done << sbi->cluster_bits) >= 2 * sbi->record_size) { + new_size = new_alloc_tmp; + attr_b->nres.data_size = attr_b->nres.alloc_size; + goto ok; + } + + if (le32_to_cpu(rec->used) < sbi->record_size) { + old_alen = next_svcn; + evcn = old_alen - 1; + goto add_alloc_in_same_attr_seg; + } + + attr_b->nres.data_size = attr_b->nres.alloc_size; + if (new_alloc_tmp < old_valid) + attr_b->nres.valid_size = attr_b->nres.data_size; + + if (type == ATTR_LIST) { + err = ni_expand_list(ni); + if (err) + goto undo_2; + if (next_svcn < vcn) + goto pack_runs; + + /* Layout of records is changed. */ + goto again; + } + + if (!nb->attr_list.size) { + err = ni_create_attr_list(nb); + /* In case of error layout of records is not changed. */ + if (err) + goto undo_2; + /* Layout of records is changed. */ + } + + if (next_svcn >= vcn) { + /* This is MFT data, repeat. */ + goto again; + } + + /* Insert new attribute segment. */ + err = ni_insert_nonresident(ni, type, name, name_len, run, + next_svcn, vcn - next_svcn, + attr_b->flags, &attr, &mi, NULL); + + /* + * Layout of records maybe changed. + * Find base attribute to update. + */ + le_b = NULL; + attr_b = ni_find_attr(ni, NULL, &le_b, type, name, name_len, + NULL, &mi_b); + if (!attr_b) { + err = -EINVAL; + goto bad_inode; + } + + if (err) { + /* ni_insert_nonresident failed. */ + attr = NULL; + goto undo_2; + } + + /* keep runs for $MFT::$ATTR_DATA and $MFT::$ATTR_BITMAP. */ + if (ni->mi.rno != MFT_REC_MFT) + run_truncate_head(run, evcn + 1); + + svcn = le64_to_cpu(attr->nres.svcn); + evcn = le64_to_cpu(attr->nres.evcn); + + /* + * Attribute is in consistency state. + * Save this point to restore to if next steps fail. + */ + old_valid = old_size = old_alloc = (u64)vcn << cluster_bits; + attr_b->nres.valid_size = attr_b->nres.data_size = + attr_b->nres.alloc_size = cpu_to_le64(old_size); + mi_b->dirty = dirty = true; + goto again_1; + } + + if (new_size != old_size || + (new_alloc != old_alloc && !keep_prealloc)) { + /* + * Truncate clusters. In simple case we have to: + * - update packed run in 'mi' + * - update attr->nres.evcn + * - update attr_b->nres.data_size/attr_b->nres.alloc_size + * - mark and trim clusters as free (vcn, lcn, len) + */ + CLST dlen = 0; + + vcn = max(svcn, new_alen); + new_alloc_tmp = (u64)vcn << cluster_bits; + + if (vcn > svcn) { + err = mi_pack_runs(mi, attr, run, vcn - svcn); + if (err) + goto out; + } else if (le && le->vcn) { + u16 le_sz = le16_to_cpu(le->size); + + /* + * NOTE: List entries for one attribute are always + * the same size. We deal with last entry (vcn==0) + * and it is not first in entries array + * (list entry for std attribute always first). + * So it is safe to step back. + */ + mi_remove_attr(NULL, mi, attr); + + if (!al_remove_le(ni, le)) { + err = -EINVAL; + goto bad_inode; + } + + le = (struct ATTR_LIST_ENTRY *)((u8 *)le - le_sz); + } else { + attr->nres.evcn = cpu_to_le64((u64)vcn - 1); + mi->dirty = true; + } + + attr_b->nres.alloc_size = cpu_to_le64(new_alloc_tmp); + + if (vcn == new_alen) { + attr_b->nres.data_size = cpu_to_le64(new_size); + if (new_size < old_valid) + attr_b->nres.valid_size = + attr_b->nres.data_size; + } else { + if (new_alloc_tmp <= + le64_to_cpu(attr_b->nres.data_size)) + attr_b->nres.data_size = + attr_b->nres.alloc_size; + if (new_alloc_tmp < + le64_to_cpu(attr_b->nres.valid_size)) + attr_b->nres.valid_size = + attr_b->nres.alloc_size; + } + mi_b->dirty = dirty = true; + + err = run_deallocate_ex(sbi, run, vcn, evcn - vcn + 1, &dlen, + true, run_da); + if (err) + goto out; + + if (is_ext) { + /* dlen - really deallocated clusters. */ + le64_sub_cpu(&attr_b->nres.total_size, + (u64)dlen << cluster_bits); + } + + run_truncate(run, vcn); + + if (new_alloc_tmp <= new_alloc) + goto ok; + + old_size = new_alloc_tmp; + vcn = svcn - 1; + + if (le == le_b) { + attr = attr_b; + mi = mi_b; + evcn = svcn - 1; + svcn = 0; + goto next_le; + } + + if (le->type != type || le->name_len != name_len || + memcmp(le_name(le), name, name_len * sizeof(short))) { + err = -EINVAL; + goto bad_inode; + } + + err = ni_load_mi(ni, le, &mi); + if (err) + goto out; + + attr = mi_find_attr(ni, mi, NULL, type, name, name_len, + &le->id); + if (!attr) { + err = -EINVAL; + goto bad_inode; + } + goto next_le_1; + } + +ok: + if (new_valid) { + __le64 valid = cpu_to_le64(min(*new_valid, new_size)); + + if (attr_b->nres.valid_size != valid) { + attr_b->nres.valid_size = valid; + mi_b->dirty = true; + } + } + +ok1: + if (ret) + *ret = attr_b; + + if ((type == ATTR_DATA || (type == ATTR_ALLOC && name == I30_NAME))) { + /* Update inode_set_bytes. */ + if (attr_b->non_res && + inode_get_bytes(&ni->vfs_inode) != new_alloc) { + inode_set_bytes(&ni->vfs_inode, new_alloc); + dirty = true; + } + + i_size_write(&ni->vfs_inode, new_size); + + /* Don't forget to update duplicate information in parent. */ + if (dirty) { + ni->ni_flags |= NI_FLAG_UPDATE_PARENT; + mark_inode_dirty(&ni->vfs_inode); + } + } + + return 0; + +undo_2: + vcn -= alen; + attr_b->nres.data_size = cpu_to_le64(old_size); + attr_b->nres.valid_size = cpu_to_le64(old_valid); + attr_b->nres.alloc_size = cpu_to_le64(old_alloc); + + /* Restore 'attr' and 'mi'. */ + if (attr) + goto restore_run; + + if (le64_to_cpu(attr_b->nres.svcn) <= svcn && + svcn <= le64_to_cpu(attr_b->nres.evcn)) { + attr = attr_b; + le = le_b; + mi = mi_b; + } else if (!le_b) { + err = -EINVAL; + goto bad_inode; + } else { + le = le_b; + attr = ni_find_attr(ni, attr_b, &le, type, name, name_len, + &svcn, &mi); + if (!attr) + goto bad_inode; + } + +restore_run: + if (mi_pack_runs(mi, attr, run, evcn - svcn + 1)) + is_bad = true; + +undo_1: + run_deallocate_ex(sbi, run, vcn, alen, NULL, false, run_da); + + run_truncate(run, vcn); +out: + if (is_bad) { +bad_inode: + _ntfs_bad_inode(&ni->vfs_inode); + } + return err; +} + +/* + * attr_data_get_block - Returns 'lcn' and 'len' for given 'vcn'. + * + * @new == NULL means just to get current mapping for 'vcn' + * @new != NULL means allocate real cluster if 'vcn' maps to hole + * @zero - zeroout new allocated clusters + * + * NOTE: + * - @new != NULL is called only for sparsed or compressed attributes. + * - new allocated clusters are zeroed via blkdev_issue_zeroout. + */ +int attr_data_get_block(struct ntfs_inode *ni, CLST vcn, CLST clen, CLST *lcn, + CLST *len, bool *new, bool zero, void **res, bool no_da) +{ + int err; + + if (new) + *new = false; + if (res) + *res = NULL; + + /* Try to find in cache. */ + down_read(&ni->file.run_lock); + if (run_lookup_entry_da(&ni->file.run, !no_da ? &ni->file.run_da : NULL, + vcn, lcn, len)) { + } else { + *len = 0; + } + up_read(&ni->file.run_lock); + + if (*len && (*lcn != SPARSE_LCN || !new)) + return 0; /* Fast normal way without allocation. */ + + /* No cluster in cache or we need to allocate cluster in hole. */ + ni_lock(ni); + down_write(&ni->file.run_lock); + + err = attr_data_get_block_locked(ni, vcn, clen, lcn, len, new, zero, + res, no_da); + + up_write(&ni->file.run_lock); + ni_unlock(ni); + + return err; +} + +/* + * attr_data_get_block_locked - Helper for attr_data_get_block. + */ +int attr_data_get_block_locked(struct ntfs_inode *ni, CLST vcn, CLST clen, + CLST *lcn, CLST *len, bool *new, bool zero, + void **res, bool no_da) +{ + int err = 0; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct runs_tree *run = &ni->file.run; + struct runs_tree *run_da = &ni->file.run_da; + bool da = sbi->options->delalloc && !no_da; + u8 cluster_bits; + struct ATTRIB *attr, *attr_b; + struct ATTR_LIST_ENTRY *le, *le_b; + struct mft_inode *mi, *mi_b; + CLST hint, svcn, to_alloc, evcn1, next_svcn, asize, end, vcn0; + CLST alloc, evcn; + unsigned fr; + u64 total_size, total_size0; + int step; + +again: + if (run_lookup_entry_da(run, da ? &ni->file.run_da : NULL, vcn, lcn, + len)) { + } else { + *len = 0; + } + + if (*len) { + if (*lcn != SPARSE_LCN || !new) + goto out; /* normal way without allocation. */ + if (clen > *len) + clen = *len; + } + + cluster_bits = sbi->cluster_bits; + step = 0; + + le_b = NULL; + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, NULL, &mi_b); + if (!attr_b) { + err = -ENOENT; + goto out; + } + + if (!attr_b->non_res) { + u32 data_size = le32_to_cpu(attr_b->res.data_size); + *lcn = RESIDENT_LCN; + *len = data_size; + if (res) { + *res = NULL; + if (data_size) { + struct page *page = alloc_page(GFP_KERNEL); + if (!page) { + err = -ENOMEM; + goto out; + } + + *res = page_address(page); + memcpy(*res, resident_data(attr_b), data_size); + } + } + goto out; + } + + asize = le64_to_cpu(attr_b->nres.alloc_size) >> cluster_bits; + if (vcn >= asize) { + if (new) { + err = -EINVAL; + } else { + *len = 1; + *lcn = EOF_LCN; + } + goto out; + } + + svcn = le64_to_cpu(attr_b->nres.svcn); + evcn1 = le64_to_cpu(attr_b->nres.evcn) + 1; + + attr = attr_b; + le = le_b; + mi = mi_b; + + if (le_b && (vcn < svcn || evcn1 <= vcn)) { + attr = ni_find_attr(ni, attr_b, &le, ATTR_DATA, + ni->file.ads.name, ni->file.ads.len, &vcn, + &mi); + if (!attr) { + err = -EINVAL; + goto out; + } + svcn = le64_to_cpu(attr->nres.svcn); + evcn1 = le64_to_cpu(attr->nres.evcn) + 1; + } + + /* Load in cache actual information. */ + err = attr_load_runs(attr, ni, run, NULL); + if (err) + goto out; + + /* Check for compressed frame. */ + err = attr_is_frame_compressed(ni, attr_b, vcn >> NTFS_LZNT_CUNIT, + &hint, run); + if (err) + goto out; + + if (hint) { + /* if frame is compressed - don't touch it. */ + *lcn = COMPRESSED_LCN; + /* length to the end of frame. */ + *len = NTFS_LZNT_CLUSTERS - (vcn & (NTFS_LZNT_CLUSTERS - 1)); + err = 0; + goto out; + } + + if (!*len) { + if (run_lookup_entry_da(run, da ? run_da : NULL, vcn, lcn, + len)) { + if (*lcn != SPARSE_LCN || !new) + goto ok; /* Slow normal way without allocation. */ + + if (clen > *len) + clen = *len; + } else if (!new) { + /* Here we may return -ENOENT. + * In any case caller gets zero length. */ + goto ok; + } + } + + if (!is_attr_ext(attr_b)) { + /* The code below only for sparsed or compressed attributes. */ + err = -EINVAL; + goto out; + } + + vcn0 = vcn; + to_alloc = clen; + fr = (sbi->record_size - le32_to_cpu(mi->mrec->used) + 8) / 3 + 1; + /* Allocate frame aligned clusters. + * ntfs.sys usually uses 16 clusters per frame for sparsed or compressed. + * ntfs3 uses 1 cluster per frame for new created sparsed files. */ + if (attr_b->nres.c_unit) { + CLST clst_per_frame = 1u << attr_b->nres.c_unit; + CLST cmask = ~(clst_per_frame - 1); + + /* Get frame aligned vcn and to_alloc. */ + vcn = vcn0 & cmask; + to_alloc = ((vcn0 + clen + clst_per_frame - 1) & cmask) - vcn; + if (fr < clst_per_frame) + fr = clst_per_frame; + if (vcn != vcn0) + zero = true; + + /* Check if 'vcn' and 'vcn0' in different attribute segments. */ + if (vcn < svcn || evcn1 <= vcn) { + struct ATTRIB *attr2; + /* Load runs for truncated vcn. */ + attr2 = ni_find_attr(ni, attr_b, &le_b, ATTR_DATA, + ni->file.ads.name, + ni->file.ads.len, &vcn, &mi); + if (!attr2) { + err = -EINVAL; + goto out; + } + evcn1 = le64_to_cpu(attr2->nres.evcn) + 1; + err = attr_load_runs(attr2, ni, run, NULL); + if (err) + goto out; + } + + if (vcn0 < svcn || evcn1 <= vcn0) { + struct ATTRIB *attr2; + + attr2 = ni_find_attr(ni, attr_b, &le_b, ATTR_DATA, + ni->file.ads.name, + ni->file.ads.len, &vcn0, &mi); + if (!attr2) { + err = -EINVAL; + goto out; + } + err = attr_load_runs(attr2, ni, run, NULL); + if (err) + goto out; + } + + da = false; /* no delalloc for compressed file. */ + } + + if (vcn + to_alloc > asize) + to_alloc = asize - vcn; + + if (da) { + CLST rlen1, rlen2; + if (!ntfs_check_free_space(sbi, to_alloc, 0, true)) { + err = ni_allocate_da_blocks_locked(ni); + if (err) + goto out; + /* Layout of records may be changed. Start again without 'da'. */ + da = false; + goto again; + } + + /* run_add_entry consolidates existed ranges. */ + rlen1 = run_len(run_da); + if (!run_add_entry(run_da, vcn, SPARSE_LCN, to_alloc, false)) { + err = -ENOMEM; + goto out; + } + rlen2 = run_len(run_da); + + /* new added delay clusters = rlen2 - rlen1. */ + ntfs_add_da(sbi, rlen2 - rlen1); + *len = to_alloc; + *lcn = DELALLOC_LCN; + goto ok; + } + + /* Get the last LCN to allocate from. */ + hint = 0; + + if (vcn > evcn1) { + if (!run_add_entry(run, evcn1, SPARSE_LCN, vcn - evcn1, + false)) { + err = -ENOMEM; + goto out; + } + } else if (vcn && !run_lookup_entry(run, vcn - 1, &hint, NULL, NULL)) { + hint = -1; + } + + /* Allocate and zeroout new clusters. */ + err = attr_allocate_clusters(sbi, run, run_da, vcn, hint + 1, to_alloc, + NULL, + zero ? ALLOCATE_ZERO : ALLOCATE_ONE_FR, + len, fr, lcn, len); + if (err) + goto out; + *new = true; + step = 1; + + end = vcn + *len; + /* Save 'total_size0' to restore if error. */ + total_size0 = le64_to_cpu(attr_b->nres.total_size); + total_size = total_size0 + ((u64)*len << cluster_bits); + + if (vcn != vcn0) { + if (!run_lookup_entry(run, vcn0, lcn, len, NULL)) { + err = -EINVAL; + goto out; + } + if (*lcn == SPARSE_LCN) { + /* Internal error. Should not happened. */ + WARN_ON(1); + err = -EINVAL; + goto out; + } + /* Check case when vcn0 + len overlaps new allocated clusters. */ + if (vcn0 + *len > end) + *len = end - vcn0; + } + +repack: + err = mi_pack_runs(mi, attr, run, max(end, evcn1) - svcn); + if (err) + goto out; + + attr_b->nres.total_size = cpu_to_le64(total_size); + inode_set_bytes(&ni->vfs_inode, total_size); + ni->ni_flags |= NI_FLAG_UPDATE_PARENT; + + mi_b->dirty = true; + mark_inode_dirty(&ni->vfs_inode); + + /* Stored [vcn : next_svcn) from [vcn : end). */ + next_svcn = le64_to_cpu(attr->nres.evcn) + 1; + + if (end <= evcn1) { + if (next_svcn == evcn1) { + /* Normal way. Update attribute and exit. */ + goto ok; + } + /* Add new segment [next_svcn : evcn1 - next_svcn). */ + if (!ni->attr_list.size) { + err = ni_create_attr_list(ni); + if (err) + goto undo1; + /* Layout of records is changed. */ + le_b = NULL; + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, + ni->file.ads.name, + ni->file.ads.len, NULL, &mi_b); + if (!attr_b) { + err = -ENOENT; + goto out; + } + + attr = attr_b; + le = le_b; + mi = mi_b; + goto repack; + } + } + + /* + * The code below may require additional cluster (to extend attribute list) + * and / or one MFT record + * It is too complex to undo operations if -ENOSPC occurs deep inside + * in 'ni_insert_nonresident'. + * Return in advance -ENOSPC here if there are no free cluster and no free MFT. + */ + if (!ntfs_check_free_space(sbi, 1, 1, false)) { + /* Undo step 1. */ + err = -ENOSPC; + goto undo1; + } + + step = 2; + svcn = evcn1; + + /* Estimate next attribute. */ + attr = ni_find_attr(ni, attr, &le, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, &svcn, &mi); + + if (!attr) { + /* Insert new attribute segment. */ + goto ins_ext; + } + + /* Try to update existed attribute segment. */ + alloc = bytes_to_cluster(sbi, le64_to_cpu(attr_b->nres.alloc_size)); + evcn = le64_to_cpu(attr->nres.evcn); + + if (end < next_svcn) + end = next_svcn; + while (end > evcn) { + /* Remove segment [svcn : evcn). */ + mi_remove_attr(NULL, mi, attr); + + if (!al_remove_le(ni, le)) { + err = -EINVAL; + goto out; + } + + if (evcn + 1 >= alloc) { + /* Last attribute segment. */ + evcn1 = evcn + 1; + goto ins_ext; + } + + if (ni_load_mi(ni, le, &mi)) { + attr = NULL; + goto out; + } + + attr = mi_find_attr(ni, mi, NULL, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, &le->id); + if (!attr) { + err = -EINVAL; + goto out; + } + svcn = le64_to_cpu(attr->nres.svcn); + evcn = le64_to_cpu(attr->nres.evcn); + } + + if (end < svcn) + end = svcn; + + err = attr_load_runs(attr, ni, run, &end); + if (err) + goto out; + + evcn1 = evcn + 1; + attr->nres.svcn = cpu_to_le64(next_svcn); + err = mi_pack_runs(mi, attr, run, evcn1 - next_svcn); + if (err) + goto out; + + le->vcn = cpu_to_le64(next_svcn); + ni->attr_list.dirty = true; + mi->dirty = true; + next_svcn = le64_to_cpu(attr->nres.evcn) + 1; + +ins_ext: + if (evcn1 > next_svcn) { + err = ni_insert_nonresident(ni, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, run, next_svcn, + evcn1 - next_svcn, attr_b->flags, + &attr, &mi, NULL); + if (err) + goto out; + } +ok: + run_truncate_around(run, vcn); +out: + if (err && step > 1) { + /* Too complex to restore. */ + _ntfs_bad_inode(&ni->vfs_inode); + } + + return err; + +undo1: + /* Undo step1. */ + attr_b->nres.total_size = cpu_to_le64(total_size0); + inode_set_bytes(&ni->vfs_inode, total_size0); + + if (run_deallocate_ex(sbi, run, vcn, *len, NULL, false, run_da) || + !run_add_entry(run, vcn, SPARSE_LCN, *len, false) || + mi_pack_runs(mi, attr, run, max(end, evcn1) - svcn)) { + _ntfs_bad_inode(&ni->vfs_inode); + } + goto out; +} + +int attr_data_write_resident(struct ntfs_inode *ni, struct folio *folio) +{ + u64 vbo; + struct mft_inode *mi; + struct ATTRIB *attr; + u32 data_size; + + attr = ni_find_attr(ni, NULL, NULL, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, NULL, &mi); + if (!attr) + return -EINVAL; + + if (attr->non_res) { + /* Return special error code to check this case. */ + return E_NTFS_NONRESIDENT; + } + + vbo = folio_pos(folio); + data_size = le32_to_cpu(attr->res.data_size); + if (vbo < data_size) { + char *data = resident_data(attr); + size_t len = min(data_size - vbo, folio_size(folio)); + + memcpy_from_folio(data + vbo, folio, 0, len); + mi->dirty = true; + } + ni->i_valid = data_size; + + return 0; +} + +/* + * attr_load_runs_vcn - Load runs with VCN. + */ +int attr_load_runs_vcn(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, struct runs_tree *run, + CLST vcn) +{ + struct ATTRIB *attr; + int err; + CLST svcn, evcn; + u16 ro; + + if (!ni) { + /* Is record corrupted? */ + return -ENOENT; + } + + attr = ni_find_attr(ni, NULL, NULL, type, name, name_len, &vcn, NULL); + if (!attr) { + /* Is record corrupted? */ + return -ENOENT; + } + + svcn = le64_to_cpu(attr->nres.svcn); + evcn = le64_to_cpu(attr->nres.evcn); + + if (evcn < vcn || vcn < svcn) { + /* Is record corrupted? */ + return -EINVAL; + } + + ro = le16_to_cpu(attr->nres.run_off); + + if (ro > le32_to_cpu(attr->size)) + return -EINVAL; + + err = run_unpack_ex(run, ni->mi.sbi, ni->mi.rno, svcn, evcn, svcn, + Add2Ptr(attr, ro), le32_to_cpu(attr->size) - ro); + if (err < 0) + return err; + return 0; +} + +/* + * attr_load_runs_range - Load runs for given range [from to). + */ +int attr_load_runs_range(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, struct runs_tree *run, + u64 from, u64 to) +{ + struct ntfs_sb_info *sbi = ni->mi.sbi; + u8 cluster_bits = sbi->cluster_bits; + CLST vcn; + CLST vcn_last = (to - 1) >> cluster_bits; + CLST lcn, clen; + int err = 0; + int retry = 0; + + for (vcn = from >> cluster_bits; vcn <= vcn_last; vcn += clen) { + if (run_lookup_entry(run, vcn, &lcn, &clen, NULL)) { + retry = 0; + continue; + } + if (retry) { + err = -EINVAL; + break; + } + err = attr_load_runs_vcn(ni, type, name, name_len, run, vcn); + if (err) + break; + + clen = 0; /* Next run_lookup_entry(vcn) must be success. */ + retry++; + } + + return err; +} + +#ifdef CONFIG_NTFS3_LZX_XPRESS +/* + * attr_wof_frame_info + * + * Read header of Xpress/LZX file to get info about frame. + */ +int attr_wof_frame_info(struct ntfs_inode *ni, struct ATTRIB *attr, + struct runs_tree *run, u64 frame, u64 frames, + u8 frame_bits, u32 *ondisk_size, u64 *vbo_data) +{ + struct ntfs_sb_info *sbi = ni->mi.sbi; + u64 vbo[2], off[2], wof_size; + u32 voff; + u8 bytes_per_off; + char *addr; + struct folio *folio; + int i, err = 0; + __le32 *off32; + __le64 *off64; + + if (ni->vfs_inode.i_size < 0x100000000ull) { + /* File starts with array of 32 bit offsets. */ + bytes_per_off = sizeof(__le32); + vbo[1] = frame << 2; + *vbo_data = frames << 2; + } else { + /* File starts with array of 64 bit offsets. */ + bytes_per_off = sizeof(__le64); + vbo[1] = frame << 3; + *vbo_data = frames << 3; + } + + /* + * Read 4/8 bytes at [vbo - 4(8)] == offset where compressed frame starts. + * Read 4/8 bytes at [vbo] == offset where compressed frame ends. + */ + if (!attr->non_res) { + if (vbo[1] + bytes_per_off > le32_to_cpu(attr->res.data_size)) { + _ntfs_bad_inode(&ni->vfs_inode); + return -EINVAL; + } + addr = resident_data(attr); + + if (bytes_per_off == sizeof(__le32)) { + off32 = Add2Ptr(addr, vbo[1]); + off[0] = vbo[1] ? le32_to_cpu(off32[-1]) : 0; + off[1] = le32_to_cpu(off32[0]); + } else { + off64 = Add2Ptr(addr, vbo[1]); + off[0] = vbo[1] ? le64_to_cpu(off64[-1]) : 0; + off[1] = le64_to_cpu(off64[0]); + } + + *vbo_data += off[0]; + *ondisk_size = off[1] - off[0]; + return 0; + } + + wof_size = le64_to_cpu(attr->nres.data_size); + down_write(&ni->file.run_lock); + folio = ni->file.offs_folio; + if (!folio) { + folio = folio_alloc(GFP_KERNEL, 0); + if (!folio) { + err = -ENOMEM; + goto out; + } + folio->index = -1; + ni->file.offs_folio = folio; + } + folio_lock(folio); + addr = folio_address(folio); + + if (vbo[1]) { + voff = vbo[1] & (PAGE_SIZE - 1); + vbo[0] = vbo[1] - bytes_per_off; + i = 0; + } else { + voff = 0; + vbo[0] = 0; + off[0] = 0; + i = 1; + } + + do { + pgoff_t index = vbo[i] >> PAGE_SHIFT; + + if (index != folio->index) { + u64 from = vbo[i] & ~(u64)(PAGE_SIZE - 1); + u64 to = min(from + PAGE_SIZE, wof_size); + + if (from >= wof_size) { + _ntfs_bad_inode(&ni->vfs_inode); + err = -EINVAL; + goto out1; + } + + err = attr_load_runs_range(ni, ATTR_DATA, WOF_NAME, + ARRAY_SIZE(WOF_NAME), run, + from, to); + if (err) + goto out1; + + err = ntfs_read_run(sbi, run, addr, from, to - from); + if (err) { + folio->index = -1; + goto out1; + } + folio->index = index; + } + + if (i) { + if (bytes_per_off == sizeof(__le32)) { + off32 = Add2Ptr(addr, voff); + off[1] = le32_to_cpu(*off32); + } else { + off64 = Add2Ptr(addr, voff); + off[1] = le64_to_cpu(*off64); + } + } else if (!voff) { + if (bytes_per_off == sizeof(__le32)) { + off32 = Add2Ptr(addr, PAGE_SIZE - sizeof(u32)); + off[0] = le32_to_cpu(*off32); + } else { + off64 = Add2Ptr(addr, PAGE_SIZE - sizeof(u64)); + off[0] = le64_to_cpu(*off64); + } + } else { + /* Two values in one page. */ + if (bytes_per_off == sizeof(__le32)) { + off32 = Add2Ptr(addr, voff); + off[0] = le32_to_cpu(off32[-1]); + off[1] = le32_to_cpu(off32[0]); + } else { + off64 = Add2Ptr(addr, voff); + off[0] = le64_to_cpu(off64[-1]); + off[1] = le64_to_cpu(off64[0]); + } + break; + } + } while (++i < 2); + + *vbo_data += off[0]; + *ondisk_size = off[1] - off[0]; + +out1: + folio_unlock(folio); +out: + up_write(&ni->file.run_lock); + return err; +} +#endif + +/* + * attr_is_frame_compressed - Used to detect compressed frame. + * + * attr - base (primary) attribute segment. + * run - run to use, usually == &ni->file.run. + * Only base segments contains valid 'attr->nres.c_unit' + */ +int attr_is_frame_compressed(struct ntfs_inode *ni, struct ATTRIB *attr, + CLST frame, CLST *clst_data, struct runs_tree *run) +{ + int err; + u32 clst_frame; + CLST clen, lcn, vcn, alen, slen, vcn_next; + size_t idx; + + *clst_data = 0; + + if (!is_attr_compressed(attr)) + return 0; + + if (!attr->non_res) + return 0; + + clst_frame = 1u << attr->nres.c_unit; + vcn = frame * clst_frame; + + if (!run_lookup_entry(run, vcn, &lcn, &clen, &idx)) { + err = attr_load_runs_vcn(ni, attr->type, attr_name(attr), + attr->name_len, run, vcn); + if (err) + return err; + + if (!run_lookup_entry(run, vcn, &lcn, &clen, &idx)) + return -EINVAL; + } + + if (lcn == SPARSE_LCN) { + /* Sparsed frame. */ + return 0; + } + + if (clen >= clst_frame) { + /* + * The frame is not compressed 'cause + * it does not contain any sparse clusters. + */ + *clst_data = clst_frame; + return 0; + } + + alen = bytes_to_cluster(ni->mi.sbi, le64_to_cpu(attr->nres.alloc_size)); + slen = 0; + *clst_data = clen; + + /* + * The frame is compressed if *clst_data + slen >= clst_frame. + * Check next fragments. + */ + while ((vcn += clen) < alen) { + vcn_next = vcn; + + if (!run_get_entry(run, ++idx, &vcn, &lcn, &clen) || + vcn_next != vcn) { + err = attr_load_runs_vcn(ni, attr->type, + attr_name(attr), + attr->name_len, run, vcn_next); + if (err) + return err; + vcn = vcn_next; + + if (!run_lookup_entry(run, vcn, &lcn, &clen, &idx)) + return -EINVAL; + } + + if (lcn == SPARSE_LCN) { + slen += clen; + } else { + if (slen) { + /* + * Data_clusters + sparse_clusters = + * not enough for frame. + */ + return -EINVAL; + } + *clst_data += clen; + } + + if (*clst_data + slen >= clst_frame) { + if (!slen) { + /* + * There is no sparsed clusters in this frame + * so it is not compressed. + */ + *clst_data = clst_frame; + } else { + /* Frame is compressed. */ + } + break; + } + } + + return 0; +} + +/* + * attr_allocate_frame - Allocate/free clusters for @frame. + * + * Assumed: down_write(&ni->file.run_lock); + */ +int attr_allocate_frame(struct ntfs_inode *ni, CLST frame, size_t compr_size, + u64 new_valid) +{ + int err = 0; + struct runs_tree *run = &ni->file.run; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct ATTRIB *attr = NULL, *attr_b; + struct ATTR_LIST_ENTRY *le, *le_b; + struct mft_inode *mi, *mi_b; + CLST svcn, evcn1, next_svcn, len; + CLST vcn, end, clst_data; + u64 total_size, valid_size, data_size; + + le_b = NULL; + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, NULL, &mi_b); + if (!attr_b) + return -ENOENT; + + if (!is_attr_ext(attr_b)) + return -EINVAL; + + vcn = frame << NTFS_LZNT_CUNIT; + total_size = le64_to_cpu(attr_b->nres.total_size); + + svcn = le64_to_cpu(attr_b->nres.svcn); + evcn1 = le64_to_cpu(attr_b->nres.evcn) + 1; + data_size = le64_to_cpu(attr_b->nres.data_size); + + if (svcn <= vcn && vcn < evcn1) { + attr = attr_b; + le = le_b; + mi = mi_b; + } else if (!le_b) { + err = -EINVAL; + goto out; + } else { + le = le_b; + attr = ni_find_attr(ni, attr_b, &le, ATTR_DATA, + ni->file.ads.name, ni->file.ads.len, &vcn, + &mi); + if (!attr) { + err = -EINVAL; + goto out; + } + svcn = le64_to_cpu(attr->nres.svcn); + evcn1 = le64_to_cpu(attr->nres.evcn) + 1; + } + + err = attr_load_runs(attr, ni, run, NULL); + if (err) + goto out; + + err = attr_is_frame_compressed(ni, attr_b, frame, &clst_data, run); + if (err) + goto out; + + total_size -= (u64)clst_data << sbi->cluster_bits; + + len = bytes_to_cluster(sbi, compr_size); + + if (len == clst_data) + goto out; + + if (len < clst_data) { + err = run_deallocate_ex(sbi, run, vcn + len, clst_data - len, + NULL, true, NULL); + if (err) + goto out; + + if (!run_add_entry(run, vcn + len, SPARSE_LCN, clst_data - len, + false)) { + err = -ENOMEM; + goto out; + } + end = vcn + clst_data; + /* Run contains updated range [vcn + len : end). */ + } else { + CLST alen, hint = 0; + /* Get the last LCN to allocate from. */ + if (vcn + clst_data && + !run_lookup_entry(run, vcn + clst_data - 1, &hint, NULL, + NULL)) { + hint = -1; + } + + err = attr_allocate_clusters(sbi, run, NULL, vcn + clst_data, + hint + 1, len - clst_data, NULL, + ALLOCATE_DEF, &alen, 0, NULL, + NULL); + if (err) + goto out; + + end = vcn + len; + /* Run contains updated range [vcn + clst_data : end). */ + } + + total_size += (u64)len << sbi->cluster_bits; + +repack: + err = mi_pack_runs(mi, attr, run, max(end, evcn1) - svcn); + if (err) + goto out; + + attr_b->nres.total_size = cpu_to_le64(total_size); + inode_set_bytes(&ni->vfs_inode, total_size); + ni->ni_flags |= NI_FLAG_UPDATE_PARENT; + + mi_b->dirty = true; + mark_inode_dirty(&ni->vfs_inode); + + /* Stored [vcn : next_svcn) from [vcn : end). */ + next_svcn = le64_to_cpu(attr->nres.evcn) + 1; + + if (end <= evcn1) { + if (next_svcn == evcn1) { + /* Normal way. Update attribute and exit. */ + goto ok; + } + /* Add new segment [next_svcn : evcn1 - next_svcn). */ + if (!ni->attr_list.size) { + err = ni_create_attr_list(ni); + if (err) + goto out; + /* Layout of records is changed. */ + le_b = NULL; + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, + ni->file.ads.name, + ni->file.ads.len, NULL, &mi_b); + if (!attr_b) { + err = -ENOENT; + goto out; + } + + attr = attr_b; + le = le_b; + mi = mi_b; + goto repack; + } + } + + svcn = evcn1; + + /* Estimate next attribute. */ + attr = ni_find_attr(ni, attr, &le, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, &svcn, &mi); + + if (attr) { + CLST alloc = bytes_to_cluster( + sbi, le64_to_cpu(attr_b->nres.alloc_size)); + CLST evcn = le64_to_cpu(attr->nres.evcn); + + if (end < next_svcn) + end = next_svcn; + while (end > evcn) { + /* Remove segment [svcn : evcn). */ + mi_remove_attr(NULL, mi, attr); + + if (!al_remove_le(ni, le)) { + err = -EINVAL; + goto out; + } + + if (evcn + 1 >= alloc) { + /* Last attribute segment. */ + evcn1 = evcn + 1; + goto ins_ext; + } + + if (ni_load_mi(ni, le, &mi)) { + attr = NULL; + goto out; + } + + attr = mi_find_attr(ni, mi, NULL, ATTR_DATA, + ni->file.ads.name, ni->file.ads.len, + &le->id); + if (!attr) { + err = -EINVAL; + goto out; + } + svcn = le64_to_cpu(attr->nres.svcn); + evcn = le64_to_cpu(attr->nres.evcn); + } + + if (end < svcn) + end = svcn; + + err = attr_load_runs(attr, ni, run, &end); + if (err) + goto out; + + evcn1 = evcn + 1; + attr->nres.svcn = cpu_to_le64(next_svcn); + err = mi_pack_runs(mi, attr, run, evcn1 - next_svcn); + if (err) + goto out; + + le->vcn = cpu_to_le64(next_svcn); + ni->attr_list.dirty = true; + mi->dirty = true; + + next_svcn = le64_to_cpu(attr->nres.evcn) + 1; + } +ins_ext: + if (evcn1 > next_svcn) { + err = ni_insert_nonresident(ni, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, run, next_svcn, + evcn1 - next_svcn, attr_b->flags, + &attr, &mi, NULL); + if (err) + goto out; + } +ok: + run_truncate_around(run, vcn); +out: + if (attr_b) { + if (new_valid > data_size) + new_valid = data_size; + + valid_size = le64_to_cpu(attr_b->nres.valid_size); + if (new_valid != valid_size) { + attr_b->nres.valid_size = cpu_to_le64(valid_size); + mi_b->dirty = true; + } + } + + return err; +} + +/* + * attr_collapse_range - Collapse range in file. + */ +int attr_collapse_range(struct ntfs_inode *ni, u64 vbo, u64 bytes) +{ + int err = 0; + struct runs_tree *run = &ni->file.run; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct ATTRIB *attr = NULL, *attr_b; + struct ATTR_LIST_ENTRY *le, *le_b; + struct mft_inode *mi, *mi_b; + CLST svcn, evcn1, len, dealloc, alen, done; + CLST vcn, end; + u64 valid_size, data_size, alloc_size, total_size; + u32 mask; + u64 i_size; + __le16 a_flags; + + if (!bytes) + return 0; + + le_b = NULL; + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, NULL, &mi_b); + if (!attr_b) + return -ENOENT; + + if (!attr_b->non_res) { + /* Attribute is resident. Nothing to do? */ + return 0; + } + + mask = is_attr_ext(attr_b) ? + ((sbi->cluster_size << attr_b->nres.c_unit) - 1) : + sbi->cluster_mask; + if ((vbo | bytes) & mask) { + /* Allow to collapse only cluster aligned ranges. */ + return -EINVAL; + } + + /* i_size - size of file with delay allocated clusters. */ + i_size = ni->vfs_inode.i_size; + + if (vbo > i_size) + return -EINVAL; + + down_write(&ni->file.run_lock); + + if (vbo + bytes >= i_size) { + valid_size = min(ni->i_valid, vbo); + + /* Simple truncate file at 'vbo'. */ + truncate_setsize(&ni->vfs_inode, vbo); + err = attr_set_size(ni, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, &ni->file.run, vbo, + &valid_size, true); + + if (!err && valid_size < ni->i_valid) + ni->i_valid = valid_size; + + goto out; + } + + vcn = vbo >> sbi->cluster_bits; + len = bytes >> sbi->cluster_bits; + end = vcn + len; + dealloc = 0; + done = 0; + + /* + * Check delayed clusters. + */ + if (ni->file.run_da.count) { + struct runs_tree *run_da = &ni->file.run_da; + if (run_is_mapped_full(run_da, vcn, end - 1)) { + /* + * The requested range is full in delayed clusters. + */ + err = attr_set_size_ex(ni, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, run, + i_size - bytes, NULL, false, + NULL, true); + goto out; + } + + /* Collapse request crosses real and delayed clusters. */ + err = ni_allocate_da_blocks_locked(ni); + if (err) + goto out; + + /* Layout of records maybe changed. */ + le_b = NULL; + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, + ni->file.ads.name, ni->file.ads.len, NULL, + &mi_b); + if (!attr_b || !attr_b->non_res) { + err = -ENOENT; + goto out; + } + } + + data_size = le64_to_cpu(attr_b->nres.data_size); + alloc_size = le64_to_cpu(attr_b->nres.alloc_size); + total_size = is_attr_ext(attr_b) ? + le64_to_cpu(attr_b->nres.total_size) : + alloc_size; + alen = alloc_size >> sbi->cluster_bits; + a_flags = attr_b->flags; + svcn = le64_to_cpu(attr_b->nres.svcn); + evcn1 = le64_to_cpu(attr_b->nres.evcn) + 1; + + if (svcn <= vcn && vcn < evcn1) { + attr = attr_b; + le = le_b; + mi = mi_b; + goto check_seg; + } + + if (!le_b) { + err = -EINVAL; + goto out; + } + + le = le_b; + attr = ni_find_attr(ni, attr_b, &le, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, &vcn, &mi); + if (!attr) { + err = -EINVAL; + goto out; + } + + /* + * Enumerate all attribute segments and collapse. + */ + for (;;) { + CLST vcn1, eat, next_svcn; + + svcn = le64_to_cpu(attr->nres.svcn); + evcn1 = le64_to_cpu(attr->nres.evcn) + 1; + +check_seg: + if (svcn >= end) { + /* Shift VCN- */ + attr->nres.svcn = cpu_to_le64(svcn - len); + attr->nres.evcn = cpu_to_le64(evcn1 - 1 - len); + if (le) { + le->vcn = attr->nres.svcn; + ni->attr_list.dirty = true; + } + mi->dirty = true; + goto next_attr; + } + + run_truncate(run, 0); + err = attr_load_runs(attr, ni, run, &svcn); + if (err) + goto out; + + vcn1 = vcn + done; /* original vcn in attr/run. */ + eat = min(end, evcn1) - vcn1; + + err = run_deallocate_ex(sbi, run, vcn1, eat, &dealloc, true, + NULL); + if (err) + goto out; + + if (svcn + eat < evcn1) { + /* Collapse a part of this attribute segment. */ + if (!run_collapse_range(run, vcn1, eat, done)) { + err = -ENOMEM; + goto out; + } + + if (svcn >= vcn) { + /* Shift VCN */ + attr->nres.svcn = cpu_to_le64(vcn); + if (le && attr->nres.svcn != le->vcn) { + le->vcn = attr->nres.svcn; + ni->attr_list.dirty = true; + } + } + + err = mi_pack_runs(mi, attr, run, evcn1 - svcn - eat); + if (err) + goto out; + + next_svcn = le64_to_cpu(attr->nres.evcn) + 1; + if (next_svcn + eat + done < evcn1) { + err = ni_insert_nonresident( + ni, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, run, next_svcn, + evcn1 - eat - next_svcn, a_flags, &attr, + &mi, &le); + if (err) + goto out; + + /* Layout of records maybe changed. */ + attr_b = NULL; + } + + /* Free all allocated memory. */ + run_truncate(run, 0); + done += eat; + } else { + u16 le_sz; + + /* Delete this attribute segment. */ + mi_remove_attr(NULL, mi, attr); + if (!le) + break; + + le_sz = le16_to_cpu(le->size); + if (!al_remove_le(ni, le)) { + err = -EINVAL; + goto out; + } + + done += evcn1 - svcn; + if (evcn1 >= alen) + break; + + if (!svcn) { + /* Load next record that contains this attribute. */ + if (ni_load_mi(ni, le, &mi)) { + err = -EINVAL; + goto out; + } + + /* Look for required attribute. */ + attr = mi_find_attr(ni, mi, NULL, ATTR_DATA, + ni->file.ads.name, + ni->file.ads.len, &le->id); + if (!attr) { + err = -EINVAL; + goto out; + } + continue; + } + le = (struct ATTR_LIST_ENTRY *)((u8 *)le - le_sz); + } + +next_attr: + if (evcn1 >= alen) + break; + + attr = ni_enum_attr_ex(ni, attr, &le, &mi); + if (!attr) { + err = -EINVAL; + goto out; + } + } + + if (!attr_b) { + le_b = NULL; + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, + ni->file.ads.name, ni->file.ads.len, NULL, + &mi_b); + if (!attr_b) { + err = -ENOENT; + goto out; + } + } + + data_size -= bytes; + valid_size = ni->i_valid; + if (vbo + bytes <= valid_size) + valid_size -= bytes; + else if (vbo < valid_size) + valid_size = vbo; + + attr_b->nres.alloc_size = cpu_to_le64(alloc_size - bytes); + attr_b->nres.data_size = cpu_to_le64(data_size); + attr_b->nres.valid_size = cpu_to_le64(min(valid_size, data_size)); + total_size -= (u64)dealloc << sbi->cluster_bits; + if (is_attr_ext(attr_b)) + attr_b->nres.total_size = cpu_to_le64(total_size); + mi_b->dirty = true; + + /* Update inode size. */ + ni->i_valid = valid_size; + i_size_write(&ni->vfs_inode, data_size); + inode_set_bytes(&ni->vfs_inode, total_size); + ni->ni_flags |= NI_FLAG_UPDATE_PARENT; + mark_inode_dirty(&ni->vfs_inode); + +out: + up_write(&ni->file.run_lock); + if (err) + _ntfs_bad_inode(&ni->vfs_inode); + + return err; +} + +/* + * attr_punch_hole + * + * Not for normal files. + */ +int attr_punch_hole(struct ntfs_inode *ni, u64 vbo, u64 bytes, u32 *frame_size) +{ + int err = 0; + struct runs_tree *run = &ni->file.run; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct ATTRIB *attr = NULL, *attr_b; + struct ATTR_LIST_ENTRY *le, *le_b; + struct mft_inode *mi, *mi_b; + CLST svcn, evcn1, vcn, len, end, alen, hole, next_svcn; + u64 total_size, alloc_size; + u32 mask; + __le16 a_flags; + struct runs_tree run2; + + if (!bytes) + return 0; + + le_b = NULL; + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, NULL, &mi_b); + if (!attr_b) + return -ENOENT; + + if (!attr_b->non_res) { + u32 data_size = le32_to_cpu(attr_b->res.data_size); + u32 from, to; + + if (vbo > data_size) + return 0; + + from = vbo; + to = min_t(u64, vbo + bytes, data_size); + memset(Add2Ptr(resident_data(attr_b), from), 0, to - from); + return 0; + } + + if (!is_attr_ext(attr_b)) + return -EOPNOTSUPP; + + alloc_size = le64_to_cpu(attr_b->nres.alloc_size); + total_size = le64_to_cpu(attr_b->nres.total_size); + + if (vbo >= alloc_size) { + /* NOTE: It is allowed. */ + return 0; + } + + mask = (sbi->cluster_size << attr_b->nres.c_unit) - 1; + + bytes += vbo; + if (bytes > alloc_size) + bytes = alloc_size; + bytes -= vbo; + + if ((vbo | bytes) & mask) { + /* We have to zero a range(s). */ + if (!frame_size) { + /* Caller insists range is aligned. */ + return -EINVAL; + } + *frame_size = mask + 1; + return E_NTFS_NOTALIGNED; + } + + down_write(&ni->file.run_lock); + run_init(&run2); + run_truncate(run, 0); + + /* + * Enumerate all attribute segments and punch hole where necessary. + */ + alen = alloc_size >> sbi->cluster_bits; + vcn = vbo >> sbi->cluster_bits; + len = bytes >> sbi->cluster_bits; + end = vcn + len; + hole = 0; + + svcn = le64_to_cpu(attr_b->nres.svcn); + evcn1 = le64_to_cpu(attr_b->nres.evcn) + 1; + a_flags = attr_b->flags; + + if (svcn <= vcn && vcn < evcn1) { + attr = attr_b; + le = le_b; + mi = mi_b; + } else if (!le_b) { + err = -EINVAL; + goto bad_inode; + } else { + le = le_b; + attr = ni_find_attr(ni, attr_b, &le, ATTR_DATA, + ni->file.ads.name, ni->file.ads.len, &vcn, + &mi); + if (!attr) { + err = -EINVAL; + goto bad_inode; + } + + svcn = le64_to_cpu(attr->nres.svcn); + evcn1 = le64_to_cpu(attr->nres.evcn) + 1; + } + + while (svcn < end) { + CLST vcn1, zero, hole2 = hole; + + err = attr_load_runs(attr, ni, run, &svcn); + if (err) + goto done; + vcn1 = max(vcn, svcn); + zero = min(end, evcn1) - vcn1; + + /* + * Check range [vcn1 + zero). + * Calculate how many clusters there are. + * Don't do any destructive actions. + */ + err = run_deallocate_ex(NULL, run, vcn1, zero, &hole2, false, + NULL); + if (err) + goto done; + + /* Check if required range is already hole. */ + if (hole2 == hole) + goto next_attr; + + /* Make a clone of run to undo. */ + err = run_clone(run, &run2); + if (err) + goto done; + + /* Make a hole range (sparse) [vcn1 + zero). */ + if (!run_add_entry(run, vcn1, SPARSE_LCN, zero, false)) { + err = -ENOMEM; + goto done; + } + + /* Update run in attribute segment. */ + err = mi_pack_runs(mi, attr, run, evcn1 - svcn); + if (err) + goto done; + next_svcn = le64_to_cpu(attr->nres.evcn) + 1; + if (next_svcn < evcn1) { + /* Insert new attribute segment. */ + err = ni_insert_nonresident( + ni, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, run, next_svcn, + evcn1 - next_svcn, a_flags, &attr, &mi, &le); + if (err) + goto undo_punch; + + /* Layout of records maybe changed. */ + attr_b = NULL; + } + + /* Real deallocate. Should not fail. */ + run_deallocate_ex(sbi, &run2, vcn1, zero, &hole, true, + &ni->file.run_da); + +next_attr: + /* Free all allocated memory. */ + run_truncate(run, 0); + + if (evcn1 >= alen) + break; + + /* Get next attribute segment. */ + attr = ni_enum_attr_ex(ni, attr, &le, &mi); + if (!attr) { + err = -EINVAL; + goto bad_inode; + } + + svcn = le64_to_cpu(attr->nres.svcn); + evcn1 = le64_to_cpu(attr->nres.evcn) + 1; + } + +done: + if (!hole) + goto out; + + if (!attr_b) { + attr_b = ni_find_attr(ni, NULL, NULL, ATTR_DATA, + ni->file.ads.name, ni->file.ads.len, NULL, + &mi_b); + if (!attr_b) { + err = -EINVAL; + goto bad_inode; + } + } + + total_size -= (u64)hole << sbi->cluster_bits; + attr_b->nres.total_size = cpu_to_le64(total_size); + mi_b->dirty = true; + + /* Update inode size. */ + inode_set_bytes(&ni->vfs_inode, total_size); + ni->ni_flags |= NI_FLAG_UPDATE_PARENT; + mark_inode_dirty(&ni->vfs_inode); + +out: + run_close(&run2); + up_write(&ni->file.run_lock); + return err; + +bad_inode: + _ntfs_bad_inode(&ni->vfs_inode); + goto out; + +undo_punch: + /* + * Restore packed runs. + * 'mi_pack_runs' should not fail, cause we restore original. + */ + if (mi_pack_runs(mi, attr, &run2, evcn1 - svcn)) + goto bad_inode; + + goto done; +} + +/* + * attr_insert_range - Insert range (hole) in file. + * Not for normal files. + */ +int attr_insert_range(struct ntfs_inode *ni, u64 vbo, u64 bytes) +{ + int err = 0; + struct runs_tree *run = &ni->file.run; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct ATTRIB *attr = NULL, *attr_b; + struct ATTR_LIST_ENTRY *le, *le_b; + struct mft_inode *mi, *mi_b; + CLST vcn, svcn, evcn1, len, next_svcn; + u64 data_size, alloc_size; + u32 mask; + __le16 a_flags; + + if (!bytes) + return 0; + + le_b = NULL; + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, NULL, &mi_b); + if (!attr_b) + return -ENOENT; + + if (!is_attr_ext(attr_b)) { + /* It was checked above. See fallocate. */ + return -EOPNOTSUPP; + } + + if (!attr_b->non_res) { + data_size = le32_to_cpu(attr_b->res.data_size); + alloc_size = data_size; + mask = sbi->cluster_mask; /* cluster_size - 1 */ + } else { + data_size = le64_to_cpu(attr_b->nres.data_size); + alloc_size = le64_to_cpu(attr_b->nres.alloc_size); + mask = (sbi->cluster_size << attr_b->nres.c_unit) - 1; + } + + if (vbo >= data_size) { + /* + * Insert range after the file size is not allowed. + * If the offset is equal to or greater than the end of + * file, an error is returned. For such operations (i.e., inserting + * a hole at the end of file), ftruncate(2) should be used. + */ + return -EINVAL; + } + + if ((vbo | bytes) & mask) { + /* Allow to insert only frame aligned ranges. */ + return -EINVAL; + } + + /* + * valid_size <= data_size <= alloc_size + * Check alloc_size for maximum possible. + */ + if (bytes > sbi->maxbytes_sparse - alloc_size) + return -EFBIG; + + vcn = vbo >> sbi->cluster_bits; + len = bytes >> sbi->cluster_bits; + + down_write(&ni->file.run_lock); + + if (!attr_b->non_res) { + err = attr_set_size(ni, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, run, data_size + bytes, + NULL, false); + + le_b = NULL; + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, + ni->file.ads.name, ni->file.ads.len, NULL, + &mi_b); + if (!attr_b) { + err = -EINVAL; + goto bad_inode; + } + + if (err) + goto out; + + if (!attr_b->non_res) { + /* Still resident. */ + char *data = Add2Ptr(attr_b, + le16_to_cpu(attr_b->res.data_off)); + + memmove(data + bytes, data, bytes); + memset(data, 0, bytes); + goto done; + } + + /* Resident file becomes nonresident. */ + data_size = le64_to_cpu(attr_b->nres.data_size); + alloc_size = le64_to_cpu(attr_b->nres.alloc_size); + } + + /* + * Enumerate all attribute segments and shift start vcn. + */ + a_flags = attr_b->flags; + svcn = le64_to_cpu(attr_b->nres.svcn); + evcn1 = le64_to_cpu(attr_b->nres.evcn) + 1; + + if (svcn <= vcn && vcn < evcn1) { + attr = attr_b; + le = le_b; + mi = mi_b; + } else if (!le_b) { + err = -EINVAL; + goto bad_inode; + } else { + le = le_b; + attr = ni_find_attr(ni, attr_b, &le, ATTR_DATA, + ni->file.ads.name, ni->file.ads.len, &vcn, + &mi); + if (!attr) { + err = -EINVAL; + goto bad_inode; + } + + svcn = le64_to_cpu(attr->nres.svcn); + evcn1 = le64_to_cpu(attr->nres.evcn) + 1; + } + + run_truncate(run, 0); /* clear cached values. */ + err = attr_load_runs(attr, ni, run, NULL); + if (err) + goto out; + + err = run_insert_range(run, vcn, len); + if (err) + goto out; + + err = run_insert_range_da(&ni->file.run_da, vcn, len); + if (err) + goto out; + + /* Try to pack in current record as much as possible. */ + err = mi_pack_runs(mi, attr, run, evcn1 + len - svcn); + if (err) + goto out; + + next_svcn = le64_to_cpu(attr->nres.evcn) + 1; + + while ((attr = ni_enum_attr_ex(ni, attr, &le, &mi)) && + attr->type == ATTR_DATA && !attr->name_len) { + le64_add_cpu(&attr->nres.svcn, len); + le64_add_cpu(&attr->nres.evcn, len); + if (le) { + le->vcn = attr->nres.svcn; + ni->attr_list.dirty = true; + } + mi->dirty = true; + } + + if (next_svcn < evcn1 + len) { + err = ni_insert_nonresident(ni, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, run, next_svcn, + evcn1 + len - next_svcn, a_flags, + NULL, NULL, NULL); + + le_b = NULL; + attr_b = ni_find_attr(ni, NULL, &le_b, ATTR_DATA, + ni->file.ads.name, ni->file.ads.len, NULL, + &mi_b); + if (!attr_b) { + err = -EINVAL; + goto bad_inode; + } + + if (err) { + /* ni_insert_nonresident failed. Try to undo. */ + goto undo_insert_range; + } + } + + /* + * Update primary attribute segment. + */ + if (vbo <= ni->i_valid) + ni->i_valid += bytes; + + attr_b->nres.data_size = cpu_to_le64(data_size + bytes); + attr_b->nres.alloc_size = cpu_to_le64(alloc_size + bytes); + + /* ni->valid may be not equal valid_size (temporary). */ + if (ni->i_valid > data_size + bytes) + attr_b->nres.valid_size = attr_b->nres.data_size; + else + attr_b->nres.valid_size = cpu_to_le64(ni->i_valid); + mi_b->dirty = true; + +done: + i_size_write(&ni->vfs_inode, ni->vfs_inode.i_size + bytes); + ni->ni_flags |= NI_FLAG_UPDATE_PARENT; + mark_inode_dirty(&ni->vfs_inode); + +out: + run_truncate(run, 0); /* clear cached values. */ + + up_write(&ni->file.run_lock); + + return err; + +bad_inode: + _ntfs_bad_inode(&ni->vfs_inode); + goto out; + +undo_insert_range: + svcn = le64_to_cpu(attr_b->nres.svcn); + evcn1 = le64_to_cpu(attr_b->nres.evcn) + 1; + + if (svcn <= vcn && vcn < evcn1) { + attr = attr_b; + le = le_b; + mi = mi_b; + } else if (!le_b) { + goto bad_inode; + } else { + le = le_b; + attr = ni_find_attr(ni, attr_b, &le, ATTR_DATA, + ni->file.ads.name, ni->file.ads.len, &vcn, + &mi); + if (!attr) { + goto bad_inode; + } + + svcn = le64_to_cpu(attr->nres.svcn); + evcn1 = le64_to_cpu(attr->nres.evcn) + 1; + } + + if (attr_load_runs(attr, ni, run, NULL)) + goto bad_inode; + + if (!run_collapse_range(run, vcn, len, 0)) + goto bad_inode; + + if (mi_pack_runs(mi, attr, run, evcn1 + len - svcn)) + goto bad_inode; + + while ((attr = ni_enum_attr_ex(ni, attr, &le, &mi)) && + attr->type == ATTR_DATA && !attr->name_len) { + le64_sub_cpu(&attr->nres.svcn, len); + le64_sub_cpu(&attr->nres.evcn, len); + if (le) { + le->vcn = attr->nres.svcn; + ni->attr_list.dirty = true; + } + mi->dirty = true; + } + + goto out; +} + +/* + * attr_force_nonresident + * + * Convert default data attribute into non resident form. + */ +int attr_force_nonresident(struct ntfs_inode *ni) +{ + int err; + struct ATTRIB *attr; + struct ATTR_LIST_ENTRY *le = NULL; + struct mft_inode *mi; + + attr = ni_find_attr(ni, NULL, &le, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, NULL, &mi); + if (!attr) { + _ntfs_bad_inode(&ni->vfs_inode); + return -ENOENT; + } + + if (attr->non_res) { + /* Already non resident. */ + return 0; + } + + down_write(&ni->file.run_lock); + err = attr_make_nonresident(ni, attr, le, mi, + le32_to_cpu(attr->res.data_size), + &ni->file.run, &attr, NULL); + up_write(&ni->file.run_lock); + + return err; +} diff --git a/fs/ntfs3/attrlist.c b/fs/ntfs3/attrlist.c new file mode 100644 index 000000000..8710560cf --- /dev/null +++ b/fs/ntfs3/attrlist.c @@ -0,0 +1,439 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + */ + +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +/* + * al_is_valid_le + * + * Return: True if @le is valid. + */ +static inline bool al_is_valid_le(const struct ntfs_inode *ni, + struct ATTR_LIST_ENTRY *le) +{ + ni = ni->base; + if (!le || !ni->attr_list.le || !ni->attr_list.size) + return false; + + return PtrOffset(ni->attr_list.le, le) + le16_to_cpu(le->size) <= + ni->attr_list.size; +} + +void al_destroy(struct ntfs_inode *ni) +{ + ni = ni->base; + run_close(&ni->attr_list.run); + kvfree(ni->attr_list.le); + ni->attr_list.le = NULL; + ni->attr_list.size = 0; + ni->attr_list.dirty = false; +} + +/* + * ntfs_load_attr_list + * + * This method makes sure that the ATTRIB list, if present, + * has been properly set up. + */ +int ntfs_load_attr_list(struct ntfs_inode *ni, struct ATTRIB *attr) +{ + int err; + size_t lsize; + void *le = NULL; + + ni = ni->base; + if (ni->attr_list.size) + return 0; + + if (!attr->non_res) { + lsize = le32_to_cpu(attr->res.data_size); + if (!lsize) { + err = -EINVAL; + goto out; + } + + /* attr is resident: lsize < record_size (1K or 4K) */ + le = kvmalloc(al_aligned(lsize), GFP_KERNEL); + if (!le) { + err = -ENOMEM; + goto out; + } + memcpy(le, resident_data(attr), lsize); + } else if (attr->nres.svcn) { + err = -EINVAL; + goto out; + } else { + u16 run_off = le16_to_cpu(attr->nres.run_off); + + lsize = le64_to_cpu(attr->nres.data_size); + if (!lsize) { + err = -EINVAL; + goto out; + } + + run_init(&ni->attr_list.run); + + if (run_off > le32_to_cpu(attr->size)) { + err = -EINVAL; + goto out; + } + + err = run_unpack_ex(&ni->attr_list.run, ni->mi.sbi, ni->mi.rno, + 0, le64_to_cpu(attr->nres.evcn), 0, + Add2Ptr(attr, run_off), + le32_to_cpu(attr->size) - run_off); + if (err < 0) + goto out; + + /* attr is nonresident. + * The worst case: + * 1T (2^40) extremely fragmented file. + * cluster = 4K (2^12) => 2^28 fragments + * 2^9 fragments per one record => 2^19 records + * 2^5 bytes of ATTR_LIST_ENTRY per one record => 2^24 bytes. + * + * the result is 16M bytes per attribute list. + * Use kvmalloc to allocate in range [several Kbytes - dozen Mbytes] + */ + le = kvmalloc(al_aligned(lsize), GFP_KERNEL); + if (!le) { + err = -ENOMEM; + goto out; + } + + err = ntfs_read_run_nb(ni->mi.sbi, &ni->attr_list.run, 0, le, + lsize, NULL); + if (err) + goto out; + } + + ni->attr_list.size = lsize; + ni->attr_list.le = le; + + return 0; + +out: + ni->attr_list.le = le; + al_destroy(ni); + + return err; +} + +/* + * al_enumerate + * + * Return: + * * The next list le. + * * If @le is NULL then return the first le. + */ +struct ATTR_LIST_ENTRY *al_enumerate(struct ntfs_inode *ni, + struct ATTR_LIST_ENTRY *le) +{ + size_t off; + u16 sz; + const unsigned le_min_size = le_size(0); + + if (!le) { + le = ni->attr_list.le; + } else { + sz = le16_to_cpu(le->size); + if (sz < le_min_size) { + /* Impossible 'cause we should not return such le. */ + return NULL; + } + le = Add2Ptr(le, sz); + } + + /* Check boundary. */ + off = PtrOffset(ni->attr_list.le, le); + if (off + le_min_size > ni->attr_list.size) { + /* The regular end of list. */ + return NULL; + } + + sz = le16_to_cpu(le->size); + + /* Check le for errors. */ + if (sz < le_min_size || off + sz > ni->attr_list.size || + sz < le->name_off + le->name_len * sizeof(short)) { + return NULL; + } + + return le; +} + +/* + * al_find_le + * + * Find the first le in the list which matches type, name and VCN. + * + * Return: NULL if not found. + */ +struct ATTR_LIST_ENTRY *al_find_le(struct ntfs_inode *ni, + struct ATTR_LIST_ENTRY *le, + const struct ATTRIB *attr) +{ + CLST svcn = attr_svcn(attr); + + return al_find_ex(ni, le, attr->type, attr_name(attr), attr->name_len, + &svcn); +} + +/* + * al_find_ex + * + * Find the first le in the list which matches type, name and VCN. + * + * Return: NULL if not found. + */ +struct ATTR_LIST_ENTRY *al_find_ex(struct ntfs_inode *ni, + struct ATTR_LIST_ENTRY *le, + enum ATTR_TYPE type, const __le16 *name, + u8 name_len, const CLST *vcn) +{ + struct ATTR_LIST_ENTRY *ret = NULL; + u32 type_in = le32_to_cpu(type); + + ni = ni->base; + while ((le = al_enumerate(ni, le))) { + u64 le_vcn; + int diff = le32_to_cpu(le->type) - type_in; + + /* List entries are sorted by type, name and VCN. */ + if (diff < 0) + continue; + + if (diff > 0) + return ret; + + if (le->name_len != name_len) + continue; + + le_vcn = le64_to_cpu(le->vcn); + if (!le_vcn) { + /* + * Compare entry names only for entry with vcn == 0. + */ + diff = ntfs_cmp_names(le_name(le), name_len, name, + name_len, ni->mi.sbi->upcase, + true); + if (diff < 0) + continue; + + if (diff > 0) + return ret; + } + + if (!vcn) + return le; + + if (*vcn == le_vcn) + return le; + + if (*vcn < le_vcn) + return ret; + + ret = le; + } + + return ret; +} + +/* + * al_find_le_to_insert + * + * Find the first list entry which matches type, name and VCN. + */ +static struct ATTR_LIST_ENTRY *al_find_le_to_insert(struct ntfs_inode *ni, + enum ATTR_TYPE type, + const __le16 *name, + u8 name_len, CLST vcn) +{ + struct ATTR_LIST_ENTRY *le = NULL, *prev; + u32 type_in = le32_to_cpu(type); + + ni = ni->base; + /* List entries are sorted by type, name and VCN. */ + while ((le = al_enumerate(ni, prev = le))) { + int diff = le32_to_cpu(le->type) - type_in; + + if (diff < 0) + continue; + + if (diff > 0) + return le; + + if (!le->vcn) { + /* + * Compare entry names only for entry with vcn == 0. + */ + diff = ntfs_cmp_names(le_name(le), le->name_len, name, + name_len, ni->mi.sbi->upcase, + true); + if (diff < 0) + continue; + + if (diff > 0) + return le; + } + + if (le64_to_cpu(le->vcn) >= vcn) + return le; + } + + return prev ? Add2Ptr(prev, le16_to_cpu(prev->size)) : ni->attr_list.le; +} + +/* + * al_add_le + * + * Add an "attribute list entry" to the list. + */ +int al_add_le(struct ntfs_inode *ni, enum ATTR_TYPE type, const __le16 *name, + u8 name_len, CLST svcn, __le16 id, const struct MFT_REF *ref, + struct ATTR_LIST_ENTRY **new_le) +{ + int err; + struct ATTRIB *attr; + struct ATTR_LIST_ENTRY *le; + size_t off; + u16 sz; + size_t asize, new_asize, old_size; + u64 new_size; + typeof(ni->attr_list) *al = &ni->attr_list; + + ni = ni->base; + /* + * Compute the size of the new 'le' + */ + sz = le_size(name_len); + old_size = al->size; + new_size = old_size + sz; + asize = al_aligned(old_size); + new_asize = al_aligned(new_size); + + /* Scan forward to the point at which the new 'le' should be inserted. */ + le = al_find_le_to_insert(ni, type, name, name_len, svcn); + off = PtrOffset(al->le, le); + + if (new_size > asize) { + void *ptr = kmalloc(new_asize, GFP_NOFS); + + if (!ptr) + return -ENOMEM; + + memcpy(ptr, al->le, off); + memcpy(Add2Ptr(ptr, off + sz), le, old_size - off); + le = Add2Ptr(ptr, off); + kvfree(al->le); + al->le = ptr; + } else { + memmove(Add2Ptr(le, sz), le, old_size - off); + } + *new_le = le; + + al->size = new_size; + + le->type = type; + le->size = cpu_to_le16(sz); + le->name_len = name_len; + le->name_off = offsetof(struct ATTR_LIST_ENTRY, name); + le->vcn = cpu_to_le64(svcn); + le->ref = *ref; + le->id = id; + memcpy(le->name, name, sizeof(short) * name_len); + + err = attr_set_size_ex(ni, ATTR_LIST, NULL, 0, &al->run, new_size, + &new_size, true, &attr, false); + if (err) { + /* Undo memmove above. */ + memmove(le, Add2Ptr(le, sz), old_size - off); + al->size = old_size; + return err; + } + + al->dirty = true; + + if (attr && attr->non_res) { + err = ntfs_sb_write_run(ni->mi.sbi, &al->run, 0, al->le, + al->size, 0); + if (err) + return err; + al->dirty = false; + } + + return 0; +} + +/* + * al_remove_le - Remove @le from attribute list. + */ +bool al_remove_le(struct ntfs_inode *ni, struct ATTR_LIST_ENTRY *le) +{ + u16 size; + size_t off; + typeof(ni->attr_list) *al; + + ni = ni->base; + al = &ni->attr_list; + if (!al_is_valid_le(ni, le)) + return false; + + /* Save on stack the size of 'le' */ + size = le16_to_cpu(le->size); + off = PtrOffset(al->le, le); + + memmove(le, Add2Ptr(le, size), al->size - (off + size)); + + al->size -= size; + al->dirty = true; + + return true; +} + +int al_update(struct ntfs_inode *ni, int sync) +{ + int err; + struct ATTRIB *attr; + typeof(ni->attr_list) *al; + + ni = ni->base; + al = &ni->attr_list; + + if (!al->dirty || !al->size) + return 0; + + /* + * Attribute list increased on demand in al_add_le. + * Attribute list decreased here. + */ + err = attr_set_size_ex(ni, ATTR_LIST, NULL, 0, &al->run, al->size, NULL, + false, &attr, false); + if (err) + goto out; + + if (!attr->non_res) { + memcpy(resident_data(attr), al->le, al->size); + } else { + err = ntfs_sb_write_run(ni->mi.sbi, &al->run, 0, al->le, + al->size, sync); + if (err) + goto out; + + attr->nres.valid_size = attr->nres.data_size; + } + + ni->mi.dirty = true; + al->dirty = false; + +out: + return err; +} diff --git a/fs/ntfs3/bitfunc.c b/fs/ntfs3/bitfunc.c new file mode 100644 index 000000000..25a4d4896 --- /dev/null +++ b/fs/ntfs3/bitfunc.c @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + */ + +#include + +#include "ntfs_fs.h" + +#define BITS_IN_SIZE_T (sizeof(size_t) * 8) + +/* + * fill_mask[i] - first i bits are '1' , i = 0,1,2,3,4,5,6,7,8 + * fill_mask[i] = 0xFF >> (8-i) + */ +static const u8 fill_mask[] = { 0x00, 0x01, 0x03, 0x07, 0x0F, + 0x1F, 0x3F, 0x7F, 0xFF }; + +/* + * zero_mask[i] - first i bits are '0' , i = 0,1,2,3,4,5,6,7,8 + * zero_mask[i] = 0xFF << i + */ +static const u8 zero_mask[] = { 0xFF, 0xFE, 0xFC, 0xF8, 0xF0, + 0xE0, 0xC0, 0x80, 0x00 }; + +/* + * are_bits_clear + * + * Return: True if all bits [bit, bit+nbits) are zeros "0". + */ +bool are_bits_clear(const void *lmap, size_t bit, size_t nbits) +{ + size_t pos = bit & 7; + const u8 *map = (u8 *)lmap + (bit >> 3); + + if (pos) { + if (8 - pos >= nbits) + return !nbits || !(*map & fill_mask[pos + nbits] & + zero_mask[pos]); + + if (*map++ & zero_mask[pos]) + return false; + nbits -= 8 - pos; + } + + pos = ((size_t)map) & (sizeof(size_t) - 1); + if (pos) { + pos = sizeof(size_t) - pos; + if (nbits >= pos * 8) { + for (nbits -= pos * 8; pos; pos--, map++) { + if (*map) + return false; + } + } + } + + for (pos = nbits / BITS_IN_SIZE_T; pos; pos--, map += sizeof(size_t)) { + if (*((size_t *)map)) + return false; + } + + for (pos = (nbits % BITS_IN_SIZE_T) >> 3; pos; pos--, map++) { + if (*map) + return false; + } + + pos = nbits & 7; + if (pos && (*map & fill_mask[pos])) + return false; + + return true; +} + +/* + * are_bits_set + * + * Return: True if all bits [bit, bit+nbits) are ones "1". + */ +bool are_bits_set(const void *lmap, size_t bit, size_t nbits) +{ + u8 mask; + size_t pos = bit & 7; + const u8 *map = (u8 *)lmap + (bit >> 3); + + if (pos) { + if (8 - pos >= nbits) { + mask = fill_mask[pos + nbits] & zero_mask[pos]; + return !nbits || (*map & mask) == mask; + } + + mask = zero_mask[pos]; + if ((*map++ & mask) != mask) + return false; + nbits -= 8 - pos; + } + + pos = ((size_t)map) & (sizeof(size_t) - 1); + if (pos) { + pos = sizeof(size_t) - pos; + if (nbits >= pos * 8) { + for (nbits -= pos * 8; pos; pos--, map++) { + if (*map != 0xFF) + return false; + } + } + } + + for (pos = nbits / BITS_IN_SIZE_T; pos; pos--, map += sizeof(size_t)) { + if (*((size_t *)map) != MINUS_ONE_T) + return false; + } + + for (pos = (nbits % BITS_IN_SIZE_T) >> 3; pos; pos--, map++) { + if (*map != 0xFF) + return false; + } + + pos = nbits & 7; + if (pos) { + mask = fill_mask[pos]; + if ((*map & mask) != mask) + return false; + } + + return true; +} diff --git a/fs/ntfs3/bitmap.c b/fs/ntfs3/bitmap.c new file mode 100644 index 000000000..60b362f38 --- /dev/null +++ b/fs/ntfs3/bitmap.c @@ -0,0 +1,1564 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + * This code builds two trees of free clusters extents. + * Trees are sorted by start of extent and by length of extent. + * NTFS_MAX_WND_EXTENTS defines the maximum number of elements in trees. + * In extreme case code reads on-disk bitmap to find free clusters. + * + */ + +#include +#include +#include + +#include "ntfs.h" +#include "ntfs_fs.h" + +/* + * Maximum number of extents in tree. + */ +#define NTFS_MAX_WND_EXTENTS (32u * 1024u) + +struct rb_node_key { + struct rb_node node; + size_t key; +}; + +struct e_node { + struct rb_node_key start; /* Tree sorted by start. */ + struct rb_node_key count; /* Tree sorted by len. */ +}; + +static int wnd_rescan(struct wnd_bitmap *wnd); +static struct buffer_head *wnd_map(struct wnd_bitmap *wnd, size_t iw); +static bool wnd_is_free_hlp(struct wnd_bitmap *wnd, size_t bit, size_t bits); + +static struct kmem_cache *ntfs_enode_cachep; + +int __init ntfs3_init_bitmap(void) +{ + ntfs_enode_cachep = kmem_cache_create("ntfs3_enode_cache", + sizeof(struct e_node), 0, + SLAB_RECLAIM_ACCOUNT, NULL); + return ntfs_enode_cachep ? 0 : -ENOMEM; +} + +void ntfs3_exit_bitmap(void) +{ + kmem_cache_destroy(ntfs_enode_cachep); +} + +/* + * wnd_scan + * + * b_pos + b_len - biggest fragment. + * Scan range [wpos wbits) window @buf. + * + * Return: -1 if not found. + */ +static size_t wnd_scan(const void *buf, size_t wbit, u32 wpos, u32 wend, + size_t to_alloc, size_t *prev_tail, size_t *b_pos, + size_t *b_len) +{ + while (wpos < wend) { + size_t free_len; + u32 free_bits, end; + u32 used = find_next_zero_bit_le(buf, wend, wpos); + + if (used >= wend) { + if (*b_len < *prev_tail) { + *b_pos = wbit - *prev_tail; + *b_len = *prev_tail; + } + + *prev_tail = 0; + return -1; + } + + if (used > wpos) { + wpos = used; + if (*b_len < *prev_tail) { + *b_pos = wbit - *prev_tail; + *b_len = *prev_tail; + } + + *prev_tail = 0; + } + + /* + * Now we have a fragment [wpos, wend) staring with 0. + */ + end = wpos + to_alloc - *prev_tail; + free_bits = find_next_bit_le(buf, min(end, wend), wpos); + + free_len = *prev_tail + free_bits - wpos; + + if (*b_len < free_len) { + *b_pos = wbit + wpos - *prev_tail; + *b_len = free_len; + } + + if (free_len >= to_alloc) + return wbit + wpos - *prev_tail; + + if (free_bits >= wend) { + *prev_tail += free_bits - wpos; + return -1; + } + + wpos = free_bits + 1; + + *prev_tail = 0; + } + + return -1; +} + +/* + * wnd_close - Frees all resources. + */ +void wnd_close(struct wnd_bitmap *wnd) +{ + struct rb_node *node, *next; + + kvfree(wnd->free_bits); + wnd->free_bits = NULL; + run_close(&wnd->run); + + node = rb_first(&wnd->start_tree); + + while (node) { + next = rb_next(node); + rb_erase(node, &wnd->start_tree); + kmem_cache_free(ntfs_enode_cachep, + rb_entry(node, struct e_node, start.node)); + node = next; + } +} + +static struct rb_node *rb_lookup(struct rb_root *root, size_t v) +{ + struct rb_node **p = &root->rb_node; + struct rb_node *r = NULL; + + while (*p) { + struct rb_node_key *k; + + k = rb_entry(*p, struct rb_node_key, node); + if (v < k->key) { + p = &(*p)->rb_left; + } else if (v > k->key) { + r = &k->node; + p = &(*p)->rb_right; + } else { + return &k->node; + } + } + + return r; +} + +/* + * rb_insert_count - Helper function to insert special kind of 'count' tree. + */ +static inline bool rb_insert_count(struct rb_root *root, struct e_node *e) +{ + struct rb_node **p = &root->rb_node; + struct rb_node *parent = NULL; + size_t e_ckey = e->count.key; + size_t e_skey = e->start.key; + + while (*p) { + struct e_node *k = + rb_entry(parent = *p, struct e_node, count.node); + + if (e_ckey > k->count.key) { + p = &(*p)->rb_left; + } else if (e_ckey < k->count.key) { + p = &(*p)->rb_right; + } else if (e_skey < k->start.key) { + p = &(*p)->rb_left; + } else if (e_skey > k->start.key) { + p = &(*p)->rb_right; + } else { + WARN_ON(1); + return false; + } + } + + rb_link_node(&e->count.node, parent, p); + rb_insert_color(&e->count.node, root); + return true; +} + +/* + * rb_insert_start - Helper function to insert special kind of 'count' tree. + */ +static inline bool rb_insert_start(struct rb_root *root, struct e_node *e) +{ + struct rb_node **p = &root->rb_node; + struct rb_node *parent = NULL; + size_t e_skey = e->start.key; + + while (*p) { + struct e_node *k; + + parent = *p; + + k = rb_entry(parent, struct e_node, start.node); + if (e_skey < k->start.key) { + p = &(*p)->rb_left; + } else if (e_skey > k->start.key) { + p = &(*p)->rb_right; + } else { + WARN_ON(1); + return false; + } + } + + rb_link_node(&e->start.node, parent, p); + rb_insert_color(&e->start.node, root); + return true; +} + +/* + * wnd_add_free_ext - Adds a new extent of free space. + * @build: 1 when building tree. + */ +static void wnd_add_free_ext(struct wnd_bitmap *wnd, size_t bit, size_t len, + bool build) +{ + struct e_node *e, *e0 = NULL; + size_t ib, end_in = bit + len; + struct rb_node *n; + + if (build) { + /* Use extent_min to filter too short extents. */ + if (wnd->count >= NTFS_MAX_WND_EXTENTS && + len <= wnd->extent_min) { + wnd->uptodated = -1; + return; + } + } else { + /* Try to find extent before 'bit'. */ + n = rb_lookup(&wnd->start_tree, bit); + + if (!n) { + n = rb_first(&wnd->start_tree); + } else { + e = rb_entry(n, struct e_node, start.node); + n = rb_next(n); + if (e->start.key + e->count.key == bit) { + /* Remove left. */ + bit = e->start.key; + len += e->count.key; + rb_erase(&e->start.node, &wnd->start_tree); + rb_erase(&e->count.node, &wnd->count_tree); + wnd->count -= 1; + e0 = e; + } + } + + while (n) { + size_t next_end; + + e = rb_entry(n, struct e_node, start.node); + next_end = e->start.key + e->count.key; + if (e->start.key > end_in) + break; + + /* Remove right. */ + n = rb_next(n); + len += next_end - end_in; + end_in = next_end; + rb_erase(&e->start.node, &wnd->start_tree); + rb_erase(&e->count.node, &wnd->count_tree); + wnd->count -= 1; + + if (!e0) + e0 = e; + else + kmem_cache_free(ntfs_enode_cachep, e); + } + + if (wnd->uptodated != 1) { + /* Check bits before 'bit'. */ + ib = wnd->zone_bit == wnd->zone_end || + bit < wnd->zone_end ? + 0 : + wnd->zone_end; + + while (bit > ib && wnd_is_free_hlp(wnd, bit - 1, 1)) { + bit -= 1; + len += 1; + } + + /* Check bits after 'end_in'. */ + ib = wnd->zone_bit == wnd->zone_end || + end_in > wnd->zone_bit ? + wnd->nbits : + wnd->zone_bit; + + while (end_in < ib && wnd_is_free_hlp(wnd, end_in, 1)) { + end_in += 1; + len += 1; + } + } + } + /* Insert new fragment. */ + if (wnd->count >= NTFS_MAX_WND_EXTENTS) { + if (e0) + kmem_cache_free(ntfs_enode_cachep, e0); + + wnd->uptodated = -1; + + /* Compare with smallest fragment. */ + n = rb_last(&wnd->count_tree); + e = rb_entry(n, struct e_node, count.node); + if (len <= e->count.key) + goto out; /* Do not insert small fragments. */ + + if (build) { + struct e_node *e2; + + n = rb_prev(n); + e2 = rb_entry(n, struct e_node, count.node); + /* Smallest fragment will be 'e2->count.key'. */ + wnd->extent_min = e2->count.key; + } + + /* Replace smallest fragment by new one. */ + rb_erase(&e->start.node, &wnd->start_tree); + rb_erase(&e->count.node, &wnd->count_tree); + wnd->count -= 1; + } else { + e = e0 ? e0 : kmem_cache_alloc(ntfs_enode_cachep, GFP_ATOMIC); + if (!e) { + wnd->uptodated = -1; + goto out; + } + + if (build && len <= wnd->extent_min) + wnd->extent_min = len; + } + e->start.key = bit; + e->count.key = len; + if (len > wnd->extent_max) + wnd->extent_max = len; + + rb_insert_start(&wnd->start_tree, e); + rb_insert_count(&wnd->count_tree, e); + wnd->count += 1; + +out:; +} + +/* + * wnd_remove_free_ext - Remove a run from the cached free space. + */ +static void wnd_remove_free_ext(struct wnd_bitmap *wnd, size_t bit, size_t len) +{ + struct rb_node *n, *n3; + struct e_node *e, *e3; + size_t end_in = bit + len; + size_t end3, end, new_key, new_len, max_new_len; + + /* Try to find extent before 'bit'. */ + n = rb_lookup(&wnd->start_tree, bit); + + if (!n) + return; + + e = rb_entry(n, struct e_node, start.node); + end = e->start.key + e->count.key; + + new_key = new_len = 0; + len = e->count.key; + + /* Range [bit,end_in) must be inside 'e' or outside 'e' and 'n'. */ + if (e->start.key > bit) + ; + else if (end_in <= end) { + /* Range [bit,end_in) inside 'e'. */ + new_key = end_in; + new_len = end - end_in; + len = bit - e->start.key; + } else if (bit > end) { + bool bmax = false; + + n3 = rb_next(n); + + while (n3) { + e3 = rb_entry(n3, struct e_node, start.node); + if (e3->start.key >= end_in) + break; + + if (e3->count.key == wnd->extent_max) + bmax = true; + + end3 = e3->start.key + e3->count.key; + if (end3 > end_in) { + e3->start.key = end_in; + rb_erase(&e3->count.node, &wnd->count_tree); + e3->count.key = end3 - end_in; + rb_insert_count(&wnd->count_tree, e3); + break; + } + + n3 = rb_next(n3); + rb_erase(&e3->start.node, &wnd->start_tree); + rb_erase(&e3->count.node, &wnd->count_tree); + wnd->count -= 1; + kmem_cache_free(ntfs_enode_cachep, e3); + } + if (!bmax) + return; + n3 = rb_first(&wnd->count_tree); + wnd->extent_max = + n3 ? rb_entry(n3, struct e_node, count.node)->count.key : + 0; + return; + } + + if (e->count.key != wnd->extent_max) { + ; + } else if (rb_prev(&e->count.node)) { + ; + } else { + n3 = rb_next(&e->count.node); + max_new_len = max(len, new_len); + if (!n3) { + wnd->extent_max = max_new_len; + } else { + e3 = rb_entry(n3, struct e_node, count.node); + wnd->extent_max = max(e3->count.key, max_new_len); + } + } + + if (!len) { + if (new_len) { + e->start.key = new_key; + rb_erase(&e->count.node, &wnd->count_tree); + e->count.key = new_len; + rb_insert_count(&wnd->count_tree, e); + } else { + rb_erase(&e->start.node, &wnd->start_tree); + rb_erase(&e->count.node, &wnd->count_tree); + wnd->count -= 1; + kmem_cache_free(ntfs_enode_cachep, e); + } + goto out; + } + rb_erase(&e->count.node, &wnd->count_tree); + e->count.key = len; + rb_insert_count(&wnd->count_tree, e); + + if (!new_len) + goto out; + + if (wnd->count >= NTFS_MAX_WND_EXTENTS) { + wnd->uptodated = -1; + + /* Get minimal extent. */ + e = rb_entry(rb_last(&wnd->count_tree), struct e_node, + count.node); + if (e->count.key > new_len) + goto out; + + /* Replace minimum. */ + rb_erase(&e->start.node, &wnd->start_tree); + rb_erase(&e->count.node, &wnd->count_tree); + wnd->count -= 1; + } else { + e = kmem_cache_alloc(ntfs_enode_cachep, GFP_ATOMIC); + if (!e) + wnd->uptodated = -1; + } + + if (e) { + e->start.key = new_key; + e->count.key = new_len; + rb_insert_start(&wnd->start_tree, e); + rb_insert_count(&wnd->count_tree, e); + wnd->count += 1; + } + +out: + if (!wnd->count && 1 != wnd->uptodated) + wnd_rescan(wnd); +} + +/* + * wnd_rescan - Scan all bitmap. Used while initialization. + */ +static int wnd_rescan(struct wnd_bitmap *wnd) +{ + int err = 0; + size_t prev_tail = 0; + struct super_block *sb = wnd->sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + u64 lbo, len = 0; + u32 blocksize = sb->s_blocksize; + u8 cluster_bits = sbi->cluster_bits; + u32 wbits = 8 * sb->s_blocksize; + u32 used, frb; + size_t wpos, wbit, iw, vbo; + struct buffer_head *bh = NULL; + CLST lcn, clen; + struct file_ra_state *ra; + struct address_space *mapping = sb->s_bdev->bd_mapping; + + wnd->uptodated = 0; + wnd->extent_max = 0; + wnd->extent_min = MINUS_ONE_T; + wnd->total_zeroes = 0; + + vbo = 0; + + /* Allocate in memory instead of stack. Not critical if failed. */ + ra = kzalloc_obj(*ra, GFP_NOFS); + if (ra) { + file_ra_state_init(ra, mapping); + ra->ra_pages = (wnd->nbits / 8 + PAGE_SIZE - 1) >> PAGE_SHIFT; + } + + for (iw = 0; iw < wnd->nwnd; iw++) { + if (iw + 1 == wnd->nwnd) + wbits = wnd->bits_last; + + if (wnd->inited) { + if (!wnd->free_bits[iw]) { + /* All ones. */ + if (prev_tail) { + wnd_add_free_ext(wnd, + vbo * 8 - prev_tail, + prev_tail, true); + prev_tail = 0; + } + goto next_wnd; + } + if (wbits == wnd->free_bits[iw]) { + /* All zeroes. */ + prev_tail += wbits; + wnd->total_zeroes += wbits; + goto next_wnd; + } + } + + if (!len) { + u32 off = vbo & sbi->cluster_mask; + + if (!run_lookup_entry(&wnd->run, vbo >> cluster_bits, + &lcn, &clen, NULL)) { + err = -ENOENT; + goto out; + } + + lbo = ((u64)lcn << cluster_bits) + off; + len = ((u64)clen << cluster_bits) - off; + } + + if (ra) { + pgoff_t idx = lbo >> PAGE_SHIFT; + if (!ra_has_index(ra, idx)) + page_cache_sync_readahead(mapping, ra, NULL, + idx, 1); + } + + bh = ntfs_bread(sb, lbo >> sb->s_blocksize_bits); + if (!bh) { + err = -EIO; + goto out; + } + + used = ntfs_bitmap_weight_le(bh->b_data, wbits); + if (used < wbits) { + frb = wbits - used; + wnd->free_bits[iw] = frb; + wnd->total_zeroes += frb; + } + + wpos = 0; + wbit = vbo * 8; + + if (wbit + wbits > wnd->nbits) + wbits = wnd->nbits - wbit; + + do { + used = find_next_zero_bit_le(bh->b_data, wbits, wpos); + + if (used > wpos && prev_tail) { + wnd_add_free_ext(wnd, wbit + wpos - prev_tail, + prev_tail, true); + prev_tail = 0; + } + + wpos = used; + + if (wpos >= wbits) { + /* No free blocks. */ + prev_tail = 0; + break; + } + + frb = find_next_bit_le(bh->b_data, wbits, wpos); + if (frb >= wbits) { + /* Keep last free block. */ + prev_tail += frb - wpos; + break; + } + + wnd_add_free_ext(wnd, wbit + wpos - prev_tail, + frb + prev_tail - wpos, true); + + /* Skip free block and first '1'. */ + wpos = frb + 1; + /* Reset previous tail. */ + prev_tail = 0; + } while (wpos < wbits); + +next_wnd: + + if (bh) + put_bh(bh); + bh = NULL; + + vbo += blocksize; + if (len) { + len -= blocksize; + lbo += blocksize; + } + } + + /* Add last block. */ + if (prev_tail) + wnd_add_free_ext(wnd, wnd->nbits - prev_tail, prev_tail, true); + + /* + * Before init cycle wnd->uptodated was 0. + * If any errors or limits occurs while initialization then + * wnd->uptodated will be -1. + * If 'uptodated' is still 0 then Tree is really updated. + */ + if (!wnd->uptodated) + wnd->uptodated = 1; + + if (wnd->zone_bit != wnd->zone_end) { + size_t zlen = wnd->zone_end - wnd->zone_bit; + + wnd->zone_end = wnd->zone_bit; + wnd_zone_set(wnd, wnd->zone_bit, zlen); + } + +out: + kfree(ra); + return err; +} + +int wnd_init(struct wnd_bitmap *wnd, struct super_block *sb, size_t nbits) +{ + int err; + u32 blocksize = sb->s_blocksize; + u32 wbits = blocksize * 8; + + init_rwsem(&wnd->rw_lock); + + wnd->sb = sb; + wnd->nbits = nbits; + wnd->total_zeroes = nbits; + wnd->extent_max = MINUS_ONE_T; + wnd->zone_bit = wnd->zone_end = 0; + wnd->nwnd = bytes_to_block(sb, ntfs3_bitmap_size(nbits)); + wnd->bits_last = nbits & (wbits - 1); + if (!wnd->bits_last) + wnd->bits_last = wbits; + + wnd->free_bits = + kvmalloc_array(wnd->nwnd, sizeof(u16), GFP_KERNEL | __GFP_ZERO); + + if (!wnd->free_bits) + return -ENOMEM; + + err = wnd_rescan(wnd); + if (err) + return err; + + wnd->inited = true; + + return 0; +} + +/* + * wnd_map - Call sb_bread for requested window. + */ +static struct buffer_head *wnd_map(struct wnd_bitmap *wnd, size_t iw) +{ + size_t vbo; + CLST lcn, clen; + struct super_block *sb = wnd->sb; + struct ntfs_sb_info *sbi; + struct buffer_head *bh; + u64 lbo; + + sbi = sb->s_fs_info; + vbo = (u64)iw << sb->s_blocksize_bits; + + if (!run_lookup_entry(&wnd->run, vbo >> sbi->cluster_bits, &lcn, &clen, + NULL)) { + return ERR_PTR(-ENOENT); + } + + lbo = ((u64)lcn << sbi->cluster_bits) + (vbo & sbi->cluster_mask); + + bh = ntfs_bread(wnd->sb, lbo >> sb->s_blocksize_bits); + if (!bh) + return ERR_PTR(-EIO); + + return bh; +} + +/* + * wnd_set_free - Mark the bits range from bit to bit + bits as free. + */ +int wnd_set_free(struct wnd_bitmap *wnd, size_t bit, size_t bits) +{ + int err = 0; + struct super_block *sb = wnd->sb; + u32 wbits = 8 * sb->s_blocksize; + size_t iw = bit >> (sb->s_blocksize_bits + 3); + u32 wbit = bit & (wbits - 1); + struct buffer_head *bh; + u32 op; + + for (; iw < wnd->nwnd && bits; iw++, bit += op, bits -= op, wbit = 0) { + if (iw + 1 == wnd->nwnd) + wbits = wnd->bits_last; + + op = min_t(u32, wbits - wbit, bits); + + bh = wnd_map(wnd, iw); + if (IS_ERR(bh)) { + err = PTR_ERR(bh); + break; + } + + lock_buffer(bh); + + ntfs_bitmap_clear_le(bh->b_data, wbit, op); + + wnd->free_bits[iw] += op; + wnd->total_zeroes += op; + + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + unlock_buffer(bh); + put_bh(bh); + + wnd_add_free_ext(wnd, bit, op, false); + } + return err; +} + +/* + * wnd_set_used - Mark the bits range from bit to bit + bits as used. + */ +int wnd_set_used(struct wnd_bitmap *wnd, size_t bit, size_t bits) +{ + int err = 0; + struct super_block *sb = wnd->sb; + size_t iw = bit >> (sb->s_blocksize_bits + 3); + u32 wbits = 8 * sb->s_blocksize; + u32 wbit = bit & (wbits - 1); + struct buffer_head *bh; + u32 op; + + for (; iw < wnd->nwnd && bits; iw++, bit += op, bits -= op, wbit = 0) { + if (unlikely(iw + 1 == wnd->nwnd)) + wbits = wnd->bits_last; + + op = min_t(u32, wbits - wbit, bits); + + bh = wnd_map(wnd, iw); + if (IS_ERR(bh)) { + err = PTR_ERR(bh); + break; + } + + lock_buffer(bh); + + ntfs_bitmap_set_le(bh->b_data, wbit, op); + wnd->free_bits[iw] -= op; + wnd->total_zeroes -= op; + + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + unlock_buffer(bh); + put_bh(bh); + + if (!RB_EMPTY_ROOT(&wnd->start_tree)) + wnd_remove_free_ext(wnd, bit, op); + } + return err; +} + +/* + * wnd_set_used_safe - Mark the bits range from bit to bit + bits as used. + * + * Unlikely wnd_set_used/wnd_set_free this function is not full trusted. + * It scans every bit in bitmap and marks free bit as used. + * @done - how many bits were marked as used. + * + * NOTE: normally *done should be 0. + */ +int wnd_set_used_safe(struct wnd_bitmap *wnd, size_t bit, size_t bits, + size_t *done) +{ + size_t i, from = 0, len = 0; + int err = 0; + + *done = 0; + for (i = 0; i < bits; i++) { + if (wnd_is_free(wnd, bit + i, 1)) { + if (!len) + from = bit + i; + len += 1; + } else if (len) { + err = wnd_set_used(wnd, from, len); + *done += len; + len = 0; + if (err) + break; + } + } + + if (len) { + /* last fragment. */ + err = wnd_set_used(wnd, from, len); + *done += len; + } + return err; +} + +/* + * wnd_is_free_hlp + * + * Return: True if all clusters [bit, bit+bits) are free (bitmap only). + */ +static bool wnd_is_free_hlp(struct wnd_bitmap *wnd, size_t bit, size_t bits) +{ + struct super_block *sb = wnd->sb; + size_t iw = bit >> (sb->s_blocksize_bits + 3); + u32 wbits = 8 * sb->s_blocksize; + u32 wbit = bit & (wbits - 1); + u32 op; + + for (; iw < wnd->nwnd && bits; iw++, bits -= op, wbit = 0) { + if (unlikely(iw + 1 == wnd->nwnd)) + wbits = wnd->bits_last; + + op = min_t(u32, wbits - wbit, bits); + + if (wbits != wnd->free_bits[iw]) { + bool ret; + struct buffer_head *bh = wnd_map(wnd, iw); + + if (IS_ERR(bh)) + return false; + + ret = are_bits_clear(bh->b_data, wbit, op); + + put_bh(bh); + if (!ret) + return false; + } + } + + return true; +} + +/* + * wnd_is_free + * + * Return: True if all clusters [bit, bit+bits) are free. + */ +bool wnd_is_free(struct wnd_bitmap *wnd, size_t bit, size_t bits) +{ + bool ret; + struct rb_node *n; + size_t end; + struct e_node *e; + + if (RB_EMPTY_ROOT(&wnd->start_tree)) + goto use_wnd; + + n = rb_lookup(&wnd->start_tree, bit); + if (!n) + goto use_wnd; + + e = rb_entry(n, struct e_node, start.node); + + end = e->start.key + e->count.key; + + if (bit < end && bit + bits <= end) + return true; + +use_wnd: + ret = wnd_is_free_hlp(wnd, bit, bits); + + return ret; +} + +/* + * wnd_is_used + * + * Return: True if all clusters [bit, bit+bits) are used. + */ +bool wnd_is_used(struct wnd_bitmap *wnd, size_t bit, size_t bits) +{ + bool ret = false; + struct super_block *sb = wnd->sb; + size_t iw = bit >> (sb->s_blocksize_bits + 3); + u32 wbits = 8 * sb->s_blocksize; + u32 wbit = bit & (wbits - 1); + u32 op; + size_t end; + struct rb_node *n; + struct e_node *e; + + if (RB_EMPTY_ROOT(&wnd->start_tree)) + goto use_wnd; + + end = bit + bits; + n = rb_lookup(&wnd->start_tree, end - 1); + if (!n) + goto use_wnd; + + e = rb_entry(n, struct e_node, start.node); + if (e->start.key + e->count.key > bit) + return false; + +use_wnd: + for (; iw < wnd->nwnd && bits; iw++, bits -= op, wbit = 0) { + if (unlikely(iw + 1 == wnd->nwnd)) + wbits = wnd->bits_last; + + op = min_t(u32, wbits - wbit, bits); + + if (wnd->free_bits[iw]) { + bool ret; + struct buffer_head *bh = wnd_map(wnd, iw); + + if (IS_ERR(bh)) + goto out; + + ret = are_bits_set(bh->b_data, wbit, op); + put_bh(bh); + if (!ret) + goto out; + } + } + ret = true; + +out: + return ret; +} + +/* + * wnd_find - Look for free space. + * + * - flags - BITMAP_FIND_XXX flags + * + * Return: 0 if not found. + */ +size_t wnd_find(struct wnd_bitmap *wnd, size_t to_alloc, size_t hint, + size_t flags, size_t *allocated) +{ + struct super_block *sb; + u32 wbits, wpos, wzbit, wzend; + size_t fnd, max_alloc, b_len, b_pos; + size_t iw, prev_tail, nwnd, wbit, ebit, zbit, zend; + size_t to_alloc0 = to_alloc; + const struct e_node *e; + const struct rb_node *pr, *cr; + u8 log2_bits; + bool fbits_valid; + struct buffer_head *bh; + + /* Fast checking for available free space. */ + if (flags & BITMAP_FIND_FULL) { + size_t zeroes = wnd_zeroes(wnd); + + zeroes -= wnd->zone_end - wnd->zone_bit; + if (zeroes < to_alloc0) + goto no_space; + + if (to_alloc0 > wnd->extent_max) + goto no_space; + } else { + if (to_alloc > wnd->extent_max) + to_alloc = wnd->extent_max; + } + + if (wnd->zone_bit <= hint && hint < wnd->zone_end) + hint = wnd->zone_end; + + max_alloc = wnd->nbits; + b_len = b_pos = 0; + + if (hint >= max_alloc) + hint = 0; + + if (RB_EMPTY_ROOT(&wnd->start_tree)) { + if (wnd->uptodated == 1) { + /* Extents tree is updated -> No free space. */ + goto no_space; + } + goto scan_bitmap; + } + + e = NULL; + if (!hint) + goto allocate_biggest; + + /* Use hint: Enumerate extents by start >= hint. */ + pr = NULL; + cr = wnd->start_tree.rb_node; + + for (;;) { + e = rb_entry(cr, struct e_node, start.node); + + if (e->start.key == hint) + break; + + if (e->start.key < hint) { + pr = cr; + cr = cr->rb_right; + if (!cr) + break; + continue; + } + + cr = cr->rb_left; + if (!cr) { + e = pr ? rb_entry(pr, struct e_node, start.node) : NULL; + break; + } + } + + if (!e) + goto allocate_biggest; + + if (e->start.key + e->count.key > hint) { + /* We have found extension with 'hint' inside. */ + size_t len = e->start.key + e->count.key - hint; + + if (len >= to_alloc && hint + to_alloc <= max_alloc) { + fnd = hint; + goto found; + } + + if (!(flags & BITMAP_FIND_FULL)) { + if (len > to_alloc) + len = to_alloc; + + if (hint + len <= max_alloc) { + fnd = hint; + to_alloc = len; + goto found; + } + } + } + +allocate_biggest: + /* Allocate from biggest free extent. */ + e = rb_entry(rb_first(&wnd->count_tree), struct e_node, count.node); + if (e->count.key != wnd->extent_max) + wnd->extent_max = e->count.key; + + if (e->count.key < max_alloc) { + if (e->count.key >= to_alloc) { + ; + } else if (flags & BITMAP_FIND_FULL) { + if (e->count.key < to_alloc0) { + /* Biggest free block is less then requested. */ + goto no_space; + } + to_alloc = e->count.key; + } else if (-1 != wnd->uptodated) { + to_alloc = e->count.key; + } else { + /* Check if we can use more bits. */ + size_t op, max_check; + struct rb_root start_tree; + + memcpy(&start_tree, &wnd->start_tree, + sizeof(struct rb_root)); + memset(&wnd->start_tree, 0, sizeof(struct rb_root)); + + max_check = e->start.key + to_alloc; + if (max_check > max_alloc) + max_check = max_alloc; + for (op = e->start.key + e->count.key; op < max_check; + op++) { + if (!wnd_is_free(wnd, op, 1)) + break; + } + memcpy(&wnd->start_tree, &start_tree, + sizeof(struct rb_root)); + to_alloc = op - e->start.key; + } + + /* Prepare to return. */ + fnd = e->start.key; + if (e->start.key + to_alloc > max_alloc) + to_alloc = max_alloc - e->start.key; + goto found; + } + + if (wnd->uptodated == 1) { + /* Extents tree is updated -> no free space. */ + goto no_space; + } + + b_len = e->count.key; + b_pos = e->start.key; + +scan_bitmap: + sb = wnd->sb; + log2_bits = sb->s_blocksize_bits + 3; + + /* At most two ranges [hint, max_alloc) + [0, hint). */ +Again: + + /* TODO: Optimize request for case nbits > wbits. */ + iw = hint >> log2_bits; + wbits = sb->s_blocksize * 8; + wpos = hint & (wbits - 1); + prev_tail = 0; + fbits_valid = true; + + if (max_alloc == wnd->nbits) { + nwnd = wnd->nwnd; + } else { + size_t t = max_alloc + wbits - 1; + + nwnd = likely(t > max_alloc) ? (t >> log2_bits) : wnd->nwnd; + } + + /* Enumerate all windows. */ + for (; iw < nwnd; iw++) { + wbit = iw << log2_bits; + + if (!wnd->free_bits[iw]) { + if (prev_tail > b_len) { + b_pos = wbit - prev_tail; + b_len = prev_tail; + } + + /* Skip full used window. */ + prev_tail = 0; + wpos = 0; + continue; + } + + if (unlikely(iw + 1 == nwnd)) { + if (max_alloc == wnd->nbits) { + wbits = wnd->bits_last; + } else { + size_t t = max_alloc & (wbits - 1); + + if (t) { + wbits = t; + fbits_valid = false; + } + } + } + + if (wnd->zone_end > wnd->zone_bit) { + ebit = wbit + wbits; + zbit = max(wnd->zone_bit, wbit); + zend = min(wnd->zone_end, ebit); + + /* Here we have a window [wbit, ebit) and zone [zbit, zend). */ + if (zend <= zbit) { + /* Zone does not overlap window. */ + } else { + wzbit = zbit - wbit; + wzend = zend - wbit; + + /* Zone overlaps window. */ + if (wnd->free_bits[iw] == wzend - wzbit) { + prev_tail = 0; + wpos = 0; + continue; + } + + /* Scan two ranges window: [wbit, zbit) and [zend, ebit). */ + bh = wnd_map(wnd, iw); + + if (IS_ERR(bh)) { + /* TODO: Error */ + prev_tail = 0; + wpos = 0; + continue; + } + + /* Scan range [wbit, zbit). */ + if (wpos < wzbit) { + /* Scan range [wpos, zbit). */ + fnd = wnd_scan(bh->b_data, wbit, wpos, + wzbit, to_alloc, + &prev_tail, &b_pos, + &b_len); + if (fnd != MINUS_ONE_T) { + put_bh(bh); + goto found; + } + } + + prev_tail = 0; + + /* Scan range [zend, ebit). */ + if (wzend < wbits) { + fnd = wnd_scan(bh->b_data, wbit, + max(wzend, wpos), wbits, + to_alloc, &prev_tail, + &b_pos, &b_len); + if (fnd != MINUS_ONE_T) { + put_bh(bh); + goto found; + } + } + + wpos = 0; + put_bh(bh); + continue; + } + } + + /* Current window does not overlap zone. */ + if (!wpos && fbits_valid && wnd->free_bits[iw] == wbits) { + /* Window is empty. */ + if (prev_tail + wbits >= to_alloc) { + fnd = wbit + wpos - prev_tail; + goto found; + } + + /* Increase 'prev_tail' and process next window. */ + prev_tail += wbits; + wpos = 0; + continue; + } + + /* Read window. */ + bh = wnd_map(wnd, iw); + if (IS_ERR(bh)) { + // TODO: Error. + prev_tail = 0; + wpos = 0; + continue; + } + + /* Scan range [wpos, eBits). */ + fnd = wnd_scan(bh->b_data, wbit, wpos, wbits, to_alloc, + &prev_tail, &b_pos, &b_len); + put_bh(bh); + if (fnd != MINUS_ONE_T) + goto found; + } + + if (b_len < prev_tail) { + /* The last fragment. */ + b_len = prev_tail; + b_pos = max_alloc - prev_tail; + } + + if (hint) { + /* + * We have scanned range [hint max_alloc). + * Prepare to scan range [0 hint + to_alloc). + */ + size_t nextmax = hint + to_alloc; + + if (likely(nextmax >= hint) && nextmax < max_alloc) + max_alloc = nextmax; + hint = 0; + goto Again; + } + + if (!b_len) + goto no_space; + + wnd->extent_max = b_len; + + if (flags & BITMAP_FIND_FULL) + goto no_space; + + fnd = b_pos; + to_alloc = b_len; + +found: + if (flags & BITMAP_FIND_MARK_AS_USED) { + /* TODO: Optimize remove extent (pass 'e'?). */ + if (wnd_set_used(wnd, fnd, to_alloc)) + goto no_space; + } else if (wnd->extent_max != MINUS_ONE_T && + to_alloc > wnd->extent_max) { + wnd->extent_max = to_alloc; + } + + *allocated = fnd; + return to_alloc; + +no_space: + return 0; +} + +/* + * wnd_extend - Extend bitmap ($MFT bitmap). + */ +int wnd_extend(struct wnd_bitmap *wnd, size_t new_bits) +{ + int err; + struct super_block *sb = wnd->sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + u32 blocksize = sb->s_blocksize; + u32 wbits = blocksize * 8; + u32 b0, new_last; + size_t bits, iw, new_wnd; + size_t old_bits = wnd->nbits; + u16 *new_free; + + if (new_bits <= old_bits) + return -EINVAL; + + /* Align to 8 byte boundary. */ + new_wnd = bytes_to_block(sb, ntfs3_bitmap_size(new_bits)); + new_last = new_bits & (wbits - 1); + if (!new_last) + new_last = wbits; + + if (new_wnd != wnd->nwnd) { + new_free = kmalloc_array(new_wnd, sizeof(u16), GFP_NOFS); + if (!new_free) + return -ENOMEM; + + memcpy(new_free, wnd->free_bits, wnd->nwnd * sizeof(short)); + memset(new_free + wnd->nwnd, 0, + (new_wnd - wnd->nwnd) * sizeof(short)); + kvfree(wnd->free_bits); + wnd->free_bits = new_free; + } + + /* Zero bits [old_bits,new_bits). */ + bits = new_bits - old_bits; + b0 = old_bits & (wbits - 1); + + for (iw = old_bits >> (sb->s_blocksize_bits + 3); bits; iw += 1) { + u32 op; + size_t frb; + u64 vbo, lbo, bytes; + struct buffer_head *bh; + + if (iw + 1 == new_wnd) + wbits = new_last; + + op = b0 + bits > wbits ? wbits - b0 : bits; + vbo = (u64)iw * blocksize; + + err = ntfs_vbo_to_lbo(sbi, &wnd->run, vbo, &lbo, &bytes); + if (err) + return err; + + bh = ntfs_bread(sb, lbo >> sb->s_blocksize_bits); + if (!bh) + return -EIO; + + lock_buffer(bh); + + ntfs_bitmap_clear_le(bh->b_data, b0, blocksize * 8 - b0); + frb = wbits - ntfs_bitmap_weight_le(bh->b_data, wbits); + wnd->total_zeroes += frb - wnd->free_bits[iw]; + wnd->free_bits[iw] = frb; + + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + unlock_buffer(bh); + /* err = sync_dirty_buffer(bh); */ + put_bh(bh); + + b0 = 0; + bits -= op; + } + + wnd->nbits = new_bits; + wnd->nwnd = new_wnd; + wnd->bits_last = new_last; + + wnd_add_free_ext(wnd, old_bits, new_bits - old_bits, false); + + return 0; +} + +void wnd_zone_set(struct wnd_bitmap *wnd, size_t lcn, size_t len) +{ + size_t zlen = wnd->zone_end - wnd->zone_bit; + + if (zlen) + wnd_add_free_ext(wnd, wnd->zone_bit, zlen, false); + + if (!RB_EMPTY_ROOT(&wnd->start_tree) && len) + wnd_remove_free_ext(wnd, lcn, len); + + wnd->zone_bit = lcn; + wnd->zone_end = lcn + len; +} + +int ntfs_trim_fs(struct ntfs_sb_info *sbi, struct fstrim_range *range) +{ + int err = 0; + struct super_block *sb = sbi->sb; + struct wnd_bitmap *wnd = &sbi->used.bitmap; + u32 wbits = 8 * sb->s_blocksize; + CLST len = 0, lcn = 0, done = 0; + CLST minlen = bytes_to_cluster(sbi, range->minlen); + CLST lcn_from = bytes_to_cluster(sbi, range->start); + size_t iw = lcn_from >> (sb->s_blocksize_bits + 3); + u32 wbit = lcn_from & (wbits - 1); + CLST lcn_to; + + if (!minlen) + minlen = 1; + + if (range->len == (u64)-1) + lcn_to = wnd->nbits; + else + lcn_to = bytes_to_cluster(sbi, range->start + range->len); + + down_read_nested(&wnd->rw_lock, BITMAP_MUTEX_CLUSTERS); + + for (; iw < wnd->nwnd; iw++, wbit = 0) { + CLST lcn_wnd = iw * wbits; + struct buffer_head *bh; + + if (lcn_wnd > lcn_to) + break; + + if (!wnd->free_bits[iw]) + continue; + + if (iw + 1 == wnd->nwnd) + wbits = wnd->bits_last; + + if (lcn_wnd + wbits > lcn_to) + wbits = lcn_to - lcn_wnd; + + bh = wnd_map(wnd, iw); + if (IS_ERR(bh)) { + err = PTR_ERR(bh); + break; + } + + for (; wbit < wbits; wbit++) { + if (!test_bit_le(wbit, bh->b_data)) { + if (!len) + lcn = lcn_wnd + wbit; + len += 1; + continue; + } + if (len >= minlen) { + err = ntfs_discard(sbi, lcn, len); + if (err) + goto out; + done += len; + } + len = 0; + } + put_bh(bh); + } + + /* Process the last fragment. */ + if (len >= minlen) { + err = ntfs_discard(sbi, lcn, len); + if (err) + goto out; + done += len; + } + +out: + range->len = (u64)done << sbi->cluster_bits; + + up_read(&wnd->rw_lock); + + return err; +} + +#if BITS_PER_LONG == 64 +typedef __le64 bitmap_ulong; +#define cpu_to_ul(x) cpu_to_le64(x) +#define ul_to_cpu(x) le64_to_cpu(x) +#else +typedef __le32 bitmap_ulong; +#define cpu_to_ul(x) cpu_to_le32(x) +#define ul_to_cpu(x) le32_to_cpu(x) +#endif + +void ntfs_bitmap_set_le(void *map, unsigned int start, int len) +{ + bitmap_ulong *p = (bitmap_ulong *)map + BIT_WORD(start); + const unsigned int size = start + len; + int bits_to_set = BITS_PER_LONG - (start % BITS_PER_LONG); + bitmap_ulong mask_to_set = cpu_to_ul(BITMAP_FIRST_WORD_MASK(start)); + + while (len - bits_to_set >= 0) { + *p |= mask_to_set; + len -= bits_to_set; + bits_to_set = BITS_PER_LONG; + mask_to_set = cpu_to_ul(~0UL); + p++; + } + if (len) { + mask_to_set &= cpu_to_ul(BITMAP_LAST_WORD_MASK(size)); + *p |= mask_to_set; + } +} + +void ntfs_bitmap_clear_le(void *map, unsigned int start, int len) +{ + bitmap_ulong *p = (bitmap_ulong *)map + BIT_WORD(start); + const unsigned int size = start + len; + int bits_to_clear = BITS_PER_LONG - (start % BITS_PER_LONG); + bitmap_ulong mask_to_clear = cpu_to_ul(BITMAP_FIRST_WORD_MASK(start)); + + while (len - bits_to_clear >= 0) { + *p &= ~mask_to_clear; + len -= bits_to_clear; + bits_to_clear = BITS_PER_LONG; + mask_to_clear = cpu_to_ul(~0UL); + p++; + } + if (len) { + mask_to_clear &= cpu_to_ul(BITMAP_LAST_WORD_MASK(size)); + *p &= ~mask_to_clear; + } +} + +unsigned int ntfs_bitmap_weight_le(const void *bitmap, int bits) +{ + const ulong *bmp = bitmap; + unsigned int k, lim = bits / BITS_PER_LONG; + unsigned int w = 0; + + for (k = 0; k < lim; k++) + w += hweight_long(bmp[k]); + + if (bits % BITS_PER_LONG) { + w += hweight_long(ul_to_cpu(((bitmap_ulong *)bitmap)[k]) & + BITMAP_LAST_WORD_MASK(bits)); + } + + return w; +} diff --git a/fs/ntfs3/debug.h b/fs/ntfs3/debug.h new file mode 100644 index 000000000..53ef7489c --- /dev/null +++ b/fs/ntfs3/debug.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + * Useful functions for debugging. + * + */ + +// clang-format off +#ifndef _LINUX_NTFS3_DEBUG_H +#define _LINUX_NTFS3_DEBUG_H + +struct super_block; +struct inode; + +#ifndef Add2Ptr +#define Add2Ptr(P, I) ((void *)((u8 *)(P) + (I))) +#define PtrOffset(B, O) ((size_t)((size_t)(O) - (size_t)(B))) +#endif + +#ifdef CONFIG_PRINTK +__printf(2, 3) +void ntfs_printk(const struct super_block *sb, const char *fmt, ...); +__printf(2, 3) +void ntfs_inode_printk(struct inode *inode, const char *fmt, ...); +#else +static inline __printf(2, 3) +void ntfs_printk(const struct super_block *sb, const char *fmt, ...) +{ +} + +static inline __printf(2, 3) +void ntfs_inode_printk(struct inode *inode, const char *fmt, ...) +{ +} +#endif + +/* + * Logging macros. Thanks Joe Perches for implementation. + */ + +#define ntfs_err(sb, fmt, ...) ntfs_printk(sb, KERN_ERR fmt, ##__VA_ARGS__) +#define ntfs_warn(sb, fmt, ...) ntfs_printk(sb, KERN_WARNING fmt, ##__VA_ARGS__) +#define ntfs_info(sb, fmt, ...) ntfs_printk(sb, KERN_INFO fmt, ##__VA_ARGS__) +#define ntfs_notice(sb, fmt, ...) \ + ntfs_printk(sb, KERN_NOTICE fmt, ##__VA_ARGS__) + +#define ntfs_inode_err(inode, fmt, ...) \ + ntfs_inode_printk(inode, KERN_ERR fmt, ##__VA_ARGS__) +#define ntfs_inode_warn(inode, fmt, ...) \ + ntfs_inode_printk(inode, KERN_WARNING fmt, ##__VA_ARGS__) + +#endif /* _LINUX_NTFS3_DEBUG_H */ +// clang-format on diff --git a/fs/ntfs3/dir.c b/fs/ntfs3/dir.c new file mode 100644 index 000000000..eb9152e9f --- /dev/null +++ b/fs/ntfs3/dir.c @@ -0,0 +1,747 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + * Directory handling functions for NTFS-based filesystems. + * + */ + +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +/* Convert little endian UTF-16 to NLS string. */ +int ntfs_utf16_to_nls(struct ntfs_sb_info *sbi, const __le16 *name, u32 len, + u8 *buf, int buf_len) +{ + int ret, warn; + u8 *op; + struct nls_table *nls = sbi->options->nls; + + static_assert(sizeof(wchar_t) == sizeof(__le16)); + + if (buf_len <= 0) + return -EINVAL; + + buf_len -= 1; + + if (!nls) { + /* UTF-16 -> UTF-8 */ + ret = utf16s_to_utf8s((wchar_t *)name, len, UTF16_LITTLE_ENDIAN, + buf, buf_len); + buf[ret] = '\0'; + return ret; + } + + op = buf; + warn = 0; + + while (len--) { + u16 ec; + int charlen; + char dump[5]; + + if (buf_len < NLS_MAX_CHARSET_SIZE) { + ntfs_warn(sbi->sb, + "filename was truncated while converting."); + break; + } + + ec = le16_to_cpu(*name++); + charlen = nls->uni2char(ec, op, buf_len); + + if (charlen > 0) { + op += charlen; + buf_len -= charlen; + continue; + } + + *op++ = '_'; + buf_len -= 1; + if (warn) + continue; + + warn = 1; + hex_byte_pack(&dump[0], ec >> 8); + hex_byte_pack(&dump[2], ec); + dump[4] = 0; + + ntfs_err(sbi->sb, "failed to convert \"%s\" to %s", dump, + nls->charset); + } + + *op = '\0'; + return op - buf; +} + +// clang-format off +#define PLANE_SIZE 0x00010000 + +#define SURROGATE_PAIR 0x0000d800 +#define SURROGATE_LOW 0x00000400 +#define SURROGATE_BITS 0x000003ff +// clang-format on + +/* + * put_utf16 - Modified version of put_utf16 from fs/nls/nls_base.c + * + * Function is sparse warnings free. + */ +static inline void put_utf16(wchar_t *s, unsigned int c, + enum utf16_endian endian) +{ + static_assert(sizeof(wchar_t) == sizeof(__le16)); + static_assert(sizeof(wchar_t) == sizeof(__be16)); + + switch (endian) { + default: + *s = (wchar_t)c; + break; + case UTF16_LITTLE_ENDIAN: + *(__le16 *)s = __cpu_to_le16(c); + break; + case UTF16_BIG_ENDIAN: + *(__be16 *)s = __cpu_to_be16(c); + break; + } +} + +/* + * _utf8s_to_utf16s + * + * Modified version of 'utf8s_to_utf16s' allows to + * detect -ENAMETOOLONG without writing out of expected maximum. + */ +static int _utf8s_to_utf16s(const u8 *s, int inlen, enum utf16_endian endian, + wchar_t *pwcs, int maxout) +{ + u16 *op; + int size; + unicode_t u; + + op = pwcs; + while (inlen > 0 && *s) { + if (*s & 0x80) { + size = utf8_to_utf32(s, inlen, &u); + if (size < 0) + return -EINVAL; + s += size; + inlen -= size; + + if (u >= PLANE_SIZE) { + if (maxout < 2) + return -ENAMETOOLONG; + + u -= PLANE_SIZE; + put_utf16(op++, + SURROGATE_PAIR | + ((u >> 10) & SURROGATE_BITS), + endian); + put_utf16(op++, + SURROGATE_PAIR | SURROGATE_LOW | + (u & SURROGATE_BITS), + endian); + maxout -= 2; + } else { + if (maxout < 1) + return -ENAMETOOLONG; + + put_utf16(op++, u, endian); + maxout--; + } + } else { + if (maxout < 1) + return -ENAMETOOLONG; + + put_utf16(op++, *s++, endian); + inlen--; + maxout--; + } + } + return op - pwcs; +} + +/* + * ntfs_nls_to_utf16 - Convert input string to UTF-16. + * @name: Input name. + * @name_len: Input name length. + * @uni: Destination memory. + * @max_ulen: Destination memory. + * @endian: Endian of target UTF-16 string. + * + * This function is called: + * - to create NTFS name + * - to create symlink + * + * Return: UTF-16 string length or error (if negative). + */ +int ntfs_nls_to_utf16(struct ntfs_sb_info *sbi, const u8 *name, u32 name_len, + struct cpu_str *uni, u32 max_ulen, + enum utf16_endian endian) +{ + int ret, slen, i; + const u8 *end; + struct nls_table *nls = sbi->options->nls; + u16 *uname = uni->name; + + static_assert(sizeof(wchar_t) == sizeof(u16)); + + if (!nls) { + /* utf8 -> utf16 */ + ret = _utf8s_to_utf16s(name, name_len, endian, uname, max_ulen); + } else { + for (ret = 0, end = name + name_len; name < end; + ret++, name += slen) { + if (ret >= max_ulen) + return -ENAMETOOLONG; + + slen = nls->char2uni(name, end - name, uname + ret); + if (!slen) + return -EINVAL; + if (slen < 0) + return slen; + } + +#ifdef __BIG_ENDIAN + if (endian == UTF16_LITTLE_ENDIAN) { + i = ret; + + while (i--) { + __cpu_to_le16s(uname); + uname++; + } + } +#else + if (endian == UTF16_BIG_ENDIAN) { + i = ret; + + while (i--) { + __cpu_to_be16s(uname); + uname++; + } + } +#endif + } + + uni->len = ret; + uni->ads_len = 0; + if (ret > 0 && sbi->options->ads) { + uname = uni->name; + /* Find delimiter in range [1 : ret-2). */ + for (i = 1; i + 1 < ret; i++) { + if (uname[i] == ':') { + uni->ads_len = ret - i - 1; + uni->len = i; + uname[i] = 0; + ret = i; + + uname += i + 1; + i = uni->ads_len; + /* Return ADS name as little endian. Always */ +#ifdef __BIG_ENDIAN + if (endian == UTF16_LITTLE_ENDIAN) { + while (i--) { + __cpu_to_le16s(uname); + uname++; + } + } +#else + if (endian == UTF16_BIG_ENDIAN) { + while (i--) { + __cpu_to_be16s(uname); + uname++; + } + } +#endif + break; + } + } + } + + return ret; +} + +/* + * dir_search_u - Helper function. + */ +struct inode *dir_search_flags(struct inode *dir, const struct cpu_str *uni, + struct ntfs_fnd *fnd, u32 flags) +{ + int err = 0; + struct super_block *sb = dir->i_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + struct ntfs_inode *ni = ntfs_i(dir); + struct NTFS_DE *e; + int diff; + struct inode *inode = NULL; + struct ntfs_fnd *fnd_a = NULL; + + if (!fnd) { + fnd_a = fnd_get(); + if (!fnd_a) { + err = -ENOMEM; + goto out; + } + fnd = fnd_a; + } + + err = indx_find(&ni->dir, ni, NULL, uni, 0, sbi, &diff, &e, fnd); + + if (err) + goto out; + + if (diff) { + err = -ENOENT; + goto out; + } + + inode = ntfs_iget5_flags(sb, &e->ref, uni, flags); + if (!IS_ERR(inode) && is_bad_inode(inode)) { + iput(inode); + err = -EINVAL; + } +out: + fnd_put(fnd_a); + + return err == -ENOENT ? NULL : err ? ERR_PTR(err) : inode; +} + +static inline bool de_fname_fits(const struct NTFS_DE *e, u32 e_size, + const struct ATTR_FILE_NAME *fname) +{ + return sizeof(struct NTFS_DE) + fname_full_size(fname) <= e_size; +} + +/* + * returns false if 'ctx' if full + */ +static inline bool ntfs_dir_emit(struct ntfs_sb_info *sbi, + struct ntfs_inode *ni, const struct NTFS_DE *e, + u8 *name, struct dir_context *ctx) +{ + const struct ATTR_FILE_NAME *fname; + u64 ino; + int name_len; + u32 dt_type; + + fname = Add2Ptr(e, sizeof(struct NTFS_DE)); + + if (fname->type == FILE_NAME_DOS) + return true; + + if (!mi_is_ref(&ni->mi, &fname->home)) + return true; + + ino = ino_get(&e->ref); + + if (ino == MFT_REC_ROOT) + return true; + + /* Skip meta files. Unless option to show metafiles is set. */ + if (!sbi->options->showmeta && ntfs_is_meta_file(sbi, ino)) + return true; + + if (sbi->options->nohidden && (fname->dup.fa & FILE_ATTRIBUTE_HIDDEN)) + return true; + + if (!de_fname_fits(e, le16_to_cpu(e->size), fname)) + return true; + + name_len = ntfs_utf16_to_nls(sbi, fname->name, fname->name_len, name, + PATH_MAX); + if (name_len <= 0) { + ntfs_warn(sbi->sb, "failed to convert name for inode %llx.", + ino); + return true; + } + + /* + * NTFS: symlinks are "dir + reparse" or "file + reparse" + * Unfortunately reparse attribute is used for many purposes (several dozens). + * It is not possible here to know is this name symlink or not. + * To get exactly the type of name we should to open inode (read mft). + * getattr for opened file (fstat) correctly returns symlink. + */ + dt_type = (fname->dup.fa & FILE_ATTRIBUTE_DIRECTORY) ? DT_DIR : DT_REG; + + /* + * It is not reliable to detect the type of name using duplicated information + * stored in parent directory. + * The only correct way to get the type of name - read MFT record and find ATTR_STD. + * The code below is not good idea. + * It does additional locks/reads just to get the type of name. + * Should we use additional mount option to enable branch below? + */ + if (fname->dup.extend_data && ino != ni->mi.rno) { + struct inode *inode = ntfs_iget5(sbi->sb, &e->ref, NULL); + if (!IS_ERR_OR_NULL(inode)) { + dt_type = fs_umode_to_dtype(inode->i_mode); + iput(inode); + } + } + + return dir_emit(ctx, (s8 *)name, name_len, ino, dt_type); +} + +/* + * ntfs_read_hdr - Helper function for ntfs_readdir(). + * + * returns 0 if ok. + * returns -EINVAL if directory is corrupted. + * returns +1 if 'ctx' is full. + */ +static int ntfs_read_hdr(struct ntfs_sb_info *sbi, struct ntfs_inode *ni, + const struct INDEX_HDR *hdr, u64 vbo, u64 pos, + u8 *name, struct dir_context *ctx) +{ + const struct NTFS_DE *e; + u32 e_size; + u32 end = le32_to_cpu(hdr->used); + u32 off = le32_to_cpu(hdr->de_off); + + for (;; off += e_size) { + if (off + sizeof(struct NTFS_DE) > end) + return -EINVAL; + + e = Add2Ptr(hdr, off); + e_size = le16_to_cpu(e->size); + if (e_size < sizeof(struct NTFS_DE) || off + e_size > end) + return -EINVAL; + + if (de_is_last(e)) + return 0; + + /* Skip already enumerated. */ + if (vbo + off < pos) + continue; + + if (le16_to_cpu(e->key_size) < SIZEOF_ATTRIBUTE_FILENAME) + return -EINVAL; + + ctx->pos = vbo + off; + + /* Submit the name to the filldir callback. */ + if (!ntfs_dir_emit(sbi, ni, e, name, ctx)) { + /* ctx is full. */ + return +1; + } + } +} + +/* + * ntfs_readdir - file_operations::iterate_shared + * + * Use non sorted enumeration. + * Sorted enumeration may result infinite loop if names tree contains loop. + */ +static int ntfs_readdir(struct file *file, struct dir_context *ctx) +{ + const struct INDEX_ROOT *root; + size_t bit; + int err = 0; + struct inode *dir = file_inode(file); + struct ntfs_inode *ni = ntfs_i(dir); + struct super_block *sb = dir->i_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + loff_t i_size = i_size_read(dir); + u64 pos = ctx->pos; + u8 *name = NULL; + struct indx_node *node = NULL; + u8 index_bits = ni->dir.index_bits; + size_t max_bit = i_size >> ni->dir.index_bits; + loff_t eod = i_size + sbi->record_size; + + /* Name is a buffer of PATH_MAX length. */ + static_assert(NTFS_NAME_LEN * 4 < PATH_MAX); + + if (!pos) { + /* + * ni->dir.version increments each directory change. + * Save the initial value of ni->dir.version. + */ + file->private_data = (void *)ni->dir.version; + } + + if (pos >= eod) { + if (file->private_data == (void *)ni->dir.version) { + /* No changes since first readdir. */ + return 0; + } + + /* + * Handle directories that changed after the initial readdir(). + * + * Some user space code implements recursive removal like this instead + * of calling rmdir(2) directly: + * + * fd = opendir(path); + * while ((dent = readdir(fd))) + * unlinkat(dirfd(fd), dent->d_name, 0); + * closedir(fd); + * + * POSIX leaves unspecified what readdir() should return once the + * directory has been modified after opendir()/rewinddir(), so this + * pattern is not guaranteed to work on all filesystems or platforms. + * + * In ntfs3 the internal name tree may be reshaped while entries are + * being removed, so there is no stable anchor for continuing a + * single-pass walk based on the original readdir() order. + * + * In practice some widely used tools (for example certain rm(1) + * implementations) have used this readdir()/unlink() loop, and some + * filesystems behave in a way that effectively makes it work in the + * common case. + * + * The code below follows that practice and tries to provide + * "rmdir-like" behaviour for such callers on ntfs3, even though the + * situation is not strictly defined by the APIs. + * + * Apple documents the same readdir()/unlink() issue and a workaround + * for HFS file systems in: + * https://web.archive.org/web/20220122122948/https:/support.apple.com/kb/TA21420?locale=en_US + */ + ctx->pos = pos = 3; + file->private_data = (void *)ni->dir.version; + } + + if (!dir_emit_dots(file, ctx)) + return 0; + + name = kmalloc(PATH_MAX, GFP_KERNEL); + if (!name) + return -ENOMEM; + + if (!ni->mi_loaded && ni->attr_list.size) { + /* + * Directory inode is locked for read. + * Load all subrecords to avoid 'write' access to 'ni' during + * directory reading. + */ + ni_lock(ni); + if (!ni->mi_loaded && ni->attr_list.size) { + err = ni_load_all_mi(ni); + if (!err) + ni->mi_loaded = true; + } + ni_unlock(ni); + if (err) + goto out; + } + + /* + * Keep directory metadata stable for the whole walk. Loading subrecords + * once is not enough if concurrent writeback can still compact ATTR_LIST + * entries and free the record that ntfs_read_hdr() is currently walking. + */ + ni_lock(ni); + + root = indx_get_root(&ni->dir, ni, NULL, NULL); + if (!root) { + err = -EINVAL; + goto out_unlock; + } + + if (pos >= sbi->record_size) { + bit = (pos - sbi->record_size) >> index_bits; + } else { + /* + * Add each name from root in 'ctx'. + */ + err = ntfs_read_hdr(sbi, ni, &root->ihdr, 0, pos, name, ctx); + if (err) + goto out_unlock; + bit = 0; + } + + /* + * Enumerate indexes until the end of dir. + */ + for (; bit < max_bit; bit += 1) { + /* Get the next used index. */ + err = indx_used_bit(&ni->dir, ni, &bit); + if (err) + goto out_unlock; + + if (bit == MINUS_ONE_T) { + /* no more used indexes. end of dir. */ + break; + } + + if (bit >= max_bit) { + /* Corrupted directory. */ + err = -EINVAL; + goto out_unlock; + } + + err = indx_read_ra(&ni->dir, ni, bit << ni->dir.idx2vbn_bits, + &node, &file->f_ra); + if (err) + goto out_unlock; + + /* + * Add each name from index in 'ctx'. + */ + err = ntfs_read_hdr(sbi, ni, &node->index->ihdr, + ((u64)bit << index_bits) + sbi->record_size, + pos, name, ctx); + if (err) + goto out_unlock; + } + +out_unlock: + ni_unlock(ni); + +out: + kfree(name); + put_indx_node(node); + + if (!err) { + /* End of directory. */ + ctx->pos = eod; + } else if (err == 1) { + /* 'ctx' is full. */ + err = 0; + } else if (err == -ENOENT) { + err = 0; + ctx->pos = pos; + } else if (err < 0) { + if (err == -EINVAL) + _ntfs_bad_inode(dir); + ctx->pos = eod; + } + + return err; +} + +/* + * Return fname when @e passes the same checks as ntfs_dir_emit() before + * exposing an entry (valid key, non-DOS, fname fits in e->size). + */ +static inline const struct ATTR_FILE_NAME * +de_countable_fname(const struct NTFS_DE *e, u32 e_size) +{ + const struct ATTR_FILE_NAME *fname; + + fname = de_get_fname(e); + if (!fname || fname->type == FILE_NAME_DOS || + !de_fname_fits(e, e_size, fname)) + return NULL; + + return fname; +} + +static int ntfs_dir_count(struct inode *dir, bool *is_empty, size_t *dirs, + size_t *files) +{ + int err = 0; + struct ntfs_inode *ni = ntfs_i(dir); + struct NTFS_DE *e = NULL; + struct INDEX_ROOT *root; + struct INDEX_HDR *hdr; + const struct ATTR_FILE_NAME *fname; + u32 e_size, off, end; + size_t drs = 0, fles = 0, bit = 0; + struct indx_node *node = NULL; + size_t max_indx = i_size_read(&ni->vfs_inode) >> ni->dir.index_bits; + + if (is_empty) + *is_empty = true; + + root = indx_get_root(&ni->dir, ni, NULL, NULL); + if (!root) + return -EINVAL; + + hdr = &root->ihdr; + + for (;;) { + end = le32_to_cpu(hdr->used); + off = le32_to_cpu(hdr->de_off); + + for (; off + sizeof(struct NTFS_DE) <= end; off += e_size) { + e = Add2Ptr(hdr, off); + e_size = le16_to_cpu(e->size); + if (e_size < sizeof(struct NTFS_DE) || + off + e_size > end) { + /* Looks like corruption. */ + break; + } + + if (de_is_last(e)) + break; + + fname = de_countable_fname(e, e_size); + if (!fname) + continue; + + if (is_empty) { + *is_empty = false; + if (!dirs && !files) + goto out; + } + + if (fname->dup.fa & FILE_ATTRIBUTE_DIRECTORY) + drs += 1; + else + fles += 1; + } + + if (bit >= max_indx) + goto out; + + err = indx_used_bit(&ni->dir, ni, &bit); + if (err) + goto out; + + if (bit == MINUS_ONE_T) + goto out; + + if (bit >= max_indx) + goto out; + + err = indx_read(&ni->dir, ni, bit << ni->dir.idx2vbn_bits, + &node); + if (err) + goto out; + + hdr = &node->index->ihdr; + bit += 1; + } + +out: + put_indx_node(node); + if (dirs) + *dirs = drs; + if (files) + *files = fles; + + return err; +} + +bool dir_is_empty(struct inode *dir) +{ + bool is_empty = false; + + ntfs_dir_count(dir, &is_empty, NULL, NULL); + + return is_empty; +} + +// clang-format off +const struct file_operations ntfs_dir_operations = { + .llseek = generic_file_llseek, + .read = generic_read_dir, + .iterate_shared = ntfs_readdir, + .fsync = ntfs_file_fsync, + .open = ntfs_file_open, + .unlocked_ioctl = ntfs_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = ntfs_compat_ioctl, +#endif + .setlease = generic_setlease, +}; +// clang-format on diff --git a/fs/ntfs3/file.c b/fs/ntfs3/file.c new file mode 100644 index 000000000..2abf334bf --- /dev/null +++ b/fs/ntfs3/file.c @@ -0,0 +1,1603 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + * Regular file handling primitives for NTFS-based filesystems. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +/* + * cifx, btrfs, exfat, ext4, f2fs use this constant. + * Hope this value will become common to all fs. + */ +#define NTFS3_IOC_SHUTDOWN _IOR('X', 125, __u32) + +/* + * Helper for ntfs_should_use_dio. + */ +static u32 ntfs_dio_alignment(struct inode *inode) +{ + struct ntfs_inode *ni = ntfs_i(inode); + + if (is_resident(ni)) { + /* Check delalloc. */ + if (!ni->file.run_da.count) + return 0; + } + + /* In most cases this is bdev_logical_block_size(bdev). */ + return ni->mi.sbi->bdev_blocksize; +} + +/* + * Returns %true if the given DIO request should be attempted with DIO, or + * %false if it should fall back to buffered I/O. + */ +static bool ntfs_should_use_dio(struct kiocb *iocb, struct iov_iter *iter) +{ + struct inode *inode = file_inode(iocb->ki_filp); + u32 dio_align = ntfs_dio_alignment(inode); + + if (!dio_align) + return false; + + return IS_ALIGNED(iocb->ki_pos | iov_iter_alignment(iter), dio_align); +} + +static int ntfs_ioctl_fitrim(struct ntfs_sb_info *sbi, unsigned long arg) +{ + struct fstrim_range __user *user_range; + struct fstrim_range range; + struct block_device *dev; + int err; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + dev = sbi->sb->s_bdev; + if (!bdev_max_discard_sectors(dev)) + return -EOPNOTSUPP; + + user_range = (struct fstrim_range __user *)arg; + if (copy_from_user(&range, user_range, sizeof(range))) + return -EFAULT; + + range.minlen = max_t(u32, range.minlen, bdev_discard_granularity(dev)); + + err = ntfs_trim_fs(sbi, &range); + if (err < 0) + return err; + + if (copy_to_user(user_range, &range, sizeof(range))) + return -EFAULT; + + return 0; +} + +/* + * ntfs_fileattr_get - inode_operations::fileattr_get + */ +int ntfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + struct ntfs_sb_info *sbi = inode->i_sb->s_fs_info; + struct ntfs_inode *ni = ntfs_i(inode); + u32 flags = 0; + + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(ni))) + return -EINVAL; + + if (sbi->options->nocase) + flags |= FS_CASEFOLD_FL; + + if (inode->i_flags & S_IMMUTABLE) + flags |= FS_IMMUTABLE_FL; + + if (inode->i_flags & S_APPEND) + flags |= FS_APPEND_FL; + + if (is_compressed(ni)) + flags |= FS_COMPR_FL; + + if (is_encrypted(ni)) + flags |= FS_ENCRYPT_FL; + + if (ni->nodump) + flags |= FS_NODUMP_FL; + + fileattr_fill_flags(fa, flags); + + return 0; +} + +/* + * ntfs_fileattr_set - inode_operations::fileattr_set + */ +int ntfs_fileattr_set(struct mnt_idmap *idmap, struct dentry *dentry, + struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + struct ntfs_inode *ni = ntfs_i(inode); + u32 flags = fa->flags; + unsigned int new_fl = 0; + + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(ni))) + return -EINVAL; + + if (fileattr_has_fsx(fa)) + return -EOPNOTSUPP; + + if (flags & ~(FS_IMMUTABLE_FL | FS_APPEND_FL | FS_NODUMP_FL)) + return -EOPNOTSUPP; + + if (flags & FS_IMMUTABLE_FL) + new_fl |= S_IMMUTABLE; + + if (flags & FS_APPEND_FL) + new_fl |= S_APPEND; + + inode_set_flags(inode, new_fl, S_IMMUTABLE | S_APPEND); + + /* Save nodump flag to return in ntfs_getattr. */ + if (flags & FS_NODUMP_FL) + ni->nodump = 1; + else + ni->nodump = 0; + + inode_set_ctime_current(inode); + mark_inode_dirty(inode); + + return 0; +} + +static int ntfs_ioctl_get_volume_label(struct ntfs_sb_info *sbi, u8 __user *buf) +{ + if (copy_to_user(buf, sbi->volume.label, FSLABEL_MAX)) + return -EFAULT; + + return 0; +} + +static int ntfs_ioctl_set_volume_label(struct ntfs_sb_info *sbi, u8 __user *buf) +{ + u8 user[FSLABEL_MAX] = { 0 }; + int len; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (copy_from_user(user, buf, FSLABEL_MAX)) + return -EFAULT; + + len = strnlen(user, FSLABEL_MAX); + + return ntfs_set_label(sbi, user, len); +} + +/* + * ntfs_force_shutdown - helper function. Called from ioctl + */ +static int ntfs_force_shutdown(struct super_block *sb, u32 flags) +{ + int err; + struct ntfs_sb_info *sbi = sb->s_fs_info; + + if (unlikely(ntfs3_forced_shutdown(sb))) + return 0; + + /* No additional options yet (flags). */ + err = bdev_freeze(sb->s_bdev); + if (err) + return err; + set_bit(NTFS_FLAGS_SHUTDOWN_BIT, &sbi->flags); + bdev_thaw(sb->s_bdev); + return 0; +} + +static int ntfs_ioctl_shutdown(struct super_block *sb, unsigned long arg) +{ + u32 flags; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (get_user(flags, (__u32 __user *)arg)) + return -EFAULT; + + return ntfs_force_shutdown(sb, flags); +} + +/* + * ntfs_ioctl - file_operations::unlocked_ioctl + */ +long ntfs_ioctl(struct file *filp, u32 cmd, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + struct super_block *sb = inode->i_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(ntfs_i(inode)))) + return -EINVAL; + + switch (cmd) { + case FITRIM: + return ntfs_ioctl_fitrim(sbi, arg); + case FS_IOC_GETFSLABEL: + return ntfs_ioctl_get_volume_label(sbi, (u8 __user *)arg); + case FS_IOC_SETFSLABEL: + return ntfs_ioctl_set_volume_label(sbi, (u8 __user *)arg); + case NTFS3_IOC_SHUTDOWN: + return ntfs_ioctl_shutdown(sb, arg); + } + return -ENOTTY; /* Inappropriate ioctl for device. */ +} + +#ifdef CONFIG_COMPAT +long ntfs_compat_ioctl(struct file *filp, u32 cmd, unsigned long arg) + +{ + return ntfs_ioctl(filp, cmd, (unsigned long)compat_ptr(arg)); +} +#endif + +/* + * ntfs_getattr - inode_operations::getattr + */ +int ntfs_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, u32 flags) +{ + struct inode *inode = d_inode(path->dentry); + struct ntfs_inode *ni = ntfs_i(inode); + + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(ni))) + return -EINVAL; + + stat->result_mask |= STATX_BTIME; + stat->btime = ni->i_crtime; + stat->blksize = ni->mi.sbi->cluster_size; /* 512, 1K, ..., 2M */ + + if (inode->i_flags & S_IMMUTABLE) + stat->attributes |= STATX_ATTR_IMMUTABLE; + + if (inode->i_flags & S_APPEND) + stat->attributes |= STATX_ATTR_APPEND; + + if (ni->nodump) + stat->attributes |= STATX_ATTR_NODUMP; + + if (is_compressed(ni)) + stat->attributes |= STATX_ATTR_COMPRESSED; + + if (is_encrypted(ni)) + stat->attributes |= STATX_ATTR_ENCRYPTED; + + stat->attributes_mask |= STATX_ATTR_COMPRESSED | STATX_ATTR_ENCRYPTED | + STATX_ATTR_IMMUTABLE | STATX_ATTR_APPEND; + + generic_fillattr(idmap, request_mask, inode, stat); + + return 0; +} + +static int ntfs_extend_initialized_size(struct file *file, + struct ntfs_inode *ni, + const loff_t new_valid) +{ + struct inode *inode = &ni->vfs_inode; + const loff_t valid = ni->i_valid; + int err; + + if (valid >= new_valid) + return 0; + + if (is_resident(ni)) { + ni->i_valid = new_valid; + return 0; + } + + err = iomap_zero_range(inode, valid, new_valid - valid, NULL, + &ntfs_iomap_ops, &ntfs_iomap_folio_ops, NULL); + if (err) { + ni->i_valid = valid; + ntfs_inode_warn(inode, + "failed to extend initialized size to %llx.", + new_valid); + return err; + } + + return 0; +} + +/* Zero pagecache after 'from'. */ +static void ntfs_zero_tail(struct address_space *mapping, loff_t from) +{ + struct folio_batch fbatch; + pgoff_t index = from >> PAGE_SHIFT; + unsigned nr, i; + + folio_batch_init(&fbatch); + + nr = filemap_get_folios(mapping, &index, -1, &fbatch); + + for (i = 0; i < nr; i++) { + struct folio *folio = fbatch.folios[i]; + u32 st = folio_pos(folio) < from ? + offset_in_folio(folio, from) : + 0; + + folio_lock(folio); + folio_zero_segment(folio, st, folio_size(folio)); + + folio_unlock(folio); + } + folio_batch_release(&fbatch); +} + +static void ntfs_filemap_close(struct vm_area_struct *vma) +{ + struct inode *inode = file_inode(vma->vm_file); + struct ntfs_inode *ni = ntfs_i(inode); + u64 i_size = i_size_read(inode); + u64 from = (u64)vma->vm_pgoff << PAGE_SHIFT; + u64 to = min(i_size, from + vma->vm_end - vma->vm_start); + + if (ni->i_valid < to) { + ni->i_valid = to; + mark_inode_dirty(inode); + } + ntfs_zero_tail(inode->i_mapping, ni->i_valid); +} + +/* Copy of generic_file_vm_ops. */ +static const struct vm_operations_struct ntfs_file_vm_ops = { + .close = ntfs_filemap_close, + .fault = filemap_fault, + .map_pages = filemap_map_pages, + .page_mkwrite = filemap_page_mkwrite, +}; + +/* + * ntfs_file_mmap_prepare - file_operations::mmap_prepare + */ +static int ntfs_file_mmap_prepare(struct vm_area_desc *desc) +{ + struct file *file = desc->file; + struct inode *inode = file_inode(file); + struct ntfs_inode *ni = ntfs_i(inode); + const bool rw = vma_desc_test(desc, VMA_WRITE_BIT); + int err; + + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(ni))) + return -EINVAL; + + if (unlikely(ntfs3_forced_shutdown(inode->i_sb))) + return -EIO; + + if (is_encrypted(ni)) { + ntfs_inode_warn(inode, "mmap encrypted not supported"); + return -EOPNOTSUPP; + } + + if (is_dedup(ni)) { + ntfs_inode_warn(inode, "mmap deduplicated not supported"); + return -EOPNOTSUPP; + } + + if (is_compressed(ni)) { + if (rw) { + ntfs_inode_warn(inode, + "mmap(write) compressed not supported"); + return -EOPNOTSUPP; + } + /* Turn off readahead for compressed files. */ + file->f_ra.ra_pages = 0; + } + + if (rw) { + u64 from = (u64)desc->pgoff << PAGE_SHIFT; + u64 to = min_t(u64, i_size_read(inode), + from + vma_desc_size(desc)); + + if (is_sparsed(ni)) { + /* Allocate clusters for rw map. */ + struct ntfs_sb_info *sbi = inode->i_sb->s_fs_info; + CLST lcn, len; + CLST vcn = from >> sbi->cluster_bits; + CLST end = bytes_to_cluster(sbi, to); + bool new; + + for (; vcn < end; vcn += len) { + err = attr_data_get_block(ni, vcn, 1, &lcn, + &len, &new, true, + NULL, false); + if (err) + goto out; + } + } + + if (ni->i_valid < to) { + if (!inode_trylock(inode)) { + err = -EAGAIN; + goto out; + } + err = ntfs_extend_initialized_size(file, ni, to); + inode_unlock(inode); + if (err) + goto out; + } + } + + err = generic_file_mmap_prepare(desc); + if (!err && rw) + desc->vm_ops = &ntfs_file_vm_ops; +out: + return err; +} + +/* + * ntfs_fallocate - file_operations::ntfs_fallocate + * + * Preallocate space for a file. This implements ntfs's fallocate file + * operation, which gets called from sys_fallocate system call. User + * space requests 'len' bytes at 'vbo'. If FALLOC_FL_KEEP_SIZE is set + * we just allocate clusters without zeroing them out. Otherwise we + * allocate and zero out clusters via an expanding truncate. + */ +static long ntfs_fallocate(struct file *file, int mode, loff_t vbo, loff_t len) +{ + struct inode *inode = file_inode(file); + struct address_space *mapping = inode->i_mapping; + struct super_block *sb = inode->i_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + struct ntfs_inode *ni = ntfs_i(inode); + loff_t end = vbo + len; + loff_t vbo_down = round_down(vbo, max_t(unsigned long, + sbi->cluster_size, PAGE_SIZE)); + bool is_supported_holes = is_sparsed(ni) || is_compressed(ni); + loff_t i_size, new_size; + bool map_locked; + int err; + + /* No support for dir. */ + if (!S_ISREG(inode->i_mode)) + return -EOPNOTSUPP; + + /* + * vfs_fallocate checks all possible combinations of mode. + * Do additional checks here before ntfs_set_state(dirty). + */ + if (mode & FALLOC_FL_PUNCH_HOLE) { + if (!is_supported_holes) + return -EOPNOTSUPP; + } else if (mode & FALLOC_FL_COLLAPSE_RANGE) { + } else if (mode & FALLOC_FL_INSERT_RANGE) { + if (!is_supported_holes) + return -EOPNOTSUPP; + } else if (mode & + ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE | + FALLOC_FL_COLLAPSE_RANGE | FALLOC_FL_INSERT_RANGE)) { + ntfs_inode_warn(inode, "fallocate(0x%x) is not supported", + mode); + return -EOPNOTSUPP; + } + + ntfs_set_state(sbi, NTFS_DIRTY_DIRTY); + + inode_lock(inode); + i_size = inode->i_size; + new_size = max(end, i_size); + map_locked = false; + + if (WARN_ON(ni->ni_flags & NI_FLAG_COMPRESSED_MASK)) { + /* Should never be here, see ntfs_file_open. */ + err = -EOPNOTSUPP; + goto out; + } + + if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_COLLAPSE_RANGE | + FALLOC_FL_INSERT_RANGE)) { + inode_dio_wait(inode); + filemap_invalidate_lock(mapping); + map_locked = true; + } + + if (mode & FALLOC_FL_PUNCH_HOLE) { + u32 frame_size; + loff_t mask, vbo_a, end_a, tmp, from; + + err = filemap_write_and_wait_range(mapping, vbo_down, + LLONG_MAX); + if (err) + goto out; + + truncate_pagecache(inode, vbo_down); + + ni_lock(ni); + err = attr_punch_hole(ni, vbo, len, &frame_size); + ni_unlock(ni); + if (!err) + goto ok; + + if (err != E_NTFS_NOTALIGNED) + goto out; + + /* Process not aligned punch. */ + err = 0; + if (end > i_size) + end = i_size; + mask = frame_size - 1; + vbo_a = (vbo + mask) & ~mask; + end_a = end & ~mask; + + tmp = min(vbo_a, end); + from = min_t(loff_t, ni->i_valid, vbo); + /* Zero head of punch. */ + if (tmp > from) { + err = iomap_zero_range(inode, from, tmp - from, NULL, + &ntfs_iomap_ops, + &ntfs_iomap_folio_ops, NULL); + if (err) + goto out; + } + + /* Aligned punch_hole. Deallocate clusters. */ + if (end_a > vbo_a) { + ni_lock(ni); + err = attr_punch_hole(ni, vbo_a, end_a - vbo_a, NULL); + ni_unlock(ni); + if (err) + goto out; + } + + /* Zero tail of punch. */ + if (vbo < end_a && end_a < end) { + err = iomap_zero_range(inode, end_a, end - end_a, NULL, + &ntfs_iomap_ops, + &ntfs_iomap_folio_ops, NULL); + if (err) + goto out; + } + } else if (mode & FALLOC_FL_COLLAPSE_RANGE) { + /* + * Write tail of the last page before removed range since + * it will get removed from the page cache below. + */ + err = filemap_write_and_wait_range(mapping, vbo_down, vbo); + if (err) + goto out; + + /* + * Write data that will be shifted to preserve them + * when discarding page cache below. + */ + err = filemap_write_and_wait_range(mapping, end, LLONG_MAX); + if (err) + goto out; + + truncate_pagecache(inode, vbo_down); + + ni_lock(ni); + err = attr_collapse_range(ni, vbo, len); + ni_unlock(ni); + if (err) + goto out; + } else if (mode & FALLOC_FL_INSERT_RANGE) { + /* Check new size. */ + err = inode_newsize_ok(inode, new_size); + if (err) + goto out; + + /* Write out all dirty pages. */ + err = filemap_write_and_wait_range(mapping, vbo_down, + LLONG_MAX); + if (err) + goto out; + truncate_pagecache(inode, vbo_down); + + ni_lock(ni); + err = attr_insert_range(ni, vbo, len); + ni_unlock(ni); + if (err) + goto out; + } else { + /* Check new size. */ + u8 cluster_bits = sbi->cluster_bits; + + /* Be sure file is non resident. */ + if (is_resident(ni)) { + ni_lock(ni); + err = attr_force_nonresident(ni); + ni_unlock(ni); + if (err) + goto out; + } + + /* generic/213: expected -ENOSPC instead of -EFBIG. */ + if (!is_supported_holes) { + loff_t to_alloc = new_size - inode_get_bytes(inode); + + if (to_alloc > 0 && + (to_alloc >> cluster_bits) > + wnd_zeroes(&sbi->used.bitmap)) { + err = -ENOSPC; + goto out; + } + } + + err = inode_newsize_ok(inode, new_size); + if (err) + goto out; + + if (new_size > i_size) { + /* + * Allocate clusters, do not change 'valid' size. + */ + err = ntfs_set_size(inode, new_size); + if (err) + goto out; + } + + if (is_supported_holes) { + CLST vcn = vbo >> cluster_bits; + CLST cend = bytes_to_cluster(sbi, end); + CLST lcn, clen; + bool new; + + /* + * Allocate and zero new clusters. + */ + for (; vcn < cend; vcn += clen) { + err = attr_data_get_block(ni, vcn, cend - vcn, + &lcn, &clen, &new, + true, NULL, false); + if (err) + goto out; + } + } + + if (mode & FALLOC_FL_KEEP_SIZE) { + err = ntfs_set_size(inode, i_size); + if (err) + goto out; + } + } + +ok: + err = file_modified(file); + if (err) + goto out; + +out: + if (map_locked) + filemap_invalidate_unlock(mapping); + + if (!err) { + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + mark_inode_dirty(inode); + } + + inode_unlock(inode); + return err; +} + +/* + * ntfs_setattr - inode_operations::setattr + */ +int ntfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +{ + struct inode *inode = d_inode(dentry); + struct ntfs_inode *ni = ntfs_i(inode); + u32 ia_valid = attr->ia_valid; + umode_t mode = inode->i_mode; + int err; + + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(ni))) + return -EINVAL; + + if (unlikely(ntfs3_forced_shutdown(inode->i_sb))) + return -EIO; + + err = setattr_prepare(idmap, dentry, attr); + if (err) + goto out; + + if (ia_valid & ATTR_SIZE) { + loff_t newsize, oldsize; + + if (WARN_ON(ni->ni_flags & NI_FLAG_COMPRESSED_MASK)) { + /* Should never be here, see ntfs_file_open(). */ + err = -EOPNOTSUPP; + goto out; + } + inode_dio_wait(inode); + oldsize = i_size_read(inode); + newsize = attr->ia_size; + + if (newsize != oldsize) { + truncate_setsize(inode, newsize); + + err = ntfs_set_size(inode, newsize); + if (err) { + i_size_write(inode, oldsize); + goto out; + } + + ni->std_fa |= FILE_ATTRIBUTE_ARCHIVE; + ni->ni_flags |= NI_FLAG_UPDATE_PARENT; + inode_set_mtime_to_ts(inode, + inode_set_ctime_current(inode)); + } + } + + setattr_copy(idmap, inode, attr); + + if (!is_ni_base(ni)) { + ia_valid &= ~ATTR_SIZE; + } else if (mode != inode->i_mode) { + err = ntfs_acl_chmod(idmap, dentry); + if (err) + goto out; + + /* Linux 'w' -> Windows 'ro'. */ + if (0222 & inode->i_mode) + ni->std_fa &= ~FILE_ATTRIBUTE_READONLY; + else + ni->std_fa |= FILE_ATTRIBUTE_READONLY; + } + + if (ia_valid & (ATTR_UID | ATTR_GID | ATTR_MODE)) + ntfs_save_wsl_perm(inode, NULL); + mark_inode_dirty(inode); +out: + return err; +} + +/* + * check_read_restriction: + * common code for ntfs_file_read_iter and ntfs_file_splice_read + */ +static int check_read_restriction(struct inode *inode) +{ + struct ntfs_inode *ni = ntfs_i(inode); + + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(ni))) + return -EINVAL; + + if (unlikely(ntfs3_forced_shutdown(inode->i_sb))) + return -EIO; + + if (is_encrypted(ni)) { + ntfs_inode_warn(inode, "encrypted i/o not supported"); + return -EOPNOTSUPP; + } + +#ifndef CONFIG_NTFS3_LZX_XPRESS + if (ni->ni_flags & NI_FLAG_COMPRESSED_MASK) { + ntfs_inode_warn( + inode, + "activate CONFIG_NTFS3_LZX_XPRESS to read external compressed files"); + return -EOPNOTSUPP; + } +#endif + + if (is_dedup(ni)) { + ntfs_inode_warn(inode, "read deduplicated not supported"); + return -EOPNOTSUPP; + } + + return 0; +} + +/* + * ntfs_file_read_iter - file_operations::read_iter + */ +static ssize_t ntfs_file_read_iter(struct kiocb *iocb, struct iov_iter *iter) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct ntfs_inode *ni = ntfs_i(inode); + size_t bytes = iov_iter_count(iter); + loff_t valid, i_size, vbo, end; + unsigned int dio_flags; + ssize_t ret; + + ret = check_read_restriction(inode); + if (ret) + return ret; + + if (!bytes) + return 0; /* skip atime */ + + if (ni->file.ads.len == ARRAY_SIZE(QUERY_STREAMS) && + !memcmp(ni->file.ads.name, QUERY_STREAMS, sizeof(QUERY_STREAMS))) { + /* Query ADS. */ + if (unlikely(iocb->ki_flags & IOCB_DIRECT)) { + ntfs_inode_warn( + inode, + "direct I/O for streams is not supported"); + return -EOPNOTSUPP; + } + + inode_lock_shared(inode); + ret = ni_query_ads(ni, &iocb->ki_pos, iter); + inode_unlock_shared(inode); + return ret; + } + + if (is_compressed(ni)) { + if (iocb->ki_flags & IOCB_DIRECT) { + ntfs_inode_warn( + inode, "direct i/o + compressed not supported"); + return -EOPNOTSUPP; + } + /* Turn off readahead for compressed files. */ + file->f_ra.ra_pages = 0; + } + + /* Fallback to buffered I/O if the inode does not support direct I/O. */ + if (!(iocb->ki_flags & IOCB_DIRECT) || + !ntfs_should_use_dio(iocb, iter)) { + iocb->ki_flags &= ~IOCB_DIRECT; + return generic_file_read_iter(iocb, iter); + } + + if (iocb->ki_flags & IOCB_NOWAIT) { + if (!inode_trylock_shared(inode)) + return -EAGAIN; + } else { + inode_lock_shared(inode); + } + + vbo = iocb->ki_pos; + end = vbo + bytes; + dio_flags = 0; + valid = ni->i_valid; + i_size = inode->i_size; + + if (vbo < valid) { + if (valid < end) { + /* read cross 'valid' size. */ + dio_flags |= IOMAP_DIO_FORCE_WAIT; + } + + if (ni->file.run_da.count) { + /* Direct I/O is not compatible with delalloc. */ + ret = ni_allocate_da_blocks(ni); + if (ret) + goto out; + } + + ret = iomap_dio_rw(iocb, iter, &ntfs_iomap_ops, NULL, dio_flags, + NULL, 0); + + if (ret <= 0) + goto out; + end = vbo + ret; + if (valid < end) { + size_t to_zero = end - valid; + /* Fix iter. */ + iov_iter_revert(iter, to_zero); + iov_iter_zero(to_zero, iter); + } + } else if (vbo < i_size) { + if (end > i_size) + bytes = i_size - vbo; + iov_iter_zero(bytes, iter); + iocb->ki_pos += bytes; + ret = bytes; + } + +out: + inode_unlock_shared(inode); + file_accessed(file); + return ret; +} + +/* + * ntfs_file_splice_read - file_operations::splice_read + */ +static ssize_t ntfs_file_splice_read(struct file *file, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, + unsigned int flags) +{ + struct inode *inode = file_inode(file); + struct ntfs_inode *ni = ntfs_i(inode); + ssize_t ret; + + ret = check_read_restriction(inode); + if (ret) + return ret; + + if (is_compressed(ni)) { + /* Turn off readahead for compressed files. */ + file->f_ra.ra_pages = 0; + } + + return filemap_splice_read(file, ppos, pipe, len, flags); +} + +/* + * ntfs_get_frame_pages + * + * Return: Array of locked pages. + */ +static int ntfs_get_frame_pages(struct address_space *mapping, pgoff_t index, + struct page **pages, u32 pages_per_frame, + bool *frame_uptodate) +{ + gfp_t gfp_mask = mapping_gfp_mask(mapping); + u32 npages; + + *frame_uptodate = true; + + for (npages = 0; npages < pages_per_frame; npages++, index++) { + struct folio *folio; + + folio = __filemap_get_folio(mapping, index, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, + gfp_mask | __GFP_ZERO); + if (IS_ERR(folio)) { + while (npages--) { + folio = page_folio(pages[npages]); + folio_unlock(folio); + folio_put(folio); + } + + return -ENOMEM; + } + + if (!folio_test_uptodate(folio)) + *frame_uptodate = false; + + pages[npages] = &folio->page; + } + + return 0; +} + +/* + * ntfs_compress_write - Helper for ntfs_file_write_iter() (compressed files). + */ +static ssize_t ntfs_compress_write(struct kiocb *iocb, struct iov_iter *from) +{ + int err; + struct file *file = iocb->ki_filp; + size_t count = iov_iter_count(from); + loff_t pos = iocb->ki_pos; + struct inode *inode = file_inode(file); + loff_t i_size = i_size_read(inode); + struct address_space *mapping = inode->i_mapping; + struct ntfs_inode *ni = ntfs_i(inode); + u64 valid = ni->i_valid; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct page **pages = NULL; + struct folio *folio; + size_t written = 0; + u8 frame_bits = NTFS_LZNT_CUNIT + sbi->cluster_bits; + u32 frame_size = 1u << frame_bits; + u32 pages_per_frame = frame_size >> PAGE_SHIFT; + u32 ip, off; + CLST frame; + u64 frame_vbo; + pgoff_t index; + bool frame_uptodate; + + if (frame_size < PAGE_SIZE) { + /* + * frame_size == 8K if cluster 512 + * frame_size == 64K if cluster 4096 + */ + ntfs_inode_warn(inode, "page size is bigger than frame size"); + return -EOPNOTSUPP; + } + + pages = kmalloc_objs(struct page *, pages_per_frame, GFP_NOFS); + if (!pages) + return -ENOMEM; + + err = file_remove_privs(file); + if (err) + goto out; + + err = file_update_time(file); + if (err) + goto out; + + /* Zero range [valid : pos). */ + while (valid < pos) { + CLST lcn, clen; + + frame = valid >> frame_bits; + frame_vbo = valid & ~(u64)(frame_size - 1); + off = valid & (frame_size - 1); + + err = attr_data_get_block(ni, frame << NTFS_LZNT_CUNIT, 1, &lcn, + &clen, NULL, false, NULL, false); + if (err) + goto out; + + if (lcn == SPARSE_LCN) { + ni->i_valid = valid = + frame_vbo + ((u64)clen << sbi->cluster_bits); + continue; + } + + /* Load full frame. */ + err = ntfs_get_frame_pages(mapping, frame_vbo >> PAGE_SHIFT, + pages, pages_per_frame, + &frame_uptodate); + if (err) + goto out; + + if (!frame_uptodate && off) { + err = ni_read_frame(ni, frame_vbo, pages, + pages_per_frame, 0); + if (err) { + for (ip = 0; ip < pages_per_frame; ip++) { + folio = page_folio(pages[ip]); + folio_unlock(folio); + folio_put(folio); + } + goto out; + } + } + + ip = off >> PAGE_SHIFT; + off = offset_in_page(valid); + for (; ip < pages_per_frame; ip++, off = 0) { + folio = page_folio(pages[ip]); + folio_zero_segment(folio, off, PAGE_SIZE); + flush_dcache_folio(folio); + folio_mark_uptodate(folio); + } + + ni_lock(ni); + err = ni_write_frame(ni, pages, pages_per_frame); + ni_unlock(ni); + + for (ip = 0; ip < pages_per_frame; ip++) { + folio = page_folio(pages[ip]); + folio_mark_uptodate(folio); + folio_unlock(folio); + folio_put(folio); + } + + if (err) + goto out; + + ni->i_valid = valid = frame_vbo + frame_size; + } + + /* Copy user data [pos : pos + count). */ + while (count) { + size_t copied, bytes; + + off = pos & (frame_size - 1); + bytes = frame_size - off; + if (bytes > count) + bytes = count; + + frame_vbo = pos & ~(u64)(frame_size - 1); + index = frame_vbo >> PAGE_SHIFT; + + if (unlikely(fault_in_iov_iter_readable(from, bytes))) { + err = -EFAULT; + goto out; + } + + /* Load full frame. */ + err = ntfs_get_frame_pages(mapping, index, pages, + pages_per_frame, &frame_uptodate); + if (err) + goto out; + + if (!frame_uptodate) { + loff_t to = pos + bytes; + + if (off || (to < i_size && (to & (frame_size - 1)))) { + err = ni_read_frame(ni, frame_vbo, pages, + pages_per_frame, 0); + if (err) { + for (ip = 0; ip < pages_per_frame; + ip++) { + folio = page_folio(pages[ip]); + folio_unlock(folio); + folio_put(folio); + } + goto out; + } + } + } + + WARN_ON(!bytes); + copied = 0; + ip = off >> PAGE_SHIFT; + off = offset_in_page(pos); + + /* Copy user data to pages. */ + for (;;) { + size_t cp, tail = PAGE_SIZE - off; + + folio = page_folio(pages[ip]); + cp = copy_folio_from_iter_atomic( + folio, off, min(tail, bytes), from); + flush_dcache_folio(folio); + + copied += cp; + bytes -= cp; + if (!bytes || !cp) + break; + + if (cp < tail) { + off += cp; + } else { + ip++; + off = 0; + } + } + + ni_lock(ni); + err = ni_write_frame(ni, pages, pages_per_frame); + ni_unlock(ni); + + for (ip = 0; ip < pages_per_frame; ip++) { + folio = page_folio(pages[ip]); + folio_clear_dirty(folio); + folio_mark_uptodate(folio); + folio_unlock(folio); + folio_put(folio); + } + + if (err) + goto out; + + /* + * We can loop for a long time in here. Be nice and allow + * us to schedule out to avoid softlocking if preempt + * is disabled. + */ + cond_resched(); + + pos += copied; + written += copied; + + count = iov_iter_count(from); + } + +out: + kfree(pages); + + if (err < 0) + return err; + + iocb->ki_pos += written; + if (iocb->ki_pos > ni->i_valid) + ni->i_valid = iocb->ki_pos; + if (iocb->ki_pos > i_size) + i_size_write(inode, iocb->ki_pos); + + return written; +} + +/* + * check_write_restriction: + * common code for ntfs_file_write_iter and ntfs_file_splice_write + */ +static int check_write_restriction(struct inode *inode) +{ + struct ntfs_inode *ni = ntfs_i(inode); + + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(ni))) + return -EINVAL; + + if (unlikely(ntfs3_forced_shutdown(inode->i_sb))) + return -EIO; + + if (is_encrypted(ni)) { + ntfs_inode_warn(inode, "encrypted i/o not supported"); + return -EOPNOTSUPP; + } + + if (is_dedup(ni)) { + ntfs_inode_warn(inode, "write into deduplicated not supported"); + return -EOPNOTSUPP; + } + + if (unlikely(IS_IMMUTABLE(inode))) + return -EPERM; + + return 0; +} + +/* + * ntfs_file_write_iter - file_operations::write_iter + */ +static ssize_t ntfs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct ntfs_inode *ni = ntfs_i(inode); + loff_t vbo, endbyte; + ssize_t ret, err; + + if (!inode_trylock(inode)) { + if (iocb->ki_flags & IOCB_NOWAIT) + return -EAGAIN; + inode_lock(inode); + } + + ret = check_write_restriction(inode); + if (ret) + goto out; + + if (is_compressed(ni) && (iocb->ki_flags & IOCB_DIRECT)) { + ntfs_inode_warn(inode, "direct i/o + compressed not supported"); + ret = -EOPNOTSUPP; + goto out; + } + + ret = generic_write_checks(iocb, from); + if (ret <= 0) + goto out; + + err = file_modified(iocb->ki_filp); + if (err) { + ret = err; + goto out; + } + + if (WARN_ON(ni->ni_flags & NI_FLAG_COMPRESSED_MASK)) { + /* Should never be here, see ntfs_file_open(). */ + ret = -EOPNOTSUPP; + goto out; + } + + vbo = iocb->ki_pos; + endbyte = vbo + ret; + + if (endbyte > inode->i_size) { + /* + * Normal files: increase file size, allocate space. + * Sparse/Compressed: increase file size. No space allocated. + */ + ret = ntfs_set_size(inode, endbyte); + if (ret) + goto out; + } + + if (is_compressed(ni)) { + ret = ntfs_compress_write(iocb, from); + goto out; + } + + if (vbo > ni->i_valid) { + ret = ntfs_extend_initialized_size(file, ni, vbo); + if (ret) + goto out; + } + + /* Fallback to buffered I/O if the inode does not support direct I/O. */ + if (!(iocb->ki_flags & IOCB_DIRECT) || + !ntfs_should_use_dio(iocb, from)) { + iocb->ki_flags &= ~IOCB_DIRECT; + + ret = iomap_file_buffered_write(iocb, from, &ntfs_iomap_ops, + &ntfs_iomap_folio_ops, NULL); + inode_unlock(inode); + + if (likely(ret > 0)) + ret = generic_write_sync(iocb, ret); + + return ret; + } + + if (ni->file.run_da.count) { + /* Direct I/O is not compatible with delalloc. */ + ret = ni_allocate_da_blocks(ni); + if (ret) + goto out; + } + + ret = iomap_dio_rw(iocb, from, &ntfs_iomap_ops, NULL, + IOMAP_DIO_FORCE_WAIT, NULL, 0); + + if (ret == -ENOTBLK) { + /* Returns -ENOTBLK in case of a page invalidation failure for writes.*/ + /* The callers needs to fall back to buffered I/O in this case. */ + ret = 0; + } + + if (ret >= 0 && iov_iter_count(from)) { + vbo = iocb->ki_pos; + + iocb->ki_flags &= ~IOCB_DIRECT; + err = iomap_file_buffered_write(iocb, from, &ntfs_iomap_ops, + &ntfs_iomap_folio_ops, NULL); + if (err < 0) { + ret = err; + goto out; + } + + /* + * We need to ensure that the pages within the page cache for + * the range covered by this I/O are written to disk and + * invalidated. This is in attempt to preserve the expected + * direct I/O semantics in the case we fallback to buffered I/O + * to complete off the I/O request. + */ + ret += err; + endbyte = vbo + err - 1; + err = filemap_write_and_wait_range(inode->i_mapping, vbo, + endbyte); + if (err) { + ret = err; + goto out; + } + + invalidate_mapping_pages(inode->i_mapping, vbo >> PAGE_SHIFT, + endbyte >> PAGE_SHIFT); + } + +out: + inode_unlock(inode); + + return ret; +} + +/* + * ntfs_file_open - file_operations::open + */ +int ntfs_file_open(struct inode *inode, struct file *file) +{ + struct ntfs_inode *ni = ntfs_i(inode); + + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(ni))) + return -EINVAL; + + if (unlikely(ntfs3_forced_shutdown(inode->i_sb))) + return -EIO; + + if (unlikely((is_compressed(ni) || is_encrypted(ni)) && + (file->f_flags & O_DIRECT))) { + return -EOPNOTSUPP; + } + + /* Decompress "external compressed" file if opened for rw. */ + if ((ni->ni_flags & NI_FLAG_COMPRESSED_MASK) && + (file->f_flags & (O_WRONLY | O_RDWR | O_TRUNC))) { +#ifdef CONFIG_NTFS3_LZX_XPRESS + int err = ni_decompress_file(ni); + + if (err) + return err; +#else + ntfs_inode_warn( + inode, + "activate CONFIG_NTFS3_LZX_XPRESS to write external compressed files"); + return -EOPNOTSUPP; +#endif + } + + file->f_mode |= FMODE_NOWAIT | FMODE_CAN_ODIRECT; + + return generic_file_open(inode, file); +} + +/* + * ntfs_file_release - file_operations::release + * + * Called when an inode is released. Note that this is different + * from ntfs_file_open: open gets called at every open, but release + * gets called only when /all/ the files are closed. + */ +static int ntfs_file_release(struct inode *inode, struct file *file) +{ + int err; + struct ntfs_inode *ni; + + if (!(file->f_mode & FMODE_WRITE) || + atomic_read(&inode->i_writecount) != 1 || + inode->i_ino == MFT_REC_MFT) { + return 0; + } + + /* Close the last writer on the inode. */ + ni = ntfs_i(inode); + + /* Allocate delayed blocks (clusters). */ + err = ni_allocate_da_blocks(ni); + if (err) + goto out; + + if (ni->mi.sbi->options->prealloc) { + ni_lock(ni); + down_write(&ni->file.run_lock); + + /* Deallocate preallocated. */ + err = attr_set_size_ex(ni, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, &ni->file.run, + inode->i_size, &ni->i_valid, false, NULL, + true); + + up_write(&ni->file.run_lock); + ni_unlock(ni); + } +out: + return err; +} + +/* + * ntfs_fiemap - inode_operations::fiemap + */ +int ntfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + __u64 start, __u64 len) +{ + int err; + struct ntfs_inode *ni = ntfs_i(inode); + + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(ni))) + return -EINVAL; + + if (is_compressed(ni)) { + /* Unfortunately cp -r incorrectly treats compressed clusters. */ + ntfs_inode_warn(inode, + "fiemap is not supported for compressed file"); + return -EOPNOTSUPP; + } + + if (S_ISDIR(inode->i_mode)) { + /* TODO: add support for dirs (ATTR_ALLOC). */ + ntfs_inode_warn(inode, + "fiemap is not supported for directories"); + return -EOPNOTSUPP; + } + + if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) { + ntfs_inode_warn(inode, "fiemap(xattr) is not supported"); + return -EOPNOTSUPP; + } + + inode_lock_shared(inode); + + err = iomap_fiemap(inode, fieinfo, start, len, &ntfs_iomap_ops); + + inode_unlock_shared(inode); + return err; +} + +/* + * ntfs_file_splice_write - file_operations::splice_write + */ +static ssize_t ntfs_file_splice_write(struct pipe_inode_info *pipe, + struct file *file, loff_t *ppos, + size_t len, unsigned int flags) +{ + ssize_t err; + struct inode *inode = file_inode(file); + + err = check_write_restriction(inode); + if (err) + return err; + + return iter_file_splice_write(pipe, file, ppos, len, flags); +} + +/* + * ntfs_file_fsync - file_operations::fsync + */ +int ntfs_file_fsync(struct file *file, loff_t start, loff_t end, int datasync) +{ + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + int err, ret; + + if (unlikely(ntfs3_forced_shutdown(sb))) + return -EIO; + + ret = file_write_and_wait_range(file, start, end); + if (ret) + return ret; + + ret = write_inode_now(inode, !datasync); + + if (!ret) { + ret = ni_write_parents(ntfs_i(inode), !datasync); + } + + if (!ret) { + ntfs_set_state(sbi, NTFS_DIRTY_CLEAR); + ntfs_update_mftmirr(sbi); + } + + err = sync_blockdev(sb->s_bdev); + if (unlikely(err && !ret)) + ret = err; + if (!ret) + blkdev_issue_flush(sb->s_bdev); + return ret; +} + +/* + * ntfs_llseek - file_operations::llseek + */ +static loff_t ntfs_llseek(struct file *file, loff_t offset, int whence) +{ + struct inode *inode = file->f_mapping->host; + struct ntfs_inode *ni = ntfs_i(inode); + loff_t maxbytes = ntfs_get_maxbytes(ni); + loff_t ret; + + if (whence != SEEK_DATA && whence != SEEK_HOLE) { + ret = generic_file_llseek_size(file, offset, whence, maxbytes, + i_size_read(inode)); + } else if ((unsigned long long)offset >= i_size_read(inode)) { + ret = -ENXIO; + } else { + inode_lock_shared(inode); + /* Scan file for hole or data. */ + ret = ni_seek_data_or_hole(ni, offset, whence == SEEK_DATA); + inode_unlock_shared(inode); + + if (ret >= 0) + ret = vfs_setpos(file, ret, maxbytes); + } + return ret; +} + +// clang-format off +const struct inode_operations ntfs_file_inode_operations = { + .getattr = ntfs_getattr, + .setattr = ntfs_setattr, + .listxattr = ntfs_listxattr, + .get_acl = ntfs_get_acl, + .set_acl = ntfs_set_acl, + .fiemap = ntfs_fiemap, + .fileattr_get = ntfs_fileattr_get, + .fileattr_set = ntfs_fileattr_set, +}; + +const struct file_operations ntfs_file_operations = { + .llseek = ntfs_llseek, + .read_iter = ntfs_file_read_iter, + .write_iter = ntfs_file_write_iter, + .unlocked_ioctl = ntfs_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = ntfs_compat_ioctl, +#endif + .splice_read = ntfs_file_splice_read, + .splice_write = ntfs_file_splice_write, + .mmap_prepare = ntfs_file_mmap_prepare, + .open = ntfs_file_open, + .fsync = ntfs_file_fsync, + .fallocate = ntfs_fallocate, + .release = ntfs_file_release, + .setlease = generic_setlease, +}; +// clang-format on diff --git a/fs/ntfs3/frecord.c b/fs/ntfs3/frecord.c new file mode 100644 index 000000000..bead01a95 --- /dev/null +++ b/fs/ntfs3/frecord.c @@ -0,0 +1,3451 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + */ + +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" +#ifdef CONFIG_NTFS3_LZX_XPRESS +#include "lib/lib.h" +#endif + +static struct mft_inode *ni_ins_mi(struct ntfs_inode *ni, struct rb_root *tree, + CLST ino, struct rb_node *ins) +{ + struct rb_node **p = &tree->rb_node; + struct rb_node *pr = NULL; + + while (*p) { + struct mft_inode *mi; + + pr = *p; + mi = rb_entry(pr, struct mft_inode, node); + if (mi->rno > ino) + p = &pr->rb_left; + else if (mi->rno < ino) + p = &pr->rb_right; + else + return mi; + } + + if (!ins) + return NULL; + + rb_link_node(ins, pr, p); + rb_insert_color(ins, tree); + return rb_entry(ins, struct mft_inode, node); +} + +/* + * ni_find_mi - Find mft_inode by record number. + */ +static struct mft_inode *ni_find_mi(struct ntfs_inode *ni, CLST rno) +{ + return ni_ins_mi(ni, &ni->mi_tree, rno, NULL); +} + +/* + * ni_add_mi - Add new mft_inode into ntfs_inode. + */ +static void ni_add_mi(struct ntfs_inode *ni, struct mft_inode *mi) +{ + ni_ins_mi(ni, &ni->mi_tree, mi->rno, &mi->node); +} + +/* + * ni_remove_mi - Remove mft_inode from ntfs_inode. + */ +void ni_remove_mi(struct ntfs_inode *ni, struct mft_inode *mi) +{ + rb_erase(&mi->node, &ni->mi_tree); +} + +/* + * ni_std - Return: Pointer into std_info from primary record. + */ +struct ATTR_STD_INFO *ni_std(struct ntfs_inode *ni) +{ + const struct ATTRIB *attr; + + attr = mi_find_attr(ni, &ni->mi, NULL, ATTR_STD, NULL, 0, NULL); + return attr ? resident_data_ex(attr, sizeof(struct ATTR_STD_INFO)) : + NULL; +} + +/* + * ni_std5 + * + * Return: Pointer into std_info from primary record. + */ +struct ATTR_STD_INFO5 *ni_std5(struct ntfs_inode *ni) +{ + const struct ATTRIB *attr; + + attr = mi_find_attr(ni, &ni->mi, NULL, ATTR_STD, NULL, 0, NULL); + + return attr ? resident_data_ex(attr, sizeof(struct ATTR_STD_INFO5)) : + NULL; +} + +/* + * ni_clear - Clear resources allocated by ntfs_inode. + */ +void ni_clear(struct ntfs_inode *ni) +{ + struct rb_node *node; + + if (!ni->vfs_inode.i_nlink && ni->mi.mrec && + is_rec_inuse(ni->mi.mrec) && + !(ni->mi.sbi->flags & NTFS_FLAGS_LOG_REPLAYING)) + ni_delete_all(ni); + + al_destroy(ni); + + for (node = rb_first(&ni->mi_tree); node;) { + struct rb_node *next = rb_next(node); + struct mft_inode *mi = rb_entry(node, struct mft_inode, node); + + rb_erase(node, &ni->mi_tree); + mi_put(mi); + node = next; + } + + /* Bad inode always has mode == S_IFREG. */ + if (ni->ni_flags & NI_FLAG_DIR) + indx_clear(&ni->dir); + else { + run_close(&ni->file.run); + ntfs_sub_da(ni->mi.sbi, run_len(&ni->file.run_da)); + run_close(&ni->file.run_da); +#ifdef CONFIG_NTFS3_LZX_XPRESS + if (ni->file.offs_folio) { + /* On-demand allocated page for offsets. */ + folio_put(ni->file.offs_folio); + ni->file.offs_folio = NULL; + } +#endif + kfree(ni->file.ads.name); + ni->file.ads.name = NULL; + } + + if (ni->base && ni->base != ni) { + iput(&ni->base->vfs_inode); + ni->base = NULL; + } + + mi_clear(&ni->mi); +} + +/* + * ni_load_mi_ex - Find mft_inode by record number. + */ +int ni_load_mi_ex(struct ntfs_inode *ni, CLST rno, struct mft_inode **mi) +{ + int err; + struct mft_inode *r; + + ni = ni->base; + r = ni_find_mi(ni, rno); + if (r) + goto out; + + err = mi_get(ni->mi.sbi, rno, &r); + if (err) { + _ntfs_bad_inode(&ni->vfs_inode); + return err; + } + + ni_add_mi(ni, r); + +out: + if (mi) + *mi = r; + return 0; +} + +/* + * ni_load_mi - Load mft_inode corresponded list_entry. + */ +int ni_load_mi(struct ntfs_inode *ni, const struct ATTR_LIST_ENTRY *le, + struct mft_inode **mi) +{ + u64 rno; + + ni = ni->base; + if (!le) { + *mi = &ni->mi; + return 0; + } + + rno = ino_get(&le->ref); + if (rno == ni->mi.rno) { + *mi = &ni->mi; + return 0; + } + return ni_load_mi_ex(ni, rno, mi); +} + +/* + * ni_find_attr + * + * Return: Attribute and record this attribute belongs to. + */ +struct ATTRIB *ni_find_attr(struct ntfs_inode *ni, struct ATTRIB *attr, + struct ATTR_LIST_ENTRY **le_o, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, const CLST *vcn, + struct mft_inode **mi) +{ + struct ATTR_LIST_ENTRY *le; + struct mft_inode *m; + + ni = ni->base; + if (!ni->attr_list.size || + (!name_len && (type == ATTR_LIST || type == ATTR_STD))) { + if (le_o) + *le_o = NULL; + if (mi) + *mi = &ni->mi; + + /* Look for required attribute in primary record. */ + return mi_find_attr(ni, &ni->mi, attr, type, name, name_len, + NULL); + } + + /* First look for list entry of required type. */ + le = al_find_ex(ni, le_o ? *le_o : NULL, type, name, name_len, vcn); + if (!le) + return NULL; + + if (le_o) + *le_o = le; + + /* Load record that contains this attribute. */ + if (ni_load_mi(ni, le, &m)) + return NULL; + + /* Look for required attribute. */ + attr = mi_find_attr(ni, m, NULL, type, name, name_len, &le->id); + + if (!attr) + goto out; + + if (!attr->non_res) { + if (vcn && *vcn) + goto out; + } else if (!vcn) { + if (attr->nres.svcn) + goto out; + } else if (le64_to_cpu(attr->nres.svcn) > *vcn || + *vcn > le64_to_cpu(attr->nres.evcn)) { + goto out; + } + + if (mi) + *mi = m; + return attr; + +out: + _ntfs_bad_inode(&ni->vfs_inode); + return NULL; +} + +/* + * ni_enum_attr_ex - Enumerates attributes in ntfs_inode. + */ +struct ATTRIB *ni_enum_attr_ex(struct ntfs_inode *ni, struct ATTRIB *attr, + struct ATTR_LIST_ENTRY **le, + struct mft_inode **mi) +{ + struct mft_inode *mi2; + struct ATTR_LIST_ENTRY *le2; + + ni = ni->base; + /* Do we have an attribute list? */ + if (!ni->attr_list.size) { + *le = NULL; + if (mi) + *mi = &ni->mi; + /* Enum attributes in primary record. */ + return mi_enum_attr(ni, &ni->mi, attr); + } + + /* Get next list entry. */ + le2 = *le = al_enumerate(ni, attr ? *le : NULL); + if (!le2) + return NULL; + + /* Load record that contains the required attribute. */ + if (ni_load_mi(ni, le2, &mi2)) + return NULL; + + if (mi) + *mi = mi2; + + /* Find attribute in loaded record. */ + return rec_find_attr_le(ni, mi2, le2); +} + +/* + * ni_load_all_mi - Load all subrecords. + */ +int ni_load_all_mi(struct ntfs_inode *ni) +{ + int err; + struct ATTR_LIST_ENTRY *le; + + ni = ni->base; + if (!ni->attr_list.size) + return 0; + + le = NULL; + + while ((le = al_enumerate(ni, le))) { + CLST rno = ino_get(&le->ref); + + if (rno == ni->mi.rno) + continue; + + err = ni_load_mi_ex(ni, rno, NULL); + if (err) + return err; + } + + return 0; +} + +/* + * ni_add_subrecord - Allocate + format + attach a new subrecord. + */ +bool ni_add_subrecord(struct ntfs_inode *ni, CLST rno, struct mft_inode **mi) +{ + struct mft_inode *m; + + ni = ni->base; + m = kzalloc_obj(struct mft_inode, GFP_NOFS); + if (!m) + return false; + + if (mi_format_new(m, ni->mi.sbi, rno, 0, ni->mi.rno == MFT_REC_MFT)) { + mi_put(m); + return false; + } + + mi_get_ref(&ni->mi, &m->mrec->parent_ref); + + *mi = ni_ins_mi(ni, &ni->mi_tree, m->rno, &m->node); + if (*mi != m) + mi_put(m); + + return true; +} + +/* + * ni_remove_attr - Remove all attributes for the given type/name/id. + */ +int ni_remove_attr(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, bool base_only, + const __le16 *id) +{ + int err; + struct ATTRIB *attr; + struct ATTR_LIST_ENTRY *le; + struct mft_inode *mi; + u32 type_in; + int diff; + + ni = ni->base; + if (base_only || type == ATTR_LIST || !ni->attr_list.size) { + attr = mi_find_attr(ni, &ni->mi, NULL, type, name, name_len, + id); + if (!attr) + return -ENOENT; + + mi_remove_attr(ni, &ni->mi, attr); + return 0; + } + + type_in = le32_to_cpu(type); + le = NULL; + + for (;;) { + le = al_enumerate(ni, le); + if (!le) + return 0; + +next_le2: + diff = le32_to_cpu(le->type) - type_in; + if (diff < 0) + continue; + + if (diff > 0) + return 0; + + if (le->name_len != name_len) + continue; + + if (name_len && + memcmp(le_name(le), name, name_len * sizeof(short))) + continue; + + if (id && le->id != *id) + continue; + err = ni_load_mi(ni, le, &mi); + if (err) + return err; + + al_remove_le(ni, le); + + attr = mi_find_attr(ni, mi, NULL, type, name, name_len, id); + if (!attr) + return -ENOENT; + + mi_remove_attr(ni, mi, attr); + + if (PtrOffset(ni->attr_list.le, le) >= ni->attr_list.size) + return 0; + goto next_le2; + } +} + +/* + * ni_ins_new_attr - Insert the attribute into record. + * + * Return: Not full constructed attribute or NULL if not possible to create. + */ +static struct ATTRIB * +ni_ins_new_attr(struct ntfs_inode *ni, struct mft_inode *mi, + struct ATTR_LIST_ENTRY *le, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, u32 asize, u16 name_off, + CLST svcn, struct ATTR_LIST_ENTRY **ins_le) +{ + int err; + struct ATTRIB *attr; + bool le_added = false; + struct MFT_REF ref; + + ni = ni->base; + mi_get_ref(mi, &ref); + + if (type != ATTR_LIST && !le && ni->attr_list.size) { + err = al_add_le(ni, type, name, name_len, svcn, cpu_to_le16(-1), + &ref, &le); + if (err) { + /* No memory or no space. */ + return ERR_PTR(err); + } + le_added = true; + + /* + * al_add_le -> attr_set_size (list) -> ni_expand_list + * which moves some attributes out of primary record + * this means that name may point into moved memory + * reinit 'name' from le. + */ + name = le->name; + } + + attr = mi_insert_attr(ni, mi, type, name, name_len, asize, name_off); + if (!attr) { + if (le_added) + al_remove_le(ni, le); + return NULL; + } + + if (type == ATTR_LIST) { + /* Attr list is not in list entry array. */ + goto out; + } + + if (!le) + goto out; + + /* Update ATTRIB Id and record reference. */ + le->id = attr->id; + ni->attr_list.dirty = true; + le->ref = ref; + +out: + if (ins_le) + *ins_le = le; + return attr; +} + +/* + * ni_repack + * + * Random write access to sparsed or compressed file may result to + * not optimized packed runs. + * Here is the place to optimize it. + */ +static int ni_repack(struct ntfs_inode *ni) +{ +#if 1 + return 0; +#else + int err = 0; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct mft_inode *mi, *mi_p = NULL; + struct ATTRIB *attr = NULL, *attr_p; + struct ATTR_LIST_ENTRY *le = NULL, *le_p; + CLST alloc = 0; + u8 cluster_bits = sbi->cluster_bits; + CLST svcn, evcn = 0, svcn_p, evcn_p, next_svcn; + u32 roff, rs = sbi->record_size; + struct runs_tree run; + + run_init(&run); + + while ((attr = ni_enum_attr_ex(ni, attr, &le, &mi))) { + if (!attr->non_res) + continue; + + svcn = le64_to_cpu(attr->nres.svcn); + if (svcn != le64_to_cpu(le->vcn)) { + err = -EINVAL; + break; + } + + if (!svcn) { + alloc = le64_to_cpu(attr->nres.alloc_size) >> + cluster_bits; + mi_p = NULL; + } else if (svcn != evcn + 1) { + err = -EINVAL; + break; + } + + evcn = le64_to_cpu(attr->nres.evcn); + + if (svcn > evcn + 1) { + err = -EINVAL; + break; + } + + if (!mi_p) { + /* Do not try if not enough free space. */ + if (le32_to_cpu(mi->mrec->used) + 8 >= rs) + continue; + + /* Do not try if last attribute segment. */ + if (evcn + 1 == alloc) + continue; + run_close(&run); + } + + roff = le16_to_cpu(attr->nres.run_off); + + if (roff > le32_to_cpu(attr->size)) { + err = -EINVAL; + break; + } + + err = run_unpack(&run, sbi, ni->mi.rno, svcn, evcn, svcn, + Add2Ptr(attr, roff), + le32_to_cpu(attr->size) - roff); + if (err < 0) + break; + + if (!mi_p) { + mi_p = mi; + attr_p = attr; + svcn_p = svcn; + evcn_p = evcn; + le_p = le; + err = 0; + continue; + } + + /* + * Run contains data from two records: mi_p and mi + * Try to pack in one. + */ + err = mi_pack_runs(mi_p, attr_p, &run, evcn + 1 - svcn_p); + if (err) + break; + + next_svcn = le64_to_cpu(attr_p->nres.evcn) + 1; + + if (next_svcn >= evcn + 1) { + /* We can remove this attribute segment. */ + al_remove_le(ni, le); + mi_remove_attr(NULL, mi, attr); + le = le_p; + continue; + } + + attr->nres.svcn = le->vcn = cpu_to_le64(next_svcn); + mi->dirty = true; + ni->attr_list.dirty = true; + + if (evcn + 1 == alloc) { + err = mi_pack_runs(mi, attr, &run, + evcn + 1 - next_svcn); + if (err) + break; + mi_p = NULL; + } else { + mi_p = mi; + attr_p = attr; + svcn_p = next_svcn; + evcn_p = evcn; + le_p = le; + run_truncate_head(&run, next_svcn); + } + } + + if (err) { + ntfs_inode_warn(&ni->vfs_inode, "repack problem"); + ntfs_set_state(sbi, NTFS_DIRTY_ERROR); + + /* Pack loaded but not packed runs. */ + if (mi_p) + mi_pack_runs(mi_p, attr_p, &run, evcn_p + 1 - svcn_p); + } + + run_close(&run); + return err; +#endif +} + +/* + * ni_try_remove_attr_list + * + * Can we remove attribute list? + * Check the case when primary record contains enough space for all attributes. + */ +static int ni_try_remove_attr_list(struct ntfs_inode *ni) +{ + int err = 0; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct ATTRIB *attr, *attr_list, *attr_ins; + struct ATTR_LIST_ENTRY *le; + struct mft_inode *mi; + u32 asize, free; + struct MFT_REF ref; + struct MFT_REC *mrec; + __le16 id; + + ni = ni->base; + if (!ni->attr_list.dirty) + return 0; + + err = ni_repack(ni); + if (err) + return err; + + attr_list = mi_find_attr(ni, &ni->mi, NULL, ATTR_LIST, NULL, 0, NULL); + if (!attr_list) + return 0; + + asize = le32_to_cpu(attr_list->size); + + /* Free space in primary record without attribute list. */ + free = sbi->record_size - le32_to_cpu(ni->mi.mrec->used) + asize; + mi_get_ref(&ni->mi, &ref); + + le = NULL; + while ((le = al_enumerate(ni, le))) { + if (!memcmp(&le->ref, &ref, sizeof(ref))) + continue; + + if (le->vcn) + return 0; + + mi = ni_find_mi(ni, ino_get(&le->ref)); + if (!mi) + return 0; + + attr = mi_find_attr(ni, mi, NULL, le->type, le_name(le), + le->name_len, &le->id); + if (!attr) + return 0; + + asize = le32_to_cpu(attr->size); + if (asize > free) + return 0; + + free -= asize; + } + + /* Make a copy of primary record to restore if error. */ + mrec = kmemdup(ni->mi.mrec, sbi->record_size, GFP_NOFS); + if (!mrec) + return 0; /* Not critical. */ + + /* It seems that attribute list can be removed from primary record. */ + mi_remove_attr(NULL, &ni->mi, attr_list); + + /* + * Repeat the cycle above and copy all attributes to primary record. + * Do not remove original attributes from subrecords! + * It should be success! + */ + le = NULL; + while ((le = al_enumerate(ni, le))) { + if (!memcmp(&le->ref, &ref, sizeof(ref))) + continue; + + mi = ni_find_mi(ni, ino_get(&le->ref)); + if (!mi) { + /* Should never happened, 'cause already checked. */ + goto out; + } + + attr = mi_find_attr(ni, mi, NULL, le->type, le_name(le), + le->name_len, &le->id); + if (!attr) { + /* Should never happened, 'cause already checked. */ + goto out; + } + asize = le32_to_cpu(attr->size); + + /* Insert into primary record. */ + attr_ins = mi_insert_attr(ni, &ni->mi, le->type, le_name(le), + le->name_len, asize, + le16_to_cpu(attr->name_off)); + if (!attr_ins) { + /* + * No space in primary record (already checked). + */ + goto out; + } + + /* Copy all except id. */ + id = attr_ins->id; + memcpy(attr_ins, attr, asize); + attr_ins->id = id; + } + + /* + * Repeat the cycle above and remove all attributes from subrecords. + */ + le = NULL; + while ((le = al_enumerate(ni, le))) { + if (!memcmp(&le->ref, &ref, sizeof(ref))) + continue; + + mi = ni_find_mi(ni, ino_get(&le->ref)); + if (!mi) + continue; + + attr = mi_find_attr(ni, mi, NULL, le->type, le_name(le), + le->name_len, &le->id); + if (!attr) + continue; + + /* Remove from original record. */ + mi_remove_attr(NULL, mi, attr); + } + + run_deallocate(sbi, &ni->attr_list.run, true); + run_close(&ni->attr_list.run); + ni->attr_list.size = 0; + kvfree(ni->attr_list.le); + ni->attr_list.le = NULL; + ni->attr_list.dirty = false; + + kfree(mrec); + return 0; +out: + /* Restore primary record. */ + swap(mrec, ni->mi.mrec); + kfree(mrec); + return 0; +} + +/* + * ni_create_attr_list - Generates an attribute list for this primary record. + */ +int ni_create_attr_list(struct ntfs_inode *ni) +{ + struct ntfs_sb_info *sbi = ni->mi.sbi; + int err; + u32 lsize; + struct ATTRIB *attr; + struct ATTRIB *arr_move[7]; + struct ATTR_LIST_ENTRY *le, *le_b[7]; + struct MFT_REC *rec; + bool is_mft; + CLST rno = 0; + struct mft_inode *mi; + u32 free_b, nb, to_free, rs; + u16 sz; + + ni = ni->base; + is_mft = ni->mi.rno == MFT_REC_MFT; + rec = ni->mi.mrec; + rs = sbi->record_size; + + /* + * Compute the exact size of the attribute list. Each attribute in the + * record yields one ATTR_LIST_ENTRY of le_size(name_len) bytes. The + * minimum on-disk attribute is SIZEOF_RESIDENT (0x18) bytes, but an + * unnamed one expands to le_size(0) (0x20) here, so a record crafted + * with many such attributes needs more than a single record_size; the + * previous fixed kzalloc(record_size) could therefore be overflowed by + * an attacker-controlled record. + */ + lsize = 0; + attr = NULL; + while ((attr = mi_enum_attr(ni, &ni->mi, attr))) + lsize += le_size(attr->name_len); + + if (!lsize) + return -EINVAL; + + le = kzalloc(al_aligned(lsize), GFP_NOFS); + if (!le) + return -ENOMEM; + + mi_get_ref(&ni->mi, &le->ref); + ni->attr_list.le = le; + + attr = NULL; + nb = 0; + free_b = 0; + + for (; (attr = mi_enum_attr(ni, &ni->mi, attr)); le = Add2Ptr(le, sz)) { + sz = le_size(attr->name_len); + le->type = attr->type; + le->size = cpu_to_le16(sz); + le->name_len = attr->name_len; + le->name_off = offsetof(struct ATTR_LIST_ENTRY, name); + le->vcn = 0; + if (le != ni->attr_list.le) + le->ref = ni->attr_list.le->ref; + le->id = attr->id; + + if (attr->name_len) + memcpy(le->name, attr_name(attr), + sizeof(short) * attr->name_len); + else if (attr->type == ATTR_STD) + continue; + else if (attr->type == ATTR_LIST) + continue; + else if (is_mft && attr->type == ATTR_DATA) + continue; + + if (!nb || nb < ARRAY_SIZE(arr_move)) { + le_b[nb] = le; + arr_move[nb++] = attr; + free_b += le32_to_cpu(attr->size); + } + } + + lsize = PtrOffset(ni->attr_list.le, le); + ni->attr_list.size = lsize; + + to_free = le32_to_cpu(rec->used) + lsize + SIZEOF_RESIDENT; + if (to_free <= rs) { + to_free = 0; + } else { + to_free -= rs; + + if (to_free > free_b) { + err = -EINVAL; + goto out; + } + } + + /* Allocate child MFT. */ + err = ntfs_look_free_mft(sbi, &rno, is_mft, ni, &mi); + if (err) + goto out; + + err = -EINVAL; + /* Call mi_remove_attr() in reverse order to keep pointers 'arr_move' valid. */ + while (to_free > 0) { + struct ATTRIB *b = arr_move[--nb]; + u32 asize = le32_to_cpu(b->size); + u16 name_off = le16_to_cpu(b->name_off); + + attr = mi_insert_attr(ni, mi, b->type, Add2Ptr(b, name_off), + b->name_len, asize, name_off); + if (!attr) + goto out; + + mi_get_ref(mi, &le_b[nb]->ref); + le_b[nb]->id = attr->id; + + /* Copy all except id. */ + memcpy(attr, b, asize); + attr->id = le_b[nb]->id; + + /* Remove from primary record. */ + if (!mi_remove_attr(NULL, &ni->mi, b)) + goto out; + + if (to_free <= asize) + break; + to_free -= asize; + if (!nb) + goto out; + } + + attr = mi_insert_attr(ni, &ni->mi, ATTR_LIST, NULL, 0, + lsize + SIZEOF_RESIDENT, SIZEOF_RESIDENT); + if (!attr) + goto out; + + attr->non_res = 0; + attr->flags = 0; + attr->res.data_size = cpu_to_le32(lsize); + attr->res.data_off = SIZEOF_RESIDENT_LE; + attr->res.flags = 0; + attr->res.res = 0; + + memcpy(resident_data_ex(attr, lsize), ni->attr_list.le, lsize); + + ni->attr_list.dirty = false; + + mark_inode_dirty(&ni->vfs_inode); + return 0; + +out: + kvfree(ni->attr_list.le); + ni->attr_list.le = NULL; + ni->attr_list.size = 0; + return err; +} + +/* + * ni_ins_attr_ext - Add an external attribute to the ntfs_inode. + */ +static int ni_ins_attr_ext(struct ntfs_inode *ni, struct ATTR_LIST_ENTRY *le, + enum ATTR_TYPE type, const __le16 *name, u8 name_len, + u32 asize, CLST svcn, u16 name_off, bool force_ext, + struct ATTRIB **ins_attr, struct mft_inode **ins_mi, + struct ATTR_LIST_ENTRY **ins_le) +{ + struct ATTRIB *attr; + struct mft_inode *mi; + CLST rno; + u64 vbo; + struct rb_node *node; + int err; + bool is_mft, is_mft_data; + struct ntfs_sb_info *sbi = ni->mi.sbi; + + ni = ni->base; + is_mft = ni->mi.rno == MFT_REC_MFT; + is_mft_data = is_mft && type == ATTR_DATA && !name_len; + + if (asize > sbi->max_bytes_per_attr) { + err = -EINVAL; + goto out; + } + + /* + * Standard information and attr_list cannot be made external. + * The Log File cannot have any external attributes. + */ + if (type == ATTR_STD || type == ATTR_LIST || + ni->mi.rno == MFT_REC_LOG) { + err = -EINVAL; + goto out; + } + + /* Create attribute list if it is not already existed. */ + if (!ni->attr_list.size) { + err = ni_create_attr_list(ni); + if (err) + goto out; + } + + vbo = is_mft_data ? ((u64)svcn << sbi->cluster_bits) : 0; + + if (force_ext) + goto insert_ext; + + /* Load all subrecords into memory. */ + err = ni_load_all_mi(ni); + if (err) + goto out; + + /* Check each of loaded subrecord. */ + for (node = rb_first(&ni->mi_tree); node; node = rb_next(node)) { + mi = rb_entry(node, struct mft_inode, node); + + if (is_mft_data && + (mi_enum_attr(ni, mi, NULL) || + vbo <= ((u64)mi->rno << sbi->record_bits))) { + /* We can't accept this record 'cause MFT's bootstrapping. */ + continue; + } + if (is_mft && + mi_find_attr(ni, mi, NULL, ATTR_DATA, NULL, 0, NULL)) { + /* + * This child record already has a ATTR_DATA. + * So it can't accept any other records. + */ + continue; + } + + if ((type != ATTR_NAME || name_len) && + mi_find_attr(ni, mi, NULL, type, name, name_len, NULL)) { + /* Only indexed attributes can share same record. */ + continue; + } + + /* + * Do not try to insert this attribute + * if there is no room in record. + */ + if (le32_to_cpu(mi->mrec->used) + asize > sbi->record_size) + continue; + + /* Try to insert attribute into this subrecord. */ + attr = ni_ins_new_attr(ni, mi, le, type, name, name_len, asize, + name_off, svcn, ins_le); + if (!attr) + continue; + if (IS_ERR(attr)) + return PTR_ERR(attr); + + if (ins_attr) + *ins_attr = attr; + if (ins_mi) + *ins_mi = mi; + return 0; + } + +insert_ext: + /* We have to allocate a new child subrecord. */ + err = ntfs_look_free_mft(sbi, &rno, is_mft_data, ni, &mi); + if (err) + goto out; + + if (is_mft_data && vbo <= ((u64)rno << sbi->record_bits)) { + err = -EINVAL; + goto out1; + } + + attr = ni_ins_new_attr(ni, mi, le, type, name, name_len, asize, + name_off, svcn, ins_le); + if (!attr) { + err = -EINVAL; + goto out2; + } + + if (IS_ERR(attr)) { + err = PTR_ERR(attr); + goto out2; + } + + if (ins_attr) + *ins_attr = attr; + if (ins_mi) + *ins_mi = mi; + + return 0; + +out2: + ni_remove_mi(ni, mi); + +out1: + mi_put(mi); + ntfs_mark_rec_free(sbi, rno, is_mft); + +out: + return err; +} + +/* + * ni_insert_attr - Insert an attribute into the file. + * + * If the primary record has room, it will just insert the attribute. + * If not, it may make the attribute external. + * For $MFT::Data it may make room for the attribute by + * making other attributes external. + * + * NOTE: + * The ATTR_LIST and ATTR_STD cannot be made external. + * This function does not fill new attribute full. + * It only fills 'size'/'type'/'id'/'name_len' fields. + */ +static int ni_insert_attr(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, u32 asize, + u16 name_off, CLST svcn, struct ATTRIB **ins_attr, + struct mft_inode **ins_mi, + struct ATTR_LIST_ENTRY **ins_le) +{ + struct ntfs_sb_info *sbi = ni->mi.sbi; + int err; + struct ATTRIB *attr, *eattr; + struct MFT_REC *rec; + bool is_mft; + struct ATTR_LIST_ENTRY *le; + u32 list_reserve, max_free, free, used, t32; + __le16 id; + u16 t16; + + ni = ni->base; + is_mft = ni->mi.rno == MFT_REC_MFT; + rec = ni->mi.mrec; + + list_reserve = SIZEOF_NONRESIDENT + 3 * (1 + 2 * sizeof(u32)); + used = le32_to_cpu(rec->used); + free = sbi->record_size - used; + + if (is_mft && type != ATTR_LIST) { + /* Reserve space for the ATTRIB list. */ + if (free < list_reserve) + free = 0; + else + free -= list_reserve; + } + + if (asize <= free) { + attr = ni_ins_new_attr(ni, &ni->mi, NULL, type, name, name_len, + asize, name_off, svcn, ins_le); + if (IS_ERR(attr)) { + err = PTR_ERR(attr); + goto out; + } + + if (attr) { + if (ins_attr) + *ins_attr = attr; + if (ins_mi) + *ins_mi = &ni->mi; + err = 0; + goto out; + } + } + + if (!is_mft || type != ATTR_DATA || svcn) { + /* This ATTRIB will be external. */ + err = ni_ins_attr_ext(ni, NULL, type, name, name_len, asize, + svcn, name_off, false, ins_attr, ins_mi, + ins_le); + goto out; + } + + /* + * Here we have: "is_mft && type == ATTR_DATA && !svcn" + * + * The first chunk of the $MFT::Data ATTRIB must be the base record. + * Evict as many other attributes as possible. + */ + max_free = free; + + /* Estimate the result of moving all possible attributes away. */ + attr = NULL; + + while ((attr = mi_enum_attr(ni, &ni->mi, attr))) { + if (attr->type == ATTR_STD) + continue; + if (attr->type == ATTR_LIST) + continue; + max_free += le32_to_cpu(attr->size); + } + + if (max_free < asize + list_reserve) { + /* Impossible to insert this attribute into primary record. */ + err = -EINVAL; + goto out; + } + + /* Start real attribute moving. */ + attr = NULL; + + for (;;) { + attr = mi_enum_attr(ni, &ni->mi, attr); + if (!attr) { + /* We should never be here 'cause we have already check this case. */ + err = -EINVAL; + goto out; + } + + /* Skip attributes that MUST be primary record. */ + if (attr->type == ATTR_STD || attr->type == ATTR_LIST) + continue; + + le = NULL; + if (ni->attr_list.size) { + le = al_find_le(ni, NULL, attr); + if (!le) { + /* Really this is a serious bug. */ + err = -EINVAL; + goto out; + } + } + + t32 = le32_to_cpu(attr->size); + t16 = le16_to_cpu(attr->name_off); + err = ni_ins_attr_ext(ni, le, attr->type, Add2Ptr(attr, t16), + attr->name_len, t32, attr_svcn(attr), t16, + false, &eattr, NULL, NULL); + if (err) + return err; + + id = eattr->id; + memcpy(eattr, attr, t32); + eattr->id = id; + + /* Remove from primary record. */ + mi_remove_attr(NULL, &ni->mi, attr); + + /* attr now points to next attribute. */ + if (attr->type == ATTR_END) + goto out; + } + while (asize + list_reserve > sbi->record_size - le32_to_cpu(rec->used)) + ; + + attr = ni_ins_new_attr(ni, &ni->mi, NULL, type, name, name_len, asize, + name_off, svcn, ins_le); + if (!attr) { + err = -EINVAL; + goto out; + } + + if (IS_ERR(attr)) { + err = PTR_ERR(attr); + goto out; + } + + if (ins_attr) + *ins_attr = attr; + if (ins_mi) + *ins_mi = &ni->mi; + +out: + return err; +} + +/* ni_expand_mft_list - Split ATTR_DATA of $MFT. */ +static int ni_expand_mft_list(struct ntfs_inode *ni) +{ + int err = 0; + struct runs_tree *run = &ni->file.run; + u32 asize, run_size, done = 0; + struct ATTRIB *attr; + struct rb_node *node; + CLST mft_min, mft_new, svcn, evcn, plen; + struct mft_inode *mi, *mi_min, *mi_new; + struct ntfs_sb_info *sbi = ni->mi.sbi; + + ni = ni->base; + /* Find the nearest MFT. */ + mft_min = 0; + mft_new = 0; + mi_min = NULL; + + for (node = rb_first(&ni->mi_tree); node; node = rb_next(node)) { + mi = rb_entry(node, struct mft_inode, node); + + attr = mi_enum_attr(ni, mi, NULL); + + if (!attr) { + mft_min = mi->rno; + mi_min = mi; + break; + } + } + + if (ntfs_look_free_mft(sbi, &mft_new, true, ni, &mi_new)) { + mft_new = 0; + /* Really this is not critical. */ + } else if (mft_min > mft_new) { + mft_min = mft_new; + mi_min = mi_new; + } else { + ntfs_mark_rec_free(sbi, mft_new, true); + mft_new = 0; + ni_remove_mi(ni, mi_new); + } + + attr = mi_find_attr(ni, &ni->mi, NULL, ATTR_DATA, NULL, 0, NULL); + if (!attr) { + err = -EINVAL; + goto out; + } + + asize = le32_to_cpu(attr->size); + + evcn = le64_to_cpu(attr->nres.evcn); + svcn = bytes_to_cluster(sbi, (u64)(mft_min + 1) << sbi->record_bits); + if (evcn + 1 >= svcn) { + err = -EINVAL; + goto out; + } + + /* + * Split primary attribute [0 evcn] in two parts [0 svcn) + [svcn evcn]. + * + * Update first part of ATTR_DATA in 'primary MFT. + */ + err = run_pack(run, 0, svcn, Add2Ptr(attr, SIZEOF_NONRESIDENT), + asize - SIZEOF_NONRESIDENT, &plen); + if (err < 0) + goto out; + + run_size = ALIGN(err, 8); + err = 0; + + if (plen < svcn) { + err = -EINVAL; + goto out; + } + + attr->nres.evcn = cpu_to_le64(svcn - 1); + attr->size = cpu_to_le32(run_size + SIZEOF_NONRESIDENT); + /* 'done' - How many bytes of primary MFT becomes free. */ + done = asize - run_size - SIZEOF_NONRESIDENT; + le32_sub_cpu(&ni->mi.mrec->used, done); + + /* Estimate packed size (run_buf=NULL). */ + err = run_pack(run, svcn, evcn + 1 - svcn, NULL, sbi->record_size, + &plen); + if (err < 0) + goto out; + + run_size = ALIGN(err, 8); + err = 0; + + if (plen < evcn + 1 - svcn) { + err = -EINVAL; + goto out; + } + + /* + * This function may implicitly call expand attr_list. + * Insert second part of ATTR_DATA in 'mi_min'. + */ + attr = ni_ins_new_attr(ni, mi_min, NULL, ATTR_DATA, NULL, 0, + SIZEOF_NONRESIDENT + run_size, + SIZEOF_NONRESIDENT, svcn, NULL); + if (!attr) { + err = -EINVAL; + goto out; + } + + if (IS_ERR(attr)) { + err = PTR_ERR(attr); + goto out; + } + + attr->non_res = 1; + attr->name_off = SIZEOF_NONRESIDENT_LE; + attr->flags = 0; + + /* This function can't fail - cause already checked above. */ + run_pack(run, svcn, evcn + 1 - svcn, Add2Ptr(attr, SIZEOF_NONRESIDENT), + run_size, &plen); + + attr->nres.svcn = cpu_to_le64(svcn); + attr->nres.evcn = cpu_to_le64(evcn); + attr->nres.run_off = cpu_to_le16(SIZEOF_NONRESIDENT); + +out: + if (mft_new) { + ntfs_mark_rec_free(sbi, mft_new, true); + ni_remove_mi(ni, mi_new); + } + + return !err && !done ? -EOPNOTSUPP : err; +} + +/* + * ni_expand_list - Move all possible attributes out of primary record. + */ +int ni_expand_list(struct ntfs_inode *ni) +{ + int err = 0; + u32 asize, done = 0; + struct ATTRIB *attr, *ins_attr = NULL; + struct ATTR_LIST_ENTRY *le; + bool is_mft = ni->mi.rno == MFT_REC_MFT; + struct MFT_REF ref; + + ni = ni->base; + mi_get_ref(&ni->mi, &ref); + le = NULL; + + while ((le = al_enumerate(ni, le))) { + if (le->type == ATTR_STD) + continue; + + if (memcmp(&ref, &le->ref, sizeof(struct MFT_REF))) + continue; + + if (is_mft && le->type == ATTR_DATA) + continue; + + /* Find attribute in primary record. */ + attr = rec_find_attr_le(ni, &ni->mi, le); + if (!attr) { + err = -EINVAL; + goto out; + } + + asize = le32_to_cpu(attr->size); + + /* Always insert into new record to avoid collisions (deep recursive). */ + err = ni_ins_attr_ext(ni, le, attr->type, attr_name(attr), + attr->name_len, asize, attr_svcn(attr), + le16_to_cpu(attr->name_off), true, + &ins_attr, NULL, NULL); + + if (err || !ins_attr) + goto out; + + memcpy(ins_attr, attr, asize); + ins_attr->id = le->id; + /* Remove from primary record. */ + mi_remove_attr(NULL, &ni->mi, attr); + + done += asize; + goto out; + } + + if (!is_mft) { + err = -EFBIG; /* Attr list is too big(?) */ + goto out; + } + + /* Split MFT data as much as possible. */ + err = ni_expand_mft_list(ni); + +out: + return !err && !done ? -EOPNOTSUPP : err; +} + +/* + * ni_insert_nonresident - Insert new nonresident attribute. + */ +int ni_insert_nonresident(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, + const struct runs_tree *run, CLST svcn, CLST len, + __le16 flags, struct ATTRIB **new_attr, + struct mft_inode **mi, struct ATTR_LIST_ENTRY **le) +{ + int err; + CLST plen; + struct ATTRIB *attr; + bool is_ext = (flags & (ATTR_FLAG_SPARSED | ATTR_FLAG_COMPRESSED)) && + !svcn; + u32 name_size = ALIGN(name_len * sizeof(short), 8); + u32 name_off = is_ext ? SIZEOF_NONRESIDENT_EX : SIZEOF_NONRESIDENT; + u32 run_off = name_off + name_size; + u32 run_size, asize; + struct ntfs_sb_info *sbi = ni->mi.sbi; + + ni = ni->base; + /* Estimate packed size (run_buf=NULL). */ + err = run_pack(run, svcn, len, NULL, sbi->max_bytes_per_attr - run_off, + &plen); + if (err < 0) + goto out; + + run_size = ALIGN(err, 8); + + if (plen < len) { + err = -EINVAL; + goto out; + } + + asize = run_off + run_size; + + if (asize > sbi->max_bytes_per_attr) { + err = -EINVAL; + goto out; + } + + err = ni_insert_attr(ni, type, name, name_len, asize, name_off, svcn, + &attr, mi, le); + + if (err) + goto out; + + attr->non_res = 1; + attr->name_off = cpu_to_le16(name_off); + attr->flags = flags; + + /* This function can't fail - cause already checked above. */ + run_pack(run, svcn, len, Add2Ptr(attr, run_off), run_size, &plen); + + attr->nres.svcn = cpu_to_le64(svcn); + attr->nres.evcn = cpu_to_le64((u64)svcn + len - 1); + + if (new_attr) + *new_attr = attr; + + *(__le64 *)&attr->nres.run_off = cpu_to_le64(run_off); + + attr->nres.alloc_size = + svcn ? 0 : cpu_to_le64((u64)len << ni->mi.sbi->cluster_bits); + attr->nres.data_size = attr->nres.alloc_size; + attr->nres.valid_size = attr->nres.alloc_size; + + if (is_ext) { + if (flags & ATTR_FLAG_COMPRESSED) + attr->nres.c_unit = NTFS_LZNT_CUNIT; + attr->nres.total_size = attr->nres.alloc_size; + } + +out: + return err; +} + +/* + * ni_insert_resident - Inserts new resident attribute. + */ +int ni_insert_resident(struct ntfs_inode *ni, u32 data_size, + enum ATTR_TYPE type, const __le16 *name, u8 name_len, + struct ATTRIB **new_attr, struct mft_inode **mi, + struct ATTR_LIST_ENTRY **le) +{ + int err; + u32 name_size = ALIGN(name_len * sizeof(short), 8); + u32 asize = SIZEOF_RESIDENT + name_size + ALIGN(data_size, 8); + struct ATTRIB *attr; + + ni = ni->base; + err = ni_insert_attr(ni, type, name, name_len, asize, SIZEOF_RESIDENT, + 0, &attr, mi, le); + if (err) + return err; + + attr->non_res = 0; + attr->flags = 0; + + attr->res.data_size = cpu_to_le32(data_size); + attr->res.data_off = cpu_to_le16(SIZEOF_RESIDENT + name_size); + if (type == ATTR_NAME) { + attr->res.flags = RESIDENT_FLAG_INDEXED; + + /* is_attr_indexed(attr)) == true */ + le16_add_cpu(&ni->mi.mrec->hard_links, 1); + ni->mi.dirty = true; + } + attr->res.res = 0; + + if (new_attr) + *new_attr = attr; + + return 0; +} + +/* + * ni_remove_attr_le - Remove attribute from record. + */ +void ni_remove_attr_le(struct ntfs_inode *ni, struct ATTRIB *attr, + struct mft_inode *mi, struct ATTR_LIST_ENTRY *le) +{ + ni = ni->base; + mi_remove_attr(ni, mi, attr); + + if (le) + al_remove_le(ni, le); +} + +/* + * ni_delete_all - Remove all attributes and frees allocates space. + * + * ntfs_evict_inode->ntfs_clear_inode->ni_delete_all (if no links). + */ +int ni_delete_all(struct ntfs_inode *ni) +{ + int err; + struct ATTR_LIST_ENTRY *le = NULL; + struct ATTRIB *attr = NULL; + struct rb_node *node; + u16 roff; + u32 asize; + CLST svcn, evcn; + struct ntfs_sb_info *sbi = ni->mi.sbi; + bool nt3 = is_ntfs3(sbi); + struct MFT_REF ref; + + ni = ni->base; + while ((attr = ni_enum_attr_ex(ni, attr, &le, NULL))) { + if (!nt3 || attr->name_len) { + ; + } else if (attr->type == ATTR_REPARSE) { + mi_get_ref(&ni->mi, &ref); + ntfs_remove_reparse(sbi, 0, &ref); + } else if (attr->type == ATTR_ID && !attr->non_res && + le32_to_cpu(attr->res.data_size) >= + sizeof(struct GUID)) { + ntfs_objid_remove(sbi, resident_data(attr)); + } + + if (!attr->non_res) + continue; + + svcn = le64_to_cpu(attr->nres.svcn); + evcn = le64_to_cpu(attr->nres.evcn); + + if (evcn + 1 <= svcn) + continue; + + asize = le32_to_cpu(attr->size); + roff = le16_to_cpu(attr->nres.run_off); + + if (roff > asize) { + /* ni_enum_attr_ex checks this case. */ + continue; + } + + /* run==1 means unpack and deallocate. */ + run_unpack_ex(RUN_DEALLOCATE, sbi, ni->mi.rno, svcn, evcn, svcn, + Add2Ptr(attr, roff), asize - roff); + } + + if (ni->attr_list.size) { + run_deallocate(ni->mi.sbi, &ni->attr_list.run, true); + al_destroy(ni); + } + + /* Free all subrecords. */ + for (node = rb_first(&ni->mi_tree); node;) { + struct rb_node *next = rb_next(node); + struct mft_inode *mi = rb_entry(node, struct mft_inode, node); + + clear_rec_inuse(mi->mrec); + mi->dirty = true; + mi_write(mi, 0); + + ntfs_mark_rec_free(sbi, mi->rno, false); + ni_remove_mi(ni, mi); + mi_put(mi); + node = next; + } + + /* Free base record. */ + clear_rec_inuse(ni->mi.mrec); + ni->mi.dirty = true; + err = mi_write(&ni->mi, 0); + + ntfs_mark_rec_free(sbi, ni->mi.rno, false); + + return err; +} + +/* ni_fname_name + * + * Return: File name attribute by its value. + */ +struct ATTR_FILE_NAME *ni_fname_name(struct ntfs_inode *ni, + const struct le_str *uni, + const struct MFT_REF *home_dir, + struct mft_inode **mi, + struct ATTR_LIST_ENTRY **le) +{ + struct ATTRIB *attr = NULL; + struct ATTR_FILE_NAME *fname; + + ni = ni->base; + if (le) + *le = NULL; + + /* Enumerate all names. */ +next: + attr = ni_find_attr(ni, attr, le, ATTR_NAME, NULL, 0, NULL, mi); + if (!attr) + return NULL; + + fname = resident_data_ex(attr, SIZEOF_ATTRIBUTE_FILENAME); + if (!fname) + goto next; + + if (home_dir && memcmp(home_dir, &fname->home, sizeof(*home_dir))) + goto next; + + if (!uni) + return fname; + + if (uni->len != fname->name_len) + goto next; + + if (ntfs_cmp_names(uni->name, uni->len, fname->name, uni->len, NULL, + false)) + goto next; + return fname; +} + +/* + * ni_fname_type + * + * Return: File name attribute with given type. + */ +struct ATTR_FILE_NAME *ni_fname_type(struct ntfs_inode *ni, u8 name_type, + struct mft_inode **mi, + struct ATTR_LIST_ENTRY **le) +{ + struct ATTRIB *attr = NULL; + struct ATTR_FILE_NAME *fname; + + *le = NULL; + ni = ni->base; + + if (name_type == FILE_NAME_POSIX) + return NULL; + + /* Enumerate all names. */ + for (;;) { + attr = ni_find_attr(ni, attr, le, ATTR_NAME, NULL, 0, NULL, mi); + if (!attr) + return NULL; + + fname = resident_data_ex(attr, SIZEOF_ATTRIBUTE_FILENAME); + if (fname && name_type == fname->type) + return fname; + } +} + +/* + * ni_new_attr_flags + * + * Process compressed/sparsed in special way. + * NOTE: You need to set ni->std_fa = new_fa + * after this function to keep internal structures in consistency. + */ +int ni_new_attr_flags(struct ntfs_inode *ni, enum FILE_ATTRIBUTE new_fa) +{ + struct ATTRIB *attr; + struct mft_inode *mi; + __le16 new_aflags; + u32 new_asize; + + ni = ni->base; + attr = ni_find_attr(ni, NULL, NULL, ATTR_DATA, NULL, 0, NULL, &mi); + if (!attr) + return -EINVAL; + + new_aflags = attr->flags; + + if (new_fa & FILE_ATTRIBUTE_SPARSE_FILE) + new_aflags |= ATTR_FLAG_SPARSED; + else + new_aflags &= ~ATTR_FLAG_SPARSED; + + if (new_fa & FILE_ATTRIBUTE_COMPRESSED) + new_aflags |= ATTR_FLAG_COMPRESSED; + else + new_aflags &= ~ATTR_FLAG_COMPRESSED; + + if (new_aflags == attr->flags) + return 0; + + if ((new_aflags & (ATTR_FLAG_COMPRESSED | ATTR_FLAG_SPARSED)) == + (ATTR_FLAG_COMPRESSED | ATTR_FLAG_SPARSED)) { + ntfs_inode_warn(&ni->vfs_inode, + "file can't be sparsed and compressed"); + return -EOPNOTSUPP; + } + + if (!attr->non_res) + goto out; + + if (attr->nres.data_size) { + ntfs_inode_warn( + &ni->vfs_inode, + "one can change sparsed/compressed only for empty files"); + return -EOPNOTSUPP; + } + + /* Resize nonresident empty attribute in-place only. */ + new_asize = (new_aflags & (ATTR_FLAG_COMPRESSED | ATTR_FLAG_SPARSED)) ? + (SIZEOF_NONRESIDENT_EX + 8) : + (SIZEOF_NONRESIDENT + 8); + + if (!mi_resize_attr(mi, attr, new_asize - le32_to_cpu(attr->size))) + return -EOPNOTSUPP; + + if (new_aflags & ATTR_FLAG_SPARSED) { + attr->name_off = SIZEOF_NONRESIDENT_EX_LE; + /* Windows uses 16 clusters per frame but supports one cluster per frame too. */ + attr->nres.c_unit = 0; + ni->vfs_inode.i_mapping->a_ops = &ntfs_aops; + } else if (new_aflags & ATTR_FLAG_COMPRESSED) { + attr->name_off = SIZEOF_NONRESIDENT_EX_LE; + /* The only allowed: 16 clusters per frame. */ + attr->nres.c_unit = NTFS_LZNT_CUNIT; + ni->vfs_inode.i_mapping->a_ops = &ntfs_aops_cmpr; + } else { + attr->name_off = SIZEOF_NONRESIDENT_LE; + /* Normal files. */ + attr->nres.c_unit = 0; + ni->vfs_inode.i_mapping->a_ops = &ntfs_aops; + } + attr->nres.run_off = attr->name_off; +out: + attr->flags = new_aflags; + mi->dirty = true; + + return 0; +} + +/* + * ni_parse_reparse + * + * buffer - memory for reparse buffer header + */ +enum REPARSE_SIGN ni_parse_reparse(struct ntfs_inode *ni, struct ATTRIB *attr, + struct REPARSE_DATA_BUFFER *buffer) +{ + const struct REPARSE_DATA_BUFFER *rp = NULL; + u8 bits; + u16 len; + typeof(rp->CompressReparseBuffer) *cmpr; + + ni = ni->base; + /* Try to estimate reparse point. */ + if (!attr->non_res) { + rp = resident_data_ex(attr, sizeof(struct REPARSE_DATA_BUFFER)); + } else if (le64_to_cpu(attr->nres.data_size) >= + sizeof(struct REPARSE_DATA_BUFFER)) { + struct runs_tree run; + + run_init(&run); + + if (!attr_load_runs_vcn(ni, ATTR_REPARSE, NULL, 0, &run, 0) && + !ntfs_read_run_nb(ni->mi.sbi, &run, 0, buffer, + sizeof(struct REPARSE_DATA_BUFFER), + NULL)) { + rp = buffer; + } + + run_close(&run); + } + + if (!rp) + return REPARSE_NONE; + + len = le16_to_cpu(rp->ReparseDataLength); + switch (rp->ReparseTag) { + case (IO_REPARSE_TAG_MICROSOFT | IO_REPARSE_TAG_SYMBOLIC_LINK): + break; /* Symbolic link. */ + case IO_REPARSE_TAG_MOUNT_POINT: + break; /* Mount points and junctions. */ + case IO_REPARSE_TAG_SYMLINK: + break; + case IO_REPARSE_TAG_COMPRESS: + /* + * WOF - Windows Overlay Filter - Used to compress files with + * LZX/Xpress. + * + * Unlike native NTFS file compression, the Windows + * Overlay Filter supports only read operations. This means + * that it doesn't need to sector-align each compressed chunk, + * so the compressed data can be packed more tightly together. + * If you open the file for writing, the WOF just decompresses + * the entire file, turning it back into a plain file. + * + * Ntfs3 driver decompresses the entire file only on write or + * change size requests. + */ + + cmpr = &rp->CompressReparseBuffer; + if (len < sizeof(*cmpr) || + cmpr->WofVersion != WOF_CURRENT_VERSION || + cmpr->WofProvider != WOF_PROVIDER_SYSTEM || + cmpr->ProviderVer != WOF_PROVIDER_CURRENT_VERSION) { + return REPARSE_NONE; + } + + switch (cmpr->CompressionFormat) { + case WOF_COMPRESSION_XPRESS4K: + bits = 0xc; // 4k + break; + case WOF_COMPRESSION_XPRESS8K: + bits = 0xd; // 8k + break; + case WOF_COMPRESSION_XPRESS16K: + bits = 0xe; // 16k + break; + case WOF_COMPRESSION_LZX32K: + bits = 0xf; // 32k + break; + default: + bits = 0x10; // 64k + break; + } + ni_set_ext_compress_bits(ni, bits); + return REPARSE_COMPRESSED; + + case IO_REPARSE_TAG_DEDUP: + ni->ni_flags |= NI_FLAG_DEDUPLICATED; + return REPARSE_DEDUPLICATED; + + default: + if (rp->ReparseTag & IO_REPARSE_TAG_NAME_SURROGATE) + break; + + return REPARSE_NONE; + } + + if (buffer != rp) + memcpy(buffer, rp, sizeof(struct REPARSE_DATA_BUFFER)); + + /* Looks like normal symlink. */ + return REPARSE_LINK; +} + +static struct folio *ntfs_lock_new_page(struct address_space *mapping, + pgoff_t index, gfp_t gfp) +{ + struct folio *folio = __filemap_get_folio( + mapping, index, FGP_LOCK | FGP_ACCESSED | FGP_CREAT, gfp); + + if (IS_ERR(folio)) + return folio; + + if (!folio_test_uptodate(folio)) { + struct page *page = folio_file_page(folio, index); + + if (IS_ERR(page)) + return ERR_CAST(page); + return page_folio(page); + } + + /* Use a temporary page to avoid data corruption */ + folio_unlock(folio); + folio_put(folio); + folio = folio_alloc(gfp, 0); + if (!folio) + return ERR_PTR(-ENOMEM); + __folio_set_locked(folio); + return folio; +} + +/* + * ni_read_folio_cmpr + * + * When decompressing, we typically obtain more than one page per reference. + * We inject the additional pages into the page cache. + */ +int ni_read_folio_cmpr(struct ntfs_inode *ni, struct folio *folio) +{ + int err; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct address_space *mapping = folio->mapping; + pgoff_t index; + u64 frame_vbo, vbo = folio_pos(folio); + struct page **pages = NULL; /* Array of at most 16 pages. stack? */ + u8 frame_bits; + CLST frame; + u32 i, idx, frame_size, pages_per_frame; + gfp_t gfp_mask; + struct page *pg; + struct folio *f; + + if (vbo >= i_size_read(&ni->vfs_inode)) { + folio_zero_range(folio, 0, folio_size(folio)); + folio_mark_uptodate(folio); + err = 0; + goto out; + } + + if (ni->ni_flags & NI_FLAG_COMPRESSED_MASK) { + /* Xpress or LZX. */ + frame_bits = ni_ext_compress_bits(ni); + } else { + /* LZNT compression. */ + frame_bits = NTFS_LZNT_CUNIT + sbi->cluster_bits; + } + frame_size = 1u << frame_bits; + frame = vbo >> frame_bits; + frame_vbo = (u64)frame << frame_bits; + idx = (vbo - frame_vbo) >> PAGE_SHIFT; + + pages_per_frame = frame_size >> PAGE_SHIFT; + pages = kzalloc_objs(struct page *, pages_per_frame, GFP_NOFS); + if (!pages) { + err = -ENOMEM; + goto out; + } + + pages[idx] = &folio->page; + index = frame_vbo >> PAGE_SHIFT; + gfp_mask = mapping_gfp_mask(mapping); + + for (i = 0; i < pages_per_frame; i++, index++) { + if (i == idx) + continue; + + f = ntfs_lock_new_page(mapping, index, gfp_mask); + if (IS_ERR(f)) { + err = PTR_ERR(f); + goto out1; + } + pages[i] = &f->page; + } + + ni_lock(ni); + err = ni_read_frame(ni, frame_vbo, pages, pages_per_frame, 0); + ni_unlock(ni); + +out1: + for (i = 0; i < pages_per_frame; i++) { + pg = pages[i]; + if (i == idx || !pg) + continue; + unlock_page(pg); + put_page(pg); + } + +out: + /* At this point, err contains 0 or -EIO depending on the "critical" page. */ + kfree(pages); + folio_unlock(folio); + + return err; +} + +#ifdef CONFIG_NTFS3_LZX_XPRESS +/* + * ni_decompress_file - Decompress LZX/Xpress compressed file. + * + * Remove ATTR_DATA::WofCompressedData. + * Remove ATTR_REPARSE. + */ +int ni_decompress_file(struct ntfs_inode *ni) +{ + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct inode *inode = &ni->vfs_inode; + loff_t i_size = i_size_read(inode); + struct address_space *mapping = inode->i_mapping; + gfp_t gfp_mask = mapping_gfp_mask(mapping); + struct page **pages = NULL; + struct ATTR_LIST_ENTRY *le; + struct ATTRIB *attr; + CLST vcn, cend, lcn, clen, end; + pgoff_t index; + u64 vbo; + u8 frame_bits; + u32 i, frame_size, pages_per_frame, bytes; + struct mft_inode *mi; + int err; + + ni = ni->base; + /* Clusters for decompressed data. */ + cend = bytes_to_cluster(sbi, i_size); + + if (!i_size) + goto remove_wof; + + /* Check in advance. */ + if (cend > wnd_zeroes(&sbi->used.bitmap)) { + err = -ENOSPC; + goto out; + } + + frame_bits = ni_ext_compress_bits(ni); + frame_size = 1u << frame_bits; + pages_per_frame = frame_size >> PAGE_SHIFT; + pages = kzalloc_objs(struct page *, pages_per_frame, GFP_NOFS); + if (!pages) { + err = -ENOMEM; + goto out; + } + + /* + * Step 1: Decompress data and copy to new allocated clusters. + */ + index = 0; + for (vbo = 0; vbo < i_size; vbo += bytes) { + bool new; + + bytes = vbo + frame_size > i_size ? (i_size - vbo) : frame_size; + end = bytes_to_cluster(sbi, vbo + bytes); + + for (vcn = vbo >> sbi->cluster_bits; vcn < end; vcn += clen) { + err = attr_data_get_block(ni, vcn, cend - vcn, &lcn, + &clen, &new, false, NULL, + false); + if (err) + goto out; + } + + for (i = 0; i < pages_per_frame; i++, index++) { + struct folio *f; + + f = ntfs_lock_new_page(mapping, index, gfp_mask); + if (IS_ERR(f)) { + while (i--) { + unlock_page(pages[i]); + put_page(pages[i]); + } + err = PTR_ERR(f); + goto out; + } + pages[i] = &f->page; + } + + err = ni_read_frame(ni, vbo, pages, pages_per_frame, 1); + + for (i = 0; i < pages_per_frame; i++) { + unlock_page(pages[i]); + put_page(pages[i]); + } + + if (err) + goto out; + + cond_resched(); + } + +remove_wof: + /* + * Step 2: Deallocate attributes ATTR_DATA::WofCompressedData + * and ATTR_REPARSE. + */ + attr = NULL; + le = NULL; + while ((attr = ni_enum_attr_ex(ni, attr, &le, NULL))) { + CLST svcn, evcn; + u32 asize, roff; + + if (attr->type == ATTR_REPARSE) { + struct MFT_REF ref; + + mi_get_ref(&ni->mi, &ref); + ntfs_remove_reparse(sbi, 0, &ref); + } + + if (!attr->non_res) + continue; + + if (attr->type != ATTR_REPARSE && + (attr->type != ATTR_DATA || + attr->name_len != ARRAY_SIZE(WOF_NAME) || + memcmp(attr_name(attr), WOF_NAME, sizeof(WOF_NAME)))) + continue; + + svcn = le64_to_cpu(attr->nres.svcn); + evcn = le64_to_cpu(attr->nres.evcn); + + if (evcn + 1 <= svcn) + continue; + + asize = le32_to_cpu(attr->size); + roff = le16_to_cpu(attr->nres.run_off); + + if (roff > asize) { + err = -EINVAL; + goto out; + } + + /*run==1 Means unpack and deallocate. */ + run_unpack_ex(RUN_DEALLOCATE, sbi, ni->mi.rno, svcn, evcn, svcn, + Add2Ptr(attr, roff), asize - roff); + } + + /* + * Step 3: Remove attribute ATTR_DATA::WofCompressedData. + */ + err = ni_remove_attr(ni, ATTR_DATA, WOF_NAME, ARRAY_SIZE(WOF_NAME), + false, NULL); + if (err) + goto out; + + /* + * Step 4: Remove ATTR_REPARSE. + */ + err = ni_remove_attr(ni, ATTR_REPARSE, NULL, 0, false, NULL); + if (err) + goto out; + + /* + * Step 5: Remove sparse flag from data attribute. + */ + attr = ni_find_attr(ni, NULL, NULL, ATTR_DATA, NULL, 0, NULL, &mi); + if (!attr) { + err = -EINVAL; + goto out; + } + + if (attr->non_res && is_attr_sparsed(attr)) { + /* Sparsed attribute header is 8 bytes bigger than normal. */ + struct MFT_REC *rec = mi->mrec; + u32 used = le32_to_cpu(rec->used); + u32 asize = le32_to_cpu(attr->size); + u16 roff = le16_to_cpu(attr->nres.run_off); + char *rbuf = Add2Ptr(attr, roff); + + memmove(rbuf - 8, rbuf, used - PtrOffset(rec, rbuf)); + attr->size = cpu_to_le32(asize - 8); + attr->flags &= ~ATTR_FLAG_SPARSED; + attr->nres.run_off = cpu_to_le16(roff - 8); + attr->nres.c_unit = 0; + rec->used = cpu_to_le32(used - 8); + mi->dirty = true; + ni->std_fa &= ~(FILE_ATTRIBUTE_SPARSE_FILE | + FILE_ATTRIBUTE_REPARSE_POINT); + + mark_inode_dirty(inode); + } + + /* Clear cached flag. */ + ni->ni_flags &= ~NI_FLAG_COMPRESSED_MASK; + if (ni->file.offs_folio) { + folio_put(ni->file.offs_folio); + ni->file.offs_folio = NULL; + } + mapping->a_ops = &ntfs_aops; + +out: + kfree(pages); + if (err) + _ntfs_bad_inode(inode); + + return err; +} + +/* + * decompress_lzx_xpress - External compression LZX/Xpress. + */ +static int decompress_lzx_xpress(struct ntfs_sb_info *sbi, const char *cmpr, + size_t cmpr_size, void *unc, size_t unc_size, + u32 frame_size) +{ + int err; + void *ctx; + + if (cmpr_size == unc_size) { + /* Frame not compressed. */ + memcpy(unc, cmpr, unc_size); + return 0; + } + + err = 0; + if (frame_size == 0x8000) { + mutex_lock(&sbi->compress.mtx_lzx); + /* LZX: Frame compressed. */ + ctx = sbi->compress.lzx; + if (!ctx) { + /* Lazy initialize LZX decompress context. */ + ctx = lzx_allocate_decompressor(); + if (!ctx) { + err = -ENOMEM; + goto out1; + } + + sbi->compress.lzx = ctx; + } + + if (lzx_decompress(ctx, cmpr, cmpr_size, unc, unc_size)) { + /* Treat all errors as "invalid argument". */ + err = -EINVAL; + } +out1: + mutex_unlock(&sbi->compress.mtx_lzx); + } else { + /* XPRESS: Frame compressed. */ + mutex_lock(&sbi->compress.mtx_xpress); + ctx = sbi->compress.xpress; + if (!ctx) { + /* Lazy initialize Xpress decompress context. */ + ctx = xpress_allocate_decompressor(); + if (!ctx) { + err = -ENOMEM; + goto out2; + } + + sbi->compress.xpress = ctx; + } + + if (xpress_decompress(ctx, cmpr, cmpr_size, unc, unc_size)) { + /* Treat all errors as "invalid argument". */ + err = -EINVAL; + } +out2: + mutex_unlock(&sbi->compress.mtx_xpress); + } + return err; +} +#endif + +/* + * ni_read_frame + * + * Pages - Array of locked pages. + */ +int ni_read_frame(struct ntfs_inode *ni, u64 frame_vbo, struct page **pages, + u32 pages_per_frame, int copy) +{ + int err; + struct ntfs_sb_info *sbi = ni->mi.sbi; + u8 cluster_bits = sbi->cluster_bits; + char *frame_ondisk = NULL; + char *frame_mem = NULL; + struct ATTR_LIST_ENTRY *le = NULL; + struct runs_tree *run = &ni->file.run; + u64 valid_size = ni->i_valid; + u64 vbo_disk; + size_t unc_size = 0; + u32 frame_size, i, ondisk_size; + struct page *pg; + struct ATTRIB *attr; + CLST frame, clst_data; + + ni = ni->base; + /* + * To simplify decompress algorithm do vmap for source + * and target pages. + */ + frame_size = pages_per_frame << PAGE_SHIFT; + frame_mem = vmap(pages, pages_per_frame, VM_MAP, PAGE_KERNEL); + if (!frame_mem) { + err = -ENOMEM; + goto out; + } + + attr = ni_find_attr(ni, NULL, &le, ATTR_DATA, NULL, 0, NULL, NULL); + if (!attr) { + err = -ENOENT; + goto out1; + } + + if (!attr->non_res) { + u32 data_size = le32_to_cpu(attr->res.data_size); + + memset(frame_mem, 0, frame_size); + if (frame_vbo < data_size) { + ondisk_size = data_size - frame_vbo; + memcpy(frame_mem, resident_data(attr) + frame_vbo, + min(ondisk_size, frame_size)); + } + err = 0; + goto out1; + } + + if (frame_vbo >= valid_size) { + memset(frame_mem, 0, frame_size); + err = 0; + goto out1; + } + + if (ni->ni_flags & NI_FLAG_COMPRESSED_MASK) { +#ifndef CONFIG_NTFS3_LZX_XPRESS + err = -EOPNOTSUPP; + goto out1; +#else + loff_t i_size = i_size_read(&ni->vfs_inode); + u32 frame_bits = ni_ext_compress_bits(ni); + u64 frame64 = frame_vbo >> frame_bits; + u64 frames, vbo_data; + + if (frame_size != (1u << frame_bits)) { + err = -EINVAL; + goto out1; + } + switch (frame_size) { + case 0x1000: + case 0x2000: + case 0x4000: + case 0x8000: + break; + default: + /* Unknown compression. */ + err = -EOPNOTSUPP; + goto out1; + } + + attr = ni_find_attr(ni, attr, &le, ATTR_DATA, WOF_NAME, + ARRAY_SIZE(WOF_NAME), NULL, NULL); + if (!attr) { + ntfs_inode_err( + &ni->vfs_inode, + "external compressed file should contains data attribute \"WofCompressedData\""); + err = -EINVAL; + goto out1; + } + + if (!attr->non_res) { + run = NULL; + } else { + run = run_alloc(); + if (!run) { + err = -ENOMEM; + goto out1; + } + } + + frames = (i_size - 1) >> frame_bits; + + err = attr_wof_frame_info(ni, attr, run, frame64, frames, + frame_bits, &ondisk_size, &vbo_data); + if (err) + goto out1; + + if (frame64 == frames) { + unc_size = 1 + ((i_size - 1) & (frame_size - 1)); + ondisk_size = attr_size(attr) - vbo_data; + } else { + unc_size = frame_size; + } + + if (ondisk_size > frame_size) { + err = -EINVAL; + goto out1; + } + + if (!attr->non_res) { + if (vbo_data + ondisk_size > + le32_to_cpu(attr->res.data_size)) { + err = -EINVAL; + goto out1; + } + + err = decompress_lzx_xpress( + sbi, Add2Ptr(resident_data(attr), vbo_data), + ondisk_size, frame_mem, unc_size, frame_size); + goto out1; + } + vbo_disk = vbo_data; + /* Load all runs to read [vbo_disk-vbo_to). */ + err = attr_load_runs_range(ni, ATTR_DATA, WOF_NAME, + ARRAY_SIZE(WOF_NAME), run, vbo_disk, + vbo_data + ondisk_size); + if (err) + goto out1; +#endif + } else if (is_attr_compressed(attr)) { + /* LZNT compression. */ + if (sbi->cluster_size > NTFS_LZNT_MAX_CLUSTER) { + err = -EOPNOTSUPP; + goto out1; + } + + if (attr->nres.c_unit != NTFS_LZNT_CUNIT) { + err = -EOPNOTSUPP; + goto out1; + } + + down_write(&ni->file.run_lock); + run_truncate_around(run, le64_to_cpu(attr->nres.svcn)); + frame = frame_vbo >> (cluster_bits + NTFS_LZNT_CUNIT); + err = attr_is_frame_compressed(ni, attr, frame, &clst_data, + run); + up_write(&ni->file.run_lock); + if (err) + goto out1; + + if (!clst_data) { + memset(frame_mem, 0, frame_size); + goto out1; + } + + frame_size = sbi->cluster_size << NTFS_LZNT_CUNIT; + ondisk_size = clst_data << cluster_bits; + + if (clst_data >= NTFS_LZNT_CLUSTERS) { + /* Frame is not compressed. */ + down_read(&ni->file.run_lock); + err = ntfs_read_run(sbi, run, frame_mem, frame_vbo, + ondisk_size); + up_read(&ni->file.run_lock); + goto out1; + } + vbo_disk = frame_vbo; + } else { + __builtin_unreachable(); + err = -EINVAL; + goto out1; + } + + /* Allocate memory to read compressed data to. */ + frame_ondisk = kvmalloc(ondisk_size, GFP_KERNEL); + if (!frame_ondisk) { + err = -ENOMEM; + goto out1; + } + + /* Read 'ondisk_size' bytes from disk. */ + down_read(&ni->file.run_lock); + err = ntfs_read_run(sbi, run, frame_ondisk, vbo_disk, ondisk_size); + up_read(&ni->file.run_lock); + if (err) + goto out2; + +#ifdef CONFIG_NTFS3_LZX_XPRESS + if (run != &ni->file.run) { + /* LZX or XPRESS */ + err = decompress_lzx_xpress(sbi, frame_ondisk, ondisk_size, + frame_mem, unc_size, frame_size); + } else +#endif + { + /* LZNT - Native NTFS compression. */ + unc_size = decompress_lznt(frame_ondisk, ondisk_size, frame_mem, + frame_size); + if ((ssize_t)unc_size < 0) + err = unc_size; + else if (!unc_size || unc_size > frame_size) + err = -EINVAL; + else if (unc_size < frame_size) { + /* + * Partial decompress: zero the [unc_size, frame_size) + * tail. decompress_lznt() leaves it untouched, so + * without this the freshly vmapped pages would expose + * uninitialized kernel memory to userspace. + */ + memset(frame_mem + unc_size, 0, frame_size - unc_size); + } + } + if (!err && valid_size < frame_vbo + frame_size) { + size_t ok = valid_size - frame_vbo; + + memset(frame_mem + ok, 0, frame_size - ok); + } + +out2: + kvfree(frame_ondisk); +out1: +#ifdef CONFIG_NTFS3_LZX_XPRESS + if (run != &ni->file.run) + run_free(run); + if (!err && copy) { + /* We are called from 'ni_decompress_file' */ + /* Copy decompressed LZX or XPRESS data into new place. */ + down_read(&ni->file.run_lock); + err = ntfs_write_run(sbi, &ni->file.run, frame_mem, frame_vbo, + frame_size); + up_read(&ni->file.run_lock); + } +#endif + vunmap(frame_mem); +out: + for (i = 0; i < pages_per_frame; i++) { + pg = pages[i]; + SetPageUptodate(pg); + } + + return err; +} + +/* + * ni_write_frame + * + * Pages - Array of locked pages. + */ +int ni_write_frame(struct ntfs_inode *ni, struct page **pages, + u32 pages_per_frame) +{ + int err; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct folio *folio = page_folio(pages[0]); + u8 frame_bits = NTFS_LZNT_CUNIT + sbi->cluster_bits; + u32 frame_size = sbi->cluster_size << NTFS_LZNT_CUNIT; + u64 frame_vbo = folio_pos(folio); + CLST frame = frame_vbo >> frame_bits; + char *frame_ondisk = NULL; + struct ATTR_LIST_ENTRY *le = NULL; + char *frame_mem; + struct ATTRIB *attr; + struct mft_inode *mi; + size_t compr_size, ondisk_size; + struct lznt *lznt; + + ni = ni->base; + attr = ni_find_attr(ni, NULL, &le, ATTR_DATA, NULL, 0, NULL, &mi); + if (!attr) { + err = -ENOENT; + goto out; + } + + if (WARN_ON(!is_attr_compressed(attr))) { + err = -EINVAL; + goto out; + } + + if (sbi->cluster_size > NTFS_LZNT_MAX_CLUSTER) { + err = -EOPNOTSUPP; + goto out; + } + + if (!attr->non_res) { + down_write(&ni->file.run_lock); + err = attr_make_nonresident(ni, attr, le, mi, + le32_to_cpu(attr->res.data_size), + &ni->file.run, &attr, pages[0]); + up_write(&ni->file.run_lock); + if (err) + goto out; + } + + if (attr->nres.c_unit != NTFS_LZNT_CUNIT) { + err = -EOPNOTSUPP; + goto out; + } + + /* Allocate memory to write compressed data to. */ + frame_ondisk = kvmalloc(frame_size, GFP_KERNEL); + if (!frame_ondisk) { + err = -ENOMEM; + goto out; + } + + /* Map in-memory frame for read-only. */ + frame_mem = vmap(pages, pages_per_frame, VM_MAP, PAGE_KERNEL_RO); + if (!frame_mem) { + err = -ENOMEM; + goto out1; + } + + mutex_lock(&sbi->compress.mtx_lznt); + lznt = NULL; + if (!sbi->compress.lznt) { + /* + * LZNT implements two levels of compression: + * 0 - Standard compression + * 1 - Best compression, requires a lot of cpu + * use mount option? + */ + lznt = get_lznt_ctx(0); + if (!lznt) { + mutex_unlock(&sbi->compress.mtx_lznt); + err = -ENOMEM; + goto out2; + } + + sbi->compress.lznt = lznt; + lznt = NULL; + } + + /* Compress: frame_mem -> frame_ondisk */ + compr_size = compress_lznt(frame_mem, frame_size, frame_ondisk, + frame_size, sbi->compress.lznt); + mutex_unlock(&sbi->compress.mtx_lznt); + kfree(lznt); + + if (compr_size + sbi->cluster_size > frame_size) { + /* Frame is not compressed. */ + compr_size = frame_size; + ondisk_size = frame_size; + } else if (compr_size) { + /* Frame is compressed. */ + ondisk_size = ntfs_up_cluster(sbi, compr_size); + memset(frame_ondisk + compr_size, 0, ondisk_size - compr_size); + } else { + /* Frame is sparsed. */ + ondisk_size = 0; + } + + down_write(&ni->file.run_lock); + run_truncate_around(&ni->file.run, le64_to_cpu(attr->nres.svcn)); + err = attr_allocate_frame(ni, frame, compr_size, ni->i_valid); + up_write(&ni->file.run_lock); + if (err) + goto out2; + + if (!ondisk_size) + goto out2; + + down_read(&ni->file.run_lock); + err = ntfs_write_run(sbi, &ni->file.run, + ondisk_size < frame_size ? frame_ondisk : + frame_mem, + frame_vbo, ondisk_size); + up_read(&ni->file.run_lock); + +out2: + vunmap(frame_mem); +out1: + kvfree(frame_ondisk); +out: + return err; +} + +/* + * ni_remove_name - Removes name 'de' from MFT and from directory. + * 'de2' and 'undo_step' are used to restore MFT/dir, if error occurs. + */ +int ni_remove_name(struct ntfs_inode *dir_ni, struct ntfs_inode *ni, + struct NTFS_DE *de, struct NTFS_DE **de2, int *undo_step) +{ + int err; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct ATTR_FILE_NAME *de_name = (struct ATTR_FILE_NAME *)(de + 1); + struct ATTR_FILE_NAME *fname; + struct ATTR_LIST_ENTRY *le; + struct mft_inode *mi; + u16 de_key_size = le16_to_cpu(de->key_size); + u8 name_type; + + ni = ni->base; + *undo_step = 0; + + /* Find name in record. */ + mi_get_ref(&dir_ni->mi, &de_name->home); + + fname = ni_fname_name(ni, (struct le_str *)&de_name->name_len, + &de_name->home, &mi, &le); + if (!fname) + return -ENOENT; + + memcpy(&de_name->dup, &fname->dup, sizeof(struct NTFS_DUP_INFO)); + name_type = paired_name(fname->type); + + /* Mark ntfs as dirty. It will be cleared at umount. */ + ntfs_set_state(sbi, NTFS_DIRTY_DIRTY); + + /* Step 1: Remove name from directory. */ + err = indx_delete_entry(&dir_ni->dir, dir_ni, fname, de_key_size, sbi); + if (err) + return err; + + /* Step 2: Remove name from MFT. */ + ni_remove_attr_le(ni, attr_from_name(fname), mi, le); + + *undo_step = 2; + + /* Get paired name. */ + fname = ni_fname_type(ni, name_type, &mi, &le); + if (fname) { + u16 de2_key_size = fname_full_size(fname); + + *de2 = Add2Ptr(de, 1024); + (*de2)->key_size = cpu_to_le16(de2_key_size); + + memcpy(*de2 + 1, fname, de2_key_size); + + /* Step 3: Remove paired name from directory. */ + err = indx_delete_entry(&dir_ni->dir, dir_ni, fname, + de2_key_size, sbi); + if (err) + return err; + + /* Step 4: Remove paired name from MFT. */ + ni_remove_attr_le(ni, attr_from_name(fname), mi, le); + + *undo_step = 4; + } + return 0; +} + +/* + * ni_remove_name_undo - Paired function for ni_remove_name. + * + * Return: True if ok + */ +bool ni_remove_name_undo(struct ntfs_inode *dir_ni, struct ntfs_inode *ni, + struct NTFS_DE *de, struct NTFS_DE *de2, int undo_step) +{ + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct ATTRIB *attr; + u16 de_key_size; + + ni = ni->base; + switch (undo_step) { + case 4: + de_key_size = le16_to_cpu(de2->key_size); + if (ni_insert_resident(ni, de_key_size, ATTR_NAME, NULL, 0, + &attr, NULL, NULL)) + return false; + memcpy(Add2Ptr(attr, SIZEOF_RESIDENT), de2 + 1, de_key_size); + + mi_get_ref(&ni->mi, &de2->ref); + de2->size = cpu_to_le16(ALIGN(de_key_size, 8) + + sizeof(struct NTFS_DE)); + de2->flags = 0; + de2->res = 0; + + if (indx_insert_entry(&dir_ni->dir, dir_ni, de2, sbi, NULL, 1)) + return false; + fallthrough; + + case 2: + de_key_size = le16_to_cpu(de->key_size); + + if (ni_insert_resident(ni, de_key_size, ATTR_NAME, NULL, 0, + &attr, NULL, NULL)) + return false; + + memcpy(Add2Ptr(attr, SIZEOF_RESIDENT), de + 1, de_key_size); + mi_get_ref(&ni->mi, &de->ref); + + if (indx_insert_entry(&dir_ni->dir, dir_ni, de, sbi, NULL, 1)) + return false; + } + + return true; +} + +/* + * ni_add_name - Add new name into MFT and into directory. + */ +int ni_add_name(struct ntfs_inode *dir_ni, struct ntfs_inode *ni, + struct NTFS_DE *de) +{ + int err; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct ATTRIB *attr; + struct ATTR_LIST_ENTRY *le; + struct mft_inode *mi; + struct ATTR_FILE_NAME *fname; + struct ATTR_FILE_NAME *de_name = (struct ATTR_FILE_NAME *)(de + 1); + u16 de_key_size = le16_to_cpu(de->key_size); + + ni = ni->base; + if (sbi->options->windows_names && + !valid_windows_name(sbi, (struct le_str *)&de_name->name_len)) + return -EINVAL; + + /* If option "hide_dot_files" then set hidden attribute for dot files. */ + if (ni->mi.sbi->options->hide_dot_files) { + if (de_name->name_len > 0 && + le16_to_cpu(de_name->name[0]) == '.') + ni->std_fa |= FILE_ATTRIBUTE_HIDDEN; + else + ni->std_fa &= ~FILE_ATTRIBUTE_HIDDEN; + } + + mi_get_ref(&ni->mi, &de->ref); + mi_get_ref(&dir_ni->mi, &de_name->home); + + /* Fill duplicate from any ATTR_NAME. */ + fname = ni_fname_name(ni, NULL, NULL, NULL, NULL); + if (fname) + memcpy(&de_name->dup, &fname->dup, sizeof(fname->dup)); + de_name->dup.fa = ni->std_fa; + + /* Insert new name into MFT. */ + err = ni_insert_resident(ni, de_key_size, ATTR_NAME, NULL, 0, &attr, + &mi, &le); + if (err) + return err; + + memcpy(Add2Ptr(attr, SIZEOF_RESIDENT), de_name, de_key_size); + + /* Insert new name into directory. */ + err = indx_insert_entry(&dir_ni->dir, dir_ni, de, sbi, NULL, 0); + if (err) + ni_remove_attr_le(ni, attr, mi, le); + + return err; +} + +/* + * ni_rename - Remove one name and insert new name. + */ +int ni_rename(struct ntfs_inode *dir_ni, struct ntfs_inode *new_dir_ni, + struct ntfs_inode *ni, struct NTFS_DE *de, struct NTFS_DE *new_de) +{ + int err; + struct NTFS_DE *de2 = NULL; + int undo = 0; + const int way = 1; /* Hope compiler removes below 'else'. */ + + /* + * There are two possible ways to rename: + * 1) Add new name and remove old name. + * 2) Remove old name and add new name. + * + * In most cases (not all!) adding new name into MFT and into directory can + * allocate additional cluster(s). + * Second way may result to bad inode if we can't add new name + * and then can't restore (add) old name. + */ + if (way == 1) { + /* + * Way 1 - Add new + remove old. + */ + err = ni_add_name(new_dir_ni, ni, new_de); + if (!err) { + err = ni_remove_name(dir_ni, ni, de, &de2, &undo); + if (err && + ni_remove_name(new_dir_ni, ni, new_de, &de2, &undo)) + _ntfs_bad_inode(&ni->vfs_inode); + } + } else { + /* + * Way 2 - Remove old + add new. + */ + err = ni_remove_name(dir_ni, ni, de, &de2, &undo); + if (!err) { + err = ni_add_name(new_dir_ni, ni, new_de); + if (err && + !ni_remove_name_undo(dir_ni, ni, de, de2, undo)) + _ntfs_bad_inode(&ni->vfs_inode); + } + } + + return err; +} + +/* + * ni_is_dirty - Return: True if 'ni' requires ni_write_inode. + */ +bool ni_is_dirty(struct inode *inode) +{ + struct ntfs_inode *ni = ntfs_i(inode); + struct rb_node *node; + + if (ni->mi.dirty || ni->attr_list.dirty || + (ni->ni_flags & NI_FLAG_UPDATE_PARENT)) + return true; + + for (node = rb_first(&ni->mi_tree); node; node = rb_next(node)) { + if (rb_entry(node, struct mft_inode, node)->dirty) + return true; + } + + return false; +} + +/* + * ni_seek_data_or_hole + * + * Helper function for ntfs_llseek( SEEK_DATA/SEEK_HOLE ) + */ +loff_t ni_seek_data_or_hole(struct ntfs_inode *ni, loff_t offset, bool data) +{ + int err; + u8 cluster_bits = ni->mi.sbi->cluster_bits; + CLST vcn, lcn, clen; + loff_t vbo; + + /* Enumerate all fragments. */ + for (vcn = offset >> cluster_bits;; vcn += clen) { + err = attr_data_get_block(ni, vcn, 1, &lcn, &clen, NULL, false, + NULL, false); + if (err) { + return err; + } + + if (!clen) { + /* Corrupted file. */ + return -EINVAL; + } + + if (lcn == RESIDENT_LCN) { + /* clen - resident size in bytes. clen == ni->vfs_inode.i_size */ + if (offset >= clen) { + /* check eof. */ + return -ENXIO; + } + + if (data) { + return offset; + } + + return clen; + } + + if (lcn == EOF_LCN) { + if (data) { + return -ENXIO; + } + + /* implicit hole at the end of file. */ + return ni->vfs_inode.i_size; + } + + if (data) { + /* + * Adjust the file offset to the next location in the file greater than + * or equal to offset containing data. If offset points to data, then + * the file offset is set to offset. + */ + if (lcn != SPARSE_LCN) { + /* Normal cluster. */ + break; + } + + if ((ni->std_fa & FILE_ATTRIBUTE_COMPRESSED) && + (vcn & (NTFS_LZNT_CLUSTERS - 1))) { + /* Compressed cluster in compressed frame. */ + break; + } + } else { + /* + * Adjust the file offset to the next hole in the file greater than or + * equal to offset. If offset points into the middle of a hole, then the + * file offset is set to offset. If there is no hole past offset, then the + * file offset is adjusted to the end of the file + * (i.e., there is an implicit hole at the end of any file). + */ + if (lcn == SPARSE_LCN && + /* native compression hole begins at aligned vcn. */ + (!(ni->std_fa & FILE_ATTRIBUTE_COMPRESSED) || + !(vcn & (NTFS_LZNT_CLUSTERS - 1)))) { + /* Hole in sparsed or compressed file frame. */ + break; + } + } + } + + vbo = (u64)vcn << cluster_bits; + return max(vbo, offset); +} + +/* + * ni_write_parents + * + * Helper function for ntfs_file_fsync. + */ +int ni_write_parents(struct ntfs_inode *ni, int sync) +{ + int err = 0; + struct ATTRIB *attr = NULL; + struct ATTR_LIST_ENTRY *le = NULL; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct super_block *sb = sbi->sb; + + if (!is_ni_base(ni)) + return 0; + + while ((attr = ni_find_attr(ni, attr, &le, ATTR_NAME, NULL, 0, NULL, + NULL))) { + struct inode *dir; + struct ATTR_FILE_NAME *fname; + + fname = resident_data_ex(attr, SIZEOF_ATTRIBUTE_FILENAME); + if (!fname) + continue; + + /* Check simple case when parent inode equals current inode. */ + if (ino_get(&fname->home) == ni->vfs_inode.i_ino) { + if (MFT_REC_ROOT != ni->vfs_inode.i_ino) { + ntfs_set_state(sbi, NTFS_DIRTY_ERROR); + err = -EINVAL; + } + continue; + } + + dir = ntfs_iget5(sb, &fname->home, NULL); + if (IS_ERR(dir)) { + ntfs_inode_warn( + &ni->vfs_inode, + "failed to open parent directory r=%llx to write", + (u64)ino_get(&fname->home)); + continue; + } + + if (!is_bad_inode(dir)) { + int err2 = write_inode_now(dir, sync); + if (!err) + err = err2; + } + iput(dir); + } + + return err; +} + +/* + * ni_update_parent + * + * Update duplicate info of ATTR_FILE_NAME in MFT and in parent directories. + */ +static bool ni_update_parent(struct ntfs_inode *ni, struct NTFS_DUP_INFO *dup, + int sync) +{ + struct ATTRIB *attr; + struct mft_inode *mi; + struct ATTR_LIST_ENTRY *le = NULL; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct super_block *sb = sbi->sb; + bool re_dirty = false; + + if (ni->mi.mrec->flags & RECORD_FLAG_DIR) { + dup->fa |= FILE_ATTRIBUTE_DIRECTORY; + attr = NULL; + dup->alloc_size = 0; + dup->data_size = 0; + } else { + dup->fa &= ~FILE_ATTRIBUTE_DIRECTORY; + + attr = ni_find_attr(ni, NULL, &le, ATTR_DATA, NULL, 0, NULL, + &mi); + if (!attr) { + dup->alloc_size = dup->data_size = 0; + } else if (!attr->non_res) { + u32 data_size = le32_to_cpu(attr->res.data_size); + + dup->alloc_size = cpu_to_le64(ALIGN(data_size, 8)); + dup->data_size = cpu_to_le64(data_size); + } else { + u64 new_valid = ni->i_valid; + u64 data_size = le64_to_cpu(attr->nres.data_size); + __le64 valid_le; + + dup->alloc_size = is_attr_ext(attr) ? + attr->nres.total_size : + attr->nres.alloc_size; + dup->data_size = attr->nres.data_size; + + if (new_valid > data_size) + new_valid = data_size; + + valid_le = cpu_to_le64(new_valid); + if (valid_le != attr->nres.valid_size) { + attr->nres.valid_size = valid_le; + mi->dirty = true; + } + } + } + + dup->extend_data = 0; + + if (dup->fa & FILE_ATTRIBUTE_REPARSE_POINT) { + attr = ni_find_attr(ni, NULL, NULL, ATTR_REPARSE, NULL, 0, NULL, + NULL); + + if (attr) { + const struct REPARSE_POINT *rp; + + rp = resident_data_ex(attr, + sizeof(struct REPARSE_POINT)); + /* If ATTR_REPARSE exists 'rp' can't be NULL. */ + if (rp) + dup->extend_data = rp->ReparseTag; + } + } else if (ni->ni_flags & NI_FLAG_EA) { + attr = ni_find_attr(ni, attr, &le, ATTR_EA_INFO, NULL, 0, NULL, + NULL); + if (attr) { + const struct EA_INFO *info; + + info = resident_data_ex(attr, sizeof(struct EA_INFO)); + /* If ATTR_EA_INFO exists 'info' can't be NULL. */ + if (info) + dup->extend_data = info->size; + } + } + + attr = NULL; + le = NULL; + + while ((attr = ni_find_attr(ni, attr, &le, ATTR_NAME, NULL, 0, NULL, + &mi))) { + struct inode *dir; + struct ATTR_FILE_NAME *fname; + + fname = resident_data_ex(attr, SIZEOF_ATTRIBUTE_FILENAME); + if (!fname || !memcmp(&fname->dup, dup, sizeof(fname->dup))) + continue; + + /* Check simple case when parent inode equals current inode. */ + if (ino_get(&fname->home) == ni->vfs_inode.i_ino) { + ntfs_set_state(sbi, NTFS_DIRTY_ERROR); + continue; + } + + /* ntfs_iget5 may sleep. */ + dir = ntfs_iget5(sb, &fname->home, NULL); + if (IS_ERR(dir)) { + ntfs_inode_warn( + &ni->vfs_inode, + "failed to open parent directory r=%llx to update", + (u64)ino_get(&fname->home)); + continue; + } + + if (!is_bad_inode(dir)) { + struct ntfs_inode *dir_ni = ntfs_i(dir); + + if (!ni_trylock(dir_ni)) { + re_dirty = true; + } else { + indx_update_dup(dir_ni, sbi, fname, dup, sync); + ni_unlock(dir_ni); + memcpy(&fname->dup, dup, sizeof(fname->dup)); + mi->dirty = true; + } + } + iput(dir); + } + + return re_dirty; +} + +/* + * ni_write_inode - Write MFT base record and all subrecords to disk. + */ +int ni_write_inode(struct inode *inode, int sync, const char *hint) +{ + int err = 0, err2; + struct ntfs_inode *ni = ntfs_i(inode); + struct super_block *sb = inode->i_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + bool re_dirty = false; + struct ATTR_STD_INFO *std; + struct rb_node *node, *next; + struct NTFS_DUP_INFO dup; + + if (is_bad_inode(inode) || sb_rdonly(sb)) + return 0; + + if (!is_ni_base(ni)) + return 0; + + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(ni))) + return -EINVAL; + + if (unlikely(ntfs3_forced_shutdown(sb))) + return -EIO; + + if (!ni_trylock(ni)) { + /* 'ni' is under modification, skip for now. */ + mark_inode_dirty_sync(inode); + return 0; + } + + if (!ni->mi.mrec) + goto out; + + if (is_rec_inuse(ni->mi.mrec) && + !(sbi->flags & NTFS_FLAGS_LOG_REPLAYING) && inode->i_nlink) { + bool modified = false; + struct timespec64 ts; + + /* Update times in standard attribute. */ + std = ni_std(ni); + if (!std) { + err = -EINVAL; + goto out; + } + + /* Update the access times if they have changed. */ + ts = inode_get_mtime(inode); + dup.m_time = kernel2nt(&ts); + if (std->m_time != dup.m_time) { + std->m_time = dup.m_time; + modified = true; + } + + ts = inode_get_ctime(inode); + dup.c_time = kernel2nt(&ts); + if (std->c_time != dup.c_time) { + std->c_time = dup.c_time; + modified = true; + } + + ts = inode_get_atime(inode); + dup.a_time = kernel2nt(&ts); + if (std->a_time != dup.a_time) { + std->a_time = dup.a_time; + modified = true; + } + + dup.fa = ni->std_fa; + if (std->fa != dup.fa) { + std->fa = dup.fa; + modified = true; + } + + /* std attribute is always in primary MFT record. */ + if (modified) + ni->mi.dirty = true; + + if (!ntfs_is_meta_file(sbi, inode->i_ino) && + (modified || (ni->ni_flags & NI_FLAG_UPDATE_PARENT)) + /* Avoid __wait_on_freeing_inode(inode). */ + && (sb->s_flags & SB_ACTIVE)) { + dup.cr_time = std->cr_time; + /* Not critical if this function fail. */ + re_dirty = ni_update_parent(ni, &dup, sync); + + if (re_dirty) + ni->ni_flags |= NI_FLAG_UPDATE_PARENT; + else + ni->ni_flags &= ~NI_FLAG_UPDATE_PARENT; + } + + /* Update attribute list. */ + if (ni->attr_list.size && ni->attr_list.dirty) { + if (inode->i_ino != MFT_REC_MFT || sync) { + err = ni_try_remove_attr_list(ni); + if (err) + goto out; + } + + err = al_update(ni, sync); + if (err) + goto out; + } + } + + for (node = rb_first(&ni->mi_tree); node; node = next) { + struct mft_inode *mi = rb_entry(node, struct mft_inode, node); + bool is_empty; + + next = rb_next(node); + + if (!mi->dirty) + continue; + + is_empty = !mi_enum_attr(ni, mi, NULL); + + if (is_empty) + clear_rec_inuse(mi->mrec); + + err2 = mi_write(mi, sync); + if (!err && err2) + err = err2; + + if (is_empty) { + ntfs_mark_rec_free(sbi, mi->rno, false); + rb_erase(node, &ni->mi_tree); + mi_put(mi); + } + } + + if (ni->mi.dirty) { + err2 = mi_write(&ni->mi, sync); + if (!err && err2) + err = err2; + } +out: + ni_unlock(ni); + + if (err) { + ntfs_inode_err(inode, "%s failed, %d.", hint, err); + ntfs_set_state(sbi, NTFS_DIRTY_ERROR); + return err; + } + + if (re_dirty) + mark_inode_dirty_sync(inode); + + return 0; +} + +/* + * Force to allocate all delay allocated clusters. + */ +int ni_allocate_da_blocks(struct ntfs_inode *ni) +{ + int err; + + ni_lock(ni); + down_write(&ni->file.run_lock); + + err = ni_allocate_da_blocks_locked(ni); + + up_write(&ni->file.run_lock); + ni_unlock(ni); + + return err; +} + +/* + * Force to allocate all delay allocated clusters. + */ +int ni_allocate_da_blocks_locked(struct ntfs_inode *ni) +{ + int err = 0; + + if (!ni->file.run_da.count) + return 0; + + if (is_sparsed(ni)) { + CLST vcn, lcn, clen, alen; + bool new; + + /* + * Sparse file allocates clusters in 'attr_data_get_block_locked' + */ + while (run_get_entry(&ni->file.run_da, 0, &vcn, &lcn, &clen)) { + /* TODO: zero=true? */ + err = attr_data_get_block_locked(ni, vcn, clen, &lcn, + &alen, &new, true, + NULL, true); + if (err) + break; + if (!new) { + err = -EINVAL; + break; + } + } + } else { + /* + * Normal file allocates clusters in 'attr_set_size' + */ + err = attr_set_size_ex(ni, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, &ni->file.run, + ni->vfs_inode.i_size, &ni->i_valid, + false, NULL, true); + } + + return err; +} + +/* + * Helper function to read ADS. + * bytes = iov_iter_count(iter) is not 0. Checked by caller; + */ +ssize_t ni_query_ads(struct ntfs_inode *ni, loff_t *pos, struct iov_iter *iter) +{ + ssize_t ret = 0; + struct ntfs_sb_info *sbi = ni->mi.sbi; + size_t bytes = iov_iter_count(iter); + loff_t end = *pos + bytes; + char *buf = NULL; + struct ATTR_LIST_ENTRY *le = NULL; + u64 vbo = 0; + struct ATTRIB *attr; + size_t done, used; + int err; + + /* + * Enumerate ADS. + */ + ni = ni->base; + attr = NULL; + while ((attr = ni_enum_attr_ex(ni, attr, &le, NULL))) { + if (attr->type != ATTR_DATA || !attr->name_len) + continue; + + if (!buf) { + buf = kmalloc(PAGE_SIZE, GFP_NOFS); + if (!buf) + return -ENOMEM; + } + + /* attr - named DATA attribute (ADS). */ + err = ntfs_utf16_to_nls(sbi, attr_name(attr), attr->name_len, + buf, PAGE_SIZE); + if (err < 0) { + ret = err; + break; + } + + /* + * err is the length of ADS name in bytes. + * Copy pseudo data interval [vbo, err + 1). + * into 'iter': [*pos, bytes) + */ + /* Add \n as streams name separator. */ + buf[err++] = '\n'; + + if (vbo >= end) { + iov_iter_zero(bytes, iter); + break; + } + + if (vbo + err > *pos) { + size_t off = *pos - vbo; + used = err - off; + done = copy_to_iter(Add2Ptr(buf, off), min(used, bytes), + iter); + } else { + done = 0; + } + + ret += done; + *pos += done; + if (done >= bytes) + break; + bytes -= done; + vbo += err; + } + kfree(buf); + return ret; +} diff --git a/fs/ntfs3/fslog.c b/fs/ntfs3/fslog.c new file mode 100644 index 000000000..ed50c1d0c --- /dev/null +++ b/fs/ntfs3/fslog.c @@ -0,0 +1,5408 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + */ + +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +/* + * LOG FILE structs + */ + +// clang-format off + +#define MaxLogFileSize 0x100000000ull +#define DefaultLogPageSize 4096 +#define MinLogRecordPages 0x30 + +struct RESTART_HDR { + struct NTFS_RECORD_HEADER rhdr; // 'RSTR' + __le32 sys_page_size; // 0x10: Page size of the system which initialized the log. + __le32 page_size; // 0x14: Log page size used for this log file. + __le16 ra_off; // 0x18: + __le16 minor_ver; // 0x1A: + __le16 major_ver; // 0x1C: + __le16 fixups[]; +}; + +#define LFS_NO_CLIENT 0xffff +#define LFS_NO_CLIENT_LE cpu_to_le16(0xffff) + +struct CLIENT_REC { + __le64 oldest_lsn; + __le64 restart_lsn; // 0x08: + __le16 prev_client; // 0x10: + __le16 next_client; // 0x12: + __le16 seq_num; // 0x14: + u8 align[6]; // 0x16: + __le32 name_bytes; // 0x1C: In bytes. + __le16 name[64]; // 0x20: Name of client. +}; + +static_assert(sizeof(struct CLIENT_REC) == 0xa0); + +/* Two copies of these will exist at the beginning of the log file */ +struct RESTART_AREA { + __le64 current_lsn; // 0x00: Current logical end of log file. + __le16 log_clients; // 0x08: Maximum number of clients. + __le16 client_idx[2]; // 0x0A: Free/use index into the client record arrays. + __le16 flags; // 0x0E: See RESTART_SINGLE_PAGE_IO. + __le32 seq_num_bits; // 0x10: The number of bits in sequence number. + __le16 ra_len; // 0x14: + __le16 client_off; // 0x16: + __le64 l_size; // 0x18: Usable log file size. + __le32 last_lsn_data_len; // 0x20: + __le16 rec_hdr_len; // 0x24: Log page data offset. + __le16 data_off; // 0x26: Log page data length. + __le32 open_log_count; // 0x28: + __le32 align[5]; // 0x2C: + struct CLIENT_REC clients[]; // 0x40: +}; + +struct LOG_REC_HDR { + __le16 redo_op; // 0x00: NTFS_LOG_OPERATION + __le16 undo_op; // 0x02: NTFS_LOG_OPERATION + __le16 redo_off; // 0x04: Offset to Redo record. + __le16 redo_len; // 0x06: Redo length. + __le16 undo_off; // 0x08: Offset to Undo record. + __le16 undo_len; // 0x0A: Undo length. + __le16 target_attr; // 0x0C: + __le16 lcns_follow; // 0x0E: + __le16 record_off; // 0x10: + __le16 attr_off; // 0x12: + __le16 cluster_off; // 0x14: + __le16 reserved; // 0x16: + __le64 target_vcn; // 0x18: + __le64 page_lcns[]; // 0x20: +}; + +static_assert(sizeof(struct LOG_REC_HDR) == 0x20); + +#define RESTART_ENTRY_ALLOCATED 0xFFFFFFFF +#define RESTART_ENTRY_ALLOCATED_LE cpu_to_le32(0xFFFFFFFF) + +struct RESTART_TABLE { + __le16 size; // 0x00: In bytes + __le16 used; // 0x02: Entries + __le16 total; // 0x04: Entries + __le16 res[3]; // 0x06: + __le32 free_goal; // 0x0C: + __le32 first_free; // 0x10: + __le32 last_free; // 0x14: + +}; + +static_assert(sizeof(struct RESTART_TABLE) == 0x18); + +struct ATTR_NAME_ENTRY { + __le16 off; // Offset in the Open attribute Table. + __le16 name_bytes; + __le16 name[]; +}; + +struct OPEN_ATTR_ENRTY { + __le32 next; // 0x00: RESTART_ENTRY_ALLOCATED if allocated + __le32 bytes_per_index; // 0x04: + enum ATTR_TYPE type; // 0x08: + u8 is_dirty_pages; // 0x0C: + u8 is_attr_name; // 0x0B: Faked field to manage 'ptr' + u8 name_len; // 0x0C: Faked field to manage 'ptr' + u8 res; + struct MFT_REF ref; // 0x10: File Reference of file containing attribute + __le64 open_record_lsn; // 0x18: + void *ptr; // 0x20: +}; + +/* 32 bit version of 'struct OPEN_ATTR_ENRTY' */ +struct OPEN_ATTR_ENRTY_32 { + __le32 next; // 0x00: RESTART_ENTRY_ALLOCATED if allocated + __le32 ptr; // 0x04: + struct MFT_REF ref; // 0x08: + __le64 open_record_lsn; // 0x10: + u8 is_dirty_pages; // 0x18: + u8 is_attr_name; // 0x19: + u8 res1[2]; + enum ATTR_TYPE type; // 0x1C: + u8 name_len; // 0x20: In wchar + u8 res2[3]; + __le32 AttributeName; // 0x24: + __le32 bytes_per_index; // 0x28: +}; + +#define SIZEOF_OPENATTRIBUTEENTRY0 0x2c +// static_assert( 0x2C == sizeof(struct OPEN_ATTR_ENRTY_32) ); +static_assert(sizeof(struct OPEN_ATTR_ENRTY) < SIZEOF_OPENATTRIBUTEENTRY0); + +/* + * One entry exists in the Dirty Pages Table for each page which is dirty at + * the time the Restart Area is written. + */ +struct DIR_PAGE_ENTRY { + __le32 next; // 0x00: RESTART_ENTRY_ALLOCATED if allocated + __le32 target_attr; // 0x04: Index into the Open attribute Table + __le32 transfer_len; // 0x08: + __le32 lcns_follow; // 0x0C: + __le64 vcn; // 0x10: Vcn of dirty page + __le64 oldest_lsn; // 0x18: + __le64 page_lcns[]; // 0x20: +}; + +static_assert(sizeof(struct DIR_PAGE_ENTRY) == 0x20); + +/* 32 bit version of 'struct DIR_PAGE_ENTRY' */ +struct DIR_PAGE_ENTRY_32 { + __le32 next; // 0x00: RESTART_ENTRY_ALLOCATED if allocated + __le32 target_attr; // 0x04: Index into the Open attribute Table + __le32 transfer_len; // 0x08: + __le32 lcns_follow; // 0x0C: + __le32 reserved; // 0x10: + __le32 vcn_low; // 0x14: Vcn of dirty page + __le32 vcn_hi; // 0x18: Vcn of dirty page + __le32 oldest_lsn_low; // 0x1C: + __le32 oldest_lsn_hi; // 0x1C: + __le32 page_lcns_low; // 0x24: + __le32 page_lcns_hi; // 0x24: +}; + +static_assert(offsetof(struct DIR_PAGE_ENTRY_32, vcn_low) == 0x14); +static_assert(sizeof(struct DIR_PAGE_ENTRY_32) == 0x2c); + +enum transact_state { + TransactionUninitialized = 0, + TransactionActive, + TransactionPrepared, + TransactionCommitted +}; + +struct TRANSACTION_ENTRY { + __le32 next; // 0x00: RESTART_ENTRY_ALLOCATED if allocated + u8 transact_state; // 0x04: + u8 reserved[3]; // 0x05: + __le64 first_lsn; // 0x08: + __le64 prev_lsn; // 0x10: + __le64 undo_next_lsn; // 0x18: + __le32 undo_records; // 0x20: Number of undo log records pending abort + __le32 undo_len; // 0x24: Total undo size +}; + +static_assert(sizeof(struct TRANSACTION_ENTRY) == 0x28); + +struct NTFS_RESTART { + __le32 major_ver; // 0x00: + __le32 minor_ver; // 0x04: + __le64 check_point_start; // 0x08: + __le64 open_attr_table_lsn; // 0x10: + __le64 attr_names_lsn; // 0x18: + __le64 dirty_pages_table_lsn; // 0x20: + __le64 transact_table_lsn; // 0x28: + __le32 open_attr_len; // 0x30: In bytes + __le32 attr_names_len; // 0x34: In bytes + __le32 dirty_pages_len; // 0x38: In bytes + __le32 transact_table_len; // 0x3C: In bytes +}; + +static_assert(sizeof(struct NTFS_RESTART) == 0x40); + +struct NEW_ATTRIBUTE_SIZES { + __le64 alloc_size; + __le64 valid_size; + __le64 data_size; + __le64 total_size; +}; + +struct BITMAP_RANGE { + __le32 bitmap_off; + __le32 bits; +}; + +struct LCN_RANGE { + __le64 lcn; + __le64 len; +}; + +/* The following type defines the different log record types. */ +#define LfsClientRecord cpu_to_le32(1) +#define LfsClientRestart cpu_to_le32(2) + +/* This is used to uniquely identify a client for a particular log file. */ +struct CLIENT_ID { + __le16 seq_num; + __le16 client_idx; +}; + +/* This is the header that begins every Log Record in the log file. */ +struct LFS_RECORD_HDR { + __le64 this_lsn; // 0x00: + __le64 client_prev_lsn; // 0x08: + __le64 client_undo_next_lsn; // 0x10: + __le32 client_data_len; // 0x18: + struct CLIENT_ID client; // 0x1C: Owner of this log record. + __le32 record_type; // 0x20: LfsClientRecord or LfsClientRestart. + __le32 transact_id; // 0x24: + __le16 flags; // 0x28: LOG_RECORD_MULTI_PAGE + u8 align[6]; // 0x2A: +}; + +#define LOG_RECORD_MULTI_PAGE cpu_to_le16(1) + +static_assert(sizeof(struct LFS_RECORD_HDR) == 0x30); + +struct LFS_RECORD { + __le16 next_record_off; // 0x00: Offset of the free space in the page, + u8 align[6]; // 0x02: + __le64 last_end_lsn; // 0x08: lsn for the last log record which ends on the page, +}; + +static_assert(sizeof(struct LFS_RECORD) == 0x10); + +struct RECORD_PAGE_HDR { + struct NTFS_RECORD_HEADER rhdr; // 'RCRD' + __le32 rflags; // 0x10: See LOG_PAGE_LOG_RECORD_END + __le16 page_count; // 0x14: + __le16 page_pos; // 0x16: + struct LFS_RECORD record_hdr; // 0x18: + __le16 fixups[10]; // 0x28: + __le32 file_off; // 0x3c: Used when major version >= 2 +}; + +// clang-format on + +// Page contains the end of a log record. +#define LOG_PAGE_LOG_RECORD_END cpu_to_le32(0x00000001) + +static inline bool is_log_record_end(const struct RECORD_PAGE_HDR *hdr) +{ + return hdr->rflags & LOG_PAGE_LOG_RECORD_END; +} + +static_assert(offsetof(struct RECORD_PAGE_HDR, file_off) == 0x3c); + +/* + * END of NTFS LOG structures + */ + +/* Define some tuning parameters to keep the restart tables a reasonable size. */ +#define INITIAL_NUMBER_TRANSACTIONS 5 + +enum NTFS_LOG_OPERATION { + + Noop = 0x00, + CompensationLogRecord = 0x01, + InitializeFileRecordSegment = 0x02, + DeallocateFileRecordSegment = 0x03, + WriteEndOfFileRecordSegment = 0x04, + CreateAttribute = 0x05, + DeleteAttribute = 0x06, + UpdateResidentValue = 0x07, + UpdateNonresidentValue = 0x08, + UpdateMappingPairs = 0x09, + DeleteDirtyClusters = 0x0A, + SetNewAttributeSizes = 0x0B, + AddIndexEntryRoot = 0x0C, + DeleteIndexEntryRoot = 0x0D, + AddIndexEntryAllocation = 0x0E, + DeleteIndexEntryAllocation = 0x0F, + WriteEndOfIndexBuffer = 0x10, + SetIndexEntryVcnRoot = 0x11, + SetIndexEntryVcnAllocation = 0x12, + UpdateFileNameRoot = 0x13, + UpdateFileNameAllocation = 0x14, + SetBitsInNonresidentBitMap = 0x15, + ClearBitsInNonresidentBitMap = 0x16, + HotFix = 0x17, + EndTopLevelAction = 0x18, + PrepareTransaction = 0x19, + CommitTransaction = 0x1A, + ForgetTransaction = 0x1B, + OpenNonresidentAttribute = 0x1C, + OpenAttributeTableDump = 0x1D, + AttributeNamesDump = 0x1E, + DirtyPageTableDump = 0x1F, + TransactionTableDump = 0x20, + UpdateRecordDataRoot = 0x21, + UpdateRecordDataAllocation = 0x22, + + UpdateRelativeDataInIndex = + 0x23, // NtOfsRestartUpdateRelativeDataInIndex + UpdateRelativeDataInIndex2 = 0x24, + ZeroEndOfFileRecord = 0x25, +}; + +/* + * Array for log records which require a target attribute. + * A true indicates that the corresponding restart operation + * requires a target attribute. + */ +static const u8 AttributeRequired[] = { + 0xFC, 0xFB, 0xFF, 0x10, 0x06, +}; + +static inline bool is_target_required(u16 op) +{ + bool ret = op <= UpdateRecordDataAllocation && + (AttributeRequired[op >> 3] >> (op & 7) & 1); + return ret; +} + +static inline bool can_skip_action(enum NTFS_LOG_OPERATION op) +{ + switch (op) { + case Noop: + case DeleteDirtyClusters: + case HotFix: + case EndTopLevelAction: + case PrepareTransaction: + case CommitTransaction: + case ForgetTransaction: + case CompensationLogRecord: + case OpenNonresidentAttribute: + case OpenAttributeTableDump: + case AttributeNamesDump: + case DirtyPageTableDump: + case TransactionTableDump: + return true; + default: + return false; + } +} + +enum { lcb_ctx_undo_next, lcb_ctx_prev, lcb_ctx_next }; + +/* Bytes per restart table. */ +static inline u32 bytes_per_rt(const struct RESTART_TABLE *rt) +{ + return le16_to_cpu(rt->used) * le16_to_cpu(rt->size) + + sizeof(struct RESTART_TABLE); +} + +/* Log record length. */ +static inline u32 lrh_length(const struct LOG_REC_HDR *lr) +{ + u16 t16 = le16_to_cpu(lr->lcns_follow); + + return struct_size(lr, page_lcns, max_t(u16, 1, t16)); +} + +struct lcb { + struct LFS_RECORD_HDR *lrh; // Log record header of the current lsn. + struct LOG_REC_HDR *log_rec; + u32 ctx_mode; // lcb_ctx_undo_next/lcb_ctx_prev/lcb_ctx_next + struct CLIENT_ID client; + bool alloc; // If true the we should deallocate 'log_rec'. +}; + +static void lcb_put(struct lcb *lcb) +{ + if (lcb->alloc) + kfree(lcb->log_rec); + kfree(lcb->lrh); + kfree(lcb); +} + +/* Find the oldest lsn from active clients. */ +static inline void oldest_client_lsn(const struct CLIENT_REC *ca, + __le16 next_client, u64 *oldest_lsn) +{ + while (next_client != LFS_NO_CLIENT_LE) { + const struct CLIENT_REC *cr = ca + le16_to_cpu(next_client); + u64 lsn = le64_to_cpu(cr->oldest_lsn); + + /* Ignore this block if it's oldest lsn is 0. */ + if (lsn && lsn < *oldest_lsn) + *oldest_lsn = lsn; + + next_client = cr->next_client; + } +} + +static inline bool is_rst_page_hdr_valid(u32 file_off, + const struct RESTART_HDR *rhdr) +{ + u32 sys_page = le32_to_cpu(rhdr->sys_page_size); + u32 page_size = le32_to_cpu(rhdr->page_size); + u32 end_usa; + u16 ro; + + if (sys_page < SECTOR_SIZE || page_size < SECTOR_SIZE || + sys_page & (sys_page - 1) || page_size & (page_size - 1)) { + return false; + } + + /* Check that if the file offset isn't 0, it is the system page size. */ + if (file_off && file_off != sys_page) + return false; + + /* Check support version 1.1+. */ + if (le16_to_cpu(rhdr->major_ver) <= 1 && !rhdr->minor_ver) + return false; + + if (le16_to_cpu(rhdr->major_ver) > 2) + return false; + + ro = le16_to_cpu(rhdr->ra_off); + if (!IS_ALIGNED(ro, 8) || ro > sys_page) + return false; + + end_usa = ((sys_page >> SECTOR_SHIFT) + 1) * sizeof(short); + end_usa += le16_to_cpu(rhdr->rhdr.fix_off); + + if (ro < end_usa) + return false; + + return true; +} + +static inline bool is_rst_area_valid(const struct RESTART_HDR *rhdr) +{ + const struct RESTART_AREA *ra; + u16 cl, fl, ul; + u32 off, l_size, seq_bits; + u16 ro = le16_to_cpu(rhdr->ra_off); + u32 sys_page = le32_to_cpu(rhdr->sys_page_size); + + if (ro + offsetof(struct RESTART_AREA, l_size) > + SECTOR_SIZE - sizeof(short)) + return false; + + ra = Add2Ptr(rhdr, ro); + cl = le16_to_cpu(ra->log_clients); + + if (cl > 1) + return false; + + off = le16_to_cpu(ra->client_off); + + if (!IS_ALIGNED(off, 8) || ro + off > SECTOR_SIZE - sizeof(short)) + return false; + + off += cl * sizeof(struct CLIENT_REC); + + if (off > sys_page) + return false; + + /* + * Check the restart length field and whether the entire + * restart area is contained that length. + */ + if (le16_to_cpu(rhdr->ra_off) + le16_to_cpu(ra->ra_len) > sys_page || + off > le16_to_cpu(ra->ra_len)) { + return false; + } + + /* + * As a final check make sure that the use list and the free list + * are either empty or point to a valid client. + */ + fl = le16_to_cpu(ra->client_idx[0]); + ul = le16_to_cpu(ra->client_idx[1]); + if ((fl != LFS_NO_CLIENT && fl >= cl) || + (ul != LFS_NO_CLIENT && ul >= cl)) + return false; + + /* Make sure the sequence number bits match the log file size. */ + l_size = le64_to_cpu(ra->l_size); + + seq_bits = sizeof(u64) * 8 + 3; + while (l_size) { + l_size >>= 1; + seq_bits -= 1; + } + + if (seq_bits != le32_to_cpu(ra->seq_num_bits)) + return false; + + /* The log page data offset and record header length must be quad-aligned. */ + if (!IS_ALIGNED(le16_to_cpu(ra->data_off), 8) || + !IS_ALIGNED(le16_to_cpu(ra->rec_hdr_len), 8)) + return false; + + return true; +} + +static inline bool is_client_area_valid(const struct RESTART_HDR *rhdr, + bool usa_error) +{ + u16 ro = le16_to_cpu(rhdr->ra_off); + const struct RESTART_AREA *ra = Add2Ptr(rhdr, ro); + u16 ra_len = le16_to_cpu(ra->ra_len); + const struct CLIENT_REC *ca; + u32 i; + + if (usa_error && ra_len + ro > SECTOR_SIZE - sizeof(short)) + return false; + + /* Find the start of the client array. */ + ca = Add2Ptr(ra, le16_to_cpu(ra->client_off)); + + /* + * Start with the free list. + * Check that all the clients are valid and that there isn't a cycle. + * Do the in-use list on the second pass. + */ + for (i = 0; i < 2; i++) { + u16 client_idx = le16_to_cpu(ra->client_idx[i]); + bool first_client = true; + u16 clients = le16_to_cpu(ra->log_clients); + + while (client_idx != LFS_NO_CLIENT) { + const struct CLIENT_REC *cr; + + if (!clients || + client_idx >= le16_to_cpu(ra->log_clients)) + return false; + + clients -= 1; + cr = ca + client_idx; + + client_idx = le16_to_cpu(cr->next_client); + + if (first_client) { + first_client = false; + if (cr->prev_client != LFS_NO_CLIENT_LE) + return false; + } + } + } + + return true; +} + +/* + * remove_client + * + * Remove a client record from a client record list an restart area. + */ +static inline void remove_client(struct CLIENT_REC *ca, + const struct CLIENT_REC *cr, __le16 *head) +{ + if (cr->prev_client == LFS_NO_CLIENT_LE) + *head = cr->next_client; + else + ca[le16_to_cpu(cr->prev_client)].next_client = cr->next_client; + + if (cr->next_client != LFS_NO_CLIENT_LE) + ca[le16_to_cpu(cr->next_client)].prev_client = cr->prev_client; +} + +/* + * add_client - Add a client record to the start of a list. + */ +static inline void add_client(struct CLIENT_REC *ca, u16 index, __le16 *head) +{ + struct CLIENT_REC *cr = ca + index; + + cr->prev_client = LFS_NO_CLIENT_LE; + cr->next_client = *head; + + if (*head != LFS_NO_CLIENT_LE) + ca[le16_to_cpu(*head)].prev_client = cpu_to_le16(index); + + *head = cpu_to_le16(index); +} + +/* + * Enumerate restart table. + * + * @t - table to enumerate. + * @c - current enumerated element. + * + * enumeration starts with @c == NULL + * returns next element or NULL + */ +static inline void *enum_rstbl(struct RESTART_TABLE *t, void *c) +{ + __le32 *e; + u32 bprt; + u16 rsize; + + if (!t) + return NULL; + + rsize = le16_to_cpu(t->size); + + if (!c) { + /* start enumeration. */ + if (!t->total) + return NULL; + e = Add2Ptr(t, sizeof(struct RESTART_TABLE)); + } else { + e = Add2Ptr(c, rsize); + } + + /* Loop until we hit the first one allocated, or the end of the list. */ + for (bprt = bytes_per_rt(t); PtrOffset(t, e) < bprt; + e = Add2Ptr(e, rsize)) { + if (*e == RESTART_ENTRY_ALLOCATED_LE) + return e; + } + return NULL; +} + +/* + * dp_range_ok - true if [j, j + count) fits in a page_lcns[cap] array. + */ +static inline bool dp_range_ok(size_t j, u32 count, u32 cap) +{ + return j < cap && count <= cap - j; +} + +/* + * find_dp - Search for a @vcn in Dirty Page Table. + */ +static inline struct DIR_PAGE_ENTRY *find_dp(struct RESTART_TABLE *dptbl, + u32 target_attr, u64 vcn) +{ + __le32 ta = cpu_to_le32(target_attr); + struct DIR_PAGE_ENTRY *dp = NULL; + + while ((dp = enum_rstbl(dptbl, dp))) { + u64 dp_vcn = le64_to_cpu(dp->vcn); + + if (dp->target_attr == ta && vcn >= dp_vcn && + vcn < dp_vcn + le32_to_cpu(dp->lcns_follow)) { + return dp; + } + } + return NULL; +} + +static inline u32 norm_file_page(u32 page_size, u32 *l_size, bool use_default) +{ + if (use_default) + page_size = DefaultLogPageSize; + + /* Round the file size down to a system page boundary. */ + *l_size &= ~(page_size - 1); + + /* File should contain at least 2 restart pages and MinLogRecordPages pages. */ + if (*l_size < (MinLogRecordPages + 2) * page_size) + return 0; + + return page_size; +} + +static bool check_log_rec(const struct LOG_REC_HDR *lr, u32 bytes, u32 tr, + u32 bytes_per_attr_entry) +{ + u16 t16; + + if (bytes < sizeof(struct LOG_REC_HDR)) + return false; + if (!tr) + return false; + + if ((tr - sizeof(struct RESTART_TABLE)) % + sizeof(struct TRANSACTION_ENTRY)) + return false; + + if (le16_to_cpu(lr->redo_off) & 7) + return false; + + if (le16_to_cpu(lr->undo_off) & 7) + return false; + + if (lr->target_attr) + goto check_lcns; + + if (is_target_required(le16_to_cpu(lr->redo_op))) + return false; + + if (is_target_required(le16_to_cpu(lr->undo_op))) + return false; + +check_lcns: + if (!lr->lcns_follow) + goto check_length; + + t16 = le16_to_cpu(lr->target_attr); + if ((t16 - sizeof(struct RESTART_TABLE)) % bytes_per_attr_entry) + return false; + +check_length: + if (bytes < lrh_length(lr)) + return false; + + return true; +} + +static bool check_rstbl(const struct RESTART_TABLE *rt, size_t bytes) +{ + u32 ts; + u32 i, off; + u16 rsize = le16_to_cpu(rt->size); + u16 ne = le16_to_cpu(rt->used); + u32 ff = le32_to_cpu(rt->first_free); + u32 lf = le32_to_cpu(rt->last_free); + + ts = rsize * ne + sizeof(struct RESTART_TABLE); + + if (!rsize || rsize > bytes || + rsize + sizeof(struct RESTART_TABLE) > bytes || bytes < ts || + le16_to_cpu(rt->total) > ne || ff > ts - sizeof(__le32) || + lf > ts - sizeof(__le32) || + (ff && ff < sizeof(struct RESTART_TABLE)) || + (lf && lf < sizeof(struct RESTART_TABLE))) { + return false; + } + + /* + * Verify each entry is either allocated or points + * to a valid offset the table. + */ + for (i = 0; i < ne; i++) { + off = le32_to_cpu(*(__le32 *)Add2Ptr( + rt, i * rsize + sizeof(struct RESTART_TABLE))); + + if (off != RESTART_ENTRY_ALLOCATED && off && + (off < sizeof(struct RESTART_TABLE) || + ((off - sizeof(struct RESTART_TABLE)) % rsize))) { + return false; + } + } + + /* + * Walk through the list headed by the first entry to make + * sure none of the entries are currently being used. + * + * Bound traversal by ne (rt->used) to defeat a crafted on-disk + * cycle in the free chain. Each entry in a legitimate free + * list is unique, so a chain that visits more than ne slots + * is malformed. Without this guard, an attacker-controlled + * RESTART_TABLE with a self-loop or A->B->A cycle whose + * offsets satisfy the existing alignment + in-bounds guards + * spins forever at mount time. + */ + for (off = ff, i = 0; off; i++) { + if (i > ne) + return false; + + if (off == RESTART_ENTRY_ALLOCATED) + return false; + + off = le32_to_cpu(*(__le32 *)Add2Ptr(rt, off)); + + if (off > ts - sizeof(__le32)) + return false; + } + + return true; +} + +static bool check_dp_table(const struct RESTART_TABLE *dptbl) +{ + u32 rsize = le16_to_cpu(dptbl->size); + struct DIR_PAGE_ENTRY *dp = NULL; + + while ((dp = enum_rstbl((struct RESTART_TABLE *)dptbl, dp))) { + if (struct_size(dp, page_lcns, le32_to_cpu(dp->lcns_follow)) > + rsize) + return false; + } + + return true; +} + +/* + * free_rsttbl_idx - Free a previously allocated index a Restart Table. + */ +static inline void free_rsttbl_idx(struct RESTART_TABLE *rt, u32 off) +{ + __le32 *e; + u32 lf = le32_to_cpu(rt->last_free); + __le32 off_le = cpu_to_le32(off); + + e = Add2Ptr(rt, off); + + if (off < le32_to_cpu(rt->free_goal)) { + *e = rt->first_free; + rt->first_free = off_le; + if (!lf) + rt->last_free = off_le; + } else { + if (lf) + *(__le32 *)Add2Ptr(rt, lf) = off_le; + else + rt->first_free = off_le; + + rt->last_free = off_le; + *e = 0; + } + + le16_sub_cpu(&rt->total, 1); +} + +static inline struct RESTART_TABLE *init_rsttbl(u16 esize, u16 used) +{ + __le32 *e, *last_free; + u32 off; + u32 bytes = esize * used + sizeof(struct RESTART_TABLE); + u32 lf = sizeof(struct RESTART_TABLE) + (used - 1) * esize; + struct RESTART_TABLE *t = kzalloc(bytes, GFP_NOFS); + + if (!t) + return NULL; + + t->size = cpu_to_le16(esize); + t->used = cpu_to_le16(used); + t->free_goal = cpu_to_le32(~0u); + t->first_free = cpu_to_le32(sizeof(struct RESTART_TABLE)); + t->last_free = cpu_to_le32(lf); + + e = (__le32 *)(t + 1); + last_free = Add2Ptr(t, lf); + + for (off = sizeof(struct RESTART_TABLE) + esize; e < last_free; + e = Add2Ptr(e, esize), off += esize) { + *e = cpu_to_le32(off); + } + return t; +} + +static inline struct RESTART_TABLE *extend_rsttbl(struct RESTART_TABLE *tbl, + u32 add, u32 free_goal) +{ + u16 esize = le16_to_cpu(tbl->size); + __le32 osize = cpu_to_le32(bytes_per_rt(tbl)); + u32 used = le16_to_cpu(tbl->used); + struct RESTART_TABLE *rt; + + if (used + add > U16_MAX) + return NULL; + + rt = init_rsttbl(esize, used + add); + if (!rt) + return NULL; + + memcpy(rt + 1, tbl + 1, esize * used); + + rt->free_goal = free_goal == ~0u ? + cpu_to_le32(~0u) : + cpu_to_le32(sizeof(struct RESTART_TABLE) + + free_goal * esize); + + if (tbl->first_free) { + rt->first_free = tbl->first_free; + *(__le32 *)Add2Ptr(rt, le32_to_cpu(tbl->last_free)) = osize; + } else { + rt->first_free = osize; + } + + rt->total = tbl->total; + + kfree(tbl); + return rt; +} + +/* + * alloc_rsttbl_idx + * + * Allocate an index from within a previously initialized Restart Table. + */ +static inline void *alloc_rsttbl_idx(struct RESTART_TABLE **tbl) +{ + u32 off; + __le32 *e; + struct RESTART_TABLE *t = *tbl; + + if (!t->first_free) { + *tbl = t = extend_rsttbl(t, 16, ~0u); + if (!t) + return NULL; + } + + off = le32_to_cpu(t->first_free); + + /* Dequeue this entry and zero it. */ + e = Add2Ptr(t, off); + + t->first_free = *e; + + memset(e, 0, le16_to_cpu(t->size)); + + *e = RESTART_ENTRY_ALLOCATED_LE; + + /* If list is going empty, then we fix the last_free as well. */ + if (!t->first_free) + t->last_free = 0; + + le16_add_cpu(&t->total, 1); + + return Add2Ptr(t, off); +} + +/* + * alloc_rsttbl_from_idx + * + * Allocate a specific index from within a previously initialized Restart Table. + */ +static inline void *alloc_rsttbl_from_idx(struct RESTART_TABLE **tbl, u32 vbo) +{ + u32 off; + __le32 *e; + struct RESTART_TABLE *rt = *tbl; + u32 bytes = bytes_per_rt(rt); + u16 esize = le16_to_cpu(rt->size); + + /* If the entry is not the table, we will have to extend the table. */ + if (vbo >= bytes) { + /* + * Extend the size by computing the number of entries between + * the existing size and the desired index and adding 1 to that. + */ + u32 bytes2idx = vbo - bytes; + + /* + * There should always be an integral number of entries + * being added. Now extend the table. + */ + *tbl = rt = extend_rsttbl(rt, bytes2idx / esize + 1, bytes); + if (!rt) + return NULL; + } + + /* See if the entry is already allocated, and just return if it is. */ + e = Add2Ptr(rt, vbo); + + if (*e == RESTART_ENTRY_ALLOCATED_LE) + return e; + + /* + * Walk through the table, looking for the entry we're + * interested and the previous entry. + */ + off = le32_to_cpu(rt->first_free); + e = Add2Ptr(rt, off); + + if (off == vbo) { + /* this is a match */ + rt->first_free = *e; + goto skip_looking; + } + + /* + * Need to walk through the list looking for the predecessor + * of our entry. + */ + for (;;) { + /* Remember the entry just found */ + u32 last_off = off; + __le32 *last_e = e; + + /* Should never run of entries. */ + + /* Lookup up the next entry the list. */ + off = le32_to_cpu(*last_e); + e = Add2Ptr(rt, off); + + /* If this is our match we are done. */ + if (off == vbo) { + *last_e = *e; + + /* + * If this was the last entry, we update that + * table as well. + */ + if (le32_to_cpu(rt->last_free) == off) + rt->last_free = cpu_to_le32(last_off); + break; + } + } + +skip_looking: + /* If the list is now empty, we fix the last_free as well. */ + if (!rt->first_free) + rt->last_free = 0; + + /* Zero this entry. */ + memset(e, 0, esize); + *e = RESTART_ENTRY_ALLOCATED_LE; + + le16_add_cpu(&rt->total, 1); + + return e; +} + +struct restart_info { + u64 last_lsn; + struct RESTART_HDR *r_page; + u32 vbo; + bool chkdsk_was_run; + bool valid_page; + bool initialized; + bool restart; +}; + +#define RESTART_SINGLE_PAGE_IO cpu_to_le16(0x0001) + +#define NTFSLOG_WRAPPED 0x00000001 +#define NTFSLOG_MULTIPLE_PAGE_IO 0x00000002 +#define NTFSLOG_NO_LAST_LSN 0x00000004 +#define NTFSLOG_REUSE_TAIL 0x00000010 +#define NTFSLOG_NO_OLDEST_LSN 0x00000020 + +/* Helper struct to work with NTFS $LogFile. */ +struct ntfs_log { + struct ntfs_inode *ni; + + u32 l_size; + u32 orig_file_size; + u32 sys_page_size; + u32 sys_page_mask; + u32 page_size; + u32 page_mask; // page_size - 1 + u8 page_bits; + struct RECORD_PAGE_HDR *one_page_buf; + + struct RESTART_TABLE *open_attr_tbl; + u32 transaction_id; + u32 clst_per_page; + + u32 first_page; + u32 next_page; + u32 ra_off; + u32 data_off; + u32 restart_size; + u32 data_size; + u16 record_header_len; + u64 seq_num; + u32 seq_num_bits; + u32 file_data_bits; + u32 seq_num_mask; /* (1 << file_data_bits) - 1 */ + + struct RESTART_AREA *ra; /* In-memory image of the next restart area. */ + u32 ra_size; /* The usable size of the restart area. */ + + /* + * If true, then the in-memory restart area is to be written + * to the first position on the disk. + */ + bool init_ra; + bool set_dirty; /* True if we need to set dirty flag. */ + + u64 oldest_lsn; + + u32 oldest_lsn_off; + u64 last_lsn; + + u32 total_avail; + u32 total_avail_pages; + u32 total_undo_commit; + u32 max_current_avail; + u32 current_avail; + u32 reserved; + + short major_ver; + short minor_ver; + + u32 l_flags; /* See NTFSLOG_XXX */ + u32 current_openlog_count; /* On-disk value for open_log_count. */ + + struct CLIENT_ID client_id; + u32 client_undo_commit; + + struct restart_info rst_info, rst_info2; + + struct file_ra_state read_ahead; +}; + +static inline u32 lsn_to_vbo(struct ntfs_log *log, const u64 lsn) +{ + u32 vbo = (lsn << log->seq_num_bits) >> (log->seq_num_bits - 3); + + return vbo; +} + +/* Compute the offset in the log file of the next log page. */ +static inline u32 next_page_off(struct ntfs_log *log, u32 off) +{ + off = (off & ~log->sys_page_mask) + log->page_size; + return off >= log->l_size ? log->first_page : off; +} + +static inline u32 lsn_to_page_off(struct ntfs_log *log, u64 lsn) +{ + return (((u32)lsn) << 3) & log->page_mask; +} + +static inline u64 vbo_to_lsn(struct ntfs_log *log, u32 off, u64 Seq) +{ + return (off >> 3) + (Seq << log->file_data_bits); +} + +static inline bool is_lsn_in_file(struct ntfs_log *log, u64 lsn) +{ + return lsn >= log->oldest_lsn && + lsn <= le64_to_cpu(log->ra->current_lsn); +} + +static inline u32 hdr_file_off(struct ntfs_log *log, + struct RECORD_PAGE_HDR *hdr) +{ + if (log->major_ver < 2) + return le64_to_cpu(hdr->rhdr.lsn); + + return le32_to_cpu(hdr->file_off); +} + +static inline u64 base_lsn(struct ntfs_log *log, + const struct RECORD_PAGE_HDR *hdr, u64 lsn) +{ + u64 h_lsn = le64_to_cpu(hdr->rhdr.lsn); + u64 ret = (((h_lsn >> log->file_data_bits) + + (lsn < (lsn_to_vbo(log, h_lsn) & ~log->page_mask) ? 1 : 0)) + << log->file_data_bits) + + ((((is_log_record_end(hdr) && + h_lsn <= le64_to_cpu(hdr->record_hdr.last_end_lsn)) ? + le16_to_cpu(hdr->record_hdr.next_record_off) : + log->page_size) + + lsn) >> + 3); + + return ret; +} + +static inline bool verify_client_lsn(struct ntfs_log *log, + const struct CLIENT_REC *client, u64 lsn) +{ + return lsn >= le64_to_cpu(client->oldest_lsn) && + lsn <= le64_to_cpu(log->ra->current_lsn) && lsn; +} + +static int read_log_page(struct ntfs_log *log, u32 vbo, + struct RECORD_PAGE_HDR **buffer, bool *usa_error) +{ + int err = 0; + u32 page_idx = vbo >> log->page_bits; + u32 page_off = vbo & log->page_mask; + u32 bytes = log->page_size - page_off; + void *to_free = NULL; + u32 page_vbo = page_idx << log->page_bits; + struct RECORD_PAGE_HDR *page_buf; + struct ntfs_inode *ni = log->ni; + bool bBAAD; + + if (vbo >= log->l_size) + return -EINVAL; + + if (!*buffer) { + to_free = kmalloc(log->page_size, GFP_NOFS); + if (!to_free) + return -ENOMEM; + *buffer = to_free; + } + + page_buf = page_off ? log->one_page_buf : *buffer; + + err = ntfs_read_run_nb_ra(ni->mi.sbi, &ni->file.run, page_vbo, page_buf, + log->page_size, NULL, &log->read_ahead); + if (err) + goto out; + + if (page_buf->rhdr.sign != NTFS_FFFF_SIGNATURE) + ntfs_fix_post_read(&page_buf->rhdr, log->page_size, false); + + if (page_buf != *buffer) + memcpy(*buffer, Add2Ptr(page_buf, page_off), bytes); + + bBAAD = page_buf->rhdr.sign == NTFS_BAAD_SIGNATURE; + + if (usa_error) + *usa_error = bBAAD; + /* Check that the update sequence array for this page is valid */ + /* If we don't allow errors, raise an error status */ + else if (bBAAD) + err = -EINVAL; + +out: + if (err && to_free) { + kfree(to_free); + *buffer = NULL; + } + + return err; +} + +/* + * log_read_rst + * + * It walks through 512 blocks of the file looking for a valid + * restart page header. It will stop the first time we find a + * valid page header. + */ +static int log_read_rst(struct ntfs_log *log, bool first, + struct restart_info *info) +{ + u32 skip; + u64 vbo; + struct RESTART_HDR *r_page = NULL; + + /* Determine which restart area we are looking for. */ + if (first) { + vbo = 0; + skip = 512; + } else { + vbo = 512; + skip = 0; + } + + /* Loop continuously until we succeed. */ + for (; vbo < log->l_size; vbo = 2 * vbo + skip, skip = 0) { + bool usa_error; + bool brst, bchk; + struct RESTART_AREA *ra; + + /* Read a page header at the current offset. */ + if (read_log_page(log, vbo, (struct RECORD_PAGE_HDR **)&r_page, + &usa_error)) { + /* Ignore any errors. */ + continue; + } + + /* Exit if the signature is a log record page. */ + if (r_page->rhdr.sign == NTFS_RCRD_SIGNATURE) { + info->initialized = true; + break; + } + + brst = r_page->rhdr.sign == NTFS_RSTR_SIGNATURE; + bchk = r_page->rhdr.sign == NTFS_CHKD_SIGNATURE; + + if (!bchk && !brst) { + if (r_page->rhdr.sign != NTFS_FFFF_SIGNATURE) { + /* + * Remember if the signature does not + * indicate uninitialized file. + */ + info->initialized = true; + } + continue; + } + + ra = NULL; + info->valid_page = false; + info->initialized = true; + info->vbo = vbo; + + /* Let's check the restart area if this is a valid page. */ + if (!is_rst_page_hdr_valid(vbo, r_page)) + goto check_result; + ra = Add2Ptr(r_page, le16_to_cpu(r_page->ra_off)); + + if (!is_rst_area_valid(r_page)) + goto check_result; + + /* + * We have a valid restart page header and restart area. + * If chkdsk was run or we have no clients then we have + * no more checking to do. + */ + if (bchk || ra->client_idx[1] == LFS_NO_CLIENT_LE) { + info->valid_page = true; + goto check_result; + } + + if (is_client_area_valid(r_page, usa_error)) { + info->valid_page = true; + ra = Add2Ptr(r_page, le16_to_cpu(r_page->ra_off)); + } + +check_result: + /* + * If chkdsk was run then update the caller's + * values and return. + */ + if (r_page->rhdr.sign == NTFS_CHKD_SIGNATURE) { + info->chkdsk_was_run = true; + info->last_lsn = le64_to_cpu(r_page->rhdr.lsn); + info->restart = true; + info->r_page = r_page; + return 0; + } + + /* + * If we have a valid page then copy the values + * we need from it. + */ + if (info->valid_page) { + info->last_lsn = le64_to_cpu(ra->current_lsn); + info->restart = true; + info->r_page = r_page; + return 0; + } + } + + kfree(r_page); + + return 0; +} + +/* + * Ilog_init_pg_hdr - Init @log from restart page header. + */ +static void log_init_pg_hdr(struct ntfs_log *log, u16 major_ver, u16 minor_ver) +{ + log->sys_page_size = log->page_size; + log->sys_page_mask = log->page_mask; + + log->clst_per_page = log->page_size >> log->ni->mi.sbi->cluster_bits; + if (!log->clst_per_page) + log->clst_per_page = 1; + + log->first_page = major_ver >= 2 ? 0x22 * log->page_size : + 4 * log->page_size; + log->major_ver = major_ver; + log->minor_ver = minor_ver; +} + +/* + * log_create - Init @log in cases when we don't have a restart area to use. + */ +static void log_create(struct ntfs_log *log, const u64 last_lsn, + u32 open_log_count, bool wrapped, bool use_multi_page) +{ + /* All file offsets must be quadword aligned. */ + log->file_data_bits = blksize_bits(log->l_size) - 3; + log->seq_num_mask = (8 << log->file_data_bits) - 1; + log->seq_num_bits = sizeof(u64) * 8 - log->file_data_bits; + log->seq_num = (last_lsn >> log->file_data_bits) + 2; + log->next_page = log->first_page; + log->oldest_lsn = log->seq_num << log->file_data_bits; + log->oldest_lsn_off = 0; + log->last_lsn = log->oldest_lsn; + + log->l_flags |= NTFSLOG_NO_LAST_LSN | NTFSLOG_NO_OLDEST_LSN; + + /* Set the correct flags for the I/O and indicate if we have wrapped. */ + if (wrapped) + log->l_flags |= NTFSLOG_WRAPPED; + + if (use_multi_page) + log->l_flags |= NTFSLOG_MULTIPLE_PAGE_IO; + + /* Compute the log page values. */ + log->data_off = ALIGN( + offsetof(struct RECORD_PAGE_HDR, fixups) + + sizeof(short) * ((log->page_size >> SECTOR_SHIFT) + 1), + 8); + log->data_size = log->page_size - log->data_off; + log->record_header_len = sizeof(struct LFS_RECORD_HDR); + + /* Remember the different page sizes for reservation. */ + log->reserved = log->data_size - log->record_header_len; + + /* Compute the restart page values. */ + log->ra_off = ALIGN( + offsetof(struct RESTART_HDR, fixups) + + sizeof(short) * + ((log->sys_page_size >> SECTOR_SHIFT) + 1), + 8); + log->restart_size = log->sys_page_size - log->ra_off; + log->ra_size = struct_size(log->ra, clients, 1); + log->current_openlog_count = open_log_count; + + /* + * The total available log file space is the number of + * log file pages times the space available on each page. + */ + log->total_avail_pages = log->l_size - log->first_page; + log->total_avail = log->total_avail_pages >> log->page_bits; + + /* + * We assume that we can't use the end of the page less than + * the file record size. + * Then we won't need to reserve more than the caller asks for. + */ + log->max_current_avail = log->total_avail * log->reserved; + log->total_avail = log->total_avail * log->data_size; + log->current_avail = log->max_current_avail; +} + +/* + * log_create_ra - Fill a restart area from the values stored in @log. + */ +static struct RESTART_AREA *log_create_ra(struct ntfs_log *log) +{ + struct CLIENT_REC *cr; + struct RESTART_AREA *ra = kzalloc(log->restart_size, GFP_NOFS); + + if (!ra) + return NULL; + + ra->current_lsn = cpu_to_le64(log->last_lsn); + ra->log_clients = cpu_to_le16(1); + ra->client_idx[1] = LFS_NO_CLIENT_LE; + if (log->l_flags & NTFSLOG_MULTIPLE_PAGE_IO) + ra->flags = RESTART_SINGLE_PAGE_IO; + ra->seq_num_bits = cpu_to_le32(log->seq_num_bits); + ra->ra_len = cpu_to_le16(log->ra_size); + ra->client_off = cpu_to_le16(offsetof(struct RESTART_AREA, clients)); + ra->l_size = cpu_to_le64(log->l_size); + ra->rec_hdr_len = cpu_to_le16(log->record_header_len); + ra->data_off = cpu_to_le16(log->data_off); + ra->open_log_count = cpu_to_le32(log->current_openlog_count + 1); + + cr = ra->clients; + + cr->prev_client = LFS_NO_CLIENT_LE; + cr->next_client = LFS_NO_CLIENT_LE; + + return ra; +} + +static u32 final_log_off(struct ntfs_log *log, u64 lsn, u32 data_len) +{ + u32 base_vbo = lsn << 3; + u32 final_log_off = (base_vbo & log->seq_num_mask) & ~log->page_mask; + u32 page_off = base_vbo & log->page_mask; + u32 tail = log->page_size - page_off; + + page_off -= 1; + + /* Add the length of the header. */ + data_len += log->record_header_len; + + /* + * If this lsn is contained this log page we are done. + * Otherwise we need to walk through several log pages. + */ + if (data_len > tail) { + data_len -= tail; + tail = log->data_size; + page_off = log->data_off - 1; + + for (;;) { + final_log_off = next_page_off(log, final_log_off); + + /* + * We are done if the remaining bytes + * fit on this page. + */ + if (data_len <= tail) + break; + data_len -= tail; + } + } + + /* + * We add the remaining bytes to our starting position on this page + * and then add that value to the file offset of this log page. + */ + return final_log_off + data_len + page_off; +} + +static int next_log_lsn(struct ntfs_log *log, const struct LFS_RECORD_HDR *rh, + u64 *lsn) +{ + int err; + u64 this_lsn = le64_to_cpu(rh->this_lsn); + u32 vbo = lsn_to_vbo(log, this_lsn); + u32 end = + final_log_off(log, this_lsn, le32_to_cpu(rh->client_data_len)); + u32 hdr_off = end & ~log->sys_page_mask; + u64 seq = this_lsn >> log->file_data_bits; + struct RECORD_PAGE_HDR *page = NULL; + + /* Remember if we wrapped. */ + if (end <= vbo) + seq += 1; + + /* Log page header for this page. */ + err = read_log_page(log, hdr_off, &page, NULL); + if (err) + return err; + + /* + * If the lsn we were given was not the last lsn on this page, + * then the starting offset for the next lsn is on a quad word + * boundary following the last file offset for the current lsn. + * Otherwise the file offset is the start of the data on the next page. + */ + if (this_lsn == le64_to_cpu(page->rhdr.lsn)) { + /* If we wrapped, we need to increment the sequence number. */ + hdr_off = next_page_off(log, hdr_off); + if (hdr_off == log->first_page) + seq += 1; + + vbo = hdr_off + log->data_off; + } else { + vbo = ALIGN(end, 8); + } + + /* Compute the lsn based on the file offset and the sequence count. */ + *lsn = vbo_to_lsn(log, vbo, seq); + + /* + * If this lsn is within the legal range for the file, we return true. + * Otherwise false indicates that there are no more lsn's. + */ + if (!is_lsn_in_file(log, *lsn)) + *lsn = 0; + + kfree(page); + + return 0; +} + +/* + * current_log_avail - Calculate the number of bytes available for log records. + */ +static u32 current_log_avail(struct ntfs_log *log) +{ + u32 oldest_off, next_free_off, free_bytes; + + if (log->l_flags & NTFSLOG_NO_LAST_LSN) { + /* The entire file is available. */ + return log->max_current_avail; + } + + /* + * If there is a last lsn the restart area then we know that we will + * have to compute the free range. + * If there is no oldest lsn then start at the first page of the file. + */ + oldest_off = (log->l_flags & NTFSLOG_NO_OLDEST_LSN) ? + log->first_page : + (log->oldest_lsn_off & ~log->sys_page_mask); + + /* + * We will use the next log page offset to compute the next free page. + * If we are going to reuse this page go to the next page. + * If we are at the first page then use the end of the file. + */ + next_free_off = (log->l_flags & NTFSLOG_REUSE_TAIL) ? + log->next_page + log->page_size : + log->next_page == log->first_page ? log->l_size : + log->next_page; + + /* If the two offsets are the same then there is no available space. */ + if (oldest_off == next_free_off) + return 0; + /* + * If the free offset follows the oldest offset then subtract + * this range from the total available pages. + */ + free_bytes = + oldest_off < next_free_off ? + log->total_avail_pages - (next_free_off - oldest_off) : + oldest_off - next_free_off; + + free_bytes >>= log->page_bits; + return free_bytes * log->reserved; +} + +static bool check_subseq_log_page(struct ntfs_log *log, + const struct RECORD_PAGE_HDR *rp, u32 vbo, + u64 seq) +{ + u64 lsn_seq; + const struct NTFS_RECORD_HEADER *rhdr = &rp->rhdr; + u64 lsn = le64_to_cpu(rhdr->lsn); + + if (rhdr->sign == NTFS_FFFF_SIGNATURE || !rhdr->sign) + return false; + + /* + * If the last lsn on the page occurs was written after the page + * that caused the original error then we have a fatal error. + */ + lsn_seq = lsn >> log->file_data_bits; + + /* + * If the sequence number for the lsn the page is equal or greater + * than lsn we expect, then this is a subsequent write. + */ + return lsn_seq >= seq || + (lsn_seq == seq - 1 && log->first_page == vbo && + vbo != (lsn_to_vbo(log, lsn) & ~log->page_mask)); +} + +/* + * last_log_lsn + * + * Walks through the log pages for a file, searching for the + * last log page written to the file. + */ +static int last_log_lsn(struct ntfs_log *log) +{ + int err; + bool usa_error = false; + bool replace_page = false; + bool reuse_page = log->l_flags & NTFSLOG_REUSE_TAIL; + bool wrapped_file, wrapped; + + u32 page_cnt = 1, page_pos = 1; + u32 page_off = 0, page_off1 = 0, saved_off = 0; + u32 final_off, second_off, final_off_prev = 0, second_off_prev = 0; + u32 first_file_off = 0, second_file_off = 0; + u32 part_io_count = 0; + u32 tails = 0; + u32 this_off, curpage_off, nextpage_off, remain_pages; + + u64 expected_seq, seq_base = 0, lsn_base = 0; + u64 best_lsn, best_lsn1, best_lsn2; + u64 lsn_cur, lsn1, lsn2; + u64 last_ok_lsn = reuse_page ? log->last_lsn : 0; + + u16 cur_pos, best_page_pos; + + struct RECORD_PAGE_HDR *page = NULL; + struct RECORD_PAGE_HDR *tst_page = NULL; + struct RECORD_PAGE_HDR *first_tail = NULL; + struct RECORD_PAGE_HDR *second_tail = NULL; + struct RECORD_PAGE_HDR *tail_page = NULL; + struct RECORD_PAGE_HDR *second_tail_prev = NULL; + struct RECORD_PAGE_HDR *first_tail_prev = NULL; + struct RECORD_PAGE_HDR *page_bufs = NULL; + struct RECORD_PAGE_HDR *best_page; + + if (log->major_ver >= 2) { + final_off = 0x02 * log->page_size; + second_off = 0x12 * log->page_size; + + // 0x10 == 0x12 - 0x2 + page_bufs = kmalloc(log->page_size * 0x10, GFP_NOFS); + if (!page_bufs) + return -ENOMEM; + } else { + second_off = log->first_page - log->page_size; + final_off = second_off - log->page_size; + } + +next_tail: + /* Read second tail page (at pos 3/0x12000). */ + if (read_log_page(log, second_off, &second_tail, &usa_error) || + usa_error || second_tail->rhdr.sign != NTFS_RCRD_SIGNATURE) { + kfree(second_tail); + second_tail = NULL; + second_file_off = 0; + lsn2 = 0; + } else { + second_file_off = hdr_file_off(log, second_tail); + lsn2 = le64_to_cpu(second_tail->record_hdr.last_end_lsn); + } + + /* Read first tail page (at pos 2/0x2000). */ + if (read_log_page(log, final_off, &first_tail, &usa_error) || + usa_error || first_tail->rhdr.sign != NTFS_RCRD_SIGNATURE) { + kfree(first_tail); + first_tail = NULL; + first_file_off = 0; + lsn1 = 0; + } else { + first_file_off = hdr_file_off(log, first_tail); + lsn1 = le64_to_cpu(first_tail->record_hdr.last_end_lsn); + } + + if (log->major_ver < 2) { + int best_page; + + first_tail_prev = first_tail; + final_off_prev = first_file_off; + second_tail_prev = second_tail; + second_off_prev = second_file_off; + tails = 1; + + if (!first_tail && !second_tail) + goto tail_read; + + if (first_tail && second_tail) + best_page = lsn1 < lsn2 ? 1 : 0; + else if (first_tail) + best_page = 0; + else + best_page = 1; + + page_off = best_page ? second_file_off : first_file_off; + seq_base = (best_page ? lsn2 : lsn1) >> log->file_data_bits; + goto tail_read; + } + + best_lsn1 = first_tail ? base_lsn(log, first_tail, first_file_off) : 0; + best_lsn2 = second_tail ? base_lsn(log, second_tail, second_file_off) : + 0; + + if (first_tail && second_tail) { + if (best_lsn1 > best_lsn2) { + best_lsn = best_lsn1; + best_page = first_tail; + this_off = first_file_off; + } else { + best_lsn = best_lsn2; + best_page = second_tail; + this_off = second_file_off; + } + } else if (first_tail) { + best_lsn = best_lsn1; + best_page = first_tail; + this_off = first_file_off; + } else if (second_tail) { + best_lsn = best_lsn2; + best_page = second_tail; + this_off = second_file_off; + } else { + goto tail_read; + } + + best_page_pos = le16_to_cpu(best_page->page_pos); + + if (!tails) { + if (best_page_pos == page_pos) { + seq_base = best_lsn >> log->file_data_bits; + saved_off = page_off = le32_to_cpu(best_page->file_off); + lsn_base = best_lsn; + + memmove(page_bufs, best_page, log->page_size); + + page_cnt = le16_to_cpu(best_page->page_count); + if (page_cnt > 1) + page_pos += 1; + + tails = 1; + } + } else if (seq_base == (best_lsn >> log->file_data_bits) && + saved_off + log->page_size == this_off && + lsn_base < best_lsn && + (page_pos != page_cnt || best_page_pos == page_pos || + best_page_pos == 1) && + (page_pos >= page_cnt || best_page_pos == page_pos)) { + u16 bppc = le16_to_cpu(best_page->page_count); + + saved_off += log->page_size; + lsn_base = best_lsn; + + memmove(Add2Ptr(page_bufs, tails * log->page_size), best_page, + log->page_size); + + tails += 1; + + if (best_page_pos != bppc) { + page_cnt = bppc; + page_pos = best_page_pos; + + if (page_cnt > 1) + page_pos += 1; + } else { + page_pos = page_cnt = 1; + } + } else { + kfree(first_tail); + kfree(second_tail); + goto tail_read; + } + + kfree(first_tail_prev); + first_tail_prev = first_tail; + final_off_prev = first_file_off; + first_tail = NULL; + + kfree(second_tail_prev); + second_tail_prev = second_tail; + second_off_prev = second_file_off; + second_tail = NULL; + + final_off += log->page_size; + second_off += log->page_size; + + if (tails < 0x10) + goto next_tail; +tail_read: + first_tail = first_tail_prev; + final_off = final_off_prev; + + second_tail = second_tail_prev; + second_off = second_off_prev; + + page_cnt = page_pos = 1; + + curpage_off = seq_base == log->seq_num ? min(log->next_page, page_off) : + log->next_page; + + wrapped_file = + curpage_off == log->first_page && + !(log->l_flags & (NTFSLOG_NO_LAST_LSN | NTFSLOG_REUSE_TAIL)); + + expected_seq = wrapped_file ? (log->seq_num + 1) : log->seq_num; + + nextpage_off = curpage_off; + +next_page: + tail_page = NULL; + /* Read the next log page. */ + err = read_log_page(log, curpage_off, &page, &usa_error); + + /* Compute the next log page offset the file. */ + nextpage_off = next_page_off(log, curpage_off); + wrapped = nextpage_off == log->first_page; + + if (tails > 1) { + struct RECORD_PAGE_HDR *cur_page = + Add2Ptr(page_bufs, curpage_off - page_off); + + if (curpage_off == saved_off) { + tail_page = cur_page; + goto use_tail_page; + } + + if (page_off > curpage_off || curpage_off >= saved_off) + goto use_tail_page; + + if (page_off1) + goto use_cur_page; + + if (!err && !usa_error && + page->rhdr.sign == NTFS_RCRD_SIGNATURE && + cur_page->rhdr.lsn == page->rhdr.lsn && + cur_page->record_hdr.next_record_off == + page->record_hdr.next_record_off && + ((page_pos == page_cnt && + le16_to_cpu(page->page_pos) == 1) || + (page_pos != page_cnt && + le16_to_cpu(page->page_pos) == page_pos + 1 && + le16_to_cpu(page->page_count) == page_cnt))) { + cur_page = NULL; + goto use_tail_page; + } + + page_off1 = page_off; + +use_cur_page: + + lsn_cur = le64_to_cpu(cur_page->rhdr.lsn); + + if (last_ok_lsn != + le64_to_cpu(cur_page->record_hdr.last_end_lsn) && + ((lsn_cur >> log->file_data_bits) + + ((curpage_off < + (lsn_to_vbo(log, lsn_cur) & ~log->page_mask)) ? + 1 : + 0)) != expected_seq) { + goto check_tail; + } + + if (!is_log_record_end(cur_page)) { + tail_page = NULL; + last_ok_lsn = lsn_cur; + goto next_page_1; + } + + log->seq_num = expected_seq; + log->l_flags &= ~NTFSLOG_NO_LAST_LSN; + log->last_lsn = le64_to_cpu(cur_page->record_hdr.last_end_lsn); + log->ra->current_lsn = cur_page->record_hdr.last_end_lsn; + + if (log->record_header_len <= + log->page_size - + le16_to_cpu(cur_page->record_hdr.next_record_off)) { + log->l_flags |= NTFSLOG_REUSE_TAIL; + log->next_page = curpage_off; + } else { + log->l_flags &= ~NTFSLOG_REUSE_TAIL; + log->next_page = nextpage_off; + } + + if (wrapped_file) + log->l_flags |= NTFSLOG_WRAPPED; + + last_ok_lsn = le64_to_cpu(cur_page->record_hdr.last_end_lsn); + goto next_page_1; + } + + /* + * If we are at the expected first page of a transfer check to see + * if either tail copy is at this offset. + * If this page is the last page of a transfer, check if we wrote + * a subsequent tail copy. + */ + if (page_cnt == page_pos || page_cnt == page_pos + 1) { + /* + * Check if the offset matches either the first or second + * tail copy. It is possible it will match both. + */ + if (curpage_off == final_off) + tail_page = first_tail; + + /* + * If we already matched on the first page then + * check the ending lsn's. + */ + if (curpage_off == second_off) { + if (!tail_page || + (second_tail && + le64_to_cpu(second_tail->record_hdr.last_end_lsn) > + le64_to_cpu(first_tail->record_hdr + .last_end_lsn))) { + tail_page = second_tail; + } + } + } + +use_tail_page: + if (tail_page) { + /* We have a candidate for a tail copy. */ + lsn_cur = le64_to_cpu(tail_page->record_hdr.last_end_lsn); + + if (last_ok_lsn < lsn_cur) { + /* + * If the sequence number is not expected, + * then don't use the tail copy. + */ + if (expected_seq != (lsn_cur >> log->file_data_bits)) + tail_page = NULL; + } else if (last_ok_lsn > lsn_cur) { + /* + * If the last lsn is greater than the one on + * this page then forget this tail. + */ + tail_page = NULL; + } + } + + /* + *If we have an error on the current page, + * we will break of this loop. + */ + if (err || usa_error) + goto check_tail; + + /* + * Done if the last lsn on this page doesn't match the previous known + * last lsn or the sequence number is not expected. + */ + lsn_cur = le64_to_cpu(page->rhdr.lsn); + if (last_ok_lsn != lsn_cur && + expected_seq != (lsn_cur >> log->file_data_bits)) { + goto check_tail; + } + + /* + * Check that the page position and page count values are correct. + * If this is the first page of a transfer the position must be 1 + * and the count will be unknown. + */ + if (page_cnt == page_pos) { + if (page->page_pos != cpu_to_le16(1) && + (!reuse_page || page->page_pos != page->page_count)) { + /* + * If the current page is the first page we are + * looking at and we are reusing this page then + * it can be either the first or last page of a + * transfer. Otherwise it can only be the first. + */ + goto check_tail; + } + } else if (le16_to_cpu(page->page_count) != page_cnt || + le16_to_cpu(page->page_pos) != page_pos + 1) { + /* + * The page position better be 1 more than the last page + * position and the page count better match. + */ + goto check_tail; + } + + /* + * We have a valid page the file and may have a valid page + * the tail copy area. + * If the tail page was written after the page the file then + * break of the loop. + */ + if (tail_page && + le64_to_cpu(tail_page->record_hdr.last_end_lsn) > lsn_cur) { + /* Remember if we will replace the page. */ + replace_page = true; + goto check_tail; + } + + tail_page = NULL; + + if (is_log_record_end(page)) { + /* + * Since we have read this page we know the sequence number + * is the same as our expected value. + */ + log->seq_num = expected_seq; + log->last_lsn = le64_to_cpu(page->record_hdr.last_end_lsn); + log->ra->current_lsn = page->record_hdr.last_end_lsn; + log->l_flags &= ~NTFSLOG_NO_LAST_LSN; + + /* + * If there is room on this page for another header then + * remember we want to reuse the page. + */ + if (log->record_header_len <= + log->page_size - + le16_to_cpu(page->record_hdr.next_record_off)) { + log->l_flags |= NTFSLOG_REUSE_TAIL; + log->next_page = curpage_off; + } else { + log->l_flags &= ~NTFSLOG_REUSE_TAIL; + log->next_page = nextpage_off; + } + + /* Remember if we wrapped the log file. */ + if (wrapped_file) + log->l_flags |= NTFSLOG_WRAPPED; + } + + /* + * Remember the last page count and position. + * Also remember the last known lsn. + */ + page_cnt = le16_to_cpu(page->page_count); + page_pos = le16_to_cpu(page->page_pos); + last_ok_lsn = le64_to_cpu(page->rhdr.lsn); + +next_page_1: + + if (wrapped) { + expected_seq += 1; + wrapped_file = 1; + } + + curpage_off = nextpage_off; + kfree(page); + page = NULL; + reuse_page = 0; + goto next_page; + +check_tail: + if (tail_page) { + log->seq_num = expected_seq; + log->last_lsn = le64_to_cpu(tail_page->record_hdr.last_end_lsn); + log->ra->current_lsn = tail_page->record_hdr.last_end_lsn; + log->l_flags &= ~NTFSLOG_NO_LAST_LSN; + + if (log->page_size - + le16_to_cpu( + tail_page->record_hdr.next_record_off) >= + log->record_header_len) { + log->l_flags |= NTFSLOG_REUSE_TAIL; + log->next_page = curpage_off; + } else { + log->l_flags &= ~NTFSLOG_REUSE_TAIL; + log->next_page = nextpage_off; + } + + if (wrapped) + log->l_flags |= NTFSLOG_WRAPPED; + } + + /* Remember that the partial IO will start at the next page. */ + second_off = nextpage_off; + + /* + * If the next page is the first page of the file then update + * the sequence number for log records which begon the next page. + */ + if (wrapped) + expected_seq += 1; + + /* + * If we have a tail copy or are performing single page I/O we can + * immediately look at the next page. + */ + if (replace_page || (log->ra->flags & RESTART_SINGLE_PAGE_IO)) { + page_cnt = 2; + page_pos = 1; + goto check_valid; + } + + if (page_pos != page_cnt) + goto check_valid; + /* + * If the next page causes us to wrap to the beginning of the log + * file then we know which page to check next. + */ + if (wrapped) { + page_cnt = 2; + page_pos = 1; + goto check_valid; + } + + cur_pos = 2; + +next_test_page: + kfree(tst_page); + tst_page = NULL; + + /* Walk through the file, reading log pages. */ + err = read_log_page(log, nextpage_off, &tst_page, &usa_error); + + /* + * If we get a USA error then assume that we correctly found + * the end of the original transfer. + */ + if (usa_error) + goto file_is_valid; + + /* + * If we were able to read the page, we examine it to see if it + * is the same or different Io block. + */ + if (err) + goto next_test_page_1; + + if (le16_to_cpu(tst_page->page_pos) == cur_pos && + check_subseq_log_page(log, tst_page, nextpage_off, expected_seq)) { + page_cnt = le16_to_cpu(tst_page->page_count) + 1; + page_pos = le16_to_cpu(tst_page->page_pos); + goto check_valid; + } else { + goto file_is_valid; + } + +next_test_page_1: + + nextpage_off = next_page_off(log, curpage_off); + wrapped = nextpage_off == log->first_page; + + if (wrapped) { + expected_seq += 1; + page_cnt = 2; + page_pos = 1; + } + + cur_pos += 1; + part_io_count += 1; + if (!wrapped) + goto next_test_page; + +check_valid: + /* Skip over the remaining pages this transfer. */ + remain_pages = page_cnt - page_pos - 1; + part_io_count += remain_pages; + + while (remain_pages--) { + nextpage_off = next_page_off(log, curpage_off); + wrapped = nextpage_off == log->first_page; + + if (wrapped) + expected_seq += 1; + } + + /* Call our routine to check this log page. */ + kfree(tst_page); + tst_page = NULL; + + err = read_log_page(log, nextpage_off, &tst_page, &usa_error); + if (!err && !usa_error && + check_subseq_log_page(log, tst_page, nextpage_off, expected_seq)) { + err = -EINVAL; + goto out; + } + +file_is_valid: + + /* We have a valid file. */ + if (page_off1 || tail_page) { + struct RECORD_PAGE_HDR *tmp_page; + + if (sb_rdonly(log->ni->mi.sbi->sb)) { + err = -EROFS; + goto out; + } + + if (page_off1) { + tmp_page = Add2Ptr(page_bufs, page_off1 - page_off); + tails -= (page_off1 - page_off) / log->page_size; + if (!tail_page) + tails -= 1; + } else { + tmp_page = tail_page; + tails = 1; + } + + while (tails--) { + u64 off = hdr_file_off(log, tmp_page); + + if (!page) { + page = kmalloc(log->page_size, GFP_NOFS); + if (!page) { + err = -ENOMEM; + goto out; + } + } + + /* + * Correct page and copy the data from this page + * into it and flush it to disk. + */ + memcpy(page, tmp_page, log->page_size); + + /* Fill last flushed lsn value flush the page. */ + if (log->major_ver < 2) + page->rhdr.lsn = page->record_hdr.last_end_lsn; + else + page->file_off = 0; + + page->page_pos = page->page_count = cpu_to_le16(1); + + ntfs_fix_pre_write(&page->rhdr, log->page_size); + + err = ntfs_sb_write_run(log->ni->mi.sbi, + &log->ni->file.run, off, page, + log->page_size, 0); + + if (err) + goto out; + + if (part_io_count && second_off == off) { + second_off += log->page_size; + part_io_count -= 1; + } + + tmp_page = Add2Ptr(tmp_page, log->page_size); + } + } + + if (part_io_count) { + if (sb_rdonly(log->ni->mi.sbi->sb)) { + err = -EROFS; + goto out; + } + } + +out: + kfree(second_tail); + kfree(first_tail); + kfree(page); + kfree(tst_page); + kfree(page_bufs); + + return err; +} + +/* + * read_log_rec_buf - Copy a log record from the file to a buffer. + * + * The log record may span several log pages and may even wrap the file. + */ +static int read_log_rec_buf(struct ntfs_log *log, + const struct LFS_RECORD_HDR *rh, void *buffer) +{ + int err; + struct RECORD_PAGE_HDR *ph = NULL; + u64 lsn = le64_to_cpu(rh->this_lsn); + u32 vbo = lsn_to_vbo(log, lsn) & ~log->page_mask; + u32 off = lsn_to_page_off(log, lsn) + log->record_header_len; + u32 data_len = le32_to_cpu(rh->client_data_len); + + /* + * While there are more bytes to transfer, + * we continue to attempt to perform the read. + */ + for (;;) { + bool usa_error; + u32 tail; + + /* off comes from the on-disk restart area; bound it. */ + if (off > log->page_size) { + err = -EINVAL; + goto out; + } + + tail = log->page_size - off; + + if (tail >= data_len) + tail = data_len; + + data_len -= tail; + + err = read_log_page(log, vbo, &ph, &usa_error); + if (err) + goto out; + + /* + * The last lsn on this page better be greater or equal + * to the lsn we are copying. + */ + if (lsn > le64_to_cpu(ph->rhdr.lsn)) { + err = -EINVAL; + goto out; + } + + memcpy(buffer, Add2Ptr(ph, off), tail); + + /* If there are no more bytes to transfer, we exit the loop. */ + if (!data_len) { + if (!is_log_record_end(ph) || + lsn > le64_to_cpu(ph->record_hdr.last_end_lsn)) { + err = -EINVAL; + goto out; + } + break; + } + + if (ph->rhdr.lsn == ph->record_hdr.last_end_lsn || + lsn > le64_to_cpu(ph->rhdr.lsn)) { + err = -EINVAL; + goto out; + } + + vbo = next_page_off(log, vbo); + off = log->data_off; + + /* + * Adjust our pointer the user's buffer to transfer + * the next block to. + */ + buffer = Add2Ptr(buffer, tail); + } + +out: + kfree(ph); + return err; +} + +static int read_rst_area(struct ntfs_log *log, struct NTFS_RESTART **rst_, + u64 *lsn) +{ + int err; + struct LFS_RECORD_HDR *rh = NULL; + const struct CLIENT_REC *cr = + Add2Ptr(log->ra, le16_to_cpu(log->ra->client_off)); + u64 lsnr, lsnc = le64_to_cpu(cr->restart_lsn); + u32 len; + struct NTFS_RESTART *rst; + + *lsn = 0; + *rst_ = NULL; + + /* If the client doesn't have a restart area, go ahead and exit now. */ + if (!lsnc) + return 0; + + err = read_log_page(log, lsn_to_vbo(log, lsnc), + (struct RECORD_PAGE_HDR **)&rh, NULL); + if (err) + return err; + + rst = NULL; + lsnr = le64_to_cpu(rh->this_lsn); + + if (lsnc != lsnr) { + /* If the lsn values don't match, then the disk is corrupt. */ + err = -EINVAL; + goto out; + } + + *lsn = lsnr; + len = le32_to_cpu(rh->client_data_len); + + if (!len) { + err = 0; + goto out; + } + + if (len < sizeof(struct NTFS_RESTART)) { + err = -EINVAL; + goto out; + } + + rst = kmalloc(len, GFP_NOFS); + if (!rst) { + err = -ENOMEM; + goto out; + } + + /* Copy the data into the 'rst' buffer. */ + err = read_log_rec_buf(log, rh, rst); + if (err) + goto out; + + *rst_ = rst; + rst = NULL; + +out: + kfree(rh); + kfree(rst); + + return err; +} + +static int find_log_rec(struct ntfs_log *log, u64 lsn, struct lcb *lcb) +{ + int err; + struct LFS_RECORD_HDR *rh = lcb->lrh; + u32 rec_len, len; + + /* Read the record header for this lsn. */ + if (!rh) { + err = read_log_page(log, lsn_to_vbo(log, lsn), + (struct RECORD_PAGE_HDR **)&rh, NULL); + + lcb->lrh = rh; + if (err) + return err; + } + + /* + * If the lsn the log record doesn't match the desired + * lsn then the disk is corrupt. + */ + if (lsn != le64_to_cpu(rh->this_lsn)) + return -EINVAL; + + len = le32_to_cpu(rh->client_data_len); + + /* + * Check that the length field isn't greater than the total + * available space the log file. + */ + rec_len = len + log->record_header_len; + if (rec_len >= log->total_avail) + return -EINVAL; + + /* + * If the entire log record is on this log page, + * put a pointer to the log record the context block. + */ + if (rh->flags & LOG_RECORD_MULTI_PAGE) { + void *lr = kmalloc(len, GFP_NOFS); + + if (!lr) + return -ENOMEM; + + lcb->log_rec = lr; + lcb->alloc = true; + + /* Copy the data into the buffer returned. */ + err = read_log_rec_buf(log, rh, lr); + if (err) + return err; + } else { + /* If beyond the end of the current page -> an error. */ + u32 page_off = lsn_to_page_off(log, lsn); + + if (page_off + len + log->record_header_len > log->page_size) + return -EINVAL; + + lcb->log_rec = Add2Ptr(rh, sizeof(struct LFS_RECORD_HDR)); + lcb->alloc = false; + } + + return 0; +} + +/* + * read_log_rec_lcb - Init the query operation. + */ +static int read_log_rec_lcb(struct ntfs_log *log, u64 lsn, u32 ctx_mode, + struct lcb **lcb_) +{ + int err; + const struct CLIENT_REC *cr; + struct lcb *lcb; + + switch (ctx_mode) { + case lcb_ctx_undo_next: + case lcb_ctx_prev: + case lcb_ctx_next: + break; + default: + return -EINVAL; + } + + /* Check that the given lsn is the legal range for this client. */ + cr = Add2Ptr(log->ra, le16_to_cpu(log->ra->client_off)); + + if (!verify_client_lsn(log, cr, lsn)) + return -EINVAL; + + lcb = kzalloc_obj(struct lcb, GFP_NOFS); + if (!lcb) + return -ENOMEM; + lcb->client = log->client_id; + lcb->ctx_mode = ctx_mode; + + /* Find the log record indicated by the given lsn. */ + err = find_log_rec(log, lsn, lcb); + if (err) + goto out; + + *lcb_ = lcb; + return 0; + +out: + lcb_put(lcb); + *lcb_ = NULL; + return err; +} + +/* + * find_client_next_lsn + * + * Attempt to find the next lsn to return to a client based on the context mode. + */ +static int find_client_next_lsn(struct ntfs_log *log, struct lcb *lcb, u64 *lsn) +{ + int err; + u64 next_lsn; + struct LFS_RECORD_HDR *hdr; + + hdr = lcb->lrh; + *lsn = 0; + + if (lcb_ctx_next != lcb->ctx_mode) + goto check_undo_next; + + /* Loop as long as another lsn can be found. */ + for (;;) { + u64 current_lsn; + + err = next_log_lsn(log, hdr, ¤t_lsn); + if (err) + goto out; + + if (!current_lsn) + break; + + if (hdr != lcb->lrh) + kfree(hdr); + + hdr = NULL; + err = read_log_page(log, lsn_to_vbo(log, current_lsn), + (struct RECORD_PAGE_HDR **)&hdr, NULL); + if (err) + goto out; + + if (memcmp(&hdr->client, &lcb->client, + sizeof(struct CLIENT_ID))) { + /*err = -EINVAL; */ + } else if (LfsClientRecord == hdr->record_type) { + kfree(lcb->lrh); + lcb->lrh = hdr; + *lsn = current_lsn; + return 0; + } + } + +out: + if (hdr != lcb->lrh) + kfree(hdr); + return err; + +check_undo_next: + if (lcb_ctx_undo_next == lcb->ctx_mode) + next_lsn = le64_to_cpu(hdr->client_undo_next_lsn); + else if (lcb_ctx_prev == lcb->ctx_mode) + next_lsn = le64_to_cpu(hdr->client_prev_lsn); + else + return 0; + + if (!next_lsn) + return 0; + + if (!verify_client_lsn( + log, Add2Ptr(log->ra, le16_to_cpu(log->ra->client_off)), + next_lsn)) + return 0; + + hdr = NULL; + err = read_log_page(log, lsn_to_vbo(log, next_lsn), + (struct RECORD_PAGE_HDR **)&hdr, NULL); + if (err) + return err; + kfree(lcb->lrh); + lcb->lrh = hdr; + + *lsn = next_lsn; + + return 0; +} + +static int read_next_log_rec(struct ntfs_log *log, struct lcb *lcb, u64 *lsn) +{ + int err; + + err = find_client_next_lsn(log, lcb, lsn); + if (err) + return err; + + if (!*lsn) + return 0; + + if (lcb->alloc) + kfree(lcb->log_rec); + + lcb->log_rec = NULL; + lcb->alloc = false; + kfree(lcb->lrh); + lcb->lrh = NULL; + + return find_log_rec(log, *lsn, lcb); +} + +bool check_index_header(const struct INDEX_HDR *hdr, size_t bytes) +{ + const bool has_subnode = hdr_has_subnode(hdr); + __le16 mask; + u32 min_de, de_off, used, total; + + if (has_subnode) { + min_de = sizeof(struct NTFS_DE) + sizeof(u64); + mask = NTFS_IE_HAS_SUBNODES; + } else { + min_de = sizeof(struct NTFS_DE); + mask = 0; + } + + de_off = le32_to_cpu(hdr->de_off); + used = le32_to_cpu(hdr->used); + total = le32_to_cpu(hdr->total); + + if (de_off > bytes - min_de || used > bytes || total > bytes || + de_off + min_de > used || used > total) { + return false; + } + + for (;;) { + const struct NTFS_DE *e = Add2Ptr(hdr, de_off); + u16 esize = le16_to_cpu(e->size); + u16 key_size = le16_to_cpu(e->key_size); + u16 data_size; + + if (!IS_ALIGNED(esize, 8) || esize < min_de || + (e->flags & NTFS_IE_HAS_SUBNODES) != mask) { + return false; + } + + if (size_add(de_off, esize) > used) + return false; + + if (de_is_last(e)) { + if (key_size) + return false; + break; + } + + data_size = esize - min_de; + if (key_size > data_size) + return false; + + de_off += esize; + } + + return true; +} + +static inline bool check_index_buffer(const struct INDEX_BUFFER *ib, u32 bytes) +{ + u16 fo; + const struct NTFS_RECORD_HEADER *r = &ib->rhdr; + + if (r->sign != NTFS_INDX_SIGNATURE) + return false; + + fo = (SECTOR_SIZE - ((bytes >> SECTOR_SHIFT) + 1) * sizeof(short)); + + if (le16_to_cpu(r->fix_off) > fo) + return false; + + if ((le16_to_cpu(r->fix_num) - 1) * SECTOR_SIZE != bytes) + return false; + + return check_index_header(&ib->ihdr, + bytes - offsetof(struct INDEX_BUFFER, ihdr)); +} + +static inline bool check_index_root(const struct ATTRIB *attr, + struct ntfs_sb_info *sbi) +{ + bool ret; + const struct INDEX_ROOT *root = resident_data(attr); + u8 index_bits = le32_to_cpu(root->index_block_size) >= + sbi->cluster_size ? + sbi->cluster_bits : + SECTOR_SHIFT; + u8 block_clst = root->index_block_clst; + + if (le32_to_cpu(attr->res.data_size) < sizeof(struct INDEX_ROOT) || + (root->type != ATTR_NAME && root->type != ATTR_ZERO) || + (root->type == ATTR_NAME && + root->rule != NTFS_COLLATION_TYPE_FILENAME) || + (le32_to_cpu(root->index_block_size) != + (block_clst << index_bits)) || + (block_clst != 1 && block_clst != 2 && block_clst != 4 && + block_clst != 8 && block_clst != 0x10 && block_clst != 0x20 && + block_clst != 0x40 && block_clst != 0x80)) { + return false; + } + + ret = check_index_header(&root->ihdr, + le32_to_cpu(attr->res.data_size) - + offsetof(struct INDEX_ROOT, ihdr)); + return ret; +} + +static inline bool check_attr(const struct MFT_REC *rec, + const struct ATTRIB *attr, + struct ntfs_sb_info *sbi) +{ + u32 asize = le32_to_cpu(attr->size); + u32 rsize = 0; + u64 dsize, svcn, evcn; + u16 run_off; + + /* Check the fixed part of the attribute record header. */ + if (asize >= sbi->record_size || + asize + PtrOffset(rec, attr) >= sbi->record_size || + (attr->name_len && + le16_to_cpu(attr->name_off) + attr->name_len * sizeof(short) > + asize)) { + return false; + } + + /* Check the attribute fields. */ + switch (attr->non_res) { + case 0: + rsize = le32_to_cpu(attr->res.data_size); + if (rsize >= asize || + le16_to_cpu(attr->res.data_off) + rsize > asize) { + return false; + } + break; + + case 1: + dsize = le64_to_cpu(attr->nres.data_size); + svcn = le64_to_cpu(attr->nres.svcn); + evcn = le64_to_cpu(attr->nres.evcn); + run_off = le16_to_cpu(attr->nres.run_off); + + if (svcn > evcn + 1 || run_off >= asize || + le64_to_cpu(attr->nres.valid_size) > dsize || + dsize > le64_to_cpu(attr->nres.alloc_size)) { + return false; + } + + if (run_off > asize) + return false; + + if (run_unpack(NULL, sbi, 0, svcn, evcn, svcn, + Add2Ptr(attr, run_off), asize - run_off) < 0) { + return false; + } + + return true; + + default: + return false; + } + + switch (attr->type) { + case ATTR_NAME: + if (fname_full_size(Add2Ptr( + attr, le16_to_cpu(attr->res.data_off))) > asize) { + return false; + } + break; + + case ATTR_ROOT: + return check_index_root(attr, sbi); + + case ATTR_STD: + if (rsize < sizeof(struct ATTR_STD_INFO5) && + rsize != sizeof(struct ATTR_STD_INFO)) { + return false; + } + break; + + case ATTR_LIST: + case ATTR_ID: + case ATTR_SECURE: + case ATTR_LABEL: + case ATTR_VOL_INFO: + case ATTR_DATA: + case ATTR_ALLOC: + case ATTR_BITMAP: + case ATTR_REPARSE: + case ATTR_EA_INFO: + case ATTR_EA: + case ATTR_PROPERTYSET: + case ATTR_LOGGED_UTILITY_STREAM: + break; + + default: + return false; + } + + return true; +} + +static inline bool check_file_record(const struct MFT_REC *rec, + const struct MFT_REC *rec2, + struct ntfs_sb_info *sbi) +{ + const struct ATTRIB *attr; + u16 fo = le16_to_cpu(rec->rhdr.fix_off); + u16 fn = le16_to_cpu(rec->rhdr.fix_num); + u16 ao = le16_to_cpu(rec->attr_off); + u32 rs = sbi->record_size; + u32 used = le32_to_cpu(rec->used); + + /* Check the file record header for consistency. */ + if (rec->rhdr.sign != NTFS_FILE_SIGNATURE || + fo > (SECTOR_SIZE - ((rs >> SECTOR_SHIFT) + 1) * sizeof(short)) || + (fn - 1) * SECTOR_SIZE != rs || ao < MFTRECORD_FIXUP_OFFSET_1 || + ao > sbi->record_size - SIZEOF_RESIDENT || !is_rec_inuse(rec) || + le32_to_cpu(rec->total) != rs || used > rs || used < ao) { + return false; + } + + /* Loop to check all of the attributes. */ + for (attr = Add2Ptr(rec, ao); attr->type != ATTR_END; + attr = Add2Ptr(attr, le32_to_cpu(attr->size))) { + if (check_attr(rec, attr, sbi)) + continue; + return false; + } + + /* + * The do_action() handlers compute memmove lengths as + * "rec->used - ", which underflows when + * rec->used is smaller than the attribute walk reached. At this + * point attr is the ATTR_END marker; rec->used must cover it. + */ + if (used < PtrOffset(rec, attr) + sizeof(attr->type)) + return false; + + return true; +} + +static inline int check_lsn(const struct NTFS_RECORD_HEADER *hdr, + const u64 *rlsn) +{ + u64 lsn; + + if (!rlsn) + return true; + + lsn = le64_to_cpu(hdr->lsn); + + if (hdr->sign == NTFS_HOLE_SIGNATURE) + return false; + + if (*rlsn > lsn) + return true; + + return false; +} + +static inline bool check_if_attr(const struct MFT_REC *rec, + const struct LOG_REC_HDR *lrh) +{ + u16 ro = le16_to_cpu(lrh->record_off); + u16 o = le16_to_cpu(rec->attr_off); + const struct ATTRIB *attr = Add2Ptr(rec, o); + + while (o < ro) { + u32 asize; + + if (attr->type == ATTR_END) + break; + + asize = le32_to_cpu(attr->size); + if (!asize) + break; + + o += asize; + attr = Add2Ptr(attr, asize); + } + + return o == ro; +} + +static inline bool check_if_index_root(const struct MFT_REC *rec, + const struct LOG_REC_HDR *lrh) +{ + u16 ro = le16_to_cpu(lrh->record_off); + u16 o = le16_to_cpu(rec->attr_off); + const struct ATTRIB *attr = Add2Ptr(rec, o); + + while (o < ro) { + u32 asize; + + if (attr->type == ATTR_END) + break; + + asize = le32_to_cpu(attr->size); + if (!asize) + break; + + o += asize; + attr = Add2Ptr(attr, asize); + } + + return o == ro && attr->type == ATTR_ROOT; +} + +static inline bool check_if_root_index(const struct ATTRIB *attr, + const struct INDEX_HDR *hdr, + const struct LOG_REC_HDR *lrh) +{ + u16 ao = le16_to_cpu(lrh->attr_off); + u32 de_off = le32_to_cpu(hdr->de_off); + u32 o = PtrOffset(attr, hdr) + de_off; + const struct NTFS_DE *e = Add2Ptr(hdr, de_off); + u32 asize = le32_to_cpu(attr->size); + + while (o < ao) { + u16 esize; + + if (o >= asize) + break; + + esize = le16_to_cpu(e->size); + if (!esize) + break; + + o += esize; + e = Add2Ptr(e, esize); + } + + return o == ao; +} + +static inline bool check_if_alloc_index(const struct INDEX_HDR *hdr, + u32 attr_off) +{ + u32 de_off = le32_to_cpu(hdr->de_off); + u32 o = offsetof(struct INDEX_BUFFER, ihdr) + de_off; + const struct NTFS_DE *e = Add2Ptr(hdr, de_off); + u32 used = le32_to_cpu(hdr->used); + + while (o < attr_off) { + u16 esize; + + if (de_off >= used) + break; + + esize = le16_to_cpu(e->size); + if (!esize) + break; + + o += esize; + de_off += esize; + e = Add2Ptr(e, esize); + } + + return o == attr_off; +} + +static inline void change_attr_size(struct MFT_REC *rec, struct ATTRIB *attr, + u32 nsize) +{ + u32 asize = le32_to_cpu(attr->size); + int dsize = nsize - asize; + u8 *next = Add2Ptr(attr, asize); + u32 used = le32_to_cpu(rec->used); + + memmove(Add2Ptr(attr, nsize), next, used - PtrOffset(rec, next)); + + rec->used = cpu_to_le32(used + dsize); + attr->size = cpu_to_le32(nsize); +} + +struct OpenAttr { + struct ATTRIB *attr; + struct runs_tree *run1; + struct runs_tree run0; + struct ntfs_inode *ni; + // CLST rno; +}; + +/* + * cmp_type_and_name + * + * Return: 0 if 'attr' has the same type and name. + */ +static inline int cmp_type_and_name(const struct ATTRIB *a1, + const struct ATTRIB *a2) +{ + return a1->type != a2->type || a1->name_len != a2->name_len || + (a1->name_len && memcmp(attr_name(a1), attr_name(a2), + a1->name_len * sizeof(short))); +} + +static struct OpenAttr *find_loaded_attr(struct ntfs_log *log, + const struct ATTRIB *attr, CLST rno) +{ + struct OPEN_ATTR_ENRTY *oe = NULL; + + while ((oe = enum_rstbl(log->open_attr_tbl, oe))) { + struct OpenAttr *op_attr; + + if (ino_get(&oe->ref) != rno) + continue; + + op_attr = (struct OpenAttr *)oe->ptr; + if (!cmp_type_and_name(op_attr->attr, attr)) + return op_attr; + } + return NULL; +} + +static struct ATTRIB *attr_create_nonres_log(struct ntfs_sb_info *sbi, + enum ATTR_TYPE type, u64 size, + const u16 *name, size_t name_len, + __le16 flags) +{ + struct ATTRIB *attr; + u32 name_size = ALIGN(name_len * sizeof(short), 8); + bool is_ext = flags & (ATTR_FLAG_COMPRESSED | ATTR_FLAG_SPARSED); + u32 asize = name_size + + (is_ext ? SIZEOF_NONRESIDENT_EX : SIZEOF_NONRESIDENT); + + attr = kzalloc(asize, GFP_NOFS); + if (!attr) + return NULL; + + attr->type = type; + attr->size = cpu_to_le32(asize); + attr->flags = flags; + attr->non_res = 1; + attr->name_len = name_len; + + attr->nres.evcn = cpu_to_le64((u64)bytes_to_cluster(sbi, size) - 1); + attr->nres.alloc_size = cpu_to_le64(ntfs_up_cluster(sbi, size)); + attr->nres.data_size = cpu_to_le64(size); + attr->nres.valid_size = attr->nres.data_size; + if (is_ext) { + attr->name_off = SIZEOF_NONRESIDENT_EX_LE; + if (is_attr_compressed(attr)) + attr->nres.c_unit = NTFS_LZNT_CUNIT; + + attr->nres.run_off = + cpu_to_le16(SIZEOF_NONRESIDENT_EX + name_size); + memcpy(Add2Ptr(attr, SIZEOF_NONRESIDENT_EX), name, + name_len * sizeof(short)); + } else { + attr->name_off = SIZEOF_NONRESIDENT_LE; + attr->nres.run_off = + cpu_to_le16(SIZEOF_NONRESIDENT + name_size); + memcpy(Add2Ptr(attr, SIZEOF_NONRESIDENT), name, + name_len * sizeof(short)); + } + + return attr; +} + +/* + * update_oa_attr - Synchronize OpenAttr's attribute pointer with modified attribute + * @oa2: OpenAttr structure in memory that needs to be updated + * @attr: Modified attribute from MFT record to duplicate + * + * Returns true on success, false on allocation failure. + */ +static bool update_oa_attr(struct OpenAttr *oa2, struct ATTRIB *attr) +{ + void *p2; + + p2 = kmemdup(attr, le32_to_cpu(attr->size), GFP_NOFS); + if (p2) { + kfree(oa2->attr); + oa2->attr = p2; + return true; + } + return false; +} + +/* + * do_action - Common routine for the Redo and Undo Passes. + * @rlsn: If it is NULL then undo. + */ +static int do_action(struct ntfs_log *log, struct OPEN_ATTR_ENRTY *oe, + const struct LOG_REC_HDR *lrh, u32 op, void *data, + u32 dlen, u32 rec_len, const u64 *rlsn) +{ + int err = 0; + struct ntfs_sb_info *sbi = log->ni->mi.sbi; + struct inode *inode = NULL, *inode_parent; + struct mft_inode *mi = NULL, *mi2_child = NULL; + CLST rno = 0, rno_base = 0; + struct INDEX_BUFFER *ib = NULL; + struct MFT_REC *rec = NULL; + struct ATTRIB *attr = NULL, *attr2; + struct INDEX_HDR *hdr; + struct INDEX_ROOT *root; + struct NTFS_DE *e, *e1, *e2; + struct NEW_ATTRIBUTE_SIZES *new_sz; + struct ATTR_FILE_NAME *fname; + struct OpenAttr *oa, *oa2; + u32 nsize, t32, asize, used, esize, off, bits; + u16 id, id2; + u32 record_size = sbi->record_size; + u64 t64; + u16 roff = le16_to_cpu(lrh->record_off); + u16 aoff = le16_to_cpu(lrh->attr_off); + u64 lco = 0; + u64 cbo = (u64)le16_to_cpu(lrh->cluster_off) << SECTOR_SHIFT; + u64 tvo = le64_to_cpu(lrh->target_vcn) << sbi->cluster_bits; + u64 vbo = cbo + tvo; + void *buffer_le = NULL; + u32 bytes = 0; + bool a_dirty = false; + u16 data_off; + + oa = oe->ptr; + + /* Big switch to prepare. */ + switch (op) { + /* ============================================================ + * Process MFT records, as described by the current log record. + * ============================================================ + */ + case InitializeFileRecordSegment: + case DeallocateFileRecordSegment: + case WriteEndOfFileRecordSegment: + case CreateAttribute: + case DeleteAttribute: + case UpdateResidentValue: + case UpdateMappingPairs: + case SetNewAttributeSizes: + case AddIndexEntryRoot: + case DeleteIndexEntryRoot: + case SetIndexEntryVcnRoot: + case UpdateFileNameRoot: + case UpdateRecordDataRoot: + case ZeroEndOfFileRecord: + rno = vbo >> sbi->record_bits; + inode = ilookup(sbi->sb, rno); + if (inode) { + mi = &ntfs_i(inode)->mi; + } else { + /* Read from disk. */ + err = mi_get(sbi, rno, &mi); + if (err && op == InitializeFileRecordSegment) { + mi = kzalloc_obj(struct mft_inode, GFP_NOFS); + if (!mi) + return -ENOMEM; + err = mi_format_new(mi, sbi, rno, 0, false); + } + if (err) + return err; + } + rec = mi->mrec; + + if (op == DeallocateFileRecordSegment) + goto skip_load_parent; + + if (rec->rhdr.sign == NTFS_BAAD_SIGNATURE) + goto dirty_vol; + if (!check_lsn(&rec->rhdr, rlsn)) + goto out; + if (!check_file_record(rec, NULL, sbi)) + goto dirty_vol; + attr = Add2Ptr(rec, roff); + + if (is_rec_base(rec) || InitializeFileRecordSegment == op) { + rno_base = rno; + goto skip_load_parent; + } + + rno_base = ino_get(&rec->parent_ref); + inode_parent = ntfs_iget5(sbi->sb, &rec->parent_ref, NULL); + if (IS_ERR(inode_parent)) + goto skip_load_parent; + + if (is_bad_inode(inode_parent)) { + iput(inode_parent); + goto skip_load_parent; + } + + if (ni_load_mi_ex(ntfs_i(inode_parent), rno, &mi2_child)) { + iput(inode_parent); + } else { + if (mi2_child->mrec != mi->mrec) + memcpy(mi2_child->mrec, mi->mrec, + sbi->record_size); + + if (inode) + iput(inode); + else + mi_put(mi); + + inode = inode_parent; + mi = mi2_child; + rec = mi2_child->mrec; + attr = Add2Ptr(rec, roff); + } + +skip_load_parent: + inode_parent = NULL; + break; + + /* + * Process attributes, as described by the current log record. + */ + case UpdateNonresidentValue: + case AddIndexEntryAllocation: + case DeleteIndexEntryAllocation: + case WriteEndOfIndexBuffer: + case SetIndexEntryVcnAllocation: + case UpdateFileNameAllocation: + case SetBitsInNonresidentBitMap: + case ClearBitsInNonresidentBitMap: + case UpdateRecordDataAllocation: + attr = oa->attr; + bytes = UpdateNonresidentValue == op ? dlen : 0; + lco = (u64)le16_to_cpu(lrh->lcns_follow) << sbi->cluster_bits; + + if (attr->type == ATTR_ALLOC) { + t32 = le32_to_cpu(oe->bytes_per_index); + if (bytes < t32) + bytes = t32; + } + + if (!bytes) + bytes = lco - cbo; + + bytes += roff; + if (attr->type == ATTR_ALLOC) + bytes = (bytes + 511) & ~511; // align + + buffer_le = kmalloc(bytes, GFP_NOFS); + if (!buffer_le) + return -ENOMEM; + + err = ntfs_read_run_nb(sbi, oa->run1, vbo, buffer_le, bytes, + NULL); + if (err) + goto out; + + if (attr->type == ATTR_ALLOC && *(int *)buffer_le) + ntfs_fix_post_read(buffer_le, bytes, false); + break; + + default: + WARN_ON(1); + } + + /* Big switch to do operation. */ + switch (op) { + case InitializeFileRecordSegment: + if (roff + dlen > record_size) + goto dirty_vol; + + memcpy(Add2Ptr(rec, roff), data, dlen); + mi->dirty = true; + break; + + case DeallocateFileRecordSegment: + clear_rec_inuse(rec); + le16_add_cpu(&rec->seq, 1); + mi->dirty = true; + break; + + case WriteEndOfFileRecordSegment: + attr2 = (struct ATTRIB *)data; + if (!check_if_attr(rec, lrh) || roff + dlen > record_size) + goto dirty_vol; + + memmove(attr, attr2, dlen); + rec->used = cpu_to_le32(ALIGN(roff + dlen, 8)); + + mi->dirty = true; + break; + + case CreateAttribute: + attr2 = (struct ATTRIB *)data; + asize = le32_to_cpu(attr2->size); + used = le32_to_cpu(rec->used); + + if (!check_if_attr(rec, lrh) || dlen < SIZEOF_RESIDENT || + !IS_ALIGNED(asize, 8) || + Add2Ptr(attr2, asize) > Add2Ptr(lrh, rec_len) || + dlen > record_size - used) { + goto dirty_vol; + } + + memmove(Add2Ptr(attr, asize), attr, used - roff); + memcpy(attr, attr2, asize); + + rec->used = cpu_to_le32(used + asize); + id = le16_to_cpu(rec->next_attr_id); + id2 = le16_to_cpu(attr2->id); + if (id <= id2) + rec->next_attr_id = cpu_to_le16(id2 + 1); + if (is_attr_indexed(attr)) + le16_add_cpu(&rec->hard_links, 1); + + oa2 = find_loaded_attr(log, attr, rno_base); + if (oa2) + update_oa_attr(oa2, attr); + + mi->dirty = true; + break; + + case DeleteAttribute: + asize = le32_to_cpu(attr->size); + used = le32_to_cpu(rec->used); + + if (!check_if_attr(rec, lrh)) + goto dirty_vol; + + rec->used = cpu_to_le32(used - asize); + if (is_attr_indexed(attr)) + le16_add_cpu(&rec->hard_links, -1); + + memmove(attr, Add2Ptr(attr, asize), used - asize - roff); + + mi->dirty = true; + break; + + case UpdateResidentValue: + nsize = aoff + dlen; + + if (!check_if_attr(rec, lrh)) + goto dirty_vol; + + asize = le32_to_cpu(attr->size); + used = le32_to_cpu(rec->used); + + if (lrh->redo_len == lrh->undo_len) { + if (nsize > asize) + goto dirty_vol; + goto move_data; + } + + if (nsize > asize && nsize - asize > record_size - used) + goto dirty_vol; + + nsize = ALIGN(nsize, 8); + data_off = le16_to_cpu(attr->res.data_off); + + /* + * aoff comes from the on-disk lrh->attr_off. Forbid + * writes that begin below the resident attribute's + * data_off (which would overwrite the resident header), + * and forbid aoff + dlen < data_off, which would make + * the data_size assignment below underflow to ~4 GiB. + */ + if (aoff < data_off || aoff + dlen < data_off || + aoff + dlen > asize) + goto dirty_vol; + + if (nsize < asize) { + memmove(Add2Ptr(attr, aoff), data, dlen); + data = NULL; // To skip below memmove(). + } + + memmove(Add2Ptr(attr, nsize), Add2Ptr(attr, asize), + used - le16_to_cpu(lrh->record_off) - asize); + + rec->used = cpu_to_le32(used + nsize - asize); + attr->size = cpu_to_le32(nsize); + attr->res.data_size = cpu_to_le32(aoff + dlen - data_off); + +move_data: + if (data) + memmove(Add2Ptr(attr, aoff), data, dlen); + + oa2 = find_loaded_attr(log, attr, rno_base); + if (oa2 && update_oa_attr(oa2, attr)) + oa2->run1 = &oa2->run0; + + mi->dirty = true; + break; + + case UpdateMappingPairs: + nsize = aoff + dlen; + asize = le32_to_cpu(attr->size); + used = le32_to_cpu(rec->used); + + if (!check_if_attr(rec, lrh) || !attr->non_res || + aoff < le16_to_cpu(attr->nres.run_off) || aoff > asize || + (nsize > asize && nsize - asize > record_size - used)) { + goto dirty_vol; + } + + nsize = ALIGN(nsize, 8); + + memmove(Add2Ptr(attr, nsize), Add2Ptr(attr, asize), + used - le16_to_cpu(lrh->record_off) - asize); + rec->used = cpu_to_le32(used + nsize - asize); + attr->size = cpu_to_le32(nsize); + memmove(Add2Ptr(attr, aoff), data, dlen); + + if (run_get_highest_vcn(le64_to_cpu(attr->nres.svcn), + attr_run(attr), + le32_to_cpu(attr->size) - + le16_to_cpu(attr->nres.run_off), + &t64)) { + goto dirty_vol; + } + + attr->nres.evcn = cpu_to_le64(t64); + oa2 = find_loaded_attr(log, attr, rno_base); + if (oa2 && oa2->attr->non_res) + oa2->attr->nres.evcn = attr->nres.evcn; + + mi->dirty = true; + break; + + case SetNewAttributeSizes: + new_sz = data; + if (!check_if_attr(rec, lrh) || !attr->non_res) + goto dirty_vol; + + attr->nres.alloc_size = new_sz->alloc_size; + attr->nres.data_size = new_sz->data_size; + attr->nres.valid_size = new_sz->valid_size; + + if (dlen >= sizeof(struct NEW_ATTRIBUTE_SIZES)) + attr->nres.total_size = new_sz->total_size; + + oa2 = find_loaded_attr(log, attr, rno_base); + if (oa2) + update_oa_attr(oa2, attr); + + mi->dirty = true; + break; + + case AddIndexEntryRoot: + e = (struct NTFS_DE *)data; + esize = le16_to_cpu(e->size); + root = resident_data(attr); + hdr = &root->ihdr; + used = le32_to_cpu(hdr->used); + + if (!check_if_index_root(rec, lrh) || + !check_if_root_index(attr, hdr, lrh) || + Add2Ptr(data, esize) > Add2Ptr(lrh, rec_len) || + esize > le32_to_cpu(rec->total) - le32_to_cpu(rec->used)) { + goto dirty_vol; + } + + e1 = Add2Ptr(attr, le16_to_cpu(lrh->attr_off)); + + change_attr_size(rec, attr, le32_to_cpu(attr->size) + esize); + + memmove(Add2Ptr(e1, esize), e1, + PtrOffset(e1, Add2Ptr(hdr, used))); + memmove(e1, e, esize); + + le32_add_cpu(&attr->res.data_size, esize); + hdr->used = cpu_to_le32(used + esize); + le32_add_cpu(&hdr->total, esize); + + mi->dirty = true; + break; + + case DeleteIndexEntryRoot: + root = resident_data(attr); + hdr = &root->ihdr; + used = le32_to_cpu(hdr->used); + + if (!check_if_index_root(rec, lrh) || + !check_if_root_index(attr, hdr, lrh)) { + goto dirty_vol; + } + + e1 = Add2Ptr(attr, le16_to_cpu(lrh->attr_off)); + esize = le16_to_cpu(e1->size); + if (PtrOffset(e1, Add2Ptr(hdr, used)) < esize) + goto dirty_vol; + + e2 = Add2Ptr(e1, esize); + + memmove(e1, e2, PtrOffset(e2, Add2Ptr(hdr, used))); + + le32_sub_cpu(&attr->res.data_size, esize); + hdr->used = cpu_to_le32(used - esize); + le32_sub_cpu(&hdr->total, esize); + + change_attr_size(rec, attr, le32_to_cpu(attr->size) - esize); + + mi->dirty = true; + break; + + case SetIndexEntryVcnRoot: + root = resident_data(attr); + hdr = &root->ihdr; + + if (!check_if_index_root(rec, lrh) || + !check_if_root_index(attr, hdr, lrh)) { + goto dirty_vol; + } + + e = Add2Ptr(attr, le16_to_cpu(lrh->attr_off)); + + de_set_vbn_le(e, *(__le64 *)data); + mi->dirty = true; + break; + + case UpdateFileNameRoot: + root = resident_data(attr); + hdr = &root->ihdr; + + if (!check_if_index_root(rec, lrh) || + !check_if_root_index(attr, hdr, lrh)) { + goto dirty_vol; + } + + e = Add2Ptr(attr, le16_to_cpu(lrh->attr_off)); + fname = (struct ATTR_FILE_NAME *)(e + 1); + memmove(&fname->dup, data, sizeof(fname->dup)); // + mi->dirty = true; + break; + + case UpdateRecordDataRoot: + root = resident_data(attr); + hdr = &root->ihdr; + + if (!check_if_index_root(rec, lrh) || + !check_if_root_index(attr, hdr, lrh)) { + goto dirty_vol; + } + + e = Add2Ptr(attr, le16_to_cpu(lrh->attr_off)); + + /* + * e->view.data_off and dlen come from the on-disk + * INDEX_ROOT entry / LRH. The neighbouring read sites + * (e.g. fs/ntfs3/index.c) check that + * view.data_off + view.data_size <= e->size; mirror that + * bound here so the memmove cannot reach past the entry. + */ + if (le16_to_cpu(e->view.data_off) > le16_to_cpu(e->size) || + le16_to_cpu(e->view.data_off) + dlen > le16_to_cpu(e->size)) + goto dirty_vol; + + memmove(Add2Ptr(e, le16_to_cpu(e->view.data_off)), data, dlen); + + mi->dirty = true; + break; + + case ZeroEndOfFileRecord: + if (roff + dlen > record_size) + goto dirty_vol; + + memset(attr, 0, dlen); + mi->dirty = true; + break; + + case UpdateNonresidentValue: + if (lco < cbo + roff + dlen) + goto dirty_vol; + + memcpy(Add2Ptr(buffer_le, roff), data, dlen); + + a_dirty = true; + if (attr->type == ATTR_ALLOC) + ntfs_fix_pre_write(buffer_le, bytes); + break; + + case AddIndexEntryAllocation: + ib = Add2Ptr(buffer_le, roff); + hdr = &ib->ihdr; + e = data; + esize = le16_to_cpu(e->size); + e1 = Add2Ptr(ib, aoff); + + if (is_baad(&ib->rhdr)) + goto dirty_vol; + if (!check_lsn(&ib->rhdr, rlsn)) + goto out; + + used = le32_to_cpu(hdr->used); + + if (!check_index_buffer(ib, bytes) || + !check_if_alloc_index(hdr, aoff) || + Add2Ptr(e, esize) > Add2Ptr(lrh, rec_len) || + used + esize > le32_to_cpu(hdr->total)) { + goto dirty_vol; + } + + memmove(Add2Ptr(e1, esize), e1, + PtrOffset(e1, Add2Ptr(hdr, used))); + memcpy(e1, e, esize); + + hdr->used = cpu_to_le32(used + esize); + + a_dirty = true; + + ntfs_fix_pre_write(&ib->rhdr, bytes); + break; + + case DeleteIndexEntryAllocation: + ib = Add2Ptr(buffer_le, roff); + hdr = &ib->ihdr; + e = Add2Ptr(ib, aoff); + esize = le16_to_cpu(e->size); + + if (is_baad(&ib->rhdr)) + goto dirty_vol; + if (!check_lsn(&ib->rhdr, rlsn)) + goto out; + + if (!check_index_buffer(ib, bytes) || + !check_if_alloc_index(hdr, aoff)) { + goto dirty_vol; + } + + e1 = Add2Ptr(e, esize); + used = le32_to_cpu(hdr->used); + + /* + * Reject crafted entries whose e->size makes e + esize + * point past the INDEX_HDR's used boundary. Without this, + * PtrOffset(e1, hdr + used) underflows to a quasi-infinite + * size_t when fed to the memmove() below. + * + * Also reject esize == 0: memmove(e, e, ...) is a no-op and + * leaves hdr->used unchanged, masking the crafted entry. + */ + if (!esize || Add2Ptr(e, esize) > Add2Ptr(hdr, used) || + PtrOffset(e1, Add2Ptr(hdr, used)) < esize) + goto dirty_vol; + + nsize = esize; + + memmove(e, e1, PtrOffset(e1, Add2Ptr(hdr, used))); + + hdr->used = cpu_to_le32(used - nsize); + + a_dirty = true; + + ntfs_fix_pre_write(&ib->rhdr, bytes); + break; + + case WriteEndOfIndexBuffer: + ib = Add2Ptr(buffer_le, roff); + hdr = &ib->ihdr; + e = Add2Ptr(ib, aoff); + + if (is_baad(&ib->rhdr)) + goto dirty_vol; + if (!check_lsn(&ib->rhdr, rlsn)) + goto out; + if (!check_index_buffer(ib, bytes) || + !check_if_alloc_index(hdr, aoff) || + aoff + dlen > offsetof(struct INDEX_BUFFER, ihdr) + + le32_to_cpu(hdr->total)) { + goto dirty_vol; + } + + hdr->used = cpu_to_le32(dlen + PtrOffset(hdr, e)); + memmove(e, data, dlen); + + a_dirty = true; + ntfs_fix_pre_write(&ib->rhdr, bytes); + break; + + case SetIndexEntryVcnAllocation: + ib = Add2Ptr(buffer_le, roff); + hdr = &ib->ihdr; + e = Add2Ptr(ib, aoff); + + if (is_baad(&ib->rhdr)) + goto dirty_vol; + + if (!check_lsn(&ib->rhdr, rlsn)) + goto out; + if (!check_index_buffer(ib, bytes) || + !check_if_alloc_index(hdr, aoff)) { + goto dirty_vol; + } + + de_set_vbn_le(e, *(__le64 *)data); + + a_dirty = true; + ntfs_fix_pre_write(&ib->rhdr, bytes); + break; + + case UpdateFileNameAllocation: + ib = Add2Ptr(buffer_le, roff); + hdr = &ib->ihdr; + e = Add2Ptr(ib, aoff); + + if (is_baad(&ib->rhdr)) + goto dirty_vol; + + if (!check_lsn(&ib->rhdr, rlsn)) + goto out; + if (!check_index_buffer(ib, bytes) || + !check_if_alloc_index(hdr, aoff)) { + goto dirty_vol; + } + + fname = (struct ATTR_FILE_NAME *)(e + 1); + memmove(&fname->dup, data, sizeof(fname->dup)); + + a_dirty = true; + ntfs_fix_pre_write(&ib->rhdr, bytes); + break; + + case SetBitsInNonresidentBitMap: + off = le32_to_cpu(((struct BITMAP_RANGE *)data)->bitmap_off); + bits = le32_to_cpu(((struct BITMAP_RANGE *)data)->bits); + + if (cbo + (off + 7) / 8 > lco || + cbo + ((off + bits + 7) / 8) > lco) { + goto dirty_vol; + } + + ntfs_bitmap_set_le(Add2Ptr(buffer_le, roff), off, bits); + a_dirty = true; + break; + + case ClearBitsInNonresidentBitMap: + off = le32_to_cpu(((struct BITMAP_RANGE *)data)->bitmap_off); + bits = le32_to_cpu(((struct BITMAP_RANGE *)data)->bits); + + if (cbo + (off + 7) / 8 > lco || + cbo + ((off + bits + 7) / 8) > lco) { + goto dirty_vol; + } + + ntfs_bitmap_clear_le(Add2Ptr(buffer_le, roff), off, bits); + a_dirty = true; + break; + + case UpdateRecordDataAllocation: + ib = Add2Ptr(buffer_le, roff); + hdr = &ib->ihdr; + e = Add2Ptr(ib, aoff); + + if (is_baad(&ib->rhdr)) + goto dirty_vol; + + if (!check_lsn(&ib->rhdr, rlsn)) + goto out; + if (!check_index_buffer(ib, bytes) || + !check_if_alloc_index(hdr, aoff)) { + goto dirty_vol; + } + + /* See UpdateRecordDataRoot for the rationale. */ + if (le16_to_cpu(e->view.data_off) > le16_to_cpu(e->size) || + le16_to_cpu(e->view.data_off) + dlen > le16_to_cpu(e->size)) + goto dirty_vol; + + memmove(Add2Ptr(e, le16_to_cpu(e->view.data_off)), data, dlen); + + a_dirty = true; + ntfs_fix_pre_write(&ib->rhdr, bytes); + break; + + default: + WARN_ON(1); + } + + if (rlsn) { + __le64 t64 = cpu_to_le64(*rlsn); + + if (rec) + rec->rhdr.lsn = t64; + if (ib) + ib->rhdr.lsn = t64; + } + + if (mi && mi->dirty) { + err = mi_write(mi, 0); + if (err) + goto out; + } + + if (a_dirty) { + attr = oa->attr; + err = ntfs_sb_write_run(sbi, oa->run1, vbo, buffer_le, bytes, + 0); + if (err) + goto out; + } + +out: + + if (inode) + iput(inode); + else if (mi != mi2_child) + mi_put(mi); + + kfree(buffer_le); + + return err; + +dirty_vol: + log->set_dirty = true; + goto out; +} + +/* + * log_replay - Replays log and empties it. + * + * This function is called during mount operation. + * It replays log and empties it. + * Initialized is set false if logfile contains '-1'. + */ +int log_replay(struct ntfs_inode *ni, bool *initialized) +{ + int err; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct ntfs_log *log; + + u64 rec_lsn, checkpt_lsn = 0, rlsn = 0; + struct ATTR_NAME_ENTRY *attr_names = NULL; + u32 attr_names_bytes = 0; + u32 oatbl_bytes = 0; + struct RESTART_TABLE *dptbl = NULL; + struct RESTART_TABLE *trtbl = NULL; + const struct RESTART_TABLE *rt; + struct RESTART_TABLE *oatbl = NULL; + struct inode *inode; + struct OpenAttr *oa; + struct ntfs_inode *ni_oe; + struct ATTRIB *attr = NULL; + u64 size, vcn, undo_next_lsn; + CLST rno, lcn, lcn0, len0, clen; + void *data; + struct NTFS_RESTART *rst = NULL; + struct lcb *lcb = NULL; + struct OPEN_ATTR_ENRTY *oe; + struct ATTR_NAME_ENTRY *ane; + struct TRANSACTION_ENTRY *tr; + struct DIR_PAGE_ENTRY *dp; + u32 i, bytes_per_attr_entry; + u32 vbo, tail, off, dlen; + u32 saved_len, rec_len, transact_id; + bool use_second_page; + struct RESTART_AREA *ra2, *ra = NULL; + struct CLIENT_REC *ca, *cr; + __le16 client; + struct RESTART_HDR *rh; + const struct LFS_RECORD_HDR *frh; + const struct LOG_REC_HDR *lrh; + bool is_mapped; + bool is_ro = sb_rdonly(sbi->sb); + u64 t64; + u16 t16; + u32 t32; + + log = kzalloc_obj(struct ntfs_log, GFP_NOFS); + if (!log) + return -ENOMEM; + + log->ni = ni; + log->l_size = log->orig_file_size = ni->vfs_inode.i_size; + + /* Get the size of page. NOTE: To replay we can use default page. */ + log->page_size = norm_file_page(PAGE_SIZE, &log->l_size, true); + if (!log->page_size) { + err = -EINVAL; + goto out; + } + + log->one_page_buf = kmalloc(log->page_size, GFP_NOFS); + if (!log->one_page_buf) { + err = -ENOMEM; + goto out; + } + + log->page_mask = log->page_size - 1; + log->page_bits = blksize_bits(log->page_size); + + /* Look for a restart area on the disk. */ + err = log_read_rst(log, true, &log->rst_info); + if (err) + goto out; + + /* remember 'initialized' */ + *initialized = log->rst_info.initialized; + + if (!log->rst_info.restart) { + if (log->rst_info.initialized) { + /* No restart area but the file is not initialized. */ + err = -EINVAL; + goto out; + } + + log_init_pg_hdr(log, 1, 1); + log_create(log, 0, get_random_u32(), false, false); + + ra = log_create_ra(log); + if (!ra) { + err = -ENOMEM; + goto out; + } + log->ra = ra; + log->init_ra = true; + + goto process_log; + } + + /* + * If the restart offset above wasn't zero then we won't + * look for a second restart. + */ + if (log->rst_info.vbo) + goto check_restart_area; + + err = log_read_rst(log, false, &log->rst_info2); + if (err) + goto out; + + /* Determine which restart area to use. */ + if (!log->rst_info2.restart || + log->rst_info2.last_lsn <= log->rst_info.last_lsn) + goto use_first_page; + + use_second_page = true; + + if (log->rst_info.chkdsk_was_run && + log->page_size != log->rst_info.vbo) { + struct RECORD_PAGE_HDR *sp = NULL; + bool usa_error; + + if (!read_log_page(log, log->page_size, &sp, &usa_error) && + sp->rhdr.sign == NTFS_CHKD_SIGNATURE) { + use_second_page = false; + } + kfree(sp); + } + + if (use_second_page) { + kfree(log->rst_info.r_page); + memcpy(&log->rst_info, &log->rst_info2, + sizeof(struct restart_info)); + log->rst_info2.r_page = NULL; + } + +use_first_page: + kfree(log->rst_info2.r_page); + +check_restart_area: + /* + * If the restart area is at offset 0, we want + * to write the second restart area first. + */ + log->init_ra = !!log->rst_info.vbo; + + /* If we have a valid page then grab a pointer to the restart area. */ + ra2 = log->rst_info.valid_page ? + Add2Ptr(log->rst_info.r_page, + le16_to_cpu(log->rst_info.r_page->ra_off)) : + NULL; + + if (log->rst_info.chkdsk_was_run || + (ra2 && ra2->client_idx[1] == LFS_NO_CLIENT_LE)) { + bool wrapped = false; + bool use_multi_page = false; + u32 open_log_count; + + /* Do some checks based on whether we have a valid log page. */ + open_log_count = log->rst_info.valid_page ? + le32_to_cpu(ra2->open_log_count) : + get_random_u32(); + + log_init_pg_hdr(log, 1, 1); + + log_create(log, log->rst_info.last_lsn, open_log_count, wrapped, + use_multi_page); + + ra = log_create_ra(log); + if (!ra) { + err = -ENOMEM; + goto out; + } + log->ra = ra; + + /* Put the restart areas and initialize + * the log file as required. + */ + goto process_log; + } + + if (!ra2) { + err = -EINVAL; + goto out; + } + + /* + * If the log page or the system page sizes have changed, we can't + * use the log file. We must use the system page size instead of the + * default size if there is not a clean shutdown. + */ + t32 = le32_to_cpu(log->rst_info.r_page->sys_page_size); + if (log->page_size != t32) { + u32 old_page_size = log->page_size; + + log->l_size = log->orig_file_size; + log->page_size = norm_file_page(t32, &log->l_size, + t32 == DefaultLogPageSize); + + /* + * If the adopted on-disk page size is larger than the size used + * to allocate one_page_buf above, grow the scratch buffer so a + * later read_log_page() cannot overflow it. + */ + if (log->page_size > old_page_size) { + void *buf; + + buf = krealloc(log->one_page_buf, log->page_size, + GFP_NOFS); + if (!buf) { + err = -ENOMEM; + goto out; + } + log->one_page_buf = buf; + } + } + + if (log->page_size != t32 || + log->page_size != le32_to_cpu(log->rst_info.r_page->page_size)) { + err = -EINVAL; + goto out; + } + + log->page_mask = log->page_size - 1; + log->page_bits = blksize_bits(log->page_size); + + /* If the file size has shrunk then we won't mount it. */ + if (log->l_size < le64_to_cpu(ra2->l_size)) { + err = -EINVAL; + goto out; + } + + log_init_pg_hdr(log, le16_to_cpu(log->rst_info.r_page->major_ver), + le16_to_cpu(log->rst_info.r_page->minor_ver)); + + log->l_size = le64_to_cpu(ra2->l_size); + log->seq_num_bits = le32_to_cpu(ra2->seq_num_bits); + log->file_data_bits = sizeof(u64) * 8 - log->seq_num_bits; + log->seq_num_mask = (8 << log->file_data_bits) - 1; + log->last_lsn = le64_to_cpu(ra2->current_lsn); + log->seq_num = log->last_lsn >> log->file_data_bits; + log->ra_off = le16_to_cpu(log->rst_info.r_page->ra_off); + log->restart_size = log->sys_page_size - log->ra_off; + log->record_header_len = le16_to_cpu(ra2->rec_hdr_len); + log->ra_size = le16_to_cpu(ra2->ra_len); + log->data_off = le16_to_cpu(ra2->data_off); + log->data_size = log->page_size - log->data_off; + log->reserved = log->data_size - log->record_header_len; + + vbo = lsn_to_vbo(log, log->last_lsn); + + if (vbo < log->first_page) { + /* This is a pseudo lsn. */ + log->l_flags |= NTFSLOG_NO_LAST_LSN; + log->next_page = log->first_page; + goto find_oldest; + } + + /* Find the end of this log record. */ + off = final_log_off(log, log->last_lsn, + le32_to_cpu(ra2->last_lsn_data_len)); + + /* If we wrapped the file then increment the sequence number. */ + if (off <= vbo) { + log->seq_num += 1; + log->l_flags |= NTFSLOG_WRAPPED; + } + + /* Now compute the next log page to use. */ + vbo &= ~log->sys_page_mask; + tail = log->page_size - (off & log->page_mask) - 1; + + /* + *If we can fit another log record on the page, + * move back a page the log file. + */ + if (tail >= log->record_header_len) { + log->l_flags |= NTFSLOG_REUSE_TAIL; + log->next_page = vbo; + } else { + log->next_page = next_page_off(log, vbo); + } + +find_oldest: + /* + * Find the oldest client lsn. Use the last + * flushed lsn as a starting point. + */ + log->oldest_lsn = log->last_lsn; + oldest_client_lsn(Add2Ptr(ra2, le16_to_cpu(ra2->client_off)), + ra2->client_idx[1], &log->oldest_lsn); + log->oldest_lsn_off = lsn_to_vbo(log, log->oldest_lsn); + + if (log->oldest_lsn_off < log->first_page) + log->l_flags |= NTFSLOG_NO_OLDEST_LSN; + + if (!(ra2->flags & RESTART_SINGLE_PAGE_IO)) + log->l_flags |= NTFSLOG_WRAPPED | NTFSLOG_MULTIPLE_PAGE_IO; + + log->current_openlog_count = le32_to_cpu(ra2->open_log_count); + log->total_avail_pages = log->l_size - log->first_page; + log->total_avail = log->total_avail_pages >> log->page_bits; + log->max_current_avail = log->total_avail * log->reserved; + log->total_avail = log->total_avail * log->data_size; + + log->current_avail = current_log_avail(log); + + ra = kzalloc(log->restart_size, GFP_NOFS); + if (!ra) { + err = -ENOMEM; + goto out; + } + log->ra = ra; + + t16 = le16_to_cpu(ra2->client_off); + if (t16 == offsetof(struct RESTART_AREA, clients)) { + memcpy(ra, ra2, log->ra_size); + } else { + memcpy(ra, ra2, offsetof(struct RESTART_AREA, clients)); + memcpy(ra->clients, Add2Ptr(ra2, t16), + le16_to_cpu(ra2->ra_len) - t16); + + log->current_openlog_count = get_random_u32(); + ra->open_log_count = cpu_to_le32(log->current_openlog_count); + log->ra_size = offsetof(struct RESTART_AREA, clients) + + sizeof(struct CLIENT_REC); + ra->client_off = + cpu_to_le16(offsetof(struct RESTART_AREA, clients)); + ra->ra_len = cpu_to_le16(log->ra_size); + } + + le32_add_cpu(&ra->open_log_count, 1); + + /* Now we need to walk through looking for the last lsn. */ + err = last_log_lsn(log); + if (err) + goto out; + + log->current_avail = current_log_avail(log); + + /* Remember which restart area to write first. */ + log->init_ra = log->rst_info.vbo; + +process_log: + /* 1.0, 1.1, 2.0 log->major_ver/minor_ver - short values. */ + switch ((log->major_ver << 16) + log->minor_ver) { + case 0x10000: + case 0x10001: + case 0x20000: + break; + default: + ntfs_warn(sbi->sb, "\x24LogFile version %d.%d is not supported", + log->major_ver, log->minor_ver); + err = -EOPNOTSUPP; + log->set_dirty = true; + goto out; + } + + /* One client "NTFS" per logfile. */ + ca = Add2Ptr(ra, le16_to_cpu(ra->client_off)); + + for (client = ra->client_idx[1];; client = cr->next_client) { + if (client == LFS_NO_CLIENT_LE) { + /* Insert "NTFS" client LogFile. */ + client = ra->client_idx[0]; + if (client == LFS_NO_CLIENT_LE) { + err = -EINVAL; + goto out; + } + + t16 = le16_to_cpu(client); + cr = ca + t16; + + remove_client(ca, cr, &ra->client_idx[0]); + + cr->restart_lsn = 0; + cr->oldest_lsn = cpu_to_le64(log->oldest_lsn); + cr->name_bytes = cpu_to_le32(8); + cr->name[0] = cpu_to_le16('N'); + cr->name[1] = cpu_to_le16('T'); + cr->name[2] = cpu_to_le16('F'); + cr->name[3] = cpu_to_le16('S'); + + add_client(ca, t16, &ra->client_idx[1]); + break; + } + + cr = ca + le16_to_cpu(client); + + if (cpu_to_le32(8) == cr->name_bytes && + cpu_to_le16('N') == cr->name[0] && + cpu_to_le16('T') == cr->name[1] && + cpu_to_le16('F') == cr->name[2] && + cpu_to_le16('S') == cr->name[3]) + break; + } + + /* Update the client handle with the client block information. */ + log->client_id.seq_num = cr->seq_num; + log->client_id.client_idx = client; + + err = read_rst_area(log, &rst, &checkpt_lsn); + if (err) + goto out; + + if (!rst) + goto out; + + bytes_per_attr_entry = !rst->major_ver ? 0x2C : 0x28; + + if (rst->check_point_start) + checkpt_lsn = le64_to_cpu(rst->check_point_start); + + /* Allocate and Read the Transaction Table. */ + if (!rst->transact_table_len) + goto check_dirty_page_table; /* reduce tab pressure. */ + + t64 = le64_to_cpu(rst->transact_table_lsn); + err = read_log_rec_lcb(log, t64, lcb_ctx_prev, &lcb); + if (err) + goto out; + + lrh = lcb->log_rec; + frh = lcb->lrh; + rec_len = le32_to_cpu(frh->client_data_len); + + if (!check_log_rec(lrh, rec_len, le32_to_cpu(frh->transact_id), + bytes_per_attr_entry)) { + err = -EINVAL; + goto out; + } + + t16 = le16_to_cpu(lrh->redo_off); + + rt = Add2Ptr(lrh, t16); + t32 = rec_len - t16; + + /* Now check that this is a valid restart table. */ + if (!check_rstbl(rt, t32)) { + err = -EINVAL; + goto out; + } + + trtbl = kmemdup(rt, t32, GFP_NOFS); + if (!trtbl) { + err = -ENOMEM; + goto out; + } + + lcb_put(lcb); + lcb = NULL; + +check_dirty_page_table: + /* The next record back should be the Dirty Pages Table. */ + if (!rst->dirty_pages_len) + goto check_attribute_names; /* reduce tab pressure. */ + + t64 = le64_to_cpu(rst->dirty_pages_table_lsn); + err = read_log_rec_lcb(log, t64, lcb_ctx_prev, &lcb); + if (err) + goto out; + + lrh = lcb->log_rec; + frh = lcb->lrh; + rec_len = le32_to_cpu(frh->client_data_len); + + if (!check_log_rec(lrh, rec_len, le32_to_cpu(frh->transact_id), + bytes_per_attr_entry)) { + err = -EINVAL; + goto out; + } + + t16 = le16_to_cpu(lrh->redo_off); + + rt = Add2Ptr(lrh, t16); + t32 = rec_len - t16; + + /* Now check that this is a valid restart table. */ + if (!check_rstbl(rt, t32)) { + err = -EINVAL; + goto out; + } + + if (!check_dp_table(rt)) { + err = -EINVAL; + goto out; + } + + dptbl = kmemdup(rt, t32, GFP_NOFS); + if (!dptbl) { + err = -ENOMEM; + goto out; + } + + /* Convert Ra version '0' into version '1'. */ + if (rst->major_ver) + goto end_conv_1; /* reduce tab pressure. */ + + t16 = le16_to_cpu(dptbl->size); + if (t16 < sizeof(struct DIR_PAGE_ENTRY)) { + log->set_dirty = true; + goto out; + } + + t32 = (t16 - sizeof(struct DIR_PAGE_ENTRY)) / sizeof(u64); + + dp = NULL; + while ((dp = enum_rstbl(dptbl, dp))) { + struct DIR_PAGE_ENTRY_32 *dp0 = (struct DIR_PAGE_ENTRY_32 *)dp; + u32 lcns = le32_to_cpu(dp->lcns_follow); + + if (lcns > t32) { + log->set_dirty = true; + goto out; + } + + memmove(&dp->vcn, &dp0->vcn_low, + 2 * sizeof(u64) + lcns * sizeof(u64)); + } + +end_conv_1: + lcb_put(lcb); + lcb = NULL; + + /* + * Go through the table and remove the duplicates, + * remembering the oldest lsn values. + */ + if (sbi->cluster_size <= log->page_size) + goto trace_dp_table; /* reduce tab pressure. */ + dp = NULL; + while ((dp = enum_rstbl(dptbl, dp))) { + struct DIR_PAGE_ENTRY *next = dp; + + while ((next = enum_rstbl(dptbl, next))) { + if (next->target_attr == dp->target_attr && + next->vcn == dp->vcn) { + if (le64_to_cpu(next->oldest_lsn) < + le64_to_cpu(dp->oldest_lsn)) { + dp->oldest_lsn = next->oldest_lsn; + } + + free_rsttbl_idx(dptbl, PtrOffset(dptbl, next)); + } + } + } +trace_dp_table: +check_attribute_names: + /* The next record should be the Attribute Names. */ + if (!rst->attr_names_len) + goto check_attr_table; /* reduce tab pressure. */ + + t64 = le64_to_cpu(rst->attr_names_lsn); + err = read_log_rec_lcb(log, t64, lcb_ctx_prev, &lcb); + if (err) + goto out; + + lrh = lcb->log_rec; + frh = lcb->lrh; + rec_len = le32_to_cpu(frh->client_data_len); + + if (!check_log_rec(lrh, rec_len, le32_to_cpu(frh->transact_id), + bytes_per_attr_entry)) { + err = -EINVAL; + goto out; + } + + t32 = lrh_length(lrh); + attr_names_bytes = rec_len - t32; + + attr_names = kmemdup(Add2Ptr(lrh, t32), attr_names_bytes, GFP_NOFS); + if (!attr_names) { + err = -ENOMEM; + goto out; + } + + lcb_put(lcb); + lcb = NULL; + +check_attr_table: + /* The next record should be the attribute Table. */ + if (!rst->open_attr_len) + goto check_attribute_names2; /* reduce tab pressure. */ + + t64 = le64_to_cpu(rst->open_attr_table_lsn); + err = read_log_rec_lcb(log, t64, lcb_ctx_prev, &lcb); + if (err) + goto out; + + lrh = lcb->log_rec; + frh = lcb->lrh; + rec_len = le32_to_cpu(frh->client_data_len); + + if (!check_log_rec(lrh, rec_len, le32_to_cpu(frh->transact_id), + bytes_per_attr_entry)) { + err = -EINVAL; + goto out; + } + + t16 = le16_to_cpu(lrh->redo_off); + + rt = Add2Ptr(lrh, t16); + oatbl_bytes = rec_len - t16; + + if (!check_rstbl(rt, oatbl_bytes)) { + err = -EINVAL; + goto out; + } + + oatbl = kmemdup(rt, oatbl_bytes, GFP_NOFS); + if (!oatbl) { + err = -ENOMEM; + goto out; + } + + log->open_attr_tbl = oatbl; + + /* Clear all of the Attr pointers. */ + oe = NULL; + while ((oe = enum_rstbl(oatbl, oe))) { + if (!rst->major_ver) { + struct OPEN_ATTR_ENRTY_32 oe0; + + /* Really 'oe' points to OPEN_ATTR_ENRTY_32. */ + memcpy(&oe0, oe, SIZEOF_OPENATTRIBUTEENTRY0); + + oe->bytes_per_index = oe0.bytes_per_index; + oe->type = oe0.type; + oe->is_dirty_pages = oe0.is_dirty_pages; + oe->name_len = 0; + oe->ref = oe0.ref; + oe->open_record_lsn = oe0.open_record_lsn; + } + + oe->is_attr_name = 0; + oe->ptr = NULL; + } + + lcb_put(lcb); + lcb = NULL; + +check_attribute_names2: + if (attr_names && oatbl) { + off = 0; + for (;;) { + /* Check we can use attribute name entry 'ane'. */ + static_assert(sizeof(*ane) == 4); + if (off + sizeof(*ane) > attr_names_bytes) { + /* just ignore the rest. */ + break; + } + + ane = Add2Ptr(attr_names, off); + t16 = le16_to_cpu(ane->off); + if (!t16) { + /* this is the only valid exit. */ + break; + } + + /* Check we can use open attribute entry 'oe'. */ + if (t16 + sizeof(*oe) > oatbl_bytes) { + /* just ignore the rest. */ + break; + } + + /* TODO: Clear table on exit! */ + oe = Add2Ptr(oatbl, t16); + t16 = le16_to_cpu(ane->name_bytes); + off += t16 + sizeof(*ane); + if (off > attr_names_bytes) { + /* just ignore the rest. */ + break; + } + oe->name_len = t16 / sizeof(short); + oe->ptr = ane->name; + oe->is_attr_name = 2; + } + } + + /* + * If the checkpt_lsn is zero, then this is a freshly + * formatted disk and we have no work to do. + */ + if (!checkpt_lsn) { + err = 0; + goto out; + } + + if (!oatbl) { + oatbl = init_rsttbl(bytes_per_attr_entry, 8); + if (!oatbl) { + err = -ENOMEM; + goto out; + } + } + + log->open_attr_tbl = oatbl; + + /* Start the analysis pass from the Checkpoint lsn. */ + rec_lsn = checkpt_lsn; + + /* Read the first lsn. */ + err = read_log_rec_lcb(log, checkpt_lsn, lcb_ctx_next, &lcb); + if (err) + goto out; + + /* Loop to read all subsequent records to the end of the log file. */ +next_log_record_analyze: + err = read_next_log_rec(log, lcb, &rec_lsn); + if (err) + goto out; + + if (!rec_lsn) + goto end_log_records_enumerate; + + frh = lcb->lrh; + transact_id = le32_to_cpu(frh->transact_id); + rec_len = le32_to_cpu(frh->client_data_len); + lrh = lcb->log_rec; + + if (!check_log_rec(lrh, rec_len, transact_id, bytes_per_attr_entry)) { + err = -EINVAL; + goto out; + } + + /* + * The first lsn after the previous lsn remembered + * the checkpoint is the first candidate for the rlsn. + */ + if (!rlsn) + rlsn = rec_lsn; + + if (LfsClientRecord != frh->record_type) + goto next_log_record_analyze; + + /* + * Now update the Transaction Table for this transaction. If there + * is no entry present or it is unallocated we allocate the entry. + */ + if (!trtbl) { + trtbl = init_rsttbl(sizeof(struct TRANSACTION_ENTRY), + INITIAL_NUMBER_TRANSACTIONS); + if (!trtbl) { + err = -ENOMEM; + goto out; + } + } + + tr = Add2Ptr(trtbl, transact_id); + + if (transact_id >= bytes_per_rt(trtbl) || + tr->next != RESTART_ENTRY_ALLOCATED_LE) { + tr = alloc_rsttbl_from_idx(&trtbl, transact_id); + if (!tr) { + err = -ENOMEM; + goto out; + } + tr->transact_state = TransactionActive; + tr->first_lsn = cpu_to_le64(rec_lsn); + } + + tr->prev_lsn = tr->undo_next_lsn = cpu_to_le64(rec_lsn); + + /* + * If this is a compensation log record, then change + * the undo_next_lsn to be the undo_next_lsn of this record. + */ + if (lrh->undo_op == cpu_to_le16(CompensationLogRecord)) + tr->undo_next_lsn = frh->client_undo_next_lsn; + + /* Dispatch to handle log record depending on type. */ + switch (le16_to_cpu(lrh->redo_op)) { + case InitializeFileRecordSegment: + case DeallocateFileRecordSegment: + case WriteEndOfFileRecordSegment: + case CreateAttribute: + case DeleteAttribute: + case UpdateResidentValue: + case UpdateNonresidentValue: + case UpdateMappingPairs: + case SetNewAttributeSizes: + case AddIndexEntryRoot: + case DeleteIndexEntryRoot: + case AddIndexEntryAllocation: + case DeleteIndexEntryAllocation: + case WriteEndOfIndexBuffer: + case SetIndexEntryVcnRoot: + case SetIndexEntryVcnAllocation: + case UpdateFileNameRoot: + case UpdateFileNameAllocation: + case SetBitsInNonresidentBitMap: + case ClearBitsInNonresidentBitMap: + case UpdateRecordDataRoot: + case UpdateRecordDataAllocation: + case ZeroEndOfFileRecord: + t16 = le16_to_cpu(lrh->target_attr); + t64 = le64_to_cpu(lrh->target_vcn); + dp = find_dp(dptbl, t16, t64); + + if (dp) + goto copy_lcns; + + /* + * Calculate the number of clusters per page the system + * which wrote the checkpoint, possibly creating the table. + */ + if (dptbl) { + t32 = (le16_to_cpu(dptbl->size) - + sizeof(struct DIR_PAGE_ENTRY)) / + sizeof(u64); + } else { + t32 = log->clst_per_page; + kfree(dptbl); + dptbl = init_rsttbl(struct_size(dp, page_lcns, t32), + 32); + if (!dptbl) { + err = -ENOMEM; + goto out; + } + } + + dp = alloc_rsttbl_idx(&dptbl); + if (!dp) { + err = -ENOMEM; + goto out; + } + dp->target_attr = cpu_to_le32(t16); + dp->transfer_len = cpu_to_le32(t32 << sbi->cluster_bits); + dp->lcns_follow = cpu_to_le32(t32); + dp->vcn = cpu_to_le64(t64 & ~((u64)t32 - 1)); + dp->oldest_lsn = cpu_to_le64(rec_lsn); + +copy_lcns: + /* + * Copy the Lcns from the log record into the Dirty Page Entry. + * TODO: For different page size support, must somehow make + * whole routine a loop, case Lcns do not fit below. + */ + t16 = le16_to_cpu(lrh->lcns_follow); + t32 = le32_to_cpu(dp->lcns_follow); + if (le64_to_cpu(lrh->target_vcn) < le64_to_cpu(dp->vcn)) { + err = -EINVAL; + goto out; + } + + /* + * find_dp() only validates that target_vcn is the first + * cluster covered by dp. The walk through lrh->lcns_follow + * further entries must stay within the allocated + * dp->page_lcns[] array, which is sized by dp->lcns_follow. + */ + if (le64_to_cpu(lrh->target_vcn) - le64_to_cpu(dp->vcn) + t16 > + le32_to_cpu(dp->lcns_follow)) { + err = -EINVAL; + log->set_dirty = true; + goto out; + } + + for (i = 0; i < t16; i++) { + size_t j = (size_t)(le64_to_cpu(lrh->target_vcn) - + le64_to_cpu(dp->vcn)); + if (j >= t32 || i >= t32 - j) { + err = -EINVAL; + goto out; + } + dp->page_lcns[j + i] = lrh->page_lcns[i]; + } + goto next_log_record_analyze; + + case DeleteDirtyClusters: { + u32 range_count = + le16_to_cpu(lrh->redo_len) / sizeof(struct LCN_RANGE); + const struct LCN_RANGE *r = + Add2Ptr(lrh, le16_to_cpu(lrh->redo_off)); + + /* Loop through all of the Lcn ranges this log record. */ + for (i = 0; i < range_count; i++, r++) { + u64 lcn0 = le64_to_cpu(r->lcn); + u64 lcn_e = lcn0 + le64_to_cpu(r->len) - 1; + + dp = NULL; + while ((dp = enum_rstbl(dptbl, dp))) { + u32 j; + + t32 = le32_to_cpu(dp->lcns_follow); + for (j = 0; j < t32; j++) { + t64 = le64_to_cpu(dp->page_lcns[j]); + if (t64 >= lcn0 && t64 <= lcn_e) + dp->page_lcns[j] = 0; + } + } + } + goto next_log_record_analyze; + } + + case OpenNonresidentAttribute: + t16 = le16_to_cpu(lrh->target_attr); + if (t16 >= bytes_per_rt(oatbl)) { + /* + * Compute how big the table needs to be. + * Add 10 extra entries for some cushion. + */ + u32 new_e = t16 / le16_to_cpu(oatbl->size); + + new_e += 10 - le16_to_cpu(oatbl->used); + + oatbl = extend_rsttbl(oatbl, new_e, ~0u); + log->open_attr_tbl = oatbl; + if (!oatbl) { + err = -ENOMEM; + goto out; + } + } + + /* Point to the entry being opened. */ + oe = alloc_rsttbl_from_idx(&oatbl, t16); + log->open_attr_tbl = oatbl; + if (!oe) { + err = -ENOMEM; + goto out; + } + + /* Initialize this entry from the log record. */ + t16 = le16_to_cpu(lrh->redo_off); + if (!rst->major_ver) { + /* Convert version '0' into version '1'. */ + struct OPEN_ATTR_ENRTY_32 *oe0 = Add2Ptr(lrh, t16); + + oe->bytes_per_index = oe0->bytes_per_index; + oe->type = oe0->type; + oe->is_dirty_pages = oe0->is_dirty_pages; + oe->name_len = 0; //oe0.name_len; + oe->ref = oe0->ref; + oe->open_record_lsn = oe0->open_record_lsn; + } else { + memcpy(oe, Add2Ptr(lrh, t16), bytes_per_attr_entry); + } + + t16 = le16_to_cpu(lrh->undo_len); + if (t16) { + oe->ptr = kmalloc(t16, GFP_NOFS); + if (!oe->ptr) { + err = -ENOMEM; + goto out; + } + oe->name_len = t16 / sizeof(short); + memcpy(oe->ptr, + Add2Ptr(lrh, le16_to_cpu(lrh->undo_off)), t16); + oe->is_attr_name = 1; + } else { + oe->ptr = NULL; + oe->is_attr_name = 0; + } + + goto next_log_record_analyze; + + case HotFix: + t16 = le16_to_cpu(lrh->target_attr); + t64 = le64_to_cpu(lrh->target_vcn); + dp = find_dp(dptbl, t16, t64); + if (dp) { + size_t j = le64_to_cpu(lrh->target_vcn) - + le64_to_cpu(dp->vcn); + if (dp->page_lcns[j]) + dp->page_lcns[j] = lrh->page_lcns[0]; + } + goto next_log_record_analyze; + + case EndTopLevelAction: + tr = Add2Ptr(trtbl, transact_id); + tr->prev_lsn = cpu_to_le64(rec_lsn); + tr->undo_next_lsn = frh->client_undo_next_lsn; + goto next_log_record_analyze; + + case PrepareTransaction: + tr = Add2Ptr(trtbl, transact_id); + tr->transact_state = TransactionPrepared; + goto next_log_record_analyze; + + case CommitTransaction: + tr = Add2Ptr(trtbl, transact_id); + tr->transact_state = TransactionCommitted; + goto next_log_record_analyze; + + case ForgetTransaction: + free_rsttbl_idx(trtbl, transact_id); + goto next_log_record_analyze; + + case Noop: + case OpenAttributeTableDump: + case AttributeNamesDump: + case DirtyPageTableDump: + case TransactionTableDump: + /* The following cases require no action the Analysis Pass. */ + goto next_log_record_analyze; + + default: + /* + * All codes will be explicitly handled. + * If we see a code we do not expect, then we are trouble. + */ + goto next_log_record_analyze; + } + +end_log_records_enumerate: + lcb_put(lcb); + lcb = NULL; + + /* + * Scan the Dirty Page Table and Transaction Table for + * the lowest lsn, and return it as the Redo lsn. + */ + dp = NULL; + while ((dp = enum_rstbl(dptbl, dp))) { + t64 = le64_to_cpu(dp->oldest_lsn); + if (t64 && t64 < rlsn) + rlsn = t64; + } + + tr = NULL; + while ((tr = enum_rstbl(trtbl, tr))) { + t64 = le64_to_cpu(tr->first_lsn); + if (t64 && t64 < rlsn) + rlsn = t64; + } + + /* + * Only proceed if the Dirty Page Table or Transaction + * table are not empty. + */ + if ((!dptbl || !dptbl->total) && (!trtbl || !trtbl->total)) + goto end_replay; + + sbi->flags |= NTFS_FLAGS_NEED_REPLAY; + if (is_ro) + goto out; + + /* Reopen all of the attributes with dirty pages. */ + oe = NULL; +next_open_attribute: + + oe = enum_rstbl(oatbl, oe); + if (!oe) { + err = 0; + dp = NULL; + goto next_dirty_page; + } + + oa = kzalloc_obj(struct OpenAttr, GFP_NOFS); + if (!oa) { + err = -ENOMEM; + goto out; + } + + inode = ntfs_iget5(sbi->sb, &oe->ref, NULL); + if (IS_ERR(inode)) + goto fake_attr; + + if (is_bad_inode(inode)) { + iput(inode); +fake_attr: + if (oa->ni) { + iput(&oa->ni->vfs_inode); + oa->ni = NULL; + } + + attr = attr_create_nonres_log(sbi, oe->type, 0, oe->ptr, + oe->name_len, 0); + if (!attr) { + kfree(oa); + err = -ENOMEM; + goto out; + } + oa->attr = attr; + oa->run1 = &oa->run0; + goto final_oe; + } + + ni_oe = ntfs_i(inode); + oa->ni = ni_oe; + + attr = ni_find_attr(ni_oe, NULL, NULL, oe->type, oe->ptr, oe->name_len, + NULL, NULL); + + if (!attr) + goto fake_attr; + + t32 = le32_to_cpu(attr->size); + oa->attr = kmemdup(attr, t32, GFP_NOFS); + if (!oa->attr) + goto fake_attr; + + if (!S_ISDIR(inode->i_mode)) { + if (attr->type == ATTR_DATA && !attr->name_len) { + oa->run1 = &ni_oe->file.run; + goto final_oe; + } + } else { + if (attr->type == ATTR_ALLOC && + attr->name_len == ARRAY_SIZE(I30_NAME) && + !memcmp(attr_name(attr), I30_NAME, sizeof(I30_NAME))) { + oa->run1 = &ni_oe->dir.alloc_run; + goto final_oe; + } + } + + if (attr->non_res) { + u16 roff = le16_to_cpu(attr->nres.run_off); + CLST svcn = le64_to_cpu(attr->nres.svcn); + + if (roff > t32) { + kfree(oa->attr); + oa->attr = NULL; + goto fake_attr; + } + + err = run_unpack(&oa->run0, sbi, inode->i_ino, svcn, + le64_to_cpu(attr->nres.evcn), svcn, + Add2Ptr(attr, roff), t32 - roff); + if (err < 0) { + kfree(oa->attr); + oa->attr = NULL; + goto fake_attr; + } + err = 0; + } + oa->run1 = &oa->run0; + attr = oa->attr; + +final_oe: + if (oe->is_attr_name == 1) + kfree(oe->ptr); + oe->is_attr_name = 0; + oe->ptr = oa; + oe->name_len = attr->name_len; + + goto next_open_attribute; + + /* + * Now loop through the dirty page table to extract all of the Vcn/Lcn. + * Mapping that we have, and insert it into the appropriate run. + */ +next_dirty_page: + dp = enum_rstbl(dptbl, dp); + if (!dp) + goto do_redo_1; + + oe = Add2Ptr(oatbl, le32_to_cpu(dp->target_attr)); + + if (oe->next != RESTART_ENTRY_ALLOCATED_LE) + goto next_dirty_page; + + oa = oe->ptr; + if (!oa) + goto next_dirty_page; + + i = -1; +next_dirty_page_vcn: + i += 1; + if (i >= le32_to_cpu(dp->lcns_follow)) + goto next_dirty_page; + + vcn = le64_to_cpu(dp->vcn) + i; + size = (vcn + 1) << sbi->cluster_bits; + + if (!dp->page_lcns[i]) + goto next_dirty_page_vcn; + + rno = ino_get(&oe->ref); + if (rno <= MFT_REC_MIRR && + size < (MFT_REC_VOL + 1) * sbi->record_size && + oe->type == ATTR_DATA) { + goto next_dirty_page_vcn; + } + + lcn = le64_to_cpu(dp->page_lcns[i]); + + if ((!run_lookup_entry(oa->run1, vcn, &lcn0, &len0, NULL) || + lcn0 != lcn) && + !run_add_entry(oa->run1, vcn, lcn, 1, false)) { + err = -ENOMEM; + goto out; + } + attr = oa->attr; + if (size > le64_to_cpu(attr->nres.alloc_size)) { + attr->nres.valid_size = attr->nres.data_size = + attr->nres.alloc_size = cpu_to_le64(size); + } + goto next_dirty_page_vcn; + +do_redo_1: + /* + * Perform the Redo Pass, to restore all of the dirty pages to the same + * contents that they had immediately before the crash. If the dirty + * page table is empty, then we can skip the entire Redo Pass. + */ + if (!dptbl || !dptbl->total) + goto do_undo_action; + + rec_lsn = rlsn; + + /* + * Read the record at the Redo lsn, before falling + * into common code to handle each record. + */ + err = read_log_rec_lcb(log, rlsn, lcb_ctx_next, &lcb); + if (err) + goto out; + + /* + * Now loop to read all of our log records forwards, until + * we hit the end of the file, cleaning up at the end. + */ +do_action_next: + frh = lcb->lrh; + + if (LfsClientRecord != frh->record_type) + goto read_next_log_do_action; + + transact_id = le32_to_cpu(frh->transact_id); + rec_len = le32_to_cpu(frh->client_data_len); + lrh = lcb->log_rec; + + if (!check_log_rec(lrh, rec_len, transact_id, bytes_per_attr_entry)) { + err = -EINVAL; + goto out; + } + + /* Ignore log records that do not update pages. */ + if (lrh->lcns_follow) + goto find_dirty_page; + + goto read_next_log_do_action; + +find_dirty_page: + t16 = le16_to_cpu(lrh->target_attr); + t64 = le64_to_cpu(lrh->target_vcn); + dp = find_dp(dptbl, t16, t64); + + if (!dp) + goto read_next_log_do_action; + + if (rec_lsn < le64_to_cpu(dp->oldest_lsn)) + goto read_next_log_do_action; + + t16 = le16_to_cpu(lrh->target_attr); + if (t16 >= bytes_per_rt(oatbl)) { + err = -EINVAL; + goto out; + } + + oe = Add2Ptr(oatbl, t16); + + if (oe->next != RESTART_ENTRY_ALLOCATED_LE) { + err = -EINVAL; + goto out; + } + + oa = oe->ptr; + + if (!oa) { + err = -EINVAL; + goto out; + } + attr = oa->attr; + + vcn = le64_to_cpu(lrh->target_vcn); + + if (!run_lookup_entry(oa->run1, vcn, &lcn, NULL, NULL) || + lcn == SPARSE_LCN) { + goto read_next_log_do_action; + } + + /* Point to the Redo data and get its length. */ + data = Add2Ptr(lrh, le16_to_cpu(lrh->redo_off)); + dlen = le16_to_cpu(lrh->redo_len); + + /* Shorten length by any Lcns which were deleted. */ + saved_len = dlen; + + if (!dp_range_ok(le64_to_cpu(lrh->target_vcn) - le64_to_cpu(dp->vcn), + le16_to_cpu(lrh->lcns_follow), + le32_to_cpu(dp->lcns_follow))) { + err = -EINVAL; + goto out; + } + + for (i = le16_to_cpu(lrh->lcns_follow); i; i--) { + size_t j; + u32 alen, voff; + + voff = le16_to_cpu(lrh->record_off) + + le16_to_cpu(lrh->attr_off); + voff += le16_to_cpu(lrh->cluster_off) << SECTOR_SHIFT; + + /* If the Vcn question is allocated, we can just get out. */ + j = le64_to_cpu(lrh->target_vcn) - le64_to_cpu(dp->vcn); + if (dp->page_lcns[j + i - 1]) + break; + + if (!saved_len) + saved_len = 1; + + /* + * Calculate the allocated space left relative to the + * log record Vcn, after removing this unallocated Vcn. + */ + alen = (i - 1) << sbi->cluster_bits; + + /* + * If the update described this log record goes beyond + * the allocated space, then we will have to reduce the length. + */ + if (voff >= alen) + dlen = 0; + else if (voff + dlen > alen) + dlen = alen - voff; + } + + /* + * If the resulting dlen from above is now zero, + * we can skip this log record. + */ + if (!dlen && saved_len) + goto read_next_log_do_action; + + t16 = le16_to_cpu(lrh->redo_op); + if (can_skip_action(t16)) + goto read_next_log_do_action; + + /* Apply the Redo operation a common routine. */ + err = do_action(log, oe, lrh, t16, data, dlen, rec_len, &rec_lsn); + if (err) + goto out; + + /* Keep reading and looping back until end of file. */ +read_next_log_do_action: + err = read_next_log_rec(log, lcb, &rec_lsn); + if (!err && rec_lsn) + goto do_action_next; + + lcb_put(lcb); + lcb = NULL; + +do_undo_action: + /* Scan Transaction Table. */ + tr = NULL; +transaction_table_next: + tr = enum_rstbl(trtbl, tr); + if (!tr) + goto undo_action_done; + + if (TransactionActive != tr->transact_state || !tr->undo_next_lsn) { + free_rsttbl_idx(trtbl, PtrOffset(trtbl, tr)); + goto transaction_table_next; + } + + log->transaction_id = PtrOffset(trtbl, tr); + undo_next_lsn = le64_to_cpu(tr->undo_next_lsn); + + /* + * We only have to do anything if the transaction has + * something its undo_next_lsn field. + */ + if (!undo_next_lsn) + goto commit_undo; + + /* Read the first record to be undone by this transaction. */ + err = read_log_rec_lcb(log, undo_next_lsn, lcb_ctx_undo_next, &lcb); + if (err) + goto out; + + /* + * Now loop to read all of our log records forwards, + * until we hit the end of the file, cleaning up at the end. + */ +undo_action_next: + + lrh = lcb->log_rec; + frh = lcb->lrh; + transact_id = le32_to_cpu(frh->transact_id); + rec_len = le32_to_cpu(frh->client_data_len); + + if (!check_log_rec(lrh, rec_len, transact_id, bytes_per_attr_entry)) { + err = -EINVAL; + goto out; + } + + if (lrh->undo_op == cpu_to_le16(Noop)) + goto read_next_log_undo_action; + + oe = Add2Ptr(oatbl, le16_to_cpu(lrh->target_attr)); + oa = oe->ptr; + + t16 = le16_to_cpu(lrh->lcns_follow); + if (!t16) + goto add_allocated_vcns; + + is_mapped = run_lookup_entry(oa->run1, le64_to_cpu(lrh->target_vcn), + &lcn, &clen, NULL); + + /* + * If the mapping isn't already the table or the mapping + * corresponds to a hole the mapping, we need to make sure + * there is no partial page already memory. + */ + if (is_mapped && lcn != SPARSE_LCN && clen >= t16) + goto add_allocated_vcns; + + vcn = le64_to_cpu(lrh->target_vcn); + vcn &= ~(u64)(log->clst_per_page - 1); + +add_allocated_vcns: + for (i = 0, vcn = le64_to_cpu(lrh->target_vcn), + size = (vcn + 1) << sbi->cluster_bits; + i < t16; i++, vcn += 1, size += sbi->cluster_size) { + attr = oa->attr; + if (!attr->non_res) { + if (size > le32_to_cpu(attr->res.data_size)) + attr->res.data_size = cpu_to_le32(size); + } else { + if (size > le64_to_cpu(attr->nres.data_size)) + attr->nres.valid_size = attr->nres.data_size = + attr->nres.alloc_size = + cpu_to_le64(size); + } + } + + t16 = le16_to_cpu(lrh->undo_op); + if (can_skip_action(t16)) + goto read_next_log_undo_action; + + /* Point to the Redo data and get its length. */ + data = Add2Ptr(lrh, le16_to_cpu(lrh->undo_off)); + dlen = le16_to_cpu(lrh->undo_len); + + /* It is time to apply the undo action. */ + err = do_action(log, oe, lrh, t16, data, dlen, rec_len, NULL); + +read_next_log_undo_action: + /* + * Keep reading and looping back until we have read the + * last record for this transaction. + */ + err = read_next_log_rec(log, lcb, &rec_lsn); + if (err) + goto out; + + if (rec_lsn) + goto undo_action_next; + + lcb_put(lcb); + lcb = NULL; + +commit_undo: + free_rsttbl_idx(trtbl, log->transaction_id); + + log->transaction_id = 0; + + goto transaction_table_next; + +undo_action_done: + + ntfs_update_mftmirr(sbi); + + sbi->flags &= ~NTFS_FLAGS_NEED_REPLAY; + +end_replay: + + err = 0; + if (is_ro) + goto out; + + rh = kzalloc(log->page_size, GFP_NOFS); + if (!rh) { + err = -ENOMEM; + goto out; + } + + rh->rhdr.sign = NTFS_RSTR_SIGNATURE; + rh->rhdr.fix_off = cpu_to_le16(offsetof(struct RESTART_HDR, fixups)); + t16 = (log->page_size >> SECTOR_SHIFT) + 1; + rh->rhdr.fix_num = cpu_to_le16(t16); + rh->sys_page_size = cpu_to_le32(log->page_size); + rh->page_size = cpu_to_le32(log->page_size); + + t16 = ALIGN(offsetof(struct RESTART_HDR, fixups) + sizeof(short) * t16, + 8); + rh->ra_off = cpu_to_le16(t16); + rh->minor_ver = cpu_to_le16(1); // 0x1A: + rh->major_ver = cpu_to_le16(1); // 0x1C: + + ra2 = Add2Ptr(rh, t16); + memcpy(ra2, ra, sizeof(struct RESTART_AREA)); + + ra2->client_idx[0] = 0; + ra2->client_idx[1] = LFS_NO_CLIENT_LE; + ra2->flags = cpu_to_le16(2); + + le32_add_cpu(&ra2->open_log_count, 1); + + ntfs_fix_pre_write(&rh->rhdr, log->page_size); + + err = ntfs_sb_write_run(sbi, &ni->file.run, 0, rh, log->page_size, 0); + if (!err) + err = ntfs_sb_write_run(sbi, &log->ni->file.run, log->page_size, + rh, log->page_size, 0); + + kfree(rh); + if (err) + goto out; + +out: + kfree(rst); + if (lcb) + lcb_put(lcb); + + /* + * Scan the Open Attribute Table to close all of + * the open attributes. + */ + oe = NULL; + while ((oe = enum_rstbl(oatbl, oe))) { + rno = ino_get(&oe->ref); + + if (oe->is_attr_name == 1) { + kfree(oe->ptr); + oe->ptr = NULL; + continue; + } + + if (oe->is_attr_name) + continue; + + oa = oe->ptr; + if (!oa) + continue; + + run_close(&oa->run0); + kfree(oa->attr); + if (oa->ni) + iput(&oa->ni->vfs_inode); + kfree(oa); + } + + kfree(trtbl); + kfree(oatbl); + kfree(dptbl); + kfree(attr_names); + kfree(log->rst_info.r_page); + + kfree(ra); + kfree(log->one_page_buf); + + if (err) + sbi->flags |= NTFS_FLAGS_NEED_REPLAY; + + if (err == -EROFS) + err = 0; + else if (log->set_dirty) + ntfs_set_state(sbi, NTFS_DIRTY_ERROR); + + kfree(log); + + return err; +} diff --git a/fs/ntfs3/fsntfs.c b/fs/ntfs3/fsntfs.c new file mode 100644 index 000000000..97c04ab27 --- /dev/null +++ b/fs/ntfs3/fsntfs.c @@ -0,0 +1,2717 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + */ + +#include +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +// clang-format off +const struct cpu_str NAME_MFT = { + 4, 0, { '$', 'M', 'F', 'T' }, +}; +const struct cpu_str NAME_MIRROR = { + 8, 0, { '$', 'M', 'F', 'T', 'M', 'i', 'r', 'r' }, +}; +const struct cpu_str NAME_LOGFILE = { + 8, 0, { '$', 'L', 'o', 'g', 'F', 'i', 'l', 'e' }, +}; +const struct cpu_str NAME_VOLUME = { + 7, 0, { '$', 'V', 'o', 'l', 'u', 'm', 'e' }, +}; +const struct cpu_str NAME_ATTRDEF = { + 8, 0, { '$', 'A', 't', 't', 'r', 'D', 'e', 'f' }, +}; +const struct cpu_str NAME_ROOT = { + 1, 0, { '.' }, +}; +const struct cpu_str NAME_BITMAP = { + 7, 0, { '$', 'B', 'i', 't', 'm', 'a', 'p' }, +}; +const struct cpu_str NAME_BOOT = { + 5, 0, { '$', 'B', 'o', 'o', 't' }, +}; +const struct cpu_str NAME_BADCLUS = { + 8, 0, { '$', 'B', 'a', 'd', 'C', 'l', 'u', 's' }, +}; +const struct cpu_str NAME_QUOTA = { + 6, 0, { '$', 'Q', 'u', 'o', 't', 'a' }, +}; +const struct cpu_str NAME_SECURE = { + 7, 0, { '$', 'S', 'e', 'c', 'u', 'r', 'e' }, +}; +const struct cpu_str NAME_UPCASE = { + 7, 0, { '$', 'U', 'p', 'C', 'a', 's', 'e' }, +}; +const struct cpu_str NAME_EXTEND = { + 7, 0, { '$', 'E', 'x', 't', 'e', 'n', 'd' }, +}; +const struct cpu_str NAME_OBJID = { + 6, 0, { '$', 'O', 'b', 'j', 'I', 'd' }, +}; +const struct cpu_str NAME_REPARSE = { + 8, 0, { '$', 'R', 'e', 'p', 'a', 'r', 's', 'e' }, +}; +const struct cpu_str NAME_USNJRNL = { + 8, 0, { '$', 'U', 's', 'n', 'J', 'r', 'n', 'l' }, +}; +const __le16 BAD_NAME[4] = { + cpu_to_le16('$'), cpu_to_le16('B'), cpu_to_le16('a'), cpu_to_le16('d'), +}; +const __le16 I30_NAME[4] = { + cpu_to_le16('$'), cpu_to_le16('I'), cpu_to_le16('3'), cpu_to_le16('0'), +}; +const __le16 SII_NAME[4] = { + cpu_to_le16('$'), cpu_to_le16('S'), cpu_to_le16('I'), cpu_to_le16('I'), +}; +const __le16 SDH_NAME[4] = { + cpu_to_le16('$'), cpu_to_le16('S'), cpu_to_le16('D'), cpu_to_le16('H'), +}; +const __le16 SDS_NAME[4] = { + cpu_to_le16('$'), cpu_to_le16('S'), cpu_to_le16('D'), cpu_to_le16('S'), +}; +const __le16 SO_NAME[2] = { + cpu_to_le16('$'), cpu_to_le16('O'), +}; +const __le16 SQ_NAME[2] = { + cpu_to_le16('$'), cpu_to_le16('Q'), +}; +const __le16 SR_NAME[2] = { + cpu_to_le16('$'), cpu_to_le16('R'), +}; +const __le16 QUERY_STREAMS[13] = { + cpu_to_le16('q'), cpu_to_le16('u'), cpu_to_le16('e'), cpu_to_le16('r'), + cpu_to_le16('y'), cpu_to_le16('_'), cpu_to_le16('s'), cpu_to_le16('t'), + cpu_to_le16('r'), cpu_to_le16('e'), cpu_to_le16('a'), cpu_to_le16('m'), + cpu_to_le16('s'), +}; + + +#ifdef CONFIG_NTFS3_LZX_XPRESS +const __le16 WOF_NAME[17] = { + cpu_to_le16('W'), cpu_to_le16('o'), cpu_to_le16('f'), cpu_to_le16('C'), + cpu_to_le16('o'), cpu_to_le16('m'), cpu_to_le16('p'), cpu_to_le16('r'), + cpu_to_le16('e'), cpu_to_le16('s'), cpu_to_le16('s'), cpu_to_le16('e'), + cpu_to_le16('d'), cpu_to_le16('D'), cpu_to_le16('a'), cpu_to_le16('t'), + cpu_to_le16('a'), +}; +#endif + +static const __le16 CON_NAME[3] = { + cpu_to_le16('C'), cpu_to_le16('O'), cpu_to_le16('N'), +}; + +static const __le16 NUL_NAME[3] = { + cpu_to_le16('N'), cpu_to_le16('U'), cpu_to_le16('L'), +}; + +static const __le16 AUX_NAME[3] = { + cpu_to_le16('A'), cpu_to_le16('U'), cpu_to_le16('X'), +}; + +static const __le16 PRN_NAME[3] = { + cpu_to_le16('P'), cpu_to_le16('R'), cpu_to_le16('N'), +}; + +static const __le16 COM_NAME[3] = { + cpu_to_le16('C'), cpu_to_le16('O'), cpu_to_le16('M'), +}; + +static const __le16 LPT_NAME[3] = { + cpu_to_le16('L'), cpu_to_le16('P'), cpu_to_le16('T'), +}; +// clang-format on + +/* + * ntfs_fix_pre_write - Insert fixups into @rhdr before writing to disk. + */ +bool ntfs_fix_pre_write(struct NTFS_RECORD_HEADER *rhdr, size_t bytes) +{ + u16 *fixup, *ptr; + u16 sample; + u16 fo = le16_to_cpu(rhdr->fix_off); + u16 fn = le16_to_cpu(rhdr->fix_num); + + if ((fo & 1) || fo + fn * sizeof(short) > SECTOR_SIZE || !fn-- || + fn * SECTOR_SIZE > bytes) { + return false; + } + + /* Get fixup pointer. */ + fixup = Add2Ptr(rhdr, fo); + + if (*fixup >= 0x7FFF) + *fixup = 1; + else + *fixup += 1; + + sample = *fixup; + + ptr = Add2Ptr(rhdr, SECTOR_SIZE - sizeof(short)); + + while (fn--) { + *++fixup = *ptr; + *ptr = sample; + ptr += SECTOR_SIZE / sizeof(short); + } + return true; +} + +/* + * ntfs_fix_post_read - Remove fixups after reading from disk. + * + * Return: < 0 if error, 0 if ok, 1 if need to update fixups. + */ +int ntfs_fix_post_read(struct NTFS_RECORD_HEADER *rhdr, size_t bytes, + bool simple) +{ + int ret; + u16 *fixup, *ptr; + u16 sample, fo, fn; + + fo = le16_to_cpu(rhdr->fix_off); + fn = simple ? ((bytes >> SECTOR_SHIFT) + 1) : + le16_to_cpu(rhdr->fix_num); + + /* Check errors. */ + if ((fo & 1) || fo + fn * sizeof(short) > SECTOR_SIZE || !fn-- || + fn * SECTOR_SIZE > bytes) { + return -E_NTFS_CORRUPT; + } + + /* Get fixup pointer. */ + fixup = Add2Ptr(rhdr, fo); + sample = *fixup; + ptr = Add2Ptr(rhdr, SECTOR_SIZE - sizeof(short)); + ret = 0; + + while (fn--) { + /* Test current word. */ + if (*ptr != sample) { + /* Fixup does not match! Is it serious error? */ + ret = -E_NTFS_FIXUP; + } + + /* Replace fixup. */ + *ptr = *++fixup; + ptr += SECTOR_SIZE / sizeof(short); + } + + return ret; +} + +/* + * ntfs_extend_init - Load $Extend file. + */ +int ntfs_extend_init(struct ntfs_sb_info *sbi) +{ + int err; + struct super_block *sb = sbi->sb; + struct inode *inode, *inode2; + struct MFT_REF ref; + + if (sbi->volume.major_ver < 3) { + ntfs_notice(sb, "Skip $Extend 'cause NTFS version"); + return 0; + } + + ref.low = cpu_to_le32(MFT_REC_EXTEND); + ref.high = 0; + ref.seq = cpu_to_le16(MFT_REC_EXTEND); + inode = ntfs_iget5(sb, &ref, &NAME_EXTEND); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + ntfs_err(sb, "Failed to load $Extend (%d).", err); + inode = NULL; + goto out; + } + + /* If ntfs_iget5() reads from disk it never returns bad inode. */ + if (!S_ISDIR(inode->i_mode)) { + err = -EINVAL; + goto out; + } + + /* Try to find $ObjId */ + inode2 = dir_search(inode, &NAME_OBJID); + if (inode2 && !IS_ERR(inode2)) { + if (is_bad_inode(inode2)) { + iput(inode2); + } else { + sbi->objid.ni = ntfs_i(inode2); + sbi->objid_no = inode2->i_ino; + } + } + + /* Try to find $Quota */ + inode2 = dir_search(inode, &NAME_QUOTA); + if (inode2 && !IS_ERR(inode2)) { + sbi->quota_no = inode2->i_ino; + iput(inode2); + } + + /* Try to find $Reparse */ + inode2 = dir_search(inode, &NAME_REPARSE); + if (inode2 && !IS_ERR(inode2)) { + sbi->reparse.ni = ntfs_i(inode2); + sbi->reparse_no = inode2->i_ino; + } + + /* Try to find $UsnJrnl */ + inode2 = dir_search(inode, &NAME_USNJRNL); + if (inode2 && !IS_ERR(inode2)) { + sbi->usn_jrnl_no = inode2->i_ino; + iput(inode2); + } + + err = 0; +out: + iput(inode); + return err; +} + +int ntfs_loadlog_and_replay(struct ntfs_inode *ni, struct ntfs_sb_info *sbi) +{ + int err = 0; + struct super_block *sb = sbi->sb; + bool initialized = false; + struct MFT_REF ref; + struct inode *inode; + + /* Check for 4GB. */ + if (ni->vfs_inode.i_size >= 0x100000000ull) { + ntfs_err(sb, "\x24LogFile is large than 4G."); + err = -EINVAL; + goto out; + } + + sbi->flags |= NTFS_FLAGS_LOG_REPLAYING; + + ref.low = cpu_to_le32(MFT_REC_MFT); + ref.high = 0; + ref.seq = cpu_to_le16(1); + + inode = ntfs_iget5(sb, &ref, NULL); + + if (IS_ERR(inode)) + inode = NULL; + + if (!inode) { + /* Try to use MFT copy. */ + u64 t64 = sbi->mft.lbo; + + sbi->mft.lbo = sbi->mft.lbo2; + inode = ntfs_iget5(sb, &ref, NULL); + sbi->mft.lbo = t64; + if (IS_ERR(inode)) + inode = NULL; + } + + if (!inode) { + err = -EINVAL; + ntfs_err(sb, "Failed to load $MFT."); + goto out; + } + + sbi->mft.ni = ntfs_i(inode); + + /* LogFile should not contains attribute list. */ + err = ni_load_all_mi(sbi->mft.ni); + if (!err) + err = log_replay(ni, &initialized); + + iput(inode); + sbi->mft.ni = NULL; + + sync_blockdev(sb->s_bdev); + invalidate_bdev(sb->s_bdev); + + if (sbi->flags & NTFS_FLAGS_NEED_REPLAY) { + err = 0; + goto out; + } + + if (sb_rdonly(sb) || !initialized) + goto out; + + /* Fill LogFile by '-1' if it is initialized. */ + err = ntfs_bio_fill_1(sbi, &ni->file.run); + +out: + sbi->flags &= ~NTFS_FLAGS_LOG_REPLAYING; + + return err; +} + +/* + * ntfs_look_for_free_space - Look for a free space in bitmap. + */ +int ntfs_look_for_free_space(struct ntfs_sb_info *sbi, CLST lcn, CLST len, + CLST *new_lcn, CLST *new_len, + enum ALLOCATE_OPT opt) +{ + int err; + CLST alen; + struct super_block *sb = sbi->sb; + size_t alcn, zlen, zeroes, zlcn, zlen2, ztrim, new_zlen; + struct wnd_bitmap *wnd = &sbi->used.bitmap; + + down_write_nested(&wnd->rw_lock, BITMAP_MUTEX_CLUSTERS); + if (opt & ALLOCATE_MFT) { + zlen = wnd_zone_len(wnd); + + if (!zlen) { + err = ntfs_refresh_zone(sbi); + if (err) + goto up_write; + + zlen = wnd_zone_len(wnd); + } + + if (!zlen) { + ntfs_err(sbi->sb, "no free space to extend mft"); + err = -ENOSPC; + goto up_write; + } + + lcn = wnd_zone_bit(wnd); + alen = min_t(CLST, len, zlen); + + wnd_zone_set(wnd, lcn + alen, zlen - alen); + + err = wnd_set_used(wnd, lcn, alen); + if (err) + goto up_write; + + alcn = lcn; + goto space_found; + } + /* + * 'Cause cluster 0 is always used this value means that we should use + * cached value of 'next_free_lcn' to improve performance. + */ + if (!lcn) + lcn = sbi->used.next_free_lcn; + + if (lcn >= wnd->nbits) + lcn = 0; + + alen = wnd_find(wnd, len, lcn, BITMAP_FIND_MARK_AS_USED, &alcn); + if (alen) + goto space_found; + + /* Try to use clusters from MftZone. */ + zlen = wnd_zone_len(wnd); + zeroes = wnd_zeroes(wnd); + + /* Check too big request */ + if (len > zeroes + zlen || zlen <= NTFS_MIN_MFT_ZONE) { + err = -ENOSPC; + goto up_write; + } + + /* How many clusters to cat from zone. */ + zlcn = wnd_zone_bit(wnd); + zlen2 = zlen >> 1; + ztrim = clamp_val(len, zlen2, zlen); + new_zlen = max_t(size_t, zlen - ztrim, NTFS_MIN_MFT_ZONE); + + wnd_zone_set(wnd, zlcn, new_zlen); + + /* Allocate continues clusters. */ + alen = wnd_find(wnd, len, 0, + BITMAP_FIND_MARK_AS_USED | BITMAP_FIND_FULL, &alcn); + if (!alen) { + err = -ENOSPC; + goto up_write; + } + +space_found: + err = 0; + *new_len = alen; + *new_lcn = alcn; + + ntfs_unmap_meta(sb, alcn, alen); + + /* Set hint for next requests. */ + if (!(opt & ALLOCATE_MFT)) + sbi->used.next_free_lcn = alcn + alen; +up_write: + up_write(&wnd->rw_lock); + return err; +} + +/* + * ntfs_check_free_space + * + * Check if it is possible to allocate 'clen' clusters and 'mlen' Mft records + */ +bool ntfs_check_free_space(struct ntfs_sb_info *sbi, CLST clen, CLST mlen, + bool da) +{ + size_t free, zlen, avail; + struct wnd_bitmap *wnd; + CLST da_clusters = ntfs_get_da(sbi); + + wnd = &sbi->used.bitmap; + down_read_nested(&wnd->rw_lock, BITMAP_MUTEX_CLUSTERS); + free = wnd_zeroes(wnd); + + if (free >= da_clusters) { + free -= da_clusters; + } else { + free = 0; + } + + zlen = min_t(size_t, NTFS_MIN_MFT_ZONE, wnd_zone_len(wnd)); + up_read(&wnd->rw_lock); + + if (free < zlen + clen) { + return false; + } + + avail = free - (zlen + clen); + + /* + * When delalloc is active then keep in mind some reserved space. + * The worst case: 1 mft record per each ~500 clusters. + */ + if (da) { + /* 1 mft record per each 1024 clusters. */ + mlen += da_clusters >> 10; + } + + if (mlen || !avail) { + wnd = &sbi->mft.bitmap; + down_read_nested(&wnd->rw_lock, BITMAP_MUTEX_MFT); + free = wnd_zeroes(wnd); + zlen = wnd_zone_len(wnd); + up_read(&wnd->rw_lock); + + if (free < zlen + mlen && + avail < bytes_to_cluster(sbi, mlen << sbi->record_bits)) { + return false; + } + } + + return true; +} + +/* + * ntfs_extend_mft - Allocate additional MFT records. + * + * sbi->mft.bitmap is locked for write. + * + * NOTE: recursive: + * ntfs_look_free_mft -> + * ntfs_extend_mft -> + * attr_set_size -> + * ni_insert_nonresident -> + * ni_insert_attr -> + * ni_ins_attr_ext -> + * ntfs_look_free_mft -> + * ntfs_extend_mft + * + * To avoid recursive always allocate space for two new MFT records + * see attrib.c: "at least two MFT to avoid recursive loop". + */ +static int ntfs_extend_mft(struct ntfs_sb_info *sbi) +{ + int err; + struct ntfs_inode *ni = sbi->mft.ni; + size_t new_mft_total; + u64 new_mft_bytes, new_bitmap_bytes; + struct ATTRIB *attr; + struct wnd_bitmap *wnd = &sbi->mft.bitmap; + + new_mft_total = ALIGN(wnd->nbits + NTFS_MFT_INCREASE_STEP, 128); + new_mft_bytes = (u64)new_mft_total << sbi->record_bits; + + /* Step 1: Resize $MFT::DATA. */ + down_write(&ni->file.run_lock); + err = attr_set_size_ex(ni, ATTR_DATA, NULL, 0, &ni->file.run, + new_mft_bytes, NULL, false, &attr, false); + + if (err) { + up_write(&ni->file.run_lock); + goto out; + } + + attr->nres.valid_size = attr->nres.data_size; + new_mft_total = le64_to_cpu(attr->nres.alloc_size) >> sbi->record_bits; + ni->mi.dirty = true; + + /* Step 2: Resize $MFT::BITMAP. */ + new_bitmap_bytes = ntfs3_bitmap_size(new_mft_total); + + err = attr_set_size(ni, ATTR_BITMAP, NULL, 0, &sbi->mft.bitmap.run, + new_bitmap_bytes, &new_bitmap_bytes, true); + + /* Refresh MFT Zone if necessary. */ + down_write_nested(&sbi->used.bitmap.rw_lock, BITMAP_MUTEX_CLUSTERS); + + ntfs_refresh_zone(sbi); + + up_write(&sbi->used.bitmap.rw_lock); + up_write(&ni->file.run_lock); + + if (err) + goto out; + + err = wnd_extend(wnd, new_mft_total); + + if (err) + goto out; + + ntfs_clear_mft_tail(sbi, sbi->mft.used, new_mft_total); + + err = _ni_write_inode(&ni->vfs_inode, 0); +out: + return err; +} + +/* + * ntfs_look_free_mft - Look for a free MFT record. + */ +int ntfs_look_free_mft(struct ntfs_sb_info *sbi, CLST *rno, bool mft, + struct ntfs_inode *ni, struct mft_inode **mi) +{ + int err = 0; + size_t zbit, zlen, from, to, fr; + size_t mft_total; + struct MFT_REF ref; + struct super_block *sb = sbi->sb; + struct wnd_bitmap *wnd = &sbi->mft.bitmap; + u32 ir; + + static_assert(sizeof(sbi->mft.reserved_bitmap) * 8 >= + MFT_REC_FREE - MFT_REC_RESERVED); + + if (!mft) + down_write_nested(&wnd->rw_lock, BITMAP_MUTEX_MFT); + + zlen = wnd_zone_len(wnd); + + /* Always reserve space for MFT. */ + if (zlen) { + if (mft) { + zbit = wnd_zone_bit(wnd); + *rno = zbit; + wnd_zone_set(wnd, zbit + 1, zlen - 1); + } + goto found; + } + + /* No MFT zone. Find the nearest to '0' free MFT. */ + if (!wnd_find(wnd, 1, MFT_REC_FREE, 0, &zbit)) { + /* Resize MFT */ + mft_total = wnd->nbits; + + err = ntfs_extend_mft(sbi); + if (!err) { + zbit = mft_total; + goto reserve_mft; + } + + if (!mft || MFT_REC_FREE == sbi->mft.next_reserved) + goto out; + + err = 0; + + /* + * Look for free record reserved area [11-16) == + * [MFT_REC_RESERVED, MFT_REC_FREE ) MFT bitmap always + * marks it as used. + */ + if (!sbi->mft.reserved_bitmap) { + /* Once per session create internal bitmap for 5 bits. */ + sbi->mft.reserved_bitmap = 0xFF; + + ref.high = 0; + for (ir = MFT_REC_RESERVED; ir < MFT_REC_FREE; ir++) { + struct inode *i; + struct ntfs_inode *ni; + struct MFT_REC *mrec; + + ref.low = cpu_to_le32(ir); + ref.seq = cpu_to_le16(ir); + + i = ntfs_iget5(sb, &ref, NULL); + if (IS_ERR(i)) { +next: + ntfs_notice( + sb, + "Invalid reserved record %x", + ref.low); + continue; + } + if (is_bad_inode(i)) { + iput(i); + goto next; + } + + ni = ntfs_i(i); + + mrec = ni->mi.mrec; + + if (!is_rec_base(mrec)) + goto next; + + if (mrec->hard_links) + goto next; + + if (!ni_std(ni)) + goto next; + + if (ni_find_attr(ni, NULL, NULL, ATTR_NAME, + NULL, 0, NULL, NULL)) + goto next; + + __clear_bit(ir - MFT_REC_RESERVED, + &sbi->mft.reserved_bitmap); + } + } + + /* Scan 5 bits for zero. Bit 0 == MFT_REC_RESERVED */ + zbit = find_next_zero_bit(&sbi->mft.reserved_bitmap, + MFT_REC_FREE, MFT_REC_RESERVED); + if (zbit >= MFT_REC_FREE) { + sbi->mft.next_reserved = MFT_REC_FREE; + goto out; + } + + zlen = 1; + sbi->mft.next_reserved = zbit; + } else { +reserve_mft: + zlen = zbit == MFT_REC_FREE ? (MFT_REC_USER - MFT_REC_FREE) : 4; + if (zbit + zlen > wnd->nbits) + zlen = wnd->nbits - zbit; + + while (zlen > 1 && !wnd_is_free(wnd, zbit, zlen)) + zlen -= 1; + + /* [zbit, zbit + zlen) will be used for MFT itself. */ + from = sbi->mft.used; + if (from < zbit) + from = zbit; + to = zbit + zlen; + if (from < to) { + ntfs_clear_mft_tail(sbi, from, to); + sbi->mft.used = to; + } + } + + if (mft) { + *rno = zbit; + zbit += 1; + zlen -= 1; + } + + wnd_zone_set(wnd, zbit, zlen); + +found: + if (!mft) { + /* The request to get record for general purpose. */ + if (sbi->mft.next_free < MFT_REC_USER) + sbi->mft.next_free = MFT_REC_USER; + + for (;;) { + if (sbi->mft.next_free >= sbi->mft.bitmap.nbits) { + } else if (!wnd_find(wnd, 1, MFT_REC_USER, 0, &fr)) { + sbi->mft.next_free = sbi->mft.bitmap.nbits; + } else { + *rno = fr; + sbi->mft.next_free = *rno + 1; + break; + } + + err = ntfs_extend_mft(sbi); + if (err) + goto out; + } + } + + if (ni && !ni_add_subrecord(ni, *rno, mi)) { + err = -ENOMEM; + goto out; + } + + /* We have found a record that are not reserved for next MFT. */ + if (*rno >= MFT_REC_FREE) + wnd_set_used(wnd, *rno, 1); + else if (*rno >= MFT_REC_RESERVED && sbi->mft.reserved_bitmap_inited) + __set_bit(*rno - MFT_REC_RESERVED, &sbi->mft.reserved_bitmap); + +out: + if (!mft) + up_write(&wnd->rw_lock); + + return err; +} + +/* + * ntfs_mark_rec_free - Mark record as free. + * is_mft - true if we are changing MFT + */ +void ntfs_mark_rec_free(struct ntfs_sb_info *sbi, CLST rno, bool is_mft) +{ + struct wnd_bitmap *wnd = &sbi->mft.bitmap; + + if (!is_mft) + down_write_nested(&wnd->rw_lock, BITMAP_MUTEX_MFT); + if (rno >= wnd->nbits) + goto out; + + if (rno >= MFT_REC_FREE) { + if (!wnd_is_used(wnd, rno, 1)) + ntfs_set_state(sbi, NTFS_DIRTY_ERROR); + else + wnd_set_free(wnd, rno, 1); + } else if (rno >= MFT_REC_RESERVED && sbi->mft.reserved_bitmap_inited) { + __clear_bit(rno - MFT_REC_RESERVED, &sbi->mft.reserved_bitmap); + } + + if (rno < wnd_zone_bit(wnd)) + wnd_zone_set(wnd, rno, 1); + else if (rno < sbi->mft.next_free && rno >= MFT_REC_USER) + sbi->mft.next_free = rno; + +out: + if (!is_mft) + up_write(&wnd->rw_lock); +} + +/* + * ntfs_clear_mft_tail - Format empty records [from, to). + * + * sbi->mft.bitmap is locked for write. + */ +int ntfs_clear_mft_tail(struct ntfs_sb_info *sbi, size_t from, size_t to) +{ + int err; + u32 rs; + u64 vbo; + struct runs_tree *run; + struct ntfs_inode *ni; + + if (from >= to) + return 0; + + rs = sbi->record_size; + ni = sbi->mft.ni; + run = &ni->file.run; + + down_read(&ni->file.run_lock); + vbo = (u64)from * rs; + for (; from < to; from++, vbo += rs) { + struct ntfs_buffers nb; + + err = ntfs_get_bh(sbi, run, vbo, rs, &nb); + if (err) + goto out; + + err = ntfs_write_bh(sbi, &sbi->new_rec->rhdr, &nb, 0); + nb_put(&nb); + if (err) + goto out; + } + +out: + sbi->mft.used = from; + up_read(&ni->file.run_lock); + return err; +} + +/* + * ntfs_refresh_zone - Refresh MFT zone. + * + * sbi->used.bitmap is locked for rw. + * sbi->mft.bitmap is locked for write. + * sbi->mft.ni->file.run_lock for write. + */ +int ntfs_refresh_zone(struct ntfs_sb_info *sbi) +{ + CLST lcn, vcn, len; + size_t lcn_s, zlen; + struct wnd_bitmap *wnd = &sbi->used.bitmap; + struct ntfs_inode *ni = sbi->mft.ni; + + /* Do not change anything unless we have non empty MFT zone. */ + if (wnd_zone_len(wnd)) + return 0; + + vcn = bytes_to_cluster(sbi, + (u64)sbi->mft.bitmap.nbits << sbi->record_bits); + + if (!run_lookup_entry(&ni->file.run, vcn - 1, &lcn, &len, NULL)) + lcn = SPARSE_LCN; + + /* We should always find Last Lcn for MFT. */ + if (lcn == SPARSE_LCN) + return -EINVAL; + + lcn_s = lcn + 1; + + /* Try to allocate clusters after last MFT run. */ + zlen = wnd_find(wnd, sbi->zone_max, lcn_s, 0, &lcn_s); + wnd_zone_set(wnd, lcn_s, zlen); + + return 0; +} + +/* + * ntfs_update_mftmirr - Update $MFTMirr data. + */ +void ntfs_update_mftmirr(struct ntfs_sb_info *sbi) +{ + struct super_block *sb = sbi->sb; + u32 blocksize, bytes; + sector_t block1, block2; + + /* + * sb can be NULL here. In this case sbi->flags should be 0 too. + */ + if (!sb || !(sbi->flags & NTFS_FLAGS_MFTMIRR) || + unlikely(ntfs3_forced_shutdown(sb))) + return; + + blocksize = sb->s_blocksize; + bytes = sbi->mft.recs_mirr << sbi->record_bits; + block1 = sbi->mft.lbo >> sb->s_blocksize_bits; + block2 = sbi->mft.lbo2 >> sb->s_blocksize_bits; + + for (; bytes >= blocksize; bytes -= blocksize) { + struct buffer_head *bh1, *bh2; + + bh1 = sb_bread(sb, block1++); + if (!bh1) + return; + + bh2 = sb_getblk(sb, block2++); + if (!bh2) { + put_bh(bh1); + return; + } + + wait_on_buffer(bh2); + lock_buffer(bh2); + memcpy(bh2->b_data, bh1->b_data, blocksize); + set_buffer_uptodate(bh2); + mark_buffer_dirty(bh2); + unlock_buffer(bh2); + + put_bh(bh1); + bh1 = NULL; + put_bh(bh2); + } + + sbi->flags &= ~NTFS_FLAGS_MFTMIRR; +} + +/* + * ntfs_bad_inode + * + * Marks inode as bad and marks fs as 'dirty' + */ +void ntfs_bad_inode(struct inode *inode, const char *hint) +{ + struct ntfs_sb_info *sbi = inode->i_sb->s_fs_info; + struct ntfs_inode *ni = ntfs_i(inode); + + ntfs_inode_err(inode, "%s", hint); + + /* Do not call make_bad_inode()! */ + ni->ni_bad = true; + + /* Avoid recursion if bad inode is $Volume. */ + if (inode->i_ino != MFT_REC_VOL && + !(sbi->flags & NTFS_FLAGS_LOG_REPLAYING)) { + ntfs_set_state(sbi, NTFS_DIRTY_ERROR); + } +} + +/* + * ntfs_set_state + * + * Mount: ntfs_set_state(NTFS_DIRTY_DIRTY) + * Umount: ntfs_set_state(NTFS_DIRTY_CLEAR) + * NTFS error: ntfs_set_state(NTFS_DIRTY_ERROR) + */ +int ntfs_set_state(struct ntfs_sb_info *sbi, enum NTFS_DIRTY_FLAGS dirty) +{ + int err; + struct ATTRIB *attr; + struct VOLUME_INFO *info; + struct mft_inode *mi; + struct ntfs_inode *ni; + __le16 info_flags; + + /* + * Do not change state if fs was real_dirty. + * Do not change state if fs already dirty(clear). + * Do not change any thing if mounted read only. + */ + if (sbi->volume.real_dirty || sb_rdonly(sbi->sb)) + return 0; + + /* Check cached value. */ + if ((dirty == NTFS_DIRTY_CLEAR ? 0 : VOLUME_FLAG_DIRTY) == + (sbi->volume.flags & VOLUME_FLAG_DIRTY)) + return 0; + + ni = sbi->volume.ni; + if (!ni) + return -EINVAL; + + mutex_lock_nested(&ni->ni_lock, NTFS_INODE_MUTEX_DIRTY); + + attr = ni_find_attr(ni, NULL, NULL, ATTR_VOL_INFO, NULL, 0, NULL, &mi); + if (!attr) { + err = -EINVAL; + goto out; + } + + info = resident_data_ex(attr, SIZEOF_ATTRIBUTE_VOLUME_INFO); + if (!info) { + err = -EINVAL; + goto out; + } + + info_flags = info->flags; + + switch (dirty) { + case NTFS_DIRTY_ERROR: + ntfs_notice(sbi->sb, "Mark volume as dirty due to NTFS errors"); + sbi->volume.real_dirty = true; + fallthrough; + case NTFS_DIRTY_DIRTY: + info->flags |= VOLUME_FLAG_DIRTY; + break; + case NTFS_DIRTY_CLEAR: + info->flags &= ~VOLUME_FLAG_DIRTY; + break; + } + /* Cache current volume flags. */ + if (info_flags != info->flags) { + sbi->volume.flags = info->flags; + mi->dirty = true; + } + err = 0; + +out: + ni_unlock(ni); + if (err) + return err; + + mark_inode_dirty_sync(&ni->vfs_inode); + /* verify(!ntfs_update_mftmirr()); */ + + /* write mft record on disk. */ + err = _ni_write_inode(&ni->vfs_inode, 1); + + return err; +} + +/* + * security_hash - Calculates a hash of security descriptor. + */ +static inline __le32 security_hash(const void *sd, size_t bytes) +{ + u32 hash = 0; + const __le32 *ptr = sd; + + bytes >>= 2; + while (bytes--) + hash = ((hash >> 0x1D) | (hash << 3)) + le32_to_cpu(*ptr++); + return cpu_to_le32(hash); +} + +/* + * simple wrapper for sb_bread_unmovable. + */ +struct buffer_head *ntfs_bread(struct super_block *sb, sector_t block) +{ + struct ntfs_sb_info *sbi = sb->s_fs_info; + struct buffer_head *bh; + + if (unlikely(block >= sbi->volume.blocks)) { + /* prevent generic message "attempt to access beyond end of device" */ + ntfs_err(sb, "try to read out of volume at offset 0x%llx", + (u64)block << sb->s_blocksize_bits); + return NULL; + } + + bh = sb_bread_unmovable(sb, block); + if (bh) + return bh; + + ntfs_err(sb, "failed to read volume at offset 0x%llx", + (u64)block << sb->s_blocksize_bits); + return NULL; +} + +int ntfs_sb_write(struct super_block *sb, u64 lbo, size_t bytes, + const void *buf, int wait) +{ + u32 blocksize = sb->s_blocksize; + struct block_device *bdev = sb->s_bdev; + sector_t block = lbo >> sb->s_blocksize_bits; + u32 off = lbo & (blocksize - 1); + u32 op = blocksize - off; + struct buffer_head *bh; + + if (!wait && (sb->s_flags & SB_SYNCHRONOUS)) + wait = 1; + + for (; bytes; block += 1, off = 0, op = blocksize) { + if (op > bytes) + op = bytes; + + if (op < blocksize) { + bh = __bread(bdev, block, blocksize); + if (!bh) { + ntfs_err(sb, "failed to read block %llx", + (u64)block); + return -EIO; + } + } else { + bh = __getblk(bdev, block, blocksize); + if (!bh) + return -ENOMEM; + } + + wait_on_buffer(bh); + lock_buffer(bh); + if (buf) { + memcpy(bh->b_data + off, buf, op); + buf = Add2Ptr(buf, op); + } else { + memset(bh->b_data + off, -1, op); + } + + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + unlock_buffer(bh); + + if (wait) { + int err = sync_dirty_buffer(bh); + + if (err) { + ntfs_err( + sb, + "failed to sync buffer at block %llx, error %d", + (u64)block, err); + put_bh(bh); + return err; + } + } + + put_bh(bh); + + bytes -= op; + } + return 0; +} + +int ntfs_sb_write_run(struct ntfs_sb_info *sbi, const struct runs_tree *run, + u64 vbo, const void *buf, size_t bytes, int sync) +{ + struct super_block *sb = sbi->sb; + u8 cluster_bits = sbi->cluster_bits; + u32 off = vbo & sbi->cluster_mask; + CLST lcn, clen, vcn = vbo >> cluster_bits, vcn_next; + u64 lbo, len; + size_t idx; + + if (!run_lookup_entry(run, vcn, &lcn, &clen, &idx)) + return -ENOENT; + + if (lcn == SPARSE_LCN) + return -EINVAL; + + lbo = ((u64)lcn << cluster_bits) + off; + len = ((u64)clen << cluster_bits) - off; + + for (;;) { + u32 op = min_t(u64, len, bytes); + int err = ntfs_sb_write(sb, lbo, op, buf, sync); + + if (err) + return err; + + bytes -= op; + if (!bytes) + break; + + vcn_next = vcn + clen; + if (!run_get_entry(run, ++idx, &vcn, &lcn, &clen) || + vcn != vcn_next) + return -ENOENT; + + if (lcn == SPARSE_LCN) + return -EINVAL; + + if (buf) + buf = Add2Ptr(buf, op); + + lbo = ((u64)lcn << cluster_bits); + len = ((u64)clen << cluster_bits); + } + + return 0; +} + +struct buffer_head *ntfs_bread_run(struct ntfs_sb_info *sbi, + const struct runs_tree *run, u64 vbo) +{ + struct super_block *sb = sbi->sb; + u8 cluster_bits = sbi->cluster_bits; + CLST lcn; + u64 lbo; + + if (!run_lookup_entry(run, vbo >> cluster_bits, &lcn, NULL, NULL)) + return ERR_PTR(-ENOENT); + + lbo = ((u64)lcn << cluster_bits) + (vbo & sbi->cluster_mask); + + return ntfs_bread(sb, lbo >> sb->s_blocksize_bits); +} + +int ntfs_read_run_nb_ra(struct ntfs_sb_info *sbi, const struct runs_tree *run, + u64 vbo, void *buf, u32 bytes, struct ntfs_buffers *nb, + struct file_ra_state *ra) +{ + int err; + struct super_block *sb = sbi->sb; + struct address_space *mapping = sb->s_bdev->bd_mapping; + u32 blocksize = sb->s_blocksize; + u8 cluster_bits = sbi->cluster_bits; + u32 off = vbo & sbi->cluster_mask; + u32 nbh = 0; + CLST vcn_next, vcn = vbo >> cluster_bits; + CLST lcn, clen; + u64 lbo, len; + size_t idx; + struct buffer_head *bh; + + if (!run) { + /* First reading of $Volume + $MFTMirr + $LogFile goes here. */ + if (vbo > MFT_REC_VOL * sbi->record_size) { + err = -ENOENT; + goto out; + } + + /* Use absolute boot's 'MFTCluster' to read record. */ + lbo = vbo + sbi->mft.lbo; + len = sbi->record_size; + } else if (!run_lookup_entry(run, vcn, &lcn, &clen, &idx)) { + err = -ENOENT; + goto out; + } else { + if (lcn == SPARSE_LCN) { + err = -EINVAL; + goto out; + } + + lbo = ((u64)lcn << cluster_bits) + off; + len = ((u64)clen << cluster_bits) - off; + } + + off = lbo & (blocksize - 1); + if (nb) { + nb->off = off; + nb->bytes = bytes; + } + + if (ra && !ra->ra_pages) + file_ra_state_init(ra, mapping); + + for (;;) { + u32 len32 = len >= bytes ? bytes : len; + sector_t block = lbo >> sb->s_blocksize_bits; + + if (ra) { + pgoff_t index = lbo >> PAGE_SHIFT; + if (!ra_has_index(ra, index)) { + page_cache_sync_readahead(mapping, ra, NULL, + index, 1); + ra->prev_pos = (loff_t)index << PAGE_SHIFT; + } + } + + do { + u32 op = blocksize - off; + + if (op > len32) + op = len32; + + bh = ntfs_bread(sb, block); + if (!bh) { + err = -EIO; + goto out; + } + + if (buf) { + memcpy(buf, bh->b_data + off, op); + buf = Add2Ptr(buf, op); + } + + if (!nb) { + put_bh(bh); + } else if (nbh >= ARRAY_SIZE(nb->bh)) { + err = -EINVAL; + goto out; + } else { + nb->bh[nbh++] = bh; + nb->nbufs = nbh; + } + + bytes -= op; + if (!bytes) + return 0; + len32 -= op; + block += 1; + off = 0; + + } while (len32); + + if (!run) { + err = -EINVAL; + goto out; + } + + /* Get next fragment to read. */ + vcn_next = vcn + clen; + if (!run_get_entry(run, ++idx, &vcn, &lcn, &clen) || + vcn != vcn_next) { + err = -ENOENT; + goto out; + } + + if (lcn == SPARSE_LCN) { + err = -EINVAL; + goto out; + } + + lbo = ((u64)lcn << cluster_bits); + len = ((u64)clen << cluster_bits); + } + +out: + if (!nbh) + return err; + + while (nbh) { + put_bh(nb->bh[--nbh]); + nb->bh[nbh] = NULL; + } + + nb->nbufs = 0; + return err; +} + +/* + * ntfs_read_bh + * + * Return: < 0 if error, 0 if ok, -E_NTFS_FIXUP if need to update fixups. + */ +int ntfs_read_bh_ra(struct ntfs_sb_info *sbi, const struct runs_tree *run, + u64 vbo, struct NTFS_RECORD_HEADER *rhdr, u32 bytes, + struct ntfs_buffers *nb, struct file_ra_state *ra) +{ + int err = ntfs_read_run_nb_ra(sbi, run, vbo, rhdr, bytes, nb, ra); + + if (err) + return err; + return ntfs_fix_post_read(rhdr, nb->bytes, true); +} + +int ntfs_get_bh(struct ntfs_sb_info *sbi, const struct runs_tree *run, u64 vbo, + u32 bytes, struct ntfs_buffers *nb) +{ + int err = 0; + struct super_block *sb = sbi->sb; + u32 blocksize = sb->s_blocksize; + u8 cluster_bits = sbi->cluster_bits; + CLST vcn_next, vcn = vbo >> cluster_bits; + u32 off; + u32 nbh = 0; + CLST lcn, clen; + u64 lbo, len; + size_t idx; + + nb->bytes = bytes; + + if (!run_lookup_entry(run, vcn, &lcn, &clen, &idx)) { + err = -ENOENT; + goto out; + } + + off = vbo & sbi->cluster_mask; + lbo = ((u64)lcn << cluster_bits) + off; + len = ((u64)clen << cluster_bits) - off; + + nb->off = off = lbo & (blocksize - 1); + + for (;;) { + u32 len32 = min_t(u64, len, bytes); + sector_t block = lbo >> sb->s_blocksize_bits; + + do { + u32 op; + struct buffer_head *bh; + + if (nbh >= ARRAY_SIZE(nb->bh)) { + err = -EINVAL; + goto out; + } + + op = blocksize - off; + if (op > len32) + op = len32; + + if (op == blocksize) { + bh = sb_getblk(sb, block); + if (!bh) { + err = -ENOMEM; + goto out; + } + wait_on_buffer(bh); + lock_buffer(bh); + if (!buffer_uptodate(bh)) { + memset(bh->b_data, 0, blocksize); + set_buffer_uptodate(bh); + } + unlock_buffer(bh); + } else { + bh = ntfs_bread(sb, block); + if (!bh) { + err = -EIO; + goto out; + } + } + + nb->bh[nbh++] = bh; + bytes -= op; + if (!bytes) { + nb->nbufs = nbh; + return 0; + } + + block += 1; + len32 -= op; + off = 0; + } while (len32); + + vcn_next = vcn + clen; + if (!run_get_entry(run, ++idx, &vcn, &lcn, &clen) || + vcn != vcn_next) { + err = -ENOENT; + goto out; + } + + lbo = ((u64)lcn << cluster_bits); + len = ((u64)clen << cluster_bits); + } + +out: + while (nbh) { + put_bh(nb->bh[--nbh]); + nb->bh[nbh] = NULL; + } + + nb->nbufs = 0; + + return err; +} + +int ntfs_write_bh(struct ntfs_sb_info *sbi, struct NTFS_RECORD_HEADER *rhdr, + struct ntfs_buffers *nb, int sync) +{ + int err = 0; + struct super_block *sb = sbi->sb; + u32 block_size = sb->s_blocksize; + u32 bytes = nb->bytes; + u32 off = nb->off; + u16 fo = le16_to_cpu(rhdr->fix_off); + u16 fn = le16_to_cpu(rhdr->fix_num); + u32 idx; + __le16 *fixup = NULL; + __le16 sample = cpu_to_le16(-1u); + + if ((fo & 1) || fo + fn * sizeof(short) > SECTOR_SIZE || !fn-- || + fn * SECTOR_SIZE > bytes) { + return -EINVAL; + } + + for (idx = 0; bytes && idx < nb->nbufs; idx += 1, off = 0) { + u32 op = block_size - off; + char *bh_data; + struct buffer_head *bh = nb->bh[idx]; + __le16 *ptr, *end_data; + + if (op > bytes) + op = bytes; + + wait_on_buffer(bh); + lock_buffer(bh); + + bh_data = bh->b_data + off; + end_data = Add2Ptr(bh_data, op); + memcpy(bh_data, rhdr, op); + + if (!idx) { + u16 t16; + + fixup = Add2Ptr(bh_data, fo); + sample = *fixup; + t16 = le16_to_cpu(sample); + if (t16 >= 0x7FFF) { + sample = *fixup = cpu_to_le16(1); + } else { + sample = cpu_to_le16(t16 + 1); + *fixup = sample; + } + + *(__le16 *)Add2Ptr(rhdr, fo) = sample; + } + + ptr = Add2Ptr(bh_data, SECTOR_SIZE - sizeof(short)); + + do { + *++fixup = *ptr; + *ptr = sample; + ptr += SECTOR_SIZE / sizeof(short); + } while (ptr < end_data); + + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + unlock_buffer(bh); + + if (sync) { + int err2 = sync_dirty_buffer(bh); + + if (!err && err2) + err = err2; + } + + bytes -= op; + rhdr = Add2Ptr(rhdr, op); + } + + return err; +} + +/* + * ntfs_read_write_run - Read/Write disk's page cache. + */ +int ntfs_read_write_run(struct ntfs_sb_info *sbi, const struct runs_tree *run, + void *buf, u64 vbo, size_t bytes, int wr) +{ + struct super_block *sb = sbi->sb; + struct address_space *mapping = sb->s_bdev->bd_mapping; + u8 cluster_bits = sbi->cluster_bits; + CLST vcn_next, vcn = vbo >> cluster_bits; + CLST lcn, clen; + u64 lbo, len; + size_t idx; + u32 off, op; + struct folio *folio; + char *kaddr; + + if (!bytes) + return 0; + + if (!run_lookup_entry(run, vcn, &lcn, &clen, &idx)) + return -ENOENT; + + if (lcn == SPARSE_LCN) + return -EINVAL; + + off = vbo & sbi->cluster_mask; + lbo = ((u64)lcn << cluster_bits) + off; + len = ((u64)clen << cluster_bits) - off; + + for (;;) { + /* Read range [lbo, lbo+len). */ + folio = read_mapping_folio(mapping, lbo >> PAGE_SHIFT, NULL); + + if (IS_ERR(folio)) + return PTR_ERR(folio); + + off = offset_in_page(lbo); + op = PAGE_SIZE - off; + + if (op > len) + op = len; + if (op > bytes) + op = bytes; + + kaddr = kmap_local_folio(folio, 0); + if (wr) { + memcpy(kaddr + off, buf, op); + folio_mark_dirty(folio); + } else { + memcpy(buf, kaddr + off, op); + flush_dcache_folio(folio); + } + kunmap_local(kaddr); + folio_put(folio); + + bytes -= op; + if (!bytes) + return 0; + + buf += op; + len -= op; + if (len) { + /* next volume's page. */ + lbo += op; + continue; + } + + /* get next range. */ + vcn_next = vcn + clen; + if (!run_get_entry(run, ++idx, &vcn, &lcn, &clen) || + vcn != vcn_next) { + return -ENOENT; + } + + if (lcn == SPARSE_LCN) + return -EINVAL; + + lbo = ((u64)lcn << cluster_bits); + len = ((u64)clen << cluster_bits); + } +} + +/* + * ntfs_bio_fill_1 - Helper for ntfs_loadlog_and_replay(). + * + * Fill on-disk logfile range by (-1) + * this means empty logfile. + */ +int ntfs_bio_fill_1(struct ntfs_sb_info *sbi, const struct runs_tree *run) +{ + int err = 0; + struct super_block *sb = sbi->sb; + struct block_device *bdev = sb->s_bdev; + u8 cluster_bits = sbi->cluster_bits; + struct bio *new, *bio = NULL; + CLST lcn, clen; + u64 lbo, len; + size_t run_idx; + struct page *fill; + void *kaddr; + struct blk_plug plug; + + fill = alloc_page(GFP_KERNEL); + if (!fill) + return -ENOMEM; + + kaddr = kmap_atomic(fill); + memset(kaddr, -1, PAGE_SIZE); + kunmap_atomic(kaddr); + flush_dcache_page(fill); + lock_page(fill); + + if (!run_lookup_entry(run, 0, &lcn, &clen, &run_idx)) { + err = -ENOENT; + goto out; + } + + /* + * TODO: Try blkdev_issue_write_same. + */ + blk_start_plug(&plug); + do { + lbo = (u64)lcn << cluster_bits; + len = (u64)clen << cluster_bits; +new_bio: + new = bio_alloc(bdev, BIO_MAX_VECS, REQ_OP_WRITE, GFP_NOFS); + if (bio) { + bio_chain(bio, new); + submit_bio(bio); + } + bio = new; + bio->bi_iter.bi_sector = lbo >> 9; + + for (;;) { + u32 add = len > PAGE_SIZE ? PAGE_SIZE : len; + + if (bio_add_page(bio, fill, add, 0) < add) + goto new_bio; + + lbo += add; + if (len <= add) + break; + len -= add; + } + } while (run_get_entry(run, ++run_idx, NULL, &lcn, &clen)); + + if (!err) + err = submit_bio_wait(bio); + bio_put(bio); + + blk_finish_plug(&plug); +out: + unlock_page(fill); + put_page(fill); + + return err; +} + +int ntfs_vbo_to_lbo(struct ntfs_sb_info *sbi, const struct runs_tree *run, + u64 vbo, u64 *lbo, u64 *bytes) +{ + u32 off; + CLST lcn, len; + u8 cluster_bits = sbi->cluster_bits; + + if (!run_lookup_entry(run, vbo >> cluster_bits, &lcn, &len, NULL)) + return -ENOENT; + + off = vbo & sbi->cluster_mask; + *lbo = lcn == SPARSE_LCN ? -1 : (((u64)lcn << cluster_bits) + off); + *bytes = ((u64)len << cluster_bits) - off; + + return 0; +} + +struct ntfs_inode *ntfs_new_inode(struct ntfs_sb_info *sbi, CLST rno, + enum RECORD_FLAG flag) +{ + int err = 0; + struct super_block *sb = sbi->sb; + struct inode *inode = new_inode(sb); + struct ntfs_inode *ni; + + if (!inode) + return ERR_PTR(-ENOMEM); + + ni = ntfs_i(inode); + + err = mi_format_new(&ni->mi, sbi, rno, flag, false); + if (err) + goto out; + + inode->i_ino = rno; + if (insert_inode_locked(inode) < 0) { + err = -EIO; + goto out; + } + + ni->base = ni; + +out: + if (err) { + make_bad_inode(inode); + iput(inode); + ni = ERR_PTR(err); + } + return ni; +} + +/* + * O:BAG:BAD:(A;OICI;FA;;;WD) + * Owner S-1-5-32-544 (Administrators) + * Group S-1-5-32-544 (Administrators) + * ACE: allow S-1-1-0 (Everyone) with FILE_ALL_ACCESS + */ +const u8 s_default_security[] __aligned(8) = { + 0x01, 0x00, 0x04, 0x80, 0x30, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x02, 0x00, 0x1C, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x03, 0x14, 0x00, 0xFF, 0x01, 0x1F, 0x00, + 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x20, 0x00, 0x00, 0x00, + 0x20, 0x02, 0x00, 0x00, 0x01, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, + 0x20, 0x00, 0x00, 0x00, 0x20, 0x02, 0x00, 0x00, +}; + +static_assert(sizeof(s_default_security) == 0x50); + +static inline u32 sid_length(const struct SID *sid) +{ + return struct_size(sid, SubAuthority, sid->SubAuthorityCount); +} + +/* + * is_acl_valid + * + * Thanks Mark Harmstone for idea. + */ +static bool is_acl_valid(const struct ACL *acl, u32 len) +{ + const struct ACE_HEADER *ace; + u32 i; + u16 ace_count, ace_size; + + if (acl->AclRevision != ACL_REVISION && + acl->AclRevision != ACL_REVISION_DS) { + /* + * This value should be ACL_REVISION, unless the ACL contains an + * object-specific ACE, in which case this value must be ACL_REVISION_DS. + * All ACEs in an ACL must be at the same revision level. + */ + return false; + } + + if (acl->Sbz1) + return false; + + if (le16_to_cpu(acl->AclSize) > len) + return false; + + if (acl->Sbz2) + return false; + + len -= sizeof(struct ACL); + ace = (struct ACE_HEADER *)&acl[1]; + ace_count = le16_to_cpu(acl->AceCount); + + for (i = 0; i < ace_count; i++) { + if (len < sizeof(struct ACE_HEADER)) + return false; + + ace_size = le16_to_cpu(ace->AceSize); + if (len < ace_size) + return false; + + len -= ace_size; + ace = Add2Ptr(ace, ace_size); + } + + return true; +} + +bool is_sd_valid(const struct SECURITY_DESCRIPTOR_RELATIVE *sd, u32 len) +{ + u32 sd_owner, sd_group, sd_sacl, sd_dacl; + + if (len < sizeof(struct SECURITY_DESCRIPTOR_RELATIVE)) + return false; + + if (sd->Revision != 1) + return false; + + if (sd->Sbz1) + return false; + + if (!(sd->Control & SE_SELF_RELATIVE)) + return false; + + sd_owner = le32_to_cpu(sd->Owner); + if (sd_owner) { + const struct SID *owner = Add2Ptr(sd, sd_owner); + + if (sd_owner + offsetof(struct SID, SubAuthority) > len) + return false; + + if (owner->Revision != 1) + return false; + + if (sd_owner + sid_length(owner) > len) + return false; + } + + sd_group = le32_to_cpu(sd->Group); + if (sd_group) { + const struct SID *group = Add2Ptr(sd, sd_group); + + if (sd_group + offsetof(struct SID, SubAuthority) > len) + return false; + + if (group->Revision != 1) + return false; + + if (sd_group + sid_length(group) > len) + return false; + } + + sd_sacl = le32_to_cpu(sd->Sacl); + if (sd_sacl) { + const struct ACL *sacl = Add2Ptr(sd, sd_sacl); + + if (sd_sacl + sizeof(struct ACL) > len) + return false; + + if (!is_acl_valid(sacl, len - sd_sacl)) + return false; + } + + sd_dacl = le32_to_cpu(sd->Dacl); + if (sd_dacl) { + const struct ACL *dacl = Add2Ptr(sd, sd_dacl); + + if (sd_dacl + sizeof(struct ACL) > len) + return false; + + if (!is_acl_valid(dacl, len - sd_dacl)) + return false; + } + + return true; +} + +/* + * ntfs_security_init - Load and parse $Secure. + */ +int ntfs_security_init(struct ntfs_sb_info *sbi) +{ + int err; + struct super_block *sb = sbi->sb; + struct inode *inode; + struct ntfs_inode *ni; + struct MFT_REF ref; + struct ATTRIB *attr; + struct ATTR_LIST_ENTRY *le; + u64 sds_size; + size_t off; + struct NTFS_DE *ne; + struct NTFS_DE_SII *sii_e; + struct ntfs_fnd *fnd_sii = NULL; + const struct INDEX_ROOT *root_sii; + const struct INDEX_ROOT *root_sdh; + struct ntfs_index *indx_sdh = &sbi->security.index_sdh; + struct ntfs_index *indx_sii = &sbi->security.index_sii; + + ref.low = cpu_to_le32(MFT_REC_SECURE); + ref.high = 0; + ref.seq = cpu_to_le16(MFT_REC_SECURE); + + inode = ntfs_iget5(sb, &ref, &NAME_SECURE); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + ntfs_err(sb, "Failed to load $Secure (%d).", err); + inode = NULL; + goto out; + } + + ni = ntfs_i(inode); + + le = NULL; + + attr = ni_find_attr(ni, NULL, &le, ATTR_ROOT, SDH_NAME, + ARRAY_SIZE(SDH_NAME), NULL, NULL); + if (!attr || + !(root_sdh = resident_data_ex(attr, sizeof(struct INDEX_ROOT))) || + root_sdh->type != ATTR_ZERO || + root_sdh->rule != NTFS_COLLATION_TYPE_SECURITY_HASH || + offsetof(struct INDEX_ROOT, ihdr) + + le32_to_cpu(root_sdh->ihdr.used) > + le32_to_cpu(attr->res.data_size)) { + ntfs_err(sb, "$Secure::$SDH is corrupted."); + err = -EINVAL; + goto out; + } + + err = indx_init(indx_sdh, sbi, attr, INDEX_MUTEX_SDH); + if (err) { + ntfs_err(sb, "Failed to initialize $Secure::$SDH (%d).", err); + goto out; + } + + attr = ni_find_attr(ni, attr, &le, ATTR_ROOT, SII_NAME, + ARRAY_SIZE(SII_NAME), NULL, NULL); + if (!attr || + !(root_sii = resident_data_ex(attr, sizeof(struct INDEX_ROOT))) || + root_sii->type != ATTR_ZERO || + root_sii->rule != NTFS_COLLATION_TYPE_UINT || + offsetof(struct INDEX_ROOT, ihdr) + + le32_to_cpu(root_sii->ihdr.used) > + le32_to_cpu(attr->res.data_size)) { + ntfs_err(sb, "$Secure::$SII is corrupted."); + err = -EINVAL; + goto out; + } + + err = indx_init(indx_sii, sbi, attr, INDEX_MUTEX_SII); + if (err) { + ntfs_err(sb, "Failed to initialize $Secure::$SII (%d).", err); + goto out; + } + + fnd_sii = fnd_get(); + if (!fnd_sii) { + err = -ENOMEM; + goto out; + } + + sds_size = inode->i_size; + + /* Find the last valid Id. */ + sbi->security.next_id = SECURITY_ID_FIRST; + /* Always write new security at the end of bucket. */ + sbi->security.next_off = + ALIGN(sds_size - SecurityDescriptorsBlockSize, 16); + + off = 0; + ne = NULL; + + for (;;) { + u32 next_id; + + err = indx_find_raw(indx_sii, ni, root_sii, &ne, &off, fnd_sii); + if (err || !ne) + break; + + sii_e = (struct NTFS_DE_SII *)ne; + if (le16_to_cpu(ne->view.data_size) < sizeof(sii_e->sec_hdr)) + continue; + + next_id = le32_to_cpu(sii_e->sec_id) + 1; + if (next_id >= sbi->security.next_id) + sbi->security.next_id = next_id; + } + + sbi->security.ni = ni; + inode = NULL; +out: + iput(inode); + fnd_put(fnd_sii); + + return err; +} + +/* + * ntfs_get_security_by_id - Read security descriptor by id. + */ +int ntfs_get_security_by_id(struct ntfs_sb_info *sbi, __le32 security_id, + struct SECURITY_DESCRIPTOR_RELATIVE **sd, + size_t *size) +{ + int err; + int diff; + struct ntfs_inode *ni = sbi->security.ni; + struct ntfs_index *indx = &sbi->security.index_sii; + void *p = NULL; + struct NTFS_DE_SII *sii_e; + struct ntfs_fnd *fnd_sii; + struct SECURITY_HDR d_security; + const struct INDEX_ROOT *root_sii; + u32 t32; + + *sd = NULL; + + mutex_lock_nested(&ni->ni_lock, NTFS_INODE_MUTEX_SECURITY); + + fnd_sii = fnd_get(); + if (!fnd_sii) { + err = -ENOMEM; + goto out; + } + + root_sii = indx_get_root(indx, ni, NULL, NULL); + if (!root_sii) { + err = -EINVAL; + goto out; + } + + /* Try to find this SECURITY descriptor in SII indexes. */ + err = indx_find(indx, ni, root_sii, &security_id, sizeof(security_id), + NULL, &diff, (struct NTFS_DE **)&sii_e, fnd_sii); + if (err) + goto out; + + if (diff) + goto out; + + t32 = le32_to_cpu(sii_e->sec_hdr.size); + if (t32 < sizeof(struct SECURITY_HDR)) { + err = -EINVAL; + goto out; + } + + if (t32 > sizeof(struct SECURITY_HDR) + 0x10000) { + /* Looks like too big security. 0x10000 - is arbitrary big number. */ + err = -EFBIG; + goto out; + } + + *size = t32 - sizeof(struct SECURITY_HDR); + + p = kmalloc(*size, GFP_NOFS); + if (!p) { + err = -ENOMEM; + goto out; + } + + err = ntfs_read_run_nb(sbi, &ni->file.run, + le64_to_cpu(sii_e->sec_hdr.off), &d_security, + sizeof(d_security), NULL); + if (err) + goto out; + + if (memcmp(&d_security, &sii_e->sec_hdr, sizeof(d_security))) { + err = -EINVAL; + goto out; + } + + err = ntfs_read_run_nb(sbi, &ni->file.run, + le64_to_cpu(sii_e->sec_hdr.off) + + sizeof(struct SECURITY_HDR), + p, *size, NULL); + if (err) + goto out; + + *sd = p; + p = NULL; + +out: + kfree(p); + fnd_put(fnd_sii); + ni_unlock(ni); + + return err; +} + +/* + * ntfs_insert_security - Insert security descriptor into $Secure::SDS. + * + * SECURITY Descriptor Stream data is organized into chunks of 256K bytes + * and it contains a mirror copy of each security descriptor. When writing + * to a security descriptor at location X, another copy will be written at + * location (X+256K). + * When writing a security descriptor that will cross the 256K boundary, + * the pointer will be advanced by 256K to skip + * over the mirror portion. + */ +int ntfs_insert_security(struct ntfs_sb_info *sbi, + const struct SECURITY_DESCRIPTOR_RELATIVE *sd, + u32 size_sd, __le32 *security_id, bool *inserted) +{ + int err, diff; + struct ntfs_inode *ni = sbi->security.ni; + struct ntfs_index *indx_sdh = &sbi->security.index_sdh; + struct ntfs_index *indx_sii = &sbi->security.index_sii; + struct NTFS_DE_SDH *e; + struct NTFS_DE_SDH sdh_e; + struct NTFS_DE_SII sii_e; + struct SECURITY_HDR *d_security; + u32 new_sec_size = size_sd + sizeof(struct SECURITY_HDR); + u32 aligned_sec_size = ALIGN(new_sec_size, 16); + struct SECURITY_KEY hash_key; + struct ntfs_fnd *fnd_sdh = NULL; + const struct INDEX_ROOT *root_sdh; + const struct INDEX_ROOT *root_sii; + u64 mirr_off, new_sds_size; + u32 next, left; + + static_assert((1 << Log2OfSecurityDescriptorsBlockSize) == + SecurityDescriptorsBlockSize); + + hash_key.hash = security_hash(sd, size_sd); + hash_key.sec_id = SECURITY_ID_INVALID; + + if (inserted) + *inserted = false; + *security_id = SECURITY_ID_INVALID; + + /* Allocate a temporal buffer. */ + d_security = kzalloc(aligned_sec_size, GFP_NOFS); + if (!d_security) + return -ENOMEM; + + mutex_lock_nested(&ni->ni_lock, NTFS_INODE_MUTEX_SECURITY); + + fnd_sdh = fnd_get(); + if (!fnd_sdh) { + err = -ENOMEM; + goto out; + } + + root_sdh = indx_get_root(indx_sdh, ni, NULL, NULL); + if (!root_sdh) { + err = -EINVAL; + goto out; + } + + root_sii = indx_get_root(indx_sii, ni, NULL, NULL); + if (!root_sii) { + err = -EINVAL; + goto out; + } + + /* + * Check if such security already exists. + * Use "SDH" and hash -> to get the offset in "SDS". + */ + err = indx_find(indx_sdh, ni, root_sdh, &hash_key, sizeof(hash_key), + &d_security->key.sec_id, &diff, (struct NTFS_DE **)&e, + fnd_sdh); + if (err) + goto out; + + while (e) { + if (le32_to_cpu(e->sec_hdr.size) == new_sec_size) { + err = ntfs_read_run_nb(sbi, &ni->file.run, + le64_to_cpu(e->sec_hdr.off), + d_security, new_sec_size, NULL); + if (err) + goto out; + + if (le32_to_cpu(d_security->size) == new_sec_size && + d_security->key.hash == hash_key.hash && + !memcmp(d_security + 1, sd, size_sd)) { + /* Such security already exists. */ + *security_id = d_security->key.sec_id; + err = 0; + goto out; + } + } + + err = indx_find_sort(indx_sdh, ni, root_sdh, + (struct NTFS_DE **)&e, fnd_sdh); + if (err) + goto out; + + if (!e || e->key.hash != hash_key.hash) + break; + } + + /* Zero unused space. */ + next = sbi->security.next_off & (SecurityDescriptorsBlockSize - 1); + left = SecurityDescriptorsBlockSize - next; + + /* Zero gap until SecurityDescriptorsBlockSize. */ + if (left < new_sec_size) { + /* Zero "left" bytes from sbi->security.next_off. */ + sbi->security.next_off += SecurityDescriptorsBlockSize + left; + } + + /* Zero tail of previous security. */ + //used = ni->vfs_inode.i_size & (SecurityDescriptorsBlockSize - 1); + + /* + * Example: + * 0x40438 == ni->vfs_inode.i_size + * 0x00440 == sbi->security.next_off + * need to zero [0x438-0x440) + * if (next > used) { + * u32 tozero = next - used; + * zero "tozero" bytes from sbi->security.next_off - tozero + */ + + /* Format new security descriptor. */ + d_security->key.hash = hash_key.hash; + d_security->key.sec_id = cpu_to_le32(sbi->security.next_id); + d_security->off = cpu_to_le64(sbi->security.next_off); + d_security->size = cpu_to_le32(new_sec_size); + memcpy(d_security + 1, sd, size_sd); + + /* Write main SDS bucket. */ + err = ntfs_sb_write_run(sbi, &ni->file.run, sbi->security.next_off, + d_security, aligned_sec_size, 0); + + if (err) + goto out; + + mirr_off = sbi->security.next_off + SecurityDescriptorsBlockSize; + new_sds_size = mirr_off + aligned_sec_size; + + if (new_sds_size > ni->vfs_inode.i_size) { + err = attr_set_size(ni, ATTR_DATA, SDS_NAME, + ARRAY_SIZE(SDS_NAME), &ni->file.run, + new_sds_size, &new_sds_size, false); + if (err) + goto out; + } + + /* Write copy SDS bucket. */ + err = ntfs_sb_write_run(sbi, &ni->file.run, mirr_off, d_security, + aligned_sec_size, 0); + if (err) + goto out; + + /* Fill SII entry. */ + sii_e.de.view.data_off = + cpu_to_le16(offsetof(struct NTFS_DE_SII, sec_hdr)); + sii_e.de.view.data_size = cpu_to_le16(sizeof(struct SECURITY_HDR)); + sii_e.de.view.res = 0; + sii_e.de.size = cpu_to_le16(sizeof(struct NTFS_DE_SII)); + sii_e.de.key_size = cpu_to_le16(sizeof(d_security->key.sec_id)); + sii_e.de.flags = 0; + sii_e.de.res = 0; + sii_e.sec_id = d_security->key.sec_id; + memcpy(&sii_e.sec_hdr, d_security, sizeof(struct SECURITY_HDR)); + + err = indx_insert_entry(indx_sii, ni, &sii_e.de, NULL, NULL, 0); + if (err) + goto out; + + /* Fill SDH entry. */ + sdh_e.de.view.data_off = + cpu_to_le16(offsetof(struct NTFS_DE_SDH, sec_hdr)); + sdh_e.de.view.data_size = cpu_to_le16(sizeof(struct SECURITY_HDR)); + sdh_e.de.view.res = 0; + sdh_e.de.size = cpu_to_le16(SIZEOF_SDH_DIRENTRY); + sdh_e.de.key_size = cpu_to_le16(sizeof(sdh_e.key)); + sdh_e.de.flags = 0; + sdh_e.de.res = 0; + sdh_e.key.hash = d_security->key.hash; + sdh_e.key.sec_id = d_security->key.sec_id; + memcpy(&sdh_e.sec_hdr, d_security, sizeof(struct SECURITY_HDR)); + sdh_e.magic[0] = cpu_to_le16('I'); + sdh_e.magic[1] = cpu_to_le16('I'); + + fnd_clear(fnd_sdh); + err = indx_insert_entry(indx_sdh, ni, &sdh_e.de, (void *)(size_t)1, + fnd_sdh, 0); + if (err) + goto out; + + *security_id = d_security->key.sec_id; + if (inserted) + *inserted = true; + + /* Update Id and offset for next descriptor. */ + sbi->security.next_id += 1; + sbi->security.next_off += aligned_sec_size; + +out: + fnd_put(fnd_sdh); + mark_inode_dirty(&ni->vfs_inode); + ni_unlock(ni); + kfree(d_security); + + return err; +} + +/* + * ntfs_reparse_init - Load and parse $Extend/$Reparse. + */ +int ntfs_reparse_init(struct ntfs_sb_info *sbi) +{ + int err; + struct ntfs_inode *ni = sbi->reparse.ni; + struct ntfs_index *indx = &sbi->reparse.index_r; + struct ATTRIB *attr; + struct ATTR_LIST_ENTRY *le; + const struct INDEX_ROOT *root_r; + + if (!ni) + return 0; + + le = NULL; + attr = ni_find_attr(ni, NULL, &le, ATTR_ROOT, SR_NAME, + ARRAY_SIZE(SR_NAME), NULL, NULL); + if (!attr) { + err = -EINVAL; + goto out; + } + + root_r = resident_data_ex(attr, sizeof(struct INDEX_ROOT)); + if (!root_r || root_r->type != ATTR_ZERO || + root_r->rule != NTFS_COLLATION_TYPE_UINTS) { + err = -EINVAL; + goto out; + } + + err = indx_init(indx, sbi, attr, INDEX_MUTEX_SR); + if (err) + goto out; + +out: + return err; +} + +/* + * ntfs_objid_init - Load and parse $Extend/$ObjId. + */ +int ntfs_objid_init(struct ntfs_sb_info *sbi) +{ + int err; + struct ntfs_inode *ni = sbi->objid.ni; + struct ntfs_index *indx = &sbi->objid.index_o; + struct ATTRIB *attr; + struct ATTR_LIST_ENTRY *le; + const struct INDEX_ROOT *root; + + if (!ni) + return 0; + + le = NULL; + attr = ni_find_attr(ni, NULL, &le, ATTR_ROOT, SO_NAME, + ARRAY_SIZE(SO_NAME), NULL, NULL); + if (!attr) { + err = -EINVAL; + goto out; + } + + root = resident_data_ex(attr, sizeof(struct INDEX_ROOT)); + if (!root || root->type != ATTR_ZERO || + root->rule != NTFS_COLLATION_TYPE_UINTS) { + err = -EINVAL; + goto out; + } + + err = indx_init(indx, sbi, attr, INDEX_MUTEX_SO); + if (err) + goto out; + +out: + return err; +} + +int ntfs_objid_remove(struct ntfs_sb_info *sbi, struct GUID *guid) +{ + int err; + struct ntfs_inode *ni = sbi->objid.ni; + struct ntfs_index *indx = &sbi->objid.index_o; + + if (!ni) + return -EINVAL; + + mutex_lock_nested(&ni->ni_lock, NTFS_INODE_MUTEX_OBJID); + + err = indx_delete_entry(indx, ni, guid, sizeof(*guid), NULL); + + mark_inode_dirty(&ni->vfs_inode); + ni_unlock(ni); + + return err; +} + +int ntfs_insert_reparse(struct ntfs_sb_info *sbi, __le32 rtag, + const struct MFT_REF *ref) +{ + int err; + struct ntfs_inode *ni = sbi->reparse.ni; + struct ntfs_index *indx = &sbi->reparse.index_r; + struct NTFS_DE_R re; + + if (!ni) + return -EINVAL; + + memset(&re, 0, sizeof(re)); + + re.de.view.data_off = cpu_to_le16(offsetof(struct NTFS_DE_R, zero)); + re.de.size = cpu_to_le16(sizeof(struct NTFS_DE_R)); + re.de.key_size = cpu_to_le16(sizeof(re.key)); + + re.key.ReparseTag = rtag; + memcpy(&re.key.ref, ref, sizeof(*ref)); + + mutex_lock_nested(&ni->ni_lock, NTFS_INODE_MUTEX_REPARSE); + + err = indx_insert_entry(indx, ni, &re.de, NULL, NULL, 0); + + mark_inode_dirty(&ni->vfs_inode); + ni_unlock(ni); + + return err; +} + +int ntfs_remove_reparse(struct ntfs_sb_info *sbi, __le32 rtag, + const struct MFT_REF *ref) +{ + int err, diff; + struct ntfs_inode *ni = sbi->reparse.ni; + struct ntfs_index *indx = &sbi->reparse.index_r; + struct ntfs_fnd *fnd = NULL; + struct REPARSE_KEY rkey; + struct NTFS_DE_R *re; + struct INDEX_ROOT *root_r; + + if (!ni) + return -EINVAL; + + rkey.ReparseTag = rtag; + rkey.ref = *ref; + + mutex_lock_nested(&ni->ni_lock, NTFS_INODE_MUTEX_REPARSE); + + if (rtag) { + err = indx_delete_entry(indx, ni, &rkey, sizeof(rkey), NULL); + goto out1; + } + + fnd = fnd_get(); + if (!fnd) { + err = -ENOMEM; + goto out1; + } + + root_r = indx_get_root(indx, ni, NULL, NULL); + if (!root_r) { + err = -EINVAL; + goto out; + } + + /* 1 - forces to ignore rkey.ReparseTag when comparing keys. */ + err = indx_find(indx, ni, root_r, &rkey, sizeof(rkey), (void *)1, &diff, + (struct NTFS_DE **)&re, fnd); + if (err) + goto out; + + if (memcmp(&re->key.ref, ref, sizeof(*ref))) { + /* Impossible. Looks like volume corrupt? */ + goto out; + } + + memcpy(&rkey, &re->key, sizeof(rkey)); + + fnd_put(fnd); + fnd = NULL; + + err = indx_delete_entry(indx, ni, &rkey, sizeof(rkey), NULL); + if (err) + goto out; + +out: + fnd_put(fnd); + +out1: + mark_inode_dirty(&ni->vfs_inode); + ni_unlock(ni); + + return err; +} + +static inline void ntfs_unmap_and_discard(struct ntfs_sb_info *sbi, CLST lcn, + CLST len) +{ + ntfs_unmap_meta(sbi->sb, lcn, len); + ntfs_discard(sbi, lcn, len); +} + +void mark_as_free_ex(struct ntfs_sb_info *sbi, CLST lcn, CLST len, bool trim) +{ + CLST end, i, zone_len, zlen; + struct wnd_bitmap *wnd = &sbi->used.bitmap; + bool dirty = false; + + down_write_nested(&wnd->rw_lock, BITMAP_MUTEX_CLUSTERS); + if (!wnd_is_used(wnd, lcn, len)) { + /* mark volume as dirty out of wnd->rw_lock */ + dirty = true; + + end = lcn + len; + len = 0; + for (i = lcn; i < end; i++) { + if (wnd_is_used(wnd, i, 1)) { + if (!len) + lcn = i; + len += 1; + continue; + } + + if (!len) + continue; + + if (trim) + ntfs_unmap_and_discard(sbi, lcn, len); + + wnd_set_free(wnd, lcn, len); + len = 0; + } + + if (!len) + goto out; + } + + if (trim) + ntfs_unmap_and_discard(sbi, lcn, len); + wnd_set_free(wnd, lcn, len); + + /* append to MFT zone, if possible. */ + zone_len = wnd_zone_len(wnd); + zlen = min(zone_len + len, sbi->zone_max); + + if (zlen == zone_len) { + /* MFT zone already has maximum size. */ + } else if (!zone_len) { + /* Create MFT zone only if 'zlen' is large enough. */ + if (zlen == sbi->zone_max) + wnd_zone_set(wnd, lcn, zlen); + } else { + CLST zone_lcn = wnd_zone_bit(wnd); + + if (lcn + len == zone_lcn) { + /* Append into head MFT zone. */ + wnd_zone_set(wnd, lcn, zlen); + } else if (zone_lcn + zone_len == lcn) { + /* Append into tail MFT zone. */ + wnd_zone_set(wnd, zone_lcn, zlen); + } + } + +out: + up_write(&wnd->rw_lock); + if (dirty) + ntfs_set_state(sbi, NTFS_DIRTY_ERROR); +} + +/* + * run_deallocate - Deallocate clusters. + */ +int run_deallocate(struct ntfs_sb_info *sbi, const struct runs_tree *run, + bool trim) +{ + CLST lcn, len; + size_t idx = 0; + + while (run_get_entry(run, idx++, NULL, &lcn, &len)) { + if (lcn == SPARSE_LCN) + continue; + + mark_as_free_ex(sbi, lcn, len, trim); + } + + return 0; +} + +static inline bool name_has_forbidden_chars(const struct le_str *fname) +{ + int i, ch; + + /* check for forbidden chars */ + for (i = 0; i < fname->len; ++i) { + ch = le16_to_cpu(fname->name[i]); + + /* control chars */ + if (ch < 0x20) + return true; + + switch (ch) { + /* disallowed by Windows */ + case '\\': + case '/': + case ':': + case '*': + case '?': + case '<': + case '>': + case '|': + case '\"': + return true; + + default: + /* allowed char */ + break; + } + } + + /* file names cannot end with space or . */ + if (fname->len > 0) { + ch = le16_to_cpu(fname->name[fname->len - 1]); + if (ch == ' ' || ch == '.') + return true; + } + + return false; +} + +static inline bool is_reserved_name(const struct ntfs_sb_info *sbi, + const struct le_str *fname) +{ + int port_digit; + const __le16 *name = fname->name; + int len = fname->len; + const u16 *upcase = sbi->upcase; + + /* check for 3 chars reserved names (device names) */ + /* name by itself or with any extension is forbidden */ + if (len == 3 || (len > 3 && le16_to_cpu(name[3]) == '.')) + if (!ntfs_cmp_names(name, 3, CON_NAME, 3, upcase, false) || + !ntfs_cmp_names(name, 3, NUL_NAME, 3, upcase, false) || + !ntfs_cmp_names(name, 3, AUX_NAME, 3, upcase, false) || + !ntfs_cmp_names(name, 3, PRN_NAME, 3, upcase, false)) + return true; + + /* check for 4 chars reserved names (port name followed by 1..9) */ + /* name by itself or with any extension is forbidden */ + if (len == 4 || (len > 4 && le16_to_cpu(name[4]) == '.')) { + port_digit = le16_to_cpu(name[3]); + if (port_digit >= '1' && port_digit <= '9') + if (!ntfs_cmp_names(name, 3, COM_NAME, 3, upcase, + false) || + !ntfs_cmp_names(name, 3, LPT_NAME, 3, upcase, + false)) + return true; + } + + return false; +} + +/* + * valid_windows_name - Check if a file name is valid in Windows. + */ +bool valid_windows_name(struct ntfs_sb_info *sbi, const struct le_str *fname) +{ + return !name_has_forbidden_chars(fname) && + !is_reserved_name(sbi, fname); +} + +/* + * ntfs_set_label - updates current ntfs label. + */ +int ntfs_set_label(struct ntfs_sb_info *sbi, u8 *label, int len) +{ + int err; + struct ATTRIB *attr; + u32 uni_bytes; + struct ntfs_inode *ni = sbi->volume.ni; + struct cpu_str *uni = kmalloc(PATH_MAX, GFP_KERNEL); + + if (!uni) + return -ENOMEM; + + err = ntfs_nls_to_utf16(sbi, label, len, uni, (PATH_MAX - 2) / 2, + UTF16_LITTLE_ENDIAN); + if (err < 0) + goto out; + + if (uni->ads_len) { + /* Undo delimiter parse */ + uni->len += uni->ads_len + 1; + uni->ads_len = 0; + } + + uni_bytes = uni->len * sizeof(u16); + if (uni_bytes > NTFS_LABEL_MAX_LENGTH * sizeof(u16)) { + ntfs_warn(sbi->sb, "new label is too long"); + err = -EFBIG; + goto out; + } + + ni_lock(ni); + + /* Ignore any errors. */ + ni_remove_attr(ni, ATTR_LABEL, NULL, 0, false, NULL); + + err = ni_insert_resident(ni, uni_bytes, ATTR_LABEL, NULL, 0, &attr, + NULL, NULL); + if (err < 0) + goto unlock_out; + + /* write new label in on-disk struct. */ + memcpy(resident_data(attr), uni->name, uni_bytes); + + /* update cached value of current label. */ + if (len >= ARRAY_SIZE(sbi->volume.label)) + len = ARRAY_SIZE(sbi->volume.label) - 1; + memcpy(sbi->volume.label, label, len); + sbi->volume.label[len] = 0; + mark_inode_dirty_sync(&ni->vfs_inode); + +unlock_out: + ni_unlock(ni); + + if (!err) + err = _ni_write_inode(&ni->vfs_inode, 0); + +out: + kfree(uni); + return err; +} diff --git a/fs/ntfs3/index.c b/fs/ntfs3/index.c new file mode 100644 index 000000000..689712d34 --- /dev/null +++ b/fs/ntfs3/index.c @@ -0,0 +1,2835 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + */ + +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +static const struct INDEX_NAMES { + const __le16 *name; + u8 name_len; +} s_index_names[INDEX_MUTEX_TOTAL] = { + { I30_NAME, ARRAY_SIZE(I30_NAME) }, { SII_NAME, ARRAY_SIZE(SII_NAME) }, + { SDH_NAME, ARRAY_SIZE(SDH_NAME) }, { SO_NAME, ARRAY_SIZE(SO_NAME) }, + { SQ_NAME, ARRAY_SIZE(SQ_NAME) }, { SR_NAME, ARRAY_SIZE(SR_NAME) }, +}; + +/* + * cmp_fnames - Compare two names in index. + * + * if l1 != 0 + * Both names are little endian on-disk ATTR_FILE_NAME structs. + * else + * key1 - cpu_str, key2 - ATTR_FILE_NAME + */ +static int cmp_fnames(const void *key1, size_t l1, const void *key2, size_t l2, + const void *data) +{ + const struct ATTR_FILE_NAME *f2 = key2; + const struct ntfs_sb_info *sbi = data; + const struct ATTR_FILE_NAME *f1; + u16 fsize2; + bool both_case; + + if (l2 <= offsetof(struct ATTR_FILE_NAME, name)) + return -1; + + fsize2 = fname_full_size(f2); + if (l2 < fsize2) + return -1; + + both_case = f2->type != FILE_NAME_DOS && !sbi->options->nocase; + if (!l1) { + const struct le_str *s2 = (struct le_str *)&f2->name_len; + + /* + * If names are equal (case insensitive) + * try to compare it case sensitive. + */ + return ntfs_cmp_names_cpu(key1, s2, sbi->upcase, both_case); + } + + f1 = key1; + return ntfs_cmp_names(f1->name, f1->name_len, f2->name, f2->name_len, + sbi->upcase, both_case); +} + +/* + * cmp_uint - $SII of $Secure and $Q of Quota + */ +static int cmp_uint(const void *key1, size_t l1, const void *key2, size_t l2, + const void *data) +{ + const u32 *k1 = key1; + const u32 *k2 = key2; + + if (l2 < sizeof(u32)) + return -1; + + if (*k1 < *k2) + return -1; + if (*k1 > *k2) + return 1; + return 0; +} + +/* + * cmp_sdh - $SDH of $Secure + */ +static int cmp_sdh(const void *key1, size_t l1, const void *key2, size_t l2, + const void *data) +{ + const struct SECURITY_KEY *k1 = key1; + const struct SECURITY_KEY *k2 = key2; + u32 t1, t2; + + if (l2 < sizeof(struct SECURITY_KEY)) + return -1; + + t1 = le32_to_cpu(k1->hash); + t2 = le32_to_cpu(k2->hash); + + /* First value is a hash value itself. */ + if (t1 < t2) + return -1; + if (t1 > t2) + return 1; + + /* Second value is security Id. */ + if (data) { + t1 = le32_to_cpu(k1->sec_id); + t2 = le32_to_cpu(k2->sec_id); + if (t1 < t2) + return -1; + if (t1 > t2) + return 1; + } + + return 0; +} + +/* + * cmp_uints - $O of ObjId and "$R" for Reparse. + */ +static int cmp_uints(const void *key1, size_t l1, const void *key2, size_t l2, + const void *data) +{ + const __le32 *k1 = key1; + const __le32 *k2 = key2; + size_t count; + + if ((size_t)data == 1) { + /* + * ni_delete_all -> ntfs_remove_reparse -> + * delete all with this reference. + * k1, k2 - pointers to REPARSE_KEY + */ + + k1 += 1; // Skip REPARSE_KEY.ReparseTag + k2 += 1; // Skip REPARSE_KEY.ReparseTag + if (l2 <= sizeof(int)) + return -1; + l2 -= sizeof(int); + if (l1 <= sizeof(int)) + return 1; + l1 -= sizeof(int); + } + + if (l2 < sizeof(int)) + return -1; + + for (count = min(l1, l2) >> 2; count > 0; --count, ++k1, ++k2) { + u32 t1 = le32_to_cpu(*k1); + u32 t2 = le32_to_cpu(*k2); + + if (t1 > t2) + return 1; + if (t1 < t2) + return -1; + } + + if (l1 > l2) + return 1; + if (l1 < l2) + return -1; + + return 0; +} + +static inline NTFS_CMP_FUNC get_cmp_func(const struct INDEX_ROOT *root) +{ + switch (root->type) { + case ATTR_NAME: + if (root->rule == NTFS_COLLATION_TYPE_FILENAME) + return &cmp_fnames; + break; + case ATTR_ZERO: + switch (root->rule) { + case NTFS_COLLATION_TYPE_UINT: + return &cmp_uint; + case NTFS_COLLATION_TYPE_SECURITY_HASH: + return &cmp_sdh; + case NTFS_COLLATION_TYPE_UINTS: + return &cmp_uints; + default: + break; + } + break; + default: + break; + } + + return NULL; +} + +struct bmp_buf { + struct ATTRIB *b; + struct mft_inode *mi; + struct buffer_head *bh; + ulong *buf; + size_t bit; + u32 nbits; + u64 new_valid; +}; + +static int bmp_buf_get(struct ntfs_index *indx, struct ntfs_inode *ni, + size_t bit, struct bmp_buf *bbuf) +{ + struct ATTRIB *b; + size_t data_size, valid_size, vbo, off = bit >> 3; + struct ntfs_sb_info *sbi = ni->mi.sbi; + CLST vcn = off >> sbi->cluster_bits; + struct ATTR_LIST_ENTRY *le = NULL; + struct buffer_head *bh; + struct super_block *sb; + u32 blocksize; + const struct INDEX_NAMES *in = &s_index_names[indx->type]; + + bbuf->bh = NULL; + + b = ni_find_attr(ni, NULL, &le, ATTR_BITMAP, in->name, in->name_len, + &vcn, &bbuf->mi); + bbuf->b = b; + if (!b) + return -EINVAL; + + if (!b->non_res) { + data_size = le32_to_cpu(b->res.data_size); + + if (off >= data_size) + return -EINVAL; + + bbuf->buf = (ulong *)resident_data(b); + bbuf->bit = 0; + bbuf->nbits = data_size * 8; + + return 0; + } + + data_size = le64_to_cpu(b->nres.data_size); + if (WARN_ON(off >= data_size)) { + /* Looks like filesystem error. */ + return -EINVAL; + } + + valid_size = le64_to_cpu(b->nres.valid_size); + + bh = ntfs_bread_run(sbi, &indx->bitmap_run, off); + if (!bh) + return -EIO; + + if (IS_ERR(bh)) + return PTR_ERR(bh); + + bbuf->bh = bh; + + wait_on_buffer(bh); + lock_buffer(bh); + + sb = sbi->sb; + blocksize = sb->s_blocksize; + + vbo = off & ~(size_t)sbi->block_mask; + + bbuf->new_valid = vbo + blocksize; + if (bbuf->new_valid <= valid_size) + bbuf->new_valid = 0; + else if (bbuf->new_valid > data_size) + bbuf->new_valid = data_size; + + if (vbo >= valid_size) { + memset(bh->b_data, 0, blocksize); + } else if (vbo + blocksize > valid_size) { + u32 voff = valid_size & sbi->block_mask; + + memset(bh->b_data + voff, 0, blocksize - voff); + } + + bbuf->buf = (ulong *)bh->b_data; + bbuf->bit = 8 * (off & ~(size_t)sbi->block_mask); + bbuf->nbits = 8 * blocksize; + + return 0; +} + +static void bmp_buf_put(struct bmp_buf *bbuf, bool dirty) +{ + struct buffer_head *bh = bbuf->bh; + struct ATTRIB *b = bbuf->b; + + if (!bh) { + if (b && !b->non_res && dirty) + bbuf->mi->dirty = true; + return; + } + + if (!dirty) + goto out; + + if (bbuf->new_valid) { + b->nres.valid_size = cpu_to_le64(bbuf->new_valid); + bbuf->mi->dirty = true; + } + + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + +out: + unlock_buffer(bh); + put_bh(bh); +} + +/* + * indx_mark_used - Mark the bit @bit as used. + */ +static int indx_mark_used(struct ntfs_index *indx, struct ntfs_inode *ni, + size_t bit) +{ + int err; + struct bmp_buf bbuf; + + err = bmp_buf_get(indx, ni, bit, &bbuf); + if (err) + return err; + + __set_bit_le(bit - bbuf.bit, bbuf.buf); + + bmp_buf_put(&bbuf, true); + + return 0; +} + +/* + * indx_mark_free - Mark the bit @bit as free. + */ +static int indx_mark_free(struct ntfs_index *indx, struct ntfs_inode *ni, + size_t bit) +{ + int err; + struct bmp_buf bbuf; + + err = bmp_buf_get(indx, ni, bit, &bbuf); + if (err) + return err; + + __clear_bit_le(bit - bbuf.bit, bbuf.buf); + + bmp_buf_put(&bbuf, true); + + return 0; +} + +/* + * scan_nres_bitmap + * + * If ntfs_readdir calls this function (indx_used_bit -> scan_nres_bitmap), + * inode is shared locked and no ni_lock. + * Use rw_semaphore for read/write access to bitmap_run. + */ +static int scan_nres_bitmap(struct ntfs_inode *ni, struct ATTRIB *bitmap, + struct ntfs_index *indx, size_t from, + bool (*fn)(const ulong *buf, u32 bit, u32 bits, + size_t *ret), + size_t *ret) +{ + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct super_block *sb = sbi->sb; + struct runs_tree *run = &indx->bitmap_run; + struct rw_semaphore *lock = &indx->run_lock; + u32 nbits = sb->s_blocksize * 8; + u32 blocksize = sb->s_blocksize; + u64 valid_size = le64_to_cpu(bitmap->nres.valid_size); + u64 data_size = le64_to_cpu(bitmap->nres.data_size); + sector_t eblock = bytes_to_block(sb, data_size); + size_t vbo = from >> 3; + sector_t blk = (vbo & sbi->cluster_mask) >> sb->s_blocksize_bits; + sector_t vblock = vbo >> sb->s_blocksize_bits; + sector_t blen, block; + CLST lcn, clen, vcn, vcn_next; + size_t idx; + struct buffer_head *bh; + bool ok; + + *ret = MINUS_ONE_T; + + if (vblock >= eblock) + return 0; + + from &= nbits - 1; + vcn = vbo >> sbi->cluster_bits; + + down_read(lock); + ok = run_lookup_entry(run, vcn, &lcn, &clen, &idx); + up_read(lock); + +next_run: + if (!ok) { + int err; + const struct INDEX_NAMES *name = &s_index_names[indx->type]; + + down_write(lock); + err = attr_load_runs_vcn(ni, ATTR_BITMAP, name->name, + name->name_len, run, vcn); + up_write(lock); + if (err) + return err; + down_read(lock); + ok = run_lookup_entry(run, vcn, &lcn, &clen, &idx); + up_read(lock); + if (!ok) + return -EINVAL; + } + + blen = (sector_t)clen * sbi->blocks_per_cluster; + block = (sector_t)lcn * sbi->blocks_per_cluster; + + for (; blk < blen; blk++, from = 0) { + bh = ntfs_bread(sb, block + blk); + if (!bh) + return -EIO; + + vbo = (u64)vblock << sb->s_blocksize_bits; + if (vbo >= valid_size) { + memset(bh->b_data, 0, blocksize); + } else if (vbo + blocksize > valid_size) { + u32 voff = valid_size & sbi->block_mask; + + memset(bh->b_data + voff, 0, blocksize - voff); + } + + if (vbo + blocksize > data_size) + nbits = 8 * (data_size - vbo); + + ok = nbits > from ? + (*fn)((ulong *)bh->b_data, from, nbits, ret) : + false; + put_bh(bh); + + if (ok) { + *ret += 8 * vbo; + return 0; + } + + if (++vblock >= eblock) { + *ret = MINUS_ONE_T; + return 0; + } + } + blk = 0; + vcn_next = vcn + clen; + down_read(lock); + ok = run_get_entry(run, ++idx, &vcn, &lcn, &clen) && vcn == vcn_next; + if (!ok) + vcn = vcn_next; + up_read(lock); + goto next_run; +} + +static bool scan_for_free(const ulong *buf, u32 bit, u32 bits, size_t *ret) +{ + size_t pos = find_next_zero_bit_le(buf, bits, bit); + + if (pos >= bits) + return false; + *ret = pos; + return true; +} + +/* + * indx_find_free - Look for free bit. + * + * Return: -1 if no free bits. + */ +static int indx_find_free(struct ntfs_index *indx, struct ntfs_inode *ni, + size_t *bit, struct ATTRIB **bitmap) +{ + struct ATTRIB *b; + struct ATTR_LIST_ENTRY *le = NULL; + const struct INDEX_NAMES *in = &s_index_names[indx->type]; + int err; + + b = ni_find_attr(ni, NULL, &le, ATTR_BITMAP, in->name, in->name_len, + NULL, NULL); + + if (!b) + return -ENOENT; + + *bitmap = b; + *bit = MINUS_ONE_T; + + if (!b->non_res) { + u32 nbits = 8 * le32_to_cpu(b->res.data_size); + size_t pos = find_next_zero_bit_le(resident_data(b), nbits, 0); + + if (pos < nbits) + *bit = pos; + } else { + err = scan_nres_bitmap(ni, b, indx, 0, &scan_for_free, bit); + + if (err) + return err; + } + + return 0; +} + +static bool scan_for_used(const ulong *buf, u32 bit, u32 bits, size_t *ret) +{ + size_t pos = find_next_bit_le(buf, bits, bit); + + if (pos >= bits) + return false; + *ret = pos; + return true; +} + +/* + * indx_used_bit - Look for used bit. + * + * Return: MINUS_ONE_T if no used bits. + */ +int indx_used_bit(struct ntfs_index *indx, struct ntfs_inode *ni, size_t *bit) +{ + struct ATTRIB *b; + struct ATTR_LIST_ENTRY *le = NULL; + size_t from = *bit; + const struct INDEX_NAMES *in = &s_index_names[indx->type]; + int err; + + b = ni_find_attr(ni, NULL, &le, ATTR_BITMAP, in->name, in->name_len, + NULL, NULL); + + if (!b) + return -ENOENT; + + *bit = MINUS_ONE_T; + + if (!b->non_res) { + u32 nbits = le32_to_cpu(b->res.data_size) * 8; + size_t pos = find_next_bit_le(resident_data(b), nbits, from); + + if (pos < nbits) + *bit = pos; + } else { + err = scan_nres_bitmap(ni, b, indx, from, &scan_for_used, bit); + if (err) + return err; + } + + return 0; +} + +/* + * hdr_find_split + * + * Find a point at which the index allocation buffer would like to be split. + * NOTE: This function should never return 'END' entry NULL returns on error. + */ +static const struct NTFS_DE *hdr_find_split(const struct INDEX_HDR *hdr) +{ + size_t o; + const struct NTFS_DE *e = hdr_first_de(hdr); + u32 used_2 = le32_to_cpu(hdr->used) >> 1; + u16 esize; + + if (!e || de_is_last(e)) + return NULL; + + esize = le16_to_cpu(e->size); + for (o = le32_to_cpu(hdr->de_off) + esize; o < used_2; o += esize) { + const struct NTFS_DE *p = e; + + e = Add2Ptr(hdr, o); + + /* We must not return END entry. */ + if (de_is_last(e)) + return p; + + esize = le16_to_cpu(e->size); + } + + return e; +} + +/* + * hdr_insert_head - Insert some entries at the beginning of the buffer. + * + * It is used to insert entries into a newly-created buffer. + */ +static const struct NTFS_DE *hdr_insert_head(struct INDEX_HDR *hdr, + const void *ins, u32 ins_bytes) +{ + u32 to_move; + struct NTFS_DE *e = hdr_first_de(hdr); + u32 used = le32_to_cpu(hdr->used); + + if (!e) + return NULL; + + /* Now we just make room for the inserted entries and jam it in. */ + to_move = used - le32_to_cpu(hdr->de_off); + memmove(Add2Ptr(e, ins_bytes), e, to_move); + memcpy(e, ins, ins_bytes); + hdr->used = cpu_to_le32(used + ins_bytes); + + return e; +} + +/* + * index_hdr_check + * + * return true if INDEX_HDR is valid + */ +static bool index_hdr_check(const struct INDEX_HDR *hdr, u32 bytes) +{ + const bool has_subnode = hdr_has_subnode(hdr); + const u16 min_size = + sizeof(struct NTFS_DE) + (has_subnode ? sizeof(u64) : 0); + u32 end = le32_to_cpu(hdr->used); + u32 tot = le32_to_cpu(hdr->total); + u32 off = le32_to_cpu(hdr->de_off); + const struct NTFS_DE *e; + + if (!IS_ALIGNED(off, 8) || tot > bytes || end > tot || + size_add(off, min_size) > end) { + /* incorrect index buffer. */ + return false; + } + + /* Ensure every key stays inside its entry before lookup walks it. */ + e = (const struct NTFS_DE *)((const u8 *)hdr + off); + for (;;) { + u16 e_size = le16_to_cpu(e->size); + u16 key_size = le16_to_cpu(e->key_size); + u16 data_size; + + if (!IS_ALIGNED(e_size, 8) || e_size < min_size || + de_has_vcn(e) != has_subnode) { + /* incorrect index entry. */ + return false; + } + + if (size_add(off, e_size) > end) + return false; + + if (de_is_last(e)) { + if (key_size) + return false; + + break; + } + + data_size = e_size - min_size; + if (key_size > data_size) + return false; + + off += e_size; + e = (const struct NTFS_DE *)((const u8 *)hdr + off); + } + + return true; +} + +/* + * index_buf_check + * + * return true if INDEX_BUFFER seems is valid + */ +static bool index_buf_check(const struct INDEX_BUFFER *ib, u32 bytes, + const CLST *vbn) +{ + const struct NTFS_RECORD_HEADER *rhdr = &ib->rhdr; + u16 fo = le16_to_cpu(rhdr->fix_off); + u16 fn = le16_to_cpu(rhdr->fix_num); + + if (bytes <= offsetof(struct INDEX_BUFFER, ihdr) || + rhdr->sign != NTFS_INDX_SIGNATURE || + fo < sizeof(struct INDEX_BUFFER) + /* Check index buffer vbn. */ + || (vbn && *vbn != le64_to_cpu(ib->vbn)) || (fo % sizeof(short)) || + fo + fn * sizeof(short) >= bytes || + fn != ((bytes >> SECTOR_SHIFT) + 1)) { + /* incorrect index buffer. */ + return false; + } + + return index_hdr_check(&ib->ihdr, + bytes - offsetof(struct INDEX_BUFFER, ihdr)); +} + +void fnd_clear(struct ntfs_fnd *fnd) +{ + int i; + + for (i = fnd->level - 1; i >= 0; i--) { + struct indx_node *n = fnd->nodes[i]; + + if (!n) + continue; + + put_indx_node(n); + fnd->nodes[i] = NULL; + } + fnd->level = 0; + fnd->root_de = NULL; +} + +static int fnd_push(struct ntfs_fnd *fnd, struct indx_node *n, + struct NTFS_DE *e) +{ + int i = fnd->level; + + if (i < 0 || i >= ARRAY_SIZE(fnd->nodes)) + return -EINVAL; + fnd->nodes[i] = n; + fnd->de[i] = e; + fnd->level += 1; + return 0; +} + +static struct indx_node *fnd_pop(struct ntfs_fnd *fnd) +{ + struct indx_node *n; + int i = fnd->level; + + i -= 1; + n = fnd->nodes[i]; + fnd->nodes[i] = NULL; + fnd->level = i; + + return n; +} + +static bool fnd_is_empty(struct ntfs_fnd *fnd) +{ + if (!fnd->level) + return !fnd->root_de; + + return !fnd->de[fnd->level - 1]; +} + +/* + * hdr_find_e - Locate an entry the index buffer. + * + * If no matching entry is found, it returns the first entry which is greater + * than the desired entry If the search key is greater than all the entries the + * buffer, it returns the 'end' entry. This function does a binary search of the + * current index buffer, for the first entry that is <= to the search value. + * + * Return: NULL if error. + */ +static struct NTFS_DE *hdr_find_e(const struct ntfs_index *indx, + const struct INDEX_HDR *hdr, const void *key, + size_t key_len, const void *ctx, int *diff, + NTFS_CMP_FUNC cmp) +{ + struct NTFS_DE *e, *found = NULL; + int min_idx = 0, mid_idx, max_idx = 0; + int diff2; + int table_size = 8; + u32 e_size, e_key_len; + u32 end = le32_to_cpu(hdr->used); + u32 off = le32_to_cpu(hdr->de_off); + u32 total = le32_to_cpu(hdr->total); + u16 offs[128]; + +fill_table: + if (end > total) + return NULL; + + if (size_add(off, sizeof(struct NTFS_DE)) > end) + return NULL; + + e = Add2Ptr(hdr, off); + e_size = le16_to_cpu(e->size); + + if (e_size < sizeof(struct NTFS_DE) || off + e_size > end) + return NULL; + + if (!de_is_last(e)) { + offs[max_idx] = off; + off += e_size; + + max_idx++; + if (max_idx < table_size) + goto fill_table; + + max_idx--; + } + +binary_search: + e_key_len = le16_to_cpu(e->key_size); + + /* Validate key_size fits within the entry data area. */ + if (e_key_len > le16_to_cpu(e->size) - sizeof(struct NTFS_DE)) + return NULL; + + diff2 = (*cmp)(key, key_len, e + 1, e_key_len, ctx); + if (diff2 > 0) { + if (found) { + min_idx = mid_idx + 1; + } else { + if (de_is_last(e)) + return NULL; + + max_idx = 0; + table_size = min(table_size * 2, (int)ARRAY_SIZE(offs)); + goto fill_table; + } + } else if (diff2 < 0) { + if (found) + max_idx = mid_idx - 1; + else + max_idx--; + + found = e; + } else { + *diff = 0; + return e; + } + + if (min_idx > max_idx) { + *diff = -1; + return found; + } + + mid_idx = (min_idx + max_idx) >> 1; + e = Add2Ptr(hdr, offs[mid_idx]); + + goto binary_search; +} + +/* + * hdr_insert_de - Insert an index entry into the buffer. + * + * 'before' should be a pointer previously returned from hdr_find_e. + */ +static struct NTFS_DE *hdr_insert_de(const struct ntfs_index *indx, + struct INDEX_HDR *hdr, + const struct NTFS_DE *de, + struct NTFS_DE *before, const void *ctx, + NTFS_CMP_FUNC cmp) +{ + int diff; + size_t off = PtrOffset(hdr, before); + u32 used = le32_to_cpu(hdr->used); + u32 total = le32_to_cpu(hdr->total); + u16 de_size = le16_to_cpu(de->size); + + /* First, check to see if there's enough room. */ + if (used + de_size > total) + return NULL; + + /* We know there's enough space, so we know we'll succeed. */ + if (before) { + /* Check that before is inside Index. */ + if (off >= used || off < le32_to_cpu(hdr->de_off) || + off + le16_to_cpu(before->size) > total) { + return NULL; + } + goto ok; + } + /* No insert point is applied. Get it manually. */ + before = hdr_find_e(indx, hdr, de + 1, le16_to_cpu(de->key_size), ctx, + &diff, cmp); + if (!before) + return NULL; + off = PtrOffset(hdr, before); + +ok: + /* Now we just make room for the entry and jam it in. */ + memmove(Add2Ptr(before, de_size), before, used - off); + + hdr->used = cpu_to_le32(used + de_size); + memcpy(before, de, de_size); + + return before; +} + +/* + * hdr_delete_de - Remove an entry from the index buffer. + */ +static inline struct NTFS_DE *hdr_delete_de(struct INDEX_HDR *hdr, + struct NTFS_DE *re) +{ + u32 used = le32_to_cpu(hdr->used); + u16 esize = le16_to_cpu(re->size); + u32 off = PtrOffset(hdr, re); + int bytes = used - (off + esize); + + /* check INDEX_HDR valid before using INDEX_HDR */ + if (!check_index_header(hdr, le32_to_cpu(hdr->total))) + return NULL; + + if (off >= used || esize < sizeof(struct NTFS_DE) || + bytes < sizeof(struct NTFS_DE)) + return NULL; + + hdr->used = cpu_to_le32(used - esize); + memmove(re, Add2Ptr(re, esize), bytes); + + return re; +} + +void indx_clear(struct ntfs_index *indx) +{ + run_close(&indx->alloc_run); + run_close(&indx->bitmap_run); +} + +int indx_init(struct ntfs_index *indx, struct ntfs_sb_info *sbi, + const struct ATTRIB *attr, enum index_mutex_classed type) +{ + u32 t32; + const struct INDEX_ROOT *root = resident_data(attr); + + t32 = le32_to_cpu(attr->res.data_size); + if (t32 <= offsetof(struct INDEX_ROOT, ihdr) || + !index_hdr_check(&root->ihdr, + t32 - offsetof(struct INDEX_ROOT, ihdr))) { + goto out; + } + + /* Check root fields. */ + if (!root->index_block_clst) + goto out; + + indx->type = type; + indx->idx2vbn_bits = __ffs(root->index_block_clst); + + t32 = le32_to_cpu(root->index_block_size); + indx->index_bits = blksize_bits(t32); + + /* Check index record size. */ + if (t32 < sbi->cluster_size) { + /* Index record is smaller than a cluster, use 512 blocks. */ + if (t32 != root->index_block_clst * SECTOR_SIZE) + goto out; + + /* Check alignment to a cluster. */ + if ((sbi->cluster_size >> SECTOR_SHIFT) & + (root->index_block_clst - 1)) { + goto out; + } + + indx->vbn2vbo_bits = SECTOR_SHIFT; + } else { + /* Index record must be a multiple of cluster size. */ + if (t32 != root->index_block_clst << sbi->cluster_bits) + goto out; + + indx->vbn2vbo_bits = sbi->cluster_bits; + } + + init_rwsem(&indx->run_lock); + + return 0; + +out: + ntfs_set_state(sbi, NTFS_DIRTY_DIRTY); + return -EINVAL; +} + +static struct indx_node *indx_new(struct ntfs_index *indx, + struct ntfs_inode *ni, CLST vbn, + const __le64 *sub_vbn) +{ + int err; + struct NTFS_DE *e; + struct indx_node *r; + struct INDEX_HDR *hdr; + struct INDEX_BUFFER *index; + u64 vbo = (u64)vbn << indx->vbn2vbo_bits; + u32 bytes = 1u << indx->index_bits; + u16 fn; + u32 eo; + + r = kzalloc_obj(struct indx_node, GFP_NOFS); + if (!r) + return ERR_PTR(-ENOMEM); + + index = kzalloc(bytes, GFP_NOFS); + if (!index) { + kfree(r); + return ERR_PTR(-ENOMEM); + } + + err = ntfs_get_bh(ni->mi.sbi, &indx->alloc_run, vbo, bytes, &r->nb); + + if (err) { + kfree(index); + kfree(r); + return ERR_PTR(err); + } + + /* Create header. */ + index->rhdr.sign = NTFS_INDX_SIGNATURE; + index->rhdr.fix_off = cpu_to_le16(sizeof(struct INDEX_BUFFER)); // 0x28 + fn = (bytes >> SECTOR_SHIFT) + 1; // 9 + index->rhdr.fix_num = cpu_to_le16(fn); + index->vbn = cpu_to_le64(vbn); + hdr = &index->ihdr; + eo = ALIGN(sizeof(struct INDEX_BUFFER) + fn * sizeof(short), 8); + hdr->de_off = cpu_to_le32(eo); + + e = Add2Ptr(hdr, eo); + + if (sub_vbn) { + e->flags = NTFS_IE_LAST | NTFS_IE_HAS_SUBNODES; + e->size = cpu_to_le16(sizeof(struct NTFS_DE) + sizeof(u64)); + hdr->used = + cpu_to_le32(eo + sizeof(struct NTFS_DE) + sizeof(u64)); + de_set_vbn_le(e, *sub_vbn); + hdr->flags = NTFS_INDEX_HDR_HAS_SUBNODES; + } else { + e->size = cpu_to_le16(sizeof(struct NTFS_DE)); + hdr->used = cpu_to_le32(eo + sizeof(struct NTFS_DE)); + e->flags = NTFS_IE_LAST; + } + + hdr->total = cpu_to_le32(bytes - offsetof(struct INDEX_BUFFER, ihdr)); + + r->index = index; + return r; +} + +struct INDEX_ROOT *indx_get_root(struct ntfs_index *indx, struct ntfs_inode *ni, + struct ATTRIB **attr, struct mft_inode **mi) +{ + struct ATTR_LIST_ENTRY *le = NULL; + struct ATTRIB *a; + const struct INDEX_NAMES *in = &s_index_names[indx->type]; + struct INDEX_ROOT *root; + + a = ni_find_attr(ni, NULL, &le, ATTR_ROOT, in->name, in->name_len, NULL, + mi); + if (!a) + return NULL; + + if (attr) + *attr = a; + + root = resident_data_ex(a, sizeof(struct INDEX_ROOT)); + + /* length check */ + if (root && + offsetof(struct INDEX_ROOT, ihdr) + le32_to_cpu(root->ihdr.used) > + le32_to_cpu(a->res.data_size)) { + return NULL; + } + + return root; +} + +static int indx_write(struct ntfs_index *indx, struct ntfs_inode *ni, + struct indx_node *node, int sync) +{ + struct INDEX_BUFFER *ib = node->index; + + return ntfs_write_bh(ni->mi.sbi, &ib->rhdr, &node->nb, sync); +} + +/* + * indx_read_ra + * + * If ntfs_readdir calls this function + * inode is shared locked and no ni_lock. + * Use rw_semaphore for read/write access to alloc_run. + */ +int indx_read_ra(struct ntfs_index *indx, struct ntfs_inode *ni, CLST vbn, + struct indx_node **node, struct file_ra_state *ra) +{ + int err; + struct INDEX_BUFFER *ib; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct runs_tree *run = &indx->alloc_run; + struct rw_semaphore *lock = &indx->run_lock; + u64 vbo = (u64)vbn << indx->vbn2vbo_bits; + u32 bytes = 1u << indx->index_bits; + struct indx_node *in = *node; + const struct INDEX_NAMES *name; + + if (!in) { + in = kzalloc_obj(struct indx_node, GFP_NOFS); + if (!in) + return -ENOMEM; + } else { + nb_put(&in->nb); + } + + ib = in->index; + if (!ib) { + ib = kmalloc(bytes, GFP_NOFS); + if (!ib) { + err = -ENOMEM; + goto out; + } + } + + down_read(lock); + err = ntfs_read_bh_ra(sbi, run, vbo, &ib->rhdr, bytes, &in->nb, ra); + up_read(lock); + if (!err) + goto ok; + + if (err == -E_NTFS_FIXUP) + goto ok; + + if (err != -ENOENT) + goto out; + + name = &s_index_names[indx->type]; + down_write(lock); + err = attr_load_runs_range(ni, ATTR_ALLOC, name->name, name->name_len, + run, vbo, vbo + bytes); + up_write(lock); + if (err) + goto out; + + down_read(lock); + err = ntfs_read_bh_ra(sbi, run, vbo, &ib->rhdr, bytes, &in->nb, ra); + up_read(lock); + if (err == -E_NTFS_FIXUP) + goto ok; + + if (err) + goto out; + +ok: + if (!index_buf_check(ib, bytes, &vbn)) { + _ntfs_bad_inode(&ni->vfs_inode); + err = -EINVAL; + goto out; + } + + if (err == -E_NTFS_FIXUP) { + ntfs_write_bh(sbi, &ib->rhdr, &in->nb, 0); + err = 0; + } + + /* check for index header length */ + if (offsetof(struct INDEX_BUFFER, ihdr) + le32_to_cpu(ib->ihdr.used) > + bytes) { + err = -EINVAL; + goto out; + } + + in->index = ib; + *node = in; + +out: + if (err == -E_NTFS_CORRUPT) { + _ntfs_bad_inode(&ni->vfs_inode); + err = -EINVAL; + } + + if (ib != in->index) + kfree(ib); + + if (*node != in) { + nb_put(&in->nb); + kfree(in); + } + + return err; +} + +/* + * indx_find - Scan NTFS directory for given entry. + */ +int indx_find(struct ntfs_index *indx, struct ntfs_inode *ni, + const struct INDEX_ROOT *root, const void *key, size_t key_len, + const void *ctx, int *diff, struct NTFS_DE **entry, + struct ntfs_fnd *fnd) +{ + int err; + struct NTFS_DE *e; + struct indx_node *node; + NTFS_CMP_FUNC cmp; + + if (!root) + root = indx_get_root(&ni->dir, ni, NULL, NULL); + + if (!root) { + /* Should not happen. */ + return -EINVAL; + } + + cmp = get_cmp_func(root); + if (unlikely(!cmp)) { + WARN_ON_ONCE(1); + return -EINVAL; + } + + /* Check cache. */ + e = fnd->level ? fnd->de[fnd->level - 1] : fnd->root_de; + if (e && !de_is_last(e) && + !(*cmp)(key, key_len, e + 1, le16_to_cpu(e->key_size), ctx)) { + *entry = e; + *diff = 0; + return 0; + } + + /* Soft finder reset. */ + fnd_clear(fnd); + + /* Lookup entry that is <= to the search value. */ + e = hdr_find_e(indx, &root->ihdr, key, key_len, ctx, diff, cmp); + if (!e) + return -EINVAL; + + fnd->root_de = e; + + for (;;) { + node = NULL; + if (*diff >= 0 || !de_has_vcn_ex(e)) + break; + + /* Read next level. */ + err = indx_read(indx, ni, de_get_vbn(e), &node); + if (err) { + /* io error? */ + return err; + } + + /* Lookup entry that is <= to the search value. */ + e = hdr_find_e(indx, &node->index->ihdr, key, key_len, ctx, + diff, cmp); + if (!e) { + put_indx_node(node); + return -EINVAL; + } + + err = fnd_push(fnd, node, e); + + if (err) { + put_indx_node(node); + return err; + } + } + + *entry = e; + return 0; +} + +int indx_find_sort(struct ntfs_index *indx, struct ntfs_inode *ni, + const struct INDEX_ROOT *root, struct NTFS_DE **entry, + struct ntfs_fnd *fnd) +{ + int err; + struct indx_node *n = NULL; + struct NTFS_DE *e; + size_t iter = 0; + int level = fnd->level; + + if (!*entry) { + /* Start find. */ + e = hdr_first_de(&root->ihdr); + if (!e) + return 0; + fnd_clear(fnd); + fnd->root_de = e; + } else if (!level) { + if (de_is_last(fnd->root_de)) { + *entry = NULL; + return 0; + } + + e = hdr_next_de(&root->ihdr, fnd->root_de); + if (!e) + return -EINVAL; + fnd->root_de = e; + } else { + n = fnd->nodes[level - 1]; + e = fnd->de[level - 1]; + + if (de_is_last(e)) + goto pop_level; + + e = hdr_next_de(&n->index->ihdr, e); + if (!e) + return -EINVAL; + + fnd->de[level - 1] = e; + } + + /* Just to avoid tree cycle. */ +next_iter: + if (iter++ >= 1000) + return -EINVAL; + + while (de_has_vcn_ex(e)) { + if (le16_to_cpu(e->size) < + sizeof(struct NTFS_DE) + sizeof(u64)) { + if (n) { + fnd_pop(fnd); + kfree(n); + } + return -EINVAL; + } + + /* Read next level. */ + err = indx_read(indx, ni, de_get_vbn(e), &n); + if (err) + return err; + + /* Try next level. */ + e = hdr_first_de(&n->index->ihdr); + if (!e) { + kfree(n); + return -EINVAL; + } + + fnd_push(fnd, n, e); + } + + if (le16_to_cpu(e->size) > sizeof(struct NTFS_DE)) { + *entry = e; + return 0; + } + +pop_level: + for (;;) { + if (!de_is_last(e)) + goto next_iter; + + /* Pop one level. */ + if (n) { + fnd_pop(fnd); + kfree(n->index); + kfree(n); + } + + level = fnd->level; + + if (level) { + n = fnd->nodes[level - 1]; + e = fnd->de[level - 1]; + } else if (fnd->root_de) { + n = NULL; + e = fnd->root_de; + fnd->root_de = NULL; + } else { + *entry = NULL; + return 0; + } + + if (le16_to_cpu(e->size) > sizeof(struct NTFS_DE)) { + *entry = e; + if (!fnd->root_de) + fnd->root_de = e; + return 0; + } + } +} + +int indx_find_raw(struct ntfs_index *indx, struct ntfs_inode *ni, + const struct INDEX_ROOT *root, struct NTFS_DE **entry, + size_t *off, struct ntfs_fnd *fnd) +{ + int err; + struct indx_node *n = NULL; + struct NTFS_DE *e = NULL; + struct NTFS_DE *e2; + size_t bit; + CLST next_used_vbn; + CLST next_vbn; + u32 record_size = ni->mi.sbi->record_size; + + /* Use non sorted algorithm. */ + if (!*entry) { + /* This is the first call. */ + e = hdr_first_de(&root->ihdr); + if (!e) + return 0; + fnd_clear(fnd); + fnd->root_de = e; + + /* The first call with setup of initial element. */ + if (*off >= record_size) { + next_vbn = (((*off - record_size) >> indx->index_bits)) + << indx->idx2vbn_bits; + /* Jump inside cycle 'for'. */ + goto next; + } + + /* Start enumeration from root. */ + *off = 0; + } else if (!fnd->root_de) + return -EINVAL; + + for (;;) { + /* Check if current entry can be used. */ + if (e && le16_to_cpu(e->size) > sizeof(struct NTFS_DE)) + goto ok; + + if (!fnd->level) { + /* Continue to enumerate root. */ + if (!de_is_last(fnd->root_de)) { + e = hdr_next_de(&root->ihdr, fnd->root_de); + if (!e) + return -EINVAL; + fnd->root_de = e; + continue; + } + + /* Start to enumerate indexes from 0. */ + next_vbn = 0; + } else { + /* Continue to enumerate indexes. */ + e2 = fnd->de[fnd->level - 1]; + + n = fnd->nodes[fnd->level - 1]; + + if (!de_is_last(e2)) { + e = hdr_next_de(&n->index->ihdr, e2); + if (!e) + return -EINVAL; + fnd->de[fnd->level - 1] = e; + continue; + } + + /* Continue with next index. */ + next_vbn = le64_to_cpu(n->index->vbn) + + root->index_block_clst; + } + +next: + /* Release current index. */ + if (n) { + fnd_pop(fnd); + put_indx_node(n); + n = NULL; + } + + /* Skip all free indexes. */ + bit = next_vbn >> indx->idx2vbn_bits; + err = indx_used_bit(indx, ni, &bit); + if (err == -ENOENT || bit == MINUS_ONE_T) { + /* No used indexes. */ + *entry = NULL; + return 0; + } + + next_used_vbn = bit << indx->idx2vbn_bits; + + /* Read buffer into memory. */ + err = indx_read(indx, ni, next_used_vbn, &n); + if (err) + return err; + + e = hdr_first_de(&n->index->ihdr); + fnd_push(fnd, n, e); + if (!e) + return -EINVAL; + } + +ok: + /* Return offset to restore enumerator if necessary. */ + if (!n) { + /* 'e' points in root, */ + *off = PtrOffset(&root->ihdr, e); + } else { + /* 'e' points in index, */ + *off = (le64_to_cpu(n->index->vbn) << indx->vbn2vbo_bits) + + record_size + PtrOffset(&n->index->ihdr, e); + } + + *entry = e; + return 0; +} + +/* + * indx_create_allocate - Create "Allocation + Bitmap" attributes. + */ +static int indx_create_allocate(struct ntfs_index *indx, struct ntfs_inode *ni, + CLST *vbn) +{ + int err; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct ATTRIB *bitmap; + struct ATTRIB *alloc; + u32 data_size = 1u << indx->index_bits; + u32 alloc_size = ntfs_up_cluster(sbi, data_size); + CLST len = alloc_size >> sbi->cluster_bits; + const struct INDEX_NAMES *in = &s_index_names[indx->type]; + CLST alen; + struct runs_tree run; + + run_init(&run); + + err = attr_allocate_clusters(sbi, &run, NULL, 0, 0, len, NULL, + ALLOCATE_DEF, &alen, 0, NULL, NULL); + if (err) + goto out; + + err = ni_insert_nonresident(ni, ATTR_ALLOC, in->name, in->name_len, + &run, 0, len, 0, &alloc, NULL, NULL); + if (err) + goto out1; + + alloc->nres.valid_size = alloc->nres.data_size = cpu_to_le64(data_size); + + err = ni_insert_resident(ni, ntfs3_bitmap_size(1), ATTR_BITMAP, + in->name, in->name_len, &bitmap, NULL, NULL); + if (err) + goto out2; + + if (in->name == I30_NAME) { + i_size_write(&ni->vfs_inode, data_size); + inode_set_bytes(&ni->vfs_inode, alloc_size); + } + + memcpy(&indx->alloc_run, &run, sizeof(run)); + + *vbn = 0; + + return 0; + +out2: + mi_remove_attr(NULL, &ni->mi, alloc); + +out1: + run_deallocate(sbi, &run, false); + +out: + run_close(&run); + return err; +} + +/* + * indx_add_allocate - Add clusters to index. + */ +static int indx_add_allocate(struct ntfs_index *indx, struct ntfs_inode *ni, + CLST *vbn) +{ + int err; + size_t bit; + u64 data_size; + u64 bmp_size, bmp_size_v; + struct ATTRIB *bmp, *alloc; + struct mft_inode *mi; + const struct INDEX_NAMES *in = &s_index_names[indx->type]; + + err = indx_find_free(indx, ni, &bit, &bmp); + if (err) + goto out1; + + if (bit != MINUS_ONE_T) { + bmp = NULL; + bmp_size = bmp_size_v = 0; + } else { + if (bmp->non_res) { + bmp_size = le64_to_cpu(bmp->nres.data_size); + bmp_size_v = le64_to_cpu(bmp->nres.valid_size); + } else { + bmp_size = bmp_size_v = le32_to_cpu(bmp->res.data_size); + } + + /* + * Index blocks exist, but $BITMAP has zero valid bits. + * This implies an on-disk corruption and must be rejected. + */ + if (in->name == I30_NAME && + unlikely(bmp_size_v == 0 && indx->alloc_run.count)) { + err = -EINVAL; + goto out1; + } + + bit = bmp_size << 3; + } + + data_size = (u64)(bit + 1) << indx->index_bits; + + if (bmp) { + /* Increase bitmap. */ + err = attr_set_size(ni, ATTR_BITMAP, in->name, in->name_len, + &indx->bitmap_run, + ntfs3_bitmap_size(bit + 1), NULL, true); + if (err) + goto out1; + } + + alloc = ni_find_attr(ni, NULL, NULL, ATTR_ALLOC, in->name, in->name_len, + NULL, &mi); + if (!alloc) { + err = -EINVAL; + if (bmp) + goto out2; + goto out1; + } + + if (data_size <= le64_to_cpu(alloc->nres.data_size)) { + /* Reuse index. */ + goto out; + } + + /* Increase allocation. */ + err = attr_set_size(ni, ATTR_ALLOC, in->name, in->name_len, + &indx->alloc_run, data_size, &data_size, true); + if (err) { + if (bmp) + goto out2; + goto out1; + } + + if (in->name == I30_NAME) + i_size_write(&ni->vfs_inode, data_size); + +out: + *vbn = bit << indx->idx2vbn_bits; + + return 0; + +out2: + /* Ops. No space? */ + attr_set_size(ni, ATTR_BITMAP, in->name, in->name_len, + &indx->bitmap_run, bmp_size, &bmp_size_v, false); + +out1: + return err; +} + +/* + * indx_insert_into_root - Attempt to insert an entry into the index root. + * + * @undo - True if we undoing previous remove. + * If necessary, it will twiddle the index b-tree. + */ +static int indx_insert_into_root(struct ntfs_index *indx, struct ntfs_inode *ni, + const struct NTFS_DE *new_de, + struct NTFS_DE *root_de, const void *ctx, + struct ntfs_fnd *fnd, bool undo, + NTFS_CMP_FUNC cmp) +{ + int err = 0; + struct NTFS_DE *e, *e0, *re; + struct mft_inode *mi; + struct ATTRIB *attr; + struct INDEX_HDR *hdr; + struct indx_node *n; + CLST new_vbn; + __le64 *sub_vbn, t_vbn; + u16 new_de_size; + u32 hdr_used, hdr_total, asize, to_move; + u32 root_size, new_root_size; + struct ntfs_sb_info *sbi; + int ds_root; + struct INDEX_ROOT *root, *a_root; + + /* Get the record this root placed in. */ + root = indx_get_root(indx, ni, &attr, &mi); + if (!root) + return -EINVAL; + + /* + * Try easy case: + * hdr_insert_de will succeed if there's + * room the root for the new entry. + */ + hdr = &root->ihdr; + sbi = ni->mi.sbi; + new_de_size = le16_to_cpu(new_de->size); + hdr_used = le32_to_cpu(hdr->used); + hdr_total = le32_to_cpu(hdr->total); + asize = le32_to_cpu(attr->size); + root_size = le32_to_cpu(attr->res.data_size); + + ds_root = new_de_size + hdr_used - hdr_total; + + /* If 'undo' is set then reduce requirements. */ + if ((undo || asize + ds_root < sbi->max_bytes_per_attr) && + mi_resize_attr(mi, attr, ds_root)) { + hdr->total = cpu_to_le32(hdr_total + ds_root); + e = hdr_insert_de(indx, hdr, new_de, root_de, ctx, cmp); + WARN_ON(!e); + fnd_clear(fnd); + fnd->root_de = e; + + return 0; + } + + /* Make a copy of root attribute to restore if error. */ + a_root = kmemdup(attr, asize, GFP_NOFS); + if (!a_root) + return -ENOMEM; + + /* + * Copy all the non-end entries from + * the index root to the new buffer. + */ + to_move = 0; + e0 = hdr_first_de(hdr); + + /* Calculate the size to copy. */ + for (e = e0;; e = hdr_next_de(hdr, e)) { + if (!e) { + err = -EINVAL; + goto out_free_root; + } + + if (de_is_last(e)) + break; + to_move += le16_to_cpu(e->size); + } + + if (!to_move) { + re = NULL; + } else { + re = kmemdup(e0, to_move, GFP_NOFS); + if (!re) { + err = -ENOMEM; + goto out_free_root; + } + } + + sub_vbn = NULL; + if (de_has_vcn(e)) { + t_vbn = de_get_vbn_le(e); + sub_vbn = &t_vbn; + } + + new_root_size = sizeof(struct INDEX_ROOT) + sizeof(struct NTFS_DE) + + sizeof(u64); + ds_root = new_root_size - root_size; + + if (ds_root > 0 && asize + ds_root > sbi->max_bytes_per_attr) { + /* Make root external. */ + err = -EOPNOTSUPP; + goto out_free_re; + } + + if (ds_root) + mi_resize_attr(mi, attr, ds_root); + + /* Fill first entry (vcn will be set later). */ + e = (struct NTFS_DE *)(root + 1); + memset(e, 0, sizeof(struct NTFS_DE)); + e->size = cpu_to_le16(sizeof(struct NTFS_DE) + sizeof(u64)); + e->flags = NTFS_IE_HAS_SUBNODES | NTFS_IE_LAST; + + hdr->flags = NTFS_INDEX_HDR_HAS_SUBNODES; + hdr->used = hdr->total = + cpu_to_le32(new_root_size - offsetof(struct INDEX_ROOT, ihdr)); + + fnd->root_de = hdr_first_de(hdr); + mi->dirty = true; + + /* Create alloc and bitmap attributes (if not). */ + err = run_is_empty(&indx->alloc_run) ? + indx_create_allocate(indx, ni, &new_vbn) : + indx_add_allocate(indx, ni, &new_vbn); + + /* Layout of record may be changed, so rescan root. */ + root = indx_get_root(indx, ni, &attr, &mi); + if (!root) { + /* Bug? */ + ntfs_set_state(sbi, NTFS_DIRTY_ERROR); + err = -EINVAL; + goto out_free_re; + } + + if (err) { + /* Restore root. */ + if (mi_resize_attr(mi, attr, -ds_root)) { + memcpy(attr, a_root, asize); + } else { + /* Bug? */ + ntfs_set_state(sbi, NTFS_DIRTY_ERROR); + } + goto out_free_re; + } + + e = (struct NTFS_DE *)(root + 1); + *(__le64 *)(e + 1) = cpu_to_le64(new_vbn); + mi->dirty = true; + + /* Now we can create/format the new buffer and copy the entries into. */ + n = indx_new(indx, ni, new_vbn, sub_vbn); + if (IS_ERR(n)) { + err = PTR_ERR(n); + goto out_free_re; + } + + hdr = &n->index->ihdr; + hdr_used = le32_to_cpu(hdr->used); + hdr_total = le32_to_cpu(hdr->total); + + /* + * The destination INDEX_BUFFER has 'hdr_total' bytes of payload + * available after the header, of which 'hdr_used' are already + * consumed by the single terminal END entry installed by + * indx_new(). A crafted image can present a resident root whose + * non-last entries (summing to 'to_move') exceed what fits in + * this buffer; copying them unchecked would overrun the + * kmalloc(1u << indx->index_bits) allocation backing the new + * buffer. Reject the copy in that case. + */ + if (to_move > hdr_total - hdr_used) { + err = -EINVAL; + ntfs_set_state(sbi, NTFS_DIRTY_ERROR); + goto out_put_n; + } + + /* Copy root entries into new buffer. */ + hdr_insert_head(hdr, re, to_move); + + /* Update bitmap attribute. */ + indx_mark_used(indx, ni, new_vbn >> indx->idx2vbn_bits); + + /* Check if we can insert new entry new index buffer. */ + if (hdr_used + new_de_size > hdr_total) { + /* + * This occurs if MFT record is the same or bigger than index + * buffer. Move all root new index and have no space to add + * new entry classic case when MFT record is 1K and index + * buffer 4K the problem should not occurs. + */ + kfree(re); + indx_write(indx, ni, n, 0); + + put_indx_node(n); + fnd_clear(fnd); + err = indx_insert_entry(indx, ni, new_de, ctx, fnd, undo); + goto out_free_root; + } + + /* + * Now root is a parent for new index buffer. + * Insert NewEntry a new buffer. + */ + e = hdr_insert_de(indx, hdr, new_de, NULL, ctx, cmp); + if (!e) { + err = -EINVAL; + goto out_put_n; + } + fnd_push(fnd, n, e); + + /* Just write updates index into disk. */ + indx_write(indx, ni, n, 0); + + n = NULL; + +out_put_n: + put_indx_node(n); +out_free_re: + kfree(re); +out_free_root: + kfree(a_root); + return err; +} + +/* + * indx_insert_into_buffer + * + * Attempt to insert an entry into an Index Allocation Buffer. + * If necessary, it will split the buffer. + */ +static int indx_insert_into_buffer(struct ntfs_index *indx, + struct ntfs_inode *ni, + struct INDEX_ROOT *root, + const struct NTFS_DE *new_de, + const void *ctx, int level, + struct ntfs_fnd *fnd, NTFS_CMP_FUNC cmp) +{ + int err; + const struct NTFS_DE *sp; /* split_point */ + struct NTFS_DE *e, *de_t, *up_e; + struct indx_node *n2; + struct indx_node *n1 = fnd->nodes[level]; + struct INDEX_HDR *hdr1 = &n1->index->ihdr; + struct INDEX_HDR *hdr2; + u32 to_copy, used, used1; + CLST new_vbn; + __le64 t_vbn, *sub_vbn; + u16 sp_size; + void *hdr1_saved = NULL; + + /* Try the most easy case. */ + e = fnd->level - 1 == level ? fnd->de[level] : NULL; + e = hdr_insert_de(indx, hdr1, new_de, e, ctx, cmp); + fnd->de[level] = e; + if (e) { + /* Just write updated index into disk. */ + indx_write(indx, ni, n1, 0); + return 0; + } + + /* + * No space to insert into buffer. Split it. + * To split we: + * - Save split point ('cause index buffers will be changed) + * - Allocate new buffer (up_e) and copy all entries <= sp into new buffer + * - Remove all entries (sp including) from hdr1 + * - Insert new_de into left or right buffer (depending on sp <=> new_de) + * - Insert sp into parent buffer (or root) + * - Make sp a parent for new buffer + */ + sp = hdr_find_split(hdr1); + if (!sp) + return -EINVAL; + + sp_size = le16_to_cpu(sp->size); + up_e = kmalloc(sp_size + sizeof(u64), GFP_NOFS); + if (!up_e) + return -ENOMEM; + memcpy(up_e, sp, sp_size); + + /* Make a copy for undo. */ + used1 = le32_to_cpu(hdr1->used); + + /* + * hdr_find_split does not validate per-entry sizes, so a crafted + * NTFS_DE whose le16 size field is out of range can place sp such + * that (PtrOffset(hdr1, sp) + sp_size) exceeds used1. Without this + * guard the u32 'used = used1 - to_copy - sp_size' underflows and + * the subsequent memmove count becomes a near-4-GiB value, + * triggering an out-of-bounds kernel write. + */ + if (PtrOffset(hdr1, sp) + sp_size > used1) { + err = -EINVAL; + goto out; + } + + hdr1_saved = kmemdup(hdr1, used1, GFP_NOFS); + if (!hdr1_saved) { + err = -ENOMEM; + goto out; + } + + if (!hdr1->flags) { + up_e->flags |= NTFS_IE_HAS_SUBNODES; + up_e->size = cpu_to_le16(sp_size + sizeof(u64)); + sub_vbn = NULL; + } else { + t_vbn = de_get_vbn_le(up_e); + sub_vbn = &t_vbn; + } + + /* Allocate on disk a new index allocation buffer. */ + err = indx_add_allocate(indx, ni, &new_vbn); + if (err) + goto out; + + /* Allocate and format memory a new index buffer. */ + n2 = indx_new(indx, ni, new_vbn, sub_vbn); + if (IS_ERR(n2)) { + err = PTR_ERR(n2); + goto out; + } + + hdr2 = &n2->index->ihdr; + + /* Make sp a parent for new buffer. */ + de_set_vbn(up_e, new_vbn); + + /* Copy all the entries <= sp into the new buffer. */ + de_t = hdr_first_de(hdr1); + to_copy = PtrOffset(de_t, sp); + hdr_insert_head(hdr2, de_t, to_copy); + + /* Remove all entries (sp including) from hdr1. */ + used = used1 - to_copy - sp_size; + memmove(de_t, Add2Ptr(sp, sp_size), used - le32_to_cpu(hdr1->de_off)); + hdr1->used = cpu_to_le32(used); + + /* + * Insert new entry into left or right buffer + * (depending on sp <=> new_de). + */ + hdr_insert_de(indx, + (*cmp)(new_de + 1, le16_to_cpu(new_de->key_size), + up_e + 1, le16_to_cpu(up_e->key_size), ctx) < 0 ? + hdr2 : + hdr1, + new_de, NULL, ctx, cmp); + + indx_mark_used(indx, ni, new_vbn >> indx->idx2vbn_bits); + + indx_write(indx, ni, n1, 0); + indx_write(indx, ni, n2, 0); + + put_indx_node(n2); + + /* + * We've finished splitting everybody, so we are ready to + * insert the promoted entry into the parent. + */ + if (!level) { + /* Insert split_point in root. */ + err = indx_insert_into_root(indx, ni, up_e, NULL, ctx, fnd, 0, + cmp); + } else { + /* + * The target buffer's parent is another index buffer. + * Insert split_point in parent index ( call itself recursively ) + * TODO: Remove recursion. + */ + err = indx_insert_into_buffer(indx, ni, root, up_e, ctx, + level - 1, fnd, cmp); + } + + if (err) { + /* + * Undo critical operations. + */ + indx_mark_free(indx, ni, new_vbn >> indx->idx2vbn_bits); + unsafe_memcpy(hdr1, hdr1_saved, used1, + "There are entries after the structure"); + indx_write(indx, ni, n1, 0); + } + +out: + kfree(up_e); + kfree(hdr1_saved); + + return err; +} + +/* + * indx_insert_entry - Insert new entry into index. + * + * @undo - True if we undoing previous remove. + */ +int indx_insert_entry(struct ntfs_index *indx, struct ntfs_inode *ni, + const struct NTFS_DE *new_de, const void *ctx, + struct ntfs_fnd *fnd, bool undo) +{ + int err; + int diff; + struct NTFS_DE *e; + struct ntfs_fnd *fnd_a = NULL; + struct INDEX_ROOT *root; + NTFS_CMP_FUNC cmp; + + if (!fnd) { + fnd_a = fnd_get(); + if (!fnd_a) { + err = -ENOMEM; + goto out1; + } + fnd = fnd_a; + } + + root = indx_get_root(indx, ni, NULL, NULL); + if (!root) { + err = -EINVAL; + goto out; + } + + cmp = get_cmp_func(root); + if (unlikely(!cmp)) { + WARN_ON_ONCE(1); + return -EINVAL; + } + + if (fnd_is_empty(fnd)) { + /* + * Find the spot the tree where we want to + * insert the new entry. + */ + err = indx_find(indx, ni, root, new_de + 1, + le16_to_cpu(new_de->key_size), ctx, &diff, &e, + fnd); + if (err) + goto out; + + if (!diff) { + err = -EEXIST; + goto out; + } + } + + if (!fnd->level) { + /* + * The root is also a leaf, so we'll insert the + * new entry into it. + */ + err = indx_insert_into_root(indx, ni, new_de, fnd->root_de, ctx, + fnd, undo, cmp); + } else { + /* + * Found a leaf buffer, so we'll insert the new entry into it. + */ + err = indx_insert_into_buffer(indx, ni, root, new_de, ctx, + fnd->level - 1, fnd, cmp); + } + + indx->version += 1; +out: + fnd_put(fnd_a); +out1: + return err; +} + +/* + * indx_find_buffer - Locate a buffer from the tree. + */ +static struct indx_node *indx_find_buffer(struct ntfs_index *indx, + struct ntfs_inode *ni, + const struct INDEX_ROOT *root, + __le64 vbn, struct indx_node *n, + int depth) +{ + int err; + const struct NTFS_DE *e; + struct indx_node *r; + const struct INDEX_HDR *hdr = n ? &n->index->ihdr : &root->ihdr; + + /* + * Limit recursion depth to prevent stack overflow from crafted + * images. Use the same bound as the fnd->nodes array (20). + */ + if (depth > ARRAY_SIZE(((struct ntfs_fnd *)NULL)->nodes)) + return ERR_PTR(-EINVAL); + + /* Step 1: Scan one level. */ + for (e = hdr_first_de(hdr);; e = hdr_next_de(hdr, e)) { + if (!e) + return ERR_PTR(-EINVAL); + + if (de_has_vcn(e) && vbn == de_get_vbn_le(e)) + return n; + + if (de_is_last(e)) + break; + } + + /* Step2: Do recursion. */ + e = Add2Ptr(hdr, le32_to_cpu(hdr->de_off)); + for (;;) { + if (de_has_vcn_ex(e)) { + err = indx_read(indx, ni, de_get_vbn(e), &n); + if (err) + return ERR_PTR(err); + + r = indx_find_buffer(indx, ni, root, vbn, n, depth + 1); + if (r) + return r; + } + + if (de_is_last(e)) + break; + + e = Add2Ptr(e, le16_to_cpu(e->size)); + } + + return NULL; +} + +/* + * indx_shrink - Deallocate unused tail indexes. + */ +static int indx_shrink(struct ntfs_index *indx, struct ntfs_inode *ni, + size_t bit) +{ + int err = 0; + u64 bpb, new_data; + size_t nbits; + struct ATTRIB *b; + struct ATTR_LIST_ENTRY *le = NULL; + const struct INDEX_NAMES *in = &s_index_names[indx->type]; + + b = ni_find_attr(ni, NULL, &le, ATTR_BITMAP, in->name, in->name_len, + NULL, NULL); + + if (!b) + return -ENOENT; + + if (!b->non_res) { + unsigned long pos; + const unsigned long *bm = resident_data(b); + + nbits = (size_t)le32_to_cpu(b->res.data_size) * 8; + + if (bit >= nbits) + return 0; + + pos = find_next_bit_le(bm, nbits, bit); + if (pos < nbits) + return 0; + } else { + size_t used = MINUS_ONE_T; + + nbits = le64_to_cpu(b->nres.data_size) * 8; + + if (bit >= nbits) + return 0; + + err = scan_nres_bitmap(ni, b, indx, bit, &scan_for_used, &used); + if (err) + return err; + + if (used != MINUS_ONE_T) + return 0; + } + + new_data = (u64)bit << indx->index_bits; + + err = attr_set_size(ni, ATTR_ALLOC, in->name, in->name_len, + &indx->alloc_run, new_data, &new_data, false); + if (err) + return err; + + if (in->name == I30_NAME) + i_size_write(&ni->vfs_inode, new_data); + + bpb = ntfs3_bitmap_size(bit); + if (bpb * 8 == nbits) + return 0; + + err = attr_set_size(ni, ATTR_BITMAP, in->name, in->name_len, + &indx->bitmap_run, bpb, &bpb, false); + + return err; +} + +static int indx_free_children(struct ntfs_index *indx, struct ntfs_inode *ni, + const struct NTFS_DE *e, bool trim) +{ + int err; + struct indx_node *n = NULL; + struct INDEX_HDR *hdr; + CLST vbn = de_get_vbn(e); + size_t i; + + err = indx_read(indx, ni, vbn, &n); + if (err) + return err; + + hdr = &n->index->ihdr; + /* First, recurse into the children, if any. */ + if (hdr_has_subnode(hdr)) { + for (e = hdr_first_de(hdr); e; e = hdr_next_de(hdr, e)) { + indx_free_children(indx, ni, e, false); + if (de_is_last(e)) + break; + } + } + + put_indx_node(n); + + i = vbn >> indx->idx2vbn_bits; + /* + * We've gotten rid of the children; add this buffer to the free list. + */ + indx_mark_free(indx, ni, i); + + if (!trim) + return 0; + + /* + * If there are no used indexes after current free index + * then we can truncate allocation and bitmap. + * Use bitmap to estimate the case. + */ + indx_shrink(indx, ni, i + 1); + return 0; +} + +/* + * indx_get_entry_to_replace + * + * Find a replacement entry for a deleted entry. + * Always returns a node entry: + * NTFS_IE_HAS_SUBNODES is set the flags and the size includes the sub_vcn. + */ +static int indx_get_entry_to_replace(struct ntfs_index *indx, + struct ntfs_inode *ni, + const struct NTFS_DE *de_next, + struct NTFS_DE **de_to_replace, + struct ntfs_fnd *fnd) +{ + int err; + int level = -1; + CLST vbn; + struct NTFS_DE *e, *te, *re; + struct indx_node *n; + struct INDEX_BUFFER *ib; + + *de_to_replace = NULL; + + /* Find first leaf entry down from de_next. */ + vbn = de_get_vbn(de_next); + for (;;) { + n = NULL; + err = indx_read(indx, ni, vbn, &n); + if (err) + goto out; + + e = hdr_first_de(&n->index->ihdr); + fnd_push(fnd, n, e); + if (!e) { + err = -EINVAL; + goto out; + } + + if (!de_is_last(e)) { + /* + * This buffer is non-empty, so its first entry + * could be used as the replacement entry. + */ + level = fnd->level - 1; + } + + if (!de_has_vcn(e)) + break; + + /* This buffer is a node. Continue to go down. */ + vbn = de_get_vbn(e); + } + + if (level == -1) + goto out; + + n = fnd->nodes[level]; + te = hdr_first_de(&n->index->ihdr); + if (!te) { + err = -EINVAL; + goto out; + } + /* Copy the candidate entry into the replacement entry buffer. */ + re = kmalloc(le16_to_cpu(te->size) + sizeof(u64), GFP_NOFS); + if (!re) { + err = -ENOMEM; + goto out; + } + + *de_to_replace = re; + memcpy(re, te, le16_to_cpu(te->size)); + + if (!de_has_vcn(re)) { + /* + * The replacement entry we found doesn't have a sub_vcn. + * increase its size to hold one. + */ + le16_add_cpu(&re->size, sizeof(u64)); + re->flags |= NTFS_IE_HAS_SUBNODES; + } else { + /* + * The replacement entry we found was a node entry, which + * means that all its child buffers are empty. Return them + * to the free pool. + */ + indx_free_children(indx, ni, te, true); + } + + /* + * Expunge the replacement entry from its former location, + * and then write that buffer. + */ + ib = n->index; + e = hdr_delete_de(&ib->ihdr, te); + + fnd->de[level] = e; + indx_write(indx, ni, n, 0); + + if (ib_is_leaf(ib) && ib_is_empty(ib)) { + /* An empty leaf. */ + return 0; + } + +out: + fnd_clear(fnd); + return err; +} + +/* + * indx_delete_entry - Delete an entry from the index. + */ +int indx_delete_entry(struct ntfs_index *indx, struct ntfs_inode *ni, + const void *key, u32 key_len, const void *ctx) +{ + int err, diff; + struct INDEX_ROOT *root; + struct INDEX_HDR *hdr; + struct ntfs_fnd *fnd, *fnd2; + struct INDEX_BUFFER *ib; + struct NTFS_DE *e, *re, *next, *prev, *me; + struct indx_node *n, *n2d = NULL; + __le64 sub_vbn; + int level, level2; + struct ATTRIB *attr; + struct mft_inode *mi; + u32 e_size, root_size, new_root_size; + size_t trim_bit; + const struct INDEX_NAMES *in; + NTFS_CMP_FUNC cmp; + + fnd = fnd_get(); + if (!fnd) { + err = -ENOMEM; + goto out2; + } + + fnd2 = fnd_get(); + if (!fnd2) { + err = -ENOMEM; + goto out1; + } + + root = indx_get_root(indx, ni, &attr, &mi); + if (!root) { + err = -EINVAL; + goto out; + } + + cmp = get_cmp_func(root); + if (unlikely(!cmp)) { + WARN_ON_ONCE(1); + return -EINVAL; + } + + /* Locate the entry to remove. */ + err = indx_find(indx, ni, root, key, key_len, ctx, &diff, &e, fnd); + if (err) + goto out; + + if (!e || diff) { + err = -ENOENT; + goto out; + } + + level = fnd->level; + + if (level) { + n = fnd->nodes[level - 1]; + e = fnd->de[level - 1]; + ib = n->index; + hdr = &ib->ihdr; + } else { + hdr = &root->ihdr; + e = fnd->root_de; + n = NULL; + ib = NULL; + } + + e_size = le16_to_cpu(e->size); + + if (!de_has_vcn_ex(e)) { + /* The entry to delete is a leaf, so we can just rip it out. */ + hdr_delete_de(hdr, e); + + if (!level) { + hdr->total = hdr->used; + + /* Shrink resident root attribute. */ + mi_resize_attr(mi, attr, 0 - e_size); + goto out; + } + + indx_write(indx, ni, n, 0); + + /* + * Check to see if removing that entry made + * the leaf empty. + */ + if (ib && ib_is_leaf(ib) && ib_is_empty(ib)) { + fnd_pop(fnd); + fnd_push(fnd2, n, e); + } + } else { + /* + * The entry we wish to delete is a node buffer, so we + * have to find a replacement for it. + */ + next = de_get_next(e); + + err = indx_get_entry_to_replace(indx, ni, next, &re, fnd2); + if (err) + goto out; + + if (re) { + de_set_vbn_le(re, de_get_vbn_le(e)); + hdr_delete_de(hdr, e); + + err = level ? indx_insert_into_buffer(indx, ni, root, + re, ctx, + fnd->level - 1, + fnd, cmp) : + indx_insert_into_root(indx, ni, re, e, + ctx, fnd, 0, cmp); + kfree(re); + + if (err) + goto out; + } else { + /* + * There is no replacement for the current entry. + * This means that the subtree rooted at its node + * is empty, and can be deleted, which turn means + * that the node can just inherit the deleted + * entry sub_vcn. + */ + indx_free_children(indx, ni, next, true); + + de_set_vbn_le(next, de_get_vbn_le(e)); + hdr_delete_de(hdr, e); + if (level) { + indx_write(indx, ni, n, 0); + } else { + hdr->total = hdr->used; + + /* Shrink resident root attribute. */ + mi_resize_attr(mi, attr, 0 - e_size); + } + } + } + + /* Delete a branch of tree. */ + if (!fnd2 || !fnd2->level) + goto out; + + /* Reinit root 'cause it can be changed. */ + root = indx_get_root(indx, ni, &attr, &mi); + if (!root) { + err = -EINVAL; + goto out; + } + + n2d = NULL; + sub_vbn = fnd2->nodes[0]->index->vbn; + level2 = 0; + level = fnd->level; + + hdr = level ? &fnd->nodes[level - 1]->index->ihdr : &root->ihdr; + + /* Scan current level. */ + for (e = hdr_first_de(hdr);; e = hdr_next_de(hdr, e)) { + if (!e) { + err = -EINVAL; + goto out; + } + + if (de_has_vcn(e) && sub_vbn == de_get_vbn_le(e)) + break; + + if (de_is_last(e)) { + e = NULL; + break; + } + } + + if (!e) { + /* Do slow search from root. */ + struct indx_node *in; + + fnd_clear(fnd); + + in = indx_find_buffer(indx, ni, root, sub_vbn, NULL, 0); + if (IS_ERR(in)) { + err = PTR_ERR(in); + goto out; + } + + if (in) + fnd_push(fnd, in, NULL); + } + + /* Merge fnd2 -> fnd. */ + for (level = 0; level < fnd2->level; level++) { + fnd_push(fnd, fnd2->nodes[level], fnd2->de[level]); + fnd2->nodes[level] = NULL; + } + fnd2->level = 0; + + hdr = NULL; + for (level = fnd->level; level; level--) { + struct indx_node *in = fnd->nodes[level - 1]; + + ib = in->index; + if (ib_is_empty(ib)) { + sub_vbn = ib->vbn; + } else { + hdr = &ib->ihdr; + n2d = in; + level2 = level; + break; + } + } + + if (!hdr) + hdr = &root->ihdr; + + e = hdr_first_de(hdr); + if (!e) { + err = -EINVAL; + goto out; + } + + if (hdr != &root->ihdr || !de_is_last(e)) { + prev = NULL; + while (!de_is_last(e)) { + if (de_has_vcn(e) && sub_vbn == de_get_vbn_le(e)) + break; + prev = e; + e = hdr_next_de(hdr, e); + if (!e) { + err = -EINVAL; + goto out; + } + } + + if (sub_vbn != de_get_vbn_le(e)) { + /* + * Didn't find the parent entry, although this buffer + * is the parent trail. Something is corrupt. + */ + err = -EINVAL; + goto out; + } + + if (de_is_last(e)) { + /* + * Since we can't remove the end entry, we'll remove + * its predecessor instead. This means we have to + * transfer the predecessor's sub_vcn to the end entry. + * Note: This index block is not empty, so the + * predecessor must exist. + */ + if (!prev) { + err = -EINVAL; + goto out; + } + + if (de_has_vcn(prev)) { + de_set_vbn_le(e, de_get_vbn_le(prev)); + } else if (de_has_vcn(e)) { + le16_sub_cpu(&e->size, sizeof(u64)); + e->flags &= ~NTFS_IE_HAS_SUBNODES; + le32_sub_cpu(&hdr->used, sizeof(u64)); + } + e = prev; + } + + /* + * Copy the current entry into a temporary buffer (stripping + * off its down-pointer, if any) and delete it from the current + * buffer or root, as appropriate. + */ + e_size = le16_to_cpu(e->size); + me = kmemdup(e, e_size, GFP_NOFS); + if (!me) { + err = -ENOMEM; + goto out; + } + + if (de_has_vcn(me)) { + me->flags &= ~NTFS_IE_HAS_SUBNODES; + le16_sub_cpu(&me->size, sizeof(u64)); + } + + hdr_delete_de(hdr, e); + + if (hdr == &root->ihdr) { + level = 0; + hdr->total = hdr->used; + + /* Shrink resident root attribute. */ + mi_resize_attr(mi, attr, 0 - e_size); + } else { + indx_write(indx, ni, n2d, 0); + level = level2; + } + + /* Mark unused buffers as free. */ + trim_bit = -1; + for (; level < fnd->level; level++) { + ib = fnd->nodes[level]->index; + if (ib_is_empty(ib)) { + size_t k = le64_to_cpu(ib->vbn) >> + indx->idx2vbn_bits; + + indx_mark_free(indx, ni, k); + if (k < trim_bit) + trim_bit = k; + } + } + + fnd_clear(fnd); + /*fnd->root_de = NULL;*/ + + /* + * Re-insert the entry into the tree. + * Find the spot the tree where we want to insert the new entry. + */ + err = indx_insert_entry(indx, ni, me, ctx, fnd, 0); + kfree(me); + if (err) + goto out; + + if (trim_bit != -1) + indx_shrink(indx, ni, trim_bit); + } else { + /* + * This tree needs to be collapsed down to an empty root. + * Recreate the index root as an empty leaf and free all + * the bits the index allocation bitmap. + */ + fnd_clear(fnd); + fnd_clear(fnd2); + + in = &s_index_names[indx->type]; + + err = attr_set_size(ni, ATTR_ALLOC, in->name, in->name_len, + &indx->alloc_run, 0, NULL, false); + if (in->name == I30_NAME) + i_size_write(&ni->vfs_inode, 0); + + err = ni_remove_attr(ni, ATTR_ALLOC, in->name, in->name_len, + false, NULL); + run_close(&indx->alloc_run); + + err = attr_set_size(ni, ATTR_BITMAP, in->name, in->name_len, + &indx->bitmap_run, 0, NULL, false); + err = ni_remove_attr(ni, ATTR_BITMAP, in->name, in->name_len, + false, NULL); + run_close(&indx->bitmap_run); + + root = indx_get_root(indx, ni, &attr, &mi); + if (!root) { + err = -EINVAL; + goto out; + } + + root_size = le32_to_cpu(attr->res.data_size); + new_root_size = + sizeof(struct INDEX_ROOT) + sizeof(struct NTFS_DE); + + if (new_root_size != root_size && + !mi_resize_attr(mi, attr, new_root_size - root_size)) { + err = -EINVAL; + goto out; + } + + /* Fill first entry. */ + e = (struct NTFS_DE *)(root + 1); + e->ref.low = 0; + e->ref.high = 0; + e->ref.seq = 0; + e->size = cpu_to_le16(sizeof(struct NTFS_DE)); + e->flags = NTFS_IE_LAST; // 0x02 + e->key_size = 0; + e->res = 0; + + hdr = &root->ihdr; + hdr->flags = 0; + hdr->used = hdr->total = cpu_to_le32( + new_root_size - offsetof(struct INDEX_ROOT, ihdr)); + mi->dirty = true; + } + + indx->version += 1; +out: + fnd_put(fnd2); +out1: + fnd_put(fnd); +out2: + return err; +} + +/* + * Update duplicated information in directory entry + * 'dup' - info from MFT record + */ +int indx_update_dup(struct ntfs_inode *ni, struct ntfs_sb_info *sbi, + const struct ATTR_FILE_NAME *fname, + const struct NTFS_DUP_INFO *dup, int sync) +{ + int err, diff; + struct NTFS_DE *e = NULL; + struct ATTR_FILE_NAME *e_fname; + struct ntfs_fnd *fnd; + struct INDEX_ROOT *root; + struct mft_inode *mi; + struct ntfs_index *indx = &ni->dir; + NTFS_CMP_FUNC cmp; + + fnd = fnd_get(); + if (!fnd) + return -ENOMEM; + + root = indx_get_root(indx, ni, NULL, &mi); + if (!root) { + err = -EINVAL; + goto out; + } + + cmp = get_cmp_func(root); + if (unlikely(!cmp)) { + WARN_ON_ONCE(1); + return -EINVAL; + } + + /* Find entry in directory. */ + err = indx_find(indx, ni, root, fname, fname_full_size(fname), sbi, + &diff, &e, fnd); + if (err) + goto out; + + if (!e) { + err = -EINVAL; + goto out; + } + + if (diff) { + err = -EINVAL; + goto out; + } + + e_fname = (struct ATTR_FILE_NAME *)(e + 1); + + if (!memcmp(&e_fname->dup, dup, sizeof(*dup))) { + /* + * Nothing to update in index! Try to avoid this call. + */ + goto out; + } + + memcpy(&e_fname->dup, dup, sizeof(*dup)); + + if (fnd->level) { + /* Directory entry in index. */ + err = indx_write(indx, ni, fnd->nodes[fnd->level - 1], sync); + } else { + /* Directory entry in directory MFT record. */ + mi->dirty = true; + if (sync) + err = mi_write(mi, 1); + else + mark_inode_dirty(&ni->vfs_inode); + } + +out: + fnd_put(fnd); + return err; +} diff --git a/fs/ntfs3/inode.c b/fs/ntfs3/inode.c new file mode 100644 index 000000000..56b4f6469 --- /dev/null +++ b/fs/ntfs3/inode.c @@ -0,0 +1,2329 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +struct IGET5_PARAM { + const struct MFT_REF *ref; + const struct cpu_str *name; +}; + +/* + * ntfs_read_mft - Read record and parse MFT. + */ +static int ntfs_read_mft(struct inode *inode, const struct cpu_str *name, + const struct MFT_REF *ref) +{ + int err = 0; + struct ntfs_inode *ni = ntfs_i(inode); + struct super_block *sb = inode->i_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + mode_t mode = 0; + struct ATTR_STD_INFO5 *std5 = NULL; + struct ATTR_LIST_ENTRY *le; + struct ATTRIB *attr; + bool is_match = false; + bool is_root = false; + bool is_dir; + u64 ino = inode->i_ino; + u32 rp_fa = 0, asize, t32; + u16 roff, rsize, names = 0, links = 0; + const struct ATTR_FILE_NAME *fname = NULL; + const struct INDEX_ROOT *root = NULL; + struct REPARSE_DATA_BUFFER rp; // 0x18 bytes + u64 t64; + struct MFT_REC *rec; + struct runs_tree *run; + struct timespec64 ts; + const __le16 *aname; + + inode->i_op = NULL; + /* Setup 'uid' and 'gid' */ + inode->i_uid = sbi->options->fs_uid; + inode->i_gid = sbi->options->fs_gid; + + err = mi_init(&ni->mi, sbi, ino); + if (err) + goto out; + + if (!sbi->mft.ni && ino == MFT_REC_MFT && !sb->s_root) { + t64 = sbi->mft.lbo >> sbi->cluster_bits; + t32 = bytes_to_cluster(sbi, MFT_REC_VOL * sbi->record_size); + sbi->mft.ni = ni; + init_rwsem(&ni->file.run_lock); + + if (!run_add_entry(&ni->file.run, 0, t64, t32, true)) { + err = -ENOMEM; + goto out; + } + } + + err = mi_read(&ni->mi, ino == MFT_REC_MFT); + + if (err) + goto out; + + rec = ni->mi.mrec; + + if (sbi->flags & NTFS_FLAGS_LOG_REPLAYING) { + ; + } else if (ref->seq != rec->seq) { + err = -EINVAL; + ntfs_err(sb, "MFT: r=%llx, expect seq=%x instead of %x!", ino, + le16_to_cpu(ref->seq), le16_to_cpu(rec->seq)); + goto out; + } else if (!is_rec_inuse(rec)) { + err = -ESTALE; + ntfs_err(sb, "Inode r=%x is not in use!", (u32)ino); + goto out; + } + + if (le32_to_cpu(rec->total) != sbi->record_size) { + /* Bad inode? */ + err = -EINVAL; + goto out; + } + + if (!is_rec_base(rec)) { + err = -EINVAL; + goto out; + } + + /* Record should contain $I30 root. */ + is_dir = rec->flags & RECORD_FLAG_DIR; + + /* MFT_REC_MFT is not a dir */ + if (is_dir && ino == MFT_REC_MFT) { + err = -EINVAL; + goto out; + } + + inode->i_generation = le16_to_cpu(rec->seq); + + /* Enumerate all struct Attributes MFT. */ + le = NULL; + attr = NULL; + + /* + * To reduce tab pressure use goto instead of + * while( (attr = ni_enum_attr_ex(ni, attr, &le, NULL) )) + */ +next_attr: + run = NULL; + err = -EINVAL; + attr = ni_enum_attr_ex(ni, attr, &le, NULL); + if (!attr) + goto end_enum; + + if (le && le->vcn) { + /* This is non primary attribute segment. Ignore if not MFT. */ + if (ino != MFT_REC_MFT) + goto next_attr; + + if (attr->type == ATTR_DATA) + run = &ni->file.run; + else if (attr->type == ATTR_BITMAP) + run = &sbi->mft.bitmap.run; + else + goto next_attr; + + asize = le32_to_cpu(attr->size); + goto attr_unpack_run; + } + + roff = attr->non_res ? 0 : le16_to_cpu(attr->res.data_off); + rsize = attr->non_res ? 0 : le32_to_cpu(attr->res.data_size); + asize = le32_to_cpu(attr->size); + aname = attr_name(attr); + + /* + * Really this check was done in 'ni_enum_attr_ex' -> ... 'mi_enum_attr'. + * There not critical to check this case again + */ + if (attr->name_len && + sizeof(short) * attr->name_len + le16_to_cpu(attr->name_off) > + asize) + goto out; + + if (attr->non_res) { + t64 = le64_to_cpu(attr->nres.alloc_size); + if (le64_to_cpu(attr->nres.data_size) > t64 || + le64_to_cpu(attr->nres.valid_size) > t64) + goto out; + } + + switch (attr->type) { + case ATTR_STD: + if (attr->non_res || + asize < sizeof(struct ATTR_STD_INFO) + roff || + rsize < sizeof(struct ATTR_STD_INFO)) + goto out; + + if (std5) + goto next_attr; + + std5 = Add2Ptr(attr, roff); + + nt2kernel(std5->cr_time, &ni->i_crtime); + nt2kernel(std5->a_time, &ts); + inode_set_atime_to_ts(inode, ts); + nt2kernel(std5->c_time, &ts); + inode_set_ctime_to_ts(inode, ts); + nt2kernel(std5->m_time, &ts); + inode_set_mtime_to_ts(inode, ts); + + ni->std_fa = std5->fa; + + if (asize >= sizeof(struct ATTR_STD_INFO5) + roff && + rsize >= sizeof(struct ATTR_STD_INFO5)) + ni->std_security_id = std5->security_id; + goto next_attr; + + case ATTR_LIST: + if (attr->name_len || le || ino == MFT_REC_LOG) + goto out; + + err = ntfs_load_attr_list(ni, attr); + if (err) + goto out; + + le = NULL; + attr = NULL; + goto next_attr; + + case ATTR_NAME: + if (attr->non_res || asize < SIZEOF_ATTRIBUTE_FILENAME + roff || + rsize < SIZEOF_ATTRIBUTE_FILENAME) + goto out; + + names += 1; + fname = Add2Ptr(attr, roff); + if (fname->type == FILE_NAME_DOS) + goto next_attr; + + links += 1; + if (name && name->len == fname->name_len && + !ntfs_cmp_names_cpu(name, (struct le_str *)&fname->name_len, + NULL, false)) + is_match = true; + + goto next_attr; + + case ATTR_DATA: + if (is_dir) { + /* Ignore data attribute in dir record. */ + goto next_attr; + } + + if (ino == MFT_REC_BADCLUST && !attr->non_res) + goto next_attr; + + if (attr->name_len && + ((ino != MFT_REC_BADCLUST || !attr->non_res || + attr->name_len != ARRAY_SIZE(BAD_NAME) || + memcmp(aname, BAD_NAME, sizeof(BAD_NAME))) && + (ino != MFT_REC_SECURE || !attr->non_res || + attr->name_len != ARRAY_SIZE(SDS_NAME) || + memcmp(aname, SDS_NAME, sizeof(SDS_NAME))))) { + /* File contains stream attribute. Ignore it. */ + goto next_attr; + } + + if (is_attr_sparsed(attr)) + ni->std_fa |= FILE_ATTRIBUTE_SPARSE_FILE; + else + ni->std_fa &= ~FILE_ATTRIBUTE_SPARSE_FILE; + + if (is_attr_compressed(attr)) + ni->std_fa |= FILE_ATTRIBUTE_COMPRESSED; + else + ni->std_fa &= ~FILE_ATTRIBUTE_COMPRESSED; + + if (is_attr_encrypted(attr)) + ni->std_fa |= FILE_ATTRIBUTE_ENCRYPTED; + else + ni->std_fa &= ~FILE_ATTRIBUTE_ENCRYPTED; + + mode = S_IFREG | (0777 & sbi->options->fs_fmask_inv); + + if (!attr->non_res) { + ni->i_valid = inode->i_size = rsize; + inode_set_bytes(inode, rsize); + ni->ni_flags |= NI_FLAG_RESIDENT; + goto next_attr; + } + + inode_set_bytes(inode, attr_ondisk_size(attr)); + + ni->i_valid = le64_to_cpu(attr->nres.valid_size); + inode->i_size = le64_to_cpu(attr->nres.data_size); + if (!attr->nres.alloc_size) + goto next_attr; + + run = ino == MFT_REC_BITMAP ? &sbi->used.bitmap.run : + &ni->file.run; + break; + + case ATTR_ROOT: + if (attr->non_res) + goto out; + + root = Add2Ptr(attr, roff); + + if (attr->name_len != ARRAY_SIZE(I30_NAME) || + memcmp(attr_name(attr), I30_NAME, sizeof(I30_NAME))) + goto next_attr; + + if (root->type != ATTR_NAME || + root->rule != NTFS_COLLATION_TYPE_FILENAME) + goto out; + + if (!is_dir) + goto next_attr; + + is_root = true; + ni->ni_flags |= NI_FLAG_DIR; + + err = indx_init(&ni->dir, sbi, attr, INDEX_MUTEX_I30); + if (err) + goto out; + + mode = sb->s_root ? + (S_IFDIR | (0777 & sbi->options->fs_dmask_inv)) : + (S_IFDIR | 0777); + goto next_attr; + + case ATTR_ALLOC: + if (!is_root || attr->name_len != ARRAY_SIZE(I30_NAME) || + memcmp(attr_name(attr), I30_NAME, sizeof(I30_NAME))) + goto next_attr; + + inode->i_size = le64_to_cpu(attr->nres.data_size); + ni->i_valid = le64_to_cpu(attr->nres.valid_size); + inode_set_bytes(inode, le64_to_cpu(attr->nres.alloc_size)); + + run = &ni->dir.alloc_run; + break; + + case ATTR_BITMAP: + if (ino == MFT_REC_MFT) { + if (!attr->non_res) + goto out; +#ifndef CONFIG_NTFS3_64BIT_CLUSTER + /* 0x20000000 = 2^32 / 8 */ + if (le64_to_cpu(attr->nres.alloc_size) >= 0x20000000) + goto out; +#endif + run = &sbi->mft.bitmap.run; + break; + } else if (is_dir && attr->name_len == ARRAY_SIZE(I30_NAME) && + !memcmp(attr_name(attr), I30_NAME, + sizeof(I30_NAME)) && + attr->non_res) { + run = &ni->dir.bitmap_run; + break; + } + goto next_attr; + + case ATTR_REPARSE: + if (attr->name_len) + goto next_attr; + + rp_fa = ni_parse_reparse(ni, attr, &rp); + switch (rp_fa) { + case REPARSE_LINK: + /* + * Normal symlink. + * Assume one unicode symbol == one utf8. + */ + inode->i_size = le16_to_cpu(rp.SymbolicLinkReparseBuffer + .PrintNameLength) / + sizeof(u16); + ni->i_valid = inode->i_size; + /* Clear directory bit. */ + if (ni->ni_flags & NI_FLAG_DIR) { + indx_clear(&ni->dir); + memset(&ni->dir, 0, sizeof(ni->dir)); + ni->ni_flags &= ~NI_FLAG_DIR; + } else { + run_close(&ni->file.run); + } + mode = S_IFLNK | 0777; + is_dir = false; + if (attr->non_res) { + run = &ni->file.run; + goto attr_unpack_run; // Double break. + } + break; + + case REPARSE_COMPRESSED: + break; + + case REPARSE_DEDUPLICATED: + break; + } + goto next_attr; + + case ATTR_EA_INFO: + if (!attr->name_len && + resident_data_ex(attr, sizeof(struct EA_INFO))) { + ni->ni_flags |= NI_FLAG_EA; + /* + * ntfs_get_wsl_perm updates inode->i_uid, inode->i_gid, inode->i_mode + */ + inode->i_mode = mode; + ntfs_get_wsl_perm(inode); + mode = inode->i_mode; + } + goto next_attr; + + default: + goto next_attr; + } + +attr_unpack_run: + roff = le16_to_cpu(attr->nres.run_off); + + if (roff > asize) { + err = -EINVAL; + goto out; + } + + t64 = le64_to_cpu(attr->nres.svcn); + + err = run_unpack_ex(run, sbi, ino, t64, le64_to_cpu(attr->nres.evcn), + t64, Add2Ptr(attr, roff), asize - roff); + if (err < 0) + goto out; + err = 0; + goto next_attr; + +end_enum: + + if (!std5) + goto out; + + if (is_bad_inode(inode)) + goto out; + + if (!is_match && name) { + err = -ENOENT; + goto out; + } + + if (std5->fa & FILE_ATTRIBUTE_READONLY) + mode &= ~0222; + + if (!names) { + err = -EINVAL; + goto out; + } + + if (names != le16_to_cpu(rec->hard_links)) { + /* Correct minor error on the fly. Do not mark inode as dirty. */ + ntfs_inode_warn(inode, "Correct links count -> %u.", names); + rec->hard_links = cpu_to_le16(names); + ni->mi.dirty = true; + } + + if (!links) { + err = -EINVAL; + goto out; + } + + set_nlink(inode, links); + + if (S_ISDIR(mode)) { + ni->std_fa |= FILE_ATTRIBUTE_DIRECTORY; + + /* + * Dot and dot-dot should be included in count but was not + * included in enumeration. + * Usually a hard links to directories are disabled. + */ + inode->i_op = &ntfs_dir_inode_operations; + inode->i_fop = &ntfs_dir_operations; + ni->i_valid = 0; + } else if (S_ISLNK(mode)) { + ni->std_fa &= ~FILE_ATTRIBUTE_DIRECTORY; + inode->i_op = &ntfs_link_inode_operations; + inode->i_fop = NULL; + inode_nohighmem(inode); + } else if (S_ISREG(mode)) { + ni->std_fa &= ~FILE_ATTRIBUTE_DIRECTORY; + inode->i_op = &ntfs_file_inode_operations; + inode->i_fop = &ntfs_file_operations; + inode->i_mapping->a_ops = is_compressed(ni) ? &ntfs_aops_cmpr : + &ntfs_aops; + if (ino != MFT_REC_MFT) + init_rwsem(&ni->file.run_lock); + } else if (S_ISCHR(mode) || S_ISBLK(mode) || S_ISFIFO(mode) || + S_ISSOCK(mode)) { + inode->i_op = &ntfs_special_inode_operations; + init_special_inode(inode, mode, inode->i_rdev); + } else if (fname && fname->home.low == cpu_to_le32(MFT_REC_EXTEND) && + fname->home.seq == cpu_to_le16(MFT_REC_EXTEND)) { + /* Records in $Extend are not a files or general directories. */ + inode->i_op = &ntfs_file_inode_operations; + mode = S_IFREG; + init_rwsem(&ni->file.run_lock); + } else { + err = -EINVAL; + goto out; + } + + if ((sbi->options->sys_immutable && + (std5->fa & FILE_ATTRIBUTE_SYSTEM)) && + !S_ISFIFO(mode) && !S_ISSOCK(mode) && !S_ISLNK(mode)) { + inode->i_flags |= S_IMMUTABLE; + } else { + inode->i_flags &= ~S_IMMUTABLE; + } + + inode->i_mode = mode; + if (!(ni->ni_flags & NI_FLAG_EA)) { + /* If no xattr then no security (stored in xattr). */ + inode->i_flags |= S_NOSEC; + } + + if (ino == MFT_REC_MFT && !sb->s_root) + sbi->mft.ni = NULL; + + return 0; + +out: + if (ino == MFT_REC_MFT && !sb->s_root) + sbi->mft.ni = NULL; + + return err; +} + +/* + * ntfs_init_ads_node + * + * This function scans base inode for given ADS. + * And init inode associated with this ADS + */ +static int ntfs_init_ads_node(struct inode *inode, const __le16 *ads_name, + u8 ads_len, u32 flags) +{ + int err = -EINVAL; + struct ntfs_inode *ni = ntfs_i(inode); + struct ntfs_inode *nb = ni->base; + struct ntfs_sb_info *sbi = nb->mi.sbi; + struct ATTR_LIST_ENTRY *le = NULL; + struct ATTRIB *attr = NULL; + u16 roff, asize; + u64 svcn; + + if (nb->ni_flags & NI_FLAG_DIR) + return -EINVAL; /* no ADS for directories. */ + + ni->mi.sbi = sbi; + ni->mi.rno = inode->i_ino; + + if (ads_len == ARRAY_SIZE(QUERY_STREAMS) && + !memcmp(ads_name, QUERY_STREAMS, sizeof(QUERY_STREAMS))) { + goto ok; /* use goto to reduce tab pressure. */ + } + + /* Enumerate all attributes in record. */ + while ((attr = ni_enum_attr_ex(nb, attr, &le, NULL))) { + if (attr->type == ATTR_DATA && attr->name_len && + ads_len == attr->name_len && + !memcmp(ads_name, attr_name(attr), ads_len * sizeof(u16))) { + /* We have found the ADS to open. */ + break; + } + } + + if (!attr) { + if (!(flags & LOOKUP_CREATE)) { + /* Do not create ADS. */ + return -ENOENT; + } + + /* Create new ADS. */ + err = ni_insert_resident(nb, 0, ATTR_DATA, ads_name, ads_len, + &attr, NULL, NULL); + if (err) { + /* Looks like the only reasons: ENOSPC/ENOMEM .*/ + return err; + } + } + + if (is_attr_sparsed(attr)) + ni->std_fa |= FILE_ATTRIBUTE_SPARSE_FILE; + else + ni->std_fa &= ~FILE_ATTRIBUTE_SPARSE_FILE; + + if (is_attr_compressed(attr)) + ni->std_fa |= FILE_ATTRIBUTE_COMPRESSED; + else + ni->std_fa &= ~FILE_ATTRIBUTE_COMPRESSED; + + if (is_attr_encrypted(attr)) + ni->std_fa |= FILE_ATTRIBUTE_ENCRYPTED; + else + ni->std_fa &= ~FILE_ATTRIBUTE_ENCRYPTED; + + if (!attr->non_res) { + ni->ni_flags |= NI_FLAG_RESIDENT; + ni->i_valid = inode->i_size = le32_to_cpu(attr->res.data_size); + inode_set_bytes(inode, inode->i_size); + goto ok; + } + + inode_set_bytes(inode, attr_ondisk_size(attr)); + ni->i_valid = le64_to_cpu(attr->nres.valid_size); + inode->i_size = le64_to_cpu(attr->nres.data_size); + + if (!attr->nres.alloc_size) + goto ok; + + roff = le16_to_cpu(attr->nres.run_off); + asize = le32_to_cpu(attr->size); + + if (roff > asize) { + /* This case should be checked in mi_enum_attr */ + return -EINVAL; + } + + svcn = le64_to_cpu(attr->nres.svcn); + err = run_unpack_ex(&ni->file.run, sbi, ni->mi.rno, svcn, + le64_to_cpu(attr->nres.evcn), svcn, + Add2Ptr(attr, roff), asize - roff); + if (err < 0) { + /* run_unpack_ex marks volume dirty, if logical error. */ + return err; + } + +ok: + /* Keep ADS name (little endian). */ + ni->file.ads.name = kmemdup(ads_name, ads_len * sizeof(u16), GFP_NOFS); + if (!ni->file.ads.name) + return -ENOMEM; + ni->file.ads.len = ads_len; + + set_nlink(inode, 1); + + init_rwsem(&ni->file.run_lock); + /* Most fields are the same as the base's? */ + inode->i_op = nb->vfs_inode.i_op; + inode->i_fop = nb->vfs_inode.i_fop; + inode->i_mapping->a_ops = nb->vfs_inode.i_mapping->a_ops; + inode->i_flags = nb->vfs_inode.i_flags; + inode->i_mode = nb->vfs_inode.i_mode; + inode->i_uid = nb->vfs_inode.i_uid; + inode->i_gid = nb->vfs_inode.i_gid; + inode->i_generation = nb->vfs_inode.i_generation; + + return 0; +} + +/* + * ntfs_test_inode + * + * Return: 1 if match. + */ +static int ntfs_test_inode(struct inode *inode, void *data) +{ + const struct IGET5_PARAM *ig5 = data; + struct ntfs_inode *ni; + const struct cpu_str *name; + + if (ino_get(ig5->ref) != inode->i_ino) + return 0; + + ni = ntfs_i(inode); + + if (ni->ni_flags & NI_FLAG_DIR) { + /* No ads for directories. */ + return 1; + } + + name = ig5->name; + if (!name || !name->ads_len) { + if (!ni->file.ads.len) { + /* default file (not ads) match. */ + return 1; + } + } else if (ni->file.ads.len == name->ads_len && + !memcmp(ni->file.ads.name, &name->name[name->len + 1], + name->ads_len * sizeof(u16))) { + /* ads name match. */ + return 1; + } + + return 0; +} + +static int ntfs_set_inode(struct inode *inode, void *data) +{ + const struct IGET5_PARAM *ig5 = data; + + inode->i_ino = ino_get(ig5->ref); + return 0; +} + +struct inode *ntfs_iget5_flags(struct super_block *sb, + const struct MFT_REF *ref, + const struct cpu_str *name, u32 flags) +{ + int err; + /* Pack params to pass in iget5_locked. */ + struct IGET5_PARAM ig5 = { ref, name }; + u64 ino = ino_get(ref); + struct inode *inode, *base = NULL; + bool ads = name && name->ads_len; + struct ntfs_inode *ni; + + if (ads) { + /* First get base inode */ + base = ntfs_iget5_flags(sb, ref, NULL, 0); + if (IS_ERR(base)) + return base; + } + + inode = iget5_locked(sb, ino, ntfs_test_inode, ntfs_set_inode, &ig5); + if (unlikely(!inode)) { + err = -ENOMEM; + goto out; + } + + ni = ntfs_i(inode); + + /* If this is a freshly allocated inode, need to read it now. */ + if (inode_state_read_once(inode) & I_NEW) { + if (!base) { + /* default inode. generic file/dir. */ + ni->base = ni; + } else { + /* inode + ads */ + ni->base = ntfs_i(base); + base = NULL; /* keep reference incremented (instead of ihold). */ + } + + if (ads) { + /* base record is loaded. Init ads node. */ + err = ntfs_init_ads_node( + inode, (__le16 *)&name->name[name->len + 1], + name->ads_len, flags); + } else { + err = ntfs_read_mft(inode, name, ref); + } + + if (!err) { + unlock_new_inode(inode); + } else { + iget_failed(inode); + /* Do not mark volume dirty if ADS not found. */ + if (ads) + goto out; + } + } else if (!ads && ref->seq != ni->mi.mrec->seq) { + /* + * Sequence number is not expected. + * Looks like inode was reused but caller uses the old reference + */ + iput(inode); + err = -ESTALE; + } else { + err = 0; + } + + if (err) + ntfs_set_state(sb->s_fs_info, NTFS_DIRTY_ERROR); + +out: + if (base) + iput(base); + + if (err) + return ERR_PTR(err); + + return inode; +} + +static sector_t ntfs_bmap(struct address_space *mapping, sector_t block) +{ + struct inode *inode = mapping->host; + struct ntfs_inode *ni = ntfs_i(inode); + + /* + * We can get here for an inline file via the FIBMAP ioctl + */ + if (is_resident(ni)) + return 0; + + if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) && + !run_is_empty(&ni->file.run_da)) { + /* + * With delalloc data we want to sync the file so + * that we can make sure we allocate blocks for file and data + * is in place for the user to see it + */ + ni_allocate_da_blocks(ni); + } + + return iomap_bmap(mapping, block, &ntfs_iomap_ops); +} + +static void ntfs_iomap_read_end_io(struct bio *bio) +{ + int error = blk_status_to_errno(bio->bi_status); + struct folio_iter fi; + + bio_for_each_folio_all(fi, bio) { + struct folio *folio = fi.folio; + struct inode *inode = folio->mapping->host; + struct ntfs_inode *ni = ntfs_i(inode); + u64 valid = ni->i_valid; + u32 f_size = folio_size(folio); + loff_t f_pos = folio_pos(folio); + + if (valid < f_pos + f_size) { + u32 z_from = valid <= f_pos ? + 0 : + offset_in_folio(folio, valid); + /* The only thing ntfs_iomap_read_end_io used for. */ + folio_zero_segment(folio, z_from, f_size); + } + + iomap_finish_folio_read(folio, fi.offset, fi.length, error); + } + bio_put(bio); +} + +static void ntfs_iomap_bio_submit_read(const struct iomap_iter *iter, + struct iomap_read_folio_ctx *ctx) +{ + iomap_bio_submit_read_endio(iter, ctx, ntfs_iomap_read_end_io); +} + +// clang-format off +static const struct iomap_read_ops ntfs_iomap_bio_read_ops = { + .read_folio_range = iomap_bio_read_folio_range, + .submit_read = ntfs_iomap_bio_submit_read, +}; +// clang-format on + +static int ntfs_read_folio(struct file *file, struct folio *folio) +{ + int err; + struct address_space *mapping = folio->mapping; + struct inode *inode = mapping->host; + struct ntfs_inode *ni = ntfs_i(inode); + loff_t vbo = folio_pos(folio); + struct iomap_read_folio_ctx ctx = { + .cur_folio = folio, + .ops = &ntfs_iomap_bio_read_ops, + }; + + if (unlikely(is_bad_ni(ni))) { + folio_unlock(folio); + return -EIO; + } + + if (ni->i_valid <= vbo) { + folio_zero_range(folio, 0, folio_size(folio)); + folio_mark_uptodate(folio); + folio_unlock(folio); + return 0; + } + + if (is_compressed(ni)) { + /* ni_lock is taken inside ni_read_folio_cmpr after page locks */ + err = ni_read_folio_cmpr(ni, folio); + return err; + } + + iomap_read_folio(&ntfs_iomap_ops, &ctx, NULL); + return 0; +} + +static void ntfs_readahead(struct readahead_control *rac) +{ + struct address_space *mapping = rac->mapping; + struct inode *inode = mapping->host; + struct ntfs_inode *ni = ntfs_i(inode); + struct iomap_read_folio_ctx ctx = { + .ops = &ntfs_iomap_bio_read_ops, + .rac = rac, + }; + + if (is_resident(ni)) { + /* No readahead for resident. */ + return; + } + + if (is_compressed(ni)) { + /* No readahead for compressed. */ + return; + } + + iomap_readahead(&ntfs_iomap_ops, &ctx, NULL); +} + +int ntfs_set_size(struct inode *inode, u64 new_size) +{ + struct super_block *sb = inode->i_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + struct ntfs_inode *ni = ntfs_i(inode); + int err; + + /* Check for maximum file size. */ + if (is_sparsed(ni) || is_compressed(ni)) { + if (new_size > sbi->maxbytes_sparse) { + return -EFBIG; + } + } else if (new_size > sbi->maxbytes) { + return -EFBIG; + } + + /* Mark rw ntfs as dirty. It will be cleared at umount. */ + ntfs_set_state(sbi, NTFS_DIRTY_DIRTY); + + ni_lock(ni); + down_write(&ni->file.run_lock); + if (new_size < ni->i_valid) + ni->i_valid = new_size; + + /* last 'true' means keep preallocated. */ + err = attr_set_size(ni, ATTR_DATA, ni->file.ads.name, ni->file.ads.len, + &ni->file.run, new_size, &ni->i_valid, true); + + up_write(&ni->file.run_lock); + ni_unlock(ni); + + return err; +} + +/* + * Special value to detect ntfs_writeback_range call + */ +#define WB_NO_DA (struct iomap *)1 +/* + * Function to get mapping vbo -> lbo. + * used with: + * - iomap_zero_range + * - iomap_truncate_page + * - iomap_dio_rw + * - iomap_file_buffered_write + * - iomap_bmap + * - iomap_fiemap + * - iomap_bio_read_folio + * - iomap_bio_readahead + */ +static int ntfs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, + unsigned int flags, struct iomap *iomap, + struct iomap *srcmap) +{ + struct ntfs_inode *ni = ntfs_i(inode); + struct ntfs_sb_info *sbi = ni->mi.sbi; + u8 cluster_bits = sbi->cluster_bits; + CLST vcn = offset >> cluster_bits; + u32 off = offset & sbi->cluster_mask; + bool rw = flags & IOMAP_WRITE; + loff_t endbyte = offset + length; + void *res = NULL; + int err; + CLST lcn, clen, clen_max = 1; + bool new_clst = false; + bool no_da; + bool zero = false; + if (unlikely(ntfs3_forced_shutdown(sbi->sb))) + return -EIO; + + if (flags & IOMAP_REPORT) { + if (offset > ntfs_get_maxbytes(ni)) { + /* called from fiemap/bmap. */ + return -EINVAL; + } + + if (offset >= inode->i_size) { + /* special code for report. */ + return -ENOENT; + } + } + + if (IOMAP_ZERO == flags && (endbyte & sbi->cluster_mask)) { + rw = true; + } else if (rw) { + clen_max = bytes_to_cluster(sbi, endbyte) - vcn; + } + + /* + * Force to allocate clusters if directIO(write) or writeback_range. + * NOTE: attr_data_get_block allocates clusters only for sparse file. + * Normal file allocates clusters in attr_set_size. + */ + no_da = flags == (IOMAP_DIRECT | IOMAP_WRITE) || srcmap == WB_NO_DA; + + err = attr_data_get_block(ni, vcn, clen_max, &lcn, &clen, + rw ? &new_clst : NULL, zero, &res, no_da); + + if (err) { + return err; + } + + if (lcn == EOF_LCN) { + /* request out of file. */ + if (flags & IOMAP_REPORT) { + /* special code for report. */ + return -ENOENT; + } + + if (rw) { + /* should never be here. */ + return -EINVAL; + } + lcn = SPARSE_LCN; + } + + iomap->flags = new_clst ? IOMAP_F_NEW : 0; + + if (lcn == RESIDENT_LCN) { + if (offset >= clen) { + if (res) + __free_page(virt_to_page(res)); + if (flags & IOMAP_REPORT) { + /* special code for report. */ + return -ENOENT; + } + return -EFAULT; + } + + iomap->private = iomap->inline_data = res; + iomap->type = IOMAP_INLINE; + iomap->offset = 0; + iomap->length = clen; /* resident size in bytes. */ + return 0; + } + + if (!clen) { + /* broken file? */ + return -EINVAL; + } + + iomap->bdev = inode->i_sb->s_bdev; + iomap->offset = offset; + iomap->length = ((loff_t)clen << cluster_bits) - off; + + if (lcn == COMPRESSED_LCN) { + /* should never be here. */ + return -EOPNOTSUPP; + } + + if (lcn == DELALLOC_LCN) { + iomap->type = IOMAP_DELALLOC; + iomap->addr = IOMAP_NULL_ADDR; + } else { + /* Translate clusters into bytes. */ + iomap->addr = ((loff_t)lcn << cluster_bits) + off; + if (length && iomap->length > length) + iomap->length = length; + else + endbyte = offset + iomap->length; + + if (lcn == SPARSE_LCN) { + iomap->addr = IOMAP_NULL_ADDR; + iomap->type = IOMAP_HOLE; + // if (IOMAP_ZERO == flags && !off) { + // iomap->length = (endbyte - offset) & + // sbi->cluster_mask_inv; + // } + } else if (endbyte <= ni->i_valid) { + iomap->type = IOMAP_MAPPED; + } else if (offset < ni->i_valid) { + iomap->type = IOMAP_MAPPED; + if (flags & IOMAP_REPORT) + iomap->length = ni->i_valid - offset; + } else if (rw || (flags & IOMAP_ZERO)) { + iomap->type = IOMAP_MAPPED; + } else { + iomap->type = IOMAP_UNWRITTEN; + } + } + + if ((flags & IOMAP_ZERO) && + (iomap->type == IOMAP_MAPPED || iomap->type == IOMAP_DELALLOC)) { + /* Avoid too large requests. */ + u32 tail; + u32 off_a = offset & (PAGE_SIZE - 1); + if (off_a) + tail = PAGE_SIZE - off_a; + else + tail = PAGE_SIZE; + + if (iomap->length > tail) + iomap->length = tail; + } + + return 0; +} + +static int ntfs_iomap_end(struct inode *inode, loff_t pos, loff_t length, + ssize_t written, unsigned int flags, + struct iomap *iomap) +{ + int err = 0; + struct ntfs_inode *ni = ntfs_i(inode); + loff_t endbyte = pos + written; + + if ((flags & IOMAP_WRITE) || (flags & IOMAP_ZERO)) { + if (iomap->type == IOMAP_INLINE) { + u32 data_size; + struct ATTRIB *attr; + struct mft_inode *mi; + + attr = ni_find_attr(ni, NULL, NULL, ATTR_DATA, + ni->file.ads.name, ni->file.ads.len, + NULL, &mi); + if (!attr || attr->non_res) { + err = -EINVAL; + goto out; + } + + data_size = le32_to_cpu(attr->res.data_size); + if (!(pos < data_size && endbyte <= data_size)) { + err = -EINVAL; + goto out; + } + + /* Update resident data. */ + memcpy(resident_data(attr) + pos, + iomap_inline_data(iomap, pos), written); + mi->dirty = true; + ni->i_valid = data_size; + } else if (ni->i_valid < endbyte) { + ni->i_valid = endbyte; + mark_inode_dirty(inode); + } + } + + if ((flags & IOMAP_ZERO) && + (iomap->type == IOMAP_MAPPED || iomap->type == IOMAP_DELALLOC)) { + /* Pair for code in ntfs_iomap_begin. */ + balance_dirty_pages_ratelimited(inode->i_mapping); + cond_resched(); + } + +out: + if (iomap->type == IOMAP_INLINE) { + __free_page(virt_to_page(iomap->private)); + iomap->private = NULL; + } + + return err; +} + +/* + * write_begin + put_folio + write_end. + * iomap_zero_range + * iomap_truncate_page + * iomap_file_buffered_write + */ +static void ntfs_iomap_put_folio(struct inode *inode, loff_t pos, + unsigned int len, struct folio *folio) +{ + struct ntfs_inode *ni = ntfs_i(inode); + loff_t end = pos + len; + u32 f_size = folio_size(folio); + loff_t f_pos = folio_pos(folio); + loff_t f_end = f_pos + f_size; + + if (ni->i_valid <= end && end < f_end) { + /* zero range [end - f_end). */ + /* The only thing ntfs_iomap_put_folio used for. */ + folio_zero_segment(folio, offset_in_folio(folio, end), f_size); + } + folio_unlock(folio); + folio_put(folio); +} + +/* + * iomap_writeback_ops::writeback_range + */ +static ssize_t ntfs_writeback_range(struct iomap_writepage_ctx *wpc, + struct folio *folio, u64 offset, + unsigned int len, u64 end_pos) +{ + struct iomap *iomap = &wpc->iomap; + /* Check iomap position. */ + if (iomap->offset + iomap->length <= offset || offset < iomap->offset) { + int err; + struct inode *inode = wpc->inode; + struct ntfs_inode *ni = ntfs_i(inode); + struct ntfs_sb_info *sbi = ntfs_sb(inode->i_sb); + loff_t i_size_up = ntfs_up_cluster(sbi, inode->i_size); + loff_t len_max = i_size_up - offset; + + err = ni->file.run_da.count ? ni_allocate_da_blocks(ni) : 0; + + if (!err) { + /* Use local special value 'WB_NO_DA' to disable delalloc. */ + err = ntfs_iomap_begin(inode, offset, len_max, + IOMAP_WRITE, iomap, WB_NO_DA); + } + + if (err) { + ntfs_set_state(sbi, NTFS_DIRTY_DIRTY); + return err; + } + } + + return iomap_add_to_ioend(wpc, folio, offset, end_pos, len); +} + +static const struct iomap_writeback_ops ntfs_writeback_ops = { + .writeback_range = ntfs_writeback_range, + .writeback_submit = iomap_ioend_writeback_submit, +}; + +static int ntfs_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + int err; + struct inode *inode = mapping->host; + struct ntfs_inode *ni = ntfs_i(inode); + struct iomap_writepage_ctx wpc = { + .inode = inode, + .wbc = wbc, + .ops = &ntfs_writeback_ops, + }; + + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(ni))) + return -EINVAL; + + if (unlikely(ntfs3_forced_shutdown(inode->i_sb))) + return -EIO; + + if (is_resident(ni)) { + struct folio *folio = NULL; + err = 0; + + while ((folio = writeback_iter(mapping, wbc, folio, &err))) { + int err2; + + ni_lock(ni); + err2 = attr_data_write_resident(ni, folio); + ni_unlock(ni); + + folio_unlock(folio); + if (err2) { + mapping_set_error(mapping, err2); + if (!err) + err = err2; + } + } + + return err; + } + + return iomap_writepages(&wpc); +} + +int ntfs3_write_inode(struct inode *inode, struct writeback_control *wbc) +{ + return _ni_write_inode(inode, wbc->sync_mode == WB_SYNC_ALL); +} + +int ntfs_sync_inode(struct inode *inode) +{ + return _ni_write_inode(inode, 1); +} + +/* + * Helper function to read file. + * Used to read $AttrDef and $UpCase + */ +int inode_read_data(struct inode *inode, void *data, size_t bytes) +{ + pgoff_t idx; + struct address_space *mapping = inode->i_mapping; + + for (idx = 0; bytes; idx++) { + size_t op = bytes > PAGE_SIZE ? PAGE_SIZE : bytes; + struct page *page = read_mapping_page(mapping, idx, NULL); + void *kaddr; + + if (IS_ERR(page)) + return PTR_ERR(page); + + kaddr = kmap_atomic(page); + memcpy(data, kaddr, op); + kunmap_atomic(kaddr); + + put_page(page); + + bytes -= op; + data = Add2Ptr(data, PAGE_SIZE); + } + return 0; +} + +/* + * ntfs_reparse_bytes + * + * Number of bytes for REPARSE_DATA_BUFFER(IO_REPARSE_TAG_SYMLINK) + * for unicode string of @uni_len length. + */ +static inline u32 ntfs_reparse_bytes(u32 uni_len, bool is_absolute) +{ + /* Header + unicode string + decorated unicode string. */ + return sizeof(short) * (2 * uni_len + (is_absolute ? 4 : 0)) + + offsetof(struct REPARSE_DATA_BUFFER, + SymbolicLinkReparseBuffer.PathBuffer); +} + +static struct REPARSE_DATA_BUFFER * +ntfs_create_reparse_buffer(struct ntfs_sb_info *sbi, const char *symname, + u32 size, u16 *nsize) +{ + int i, err; + struct REPARSE_DATA_BUFFER *rp; + __le16 *rp_name; + typeof(rp->SymbolicLinkReparseBuffer) *rs; + bool is_absolute; + + is_absolute = symname[0] && symname[1] == ':'; + + rp = kzalloc(ntfs_reparse_bytes(2 * size + 2, is_absolute), GFP_NOFS); + if (!rp) + return ERR_PTR(-ENOMEM); + + rs = &rp->SymbolicLinkReparseBuffer; + rp_name = rs->PathBuffer; + + /* Convert link name to UTF-16. */ + err = ntfs_nls_to_utf16(sbi, symname, size, + (struct cpu_str *)(rp_name - 1), 2 * size, + UTF16_LITTLE_ENDIAN); + if (err < 0) + goto out; + + /* err = the length of unicode name of symlink. */ + *nsize = ntfs_reparse_bytes(err, is_absolute); + + if (*nsize > sbi->reparse.max_size) { + err = -EFBIG; + goto out; + } + + /* Translate Linux '/' into Windows '\'. */ + for (i = 0; i < err; i++) { + if (rp_name[i] == cpu_to_le16('/')) + rp_name[i] = cpu_to_le16('\\'); + } + + rp->ReparseTag = IO_REPARSE_TAG_SYMLINK; + rp->ReparseDataLength = + cpu_to_le16(*nsize - offsetof(struct REPARSE_DATA_BUFFER, + SymbolicLinkReparseBuffer)); + + /* PrintName + SubstituteName. */ + rs->SubstituteNameOffset = cpu_to_le16(sizeof(short) * err); + rs->SubstituteNameLength = + cpu_to_le16(sizeof(short) * err + (is_absolute ? 8 : 0)); + rs->PrintNameLength = rs->SubstituteNameOffset; + + /* + * TODO: Use relative path if possible to allow Windows to + * parse this path. + * 0-absolute path, 1- relative path (SYMLINK_FLAG_RELATIVE). + */ + rs->Flags = cpu_to_le32(is_absolute ? 0 : SYMLINK_FLAG_RELATIVE); + + memmove(rp_name + err + (is_absolute ? 4 : 0), rp_name, + sizeof(short) * err); + + if (is_absolute) { + /* Decorate SubstituteName. */ + rp_name += err; + rp_name[0] = cpu_to_le16('\\'); + rp_name[1] = cpu_to_le16('?'); + rp_name[2] = cpu_to_le16('?'); + rp_name[3] = cpu_to_le16('\\'); + } + + return rp; +out: + kfree(rp); + return ERR_PTR(err); +} + +/* + * ntfs_create_inode + * + * Helper function for: + * - ntfs_create + * - ntfs_mknod + * - ntfs_symlink + * - ntfs_mkdir + * - ntfs_atomic_open + * + * NOTE: if fnd != NULL (ntfs_atomic_open) then @dir is locked + */ +int ntfs_create_inode(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const struct cpu_str *uni, + umode_t mode, dev_t dev, const char *symname, u32 size, + struct ntfs_fnd *fnd) +{ + int err; + struct super_block *sb = dir->i_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + const struct qstr *name = &dentry->d_name; + CLST ino = 0; + struct ntfs_inode *dir_ni = ntfs_i(dir); + struct ntfs_inode *ni = NULL; + struct inode *inode = NULL; + struct ATTRIB *attr; + struct ATTR_STD_INFO5 *std5; + struct ATTR_FILE_NAME *fname; + struct MFT_REC *rec; + u32 asize, dsize, sd_size; + enum FILE_ATTRIBUTE fa; + __le32 security_id = SECURITY_ID_INVALID; + CLST vcn; + const void *sd; + u16 t16, nsize = 0, aid = 0; + struct INDEX_ROOT *root, *dir_root; + struct NTFS_DE *e, *new_de = NULL; + struct REPARSE_DATA_BUFFER *rp = NULL; + bool rp_inserted = false; + + /* New file will be resident or non resident. */ + const bool new_file_resident = 1; + + if (!fnd) + ni_lock_dir(dir_ni); + + if (sbi->options->ads) { + const char *ads = strchr(name->name + 1, ':'); + if (ads && ads[1]) { + ntfs_warn(sb, "failed to create ads"); + err = -EINVAL; + goto out1; + } + } + + dir_root = indx_get_root(&dir_ni->dir, dir_ni, NULL, NULL); + if (!dir_root) { + err = -EINVAL; + goto out1; + } + + if (S_ISDIR(mode)) { + /* Use parent's directory attributes. */ + fa = dir_ni->std_fa | FILE_ATTRIBUTE_DIRECTORY | + FILE_ATTRIBUTE_ARCHIVE; + /* + * By default child directory inherits parent attributes. + * Root directory is hidden + system. + * Make an exception for children in root. + */ + if (dir->i_ino == MFT_REC_ROOT) + fa &= ~(FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_SYSTEM); + } else if (S_ISLNK(mode)) { + /* It is good idea that link should be the same type (file/dir) as target */ + fa = FILE_ATTRIBUTE_REPARSE_POINT; + + /* + * Linux: there are dir/file/symlink and so on. + * NTFS: symlinks are "dir + reparse" or "file + reparse" + * It is good idea to create: + * dir + reparse if 'symname' points to directory + * or + * file + reparse if 'symname' points to file + * Unfortunately kern_path hangs if symname contains 'dir'. + */ + + /* + * struct path path; + * + * if (!kern_path(symname, LOOKUP_FOLLOW, &path)){ + * struct inode *target = d_inode(path.dentry); + * + * if (S_ISDIR(target->i_mode)) + * fa |= FILE_ATTRIBUTE_DIRECTORY; + * // if ( target->i_sb == sb ){ + * // use relative path? + * // } + * path_put(&path); + * } + */ + } else if (S_ISREG(mode)) { + if (sbi->options->sparse) { + /* Sparsed regular file, cause option 'sparse'. */ + fa = FILE_ATTRIBUTE_SPARSE_FILE | + FILE_ATTRIBUTE_ARCHIVE; + } else if (dir_ni->std_fa & FILE_ATTRIBUTE_COMPRESSED) { + /* Compressed regular file, if parent is compressed. */ + fa = FILE_ATTRIBUTE_COMPRESSED | FILE_ATTRIBUTE_ARCHIVE; + } else { + /* Regular file, default attributes. */ + fa = FILE_ATTRIBUTE_ARCHIVE; + } + } else { + fa = FILE_ATTRIBUTE_ARCHIVE; + } + + /* If option "hide_dot_files" then set hidden attribute for dot files. */ + if (sbi->options->hide_dot_files && name->name[0] == '.') + fa |= FILE_ATTRIBUTE_HIDDEN; + + if (!(mode & 0222)) + fa |= FILE_ATTRIBUTE_READONLY; + + new_de = kzalloc(PATH_MAX, GFP_KERNEL); + if (!new_de) { + err = -ENOMEM; + goto out1; + } + + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(dir_ni))) { + err = -EINVAL; + goto out2; + } + + if (unlikely(ntfs3_forced_shutdown(sb))) { + err = -EIO; + goto out2; + } + + /* Mark rw ntfs as dirty. it will be cleared at umount. */ + ntfs_set_state(sbi, NTFS_DIRTY_DIRTY); + + /* Step 1: allocate and fill new mft record. */ + err = ntfs_look_free_mft(sbi, &ino, false, NULL, NULL); + if (err) + goto out2; + + ni = ntfs_new_inode(sbi, ino, S_ISDIR(mode) ? RECORD_FLAG_DIR : 0); + if (IS_ERR(ni)) { + err = PTR_ERR(ni); + ni = NULL; + goto out3; + } + inode = &ni->vfs_inode; + inode_init_owner(idmap, inode, dir, mode); + mode = inode->i_mode; + + ni->i_crtime = current_time(inode); + + rec = ni->mi.mrec; + rec->hard_links = cpu_to_le16(1); + attr = Add2Ptr(rec, le16_to_cpu(rec->attr_off)); + + /* Get default security id. */ + sd = s_default_security; + sd_size = sizeof(s_default_security); + + if (is_ntfs3(sbi)) { + security_id = dir_ni->std_security_id; + if (le32_to_cpu(security_id) < SECURITY_ID_FIRST) { + security_id = sbi->security.def_security_id; + + if (security_id == SECURITY_ID_INVALID && + !ntfs_insert_security(sbi, sd, sd_size, + &security_id, NULL)) + sbi->security.def_security_id = security_id; + } + } + + /* Insert standard info. */ + std5 = Add2Ptr(attr, SIZEOF_RESIDENT); + + if (security_id == SECURITY_ID_INVALID) { + dsize = sizeof(struct ATTR_STD_INFO); + } else { + dsize = sizeof(struct ATTR_STD_INFO5); + std5->security_id = security_id; + ni->std_security_id = security_id; + } + asize = SIZEOF_RESIDENT + dsize; + + attr->type = ATTR_STD; + attr->size = cpu_to_le32(asize); + attr->id = cpu_to_le16(aid++); + attr->res.data_off = SIZEOF_RESIDENT_LE; + attr->res.data_size = cpu_to_le32(dsize); + + std5->cr_time = std5->m_time = std5->c_time = std5->a_time = + kernel2nt(&ni->i_crtime); + + std5->fa = ni->std_fa = fa; + + attr = Add2Ptr(attr, asize); + + /* Insert file name. */ + err = fill_name_de(sbi, new_de, name, uni); + if (err) + goto out4; + + mi_get_ref(&ni->mi, &new_de->ref); + + fname = (struct ATTR_FILE_NAME *)(new_de + 1); + + if (sbi->options->windows_names && + !valid_windows_name(sbi, (struct le_str *)&fname->name_len)) { + err = -EINVAL; + goto out4; + } + + mi_get_ref(&dir_ni->mi, &fname->home); + fname->dup.cr_time = fname->dup.m_time = fname->dup.c_time = + fname->dup.a_time = std5->cr_time; + fname->dup.alloc_size = fname->dup.data_size = 0; + fname->dup.fa = std5->fa; + fname->dup.extend_data = S_ISLNK(mode) ? IO_REPARSE_TAG_SYMLINK : 0; + + dsize = le16_to_cpu(new_de->key_size); + asize = ALIGN(SIZEOF_RESIDENT + dsize, 8); + + attr->type = ATTR_NAME; + attr->size = cpu_to_le32(asize); + attr->res.data_off = SIZEOF_RESIDENT_LE; + attr->res.flags = RESIDENT_FLAG_INDEXED; + attr->id = cpu_to_le16(aid++); + attr->res.data_size = cpu_to_le32(dsize); + memcpy(Add2Ptr(attr, SIZEOF_RESIDENT), fname, dsize); + + attr = Add2Ptr(attr, asize); + + if (security_id == SECURITY_ID_INVALID) { + /* Insert security attribute. */ + asize = SIZEOF_RESIDENT + ALIGN(sd_size, 8); + + attr->type = ATTR_SECURE; + attr->size = cpu_to_le32(asize); + attr->id = cpu_to_le16(aid++); + attr->res.data_off = SIZEOF_RESIDENT_LE; + attr->res.data_size = cpu_to_le32(sd_size); + memcpy(Add2Ptr(attr, SIZEOF_RESIDENT), sd, sd_size); + + attr = Add2Ptr(attr, asize); + } + + attr->id = cpu_to_le16(aid++); + if (fa & FILE_ATTRIBUTE_DIRECTORY) { + /* + * Regular directory or symlink to directory. + * Create root attribute. + */ + dsize = sizeof(struct INDEX_ROOT) + sizeof(struct NTFS_DE); + asize = sizeof(I30_NAME) + SIZEOF_RESIDENT + dsize; + + attr->type = ATTR_ROOT; + attr->size = cpu_to_le32(asize); + + attr->name_len = ARRAY_SIZE(I30_NAME); + attr->name_off = SIZEOF_RESIDENT_LE; + attr->res.data_off = + cpu_to_le16(sizeof(I30_NAME) + SIZEOF_RESIDENT); + attr->res.data_size = cpu_to_le32(dsize); + memcpy(Add2Ptr(attr, SIZEOF_RESIDENT), I30_NAME, + sizeof(I30_NAME)); + + root = Add2Ptr(attr, sizeof(I30_NAME) + SIZEOF_RESIDENT); + memcpy(root, dir_root, offsetof(struct INDEX_ROOT, ihdr)); + root->ihdr.de_off = cpu_to_le32(sizeof(struct INDEX_HDR)); + root->ihdr.used = cpu_to_le32(sizeof(struct INDEX_HDR) + + sizeof(struct NTFS_DE)); + root->ihdr.total = root->ihdr.used; + + e = Add2Ptr(root, sizeof(struct INDEX_ROOT)); + e->size = cpu_to_le16(sizeof(struct NTFS_DE)); + e->flags = NTFS_IE_LAST; + } else if (S_ISLNK(mode)) { + /* + * Symlink to file. + * Create empty resident data attribute. + */ + asize = SIZEOF_RESIDENT; + + /* Insert empty ATTR_DATA */ + attr->type = ATTR_DATA; + attr->size = cpu_to_le32(SIZEOF_RESIDENT); + attr->name_off = SIZEOF_RESIDENT_LE; + attr->res.data_off = SIZEOF_RESIDENT_LE; + } else if (!new_file_resident && S_ISREG(mode)) { + /* + * Regular file. Create empty non resident data attribute. + */ + attr->type = ATTR_DATA; + attr->non_res = 1; + attr->nres.evcn = cpu_to_le64(-1ll); + if (fa & FILE_ATTRIBUTE_SPARSE_FILE) { + attr->size = cpu_to_le32(SIZEOF_NONRESIDENT_EX + 8); + attr->name_off = SIZEOF_NONRESIDENT_EX_LE; + attr->flags = ATTR_FLAG_SPARSED; + asize = SIZEOF_NONRESIDENT_EX + 8; + } else if (fa & FILE_ATTRIBUTE_COMPRESSED) { + attr->size = cpu_to_le32(SIZEOF_NONRESIDENT_EX + 8); + attr->name_off = SIZEOF_NONRESIDENT_EX_LE; + attr->flags = ATTR_FLAG_COMPRESSED; + attr->nres.c_unit = NTFS_LZNT_CUNIT; + asize = SIZEOF_NONRESIDENT_EX + 8; + } else { + attr->size = cpu_to_le32(SIZEOF_NONRESIDENT + 8); + attr->name_off = SIZEOF_NONRESIDENT_LE; + asize = SIZEOF_NONRESIDENT + 8; + } + attr->nres.run_off = attr->name_off; + } else { + /* + * Node. Create empty resident data attribute. + */ + attr->type = ATTR_DATA; + attr->size = cpu_to_le32(SIZEOF_RESIDENT); + attr->name_off = SIZEOF_RESIDENT_LE; + if (fa & FILE_ATTRIBUTE_SPARSE_FILE) + attr->flags = ATTR_FLAG_SPARSED; + else if (fa & FILE_ATTRIBUTE_COMPRESSED) + attr->flags = ATTR_FLAG_COMPRESSED; + attr->res.data_off = SIZEOF_RESIDENT_LE; + asize = SIZEOF_RESIDENT; + ni->ni_flags |= NI_FLAG_RESIDENT; + } + + if (S_ISDIR(mode)) { + ni->ni_flags |= NI_FLAG_DIR; + err = indx_init(&ni->dir, sbi, attr, INDEX_MUTEX_I30); + if (err) + goto out4; + } else if (S_ISLNK(mode)) { + rp = ntfs_create_reparse_buffer(sbi, symname, size, &nsize); + + if (IS_ERR(rp)) { + err = PTR_ERR(rp); + rp = NULL; + goto out4; + } + + /* + * Insert ATTR_REPARSE. + */ + attr = Add2Ptr(attr, asize); + attr->type = ATTR_REPARSE; + attr->id = cpu_to_le16(aid++); + + /* Resident or non resident? */ + asize = ALIGN(SIZEOF_RESIDENT + nsize, 8); + t16 = PtrOffset(rec, attr); + + /* + * Below function 'ntfs_save_wsl_perm' requires 0x78 bytes. + * It is good idea to keep extended attributes resident. + */ + if (asize + t16 + 0x78 + 8 > sbi->record_size) { + CLST alen; + CLST clst = bytes_to_cluster(sbi, nsize); + + /* Bytes per runs. */ + t16 = sbi->record_size - t16 - SIZEOF_NONRESIDENT; + + attr->non_res = 1; + attr->nres.evcn = cpu_to_le64(clst - 1); + attr->name_off = SIZEOF_NONRESIDENT_LE; + attr->nres.run_off = attr->name_off; + attr->nres.data_size = cpu_to_le64(nsize); + attr->nres.valid_size = attr->nres.data_size; + attr->nres.alloc_size = + cpu_to_le64(ntfs_up_cluster(sbi, nsize)); + + err = attr_allocate_clusters(sbi, &ni->file.run, NULL, + 0, 0, clst, NULL, + ALLOCATE_DEF, &alen, 0, + NULL, NULL); + if (err) + goto out5; + + err = run_pack(&ni->file.run, 0, clst, + Add2Ptr(attr, SIZEOF_NONRESIDENT), t16, + &vcn); + if (err < 0) + goto out5; + + if (vcn != clst) { + err = -EINVAL; + goto out5; + } + + asize = SIZEOF_NONRESIDENT + ALIGN(err, 8); + /* Write non resident data. */ + err = ntfs_sb_write_run(sbi, &ni->file.run, 0, rp, + nsize, 0); + if (err) + goto out5; + } else { + attr->res.data_off = SIZEOF_RESIDENT_LE; + attr->res.data_size = cpu_to_le32(nsize); + memcpy(Add2Ptr(attr, SIZEOF_RESIDENT), rp, nsize); + } + /* Size of symlink equals the length of input string. */ + inode->i_size = size; + + attr->size = cpu_to_le32(asize); + + err = ntfs_insert_reparse(sbi, IO_REPARSE_TAG_SYMLINK, + &new_de->ref); + if (err) + goto out5; + + rp_inserted = true; + } + + attr = Add2Ptr(attr, asize); + attr->type = ATTR_END; + + rec->used = cpu_to_le32(PtrOffset(rec, attr) + 8); + rec->next_attr_id = cpu_to_le16(aid); + + inode->i_generation = le16_to_cpu(rec->seq); + + if (S_ISDIR(mode)) { + inode->i_op = &ntfs_dir_inode_operations; + inode->i_fop = &ntfs_dir_operations; + } else if (S_ISLNK(mode)) { + inode->i_op = &ntfs_link_inode_operations; + inode->i_fop = NULL; + inode->i_mapping->a_ops = &ntfs_aops; + inode->i_size = size; + inode_nohighmem(inode); + } else if (S_ISREG(mode)) { + inode->i_op = &ntfs_file_inode_operations; + inode->i_fop = &ntfs_file_operations; + inode->i_mapping->a_ops = is_compressed(ni) ? &ntfs_aops_cmpr : + &ntfs_aops; + init_rwsem(&ni->file.run_lock); + } else { + inode->i_op = &ntfs_special_inode_operations; + init_special_inode(inode, mode, dev); + } + +#ifdef CONFIG_NTFS3_FS_POSIX_ACL + if (!S_ISLNK(mode) && (sb->s_flags & SB_POSIXACL)) { + err = ntfs_init_acl(idmap, inode, dir); + if (err) + goto out5; + } else +#endif + { + inode->i_flags |= S_NOSEC; + } + + if (!S_ISLNK(mode)) { + /* + * ntfs_init_acl and ntfs_save_wsl_perm update extended attribute. + * The packed size of extended attribute is stored in direntry too. + * 'fname' here points to inside new_de. + */ + err = ntfs_save_wsl_perm(inode, &fname->dup.extend_data); + if (err) + goto out6; + + /* + * update ea_size in file_name attribute too. + * Use ni_find_attr cause layout of MFT record may be changed + * in ntfs_init_acl and ntfs_save_wsl_perm. + */ + attr = ni_find_attr(ni, NULL, NULL, ATTR_NAME, NULL, 0, NULL, + NULL); + if (attr) { + struct ATTR_FILE_NAME *fn; + + fn = resident_data_ex(attr, SIZEOF_ATTRIBUTE_FILENAME); + if (fn) + fn->dup.extend_data = fname->dup.extend_data; + } + } + + /* We do not need to update parent directory later */ + ni->ni_flags &= ~NI_FLAG_UPDATE_PARENT; + + /* Step 2: Add new name in index. */ + err = indx_insert_entry(&dir_ni->dir, dir_ni, new_de, sbi, fnd, 0); + if (err) + goto out6; + + /* + * Call 'd_instantiate' after inode->i_op is set + * but before finish_open. + */ + d_instantiate(dentry, inode); + + /* Set original time. inode times (i_ctime) may be changed in ntfs_init_acl. */ + inode_set_atime_to_ts(inode, ni->i_crtime); + inode_set_ctime_to_ts(inode, ni->i_crtime); + inode_set_mtime_to_ts(inode, ni->i_crtime); + inode_set_mtime_to_ts(dir, ni->i_crtime); + inode_set_ctime_to_ts(dir, ni->i_crtime); + + mark_inode_dirty(dir); + mark_inode_dirty(inode); + + /* Normal exit. */ + goto out2; + +out6: + attr = ni_find_attr(ni, NULL, NULL, ATTR_EA, NULL, 0, NULL, NULL); + if (attr && attr->non_res) { + /* Delete ATTR_EA, if non-resident. */ + struct runs_tree run; + run_init(&run); + attr_set_size(ni, ATTR_EA, NULL, 0, &run, 0, NULL, false); + run_close(&run); + } + + if (rp_inserted) + ntfs_remove_reparse(sbi, IO_REPARSE_TAG_SYMLINK, &new_de->ref); + +out5: + if (!S_ISDIR(mode)) + run_deallocate(sbi, &ni->file.run, false); + +out4: + clear_rec_inuse(rec); + clear_nlink(inode); + ni->mi.dirty = false; + discard_new_inode(inode); +out3: + ntfs_mark_rec_free(sbi, ino, false); + +out2: + kfree(new_de); + kfree(rp); + +out1: + if (!fnd) + ni_unlock(dir_ni); + + if (!err) + unlock_new_inode(inode); + + return err; +} + +int ntfs_link_inode(struct inode *inode, struct dentry *dentry) +{ + int err; + struct ntfs_inode *ni = ntfs_i(inode); + struct ntfs_sb_info *sbi = inode->i_sb->s_fs_info; + struct NTFS_DE *de; + + de = kzalloc(PATH_MAX, GFP_KERNEL); + if (!de) + return -ENOMEM; + + /* Mark rw ntfs as dirty. It will be cleared at umount. */ + ntfs_set_state(sbi, NTFS_DIRTY_DIRTY); + + /* Construct 'de'. */ + err = fill_name_de(sbi, de, &dentry->d_name, NULL); + if (err) + goto out; + + err = ni_add_name(ntfs_i(d_inode(dentry->d_parent)), ni, de); +out: + kfree(de); + return err; +} + +/* + * ntfs_unlink_inode + * + * inode_operations::unlink + * inode_operations::rmdir + */ +int ntfs_unlink_inode(struct inode *dir, const struct dentry *dentry) +{ + int err; + struct ntfs_sb_info *sbi = dir->i_sb->s_fs_info; + struct inode *inode = d_inode(dentry); + struct ntfs_inode *ni = ntfs_i(inode); + struct ntfs_inode *dir_ni = ntfs_i(dir); + struct NTFS_DE *de, *de2 = NULL; + int undo_remove; + + if (ntfs_is_meta_file(sbi, ni->mi.rno)) + return -EINVAL; + + de = kzalloc(PATH_MAX, GFP_KERNEL); + if (!de) + return -ENOMEM; + + ni_lock(ni); + + if (S_ISDIR(inode->i_mode) && !dir_is_empty(inode)) { + err = -ENOTEMPTY; + goto out; + } + + err = fill_name_de(sbi, de, &dentry->d_name, NULL); + if (err < 0) + goto out; + + undo_remove = 0; + err = ni_remove_name(dir_ni, ni, de, &de2, &undo_remove); + + if (!err) { + drop_nlink(inode); + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + mark_inode_dirty(dir); + inode_set_ctime_to_ts(inode, inode_get_ctime(dir)); + if (inode->i_nlink) + mark_inode_dirty(inode); + } else if (!ni_remove_name_undo(dir_ni, ni, de, de2, undo_remove)) { + _ntfs_bad_inode(inode); + } else { + if (ni_is_dirty(dir)) + mark_inode_dirty(dir); + if (ni_is_dirty(inode)) + mark_inode_dirty(inode); + } + +out: + ni_unlock(ni); + kfree(de); + return err; +} + +void ntfs_evict_inode(struct inode *inode) +{ + truncate_inode_pages_final(&inode->i_data); + + clear_inode(inode); + + ni_clear(ntfs_i(inode)); +} + +/* + * ntfs_translate_junction + * + * Translate a Windows junction target to the Linux equivalent. + * On junctions, targets are always absolute (they include the drive + * letter). We have no way of knowing if the target is for the current + * mounted device or not so we just assume it is. + */ +static int ntfs_translate_junction(const struct super_block *sb, + const struct dentry *link_de, char *target, + int target_len, int target_max) +{ + int tl_len, err = target_len; + char *link_path_buffer = NULL, *link_path; + char *translated = NULL; + char *target_start; + int copy_len; + + link_path_buffer = kmalloc(PATH_MAX, GFP_NOFS); + if (!link_path_buffer) { + err = -ENOMEM; + goto out; + } + /* Get link path, relative to mount point */ + link_path = dentry_path_raw(link_de, link_path_buffer, PATH_MAX); + if (IS_ERR(link_path)) { + ntfs_err(sb, "Error getting link path"); + err = -EINVAL; + goto out; + } + + translated = kmalloc(PATH_MAX, GFP_NOFS); + if (!translated) { + err = -ENOMEM; + goto out; + } + + /* Make translated path a relative path to mount point */ + strcpy(translated, "./"); + ++link_path; /* Skip leading / */ + for (tl_len = sizeof("./") - 1; *link_path; ++link_path) { + if (*link_path == '/') { + if (PATH_MAX - tl_len < sizeof("../")) { + ntfs_err(sb, + "Link path %s has too many components", + link_path); + err = -EINVAL; + goto out; + } + strcpy(translated + tl_len, "../"); + tl_len += sizeof("../") - 1; + } + } + + /* Skip drive letter */ + target_start = target; + while (*target_start && *target_start != ':') + ++target_start; + + if (!*target_start) { + ntfs_err(sb, "Link target (%s) missing drive separator", + target); + err = -EINVAL; + goto out; + } + + /* Skip drive separator and leading /, if exists */ + target_start += 1 + (target_start[1] == '/'); + copy_len = target_len - (target_start - target); + + if (PATH_MAX - tl_len <= copy_len) { + ntfs_err(sb, "Link target %s too large for buffer (%d <= %d)", + target_start, PATH_MAX - tl_len, copy_len); + err = -EINVAL; + goto out; + } + + /* translated path has a trailing / and target_start does not */ + strcpy(translated + tl_len, target_start); + tl_len += copy_len; + if (target_max <= tl_len) { + ntfs_err(sb, "Target path %s too large for buffer (%d <= %d)", + translated, target_max, tl_len); + err = -EINVAL; + goto out; + } + strcpy(target, translated); + err = tl_len; + +out: + kfree(link_path_buffer); + kfree(translated); + return err; +} + +static noinline int ntfs_readlink_hlp(const struct dentry *link_de, + struct inode *inode, char *buffer, + int buflen) +{ + int i, err = -EINVAL; + struct ntfs_inode *ni = ntfs_i(inode); + struct super_block *sb = inode->i_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + u64 size; + u16 ulen = 0; + void *to_free = NULL; + struct REPARSE_DATA_BUFFER *rp; + const __le16 *uname; + struct ATTRIB *attr; + + /* Reparse data present. Try to parse it. */ + static_assert(!offsetof(struct REPARSE_DATA_BUFFER, ReparseTag)); + static_assert(sizeof(u32) == sizeof(rp->ReparseTag)); + + *buffer = 0; + + attr = ni_find_attr(ni, NULL, NULL, ATTR_REPARSE, NULL, 0, NULL, NULL); + if (!attr) + goto out; + + if (!attr->non_res) { + rp = resident_data_ex(attr, sizeof(struct REPARSE_DATA_BUFFER)); + if (!rp) + goto out; + size = le32_to_cpu(attr->res.data_size); + } else { + size = le64_to_cpu(attr->nres.data_size); + rp = NULL; + } + + if (size > sbi->reparse.max_size || size <= sizeof(u32)) + goto out; + + if (!rp) { + rp = kmalloc(size, GFP_NOFS); + if (!rp) { + err = -ENOMEM; + goto out; + } + to_free = rp; + /* Read into temporal buffer. */ + err = ntfs_read_run_nb(sbi, &ni->file.run, 0, rp, size, NULL); + if (err) + goto out; + } + + /* Microsoft Tag. */ + switch (rp->ReparseTag) { + case IO_REPARSE_TAG_MOUNT_POINT: + /* Mount points and junctions. */ + /* Can we use 'Rp->MountPointReparseBuffer.PrintNameLength'? */ + if (size <= offsetof(struct REPARSE_DATA_BUFFER, + MountPointReparseBuffer.PathBuffer)) + goto out; + uname = Add2Ptr(rp, + offsetof(struct REPARSE_DATA_BUFFER, + MountPointReparseBuffer.PathBuffer) + + le16_to_cpu(rp->MountPointReparseBuffer + .PrintNameOffset)); + ulen = le16_to_cpu(rp->MountPointReparseBuffer.PrintNameLength); + break; + + case IO_REPARSE_TAG_SYMLINK: + /* FolderSymbolicLink */ + /* Can we use 'Rp->SymbolicLinkReparseBuffer.PrintNameLength'? */ + if (size <= offsetof(struct REPARSE_DATA_BUFFER, + SymbolicLinkReparseBuffer.PathBuffer)) + goto out; + uname = Add2Ptr( + rp, offsetof(struct REPARSE_DATA_BUFFER, + SymbolicLinkReparseBuffer.PathBuffer) + + le16_to_cpu(rp->SymbolicLinkReparseBuffer + .PrintNameOffset)); + ulen = le16_to_cpu( + rp->SymbolicLinkReparseBuffer.PrintNameLength); + break; + + case IO_REPARSE_TAG_CLOUD: + case IO_REPARSE_TAG_CLOUD_1: + case IO_REPARSE_TAG_CLOUD_2: + case IO_REPARSE_TAG_CLOUD_3: + case IO_REPARSE_TAG_CLOUD_4: + case IO_REPARSE_TAG_CLOUD_5: + case IO_REPARSE_TAG_CLOUD_6: + case IO_REPARSE_TAG_CLOUD_7: + case IO_REPARSE_TAG_CLOUD_8: + case IO_REPARSE_TAG_CLOUD_9: + case IO_REPARSE_TAG_CLOUD_A: + case IO_REPARSE_TAG_CLOUD_B: + case IO_REPARSE_TAG_CLOUD_C: + case IO_REPARSE_TAG_CLOUD_D: + case IO_REPARSE_TAG_CLOUD_E: + case IO_REPARSE_TAG_CLOUD_F: + err = sizeof("OneDrive") - 1; + if (err > buflen) + err = buflen; + memcpy(buffer, "OneDrive", err); + goto out; + + default: + if (IsReparseTagMicrosoft(rp->ReparseTag)) { + /* Unknown Microsoft Tag. */ + goto out; + } + if (!IsReparseTagNameSurrogate(rp->ReparseTag) || + size <= sizeof(struct REPARSE_POINT)) { + goto out; + } + + /* Users tag. */ + uname = Add2Ptr(rp, sizeof(struct REPARSE_POINT)); + ulen = le16_to_cpu(rp->ReparseDataLength) - + sizeof(struct REPARSE_POINT); + } + + /* Convert nlen from bytes to UNICODE chars. */ + ulen >>= 1; + + /* Check that name is available. */ + if (!ulen || uname + ulen > (__le16 *)Add2Ptr(rp, size)) + goto out; + + /* If name is already zero terminated then truncate it now. */ + if (!uname[ulen - 1]) + ulen -= 1; + + err = ntfs_utf16_to_nls(sbi, uname, ulen, buffer, buflen); + + if (err < 0) + goto out; + + /* Translate Windows '\' into Linux '/'. */ + for (i = 0; i < err; i++) { + if (buffer[i] == '\\') + buffer[i] = '/'; + } + + /* Always set last zero. */ + buffer[err] = 0; + + /* If this is a junction, translate the link target. */ + if (rp->ReparseTag == IO_REPARSE_TAG_MOUNT_POINT) + err = ntfs_translate_junction(sb, link_de, buffer, err, buflen); + +out: + kfree(to_free); + return err; +} + +static const char *ntfs_get_link(struct dentry *de, struct inode *inode, + struct delayed_call *done) +{ + int err; + char *ret; + + if (!de) + return ERR_PTR(-ECHILD); + + ret = kmalloc(PAGE_SIZE, GFP_NOFS); + if (!ret) + return ERR_PTR(-ENOMEM); + + err = ntfs_readlink_hlp(de, inode, ret, PAGE_SIZE); + if (err < 0) { + kfree(ret); + return ERR_PTR(err); + } + + set_delayed_call(done, kfree_link, ret); + + return ret; +} + +// clang-format off +const struct inode_operations ntfs_link_inode_operations = { + .get_link = ntfs_get_link, + .setattr = ntfs_setattr, + .listxattr = ntfs_listxattr, + .fileattr_get = ntfs_fileattr_get, + .fileattr_set = ntfs_fileattr_set, +}; + +const struct address_space_operations ntfs_aops = { + .read_folio = ntfs_read_folio, + .readahead = ntfs_readahead, + .writepages = ntfs_writepages, + .bmap = ntfs_bmap, + .dirty_folio = iomap_dirty_folio, + .migrate_folio = filemap_migrate_folio, + .release_folio = iomap_release_folio, + .invalidate_folio = iomap_invalidate_folio, +}; + +const struct address_space_operations ntfs_aops_cmpr = { + .read_folio = ntfs_read_folio, + .dirty_folio = iomap_dirty_folio, + .release_folio = iomap_release_folio, + .invalidate_folio = iomap_invalidate_folio, +}; + +static DEFINE_IOMAP_ITER_NEXT_END(ntfs_iomap_next, ntfs_iomap_begin, + ntfs_iomap_end); + +const struct iomap_ops ntfs_iomap_ops = { + .iomap_next = ntfs_iomap_next, +}; + +const struct iomap_write_ops ntfs_iomap_folio_ops = { + .put_folio = ntfs_iomap_put_folio, +}; +// clang-format on diff --git a/fs/ntfs3/lib/decompress_common.c b/fs/ntfs3/lib/decompress_common.c new file mode 100644 index 000000000..e96652240 --- /dev/null +++ b/fs/ntfs3/lib/decompress_common.c @@ -0,0 +1,319 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * decompress_common.c - Code shared by the XPRESS and LZX decompressors + * + * Copyright (C) 2015 Eric Biggers + */ + +#include "decompress_common.h" + +/* + * make_huffman_decode_table() - + * + * Build a decoding table for a canonical prefix code, or "Huffman code". + * + * This is an internal function, not part of the library API! + * + * This takes as input the length of the codeword for each symbol in the + * alphabet and produces as output a table that can be used for fast + * decoding of prefix-encoded symbols using read_huffsym(). + * + * Strictly speaking, a canonical prefix code might not be a Huffman + * code. But this algorithm will work either way; and in fact, since + * Huffman codes are defined in terms of symbol frequencies, there is no + * way for the decompressor to know whether the code is a true Huffman + * code or not until all symbols have been decoded. + * + * Because the prefix code is assumed to be "canonical", it can be + * reconstructed directly from the codeword lengths. A prefix code is + * canonical if and only if a longer codeword never lexicographically + * precedes a shorter codeword, and the lexicographic ordering of + * codewords of the same length is the same as the lexicographic ordering + * of the corresponding symbols. Consequently, we can sort the symbols + * primarily by codeword length and secondarily by symbol value, then + * reconstruct the prefix code by generating codewords lexicographically + * in that order. + * + * This function does not, however, generate the prefix code explicitly. + * Instead, it directly builds a table for decoding symbols using the + * code. The basic idea is this: given the next 'max_codeword_len' bits + * in the input, we can look up the decoded symbol by indexing a table + * containing 2**max_codeword_len entries. A codeword with length + * 'max_codeword_len' will have exactly one entry in this table, whereas + * a codeword shorter than 'max_codeword_len' will have multiple entries + * in this table. Precisely, a codeword of length n will be represented + * by 2**(max_codeword_len - n) entries in this table. The 0-based index + * of each such entry will contain the corresponding codeword as a prefix + * when zero-padded on the left to 'max_codeword_len' binary digits. + * + * That's the basic idea, but we implement two optimizations regarding + * the format of the decode table itself: + * + * - For many compression formats, the maximum codeword length is too + * long for it to be efficient to build the full decoding table + * whenever a new prefix code is used. Instead, we can build the table + * using only 2**table_bits entries, where 'table_bits' is some number + * less than or equal to 'max_codeword_len'. Then, only codewords of + * length 'table_bits' and shorter can be directly looked up. For + * longer codewords, the direct lookup instead produces the root of a + * binary tree. Using this tree, the decoder can do traditional + * bit-by-bit decoding of the remainder of the codeword. Child nodes + * are allocated in extra entries at the end of the table; leaf nodes + * contain symbols. Note that the long-codeword case is, in general, + * not performance critical, since in Huffman codes the most frequently + * used symbols are assigned the shortest codeword lengths. + * + * - When we decode a symbol using a direct lookup of the table, we still + * need to know its length so that the bitstream can be advanced by the + * appropriate number of bits. The simple solution is to simply retain + * the 'lens' array and use the decoded symbol as an index into it. + * However, this requires two separate array accesses in the fast path. + * The optimization is to store the length directly in the decode + * table. We use the bottom 11 bits for the symbol and the top 5 bits + * for the length. In addition, to combine this optimization with the + * previous one, we introduce a special case where the top 2 bits of + * the length are both set if the entry is actually the root of a + * binary tree. + * + * @decode_table: + * The array in which to create the decoding table. This must have + * a length of at least ((2**table_bits) + 2 * num_syms) entries. + * + * @num_syms: + * The number of symbols in the alphabet; also, the length of the + * 'lens' array. Must be less than or equal to 2048. + * + * @table_bits: + * The order of the decode table size, as explained above. Must be + * less than or equal to 13. + * + * @lens: + * An array of length @num_syms, indexable by symbol, that gives the + * length of the codeword, in bits, for that symbol. The length can + * be 0, which means that the symbol does not have a codeword + * assigned. + * + * @max_codeword_len: + * The longest codeword length allowed in the compression format. + * All entries in 'lens' must be less than or equal to this value. + * This must be less than or equal to 23. + * + * @working_space + * A temporary array of length '2 * (max_codeword_len + 1) + + * num_syms'. + * + * Returns 0 on success, or -1 if the lengths do not form a valid prefix + * code. + */ +int make_huffman_decode_table(u16 decode_table[], const u32 num_syms, + const u32 table_bits, const u8 lens[], + const u32 max_codeword_len, + u16 working_space[]) +{ + const u32 table_num_entries = 1 << table_bits; + u16 * const len_counts = &working_space[0]; + u16 * const offsets = &working_space[1 * (max_codeword_len + 1)]; + u16 * const sorted_syms = &working_space[2 * (max_codeword_len + 1)]; + int left; + void *decode_table_ptr; + u32 sym_idx; + u32 codeword_len; + u32 stores_per_loop; + u32 decode_table_pos; + u32 len; + u32 sym; + + /* Count how many symbols have each possible codeword length. + * Note that a length of 0 indicates the corresponding symbol is not + * used in the code and therefore does not have a codeword. + */ + for (len = 0; len <= max_codeword_len; len++) + len_counts[len] = 0; + for (sym = 0; sym < num_syms; sym++) + len_counts[lens[sym]]++; + + /* We can assume all lengths are <= max_codeword_len, but we + * cannot assume they form a valid prefix code. A codeword of + * length n should require a proportion of the codespace equaling + * (1/2)^n. The code is valid if and only if the codespace is + * exactly filled by the lengths, by this measure. + */ + left = 1; + for (len = 1; len <= max_codeword_len; len++) { + left <<= 1; + left -= len_counts[len]; + if (left < 0) { + /* The lengths overflow the codespace; that is, the code + * is over-subscribed. + */ + return -1; + } + } + + if (left) { + /* The lengths do not fill the codespace; that is, they form an + * incomplete set. + */ + if (left == (1 << max_codeword_len)) { + /* The code is completely empty. This is arguably + * invalid, but in fact it is valid in LZX and XPRESS, + * so we must allow it. By definition, no symbols can + * be decoded with an empty code. Consequently, we + * technically don't even need to fill in the decode + * table. However, to avoid accessing uninitialized + * memory if the algorithm nevertheless attempts to + * decode symbols using such a code, we zero out the + * decode table. + */ + memset(decode_table, 0, + table_num_entries * sizeof(decode_table[0])); + return 0; + } + return -1; + } + + /* Sort the symbols primarily by length and secondarily by symbol order. + */ + + /* Initialize 'offsets' so that offsets[len] for 1 <= len <= + * max_codeword_len is the number of codewords shorter than 'len' bits. + */ + offsets[1] = 0; + for (len = 1; len < max_codeword_len; len++) + offsets[len + 1] = offsets[len] + len_counts[len]; + + /* Use the 'offsets' array to sort the symbols. Note that we do not + * include symbols that are not used in the code. Consequently, fewer + * than 'num_syms' entries in 'sorted_syms' may be filled. + */ + for (sym = 0; sym < num_syms; sym++) + if (lens[sym]) + sorted_syms[offsets[lens[sym]]++] = sym; + + /* Fill entries for codewords with length <= table_bits + * --- that is, those short enough for a direct mapping. + * + * The table will start with entries for the shortest codeword(s), which + * have the most entries. From there, the number of entries per + * codeword will decrease. + */ + decode_table_ptr = decode_table; + sym_idx = 0; + codeword_len = 1; + stores_per_loop = (1 << (table_bits - codeword_len)); + for (; stores_per_loop != 0; codeword_len++, stores_per_loop >>= 1) { + u32 end_sym_idx = sym_idx + len_counts[codeword_len]; + + for (; sym_idx < end_sym_idx; sym_idx++) { + u16 entry; + u16 *p; + u32 n; + + entry = ((u32)codeword_len << 11) | sorted_syms[sym_idx]; + p = (u16 *)decode_table_ptr; + n = stores_per_loop; + + do { + *p++ = entry; + } while (--n); + + decode_table_ptr = p; + } + } + + /* If we've filled in the entire table, we are done. Otherwise, + * there are codewords longer than table_bits for which we must + * generate binary trees. + */ + decode_table_pos = (u16 *)decode_table_ptr - decode_table; + if (decode_table_pos != table_num_entries) { + u32 j; + u32 next_free_tree_slot; + u32 cur_codeword; + + /* First, zero out the remaining entries. This is + * necessary so that these entries appear as + * "unallocated" in the next part. Each of these entries + * will eventually be filled with the representation of + * the root node of a binary tree. + */ + j = decode_table_pos; + do { + decode_table[j] = 0; + } while (++j != table_num_entries); + + /* We allocate child nodes starting at the end of the + * direct lookup table. Note that there should be + * 2*num_syms extra entries for this purpose, although + * fewer than this may actually be needed. + */ + next_free_tree_slot = table_num_entries; + + /* Iterate through each codeword with length greater than + * 'table_bits', primarily in order of codeword length + * and secondarily in order of symbol. + */ + for (cur_codeword = decode_table_pos << 1; + codeword_len <= max_codeword_len; + codeword_len++, cur_codeword <<= 1) { + u32 end_sym_idx = sym_idx + len_counts[codeword_len]; + + for (; sym_idx < end_sym_idx; sym_idx++, cur_codeword++) { + /* 'sorted_sym' is the symbol represented by the + * codeword. + */ + u32 sorted_sym = sorted_syms[sym_idx]; + u32 extra_bits = codeword_len - table_bits; + u32 node_idx = cur_codeword >> extra_bits; + + /* Go through each bit of the current codeword + * beyond the prefix of length @table_bits and + * walk the appropriate binary tree, allocating + * any slots that have not yet been allocated. + * + * Note that the 'pointer' entry to the binary + * tree, which is stored in the direct lookup + * portion of the table, is represented + * identically to other internal (non-leaf) + * nodes of the binary tree; it can be thought + * of as simply the root of the tree. The + * representation of these internal nodes is + * simply the index of the left child combined + * with the special bits 0xC000 to distinguish + * the entry from direct mapping and leaf node + * entries. + */ + do { + /* At least one bit remains in the + * codeword, but the current node is an + * unallocated leaf. Change it to an + * internal node. + */ + if (decode_table[node_idx] == 0) { + decode_table[node_idx] = + next_free_tree_slot | 0xC000; + decode_table[next_free_tree_slot++] = 0; + decode_table[next_free_tree_slot++] = 0; + } + + /* Go to the left child if the next bit + * in the codeword is 0; otherwise go to + * the right child. + */ + node_idx = decode_table[node_idx] & 0x3FFF; + --extra_bits; + node_idx += (cur_codeword >> extra_bits) & 1; + } while (extra_bits != 0); + + /* We've traversed the tree using the entire + * codeword, and we're now at the entry where + * the actual symbol will be stored. This is + * distinguished from internal nodes by not + * having its high two bits set. + */ + decode_table[node_idx] = sorted_sym; + } + } + } + return 0; +} diff --git a/fs/ntfs3/lib/decompress_common.h b/fs/ntfs3/lib/decompress_common.h new file mode 100644 index 000000000..f0cad9c4a --- /dev/null +++ b/fs/ntfs3/lib/decompress_common.h @@ -0,0 +1,343 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * decompress_common.h - Code shared by the XPRESS and LZX decompressors + * + * Copyright (C) 2015 Eric Biggers + */ + +#ifndef _LINUX_NTFS3_LIB_DECOMPRESS_COMMON_H +#define _LINUX_NTFS3_LIB_DECOMPRESS_COMMON_H + +#include +#include +#include +#include +#include + + +/* "Force inline" macro (not required, but helpful for performance) */ +#define forceinline __always_inline + +/* Enable whole-word match copying on selected architectures */ +#if defined(__i386__) || defined(__x86_64__) || defined(__ARM_FEATURE_UNALIGNED) +# define FAST_UNALIGNED_ACCESS +#endif + +/* Size of a machine word */ +#define WORDBYTES (sizeof(size_t)) + +static forceinline void +copy_unaligned_word(const void *src, void *dst) +{ + put_unaligned(get_unaligned((const size_t *)src), (size_t *)dst); +} + + +/* Generate a "word" with platform-dependent size whose bytes all contain the + * value 'b'. + */ +static forceinline size_t repeat_byte(u8 b) +{ + size_t v; + + v = b; + v |= v << 8; + v |= v << 16; + v |= v << ((WORDBYTES == 8) ? 32 : 0); + return v; +} + +/* Structure that encapsulates a block of in-memory data being interpreted as a + * stream of bits, optionally with interwoven literal bytes. Bits are assumed + * to be stored in little endian 16-bit coding units, with the bits ordered high + * to low. + */ +struct input_bitstream { + + /* Bits that have been read from the input buffer. The bits are + * left-justified; the next bit is always bit 31. + */ + u32 bitbuf; + + /* Number of bits currently held in @bitbuf. */ + u32 bitsleft; + + /* Pointer to the next byte to be retrieved from the input buffer. */ + const u8 *next; + + /* Pointer to just past the end of the input buffer. */ + const u8 *end; +}; + +/* Initialize a bitstream to read from the specified input buffer. */ +static forceinline void init_input_bitstream(struct input_bitstream *is, + const void *buffer, u32 size) +{ + is->bitbuf = 0; + is->bitsleft = 0; + is->next = buffer; + is->end = is->next + size; +} + +/* Ensure the bit buffer variable for the bitstream contains at least @num_bits + * bits. Following this, bitstream_peek_bits() and/or bitstream_remove_bits() + * may be called on the bitstream to peek or remove up to @num_bits bits. Note + * that @num_bits must be <= 16. + */ +static forceinline void bitstream_ensure_bits(struct input_bitstream *is, + u32 num_bits) +{ + if (is->bitsleft < num_bits) { + if (is->end - is->next >= 2) { + is->bitbuf |= (u32)get_unaligned_le16(is->next) + << (16 - is->bitsleft); + is->next += 2; + } + is->bitsleft += 16; + } +} + +/* Return the next @num_bits bits from the bitstream, without removing them. + * There must be at least @num_bits remaining in the buffer variable, from a + * previous call to bitstream_ensure_bits(). + */ +static forceinline u32 +bitstream_peek_bits(const struct input_bitstream *is, const u32 num_bits) +{ + return (is->bitbuf >> 1) >> (sizeof(is->bitbuf) * 8 - num_bits - 1); +} + +/* Remove @num_bits from the bitstream. There must be at least @num_bits + * remaining in the buffer variable, from a previous call to + * bitstream_ensure_bits(). + */ +static forceinline void +bitstream_remove_bits(struct input_bitstream *is, u32 num_bits) +{ + is->bitbuf <<= num_bits; + is->bitsleft -= num_bits; +} + +/* Remove and return @num_bits bits from the bitstream. There must be at least + * @num_bits remaining in the buffer variable, from a previous call to + * bitstream_ensure_bits(). + */ +static forceinline u32 +bitstream_pop_bits(struct input_bitstream *is, u32 num_bits) +{ + u32 bits = bitstream_peek_bits(is, num_bits); + + bitstream_remove_bits(is, num_bits); + return bits; +} + +/* Read and return the next @num_bits bits from the bitstream. */ +static forceinline u32 +bitstream_read_bits(struct input_bitstream *is, u32 num_bits) +{ + bitstream_ensure_bits(is, num_bits); + return bitstream_pop_bits(is, num_bits); +} + +/* Read and return the next literal byte embedded in the bitstream. */ +static forceinline u8 +bitstream_read_byte(struct input_bitstream *is) +{ + if (unlikely(is->end == is->next)) + return 0; + return *is->next++; +} + +/* Read and return the next 16-bit integer embedded in the bitstream. */ +static forceinline u16 +bitstream_read_u16(struct input_bitstream *is) +{ + u16 v; + + if (unlikely(is->end - is->next < 2)) + return 0; + v = get_unaligned_le16(is->next); + is->next += 2; + return v; +} + +/* Read and return the next 32-bit integer embedded in the bitstream. */ +static forceinline u32 +bitstream_read_u32(struct input_bitstream *is) +{ + u32 v; + + if (unlikely(is->end - is->next < 4)) + return 0; + v = get_unaligned_le32(is->next); + is->next += 4; + return v; +} + +/* Read into @dst_buffer an array of literal bytes embedded in the bitstream. + * Return either a pointer to the byte past the last written, or NULL if the + * read overflows the input buffer. + */ +static forceinline void *bitstream_read_bytes(struct input_bitstream *is, + void *dst_buffer, size_t count) +{ + if ((size_t)(is->end - is->next) < count) + return NULL; + memcpy(dst_buffer, is->next, count); + is->next += count; + return (u8 *)dst_buffer + count; +} + +/* Align the input bitstream on a coding-unit boundary. */ +static forceinline void bitstream_align(struct input_bitstream *is) +{ + is->bitsleft = 0; + is->bitbuf = 0; +} + +extern int make_huffman_decode_table(u16 decode_table[], const u32 num_syms, + const u32 num_bits, const u8 lens[], + const u32 max_codeword_len, + u16 working_space[]); + + +/* Reads and returns the next Huffman-encoded symbol from a bitstream. If the + * input data is exhausted, the Huffman symbol is decoded as if the missing bits + * are all zeroes. + */ +static forceinline u32 read_huffsym(struct input_bitstream *istream, + const u16 decode_table[], + u32 table_bits, + u32 max_codeword_len) +{ + u32 entry; + u32 key_bits; + + bitstream_ensure_bits(istream, max_codeword_len); + + /* Index the decode table by the next table_bits bits of the input. */ + key_bits = bitstream_peek_bits(istream, table_bits); + entry = decode_table[key_bits]; + if (entry < 0xC000) { + /* Fast case: The decode table directly provided the + * symbol and codeword length. The low 11 bits are the + * symbol, and the high 5 bits are the codeword length. + */ + bitstream_remove_bits(istream, entry >> 11); + return entry & 0x7FF; + } + /* Slow case: The codeword for the symbol is longer than + * table_bits, so the symbol does not have an entry + * directly in the first (1 << table_bits) entries of the + * decode table. Traverse the appropriate binary tree + * bit-by-bit to decode the symbol. + */ + bitstream_remove_bits(istream, table_bits); + do { + key_bits = (entry & 0x3FFF) + bitstream_pop_bits(istream, 1); + } while ((entry = decode_table[key_bits]) >= 0xC000); + return entry; +} + +/* + * Copy an LZ77 match at (dst - offset) to dst. + * + * The length and offset must be already validated --- that is, (dst - offset) + * can't underrun the output buffer, and (dst + length) can't overrun the output + * buffer. Also, the length cannot be 0. + * + * @bufend points to the byte past the end of the output buffer. This function + * won't write any data beyond this position. + * + * Returns dst + length. + */ +static forceinline u8 *lz_copy(u8 *dst, u32 length, u32 offset, const u8 *bufend, + u32 min_length) +{ + const u8 *src = dst - offset; + + /* + * Try to copy one machine word at a time. On i386 and x86_64 this is + * faster than copying one byte at a time, unless the data is + * near-random and all the matches have very short lengths. Note that + * since this requires unaligned memory accesses, it won't necessarily + * be faster on every architecture. + * + * Also note that we might copy more than the length of the match. For + * example, if a word is 8 bytes and the match is of length 5, then + * we'll simply copy 8 bytes. This is okay as long as we don't write + * beyond the end of the output buffer, hence the check for (bufend - + * end >= WORDBYTES - 1). + */ +#ifdef FAST_UNALIGNED_ACCESS + u8 * const end = dst + length; + + if (bufend - end >= (ptrdiff_t)(WORDBYTES - 1)) { + + if (offset >= WORDBYTES) { + /* The source and destination words don't overlap. */ + + /* To improve branch prediction, one iteration of this + * loop is unrolled. Most matches are short and will + * fail the first check. But if that check passes, then + * it becomes increasing likely that the match is long + * and we'll need to continue copying. + */ + + copy_unaligned_word(src, dst); + src += WORDBYTES; + dst += WORDBYTES; + + if (dst < end) { + do { + copy_unaligned_word(src, dst); + src += WORDBYTES; + dst += WORDBYTES; + } while (dst < end); + } + return end; + } else if (offset == 1) { + + /* Offset 1 matches are equivalent to run-length + * encoding of the previous byte. This case is common + * if the data contains many repeated bytes. + */ + size_t v = repeat_byte(*(dst - 1)); + + do { + put_unaligned(v, (size_t *)dst); + src += WORDBYTES; + dst += WORDBYTES; + } while (dst < end); + return end; + } + /* + * We don't bother with special cases for other 'offset < + * WORDBYTES', which are usually rarer than 'offset == 1'. Extra + * checks will just slow things down. Actually, it's possible + * to handle all the 'offset < WORDBYTES' cases using the same + * code, but it still becomes more complicated doesn't seem any + * faster overall; it definitely slows down the more common + * 'offset == 1' case. + */ + } +#endif /* FAST_UNALIGNED_ACCESS */ + + /* Fall back to a bytewise copy. */ + + if (min_length >= 2) { + *dst++ = *src++; + length--; + } + if (min_length >= 3) { + *dst++ = *src++; + length--; + } + do { + *dst++ = *src++; + } while (--length); + + return dst; +} + +#endif /* _LINUX_NTFS3_LIB_DECOMPRESS_COMMON_H */ diff --git a/fs/ntfs3/lib/lib.h b/fs/ntfs3/lib/lib.h new file mode 100644 index 000000000..90309a5ae --- /dev/null +++ b/fs/ntfs3/lib/lib.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Adapted for linux kernel by Alexander Mamaev: + * - remove implementations of get_unaligned_ + * - assume GCC is always defined + * - ISO C90 + * - linux kernel code style + */ + +#ifndef _LINUX_NTFS3_LIB_LIB_H +#define _LINUX_NTFS3_LIB_LIB_H + +#include + +/* globals from xpress_decompress.c */ +struct xpress_decompressor *xpress_allocate_decompressor(void); +void xpress_free_decompressor(struct xpress_decompressor *d); +int xpress_decompress(struct xpress_decompressor *__restrict d, + const void *__restrict compressed_data, + size_t compressed_size, + void *__restrict uncompressed_data, + size_t uncompressed_size); + +/* globals from lzx_decompress.c */ +struct lzx_decompressor *lzx_allocate_decompressor(void); +void lzx_free_decompressor(struct lzx_decompressor *d); +int lzx_decompress(struct lzx_decompressor *__restrict d, + const void *__restrict compressed_data, + size_t compressed_size, void *__restrict uncompressed_data, + size_t uncompressed_size); + +#endif /* _LINUX_NTFS3_LIB_LIB_H */ diff --git a/fs/ntfs3/lib/lzx_decompress.c b/fs/ntfs3/lib/lzx_decompress.c new file mode 100644 index 000000000..9196bf300 --- /dev/null +++ b/fs/ntfs3/lib/lzx_decompress.c @@ -0,0 +1,669 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * lzx_decompress.c - A decompressor for the LZX compression format, which can + * be used in "System Compressed" files. This is based on the code from wimlib. + * This code only supports a window size (dictionary size) of 32768 bytes, since + * this is the only size used in System Compression. + * + * Copyright (C) 2015 Eric Biggers + */ + +#include "decompress_common.h" +#include "lib.h" + +/* Number of literal byte values */ +#define LZX_NUM_CHARS 256 + +/* The smallest and largest allowed match lengths */ +#define LZX_MIN_MATCH_LEN 2 +#define LZX_MAX_MATCH_LEN 257 + +/* Number of distinct match lengths that can be represented */ +#define LZX_NUM_LENS (LZX_MAX_MATCH_LEN - LZX_MIN_MATCH_LEN + 1) + +/* Number of match lengths for which no length symbol is required */ +#define LZX_NUM_PRIMARY_LENS 7 +#define LZX_NUM_LEN_HEADERS (LZX_NUM_PRIMARY_LENS + 1) + +/* Valid values of the 3-bit block type field */ +#define LZX_BLOCKTYPE_VERBATIM 1 +#define LZX_BLOCKTYPE_ALIGNED 2 +#define LZX_BLOCKTYPE_UNCOMPRESSED 3 + +/* Number of offset slots for a window size of 32768 */ +#define LZX_NUM_OFFSET_SLOTS 30 + +/* Number of symbols in the main code for a window size of 32768 */ +#define LZX_MAINCODE_NUM_SYMBOLS \ + (LZX_NUM_CHARS + (LZX_NUM_OFFSET_SLOTS * LZX_NUM_LEN_HEADERS)) + +/* Number of symbols in the length code */ +#define LZX_LENCODE_NUM_SYMBOLS (LZX_NUM_LENS - LZX_NUM_PRIMARY_LENS) + +/* Number of symbols in the precode */ +#define LZX_PRECODE_NUM_SYMBOLS 20 + +/* Number of bits in which each precode codeword length is represented */ +#define LZX_PRECODE_ELEMENT_SIZE 4 + +/* Number of low-order bits of each match offset that are entropy-encoded in + * aligned offset blocks + */ +#define LZX_NUM_ALIGNED_OFFSET_BITS 3 + +/* Number of symbols in the aligned offset code */ +#define LZX_ALIGNEDCODE_NUM_SYMBOLS (1 << LZX_NUM_ALIGNED_OFFSET_BITS) + +/* Mask for the match offset bits that are entropy-encoded in aligned offset + * blocks + */ +#define LZX_ALIGNED_OFFSET_BITMASK ((1 << LZX_NUM_ALIGNED_OFFSET_BITS) - 1) + +/* Number of bits in which each aligned offset codeword length is represented */ +#define LZX_ALIGNEDCODE_ELEMENT_SIZE 3 + +/* Maximum lengths (in bits) of the codewords in each Huffman code */ +#define LZX_MAX_MAIN_CODEWORD_LEN 16 +#define LZX_MAX_LEN_CODEWORD_LEN 16 +#define LZX_MAX_PRE_CODEWORD_LEN ((1 << LZX_PRECODE_ELEMENT_SIZE) - 1) +#define LZX_MAX_ALIGNED_CODEWORD_LEN ((1 << LZX_ALIGNEDCODE_ELEMENT_SIZE) - 1) + +/* The default "filesize" value used in pre/post-processing. In the LZX format + * used in cabinet files this value must be given to the decompressor, whereas + * in the LZX format used in WIM files and system-compressed files this value is + * fixed at 12000000. + */ +#define LZX_DEFAULT_FILESIZE 12000000 + +/* Assumed block size when the encoded block size begins with a 0 bit. */ +#define LZX_DEFAULT_BLOCK_SIZE 32768 + +/* Number of offsets in the recent (or "repeat") offsets queue. */ +#define LZX_NUM_RECENT_OFFSETS 3 + +/* These values are chosen for fast decompression. */ +#define LZX_MAINCODE_TABLEBITS 11 +#define LZX_LENCODE_TABLEBITS 10 +#define LZX_PRECODE_TABLEBITS 6 +#define LZX_ALIGNEDCODE_TABLEBITS 7 + +#define LZX_READ_LENS_MAX_OVERRUN 50 + +/* Mapping: offset slot => first match offset that uses that offset slot. + */ +static const u32 lzx_offset_slot_base[LZX_NUM_OFFSET_SLOTS + 1] = { + 0, 1, 2, 3, 4, /* 0 --- 4 */ + 6, 8, 12, 16, 24, /* 5 --- 9 */ + 32, 48, 64, 96, 128, /* 10 --- 14 */ + 192, 256, 384, 512, 768, /* 15 --- 19 */ + 1024, 1536, 2048, 3072, 4096, /* 20 --- 24 */ + 6144, 8192, 12288, 16384, 24576, /* 25 --- 29 */ + 32768, /* extra */ +}; + +/* Mapping: offset slot => how many extra bits must be read and added to the + * corresponding offset slot base to decode the match offset. + */ +static const u8 lzx_extra_offset_bits[LZX_NUM_OFFSET_SLOTS] = { + 0, 0, 0, 0, 1, + 1, 2, 2, 3, 3, + 4, 4, 5, 5, 6, + 6, 7, 7, 8, 8, + 9, 9, 10, 10, 11, + 11, 12, 12, 13, 13, +}; + +/* Reusable heap-allocated memory for LZX decompression */ +struct lzx_decompressor { + + /* Huffman decoding tables, and arrays that map symbols to codeword + * lengths + */ + + u16 maincode_decode_table[(1 << LZX_MAINCODE_TABLEBITS) + + (LZX_MAINCODE_NUM_SYMBOLS * 2)]; + u8 maincode_lens[LZX_MAINCODE_NUM_SYMBOLS + LZX_READ_LENS_MAX_OVERRUN]; + + + u16 lencode_decode_table[(1 << LZX_LENCODE_TABLEBITS) + + (LZX_LENCODE_NUM_SYMBOLS * 2)]; + u8 lencode_lens[LZX_LENCODE_NUM_SYMBOLS + LZX_READ_LENS_MAX_OVERRUN]; + + + u16 alignedcode_decode_table[(1 << LZX_ALIGNEDCODE_TABLEBITS) + + (LZX_ALIGNEDCODE_NUM_SYMBOLS * 2)]; + u8 alignedcode_lens[LZX_ALIGNEDCODE_NUM_SYMBOLS]; + + u16 precode_decode_table[(1 << LZX_PRECODE_TABLEBITS) + + (LZX_PRECODE_NUM_SYMBOLS * 2)]; + u8 precode_lens[LZX_PRECODE_NUM_SYMBOLS]; + + /* Temporary space for make_huffman_decode_table() */ + u16 working_space[2 * (1 + LZX_MAX_MAIN_CODEWORD_LEN) + + LZX_MAINCODE_NUM_SYMBOLS]; +}; + +static void undo_e8_translation(void *target, s32 input_pos) +{ + s32 abs_offset, rel_offset; + + abs_offset = get_unaligned_le32(target); + if (abs_offset >= 0) { + if (abs_offset < LZX_DEFAULT_FILESIZE) { + /* "good translation" */ + rel_offset = abs_offset - input_pos; + put_unaligned_le32(rel_offset, target); + } + } else { + if (abs_offset >= -input_pos) { + /* "compensating translation" */ + rel_offset = abs_offset + LZX_DEFAULT_FILESIZE; + put_unaligned_le32(rel_offset, target); + } + } +} + +/* + * Undo the 'E8' preprocessing used in LZX. Before compression, the + * uncompressed data was preprocessed by changing the targets of suspected x86 + * CALL instructions from relative offsets to absolute offsets. After + * match/literal decoding, the decompressor must undo the translation. + */ +static void lzx_postprocess(u8 *data, u32 size) +{ + /* + * A worthwhile optimization is to push the end-of-buffer check into the + * relatively rare E8 case. This is possible if we replace the last six + * bytes of data with E8 bytes; then we are guaranteed to hit an E8 byte + * before reaching end-of-buffer. In addition, this scheme guarantees + * that no translation can begin following an E8 byte in the last 10 + * bytes because a 4-byte offset containing E8 as its high byte is a + * large negative number that is not valid for translation. That is + * exactly what we need. + */ + u8 *tail; + u8 saved_bytes[6]; + u8 *p; + + if (size <= 10) + return; + + tail = &data[size - 6]; + memcpy(saved_bytes, tail, 6); + memset(tail, 0xE8, 6); + p = data; + for (;;) { + while (*p != 0xE8) + p++; + if (p >= tail) + break; + undo_e8_translation(p + 1, p - data); + p += 5; + } + memcpy(tail, saved_bytes, 6); +} + +/* Read a Huffman-encoded symbol using the precode. */ +static forceinline u32 read_presym(const struct lzx_decompressor *d, + struct input_bitstream *is) +{ + return read_huffsym(is, d->precode_decode_table, + LZX_PRECODE_TABLEBITS, LZX_MAX_PRE_CODEWORD_LEN); +} + +/* Read a Huffman-encoded symbol using the main code. */ +static forceinline u32 read_mainsym(const struct lzx_decompressor *d, + struct input_bitstream *is) +{ + return read_huffsym(is, d->maincode_decode_table, + LZX_MAINCODE_TABLEBITS, LZX_MAX_MAIN_CODEWORD_LEN); +} + +/* Read a Huffman-encoded symbol using the length code. */ +static forceinline u32 read_lensym(const struct lzx_decompressor *d, + struct input_bitstream *is) +{ + return read_huffsym(is, d->lencode_decode_table, + LZX_LENCODE_TABLEBITS, LZX_MAX_LEN_CODEWORD_LEN); +} + +/* Read a Huffman-encoded symbol using the aligned offset code. */ +static forceinline u32 read_alignedsym(const struct lzx_decompressor *d, + struct input_bitstream *is) +{ + return read_huffsym(is, d->alignedcode_decode_table, + LZX_ALIGNEDCODE_TABLEBITS, + LZX_MAX_ALIGNED_CODEWORD_LEN); +} + +/* + * Read the precode from the compressed input bitstream, then use it to decode + * @num_lens codeword length values. + * + * @is: The input bitstream. + * + * @lens: An array that contains the length values from the previous time + * the codeword lengths for this Huffman code were read, or all 0's + * if this is the first time. This array must have at least + * (@num_lens + LZX_READ_LENS_MAX_OVERRUN) entries. + * + * @num_lens: Number of length values to decode. + * + * Returns 0 on success, or -1 if the data was invalid. + */ +static int lzx_read_codeword_lens(struct lzx_decompressor *d, + struct input_bitstream *is, + u8 *lens, u32 num_lens) +{ + u8 *len_ptr = lens; + u8 *lens_end = lens + num_lens; + int i; + + /* Read the lengths of the precode codewords. These are given + * explicitly. + */ + for (i = 0; i < LZX_PRECODE_NUM_SYMBOLS; i++) { + d->precode_lens[i] = + bitstream_read_bits(is, LZX_PRECODE_ELEMENT_SIZE); + } + + /* Make the decoding table for the precode. */ + if (make_huffman_decode_table(d->precode_decode_table, + LZX_PRECODE_NUM_SYMBOLS, + LZX_PRECODE_TABLEBITS, + d->precode_lens, + LZX_MAX_PRE_CODEWORD_LEN, + d->working_space)) + return -1; + + /* Decode the codeword lengths. */ + do { + u32 presym; + u8 len; + + /* Read the next precode symbol. */ + presym = read_presym(d, is); + if (presym < 17) { + /* Difference from old length */ + len = *len_ptr - presym; + if ((s8)len < 0) + len += 17; + *len_ptr++ = len; + } else { + /* Special RLE values */ + + u32 run_len; + + if (presym == 17) { + /* Run of 0's */ + run_len = 4 + bitstream_read_bits(is, 4); + len = 0; + } else if (presym == 18) { + /* Longer run of 0's */ + run_len = 20 + bitstream_read_bits(is, 5); + len = 0; + } else { + /* Run of identical lengths */ + run_len = 4 + bitstream_read_bits(is, 1); + presym = read_presym(d, is); + if (presym > 17) + return -1; + len = *len_ptr - presym; + if ((s8)len < 0) + len += 17; + } + + do { + *len_ptr++ = len; + } while (--run_len); + /* Worst case overrun is when presym == 18, + * run_len == 20 + 31, and only 1 length was remaining. + * So LZX_READ_LENS_MAX_OVERRUN == 50. + * + * Overrun while reading the first half of maincode_lens + * can corrupt the previous values in the second half. + * This doesn't really matter because the resulting + * lengths will still be in range, and data that + * generates overruns is invalid anyway. + */ + } + } while (len_ptr < lens_end); + + return 0; +} + +/* + * Read the header of an LZX block and save the block type and (uncompressed) + * size in *block_type_ret and *block_size_ret, respectively. + * + * If the block is compressed, also update the Huffman decode @tables with the + * new Huffman codes. If the block is uncompressed, also update the match + * offset @queue with the new match offsets. + * + * Return 0 on success, or -1 if the data was invalid. + */ +static int lzx_read_block_header(struct lzx_decompressor *d, + struct input_bitstream *is, + int *block_type_ret, + u32 *block_size_ret, + u32 recent_offsets[]) +{ + int block_type; + u32 block_size; + int i; + + bitstream_ensure_bits(is, 4); + + /* The first three bits tell us what kind of block it is, and should be + * one of the LZX_BLOCKTYPE_* values. + */ + block_type = bitstream_pop_bits(is, 3); + + /* Read the block size. */ + if (bitstream_pop_bits(is, 1)) { + block_size = LZX_DEFAULT_BLOCK_SIZE; + } else { + block_size = 0; + block_size |= bitstream_read_bits(is, 8); + block_size <<= 8; + block_size |= bitstream_read_bits(is, 8); + } + + switch (block_type) { + + case LZX_BLOCKTYPE_ALIGNED: + + /* Read the aligned offset code and prepare its decode table. + */ + + for (i = 0; i < LZX_ALIGNEDCODE_NUM_SYMBOLS; i++) { + d->alignedcode_lens[i] = + bitstream_read_bits(is, + LZX_ALIGNEDCODE_ELEMENT_SIZE); + } + + if (make_huffman_decode_table(d->alignedcode_decode_table, + LZX_ALIGNEDCODE_NUM_SYMBOLS, + LZX_ALIGNEDCODE_TABLEBITS, + d->alignedcode_lens, + LZX_MAX_ALIGNED_CODEWORD_LEN, + d->working_space)) + return -1; + + /* Fall though, since the rest of the header for aligned offset + * blocks is the same as that for verbatim blocks. + */ + fallthrough; + + case LZX_BLOCKTYPE_VERBATIM: + + /* Read the main code and prepare its decode table. + * + * Note that the codeword lengths in the main code are encoded + * in two parts: one part for literal symbols, and one part for + * match symbols. + */ + + if (lzx_read_codeword_lens(d, is, d->maincode_lens, + LZX_NUM_CHARS)) + return -1; + + if (lzx_read_codeword_lens(d, is, + d->maincode_lens + LZX_NUM_CHARS, + LZX_MAINCODE_NUM_SYMBOLS - LZX_NUM_CHARS)) + return -1; + + if (make_huffman_decode_table(d->maincode_decode_table, + LZX_MAINCODE_NUM_SYMBOLS, + LZX_MAINCODE_TABLEBITS, + d->maincode_lens, + LZX_MAX_MAIN_CODEWORD_LEN, + d->working_space)) + return -1; + + /* Read the length code and prepare its decode table. */ + + if (lzx_read_codeword_lens(d, is, d->lencode_lens, + LZX_LENCODE_NUM_SYMBOLS)) + return -1; + + if (make_huffman_decode_table(d->lencode_decode_table, + LZX_LENCODE_NUM_SYMBOLS, + LZX_LENCODE_TABLEBITS, + d->lencode_lens, + LZX_MAX_LEN_CODEWORD_LEN, + d->working_space)) + return -1; + + break; + + case LZX_BLOCKTYPE_UNCOMPRESSED: + + /* Before reading the three recent offsets from the uncompressed + * block header, the stream must be aligned on a 16-bit + * boundary. But if the stream is *already* aligned, then the + * next 16 bits must be discarded. + */ + bitstream_ensure_bits(is, 1); + bitstream_align(is); + + recent_offsets[0] = bitstream_read_u32(is); + recent_offsets[1] = bitstream_read_u32(is); + recent_offsets[2] = bitstream_read_u32(is); + + /* Offsets of 0 are invalid. */ + if (recent_offsets[0] == 0 || recent_offsets[1] == 0 || + recent_offsets[2] == 0) + return -1; + break; + + default: + /* Unrecognized block type. */ + return -1; + } + + *block_type_ret = block_type; + *block_size_ret = block_size; + return 0; +} + +/* Decompress a block of LZX-compressed data. */ +static int lzx_decompress_block(const struct lzx_decompressor *d, + struct input_bitstream *is, + int block_type, u32 block_size, + u8 * const out_begin, u8 *out_next, + u32 recent_offsets[]) +{ + u8 * const block_end = out_next + block_size; + u32 ones_if_aligned = 0U - (block_type == LZX_BLOCKTYPE_ALIGNED); + + do { + u32 mainsym; + u32 match_len; + u32 match_offset; + u32 offset_slot; + u32 num_extra_bits; + + mainsym = read_mainsym(d, is); + if (mainsym < LZX_NUM_CHARS) { + /* Literal */ + *out_next++ = mainsym; + continue; + } + + /* Match */ + + /* Decode the length header and offset slot. */ + mainsym -= LZX_NUM_CHARS; + match_len = mainsym % LZX_NUM_LEN_HEADERS; + offset_slot = mainsym / LZX_NUM_LEN_HEADERS; + + /* If needed, read a length symbol to decode the full length. */ + if (match_len == LZX_NUM_PRIMARY_LENS) + match_len += read_lensym(d, is); + match_len += LZX_MIN_MATCH_LEN; + + if (offset_slot < LZX_NUM_RECENT_OFFSETS) { + /* Repeat offset */ + + /* Note: This isn't a real LRU queue, since using the R2 + * offset doesn't bump the R1 offset down to R2. This + * quirk allows all 3 recent offsets to be handled by + * the same code. (For R0, the swap is a no-op.) + */ + match_offset = recent_offsets[offset_slot]; + swap(recent_offsets[offset_slot], recent_offsets[0]); + } else { + /* Explicit offset */ + + /* Look up the number of extra bits that need to be read + * to decode offsets with this offset slot. + */ + num_extra_bits = lzx_extra_offset_bits[offset_slot]; + + /* Start with the offset slot base value. */ + match_offset = lzx_offset_slot_base[offset_slot]; + + /* In aligned offset blocks, the low-order 3 bits of + * each offset are encoded using the aligned offset + * code. Otherwise, all the extra bits are literal. + */ + + if ((num_extra_bits & ones_if_aligned) >= LZX_NUM_ALIGNED_OFFSET_BITS) { + match_offset += + bitstream_read_bits(is, num_extra_bits - + LZX_NUM_ALIGNED_OFFSET_BITS) + << LZX_NUM_ALIGNED_OFFSET_BITS; + match_offset += read_alignedsym(d, is); + } else { + match_offset += bitstream_read_bits(is, num_extra_bits); + } + + /* Adjust the offset. */ + match_offset -= (LZX_NUM_RECENT_OFFSETS - 1); + + /* Update the recent offsets. */ + recent_offsets[2] = recent_offsets[1]; + recent_offsets[1] = recent_offsets[0]; + recent_offsets[0] = match_offset; + } + + /* Validate the match, then copy it to the current position. */ + + if (match_len > (size_t)(block_end - out_next)) + return -1; + + if (match_offset > (size_t)(out_next - out_begin)) + return -1; + + out_next = lz_copy(out_next, match_len, match_offset, + block_end, LZX_MIN_MATCH_LEN); + + } while (out_next != block_end); + + return 0; +} + +/* + * lzx_allocate_decompressor - Allocate an LZX decompressor + * + * Return the pointer to the decompressor on success, or return NULL and set + * errno on failure. + */ +struct lzx_decompressor *lzx_allocate_decompressor(void) +{ + return kmalloc_obj(struct lzx_decompressor, GFP_NOFS); +} + +/* + * lzx_decompress - Decompress a buffer of LZX-compressed data + * + * @decompressor: A decompressor allocated with lzx_allocate_decompressor() + * @compressed_data: The buffer of data to decompress + * @compressed_size: Number of bytes of compressed data + * @uncompressed_data: The buffer in which to store the decompressed data + * @uncompressed_size: The number of bytes the data decompresses into + * + * Return 0 on success, or return -1 and set errno on failure. + */ +int lzx_decompress(struct lzx_decompressor *decompressor, + const void *compressed_data, size_t compressed_size, + void *uncompressed_data, size_t uncompressed_size) +{ + struct lzx_decompressor *d = decompressor; + u8 * const out_begin = uncompressed_data; + u8 *out_next = out_begin; + u8 * const out_end = out_begin + uncompressed_size; + struct input_bitstream is; + u32 recent_offsets[LZX_NUM_RECENT_OFFSETS] = {1, 1, 1}; + int e8_status = 0; + + init_input_bitstream(&is, compressed_data, compressed_size); + + /* Codeword lengths begin as all 0's for delta encoding purposes. */ + memset(d->maincode_lens, 0, LZX_MAINCODE_NUM_SYMBOLS); + memset(d->lencode_lens, 0, LZX_LENCODE_NUM_SYMBOLS); + + /* Decompress blocks until we have all the uncompressed data. */ + + while (out_next != out_end) { + int block_type; + u32 block_size; + + if (lzx_read_block_header(d, &is, &block_type, &block_size, + recent_offsets)) + goto invalid; + + if (block_size < 1 || block_size > (size_t)(out_end - out_next)) + goto invalid; + + if (block_type != LZX_BLOCKTYPE_UNCOMPRESSED) { + + /* Compressed block */ + + if (lzx_decompress_block(d, + &is, + block_type, + block_size, + out_begin, + out_next, + recent_offsets)) + goto invalid; + + e8_status |= d->maincode_lens[0xe8]; + out_next += block_size; + } else { + /* Uncompressed block */ + + out_next = bitstream_read_bytes(&is, out_next, + block_size); + if (!out_next) + goto invalid; + + if (block_size & 1) + bitstream_read_byte(&is); + + e8_status = 1; + } + } + + /* Postprocess the data unless it cannot possibly contain 0xe8 bytes. */ + if (e8_status) + lzx_postprocess(uncompressed_data, uncompressed_size); + + return 0; + +invalid: + return -1; +} + +/* + * lzx_free_decompressor - Free an LZX decompressor + * + * @decompressor: A decompressor that was allocated with + * lzx_allocate_decompressor(), or NULL. + */ +void lzx_free_decompressor(struct lzx_decompressor *decompressor) +{ + kfree(decompressor); +} diff --git a/fs/ntfs3/lib/xpress_decompress.c b/fs/ntfs3/lib/xpress_decompress.c new file mode 100644 index 000000000..402cf21ae --- /dev/null +++ b/fs/ntfs3/lib/xpress_decompress.c @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * xpress_decompress.c - A decompressor for the XPRESS compression format + * (Huffman variant), which can be used in "System Compressed" files. This is + * based on the code from wimlib. + * + * Copyright (C) 2015 Eric Biggers + */ + +#include "decompress_common.h" +#include "lib.h" + +#define XPRESS_NUM_SYMBOLS 512 +#define XPRESS_MAX_CODEWORD_LEN 15 +#define XPRESS_MIN_MATCH_LEN 3 + +/* This value is chosen for fast decompression. */ +#define XPRESS_TABLEBITS 12 + +/* Reusable heap-allocated memory for XPRESS decompression */ +struct xpress_decompressor { + + /* The Huffman decoding table */ + u16 decode_table[(1 << XPRESS_TABLEBITS) + 2 * XPRESS_NUM_SYMBOLS]; + + /* An array that maps symbols to codeword lengths */ + u8 lens[XPRESS_NUM_SYMBOLS]; + + /* Temporary space for make_huffman_decode_table() */ + u16 working_space[2 * (1 + XPRESS_MAX_CODEWORD_LEN) + + XPRESS_NUM_SYMBOLS]; +}; + +/* + * xpress_allocate_decompressor - Allocate an XPRESS decompressor + * + * Return the pointer to the decompressor on success, or return NULL and set + * errno on failure. + */ +struct xpress_decompressor *xpress_allocate_decompressor(void) +{ + return kmalloc_obj(struct xpress_decompressor, GFP_NOFS); +} + +/* + * xpress_decompress - Decompress a buffer of XPRESS-compressed data + * + * @decompressor: A decompressor that was allocated with + * xpress_allocate_decompressor() + * @compressed_data: The buffer of data to decompress + * @compressed_size: Number of bytes of compressed data + * @uncompressed_data: The buffer in which to store the decompressed data + * @uncompressed_size: The number of bytes the data decompresses into + * + * Return 0 on success, or return -1 and set errno on failure. + */ +int xpress_decompress(struct xpress_decompressor *decompressor, + const void *compressed_data, size_t compressed_size, + void *uncompressed_data, size_t uncompressed_size) +{ + struct xpress_decompressor *d = decompressor; + const u8 * const in_begin = compressed_data; + u8 * const out_begin = uncompressed_data; + u8 *out_next = out_begin; + u8 * const out_end = out_begin + uncompressed_size; + struct input_bitstream is; + u32 i; + + /* Read the Huffman codeword lengths. */ + if (compressed_size < XPRESS_NUM_SYMBOLS / 2) + goto invalid; + for (i = 0; i < XPRESS_NUM_SYMBOLS / 2; i++) { + d->lens[i*2 + 0] = in_begin[i] & 0xF; + d->lens[i*2 + 1] = in_begin[i] >> 4; + } + + /* Build a decoding table for the Huffman code. */ + if (make_huffman_decode_table(d->decode_table, XPRESS_NUM_SYMBOLS, + XPRESS_TABLEBITS, d->lens, + XPRESS_MAX_CODEWORD_LEN, + d->working_space)) + goto invalid; + + /* Decode the matches and literals. */ + + init_input_bitstream(&is, in_begin + XPRESS_NUM_SYMBOLS / 2, + compressed_size - XPRESS_NUM_SYMBOLS / 2); + + while (out_next != out_end) { + u32 sym; + u32 log2_offset; + u32 length; + u32 offset; + + sym = read_huffsym(&is, d->decode_table, + XPRESS_TABLEBITS, XPRESS_MAX_CODEWORD_LEN); + if (sym < 256) { + /* Literal */ + *out_next++ = sym; + } else { + /* Match */ + length = sym & 0xf; + log2_offset = (sym >> 4) & 0xf; + + bitstream_ensure_bits(&is, 16); + + offset = ((u32)1 << log2_offset) | + bitstream_pop_bits(&is, log2_offset); + + if (length == 0xf) { + length += bitstream_read_byte(&is); + if (length == 0xf + 0xff) + length = bitstream_read_u16(&is); + } + length += XPRESS_MIN_MATCH_LEN; + + if (offset > (size_t)(out_next - out_begin)) + goto invalid; + + if (length > (size_t)(out_end - out_next)) + goto invalid; + + out_next = lz_copy(out_next, length, offset, out_end, + XPRESS_MIN_MATCH_LEN); + } + } + return 0; + +invalid: + return -1; +} + +/* + * xpress_free_decompressor - Free an XPRESS decompressor + * + * @decompressor: A decompressor that was allocated with + * xpress_allocate_decompressor(), or NULL. + */ +void xpress_free_decompressor(struct xpress_decompressor *decompressor) +{ + kfree(decompressor); +} diff --git a/fs/ntfs3/lznt.c b/fs/ntfs3/lznt.c new file mode 100644 index 000000000..5dcb76747 --- /dev/null +++ b/fs/ntfs3/lznt.c @@ -0,0 +1,458 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + */ + +#include +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs_fs.h" + +// clang-format off +/* Src buffer is zero. */ +#define LZNT_ERROR_ALL_ZEROS 1 +#define LZNT_CHUNK_SIZE 0x1000 +// clang-format on + +struct lznt_hash { + const u8 *p1; + const u8 *p2; +}; + +struct lznt { + const u8 *unc; + const u8 *unc_end; + const u8 *best_match; + size_t max_len; + bool std; + + struct lznt_hash hash[LZNT_CHUNK_SIZE]; +}; + +static inline size_t get_match_len(const u8 *ptr, const u8 *end, const u8 *prev, + size_t max_len) +{ + size_t len = 0; + + while (ptr + len < end && ptr[len] == prev[len] && ++len < max_len) + ; + return len; +} + +static size_t longest_match_std(const u8 *src, struct lznt *ctx) +{ + size_t hash_index; + size_t len1 = 0, len2 = 0; + const u8 **hash; + + hash_index = + ((40543U * ((((src[0] << 4) ^ src[1]) << 4) ^ src[2])) >> 4) & + (LZNT_CHUNK_SIZE - 1); + + hash = &(ctx->hash[hash_index].p1); + + if (hash[0] >= ctx->unc && hash[0] < src && hash[0][0] == src[0] && + hash[0][1] == src[1] && hash[0][2] == src[2]) { + len1 = 3; + if (ctx->max_len > 3) + len1 += get_match_len(src + 3, ctx->unc_end, + hash[0] + 3, ctx->max_len - 3); + } + + if (hash[1] >= ctx->unc && hash[1] < src && hash[1][0] == src[0] && + hash[1][1] == src[1] && hash[1][2] == src[2]) { + len2 = 3; + if (ctx->max_len > 3) + len2 += get_match_len(src + 3, ctx->unc_end, + hash[1] + 3, ctx->max_len - 3); + } + + /* Compare two matches and select the best one. */ + if (len1 < len2) { + ctx->best_match = hash[1]; + len1 = len2; + } else { + ctx->best_match = hash[0]; + } + + hash[1] = hash[0]; + hash[0] = src; + return len1; +} + +static size_t longest_match_best(const u8 *src, struct lznt *ctx) +{ + size_t max_len; + const u8 *ptr; + + if (ctx->unc >= src || !ctx->max_len) + return 0; + + max_len = 0; + for (ptr = ctx->unc; ptr < src; ++ptr) { + size_t len = + get_match_len(src, ctx->unc_end, ptr, ctx->max_len); + if (len >= max_len) { + max_len = len; + ctx->best_match = ptr; + } + } + + return max_len >= 3 ? max_len : 0; +} + +static const size_t s_max_len[] = { + 0x1002, 0x802, 0x402, 0x202, 0x102, 0x82, 0x42, 0x22, 0x12, +}; + +static const size_t s_max_off[] = { + 0x10, 0x20, 0x40, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, +}; + +static inline u16 make_pair(size_t offset, size_t len, size_t index) +{ + return ((offset - 1) << (12 - index)) | + ((len - 3) & (((1 << (12 - index)) - 1))); +} + +static inline size_t parse_pair(u16 pair, size_t *offset, size_t index) +{ + *offset = 1 + (pair >> (12 - index)); + return 3 + (pair & ((1 << (12 - index)) - 1)); +} + +/* + * compress_chunk + * + * Return: + * * 0 - Ok, @cmpr contains @cmpr_chunk_size bytes of compressed data. + * * 1 - Input buffer is full zero. + * * -2 - The compressed buffer is too small to hold the compressed data. + */ +static inline int compress_chunk(size_t (*match)(const u8 *, struct lznt *), + const u8 *unc, const u8 *unc_end, u8 *cmpr, + u8 *cmpr_end, size_t *cmpr_chunk_size, + struct lznt *ctx) +{ + size_t cnt = 0; + size_t idx = 0; + const u8 *up = unc; + u8 *cp = cmpr + 3; + u8 *cp2 = cmpr + 2; + u8 not_zero = 0; + /* Control byte of 8-bit values: ( 0 - means byte as is, 1 - short pair ). */ + u8 ohdr = 0; + u8 *last; + u16 t16; + + if (unc + LZNT_CHUNK_SIZE < unc_end) + unc_end = unc + LZNT_CHUNK_SIZE; + + last = min(cmpr + LZNT_CHUNK_SIZE + sizeof(short), cmpr_end); + + ctx->unc = unc; + ctx->unc_end = unc_end; + ctx->max_len = s_max_len[0]; + + while (up < unc_end) { + size_t max_len; + + while (unc + s_max_off[idx] < up) + ctx->max_len = s_max_len[++idx]; + + /* Find match. */ + max_len = up + 3 <= unc_end ? (*match)(up, ctx) : 0; + + if (!max_len) { + if (cp >= last) + goto NotCompressed; + not_zero |= *cp++ = *up++; + } else if (cp + 1 >= last) { + goto NotCompressed; + } else { + t16 = make_pair(up - ctx->best_match, max_len, idx); + *cp++ = t16; + *cp++ = t16 >> 8; + + ohdr |= 1 << cnt; + up += max_len; + } + + cnt = (cnt + 1) & 7; + if (!cnt) { + *cp2 = ohdr; + ohdr = 0; + cp2 = cp; + cp += 1; + } + } + + if (cp2 < last) + *cp2 = ohdr; + else + cp -= 1; + + *cmpr_chunk_size = cp - cmpr; + + t16 = (*cmpr_chunk_size - 3) | 0xB000; + cmpr[0] = t16; + cmpr[1] = t16 >> 8; + + return not_zero ? 0 : LZNT_ERROR_ALL_ZEROS; + +NotCompressed: + + if ((cmpr + LZNT_CHUNK_SIZE + sizeof(short)) > last) + return -2; + + /* + * Copy non cmpr data. + * 0x3FFF == ((LZNT_CHUNK_SIZE + 2 - 3) | 0x3000) + */ + cmpr[0] = 0xff; + cmpr[1] = 0x3f; + + memcpy(cmpr + sizeof(short), unc, LZNT_CHUNK_SIZE); + *cmpr_chunk_size = LZNT_CHUNK_SIZE + sizeof(short); + + return 0; +} + +static inline ssize_t decompress_chunk(u8 *unc, u8 *unc_end, const u8 *cmpr, + const u8 *cmpr_end) +{ + u8 *up = unc; + u8 ch = *cmpr++; + size_t bit = 0; + size_t index = 0; + u16 pair; + size_t offset, length; + + /* Do decompression until pointers are inside range. */ + while (up < unc_end && cmpr < cmpr_end) { + // return err if more than LZNT_CHUNK_SIZE bytes are written + if (up - unc > LZNT_CHUNK_SIZE) + return -EINVAL; + /* Correct index */ + while (index < ARRAY_SIZE(s_max_off) - 1 && + unc + s_max_off[index] < up) { + index += 1; + } + + /* Check the current flag for zero. */ + if (!(ch & (1 << bit))) { + /* Just copy byte. */ + *up++ = *cmpr++; + goto next; + } + + /* Check for boundary. */ + if (cmpr + 1 >= cmpr_end) + return -EINVAL; + + /* Read a short from little endian stream. */ + pair = cmpr[1]; + pair <<= 8; + pair |= cmpr[0]; + + cmpr += 2; + + /* Translate packed information into offset and length. */ + length = parse_pair(pair, &offset, index); + + /* Check offset for boundary. */ + if (unc + offset > up) + return -EINVAL; + + /* Truncate the length if necessary. */ + if (up + length >= unc_end) + length = unc_end - up; + + /* Now we copy bytes. This is the heart of LZ algorithm. */ + for (; length > 0; length--, up++) + *up = *(up - offset); + +next: + /* Advance flag bit value. */ + bit = (bit + 1) & 7; + + if (!bit) { + if (cmpr >= cmpr_end) + break; + + ch = *cmpr++; + } + } + + /* Return the size of uncompressed data. */ + return up - unc; +} + +/* + * get_lznt_ctx + * @level: 0 - Standard compression. + * !0 - Best compression, requires a lot of cpu. + */ +struct lznt *get_lznt_ctx(int level) +{ + struct lznt *r = kzalloc(level ? offsetof(struct lznt, hash) : + sizeof(struct lznt), + GFP_NOFS); + + if (r) + r->std = !level; + return r; +} + +/* + * compress_lznt - Compresses @unc into @cmpr + * + * Return: + * * +x - Ok, @cmpr contains 'final_compressed_size' bytes of compressed data. + * * 0 - Input buffer is full zero. + */ +size_t compress_lznt(const void *unc, size_t unc_size, void *cmpr, + size_t cmpr_size, struct lznt *ctx) +{ + int err; + size_t (*match)(const u8 *src, struct lznt *ctx); + u8 *p = cmpr; + u8 *end = p + cmpr_size; + const u8 *unc_chunk = unc; + const u8 *unc_end = unc_chunk + unc_size; + bool is_zero = true; + + if (ctx->std) { + match = &longest_match_std; + memset(ctx->hash, 0, sizeof(ctx->hash)); + } else { + match = &longest_match_best; + } + + /* Compression cycle. */ + for (; unc_chunk < unc_end; unc_chunk += LZNT_CHUNK_SIZE) { + cmpr_size = 0; + err = compress_chunk(match, unc_chunk, unc_end, p, end, + &cmpr_size, ctx); + if (err < 0) + return unc_size; + + if (is_zero && err != LZNT_ERROR_ALL_ZEROS) + is_zero = false; + + p += cmpr_size; + } + + if (p <= end - 2) + p[0] = p[1] = 0; + + return is_zero ? 0 : PtrOffset(cmpr, p); +} + +/* + * decompress_lznt - Decompress @cmpr into @unc. + */ +ssize_t decompress_lznt(const void *cmpr, size_t cmpr_size, void *unc, + size_t unc_size) +{ + const u8 *cmpr_chunk = cmpr; + const u8 *cmpr_end = cmpr_chunk + cmpr_size; + u8 *unc_chunk = unc; + u8 *unc_end = unc_chunk + unc_size; + u16 chunk_hdr; + + if (cmpr_size < sizeof(short)) + return -EINVAL; + + /* Read chunk header. */ + chunk_hdr = cmpr_chunk[1]; + chunk_hdr <<= 8; + chunk_hdr |= cmpr_chunk[0]; + + /* Loop through decompressing chunks. */ + for (;;) { + size_t chunk_size_saved; + size_t unc_use; + size_t cmpr_use = 3 + (chunk_hdr & (LZNT_CHUNK_SIZE - 1)); + + /* Check that the chunk actually fits the supplied buffer. */ + if (cmpr_chunk + cmpr_use > cmpr_end) + return -EINVAL; + + /* First make sure the chunk contains compressed data. */ + if (chunk_hdr & 0x8000) { + /* Decompress a chunk and return if we get an error. */ + ssize_t err = + decompress_chunk(unc_chunk, unc_end, + cmpr_chunk + sizeof(chunk_hdr), + cmpr_chunk + cmpr_use); + if (err < 0) + return err; + unc_use = err; + } else { + /* This chunk does not contain compressed data. */ + unc_use = unc_chunk + LZNT_CHUNK_SIZE > unc_end ? + unc_end - unc_chunk : + LZNT_CHUNK_SIZE; + + if (cmpr_chunk + sizeof(chunk_hdr) + unc_use > + cmpr_end) { + return -EINVAL; + } + + memcpy(unc_chunk, cmpr_chunk + sizeof(chunk_hdr), + unc_use); + } + + /* Advance pointers. */ + cmpr_chunk += cmpr_use; + unc_chunk += unc_use; + + /* Check for the end of unc buffer. */ + if (unc_chunk >= unc_end) + break; + + /* Proceed the next chunk. */ + if (cmpr_chunk > cmpr_end - 2) + break; + + chunk_size_saved = LZNT_CHUNK_SIZE; + + /* Read chunk header. */ + chunk_hdr = cmpr_chunk[1]; + chunk_hdr <<= 8; + chunk_hdr |= cmpr_chunk[0]; + + if (!chunk_hdr) + break; + + /* Check the size of unc buffer. */ + if (unc_use < chunk_size_saved) { + size_t t1 = chunk_size_saved - unc_use; + u8 *t2 = unc_chunk + t1; + + /* 'Zero' memory. */ + if (t2 >= unc_end) + break; + + memset(unc_chunk, 0, t1); + unc_chunk = t2; + } + } + + /* Check compression boundary. */ + if (cmpr_chunk > cmpr_end) + return -EINVAL; + + /* + * The unc size is just a difference between current + * pointer and original one. + */ + return PtrOffset(unc, unc_chunk); +} diff --git a/fs/ntfs3/namei.c b/fs/ntfs3/namei.c new file mode 100644 index 000000000..ec59bbabd --- /dev/null +++ b/fs/ntfs3/namei.c @@ -0,0 +1,564 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + */ + +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +/* + * fill_name_de - Format NTFS_DE in @buf. + */ +int fill_name_de(struct ntfs_sb_info *sbi, void *buf, const struct qstr *name, + const struct cpu_str *uni) +{ + int err; + struct NTFS_DE *e = buf; + u16 data_size, real_size, aligned_size; + struct ATTR_FILE_NAME *fname = (struct ATTR_FILE_NAME *)(e + 1); + +#ifndef CONFIG_NTFS3_64BIT_CLUSTER + e->ref.high = fname->home.high = 0; +#endif + if (uni) { +#ifdef __BIG_ENDIAN + int ulen = uni->len; + __le16 *uname = fname->name; + const u16 *name_cpu = uni->name; + + while (ulen--) + *uname++ = cpu_to_le16(*name_cpu++); +#else + memcpy(fname->name, uni->name, uni->len * sizeof(u16)); +#endif + fname->name_len = uni->len; + + } else { + /* Convert input string to unicode. */ + err = ntfs_nls_to_utf16(sbi, name->name, name->len, + (struct cpu_str *)&fname->name_len, + NTFS_NAME_LEN, UTF16_LITTLE_ENDIAN); + if (err < 0) + return err; + } + + fname->type = FILE_NAME_POSIX; + data_size = fname_full_size(fname); + + real_size = data_size + sizeof(struct NTFS_DE); + aligned_size = ALIGN(data_size, 8) + sizeof(struct NTFS_DE); + if (aligned_size > real_size) + memset((char *)buf + real_size, 0, aligned_size - real_size); + + e->size = cpu_to_le16(aligned_size); + e->key_size = cpu_to_le16(data_size); + e->flags = 0; + e->res = 0; + + return 0; +} + +/* + * ntfs_lookup - inode_operations::lookup + */ +static struct dentry *ntfs_lookup(struct inode *dir, struct dentry *dentry, + u32 flags) +{ + struct ntfs_inode *ni = ntfs_i(dir); + struct cpu_str *uni = kmalloc(PATH_MAX, GFP_KERNEL); + struct inode *inode; + int err; + + if (!uni) + return ERR_PTR(-ENOMEM); + + err = ntfs_nls_to_utf16(ni->mi.sbi, dentry->d_name.name, + dentry->d_name.len, uni, NTFS_NAME_LEN, + UTF16_HOST_ENDIAN); + + if (err < 0) { + kfree(uni); + return ERR_PTR(err); + } + + ni_lock_dir(ni); + inode = dir_search_flags(dir, uni, NULL, flags); + ni_unlock(ni); + + kfree(uni); + + /* + * Check for a null pointer + * If the MFT record of ntfs inode is not a base record, inode->i_op can be NULL. + * This causes null pointer dereference in d_splice_alias(). + */ + if (!IS_ERR_OR_NULL(inode) && !inode->i_op) { + iput(inode); + return ERR_PTR(-EINVAL); + } + + return d_splice_alias(inode, dentry); +} + +/* + * ntfs_create - inode_operations::create + */ +static int ntfs_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + return ntfs_create_inode(idmap, dir, dentry, NULL, S_IFREG | mode, 0, + NULL, 0, NULL); +} + +/* + * ntfs_mknod - inode_operations::mknod + */ +static int ntfs_mknod(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) +{ + return ntfs_create_inode(idmap, dir, dentry, NULL, mode, rdev, NULL, 0, + NULL); +} + +/* + * ntfs_link - inode_operations::link + */ +static int ntfs_link(struct dentry *ode, struct inode *dir, struct dentry *de) +{ + int err; + struct inode *inode = d_inode(ode); + struct ntfs_inode *ni = ntfs_i(inode); + + if (S_ISDIR(inode->i_mode)) + return -EPERM; + + if (inode->i_nlink >= NTFS_LINK_MAX) + return -EMLINK; + + ni_lock_dir(ntfs_i(dir)); + if (inode != dir) + ni_lock(ni); + + inc_nlink(inode); + ihold(inode); + + err = ntfs_link_inode(inode, de); + + if (!err) { + inode_set_ctime_current(inode); + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + mark_inode_dirty(inode); + mark_inode_dirty(dir); + d_instantiate(de, inode); + } else { + drop_nlink(inode); + iput(inode); + } + + if (inode != dir) + ni_unlock(ni); + ni_unlock(ntfs_i(dir)); + + return err; +} + +/* + * ntfs_unlink - inode_operations::unlink + */ +static int ntfs_unlink(struct inode *dir, struct dentry *dentry) +{ + struct ntfs_inode *dir_ni = ntfs_i(dir); + struct inode *inode = d_inode(dentry); + struct ntfs_inode *ni = ntfs_i(inode); + int err; + + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(ni))) + return -EINVAL; + + if (unlikely(ntfs3_forced_shutdown(dir->i_sb))) + return -EIO; + + if (likely(is_ni_base(ni))) { + ni_lock_dir(dir_ni); + /* Remove general file/dir. */ + err = ntfs_unlink_inode(dir, dentry); + ni_unlock(dir_ni); + } else { + ni_lock(ni); + /* Remove ADS. */ + err = ni_remove_attr(ni, ATTR_DATA, ni->file.ads.name, + ni->file.ads.len, false, NULL); + ni_unlock(ni); + + if (!err) + drop_nlink(inode); + } + + return err; +} + +/* + * ntfs_symlink - inode_operations::symlink + */ +static int ntfs_symlink(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const char *symname) +{ + u32 size = strlen(symname); + + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(ntfs_i(dir)))) + return -EINVAL; + + if (unlikely(ntfs3_forced_shutdown(dir->i_sb))) + return -EIO; + + return ntfs_create_inode(idmap, dir, dentry, NULL, S_IFLNK | 0777, 0, + symname, size, NULL); +} + +/* + * ntfs_mkdir - inode_operations::mkdir + */ +static struct dentry *ntfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + return ERR_PTR(ntfs_create_inode(idmap, dir, dentry, NULL, + mode, 0, NULL, 0, NULL)); +} + +/* + * ntfs_rmdir - inode_operations::rmdir + */ +static int ntfs_rmdir(struct inode *dir, struct dentry *dentry) +{ + struct ntfs_inode *ni = ntfs_i(dir); + int err; + + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(ni))) + return -EINVAL; + + if (unlikely(ntfs3_forced_shutdown(dir->i_sb))) + return -EIO; + + ni_lock_dir(ni); + + err = ntfs_unlink_inode(dir, dentry); + + ni_unlock(ni); + + return err; +} + +/* + * ntfs_rename - inode_operations::rename + */ +static int ntfs_rename(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, struct inode *new_dir, + struct dentry *new_dentry, u32 flags) +{ + int err; + struct super_block *sb = dir->i_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + struct ntfs_inode *dir_ni = ntfs_i(dir); + struct ntfs_inode *new_dir_ni = ntfs_i(new_dir); + struct inode *inode = d_inode(dentry); + struct ntfs_inode *ni = ntfs_i(inode); + struct inode *new_inode = d_inode(new_dentry); + struct NTFS_DE *de, *new_de; + bool is_same; + /* + * de - memory of PATH_MAX bytes: + * [0-1024) - original name (dentry->d_name) + * [1024-2048) - paired to original name, usually DOS variant of dentry->d_name + * [2048-3072) - new name (new_dentry->d_name) + */ + static_assert(SIZEOF_ATTRIBUTE_FILENAME_MAX + SIZEOF_RESIDENT < 1024); + static_assert(SIZEOF_ATTRIBUTE_FILENAME_MAX + sizeof(struct NTFS_DE) < + 1024); + static_assert(PATH_MAX >= 4 * 1024); + + if (!is_ni_base(ni)) { + /* No rename for ADS. */ + return -EOPNOTSUPP; + } + + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(ni))) + return -EINVAL; + + if (unlikely(ntfs3_forced_shutdown(sb))) + return -EIO; + + if (flags & ~RENAME_NOREPLACE) + return -EINVAL; + + is_same = dentry->d_name.len == new_dentry->d_name.len && + !memcmp(dentry->d_name.name, new_dentry->d_name.name, + dentry->d_name.len); + + if (is_same && dir == new_dir) { + /* Nothing to do. */ + return 0; + } + + if (ntfs_is_meta_file(sbi, inode->i_ino)) { + /* Should we print an error? */ + return -EINVAL; + } + + if (new_inode) { + /* Target name exists. Unlink it. */ + dget(new_dentry); + ni_lock_dir(new_dir_ni); + err = ntfs_unlink_inode(new_dir, new_dentry); + ni_unlock(new_dir_ni); + dput(new_dentry); + if (err) + return err; + } + + de = kmalloc(PATH_MAX, GFP_KERNEL); + if (!de) + return -ENOMEM; + + /* Translate dentry->d_name into unicode form. */ + err = fill_name_de(sbi, de, &dentry->d_name, NULL); + if (err < 0) + goto out; + + if (is_same) { + /* Reuse 'de'. */ + new_de = de; + } else { + /* Translate new_dentry->d_name into unicode form. */ + new_de = Add2Ptr(de, 2048); + err = fill_name_de(sbi, new_de, &new_dentry->d_name, NULL); + if (err < 0) + goto out; + } + + ni_lock_dir(dir_ni); + ni_lock(ni); + if (dir_ni != new_dir_ni) + ni_lock_dir2(new_dir_ni); + + err = ni_rename(dir_ni, new_dir_ni, ni, de, new_de); + if (!err) { + simple_rename_timestamp(dir, dentry, new_dir, new_dentry); + mark_inode_dirty(inode); + mark_inode_dirty(dir); + if (dir != new_dir) + mark_inode_dirty(new_dir); + + if (IS_DIRSYNC(dir)) + ntfs_sync_inode(dir); + + if (IS_DIRSYNC(new_dir)) + ntfs_sync_inode(new_dir); + } + + if (dir_ni != new_dir_ni) + ni_unlock(new_dir_ni); + ni_unlock(ni); + ni_unlock(dir_ni); +out: + kfree(de); + return err; +} + +struct dentry *ntfs3_get_parent(struct dentry *child) +{ + struct inode *inode = d_inode(child); + struct ntfs_inode *ni = ntfs_i(inode); + + struct ATTR_LIST_ENTRY *le = NULL; + struct ATTRIB *attr = NULL; + struct ATTR_FILE_NAME *fname; + + while ((attr = ni_find_attr(ni, attr, &le, ATTR_NAME, NULL, 0, NULL, + NULL))) { + fname = resident_data_ex(attr, SIZEOF_ATTRIBUTE_FILENAME); + if (!fname) + continue; + + return d_obtain_alias( + ntfs_iget5(inode->i_sb, &fname->home, NULL)); + } + + return ERR_PTR(-ENOENT); +} + +/* + * dentry_operations::d_hash + */ +static int ntfs_d_hash(const struct dentry *dentry, struct qstr *name) +{ + struct ntfs_sb_info *sbi; + const char *n = name->name; + unsigned int len = name->len; + unsigned long hash; + struct cpu_str *uni; + unsigned int c; + int err; + + /* First try fast implementation. */ + hash = init_name_hash(dentry); + + for (;;) { + if (!len--) { + name->hash = end_name_hash(hash); + return 0; + } + + c = *n++; + if (c >= 0x80) + break; + + hash = partial_name_hash(toupper(c), hash); + } + + /* + * Try slow way with current upcase table + */ + uni = kmalloc(PATH_MAX, GFP_NOWAIT); + if (!uni) + return -ENOMEM; + + sbi = dentry->d_sb->s_fs_info; + + err = ntfs_nls_to_utf16(sbi, name->name, name->len, uni, NTFS_NAME_LEN, + UTF16_HOST_ENDIAN); + if (err < 0) + goto out; + + if (!err) { + err = -EINVAL; + goto out; + } + + hash = ntfs_names_hash(uni->name, uni->len, sbi->upcase, + init_name_hash(dentry)); + name->hash = end_name_hash(hash); + err = 0; + +out: + kfree(uni); + return err; +} + +/* + * dentry_operations::d_compare + */ +static int ntfs_d_compare(const struct dentry *dentry, unsigned int len1, + const char *str, const struct qstr *name) +{ + struct ntfs_sb_info *sbi; + int ret; + const char *n1 = str; + const char *n2 = name->name; + unsigned int len2 = name->len; + unsigned int lm = min(len1, len2); + unsigned char c1, c2; + struct cpu_str *uni1; + struct le_str *uni2; + + /* First try fast implementation. */ + for (;;) { + if (!lm--) + return len1 != len2; + + if ((c1 = *n1++) == (c2 = *n2++)) + continue; + + if (c1 >= 0x80 || c2 >= 0x80) + break; + + if (toupper(c1) != toupper(c2)) + return 1; + } + + /* + * Try slow way with current upcase table + */ + sbi = dentry->d_sb->s_fs_info; + uni1 = kmalloc(PATH_MAX, GFP_NOWAIT); + if (!uni1) + return -ENOMEM; + + ret = ntfs_nls_to_utf16(sbi, str, len1, uni1, NTFS_NAME_LEN, + UTF16_HOST_ENDIAN); + if (ret < 0) + goto out; + + if (!ret) { + ret = -EINVAL; + goto out; + } + + uni2 = Add2Ptr(uni1, 2048); + + ret = ntfs_nls_to_utf16(sbi, name->name, name->len, + (struct cpu_str *)uni2, NTFS_NAME_LEN, + UTF16_LITTLE_ENDIAN); + if (ret < 0) + goto out; + + if (!ret) { + ret = -EINVAL; + goto out; + } + + ret = !ntfs_cmp_names_cpu(uni1, uni2, sbi->upcase, false) ? 0 : 1; + +out: + kfree(uni1); + return ret; +} + +// clang-format off +const struct inode_operations ntfs_dir_inode_operations = { + .lookup = ntfs_lookup, + .create = ntfs_create, + .link = ntfs_link, + .unlink = ntfs_unlink, + .symlink = ntfs_symlink, + .mkdir = ntfs_mkdir, + .rmdir = ntfs_rmdir, + .mknod = ntfs_mknod, + .rename = ntfs_rename, + .get_acl = ntfs_get_acl, + .set_acl = ntfs_set_acl, + .setattr = ntfs_setattr, + .getattr = ntfs_getattr, + .listxattr = ntfs_listxattr, + .fiemap = ntfs_fiemap, + .fileattr_get = ntfs_fileattr_get, + .fileattr_set = ntfs_fileattr_set, +}; + +const struct inode_operations ntfs_special_inode_operations = { + .setattr = ntfs_setattr, + .getattr = ntfs_getattr, + .listxattr = ntfs_listxattr, + .get_acl = ntfs_get_acl, + .set_acl = ntfs_set_acl, + .fileattr_get = ntfs_fileattr_get, + .fileattr_set = ntfs_fileattr_set, +}; + +const struct dentry_operations ntfs_dentry_ops = { + .d_hash = ntfs_d_hash, + .d_compare = ntfs_d_compare, +}; + +// clang-format on diff --git a/fs/ntfs3/ntfs.h b/fs/ntfs3/ntfs.h new file mode 100644 index 000000000..4589b1632 --- /dev/null +++ b/fs/ntfs3/ntfs.h @@ -0,0 +1,1235 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + * on-disk ntfs structs + */ + +// clang-format off +#ifndef _LINUX_NTFS3_NTFS_H +#define _LINUX_NTFS3_NTFS_H + +#include +#include +#include +#include +#include +#include + +#include "debug.h" + +/* TODO: Check 4K MFT record and 512 bytes cluster. */ + +/* Check each run for marked clusters. */ +#define NTFS3_CHECK_FREE_CLST + +#define NTFS_NAME_LEN 255 + +/* + * ntfs.sys used 500 maximum links on-disk struct allows up to 0xffff. + * xfstest generic/041 creates 3003 hardlinks. + */ +#define NTFS_LINK_MAX 4000 + +/* + * Activate to use 64 bit clusters instead of 32 bits in ntfs.sys. + * Logical and virtual cluster number if needed, may be + * redefined to use 64 bit value. + */ +//#define CONFIG_NTFS3_64BIT_CLUSTER + +#define NTFS_LZNT_MAX_CLUSTER 4096 +#define NTFS_LZNT_CUNIT 4 +#define NTFS_LZNT_CLUSTERS (1u<low) | ((u64)le16_to_cpu(ref->high) << 32); +#else + return le32_to_cpu(ref->low); +#endif +} + +struct NTFS_BOOT { + u8 jump_code[3]; // 0x00: Jump to boot code. + u8 system_id[8]; // 0x03: System ID, equals "NTFS " + + // NOTE: This member is not aligned(!) + // bytes_per_sector[0] must be 0. + // bytes_per_sector[1] must be multiplied by 256. + u8 bytes_per_sector[2]; // 0x0B: Bytes per sector. + + u8 sectors_per_clusters;// 0x0D: Sectors per cluster. + u8 unused1[7]; + u8 media_type; // 0x15: Media type (0xF8 - harddisk) + u8 unused2[2]; + __le16 sct_per_track; // 0x18: number of sectors per track. + __le16 heads; // 0x1A: number of heads per cylinder. + __le32 hidden_sectors; // 0x1C: number of 'hidden' sectors. + u8 unused3[4]; + u8 bios_drive_num; // 0x24: BIOS drive number =0x80. + u8 unused4; + u8 signature_ex; // 0x26: Extended BOOT signature =0x80. + u8 unused5; + __le64 sectors_per_volume;// 0x28: Size of volume in sectors. + __le64 mft_clst; // 0x30: First cluster of $MFT + __le64 mft2_clst; // 0x38: First cluster of $MFTMirr + s8 record_size; // 0x40: Size of MFT record in clusters(sectors). + u8 unused6[3]; + s8 index_size; // 0x44: Size of INDX record in clusters(sectors). + u8 unused7[3]; + __le64 serial_num; // 0x48: Volume serial number + __le32 check_sum; // 0x50: Simple additive checksum of all + // of the u32's which precede the 'check_sum'. + + u8 boot_code[0x200 - 0x50 - 2 - 4]; // 0x54: + u8 boot_magic[2]; // 0x1FE: Boot signature =0x55 + 0xAA +}; + +static_assert(sizeof(struct NTFS_BOOT) == 0x200); + +enum NTFS_SIGNATURE { + NTFS_FILE_SIGNATURE = cpu_to_le32(0x454C4946), // 'FILE' + NTFS_INDX_SIGNATURE = cpu_to_le32(0x58444E49), // 'INDX' + NTFS_CHKD_SIGNATURE = cpu_to_le32(0x444B4843), // 'CHKD' + NTFS_RSTR_SIGNATURE = cpu_to_le32(0x52545352), // 'RSTR' + NTFS_RCRD_SIGNATURE = cpu_to_le32(0x44524352), // 'RCRD' + NTFS_BAAD_SIGNATURE = cpu_to_le32(0x44414142), // 'BAAD' + NTFS_HOLE_SIGNATURE = cpu_to_le32(0x454C4F48), // 'HOLE' + NTFS_FFFF_SIGNATURE = cpu_to_le32(0xffffffff), +}; + +static_assert(sizeof(enum NTFS_SIGNATURE) == 4); + +/* MFT Record header structure. */ +struct NTFS_RECORD_HEADER { + /* Record magic number, equals 'FILE'/'INDX'/'RSTR'/'RCRD'. */ + enum NTFS_SIGNATURE sign; // 0x00: + __le16 fix_off; // 0x04: + __le16 fix_num; // 0x06: + __le64 lsn; // 0x08: Log file sequence number, +}; + +static_assert(sizeof(struct NTFS_RECORD_HEADER) == 0x10); + +static inline int is_baad(const struct NTFS_RECORD_HEADER *hdr) +{ + return hdr->sign == NTFS_BAAD_SIGNATURE; +} + +/* Possible bits in struct MFT_REC.flags. */ +enum RECORD_FLAG { + RECORD_FLAG_IN_USE = cpu_to_le16(0x0001), + RECORD_FLAG_DIR = cpu_to_le16(0x0002), + RECORD_FLAG_SYSTEM = cpu_to_le16(0x0004), + RECORD_FLAG_INDEX = cpu_to_le16(0x0008), +}; + +/* MFT Record structure. */ +struct MFT_REC { + struct NTFS_RECORD_HEADER rhdr; // 'FILE' + + __le16 seq; // 0x10: Sequence number for this record. + __le16 hard_links; // 0x12: The number of hard links to record. + __le16 attr_off; // 0x14: Offset to attributes. + __le16 flags; // 0x16: See RECORD_FLAG. + __le32 used; // 0x18: The size of used part. + __le32 total; // 0x1C: Total record size. + + struct MFT_REF parent_ref; // 0x20: Parent MFT record. + __le16 next_attr_id; // 0x28: The next attribute Id. + + __le16 res; // 0x2A: High part of MFT record? + __le32 mft_record; // 0x2C: Current MFT record number. + __le16 fixups[]; // 0x30: +}; + +#define MFTRECORD_FIXUP_OFFSET_1 offsetof(struct MFT_REC, res) +#define MFTRECORD_FIXUP_OFFSET_3 offsetof(struct MFT_REC, fixups) +/* + * define MFTRECORD_FIXUP_OFFSET as MFTRECORD_FIXUP_OFFSET_3 (0x30) + * to format new mft records with bigger header (as current ntfs.sys does) + * + * define MFTRECORD_FIXUP_OFFSET as MFTRECORD_FIXUP_OFFSET_1 (0x2A) + * to format new mft records with smaller header (as old ntfs.sys did) + * Both variants are valid. + */ +#define MFTRECORD_FIXUP_OFFSET MFTRECORD_FIXUP_OFFSET_1 + +static_assert(MFTRECORD_FIXUP_OFFSET_1 == 0x2A); +static_assert(MFTRECORD_FIXUP_OFFSET_3 == 0x30); + +static inline bool is_rec_base(const struct MFT_REC *rec) +{ + const struct MFT_REF *r = &rec->parent_ref; + + return !r->low && !r->high && !r->seq; +} + +static inline bool is_mft_rec5(const struct MFT_REC *rec) +{ + return le16_to_cpu(rec->rhdr.fix_off) >= + offsetof(struct MFT_REC, fixups); +} + +static inline bool is_rec_inuse(const struct MFT_REC *rec) +{ + return rec->flags & RECORD_FLAG_IN_USE; +} + +static inline bool clear_rec_inuse(struct MFT_REC *rec) +{ + return rec->flags &= ~RECORD_FLAG_IN_USE; +} + +/* Possible values of ATTR_RESIDENT.flags */ +#define RESIDENT_FLAG_INDEXED 0x01 + +struct ATTR_RESIDENT { + __le32 data_size; // 0x10: The size of data. + __le16 data_off; // 0x14: Offset to data. + u8 flags; // 0x16: Resident flags ( 1 - indexed ). + u8 res; // 0x17: +}; // sizeof() = 0x18 + +struct ATTR_NONRESIDENT { + __le64 svcn; // 0x10: Starting VCN of this segment. + __le64 evcn; // 0x18: End VCN of this segment. + __le16 run_off; // 0x20: Offset to packed runs. + // Unit of Compression size for this stream, expressed + // as a log of the cluster size. + // + // 0 means file is not compressed + // 1, 2, 3, and 4 are potentially legal values if the + // stream is compressed, however the implementation + // may only choose to use 4, or possibly 3. + // Note that 4 means cluster size time 16. + // If convenient the implementation may wish to accept a + // reasonable range of legal values here (1-5?), + // even if the implementation only generates + // a smaller set of values itself. + u8 c_unit; // 0x22: + u8 res1[5]; // 0x23: + __le64 alloc_size; // 0x28: The allocated size of attribute in bytes. + // (multiple of cluster size) + __le64 data_size; // 0x30: The size of attribute in bytes <= alloc_size. + __le64 valid_size; // 0x38: The size of valid part in bytes <= data_size. + __le64 total_size; // 0x40: The sum of the allocated clusters for a file. + // (present only for the first segment (0 == vcn) + // of compressed attribute) + +}; // sizeof()=0x40 or 0x48 (if compressed) + +/* Possible values of ATTRIB.flags: */ +#define ATTR_FLAG_COMPRESSED cpu_to_le16(0x0001) +#define ATTR_FLAG_COMPRESSED_MASK cpu_to_le16(0x00FF) +#define ATTR_FLAG_ENCRYPTED cpu_to_le16(0x4000) +#define ATTR_FLAG_SPARSED cpu_to_le16(0x8000) + +struct ATTRIB { + enum ATTR_TYPE type; // 0x00: The type of this attribute. + __le32 size; // 0x04: The size of this attribute. + u8 non_res; // 0x08: Is this attribute non-resident? + u8 name_len; // 0x09: This attribute name length. + __le16 name_off; // 0x0A: Offset to the attribute name. + __le16 flags; // 0x0C: See ATTR_FLAG_XXX. + __le16 id; // 0x0E: Unique id (per record). + + union { + struct ATTR_RESIDENT res; // 0x10 + struct ATTR_NONRESIDENT nres; // 0x10 + }; +}; + +/* Define attribute sizes. */ +#define SIZEOF_RESIDENT 0x18 +#define SIZEOF_NONRESIDENT_EX 0x48 +#define SIZEOF_NONRESIDENT 0x40 + +#define SIZEOF_RESIDENT_LE cpu_to_le16(0x18) +#define SIZEOF_NONRESIDENT_EX_LE cpu_to_le16(0x48) +#define SIZEOF_NONRESIDENT_LE cpu_to_le16(0x40) + +static inline u64 attr_ondisk_size(const struct ATTRIB *attr) +{ + return attr->non_res ? ((attr->flags & + (ATTR_FLAG_COMPRESSED | ATTR_FLAG_SPARSED)) ? + le64_to_cpu(attr->nres.total_size) : + le64_to_cpu(attr->nres.alloc_size)) + : ALIGN(le32_to_cpu(attr->res.data_size), 8); +} + +static inline u64 attr_size(const struct ATTRIB *attr) +{ + return attr->non_res ? le64_to_cpu(attr->nres.data_size) : + le32_to_cpu(attr->res.data_size); +} + +static inline bool is_attr_encrypted(const struct ATTRIB *attr) +{ + return attr->flags & ATTR_FLAG_ENCRYPTED; +} + +static inline bool is_attr_sparsed(const struct ATTRIB *attr) +{ + return attr->flags & ATTR_FLAG_SPARSED; +} + +static inline bool is_attr_compressed(const struct ATTRIB *attr) +{ + return attr->flags & ATTR_FLAG_COMPRESSED; +} + +static inline bool is_attr_ext(const struct ATTRIB *attr) +{ + return attr->flags & (ATTR_FLAG_SPARSED | ATTR_FLAG_COMPRESSED); +} + +static inline bool is_attr_indexed(const struct ATTRIB *attr) +{ + return !attr->non_res && (attr->res.flags & RESIDENT_FLAG_INDEXED); +} + +static inline __le16 const *attr_name(const struct ATTRIB *attr) +{ + return Add2Ptr(attr, le16_to_cpu(attr->name_off)); +} + +static inline u64 attr_svcn(const struct ATTRIB *attr) +{ + return attr->non_res ? le64_to_cpu(attr->nres.svcn) : 0; +} + +static_assert(sizeof(struct ATTRIB) == 0x48); +static_assert(sizeof(((struct ATTRIB *)NULL)->res) == 0x08); +static_assert(sizeof(((struct ATTRIB *)NULL)->nres) == 0x38); + +static inline void *resident_data_ex(const struct ATTRIB *attr, u32 datasize) +{ + u32 asize, rsize; + u16 off; + + if (attr->non_res) + return NULL; + + asize = le32_to_cpu(attr->size); + off = le16_to_cpu(attr->res.data_off); + + if (asize < datasize + off) + return NULL; + + rsize = le32_to_cpu(attr->res.data_size); + if (rsize < datasize) + return NULL; + + return Add2Ptr(attr, off); +} + +static inline void *resident_data(const struct ATTRIB *attr) +{ + return Add2Ptr(attr, le16_to_cpu(attr->res.data_off)); +} + +static inline void *attr_run(const struct ATTRIB *attr) +{ + return Add2Ptr(attr, le16_to_cpu(attr->nres.run_off)); +} + +/* Standard information attribute (0x10). */ +struct ATTR_STD_INFO { + __le64 cr_time; // 0x00: File creation file. + __le64 m_time; // 0x08: File modification time. + __le64 c_time; // 0x10: Last time any attribute was modified. + __le64 a_time; // 0x18: File last access time. + enum FILE_ATTRIBUTE fa; // 0x20: Standard DOS attributes & more. + __le32 max_ver_num; // 0x24: Maximum Number of Versions. + __le32 ver_num; // 0x28: Version Number. + __le32 class_id; // 0x2C: Class Id from bidirectional Class Id index. +}; + +static_assert(sizeof(struct ATTR_STD_INFO) == 0x30); + +#define SECURITY_ID_INVALID 0x00000000 +#define SECURITY_ID_FIRST 0x00000100 + +struct ATTR_STD_INFO5 { + __le64 cr_time; // 0x00: File creation file. + __le64 m_time; // 0x08: File modification time. + __le64 c_time; // 0x10: Last time any attribute was modified. + __le64 a_time; // 0x18: File last access time. + enum FILE_ATTRIBUTE fa; // 0x20: Standard DOS attributes & more. + __le32 max_ver_num; // 0x24: Maximum Number of Versions. + __le32 ver_num; // 0x28: Version Number. + __le32 class_id; // 0x2C: Class Id from bidirectional Class Id index. + + __le32 owner_id; // 0x30: Owner Id of the user owning the file. + __le32 security_id; // 0x34: The Security Id is a key in the $SII Index and $SDS. + __le64 quota_charge; // 0x38: + __le64 usn; // 0x40: Last Update Sequence Number of the file. This is a direct + // index into the file $UsnJrnl. If zero, the USN Journal is + // disabled. +}; + +static_assert(sizeof(struct ATTR_STD_INFO5) == 0x48); + +/* Attribute list entry structure (0x20) */ +struct ATTR_LIST_ENTRY { + enum ATTR_TYPE type; // 0x00: The type of attribute. + __le16 size; // 0x04: The size of this record. + u8 name_len; // 0x06: The length of attribute name. + u8 name_off; // 0x07: The offset to attribute name. + __le64 vcn; // 0x08: Starting VCN of this attribute. + struct MFT_REF ref; // 0x10: MFT record number with attribute. + __le16 id; // 0x18: struct ATTRIB ID. + __le16 name[]; // 0x1A: To get real name use name_off. + +}; // sizeof(0x20) + +static inline u32 le_size(u8 name_len) +{ + return ALIGN(offsetof(struct ATTR_LIST_ENTRY, name) + + name_len * sizeof(short), 8); +} + +/* Returns 0 if 'attr' has the same type and name. */ +static inline int le_cmp(const struct ATTR_LIST_ENTRY *le, + const struct ATTRIB *attr) +{ + return le->type != attr->type || le->name_len != attr->name_len || + (!le->name_len && + memcmp(Add2Ptr(le, le->name_off), + Add2Ptr(attr, le16_to_cpu(attr->name_off)), + le->name_len * sizeof(short))); +} + +static inline __le16 const *le_name(const struct ATTR_LIST_ENTRY *le) +{ + return Add2Ptr(le, le->name_off); +} + +/* File name types (the field type in struct ATTR_FILE_NAME). */ +#define FILE_NAME_POSIX 0 +#define FILE_NAME_UNICODE 1 +#define FILE_NAME_DOS 2 +#define FILE_NAME_UNICODE_AND_DOS (FILE_NAME_DOS | FILE_NAME_UNICODE) + +/* Filename attribute structure (0x30). */ +struct NTFS_DUP_INFO { + __le64 cr_time; // 0x00: File creation file. + __le64 m_time; // 0x08: File modification time. + __le64 c_time; // 0x10: Last time any attribute was modified. + __le64 a_time; // 0x18: File last access time. + __le64 alloc_size; // 0x20: Data attribute allocated size, multiple of cluster size. + __le64 data_size; // 0x28: Data attribute size <= Dataalloc_size. + enum FILE_ATTRIBUTE fa; // 0x30: Standard DOS attributes & more. + __le32 extend_data; // 0x34: Extended data. + +}; // 0x38 + +struct ATTR_FILE_NAME { + struct MFT_REF home; // 0x00: MFT record for directory. + struct NTFS_DUP_INFO dup;// 0x08: + u8 name_len; // 0x40: File name length in words. + u8 type; // 0x41: File name type. + __le16 name[]; // 0x42: File name. +}; + +static_assert(sizeof(((struct ATTR_FILE_NAME *)NULL)->dup) == 0x38); +static_assert(offsetof(struct ATTR_FILE_NAME, name) == 0x42); +#define SIZEOF_ATTRIBUTE_FILENAME 0x44 +#define SIZEOF_ATTRIBUTE_FILENAME_MAX (0x42 + 255 * 2) + +static inline struct ATTRIB *attr_from_name(struct ATTR_FILE_NAME *fname) +{ + return (struct ATTRIB *)((char *)fname - SIZEOF_RESIDENT); +} + +static inline u16 fname_full_size(const struct ATTR_FILE_NAME *fname) +{ + /* Don't return struct_size(fname, name, fname->name_len); */ + return offsetof(struct ATTR_FILE_NAME, name) + + fname->name_len * sizeof(short); +} + +static inline u8 paired_name(u8 type) +{ + if (type == FILE_NAME_UNICODE) + return FILE_NAME_DOS; + if (type == FILE_NAME_DOS) + return FILE_NAME_UNICODE; + return FILE_NAME_POSIX; +} + +/* Index entry defines ( the field flags in NtfsDirEntry ). */ +#define NTFS_IE_HAS_SUBNODES cpu_to_le16(1) +#define NTFS_IE_LAST cpu_to_le16(2) + +/* Directory entry structure. */ +struct NTFS_DE { + union { + struct MFT_REF ref; // 0x00: MFT record number with this file. + struct { + __le16 data_off; // 0x00: + __le16 data_size; // 0x02: + __le32 res; // 0x04: Must be 0. + } view; + }; + __le16 size; // 0x08: The size of this entry. + __le16 key_size; // 0x0A: The size of File name length in bytes + 0x42. + __le16 flags; // 0x0C: Entry flags: NTFS_IE_XXX. + __le16 res; // 0x0E: + + // Here any indexed attribute can be placed. + // One of them is: + // struct ATTR_FILE_NAME AttrFileName; + // + + // The last 8 bytes of this structure contains + // the VBN of subnode. + // !!! Note !!! + // This field is presented only if (flags & NTFS_IE_HAS_SUBNODES) + // __le64 vbn; +}; + +static_assert(sizeof(struct NTFS_DE) == 0x10); + +static inline void de_set_vbn_le(struct NTFS_DE *e, __le64 vcn) +{ + __le64 *v = Add2Ptr(e, le16_to_cpu(e->size) - sizeof(__le64)); + + *v = vcn; +} + +static inline void de_set_vbn(struct NTFS_DE *e, CLST vcn) +{ + __le64 *v = Add2Ptr(e, le16_to_cpu(e->size) - sizeof(__le64)); + + *v = cpu_to_le64(vcn); +} + +static inline __le64 de_get_vbn_le(const struct NTFS_DE *e) +{ + return *(__le64 *)Add2Ptr(e, le16_to_cpu(e->size) - sizeof(__le64)); +} + +static inline CLST de_get_vbn(const struct NTFS_DE *e) +{ + __le64 *v = Add2Ptr(e, le16_to_cpu(e->size) - sizeof(__le64)); + + return le64_to_cpu(*v); +} + +static inline struct NTFS_DE *de_get_next(const struct NTFS_DE *e) +{ + return Add2Ptr(e, le16_to_cpu(e->size)); +} + +static inline struct ATTR_FILE_NAME *de_get_fname(const struct NTFS_DE *e) +{ + return le16_to_cpu(e->key_size) >= SIZEOF_ATTRIBUTE_FILENAME ? + Add2Ptr(e, sizeof(struct NTFS_DE)) : + NULL; +} + +static inline bool de_is_last(const struct NTFS_DE *e) +{ + return e->flags & NTFS_IE_LAST; +} + +static inline bool de_has_vcn(const struct NTFS_DE *e) +{ + return e->flags & NTFS_IE_HAS_SUBNODES; +} + +static inline bool de_has_vcn_ex(const struct NTFS_DE *e) +{ + return (e->flags & NTFS_IE_HAS_SUBNODES) && + (u64)(-1) != *((u64 *)Add2Ptr(e, le16_to_cpu(e->size) - + sizeof(__le64))); +} + +#define MAX_BYTES_PER_NAME_ENTRY \ + ALIGN(sizeof(struct NTFS_DE) + \ + offsetof(struct ATTR_FILE_NAME, name) + \ + NTFS_NAME_LEN * sizeof(short), 8) + +#define NTFS_INDEX_HDR_HAS_SUBNODES cpu_to_le32(1) + +struct INDEX_HDR { + __le32 de_off; // 0x00: The offset from the start of this structure + // to the first NTFS_DE. + __le32 used; // 0x04: The size of this structure plus all + // entries (quad-word aligned). + __le32 total; // 0x08: The allocated size of for this structure plus all entries. + __le32 flags; // 0x0C: 0x00 = Small directory, 0x01 = Large directory. + + // + // de_off + used <= total + // +}; + +static_assert(sizeof(struct INDEX_HDR) == 0x10); + +static inline struct NTFS_DE *hdr_first_de(const struct INDEX_HDR *hdr) +{ + u32 de_off = le32_to_cpu(hdr->de_off); + u32 used = le32_to_cpu(hdr->used); + struct NTFS_DE *e; + u16 esize; + + if (de_off >= used || size_add(de_off, sizeof(struct NTFS_DE)) > used) + return NULL; + + e = Add2Ptr(hdr, de_off); + esize = le16_to_cpu(e->size); + if (esize < sizeof(struct NTFS_DE) || de_off + esize > used) + return NULL; + + return e; +} + +static inline struct NTFS_DE *hdr_next_de(const struct INDEX_HDR *hdr, + const struct NTFS_DE *e) +{ + size_t off = PtrOffset(hdr, e); + u32 used = le32_to_cpu(hdr->used); + u16 esize; + + if (off >= used) + return NULL; + + esize = le16_to_cpu(e->size); + + if (esize < sizeof(struct NTFS_DE) || + off + esize + sizeof(struct NTFS_DE) > used) + return NULL; + + return Add2Ptr(e, esize); +} + +static inline bool hdr_has_subnode(const struct INDEX_HDR *hdr) +{ + return hdr->flags & NTFS_INDEX_HDR_HAS_SUBNODES; +} + +struct INDEX_BUFFER { + struct NTFS_RECORD_HEADER rhdr; // 'INDX' + __le64 vbn; // 0x10: vcn if index >= cluster or vsn id index < cluster + struct INDEX_HDR ihdr; // 0x18: +}; + +static_assert(sizeof(struct INDEX_BUFFER) == 0x28); + +static inline bool ib_is_empty(const struct INDEX_BUFFER *ib) +{ + const struct NTFS_DE *first = hdr_first_de(&ib->ihdr); + + return !first || de_is_last(first); +} + +static inline bool ib_is_leaf(const struct INDEX_BUFFER *ib) +{ + return !(ib->ihdr.flags & NTFS_INDEX_HDR_HAS_SUBNODES); +} + +/* Index root structure ( 0x90 ). */ +enum COLLATION_RULE { + NTFS_COLLATION_TYPE_BINARY = cpu_to_le32(0), + // $I30 + NTFS_COLLATION_TYPE_FILENAME = cpu_to_le32(0x01), + // $SII of $Secure and $Q of Quota + NTFS_COLLATION_TYPE_UINT = cpu_to_le32(0x10), + // $O of Quota + NTFS_COLLATION_TYPE_SID = cpu_to_le32(0x11), + // $SDH of $Secure + NTFS_COLLATION_TYPE_SECURITY_HASH = cpu_to_le32(0x12), + // $O of ObjId and "$R" for Reparse + NTFS_COLLATION_TYPE_UINTS = cpu_to_le32(0x13) +}; + +static_assert(sizeof(enum COLLATION_RULE) == 4); + +// +struct INDEX_ROOT { + enum ATTR_TYPE type; // 0x00: The type of attribute to index on. + enum COLLATION_RULE rule; // 0x04: The rule. + __le32 index_block_size;// 0x08: The size of index record. + u8 index_block_clst; // 0x0C: The number of clusters or sectors per index. + u8 res[3]; + struct INDEX_HDR ihdr; // 0x10: +}; + +static_assert(sizeof(struct INDEX_ROOT) == 0x20); +static_assert(offsetof(struct INDEX_ROOT, ihdr) == 0x10); + +#define VOLUME_FLAG_DIRTY cpu_to_le16(0x0001) +#define VOLUME_FLAG_RESIZE_LOG_FILE cpu_to_le16(0x0002) + +struct VOLUME_INFO { + __le64 res1; // 0x00 + u8 major_ver; // 0x08: NTFS major version number (before .) + u8 minor_ver; // 0x09: NTFS minor version number (after .) + __le16 flags; // 0x0A: Volume flags, see VOLUME_FLAG_XXX + +}; // sizeof=0xC + +#define SIZEOF_ATTRIBUTE_VOLUME_INFO 0xc + +#define NTFS_LABEL_MAX_LENGTH (0x100 / sizeof(short)) +#define NTFS_ATTR_INDEXABLE cpu_to_le32(0x00000002) +#define NTFS_ATTR_DUPALLOWED cpu_to_le32(0x00000004) +#define NTFS_ATTR_MUST_BE_INDEXED cpu_to_le32(0x00000010) +#define NTFS_ATTR_MUST_BE_NAMED cpu_to_le32(0x00000020) +#define NTFS_ATTR_MUST_BE_RESIDENT cpu_to_le32(0x00000040) +#define NTFS_ATTR_LOG_ALWAYS cpu_to_le32(0x00000080) + +/* $AttrDef file entry. */ +struct ATTR_DEF_ENTRY { + __le16 name[0x40]; // 0x00: Attr name. + enum ATTR_TYPE type; // 0x80: struct ATTRIB type. + __le32 res; // 0x84: + enum COLLATION_RULE rule; // 0x88: + __le32 flags; // 0x8C: NTFS_ATTR_XXX (see above). + __le64 min_sz; // 0x90: Minimum attribute data size. + __le64 max_sz; // 0x98: Maximum attribute data size. +}; + +static_assert(sizeof(struct ATTR_DEF_ENTRY) == 0xa0); + +/* Object ID (0x40) */ +struct OBJECT_ID { + struct GUID ObjId; // 0x00: Unique Id assigned to file. + + // Birth Volume Id is the Object Id of the Volume on. + // which the Object Id was allocated. It never changes. + struct GUID BirthVolumeId; //0x10: + + // Birth Object Id is the first Object Id that was + // ever assigned to this MFT Record. I.e. If the Object Id + // is changed for some reason, this field will reflect the + // original value of the Object Id. + struct GUID BirthObjectId; // 0x20: + + // Domain Id is currently unused but it is intended to be + // used in a network environment where the local machine is + // part of a Windows 2000 Domain. This may be used in a Windows + // 2000 Advanced Server managed domain. + struct GUID DomainId; // 0x30: +}; + +static_assert(sizeof(struct OBJECT_ID) == 0x40); + +/* O Directory entry structure ( rule = 0x13 ) */ +struct NTFS_DE_O { + struct NTFS_DE de; + struct GUID ObjId; // 0x10: Unique Id assigned to file. + struct MFT_REF ref; // 0x20: MFT record number with this file. + + // Birth Volume Id is the Object Id of the Volume on + // which the Object Id was allocated. It never changes. + struct GUID BirthVolumeId; // 0x28: + + // Birth Object Id is the first Object Id that was + // ever assigned to this MFT Record. I.e. If the Object Id + // is changed for some reason, this field will reflect the + // original value of the Object Id. + // This field is valid if data_size == 0x48. + struct GUID BirthObjectId; // 0x38: + + // Domain Id is currently unused but it is intended + // to be used in a network environment where the local + // machine is part of a Windows 2000 Domain. This may be + // used in a Windows 2000 Advanced Server managed domain. + struct GUID BirthDomainId; // 0x48: +}; + +static_assert(sizeof(struct NTFS_DE_O) == 0x58); + +/* Q Directory entry structure ( rule = 0x11 ) */ +struct NTFS_DE_Q { + struct NTFS_DE de; + __le32 owner_id; // 0x10: Unique Id assigned to file + + /* here is 0x30 bytes of user quota. NOTE: 4 byte aligned! */ + __le32 Version; // 0x14: 0x02 + __le32 Flags; // 0x18: Quota flags, see above + __le64 BytesUsed; // 0x1C: + __le64 ChangeTime; // 0x24: + __le64 WarningLimit; // 0x28: + __le64 HardLimit; // 0x34: + __le64 ExceededTime; // 0x3C: + + // SID is placed here +}__packed; // sizeof() = 0x44 + +static_assert(sizeof(struct NTFS_DE_Q) == 0x44); + +#define SecurityDescriptorsBlockSize 0x40000 // 256K +#define SecurityDescriptorMaxSize 0x20000 // 128K +#define Log2OfSecurityDescriptorsBlockSize 18 + +struct SECURITY_KEY { + __le32 hash; // Hash value for descriptor + __le32 sec_id; // Security Id (guaranteed unique) +}; + +/* Security descriptors (the content of $Secure::SDS data stream) */ +struct SECURITY_HDR { + struct SECURITY_KEY key; // 0x00: Security Key. + __le64 off; // 0x08: Offset of this entry in the file. + __le32 size; // 0x10: Size of this entry, 8 byte aligned. + /* + * Security descriptor itself is placed here. + * Total size is 16 byte aligned. + */ +} __packed; + +static_assert(sizeof(struct SECURITY_HDR) == 0x14); + +/* SII Directory entry structure */ +struct NTFS_DE_SII { + struct NTFS_DE de; + __le32 sec_id; // 0x10: Key: sizeof(security_id) = wKeySize + struct SECURITY_HDR sec_hdr; // 0x14: +} __packed; + +static_assert(offsetof(struct NTFS_DE_SII, sec_hdr) == 0x14); +static_assert(sizeof(struct NTFS_DE_SII) == 0x28); + +/* SDH Directory entry structure */ +struct NTFS_DE_SDH { + struct NTFS_DE de; + struct SECURITY_KEY key; // 0x10: Key + struct SECURITY_HDR sec_hdr; // 0x18: Data + __le16 magic[2]; // 0x2C: 0x00490049 "I I" +}; + +#define SIZEOF_SDH_DIRENTRY 0x30 + +struct REPARSE_KEY { + __le32 ReparseTag; // 0x00: Reparse Tag + struct MFT_REF ref; // 0x04: MFT record number with this file +}; // sizeof() = 0x0C + +static_assert(offsetof(struct REPARSE_KEY, ref) == 0x04); +#define SIZEOF_REPARSE_KEY 0x0C + +/* Reparse Directory entry structure */ +struct NTFS_DE_R { + struct NTFS_DE de; + struct REPARSE_KEY key; // 0x10: Reparse Key. + u32 zero; // 0x1c: +}; // sizeof() = 0x20 + +static_assert(sizeof(struct NTFS_DE_R) == 0x20); + +/* CompressReparseBuffer.WofVersion */ +#define WOF_CURRENT_VERSION cpu_to_le32(1) +/* CompressReparseBuffer.WofProvider */ +#define WOF_PROVIDER_WIM cpu_to_le32(1) +/* CompressReparseBuffer.WofProvider */ +#define WOF_PROVIDER_SYSTEM cpu_to_le32(2) +/* CompressReparseBuffer.ProviderVer */ +#define WOF_PROVIDER_CURRENT_VERSION cpu_to_le32(1) + +#define WOF_COMPRESSION_XPRESS4K cpu_to_le32(0) // 4k +#define WOF_COMPRESSION_LZX32K cpu_to_le32(1) // 32k +#define WOF_COMPRESSION_XPRESS8K cpu_to_le32(2) // 8k +#define WOF_COMPRESSION_XPRESS16K cpu_to_le32(3) // 16k + +/* + * ATTR_REPARSE (0xC0) + * + * The reparse struct GUID structure is used by all 3rd party layered drivers to + * store data in a reparse point. For non-Microsoft tags, The struct GUID field + * cannot be GUID_NULL. + * The constraints on reparse tags are defined below. + * Microsoft tags can also be used with this format of the reparse point buffer. + */ +struct REPARSE_POINT { + __le32 ReparseTag; // 0x00: + __le16 ReparseDataLength;// 0x04: + __le16 Reserved; + + struct GUID Guid; // 0x08: + + // + // Here GenericReparseBuffer is placed + // +}; + +static_assert(sizeof(struct REPARSE_POINT) == 0x18); + +/* + * The value of the following constant needs to satisfy the following + * conditions: + * (1) Be at least as large as the largest of the reserved tags. + * (2) Be strictly smaller than all the tags in use. + */ +#define IO_REPARSE_TAG_RESERVED_RANGE 1 + +/* + * The reparse tags are a ULONG. The 32 bits are laid out as follows: + * + * 3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 + * 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 + * +-+-+-+-+-----------------------+-------------------------------+ + * |M|R|N|R| Reserved bits | Reparse Tag Value | + * +-+-+-+-+-----------------------+-------------------------------+ + * + * M is the Microsoft bit. When set to 1, it denotes a tag owned by Microsoft. + * All ISVs must use a tag with a 0 in this position. + * Note: If a Microsoft tag is used by non-Microsoft software, the + * behavior is not defined. + * + * R is reserved. Must be zero for non-Microsoft tags. + * + * N is name surrogate. When set to 1, the file represents another named + * entity in the system. + * + * The M and N bits are OR-able. + * The following macros check for the M and N bit values: + */ + +/* + * Macro to determine whether a reparse point tag corresponds to a tag + * owned by Microsoft. + */ +#define IsReparseTagMicrosoft(_tag) (((_tag)&IO_REPARSE_TAG_MICROSOFT)) + +/* Macro to determine whether a reparse point tag is a name surrogate. */ +#define IsReparseTagNameSurrogate(_tag) (((_tag)&IO_REPARSE_TAG_NAME_SURROGATE)) + +/* + * The following constant represents the bits that are valid to use in + * reparse tags. + */ +#define IO_REPARSE_TAG_VALID_VALUES 0xF000FFFF + +/* + * Macro to determine whether a reparse tag is a valid tag. + */ +#define IsReparseTagValid(_tag) \ + (!((_tag) & ~IO_REPARSE_TAG_VALID_VALUES) && \ + ((_tag) > IO_REPARSE_TAG_RESERVED_RANGE)) + +/* Microsoft tags for reparse points. */ + +enum IO_REPARSE_TAG { + IO_REPARSE_TAG_SYMBOLIC_LINK = cpu_to_le32(0), + IO_REPARSE_TAG_NAME_SURROGATE = cpu_to_le32(0x20000000), + IO_REPARSE_TAG_MICROSOFT = cpu_to_le32(0x80000000), + IO_REPARSE_TAG_MOUNT_POINT = cpu_to_le32(0xA0000003), + IO_REPARSE_TAG_SYMLINK = cpu_to_le32(0xA000000C), + IO_REPARSE_TAG_HSM = cpu_to_le32(0xC0000004), + IO_REPARSE_TAG_SIS = cpu_to_le32(0x80000007), + IO_REPARSE_TAG_DEDUP = cpu_to_le32(0x80000013), + IO_REPARSE_TAG_COMPRESS = cpu_to_le32(0x80000017), + + /* + * The reparse tag 0x80000008 is reserved for Microsoft internal use. + * May be published in the future. + */ + + /* Microsoft reparse tag reserved for DFS */ + IO_REPARSE_TAG_DFS = cpu_to_le32(0x8000000A), + + /* Microsoft reparse tag reserved for the file system filter manager. */ + IO_REPARSE_TAG_FILTER_MANAGER = cpu_to_le32(0x8000000B), + + /* Non-Microsoft tags for reparse points */ + + /* Tag allocated to CONGRUENT, May 2000. Used by IFSTEST. */ + IO_REPARSE_TAG_IFSTEST_CONGRUENT = cpu_to_le32(0x00000009), + + /* Tag allocated to ARKIVIO. */ + IO_REPARSE_TAG_ARKIVIO = cpu_to_le32(0x0000000C), + + /* Tag allocated to SOLUTIONSOFT. */ + IO_REPARSE_TAG_SOLUTIONSOFT = cpu_to_le32(0x2000000D), + + /* Tag allocated to COMMVAULT. */ + IO_REPARSE_TAG_COMMVAULT = cpu_to_le32(0x0000000E), + + /* OneDrive?? */ + IO_REPARSE_TAG_CLOUD = cpu_to_le32(0x9000001A), + IO_REPARSE_TAG_CLOUD_1 = cpu_to_le32(0x9000101A), + IO_REPARSE_TAG_CLOUD_2 = cpu_to_le32(0x9000201A), + IO_REPARSE_TAG_CLOUD_3 = cpu_to_le32(0x9000301A), + IO_REPARSE_TAG_CLOUD_4 = cpu_to_le32(0x9000401A), + IO_REPARSE_TAG_CLOUD_5 = cpu_to_le32(0x9000501A), + IO_REPARSE_TAG_CLOUD_6 = cpu_to_le32(0x9000601A), + IO_REPARSE_TAG_CLOUD_7 = cpu_to_le32(0x9000701A), + IO_REPARSE_TAG_CLOUD_8 = cpu_to_le32(0x9000801A), + IO_REPARSE_TAG_CLOUD_9 = cpu_to_le32(0x9000901A), + IO_REPARSE_TAG_CLOUD_A = cpu_to_le32(0x9000A01A), + IO_REPARSE_TAG_CLOUD_B = cpu_to_le32(0x9000B01A), + IO_REPARSE_TAG_CLOUD_C = cpu_to_le32(0x9000C01A), + IO_REPARSE_TAG_CLOUD_D = cpu_to_le32(0x9000D01A), + IO_REPARSE_TAG_CLOUD_E = cpu_to_le32(0x9000E01A), + IO_REPARSE_TAG_CLOUD_F = cpu_to_le32(0x9000F01A), + +}; + +#define SYMLINK_FLAG_RELATIVE 1 + +/* Microsoft reparse buffer. (see DDK for details) */ +struct REPARSE_DATA_BUFFER { + __le32 ReparseTag; // 0x00: + __le16 ReparseDataLength; // 0x04: + __le16 Reserved; + + union { + /* If ReparseTag == 0xA0000003 (IO_REPARSE_TAG_MOUNT_POINT) */ + struct { + __le16 SubstituteNameOffset; // 0x08 + __le16 SubstituteNameLength; // 0x0A + __le16 PrintNameOffset; // 0x0C + __le16 PrintNameLength; // 0x0E + __le16 PathBuffer[]; // 0x10 + } MountPointReparseBuffer; + + /* + * If ReparseTag == 0xA000000C (IO_REPARSE_TAG_SYMLINK) + * https://msdn.microsoft.com/en-us/library/cc232006.aspx + */ + struct { + __le16 SubstituteNameOffset; // 0x08 + __le16 SubstituteNameLength; // 0x0A + __le16 PrintNameOffset; // 0x0C + __le16 PrintNameLength; // 0x0E + // 0-absolute path 1- relative path, SYMLINK_FLAG_RELATIVE + __le32 Flags; // 0x10 + __le16 PathBuffer[]; // 0x14 + } SymbolicLinkReparseBuffer; + + /* If ReparseTag == 0x80000017U */ + struct { + __le32 WofVersion; // 0x08 == 1 + /* + * 1 - WIM backing provider ("WIMBoot"), + * 2 - System compressed file provider + */ + __le32 WofProvider; // 0x0C: + __le32 ProviderVer; // 0x10: == 1 WOF_FILE_PROVIDER_CURRENT_VERSION == 1 + __le32 CompressionFormat; // 0x14: 0, 1, 2, 3. See WOF_COMPRESSION_XXX + } CompressReparseBuffer; + + struct { + u8 DataBuffer[1]; // 0x08: + } GenericReparseBuffer; + }; +}; + +/* ATTR_EA_INFO (0xD0) */ + +#define FILE_NEED_EA 0x80 // See ntifs.h +/* + * FILE_NEED_EA, indicates that the file to which the EA belongs cannot be + * interpreted without understanding the associated extended attributes. + */ +struct EA_INFO { + __le16 size_pack; // 0x00: Size of buffer to hold in packed form. + __le16 count; // 0x02: Count of EA's with FILE_NEED_EA bit set. + __le32 size; // 0x04: Size of buffer to hold in unpacked form. +}; + +static_assert(sizeof(struct EA_INFO) == 8); + +/* ATTR_EA (0xE0) */ +struct EA_FULL { + __le32 size; // 0x00: (not in packed) + u8 flags; // 0x04: + u8 name_len; // 0x05: + __le16 elength; // 0x06: + u8 name[]; // 0x08: +}; + +static_assert(offsetof(struct EA_FULL, name) == 8); + +#define ACL_REVISION 2 +#define ACL_REVISION_DS 4 + +#define SE_SELF_RELATIVE cpu_to_le16(0x8000) + +struct SECURITY_DESCRIPTOR_RELATIVE { + u8 Revision; + u8 Sbz1; + __le16 Control; + __le32 Owner; + __le32 Group; + __le32 Sacl; + __le32 Dacl; +}; +static_assert(sizeof(struct SECURITY_DESCRIPTOR_RELATIVE) == 0x14); + +struct ACE_HEADER { + u8 AceType; + u8 AceFlags; + __le16 AceSize; +}; +static_assert(sizeof(struct ACE_HEADER) == 4); + +struct ACL { + u8 AclRevision; + u8 Sbz1; + __le16 AclSize; + __le16 AceCount; + __le16 Sbz2; +}; +static_assert(sizeof(struct ACL) == 8); + +struct SID { + u8 Revision; + u8 SubAuthorityCount; + u8 IdentifierAuthority[6]; + __le32 SubAuthority[]; +}; +static_assert(offsetof(struct SID, SubAuthority) == 8); + +#endif /* _LINUX_NTFS3_NTFS_H */ +// clang-format on diff --git a/fs/ntfs3/ntfs_fs.h b/fs/ntfs3/ntfs_fs.h new file mode 100644 index 000000000..5811d89d6 --- /dev/null +++ b/fs/ntfs3/ntfs_fs.h @@ -0,0 +1,1283 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + */ + +// clang-format off +#ifndef _LINUX_NTFS3_NTFS_FS_H +#define _LINUX_NTFS3_NTFS_FS_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" + +struct dentry; +struct fiemap_extent_info; +struct user_namespace; +struct page; +struct writeback_control; +enum utf16_endian; + + +#define MINUS_ONE_T ((size_t)(-1)) +/* Biggest MFT / smallest cluster */ +#define MAXIMUM_BYTES_PER_MFT 4096 +#define MAXIMUM_SHIFT_BYTES_PER_MFT 12 +#define NTFS_BLOCKS_PER_MFT_RECORD (MAXIMUM_BYTES_PER_MFT / 512) + +#define MAXIMUM_BYTES_PER_INDEX 4096 +#define MAXIMUM_SHIFT_BYTES_PER_INDEX 12 +#define NTFS_BLOCKS_PER_INODE (MAXIMUM_BYTES_PER_INDEX / 512) + +/* NTFS specific error code when fixup failed. */ +#define E_NTFS_FIXUP 555 +/* NTFS specific error code about resident->nonresident. */ +#define E_NTFS_NONRESIDENT 556 +/* NTFS specific error code about punch hole. */ +#define E_NTFS_NOTALIGNED 557 +/* NTFS specific error code when on-disk struct is corrupted. */ +#define E_NTFS_CORRUPT 558 + + +/* sbi->flags */ +#define NTFS_FLAGS_NODISCARD 0x00000001 +/* ntfs in shutdown state. */ +#define NTFS_FLAGS_SHUTDOWN_BIT 0x00000002 /* == 4*/ +/* Set when LogFile is replaying. */ +#define NTFS_FLAGS_LOG_REPLAYING 0x00000008 +/* Set when we changed first MFT's which copy must be updated in $MftMirr. */ +#define NTFS_FLAGS_MFTMIRR 0x00001000 +#define NTFS_FLAGS_NEED_REPLAY 0x04000000 + + +/* ni->ni_flags */ +/* + * Data attribute is external compressed (LZX/Xpress) + * 1 - WOF_COMPRESSION_XPRESS4K + * 2 - WOF_COMPRESSION_XPRESS8K + * 3 - WOF_COMPRESSION_XPRESS16K + * 4 - WOF_COMPRESSION_LZX32K + */ +#define NI_FLAG_COMPRESSED_MASK 0x0000000f +/* Data attribute is deduplicated. */ +#define NI_FLAG_DEDUPLICATED 0x00000010 +#define NI_FLAG_EA 0x00000020 +#define NI_FLAG_DIR 0x00000040 +#define NI_FLAG_RESIDENT 0x00000080 +#define NI_FLAG_UPDATE_PARENT 0x00000100 +// clang-format on + +struct ntfs_mount_options { + char *nls_name; + struct nls_table *nls; + + kuid_t fs_uid; + kgid_t fs_gid; + u16 fs_fmask_inv; + u16 fs_dmask_inv; + + unsigned fmask : 1; /* fmask was set. */ + unsigned dmask : 1; /*dmask was set. */ + unsigned sys_immutable : 1; /* Immutable system files. */ + unsigned discard : 1; /* Issue discard requests on deletions. */ + unsigned sparse : 1; /* Create sparse files. */ + unsigned showmeta : 1; /* Show meta files. */ + unsigned nohidden : 1; /* Do not show hidden files. */ + unsigned hide_dot_files : 1; /* Set hidden flag on dot files. */ + unsigned windows_names : 1; /* Disallow names forbidden by Windows. */ + unsigned force : 1; /* RW mount dirty volume. */ + unsigned prealloc : 1; /* Preallocate space when file is growing. */ + unsigned nocase : 1; /* case insensitive. */ + unsigned delalloc : 1; /* delay allocation. */ + unsigned ads : 1; /* ads support. */ +}; + +/* Special value to unpack and deallocate. */ +#define RUN_DEALLOCATE ((struct runs_tree *)(size_t)1) + +/* TODO: Use rb tree instead of array. */ +struct runs_tree { + struct ntfs_run *runs; + size_t count; /* Currently used size a ntfs_run storage. */ + size_t allocated; /* Currently allocated ntfs_run storage size. */ +}; + +struct ntfs_buffers { + /* Biggest MFT / smallest cluster = 4096 / 512 = 8 */ + /* Biggest index / smallest cluster = 4096 / 512 = 8 */ + struct buffer_head *bh[PAGE_SIZE >> SECTOR_SHIFT]; + u32 bytes; + u32 nbufs; + u32 off; +}; + +enum ALLOCATE_OPT { + ALLOCATE_DEF = 0, // Allocate all clusters. + ALLOCATE_MFT = 1, // Allocate for MFT. + ALLOCATE_ZERO = 2, // Zeroout new allocated clusters. + ALLOCATE_ONE_FR = 4, // Allocate one fragment only. +}; + +enum bitmap_mutex_classes { + BITMAP_MUTEX_CLUSTERS = 0, + BITMAP_MUTEX_MFT = 1, +}; + +struct wnd_bitmap { + struct super_block *sb; + struct rw_semaphore rw_lock; + + struct runs_tree run; + size_t nbits; + + size_t total_zeroes; // Total number of free bits. + u16 *free_bits; // Free bits in each window. + size_t nwnd; + u32 bits_last; // Bits in last window. + + struct rb_root start_tree; // Extents, sorted by 'start'. + struct rb_root count_tree; // Extents, sorted by 'count + start'. + size_t count; // Extents count. + + /* + * -1 Tree is activated but not updated (too many fragments). + * 0 - Tree is not activated. + * 1 - Tree is activated and updated. + */ + int uptodated; + size_t extent_min; // Minimal extent used while building. + size_t extent_max; // Upper estimate of biggest free block. + + /* Zone [bit, end) */ + size_t zone_bit; + size_t zone_end; + + bool inited; +}; + +typedef int (*NTFS_CMP_FUNC)(const void *key1, size_t len1, const void *key2, + size_t len2, const void *param); + +enum index_mutex_classed { + INDEX_MUTEX_I30 = 0, + INDEX_MUTEX_SII = 1, + INDEX_MUTEX_SDH = 2, + INDEX_MUTEX_SO = 3, + INDEX_MUTEX_SQ = 4, + INDEX_MUTEX_SR = 5, + INDEX_MUTEX_TOTAL +}; + +/* ntfs_index - Allocation unit inside directory. */ +struct ntfs_index { + struct runs_tree bitmap_run; + struct runs_tree alloc_run; + /* read/write access to 'bitmap_run'/'alloc_run' while ntfs_readdir */ + struct rw_semaphore run_lock; + size_t version; /* increment each change */ + + u8 index_bits; // log2(root->index_block_size) + u8 idx2vbn_bits; // log2(root->index_block_clst) + u8 vbn2vbo_bits; // index_block_size < cluster? 9 : cluster_bits + u8 type; // index_mutex_classed +}; + +/* Minimum MFT zone. */ +#define NTFS_MIN_MFT_ZONE 100 +/* Step to increase the MFT. */ +#define NTFS_MFT_INCREASE_STEP 1024 + +/* Ntfs file system in-core superblock data. */ +struct ntfs_sb_info { + struct super_block *sb; + + u32 discard_granularity; + u64 discard_granularity_mask_inv; // ~(discard_granularity_mask_inv-1) + u32 bdev_blocksize; // bdev_logical_block_size(bdev) + + u32 cluster_size; // bytes per cluster + u32 cluster_mask; // == cluster_size - 1 + u64 cluster_mask_inv; // ~(cluster_size - 1) + u32 block_mask; // sb->s_blocksize - 1 + u32 blocks_per_cluster; // cluster_size / sb->s_blocksize + + u32 record_size; + u32 index_size; + + u8 cluster_bits; + u8 record_bits; + + u64 maxbytes; // Maximum size for normal files. + u64 maxbytes_sparse; // Maximum size for sparse file. + + unsigned long flags; // See NTFS_FLAGS_ + + CLST zone_max; // Maximum MFT zone length in clusters + CLST bad_clusters; // The count of marked bad clusters. + + u16 max_bytes_per_attr; // Maximum attribute size in record. + u16 attr_size_tr; // Attribute size threshold (320 bytes). + + /* Records in $Extend. */ + CLST objid_no; + CLST quota_no; + CLST reparse_no; + CLST usn_jrnl_no; + + struct ATTR_DEF_ENTRY *def_table; // Attribute definition table. + u32 def_entries; + u32 ea_max_size; + + struct MFT_REC *new_rec; + + u16 *upcase; + + struct { + u64 lbo, lbo2; + struct ntfs_inode *ni; + struct wnd_bitmap bitmap; // $MFT::Bitmap + /* + * MFT records [11-24) used to expand MFT itself. + * They always marked as used in $MFT::Bitmap + * 'reserved_bitmap' contains real bitmap of these records. + */ + ulong reserved_bitmap; // Bitmap of used records [11 - 24) + size_t next_free; // The next record to allocate from + size_t used; // MFT valid size in records. + u32 recs_mirr; // Number of records in MFTMirr + u8 next_reserved; + u8 reserved_bitmap_inited; + } mft; + + struct { + struct wnd_bitmap bitmap; // $Bitmap::Data + CLST next_free_lcn; + /* Total sum of delay allocated clusters in all files. */ +#ifdef CONFIG_NTFS3_64BIT_CLUSTER + atomic64_t da; +#else + atomic_t da; +#endif + } used; + + struct { + u64 size; // In bytes. + u64 blocks; // In blocks. + u64 ser_num; + struct ntfs_inode *ni; + __le16 flags; // Cached current VOLUME_INFO::flags, VOLUME_FLAG_DIRTY. + u8 major_ver; + u8 minor_ver; + char label[FSLABEL_MAX]; + bool real_dirty; // Real fs state. + } volume; + + struct { + struct ntfs_index index_sii; + struct ntfs_index index_sdh; + struct ntfs_inode *ni; + u32 next_id; + u64 next_off; + __le32 def_security_id; + } security; + + struct { + struct ntfs_index index_r; + struct ntfs_inode *ni; + u64 max_size; // 16K + } reparse; + + struct { + struct ntfs_index index_o; + struct ntfs_inode *ni; + } objid; + + struct { + struct mutex mtx_lznt; + struct lznt *lznt; +#ifdef CONFIG_NTFS3_LZX_XPRESS + struct mutex mtx_xpress; + struct xpress_decompressor *xpress; + struct mutex mtx_lzx; + struct lzx_decompressor *lzx; +#endif + } compress; + + struct ntfs_mount_options *options; + struct ratelimit_state msg_ratelimit; + struct proc_dir_entry *procdir; +}; + +/* One MFT record(usually 1024 bytes), consists of attributes. */ +struct mft_inode { + struct rb_node node; + struct ntfs_sb_info *sbi; + + struct MFT_REC *mrec; + struct ntfs_buffers nb; + + CLST rno; + bool dirty; +}; + +/* Nested class for ntfs_inode::ni_lock. */ +enum ntfs_inode_mutex_lock_class { + NTFS_INODE_MUTEX_DIRTY = 1, + NTFS_INODE_MUTEX_SECURITY, + NTFS_INODE_MUTEX_OBJID, + NTFS_INODE_MUTEX_REPARSE, + NTFS_INODE_MUTEX_NORMAL, + NTFS_INODE_MUTEX_PARENT, + NTFS_INODE_MUTEX_PARENT2, +}; + +/* + * struct ntfs_inode + * + * Ntfs inode - extends linux inode. consists of one or more MFT inodes. + */ +struct ntfs_inode { + struct mft_inode mi; // base record + + /* + * Valid size: [0 - i_valid) - these range in file contains valid data. + * Range [i_valid - inode->i_size) - contains 0. + * Usually i_valid <= inode->i_size. + */ + u64 i_valid; + struct timespec64 i_crtime; + + struct mutex ni_lock; + + /* File attributes from std. */ + enum FILE_ATTRIBUTE std_fa; + __le32 std_security_id; + + /* + * Tree of mft_inode. + * Not empty when primary MFT record (usually 1024 bytes) can't save all attributes + * e.g. file becomes too fragmented or contains a lot of names. + */ + struct rb_root mi_tree; + + /* + * This member is used in ntfs_readdir to ensure that all subrecords are loaded + */ + u8 mi_loaded; + + /* + * Use this field to avoid any write(s). + * If inode is bad during initialization - use make_bad_inode + * If inode is bad during operations - use this field + */ + u8 ni_bad; + + /* Keep track of FS_NODUMP_FL. */ + u8 nodump; + + union { + struct ntfs_index dir; + struct { + struct rw_semaphore run_lock; + /* Unpacked runs from just one record. */ + struct runs_tree run; + /* + * Pairs [vcn, len] for all delay allocated clusters. + * Normal file always contains delayed clusters in one fragment. + * TODO: use 2 CLST per pair instead of 3. + */ + struct runs_tree run_da; +#ifdef CONFIG_NTFS3_LZX_XPRESS + struct folio *offs_folio; +#endif + /* Alternative data stream */ + struct { + __le16 *name; + u8 len; + } ads; + } file; + }; + + struct { + struct runs_tree run; + struct ATTR_LIST_ENTRY *le; // 1K aligned memory. + size_t size; + bool dirty; + } attr_list; + + size_t ni_flags; // NI_FLAG_XXX + struct ntfs_inode *base; /* ADS: points to base inode. Other: this. */ + + struct inode vfs_inode; +}; + +struct indx_node { + struct ntfs_buffers nb; + struct INDEX_BUFFER *index; +}; + +struct ntfs_fnd { + int level; + struct indx_node *nodes[20]; + struct NTFS_DE *de[20]; + struct NTFS_DE *root_de; +}; + +enum REPARSE_SIGN { + REPARSE_NONE = 0, + REPARSE_COMPRESSED = 1, + REPARSE_DEDUPLICATED = 2, + REPARSE_LINK = 3 +}; + +static inline bool is_ni_base(const struct ntfs_inode *ni) +{ + return ni == ni->base; +} + +/* Functions from attrib.c */ +int attr_allocate_clusters(struct ntfs_sb_info *sbi, struct runs_tree *run, + struct runs_tree *run_da, CLST vcn, CLST lcn, + CLST len, CLST *pre_alloc, enum ALLOCATE_OPT opt, + CLST *alen, const size_t fr, CLST *new_lcn, + CLST *new_len); +int attr_make_nonresident(struct ntfs_inode *ni, struct ATTRIB *attr, + struct ATTR_LIST_ENTRY *le, struct mft_inode *mi, + u64 new_size, struct runs_tree *run, + struct ATTRIB **ins_attr, struct page *page); +int attr_set_size_ex(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, struct runs_tree *run, + u64 new_size, const u64 *new_valid, bool keep_prealloc, + struct ATTRIB **ret, bool no_da); +static inline int attr_set_size(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, + struct runs_tree *run, u64 new_size, + const u64 *new_valid, bool keep_prealloc) +{ + return attr_set_size_ex(ni, type, name, name_len, run, new_size, + new_valid, keep_prealloc, NULL, false); +} +int attr_data_get_block(struct ntfs_inode *ni, CLST vcn, CLST clen, CLST *lcn, + CLST *len, bool *new, bool zero, void **res, + bool no_da); +int attr_data_get_block_locked(struct ntfs_inode *ni, CLST vcn, CLST clen, + CLST *lcn, CLST *len, bool *new, bool zero, + void **res, bool no_da); +int attr_data_write_resident(struct ntfs_inode *ni, struct folio *folio); +int attr_load_runs_vcn(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, struct runs_tree *run, + CLST vcn); +int attr_load_runs_range(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, struct runs_tree *run, + u64 from, u64 to); +int attr_wof_frame_info(struct ntfs_inode *ni, struct ATTRIB *attr, + struct runs_tree *run, u64 frame, u64 frames, + u8 frame_bits, u32 *ondisk_size, u64 *vbo_data); +int attr_is_frame_compressed(struct ntfs_inode *ni, struct ATTRIB *attr, + CLST frame, CLST *clst_data, + struct runs_tree *run); +int attr_allocate_frame(struct ntfs_inode *ni, CLST frame, size_t compr_size, + u64 new_valid); +int attr_collapse_range(struct ntfs_inode *ni, u64 vbo, u64 bytes); +int attr_insert_range(struct ntfs_inode *ni, u64 vbo, u64 bytes); +int attr_punch_hole(struct ntfs_inode *ni, u64 vbo, u64 bytes, u32 *frame_size); +int attr_force_nonresident(struct ntfs_inode *ni); + +/* Functions from attrlist.c */ +void al_destroy(struct ntfs_inode *ni); +bool al_verify(struct ntfs_inode *ni); +int ntfs_load_attr_list(struct ntfs_inode *ni, struct ATTRIB *attr); +struct ATTR_LIST_ENTRY *al_enumerate(struct ntfs_inode *ni, + struct ATTR_LIST_ENTRY *le); +struct ATTR_LIST_ENTRY *al_find_le(struct ntfs_inode *ni, + struct ATTR_LIST_ENTRY *le, + const struct ATTRIB *attr); +struct ATTR_LIST_ENTRY *al_find_ex(struct ntfs_inode *ni, + struct ATTR_LIST_ENTRY *le, + enum ATTR_TYPE type, const __le16 *name, + u8 name_len, const CLST *vcn); +int al_add_le(struct ntfs_inode *ni, enum ATTR_TYPE type, const __le16 *name, + u8 name_len, CLST svcn, __le16 id, const struct MFT_REF *ref, + struct ATTR_LIST_ENTRY **new_le); +bool al_remove_le(struct ntfs_inode *ni, struct ATTR_LIST_ENTRY *le); +int al_update(struct ntfs_inode *ni, int sync); +static inline size_t al_aligned(size_t size) +{ + return size_add(size, 1023) & ~(size_t)1023; +} + +/* Globals from bitfunc.c */ +bool are_bits_clear(const void *map, size_t bit, size_t nbits); +bool are_bits_set(const void *map, size_t bit, size_t nbits); +size_t get_set_bits_ex(const void *map, size_t bit, size_t nbits); + +/* Globals from dir.c */ +int ntfs_utf16_to_nls(struct ntfs_sb_info *sbi, const __le16 *name, u32 len, + u8 *buf, int buf_len); +int ntfs_nls_to_utf16(struct ntfs_sb_info *sbi, const u8 *name, u32 name_len, + struct cpu_str *uni, u32 max_ulen, + enum utf16_endian endian); +struct inode *dir_search_flags(struct inode *dir, const struct cpu_str *uni, + struct ntfs_fnd *fnd, u32 flags); +static inline struct inode *dir_search(struct inode *dir, + const struct cpu_str *uni) +{ + return dir_search_flags(dir, uni, NULL, 0); +} + +bool dir_is_empty(struct inode *dir); +extern const struct file_operations ntfs_dir_operations; + +/* Globals from file.c */ +int ntfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa); +int ntfs_fileattr_set(struct mnt_idmap *idmap, struct dentry *dentry, + struct file_kattr *fa); +int ntfs_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, u32 flags); +int ntfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr); +int ntfs_file_open(struct inode *inode, struct file *file); +int ntfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + __u64 start, __u64 len); +int ntfs_file_fsync(struct file *file, loff_t start, loff_t end, int datasync); +long ntfs_ioctl(struct file *filp, u32 cmd, unsigned long arg); +long ntfs_compat_ioctl(struct file *filp, u32 cmd, unsigned long arg); +extern const struct inode_operations ntfs_special_inode_operations; +extern const struct inode_operations ntfs_file_inode_operations; +extern const struct file_operations ntfs_file_operations; + +/* Globals from frecord.c */ +void ni_remove_mi(struct ntfs_inode *ni, struct mft_inode *mi); +struct ATTR_STD_INFO *ni_std(struct ntfs_inode *ni); +struct ATTR_STD_INFO5 *ni_std5(struct ntfs_inode *ni); +void ni_clear(struct ntfs_inode *ni); +int ni_load_mi_ex(struct ntfs_inode *ni, CLST rno, struct mft_inode **mi); +int ni_load_mi(struct ntfs_inode *ni, const struct ATTR_LIST_ENTRY *le, + struct mft_inode **mi); +struct ATTRIB *ni_find_attr(struct ntfs_inode *ni, struct ATTRIB *attr, + struct ATTR_LIST_ENTRY **entry_o, + enum ATTR_TYPE type, const __le16 *name, + u8 name_len, const CLST *vcn, + struct mft_inode **mi); +struct ATTRIB *ni_enum_attr_ex(struct ntfs_inode *ni, struct ATTRIB *attr, + struct ATTR_LIST_ENTRY **le, + struct mft_inode **mi); +int ni_load_all_mi(struct ntfs_inode *ni); +bool ni_add_subrecord(struct ntfs_inode *ni, CLST rno, struct mft_inode **mi); +int ni_remove_attr(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, bool base_only, + const __le16 *id); +int ni_create_attr_list(struct ntfs_inode *ni); +int ni_expand_list(struct ntfs_inode *ni); +int ni_insert_nonresident(struct ntfs_inode *ni, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, + const struct runs_tree *run, CLST svcn, CLST len, + __le16 flags, struct ATTRIB **new_attr, + struct mft_inode **mi, struct ATTR_LIST_ENTRY **le); +int ni_insert_resident(struct ntfs_inode *ni, u32 data_size, + enum ATTR_TYPE type, const __le16 *name, u8 name_len, + struct ATTRIB **new_attr, struct mft_inode **mi, + struct ATTR_LIST_ENTRY **le); +void ni_remove_attr_le(struct ntfs_inode *ni, struct ATTRIB *attr, + struct mft_inode *mi, struct ATTR_LIST_ENTRY *le); +int ni_delete_all(struct ntfs_inode *ni); +struct ATTR_FILE_NAME *ni_fname_name(struct ntfs_inode *ni, + const struct le_str *uni, + const struct MFT_REF *home, + struct mft_inode **mi, + struct ATTR_LIST_ENTRY **entry); +struct ATTR_FILE_NAME *ni_fname_type(struct ntfs_inode *ni, u8 name_type, + struct mft_inode **mi, + struct ATTR_LIST_ENTRY **entry); +int ni_new_attr_flags(struct ntfs_inode *ni, enum FILE_ATTRIBUTE new_fa); +enum REPARSE_SIGN ni_parse_reparse(struct ntfs_inode *ni, struct ATTRIB *attr, + struct REPARSE_DATA_BUFFER *buffer); +int ni_write_inode(struct inode *inode, int sync, const char *hint); +#define _ni_write_inode(i, w) ni_write_inode(i, w, __func__) +int ni_read_folio_cmpr(struct ntfs_inode *ni, struct folio *folio); +int ni_decompress_file(struct ntfs_inode *ni); +int ni_read_frame(struct ntfs_inode *ni, u64 frame_vbo, struct page **pages, + u32 pages_per_frame, int copy); +int ni_write_frame(struct ntfs_inode *ni, struct page **pages, + u32 pages_per_frame); +int ni_remove_name(struct ntfs_inode *dir_ni, struct ntfs_inode *ni, + struct NTFS_DE *de, struct NTFS_DE **de2, int *undo_step); + +bool ni_remove_name_undo(struct ntfs_inode *dir_ni, struct ntfs_inode *ni, + struct NTFS_DE *de, struct NTFS_DE *de2, + int undo_step); + +int ni_add_name(struct ntfs_inode *dir_ni, struct ntfs_inode *ni, + struct NTFS_DE *de); + +int ni_rename(struct ntfs_inode *dir_ni, struct ntfs_inode *new_dir_ni, + struct ntfs_inode *ni, struct NTFS_DE *de, + struct NTFS_DE *new_de); + +bool ni_is_dirty(struct inode *inode); +loff_t ni_seek_data_or_hole(struct ntfs_inode *ni, loff_t offset, bool data); +int ni_write_parents(struct ntfs_inode *ni, int sync); +int ni_allocate_da_blocks(struct ntfs_inode *ni); +int ni_allocate_da_blocks_locked(struct ntfs_inode *ni); +ssize_t ni_query_ads(struct ntfs_inode *ni, loff_t *pos, struct iov_iter *iter); + +/* Globals from fslog.c */ +bool check_index_header(const struct INDEX_HDR *hdr, size_t bytes); +int log_replay(struct ntfs_inode *ni, bool *initialized); + +/* Globals from fsntfs.c */ +struct buffer_head *ntfs_bread(struct super_block *sb, sector_t block); +bool ntfs_fix_pre_write(struct NTFS_RECORD_HEADER *rhdr, size_t bytes); +int ntfs_fix_post_read(struct NTFS_RECORD_HEADER *rhdr, size_t bytes, + bool simple); +int ntfs_extend_init(struct ntfs_sb_info *sbi); +int ntfs_loadlog_and_replay(struct ntfs_inode *ni, struct ntfs_sb_info *sbi); +int ntfs_look_for_free_space(struct ntfs_sb_info *sbi, CLST lcn, CLST len, + CLST *new_lcn, CLST *new_len, + enum ALLOCATE_OPT opt); +bool ntfs_check_free_space(struct ntfs_sb_info *sbi, CLST clen, CLST mlen, + bool da); +int ntfs_look_free_mft(struct ntfs_sb_info *sbi, CLST *rno, bool mft, + struct ntfs_inode *ni, struct mft_inode **mi); +void ntfs_mark_rec_free(struct ntfs_sb_info *sbi, CLST rno, bool is_mft); +int ntfs_clear_mft_tail(struct ntfs_sb_info *sbi, size_t from, size_t to); +int ntfs_refresh_zone(struct ntfs_sb_info *sbi); +void ntfs_update_mftmirr(struct ntfs_sb_info *sbi); +void ntfs_bad_inode(struct inode *inode, const char *hint); +#define _ntfs_bad_inode(i) ntfs_bad_inode(i, __func__) +enum NTFS_DIRTY_FLAGS { + NTFS_DIRTY_CLEAR = 0, + NTFS_DIRTY_DIRTY = 1, + NTFS_DIRTY_ERROR = 2, +}; +int ntfs_set_state(struct ntfs_sb_info *sbi, enum NTFS_DIRTY_FLAGS dirty); +int ntfs_sb_write(struct super_block *sb, u64 lbo, size_t bytes, + const void *buffer, int wait); +int ntfs_sb_write_run(struct ntfs_sb_info *sbi, const struct runs_tree *run, + u64 vbo, const void *buf, size_t bytes, int sync); +struct buffer_head *ntfs_bread_run(struct ntfs_sb_info *sbi, + const struct runs_tree *run, u64 vbo); +int ntfs_read_run_nb_ra(struct ntfs_sb_info *sbi, const struct runs_tree *run, + u64 vbo, void *buf, u32 bytes, struct ntfs_buffers *nb, + struct file_ra_state *ra); +static inline int ntfs_read_run_nb(struct ntfs_sb_info *sbi, + const struct runs_tree *run, u64 vbo, + void *buf, u32 bytes, + struct ntfs_buffers *nb) +{ + return ntfs_read_run_nb_ra(sbi, run, vbo, buf, bytes, nb, NULL); +} +int ntfs_read_bh_ra(struct ntfs_sb_info *sbi, const struct runs_tree *run, + u64 vbo, struct NTFS_RECORD_HEADER *rhdr, u32 bytes, + struct ntfs_buffers *nb, struct file_ra_state *ra); +static inline int ntfs_read_bh(struct ntfs_sb_info *sbi, + const struct runs_tree *run, u64 vbo, + struct NTFS_RECORD_HEADER *rhdr, u32 bytes, + struct ntfs_buffers *nb) +{ + return ntfs_read_bh_ra(sbi, run, vbo, rhdr, bytes, nb, NULL); +} +int ntfs_get_bh(struct ntfs_sb_info *sbi, const struct runs_tree *run, u64 vbo, + u32 bytes, struct ntfs_buffers *nb); +int ntfs_write_bh(struct ntfs_sb_info *sbi, struct NTFS_RECORD_HEADER *rhdr, + struct ntfs_buffers *nb, int sync); +int ntfs_read_write_run(struct ntfs_sb_info *sbi, const struct runs_tree *run, + void *buf, u64 vbo, size_t bytes, int wr); +static inline int ntfs_read_run(struct ntfs_sb_info *sbi, + const struct runs_tree *run, void *buf, u64 vbo, + size_t bytes) +{ + return ntfs_read_write_run(sbi, run, buf, vbo, bytes, 0); +} +static inline int ntfs_write_run(struct ntfs_sb_info *sbi, + const struct runs_tree *run, void *buf, + u64 vbo, size_t bytes) +{ + return ntfs_read_write_run(sbi, run, buf, vbo, bytes, 1); +} + +int ntfs_bio_fill_1(struct ntfs_sb_info *sbi, const struct runs_tree *run); +int ntfs_vbo_to_lbo(struct ntfs_sb_info *sbi, const struct runs_tree *run, + u64 vbo, u64 *lbo, u64 *bytes); +struct ntfs_inode *ntfs_new_inode(struct ntfs_sb_info *sbi, CLST nRec, + enum RECORD_FLAG flag); +extern const u8 s_default_security[0x50]; +bool is_sd_valid(const struct SECURITY_DESCRIPTOR_RELATIVE *sd, u32 len); +int ntfs_security_init(struct ntfs_sb_info *sbi); +int ntfs_get_security_by_id(struct ntfs_sb_info *sbi, __le32 security_id, + struct SECURITY_DESCRIPTOR_RELATIVE **sd, + size_t *size); +int ntfs_insert_security(struct ntfs_sb_info *sbi, + const struct SECURITY_DESCRIPTOR_RELATIVE *sd, + u32 size, __le32 *security_id, bool *inserted); +int ntfs_reparse_init(struct ntfs_sb_info *sbi); +int ntfs_objid_init(struct ntfs_sb_info *sbi); +int ntfs_objid_remove(struct ntfs_sb_info *sbi, struct GUID *guid); +int ntfs_insert_reparse(struct ntfs_sb_info *sbi, __le32 rtag, + const struct MFT_REF *ref); +int ntfs_remove_reparse(struct ntfs_sb_info *sbi, __le32 rtag, + const struct MFT_REF *ref); +void mark_as_free_ex(struct ntfs_sb_info *sbi, CLST lcn, CLST len, bool trim); +int run_deallocate(struct ntfs_sb_info *sbi, const struct runs_tree *run, + bool trim); +bool valid_windows_name(struct ntfs_sb_info *sbi, const struct le_str *name); +int ntfs_set_label(struct ntfs_sb_info *sbi, u8 *label, int len); + +/* Globals from index.c */ +int indx_used_bit(struct ntfs_index *indx, struct ntfs_inode *ni, size_t *bit); +void fnd_clear(struct ntfs_fnd *fnd); +static inline struct ntfs_fnd *fnd_get(void) +{ + return kzalloc_obj(struct ntfs_fnd, GFP_NOFS); +} +static inline void fnd_put(struct ntfs_fnd *fnd) +{ + if (fnd) { + fnd_clear(fnd); + kfree(fnd); + } +} +void indx_clear(struct ntfs_index *idx); +int indx_init(struct ntfs_index *indx, struct ntfs_sb_info *sbi, + const struct ATTRIB *attr, enum index_mutex_classed type); +struct INDEX_ROOT *indx_get_root(struct ntfs_index *indx, struct ntfs_inode *ni, + struct ATTRIB **attr, struct mft_inode **mi); +int indx_read_ra(struct ntfs_index *idx, struct ntfs_inode *ni, CLST vbn, + struct indx_node **node, struct file_ra_state *ra); +static inline int indx_read(struct ntfs_index *idx, struct ntfs_inode *ni, + CLST vbn, struct indx_node **node) +{ + return indx_read_ra(idx, ni, vbn, node, NULL); +} +int indx_find(struct ntfs_index *indx, struct ntfs_inode *dir, + const struct INDEX_ROOT *root, const void *Key, size_t KeyLen, + const void *param, int *diff, struct NTFS_DE **entry, + struct ntfs_fnd *fnd); +int indx_find_sort(struct ntfs_index *indx, struct ntfs_inode *ni, + const struct INDEX_ROOT *root, struct NTFS_DE **entry, + struct ntfs_fnd *fnd); +int indx_find_raw(struct ntfs_index *indx, struct ntfs_inode *ni, + const struct INDEX_ROOT *root, struct NTFS_DE **entry, + size_t *off, struct ntfs_fnd *fnd); +int indx_insert_entry(struct ntfs_index *indx, struct ntfs_inode *ni, + const struct NTFS_DE *new_de, const void *param, + struct ntfs_fnd *fnd, bool undo); +int indx_delete_entry(struct ntfs_index *indx, struct ntfs_inode *ni, + const void *key, u32 key_len, const void *param); +int indx_update_dup(struct ntfs_inode *ni, struct ntfs_sb_info *sbi, + const struct ATTR_FILE_NAME *fname, + const struct NTFS_DUP_INFO *dup, int sync); + +/* Globals from inode.c */ +struct inode *ntfs_iget5_flags(struct super_block *sb, + const struct MFT_REF *ref, + const struct cpu_str *name, u32 flags); +static inline struct inode *ntfs_iget5(struct super_block *sb, + const struct MFT_REF *ref, + const struct cpu_str *name) +{ + return ntfs_iget5_flags(sb, ref, name, 0); +} +int ntfs_set_size(struct inode *inode, u64 new_size); +int ntfs3_write_inode(struct inode *inode, struct writeback_control *wbc); +int ntfs_sync_inode(struct inode *inode); +int inode_read_data(struct inode *inode, void *data, size_t bytes); +int ntfs_create_inode(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const struct cpu_str *uni, + umode_t mode, dev_t dev, const char *symname, u32 size, + struct ntfs_fnd *fnd); +int ntfs_link_inode(struct inode *inode, struct dentry *dentry); +int ntfs_unlink_inode(struct inode *dir, const struct dentry *dentry); +void ntfs_evict_inode(struct inode *inode); +extern const struct iomap_ops ntfs_iomap_ops; +extern const struct iomap_write_ops ntfs_iomap_folio_ops; +extern const struct inode_operations ntfs_link_inode_operations; +extern const struct address_space_operations ntfs_aops; +extern const struct address_space_operations ntfs_aops_cmpr; + +/* Globals from name_i.c */ +int fill_name_de(struct ntfs_sb_info *sbi, void *buf, const struct qstr *name, + const struct cpu_str *uni); +struct dentry *ntfs3_get_parent(struct dentry *child); + +extern const struct inode_operations ntfs_dir_inode_operations; +extern const struct inode_operations ntfs_special_inode_operations; +extern const struct dentry_operations ntfs_dentry_ops; + +/* Globals from record.c */ +int mi_get(struct ntfs_sb_info *sbi, CLST rno, struct mft_inode **mi); +void mi_put(struct mft_inode *mi); +int mi_init(struct mft_inode *mi, struct ntfs_sb_info *sbi, CLST rno); +int mi_read(struct mft_inode *mi, bool is_mft); +struct ATTRIB *mi_enum_attr(struct ntfs_inode *ni, struct mft_inode *mi, + struct ATTRIB *attr); +struct ATTRIB *mi_find_attr(struct ntfs_inode *ni, struct mft_inode *mi, + struct ATTRIB *attr, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, const __le16 *id); +static inline struct ATTRIB *rec_find_attr_le(struct ntfs_inode *ni, + struct mft_inode *rec, + struct ATTR_LIST_ENTRY *le) +{ + return mi_find_attr(ni, rec, NULL, le->type, le_name(le), le->name_len, + &le->id); +} +int mi_write(struct mft_inode *mi, int wait); +int mi_format_new(struct mft_inode *mi, struct ntfs_sb_info *sbi, CLST rno, + __le16 flags, bool is_mft); +struct ATTRIB *mi_insert_attr(struct ntfs_inode *ni, struct mft_inode *mi, + enum ATTR_TYPE type, const __le16 *name, + u8 name_len, u32 asize, u16 name_off); + +bool mi_remove_attr(struct ntfs_inode *ni, struct mft_inode *mi, + struct ATTRIB *attr); +bool mi_resize_attr(struct mft_inode *mi, struct ATTRIB *attr, int bytes); +int mi_pack_runs(struct mft_inode *mi, struct ATTRIB *attr, + const struct runs_tree *run, CLST len); +static inline bool mi_is_ref(const struct mft_inode *mi, + const struct MFT_REF *ref) +{ + if (le32_to_cpu(ref->low) != mi->rno) + return false; + if (ref->seq != mi->mrec->seq) + return false; + +#ifdef CONFIG_NTFS3_64BIT_CLUSTER + return le16_to_cpu(ref->high) == (mi->rno >> 32); +#else + return !ref->high; +#endif +} + +static inline void mi_get_ref(const struct mft_inode *mi, struct MFT_REF *ref) +{ + ref->low = cpu_to_le32(mi->rno); +#ifdef CONFIG_NTFS3_64BIT_CLUSTER + ref->high = cpu_to_le16(mi->rno >> 32); +#else + ref->high = 0; +#endif + ref->seq = mi->mrec->seq; +} + +/* Globals from run.c */ +bool run_lookup_entry(const struct runs_tree *run, CLST vcn, CLST *lcn, + CLST *len, size_t *index); +bool run_lookup_entry_da(const struct runs_tree *run, + const struct runs_tree *run_da, CLST vcn, CLST *lcn, + CLST *len); +void run_truncate(struct runs_tree *run, CLST vcn); +void run_truncate_head(struct runs_tree *run, CLST vcn); +void run_truncate_around(struct runs_tree *run, CLST vcn); +bool run_add_entry(struct runs_tree *run, CLST vcn, CLST lcn, CLST len, + bool is_mft); +bool run_collapse_range(struct runs_tree *run, CLST vcn, CLST len, CLST sub); +int run_insert_range(struct runs_tree *run, CLST vcn, CLST len); +int run_insert_range_da(struct runs_tree *run, CLST vcn, CLST len); +bool run_get_entry(const struct runs_tree *run, size_t index, CLST *vcn, + CLST *lcn, CLST *len); +bool run_is_mapped_full(const struct runs_tree *run, CLST svcn, CLST evcn); + +int run_pack(const struct runs_tree *run, CLST svcn, CLST len, u8 *run_buf, + u32 run_buf_size, CLST *packed_vcns); +int run_unpack(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino, + CLST svcn, CLST evcn, CLST vcn, const u8 *run_buf, + int run_buf_size); + +#ifdef NTFS3_CHECK_FREE_CLST +int run_unpack_ex(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino, + CLST svcn, CLST evcn, CLST vcn, const u8 *run_buf, + int run_buf_size); +#else +#define run_unpack_ex run_unpack +#endif +int run_get_highest_vcn(CLST vcn, const u8 *run_buf, size_t run_buf_size, + u64 *highest_vcn); +int run_clone(const struct runs_tree *run, struct runs_tree *new_run); +bool run_remove_range(struct runs_tree *run, CLST vcn, CLST len, CLST *done); +CLST run_len(const struct runs_tree *run); +CLST run_get_max_vcn(const struct runs_tree *run); + +/* Globals from super.c */ +void *ntfs_set_shared(void *ptr, u32 bytes); +void *ntfs_put_shared(void *ptr); +void ntfs_unmap_meta(struct super_block *sb, CLST lcn, CLST len); +int ntfs_discard(struct ntfs_sb_info *sbi, CLST Lcn, CLST Len); + +/* Globals from bitmap.c*/ +int __init ntfs3_init_bitmap(void); +void ntfs3_exit_bitmap(void); +void wnd_close(struct wnd_bitmap *wnd); +static inline size_t wnd_zeroes(const struct wnd_bitmap *wnd) +{ + return wnd->total_zeroes; +} +int wnd_init(struct wnd_bitmap *wnd, struct super_block *sb, size_t nbits); +int wnd_set_free(struct wnd_bitmap *wnd, size_t bit, size_t bits); +int wnd_set_used(struct wnd_bitmap *wnd, size_t bit, size_t bits); +int wnd_set_used_safe(struct wnd_bitmap *wnd, size_t bit, size_t bits, + size_t *done); +bool wnd_is_free(struct wnd_bitmap *wnd, size_t bit, size_t bits); +bool wnd_is_used(struct wnd_bitmap *wnd, size_t bit, size_t bits); + +/* Possible values for 'flags' 'wnd_find'. */ +#define BITMAP_FIND_MARK_AS_USED 0x01 +#define BITMAP_FIND_FULL 0x02 +size_t wnd_find(struct wnd_bitmap *wnd, size_t to_alloc, size_t hint, + size_t flags, size_t *allocated); +int wnd_extend(struct wnd_bitmap *wnd, size_t new_bits); +void wnd_zone_set(struct wnd_bitmap *wnd, size_t Lcn, size_t Len); +int ntfs_trim_fs(struct ntfs_sb_info *sbi, struct fstrim_range *range); + +void ntfs_bitmap_set_le(void *map, unsigned int start, int len); +void ntfs_bitmap_clear_le(void *map, unsigned int start, int len); +unsigned int ntfs_bitmap_weight_le(const void *bitmap, int bits); + +/* Globals from upcase.c */ +int ntfs_cmp_names(const __le16 *s1, size_t l1, const __le16 *s2, size_t l2, + const u16 *upcase, bool bothcase); +int ntfs_cmp_names_cpu(const struct cpu_str *uni1, const struct le_str *uni2, + const u16 *upcase, bool bothcase); +unsigned long ntfs_names_hash(const u16 *name, size_t len, const u16 *upcase, + unsigned long hash); + +/* globals from xattr.c */ +#ifdef CONFIG_NTFS3_FS_POSIX_ACL +struct posix_acl *ntfs_get_acl(struct mnt_idmap *idmap, struct dentry *dentry, + int type); +int ntfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type); +int ntfs_init_acl(struct mnt_idmap *idmap, struct inode *inode, + struct inode *dir); +#else +#define ntfs_get_acl NULL +#define ntfs_set_acl NULL +#endif + +int ntfs_acl_chmod(struct mnt_idmap *idmap, struct dentry *dentry); +ssize_t ntfs_listxattr(struct dentry *dentry, char *buffer, size_t size); +extern const struct xattr_handler *const ntfs_xattr_handlers[]; + +int ntfs_save_wsl_perm(struct inode *inode, __le32 *ea_size); +void ntfs_get_wsl_perm(struct inode *inode); + +/* globals from lznt.c */ +struct lznt *get_lznt_ctx(int level); +size_t compress_lznt(const void *uncompressed, size_t uncompressed_size, + void *compressed, size_t compressed_size, + struct lznt *ctx); +ssize_t decompress_lznt(const void *compressed, size_t compressed_size, + void *uncompressed, size_t uncompressed_size); + +static inline bool is_ntfs3(struct ntfs_sb_info *sbi) +{ + return sbi->volume.major_ver >= 3; +} + +/* (sb->s_flags & SB_ACTIVE) */ +static inline bool is_mounted(struct ntfs_sb_info *sbi) +{ + return !!sbi->sb->s_root; +} + +static inline bool ntfs_is_meta_file(struct ntfs_sb_info *sbi, CLST rno) +{ + return rno < MFT_REC_FREE || rno == sbi->objid_no || + rno == sbi->quota_no || rno == sbi->reparse_no || + rno == sbi->usn_jrnl_no; +} + +static inline size_t wnd_zone_bit(const struct wnd_bitmap *wnd) +{ + return wnd->zone_bit; +} + +static inline size_t wnd_zone_len(const struct wnd_bitmap *wnd) +{ + return wnd->zone_end - wnd->zone_bit; +} + +static inline void run_init(struct runs_tree *run) +{ + run->runs = NULL; + run->count = 0; + run->allocated = 0; +} + +static inline struct runs_tree *run_alloc(void) +{ + return kzalloc_obj(struct runs_tree, GFP_NOFS); +} + +static inline void run_close(struct runs_tree *run) +{ + kvfree(run->runs); + memset(run, 0, sizeof(*run)); +} + +static inline void run_free(struct runs_tree *run) +{ + if (run) { + kvfree(run->runs); + kfree(run); + } +} + +static inline bool run_is_empty(struct runs_tree *run) +{ + return !run->count; +} + +/* NTFS uses quad aligned bitmaps. */ +static inline size_t ntfs3_bitmap_size(size_t bits) +{ + return BITS_TO_U64(bits) * sizeof(u64); +} + +#define _100ns2seconds 10000000 +#define SecondsToStartOf1970 0x00000002B6109100 + +#define NTFS_TIME_GRAN 100 + +/* + * kernel2nt - Converts in-memory kernel timestamp into nt time. + */ +static inline __le64 kernel2nt(const struct timespec64 *ts) +{ + // 10^7 units of 100 nanoseconds one second + return cpu_to_le64(_100ns2seconds * + (ts->tv_sec + SecondsToStartOf1970) + + ts->tv_nsec / NTFS_TIME_GRAN); +} + +/* + * nt2kernel - Converts on-disk nt time into kernel timestamp. + */ +static inline void nt2kernel(const __le64 tm, struct timespec64 *ts) +{ + s32 t32; + /* use signed 64 bit to support timestamps prior to epoch. xfstest 258. */ + s64 t = le64_to_cpu(tm) - _100ns2seconds * SecondsToStartOf1970; + + ts->tv_sec = div_s64_rem(t, _100ns2seconds, &t32); + ts->tv_nsec = t32 * 100; +} + +static inline struct ntfs_sb_info *ntfs_sb(struct super_block *sb) +{ + return sb->s_fs_info; +} + +static inline int ntfs3_forced_shutdown(struct super_block *sb) +{ + return test_bit(NTFS_FLAGS_SHUTDOWN_BIT, &ntfs_sb(sb)->flags); +} + +/* Returns total sum of delay allocated clusters in all files. */ +static inline CLST ntfs_get_da(struct ntfs_sb_info *sbi) +{ +#ifdef CONFIG_NTFS3_64BIT_CLUSTER + return atomic64_read(&sbi->used.da); +#else + return atomic_read(&sbi->used.da); +#endif +} + +/* Update total count of delay allocated clusters. */ +static inline void ntfs_add_da(struct ntfs_sb_info *sbi, CLST da) +{ +#ifdef CONFIG_NTFS3_64BIT_CLUSTER + atomic64_add(da, &sbi->used.da); +#else + atomic_add(da, &sbi->used.da); +#endif +} + +/* Update total count of delay allocated clusters. */ +static inline void ntfs_sub_da(struct ntfs_sb_info *sbi, CLST da) +{ +#ifdef CONFIG_NTFS3_64BIT_CLUSTER + atomic64_sub(da, &sbi->used.da); +#else + atomic_sub(da, &sbi->used.da); +#endif +} + +/* + * ntfs_up_cluster - Align up on cluster boundary. + */ +static inline u64 ntfs_up_cluster(const struct ntfs_sb_info *sbi, u64 size) +{ + return (size + sbi->cluster_mask) & sbi->cluster_mask_inv; +} + +/* + * ntfs_up_block - Align up on cluster boundary. + */ +static inline u64 ntfs_up_block(const struct super_block *sb, u64 size) +{ + return (size + sb->s_blocksize - 1) & ~(u64)(sb->s_blocksize - 1); +} + +static inline CLST bytes_to_cluster(const struct ntfs_sb_info *sbi, u64 size) +{ + return (size + sbi->cluster_mask) >> sbi->cluster_bits; +} + +static inline u64 bytes_to_block(const struct super_block *sb, u64 size) +{ + return (size + sb->s_blocksize - 1) >> sb->s_blocksize_bits; +} + +static inline struct ntfs_inode *ntfs_i(struct inode *inode) +{ + return container_of(inode, struct ntfs_inode, vfs_inode); +} + +static inline bool is_compressed(const struct ntfs_inode *ni) +{ + return (ni->std_fa & FILE_ATTRIBUTE_COMPRESSED) || + (ni->ni_flags & NI_FLAG_COMPRESSED_MASK); +} + +static inline bool is_bad_ni(const struct ntfs_inode *ni) +{ + return ni->ni_bad; +} + +static inline int ni_ext_compress_bits(const struct ntfs_inode *ni) +{ + return 0xb + (ni->ni_flags & NI_FLAG_COMPRESSED_MASK); +} + +/* Bits - 0xc, 0xd, 0xe, 0xf, 0x10 */ +static inline void ni_set_ext_compress_bits(struct ntfs_inode *ni, u8 bits) +{ + ni->ni_flags |= (bits - 0xb) & NI_FLAG_COMPRESSED_MASK; +} + +static inline bool is_dedup(const struct ntfs_inode *ni) +{ + return ni->ni_flags & NI_FLAG_DEDUPLICATED; +} + +static inline bool is_encrypted(const struct ntfs_inode *ni) +{ + return ni->std_fa & FILE_ATTRIBUTE_ENCRYPTED; +} + +static inline bool is_sparsed(const struct ntfs_inode *ni) +{ + return ni->std_fa & FILE_ATTRIBUTE_SPARSE_FILE; +} + +static inline int is_resident(struct ntfs_inode *ni) +{ + return ni->ni_flags & NI_FLAG_RESIDENT; +} + +static inline loff_t ntfs_get_maxbytes(struct ntfs_inode *ni) +{ + struct ntfs_sb_info *sbi = ni->mi.sbi; + return is_sparsed(ni) || is_compressed(ni) ? sbi->maxbytes_sparse : + sbi->maxbytes; +} + +static inline void le16_sub_cpu(__le16 *var, u16 val) +{ + *var = cpu_to_le16(le16_to_cpu(*var) - val); +} + +static inline void le32_sub_cpu(__le32 *var, u32 val) +{ + *var = cpu_to_le32(le32_to_cpu(*var) - val); +} + +static inline void nb_put(struct ntfs_buffers *nb) +{ + u32 i, nbufs = nb->nbufs; + + if (!nbufs) + return; + + for (i = 0; i < nbufs; i++) + put_bh(nb->bh[i]); + nb->nbufs = 0; +} + +static inline void put_indx_node(struct indx_node *in) +{ + if (!in) + return; + + kfree(in->index); + nb_put(&in->nb); + kfree(in); +} + +static inline void mi_clear(struct mft_inode *mi) +{ + nb_put(&mi->nb); + kfree(mi->mrec); + mi->mrec = NULL; +} + +static inline void ni_lock(struct ntfs_inode *ni) +{ + mutex_lock_nested(&ni->base->ni_lock, NTFS_INODE_MUTEX_NORMAL); +} + +static inline void ni_lock_dir(struct ntfs_inode *ni) +{ + mutex_lock_nested(&ni->base->ni_lock, NTFS_INODE_MUTEX_PARENT); +} + +static inline void ni_lock_dir2(struct ntfs_inode *ni) +{ + mutex_lock_nested(&ni->base->ni_lock, NTFS_INODE_MUTEX_PARENT2); +} + +static inline void ni_unlock(struct ntfs_inode *ni) +{ + mutex_unlock(&ni->base->ni_lock); +} + +static inline int ni_trylock(struct ntfs_inode *ni) +{ + return mutex_trylock(&ni->base->ni_lock); +} + +static inline int attr_load_runs_attr(struct ntfs_inode *ni, + struct ATTRIB *attr, + struct runs_tree *run, CLST vcn) +{ + return attr_load_runs_vcn(ni, attr->type, attr_name(attr), + attr->name_len, run, vcn); +} + +static inline void le64_sub_cpu(__le64 *var, u64 val) +{ + *var = cpu_to_le64(le64_to_cpu(*var) - val); +} + +#endif /* _LINUX_NTFS3_NTFS_FS_H */ diff --git a/fs/ntfs3/record.c b/fs/ntfs3/record.c new file mode 100644 index 000000000..4f12ce15b --- /dev/null +++ b/fs/ntfs3/record.c @@ -0,0 +1,690 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + */ + +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +static inline int compare_attr(const struct ATTRIB *left, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, + const u16 *upcase) +{ + /* First, compare the type codes. */ + int diff = le32_to_cpu(left->type) - le32_to_cpu(type); + + if (diff) + return diff; + + /* They have the same type code, so we have to compare the names. */ + return ntfs_cmp_names(attr_name(left), left->name_len, name, name_len, + upcase, true); +} + +/* + * mi_new_attt_id + * + * Return: Unused attribute id that is less than mrec->next_attr_id. + */ +static __le16 mi_new_attt_id(struct ntfs_inode *ni, struct mft_inode *mi) +{ + u16 free_id, max_id, t16; + struct MFT_REC *rec = mi->mrec; + struct ATTRIB *attr; + __le16 id; + + id = rec->next_attr_id; + free_id = le16_to_cpu(id); + if (free_id < 0x7FFF) { + rec->next_attr_id = cpu_to_le16(free_id + 1); + return id; + } + + /* One record can store up to 1024/24 ~= 42 attributes. */ + free_id = 0; + max_id = 0; + + attr = NULL; + + for (;;) { + attr = mi_enum_attr(ni, mi, attr); + if (!attr) { + rec->next_attr_id = cpu_to_le16(max_id + 1); + mi->dirty = true; + return cpu_to_le16(free_id); + } + + t16 = le16_to_cpu(attr->id); + if (t16 == free_id) { + free_id += 1; + attr = NULL; + } else if (max_id < t16) + max_id = t16; + } +} + +int mi_get(struct ntfs_sb_info *sbi, CLST rno, struct mft_inode **mi) +{ + int err; + struct mft_inode *m = kzalloc_obj(struct mft_inode, GFP_NOFS); + + if (!m) + return -ENOMEM; + + err = mi_init(m, sbi, rno); + if (err) { + kfree(m); + return err; + } + + err = mi_read(m, false); + if (err) { + mi_put(m); + return err; + } + + *mi = m; + return 0; +} + +void mi_put(struct mft_inode *mi) +{ + mi_clear(mi); + kfree(mi); +} + +int mi_init(struct mft_inode *mi, struct ntfs_sb_info *sbi, CLST rno) +{ + mi->sbi = sbi; + mi->rno = rno; + mi->mrec = kmalloc(sbi->record_size, GFP_NOFS); + if (!mi->mrec) + return -ENOMEM; + + return 0; +} + +/* + * mi_read - Read MFT data. + */ +int mi_read(struct mft_inode *mi, bool is_mft) +{ + int err; + struct MFT_REC *rec = mi->mrec; + struct ntfs_sb_info *sbi = mi->sbi; + u32 bpr = sbi->record_size; + u64 vbo = (u64)mi->rno << sbi->record_bits; + struct ntfs_inode *mft_ni = sbi->mft.ni; + struct runs_tree *run = mft_ni ? &mft_ni->file.run : NULL; + struct rw_semaphore *rw_lock = NULL; + + if (is_mounted(sbi)) { + if (!is_mft && mft_ni) { + rw_lock = &mft_ni->file.run_lock; + down_read(rw_lock); + } + } + + err = ntfs_read_bh(sbi, run, vbo, &rec->rhdr, bpr, &mi->nb); + if (rw_lock) + up_read(rw_lock); + if (!err) + goto ok; + + if (err == -E_NTFS_FIXUP) { + mi->dirty = true; + goto ok; + } + + if (err != -ENOENT) + goto out; + + if (rw_lock) { + ni_lock(mft_ni); + down_write(rw_lock); + } + err = attr_load_runs_vcn(mft_ni, ATTR_DATA, NULL, 0, run, + vbo >> sbi->cluster_bits); + if (rw_lock) { + up_write(rw_lock); + ni_unlock(mft_ni); + } + if (err) + goto out; + + if (rw_lock) + down_read(rw_lock); + err = ntfs_read_bh(sbi, run, vbo, &rec->rhdr, bpr, &mi->nb); + if (rw_lock) + up_read(rw_lock); + + if (err == -E_NTFS_FIXUP) { + mi->dirty = true; + goto ok; + } + if (err) + goto out; + +ok: + /* Check field 'total' only here. */ + if (le32_to_cpu(rec->total) != bpr) { + err = -EINVAL; + goto out; + } + + return 0; + +out: + if (err == -E_NTFS_CORRUPT) { + ntfs_err(sbi->sb, "mft corrupted"); + ntfs_set_state(sbi, NTFS_DIRTY_ERROR); + err = -EINVAL; + } + + return err; +} + +/* + * mi_enum_attr - start/continue attributes enumeration in record. + * + * NOTE: mi->mrec - memory of size sbi->record_size + * here we sure that mi->mrec->total == sbi->record_size (see mi_read) + */ +struct ATTRIB *mi_enum_attr(struct ntfs_inode *ni, struct mft_inode *mi, + struct ATTRIB *attr) +{ + const struct MFT_REC *rec = mi->mrec; + u32 used = le32_to_cpu(rec->used); + u32 t32, off, asize, prev_type; + u16 t16; + u64 svcn, evcn, data_size, alloc_size, tot_size; + + if (!attr) { + u32 total = le32_to_cpu(rec->total); + + off = le16_to_cpu(rec->attr_off); + + if (used > total) + goto out; + + if (off >= used || off < MFTRECORD_FIXUP_OFFSET_1 || + !IS_ALIGNED(off, 8)) { + goto out; + } + + /* Skip non-resident records. */ + if (!is_rec_inuse(rec)) + return NULL; + + prev_type = 0; + attr = Add2Ptr(rec, off); + } else { + /* + * We don't need to check previous attr here. There is + * a bounds checking in the previous round. + */ + off = PtrOffset(rec, attr); + + asize = le32_to_cpu(attr->size); + + prev_type = le32_to_cpu(attr->type); + attr = Add2Ptr(attr, asize); + off += asize; + } + + /* + * Can we use the first fields: + * attr->type, + * attr->size + */ + if (off + 8 > used) { + static_assert(ALIGN(sizeof(enum ATTR_TYPE), 8) == 8); + goto out; + } + + if (attr->type == ATTR_END) { + /* End of enumeration. */ + return NULL; + } + + /* 0x100 is last known attribute for now. */ + t32 = le32_to_cpu(attr->type); + if (!t32 || (t32 & 0xf) || (t32 > 0x100)) + goto out; + + /* attributes in record must be ordered by type */ + if (t32 < prev_type) + goto out; + + asize = le32_to_cpu(attr->size); + + if (!IS_ALIGNED(asize, 8)) + goto out; + + /* Check overflow and boundary. */ + if (off + asize < off || off + asize > used) + goto out; + + /* Can we use the field attr->non_res. */ + if (off + 9 > used) + goto out; + + /* Check size of attribute. */ + if (!attr->non_res) { + /* Check resident fields. */ + if (asize < SIZEOF_RESIDENT) + goto out; + + t16 = le16_to_cpu(attr->res.data_off); + if (t16 > asize) + goto out; + + if (le32_to_cpu(attr->res.data_size) > asize - t16) + goto out; + + t32 = sizeof(short) * attr->name_len; + if (t32 && le16_to_cpu(attr->name_off) + t32 > t16) + goto out; + + return attr; + } + + /* Check nonresident fields. */ + if (attr->non_res != 1) + goto out; + + /* Can we use memory including attr->nres.valid_size? */ + if (asize < SIZEOF_NONRESIDENT) + goto out; + + t16 = le16_to_cpu(attr->nres.run_off); + if (t16 > asize) + goto out; + + t32 = sizeof(short) * attr->name_len; + if (t32 && le16_to_cpu(attr->name_off) + t32 > t16) + goto out; + + /* + * Check start/end vcn. svcn == 0 with evcn == -1 (U64_MAX) is the + * sentinel for an empty non-resident attribute (no allocated + * clusters) and must be accepted: "svcn > evcn + 1" tolerates it, + * since "(u64)-1 + 1" is 0 and "0 > 0" is false. + * + * For a non-empty attribute evcn is a cluster index and must lie + * within the volume (sbi->used.bitmap.nbits, set up in + * ntfs_init_from_boot() before any caller of mi_enum_attr() runs). + * Bounding evcn also prevents a malformed value close to U64_MAX + * from slipping through the near-wrap "evcn + 1" upper bound. + */ + svcn = le64_to_cpu(attr->nres.svcn); + evcn = le64_to_cpu(attr->nres.evcn); + if (svcn > evcn + 1) + goto out; + + if (is_attr_ext(attr)) { + /* sparsed/compressed attribute. */ +#ifdef CONFIG_NTFS3_64BIT_CLUSTER + /* No limits. */ +#else + /* Check evcn fits into 32 bits. */ + if (evcn != U64_MAX && evcn >= (1ull << 32)) + goto out; +#endif + } else { + /* Check out of volume for normal attribute. */ + if (evcn != U64_MAX && evcn >= mi->sbi->used.bitmap.nbits) + goto out; + } + + data_size = le64_to_cpu(attr->nres.data_size); + if (le64_to_cpu(attr->nres.valid_size) > data_size) + goto out; + + alloc_size = le64_to_cpu(attr->nres.alloc_size); + if (data_size > alloc_size) + goto out; + + t32 = mi->sbi->cluster_mask; + if (alloc_size & t32) + goto out; + + if (!attr->nres.svcn && is_attr_ext(attr)) { + /* First segment of sparse/compressed attribute */ + /* Can we use memory including attr->nres.total_size? */ + if (asize < SIZEOF_NONRESIDENT_EX) + goto out; + + tot_size = le64_to_cpu(attr->nres.total_size); + if (tot_size & t32) + goto out; + + if (tot_size > alloc_size) + goto out; + } else { + if (attr->nres.c_unit) + goto out; + + if (alloc_size > mi->sbi->volume.size) + goto out; + } + + return attr; + +out: + _ntfs_bad_inode(&ni->vfs_inode); + return NULL; +} + +/* + * mi_find_attr - Find the attribute by type and name and id. + */ +struct ATTRIB *mi_find_attr(struct ntfs_inode *ni, struct mft_inode *mi, + struct ATTRIB *attr, enum ATTR_TYPE type, + const __le16 *name, u8 name_len, const __le16 *id) +{ + u32 type_in = le32_to_cpu(type); + u32 atype; + +next_attr: + attr = mi_enum_attr(ni, mi, attr); + if (!attr) + return NULL; + + atype = le32_to_cpu(attr->type); + if (atype > type_in) + return NULL; + + if (atype < type_in) + goto next_attr; + + if (attr->name_len != name_len) + goto next_attr; + + if (name_len && memcmp(attr_name(attr), name, name_len * sizeof(short))) + goto next_attr; + + if (id && *id != attr->id) + goto next_attr; + + return attr; +} + +int mi_write(struct mft_inode *mi, int wait) +{ + struct MFT_REC *rec; + int err; + struct ntfs_sb_info *sbi; + + if (!mi->dirty) + return 0; + + sbi = mi->sbi; + rec = mi->mrec; + + err = ntfs_write_bh(sbi, &rec->rhdr, &mi->nb, wait); + if (err) + return err; + + if (mi->rno < sbi->mft.recs_mirr) + sbi->flags |= NTFS_FLAGS_MFTMIRR; + + mi->dirty = false; + + return 0; +} + +int mi_format_new(struct mft_inode *mi, struct ntfs_sb_info *sbi, CLST rno, + __le16 flags, bool is_mft) +{ + int err; + u16 seq = 1; + struct MFT_REC *rec; + u64 vbo = (u64)rno << sbi->record_bits; + + err = mi_init(mi, sbi, rno); + if (err) + return err; + + rec = mi->mrec; + + if (rno == MFT_REC_MFT) { + ; + } else if (rno < MFT_REC_FREE) { + seq = rno; + } else if (rno >= sbi->mft.used) { + ; + } else if (mi_read(mi, is_mft)) { + ; + } else if (rec->rhdr.sign == NTFS_FILE_SIGNATURE) { + /* Record is reused. Update its sequence number. */ + seq = le16_to_cpu(rec->seq) + 1; + if (!seq) + seq = 1; + } + + memcpy(rec, sbi->new_rec, sbi->record_size); + + rec->seq = cpu_to_le16(seq); + rec->flags = RECORD_FLAG_IN_USE | flags; + if (MFTRECORD_FIXUP_OFFSET == MFTRECORD_FIXUP_OFFSET_3) + rec->mft_record = cpu_to_le32(rno); + + mi->dirty = true; + + if (!mi->nb.nbufs) { + struct ntfs_inode *ni = sbi->mft.ni; + bool lock = false; + + if (is_mounted(sbi) && !is_mft) { + down_read(&ni->file.run_lock); + lock = true; + } + + err = ntfs_get_bh(sbi, &ni->file.run, vbo, sbi->record_size, + &mi->nb); + if (lock) + up_read(&ni->file.run_lock); + } + + return err; +} + +/* + * mi_insert_attr - Reserve space for new attribute. + * + * Return: Not full constructed attribute or NULL if not possible to create. + */ +struct ATTRIB *mi_insert_attr(struct ntfs_inode *ni, struct mft_inode *mi, + enum ATTR_TYPE type, const __le16 *name, + u8 name_len, u32 asize, u16 name_off) +{ + size_t tail; + struct ATTRIB *attr; + __le16 id; + struct MFT_REC *rec = mi->mrec; + struct ntfs_sb_info *sbi = mi->sbi; + u32 used = le32_to_cpu(rec->used); + const u16 *upcase = sbi->upcase; + + /* Can we insert mi attribute? */ + if (used + asize > sbi->record_size) + return NULL; + + /* + * Scan through the list of attributes to find the point + * at which we should insert it. + */ + attr = NULL; + while ((attr = mi_enum_attr(ni, mi, attr))) { + int diff = compare_attr(attr, type, name, name_len, upcase); + + if (diff < 0) + continue; + + if (!diff && !is_attr_indexed(attr)) + return NULL; + break; + } + + if (!attr) { + /* Append. */ + tail = 8; + attr = Add2Ptr(rec, used - 8); + } else { + /* Insert before 'attr'. */ + tail = used - PtrOffset(rec, attr); + } + + id = mi_new_attt_id(ni, mi); + + memmove(Add2Ptr(attr, asize), attr, tail); + memset(attr, 0, asize); + + attr->type = type; + attr->size = cpu_to_le32(asize); + attr->name_len = name_len; + attr->name_off = cpu_to_le16(name_off); + attr->id = id; + + memmove(Add2Ptr(attr, name_off), name, name_len * sizeof(short)); + rec->used = cpu_to_le32(used + asize); + + mi->dirty = true; + + return attr; +} + +/* + * mi_remove_attr - Remove the attribute from record. + * + * NOTE: The source attr will point to next attribute. + */ +bool mi_remove_attr(struct ntfs_inode *ni, struct mft_inode *mi, + struct ATTRIB *attr) +{ + struct MFT_REC *rec = mi->mrec; + u32 aoff = PtrOffset(rec, attr); + u32 used = le32_to_cpu(rec->used); + u32 asize = le32_to_cpu(attr->size); + + if (aoff + asize > used) + return false; + + if (ni && is_attr_indexed(attr) && attr->type == ATTR_NAME) { + u16 links = le16_to_cpu(ni->mi.mrec->hard_links); + if (!links) { + /* minor error. Not critical. */ + } else { + ni->mi.mrec->hard_links = cpu_to_le16(links - 1); + ni->mi.dirty = true; + } + } + + used -= asize; + memmove(attr, Add2Ptr(attr, asize), used - aoff); + rec->used = cpu_to_le32(used); + mi->dirty = true; + + return true; +} + +/* bytes = "new attribute size" - "old attribute size" */ +bool mi_resize_attr(struct mft_inode *mi, struct ATTRIB *attr, int bytes) +{ + struct MFT_REC *rec = mi->mrec; + u32 aoff = PtrOffset(rec, attr); + u32 total, used = le32_to_cpu(rec->used); + u32 nsize, asize = le32_to_cpu(attr->size); + u32 rsize = le32_to_cpu(attr->res.data_size); + int tail = (int)(used - aoff - asize); + int dsize; + char *next; + + if (tail < 0 || aoff >= used) + return false; + + if (!bytes) + return true; + + total = le32_to_cpu(rec->total); + next = Add2Ptr(attr, asize); + + if (bytes > 0) { + dsize = ALIGN(bytes, 8); + if (used + dsize > total) + return false; + nsize = asize + dsize; + /* Move tail */ + memmove(next + dsize, next, tail); + memset(next, 0, dsize); + used += dsize; + rsize += dsize; + } else { + dsize = ALIGN(-bytes, 8); + if (dsize > asize) + return false; + nsize = asize - dsize; + memmove(next - dsize, next, tail); + used -= dsize; + rsize -= dsize; + } + + rec->used = cpu_to_le32(used); + attr->size = cpu_to_le32(nsize); + if (!attr->non_res) + attr->res.data_size = cpu_to_le32(rsize); + mi->dirty = true; + + return true; +} + +/* + * Pack runs in MFT record. + * If failed record is not changed. + */ +int mi_pack_runs(struct mft_inode *mi, struct ATTRIB *attr, + const struct runs_tree *run, CLST len) +{ + int err = 0; + struct ntfs_sb_info *sbi = mi->sbi; + u32 new_run_size; + CLST plen; + struct MFT_REC *rec = mi->mrec; + CLST svcn = le64_to_cpu(attr->nres.svcn); + u32 used = le32_to_cpu(rec->used); + u32 aoff = PtrOffset(rec, attr); + u32 asize = le32_to_cpu(attr->size); + char *next = Add2Ptr(attr, asize); + u16 run_off = le16_to_cpu(attr->nres.run_off); + u32 run_size = asize - run_off; + u32 tail = used - aoff - asize; + u32 dsize = sbi->record_size - used; + + /* Make a maximum gap in current record. */ + memmove(next + dsize, next, tail); + + /* Pack as much as possible. */ + err = run_pack(run, svcn, len, Add2Ptr(attr, run_off), run_size + dsize, + &plen); + if (err < 0) { + memmove(next, next + dsize, tail); + return err; + } + + new_run_size = ALIGN(err, 8); + + memmove(next + new_run_size - run_size, next + dsize, tail); + + attr->size = cpu_to_le32(asize + new_run_size - run_size); + attr->nres.evcn = cpu_to_le64(svcn + plen - 1); + rec->used = cpu_to_le32(used + new_run_size - run_size); + mi->dirty = true; + + return 0; +} diff --git a/fs/ntfs3/run.c b/fs/ntfs3/run.c new file mode 100644 index 000000000..6e3ef89fc --- /dev/null +++ b/fs/ntfs3/run.c @@ -0,0 +1,1423 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + * TODO: try to use extents tree (instead of array) + */ + +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +/* runs_tree is a continues memory. Try to avoid big size. */ +#define NTFS3_RUN_MAX_BYTES 0x10000 + +struct ntfs_run { + CLST vcn; /* Virtual cluster number. */ + CLST len; /* Length in clusters. */ + CLST lcn; /* Logical cluster number. */ +}; + +/* + * run_lookup - Lookup the index of a MCB entry that is first <= vcn. + * + * Case of success it will return non-zero value and set + * @index parameter to index of entry been found. + * Case of entry missing from list 'index' will be set to + * point to insertion position for the entry question. + */ +static bool run_lookup(const struct runs_tree *run, CLST vcn, size_t *index) +{ + size_t min_idx, max_idx, mid_idx; + struct ntfs_run *r; + + if (!run->count) { + *index = 0; + return false; + } + + min_idx = 0; + max_idx = run->count - 1; + + /* Check boundary cases specially, 'cause they cover the often requests. */ + r = run->runs; + if (vcn < r->vcn) { + *index = 0; + return false; + } + + if (vcn < r->vcn + r->len) { + *index = 0; + return true; + } + + r += max_idx; + if (vcn >= r->vcn + r->len) { + *index = run->count; + return false; + } + + if (vcn >= r->vcn) { + *index = max_idx; + return true; + } + + do { + mid_idx = min_idx + ((max_idx - min_idx) >> 1); + r = run->runs + mid_idx; + + if (vcn < r->vcn) { + max_idx = mid_idx - 1; + if (!mid_idx) + break; + } else if (vcn >= r->vcn + r->len) { + min_idx = mid_idx + 1; + } else { + *index = mid_idx; + return true; + } + } while (min_idx <= max_idx); + + *index = max_idx + 1; + return false; +} + +/* + * run_consolidate - Consolidate runs starting from a given one. + */ +static void run_consolidate(struct runs_tree *run, size_t index) +{ + size_t i; + struct ntfs_run *r = run->runs + index; + + while (index + 1 < run->count) { + /* + * I should merge current run with next + * if start of the next run lies inside one being tested. + */ + struct ntfs_run *n = r + 1; + CLST end = r->vcn + r->len; + CLST dl; + + /* Stop if runs are not aligned one to another. */ + if (n->vcn > end) + break; + + dl = end - n->vcn; + + /* + * If range at index overlaps with next one + * then I will either adjust it's start position + * or (if completely matches) dust remove one from the list. + */ + if (dl > 0) { + if (n->len <= dl) + goto remove_next_range; + + n->len -= dl; + n->vcn += dl; + if (n->lcn != SPARSE_LCN) + n->lcn += dl; + dl = 0; + } + + /* + * Stop if sparse mode does not match + * both current and next runs. + */ + if ((n->lcn == SPARSE_LCN) != (r->lcn == SPARSE_LCN)) { + index += 1; + r = n; + continue; + } + + /* + * Check if volume block + * of a next run lcn does not match + * last volume block of the current run. + */ + if (n->lcn != SPARSE_LCN && n->lcn != r->lcn + r->len) + break; + + /* + * Next and current are siblings. + * Eat/join. + */ + r->len += n->len - dl; + +remove_next_range: + i = run->count - (index + 1); + if (i > 1) + memmove(n, n + 1, sizeof(*n) * (i - 1)); + + run->count -= 1; + } +} + +/* + * run_is_mapped_full + * + * Return: True if range [svcn - evcn] is mapped. + */ +bool run_is_mapped_full(const struct runs_tree *run, CLST svcn, CLST evcn) +{ + size_t i; + const struct ntfs_run *r, *end; + CLST next_vcn; + + if (!run_lookup(run, svcn, &i)) + return false; + + end = run->runs + run->count; + r = run->runs + i; + + for (;;) { + next_vcn = r->vcn + r->len; + if (next_vcn > evcn) + return true; + + if (++r >= end) + return false; + + if (r->vcn != next_vcn) + return false; + } +} + +bool run_lookup_entry(const struct runs_tree *run, CLST vcn, CLST *lcn, + CLST *len, size_t *index) +{ + size_t idx; + CLST gap; + struct ntfs_run *r; + + /* Fail immediately if nrun was not touched yet. */ + if (!run->runs) + return false; + + if (!run_lookup(run, vcn, &idx)) + return false; + + r = run->runs + idx; + + if (vcn >= r->vcn + r->len) + return false; + + gap = vcn - r->vcn; + if (r->len <= gap) + return false; + + *lcn = r->lcn == SPARSE_LCN ? SPARSE_LCN : (r->lcn + gap); + + if (len) + *len = r->len - gap; + if (index) + *index = idx; + + return true; +} + +/* + * run_overlaps + * + * true if run overlaps with range [svcn, svcn + len) + */ +static bool run_overlaps(const struct runs_tree *run, CLST svcn, CLST len, + CLST *vcn, CLST *clen) +{ + size_t i; + const struct ntfs_run *r = run->runs; + CLST end = svcn + len; + + for (i = 0; i < run->count; i++, r++) { + /* Check if [r->vcn, r->vcn+r->len) overlaps [svcn, end). */ + if (r->vcn < end && svcn < r->vcn + r->len) { + if (vcn) + *vcn = r->vcn; + if (clen) + *clen = r->len; + return true; + } + } + + return false; +} + +/* + * run_lookup_entry_da + * + * - lookup vcn in delalloc run + * - lookup vcn in real run + * - correct result if real run overlaps with delalloc + */ +bool run_lookup_entry_da(const struct runs_tree *run, + const struct runs_tree *run_da, CLST vcn, CLST *lcn, + CLST *len) +{ + CLST vcn1, len1; + + if (run_da && run_lookup_entry(run_da, vcn, lcn, len, NULL)) { + *lcn = DELALLOC_LCN; + return true; + } + + if (!run_lookup_entry(run, vcn, lcn, len, NULL)) + return false; + + if (run_da && run_overlaps(run_da, vcn, *len, &vcn1, &len1)) { + /* Correct return value. */ + if (vcn1 > vcn) { + *len = vcn1 - vcn; + } else { + *lcn = DELALLOC_LCN; + *len = len1; + } + } + + return true; +} + +/* + * run_truncate_head - Decommit the range before vcn. + */ +void run_truncate_head(struct runs_tree *run, CLST vcn) +{ + size_t index; + struct ntfs_run *r; + + if (run_lookup(run, vcn, &index)) { + r = run->runs + index; + + if (vcn > r->vcn) { + CLST dlen = vcn - r->vcn; + + r->vcn = vcn; + r->len -= dlen; + if (r->lcn != SPARSE_LCN) + r->lcn += dlen; + } + + if (!index) + return; + } + r = run->runs; + memmove(r, r + index, sizeof(*r) * (run->count - index)); + + run->count -= index; + + if (!run->count) { + kvfree(run->runs); + run->runs = NULL; + run->allocated = 0; + } +} + +/* + * run_truncate - Decommit the range after vcn. + */ +void run_truncate(struct runs_tree *run, CLST vcn) +{ + size_t index; + + /* + * If I hit the range then + * I have to truncate one. + * If range to be truncated is becoming empty + * then it will entirely be removed. + */ + if (run_lookup(run, vcn, &index)) { + struct ntfs_run *r = run->runs + index; + + r->len = vcn - r->vcn; + + if (r->len > 0) + index += 1; + } + + /* + * At this point 'index' is set to position that + * should be thrown away (including index itself) + * Simple one - just set the limit. + */ + run->count = index; + + /* Do not reallocate array 'runs'. Only free if possible. */ + if (!index) { + kvfree(run->runs); + run->runs = NULL; + run->allocated = 0; + } +} + +/* + * run_truncate_around - Trim head and tail if necessary. + */ +void run_truncate_around(struct runs_tree *run, CLST vcn) +{ + run_truncate_head(run, vcn); + + if (run->count >= NTFS3_RUN_MAX_BYTES / sizeof(struct ntfs_run) / 2) + run_truncate(run, (run->runs + (run->count >> 1))->vcn); +} + +/* + * run_add_entry + * + * Sets location to known state. + * Run to be added may overlap with existing location. + * + * Return: false if of memory. + */ +bool run_add_entry(struct runs_tree *run, CLST vcn, CLST lcn, CLST len, + bool is_mft) +{ + size_t used, index; + struct ntfs_run *r; + bool inrange; + CLST tail_vcn = 0, tail_len = 0, tail_lcn = 0; + bool should_add_tail = false; + + /* + * Lookup the insertion point. + * + * Execute bsearch for the entry containing + * start position question. + */ + inrange = run_lookup(run, vcn, &index); + + /* + * Shortcut here would be case of + * range not been found but one been added + * continues previous run. + * This case I can directly make use of + * existing range as my start point. + */ + if (!inrange && index > 0) { + struct ntfs_run *t = run->runs + index - 1; + + if (t->vcn + t->len == vcn && + (t->lcn == SPARSE_LCN) == (lcn == SPARSE_LCN) && + (lcn == SPARSE_LCN || lcn == t->lcn + t->len)) { + inrange = true; + index -= 1; + } + } + + /* + * At this point 'index' either points to the range + * containing start position or to the insertion position + * for a new range. + * So first let's check if range I'm probing is here already. + */ + if (!inrange) { +requires_new_range: + /* + * Range was not found. + * Insert at position 'index' + */ + used = run->count * sizeof(struct ntfs_run); + + /* + * Check allocated space. + * If one is not enough to get one more entry + * then it will be reallocated. + */ + if (run->allocated < used + sizeof(struct ntfs_run)) { + size_t bytes; + struct ntfs_run *new_ptr; + + /* Use power of 2 for 'bytes'. */ + if (!used) { + bytes = 64; + } else if (used <= 16 * PAGE_SIZE) { + if (is_power_of_2(run->allocated)) + bytes = run->allocated << 1; + else + bytes = (size_t)1 + << (2 + blksize_bits(used)); + } else { + bytes = run->allocated + (16 * PAGE_SIZE); + } + + WARN_ON(!is_mft && bytes > NTFS3_RUN_MAX_BYTES); + + new_ptr = kvmalloc(bytes, GFP_KERNEL); + + if (!new_ptr) + return false; + + r = new_ptr + index; + memcpy(new_ptr, run->runs, + index * sizeof(struct ntfs_run)); + memcpy(r + 1, run->runs + index, + sizeof(struct ntfs_run) * (run->count - index)); + + kvfree(run->runs); + run->runs = new_ptr; + run->allocated = bytes; + + } else { + size_t i = run->count - index; + + r = run->runs + index; + + /* memmove appears to be a bottle neck here... */ + if (i > 0) + memmove(r + 1, r, sizeof(struct ntfs_run) * i); + } + + r->vcn = vcn; + r->lcn = lcn; + r->len = len; + run->count += 1; + } else { + r = run->runs + index; + + /* + * If one of ranges was not allocated then we + * have to split location we just matched and + * insert current one. + * A common case this requires tail to be reinserted + * a recursive call. + */ + if (((lcn == SPARSE_LCN) != (r->lcn == SPARSE_LCN)) || + (lcn != SPARSE_LCN && lcn != r->lcn + (vcn - r->vcn))) { + CLST to_eat = vcn - r->vcn; + CLST Tovcn = to_eat + len; + + should_add_tail = Tovcn < r->len; + + if (should_add_tail) { + tail_lcn = r->lcn == SPARSE_LCN ? + SPARSE_LCN : + (r->lcn + Tovcn); + tail_vcn = r->vcn + Tovcn; + tail_len = r->len - Tovcn; + } + + if (to_eat > 0) { + r->len = to_eat; + inrange = false; + index += 1; + goto requires_new_range; + } + + /* lcn should match one were going to add. */ + r->lcn = lcn; + } + + /* + * If existing range fits then were done. + * Otherwise extend found one and fall back to range join code. + */ + if (r->vcn + r->len < vcn + len) + r->len += len - ((r->vcn + r->len) - vcn); + } + + /* + * And normalize it starting from insertion point. + * It's possible that no insertion needed case if + * start point lies within the range of an entry + * that 'index' points to. + */ + if (inrange && index > 0) + index -= 1; + run_consolidate(run, index); + run_consolidate(run, index + 1); + + /* + * A special case. + * We have to add extra range a tail. + */ + if (should_add_tail && + !run_add_entry(run, tail_vcn, tail_lcn, tail_len, is_mft)) + return false; + + return true; +} + +/* + * run_collapse_range + * + * Helper for attr_collapse_range(), + * which is helper for fallocate(collapse_range). + */ +bool run_collapse_range(struct runs_tree *run, CLST vcn, CLST len, CLST sub) +{ + size_t index, eat; + struct ntfs_run *r, *e, *eat_start, *eat_end; + CLST end; + + if (!run_lookup(run, vcn, &index) && index >= run->count) { + return true; + } + + e = run->runs + run->count; + r = run->runs + index; + end = vcn + len; + + if (vcn > r->vcn) { + if (r->vcn + r->len <= end) { + /* Collapse tail of run .*/ + r->len = vcn - r->vcn; + } else if (r->lcn == SPARSE_LCN) { + /* Collapse a middle part of sparsed run. */ + r->len -= len; + } else { + /* Collapse a middle part of normal run, split. */ + if (!run_add_entry(run, vcn, SPARSE_LCN, len, false)) + return false; + return run_collapse_range(run, vcn, len, sub); + } + + r += 1; + } + + eat_start = r; + eat_end = r; + + for (; r < e; r++) { + CLST d; + + if (r->vcn >= end) { + r->vcn -= len; + continue; + } + + if (r->vcn + r->len <= end) { + /* Eat this run. */ + eat_end = r + 1; + continue; + } + + d = end - r->vcn; + if (r->lcn != SPARSE_LCN) + r->lcn += d; + r->len -= d; + r->vcn -= len - d; + } + + eat = eat_end - eat_start; + memmove(eat_start, eat_end, (e - eat_end) * sizeof(*r)); + run->count -= eat; + + if (sub) { + e -= eat; + for (r = run->runs; r < e; r++) { + r->vcn -= sub; + } + } + + return true; +} + +/* run_insert_range + * + * Helper for attr_insert_range(), + * which is helper for fallocate(insert_range). + */ +int run_insert_range(struct runs_tree *run, CLST vcn, CLST len) +{ + size_t index; + struct ntfs_run *r, *e; + + if (WARN_ON(!run_lookup(run, vcn, &index))) + return -EINVAL; /* Should never be here. */ + + e = run->runs + run->count; + r = run->runs + index; + + if (vcn > r->vcn) + r += 1; + + for (; r < e; r++) + r->vcn += len; + + r = run->runs + index; + + if (vcn > r->vcn) { + /* split fragment. */ + CLST len1 = vcn - r->vcn; + CLST len2 = r->len - len1; + CLST lcn2 = r->lcn == SPARSE_LCN ? SPARSE_LCN : (r->lcn + len1); + + r->len = len1; + + if (!run_add_entry(run, vcn + len, lcn2, len2, false)) + return -ENOMEM; + } + + if (!run_add_entry(run, vcn, SPARSE_LCN, len, false)) + return -ENOMEM; + + return 0; +} + +/* run_insert_range_da + * + * Helper for attr_insert_range(), + * which is helper for fallocate(insert_range). + */ +int run_insert_range_da(struct runs_tree *run, CLST vcn, CLST len) +{ + struct ntfs_run *r, *r0 = NULL, *e = run->runs + run->count; + ; + + for (r = run->runs; r < e; r++) { + CLST end = r->vcn + r->len; + + if (vcn >= end) + continue; + + if (!r0 && r->vcn < vcn) { + r0 = r; + } else { + r->vcn += len; + } + } + + if (r0) { + /* split fragment. */ + CLST len1 = vcn - r0->vcn; + CLST len2 = r0->len - len1; + + r0->len = len1; + if (!run_add_entry(run, vcn + len, SPARSE_LCN, len2, false)) + return -ENOMEM; + } + + return 0; +} + +/* + * run_get_entry - Return index-th mapped region. + */ +bool run_get_entry(const struct runs_tree *run, size_t index, CLST *vcn, + CLST *lcn, CLST *len) +{ + const struct ntfs_run *r; + + if (index >= run->count) + return false; + + r = run->runs + index; + + if (!r->len) + return false; + + if (vcn) + *vcn = r->vcn; + if (lcn) + *lcn = r->lcn; + if (len) + *len = r->len; + return true; +} + +/* + * run_packed_size - Calculate the size of packed int64. + */ +#ifdef __BIG_ENDIAN +static inline int run_packed_size(const s64 n) +{ + const u8 *p = (const u8 *)&n + sizeof(n) - 1; + + if (n >= 0) { + if (p[-7] || p[-6] || p[-5] || p[-4]) + p -= 4; + if (p[-3] || p[-2]) + p -= 2; + if (p[-1]) + p -= 1; + if (p[0] & 0x80) + p -= 1; + } else { + if (p[-7] != 0xff || p[-6] != 0xff || p[-5] != 0xff || + p[-4] != 0xff) + p -= 4; + if (p[-3] != 0xff || p[-2] != 0xff) + p -= 2; + if (p[-1] != 0xff) + p -= 1; + if (!(p[0] & 0x80)) + p -= 1; + } + return (const u8 *)&n + sizeof(n) - p; +} + +/* Full trusted function. It does not check 'size' for errors. */ +static inline void run_pack_s64(u8 *run_buf, u8 size, s64 v) +{ + const u8 *p = (u8 *)&v; + + switch (size) { + case 8: + run_buf[7] = p[0]; + fallthrough; + case 7: + run_buf[6] = p[1]; + fallthrough; + case 6: + run_buf[5] = p[2]; + fallthrough; + case 5: + run_buf[4] = p[3]; + fallthrough; + case 4: + run_buf[3] = p[4]; + fallthrough; + case 3: + run_buf[2] = p[5]; + fallthrough; + case 2: + run_buf[1] = p[6]; + fallthrough; + case 1: + run_buf[0] = p[7]; + } +} + +/* Full trusted function. It does not check 'size' for errors. */ +static inline s64 run_unpack_s64(const u8 *run_buf, u8 size, s64 v) +{ + u8 *p = (u8 *)&v; + + switch (size) { + case 8: + p[0] = run_buf[7]; + fallthrough; + case 7: + p[1] = run_buf[6]; + fallthrough; + case 6: + p[2] = run_buf[5]; + fallthrough; + case 5: + p[3] = run_buf[4]; + fallthrough; + case 4: + p[4] = run_buf[3]; + fallthrough; + case 3: + p[5] = run_buf[2]; + fallthrough; + case 2: + p[6] = run_buf[1]; + fallthrough; + case 1: + p[7] = run_buf[0]; + } + return v; +} + +#else + +static inline int run_packed_size(const s64 n) +{ + const u8 *p = (const u8 *)&n; + + if (n >= 0) { + if (p[7] || p[6] || p[5] || p[4]) + p += 4; + if (p[3] || p[2]) + p += 2; + if (p[1]) + p += 1; + if (p[0] & 0x80) + p += 1; + } else { + if (p[7] != 0xff || p[6] != 0xff || p[5] != 0xff || + p[4] != 0xff) + p += 4; + if (p[3] != 0xff || p[2] != 0xff) + p += 2; + if (p[1] != 0xff) + p += 1; + if (!(p[0] & 0x80)) + p += 1; + } + + return 1 + p - (const u8 *)&n; +} + +/* Full trusted function. It does not check 'size' for errors. */ +static inline void run_pack_s64(u8 *run_buf, u8 size, s64 v) +{ + const u8 *p = (u8 *)&v; + + /* memcpy( run_buf, &v, size); Is it faster? */ + switch (size) { + case 8: + run_buf[7] = p[7]; + fallthrough; + case 7: + run_buf[6] = p[6]; + fallthrough; + case 6: + run_buf[5] = p[5]; + fallthrough; + case 5: + run_buf[4] = p[4]; + fallthrough; + case 4: + run_buf[3] = p[3]; + fallthrough; + case 3: + run_buf[2] = p[2]; + fallthrough; + case 2: + run_buf[1] = p[1]; + fallthrough; + case 1: + run_buf[0] = p[0]; + } +} + +/* full trusted function. It does not check 'size' for errors */ +static inline s64 run_unpack_s64(const u8 *run_buf, u8 size, s64 v) +{ + u8 *p = (u8 *)&v; + + /* memcpy( &v, run_buf, size); Is it faster? */ + switch (size) { + case 8: + p[7] = run_buf[7]; + fallthrough; + case 7: + p[6] = run_buf[6]; + fallthrough; + case 6: + p[5] = run_buf[5]; + fallthrough; + case 5: + p[4] = run_buf[4]; + fallthrough; + case 4: + p[3] = run_buf[3]; + fallthrough; + case 3: + p[2] = run_buf[2]; + fallthrough; + case 2: + p[1] = run_buf[1]; + fallthrough; + case 1: + p[0] = run_buf[0]; + } + return v; +} +#endif + +/* + * run_pack - Pack runs into buffer. + * + * packed_vcns - How much runs we have packed. + * packed_size - How much bytes we have used run_buf. + */ +int run_pack(const struct runs_tree *run, CLST svcn, CLST len, u8 *run_buf, + u32 run_buf_size, CLST *packed_vcns) +{ + CLST next_vcn, vcn, lcn; + CLST prev_lcn = 0; + CLST evcn1 = svcn + len; + const struct ntfs_run *r, *r_end; + int packed_size = 0; + size_t i; + s64 dlcn; + int offset_size, size_size, tmp; + + *packed_vcns = 0; + + if (!len) + goto out; + + /* Check all required entries [svcn, encv1) available. */ + if (!run_lookup(run, svcn, &i)) + return -ENOENT; + + r_end = run->runs + run->count; + r = run->runs + i; + + for (next_vcn = r->vcn + r->len; next_vcn < evcn1; + next_vcn = r->vcn + r->len) { + if (++r >= r_end || r->vcn != next_vcn) + return -ENOENT; + } + + /* Repeat cycle above and pack runs. Assume no errors. */ + r = run->runs + i; + len = svcn - r->vcn; + vcn = svcn; + lcn = r->lcn == SPARSE_LCN ? SPARSE_LCN : (r->lcn + len); + len = r->len - len; + + for (;;) { + next_vcn = vcn + len; + if (next_vcn > evcn1) + len = evcn1 - vcn; + + /* How much bytes required to pack len. */ + size_size = run_packed_size(len); + + /* offset_size - How much bytes is packed dlcn. */ + if (lcn == SPARSE_LCN) { + offset_size = 0; + dlcn = 0; + } else { + /* NOTE: lcn can be less than prev_lcn! */ + dlcn = (s64)lcn - prev_lcn; + offset_size = run_packed_size(dlcn); + prev_lcn = lcn; + } + + tmp = run_buf_size - packed_size - 2 - offset_size; + if (tmp <= 0) + goto out; + + /* Can we store this entire run. */ + if (tmp < size_size) + goto out; + + if (run_buf) { + /* Pack run header. */ + run_buf[0] = ((u8)(size_size | (offset_size << 4))); + run_buf += 1; + + /* Pack the length of run. */ + run_pack_s64(run_buf, size_size, len); + + run_buf += size_size; + /* Pack the offset from previous LCN. */ + run_pack_s64(run_buf, offset_size, dlcn); + run_buf += offset_size; + } + + packed_size += 1 + offset_size + size_size; + *packed_vcns += len; + + if (packed_size + 1 >= run_buf_size || next_vcn >= evcn1) + goto out; + + r += 1; + vcn = r->vcn; + lcn = r->lcn; + len = r->len; + } + +out: + /* Store last zero. */ + if (run_buf) + run_buf[0] = 0; + + return packed_size + 1; +} + +/* + * run_unpack - Unpack packed runs from @run_buf. + * + * Return: Error if negative, or real used bytes. + */ +int run_unpack(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino, + CLST svcn, CLST evcn, CLST vcn, const u8 *run_buf, + int run_buf_size) +{ + u64 prev_lcn, vcn64, lcn, next_vcn; + const u8 *run_last, *run_0; + bool is_mft = ino == MFT_REC_MFT; + + if (run_buf_size < 0) + return -EINVAL; + + /* Check for empty. */ + if (evcn + 1 == svcn) + return 0; + + if (evcn < svcn) + return -EINVAL; + + run_0 = run_buf; + run_last = run_buf + run_buf_size; + prev_lcn = 0; + vcn64 = svcn; + + /* Read all runs the chain. */ + /* size_size - How much bytes is packed len. */ + while (run_buf < run_last) { + /* size_size - How much bytes is packed len. */ + u8 size_size = *run_buf & 0xF; + /* offset_size - How much bytes is packed dlcn. */ + u8 offset_size = *run_buf++ >> 4; + u64 len; + + if (!size_size) + break; + + /* + * Unpack runs. + * NOTE: Runs are stored little endian order + * "len" is unsigned value, "dlcn" is signed. + * Large positive number requires to store 5 bytes + * e.g.: 05 FF 7E FF FF 00 00 00 + */ + if (size_size > sizeof(len)) + return -EINVAL; + + if (run_buf + size_size > run_last) + return -EINVAL; + + len = run_unpack_s64(run_buf, size_size, 0); + /* Skip size_size. */ + run_buf += size_size; + + if (!len) + return -EINVAL; + + if (!offset_size) + lcn = SPARSE_LCN64; + else if (offset_size <= sizeof(s64)) { + s64 dlcn; + + if (run_buf + offset_size > run_last) + return -EINVAL; + + /* Initial value of dlcn is -1 or 0. */ + dlcn = (run_buf[offset_size - 1] & 0x80) ? (s64)-1 : 0; + dlcn = run_unpack_s64(run_buf, offset_size, dlcn); + /* Skip offset_size. */ + run_buf += offset_size; + + if (!dlcn) + return -EINVAL; + + /* Check special combination: 0 + SPARSE_LCN64. */ + if (!prev_lcn && dlcn == SPARSE_LCN64) { + lcn = SPARSE_LCN64; + } else if (check_add_overflow(prev_lcn, dlcn, &lcn)) { + return -EINVAL; + } + prev_lcn = lcn; + } else { + /* The size of 'dlcn' can't be > 8. */ + return -EINVAL; + } + + if (check_add_overflow(vcn64, len, &next_vcn)) + return -EINVAL; + + /* Check boundary. */ + if (next_vcn > evcn + 1) + return -EINVAL; + +#ifndef CONFIG_NTFS3_64BIT_CLUSTER + if (next_vcn > 0x100000000ull || (lcn + len) > 0x100000000ull) { + ntfs_err( + sbi->sb, + "This driver is compiled without CONFIG_NTFS3_64BIT_CLUSTER (like windows driver).\n" + "Volume contains 64 bits run: vcn %llx, lcn %llx, len %llx.\n" + "Activate CONFIG_NTFS3_64BIT_CLUSTER to process this case", + vcn64, lcn, len); + return -EOPNOTSUPP; + } +#endif + if (lcn != SPARSE_LCN64) { + u64 lcn_end; + + if (check_add_overflow(lcn, len, &lcn_end)) + return -EINVAL; + if (lcn_end > sbi->used.bitmap.nbits) { + /* LCN range is out of volume. */ + return -EINVAL; + } + } + + if (!run) + ; /* Called from check_attr(fslog.c) to check run. */ + else if (run == RUN_DEALLOCATE) { + /* + * Called from ni_delete_all to free clusters + * without storing in run. + */ + if (lcn != SPARSE_LCN64) + mark_as_free_ex(sbi, lcn, len, true); + } else if (vcn64 >= vcn) { + if (!run_add_entry(run, vcn64, lcn, len, is_mft)) + return -ENOMEM; + } else if (next_vcn > vcn) { + u64 dlen = vcn - vcn64; + + if (!run_add_entry(run, vcn, lcn + dlen, len - dlen, + is_mft)) + return -ENOMEM; + } + + vcn64 = next_vcn; + } + + if (vcn64 != evcn + 1) { + /* Not expected length of unpacked runs. */ + return -EINVAL; + } + + return run_buf - run_0; +} + +#ifdef NTFS3_CHECK_FREE_CLST +/* + * run_unpack_ex - Unpack packed runs from "run_buf". + * + * Checks unpacked runs to be used in bitmap. + * + * Return: Error if negative, or real used bytes. + */ +int run_unpack_ex(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino, + CLST svcn, CLST evcn, CLST vcn, const u8 *run_buf, + int run_buf_size) +{ + int ret, err; + CLST next_vcn, lcn, len; + size_t index, done; + bool ok, zone; + struct wnd_bitmap *wnd; + + ret = run_unpack(run, sbi, ino, svcn, evcn, vcn, run_buf, run_buf_size); + if (ret <= 0) + return ret; + + if (!sbi->used.bitmap.sb || !run || run == RUN_DEALLOCATE) + return ret; + + if (ino == MFT_REC_BADCLUST) + return ret; + + next_vcn = vcn = svcn; + wnd = &sbi->used.bitmap; + + for (ok = run_lookup_entry(run, vcn, &lcn, &len, &index); + next_vcn <= evcn; + ok = run_get_entry(run, ++index, &vcn, &lcn, &len)) { + if (!ok || next_vcn != vcn) + return -EINVAL; + + next_vcn = vcn + len; + + if (lcn == SPARSE_LCN) + continue; + + if (sbi->flags & NTFS_FLAGS_NEED_REPLAY) + continue; + + down_read_nested(&wnd->rw_lock, BITMAP_MUTEX_CLUSTERS); + zone = max(wnd->zone_bit, lcn) < min(wnd->zone_end, lcn + len); + /* Check for free blocks. */ + ok = !zone && wnd_is_used(wnd, lcn, len); + up_read(&wnd->rw_lock); + if (ok) + continue; + + /* Looks like volume is corrupted. */ + ntfs_set_state(sbi, NTFS_DIRTY_ERROR); + + if (!down_write_trylock(&wnd->rw_lock)) + continue; + + if (zone) { + /* + * Range [lcn, lcn + len) intersects with zone. + * To avoid complex with zone just turn it off. + */ + wnd_zone_set(wnd, 0, 0); + } + + /* Mark all zero bits as used in range [lcn, lcn+len). */ + err = wnd_set_used_safe(wnd, lcn, len, &done); + if (zone) { + /* Restore zone. Lock mft run. */ + struct rw_semaphore *lock = + is_mounted(sbi) ? &sbi->mft.ni->file.run_lock : + NULL; + if (lock) { + if (down_read_trylock(lock)) { + ntfs_refresh_zone(sbi); + up_read(lock); + } + } else { + ntfs_refresh_zone(sbi); + } + } + up_write(&wnd->rw_lock); + if (err) + return err; + } + + return ret; +} +#endif + +/* + * run_get_highest_vcn + * + * Return the highest vcn from a mapping pairs array + * it used while replaying log file. + */ +int run_get_highest_vcn(CLST vcn, const u8 *run_buf, size_t run_buf_size, + u64 *highest_vcn) +{ + const u8 *run_last = run_buf + run_buf_size; + u64 vcn64 = vcn; + u8 size_size; + + while (run_buf < run_last && (size_size = *run_buf & 0xF)) { + u8 offset_size = *run_buf++ >> 4; + u64 len; + + if (size_size > 8 || offset_size > 8) + return -EINVAL; + + if (run_buf + size_size + offset_size > run_last) + return -EINVAL; + + len = run_unpack_s64(run_buf, size_size, 0); + if (!len) + return -EINVAL; + + run_buf += size_size + offset_size; + if (check_add_overflow(vcn64, len, &vcn64)) + return -EINVAL; + +#ifndef CONFIG_NTFS3_64BIT_CLUSTER + if (vcn64 > 0x100000000ull) + return -EINVAL; +#endif + } + + *highest_vcn = vcn64 - 1; + return 0; +} + +/* + * run_clone + * + * Make a copy of run + */ +int run_clone(const struct runs_tree *run, struct runs_tree *new_run) +{ + size_t bytes = run->count * sizeof(struct ntfs_run); + + if (bytes > new_run->allocated) { + struct ntfs_run *new_ptr = kvmalloc(bytes, GFP_KERNEL); + + if (!new_ptr) + return -ENOMEM; + + kvfree(new_run->runs); + new_run->runs = new_ptr; + new_run->allocated = bytes; + } + + memcpy(new_run->runs, run->runs, bytes); + new_run->count = run->count; + return 0; +} + +/* + * run_remove_range + * + */ +bool run_remove_range(struct runs_tree *run, CLST vcn, CLST len, CLST *done) +{ + size_t index, eat; + struct ntfs_run *r, *e, *eat_start, *eat_end; + CLST end, d; + + *done = 0; + + /* Fast check. */ + if (!run->count) + return true; + + if (!run_lookup(run, vcn, &index) && index >= run->count) { + /* No entries in this run. */ + return true; + } + + e = run->runs + run->count; + r = run->runs + index; + end = vcn + len; + + if (vcn > r->vcn) { + CLST r_end = r->vcn + r->len; + d = vcn - r->vcn; + + if (r_end > end) { + /* Remove a middle part, split. */ + CLST tail_lcn = r->lcn == SPARSE_LCN ? + SPARSE_LCN : + (r->lcn + (end - r->vcn)); + + *done += len; + r->len = d; + return run_add_entry(run, end, tail_lcn, r_end - end, + false); + } + /* Remove tail of run .*/ + *done += r->len - d; + r->len = d; + r += 1; + } + + eat_start = r; + eat_end = r; + + for (; r < e; r++) { + if (r->vcn >= end) + continue; + + if (r->vcn + r->len <= end) { + /* Eat this run. */ + *done += r->len; + eat_end = r + 1; + continue; + } + + d = end - r->vcn; + *done += d; + if (r->lcn != SPARSE_LCN) + r->lcn += d; + r->len -= d; + r->vcn = end; + } + + eat = eat_end - eat_start; + memmove(eat_start, eat_end, (e - eat_end) * sizeof(*r)); + run->count -= eat; + + return true; +} + +CLST run_len(const struct runs_tree *run) +{ + const struct ntfs_run *r, *e; + CLST len = 0; + + for (r = run->runs, e = r + run->count; r < e; r++) { + len += r->len; + } + + return len; +} + +CLST run_get_max_vcn(const struct runs_tree *run) +{ + const struct ntfs_run *r; + if (!run->count) + return 0; + + r = run->runs + run->count - 1; + return r->vcn + r->len; +} diff --git a/fs/ntfs3/super.c b/fs/ntfs3/super.c new file mode 100644 index 000000000..f4a42a0c7 --- /dev/null +++ b/fs/ntfs3/super.c @@ -0,0 +1,2025 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + * + * terminology + * + * cluster - allocation unit - 512,1K,2K,4K,...,2M + * vcn - virtual cluster number - Offset inside the file in clusters. + * vbo - virtual byte offset - Offset inside the file in bytes. + * lcn - logical cluster number - 0 based cluster in clusters heap. + * lbo - logical byte offset - Absolute position inside volume. + * run - maps VCN to LCN - Stored in attributes in packed form. + * attr - attribute segment - std/name/data etc records inside MFT. + * mi - MFT inode - One MFT record(usually 1024 bytes or 4K), consists of attributes. + * ni - NTFS inode - Extends linux inode. consists of one or more mft inodes. + * index - unit inside directory - 2K, 4K, <=page size, does not depend on cluster size. + * resident attribute - Attribute with content stored directly in the MFT record + * non-resident attribute - Attribute with content stored in clusters + * data_size - Size of attribute content in bytes. Equal to inode->i_size + * valid_size - Number of bytes written to the non-resident attribute + * allocated_size - Total size of clusters allocated for non-resident content + * total_size - Actual size of allocated clusters for sparse or compressed attributes + * - Constraint: valid_size <= data_size <= allocated_size + * ADS - Alternative data stream: Named data attribute (0x80) + * + * WSL - Windows Subsystem for Linux + * https://docs.microsoft.com/en-us/windows/wsl/file-permissions + * It stores uid/gid/mode/dev in xattr + * + * ntfs allows up to 2^64 clusters per volume. + * It means you should use 64 bits lcn to operate with ntfs. + * Implementation of ntfs.sys uses only 32 bits lcn. + * Default ntfs3 uses 32 bits lcn too. + * ntfs3 built with CONFIG_NTFS3_64BIT_CLUSTER (ntfs3_64) uses 64 bits per lcn. + * + * + * ntfs limits, cluster size is 4K (2^12) + * ----------------------------------------------------------------------------- + * | Volume size | Clusters | ntfs.sys | ntfs3 | ntfs3_64 | mkntfs | chkdsk | + * ----------------------------------------------------------------------------- + * | < 16T, 2^44 | < 2^32 | yes | yes | yes | yes | yes | + * | > 16T, 2^44 | > 2^32 | no | no | yes | yes | yes | + * ----------------------------------------------------------|------------------ + * + * To mount large volumes as ntfs one should use large cluster size (up to 2M) + * The maximum volume size in this case is 2^32 * 2^21 = 2^53 = 8P + * + * ntfs limits, cluster size is 2M (2^21) + * ----------------------------------------------------------------------------- + * | < 8P, 2^53 | < 2^32 | yes | yes | yes | yes | yes | + * | > 8P, 2^53 | > 2^32 | no | no | yes | yes | yes | + * ----------------------------------------------------------|------------------ + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" +#ifdef CONFIG_NTFS3_LZX_XPRESS +#include "lib/lib.h" +#endif + +#ifdef CONFIG_PRINTK +/* + * ntfs_printk - Trace warnings/notices/errors. + * + * Thanks Joe Perches for implementation + */ +void ntfs_printk(const struct super_block *sb, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + int level; + struct ntfs_sb_info *sbi = sb->s_fs_info; + + /* Should we use different ratelimits for warnings/notices/errors? */ + if (!___ratelimit(&sbi->msg_ratelimit, "ntfs3")) + return; + + va_start(args, fmt); + + level = printk_get_level(fmt); + vaf.fmt = printk_skip_level(fmt); + vaf.va = &args; + printk("%c%cntfs3(%s): %pV\n", KERN_SOH_ASCII, level, sb->s_id, &vaf); + + va_end(args); +} + +static char s_name_buf[512]; +static atomic_t s_name_buf_cnt = ATOMIC_INIT(1); // 1 means 'free s_name_buf'. + +/* + * ntfs_inode_printk + * + * Print warnings/notices/errors about inode using name or inode number. + */ +void ntfs_inode_printk(struct inode *inode, const char *fmt, ...) +{ + struct super_block *sb = inode->i_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + char *name; + va_list args; + struct va_format vaf; + int level; + + if (!___ratelimit(&sbi->msg_ratelimit, "ntfs3")) + return; + + /* Use static allocated buffer, if possible. */ + name = atomic_dec_and_test(&s_name_buf_cnt) ? + s_name_buf : + kmalloc(sizeof(s_name_buf), GFP_NOFS); + + if (name) { + struct dentry *de = d_find_alias(inode); + + if (de) { + int len; + spin_lock(&de->d_lock); + len = snprintf(name, sizeof(s_name_buf), " \"%s\"", + de->d_name.name); + spin_unlock(&de->d_lock); + if (len <= 0) + name[0] = 0; + else if (len >= sizeof(s_name_buf)) + name[sizeof(s_name_buf) - 1] = 0; + } else { + name[0] = 0; + } + dput(de); /* Cocci warns if placed in branch "if (de)" */ + } + + va_start(args, fmt); + + level = printk_get_level(fmt); + vaf.fmt = printk_skip_level(fmt); + vaf.va = &args; + + printk("%c%cntfs3(%s): ino=%llx,%s %pV\n", KERN_SOH_ASCII, level, + sb->s_id, inode->i_ino, name ? name : "", &vaf); + + va_end(args); + + atomic_inc(&s_name_buf_cnt); + if (name != s_name_buf) + kfree(name); +} +#endif + +/* + * Shared memory struct. + * + * On-disk ntfs's upcase table is created by ntfs formatter. + * 'upcase' table is 128K bytes of memory. + * We should read it into memory when mounting. + * Several ntfs volumes likely use the same 'upcase' table. + * It is good idea to share in-memory 'upcase' table between different volumes. + * Unfortunately winxp/vista/win7 use different upcase tables. + */ +static DEFINE_SPINLOCK(s_shared_lock); + +static struct { + void *ptr; + u32 len; + int cnt; +} s_shared[8]; + +/* + * ntfs_set_shared + * + * Return: + * * @ptr - If pointer was saved in shared memory. + * * NULL - If pointer was not shared. + */ +void *ntfs_set_shared(void *ptr, u32 bytes) +{ + void *ret = NULL; + int i, j = -1; + + spin_lock(&s_shared_lock); + for (i = 0; i < ARRAY_SIZE(s_shared); i++) { + if (!s_shared[i].cnt) { + j = i; + } else if (bytes == s_shared[i].len && + !memcmp(s_shared[i].ptr, ptr, bytes)) { + s_shared[i].cnt += 1; + ret = s_shared[i].ptr; + break; + } + } + + if (!ret && j != -1) { + s_shared[j].ptr = ptr; + s_shared[j].len = bytes; + s_shared[j].cnt = 1; + ret = ptr; + } + spin_unlock(&s_shared_lock); + + return ret; +} + +/* + * ntfs_put_shared + * + * Return: + * * @ptr - If pointer is not shared anymore. + * * NULL - If pointer is still shared. + */ +void *ntfs_put_shared(void *ptr) +{ + void *ret = ptr; + int i; + + spin_lock(&s_shared_lock); + for (i = 0; i < ARRAY_SIZE(s_shared); i++) { + if (s_shared[i].cnt && s_shared[i].ptr == ptr) { + if (--s_shared[i].cnt) + ret = NULL; + break; + } + } + spin_unlock(&s_shared_lock); + + return ret; +} + +static inline void put_mount_options(struct ntfs_mount_options *options) +{ + kfree(options->nls_name); + unload_nls(options->nls); + kfree(options); +} + +enum Opt { + Opt_uid, + Opt_gid, + Opt_umask, + Opt_dmask, + Opt_fmask, + Opt_immutable, + Opt_discard, + Opt_force, + Opt_sparse, + Opt_nohidden, + Opt_hide_dot_files, + Opt_windows_names, + Opt_showmeta, + Opt_acl, + Opt_acl_bool, + Opt_iocharset, + Opt_prealloc, + Opt_prealloc_bool, + Opt_nocase, + Opt_delalloc, + Opt_delalloc_bool, + Opt_ads, + Opt_ads_bool, + Opt_err, +}; + +// clang-format off +static const struct fs_parameter_spec ntfs_fs_parameters[] = { + fsparam_uid("uid", Opt_uid), + fsparam_gid("gid", Opt_gid), + fsparam_u32oct("umask", Opt_umask), + fsparam_u32oct("dmask", Opt_dmask), + fsparam_u32oct("fmask", Opt_fmask), + fsparam_flag("sys_immutable", Opt_immutable), + fsparam_flag("discard", Opt_discard), + fsparam_flag("force", Opt_force), + fsparam_flag("sparse", Opt_sparse), + fsparam_flag("nohidden", Opt_nohidden), + fsparam_flag("hide_dot_files", Opt_hide_dot_files), + fsparam_flag("windows_names", Opt_windows_names), + fsparam_flag("showmeta", Opt_showmeta), + fsparam_flag("acl", Opt_acl), + fsparam_bool("acl", Opt_acl_bool), + fsparam_string("iocharset", Opt_iocharset), + fsparam_flag("prealloc", Opt_prealloc), + fsparam_bool("prealloc", Opt_prealloc_bool), + fsparam_flag("nocase", Opt_nocase), + fsparam_flag("delalloc", Opt_delalloc), + fsparam_bool("delalloc", Opt_delalloc_bool), + fsparam_flag("ads", Opt_ads), + fsparam_bool("ads", Opt_ads_bool), + {} +}; +// clang-format on + +/* + * Load nls table or if @nls is utf8 then return NULL. + * + */ +static struct nls_table *ntfs_load_nls(const char *nls) +{ + struct nls_table *ret; + + if (!nls) + nls = CONFIG_NLS_DEFAULT; + + if (strcmp(nls, "utf8") == 0) + return NULL; + + if (strcmp(nls, CONFIG_NLS_DEFAULT) == 0) + return load_nls_default(); + + ret = load_nls(nls); + if (ret) + return ret; + + return ERR_PTR(-EINVAL); +} + +static int ntfs_fs_parse_param(struct fs_context *fc, + struct fs_parameter *param) +{ + struct ntfs_mount_options *opts = fc->fs_private; + struct fs_parse_result result; + int opt; + + opt = fs_parse(fc, ntfs_fs_parameters, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_uid: + opts->fs_uid = result.uid; + break; + case Opt_gid: + opts->fs_gid = result.gid; + break; + case Opt_umask: + if (result.uint_32 & ~07777) + return invalf(fc, "ntfs3: Invalid value for umask."); + opts->fs_fmask_inv = ~result.uint_32; + opts->fs_dmask_inv = ~result.uint_32; + opts->fmask = 1; + opts->dmask = 1; + break; + case Opt_dmask: + if (result.uint_32 & ~07777) + return invalf(fc, "ntfs3: Invalid value for dmask."); + opts->fs_dmask_inv = ~result.uint_32; + opts->dmask = 1; + break; + case Opt_fmask: + if (result.uint_32 & ~07777) + return invalf(fc, "ntfs3: Invalid value for fmask."); + opts->fs_fmask_inv = ~result.uint_32; + opts->fmask = 1; + break; + case Opt_immutable: + opts->sys_immutable = 1; + break; + case Opt_discard: + opts->discard = 1; + break; + case Opt_force: + opts->force = 1; + break; + case Opt_sparse: + opts->sparse = 1; + break; + case Opt_nohidden: + opts->nohidden = 1; + break; + case Opt_hide_dot_files: + opts->hide_dot_files = 1; + break; + case Opt_windows_names: + opts->windows_names = 1; + break; + case Opt_showmeta: + opts->showmeta = 1; + break; + case Opt_acl_bool: + if (result.boolean) { + fallthrough; + case Opt_acl: +#ifdef CONFIG_NTFS3_FS_POSIX_ACL + fc->sb_flags |= SB_POSIXACL; +#else + return invalf( + fc, "ntfs3: Support for ACL not compiled in!"); +#endif + } else + fc->sb_flags &= ~SB_POSIXACL; + break; + case Opt_iocharset: + kfree(opts->nls_name); + opts->nls_name = param->string; + param->string = NULL; + break; + case Opt_prealloc: + opts->prealloc = 1; + break; + case Opt_prealloc_bool: + opts->prealloc = result.boolean; + break; + case Opt_nocase: + opts->nocase = 1; + break; + case Opt_delalloc: + opts->delalloc = 1; + break; + case Opt_delalloc_bool: + opts->delalloc = result.boolean; + break; + case Opt_ads: + opts->ads = 1; + break; + case Opt_ads_bool: + opts->ads = result.boolean; + break; + default: + /* Should not be here unless we forget add case. */ + return -EINVAL; + } + return 0; +} + +static int ntfs_fs_reconfigure(struct fs_context *fc) +{ + struct super_block *sb = fc->root->d_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + struct ntfs_mount_options *new_opts = fc->fs_private; + int ro_rw; + + ro_rw = sb_rdonly(sb) && !(fc->sb_flags & SB_RDONLY); + if (ro_rw && (sbi->flags & NTFS_FLAGS_NEED_REPLAY)) { + errorf(fc, + "ntfs3: Couldn't remount rw because journal is not replayed. Please umount/remount instead\n"); + return -EINVAL; + } + + new_opts->nls = ntfs_load_nls(new_opts->nls_name); + if (IS_ERR(new_opts->nls)) { + new_opts->nls = NULL; + errorf(fc, "ntfs3: Cannot load iocharset %s", + new_opts->nls_name); + return -EINVAL; + } + if (new_opts->nls != sbi->options->nls) + return invalf( + fc, + "ntfs3: Cannot use different iocharset when remounting!"); + + if (ro_rw && (sbi->volume.flags & VOLUME_FLAG_DIRTY) && + !new_opts->force) { + errorf(fc, + "ntfs3: Volume is dirty and \"force\" flag is not set!"); + return -EINVAL; + } + + sync_filesystem(sb); + swap(sbi->options, fc->fs_private); + + return 0; +} + +#ifdef CONFIG_PROC_FS +static struct proc_dir_entry *proc_info_root; + +/* + * ntfs3_volinfo: + * + * The content of /proc/fs/ntfs3//volinfo + * + * ntfs3.1 + * cluster size + * number of clusters + * total number of mft records + * number of used mft records ~= number of files + folders + * real state of ntfs "dirty"/"clean" + * current state of ntfs "dirty"/"clean" +*/ +static int ntfs3_volinfo(struct seq_file *m, void *o) +{ + struct super_block *sb = m->private; + struct ntfs_sb_info *sbi = sb->s_fs_info; + + seq_printf(m, "ntfs%d.%d\n%u\n%zu\n%zu\n%zu\n%s\n%s\n", + sbi->volume.major_ver, sbi->volume.minor_ver, + sbi->cluster_size, sbi->used.bitmap.nbits, + sbi->mft.bitmap.nbits, + sbi->mft.bitmap.nbits - wnd_zeroes(&sbi->mft.bitmap), + sbi->volume.real_dirty ? "dirty" : "clean", + (sbi->volume.flags & VOLUME_FLAG_DIRTY) ? "dirty" : "clean"); + + return 0; +} + +static int ntfs3_volinfo_open(struct inode *inode, struct file *file) +{ + return single_open(file, ntfs3_volinfo, pde_data(inode)); +} + +/* read /proc/fs/ntfs3//label */ +static int ntfs3_label_show(struct seq_file *m, void *o) +{ + struct super_block *sb = m->private; + struct ntfs_sb_info *sbi = sb->s_fs_info; + + seq_printf(m, "%s\n", sbi->volume.label); + + return 0; +} + +/* write /proc/fs/ntfs3//label */ +static ssize_t ntfs3_label_write(struct file *file, const char __user *buffer, + size_t count, loff_t *ppos) +{ + int err; + struct super_block *sb = pde_data(file_inode(file)); + ssize_t ret = count; + u8 *label; + + if (sb_rdonly(sb)) + return -EROFS; + + label = kmalloc(count, GFP_NOFS); + + if (!label) + return -ENOMEM; + + if (copy_from_user(label, buffer, ret)) { + ret = -EFAULT; + goto out; + } + while (ret > 0 && label[ret - 1] == '\n') + ret -= 1; + + err = ntfs_set_label(sb->s_fs_info, label, ret); + + if (err < 0) { + ntfs_err(sb, "failed (%d) to write label", err); + ret = err; + goto out; + } + + *ppos += count; + ret = count; +out: + kfree(label); + return ret; +} + +static int ntfs3_label_open(struct inode *inode, struct file *file) +{ + return single_open(file, ntfs3_label_show, pde_data(inode)); +} + +static const struct proc_ops ntfs3_volinfo_fops = { + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_open = ntfs3_volinfo_open, +}; + +static const struct proc_ops ntfs3_label_fops = { + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_open = ntfs3_label_open, + .proc_write = ntfs3_label_write, +}; + +static void ntfs_create_procdir(struct super_block *sb) +{ + struct proc_dir_entry *e; + + if (!proc_info_root) + return; + + e = proc_mkdir(sb->s_id, proc_info_root); + if (e) { + struct ntfs_sb_info *sbi = sb->s_fs_info; + + proc_create_data("volinfo", 0444, e, &ntfs3_volinfo_fops, sb); + proc_create_data("label", 0644, e, &ntfs3_label_fops, sb); + sbi->procdir = e; + } +} + +static void ntfs_remove_procdir(struct super_block *sb) +{ + struct ntfs_sb_info *sbi = sb->s_fs_info; + + if (!sbi->procdir) + return; + + remove_proc_entry("label", sbi->procdir); + remove_proc_entry("volinfo", sbi->procdir); + remove_proc_entry(sb->s_id, proc_info_root); + sbi->procdir = NULL; +} + +static void ntfs_create_proc_root(void) +{ + proc_info_root = proc_mkdir("fs/ntfs3", NULL); +} + +static void ntfs_remove_proc_root(void) +{ + if (proc_info_root) { + remove_proc_entry("fs/ntfs3", NULL); + proc_info_root = NULL; + } +} +#else +// clang-format off +static void ntfs_create_procdir(struct super_block *sb){} +static void ntfs_remove_procdir(struct super_block *sb){} +static void ntfs_create_proc_root(void){} +static void ntfs_remove_proc_root(void){} +// clang-format on +#endif + +static struct kmem_cache *ntfs_inode_cachep; + +static struct inode *ntfs_alloc_inode(struct super_block *sb) +{ + struct ntfs_inode *ni = alloc_inode_sb(sb, ntfs_inode_cachep, GFP_NOFS); + + if (!ni) + return NULL; + + memset(ni, 0, offsetof(struct ntfs_inode, vfs_inode)); + mutex_init(&ni->ni_lock); + return &ni->vfs_inode; +} + +static void ntfs_free_inode(struct inode *inode) +{ + struct ntfs_inode *ni = ntfs_i(inode); + + mutex_destroy(&ni->ni_lock); + kmem_cache_free(ntfs_inode_cachep, ni); +} + +static void init_once(void *foo) +{ + struct ntfs_inode *ni = foo; + + inode_init_once(&ni->vfs_inode); +} + +/* + * Noinline to reduce binary size. + */ +static noinline void ntfs3_put_sbi(struct ntfs_sb_info *sbi) +{ + wnd_close(&sbi->mft.bitmap); + wnd_close(&sbi->used.bitmap); + + if (sbi->mft.ni) { + iput(&sbi->mft.ni->vfs_inode); + sbi->mft.ni = NULL; + } + + if (sbi->security.ni) { + iput(&sbi->security.ni->vfs_inode); + sbi->security.ni = NULL; + } + + if (sbi->reparse.ni) { + iput(&sbi->reparse.ni->vfs_inode); + sbi->reparse.ni = NULL; + } + + if (sbi->objid.ni) { + iput(&sbi->objid.ni->vfs_inode); + sbi->objid.ni = NULL; + } + + if (sbi->volume.ni) { + iput(&sbi->volume.ni->vfs_inode); + sbi->volume.ni = NULL; + } + + ntfs_update_mftmirr(sbi); + + indx_clear(&sbi->security.index_sii); + indx_clear(&sbi->security.index_sdh); + indx_clear(&sbi->reparse.index_r); + indx_clear(&sbi->objid.index_o); +} + +static void ntfs3_free_sbi(struct ntfs_sb_info *sbi) +{ + kfree(sbi->new_rec); + kvfree(ntfs_put_shared(sbi->upcase)); + kvfree(sbi->def_table); + kfree(sbi->compress.lznt); +#ifdef CONFIG_NTFS3_LZX_XPRESS + xpress_free_decompressor(sbi->compress.xpress); + lzx_free_decompressor(sbi->compress.lzx); +#endif + kfree(sbi); +} + +static void ntfs_put_super(struct super_block *sb) +{ + struct ntfs_sb_info *sbi = sb->s_fs_info; + + ntfs_remove_procdir(sb); + + /* Mark rw ntfs as clear, if possible. */ + ntfs_set_state(sbi, NTFS_DIRTY_CLEAR); + + if (sbi->options) { + put_mount_options(sbi->options); + sbi->options = NULL; + } + + ntfs3_put_sbi(sbi); +} + +static int ntfs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + struct wnd_bitmap *wnd = &sbi->used.bitmap; + CLST da_clusters = ntfs_get_da(sbi); + + buf->f_type = sb->s_magic; + buf->f_bsize = buf->f_frsize = sbi->cluster_size; + buf->f_blocks = wnd->nbits; + + buf->f_bfree = wnd_zeroes(wnd); + if (buf->f_bfree > da_clusters) { + buf->f_bfree -= da_clusters; + } else { + buf->f_bfree = 0; + } + buf->f_bavail = buf->f_bfree; + + buf->f_fsid.val[0] = sbi->volume.ser_num; + buf->f_fsid.val[1] = sbi->volume.ser_num >> 32; + buf->f_namelen = NTFS_NAME_LEN; + + return 0; +} + +static int ntfs_show_options(struct seq_file *m, struct dentry *root) +{ + struct super_block *sb = root->d_sb; + struct ntfs_sb_info *sbi = sb->s_fs_info; + struct ntfs_mount_options *opts = sbi->options; + struct user_namespace *user_ns = seq_user_ns(m); + + seq_printf(m, ",uid=%u", from_kuid_munged(user_ns, opts->fs_uid)); + seq_printf(m, ",gid=%u", from_kgid_munged(user_ns, opts->fs_gid)); + if (opts->dmask) + seq_printf(m, ",dmask=%04o", opts->fs_dmask_inv ^ 0xffff); + if (opts->fmask) + seq_printf(m, ",fmask=%04o", opts->fs_fmask_inv ^ 0xffff); + if (opts->sys_immutable) + seq_puts(m, ",sys_immutable"); + if (opts->discard) + seq_puts(m, ",discard"); + if (opts->force) + seq_puts(m, ",force"); + if (opts->sparse) + seq_puts(m, ",sparse"); + if (opts->nohidden) + seq_puts(m, ",nohidden"); + if (opts->hide_dot_files) + seq_puts(m, ",hide_dot_files"); + if (opts->windows_names) + seq_puts(m, ",windows_names"); + if (opts->showmeta) + seq_puts(m, ",showmeta"); + if (sb->s_flags & SB_POSIXACL) + seq_puts(m, ",acl"); + if (opts->nls) + seq_printf(m, ",iocharset=%s", opts->nls->charset); + else + seq_puts(m, ",iocharset=utf8"); + if (opts->prealloc) + seq_puts(m, ",prealloc"); + if (opts->nocase) + seq_puts(m, ",nocase"); + if (opts->delalloc) + seq_puts(m, ",delalloc"); + if (opts->ads) + seq_puts(m, ",ads"); + + return 0; +} + +/* + * ntfs_shutdown - super_operations::shutdown + */ +static void ntfs_shutdown(struct super_block *sb) +{ + set_bit(NTFS_FLAGS_SHUTDOWN_BIT, &ntfs_sb(sb)->flags); +} + +/* + * ntfs_sync_fs - super_operations::sync_fs + */ +static int ntfs_sync_fs(struct super_block *sb, int wait) +{ + int err = 0, err2; + struct ntfs_sb_info *sbi = sb->s_fs_info; + struct ntfs_inode *ni; + struct inode *inode; + + if (unlikely(ntfs3_forced_shutdown(sb))) + return -EIO; + + ni = sbi->security.ni; + if (ni) { + inode = &ni->vfs_inode; + err2 = _ni_write_inode(inode, wait); + if (err2 && !err) + err = err2; + } + + ni = sbi->objid.ni; + if (ni) { + inode = &ni->vfs_inode; + err2 = _ni_write_inode(inode, wait); + if (err2 && !err) + err = err2; + } + + ni = sbi->reparse.ni; + if (ni) { + inode = &ni->vfs_inode; + err2 = _ni_write_inode(inode, wait); + if (err2 && !err) + err = err2; + } + + if (!err) + ntfs_set_state(sbi, NTFS_DIRTY_CLEAR); + + ntfs_update_mftmirr(sbi); + + if (wait) { + sync_blockdev(sb->s_bdev); + blkdev_issue_flush(sb->s_bdev); + } + + return err; +} + +static const struct super_operations ntfs_sops = { + .alloc_inode = ntfs_alloc_inode, + .free_inode = ntfs_free_inode, + .evict_inode = ntfs_evict_inode, + .put_super = ntfs_put_super, + .statfs = ntfs_statfs, + .show_options = ntfs_show_options, + .shutdown = ntfs_shutdown, + .sync_fs = ntfs_sync_fs, + .write_inode = ntfs3_write_inode, +}; + +static struct inode *ntfs_export_get_inode(struct super_block *sb, u64 ino, + u32 generation) +{ + struct MFT_REF ref; + struct inode *inode; + + ref.low = cpu_to_le32(ino); +#ifdef CONFIG_NTFS3_64BIT_CLUSTER + ref.high = cpu_to_le16(ino >> 32); +#else + ref.high = 0; +#endif + ref.seq = cpu_to_le16(generation); + + inode = ntfs_iget5(sb, &ref, NULL); + if (!IS_ERR(inode) && is_bad_inode(inode)) { + iput(inode); + inode = ERR_PTR(-ESTALE); + } + + return inode; +} + +static struct dentry *ntfs_fh_to_dentry(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + return generic_fh_to_dentry(sb, fid, fh_len, fh_type, + ntfs_export_get_inode); +} + +static struct dentry *ntfs_fh_to_parent(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + return generic_fh_to_parent(sb, fid, fh_len, fh_type, + ntfs_export_get_inode); +} + +/* TODO: == ntfs_sync_inode */ +static int ntfs_nfs_commit_metadata(struct inode *inode) +{ + return _ni_write_inode(inode, 1); +} + +static const struct export_operations ntfs_export_ops = { + .encode_fh = generic_encode_ino32_fh, + .fh_to_dentry = ntfs_fh_to_dentry, + .fh_to_parent = ntfs_fh_to_parent, + .get_parent = ntfs3_get_parent, + .commit_metadata = ntfs_nfs_commit_metadata, +}; + +/* + * format_size_gb - Return Gb,Mb to print with "%u.%02u Gb". + */ +static u32 format_size_gb(const u64 bytes, u32 *mb) +{ + /* Do simple right 30 bit shift of 64 bit value. */ + u64 kbytes = bytes >> 10; + u32 kbytes32 = kbytes; + + *mb = (100 * (kbytes32 & 0xfffff) + 0x7ffff) >> 20; + if (*mb >= 100) + *mb = 99; + + return (kbytes32 >> 20) | (((u32)(kbytes >> 32)) << 12); +} + +static u32 true_sectors_per_clst(const struct NTFS_BOOT *boot) +{ + if (boot->sectors_per_clusters <= 0x80) + return boot->sectors_per_clusters; + if (boot->sectors_per_clusters >= 0xf4) /* limit shift to 2MB max */ + return 1U << (-(s8)boot->sectors_per_clusters); + return -EINVAL; +} + +/* + * ntfs_init_from_boot - Init internal info from on-disk boot sector. + * + * NTFS mount begins from boot - special formatted 512 bytes. + * There are two boots: the first and the last 512 bytes of volume. + * The content of boot is not changed during ntfs life. + * + * NOTE: ntfs.sys checks only first (primary) boot. + * chkdsk checks both boots. + */ +static int ntfs_init_from_boot(struct super_block *sb, u32 sector_size, + u64 dev_size, struct NTFS_BOOT **boot2) +{ + struct ntfs_sb_info *sbi = sb->s_fs_info; + int err; + u32 mb, gb, boot_sector_size, sct_per_clst, record_size; + u64 sectors, clusters, mlcn, mlcn2, mft_pos, mft2_pos, dev_size0; + struct NTFS_BOOT *boot; + struct buffer_head *bh; + struct MFT_REC *rec; + u16 fn, ao; + u8 cluster_bits; + u32 boot_off = 0; + sector_t boot_block = 0; + const char *hint = "Primary boot"; + + /* Save original dev_size. Used with alternative boot. */ + dev_size0 = dev_size; + + sbi->volume.blocks = dev_size >> PAGE_SHIFT; + + /* Set dummy blocksize to read boot_block. */ + if (!sb_min_blocksize(sb, PAGE_SIZE)) { + return -EINVAL; + } + +read_boot: + bh = ntfs_bread(sb, boot_block); + if (!bh) + return boot_block ? -EINVAL : -EIO; + + err = -EINVAL; + + /* Corrupted image; do not read OOB */ + if (bh->b_size - sizeof(*boot) < boot_off) + goto out; + + boot = (struct NTFS_BOOT *)Add2Ptr(bh->b_data, boot_off); + + if (memcmp(boot->system_id, "NTFS ", sizeof("NTFS ") - 1)) { + ntfs_err(sb, "%s signature is not NTFS.", hint); + goto out; + } + + /* 0x55AA is not mandaroty. Thanks Maxim Suhanov*/ + /*if (0x55 != boot->boot_magic[0] || 0xAA != boot->boot_magic[1]) + * goto out; + */ + + boot_sector_size = ((u32)boot->bytes_per_sector[1] << 8) | + boot->bytes_per_sector[0]; + if (boot_sector_size < SECTOR_SIZE || + !is_power_of_2(boot_sector_size)) { + ntfs_err(sb, "%s: invalid bytes per sector %u.", hint, + boot_sector_size); + goto out; + } + + /* cluster size: 512, 1K, 2K, 4K, ... 2M */ + sct_per_clst = true_sectors_per_clst(boot); + if ((int)sct_per_clst < 0 || !is_power_of_2(sct_per_clst)) { + ntfs_err(sb, "%s: invalid sectors per cluster %u.", hint, + sct_per_clst); + goto out; + } + + sbi->cluster_size = boot_sector_size * sct_per_clst; + sbi->cluster_bits = cluster_bits = blksize_bits(sbi->cluster_size); + sbi->cluster_mask = sbi->cluster_size - 1; + sbi->cluster_mask_inv = ~(u64)sbi->cluster_mask; + + mlcn = le64_to_cpu(boot->mft_clst); + mlcn2 = le64_to_cpu(boot->mft2_clst); + sectors = le64_to_cpu(boot->sectors_per_volume); + + /* + * Convert mlcn/mlcn2 to sector positions before comparing with + * 'sectors'. All three are u64 values that come from the boot + * sector, so use check_mul_overflow() to keep a wraparound from + * silently bypassing the comparison. + */ + if (check_mul_overflow(mlcn, (u64)sct_per_clst, &mft_pos) || + check_mul_overflow(mlcn2, (u64)sct_per_clst, &mft2_pos) || + mft_pos >= sectors || mft2_pos >= sectors) { + ntfs_err( + sb, + "%s: start of MFT 0x%llx (0x%llx) is out of volume 0x%llx.", + hint, mlcn, mlcn2, sectors); + goto out; + } + + if (boot->record_size >= 0) { + record_size = (u32)boot->record_size << cluster_bits; + } else if (-boot->record_size <= MAXIMUM_SHIFT_BYTES_PER_MFT) { + record_size = 1u << (-boot->record_size); + } else { + ntfs_err(sb, "%s: invalid record size %d.", hint, + boot->record_size); + goto out; + } + + sbi->record_size = record_size; + sbi->record_bits = blksize_bits(record_size); + sbi->attr_size_tr = (5 * record_size >> 4); // ~320 bytes + + /* Check MFT record size. */ + if (record_size < SECTOR_SIZE || !is_power_of_2(record_size)) { + ntfs_err(sb, "%s: invalid bytes per MFT record %u (%d).", hint, + record_size, boot->record_size); + goto out; + } + + if (record_size > MAXIMUM_BYTES_PER_MFT) { + ntfs_err(sb, "Unsupported bytes per MFT record %u.", + record_size); + goto out; + } + + if (boot->index_size >= 0) { + sbi->index_size = (u32)boot->index_size << cluster_bits; + } else if (-boot->index_size <= MAXIMUM_SHIFT_BYTES_PER_INDEX) { + sbi->index_size = 1u << (-boot->index_size); + } else { + ntfs_err(sb, "%s: invalid index size %d.", hint, + boot->index_size); + goto out; + } + + /* Check index record size. */ + if (sbi->index_size < SECTOR_SIZE || !is_power_of_2(sbi->index_size)) { + ntfs_err(sb, "%s: invalid bytes per index %u(%d).", hint, + sbi->index_size, boot->index_size); + goto out; + } + + if (sbi->index_size > MAXIMUM_BYTES_PER_INDEX) { + ntfs_err(sb, "%s: unsupported bytes per index %u.", hint, + sbi->index_size); + goto out; + } + + sbi->volume.size = sectors * boot_sector_size; + + gb = format_size_gb(sbi->volume.size + boot_sector_size, &mb); + + /* + * - Volume formatted and mounted with the same sector size. + * - Volume formatted 4K and mounted as 512. + * - Volume formatted 512 and mounted as 4K. + */ + if (boot_sector_size != sector_size) { + ntfs_warn( + sb, + "Different NTFS sector size (%u) and media sector size (%u).", + boot_sector_size, sector_size); + dev_size += sector_size - 1; + } + + sbi->bdev_blocksize = max(boot_sector_size, sector_size); + sbi->mft.lbo = mlcn << cluster_bits; + sbi->mft.lbo2 = mlcn2 << cluster_bits; + + /* Compare boot's cluster and sector. */ + if (sbi->cluster_size < boot_sector_size) { + ntfs_err(sb, "%s: invalid bytes per cluster (%u).", hint, + sbi->cluster_size); + goto out; + } + + /* Compare boot's cluster and media sector. */ + if (sbi->cluster_size < sector_size) { + /* No way to use ntfs_get_block in this case. */ + ntfs_err( + sb, + "Failed to mount 'cause NTFS's cluster size (%u) is less than media sector size (%u).", + sbi->cluster_size, sector_size); + goto out; + } + + sbi->max_bytes_per_attr = + record_size - ALIGN(MFTRECORD_FIXUP_OFFSET, 8) - + ALIGN(((record_size >> SECTOR_SHIFT) * sizeof(short)), 8) - + ALIGN(sizeof(enum ATTR_TYPE), 8); + + sbi->volume.ser_num = le64_to_cpu(boot->serial_num); + + /* Warning if RAW volume. */ + if (dev_size < sbi->volume.size + boot_sector_size) { + u32 mb0, gb0; + + gb0 = format_size_gb(dev_size, &mb0); + ntfs_warn( + sb, + "RAW NTFS volume: Filesystem size %u.%02u Gb > volume size %u.%02u Gb. Mount in read-only.", + gb, mb, gb0, mb0); + sb->s_flags |= SB_RDONLY; + } + + clusters = sbi->volume.size >> cluster_bits; +#ifndef CONFIG_NTFS3_64BIT_CLUSTER + /* 32 bits per cluster. */ + if (clusters >> 32) { + ntfs_notice( + sb, + "NTFS %u.%02u Gb is too big to use 32 bits per cluster.", + gb, mb); + goto out; + } +#elif BITS_PER_LONG < 64 +#error "CONFIG_NTFS3_64BIT_CLUSTER incompatible in 32 bit OS" +#endif + + sbi->used.bitmap.nbits = clusters; + + rec = kzalloc(record_size, GFP_NOFS); + if (!rec) { + err = -ENOMEM; + goto out; + } + + sbi->new_rec = rec; + rec->rhdr.sign = NTFS_FILE_SIGNATURE; + rec->rhdr.fix_off = cpu_to_le16(MFTRECORD_FIXUP_OFFSET); + fn = (sbi->record_size >> SECTOR_SHIFT) + 1; + rec->rhdr.fix_num = cpu_to_le16(fn); + ao = ALIGN(MFTRECORD_FIXUP_OFFSET + sizeof(short) * fn, 8); + rec->attr_off = cpu_to_le16(ao); + rec->used = cpu_to_le32(ao + ALIGN(sizeof(enum ATTR_TYPE), 8)); + rec->total = cpu_to_le32(sbi->record_size); + ((struct ATTRIB *)Add2Ptr(rec, ao))->type = ATTR_END; + + if (!sb_set_blocksize(sb, min_t(u32, sbi->cluster_size, PAGE_SIZE))) { + err = -EINVAL; + goto out; + } + + sbi->block_mask = sb->s_blocksize - 1; + sbi->blocks_per_cluster = sbi->cluster_size >> sb->s_blocksize_bits; + sbi->volume.blocks = sbi->volume.size >> sb->s_blocksize_bits; + + /* Maximum size for normal files. */ + sbi->maxbytes = (clusters << cluster_bits) - 1; + +#ifdef CONFIG_NTFS3_64BIT_CLUSTER + if (clusters >= (1ull << (64 - cluster_bits))) + sbi->maxbytes = -1; + sbi->maxbytes_sparse = MAX_LFS_FILESIZE; + sb->s_maxbytes = MAX_LFS_FILESIZE; +#else + /* Maximum size for sparse file. */ + sbi->maxbytes_sparse = (1ull << (cluster_bits + 32)) - 1; + sb->s_maxbytes = 0xFFFFFFFFull << cluster_bits; +#endif + + /* + * Compute the MFT zone at two steps. + * It would be nice if we are able to allocate 1/8 of + * total clusters for MFT but not more then 512 MB. + */ + sbi->zone_max = min_t(CLST, 0x20000000 >> cluster_bits, clusters >> 3); + + err = 0; + + if (bh->b_blocknr && !sb_rdonly(sb)) { + /* + * Alternative boot is ok but primary is not ok. + * Do not update primary boot here 'cause it may be faked boot. + * Let ntfs to be mounted and update boot later. + */ + *boot2 = kmemdup(boot, sizeof(*boot), GFP_NOFS | __GFP_NOWARN); + } + +out: + brelse(bh); + + if (err == -EINVAL && !boot_block && dev_size0 > PAGE_SHIFT) { + u32 block_size = min_t(u32, sector_size, PAGE_SIZE); + u64 lbo = dev_size0 - sizeof(*boot); + + boot_block = lbo >> blksize_bits(block_size); + boot_off = lbo & (block_size - 1); + if (boot_block && block_size >= boot_off + sizeof(*boot)) { + /* + * Try alternative boot (last sector) + */ + if (!sb_set_blocksize(sb, block_size)) + return -EINVAL; + hint = "Alternative boot"; + dev_size = dev_size0; /* restore original size. */ + goto read_boot; + } + } + + return err; +} + +/* + * ntfs_fill_super - Try to mount. + */ +static int ntfs_fill_super(struct super_block *sb, struct fs_context *fc) +{ + int err; + struct ntfs_sb_info *sbi = sb->s_fs_info; + struct block_device *bdev = sb->s_bdev; + struct ntfs_mount_options *fc_opts; + struct ntfs_mount_options *options = NULL; + struct inode *inode; + struct ntfs_inode *ni; + size_t i, tt, bad_len, bad_frags; + CLST vcn, lcn, len; + struct ATTRIB *attr; + const struct VOLUME_INFO *info; + u32 done, bytes; + struct ATTR_DEF_ENTRY *t; + u16 *shared; + struct MFT_REF ref; + bool ro = sb_rdonly(sb); + struct NTFS_BOOT *boot2 = NULL; + + ref.high = 0; + + sbi->sb = sb; + fc_opts = fc->fs_private; + if (!fc_opts) { + errorf(fc, "missing mount options"); + return -EINVAL; + } + options = kmemdup(fc_opts, sizeof(*fc_opts), GFP_KERNEL); + if (!options) + return -ENOMEM; + + if (fc_opts->nls_name) { + options->nls_name = kstrdup(fc_opts->nls_name, GFP_KERNEL); + if (!options->nls_name) { + kfree(options); + return -ENOMEM; + } + } + sbi->options = options; + sb->s_flags |= SB_NODIRATIME; + sb->s_magic = 0x7366746e; // "ntfs" + sb->s_op = &ntfs_sops; + sb->s_export_op = &ntfs_export_ops; + sb->s_time_gran = NTFS_TIME_GRAN; // 100 nsec + sb->s_xattr = ntfs_xattr_handlers; + set_default_d_op(sb, options->nocase ? &ntfs_dentry_ops : NULL); + + options->nls = ntfs_load_nls(options->nls_name); + if (IS_ERR(options->nls)) { + options->nls = NULL; + errorf(fc, "Cannot load nls %s", options->nls_name); + err = -EINVAL; + goto out; + } + + if (bdev_max_discard_sectors(bdev) && bdev_discard_granularity(bdev)) { + sbi->discard_granularity = bdev_discard_granularity(bdev); + sbi->discard_granularity_mask_inv = + ~(u64)(sbi->discard_granularity - 1); + } + + /* Parse boot. */ + err = ntfs_init_from_boot(sb, bdev_logical_block_size(bdev), + bdev_nr_bytes(bdev), &boot2); + if (err) + goto out; + + /* + * Load $Volume. This should be done before $LogFile + * 'cause 'sbi->volume.ni' is used in 'ntfs_set_state'. + */ + ref.low = cpu_to_le32(MFT_REC_VOL); + ref.seq = cpu_to_le16(MFT_REC_VOL); + inode = ntfs_iget5(sb, &ref, &NAME_VOLUME); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + ntfs_err(sb, "Failed to load $Volume (%d).", err); + goto out; + } + + ni = ntfs_i(inode); + + /* Load and save label (not necessary). */ + attr = ni_find_attr(ni, NULL, NULL, ATTR_LABEL, NULL, 0, NULL, NULL); + + if (!attr) { + /* It is ok if no ATTR_LABEL */ + } else if (!attr->non_res && !is_attr_ext(attr)) { + /* $AttrDef allows labels to be up to 128 symbols. */ + err = utf16s_to_utf8s(resident_data(attr), + le32_to_cpu(attr->res.data_size) >> 1, + UTF16_LITTLE_ENDIAN, sbi->volume.label, + sizeof(sbi->volume.label)); + if (err < 0) { + sbi->volume.label[0] = 0; + } else if (err >= sizeof(sbi->volume.label)) { + sbi->volume.label[sizeof(sbi->volume.label) - 1] = 0; + } else { + sbi->volume.label[err] = 0; + } + } else { + /* Should we break mounting here? */ + //err = -EINVAL; + //goto put_inode_out; + } + + attr = ni_find_attr(ni, attr, NULL, ATTR_VOL_INFO, NULL, 0, NULL, NULL); + if (!attr || is_attr_ext(attr) || + !(info = resident_data_ex(attr, SIZEOF_ATTRIBUTE_VOLUME_INFO))) { + ntfs_err(sb, "$Volume is corrupted."); + err = -EINVAL; + goto put_inode_out; + } + + sbi->volume.major_ver = info->major_ver; + sbi->volume.minor_ver = info->minor_ver; + sbi->volume.flags = info->flags; + sbi->volume.ni = ni; + if (info->flags & VOLUME_FLAG_DIRTY) { + sbi->volume.real_dirty = true; + ntfs_info(sb, "It is recommended to use chkdsk."); + } + + /* Load $MFTMirr to estimate recs_mirr. */ + ref.low = cpu_to_le32(MFT_REC_MIRR); + ref.seq = cpu_to_le16(MFT_REC_MIRR); + inode = ntfs_iget5(sb, &ref, &NAME_MIRROR); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + ntfs_err(sb, "Failed to load $MFTMirr (%d).", err); + goto out; + } + + sbi->mft.recs_mirr = ntfs_up_cluster(sbi, inode->i_size) >> + sbi->record_bits; + + iput(inode); + + /* Load LogFile to replay. */ + ref.low = cpu_to_le32(MFT_REC_LOG); + ref.seq = cpu_to_le16(MFT_REC_LOG); + inode = ntfs_iget5(sb, &ref, &NAME_LOGFILE); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + ntfs_err(sb, "Failed to load \x24LogFile (%d).", err); + goto out; + } + + ni = ntfs_i(inode); + + err = ntfs_loadlog_and_replay(ni, sbi); + if (err) + goto put_inode_out; + + iput(inode); + + if ((sbi->flags & NTFS_FLAGS_NEED_REPLAY) && !ro) { + ntfs_warn(sb, "failed to replay log file. Can't mount rw!"); + err = -EINVAL; + goto out; + } + + if ((sbi->volume.flags & VOLUME_FLAG_DIRTY) && !ro && !options->force) { + ntfs_warn(sb, "volume is dirty and \"force\" flag is not set!"); + err = -EINVAL; + goto out; + } + + /* Load $MFT. */ + ref.low = cpu_to_le32(MFT_REC_MFT); + ref.seq = cpu_to_le16(1); + + inode = ntfs_iget5(sb, &ref, &NAME_MFT); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + ntfs_err(sb, "Failed to load $MFT (%d).", err); + goto out; + } + + ni = ntfs_i(inode); + + sbi->mft.used = ni->i_valid >> sbi->record_bits; + tt = inode->i_size >> sbi->record_bits; + sbi->mft.next_free = MFT_REC_USER; + + err = ni_load_all_mi(ni); + if (err) { + ntfs_err(sb, "Failed to load $MFT's subrecords (%d).", err); + goto put_inode_out; + } + + /* Merge MFT bitmap runs from extent records loaded by ni_load_all_mi. */ + { + struct ATTRIB *a = NULL; + struct ATTR_LIST_ENTRY *le = NULL; + + while ((a = ni_enum_attr_ex(ni, a, &le, NULL))) { + CLST svcn, evcn; + u16 roff; + + if (a->type != ATTR_BITMAP || !a->non_res) + continue; + + svcn = le64_to_cpu(a->nres.svcn); + if (!svcn) + continue; /* Base record runs already loaded. */ + + evcn = le64_to_cpu(a->nres.evcn); + roff = le16_to_cpu(a->nres.run_off); + + err = run_unpack_ex(&sbi->mft.bitmap.run, sbi, + MFT_REC_MFT, svcn, evcn, svcn, + Add2Ptr(a, roff), + le32_to_cpu(a->size) - roff); + if (err < 0) { + ntfs_err( + sb, + "Failed to unpack $MFT bitmap extent (%d).", + err); + goto put_inode_out; + } + err = 0; + } + } + + err = wnd_init(&sbi->mft.bitmap, sb, tt); + if (err) + goto put_inode_out; + + sbi->mft.ni = ni; + + /* Load $Bitmap. */ + ref.low = cpu_to_le32(MFT_REC_BITMAP); + ref.seq = cpu_to_le16(MFT_REC_BITMAP); + inode = ntfs_iget5(sb, &ref, &NAME_BITMAP); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + ntfs_err(sb, "Failed to load $Bitmap (%d).", err); + goto out; + } + +#ifndef CONFIG_NTFS3_64BIT_CLUSTER + if (inode->i_size >> 32) { + err = -EINVAL; + goto put_inode_out; + } +#endif + + /* Check bitmap boundary. */ + tt = sbi->used.bitmap.nbits; + if (inode->i_size < ntfs3_bitmap_size(tt)) { + ntfs_err(sb, "$Bitmap is corrupted."); + err = -EINVAL; + goto put_inode_out; + } + + err = wnd_init(&sbi->used.bitmap, sb, tt); + if (err) { + ntfs_err(sb, "Failed to initialize $Bitmap (%d).", err); + goto put_inode_out; + } + + iput(inode); + + /* Compute the MFT zone. */ + err = ntfs_refresh_zone(sbi); + if (err) { + ntfs_err(sb, "Failed to initialize MFT zone (%d).", err); + goto out; + } + + /* Load $BadClus. */ + ref.low = cpu_to_le32(MFT_REC_BADCLUST); + ref.seq = cpu_to_le16(MFT_REC_BADCLUST); + inode = ntfs_iget5(sb, &ref, &NAME_BADCLUS); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + ntfs_err(sb, "Failed to load $BadClus (%d).", err); + goto out; + } + + ni = ntfs_i(inode); + bad_len = bad_frags = 0; + for (i = 0; run_get_entry(&ni->file.run, i, &vcn, &lcn, &len); i++) { + if (lcn == SPARSE_LCN) + continue; + + bad_len += len; + bad_frags += 1; + if (ro) + continue; + + if (wnd_set_used_safe(&sbi->used.bitmap, lcn, len, &tt) || tt) { + /* Bad blocks marked as free in bitmap. */ + ntfs_set_state(sbi, NTFS_DIRTY_ERROR); + } + } + if (bad_len) { + /* + * Notice about bad blocks. + * In normal cases these blocks are marked as used in bitmap. + * And we never allocate space in it. + */ + ntfs_notice(sb, + "Volume contains %zu bad blocks in %zu fragments.", + bad_len, bad_frags); + } + iput(inode); + + /* Load $AttrDef. */ + ref.low = cpu_to_le32(MFT_REC_ATTR); + ref.seq = cpu_to_le16(MFT_REC_ATTR); + inode = ntfs_iget5(sb, &ref, &NAME_ATTRDEF); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + ntfs_err(sb, "Failed to load $AttrDef (%d)", err); + goto out; + } + + /* + * Typical $AttrDef contains up to 20 entries. + * Check for extremely large/small size. + */ + if (inode->i_size < sizeof(struct ATTR_DEF_ENTRY) || + inode->i_size > 100 * sizeof(struct ATTR_DEF_ENTRY)) { + ntfs_err(sb, "Looks like $AttrDef is corrupted (size=%llu).", + inode->i_size); + err = -EINVAL; + goto put_inode_out; + } + + bytes = inode->i_size; + sbi->def_table = t = kvmalloc(bytes, GFP_KERNEL); + if (!t) { + err = -ENOMEM; + goto put_inode_out; + } + + /* Read the entire file. */ + err = inode_read_data(inode, sbi->def_table, bytes); + if (err) { + ntfs_err(sb, "Failed to read $AttrDef (%d).", err); + goto put_inode_out; + } + + if (ATTR_STD != t->type) { + ntfs_err(sb, "$AttrDef is corrupted."); + err = -EINVAL; + goto put_inode_out; + } + + t += 1; + sbi->def_entries = 1; + done = sizeof(struct ATTR_DEF_ENTRY); + + while (done + sizeof(struct ATTR_DEF_ENTRY) <= bytes) { + u32 t32 = le32_to_cpu(t->type); + u64 sz = le64_to_cpu(t->max_sz); + + if ((t32 & 0xF) || le32_to_cpu(t[-1].type) >= t32) + break; + + if (t->type == ATTR_REPARSE) + sbi->reparse.max_size = sz; + else if (t->type == ATTR_EA) + sbi->ea_max_size = sz; + + done += sizeof(struct ATTR_DEF_ENTRY); + t += 1; + sbi->def_entries += 1; + } + iput(inode); + + /* Load $UpCase. */ + ref.low = cpu_to_le32(MFT_REC_UPCASE); + ref.seq = cpu_to_le16(MFT_REC_UPCASE); + inode = ntfs_iget5(sb, &ref, &NAME_UPCASE); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + ntfs_err(sb, "Failed to load $UpCase (%d).", err); + goto out; + } + + if (inode->i_size != 0x10000 * sizeof(short)) { + err = -EINVAL; + ntfs_err(sb, "$UpCase is corrupted."); + goto put_inode_out; + } + + /* Read the entire file. */ + err = inode_read_data(inode, sbi->upcase, 0x10000 * sizeof(short)); + if (err) { + ntfs_err(sb, "Failed to read $UpCase (%d).", err); + goto put_inode_out; + } + +#ifdef __BIG_ENDIAN + { + u16 *dst = sbi->upcase; + + for (i = 0; i < 0x10000; i++) + __swab16s(dst++); + } +#endif + + shared = ntfs_set_shared(sbi->upcase, 0x10000 * sizeof(short)); + if (shared && sbi->upcase != shared) { + kvfree(sbi->upcase); + sbi->upcase = shared; + } + + iput(inode); + + if (is_ntfs3(sbi)) { + /* Load $Secure. */ + err = ntfs_security_init(sbi); + if (err) { + ntfs_err(sb, "Failed to initialize $Secure (%d).", err); + goto out; + } + + /* Load $Extend. */ + err = ntfs_extend_init(sbi); + if (err) { + ntfs_warn(sb, "Failed to initialize $Extend."); + goto load_root; + } + + /* Load $Extend/$Reparse. */ + err = ntfs_reparse_init(sbi); + if (err) { + ntfs_warn(sb, "Failed to initialize $Extend/$Reparse."); + goto load_root; + } + + /* Load $Extend/$ObjId. */ + err = ntfs_objid_init(sbi); + if (err) { + ntfs_warn(sb, "Failed to initialize $Extend/$ObjId."); + goto load_root; + } + } + +load_root: + /* Load root. */ + ref.low = cpu_to_le32(MFT_REC_ROOT); + ref.seq = cpu_to_le16(MFT_REC_ROOT); + inode = ntfs_iget5(sb, &ref, &NAME_ROOT); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + ntfs_err(sb, "Failed to load root (%d).", err); + goto out; + } + + /* + * Final check. Looks like this case should never occurs. + */ + if (!inode->i_op) { + err = -EINVAL; + ntfs_err(sb, "Failed to load root (%d).", err); + goto put_inode_out; + } + + sb->s_root = d_make_root(inode); + if (!sb->s_root) { + err = -ENOMEM; + goto out; + } + + if (boot2) { + /* + * Alternative boot is ok but primary is not ok. + * Volume is recognized as NTFS. Update primary boot. + */ + struct buffer_head *bh0 = sb_getblk(sb, 0); + if (bh0) { + wait_on_buffer(bh0); + lock_buffer(bh0); + memcpy(bh0->b_data, boot2, sizeof(*boot2)); + set_buffer_uptodate(bh0); + mark_buffer_dirty(bh0); + unlock_buffer(bh0); + if (!sync_dirty_buffer(bh0)) + ntfs_warn(sb, "primary boot is updated"); + put_bh(bh0); + } + + kfree(boot2); + } + + ntfs_create_procdir(sb); + + return 0; + +put_inode_out: + iput(inode); +out: + /* sbi->options == options */ + if (options) { + put_mount_options(sbi->options); + sbi->options = NULL; + } + + ntfs3_put_sbi(sbi); + kfree(boot2); + return err; +} + +void ntfs_unmap_meta(struct super_block *sb, CLST lcn, CLST len) +{ + struct ntfs_sb_info *sbi = sb->s_fs_info; + struct block_device *bdev = sb->s_bdev; + sector_t devblock = (u64)lcn * sbi->blocks_per_cluster; + unsigned long blocks = (u64)len * sbi->blocks_per_cluster; + unsigned long cnt = 0; + unsigned long limit = global_zone_page_state(NR_FREE_PAGES) + << (PAGE_SHIFT - sb->s_blocksize_bits); + + if (limit >= 0x2000) + limit -= 0x1000; + else if (limit < 32) + limit = 32; + else + limit >>= 1; + + while (blocks--) { + clean_bdev_aliases(bdev, devblock++, 1); + if (cnt++ >= limit) { + sync_blockdev(bdev); + cnt = 0; + } + } +} + +/* + * ntfs_discard - Issue a discard request (trim for SSD). + */ +int ntfs_discard(struct ntfs_sb_info *sbi, CLST lcn, CLST len) +{ + int err; + u64 lbo, bytes, start, end; + struct super_block *sb; + + if (sbi->used.next_free_lcn == lcn + len) + sbi->used.next_free_lcn = lcn; + + if (sbi->flags & NTFS_FLAGS_NODISCARD) + return -EOPNOTSUPP; + + if (!sbi->options->discard) + return -EOPNOTSUPP; + + lbo = (u64)lcn << sbi->cluster_bits; + bytes = (u64)len << sbi->cluster_bits; + + /* Align up 'start' on discard_granularity. */ + start = (lbo + sbi->discard_granularity - 1) & + sbi->discard_granularity_mask_inv; + /* Align down 'end' on discard_granularity. */ + end = (lbo + bytes) & sbi->discard_granularity_mask_inv; + + sb = sbi->sb; + if (start >= end) + return 0; + + err = blkdev_issue_discard(sb->s_bdev, start >> 9, (end - start) >> 9, + GFP_NOFS); + + if (err == -EOPNOTSUPP) + sbi->flags |= NTFS_FLAGS_NODISCARD; + + return err; +} + +static int ntfs_fs_get_tree(struct fs_context *fc) +{ + return get_tree_bdev(fc, ntfs_fill_super); +} + +/* + * ntfs_fs_free - Free fs_context. + * + * Note that this will be called after fill_super and reconfigure + * even when they pass. So they have to take pointers if they pass. + */ +static void ntfs_fs_free(struct fs_context *fc) +{ + struct ntfs_mount_options *opts = fc->fs_private; + struct ntfs_sb_info *sbi = fc->s_fs_info; + + if (sbi) { + ntfs3_put_sbi(sbi); + ntfs3_free_sbi(sbi); + } + + if (opts) + put_mount_options(opts); +} + +// clang-format off +static const struct fs_context_operations ntfs_context_ops = { + .parse_param = ntfs_fs_parse_param, + .get_tree = ntfs_fs_get_tree, + .reconfigure = ntfs_fs_reconfigure, + .free = ntfs_fs_free, +}; +// clang-format on + +/* + * ntfs_init_fs_context - Initialize sbi and opts + * + * This will called when mount/remount. We will first initialize + * options so that if remount we can use just that. + */ +static int ntfs_init_fs_context(struct fs_context *fc) +{ + struct ntfs_mount_options *opts; + struct ntfs_sb_info *sbi; + + opts = kzalloc_obj(struct ntfs_mount_options, GFP_NOFS); + if (!opts) + return -ENOMEM; + + /* Default options. */ + opts->fs_uid = current_uid(); + opts->fs_gid = current_gid(); + opts->fs_fmask_inv = opts->fs_dmask_inv = ~current_umask(); + opts->prealloc = 1; + opts->ads = 1; + +#ifdef CONFIG_NTFS3_FS_POSIX_ACL + /* Set the default value 'acl' */ + fc->sb_flags |= SB_POSIXACL; +#endif + + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE) + goto ok; + + sbi = kzalloc_obj(struct ntfs_sb_info, GFP_NOFS); + if (!sbi) + goto free_opts; + + sbi->upcase = kvmalloc(0x10000 * sizeof(short), GFP_KERNEL); + if (!sbi->upcase) + goto free_sbi; + + ratelimit_state_init(&sbi->msg_ratelimit, DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + + mutex_init(&sbi->compress.mtx_lznt); +#ifdef CONFIG_NTFS3_LZX_XPRESS + mutex_init(&sbi->compress.mtx_xpress); + mutex_init(&sbi->compress.mtx_lzx); +#endif + + fc->s_fs_info = sbi; +ok: + fc->fs_private = opts; + fc->ops = &ntfs_context_ops; + + return 0; +free_sbi: + kfree(sbi); +free_opts: + kfree(opts); + return -ENOMEM; +} + +static void ntfs3_kill_sb(struct super_block *sb) +{ + struct ntfs_sb_info *sbi = sb->s_fs_info; + + kill_block_super(sb); + + if (sbi->options) + put_mount_options(sbi->options); + ntfs3_free_sbi(sbi); +} + +// clang-format off +static struct file_system_type ntfs_fs_type = { + .owner = THIS_MODULE, + .name = "ntfs3", + .init_fs_context = ntfs_init_fs_context, + .parameters = ntfs_fs_parameters, + .kill_sb = ntfs3_kill_sb, + .fs_flags = FS_REQUIRES_DEV | FS_ALLOW_IDMAP, +}; +// clang-format on + +static int __init init_ntfs_fs(void) +{ + int err; + + if (IS_ENABLED(CONFIG_NTFS3_FS_POSIX_ACL)) + pr_info("ntfs3: Enabled Linux POSIX ACLs support\n"); + if (IS_ENABLED(CONFIG_NTFS3_64BIT_CLUSTER)) + pr_notice( + "ntfs3: Warning: Activated 64 bits per cluster. Windows does not support this\n"); + if (IS_ENABLED(CONFIG_NTFS3_LZX_XPRESS)) + pr_info("ntfs3: Read-only LZX/Xpress compression included\n"); + + ntfs_create_proc_root(); + + err = ntfs3_init_bitmap(); + if (err) + goto out2; + + ntfs_inode_cachep = kmem_cache_create( + "ntfs_inode_cache", sizeof(struct ntfs_inode), 0, + (SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT), init_once); + if (!ntfs_inode_cachep) { + err = -ENOMEM; + goto out1; + } + + err = register_filesystem(&ntfs_fs_type); + if (err) + goto out; + + return 0; +out: + kmem_cache_destroy(ntfs_inode_cachep); +out1: + ntfs3_exit_bitmap(); +out2: + ntfs_remove_proc_root(); + return err; +} + +static void __exit exit_ntfs_fs(void) +{ + rcu_barrier(); + kmem_cache_destroy(ntfs_inode_cachep); + unregister_filesystem(&ntfs_fs_type); + ntfs3_exit_bitmap(); + ntfs_remove_proc_root(); +} + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("ntfs3 read/write filesystem"); +#ifdef CONFIG_NTFS3_FS_POSIX_ACL +MODULE_INFO(behaviour, "Enabled Linux POSIX ACLs support"); +#endif +#ifdef CONFIG_NTFS3_64BIT_CLUSTER +MODULE_INFO( + cluster, + "Warning: Activated 64 bits per cluster. Windows does not support this"); +#endif +#ifdef CONFIG_NTFS3_LZX_XPRESS +MODULE_INFO(compression, "Read-only lzx/xpress compression included"); +#endif + +MODULE_AUTHOR("Konstantin Komarov"); +MODULE_ALIAS_FS("ntfs3"); + +module_init(init_ntfs_fs); +module_exit(exit_ntfs_fs); diff --git a/fs/ntfs3/upcase.c b/fs/ntfs3/upcase.c new file mode 100644 index 000000000..7681eefac --- /dev/null +++ b/fs/ntfs3/upcase.c @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + */ + +#include +#include + +#include "ntfs_fs.h" + +static inline u16 upcase_unicode_char(const u16 *upcase, u16 chr) +{ + if (chr < 'a') + return chr; + + if (chr <= 'z') + return chr - ('a' - 'A'); + + return upcase[chr]; +} + +/* + * ntfs_cmp_names + * + * Thanks Kari Argillander for idea and implementation 'bothcase' + * + * Straight way to compare names: + * - Case insensitive + * - If name equals and 'bothcases' then + * - Case sensitive + * 'Straight way' code scans input names twice in worst case. + * Optimized code scans input names only once. + */ +int ntfs_cmp_names(const __le16 *s1, size_t l1, const __le16 *s2, size_t l2, + const u16 *upcase, bool bothcase) +{ + int diff1 = 0; + int diff2; + size_t len = min(l1, l2); + + if (!bothcase && upcase) + goto case_insentive; + + for (; len; s1++, s2++, len--) { + diff1 = le16_to_cpu(*s1) - le16_to_cpu(*s2); + if (diff1) { + if (bothcase && upcase) + goto case_insentive; + + return diff1; + } + } + return l1 - l2; + +case_insentive: + for (; len; s1++, s2++, len--) { + diff2 = upcase_unicode_char(upcase, le16_to_cpu(*s1)) - + upcase_unicode_char(upcase, le16_to_cpu(*s2)); + if (diff2) + return diff2; + } + + diff2 = l1 - l2; + return diff2 ? diff2 : diff1; +} + +int ntfs_cmp_names_cpu(const struct cpu_str *uni1, const struct le_str *uni2, + const u16 *upcase, bool bothcase) +{ + const u16 *s1 = uni1->name; + const __le16 *s2 = uni2->name; + size_t l1 = uni1->len; + size_t l2 = uni2->len; + size_t len = min(l1, l2); + int diff1 = 0; + int diff2; + + if (!bothcase && upcase) + goto case_insentive; + + for (; len; s1++, s2++, len--) { + diff1 = *s1 - le16_to_cpu(*s2); + if (diff1) { + if (bothcase && upcase) + goto case_insentive; + + return diff1; + } + } + return l1 - l2; + +case_insentive: + for (; len; s1++, s2++, len--) { + diff2 = upcase_unicode_char(upcase, *s1) - + upcase_unicode_char(upcase, le16_to_cpu(*s2)); + if (diff2) + return diff2; + } + + diff2 = l1 - l2; + return diff2 ? diff2 : diff1; +} + +/* Helper function for ntfs_d_hash. */ +unsigned long ntfs_names_hash(const u16 *name, size_t len, const u16 *upcase, + unsigned long hash) +{ + while (len--) { + unsigned int c = upcase_unicode_char(upcase, *name++); + hash = partial_name_hash(c, hash); + } + + return hash; +} diff --git a/fs/ntfs3/xattr.c b/fs/ntfs3/xattr.c new file mode 100644 index 000000000..594ef6860 --- /dev/null +++ b/fs/ntfs3/xattr.c @@ -0,0 +1,1075 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * + * Copyright (C) 2019-2021 Paragon Software GmbH, All rights reserved. + * + */ + +#include +#include +#include +#include + +#include "debug.h" +#include "ntfs.h" +#include "ntfs_fs.h" + +// clang-format off +#define SYSTEM_DOS_ATTRIB "system.dos_attrib" +#define SYSTEM_NTFS_ATTRIB "system.ntfs_attrib" +#define SYSTEM_NTFS_ATTRIB_BE "system.ntfs_attrib_be" +#define SYSTEM_NTFS_SECURITY "system.ntfs_security" +// clang-format on + +static inline size_t unpacked_ea_size(const struct EA_FULL *ea) +{ + return ea->size ? le32_to_cpu(ea->size) : + ALIGN(struct_size(ea, name, + 1 + ea->name_len + + le16_to_cpu(ea->elength)), + 4); +} + +static inline size_t packed_ea_size(const struct EA_FULL *ea) +{ + return struct_size(ea, name, + 1 + ea->name_len + le16_to_cpu(ea->elength)) - + offsetof(struct EA_FULL, flags); +} + +/* + * find_ea + * + * Assume there is at least one xattr in the list. + */ +static inline bool find_ea(const struct EA_FULL *ea_all, u32 bytes, + const char *name, u8 name_len, u32 *off, u32 *ea_sz) +{ + u32 ea_size; + + *off = 0; + if (!ea_all) + return false; + + for (; *off < bytes; *off += ea_size) { + const struct EA_FULL *ea = Add2Ptr(ea_all, *off); + ea_size = unpacked_ea_size(ea); + if (ea->name_len == name_len && + !memcmp(ea->name, name, name_len)) { + if (ea_sz) + *ea_sz = ea_size; + return true; + } + } + + return false; +} + +/* + * ntfs_read_ea - Read all extended attributes. + * @ea: New allocated memory. + * @info: Pointer into resident data. + */ +static int ntfs_read_ea(struct ntfs_inode *ni, struct EA_FULL **ea, + size_t add_bytes, const struct EA_INFO **info) +{ + int err = -EINVAL; + struct ntfs_sb_info *sbi = ni->mi.sbi; + struct ATTR_LIST_ENTRY *le = NULL; + struct ATTRIB *attr_info, *attr_ea; + void *ea_p; + u32 size, off, ea_size; + + static_assert(le32_to_cpu(ATTR_EA_INFO) < le32_to_cpu(ATTR_EA)); + + *ea = NULL; + *info = NULL; + + attr_info = + ni_find_attr(ni, NULL, &le, ATTR_EA_INFO, NULL, 0, NULL, NULL); + attr_ea = + ni_find_attr(ni, attr_info, &le, ATTR_EA, NULL, 0, NULL, NULL); + + if (!attr_ea || !attr_info) + return 0; + + *info = resident_data_ex(attr_info, sizeof(struct EA_INFO)); + if (!*info) + goto out; + + /* Check Ea limit. */ + size = le32_to_cpu((*info)->size); + if (size > sbi->ea_max_size) { + err = -EFBIG; + goto out; + } + + if (attr_size(attr_ea) > sbi->ea_max_size) { + err = -EFBIG; + goto out; + } + + if (!size) { + /* EA info persists, but xattr is empty. Looks like EA problem. */ + goto out; + } + + /* Allocate memory for packed Ea. */ + ea_p = kmalloc(size_add(size, add_bytes), GFP_NOFS); + if (!ea_p) + return -ENOMEM; + + if (attr_ea->non_res) { + struct runs_tree run; + + run_init(&run); + + err = attr_load_runs_range(ni, ATTR_EA, NULL, 0, &run, 0, size); + if (!err) + err = ntfs_read_run_nb(sbi, &run, 0, ea_p, size, NULL); + run_close(&run); + + if (err) + goto out1; + } else { + void *p = resident_data_ex(attr_ea, size); + + if (!p) + goto out1; + memcpy(ea_p, p, size); + } + + memset(Add2Ptr(ea_p, size), 0, add_bytes); + + err = -EINVAL; + /* Check all attributes for consistency. */ + for (off = 0; off < size; off += ea_size) { + const struct EA_FULL *ef = Add2Ptr(ea_p, off); + u32 bytes = size - off; + size_t need; + + /* Check if we can use field ea->size. */ + if (bytes < sizeof(ef->size)) + goto out1; + + /* Check if we can use fields ef->name_len and ef->elength. */ + if (bytes < offsetof(struct EA_FULL, name)) + goto out1; + + /* Size needed to hold this record's name and value. */ + need = struct_size(ef, name, + 1 + ef->name_len + le16_to_cpu(ef->elength)); + + if (ef->size) { + ea_size = le32_to_cpu(ef->size); + /* ef->size must fit the list and cover the record. */ + if (ea_size > bytes || ea_size < need) + goto out1; + continue; + } + + ea_size = ALIGN(need, 4); + if (ea_size > bytes) + goto out1; + } + + *ea = ea_p; + return 0; + +out1: + kfree(ea_p); +out: + ntfs_set_state(sbi, NTFS_DIRTY_DIRTY); + return err; +} + +/* + * ntfs_list_ea + * + * Copy a list of xattrs names into the buffer + * provided, or compute the buffer size required. + * + * Return: + * * Number of bytes used / required on + * * -ERRNO - on failure + */ +static ssize_t ntfs_list_ea(struct ntfs_inode *ni, char *buffer, + size_t bytes_per_buffer) +{ + const struct EA_INFO *info; + struct EA_FULL *ea_all = NULL; + u32 off, size; + int err; + size_t ret; + + err = ntfs_read_ea(ni, &ea_all, 0, &info); + if (err) + return err; + + if (!info || !ea_all) + return 0; + + size = le32_to_cpu(info->size); + + /* Enumerate all xattrs. */ + ret = 0; + off = 0; + while (off + sizeof(struct EA_FULL) < size) { + const struct EA_FULL *ea = Add2Ptr(ea_all, off); + int ea_size = unpacked_ea_size(ea); + u8 name_len = ea->name_len; + + if (!name_len) + break; + + if (name_len > ea_size) { + ntfs_set_state(ni->mi.sbi, NTFS_DIRTY_ERROR); + err = -EINVAL; /* corrupted fs. */ + break; + } + + if (buffer) { + /* Check if we can use field ea->name */ + if (off + ea_size > size) + break; + + if (ret + name_len + 1 > bytes_per_buffer) { + err = -ERANGE; + goto out; + } + + memcpy(buffer + ret, ea->name, name_len); + buffer[ret + name_len] = 0; + } + + ret += name_len + 1; + off += ea_size; + } + +out: + kfree(ea_all); + return err ? err : ret; +} + +static int ntfs_get_ea(struct inode *inode, const char *name, size_t name_len, + void *buffer, size_t size, size_t *required) +{ + struct ntfs_inode *ni = ntfs_i(inode); + const struct EA_INFO *info; + struct EA_FULL *ea_all = NULL; + const struct EA_FULL *ea; + u32 off, len; + int err; + + if (!(ni->ni_flags & NI_FLAG_EA)) + return -ENODATA; + + if (!required) + ni_lock(ni); + + len = 0; + + if (name_len > 255) { + err = -ENAMETOOLONG; + goto out; + } + + err = ntfs_read_ea(ni, &ea_all, 0, &info); + if (err) + goto out; + + if (!info) + goto out; + + /* Enumerate all xattrs. */ + if (!find_ea(ea_all, le32_to_cpu(info->size), name, name_len, &off, + NULL)) { + err = -ENODATA; + goto out; + } + ea = Add2Ptr(ea_all, off); + + len = le16_to_cpu(ea->elength); + if (!buffer) { + err = 0; + goto out; + } + + if (len > size) { + err = -ERANGE; + if (required) + *required = len; + goto out; + } + + memcpy(buffer, ea->name + ea->name_len + 1, len); + err = 0; + +out: + kfree(ea_all); + if (!required) + ni_unlock(ni); + + return err ? err : len; +} + +static noinline int ntfs_set_ea(struct inode *inode, const char *name, + size_t name_len, const void *value, + size_t val_size, int flags, bool locked, + __le32 *ea_size) +{ + struct ntfs_inode *ni = ntfs_i(inode); + struct ntfs_sb_info *sbi = ni->mi.sbi; + int err; + struct EA_INFO ea_info; + const struct EA_INFO *info; + struct EA_FULL *new_ea; + struct EA_FULL *ea_all = NULL; + size_t add, new_pack; + u32 off, size, ea_sz; + __le16 size_pack; + struct ATTRIB *attr; + struct ATTR_LIST_ENTRY *le; + struct mft_inode *mi; + struct runs_tree ea_run; + u64 new_sz; + void *p; + + if (!locked) + ni_lock(ni); + + run_init(&ea_run); + + if (name_len > 255) { + err = -ENAMETOOLONG; + goto out; + } + + add = ALIGN(struct_size(ea_all, name, 1 + name_len + val_size), 4); + + err = ntfs_read_ea(ni, &ea_all, add, &info); + if (err) + goto out; + + if (!info) { + memset(&ea_info, 0, sizeof(ea_info)); + size = 0; + size_pack = 0; + } else { + memcpy(&ea_info, info, sizeof(ea_info)); + size = le32_to_cpu(ea_info.size); + size_pack = ea_info.size_pack; + } + + if (info && find_ea(ea_all, size, name, name_len, &off, &ea_sz)) { + struct EA_FULL *ea; + + if (flags & XATTR_CREATE) { + err = -EEXIST; + goto out; + } + + ea = Add2Ptr(ea_all, off); + + /* + * Check simple case when we try to insert xattr with the same value + * e.g. ntfs_save_wsl_perm + */ + if (val_size && le16_to_cpu(ea->elength) == val_size && + !memcmp(ea->name + ea->name_len + 1, value, val_size)) { + /* xattr already contains the required value. */ + goto out; + } + + /* Remove current xattr. */ + if (ea->flags & FILE_NEED_EA) + le16_add_cpu(&ea_info.count, -1); + + le16_add_cpu(&ea_info.size_pack, 0 - packed_ea_size(ea)); + + memmove(ea, Add2Ptr(ea, ea_sz), size - off - ea_sz); + + size -= ea_sz; + memset(Add2Ptr(ea_all, size), 0, ea_sz); + + ea_info.size = cpu_to_le32(size); + + if ((flags & XATTR_REPLACE) && !val_size) { + /* Remove xattr. */ + goto update_ea; + } + } else { + if (flags & XATTR_REPLACE) { + err = -ENODATA; + goto out; + } + + if (!ea_all) { + ea_all = kzalloc(add, GFP_NOFS); + if (!ea_all) { + err = -ENOMEM; + goto out; + } + } + } + + /* Append new xattr. */ + new_ea = Add2Ptr(ea_all, size); + new_ea->size = cpu_to_le32(add); + new_ea->flags = 0; + new_ea->name_len = name_len; + new_ea->elength = cpu_to_le16(val_size); + memcpy(new_ea->name, name, name_len); + new_ea->name[name_len] = 0; + memcpy(new_ea->name + name_len + 1, value, val_size); + new_pack = le16_to_cpu(ea_info.size_pack) + packed_ea_size(new_ea); + ea_info.size_pack = cpu_to_le16(new_pack); + /* New size of ATTR_EA. */ + size += add; + ea_info.size = cpu_to_le32(size); + + /* + * 1. Check ea_info.size_pack for overflow. + * 2. New attribute size must fit value from $AttrDef + */ + if (new_pack > 0xffff || size > sbi->ea_max_size) { + ntfs_inode_warn( + inode, + "The size of extended attributes must not exceed 64KiB"); + err = -EFBIG; // -EINVAL? + goto out; + } + +update_ea: + + if (!info) { + /* Create xattr. */ + if (!size) { + err = 0; + goto out; + } + + err = ni_insert_resident(ni, sizeof(struct EA_INFO), + ATTR_EA_INFO, NULL, 0, NULL, NULL, + NULL); + if (err) + goto out; + + err = ni_insert_resident(ni, 0, ATTR_EA, NULL, 0, NULL, NULL, + NULL); + if (err) + goto out; + } + + new_sz = size; + err = attr_set_size(ni, ATTR_EA, NULL, 0, &ea_run, new_sz, &new_sz, + false); + if (err) + goto out; + + le = NULL; + attr = ni_find_attr(ni, NULL, &le, ATTR_EA_INFO, NULL, 0, NULL, &mi); + if (!attr) { + err = -EINVAL; + goto out; + } + + if (!size) { + /* Delete xattr, ATTR_EA_INFO */ + ni_remove_attr_le(ni, attr, mi, le); + } else { + p = resident_data_ex(attr, sizeof(struct EA_INFO)); + if (!p) { + err = -EINVAL; + goto out; + } + memcpy(p, &ea_info, sizeof(struct EA_INFO)); + mi->dirty = true; + } + + le = NULL; + attr = ni_find_attr(ni, NULL, &le, ATTR_EA, NULL, 0, NULL, &mi); + if (!attr) { + err = -EINVAL; + goto out; + } + + if (!size) { + /* Delete xattr, ATTR_EA */ + ni_remove_attr_le(ni, attr, mi, le); + } else if (attr->non_res) { + err = attr_load_runs_range(ni, ATTR_EA, NULL, 0, &ea_run, 0, + size); + if (err) + goto out; + + err = ntfs_sb_write_run(sbi, &ea_run, 0, ea_all, size, 0); + if (err) + goto out; + } else { + p = resident_data_ex(attr, size); + if (!p) { + err = -EINVAL; + goto out; + } + memcpy(p, ea_all, size); + mi->dirty = true; + } + + /* Check if we delete the last xattr. */ + if (size) + ni->ni_flags |= NI_FLAG_EA; + else + ni->ni_flags &= ~NI_FLAG_EA; + + if (ea_info.size_pack != size_pack) + ni->ni_flags |= NI_FLAG_UPDATE_PARENT; + if (ea_size) + *ea_size = ea_info.size; + mark_inode_dirty(&ni->vfs_inode); + +out: + if (!locked) + ni_unlock(ni); + + run_close(&ea_run); + kfree(ea_all); + + return err; +} + +#ifdef CONFIG_NTFS3_FS_POSIX_ACL + +/* + * ntfs_get_acl - inode_operations::get_acl + */ +struct posix_acl *ntfs_get_acl(struct mnt_idmap *idmap, struct dentry *dentry, + int type) +{ + struct inode *inode = d_inode(dentry); + struct ntfs_inode *ni = ntfs_i(inode); + const char *name; + size_t name_len; + struct posix_acl *acl; + size_t req; + int err; + void *buf; + + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(ni))) + return ERR_PTR(-EINVAL); + + buf = kmalloc(PATH_MAX, GFP_KERNEL); + if (!buf) + return ERR_PTR(-ENOMEM); + + /* Possible values of 'type' was already checked above. */ + if (type == ACL_TYPE_ACCESS) { + name = XATTR_NAME_POSIX_ACL_ACCESS; + name_len = sizeof(XATTR_NAME_POSIX_ACL_ACCESS) - 1; + } else { + name = XATTR_NAME_POSIX_ACL_DEFAULT; + name_len = sizeof(XATTR_NAME_POSIX_ACL_DEFAULT) - 1; + } + + ni_lock(ni); + + err = ntfs_get_ea(inode, name, name_len, buf, PATH_MAX, &req); + + ni_unlock(ni); + + /* Translate extended attribute to acl. */ + if (err >= 0) { + acl = posix_acl_from_xattr(&init_user_ns, buf, err); + } else if (err == -ENODATA) { + acl = NULL; + } else { + acl = ERR_PTR(err); + } + + if (!IS_ERR(acl)) + set_cached_acl(inode, type, acl); + + kfree(buf); + + return acl; +} + +static noinline int ntfs_set_acl_ex(struct mnt_idmap *idmap, + struct inode *inode, struct posix_acl *acl, + int type, bool init_acl) +{ + const char *name; + size_t size, name_len; + void *value; + int err; + int flags; + umode_t mode; + + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(ntfs_i(inode)))) + return -EINVAL; + + if (S_ISLNK(inode->i_mode)) + return -EOPNOTSUPP; + + mode = inode->i_mode; + switch (type) { + case ACL_TYPE_ACCESS: + /* Do not change i_mode if we are in init_acl */ + if (acl && !init_acl) { + err = posix_acl_update_mode(idmap, inode, &mode, &acl); + if (err) + return err; + } + name = XATTR_NAME_POSIX_ACL_ACCESS; + name_len = sizeof(XATTR_NAME_POSIX_ACL_ACCESS) - 1; + break; + + case ACL_TYPE_DEFAULT: + if (!S_ISDIR(inode->i_mode)) + return acl ? -EACCES : 0; + name = XATTR_NAME_POSIX_ACL_DEFAULT; + name_len = sizeof(XATTR_NAME_POSIX_ACL_DEFAULT) - 1; + break; + + default: + return -EINVAL; + } + + if (!acl) { + /* Remove xattr if it can be presented via mode. */ + size = 0; + value = NULL; + flags = XATTR_REPLACE; + } else { + value = posix_acl_to_xattr(&init_user_ns, acl, &size, GFP_NOFS); + if (!value) + return -ENOMEM; + flags = 0; + } + + err = ntfs_set_ea(inode, name, name_len, value, size, flags, 0, NULL); + if (err == -ENODATA && !size) + err = 0; /* Removing non existed xattr. */ + if (err) + goto out; + + if (inode->i_mode != mode) { + umode_t old_mode = inode->i_mode; + inode->i_mode = mode; + err = ntfs_save_wsl_perm(inode, NULL); + if (err) { + inode->i_mode = old_mode; + goto out; + } + } + set_cached_acl(inode, type, acl); + inode_set_ctime_current(inode); + mark_inode_dirty(inode); + +out: + kfree(value); + + return err; +} + +/* + * ntfs_set_acl - inode_operations::set_acl + */ +int ntfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type) +{ + return ntfs_set_acl_ex(idmap, d_inode(dentry), acl, type, false); +} + +/* + * ntfs_init_acl - Initialize the ACLs of a new inode. + * + * Called from ntfs_create_inode(). + */ +int ntfs_init_acl(struct mnt_idmap *idmap, struct inode *inode, + struct inode *dir) +{ + struct posix_acl *default_acl, *acl; + int err; + + err = posix_acl_create(dir, &inode->i_mode, &default_acl, &acl); + if (err) + return err; + + if (default_acl) { + err = ntfs_set_acl_ex(idmap, inode, default_acl, + ACL_TYPE_DEFAULT, true); + posix_acl_release(default_acl); + } else { + inode->i_default_acl = NULL; + } + + if (acl) { + if (!err) + err = ntfs_set_acl_ex(idmap, inode, acl, + ACL_TYPE_ACCESS, true); + posix_acl_release(acl); + } else { + inode->i_acl = NULL; + } + + return err; +} +#endif + +/* + * ntfs_acl_chmod - Helper for ntfs_setattr(). + */ +int ntfs_acl_chmod(struct mnt_idmap *idmap, struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + struct super_block *sb = inode->i_sb; + + if (!(sb->s_flags & SB_POSIXACL)) + return 0; + + if (S_ISLNK(inode->i_mode)) + return -EOPNOTSUPP; + + return posix_acl_chmod(idmap, dentry, inode->i_mode); +} + +/* + * ntfs_listxattr - inode_operations::listxattr + */ +ssize_t ntfs_listxattr(struct dentry *dentry, char *buffer, size_t size) +{ + struct inode *inode = d_inode(dentry); + struct ntfs_inode *ni = ntfs_i(inode); + ssize_t ret; + + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(ni))) + return -EINVAL; + + if (!(ni->ni_flags & NI_FLAG_EA)) { + /* no xattr in file */ + return 0; + } + + ni_lock(ni); + + ret = ntfs_list_ea(ni, buffer, size); + + ni_unlock(ni); + + return ret; +} + +static int ntfs_getxattr(const struct xattr_handler *handler, struct dentry *de, + struct inode *inode, const char *name, void *buffer, + size_t size) +{ + int err; + struct ntfs_inode *ni = ntfs_i(inode); + + /* Avoid any operation if inode is bad. */ + if (unlikely(is_bad_ni(ni))) + return -EINVAL; + + if (unlikely(ntfs3_forced_shutdown(inode->i_sb))) + return -EIO; + + /* Dispatch request. */ + if (!strcmp(name, SYSTEM_DOS_ATTRIB)) { + /* system.dos_attrib */ + if (!buffer) { + err = sizeof(u8); + } else if (size < sizeof(u8)) { + err = -ENODATA; + } else { + err = sizeof(u8); + *(u8 *)buffer = le32_to_cpu(ni->std_fa); + } + goto out; + } + + if (!strcmp(name, SYSTEM_NTFS_ATTRIB) || + !strcmp(name, SYSTEM_NTFS_ATTRIB_BE)) { + /* system.ntfs_attrib */ + if (!buffer) { + err = sizeof(u32); + } else if (size < sizeof(u32)) { + err = -ENODATA; + } else { + err = sizeof(u32); + *(u32 *)buffer = le32_to_cpu(ni->std_fa); + if (!strcmp(name, SYSTEM_NTFS_ATTRIB_BE)) + *(__be32 *)buffer = cpu_to_be32(*(u32 *)buffer); + } + goto out; + } + + if (!strcmp(name, SYSTEM_NTFS_SECURITY)) { + /* system.ntfs_security*/ + struct SECURITY_DESCRIPTOR_RELATIVE *sd = NULL; + size_t sd_size = 0; + + if (!is_ntfs3(ni->mi.sbi)) { + /* We should get nt4 security. */ + err = -EINVAL; + goto out; + } else if (le32_to_cpu(ni->std_security_id) < + SECURITY_ID_FIRST) { + err = -ENOENT; + goto out; + } + + err = ntfs_get_security_by_id(ni->mi.sbi, ni->std_security_id, + &sd, &sd_size); + if (err) + goto out; + + if (!is_sd_valid(sd, sd_size)) { + ntfs_inode_warn( + inode, + "looks like you get incorrect security descriptor id=%u", + ni->std_security_id); + } + + if (!buffer) { + err = sd_size; + } else if (size < sd_size) { + err = -ENODATA; + } else { + err = sd_size; + memcpy(buffer, sd, sd_size); + } + kfree(sd); + goto out; + } + + /* Deal with NTFS extended attribute. */ + err = ntfs_get_ea(inode, name, strlen(name), buffer, size, NULL); + +out: + return err; +} + +static bool ntfs_is_reserved_lxattr(const char *name) +{ + return !strcmp(name, "$LXUID") || !strcmp(name, "$LXGID") || + !strcmp(name, "$LXMOD") || !strcmp(name, "$LXDEV"); +} + +/* + * ntfs_setxattr - inode_operations::setxattr + */ +static noinline int ntfs_setxattr(const struct xattr_handler *handler, + struct mnt_idmap *idmap, struct dentry *de, + struct inode *inode, const char *name, + const void *value, size_t size, int flags) +{ + int err = -EINVAL; + struct ntfs_inode *ni = ntfs_i(inode); + enum FILE_ATTRIBUTE new_fa; + + /* Dispatch request. */ + if (!strcmp(name, SYSTEM_DOS_ATTRIB)) { + if (sizeof(u8) != size) + goto out; + /* system.dos_attrib only covers the low DOS attribute byte. */ + new_fa = (ni->std_fa & ~cpu_to_le32(0xff)) | + cpu_to_le32(*(u8 *)value); + goto set_new_fa; + } + + if (!strcmp(name, SYSTEM_NTFS_ATTRIB) || + !strcmp(name, SYSTEM_NTFS_ATTRIB_BE)) { + if (size != sizeof(u32)) + goto out; + if (!strcmp(name, SYSTEM_NTFS_ATTRIB_BE)) + new_fa = cpu_to_le32(be32_to_cpu(*(__be32 *)value)); + else + new_fa = cpu_to_le32(*(u32 *)value); + + if (S_ISREG(inode->i_mode)) { + /* Process compressed/sparsed in special way. */ + ni_lock(ni); + err = ni_new_attr_flags(ni, new_fa); + ni_unlock(ni); + if (err) + goto out; + } +set_new_fa: + /* + * Thanks Mark Harmstone: + * Keep directory bit consistency. + */ + if (S_ISDIR(inode->i_mode)) + new_fa |= FILE_ATTRIBUTE_DIRECTORY; + else + new_fa &= ~FILE_ATTRIBUTE_DIRECTORY; + + if (ni->std_fa != new_fa) { + ni->std_fa = new_fa; + if (new_fa & FILE_ATTRIBUTE_READONLY) + inode->i_mode &= ~0222; + else + inode->i_mode |= 0222; + /* Std attribute always in primary record. */ + ni->mi.dirty = true; + mark_inode_dirty(inode); + } + err = 0; + + goto out; + } + + if (!strcmp(name, SYSTEM_NTFS_SECURITY)) { + /* system.ntfs_security*/ + __le32 security_id; + bool inserted; + struct ATTR_STD_INFO5 *std; + + if (!is_ntfs3(ni->mi.sbi)) { + /* + * We should replace ATTR_SECURE. + * Skip this way cause it is nt4 feature. + */ + err = -EINVAL; + goto out; + } + + if (!is_sd_valid(value, size)) { + err = -EINVAL; + ntfs_inode_warn( + inode, + "you try to set invalid security descriptor"); + goto out; + } + + err = ntfs_insert_security(ni->mi.sbi, value, size, + &security_id, &inserted); + if (err) + goto out; + + ni_lock(ni); + std = ni_std5(ni); + if (!std) { + err = -EINVAL; + } else if (std->security_id != security_id) { + std->security_id = ni->std_security_id = security_id; + /* Std attribute always in primary record. */ + ni->mi.dirty = true; + mark_inode_dirty(&ni->vfs_inode); + } + ni_unlock(ni); + goto out; + } + + /* Do not allow non privileged users to change $LXUID/$LXGID... */ + if (ntfs_is_reserved_lxattr(name) && !capable(CAP_SYS_ADMIN)) { + err = -EPERM; + goto out; + } + + /* Deal with NTFS extended attribute. */ + err = ntfs_set_ea(inode, name, strlen(name), value, size, flags, 0, + NULL); + +out: + inode_set_ctime_current(inode); + mark_inode_dirty(inode); + + return err; +} + +/* + * ntfs_save_wsl_perm + * + * save uid/gid/mode in xattr + */ +int ntfs_save_wsl_perm(struct inode *inode, __le32 *ea_size) +{ + int err; + __le32 value; + struct ntfs_inode *ni = ntfs_i(inode); + + ni_lock(ni); + value = cpu_to_le32(i_uid_read(inode)); + err = ntfs_set_ea(inode, "$LXUID", sizeof("$LXUID") - 1, &value, + sizeof(value), 0, true, ea_size); + if (err) + goto out; + + value = cpu_to_le32(i_gid_read(inode)); + err = ntfs_set_ea(inode, "$LXGID", sizeof("$LXGID") - 1, &value, + sizeof(value), 0, true, ea_size); + if (err) + goto out; + + value = cpu_to_le32(inode->i_mode); + err = ntfs_set_ea(inode, "$LXMOD", sizeof("$LXMOD") - 1, &value, + sizeof(value), 0, true, ea_size); + if (err) + goto out; + + if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) { + value = cpu_to_le32(inode->i_rdev); + err = ntfs_set_ea(inode, "$LXDEV", sizeof("$LXDEV") - 1, &value, + sizeof(value), 0, true, ea_size); + if (err) + goto out; + } + +out: + ni_unlock(ni); + /* In case of error should we delete all WSL xattr? */ + return err; +} + +/* + * ntfs_get_wsl_perm + * + * get uid/gid/mode from xattr + * it is called from ntfs_iget5->ntfs_read_mft + */ +void ntfs_get_wsl_perm(struct inode *inode) +{ + size_t sz; + __le32 value[3]; + + if (ntfs_get_ea(inode, "$LXUID", sizeof("$LXUID") - 1, &value[0], + sizeof(value[0]), &sz) == sizeof(value[0]) && + ntfs_get_ea(inode, "$LXGID", sizeof("$LXGID") - 1, &value[1], + sizeof(value[1]), &sz) == sizeof(value[1]) && + ntfs_get_ea(inode, "$LXMOD", sizeof("$LXMOD") - 1, &value[2], + sizeof(value[2]), &sz) == sizeof(value[2])) { + i_uid_write(inode, (uid_t)le32_to_cpu(value[0])); + i_gid_write(inode, (gid_t)le32_to_cpu(value[1])); + inode->i_mode = le32_to_cpu(value[2]); + + if (ntfs_get_ea(inode, "$LXDEV", sizeof("$LXDEV") - 1, + &value[0], sizeof(value), + &sz) == sizeof(value[0])) { + inode->i_rdev = le32_to_cpu(value[0]); + } + } +} + +static bool ntfs_xattr_user_list(struct dentry *dentry) +{ + return true; +} + +// clang-format off +static const struct xattr_handler ntfs_other_xattr_handler = { + .prefix = "", + .get = ntfs_getxattr, + .set = ntfs_setxattr, + .list = ntfs_xattr_user_list, +}; + +const struct xattr_handler * const ntfs_xattr_handlers[] = { + &ntfs_other_xattr_handler, + NULL, +}; +// clang-format on diff --git a/fs/nullfs.c b/fs/nullfs.c new file mode 100644 index 000000000..e06352c7b --- /dev/null +++ b/fs/nullfs.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (c) 2026 Christian Brauner */ +#include +#include +#include + +#include "mount.h" + +static const struct super_operations nullfs_super_operations = { + .statfs = simple_statfs, +}; + +static int nullfs_fs_fill_super(struct super_block *s, struct fs_context *fc) +{ + struct inode *inode; + + s->s_maxbytes = MAX_LFS_FILESIZE; + s->s_blocksize = PAGE_SIZE; + s->s_blocksize_bits = PAGE_SHIFT; + s->s_magic = NULL_FS_MAGIC; + s->s_op = &nullfs_super_operations; + s->s_export_op = NULL; + s->s_xattr = NULL; + s->s_time_gran = 1; + s->s_d_flags = 0; + + inode = new_inode(s); + if (!inode) + return -ENOMEM; + + /* nullfs is permanently empty... */ + make_empty_dir_inode(inode); + simple_inode_init_ts(inode); + inode->i_ino = 1; + /* ... and immutable. */ + inode->i_flags |= S_IMMUTABLE; + + s->s_root = d_make_root(inode); + if (!s->s_root) + return -ENOMEM; + + return 0; +} + +static int nullfs_fs_get_tree(struct fs_context *fc) +{ + return get_tree_nodev(fc, nullfs_fs_fill_super); +} + +static const struct fs_context_operations nullfs_fs_context_ops = { + .get_tree = nullfs_fs_get_tree, +}; + +static int nullfs_init_fs_context(struct fs_context *fc) +{ + fc->ops = &nullfs_fs_context_ops; + fc->sb_flags |= SB_NOUSER; + fc->s_iflags |= SB_I_NOEXEC | SB_I_NODEV; + return 0; +} + +struct file_system_type nullfs_fs_type = { + .name = "nullfs", + .init_fs_context = nullfs_init_fs_context, + .kill_sb = kill_anon_super, +}; diff --git a/fs/ocfs2/Kconfig b/fs/ocfs2/Kconfig new file mode 100644 index 000000000..2514d36cb --- /dev/null +++ b/fs/ocfs2/Kconfig @@ -0,0 +1,79 @@ +# SPDX-License-Identifier: GPL-2.0-only +config OCFS2_FS + tristate "OCFS2 file system support" + depends on INET && SYSFS && CONFIGFS_FS + select BUFFER_HEAD + select JBD2 + select CRC32 + select QUOTA + select QUOTA_TREE + select FS_POSIX_ACL + select LEGACY_DIRECT_IO + help + OCFS2 is a general purpose extent based shared disk cluster file + system with many similarities to ext3. It supports 64 bit inode + numbers, and has automatically extending metadata groups which may + also make it attractive for non-clustered use. + + You'll want to install the ocfs2-tools package in order to at least + get "mount.ocfs2". + + Project web page: https://ocfs2.wiki.kernel.org/ + Tools web page: https://github.com/markfasheh/ocfs2-tools + OCFS2 mailing lists: https://subspace.kernel.org/lists.linux.dev.html + + For more information on OCFS2, see the file + . + +config OCFS2_FS_O2CB + tristate "O2CB Kernelspace Clustering" + depends on OCFS2_FS + default y + help + OCFS2 includes a simple kernelspace clustering package, the OCFS2 + Cluster Base. It only requires a very small userspace component + to configure it. This comes with the standard ocfs2-tools package. + O2CB is limited to maintaining a cluster for OCFS2 file systems. + It cannot manage any other cluster applications. + + It is always safe to say Y here, as the clustering method is + run-time selectable. + +config OCFS2_FS_USERSPACE_CLUSTER + tristate "OCFS2 Userspace Clustering" + depends on OCFS2_FS && DLM + default y + help + This option will allow OCFS2 to use userspace clustering services + in conjunction with the DLM in fs/dlm. If you are using a + userspace cluster manager, say Y here. + + It is safe to say Y, as the clustering method is run-time + selectable. + +config OCFS2_FS_STATS + bool "OCFS2 statistics" + depends on OCFS2_FS && DEBUG_FS + default y + help + This option allows some fs statistics to be captured. Enabling + this option may increase the memory consumption. + +config OCFS2_DEBUG_MASKLOG + bool "OCFS2 logging support" + depends on OCFS2_FS + default y + help + The ocfs2 filesystem has an extensive logging system. The system + allows selection of events to log via files in /sys/o2cb/logmask/. + This option will enlarge your kernel, but it allows debugging of + ocfs2 filesystem issues. + +config OCFS2_DEBUG_FS + bool "OCFS2 expensive checks" + depends on OCFS2_FS + default n + help + This option will enable expensive consistency checks. Enable + this option for debugging only as it is likely to decrease + performance of the filesystem. diff --git a/fs/ocfs2/Makefile b/fs/ocfs2/Makefile new file mode 100644 index 000000000..cc9b32b9d --- /dev/null +++ b/fs/ocfs2/Makefile @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: GPL-2.0 +ccflags-y := -I$(src) + +obj-$(CONFIG_OCFS2_FS) += \ + ocfs2.o \ + ocfs2_stackglue.o + +obj-$(CONFIG_OCFS2_FS_O2CB) += ocfs2_stack_o2cb.o +obj-$(CONFIG_OCFS2_FS_USERSPACE_CLUSTER) += ocfs2_stack_user.o + +ocfs2-objs := \ + alloc.o \ + aops.o \ + blockcheck.o \ + buffer_head_io.o \ + dcache.o \ + dir.o \ + dlmglue.o \ + export.o \ + extent_map.o \ + file.o \ + heartbeat.o \ + inode.o \ + ioctl.o \ + journal.o \ + localalloc.o \ + locks.o \ + mmap.o \ + namei.o \ + refcounttree.o \ + reservations.o \ + move_extents.o \ + resize.o \ + slot_map.o \ + suballoc.o \ + super.o \ + symlink.o \ + sysfile.o \ + uptodate.o \ + quota_local.o \ + quota_global.o \ + xattr.o \ + acl.o \ + filecheck.o + +ocfs2_stackglue-objs := stackglue.o +ocfs2_stack_o2cb-objs := stack_o2cb.o +ocfs2_stack_user-objs := stack_user.o + +obj-$(CONFIG_OCFS2_FS) += dlmfs/ +# cluster/ is always needed when OCFS2_FS for masklog support +obj-$(CONFIG_OCFS2_FS) += cluster/ +obj-$(CONFIG_OCFS2_FS_O2CB) += dlm/ diff --git a/fs/ocfs2/acl.c b/fs/ocfs2/acl.c new file mode 100644 index 000000000..090ec60fb --- /dev/null +++ b/fs/ocfs2/acl.c @@ -0,0 +1,452 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * acl.c + * + * Copyright (C) 2004, 2008 Oracle. All rights reserved. + * + * CREDITS: + * Lots of code in this file is copy from linux/fs/ext3/acl.c. + * Copyright (C) 2001-2003 Andreas Gruenbacher, + */ + +#include +#include +#include +#include +#include + +#include + +#include "ocfs2.h" +#include "alloc.h" +#include "dlmglue.h" +#include "file.h" +#include "inode.h" +#include "journal.h" +#include "ocfs2_fs.h" + +#include "xattr.h" +#include "acl.h" + +/* + * Convert from xattr value to acl struct. + */ +static struct posix_acl *ocfs2_acl_from_xattr(const void *value, size_t size) +{ + int n, count; + struct posix_acl *acl; + + if (!value) + return NULL; + if (size < sizeof(struct posix_acl_entry)) + return ERR_PTR(-EINVAL); + + count = size / sizeof(struct posix_acl_entry); + + acl = posix_acl_alloc(count, GFP_NOFS); + if (!acl) + return ERR_PTR(-ENOMEM); + for (n = 0; n < count; n++) { + struct ocfs2_acl_entry *entry = + (struct ocfs2_acl_entry *)value; + + acl->a_entries[n].e_tag = le16_to_cpu(entry->e_tag); + acl->a_entries[n].e_perm = le16_to_cpu(entry->e_perm); + switch(acl->a_entries[n].e_tag) { + case ACL_USER: + acl->a_entries[n].e_uid = + make_kuid(&init_user_ns, + le32_to_cpu(entry->e_id)); + break; + case ACL_GROUP: + acl->a_entries[n].e_gid = + make_kgid(&init_user_ns, + le32_to_cpu(entry->e_id)); + break; + default: + break; + } + value += sizeof(struct posix_acl_entry); + + } + return acl; +} + +/* + * Convert acl struct to xattr value. + */ +static void *ocfs2_acl_to_xattr(const struct posix_acl *acl, size_t *size) +{ + struct ocfs2_acl_entry *entry = NULL; + char *ocfs2_acl; + size_t n; + + *size = acl->a_count * sizeof(struct posix_acl_entry); + + ocfs2_acl = kmalloc(*size, GFP_NOFS); + if (!ocfs2_acl) + return ERR_PTR(-ENOMEM); + + entry = (struct ocfs2_acl_entry *)ocfs2_acl; + for (n = 0; n < acl->a_count; n++, entry++) { + entry->e_tag = cpu_to_le16(acl->a_entries[n].e_tag); + entry->e_perm = cpu_to_le16(acl->a_entries[n].e_perm); + switch(acl->a_entries[n].e_tag) { + case ACL_USER: + entry->e_id = cpu_to_le32( + from_kuid(&init_user_ns, + acl->a_entries[n].e_uid)); + break; + case ACL_GROUP: + entry->e_id = cpu_to_le32( + from_kgid(&init_user_ns, + acl->a_entries[n].e_gid)); + break; + default: + entry->e_id = cpu_to_le32(ACL_UNDEFINED_ID); + break; + } + } + return ocfs2_acl; +} + +static struct posix_acl *ocfs2_get_acl_nolock(struct inode *inode, int type, + struct buffer_head *di_bh) +{ + int name_index; + char *value = NULL; + struct posix_acl *acl; + int retval; + + switch (type) { + case ACL_TYPE_ACCESS: + name_index = OCFS2_XATTR_INDEX_POSIX_ACL_ACCESS; + break; + case ACL_TYPE_DEFAULT: + name_index = OCFS2_XATTR_INDEX_POSIX_ACL_DEFAULT; + break; + default: + return ERR_PTR(-EINVAL); + } + + retval = ocfs2_xattr_get_nolock(inode, di_bh, name_index, "", NULL, 0); + if (retval > 0) { + value = kmalloc(retval, GFP_NOFS); + if (!value) + return ERR_PTR(-ENOMEM); + retval = ocfs2_xattr_get_nolock(inode, di_bh, name_index, + "", value, retval); + } + + if (retval > 0) + acl = ocfs2_acl_from_xattr(value, retval); + else if (retval == -ENODATA || retval == 0) + acl = NULL; + else + acl = ERR_PTR(retval); + + kfree(value); + + return acl; +} + +/* + * Helper function to set i_mode in memory and disk. Some call paths + * will not have di_bh or a journal handle to pass, in which case it + * will create it's own. + */ +static int ocfs2_acl_set_mode(struct inode *inode, struct buffer_head *di_bh, + handle_t *handle, umode_t new_mode) +{ + int ret, commit_handle = 0; + struct ocfs2_dinode *di; + + if (di_bh == NULL) { + ret = ocfs2_read_inode_block(inode, &di_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + } else + get_bh(di_bh); + + if (handle == NULL) { + handle = ocfs2_start_trans(OCFS2_SB(inode->i_sb), + OCFS2_INODE_UPDATE_CREDITS); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out_brelse; + } + + commit_handle = 1; + } + + di = (struct ocfs2_dinode *)di_bh->b_data; + ret = ocfs2_journal_access_di(handle, INODE_CACHE(inode), di_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + inode->i_mode = new_mode; + inode_set_ctime_current(inode); + di->i_mode = cpu_to_le16(inode->i_mode); + di->i_ctime = cpu_to_le64(inode_get_ctime_sec(inode)); + di->i_ctime_nsec = cpu_to_le32(inode_get_ctime_nsec(inode)); + ocfs2_update_inode_fsync_trans(handle, inode, 0); + + ocfs2_journal_dirty(handle, di_bh); + +out_commit: + if (commit_handle) + ocfs2_commit_trans(OCFS2_SB(inode->i_sb), handle); +out_brelse: + brelse(di_bh); +out: + return ret; +} + +/* + * Set the access or default ACL of an inode. + */ +static int ocfs2_set_acl(handle_t *handle, + struct inode *inode, + struct buffer_head *di_bh, + int type, + struct posix_acl *acl, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_alloc_context *data_ac) +{ + int name_index; + void *value = NULL; + size_t size = 0; + int ret; + + if (S_ISLNK(inode->i_mode)) + return -EOPNOTSUPP; + + switch (type) { + case ACL_TYPE_ACCESS: + name_index = OCFS2_XATTR_INDEX_POSIX_ACL_ACCESS; + break; + case ACL_TYPE_DEFAULT: + name_index = OCFS2_XATTR_INDEX_POSIX_ACL_DEFAULT; + if (!S_ISDIR(inode->i_mode)) + return acl ? -EACCES : 0; + break; + default: + return -EINVAL; + } + + if (acl) { + value = ocfs2_acl_to_xattr(acl, &size); + if (IS_ERR(value)) + return (int)PTR_ERR(value); + } + + if (handle) + ret = ocfs2_xattr_set_handle(handle, inode, di_bh, name_index, + "", value, size, 0, + meta_ac, data_ac); + else + ret = ocfs2_xattr_set(inode, name_index, "", value, size, 0); + + kfree(value); + if (!ret) + set_cached_acl(inode, type, acl); + + return ret; +} + +int ocfs2_iop_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type) +{ + struct buffer_head *bh = NULL; + int status, had_lock; + struct ocfs2_lock_holder oh; + struct inode *inode = d_inode(dentry); + + had_lock = ocfs2_inode_lock_tracker(inode, &bh, 1, &oh); + if (had_lock < 0) + return had_lock; + if (type == ACL_TYPE_ACCESS && acl) { + umode_t mode; + + status = posix_acl_update_mode(&nop_mnt_idmap, inode, &mode, + &acl); + if (status) + goto unlock; + + status = ocfs2_acl_set_mode(inode, bh, NULL, mode); + if (status) + goto unlock; + } + status = ocfs2_set_acl(NULL, inode, bh, type, acl, NULL, NULL); +unlock: + ocfs2_inode_unlock_tracker(inode, 1, &oh, had_lock); + brelse(bh); + return status; +} + +struct posix_acl *ocfs2_iop_get_acl(struct inode *inode, int type, bool rcu) +{ + struct ocfs2_super *osb; + struct buffer_head *di_bh = NULL; + struct posix_acl *acl; + int had_lock; + struct ocfs2_lock_holder oh; + + if (rcu) + return ERR_PTR(-ECHILD); + + osb = OCFS2_SB(inode->i_sb); + if (!(osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL)) + return NULL; + + had_lock = ocfs2_inode_lock_tracker(inode, &di_bh, 0, &oh); + if (had_lock < 0) + return ERR_PTR(had_lock); + + down_read(&OCFS2_I(inode)->ip_xattr_sem); + acl = ocfs2_get_acl_nolock(inode, type, di_bh); + up_read(&OCFS2_I(inode)->ip_xattr_sem); + + ocfs2_inode_unlock_tracker(inode, 0, &oh, had_lock); + brelse(di_bh); + return acl; +} + +int ocfs2_acl_chmod(struct inode *inode, struct buffer_head *bh) +{ + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct posix_acl *acl; + int ret; + + if (S_ISLNK(inode->i_mode)) + return -EOPNOTSUPP; + + if (!(osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL)) + return 0; + + down_read(&OCFS2_I(inode)->ip_xattr_sem); + acl = ocfs2_get_acl_nolock(inode, ACL_TYPE_ACCESS, bh); + up_read(&OCFS2_I(inode)->ip_xattr_sem); + if (IS_ERR_OR_NULL(acl)) + return PTR_ERR_OR_ZERO(acl); + ret = __posix_acl_chmod(&acl, GFP_KERNEL, inode->i_mode); + if (ret) + return ret; + ret = ocfs2_set_acl(NULL, inode, NULL, ACL_TYPE_ACCESS, + acl, NULL, NULL); + posix_acl_release(acl); + return ret; +} + +/* + * Initialize the ACLs of a new inode. If parent directory has default ACL, + * then clone to new inode. Called from ocfs2_mknod. + */ +void ocfs2_acl_init_release(struct ocfs2_acl_state *state) +{ + posix_acl_release(state->default_acl); + posix_acl_release(state->acl); + state->default_acl = NULL; + state->acl = NULL; +} + +int ocfs2_acl_init_prepare(struct inode *inode, struct inode *dir, + struct buffer_head *dir_bh, + struct ocfs2_acl_state *state) +{ + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + int ret = 0; + + state->default_acl = NULL; + state->acl = NULL; + state->mode = inode->i_mode; + + if (S_ISLNK(inode->i_mode)) + return 0; + + if (osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) { + down_read(&OCFS2_I(dir)->ip_xattr_sem); + state->default_acl = + ocfs2_get_acl_nolock(dir, ACL_TYPE_DEFAULT, dir_bh); + up_read(&OCFS2_I(dir)->ip_xattr_sem); + if (IS_ERR(state->default_acl)) { + ret = PTR_ERR(state->default_acl); + state->default_acl = NULL; + return ret; + } + if (state->default_acl) { + state->acl = posix_acl_dup(state->default_acl); + if (!state->acl) { + ret = -ENOMEM; + goto cleanup; + } + ret = __posix_acl_create(&state->acl, GFP_NOFS, + &state->mode); + if (ret < 0) + goto cleanup; + if (ret == 0) { + posix_acl_release(state->acl); + state->acl = NULL; + } + if (!S_ISDIR(inode->i_mode)) { + posix_acl_release(state->default_acl); + state->default_acl = NULL; + } + } else { + state->mode &= ~current_umask(); + } + } else { + state->mode &= ~current_umask(); + } + + return 0; +cleanup: + ocfs2_acl_init_release(state); + return ret; +} + +int ocfs2_init_acl(handle_t *handle, struct inode *inode, + struct buffer_head *di_bh, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_alloc_context *data_ac, + struct ocfs2_acl_state *state) +{ + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + int ret = 0; + + if (S_ISLNK(inode->i_mode)) + return 0; + + if (osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) { + if (S_ISDIR(inode->i_mode) && state->default_acl) { + ret = ocfs2_set_acl(handle, inode, di_bh, + ACL_TYPE_DEFAULT, + state->default_acl, meta_ac, + data_ac); + if (ret) + return ret; + } + } + + ret = ocfs2_acl_set_mode(inode, di_bh, handle, state->mode); + if (ret) { + mlog_errno(ret); + return ret; + } + + if (osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) { + if (state->acl) { + ret = ocfs2_set_acl(handle, inode, di_bh, + ACL_TYPE_ACCESS, state->acl, + meta_ac, data_ac); + } + } + + return ret; +} diff --git a/fs/ocfs2/acl.h b/fs/ocfs2/acl.h new file mode 100644 index 000000000..a91f9ce27 --- /dev/null +++ b/fs/ocfs2/acl.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * acl.h + * + * Copyright (C) 2004, 2008 Oracle. All rights reserved. + */ + +#ifndef OCFS2_ACL_H +#define OCFS2_ACL_H + +#include + +struct ocfs2_acl_entry { + __le16 e_tag; + __le16 e_perm; + __le32 e_id; +}; + +struct posix_acl *ocfs2_iop_get_acl(struct inode *inode, int type, bool rcu); +int ocfs2_iop_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type); +extern int ocfs2_acl_chmod(struct inode *, struct buffer_head *); +struct ocfs2_acl_state { + struct posix_acl *default_acl; + struct posix_acl *acl; + umode_t mode; +}; + +int ocfs2_acl_init_prepare(struct inode *inode, struct inode *dir, + struct buffer_head *dir_bh, + struct ocfs2_acl_state *state); +void ocfs2_acl_init_release(struct ocfs2_acl_state *state); +int ocfs2_init_acl(handle_t *handle, struct inode *inode, + struct buffer_head *di_bh, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_alloc_context *data_ac, + struct ocfs2_acl_state *state); + +#endif /* OCFS2_ACL_H */ diff --git a/fs/ocfs2/alloc.c b/fs/ocfs2/alloc.c new file mode 100644 index 000000000..be09e766a --- /dev/null +++ b/fs/ocfs2/alloc.c @@ -0,0 +1,7741 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * alloc.c + * + * Extent allocs and frees + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ocfs2.h" + +#include "alloc.h" +#include "aops.h" +#include "blockcheck.h" +#include "dlmglue.h" +#include "extent_map.h" +#include "inode.h" +#include "journal.h" +#include "localalloc.h" +#include "suballoc.h" +#include "sysfile.h" +#include "file.h" +#include "super.h" +#include "uptodate.h" +#include "xattr.h" +#include "refcounttree.h" +#include "ocfs2_trace.h" + +#include "buffer_head_io.h" + +enum ocfs2_contig_type { + CONTIG_NONE = 0, + CONTIG_LEFT, + CONTIG_RIGHT, + CONTIG_LEFTRIGHT, +}; + +static enum ocfs2_contig_type + ocfs2_extent_rec_contig(struct super_block *sb, + struct ocfs2_extent_rec *ext, + struct ocfs2_extent_rec *insert_rec); +/* + * Operations for a specific extent tree type. + * + * To implement an on-disk btree (extent tree) type in ocfs2, add + * an ocfs2_extent_tree_operations structure and the matching + * ocfs2_init__extent_tree() function. That's pretty much it + * for the allocation portion of the extent tree. + */ +struct ocfs2_extent_tree_operations { + /* + * last_eb_blk is the block number of the right most leaf extent + * block. Most on-disk structures containing an extent tree store + * this value for fast access. The ->eo_set_last_eb_blk() and + * ->eo_get_last_eb_blk() operations access this value. They are + * both required. + */ + void (*eo_set_last_eb_blk)(struct ocfs2_extent_tree *et, + u64 blkno); + u64 (*eo_get_last_eb_blk)(struct ocfs2_extent_tree *et); + + /* + * The on-disk structure usually keeps track of how many total + * clusters are stored in this extent tree. This function updates + * that value. new_clusters is the delta, and must be + * added to the total. Required. + */ + void (*eo_update_clusters)(struct ocfs2_extent_tree *et, + u32 new_clusters); + + /* + * If this extent tree is supported by an extent map, insert + * a record into the map. + */ + void (*eo_extent_map_insert)(struct ocfs2_extent_tree *et, + struct ocfs2_extent_rec *rec); + + /* + * If this extent tree is supported by an extent map, truncate the + * map to clusters, + */ + void (*eo_extent_map_truncate)(struct ocfs2_extent_tree *et, + u32 clusters); + + /* + * If ->eo_insert_check() exists, it is called before rec is + * inserted into the extent tree. It is optional. + */ + int (*eo_insert_check)(struct ocfs2_extent_tree *et, + struct ocfs2_extent_rec *rec); + int (*eo_sanity_check)(struct ocfs2_extent_tree *et); + + /* + * -------------------------------------------------------------- + * The remaining are internal to ocfs2_extent_tree and don't have + * accessor functions + */ + + /* + * ->eo_fill_root_el() takes et->et_object and sets et->et_root_el. + * It is required. + */ + void (*eo_fill_root_el)(struct ocfs2_extent_tree *et); + + /* + * ->eo_fill_max_leaf_clusters sets et->et_max_leaf_clusters if + * it exists. If it does not, et->et_max_leaf_clusters is set + * to 0 (unlimited). Optional. + */ + void (*eo_fill_max_leaf_clusters)(struct ocfs2_extent_tree *et); + + /* + * ->eo_extent_contig test whether the 2 ocfs2_extent_rec + * are contiguous or not. Optional. Don't need to set it if use + * ocfs2_extent_rec as the tree leaf. + */ + enum ocfs2_contig_type + (*eo_extent_contig)(struct ocfs2_extent_tree *et, + struct ocfs2_extent_rec *ext, + struct ocfs2_extent_rec *insert_rec); +}; + + +/* + * Pre-declare ocfs2_dinode_et_ops so we can use it as a sanity check + * in the methods. + */ +static u64 ocfs2_dinode_get_last_eb_blk(struct ocfs2_extent_tree *et); +static void ocfs2_dinode_set_last_eb_blk(struct ocfs2_extent_tree *et, + u64 blkno); +static void ocfs2_dinode_update_clusters(struct ocfs2_extent_tree *et, + u32 clusters); +static void ocfs2_dinode_extent_map_insert(struct ocfs2_extent_tree *et, + struct ocfs2_extent_rec *rec); +static void ocfs2_dinode_extent_map_truncate(struct ocfs2_extent_tree *et, + u32 clusters); +static int ocfs2_dinode_insert_check(struct ocfs2_extent_tree *et, + struct ocfs2_extent_rec *rec); +static int ocfs2_dinode_sanity_check(struct ocfs2_extent_tree *et); +static void ocfs2_dinode_fill_root_el(struct ocfs2_extent_tree *et); + +static int ocfs2_reuse_blk_from_dealloc(handle_t *handle, + struct ocfs2_extent_tree *et, + struct buffer_head **new_eb_bh, + int blk_wanted, int *blk_given); +static int ocfs2_is_dealloc_empty(struct ocfs2_extent_tree *et); + +static const struct ocfs2_extent_tree_operations ocfs2_dinode_et_ops = { + .eo_set_last_eb_blk = ocfs2_dinode_set_last_eb_blk, + .eo_get_last_eb_blk = ocfs2_dinode_get_last_eb_blk, + .eo_update_clusters = ocfs2_dinode_update_clusters, + .eo_extent_map_insert = ocfs2_dinode_extent_map_insert, + .eo_extent_map_truncate = ocfs2_dinode_extent_map_truncate, + .eo_insert_check = ocfs2_dinode_insert_check, + .eo_sanity_check = ocfs2_dinode_sanity_check, + .eo_fill_root_el = ocfs2_dinode_fill_root_el, +}; + +static void ocfs2_dinode_set_last_eb_blk(struct ocfs2_extent_tree *et, + u64 blkno) +{ + struct ocfs2_dinode *di = et->et_object; + + BUG_ON(et->et_ops != &ocfs2_dinode_et_ops); + di->i_last_eb_blk = cpu_to_le64(blkno); +} + +static u64 ocfs2_dinode_get_last_eb_blk(struct ocfs2_extent_tree *et) +{ + struct ocfs2_dinode *di = et->et_object; + + BUG_ON(et->et_ops != &ocfs2_dinode_et_ops); + return le64_to_cpu(di->i_last_eb_blk); +} + +static void ocfs2_dinode_update_clusters(struct ocfs2_extent_tree *et, + u32 clusters) +{ + struct ocfs2_inode_info *oi = cache_info_to_inode(et->et_ci); + struct ocfs2_dinode *di = et->et_object; + + le32_add_cpu(&di->i_clusters, clusters); + spin_lock(&oi->ip_lock); + oi->ip_clusters = le32_to_cpu(di->i_clusters); + spin_unlock(&oi->ip_lock); +} + +static void ocfs2_dinode_extent_map_insert(struct ocfs2_extent_tree *et, + struct ocfs2_extent_rec *rec) +{ + struct inode *inode = &cache_info_to_inode(et->et_ci)->vfs_inode; + + ocfs2_extent_map_insert_rec(inode, rec); +} + +static void ocfs2_dinode_extent_map_truncate(struct ocfs2_extent_tree *et, + u32 clusters) +{ + struct inode *inode = &cache_info_to_inode(et->et_ci)->vfs_inode; + + ocfs2_extent_map_trunc(inode, clusters); +} + +static int ocfs2_dinode_insert_check(struct ocfs2_extent_tree *et, + struct ocfs2_extent_rec *rec) +{ + struct ocfs2_inode_info *oi = cache_info_to_inode(et->et_ci); + struct ocfs2_super *osb = OCFS2_SB(oi->vfs_inode.i_sb); + + BUG_ON(oi->ip_dyn_features & OCFS2_INLINE_DATA_FL); + mlog_bug_on_msg(!ocfs2_sparse_alloc(osb) && + (oi->ip_clusters != le32_to_cpu(rec->e_cpos)), + "Device %s, asking for sparse allocation: inode %llu, " + "cpos %u, clusters %u\n", + osb->dev_str, + (unsigned long long)oi->ip_blkno, + rec->e_cpos, oi->ip_clusters); + + return 0; +} + +static int ocfs2_dinode_sanity_check(struct ocfs2_extent_tree *et) +{ + struct ocfs2_dinode *di = et->et_object; + + BUG_ON(et->et_ops != &ocfs2_dinode_et_ops); + BUG_ON(!OCFS2_IS_VALID_DINODE(di)); + + return 0; +} + +static void ocfs2_dinode_fill_root_el(struct ocfs2_extent_tree *et) +{ + struct ocfs2_dinode *di = et->et_object; + + et->et_root_el = &di->id2.i_list; +} + + +static void ocfs2_xattr_value_fill_root_el(struct ocfs2_extent_tree *et) +{ + struct ocfs2_xattr_value_buf *vb = et->et_object; + + et->et_root_el = &vb->vb_xv->xr_list; +} + +static void ocfs2_xattr_value_set_last_eb_blk(struct ocfs2_extent_tree *et, + u64 blkno) +{ + struct ocfs2_xattr_value_buf *vb = et->et_object; + + vb->vb_xv->xr_last_eb_blk = cpu_to_le64(blkno); +} + +static u64 ocfs2_xattr_value_get_last_eb_blk(struct ocfs2_extent_tree *et) +{ + struct ocfs2_xattr_value_buf *vb = et->et_object; + + return le64_to_cpu(vb->vb_xv->xr_last_eb_blk); +} + +static void ocfs2_xattr_value_update_clusters(struct ocfs2_extent_tree *et, + u32 clusters) +{ + struct ocfs2_xattr_value_buf *vb = et->et_object; + + le32_add_cpu(&vb->vb_xv->xr_clusters, clusters); +} + +static const struct ocfs2_extent_tree_operations ocfs2_xattr_value_et_ops = { + .eo_set_last_eb_blk = ocfs2_xattr_value_set_last_eb_blk, + .eo_get_last_eb_blk = ocfs2_xattr_value_get_last_eb_blk, + .eo_update_clusters = ocfs2_xattr_value_update_clusters, + .eo_fill_root_el = ocfs2_xattr_value_fill_root_el, +}; + +static void ocfs2_xattr_tree_fill_root_el(struct ocfs2_extent_tree *et) +{ + struct ocfs2_xattr_block *xb = et->et_object; + + et->et_root_el = &xb->xb_attrs.xb_root.xt_list; +} + +static void ocfs2_xattr_tree_fill_max_leaf_clusters(struct ocfs2_extent_tree *et) +{ + struct super_block *sb = ocfs2_metadata_cache_get_super(et->et_ci); + et->et_max_leaf_clusters = + ocfs2_clusters_for_bytes(sb, OCFS2_MAX_XATTR_TREE_LEAF_SIZE); +} + +static void ocfs2_xattr_tree_set_last_eb_blk(struct ocfs2_extent_tree *et, + u64 blkno) +{ + struct ocfs2_xattr_block *xb = et->et_object; + struct ocfs2_xattr_tree_root *xt = &xb->xb_attrs.xb_root; + + xt->xt_last_eb_blk = cpu_to_le64(blkno); +} + +static u64 ocfs2_xattr_tree_get_last_eb_blk(struct ocfs2_extent_tree *et) +{ + struct ocfs2_xattr_block *xb = et->et_object; + struct ocfs2_xattr_tree_root *xt = &xb->xb_attrs.xb_root; + + return le64_to_cpu(xt->xt_last_eb_blk); +} + +static void ocfs2_xattr_tree_update_clusters(struct ocfs2_extent_tree *et, + u32 clusters) +{ + struct ocfs2_xattr_block *xb = et->et_object; + + le32_add_cpu(&xb->xb_attrs.xb_root.xt_clusters, clusters); +} + +static const struct ocfs2_extent_tree_operations ocfs2_xattr_tree_et_ops = { + .eo_set_last_eb_blk = ocfs2_xattr_tree_set_last_eb_blk, + .eo_get_last_eb_blk = ocfs2_xattr_tree_get_last_eb_blk, + .eo_update_clusters = ocfs2_xattr_tree_update_clusters, + .eo_fill_root_el = ocfs2_xattr_tree_fill_root_el, + .eo_fill_max_leaf_clusters = ocfs2_xattr_tree_fill_max_leaf_clusters, +}; + +static void ocfs2_dx_root_set_last_eb_blk(struct ocfs2_extent_tree *et, + u64 blkno) +{ + struct ocfs2_dx_root_block *dx_root = et->et_object; + + dx_root->dr_last_eb_blk = cpu_to_le64(blkno); +} + +static u64 ocfs2_dx_root_get_last_eb_blk(struct ocfs2_extent_tree *et) +{ + struct ocfs2_dx_root_block *dx_root = et->et_object; + + return le64_to_cpu(dx_root->dr_last_eb_blk); +} + +static void ocfs2_dx_root_update_clusters(struct ocfs2_extent_tree *et, + u32 clusters) +{ + struct ocfs2_dx_root_block *dx_root = et->et_object; + + le32_add_cpu(&dx_root->dr_clusters, clusters); +} + +static int ocfs2_dx_root_sanity_check(struct ocfs2_extent_tree *et) +{ + struct ocfs2_dx_root_block *dx_root = et->et_object; + + BUG_ON(!OCFS2_IS_VALID_DX_ROOT(dx_root)); + + return 0; +} + +static void ocfs2_dx_root_fill_root_el(struct ocfs2_extent_tree *et) +{ + struct ocfs2_dx_root_block *dx_root = et->et_object; + + et->et_root_el = &dx_root->dr_list; +} + +static const struct ocfs2_extent_tree_operations ocfs2_dx_root_et_ops = { + .eo_set_last_eb_blk = ocfs2_dx_root_set_last_eb_blk, + .eo_get_last_eb_blk = ocfs2_dx_root_get_last_eb_blk, + .eo_update_clusters = ocfs2_dx_root_update_clusters, + .eo_sanity_check = ocfs2_dx_root_sanity_check, + .eo_fill_root_el = ocfs2_dx_root_fill_root_el, +}; + +static void ocfs2_refcount_tree_fill_root_el(struct ocfs2_extent_tree *et) +{ + struct ocfs2_refcount_block *rb = et->et_object; + + et->et_root_el = &rb->rf_list; +} + +static void ocfs2_refcount_tree_set_last_eb_blk(struct ocfs2_extent_tree *et, + u64 blkno) +{ + struct ocfs2_refcount_block *rb = et->et_object; + + rb->rf_last_eb_blk = cpu_to_le64(blkno); +} + +static u64 ocfs2_refcount_tree_get_last_eb_blk(struct ocfs2_extent_tree *et) +{ + struct ocfs2_refcount_block *rb = et->et_object; + + return le64_to_cpu(rb->rf_last_eb_blk); +} + +static void ocfs2_refcount_tree_update_clusters(struct ocfs2_extent_tree *et, + u32 clusters) +{ + struct ocfs2_refcount_block *rb = et->et_object; + + le32_add_cpu(&rb->rf_clusters, clusters); +} + +static enum ocfs2_contig_type +ocfs2_refcount_tree_extent_contig(struct ocfs2_extent_tree *et, + struct ocfs2_extent_rec *ext, + struct ocfs2_extent_rec *insert_rec) +{ + return CONTIG_NONE; +} + +static const struct ocfs2_extent_tree_operations ocfs2_refcount_tree_et_ops = { + .eo_set_last_eb_blk = ocfs2_refcount_tree_set_last_eb_blk, + .eo_get_last_eb_blk = ocfs2_refcount_tree_get_last_eb_blk, + .eo_update_clusters = ocfs2_refcount_tree_update_clusters, + .eo_fill_root_el = ocfs2_refcount_tree_fill_root_el, + .eo_extent_contig = ocfs2_refcount_tree_extent_contig, +}; + +static void __ocfs2_init_extent_tree(struct ocfs2_extent_tree *et, + struct ocfs2_caching_info *ci, + struct buffer_head *bh, + ocfs2_journal_access_func access, + void *obj, + const struct ocfs2_extent_tree_operations *ops) +{ + et->et_ops = ops; + et->et_root_bh = bh; + et->et_ci = ci; + et->et_root_journal_access = access; + if (!obj) + obj = (void *)bh->b_data; + et->et_object = obj; + et->et_dealloc = NULL; + + et->et_ops->eo_fill_root_el(et); + if (!et->et_ops->eo_fill_max_leaf_clusters) + et->et_max_leaf_clusters = 0; + else + et->et_ops->eo_fill_max_leaf_clusters(et); +} + +void ocfs2_init_dinode_extent_tree(struct ocfs2_extent_tree *et, + struct ocfs2_caching_info *ci, + struct buffer_head *bh) +{ + __ocfs2_init_extent_tree(et, ci, bh, ocfs2_journal_access_di, + NULL, &ocfs2_dinode_et_ops); +} + +void ocfs2_init_xattr_tree_extent_tree(struct ocfs2_extent_tree *et, + struct ocfs2_caching_info *ci, + struct buffer_head *bh) +{ + __ocfs2_init_extent_tree(et, ci, bh, ocfs2_journal_access_xb, + NULL, &ocfs2_xattr_tree_et_ops); +} + +void ocfs2_init_xattr_value_extent_tree(struct ocfs2_extent_tree *et, + struct ocfs2_caching_info *ci, + struct ocfs2_xattr_value_buf *vb) +{ + __ocfs2_init_extent_tree(et, ci, vb->vb_bh, vb->vb_access, vb, + &ocfs2_xattr_value_et_ops); +} + +void ocfs2_init_dx_root_extent_tree(struct ocfs2_extent_tree *et, + struct ocfs2_caching_info *ci, + struct buffer_head *bh) +{ + __ocfs2_init_extent_tree(et, ci, bh, ocfs2_journal_access_dr, + NULL, &ocfs2_dx_root_et_ops); +} + +void ocfs2_init_refcount_extent_tree(struct ocfs2_extent_tree *et, + struct ocfs2_caching_info *ci, + struct buffer_head *bh) +{ + __ocfs2_init_extent_tree(et, ci, bh, ocfs2_journal_access_rb, + NULL, &ocfs2_refcount_tree_et_ops); +} + +static inline void ocfs2_et_set_last_eb_blk(struct ocfs2_extent_tree *et, + u64 new_last_eb_blk) +{ + et->et_ops->eo_set_last_eb_blk(et, new_last_eb_blk); +} + +static inline u64 ocfs2_et_get_last_eb_blk(struct ocfs2_extent_tree *et) +{ + return et->et_ops->eo_get_last_eb_blk(et); +} + +static inline void ocfs2_et_update_clusters(struct ocfs2_extent_tree *et, + u32 clusters) +{ + et->et_ops->eo_update_clusters(et, clusters); +} + +static inline void ocfs2_et_extent_map_insert(struct ocfs2_extent_tree *et, + struct ocfs2_extent_rec *rec) +{ + if (et->et_ops->eo_extent_map_insert) + et->et_ops->eo_extent_map_insert(et, rec); +} + +static inline void ocfs2_et_extent_map_truncate(struct ocfs2_extent_tree *et, + u32 clusters) +{ + if (et->et_ops->eo_extent_map_truncate) + et->et_ops->eo_extent_map_truncate(et, clusters); +} + +static inline int ocfs2_et_root_journal_access(handle_t *handle, + struct ocfs2_extent_tree *et, + int type) +{ + return et->et_root_journal_access(handle, et->et_ci, et->et_root_bh, + type); +} + +static inline enum ocfs2_contig_type + ocfs2_et_extent_contig(struct ocfs2_extent_tree *et, + struct ocfs2_extent_rec *rec, + struct ocfs2_extent_rec *insert_rec) +{ + if (et->et_ops->eo_extent_contig) + return et->et_ops->eo_extent_contig(et, rec, insert_rec); + + return ocfs2_extent_rec_contig( + ocfs2_metadata_cache_get_super(et->et_ci), + rec, insert_rec); +} + +static inline int ocfs2_et_insert_check(struct ocfs2_extent_tree *et, + struct ocfs2_extent_rec *rec) +{ + int ret = 0; + + if (et->et_ops->eo_insert_check) + ret = et->et_ops->eo_insert_check(et, rec); + return ret; +} + +static inline int ocfs2_et_sanity_check(struct ocfs2_extent_tree *et) +{ + int ret = 0; + + if (et->et_ops->eo_sanity_check) + ret = et->et_ops->eo_sanity_check(et); + return ret; +} + +static int ocfs2_cache_extent_block_free(struct ocfs2_cached_dealloc_ctxt *ctxt, + struct ocfs2_extent_block *eb); +static void ocfs2_adjust_rightmost_records(handle_t *handle, + struct ocfs2_extent_tree *et, + struct ocfs2_path *path, + struct ocfs2_extent_rec *insert_rec); +/* + * Reset the actual path elements so that we can reuse the structure + * to build another path. Generally, this involves freeing the buffer + * heads. + */ +void ocfs2_reinit_path(struct ocfs2_path *path, int keep_root) +{ + int i, start = 0, depth = 0; + struct ocfs2_path_item *node; + + if (keep_root) + start = 1; + + for(i = start; i < path_num_items(path); i++) { + node = &path->p_node[i]; + + brelse(node->bh); + node->bh = NULL; + node->el = NULL; + } + + /* + * Tree depth may change during truncate, or insert. If we're + * keeping the root extent list, then make sure that our path + * structure reflects the proper depth. + */ + if (keep_root) + depth = le16_to_cpu(path_root_el(path)->l_tree_depth); + else + path_root_access(path) = NULL; + + path->p_tree_depth = depth; +} + +void ocfs2_free_path(struct ocfs2_path *path) +{ + if (path) { + ocfs2_reinit_path(path, 0); + kfree(path); + } +} + +/* + * All the elements of src into dest. After this call, src could be freed + * without affecting dest. + * + * Both paths should have the same root. Any non-root elements of dest + * will be freed. + */ +static void ocfs2_cp_path(struct ocfs2_path *dest, struct ocfs2_path *src) +{ + int i; + + BUG_ON(path_root_bh(dest) != path_root_bh(src)); + BUG_ON(path_root_el(dest) != path_root_el(src)); + BUG_ON(path_root_access(dest) != path_root_access(src)); + + ocfs2_reinit_path(dest, 1); + + for(i = 1; i < OCFS2_MAX_PATH_DEPTH; i++) { + dest->p_node[i].bh = src->p_node[i].bh; + dest->p_node[i].el = src->p_node[i].el; + + if (dest->p_node[i].bh) + get_bh(dest->p_node[i].bh); + } +} + +/* + * Make the *dest path the same as src and re-initialize src path to + * have a root only. + */ +static void ocfs2_mv_path(struct ocfs2_path *dest, struct ocfs2_path *src) +{ + int i; + + BUG_ON(path_root_bh(dest) != path_root_bh(src)); + BUG_ON(path_root_access(dest) != path_root_access(src)); + + for(i = 1; i < OCFS2_MAX_PATH_DEPTH; i++) { + brelse(dest->p_node[i].bh); + + dest->p_node[i].bh = src->p_node[i].bh; + dest->p_node[i].el = src->p_node[i].el; + + src->p_node[i].bh = NULL; + src->p_node[i].el = NULL; + } +} + +/* + * Insert an extent block at given index. + * + * This will not take an additional reference on eb_bh. + */ +static inline void ocfs2_path_insert_eb(struct ocfs2_path *path, int index, + struct buffer_head *eb_bh) +{ + struct ocfs2_extent_block *eb = (struct ocfs2_extent_block *)eb_bh->b_data; + + /* + * Right now, no root bh is an extent block, so this helps + * catch code errors with dinode trees. The assertion can be + * safely removed if we ever need to insert extent block + * structures at the root. + */ + BUG_ON(index == 0); + + path->p_node[index].bh = eb_bh; + path->p_node[index].el = &eb->h_list; +} + +static struct ocfs2_path *ocfs2_new_path(struct buffer_head *root_bh, + struct ocfs2_extent_list *root_el, + ocfs2_journal_access_func access) +{ + struct ocfs2_path *path; + + BUG_ON(le16_to_cpu(root_el->l_tree_depth) >= OCFS2_MAX_PATH_DEPTH); + + path = kzalloc_obj(*path, GFP_NOFS); + if (path) { + path->p_tree_depth = le16_to_cpu(root_el->l_tree_depth); + get_bh(root_bh); + path_root_bh(path) = root_bh; + path_root_el(path) = root_el; + path_root_access(path) = access; + } + + return path; +} + +struct ocfs2_path *ocfs2_new_path_from_path(struct ocfs2_path *path) +{ + return ocfs2_new_path(path_root_bh(path), path_root_el(path), + path_root_access(path)); +} + +struct ocfs2_path *ocfs2_new_path_from_et(struct ocfs2_extent_tree *et) +{ + return ocfs2_new_path(et->et_root_bh, et->et_root_el, + et->et_root_journal_access); +} + +/* + * Journal the buffer at depth idx. All idx>0 are extent_blocks, + * otherwise it's the root_access function. + * + * I don't like the way this function's name looks next to + * ocfs2_journal_access_path(), but I don't have a better one. + */ +int ocfs2_path_bh_journal_access(handle_t *handle, + struct ocfs2_caching_info *ci, + struct ocfs2_path *path, + int idx) +{ + ocfs2_journal_access_func access = path_root_access(path); + + if (!access) + access = ocfs2_journal_access; + + if (idx) + access = ocfs2_journal_access_eb; + + return access(handle, ci, path->p_node[idx].bh, + OCFS2_JOURNAL_ACCESS_WRITE); +} + +/* + * Convenience function to journal all components in a path. + */ +int ocfs2_journal_access_path(struct ocfs2_caching_info *ci, + handle_t *handle, + struct ocfs2_path *path) +{ + int i, ret = 0; + + if (!path) + goto out; + + for(i = 0; i < path_num_items(path); i++) { + ret = ocfs2_path_bh_journal_access(handle, ci, path, i); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + } + +out: + return ret; +} + +/* + * Return the index of the extent record which contains cluster #v_cluster. + * -1 is returned if it was not found. + * + * Should work fine on interior and exterior nodes. + */ +int ocfs2_search_extent_list(struct ocfs2_extent_list *el, u32 v_cluster) +{ + int ret = -1; + int i; + struct ocfs2_extent_rec *rec; + u32 rec_end, rec_start, clusters; + + for(i = 0; i < le16_to_cpu(el->l_next_free_rec); i++) { + rec = &el->l_recs[i]; + + rec_start = le32_to_cpu(rec->e_cpos); + clusters = ocfs2_rec_clusters(el, rec); + + rec_end = rec_start + clusters; + + if (v_cluster >= rec_start && v_cluster < rec_end) { + ret = i; + break; + } + } + + return ret; +} + +/* + * NOTE: ocfs2_block_extent_contig(), ocfs2_extents_adjacent() and + * ocfs2_extent_rec_contig only work properly against leaf nodes! + */ +static int ocfs2_block_extent_contig(struct super_block *sb, + struct ocfs2_extent_rec *ext, + u64 blkno) +{ + u64 blk_end = le64_to_cpu(ext->e_blkno); + + blk_end += ocfs2_clusters_to_blocks(sb, + le16_to_cpu(ext->e_leaf_clusters)); + + return blkno == blk_end; +} + +static int ocfs2_extents_adjacent(struct ocfs2_extent_rec *left, + struct ocfs2_extent_rec *right) +{ + u32 left_range; + + left_range = le32_to_cpu(left->e_cpos) + + le16_to_cpu(left->e_leaf_clusters); + + return (left_range == le32_to_cpu(right->e_cpos)); +} + +static enum ocfs2_contig_type + ocfs2_extent_rec_contig(struct super_block *sb, + struct ocfs2_extent_rec *ext, + struct ocfs2_extent_rec *insert_rec) +{ + u64 blkno = le64_to_cpu(insert_rec->e_blkno); + + /* + * Refuse to coalesce extent records with different flag + * fields - we don't want to mix unwritten extents with user + * data. + */ + if (ext->e_flags != insert_rec->e_flags) + return CONTIG_NONE; + + if (ocfs2_extents_adjacent(ext, insert_rec) && + ocfs2_block_extent_contig(sb, ext, blkno)) + return CONTIG_RIGHT; + + blkno = le64_to_cpu(ext->e_blkno); + if (ocfs2_extents_adjacent(insert_rec, ext) && + ocfs2_block_extent_contig(sb, insert_rec, blkno)) + return CONTIG_LEFT; + + return CONTIG_NONE; +} + +/* + * NOTE: We can have pretty much any combination of contiguousness and + * appending. + * + * The usefulness of APPEND_TAIL is more in that it lets us know that + * we'll have to update the path to that leaf. + */ +enum ocfs2_append_type { + APPEND_NONE = 0, + APPEND_TAIL, +}; + +enum ocfs2_split_type { + SPLIT_NONE = 0, + SPLIT_LEFT, + SPLIT_RIGHT, +}; + +struct ocfs2_insert_type { + enum ocfs2_split_type ins_split; + enum ocfs2_append_type ins_appending; + enum ocfs2_contig_type ins_contig; + int ins_contig_index; + int ins_tree_depth; +}; + +struct ocfs2_merge_ctxt { + enum ocfs2_contig_type c_contig_type; + int c_has_empty_extent; + int c_split_covers_rec; +}; + +static int ocfs2_validate_extent_block(struct super_block *sb, + struct buffer_head *bh) +{ + int rc; + struct ocfs2_extent_block *eb = + (struct ocfs2_extent_block *)bh->b_data; + + trace_ocfs2_validate_extent_block((unsigned long long)bh->b_blocknr); + + BUG_ON(!buffer_uptodate(bh)); + + /* + * If the ecc fails, we return the error but otherwise + * leave the filesystem running. We know any error is + * local to this block. + */ + rc = ocfs2_validate_meta_ecc(sb, bh->b_data, &eb->h_check); + if (rc) { + mlog(ML_ERROR, "Checksum failed for extent block %llu\n", + (unsigned long long)bh->b_blocknr); + return rc; + } + + /* + * Errors after here are fatal. + */ + + if (!OCFS2_IS_VALID_EXTENT_BLOCK(eb)) { + rc = ocfs2_error(sb, + "Extent block #%llu has bad signature %.*s\n", + (unsigned long long)bh->b_blocknr, 7, + eb->h_signature); + goto bail; + } + + if (le64_to_cpu(eb->h_blkno) != bh->b_blocknr) { + rc = ocfs2_error(sb, + "Extent block #%llu has an invalid h_blkno of %llu\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(eb->h_blkno)); + goto bail; + } + + if (le32_to_cpu(eb->h_fs_generation) != OCFS2_SB(sb)->fs_generation) { + rc = ocfs2_error(sb, + "Extent block #%llu has an invalid h_fs_generation of #%u\n", + (unsigned long long)bh->b_blocknr, + le32_to_cpu(eb->h_fs_generation)); + goto bail; + } + + if (le16_to_cpu(eb->h_list.l_count) != ocfs2_extent_recs_per_eb(sb)) { + rc = ocfs2_error(sb, + "Extent block #%llu has invalid l_count %u (expected %u)\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(eb->h_list.l_count), + ocfs2_extent_recs_per_eb(sb)); + goto bail; + } + + if (le16_to_cpu(eb->h_list.l_next_free_rec) > le16_to_cpu(eb->h_list.l_count)) { + rc = ocfs2_error(sb, + "Extent block #%llu has invalid l_next_free_rec %u (l_count %u)\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(eb->h_list.l_next_free_rec), + le16_to_cpu(eb->h_list.l_count)); + goto bail; + } + +bail: + return rc; +} + +int ocfs2_read_extent_block(struct ocfs2_caching_info *ci, u64 eb_blkno, + struct buffer_head **bh) +{ + int rc; + struct buffer_head *tmp = *bh; + + rc = ocfs2_read_block(ci, eb_blkno, &tmp, + ocfs2_validate_extent_block); + + /* If ocfs2_read_block() got us a new bh, pass it up. */ + if (!rc && !*bh) + *bh = tmp; + + return rc; +} + + +/* + * How many free extents have we got before we need more meta data? + */ +int ocfs2_num_free_extents(struct ocfs2_extent_tree *et) +{ + int retval; + struct ocfs2_extent_list *el = NULL; + struct ocfs2_extent_block *eb; + struct buffer_head *eb_bh = NULL; + u64 last_eb_blk = 0; + + el = et->et_root_el; + last_eb_blk = ocfs2_et_get_last_eb_blk(et); + + if (last_eb_blk) { + retval = ocfs2_read_extent_block(et->et_ci, last_eb_blk, + &eb_bh); + if (retval < 0) { + mlog_errno(retval); + goto bail; + } + eb = (struct ocfs2_extent_block *) eb_bh->b_data; + el = &eb->h_list; + } + + if (el->l_tree_depth != 0) { + retval = ocfs2_error(ocfs2_metadata_cache_get_super(et->et_ci), + "Owner %llu has leaf extent block %llu with an invalid l_tree_depth of %u\n", + (unsigned long long)ocfs2_metadata_cache_owner(et->et_ci), + (unsigned long long)last_eb_blk, + le16_to_cpu(el->l_tree_depth)); + goto bail; + } + + retval = le16_to_cpu(el->l_count) - le16_to_cpu(el->l_next_free_rec); +bail: + brelse(eb_bh); + + trace_ocfs2_num_free_extents(retval); + return retval; +} + +/* expects array to already be allocated + * + * sets h_signature, h_blkno, h_suballoc_bit, h_suballoc_slot, and + * l_count for you + */ +static int ocfs2_create_new_meta_bhs(handle_t *handle, + struct ocfs2_extent_tree *et, + int wanted, + struct ocfs2_alloc_context *meta_ac, + struct buffer_head *bhs[]) +{ + int count, status, i; + u16 suballoc_bit_start; + u32 num_got; + u64 suballoc_loc, first_blkno; + struct ocfs2_super *osb = + OCFS2_SB(ocfs2_metadata_cache_get_super(et->et_ci)); + struct ocfs2_extent_block *eb; + + count = 0; + while (count < wanted) { + status = ocfs2_claim_metadata(handle, + meta_ac, + wanted - count, + &suballoc_loc, + &suballoc_bit_start, + &num_got, + &first_blkno); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + for(i = count; i < (num_got + count); i++) { + bhs[i] = sb_getblk(osb->sb, first_blkno); + if (bhs[i] == NULL) { + status = -ENOMEM; + mlog_errno(status); + goto bail; + } + ocfs2_set_new_buffer_uptodate(et->et_ci, bhs[i]); + + status = ocfs2_journal_access_eb(handle, et->et_ci, + bhs[i], + OCFS2_JOURNAL_ACCESS_CREATE); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + memset(bhs[i]->b_data, 0, osb->sb->s_blocksize); + eb = (struct ocfs2_extent_block *) bhs[i]->b_data; + /* Ok, setup the minimal stuff here. */ + strscpy(eb->h_signature, OCFS2_EXTENT_BLOCK_SIGNATURE); + eb->h_blkno = cpu_to_le64(first_blkno); + eb->h_fs_generation = cpu_to_le32(osb->fs_generation); + eb->h_suballoc_slot = + cpu_to_le16(meta_ac->ac_alloc_slot); + eb->h_suballoc_loc = cpu_to_le64(suballoc_loc); + eb->h_suballoc_bit = cpu_to_le16(suballoc_bit_start); + eb->h_list.l_count = + cpu_to_le16(ocfs2_extent_recs_per_eb(osb->sb)); + + suballoc_bit_start++; + first_blkno++; + + /* We'll also be dirtied by the caller, so + * this isn't absolutely necessary. */ + ocfs2_journal_dirty(handle, bhs[i]); + } + + count += num_got; + } + + status = 0; +bail: + if (status < 0) { + for(i = 0; i < wanted; i++) { + brelse(bhs[i]); + bhs[i] = NULL; + } + } + return status; +} + +/* + * Helper function for ocfs2_add_branch() and ocfs2_shift_tree_depth(). + * + * Returns the sum of the rightmost extent rec logical offset and + * cluster count. + * + * ocfs2_add_branch() uses this to determine what logical cluster + * value should be populated into the leftmost new branch records. + * + * ocfs2_shift_tree_depth() uses this to determine the # clusters + * value for the new topmost tree record. + */ +static inline u32 ocfs2_sum_rightmost_rec(struct ocfs2_extent_list *el) +{ + int i; + + i = le16_to_cpu(el->l_next_free_rec) - 1; + + return le32_to_cpu(el->l_recs[i].e_cpos) + + ocfs2_rec_clusters(el, &el->l_recs[i]); +} + +/* + * Change range of the branches in the right most path according to the leaf + * extent block's rightmost record. + */ +static int ocfs2_adjust_rightmost_branch(handle_t *handle, + struct ocfs2_extent_tree *et) +{ + int status; + struct ocfs2_path *path = NULL; + struct ocfs2_extent_list *el; + struct ocfs2_extent_rec *rec; + + path = ocfs2_new_path_from_et(et); + if (!path) { + status = -ENOMEM; + return status; + } + + status = ocfs2_find_path(et->et_ci, path, UINT_MAX); + if (status < 0) { + mlog_errno(status); + goto out; + } + + status = ocfs2_extend_trans(handle, path_num_items(path)); + if (status < 0) { + mlog_errno(status); + goto out; + } + + status = ocfs2_journal_access_path(et->et_ci, handle, path); + if (status < 0) { + mlog_errno(status); + goto out; + } + + el = path_leaf_el(path); + rec = &el->l_recs[le16_to_cpu(el->l_next_free_rec) - 1]; + + ocfs2_adjust_rightmost_records(handle, et, path, rec); + +out: + ocfs2_free_path(path); + return status; +} + +/* + * Add an entire tree branch to our inode. eb_bh is the extent block + * to start at, if we don't want to start the branch at the root + * structure. + * + * last_eb_bh is required as we have to update it's next_leaf pointer + * for the new last extent block. + * + * the new branch will be 'empty' in the sense that every block will + * contain a single record with cluster count == 0. + */ +static int ocfs2_add_branch(handle_t *handle, + struct ocfs2_extent_tree *et, + struct buffer_head *eb_bh, + struct buffer_head **last_eb_bh, + struct ocfs2_alloc_context *meta_ac) +{ + int status, new_blocks, i, block_given = 0; + u64 next_blkno, new_last_eb_blk; + struct buffer_head *bh; + struct buffer_head **new_eb_bhs = NULL; + struct ocfs2_extent_block *eb; + struct ocfs2_extent_list *eb_el; + struct ocfs2_extent_list *el; + u32 new_cpos, root_end; + + BUG_ON(!last_eb_bh || !*last_eb_bh); + + if (eb_bh) { + eb = (struct ocfs2_extent_block *) eb_bh->b_data; + el = &eb->h_list; + } else + el = et->et_root_el; + + /* we never add a branch to a leaf. */ + BUG_ON(!el->l_tree_depth); + + new_blocks = le16_to_cpu(el->l_tree_depth); + + eb = (struct ocfs2_extent_block *)(*last_eb_bh)->b_data; + new_cpos = ocfs2_sum_rightmost_rec(&eb->h_list); + root_end = ocfs2_sum_rightmost_rec(et->et_root_el); + + /* + * If there is a gap before the root end and the real end + * of the rightmost leaf block, we need to remove the gap + * between new_cpos and root_end first so that the tree + * is consistent after we add a new branch(it will start + * from new_cpos). + */ + if (root_end > new_cpos) { + trace_ocfs2_adjust_rightmost_branch( + (unsigned long long) + ocfs2_metadata_cache_owner(et->et_ci), + root_end, new_cpos); + + status = ocfs2_adjust_rightmost_branch(handle, et); + if (status) { + mlog_errno(status); + goto bail; + } + } + + /* allocate the number of new eb blocks we need */ + new_eb_bhs = kzalloc_objs(struct buffer_head *, new_blocks); + if (!new_eb_bhs) { + status = -ENOMEM; + mlog_errno(status); + goto bail; + } + + /* Firstyly, try to reuse dealloc since we have already estimated how + * many extent blocks we may use. + */ + if (!ocfs2_is_dealloc_empty(et)) { + status = ocfs2_reuse_blk_from_dealloc(handle, et, + new_eb_bhs, new_blocks, + &block_given); + if (status < 0) { + mlog_errno(status); + goto bail; + } + } + + BUG_ON(block_given > new_blocks); + + if (block_given < new_blocks) { + BUG_ON(!meta_ac); + status = ocfs2_create_new_meta_bhs(handle, et, + new_blocks - block_given, + meta_ac, + &new_eb_bhs[block_given]); + if (status < 0) { + mlog_errno(status); + goto bail; + } + } + + /* Note: new_eb_bhs[new_blocks - 1] is the guy which will be + * linked with the rest of the tree. + * conversely, new_eb_bhs[0] is the new bottommost leaf. + * + * when we leave the loop, new_last_eb_blk will point to the + * newest leaf, and next_blkno will point to the topmost extent + * block. */ + next_blkno = new_last_eb_blk = 0; + for(i = 0; i < new_blocks; i++) { + bh = new_eb_bhs[i]; + eb = (struct ocfs2_extent_block *) bh->b_data; + /* ocfs2_create_new_meta_bhs() should create it right! */ + BUG_ON(!OCFS2_IS_VALID_EXTENT_BLOCK(eb)); + eb_el = &eb->h_list; + + status = ocfs2_journal_access_eb(handle, et->et_ci, bh, + OCFS2_JOURNAL_ACCESS_CREATE); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + eb->h_next_leaf_blk = 0; + eb_el->l_tree_depth = cpu_to_le16(i); + eb_el->l_next_free_rec = cpu_to_le16(1); + /* + * This actually counts as an empty extent as + * c_clusters == 0 + */ + eb_el->l_recs[0].e_cpos = cpu_to_le32(new_cpos); + eb_el->l_recs[0].e_blkno = cpu_to_le64(next_blkno); + /* + * eb_el isn't always an interior node, but even leaf + * nodes want a zero'd flags and reserved field so + * this gets the whole 32 bits regardless of use. + */ + eb_el->l_recs[0].e_int_clusters = cpu_to_le32(0); + if (!eb_el->l_tree_depth) + new_last_eb_blk = le64_to_cpu(eb->h_blkno); + + ocfs2_journal_dirty(handle, bh); + next_blkno = le64_to_cpu(eb->h_blkno); + } + + /* This is a bit hairy. We want to update up to three blocks + * here without leaving any of them in an inconsistent state + * in case of error. We don't have to worry about + * journal_dirty erroring as it won't unless we've aborted the + * handle (in which case we would never be here) so reserving + * the write with journal_access is all we need to do. */ + status = ocfs2_journal_access_eb(handle, et->et_ci, *last_eb_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto bail; + } + status = ocfs2_et_root_journal_access(handle, et, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto bail; + } + if (eb_bh) { + status = ocfs2_journal_access_eb(handle, et->et_ci, eb_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto bail; + } + } + + /* Link the new branch into the rest of the tree (el will + * either be on the root_bh, or the extent block passed in. */ + i = le16_to_cpu(el->l_next_free_rec); + el->l_recs[i].e_blkno = cpu_to_le64(next_blkno); + el->l_recs[i].e_cpos = cpu_to_le32(new_cpos); + el->l_recs[i].e_int_clusters = 0; + le16_add_cpu(&el->l_next_free_rec, 1); + + /* fe needs a new last extent block pointer, as does the + * next_leaf on the previously last-extent-block. */ + ocfs2_et_set_last_eb_blk(et, new_last_eb_blk); + + eb = (struct ocfs2_extent_block *) (*last_eb_bh)->b_data; + eb->h_next_leaf_blk = cpu_to_le64(new_last_eb_blk); + + ocfs2_journal_dirty(handle, *last_eb_bh); + ocfs2_journal_dirty(handle, et->et_root_bh); + if (eb_bh) + ocfs2_journal_dirty(handle, eb_bh); + + /* + * Some callers want to track the rightmost leaf so pass it + * back here. + */ + brelse(*last_eb_bh); + get_bh(new_eb_bhs[0]); + *last_eb_bh = new_eb_bhs[0]; + + status = 0; +bail: + if (new_eb_bhs) { + for (i = 0; i < new_blocks; i++) + brelse(new_eb_bhs[i]); + kfree(new_eb_bhs); + } + + return status; +} + +/* + * adds another level to the allocation tree. + * returns back the new extent block so you can add a branch to it + * after this call. + */ +static int ocfs2_shift_tree_depth(handle_t *handle, + struct ocfs2_extent_tree *et, + struct ocfs2_alloc_context *meta_ac, + struct buffer_head **ret_new_eb_bh) +{ + int status, i, block_given = 0; + u32 new_clusters; + struct buffer_head *new_eb_bh = NULL; + struct ocfs2_extent_block *eb; + struct ocfs2_extent_list *root_el; + struct ocfs2_extent_list *eb_el; + + if (!ocfs2_is_dealloc_empty(et)) { + status = ocfs2_reuse_blk_from_dealloc(handle, et, + &new_eb_bh, 1, + &block_given); + } else if (meta_ac) { + status = ocfs2_create_new_meta_bhs(handle, et, 1, meta_ac, + &new_eb_bh); + + } else { + BUG(); + } + + if (status < 0) { + mlog_errno(status); + goto bail; + } + + eb = (struct ocfs2_extent_block *) new_eb_bh->b_data; + /* ocfs2_create_new_meta_bhs() should create it right! */ + BUG_ON(!OCFS2_IS_VALID_EXTENT_BLOCK(eb)); + + eb_el = &eb->h_list; + root_el = et->et_root_el; + + status = ocfs2_journal_access_eb(handle, et->et_ci, new_eb_bh, + OCFS2_JOURNAL_ACCESS_CREATE); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + /* copy the root extent list data into the new extent block */ + eb_el->l_tree_depth = root_el->l_tree_depth; + eb_el->l_next_free_rec = root_el->l_next_free_rec; + for (i = 0; i < le16_to_cpu(root_el->l_next_free_rec); i++) + eb_el->l_recs[i] = root_el->l_recs[i]; + + ocfs2_journal_dirty(handle, new_eb_bh); + + status = ocfs2_et_root_journal_access(handle, et, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + new_clusters = ocfs2_sum_rightmost_rec(eb_el); + + /* update root_bh now */ + le16_add_cpu(&root_el->l_tree_depth, 1); + root_el->l_recs[0].e_cpos = 0; + root_el->l_recs[0].e_blkno = eb->h_blkno; + root_el->l_recs[0].e_int_clusters = cpu_to_le32(new_clusters); + for (i = 1; i < le16_to_cpu(root_el->l_next_free_rec); i++) + memset(&root_el->l_recs[i], 0, sizeof(struct ocfs2_extent_rec)); + root_el->l_next_free_rec = cpu_to_le16(1); + + /* If this is our 1st tree depth shift, then last_eb_blk + * becomes the allocated extent block */ + if (root_el->l_tree_depth == cpu_to_le16(1)) + ocfs2_et_set_last_eb_blk(et, le64_to_cpu(eb->h_blkno)); + + ocfs2_journal_dirty(handle, et->et_root_bh); + + *ret_new_eb_bh = new_eb_bh; + new_eb_bh = NULL; + status = 0; +bail: + brelse(new_eb_bh); + + return status; +} + +/* + * Should only be called when there is no space left in any of the + * leaf nodes. What we want to do is find the lowest tree depth + * non-leaf extent block with room for new records. There are three + * valid results of this search: + * + * 1) a lowest extent block is found, then we pass it back in + * *lowest_eb_bh and return '0' + * + * 2) the search fails to find anything, but the root_el has room. We + * pass NULL back in *lowest_eb_bh, but still return '0' + * + * 3) the search fails to find anything AND the root_el is full, in + * which case we return > 0 + * + * return status < 0 indicates an error. + */ +static int ocfs2_find_branch_target(struct ocfs2_extent_tree *et, + struct buffer_head **target_bh) +{ + int status = 0, i; + u64 blkno; + struct ocfs2_extent_block *eb; + struct ocfs2_extent_list *el; + struct buffer_head *bh = NULL; + struct buffer_head *lowest_bh = NULL; + + *target_bh = NULL; + + el = et->et_root_el; + + while(le16_to_cpu(el->l_tree_depth) > 1) { + if (le16_to_cpu(el->l_next_free_rec) == 0) { + status = ocfs2_error(ocfs2_metadata_cache_get_super(et->et_ci), + "Owner %llu has empty extent list (next_free_rec == 0)\n", + (unsigned long long)ocfs2_metadata_cache_owner(et->et_ci)); + goto bail; + } + i = le16_to_cpu(el->l_next_free_rec) - 1; + blkno = le64_to_cpu(el->l_recs[i].e_blkno); + if (!blkno) { + status = ocfs2_error(ocfs2_metadata_cache_get_super(et->et_ci), + "Owner %llu has extent list where extent # %d has no physical block start\n", + (unsigned long long)ocfs2_metadata_cache_owner(et->et_ci), i); + goto bail; + } + + brelse(bh); + bh = NULL; + + status = ocfs2_read_extent_block(et->et_ci, blkno, &bh); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + eb = (struct ocfs2_extent_block *) bh->b_data; + el = &eb->h_list; + + if (le16_to_cpu(el->l_next_free_rec) < + le16_to_cpu(el->l_count)) { + brelse(lowest_bh); + lowest_bh = bh; + get_bh(lowest_bh); + } + } + + /* If we didn't find one and the fe doesn't have any room, + * then return '1' */ + el = et->et_root_el; + if (!lowest_bh && (el->l_next_free_rec == el->l_count)) + status = 1; + + *target_bh = lowest_bh; +bail: + brelse(bh); + + return status; +} + +/* + * Grow a b-tree so that it has more records. + * + * We might shift the tree depth in which case existing paths should + * be considered invalid. + * + * Tree depth after the grow is returned via *final_depth. + * + * *last_eb_bh will be updated by ocfs2_add_branch(). + */ +static int ocfs2_grow_tree(handle_t *handle, struct ocfs2_extent_tree *et, + int *final_depth, struct buffer_head **last_eb_bh, + struct ocfs2_alloc_context *meta_ac) +{ + int ret, shift; + struct ocfs2_extent_list *el = et->et_root_el; + int depth = le16_to_cpu(el->l_tree_depth); + struct buffer_head *bh = NULL; + + BUG_ON(meta_ac == NULL && ocfs2_is_dealloc_empty(et)); + + shift = ocfs2_find_branch_target(et, &bh); + if (shift < 0) { + ret = shift; + mlog_errno(ret); + goto out; + } + + /* We traveled all the way to the bottom of the allocation tree + * and didn't find room for any more extents - we need to add + * another tree level */ + if (shift) { + BUG_ON(bh); + trace_ocfs2_grow_tree( + (unsigned long long) + ocfs2_metadata_cache_owner(et->et_ci), + depth); + + /* ocfs2_shift_tree_depth will return us a buffer with + * the new extent block (so we can pass that to + * ocfs2_add_branch). */ + ret = ocfs2_shift_tree_depth(handle, et, meta_ac, &bh); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + depth++; + if (depth == 1) { + /* + * Special case: we have room now if we shifted from + * tree_depth 0, so no more work needs to be done. + * + * We won't be calling add_branch, so pass + * back *last_eb_bh as the new leaf. At depth + * zero, it should always be null so there's + * no reason to brelse. + */ + BUG_ON(*last_eb_bh); + get_bh(bh); + *last_eb_bh = bh; + goto out; + } + } + + /* call ocfs2_add_branch to add the final part of the tree with + * the new data. */ + ret = ocfs2_add_branch(handle, et, bh, last_eb_bh, + meta_ac); + if (ret < 0) + mlog_errno(ret); + +out: + if (final_depth) + *final_depth = depth; + brelse(bh); + return ret; +} + +/* + * This function will discard the rightmost extent record. + */ +static void ocfs2_shift_records_right(struct ocfs2_extent_list *el) +{ + int next_free = le16_to_cpu(el->l_next_free_rec); + int count = le16_to_cpu(el->l_count); + unsigned int num_bytes; + + BUG_ON(!next_free); + /* This will cause us to go off the end of our extent list. */ + BUG_ON(next_free >= count); + + num_bytes = sizeof(struct ocfs2_extent_rec) * next_free; + + memmove(&el->l_recs[1], &el->l_recs[0], num_bytes); +} + +static void ocfs2_rotate_leaf(struct ocfs2_extent_list *el, + struct ocfs2_extent_rec *insert_rec) +{ + int i, insert_index, next_free, has_empty, num_bytes; + u32 insert_cpos = le32_to_cpu(insert_rec->e_cpos); + struct ocfs2_extent_rec *rec; + + next_free = le16_to_cpu(el->l_next_free_rec); + has_empty = ocfs2_is_empty_extent(&el->l_recs[0]); + + BUG_ON(!next_free); + + /* The tree code before us didn't allow enough room in the leaf. */ + BUG_ON(el->l_next_free_rec == el->l_count && !has_empty); + + /* + * The easiest way to approach this is to just remove the + * empty extent and temporarily decrement next_free. + */ + if (has_empty) { + /* + * If next_free was 1 (only an empty extent), this + * loop won't execute, which is fine. We still want + * the decrement above to happen. + */ + for(i = 0; i < (next_free - 1); i++) + el->l_recs[i] = el->l_recs[i+1]; + + next_free--; + } + + /* + * Figure out what the new record index should be. + */ + for(i = 0; i < next_free; i++) { + rec = &el->l_recs[i]; + + if (insert_cpos < le32_to_cpu(rec->e_cpos)) + break; + } + insert_index = i; + + trace_ocfs2_rotate_leaf(insert_cpos, insert_index, + has_empty, next_free, + le16_to_cpu(el->l_count)); + + BUG_ON(insert_index < 0); + BUG_ON(insert_index >= le16_to_cpu(el->l_count)); + BUG_ON(insert_index > next_free); + + /* + * No need to memmove if we're just adding to the tail. + */ + if (insert_index != next_free) { + BUG_ON(next_free >= le16_to_cpu(el->l_count)); + + num_bytes = next_free - insert_index; + num_bytes *= sizeof(struct ocfs2_extent_rec); + memmove(&el->l_recs[insert_index + 1], + &el->l_recs[insert_index], + num_bytes); + } + + /* + * Either we had an empty extent, and need to re-increment or + * there was no empty extent on a non full rightmost leaf node, + * in which case we still need to increment. + */ + next_free++; + el->l_next_free_rec = cpu_to_le16(next_free); + /* + * Make sure none of the math above just messed up our tree. + */ + BUG_ON(le16_to_cpu(el->l_next_free_rec) > le16_to_cpu(el->l_count)); + + el->l_recs[insert_index] = *insert_rec; + +} + +static void ocfs2_remove_empty_extent(struct ocfs2_extent_list *el) +{ + int size, num_recs = le16_to_cpu(el->l_next_free_rec); + + BUG_ON(num_recs == 0); + + if (ocfs2_is_empty_extent(&el->l_recs[0])) { + num_recs--; + size = num_recs * sizeof(struct ocfs2_extent_rec); + memmove(&el->l_recs[0], &el->l_recs[1], size); + memset(&el->l_recs[num_recs], 0, + sizeof(struct ocfs2_extent_rec)); + el->l_next_free_rec = cpu_to_le16(num_recs); + } +} + +/* + * Create an empty extent record . + * + * l_next_free_rec may be updated. + * + * If an empty extent already exists do nothing. + */ +static void ocfs2_create_empty_extent(struct ocfs2_extent_list *el) +{ + int next_free = le16_to_cpu(el->l_next_free_rec); + + BUG_ON(le16_to_cpu(el->l_tree_depth) != 0); + + if (next_free == 0) + goto set_and_inc; + + if (ocfs2_is_empty_extent(&el->l_recs[0])) + return; + + mlog_bug_on_msg(el->l_count == el->l_next_free_rec, + "Asked to create an empty extent in a full list:\n" + "count = %u, tree depth = %u", + le16_to_cpu(el->l_count), + le16_to_cpu(el->l_tree_depth)); + + ocfs2_shift_records_right(el); + +set_and_inc: + le16_add_cpu(&el->l_next_free_rec, 1); + memset(&el->l_recs[0], 0, sizeof(struct ocfs2_extent_rec)); +} + +/* + * For a rotation which involves two leaf nodes, the "root node" is + * the lowest level tree node which contains a path to both leafs. This + * resulting set of information can be used to form a complete "subtree" + * + * This function is passed two full paths from the dinode down to a + * pair of adjacent leaves. It's task is to figure out which path + * index contains the subtree root - this can be the root index itself + * in a worst-case rotation. + * + * The array index of the subtree root is passed back. + */ +int ocfs2_find_subtree_root(struct ocfs2_extent_tree *et, + struct ocfs2_path *left, + struct ocfs2_path *right) +{ + int i = 0; + + /* + * Check that the caller passed in two paths from the same tree. + */ + BUG_ON(path_root_bh(left) != path_root_bh(right)); + + do { + i++; + + /* + * The caller didn't pass two adjacent paths. + */ + mlog_bug_on_msg(i > left->p_tree_depth, + "Owner %llu, left depth %u, right depth %u\n" + "left leaf blk %llu, right leaf blk %llu\n", + (unsigned long long)ocfs2_metadata_cache_owner(et->et_ci), + left->p_tree_depth, right->p_tree_depth, + (unsigned long long)path_leaf_bh(left)->b_blocknr, + (unsigned long long)path_leaf_bh(right)->b_blocknr); + } while (left->p_node[i].bh->b_blocknr == + right->p_node[i].bh->b_blocknr); + + return i - 1; +} + +typedef void (path_insert_t)(void *, struct buffer_head *); + +/* + * Traverse a btree path in search of cpos, starting at root_el. + * + * This code can be called with a cpos larger than the tree, in which + * case it will return the rightmost path. + */ +static int __ocfs2_find_path(struct ocfs2_caching_info *ci, + struct ocfs2_extent_list *root_el, u32 cpos, + path_insert_t *func, void *data) +{ + int i, ret = 0; + u32 range; + u64 blkno; + struct buffer_head *bh = NULL; + struct ocfs2_extent_block *eb; + struct ocfs2_extent_list *el; + struct ocfs2_extent_rec *rec; + + el = root_el; + while (el->l_tree_depth) { + if (unlikely(le16_to_cpu(el->l_tree_depth) >= OCFS2_MAX_PATH_DEPTH)) { + ocfs2_error(ocfs2_metadata_cache_get_super(ci), + "Owner %llu has invalid tree depth %u in extent list\n", + (unsigned long long)ocfs2_metadata_cache_owner(ci), + le16_to_cpu(el->l_tree_depth)); + ret = -EROFS; + goto out; + } + if (!el->l_next_free_rec || !el->l_count) { + ocfs2_error(ocfs2_metadata_cache_get_super(ci), + "Owner %llu has empty extent list at depth %u\n" + "(next free=%u count=%u)\n", + (unsigned long long)ocfs2_metadata_cache_owner(ci), + le16_to_cpu(el->l_tree_depth), + le16_to_cpu(el->l_next_free_rec), le16_to_cpu(el->l_count)); + ret = -EROFS; + goto out; + } + + for(i = 0; i < le16_to_cpu(el->l_next_free_rec) - 1; i++) { + rec = &el->l_recs[i]; + + /* + * In the case that cpos is off the allocation + * tree, this should just wind up returning the + * rightmost record. + */ + range = le32_to_cpu(rec->e_cpos) + + ocfs2_rec_clusters(el, rec); + if (cpos >= le32_to_cpu(rec->e_cpos) && cpos < range) + break; + } + + blkno = le64_to_cpu(el->l_recs[i].e_blkno); + if (blkno == 0) { + ocfs2_error(ocfs2_metadata_cache_get_super(ci), + "Owner %llu has bad blkno in extent list at depth %u (index %d)\n", + (unsigned long long)ocfs2_metadata_cache_owner(ci), + le16_to_cpu(el->l_tree_depth), i); + ret = -EROFS; + goto out; + } + + brelse(bh); + bh = NULL; + ret = ocfs2_read_extent_block(ci, blkno, &bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + eb = (struct ocfs2_extent_block *) bh->b_data; + el = &eb->h_list; + + if (func) + func(data, bh); + } + +out: + /* + * Catch any trailing bh that the loop didn't handle. + */ + brelse(bh); + + return ret; +} + +/* + * Given an initialized path (that is, it has a valid root extent + * list), this function will traverse the btree in search of the path + * which would contain cpos. + * + * The path traveled is recorded in the path structure. + * + * Note that this will not do any comparisons on leaf node extent + * records, so it will work fine in the case that we just added a tree + * branch. + */ +struct find_path_data { + int index; + struct ocfs2_path *path; +}; +static void find_path_ins(void *data, struct buffer_head *bh) +{ + struct find_path_data *fp = data; + + get_bh(bh); + ocfs2_path_insert_eb(fp->path, fp->index, bh); + fp->index++; +} +int ocfs2_find_path(struct ocfs2_caching_info *ci, + struct ocfs2_path *path, u32 cpos) +{ + struct find_path_data data; + + data.index = 1; + data.path = path; + return __ocfs2_find_path(ci, path_root_el(path), cpos, + find_path_ins, &data); +} + +static void find_leaf_ins(void *data, struct buffer_head *bh) +{ + struct ocfs2_extent_block *eb =(struct ocfs2_extent_block *)bh->b_data; + struct ocfs2_extent_list *el = &eb->h_list; + struct buffer_head **ret = data; + + /* We want to retain only the leaf block. */ + if (le16_to_cpu(el->l_tree_depth) == 0) { + get_bh(bh); + *ret = bh; + } +} +/* + * Find the leaf block in the tree which would contain cpos. No + * checking of the actual leaf is done. + * + * Some paths want to call this instead of allocating a path structure + * and calling ocfs2_find_path(). + * + * This function doesn't handle non btree extent lists. + */ +int ocfs2_find_leaf(struct ocfs2_caching_info *ci, + struct ocfs2_extent_list *root_el, u32 cpos, + struct buffer_head **leaf_bh) +{ + int ret; + struct buffer_head *bh = NULL; + + ret = __ocfs2_find_path(ci, root_el, cpos, find_leaf_ins, &bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + *leaf_bh = bh; +out: + return ret; +} + +/* + * Adjust the adjacent records (left_rec, right_rec) involved in a rotation. + * + * Basically, we've moved stuff around at the bottom of the tree and + * we need to fix up the extent records above the changes to reflect + * the new changes. + * + * left_rec: the record on the left. + * right_rec: the record to the right of left_rec + * right_child_el: is the child list pointed to by right_rec + * + * By definition, this only works on interior nodes. + */ +static void ocfs2_adjust_adjacent_records(struct ocfs2_extent_rec *left_rec, + struct ocfs2_extent_rec *right_rec, + struct ocfs2_extent_list *right_child_el) +{ + u32 left_clusters, right_end; + + /* + * Interior nodes never have holes. Their cpos is the cpos of + * the leftmost record in their child list. Their cluster + * count covers the full theoretical range of their child list + * - the range between their cpos and the cpos of the record + * immediately to their right. + */ + left_clusters = le32_to_cpu(right_child_el->l_recs[0].e_cpos); + if (!ocfs2_rec_clusters(right_child_el, &right_child_el->l_recs[0])) { + BUG_ON(right_child_el->l_tree_depth); + BUG_ON(le16_to_cpu(right_child_el->l_next_free_rec) <= 1); + left_clusters = le32_to_cpu(right_child_el->l_recs[1].e_cpos); + } + left_clusters -= le32_to_cpu(left_rec->e_cpos); + left_rec->e_int_clusters = cpu_to_le32(left_clusters); + + /* + * Calculate the rightmost cluster count boundary before + * moving cpos - we will need to adjust clusters after + * updating e_cpos to keep the same highest cluster count. + */ + right_end = le32_to_cpu(right_rec->e_cpos); + right_end += le32_to_cpu(right_rec->e_int_clusters); + + right_rec->e_cpos = left_rec->e_cpos; + le32_add_cpu(&right_rec->e_cpos, left_clusters); + + right_end -= le32_to_cpu(right_rec->e_cpos); + right_rec->e_int_clusters = cpu_to_le32(right_end); +} + +/* + * Adjust the adjacent root node records involved in a + * rotation. left_el_blkno is passed in as a key so that we can easily + * find it's index in the root list. + */ +static void ocfs2_adjust_root_records(struct ocfs2_extent_list *root_el, + struct ocfs2_extent_list *left_el, + struct ocfs2_extent_list *right_el, + u64 left_el_blkno) +{ + int i; + + BUG_ON(le16_to_cpu(root_el->l_tree_depth) <= + le16_to_cpu(left_el->l_tree_depth)); + + for(i = 0; i < le16_to_cpu(root_el->l_next_free_rec) - 1; i++) { + if (le64_to_cpu(root_el->l_recs[i].e_blkno) == left_el_blkno) + break; + } + + /* + * The path walking code should have never returned a root and + * two paths which are not adjacent. + */ + BUG_ON(i >= (le16_to_cpu(root_el->l_next_free_rec) - 1)); + + ocfs2_adjust_adjacent_records(&root_el->l_recs[i], + &root_el->l_recs[i + 1], right_el); +} + +/* + * We've changed a leaf block (in right_path) and need to reflect that + * change back up the subtree. + * + * This happens in multiple places: + * - When we've moved an extent record from the left path leaf to the right + * path leaf to make room for an empty extent in the left path leaf. + * - When our insert into the right path leaf is at the leftmost edge + * and requires an update of the path immediately to it's left. This + * can occur at the end of some types of rotation and appending inserts. + * - When we've adjusted the last extent record in the left path leaf and the + * 1st extent record in the right path leaf during cross extent block merge. + */ +static void ocfs2_complete_edge_insert(handle_t *handle, + struct ocfs2_path *left_path, + struct ocfs2_path *right_path, + int subtree_index) +{ + int i, idx; + struct ocfs2_extent_list *el, *left_el, *right_el; + struct ocfs2_extent_rec *left_rec, *right_rec; + struct buffer_head *root_bh; + + /* + * Update the counts and position values within all the + * interior nodes to reflect the leaf rotation we just did. + * + * The root node is handled below the loop. + * + * We begin the loop with right_el and left_el pointing to the + * leaf lists and work our way up. + * + * NOTE: within this loop, left_el and right_el always refer + * to the *child* lists. + */ + left_el = path_leaf_el(left_path); + right_el = path_leaf_el(right_path); + for(i = left_path->p_tree_depth - 1; i > subtree_index; i--) { + trace_ocfs2_complete_edge_insert(i); + + /* + * One nice property of knowing that all of these + * nodes are below the root is that we only deal with + * the leftmost right node record and the rightmost + * left node record. + */ + el = left_path->p_node[i].el; + idx = le16_to_cpu(left_el->l_next_free_rec) - 1; + left_rec = &el->l_recs[idx]; + + el = right_path->p_node[i].el; + right_rec = &el->l_recs[0]; + + ocfs2_adjust_adjacent_records(left_rec, right_rec, right_el); + + ocfs2_journal_dirty(handle, left_path->p_node[i].bh); + ocfs2_journal_dirty(handle, right_path->p_node[i].bh); + + /* + * Setup our list pointers now so that the current + * parents become children in the next iteration. + */ + left_el = left_path->p_node[i].el; + right_el = right_path->p_node[i].el; + } + + /* + * At the root node, adjust the two adjacent records which + * begin our path to the leaves. + */ + + el = left_path->p_node[subtree_index].el; + left_el = left_path->p_node[subtree_index + 1].el; + right_el = right_path->p_node[subtree_index + 1].el; + + ocfs2_adjust_root_records(el, left_el, right_el, + left_path->p_node[subtree_index + 1].bh->b_blocknr); + + root_bh = left_path->p_node[subtree_index].bh; + + ocfs2_journal_dirty(handle, root_bh); +} + +static int ocfs2_rotate_subtree_right(handle_t *handle, + struct ocfs2_extent_tree *et, + struct ocfs2_path *left_path, + struct ocfs2_path *right_path, + int subtree_index) +{ + int ret, i; + struct buffer_head *right_leaf_bh; + struct buffer_head *left_leaf_bh = NULL; + struct buffer_head *root_bh; + struct ocfs2_extent_list *right_el, *left_el; + struct ocfs2_extent_rec move_rec; + + left_leaf_bh = path_leaf_bh(left_path); + left_el = path_leaf_el(left_path); + + if (left_el->l_next_free_rec != left_el->l_count) { + ocfs2_error(ocfs2_metadata_cache_get_super(et->et_ci), + "Inode %llu has non-full interior leaf node %llu (next free = %u)\n", + (unsigned long long)ocfs2_metadata_cache_owner(et->et_ci), + (unsigned long long)left_leaf_bh->b_blocknr, + le16_to_cpu(left_el->l_next_free_rec)); + return -EROFS; + } + + /* + * This extent block may already have an empty record, so we + * return early if so. + */ + if (ocfs2_is_empty_extent(&left_el->l_recs[0])) + return 0; + + root_bh = left_path->p_node[subtree_index].bh; + BUG_ON(root_bh != right_path->p_node[subtree_index].bh); + + ret = ocfs2_path_bh_journal_access(handle, et->et_ci, right_path, + subtree_index); + if (ret) { + mlog_errno(ret); + goto out; + } + + for(i = subtree_index + 1; i < path_num_items(right_path); i++) { + ret = ocfs2_path_bh_journal_access(handle, et->et_ci, + right_path, i); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_path_bh_journal_access(handle, et->et_ci, + left_path, i); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + right_leaf_bh = path_leaf_bh(right_path); + right_el = path_leaf_el(right_path); + + /* This is a code error, not a disk corruption. */ + mlog_bug_on_msg(!right_el->l_next_free_rec, "Inode %llu: Rotate fails " + "because rightmost leaf block %llu is empty\n", + (unsigned long long)ocfs2_metadata_cache_owner(et->et_ci), + (unsigned long long)right_leaf_bh->b_blocknr); + + ocfs2_create_empty_extent(right_el); + + ocfs2_journal_dirty(handle, right_leaf_bh); + + /* Do the copy now. */ + i = le16_to_cpu(left_el->l_next_free_rec) - 1; + move_rec = left_el->l_recs[i]; + right_el->l_recs[0] = move_rec; + + /* + * Clear out the record we just copied and shift everything + * over, leaving an empty extent in the left leaf. + * + * We temporarily subtract from next_free_rec so that the + * shift will lose the tail record (which is now defunct). + */ + le16_add_cpu(&left_el->l_next_free_rec, -1); + ocfs2_shift_records_right(left_el); + memset(&left_el->l_recs[0], 0, sizeof(struct ocfs2_extent_rec)); + le16_add_cpu(&left_el->l_next_free_rec, 1); + + ocfs2_journal_dirty(handle, left_leaf_bh); + + ocfs2_complete_edge_insert(handle, left_path, right_path, + subtree_index); + +out: + return ret; +} + +/* + * Given a full path, determine what cpos value would return us a path + * containing the leaf immediately to the left of the current one. + * + * Will return zero if the path passed in is already the leftmost path. + */ +int ocfs2_find_cpos_for_left_leaf(struct super_block *sb, + struct ocfs2_path *path, u32 *cpos) +{ + int i, j, ret = 0; + u64 blkno; + struct ocfs2_extent_list *el; + + BUG_ON(path->p_tree_depth == 0); + + *cpos = 0; + + blkno = path_leaf_bh(path)->b_blocknr; + + /* Start at the tree node just above the leaf and work our way up. */ + i = path->p_tree_depth - 1; + while (i >= 0) { + el = path->p_node[i].el; + + /* + * Find the extent record just before the one in our + * path. + */ + for(j = 0; j < le16_to_cpu(el->l_next_free_rec); j++) { + if (le64_to_cpu(el->l_recs[j].e_blkno) == blkno) { + if (j == 0) { + if (i == 0) { + /* + * We've determined that the + * path specified is already + * the leftmost one - return a + * cpos of zero. + */ + goto out; + } + /* + * The leftmost record points to our + * leaf - we need to travel up the + * tree one level. + */ + goto next_node; + } + + *cpos = le32_to_cpu(el->l_recs[j - 1].e_cpos); + *cpos = *cpos + ocfs2_rec_clusters(el, + &el->l_recs[j - 1]); + *cpos = *cpos - 1; + goto out; + } + } + + /* + * If we got here, we never found a valid node where + * the tree indicated one should be. + */ + ocfs2_error(sb, "Invalid extent tree at extent block %llu\n", + (unsigned long long)blkno); + ret = -EROFS; + goto out; + +next_node: + blkno = path->p_node[i].bh->b_blocknr; + i--; + } + +out: + return ret; +} + +/* + * Extend the transaction by enough credits to complete the rotation, + * and still leave at least the original number of credits allocated + * to this transaction. + */ +static int ocfs2_extend_rotate_transaction(handle_t *handle, int subtree_depth, + int op_credits, + struct ocfs2_path *path) +{ + int ret = 0; + int credits = (path->p_tree_depth - subtree_depth) * 2 + 1 + op_credits; + + if (jbd2_handle_buffer_credits(handle) < credits) + ret = ocfs2_extend_trans(handle, + credits - jbd2_handle_buffer_credits(handle)); + + return ret; +} + +/* + * Trap the case where we're inserting into the theoretical range past + * the _actual_ left leaf range. Otherwise, we'll rotate a record + * whose cpos is less than ours into the right leaf. + * + * It's only necessary to look at the rightmost record of the left + * leaf because the logic that calls us should ensure that the + * theoretical ranges in the path components above the leaves are + * correct. + */ +static int ocfs2_rotate_requires_path_adjustment(struct ocfs2_path *left_path, + u32 insert_cpos) +{ + struct ocfs2_extent_list *left_el; + struct ocfs2_extent_rec *rec; + int next_free; + + left_el = path_leaf_el(left_path); + next_free = le16_to_cpu(left_el->l_next_free_rec); + rec = &left_el->l_recs[next_free - 1]; + + if (insert_cpos > le32_to_cpu(rec->e_cpos)) + return 1; + return 0; +} + +static int ocfs2_leftmost_rec_contains(struct ocfs2_extent_list *el, u32 cpos) +{ + int next_free = le16_to_cpu(el->l_next_free_rec); + unsigned int range; + struct ocfs2_extent_rec *rec; + + if (next_free == 0) + return 0; + + rec = &el->l_recs[0]; + if (ocfs2_is_empty_extent(rec)) { + /* Empty list. */ + if (next_free == 1) + return 0; + rec = &el->l_recs[1]; + } + + range = le32_to_cpu(rec->e_cpos) + ocfs2_rec_clusters(el, rec); + if (cpos >= le32_to_cpu(rec->e_cpos) && cpos < range) + return 1; + return 0; +} + +/* + * Rotate all the records in a btree right one record, starting at insert_cpos. + * + * The path to the rightmost leaf should be passed in. + * + * The array is assumed to be large enough to hold an entire path (tree depth). + * + * Upon successful return from this function: + * + * - The 'right_path' array will contain a path to the leaf block + * whose range contains e_cpos. + * - That leaf block will have a single empty extent in list index 0. + * - In the case that the rotation requires a post-insert update, + * *ret_left_path will contain a valid path which can be passed to + * ocfs2_insert_path(). + */ +static int ocfs2_rotate_tree_right(handle_t *handle, + struct ocfs2_extent_tree *et, + enum ocfs2_split_type split, + u32 insert_cpos, + struct ocfs2_path *right_path, + struct ocfs2_path **ret_left_path) +{ + int ret, start, orig_credits = jbd2_handle_buffer_credits(handle); + u32 cpos; + struct ocfs2_path *left_path = NULL; + struct super_block *sb = ocfs2_metadata_cache_get_super(et->et_ci); + + *ret_left_path = NULL; + + left_path = ocfs2_new_path_from_path(right_path); + if (!left_path) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + ret = ocfs2_find_cpos_for_left_leaf(sb, right_path, &cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + + trace_ocfs2_rotate_tree_right( + (unsigned long long)ocfs2_metadata_cache_owner(et->et_ci), + insert_cpos, cpos); + + /* + * What we want to do here is: + * + * 1) Start with the rightmost path. + * + * 2) Determine a path to the leaf block directly to the left + * of that leaf. + * + * 3) Determine the 'subtree root' - the lowest level tree node + * which contains a path to both leaves. + * + * 4) Rotate the subtree. + * + * 5) Find the next subtree by considering the left path to be + * the new right path. + * + * The check at the top of this while loop also accepts + * insert_cpos == cpos because cpos is only a _theoretical_ + * value to get us the left path - insert_cpos might very well + * be filling that hole. + * + * Stop at a cpos of '0' because we either started at the + * leftmost branch (i.e., a tree with one branch and a + * rotation inside of it), or we've gone as far as we can in + * rotating subtrees. + */ + while (cpos && insert_cpos <= cpos) { + trace_ocfs2_rotate_tree_right( + (unsigned long long) + ocfs2_metadata_cache_owner(et->et_ci), + insert_cpos, cpos); + + ret = ocfs2_find_path(et->et_ci, left_path, cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + + mlog_bug_on_msg(path_leaf_bh(left_path) == + path_leaf_bh(right_path), + "Owner %llu: error during insert of %u " + "(left path cpos %u) results in two identical " + "paths ending at %llu\n", + (unsigned long long)ocfs2_metadata_cache_owner(et->et_ci), + insert_cpos, cpos, + (unsigned long long) + path_leaf_bh(left_path)->b_blocknr); + + if (split == SPLIT_NONE && + ocfs2_rotate_requires_path_adjustment(left_path, + insert_cpos)) { + + /* + * We've rotated the tree as much as we + * should. The rest is up to + * ocfs2_insert_path() to complete, after the + * record insertion. We indicate this + * situation by returning the left path. + * + * The reason we don't adjust the records here + * before the record insert is that an error + * later might break the rule where a parent + * record e_cpos will reflect the actual + * e_cpos of the 1st nonempty record of the + * child list. + */ + *ret_left_path = left_path; + goto out_ret_path; + } + + start = ocfs2_find_subtree_root(et, left_path, right_path); + + trace_ocfs2_rotate_subtree(start, + (unsigned long long) + right_path->p_node[start].bh->b_blocknr, + right_path->p_tree_depth); + + ret = ocfs2_extend_rotate_transaction(handle, start, + orig_credits, right_path); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_rotate_subtree_right(handle, et, left_path, + right_path, start); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (split != SPLIT_NONE && + ocfs2_leftmost_rec_contains(path_leaf_el(right_path), + insert_cpos)) { + /* + * A rotate moves the rightmost left leaf + * record over to the leftmost right leaf + * slot. If we're doing an extent split + * instead of a real insert, then we have to + * check that the extent to be split wasn't + * just moved over. If it was, then we can + * exit here, passing left_path back - + * ocfs2_split_extent() is smart enough to + * search both leaves. + */ + *ret_left_path = left_path; + goto out_ret_path; + } + + /* + * There is no need to re-read the next right path + * as we know that it'll be our current left + * path. Optimize by copying values instead. + */ + ocfs2_mv_path(right_path, left_path); + + ret = ocfs2_find_cpos_for_left_leaf(sb, right_path, &cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + } + +out: + ocfs2_free_path(left_path); + +out_ret_path: + return ret; +} + +static int ocfs2_update_edge_lengths(handle_t *handle, + struct ocfs2_extent_tree *et, + struct ocfs2_path *path) +{ + int i, idx, ret; + struct ocfs2_extent_rec *rec; + struct ocfs2_extent_list *el; + struct ocfs2_extent_block *eb; + u32 range; + + ret = ocfs2_journal_access_path(et->et_ci, handle, path); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* Path should always be rightmost. */ + eb = (struct ocfs2_extent_block *)path_leaf_bh(path)->b_data; + BUG_ON(eb->h_next_leaf_blk != 0ULL); + + el = &eb->h_list; + BUG_ON(le16_to_cpu(el->l_next_free_rec) == 0); + idx = le16_to_cpu(el->l_next_free_rec) - 1; + rec = &el->l_recs[idx]; + range = le32_to_cpu(rec->e_cpos) + ocfs2_rec_clusters(el, rec); + + for (i = 0; i < path->p_tree_depth; i++) { + el = path->p_node[i].el; + idx = le16_to_cpu(el->l_next_free_rec) - 1; + rec = &el->l_recs[idx]; + + rec->e_int_clusters = cpu_to_le32(range); + le32_add_cpu(&rec->e_int_clusters, -le32_to_cpu(rec->e_cpos)); + + ocfs2_journal_dirty(handle, path->p_node[i].bh); + } +out: + return ret; +} + +static void ocfs2_unlink_path(handle_t *handle, + struct ocfs2_extent_tree *et, + struct ocfs2_cached_dealloc_ctxt *dealloc, + struct ocfs2_path *path, int unlink_start) +{ + int ret, i; + struct ocfs2_extent_block *eb; + struct ocfs2_extent_list *el; + struct buffer_head *bh; + + for(i = unlink_start; i < path_num_items(path); i++) { + bh = path->p_node[i].bh; + + eb = (struct ocfs2_extent_block *)bh->b_data; + /* + * Not all nodes might have had their final count + * decremented by the caller - handle this here. + */ + el = &eb->h_list; + if (le16_to_cpu(el->l_next_free_rec) > 1) { + mlog(ML_ERROR, + "Inode %llu, attempted to remove extent block " + "%llu with %u records\n", + (unsigned long long)ocfs2_metadata_cache_owner(et->et_ci), + (unsigned long long)le64_to_cpu(eb->h_blkno), + le16_to_cpu(el->l_next_free_rec)); + + ocfs2_journal_dirty(handle, bh); + ocfs2_remove_from_cache(et->et_ci, bh); + continue; + } + + el->l_next_free_rec = 0; + memset(&el->l_recs[0], 0, sizeof(struct ocfs2_extent_rec)); + + ocfs2_journal_dirty(handle, bh); + + ret = ocfs2_cache_extent_block_free(dealloc, eb); + if (ret) + mlog_errno(ret); + + ocfs2_remove_from_cache(et->et_ci, bh); + } +} + +static void ocfs2_unlink_subtree(handle_t *handle, + struct ocfs2_extent_tree *et, + struct ocfs2_path *left_path, + struct ocfs2_path *right_path, + int subtree_index, + struct ocfs2_cached_dealloc_ctxt *dealloc) +{ + int i; + struct buffer_head *root_bh = left_path->p_node[subtree_index].bh; + struct ocfs2_extent_list *root_el = left_path->p_node[subtree_index].el; + struct ocfs2_extent_block *eb; + + eb = (struct ocfs2_extent_block *)right_path->p_node[subtree_index + 1].bh->b_data; + + for(i = 1; i < le16_to_cpu(root_el->l_next_free_rec); i++) + if (root_el->l_recs[i].e_blkno == eb->h_blkno) + break; + + BUG_ON(i >= le16_to_cpu(root_el->l_next_free_rec)); + + memset(&root_el->l_recs[i], 0, sizeof(struct ocfs2_extent_rec)); + le16_add_cpu(&root_el->l_next_free_rec, -1); + + eb = (struct ocfs2_extent_block *)path_leaf_bh(left_path)->b_data; + eb->h_next_leaf_blk = 0; + + ocfs2_journal_dirty(handle, root_bh); + ocfs2_journal_dirty(handle, path_leaf_bh(left_path)); + + ocfs2_unlink_path(handle, et, dealloc, right_path, + subtree_index + 1); +} + +static int ocfs2_rotate_subtree_left(handle_t *handle, + struct ocfs2_extent_tree *et, + struct ocfs2_path *left_path, + struct ocfs2_path *right_path, + int subtree_index, + struct ocfs2_cached_dealloc_ctxt *dealloc, + int *deleted) +{ + int ret, i, del_right_subtree = 0, right_has_empty = 0; + struct buffer_head *root_bh, *et_root_bh = path_root_bh(right_path); + struct ocfs2_extent_list *right_leaf_el, *left_leaf_el; + struct ocfs2_extent_block *eb; + + *deleted = 0; + + right_leaf_el = path_leaf_el(right_path); + left_leaf_el = path_leaf_el(left_path); + root_bh = left_path->p_node[subtree_index].bh; + BUG_ON(root_bh != right_path->p_node[subtree_index].bh); + + if (!ocfs2_is_empty_extent(&left_leaf_el->l_recs[0])) + return 0; + + eb = (struct ocfs2_extent_block *)path_leaf_bh(right_path)->b_data; + if (ocfs2_is_empty_extent(&right_leaf_el->l_recs[0])) { + /* + * It's legal for us to proceed if the right leaf is + * the rightmost one and it has an empty extent. There + * are two cases to handle - whether the leaf will be + * empty after removal or not. If the leaf isn't empty + * then just remove the empty extent up front. The + * next block will handle empty leaves by flagging + * them for unlink. + * + * Non rightmost leaves will throw -EAGAIN and the + * caller can manually move the subtree and retry. + */ + + if (eb->h_next_leaf_blk != 0ULL) + return -EAGAIN; + + if (le16_to_cpu(right_leaf_el->l_next_free_rec) > 1) { + ret = ocfs2_journal_access_eb(handle, et->et_ci, + path_leaf_bh(right_path), + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + + ocfs2_remove_empty_extent(right_leaf_el); + } else + right_has_empty = 1; + } + + if (eb->h_next_leaf_blk == 0ULL && + le16_to_cpu(right_leaf_el->l_next_free_rec) == 1) { + /* + * We have to update i_last_eb_blk during the meta + * data delete. + */ + ret = ocfs2_et_root_journal_access(handle, et, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + + del_right_subtree = 1; + } + + /* + * Getting here with an empty extent in the right path implies + * that it's the rightmost path and will be deleted. + */ + BUG_ON(right_has_empty && !del_right_subtree); + + ret = ocfs2_path_bh_journal_access(handle, et->et_ci, right_path, + subtree_index); + if (ret) { + mlog_errno(ret); + goto out; + } + + for(i = subtree_index + 1; i < path_num_items(right_path); i++) { + ret = ocfs2_path_bh_journal_access(handle, et->et_ci, + right_path, i); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_path_bh_journal_access(handle, et->et_ci, + left_path, i); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + if (!right_has_empty) { + /* + * Only do this if we're moving a real + * record. Otherwise, the action is delayed until + * after removal of the right path in which case we + * can do a simple shift to remove the empty extent. + */ + ocfs2_rotate_leaf(left_leaf_el, &right_leaf_el->l_recs[0]); + memset(&right_leaf_el->l_recs[0], 0, + sizeof(struct ocfs2_extent_rec)); + } + if (eb->h_next_leaf_blk == 0ULL) { + /* + * Move recs over to get rid of empty extent, decrease + * next_free. This is allowed to remove the last + * extent in our leaf (setting l_next_free_rec to + * zero) - the delete code below won't care. + */ + ocfs2_remove_empty_extent(right_leaf_el); + } + + ocfs2_journal_dirty(handle, path_leaf_bh(left_path)); + ocfs2_journal_dirty(handle, path_leaf_bh(right_path)); + + if (del_right_subtree) { + ocfs2_unlink_subtree(handle, et, left_path, right_path, + subtree_index, dealloc); + ret = ocfs2_update_edge_lengths(handle, et, left_path); + if (ret) { + mlog_errno(ret); + goto out; + } + + eb = (struct ocfs2_extent_block *)path_leaf_bh(left_path)->b_data; + ocfs2_et_set_last_eb_blk(et, le64_to_cpu(eb->h_blkno)); + + /* + * Removal of the extent in the left leaf was skipped + * above so we could delete the right path + * 1st. + */ + if (right_has_empty) + ocfs2_remove_empty_extent(left_leaf_el); + + ocfs2_journal_dirty(handle, et_root_bh); + + *deleted = 1; + } else + ocfs2_complete_edge_insert(handle, left_path, right_path, + subtree_index); + +out: + return ret; +} + +/* + * Given a full path, determine what cpos value would return us a path + * containing the leaf immediately to the right of the current one. + * + * Will return zero if the path passed in is already the rightmost path. + * + * This looks similar, but is subtly different to + * ocfs2_find_cpos_for_left_leaf(). + */ +int ocfs2_find_cpos_for_right_leaf(struct super_block *sb, + struct ocfs2_path *path, u32 *cpos) +{ + int i, j, ret = 0; + u64 blkno; + struct ocfs2_extent_list *el; + + *cpos = 0; + + if (path->p_tree_depth == 0) + return 0; + + blkno = path_leaf_bh(path)->b_blocknr; + + /* Start at the tree node just above the leaf and work our way up. */ + i = path->p_tree_depth - 1; + while (i >= 0) { + int next_free; + + el = path->p_node[i].el; + + /* + * Find the extent record just after the one in our + * path. + */ + next_free = le16_to_cpu(el->l_next_free_rec); + for(j = 0; j < le16_to_cpu(el->l_next_free_rec); j++) { + if (le64_to_cpu(el->l_recs[j].e_blkno) == blkno) { + if (j == (next_free - 1)) { + if (i == 0) { + /* + * We've determined that the + * path specified is already + * the rightmost one - return a + * cpos of zero. + */ + goto out; + } + /* + * The rightmost record points to our + * leaf - we need to travel up the + * tree one level. + */ + goto next_node; + } + + *cpos = le32_to_cpu(el->l_recs[j + 1].e_cpos); + goto out; + } + } + + /* + * If we got here, we never found a valid node where + * the tree indicated one should be. + */ + ocfs2_error(sb, "Invalid extent tree at extent block %llu\n", + (unsigned long long)blkno); + ret = -EROFS; + goto out; + +next_node: + blkno = path->p_node[i].bh->b_blocknr; + i--; + } + +out: + return ret; +} + +static int ocfs2_rotate_rightmost_leaf_left(handle_t *handle, + struct ocfs2_extent_tree *et, + struct ocfs2_path *path) +{ + int ret; + struct buffer_head *bh = path_leaf_bh(path); + struct ocfs2_extent_list *el = path_leaf_el(path); + + if (!ocfs2_is_empty_extent(&el->l_recs[0])) + return 0; + + ret = ocfs2_path_bh_journal_access(handle, et->et_ci, path, + path_num_items(path) - 1); + if (ret) { + mlog_errno(ret); + goto out; + } + + ocfs2_remove_empty_extent(el); + ocfs2_journal_dirty(handle, bh); + +out: + return ret; +} + +static int __ocfs2_rotate_tree_left(handle_t *handle, + struct ocfs2_extent_tree *et, + int orig_credits, + struct ocfs2_path *path, + struct ocfs2_cached_dealloc_ctxt *dealloc, + struct ocfs2_path **empty_extent_path) +{ + int ret, subtree_root, deleted; + u32 right_cpos; + struct ocfs2_path *left_path = NULL; + struct ocfs2_path *right_path = NULL; + struct super_block *sb = ocfs2_metadata_cache_get_super(et->et_ci); + + if (!ocfs2_is_empty_extent(&(path_leaf_el(path)->l_recs[0]))) + return 0; + + *empty_extent_path = NULL; + + ret = ocfs2_find_cpos_for_right_leaf(sb, path, &right_cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + + left_path = ocfs2_new_path_from_path(path); + if (!left_path) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + ocfs2_cp_path(left_path, path); + + right_path = ocfs2_new_path_from_path(path); + if (!right_path) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + while (right_cpos) { + ret = ocfs2_find_path(et->et_ci, right_path, right_cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + + subtree_root = ocfs2_find_subtree_root(et, left_path, + right_path); + + trace_ocfs2_rotate_subtree(subtree_root, + (unsigned long long) + right_path->p_node[subtree_root].bh->b_blocknr, + right_path->p_tree_depth); + + ret = ocfs2_extend_rotate_transaction(handle, 0, + orig_credits, left_path); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * Caller might still want to make changes to the + * tree root, so re-add it to the journal here. + */ + ret = ocfs2_path_bh_journal_access(handle, et->et_ci, + left_path, 0); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_rotate_subtree_left(handle, et, left_path, + right_path, subtree_root, + dealloc, &deleted); + if (ret == -EAGAIN) { + /* + * The rotation has to temporarily stop due to + * the right subtree having an empty + * extent. Pass it back to the caller for a + * fixup. + */ + *empty_extent_path = right_path; + right_path = NULL; + goto out; + } + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * The subtree rotate might have removed records on + * the rightmost edge. If so, then rotation is + * complete. + */ + if (deleted) + break; + + ocfs2_mv_path(left_path, right_path); + + ret = ocfs2_find_cpos_for_right_leaf(sb, left_path, + &right_cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + } + +out: + ocfs2_free_path(right_path); + ocfs2_free_path(left_path); + + return ret; +} + +static int ocfs2_remove_rightmost_path(handle_t *handle, + struct ocfs2_extent_tree *et, + struct ocfs2_path *path, + struct ocfs2_cached_dealloc_ctxt *dealloc) +{ + int ret, subtree_index; + u32 cpos; + struct ocfs2_path *left_path = NULL; + struct ocfs2_extent_block *eb; + struct ocfs2_extent_list *el; + + ret = ocfs2_et_sanity_check(et); + if (ret) + goto out; + + ret = ocfs2_journal_access_path(et->et_ci, handle, path); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_find_cpos_for_left_leaf(ocfs2_metadata_cache_get_super(et->et_ci), + path, &cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (cpos) { + /* + * We have a path to the left of this one - it needs + * an update too. + */ + left_path = ocfs2_new_path_from_path(path); + if (!left_path) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + ret = ocfs2_find_path(et->et_ci, left_path, cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_journal_access_path(et->et_ci, handle, left_path); + if (ret) { + mlog_errno(ret); + goto out; + } + + subtree_index = ocfs2_find_subtree_root(et, left_path, path); + + ocfs2_unlink_subtree(handle, et, left_path, path, + subtree_index, dealloc); + ret = ocfs2_update_edge_lengths(handle, et, left_path); + if (ret) { + mlog_errno(ret); + goto out; + } + + eb = (struct ocfs2_extent_block *)path_leaf_bh(left_path)->b_data; + ocfs2_et_set_last_eb_blk(et, le64_to_cpu(eb->h_blkno)); + } else { + /* + * 'path' is also the leftmost path which + * means it must be the only one. This gets + * handled differently because we want to + * revert the root back to having extents + * in-line. + */ + ocfs2_unlink_path(handle, et, dealloc, path, 1); + + el = et->et_root_el; + el->l_tree_depth = 0; + el->l_next_free_rec = 0; + memset(&el->l_recs[0], 0, sizeof(struct ocfs2_extent_rec)); + + ocfs2_et_set_last_eb_blk(et, 0); + } + + ocfs2_journal_dirty(handle, path_root_bh(path)); + +out: + ocfs2_free_path(left_path); + return ret; +} + +static int ocfs2_remove_rightmost_empty_extent(struct ocfs2_super *osb, + struct ocfs2_extent_tree *et, + struct ocfs2_path *path, + struct ocfs2_cached_dealloc_ctxt *dealloc) +{ + handle_t *handle; + int ret; + int credits = path->p_tree_depth * 2 + 1; + + handle = ocfs2_start_trans(osb, credits); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + return ret; + } + + ret = ocfs2_remove_rightmost_path(handle, et, path, dealloc); + if (ret) + mlog_errno(ret); + + ocfs2_commit_trans(osb, handle); + return ret; +} + +/* + * Left rotation of btree records. + * + * In many ways, this is (unsurprisingly) the opposite of right + * rotation. We start at some non-rightmost path containing an empty + * extent in the leaf block. The code works its way to the rightmost + * path by rotating records to the left in every subtree. + * + * This is used by any code which reduces the number of extent records + * in a leaf. After removal, an empty record should be placed in the + * leftmost list position. + * + * This won't handle a length update of the rightmost path records if + * the rightmost tree leaf record is removed so the caller is + * responsible for detecting and correcting that. + */ +static int ocfs2_rotate_tree_left(handle_t *handle, + struct ocfs2_extent_tree *et, + struct ocfs2_path *path, + struct ocfs2_cached_dealloc_ctxt *dealloc) +{ + int ret, orig_credits = jbd2_handle_buffer_credits(handle); + struct ocfs2_path *tmp_path = NULL, *restart_path = NULL; + struct ocfs2_extent_block *eb; + struct ocfs2_extent_list *el; + + el = path_leaf_el(path); + if (!ocfs2_is_empty_extent(&el->l_recs[0])) + return 0; + + if (path->p_tree_depth == 0) { +rightmost_no_delete: + /* + * Inline extents. This is trivially handled, so do + * it up front. + */ + ret = ocfs2_rotate_rightmost_leaf_left(handle, et, path); + if (ret) + mlog_errno(ret); + goto out; + } + + /* + * Handle rightmost branch now. There's several cases: + * 1) simple rotation leaving records in there. That's trivial. + * 2) rotation requiring a branch delete - there's no more + * records left. Two cases of this: + * a) There are branches to the left. + * b) This is also the leftmost (the only) branch. + * + * 1) is handled via ocfs2_rotate_rightmost_leaf_left() + * 2a) we need the left branch so that we can update it with the unlink + * 2b) we need to bring the root back to inline extents. + */ + + eb = (struct ocfs2_extent_block *)path_leaf_bh(path)->b_data; + el = &eb->h_list; + if (eb->h_next_leaf_blk == 0) { + /* + * This gets a bit tricky if we're going to delete the + * rightmost path. Get the other cases out of the way + * 1st. + */ + if (le16_to_cpu(el->l_next_free_rec) > 1) + goto rightmost_no_delete; + + if (le16_to_cpu(el->l_next_free_rec) == 0) { + ret = ocfs2_error(ocfs2_metadata_cache_get_super(et->et_ci), + "Owner %llu has empty extent block at %llu\n", + (unsigned long long)ocfs2_metadata_cache_owner(et->et_ci), + (unsigned long long)le64_to_cpu(eb->h_blkno)); + goto out; + } + + /* + * XXX: The caller can not trust "path" any more after + * this as it will have been deleted. What do we do? + * + * In theory the rotate-for-merge code will never get + * here because it'll always ask for a rotate in a + * nonempty list. + */ + + ret = ocfs2_remove_rightmost_path(handle, et, path, + dealloc); + if (ret) + mlog_errno(ret); + goto out; + } + + /* + * Now we can loop, remembering the path we get from -EAGAIN + * and restarting from there. + */ +try_rotate: + ret = __ocfs2_rotate_tree_left(handle, et, orig_credits, path, + dealloc, &restart_path); + if (ret && ret != -EAGAIN) { + mlog_errno(ret); + goto out; + } + + while (ret == -EAGAIN) { + tmp_path = restart_path; + restart_path = NULL; + + ret = __ocfs2_rotate_tree_left(handle, et, orig_credits, + tmp_path, dealloc, + &restart_path); + if (ret && ret != -EAGAIN) { + mlog_errno(ret); + goto out; + } + + ocfs2_free_path(tmp_path); + tmp_path = NULL; + + if (ret == 0) + goto try_rotate; + } + +out: + ocfs2_free_path(tmp_path); + ocfs2_free_path(restart_path); + return ret; +} + +static void ocfs2_cleanup_merge(struct ocfs2_extent_list *el, + int index) +{ + struct ocfs2_extent_rec *rec = &el->l_recs[index]; + unsigned int size; + + if (rec->e_leaf_clusters == 0) { + /* + * We consumed all of the merged-from record. An empty + * extent cannot exist anywhere but the 1st array + * position, so move things over if the merged-from + * record doesn't occupy that position. + * + * This creates a new empty extent so the caller + * should be smart enough to have removed any existing + * ones. + */ + if (index > 0) { + BUG_ON(ocfs2_is_empty_extent(&el->l_recs[0])); + size = index * sizeof(struct ocfs2_extent_rec); + memmove(&el->l_recs[1], &el->l_recs[0], size); + } + + /* + * Always memset - the caller doesn't check whether it + * created an empty extent, so there could be junk in + * the other fields. + */ + memset(&el->l_recs[0], 0, sizeof(struct ocfs2_extent_rec)); + } +} + +static int ocfs2_get_right_path(struct ocfs2_extent_tree *et, + struct ocfs2_path *left_path, + struct ocfs2_path **ret_right_path) +{ + int ret; + u32 right_cpos; + struct ocfs2_path *right_path = NULL; + struct ocfs2_extent_list *left_el; + + *ret_right_path = NULL; + + /* This function shouldn't be called for non-trees. */ + BUG_ON(left_path->p_tree_depth == 0); + + left_el = path_leaf_el(left_path); + BUG_ON(left_el->l_next_free_rec != left_el->l_count); + + ret = ocfs2_find_cpos_for_right_leaf(ocfs2_metadata_cache_get_super(et->et_ci), + left_path, &right_cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* This function shouldn't be called for the rightmost leaf. */ + BUG_ON(right_cpos == 0); + + right_path = ocfs2_new_path_from_path(left_path); + if (!right_path) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + ret = ocfs2_find_path(et->et_ci, right_path, right_cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + + *ret_right_path = right_path; +out: + if (ret) + ocfs2_free_path(right_path); + return ret; +} + +/* + * Remove split_rec clusters from the record at index and merge them + * onto the beginning of the record "next" to it. + * For index < l_count - 1, the next means the extent rec at index + 1. + * For index == l_count - 1, the "next" means the 1st extent rec of the + * next extent block. + */ +static int ocfs2_merge_rec_right(struct ocfs2_path *left_path, + handle_t *handle, + struct ocfs2_extent_tree *et, + struct ocfs2_extent_rec *split_rec, + int index) +{ + int ret, next_free, i; + unsigned int split_clusters = le16_to_cpu(split_rec->e_leaf_clusters); + struct ocfs2_extent_rec *left_rec; + struct ocfs2_extent_rec *right_rec; + struct ocfs2_extent_list *right_el; + struct ocfs2_path *right_path = NULL; + int subtree_index = 0; + struct ocfs2_extent_list *el = path_leaf_el(left_path); + struct buffer_head *bh = path_leaf_bh(left_path); + struct buffer_head *root_bh = NULL; + + BUG_ON(index >= le16_to_cpu(el->l_next_free_rec)); + left_rec = &el->l_recs[index]; + + if (index == le16_to_cpu(el->l_next_free_rec) - 1 && + le16_to_cpu(el->l_next_free_rec) == le16_to_cpu(el->l_count)) { + /* we meet with a cross extent block merge. */ + ret = ocfs2_get_right_path(et, left_path, &right_path); + if (ret) { + mlog_errno(ret); + return ret; + } + + right_el = path_leaf_el(right_path); + next_free = le16_to_cpu(right_el->l_next_free_rec); + BUG_ON(next_free <= 0); + right_rec = &right_el->l_recs[0]; + if (ocfs2_is_empty_extent(right_rec)) { + BUG_ON(next_free <= 1); + right_rec = &right_el->l_recs[1]; + } + + BUG_ON(le32_to_cpu(left_rec->e_cpos) + + le16_to_cpu(left_rec->e_leaf_clusters) != + le32_to_cpu(right_rec->e_cpos)); + + subtree_index = ocfs2_find_subtree_root(et, left_path, + right_path); + + ret = ocfs2_extend_rotate_transaction(handle, subtree_index, + jbd2_handle_buffer_credits(handle), + right_path); + if (ret) { + mlog_errno(ret); + goto out; + } + + root_bh = left_path->p_node[subtree_index].bh; + BUG_ON(root_bh != right_path->p_node[subtree_index].bh); + + ret = ocfs2_path_bh_journal_access(handle, et->et_ci, right_path, + subtree_index); + if (ret) { + mlog_errno(ret); + goto out; + } + + for (i = subtree_index + 1; + i < path_num_items(right_path); i++) { + ret = ocfs2_path_bh_journal_access(handle, et->et_ci, + right_path, i); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_path_bh_journal_access(handle, et->et_ci, + left_path, i); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + } else { + BUG_ON(index == le16_to_cpu(el->l_next_free_rec) - 1); + right_rec = &el->l_recs[index + 1]; + } + + ret = ocfs2_path_bh_journal_access(handle, et->et_ci, left_path, + path_num_items(left_path) - 1); + if (ret) { + mlog_errno(ret); + goto out; + } + + le16_add_cpu(&left_rec->e_leaf_clusters, -split_clusters); + + le32_add_cpu(&right_rec->e_cpos, -split_clusters); + le64_add_cpu(&right_rec->e_blkno, + -ocfs2_clusters_to_blocks(ocfs2_metadata_cache_get_super(et->et_ci), + split_clusters)); + le16_add_cpu(&right_rec->e_leaf_clusters, split_clusters); + + ocfs2_cleanup_merge(el, index); + + ocfs2_journal_dirty(handle, bh); + if (right_path) { + ocfs2_journal_dirty(handle, path_leaf_bh(right_path)); + ocfs2_complete_edge_insert(handle, left_path, right_path, + subtree_index); + } +out: + ocfs2_free_path(right_path); + return ret; +} + +static int ocfs2_get_left_path(struct ocfs2_extent_tree *et, + struct ocfs2_path *right_path, + struct ocfs2_path **ret_left_path) +{ + int ret; + u32 left_cpos; + struct ocfs2_path *left_path = NULL; + + *ret_left_path = NULL; + + /* This function shouldn't be called for non-trees. */ + BUG_ON(right_path->p_tree_depth == 0); + + ret = ocfs2_find_cpos_for_left_leaf(ocfs2_metadata_cache_get_super(et->et_ci), + right_path, &left_cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* This function shouldn't be called for the leftmost leaf. */ + BUG_ON(left_cpos == 0); + + left_path = ocfs2_new_path_from_path(right_path); + if (!left_path) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + ret = ocfs2_find_path(et->et_ci, left_path, left_cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + + *ret_left_path = left_path; +out: + if (ret) + ocfs2_free_path(left_path); + return ret; +} + +/* + * Remove split_rec clusters from the record at index and merge them + * onto the tail of the record "before" it. + * For index > 0, the "before" means the extent rec at index - 1. + * + * For index == 0, the "before" means the last record of the previous + * extent block. And there is also a situation that we may need to + * remove the rightmost leaf extent block in the right_path and change + * the right path to indicate the new rightmost path. + */ +static int ocfs2_merge_rec_left(struct ocfs2_path *right_path, + handle_t *handle, + struct ocfs2_extent_tree *et, + struct ocfs2_extent_rec *split_rec, + struct ocfs2_cached_dealloc_ctxt *dealloc, + int index) +{ + int ret, i, subtree_index = 0, has_empty_extent = 0; + unsigned int split_clusters = le16_to_cpu(split_rec->e_leaf_clusters); + struct ocfs2_extent_rec *left_rec; + struct ocfs2_extent_rec *right_rec; + struct ocfs2_extent_list *el = path_leaf_el(right_path); + struct buffer_head *bh = path_leaf_bh(right_path); + struct buffer_head *root_bh = NULL; + struct ocfs2_path *left_path = NULL; + struct ocfs2_extent_list *left_el; + + BUG_ON(index < 0); + + right_rec = &el->l_recs[index]; + if (index == 0) { + /* we meet with a cross extent block merge. */ + ret = ocfs2_get_left_path(et, right_path, &left_path); + if (ret) { + mlog_errno(ret); + return ret; + } + + left_el = path_leaf_el(left_path); + BUG_ON(le16_to_cpu(left_el->l_next_free_rec) != + le16_to_cpu(left_el->l_count)); + + left_rec = &left_el->l_recs[ + le16_to_cpu(left_el->l_next_free_rec) - 1]; + BUG_ON(le32_to_cpu(left_rec->e_cpos) + + le16_to_cpu(left_rec->e_leaf_clusters) != + le32_to_cpu(split_rec->e_cpos)); + + subtree_index = ocfs2_find_subtree_root(et, left_path, + right_path); + + ret = ocfs2_extend_rotate_transaction(handle, subtree_index, + jbd2_handle_buffer_credits(handle), + left_path); + if (ret) { + mlog_errno(ret); + goto out; + } + + root_bh = left_path->p_node[subtree_index].bh; + BUG_ON(root_bh != right_path->p_node[subtree_index].bh); + + ret = ocfs2_path_bh_journal_access(handle, et->et_ci, right_path, + subtree_index); + if (ret) { + mlog_errno(ret); + goto out; + } + + for (i = subtree_index + 1; + i < path_num_items(right_path); i++) { + ret = ocfs2_path_bh_journal_access(handle, et->et_ci, + right_path, i); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_path_bh_journal_access(handle, et->et_ci, + left_path, i); + if (ret) { + mlog_errno(ret); + goto out; + } + } + } else { + left_rec = &el->l_recs[index - 1]; + if (ocfs2_is_empty_extent(&el->l_recs[0])) + has_empty_extent = 1; + } + + ret = ocfs2_path_bh_journal_access(handle, et->et_ci, right_path, + path_num_items(right_path) - 1); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (has_empty_extent && index == 1) { + /* + * The easy case - we can just plop the record right in. + */ + *left_rec = *split_rec; + } else + le16_add_cpu(&left_rec->e_leaf_clusters, split_clusters); + + le32_add_cpu(&right_rec->e_cpos, split_clusters); + le64_add_cpu(&right_rec->e_blkno, + ocfs2_clusters_to_blocks(ocfs2_metadata_cache_get_super(et->et_ci), + split_clusters)); + le16_add_cpu(&right_rec->e_leaf_clusters, -split_clusters); + + ocfs2_cleanup_merge(el, index); + + ocfs2_journal_dirty(handle, bh); + if (left_path) { + ocfs2_journal_dirty(handle, path_leaf_bh(left_path)); + + /* + * In the situation that the right_rec is empty and the extent + * block is empty also, ocfs2_complete_edge_insert can't handle + * it and we need to delete the right extent block. + */ + if (le16_to_cpu(right_rec->e_leaf_clusters) == 0 && + le16_to_cpu(el->l_next_free_rec) == 1) { + /* extend credit for ocfs2_remove_rightmost_path */ + ret = ocfs2_extend_rotate_transaction(handle, 0, + jbd2_handle_buffer_credits(handle), + right_path); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_remove_rightmost_path(handle, et, + right_path, + dealloc); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* Now the rightmost extent block has been deleted. + * So we use the new rightmost path. + */ + ocfs2_mv_path(right_path, left_path); + } else + ocfs2_complete_edge_insert(handle, left_path, + right_path, subtree_index); + } +out: + ocfs2_free_path(left_path); + return ret; +} + +static int ocfs2_try_to_merge_extent(handle_t *handle, + struct ocfs2_extent_tree *et, + struct ocfs2_path *path, + int split_index, + struct ocfs2_extent_rec *split_rec, + struct ocfs2_cached_dealloc_ctxt *dealloc, + struct ocfs2_merge_ctxt *ctxt) +{ + int ret = 0; + struct ocfs2_extent_list *el = path_leaf_el(path); + struct ocfs2_extent_rec *rec = &el->l_recs[split_index]; + + BUG_ON(ctxt->c_contig_type == CONTIG_NONE); + + if (ctxt->c_split_covers_rec && ctxt->c_has_empty_extent) { + /* extend credit for ocfs2_remove_rightmost_path */ + ret = ocfs2_extend_rotate_transaction(handle, 0, + jbd2_handle_buffer_credits(handle), + path); + if (ret) { + mlog_errno(ret); + goto out; + } + /* + * The merge code will need to create an empty + * extent to take the place of the newly + * emptied slot. Remove any pre-existing empty + * extents - having more than one in a leaf is + * illegal. + */ + ret = ocfs2_rotate_tree_left(handle, et, path, dealloc); + if (ret) { + mlog_errno(ret); + goto out; + } + split_index--; + rec = &el->l_recs[split_index]; + } + + if (ctxt->c_contig_type == CONTIG_LEFTRIGHT) { + /* + * Left-right contig implies this. + */ + BUG_ON(!ctxt->c_split_covers_rec); + + /* + * Since the leftright insert always covers the entire + * extent, this call will delete the insert record + * entirely, resulting in an empty extent record added to + * the extent block. + * + * Since the adding of an empty extent shifts + * everything back to the right, there's no need to + * update split_index here. + * + * When the split_index is zero, we need to merge it to the + * previous extent block. It is more efficient and easier + * if we do merge_right first and merge_left later. + */ + ret = ocfs2_merge_rec_right(path, handle, et, split_rec, + split_index); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * We can only get this from logic error above. + */ + BUG_ON(!ocfs2_is_empty_extent(&el->l_recs[0])); + + /* extend credit for ocfs2_remove_rightmost_path */ + ret = ocfs2_extend_rotate_transaction(handle, 0, + jbd2_handle_buffer_credits(handle), + path); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* The merge left us with an empty extent, remove it. */ + ret = ocfs2_rotate_tree_left(handle, et, path, dealloc); + if (ret) { + mlog_errno(ret); + goto out; + } + + rec = &el->l_recs[split_index]; + + /* + * Note that we don't pass split_rec here on purpose - + * we've merged it into the rec already. + */ + ret = ocfs2_merge_rec_left(path, handle, et, rec, + dealloc, split_index); + + if (ret) { + mlog_errno(ret); + goto out; + } + + /* extend credit for ocfs2_remove_rightmost_path */ + ret = ocfs2_extend_rotate_transaction(handle, 0, + jbd2_handle_buffer_credits(handle), + path); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_rotate_tree_left(handle, et, path, dealloc); + /* + * Error from this last rotate is not critical, so + * print but don't bubble it up. + */ + if (ret) + mlog_errno(ret); + ret = 0; + } else { + /* + * Merge a record to the left or right. + * + * 'contig_type' is relative to the existing record, + * so for example, if we're "right contig", it's to + * the record on the left (hence the left merge). + */ + if (ctxt->c_contig_type == CONTIG_RIGHT) { + ret = ocfs2_merge_rec_left(path, handle, et, + split_rec, dealloc, + split_index); + if (ret) { + mlog_errno(ret); + goto out; + } + } else { + ret = ocfs2_merge_rec_right(path, handle, + et, split_rec, + split_index); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + if (ctxt->c_split_covers_rec) { + /* extend credit for ocfs2_remove_rightmost_path */ + ret = ocfs2_extend_rotate_transaction(handle, 0, + jbd2_handle_buffer_credits(handle), + path); + if (ret) { + mlog_errno(ret); + ret = 0; + goto out; + } + + /* + * The merge may have left an empty extent in + * our leaf. Try to rotate it away. + */ + ret = ocfs2_rotate_tree_left(handle, et, path, + dealloc); + if (ret) + mlog_errno(ret); + ret = 0; + } + } + +out: + return ret; +} + +static void ocfs2_subtract_from_rec(struct super_block *sb, + enum ocfs2_split_type split, + struct ocfs2_extent_rec *rec, + struct ocfs2_extent_rec *split_rec) +{ + u64 len_blocks; + + len_blocks = ocfs2_clusters_to_blocks(sb, + le16_to_cpu(split_rec->e_leaf_clusters)); + + if (split == SPLIT_LEFT) { + /* + * Region is on the left edge of the existing + * record. + */ + le32_add_cpu(&rec->e_cpos, + le16_to_cpu(split_rec->e_leaf_clusters)); + le64_add_cpu(&rec->e_blkno, len_blocks); + le16_add_cpu(&rec->e_leaf_clusters, + -le16_to_cpu(split_rec->e_leaf_clusters)); + } else { + /* + * Region is on the right edge of the existing + * record. + */ + le16_add_cpu(&rec->e_leaf_clusters, + -le16_to_cpu(split_rec->e_leaf_clusters)); + } +} + +/* + * Do the final bits of extent record insertion at the target leaf + * list. If this leaf is part of an allocation tree, it is assumed + * that the tree above has been prepared. + */ +static void ocfs2_insert_at_leaf(struct ocfs2_extent_tree *et, + struct ocfs2_extent_rec *insert_rec, + struct ocfs2_extent_list *el, + struct ocfs2_insert_type *insert) +{ + int i = insert->ins_contig_index; + unsigned int range; + struct ocfs2_extent_rec *rec; + + BUG_ON(le16_to_cpu(el->l_tree_depth) != 0); + + if (insert->ins_split != SPLIT_NONE) { + i = ocfs2_search_extent_list(el, le32_to_cpu(insert_rec->e_cpos)); + BUG_ON(i == -1); + rec = &el->l_recs[i]; + ocfs2_subtract_from_rec(ocfs2_metadata_cache_get_super(et->et_ci), + insert->ins_split, rec, + insert_rec); + goto rotate; + } + + /* + * Contiguous insert - either left or right. + */ + if (insert->ins_contig != CONTIG_NONE) { + rec = &el->l_recs[i]; + if (insert->ins_contig == CONTIG_LEFT) { + rec->e_blkno = insert_rec->e_blkno; + rec->e_cpos = insert_rec->e_cpos; + } + le16_add_cpu(&rec->e_leaf_clusters, + le16_to_cpu(insert_rec->e_leaf_clusters)); + return; + } + + /* + * Handle insert into an empty leaf. + */ + if (le16_to_cpu(el->l_next_free_rec) == 0 || + ((le16_to_cpu(el->l_next_free_rec) == 1) && + ocfs2_is_empty_extent(&el->l_recs[0]))) { + el->l_recs[0] = *insert_rec; + el->l_next_free_rec = cpu_to_le16(1); + return; + } + + /* + * Appending insert. + */ + if (insert->ins_appending == APPEND_TAIL) { + i = le16_to_cpu(el->l_next_free_rec) - 1; + rec = &el->l_recs[i]; + range = le32_to_cpu(rec->e_cpos) + + le16_to_cpu(rec->e_leaf_clusters); + BUG_ON(le32_to_cpu(insert_rec->e_cpos) < range); + + mlog_bug_on_msg(le16_to_cpu(el->l_next_free_rec) >= + le16_to_cpu(el->l_count), + "owner %llu, depth %u, count %u, next free %u, " + "rec.cpos %u, rec.clusters %u, " + "insert.cpos %u, insert.clusters %u\n", + ocfs2_metadata_cache_owner(et->et_ci), + le16_to_cpu(el->l_tree_depth), + le16_to_cpu(el->l_count), + le16_to_cpu(el->l_next_free_rec), + le32_to_cpu(el->l_recs[i].e_cpos), + le16_to_cpu(el->l_recs[i].e_leaf_clusters), + le32_to_cpu(insert_rec->e_cpos), + le16_to_cpu(insert_rec->e_leaf_clusters)); + i++; + el->l_recs[i] = *insert_rec; + le16_add_cpu(&el->l_next_free_rec, 1); + return; + } + +rotate: + /* + * Ok, we have to rotate. + * + * At this point, it is safe to assume that inserting into an + * empty leaf and appending to a leaf have both been handled + * above. + * + * This leaf needs to have space, either by the empty 1st + * extent record, or by virtue of an l_next_free_rec < l_count. + */ + ocfs2_rotate_leaf(el, insert_rec); +} + +static void ocfs2_adjust_rightmost_records(handle_t *handle, + struct ocfs2_extent_tree *et, + struct ocfs2_path *path, + struct ocfs2_extent_rec *insert_rec) +{ + int i, next_free; + struct buffer_head *bh; + struct ocfs2_extent_list *el; + struct ocfs2_extent_rec *rec; + + /* + * Update everything except the leaf block. + */ + for (i = 0; i < path->p_tree_depth; i++) { + bh = path->p_node[i].bh; + el = path->p_node[i].el; + + next_free = le16_to_cpu(el->l_next_free_rec); + if (next_free == 0) { + ocfs2_error(ocfs2_metadata_cache_get_super(et->et_ci), + "Owner %llu has a bad extent list\n", + (unsigned long long)ocfs2_metadata_cache_owner(et->et_ci)); + return; + } + + rec = &el->l_recs[next_free - 1]; + + rec->e_int_clusters = insert_rec->e_cpos; + le32_add_cpu(&rec->e_int_clusters, + le16_to_cpu(insert_rec->e_leaf_clusters)); + le32_add_cpu(&rec->e_int_clusters, + -le32_to_cpu(rec->e_cpos)); + + ocfs2_journal_dirty(handle, bh); + } +} + +static int ocfs2_append_rec_to_path(handle_t *handle, + struct ocfs2_extent_tree *et, + struct ocfs2_extent_rec *insert_rec, + struct ocfs2_path *right_path, + struct ocfs2_path **ret_left_path) +{ + int ret, next_free; + struct ocfs2_extent_list *el; + struct ocfs2_path *left_path = NULL; + + *ret_left_path = NULL; + + /* + * This shouldn't happen for non-trees. The extent rec cluster + * count manipulation below only works for interior nodes. + */ + BUG_ON(right_path->p_tree_depth == 0); + + /* + * If our appending insert is at the leftmost edge of a leaf, + * then we might need to update the rightmost records of the + * neighboring path. + */ + el = path_leaf_el(right_path); + next_free = le16_to_cpu(el->l_next_free_rec); + if (next_free == 0 || + (next_free == 1 && ocfs2_is_empty_extent(&el->l_recs[0]))) { + u32 left_cpos; + + ret = ocfs2_find_cpos_for_left_leaf(ocfs2_metadata_cache_get_super(et->et_ci), + right_path, &left_cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + + trace_ocfs2_append_rec_to_path( + (unsigned long long) + ocfs2_metadata_cache_owner(et->et_ci), + le32_to_cpu(insert_rec->e_cpos), + left_cpos); + + /* + * No need to worry if the append is already in the + * leftmost leaf. + */ + if (left_cpos) { + left_path = ocfs2_new_path_from_path(right_path); + if (!left_path) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + ret = ocfs2_find_path(et->et_ci, left_path, + left_cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * ocfs2_insert_path() will pass the left_path to the + * journal for us. + */ + } + } + + ret = ocfs2_journal_access_path(et->et_ci, handle, right_path); + if (ret) { + mlog_errno(ret); + goto out; + } + + ocfs2_adjust_rightmost_records(handle, et, right_path, insert_rec); + + *ret_left_path = left_path; + ret = 0; +out: + if (ret != 0) + ocfs2_free_path(left_path); + + return ret; +} + +static void ocfs2_split_record(struct ocfs2_extent_tree *et, + struct ocfs2_path *left_path, + struct ocfs2_path *right_path, + struct ocfs2_extent_rec *split_rec, + enum ocfs2_split_type split) +{ + int index; + u32 cpos = le32_to_cpu(split_rec->e_cpos); + struct ocfs2_extent_list *left_el = NULL, *right_el, *insert_el, *el; + struct ocfs2_extent_rec *rec, *tmprec; + + right_el = path_leaf_el(right_path); + if (left_path) + left_el = path_leaf_el(left_path); + + el = right_el; + insert_el = right_el; + index = ocfs2_search_extent_list(el, cpos); + if (index != -1) { + if (index == 0 && left_path) { + BUG_ON(ocfs2_is_empty_extent(&el->l_recs[0])); + + /* + * This typically means that the record + * started in the left path but moved to the + * right as a result of rotation. We either + * move the existing record to the left, or we + * do the later insert there. + * + * In this case, the left path should always + * exist as the rotate code will have passed + * it back for a post-insert update. + */ + + if (split == SPLIT_LEFT) { + /* + * It's a left split. Since we know + * that the rotate code gave us an + * empty extent in the left path, we + * can just do the insert there. + */ + insert_el = left_el; + } else { + /* + * Right split - we have to move the + * existing record over to the left + * leaf. The insert will be into the + * newly created empty extent in the + * right leaf. + */ + tmprec = &right_el->l_recs[index]; + ocfs2_rotate_leaf(left_el, tmprec); + el = left_el; + + memset(tmprec, 0, sizeof(*tmprec)); + index = ocfs2_search_extent_list(left_el, cpos); + BUG_ON(index == -1); + } + } + } else { + BUG_ON(!left_path); + BUG_ON(!ocfs2_is_empty_extent(&left_el->l_recs[0])); + /* + * Left path is easy - we can just allow the insert to + * happen. + */ + el = left_el; + insert_el = left_el; + index = ocfs2_search_extent_list(el, cpos); + BUG_ON(index == -1); + } + + rec = &el->l_recs[index]; + ocfs2_subtract_from_rec(ocfs2_metadata_cache_get_super(et->et_ci), + split, rec, split_rec); + ocfs2_rotate_leaf(insert_el, split_rec); +} + +/* + * This function only does inserts on an allocation b-tree. For tree + * depth = 0, ocfs2_insert_at_leaf() is called directly. + * + * right_path is the path we want to do the actual insert + * in. left_path should only be passed in if we need to update that + * portion of the tree after an edge insert. + */ +static int ocfs2_insert_path(handle_t *handle, + struct ocfs2_extent_tree *et, + struct ocfs2_path *left_path, + struct ocfs2_path *right_path, + struct ocfs2_extent_rec *insert_rec, + struct ocfs2_insert_type *insert) +{ + int ret, subtree_index; + struct buffer_head *leaf_bh = path_leaf_bh(right_path); + + if (left_path) { + /* + * There's a chance that left_path got passed back to + * us without being accounted for in the + * journal. Extend our transaction here to be sure we + * can change those blocks. + */ + ret = ocfs2_extend_trans(handle, left_path->p_tree_depth); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_journal_access_path(et->et_ci, handle, left_path); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + } + + /* + * Pass both paths to the journal. The majority of inserts + * will be touching all components anyway. + */ + ret = ocfs2_journal_access_path(et->et_ci, handle, right_path); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + if (insert->ins_split != SPLIT_NONE) { + /* + * We could call ocfs2_insert_at_leaf() for some types + * of splits, but it's easier to just let one separate + * function sort it all out. + */ + ocfs2_split_record(et, left_path, right_path, + insert_rec, insert->ins_split); + + /* + * Split might have modified either leaf and we don't + * have a guarantee that the later edge insert will + * dirty this for us. + */ + if (left_path) + ocfs2_journal_dirty(handle, + path_leaf_bh(left_path)); + } else + ocfs2_insert_at_leaf(et, insert_rec, path_leaf_el(right_path), + insert); + + ocfs2_journal_dirty(handle, leaf_bh); + + if (left_path) { + /* + * The rotate code has indicated that we need to fix + * up portions of the tree after the insert. + * + * XXX: Should we extend the transaction here? + */ + subtree_index = ocfs2_find_subtree_root(et, left_path, + right_path); + ocfs2_complete_edge_insert(handle, left_path, right_path, + subtree_index); + } + + ret = 0; +out: + return ret; +} + +static int ocfs2_do_insert_extent(handle_t *handle, + struct ocfs2_extent_tree *et, + struct ocfs2_extent_rec *insert_rec, + struct ocfs2_insert_type *type) +{ + int ret, rotate = 0; + u32 cpos; + struct ocfs2_path *right_path = NULL; + struct ocfs2_path *left_path = NULL; + struct ocfs2_extent_list *el; + + el = et->et_root_el; + + ret = ocfs2_et_root_journal_access(handle, et, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (le16_to_cpu(el->l_tree_depth) == 0) { + ocfs2_insert_at_leaf(et, insert_rec, el, type); + goto out_update_clusters; + } + + right_path = ocfs2_new_path_from_et(et); + if (!right_path) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + /* + * Determine the path to start with. Rotations need the + * rightmost path, everything else can go directly to the + * target leaf. + */ + cpos = le32_to_cpu(insert_rec->e_cpos); + if (type->ins_appending == APPEND_NONE && + type->ins_contig == CONTIG_NONE) { + rotate = 1; + cpos = UINT_MAX; + } + + ret = ocfs2_find_path(et->et_ci, right_path, cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * Rotations and appends need special treatment - they modify + * parts of the tree's above them. + * + * Both might pass back a path immediate to the left of the + * one being inserted to. This will be cause + * ocfs2_insert_path() to modify the rightmost records of + * left_path to account for an edge insert. + * + * XXX: When modifying this code, keep in mind that an insert + * can wind up skipping both of these two special cases... + */ + if (rotate) { + ret = ocfs2_rotate_tree_right(handle, et, type->ins_split, + le32_to_cpu(insert_rec->e_cpos), + right_path, &left_path); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * ocfs2_rotate_tree_right() might have extended the + * transaction without re-journaling our tree root. + */ + ret = ocfs2_et_root_journal_access(handle, et, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + } else if (type->ins_appending == APPEND_TAIL + && type->ins_contig != CONTIG_LEFT) { + ret = ocfs2_append_rec_to_path(handle, et, insert_rec, + right_path, &left_path); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + ret = ocfs2_insert_path(handle, et, left_path, right_path, + insert_rec, type); + if (ret) { + mlog_errno(ret); + goto out; + } + +out_update_clusters: + if (type->ins_split == SPLIT_NONE) + ocfs2_et_update_clusters(et, + le16_to_cpu(insert_rec->e_leaf_clusters)); + + ocfs2_journal_dirty(handle, et->et_root_bh); + +out: + ocfs2_free_path(left_path); + ocfs2_free_path(right_path); + + return ret; +} + +static int ocfs2_figure_merge_contig_type(struct ocfs2_extent_tree *et, + struct ocfs2_path *path, + struct ocfs2_extent_list *el, int index, + struct ocfs2_extent_rec *split_rec, + struct ocfs2_merge_ctxt *ctxt) +{ + int status = 0; + enum ocfs2_contig_type ret = CONTIG_NONE; + u32 left_cpos, right_cpos; + struct ocfs2_extent_rec *rec = NULL; + struct ocfs2_extent_list *new_el; + struct ocfs2_path *left_path = NULL, *right_path = NULL; + struct buffer_head *bh; + struct ocfs2_extent_block *eb; + struct super_block *sb = ocfs2_metadata_cache_get_super(et->et_ci); + + if (index > 0) { + rec = &el->l_recs[index - 1]; + } else if (path->p_tree_depth > 0) { + status = ocfs2_find_cpos_for_left_leaf(sb, path, &left_cpos); + if (status) + goto exit; + + if (left_cpos != 0) { + left_path = ocfs2_new_path_from_path(path); + if (!left_path) { + status = -ENOMEM; + mlog_errno(status); + goto exit; + } + + status = ocfs2_find_path(et->et_ci, left_path, + left_cpos); + if (status) + goto free_left_path; + + new_el = path_leaf_el(left_path); + + if (le16_to_cpu(new_el->l_next_free_rec) != + le16_to_cpu(new_el->l_count)) { + bh = path_leaf_bh(left_path); + eb = (struct ocfs2_extent_block *)bh->b_data; + status = ocfs2_error(sb, + "Extent block #%llu has an invalid l_next_free_rec of %d. It should have matched the l_count of %d\n", + (unsigned long long)le64_to_cpu(eb->h_blkno), + le16_to_cpu(new_el->l_next_free_rec), + le16_to_cpu(new_el->l_count)); + goto free_left_path; + } + rec = &new_el->l_recs[ + le16_to_cpu(new_el->l_next_free_rec) - 1]; + } + } + + /* + * We're careful to check for an empty extent record here - + * the merge code will know what to do if it sees one. + */ + if (rec) { + if (index == 1 && ocfs2_is_empty_extent(rec)) { + if (split_rec->e_cpos == el->l_recs[index].e_cpos) + ret = CONTIG_RIGHT; + } else { + ret = ocfs2_et_extent_contig(et, rec, split_rec); + } + } + + rec = NULL; + if (index < (le16_to_cpu(el->l_next_free_rec) - 1)) + rec = &el->l_recs[index + 1]; + else if (le16_to_cpu(el->l_next_free_rec) == le16_to_cpu(el->l_count) && + path->p_tree_depth > 0) { + status = ocfs2_find_cpos_for_right_leaf(sb, path, &right_cpos); + if (status) + goto free_left_path; + + if (right_cpos == 0) + goto free_left_path; + + right_path = ocfs2_new_path_from_path(path); + if (!right_path) { + status = -ENOMEM; + mlog_errno(status); + goto free_left_path; + } + + status = ocfs2_find_path(et->et_ci, right_path, right_cpos); + if (status) + goto free_right_path; + + new_el = path_leaf_el(right_path); + rec = &new_el->l_recs[0]; + if (ocfs2_is_empty_extent(rec)) { + if (le16_to_cpu(new_el->l_next_free_rec) <= 1) { + bh = path_leaf_bh(right_path); + eb = (struct ocfs2_extent_block *)bh->b_data; + status = ocfs2_error(sb, + "Extent block #%llu has an invalid l_next_free_rec of %d\n", + (unsigned long long)le64_to_cpu(eb->h_blkno), + le16_to_cpu(new_el->l_next_free_rec)); + goto free_right_path; + } + rec = &new_el->l_recs[1]; + } + } + + if (rec) { + enum ocfs2_contig_type contig_type; + + contig_type = ocfs2_et_extent_contig(et, rec, split_rec); + + if (contig_type == CONTIG_LEFT && ret == CONTIG_RIGHT) + ret = CONTIG_LEFTRIGHT; + else if (ret == CONTIG_NONE) + ret = contig_type; + } + +free_right_path: + ocfs2_free_path(right_path); +free_left_path: + ocfs2_free_path(left_path); +exit: + if (status == 0) + ctxt->c_contig_type = ret; + + return status; +} + +static void ocfs2_figure_contig_type(struct ocfs2_extent_tree *et, + struct ocfs2_insert_type *insert, + struct ocfs2_extent_list *el, + struct ocfs2_extent_rec *insert_rec) +{ + int i; + enum ocfs2_contig_type contig_type = CONTIG_NONE; + + BUG_ON(le16_to_cpu(el->l_tree_depth) != 0); + + for(i = 0; i < le16_to_cpu(el->l_next_free_rec); i++) { + contig_type = ocfs2_et_extent_contig(et, &el->l_recs[i], + insert_rec); + if (contig_type != CONTIG_NONE) { + insert->ins_contig_index = i; + break; + } + } + insert->ins_contig = contig_type; + + if (insert->ins_contig != CONTIG_NONE) { + struct ocfs2_extent_rec *rec = + &el->l_recs[insert->ins_contig_index]; + unsigned int len = le16_to_cpu(rec->e_leaf_clusters) + + le16_to_cpu(insert_rec->e_leaf_clusters); + + /* + * Caller might want us to limit the size of extents, don't + * calculate contiguousness if we might exceed that limit. + */ + if (et->et_max_leaf_clusters && + (len > et->et_max_leaf_clusters)) + insert->ins_contig = CONTIG_NONE; + } +} + +/* + * This should only be called against the rightmost leaf extent list. + * + * ocfs2_figure_appending_type() will figure out whether we'll have to + * insert at the tail of the rightmost leaf. + * + * This should also work against the root extent list for tree's with 0 + * depth. If we consider the root extent list to be the rightmost leaf node + * then the logic here makes sense. + */ +static void ocfs2_figure_appending_type(struct ocfs2_insert_type *insert, + struct ocfs2_extent_list *el, + struct ocfs2_extent_rec *insert_rec) +{ + int i; + u32 cpos = le32_to_cpu(insert_rec->e_cpos); + struct ocfs2_extent_rec *rec; + + insert->ins_appending = APPEND_NONE; + + BUG_ON(le16_to_cpu(el->l_tree_depth) != 0); + + if (!el->l_next_free_rec) + goto set_tail_append; + + if (ocfs2_is_empty_extent(&el->l_recs[0])) { + /* Were all records empty? */ + if (le16_to_cpu(el->l_next_free_rec) == 1) + goto set_tail_append; + } + + i = le16_to_cpu(el->l_next_free_rec) - 1; + rec = &el->l_recs[i]; + + if (cpos >= + (le32_to_cpu(rec->e_cpos) + le16_to_cpu(rec->e_leaf_clusters))) + goto set_tail_append; + + return; + +set_tail_append: + insert->ins_appending = APPEND_TAIL; +} + +/* + * Helper function called at the beginning of an insert. + * + * This computes a few things that are commonly used in the process of + * inserting into the btree: + * - Whether the new extent is contiguous with an existing one. + * - The current tree depth. + * - Whether the insert is an appending one. + * - The total # of free records in the tree. + * + * All of the information is stored on the ocfs2_insert_type + * structure. + */ +static int ocfs2_figure_insert_type(struct ocfs2_extent_tree *et, + struct buffer_head **last_eb_bh, + struct ocfs2_extent_rec *insert_rec, + int *free_records, + struct ocfs2_insert_type *insert) +{ + int ret; + struct ocfs2_extent_block *eb; + struct ocfs2_extent_list *el; + struct ocfs2_path *path = NULL; + struct buffer_head *bh = NULL; + + insert->ins_split = SPLIT_NONE; + + el = et->et_root_el; + insert->ins_tree_depth = le16_to_cpu(el->l_tree_depth); + + if (el->l_tree_depth) { + /* + * If we have tree depth, we read in the + * rightmost extent block ahead of time as + * ocfs2_figure_insert_type() and ocfs2_add_branch() + * may want it later. + */ + ret = ocfs2_read_extent_block(et->et_ci, + ocfs2_et_get_last_eb_blk(et), + &bh); + if (ret) { + mlog_errno(ret); + goto out; + } + eb = (struct ocfs2_extent_block *) bh->b_data; + el = &eb->h_list; + } + + /* + * Unless we have a contiguous insert, we'll need to know if + * there is room left in our allocation tree for another + * extent record. + * + * XXX: This test is simplistic, we can search for empty + * extent records too. + */ + *free_records = le16_to_cpu(el->l_count) - + le16_to_cpu(el->l_next_free_rec); + + if (!insert->ins_tree_depth) { + ocfs2_figure_contig_type(et, insert, el, insert_rec); + ocfs2_figure_appending_type(insert, el, insert_rec); + return 0; + } + + path = ocfs2_new_path_from_et(et); + if (!path) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + /* + * In the case that we're inserting past what the tree + * currently accounts for, ocfs2_find_path() will return for + * us the rightmost tree path. This is accounted for below in + * the appending code. + */ + ret = ocfs2_find_path(et->et_ci, path, le32_to_cpu(insert_rec->e_cpos)); + if (ret) { + mlog_errno(ret); + goto out; + } + + el = path_leaf_el(path); + + /* + * Now that we have the path, there's two things we want to determine: + * 1) Contiguousness (also set contig_index if this is so) + * + * 2) Are we doing an append? We can trivially break this up + * into two types of appends: simple record append, or a + * rotate inside the tail leaf. + */ + ocfs2_figure_contig_type(et, insert, el, insert_rec); + + /* + * The insert code isn't quite ready to deal with all cases of + * left contiguousness. Specifically, if it's an insert into + * the 1st record in a leaf, it will require the adjustment of + * cluster count on the last record of the path directly to it's + * left. For now, just catch that case and fool the layers + * above us. This works just fine for tree_depth == 0, which + * is why we allow that above. + */ + if (insert->ins_contig == CONTIG_LEFT && + insert->ins_contig_index == 0) + insert->ins_contig = CONTIG_NONE; + + /* + * Ok, so we can simply compare against last_eb to figure out + * whether the path doesn't exist. This will only happen in + * the case that we're doing a tail append, so maybe we can + * take advantage of that information somehow. + */ + if (ocfs2_et_get_last_eb_blk(et) == + path_leaf_bh(path)->b_blocknr) { + /* + * Ok, ocfs2_find_path() returned us the rightmost + * tree path. This might be an appending insert. There are + * two cases: + * 1) We're doing a true append at the tail: + * -This might even be off the end of the leaf + * 2) We're "appending" by rotating in the tail + */ + ocfs2_figure_appending_type(insert, el, insert_rec); + } + +out: + ocfs2_free_path(path); + + if (ret == 0) + *last_eb_bh = bh; + else + brelse(bh); + return ret; +} + +/* + * Insert an extent into a btree. + * + * The caller needs to update the owning btree's cluster count. + */ +int ocfs2_insert_extent(handle_t *handle, + struct ocfs2_extent_tree *et, + u32 cpos, + u64 start_blk, + u32 new_clusters, + u8 flags, + struct ocfs2_alloc_context *meta_ac) +{ + int status; + int free_records; + struct buffer_head *last_eb_bh = NULL; + struct ocfs2_insert_type insert = {0, }; + struct ocfs2_extent_rec rec; + + trace_ocfs2_insert_extent_start( + (unsigned long long)ocfs2_metadata_cache_owner(et->et_ci), + cpos, new_clusters); + + memset(&rec, 0, sizeof(rec)); + rec.e_cpos = cpu_to_le32(cpos); + rec.e_blkno = cpu_to_le64(start_blk); + rec.e_leaf_clusters = cpu_to_le16(new_clusters); + rec.e_flags = flags; + status = ocfs2_et_insert_check(et, &rec); + if (status) { + mlog_errno(status); + goto bail; + } + + status = ocfs2_figure_insert_type(et, &last_eb_bh, &rec, + &free_records, &insert); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + trace_ocfs2_insert_extent(insert.ins_appending, insert.ins_contig, + insert.ins_contig_index, free_records, + insert.ins_tree_depth); + + if (insert.ins_contig == CONTIG_NONE && free_records == 0) { + status = ocfs2_grow_tree(handle, et, + &insert.ins_tree_depth, &last_eb_bh, + meta_ac); + if (status) { + mlog_errno(status); + goto bail; + } + } + + /* Finally, we can add clusters. This might rotate the tree for us. */ + status = ocfs2_do_insert_extent(handle, et, &rec, &insert); + if (status < 0) + mlog_errno(status); + else + ocfs2_et_extent_map_insert(et, &rec); + +bail: + brelse(last_eb_bh); + + return status; +} + +/* + * Allocate and add clusters into the extent b-tree. + * The new clusters(clusters_to_add) will be inserted at logical_offset. + * The extent b-tree's root is specified by et, and + * it is not limited to the file storage. Any extent tree can use this + * function if it implements the proper ocfs2_extent_tree. + */ +int ocfs2_add_clusters_in_btree(handle_t *handle, + struct ocfs2_extent_tree *et, + u32 *logical_offset, + u32 clusters_to_add, + int mark_unwritten, + struct ocfs2_alloc_context *data_ac, + struct ocfs2_alloc_context *meta_ac, + enum ocfs2_alloc_restarted *reason_ret) +{ + int status = 0, err = 0; + int need_free = 0; + int free_extents; + enum ocfs2_alloc_restarted reason = RESTART_NONE; + u32 bit_off, num_bits; + u64 block; + u8 flags = 0; + struct ocfs2_super *osb = + OCFS2_SB(ocfs2_metadata_cache_get_super(et->et_ci)); + + BUG_ON(!clusters_to_add); + + if (mark_unwritten) + flags = OCFS2_EXT_UNWRITTEN; + + free_extents = ocfs2_num_free_extents(et); + if (free_extents < 0) { + status = free_extents; + mlog_errno(status); + goto leave; + } + + /* there are two cases which could cause us to EAGAIN in the + * we-need-more-metadata case: + * 1) we haven't reserved *any* + * 2) we are so fragmented, we've needed to add metadata too + * many times. */ + if (!free_extents && !meta_ac) { + err = -1; + status = -EAGAIN; + reason = RESTART_META; + goto leave; + } else if ((!free_extents) + && (ocfs2_alloc_context_bits_left(meta_ac) + < ocfs2_extend_meta_needed(et->et_root_el))) { + err = -2; + status = -EAGAIN; + reason = RESTART_META; + goto leave; + } + + status = __ocfs2_claim_clusters(handle, data_ac, 1, + clusters_to_add, &bit_off, &num_bits); + if (status < 0) { + if (status != -ENOSPC) + mlog_errno(status); + goto leave; + } + + BUG_ON(num_bits > clusters_to_add); + + /* reserve our write early -- insert_extent may update the tree root */ + status = ocfs2_et_root_journal_access(handle, et, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + need_free = 1; + goto bail; + } + + block = ocfs2_clusters_to_blocks(osb->sb, bit_off); + trace_ocfs2_add_clusters_in_btree( + (unsigned long long)ocfs2_metadata_cache_owner(et->et_ci), + bit_off, num_bits); + status = ocfs2_insert_extent(handle, et, *logical_offset, block, + num_bits, flags, meta_ac); + if (status < 0) { + mlog_errno(status); + need_free = 1; + goto bail; + } + + ocfs2_journal_dirty(handle, et->et_root_bh); + + clusters_to_add -= num_bits; + *logical_offset += num_bits; + + if (clusters_to_add) { + err = clusters_to_add; + status = -EAGAIN; + reason = RESTART_TRANS; + } + +bail: + if (need_free) { + if (data_ac->ac_which == OCFS2_AC_USE_LOCAL) + ocfs2_free_local_alloc_bits(osb, handle, data_ac, + bit_off, num_bits); + else + ocfs2_free_clusters(handle, + data_ac->ac_inode, + data_ac->ac_bh, + ocfs2_clusters_to_blocks(osb->sb, bit_off), + num_bits); + } + +leave: + if (reason_ret) + *reason_ret = reason; + trace_ocfs2_add_clusters_in_btree_ret(status, reason, err); + return status; +} + +static void ocfs2_make_right_split_rec(struct super_block *sb, + struct ocfs2_extent_rec *split_rec, + u32 cpos, + struct ocfs2_extent_rec *rec) +{ + u32 rec_cpos = le32_to_cpu(rec->e_cpos); + u32 rec_range = rec_cpos + le16_to_cpu(rec->e_leaf_clusters); + + memset(split_rec, 0, sizeof(struct ocfs2_extent_rec)); + + split_rec->e_cpos = cpu_to_le32(cpos); + split_rec->e_leaf_clusters = cpu_to_le16(rec_range - cpos); + + split_rec->e_blkno = rec->e_blkno; + le64_add_cpu(&split_rec->e_blkno, + ocfs2_clusters_to_blocks(sb, cpos - rec_cpos)); + + split_rec->e_flags = rec->e_flags; +} + +static int ocfs2_split_and_insert(handle_t *handle, + struct ocfs2_extent_tree *et, + struct ocfs2_path *path, + struct buffer_head **last_eb_bh, + int split_index, + struct ocfs2_extent_rec *orig_split_rec, + struct ocfs2_alloc_context *meta_ac) +{ + int ret = 0, depth; + unsigned int insert_range, rec_range, do_leftright = 0; + struct ocfs2_extent_rec tmprec; + struct ocfs2_extent_list *rightmost_el; + struct ocfs2_extent_rec rec; + struct ocfs2_extent_rec split_rec = *orig_split_rec; + struct ocfs2_insert_type insert; + struct ocfs2_extent_block *eb; + +leftright: + /* + * Store a copy of the record on the stack - it might move + * around as the tree is manipulated below. + */ + rec = path_leaf_el(path)->l_recs[split_index]; + + rightmost_el = et->et_root_el; + + depth = le16_to_cpu(rightmost_el->l_tree_depth); + if (depth) { + BUG_ON(!(*last_eb_bh)); + eb = (struct ocfs2_extent_block *) (*last_eb_bh)->b_data; + rightmost_el = &eb->h_list; + } + + if (le16_to_cpu(rightmost_el->l_next_free_rec) == + le16_to_cpu(rightmost_el->l_count)) { + ret = ocfs2_grow_tree(handle, et, + &depth, last_eb_bh, meta_ac); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + memset(&insert, 0, sizeof(struct ocfs2_insert_type)); + insert.ins_appending = APPEND_NONE; + insert.ins_contig = CONTIG_NONE; + insert.ins_tree_depth = depth; + + insert_range = le32_to_cpu(split_rec.e_cpos) + + le16_to_cpu(split_rec.e_leaf_clusters); + rec_range = le32_to_cpu(rec.e_cpos) + + le16_to_cpu(rec.e_leaf_clusters); + + if (split_rec.e_cpos == rec.e_cpos) { + insert.ins_split = SPLIT_LEFT; + } else if (insert_range == rec_range) { + insert.ins_split = SPLIT_RIGHT; + } else { + /* + * Left/right split. We fake this as a right split + * first and then make a second pass as a left split. + */ + insert.ins_split = SPLIT_RIGHT; + + ocfs2_make_right_split_rec(ocfs2_metadata_cache_get_super(et->et_ci), + &tmprec, insert_range, &rec); + + split_rec = tmprec; + + BUG_ON(do_leftright); + do_leftright = 1; + } + + ret = ocfs2_do_insert_extent(handle, et, &split_rec, &insert); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (do_leftright == 1) { + u32 cpos; + struct ocfs2_extent_list *el; + + do_leftright++; + split_rec = *orig_split_rec; + + ocfs2_reinit_path(path, 1); + + cpos = le32_to_cpu(split_rec.e_cpos); + ret = ocfs2_find_path(et->et_ci, path, cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + + el = path_leaf_el(path); + split_index = ocfs2_search_extent_list(el, cpos); + if (split_index == -1) { + ocfs2_error(ocfs2_metadata_cache_get_super(et->et_ci), + "Owner %llu has an extent at cpos %u which can no longer be found\n", + (unsigned long long)ocfs2_metadata_cache_owner(et->et_ci), + cpos); + ret = -EROFS; + goto out; + } + goto leftright; + } +out: + + return ret; +} + +static int ocfs2_replace_extent_rec(handle_t *handle, + struct ocfs2_extent_tree *et, + struct ocfs2_path *path, + struct ocfs2_extent_list *el, + int split_index, + struct ocfs2_extent_rec *split_rec) +{ + int ret; + + ret = ocfs2_path_bh_journal_access(handle, et->et_ci, path, + path_num_items(path) - 1); + if (ret) { + mlog_errno(ret); + goto out; + } + + el->l_recs[split_index] = *split_rec; + + ocfs2_journal_dirty(handle, path_leaf_bh(path)); +out: + return ret; +} + +/* + * Split part or all of the extent record at split_index in the leaf + * pointed to by path. Merge with the contiguous extent record if needed. + * + * Care is taken to handle contiguousness so as to not grow the tree. + * + * meta_ac is not strictly necessary - we only truly need it if growth + * of the tree is required. All other cases will degrade into a less + * optimal tree layout. + * + * last_eb_bh should be the rightmost leaf block for any extent + * btree. Since a split may grow the tree or a merge might shrink it, + * the caller cannot trust the contents of that buffer after this call. + * + * This code is optimized for readability - several passes might be + * made over certain portions of the tree. All of those blocks will + * have been brought into cache (and pinned via the journal), so the + * extra overhead is not expressed in terms of disk reads. + */ +int ocfs2_split_extent(handle_t *handle, + struct ocfs2_extent_tree *et, + struct ocfs2_path *path, + int split_index, + struct ocfs2_extent_rec *split_rec, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_cached_dealloc_ctxt *dealloc) +{ + int ret = 0; + struct ocfs2_extent_list *el = path_leaf_el(path); + struct buffer_head *last_eb_bh = NULL; + struct ocfs2_extent_rec *rec = &el->l_recs[split_index]; + struct ocfs2_merge_ctxt ctxt; + + if (le32_to_cpu(rec->e_cpos) > le32_to_cpu(split_rec->e_cpos) || + ((le32_to_cpu(rec->e_cpos) + le16_to_cpu(rec->e_leaf_clusters)) < + (le32_to_cpu(split_rec->e_cpos) + le16_to_cpu(split_rec->e_leaf_clusters)))) { + ret = -EIO; + mlog_errno(ret); + goto out; + } + + ret = ocfs2_figure_merge_contig_type(et, path, el, + split_index, + split_rec, + &ctxt); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * The core merge / split code wants to know how much room is + * left in this allocation tree, so we pass the + * rightmost extent list. + */ + if (path->p_tree_depth) { + ret = ocfs2_read_extent_block(et->et_ci, + ocfs2_et_get_last_eb_blk(et), + &last_eb_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + if (rec->e_cpos == split_rec->e_cpos && + rec->e_leaf_clusters == split_rec->e_leaf_clusters) + ctxt.c_split_covers_rec = 1; + else + ctxt.c_split_covers_rec = 0; + + ctxt.c_has_empty_extent = ocfs2_is_empty_extent(&el->l_recs[0]); + + trace_ocfs2_split_extent(split_index, ctxt.c_contig_type, + ctxt.c_has_empty_extent, + ctxt.c_split_covers_rec); + + if (ctxt.c_contig_type == CONTIG_NONE) { + if (ctxt.c_split_covers_rec) + ret = ocfs2_replace_extent_rec(handle, et, path, el, + split_index, split_rec); + else + ret = ocfs2_split_and_insert(handle, et, path, + &last_eb_bh, split_index, + split_rec, meta_ac); + if (ret) + mlog_errno(ret); + } else { + ret = ocfs2_try_to_merge_extent(handle, et, path, + split_index, split_rec, + dealloc, &ctxt); + if (ret) + mlog_errno(ret); + } + +out: + brelse(last_eb_bh); + return ret; +} + +/* + * Change the flags of the already-existing extent at cpos for len clusters. + * + * new_flags: the flags we want to set. + * clear_flags: the flags we want to clear. + * phys: the new physical offset we want this new extent starts from. + * + * If the existing extent is larger than the request, initiate a + * split. An attempt will be made at merging with adjacent extents. + * + * The caller is responsible for passing down meta_ac if we'll need it. + */ +int ocfs2_change_extent_flag(handle_t *handle, + struct ocfs2_extent_tree *et, + u32 cpos, u32 len, u32 phys, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_cached_dealloc_ctxt *dealloc, + int new_flags, int clear_flags) +{ + int ret, index; + struct super_block *sb = ocfs2_metadata_cache_get_super(et->et_ci); + u64 start_blkno = ocfs2_clusters_to_blocks(sb, phys); + struct ocfs2_extent_rec split_rec; + struct ocfs2_path *left_path = NULL; + struct ocfs2_extent_list *el; + struct ocfs2_extent_rec *rec; + + left_path = ocfs2_new_path_from_et(et); + if (!left_path) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + ret = ocfs2_find_path(et->et_ci, left_path, cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + el = path_leaf_el(left_path); + + index = ocfs2_search_extent_list(el, cpos); + if (index == -1) { + ocfs2_error(sb, + "Owner %llu has an extent at cpos %u which can no longer be found\n", + (unsigned long long)ocfs2_metadata_cache_owner(et->et_ci), + cpos); + ret = -EROFS; + goto out; + } + + ret = -EIO; + rec = &el->l_recs[index]; + if (new_flags && (rec->e_flags & new_flags)) { + mlog(ML_ERROR, "Owner %llu tried to set %d flags on an " + "extent that already had them\n", + (unsigned long long)ocfs2_metadata_cache_owner(et->et_ci), + new_flags); + goto out; + } + + if (clear_flags && !(rec->e_flags & clear_flags)) { + mlog(ML_ERROR, "Owner %llu tried to clear %d flags on an " + "extent that didn't have them\n", + (unsigned long long)ocfs2_metadata_cache_owner(et->et_ci), + clear_flags); + goto out; + } + + memset(&split_rec, 0, sizeof(struct ocfs2_extent_rec)); + split_rec.e_cpos = cpu_to_le32(cpos); + split_rec.e_leaf_clusters = cpu_to_le16(len); + split_rec.e_blkno = cpu_to_le64(start_blkno); + split_rec.e_flags = rec->e_flags; + if (new_flags) + split_rec.e_flags |= new_flags; + if (clear_flags) + split_rec.e_flags &= ~clear_flags; + + ret = ocfs2_split_extent(handle, et, left_path, + index, &split_rec, meta_ac, + dealloc); + if (ret) + mlog_errno(ret); + +out: + ocfs2_free_path(left_path); + return ret; + +} + +/* + * Mark the already-existing extent at cpos as written for len clusters. + * This removes the unwritten extent flag. + * + * If the existing extent is larger than the request, initiate a + * split. An attempt will be made at merging with adjacent extents. + * + * The caller is responsible for passing down meta_ac if we'll need it. + */ +int ocfs2_mark_extent_written(struct inode *inode, + struct ocfs2_extent_tree *et, + handle_t *handle, u32 cpos, u32 len, u32 phys, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_cached_dealloc_ctxt *dealloc) +{ + int ret; + + trace_ocfs2_mark_extent_written( + (unsigned long long)OCFS2_I(inode)->ip_blkno, + cpos, len, phys); + + if (!ocfs2_writes_unwritten_extents(OCFS2_SB(inode->i_sb))) { + ocfs2_error(inode->i_sb, "Inode %llu has unwritten extents that are being written to, but the feature bit is not set in the super block\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno); + ret = -EROFS; + goto out; + } + + /* + * XXX: This should be fixed up so that we just re-insert the + * next extent records. + */ + ocfs2_et_extent_map_truncate(et, 0); + + ret = ocfs2_change_extent_flag(handle, et, cpos, + len, phys, meta_ac, dealloc, + 0, OCFS2_EXT_UNWRITTEN); + if (ret) + mlog_errno(ret); + +out: + return ret; +} + +static int ocfs2_split_tree(handle_t *handle, struct ocfs2_extent_tree *et, + struct ocfs2_path *path, + int index, u32 new_range, + struct ocfs2_alloc_context *meta_ac) +{ + int ret, depth, credits; + struct buffer_head *last_eb_bh = NULL; + struct ocfs2_extent_block *eb; + struct ocfs2_extent_list *rightmost_el, *el; + struct ocfs2_extent_rec split_rec; + struct ocfs2_extent_rec *rec; + struct ocfs2_insert_type insert; + + /* + * Setup the record to split before we grow the tree. + */ + el = path_leaf_el(path); + rec = &el->l_recs[index]; + ocfs2_make_right_split_rec(ocfs2_metadata_cache_get_super(et->et_ci), + &split_rec, new_range, rec); + + depth = path->p_tree_depth; + if (depth > 0) { + ret = ocfs2_read_extent_block(et->et_ci, + ocfs2_et_get_last_eb_blk(et), + &last_eb_bh); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + eb = (struct ocfs2_extent_block *) last_eb_bh->b_data; + rightmost_el = &eb->h_list; + } else + rightmost_el = path_leaf_el(path); + + credits = path->p_tree_depth + + ocfs2_extend_meta_needed(et->et_root_el); + ret = ocfs2_extend_trans(handle, credits); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (le16_to_cpu(rightmost_el->l_next_free_rec) == + le16_to_cpu(rightmost_el->l_count)) { + ret = ocfs2_grow_tree(handle, et, &depth, &last_eb_bh, + meta_ac); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + memset(&insert, 0, sizeof(struct ocfs2_insert_type)); + insert.ins_appending = APPEND_NONE; + insert.ins_contig = CONTIG_NONE; + insert.ins_split = SPLIT_RIGHT; + insert.ins_tree_depth = depth; + + ret = ocfs2_do_insert_extent(handle, et, &split_rec, &insert); + if (ret) + mlog_errno(ret); + +out: + brelse(last_eb_bh); + return ret; +} + +static int ocfs2_truncate_rec(handle_t *handle, + struct ocfs2_extent_tree *et, + struct ocfs2_path *path, int index, + struct ocfs2_cached_dealloc_ctxt *dealloc, + u32 cpos, u32 len) +{ + int ret; + u32 left_cpos, rec_range, trunc_range; + int is_rightmost_tree_rec = 0; + struct super_block *sb = ocfs2_metadata_cache_get_super(et->et_ci); + struct ocfs2_path *left_path = NULL; + struct ocfs2_extent_list *el = path_leaf_el(path); + struct ocfs2_extent_rec *rec; + struct ocfs2_extent_block *eb; + + if (ocfs2_is_empty_extent(&el->l_recs[0]) && index > 0) { + /* extend credit for ocfs2_remove_rightmost_path */ + ret = ocfs2_extend_rotate_transaction(handle, 0, + jbd2_handle_buffer_credits(handle), + path); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_rotate_tree_left(handle, et, path, dealloc); + if (ret) { + mlog_errno(ret); + goto out; + } + + index--; + } + + if (index == (le16_to_cpu(el->l_next_free_rec) - 1) && + path->p_tree_depth) { + /* + * Check whether this is the rightmost tree record. If + * we remove all of this record or part of its right + * edge then an update of the record lengths above it + * will be required. + */ + eb = (struct ocfs2_extent_block *)path_leaf_bh(path)->b_data; + if (eb->h_next_leaf_blk == 0) + is_rightmost_tree_rec = 1; + } + + rec = &el->l_recs[index]; + if (index == 0 && path->p_tree_depth && + le32_to_cpu(rec->e_cpos) == cpos) { + /* + * Changing the leftmost offset (via partial or whole + * record truncate) of an interior (or rightmost) path + * means we have to update the subtree that is formed + * by this leaf and the one to it's left. + * + * There are two cases we can skip: + * 1) Path is the leftmost one in our btree. + * 2) The leaf is rightmost and will be empty after + * we remove the extent record - the rotate code + * knows how to update the newly formed edge. + */ + + ret = ocfs2_find_cpos_for_left_leaf(sb, path, &left_cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (left_cpos && le16_to_cpu(el->l_next_free_rec) > 1) { + left_path = ocfs2_new_path_from_path(path); + if (!left_path) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + ret = ocfs2_find_path(et->et_ci, left_path, + left_cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + } + } + + ret = ocfs2_extend_rotate_transaction(handle, 0, + jbd2_handle_buffer_credits(handle), + path); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_journal_access_path(et->et_ci, handle, path); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_journal_access_path(et->et_ci, handle, left_path); + if (ret) { + mlog_errno(ret); + goto out; + } + + rec_range = le32_to_cpu(rec->e_cpos) + ocfs2_rec_clusters(el, rec); + trunc_range = cpos + len; + + if (le32_to_cpu(rec->e_cpos) == cpos && rec_range == trunc_range) { + int next_free; + + memset(rec, 0, sizeof(*rec)); + ocfs2_cleanup_merge(el, index); + + next_free = le16_to_cpu(el->l_next_free_rec); + if (is_rightmost_tree_rec && next_free > 1) { + /* + * We skip the edge update if this path will + * be deleted by the rotate code. + */ + rec = &el->l_recs[next_free - 1]; + ocfs2_adjust_rightmost_records(handle, et, path, + rec); + } + } else if (le32_to_cpu(rec->e_cpos) == cpos) { + /* Remove leftmost portion of the record. */ + le32_add_cpu(&rec->e_cpos, len); + le64_add_cpu(&rec->e_blkno, ocfs2_clusters_to_blocks(sb, len)); + le16_add_cpu(&rec->e_leaf_clusters, -len); + } else if (rec_range == trunc_range) { + /* Remove rightmost portion of the record */ + le16_add_cpu(&rec->e_leaf_clusters, -len); + if (is_rightmost_tree_rec) + ocfs2_adjust_rightmost_records(handle, et, path, rec); + } else { + /* Caller should have trapped this. */ + mlog(ML_ERROR, "Owner %llu: Invalid record truncate: (%u, %u) " + "(%u, %u)\n", + (unsigned long long)ocfs2_metadata_cache_owner(et->et_ci), + le32_to_cpu(rec->e_cpos), + le16_to_cpu(rec->e_leaf_clusters), cpos, len); + BUG(); + } + + if (left_path) { + int subtree_index; + + subtree_index = ocfs2_find_subtree_root(et, left_path, path); + ocfs2_complete_edge_insert(handle, left_path, path, + subtree_index); + } + + ocfs2_journal_dirty(handle, path_leaf_bh(path)); + + ret = ocfs2_rotate_tree_left(handle, et, path, dealloc); + if (ret) + mlog_errno(ret); + +out: + ocfs2_free_path(left_path); + return ret; +} + +int ocfs2_remove_extent(handle_t *handle, + struct ocfs2_extent_tree *et, + u32 cpos, u32 len, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_cached_dealloc_ctxt *dealloc) +{ + int ret, index; + u32 rec_range, trunc_range; + struct ocfs2_extent_rec *rec; + struct ocfs2_extent_list *el; + struct ocfs2_path *path = NULL; + + /* + * XXX: Why are we truncating to 0 instead of wherever this + * affects us? + */ + ocfs2_et_extent_map_truncate(et, 0); + + path = ocfs2_new_path_from_et(et); + if (!path) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + ret = ocfs2_find_path(et->et_ci, path, cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + + el = path_leaf_el(path); + index = ocfs2_search_extent_list(el, cpos); + if (index == -1) { + ocfs2_error(ocfs2_metadata_cache_get_super(et->et_ci), + "Owner %llu has an extent at cpos %u which can no longer be found\n", + (unsigned long long)ocfs2_metadata_cache_owner(et->et_ci), + cpos); + ret = -EROFS; + goto out; + } + + /* + * We have 3 cases of extent removal: + * 1) Range covers the entire extent rec + * 2) Range begins or ends on one edge of the extent rec + * 3) Range is in the middle of the extent rec (no shared edges) + * + * For case 1 we remove the extent rec and left rotate to + * fill the hole. + * + * For case 2 we just shrink the existing extent rec, with a + * tree update if the shrinking edge is also the edge of an + * extent block. + * + * For case 3 we do a right split to turn the extent rec into + * something case 2 can handle. + */ + rec = &el->l_recs[index]; + rec_range = le32_to_cpu(rec->e_cpos) + ocfs2_rec_clusters(el, rec); + trunc_range = cpos + len; + + BUG_ON(cpos < le32_to_cpu(rec->e_cpos) || trunc_range > rec_range); + + trace_ocfs2_remove_extent( + (unsigned long long)ocfs2_metadata_cache_owner(et->et_ci), + cpos, len, index, le32_to_cpu(rec->e_cpos), + ocfs2_rec_clusters(el, rec)); + + if (le32_to_cpu(rec->e_cpos) == cpos || rec_range == trunc_range) { + ret = ocfs2_truncate_rec(handle, et, path, index, dealloc, + cpos, len); + if (ret) { + mlog_errno(ret); + goto out; + } + } else { + ret = ocfs2_split_tree(handle, et, path, index, + trunc_range, meta_ac); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * The split could have manipulated the tree enough to + * move the record location, so we have to look for it again. + */ + ocfs2_reinit_path(path, 1); + + ret = ocfs2_find_path(et->et_ci, path, cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + + el = path_leaf_el(path); + index = ocfs2_search_extent_list(el, cpos); + if (index == -1) { + ocfs2_error(ocfs2_metadata_cache_get_super(et->et_ci), + "Owner %llu: split at cpos %u lost record\n", + (unsigned long long)ocfs2_metadata_cache_owner(et->et_ci), + cpos); + ret = -EROFS; + goto out; + } + + /* + * Double check our values here. If anything is fishy, + * it's easier to catch it at the top level. + */ + rec = &el->l_recs[index]; + rec_range = le32_to_cpu(rec->e_cpos) + + ocfs2_rec_clusters(el, rec); + if (rec_range != trunc_range) { + ocfs2_error(ocfs2_metadata_cache_get_super(et->et_ci), + "Owner %llu: error after split at cpos %u trunc len %u, existing record is (%u,%u)\n", + (unsigned long long)ocfs2_metadata_cache_owner(et->et_ci), + cpos, len, le32_to_cpu(rec->e_cpos), + ocfs2_rec_clusters(el, rec)); + ret = -EROFS; + goto out; + } + + ret = ocfs2_truncate_rec(handle, et, path, index, dealloc, + cpos, len); + if (ret) + mlog_errno(ret); + } + +out: + ocfs2_free_path(path); + return ret; +} + +/* + * ocfs2_reserve_blocks_for_rec_trunc() would look basically the + * same as ocfs2_lock_alloctors(), except for it accepts a blocks + * number to reserve some extra blocks, and it only handles meta + * data allocations. + * + * Currently, only ocfs2_remove_btree_range() uses it for truncating + * and punching holes. + */ +static int ocfs2_reserve_blocks_for_rec_trunc(struct inode *inode, + struct ocfs2_extent_tree *et, + u32 extents_to_split, + struct ocfs2_alloc_context **ac, + int extra_blocks) +{ + int ret = 0, num_free_extents; + unsigned int max_recs_needed = 2 * extents_to_split; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + *ac = NULL; + + num_free_extents = ocfs2_num_free_extents(et); + if (num_free_extents < 0) { + ret = num_free_extents; + mlog_errno(ret); + goto out; + } + + if (!num_free_extents || + (ocfs2_sparse_alloc(osb) && num_free_extents < max_recs_needed)) + extra_blocks += ocfs2_extend_meta_needed(et->et_root_el); + + if (extra_blocks) { + ret = ocfs2_reserve_new_metadata_blocks(osb, extra_blocks, ac); + if (ret < 0) { + if (ret != -ENOSPC) + mlog_errno(ret); + } + } + +out: + if (ret) { + if (*ac) { + ocfs2_free_alloc_context(*ac); + *ac = NULL; + } + } + + return ret; +} + +int ocfs2_remove_btree_range(struct inode *inode, + struct ocfs2_extent_tree *et, + u32 cpos, u32 phys_cpos, u32 len, int flags, + struct ocfs2_cached_dealloc_ctxt *dealloc, + u64 refcount_loc, bool refcount_tree_locked) +{ + int ret, credits = 0, extra_blocks = 0; + u64 phys_blkno = ocfs2_clusters_to_blocks(inode->i_sb, phys_cpos); + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct inode *tl_inode = osb->osb_tl_inode; + handle_t *handle; + struct ocfs2_alloc_context *meta_ac = NULL; + struct ocfs2_refcount_tree *ref_tree = NULL; + + if ((flags & OCFS2_EXT_REFCOUNTED) && len) { + BUG_ON(!ocfs2_is_refcount_inode(inode)); + + if (!refcount_tree_locked) { + ret = ocfs2_lock_refcount_tree(osb, refcount_loc, 1, + &ref_tree, NULL); + if (ret) { + mlog_errno(ret); + goto bail; + } + } + + ret = ocfs2_prepare_refcount_change_for_del(inode, + refcount_loc, + phys_blkno, + len, + &credits, + &extra_blocks); + if (ret < 0) { + mlog_errno(ret); + goto bail; + } + } + + ret = ocfs2_reserve_blocks_for_rec_trunc(inode, et, 1, &meta_ac, + extra_blocks); + if (ret) { + mlog_errno(ret); + goto bail; + } + + inode_lock(tl_inode); + + if (ocfs2_truncate_log_needs_flush(osb)) { + ret = __ocfs2_flush_truncate_log(osb); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + } + + handle = ocfs2_start_trans(osb, + ocfs2_remove_extent_credits(osb->sb) + credits); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out; + } + + ret = ocfs2_et_root_journal_access(handle, et, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + dquot_free_space_nodirty(inode, + ocfs2_clusters_to_bytes(inode->i_sb, len)); + + ret = ocfs2_remove_extent(handle, et, cpos, len, meta_ac, dealloc); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + ocfs2_et_update_clusters(et, -len); + ocfs2_update_inode_fsync_trans(handle, inode, 1); + + ocfs2_journal_dirty(handle, et->et_root_bh); + + if (phys_blkno) { + if (flags & OCFS2_EXT_REFCOUNTED) + ret = ocfs2_decrease_refcount(inode, handle, + ocfs2_blocks_to_clusters(osb->sb, + phys_blkno), + len, meta_ac, + dealloc, 1); + else + ret = ocfs2_truncate_log_append(osb, handle, + phys_blkno, len); + if (ret) + mlog_errno(ret); + + } + +out_commit: + ocfs2_commit_trans(osb, handle); +out: + inode_unlock(tl_inode); +bail: + if (meta_ac) + ocfs2_free_alloc_context(meta_ac); + + if (ref_tree) + ocfs2_unlock_refcount_tree(osb, ref_tree, 1); + + return ret; +} + +int ocfs2_truncate_log_needs_flush(struct ocfs2_super *osb) +{ + struct buffer_head *tl_bh = osb->osb_tl_bh; + struct ocfs2_dinode *di; + struct ocfs2_truncate_log *tl; + + di = (struct ocfs2_dinode *) tl_bh->b_data; + tl = &di->id2.i_dealloc; + + mlog_bug_on_msg(le16_to_cpu(tl->tl_used) > le16_to_cpu(tl->tl_count), + "slot %d, invalid truncate log parameters: used = " + "%u, count = %u\n", osb->slot_num, + le16_to_cpu(tl->tl_used), le16_to_cpu(tl->tl_count)); + return le16_to_cpu(tl->tl_used) == le16_to_cpu(tl->tl_count); +} + +static int ocfs2_truncate_log_can_coalesce(struct ocfs2_truncate_log *tl, + unsigned int new_start) +{ + unsigned int tail_index; + unsigned int current_tail; + + /* No records, nothing to coalesce */ + if (!le16_to_cpu(tl->tl_used)) + return 0; + + tail_index = le16_to_cpu(tl->tl_used) - 1; + current_tail = le32_to_cpu(tl->tl_recs[tail_index].t_start); + current_tail += le32_to_cpu(tl->tl_recs[tail_index].t_clusters); + + return current_tail == new_start; +} + +int ocfs2_truncate_log_append(struct ocfs2_super *osb, + handle_t *handle, + u64 start_blk, + unsigned int num_clusters) +{ + int status, index; + unsigned int start_cluster, tl_count; + struct inode *tl_inode = osb->osb_tl_inode; + struct buffer_head *tl_bh = osb->osb_tl_bh; + struct ocfs2_dinode *di; + struct ocfs2_truncate_log *tl; + + BUG_ON(inode_trylock(tl_inode)); + + start_cluster = ocfs2_blocks_to_clusters(osb->sb, start_blk); + + di = (struct ocfs2_dinode *) tl_bh->b_data; + + /* tl_bh is loaded from ocfs2_truncate_log_init(). It's validated + * by the underlying call to ocfs2_read_inode_block(), so any + * corruption is a code bug */ + BUG_ON(!OCFS2_IS_VALID_DINODE(di)); + + tl = &di->id2.i_dealloc; + tl_count = le16_to_cpu(tl->tl_count); + mlog_bug_on_msg(tl_count > ocfs2_truncate_recs_per_inode(osb->sb) || + tl_count == 0, + "Truncate record count on #%llu invalid " + "wanted %u, actual %u\n", + (unsigned long long)OCFS2_I(tl_inode)->ip_blkno, + ocfs2_truncate_recs_per_inode(osb->sb), + le16_to_cpu(tl->tl_count)); + + /* Caller should have known to flush before calling us. */ + index = le16_to_cpu(tl->tl_used); + if (index >= tl_count) { + status = -ENOSPC; + mlog_errno(status); + goto bail; + } + + status = ocfs2_journal_access_di(handle, INODE_CACHE(tl_inode), tl_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + trace_ocfs2_truncate_log_append( + (unsigned long long)OCFS2_I(tl_inode)->ip_blkno, index, + start_cluster, num_clusters); + if (ocfs2_truncate_log_can_coalesce(tl, start_cluster)) { + /* + * Move index back to the record we are coalescing with. + * ocfs2_truncate_log_can_coalesce() guarantees nonzero + */ + index--; + + num_clusters += le32_to_cpu(tl->tl_recs[index].t_clusters); + trace_ocfs2_truncate_log_append( + (unsigned long long)OCFS2_I(tl_inode)->ip_blkno, + index, le32_to_cpu(tl->tl_recs[index].t_start), + num_clusters); + } else { + tl->tl_recs[index].t_start = cpu_to_le32(start_cluster); + tl->tl_used = cpu_to_le16(index + 1); + } + tl->tl_recs[index].t_clusters = cpu_to_le32(num_clusters); + + ocfs2_journal_dirty(handle, tl_bh); + + osb->truncated_clusters += num_clusters; +bail: + return status; +} + +static int ocfs2_replay_truncate_records(struct ocfs2_super *osb, + struct inode *data_alloc_inode, + struct buffer_head *data_alloc_bh) +{ + int status = 0; + int i; + unsigned int num_clusters; + u64 start_blk; + struct ocfs2_truncate_rec rec; + struct ocfs2_dinode *di; + struct ocfs2_truncate_log *tl; + struct inode *tl_inode = osb->osb_tl_inode; + struct buffer_head *tl_bh = osb->osb_tl_bh; + handle_t *handle; + + di = (struct ocfs2_dinode *) tl_bh->b_data; + tl = &di->id2.i_dealloc; + i = le16_to_cpu(tl->tl_used) - 1; + while (i >= 0) { + handle = ocfs2_start_trans(osb, OCFS2_TRUNCATE_LOG_FLUSH_ONE_REC); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + mlog_errno(status); + goto bail; + } + + /* Caller has given us at least enough credits to + * update the truncate log dinode */ + status = ocfs2_journal_access_di(handle, INODE_CACHE(tl_inode), tl_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + ocfs2_commit_trans(osb, handle); + mlog_errno(status); + goto bail; + } + + tl->tl_used = cpu_to_le16(i); + + ocfs2_journal_dirty(handle, tl_bh); + + rec = tl->tl_recs[i]; + start_blk = ocfs2_clusters_to_blocks(data_alloc_inode->i_sb, + le32_to_cpu(rec.t_start)); + num_clusters = le32_to_cpu(rec.t_clusters); + + /* if start_blk is not set, we ignore the record as + * invalid. */ + if (start_blk) { + trace_ocfs2_replay_truncate_records( + (unsigned long long)OCFS2_I(tl_inode)->ip_blkno, + i, le32_to_cpu(rec.t_start), num_clusters); + + status = ocfs2_free_clusters(handle, data_alloc_inode, + data_alloc_bh, start_blk, + num_clusters); + if (status < 0) { + ocfs2_commit_trans(osb, handle); + mlog_errno(status); + goto bail; + } + } + + ocfs2_commit_trans(osb, handle); + i--; + } + + osb->truncated_clusters = 0; + +bail: + return status; +} + +/* Expects you to already be holding tl_inode->i_rwsem */ +int __ocfs2_flush_truncate_log(struct ocfs2_super *osb) +{ + int status; + unsigned int num_to_flush; + struct inode *tl_inode = osb->osb_tl_inode; + struct inode *data_alloc_inode = NULL; + struct buffer_head *tl_bh = osb->osb_tl_bh; + struct buffer_head *data_alloc_bh = NULL; + struct ocfs2_dinode *di; + struct ocfs2_truncate_log *tl; + struct ocfs2_journal *journal = osb->journal; + + BUG_ON(inode_trylock(tl_inode)); + + di = (struct ocfs2_dinode *) tl_bh->b_data; + + /* tl_bh is loaded from ocfs2_truncate_log_init(). It's validated + * by the underlying call to ocfs2_read_inode_block(), so any + * corruption is a code bug */ + BUG_ON(!OCFS2_IS_VALID_DINODE(di)); + + tl = &di->id2.i_dealloc; + num_to_flush = le16_to_cpu(tl->tl_used); + trace_ocfs2_flush_truncate_log( + (unsigned long long)OCFS2_I(tl_inode)->ip_blkno, + num_to_flush); + if (!num_to_flush) { + status = 0; + goto out; + } + + /* Appending truncate log(TA) and flushing truncate log(TF) are + * two separated transactions. They can be both committed but not + * checkpointed. If crash occurs then, both two transaction will be + * replayed with several already released to global bitmap clusters. + * Then truncate log will be replayed resulting in cluster double free. + */ + jbd2_journal_lock_updates(journal->j_journal); + status = jbd2_journal_flush(journal->j_journal, 0); + jbd2_journal_unlock_updates(journal->j_journal); + if (status < 0) { + mlog_errno(status); + goto out; + } + + data_alloc_inode = ocfs2_get_system_file_inode(osb, + GLOBAL_BITMAP_SYSTEM_INODE, + OCFS2_INVALID_SLOT); + if (!data_alloc_inode) { + status = -EINVAL; + mlog(ML_ERROR, "Could not get bitmap inode!\n"); + goto out; + } + + inode_lock(data_alloc_inode); + + status = ocfs2_inode_lock(data_alloc_inode, &data_alloc_bh, 1); + if (status < 0) { + mlog_errno(status); + goto out_mutex; + } + + status = ocfs2_replay_truncate_records(osb, data_alloc_inode, + data_alloc_bh); + if (status < 0) + mlog_errno(status); + + brelse(data_alloc_bh); + ocfs2_inode_unlock(data_alloc_inode, 1); + +out_mutex: + inode_unlock(data_alloc_inode); + iput(data_alloc_inode); + +out: + return status; +} + +int ocfs2_flush_truncate_log(struct ocfs2_super *osb) +{ + int status; + struct inode *tl_inode = osb->osb_tl_inode; + + inode_lock(tl_inode); + status = __ocfs2_flush_truncate_log(osb); + inode_unlock(tl_inode); + + return status; +} + +static void ocfs2_truncate_log_worker(struct work_struct *work) +{ + int status; + struct ocfs2_super *osb = + container_of(work, struct ocfs2_super, + osb_truncate_log_wq.work); + + status = ocfs2_flush_truncate_log(osb); + if (status < 0) + mlog_errno(status); + else + ocfs2_init_steal_slots(osb); +} + +#define OCFS2_TRUNCATE_LOG_FLUSH_INTERVAL (2 * HZ) +void ocfs2_schedule_truncate_log_flush(struct ocfs2_super *osb, + int cancel) +{ + if (osb->osb_tl_inode && + atomic_read(&osb->osb_tl_disable) == 0) { + /* We want to push off log flushes while truncates are + * still running. */ + if (cancel) + cancel_delayed_work(&osb->osb_truncate_log_wq); + + queue_delayed_work(osb->ocfs2_wq, &osb->osb_truncate_log_wq, + OCFS2_TRUNCATE_LOG_FLUSH_INTERVAL); + } +} + +/* + * Try to flush truncate logs if we can free enough clusters from it. + * As for return value, "< 0" means error, "0" no space and "1" means + * we have freed enough spaces and let the caller try to allocate again. + */ +int ocfs2_try_to_free_truncate_log(struct ocfs2_super *osb, + unsigned int needed) +{ + tid_t target; + int ret = 0; + unsigned int truncated_clusters; + + inode_lock(osb->osb_tl_inode); + truncated_clusters = osb->truncated_clusters; + inode_unlock(osb->osb_tl_inode); + + /* + * Check whether we can succeed in allocating if we free + * the truncate log. + */ + if (truncated_clusters < needed) + goto out; + + ret = ocfs2_flush_truncate_log(osb); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (jbd2_journal_start_commit(osb->journal->j_journal, &target)) { + jbd2_log_wait_commit(osb->journal->j_journal, target); + ret = 1; + } +out: + return ret; +} + +static int ocfs2_get_truncate_log_info(struct ocfs2_super *osb, + int slot_num, + struct inode **tl_inode, + struct buffer_head **tl_bh) +{ + int status; + struct inode *inode = NULL; + struct buffer_head *bh = NULL; + struct ocfs2_dinode *di; + struct ocfs2_truncate_log *tl; + unsigned int tl_count, tl_used; + + inode = ocfs2_get_system_file_inode(osb, + TRUNCATE_LOG_SYSTEM_INODE, + slot_num); + if (!inode) { + status = -EINVAL; + mlog(ML_ERROR, "Could not get load truncate log inode!\n"); + goto bail; + } + + status = ocfs2_read_inode_block(inode, &bh); + if (status < 0) { + iput(inode); + mlog_errno(status); + goto bail; + } + + di = (struct ocfs2_dinode *)bh->b_data; + tl = &di->id2.i_dealloc; + tl_count = le16_to_cpu(tl->tl_count); + tl_used = le16_to_cpu(tl->tl_used); + if (unlikely(tl_count > ocfs2_truncate_recs_per_inode(osb->sb) || + tl_count == 0 || + tl_used > tl_count)) { + status = -EFSCORRUPTED; + iput(inode); + brelse(bh); + mlog_errno(status); + goto bail; + } + + *tl_inode = inode; + *tl_bh = bh; +bail: + return status; +} + +/* called during the 1st stage of node recovery. we stamp a clean + * truncate log and pass back a copy for processing later. if the + * truncate log does not require processing, a *tl_copy is set to + * NULL. */ +int ocfs2_begin_truncate_log_recovery(struct ocfs2_super *osb, + int slot_num, + struct ocfs2_dinode **tl_copy) +{ + int status; + struct inode *tl_inode = NULL; + struct buffer_head *tl_bh = NULL; + struct ocfs2_dinode *di; + struct ocfs2_truncate_log *tl; + + *tl_copy = NULL; + + trace_ocfs2_begin_truncate_log_recovery(slot_num); + + status = ocfs2_get_truncate_log_info(osb, slot_num, &tl_inode, &tl_bh); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + di = (struct ocfs2_dinode *) tl_bh->b_data; + + /* tl_bh is loaded from ocfs2_get_truncate_log_info(). It's + * validated by the underlying call to ocfs2_read_inode_block(), + * so any corruption is a code bug */ + BUG_ON(!OCFS2_IS_VALID_DINODE(di)); + + tl = &di->id2.i_dealloc; + if (le16_to_cpu(tl->tl_used)) { + trace_ocfs2_truncate_log_recovery_num(le16_to_cpu(tl->tl_used)); + + /* + * Assuming the write-out below goes well, this copy will be + * passed back to recovery for processing. + */ + *tl_copy = kmemdup(tl_bh->b_data, tl_bh->b_size, GFP_KERNEL); + if (!(*tl_copy)) { + status = -ENOMEM; + mlog_errno(status); + goto bail; + } + + /* All we need to do to clear the truncate log is set + * tl_used. */ + tl->tl_used = 0; + + ocfs2_compute_meta_ecc(osb->sb, tl_bh->b_data, &di->i_check); + status = ocfs2_write_block(osb, tl_bh, INODE_CACHE(tl_inode)); + if (status < 0) { + mlog_errno(status); + goto bail; + } + } + +bail: + iput(tl_inode); + brelse(tl_bh); + + if (status < 0) { + kfree(*tl_copy); + *tl_copy = NULL; + mlog_errno(status); + } + + return status; +} + +int ocfs2_complete_truncate_log_recovery(struct ocfs2_super *osb, + struct ocfs2_dinode *tl_copy) +{ + int status = 0; + int i; + unsigned int clusters, num_recs, start_cluster; + u64 start_blk; + handle_t *handle; + struct inode *tl_inode = osb->osb_tl_inode; + struct ocfs2_truncate_log *tl; + + if (OCFS2_I(tl_inode)->ip_blkno == le64_to_cpu(tl_copy->i_blkno)) { + mlog(ML_ERROR, "Asked to recover my own truncate log!\n"); + return -EINVAL; + } + + tl = &tl_copy->id2.i_dealloc; + num_recs = le16_to_cpu(tl->tl_used); + trace_ocfs2_complete_truncate_log_recovery( + (unsigned long long)le64_to_cpu(tl_copy->i_blkno), + num_recs); + + inode_lock(tl_inode); + for(i = 0; i < num_recs; i++) { + if (ocfs2_truncate_log_needs_flush(osb)) { + status = __ocfs2_flush_truncate_log(osb); + if (status < 0) { + mlog_errno(status); + goto bail_up; + } + } + + handle = ocfs2_start_trans(osb, OCFS2_TRUNCATE_LOG_UPDATE); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + mlog_errno(status); + goto bail_up; + } + + clusters = le32_to_cpu(tl->tl_recs[i].t_clusters); + start_cluster = le32_to_cpu(tl->tl_recs[i].t_start); + start_blk = ocfs2_clusters_to_blocks(osb->sb, start_cluster); + + status = ocfs2_truncate_log_append(osb, handle, + start_blk, clusters); + ocfs2_commit_trans(osb, handle); + if (status < 0) { + mlog_errno(status); + goto bail_up; + } + } + +bail_up: + inode_unlock(tl_inode); + + return status; +} + +void ocfs2_truncate_log_shutdown(struct ocfs2_super *osb) +{ + int status; + struct inode *tl_inode = osb->osb_tl_inode; + + atomic_set(&osb->osb_tl_disable, 1); + + if (tl_inode) { + cancel_delayed_work(&osb->osb_truncate_log_wq); + flush_workqueue(osb->ocfs2_wq); + + status = ocfs2_flush_truncate_log(osb); + if (status < 0) + mlog_errno(status); + + brelse(osb->osb_tl_bh); + iput(osb->osb_tl_inode); + } +} + +int ocfs2_truncate_log_init(struct ocfs2_super *osb) +{ + int status; + struct inode *tl_inode = NULL; + struct buffer_head *tl_bh = NULL; + + status = ocfs2_get_truncate_log_info(osb, + osb->slot_num, + &tl_inode, + &tl_bh); + if (status < 0) + mlog_errno(status); + + /* ocfs2_truncate_log_shutdown keys on the existence of + * osb->osb_tl_inode so we don't set any of the osb variables + * until we're sure all is well. */ + INIT_DELAYED_WORK(&osb->osb_truncate_log_wq, + ocfs2_truncate_log_worker); + atomic_set(&osb->osb_tl_disable, 0); + osb->osb_tl_bh = tl_bh; + osb->osb_tl_inode = tl_inode; + + return status; +} + +/* + * Delayed de-allocation of suballocator blocks. + * + * Some sets of block de-allocations might involve multiple suballocator inodes. + * + * The locking for this can get extremely complicated, especially when + * the suballocator inodes to delete from aren't known until deep + * within an unrelated codepath. + * + * ocfs2_extent_block structures are a good example of this - an inode + * btree could have been grown by any number of nodes each allocating + * out of their own suballoc inode. + * + * These structures allow the delay of block de-allocation until a + * later time, when locking of multiple cluster inodes won't cause + * deadlock. + */ + +/* + * Describe a single bit freed from a suballocator. For the block + * suballocators, it represents one block. For the global cluster + * allocator, it represents some clusters and free_bit indicates + * clusters number. + */ +struct ocfs2_cached_block_free { + struct ocfs2_cached_block_free *free_next; + u64 free_bg; + u64 free_blk; + unsigned int free_bit; +}; + +struct ocfs2_per_slot_free_list { + struct ocfs2_per_slot_free_list *f_next_suballocator; + int f_inode_type; + int f_slot; + struct ocfs2_cached_block_free *f_first; +}; + +static int ocfs2_free_cached_blocks(struct ocfs2_super *osb, + int sysfile_type, + int slot, + struct ocfs2_cached_block_free *head) +{ + int ret; + u64 bg_blkno; + handle_t *handle; + struct inode *inode; + struct buffer_head *di_bh = NULL; + struct ocfs2_cached_block_free *tmp; + + inode = ocfs2_get_system_file_inode(osb, sysfile_type, slot); + if (!inode) { + ret = -EINVAL; + mlog_errno(ret); + goto out; + } + + inode_lock(inode); + + ret = ocfs2_inode_lock(inode, &di_bh, 1); + if (ret) { + mlog_errno(ret); + goto out_mutex; + } + + while (head) { + if (head->free_bg) + bg_blkno = head->free_bg; + else + bg_blkno = ocfs2_which_suballoc_group(head->free_blk, + head->free_bit); + handle = ocfs2_start_trans(osb, OCFS2_SUBALLOC_FREE); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out_unlock; + } + + trace_ocfs2_free_cached_blocks( + (unsigned long long)head->free_blk, head->free_bit); + + ret = ocfs2_free_suballoc_bits(handle, inode, di_bh, + head->free_bit, bg_blkno, 1); + if (ret) + mlog_errno(ret); + + ocfs2_commit_trans(osb, handle); + + tmp = head; + head = head->free_next; + kfree(tmp); + } + +out_unlock: + ocfs2_inode_unlock(inode, 1); + brelse(di_bh); +out_mutex: + inode_unlock(inode); + iput(inode); +out: + while(head) { + /* Premature exit may have left some dangling items. */ + tmp = head; + head = head->free_next; + kfree(tmp); + } + + return ret; +} + +int ocfs2_cache_cluster_dealloc(struct ocfs2_cached_dealloc_ctxt *ctxt, + u64 blkno, unsigned int bit) +{ + int ret = 0; + struct ocfs2_cached_block_free *item; + + item = kzalloc_obj(*item, GFP_NOFS); + if (item == NULL) { + ret = -ENOMEM; + mlog_errno(ret); + return ret; + } + + trace_ocfs2_cache_cluster_dealloc((unsigned long long)blkno, bit); + + item->free_blk = blkno; + item->free_bit = bit; + item->free_next = ctxt->c_global_allocator; + + ctxt->c_global_allocator = item; + return ret; +} + +static int ocfs2_free_cached_clusters(struct ocfs2_super *osb, + struct ocfs2_cached_block_free *head) +{ + struct ocfs2_cached_block_free *tmp; + struct inode *tl_inode = osb->osb_tl_inode; + handle_t *handle; + int ret = 0; + + inode_lock(tl_inode); + + while (head) { + if (ocfs2_truncate_log_needs_flush(osb)) { + ret = __ocfs2_flush_truncate_log(osb); + if (ret < 0) { + mlog_errno(ret); + break; + } + } + + handle = ocfs2_start_trans(osb, OCFS2_TRUNCATE_LOG_UPDATE); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + break; + } + + ret = ocfs2_truncate_log_append(osb, handle, head->free_blk, + head->free_bit); + + ocfs2_commit_trans(osb, handle); + tmp = head; + head = head->free_next; + kfree(tmp); + + if (ret < 0) { + mlog_errno(ret); + break; + } + } + + inode_unlock(tl_inode); + + while (head) { + /* Premature exit may have left some dangling items. */ + tmp = head; + head = head->free_next; + kfree(tmp); + } + + return ret; +} + +int ocfs2_run_deallocs(struct ocfs2_super *osb, + struct ocfs2_cached_dealloc_ctxt *ctxt) +{ + int ret = 0, ret2; + struct ocfs2_per_slot_free_list *fl; + + if (!ctxt) + return 0; + + while (ctxt->c_first_suballocator) { + fl = ctxt->c_first_suballocator; + + if (fl->f_first) { + trace_ocfs2_run_deallocs(fl->f_inode_type, + fl->f_slot); + ret2 = ocfs2_free_cached_blocks(osb, + fl->f_inode_type, + fl->f_slot, + fl->f_first); + if (ret2) + mlog_errno(ret2); + if (!ret) + ret = ret2; + } + + ctxt->c_first_suballocator = fl->f_next_suballocator; + kfree(fl); + } + + if (ctxt->c_global_allocator) { + ret2 = ocfs2_free_cached_clusters(osb, + ctxt->c_global_allocator); + if (ret2) + mlog_errno(ret2); + if (!ret) + ret = ret2; + + ctxt->c_global_allocator = NULL; + } + + return ret; +} + +static struct ocfs2_per_slot_free_list * +ocfs2_find_per_slot_free_list(int type, + int slot, + struct ocfs2_cached_dealloc_ctxt *ctxt) +{ + struct ocfs2_per_slot_free_list *fl = ctxt->c_first_suballocator; + + while (fl) { + if (fl->f_inode_type == type && fl->f_slot == slot) + return fl; + + fl = fl->f_next_suballocator; + } + + fl = kmalloc_obj(*fl, GFP_NOFS); + if (fl) { + fl->f_inode_type = type; + fl->f_slot = slot; + fl->f_first = NULL; + fl->f_next_suballocator = ctxt->c_first_suballocator; + + ctxt->c_first_suballocator = fl; + } + return fl; +} + +static struct ocfs2_per_slot_free_list * +ocfs2_find_preferred_free_list(int type, + int preferred_slot, + int *real_slot, + struct ocfs2_cached_dealloc_ctxt *ctxt) +{ + struct ocfs2_per_slot_free_list *fl = ctxt->c_first_suballocator; + + while (fl) { + if (fl->f_inode_type == type && fl->f_slot == preferred_slot) { + *real_slot = fl->f_slot; + return fl; + } + + fl = fl->f_next_suballocator; + } + + /* If we can't find any free list matching preferred slot, just use + * the first one. + */ + fl = ctxt->c_first_suballocator; + *real_slot = fl->f_slot; + + return fl; +} + +/* Return Value 1 indicates empty */ +static int ocfs2_is_dealloc_empty(struct ocfs2_extent_tree *et) +{ + struct ocfs2_per_slot_free_list *fl = NULL; + + if (!et->et_dealloc) + return 1; + + fl = et->et_dealloc->c_first_suballocator; + if (!fl) + return 1; + + if (!fl->f_first) + return 1; + + return 0; +} + +/* If extent was deleted from tree due to extent rotation and merging, and + * no metadata is reserved ahead of time. Try to reuse some extents + * just deleted. This is only used to reuse extent blocks. + * It is supposed to find enough extent blocks in dealloc if our estimation + * on metadata is accurate. + */ +static int ocfs2_reuse_blk_from_dealloc(handle_t *handle, + struct ocfs2_extent_tree *et, + struct buffer_head **new_eb_bh, + int blk_wanted, int *blk_given) +{ + int i, status = 0, real_slot; + struct ocfs2_cached_dealloc_ctxt *dealloc; + struct ocfs2_per_slot_free_list *fl; + struct ocfs2_cached_block_free *bf; + struct ocfs2_extent_block *eb; + struct ocfs2_super *osb = + OCFS2_SB(ocfs2_metadata_cache_get_super(et->et_ci)); + + *blk_given = 0; + + /* If extent tree doesn't have a dealloc, this is not faulty. Just + * tell upper caller dealloc can't provide any block and it should + * ask for alloc to claim more space. + */ + dealloc = et->et_dealloc; + if (!dealloc) + goto bail; + + for (i = 0; i < blk_wanted; i++) { + /* Prefer to use local slot */ + fl = ocfs2_find_preferred_free_list(EXTENT_ALLOC_SYSTEM_INODE, + osb->slot_num, &real_slot, + dealloc); + /* If no more block can be reused, we should claim more + * from alloc. Just return here normally. + */ + if (!fl) { + status = 0; + break; + } + + bf = fl->f_first; + fl->f_first = bf->free_next; + + new_eb_bh[i] = sb_getblk(osb->sb, bf->free_blk); + if (new_eb_bh[i] == NULL) { + status = -ENOMEM; + mlog_errno(status); + goto bail; + } + + mlog(0, "Reusing block(%llu) from " + "dealloc(local slot:%d, real slot:%d)\n", + bf->free_blk, osb->slot_num, real_slot); + + ocfs2_set_new_buffer_uptodate(et->et_ci, new_eb_bh[i]); + + status = ocfs2_journal_access_eb(handle, et->et_ci, + new_eb_bh[i], + OCFS2_JOURNAL_ACCESS_CREATE); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + memset(new_eb_bh[i]->b_data, 0, osb->sb->s_blocksize); + eb = (struct ocfs2_extent_block *) new_eb_bh[i]->b_data; + + /* We can't guarantee that buffer head is still cached, so + * polutlate the extent block again. + */ + strscpy(eb->h_signature, OCFS2_EXTENT_BLOCK_SIGNATURE); + eb->h_blkno = cpu_to_le64(bf->free_blk); + eb->h_fs_generation = cpu_to_le32(osb->fs_generation); + eb->h_suballoc_slot = cpu_to_le16(real_slot); + eb->h_suballoc_loc = cpu_to_le64(bf->free_bg); + eb->h_suballoc_bit = cpu_to_le16(bf->free_bit); + eb->h_list.l_count = + cpu_to_le16(ocfs2_extent_recs_per_eb(osb->sb)); + + /* We'll also be dirtied by the caller, so + * this isn't absolutely necessary. + */ + ocfs2_journal_dirty(handle, new_eb_bh[i]); + + if (!fl->f_first) { + dealloc->c_first_suballocator = fl->f_next_suballocator; + kfree(fl); + } + kfree(bf); + } + + *blk_given = i; + +bail: + if (unlikely(status < 0)) { + for (i = 0; i < blk_wanted; i++) + brelse(new_eb_bh[i]); + } + + return status; +} + +int ocfs2_cache_block_dealloc(struct ocfs2_cached_dealloc_ctxt *ctxt, + int type, int slot, u64 suballoc, + u64 blkno, unsigned int bit) +{ + int ret; + struct ocfs2_per_slot_free_list *fl; + struct ocfs2_cached_block_free *item; + + fl = ocfs2_find_per_slot_free_list(type, slot, ctxt); + if (fl == NULL) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + item = kzalloc_obj(*item, GFP_NOFS); + if (item == NULL) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + trace_ocfs2_cache_block_dealloc(type, slot, + (unsigned long long)suballoc, + (unsigned long long)blkno, bit); + + item->free_bg = suballoc; + item->free_blk = blkno; + item->free_bit = bit; + item->free_next = fl->f_first; + + fl->f_first = item; + + ret = 0; +out: + return ret; +} + +static int ocfs2_cache_extent_block_free(struct ocfs2_cached_dealloc_ctxt *ctxt, + struct ocfs2_extent_block *eb) +{ + return ocfs2_cache_block_dealloc(ctxt, EXTENT_ALLOC_SYSTEM_INODE, + le16_to_cpu(eb->h_suballoc_slot), + le64_to_cpu(eb->h_suballoc_loc), + le64_to_cpu(eb->h_blkno), + le16_to_cpu(eb->h_suballoc_bit)); +} + +static int ocfs2_zero_func(handle_t *handle, struct buffer_head *bh) +{ + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + return 0; +} + +void ocfs2_map_and_dirty_folio(struct inode *inode, handle_t *handle, + size_t from, size_t to, struct folio *folio, int zero, + u64 *phys) +{ + int ret, partial = 0; + loff_t start_byte = folio_pos(folio) + from; + loff_t length = to - from; + + ret = ocfs2_map_folio_blocks(folio, phys, inode, from, to, 0); + if (ret) + mlog_errno(ret); + + if (zero) + folio_zero_segment(folio, from, to); + + /* + * Need to set the buffers we zero'd into uptodate + * here if they aren't - ocfs2_map_page_blocks() + * might've skipped some + */ + ret = walk_page_buffers(handle, folio_buffers(folio), + from, to, &partial, + ocfs2_zero_func); + if (ret < 0) + mlog_errno(ret); + else if (ocfs2_should_order_data(inode)) { + ret = ocfs2_jbd2_inode_add_write(handle, inode, + start_byte, length); + if (ret < 0) + mlog_errno(ret); + } + + if (!partial) + folio_mark_uptodate(folio); + + flush_dcache_folio(folio); +} + +static void ocfs2_zero_cluster_folios(struct inode *inode, loff_t start, + loff_t end, struct folio **folios, int numfolios, + u64 phys, handle_t *handle) +{ + int i; + struct super_block *sb = inode->i_sb; + + BUG_ON(!ocfs2_sparse_alloc(OCFS2_SB(sb))); + + if (numfolios == 0) + goto out; + + for (i = 0; i < numfolios; i++) { + struct folio *folio = folios[i]; + size_t to = folio_size(folio); + size_t from = offset_in_folio(folio, start); + + if (to > end - folio_pos(folio)) + to = end - folio_pos(folio); + + ocfs2_map_and_dirty_folio(inode, handle, from, to, folio, 1, + &phys); + + start = folio_next_pos(folio); + } +out: + if (folios) + ocfs2_unlock_and_free_folios(folios, numfolios); +} + +static int ocfs2_grab_folios(struct inode *inode, loff_t start, loff_t end, + struct folio **folios, int *num) +{ + int numfolios, ret = 0; + struct address_space *mapping = inode->i_mapping; + unsigned long index; + loff_t last_page_bytes; + + BUG_ON(start > end); + + numfolios = 0; + last_page_bytes = PAGE_ALIGN(end); + index = start >> PAGE_SHIFT; + do { + folios[numfolios] = __filemap_get_folio(mapping, index, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, GFP_NOFS); + if (IS_ERR(folios[numfolios])) { + ret = PTR_ERR(folios[numfolios]); + mlog_errno(ret); + folios[numfolios] = NULL; + goto out; + } + + index = folio_next_index(folios[numfolios]); + numfolios++; + } while (index < (last_page_bytes >> PAGE_SHIFT)); + +out: + if (ret != 0) { + ocfs2_unlock_and_free_folios(folios, numfolios); + numfolios = 0; + } + + *num = numfolios; + + return ret; +} + +static int ocfs2_grab_eof_folios(struct inode *inode, loff_t start, loff_t end, + struct folio **folios, int *num) +{ + struct super_block *sb = inode->i_sb; + + BUG_ON(start >> OCFS2_SB(sb)->s_clustersize_bits != + (end - 1) >> OCFS2_SB(sb)->s_clustersize_bits); + + return ocfs2_grab_folios(inode, start, end, folios, num); +} + +/* + * Zero partial cluster for a hole punch or truncate. This avoids exposing + * nonzero data on subsequent file extends. + * + * We need to call this before i_size is updated on the inode because + * otherwise block_write_full_folio() will skip writeout of pages past + * i_size. + */ +int ocfs2_zero_range_for_truncate(struct inode *inode, handle_t *handle, + u64 range_start, u64 range_end) +{ + int ret = 0, numfolios; + struct folio **folios = NULL; + u64 phys; + unsigned int ext_flags; + struct super_block *sb = inode->i_sb; + + /* + * File systems which don't support sparse files zero on every + * extend. + */ + if (!ocfs2_sparse_alloc(OCFS2_SB(sb))) + return 0; + + /* + * Avoid zeroing folios fully beyond current i_size. It is pointless as + * underlying blocks of those folios should be already zeroed out and + * page writeback will skip them anyway. + */ + range_end = min_t(u64, range_end, i_size_read(inode)); + if (range_start >= range_end) + return 0; + + folios = kzalloc_objs(struct folio *, ocfs2_pages_per_cluster(sb), + GFP_NOFS); + if (folios == NULL) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + ret = ocfs2_extent_map_get_blocks(inode, + range_start >> sb->s_blocksize_bits, + &phys, NULL, &ext_flags); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * Tail is a hole, or is marked unwritten. In either case, we + * can count on read and write to return/push zero's. + */ + if (phys == 0 || ext_flags & OCFS2_EXT_UNWRITTEN) + goto out; + + ret = ocfs2_grab_eof_folios(inode, range_start, range_end, folios, + &numfolios); + if (ret) { + mlog_errno(ret); + goto out; + } + + ocfs2_zero_cluster_folios(inode, range_start, range_end, folios, + numfolios, phys, handle); + + /* + * Initiate writeout of the folios we zero'd here. We don't + * wait on them - the truncate_inode_pages() call later will + * do that for us. + */ + ret = filemap_fdatawrite_range(inode->i_mapping, range_start, + range_end - 1); + if (ret) + mlog_errno(ret); + +out: + kfree(folios); + + return ret; +} + +static void ocfs2_zero_dinode_id2_with_xattr(struct inode *inode, + struct ocfs2_dinode *di) +{ + unsigned int blocksize = 1 << inode->i_sb->s_blocksize_bits; + unsigned int xattrsize = le16_to_cpu(di->i_xattr_inline_size); + + if (le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_XATTR_FL) + memset(&di->id2, 0, blocksize - + offsetof(struct ocfs2_dinode, id2) - + xattrsize); + else + memset(&di->id2, 0, blocksize - + offsetof(struct ocfs2_dinode, id2)); +} + +void ocfs2_dinode_new_extent_list(struct inode *inode, + struct ocfs2_dinode *di) +{ + ocfs2_zero_dinode_id2_with_xattr(inode, di); + di->id2.i_list.l_tree_depth = 0; + di->id2.i_list.l_next_free_rec = 0; + di->id2.i_list.l_count = cpu_to_le16( + ocfs2_extent_recs_per_inode_with_xattr(inode->i_sb, di)); +} + +void ocfs2_set_inode_data_inline(struct inode *inode, struct ocfs2_dinode *di) +{ + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_inline_data *idata = &di->id2.i_data; + + spin_lock(&oi->ip_lock); + oi->ip_dyn_features |= OCFS2_INLINE_DATA_FL; + di->i_dyn_features = cpu_to_le16(oi->ip_dyn_features); + spin_unlock(&oi->ip_lock); + + /* + * We clear the entire i_data structure here so that all + * fields can be properly initialized. + */ + ocfs2_zero_dinode_id2_with_xattr(inode, di); + + idata->id_count = cpu_to_le16( + ocfs2_max_inline_data_with_xattr(inode->i_sb, di)); +} + +int ocfs2_convert_inline_data_to_extents(struct inode *inode, + struct buffer_head *di_bh) +{ + int ret, has_data, num_folios = 0; + int need_free = 0; + u32 bit_off, num; + handle_t *handle; + u64 block; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data; + struct ocfs2_alloc_context *data_ac = NULL; + struct folio *folio = NULL; + struct ocfs2_extent_tree et; + int did_quota = 0; + + has_data = i_size_read(inode) ? 1 : 0; + + if (has_data) { + ret = ocfs2_reserve_clusters(osb, 1, &data_ac); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + handle = ocfs2_start_trans(osb, + ocfs2_inline_to_extents_credits(osb->sb)); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out; + } + + ret = ocfs2_journal_access_di(handle, INODE_CACHE(inode), di_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + if (has_data) { + unsigned int page_end = min_t(unsigned, PAGE_SIZE, + osb->s_clustersize); + u64 phys; + + ret = dquot_alloc_space_nodirty(inode, + ocfs2_clusters_to_bytes(osb->sb, 1)); + if (ret) + goto out_commit; + did_quota = 1; + + data_ac->ac_resv = &oi->ip_la_data_resv; + + ret = ocfs2_claim_clusters(handle, data_ac, 1, &bit_off, + &num); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + /* + * Save two copies, one for insert, and one that can + * be changed by ocfs2_map_and_dirty_folio() below. + */ + block = phys = ocfs2_clusters_to_blocks(inode->i_sb, bit_off); + + ret = ocfs2_grab_eof_folios(inode, 0, page_end, &folio, + &num_folios); + if (ret) { + mlog_errno(ret); + need_free = 1; + goto out_commit; + } + + /* + * This should populate the 1st page for us and mark + * it up to date. + */ + ret = ocfs2_read_inline_data(inode, folio, di_bh); + if (ret) { + mlog_errno(ret); + need_free = 1; + goto out_unlock; + } + + ocfs2_map_and_dirty_folio(inode, handle, 0, page_end, folio, 0, + &phys); + } + + spin_lock(&oi->ip_lock); + oi->ip_dyn_features &= ~OCFS2_INLINE_DATA_FL; + di->i_dyn_features = cpu_to_le16(oi->ip_dyn_features); + spin_unlock(&oi->ip_lock); + + ocfs2_update_inode_fsync_trans(handle, inode, 1); + ocfs2_dinode_new_extent_list(inode, di); + + ocfs2_journal_dirty(handle, di_bh); + + if (has_data) { + /* + * An error at this point should be extremely rare. If + * this proves to be false, we could always re-build + * the in-inode data from our pages. + */ + ocfs2_init_dinode_extent_tree(&et, INODE_CACHE(inode), di_bh); + ret = ocfs2_insert_extent(handle, &et, 0, block, 1, 0, NULL); + if (ret) { + mlog_errno(ret); + need_free = 1; + goto out_unlock; + } + + inode->i_blocks = ocfs2_inode_sector_count(inode); + } + +out_unlock: + if (folio) + ocfs2_unlock_and_free_folios(&folio, num_folios); + +out_commit: + if (ret < 0 && did_quota) + dquot_free_space_nodirty(inode, + ocfs2_clusters_to_bytes(osb->sb, 1)); + + if (need_free) { + if (data_ac->ac_which == OCFS2_AC_USE_LOCAL) + ocfs2_free_local_alloc_bits(osb, handle, data_ac, + bit_off, num); + else + ocfs2_free_clusters(handle, + data_ac->ac_inode, + data_ac->ac_bh, + ocfs2_clusters_to_blocks(osb->sb, bit_off), + num); + } + + ocfs2_commit_trans(osb, handle); + +out: + if (data_ac) + ocfs2_free_alloc_context(data_ac); + return ret; +} + +/* + * It is expected, that by the time you call this function, + * inode->i_size and fe->i_size have been adjusted. + * + * WARNING: This will kfree the truncate context + */ +int ocfs2_commit_truncate(struct ocfs2_super *osb, + struct inode *inode, + struct buffer_head *di_bh) +{ + int status = 0, i, flags = 0; + u32 new_highest_cpos, range, trunc_cpos, trunc_len, phys_cpos, coff; + u64 blkno = 0; + struct ocfs2_extent_list *el; + struct ocfs2_extent_rec *rec; + struct ocfs2_path *path = NULL; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data; + struct ocfs2_extent_list *root_el = &(di->id2.i_list); + u64 refcount_loc = le64_to_cpu(di->i_refcount_loc); + struct ocfs2_extent_tree et; + struct ocfs2_cached_dealloc_ctxt dealloc; + struct ocfs2_refcount_tree *ref_tree = NULL; + + ocfs2_init_dinode_extent_tree(&et, INODE_CACHE(inode), di_bh); + ocfs2_init_dealloc_ctxt(&dealloc); + + new_highest_cpos = ocfs2_clusters_for_bytes(osb->sb, + i_size_read(inode)); + + path = ocfs2_new_path(di_bh, &di->id2.i_list, + ocfs2_journal_access_di); + if (!path) { + status = -ENOMEM; + mlog_errno(status); + goto bail; + } + + ocfs2_extent_map_trunc(inode, new_highest_cpos); + +start: + /* + * Check that we still have allocation to delete. + */ + if (OCFS2_I(inode)->ip_clusters == 0) { + status = 0; + goto bail; + } + + /* + * Truncate always works against the rightmost tree branch. + */ + status = ocfs2_find_path(INODE_CACHE(inode), path, UINT_MAX); + if (status) { + mlog_errno(status); + goto bail; + } + + trace_ocfs2_commit_truncate( + (unsigned long long)OCFS2_I(inode)->ip_blkno, + new_highest_cpos, + OCFS2_I(inode)->ip_clusters, + path->p_tree_depth); + + /* + * By now, el will point to the extent list on the bottom most + * portion of this tree. Only the tail record is considered in + * each pass. + * + * We handle the following cases, in order: + * - empty extent: delete the remaining branch + * - remove the entire record + * - remove a partial record + * - no record needs to be removed (truncate has completed) + */ + el = path_leaf_el(path); + if (le16_to_cpu(el->l_next_free_rec) == 0) { + ocfs2_error(inode->i_sb, + "Inode %llu has empty extent block at %llu\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno, + (unsigned long long)path_leaf_bh(path)->b_blocknr); + status = -EROFS; + goto bail; + } + + i = le16_to_cpu(el->l_next_free_rec) - 1; + rec = &el->l_recs[i]; + flags = rec->e_flags; + range = le32_to_cpu(rec->e_cpos) + ocfs2_rec_clusters(el, rec); + + if (i == 0 && ocfs2_is_empty_extent(rec)) { + /* + * Lower levels depend on this never happening, but it's best + * to check it up here before changing the tree. + */ + if (root_el->l_tree_depth && rec->e_int_clusters == 0) { + mlog(ML_ERROR, "Inode %llu has an empty " + "extent record, depth %u\n", inode->i_ino, + le16_to_cpu(root_el->l_tree_depth)); + status = ocfs2_remove_rightmost_empty_extent(osb, + &et, path, &dealloc); + if (status) { + mlog_errno(status); + goto bail; + } + + ocfs2_reinit_path(path, 1); + goto start; + } else { + trunc_cpos = le32_to_cpu(rec->e_cpos); + trunc_len = 0; + blkno = 0; + } + } else if (le32_to_cpu(rec->e_cpos) >= new_highest_cpos) { + /* + * Truncate entire record. + */ + trunc_cpos = le32_to_cpu(rec->e_cpos); + trunc_len = ocfs2_rec_clusters(el, rec); + blkno = le64_to_cpu(rec->e_blkno); + } else if (range > new_highest_cpos) { + /* + * Partial truncate. it also should be + * the last truncate we're doing. + */ + trunc_cpos = new_highest_cpos; + trunc_len = range - new_highest_cpos; + coff = new_highest_cpos - le32_to_cpu(rec->e_cpos); + blkno = le64_to_cpu(rec->e_blkno) + + ocfs2_clusters_to_blocks(inode->i_sb, coff); + } else { + /* + * Truncate completed, leave happily. + */ + status = 0; + goto bail; + } + + phys_cpos = ocfs2_blocks_to_clusters(inode->i_sb, blkno); + + if ((flags & OCFS2_EXT_REFCOUNTED) && trunc_len && !ref_tree) { + status = ocfs2_lock_refcount_tree(osb, refcount_loc, 1, + &ref_tree, NULL); + if (status) { + mlog_errno(status); + goto bail; + } + } + + status = ocfs2_remove_btree_range(inode, &et, trunc_cpos, + phys_cpos, trunc_len, flags, &dealloc, + refcount_loc, true); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + ocfs2_reinit_path(path, 1); + + /* + * The check above will catch the case where we've truncated + * away all allocation. + */ + goto start; + +bail: + if (ref_tree) + ocfs2_unlock_refcount_tree(osb, ref_tree, 1); + + ocfs2_schedule_truncate_log_flush(osb, 1); + + ocfs2_run_deallocs(osb, &dealloc); + + ocfs2_free_path(path); + + return status; +} + +/* + * 'start' is inclusive, 'end' is not. + */ +int ocfs2_truncate_inline(struct inode *inode, struct buffer_head *di_bh, + unsigned int start, unsigned int end, int trunc) +{ + int ret; + unsigned int numbytes; + handle_t *handle; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data; + struct ocfs2_inline_data *idata = &di->id2.i_data; + + /* No need to punch hole beyond i_size. */ + if (start >= i_size_read(inode)) + return 0; + + if (end > i_size_read(inode)) + end = i_size_read(inode); + + BUG_ON(start > end); + + if (!(OCFS2_I(inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL) || + !(le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_DATA_FL) || + !ocfs2_supports_inline_data(osb)) { + ocfs2_error(inode->i_sb, + "Inline data flags for inode %llu don't agree! Disk: 0x%x, Memory: 0x%x, Superblock: 0x%x\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno, + le16_to_cpu(di->i_dyn_features), + OCFS2_I(inode)->ip_dyn_features, + osb->s_feature_incompat); + ret = -EROFS; + goto out; + } + + handle = ocfs2_start_trans(osb, OCFS2_INODE_UPDATE_CREDITS); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out; + } + + ret = ocfs2_journal_access_di(handle, INODE_CACHE(inode), di_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + numbytes = end - start; + memset(idata->id_data + start, 0, numbytes); + + /* + * No need to worry about the data page here - it's been + * truncated already and inline data doesn't need it for + * pushing zero's to disk, so we'll let read_folio pick it up + * later. + */ + if (trunc) { + i_size_write(inode, start); + di->i_size = cpu_to_le64(start); + } + + inode->i_blocks = ocfs2_inode_sector_count(inode); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + + di->i_ctime = di->i_mtime = cpu_to_le64(inode_get_ctime_sec(inode)); + di->i_ctime_nsec = di->i_mtime_nsec = cpu_to_le32(inode_get_ctime_nsec(inode)); + + ocfs2_update_inode_fsync_trans(handle, inode, 1); + ocfs2_journal_dirty(handle, di_bh); + +out_commit: + ocfs2_commit_trans(osb, handle); + +out: + return ret; +} + +static int ocfs2_trim_extent(struct super_block *sb, + struct ocfs2_group_desc *gd, + u64 group, u32 start, u32 count) +{ + u64 discard, bcount; + struct ocfs2_super *osb = OCFS2_SB(sb); + + bcount = ocfs2_clusters_to_blocks(sb, count); + discard = ocfs2_clusters_to_blocks(sb, start); + + /* + * For the first cluster group, the gd->bg_blkno is not at the start + * of the group, but at an offset from the start. If we add it while + * calculating discard for first group, we will wrongly start fstrim a + * few blocks after the desried start block and the range can cross + * over into the next cluster group. So, add it only if this is not + * the first cluster group. + */ + if (group != osb->first_cluster_group_blkno) + discard += le64_to_cpu(gd->bg_blkno); + + trace_ocfs2_trim_extent(sb, (unsigned long long)discard, bcount); + + return sb_issue_discard(sb, discard, bcount, GFP_NOFS, 0); +} + +static int ocfs2_trim_group(struct super_block *sb, + struct ocfs2_group_desc *gd, u64 group, + u32 start, u32 max, u32 minbits) +{ + int ret = 0, count = 0, next; + void *bitmap = gd->bg_bitmap; + + if (le16_to_cpu(gd->bg_free_bits_count) < minbits) + return 0; + + trace_ocfs2_trim_group((unsigned long long)le64_to_cpu(gd->bg_blkno), + start, max, minbits); + + while (start < max) { + start = ocfs2_find_next_zero_bit(bitmap, max, start); + if (start >= max) + break; + next = ocfs2_find_next_bit(bitmap, max, start); + + if ((next - start) >= minbits) { + ret = ocfs2_trim_extent(sb, gd, group, + start, next - start); + if (ret < 0) { + mlog_errno(ret); + break; + } + count += next - start; + } + start = next + 1; + + if (fatal_signal_pending(current)) { + count = -ERESTARTSYS; + break; + } + + if ((le16_to_cpu(gd->bg_free_bits_count) - count) < minbits) + break; + } + + if (ret < 0) + count = ret; + + return count; +} + +static +int ocfs2_trim_mainbm(struct super_block *sb, struct fstrim_range *range) +{ + struct ocfs2_super *osb = OCFS2_SB(sb); + u64 start, len, trimmed = 0, first_group, last_group = 0, group = 0; + int ret, cnt; + u32 first_bit, last_bit, minlen; + struct buffer_head *main_bm_bh = NULL; + struct inode *main_bm_inode = NULL; + struct buffer_head *gd_bh = NULL; + struct ocfs2_dinode *main_bm; + struct ocfs2_group_desc *gd = NULL; + + start = range->start >> osb->s_clustersize_bits; + len = range->len >> osb->s_clustersize_bits; + minlen = range->minlen >> osb->s_clustersize_bits; + + if (minlen >= osb->bitmap_cpg || range->len < osb->s_clustersize) + return -EINVAL; + + trace_ocfs2_trim_mainbm(start, len, minlen); + +next_group: + main_bm_inode = ocfs2_get_system_file_inode(osb, + GLOBAL_BITMAP_SYSTEM_INODE, + OCFS2_INVALID_SLOT); + if (!main_bm_inode) { + ret = -EIO; + mlog_errno(ret); + goto out; + } + + inode_lock(main_bm_inode); + + ret = ocfs2_inode_lock(main_bm_inode, &main_bm_bh, 0); + if (ret < 0) { + mlog_errno(ret); + goto out_mutex; + } + main_bm = (struct ocfs2_dinode *)main_bm_bh->b_data; + + /* + * Do some check before trim the first group. + */ + if (!group) { + if (start >= le32_to_cpu(main_bm->i_clusters)) { + ret = -EINVAL; + goto out_unlock; + } + + if (start + len > le32_to_cpu(main_bm->i_clusters)) + len = le32_to_cpu(main_bm->i_clusters) - start; + + /* + * Determine first and last group to examine based on + * start and len + */ + first_group = ocfs2_which_cluster_group(main_bm_inode, start); + if (first_group == osb->first_cluster_group_blkno) + first_bit = start; + else + first_bit = start - ocfs2_blocks_to_clusters(sb, + first_group); + last_group = ocfs2_which_cluster_group(main_bm_inode, + start + len - 1); + group = first_group; + } + + do { + if (first_bit + len >= osb->bitmap_cpg) + last_bit = osb->bitmap_cpg; + else + last_bit = first_bit + len; + + ret = ocfs2_read_group_descriptor(main_bm_inode, + main_bm, group, + &gd_bh); + if (ret < 0) { + mlog_errno(ret); + break; + } + + gd = (struct ocfs2_group_desc *)gd_bh->b_data; + cnt = ocfs2_trim_group(sb, gd, group, + first_bit, last_bit, minlen); + brelse(gd_bh); + gd_bh = NULL; + if (cnt < 0) { + ret = cnt; + mlog_errno(ret); + break; + } + + trimmed += cnt; + len -= osb->bitmap_cpg - first_bit; + first_bit = 0; + if (group == osb->first_cluster_group_blkno) + group = ocfs2_clusters_to_blocks(sb, osb->bitmap_cpg); + else + group += ocfs2_clusters_to_blocks(sb, osb->bitmap_cpg); + } while (0); + +out_unlock: + ocfs2_inode_unlock(main_bm_inode, 0); + brelse(main_bm_bh); + main_bm_bh = NULL; +out_mutex: + inode_unlock(main_bm_inode); + iput(main_bm_inode); + + /* + * If all the groups trim are not done or failed, but we should release + * main_bm related locks for avoiding the current IO starve, then go to + * trim the next group + */ + if (ret >= 0 && group <= last_group) { + cond_resched(); + goto next_group; + } +out: + range->len = trimmed * osb->s_clustersize; + return ret; +} + +int ocfs2_trim_fs(struct super_block *sb, struct fstrim_range *range) +{ + int ret; + struct ocfs2_super *osb = OCFS2_SB(sb); + struct ocfs2_trim_fs_info info, *pinfo = NULL; + + ocfs2_trim_fs_lock_res_init(osb); + + trace_ocfs2_trim_fs(range->start, range->len, range->minlen); + + ret = ocfs2_trim_fs_lock(osb, NULL, 1); + if (ret < 0) { + if (ret != -EAGAIN) { + mlog_errno(ret); + ocfs2_trim_fs_lock_res_uninit(osb); + return ret; + } + + mlog(ML_NOTICE, "Wait for trim on device (%s) to " + "finish, which is running from another node.\n", + osb->dev_str); + ret = ocfs2_trim_fs_lock(osb, &info, 0); + if (ret < 0) { + mlog_errno(ret); + ocfs2_trim_fs_lock_res_uninit(osb); + return ret; + } + + if (info.tf_valid && info.tf_success && + info.tf_start == range->start && + info.tf_len == range->len && + info.tf_minlen == range->minlen) { + /* Avoid sending duplicated trim to a shared device */ + mlog(ML_NOTICE, "The same trim on device (%s) was " + "just done from node (%u), return.\n", + osb->dev_str, info.tf_nodenum); + range->len = info.tf_trimlen; + goto out; + } + } + + info.tf_nodenum = osb->node_num; + info.tf_start = range->start; + info.tf_len = range->len; + info.tf_minlen = range->minlen; + + ret = ocfs2_trim_mainbm(sb, range); + + info.tf_trimlen = range->len; + info.tf_success = (ret < 0 ? 0 : 1); + pinfo = &info; +out: + ocfs2_trim_fs_unlock(osb, pinfo); + ocfs2_trim_fs_lock_res_uninit(osb); + return ret; +} diff --git a/fs/ocfs2/alloc.h b/fs/ocfs2/alloc.h new file mode 100644 index 000000000..1c0c83362 --- /dev/null +++ b/fs/ocfs2/alloc.h @@ -0,0 +1,308 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * alloc.h + * + * Function prototypes + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#ifndef OCFS2_ALLOC_H +#define OCFS2_ALLOC_H + + +/* + * For xattr tree leaf, we limit the leaf byte size to be 64K. + */ +#define OCFS2_MAX_XATTR_TREE_LEAF_SIZE 65536 + +/* + * ocfs2_extent_tree and ocfs2_extent_tree_operations are used to abstract + * the b-tree operations in ocfs2. Now all the b-tree operations are not + * limited to ocfs2_dinode only. Any data which need to allocate clusters + * to store can use b-tree. And it only needs to implement its ocfs2_extent_tree + * and operation. + * + * ocfs2_extent_tree becomes the first-class object for extent tree + * manipulation. Callers of the alloc.c code need to fill it via one of + * the ocfs2_init_*_extent_tree() operations below. + * + * ocfs2_extent_tree contains info for the root of the b-tree, it must have a + * root ocfs2_extent_list and a root_bh so that they can be used in the b-tree + * functions. It needs the ocfs2_caching_info structure associated with + * I/O on the tree. With metadata ecc, we now call different journal_access + * functions for each type of metadata, so it must have the + * root_journal_access function. + * ocfs2_extent_tree_operations abstract the normal operations we do for + * the root of extent b-tree. + */ +struct ocfs2_extent_tree_operations; +struct ocfs2_extent_tree { + const struct ocfs2_extent_tree_operations *et_ops; + struct buffer_head *et_root_bh; + struct ocfs2_extent_list *et_root_el; + struct ocfs2_caching_info *et_ci; + ocfs2_journal_access_func et_root_journal_access; + void *et_object; + unsigned int et_max_leaf_clusters; + struct ocfs2_cached_dealloc_ctxt *et_dealloc; +}; + +/* + * ocfs2_init_*_extent_tree() will fill an ocfs2_extent_tree from the + * specified object buffer. + */ +void ocfs2_init_dinode_extent_tree(struct ocfs2_extent_tree *et, + struct ocfs2_caching_info *ci, + struct buffer_head *bh); +void ocfs2_init_xattr_tree_extent_tree(struct ocfs2_extent_tree *et, + struct ocfs2_caching_info *ci, + struct buffer_head *bh); +struct ocfs2_xattr_value_buf; +void ocfs2_init_xattr_value_extent_tree(struct ocfs2_extent_tree *et, + struct ocfs2_caching_info *ci, + struct ocfs2_xattr_value_buf *vb); +void ocfs2_init_dx_root_extent_tree(struct ocfs2_extent_tree *et, + struct ocfs2_caching_info *ci, + struct buffer_head *bh); +void ocfs2_init_refcount_extent_tree(struct ocfs2_extent_tree *et, + struct ocfs2_caching_info *ci, + struct buffer_head *bh); + +/* + * Read an extent block into *bh. If *bh is NULL, a bh will be + * allocated. This is a cached read. The extent block will be validated + * with ocfs2_validate_extent_block(). + */ +int ocfs2_read_extent_block(struct ocfs2_caching_info *ci, u64 eb_blkno, + struct buffer_head **bh); + +struct ocfs2_alloc_context; +int ocfs2_insert_extent(handle_t *handle, + struct ocfs2_extent_tree *et, + u32 cpos, + u64 start_blk, + u32 new_clusters, + u8 flags, + struct ocfs2_alloc_context *meta_ac); + +enum ocfs2_alloc_restarted { + RESTART_NONE = 0, + RESTART_TRANS, + RESTART_META +}; +int ocfs2_add_clusters_in_btree(handle_t *handle, + struct ocfs2_extent_tree *et, + u32 *logical_offset, + u32 clusters_to_add, + int mark_unwritten, + struct ocfs2_alloc_context *data_ac, + struct ocfs2_alloc_context *meta_ac, + enum ocfs2_alloc_restarted *reason_ret); +struct ocfs2_cached_dealloc_ctxt; +struct ocfs2_path; +int ocfs2_split_extent(handle_t *handle, + struct ocfs2_extent_tree *et, + struct ocfs2_path *path, + int split_index, + struct ocfs2_extent_rec *split_rec, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_cached_dealloc_ctxt *dealloc); +int ocfs2_mark_extent_written(struct inode *inode, + struct ocfs2_extent_tree *et, + handle_t *handle, u32 cpos, u32 len, u32 phys, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_cached_dealloc_ctxt *dealloc); +int ocfs2_change_extent_flag(handle_t *handle, + struct ocfs2_extent_tree *et, + u32 cpos, u32 len, u32 phys, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_cached_dealloc_ctxt *dealloc, + int new_flags, int clear_flags); +int ocfs2_remove_extent(handle_t *handle, struct ocfs2_extent_tree *et, + u32 cpos, u32 len, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_cached_dealloc_ctxt *dealloc); +int ocfs2_remove_btree_range(struct inode *inode, + struct ocfs2_extent_tree *et, + u32 cpos, u32 phys_cpos, u32 len, int flags, + struct ocfs2_cached_dealloc_ctxt *dealloc, + u64 refcount_loc, bool refcount_tree_locked); + +int ocfs2_num_free_extents(struct ocfs2_extent_tree *et); + +/* + * how many new metadata chunks would an allocation need at maximum? + * + * Please note that the caller must make sure that root_el is the root + * of extent tree. So for an inode, it should be &fe->id2.i_list. Otherwise + * the result may be wrong. + */ +static inline int ocfs2_extend_meta_needed(struct ocfs2_extent_list *root_el) +{ + /* + * Rather than do all the work of determining how much we need + * (involves a ton of reads and locks), just ask for the + * maximal limit. That's a tree depth shift. So, one block for + * level of the tree (current l_tree_depth), one block for the + * new tree_depth==0 extent_block, and one block at the new + * top-of-the tree. + */ + return le16_to_cpu(root_el->l_tree_depth) + 2; +} + +void ocfs2_dinode_new_extent_list(struct inode *inode, struct ocfs2_dinode *di); +void ocfs2_set_inode_data_inline(struct inode *inode, struct ocfs2_dinode *di); +int ocfs2_convert_inline_data_to_extents(struct inode *inode, + struct buffer_head *di_bh); + +int ocfs2_truncate_log_init(struct ocfs2_super *osb); +void ocfs2_truncate_log_shutdown(struct ocfs2_super *osb); +void ocfs2_schedule_truncate_log_flush(struct ocfs2_super *osb, + int cancel); +int ocfs2_flush_truncate_log(struct ocfs2_super *osb); +int ocfs2_begin_truncate_log_recovery(struct ocfs2_super *osb, + int slot_num, + struct ocfs2_dinode **tl_copy); +int ocfs2_complete_truncate_log_recovery(struct ocfs2_super *osb, + struct ocfs2_dinode *tl_copy); +int ocfs2_truncate_log_needs_flush(struct ocfs2_super *osb); +int ocfs2_truncate_log_append(struct ocfs2_super *osb, + handle_t *handle, + u64 start_blk, + unsigned int num_clusters); +int __ocfs2_flush_truncate_log(struct ocfs2_super *osb); +int ocfs2_try_to_free_truncate_log(struct ocfs2_super *osb, + unsigned int needed); + +/* + * Process local structure which describes the block unlinks done + * during an operation. This is populated via + * ocfs2_cache_block_dealloc(). + * + * ocfs2_run_deallocs() should be called after the potentially + * de-allocating routines. No journal handles should be open, and most + * locks should have been dropped. + */ +struct ocfs2_cached_dealloc_ctxt { + struct ocfs2_per_slot_free_list *c_first_suballocator; + struct ocfs2_cached_block_free *c_global_allocator; +}; +static inline void ocfs2_init_dealloc_ctxt(struct ocfs2_cached_dealloc_ctxt *c) +{ + c->c_first_suballocator = NULL; + c->c_global_allocator = NULL; +} +int ocfs2_cache_cluster_dealloc(struct ocfs2_cached_dealloc_ctxt *ctxt, + u64 blkno, unsigned int bit); +int ocfs2_cache_block_dealloc(struct ocfs2_cached_dealloc_ctxt *ctxt, + int type, int slot, u64 suballoc, u64 blkno, + unsigned int bit); +static inline int ocfs2_dealloc_has_cluster(struct ocfs2_cached_dealloc_ctxt *c) +{ + return c->c_global_allocator != NULL; +} +int ocfs2_run_deallocs(struct ocfs2_super *osb, + struct ocfs2_cached_dealloc_ctxt *ctxt); + +struct ocfs2_truncate_context { + struct ocfs2_cached_dealloc_ctxt tc_dealloc; + int tc_ext_alloc_locked; /* is it cluster locked? */ + /* these get destroyed once it's passed to ocfs2_commit_truncate. */ + struct buffer_head *tc_last_eb_bh; +}; + +int ocfs2_zero_range_for_truncate(struct inode *inode, handle_t *handle, + u64 range_start, u64 range_end); +int ocfs2_commit_truncate(struct ocfs2_super *osb, + struct inode *inode, + struct buffer_head *di_bh); +int ocfs2_truncate_inline(struct inode *inode, struct buffer_head *di_bh, + unsigned int start, unsigned int end, int trunc); + +int ocfs2_find_leaf(struct ocfs2_caching_info *ci, + struct ocfs2_extent_list *root_el, u32 cpos, + struct buffer_head **leaf_bh); +int ocfs2_search_extent_list(struct ocfs2_extent_list *el, u32 v_cluster); + +int ocfs2_trim_fs(struct super_block *sb, struct fstrim_range *range); +/* + * Helper function to look at the # of clusters in an extent record. + */ +static inline unsigned int ocfs2_rec_clusters(struct ocfs2_extent_list *el, + struct ocfs2_extent_rec *rec) +{ + /* + * Cluster count in extent records is slightly different + * between interior nodes and leaf nodes. This is to support + * unwritten extents which need a flags field in leaf node + * records, thus shrinking the available space for a clusters + * field. + */ + if (el->l_tree_depth) + return le32_to_cpu(rec->e_int_clusters); + else + return le16_to_cpu(rec->e_leaf_clusters); +} + +/* + * This is only valid for leaf nodes, which are the only ones that can + * have empty extents anyway. + */ +static inline int ocfs2_is_empty_extent(struct ocfs2_extent_rec *rec) +{ + return !rec->e_leaf_clusters; +} + +void ocfs2_map_and_dirty_folio(struct inode *inode, handle_t *handle, + size_t from, size_t to, struct folio *folio, int zero, + u64 *phys); +/* + * Structures which describe a path through a btree, and functions to + * manipulate them. + * + * The idea here is to be as generic as possible with the tree + * manipulation code. + */ +struct ocfs2_path_item { + struct buffer_head *bh; + struct ocfs2_extent_list *el; +}; + +#define OCFS2_MAX_PATH_DEPTH 5 + +struct ocfs2_path { + int p_tree_depth; + ocfs2_journal_access_func p_root_access; + struct ocfs2_path_item p_node[OCFS2_MAX_PATH_DEPTH]; +}; + +#define path_root_bh(_path) ((_path)->p_node[0].bh) +#define path_root_el(_path) ((_path)->p_node[0].el) +#define path_root_access(_path)((_path)->p_root_access) +#define path_leaf_bh(_path) ((_path)->p_node[(_path)->p_tree_depth].bh) +#define path_leaf_el(_path) ((_path)->p_node[(_path)->p_tree_depth].el) +#define path_num_items(_path) ((_path)->p_tree_depth + 1) + +void ocfs2_reinit_path(struct ocfs2_path *path, int keep_root); +void ocfs2_free_path(struct ocfs2_path *path); +int ocfs2_find_path(struct ocfs2_caching_info *ci, + struct ocfs2_path *path, + u32 cpos); +struct ocfs2_path *ocfs2_new_path_from_path(struct ocfs2_path *path); +struct ocfs2_path *ocfs2_new_path_from_et(struct ocfs2_extent_tree *et); +int ocfs2_path_bh_journal_access(handle_t *handle, + struct ocfs2_caching_info *ci, + struct ocfs2_path *path, + int idx); +int ocfs2_journal_access_path(struct ocfs2_caching_info *ci, + handle_t *handle, + struct ocfs2_path *path); +int ocfs2_find_cpos_for_right_leaf(struct super_block *sb, + struct ocfs2_path *path, u32 *cpos); +int ocfs2_find_cpos_for_left_leaf(struct super_block *sb, + struct ocfs2_path *path, u32 *cpos); +int ocfs2_find_subtree_root(struct ocfs2_extent_tree *et, + struct ocfs2_path *left, + struct ocfs2_path *right); +#endif /* OCFS2_ALLOC_H */ diff --git a/fs/ocfs2/aops.c b/fs/ocfs2/aops.c new file mode 100644 index 000000000..4acdbb708 --- /dev/null +++ b/fs/ocfs2/aops.c @@ -0,0 +1,2479 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ocfs2.h" + +#include "alloc.h" +#include "aops.h" +#include "dlmglue.h" +#include "extent_map.h" +#include "file.h" +#include "inode.h" +#include "journal.h" +#include "suballoc.h" +#include "super.h" +#include "symlink.h" +#include "refcounttree.h" +#include "ocfs2_trace.h" + +#include "buffer_head_io.h" +#include "dir.h" +#include "namei.h" +#include "sysfile.h" + +#define OCFS2_DIO_MARK_EXTENT_BATCH 200 + +static int ocfs2_symlink_get_block(struct inode *inode, sector_t iblock, + struct buffer_head *bh_result, int create) +{ + int err = -EIO; + int status; + struct ocfs2_dinode *fe = NULL; + struct buffer_head *bh = NULL; + struct buffer_head *buffer_cache_bh = NULL; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + trace_ocfs2_symlink_get_block( + (unsigned long long)OCFS2_I(inode)->ip_blkno, + (unsigned long long)iblock, bh_result, create); + + BUG_ON(ocfs2_inode_is_fast_symlink(inode)); + + if ((iblock << inode->i_sb->s_blocksize_bits) > PATH_MAX + 1) { + mlog(ML_ERROR, "block offset > PATH_MAX: %llu", + (unsigned long long)iblock); + goto bail; + } + + status = ocfs2_read_inode_block(inode, &bh); + if (status < 0) { + mlog_errno(status); + goto bail; + } + fe = (struct ocfs2_dinode *) bh->b_data; + + if ((u64)iblock >= ocfs2_clusters_to_blocks(inode->i_sb, + le32_to_cpu(fe->i_clusters))) { + err = -ENOMEM; + mlog(ML_ERROR, "block offset is outside the allocated size: " + "%llu\n", (unsigned long long)iblock); + goto bail; + } + + /* We don't use the page cache to create symlink data, so if + * need be, copy it over from the buffer cache. */ + if (!buffer_uptodate(bh_result) && ocfs2_inode_is_new(inode)) { + u64 blkno = le64_to_cpu(fe->id2.i_list.l_recs[0].e_blkno) + + iblock; + buffer_cache_bh = sb_getblk(osb->sb, blkno); + if (!buffer_cache_bh) { + err = -ENOMEM; + mlog(ML_ERROR, "couldn't getblock for symlink!\n"); + goto bail; + } + + /* we haven't locked out transactions, so a commit + * could've happened. Since we've got a reference on + * the bh, even if it commits while we're doing the + * copy, the data is still good. */ + if (buffer_jbd(buffer_cache_bh) && ocfs2_inode_is_new(inode)) { + memcpy_to_folio(bh_result->b_folio, + bh_result->b_size * iblock, + buffer_cache_bh->b_data, + bh_result->b_size); + set_buffer_uptodate(bh_result); + } + brelse(buffer_cache_bh); + } + + map_bh(bh_result, inode->i_sb, + le64_to_cpu(fe->id2.i_list.l_recs[0].e_blkno) + iblock); + + err = 0; + +bail: + brelse(bh); + + return err; +} + +static int ocfs2_lock_get_block(struct inode *inode, sector_t iblock, + struct buffer_head *bh_result, int create) +{ + int ret = 0; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + + down_read(&oi->ip_alloc_sem); + ret = ocfs2_get_block(inode, iblock, bh_result, create); + up_read(&oi->ip_alloc_sem); + + return ret; +} + +int ocfs2_get_block(struct inode *inode, sector_t iblock, + struct buffer_head *bh_result, int create) +{ + int err = 0; + unsigned int ext_flags; + u64 max_blocks = bh_result->b_size >> inode->i_blkbits; + u64 p_blkno, count, past_eof; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + trace_ocfs2_get_block((unsigned long long)OCFS2_I(inode)->ip_blkno, + (unsigned long long)iblock, bh_result, create); + + if (OCFS2_I(inode)->ip_flags & OCFS2_INODE_SYSTEM_FILE) + mlog(ML_NOTICE, "get_block on system inode 0x%p (%llu)\n", + inode, inode->i_ino); + + if (S_ISLNK(inode->i_mode)) { + /* this always does I/O for some reason. */ + err = ocfs2_symlink_get_block(inode, iblock, bh_result, create); + goto bail; + } + + err = ocfs2_extent_map_get_blocks(inode, iblock, &p_blkno, &count, + &ext_flags); + if (err) { + mlog(ML_ERROR, "get_blocks() failed, inode: 0x%p, " + "block: %llu\n", inode, (unsigned long long)iblock); + goto bail; + } + + if (max_blocks < count) + count = max_blocks; + + /* + * ocfs2 never allocates in this function - the only time we + * need to use BH_New is when we're extending i_size on a file + * system which doesn't support holes, in which case BH_New + * allows __block_write_begin() to zero. + * + * If we see this on a sparse file system, then a truncate has + * raced us and removed the cluster. In this case, we clear + * the buffers dirty and uptodate bits and let the buffer code + * ignore it as a hole. + */ + if (create && p_blkno == 0 && ocfs2_sparse_alloc(osb)) { + clear_buffer_dirty(bh_result); + clear_buffer_uptodate(bh_result); + goto bail; + } + + /* Treat the unwritten extent as a hole for zeroing purposes. */ + if (p_blkno && !(ext_flags & OCFS2_EXT_UNWRITTEN)) + map_bh(bh_result, inode->i_sb, p_blkno); + + bh_result->b_size = count << inode->i_blkbits; + + if (!ocfs2_sparse_alloc(osb)) { + if (p_blkno == 0) { + err = -EIO; + mlog(ML_ERROR, + "iblock = %llu p_blkno = %llu blkno=(%llu)\n", + (unsigned long long)iblock, + (unsigned long long)p_blkno, + (unsigned long long)OCFS2_I(inode)->ip_blkno); + mlog(ML_ERROR, "Size %llu, clusters %u\n", (unsigned long long)i_size_read(inode), OCFS2_I(inode)->ip_clusters); + dump_stack(); + goto bail; + } + } + + past_eof = ocfs2_blocks_for_bytes(inode->i_sb, i_size_read(inode)); + + trace_ocfs2_get_block_end((unsigned long long)OCFS2_I(inode)->ip_blkno, + (unsigned long long)past_eof); + if (create && (iblock >= past_eof)) + set_buffer_new(bh_result); + +bail: + if (err < 0) + err = -EIO; + + return err; +} + +int ocfs2_read_inline_data(struct inode *inode, struct folio *folio, + struct buffer_head *di_bh) +{ + loff_t size; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data; + + if (!(le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_DATA_FL)) { + ocfs2_error(inode->i_sb, "Inode %llu lost inline data flag\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno); + return -EROFS; + } + + size = i_size_read(inode); + + if (size > folio_size(folio) || + size > ocfs2_max_inline_data_with_xattr(inode->i_sb, di)) { + ocfs2_error(inode->i_sb, + "Inode %llu has with inline data has bad size: %Lu\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno, + (unsigned long long)size); + return -EROFS; + } + + folio_fill_tail(folio, 0, di->id2.i_data.id_data, size); + folio_mark_uptodate(folio); + + return 0; +} + +static int ocfs2_readpage_inline(struct inode *inode, struct folio *folio) +{ + int ret; + struct buffer_head *di_bh = NULL; + + BUG_ON(!folio_test_locked(folio)); + BUG_ON(!(OCFS2_I(inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL)); + + ret = ocfs2_read_inode_block(inode, &di_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_read_inline_data(inode, folio, di_bh); +out: + folio_unlock(folio); + + brelse(di_bh); + return ret; +} + +static int ocfs2_read_folio(struct file *file, struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + loff_t start = folio_pos(folio); + int ret, unlock = 1; + + trace_ocfs2_readpage((unsigned long long)oi->ip_blkno, folio->index); + + ret = ocfs2_inode_lock_with_folio(inode, NULL, 0, folio); + if (ret != 0) { + if (ret == AOP_TRUNCATED_PAGE) + unlock = 0; + mlog_errno(ret); + goto out; + } + + if (down_read_trylock(&oi->ip_alloc_sem) == 0) { + /* + * Unlock the folio and cycle ip_alloc_sem so that we don't + * busyloop waiting for ip_alloc_sem to unlock + */ + ret = AOP_TRUNCATED_PAGE; + folio_unlock(folio); + unlock = 0; + down_read(&oi->ip_alloc_sem); + up_read(&oi->ip_alloc_sem); + goto out_inode_unlock; + } + + /* + * i_size might have just been updated as we grabbed the meta lock. We + * might now be discovering a truncate that hit on another node. + * block_read_full_folio->get_block freaks out if it is asked to read + * beyond the end of a file, so we check here. Callers + * (generic_file_read, vm_ops->fault) are clever enough to check i_size + * and notice that the folio they just read isn't needed. + * + * XXX sys_readahead() seems to get that wrong? + */ + if (start >= i_size_read(inode)) { + folio_zero_segment(folio, 0, folio_size(folio)); + folio_mark_uptodate(folio); + ret = 0; + goto out_alloc; + } + + if (oi->ip_dyn_features & OCFS2_INLINE_DATA_FL) + ret = ocfs2_readpage_inline(inode, folio); + else + ret = block_read_full_folio(folio, ocfs2_get_block); + unlock = 0; + +out_alloc: + up_read(&oi->ip_alloc_sem); +out_inode_unlock: + ocfs2_inode_unlock(inode, 0); +out: + if (unlock) + folio_unlock(folio); + return ret; +} + +/* + * This is used only for read-ahead. Failures or difficult to handle + * situations are safe to ignore. + * + * Right now, we don't bother with BH_Boundary - in-inode extent lists + * are quite large (243 extents on 4k blocks), so most inodes don't + * grow out to a tree. If need be, detecting boundary extents could + * trivially be added in a future version of ocfs2_get_block(). + */ +static void ocfs2_readahead(struct readahead_control *rac) +{ + int ret; + struct inode *inode = rac->mapping->host; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + + /* + * Use the nonblocking flag for the dlm code to avoid page + * lock inversion, but don't bother with retrying. + */ + ret = ocfs2_inode_lock_full(inode, NULL, 0, OCFS2_LOCK_NONBLOCK); + if (ret) + return; + + if (down_read_trylock(&oi->ip_alloc_sem) == 0) + goto out_unlock; + + /* + * Don't bother with inline-data. There isn't anything + * to read-ahead in that case anyway... + */ + if (oi->ip_dyn_features & OCFS2_INLINE_DATA_FL) + goto out_up; + + /* + * Check whether a remote node truncated this file - we just + * drop out in that case as it's not worth handling here. + */ + if (readahead_pos(rac) >= i_size_read(inode)) + goto out_up; + + mpage_readahead(rac, ocfs2_get_block); + +out_up: + up_read(&oi->ip_alloc_sem); +out_unlock: + ocfs2_inode_unlock(inode, 0); +} + +/* Note: Because we don't support holes, our allocation has + * already happened (allocation writes zeros to the file data) + * so we don't have to worry about ordered writes in + * ocfs2_writepages. + * + * ->writepages is called during the process of invalidating the page cache + * during blocked lock processing. It can't block on any cluster locks + * to during block mapping. It's relying on the fact that the block + * mapping can't have disappeared under the dirty pages that it is + * being asked to write back. + */ +static int ocfs2_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + return mpage_writepages(mapping, wbc, ocfs2_get_block); +} + +/* Taken from ext3. We don't necessarily need the full blown + * functionality yet, but IMHO it's better to cut and paste the whole + * thing so we can avoid introducing our own bugs (and easily pick up + * their fixes when they happen) --Mark */ +int walk_page_buffers( handle_t *handle, + struct buffer_head *head, + unsigned from, + unsigned to, + int *partial, + int (*fn)( handle_t *handle, + struct buffer_head *bh)) +{ + struct buffer_head *bh; + unsigned block_start, block_end; + unsigned blocksize = head->b_size; + int err, ret = 0; + struct buffer_head *next; + + for ( bh = head, block_start = 0; + ret == 0 && (bh != head || !block_start); + block_start = block_end, bh = next) + { + next = bh->b_this_page; + block_end = block_start + blocksize; + if (block_end <= from || block_start >= to) { + if (partial && !buffer_uptodate(bh)) + *partial = 1; + continue; + } + err = (*fn)(handle, bh); + if (!ret) + ret = err; + } + return ret; +} + +static sector_t ocfs2_bmap(struct address_space *mapping, sector_t block) +{ + sector_t status; + u64 p_blkno = 0; + int err = 0; + struct inode *inode = mapping->host; + + trace_ocfs2_bmap((unsigned long long)OCFS2_I(inode)->ip_blkno, + (unsigned long long)block); + + /* + * The swap code (ab-)uses ->bmap to get a block mapping and then + * bypasseѕ the file system for actual I/O. We really can't allow + * that on refcounted inodes, so we have to skip out here. And yes, + * 0 is the magic code for a bmap error.. + */ + if (ocfs2_is_refcount_inode(inode)) + return 0; + + /* We don't need to lock journal system files, since they aren't + * accessed concurrently from multiple nodes. + */ + if (!INODE_JOURNAL(inode)) { + err = ocfs2_inode_lock(inode, NULL, 0); + if (err) { + if (err != -ENOENT) + mlog_errno(err); + goto bail; + } + down_read(&OCFS2_I(inode)->ip_alloc_sem); + } + + if (!(OCFS2_I(inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL)) + err = ocfs2_extent_map_get_blocks(inode, block, &p_blkno, NULL, + NULL); + + if (!INODE_JOURNAL(inode)) { + up_read(&OCFS2_I(inode)->ip_alloc_sem); + ocfs2_inode_unlock(inode, 0); + } + + if (err) { + mlog(ML_ERROR, "get_blocks() failed, block = %llu\n", + (unsigned long long)block); + mlog_errno(err); + goto bail; + } + +bail: + status = err ? 0 : p_blkno; + + return status; +} + +static bool ocfs2_release_folio(struct folio *folio, gfp_t wait) +{ + if (!folio_buffers(folio)) + return false; + return try_to_free_buffers(folio); +} + +static void ocfs2_figure_cluster_boundaries(struct ocfs2_super *osb, + u32 cpos, + unsigned int *start, + unsigned int *end) +{ + unsigned int cluster_start = 0, cluster_end = PAGE_SIZE; + + if (unlikely(PAGE_SHIFT > osb->s_clustersize_bits)) { + unsigned int cpp; + + cpp = 1 << (PAGE_SHIFT - osb->s_clustersize_bits); + + cluster_start = cpos % cpp; + cluster_start = cluster_start << osb->s_clustersize_bits; + + cluster_end = cluster_start + osb->s_clustersize; + } + + BUG_ON(cluster_start > PAGE_SIZE); + BUG_ON(cluster_end > PAGE_SIZE); + + if (start) + *start = cluster_start; + if (end) + *end = cluster_end; +} + +/* + * 'from' and 'to' are the region in the page to avoid zeroing. + * + * If pagesize > clustersize, this function will avoid zeroing outside + * of the cluster boundary. + * + * from == to == 0 is code for "zero the entire cluster region" + */ +static void ocfs2_clear_folio_regions(struct folio *folio, + struct ocfs2_super *osb, u32 cpos, + unsigned from, unsigned to) +{ + void *kaddr; + unsigned int cluster_start, cluster_end; + + ocfs2_figure_cluster_boundaries(osb, cpos, &cluster_start, &cluster_end); + + kaddr = kmap_local_folio(folio, 0); + + if (from || to) { + if (from > cluster_start) + memset(kaddr + cluster_start, 0, from - cluster_start); + if (to < cluster_end) + memset(kaddr + to, 0, cluster_end - to); + } else { + memset(kaddr + cluster_start, 0, cluster_end - cluster_start); + } + + kunmap_local(kaddr); +} + +/* + * Nonsparse file systems fully allocate before we get to the write + * code. This prevents ocfs2_write() from tagging the write as an + * allocating one, which means ocfs2_map_folio_blocks() might try to + * read-in the blocks at the tail of our file. Avoid reading them by + * testing i_size against each block offset. + */ +static int ocfs2_should_read_blk(struct inode *inode, struct folio *folio, + unsigned int block_start) +{ + u64 offset = folio_pos(folio) + block_start; + + if (ocfs2_sparse_alloc(OCFS2_SB(inode->i_sb))) + return 1; + + if (i_size_read(inode) > offset) + return 1; + + return 0; +} + +/* + * Some of this taken from __block_write_begin(). We already have our + * mapping by now though, and the entire write will be allocating or + * it won't, so not much need to use BH_New. + * + * This will also skip zeroing, which is handled externally. + */ +int ocfs2_map_folio_blocks(struct folio *folio, u64 *p_blkno, + struct inode *inode, unsigned int from, + unsigned int to, int new) +{ + int ret = 0; + struct buffer_head *head, *bh, *wait[2], **wait_bh = wait; + unsigned int block_end, block_start; + unsigned int bsize = i_blocksize(inode); + + head = folio_buffers(folio); + if (!head) + head = create_empty_buffers(folio, bsize, 0); + + for (bh = head, block_start = 0; bh != head || !block_start; + bh = bh->b_this_page, block_start += bsize) { + block_end = block_start + bsize; + + clear_buffer_new(bh); + + /* + * Ignore blocks outside of our i/o range - + * they may belong to unallocated clusters. + */ + if (block_start >= to || block_end <= from) { + if (folio_test_uptodate(folio)) + set_buffer_uptodate(bh); + continue; + } + + /* + * For an allocating write with cluster size >= page + * size, we always write the entire page. + */ + if (new) + set_buffer_new(bh); + + if (!buffer_mapped(bh)) { + map_bh(bh, inode->i_sb, *p_blkno); + clean_bdev_bh_alias(bh); + } + + if (folio_test_uptodate(folio)) { + set_buffer_uptodate(bh); + } else if (!buffer_uptodate(bh) && !buffer_delay(bh) && + !buffer_new(bh) && + ocfs2_should_read_blk(inode, folio, block_start) && + (block_start < from || block_end > to)) { + bh_read_nowait(bh, 0); + *wait_bh++=bh; + } + + *p_blkno = *p_blkno + 1; + } + + /* + * If we issued read requests - let them complete. + */ + while(wait_bh > wait) { + wait_on_buffer(*--wait_bh); + if (!buffer_uptodate(*wait_bh)) + ret = -EIO; + } + + if (ret == 0 || !new) + return ret; + + /* + * If we get -EIO above, zero out any newly allocated blocks + * to avoid exposing stale data. + */ + bh = head; + block_start = 0; + do { + block_end = block_start + bsize; + if (block_end <= from) + goto next_bh; + if (block_start >= to) + break; + + folio_zero_range(folio, block_start, bh->b_size); + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + +next_bh: + block_start = block_end; + bh = bh->b_this_page; + } while (bh != head); + + return ret; +} + +#if (PAGE_SIZE >= OCFS2_MAX_CLUSTERSIZE) +#define OCFS2_MAX_CTXT_PAGES 1 +#else +#define OCFS2_MAX_CTXT_PAGES (OCFS2_MAX_CLUSTERSIZE / PAGE_SIZE) +#endif + +#define OCFS2_MAX_CLUSTERS_PER_PAGE (PAGE_SIZE / OCFS2_MIN_CLUSTERSIZE) + +struct ocfs2_unwritten_extent { + struct list_head ue_node; + struct list_head ue_ip_node; + u32 ue_cpos; + u32 ue_phys; +}; + +/* + * Describe the state of a single cluster to be written to. + */ +struct ocfs2_write_cluster_desc { + u32 c_cpos; + u32 c_phys; + /* + * Give this a unique field because c_phys eventually gets + * filled. + */ + unsigned c_new; + unsigned c_clear_unwritten; + unsigned c_needs_zero; +}; + +struct ocfs2_write_ctxt { + /* Logical cluster position / len of write */ + u32 w_cpos; + u32 w_clen; + + /* First cluster allocated in a nonsparse extend */ + u32 w_first_new_cpos; + + /* Type of caller. Must be one of buffer, mmap, direct. */ + ocfs2_write_type_t w_type; + + struct ocfs2_write_cluster_desc w_desc[OCFS2_MAX_CLUSTERS_PER_PAGE]; + + /* + * This is true if page_size > cluster_size. + * + * It triggers a set of special cases during write which might + * have to deal with allocating writes to partial pages. + */ + unsigned int w_large_pages; + + /* + * Folios involved in this write. + * + * w_target_folio is the folio being written to by the user. + * + * w_folios is an array of folios which always contains + * w_target_folio, and in the case of an allocating write with + * page_size < cluster size, it will contain zero'd and mapped + * pages adjacent to w_target_folio which need to be written + * out in so that future reads from that region will get + * zero's. + */ + unsigned int w_num_folios; + struct folio *w_folios[OCFS2_MAX_CTXT_PAGES]; + struct folio *w_target_folio; + + /* + * w_target_locked is used for page_mkwrite path indicating no unlocking + * against w_target_folio in ocfs2_write_end_nolock. + */ + unsigned int w_target_locked:1; + + /* + * ocfs2_write_end() uses this to know what the real range to + * write in the target should be. + */ + unsigned int w_target_from; + unsigned int w_target_to; + + /* + * We could use journal_current_handle() but this is cleaner, + * IMHO -Mark + */ + handle_t *w_handle; + + struct buffer_head *w_di_bh; + + struct ocfs2_cached_dealloc_ctxt w_dealloc; + + struct list_head w_unwritten_list; + unsigned int w_unwritten_count; +}; + +void ocfs2_unlock_and_free_folios(struct folio **folios, int num_folios) +{ + int i; + + for(i = 0; i < num_folios; i++) { + if (!folios[i]) + continue; + folio_unlock(folios[i]); + folio_mark_accessed(folios[i]); + folio_put(folios[i]); + } +} + +static void ocfs2_unlock_folios(struct ocfs2_write_ctxt *wc) +{ + int i; + + /* + * w_target_locked is only set to true in the page_mkwrite() case. + * The intent is to allow us to lock the target page from write_begin() + * to write_end(). The caller must hold a ref on w_target_folio. + */ + if (wc->w_target_locked) { + BUG_ON(!wc->w_target_folio); + for (i = 0; i < wc->w_num_folios; i++) { + if (wc->w_target_folio == wc->w_folios[i]) { + wc->w_folios[i] = NULL; + break; + } + } + folio_mark_accessed(wc->w_target_folio); + folio_put(wc->w_target_folio); + } + ocfs2_unlock_and_free_folios(wc->w_folios, wc->w_num_folios); +} + +static void ocfs2_free_unwritten_list(struct inode *inode, + struct list_head *head) +{ + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_unwritten_extent *ue = NULL, *tmp = NULL; + + list_for_each_entry_safe(ue, tmp, head, ue_node) { + list_del(&ue->ue_node); + spin_lock(&oi->ip_lock); + list_del(&ue->ue_ip_node); + spin_unlock(&oi->ip_lock); + kfree(ue); + } +} + +static void ocfs2_free_write_ctxt(struct inode *inode, + struct ocfs2_write_ctxt *wc) +{ + ocfs2_free_unwritten_list(inode, &wc->w_unwritten_list); + ocfs2_unlock_folios(wc); + brelse(wc->w_di_bh); + kfree(wc); +} + +static int ocfs2_alloc_write_ctxt(struct ocfs2_write_ctxt **wcp, + struct ocfs2_super *osb, loff_t pos, + unsigned len, ocfs2_write_type_t type, + struct buffer_head *di_bh) +{ + u32 cend; + struct ocfs2_write_ctxt *wc; + + wc = kzalloc_obj(struct ocfs2_write_ctxt, GFP_NOFS); + if (!wc) + return -ENOMEM; + + wc->w_cpos = pos >> osb->s_clustersize_bits; + wc->w_first_new_cpos = UINT_MAX; + cend = (pos + len - 1) >> osb->s_clustersize_bits; + wc->w_clen = cend - wc->w_cpos + 1; + get_bh(di_bh); + wc->w_di_bh = di_bh; + wc->w_type = type; + + if (unlikely(PAGE_SHIFT > osb->s_clustersize_bits)) + wc->w_large_pages = 1; + else + wc->w_large_pages = 0; + + ocfs2_init_dealloc_ctxt(&wc->w_dealloc); + INIT_LIST_HEAD(&wc->w_unwritten_list); + + *wcp = wc; + + return 0; +} + +/* + * If a page has any new buffers, zero them out here, and mark them uptodate + * and dirty so they'll be written out (in order to prevent uninitialised + * block data from leaking). And clear the new bit. + */ +static void ocfs2_zero_new_buffers(struct folio *folio, size_t from, size_t to) +{ + unsigned int block_start, block_end; + struct buffer_head *head, *bh; + + BUG_ON(!folio_test_locked(folio)); + head = folio_buffers(folio); + if (!head) + return; + + bh = head; + block_start = 0; + do { + block_end = block_start + bh->b_size; + + if (buffer_new(bh)) { + if (block_end > from && block_start < to) { + if (!folio_test_uptodate(folio)) { + unsigned start, end; + + start = max(from, block_start); + end = min(to, block_end); + + folio_zero_segment(folio, start, end); + set_buffer_uptodate(bh); + } + + clear_buffer_new(bh); + mark_buffer_dirty(bh); + } + } + + block_start = block_end; + bh = bh->b_this_page; + } while (bh != head); +} + +/* + * Only called when we have a failure during allocating write to write + * zero's to the newly allocated region. + */ +static void ocfs2_write_failure(struct inode *inode, + struct ocfs2_write_ctxt *wc, + loff_t user_pos, unsigned user_len) +{ + int i; + unsigned from = user_pos & (PAGE_SIZE - 1), + to = user_pos + user_len; + + if (wc->w_target_folio) + ocfs2_zero_new_buffers(wc->w_target_folio, from, to); + + for (i = 0; i < wc->w_num_folios; i++) { + struct folio *folio = wc->w_folios[i]; + + if (folio && folio_buffers(folio)) { + if (ocfs2_should_order_data(inode)) + ocfs2_jbd2_inode_add_write(wc->w_handle, inode, + user_pos, user_len); + + block_commit_write(folio, from, to); + } + } +} + +static int ocfs2_prepare_folio_for_write(struct inode *inode, u64 *p_blkno, + struct ocfs2_write_ctxt *wc, struct folio *folio, u32 cpos, + loff_t user_pos, unsigned user_len, int new) +{ + int ret; + unsigned int map_from = 0, map_to = 0; + unsigned int cluster_start, cluster_end; + unsigned int user_data_from = 0, user_data_to = 0; + + ocfs2_figure_cluster_boundaries(OCFS2_SB(inode->i_sb), cpos, + &cluster_start, &cluster_end); + + /* treat the write as new if the a hole/lseek spanned across + * the page boundary. + */ + new = new | ((i_size_read(inode) <= folio_pos(folio)) && + (folio_pos(folio) <= user_pos)); + + if (folio == wc->w_target_folio) { + map_from = user_pos & (PAGE_SIZE - 1); + map_to = map_from + user_len; + + if (new) + ret = ocfs2_map_folio_blocks(folio, p_blkno, inode, + cluster_start, cluster_end, new); + else + ret = ocfs2_map_folio_blocks(folio, p_blkno, inode, + map_from, map_to, new); + if (ret) { + mlog_errno(ret); + goto out; + } + + user_data_from = map_from; + user_data_to = map_to; + if (new) { + map_from = cluster_start; + map_to = cluster_end; + } + } else { + /* + * If we haven't allocated the new folio yet, we + * shouldn't be writing it out without copying user + * data. This is likely a math error from the caller. + */ + BUG_ON(!new); + + map_from = cluster_start; + map_to = cluster_end; + + ret = ocfs2_map_folio_blocks(folio, p_blkno, inode, + cluster_start, cluster_end, new); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + /* + * Parts of newly allocated folios need to be zero'd. + * + * Above, we have also rewritten 'to' and 'from' - as far as + * the rest of the function is concerned, the entire cluster + * range inside of a folio needs to be written. + * + * We can skip this if the folio is uptodate - it's already + * been zero'd from being read in as a hole. + */ + if (new && !folio_test_uptodate(folio)) + ocfs2_clear_folio_regions(folio, OCFS2_SB(inode->i_sb), + cpos, user_data_from, user_data_to); + + flush_dcache_folio(folio); + +out: + return ret; +} + +/* + * This function will only grab one clusters worth of pages. + */ +static int ocfs2_grab_folios_for_write(struct address_space *mapping, + struct ocfs2_write_ctxt *wc, u32 cpos, loff_t user_pos, + unsigned user_len, int new, struct folio *mmap_folio) +{ + int ret = 0, i; + unsigned long start, target_index, end_index, index; + struct inode *inode = mapping->host; + loff_t last_byte; + + target_index = user_pos >> PAGE_SHIFT; + + /* + * Figure out how many pages we'll be manipulating here. For + * non allocating write, we just change the one + * page. Otherwise, we'll need a whole clusters worth. If we're + * writing past i_size, we only need enough pages to cover the + * last page of the write. + */ + if (new) { + wc->w_num_folios = ocfs2_pages_per_cluster(inode->i_sb); + start = ocfs2_align_clusters_to_page_index(inode->i_sb, cpos); + /* + * We need the index *past* the last page we could possibly + * touch. This is the page past the end of the write or + * i_size, whichever is greater. + */ + last_byte = max(user_pos + user_len, i_size_read(inode)); + BUG_ON(last_byte < 1); + end_index = ((last_byte - 1) >> PAGE_SHIFT) + 1; + if ((start + wc->w_num_folios) > end_index) + wc->w_num_folios = end_index - start; + } else { + wc->w_num_folios = 1; + start = target_index; + } + end_index = (user_pos + user_len - 1) >> PAGE_SHIFT; + + for(i = 0; i < wc->w_num_folios; i++) { + index = start + i; + + if (index >= target_index && index <= end_index && + wc->w_type == OCFS2_WRITE_MMAP) { + /* + * ocfs2_pagemkwrite() is a little different + * and wants us to directly use the page + * passed in. + */ + folio_lock(mmap_folio); + + /* Exit and let the caller retry */ + if (mmap_folio->mapping != mapping) { + WARN_ON(mmap_folio->mapping); + folio_unlock(mmap_folio); + ret = -EAGAIN; + goto out; + } + + folio_get(mmap_folio); + wc->w_folios[i] = mmap_folio; + wc->w_target_locked = true; + } else if (index >= target_index && index <= end_index && + wc->w_type == OCFS2_WRITE_DIRECT) { + /* Direct write has no mapping page. */ + wc->w_folios[i] = NULL; + continue; + } else { + wc->w_folios[i] = __filemap_get_folio(mapping, index, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, + GFP_NOFS); + if (IS_ERR(wc->w_folios[i])) { + ret = PTR_ERR(wc->w_folios[i]); + mlog_errno(ret); + wc->w_folios[i] = NULL; + goto out; + } + } + folio_wait_stable(wc->w_folios[i]); + + if (index == target_index) + wc->w_target_folio = wc->w_folios[i]; + } +out: + if (ret) + wc->w_target_locked = false; + return ret; +} + +/* + * Prepare a single cluster for write one cluster into the file. + */ +static int ocfs2_write_cluster(struct address_space *mapping, + u32 *phys, unsigned int new, + unsigned int clear_unwritten, + unsigned int should_zero, + struct ocfs2_alloc_context *data_ac, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_write_ctxt *wc, u32 cpos, + loff_t user_pos, unsigned user_len) +{ + int ret, i; + u64 p_blkno; + struct inode *inode = mapping->host; + struct ocfs2_extent_tree et; + int bpc = ocfs2_clusters_to_blocks(inode->i_sb, 1); + + if (new) { + u32 tmp_pos; + + /* + * This is safe to call with the page locks - it won't take + * any additional semaphores or cluster locks. + */ + tmp_pos = cpos; + ret = ocfs2_add_inode_data(OCFS2_SB(inode->i_sb), inode, + &tmp_pos, 1, !clear_unwritten, + wc->w_di_bh, wc->w_handle, + data_ac, meta_ac, NULL); + /* + * This shouldn't happen because we must have already + * calculated the correct meta data allocation required. The + * internal tree allocation code should know how to increase + * transaction credits itself. + * + * If need be, we could handle -EAGAIN for a + * RESTART_TRANS here. + */ + mlog_bug_on_msg(ret == -EAGAIN, + "Inode %llu: EAGAIN return during allocation.\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + } else if (clear_unwritten) { + ocfs2_init_dinode_extent_tree(&et, INODE_CACHE(inode), + wc->w_di_bh); + ret = ocfs2_mark_extent_written(inode, &et, + wc->w_handle, cpos, 1, *phys, + meta_ac, &wc->w_dealloc); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + } + + /* + * The only reason this should fail is due to an inability to + * find the extent added. + */ + ret = ocfs2_get_clusters(inode, cpos, phys, NULL, NULL); + if (ret < 0) { + mlog(ML_ERROR, "Get physical blkno failed for inode %llu, " + "at logical cluster %u", + (unsigned long long)OCFS2_I(inode)->ip_blkno, cpos); + goto out; + } + + BUG_ON(*phys == 0); + + p_blkno = ocfs2_clusters_to_blocks(inode->i_sb, *phys); + if (!should_zero) + p_blkno += (user_pos >> inode->i_sb->s_blocksize_bits) & (u64)(bpc - 1); + + for (i = 0; i < wc->w_num_folios; i++) { + int tmpret; + + /* This is the direct io target page. */ + if (wc->w_folios[i] == NULL) { + p_blkno += (1 << (PAGE_SHIFT - inode->i_sb->s_blocksize_bits)); + continue; + } + + tmpret = ocfs2_prepare_folio_for_write(inode, &p_blkno, wc, + wc->w_folios[i], cpos, user_pos, user_len, + should_zero); + if (tmpret) { + mlog_errno(tmpret); + if (ret == 0) + ret = tmpret; + } + } + + /* + * We only have cleanup to do in case of allocating write. + */ + if (ret && new) + ocfs2_write_failure(inode, wc, user_pos, user_len); + +out: + + return ret; +} + +static int ocfs2_write_cluster_by_desc(struct address_space *mapping, + struct ocfs2_alloc_context *data_ac, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_write_ctxt *wc, + loff_t pos, unsigned len) +{ + int ret, i; + loff_t cluster_off; + unsigned int local_len = len; + struct ocfs2_write_cluster_desc *desc; + struct ocfs2_super *osb = OCFS2_SB(mapping->host->i_sb); + + for (i = 0; i < wc->w_clen; i++) { + desc = &wc->w_desc[i]; + + /* + * We have to make sure that the total write passed in + * doesn't extend past a single cluster. + */ + local_len = len; + cluster_off = pos & (osb->s_clustersize - 1); + if ((cluster_off + local_len) > osb->s_clustersize) + local_len = osb->s_clustersize - cluster_off; + + ret = ocfs2_write_cluster(mapping, &desc->c_phys, + desc->c_new, + desc->c_clear_unwritten, + desc->c_needs_zero, + data_ac, meta_ac, + wc, desc->c_cpos, pos, local_len); + if (ret) { + mlog_errno(ret); + goto out; + } + + len -= local_len; + pos += local_len; + } + + ret = 0; +out: + return ret; +} + +/* + * ocfs2_write_end() wants to know which parts of the target page it + * should complete the write on. It's easiest to compute them ahead of + * time when a more complete view of the write is available. + */ +static void ocfs2_set_target_boundaries(struct ocfs2_super *osb, + struct ocfs2_write_ctxt *wc, + loff_t pos, unsigned len, int alloc) +{ + struct ocfs2_write_cluster_desc *desc; + + wc->w_target_from = pos & (PAGE_SIZE - 1); + wc->w_target_to = wc->w_target_from + len; + + if (alloc == 0) + return; + + /* + * Allocating write - we may have different boundaries based + * on page size and cluster size. + * + * NOTE: We can no longer compute one value from the other as + * the actual write length and user provided length may be + * different. + */ + + if (wc->w_large_pages) { + /* + * We only care about the 1st and last cluster within + * our range and whether they should be zero'd or not. Either + * value may be extended out to the start/end of a + * newly allocated cluster. + */ + desc = &wc->w_desc[0]; + if (desc->c_needs_zero) + ocfs2_figure_cluster_boundaries(osb, + desc->c_cpos, + &wc->w_target_from, + NULL); + + desc = &wc->w_desc[wc->w_clen - 1]; + if (desc->c_needs_zero) + ocfs2_figure_cluster_boundaries(osb, + desc->c_cpos, + NULL, + &wc->w_target_to); + } else { + wc->w_target_from = 0; + wc->w_target_to = PAGE_SIZE; + } +} + +/* + * Check if this extent is marked UNWRITTEN by direct io. If so, we need not to + * do the zero work. And should not to clear UNWRITTEN since it will be cleared + * by the direct io procedure. + * If this is a new extent that allocated by direct io, we should mark it in + * the ip_unwritten_list. + */ +static int ocfs2_unwritten_check(struct inode *inode, + struct ocfs2_write_ctxt *wc, + struct ocfs2_write_cluster_desc *desc) +{ + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_unwritten_extent *ue = NULL, *new = NULL; + int ret = 0; + + if (!desc->c_needs_zero) + return 0; + +retry: + spin_lock(&oi->ip_lock); + /* Needs not to zero no metter buffer or direct. The one who is zero + * the cluster is doing zero. And he will clear unwritten after all + * cluster io finished. */ + list_for_each_entry(ue, &oi->ip_unwritten_list, ue_ip_node) { + if (desc->c_cpos == ue->ue_cpos) { + BUG_ON(desc->c_new); + desc->c_needs_zero = 0; + desc->c_clear_unwritten = 0; + goto unlock; + } + } + + if (wc->w_type != OCFS2_WRITE_DIRECT) + goto unlock; + + if (new == NULL) { + spin_unlock(&oi->ip_lock); + new = kmalloc_obj(struct ocfs2_unwritten_extent, GFP_NOFS); + if (new == NULL) { + ret = -ENOMEM; + goto out; + } + goto retry; + } + /* This direct write will doing zero. */ + new->ue_cpos = desc->c_cpos; + new->ue_phys = desc->c_phys; + desc->c_clear_unwritten = 0; + list_add_tail(&new->ue_ip_node, &oi->ip_unwritten_list); + list_add_tail(&new->ue_node, &wc->w_unwritten_list); + wc->w_unwritten_count++; + new = NULL; +unlock: + spin_unlock(&oi->ip_lock); +out: + kfree(new); + return ret; +} + +/* + * Populate each single-cluster write descriptor in the write context + * with information about the i/o to be done. + * + * Returns the number of clusters that will have to be allocated, as + * well as a worst case estimate of the number of extent records that + * would have to be created during a write to an unwritten region. + */ +static int ocfs2_populate_write_desc(struct inode *inode, + struct ocfs2_write_ctxt *wc, + unsigned int *clusters_to_alloc, + unsigned int *extents_to_split) +{ + int ret; + struct ocfs2_write_cluster_desc *desc; + unsigned int num_clusters = 0; + unsigned int ext_flags = 0; + u32 phys = 0; + int i; + + *clusters_to_alloc = 0; + *extents_to_split = 0; + + for (i = 0; i < wc->w_clen; i++) { + desc = &wc->w_desc[i]; + desc->c_cpos = wc->w_cpos + i; + + if (num_clusters == 0) { + /* + * Need to look up the next extent record. + */ + ret = ocfs2_get_clusters(inode, desc->c_cpos, &phys, + &num_clusters, &ext_flags); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* We should already CoW the refcountd extent. */ + BUG_ON(ext_flags & OCFS2_EXT_REFCOUNTED); + + /* + * Assume worst case - that we're writing in + * the middle of the extent. + * + * We can assume that the write proceeds from + * left to right, in which case the extent + * insert code is smart enough to coalesce the + * next splits into the previous records created. + */ + if (ext_flags & OCFS2_EXT_UNWRITTEN) + *extents_to_split = *extents_to_split + 2; + } else if (phys) { + /* + * Only increment phys if it doesn't describe + * a hole. + */ + phys++; + } + + /* + * If w_first_new_cpos is < UINT_MAX, we have a non-sparse + * file that got extended. w_first_new_cpos tells us + * where the newly allocated clusters are so we can + * zero them. + */ + if (desc->c_cpos >= wc->w_first_new_cpos) { + BUG_ON(phys == 0); + desc->c_needs_zero = 1; + } + + desc->c_phys = phys; + if (phys == 0) { + desc->c_new = 1; + desc->c_needs_zero = 1; + desc->c_clear_unwritten = 1; + *clusters_to_alloc = *clusters_to_alloc + 1; + } + + if (ext_flags & OCFS2_EXT_UNWRITTEN) { + desc->c_clear_unwritten = 1; + desc->c_needs_zero = 1; + } + + ret = ocfs2_unwritten_check(inode, wc, desc); + if (ret) { + mlog_errno(ret); + goto out; + } + + num_clusters--; + } + + ret = 0; +out: + return ret; +} + +static int ocfs2_write_begin_inline(struct address_space *mapping, + struct inode *inode, + struct ocfs2_write_ctxt *wc) +{ + int ret; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct folio *folio; + handle_t *handle; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)wc->w_di_bh->b_data; + + handle = ocfs2_start_trans(osb, OCFS2_INODE_UPDATE_CREDITS); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out; + } + + folio = __filemap_get_folio(mapping, 0, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, GFP_NOFS); + if (IS_ERR(folio)) { + ocfs2_commit_trans(osb, handle); + ret = PTR_ERR(folio); + mlog_errno(ret); + goto out; + } + /* + * If we don't set w_num_folios then this folio won't get unlocked + * and freed on cleanup of the write context. + */ + wc->w_target_folio = folio; + wc->w_folios[0] = folio; + wc->w_num_folios = 1; + + ret = ocfs2_journal_access_di(handle, INODE_CACHE(inode), wc->w_di_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + ocfs2_commit_trans(osb, handle); + + mlog_errno(ret); + goto out; + } + + if (!(OCFS2_I(inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL)) + ocfs2_set_inode_data_inline(inode, di); + + if (!folio_test_uptodate(folio)) { + ret = ocfs2_read_inline_data(inode, folio, wc->w_di_bh); + if (ret) { + ocfs2_commit_trans(osb, handle); + + goto out; + } + } + + wc->w_handle = handle; +out: + return ret; +} + +int ocfs2_size_fits_inline_data(struct buffer_head *di_bh, u64 new_size) +{ + struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data; + + if (new_size <= le16_to_cpu(di->id2.i_data.id_count)) + return 1; + return 0; +} + +static int ocfs2_try_to_write_inline_data(struct address_space *mapping, + struct inode *inode, loff_t pos, size_t len, + struct folio *mmap_folio, struct ocfs2_write_ctxt *wc) +{ + int ret, written = 0; + loff_t end = pos + len; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_dinode *di = NULL; + + trace_ocfs2_try_to_write_inline_data((unsigned long long)oi->ip_blkno, + len, (unsigned long long)pos, + oi->ip_dyn_features); + + /* + * Handle inodes which already have inline data 1st. + */ + if (oi->ip_dyn_features & OCFS2_INLINE_DATA_FL) { + if (mmap_folio == NULL && + ocfs2_size_fits_inline_data(wc->w_di_bh, end)) + goto do_inline_write; + + /* + * The write won't fit - we have to give this inode an + * inline extent list now. + */ + ret = ocfs2_convert_inline_data_to_extents(inode, wc->w_di_bh); + if (ret) + mlog_errno(ret); + goto out; + } + + /* + * Check whether the inode can accept inline data. + */ + if (oi->ip_clusters != 0 || i_size_read(inode) != 0) + return 0; + + /* + * Check whether the write can fit. + */ + di = (struct ocfs2_dinode *)wc->w_di_bh->b_data; + if (mmap_folio || + end > ocfs2_max_inline_data_with_xattr(inode->i_sb, di)) + return 0; + +do_inline_write: + ret = ocfs2_write_begin_inline(mapping, inode, wc); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * This signals to the caller that the data can be written + * inline. + */ + written = 1; +out: + return written ? written : ret; +} + +/* + * This function only does anything for file systems which can't + * handle sparse files. + * + * What we want to do here is fill in any hole between the current end + * of allocation and the end of our write. That way the rest of the + * write path can treat it as an non-allocating write, which has no + * special case code for sparse/nonsparse files. + */ +static int ocfs2_expand_nonsparse_inode(struct inode *inode, + struct buffer_head *di_bh, + loff_t pos, unsigned len, + struct ocfs2_write_ctxt *wc) +{ + int ret; + loff_t newsize = pos + len; + + BUG_ON(ocfs2_sparse_alloc(OCFS2_SB(inode->i_sb))); + + if (newsize <= i_size_read(inode)) + return 0; + + ret = ocfs2_extend_no_holes(inode, di_bh, newsize, pos); + if (ret) + mlog_errno(ret); + + /* There is no wc if this is call from direct. */ + if (wc) + wc->w_first_new_cpos = + ocfs2_clusters_for_bytes(inode->i_sb, i_size_read(inode)); + + return ret; +} + +static int ocfs2_zero_tail(struct inode *inode, struct buffer_head *di_bh, + loff_t pos) +{ + int ret = 0; + + BUG_ON(!ocfs2_sparse_alloc(OCFS2_SB(inode->i_sb))); + if (pos > i_size_read(inode)) + ret = ocfs2_zero_extend(inode, di_bh, pos); + + return ret; +} + +int ocfs2_write_begin_nolock(struct address_space *mapping, + loff_t pos, unsigned len, ocfs2_write_type_t type, + struct folio **foliop, void **fsdata, + struct buffer_head *di_bh, struct folio *mmap_folio) +{ + int ret, cluster_of_pages, credits = OCFS2_INODE_UPDATE_CREDITS; + unsigned int clusters_to_alloc, extents_to_split, clusters_need = 0; + struct ocfs2_write_ctxt *wc; + struct inode *inode = mapping->host; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct ocfs2_dinode *di; + struct ocfs2_alloc_context *data_ac = NULL; + struct ocfs2_alloc_context *meta_ac = NULL; + handle_t *handle; + struct ocfs2_extent_tree et; + int try_free = 1, ret1; + +try_again: + ret = ocfs2_alloc_write_ctxt(&wc, osb, pos, len, type, di_bh); + if (ret) { + mlog_errno(ret); + return ret; + } + + if (ocfs2_supports_inline_data(osb)) { + ret = ocfs2_try_to_write_inline_data(mapping, inode, pos, len, + mmap_folio, wc); + if (ret == 1) { + ret = 0; + goto success; + } + if (ret < 0) { + mlog_errno(ret); + goto out; + } + } + + /* Direct io change i_size late, should not zero tail here. */ + if (type != OCFS2_WRITE_DIRECT) { + if (ocfs2_sparse_alloc(osb)) + ret = ocfs2_zero_tail(inode, di_bh, pos); + else + ret = ocfs2_expand_nonsparse_inode(inode, di_bh, pos, + len, wc); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + ret = ocfs2_check_range_for_refcount(inode, pos, len); + if (ret < 0) { + mlog_errno(ret); + goto out; + } else if (ret == 1) { + clusters_need = wc->w_clen; + ret = ocfs2_refcount_cow(inode, di_bh, + wc->w_cpos, wc->w_clen, UINT_MAX); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + ret = ocfs2_populate_write_desc(inode, wc, &clusters_to_alloc, + &extents_to_split); + if (ret) { + mlog_errno(ret); + goto out; + } + clusters_need += clusters_to_alloc; + + di = (struct ocfs2_dinode *)wc->w_di_bh->b_data; + + trace_ocfs2_write_begin_nolock( + (unsigned long long)OCFS2_I(inode)->ip_blkno, + (long long)i_size_read(inode), + le32_to_cpu(di->i_clusters), + pos, len, type, mmap_folio, + clusters_to_alloc, extents_to_split); + + /* + * We set w_target_from, w_target_to here so that + * ocfs2_write_end() knows which range in the target page to + * write out. An allocation requires that we write the entire + * cluster range. + */ + if (clusters_to_alloc || extents_to_split) { + /* + * XXX: We are stretching the limits of + * ocfs2_lock_allocators(). It greatly over-estimates + * the work to be done. + */ + ocfs2_init_dinode_extent_tree(&et, INODE_CACHE(inode), + wc->w_di_bh); + ret = ocfs2_lock_allocators(inode, &et, + clusters_to_alloc, extents_to_split, + &data_ac, &meta_ac); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (data_ac) + data_ac->ac_resv = &OCFS2_I(inode)->ip_la_data_resv; + + credits = ocfs2_calc_extend_credits(inode->i_sb, + &di->id2.i_list); + } else if (type == OCFS2_WRITE_DIRECT) + /* direct write needs not to start trans if no extents alloc. */ + goto success; + + /* + * We have to zero sparse allocated clusters, unwritten extent clusters, + * and non-sparse clusters we just extended. For non-sparse writes, + * we know zeros will only be needed in the first and/or last cluster. + */ + if (wc->w_clen && (wc->w_desc[0].c_needs_zero || + wc->w_desc[wc->w_clen - 1].c_needs_zero)) + cluster_of_pages = 1; + else + cluster_of_pages = 0; + + ocfs2_set_target_boundaries(osb, wc, pos, len, cluster_of_pages); + + handle = ocfs2_start_trans(osb, credits); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out; + } + + wc->w_handle = handle; + + if (clusters_to_alloc) { + ret = dquot_alloc_space_nodirty(inode, + ocfs2_clusters_to_bytes(osb->sb, clusters_to_alloc)); + if (ret) + goto out_commit; + } + + ret = ocfs2_journal_access_di(handle, INODE_CACHE(inode), wc->w_di_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out_quota; + } + + /* + * Fill our folio array first. That way we've grabbed enough so + * that we can zero and flush if we error after adding the + * extent. + */ + ret = ocfs2_grab_folios_for_write(mapping, wc, wc->w_cpos, pos, len, + cluster_of_pages, mmap_folio); + if (ret) { + /* + * ocfs2_grab_folios_for_write() returns -EAGAIN if it + * could not lock the target folio. In this case, we exit + * with no error and no target folio. This will trigger + * the caller, page_mkwrite(), to re-try the operation. + */ + if (type == OCFS2_WRITE_MMAP && ret == -EAGAIN) { + BUG_ON(wc->w_target_folio); + ret = 0; + goto out_quota; + } + + mlog_errno(ret); + goto out_quota; + } + + ret = ocfs2_write_cluster_by_desc(mapping, data_ac, meta_ac, wc, pos, + len); + if (ret) { + mlog_errno(ret); + goto out_quota; + } + + if (data_ac) + ocfs2_free_alloc_context(data_ac); + if (meta_ac) + ocfs2_free_alloc_context(meta_ac); + +success: + if (foliop) + *foliop = wc->w_target_folio; + *fsdata = wc; + return 0; +out_quota: + if (clusters_to_alloc) + dquot_free_space(inode, + ocfs2_clusters_to_bytes(osb->sb, clusters_to_alloc)); +out_commit: + ocfs2_commit_trans(osb, handle); + +out: + /* + * The mmapped page won't be unlocked in ocfs2_free_write_ctxt(), + * even in case of error here like ENOSPC and ENOMEM. So, we need + * to unlock the target page manually to prevent deadlocks when + * retrying again on ENOSPC, or when returning non-VM_FAULT_LOCKED + * to VM code. + */ + if (wc->w_target_locked) + folio_unlock(mmap_folio); + + ocfs2_free_write_ctxt(inode, wc); + + if (data_ac) { + ocfs2_free_alloc_context(data_ac); + data_ac = NULL; + } + if (meta_ac) { + ocfs2_free_alloc_context(meta_ac); + meta_ac = NULL; + } + + if (ret == -ENOSPC && try_free) { + /* + * Try to free some truncate log so that we can have enough + * clusters to allocate. + */ + try_free = 0; + + ret1 = ocfs2_try_to_free_truncate_log(osb, clusters_need); + if (ret1 == 1) + goto try_again; + + if (ret1 < 0) + mlog_errno(ret1); + } + + return ret; +} + +static int ocfs2_write_begin(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, + struct folio **foliop, void **fsdata) +{ + int ret; + struct buffer_head *di_bh = NULL; + struct inode *inode = mapping->host; + + ret = ocfs2_inode_lock(inode, &di_bh, 1); + if (ret) { + mlog_errno(ret); + return ret; + } + + /* + * Take alloc sem here to prevent concurrent lookups. That way + * the mapping, zeroing and tree manipulation within + * ocfs2_write() will be safe against ->read_folio(). This + * should also serve to lock out allocation from a shared + * writeable region. + */ + down_write(&OCFS2_I(inode)->ip_alloc_sem); + + ret = ocfs2_write_begin_nolock(mapping, pos, len, OCFS2_WRITE_BUFFER, + foliop, fsdata, di_bh, NULL); + if (ret) { + mlog_errno(ret); + goto out_fail; + } + + brelse(di_bh); + + return 0; + +out_fail: + up_write(&OCFS2_I(inode)->ip_alloc_sem); + + brelse(di_bh); + ocfs2_inode_unlock(inode, 1); + + return ret; +} + +static void ocfs2_write_end_inline(struct inode *inode, loff_t pos, + unsigned len, unsigned *copied, + struct ocfs2_dinode *di, + struct ocfs2_write_ctxt *wc) +{ + if (unlikely(*copied < len)) { + if (!folio_test_uptodate(wc->w_target_folio)) { + *copied = 0; + return; + } + } + + memcpy_from_folio(di->id2.i_data.id_data + pos, wc->w_target_folio, + pos, *copied); + + trace_ocfs2_write_end_inline( + (unsigned long long)OCFS2_I(inode)->ip_blkno, + (unsigned long long)pos, *copied, + le16_to_cpu(di->id2.i_data.id_count), + le16_to_cpu(di->i_dyn_features)); +} + +int ocfs2_write_end_nolock(struct address_space *mapping, loff_t pos, + unsigned len, unsigned copied, void *fsdata) +{ + int i, ret; + size_t from, to, start = pos & (PAGE_SIZE - 1); + struct inode *inode = mapping->host; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct ocfs2_write_ctxt *wc = fsdata; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)wc->w_di_bh->b_data; + handle_t *handle = wc->w_handle; + + BUG_ON(!list_empty(&wc->w_unwritten_list)); + + if (handle) { + ret = ocfs2_journal_access_di(handle, INODE_CACHE(inode), + wc->w_di_bh, OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + copied = ret; + mlog_errno(ret); + goto out; + } + } + + if (OCFS2_I(inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL) { + ocfs2_write_end_inline(inode, pos, len, &copied, di, wc); + goto out_write_size; + } + + if (unlikely(copied < len) && wc->w_target_folio) { + loff_t new_isize; + + if (!folio_test_uptodate(wc->w_target_folio)) + copied = 0; + + new_isize = max_t(loff_t, i_size_read(inode), pos + copied); + if (new_isize > folio_pos(wc->w_target_folio)) + ocfs2_zero_new_buffers(wc->w_target_folio, start+copied, + start+len); + else { + /* + * When folio is fully beyond new isize (data copy + * failed), do not bother zeroing the folio. Invalidate + * it instead so that writeback does not get confused + * put page & buffer dirty bits into inconsistent + * state. + */ + block_invalidate_folio(wc->w_target_folio, 0, + folio_size(wc->w_target_folio)); + } + } + if (wc->w_target_folio) + flush_dcache_folio(wc->w_target_folio); + + for (i = 0; i < wc->w_num_folios; i++) { + struct folio *folio = wc->w_folios[i]; + + /* This is the direct io target folio */ + if (folio == NULL) + continue; + + if (folio == wc->w_target_folio) { + from = wc->w_target_from; + to = wc->w_target_to; + + BUG_ON(from > folio_size(folio) || + to > folio_size(folio) || + to < from); + } else { + /* + * Pages adjacent to the target (if any) imply + * a hole-filling write in which case we want + * to flush their entire range. + */ + from = 0; + to = folio_size(folio); + } + + if (folio_buffers(folio)) { + if (handle && ocfs2_should_order_data(inode)) { + loff_t start_byte = folio_pos(folio) + from; + loff_t length = to - from; + ocfs2_jbd2_inode_add_write(handle, inode, + start_byte, length); + } + block_commit_write(folio, from, to); + } + } + +out_write_size: + /* Direct io do not update i_size here. */ + if (wc->w_type != OCFS2_WRITE_DIRECT) { + pos += copied; + if (pos > i_size_read(inode)) { + i_size_write(inode, pos); + mark_inode_dirty(inode); + } + inode->i_blocks = ocfs2_inode_sector_count(inode); + di->i_size = cpu_to_le64((u64)i_size_read(inode)); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + di->i_mtime = di->i_ctime = cpu_to_le64(inode_get_mtime_sec(inode)); + di->i_mtime_nsec = di->i_ctime_nsec = cpu_to_le32(inode_get_mtime_nsec(inode)); + if (handle) + ocfs2_update_inode_fsync_trans(handle, inode, 1); + } + if (handle) + ocfs2_journal_dirty(handle, wc->w_di_bh); + +out: + /* unlock pages before dealloc since it needs acquiring j_trans_barrier + * lock, or it will cause a deadlock since journal commit threads holds + * this lock and will ask for the page lock when flushing the data. + * put it here to preserve the unlock order. + */ + ocfs2_unlock_folios(wc); + + if (handle) + ocfs2_commit_trans(osb, handle); + + ocfs2_run_deallocs(osb, &wc->w_dealloc); + + brelse(wc->w_di_bh); + kfree(wc); + + return copied; +} + +static int ocfs2_write_end(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct folio *folio, void *fsdata) +{ + int ret; + struct inode *inode = mapping->host; + + ret = ocfs2_write_end_nolock(mapping, pos, len, copied, fsdata); + + up_write(&OCFS2_I(inode)->ip_alloc_sem); + ocfs2_inode_unlock(inode, 1); + + return ret; +} + +struct ocfs2_dio_write_ctxt { + struct list_head dw_zero_list; + unsigned dw_zero_count; + int dw_orphaned; + pid_t dw_writer_pid; +}; + +static struct ocfs2_dio_write_ctxt * +ocfs2_dio_alloc_write_ctx(struct buffer_head *bh, int *alloc) +{ + struct ocfs2_dio_write_ctxt *dwc = NULL; + + if (bh->b_private) + return bh->b_private; + + dwc = kmalloc_obj(struct ocfs2_dio_write_ctxt, GFP_NOFS); + if (dwc == NULL) + return NULL; + INIT_LIST_HEAD(&dwc->dw_zero_list); + dwc->dw_zero_count = 0; + dwc->dw_orphaned = 0; + dwc->dw_writer_pid = task_pid_nr(current); + bh->b_private = dwc; + *alloc = 1; + + return dwc; +} + +static void ocfs2_dio_free_write_ctx(struct inode *inode, + struct ocfs2_dio_write_ctxt *dwc) +{ + ocfs2_free_unwritten_list(inode, &dwc->dw_zero_list); + kfree(dwc); +} + +/* + * TODO: Make this into a generic get_blocks function. + * + * From do_direct_io in direct-io.c: + * "So what we do is to permit the ->get_blocks function to populate + * bh.b_size with the size of IO which is permitted at this offset and + * this i_blkbits." + * + * This function is called directly from get_more_blocks in direct-io.c. + * + * called like this: dio->get_blocks(dio->inode, fs_startblk, + * fs_count, map_bh, dio->rw == WRITE); + */ +static int ocfs2_dio_wr_get_block(struct inode *inode, sector_t iblock, + struct buffer_head *bh_result, int create) +{ + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_write_ctxt *wc; + struct ocfs2_write_cluster_desc *desc = NULL; + struct ocfs2_dio_write_ctxt *dwc = NULL; + struct buffer_head *di_bh = NULL; + u64 p_blkno; + unsigned int i_blkbits = inode->i_sb->s_blocksize_bits; + loff_t pos = iblock << i_blkbits; + sector_t endblk = (i_size_read(inode) - 1) >> i_blkbits; + unsigned len, total_len = bh_result->b_size; + int ret = 0, first_get_block = 0; + + len = osb->s_clustersize - (pos & (osb->s_clustersize - 1)); + len = min(total_len, len); + + /* + * bh_result->b_size is count in get_more_blocks according to write + * "pos" and "end", we need map twice to return different buffer state: + * 1. area in file size, not set NEW; + * 2. area out file size, set NEW. + * + * iblock endblk + * |--------|---------|---------|--------- + * |<-------area in file------->| + */ + + if ((iblock <= endblk) && + ((iblock + ((len - 1) >> i_blkbits)) > endblk)) + len = (endblk - iblock + 1) << i_blkbits; + + mlog(0, "get block of %llu at %llu:%u req %u\n", + inode->i_ino, pos, len, total_len); + + /* + * Because we need to change file size in ocfs2_dio_end_io_write(), or + * we may need to add it to orphan dir. So can not fall to fast path + * while file size will be changed. + */ + if (pos + total_len <= i_size_read(inode)) { + + /* This is the fast path for re-write. */ + ret = ocfs2_lock_get_block(inode, iblock, bh_result, create); + if (buffer_mapped(bh_result) && + !buffer_new(bh_result) && + ret == 0) + goto out; + + /* Clear state set by ocfs2_get_block. */ + bh_result->b_state = 0; + } + + dwc = ocfs2_dio_alloc_write_ctx(bh_result, &first_get_block); + if (unlikely(dwc == NULL)) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + if (ocfs2_clusters_for_bytes(inode->i_sb, pos + total_len) > + ocfs2_clusters_for_bytes(inode->i_sb, i_size_read(inode)) && + !dwc->dw_orphaned) { + /* + * when we are going to alloc extents beyond file size, add the + * inode to orphan dir, so we can recall those spaces when + * system crashed during write. + */ + ret = ocfs2_add_inode_to_orphan(osb, inode); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + dwc->dw_orphaned = 1; + } + + ret = ocfs2_inode_lock(inode, &di_bh, 1); + if (ret) { + mlog_errno(ret); + goto out; + } + + down_write(&oi->ip_alloc_sem); + + if (first_get_block) { + if (ocfs2_sparse_alloc(osb)) + ret = ocfs2_zero_tail(inode, di_bh, pos); + else + ret = ocfs2_expand_nonsparse_inode(inode, di_bh, pos, + total_len, NULL); + if (ret < 0) { + mlog_errno(ret); + goto unlock; + } + } + + ret = ocfs2_write_begin_nolock(inode->i_mapping, pos, len, + OCFS2_WRITE_DIRECT, NULL, + (void **)&wc, di_bh, NULL); + if (ret) { + mlog_errno(ret); + goto unlock; + } + + desc = &wc->w_desc[0]; + + p_blkno = ocfs2_clusters_to_blocks(inode->i_sb, desc->c_phys); + BUG_ON(p_blkno == 0); + p_blkno += iblock & (u64)(ocfs2_clusters_to_blocks(inode->i_sb, 1) - 1); + + map_bh(bh_result, inode->i_sb, p_blkno); + bh_result->b_size = len; + if (desc->c_needs_zero) + set_buffer_new(bh_result); + + if (iblock > endblk) + set_buffer_new(bh_result); + + /* May sleep in end_io. It should not happen in a irq context. So defer + * it to dio work queue. */ + set_buffer_defer_completion(bh_result); + + if (!list_empty(&wc->w_unwritten_list)) { + struct ocfs2_unwritten_extent *ue = NULL; + + ue = list_first_entry(&wc->w_unwritten_list, + struct ocfs2_unwritten_extent, + ue_node); + BUG_ON(ue->ue_cpos != desc->c_cpos); + /* The physical address may be 0, fill it. */ + ue->ue_phys = desc->c_phys; + + list_splice_tail_init(&wc->w_unwritten_list, &dwc->dw_zero_list); + dwc->dw_zero_count += wc->w_unwritten_count; + } + + ret = ocfs2_write_end_nolock(inode->i_mapping, pos, len, len, wc); + BUG_ON(ret != len); + ret = 0; +unlock: + up_write(&oi->ip_alloc_sem); + ocfs2_inode_unlock(inode, 1); + brelse(di_bh); +out: + return ret; +} + +static int ocfs2_dio_end_io_write(struct inode *inode, + struct ocfs2_dio_write_ctxt *dwc, + loff_t offset, + ssize_t bytes) +{ + struct ocfs2_cached_dealloc_ctxt dealloc; + struct ocfs2_extent_tree et; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_unwritten_extent *ue = NULL; + struct buffer_head *di_bh = NULL; + struct ocfs2_dinode *di; + struct ocfs2_alloc_context *data_ac = NULL; + struct ocfs2_alloc_context *meta_ac = NULL; + handle_t *handle = NULL; + loff_t end = offset + bytes; + int ret = 0, credits = 0, batch = 0; + + ocfs2_init_dealloc_ctxt(&dealloc); + + /* We do clear unwritten, delete orphan, change i_size here. If neither + * of these happen, we can skip all this. */ + if (list_empty(&dwc->dw_zero_list) && + end <= i_size_read(inode) && + !dwc->dw_orphaned) + goto out; + + ret = ocfs2_inode_lock(inode, &di_bh, 1); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + down_write(&oi->ip_alloc_sem); + di = (struct ocfs2_dinode *)di_bh->b_data; + + ocfs2_init_dinode_extent_tree(&et, INODE_CACHE(inode), di_bh); + + /* Attach dealloc with extent tree in case that we may reuse extents + * which are already unlinked from current extent tree due to extent + * rotation and merging. + */ + et.et_dealloc = &dealloc; + + ret = ocfs2_lock_allocators(inode, &et, 0, dwc->dw_zero_count*2, + &data_ac, &meta_ac); + if (ret) { + mlog_errno(ret); + goto unlock; + } + + credits = ocfs2_calc_extend_credits(inode->i_sb, &di->id2.i_list); + + list_for_each_entry(ue, &dwc->dw_zero_list, ue_node) { + if (!handle) { + handle = ocfs2_start_trans(osb, credits); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto unlock; + } + ret = ocfs2_journal_access_di(handle, INODE_CACHE(inode), di_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto commit; + } + } + ret = ocfs2_assure_trans_credits(handle, credits); + if (ret < 0) { + mlog_errno(ret); + goto commit; + } + ret = ocfs2_mark_extent_written(inode, &et, handle, + ue->ue_cpos, 1, + ue->ue_phys, + meta_ac, &dealloc); + if (ret < 0) { + mlog_errno(ret); + goto commit; + } + + if (++batch == OCFS2_DIO_MARK_EXTENT_BATCH) { + ocfs2_commit_trans(osb, handle); + handle = NULL; + batch = 0; + } + } + + if (end > i_size_read(inode)) { + if (!handle) { + handle = ocfs2_start_trans(osb, credits); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto unlock; + } + } + ret = ocfs2_set_inode_size(handle, inode, di_bh, end); + if (ret < 0) + mlog_errno(ret); + } + +commit: + if (handle) + ocfs2_commit_trans(osb, handle); +unlock: + up_write(&oi->ip_alloc_sem); + + if (data_ac) { + ocfs2_free_alloc_context(data_ac); + data_ac = NULL; + } + if (meta_ac) { + ocfs2_free_alloc_context(meta_ac); + meta_ac = NULL; + } + + /* everything looks good, let's start the cleanup */ + if (!ret && dwc->dw_orphaned) { + BUG_ON(dwc->dw_writer_pid != task_pid_nr(current)); + + ret = ocfs2_del_inode_from_orphan(osb, inode, di_bh, 0, 0); + if (ret < 0) + mlog_errno(ret); + } + ocfs2_inode_unlock(inode, 1); + brelse(di_bh); +out: + ocfs2_run_deallocs(osb, &dealloc); + ocfs2_dio_free_write_ctx(inode, dwc); + + return ret; +} + +/* + * ocfs2_dio_end_io is called by the dio core when a dio is finished. We're + * particularly interested in the aio/dio case. We use the rw_lock DLM lock + * to protect io on one node from truncation on another. + */ +static int ocfs2_dio_end_io(struct kiocb *iocb, + loff_t offset, + ssize_t bytes, + void *private) +{ + struct inode *inode = file_inode(iocb->ki_filp); + int level; + int ret = 0; + + /* this io's submitter should not have unlocked this before we could */ + BUG_ON(!ocfs2_iocb_is_rw_locked(iocb)); + + if (bytes <= 0) + mlog_ratelimited(ML_ERROR, "Direct IO failed, bytes = %lld", + (long long)bytes); + if (private) { + if (bytes > 0) + ret = ocfs2_dio_end_io_write(inode, private, offset, + bytes); + else + ocfs2_dio_free_write_ctx(inode, private); + } + + ocfs2_iocb_clear_rw_locked(iocb); + + level = ocfs2_iocb_rw_locked_level(iocb); + ocfs2_rw_unlock(inode, level); + return ret; +} + +static ssize_t ocfs2_direct_IO(struct kiocb *iocb, struct iov_iter *iter) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file->f_mapping->host; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + get_block_t *get_block; + + /* + * Fallback to buffered I/O if we see an inode without + * extents. + */ + if (OCFS2_I(inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL) + return 0; + + /* Fallback to buffered I/O if we do not support append dio. */ + if (iocb->ki_pos + iter->count > i_size_read(inode) && + !ocfs2_supports_append_dio(osb)) + return 0; + + if (iov_iter_rw(iter) == READ) + get_block = ocfs2_lock_get_block; + else + get_block = ocfs2_dio_wr_get_block; + + return __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev, + iter, get_block, + ocfs2_dio_end_io, 0); +} + +const struct address_space_operations ocfs2_aops = { + .dirty_folio = block_dirty_folio, + .read_folio = ocfs2_read_folio, + .readahead = ocfs2_readahead, + .writepages = ocfs2_writepages, + .write_begin = ocfs2_write_begin, + .write_end = ocfs2_write_end, + .bmap = ocfs2_bmap, + .direct_IO = ocfs2_direct_IO, + .invalidate_folio = block_invalidate_folio, + .release_folio = ocfs2_release_folio, + .migrate_folio = buffer_migrate_folio, + .is_partially_uptodate = block_is_partially_uptodate, + .error_remove_folio = generic_error_remove_folio, +}; diff --git a/fs/ocfs2/aops.h b/fs/ocfs2/aops.h new file mode 100644 index 000000000..114efc911 --- /dev/null +++ b/fs/ocfs2/aops.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2002, 2004, 2005 Oracle. All rights reserved. + */ + +#ifndef OCFS2_AOPS_H +#define OCFS2_AOPS_H + +#include + +int ocfs2_map_folio_blocks(struct folio *folio, u64 *p_blkno, + struct inode *inode, unsigned int from, + unsigned int to, int new); + +void ocfs2_unlock_and_free_folios(struct folio **folios, int num_folios); + +int walk_page_buffers( handle_t *handle, + struct buffer_head *head, + unsigned from, + unsigned to, + int *partial, + int (*fn)( handle_t *handle, + struct buffer_head *bh)); + +int ocfs2_write_end_nolock(struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, void *fsdata); + +typedef enum { + OCFS2_WRITE_BUFFER = 0, + OCFS2_WRITE_DIRECT, + OCFS2_WRITE_MMAP, +} ocfs2_write_type_t; + +int ocfs2_write_begin_nolock(struct address_space *mapping, + loff_t pos, unsigned len, ocfs2_write_type_t type, + struct folio **foliop, void **fsdata, + struct buffer_head *di_bh, struct folio *mmap_folio); + +int ocfs2_read_inline_data(struct inode *inode, struct folio *folio, + struct buffer_head *di_bh); +int ocfs2_size_fits_inline_data(struct buffer_head *di_bh, u64 new_size); + +int ocfs2_get_block(struct inode *inode, sector_t iblock, + struct buffer_head *bh_result, int create); +/* all ocfs2_dio_end_io()'s fault */ +#define ocfs2_iocb_is_rw_locked(iocb) \ + test_bit(0, (unsigned long *)&iocb->private) +static inline void ocfs2_iocb_set_rw_locked(struct kiocb *iocb, int level) +{ + set_bit(0, (unsigned long *)&iocb->private); + if (level) + set_bit(1, (unsigned long *)&iocb->private); + else + clear_bit(1, (unsigned long *)&iocb->private); +} + +/* + * Using a named enum representing lock types in terms of #N bit stored in + * iocb->private, which is going to be used for communication between + * ocfs2_dio_end_io() and ocfs2_file_write/read_iter(). + */ +enum ocfs2_iocb_lock_bits { + OCFS2_IOCB_RW_LOCK = 0, + OCFS2_IOCB_RW_LOCK_LEVEL, + OCFS2_IOCB_NUM_LOCKS +}; + +#define ocfs2_iocb_init_rw_locked(iocb) \ + (iocb->private = NULL) +#define ocfs2_iocb_clear_rw_locked(iocb) \ + clear_bit(OCFS2_IOCB_RW_LOCK, (unsigned long *)&iocb->private) +#define ocfs2_iocb_rw_locked_level(iocb) \ + test_bit(OCFS2_IOCB_RW_LOCK_LEVEL, (unsigned long *)&iocb->private) + +#endif /* OCFS2_FILE_H */ diff --git a/fs/ocfs2/blockcheck.c b/fs/ocfs2/blockcheck.c new file mode 100644 index 000000000..863a53160 --- /dev/null +++ b/fs/ocfs2/blockcheck.c @@ -0,0 +1,605 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * blockcheck.c + * + * Checksum and ECC codes for the OCFS2 userspace library. + * + * Copyright (C) 2006, 2008 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ocfs2.h" + +#include "blockcheck.h" + + +/* + * We use the following conventions: + * + * d = # data bits + * p = # parity bits + * c = # total code bits (d + p) + */ + + +/* + * Calculate the bit offset in the hamming code buffer based on the bit's + * offset in the data buffer. Since the hamming code reserves all + * power-of-two bits for parity, the data bit number and the code bit + * number are offset by all the parity bits beforehand. + * + * Recall that bit numbers in hamming code are 1-based. This function + * takes the 0-based data bit from the caller. + * + * An example. Take bit 1 of the data buffer. 1 is a power of two (2^0), + * so it's a parity bit. 2 is a power of two (2^1), so it's a parity bit. + * 3 is not a power of two. So bit 1 of the data buffer ends up as bit 3 + * in the code buffer. + * + * The caller can pass in *p if it wants to keep track of the most recent + * number of parity bits added. This allows the function to start the + * calculation at the last place. + */ +static unsigned int calc_code_bit(unsigned int i, unsigned int *p_cache) +{ + unsigned int b, p = 0; + + /* + * Data bits are 0-based, but we're talking code bits, which + * are 1-based. + */ + b = i + 1; + + /* Use the cache if it is there */ + if (p_cache) + p = *p_cache; + b += p; + + /* + * For every power of two below our bit number, bump our bit. + * + * We compare with (b + 1) because we have to compare with what b + * would be _if_ it were bumped up by the parity bit. Capice? + * + * p is set above. + */ + for (; (1 << p) < (b + 1); p++) + b++; + + if (p_cache) + *p_cache = p; + + return b; +} + +/* + * This is the low level encoder function. It can be called across + * multiple hunks just like the crc32 code. 'd' is the number of bits + * _in_this_hunk_. nr is the bit offset of this hunk. So, if you had + * two 512B buffers, you would do it like so: + * + * parity = ocfs2_hamming_encode(0, buf1, 512 * 8, 0); + * parity = ocfs2_hamming_encode(parity, buf2, 512 * 8, 512 * 8); + * + * If you just have one buffer, use ocfs2_hamming_encode_block(). + */ +u32 ocfs2_hamming_encode(u32 parity, void *data, unsigned int d, unsigned int nr) +{ + unsigned int i, b, p = 0; + + BUG_ON(!d); + + /* + * b is the hamming code bit number. Hamming code specifies a + * 1-based array, but C uses 0-based. So 'i' is for C, and 'b' is + * for the algorithm. + * + * The i++ in the for loop is so that the start offset passed + * to ocfs2_find_next_bit_set() is one greater than the previously + * found bit. + */ + for (i = 0; (i = ocfs2_find_next_bit(data, d, i)) < d; i++) + { + /* + * i is the offset in this hunk, nr + i is the total bit + * offset. + */ + b = calc_code_bit(nr + i, &p); + + /* + * Data bits in the resultant code are checked by + * parity bits that are part of the bit number + * representation. Huh? + * + * + * In other words, the parity bit at position 2^k + * checks bits in positions having bit k set in + * their binary representation. Conversely, for + * instance, bit 13, i.e. 1101(2), is checked by + * bits 1000(2) = 8, 0100(2)=4 and 0001(2) = 1. + * + * + * Note that 'k' is the _code_ bit number. 'b' in + * our loop. + */ + parity ^= b; + } + + /* While the data buffer was treated as little endian, the + * return value is in host endian. */ + return parity; +} + +u32 ocfs2_hamming_encode_block(void *data, unsigned int blocksize) +{ + return ocfs2_hamming_encode(0, data, blocksize * 8, 0); +} + +/* + * Like ocfs2_hamming_encode(), this can handle hunks. nr is the bit + * offset of the current hunk. If bit to be fixed is not part of the + * current hunk, this does nothing. + * + * If you only have one hunk, use ocfs2_hamming_fix_block(). + */ +void ocfs2_hamming_fix(void *data, unsigned int d, unsigned int nr, + unsigned int fix) +{ + unsigned int i, b; + + BUG_ON(!d); + + /* + * If the bit to fix has an hweight of 1, it's a parity bit. One + * busted parity bit is its own error. Nothing to do here. + */ + if (hweight32(fix) == 1) + return; + + /* + * nr + d is the bit right past the data hunk we're looking at. + * If fix after that, nothing to do + */ + if (fix >= calc_code_bit(nr + d, NULL)) + return; + + /* + * nr is the offset in the data hunk we're starting at. Let's + * start b at the offset in the code buffer. See hamming_encode() + * for a more detailed description of 'b'. + */ + b = calc_code_bit(nr, NULL); + /* If the fix is before this hunk, nothing to do */ + if (fix < b) + return; + + for (i = 0; i < d; i++, b++) + { + /* Skip past parity bits */ + while (hweight32(b) == 1) + b++; + + /* + * i is the offset in this data hunk. + * nr + i is the offset in the total data buffer. + * b is the offset in the total code buffer. + * + * Thus, when b == fix, bit i in the current hunk needs + * fixing. + */ + if (b == fix) + { + if (ocfs2_test_bit(i, data)) + ocfs2_clear_bit(i, data); + else + ocfs2_set_bit(i, data); + break; + } + } +} + +void ocfs2_hamming_fix_block(void *data, unsigned int blocksize, + unsigned int fix) +{ + ocfs2_hamming_fix(data, blocksize * 8, 0, fix); +} + + +/* + * Debugfs handling. + */ + +#ifdef CONFIG_DEBUG_FS + +static int blockcheck_u64_get(void *data, u64 *val) +{ + *val = *(u64 *)data; + return 0; +} +DEFINE_DEBUGFS_ATTRIBUTE(blockcheck_fops, blockcheck_u64_get, NULL, "%llu\n"); + +static void ocfs2_blockcheck_debug_remove(struct ocfs2_blockcheck_stats *stats) +{ + if (stats) { + debugfs_remove_recursive(stats->b_debug_dir); + stats->b_debug_dir = NULL; + } +} + +static void ocfs2_blockcheck_debug_install(struct ocfs2_blockcheck_stats *stats, + struct dentry *parent) +{ + struct dentry *dir; + + dir = debugfs_create_dir("blockcheck", parent); + stats->b_debug_dir = dir; + + debugfs_create_file("blocks_checked", S_IFREG | S_IRUSR, dir, + &stats->b_check_count, &blockcheck_fops); + + debugfs_create_file("checksums_failed", S_IFREG | S_IRUSR, dir, + &stats->b_failure_count, &blockcheck_fops); + + debugfs_create_file("ecc_recoveries", S_IFREG | S_IRUSR, dir, + &stats->b_recover_count, &blockcheck_fops); + +} +#else +static inline void ocfs2_blockcheck_debug_install(struct ocfs2_blockcheck_stats *stats, + struct dentry *parent) +{ +} + +static inline void ocfs2_blockcheck_debug_remove(struct ocfs2_blockcheck_stats *stats) +{ +} +#endif /* CONFIG_DEBUG_FS */ + +/* Always-called wrappers for starting and stopping the debugfs files */ +void ocfs2_blockcheck_stats_debugfs_install(struct ocfs2_blockcheck_stats *stats, + struct dentry *parent) +{ + ocfs2_blockcheck_debug_install(stats, parent); +} + +void ocfs2_blockcheck_stats_debugfs_remove(struct ocfs2_blockcheck_stats *stats) +{ + ocfs2_blockcheck_debug_remove(stats); +} + +static void ocfs2_blockcheck_inc_check(struct ocfs2_blockcheck_stats *stats) +{ + u64 new_count; + + if (!stats) + return; + + spin_lock(&stats->b_lock); + stats->b_check_count++; + new_count = stats->b_check_count; + spin_unlock(&stats->b_lock); + + if (!new_count) + mlog(ML_NOTICE, "Block check count has wrapped\n"); +} + +static void ocfs2_blockcheck_inc_failure(struct ocfs2_blockcheck_stats *stats) +{ + u64 new_count; + + if (!stats) + return; + + spin_lock(&stats->b_lock); + stats->b_failure_count++; + new_count = stats->b_failure_count; + spin_unlock(&stats->b_lock); + + if (!new_count) + mlog(ML_NOTICE, "Checksum failure count has wrapped\n"); +} + +static void ocfs2_blockcheck_inc_recover(struct ocfs2_blockcheck_stats *stats) +{ + u64 new_count; + + if (!stats) + return; + + spin_lock(&stats->b_lock); + stats->b_recover_count++; + new_count = stats->b_recover_count; + spin_unlock(&stats->b_lock); + + if (!new_count) + mlog(ML_NOTICE, "ECC recovery count has wrapped\n"); +} + + + +/* + * These are the low-level APIs for using the ocfs2_block_check structure. + */ + +/* + * This function generates check information for a block. + * data is the block to be checked. bc is a pointer to the + * ocfs2_block_check structure describing the crc32 and the ecc. + * + * bc should be a pointer inside data, as the function will + * take care of zeroing it before calculating the check information. If + * bc does not point inside data, the caller must make sure any inline + * ocfs2_block_check structures are zeroed. + * + * The data buffer must be in on-disk endian (little endian for ocfs2). + * bc will be filled with little-endian values and will be ready to go to + * disk. + */ +void ocfs2_block_check_compute(void *data, size_t blocksize, + struct ocfs2_block_check *bc) +{ + u32 crc; + u32 ecc; + + memset(bc, 0, sizeof(struct ocfs2_block_check)); + + crc = crc32_le(~0, data, blocksize); + ecc = ocfs2_hamming_encode_block(data, blocksize); + + /* + * No ecc'd ocfs2 structure is larger than 4K, so ecc will be no + * larger than 16 bits. + */ + BUG_ON(ecc > USHRT_MAX); + + bc->bc_crc32e = cpu_to_le32(crc); + bc->bc_ecc = cpu_to_le16((u16)ecc); +} + +/* + * This function validates existing check information. Like _compute, + * the function will take care of zeroing bc before calculating check codes. + * If bc is not a pointer inside data, the caller must have zeroed any + * inline ocfs2_block_check structures. + * + * Again, the data passed in should be the on-disk endian. + */ +int ocfs2_block_check_validate(void *data, size_t blocksize, + struct ocfs2_block_check *bc, + struct ocfs2_blockcheck_stats *stats) +{ + int rc = 0; + u32 bc_crc32e; + u16 bc_ecc; + u32 crc, ecc; + + ocfs2_blockcheck_inc_check(stats); + + bc_crc32e = le32_to_cpu(bc->bc_crc32e); + bc_ecc = le16_to_cpu(bc->bc_ecc); + + memset(bc, 0, sizeof(struct ocfs2_block_check)); + + /* Fast path - if the crc32 validates, we're good to go */ + crc = crc32_le(~0, data, blocksize); + if (crc == bc_crc32e) + goto out; + + ocfs2_blockcheck_inc_failure(stats); + mlog(ML_ERROR, + "CRC32 failed: stored: 0x%x, computed 0x%x. Applying ECC.\n", + (unsigned int)bc_crc32e, (unsigned int)crc); + + /* Ok, try ECC fixups */ + ecc = ocfs2_hamming_encode_block(data, blocksize); + ocfs2_hamming_fix_block(data, blocksize, ecc ^ bc_ecc); + + /* And check the crc32 again */ + crc = crc32_le(~0, data, blocksize); + if (crc == bc_crc32e) { + ocfs2_blockcheck_inc_recover(stats); + goto out; + } + + mlog(ML_ERROR, "Fixed CRC32 failed: stored: 0x%x, computed 0x%x\n", + (unsigned int)bc_crc32e, (unsigned int)crc); + + rc = -EIO; + +out: + bc->bc_crc32e = cpu_to_le32(bc_crc32e); + bc->bc_ecc = cpu_to_le16(bc_ecc); + + return rc; +} + +/* + * This function generates check information for a list of buffer_heads. + * bhs is the blocks to be checked. bc is a pointer to the + * ocfs2_block_check structure describing the crc32 and the ecc. + * + * bc should be a pointer inside data, as the function will + * take care of zeroing it before calculating the check information. If + * bc does not point inside data, the caller must make sure any inline + * ocfs2_block_check structures are zeroed. + * + * The data buffer must be in on-disk endian (little endian for ocfs2). + * bc will be filled with little-endian values and will be ready to go to + * disk. + */ +void ocfs2_block_check_compute_bhs(struct buffer_head **bhs, int nr, + struct ocfs2_block_check *bc) +{ + int i; + u32 crc, ecc; + + BUG_ON(nr < 0); + + if (!nr) + return; + + memset(bc, 0, sizeof(struct ocfs2_block_check)); + + for (i = 0, crc = ~0, ecc = 0; i < nr; i++) { + crc = crc32_le(crc, bhs[i]->b_data, bhs[i]->b_size); + /* + * The number of bits in a buffer is obviously b_size*8. + * The offset of this buffer is b_size*i, so the bit offset + * of this buffer is b_size*8*i. + */ + ecc = (u16)ocfs2_hamming_encode(ecc, bhs[i]->b_data, + bhs[i]->b_size * 8, + bhs[i]->b_size * 8 * i); + } + + /* + * No ecc'd ocfs2 structure is larger than 4K, so ecc will be no + * larger than 16 bits. + */ + BUG_ON(ecc > USHRT_MAX); + + bc->bc_crc32e = cpu_to_le32(crc); + bc->bc_ecc = cpu_to_le16((u16)ecc); +} + +/* + * This function validates existing check information on a list of + * buffer_heads. Like _compute_bhs, the function will take care of + * zeroing bc before calculating check codes. If bc is not a pointer + * inside data, the caller must have zeroed any inline + * ocfs2_block_check structures. + * + * Again, the data passed in should be the on-disk endian. + */ +int ocfs2_block_check_validate_bhs(struct buffer_head **bhs, int nr, + struct ocfs2_block_check *bc, + struct ocfs2_blockcheck_stats *stats) +{ + int i, rc = 0; + u32 bc_crc32e; + u16 bc_ecc; + u32 crc, ecc, fix; + + BUG_ON(nr < 0); + + if (!nr) + return 0; + + ocfs2_blockcheck_inc_check(stats); + + bc_crc32e = le32_to_cpu(bc->bc_crc32e); + bc_ecc = le16_to_cpu(bc->bc_ecc); + + memset(bc, 0, sizeof(struct ocfs2_block_check)); + + /* Fast path - if the crc32 validates, we're good to go */ + for (i = 0, crc = ~0; i < nr; i++) + crc = crc32_le(crc, bhs[i]->b_data, bhs[i]->b_size); + if (crc == bc_crc32e) + goto out; + + ocfs2_blockcheck_inc_failure(stats); + mlog(ML_ERROR, + "CRC32 failed: stored: %u, computed %u. Applying ECC.\n", + (unsigned int)bc_crc32e, (unsigned int)crc); + + /* Ok, try ECC fixups */ + for (i = 0, ecc = 0; i < nr; i++) { + /* + * The number of bits in a buffer is obviously b_size*8. + * The offset of this buffer is b_size*i, so the bit offset + * of this buffer is b_size*8*i. + */ + ecc = (u16)ocfs2_hamming_encode(ecc, bhs[i]->b_data, + bhs[i]->b_size * 8, + bhs[i]->b_size * 8 * i); + } + fix = ecc ^ bc_ecc; + for (i = 0; i < nr; i++) { + /* + * Try the fix against each buffer. It will only affect + * one of them. + */ + ocfs2_hamming_fix(bhs[i]->b_data, bhs[i]->b_size * 8, + bhs[i]->b_size * 8 * i, fix); + } + + /* And check the crc32 again */ + for (i = 0, crc = ~0; i < nr; i++) + crc = crc32_le(crc, bhs[i]->b_data, bhs[i]->b_size); + if (crc == bc_crc32e) { + ocfs2_blockcheck_inc_recover(stats); + goto out; + } + + mlog(ML_ERROR, "Fixed CRC32 failed: stored: %u, computed %u\n", + (unsigned int)bc_crc32e, (unsigned int)crc); + + rc = -EIO; + +out: + bc->bc_crc32e = cpu_to_le32(bc_crc32e); + bc->bc_ecc = cpu_to_le16(bc_ecc); + + return rc; +} + +/* + * These are the main API. They check the superblock flag before + * calling the underlying operations. + * + * They expect the buffer(s) to be in disk format. + */ +void ocfs2_compute_meta_ecc(struct super_block *sb, void *data, + struct ocfs2_block_check *bc) +{ + if (ocfs2_meta_ecc(OCFS2_SB(sb))) + ocfs2_block_check_compute(data, sb->s_blocksize, bc); +} + +int ocfs2_validate_meta_ecc(struct super_block *sb, void *data, + struct ocfs2_block_check *bc) +{ + int rc = 0; + struct ocfs2_super *osb = OCFS2_SB(sb); + + if (ocfs2_meta_ecc(osb)) + rc = ocfs2_block_check_validate(data, sb->s_blocksize, bc, + &osb->osb_ecc_stats); + + return rc; +} + +void ocfs2_compute_meta_ecc_bhs(struct super_block *sb, + struct buffer_head **bhs, int nr, + struct ocfs2_block_check *bc) +{ + if (ocfs2_meta_ecc(OCFS2_SB(sb))) + ocfs2_block_check_compute_bhs(bhs, nr, bc); +} + +int ocfs2_validate_meta_ecc_bhs(struct super_block *sb, + struct buffer_head **bhs, int nr, + struct ocfs2_block_check *bc) +{ + int rc = 0; + struct ocfs2_super *osb = OCFS2_SB(sb); + + if (ocfs2_meta_ecc(osb)) + rc = ocfs2_block_check_validate_bhs(bhs, nr, bc, + &osb->osb_ecc_stats); + + return rc; +} + diff --git a/fs/ocfs2/blockcheck.h b/fs/ocfs2/blockcheck.h new file mode 100644 index 000000000..d0578e98e --- /dev/null +++ b/fs/ocfs2/blockcheck.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * blockcheck.h + * + * Checksum and ECC codes for the OCFS2 userspace library. + * + * Copyright (C) 2004, 2008 Oracle. All rights reserved. + */ + +#ifndef OCFS2_BLOCKCHECK_H +#define OCFS2_BLOCKCHECK_H + + +/* Count errors and error correction from blockcheck.c */ +struct ocfs2_blockcheck_stats { + spinlock_t b_lock; + u64 b_check_count; /* Number of blocks we've checked */ + u64 b_failure_count; /* Number of failed checksums */ + u64 b_recover_count; /* Number of blocks fixed by ecc */ + + /* + * debugfs entries, used if this is passed to + * ocfs2_blockcheck_stats_debugfs_install() + */ + struct dentry *b_debug_dir; /* Parent of the debugfs files */ +}; + + +/* High level block API */ +void ocfs2_compute_meta_ecc(struct super_block *sb, void *data, + struct ocfs2_block_check *bc); +int ocfs2_validate_meta_ecc(struct super_block *sb, void *data, + struct ocfs2_block_check *bc); +void ocfs2_compute_meta_ecc_bhs(struct super_block *sb, + struct buffer_head **bhs, int nr, + struct ocfs2_block_check *bc); +int ocfs2_validate_meta_ecc_bhs(struct super_block *sb, + struct buffer_head **bhs, int nr, + struct ocfs2_block_check *bc); + +/* Lower level API */ +void ocfs2_block_check_compute(void *data, size_t blocksize, + struct ocfs2_block_check *bc); +int ocfs2_block_check_validate(void *data, size_t blocksize, + struct ocfs2_block_check *bc, + struct ocfs2_blockcheck_stats *stats); +void ocfs2_block_check_compute_bhs(struct buffer_head **bhs, int nr, + struct ocfs2_block_check *bc); +int ocfs2_block_check_validate_bhs(struct buffer_head **bhs, int nr, + struct ocfs2_block_check *bc, + struct ocfs2_blockcheck_stats *stats); + +/* Debug Initialization */ +void ocfs2_blockcheck_stats_debugfs_install(struct ocfs2_blockcheck_stats *stats, + struct dentry *parent); +void ocfs2_blockcheck_stats_debugfs_remove(struct ocfs2_blockcheck_stats *stats); + +/* + * Hamming code functions + */ + +/* + * Encoding hamming code parity bits for a buffer. + * + * This is the low level encoder function. It can be called across + * multiple hunks just like the crc32 code. 'd' is the number of bits + * _in_this_hunk_. nr is the bit offset of this hunk. So, if you had + * two 512B buffers, you would do it like so: + * + * parity = ocfs2_hamming_encode(0, buf1, 512 * 8, 0); + * parity = ocfs2_hamming_encode(parity, buf2, 512 * 8, 512 * 8); + * + * If you just have one buffer, use ocfs2_hamming_encode_block(). + */ +u32 ocfs2_hamming_encode(u32 parity, void *data, unsigned int d, + unsigned int nr); +/* + * Fix a buffer with a bit error. The 'fix' is the original parity + * xor'd with the parity calculated now. + * + * Like ocfs2_hamming_encode(), this can handle hunks. nr is the bit + * offset of the current hunk. If bit to be fixed is not part of the + * current hunk, this does nothing. + * + * If you only have one buffer, use ocfs2_hamming_fix_block(). + */ +void ocfs2_hamming_fix(void *data, unsigned int d, unsigned int nr, + unsigned int fix); + +/* Convenience wrappers for a single buffer of data */ +extern u32 ocfs2_hamming_encode_block(void *data, unsigned int blocksize); +extern void ocfs2_hamming_fix_block(void *data, unsigned int blocksize, + unsigned int fix); +#endif diff --git a/fs/ocfs2/buffer_head_io.c b/fs/ocfs2/buffer_head_io.c new file mode 100644 index 000000000..7bfe377af --- /dev/null +++ b/fs/ocfs2/buffer_head_io.c @@ -0,0 +1,456 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * io.c + * + * Buffer cache handling + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#include +#include +#include +#include + +#include + +#include "ocfs2.h" + +#include "alloc.h" +#include "inode.h" +#include "journal.h" +#include "uptodate.h" +#include "buffer_head_io.h" +#include "ocfs2_trace.h" + +/* + * Bits on bh->b_state used by ocfs2. + * + * These MUST be after the JBD2 bits. Hence, we use BH_JBDPrivateStart. + */ +enum ocfs2_state_bits { + BH_NeedsValidate = BH_JBDPrivateStart, +}; + +/* Expand the magic b_state functions */ +BUFFER_FNS(NeedsValidate, needs_validate); + +int ocfs2_write_block(struct ocfs2_super *osb, struct buffer_head *bh, + struct ocfs2_caching_info *ci) +{ + int ret = 0; + + trace_ocfs2_write_block((unsigned long long)bh->b_blocknr, ci); + + BUG_ON(bh->b_blocknr < OCFS2_SUPER_BLOCK_BLKNO); + BUG_ON(buffer_jbd(bh)); + + /* No need to check for a soft readonly file system here. non + * journalled writes are only ever done on system files which + * can get modified during recovery even if read-only. */ + if (ocfs2_is_hard_readonly(osb)) { + ret = -EROFS; + mlog_errno(ret); + goto out; + } + + ocfs2_metadata_cache_io_lock(ci); + + lock_buffer(bh); + set_buffer_uptodate(bh); + + /* remove from dirty list before I/O. */ + clear_buffer_dirty(bh); + + bh_submit(bh, REQ_OP_WRITE, bh_end_write); + + wait_on_buffer(bh); + + if (buffer_uptodate(bh)) { + ocfs2_set_buffer_uptodate(ci, bh); + } else { + /* We don't need to remove the clustered uptodate + * information for this bh as it's not marked locally + * uptodate. */ + ret = -EIO; + mlog_errno(ret); + } + + ocfs2_metadata_cache_io_unlock(ci); +out: + return ret; +} + +/* Caller must provide a bhs[] with all NULL or non-NULL entries, so it + * will be easier to handle read failure. + */ +int ocfs2_read_blocks_sync(struct ocfs2_super *osb, u64 block, + unsigned int nr, struct buffer_head *bhs[]) +{ + int status = 0; + unsigned int i; + struct buffer_head *bh; + int new_bh = 0; + + trace_ocfs2_read_blocks_sync((unsigned long long)block, nr); + + if (!nr) + goto bail; + + /* Don't put buffer head and re-assign it to NULL if it is allocated + * outside since the caller can't be aware of this alternation! + */ + new_bh = (bhs[0] == NULL); + + for (i = 0 ; i < nr ; i++) { + if (bhs[i] == NULL) { + bhs[i] = sb_getblk(osb->sb, block++); + if (bhs[i] == NULL) { + status = -ENOMEM; + mlog_errno(status); + break; + } + } + bh = bhs[i]; + + if (buffer_jbd(bh)) { + trace_ocfs2_read_blocks_sync_jbd( + (unsigned long long)bh->b_blocknr); + continue; + } + + if (buffer_dirty(bh)) { + /* This should probably be a BUG, or + * at least return an error. */ + mlog(ML_ERROR, + "trying to sync read a dirty " + "buffer! (blocknr = %llu), skipping\n", + (unsigned long long)bh->b_blocknr); + continue; + } + + lock_buffer(bh); + if (buffer_jbd(bh)) { +#ifdef CATCH_BH_JBD_RACES + mlog(ML_ERROR, + "block %llu had the JBD bit set " + "while I was in lock_buffer!", + (unsigned long long)bh->b_blocknr); + BUG(); +#else + unlock_buffer(bh); + continue; +#endif + } + + bh_submit(bh, REQ_OP_READ, bh_end_read); + } + +read_failure: + for (i = nr; i > 0; i--) { + bh = bhs[i - 1]; + + if (unlikely(status)) { + if (new_bh && bh) { + /* If middle bh fails, let previous bh + * finish its read and then put it to + * avoid bh leak + */ + if (!buffer_jbd(bh)) + wait_on_buffer(bh); + put_bh(bh); + bhs[i - 1] = NULL; + } else if (bh && buffer_uptodate(bh)) { + clear_buffer_uptodate(bh); + } + continue; + } + + /* No need to wait on the buffer if it's managed by JBD. */ + if (!buffer_jbd(bh)) + wait_on_buffer(bh); + + if (!buffer_uptodate(bh)) { + /* Status won't be cleared from here on out, + * so we can safely record this and loop back + * to cleanup the other buffers. */ + status = -EIO; + goto read_failure; + } + } + +bail: + return status; +} + +/* Caller must provide a bhs[] with all NULL or non-NULL entries, so it + * will be easier to handle read failure. + */ +int ocfs2_read_blocks(struct ocfs2_caching_info *ci, u64 block, int nr, + struct buffer_head *bhs[], int flags, + int (*validate)(struct super_block *sb, + struct buffer_head *bh)) +{ + int status = 0; + int i, ignore_cache = 0; + struct buffer_head *bh; + struct super_block *sb = ocfs2_metadata_cache_get_super(ci); + int new_bh = 0; + + trace_ocfs2_read_blocks_begin(ci, (unsigned long long)block, nr, flags); + + BUG_ON(!ci); + BUG_ON((flags & OCFS2_BH_READAHEAD) && + (flags & OCFS2_BH_IGNORE_CACHE)); + + if (bhs == NULL) { + status = -EINVAL; + mlog_errno(status); + goto bail; + } + + if (nr < 0) { + mlog(ML_ERROR, "asked to read %d blocks!\n", nr); + status = -EINVAL; + mlog_errno(status); + goto bail; + } + + if (nr == 0) { + status = 0; + goto bail; + } + + /* Don't put buffer head and re-assign it to NULL if it is allocated + * outside since the caller can't be aware of this alternation! + */ + new_bh = (bhs[0] == NULL); + + ocfs2_metadata_cache_io_lock(ci); + for (i = 0 ; i < nr ; i++) { + if (bhs[i] == NULL) { + bhs[i] = sb_getblk(sb, block++); + if (bhs[i] == NULL) { + status = -ENOMEM; + mlog_errno(status); + /* Don't forget to put previous bh! */ + break; + } + } + bh = bhs[i]; + ignore_cache = (flags & OCFS2_BH_IGNORE_CACHE); + + /* There are three read-ahead cases here which we need to + * be concerned with. All three assume a buffer has + * previously been submitted with OCFS2_BH_READAHEAD + * and it hasn't yet completed I/O. + * + * 1) The current request is sync to disk. This rarely + * happens these days, and never when performance + * matters - the code can just wait on the buffer + * lock and re-submit. + * + * 2) The current request is cached, but not + * readahead. ocfs2_buffer_uptodate() will return + * false anyway, so we'll wind up waiting on the + * buffer lock to do I/O. We re-check the request + * with after getting the lock to avoid a re-submit. + * + * 3) The current request is readahead (and so must + * also be a caching one). We short circuit if the + * buffer is locked (under I/O) and if it's in the + * uptodate cache. The re-check from #2 catches the + * case that the previous read-ahead completes just + * before our is-it-in-flight check. + */ + + if (!ignore_cache && !ocfs2_buffer_uptodate(ci, bh)) { + trace_ocfs2_read_blocks_from_disk( + (unsigned long long)bh->b_blocknr, + (unsigned long long)ocfs2_metadata_cache_owner(ci)); + /* We're using ignore_cache here to say + * "go to disk" */ + ignore_cache = 1; + } + + trace_ocfs2_read_blocks_bh((unsigned long long)bh->b_blocknr, + ignore_cache, buffer_jbd(bh), buffer_dirty(bh)); + + if (buffer_jbd(bh)) { + continue; + } + + if (ignore_cache) { + if (buffer_dirty(bh)) { + /* This should probably be a BUG, or + * at least return an error. */ + continue; + } + + /* A read-ahead request was made - if the + * buffer is already under read-ahead from a + * previously submitted request than we are + * done here. */ + if ((flags & OCFS2_BH_READAHEAD) + && ocfs2_buffer_read_ahead(ci, bh)) + continue; + + lock_buffer(bh); + if (buffer_jbd(bh)) { +#ifdef CATCH_BH_JBD_RACES + mlog(ML_ERROR, "block %llu had the JBD bit set " + "while I was in lock_buffer!", + (unsigned long long)bh->b_blocknr); + BUG(); +#else + unlock_buffer(bh); + continue; +#endif + } + + /* Re-check ocfs2_buffer_uptodate() as a + * previously read-ahead buffer may have + * completed I/O while we were waiting for the + * buffer lock. */ + if (!(flags & OCFS2_BH_IGNORE_CACHE) + && !(flags & OCFS2_BH_READAHEAD) + && ocfs2_buffer_uptodate(ci, bh)) { + unlock_buffer(bh); + continue; + } + + if (validate) + set_buffer_needs_validate(bh); + bh_submit(bh, REQ_OP_READ, bh_end_read); + continue; + } + } + +read_failure: + for (i = (nr - 1); i >= 0; i--) { + bh = bhs[i]; + + if (!(flags & OCFS2_BH_READAHEAD)) { + if (unlikely(status)) { + /* Clear the buffers on error including those + * ever succeeded in reading + */ + if (new_bh && bh) { + /* If middle bh fails, let previous bh + * finish its read and then put it to + * avoid bh leak + */ + if (!buffer_jbd(bh)) + wait_on_buffer(bh); + put_bh(bh); + bhs[i] = NULL; + } + continue; + } + /* We know this can't have changed as we hold the + * owner sem. Avoid doing any work on the bh if the + * journal has it. */ + if (!buffer_jbd(bh)) + wait_on_buffer(bh); + + if (!buffer_uptodate(bh)) { + /* Status won't be cleared from here on out, + * so we can safely record this and loop back + * to cleanup the other buffers. Don't need to + * remove the clustered uptodate information + * for this bh as it's not marked locally + * uptodate. */ + status = -EIO; + clear_buffer_needs_validate(bh); + goto read_failure; + } + + if (buffer_needs_validate(bh)) { + /* We never set NeedsValidate if the + * buffer was held by the journal, so + * that better not have changed */ + BUG_ON(buffer_jbd(bh)); + clear_buffer_needs_validate(bh); + status = validate(sb, bh); + if (status) { + if (buffer_uptodate(bh)) + clear_buffer_uptodate(bh); + goto read_failure; + } + } + } + + /* Always set the buffer in the cache, even if it was + * a forced read, or read-ahead which hasn't yet + * completed. */ + if (bh) + ocfs2_set_buffer_uptodate(ci, bh); + } + ocfs2_metadata_cache_io_unlock(ci); + + trace_ocfs2_read_blocks_end((unsigned long long)block, nr, + flags, ignore_cache); + +bail: + + return status; +} + +/* Check whether the blkno is the super block or one of the backups. */ +static void ocfs2_check_super_or_backup(struct super_block *sb, + sector_t blkno) +{ + int i; + u64 backup_blkno; + + if (blkno == OCFS2_SUPER_BLOCK_BLKNO) + return; + + for (i = 0; i < OCFS2_MAX_BACKUP_SUPERBLOCKS; i++) { + backup_blkno = ocfs2_backup_super_blkno(sb, i); + if (backup_blkno == blkno) + return; + } + + BUG(); +} + +/* + * Write super block and backups doesn't need to collaborate with journal, + * so we don't need to lock ip_io_mutex and ci doesn't need to bea passed + * into this function. + */ +int ocfs2_write_super_or_backup(struct ocfs2_super *osb, + struct buffer_head *bh) +{ + int ret = 0; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)bh->b_data; + + BUG_ON(buffer_jbd(bh)); + ocfs2_check_super_or_backup(osb->sb, bh->b_blocknr); + + if (unlikely(ocfs2_emergency_state(osb))) { + ret = -EROFS; + mlog_errno(ret); + goto out; + } + + lock_buffer(bh); + set_buffer_uptodate(bh); + + /* remove from dirty list before I/O. */ + clear_buffer_dirty(bh); + + ocfs2_compute_meta_ecc(osb->sb, bh->b_data, &di->i_check); + bh_submit(bh, REQ_OP_WRITE, bh_end_write); + + wait_on_buffer(bh); + + if (!buffer_uptodate(bh)) { + ret = -EIO; + mlog_errno(ret); + } + +out: + return ret; +} diff --git a/fs/ocfs2/buffer_head_io.h b/fs/ocfs2/buffer_head_io.h new file mode 100644 index 000000000..2d51649fc --- /dev/null +++ b/fs/ocfs2/buffer_head_io.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * ocfs2_buffer_head.h + * + * Buffer cache handling functions defined + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#ifndef OCFS2_BUFFER_HEAD_IO_H +#define OCFS2_BUFFER_HEAD_IO_H + +#include + +int ocfs2_write_block(struct ocfs2_super *osb, + struct buffer_head *bh, + struct ocfs2_caching_info *ci); +int ocfs2_read_blocks_sync(struct ocfs2_super *osb, u64 block, + unsigned int nr, struct buffer_head *bhs[]); + +/* + * If not NULL, validate() will be called on a buffer that is freshly + * read from disk. It will not be called if the buffer was in cache. + * Note that if validate() is being used for this buffer, it needs to + * be set even for a READAHEAD call, as it marks the buffer for later + * validation. + */ +int ocfs2_read_blocks(struct ocfs2_caching_info *ci, u64 block, int nr, + struct buffer_head *bhs[], int flags, + int (*validate)(struct super_block *sb, + struct buffer_head *bh)); + +int ocfs2_write_super_or_backup(struct ocfs2_super *osb, + struct buffer_head *bh); + +#define OCFS2_BH_IGNORE_CACHE 1 +#define OCFS2_BH_READAHEAD 8 + +static inline int ocfs2_read_block(struct ocfs2_caching_info *ci, u64 off, + struct buffer_head **bh, + int (*validate)(struct super_block *sb, + struct buffer_head *bh)) +{ + int status = 0; + + if (bh == NULL) { + printk("ocfs2: bh == NULL\n"); + status = -EINVAL; + goto bail; + } + + status = ocfs2_read_blocks(ci, off, 1, bh, 0, validate); + +bail: + return status; +} + +#endif /* OCFS2_BUFFER_HEAD_IO_H */ diff --git a/fs/ocfs2/cluster/Makefile b/fs/ocfs2/cluster/Makefile new file mode 100644 index 000000000..0e5b73215 --- /dev/null +++ b/fs/ocfs2/cluster/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_OCFS2_FS) += ocfs2_nodemanager.o + +ocfs2_nodemanager-objs := heartbeat.o masklog.o sys.o nodemanager.o \ + quorum.o tcp.o netdebug.o diff --git a/fs/ocfs2/cluster/heartbeat.c b/fs/ocfs2/cluster/heartbeat.c new file mode 100644 index 000000000..1c3def99b --- /dev/null +++ b/fs/ocfs2/cluster/heartbeat.c @@ -0,0 +1,2755 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2004, 2005 Oracle. All rights reserved. + */ + +#include "linux/kstrtox.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "heartbeat.h" +#include "tcp.h" +#include "nodemanager.h" +#include "quorum.h" + +#include "masklog.h" + + +/* + * The first heartbeat pass had one global thread that would serialize all hb + * callback calls. This global serializing sem should only be removed once + * we've made sure that all callees can deal with being called concurrently + * from multiple hb region threads. + */ +static DECLARE_RWSEM(o2hb_callback_sem); + +/* + * multiple hb threads are watching multiple regions. A node is live + * whenever any of the threads sees activity from the node in its region. + */ +static DEFINE_SPINLOCK(o2hb_live_lock); +/* + * Serializes region pin/unpin dependency management (o2hb_dependent_users + * and the o2nm_depend_item()/o2nm_undepend_item() calls). o2hb_region_pin() + * has to drop o2hb_live_lock across the sleeping o2nm_depend_item(), so the + * spinlock alone can no longer keep pin and unpin mutually exclusive; this + * mutex, taken outside o2hb_live_lock, does. + */ +static DEFINE_MUTEX(o2hb_dependency_mutex); +static struct list_head o2hb_live_slots[O2NM_MAX_NODES]; +static unsigned long o2hb_live_node_bitmap[BITS_TO_LONGS(O2NM_MAX_NODES)]; +static LIST_HEAD(o2hb_node_events); +static DECLARE_WAIT_QUEUE_HEAD(o2hb_steady_queue); + +/* + * In global heartbeat, we maintain a series of region bitmaps. + * - o2hb_region_bitmap allows us to limit the region number to max region. + * - o2hb_live_region_bitmap tracks live regions (seen steady iterations). + * - o2hb_quorum_region_bitmap tracks live regions that have seen all nodes + * heartbeat on it. + * - o2hb_failed_region_bitmap tracks the regions that have seen io timeouts. + */ +static unsigned long o2hb_region_bitmap[BITS_TO_LONGS(O2NM_MAX_REGIONS)]; +static unsigned long o2hb_live_region_bitmap[BITS_TO_LONGS(O2NM_MAX_REGIONS)]; +static unsigned long o2hb_quorum_region_bitmap[BITS_TO_LONGS(O2NM_MAX_REGIONS)]; +static unsigned long o2hb_failed_region_bitmap[BITS_TO_LONGS(O2NM_MAX_REGIONS)]; + +#define O2HB_DB_TYPE_LIVENODES 0 +#define O2HB_DB_TYPE_LIVEREGIONS 1 +#define O2HB_DB_TYPE_QUORUMREGIONS 2 +#define O2HB_DB_TYPE_FAILEDREGIONS 3 +#define O2HB_DB_TYPE_REGION_LIVENODES 4 +#define O2HB_DB_TYPE_REGION_NUMBER 5 +#define O2HB_DB_TYPE_REGION_ELAPSED_TIME 6 +#define O2HB_DB_TYPE_REGION_PINNED 7 +struct o2hb_debug_buf { + int db_type; + int db_size; + int db_len; + void *db_data; +}; + +static struct o2hb_debug_buf *o2hb_db_livenodes; +static struct o2hb_debug_buf *o2hb_db_liveregions; +static struct o2hb_debug_buf *o2hb_db_quorumregions; +static struct o2hb_debug_buf *o2hb_db_failedregions; + +#define O2HB_DEBUG_DIR "o2hb" +#define O2HB_DEBUG_LIVENODES "livenodes" +#define O2HB_DEBUG_LIVEREGIONS "live_regions" +#define O2HB_DEBUG_QUORUMREGIONS "quorum_regions" +#define O2HB_DEBUG_FAILEDREGIONS "failed_regions" +#define O2HB_DEBUG_REGION_NUMBER "num" +#define O2HB_DEBUG_REGION_ELAPSED_TIME "elapsed_time_in_ms" +#define O2HB_DEBUG_REGION_PINNED "pinned" + +static struct dentry *o2hb_debug_dir; + +static LIST_HEAD(o2hb_all_regions); + +static struct o2hb_callback { + struct list_head list; +} o2hb_callbacks[O2HB_NUM_CB]; + +static struct o2hb_callback *hbcall_from_type(enum o2hb_callback_type type); + +enum o2hb_heartbeat_modes { + O2HB_HEARTBEAT_LOCAL = 0, + O2HB_HEARTBEAT_GLOBAL, + O2HB_HEARTBEAT_NUM_MODES, +}; + +static const char *o2hb_heartbeat_mode_desc[O2HB_HEARTBEAT_NUM_MODES] = { + "local", /* O2HB_HEARTBEAT_LOCAL */ + "global", /* O2HB_HEARTBEAT_GLOBAL */ +}; + +unsigned int o2hb_dead_threshold = O2HB_DEFAULT_DEAD_THRESHOLD; +static unsigned int o2hb_heartbeat_mode = O2HB_HEARTBEAT_LOCAL; + +/* + * o2hb_dependent_users tracks the number of registered callbacks that depend + * on heartbeat. o2net and o2dlm are two entities that register this callback. + * However only o2dlm depends on the heartbeat. It does not want the heartbeat + * to stop while a dlm domain is still active. + */ +static unsigned int o2hb_dependent_users; + +/* + * In global heartbeat mode, all regions are pinned if there are one or more + * dependent users and the quorum region count is <= O2HB_PIN_CUT_OFF. All + * regions are unpinned if the region count exceeds the cut off or the number + * of dependent users falls to zero. + */ +#define O2HB_PIN_CUT_OFF 3 + +/* + * In local heartbeat mode, we assume the dlm domain name to be the same as + * region uuid. This is true for domains created for the file system but not + * necessarily true for userdlm domains. This is a known limitation. + * + * In global heartbeat mode, we pin/unpin all o2hb regions. This solution + * works for both file system and userdlm domains. + */ +static int o2hb_region_pin(const char *region_uuid, bool from_callback); +static void o2hb_region_unpin(const char *region_uuid); + +/* Only sets a new threshold if there are no active regions. + * + * No locking or otherwise interesting code is required for reading + * o2hb_dead_threshold as it can't change once regions are active and + * it's not interesting to anyone until then anyway. */ +static void o2hb_dead_threshold_set(unsigned int threshold) +{ + if (threshold > O2HB_MIN_DEAD_THRESHOLD) { + spin_lock(&o2hb_live_lock); + if (list_empty(&o2hb_all_regions)) + o2hb_dead_threshold = threshold; + spin_unlock(&o2hb_live_lock); + } +} + +static int o2hb_global_heartbeat_mode_set(unsigned int hb_mode) +{ + int ret = -1; + + if (hb_mode < O2HB_HEARTBEAT_NUM_MODES) { + spin_lock(&o2hb_live_lock); + if (list_empty(&o2hb_all_regions)) { + o2hb_heartbeat_mode = hb_mode; + ret = 0; + } + spin_unlock(&o2hb_live_lock); + } + + return ret; +} + +struct o2hb_node_event { + struct list_head hn_item; + enum o2hb_callback_type hn_event_type; + struct o2nm_node *hn_node; + int hn_node_num; +}; + +struct o2hb_disk_slot { + struct o2hb_disk_heartbeat_block *ds_raw_block; + u8 ds_node_num; + u64 ds_last_time; + u64 ds_last_generation; + u16 ds_equal_samples; + u16 ds_changed_samples; + struct list_head ds_live_item; +}; + +/* each thread owns a region.. when we're asked to tear down the region + * we ask the thread to stop, who cleans up the region */ +struct o2hb_region { + struct config_item hr_item; + + struct list_head hr_all_item; + unsigned hr_unclean_stop:1, + hr_aborted_start:1, + hr_item_pinned:1, + hr_item_dropped:1, + hr_node_deleted:1; + + /* protected by the hr_callback_sem */ + struct task_struct *hr_task; + u8 hr_node_num; + + unsigned int hr_blocks; + unsigned long long hr_start_block; + + unsigned int hr_block_bits; + unsigned int hr_block_bytes; + + unsigned int hr_slots_per_page; + unsigned int hr_num_pages; + + struct page **hr_slot_data; + struct file *hr_bdev_file; + struct o2hb_disk_slot *hr_slots; + + /* live node map of this region */ + unsigned long hr_live_node_bitmap[BITS_TO_LONGS(O2NM_MAX_NODES)]; + unsigned int hr_region_num; + + struct dentry *hr_debug_dir; + struct o2hb_debug_buf *hr_db_livenodes; + struct o2hb_debug_buf *hr_db_regnum; + struct o2hb_debug_buf *hr_db_elapsed_time; + struct o2hb_debug_buf *hr_db_pinned; + + /* let the person setting up hb wait for it to return until it + * has reached a 'steady' state. This will be fixed when we have + * a more complete api that doesn't lead to this sort of fragility. */ + atomic_t hr_steady_iterations; + + /* terminate o2hb thread if it does not reach steady state + * (hr_steady_iterations == 0) within hr_unsteady_iterations */ + atomic_t hr_unsteady_iterations; + + unsigned int hr_timeout_ms; + + /* randomized as the region goes up and down so that a node + * recognizes a node going up and down in one iteration */ + u64 hr_generation; + + struct delayed_work hr_write_timeout_work; + unsigned long hr_last_timeout_start; + + /* negotiate timer, used to negotiate extending hb timeout. */ + struct delayed_work hr_nego_timeout_work; + unsigned long hr_nego_node_bitmap[BITS_TO_LONGS(O2NM_MAX_NODES)]; + + /* Used during o2hb_check_slot to hold a copy of the block + * being checked because we temporarily have to zero out the + * crc field. */ + struct o2hb_disk_heartbeat_block *hr_tmp_block; + + /* Message key for negotiate timeout message. */ + unsigned int hr_key; + struct list_head hr_handler_list; + /* Serializes timeout arming against failed-start and teardown. */ + struct mutex hr_arming_mutex; + bool hr_stopping; + + /* last hb status, 0 for success, other value for error. */ + int hr_last_hb_status; +}; + +static inline struct block_device *reg_bdev(struct o2hb_region *reg) +{ + return reg->hr_bdev_file ? file_bdev(reg->hr_bdev_file) : NULL; +} + +struct o2hb_bio_wait_ctxt { + atomic_t wc_num_reqs; + struct completion wc_io_complete; + int wc_error; + /* On-stack bio used by the synchronous write path only. */ + struct bio wc_write_bio; + struct bio_vec wc_write_bvec; +}; + +#define O2HB_NEGO_TIMEOUT_MS (O2HB_MAX_WRITE_TIMEOUT_MS/2) + +enum { + O2HB_NEGO_TIMEOUT_MSG = 1, + O2HB_NEGO_APPROVE_MSG = 2, +}; + +struct o2hb_nego_msg { + u8 node_num; +}; + +static void o2hb_write_timeout(struct work_struct *work) +{ + int failed, quorum; + struct o2hb_region *reg = + container_of(work, struct o2hb_region, + hr_write_timeout_work.work); + + mlog(ML_ERROR, "Heartbeat write timeout to device %pg after %u " + "milliseconds\n", reg_bdev(reg), + jiffies_to_msecs(jiffies - reg->hr_last_timeout_start)); + + if (o2hb_global_heartbeat_active()) { + spin_lock(&o2hb_live_lock); + if (test_bit(reg->hr_region_num, o2hb_quorum_region_bitmap)) + set_bit(reg->hr_region_num, o2hb_failed_region_bitmap); + failed = bitmap_weight(o2hb_failed_region_bitmap, + O2NM_MAX_REGIONS); + quorum = bitmap_weight(o2hb_quorum_region_bitmap, + O2NM_MAX_REGIONS); + spin_unlock(&o2hb_live_lock); + + mlog(ML_HEARTBEAT, "Number of regions %d, failed regions %d\n", + quorum, failed); + + /* + * Fence if the number of failed regions >= half the number + * of quorum regions + */ + if ((failed << 1) < quorum) + return; + } + + o2quo_disk_timeout(); +} + +static void o2hb_arm_timeout(struct o2hb_region *reg) +{ + mutex_lock(®->hr_arming_mutex); + + if (reg->hr_stopping) + goto out_unlock; + + /* Arm writeout only after thread reaches steady state */ + if (atomic_read(®->hr_steady_iterations) != 0) + goto out_unlock; + + mlog(ML_HEARTBEAT, "Queue write timeout for %u ms\n", + O2HB_MAX_WRITE_TIMEOUT_MS); + + if (o2hb_global_heartbeat_active()) { + spin_lock(&o2hb_live_lock); + clear_bit(reg->hr_region_num, o2hb_failed_region_bitmap); + spin_unlock(&o2hb_live_lock); + } + cancel_delayed_work(®->hr_write_timeout_work); + schedule_delayed_work(®->hr_write_timeout_work, + msecs_to_jiffies(O2HB_MAX_WRITE_TIMEOUT_MS)); + + cancel_delayed_work(®->hr_nego_timeout_work); + /* negotiate timeout must be less than write timeout. */ + schedule_delayed_work(®->hr_nego_timeout_work, + msecs_to_jiffies(O2HB_NEGO_TIMEOUT_MS)); + bitmap_zero(reg->hr_nego_node_bitmap, O2NM_MAX_NODES); + +out_unlock: + mutex_unlock(®->hr_arming_mutex); +} + +static void o2hb_queue_nego_timeout(struct o2hb_region *reg, + unsigned long delay) +{ + mutex_lock(®->hr_arming_mutex); + if (!reg->hr_stopping) + schedule_delayed_work(®->hr_nego_timeout_work, delay); + mutex_unlock(®->hr_arming_mutex); +} + +static void o2hb_disarm_timeout(struct o2hb_region *reg) +{ + cancel_delayed_work_sync(®->hr_write_timeout_work); + cancel_delayed_work_sync(®->hr_nego_timeout_work); +} + +static void o2hb_set_region_stopping(struct o2hb_region *reg, bool stopping) +{ + mutex_lock(®->hr_arming_mutex); + reg->hr_stopping = stopping; + mutex_unlock(®->hr_arming_mutex); +} + +static void o2hb_quiesce_timeout(struct o2hb_region *reg) +{ + o2hb_set_region_stopping(reg, true); + o2hb_disarm_timeout(reg); +} + +static int o2hb_send_nego_msg(int key, int type, u8 target, u8 node_num) +{ + struct o2hb_nego_msg msg; + int status, ret; + + msg.node_num = node_num; +again: + ret = o2net_send_message(type, key, &msg, sizeof(msg), + target, &status); + + if (ret == -EAGAIN || ret == -ENOMEM) { + msleep(100); + goto again; + } + + return ret; +} + +static void o2hb_nego_timeout(struct work_struct *work) +{ + unsigned long live_node_bitmap[BITS_TO_LONGS(O2NM_MAX_NODES)]; + int master_node, i, ret; + struct o2hb_region *reg; + u8 node_num; + + reg = container_of(work, struct o2hb_region, hr_nego_timeout_work.work); + node_num = reg->hr_node_num; + /* don't negotiate timeout if last hb failed since it is very + * possible io failed. Should let write timeout fence self. + */ + if (reg->hr_last_hb_status) + return; + + o2hb_fill_node_map(live_node_bitmap, O2NM_MAX_NODES); + /* lowest node as master node to make negotiate decision. */ + master_node = find_first_bit(live_node_bitmap, O2NM_MAX_NODES); + + if (master_node == node_num) { + if (!test_bit(master_node, reg->hr_nego_node_bitmap)) { + printk(KERN_NOTICE "o2hb: node %d hb write hung for %ds on region %s (%pg).\n", + node_num, O2HB_NEGO_TIMEOUT_MS / 1000, + config_item_name(®->hr_item), reg_bdev(reg)); + set_bit(master_node, reg->hr_nego_node_bitmap); + } + if (!bitmap_equal(reg->hr_nego_node_bitmap, live_node_bitmap, + O2NM_MAX_NODES)) { + /* check negotiate bitmap every second to do timeout + * approve decision. + */ + o2hb_queue_nego_timeout(reg, msecs_to_jiffies(1000)); + + return; + } + + printk(KERN_NOTICE "o2hb: all nodes hb write hung, maybe region %s (%pg) is down.\n", + config_item_name(®->hr_item), + reg_bdev(reg)); + /* approve negotiate timeout request. */ + o2hb_arm_timeout(reg); + + i = -1; + while ((i = find_next_bit(live_node_bitmap, + O2NM_MAX_NODES, i + 1)) < O2NM_MAX_NODES) { + if (i == master_node) + continue; + + mlog(ML_HEARTBEAT, "send NEGO_APPROVE msg to node %d\n", i); + ret = o2hb_send_nego_msg(reg->hr_key, + O2HB_NEGO_APPROVE_MSG, i, node_num); + if (ret) + mlog(ML_ERROR, "send NEGO_APPROVE msg to node %d fail %d\n", + i, ret); + } + } else { + /* negotiate timeout with master node. */ + printk(KERN_NOTICE "o2hb: node %d hb write hung for %ds on region %s (%pg), negotiate timeout with node %d.\n", + node_num, O2HB_NEGO_TIMEOUT_MS / 1000, config_item_name(®->hr_item), + reg_bdev(reg), master_node); + ret = o2hb_send_nego_msg(reg->hr_key, O2HB_NEGO_TIMEOUT_MSG, + master_node, node_num); + if (ret) + mlog(ML_ERROR, "send NEGO_TIMEOUT msg to node %d fail %d\n", + master_node, ret); + } +} + +static int o2hb_nego_timeout_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data) +{ + struct o2hb_region *reg = data; + struct o2hb_nego_msg *nego_msg; + + nego_msg = (struct o2hb_nego_msg *)msg->buf; + printk(KERN_NOTICE "o2hb: receive negotiate timeout message from node %d on region %s (%pg).\n", + nego_msg->node_num, config_item_name(®->hr_item), + reg_bdev(reg)); + if (nego_msg->node_num < O2NM_MAX_NODES) + set_bit(nego_msg->node_num, reg->hr_nego_node_bitmap); + else + mlog(ML_ERROR, "got nego timeout message from bad node.\n"); + + return 0; +} + +static int o2hb_nego_approve_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data) +{ + struct o2hb_region *reg = data; + + printk(KERN_NOTICE "o2hb: negotiate timeout approved by master node on region %s (%pg).\n", + config_item_name(®->hr_item), reg_bdev(reg)); + o2hb_arm_timeout(reg); + return 0; +} + +static inline void o2hb_bio_wait_init(struct o2hb_bio_wait_ctxt *wc) +{ + atomic_set(&wc->wc_num_reqs, 1); + init_completion(&wc->wc_io_complete); + wc->wc_error = 0; +} + +/* Used in error paths too */ +static inline void o2hb_bio_wait_dec(struct o2hb_bio_wait_ctxt *wc, + unsigned int num) +{ + /* sadly atomic_sub_and_test() isn't available on all platforms. The + * good news is that the fast path only completes one at a time */ + while(num--) { + if (atomic_dec_and_test(&wc->wc_num_reqs)) { + BUG_ON(num > 0); + complete(&wc->wc_io_complete); + } + } +} + +static void o2hb_wait_on_io(struct o2hb_bio_wait_ctxt *wc) +{ + o2hb_bio_wait_dec(wc, 1); + wait_for_completion(&wc->wc_io_complete); +} + +static void o2hb_bio_end_io(struct bio *bio) +{ + struct o2hb_bio_wait_ctxt *wc = bio->bi_private; + + if (bio->bi_status) { + mlog(ML_ERROR, "IO Error %d\n", bio->bi_status); + wc->wc_error = blk_status_to_errno(bio->bi_status); + } + + o2hb_bio_wait_dec(wc, 1); + bio_put(bio); +} + +/* + * End I/O for the synchronous write path. The write bio is embedded in + * the wait ctxt rather than allocated, so it must not be freed here; it + * is torn down with bio_uninit() once the caller has waited on it. + */ +static void o2hb_write_bio_end_io(struct bio *bio) +{ + struct o2hb_bio_wait_ctxt *wc = bio->bi_private; + + if (bio->bi_status) { + mlog(ML_ERROR, "IO Error %d\n", bio->bi_status); + wc->wc_error = blk_status_to_errno(bio->bi_status); + } + + o2hb_bio_wait_dec(wc, 1); +} + +/* Setup a Bio to cover I/O against num_slots slots starting at + * start_slot. */ +static struct bio *o2hb_setup_one_bio(struct o2hb_region *reg, + struct o2hb_bio_wait_ctxt *wc, + unsigned int *current_slot, + unsigned int max_slots, blk_opf_t opf) +{ + int len, current_page; + unsigned int vec_len, vec_start; + unsigned int bits = reg->hr_block_bits; + unsigned int spp = reg->hr_slots_per_page; + unsigned int cs = *current_slot; + struct bio *bio; + struct page *page; + + /* + * The heartbeat runs in process context and can sleep, so use + * GFP_NOFS. It is backed by the fs_bio_set mempool and thus cannot + * fail, while avoiding recursion back into the filesystem. + */ + bio = bio_alloc(reg_bdev(reg), 16, opf, GFP_NOFS); + + /* Must put everything in 512 byte sectors for the bio... */ + bio->bi_iter.bi_sector = (reg->hr_start_block + cs) << (bits - 9); + bio->bi_private = wc; + bio->bi_end_io = o2hb_bio_end_io; + + vec_start = (cs << bits) % PAGE_SIZE; + while(cs < max_slots) { + current_page = cs / spp; + page = reg->hr_slot_data[current_page]; + + vec_len = min(PAGE_SIZE - vec_start, + (max_slots-cs) * (PAGE_SIZE/spp) ); + + mlog(ML_HB_BIO, "page %d, vec_len = %u, vec_start = %u\n", + current_page, vec_len, vec_start); + + len = bio_add_page(bio, page, vec_len, vec_start); + if (len != vec_len) break; + + cs += vec_len / (PAGE_SIZE/spp); + vec_start = 0; + } + + *current_slot = cs; + return bio; +} + +static int o2hb_read_slots(struct o2hb_region *reg, + unsigned int begin_slot, + unsigned int max_slots) +{ + unsigned int current_slot = begin_slot; + struct o2hb_bio_wait_ctxt wc; + struct bio *bio; + + o2hb_bio_wait_init(&wc); + + while(current_slot < max_slots) { + bio = o2hb_setup_one_bio(reg, &wc, ¤t_slot, max_slots, + REQ_OP_READ); + atomic_inc(&wc.wc_num_reqs); + submit_bio(bio); + } + + o2hb_wait_on_io(&wc); + + return wc.wc_error; +} + +static int o2hb_issue_node_write(struct o2hb_region *reg, + struct o2hb_bio_wait_ctxt *write_wc) +{ + unsigned int slot; + unsigned int bits = reg->hr_block_bits; + unsigned int spp = reg->hr_slots_per_page; + unsigned int vec_start, vec_len; + struct page *page; + struct bio *bio = &write_wc->wc_write_bio; + + o2hb_bio_wait_init(write_wc); + + slot = reg->hr_node_num; + if (slot >= O2NM_MAX_NODES) + return -EINVAL; + + /* + * The heartbeat write always covers our own single slot, i.e. one + * block that lives within a single page. Use an on-stack bio (embedded + * in write_wc) so this fence-critical path never has to allocate. + */ + bio_init(bio, reg_bdev(reg), &write_wc->wc_write_bvec, 1, + REQ_OP_WRITE | REQ_SYNC); + bio->bi_iter.bi_sector = (reg->hr_start_block + slot) << (bits - 9); + bio->bi_private = write_wc; + bio->bi_end_io = o2hb_write_bio_end_io; + + page = reg->hr_slot_data[slot / spp]; + vec_start = (slot << bits) % PAGE_SIZE; + vec_len = PAGE_SIZE / spp; + __bio_add_page(bio, page, vec_len, vec_start); + + atomic_inc(&write_wc->wc_num_reqs); + submit_bio(bio); + + return 0; +} + +static u32 o2hb_compute_block_crc_le(struct o2hb_region *reg, + struct o2hb_disk_heartbeat_block *hb_block) +{ + __le32 old_cksum; + u32 ret; + + /* We want to compute the block crc with a 0 value in the + * hb_cksum field. Save it off here and replace after the + * crc. */ + old_cksum = hb_block->hb_cksum; + hb_block->hb_cksum = 0; + + ret = crc32_le(0, (unsigned char *) hb_block, reg->hr_block_bytes); + + hb_block->hb_cksum = old_cksum; + + return ret; +} + +static void o2hb_dump_slot(struct o2hb_disk_heartbeat_block *hb_block) +{ + mlog(ML_ERROR, "Dump slot information: seq = 0x%llx, node = %u, " + "cksum = 0x%x, generation 0x%llx\n", + (long long)le64_to_cpu(hb_block->hb_seq), + hb_block->hb_node, le32_to_cpu(hb_block->hb_cksum), + (long long)le64_to_cpu(hb_block->hb_generation)); +} + +static int o2hb_verify_crc(struct o2hb_region *reg, + struct o2hb_disk_heartbeat_block *hb_block) +{ + u32 read, computed; + + read = le32_to_cpu(hb_block->hb_cksum); + computed = o2hb_compute_block_crc_le(reg, hb_block); + + return read == computed; +} + +/* + * Compare the slot data with what we wrote in the last iteration. + * If the match fails, print an appropriate error message. This is to + * detect errors like... another node hearting on the same slot, + * flaky device that is losing writes, etc. + * Returns 1 if check succeeds, 0 otherwise. + */ +static int o2hb_check_own_slot(struct o2hb_region *reg) +{ + struct o2hb_disk_slot *slot; + struct o2hb_disk_heartbeat_block *hb_block; + char *errstr; + u8 node_num = reg->hr_node_num; + + if (node_num >= O2NM_MAX_NODES) + return 0; + + slot = ®->hr_slots[node_num]; + /* Don't check on our 1st timestamp */ + if (!slot->ds_last_time) + return 0; + + hb_block = slot->ds_raw_block; + if (le64_to_cpu(hb_block->hb_seq) == slot->ds_last_time && + le64_to_cpu(hb_block->hb_generation) == slot->ds_last_generation && + hb_block->hb_node == slot->ds_node_num) + return 1; + +#define ERRSTR1 "Another node is heartbeating on device" +#define ERRSTR2 "Heartbeat generation mismatch on device" +#define ERRSTR3 "Heartbeat sequence mismatch on device" + + if (hb_block->hb_node != slot->ds_node_num) + errstr = ERRSTR1; + else if (le64_to_cpu(hb_block->hb_generation) != + slot->ds_last_generation) + errstr = ERRSTR2; + else + errstr = ERRSTR3; + + mlog(ML_ERROR, "%s (%pg): expected(%u:0x%llx, 0x%llx), " + "ondisk(%u:0x%llx, 0x%llx)\n", errstr, reg_bdev(reg), + slot->ds_node_num, (unsigned long long)slot->ds_last_generation, + (unsigned long long)slot->ds_last_time, hb_block->hb_node, + (unsigned long long)le64_to_cpu(hb_block->hb_generation), + (unsigned long long)le64_to_cpu(hb_block->hb_seq)); + + return 0; +} + +static inline void o2hb_prepare_block(struct o2hb_region *reg, + u64 generation) +{ + int node_num; + u64 cputime; + struct o2hb_disk_slot *slot; + struct o2hb_disk_heartbeat_block *hb_block; + + node_num = reg->hr_node_num; + if (node_num >= O2NM_MAX_NODES) + return; + + slot = ®->hr_slots[node_num]; + + hb_block = (struct o2hb_disk_heartbeat_block *)slot->ds_raw_block; + memset(hb_block, 0, reg->hr_block_bytes); + /* TODO: time stuff */ + cputime = ktime_get_real_seconds(); + if (!cputime) + cputime = 1; + + hb_block->hb_seq = cpu_to_le64(cputime); + hb_block->hb_node = node_num; + hb_block->hb_generation = cpu_to_le64(generation); + hb_block->hb_dead_ms = cpu_to_le32(o2hb_dead_threshold * O2HB_REGION_TIMEOUT_MS); + + /* This step must always happen last! */ + hb_block->hb_cksum = cpu_to_le32(o2hb_compute_block_crc_le(reg, + hb_block)); + + mlog(ML_HB_BIO, "our node generation = 0x%llx, cksum = 0x%x\n", + (long long)generation, + le32_to_cpu(hb_block->hb_cksum)); +} + +static void o2hb_fire_callbacks(struct o2hb_callback *hbcall, + struct o2nm_node *node, + int idx) +{ + struct o2hb_callback_func *f; + + list_for_each_entry(f, &hbcall->list, hc_item) { + mlog(ML_HEARTBEAT, "calling funcs %p\n", f); + (f->hc_func)(node, idx, f->hc_data); + } +} + +/* Will run the list in order until we process the passed event */ +static void o2hb_run_event_list(struct o2hb_node_event *queued_event) +{ + struct o2hb_callback *hbcall; + struct o2hb_node_event *event; + + /* Holding callback sem assures we don't alter the callback + * lists when doing this, and serializes ourselves with other + * processes wanting callbacks. */ + down_write(&o2hb_callback_sem); + + spin_lock(&o2hb_live_lock); + while (!list_empty(&o2hb_node_events) + && !list_empty(&queued_event->hn_item)) { + event = list_entry(o2hb_node_events.next, + struct o2hb_node_event, + hn_item); + list_del_init(&event->hn_item); + spin_unlock(&o2hb_live_lock); + + mlog(ML_HEARTBEAT, "Node %s event for %d\n", + event->hn_event_type == O2HB_NODE_UP_CB ? "UP" : "DOWN", + event->hn_node_num); + + hbcall = hbcall_from_type(event->hn_event_type); + + /* We should *never* have gotten on to the list with a + * bad type... This isn't something that we should try + * to recover from. */ + BUG_ON(IS_ERR(hbcall)); + + o2hb_fire_callbacks(hbcall, event->hn_node, event->hn_node_num); + + spin_lock(&o2hb_live_lock); + } + spin_unlock(&o2hb_live_lock); + + up_write(&o2hb_callback_sem); +} + +static void o2hb_queue_node_event(struct o2hb_node_event *event, + enum o2hb_callback_type type, + struct o2nm_node *node, + int node_num) +{ + assert_spin_locked(&o2hb_live_lock); + + BUG_ON((!node) && (type != O2HB_NODE_DOWN_CB)); + + event->hn_event_type = type; + event->hn_node = node; + event->hn_node_num = node_num; + + mlog(ML_HEARTBEAT, "Queue node %s event for node %d\n", + type == O2HB_NODE_UP_CB ? "UP" : "DOWN", node_num); + + list_add_tail(&event->hn_item, &o2hb_node_events); +} + +static void o2hb_shutdown_slot(struct o2hb_disk_slot *slot) +{ + struct o2hb_node_event event = + { .hn_item = LIST_HEAD_INIT(event.hn_item), }; + struct o2nm_node *node; + int queued = 0; + + node = o2nm_get_node_by_num(slot->ds_node_num); + if (!node) + return; + + spin_lock(&o2hb_live_lock); + if (!list_empty(&slot->ds_live_item)) { + mlog(ML_HEARTBEAT, "Shutdown, node %d leaves region\n", + slot->ds_node_num); + + list_del_init(&slot->ds_live_item); + + if (list_empty(&o2hb_live_slots[slot->ds_node_num])) { + clear_bit(slot->ds_node_num, o2hb_live_node_bitmap); + + o2hb_queue_node_event(&event, O2HB_NODE_DOWN_CB, node, + slot->ds_node_num); + queued = 1; + } + } + spin_unlock(&o2hb_live_lock); + + if (queued) + o2hb_run_event_list(&event); + + o2nm_node_put(node); +} + +static void o2hb_set_quorum_device(struct o2hb_region *reg) +{ + if (!o2hb_global_heartbeat_active()) + return; + + /* Prevent race with o2hb_heartbeat_group_drop_item() */ + if (kthread_should_stop()) + return; + + /* Tag region as quorum only after thread reaches steady state */ + if (atomic_read(®->hr_steady_iterations) != 0) + return; + + spin_lock(&o2hb_live_lock); + + if (test_bit(reg->hr_region_num, o2hb_quorum_region_bitmap)) + goto unlock; + + /* + * A region can be added to the quorum only when it sees all + * live nodes heartbeat on it. In other words, the region has been + * added to all nodes. + */ + if (!bitmap_equal(reg->hr_live_node_bitmap, o2hb_live_node_bitmap, + O2NM_MAX_NODES)) + goto unlock; + + printk(KERN_NOTICE "o2hb: Region %s (%pg) is now a quorum device\n", + config_item_name(®->hr_item), reg_bdev(reg)); + + set_bit(reg->hr_region_num, o2hb_quorum_region_bitmap); + + /* + * If global heartbeat active, unpin all regions if the + * region count > CUT_OFF + */ + if (bitmap_weight(o2hb_quorum_region_bitmap, + O2NM_MAX_REGIONS) > O2HB_PIN_CUT_OFF) + o2hb_region_unpin(NULL); +unlock: + spin_unlock(&o2hb_live_lock); +} + +static int o2hb_check_slot(struct o2hb_region *reg, + struct o2hb_disk_slot *slot) +{ + int changed = 0, gen_changed = 0; + struct o2hb_node_event event = + { .hn_item = LIST_HEAD_INIT(event.hn_item), }; + struct o2nm_node *node; + struct o2hb_disk_heartbeat_block *hb_block = reg->hr_tmp_block; + u64 cputime; + unsigned int dead_ms = o2hb_dead_threshold * O2HB_REGION_TIMEOUT_MS; + unsigned int slot_dead_ms; + int tmp; + int queued = 0; + + memcpy(hb_block, slot->ds_raw_block, reg->hr_block_bytes); + + /* + * If a node is no longer configured but is still in the livemap, we + * may need to clear that bit from the livemap. + */ + node = o2nm_get_node_by_num(slot->ds_node_num); + if (!node) { + spin_lock(&o2hb_live_lock); + tmp = test_bit(slot->ds_node_num, o2hb_live_node_bitmap); + spin_unlock(&o2hb_live_lock); + if (!tmp) + return 0; + } + + if (!o2hb_verify_crc(reg, hb_block)) { + /* all paths from here will drop o2hb_live_lock for + * us. */ + spin_lock(&o2hb_live_lock); + + /* Don't print an error on the console in this case - + * a freshly formatted heartbeat area will not have a + * crc set on it. */ + if (list_empty(&slot->ds_live_item)) + goto out; + + /* The node is live but pushed out a bad crc. We + * consider it a transient miss but don't populate any + * other values as they may be junk. */ + mlog(ML_ERROR, "Node %d has written a bad crc to %pg\n", + slot->ds_node_num, reg_bdev(reg)); + o2hb_dump_slot(hb_block); + + slot->ds_equal_samples++; + goto fire_callbacks; + } + + /* we don't care if these wrap.. the state transitions below + * clear at the right places */ + cputime = le64_to_cpu(hb_block->hb_seq); + if (slot->ds_last_time != cputime) + slot->ds_changed_samples++; + else + slot->ds_equal_samples++; + slot->ds_last_time = cputime; + + /* The node changed heartbeat generations. We assume this to + * mean it dropped off but came back before we timed out. We + * want to consider it down for the time being but don't want + * to lose any changed_samples state we might build up to + * considering it live again. */ + if (slot->ds_last_generation != le64_to_cpu(hb_block->hb_generation)) { + gen_changed = 1; + slot->ds_equal_samples = 0; + mlog(ML_HEARTBEAT, "Node %d changed generation (0x%llx " + "to 0x%llx)\n", slot->ds_node_num, + (long long)slot->ds_last_generation, + (long long)le64_to_cpu(hb_block->hb_generation)); + } + + slot->ds_last_generation = le64_to_cpu(hb_block->hb_generation); + + mlog(ML_HEARTBEAT, "Slot %d gen 0x%llx cksum 0x%x " + "seq %llu last %llu changed %u equal %u\n", + slot->ds_node_num, (long long)slot->ds_last_generation, + le32_to_cpu(hb_block->hb_cksum), + (unsigned long long)le64_to_cpu(hb_block->hb_seq), + (unsigned long long)slot->ds_last_time, slot->ds_changed_samples, + slot->ds_equal_samples); + + spin_lock(&o2hb_live_lock); + +fire_callbacks: + /* dead nodes only come to life after some number of + * changes at any time during their dead time */ + if (list_empty(&slot->ds_live_item) && + slot->ds_changed_samples >= O2HB_LIVE_THRESHOLD) { + mlog(ML_HEARTBEAT, "Node %d (id 0x%llx) joined my region\n", + slot->ds_node_num, (long long)slot->ds_last_generation); + + set_bit(slot->ds_node_num, reg->hr_live_node_bitmap); + + /* first on the list generates a callback */ + if (list_empty(&o2hb_live_slots[slot->ds_node_num])) { + mlog(ML_HEARTBEAT, "o2hb: Add node %d to live nodes " + "bitmap\n", slot->ds_node_num); + set_bit(slot->ds_node_num, o2hb_live_node_bitmap); + + o2hb_queue_node_event(&event, O2HB_NODE_UP_CB, node, + slot->ds_node_num); + + changed = 1; + queued = 1; + } + + list_add_tail(&slot->ds_live_item, + &o2hb_live_slots[slot->ds_node_num]); + + slot->ds_equal_samples = 0; + + /* We want to be sure that all nodes agree on the + * number of milliseconds before a node will be + * considered dead. The self-fencing timeout is + * computed from this value, and a discrepancy might + * result in heartbeat calling a node dead when it + * hasn't self-fenced yet. */ + slot_dead_ms = le32_to_cpu(hb_block->hb_dead_ms); + if (slot_dead_ms && slot_dead_ms != dead_ms) { + /* TODO: Perhaps we can fail the region here. */ + mlog(ML_ERROR, "Node %d on device %pg has a dead count " + "of %u ms, but our count is %u ms.\n" + "Please double check your configuration values " + "for 'O2CB_HEARTBEAT_THRESHOLD'\n", + slot->ds_node_num, reg_bdev(reg), + slot_dead_ms, dead_ms); + } + goto out; + } + + /* if the list is dead, we're done.. */ + if (list_empty(&slot->ds_live_item)) + goto out; + + /* live nodes only go dead after enough consecutive missed + * samples.. reset the missed counter whenever we see + * activity */ + if (slot->ds_equal_samples >= o2hb_dead_threshold || gen_changed) { + mlog(ML_HEARTBEAT, "Node %d left my region\n", + slot->ds_node_num); + + clear_bit(slot->ds_node_num, reg->hr_live_node_bitmap); + + /* last off the live_slot generates a callback */ + list_del_init(&slot->ds_live_item); + if (list_empty(&o2hb_live_slots[slot->ds_node_num])) { + mlog(ML_HEARTBEAT, "o2hb: Remove node %d from live " + "nodes bitmap\n", slot->ds_node_num); + clear_bit(slot->ds_node_num, o2hb_live_node_bitmap); + + /* node can be null */ + o2hb_queue_node_event(&event, O2HB_NODE_DOWN_CB, + node, slot->ds_node_num); + + changed = 1; + queued = 1; + } + + /* We don't clear this because the node is still + * actually writing new blocks. */ + if (!gen_changed) + slot->ds_changed_samples = 0; + goto out; + } + if (slot->ds_changed_samples) { + slot->ds_changed_samples = 0; + slot->ds_equal_samples = 0; + } +out: + spin_unlock(&o2hb_live_lock); + + if (queued) + o2hb_run_event_list(&event); + + if (node) + o2nm_node_put(node); + return changed; +} + +static int o2hb_highest_node(unsigned long *nodes, int numbits) +{ + return find_last_bit(nodes, numbits); +} + +static int o2hb_lowest_node(unsigned long *nodes, int numbits) +{ + return find_first_bit(nodes, numbits); +} + +static int o2hb_do_disk_heartbeat(struct o2hb_region *reg) +{ + int i, ret, highest_node, lowest_node; + int membership_change = 0, own_slot_ok = 0; + unsigned long configured_nodes[BITS_TO_LONGS(O2NM_MAX_NODES)]; + unsigned long live_node_bitmap[BITS_TO_LONGS(O2NM_MAX_NODES)]; + struct o2hb_bio_wait_ctxt write_wc; + + ret = o2nm_configured_node_map(configured_nodes, + sizeof(configured_nodes)); + if (ret) { + mlog_errno(ret); + goto bail; + } + + /* + * If a node is not configured but is in the livemap, we still need + * to read the slot so as to be able to remove it from the livemap. + */ + o2hb_fill_node_map(live_node_bitmap, O2NM_MAX_NODES); + i = -1; + while ((i = find_next_bit(live_node_bitmap, + O2NM_MAX_NODES, i + 1)) < O2NM_MAX_NODES) { + set_bit(i, configured_nodes); + } + + highest_node = o2hb_highest_node(configured_nodes, O2NM_MAX_NODES); + lowest_node = o2hb_lowest_node(configured_nodes, O2NM_MAX_NODES); + if (highest_node >= O2NM_MAX_NODES || lowest_node >= O2NM_MAX_NODES) { + mlog(ML_NOTICE, "o2hb: No configured nodes found!\n"); + ret = -EINVAL; + goto bail; + } + + /* No sense in reading the slots of nodes that don't exist + * yet. Of course, if the node definitions have holes in them + * then we're reading an empty slot anyway... Consider this + * best-effort. */ + ret = o2hb_read_slots(reg, lowest_node, highest_node + 1); + if (ret < 0) { + mlog_errno(ret); + goto bail; + } + + /* With an up to date view of the slots, we can check that no + * other node has been improperly configured to heartbeat in + * our slot. */ + own_slot_ok = o2hb_check_own_slot(reg); + + /* fill in the proper info for our next heartbeat */ + o2hb_prepare_block(reg, reg->hr_generation); + + ret = o2hb_issue_node_write(reg, &write_wc); + if (ret < 0) { + mlog_errno(ret); + goto bail; + } + + i = -1; + while((i = find_next_bit(configured_nodes, + O2NM_MAX_NODES, i + 1)) < O2NM_MAX_NODES) { + membership_change |= o2hb_check_slot(reg, ®->hr_slots[i]); + } + + /* + * We have to be sure we've advertised ourselves on disk + * before we can go to steady state. This ensures that + * people we find in our steady state have seen us. + */ + o2hb_wait_on_io(&write_wc); + bio_uninit(&write_wc.wc_write_bio); + if (write_wc.wc_error) { + /* Do not re-arm the write timeout on I/O error - we + * can't be sure that the new block ever made it to + * disk */ + mlog(ML_ERROR, "Write error %d on device \"%pg\"\n", + write_wc.wc_error, reg_bdev(reg)); + ret = write_wc.wc_error; + goto bail; + } + + /* Skip disarming the timeout if own slot has stale/bad data */ + if (own_slot_ok) { + o2hb_set_quorum_device(reg); + o2hb_arm_timeout(reg); + reg->hr_last_timeout_start = jiffies; + } + +bail: + /* let the person who launched us know when things are steady */ + if (atomic_read(®->hr_steady_iterations) != 0) { + if (!ret && own_slot_ok && !membership_change) { + if (atomic_dec_and_test(®->hr_steady_iterations)) + wake_up(&o2hb_steady_queue); + } + } + + if (atomic_read(®->hr_steady_iterations) != 0) { + if (atomic_dec_and_test(®->hr_unsteady_iterations)) { + printk(KERN_NOTICE "o2hb: Unable to stabilize " + "heartbeat on region %s (%pg)\n", + config_item_name(®->hr_item), + reg_bdev(reg)); + atomic_set(®->hr_steady_iterations, 0); + reg->hr_aborted_start = 1; + wake_up(&o2hb_steady_queue); + ret = -EIO; + } + } + + return ret; +} + +/* + * we ride the region ref that the region dir holds. before the region + * dir is removed and drops it ref it will wait to tear down this + * thread. + */ +static int o2hb_thread(void *data) +{ + int i, ret; + struct o2hb_region *reg = data; + struct o2hb_bio_wait_ctxt write_wc; + ktime_t before_hb, after_hb; + unsigned int elapsed_msec; + + mlog(ML_HEARTBEAT|ML_KTHREAD, "hb thread running\n"); + + set_user_nice(current, MIN_NICE); + + /* Pin node */ + ret = o2nm_depend_node(reg->hr_node_num); + if (ret) { + mlog(ML_ERROR, "Node has been deleted, ret = %d\n", ret); + reg->hr_node_deleted = 1; + wake_up(&o2hb_steady_queue); + return 0; + } + + while (!kthread_should_stop() && + !reg->hr_unclean_stop && !reg->hr_aborted_start && + o2nm_this_node() == reg->hr_node_num) { + /* We track the time spent inside + * o2hb_do_disk_heartbeat so that we avoid more than + * hr_timeout_ms between disk writes. On busy systems + * this should result in a heartbeat which is less + * likely to time itself out. */ + before_hb = ktime_get_real(); + + ret = o2hb_do_disk_heartbeat(reg); + reg->hr_last_hb_status = ret; + + after_hb = ktime_get_real(); + + elapsed_msec = (unsigned int) + ktime_ms_delta(after_hb, before_hb); + + mlog(ML_HEARTBEAT, + "start = %lld, end = %lld, msec = %u, ret = %d\n", + before_hb, after_hb, elapsed_msec, ret); + + if (!kthread_should_stop() && + elapsed_msec < reg->hr_timeout_ms) { + /* the kthread api has blocked signals for us so no + * need to record the return value. */ + msleep_interruptible(reg->hr_timeout_ms - elapsed_msec); + } + } + + o2hb_disarm_timeout(reg); + + /* unclean stop is only used in very bad situation */ + for(i = 0; !reg->hr_unclean_stop && i < reg->hr_blocks; i++) + o2hb_shutdown_slot(®->hr_slots[i]); + + /* Explicit down notification - avoid forcing the other nodes + * to timeout on this region when we could just as easily + * write a clear generation - thus indicating to them that + * this node has left this region. + */ + if (!reg->hr_unclean_stop && !reg->hr_aborted_start) { + o2hb_prepare_block(reg, 0); + ret = o2hb_issue_node_write(reg, &write_wc); + if (ret == 0) { + o2hb_wait_on_io(&write_wc); + bio_uninit(&write_wc.wc_write_bio); + } else { + mlog_errno(ret); + } + } + + /* Unpin node */ + o2nm_undepend_node(reg->hr_node_num); + + mlog(ML_HEARTBEAT|ML_KTHREAD, "o2hb thread exiting\n"); + + return 0; +} + +#ifdef CONFIG_DEBUG_FS +static int o2hb_debug_open(struct inode *inode, struct file *file) +{ + struct o2hb_debug_buf *db = inode->i_private; + struct o2hb_region *reg; + unsigned long map[BITS_TO_LONGS(O2NM_MAX_NODES)]; + unsigned long lts; + char *buf = NULL; + int i = -1; + int out = 0; + + /* max_nodes should be the largest bitmap we pass here */ + BUG_ON(sizeof(map) < db->db_size); + + buf = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!buf) + goto bail; + + switch (db->db_type) { + case O2HB_DB_TYPE_LIVENODES: + case O2HB_DB_TYPE_LIVEREGIONS: + case O2HB_DB_TYPE_QUORUMREGIONS: + case O2HB_DB_TYPE_FAILEDREGIONS: + spin_lock(&o2hb_live_lock); + memcpy(map, db->db_data, db->db_size); + spin_unlock(&o2hb_live_lock); + break; + + case O2HB_DB_TYPE_REGION_LIVENODES: + spin_lock(&o2hb_live_lock); + reg = (struct o2hb_region *)db->db_data; + memcpy(map, reg->hr_live_node_bitmap, db->db_size); + spin_unlock(&o2hb_live_lock); + break; + + case O2HB_DB_TYPE_REGION_NUMBER: + reg = (struct o2hb_region *)db->db_data; + out += scnprintf(buf + out, PAGE_SIZE - out, "%d\n", + reg->hr_region_num); + goto done; + + case O2HB_DB_TYPE_REGION_ELAPSED_TIME: + reg = (struct o2hb_region *)db->db_data; + lts = reg->hr_last_timeout_start; + /* If 0, it has never been set before */ + if (lts) + lts = jiffies_to_msecs(jiffies - lts); + out += scnprintf(buf + out, PAGE_SIZE - out, "%lu\n", lts); + goto done; + + case O2HB_DB_TYPE_REGION_PINNED: + reg = (struct o2hb_region *)db->db_data; + out += scnprintf(buf + out, PAGE_SIZE - out, "%u\n", + !!reg->hr_item_pinned); + goto done; + + default: + goto done; + } + + while ((i = find_next_bit(map, db->db_len, i + 1)) < db->db_len) + out += scnprintf(buf + out, PAGE_SIZE - out, "%d ", i); + out += scnprintf(buf + out, PAGE_SIZE - out, "\n"); + +done: + i_size_write(inode, out); + + file->private_data = buf; + + return 0; +bail: + return -ENOMEM; +} + +static int o2hb_debug_release(struct inode *inode, struct file *file) +{ + kfree(file->private_data); + return 0; +} + +static ssize_t o2hb_debug_read(struct file *file, char __user *buf, + size_t nbytes, loff_t *ppos) +{ + return simple_read_from_buffer(buf, nbytes, ppos, file->private_data, + i_size_read(file->f_mapping->host)); +} +#else +static int o2hb_debug_open(struct inode *inode, struct file *file) +{ + return 0; +} +static int o2hb_debug_release(struct inode *inode, struct file *file) +{ + return 0; +} +static ssize_t o2hb_debug_read(struct file *file, char __user *buf, + size_t nbytes, loff_t *ppos) +{ + return 0; +} +#endif /* CONFIG_DEBUG_FS */ + +static const struct file_operations o2hb_debug_fops = { + .open = o2hb_debug_open, + .release = o2hb_debug_release, + .read = o2hb_debug_read, + .llseek = generic_file_llseek, +}; + +void o2hb_exit(void) +{ + debugfs_remove_recursive(o2hb_debug_dir); + kfree(o2hb_db_livenodes); + kfree(o2hb_db_liveregions); + kfree(o2hb_db_quorumregions); + kfree(o2hb_db_failedregions); +} + +static void o2hb_debug_create(const char *name, struct dentry *dir, + struct o2hb_debug_buf **db, int db_len, int type, + int size, int len, void *data) +{ + *db = kmalloc(db_len, GFP_KERNEL); + if (!*db) + return; + + (*db)->db_type = type; + (*db)->db_size = size; + (*db)->db_len = len; + (*db)->db_data = data; + + debugfs_create_file(name, S_IFREG|S_IRUSR, dir, *db, &o2hb_debug_fops); +} + +static void o2hb_debug_init(void) +{ + o2hb_debug_dir = debugfs_create_dir(O2HB_DEBUG_DIR, NULL); + + o2hb_debug_create(O2HB_DEBUG_LIVENODES, o2hb_debug_dir, + &o2hb_db_livenodes, sizeof(*o2hb_db_livenodes), + O2HB_DB_TYPE_LIVENODES, sizeof(o2hb_live_node_bitmap), + O2NM_MAX_NODES, o2hb_live_node_bitmap); + + o2hb_debug_create(O2HB_DEBUG_LIVEREGIONS, o2hb_debug_dir, + &o2hb_db_liveregions, sizeof(*o2hb_db_liveregions), + O2HB_DB_TYPE_LIVEREGIONS, + sizeof(o2hb_live_region_bitmap), O2NM_MAX_REGIONS, + o2hb_live_region_bitmap); + + o2hb_debug_create(O2HB_DEBUG_QUORUMREGIONS, o2hb_debug_dir, + &o2hb_db_quorumregions, + sizeof(*o2hb_db_quorumregions), + O2HB_DB_TYPE_QUORUMREGIONS, + sizeof(o2hb_quorum_region_bitmap), O2NM_MAX_REGIONS, + o2hb_quorum_region_bitmap); + + o2hb_debug_create(O2HB_DEBUG_FAILEDREGIONS, o2hb_debug_dir, + &o2hb_db_failedregions, + sizeof(*o2hb_db_failedregions), + O2HB_DB_TYPE_FAILEDREGIONS, + sizeof(o2hb_failed_region_bitmap), O2NM_MAX_REGIONS, + o2hb_failed_region_bitmap); +} + +void o2hb_init(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(o2hb_callbacks); i++) + INIT_LIST_HEAD(&o2hb_callbacks[i].list); + + for (i = 0; i < ARRAY_SIZE(o2hb_live_slots); i++) + INIT_LIST_HEAD(&o2hb_live_slots[i]); + + bitmap_zero(o2hb_live_node_bitmap, O2NM_MAX_NODES); + bitmap_zero(o2hb_region_bitmap, O2NM_MAX_REGIONS); + bitmap_zero(o2hb_live_region_bitmap, O2NM_MAX_REGIONS); + bitmap_zero(o2hb_quorum_region_bitmap, O2NM_MAX_REGIONS); + bitmap_zero(o2hb_failed_region_bitmap, O2NM_MAX_REGIONS); + + o2hb_dependent_users = 0; + + o2hb_debug_init(); +} + +static void __o2hb_fill_node_map(unsigned long *map, unsigned int bits) +{ + bitmap_copy(map, o2hb_live_node_bitmap, bits); +} + +void o2hb_callback_read_lock(void) +{ + down_read(&o2hb_callback_sem); +} + +void o2hb_callback_read_unlock(void) +{ + up_read(&o2hb_callback_sem); +} + +void o2hb_synchronize_callbacks(void) +{ + down_write(&o2hb_callback_sem); + up_write(&o2hb_callback_sem); +} + +/* + * Callers must already hold o2hb_callback_sem for read or write so the copy + * stays serialized with callback delivery. + */ +void o2hb_fill_node_map_locked(unsigned long *map, unsigned int bits) +{ + spin_lock(&o2hb_live_lock); + __o2hb_fill_node_map(map, bits); + spin_unlock(&o2hb_live_lock); +} + +/* + * get a map of all nodes that are heartbeating in any regions + */ +void o2hb_fill_node_map(unsigned long *map, unsigned int bits) +{ + /* callers want to serialize this map and callbacks so that they + * can trust that they don't miss nodes coming to the party */ + o2hb_callback_read_lock(); + o2hb_fill_node_map_locked(map, bits); + o2hb_callback_read_unlock(); +} +EXPORT_SYMBOL_GPL(o2hb_fill_node_map); + +/* + * heartbeat configfs bits. The heartbeat set is a default set under + * the cluster set in nodemanager.c. + */ + +static struct o2hb_region *to_o2hb_region(struct config_item *item) +{ + return item ? container_of(item, struct o2hb_region, hr_item) : NULL; +} + +static void o2hb_unmap_slot_data(struct o2hb_region *reg) +{ + int i; + struct page *page; + + if (reg->hr_slot_data) { + for (i = 0; i < reg->hr_num_pages; i++) { + page = reg->hr_slot_data[i]; + if (page) { + __free_page(page); + reg->hr_slot_data[i] = NULL; + } + } + kfree(reg->hr_slot_data); + reg->hr_slot_data = NULL; + } + + kfree(reg->hr_slots); + reg->hr_slots = NULL; + + kfree(reg->hr_tmp_block); + reg->hr_tmp_block = NULL; +} + +/* drop_item only drops its ref after killing the thread, nothing should + * be using the region anymore. this has to clean up any state that + * attributes might have built up. + */ +static void o2hb_region_release(struct config_item *item) +{ + struct o2hb_region *reg = to_o2hb_region(item); + + mlog(ML_HEARTBEAT, "hb region release (%pg)\n", reg_bdev(reg)); + + o2hb_quiesce_timeout(reg); + o2net_unregister_and_flush_handler_list(®->hr_handler_list); + o2hb_unmap_slot_data(reg); + + if (reg->hr_bdev_file) + fput(reg->hr_bdev_file); + + debugfs_remove_recursive(reg->hr_debug_dir); + kfree(reg->hr_db_livenodes); + kfree(reg->hr_db_regnum); + kfree(reg->hr_db_elapsed_time); + kfree(reg->hr_db_pinned); + + spin_lock(&o2hb_live_lock); + list_del(®->hr_all_item); + spin_unlock(&o2hb_live_lock); + + kfree(reg); +} + +static int o2hb_read_block_input(struct o2hb_region *reg, + const char *page, + unsigned long *ret_bytes, + unsigned int *ret_bits) +{ + unsigned long bytes; + char *p = (char *)page; + int ret; + + ret = kstrtoul(p, 0, &bytes); + if (ret) + return ret; + + /* Heartbeat and fs min / max block sizes are the same. */ + if (bytes > 4096 || bytes < 512) + return -ERANGE; + if (hweight16(bytes) != 1) + return -EINVAL; + + if (ret_bytes) + *ret_bytes = bytes; + if (ret_bits) + *ret_bits = ffs(bytes) - 1; + + return 0; +} + +static ssize_t o2hb_region_block_bytes_show(struct config_item *item, + char *page) +{ + return sprintf(page, "%u\n", to_o2hb_region(item)->hr_block_bytes); +} + +static ssize_t o2hb_region_block_bytes_store(struct config_item *item, + const char *page, + size_t count) +{ + struct o2hb_region *reg = to_o2hb_region(item); + int status; + unsigned long block_bytes; + unsigned int block_bits; + + if (reg->hr_bdev_file) + return -EINVAL; + + status = o2hb_read_block_input(reg, page, &block_bytes, + &block_bits); + if (status) + return status; + + reg->hr_block_bytes = (unsigned int)block_bytes; + reg->hr_block_bits = block_bits; + + return count; +} + +static ssize_t o2hb_region_start_block_show(struct config_item *item, + char *page) +{ + return sprintf(page, "%llu\n", to_o2hb_region(item)->hr_start_block); +} + +static ssize_t o2hb_region_start_block_store(struct config_item *item, + const char *page, + size_t count) +{ + struct o2hb_region *reg = to_o2hb_region(item); + unsigned long long tmp; + char *p = (char *)page; + ssize_t ret; + + if (reg->hr_bdev_file) + return -EINVAL; + + ret = kstrtoull(p, 0, &tmp); + if (ret) + return -EINVAL; + + reg->hr_start_block = tmp; + + return count; +} + +static ssize_t o2hb_region_blocks_show(struct config_item *item, char *page) +{ + return sprintf(page, "%d\n", to_o2hb_region(item)->hr_blocks); +} + +static ssize_t o2hb_region_blocks_store(struct config_item *item, + const char *page, + size_t count) +{ + struct o2hb_region *reg = to_o2hb_region(item); + unsigned long tmp; + char *p = (char *)page; + int ret; + + if (reg->hr_bdev_file) + return -EINVAL; + + ret = kstrtoul(p, 0, &tmp); + if (ret) + return ret; + + if (tmp > O2NM_MAX_NODES || tmp == 0) + return -ERANGE; + + reg->hr_blocks = (unsigned int)tmp; + + return count; +} + +static ssize_t o2hb_region_dev_show(struct config_item *item, char *page) +{ + unsigned int ret = 0; + + if (to_o2hb_region(item)->hr_bdev_file) + ret = sprintf(page, "%pg\n", reg_bdev(to_o2hb_region(item))); + + return ret; +} + +static void o2hb_init_region_params(struct o2hb_region *reg) +{ + reg->hr_slots_per_page = PAGE_SIZE >> reg->hr_block_bits; + reg->hr_timeout_ms = O2HB_REGION_TIMEOUT_MS; + + mlog(ML_HEARTBEAT, "hr_start_block = %llu, hr_blocks = %u\n", + reg->hr_start_block, reg->hr_blocks); + mlog(ML_HEARTBEAT, "hr_block_bytes = %u, hr_block_bits = %u\n", + reg->hr_block_bytes, reg->hr_block_bits); + mlog(ML_HEARTBEAT, "hr_timeout_ms = %u\n", reg->hr_timeout_ms); + mlog(ML_HEARTBEAT, "dead threshold = %u\n", o2hb_dead_threshold); +} + +static int o2hb_map_slot_data(struct o2hb_region *reg) +{ + int i, j; + int ret = -ENOMEM; + unsigned int last_slot; + unsigned int spp = reg->hr_slots_per_page; + struct page *page; + char *raw; + struct o2hb_disk_slot *slot; + + reg->hr_tmp_block = kmalloc(reg->hr_block_bytes, GFP_KERNEL); + if (!reg->hr_tmp_block) + goto out; + + reg->hr_slots = kzalloc_objs(struct o2hb_disk_slot, reg->hr_blocks); + if (!reg->hr_slots) + goto out; + + for (i = 0; i < reg->hr_blocks; i++) { + slot = ®->hr_slots[i]; + slot->ds_node_num = i; + INIT_LIST_HEAD(&slot->ds_live_item); + slot->ds_raw_block = NULL; + } + + reg->hr_num_pages = (reg->hr_blocks + spp - 1) / spp; + mlog(ML_HEARTBEAT, "Going to require %u pages to cover %u blocks " + "at %u blocks per page\n", + reg->hr_num_pages, reg->hr_blocks, spp); + + reg->hr_slot_data = kzalloc_objs(struct page *, reg->hr_num_pages); + if (!reg->hr_slot_data) + goto out; + + for (i = 0; i < reg->hr_num_pages; i++) { + page = alloc_page(GFP_KERNEL); + if (!page) + goto out; + + reg->hr_slot_data[i] = page; + + last_slot = i * spp; + raw = page_address(page); + for (j = 0; + (j < spp) && ((j + last_slot) < reg->hr_blocks); + j++) { + BUG_ON((j + last_slot) >= reg->hr_blocks); + + slot = ®->hr_slots[j + last_slot]; + slot->ds_raw_block = + (struct o2hb_disk_heartbeat_block *) raw; + + raw += reg->hr_block_bytes; + } + } + + return 0; + +out: + o2hb_unmap_slot_data(reg); + return ret; +} + +/* Read in all the slots available and populate the tracking + * structures so that we can start with a baseline idea of what's + * there. */ +static int o2hb_populate_slot_data(struct o2hb_region *reg) +{ + int ret, i; + struct o2hb_disk_slot *slot; + struct o2hb_disk_heartbeat_block *hb_block; + + ret = o2hb_read_slots(reg, 0, reg->hr_blocks); + if (ret) + goto out; + + /* We only want to get an idea of the values initially in each + * slot, so we do no verification - o2hb_check_slot will + * actually determine if each configured slot is valid and + * whether any values have changed. */ + for(i = 0; i < reg->hr_blocks; i++) { + slot = ®->hr_slots[i]; + hb_block = (struct o2hb_disk_heartbeat_block *) slot->ds_raw_block; + + /* Only fill the values that o2hb_check_slot uses to + * determine changing slots */ + slot->ds_last_time = le64_to_cpu(hb_block->hb_seq); + slot->ds_last_generation = le64_to_cpu(hb_block->hb_generation); + } + +out: + return ret; +} + +/* + * this is acting as commit; we set up all of hr_bdev_file and hr_task or + * nothing + */ +static ssize_t o2hb_region_dev_store(struct config_item *item, + const char *page, + size_t count) +{ + struct o2hb_region *reg = to_o2hb_region(item); + struct task_struct *hb_task; + long fd; + int sectsize; + char *p = (char *)page; + ssize_t ret = -EINVAL; + int live_threshold; + + if (reg->hr_bdev_file) + return -EINVAL; + + /* We can't heartbeat without having had our node number + * configured yet. */ + reg->hr_node_num = o2nm_this_node(); + if (reg->hr_node_num == O2NM_MAX_NODES) + return -EINVAL; + + ret = kstrtol(p, 0, &fd); + if (ret < 0) + return -EINVAL; + + if (fd < 0 || fd >= INT_MAX) + return -EINVAL; + + CLASS(fd, f)(fd); + if (fd_empty(f)) + return -EINVAL; + + if (reg->hr_blocks == 0 || reg->hr_start_block == 0 || + reg->hr_block_bytes == 0) + return -EINVAL; + + if (!S_ISBLK(fd_file(f)->f_mapping->host->i_mode)) + return -EINVAL; + + reg->hr_bdev_file = bdev_file_open_by_dev(fd_file(f)->f_mapping->host->i_rdev, + BLK_OPEN_WRITE | BLK_OPEN_READ, NULL, NULL); + if (IS_ERR(reg->hr_bdev_file)) { + ret = PTR_ERR(reg->hr_bdev_file); + reg->hr_bdev_file = NULL; + return ret; + } + + sectsize = bdev_logical_block_size(reg_bdev(reg)); + if (sectsize != reg->hr_block_bytes) { + mlog(ML_ERROR, + "blocksize %u incorrect for device, expected %d", + reg->hr_block_bytes, sectsize); + ret = -EINVAL; + goto out; + } + + reg->hr_aborted_start = 0; + reg->hr_node_deleted = 0; + o2hb_init_region_params(reg); + + /* Generation of zero is invalid */ + do { + get_random_bytes(®->hr_generation, + sizeof(reg->hr_generation)); + } while (reg->hr_generation == 0); + + ret = o2hb_map_slot_data(reg); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = o2hb_populate_slot_data(reg); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * A node is considered live after it has beat LIVE_THRESHOLD + * times. We're not steady until we've given them a chance + * _after_ our first read. + * The default threshold is bare minimum so as to limit the delay + * during mounts. For global heartbeat, the threshold doubled for the + * first region. + */ + live_threshold = O2HB_LIVE_THRESHOLD; + if (o2hb_global_heartbeat_active()) { + spin_lock(&o2hb_live_lock); + if (bitmap_weight(o2hb_region_bitmap, O2NM_MAX_REGIONS) == 1) + live_threshold <<= 1; + spin_unlock(&o2hb_live_lock); + } + ++live_threshold; + atomic_set(®->hr_steady_iterations, live_threshold); + /* unsteady_iterations is triple the steady_iterations */ + atomic_set(®->hr_unsteady_iterations, (live_threshold * 3)); + o2hb_set_region_stopping(reg, false); + + hb_task = kthread_run(o2hb_thread, reg, "o2hb-%s", + reg->hr_item.ci_name); + if (IS_ERR(hb_task)) { + ret = PTR_ERR(hb_task); + mlog_errno(ret); + goto out; + } + + spin_lock(&o2hb_live_lock); + reg->hr_task = hb_task; + spin_unlock(&o2hb_live_lock); + + ret = wait_event_interruptible(o2hb_steady_queue, + atomic_read(®->hr_steady_iterations) == 0 || + reg->hr_node_deleted); + if (ret) { + atomic_set(®->hr_steady_iterations, 0); + reg->hr_aborted_start = 1; + } + + if (reg->hr_aborted_start) { + ret = -EIO; + goto out; + } + + if (reg->hr_node_deleted) { + ret = -EINVAL; + goto out; + } + + /* Ok, we were woken. Make sure it wasn't by drop_item() */ + spin_lock(&o2hb_live_lock); + hb_task = reg->hr_task; + if (o2hb_global_heartbeat_active()) + set_bit(reg->hr_region_num, o2hb_live_region_bitmap); + spin_unlock(&o2hb_live_lock); + + if (hb_task) + ret = count; + else + ret = -EIO; + + if (hb_task && o2hb_global_heartbeat_active()) + printk(KERN_NOTICE "o2hb: Heartbeat started on region %s (%pg)\n", + config_item_name(®->hr_item), reg_bdev(reg)); + +out: + if (ret < 0) { + o2hb_quiesce_timeout(reg); + + spin_lock(&o2hb_live_lock); + hb_task = reg->hr_task; + reg->hr_task = NULL; + spin_unlock(&o2hb_live_lock); + + if (hb_task) + kthread_stop(hb_task); + + o2hb_unmap_slot_data(reg); + + fput(reg->hr_bdev_file); + reg->hr_bdev_file = NULL; + } + return ret; +} + +static ssize_t o2hb_region_pid_show(struct config_item *item, char *page) +{ + struct o2hb_region *reg = to_o2hb_region(item); + pid_t pid = 0; + + spin_lock(&o2hb_live_lock); + if (reg->hr_task) + pid = task_pid_nr(reg->hr_task); + spin_unlock(&o2hb_live_lock); + + if (!pid) + return 0; + + return sprintf(page, "%u\n", pid); +} + +CONFIGFS_ATTR(o2hb_region_, block_bytes); +CONFIGFS_ATTR(o2hb_region_, start_block); +CONFIGFS_ATTR(o2hb_region_, blocks); +CONFIGFS_ATTR(o2hb_region_, dev); +CONFIGFS_ATTR_RO(o2hb_region_, pid); + +static struct configfs_attribute *o2hb_region_attrs[] = { + &o2hb_region_attr_block_bytes, + &o2hb_region_attr_start_block, + &o2hb_region_attr_blocks, + &o2hb_region_attr_dev, + &o2hb_region_attr_pid, + NULL, +}; + +static const struct configfs_item_operations o2hb_region_item_ops = { + .release = o2hb_region_release, +}; + +static const struct config_item_type o2hb_region_type = { + .ct_item_ops = &o2hb_region_item_ops, + .ct_attrs = o2hb_region_attrs, + .ct_owner = THIS_MODULE, +}; + +/* heartbeat set */ + +struct o2hb_heartbeat_group { + struct config_group hs_group; + /* some stuff? */ +}; + +static struct o2hb_heartbeat_group *to_o2hb_heartbeat_group(struct config_group *group) +{ + return group ? + container_of(group, struct o2hb_heartbeat_group, hs_group) + : NULL; +} + +static void o2hb_debug_region_init(struct o2hb_region *reg, + struct dentry *parent) +{ + struct dentry *dir; + + dir = debugfs_create_dir(config_item_name(®->hr_item), parent); + reg->hr_debug_dir = dir; + + o2hb_debug_create(O2HB_DEBUG_LIVENODES, dir, &(reg->hr_db_livenodes), + sizeof(*(reg->hr_db_livenodes)), + O2HB_DB_TYPE_REGION_LIVENODES, + sizeof(reg->hr_live_node_bitmap), O2NM_MAX_NODES, + reg); + + o2hb_debug_create(O2HB_DEBUG_REGION_NUMBER, dir, &(reg->hr_db_regnum), + sizeof(*(reg->hr_db_regnum)), + O2HB_DB_TYPE_REGION_NUMBER, 0, O2NM_MAX_NODES, reg); + + o2hb_debug_create(O2HB_DEBUG_REGION_ELAPSED_TIME, dir, + &(reg->hr_db_elapsed_time), + sizeof(*(reg->hr_db_elapsed_time)), + O2HB_DB_TYPE_REGION_ELAPSED_TIME, 0, 0, reg); + + o2hb_debug_create(O2HB_DEBUG_REGION_PINNED, dir, &(reg->hr_db_pinned), + sizeof(*(reg->hr_db_pinned)), + O2HB_DB_TYPE_REGION_PINNED, 0, 0, reg); + +} + +static struct config_item *o2hb_heartbeat_group_make_item(struct config_group *group, + const char *name) +{ + struct o2hb_region *reg = NULL; + int ret; + + reg = kzalloc_obj(struct o2hb_region); + if (reg == NULL) + return ERR_PTR(-ENOMEM); + + if (strlen(name) > O2HB_MAX_REGION_NAME_LEN) { + ret = -ENAMETOOLONG; + goto free; + } + reg->hr_node_num = O2NM_MAX_NODES; + + spin_lock(&o2hb_live_lock); + reg->hr_region_num = 0; + if (o2hb_global_heartbeat_active()) { + reg->hr_region_num = find_first_zero_bit(o2hb_region_bitmap, + O2NM_MAX_REGIONS); + if (reg->hr_region_num >= O2NM_MAX_REGIONS) { + spin_unlock(&o2hb_live_lock); + ret = -EFBIG; + goto free; + } + set_bit(reg->hr_region_num, o2hb_region_bitmap); + } + list_add_tail(®->hr_all_item, &o2hb_all_regions); + spin_unlock(&o2hb_live_lock); + + config_item_init_type_name(®->hr_item, name, &o2hb_region_type); + + /* this is the same way to generate msg key as dlm, for local heartbeat, + * name is also the same, so make initial crc value different to avoid + * message key conflict. + */ + reg->hr_key = crc32_le(reg->hr_region_num + O2NM_MAX_REGIONS, + name, strlen(name)); + mutex_init(®->hr_arming_mutex); + reg->hr_stopping = true; + INIT_DELAYED_WORK(®->hr_write_timeout_work, o2hb_write_timeout); + INIT_DELAYED_WORK(®->hr_nego_timeout_work, o2hb_nego_timeout); + INIT_LIST_HEAD(®->hr_handler_list); + ret = o2net_register_handler(O2HB_NEGO_TIMEOUT_MSG, reg->hr_key, + sizeof(struct o2hb_nego_msg), + o2hb_nego_timeout_handler, + reg, NULL, ®->hr_handler_list); + if (ret) + goto remove_item; + + ret = o2net_register_handler(O2HB_NEGO_APPROVE_MSG, reg->hr_key, + sizeof(struct o2hb_nego_msg), + o2hb_nego_approve_handler, + reg, NULL, ®->hr_handler_list); + if (ret) + goto unregister_handler; + + o2hb_debug_region_init(reg, o2hb_debug_dir); + + return ®->hr_item; + +unregister_handler: + o2net_unregister_and_flush_handler_list(®->hr_handler_list); +remove_item: + spin_lock(&o2hb_live_lock); + list_del(®->hr_all_item); + if (o2hb_global_heartbeat_active()) + clear_bit(reg->hr_region_num, o2hb_region_bitmap); + spin_unlock(&o2hb_live_lock); +free: + kfree(reg); + return ERR_PTR(ret); +} + +static void o2hb_heartbeat_group_drop_item(struct config_group *group, + struct config_item *item) +{ + struct task_struct *hb_task; + struct o2hb_region *reg = to_o2hb_region(item); + int quorum_region = 0; + + o2hb_quiesce_timeout(reg); + + /* stop the thread when the user removes the region dir */ + spin_lock(&o2hb_live_lock); + hb_task = reg->hr_task; + reg->hr_task = NULL; + reg->hr_item_dropped = 1; + spin_unlock(&o2hb_live_lock); + + if (hb_task) + kthread_stop(hb_task); + + if (o2hb_global_heartbeat_active()) { + spin_lock(&o2hb_live_lock); + clear_bit(reg->hr_region_num, o2hb_region_bitmap); + clear_bit(reg->hr_region_num, o2hb_live_region_bitmap); + if (test_bit(reg->hr_region_num, o2hb_quorum_region_bitmap)) + quorum_region = 1; + clear_bit(reg->hr_region_num, o2hb_quorum_region_bitmap); + spin_unlock(&o2hb_live_lock); + printk(KERN_NOTICE "o2hb: Heartbeat %s on region %s (%pg)\n", + ((atomic_read(®->hr_steady_iterations) == 0) ? + "stopped" : "start aborted"), config_item_name(item), + reg_bdev(reg)); + } + + /* + * If we're racing a dev_write(), we need to wake them. They will + * check reg->hr_task + */ + if (atomic_read(®->hr_steady_iterations) != 0) { + reg->hr_aborted_start = 1; + atomic_set(®->hr_steady_iterations, 0); + wake_up(&o2hb_steady_queue); + } + + config_item_put(item); + + if (!o2hb_global_heartbeat_active() || !quorum_region) + return; + + /* + * If global heartbeat active and there are dependent users, + * pin all regions if quorum region count <= CUT_OFF + */ + mutex_lock(&o2hb_dependency_mutex); + spin_lock(&o2hb_live_lock); + + if (!o2hb_dependent_users) + goto unlock; + + if (bitmap_weight(o2hb_quorum_region_bitmap, + O2NM_MAX_REGIONS) <= O2HB_PIN_CUT_OFF) + o2hb_region_pin(NULL, true); + +unlock: + spin_unlock(&o2hb_live_lock); + mutex_unlock(&o2hb_dependency_mutex); +} + +static ssize_t o2hb_heartbeat_group_dead_threshold_show(struct config_item *item, + char *page) +{ + return sprintf(page, "%u\n", o2hb_dead_threshold); +} + +static ssize_t o2hb_heartbeat_group_dead_threshold_store(struct config_item *item, + const char *page, size_t count) +{ + unsigned long tmp; + char *p = (char *)page; + int ret; + + ret = kstrtoul(p, 10, &tmp); + if (ret) + return ret; + + /* this will validate ranges for us. */ + o2hb_dead_threshold_set((unsigned int) tmp); + + return count; +} + +static ssize_t o2hb_heartbeat_group_mode_show(struct config_item *item, + char *page) +{ + return sprintf(page, "%s\n", + o2hb_heartbeat_mode_desc[o2hb_heartbeat_mode]); +} + +static ssize_t o2hb_heartbeat_group_mode_store(struct config_item *item, + const char *page, size_t count) +{ + unsigned int i; + int ret; + size_t len; + + len = (page[count - 1] == '\n') ? count - 1 : count; + if (!len) + return -EINVAL; + + for (i = 0; i < O2HB_HEARTBEAT_NUM_MODES; ++i) { + if (strncasecmp(page, o2hb_heartbeat_mode_desc[i], len)) + continue; + + ret = o2hb_global_heartbeat_mode_set(i); + if (!ret) + printk(KERN_NOTICE "o2hb: Heartbeat mode set to %s\n", + o2hb_heartbeat_mode_desc[i]); + return count; + } + + return -EINVAL; + +} + +CONFIGFS_ATTR(o2hb_heartbeat_group_, dead_threshold); +CONFIGFS_ATTR(o2hb_heartbeat_group_, mode); + +static struct configfs_attribute *o2hb_heartbeat_group_attrs[] = { + &o2hb_heartbeat_group_attr_dead_threshold, + &o2hb_heartbeat_group_attr_mode, + NULL, +}; + +static const struct configfs_group_operations o2hb_heartbeat_group_group_ops = { + .make_item = o2hb_heartbeat_group_make_item, + .drop_item = o2hb_heartbeat_group_drop_item, +}; + +static const struct config_item_type o2hb_heartbeat_group_type = { + .ct_group_ops = &o2hb_heartbeat_group_group_ops, + .ct_attrs = o2hb_heartbeat_group_attrs, + .ct_owner = THIS_MODULE, +}; + +/* this is just here to avoid touching group in heartbeat.h which the + * entire damn world #includes */ +struct config_group *o2hb_alloc_hb_set(void) +{ + struct o2hb_heartbeat_group *hs = NULL; + struct config_group *ret = NULL; + + hs = kzalloc_obj(struct o2hb_heartbeat_group); + if (hs == NULL) + goto out; + + config_group_init_type_name(&hs->hs_group, "heartbeat", + &o2hb_heartbeat_group_type); + + ret = &hs->hs_group; +out: + if (ret == NULL) + kfree(hs); + return ret; +} + +void o2hb_free_hb_set(struct config_group *group) +{ + struct o2hb_heartbeat_group *hs = to_o2hb_heartbeat_group(group); + kfree(hs); +} + +/* hb callback registration and issuing */ + +static struct o2hb_callback *hbcall_from_type(enum o2hb_callback_type type) +{ + if (type == O2HB_NUM_CB) + return ERR_PTR(-EINVAL); + + return &o2hb_callbacks[type]; +} + +void o2hb_setup_callback(struct o2hb_callback_func *hc, + enum o2hb_callback_type type, + o2hb_cb_func *func, + void *data, + int priority) +{ + INIT_LIST_HEAD(&hc->hc_item); + hc->hc_func = func; + hc->hc_data = data; + hc->hc_priority = priority; + hc->hc_type = type; + hc->hc_magic = O2HB_CB_MAGIC; +} +EXPORT_SYMBOL_GPL(o2hb_setup_callback); + +/* + * In local heartbeat mode, region_uuid passed matches the dlm domain name. + * In global heartbeat mode, region_uuid passed is NULL. + * + * In local, we only pin the matching region. In global we pin all the active + * regions. + */ +static int o2hb_region_pin(const char *region_uuid, bool from_callback) +{ + int ret = 0, found; + struct o2hb_region *reg, *pinned; + char *uuid; + + assert_spin_locked(&o2hb_live_lock); + + do { + found = 0; + pinned = NULL; + + list_for_each_entry(reg, &o2hb_all_regions, hr_all_item) { + if (reg->hr_item_dropped) + continue; + + uuid = config_item_name(®->hr_item); + + /* local heartbeat */ + if (region_uuid) { + if (strcmp(region_uuid, uuid)) + continue; + found = 1; + } + + if (reg->hr_item_pinned || reg->hr_item_dropped) { + if (found) + break; + continue; + } + + /* + * Found a region that needs pinning. Take a reference + * so it stays alive while we drop the lock below. + */ + pinned = reg; + config_item_get(®->hr_item); + break; + } + + if (!pinned) + break; + + uuid = config_item_name(&pinned->hr_item); + + /* + * o2nm_depend_item() -> configfs_depend_item() can sleep (it + * takes the configfs root inode rwsem), so it must not run + * under o2hb_live_lock. Drop the lock across it; @pinned is + * kept alive by the reference taken above. The region list may + * change while unlocked, so we rescan from the top afterwards. + */ + spin_unlock(&o2hb_live_lock); + + /* Ignore ENOENT only for local hb (userdlm domain) */ + if (from_callback) + ret = o2nm_depend_item_unlocked(&pinned->hr_item); + else + ret = o2nm_depend_item(&pinned->hr_item); + + spin_lock(&o2hb_live_lock); + if (!ret) { + /* + * o2hb_live_lock was dropped across o2nm_depend_item(). + * o2hb_set_quorum_device() runs in the heartbeat thread + * without o2hb_dependency_mutex, so for global heartbeat + * it may have crossed O2HB_PIN_CUT_OFF and unpinned the + * regions while we slept. If that happened this pin is + * no longer wanted; undo it and stop rather than + * resurrecting it on the rescan below. + */ + if (!region_uuid && + bitmap_weight(o2hb_quorum_region_bitmap, + O2NM_MAX_REGIONS) > O2HB_PIN_CUT_OFF) { + o2nm_undepend_item(&pinned->hr_item); + spin_unlock(&o2hb_live_lock); + config_item_put(&pinned->hr_item); + spin_lock(&o2hb_live_lock); + break; + } + mlog(ML_CLUSTER, "Pin region %s\n", uuid); + pinned->hr_item_pinned = 1; + } else if (ret == -ENOENT && (found || !region_uuid)) { + /* + * For local hb (found): ignore ENOENT from userdlm + * domains as before. For global hb (!region_uuid): + * the region may have been detached from configfs + * while the lock was dropped — skip it and continue + * pinning the remaining regions. + */ + ret = 0; + } else { + mlog(ML_ERROR, "Pin region %s fails with %d\n", + uuid, ret); + } + + /* + * config_item_put() may drop the last reference and run + * o2hb_region_release(), which also grabs o2hb_live_lock and + * can sleep, so it must happen with the lock released. + */ + spin_unlock(&o2hb_live_lock); + config_item_put(&pinned->hr_item); + spin_lock(&o2hb_live_lock); + + /* local hb pins a single matching region */ + } while (!ret && !region_uuid); + + return ret; +} + +/* + * In local heartbeat mode, region_uuid passed matches the dlm domain name. + * In global heartbeat mode, region_uuid passed is NULL. + * + * In local, we only unpin the matching region. In global we unpin all the + * active regions. + */ +static void o2hb_region_unpin(const char *region_uuid) +{ + struct o2hb_region *reg; + char *uuid; + int found = 0; + + assert_spin_locked(&o2hb_live_lock); + + list_for_each_entry(reg, &o2hb_all_regions, hr_all_item) { + if (reg->hr_item_dropped) + continue; + + uuid = config_item_name(®->hr_item); + if (region_uuid) { + if (strcmp(region_uuid, uuid)) + continue; + found = 1; + } + + if (reg->hr_item_pinned) { + mlog(ML_CLUSTER, "Unpin region %s\n", uuid); + o2nm_undepend_item(®->hr_item); + reg->hr_item_pinned = 0; + } + if (found) + break; + } +} + +static int o2hb_region_inc_user(const char *region_uuid) +{ + int ret = 0; + + mutex_lock(&o2hb_dependency_mutex); + spin_lock(&o2hb_live_lock); + + /* local heartbeat */ + if (!o2hb_global_heartbeat_active()) { + ret = o2hb_region_pin(region_uuid, false); + goto unlock; + } + + /* + * if global heartbeat active and this is the first dependent user, + * pin all regions if quorum region count <= CUT_OFF + */ + o2hb_dependent_users++; + if (o2hb_dependent_users > 1) + goto unlock; + + if (bitmap_weight(o2hb_quorum_region_bitmap, + O2NM_MAX_REGIONS) <= O2HB_PIN_CUT_OFF) { + ret = o2hb_region_pin(NULL, false); + if (ret) { + o2hb_region_unpin(NULL); + o2hb_dependent_users--; + } + } + +unlock: + spin_unlock(&o2hb_live_lock); + mutex_unlock(&o2hb_dependency_mutex); + return ret; +} + +static void o2hb_region_dec_user(const char *region_uuid) +{ + mutex_lock(&o2hb_dependency_mutex); + spin_lock(&o2hb_live_lock); + + /* local heartbeat */ + if (!o2hb_global_heartbeat_active()) { + o2hb_region_unpin(region_uuid); + goto unlock; + } + + /* + * if global heartbeat active and there are no dependent users, + * unpin all quorum regions + */ + o2hb_dependent_users--; + if (!o2hb_dependent_users) + o2hb_region_unpin(NULL); + +unlock: + spin_unlock(&o2hb_live_lock); + mutex_unlock(&o2hb_dependency_mutex); +} + +int o2hb_register_callback(const char *region_uuid, + struct o2hb_callback_func *hc) +{ + struct o2hb_callback_func *f; + struct o2hb_callback *hbcall; + int ret; + + BUG_ON(hc->hc_magic != O2HB_CB_MAGIC); + BUG_ON(!list_empty(&hc->hc_item)); + + hbcall = hbcall_from_type(hc->hc_type); + if (IS_ERR(hbcall)) { + ret = PTR_ERR(hbcall); + goto out; + } + + if (region_uuid) { + ret = o2hb_region_inc_user(region_uuid); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + down_write(&o2hb_callback_sem); + + list_for_each_entry(f, &hbcall->list, hc_item) { + if (hc->hc_priority < f->hc_priority) { + list_add_tail(&hc->hc_item, &f->hc_item); + break; + } + } + if (list_empty(&hc->hc_item)) + list_add_tail(&hc->hc_item, &hbcall->list); + + up_write(&o2hb_callback_sem); + ret = 0; +out: + mlog(ML_CLUSTER, "returning %d on behalf of %p for funcs %p\n", + ret, __builtin_return_address(0), hc); + return ret; +} +EXPORT_SYMBOL_GPL(o2hb_register_callback); + +void o2hb_unregister_callback(const char *region_uuid, + struct o2hb_callback_func *hc) +{ + BUG_ON(hc->hc_magic != O2HB_CB_MAGIC); + + mlog(ML_CLUSTER, "on behalf of %p for funcs %p\n", + __builtin_return_address(0), hc); + + /* XXX Can this happen _with_ a region reference? */ + if (list_empty(&hc->hc_item)) + return; + + if (region_uuid) + o2hb_region_dec_user(region_uuid); + + down_write(&o2hb_callback_sem); + + list_del_init(&hc->hc_item); + + up_write(&o2hb_callback_sem); +} +EXPORT_SYMBOL_GPL(o2hb_unregister_callback); + +int o2hb_check_node_heartbeating_no_sem(u8 node_num) +{ + unsigned long testing_map[BITS_TO_LONGS(O2NM_MAX_NODES)]; + + spin_lock(&o2hb_live_lock); + __o2hb_fill_node_map(testing_map, O2NM_MAX_NODES); + spin_unlock(&o2hb_live_lock); + if (!test_bit(node_num, testing_map)) { + mlog(ML_HEARTBEAT, + "node (%u) does not have heartbeating enabled.\n", + node_num); + return 0; + } + + return 1; +} +EXPORT_SYMBOL_GPL(o2hb_check_node_heartbeating_no_sem); + +int o2hb_check_node_heartbeating_from_callback(u8 node_num) +{ + unsigned long testing_map[BITS_TO_LONGS(O2NM_MAX_NODES)]; + + o2hb_fill_node_map_locked(testing_map, O2NM_MAX_NODES); + if (!test_bit(node_num, testing_map)) { + mlog(ML_HEARTBEAT, + "node (%u) does not have heartbeating enabled.\n", + node_num); + return 0; + } + + return 1; +} +EXPORT_SYMBOL_GPL(o2hb_check_node_heartbeating_from_callback); + +/* + * this is just a hack until we get the plumbing which flips file systems + * read only and drops the hb ref instead of killing the node dead. + */ +void o2hb_stop_all_regions(void) +{ + struct o2hb_region *reg; + + mlog(ML_ERROR, "stopping heartbeat on all active regions.\n"); + + spin_lock(&o2hb_live_lock); + + list_for_each_entry(reg, &o2hb_all_regions, hr_all_item) + reg->hr_unclean_stop = 1; + + spin_unlock(&o2hb_live_lock); +} +EXPORT_SYMBOL_GPL(o2hb_stop_all_regions); + +int o2hb_get_all_regions(char *region_uuids, u8 max_regions) +{ + struct o2hb_region *reg; + int numregs = 0; + char *p; + + spin_lock(&o2hb_live_lock); + + p = region_uuids; + list_for_each_entry(reg, &o2hb_all_regions, hr_all_item) { + if (reg->hr_item_dropped) + continue; + + mlog(0, "Region: %s\n", config_item_name(®->hr_item)); + if (numregs < max_regions) { + memcpy(p, config_item_name(®->hr_item), + O2HB_MAX_REGION_NAME_LEN); + p += O2HB_MAX_REGION_NAME_LEN; + } + numregs++; + } + + spin_unlock(&o2hb_live_lock); + + return numregs; +} +EXPORT_SYMBOL_GPL(o2hb_get_all_regions); + +int o2hb_global_heartbeat_active(void) +{ + return (o2hb_heartbeat_mode == O2HB_HEARTBEAT_GLOBAL); +} +EXPORT_SYMBOL(o2hb_global_heartbeat_active); diff --git a/fs/ocfs2/cluster/heartbeat.h b/fs/ocfs2/cluster/heartbeat.h new file mode 100644 index 000000000..2ca2b6575 --- /dev/null +++ b/fs/ocfs2/cluster/heartbeat.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * heartbeat.h + * + * Function prototypes + * + * Copyright (C) 2004 Oracle. All rights reserved. + */ + +#ifndef O2CLUSTER_HEARTBEAT_H +#define O2CLUSTER_HEARTBEAT_H + +#include "ocfs2_heartbeat.h" + +#define O2HB_REGION_TIMEOUT_MS 2000 + +#define O2HB_MAX_REGION_NAME_LEN 32 + +/* number of changes to be seen as live */ +#define O2HB_LIVE_THRESHOLD 2 +/* number of equal samples to be seen as dead */ +extern unsigned int o2hb_dead_threshold; +#define O2HB_DEFAULT_DEAD_THRESHOLD 31 +/* Otherwise MAX_WRITE_TIMEOUT will be zero... */ +#define O2HB_MIN_DEAD_THRESHOLD 2 +#define O2HB_MAX_WRITE_TIMEOUT_MS (O2HB_REGION_TIMEOUT_MS * (o2hb_dead_threshold - 1)) + +#define O2HB_CB_MAGIC 0x51d1e4ec + +/* callback stuff */ +enum o2hb_callback_type { + O2HB_NODE_DOWN_CB = 0, + O2HB_NODE_UP_CB, + O2HB_NUM_CB +}; + +struct o2nm_node; +typedef void (o2hb_cb_func)(struct o2nm_node *, int, void *); + +struct o2hb_callback_func { + u32 hc_magic; + struct list_head hc_item; + o2hb_cb_func *hc_func; + void *hc_data; + int hc_priority; + enum o2hb_callback_type hc_type; +}; + +struct config_group *o2hb_alloc_hb_set(void); +void o2hb_free_hb_set(struct config_group *group); + +void o2hb_setup_callback(struct o2hb_callback_func *hc, + enum o2hb_callback_type type, + o2hb_cb_func *func, + void *data, + int priority); +int o2hb_register_callback(const char *region_uuid, + struct o2hb_callback_func *hc); +void o2hb_unregister_callback(const char *region_uuid, + struct o2hb_callback_func *hc); +void o2hb_callback_read_lock(void); +void o2hb_callback_read_unlock(void); +void o2hb_synchronize_callbacks(void); +void o2hb_fill_node_map_locked(unsigned long *map, + unsigned int bits); +void o2hb_fill_node_map(unsigned long *map, + unsigned int bits); +void o2hb_exit(void); +void o2hb_init(void); +int o2hb_check_node_heartbeating_no_sem(u8 node_num); +int o2hb_check_node_heartbeating_from_callback(u8 node_num); +void o2hb_stop_all_regions(void); +int o2hb_get_all_regions(char *region_uuids, u8 numregions); +int o2hb_global_heartbeat_active(void); + +#endif /* O2CLUSTER_HEARTBEAT_H */ diff --git a/fs/ocfs2/cluster/masklog.c b/fs/ocfs2/cluster/masklog.c new file mode 100644 index 000000000..563881ddb --- /dev/null +++ b/fs/ocfs2/cluster/masklog.c @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2004, 2005 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include + +#include "masklog.h" + +struct mlog_bits mlog_and_bits = MLOG_BITS_RHS(MLOG_INITIAL_AND_MASK); +EXPORT_SYMBOL_GPL(mlog_and_bits); +struct mlog_bits mlog_not_bits = MLOG_BITS_RHS(0); +EXPORT_SYMBOL_GPL(mlog_not_bits); + +static ssize_t mlog_mask_show(u64 mask, char *buf) +{ + char *state; + + if (__mlog_test_u64(mask, mlog_and_bits)) + state = "allow"; + else if (__mlog_test_u64(mask, mlog_not_bits)) + state = "deny"; + else + state = "off"; + + return snprintf(buf, PAGE_SIZE, "%s\n", state); +} + +static ssize_t mlog_mask_store(u64 mask, const char *buf, size_t count) +{ + if (!strncasecmp(buf, "allow", 5)) { + __mlog_set_u64(mask, mlog_and_bits); + __mlog_clear_u64(mask, mlog_not_bits); + } else if (!strncasecmp(buf, "deny", 4)) { + __mlog_set_u64(mask, mlog_not_bits); + __mlog_clear_u64(mask, mlog_and_bits); + } else if (!strncasecmp(buf, "off", 3)) { + __mlog_clear_u64(mask, mlog_not_bits); + __mlog_clear_u64(mask, mlog_and_bits); + } else + return -EINVAL; + + return count; +} + +void __mlog_printk(const u64 *mask, const char *func, int line, + const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + const char *level; + const char *prefix = ""; + + if (!__mlog_test_u64(*mask, mlog_and_bits) || + __mlog_test_u64(*mask, mlog_not_bits)) + return; + + if (*mask & ML_ERROR) { + level = KERN_ERR; + prefix = "ERROR: "; + } else if (*mask & ML_NOTICE) { + level = KERN_NOTICE; + } else { + level = KERN_INFO; + } + + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + printk("%s(%s,%u,%u):%s:%d %s%pV", + level, current->comm, task_pid_nr(current), + raw_smp_processor_id(), func, line, prefix, &vaf); + + va_end(args); +} +EXPORT_SYMBOL_GPL(__mlog_printk); + +struct mlog_attribute { + struct attribute attr; + u64 mask; +}; + +#define to_mlog_attr(_attr) container_of(_attr, struct mlog_attribute, attr) + +#define define_mask(_name) { \ + .attr = { \ + .name = #_name, \ + .mode = S_IRUGO | S_IWUSR, \ + }, \ + .mask = ML_##_name, \ +} + +static struct mlog_attribute mlog_attrs[MLOG_MAX_BITS] = { + define_mask(TCP), + define_mask(MSG), + define_mask(SOCKET), + define_mask(HEARTBEAT), + define_mask(HB_BIO), + define_mask(DLMFS), + define_mask(DLM), + define_mask(DLM_DOMAIN), + define_mask(DLM_THREAD), + define_mask(DLM_MASTER), + define_mask(DLM_RECOVERY), + define_mask(DLM_GLUE), + define_mask(VOTE), + define_mask(CONN), + define_mask(QUORUM), + define_mask(BASTS), + define_mask(CLUSTER), + define_mask(ERROR), + define_mask(NOTICE), + define_mask(KTHREAD), +}; + +static struct attribute *mlog_default_attrs[MLOG_MAX_BITS] = {NULL, }; +ATTRIBUTE_GROUPS(mlog_default); + +static ssize_t mlog_show(struct kobject *obj, struct attribute *attr, + char *buf) +{ + struct mlog_attribute *mlog_attr = to_mlog_attr(attr); + + return mlog_mask_show(mlog_attr->mask, buf); +} + +static ssize_t mlog_store(struct kobject *obj, struct attribute *attr, + const char *buf, size_t count) +{ + struct mlog_attribute *mlog_attr = to_mlog_attr(attr); + + return mlog_mask_store(mlog_attr->mask, buf, count); +} + +static const struct sysfs_ops mlog_attr_ops = { + .show = mlog_show, + .store = mlog_store, +}; + +static struct kobj_type mlog_ktype = { + .default_groups = mlog_default_groups, + .sysfs_ops = &mlog_attr_ops, +}; + +static struct kset mlog_kset = { + .kobj = {.ktype = &mlog_ktype}, +}; + +int mlog_sys_init(struct kset *o2cb_kset) +{ + int i = 0; + + while (mlog_attrs[i].attr.mode) { + mlog_default_attrs[i] = &mlog_attrs[i].attr; + i++; + } + mlog_default_attrs[i] = NULL; + + kobject_set_name(&mlog_kset.kobj, "logmask"); + mlog_kset.kobj.kset = o2cb_kset; + return kset_register(&mlog_kset); +} + +void mlog_sys_shutdown(void) +{ + kset_unregister(&mlog_kset); +} diff --git a/fs/ocfs2/cluster/masklog.h b/fs/ocfs2/cluster/masklog.h new file mode 100644 index 000000000..630bd5a3d --- /dev/null +++ b/fs/ocfs2/cluster/masklog.h @@ -0,0 +1,196 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2005 Oracle. All rights reserved. + */ + +#ifndef O2CLUSTER_MASKLOG_H +#define O2CLUSTER_MASKLOG_H + +/* + * For now this is a trivial wrapper around printk() that gives the critical + * ability to enable sets of debugging output at run-time. In the future this + * will almost certainly be redirected to relayfs so that it can pay a + * substantially lower heisenberg tax. + * + * Callers associate the message with a bitmask and a global bitmask is + * maintained with help from /proc. If any of the bits match the message is + * output. + * + * We must have efficient bit tests on i386 and it seems gcc still emits crazy + * code for the 64bit compare. It emits very good code for the dual unsigned + * long tests, though, completely avoiding tests that can never pass if the + * caller gives a constant bitmask that fills one of the longs with all 0s. So + * the desire is to have almost all of the calls decided on by comparing just + * one of the longs. This leads to having infrequently given bits that are + * frequently matched in the high bits. + * + * _ERROR and _NOTICE are used for messages that always go to the console and + * have appropriate KERN_ prefixes. We wrap these in our function instead of + * just calling printk() so that this can eventually make its way through + * relayfs along with the debugging messages. Everything else gets KERN_DEBUG. + * The inline tests and macro dance give GCC the opportunity to quite cleverly + * only emit the appropriate printk() when the caller passes in a constant + * mask, as is almost always the case. + * + * All this bitmask nonsense is managed from the files under + * /sys/fs/o2cb/logmask/. Reading the files gives a straightforward + * indication of which bits are allowed (allow) or denied (off/deny). + * ENTRY deny + * EXIT deny + * TCP off + * MSG off + * SOCKET off + * ERROR allow + * NOTICE allow + * + * Writing changes the state of a given bit and requires a strictly formatted + * single write() call: + * + * write(fd, "allow", 5); + * + * Echoing allow/deny/off string into the logmask files can flip the bits + * on or off as expected; here is the bash script for example: + * + * log_mask="/sys/fs/o2cb/log_mask" + * for node in ENTRY EXIT TCP MSG SOCKET ERROR NOTICE; do + * echo allow >"$log_mask"/"$node" + * done + * + * The debugfs.ocfs2 tool can also flip the bits with the -l option: + * + * debugfs.ocfs2 -l TCP allow + */ + +/* for task_struct */ +#include + +/* bits that are frequently given and infrequently matched in the low word */ +/* NOTE: If you add a flag, you need to also update masklog.c! */ +#define ML_TCP 0x0000000000000001ULL /* net cluster/tcp.c */ +#define ML_MSG 0x0000000000000002ULL /* net network messages */ +#define ML_SOCKET 0x0000000000000004ULL /* net socket lifetime */ +#define ML_HEARTBEAT 0x0000000000000008ULL /* hb all heartbeat tracking */ +#define ML_HB_BIO 0x0000000000000010ULL /* hb io tracing */ +#define ML_DLMFS 0x0000000000000020ULL /* dlm user dlmfs */ +#define ML_DLM 0x0000000000000040ULL /* dlm general debugging */ +#define ML_DLM_DOMAIN 0x0000000000000080ULL /* dlm domain debugging */ +#define ML_DLM_THREAD 0x0000000000000100ULL /* dlm domain thread */ +#define ML_DLM_MASTER 0x0000000000000200ULL /* dlm master functions */ +#define ML_DLM_RECOVERY 0x0000000000000400ULL /* dlm master functions */ +#define ML_DLM_GLUE 0x0000000000000800ULL /* ocfs2 dlm glue layer */ +#define ML_VOTE 0x0000000000001000ULL /* ocfs2 node messaging */ +#define ML_CONN 0x0000000000002000ULL /* net connection management */ +#define ML_QUORUM 0x0000000000004000ULL /* net connection quorum */ +#define ML_BASTS 0x0000000000008000ULL /* dlmglue asts and basts */ +#define ML_CLUSTER 0x0000000000010000ULL /* cluster stack */ + +/* bits that are infrequently given and frequently matched in the high word */ +#define ML_ERROR 0x1000000000000000ULL /* sent to KERN_ERR */ +#define ML_NOTICE 0x2000000000000000ULL /* setn to KERN_NOTICE */ +#define ML_KTHREAD 0x4000000000000000ULL /* kernel thread activity */ + +#define MLOG_INITIAL_AND_MASK (ML_ERROR|ML_NOTICE) +#ifndef MLOG_MASK_PREFIX +#define MLOG_MASK_PREFIX 0 +#endif + +/* + * When logging is disabled, force the bit test to 0 for anything other + * than errors and notices, allowing gcc to remove the code completely. + * When enabled, allow all masks. + */ +#if defined(CONFIG_OCFS2_DEBUG_MASKLOG) +#define ML_ALLOWED_BITS ~0 +#else +#define ML_ALLOWED_BITS (ML_ERROR|ML_NOTICE) +#endif + +#define MLOG_MAX_BITS 64 + +struct mlog_bits { + unsigned long words[MLOG_MAX_BITS / BITS_PER_LONG]; +}; + +extern struct mlog_bits mlog_and_bits, mlog_not_bits; + +#if BITS_PER_LONG == 32 + +#define __mlog_test_u64(mask, bits) \ + ( (u32)(mask & 0xffffffff) & bits.words[0] || \ + ((u64)(mask) >> 32) & bits.words[1] ) +#define __mlog_set_u64(mask, bits) do { \ + bits.words[0] |= (u32)(mask & 0xffffffff); \ + bits.words[1] |= (u64)(mask) >> 32; \ +} while (0) +#define __mlog_clear_u64(mask, bits) do { \ + bits.words[0] &= ~((u32)(mask & 0xffffffff)); \ + bits.words[1] &= ~((u64)(mask) >> 32); \ +} while (0) +#define MLOG_BITS_RHS(mask) { \ + { \ + [0] = (u32)(mask & 0xffffffff), \ + [1] = (u64)(mask) >> 32, \ + } \ +} + +#else /* 32bit long above, 64bit long below */ + +#define __mlog_test_u64(mask, bits) ((mask) & bits.words[0]) +#define __mlog_set_u64(mask, bits) do { \ + bits.words[0] |= (mask); \ +} while (0) +#define __mlog_clear_u64(mask, bits) do { \ + bits.words[0] &= ~(mask); \ +} while (0) +#define MLOG_BITS_RHS(mask) { { (mask) } } + +#endif + +__printf(4, 5) +void __mlog_printk(const u64 *m, const char *func, int line, + const char *fmt, ...); + +/* + * Testing before the __mlog_printk call lets the compiler eliminate the + * call completely when (m & ML_ALLOWED_BITS) is 0. + */ +#define mlog(mask, fmt, ...) \ +do { \ + u64 _m = MLOG_MASK_PREFIX | (mask); \ + if (_m & ML_ALLOWED_BITS) \ + __mlog_printk(&_m, __func__, __LINE__, fmt, \ + ##__VA_ARGS__); \ +} while (0) + +#define mlog_ratelimited(mask, fmt, ...) \ +do { \ + static DEFINE_RATELIMIT_STATE(_rs, \ + DEFAULT_RATELIMIT_INTERVAL, \ + DEFAULT_RATELIMIT_BURST); \ + if (__ratelimit(&_rs)) \ + mlog(mask, fmt, ##__VA_ARGS__); \ +} while (0) + +#define mlog_errno(st) ({ \ + int _st = (st); \ + if (_st != -ERESTARTSYS && _st != -EINTR && \ + _st != AOP_TRUNCATED_PAGE && _st != -ENOSPC && \ + _st != -EDQUOT) \ + mlog(ML_ERROR, "status = %lld\n", (long long)_st); \ + _st; \ +}) + +#define mlog_bug_on_msg(cond, fmt, args...) do { \ + if (cond) { \ + mlog(ML_ERROR, "bug expression: " #cond "\n"); \ + mlog(ML_ERROR, fmt, ##args); \ + BUG(); \ + } \ +} while (0) + +#include +#include +int mlog_sys_init(struct kset *o2cb_subsys); +void mlog_sys_shutdown(void); + +#endif /* O2CLUSTER_MASKLOG_H */ diff --git a/fs/ocfs2/cluster/netdebug.c b/fs/ocfs2/cluster/netdebug.c new file mode 100644 index 000000000..f7ab66e70 --- /dev/null +++ b/fs/ocfs2/cluster/netdebug.c @@ -0,0 +1,506 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * netdebug.c + * + * debug functionality for o2net + * + * Copyright (C) 2005, 2008 Oracle. All rights reserved. + */ + +#ifdef CONFIG_DEBUG_FS + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "tcp.h" +#include "nodemanager.h" +#define MLOG_MASK_PREFIX ML_TCP +#include "masklog.h" + +#include "tcp_internal.h" + +#define O2NET_DEBUG_DIR "o2net" +#define SC_DEBUG_NAME "sock_containers" +#define NST_DEBUG_NAME "send_tracking" +#define STATS_DEBUG_NAME "stats" +#define NODES_DEBUG_NAME "connected_nodes" + +#define SHOW_SOCK_CONTAINERS 0 +#define SHOW_SOCK_STATS 1 + +static struct dentry *o2net_dentry; + +static DEFINE_SPINLOCK(o2net_debug_lock); + +static LIST_HEAD(sock_containers); +static LIST_HEAD(send_tracking); + +void o2net_debug_add_nst(struct o2net_send_tracking *nst) +{ + spin_lock_bh(&o2net_debug_lock); + list_add(&nst->st_net_debug_item, &send_tracking); + spin_unlock_bh(&o2net_debug_lock); +} + +void o2net_debug_del_nst(struct o2net_send_tracking *nst) +{ + spin_lock_bh(&o2net_debug_lock); + if (!list_empty(&nst->st_net_debug_item)) + list_del_init(&nst->st_net_debug_item); + spin_unlock_bh(&o2net_debug_lock); +} + +static struct o2net_send_tracking + *next_nst(struct o2net_send_tracking *nst_start) +{ + struct o2net_send_tracking *nst, *ret = NULL; + + assert_spin_locked(&o2net_debug_lock); + + list_for_each_entry(nst, &nst_start->st_net_debug_item, + st_net_debug_item) { + /* discover the head of the list */ + if (&nst->st_net_debug_item == &send_tracking) + break; + + /* use st_task to detect real nsts in the list */ + if (nst->st_task != NULL) { + ret = nst; + break; + } + } + + return ret; +} + +static void *nst_seq_start(struct seq_file *seq, loff_t *pos) +{ + struct o2net_send_tracking *nst, *dummy_nst = seq->private; + + spin_lock_bh(&o2net_debug_lock); + nst = next_nst(dummy_nst); + spin_unlock_bh(&o2net_debug_lock); + + return nst; +} + +static void *nst_seq_next(struct seq_file *seq, void *v, loff_t *pos) +{ + struct o2net_send_tracking *nst, *dummy_nst = seq->private; + + spin_lock_bh(&o2net_debug_lock); + nst = next_nst(dummy_nst); + list_del_init(&dummy_nst->st_net_debug_item); + if (nst) + list_add(&dummy_nst->st_net_debug_item, + &nst->st_net_debug_item); + spin_unlock_bh(&o2net_debug_lock); + + return nst; /* unused, just needs to be null when done */ +} + +static int nst_seq_show(struct seq_file *seq, void *v) +{ + struct o2net_send_tracking *nst, *dummy_nst = seq->private; + ktime_t now; + s64 sock, send, status; + + spin_lock_bh(&o2net_debug_lock); + nst = next_nst(dummy_nst); + if (!nst) + goto out; + + now = ktime_get(); + sock = ktime_to_us(ktime_sub(now, nst->st_sock_time)); + send = ktime_to_us(ktime_sub(now, nst->st_send_time)); + status = ktime_to_us(ktime_sub(now, nst->st_status_time)); + + /* get_task_comm isn't exported. oh well. */ + seq_printf(seq, "%p:\n" + " pid: %lu\n" + " tgid: %lu\n" + " process name: %s\n" + " node: %u\n" + " sc: %p\n" + " message id: %d\n" + " message type: %u\n" + " message key: 0x%08x\n" + " sock acquiry: %lld usecs ago\n" + " send start: %lld usecs ago\n" + " wait start: %lld usecs ago\n", + nst, (unsigned long)task_pid_nr(nst->st_task), + (unsigned long)nst->st_task->tgid, + nst->st_task->comm, nst->st_node, + nst->st_sc, nst->st_id, nst->st_msg_type, + nst->st_msg_key, + (long long)sock, + (long long)send, + (long long)status); + +out: + spin_unlock_bh(&o2net_debug_lock); + + return 0; +} + +static void nst_seq_stop(struct seq_file *seq, void *v) +{ +} + +static const struct seq_operations nst_seq_ops = { + .start = nst_seq_start, + .next = nst_seq_next, + .stop = nst_seq_stop, + .show = nst_seq_show, +}; + +static int nst_fop_open(struct inode *inode, struct file *file) +{ + struct o2net_send_tracking *dummy_nst; + + dummy_nst = __seq_open_private(file, &nst_seq_ops, sizeof(*dummy_nst)); + if (!dummy_nst) + return -ENOMEM; + o2net_debug_add_nst(dummy_nst); + + return 0; +} + +static int nst_fop_release(struct inode *inode, struct file *file) +{ + struct seq_file *seq = file->private_data; + struct o2net_send_tracking *dummy_nst = seq->private; + + o2net_debug_del_nst(dummy_nst); + return seq_release_private(inode, file); +} + +static const struct file_operations nst_seq_fops = { + .open = nst_fop_open, + .read = seq_read, + .llseek = seq_lseek, + .release = nst_fop_release, +}; + +void o2net_debug_add_sc(struct o2net_sock_container *sc) +{ + spin_lock_bh(&o2net_debug_lock); + list_add(&sc->sc_net_debug_item, &sock_containers); + spin_unlock_bh(&o2net_debug_lock); +} + +void o2net_debug_del_sc(struct o2net_sock_container *sc) +{ + spin_lock_bh(&o2net_debug_lock); + list_del_init(&sc->sc_net_debug_item); + spin_unlock_bh(&o2net_debug_lock); +} + +struct o2net_sock_debug { + int dbg_ctxt; + struct o2net_sock_container *dbg_sock; +}; + +static struct o2net_sock_container + *next_sc(struct o2net_sock_container *sc_start) +{ + struct o2net_sock_container *sc, *ret = NULL; + + assert_spin_locked(&o2net_debug_lock); + + list_for_each_entry(sc, &sc_start->sc_net_debug_item, + sc_net_debug_item) { + /* discover the head of the list miscast as a sc */ + if (&sc->sc_net_debug_item == &sock_containers) + break; + + /* use sc_page to detect real scs in the list */ + if (sc->sc_page != NULL) { + ret = sc; + break; + } + } + + return ret; +} + +static void *sc_seq_start(struct seq_file *seq, loff_t *pos) +{ + struct o2net_sock_debug *sd = seq->private; + struct o2net_sock_container *sc, *dummy_sc = sd->dbg_sock; + + spin_lock_bh(&o2net_debug_lock); + sc = next_sc(dummy_sc); + spin_unlock_bh(&o2net_debug_lock); + + return sc; +} + +static void *sc_seq_next(struct seq_file *seq, void *v, loff_t *pos) +{ + struct o2net_sock_debug *sd = seq->private; + struct o2net_sock_container *sc, *dummy_sc = sd->dbg_sock; + + spin_lock_bh(&o2net_debug_lock); + sc = next_sc(dummy_sc); + list_del_init(&dummy_sc->sc_net_debug_item); + if (sc) + list_add(&dummy_sc->sc_net_debug_item, &sc->sc_net_debug_item); + spin_unlock_bh(&o2net_debug_lock); + + return sc; /* unused, just needs to be null when done */ +} + +#ifdef CONFIG_OCFS2_FS_STATS +# define sc_send_count(_s) ((_s)->sc_send_count) +# define sc_recv_count(_s) ((_s)->sc_recv_count) +# define sc_tv_acquiry_total_ns(_s) (ktime_to_ns((_s)->sc_tv_acquiry_total)) +# define sc_tv_send_total_ns(_s) (ktime_to_ns((_s)->sc_tv_send_total)) +# define sc_tv_status_total_ns(_s) (ktime_to_ns((_s)->sc_tv_status_total)) +# define sc_tv_process_total_ns(_s) (ktime_to_ns((_s)->sc_tv_process_total)) +#else +# define sc_send_count(_s) (0U) +# define sc_recv_count(_s) (0U) +# define sc_tv_acquiry_total_ns(_s) (0LL) +# define sc_tv_send_total_ns(_s) (0LL) +# define sc_tv_status_total_ns(_s) (0LL) +# define sc_tv_process_total_ns(_s) (0LL) +#endif + +/* So that debugfs.ocfs2 can determine which format is being used */ +#define O2NET_STATS_STR_VERSION 1 +static void sc_show_sock_stats(struct seq_file *seq, + struct o2net_sock_container *sc) +{ + if (!sc) + return; + + seq_printf(seq, "%d,%u,%lu,%lld,%lld,%lld,%lu,%lld\n", O2NET_STATS_STR_VERSION, + sc->sc_node->nd_num, (unsigned long)sc_send_count(sc), + (long long)sc_tv_acquiry_total_ns(sc), + (long long)sc_tv_send_total_ns(sc), + (long long)sc_tv_status_total_ns(sc), + (unsigned long)sc_recv_count(sc), + (long long)sc_tv_process_total_ns(sc)); +} + +static void sc_show_sock_container(struct seq_file *seq, + struct o2net_sock_container *sc) +{ + struct inet_sock *inet = NULL; + __be32 saddr = 0, daddr = 0; + __be16 sport = 0, dport = 0; + + if (!sc) + return; + + if (sc->sc_sock) { + inet = inet_sk(sc->sc_sock->sk); + /* the stack's structs aren't sparse endian clean */ + saddr = (__force __be32)inet->inet_saddr; + daddr = (__force __be32)inet->inet_daddr; + sport = (__force __be16)inet->inet_sport; + dport = (__force __be16)inet->inet_dport; + } + + /* XXX sigh, inet-> doesn't have sparse annotation so any + * use of it here generates a warning with -Wbitwise */ + seq_printf(seq, "%p:\n" + " krefs: %d\n" + " sock: %pI4:%u -> " + "%pI4:%u\n" + " remote node: %s\n" + " page off: %zu\n" + " handshake ok: %u\n" + " timer: %lld usecs\n" + " data ready: %lld usecs\n" + " advance start: %lld usecs\n" + " advance stop: %lld usecs\n" + " func start: %lld usecs\n" + " func stop: %lld usecs\n" + " func key: 0x%08x\n" + " func type: %u\n", + sc, + kref_read(&sc->sc_kref), + &saddr, inet ? ntohs(sport) : 0, + &daddr, inet ? ntohs(dport) : 0, + sc->sc_node->nd_name, + sc->sc_page_off, + sc->sc_handshake_ok, + (long long)ktime_to_us(sc->sc_tv_timer), + (long long)ktime_to_us(sc->sc_tv_data_ready), + (long long)ktime_to_us(sc->sc_tv_advance_start), + (long long)ktime_to_us(sc->sc_tv_advance_stop), + (long long)ktime_to_us(sc->sc_tv_func_start), + (long long)ktime_to_us(sc->sc_tv_func_stop), + sc->sc_msg_key, + sc->sc_msg_type); +} + +static int sc_seq_show(struct seq_file *seq, void *v) +{ + struct o2net_sock_debug *sd = seq->private; + struct o2net_sock_container *sc, *dummy_sc = sd->dbg_sock; + + spin_lock_bh(&o2net_debug_lock); + sc = next_sc(dummy_sc); + + if (sc) { + if (sd->dbg_ctxt == SHOW_SOCK_CONTAINERS) + sc_show_sock_container(seq, sc); + else + sc_show_sock_stats(seq, sc); + } + + spin_unlock_bh(&o2net_debug_lock); + + return 0; +} + +static void sc_seq_stop(struct seq_file *seq, void *v) +{ +} + +static const struct seq_operations sc_seq_ops = { + .start = sc_seq_start, + .next = sc_seq_next, + .stop = sc_seq_stop, + .show = sc_seq_show, +}; + +static int sc_common_open(struct file *file, int ctxt) +{ + struct o2net_sock_debug *sd; + struct o2net_sock_container *dummy_sc; + + dummy_sc = kzalloc_obj(*dummy_sc); + if (!dummy_sc) + return -ENOMEM; + + sd = __seq_open_private(file, &sc_seq_ops, sizeof(*sd)); + if (!sd) { + kfree(dummy_sc); + return -ENOMEM; + } + + sd->dbg_ctxt = ctxt; + sd->dbg_sock = dummy_sc; + + o2net_debug_add_sc(dummy_sc); + + return 0; +} + +static int sc_fop_release(struct inode *inode, struct file *file) +{ + struct seq_file *seq = file->private_data; + struct o2net_sock_debug *sd = seq->private; + struct o2net_sock_container *dummy_sc = sd->dbg_sock; + + o2net_debug_del_sc(dummy_sc); + kfree(dummy_sc); + return seq_release_private(inode, file); +} + +static int stats_fop_open(struct inode *inode, struct file *file) +{ + return sc_common_open(file, SHOW_SOCK_STATS); +} + +static const struct file_operations stats_seq_fops = { + .open = stats_fop_open, + .read = seq_read, + .llseek = seq_lseek, + .release = sc_fop_release, +}; + +static int sc_fop_open(struct inode *inode, struct file *file) +{ + return sc_common_open(file, SHOW_SOCK_CONTAINERS); +} + +static const struct file_operations sc_seq_fops = { + .open = sc_fop_open, + .read = seq_read, + .llseek = seq_lseek, + .release = sc_fop_release, +}; + +static int o2net_fill_bitmap(char *buf, int len) +{ + unsigned long map[BITS_TO_LONGS(O2NM_MAX_NODES)]; + int i = -1, out = 0; + + o2net_fill_node_map(map, O2NM_MAX_NODES); + + while ((i = find_next_bit(map, O2NM_MAX_NODES, i + 1)) < O2NM_MAX_NODES) + out += scnprintf(buf + out, PAGE_SIZE - out, "%d ", i); + out += scnprintf(buf + out, PAGE_SIZE - out, "\n"); + + return out; +} + +static int nodes_fop_open(struct inode *inode, struct file *file) +{ + char *buf; + + buf = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + i_size_write(inode, o2net_fill_bitmap(buf, PAGE_SIZE)); + + file->private_data = buf; + + return 0; +} + +static int o2net_debug_release(struct inode *inode, struct file *file) +{ + kfree(file->private_data); + return 0; +} + +static ssize_t o2net_debug_read(struct file *file, char __user *buf, + size_t nbytes, loff_t *ppos) +{ + return simple_read_from_buffer(buf, nbytes, ppos, file->private_data, + i_size_read(file->f_mapping->host)); +} + +static const struct file_operations nodes_fops = { + .open = nodes_fop_open, + .release = o2net_debug_release, + .read = o2net_debug_read, + .llseek = generic_file_llseek, +}; + +void o2net_debugfs_exit(void) +{ + debugfs_remove_recursive(o2net_dentry); +} + +void o2net_debugfs_init(void) +{ + umode_t mode = S_IFREG|S_IRUSR; + + o2net_dentry = debugfs_create_dir(O2NET_DEBUG_DIR, NULL); + + debugfs_create_file(NST_DEBUG_NAME, mode, o2net_dentry, NULL, + &nst_seq_fops); + debugfs_create_file(SC_DEBUG_NAME, mode, o2net_dentry, NULL, + &sc_seq_fops); + debugfs_create_file(STATS_DEBUG_NAME, mode, o2net_dentry, NULL, + &stats_seq_fops); + debugfs_create_file(NODES_DEBUG_NAME, mode, o2net_dentry, NULL, + &nodes_fops); +} + +#endif /* CONFIG_DEBUG_FS */ diff --git a/fs/ocfs2/cluster/nodemanager.c b/fs/ocfs2/cluster/nodemanager.c new file mode 100644 index 000000000..e08850a5d --- /dev/null +++ b/fs/ocfs2/cluster/nodemanager.c @@ -0,0 +1,889 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2004, 2005 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include + +#include "tcp.h" +#include "nodemanager.h" +#include "heartbeat.h" +#include "masklog.h" +#include "sys.h" + +/* for now we operate under the assertion that there can be only one + * cluster active at a time. Changing this will require trickling + * cluster references throughout where nodes are looked up */ +struct o2nm_cluster *o2nm_single_cluster = NULL; + +static const char *o2nm_fence_method_desc[O2NM_FENCE_METHODS] = { + "reset", /* O2NM_FENCE_RESET */ + "panic", /* O2NM_FENCE_PANIC */ +}; + +static inline void o2nm_lock_subsystem(void); +static inline void o2nm_unlock_subsystem(void); + +struct o2nm_node *o2nm_get_node_by_num(u8 node_num) +{ + struct o2nm_node *node = NULL; + + if (node_num >= O2NM_MAX_NODES || o2nm_single_cluster == NULL) + goto out; + + read_lock(&o2nm_single_cluster->cl_nodes_lock); + node = o2nm_single_cluster->cl_nodes[node_num]; + if (node) + config_item_get(&node->nd_item); + read_unlock(&o2nm_single_cluster->cl_nodes_lock); +out: + return node; +} +EXPORT_SYMBOL_GPL(o2nm_get_node_by_num); + +int o2nm_configured_node_map(unsigned long *map, unsigned bytes) +{ + struct o2nm_cluster *cluster = o2nm_single_cluster; + + BUG_ON(bytes < (sizeof(cluster->cl_nodes_bitmap))); + + if (cluster == NULL) + return -EINVAL; + + read_lock(&cluster->cl_nodes_lock); + bitmap_copy(map, cluster->cl_nodes_bitmap, O2NM_MAX_NODES); + read_unlock(&cluster->cl_nodes_lock); + + return 0; +} +EXPORT_SYMBOL_GPL(o2nm_configured_node_map); + +static struct o2nm_node *o2nm_node_ip_tree_lookup(struct o2nm_cluster *cluster, + __be32 ip_needle, + struct rb_node ***ret_p, + struct rb_node **ret_parent) +{ + struct rb_node **p = &cluster->cl_node_ip_tree.rb_node; + struct rb_node *parent = NULL; + struct o2nm_node *node, *ret = NULL; + + while (*p) { + int cmp; + + parent = *p; + node = rb_entry(parent, struct o2nm_node, nd_ip_node); + + cmp = memcmp(&ip_needle, &node->nd_ipv4_address, + sizeof(ip_needle)); + if (cmp < 0) + p = &(*p)->rb_left; + else if (cmp > 0) + p = &(*p)->rb_right; + else { + ret = node; + break; + } + } + + if (ret_p != NULL) + *ret_p = p; + if (ret_parent != NULL) + *ret_parent = parent; + + return ret; +} + +struct o2nm_node *o2nm_get_node_by_ip(__be32 addr) +{ + struct o2nm_node *node = NULL; + struct o2nm_cluster *cluster = o2nm_single_cluster; + + if (cluster == NULL) + goto out; + + read_lock(&cluster->cl_nodes_lock); + node = o2nm_node_ip_tree_lookup(cluster, addr, NULL, NULL); + if (node) + config_item_get(&node->nd_item); + read_unlock(&cluster->cl_nodes_lock); + +out: + return node; +} +EXPORT_SYMBOL_GPL(o2nm_get_node_by_ip); + +void o2nm_node_put(struct o2nm_node *node) +{ + config_item_put(&node->nd_item); +} +EXPORT_SYMBOL_GPL(o2nm_node_put); + +void o2nm_node_get(struct o2nm_node *node) +{ + config_item_get(&node->nd_item); +} +EXPORT_SYMBOL_GPL(o2nm_node_get); + +u8 o2nm_this_node(void) +{ + u8 node_num = O2NM_MAX_NODES; + + if (o2nm_single_cluster && o2nm_single_cluster->cl_has_local) + node_num = o2nm_single_cluster->cl_local_node; + + return node_num; +} +EXPORT_SYMBOL_GPL(o2nm_this_node); + +/* node configfs bits */ + +static struct o2nm_cluster *to_o2nm_cluster(struct config_item *item) +{ + return item ? + container_of(to_config_group(item), struct o2nm_cluster, + cl_group) + : NULL; +} + +static struct o2nm_node *to_o2nm_node(struct config_item *item) +{ + return item ? container_of(item, struct o2nm_node, nd_item) : NULL; +} + +static void o2nm_node_release(struct config_item *item) +{ + struct o2nm_node *node = to_o2nm_node(item); + kfree(node); +} + +static ssize_t o2nm_node_num_show(struct config_item *item, char *page) +{ + return sprintf(page, "%d\n", to_o2nm_node(item)->nd_num); +} + +static struct o2nm_cluster *to_o2nm_cluster_from_node(struct o2nm_node *node) +{ + /* through the first node_set .parent + * mycluster/nodes/mynode == o2nm_cluster->o2nm_node_group->o2nm_node */ + if (node->nd_item.ci_parent) + return to_o2nm_cluster(node->nd_item.ci_parent->ci_parent); + else + return NULL; +} + +enum { + O2NM_NODE_ATTR_NUM = 0, + O2NM_NODE_ATTR_PORT, + O2NM_NODE_ATTR_ADDRESS, +}; + +static ssize_t o2nm_node_num_store(struct config_item *item, const char *page, + size_t count) +{ + struct o2nm_node *node = to_o2nm_node(item); + struct o2nm_cluster *cluster; + unsigned long tmp; + char *p = (char *)page; + int ret = 0; + + tmp = simple_strtoul(p, &p, 0); + if (!p || (*p && (*p != '\n'))) + return -EINVAL; + + if (tmp >= O2NM_MAX_NODES) + return -ERANGE; + + /* once we're in the cl_nodes tree networking can look us up by + * node number and try to use our address and port attributes + * to connect to this node.. make sure that they've been set + * before writing the node attribute? */ + if (!test_bit(O2NM_NODE_ATTR_ADDRESS, &node->nd_set_attributes) || + !test_bit(O2NM_NODE_ATTR_PORT, &node->nd_set_attributes)) + return -EINVAL; /* XXX */ + + o2nm_lock_subsystem(); + cluster = to_o2nm_cluster_from_node(node); + if (!cluster) { + o2nm_unlock_subsystem(); + return -EINVAL; + } + + write_lock(&cluster->cl_nodes_lock); + if (cluster->cl_nodes[tmp]) + ret = -EEXIST; + else if (test_and_set_bit(O2NM_NODE_ATTR_NUM, + &node->nd_set_attributes)) + ret = -EBUSY; + else { + cluster->cl_nodes[tmp] = node; + node->nd_num = tmp; + set_bit(tmp, cluster->cl_nodes_bitmap); + } + write_unlock(&cluster->cl_nodes_lock); + o2nm_unlock_subsystem(); + + if (ret) + return ret; + + return count; +} +static ssize_t o2nm_node_ipv4_port_show(struct config_item *item, char *page) +{ + return sprintf(page, "%u\n", ntohs(to_o2nm_node(item)->nd_ipv4_port)); +} + +static ssize_t o2nm_node_ipv4_port_store(struct config_item *item, + const char *page, size_t count) +{ + struct o2nm_node *node = to_o2nm_node(item); + unsigned long tmp; + char *p = (char *)page; + + tmp = simple_strtoul(p, &p, 0); + if (!p || (*p && (*p != '\n'))) + return -EINVAL; + + if (tmp == 0) + return -EINVAL; + if (tmp >= (u16)-1) + return -ERANGE; + + if (test_and_set_bit(O2NM_NODE_ATTR_PORT, &node->nd_set_attributes)) + return -EBUSY; + node->nd_ipv4_port = htons(tmp); + + return count; +} + +static ssize_t o2nm_node_ipv4_address_show(struct config_item *item, char *page) +{ + return sprintf(page, "%pI4\n", &to_o2nm_node(item)->nd_ipv4_address); +} + +static ssize_t o2nm_node_ipv4_address_store(struct config_item *item, + const char *page, + size_t count) +{ + struct o2nm_node *node = to_o2nm_node(item); + struct o2nm_cluster *cluster; + int ret, i; + struct rb_node **p, *parent; + unsigned int octets[4]; + __be32 ipv4_addr = 0; + + ret = sscanf(page, "%3u.%3u.%3u.%3u", &octets[3], &octets[2], + &octets[1], &octets[0]); + if (ret != 4) + return -EINVAL; + + for (i = 0; i < ARRAY_SIZE(octets); i++) { + if (octets[i] > 255) + return -ERANGE; + be32_add_cpu(&ipv4_addr, octets[i] << (i * 8)); + } + + o2nm_lock_subsystem(); + cluster = to_o2nm_cluster_from_node(node); + if (!cluster) { + o2nm_unlock_subsystem(); + return -EINVAL; + } + + ret = 0; + write_lock(&cluster->cl_nodes_lock); + if (o2nm_node_ip_tree_lookup(cluster, ipv4_addr, &p, &parent)) + ret = -EEXIST; + else if (test_and_set_bit(O2NM_NODE_ATTR_ADDRESS, + &node->nd_set_attributes)) + ret = -EBUSY; + else { + rb_link_node(&node->nd_ip_node, parent, p); + rb_insert_color(&node->nd_ip_node, &cluster->cl_node_ip_tree); + } + write_unlock(&cluster->cl_nodes_lock); + o2nm_unlock_subsystem(); + + if (ret) + return ret; + + memcpy(&node->nd_ipv4_address, &ipv4_addr, sizeof(ipv4_addr)); + + return count; +} + +static ssize_t o2nm_node_local_show(struct config_item *item, char *page) +{ + return sprintf(page, "%d\n", to_o2nm_node(item)->nd_local); +} + +static ssize_t o2nm_node_local_store(struct config_item *item, const char *page, + size_t count) +{ + struct o2nm_node *node = to_o2nm_node(item); + struct o2nm_cluster *cluster; + unsigned long tmp; + bool starting = false; + char *p = (char *)page; + ssize_t ret; + + tmp = simple_strtoul(p, &p, 0); + if (!p || (*p && (*p != '\n'))) + return -EINVAL; + + tmp = !!tmp; /* boolean of whether this node wants to be local */ + + /* setting local turns on networking rx for now so we require having + * set everything else first */ + if (!test_bit(O2NM_NODE_ATTR_ADDRESS, &node->nd_set_attributes) || + !test_bit(O2NM_NODE_ATTR_NUM, &node->nd_set_attributes) || + !test_bit(O2NM_NODE_ATTR_PORT, &node->nd_set_attributes)) + return -EINVAL; /* XXX */ + + o2nm_lock_subsystem(); + cluster = to_o2nm_cluster_from_node(node); + if (!cluster) { + ret = -EINVAL; + goto out; + } + + /* the only failure case is trying to set a new local node + * when a different one is already set */ + if (tmp && tmp == cluster->cl_has_local && + cluster->cl_local_node != node->nd_num) { + ret = -EBUSY; + goto out; + } + + /* bring up the rx thread if we're setting the new local node. */ + if (tmp && !cluster->cl_has_local) { + ret = o2net_start_listening(node); + if (ret) + goto out; + starting = true; + } + + if (!tmp && cluster->cl_has_local && + cluster->cl_local_node == node->nd_num) { + o2net_stop_listening(node); + cluster->cl_has_local = 0; + cluster->cl_local_node = O2NM_INVALID_NODE_NUM; + } + + node->nd_local = tmp; + if (node->nd_local) { + cluster->cl_has_local = tmp; + cluster->cl_local_node = node->nd_num; + if (starting) + o2net_complete_start_listening(node); + } + + ret = count; + +out: + o2nm_unlock_subsystem(); + return ret; +} + +CONFIGFS_ATTR(o2nm_node_, num); +CONFIGFS_ATTR(o2nm_node_, ipv4_port); +CONFIGFS_ATTR(o2nm_node_, ipv4_address); +CONFIGFS_ATTR(o2nm_node_, local); + +static struct configfs_attribute *o2nm_node_attrs[] = { + &o2nm_node_attr_num, + &o2nm_node_attr_ipv4_port, + &o2nm_node_attr_ipv4_address, + &o2nm_node_attr_local, + NULL, +}; + +static const struct configfs_item_operations o2nm_node_item_ops = { + .release = o2nm_node_release, +}; + +static const struct config_item_type o2nm_node_type = { + .ct_item_ops = &o2nm_node_item_ops, + .ct_attrs = o2nm_node_attrs, + .ct_owner = THIS_MODULE, +}; + +/* node set */ + +struct o2nm_node_group { + struct config_group ns_group; + /* some stuff? */ +}; + +#if 0 +static struct o2nm_node_group *to_o2nm_node_group(struct config_group *group) +{ + return group ? + container_of(group, struct o2nm_node_group, ns_group) + : NULL; +} +#endif + +static ssize_t o2nm_cluster_attr_write(const char *page, ssize_t count, + unsigned int *val) +{ + unsigned long tmp; + char *p = (char *)page; + + tmp = simple_strtoul(p, &p, 0); + if (!p || (*p && (*p != '\n'))) + return -EINVAL; + + if (tmp == 0) + return -EINVAL; + if (tmp >= (u32)-1) + return -ERANGE; + + *val = tmp; + + return count; +} + +static ssize_t o2nm_cluster_idle_timeout_ms_show(struct config_item *item, + char *page) +{ + return sprintf(page, "%u\n", to_o2nm_cluster(item)->cl_idle_timeout_ms); +} + +static ssize_t o2nm_cluster_idle_timeout_ms_store(struct config_item *item, + const char *page, size_t count) +{ + struct o2nm_cluster *cluster = to_o2nm_cluster(item); + ssize_t ret; + unsigned int val; + + ret = o2nm_cluster_attr_write(page, count, &val); + + if (ret > 0) { + if (cluster->cl_idle_timeout_ms != val + && o2net_num_connected_peers()) { + mlog(ML_NOTICE, + "o2net: cannot change idle timeout after " + "the first peer has agreed to it." + " %d connected peers\n", + o2net_num_connected_peers()); + ret = -EINVAL; + } else if (val <= cluster->cl_keepalive_delay_ms) { + mlog(ML_NOTICE, "o2net: idle timeout must be larger " + "than keepalive delay\n"); + ret = -EINVAL; + } else { + cluster->cl_idle_timeout_ms = val; + } + } + + return ret; +} + +static ssize_t o2nm_cluster_keepalive_delay_ms_show( + struct config_item *item, char *page) +{ + return sprintf(page, "%u\n", + to_o2nm_cluster(item)->cl_keepalive_delay_ms); +} + +static ssize_t o2nm_cluster_keepalive_delay_ms_store( + struct config_item *item, const char *page, size_t count) +{ + struct o2nm_cluster *cluster = to_o2nm_cluster(item); + ssize_t ret; + unsigned int val; + + ret = o2nm_cluster_attr_write(page, count, &val); + + if (ret > 0) { + if (cluster->cl_keepalive_delay_ms != val + && o2net_num_connected_peers()) { + mlog(ML_NOTICE, + "o2net: cannot change keepalive delay after" + " the first peer has agreed to it." + " %d connected peers\n", + o2net_num_connected_peers()); + ret = -EINVAL; + } else if (val >= cluster->cl_idle_timeout_ms) { + mlog(ML_NOTICE, "o2net: keepalive delay must be " + "smaller than idle timeout\n"); + ret = -EINVAL; + } else { + cluster->cl_keepalive_delay_ms = val; + } + } + + return ret; +} + +static ssize_t o2nm_cluster_reconnect_delay_ms_show( + struct config_item *item, char *page) +{ + return sprintf(page, "%u\n", + to_o2nm_cluster(item)->cl_reconnect_delay_ms); +} + +static ssize_t o2nm_cluster_reconnect_delay_ms_store( + struct config_item *item, const char *page, size_t count) +{ + return o2nm_cluster_attr_write(page, count, + &to_o2nm_cluster(item)->cl_reconnect_delay_ms); +} + +static ssize_t o2nm_cluster_fence_method_show( + struct config_item *item, char *page) +{ + struct o2nm_cluster *cluster = to_o2nm_cluster(item); + ssize_t ret = 0; + + if (cluster) + ret = sprintf(page, "%s\n", + o2nm_fence_method_desc[cluster->cl_fence_method]); + return ret; +} + +static ssize_t o2nm_cluster_fence_method_store( + struct config_item *item, const char *page, size_t count) +{ + unsigned int i; + + if (page[count - 1] != '\n') + goto bail; + + for (i = 0; i < O2NM_FENCE_METHODS; ++i) { + if (count != strlen(o2nm_fence_method_desc[i]) + 1) + continue; + if (strncasecmp(page, o2nm_fence_method_desc[i], count - 1)) + continue; + if (to_o2nm_cluster(item)->cl_fence_method != i) { + printk(KERN_INFO "ocfs2: Changing fence method to %s\n", + o2nm_fence_method_desc[i]); + to_o2nm_cluster(item)->cl_fence_method = i; + } + return count; + } + +bail: + return -EINVAL; +} + +CONFIGFS_ATTR(o2nm_cluster_, idle_timeout_ms); +CONFIGFS_ATTR(o2nm_cluster_, keepalive_delay_ms); +CONFIGFS_ATTR(o2nm_cluster_, reconnect_delay_ms); +CONFIGFS_ATTR(o2nm_cluster_, fence_method); + +static struct configfs_attribute *o2nm_cluster_attrs[] = { + &o2nm_cluster_attr_idle_timeout_ms, + &o2nm_cluster_attr_keepalive_delay_ms, + &o2nm_cluster_attr_reconnect_delay_ms, + &o2nm_cluster_attr_fence_method, + NULL, +}; + +static struct config_item *o2nm_node_group_make_item(struct config_group *group, + const char *name) +{ + struct o2nm_node *node = NULL; + + if (strlen(name) > O2NM_MAX_NAME_LEN) + return ERR_PTR(-ENAMETOOLONG); + + node = kzalloc_obj(struct o2nm_node); + if (node == NULL) + return ERR_PTR(-ENOMEM); + + strscpy(node->nd_name, name); /* use item.ci_namebuf instead? */ + config_item_init_type_name(&node->nd_item, name, &o2nm_node_type); + spin_lock_init(&node->nd_lock); + + mlog(ML_CLUSTER, "o2nm: Registering node %s\n", name); + + return &node->nd_item; +} + +static void o2nm_node_group_drop_item(struct config_group *group, + struct config_item *item) +{ + struct o2nm_node *node = to_o2nm_node(item); + struct o2nm_cluster *cluster = to_o2nm_cluster(group->cg_item.ci_parent); + + if (cluster->cl_nodes[node->nd_num] == node) { + o2net_disconnect_node(node); + + if (cluster->cl_has_local && + (cluster->cl_local_node == node->nd_num)) { + cluster->cl_has_local = 0; + cluster->cl_local_node = O2NM_INVALID_NODE_NUM; + o2net_stop_listening(node); + } + } + + /* XXX call into net to stop this node from trading messages */ + + write_lock(&cluster->cl_nodes_lock); + + /* XXX sloppy */ + if (node->nd_ipv4_address) + rb_erase(&node->nd_ip_node, &cluster->cl_node_ip_tree); + + /* nd_num might be 0 if the node number hasn't been set.. */ + if (cluster->cl_nodes[node->nd_num] == node) { + cluster->cl_nodes[node->nd_num] = NULL; + clear_bit(node->nd_num, cluster->cl_nodes_bitmap); + } + write_unlock(&cluster->cl_nodes_lock); + + mlog(ML_CLUSTER, "o2nm: Unregistered node %s\n", + config_item_name(&node->nd_item)); + + config_item_put(item); +} + +static const struct configfs_group_operations o2nm_node_group_group_ops = { + .make_item = o2nm_node_group_make_item, + .drop_item = o2nm_node_group_drop_item, +}; + +static const struct config_item_type o2nm_node_group_type = { + .ct_group_ops = &o2nm_node_group_group_ops, + .ct_owner = THIS_MODULE, +}; + +/* cluster */ + +static void o2nm_cluster_release(struct config_item *item) +{ + struct o2nm_cluster *cluster = to_o2nm_cluster(item); + + kfree(cluster); +} + +static const struct configfs_item_operations o2nm_cluster_item_ops = { + .release = o2nm_cluster_release, +}; + +static const struct config_item_type o2nm_cluster_type = { + .ct_item_ops = &o2nm_cluster_item_ops, + .ct_attrs = o2nm_cluster_attrs, + .ct_owner = THIS_MODULE, +}; + +/* cluster set */ + +struct o2nm_cluster_group { + struct configfs_subsystem cs_subsys; + /* some stuff? */ +}; + +#if 0 +static struct o2nm_cluster_group *to_o2nm_cluster_group(struct config_group *group) +{ + return group ? + container_of(to_configfs_subsystem(group), struct o2nm_cluster_group, cs_subsys) + : NULL; +} +#endif + +static struct config_group *o2nm_cluster_group_make_group(struct config_group *group, + const char *name) +{ + struct o2nm_cluster *cluster = NULL; + struct o2nm_node_group *ns = NULL; + struct config_group *o2hb_group = NULL, *ret = NULL; + + /* this runs under the parent dir's i_rwsem; there can be only + * one caller in here at a time */ + if (o2nm_single_cluster) + return ERR_PTR(-ENOSPC); + + cluster = kzalloc_obj(struct o2nm_cluster); + ns = kzalloc_obj(struct o2nm_node_group); + o2hb_group = o2hb_alloc_hb_set(); + if (cluster == NULL || ns == NULL || o2hb_group == NULL) + goto out; + + config_group_init_type_name(&cluster->cl_group, name, + &o2nm_cluster_type); + configfs_add_default_group(&ns->ns_group, &cluster->cl_group); + + config_group_init_type_name(&ns->ns_group, "node", + &o2nm_node_group_type); + configfs_add_default_group(o2hb_group, &cluster->cl_group); + + rwlock_init(&cluster->cl_nodes_lock); + cluster->cl_node_ip_tree = RB_ROOT; + cluster->cl_reconnect_delay_ms = O2NET_RECONNECT_DELAY_MS_DEFAULT; + cluster->cl_idle_timeout_ms = O2NET_IDLE_TIMEOUT_MS_DEFAULT; + cluster->cl_keepalive_delay_ms = O2NET_KEEPALIVE_DELAY_MS_DEFAULT; + cluster->cl_fence_method = O2NM_FENCE_RESET; + + ret = &cluster->cl_group; + o2nm_single_cluster = cluster; + +out: + if (ret == NULL) { + kfree(cluster); + kfree(ns); + o2hb_free_hb_set(o2hb_group); + ret = ERR_PTR(-ENOMEM); + } + + return ret; +} + +static void o2nm_cluster_group_drop_item(struct config_group *group, struct config_item *item) +{ + struct o2nm_cluster *cluster = to_o2nm_cluster(item); + + BUG_ON(o2nm_single_cluster != cluster); + o2nm_single_cluster = NULL; + + configfs_remove_default_groups(&cluster->cl_group); + config_item_put(item); +} + +static const struct configfs_group_operations o2nm_cluster_group_group_ops = { + .make_group = o2nm_cluster_group_make_group, + .drop_item = o2nm_cluster_group_drop_item, +}; + +static const struct config_item_type o2nm_cluster_group_type = { + .ct_group_ops = &o2nm_cluster_group_group_ops, + .ct_owner = THIS_MODULE, +}; + +static struct o2nm_cluster_group o2nm_cluster_group = { + .cs_subsys = { + .su_group = { + .cg_item = { + .ci_namebuf = "cluster", + .ci_type = &o2nm_cluster_group_type, + }, + }, + }, +}; + +static inline void o2nm_lock_subsystem(void) +{ + mutex_lock(&o2nm_cluster_group.cs_subsys.su_mutex); +} + +static inline void o2nm_unlock_subsystem(void) +{ + mutex_unlock(&o2nm_cluster_group.cs_subsys.su_mutex); +} + +int o2nm_depend_item(struct config_item *item) +{ + return configfs_depend_item(&o2nm_cluster_group.cs_subsys, item); +} + +int o2nm_depend_item_unlocked(struct config_item *item) +{ + return configfs_depend_item_unlocked(&o2nm_cluster_group.cs_subsys, + item); +} + +void o2nm_undepend_item(struct config_item *item) +{ + configfs_undepend_item(item); +} + +int o2nm_depend_node(u8 node_num) +{ + int ret = 0; + struct o2nm_node *local_node; + + local_node = o2nm_get_node_by_num(node_num); + if (!local_node) { + ret = -EINVAL; + goto out; + } + + ret = o2nm_depend_item(&local_node->nd_item); + o2nm_node_put(local_node); + +out: + return ret; +} + +void o2nm_undepend_node(u8 node_num) +{ + struct o2nm_node *local_node; + + local_node = o2nm_get_node_by_num(node_num); + BUG_ON(!local_node); + + o2nm_undepend_item(&local_node->nd_item); + o2nm_node_put(local_node); +} + +int o2nm_depend_this_node(void) +{ + return o2nm_depend_node(o2nm_this_node()); +} + +void o2nm_undepend_this_node(void) +{ + o2nm_undepend_node(o2nm_this_node()); +} + + +static void __exit exit_o2nm(void) +{ + /* XXX sync with hb callbacks and shut down hb? */ + o2net_unregister_hb_callbacks(); + configfs_unregister_subsystem(&o2nm_cluster_group.cs_subsys); + o2cb_sys_shutdown(); + + o2net_exit(); + o2hb_exit(); +} + +static int __init init_o2nm(void) +{ + int ret; + + o2hb_init(); + + ret = o2net_init(); + if (ret) + goto out_o2hb; + + ret = o2net_register_hb_callbacks(); + if (ret) + goto out_o2net; + + config_group_init(&o2nm_cluster_group.cs_subsys.su_group); + mutex_init(&o2nm_cluster_group.cs_subsys.su_mutex); + ret = configfs_register_subsystem(&o2nm_cluster_group.cs_subsys); + if (ret) { + printk(KERN_ERR "nodemanager: Registration returned %d\n", ret); + goto out_callbacks; + } + + ret = o2cb_sys_init(); + if (!ret) + goto out; + + configfs_unregister_subsystem(&o2nm_cluster_group.cs_subsys); +out_callbacks: + o2net_unregister_hb_callbacks(); +out_o2net: + o2net_exit(); +out_o2hb: + o2hb_exit(); +out: + return ret; +} + +MODULE_AUTHOR("Oracle"); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("OCFS2 cluster management"); + +module_init(init_o2nm) +module_exit(exit_o2nm) diff --git a/fs/ocfs2/cluster/nodemanager.h b/fs/ocfs2/cluster/nodemanager.h new file mode 100644 index 000000000..ca3483fb5 --- /dev/null +++ b/fs/ocfs2/cluster/nodemanager.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * nodemanager.h + * + * Function prototypes + * + * Copyright (C) 2004 Oracle. All rights reserved. + */ + +#ifndef O2CLUSTER_NODEMANAGER_H +#define O2CLUSTER_NODEMANAGER_H + +#include "ocfs2_nodemanager.h" + +/* This totally doesn't belong here. */ +#include +#include + +enum o2nm_fence_method { + O2NM_FENCE_RESET = 0, + O2NM_FENCE_PANIC, + O2NM_FENCE_METHODS, /* Number of fence methods */ +}; + +struct o2nm_node { + spinlock_t nd_lock; + struct config_item nd_item; + char nd_name[O2NM_MAX_NAME_LEN+1]; /* replace? */ + __u8 nd_num; + /* only one address per node, as attributes, for now. */ + __be32 nd_ipv4_address; + __be16 nd_ipv4_port; + struct rb_node nd_ip_node; + /* there can be only one local node for now */ + int nd_local; + + unsigned long nd_set_attributes; +}; + +struct o2nm_cluster { + struct config_group cl_group; + unsigned cl_has_local:1; + u8 cl_local_node; + rwlock_t cl_nodes_lock; + struct o2nm_node *cl_nodes[O2NM_MAX_NODES]; + struct rb_root cl_node_ip_tree; + unsigned int cl_idle_timeout_ms; + unsigned int cl_keepalive_delay_ms; + unsigned int cl_reconnect_delay_ms; + enum o2nm_fence_method cl_fence_method; + + /* this bitmap is part of a hack for disk bitmap.. will go eventually. - zab */ + unsigned long cl_nodes_bitmap[BITS_TO_LONGS(O2NM_MAX_NODES)]; +}; + +extern struct o2nm_cluster *o2nm_single_cluster; + +u8 o2nm_this_node(void); + +int o2nm_configured_node_map(unsigned long *map, unsigned bytes); +struct o2nm_node *o2nm_get_node_by_num(u8 node_num); +struct o2nm_node *o2nm_get_node_by_ip(__be32 addr); +void o2nm_node_get(struct o2nm_node *node); +void o2nm_node_put(struct o2nm_node *node); + +int o2nm_depend_item(struct config_item *item); +int o2nm_depend_item_unlocked(struct config_item *item); +void o2nm_undepend_item(struct config_item *item); +int o2nm_depend_node(u8 node_num); +void o2nm_undepend_node(u8 node_num); +int o2nm_depend_this_node(void); +void o2nm_undepend_this_node(void); + +#endif /* O2CLUSTER_NODEMANAGER_H */ diff --git a/fs/ocfs2/cluster/ocfs2_heartbeat.h b/fs/ocfs2/cluster/ocfs2_heartbeat.h new file mode 100644 index 000000000..6088c9f97 --- /dev/null +++ b/fs/ocfs2/cluster/ocfs2_heartbeat.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * ocfs2_heartbeat.h + * + * On-disk structures for ocfs2_heartbeat + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#ifndef _OCFS2_HEARTBEAT_H +#define _OCFS2_HEARTBEAT_H + +struct o2hb_disk_heartbeat_block { + __le64 hb_seq; + __u8 hb_node; + __u8 hb_pad1[3]; + __le32 hb_cksum; + __le64 hb_generation; + __le32 hb_dead_ms; +}; + +#endif /* _OCFS2_HEARTBEAT_H */ diff --git a/fs/ocfs2/cluster/ocfs2_nodemanager.h b/fs/ocfs2/cluster/ocfs2_nodemanager.h new file mode 100644 index 000000000..c9a0b7744 --- /dev/null +++ b/fs/ocfs2/cluster/ocfs2_nodemanager.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * ocfs2_nodemanager.h + * + * Header describing the interface between userspace and the kernel + * for the ocfs2_nodemanager module. + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#ifndef _OCFS2_NODEMANAGER_H +#define _OCFS2_NODEMANAGER_H + +#define O2NM_API_VERSION 5 + +#define O2NM_MAX_NODES 255 +#define O2NM_INVALID_NODE_NUM 255 + +/* host name, group name, cluster name all 64 bytes */ +#define O2NM_MAX_NAME_LEN 64 // __NEW_UTS_LEN + +/* + * Maximum number of global heartbeat regions allowed. + * **CAUTION** Changing this number will break dlm compatibility. + */ +#define O2NM_MAX_REGIONS 32 + +#endif /* _OCFS2_NODEMANAGER_H */ diff --git a/fs/ocfs2/cluster/quorum.c b/fs/ocfs2/cluster/quorum.c new file mode 100644 index 000000000..bfb8b4568 --- /dev/null +++ b/fs/ocfs2/cluster/quorum.c @@ -0,0 +1,326 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * + * Copyright (C) 2005 Oracle. All rights reserved. + */ + +/* This quorum hack is only here until we transition to some more rational + * approach that is driven from userspace. Honest. No foolin'. + * + * Imagine two nodes lose network connectivity to each other but they're still + * up and operating in every other way. Presumably a network timeout indicates + * that a node is broken and should be recovered. They can't both recover each + * other and both carry on without serialising their access to the file system. + * They need to decide who is authoritative. Now extend that problem to + * arbitrary groups of nodes losing connectivity between each other. + * + * So we declare that a node which has given up on connecting to a majority + * of nodes who are still heartbeating will fence itself. + * + * There are huge opportunities for races here. After we give up on a node's + * connection we need to wait long enough to give heartbeat an opportunity + * to declare the node as truly dead. We also need to be careful with the + * race between when we see a node start heartbeating and when we connect + * to it. + * + * So nodes that are in this transition put a hold on the quorum decision + * with a counter. As they fall out of this transition they drop the count + * and if they're the last, they fire off the decision. + */ +#include +#include +#include + +#include "heartbeat.h" +#include "nodemanager.h" +#define MLOG_MASK_PREFIX ML_QUORUM +#include "masklog.h" +#include "quorum.h" + +static struct o2quo_state { + spinlock_t qs_lock; + struct work_struct qs_work; + int qs_pending; + int qs_heartbeating; + unsigned long qs_hb_bm[BITS_TO_LONGS(O2NM_MAX_NODES)]; + int qs_connected; + unsigned long qs_conn_bm[BITS_TO_LONGS(O2NM_MAX_NODES)]; + int qs_holds; + unsigned long qs_hold_bm[BITS_TO_LONGS(O2NM_MAX_NODES)]; +} o2quo_state; + +/* this is horribly heavy-handed. It should instead flip the file + * system RO and call some userspace script. */ +static void o2quo_fence_self(void) +{ + /* panic spins with interrupts enabled. with preempt + * threads can still schedule, etc, etc */ + o2hb_stop_all_regions(); + + switch (o2nm_single_cluster->cl_fence_method) { + case O2NM_FENCE_PANIC: + panic("*** ocfs2 is very sorry to be fencing this system by " + "panicking ***\n"); + break; + default: + WARN_ON(o2nm_single_cluster->cl_fence_method >= + O2NM_FENCE_METHODS); + fallthrough; + case O2NM_FENCE_RESET: + printk(KERN_ERR "*** ocfs2 is very sorry to be fencing this " + "system by restarting ***\n"); + emergency_restart(); + break; + } +} + +/* Indicate that a timeout occurred on a heartbeat region write. The + * other nodes in the cluster may consider us dead at that time so we + * want to "fence" ourselves so that we don't scribble on the disk + * after they think they've recovered us. This can't solve all + * problems related to writeout after recovery but this hack can at + * least close some of those gaps. When we have real fencing, this can + * go away as our node would be fenced externally before other nodes + * begin recovery. */ +void o2quo_disk_timeout(void) +{ + o2quo_fence_self(); +} + +static void o2quo_make_decision(struct work_struct *work) +{ + int quorum; + int lowest_hb, lowest_reachable = 0, fence = 0; + struct o2quo_state *qs = &o2quo_state; + + spin_lock_bh(&qs->qs_lock); + + lowest_hb = find_first_bit(qs->qs_hb_bm, O2NM_MAX_NODES); + if (lowest_hb != O2NM_MAX_NODES) + lowest_reachable = test_bit(lowest_hb, qs->qs_conn_bm); + + mlog(0, "heartbeating: %d, connected: %d, " + "lowest: %d (%sreachable)\n", qs->qs_heartbeating, + qs->qs_connected, lowest_hb, lowest_reachable ? "" : "un"); + + if (!test_bit(o2nm_this_node(), qs->qs_hb_bm) || + qs->qs_heartbeating == 1) + goto out; + + if (qs->qs_heartbeating & 1) { + /* the odd numbered cluster case is straight forward -- + * if we can't talk to the majority we're hosed */ + quorum = (qs->qs_heartbeating + 1)/2; + if (qs->qs_connected < quorum) { + mlog(ML_ERROR, "fencing this node because it is " + "only connected to %u nodes and %u is needed " + "to make a quorum out of %u heartbeating nodes\n", + qs->qs_connected, quorum, + qs->qs_heartbeating); + fence = 1; + } + } else { + /* the even numbered cluster adds the possibility of each half + * of the cluster being able to talk amongst themselves.. in + * that case we're hosed if we can't talk to the group that has + * the lowest numbered node */ + quorum = qs->qs_heartbeating / 2; + if (qs->qs_connected < quorum) { + mlog(ML_ERROR, "fencing this node because it is " + "only connected to %u nodes and %u is needed " + "to make a quorum out of %u heartbeating nodes\n", + qs->qs_connected, quorum, + qs->qs_heartbeating); + fence = 1; + } + else if ((qs->qs_connected == quorum) && + !lowest_reachable) { + mlog(ML_ERROR, "fencing this node because it is " + "connected to a half-quorum of %u out of %u " + "nodes which doesn't include the lowest active " + "node %u\n", quorum, qs->qs_heartbeating, + lowest_hb); + fence = 1; + } + } + +out: + if (fence) { + spin_unlock_bh(&qs->qs_lock); + o2quo_fence_self(); + } else { + mlog(ML_NOTICE, "not fencing this node, heartbeating: %d, " + "connected: %d, lowest: %d (%sreachable)\n", + qs->qs_heartbeating, qs->qs_connected, lowest_hb, + lowest_reachable ? "" : "un"); + spin_unlock_bh(&qs->qs_lock); + + } + +} + +static void o2quo_set_hold(struct o2quo_state *qs, u8 node) +{ + assert_spin_locked(&qs->qs_lock); + + if (!test_and_set_bit(node, qs->qs_hold_bm)) { + qs->qs_holds++; + mlog_bug_on_msg(qs->qs_holds == O2NM_MAX_NODES, + "node %u\n", node); + mlog(0, "node %u, %d total\n", node, qs->qs_holds); + } +} + +static void o2quo_clear_hold(struct o2quo_state *qs, u8 node) +{ + assert_spin_locked(&qs->qs_lock); + + if (test_and_clear_bit(node, qs->qs_hold_bm)) { + mlog(0, "node %u, %d total\n", node, qs->qs_holds - 1); + if (--qs->qs_holds == 0) { + if (qs->qs_pending) { + qs->qs_pending = 0; + schedule_work(&qs->qs_work); + } + } + mlog_bug_on_msg(qs->qs_holds < 0, "node %u, holds %d\n", + node, qs->qs_holds); + } +} + +/* as a node comes up we delay the quorum decision until we know the fate of + * the connection. the hold will be dropped in conn_up or hb_down. it might be + * perpetuated by con_err until hb_down. if we already have a conn, we might + * be dropping a hold that conn_up got. */ +void o2quo_hb_up(u8 node) +{ + struct o2quo_state *qs = &o2quo_state; + + spin_lock_bh(&qs->qs_lock); + + qs->qs_heartbeating++; + mlog_bug_on_msg(qs->qs_heartbeating == O2NM_MAX_NODES, + "node %u\n", node); + mlog_bug_on_msg(test_bit(node, qs->qs_hb_bm), "node %u\n", node); + set_bit(node, qs->qs_hb_bm); + + mlog(0, "node %u, %d total\n", node, qs->qs_heartbeating); + + if (!test_bit(node, qs->qs_conn_bm)) + o2quo_set_hold(qs, node); + else + o2quo_clear_hold(qs, node); + + spin_unlock_bh(&qs->qs_lock); +} + +/* hb going down releases any holds we might have had due to this node from + * conn_up, conn_err, or hb_up */ +void o2quo_hb_down(u8 node) +{ + struct o2quo_state *qs = &o2quo_state; + + spin_lock_bh(&qs->qs_lock); + + qs->qs_heartbeating--; + mlog_bug_on_msg(qs->qs_heartbeating < 0, + "node %u, %d heartbeating\n", + node, qs->qs_heartbeating); + mlog_bug_on_msg(!test_bit(node, qs->qs_hb_bm), "node %u\n", node); + clear_bit(node, qs->qs_hb_bm); + + mlog(0, "node %u, %d total\n", node, qs->qs_heartbeating); + + o2quo_clear_hold(qs, node); + + spin_unlock_bh(&qs->qs_lock); +} + +/* this tells us that we've decided that the node is still heartbeating + * even though we've lost it's conn. it must only be called after conn_err + * and indicates that we must now make a quorum decision in the future, + * though we might be doing so after waiting for holds to drain. Here + * we'll be dropping the hold from conn_err. */ +void o2quo_hb_still_up(u8 node) +{ + struct o2quo_state *qs = &o2quo_state; + + spin_lock_bh(&qs->qs_lock); + + mlog(0, "node %u\n", node); + + qs->qs_pending = 1; + o2quo_clear_hold(qs, node); + + spin_unlock_bh(&qs->qs_lock); +} + +/* This is analogous to hb_up. as a node's connection comes up we delay the + * quorum decision until we see it heartbeating. the hold will be dropped in + * hb_up or hb_down. it might be perpetuated by con_err until hb_down. if + * it's already heartbeating we might be dropping a hold that conn_up got. + * */ +void o2quo_conn_up(u8 node) +{ + struct o2quo_state *qs = &o2quo_state; + + spin_lock_bh(&qs->qs_lock); + + qs->qs_connected++; + mlog_bug_on_msg(qs->qs_connected == O2NM_MAX_NODES, + "node %u\n", node); + mlog_bug_on_msg(test_bit(node, qs->qs_conn_bm), "node %u\n", node); + set_bit(node, qs->qs_conn_bm); + + mlog(0, "node %u, %d total\n", node, qs->qs_connected); + + if (!test_bit(node, qs->qs_hb_bm)) + o2quo_set_hold(qs, node); + else + o2quo_clear_hold(qs, node); + + spin_unlock_bh(&qs->qs_lock); +} + +/* we've decided that we won't ever be connecting to the node again. if it's + * still heartbeating we grab a hold that will delay decisions until either the + * node stops heartbeating from hb_down or the caller decides that the node is + * still up and calls still_up */ +void o2quo_conn_err(u8 node) +{ + struct o2quo_state *qs = &o2quo_state; + + spin_lock_bh(&qs->qs_lock); + + if (test_bit(node, qs->qs_conn_bm)) { + qs->qs_connected--; + mlog_bug_on_msg(qs->qs_connected < 0, + "node %u, connected %d\n", + node, qs->qs_connected); + + clear_bit(node, qs->qs_conn_bm); + + if (test_bit(node, qs->qs_hb_bm)) + o2quo_set_hold(qs, node); + } + + mlog(0, "node %u, %d total\n", node, qs->qs_connected); + + + spin_unlock_bh(&qs->qs_lock); +} + +void o2quo_init(void) +{ + struct o2quo_state *qs = &o2quo_state; + + spin_lock_init(&qs->qs_lock); + INIT_WORK(&qs->qs_work, o2quo_make_decision); +} + +void o2quo_exit(void) +{ + struct o2quo_state *qs = &o2quo_state; + + flush_work(&qs->qs_work); +} diff --git a/fs/ocfs2/cluster/quorum.h b/fs/ocfs2/cluster/quorum.h new file mode 100644 index 000000000..d64bf4482 --- /dev/null +++ b/fs/ocfs2/cluster/quorum.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2005 Oracle. All rights reserved. + */ + +#ifndef O2CLUSTER_QUORUM_H +#define O2CLUSTER_QUORUM_H + +void o2quo_init(void); +void o2quo_exit(void); + +void o2quo_hb_up(u8 node); +void o2quo_hb_down(u8 node); +void o2quo_hb_still_up(u8 node); +void o2quo_conn_up(u8 node); +void o2quo_conn_err(u8 node); +void o2quo_disk_timeout(void); + +#endif /* O2CLUSTER_QUORUM_H */ diff --git a/fs/ocfs2/cluster/sys.c b/fs/ocfs2/cluster/sys.c new file mode 100644 index 000000000..022f716c7 --- /dev/null +++ b/fs/ocfs2/cluster/sys.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * sys.c + * + * OCFS2 cluster sysfs interface + * + * Copyright (C) 2005 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include + +#include "ocfs2_nodemanager.h" +#include "masklog.h" +#include "sys.h" + + +static ssize_t version_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + return snprintf(buf, PAGE_SIZE, "%u\n", O2NM_API_VERSION); +} +static struct kobj_attribute attr_version = + __ATTR(interface_revision, S_IRUGO, version_show, NULL); + +static struct attribute *o2cb_attrs[] = { + &attr_version.attr, + NULL, +}; + +static struct attribute_group o2cb_attr_group = { + .attrs = o2cb_attrs, +}; + +static struct kset *o2cb_kset; + +void o2cb_sys_shutdown(void) +{ + mlog_sys_shutdown(); + kset_unregister(o2cb_kset); +} + +int o2cb_sys_init(void) +{ + int ret; + + o2cb_kset = kset_create_and_add("o2cb", NULL, fs_kobj); + if (!o2cb_kset) + return -ENOMEM; + + ret = sysfs_create_group(&o2cb_kset->kobj, &o2cb_attr_group); + if (ret) + goto error; + + ret = mlog_sys_init(o2cb_kset); + if (ret) + goto error; + return 0; +error: + kset_unregister(o2cb_kset); + return ret; +} diff --git a/fs/ocfs2/cluster/sys.h b/fs/ocfs2/cluster/sys.h new file mode 100644 index 000000000..70aaba653 --- /dev/null +++ b/fs/ocfs2/cluster/sys.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * sys.h + * + * Function prototypes for o2cb sysfs interface + * + * Copyright (C) 2005 Oracle. All rights reserved. + */ + +#ifndef O2CLUSTER_SYS_H +#define O2CLUSTER_SYS_H + +void o2cb_sys_shutdown(void); +int o2cb_sys_init(void); + +#endif /* O2CLUSTER_SYS_H */ diff --git a/fs/ocfs2/cluster/tcp.c b/fs/ocfs2/cluster/tcp.c new file mode 100644 index 000000000..474fe1414 --- /dev/null +++ b/fs/ocfs2/cluster/tcp.c @@ -0,0 +1,2274 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * + * Copyright (C) 2004 Oracle. All rights reserved. + * + * ---- + * + * Callers for this were originally written against a very simple synchronous + * API. This implementation reflects those simple callers. Some day I'm sure + * we'll need to move to a more robust posting/callback mechanism. + * + * Transmit calls pass in kernel virtual addresses and block copying this into + * the socket's tx buffers via a usual blocking sendmsg. They'll block waiting + * for a failed socket to timeout. TX callers can also pass in a pointer to an + * 'int' which gets filled with an errno off the wire in response to the + * message they send. + * + * Handlers for unsolicited messages are registered. Each socket has a page + * that incoming data is copied into. First the header, then the data. + * Handlers are called from only one thread with a reference to this per-socket + * page. This page is destroyed after the handler call, so it can't be + * referenced beyond the call. Handlers may block but are discouraged from + * doing so. + * + * Any framing errors (bad magic, large payload lengths) close a connection. + * + * Our sock_container holds the state we associate with a socket. It's current + * framing state is held there as well as the refcounting we do around when it + * is safe to tear down the socket. The socket is only finally torn down from + * the container when the container loses all of its references -- so as long + * as you hold a ref on the container you can trust that the socket is valid + * for use with kernel socket APIs. + * + * Connections are initiated between a pair of nodes when the node with the + * higher node number gets a heartbeat callback which indicates that the lower + * numbered node has started heartbeating. The lower numbered node is passive + * and only accepts the connection if the higher numbered node is heartbeating. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "heartbeat.h" +#include "tcp.h" +#include "nodemanager.h" +#define MLOG_MASK_PREFIX ML_TCP +#include "masklog.h" +#include "quorum.h" + +#include "tcp_internal.h" + +#define SC_NODEF_FMT "node %s (num %u) at %pI4:%u" +#define SC_NODEF_ARGS(sc) sc->sc_node->nd_name, sc->sc_node->nd_num, \ + &sc->sc_node->nd_ipv4_address, \ + ntohs(sc->sc_node->nd_ipv4_port) + +/* + * In the following two log macros, the whitespace after the ',' just + * before ##args is intentional. Otherwise, gcc 2.95 will eat the + * previous token if args expands to nothing. + */ +#define msglog(hdr, fmt, args...) do { \ + typeof(hdr) __hdr = (hdr); \ + mlog(ML_MSG, "[mag %u len %u typ %u stat %d sys_stat %d " \ + "key %08x num %u] " fmt, \ + be16_to_cpu(__hdr->magic), be16_to_cpu(__hdr->data_len), \ + be16_to_cpu(__hdr->msg_type), be32_to_cpu(__hdr->status), \ + be32_to_cpu(__hdr->sys_status), be32_to_cpu(__hdr->key), \ + be32_to_cpu(__hdr->msg_num) , ##args); \ +} while (0) + +#define sclog(sc, fmt, args...) do { \ + typeof(sc) __sc = (sc); \ + mlog(ML_SOCKET, "[sc %p refs %d sock %p node %u page %p " \ + "pg_off %zu] " fmt, __sc, \ + kref_read(&__sc->sc_kref), __sc->sc_sock, \ + __sc->sc_node->nd_num, __sc->sc_page, __sc->sc_page_off , \ + ##args); \ +} while (0) + +static DEFINE_RWLOCK(o2net_handler_lock); +static struct rb_root o2net_handler_tree = RB_ROOT; + +static struct o2net_node o2net_nodes[O2NM_MAX_NODES]; + +/* XXX someday we'll need better accounting */ +static struct socket *o2net_listen_sock; + +/* + * listen work is only queued by the listening socket callbacks on the + * o2net_wq. teardown detaches the callbacks before destroying the workqueue. + * quorum work is queued as sock containers are shutdown.. stop_listening + * tears down all the node's sock containers, preventing future shutdowns + * and queued quorum work, before canceling delayed quorum work and + * destroying the work queue. Handler teardown can also race local listener + * shutdown, so keep a waitable destroying pointer until the old ordered + * queue has finished draining. + */ +static struct workqueue_struct *o2net_wq; +static struct workqueue_struct *o2net_wq_destroying; +static DEFINE_MUTEX(o2net_wq_mutex); +static DECLARE_COMPLETION(o2net_wq_destroyed); +/* Heartbeat callbacks stay registered across local-node off/on. */ +static bool o2net_listening; +static struct work_struct o2net_listen_work; + +static struct o2hb_callback_func o2net_hb_up, o2net_hb_down; +#define O2NET_HB_PRI 0x1 + +static struct o2net_handshake *o2net_hand; +static struct o2net_msg *o2net_keep_req, *o2net_keep_resp; + +static int o2net_sys_err_translations[O2NET_ERR_MAX] = + {[O2NET_ERR_NONE] = 0, + [O2NET_ERR_NO_HNDLR] = -ENOPROTOOPT, + [O2NET_ERR_OVERFLOW] = -EOVERFLOW, + [O2NET_ERR_DIED] = -EHOSTDOWN,}; + +/* can't quite avoid *all* internal declarations :/ */ +static void o2net_sc_connect_completed(struct work_struct *work); +static void o2net_rx_until_empty(struct work_struct *work); +static void o2net_shutdown_sc(struct work_struct *work); +static void o2net_listen_data_ready(struct sock *sk); +static void o2net_sc_send_keep_req(struct work_struct *work); +static void o2net_idle_timer(struct timer_list *t); +static void o2net_sc_postpone_idle(struct o2net_sock_container *sc); +static void o2net_sc_reset_idle_timer(struct o2net_sock_container *sc); + +#ifdef CONFIG_DEBUG_FS +static void o2net_init_nst(struct o2net_send_tracking *nst, u32 msgtype, + u32 msgkey, struct task_struct *task, u8 node) +{ + INIT_LIST_HEAD(&nst->st_net_debug_item); + nst->st_task = task; + nst->st_msg_type = msgtype; + nst->st_msg_key = msgkey; + nst->st_node = node; +} + +static inline void o2net_set_nst_sock_time(struct o2net_send_tracking *nst) +{ + nst->st_sock_time = ktime_get(); +} + +static inline void o2net_set_nst_send_time(struct o2net_send_tracking *nst) +{ + nst->st_send_time = ktime_get(); +} + +static inline void o2net_set_nst_status_time(struct o2net_send_tracking *nst) +{ + nst->st_status_time = ktime_get(); +} + +static inline void o2net_set_nst_sock_container(struct o2net_send_tracking *nst, + struct o2net_sock_container *sc) +{ + nst->st_sc = sc; +} + +static inline void o2net_set_nst_msg_id(struct o2net_send_tracking *nst, + u32 msg_id) +{ + nst->st_id = msg_id; +} + +static inline void o2net_set_sock_timer(struct o2net_sock_container *sc) +{ + sc->sc_tv_timer = ktime_get(); +} + +static inline void o2net_set_data_ready_time(struct o2net_sock_container *sc) +{ + sc->sc_tv_data_ready = ktime_get(); +} + +static inline void o2net_set_advance_start_time(struct o2net_sock_container *sc) +{ + sc->sc_tv_advance_start = ktime_get(); +} + +static inline void o2net_set_advance_stop_time(struct o2net_sock_container *sc) +{ + sc->sc_tv_advance_stop = ktime_get(); +} + +static inline void o2net_set_func_start_time(struct o2net_sock_container *sc) +{ + sc->sc_tv_func_start = ktime_get(); +} + +static inline void o2net_set_func_stop_time(struct o2net_sock_container *sc) +{ + sc->sc_tv_func_stop = ktime_get(); +} + +#else /* CONFIG_DEBUG_FS */ +# define o2net_init_nst(a, b, c, d, e) +# define o2net_set_nst_sock_time(a) +# define o2net_set_nst_send_time(a) +# define o2net_set_nst_status_time(a) +# define o2net_set_nst_sock_container(a, b) +# define o2net_set_nst_msg_id(a, b) +# define o2net_set_sock_timer(a) +# define o2net_set_data_ready_time(a) +# define o2net_set_advance_start_time(a) +# define o2net_set_advance_stop_time(a) +# define o2net_set_func_start_time(a) +# define o2net_set_func_stop_time(a) +#endif /* CONFIG_DEBUG_FS */ + +#ifdef CONFIG_OCFS2_FS_STATS +static ktime_t o2net_get_func_run_time(struct o2net_sock_container *sc) +{ + return ktime_sub(sc->sc_tv_func_stop, sc->sc_tv_func_start); +} + +static void o2net_update_send_stats(struct o2net_send_tracking *nst, + struct o2net_sock_container *sc) +{ + sc->sc_tv_status_total = ktime_add(sc->sc_tv_status_total, + ktime_sub(ktime_get(), + nst->st_status_time)); + sc->sc_tv_send_total = ktime_add(sc->sc_tv_send_total, + ktime_sub(nst->st_status_time, + nst->st_send_time)); + sc->sc_tv_acquiry_total = ktime_add(sc->sc_tv_acquiry_total, + ktime_sub(nst->st_send_time, + nst->st_sock_time)); + sc->sc_send_count++; +} + +static void o2net_update_recv_stats(struct o2net_sock_container *sc) +{ + sc->sc_tv_process_total = ktime_add(sc->sc_tv_process_total, + o2net_get_func_run_time(sc)); + sc->sc_recv_count++; +} + +#else + +# define o2net_update_send_stats(a, b) + +# define o2net_update_recv_stats(sc) + +#endif /* CONFIG_OCFS2_FS_STATS */ + +static inline unsigned int o2net_reconnect_delay(void) +{ + return o2nm_single_cluster->cl_reconnect_delay_ms; +} + +static inline unsigned int o2net_keepalive_delay(void) +{ + return o2nm_single_cluster->cl_keepalive_delay_ms; +} + +static inline unsigned int o2net_idle_timeout(void) +{ + return o2nm_single_cluster->cl_idle_timeout_ms; +} + +static inline int o2net_sys_err_to_errno(enum o2net_system_error err) +{ + int trans; + BUG_ON(err >= O2NET_ERR_MAX); + trans = o2net_sys_err_translations[err]; + + /* Just in case we mess up the translation table above */ + BUG_ON(err != O2NET_ERR_NONE && trans == 0); + return trans; +} + +static struct o2net_node * o2net_nn_from_num(u8 node_num) +{ + BUG_ON(node_num >= ARRAY_SIZE(o2net_nodes)); + return &o2net_nodes[node_num]; +} + +static u8 o2net_num_from_nn(struct o2net_node *nn) +{ + BUG_ON(nn == NULL); + return nn - o2net_nodes; +} + +/* ------------------------------------------------------------ */ + +static int o2net_prep_nsw(struct o2net_node *nn, struct o2net_status_wait *nsw) +{ + int ret; + + spin_lock(&nn->nn_lock); + ret = idr_alloc(&nn->nn_status_idr, nsw, 0, 0, GFP_ATOMIC); + if (ret >= 0) { + nsw->ns_id = ret; + list_add_tail(&nsw->ns_node_item, &nn->nn_status_list); + } + spin_unlock(&nn->nn_lock); + if (ret < 0) + return ret; + + init_waitqueue_head(&nsw->ns_wq); + nsw->ns_sys_status = O2NET_ERR_NONE; + nsw->ns_status = 0; + return 0; +} + +static void o2net_complete_nsw_locked(struct o2net_node *nn, + struct o2net_status_wait *nsw, + enum o2net_system_error sys_status, + s32 status) +{ + assert_spin_locked(&nn->nn_lock); + + if (!list_empty(&nsw->ns_node_item)) { + list_del_init(&nsw->ns_node_item); + nsw->ns_sys_status = sys_status; + nsw->ns_status = status; + idr_remove(&nn->nn_status_idr, nsw->ns_id); + wake_up(&nsw->ns_wq); + } +} + +static void o2net_complete_nsw(struct o2net_node *nn, + struct o2net_status_wait *nsw, + u64 id, enum o2net_system_error sys_status, + s32 status) +{ + spin_lock(&nn->nn_lock); + if (nsw == NULL) { + if (id > INT_MAX) + goto out; + + nsw = idr_find(&nn->nn_status_idr, id); + if (nsw == NULL) + goto out; + } + + o2net_complete_nsw_locked(nn, nsw, sys_status, status); + +out: + spin_unlock(&nn->nn_lock); + return; +} + +static void o2net_complete_nodes_nsw(struct o2net_node *nn) +{ + struct o2net_status_wait *nsw, *tmp; + unsigned int num_kills = 0; + + assert_spin_locked(&nn->nn_lock); + + list_for_each_entry_safe(nsw, tmp, &nn->nn_status_list, ns_node_item) { + o2net_complete_nsw_locked(nn, nsw, O2NET_ERR_DIED, 0); + num_kills++; + } + + mlog(0, "completed %d messages for node %u\n", num_kills, + o2net_num_from_nn(nn)); +} + +static int o2net_nsw_completed(struct o2net_node *nn, + struct o2net_status_wait *nsw) +{ + int completed; + spin_lock(&nn->nn_lock); + completed = list_empty(&nsw->ns_node_item); + spin_unlock(&nn->nn_lock); + return completed; +} + +/* ------------------------------------------------------------ */ + +static void sc_kref_release(struct kref *kref) +{ + struct o2net_sock_container *sc = container_of(kref, + struct o2net_sock_container, sc_kref); + BUG_ON(timer_pending(&sc->sc_idle_timeout)); + + sclog(sc, "releasing\n"); + + if (sc->sc_sock) { + sock_release(sc->sc_sock); + sc->sc_sock = NULL; + } + + o2nm_undepend_item(&sc->sc_node->nd_item); + o2nm_node_put(sc->sc_node); + sc->sc_node = NULL; + + o2net_debug_del_sc(sc); + + if (sc->sc_page) + __free_page(sc->sc_page); + kfree(sc); +} + +static void sc_put(struct o2net_sock_container *sc) +{ + sclog(sc, "put\n"); + kref_put(&sc->sc_kref, sc_kref_release); +} +static void sc_get(struct o2net_sock_container *sc) +{ + sclog(sc, "get\n"); + kref_get(&sc->sc_kref); +} +static struct o2net_sock_container *sc_alloc(struct o2nm_node *node) +{ + struct o2net_sock_container *sc, *ret = NULL; + struct page *page = NULL; + int status = 0; + + page = alloc_page(GFP_NOFS); + sc = kzalloc_obj(*sc, GFP_NOFS); + if (sc == NULL || page == NULL) + goto out; + + kref_init(&sc->sc_kref); + o2nm_node_get(node); + sc->sc_node = node; + + /* pin the node item of the remote node */ + status = o2nm_depend_item(&node->nd_item); + if (status) { + mlog_errno(status); + o2nm_node_put(node); + goto out; + } + INIT_WORK(&sc->sc_connect_work, o2net_sc_connect_completed); + INIT_WORK(&sc->sc_rx_work, o2net_rx_until_empty); + INIT_WORK(&sc->sc_shutdown_work, o2net_shutdown_sc); + INIT_DELAYED_WORK(&sc->sc_keepalive_work, o2net_sc_send_keep_req); + + timer_setup(&sc->sc_idle_timeout, o2net_idle_timer, 0); + + sclog(sc, "alloced\n"); + + ret = sc; + sc->sc_page = page; + o2net_debug_add_sc(sc); + sc = NULL; + page = NULL; + +out: + if (page) + __free_page(page); + kfree(sc); + + return ret; +} + +/* ------------------------------------------------------------ */ + +static void o2net_sc_queue_work(struct o2net_sock_container *sc, + struct work_struct *work) +{ + sc_get(sc); + if (!queue_work(o2net_wq, work)) + sc_put(sc); +} +static void o2net_sc_queue_delayed_work(struct o2net_sock_container *sc, + struct delayed_work *work, + int delay) +{ + sc_get(sc); + if (!queue_delayed_work(o2net_wq, work, delay)) + sc_put(sc); +} +static void o2net_sc_cancel_delayed_work(struct o2net_sock_container *sc, + struct delayed_work *work) +{ + if (cancel_delayed_work(work)) + sc_put(sc); +} + +static atomic_t o2net_connected_peers = ATOMIC_INIT(0); + +int o2net_num_connected_peers(void) +{ + return atomic_read(&o2net_connected_peers); +} + +static void o2net_set_nn_state(struct o2net_node *nn, + struct o2net_sock_container *sc, + unsigned valid, int err) +{ + int was_valid = nn->nn_sc_valid; + int was_err = nn->nn_persistent_error; + struct o2net_sock_container *old_sc = nn->nn_sc; + + assert_spin_locked(&nn->nn_lock); + + if (old_sc && !sc) + atomic_dec(&o2net_connected_peers); + else if (!old_sc && sc) + atomic_inc(&o2net_connected_peers); + + /* the node num comparison and single connect/accept path should stop + * an non-null sc from being overwritten with another */ + BUG_ON(sc && nn->nn_sc && nn->nn_sc != sc); + mlog_bug_on_msg(err && valid, "err %d valid %u\n", err, valid); + mlog_bug_on_msg(valid && !sc, "valid %u sc %p\n", valid, sc); + + if (was_valid && !valid && err == 0) + err = -ENOTCONN; + + mlog(ML_CONN, "node %u sc: %p -> %p, valid %u -> %u, err %d -> %d\n", + o2net_num_from_nn(nn), nn->nn_sc, sc, nn->nn_sc_valid, valid, + nn->nn_persistent_error, err); + + nn->nn_sc = sc; + nn->nn_sc_valid = valid ? 1 : 0; + nn->nn_persistent_error = err; + + /* mirrors o2net_tx_can_proceed() */ + if (nn->nn_persistent_error || nn->nn_sc_valid) + wake_up(&nn->nn_sc_wq); + + if (was_valid && !was_err && nn->nn_persistent_error) { + o2quo_conn_err(o2net_num_from_nn(nn)); + queue_delayed_work(o2net_wq, &nn->nn_still_up, + msecs_to_jiffies(O2NET_QUORUM_DELAY_MS)); + } + + if (was_valid && !valid) { + if (old_sc) + printk(KERN_NOTICE "o2net: No longer connected to " + SC_NODEF_FMT "\n", SC_NODEF_ARGS(old_sc)); + o2net_complete_nodes_nsw(nn); + } + + if (!was_valid && valid) { + o2quo_conn_up(o2net_num_from_nn(nn)); + cancel_delayed_work(&nn->nn_connect_expired); + printk(KERN_NOTICE "o2net: %s " SC_NODEF_FMT "\n", + o2nm_this_node() > sc->sc_node->nd_num ? + "Connected to" : "Accepted connection from", + SC_NODEF_ARGS(sc)); + } + + /* trigger the connecting worker func as long as we're not valid, + * it will back off if it shouldn't connect. This can be called + * from node config teardown and so needs to be careful about + * the work queue actually being up. */ + if (!valid && o2net_wq) { + unsigned long delay; + /* delay if we're within a RECONNECT_DELAY of the + * last attempt */ + delay = (nn->nn_last_connect_attempt + + msecs_to_jiffies(o2net_reconnect_delay())) + - jiffies; + if (delay > msecs_to_jiffies(o2net_reconnect_delay())) + delay = 0; + mlog(ML_CONN, "queueing conn attempt in %lu jiffies\n", delay); + queue_delayed_work(o2net_wq, &nn->nn_connect_work, delay); + + /* + * Delay the expired work after idle timeout. + * + * We might have lots of failed connection attempts that run + * through here but we only cancel the connect_expired work when + * a connection attempt succeeds. So only the first enqueue of + * the connect_expired work will do anything. The rest will see + * that it's already queued and do nothing. + */ + delay += msecs_to_jiffies(o2net_idle_timeout()); + queue_delayed_work(o2net_wq, &nn->nn_connect_expired, delay); + } + + /* keep track of the nn's sc ref for the caller */ + if ((old_sc == NULL) && sc) + sc_get(sc); + if (old_sc && (old_sc != sc)) { + o2net_sc_queue_work(old_sc, &old_sc->sc_shutdown_work); + sc_put(old_sc); + } +} + +/* see o2net_register_callbacks() */ +static void o2net_data_ready(struct sock *sk) +{ + void (*ready)(struct sock *sk); + struct o2net_sock_container *sc; + + trace_sk_data_ready(sk); + + read_lock_bh(&sk->sk_callback_lock); + sc = sk->sk_user_data; + if (sc) { + sclog(sc, "data_ready hit\n"); + o2net_set_data_ready_time(sc); + o2net_sc_queue_work(sc, &sc->sc_rx_work); + ready = sc->sc_data_ready; + } else { + ready = sk->sk_data_ready; + } + read_unlock_bh(&sk->sk_callback_lock); + + ready(sk); +} + +/* see o2net_register_callbacks() */ +static void o2net_state_change(struct sock *sk) +{ + void (*state_change)(struct sock *sk); + struct o2net_sock_container *sc; + + read_lock_bh(&sk->sk_callback_lock); + sc = sk->sk_user_data; + if (sc == NULL) { + state_change = sk->sk_state_change; + goto out; + } + + sclog(sc, "state_change to %d\n", sk->sk_state); + + state_change = sc->sc_state_change; + + switch(sk->sk_state) { + /* ignore connecting sockets as they make progress */ + case TCP_SYN_SENT: + case TCP_SYN_RECV: + break; + case TCP_ESTABLISHED: + o2net_sc_queue_work(sc, &sc->sc_connect_work); + break; + default: + printk(KERN_INFO "o2net: Connection to " SC_NODEF_FMT + " shutdown, state %d\n", + SC_NODEF_ARGS(sc), sk->sk_state); + o2net_sc_queue_work(sc, &sc->sc_shutdown_work); + break; + } +out: + read_unlock_bh(&sk->sk_callback_lock); + state_change(sk); +} + +/* + * we register callbacks so we can queue work on events before calling + * the original callbacks. our callbacks our careful to test user_data + * to discover when they've reaced with o2net_unregister_callbacks(). + */ +static void o2net_register_callbacks(struct sock *sk, + struct o2net_sock_container *sc) +{ + write_lock_bh(&sk->sk_callback_lock); + + /* accepted sockets inherit the old listen socket data ready */ + if (sk->sk_data_ready == o2net_listen_data_ready) { + sk->sk_data_ready = sk->sk_user_data; + sk->sk_user_data = NULL; + } + + BUG_ON(sk->sk_user_data != NULL); + sk->sk_user_data = sc; + sc_get(sc); + + sc->sc_data_ready = sk->sk_data_ready; + sc->sc_state_change = sk->sk_state_change; + sk->sk_data_ready = o2net_data_ready; + sk->sk_state_change = o2net_state_change; + + mutex_init(&sc->sc_send_lock); + + write_unlock_bh(&sk->sk_callback_lock); +} + +static int o2net_unregister_callbacks(struct sock *sk, + struct o2net_sock_container *sc) +{ + int ret = 0; + + write_lock_bh(&sk->sk_callback_lock); + if (sk->sk_user_data == sc) { + ret = 1; + sk->sk_user_data = NULL; + sk->sk_data_ready = sc->sc_data_ready; + sk->sk_state_change = sc->sc_state_change; + } + write_unlock_bh(&sk->sk_callback_lock); + + return ret; +} + +/* + * this is a little helper that is called by callers who have seen a problem + * with an sc and want to detach it from the nn if someone already hasn't beat + * them to it. if an error is given then the shutdown will be persistent + * and pending transmits will be canceled. + */ +static void o2net_ensure_shutdown(struct o2net_node *nn, + struct o2net_sock_container *sc, + int err) +{ + spin_lock(&nn->nn_lock); + if (nn->nn_sc == sc) + o2net_set_nn_state(nn, NULL, 0, err); + spin_unlock(&nn->nn_lock); +} + +/* + * This work queue function performs the blocking parts of socket shutdown. A + * few paths lead here. set_nn_state will trigger this callback if it sees an + * sc detached from the nn. state_change will also trigger this callback + * directly when it sees errors. In that case we need to call set_nn_state + * ourselves as state_change couldn't get the nn_lock and call set_nn_state + * itself. + */ +static void o2net_shutdown_sc(struct work_struct *work) +{ + struct o2net_sock_container *sc = + container_of(work, struct o2net_sock_container, + sc_shutdown_work); + struct o2net_node *nn = o2net_nn_from_num(sc->sc_node->nd_num); + + sclog(sc, "shutting down\n"); + + /* drop the callbacks ref and call shutdown only once */ + if (o2net_unregister_callbacks(sc->sc_sock->sk, sc)) { + /* we shouldn't flush as we're in the thread, the + * races with pending sc work structs are harmless */ + timer_delete_sync(&sc->sc_idle_timeout); + o2net_sc_cancel_delayed_work(sc, &sc->sc_keepalive_work); + sc_put(sc); + kernel_sock_shutdown(sc->sc_sock, SHUT_RDWR); + } + + /* not fatal so failed connects before the other guy has our + * heartbeat can be retried */ + o2net_ensure_shutdown(nn, sc, 0); + sc_put(sc); +} + +/* ------------------------------------------------------------ */ + +static int o2net_handler_cmp(struct o2net_msg_handler *nmh, u32 msg_type, + u32 key) +{ + int ret = memcmp(&nmh->nh_key, &key, sizeof(key)); + + if (ret == 0) + ret = memcmp(&nmh->nh_msg_type, &msg_type, sizeof(msg_type)); + + return ret; +} + +static struct o2net_msg_handler * +o2net_handler_tree_lookup(u32 msg_type, u32 key, struct rb_node ***ret_p, + struct rb_node **ret_parent) +{ + struct rb_node **p = &o2net_handler_tree.rb_node; + struct rb_node *parent = NULL; + struct o2net_msg_handler *nmh, *ret = NULL; + int cmp; + + while (*p) { + parent = *p; + nmh = rb_entry(parent, struct o2net_msg_handler, nh_node); + cmp = o2net_handler_cmp(nmh, msg_type, key); + + if (cmp < 0) + p = &(*p)->rb_left; + else if (cmp > 0) + p = &(*p)->rb_right; + else { + ret = nmh; + break; + } + } + + if (ret_p != NULL) + *ret_p = p; + if (ret_parent != NULL) + *ret_parent = parent; + + return ret; +} + +static void o2net_handler_kref_release(struct kref *kref) +{ + struct o2net_msg_handler *nmh; + nmh = container_of(kref, struct o2net_msg_handler, nh_kref); + + kfree(nmh); +} + +static void o2net_handler_put(struct o2net_msg_handler *nmh) +{ + kref_put(&nmh->nh_kref, o2net_handler_kref_release); +} + +/* max_len is protection for the handler func. incoming messages won't + * be given to the handler if their payload is longer than the max. */ +int o2net_register_handler(u32 msg_type, u32 key, u32 max_len, + o2net_msg_handler_func *func, void *data, + o2net_post_msg_handler_func *post_func, + struct list_head *unreg_list) +{ + struct o2net_msg_handler *nmh = NULL; + struct rb_node **p, *parent; + int ret = 0; + + if (max_len > O2NET_MAX_PAYLOAD_BYTES) { + mlog(0, "max_len for message handler out of range: %u\n", + max_len); + ret = -EINVAL; + goto out; + } + + if (!msg_type) { + mlog(0, "no message type provided: %u, %p\n", msg_type, func); + ret = -EINVAL; + goto out; + + } + if (!func) { + mlog(0, "no message handler provided: %u, %p\n", + msg_type, func); + ret = -EINVAL; + goto out; + } + + nmh = kzalloc_obj(struct o2net_msg_handler, GFP_NOFS); + if (nmh == NULL) { + ret = -ENOMEM; + goto out; + } + + nmh->nh_func = func; + nmh->nh_func_data = data; + nmh->nh_post_func = post_func; + nmh->nh_msg_type = msg_type; + nmh->nh_max_len = max_len; + nmh->nh_key = key; + /* the tree and list get this ref.. they're both removed in + * unregister when this ref is dropped */ + kref_init(&nmh->nh_kref); + INIT_LIST_HEAD(&nmh->nh_unregister_item); + + write_lock(&o2net_handler_lock); + if (o2net_handler_tree_lookup(msg_type, key, &p, &parent)) + ret = -EEXIST; + else { + rb_link_node(&nmh->nh_node, parent, p); + rb_insert_color(&nmh->nh_node, &o2net_handler_tree); + list_add_tail(&nmh->nh_unregister_item, unreg_list); + + mlog(ML_TCP, "registered handler func %p type %u key %08x\n", + func, msg_type, key); + /* we've had some trouble with handlers seemingly vanishing. */ + mlog_bug_on_msg(o2net_handler_tree_lookup(msg_type, key, &p, + &parent) == NULL, + "couldn't find handler we *just* registered " + "for type %u key %08x\n", msg_type, key); + } + write_unlock(&o2net_handler_lock); + +out: + if (ret) + kfree(nmh); + + return ret; +} +EXPORT_SYMBOL_GPL(o2net_register_handler); + +void o2net_unregister_handler_list(struct list_head *list) +{ + struct o2net_msg_handler *nmh, *n; + + write_lock(&o2net_handler_lock); + list_for_each_entry_safe(nmh, n, list, nh_unregister_item) { + mlog(ML_TCP, "unregistering handler func %p type %u key %08x\n", + nmh->nh_func, nmh->nh_msg_type, nmh->nh_key); + rb_erase(&nmh->nh_node, &o2net_handler_tree); + list_del_init(&nmh->nh_unregister_item); + kref_put(&nmh->nh_kref, o2net_handler_kref_release); + } + write_unlock(&o2net_handler_lock); +} +EXPORT_SYMBOL_GPL(o2net_unregister_handler_list); + +static void o2net_flush_wq(void) +{ + mutex_lock(&o2net_wq_mutex); + if (o2net_wq_destroying) { + mutex_unlock(&o2net_wq_mutex); + wait_for_completion(&o2net_wq_destroyed); + return; + } + + if (o2net_wq) + flush_workqueue(o2net_wq); + mutex_unlock(&o2net_wq_mutex); +} + +void o2net_unregister_and_flush_handler_list(struct list_head *list) +{ + o2net_unregister_handler_list(list); + o2net_flush_wq(); +} +EXPORT_SYMBOL_GPL(o2net_unregister_and_flush_handler_list); + +static struct o2net_msg_handler *o2net_handler_get(u32 msg_type, u32 key) +{ + struct o2net_msg_handler *nmh; + + read_lock(&o2net_handler_lock); + nmh = o2net_handler_tree_lookup(msg_type, key, NULL, NULL); + if (nmh) + kref_get(&nmh->nh_kref); + read_unlock(&o2net_handler_lock); + + return nmh; +} + +/* ------------------------------------------------------------ */ + +static int o2net_recv_tcp_msg(struct socket *sock, void *data, size_t len) +{ + struct kvec vec = { .iov_len = len, .iov_base = data, }; + struct msghdr msg = { .msg_flags = MSG_DONTWAIT, }; + iov_iter_kvec(&msg.msg_iter, ITER_DEST, &vec, 1, len); + return sock_recvmsg(sock, &msg, MSG_DONTWAIT); +} + +static int o2net_send_tcp_msg(struct socket *sock, struct kvec *vec, + size_t veclen, size_t total) +{ + int ret; + struct msghdr msg = {.msg_flags = 0,}; + + if (sock == NULL) { + ret = -EINVAL; + goto out; + } + + ret = kernel_sendmsg(sock, &msg, vec, veclen, total); + if (likely(ret == total)) + return 0; + mlog(ML_ERROR, "sendmsg returned %d instead of %zu\n", ret, total); + if (ret >= 0) + ret = -EPIPE; /* should be smarter, I bet */ +out: + mlog(0, "returning error: %d\n", ret); + return ret; +} + +static void o2net_sendpage(struct o2net_sock_container *sc, + void *virt, size_t size) +{ + struct o2net_node *nn = o2net_nn_from_num(sc->sc_node->nd_num); + struct msghdr msg = {}; + struct bio_vec bv; + ssize_t ret; + + bvec_set_virt(&bv, virt, size); + iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bv, 1, size); + + while (1) { + msg.msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES; + mutex_lock(&sc->sc_send_lock); + ret = sock_sendmsg(sc->sc_sock, &msg); + mutex_unlock(&sc->sc_send_lock); + + if (ret == size) + break; + if (ret == (ssize_t)-EAGAIN) { + mlog(0, "sendpage of size %zu to " SC_NODEF_FMT + " returned EAGAIN\n", size, SC_NODEF_ARGS(sc)); + cond_resched(); + continue; + } + mlog(ML_ERROR, "sendpage of size %zu to " SC_NODEF_FMT + " failed with %zd\n", size, SC_NODEF_ARGS(sc), ret); + o2net_ensure_shutdown(nn, sc, 0); + break; + } +} + +static void o2net_init_msg(struct o2net_msg *msg, u16 data_len, u16 msg_type, u32 key) +{ + memset(msg, 0, sizeof(struct o2net_msg)); + msg->magic = cpu_to_be16(O2NET_MSG_MAGIC); + msg->data_len = cpu_to_be16(data_len); + msg->msg_type = cpu_to_be16(msg_type); + msg->sys_status = cpu_to_be32(O2NET_ERR_NONE); + msg->status = 0; + msg->key = cpu_to_be32(key); +} + +static int o2net_tx_can_proceed(struct o2net_node *nn, + struct o2net_sock_container **sc_ret, + int *error) +{ + int ret = 0; + + spin_lock(&nn->nn_lock); + if (nn->nn_persistent_error) { + ret = 1; + *sc_ret = NULL; + *error = nn->nn_persistent_error; + } else if (nn->nn_sc_valid) { + kref_get(&nn->nn_sc->sc_kref); + + ret = 1; + *sc_ret = nn->nn_sc; + *error = 0; + } + spin_unlock(&nn->nn_lock); + + return ret; +} + +/* Get a map of all nodes to which this node is currently connected to */ +void o2net_fill_node_map(unsigned long *map, unsigned int bits) +{ + struct o2net_sock_container *sc; + int node, ret; + + bitmap_zero(map, bits); + for (node = 0; node < O2NM_MAX_NODES; ++node) { + if (!o2net_tx_can_proceed(o2net_nn_from_num(node), &sc, &ret)) + continue; + if (!ret) { + set_bit(node, map); + sc_put(sc); + } + } +} +EXPORT_SYMBOL_GPL(o2net_fill_node_map); + +int o2net_send_message_vec(u32 msg_type, u32 key, struct kvec *caller_vec, + size_t caller_veclen, u8 target_node, int *status) +{ + int ret = 0; + struct o2net_msg *msg = NULL; + size_t veclen, caller_bytes = 0; + struct kvec *vec = NULL; + struct o2net_sock_container *sc = NULL; + struct o2net_node *nn = o2net_nn_from_num(target_node); + struct o2net_status_wait nsw = { + .ns_node_item = LIST_HEAD_INIT(nsw.ns_node_item), + }; + struct o2net_send_tracking nst; + + o2net_init_nst(&nst, msg_type, key, current, target_node); + + if (o2net_wq == NULL) { + mlog(0, "attempt to tx without o2netd running\n"); + ret = -ESRCH; + goto out; + } + + if (caller_veclen == 0) { + mlog(0, "bad kvec array length\n"); + ret = -EINVAL; + goto out; + } + + caller_bytes = iov_length((struct iovec *)caller_vec, caller_veclen); + if (caller_bytes > O2NET_MAX_PAYLOAD_BYTES) { + mlog(0, "total payload len %zu too large\n", caller_bytes); + ret = -EINVAL; + goto out; + } + + if (target_node == o2nm_this_node()) { + ret = -ELOOP; + goto out; + } + + o2net_debug_add_nst(&nst); + + o2net_set_nst_sock_time(&nst); + + wait_event(nn->nn_sc_wq, o2net_tx_can_proceed(nn, &sc, &ret)); + if (ret) + goto out; + + o2net_set_nst_sock_container(&nst, sc); + + veclen = caller_veclen + 1; + vec = kmalloc_objs(struct kvec, veclen, GFP_ATOMIC); + if (vec == NULL) { + mlog(0, "failed to %zu element kvec!\n", veclen); + ret = -ENOMEM; + goto out; + } + + msg = kmalloc_obj(struct o2net_msg, GFP_ATOMIC); + if (!msg) { + mlog(0, "failed to allocate a o2net_msg!\n"); + ret = -ENOMEM; + goto out; + } + + o2net_init_msg(msg, caller_bytes, msg_type, key); + + vec[0].iov_len = sizeof(struct o2net_msg); + vec[0].iov_base = msg; + memcpy(&vec[1], caller_vec, caller_veclen * sizeof(struct kvec)); + + ret = o2net_prep_nsw(nn, &nsw); + if (ret) + goto out; + + msg->msg_num = cpu_to_be32(nsw.ns_id); + o2net_set_nst_msg_id(&nst, nsw.ns_id); + + o2net_set_nst_send_time(&nst); + + /* finally, convert the message header to network byte-order + * and send */ + mutex_lock(&sc->sc_send_lock); + ret = o2net_send_tcp_msg(sc->sc_sock, vec, veclen, + sizeof(struct o2net_msg) + caller_bytes); + mutex_unlock(&sc->sc_send_lock); + msglog(msg, "sending returned %d\n", ret); + if (ret < 0) { + mlog(0, "error returned from o2net_send_tcp_msg=%d\n", ret); + goto out; + } + + /* wait on other node's handler */ + o2net_set_nst_status_time(&nst); + wait_event(nsw.ns_wq, o2net_nsw_completed(nn, &nsw)); + + o2net_update_send_stats(&nst, sc); + + /* Note that we avoid overwriting the callers status return + * variable if a system error was reported on the other + * side. Callers beware. */ + ret = o2net_sys_err_to_errno(nsw.ns_sys_status); + if (status && !ret) + *status = nsw.ns_status; + + mlog(0, "woken, returning system status %d, user status %d\n", + ret, nsw.ns_status); +out: + o2net_debug_del_nst(&nst); /* must be before dropping sc and node */ + if (sc) + sc_put(sc); + kfree(vec); + kfree(msg); + o2net_complete_nsw(nn, &nsw, 0, 0, 0); + return ret; +} +EXPORT_SYMBOL_GPL(o2net_send_message_vec); + +int o2net_send_message(u32 msg_type, u32 key, void *data, u32 len, + u8 target_node, int *status) +{ + struct kvec vec = { + .iov_base = data, + .iov_len = len, + }; + return o2net_send_message_vec(msg_type, key, &vec, 1, + target_node, status); +} +EXPORT_SYMBOL_GPL(o2net_send_message); + +static int o2net_send_status_magic(struct socket *sock, struct o2net_msg *hdr, + enum o2net_system_error syserr, int err) +{ + struct kvec vec = { + .iov_base = hdr, + .iov_len = sizeof(struct o2net_msg), + }; + + BUG_ON(syserr >= O2NET_ERR_MAX); + + /* leave other fields intact from the incoming message, msg_num + * in particular */ + hdr->sys_status = cpu_to_be32(syserr); + hdr->status = cpu_to_be32(err); + hdr->magic = cpu_to_be16(O2NET_MSG_STATUS_MAGIC); // twiddle the magic + hdr->data_len = 0; + + msglog(hdr, "about to send status magic %d\n", err); + /* hdr has been in host byteorder this whole time */ + return o2net_send_tcp_msg(sock, &vec, 1, sizeof(struct o2net_msg)); +} + +/* this returns -errno if the header was unknown or too large, etc. + * after this is called the buffer us reused for the next message */ +static int o2net_process_message(struct o2net_sock_container *sc, + struct o2net_msg *hdr) +{ + struct o2net_node *nn = o2net_nn_from_num(sc->sc_node->nd_num); + int ret = 0, handler_status; + enum o2net_system_error syserr; + struct o2net_msg_handler *nmh = NULL; + void *ret_data = NULL; + + msglog(hdr, "processing message\n"); + + o2net_sc_postpone_idle(sc); + + switch(be16_to_cpu(hdr->magic)) { + case O2NET_MSG_STATUS_MAGIC: + /* special type for returning message status */ + o2net_complete_nsw(nn, NULL, + be32_to_cpu(hdr->msg_num), + be32_to_cpu(hdr->sys_status), + be32_to_cpu(hdr->status)); + goto out; + case O2NET_MSG_KEEP_REQ_MAGIC: + o2net_sendpage(sc, o2net_keep_resp, + sizeof(*o2net_keep_resp)); + goto out; + case O2NET_MSG_KEEP_RESP_MAGIC: + goto out; + case O2NET_MSG_MAGIC: + break; + default: + msglog(hdr, "bad magic\n"); + ret = -EINVAL; + goto out; + } + + /* find a handler for it */ + handler_status = 0; + nmh = o2net_handler_get(be16_to_cpu(hdr->msg_type), + be32_to_cpu(hdr->key)); + if (!nmh) { + mlog(ML_TCP, "couldn't find handler for type %u key %08x\n", + be16_to_cpu(hdr->msg_type), be32_to_cpu(hdr->key)); + syserr = O2NET_ERR_NO_HNDLR; + goto out_respond; + } + + syserr = O2NET_ERR_NONE; + + if (be16_to_cpu(hdr->data_len) > nmh->nh_max_len) + syserr = O2NET_ERR_OVERFLOW; + + if (syserr != O2NET_ERR_NONE) + goto out_respond; + + o2net_set_func_start_time(sc); + sc->sc_msg_key = be32_to_cpu(hdr->key); + sc->sc_msg_type = be16_to_cpu(hdr->msg_type); + handler_status = (nmh->nh_func)(hdr, sizeof(struct o2net_msg) + + be16_to_cpu(hdr->data_len), + nmh->nh_func_data, &ret_data); + o2net_set_func_stop_time(sc); + + o2net_update_recv_stats(sc); + +out_respond: + /* this destroys the hdr, so don't use it after this */ + mutex_lock(&sc->sc_send_lock); + ret = o2net_send_status_magic(sc->sc_sock, hdr, syserr, + handler_status); + mutex_unlock(&sc->sc_send_lock); + hdr = NULL; + mlog(0, "sending handler status %d, syserr %d returned %d\n", + handler_status, syserr, ret); + + if (nmh) { + BUG_ON(ret_data != NULL && nmh->nh_post_func == NULL); + if (nmh->nh_post_func) + (nmh->nh_post_func)(handler_status, nmh->nh_func_data, + ret_data); + } + +out: + if (nmh) + o2net_handler_put(nmh); + return ret; +} + +static int o2net_check_handshake(struct o2net_sock_container *sc) +{ + struct o2net_handshake *hand = page_address(sc->sc_page); + struct o2net_node *nn = o2net_nn_from_num(sc->sc_node->nd_num); + + if (hand->protocol_version != cpu_to_be64(O2NET_PROTOCOL_VERSION)) { + printk(KERN_NOTICE "o2net: " SC_NODEF_FMT " Advertised net " + "protocol version %llu but %llu is required. " + "Disconnecting.\n", SC_NODEF_ARGS(sc), + (unsigned long long)be64_to_cpu(hand->protocol_version), + O2NET_PROTOCOL_VERSION); + + /* don't bother reconnecting if its the wrong version. */ + o2net_ensure_shutdown(nn, sc, -ENOTCONN); + return -1; + } + + /* + * Ensure timeouts are consistent with other nodes, otherwise + * we can end up with one node thinking that the other must be down, + * but isn't. This can ultimately cause corruption. + */ + if (be32_to_cpu(hand->o2net_idle_timeout_ms) != + o2net_idle_timeout()) { + printk(KERN_NOTICE "o2net: " SC_NODEF_FMT " uses a network " + "idle timeout of %u ms, but we use %u ms locally. " + "Disconnecting.\n", SC_NODEF_ARGS(sc), + be32_to_cpu(hand->o2net_idle_timeout_ms), + o2net_idle_timeout()); + o2net_ensure_shutdown(nn, sc, -ENOTCONN); + return -1; + } + + if (be32_to_cpu(hand->o2net_keepalive_delay_ms) != + o2net_keepalive_delay()) { + printk(KERN_NOTICE "o2net: " SC_NODEF_FMT " uses a keepalive " + "delay of %u ms, but we use %u ms locally. " + "Disconnecting.\n", SC_NODEF_ARGS(sc), + be32_to_cpu(hand->o2net_keepalive_delay_ms), + o2net_keepalive_delay()); + o2net_ensure_shutdown(nn, sc, -ENOTCONN); + return -1; + } + + if (be32_to_cpu(hand->o2hb_heartbeat_timeout_ms) != + O2HB_MAX_WRITE_TIMEOUT_MS) { + printk(KERN_NOTICE "o2net: " SC_NODEF_FMT " uses a heartbeat " + "timeout of %u ms, but we use %u ms locally. " + "Disconnecting.\n", SC_NODEF_ARGS(sc), + be32_to_cpu(hand->o2hb_heartbeat_timeout_ms), + O2HB_MAX_WRITE_TIMEOUT_MS); + o2net_ensure_shutdown(nn, sc, -ENOTCONN); + return -1; + } + + sc->sc_handshake_ok = 1; + + spin_lock(&nn->nn_lock); + /* set valid and queue the idle timers only if it hasn't been + * shut down already */ + if (nn->nn_sc == sc) { + o2net_sc_reset_idle_timer(sc); + atomic_set(&nn->nn_timeout, 0); + o2net_set_nn_state(nn, sc, 1, 0); + } + spin_unlock(&nn->nn_lock); + + /* shift everything up as though it wasn't there */ + sc->sc_page_off -= sizeof(struct o2net_handshake); + if (sc->sc_page_off) + memmove(hand, hand + 1, sc->sc_page_off); + + return 0; +} + +/* this demuxes the queued rx bytes into header or payload bits and calls + * handlers as each full message is read off the socket. it returns -error, + * == 0 eof, or > 0 for progress made.*/ +static int o2net_advance_rx(struct o2net_sock_container *sc) +{ + struct o2net_msg *hdr; + int ret = 0; + void *data; + size_t datalen; + + sclog(sc, "receiving\n"); + o2net_set_advance_start_time(sc); + + if (unlikely(sc->sc_handshake_ok == 0)) { + if(sc->sc_page_off < sizeof(struct o2net_handshake)) { + data = page_address(sc->sc_page) + sc->sc_page_off; + datalen = sizeof(struct o2net_handshake) - sc->sc_page_off; + ret = o2net_recv_tcp_msg(sc->sc_sock, data, datalen); + if (ret > 0) + sc->sc_page_off += ret; + } + + if (sc->sc_page_off == sizeof(struct o2net_handshake)) { + o2net_check_handshake(sc); + if (unlikely(sc->sc_handshake_ok == 0)) + ret = -EPROTO; + } + goto out; + } + + /* do we need more header? */ + if (sc->sc_page_off < sizeof(struct o2net_msg)) { + data = page_address(sc->sc_page) + sc->sc_page_off; + datalen = sizeof(struct o2net_msg) - sc->sc_page_off; + ret = o2net_recv_tcp_msg(sc->sc_sock, data, datalen); + if (ret > 0) { + sc->sc_page_off += ret; + /* only swab incoming here.. we can + * only get here once as we cross from + * being under to over */ + if (sc->sc_page_off == sizeof(struct o2net_msg)) { + hdr = page_address(sc->sc_page); + if (be16_to_cpu(hdr->data_len) > + O2NET_MAX_PAYLOAD_BYTES) + ret = -EOVERFLOW; + } + } + if (ret <= 0) + goto out; + } + + if (sc->sc_page_off < sizeof(struct o2net_msg)) { + /* oof, still don't have a header */ + goto out; + } + + /* this was swabbed above when we first read it */ + hdr = page_address(sc->sc_page); + + msglog(hdr, "at page_off %zu\n", sc->sc_page_off); + + /* do we need more payload? */ + if (sc->sc_page_off - sizeof(struct o2net_msg) < be16_to_cpu(hdr->data_len)) { + /* need more payload */ + data = page_address(sc->sc_page) + sc->sc_page_off; + datalen = (sizeof(struct o2net_msg) + be16_to_cpu(hdr->data_len)) - + sc->sc_page_off; + ret = o2net_recv_tcp_msg(sc->sc_sock, data, datalen); + if (ret > 0) + sc->sc_page_off += ret; + if (ret <= 0) + goto out; + } + + if (sc->sc_page_off - sizeof(struct o2net_msg) == be16_to_cpu(hdr->data_len)) { + /* we can only get here once, the first time we read + * the payload.. so set ret to progress if the handler + * works out. after calling this the message is toast */ + ret = o2net_process_message(sc, hdr); + if (ret == 0) + ret = 1; + sc->sc_page_off = 0; + } + +out: + sclog(sc, "ret = %d\n", ret); + o2net_set_advance_stop_time(sc); + return ret; +} + +/* this work func is triggered by data ready. it reads until it can read no + * more. it interprets 0, eof, as fatal. if data_ready hits while we're doing + * our work the work struct will be marked and we'll be called again. */ +static void o2net_rx_until_empty(struct work_struct *work) +{ + struct o2net_sock_container *sc = + container_of(work, struct o2net_sock_container, sc_rx_work); + int ret; + + do { + ret = o2net_advance_rx(sc); + } while (ret > 0); + + if (ret <= 0 && ret != -EAGAIN) { + struct o2net_node *nn = o2net_nn_from_num(sc->sc_node->nd_num); + sclog(sc, "saw error %d, closing\n", ret); + /* not permanent so read failed handshake can retry */ + o2net_ensure_shutdown(nn, sc, 0); + } + + sc_put(sc); +} + +static void o2net_initialize_handshake(void) +{ + o2net_hand->o2hb_heartbeat_timeout_ms = cpu_to_be32( + O2HB_MAX_WRITE_TIMEOUT_MS); + o2net_hand->o2net_idle_timeout_ms = cpu_to_be32(o2net_idle_timeout()); + o2net_hand->o2net_keepalive_delay_ms = cpu_to_be32( + o2net_keepalive_delay()); + o2net_hand->o2net_reconnect_delay_ms = cpu_to_be32( + o2net_reconnect_delay()); +} + +/* ------------------------------------------------------------ */ + +/* called when a connect completes and after a sock is accepted. the + * rx path will see the response and mark the sc valid */ +static void o2net_sc_connect_completed(struct work_struct *work) +{ + struct o2net_sock_container *sc = + container_of(work, struct o2net_sock_container, + sc_connect_work); + + mlog(ML_MSG, "sc sending handshake with ver %llu id %llx\n", + (unsigned long long)O2NET_PROTOCOL_VERSION, + (unsigned long long)be64_to_cpu(o2net_hand->connector_id)); + + o2net_initialize_handshake(); + o2net_sendpage(sc, o2net_hand, sizeof(*o2net_hand)); + sc_put(sc); +} + +/* this is called as a work_struct func. */ +static void o2net_sc_send_keep_req(struct work_struct *work) +{ + struct o2net_sock_container *sc = + container_of(work, struct o2net_sock_container, + sc_keepalive_work.work); + + o2net_sendpage(sc, o2net_keep_req, sizeof(*o2net_keep_req)); + sc_put(sc); +} + +/* socket shutdown does a timer_delete_sync against this as it tears down. + * we can't start this timer until we've got to the point in sc buildup + * where shutdown is going to be involved */ +static void o2net_idle_timer(struct timer_list *t) +{ + struct o2net_sock_container *sc = timer_container_of(sc, t, + sc_idle_timeout); + struct o2net_node *nn = o2net_nn_from_num(sc->sc_node->nd_num); +#ifdef CONFIG_DEBUG_FS + unsigned long msecs = ktime_to_ms(ktime_get()) - + ktime_to_ms(sc->sc_tv_timer); +#else + unsigned long msecs = o2net_idle_timeout(); +#endif + + printk(KERN_NOTICE "o2net: Connection to " SC_NODEF_FMT " has been " + "idle for %lu.%lu secs.\n", + SC_NODEF_ARGS(sc), msecs / 1000, msecs % 1000); + + /* idle timerout happen, don't shutdown the connection, but + * make fence decision. Maybe the connection can recover before + * the decision is made. + */ + atomic_set(&nn->nn_timeout, 1); + o2quo_conn_err(o2net_num_from_nn(nn)); + queue_delayed_work(o2net_wq, &nn->nn_still_up, + msecs_to_jiffies(O2NET_QUORUM_DELAY_MS)); + + o2net_sc_reset_idle_timer(sc); + +} + +static void o2net_sc_reset_idle_timer(struct o2net_sock_container *sc) +{ + o2net_sc_cancel_delayed_work(sc, &sc->sc_keepalive_work); + o2net_sc_queue_delayed_work(sc, &sc->sc_keepalive_work, + msecs_to_jiffies(o2net_keepalive_delay())); + o2net_set_sock_timer(sc); + mod_timer(&sc->sc_idle_timeout, + jiffies + msecs_to_jiffies(o2net_idle_timeout())); +} + +static void o2net_sc_postpone_idle(struct o2net_sock_container *sc) +{ + struct o2net_node *nn = o2net_nn_from_num(sc->sc_node->nd_num); + + /* clear fence decision since the connection recover from timeout*/ + if (atomic_read(&nn->nn_timeout)) { + o2quo_conn_up(o2net_num_from_nn(nn)); + cancel_delayed_work(&nn->nn_still_up); + atomic_set(&nn->nn_timeout, 0); + } + + /* Only push out an existing timer */ + if (timer_pending(&sc->sc_idle_timeout)) + o2net_sc_reset_idle_timer(sc); +} + +/* this work func is kicked whenever a path sets the nn state which doesn't + * have valid set. This includes seeing hb come up, losing a connection, + * having a connect attempt fail, etc. This centralizes the logic which decides + * if a connect attempt should be made or if we should give up and all future + * transmit attempts should fail */ +static void o2net_start_connect(struct work_struct *work) +{ + struct o2net_node *nn = + container_of(work, struct o2net_node, nn_connect_work.work); + struct o2net_sock_container *sc = NULL; + struct o2nm_node *node = NULL, *mynode = NULL; + struct socket *sock = NULL; + struct sockaddr_in myaddr = {0, }, remoteaddr = {0, }; + int ret = 0, stop; + unsigned int timeout; + unsigned int nofs_flag; + + /* + * sock_create allocates the sock with GFP_KERNEL. We must + * prevent the filesystem from being reentered by memory reclaim. + */ + nofs_flag = memalloc_nofs_save(); + /* if we're greater we initiate tx, otherwise we accept */ + if (o2nm_this_node() <= o2net_num_from_nn(nn)) + goto out; + + /* watch for racing with tearing a node down */ + node = o2nm_get_node_by_num(o2net_num_from_nn(nn)); + if (node == NULL) + goto out; + + mynode = o2nm_get_node_by_num(o2nm_this_node()); + if (mynode == NULL) + goto out; + + spin_lock(&nn->nn_lock); + /* + * see if we already have one pending or have given up. + * For nn_timeout, it is set when we close the connection + * because of the idle time out. So it means that we have + * at least connected to that node successfully once, + * now try to connect to it again. + */ + timeout = atomic_read(&nn->nn_timeout); + stop = (nn->nn_sc || + (nn->nn_persistent_error && + (nn->nn_persistent_error != -ENOTCONN || timeout == 0))); + spin_unlock(&nn->nn_lock); + if (stop) + goto out; + + nn->nn_last_connect_attempt = jiffies; + + sc = sc_alloc(node); + if (sc == NULL) { + mlog(0, "couldn't allocate sc\n"); + ret = -ENOMEM; + goto out; + } + + ret = sock_create(PF_INET, SOCK_STREAM, IPPROTO_TCP, &sock); + if (ret < 0) { + mlog(0, "can't create socket: %d\n", ret); + goto out; + } + sc->sc_sock = sock; /* freed by sc_kref_release */ + + sock->sk->sk_allocation = GFP_ATOMIC; + sock->sk->sk_use_task_frag = false; + + myaddr.sin_family = AF_INET; + myaddr.sin_addr.s_addr = mynode->nd_ipv4_address; + myaddr.sin_port = htons(0); /* any port */ + + ret = sock->ops->bind(sock, (struct sockaddr_unsized *)&myaddr, + sizeof(myaddr)); + if (ret) { + mlog(ML_ERROR, "bind failed with %d at address %pI4\n", + ret, &mynode->nd_ipv4_address); + goto out; + } + + tcp_sock_set_nodelay(sc->sc_sock->sk); + tcp_sock_set_user_timeout(sock->sk, O2NET_TCP_USER_TIMEOUT); + + o2net_register_callbacks(sc->sc_sock->sk, sc); + + spin_lock(&nn->nn_lock); + /* handshake completion will set nn->nn_sc_valid */ + o2net_set_nn_state(nn, sc, 0, 0); + spin_unlock(&nn->nn_lock); + + remoteaddr.sin_family = AF_INET; + remoteaddr.sin_addr.s_addr = node->nd_ipv4_address; + remoteaddr.sin_port = node->nd_ipv4_port; + + ret = sc->sc_sock->ops->connect(sc->sc_sock, + (struct sockaddr_unsized *)&remoteaddr, + sizeof(remoteaddr), + O_NONBLOCK); + if (ret == -EINPROGRESS) + ret = 0; + +out: + if (ret && sc) { + printk(KERN_NOTICE "o2net: Connect attempt to " SC_NODEF_FMT + " failed with errno %d\n", SC_NODEF_ARGS(sc), ret); + /* 0 err so that another will be queued and attempted + * from set_nn_state */ + o2net_ensure_shutdown(nn, sc, 0); + } + if (sc) + sc_put(sc); + if (node) + o2nm_node_put(node); + if (mynode) + o2nm_node_put(mynode); + + memalloc_nofs_restore(nofs_flag); + return; +} + +static void o2net_connect_expired(struct work_struct *work) +{ + struct o2net_node *nn = + container_of(work, struct o2net_node, nn_connect_expired.work); + + spin_lock(&nn->nn_lock); + if (!nn->nn_sc_valid) { + printk(KERN_NOTICE "o2net: No connection established with " + "node %u after %u.%u seconds, check network and" + " cluster configuration.\n", + o2net_num_from_nn(nn), + o2net_idle_timeout() / 1000, + o2net_idle_timeout() % 1000); + + o2net_set_nn_state(nn, NULL, 0, 0); + } + spin_unlock(&nn->nn_lock); +} + +static void o2net_still_up(struct work_struct *work) +{ + struct o2net_node *nn = + container_of(work, struct o2net_node, nn_still_up.work); + + o2quo_hb_still_up(o2net_num_from_nn(nn)); +} + +/* ------------------------------------------------------------ */ + +static void o2net_hb_node_up(struct o2net_node *nn) +{ + /* ensure an immediate connect attempt */ + nn->nn_last_connect_attempt = jiffies - + (msecs_to_jiffies(o2net_reconnect_delay()) + 1); + + spin_lock(&nn->nn_lock); + atomic_set(&nn->nn_timeout, 0); + if (nn->nn_persistent_error) + o2net_set_nn_state(nn, NULL, 0, 0); + spin_unlock(&nn->nn_lock); +} + +void o2net_disconnect_node(struct o2nm_node *node) +{ + struct o2net_node *nn = o2net_nn_from_num(node->nd_num); + + /* don't reconnect until it's heartbeating again */ + spin_lock(&nn->nn_lock); + atomic_set(&nn->nn_timeout, 0); + o2net_set_nn_state(nn, NULL, 0, -ENOTCONN); + spin_unlock(&nn->nn_lock); + + cancel_delayed_work(&nn->nn_connect_expired); + cancel_delayed_work(&nn->nn_connect_work); + cancel_delayed_work(&nn->nn_still_up); + o2net_flush_wq(); +} + +static void o2net_hb_node_down_cb(struct o2nm_node *node, int node_num, + void *data) +{ + u8 this_node; + + o2quo_hb_down(node_num); + + if (!node) + goto out; + + this_node = o2nm_this_node(); + if (!READ_ONCE(o2net_listening) || this_node == O2NM_MAX_NODES) + goto out; + + if (node_num != this_node) + o2net_disconnect_node(node); + +out: + BUG_ON(atomic_read(&o2net_connected_peers) < 0); +} + +static void o2net_hb_node_up_cb(struct o2nm_node *node, int node_num, + void *data) +{ + u8 this_node; + + o2quo_hb_up(node_num); + + BUG_ON(!node); + + this_node = o2nm_this_node(); + if (!READ_ONCE(o2net_listening) || this_node == O2NM_MAX_NODES) + return; + + if (node_num != this_node) + o2net_hb_node_up(o2net_nn_from_num(node_num)); +} + +void o2net_unregister_hb_callbacks(void) +{ + o2hb_unregister_callback(NULL, &o2net_hb_up); + o2hb_unregister_callback(NULL, &o2net_hb_down); +} + +/* + * Delay heartbeat-driven network work until the local node is fully published + * through o2nm_this_node(), then replay the nodes that are already live while + * callback delivery stays blocked. + */ +void o2net_complete_start_listening(struct o2nm_node *node) +{ + unsigned long live_nodes[BITS_TO_LONGS(O2NM_MAX_NODES)]; + unsigned long node_num; + u8 local_node; + + local_node = o2nm_this_node(); + if (WARN_ON_ONCE(local_node == O2NM_MAX_NODES)) + return; + if (WARN_ON_ONCE(local_node != node->nd_num)) + return; + if (WARN_ON_ONCE(!o2net_wq)) + return; + + o2hb_callback_read_lock(); + WRITE_ONCE(o2net_listening, true); + o2hb_fill_node_map_locked(live_nodes, O2NM_MAX_NODES); + for_each_set_bit(node_num, live_nodes, O2NM_MAX_NODES) { + if (node_num == local_node) + continue; + + o2net_hb_node_up(o2net_nn_from_num(node_num)); + } + o2hb_callback_read_unlock(); +} + +int o2net_register_hb_callbacks(void) +{ + int ret; + + o2hb_setup_callback(&o2net_hb_down, O2HB_NODE_DOWN_CB, + o2net_hb_node_down_cb, NULL, O2NET_HB_PRI); + o2hb_setup_callback(&o2net_hb_up, O2HB_NODE_UP_CB, + o2net_hb_node_up_cb, NULL, O2NET_HB_PRI); + + ret = o2hb_register_callback(NULL, &o2net_hb_up); + if (ret == 0) + ret = o2hb_register_callback(NULL, &o2net_hb_down); + + if (ret) + o2net_unregister_hb_callbacks(); + + return ret; +} + +/* ------------------------------------------------------------ */ + +static int o2net_accept_one(struct socket *sock, int *more) +{ + int ret; + struct sockaddr_in sin; + struct socket *new_sock = NULL; + struct o2nm_node *node = NULL; + struct o2nm_node *local_node = NULL; + struct o2net_sock_container *sc = NULL; + struct proto_accept_arg arg = { + .flags = O_NONBLOCK, + }; + struct o2net_node *nn; + unsigned int nofs_flag; + + /* + * sock_create_lite allocates the sock with GFP_KERNEL. We must + * prevent the filesystem from being reentered by memory reclaim. + */ + nofs_flag = memalloc_nofs_save(); + + BUG_ON(sock == NULL); + *more = 0; + ret = sock_create_lite(sock->sk->sk_family, sock->sk->sk_type, + sock->sk->sk_protocol, &new_sock); + if (ret) + goto out; + + new_sock->type = sock->type; + new_sock->ops = sock->ops; + ret = sock->ops->accept(sock, new_sock, &arg); + if (ret < 0) + goto out; + + *more = 1; + new_sock->sk->sk_allocation = GFP_ATOMIC; + + tcp_sock_set_nodelay(new_sock->sk); + tcp_sock_set_user_timeout(new_sock->sk, O2NET_TCP_USER_TIMEOUT); + + ret = new_sock->ops->getname(new_sock, (struct sockaddr *) &sin, 1); + if (ret < 0) + goto out; + + node = o2nm_get_node_by_ip(sin.sin_addr.s_addr); + if (node == NULL) { + printk(KERN_NOTICE "o2net: Attempt to connect from unknown " + "node at %pI4:%d\n", &sin.sin_addr.s_addr, + ntohs(sin.sin_port)); + ret = -EINVAL; + goto out; + } + + if (o2nm_this_node() >= node->nd_num) { + local_node = o2nm_get_node_by_num(o2nm_this_node()); + if (local_node) + printk(KERN_NOTICE "o2net: Unexpected connect attempt " + "seen at node '%s' (%u, %pI4:%d) from " + "node '%s' (%u, %pI4:%d)\n", + local_node->nd_name, local_node->nd_num, + &(local_node->nd_ipv4_address), + ntohs(local_node->nd_ipv4_port), + node->nd_name, + node->nd_num, &sin.sin_addr.s_addr, + ntohs(sin.sin_port)); + ret = -EINVAL; + goto out; + } + + /* this happens all the time when the other node sees our heartbeat + * and tries to connect before we see their heartbeat */ + if (!o2hb_check_node_heartbeating_from_callback(node->nd_num)) { + mlog(ML_CONN, "attempt to connect from node '%s' at " + "%pI4:%d but it isn't heartbeating\n", + node->nd_name, &sin.sin_addr.s_addr, + ntohs(sin.sin_port)); + ret = -EINVAL; + goto out; + } + + nn = o2net_nn_from_num(node->nd_num); + + spin_lock(&nn->nn_lock); + if (nn->nn_sc) + ret = -EBUSY; + else + ret = 0; + spin_unlock(&nn->nn_lock); + if (ret) { + printk(KERN_NOTICE "o2net: Attempt to connect from node '%s' " + "at %pI4:%d but it already has an open connection\n", + node->nd_name, &sin.sin_addr.s_addr, + ntohs(sin.sin_port)); + goto out; + } + + sc = sc_alloc(node); + if (sc == NULL) { + ret = -ENOMEM; + goto out; + } + + sc->sc_sock = new_sock; + new_sock = NULL; + + spin_lock(&nn->nn_lock); + atomic_set(&nn->nn_timeout, 0); + o2net_set_nn_state(nn, sc, 0, 0); + spin_unlock(&nn->nn_lock); + + o2net_register_callbacks(sc->sc_sock->sk, sc); + o2net_sc_queue_work(sc, &sc->sc_rx_work); + + o2net_initialize_handshake(); + o2net_sendpage(sc, o2net_hand, sizeof(*o2net_hand)); + +out: + if (new_sock) + sock_release(new_sock); + if (node) + o2nm_node_put(node); + if (local_node) + o2nm_node_put(local_node); + if (sc) + sc_put(sc); + + memalloc_nofs_restore(nofs_flag); + return ret; +} + +/* + * This function is invoked in response to one or more + * pending accepts at softIRQ level. We must drain the + * entire que before returning. + */ + +static void o2net_accept_many(struct work_struct *work) +{ + struct socket *sock = o2net_listen_sock; + int more; + + /* + * It is critical to note that due to interrupt moderation + * at the network driver level, we can't assume to get a + * softIRQ for every single conn since tcp SYN packets + * can arrive back-to-back, and therefore many pending + * accepts may result in just 1 softIRQ. If we terminate + * the o2net_accept_one() loop upon seeing an err, what happens + * to the rest of the conns in the queue? If no new SYN + * arrives for hours, no softIRQ will be delivered, + * and the connections will just sit in the queue. + */ + + for (;;) { + o2net_accept_one(sock, &more); + if (!more) + break; + cond_resched(); + } +} + +static void o2net_listen_data_ready(struct sock *sk) +{ + void (*ready)(struct sock *sk); + + trace_sk_data_ready(sk); + + read_lock_bh(&sk->sk_callback_lock); + ready = sk->sk_user_data; + if (ready == NULL) { /* check for teardown race */ + ready = sk->sk_data_ready; + goto out; + } + + /* This callback may called twice when a new connection + * is being established as a child socket inherits everything + * from a parent LISTEN socket, including the data_ready cb of + * the parent. This leads to a hazard. In o2net_accept_one() + * we are still initializing the child socket but have not + * changed the inherited data_ready callback yet when + * data starts arriving. + * We avoid this hazard by checking the state. + * For the listening socket, the state will be TCP_LISTEN; for the new + * socket, will be TCP_ESTABLISHED. Also, in this case, + * sk->sk_user_data is not a valid function pointer. + */ + + if (sk->sk_state == TCP_LISTEN) { + queue_work(o2net_wq, &o2net_listen_work); + } else { + ready = NULL; + } + +out: + read_unlock_bh(&sk->sk_callback_lock); + if (ready != NULL) + ready(sk); +} + +static int o2net_open_listening_sock(__be32 addr, __be16 port) +{ + struct socket *sock = NULL; + int ret; + struct sockaddr_in sin = { + .sin_family = PF_INET, + .sin_addr = { .s_addr = addr }, + .sin_port = port, + }; + + ret = sock_create(PF_INET, SOCK_STREAM, IPPROTO_TCP, &sock); + if (ret < 0) { + printk(KERN_ERR "o2net: Error %d while creating socket\n", ret); + goto out; + } + + sock->sk->sk_allocation = GFP_ATOMIC; + + write_lock_bh(&sock->sk->sk_callback_lock); + sock->sk->sk_user_data = sock->sk->sk_data_ready; + sock->sk->sk_data_ready = o2net_listen_data_ready; + write_unlock_bh(&sock->sk->sk_callback_lock); + + o2net_listen_sock = sock; + INIT_WORK(&o2net_listen_work, o2net_accept_many); + + sock->sk->sk_reuse = SK_CAN_REUSE; + ret = sock->ops->bind(sock, (struct sockaddr_unsized *)&sin, sizeof(sin)); + if (ret < 0) { + printk(KERN_ERR "o2net: Error %d while binding socket at " + "%pI4:%u\n", ret, &addr, ntohs(port)); + goto out; + } + + ret = sock->ops->listen(sock, 64); + if (ret < 0) + printk(KERN_ERR "o2net: Error %d while listening on %pI4:%u\n", + ret, &addr, ntohs(port)); + +out: + if (ret) { + o2net_listen_sock = NULL; + if (sock) + sock_release(sock); + } + return ret; +} + +static void o2net_destroy_wq(void) +{ + struct workqueue_struct *wq; + + mutex_lock(&o2net_wq_mutex); + if (o2net_wq_destroying) { + mutex_unlock(&o2net_wq_mutex); + wait_for_completion(&o2net_wq_destroyed); + return; + } + + wq = o2net_wq; + if (!wq) { + mutex_unlock(&o2net_wq_mutex); + return; + } + + reinit_completion(&o2net_wq_destroyed); + o2net_wq_destroying = wq; + mutex_unlock(&o2net_wq_mutex); + + destroy_workqueue(wq); + + mutex_lock(&o2net_wq_mutex); + o2net_wq = NULL; + o2net_wq_destroying = NULL; + complete_all(&o2net_wq_destroyed); + mutex_unlock(&o2net_wq_mutex); +} + +/* + * called from node manager when we should bring up our network listening + * socket. node manager handles all the serialization to only call this + * once and to match it with o2net_stop_listening(). note, + * o2nm_this_node() doesn't work yet as we're being called while it + * is being set up. + */ +int o2net_start_listening(struct o2nm_node *node) +{ + int ret = 0; + struct workqueue_struct *wq; + + if (WARN_ON_ONCE(READ_ONCE(o2net_listening))) + return -EBUSY; + + mutex_lock(&o2net_wq_mutex); + if (o2net_wq_destroying) { + mutex_unlock(&o2net_wq_mutex); + return -EBUSY; + } + if (WARN_ON_ONCE(o2net_wq)) { + mutex_unlock(&o2net_wq_mutex); + return -EBUSY; + } + mutex_unlock(&o2net_wq_mutex); + + BUG_ON(o2net_listen_sock != NULL); + + mlog(ML_KTHREAD, "starting o2net thread...\n"); + wq = alloc_ordered_workqueue("o2net", WQ_MEM_RECLAIM); + if (!wq) { + mlog(ML_ERROR, "unable to launch o2net thread\n"); + return -ENOMEM; /* ? */ + } + + mutex_lock(&o2net_wq_mutex); + if (unlikely(o2net_wq_destroying || o2net_wq)) { + mutex_unlock(&o2net_wq_mutex); + destroy_workqueue(wq); + return -EBUSY; + } + o2net_wq = wq; + mutex_unlock(&o2net_wq_mutex); + + ret = o2net_open_listening_sock(node->nd_ipv4_address, + node->nd_ipv4_port); + if (ret) { + o2net_destroy_wq(); + } else + o2quo_conn_up(node->nd_num); + + return ret; +} + +/* again, o2nm_this_node() doesn't work here as we're involved in + * tearing it down */ +void o2net_stop_listening(struct o2nm_node *node) +{ + struct socket *sock = o2net_listen_sock; + size_t i; + + BUG_ON(o2net_wq == NULL); + BUG_ON(o2net_listen_sock == NULL); + + WRITE_ONCE(o2net_listening, false); + o2hb_synchronize_callbacks(); + + /* stop the listening socket from generating work */ + write_lock_bh(&sock->sk->sk_callback_lock); + sock->sk->sk_data_ready = sock->sk->sk_user_data; + sock->sk->sk_user_data = NULL; + write_unlock_bh(&sock->sk->sk_callback_lock); + + for (i = 0; i < ARRAY_SIZE(o2net_nodes); i++) { + struct o2nm_node *node = o2nm_get_node_by_num(i); + if (node) { + o2net_disconnect_node(node); + o2nm_node_put(node); + } + } + + /* finish all work and tear down the work queue */ + mlog(ML_KTHREAD, "waiting for o2net thread to exit....\n"); + o2net_destroy_wq(); + + sock_release(o2net_listen_sock); + o2net_listen_sock = NULL; + + o2quo_conn_err(node->nd_num); +} + +/* ------------------------------------------------------------ */ + +int o2net_init(void) +{ + struct folio *folio; + void *p; + unsigned long i; + + o2quo_init(); + o2net_debugfs_init(); + + folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, 0); + if (!folio) + goto out; + + p = folio_address(folio); + o2net_hand = p; + p += sizeof(struct o2net_handshake); + o2net_keep_req = p; + p += sizeof(struct o2net_msg); + o2net_keep_resp = p; + + o2net_hand->protocol_version = cpu_to_be64(O2NET_PROTOCOL_VERSION); + o2net_hand->connector_id = cpu_to_be64(1); + + o2net_keep_req->magic = cpu_to_be16(O2NET_MSG_KEEP_REQ_MAGIC); + o2net_keep_resp->magic = cpu_to_be16(O2NET_MSG_KEEP_RESP_MAGIC); + + for (i = 0; i < ARRAY_SIZE(o2net_nodes); i++) { + struct o2net_node *nn = o2net_nn_from_num(i); + + atomic_set(&nn->nn_timeout, 0); + spin_lock_init(&nn->nn_lock); + INIT_DELAYED_WORK(&nn->nn_connect_work, o2net_start_connect); + INIT_DELAYED_WORK(&nn->nn_connect_expired, + o2net_connect_expired); + INIT_DELAYED_WORK(&nn->nn_still_up, o2net_still_up); + /* until we see hb from a node we'll return einval */ + nn->nn_persistent_error = -ENOTCONN; + init_waitqueue_head(&nn->nn_sc_wq); + idr_init(&nn->nn_status_idr); + INIT_LIST_HEAD(&nn->nn_status_list); + } + + return 0; + +out: + o2net_debugfs_exit(); + o2quo_exit(); + return -ENOMEM; +} + +void o2net_exit(void) +{ + o2quo_exit(); + o2net_debugfs_exit(); + folio_put(virt_to_folio(o2net_hand)); +} diff --git a/fs/ocfs2/cluster/tcp.h b/fs/ocfs2/cluster/tcp.h new file mode 100644 index 000000000..a11bcee28 --- /dev/null +++ b/fs/ocfs2/cluster/tcp.h @@ -0,0 +1,139 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * tcp.h + * + * Function prototypes + * + * Copyright (C) 2004 Oracle. All rights reserved. + */ + +#ifndef O2CLUSTER_TCP_H +#define O2CLUSTER_TCP_H + +#include +#ifdef __KERNEL__ +#include +#include +#else +#include +#endif +#include +#include + +struct o2net_msg +{ + __be16 magic; + __be16 data_len; + __be16 msg_type; + __be16 pad1; + __be32 sys_status; + __be32 status; + __be32 key; + __be32 msg_num; + __u8 buf[]; +}; + +typedef int (o2net_msg_handler_func)(struct o2net_msg *msg, u32 len, void *data, + void **ret_data); +typedef void (o2net_post_msg_handler_func)(int status, void *data, + void *ret_data); + +#define O2NET_MAX_PAYLOAD_BYTES (4096 - sizeof(struct o2net_msg)) + +/* same as hb delay, we're waiting for another node to recognize our hb */ +#define O2NET_RECONNECT_DELAY_MS_DEFAULT 2000 + +#define O2NET_KEEPALIVE_DELAY_MS_DEFAULT 2000 +#define O2NET_IDLE_TIMEOUT_MS_DEFAULT 30000 + +#define O2NET_TCP_USER_TIMEOUT 0x7fffffff + +/* TODO: figure this out.... */ +static inline int o2net_link_down(int err, struct socket *sock) +{ + if (sock) { + if (sock->sk->sk_state != TCP_ESTABLISHED && + sock->sk->sk_state != TCP_CLOSE_WAIT) + return 1; + } + + if (err >= 0) + return 0; + switch (err) { + /* ????????????????????????? */ + case -ERESTARTSYS: + case -EBADF: + /* When the server has died, an ICMP port unreachable + * message prompts ECONNREFUSED. */ + case -ECONNREFUSED: + case -ENOTCONN: + case -ECONNRESET: + case -EPIPE: + return 1; + } + return 0; +} + +enum { + O2NET_DRIVER_UNINITED, + O2NET_DRIVER_READY, +}; + +int o2net_send_message(u32 msg_type, u32 key, void *data, u32 len, + u8 target_node, int *status); +int o2net_send_message_vec(u32 msg_type, u32 key, struct kvec *vec, + size_t veclen, u8 target_node, int *status); + +int o2net_register_handler(u32 msg_type, u32 key, u32 max_len, + o2net_msg_handler_func *func, void *data, + o2net_post_msg_handler_func *post_func, + struct list_head *unreg_list); +void o2net_unregister_handler_list(struct list_head *list); +void o2net_unregister_and_flush_handler_list(struct list_head *list); + +void o2net_fill_node_map(unsigned long *map, unsigned bytes); + +struct o2nm_node; +int o2net_register_hb_callbacks(void); +void o2net_unregister_hb_callbacks(void); +int o2net_start_listening(struct o2nm_node *node); +void o2net_complete_start_listening(struct o2nm_node *node); +void o2net_stop_listening(struct o2nm_node *node); +void o2net_disconnect_node(struct o2nm_node *node); +int o2net_num_connected_peers(void); + +int o2net_init(void); +void o2net_exit(void); + +struct o2net_send_tracking; +struct o2net_sock_container; + +#ifdef CONFIG_DEBUG_FS +void o2net_debugfs_init(void); +void o2net_debugfs_exit(void); +void o2net_debug_add_nst(struct o2net_send_tracking *nst); +void o2net_debug_del_nst(struct o2net_send_tracking *nst); +void o2net_debug_add_sc(struct o2net_sock_container *sc); +void o2net_debug_del_sc(struct o2net_sock_container *sc); +#else +static inline void o2net_debugfs_init(void) +{ +} +static inline void o2net_debugfs_exit(void) +{ +} +static inline void o2net_debug_add_nst(struct o2net_send_tracking *nst) +{ +} +static inline void o2net_debug_del_nst(struct o2net_send_tracking *nst) +{ +} +static inline void o2net_debug_add_sc(struct o2net_sock_container *sc) +{ +} +static inline void o2net_debug_del_sc(struct o2net_sock_container *sc) +{ +} +#endif /* CONFIG_DEBUG_FS */ + +#endif /* O2CLUSTER_TCP_H */ diff --git a/fs/ocfs2/cluster/tcp_internal.h b/fs/ocfs2/cluster/tcp_internal.h new file mode 100644 index 000000000..601c99bd2 --- /dev/null +++ b/fs/ocfs2/cluster/tcp_internal.h @@ -0,0 +1,226 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2005 Oracle. All rights reserved. + */ + +#ifndef O2CLUSTER_TCP_INTERNAL_H +#define O2CLUSTER_TCP_INTERNAL_H + +#define O2NET_MSG_MAGIC ((u16)0xfa55) +#define O2NET_MSG_STATUS_MAGIC ((u16)0xfa56) +#define O2NET_MSG_KEEP_REQ_MAGIC ((u16)0xfa57) +#define O2NET_MSG_KEEP_RESP_MAGIC ((u16)0xfa58) + +/* we're delaying our quorum decision so that heartbeat will have timed + * out truly dead nodes by the time we come around to making decisions + * on their number */ +#define O2NET_QUORUM_DELAY_MS ((o2hb_dead_threshold + 2) * O2HB_REGION_TIMEOUT_MS) + +/* + * This version number represents quite a lot, unfortunately. It not + * only represents the raw network message protocol on the wire but also + * locking semantics of the file system using the protocol. It should + * be somewhere else, I'm sure, but right now it isn't. + * + * With version 11, we separate out the filesystem locking portion. The + * filesystem now has a major.minor version it negotiates. Version 11 + * introduces this negotiation to the o2dlm protocol, and as such the + * version here in tcp_internal.h should not need to be bumped for + * filesystem locking changes. + * + * New in version 11 + * - Negotiation of filesystem locking in the dlm join. + * + * New in version 10: + * - Meta/data locks combined + * + * New in version 9: + * - All votes removed + * + * New in version 8: + * - Replace delete inode votes with a cluster lock + * + * New in version 7: + * - DLM join domain includes the live nodemap + * + * New in version 6: + * - DLM lockres remote refcount fixes. + * + * New in version 5: + * - Network timeout checking protocol + * + * New in version 4: + * - Remove i_generation from lock names for better stat performance. + * + * New in version 3: + * - Replace dentry votes with a cluster lock + * + * New in version 2: + * - full 64 bit i_size in the metadata lock lvbs + * - introduction of "rw" lock and pushing meta/data locking down + */ +#define O2NET_PROTOCOL_VERSION 11ULL +struct o2net_handshake { + __be64 protocol_version; + __be64 connector_id; + __be32 o2hb_heartbeat_timeout_ms; + __be32 o2net_idle_timeout_ms; + __be32 o2net_keepalive_delay_ms; + __be32 o2net_reconnect_delay_ms; +}; + +struct o2net_node { + /* this is never called from int/bh */ + spinlock_t nn_lock; + + /* set the moment an sc is allocated and a connect is started */ + struct o2net_sock_container *nn_sc; + /* _valid is only set after the handshake passes and tx can happen */ + unsigned nn_sc_valid:1; + /* if this is set tx just returns it */ + int nn_persistent_error; + /* It is only set to 1 after the idle time out. */ + atomic_t nn_timeout; + + /* threads waiting for an sc to arrive wait on the wq for generation + * to increase. it is increased when a connecting socket succeeds + * or fails or when an accepted socket is attached. */ + wait_queue_head_t nn_sc_wq; + + struct idr nn_status_idr; + struct list_head nn_status_list; + + /* connects are attempted from when heartbeat comes up until either hb + * goes down, the node is unconfigured, or a connect succeeds. + * connect_work is queued from set_nn_state both from hb up and from + * itself if a connect attempt fails and so can be self-arming. + * shutdown is careful to first mark the nn such that no connects will + * be attempted before canceling delayed connect work and flushing the + * queue. */ + struct delayed_work nn_connect_work; + unsigned long nn_last_connect_attempt; + + /* this is queued as nodes come up and is canceled when a connection is + * established. this expiring gives up on the node and errors out + * transmits */ + struct delayed_work nn_connect_expired; + + /* after we give up on a socket we wait a while before deciding + * that it is still heartbeating and that we should do some + * quorum work */ + struct delayed_work nn_still_up; +}; + +struct o2net_sock_container { + struct kref sc_kref; + /* the next two are valid for the life time of the sc */ + struct socket *sc_sock; + struct o2nm_node *sc_node; + + /* all of these sc work structs hold refs on the sc while they are + * queued. they should not be able to ref a freed sc. the teardown + * race is with o2net_wq destruction in o2net_stop_listening() */ + + /* rx and connect work are generated from socket callbacks. sc + * shutdown removes the callbacks and then flushes the work queue */ + struct work_struct sc_rx_work; + struct work_struct sc_connect_work; + /* shutdown work is triggered in two ways. the simple way is + * for a code path calls ensure_shutdown which gets a lock, removes + * the sc from the nn, and queues the work. in this case the + * work is single-shot. the work is also queued from a sock + * callback, though, and in this case the work will find the sc + * still on the nn and will call ensure_shutdown itself.. this + * ends up triggering the shutdown work again, though nothing + * will be done in that second iteration. so work queue teardown + * has to be careful to remove the sc from the nn before waiting + * on the work queue so that the shutdown work doesn't remove the + * sc and rearm itself. + */ + struct work_struct sc_shutdown_work; + + struct timer_list sc_idle_timeout; + struct delayed_work sc_keepalive_work; + + unsigned sc_handshake_ok:1; + + struct page *sc_page; + size_t sc_page_off; + + /* original handlers for the sockets */ + void (*sc_state_change)(struct sock *sk); + void (*sc_data_ready)(struct sock *sk); + + u32 sc_msg_key; + u16 sc_msg_type; + +#ifdef CONFIG_DEBUG_FS + struct list_head sc_net_debug_item; + ktime_t sc_tv_timer; + ktime_t sc_tv_data_ready; + ktime_t sc_tv_advance_start; + ktime_t sc_tv_advance_stop; + ktime_t sc_tv_func_start; + ktime_t sc_tv_func_stop; +#endif +#ifdef CONFIG_OCFS2_FS_STATS + ktime_t sc_tv_acquiry_total; + ktime_t sc_tv_send_total; + ktime_t sc_tv_status_total; + u32 sc_send_count; + u32 sc_recv_count; + ktime_t sc_tv_process_total; +#endif + struct mutex sc_send_lock; +}; + +struct o2net_msg_handler { + struct rb_node nh_node; + u32 nh_max_len; + u32 nh_msg_type; + u32 nh_key; + o2net_msg_handler_func *nh_func; + void *nh_func_data; + o2net_post_msg_handler_func + *nh_post_func; + struct kref nh_kref; + struct list_head nh_unregister_item; +}; + +enum o2net_system_error { + O2NET_ERR_NONE = 0, + O2NET_ERR_NO_HNDLR, + O2NET_ERR_OVERFLOW, + O2NET_ERR_DIED, + O2NET_ERR_MAX +}; + +struct o2net_status_wait { + enum o2net_system_error ns_sys_status; + s32 ns_status; + int ns_id; + wait_queue_head_t ns_wq; + struct list_head ns_node_item; +}; + +#ifdef CONFIG_DEBUG_FS +/* just for state dumps */ +struct o2net_send_tracking { + struct list_head st_net_debug_item; + struct task_struct *st_task; + struct o2net_sock_container *st_sc; + u32 st_id; + u32 st_msg_type; + u32 st_msg_key; + u8 st_node; + ktime_t st_sock_time; + ktime_t st_send_time; + ktime_t st_status_time; +}; +#else +struct o2net_send_tracking { + u32 dummy; +}; +#endif /* CONFIG_DEBUG_FS */ + +#endif /* O2CLUSTER_TCP_INTERNAL_H */ diff --git a/fs/ocfs2/dcache.c b/fs/ocfs2/dcache.c new file mode 100644 index 000000000..e06774fd8 --- /dev/null +++ b/fs/ocfs2/dcache.c @@ -0,0 +1,459 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * dcache.c + * + * dentry cache handling code + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#include +#include +#include +#include + +#include + +#include "ocfs2.h" + +#include "alloc.h" +#include "dcache.h" +#include "dlmglue.h" +#include "file.h" +#include "inode.h" +#include "ocfs2_trace.h" + +void ocfs2_dentry_attach_gen(struct dentry *dentry) +{ + unsigned long gen = + OCFS2_I(d_inode(dentry->d_parent))->ip_dir_lock_gen; + BUG_ON(d_inode(dentry)); + dentry->d_fsdata = (void *)gen; +} + + +static int ocfs2_dentry_revalidate(struct inode *dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + struct inode *inode; + int ret = 0; /* if all else fails, just return false */ + struct ocfs2_super *osb; + + if (flags & LOOKUP_RCU) + return -ECHILD; + + inode = d_inode(dentry); + osb = OCFS2_SB(dentry->d_sb); + + trace_ocfs2_dentry_revalidate(dentry, name->len, name->name); + + /* For a negative dentry - + * check the generation number of the parent and compare with the + * one stored in the inode. + */ + if (inode == NULL) { + unsigned long gen = (unsigned long) dentry->d_fsdata; + unsigned long pgen = OCFS2_I(dir)->ip_dir_lock_gen; + trace_ocfs2_dentry_revalidate_negative(name->len, name->name, + pgen, gen); + if (gen != pgen) + goto bail; + goto valid; + } + + BUG_ON(!osb); + + if (inode == osb->root_inode || is_bad_inode(inode)) + goto bail; + + spin_lock(&OCFS2_I(inode)->ip_lock); + /* did we or someone else delete this inode? */ + if (OCFS2_I(inode)->ip_flags & OCFS2_INODE_DELETED) { + spin_unlock(&OCFS2_I(inode)->ip_lock); + trace_ocfs2_dentry_revalidate_delete( + (unsigned long long)OCFS2_I(inode)->ip_blkno); + goto bail; + } + spin_unlock(&OCFS2_I(inode)->ip_lock); + + /* + * We don't need a cluster lock to test this because once an + * inode nlink hits zero, it never goes back. + */ + if (inode->i_nlink == 0) { + trace_ocfs2_dentry_revalidate_orphaned( + (unsigned long long)OCFS2_I(inode)->ip_blkno, + S_ISDIR(inode->i_mode)); + goto bail; + } + + /* + * If the last lookup failed to create dentry lock, let us + * redo it. + */ + if (!dentry->d_fsdata) { + trace_ocfs2_dentry_revalidate_nofsdata( + (unsigned long long)OCFS2_I(inode)->ip_blkno); + goto bail; + } + +valid: + ret = 1; + +bail: + trace_ocfs2_dentry_revalidate_ret(ret); + return ret; +} + +static int ocfs2_match_dentry(struct dentry *dentry, + u64 parent_blkno, + int skip_unhashed) +{ + struct inode *parent; + + /* + * ocfs2_lookup() does a d_splice_alias() _before_ attaching + * to the lock data, so we skip those here, otherwise + * ocfs2_dentry_attach_lock() will get its original dentry + * back. + */ + if (!dentry->d_fsdata) + return 0; + + if (skip_unhashed && d_unhashed(dentry)) + return 0; + + parent = d_inode(dentry->d_parent); + /* Name is in a different directory. */ + if (OCFS2_I(parent)->ip_blkno != parent_blkno) + return 0; + + return 1; +} + +/* + * Walk the inode alias list, and find a dentry which has a given + * parent. ocfs2_dentry_attach_lock() wants to find _any_ alias as it + * is looking for a dentry_lock reference. The downconvert thread is + * looking to unhash aliases, so we allow it to skip any that already + * have that property. + */ +struct dentry *ocfs2_find_local_alias(struct inode *inode, + u64 parent_blkno, + int skip_unhashed) +{ + struct dentry *dentry; + + spin_lock(&inode->i_lock); + for_each_alias(dentry, inode) { + spin_lock(&dentry->d_lock); + if (ocfs2_match_dentry(dentry, parent_blkno, skip_unhashed)) { + trace_ocfs2_find_local_alias(dentry->d_name.len, + dentry->d_name.name); + + dget_dlock(dentry); + spin_unlock(&dentry->d_lock); + spin_unlock(&inode->i_lock); + return dentry; + } + spin_unlock(&dentry->d_lock); + } + spin_unlock(&inode->i_lock); + return NULL; +} + +DEFINE_SPINLOCK(dentry_attach_lock); + +/* + * Attach this dentry to a cluster lock. + * + * Dentry locks cover all links in a given directory to a particular + * inode. We do this so that ocfs2 can build a lock name which all + * nodes in the cluster can agree on at all times. Shoving full names + * in the cluster lock won't work due to size restrictions. Covering + * links inside of a directory is a good compromise because it still + * allows us to use the parent directory lock to synchronize + * operations. + * + * Call this function with the parent dir semaphore and the parent dir + * cluster lock held. + * + * The dir semaphore will protect us from having to worry about + * concurrent processes on our node trying to attach a lock at the + * same time. + * + * The dir cluster lock (held at either PR or EX mode) protects us + * from unlink and rename on other nodes. + * + * A dput() can happen asynchronously due to pruning, so we cover + * attaching and detaching the dentry lock with a + * dentry_attach_lock. + * + * A node which has done lookup on a name retains a protected read + * lock until final dput. If the user requests and unlink or rename, + * the protected read is upgraded to an exclusive lock. Other nodes + * who have seen the dentry will then be informed that they need to + * downgrade their lock, which will involve d_delete on the + * dentry. This happens in ocfs2_dentry_convert_worker(). + */ +int ocfs2_dentry_attach_lock(struct dentry *dentry, + struct inode *inode, + u64 parent_blkno) +{ + int ret; + struct dentry *alias; + struct ocfs2_dentry_lock *dl = dentry->d_fsdata; + + trace_ocfs2_dentry_attach_lock(dentry->d_name.len, dentry->d_name.name, + (unsigned long long)parent_blkno, dl); + + /* + * Negative dentry. We ignore these for now. + * + * XXX: Could we can improve ocfs2_dentry_revalidate() by + * tracking these? + */ + if (!inode) + return 0; + + if (d_really_is_negative(dentry) && dentry->d_fsdata) { + /* Converting a negative dentry to positive + Clear dentry->d_fsdata */ + dentry->d_fsdata = dl = NULL; + } + + if (dl) { + mlog_bug_on_msg(dl->dl_parent_blkno != parent_blkno, + " \"%pd\": old parent: %llu, new: %llu\n", + dentry, + (unsigned long long)parent_blkno, + (unsigned long long)dl->dl_parent_blkno); + return 0; + } + + alias = ocfs2_find_local_alias(inode, parent_blkno, 0); + if (alias) { + /* + * Great, an alias exists, which means we must have a + * dentry lock already. We can just grab the lock off + * the alias and add it to the list. + * + * We're depending here on the fact that this dentry + * was found and exists in the dcache and so must have + * a reference to the dentry_lock because we can't + * race creates. Final dput() cannot happen on it + * since we have it pinned, so our reference is safe. + */ + dl = alias->d_fsdata; + mlog_bug_on_msg(!dl, "parent %llu, ino %llu\n", + (unsigned long long)parent_blkno, + (unsigned long long)OCFS2_I(inode)->ip_blkno); + + mlog_bug_on_msg(dl->dl_parent_blkno != parent_blkno, + " \"%pd\": old parent: %llu, new: %llu\n", + dentry, + (unsigned long long)parent_blkno, + (unsigned long long)dl->dl_parent_blkno); + + trace_ocfs2_dentry_attach_lock_found(dl->dl_lockres.l_name, + (unsigned long long)parent_blkno, + (unsigned long long)OCFS2_I(inode)->ip_blkno); + + goto out_attach; + } + + /* + * There are no other aliases + */ + dl = kmalloc_obj(*dl, GFP_NOFS); + if (!dl) { + ret = -ENOMEM; + mlog_errno(ret); + return ret; + } + + dl->dl_count = 0; + /* + * Does this have to happen below, for all attaches, in case + * the struct inode gets blown away by the downconvert thread? + */ + dl->dl_inode = igrab(inode); + dl->dl_parent_blkno = parent_blkno; + ocfs2_dentry_lock_res_init(dl, parent_blkno, inode); + +out_attach: + spin_lock(&dentry_attach_lock); + if (unlikely(dentry->d_fsdata && !alias)) { + /* d_fsdata is set by a racing thread which is doing + * the same thing as this thread is doing. Leave the racing + * thread going ahead and we return here. + */ + spin_unlock(&dentry_attach_lock); + iput(dl->dl_inode); + ocfs2_lock_res_free(&dl->dl_lockres); + kfree(dl); + return 0; + } + + dentry->d_fsdata = dl; + dl->dl_count++; + spin_unlock(&dentry_attach_lock); + + /* + * This actually gets us our PRMODE level lock. From now on, + * we'll have a notification if one of these names is + * destroyed on another node. + */ + ret = ocfs2_dentry_lock(dentry, 0); + if (!ret) + ocfs2_dentry_unlock(dentry, 0); + else + mlog_errno(ret); + + /* + * In case of error, manually free the allocation and do the iput(). + * We need to do this because error here means no d_instantiate(), + * which means iput() will not be called during dput(dentry). + */ + if (ret < 0 && !alias) { + ocfs2_lock_res_free(&dl->dl_lockres); + BUG_ON(dl->dl_count != 1); + spin_lock(&dentry_attach_lock); + dentry->d_fsdata = NULL; + spin_unlock(&dentry_attach_lock); + kfree(dl); + iput(inode); + } + + dput(alias); + + return ret; +} + +/* + * ocfs2_dentry_iput() and friends. + * + * At this point, our particular dentry is detached from the inodes + * alias list, so there's no way that the locking code can find it. + * + * The interesting stuff happens when we determine that our lock needs + * to go away because this is the last subdir alias in the + * system. This function needs to handle a couple things: + * + * 1) Synchronizing lock shutdown with the downconvert threads. This + * is already handled for us via the lockres release drop function + * called in ocfs2_release_dentry_lock() + * + * 2) A race may occur when we're doing our lock shutdown and + * another process wants to create a new dentry lock. Right now we + * let them race, which means that for a very short while, this + * node might have two locks on a lock resource. This should be a + * problem though because one of them is in the process of being + * thrown out. + */ +static void ocfs2_drop_dentry_lock(struct ocfs2_super *osb, + struct ocfs2_dentry_lock *dl) +{ + iput(dl->dl_inode); + ocfs2_simple_drop_lockres(osb, &dl->dl_lockres); + ocfs2_lock_res_free(&dl->dl_lockres); + kfree(dl); +} + +void ocfs2_dentry_lock_put(struct ocfs2_super *osb, + struct ocfs2_dentry_lock *dl) +{ + int unlock = 0; + + BUG_ON(dl->dl_count == 0); + + spin_lock(&dentry_attach_lock); + dl->dl_count--; + unlock = !dl->dl_count; + spin_unlock(&dentry_attach_lock); + + if (unlock) + ocfs2_drop_dentry_lock(osb, dl); +} + +static void ocfs2_dentry_iput(struct dentry *dentry, struct inode *inode) +{ + struct ocfs2_dentry_lock *dl = dentry->d_fsdata; + + if (!dl) { + /* + * No dentry lock is ok if we're disconnected or + * unhashed. + */ + if (!(dentry->d_flags & DCACHE_DISCONNECTED) && + !d_unhashed(dentry)) { + unsigned long long ino = 0ULL; + if (inode) + ino = (unsigned long long)OCFS2_I(inode)->ip_blkno; + mlog(ML_ERROR, "Dentry is missing cluster lock. " + "inode: %llu, d_flags: 0x%x, d_name: %pd\n", + ino, dentry->d_flags, dentry); + } + + goto out; + } + + mlog_bug_on_msg(dl->dl_count == 0, "dentry: %pd, count: %u\n", + dentry, dl->dl_count); + + ocfs2_dentry_lock_put(OCFS2_SB(dentry->d_sb), dl); + +out: + iput(inode); +} + +/* + * d_move(), but keep the locks in sync. + * + * When we are done, "dentry" will have the parent dir and name of + * "target", which will be thrown away. + * + * We manually update the lock of "dentry" if need be. + * + * "target" doesn't have it's dentry lock touched - we allow the later + * dput() to handle this for us. + * + * This is called during ocfs2_rename(), while holding parent + * directory locks. The dentries have already been deleted on other + * nodes via ocfs2_remote_dentry_delete(). + * + * Normally, the VFS handles the d_move() for the file system, after + * the ->rename() callback. OCFS2 wants to handle this internally, so + * the new lock can be created atomically with respect to the cluster. + */ +void ocfs2_dentry_move(struct dentry *dentry, struct dentry *target, + struct inode *old_dir, struct inode *new_dir) +{ + int ret; + struct ocfs2_super *osb = OCFS2_SB(old_dir->i_sb); + struct inode *inode = d_inode(dentry); + + /* + * Move within the same directory, so the actual lock info won't + * change. + * + * XXX: Is there any advantage to dropping the lock here? + */ + if (old_dir == new_dir) + goto out_move; + + ocfs2_dentry_lock_put(osb, dentry->d_fsdata); + + dentry->d_fsdata = NULL; + ret = ocfs2_dentry_attach_lock(dentry, inode, OCFS2_I(new_dir)->ip_blkno); + if (ret) + mlog_errno(ret); + +out_move: + d_move(dentry, target); +} + +const struct dentry_operations ocfs2_dentry_ops = { + .d_revalidate = ocfs2_dentry_revalidate, + .d_iput = ocfs2_dentry_iput, +}; diff --git a/fs/ocfs2/dcache.h b/fs/ocfs2/dcache.h new file mode 100644 index 000000000..7f246c569 --- /dev/null +++ b/fs/ocfs2/dcache.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * dcache.h + * + * Function prototypes + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#ifndef OCFS2_DCACHE_H +#define OCFS2_DCACHE_H + +extern const struct dentry_operations ocfs2_dentry_ops; + +struct ocfs2_dentry_lock { + unsigned int dl_count; + u64 dl_parent_blkno; + + /* + * The ocfs2_dentry_lock keeps an inode reference until + * dl_lockres has been destroyed. This is usually done in + * ->d_iput() anyway, so there should be minimal impact. + */ + struct inode *dl_inode; + struct ocfs2_lock_res dl_lockres; +}; + +int ocfs2_dentry_attach_lock(struct dentry *dentry, struct inode *inode, + u64 parent_blkno); + +void ocfs2_dentry_lock_put(struct ocfs2_super *osb, + struct ocfs2_dentry_lock *dl); + +struct dentry *ocfs2_find_local_alias(struct inode *inode, u64 parent_blkno, + int skip_unhashed); + +void ocfs2_dentry_move(struct dentry *dentry, struct dentry *target, + struct inode *old_dir, struct inode *new_dir); + +extern spinlock_t dentry_attach_lock; +void ocfs2_dentry_attach_gen(struct dentry *dentry); + +#endif /* OCFS2_DCACHE_H */ diff --git a/fs/ocfs2/dir.c b/fs/ocfs2/dir.c new file mode 100644 index 000000000..0075e1624 --- /dev/null +++ b/fs/ocfs2/dir.c @@ -0,0 +1,4575 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * dir.c + * + * Creates, reads, walks and deletes directory-nodes + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + * + * Portions of this code from linux/fs/ext3/dir.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * from + * + * linux/fs/minix/dir.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ocfs2.h" + +#include "alloc.h" +#include "blockcheck.h" +#include "dir.h" +#include "dlmglue.h" +#include "extent_map.h" +#include "file.h" +#include "inode.h" +#include "journal.h" +#include "namei.h" +#include "suballoc.h" +#include "super.h" +#include "sysfile.h" +#include "uptodate.h" +#include "ocfs2_trace.h" + +#include "buffer_head_io.h" + +#define NAMEI_RA_CHUNKS 2 +#define NAMEI_RA_BLOCKS 4 +#define NAMEI_RA_SIZE (NAMEI_RA_CHUNKS * NAMEI_RA_BLOCKS) + +static int ocfs2_do_extend_dir(struct super_block *sb, + handle_t *handle, + struct inode *dir, + struct buffer_head *parent_fe_bh, + struct ocfs2_alloc_context *data_ac, + struct ocfs2_alloc_context *meta_ac, + struct buffer_head **new_bh); +static int ocfs2_dir_indexed(struct inode *inode); + +/* + * These are distinct checks because future versions of the file system will + * want to have a trailing dirent structure independent of indexing. + */ +static int ocfs2_supports_dir_trailer(struct inode *dir) +{ + struct ocfs2_super *osb = OCFS2_SB(dir->i_sb); + + if (OCFS2_I(dir)->ip_dyn_features & OCFS2_INLINE_DATA_FL) + return 0; + + return ocfs2_meta_ecc(osb) || ocfs2_dir_indexed(dir); +} + +/* + * "new' here refers to the point at which we're creating a new + * directory via "mkdir()", but also when we're expanding an inline + * directory. In either case, we don't yet have the indexing bit set + * on the directory, so the standard checks will fail in when metaecc + * is turned off. Only directory-initialization type functions should + * use this then. Everything else wants ocfs2_supports_dir_trailer() + */ +static int ocfs2_new_dir_wants_trailer(struct inode *dir) +{ + struct ocfs2_super *osb = OCFS2_SB(dir->i_sb); + + return ocfs2_meta_ecc(osb) || + ocfs2_supports_indexed_dirs(osb); +} + +static inline unsigned int ocfs2_dir_trailer_blk_off(struct super_block *sb) +{ + return sb->s_blocksize - sizeof(struct ocfs2_dir_block_trailer); +} + +#define ocfs2_trailer_from_bh(_bh, _sb) ((struct ocfs2_dir_block_trailer *) ((_bh)->b_data + ocfs2_dir_trailer_blk_off((_sb)))) + +/* XXX ocfs2_block_dqtrailer() is similar but not quite - can we make + * them more consistent? */ +struct ocfs2_dir_block_trailer *ocfs2_dir_trailer_from_size(int blocksize, + void *data) +{ + char *p = data; + + p += blocksize - sizeof(struct ocfs2_dir_block_trailer); + return (struct ocfs2_dir_block_trailer *)p; +} + +/* + * XXX: This is executed once on every dirent. We should consider optimizing + * it. + */ +static int ocfs2_skip_dir_trailer(struct inode *dir, + struct ocfs2_dir_entry *de, + unsigned long offset, + unsigned long blklen) +{ + unsigned long toff = blklen - sizeof(struct ocfs2_dir_block_trailer); + + if (!ocfs2_supports_dir_trailer(dir)) + return 0; + + if (offset != toff) + return 0; + + return 1; +} + +static void ocfs2_init_dir_trailer(struct inode *inode, + struct buffer_head *bh, u16 rec_len) +{ + struct ocfs2_dir_block_trailer *trailer; + + trailer = ocfs2_trailer_from_bh(bh, inode->i_sb); + strscpy(trailer->db_signature, OCFS2_DIR_TRAILER_SIGNATURE); + trailer->db_compat_rec_len = + cpu_to_le16(sizeof(struct ocfs2_dir_block_trailer)); + trailer->db_parent_dinode = cpu_to_le64(OCFS2_I(inode)->ip_blkno); + trailer->db_blkno = cpu_to_le64(bh->b_blocknr); + trailer->db_free_rec_len = cpu_to_le16(rec_len); +} +/* + * Link an unindexed block with a dir trailer structure into the index free + * list. This function will modify dirdata_bh, but assumes you've already + * passed it to the journal. + */ +static int ocfs2_dx_dir_link_trailer(struct inode *dir, handle_t *handle, + struct buffer_head *dx_root_bh, + struct buffer_head *dirdata_bh) +{ + int ret; + struct ocfs2_dx_root_block *dx_root; + struct ocfs2_dir_block_trailer *trailer; + + ret = ocfs2_journal_access_dr(handle, INODE_CACHE(dir), dx_root_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + trailer = ocfs2_trailer_from_bh(dirdata_bh, dir->i_sb); + dx_root = (struct ocfs2_dx_root_block *)dx_root_bh->b_data; + + trailer->db_free_next = dx_root->dr_free_blk; + dx_root->dr_free_blk = cpu_to_le64(dirdata_bh->b_blocknr); + + ocfs2_journal_dirty(handle, dx_root_bh); + +out: + return ret; +} + +static int ocfs2_free_list_at_root(struct ocfs2_dir_lookup_result *res) +{ + return res->dl_prev_leaf_bh == NULL; +} + +void ocfs2_free_dir_lookup_result(struct ocfs2_dir_lookup_result *res) +{ + brelse(res->dl_dx_root_bh); + brelse(res->dl_leaf_bh); + brelse(res->dl_dx_leaf_bh); + brelse(res->dl_prev_leaf_bh); +} + +static int ocfs2_dir_indexed(struct inode *inode) +{ + if (OCFS2_I(inode)->ip_dyn_features & OCFS2_INDEXED_DIR_FL) + return 1; + return 0; +} + +static inline int ocfs2_dx_root_inline(struct ocfs2_dx_root_block *dx_root) +{ + return dx_root->dr_flags & OCFS2_DX_FLAG_INLINE; +} + +/* + * Hashing code adapted from ext3 + */ +#define DELTA 0x9E3779B9 + +static void TEA_transform(__u32 buf[4], __u32 const in[]) +{ + __u32 sum = 0; + __u32 b0 = buf[0], b1 = buf[1]; + __u32 a = in[0], b = in[1], c = in[2], d = in[3]; + int n = 16; + + do { + sum += DELTA; + b0 += ((b1 << 4)+a) ^ (b1+sum) ^ ((b1 >> 5)+b); + b1 += ((b0 << 4)+c) ^ (b0+sum) ^ ((b0 >> 5)+d); + } while (--n); + + buf[0] += b0; + buf[1] += b1; +} + +static void str2hashbuf(const char *msg, int len, __u32 *buf, int num) +{ + __u32 pad, val; + int i; + + pad = (__u32)len | ((__u32)len << 8); + pad |= pad << 16; + + val = pad; + if (len > num*4) + len = num * 4; + for (i = 0; i < len; i++) { + if ((i % 4) == 0) + val = pad; + val = msg[i] + (val << 8); + if ((i % 4) == 3) { + *buf++ = val; + val = pad; + num--; + } + } + if (--num >= 0) + *buf++ = val; + while (--num >= 0) + *buf++ = pad; +} + +static void ocfs2_dx_dir_name_hash(struct inode *dir, const char *name, int len, + struct ocfs2_dx_hinfo *hinfo) +{ + struct ocfs2_super *osb = OCFS2_SB(dir->i_sb); + const char *p; + __u32 in[8], buf[4]; + + /* + * XXX: Is this really necessary, if the index is never looked + * at by readdir? Is a hash value of '0' a bad idea? + */ + if ((len == 1 && !strncmp(".", name, 1)) || + (len == 2 && !strncmp("..", name, 2))) { + buf[0] = buf[1] = 0; + goto out; + } + +#ifdef OCFS2_DEBUG_DX_DIRS + /* + * This makes it very easy to debug indexing problems. We + * should never allow this to be selected without hand editing + * this file though. + */ + buf[0] = buf[1] = len; + goto out; +#endif + + memcpy(buf, osb->osb_dx_seed, sizeof(buf)); + + p = name; + while (len > 0) { + str2hashbuf(p, len, in, 4); + TEA_transform(buf, in); + len -= 16; + p += 16; + } + +out: + hinfo->major_hash = buf[0]; + hinfo->minor_hash = buf[1]; +} + +/* + * bh passed here can be an inode block or a dir data block, depending + * on the inode inline data flag. + */ +static int ocfs2_check_dir_entry(struct inode *dir, + struct ocfs2_dir_entry *de, + struct buffer_head *bh, + char *buf, + unsigned int size, + unsigned long offset) +{ + const char *error_msg = NULL; + unsigned long buf_offset = (char *)de - buf; + unsigned long next_offset; + int rlen; + + if (buf_offset > size || size - buf_offset < OCFS2_DIR_REC_LEN(1)) { + /* Dirent is (maybe partially) beyond the buffer + * boundaries so touching 'de' members is unsafe. + */ + mlog(ML_ERROR, "directory entry (#%llu: offset=%lu) " + "too close to end or out-of-bounds", + (unsigned long long)OCFS2_I(dir)->ip_blkno, offset); + return 0; + } + + rlen = le16_to_cpu(de->rec_len); + next_offset = buf_offset + rlen; + + if (unlikely(rlen < OCFS2_DIR_REC_LEN(1))) + error_msg = "rec_len is smaller than minimal"; + else if (unlikely(rlen % 4 != 0)) + error_msg = "rec_len % 4 != 0"; + else if (unlikely(rlen < OCFS2_DIR_REC_LEN(de->name_len))) + error_msg = "rec_len is too small for name_len"; + else if (unlikely(next_offset > size)) + error_msg = "directory entry overrun"; + else if (unlikely(next_offset > size - OCFS2_DIR_REC_LEN(1)) && + next_offset != size) + error_msg = "directory entry too close to end"; + + if (unlikely(error_msg != NULL)) + mlog(ML_ERROR, "bad entry in directory #%llu: %s - " + "offset=%lu, inode=%llu, rec_len=%d, name_len=%d\n", + (unsigned long long)OCFS2_I(dir)->ip_blkno, error_msg, + offset, (unsigned long long)le64_to_cpu(de->inode), rlen, + de->name_len); + + return error_msg == NULL ? 1 : 0; +} + +static inline int ocfs2_match(int len, + const char * const name, + struct ocfs2_dir_entry *de) +{ + if (len != de->name_len) + return 0; + if (!de->inode) + return 0; + return !memcmp(name, de->name, len); +} + +/* + * Returns 0 if not found, -1 on failure, and 1 on success + */ +static inline int ocfs2_search_dirblock(struct buffer_head *bh, + struct inode *dir, + const char *name, int namelen, + unsigned long offset, + char *first_de, + unsigned int bytes, + struct ocfs2_dir_entry **res_dir) +{ + struct ocfs2_dir_entry *de; + char *dlimit, *de_buf; + int de_len; + int ret = 0; + + de_buf = first_de; + dlimit = de_buf + bytes; + + while (de_buf < dlimit - OCFS2_DIR_MEMBER_LEN) { + /* this code is executed quadratically often */ + /* do minimal checking `by hand' */ + + de = (struct ocfs2_dir_entry *) de_buf; + + if (de->name + namelen <= dlimit && + ocfs2_match(namelen, name, de)) { + /* found a match - just to be sure, do a full check */ + if (!ocfs2_check_dir_entry(dir, de, bh, first_de, + bytes, offset)) { + ret = -1; + goto bail; + } + *res_dir = de; + ret = 1; + goto bail; + } + + /* prevent looping on a bad block */ + de_len = le16_to_cpu(de->rec_len); + if (de_len <= 0) { + ret = -1; + goto bail; + } + + de_buf += de_len; + offset += de_len; + } + +bail: + trace_ocfs2_search_dirblock(ret); + return ret; +} + +static struct buffer_head *ocfs2_find_entry_id(const char *name, + int namelen, + struct inode *dir, + struct ocfs2_dir_entry **res_dir) +{ + int ret, found; + struct buffer_head *di_bh = NULL; + struct ocfs2_dinode *di; + struct ocfs2_inline_data *data; + + ret = ocfs2_read_inode_block(dir, &di_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + di = (struct ocfs2_dinode *)di_bh->b_data; + data = &di->id2.i_data; + + found = ocfs2_search_dirblock(di_bh, dir, name, namelen, 0, + data->id_data, i_size_read(dir), res_dir); + if (found == 1) + return di_bh; + + brelse(di_bh); +out: + return NULL; +} + +static int ocfs2_validate_dir_block(struct super_block *sb, + struct buffer_head *bh) +{ + int rc; + struct ocfs2_dir_block_trailer *trailer = + ocfs2_trailer_from_bh(bh, sb); + + + /* + * We don't validate dirents here, that's handled + * in-place when the code walks them. + */ + trace_ocfs2_validate_dir_block((unsigned long long)bh->b_blocknr); + + BUG_ON(!buffer_uptodate(bh)); + + /* + * If the ecc fails, we return the error but otherwise + * leave the filesystem running. We know any error is + * local to this block. + * + * Note that we are safe to call this even if the directory + * doesn't have a trailer. Filesystems without metaecc will do + * nothing, and filesystems with it will have one. + */ + rc = ocfs2_validate_meta_ecc(sb, bh->b_data, &trailer->db_check); + if (rc) + mlog(ML_ERROR, "Checksum failed for dinode %llu\n", + (unsigned long long)bh->b_blocknr); + + return rc; +} + +/* + * Validate a directory trailer. + * + * We check the trailer here rather than in ocfs2_validate_dir_block() + * because that function doesn't have the inode to test. + */ +static int ocfs2_check_dir_trailer(struct inode *dir, struct buffer_head *bh) +{ + int rc = 0; + struct ocfs2_dir_block_trailer *trailer; + + trailer = ocfs2_trailer_from_bh(bh, dir->i_sb); + if (!OCFS2_IS_VALID_DIR_TRAILER(trailer)) { + rc = ocfs2_error(dir->i_sb, + "Invalid dirblock #%llu: signature = %.*s\n", + (unsigned long long)bh->b_blocknr, 7, + trailer->db_signature); + goto out; + } + if (le64_to_cpu(trailer->db_blkno) != bh->b_blocknr) { + rc = ocfs2_error(dir->i_sb, + "Directory block #%llu has an invalid db_blkno of %llu\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(trailer->db_blkno)); + goto out; + } + if (le64_to_cpu(trailer->db_parent_dinode) != + OCFS2_I(dir)->ip_blkno) { + rc = ocfs2_error(dir->i_sb, + "Directory block #%llu on dinode #%llu has an invalid parent_dinode of %llu\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)OCFS2_I(dir)->ip_blkno, + (unsigned long long)le64_to_cpu(trailer->db_blkno)); + goto out; + } +out: + return rc; +} + +/* + * This function forces all errors to -EIO for consistency with its + * predecessor, ocfs2_bread(). We haven't audited what returning the + * real error codes would do to callers. We log the real codes with + * mlog_errno() before we squash them. + */ +static int ocfs2_read_dir_block(struct inode *inode, u64 v_block, + struct buffer_head **bh, int flags) +{ + int rc = 0; + struct buffer_head *tmp = *bh; + + rc = ocfs2_read_virt_blocks(inode, v_block, 1, &tmp, flags, + ocfs2_validate_dir_block); + if (rc) { + mlog_errno(rc); + goto out; + } + + if (!(flags & OCFS2_BH_READAHEAD) && + ocfs2_supports_dir_trailer(inode)) { + rc = ocfs2_check_dir_trailer(inode, tmp); + if (rc) { + if (!*bh) + brelse(tmp); + mlog_errno(rc); + goto out; + } + } + + /* If ocfs2_read_virt_blocks() got us a new bh, pass it up. */ + if (!*bh) + *bh = tmp; + +out: + return rc ? -EIO : 0; +} + +/* + * Read the block at 'phys' which belongs to this directory + * inode. This function does no virtual->physical block translation - + * what's passed in is assumed to be a valid directory block. + */ +static int ocfs2_read_dir_block_direct(struct inode *dir, u64 phys, + struct buffer_head **bh) +{ + int ret; + struct buffer_head *tmp = *bh; + + ret = ocfs2_read_block(INODE_CACHE(dir), phys, &tmp, + ocfs2_validate_dir_block); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (ocfs2_supports_dir_trailer(dir)) { + ret = ocfs2_check_dir_trailer(dir, tmp); + if (ret) { + if (!*bh) + brelse(tmp); + mlog_errno(ret); + goto out; + } + } + + if (!ret && !*bh) + *bh = tmp; +out: + return ret; +} + +static int ocfs2_validate_dx_root(struct super_block *sb, + struct buffer_head *bh) +{ + int ret; + struct ocfs2_dx_root_block *dx_root; + + BUG_ON(!buffer_uptodate(bh)); + + dx_root = (struct ocfs2_dx_root_block *) bh->b_data; + + ret = ocfs2_validate_meta_ecc(sb, bh->b_data, &dx_root->dr_check); + if (ret) { + mlog(ML_ERROR, + "Checksum failed for dir index root block %llu\n", + (unsigned long long)bh->b_blocknr); + goto bail; + } + + if (!OCFS2_IS_VALID_DX_ROOT(dx_root)) { + ret = ocfs2_error(sb, + "Dir Index Root # %llu has bad signature %.*s\n", + (unsigned long long)le64_to_cpu(dx_root->dr_blkno), + 7, dx_root->dr_signature); + goto bail; + } + + if (!(dx_root->dr_flags & OCFS2_DX_FLAG_INLINE)) { + struct ocfs2_extent_list *el = &dx_root->dr_list; + + if (le16_to_cpu(el->l_count) != ocfs2_extent_recs_per_dx_root(sb)) { + ret = ocfs2_error(sb, + "Dir Index Root # %llu has invalid l_count %u (expected %u)\n", + (unsigned long long)le64_to_cpu(dx_root->dr_blkno), + le16_to_cpu(el->l_count), + ocfs2_extent_recs_per_dx_root(sb)); + goto bail; + } + + if (le16_to_cpu(el->l_next_free_rec) > le16_to_cpu(el->l_count)) { + ret = ocfs2_error(sb, + "Dir Index Root # %llu has invalid l_next_free_rec %u (l_count %u)\n", + (unsigned long long)le64_to_cpu(dx_root->dr_blkno), + le16_to_cpu(el->l_next_free_rec), + le16_to_cpu(el->l_count)); + goto bail; + } + } else { + struct ocfs2_dx_entry_list *dl_list = &dx_root->dr_entries; + + if (le16_to_cpu(dl_list->de_count) != + ocfs2_dx_entries_per_root(sb)) { + ret = ocfs2_error(sb, + "Dir Index Root # %llu has invalid de_count %u (expected %u)\n", + (unsigned long long)le64_to_cpu(dx_root->dr_blkno), + le16_to_cpu(dl_list->de_count), + ocfs2_dx_entries_per_root(sb)); + goto bail; + } + + if (le16_to_cpu(dl_list->de_num_used) > + le16_to_cpu(dl_list->de_count)) { + ret = ocfs2_error(sb, + "Dir Index Root # %llu has invalid de_num_used %u (de_count %u)\n", + (unsigned long long)le64_to_cpu(dx_root->dr_blkno), + le16_to_cpu(dl_list->de_num_used), + le16_to_cpu(dl_list->de_count)); + goto bail; + } + } + +bail: + return ret; +} + +static int ocfs2_read_dx_root(struct inode *dir, struct ocfs2_dinode *di, + struct buffer_head **dx_root_bh) +{ + int ret; + u64 blkno = le64_to_cpu(di->i_dx_root); + struct buffer_head *tmp = *dx_root_bh; + + ret = ocfs2_read_block(INODE_CACHE(dir), blkno, &tmp, + ocfs2_validate_dx_root); + + /* If ocfs2_read_block() got us a new bh, pass it up. */ + if (!ret && !*dx_root_bh) + *dx_root_bh = tmp; + + return ret; +} + +static int ocfs2_validate_dx_leaf(struct super_block *sb, + struct buffer_head *bh) +{ + int ret; + struct ocfs2_dx_leaf *dx_leaf = (struct ocfs2_dx_leaf *)bh->b_data; + + BUG_ON(!buffer_uptodate(bh)); + + ret = ocfs2_validate_meta_ecc(sb, bh->b_data, &dx_leaf->dl_check); + if (ret) { + mlog(ML_ERROR, + "Checksum failed for dir index leaf block %llu\n", + (unsigned long long)bh->b_blocknr); + return ret; + } + + if (!OCFS2_IS_VALID_DX_LEAF(dx_leaf)) + return ocfs2_error(sb, "Dir Index Leaf has bad signature %.*s\n", + 7, dx_leaf->dl_signature); + + if (le16_to_cpu(dx_leaf->dl_list.de_count) != + ocfs2_dx_entries_per_leaf(sb)) + return ocfs2_error(sb, + "Dir Index Leaf # %llu has invalid de_count %u (expected %u)\n", + (unsigned long long)le64_to_cpu(dx_leaf->dl_blkno), + le16_to_cpu(dx_leaf->dl_list.de_count), + ocfs2_dx_entries_per_leaf(sb)); + + if (le16_to_cpu(dx_leaf->dl_list.de_num_used) > + le16_to_cpu(dx_leaf->dl_list.de_count)) + return ocfs2_error(sb, + "Dir Index Leaf # %llu has invalid de_num_used %u (de_count %u)\n", + (unsigned long long)le64_to_cpu(dx_leaf->dl_blkno), + le16_to_cpu(dx_leaf->dl_list.de_num_used), + le16_to_cpu(dx_leaf->dl_list.de_count)); + + return ret; +} + +static int ocfs2_read_dx_leaf(struct inode *dir, u64 blkno, + struct buffer_head **dx_leaf_bh) +{ + int ret; + struct buffer_head *tmp = *dx_leaf_bh; + + ret = ocfs2_read_block(INODE_CACHE(dir), blkno, &tmp, + ocfs2_validate_dx_leaf); + + /* If ocfs2_read_block() got us a new bh, pass it up. */ + if (!ret && !*dx_leaf_bh) + *dx_leaf_bh = tmp; + + return ret; +} + +/* + * Read a series of dx_leaf blocks. This expects all buffer_head + * pointers to be NULL on function entry. + */ +static int ocfs2_read_dx_leaves(struct inode *dir, u64 start, int num, + struct buffer_head **dx_leaf_bhs) +{ + int ret; + + ret = ocfs2_read_blocks(INODE_CACHE(dir), start, num, dx_leaf_bhs, 0, + ocfs2_validate_dx_leaf); + if (ret) + mlog_errno(ret); + + return ret; +} + +static struct buffer_head *ocfs2_find_entry_el(const char *name, int namelen, + struct inode *dir, + struct ocfs2_dir_entry **res_dir) +{ + struct super_block *sb; + struct buffer_head *bh_use[NAMEI_RA_SIZE]; + struct buffer_head *bh, *ret = NULL; + unsigned long start, block, b; + int ra_max = 0; /* Number of bh's in the readahead + buffer, bh_use[] */ + int ra_ptr = 0; /* Current index into readahead + buffer */ + int num = 0; + int nblocks, i; + + sb = dir->i_sb; + + nblocks = i_size_read(dir) >> sb->s_blocksize_bits; + start = OCFS2_I(dir)->ip_dir_start_lookup; + if (start >= nblocks) + start = 0; + block = start; + +restart: + do { + /* + * We deal with the read-ahead logic here. + */ + if (ra_ptr >= ra_max) { + /* Refill the readahead buffer */ + ra_ptr = 0; + b = block; + for (ra_max = 0; ra_max < NAMEI_RA_SIZE; ra_max++) { + /* + * Terminate if we reach the end of the + * directory and must wrap, or if our + * search has finished at this block. + */ + if (b >= nblocks || (num && block == start)) { + bh_use[ra_max] = NULL; + break; + } + num++; + + bh = NULL; + ocfs2_read_dir_block(dir, b++, &bh, + OCFS2_BH_READAHEAD); + bh_use[ra_max] = bh; + } + } + if ((bh = bh_use[ra_ptr++]) == NULL) + goto next; + if (ocfs2_read_dir_block(dir, block, &bh, 0)) { + /* read error, skip block & hope for the best. + * ocfs2_read_dir_block() has released the bh. */ + mlog(ML_ERROR, "reading directory %llu, " + "offset %lu\n", + (unsigned long long)OCFS2_I(dir)->ip_blkno, + block); + goto next; + } + i = ocfs2_search_dirblock(bh, dir, name, namelen, + block << sb->s_blocksize_bits, + bh->b_data, sb->s_blocksize, + res_dir); + if (i == 1) { + OCFS2_I(dir)->ip_dir_start_lookup = block; + ret = bh; + goto cleanup_and_exit; + } else { + brelse(bh); + if (i < 0) + goto cleanup_and_exit; + } + next: + if (++block >= nblocks) + block = 0; + } while (block != start); + + /* + * If the directory has grown while we were searching, then + * search the last part of the directory before giving up. + */ + block = nblocks; + nblocks = i_size_read(dir) >> sb->s_blocksize_bits; + if (block < nblocks) { + start = 0; + goto restart; + } + +cleanup_and_exit: + /* Clean up the read-ahead blocks */ + for (; ra_ptr < ra_max; ra_ptr++) + brelse(bh_use[ra_ptr]); + + trace_ocfs2_find_entry_el(ret); + return ret; +} + +static int ocfs2_dx_dir_lookup_rec(struct inode *inode, + struct ocfs2_extent_list *el, + u32 major_hash, + u32 *ret_cpos, + u64 *ret_phys_blkno, + unsigned int *ret_clen) +{ + int ret = 0, i, found; + struct buffer_head *eb_bh = NULL; + struct ocfs2_extent_block *eb; + struct ocfs2_extent_rec *rec = NULL; + + if (el->l_tree_depth) { + ret = ocfs2_find_leaf(INODE_CACHE(inode), el, major_hash, + &eb_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + eb = (struct ocfs2_extent_block *) eb_bh->b_data; + el = &eb->h_list; + + if (el->l_tree_depth) { + ret = ocfs2_error(inode->i_sb, + "Inode %llu has non zero tree depth in btree tree block %llu\n", + inode->i_ino, + (unsigned long long)eb_bh->b_blocknr); + goto out; + } + } + + found = 0; + for (i = le16_to_cpu(el->l_next_free_rec) - 1; i >= 0; i--) { + rec = &el->l_recs[i]; + + if (le32_to_cpu(rec->e_cpos) <= major_hash) { + found = 1; + break; + } + } + + if (!found) { + ret = ocfs2_error(inode->i_sb, + "Inode %llu has no extent record for hash %u in btree (next_free_rec %u)\n", + inode->i_ino, major_hash, + le16_to_cpu(el->l_next_free_rec)); + goto out; + } + + if (ret_phys_blkno) + *ret_phys_blkno = le64_to_cpu(rec->e_blkno); + if (ret_cpos) + *ret_cpos = le32_to_cpu(rec->e_cpos); + if (ret_clen) + *ret_clen = le16_to_cpu(rec->e_leaf_clusters); + +out: + brelse(eb_bh); + return ret; +} + +/* + * Returns the block index, from the start of the cluster which this + * hash belongs too. + */ +static inline unsigned int __ocfs2_dx_dir_hash_idx(struct ocfs2_super *osb, + u32 minor_hash) +{ + return minor_hash & osb->osb_dx_mask; +} + +static inline unsigned int ocfs2_dx_dir_hash_idx(struct ocfs2_super *osb, + struct ocfs2_dx_hinfo *hinfo) +{ + return __ocfs2_dx_dir_hash_idx(osb, hinfo->minor_hash); +} + +static int ocfs2_dx_dir_lookup(struct inode *inode, + struct ocfs2_extent_list *el, + struct ocfs2_dx_hinfo *hinfo, + u32 *ret_cpos, + u64 *ret_phys_blkno) +{ + int ret = 0; + unsigned int cend, clen; + u32 cpos; + u64 blkno; + u32 name_hash = hinfo->major_hash; + + ret = ocfs2_dx_dir_lookup_rec(inode, el, name_hash, &cpos, &blkno, + &clen); + if (ret) { + mlog_errno(ret); + goto out; + } + + cend = cpos + clen; + if (name_hash >= cend) { + /* We want the last cluster */ + blkno += ocfs2_clusters_to_blocks(inode->i_sb, clen - 1); + cpos += clen - 1; + } else { + blkno += ocfs2_clusters_to_blocks(inode->i_sb, + name_hash - cpos); + cpos = name_hash; + } + + /* + * We now have the cluster which should hold our entry. To + * find the exact block from the start of the cluster to + * search, we take the lower bits of the hash. + */ + blkno += ocfs2_dx_dir_hash_idx(OCFS2_SB(inode->i_sb), hinfo); + + if (ret_phys_blkno) + *ret_phys_blkno = blkno; + if (ret_cpos) + *ret_cpos = cpos; + +out: + + return ret; +} + +static int ocfs2_dx_dir_search(const char *name, int namelen, + struct inode *dir, + struct ocfs2_dx_root_block *dx_root, + struct ocfs2_dir_lookup_result *res) +{ + int ret, i, found; + u64 phys; + struct buffer_head *dx_leaf_bh = NULL; + struct ocfs2_dx_leaf *dx_leaf; + struct ocfs2_dx_entry *dx_entry = NULL; + struct buffer_head *dir_ent_bh = NULL; + struct ocfs2_dir_entry *dir_ent = NULL; + struct ocfs2_dx_hinfo *hinfo = &res->dl_hinfo; + struct ocfs2_extent_list *dr_el; + struct ocfs2_dx_entry_list *entry_list; + + ocfs2_dx_dir_name_hash(dir, name, namelen, &res->dl_hinfo); + + if (ocfs2_dx_root_inline(dx_root)) { + entry_list = &dx_root->dr_entries; + goto search; + } + + dr_el = &dx_root->dr_list; + + ret = ocfs2_dx_dir_lookup(dir, dr_el, hinfo, NULL, &phys); + if (ret) { + mlog_errno(ret); + goto out; + } + + trace_ocfs2_dx_dir_search((unsigned long long)OCFS2_I(dir)->ip_blkno, + namelen, name, hinfo->major_hash, + hinfo->minor_hash, (unsigned long long)phys); + + ret = ocfs2_read_dx_leaf(dir, phys, &dx_leaf_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + dx_leaf = (struct ocfs2_dx_leaf *) dx_leaf_bh->b_data; + + trace_ocfs2_dx_dir_search_leaf_info( + le16_to_cpu(dx_leaf->dl_list.de_num_used), + le16_to_cpu(dx_leaf->dl_list.de_count)); + + entry_list = &dx_leaf->dl_list; + +search: + /* + * Empty leaf is legal, so no need to check for that. + */ + found = 0; + for (i = 0; i < le16_to_cpu(entry_list->de_num_used); i++) { + dx_entry = &entry_list->de_entries[i]; + + if (hinfo->major_hash != le32_to_cpu(dx_entry->dx_major_hash) + || hinfo->minor_hash != le32_to_cpu(dx_entry->dx_minor_hash)) + continue; + + /* + * Search unindexed leaf block now. We're not + * guaranteed to find anything. + */ + ret = ocfs2_read_dir_block_direct(dir, + le64_to_cpu(dx_entry->dx_dirent_blk), + &dir_ent_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * XXX: We should check the unindexed block here, + * before using it. + */ + + found = ocfs2_search_dirblock(dir_ent_bh, dir, name, namelen, + 0, dir_ent_bh->b_data, + dir->i_sb->s_blocksize, &dir_ent); + if (found == 1) + break; + + if (found == -1) { + /* This means we found a bad directory entry. */ + ret = -EIO; + mlog_errno(ret); + goto out; + } + + brelse(dir_ent_bh); + dir_ent_bh = NULL; + } + + if (found <= 0) { + ret = -ENOENT; + goto out; + } + + res->dl_leaf_bh = dir_ent_bh; + res->dl_entry = dir_ent; + res->dl_dx_leaf_bh = dx_leaf_bh; + res->dl_dx_entry = dx_entry; + + ret = 0; +out: + if (ret) { + brelse(dx_leaf_bh); + brelse(dir_ent_bh); + } + return ret; +} + +static int ocfs2_find_entry_dx(const char *name, int namelen, + struct inode *dir, + struct ocfs2_dir_lookup_result *lookup) +{ + int ret; + struct buffer_head *di_bh = NULL; + struct ocfs2_dinode *di; + struct buffer_head *dx_root_bh = NULL; + struct ocfs2_dx_root_block *dx_root; + + ret = ocfs2_read_inode_block(dir, &di_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + di = (struct ocfs2_dinode *)di_bh->b_data; + + ret = ocfs2_read_dx_root(dir, di, &dx_root_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + dx_root = (struct ocfs2_dx_root_block *) dx_root_bh->b_data; + + ret = ocfs2_dx_dir_search(name, namelen, dir, dx_root, lookup); + if (ret) { + if (ret != -ENOENT) + mlog_errno(ret); + goto out; + } + + lookup->dl_dx_root_bh = dx_root_bh; + dx_root_bh = NULL; +out: + brelse(di_bh); + brelse(dx_root_bh); + return ret; +} + +/* + * Try to find an entry of the provided name within 'dir'. + * + * If nothing was found, -ENOENT is returned. Otherwise, zero is + * returned and the struct 'res' will contain information useful to + * other directory manipulation functions. + * + * Caller can NOT assume anything about the contents of the + * buffer_heads - they are passed back only so that it can be passed + * into any one of the manipulation functions (add entry, delete + * entry, etc). As an example, bh in the extent directory case is a + * data block, in the inline-data case it actually points to an inode, + * in the indexed directory case, multiple buffers are involved. + */ +int ocfs2_find_entry(const char *name, int namelen, + struct inode *dir, struct ocfs2_dir_lookup_result *lookup) +{ + struct buffer_head *bh; + struct ocfs2_dir_entry *res_dir = NULL; + int ret = 0; + + if (ocfs2_dir_indexed(dir)) + return ocfs2_find_entry_dx(name, namelen, dir, lookup); + + if (unlikely(i_size_read(dir) <= 0)) { + ret = -EFSCORRUPTED; + mlog_errno(ret); + goto out; + } + /* + * The unindexed dir code only uses part of the lookup + * structure, so there's no reason to push it down further + * than this. + */ + if (OCFS2_I(dir)->ip_dyn_features & OCFS2_INLINE_DATA_FL) { + if (unlikely(i_size_read(dir) > dir->i_sb->s_blocksize)) { + ret = -EFSCORRUPTED; + mlog_errno(ret); + goto out; + } + bh = ocfs2_find_entry_id(name, namelen, dir, &res_dir); + } else { + bh = ocfs2_find_entry_el(name, namelen, dir, &res_dir); + } + + if (bh == NULL) + return -ENOENT; + + lookup->dl_leaf_bh = bh; + lookup->dl_entry = res_dir; +out: + return ret; +} + +/* + * Update inode number and type of a previously found directory entry. + */ +int ocfs2_update_entry(struct inode *dir, handle_t *handle, + struct ocfs2_dir_lookup_result *res, + struct inode *new_entry_inode) +{ + int ret; + ocfs2_journal_access_func access = ocfs2_journal_access_db; + struct ocfs2_dir_entry *de = res->dl_entry; + struct buffer_head *de_bh = res->dl_leaf_bh; + + /* + * The same code works fine for both inline-data and extent + * based directories, so no need to split this up. The only + * difference is the journal_access function. + */ + + if (OCFS2_I(dir)->ip_dyn_features & OCFS2_INLINE_DATA_FL) + access = ocfs2_journal_access_di; + + ret = access(handle, INODE_CACHE(dir), de_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + + de->inode = cpu_to_le64(OCFS2_I(new_entry_inode)->ip_blkno); + ocfs2_set_de_type(de, new_entry_inode->i_mode); + + ocfs2_journal_dirty(handle, de_bh); + +out: + return ret; +} + +/* + * __ocfs2_delete_entry deletes a directory entry by merging it with the + * previous entry + */ +static int __ocfs2_delete_entry(handle_t *handle, struct inode *dir, + struct ocfs2_dir_entry *de_del, + struct buffer_head *bh, char *first_de, + unsigned int bytes) +{ + struct ocfs2_dir_entry *de, *pde; + int i, status = -ENOENT; + ocfs2_journal_access_func access = ocfs2_journal_access_db; + + if (OCFS2_I(dir)->ip_dyn_features & OCFS2_INLINE_DATA_FL) + access = ocfs2_journal_access_di; + + i = 0; + pde = NULL; + de = (struct ocfs2_dir_entry *) first_de; + while (i < bytes) { + if (!ocfs2_check_dir_entry(dir, de, bh, first_de, bytes, i)) { + status = -EIO; + mlog_errno(status); + goto bail; + } + if (de == de_del) { + status = access(handle, INODE_CACHE(dir), bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + status = -EIO; + mlog_errno(status); + goto bail; + } + if (pde) + le16_add_cpu(&pde->rec_len, + le16_to_cpu(de->rec_len)); + de->inode = 0; + inode_inc_iversion(dir); + ocfs2_journal_dirty(handle, bh); + goto bail; + } + i += le16_to_cpu(de->rec_len); + pde = de; + de = (struct ocfs2_dir_entry *)((char *)de + le16_to_cpu(de->rec_len)); + } +bail: + return status; +} + +static unsigned int ocfs2_figure_dirent_hole(struct ocfs2_dir_entry *de) +{ + unsigned int hole; + + if (le64_to_cpu(de->inode) == 0) + hole = le16_to_cpu(de->rec_len); + else + hole = le16_to_cpu(de->rec_len) - + OCFS2_DIR_REC_LEN(de->name_len); + + return hole; +} + +static int ocfs2_find_max_rec_len(struct super_block *sb, + struct buffer_head *dirblock_bh) +{ + int size, this_hole, largest_hole = 0; + char *trailer, *de_buf, *limit, *start = dirblock_bh->b_data; + struct ocfs2_dir_entry *de; + + trailer = (char *)ocfs2_trailer_from_bh(dirblock_bh, sb); + size = ocfs2_dir_trailer_blk_off(sb); + limit = start + size; + de_buf = start; + de = (struct ocfs2_dir_entry *)de_buf; + do { + if (de_buf != trailer) { + this_hole = ocfs2_figure_dirent_hole(de); + if (this_hole > largest_hole) + largest_hole = this_hole; + } + + de_buf += le16_to_cpu(de->rec_len); + de = (struct ocfs2_dir_entry *)de_buf; + } while (de_buf < limit); + + if (largest_hole >= OCFS2_DIR_MIN_REC_LEN) + return largest_hole; + return 0; +} + +static void ocfs2_dx_list_remove_entry(struct ocfs2_dx_entry_list *entry_list, + int index) +{ + int num_used = le16_to_cpu(entry_list->de_num_used); + + if (num_used == 1 || index == (num_used - 1)) + goto clear; + + memmove(&entry_list->de_entries[index], + &entry_list->de_entries[index + 1], + (num_used - index - 1)*sizeof(struct ocfs2_dx_entry)); +clear: + num_used--; + memset(&entry_list->de_entries[num_used], 0, + sizeof(struct ocfs2_dx_entry)); + entry_list->de_num_used = cpu_to_le16(num_used); +} + +static int ocfs2_delete_entry_dx(handle_t *handle, struct inode *dir, + struct ocfs2_dir_lookup_result *lookup) +{ + int ret, index, max_rec_len, add_to_free_list = 0; + struct buffer_head *dx_root_bh = lookup->dl_dx_root_bh; + struct buffer_head *leaf_bh = lookup->dl_leaf_bh; + struct ocfs2_dx_leaf *dx_leaf; + struct ocfs2_dx_entry *dx_entry = lookup->dl_dx_entry; + struct ocfs2_dir_block_trailer *trailer; + struct ocfs2_dx_root_block *dx_root; + struct ocfs2_dx_entry_list *entry_list; + + /* + * This function gets a bit messy because we might have to + * modify the root block, regardless of whether the indexed + * entries are stored inline. + */ + + /* + * *Only* set 'entry_list' here, based on where we're looking + * for the indexed entries. Later, we might still want to + * journal both blocks, based on free list state. + */ + dx_root = (struct ocfs2_dx_root_block *)dx_root_bh->b_data; + if (ocfs2_dx_root_inline(dx_root)) { + entry_list = &dx_root->dr_entries; + } else { + dx_leaf = (struct ocfs2_dx_leaf *) lookup->dl_dx_leaf_bh->b_data; + entry_list = &dx_leaf->dl_list; + } + + /* Neither of these are a disk corruption - that should have + * been caught by lookup, before we got here. */ + BUG_ON(le16_to_cpu(entry_list->de_count) <= 0); + BUG_ON(le16_to_cpu(entry_list->de_num_used) <= 0); + + index = (char *)dx_entry - (char *)entry_list->de_entries; + index /= sizeof(*dx_entry); + + if (index >= le16_to_cpu(entry_list->de_num_used)) { + mlog(ML_ERROR, "Dir %llu: Bad dx_entry ptr idx %d, (%p, %p)\n", + (unsigned long long)OCFS2_I(dir)->ip_blkno, index, + entry_list, dx_entry); + return -EIO; + } + + /* + * We know that removal of this dirent will leave enough room + * for a new one, so add this block to the free list if it + * isn't already there. + */ + trailer = ocfs2_trailer_from_bh(leaf_bh, dir->i_sb); + if (trailer->db_free_rec_len == 0) + add_to_free_list = 1; + + /* + * Add the block holding our index into the journal before + * removing the unindexed entry. If we get an error return + * from __ocfs2_delete_entry(), then it hasn't removed the + * entry yet. Likewise, successful return means we *must* + * remove the indexed entry. + * + * We're also careful to journal the root tree block here as + * the entry count needs to be updated. Also, we might be + * adding to the start of the free list. + */ + ret = ocfs2_journal_access_dr(handle, INODE_CACHE(dir), dx_root_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (!ocfs2_dx_root_inline(dx_root)) { + ret = ocfs2_journal_access_dl(handle, INODE_CACHE(dir), + lookup->dl_dx_leaf_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + trace_ocfs2_delete_entry_dx((unsigned long long)OCFS2_I(dir)->ip_blkno, + index); + + ret = __ocfs2_delete_entry(handle, dir, lookup->dl_entry, + leaf_bh, leaf_bh->b_data, leaf_bh->b_size); + if (ret) { + mlog_errno(ret); + goto out; + } + + max_rec_len = ocfs2_find_max_rec_len(dir->i_sb, leaf_bh); + trailer->db_free_rec_len = cpu_to_le16(max_rec_len); + if (add_to_free_list) { + trailer->db_free_next = dx_root->dr_free_blk; + dx_root->dr_free_blk = cpu_to_le64(leaf_bh->b_blocknr); + ocfs2_journal_dirty(handle, dx_root_bh); + } + + /* leaf_bh was journal_accessed for us in __ocfs2_delete_entry */ + ocfs2_journal_dirty(handle, leaf_bh); + + le32_add_cpu(&dx_root->dr_num_entries, -1); + ocfs2_journal_dirty(handle, dx_root_bh); + + ocfs2_dx_list_remove_entry(entry_list, index); + + if (!ocfs2_dx_root_inline(dx_root)) + ocfs2_journal_dirty(handle, lookup->dl_dx_leaf_bh); + +out: + return ret; +} + +static inline int ocfs2_delete_entry_id(handle_t *handle, + struct inode *dir, + struct ocfs2_dir_entry *de_del, + struct buffer_head *bh) +{ + int ret; + struct buffer_head *di_bh = NULL; + struct ocfs2_dinode *di; + struct ocfs2_inline_data *data; + + ret = ocfs2_read_inode_block(dir, &di_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + di = (struct ocfs2_dinode *)di_bh->b_data; + data = &di->id2.i_data; + + ret = __ocfs2_delete_entry(handle, dir, de_del, bh, data->id_data, + i_size_read(dir)); + + brelse(di_bh); +out: + return ret; +} + +static inline int ocfs2_delete_entry_el(handle_t *handle, + struct inode *dir, + struct ocfs2_dir_entry *de_del, + struct buffer_head *bh) +{ + return __ocfs2_delete_entry(handle, dir, de_del, bh, bh->b_data, + bh->b_size); +} + +/* + * Delete a directory entry. Hide the details of directory + * implementation from the caller. + */ +int ocfs2_delete_entry(handle_t *handle, + struct inode *dir, + struct ocfs2_dir_lookup_result *res) +{ + if (ocfs2_dir_indexed(dir)) + return ocfs2_delete_entry_dx(handle, dir, res); + + if (OCFS2_I(dir)->ip_dyn_features & OCFS2_INLINE_DATA_FL) + return ocfs2_delete_entry_id(handle, dir, res->dl_entry, + res->dl_leaf_bh); + + return ocfs2_delete_entry_el(handle, dir, res->dl_entry, + res->dl_leaf_bh); +} + +/* + * Check whether 'de' has enough room to hold an entry of + * 'new_rec_len' bytes. + */ +static inline int ocfs2_dirent_would_fit(struct ocfs2_dir_entry *de, + unsigned int new_rec_len) +{ + unsigned int de_really_used; + + /* Check whether this is an empty record with enough space */ + if (le64_to_cpu(de->inode) == 0 && + le16_to_cpu(de->rec_len) >= new_rec_len) + return 1; + + /* + * Record might have free space at the end which we can + * use. + */ + de_really_used = OCFS2_DIR_REC_LEN(de->name_len); + if (le16_to_cpu(de->rec_len) >= (de_really_used + new_rec_len)) + return 1; + + return 0; +} + +static void ocfs2_dx_dir_leaf_insert_tail(struct ocfs2_dx_leaf *dx_leaf, + struct ocfs2_dx_entry *dx_new_entry) +{ + int i; + + i = le16_to_cpu(dx_leaf->dl_list.de_num_used); + dx_leaf->dl_list.de_entries[i] = *dx_new_entry; + + le16_add_cpu(&dx_leaf->dl_list.de_num_used, 1); +} + +static void ocfs2_dx_entry_list_insert(struct ocfs2_dx_entry_list *entry_list, + struct ocfs2_dx_hinfo *hinfo, + u64 dirent_blk) +{ + int i; + struct ocfs2_dx_entry *dx_entry; + + i = le16_to_cpu(entry_list->de_num_used); + dx_entry = &entry_list->de_entries[i]; + + memset(dx_entry, 0, sizeof(*dx_entry)); + dx_entry->dx_major_hash = cpu_to_le32(hinfo->major_hash); + dx_entry->dx_minor_hash = cpu_to_le32(hinfo->minor_hash); + dx_entry->dx_dirent_blk = cpu_to_le64(dirent_blk); + + le16_add_cpu(&entry_list->de_num_used, 1); +} + +static int __ocfs2_dx_dir_leaf_insert(struct inode *dir, handle_t *handle, + struct ocfs2_dx_hinfo *hinfo, + u64 dirent_blk, + struct buffer_head *dx_leaf_bh) +{ + int ret; + struct ocfs2_dx_leaf *dx_leaf; + + ret = ocfs2_journal_access_dl(handle, INODE_CACHE(dir), dx_leaf_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + + dx_leaf = (struct ocfs2_dx_leaf *)dx_leaf_bh->b_data; + ocfs2_dx_entry_list_insert(&dx_leaf->dl_list, hinfo, dirent_blk); + ocfs2_journal_dirty(handle, dx_leaf_bh); + +out: + return ret; +} + +static void ocfs2_dx_inline_root_insert(struct inode *dir, handle_t *handle, + struct ocfs2_dx_hinfo *hinfo, + u64 dirent_blk, + struct ocfs2_dx_root_block *dx_root) +{ + ocfs2_dx_entry_list_insert(&dx_root->dr_entries, hinfo, dirent_blk); +} + +static int ocfs2_dx_dir_insert(struct inode *dir, handle_t *handle, + struct ocfs2_dir_lookup_result *lookup) +{ + int ret = 0; + struct ocfs2_dx_root_block *dx_root; + struct buffer_head *dx_root_bh = lookup->dl_dx_root_bh; + + ret = ocfs2_journal_access_dr(handle, INODE_CACHE(dir), dx_root_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + + dx_root = (struct ocfs2_dx_root_block *)lookup->dl_dx_root_bh->b_data; + if (ocfs2_dx_root_inline(dx_root)) { + ocfs2_dx_inline_root_insert(dir, handle, + &lookup->dl_hinfo, + lookup->dl_leaf_bh->b_blocknr, + dx_root); + } else { + ret = __ocfs2_dx_dir_leaf_insert(dir, handle, &lookup->dl_hinfo, + lookup->dl_leaf_bh->b_blocknr, + lookup->dl_dx_leaf_bh); + if (ret) + goto out; + } + + le32_add_cpu(&dx_root->dr_num_entries, 1); + ocfs2_journal_dirty(handle, dx_root_bh); + +out: + return ret; +} + +static void ocfs2_remove_block_from_free_list(struct inode *dir, + handle_t *handle, + struct ocfs2_dir_lookup_result *lookup) +{ + struct ocfs2_dir_block_trailer *trailer, *prev; + struct ocfs2_dx_root_block *dx_root; + struct buffer_head *bh; + + trailer = ocfs2_trailer_from_bh(lookup->dl_leaf_bh, dir->i_sb); + + if (ocfs2_free_list_at_root(lookup)) { + bh = lookup->dl_dx_root_bh; + dx_root = (struct ocfs2_dx_root_block *)bh->b_data; + dx_root->dr_free_blk = trailer->db_free_next; + } else { + bh = lookup->dl_prev_leaf_bh; + prev = ocfs2_trailer_from_bh(bh, dir->i_sb); + prev->db_free_next = trailer->db_free_next; + } + + trailer->db_free_rec_len = cpu_to_le16(0); + trailer->db_free_next = cpu_to_le64(0); + + ocfs2_journal_dirty(handle, bh); + ocfs2_journal_dirty(handle, lookup->dl_leaf_bh); +} + +/* + * This expects that a journal write has been reserved on + * lookup->dl_prev_leaf_bh or lookup->dl_dx_root_bh + */ +static void ocfs2_recalc_free_list(struct inode *dir, handle_t *handle, + struct ocfs2_dir_lookup_result *lookup) +{ + int max_rec_len; + struct ocfs2_dir_block_trailer *trailer; + + /* Walk dl_leaf_bh to figure out what the new free rec_len is. */ + max_rec_len = ocfs2_find_max_rec_len(dir->i_sb, lookup->dl_leaf_bh); + if (max_rec_len) { + /* + * There's still room in this block, so no need to remove it + * from the free list. In this case, we just want to update + * the rec len accounting. + */ + trailer = ocfs2_trailer_from_bh(lookup->dl_leaf_bh, dir->i_sb); + trailer->db_free_rec_len = cpu_to_le16(max_rec_len); + ocfs2_journal_dirty(handle, lookup->dl_leaf_bh); + } else { + ocfs2_remove_block_from_free_list(dir, handle, lookup); + } +} + +/* we don't always have a dentry for what we want to add, so people + * like orphan dir can call this instead. + * + * The lookup context must have been filled from + * ocfs2_prepare_dir_for_insert. + */ +int __ocfs2_add_entry(handle_t *handle, + struct inode *dir, + const char *name, int namelen, + struct inode *inode, u64 blkno, + struct buffer_head *parent_fe_bh, + struct ocfs2_dir_lookup_result *lookup) +{ + unsigned long offset; + unsigned short rec_len; + struct ocfs2_dir_entry *de, *de1; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)parent_fe_bh->b_data; + struct super_block *sb = dir->i_sb; + int retval; + unsigned int size = sb->s_blocksize; + struct buffer_head *insert_bh = lookup->dl_leaf_bh; + char *data_start = insert_bh->b_data; + + if (ocfs2_dir_indexed(dir)) { + struct buffer_head *bh; + + /* + * An indexed dir may require that we update the free space + * list. Reserve a write to the previous node in the list so + * that we don't fail later. + * + * XXX: This can be either a dx_root_block, or an unindexed + * directory tree leaf block. + */ + if (ocfs2_free_list_at_root(lookup)) { + bh = lookup->dl_dx_root_bh; + retval = ocfs2_journal_access_dr(handle, + INODE_CACHE(dir), bh, + OCFS2_JOURNAL_ACCESS_WRITE); + } else { + bh = lookup->dl_prev_leaf_bh; + retval = ocfs2_journal_access_db(handle, + INODE_CACHE(dir), bh, + OCFS2_JOURNAL_ACCESS_WRITE); + } + if (retval) { + mlog_errno(retval); + return retval; + } + } else if (OCFS2_I(dir)->ip_dyn_features & OCFS2_INLINE_DATA_FL) { + data_start = di->id2.i_data.id_data; + size = i_size_read(dir); + + BUG_ON(insert_bh != parent_fe_bh); + } + + rec_len = OCFS2_DIR_REC_LEN(namelen); + offset = 0; + de = (struct ocfs2_dir_entry *) data_start; + while (1) { + BUG_ON((char *)de >= (size + data_start)); + + /* These checks should've already been passed by the + * prepare function, but I guess we can leave them + * here anyway. */ + if (!ocfs2_check_dir_entry(dir, de, insert_bh, data_start, + size, offset)) { + retval = -ENOENT; + goto bail; + } + if (ocfs2_match(namelen, name, de)) { + retval = -EEXIST; + goto bail; + } + + /* We're guaranteed that we should have space, so we + * can't possibly have hit the trailer...right? */ + mlog_bug_on_msg(ocfs2_skip_dir_trailer(dir, de, offset, size), + "Hit dir trailer trying to insert %.*s " + "(namelen %d) into directory %llu. " + "offset is %lu, trailer offset is %d\n", + namelen, name, namelen, + (unsigned long long)parent_fe_bh->b_blocknr, + offset, ocfs2_dir_trailer_blk_off(dir->i_sb)); + + if (ocfs2_dirent_would_fit(de, rec_len)) { + inode_set_mtime_to_ts(dir, + inode_set_ctime_current(dir)); + retval = ocfs2_mark_inode_dirty(handle, dir, parent_fe_bh); + if (retval < 0) { + mlog_errno(retval); + goto bail; + } + + if (insert_bh == parent_fe_bh) + retval = ocfs2_journal_access_di(handle, + INODE_CACHE(dir), + insert_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + else { + retval = ocfs2_journal_access_db(handle, + INODE_CACHE(dir), + insert_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + + if (!retval && ocfs2_dir_indexed(dir)) + retval = ocfs2_dx_dir_insert(dir, + handle, + lookup); + } + + if (retval) { + mlog_errno(retval); + goto bail; + } + + /* By now the buffer is marked for journaling */ + offset += le16_to_cpu(de->rec_len); + if (le64_to_cpu(de->inode)) { + de1 = (struct ocfs2_dir_entry *)((char *) de + + OCFS2_DIR_REC_LEN(de->name_len)); + de1->rec_len = + cpu_to_le16(le16_to_cpu(de->rec_len) - + OCFS2_DIR_REC_LEN(de->name_len)); + de->rec_len = cpu_to_le16(OCFS2_DIR_REC_LEN(de->name_len)); + de = de1; + } + de->file_type = FT_UNKNOWN; + if (blkno) { + de->inode = cpu_to_le64(blkno); + ocfs2_set_de_type(de, inode->i_mode); + } else + de->inode = 0; + de->name_len = namelen; + memcpy(de->name, name, namelen); + + if (ocfs2_dir_indexed(dir)) + ocfs2_recalc_free_list(dir, handle, lookup); + + inode_inc_iversion(dir); + ocfs2_journal_dirty(handle, insert_bh); + retval = 0; + goto bail; + } + + offset += le16_to_cpu(de->rec_len); + de = (struct ocfs2_dir_entry *) ((char *) de + le16_to_cpu(de->rec_len)); + } + + /* when you think about it, the assert above should prevent us + * from ever getting here. */ + retval = -ENOSPC; +bail: + if (retval) + mlog_errno(retval); + + return retval; +} + +static int ocfs2_dir_foreach_blk_id(struct inode *inode, + u64 *f_version, + struct dir_context *ctx) +{ + int ret, i; + unsigned long offset = ctx->pos; + struct buffer_head *di_bh = NULL; + struct ocfs2_dinode *di; + struct ocfs2_inline_data *data; + struct ocfs2_dir_entry *de; + + ret = ocfs2_read_inode_block(inode, &di_bh); + if (ret) { + mlog(ML_ERROR, "Unable to read inode block for dir %llu\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno); + goto out; + } + + di = (struct ocfs2_dinode *)di_bh->b_data; + data = &di->id2.i_data; + + while (ctx->pos < i_size_read(inode)) { + /* If the dir block has changed since the last call to + * readdir(2), then we might be pointing to an invalid + * dirent right now. Scan from the start of the block + * to make sure. */ + if (!inode_eq_iversion(inode, *f_version)) { + loff_t size = i_size_read(inode); + + for (i = 0; i + OCFS2_DIR_REC_LEN(1) <= size && + i < offset;) { + unsigned int rec_len; + + de = (struct ocfs2_dir_entry *) + (data->id_data + i); + /* It's too expensive to do a full + * dirent test each time round this + * loop, but we do have to test at + * least that it is non-zero. A + * failure will be detected in the + * dirent test below. */ + rec_len = le16_to_cpu(de->rec_len); + if (rec_len < OCFS2_DIR_REC_LEN(1) || + i + rec_len > size) + break; + i += rec_len; + } + ctx->pos = offset = i; + *f_version = inode_query_iversion(inode); + } + + de = (struct ocfs2_dir_entry *) (data->id_data + ctx->pos); + if (!ocfs2_check_dir_entry(inode, de, di_bh, (char *)data->id_data, + i_size_read(inode), ctx->pos)) { + /* On error, skip the f_pos to the end. */ + ctx->pos = i_size_read(inode); + break; + } + offset += le16_to_cpu(de->rec_len); + if (le64_to_cpu(de->inode)) { + if (!dir_emit(ctx, de->name, de->name_len, + le64_to_cpu(de->inode), + fs_ftype_to_dtype(de->file_type))) + goto out; + } + ctx->pos += le16_to_cpu(de->rec_len); + } +out: + brelse(di_bh); + return 0; +} + +/* + * NOTE: This function can be called against unindexed directories, + * and indexed ones. + */ +static int ocfs2_dir_foreach_blk_el(struct inode *inode, + u64 *f_version, + struct dir_context *ctx, + bool persist) +{ + unsigned long offset, blk, last_ra_blk = 0; + int i; + struct buffer_head * bh, * tmp; + struct ocfs2_dir_entry * de; + struct super_block * sb = inode->i_sb; + unsigned int ra_sectors = 16; + int stored = 0; + int ret; + + bh = NULL; + + offset = ctx->pos & (sb->s_blocksize - 1); + + while (ctx->pos < i_size_read(inode)) { + blk = ctx->pos >> sb->s_blocksize_bits; + ret = ocfs2_read_dir_block(inode, blk, &bh, 0); + if (ret) { + if (persist) + return ret; + /* Skip the corrupt dirblock and keep trying */ + ctx->pos += sb->s_blocksize - offset; + offset = 0; + continue; + } + + /* The idea here is to begin with 8k read-ahead and to stay + * 4k ahead of our current position. + * + * TODO: Use the pagecache for this. We just need to + * make sure it's cluster-safe... */ + if (!last_ra_blk + || (((last_ra_blk - blk) << 9) <= (ra_sectors / 2))) { + for (i = ra_sectors >> (sb->s_blocksize_bits - 9); + i > 0; i--) { + tmp = NULL; + if (!ocfs2_read_dir_block(inode, ++blk, &tmp, + OCFS2_BH_READAHEAD)) + brelse(tmp); + } + last_ra_blk = blk; + ra_sectors = 8; + } + + /* If the dir block has changed since the last call to + * readdir(2), then we might be pointing to an invalid + * dirent right now. Scan from the start of the block + * to make sure. */ + if (!inode_eq_iversion(inode, *f_version)) { + for (i = 0; i + OCFS2_DIR_REC_LEN(1) <= sb->s_blocksize && + i < offset;) { + unsigned int rec_len; + + de = (struct ocfs2_dir_entry *) (bh->b_data + i); + /* It's too expensive to do a full + * dirent test each time round this + * loop, but we do have to test at + * least that it is non-zero. A + * failure will be detected in the + * dirent test below. */ + rec_len = le16_to_cpu(de->rec_len); + if (rec_len < OCFS2_DIR_REC_LEN(1) || + i + rec_len > sb->s_blocksize) + break; + i += rec_len; + } + offset = i; + ctx->pos = (ctx->pos & ~((loff_t)sb->s_blocksize - 1)) + | offset; + *f_version = inode_query_iversion(inode); + } + + while (ctx->pos < i_size_read(inode) + && offset < sb->s_blocksize) { + de = (struct ocfs2_dir_entry *) (bh->b_data + offset); + if (!ocfs2_check_dir_entry(inode, de, bh, bh->b_data, + sb->s_blocksize, offset)) { + /* On error, skip the f_pos to the + next block. */ + ctx->pos = (ctx->pos | (sb->s_blocksize - 1)) + 1; + break; + } + if (le64_to_cpu(de->inode)) { + if (!dir_emit(ctx, de->name, + de->name_len, + le64_to_cpu(de->inode), + fs_ftype_to_dtype(de->file_type))) { + brelse(bh); + return 0; + } + stored++; + } + offset += le16_to_cpu(de->rec_len); + ctx->pos += le16_to_cpu(de->rec_len); + } + offset = 0; + brelse(bh); + bh = NULL; + if (!persist && stored) + break; + } + return 0; +} + +static int ocfs2_dir_foreach_blk(struct inode *inode, u64 *f_version, + struct dir_context *ctx, + bool persist) +{ + if (OCFS2_I(inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL) + return ocfs2_dir_foreach_blk_id(inode, f_version, ctx); + return ocfs2_dir_foreach_blk_el(inode, f_version, ctx, persist); +} + +/* + * This is intended to be called from inside other kernel functions, + * so we fake some arguments. + */ +int ocfs2_dir_foreach(struct inode *inode, struct dir_context *ctx) +{ + u64 version = inode_query_iversion(inode); + return ocfs2_dir_foreach_blk(inode, &version, ctx, true); +} + +/* + * ocfs2_readdir() + * + */ +int ocfs2_readdir(struct file *file, struct dir_context *ctx) +{ + int error = 0; + struct inode *inode = file_inode(file); + struct ocfs2_file_private *fp = file->private_data; + int lock_level = 0; + + trace_ocfs2_readdir((unsigned long long)OCFS2_I(inode)->ip_blkno); + + error = ocfs2_inode_lock_atime(inode, file->f_path.mnt, &lock_level, 1); + if (lock_level && error >= 0) { + /* We release EX lock which used to update atime + * and get PR lock again to reduce contention + * on commonly accessed directories. */ + ocfs2_inode_unlock(inode, 1); + lock_level = 0; + error = ocfs2_inode_lock(inode, NULL, 0); + } + if (error < 0) { + if (error != -ENOENT) + mlog_errno(error); + /* we haven't got any yet, so propagate the error. */ + goto bail_nolock; + } + + error = ocfs2_dir_foreach_blk(inode, &fp->cookie, ctx, false); + + ocfs2_inode_unlock(inode, lock_level); + if (error) + mlog_errno(error); + +bail_nolock: + + return error; +} + +/* + * NOTE: this should always be called with parent dir i_rwsem taken. + */ +int ocfs2_find_files_on_disk(const char *name, + int namelen, + u64 *blkno, + struct inode *inode, + struct ocfs2_dir_lookup_result *lookup) +{ + int status = -ENOENT; + + trace_ocfs2_find_files_on_disk(namelen, name, blkno, + (unsigned long long)OCFS2_I(inode)->ip_blkno); + + status = ocfs2_find_entry(name, namelen, inode, lookup); + if (status) + goto leave; + + *blkno = le64_to_cpu(lookup->dl_entry->inode); + + status = 0; +leave: + + return status; +} + +/* + * Convenience function for callers which just want the block number + * mapped to a name and don't require the full dirent info, etc. + */ +int ocfs2_lookup_ino_from_name(struct inode *dir, const char *name, + int namelen, u64 *blkno) +{ + int ret; + struct ocfs2_dir_lookup_result lookup = { NULL, }; + + ret = ocfs2_find_files_on_disk(name, namelen, blkno, dir, &lookup); + ocfs2_free_dir_lookup_result(&lookup); + + return ret; +} + +/* Check for a name within a directory. + * + * Return 0 if the name does not exist + * Return -EEXIST if the directory contains the name + * Return -EFSCORRUPTED if found corruption + * + * Callers should have i_rwsem + a cluster lock on dir + */ +int ocfs2_check_dir_for_entry(struct inode *dir, + const char *name, + int namelen) +{ + int ret = 0; + struct ocfs2_dir_lookup_result lookup = { NULL, }; + + trace_ocfs2_check_dir_for_entry( + (unsigned long long)OCFS2_I(dir)->ip_blkno, namelen, name); + + ret = ocfs2_find_entry(name, namelen, dir, &lookup); + if (ret == 0) { + ret = -EEXIST; + mlog_errno(ret); + } else if (ret == -ENOENT) { + ret = 0; + } + + ocfs2_free_dir_lookup_result(&lookup); + + return ret; +} + +struct ocfs2_empty_dir_priv { + struct dir_context ctx; + unsigned seen_dot; + unsigned seen_dot_dot; + unsigned seen_other; + unsigned dx_dir; +}; +static bool ocfs2_empty_dir_filldir(struct dir_context *ctx, const char *name, + int name_len, loff_t pos, u64 ino, + unsigned type) +{ + struct ocfs2_empty_dir_priv *p = + container_of(ctx, struct ocfs2_empty_dir_priv, ctx); + + /* + * Check the positions of "." and ".." records to be sure + * they're in the correct place. + * + * Indexed directories don't need to proceed past the first + * two entries, so we end the scan after seeing '..'. Despite + * that, we allow the scan to proceed In the event that we + * have a corrupted indexed directory (no dot or dot dot + * entries). This allows us to double check for existing + * entries which might not have been found in the index. + */ + if (name_len == 1 && !strncmp(".", name, 1) && pos == 0) { + p->seen_dot = 1; + return true; + } + + if (name_len == 2 && !strncmp("..", name, 2) && + pos == OCFS2_DIR_REC_LEN(1)) { + p->seen_dot_dot = 1; + + if (p->dx_dir && p->seen_dot) + return false; + + return true; + } + + p->seen_other = 1; + return false; +} + +static int ocfs2_empty_dir_dx(struct inode *inode, + struct ocfs2_empty_dir_priv *priv) +{ + int ret; + struct buffer_head *di_bh = NULL; + struct buffer_head *dx_root_bh = NULL; + struct ocfs2_dinode *di; + struct ocfs2_dx_root_block *dx_root; + + priv->dx_dir = 1; + + ret = ocfs2_read_inode_block(inode, &di_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + di = (struct ocfs2_dinode *)di_bh->b_data; + + ret = ocfs2_read_dx_root(inode, di, &dx_root_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + dx_root = (struct ocfs2_dx_root_block *)dx_root_bh->b_data; + + if (le32_to_cpu(dx_root->dr_num_entries) != 2) + priv->seen_other = 1; + +out: + brelse(di_bh); + brelse(dx_root_bh); + return ret; +} + +/* + * routine to check that the specified directory is empty (for rmdir) + * + * Returns 1 if dir is empty, zero if not, and a negative errno on error. + * + * XXX: This is a performance problem for unindexed directories. + */ +int ocfs2_empty_dir(struct inode *inode) +{ + int ret; + struct ocfs2_empty_dir_priv priv = { + .ctx.actor = ocfs2_empty_dir_filldir, + }; + + if (ocfs2_dir_indexed(inode)) { + ret = ocfs2_empty_dir_dx(inode, &priv); + if (ret) { + mlog_errno(ret); + return ret; + } + /* + * We still run ocfs2_dir_foreach to get the checks + * for "." and "..". + */ + } + + ret = ocfs2_dir_foreach(inode, &priv.ctx); + if (ret) { + mlog_errno(ret); + return ret; + } + + if (!priv.seen_dot || !priv.seen_dot_dot) { + mlog(ML_ERROR, "bad directory (dir #%llu) - no `.' or `..'\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno); + /* + * XXX: Is it really safe to allow an unlink to continue? + */ + return 1; + } + + return !priv.seen_other; +} + +/* + * Fills "." and ".." dirents in a new directory block. Returns dirent for + * "..", which might be used during creation of a directory with a trailing + * header. It is otherwise safe to ignore the return code. + */ +static struct ocfs2_dir_entry *ocfs2_fill_initial_dirents(struct inode *inode, + struct inode *parent, + char *start, + unsigned int size) +{ + struct ocfs2_dir_entry *de = (struct ocfs2_dir_entry *)start; + + de->inode = cpu_to_le64(OCFS2_I(inode)->ip_blkno); + de->name_len = 1; + de->rec_len = + cpu_to_le16(OCFS2_DIR_REC_LEN(de->name_len)); + strscpy(de->name, "."); + ocfs2_set_de_type(de, S_IFDIR); + + de = (struct ocfs2_dir_entry *) ((char *)de + le16_to_cpu(de->rec_len)); + de->inode = cpu_to_le64(OCFS2_I(parent)->ip_blkno); + de->rec_len = cpu_to_le16(size - OCFS2_DIR_REC_LEN(1)); + de->name_len = 2; + strscpy(de->name, ".."); + ocfs2_set_de_type(de, S_IFDIR); + + return de; +} + +/* + * This works together with code in ocfs2_mknod_locked() which sets + * the inline-data flag and initializes the inline-data section. + */ +static int ocfs2_fill_new_dir_id(struct ocfs2_super *osb, + handle_t *handle, + struct inode *parent, + struct inode *inode, + struct buffer_head *di_bh) +{ + int ret; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data; + struct ocfs2_inline_data *data = &di->id2.i_data; + unsigned int size = le16_to_cpu(data->id_count); + + ret = ocfs2_journal_access_di(handle, INODE_CACHE(inode), di_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + + ocfs2_fill_initial_dirents(inode, parent, data->id_data, size); + ocfs2_journal_dirty(handle, di_bh); + + i_size_write(inode, size); + set_nlink(inode, 2); + inode->i_blocks = ocfs2_inode_sector_count(inode); + + ret = ocfs2_mark_inode_dirty(handle, inode, di_bh); + if (ret < 0) + mlog_errno(ret); + +out: + return ret; +} + +static int ocfs2_fill_new_dir_el(struct ocfs2_super *osb, + handle_t *handle, + struct inode *parent, + struct inode *inode, + struct buffer_head *fe_bh, + struct ocfs2_alloc_context *data_ac, + struct buffer_head **ret_new_bh) +{ + int status; + unsigned int size = osb->sb->s_blocksize; + struct buffer_head *new_bh = NULL; + struct ocfs2_dir_entry *de; + + if (ocfs2_new_dir_wants_trailer(inode)) + size = ocfs2_dir_trailer_blk_off(parent->i_sb); + + status = ocfs2_do_extend_dir(osb->sb, handle, inode, fe_bh, + data_ac, NULL, &new_bh); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + ocfs2_set_new_buffer_uptodate(INODE_CACHE(inode), new_bh); + + status = ocfs2_journal_access_db(handle, INODE_CACHE(inode), new_bh, + OCFS2_JOURNAL_ACCESS_CREATE); + if (status < 0) { + mlog_errno(status); + goto bail; + } + memset(new_bh->b_data, 0, osb->sb->s_blocksize); + + de = ocfs2_fill_initial_dirents(inode, parent, new_bh->b_data, size); + if (ocfs2_new_dir_wants_trailer(inode)) { + int size = le16_to_cpu(de->rec_len); + + /* + * Figure out the size of the hole left over after + * insertion of '.' and '..'. The trailer wants this + * information. + */ + size -= OCFS2_DIR_REC_LEN(2); + size -= sizeof(struct ocfs2_dir_block_trailer); + + ocfs2_init_dir_trailer(inode, new_bh, size); + } + + ocfs2_journal_dirty(handle, new_bh); + + i_size_write(inode, inode->i_sb->s_blocksize); + set_nlink(inode, 2); + inode->i_blocks = ocfs2_inode_sector_count(inode); + status = ocfs2_mark_inode_dirty(handle, inode, fe_bh); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + status = 0; + if (ret_new_bh) { + *ret_new_bh = new_bh; + new_bh = NULL; + } +bail: + brelse(new_bh); + + return status; +} + +static int ocfs2_dx_dir_attach_index(struct ocfs2_super *osb, + handle_t *handle, struct inode *dir, + struct buffer_head *di_bh, + struct buffer_head *dirdata_bh, + struct ocfs2_alloc_context *meta_ac, + int dx_inline, u32 num_entries, + struct buffer_head **ret_dx_root_bh) +{ + int ret; + struct ocfs2_dinode *di = (struct ocfs2_dinode *) di_bh->b_data; + u16 dr_suballoc_bit; + u64 suballoc_loc, dr_blkno; + unsigned int num_bits; + struct buffer_head *dx_root_bh = NULL; + struct ocfs2_dx_root_block *dx_root; + struct ocfs2_dir_block_trailer *trailer = + ocfs2_trailer_from_bh(dirdata_bh, dir->i_sb); + + ret = ocfs2_claim_metadata(handle, meta_ac, 1, &suballoc_loc, + &dr_suballoc_bit, &num_bits, &dr_blkno); + if (ret) { + mlog_errno(ret); + goto out; + } + + trace_ocfs2_dx_dir_attach_index( + (unsigned long long)OCFS2_I(dir)->ip_blkno, + (unsigned long long)dr_blkno); + + dx_root_bh = sb_getblk(osb->sb, dr_blkno); + if (dx_root_bh == NULL) { + ret = -ENOMEM; + goto out; + } + ocfs2_set_new_buffer_uptodate(INODE_CACHE(dir), dx_root_bh); + + ret = ocfs2_journal_access_dr(handle, INODE_CACHE(dir), dx_root_bh, + OCFS2_JOURNAL_ACCESS_CREATE); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + dx_root = (struct ocfs2_dx_root_block *)dx_root_bh->b_data; + memset(dx_root, 0, osb->sb->s_blocksize); + strscpy(dx_root->dr_signature, OCFS2_DX_ROOT_SIGNATURE); + dx_root->dr_suballoc_slot = cpu_to_le16(meta_ac->ac_alloc_slot); + dx_root->dr_suballoc_loc = cpu_to_le64(suballoc_loc); + dx_root->dr_suballoc_bit = cpu_to_le16(dr_suballoc_bit); + dx_root->dr_fs_generation = cpu_to_le32(osb->fs_generation); + dx_root->dr_blkno = cpu_to_le64(dr_blkno); + dx_root->dr_dir_blkno = cpu_to_le64(OCFS2_I(dir)->ip_blkno); + dx_root->dr_num_entries = cpu_to_le32(num_entries); + if (le16_to_cpu(trailer->db_free_rec_len)) + dx_root->dr_free_blk = cpu_to_le64(dirdata_bh->b_blocknr); + else + dx_root->dr_free_blk = cpu_to_le64(0); + + if (dx_inline) { + dx_root->dr_flags |= OCFS2_DX_FLAG_INLINE; + dx_root->dr_entries.de_count = + cpu_to_le16(ocfs2_dx_entries_per_root(osb->sb)); + } else { + dx_root->dr_list.l_count = + cpu_to_le16(ocfs2_extent_recs_per_dx_root(osb->sb)); + } + ocfs2_journal_dirty(handle, dx_root_bh); + + ret = ocfs2_journal_access_di(handle, INODE_CACHE(dir), di_bh, + OCFS2_JOURNAL_ACCESS_CREATE); + if (ret) { + mlog_errno(ret); + goto out; + } + + di->i_dx_root = cpu_to_le64(dr_blkno); + + spin_lock(&OCFS2_I(dir)->ip_lock); + OCFS2_I(dir)->ip_dyn_features |= OCFS2_INDEXED_DIR_FL; + di->i_dyn_features = cpu_to_le16(OCFS2_I(dir)->ip_dyn_features); + spin_unlock(&OCFS2_I(dir)->ip_lock); + + ocfs2_journal_dirty(handle, di_bh); + + *ret_dx_root_bh = dx_root_bh; + dx_root_bh = NULL; + +out: + brelse(dx_root_bh); + return ret; +} + +static int ocfs2_dx_dir_format_cluster(struct ocfs2_super *osb, + handle_t *handle, struct inode *dir, + struct buffer_head **dx_leaves, + int num_dx_leaves, u64 start_blk) +{ + int ret, i; + struct ocfs2_dx_leaf *dx_leaf; + struct buffer_head *bh; + + for (i = 0; i < num_dx_leaves; i++) { + bh = sb_getblk(osb->sb, start_blk + i); + if (bh == NULL) { + ret = -ENOMEM; + goto out; + } + dx_leaves[i] = bh; + + ocfs2_set_new_buffer_uptodate(INODE_CACHE(dir), bh); + + ret = ocfs2_journal_access_dl(handle, INODE_CACHE(dir), bh, + OCFS2_JOURNAL_ACCESS_CREATE); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + dx_leaf = (struct ocfs2_dx_leaf *) bh->b_data; + + memset(dx_leaf, 0, osb->sb->s_blocksize); + strscpy(dx_leaf->dl_signature, OCFS2_DX_LEAF_SIGNATURE); + dx_leaf->dl_fs_generation = cpu_to_le32(osb->fs_generation); + dx_leaf->dl_blkno = cpu_to_le64(bh->b_blocknr); + dx_leaf->dl_list.de_count = + cpu_to_le16(ocfs2_dx_entries_per_leaf(osb->sb)); + + trace_ocfs2_dx_dir_format_cluster( + (unsigned long long)OCFS2_I(dir)->ip_blkno, + (unsigned long long)bh->b_blocknr, + le16_to_cpu(dx_leaf->dl_list.de_count)); + + ocfs2_journal_dirty(handle, bh); + } + + ret = 0; +out: + return ret; +} + +/* + * Allocates and formats a new cluster for use in an indexed dir + * leaf. This version will not do the extent insert, so that it can be + * used by operations which need careful ordering. + */ +static int __ocfs2_dx_dir_new_cluster(struct inode *dir, + u32 cpos, handle_t *handle, + struct ocfs2_alloc_context *data_ac, + struct buffer_head **dx_leaves, + int num_dx_leaves, u64 *ret_phys_blkno) +{ + int ret; + u32 phys, num; + u64 phys_blkno; + struct ocfs2_super *osb = OCFS2_SB(dir->i_sb); + + /* + * XXX: For create, this should claim cluster for the index + * *before* the unindexed insert so that we have a better + * chance of contiguousness as the directory grows in number + * of entries. + */ + ret = __ocfs2_claim_clusters(handle, data_ac, 1, 1, &phys, &num); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * Format the new cluster first. That way, we're inserting + * valid data. + */ + phys_blkno = ocfs2_clusters_to_blocks(osb->sb, phys); + ret = ocfs2_dx_dir_format_cluster(osb, handle, dir, dx_leaves, + num_dx_leaves, phys_blkno); + if (ret) { + mlog_errno(ret); + goto out; + } + + *ret_phys_blkno = phys_blkno; +out: + return ret; +} + +static int ocfs2_dx_dir_new_cluster(struct inode *dir, + struct ocfs2_extent_tree *et, + u32 cpos, handle_t *handle, + struct ocfs2_alloc_context *data_ac, + struct ocfs2_alloc_context *meta_ac, + struct buffer_head **dx_leaves, + int num_dx_leaves) +{ + int ret; + u64 phys_blkno; + + ret = __ocfs2_dx_dir_new_cluster(dir, cpos, handle, data_ac, dx_leaves, + num_dx_leaves, &phys_blkno); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_insert_extent(handle, et, cpos, phys_blkno, 1, 0, + meta_ac); + if (ret) + mlog_errno(ret); +out: + return ret; +} + +static struct buffer_head **ocfs2_dx_dir_kmalloc_leaves(struct super_block *sb, + int *ret_num_leaves) +{ + int num_dx_leaves = ocfs2_clusters_to_blocks(sb, 1); + struct buffer_head **dx_leaves; + + dx_leaves = kzalloc_objs(struct buffer_head *, num_dx_leaves, GFP_NOFS); + if (dx_leaves && ret_num_leaves) + *ret_num_leaves = num_dx_leaves; + + return dx_leaves; +} + +static int ocfs2_fill_new_dir_dx(struct ocfs2_super *osb, + handle_t *handle, + struct inode *parent, + struct inode *inode, + struct buffer_head *di_bh, + struct ocfs2_alloc_context *data_ac, + struct ocfs2_alloc_context *meta_ac) +{ + int ret; + struct buffer_head *leaf_bh = NULL; + struct buffer_head *dx_root_bh = NULL; + struct ocfs2_dx_hinfo hinfo; + struct ocfs2_dx_root_block *dx_root; + struct ocfs2_dx_entry_list *entry_list; + + /* + * Our strategy is to create the directory as though it were + * unindexed, then add the index block. This works with very + * little complication since the state of a new directory is a + * very well known quantity. + * + * Essentially, we have two dirents ("." and ".."), in the 1st + * block which need indexing. These are easily inserted into + * the index block. + */ + + ret = ocfs2_fill_new_dir_el(osb, handle, parent, inode, di_bh, + data_ac, &leaf_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_dx_dir_attach_index(osb, handle, inode, di_bh, leaf_bh, + meta_ac, 1, 2, &dx_root_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + dx_root = (struct ocfs2_dx_root_block *)dx_root_bh->b_data; + entry_list = &dx_root->dr_entries; + + /* Buffer has been journaled for us by ocfs2_dx_dir_attach_index */ + ocfs2_dx_dir_name_hash(inode, ".", 1, &hinfo); + ocfs2_dx_entry_list_insert(entry_list, &hinfo, leaf_bh->b_blocknr); + + ocfs2_dx_dir_name_hash(inode, "..", 2, &hinfo); + ocfs2_dx_entry_list_insert(entry_list, &hinfo, leaf_bh->b_blocknr); + +out: + brelse(dx_root_bh); + brelse(leaf_bh); + return ret; +} + +int ocfs2_fill_new_dir(struct ocfs2_super *osb, + handle_t *handle, + struct inode *parent, + struct inode *inode, + struct buffer_head *fe_bh, + struct ocfs2_alloc_context *data_ac, + struct ocfs2_alloc_context *meta_ac) + +{ + BUG_ON(!ocfs2_supports_inline_data(osb) && data_ac == NULL); + + if (OCFS2_I(inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL) + return ocfs2_fill_new_dir_id(osb, handle, parent, inode, fe_bh); + + if (ocfs2_supports_indexed_dirs(osb)) + return ocfs2_fill_new_dir_dx(osb, handle, parent, inode, fe_bh, + data_ac, meta_ac); + + return ocfs2_fill_new_dir_el(osb, handle, parent, inode, fe_bh, + data_ac, NULL); +} + +static int ocfs2_dx_dir_index_block(struct inode *dir, + handle_t *handle, + struct buffer_head **dx_leaves, + int num_dx_leaves, + u32 *num_dx_entries, + struct buffer_head *dirent_bh) +{ + int ret = 0, namelen, i; + char *de_buf, *limit; + struct ocfs2_dir_entry *de; + struct buffer_head *dx_leaf_bh; + struct ocfs2_dx_hinfo hinfo; + u64 dirent_blk = dirent_bh->b_blocknr; + + de_buf = dirent_bh->b_data; + limit = de_buf + dir->i_sb->s_blocksize; + + while (de_buf < limit) { + de = (struct ocfs2_dir_entry *)de_buf; + + namelen = de->name_len; + if (!namelen || !de->inode) + goto inc; + + ocfs2_dx_dir_name_hash(dir, de->name, namelen, &hinfo); + + i = ocfs2_dx_dir_hash_idx(OCFS2_SB(dir->i_sb), &hinfo); + dx_leaf_bh = dx_leaves[i]; + + ret = __ocfs2_dx_dir_leaf_insert(dir, handle, &hinfo, + dirent_blk, dx_leaf_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + *num_dx_entries = *num_dx_entries + 1; + +inc: + de_buf += le16_to_cpu(de->rec_len); + } + +out: + return ret; +} + +/* + * XXX: This expects dx_root_bh to already be part of the transaction. + */ +static void ocfs2_dx_dir_index_root_block(struct inode *dir, + struct buffer_head *dx_root_bh, + struct buffer_head *dirent_bh) +{ + char *de_buf, *limit; + struct ocfs2_dx_root_block *dx_root; + struct ocfs2_dir_entry *de; + struct ocfs2_dx_hinfo hinfo; + u64 dirent_blk = dirent_bh->b_blocknr; + + dx_root = (struct ocfs2_dx_root_block *)dx_root_bh->b_data; + + de_buf = dirent_bh->b_data; + limit = de_buf + dir->i_sb->s_blocksize; + + while (de_buf < limit) { + de = (struct ocfs2_dir_entry *)de_buf; + + if (!de->name_len || !de->inode) + goto inc; + + ocfs2_dx_dir_name_hash(dir, de->name, de->name_len, &hinfo); + + trace_ocfs2_dx_dir_index_root_block( + (unsigned long long)dir->i_ino, + hinfo.major_hash, hinfo.minor_hash, + de->name_len, de->name, + le16_to_cpu(dx_root->dr_entries.de_num_used)); + + ocfs2_dx_entry_list_insert(&dx_root->dr_entries, &hinfo, + dirent_blk); + + le32_add_cpu(&dx_root->dr_num_entries, 1); +inc: + de_buf += le16_to_cpu(de->rec_len); + } +} + +/* + * Count the number of inline directory entries in di_bh and compare + * them against the number of entries we can hold in an inline dx root + * block. + */ +static int ocfs2_new_dx_should_be_inline(struct inode *dir, + struct buffer_head *di_bh) +{ + int dirent_count = 0; + char *de_buf, *limit; + struct ocfs2_dir_entry *de; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data; + + de_buf = di->id2.i_data.id_data; + limit = de_buf + i_size_read(dir); + + while (de_buf < limit) { + de = (struct ocfs2_dir_entry *)de_buf; + + if (de->name_len && de->inode) + dirent_count++; + + de_buf += le16_to_cpu(de->rec_len); + } + + /* We are careful to leave room for one extra record. */ + return dirent_count < ocfs2_dx_entries_per_root(dir->i_sb); +} + +/* + * Expand rec_len of the rightmost dirent in a directory block so that it + * contains the end of our valid space for dirents. We do this during + * expansion from an inline directory to one with extents. The first dir block + * in that case is taken from the inline data portion of the inode block. + * + * This will also return the largest amount of contiguous space for a dirent + * in the block. That value is *not* necessarily the last dirent, even after + * expansion. The directory indexing code wants this value for free space + * accounting. We do this here since we're already walking the entire dir + * block. + * + * We add the dir trailer if this filesystem wants it. + */ +static unsigned int ocfs2_expand_last_dirent(char *start, unsigned int old_size, + struct inode *dir) +{ + struct super_block *sb = dir->i_sb; + struct ocfs2_dir_entry *de; + struct ocfs2_dir_entry *prev_de; + char *de_buf, *limit; + unsigned int new_size = sb->s_blocksize; + unsigned int bytes, this_hole; + unsigned int largest_hole = 0; + + if (ocfs2_new_dir_wants_trailer(dir)) + new_size = ocfs2_dir_trailer_blk_off(sb); + + bytes = new_size - old_size; + + limit = start + old_size; + de_buf = start; + de = (struct ocfs2_dir_entry *)de_buf; + do { + this_hole = ocfs2_figure_dirent_hole(de); + if (this_hole > largest_hole) + largest_hole = this_hole; + + prev_de = de; + de_buf += le16_to_cpu(de->rec_len); + de = (struct ocfs2_dir_entry *)de_buf; + } while (de_buf < limit); + + le16_add_cpu(&prev_de->rec_len, bytes); + + /* We need to double check this after modification of the final + * dirent. */ + this_hole = ocfs2_figure_dirent_hole(prev_de); + if (this_hole > largest_hole) + largest_hole = this_hole; + + if (largest_hole >= OCFS2_DIR_MIN_REC_LEN) + return largest_hole; + return 0; +} + +/* + * We allocate enough clusters to fulfill "blocks_wanted", but set + * i_size to exactly one block. Ocfs2_extend_dir() will handle the + * rest automatically for us. + * + * *first_block_bh is a pointer to the 1st data block allocated to the + * directory. + */ +static int ocfs2_expand_inline_dir(struct inode *dir, struct buffer_head *di_bh, + unsigned int blocks_wanted, + struct ocfs2_dir_lookup_result *lookup, + struct buffer_head **first_block_bh) +{ + u32 alloc, dx_alloc, bit_off, len, num_dx_entries = 0; + struct super_block *sb = dir->i_sb; + int ret, i, num_dx_leaves = 0, dx_inline = 0, + credits = ocfs2_inline_to_extents_credits(sb); + u64 dx_insert_blkno, blkno, + bytes = blocks_wanted << sb->s_blocksize_bits; + struct ocfs2_super *osb = OCFS2_SB(dir->i_sb); + struct ocfs2_inode_info *oi = OCFS2_I(dir); + struct ocfs2_alloc_context *data_ac = NULL; + struct ocfs2_alloc_context *meta_ac = NULL; + struct buffer_head *dirdata_bh = NULL; + struct buffer_head *dx_root_bh = NULL; + struct buffer_head **dx_leaves = NULL; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data; + handle_t *handle; + struct ocfs2_extent_tree et; + struct ocfs2_extent_tree dx_et; + int did_quota = 0, bytes_allocated = 0; + + ocfs2_init_dinode_extent_tree(&et, INODE_CACHE(dir), di_bh); + + alloc = ocfs2_clusters_for_bytes(sb, bytes); + dx_alloc = 0; + + down_write(&oi->ip_alloc_sem); + + if (ocfs2_supports_indexed_dirs(osb)) { + credits += ocfs2_add_dir_index_credits(sb); + + dx_inline = ocfs2_new_dx_should_be_inline(dir, di_bh); + if (!dx_inline) { + /* Add one more cluster for an index leaf */ + dx_alloc++; + dx_leaves = ocfs2_dx_dir_kmalloc_leaves(sb, + &num_dx_leaves); + if (!dx_leaves) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + } + + /* This gets us the dx_root */ + ret = ocfs2_reserve_new_metadata_blocks(osb, 1, &meta_ac); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + /* + * We should never need more than 2 clusters for the unindexed + * tree - maximum dirent size is far less than one block. In + * fact, the only time we'd need more than one cluster is if + * blocksize == clustersize and the dirent won't fit in the + * extra space that the expansion to a single block gives. As + * of today, that only happens on 4k/4k file systems. + */ + BUG_ON(alloc > 2); + + ret = ocfs2_reserve_clusters(osb, alloc + dx_alloc, &data_ac); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * Prepare for worst case allocation scenario of two separate + * extents in the unindexed tree. + */ + if (alloc == 2) + credits += OCFS2_SUBALLOC_ALLOC; + + handle = ocfs2_start_trans(osb, credits); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out; + } + + ret = dquot_alloc_space_nodirty(dir, + ocfs2_clusters_to_bytes(osb->sb, alloc + dx_alloc)); + if (ret) + goto out_commit; + did_quota = 1; + + if (ocfs2_supports_indexed_dirs(osb) && !dx_inline) { + /* + * Allocate our index cluster first, to maximize the + * possibility that unindexed leaves grow + * contiguously. + */ + ret = __ocfs2_dx_dir_new_cluster(dir, 0, handle, data_ac, + dx_leaves, num_dx_leaves, + &dx_insert_blkno); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + bytes_allocated += ocfs2_clusters_to_bytes(dir->i_sb, 1); + } + + /* + * Try to claim as many clusters as the bitmap can give though + * if we only get one now, that's enough to continue. The rest + * will be claimed after the conversion to extents. + */ + if (ocfs2_dir_resv_allowed(osb)) + data_ac->ac_resv = &oi->ip_la_data_resv; + ret = ocfs2_claim_clusters(handle, data_ac, 1, &bit_off, &len); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + bytes_allocated += ocfs2_clusters_to_bytes(dir->i_sb, 1); + + /* + * Operations are carefully ordered so that we set up the new + * data block first. The conversion from inline data to + * extents follows. + */ + blkno = ocfs2_clusters_to_blocks(dir->i_sb, bit_off); + dirdata_bh = sb_getblk(sb, blkno); + if (!dirdata_bh) { + ret = -ENOMEM; + mlog_errno(ret); + goto out_commit; + } + + ocfs2_set_new_buffer_uptodate(INODE_CACHE(dir), dirdata_bh); + + ret = ocfs2_journal_access_db(handle, INODE_CACHE(dir), dirdata_bh, + OCFS2_JOURNAL_ACCESS_CREATE); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + memcpy(dirdata_bh->b_data, di->id2.i_data.id_data, i_size_read(dir)); + memset(dirdata_bh->b_data + i_size_read(dir), 0, + sb->s_blocksize - i_size_read(dir)); + i = ocfs2_expand_last_dirent(dirdata_bh->b_data, i_size_read(dir), dir); + if (ocfs2_new_dir_wants_trailer(dir)) { + /* + * Prepare the dir trailer up front. It will otherwise look + * like a valid dirent. Even if inserting the index fails + * (unlikely), then all we'll have done is given first dir + * block a small amount of fragmentation. + */ + ocfs2_init_dir_trailer(dir, dirdata_bh, i); + } + + ocfs2_update_inode_fsync_trans(handle, dir, 1); + ocfs2_journal_dirty(handle, dirdata_bh); + + if (ocfs2_supports_indexed_dirs(osb) && !dx_inline) { + /* + * Dx dirs with an external cluster need to do this up + * front. Inline dx root's get handled later, after + * we've allocated our root block. We get passed back + * a total number of items so that dr_num_entries can + * be correctly set once the dx_root has been + * allocated. + */ + ret = ocfs2_dx_dir_index_block(dir, handle, dx_leaves, + num_dx_leaves, &num_dx_entries, + dirdata_bh); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + } + + /* + * Set extent, i_size, etc on the directory. After this, the + * inode should contain the same exact dirents as before and + * be fully accessible from system calls. + * + * We let the later dirent insert modify c/mtime - to the user + * the data hasn't changed. + */ + ret = ocfs2_journal_access_di(handle, INODE_CACHE(dir), di_bh, + OCFS2_JOURNAL_ACCESS_CREATE); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + spin_lock(&oi->ip_lock); + oi->ip_dyn_features &= ~OCFS2_INLINE_DATA_FL; + di->i_dyn_features = cpu_to_le16(oi->ip_dyn_features); + spin_unlock(&oi->ip_lock); + + ocfs2_dinode_new_extent_list(dir, di); + + i_size_write(dir, sb->s_blocksize); + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + + di->i_size = cpu_to_le64(sb->s_blocksize); + di->i_ctime = di->i_mtime = cpu_to_le64(inode_get_ctime_sec(dir)); + di->i_ctime_nsec = di->i_mtime_nsec = cpu_to_le32(inode_get_ctime_nsec(dir)); + ocfs2_update_inode_fsync_trans(handle, dir, 1); + + /* + * This should never fail as our extent list is empty and all + * related blocks have been journaled already. + */ + ret = ocfs2_insert_extent(handle, &et, 0, blkno, len, + 0, NULL); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + /* + * Set i_blocks after the extent insert for the most up to + * date ip_clusters value. + */ + dir->i_blocks = ocfs2_inode_sector_count(dir); + + ocfs2_journal_dirty(handle, di_bh); + + if (ocfs2_supports_indexed_dirs(osb)) { + ret = ocfs2_dx_dir_attach_index(osb, handle, dir, di_bh, + dirdata_bh, meta_ac, dx_inline, + num_dx_entries, &dx_root_bh); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + if (dx_inline) { + ocfs2_dx_dir_index_root_block(dir, dx_root_bh, + dirdata_bh); + } else { + ocfs2_init_dx_root_extent_tree(&dx_et, + INODE_CACHE(dir), + dx_root_bh); + ret = ocfs2_insert_extent(handle, &dx_et, 0, + dx_insert_blkno, 1, 0, NULL); + if (ret) + mlog_errno(ret); + } + } + + /* + * We asked for two clusters, but only got one in the 1st + * pass. Claim the 2nd cluster as a separate extent. + */ + if (alloc > len) { + ret = ocfs2_claim_clusters(handle, data_ac, 1, &bit_off, + &len); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + blkno = ocfs2_clusters_to_blocks(dir->i_sb, bit_off); + + ret = ocfs2_insert_extent(handle, &et, 1, + blkno, len, 0, NULL); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + bytes_allocated += ocfs2_clusters_to_bytes(dir->i_sb, 1); + } + + *first_block_bh = dirdata_bh; + dirdata_bh = NULL; + if (ocfs2_supports_indexed_dirs(osb)) { + unsigned int off; + + if (!dx_inline) { + /* + * We need to return the correct block within the + * cluster which should hold our entry. + */ + off = ocfs2_dx_dir_hash_idx(osb, + &lookup->dl_hinfo); + get_bh(dx_leaves[off]); + lookup->dl_dx_leaf_bh = dx_leaves[off]; + } + lookup->dl_dx_root_bh = dx_root_bh; + dx_root_bh = NULL; + } + +out_commit: + if (ret < 0 && did_quota) + dquot_free_space_nodirty(dir, bytes_allocated); + + ocfs2_commit_trans(osb, handle); + +out: + up_write(&oi->ip_alloc_sem); + if (data_ac) + ocfs2_free_alloc_context(data_ac); + if (meta_ac) + ocfs2_free_alloc_context(meta_ac); + + if (dx_leaves) { + for (i = 0; i < num_dx_leaves; i++) + brelse(dx_leaves[i]); + kfree(dx_leaves); + } + + brelse(dirdata_bh); + brelse(dx_root_bh); + + return ret; +} + +/* returns a bh of the 1st new block in the allocation. */ +static int ocfs2_do_extend_dir(struct super_block *sb, + handle_t *handle, + struct inode *dir, + struct buffer_head *parent_fe_bh, + struct ocfs2_alloc_context *data_ac, + struct ocfs2_alloc_context *meta_ac, + struct buffer_head **new_bh) +{ + int status; + int extend, did_quota = 0; + u64 p_blkno, v_blkno; + + spin_lock(&OCFS2_I(dir)->ip_lock); + extend = (i_size_read(dir) == ocfs2_clusters_to_bytes(sb, OCFS2_I(dir)->ip_clusters)); + spin_unlock(&OCFS2_I(dir)->ip_lock); + + if (extend) { + u32 offset = OCFS2_I(dir)->ip_clusters; + + status = dquot_alloc_space_nodirty(dir, + ocfs2_clusters_to_bytes(sb, 1)); + if (status) + goto bail; + did_quota = 1; + + status = ocfs2_add_inode_data(OCFS2_SB(sb), dir, &offset, + 1, 0, parent_fe_bh, handle, + data_ac, meta_ac, NULL); + BUG_ON(status == -EAGAIN); + if (status < 0) { + mlog_errno(status); + goto bail; + } + } + + v_blkno = ocfs2_blocks_for_bytes(sb, i_size_read(dir)); + status = ocfs2_extent_map_get_blocks(dir, v_blkno, &p_blkno, NULL, NULL); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + *new_bh = sb_getblk(sb, p_blkno); + if (!*new_bh) { + status = -ENOMEM; + mlog_errno(status); + goto bail; + } + status = 0; +bail: + if (did_quota && status < 0) + dquot_free_space_nodirty(dir, ocfs2_clusters_to_bytes(sb, 1)); + return status; +} + +/* + * Assumes you already have a cluster lock on the directory. + * + * 'blocks_wanted' is only used if we have an inline directory which + * is to be turned into an extent based one. The size of the dirent to + * insert might be larger than the space gained by growing to just one + * block, so we may have to grow the inode by two blocks in that case. + * + * If the directory is already indexed, dx_root_bh must be provided. + */ +static int ocfs2_extend_dir(struct ocfs2_super *osb, + struct inode *dir, + struct buffer_head *parent_fe_bh, + unsigned int blocks_wanted, + struct ocfs2_dir_lookup_result *lookup, + struct buffer_head **new_de_bh) +{ + int status = 0; + int credits, num_free_extents, drop_alloc_sem = 0; + loff_t dir_i_size; + struct ocfs2_dinode *fe = (struct ocfs2_dinode *) parent_fe_bh->b_data; + struct ocfs2_extent_list *el = &fe->id2.i_list; + struct ocfs2_alloc_context *data_ac = NULL; + struct ocfs2_alloc_context *meta_ac = NULL; + handle_t *handle = NULL; + struct buffer_head *new_bh = NULL; + struct ocfs2_dir_entry * de; + struct super_block *sb = osb->sb; + struct ocfs2_extent_tree et; + struct buffer_head *dx_root_bh = lookup->dl_dx_root_bh; + + if (OCFS2_I(dir)->ip_dyn_features & OCFS2_INLINE_DATA_FL) { + /* + * This would be a code error as an inline directory should + * never have an index root. + */ + BUG_ON(dx_root_bh); + + status = ocfs2_expand_inline_dir(dir, parent_fe_bh, + blocks_wanted, lookup, + &new_bh); + if (status) { + mlog_errno(status); + goto bail; + } + + /* Expansion from inline to an indexed directory will + * have given us this. */ + dx_root_bh = lookup->dl_dx_root_bh; + + if (blocks_wanted == 1) { + /* + * If the new dirent will fit inside the space + * created by pushing out to one block, then + * we can complete the operation + * here. Otherwise we have to expand i_size + * and format the 2nd block below. + */ + BUG_ON(new_bh == NULL); + goto bail_bh; + } + + /* + * Get rid of 'new_bh' - we want to format the 2nd + * data block and return that instead. + */ + brelse(new_bh); + new_bh = NULL; + + down_write(&OCFS2_I(dir)->ip_alloc_sem); + drop_alloc_sem = 1; + dir_i_size = i_size_read(dir); + credits = OCFS2_SIMPLE_DIR_EXTEND_CREDITS; + goto do_extend; + } + + down_write(&OCFS2_I(dir)->ip_alloc_sem); + drop_alloc_sem = 1; + dir_i_size = i_size_read(dir); + trace_ocfs2_extend_dir((unsigned long long)OCFS2_I(dir)->ip_blkno, + dir_i_size); + + /* dir->i_size is always block aligned. */ + spin_lock(&OCFS2_I(dir)->ip_lock); + if (dir_i_size == ocfs2_clusters_to_bytes(sb, OCFS2_I(dir)->ip_clusters)) { + spin_unlock(&OCFS2_I(dir)->ip_lock); + ocfs2_init_dinode_extent_tree(&et, INODE_CACHE(dir), + parent_fe_bh); + num_free_extents = ocfs2_num_free_extents(&et); + if (num_free_extents < 0) { + status = num_free_extents; + mlog_errno(status); + goto bail; + } + + if (!num_free_extents) { + status = ocfs2_reserve_new_metadata(osb, el, &meta_ac); + if (status < 0) { + if (status != -ENOSPC) + mlog_errno(status); + goto bail; + } + } + + status = ocfs2_reserve_clusters(osb, 1, &data_ac); + if (status < 0) { + if (status != -ENOSPC) + mlog_errno(status); + goto bail; + } + + if (ocfs2_dir_resv_allowed(osb)) + data_ac->ac_resv = &OCFS2_I(dir)->ip_la_data_resv; + + credits = ocfs2_calc_extend_credits(sb, el); + } else { + spin_unlock(&OCFS2_I(dir)->ip_lock); + credits = OCFS2_SIMPLE_DIR_EXTEND_CREDITS; + } + +do_extend: + if (ocfs2_dir_indexed(dir)) + credits++; /* For attaching the new dirent block to the + * dx_root */ + + handle = ocfs2_start_trans(osb, credits); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + handle = NULL; + mlog_errno(status); + goto bail; + } + + status = ocfs2_do_extend_dir(osb->sb, handle, dir, parent_fe_bh, + data_ac, meta_ac, &new_bh); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + ocfs2_set_new_buffer_uptodate(INODE_CACHE(dir), new_bh); + + status = ocfs2_journal_access_db(handle, INODE_CACHE(dir), new_bh, + OCFS2_JOURNAL_ACCESS_CREATE); + if (status < 0) { + mlog_errno(status); + goto bail; + } + memset(new_bh->b_data, 0, sb->s_blocksize); + + de = (struct ocfs2_dir_entry *) new_bh->b_data; + de->inode = 0; + if (ocfs2_supports_dir_trailer(dir)) { + de->rec_len = cpu_to_le16(ocfs2_dir_trailer_blk_off(sb)); + + ocfs2_init_dir_trailer(dir, new_bh, le16_to_cpu(de->rec_len)); + + if (ocfs2_dir_indexed(dir)) { + status = ocfs2_dx_dir_link_trailer(dir, handle, + dx_root_bh, new_bh); + if (status) { + mlog_errno(status); + goto bail; + } + } + } else { + de->rec_len = cpu_to_le16(sb->s_blocksize); + } + ocfs2_update_inode_fsync_trans(handle, dir, 1); + ocfs2_journal_dirty(handle, new_bh); + + dir_i_size += dir->i_sb->s_blocksize; + i_size_write(dir, dir_i_size); + dir->i_blocks = ocfs2_inode_sector_count(dir); + status = ocfs2_mark_inode_dirty(handle, dir, parent_fe_bh); + if (status < 0) { + mlog_errno(status); + goto bail; + } + +bail_bh: + *new_de_bh = new_bh; + get_bh(*new_de_bh); +bail: + if (handle) + ocfs2_commit_trans(osb, handle); + if (drop_alloc_sem) + up_write(&OCFS2_I(dir)->ip_alloc_sem); + + if (data_ac) + ocfs2_free_alloc_context(data_ac); + if (meta_ac) + ocfs2_free_alloc_context(meta_ac); + + brelse(new_bh); + + return status; +} + +static int ocfs2_find_dir_space_id(struct inode *dir, struct buffer_head *di_bh, + const char *name, int namelen, + struct buffer_head **ret_de_bh, + unsigned int *blocks_wanted) +{ + int ret; + struct super_block *sb = dir->i_sb; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data; + struct ocfs2_dir_entry *de, *last_de = NULL; + char *first_de, *de_buf, *limit; + unsigned long offset = 0; + unsigned int rec_len, new_rec_len, free_space; + + /* + * This calculates how many free bytes we'd have in block zero, should + * this function force expansion to an extent tree. + */ + if (ocfs2_new_dir_wants_trailer(dir)) + free_space = ocfs2_dir_trailer_blk_off(sb) - i_size_read(dir); + else + free_space = dir->i_sb->s_blocksize - i_size_read(dir); + + first_de = di->id2.i_data.id_data; + de_buf = first_de; + limit = de_buf + i_size_read(dir); + rec_len = OCFS2_DIR_REC_LEN(namelen); + + while (de_buf < limit) { + de = (struct ocfs2_dir_entry *)de_buf; + + if (!ocfs2_check_dir_entry(dir, de, di_bh, first_de, + i_size_read(dir), offset)) { + ret = -ENOENT; + goto out; + } + if (ocfs2_match(namelen, name, de)) { + ret = -EEXIST; + goto out; + } + /* + * No need to check for a trailing dirent record here as + * they're not used for inline dirs. + */ + + if (ocfs2_dirent_would_fit(de, rec_len)) { + /* Ok, we found a spot. Return this bh and let + * the caller actually fill it in. */ + *ret_de_bh = di_bh; + get_bh(*ret_de_bh); + ret = 0; + goto out; + } + + last_de = de; + de_buf += le16_to_cpu(de->rec_len); + offset += le16_to_cpu(de->rec_len); + } + + if (!last_de) { + ret = ocfs2_error(sb, "Directory entry (#%llu: size=%lld) " + "is unexpectedly short", + (unsigned long long)OCFS2_I(dir)->ip_blkno, + i_size_read(dir)); + goto out; + } + + /* + * We're going to require expansion of the directory - figure + * out how many blocks we'll need so that a place for the + * dirent can be found. + */ + *blocks_wanted = 1; + new_rec_len = le16_to_cpu(last_de->rec_len) + free_space; + if (new_rec_len < (rec_len + OCFS2_DIR_REC_LEN(last_de->name_len))) + *blocks_wanted = 2; + + ret = -ENOSPC; +out: + return ret; +} + +static int ocfs2_find_dir_space_el(struct inode *dir, const char *name, + int namelen, struct buffer_head **ret_de_bh) +{ + unsigned long offset; + struct buffer_head *bh = NULL; + unsigned short rec_len; + struct ocfs2_dir_entry *de; + struct super_block *sb = dir->i_sb; + int status; + int blocksize = dir->i_sb->s_blocksize; + + status = ocfs2_read_dir_block(dir, 0, &bh, 0); + if (status) + goto bail; + + rec_len = OCFS2_DIR_REC_LEN(namelen); + offset = 0; + de = (struct ocfs2_dir_entry *) bh->b_data; + while (1) { + if ((char *)de >= sb->s_blocksize + bh->b_data) { + brelse(bh); + bh = NULL; + + if (i_size_read(dir) <= offset) { + /* + * Caller will have to expand this + * directory. + */ + status = -ENOSPC; + goto bail; + } + status = ocfs2_read_dir_block(dir, + offset >> sb->s_blocksize_bits, + &bh, 0); + if (status) + goto bail; + + /* move to next block */ + de = (struct ocfs2_dir_entry *) bh->b_data; + } + if (!ocfs2_check_dir_entry(dir, de, bh, bh->b_data, blocksize, + offset)) { + status = -ENOENT; + goto bail; + } + if (ocfs2_match(namelen, name, de)) { + status = -EEXIST; + goto bail; + } + + if (ocfs2_skip_dir_trailer(dir, de, offset % blocksize, + blocksize)) + goto next; + + if (ocfs2_dirent_would_fit(de, rec_len)) { + /* Ok, we found a spot. Return this bh and let + * the caller actually fill it in. */ + *ret_de_bh = bh; + get_bh(*ret_de_bh); + status = 0; + goto bail; + } +next: + offset += le16_to_cpu(de->rec_len); + de = (struct ocfs2_dir_entry *)((char *) de + le16_to_cpu(de->rec_len)); + } + +bail: + brelse(bh); + if (status) + mlog_errno(status); + + return status; +} + +static int dx_leaf_sort_cmp(const void *a, const void *b) +{ + const struct ocfs2_dx_entry *entry1 = a; + const struct ocfs2_dx_entry *entry2 = b; + u32 major_hash1 = le32_to_cpu(entry1->dx_major_hash); + u32 major_hash2 = le32_to_cpu(entry2->dx_major_hash); + u32 minor_hash1 = le32_to_cpu(entry1->dx_minor_hash); + u32 minor_hash2 = le32_to_cpu(entry2->dx_minor_hash); + + if (major_hash1 > major_hash2) + return 1; + if (major_hash1 < major_hash2) + return -1; + + /* + * It is not strictly necessary to sort by minor + */ + if (minor_hash1 > minor_hash2) + return 1; + if (minor_hash1 < minor_hash2) + return -1; + return 0; +} + +static int ocfs2_dx_leaf_same_major(struct ocfs2_dx_leaf *dx_leaf) +{ + struct ocfs2_dx_entry_list *dl_list = &dx_leaf->dl_list; + int i, num = le16_to_cpu(dl_list->de_num_used); + + for (i = 0; i < (num - 1); i++) { + if (le32_to_cpu(dl_list->de_entries[i].dx_major_hash) != + le32_to_cpu(dl_list->de_entries[i + 1].dx_major_hash)) + return 0; + } + + return 1; +} + +/* + * Find the optimal value to split this leaf on. This expects the leaf + * entries to be in sorted order. + * + * leaf_cpos is the cpos of the leaf we're splitting. insert_hash is + * the hash we want to insert. + * + * This function is only concerned with the major hash - that which + * determines which cluster an item belongs to. + */ +static int ocfs2_dx_dir_find_leaf_split(struct ocfs2_dx_leaf *dx_leaf, + u32 leaf_cpos, u32 insert_hash, + u32 *split_hash) +{ + struct ocfs2_dx_entry_list *dl_list = &dx_leaf->dl_list; + int i, num_used = le16_to_cpu(dl_list->de_num_used); + int allsame; + + /* + * There's a couple rare, but nasty corner cases we have to + * check for here. All of them involve a leaf where all value + * have the same hash, which is what we look for first. + * + * Most of the time, all of the above is false, and we simply + * pick the median value for a split. + */ + allsame = ocfs2_dx_leaf_same_major(dx_leaf); + if (allsame) { + u32 val = le32_to_cpu(dl_list->de_entries[0].dx_major_hash); + + if (val == insert_hash) { + /* + * No matter where we would choose to split, + * the new entry would want to occupy the same + * block as these. Since there's no space left + * in their existing block, we know there + * won't be space after the split. + */ + return -ENOSPC; + } + + if (val == leaf_cpos) { + /* + * Because val is the same as leaf_cpos (which + * is the smallest value this leaf can have), + * yet is not equal to insert_hash, then we + * know that insert_hash *must* be larger than + * val (and leaf_cpos). At least cpos+1 in value. + * + * We also know then, that there cannot be an + * adjacent extent (otherwise we'd be looking + * at it). Choosing this value gives us a + * chance to get some contiguousness. + */ + *split_hash = leaf_cpos + 1; + return 0; + } + + if (val > insert_hash) { + /* + * val can not be the same as insert hash, and + * also must be larger than leaf_cpos. Also, + * we know that there can't be a leaf between + * cpos and val, otherwise the entries with + * hash 'val' would be there. + */ + *split_hash = val; + return 0; + } + + *split_hash = insert_hash; + return 0; + } + + /* + * Since the records are sorted and the checks above + * guaranteed that not all records in this block are the same, + * we simple travel forward, from the median, and pick the 1st + * record whose value is larger than leaf_cpos. + */ + for (i = (num_used / 2); i < num_used; i++) + if (le32_to_cpu(dl_list->de_entries[i].dx_major_hash) > + leaf_cpos) + break; + + BUG_ON(i == num_used); /* Should be impossible */ + *split_hash = le32_to_cpu(dl_list->de_entries[i].dx_major_hash); + return 0; +} + +/* + * Transfer all entries in orig_dx_leaves whose major hash is equal to or + * larger than split_hash into new_dx_leaves. We use a temporary + * buffer (tmp_dx_leaf) to make the changes to the original leaf blocks. + * + * Since the block offset inside a leaf (cluster) is a constant mask + * of minor_hash, we can optimize - an item at block offset X within + * the original cluster, will be at offset X within the new cluster. + */ +static void ocfs2_dx_dir_transfer_leaf(struct inode *dir, u32 split_hash, + handle_t *handle, + struct ocfs2_dx_leaf *tmp_dx_leaf, + struct buffer_head **orig_dx_leaves, + struct buffer_head **new_dx_leaves, + int num_dx_leaves) +{ + int i, j, num_used; + u32 major_hash; + struct ocfs2_dx_leaf *orig_dx_leaf, *new_dx_leaf; + struct ocfs2_dx_entry_list *orig_list, *tmp_list; + struct ocfs2_dx_entry *dx_entry; + + tmp_list = &tmp_dx_leaf->dl_list; + + for (i = 0; i < num_dx_leaves; i++) { + orig_dx_leaf = (struct ocfs2_dx_leaf *) orig_dx_leaves[i]->b_data; + orig_list = &orig_dx_leaf->dl_list; + new_dx_leaf = (struct ocfs2_dx_leaf *) new_dx_leaves[i]->b_data; + + num_used = le16_to_cpu(orig_list->de_num_used); + + memcpy(tmp_dx_leaf, orig_dx_leaf, dir->i_sb->s_blocksize); + tmp_list->de_num_used = cpu_to_le16(0); + memset(&tmp_list->de_entries, 0, sizeof(*dx_entry)*num_used); + + for (j = 0; j < num_used; j++) { + dx_entry = &orig_list->de_entries[j]; + major_hash = le32_to_cpu(dx_entry->dx_major_hash); + if (major_hash >= split_hash) + ocfs2_dx_dir_leaf_insert_tail(new_dx_leaf, + dx_entry); + else + ocfs2_dx_dir_leaf_insert_tail(tmp_dx_leaf, + dx_entry); + } + memcpy(orig_dx_leaf, tmp_dx_leaf, dir->i_sb->s_blocksize); + + ocfs2_journal_dirty(handle, orig_dx_leaves[i]); + ocfs2_journal_dirty(handle, new_dx_leaves[i]); + } +} + +static int ocfs2_dx_dir_rebalance_credits(struct ocfs2_super *osb, + struct ocfs2_dx_root_block *dx_root) +{ + int credits = ocfs2_clusters_to_blocks(osb->sb, 3); + + credits += ocfs2_calc_extend_credits(osb->sb, &dx_root->dr_list); + credits += ocfs2_quota_trans_credits(osb->sb); + return credits; +} + +/* + * Find the median value in dx_leaf_bh and allocate a new leaf to move + * half our entries into. + */ +static int ocfs2_dx_dir_rebalance(struct ocfs2_super *osb, struct inode *dir, + struct buffer_head *dx_root_bh, + struct buffer_head *dx_leaf_bh, + struct ocfs2_dx_hinfo *hinfo, u32 leaf_cpos, + u64 leaf_blkno) +{ + struct ocfs2_dx_leaf *dx_leaf = (struct ocfs2_dx_leaf *)dx_leaf_bh->b_data; + int credits, ret, i, num_used, did_quota = 0; + u32 cpos, split_hash, insert_hash = hinfo->major_hash; + u64 orig_leaves_start; + int num_dx_leaves; + struct buffer_head **orig_dx_leaves = NULL; + struct buffer_head **new_dx_leaves = NULL; + struct ocfs2_alloc_context *data_ac = NULL, *meta_ac = NULL; + struct ocfs2_extent_tree et; + handle_t *handle = NULL; + struct ocfs2_dx_root_block *dx_root; + struct ocfs2_dx_leaf *tmp_dx_leaf = NULL; + + trace_ocfs2_dx_dir_rebalance((unsigned long long)OCFS2_I(dir)->ip_blkno, + (unsigned long long)leaf_blkno, + insert_hash); + + ocfs2_init_dx_root_extent_tree(&et, INODE_CACHE(dir), dx_root_bh); + + dx_root = (struct ocfs2_dx_root_block *)dx_root_bh->b_data; + /* + * XXX: This is a rather large limit. We should use a more + * realistic value. + */ + if (le32_to_cpu(dx_root->dr_clusters) == UINT_MAX) + return -ENOSPC; + + num_used = le16_to_cpu(dx_leaf->dl_list.de_num_used); + if (num_used < le16_to_cpu(dx_leaf->dl_list.de_count)) { + mlog(ML_ERROR, "DX Dir: %llu, Asked to rebalance empty leaf: " + "%llu, %d\n", (unsigned long long)OCFS2_I(dir)->ip_blkno, + (unsigned long long)leaf_blkno, num_used); + ret = -EIO; + goto out; + } + + orig_dx_leaves = ocfs2_dx_dir_kmalloc_leaves(osb->sb, &num_dx_leaves); + if (!orig_dx_leaves) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + new_dx_leaves = ocfs2_dx_dir_kmalloc_leaves(osb->sb, NULL); + if (!new_dx_leaves) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + ret = ocfs2_lock_allocators(dir, &et, 1, 0, &data_ac, &meta_ac); + if (ret) { + if (ret != -ENOSPC) + mlog_errno(ret); + goto out; + } + + credits = ocfs2_dx_dir_rebalance_credits(osb, dx_root); + handle = ocfs2_start_trans(osb, credits); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + handle = NULL; + mlog_errno(ret); + goto out; + } + + ret = dquot_alloc_space_nodirty(dir, + ocfs2_clusters_to_bytes(dir->i_sb, 1)); + if (ret) + goto out_commit; + did_quota = 1; + + ret = ocfs2_journal_access_dl(handle, INODE_CACHE(dir), dx_leaf_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + /* + * This block is changing anyway, so we can sort it in place. + */ + sort(dx_leaf->dl_list.de_entries, num_used, + sizeof(struct ocfs2_dx_entry), dx_leaf_sort_cmp, + NULL); + + ocfs2_journal_dirty(handle, dx_leaf_bh); + + ret = ocfs2_dx_dir_find_leaf_split(dx_leaf, leaf_cpos, insert_hash, + &split_hash); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + trace_ocfs2_dx_dir_rebalance_split(leaf_cpos, split_hash, insert_hash); + + /* + * We have to carefully order operations here. There are items + * which want to be in the new cluster before insert, but in + * order to put those items in the new cluster, we alter the + * old cluster. A failure to insert gets nasty. + * + * So, start by reserving writes to the old + * cluster. ocfs2_dx_dir_new_cluster will reserve writes on + * the new cluster for us, before inserting it. The insert + * won't happen if there's an error before that. Once the + * insert is done then, we can transfer from one leaf into the + * other without fear of hitting any error. + */ + + /* + * The leaf transfer wants some scratch space so that we don't + * wind up doing a bunch of expensive memmove(). + */ + tmp_dx_leaf = kmalloc(osb->sb->s_blocksize, GFP_NOFS); + if (!tmp_dx_leaf) { + ret = -ENOMEM; + mlog_errno(ret); + goto out_commit; + } + + orig_leaves_start = ocfs2_block_to_cluster_start(dir->i_sb, leaf_blkno); + ret = ocfs2_read_dx_leaves(dir, orig_leaves_start, num_dx_leaves, + orig_dx_leaves); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + cpos = split_hash; + ret = ocfs2_dx_dir_new_cluster(dir, &et, cpos, handle, + data_ac, meta_ac, new_dx_leaves, + num_dx_leaves); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + for (i = 0; i < num_dx_leaves; i++) { + ret = ocfs2_journal_access_dl(handle, INODE_CACHE(dir), + orig_dx_leaves[i], + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + ret = ocfs2_journal_access_dl(handle, INODE_CACHE(dir), + new_dx_leaves[i], + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + } + + ocfs2_dx_dir_transfer_leaf(dir, split_hash, handle, tmp_dx_leaf, + orig_dx_leaves, new_dx_leaves, num_dx_leaves); + +out_commit: + if (ret < 0 && did_quota) + dquot_free_space_nodirty(dir, + ocfs2_clusters_to_bytes(dir->i_sb, 1)); + + ocfs2_update_inode_fsync_trans(handle, dir, 1); + ocfs2_commit_trans(osb, handle); + +out: + if (orig_dx_leaves || new_dx_leaves) { + for (i = 0; i < num_dx_leaves; i++) { + if (orig_dx_leaves) + brelse(orig_dx_leaves[i]); + if (new_dx_leaves) + brelse(new_dx_leaves[i]); + } + kfree(orig_dx_leaves); + kfree(new_dx_leaves); + } + + if (meta_ac) + ocfs2_free_alloc_context(meta_ac); + if (data_ac) + ocfs2_free_alloc_context(data_ac); + + kfree(tmp_dx_leaf); + return ret; +} + +static int ocfs2_find_dir_space_dx(struct ocfs2_super *osb, struct inode *dir, + struct buffer_head *di_bh, + struct buffer_head *dx_root_bh, + const char *name, int namelen, + struct ocfs2_dir_lookup_result *lookup) +{ + int ret, rebalanced = 0; + struct ocfs2_dx_root_block *dx_root; + struct buffer_head *dx_leaf_bh = NULL; + struct ocfs2_dx_leaf *dx_leaf; + u64 blkno; + u32 leaf_cpos; + + dx_root = (struct ocfs2_dx_root_block *)dx_root_bh->b_data; + +restart_search: + ret = ocfs2_dx_dir_lookup(dir, &dx_root->dr_list, &lookup->dl_hinfo, + &leaf_cpos, &blkno); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_read_dx_leaf(dir, blkno, &dx_leaf_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + dx_leaf = (struct ocfs2_dx_leaf *)dx_leaf_bh->b_data; + + if (le16_to_cpu(dx_leaf->dl_list.de_num_used) >= + le16_to_cpu(dx_leaf->dl_list.de_count)) { + if (rebalanced) { + /* + * Rebalancing should have provided us with + * space in an appropriate leaf. + * + * XXX: Is this an abnormal condition then? + * Should we print a message here? + */ + ret = -ENOSPC; + goto out; + } + + ret = ocfs2_dx_dir_rebalance(osb, dir, dx_root_bh, dx_leaf_bh, + &lookup->dl_hinfo, leaf_cpos, + blkno); + if (ret) { + if (ret != -ENOSPC) + mlog_errno(ret); + goto out; + } + + /* + * Restart the lookup. The rebalance might have + * changed which block our item fits into. Mark our + * progress, so we only execute this once. + */ + brelse(dx_leaf_bh); + dx_leaf_bh = NULL; + rebalanced = 1; + goto restart_search; + } + + lookup->dl_dx_leaf_bh = dx_leaf_bh; + dx_leaf_bh = NULL; + +out: + brelse(dx_leaf_bh); + return ret; +} + +static int ocfs2_search_dx_free_list(struct inode *dir, + struct buffer_head *dx_root_bh, + int namelen, + struct ocfs2_dir_lookup_result *lookup) +{ + int ret = -ENOSPC; + struct buffer_head *leaf_bh = NULL, *prev_leaf_bh = NULL; + struct ocfs2_dir_block_trailer *db; + u64 next_block; + int rec_len = OCFS2_DIR_REC_LEN(namelen); + struct ocfs2_dx_root_block *dx_root; + + dx_root = (struct ocfs2_dx_root_block *)dx_root_bh->b_data; + next_block = le64_to_cpu(dx_root->dr_free_blk); + + while (next_block) { + brelse(prev_leaf_bh); + prev_leaf_bh = leaf_bh; + leaf_bh = NULL; + + ret = ocfs2_read_dir_block_direct(dir, next_block, &leaf_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + db = ocfs2_trailer_from_bh(leaf_bh, dir->i_sb); + if (rec_len <= le16_to_cpu(db->db_free_rec_len)) { + lookup->dl_leaf_bh = leaf_bh; + lookup->dl_prev_leaf_bh = prev_leaf_bh; + leaf_bh = NULL; + prev_leaf_bh = NULL; + break; + } + + next_block = le64_to_cpu(db->db_free_next); + } + + if (!next_block) + ret = -ENOSPC; + +out: + + brelse(leaf_bh); + brelse(prev_leaf_bh); + return ret; +} + +static int ocfs2_expand_inline_dx_root(struct inode *dir, + struct buffer_head *dx_root_bh) +{ + int ret, num_dx_leaves, i, j, did_quota = 0; + struct buffer_head **dx_leaves = NULL; + struct ocfs2_extent_tree et; + u64 insert_blkno; + struct ocfs2_alloc_context *data_ac = NULL; + struct ocfs2_super *osb = OCFS2_SB(dir->i_sb); + handle_t *handle = NULL; + struct ocfs2_dx_root_block *dx_root; + struct ocfs2_dx_entry_list *entry_list; + struct ocfs2_dx_entry *dx_entry; + struct ocfs2_dx_leaf *target_leaf; + + ret = ocfs2_reserve_clusters(osb, 1, &data_ac); + if (ret) { + mlog_errno(ret); + goto out; + } + + dx_leaves = ocfs2_dx_dir_kmalloc_leaves(osb->sb, &num_dx_leaves); + if (!dx_leaves) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + handle = ocfs2_start_trans(osb, ocfs2_calc_dxi_expand_credits(osb->sb)); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out; + } + + ret = dquot_alloc_space_nodirty(dir, + ocfs2_clusters_to_bytes(osb->sb, 1)); + if (ret) + goto out_commit; + did_quota = 1; + + /* + * We do this up front, before the allocation, so that a + * failure to add the dx_root_bh to the journal won't result + * us losing clusters. + */ + ret = ocfs2_journal_access_dr(handle, INODE_CACHE(dir), dx_root_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + ret = __ocfs2_dx_dir_new_cluster(dir, 0, handle, data_ac, dx_leaves, + num_dx_leaves, &insert_blkno); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + /* + * Transfer the entries from our dx_root into the appropriate + * block + */ + dx_root = (struct ocfs2_dx_root_block *) dx_root_bh->b_data; + entry_list = &dx_root->dr_entries; + + for (i = 0; i < le16_to_cpu(entry_list->de_num_used); i++) { + dx_entry = &entry_list->de_entries[i]; + + j = __ocfs2_dx_dir_hash_idx(osb, + le32_to_cpu(dx_entry->dx_minor_hash)); + target_leaf = (struct ocfs2_dx_leaf *)dx_leaves[j]->b_data; + + ocfs2_dx_dir_leaf_insert_tail(target_leaf, dx_entry); + + /* Each leaf has been passed to the journal already + * via __ocfs2_dx_dir_new_cluster() */ + } + + dx_root->dr_flags &= ~OCFS2_DX_FLAG_INLINE; + + dx_root->dr_list.l_tree_depth = 0; + dx_root->dr_list.l_count = + cpu_to_le16(ocfs2_extent_recs_per_dx_root(osb->sb)); + dx_root->dr_list.l_next_free_rec = 0; + memset(&dx_root->dr_list.l_recs, 0, + osb->sb->s_blocksize - + (offsetof(struct ocfs2_dx_root_block, dr_list) + + offsetof(struct ocfs2_extent_list, l_recs))); + + /* This should never fail considering we start with an empty + * dx_root. */ + ocfs2_init_dx_root_extent_tree(&et, INODE_CACHE(dir), dx_root_bh); + ret = ocfs2_insert_extent(handle, &et, 0, insert_blkno, 1, 0, NULL); + if (ret) + mlog_errno(ret); + did_quota = 0; + + ocfs2_update_inode_fsync_trans(handle, dir, 1); + ocfs2_journal_dirty(handle, dx_root_bh); + +out_commit: + if (ret < 0 && did_quota) + dquot_free_space_nodirty(dir, + ocfs2_clusters_to_bytes(dir->i_sb, 1)); + + ocfs2_commit_trans(osb, handle); + +out: + if (data_ac) + ocfs2_free_alloc_context(data_ac); + + if (dx_leaves) { + for (i = 0; i < num_dx_leaves; i++) + brelse(dx_leaves[i]); + kfree(dx_leaves); + } + return ret; +} + +static int ocfs2_inline_dx_has_space(struct buffer_head *dx_root_bh) +{ + struct ocfs2_dx_root_block *dx_root; + struct ocfs2_dx_entry_list *entry_list; + + dx_root = (struct ocfs2_dx_root_block *) dx_root_bh->b_data; + entry_list = &dx_root->dr_entries; + + if (le16_to_cpu(entry_list->de_num_used) >= + le16_to_cpu(entry_list->de_count)) + return -ENOSPC; + + return 0; +} + +static int ocfs2_prepare_dx_dir_for_insert(struct inode *dir, + struct buffer_head *di_bh, + const char *name, + int namelen, + struct ocfs2_dir_lookup_result *lookup) +{ + int ret, free_dx_root = 1; + struct ocfs2_super *osb = OCFS2_SB(dir->i_sb); + struct buffer_head *dx_root_bh = NULL; + struct buffer_head *leaf_bh = NULL; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data; + struct ocfs2_dx_root_block *dx_root; + + ret = ocfs2_read_dx_root(dir, di, &dx_root_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + dx_root = (struct ocfs2_dx_root_block *)dx_root_bh->b_data; + if (le32_to_cpu(dx_root->dr_num_entries) == OCFS2_DX_ENTRIES_MAX) { + ret = -ENOSPC; + mlog_errno(ret); + goto out; + } + + if (ocfs2_dx_root_inline(dx_root)) { + ret = ocfs2_inline_dx_has_space(dx_root_bh); + + if (ret == 0) + goto search_el; + + /* + * We ran out of room in the root block. Expand it to + * an extent, then allow ocfs2_find_dir_space_dx to do + * the rest. + */ + ret = ocfs2_expand_inline_dx_root(dir, dx_root_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + /* + * Insert preparation for an indexed directory is split into two + * steps. The call to find_dir_space_dx reserves room in the index for + * an additional item. If we run out of space there, it's a real error + * we can't continue on. + */ + ret = ocfs2_find_dir_space_dx(osb, dir, di_bh, dx_root_bh, name, + namelen, lookup); + if (ret) { + mlog_errno(ret); + goto out; + } + +search_el: + /* + * Next, we need to find space in the unindexed tree. This call + * searches using the free space linked list. If the unindexed tree + * lacks sufficient space, we'll expand it below. The expansion code + * is smart enough to add any new blocks to the free space list. + */ + ret = ocfs2_search_dx_free_list(dir, dx_root_bh, namelen, lookup); + if (ret && ret != -ENOSPC) { + mlog_errno(ret); + goto out; + } + + /* Do this up here - ocfs2_extend_dir might need the dx_root */ + lookup->dl_dx_root_bh = dx_root_bh; + free_dx_root = 0; + + if (ret == -ENOSPC) { + ret = ocfs2_extend_dir(osb, dir, di_bh, 1, lookup, &leaf_bh); + + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * We make the assumption here that new leaf blocks are added + * to the front of our free list. + */ + lookup->dl_prev_leaf_bh = NULL; + lookup->dl_leaf_bh = leaf_bh; + } + +out: + if (free_dx_root) + brelse(dx_root_bh); + return ret; +} + +/* + * Get a directory ready for insert. Any directory allocation required + * happens here. Success returns zero, and enough context in the dir + * lookup result that ocfs2_add_entry() will be able complete the task + * with minimal performance impact. + */ +int ocfs2_prepare_dir_for_insert(struct ocfs2_super *osb, + struct inode *dir, + struct buffer_head *parent_fe_bh, + const char *name, + int namelen, + struct ocfs2_dir_lookup_result *lookup) +{ + int ret; + unsigned int blocks_wanted = 1; + struct buffer_head *bh = NULL; + + trace_ocfs2_prepare_dir_for_insert( + (unsigned long long)OCFS2_I(dir)->ip_blkno, namelen); + + /* + * Do this up front to reduce confusion. + * + * The directory might start inline, then be turned into an + * indexed one, in which case we'd need to hash deep inside + * ocfs2_find_dir_space_id(). Since + * ocfs2_prepare_dx_dir_for_insert() also needs this hash + * done, there seems no point in spreading out the calls. We + * can optimize away the case where the file system doesn't + * support indexing. + */ + if (ocfs2_supports_indexed_dirs(osb)) + ocfs2_dx_dir_name_hash(dir, name, namelen, &lookup->dl_hinfo); + + if (ocfs2_dir_indexed(dir)) { + ret = ocfs2_prepare_dx_dir_for_insert(dir, parent_fe_bh, + name, namelen, lookup); + if (ret) + mlog_errno(ret); + goto out; + } + + if (OCFS2_I(dir)->ip_dyn_features & OCFS2_INLINE_DATA_FL) { + ret = ocfs2_find_dir_space_id(dir, parent_fe_bh, name, + namelen, &bh, &blocks_wanted); + } else + ret = ocfs2_find_dir_space_el(dir, name, namelen, &bh); + + if (ret && ret != -ENOSPC) { + mlog_errno(ret); + goto out; + } + + if (ret == -ENOSPC) { + /* + * We have to expand the directory to add this name. + */ + BUG_ON(bh); + + ret = ocfs2_extend_dir(osb, dir, parent_fe_bh, blocks_wanted, + lookup, &bh); + if (ret) { + if (ret != -ENOSPC) + mlog_errno(ret); + goto out; + } + + BUG_ON(!bh); + } + + lookup->dl_leaf_bh = bh; + bh = NULL; +out: + brelse(bh); + return ret; +} + +static int ocfs2_dx_dir_remove_index(struct inode *dir, + struct buffer_head *di_bh, + struct buffer_head *dx_root_bh) +{ + int ret; + struct ocfs2_super *osb = OCFS2_SB(dir->i_sb); + struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data; + struct ocfs2_dx_root_block *dx_root; + struct inode *dx_alloc_inode = NULL; + struct buffer_head *dx_alloc_bh = NULL; + handle_t *handle; + u64 blk; + u16 bit; + u64 bg_blkno; + + dx_root = (struct ocfs2_dx_root_block *) dx_root_bh->b_data; + + dx_alloc_inode = ocfs2_get_system_file_inode(osb, + EXTENT_ALLOC_SYSTEM_INODE, + le16_to_cpu(dx_root->dr_suballoc_slot)); + if (!dx_alloc_inode) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + inode_lock(dx_alloc_inode); + + ret = ocfs2_inode_lock(dx_alloc_inode, &dx_alloc_bh, 1); + if (ret) { + mlog_errno(ret); + goto out_mutex; + } + + handle = ocfs2_start_trans(osb, OCFS2_DX_ROOT_REMOVE_CREDITS); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out_unlock; + } + + ret = ocfs2_journal_access_di(handle, INODE_CACHE(dir), di_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + spin_lock(&OCFS2_I(dir)->ip_lock); + OCFS2_I(dir)->ip_dyn_features &= ~OCFS2_INDEXED_DIR_FL; + di->i_dyn_features = cpu_to_le16(OCFS2_I(dir)->ip_dyn_features); + spin_unlock(&OCFS2_I(dir)->ip_lock); + di->i_dx_root = cpu_to_le64(0ULL); + ocfs2_update_inode_fsync_trans(handle, dir, 1); + + ocfs2_journal_dirty(handle, di_bh); + + blk = le64_to_cpu(dx_root->dr_blkno); + bit = le16_to_cpu(dx_root->dr_suballoc_bit); + if (dx_root->dr_suballoc_loc) + bg_blkno = le64_to_cpu(dx_root->dr_suballoc_loc); + else + bg_blkno = ocfs2_which_suballoc_group(blk, bit); + ret = ocfs2_free_suballoc_bits(handle, dx_alloc_inode, dx_alloc_bh, + bit, bg_blkno, 1); + if (ret) + mlog_errno(ret); + +out_commit: + ocfs2_commit_trans(osb, handle); + +out_unlock: + ocfs2_inode_unlock(dx_alloc_inode, 1); + +out_mutex: + inode_unlock(dx_alloc_inode); + brelse(dx_alloc_bh); +out: + iput(dx_alloc_inode); + return ret; +} + +int ocfs2_dx_dir_truncate(struct inode *dir, struct buffer_head *di_bh) +{ + int ret; + unsigned int clen; + u32 major_hash = UINT_MAX, p_cpos, cpos; + u64 blkno; + struct ocfs2_super *osb = OCFS2_SB(dir->i_sb); + struct buffer_head *dx_root_bh = NULL; + struct ocfs2_dx_root_block *dx_root; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data; + struct ocfs2_cached_dealloc_ctxt dealloc; + struct ocfs2_extent_tree et; + + ocfs2_init_dealloc_ctxt(&dealloc); + + if (!ocfs2_dir_indexed(dir)) + return 0; + + ret = ocfs2_read_dx_root(dir, di, &dx_root_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + dx_root = (struct ocfs2_dx_root_block *)dx_root_bh->b_data; + + if (ocfs2_dx_root_inline(dx_root)) + goto remove_index; + + ocfs2_init_dx_root_extent_tree(&et, INODE_CACHE(dir), dx_root_bh); + + /* XXX: What if dr_clusters is too large? */ + while (le32_to_cpu(dx_root->dr_clusters)) { + ret = ocfs2_dx_dir_lookup_rec(dir, &dx_root->dr_list, + major_hash, &cpos, &blkno, &clen); + if (ret) { + mlog_errno(ret); + goto out; + } + + p_cpos = ocfs2_blocks_to_clusters(dir->i_sb, blkno); + + ret = ocfs2_remove_btree_range(dir, &et, cpos, p_cpos, clen, 0, + &dealloc, 0, false); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (cpos == 0) + break; + + major_hash = cpos - 1; + } + +remove_index: + ret = ocfs2_dx_dir_remove_index(dir, di_bh, dx_root_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + ocfs2_remove_from_cache(INODE_CACHE(dir), dx_root_bh); +out: + ocfs2_schedule_truncate_log_flush(osb, 1); + ocfs2_run_deallocs(osb, &dealloc); + + brelse(dx_root_bh); + return ret; +} diff --git a/fs/ocfs2/dir.h b/fs/ocfs2/dir.h new file mode 100644 index 000000000..4b9f5a12c --- /dev/null +++ b/fs/ocfs2/dir.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * dir.h + * + * Function prototypes + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#ifndef OCFS2_DIR_H +#define OCFS2_DIR_H + +struct ocfs2_dx_hinfo { + u32 major_hash; + u32 minor_hash; +}; + +struct ocfs2_dir_lookup_result { + struct buffer_head *dl_leaf_bh; /* Unindexed leaf + * block */ + struct ocfs2_dir_entry *dl_entry; /* Target dirent in + * unindexed leaf */ + + struct buffer_head *dl_dx_root_bh; /* Root of indexed + * tree */ + + struct buffer_head *dl_dx_leaf_bh; /* Indexed leaf block */ + struct ocfs2_dx_entry *dl_dx_entry; /* Target dx_entry in + * indexed leaf */ + struct ocfs2_dx_hinfo dl_hinfo; /* Name hash results */ + + struct buffer_head *dl_prev_leaf_bh;/* Previous entry in + * dir free space + * list. NULL if + * previous entry is + * dx root block. */ +}; + +void ocfs2_free_dir_lookup_result(struct ocfs2_dir_lookup_result *res); + +int ocfs2_find_entry(const char *name, int namelen, + struct inode *dir, + struct ocfs2_dir_lookup_result *lookup); +int ocfs2_delete_entry(handle_t *handle, + struct inode *dir, + struct ocfs2_dir_lookup_result *res); +int __ocfs2_add_entry(handle_t *handle, + struct inode *dir, + const char *name, int namelen, + struct inode *inode, u64 blkno, + struct buffer_head *parent_fe_bh, + struct ocfs2_dir_lookup_result *lookup); +static inline int ocfs2_add_entry(handle_t *handle, + struct dentry *dentry, + struct inode *inode, u64 blkno, + struct buffer_head *parent_fe_bh, + struct ocfs2_dir_lookup_result *lookup) +{ + return __ocfs2_add_entry(handle, d_inode(dentry->d_parent), + dentry->d_name.name, dentry->d_name.len, + inode, blkno, parent_fe_bh, lookup); +} +int ocfs2_update_entry(struct inode *dir, handle_t *handle, + struct ocfs2_dir_lookup_result *res, + struct inode *new_entry_inode); + +int ocfs2_check_dir_for_entry(struct inode *dir, + const char *name, + int namelen); +int ocfs2_empty_dir(struct inode *inode); + +int ocfs2_find_files_on_disk(const char *name, + int namelen, + u64 *blkno, + struct inode *inode, + struct ocfs2_dir_lookup_result *res); +int ocfs2_lookup_ino_from_name(struct inode *dir, const char *name, + int namelen, u64 *blkno); +int ocfs2_readdir(struct file *file, struct dir_context *ctx); +int ocfs2_dir_foreach(struct inode *inode, struct dir_context *ctx); +int ocfs2_prepare_dir_for_insert(struct ocfs2_super *osb, + struct inode *dir, + struct buffer_head *parent_fe_bh, + const char *name, + int namelen, + struct ocfs2_dir_lookup_result *lookup); +struct ocfs2_alloc_context; +int ocfs2_fill_new_dir(struct ocfs2_super *osb, + handle_t *handle, + struct inode *parent, + struct inode *inode, + struct buffer_head *fe_bh, + struct ocfs2_alloc_context *data_ac, + struct ocfs2_alloc_context *meta_ac); + +int ocfs2_dx_dir_truncate(struct inode *dir, struct buffer_head *di_bh); + +struct ocfs2_dir_block_trailer *ocfs2_dir_trailer_from_size(int blocksize, + void *data); +#endif /* OCFS2_DIR_H */ diff --git a/fs/ocfs2/dlm/Makefile b/fs/ocfs2/dlm/Makefile new file mode 100644 index 000000000..5e700b45d --- /dev/null +++ b/fs/ocfs2/dlm/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_OCFS2_FS_O2CB) += ocfs2_dlm.o + +ocfs2_dlm-objs := dlmdomain.o dlmdebug.o dlmthread.o dlmrecovery.o \ + dlmmaster.o dlmast.o dlmconvert.o dlmlock.o dlmunlock.o diff --git a/fs/ocfs2/dlm/dlmapi.h b/fs/ocfs2/dlm/dlmapi.h new file mode 100644 index 000000000..1969db8ff --- /dev/null +++ b/fs/ocfs2/dlm/dlmapi.h @@ -0,0 +1,201 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * dlmapi.h + * + * externally exported dlm interfaces + * + * Copyright (C) 2004 Oracle. All rights reserved. + */ + +#ifndef DLMAPI_H +#define DLMAPI_H + +struct dlm_lock; +struct dlm_ctxt; + +/* NOTE: changes made to this enum should be reflected in dlmdebug.c */ +enum dlm_status { + DLM_NORMAL = 0, /* 0: request in progress */ + DLM_GRANTED, /* 1: request granted */ + DLM_DENIED, /* 2: request denied */ + DLM_DENIED_NOLOCKS, /* 3: request denied, out of system resources */ + DLM_WORKING, /* 4: async request in progress */ + DLM_BLOCKED, /* 5: lock request blocked */ + DLM_BLOCKED_ORPHAN, /* 6: lock request blocked by a orphan lock*/ + DLM_DENIED_GRACE_PERIOD, /* 7: topological change in progress */ + DLM_SYSERR, /* 8: system error */ + DLM_NOSUPPORT, /* 9: unsupported */ + DLM_CANCELGRANT, /* 10: can't cancel convert: already granted */ + DLM_IVLOCKID, /* 11: bad lockid */ + DLM_SYNC, /* 12: synchronous request granted */ + DLM_BADTYPE, /* 13: bad resource type */ + DLM_BADRESOURCE, /* 14: bad resource handle */ + DLM_MAXHANDLES, /* 15: no more resource handles */ + DLM_NOCLINFO, /* 16: can't contact cluster manager */ + DLM_NOLOCKMGR, /* 17: can't contact lock manager */ + DLM_NOPURGED, /* 18: can't contact purge daemon */ + DLM_BADARGS, /* 19: bad api args */ + DLM_VOID, /* 20: no status */ + DLM_NOTQUEUED, /* 21: NOQUEUE was specified and request failed */ + DLM_IVBUFLEN, /* 22: invalid resource name length */ + DLM_CVTUNGRANT, /* 23: attempted to convert ungranted lock */ + DLM_BADPARAM, /* 24: invalid lock mode specified */ + DLM_VALNOTVALID, /* 25: value block has been invalidated */ + DLM_REJECTED, /* 26: request rejected, unrecognized client */ + DLM_ABORT, /* 27: blocked lock request cancelled */ + DLM_CANCEL, /* 28: conversion request cancelled */ + DLM_IVRESHANDLE, /* 29: invalid resource handle */ + DLM_DEADLOCK, /* 30: deadlock recovery refused this request */ + DLM_DENIED_NOASTS, /* 31: failed to allocate AST */ + DLM_FORWARD, /* 32: request must wait for primary's response */ + DLM_TIMEOUT, /* 33: timeout value for lock has expired */ + DLM_IVGROUPID, /* 34: invalid group specification */ + DLM_VERS_CONFLICT, /* 35: version conflicts prevent request handling */ + DLM_BAD_DEVICE_PATH, /* 36: Locks device does not exist or path wrong */ + DLM_NO_DEVICE_PERMISSION, /* 37: Client has insufficient pers for device */ + DLM_NO_CONTROL_DEVICE, /* 38: Cannot set options on opened device */ + + DLM_RECOVERING, /* 39: extension, allows caller to fail a lock + request if it is being recovered */ + DLM_MIGRATING, /* 40: extension, allows caller to fail a lock + request if it is being migrated */ + DLM_MAXSTATS, /* 41: upper limit for return code validation */ +}; + +/* for pretty-printing dlm_status error names */ +const char *dlm_errname(enum dlm_status err); + +/* Eventually the DLM will use standard errno values, but in the + * meantime this lets us track dlm errors as they bubble up. When we + * bring its error reporting into line with the rest of the stack, + * these can just be replaced with calls to mlog_errno. */ +#define dlm_error(st) do { \ + if ((st) != DLM_RECOVERING && \ + (st) != DLM_MIGRATING && \ + (st) != DLM_FORWARD) \ + mlog(ML_ERROR, "dlm status = %s\n", dlm_errname((st))); \ +} while (0) + +#define DLM_LKSB_UNUSED1 0x01 +#define DLM_LKSB_PUT_LVB 0x02 +#define DLM_LKSB_GET_LVB 0x04 +#define DLM_LKSB_UNUSED2 0x08 +#define DLM_LKSB_UNUSED3 0x10 +#define DLM_LKSB_UNUSED4 0x20 +#define DLM_LKSB_UNUSED5 0x40 +#define DLM_LKSB_UNUSED6 0x80 + +#define DLM_LVB_LEN 64 + +/* Callers are only allowed access to the lvb and status members of + * this struct. */ +struct dlm_lockstatus { + enum dlm_status status; + u32 flags; + struct dlm_lock *lockid; + char lvb[DLM_LVB_LEN]; +}; + +/* Valid lock modes. */ +#define LKM_IVMODE (-1) /* invalid mode */ +#define LKM_NLMODE 0 /* null lock */ +#define LKM_CRMODE 1 /* concurrent read unsupported */ +#define LKM_CWMODE 2 /* concurrent write unsupported */ +#define LKM_PRMODE 3 /* protected read */ +#define LKM_PWMODE 4 /* protected write unsupported */ +#define LKM_EXMODE 5 /* exclusive */ +#define LKM_MAXMODE 5 +#define LKM_MODEMASK 0xff + +/* Flags passed to dlmlock and dlmunlock: + * reserved: flags used by the "real" dlm + * only a few are supported by this dlm + * (U) = unsupported by ocfs2 dlm */ +#define LKM_ORPHAN 0x00000010 /* this lock is orphanable (U) */ +#define LKM_PARENTABLE 0x00000020 /* this lock was orphaned (U) */ +#define LKM_BLOCK 0x00000040 /* blocking lock request (U) */ +#define LKM_LOCAL 0x00000080 /* local lock request */ +#define LKM_VALBLK 0x00000100 /* lock value block request */ +#define LKM_NOQUEUE 0x00000200 /* non blocking request */ +#define LKM_CONVERT 0x00000400 /* conversion request */ +#define LKM_NODLCKWT 0x00000800 /* this lock won't deadlock (U) */ +#define LKM_UNLOCK 0x00001000 /* deallocate this lock */ +#define LKM_CANCEL 0x00002000 /* cancel conversion request */ +#define LKM_DEQALL 0x00004000 /* remove all locks held by proc (U) */ +#define LKM_INVVALBLK 0x00008000 /* invalidate lock value block */ +#define LKM_SYNCSTS 0x00010000 /* return synchronous status if poss (U) */ +#define LKM_TIMEOUT 0x00020000 /* lock request contains timeout (U) */ +#define LKM_SNGLDLCK 0x00040000 /* request can self-deadlock (U) */ +#define LKM_FINDLOCAL 0x00080000 /* find local lock request (U) */ +#define LKM_PROC_OWNED 0x00100000 /* owned by process, not group (U) */ +#define LKM_XID 0x00200000 /* use transaction id for deadlock (U) */ +#define LKM_XID_CONFLICT 0x00400000 /* do not allow lock inheritance (U) */ +#define LKM_FORCE 0x00800000 /* force unlock flag */ +#define LKM_REVVALBLK 0x01000000 /* temporary solution: re-validate + lock value block (U) */ +/* unused */ +#define LKM_UNUSED1 0x00000001 /* unused */ +#define LKM_UNUSED2 0x00000002 /* unused */ +#define LKM_UNUSED3 0x00000004 /* unused */ +#define LKM_UNUSED4 0x00000008 /* unused */ +#define LKM_UNUSED5 0x02000000 /* unused */ +#define LKM_UNUSED6 0x04000000 /* unused */ +#define LKM_UNUSED7 0x08000000 /* unused */ + +/* ocfs2 extensions: internal only + * should never be used by caller */ +#define LKM_MIGRATION 0x10000000 /* extension: lockres is to be migrated + to another node */ +#define LKM_PUT_LVB 0x20000000 /* extension: lvb is being passed + should be applied to lockres */ +#define LKM_GET_LVB 0x40000000 /* extension: lvb should be copied + from lockres when lock is granted */ +#define LKM_RECOVERY 0x80000000 /* extension: flag for recovery lock + used to avoid recovery rwsem */ + + +typedef void (dlm_astlockfunc_t)(void *); +typedef void (dlm_bastlockfunc_t)(void *, int); +typedef void (dlm_astunlockfunc_t)(void *, enum dlm_status); + +enum dlm_status dlmlock(struct dlm_ctxt *dlm, + int mode, + struct dlm_lockstatus *lksb, + int flags, + const char *name, + int namelen, + dlm_astlockfunc_t *ast, + void *data, + dlm_bastlockfunc_t *bast); + +enum dlm_status dlmunlock(struct dlm_ctxt *dlm, + struct dlm_lockstatus *lksb, + int flags, + dlm_astunlockfunc_t *unlockast, + void *data); + +struct dlm_protocol_version { + u8 pv_major; + u8 pv_minor; +}; +struct dlm_ctxt * dlm_register_domain(const char *domain, u32 key, + struct dlm_protocol_version *fs_proto); + +void dlm_unregister_domain(struct dlm_ctxt *dlm); + +void dlm_print_one_lock(struct dlm_lock *lockid); + +typedef void (dlm_eviction_func)(int, void *); +struct dlm_eviction_cb { + struct list_head ec_item; + dlm_eviction_func *ec_func; + void *ec_data; +}; +void dlm_setup_eviction_cb(struct dlm_eviction_cb *cb, + dlm_eviction_func *f, + void *data); +void dlm_register_eviction_cb(struct dlm_ctxt *dlm, + struct dlm_eviction_cb *cb); +void dlm_unregister_eviction_cb(struct dlm_eviction_cb *cb); + +#endif /* DLMAPI_H */ diff --git a/fs/ocfs2/dlm/dlmast.c b/fs/ocfs2/dlm/dlmast.c new file mode 100644 index 000000000..c681ba957 --- /dev/null +++ b/fs/ocfs2/dlm/dlmast.c @@ -0,0 +1,475 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * dlmast.c + * + * AST and BAST functionality for local and remote nodes + * + * Copyright (C) 2004 Oracle. All rights reserved. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#include "../cluster/heartbeat.h" +#include "../cluster/nodemanager.h" +#include "../cluster/tcp.h" + +#include "dlmapi.h" +#include "dlmcommon.h" + +#define MLOG_MASK_PREFIX ML_DLM +#include "../cluster/masklog.h" + +static void dlm_update_lvb(struct dlm_ctxt *dlm, struct dlm_lock_resource *res, + struct dlm_lock *lock); +static int dlm_should_cancel_bast(struct dlm_ctxt *dlm, struct dlm_lock *lock); + +/* Should be called as an ast gets queued to see if the new + * lock level will obsolete a pending bast. + * For example, if dlm_thread queued a bast for an EX lock that + * was blocking another EX, but before sending the bast the + * lock owner downconverted to NL, the bast is now obsolete. + * Only the ast should be sent. + * This is needed because the lock and convert paths can queue + * asts out-of-band (not waiting for dlm_thread) in order to + * allow for LKM_NOQUEUE to get immediate responses. */ +static int dlm_should_cancel_bast(struct dlm_ctxt *dlm, struct dlm_lock *lock) +{ + assert_spin_locked(&dlm->ast_lock); + assert_spin_locked(&lock->spinlock); + + if (lock->ml.highest_blocked == LKM_IVMODE) + return 0; + BUG_ON(lock->ml.highest_blocked == LKM_NLMODE); + + if (lock->bast_pending && + list_empty(&lock->bast_list)) + /* old bast already sent, ok */ + return 0; + + if (lock->ml.type == LKM_EXMODE) + /* EX blocks anything left, any bast still valid */ + return 0; + else if (lock->ml.type == LKM_NLMODE) + /* NL blocks nothing, no reason to send any bast, cancel it */ + return 1; + else if (lock->ml.highest_blocked != LKM_EXMODE) + /* PR only blocks EX */ + return 1; + + return 0; +} + +void __dlm_queue_ast(struct dlm_ctxt *dlm, struct dlm_lock *lock) +{ + struct dlm_lock_resource *res; + + BUG_ON(!dlm); + BUG_ON(!lock); + + res = lock->lockres; + + assert_spin_locked(&dlm->ast_lock); + + if (!list_empty(&lock->ast_list)) { + mlog(ML_ERROR, "%s: res %.*s, lock %u:%llu, " + "AST list not empty, pending %d, newlevel %d\n", + dlm->name, res->lockname.len, res->lockname.name, + dlm_get_lock_cookie_node(be64_to_cpu(lock->ml.cookie)), + dlm_get_lock_cookie_seq(be64_to_cpu(lock->ml.cookie)), + lock->ast_pending, lock->ml.type); + BUG(); + } + if (lock->ast_pending) + mlog(0, "%s: res %.*s, lock %u:%llu, AST getting flushed\n", + dlm->name, res->lockname.len, res->lockname.name, + dlm_get_lock_cookie_node(be64_to_cpu(lock->ml.cookie)), + dlm_get_lock_cookie_seq(be64_to_cpu(lock->ml.cookie))); + + /* putting lock on list, add a ref */ + dlm_lock_get(lock); + spin_lock(&lock->spinlock); + + /* check to see if this ast obsoletes the bast */ + if (dlm_should_cancel_bast(dlm, lock)) { + mlog(0, "%s: res %.*s, lock %u:%llu, Cancelling BAST\n", + dlm->name, res->lockname.len, res->lockname.name, + dlm_get_lock_cookie_node(be64_to_cpu(lock->ml.cookie)), + dlm_get_lock_cookie_seq(be64_to_cpu(lock->ml.cookie))); + lock->bast_pending = 0; + list_del_init(&lock->bast_list); + lock->ml.highest_blocked = LKM_IVMODE; + /* removing lock from list, remove a ref. guaranteed + * this won't be the last ref because of the get above, + * so res->spinlock will not be taken here */ + dlm_lock_put(lock); + /* free up the reserved bast that we are cancelling. + * guaranteed that this will not be the last reserved + * ast because *both* an ast and a bast were reserved + * to get to this point. the res->spinlock will not be + * taken here */ + dlm_lockres_release_ast(dlm, res); + } + list_add_tail(&lock->ast_list, &dlm->pending_asts); + lock->ast_pending = 1; + spin_unlock(&lock->spinlock); +} + +void dlm_queue_ast(struct dlm_ctxt *dlm, struct dlm_lock *lock) +{ + BUG_ON(!dlm); + BUG_ON(!lock); + + spin_lock(&dlm->ast_lock); + __dlm_queue_ast(dlm, lock); + spin_unlock(&dlm->ast_lock); +} + + +void __dlm_queue_bast(struct dlm_ctxt *dlm, struct dlm_lock *lock) +{ + struct dlm_lock_resource *res; + + BUG_ON(!dlm); + BUG_ON(!lock); + + assert_spin_locked(&dlm->ast_lock); + + res = lock->lockres; + + BUG_ON(!list_empty(&lock->bast_list)); + if (lock->bast_pending) + mlog(0, "%s: res %.*s, lock %u:%llu, BAST getting flushed\n", + dlm->name, res->lockname.len, res->lockname.name, + dlm_get_lock_cookie_node(be64_to_cpu(lock->ml.cookie)), + dlm_get_lock_cookie_seq(be64_to_cpu(lock->ml.cookie))); + + /* putting lock on list, add a ref */ + dlm_lock_get(lock); + spin_lock(&lock->spinlock); + list_add_tail(&lock->bast_list, &dlm->pending_basts); + lock->bast_pending = 1; + spin_unlock(&lock->spinlock); +} + +static void dlm_update_lvb(struct dlm_ctxt *dlm, struct dlm_lock_resource *res, + struct dlm_lock *lock) +{ + struct dlm_lockstatus *lksb = lock->lksb; + BUG_ON(!lksb); + + /* only updates if this node masters the lockres */ + spin_lock(&res->spinlock); + if (res->owner == dlm->node_num) { + /* check the lksb flags for the direction */ + if (lksb->flags & DLM_LKSB_GET_LVB) { + mlog(0, "getting lvb from lockres for %s node\n", + lock->ml.node == dlm->node_num ? "master" : + "remote"); + memcpy(lksb->lvb, res->lvb, DLM_LVB_LEN); + } + /* Do nothing for lvb put requests - they should be done in + * place when the lock is downconverted - otherwise we risk + * racing gets and puts which could result in old lvb data + * being propagated. We leave the put flag set and clear it + * here. In the future we might want to clear it at the time + * the put is actually done. + */ + } + spin_unlock(&res->spinlock); + + /* reset any lvb flags on the lksb */ + lksb->flags &= ~(DLM_LKSB_PUT_LVB|DLM_LKSB_GET_LVB); +} + +void dlm_do_local_ast(struct dlm_ctxt *dlm, struct dlm_lock_resource *res, + struct dlm_lock *lock) +{ + dlm_astlockfunc_t *fn; + + mlog(0, "%s: res %.*s, lock %u:%llu, Local AST\n", dlm->name, + res->lockname.len, res->lockname.name, + dlm_get_lock_cookie_node(be64_to_cpu(lock->ml.cookie)), + dlm_get_lock_cookie_seq(be64_to_cpu(lock->ml.cookie))); + + fn = lock->ast; + BUG_ON(lock->ml.node != dlm->node_num); + + dlm_update_lvb(dlm, res, lock); + (*fn)(lock->astdata); +} + + +int dlm_do_remote_ast(struct dlm_ctxt *dlm, struct dlm_lock_resource *res, + struct dlm_lock *lock) +{ + int ret; + struct dlm_lockstatus *lksb; + int lksbflags; + + mlog(0, "%s: res %.*s, lock %u:%llu, Remote AST\n", dlm->name, + res->lockname.len, res->lockname.name, + dlm_get_lock_cookie_node(be64_to_cpu(lock->ml.cookie)), + dlm_get_lock_cookie_seq(be64_to_cpu(lock->ml.cookie))); + + lksb = lock->lksb; + BUG_ON(lock->ml.node == dlm->node_num); + + lksbflags = lksb->flags; + dlm_update_lvb(dlm, res, lock); + + /* lock request came from another node + * go do the ast over there */ + ret = dlm_send_proxy_ast(dlm, res, lock, lksbflags); + return ret; +} + +void dlm_do_local_bast(struct dlm_ctxt *dlm, struct dlm_lock_resource *res, + struct dlm_lock *lock, int blocked_type) +{ + dlm_bastlockfunc_t *fn = lock->bast; + + BUG_ON(lock->ml.node != dlm->node_num); + + mlog(0, "%s: res %.*s, lock %u:%llu, Local BAST, blocked %d\n", + dlm->name, res->lockname.len, res->lockname.name, + dlm_get_lock_cookie_node(be64_to_cpu(lock->ml.cookie)), + dlm_get_lock_cookie_seq(be64_to_cpu(lock->ml.cookie)), + blocked_type); + + (*fn)(lock->astdata, blocked_type); +} + + + +int dlm_proxy_ast_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data) +{ + int ret; + unsigned int locklen; + struct dlm_ctxt *dlm = data; + struct dlm_lock_resource *res = NULL; + struct dlm_lock *lock = NULL; + struct dlm_proxy_ast *past = (struct dlm_proxy_ast *) msg->buf; + char *name; + struct list_head *head = NULL; + __be64 cookie; + u32 flags; + u8 node; + + if (!dlm_grab(dlm)) { + dlm_error(DLM_REJECTED); + return DLM_REJECTED; + } + + mlog_bug_on_msg(!dlm_domain_fully_joined(dlm), + "Domain %s not fully joined!\n", dlm->name); + + name = past->name; + locklen = past->namelen; + cookie = past->cookie; + flags = be32_to_cpu(past->flags); + node = past->node_idx; + + if (locklen > DLM_LOCKID_NAME_MAX) { + ret = DLM_IVBUFLEN; + mlog(ML_ERROR, "Invalid name length (%d) in proxy ast " + "handler!\n", locklen); + goto leave; + } + + if ((flags & (LKM_PUT_LVB|LKM_GET_LVB)) == + (LKM_PUT_LVB|LKM_GET_LVB)) { + mlog(ML_ERROR, "Both PUT and GET lvb specified, (0x%x)\n", + flags); + ret = DLM_BADARGS; + goto leave; + } + + mlog(0, "lvb: %s\n", flags & LKM_PUT_LVB ? "put lvb" : + (flags & LKM_GET_LVB ? "get lvb" : "none")); + + mlog(0, "type=%d, blocked_type=%d\n", past->type, past->blocked_type); + + if (past->type != DLM_AST && + past->type != DLM_BAST) { + mlog(ML_ERROR, "Unknown ast type! %d, cookie=%u:%llu" + "name=%.*s, node=%u\n", past->type, + dlm_get_lock_cookie_node(be64_to_cpu(cookie)), + dlm_get_lock_cookie_seq(be64_to_cpu(cookie)), + locklen, name, node); + ret = DLM_IVLOCKID; + goto leave; + } + + res = dlm_lookup_lockres(dlm, name, locklen); + if (!res) { + mlog(0, "Got %sast for unknown lockres! cookie=%u:%llu, " + "name=%.*s, node=%u\n", (past->type == DLM_AST ? "" : "b"), + dlm_get_lock_cookie_node(be64_to_cpu(cookie)), + dlm_get_lock_cookie_seq(be64_to_cpu(cookie)), + locklen, name, node); + ret = DLM_IVLOCKID; + goto leave; + } + + /* cannot get a proxy ast message if this node owns it */ + BUG_ON(res->owner == dlm->node_num); + + mlog(0, "%s: res %.*s\n", dlm->name, res->lockname.len, + res->lockname.name); + + spin_lock(&res->spinlock); + if (res->state & DLM_LOCK_RES_RECOVERING) { + mlog(0, "Responding with DLM_RECOVERING!\n"); + ret = DLM_RECOVERING; + goto unlock_out; + } + if (res->state & DLM_LOCK_RES_MIGRATING) { + mlog(0, "Responding with DLM_MIGRATING!\n"); + ret = DLM_MIGRATING; + goto unlock_out; + } + /* try convert queue for both ast/bast */ + head = &res->converting; + lock = NULL; + list_for_each_entry(lock, head, list) { + if (lock->ml.cookie == cookie) + goto do_ast; + } + + /* if not on convert, try blocked for ast, granted for bast */ + if (past->type == DLM_AST) + head = &res->blocked; + else + head = &res->granted; + + list_for_each_entry(lock, head, list) { + /* if lock is found but unlock is pending ignore the bast */ + if (lock->ml.cookie == cookie) { + if (lock->unlock_pending) + break; + goto do_ast; + } + } + + mlog(0, "Got %sast for unknown lock! cookie=%u:%llu, name=%.*s, " + "node=%u\n", past->type == DLM_AST ? "" : "b", + dlm_get_lock_cookie_node(be64_to_cpu(cookie)), + dlm_get_lock_cookie_seq(be64_to_cpu(cookie)), + locklen, name, node); + + ret = DLM_NORMAL; +unlock_out: + spin_unlock(&res->spinlock); + goto leave; + +do_ast: + ret = DLM_NORMAL; + if (past->type == DLM_AST) { + /* do not alter lock refcount. switching lists. */ + list_move_tail(&lock->list, &res->granted); + mlog(0, "%s: res %.*s, lock %u:%llu, Granted type %d => %d\n", + dlm->name, res->lockname.len, res->lockname.name, + dlm_get_lock_cookie_node(be64_to_cpu(cookie)), + dlm_get_lock_cookie_seq(be64_to_cpu(cookie)), + lock->ml.type, lock->ml.convert_type); + + if (lock->ml.convert_type != LKM_IVMODE) { + lock->ml.type = lock->ml.convert_type; + lock->ml.convert_type = LKM_IVMODE; + } else { + // should already be there.... + } + + lock->lksb->status = DLM_NORMAL; + + /* if we requested the lvb, fetch it into our lksb now */ + if (flags & LKM_GET_LVB) { + BUG_ON(!(lock->lksb->flags & DLM_LKSB_GET_LVB)); + memcpy(lock->lksb->lvb, past->lvb, DLM_LVB_LEN); + } + } + spin_unlock(&res->spinlock); + + if (past->type == DLM_AST) + dlm_do_local_ast(dlm, res, lock); + else + dlm_do_local_bast(dlm, res, lock, past->blocked_type); + +leave: + if (res) + dlm_lockres_put(res); + + dlm_put(dlm); + return ret; +} + + + +int dlm_send_proxy_ast_msg(struct dlm_ctxt *dlm, struct dlm_lock_resource *res, + struct dlm_lock *lock, int msg_type, + int blocked_type, int flags) +{ + int ret = 0; + struct dlm_proxy_ast past; + struct kvec vec[2]; + size_t veclen = 1; + int status; + + mlog(0, "%s: res %.*s, to %u, type %d, blocked_type %d\n", dlm->name, + res->lockname.len, res->lockname.name, lock->ml.node, msg_type, + blocked_type); + + memset(&past, 0, sizeof(struct dlm_proxy_ast)); + past.node_idx = dlm->node_num; + past.type = msg_type; + past.blocked_type = blocked_type; + past.namelen = res->lockname.len; + memcpy(past.name, res->lockname.name, past.namelen); + past.cookie = lock->ml.cookie; + + vec[0].iov_len = sizeof(struct dlm_proxy_ast); + vec[0].iov_base = &past; + if (flags & DLM_LKSB_GET_LVB) { + be32_add_cpu(&past.flags, LKM_GET_LVB); + vec[1].iov_len = DLM_LVB_LEN; + vec[1].iov_base = lock->lksb->lvb; + veclen++; + } + + ret = o2net_send_message_vec(DLM_PROXY_AST_MSG, dlm->key, vec, veclen, + lock->ml.node, &status); + if (ret < 0) + mlog(ML_ERROR, "%s: res %.*s, error %d send AST to node %u\n", + dlm->name, res->lockname.len, res->lockname.name, ret, + lock->ml.node); + else { + if (status == DLM_RECOVERING) { + mlog(ML_ERROR, "sent AST to node %u, it thinks this " + "node is dead!\n", lock->ml.node); + BUG(); + } else if (status == DLM_MIGRATING) { + mlog(ML_ERROR, "sent AST to node %u, it returned " + "DLM_MIGRATING!\n", lock->ml.node); + BUG(); + } else if (status != DLM_NORMAL && status != DLM_IVLOCKID) { + mlog(ML_ERROR, "AST to node %u returned %d!\n", + lock->ml.node, status); + /* ignore it */ + } + ret = 0; + } + return ret; +} diff --git a/fs/ocfs2/dlm/dlmcommon.h b/fs/ocfs2/dlm/dlmcommon.h new file mode 100644 index 000000000..20f790a47 --- /dev/null +++ b/fs/ocfs2/dlm/dlmcommon.h @@ -0,0 +1,1132 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * dlmcommon.h + * + * Copyright (C) 2004 Oracle. All rights reserved. + */ + +#ifndef DLMCOMMON_H +#define DLMCOMMON_H + +#include + +#define DLM_HB_NODE_DOWN_PRI (0xf000000) +#define DLM_HB_NODE_UP_PRI (0x8000000) + +#define DLM_LOCKID_NAME_MAX 32 + +#define DLM_LOCK_RES_OWNER_UNKNOWN O2NM_MAX_NODES + +#define DLM_HASH_SIZE_DEFAULT (1 << 17) +#if DLM_HASH_SIZE_DEFAULT < PAGE_SIZE +# define DLM_HASH_PAGES 1 +#else +# define DLM_HASH_PAGES (DLM_HASH_SIZE_DEFAULT / PAGE_SIZE) +#endif +#define DLM_BUCKETS_PER_PAGE (PAGE_SIZE / sizeof(struct hlist_head)) +#define DLM_HASH_BUCKETS (DLM_HASH_PAGES * DLM_BUCKETS_PER_PAGE) + +/* Intended to make it easier for us to switch out hash functions */ +#define dlm_lockid_hash(_n, _l) full_name_hash(NULL, _n, _l) + +enum dlm_mle_type { + DLM_MLE_BLOCK = 0, + DLM_MLE_MASTER = 1, + DLM_MLE_MIGRATION = 2, + DLM_MLE_NUM_TYPES = 3, +}; + +struct dlm_master_list_entry { + struct hlist_node master_hash_node; + struct list_head hb_events; + struct dlm_ctxt *dlm; + spinlock_t spinlock; + wait_queue_head_t wq; + atomic_t woken; + struct kref mle_refs; + int inuse; + unsigned long maybe_map[BITS_TO_LONGS(O2NM_MAX_NODES)]; + unsigned long vote_map[BITS_TO_LONGS(O2NM_MAX_NODES)]; + unsigned long response_map[BITS_TO_LONGS(O2NM_MAX_NODES)]; + unsigned long node_map[BITS_TO_LONGS(O2NM_MAX_NODES)]; + u8 master; + u8 new_master; + enum dlm_mle_type type; + struct o2hb_callback_func mle_hb_up; + struct o2hb_callback_func mle_hb_down; + struct dlm_lock_resource *mleres; + unsigned char mname[DLM_LOCKID_NAME_MAX]; + unsigned int mnamelen; + unsigned int mnamehash; +}; + +enum dlm_ast_type { + DLM_AST = 0, + DLM_BAST = 1, + DLM_ASTUNLOCK = 2, +}; + + +#define LKM_VALID_FLAGS (LKM_VALBLK | LKM_CONVERT | LKM_UNLOCK | \ + LKM_CANCEL | LKM_INVVALBLK | LKM_FORCE | \ + LKM_RECOVERY | LKM_LOCAL | LKM_NOQUEUE) + +#define DLM_RECOVERY_LOCK_NAME "$RECOVERY" +#define DLM_RECOVERY_LOCK_NAME_LEN 9 + +static inline int dlm_is_recovery_lock(const char *lock_name, int name_len) +{ + if (name_len == DLM_RECOVERY_LOCK_NAME_LEN && + memcmp(lock_name, DLM_RECOVERY_LOCK_NAME, name_len)==0) + return 1; + return 0; +} + +#define DLM_RECO_STATE_ACTIVE 0x0001 +#define DLM_RECO_STATE_FINALIZE 0x0002 + +struct dlm_recovery_ctxt +{ + struct list_head resources; + struct list_head node_data; + u8 new_master; + u8 dead_node; + u16 state; + unsigned long node_map[BITS_TO_LONGS(O2NM_MAX_NODES)]; + wait_queue_head_t event; +}; + +enum dlm_ctxt_state { + DLM_CTXT_NEW = 0, + DLM_CTXT_JOINED = 1, + DLM_CTXT_IN_SHUTDOWN = 2, + DLM_CTXT_LEAVING = 3, +}; + +struct dlm_ctxt +{ + struct list_head list; + struct hlist_head **lockres_hash; + struct list_head dirty_list; + struct list_head purge_list; + struct list_head pending_asts; + struct list_head pending_basts; + struct list_head tracking_list; + unsigned int purge_count; + spinlock_t spinlock; + spinlock_t ast_lock; + spinlock_t track_lock; + char *name; + u8 node_num; + u32 key; + u8 joining_node; + u8 migrate_done; /* set to 1 means node has migrated all lock resources */ + wait_queue_head_t dlm_join_events; + unsigned long live_nodes_map[BITS_TO_LONGS(O2NM_MAX_NODES)]; + unsigned long domain_map[BITS_TO_LONGS(O2NM_MAX_NODES)]; + unsigned long exit_domain_map[BITS_TO_LONGS(O2NM_MAX_NODES)]; + unsigned long recovery_map[BITS_TO_LONGS(O2NM_MAX_NODES)]; + struct dlm_recovery_ctxt reco; + spinlock_t master_lock; + struct hlist_head **master_hash; + struct list_head mle_hb_events; + + /* these give a really vague idea of the system load */ + atomic_t mle_tot_count[DLM_MLE_NUM_TYPES]; + atomic_t mle_cur_count[DLM_MLE_NUM_TYPES]; + atomic_t res_tot_count; + atomic_t res_cur_count; + + struct dentry *dlm_debugfs_subroot; + + /* NOTE: Next three are protected by dlm_domain_lock */ + struct kref dlm_refs; + enum dlm_ctxt_state dlm_state; + unsigned int num_joins; + + struct o2hb_callback_func dlm_hb_up; + struct o2hb_callback_func dlm_hb_down; + struct task_struct *dlm_thread_task; + struct task_struct *dlm_reco_thread_task; + struct workqueue_struct *dlm_worker; + wait_queue_head_t dlm_thread_wq; + wait_queue_head_t dlm_reco_thread_wq; + wait_queue_head_t ast_wq; + wait_queue_head_t migration_wq; + + struct work_struct dispatched_work; + struct list_head work_list; + spinlock_t work_lock; + struct list_head dlm_domain_handlers; + struct list_head dlm_eviction_callbacks; + + /* The filesystem specifies this at domain registration. We + * cache it here to know what to tell other nodes. */ + struct dlm_protocol_version fs_locking_proto; + /* This is the inter-dlm communication version */ + struct dlm_protocol_version dlm_locking_proto; +}; + +static inline struct hlist_head *dlm_lockres_hash(struct dlm_ctxt *dlm, unsigned i) +{ + return dlm->lockres_hash[(i / DLM_BUCKETS_PER_PAGE) % DLM_HASH_PAGES] + (i % DLM_BUCKETS_PER_PAGE); +} + +static inline struct hlist_head *dlm_master_hash(struct dlm_ctxt *dlm, + unsigned i) +{ + return dlm->master_hash[(i / DLM_BUCKETS_PER_PAGE) % DLM_HASH_PAGES] + + (i % DLM_BUCKETS_PER_PAGE); +} + +/* these keventd work queue items are for less-frequently + * called functions that cannot be directly called from the + * net message handlers for some reason, usually because + * they need to send net messages of their own. */ +void dlm_dispatch_work(struct work_struct *work); + +struct dlm_lock_resource; +struct dlm_work_item; + +typedef void (dlm_workfunc_t)(struct dlm_work_item *, void *); + +struct dlm_request_all_locks_priv +{ + u8 reco_master; + u8 dead_node; +}; + +struct dlm_mig_lockres_priv +{ + struct dlm_lock_resource *lockres; + u8 real_master; + u8 extra_ref; +}; + +struct dlm_assert_master_priv +{ + struct dlm_lock_resource *lockres; + u8 request_from; + u32 flags; + unsigned ignore_higher:1; +}; + +struct dlm_deref_lockres_priv +{ + struct dlm_lock_resource *deref_res; + u8 deref_node; +}; + +struct dlm_work_item +{ + struct list_head list; + dlm_workfunc_t *func; + struct dlm_ctxt *dlm; + void *data; + union { + struct dlm_request_all_locks_priv ral; + struct dlm_mig_lockres_priv ml; + struct dlm_assert_master_priv am; + struct dlm_deref_lockres_priv dl; + } u; +}; + +static inline void dlm_init_work_item(struct dlm_ctxt *dlm, + struct dlm_work_item *i, + dlm_workfunc_t *f, void *data) +{ + memset(i, 0, sizeof(*i)); + i->func = f; + INIT_LIST_HEAD(&i->list); + i->data = data; + i->dlm = dlm; /* must have already done a dlm_grab on this! */ +} + + + +static inline void __dlm_set_joining_node(struct dlm_ctxt *dlm, + u8 node) +{ + assert_spin_locked(&dlm->spinlock); + + dlm->joining_node = node; + wake_up(&dlm->dlm_join_events); +} + +#define DLM_LOCK_RES_UNINITED 0x00000001 +#define DLM_LOCK_RES_RECOVERING 0x00000002 +#define DLM_LOCK_RES_READY 0x00000004 +#define DLM_LOCK_RES_DIRTY 0x00000008 +#define DLM_LOCK_RES_IN_PROGRESS 0x00000010 +#define DLM_LOCK_RES_MIGRATING 0x00000020 +#define DLM_LOCK_RES_DROPPING_REF 0x00000040 +#define DLM_LOCK_RES_BLOCK_DIRTY 0x00001000 +#define DLM_LOCK_RES_SETREF_INPROG 0x00002000 +#define DLM_LOCK_RES_RECOVERY_WAITING 0x00004000 + +/* max milliseconds to wait to sync up a network failure with a node death */ +#define DLM_NODE_DEATH_WAIT_MAX (5 * 1000) + +#define DLM_PURGE_INTERVAL_MS (8 * 1000) + +struct dlm_lock_resource +{ + /* WARNING: Please see the comment in dlm_init_lockres before + * adding fields here. */ + struct hlist_node hash_node; + struct qstr lockname; + struct kref refs; + + /* + * Please keep granted, converting, and blocked in this order, + * as some funcs want to iterate over all lists. + * + * All four lists are protected by the hash's reference. + */ + struct list_head granted; + struct list_head converting; + struct list_head blocked; + struct list_head purge; + + /* + * These two lists require you to hold an additional reference + * while they are on the list. + */ + struct list_head dirty; + struct list_head recovering; // dlm_recovery_ctxt.resources list + + /* Added during init and removed during release */ + struct list_head tracking; /* dlm->tracking_list */ + + /* unused lock resources have their last_used stamped and are + * put on a list for the dlm thread to run. */ + unsigned long last_used; + + struct dlm_ctxt *dlm; + + unsigned migration_pending:1; + atomic_t asts_reserved; + spinlock_t spinlock; + wait_queue_head_t wq; + u8 owner; //node which owns the lock resource, or unknown + u16 state; + char lvb[DLM_LVB_LEN]; + unsigned int inflight_locks; + unsigned int inflight_assert_workers; + unsigned long refmap[BITS_TO_LONGS(O2NM_MAX_NODES)]; +}; + +struct dlm_migratable_lock +{ + __be64 cookie; + + /* these 3 are just padding for the in-memory structure, but + * list and flags are actually used when sent over the wire */ + __be16 pad1; + u8 list; // 0=granted, 1=converting, 2=blocked + u8 flags; + + s8 type; + s8 convert_type; + s8 highest_blocked; + u8 node; +}; // 16 bytes + +struct dlm_lock +{ + struct dlm_migratable_lock ml; + + struct list_head list; + struct list_head ast_list; + struct list_head bast_list; + struct dlm_lock_resource *lockres; + spinlock_t spinlock; + struct kref lock_refs; + + // ast and bast must be callable while holding a spinlock! + dlm_astlockfunc_t *ast; + dlm_bastlockfunc_t *bast; + void *astdata; + struct dlm_lockstatus *lksb; + unsigned ast_pending:1, + bast_pending:1, + convert_pending:1, + lock_pending:1, + cancel_pending:1, + unlock_pending:1, + lksb_kernel_allocated:1; +}; + +enum dlm_lockres_list { + DLM_GRANTED_LIST = 0, + DLM_CONVERTING_LIST = 1, + DLM_BLOCKED_LIST = 2, +}; + +static inline int dlm_lvb_is_empty(char *lvb) +{ + int i; + for (i=0; igranted; + else if (idx == DLM_CONVERTING_LIST) + ret = &res->converting; + else if (idx == DLM_BLOCKED_LIST) + ret = &res->blocked; + else + BUG(); + return ret; +} + + + + +struct dlm_node_iter +{ + unsigned long node_map[BITS_TO_LONGS(O2NM_MAX_NODES)]; + int curnode; +}; + + +enum { + DLM_MASTER_REQUEST_MSG = 500, + DLM_UNUSED_MSG1 = 501, + DLM_ASSERT_MASTER_MSG = 502, + DLM_CREATE_LOCK_MSG = 503, + DLM_CONVERT_LOCK_MSG = 504, + DLM_PROXY_AST_MSG = 505, + DLM_UNLOCK_LOCK_MSG = 506, + DLM_DEREF_LOCKRES_MSG = 507, + DLM_MIGRATE_REQUEST_MSG = 508, + DLM_MIG_LOCKRES_MSG = 509, + DLM_QUERY_JOIN_MSG = 510, + DLM_ASSERT_JOINED_MSG = 511, + DLM_CANCEL_JOIN_MSG = 512, + DLM_EXIT_DOMAIN_MSG = 513, + DLM_MASTER_REQUERY_MSG = 514, + DLM_LOCK_REQUEST_MSG = 515, + DLM_RECO_DATA_DONE_MSG = 516, + DLM_BEGIN_RECO_MSG = 517, + DLM_FINALIZE_RECO_MSG = 518, + DLM_QUERY_REGION = 519, + DLM_QUERY_NODEINFO = 520, + DLM_BEGIN_EXIT_DOMAIN_MSG = 521, + DLM_DEREF_LOCKRES_DONE = 522, +}; + +struct dlm_reco_node_data +{ + int state; + u8 node_num; + struct list_head list; +}; + +enum { + DLM_RECO_NODE_DATA_DEAD = -1, + DLM_RECO_NODE_DATA_INIT = 0, + DLM_RECO_NODE_DATA_REQUESTING = 1, + DLM_RECO_NODE_DATA_REQUESTED = 2, + DLM_RECO_NODE_DATA_RECEIVING = 3, + DLM_RECO_NODE_DATA_DONE = 4, + DLM_RECO_NODE_DATA_FINALIZE_SENT = 5, +}; + + +enum { + DLM_MASTER_RESP_NO = 0, + DLM_MASTER_RESP_YES = 1, + DLM_MASTER_RESP_MAYBE = 2, + DLM_MASTER_RESP_ERROR = 3, +}; + + +struct dlm_master_request +{ + u8 node_idx; + u8 namelen; + __be16 pad1; + __be32 flags; + + u8 name[O2NM_MAX_NAME_LEN]; +}; + +#define DLM_ASSERT_RESPONSE_REASSERT 0x00000001 +#define DLM_ASSERT_RESPONSE_MASTERY_REF 0x00000002 + +#define DLM_ASSERT_MASTER_MLE_CLEANUP 0x00000001 +#define DLM_ASSERT_MASTER_REQUERY 0x00000002 +#define DLM_ASSERT_MASTER_FINISH_MIGRATION 0x00000004 +struct dlm_assert_master +{ + u8 node_idx; + u8 namelen; + __be16 pad1; + __be32 flags; + + u8 name[O2NM_MAX_NAME_LEN]; +}; + +#define DLM_MIGRATE_RESPONSE_MASTERY_REF 0x00000001 + +struct dlm_migrate_request +{ + u8 master; + u8 new_master; + u8 namelen; + u8 pad1; + __be32 pad2; + u8 name[O2NM_MAX_NAME_LEN]; +}; + +struct dlm_master_requery +{ + u8 pad1; + u8 pad2; + u8 node_idx; + u8 namelen; + __be32 pad3; + u8 name[O2NM_MAX_NAME_LEN]; +}; + +#define DLM_MRES_RECOVERY 0x01 +#define DLM_MRES_MIGRATION 0x02 +#define DLM_MRES_ALL_DONE 0x04 + +/* + * We would like to get one whole lockres into a single network + * message whenever possible. Generally speaking, there will be + * at most one dlm_lock on a lockres for each node in the cluster, + * plus (infrequently) any additional locks coming in from userdlm. + * + * struct _dlm_lockres_page + * { + * dlm_migratable_lockres mres; + * dlm_migratable_lock ml[DLM_MAX_MIGRATABLE_LOCKS]; + * u8 pad[DLM_MIG_LOCKRES_RESERVED]; + * }; + * + * from ../cluster/tcp.h + * O2NET_MAX_PAYLOAD_BYTES (4096 - sizeof(net_msg)) + * (roughly 4080 bytes) + * and sizeof(dlm_migratable_lockres) = 112 bytes + * and sizeof(dlm_migratable_lock) = 16 bytes + * + * Choosing DLM_MAX_MIGRATABLE_LOCKS=240 and + * DLM_MIG_LOCKRES_RESERVED=128 means we have this: + * + * (DLM_MAX_MIGRATABLE_LOCKS * sizeof(dlm_migratable_lock)) + + * sizeof(dlm_migratable_lockres) + DLM_MIG_LOCKRES_RESERVED = + * NET_MAX_PAYLOAD_BYTES + * (240 * 16) + 112 + 128 = 4080 + * + * So a lockres would need more than 240 locks before it would + * use more than one network packet to recover. Not too bad. + */ +#define DLM_MAX_MIGRATABLE_LOCKS 240 + +struct dlm_migratable_lockres +{ + u8 master; + u8 lockname_len; + u8 num_locks; // locks sent in this structure + u8 flags; + __be32 total_locks; // locks to be sent for this migration cookie + __be64 mig_cookie; // cookie for this lockres migration + // or zero if not needed + // 16 bytes + u8 lockname[DLM_LOCKID_NAME_MAX]; + // 48 bytes + u8 lvb[DLM_LVB_LEN]; + // 112 bytes + struct dlm_migratable_lock ml[]; // 16 bytes each, begins at byte 112 +}; +#define DLM_MIG_LOCKRES_MAX_LEN \ + (sizeof(struct dlm_migratable_lockres) + \ + (sizeof(struct dlm_migratable_lock) * \ + DLM_MAX_MIGRATABLE_LOCKS) ) + +/* from above, 128 bytes + * for some undetermined future use */ +#define DLM_MIG_LOCKRES_RESERVED (O2NET_MAX_PAYLOAD_BYTES - \ + DLM_MIG_LOCKRES_MAX_LEN) + +struct dlm_create_lock +{ + __be64 cookie; + + __be32 flags; + u8 pad1; + u8 node_idx; + s8 requested_type; + u8 namelen; + + u8 name[O2NM_MAX_NAME_LEN]; +}; + +struct dlm_convert_lock +{ + __be64 cookie; + + __be32 flags; + u8 pad1; + u8 node_idx; + s8 requested_type; + u8 namelen; + + u8 name[O2NM_MAX_NAME_LEN]; + + s8 lvb[]; +}; +#define DLM_CONVERT_LOCK_MAX_LEN (sizeof(struct dlm_convert_lock)+DLM_LVB_LEN) + +struct dlm_unlock_lock +{ + __be64 cookie; + + __be32 flags; + __be16 pad1; + u8 node_idx; + u8 namelen; + + u8 name[O2NM_MAX_NAME_LEN]; + + s8 lvb[]; +}; +#define DLM_UNLOCK_LOCK_MAX_LEN (sizeof(struct dlm_unlock_lock)+DLM_LVB_LEN) + +struct dlm_proxy_ast +{ + __be64 cookie; + + __be32 flags; + u8 node_idx; + u8 type; + u8 blocked_type; + u8 namelen; + + u8 name[O2NM_MAX_NAME_LEN]; + + s8 lvb[]; +}; +#define DLM_PROXY_AST_MAX_LEN (sizeof(struct dlm_proxy_ast)+DLM_LVB_LEN) + +#define DLM_MOD_KEY (0x666c6172) +enum dlm_query_join_response_code { + JOIN_DISALLOW = 0, + JOIN_OK = 1, + JOIN_OK_NO_MAP = 2, + JOIN_PROTOCOL_MISMATCH = 3, +}; + +struct dlm_query_join_packet { + u8 code; /* Response code. dlm_minor and fs_minor + are only valid if this is JOIN_OK */ + u8 dlm_minor; /* The minor version of the protocol the + dlm is speaking. */ + u8 fs_minor; /* The minor version of the protocol the + filesystem is speaking. */ + u8 reserved; +}; + +union dlm_query_join_response { + __be32 intval; + struct dlm_query_join_packet packet; +}; + +struct dlm_lock_request +{ + u8 node_idx; + u8 dead_node; + __be16 pad1; + __be32 pad2; +}; + +struct dlm_reco_data_done +{ + u8 node_idx; + u8 dead_node; + __be16 pad1; + __be32 pad2; + + /* unused for now */ + /* eventually we can use this to attempt + * lvb recovery based on each node's info */ + u8 reco_lvb[DLM_LVB_LEN]; +}; + +struct dlm_begin_reco +{ + u8 node_idx; + u8 dead_node; + __be16 pad1; + __be32 pad2; +}; + +struct dlm_query_join_request +{ + u8 node_idx; + u8 pad1[2]; + u8 name_len; + struct dlm_protocol_version dlm_proto; + struct dlm_protocol_version fs_proto; + u8 domain[O2NM_MAX_NAME_LEN]; + u8 node_map[BITS_TO_BYTES(O2NM_MAX_NODES)]; +}; + +struct dlm_assert_joined +{ + u8 node_idx; + u8 pad1[2]; + u8 name_len; + u8 domain[O2NM_MAX_NAME_LEN]; +}; + +struct dlm_cancel_join +{ + u8 node_idx; + u8 pad1[2]; + u8 name_len; + u8 domain[O2NM_MAX_NAME_LEN]; +}; + +struct dlm_query_region { + u8 qr_node; + u8 qr_numregions; + u8 qr_namelen; + u8 pad1; + u8 qr_domain[O2NM_MAX_NAME_LEN]; + u8 qr_regions[O2HB_MAX_REGION_NAME_LEN * O2NM_MAX_REGIONS]; +}; + +struct dlm_node_info { + u8 ni_nodenum; + u8 pad1; + __be16 ni_ipv4_port; + __be32 ni_ipv4_address; +}; + +struct dlm_query_nodeinfo { + u8 qn_nodenum; + u8 qn_numnodes; + u8 qn_namelen; + u8 pad1; + u8 qn_domain[O2NM_MAX_NAME_LEN]; + struct dlm_node_info qn_nodes[O2NM_MAX_NODES]; +}; + +struct dlm_exit_domain +{ + u8 node_idx; + u8 pad1[3]; +}; + +struct dlm_finalize_reco +{ + u8 node_idx; + u8 dead_node; + u8 flags; + u8 pad1; + __be32 pad2; +}; + +struct dlm_deref_lockres +{ + u32 pad1; + u16 pad2; + u8 node_idx; + u8 namelen; + + u8 name[O2NM_MAX_NAME_LEN]; +}; + +enum { + DLM_DEREF_RESPONSE_DONE = 0, + DLM_DEREF_RESPONSE_INPROG = 1, +}; + +struct dlm_deref_lockres_done { + u32 pad1; + u16 pad2; + u8 node_idx; + u8 namelen; + + u8 name[O2NM_MAX_NAME_LEN]; +}; + +static inline enum dlm_status +__dlm_lockres_state_to_status(struct dlm_lock_resource *res) +{ + enum dlm_status status = DLM_NORMAL; + + assert_spin_locked(&res->spinlock); + + if (res->state & (DLM_LOCK_RES_RECOVERING| + DLM_LOCK_RES_RECOVERY_WAITING)) + status = DLM_RECOVERING; + else if (res->state & DLM_LOCK_RES_MIGRATING) + status = DLM_MIGRATING; + else if (res->state & DLM_LOCK_RES_IN_PROGRESS) + status = DLM_FORWARD; + + return status; +} + +static inline u8 dlm_get_lock_cookie_node(u64 cookie) +{ + u8 ret; + cookie >>= 56; + ret = (u8)(cookie & 0xffULL); + return ret; +} + +static inline unsigned long long dlm_get_lock_cookie_seq(u64 cookie) +{ + unsigned long long ret; + ret = ((unsigned long long)cookie) & 0x00ffffffffffffffULL; + return ret; +} + +struct dlm_lock * dlm_new_lock(int type, u8 node, u64 cookie, + struct dlm_lockstatus *lksb); +void dlm_lock_get(struct dlm_lock *lock); +void dlm_lock_put(struct dlm_lock *lock); + +void dlm_lock_attach_lockres(struct dlm_lock *lock, + struct dlm_lock_resource *res); + +int dlm_create_lock_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data); +int dlm_convert_lock_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data); +int dlm_proxy_ast_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data); + +void dlm_revert_pending_convert(struct dlm_lock_resource *res, + struct dlm_lock *lock); +void dlm_revert_pending_lock(struct dlm_lock_resource *res, + struct dlm_lock *lock); + +int dlm_unlock_lock_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data); +void dlm_commit_pending_cancel(struct dlm_lock_resource *res, + struct dlm_lock *lock); +void dlm_commit_pending_unlock(struct dlm_lock_resource *res, + struct dlm_lock *lock); + +int dlm_launch_thread(struct dlm_ctxt *dlm); +void dlm_complete_thread(struct dlm_ctxt *dlm); +int dlm_launch_recovery_thread(struct dlm_ctxt *dlm); +void dlm_complete_recovery_thread(struct dlm_ctxt *dlm); +void dlm_wait_for_recovery(struct dlm_ctxt *dlm); +void dlm_kick_recovery_thread(struct dlm_ctxt *dlm); +int dlm_is_node_dead(struct dlm_ctxt *dlm, u8 node); +void dlm_wait_for_node_death(struct dlm_ctxt *dlm, u8 node, int timeout); +void dlm_wait_for_node_recovery(struct dlm_ctxt *dlm, u8 node, int timeout); + +void dlm_put(struct dlm_ctxt *dlm); +struct dlm_ctxt *dlm_grab(struct dlm_ctxt *dlm); +int dlm_domain_fully_joined(struct dlm_ctxt *dlm); + +void __dlm_lockres_calc_usage(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res); +void dlm_lockres_calc_usage(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res); +static inline void dlm_lockres_get(struct dlm_lock_resource *res) +{ + /* This is called on every lookup, so it might be worth + * inlining. */ + kref_get(&res->refs); +} +void dlm_lockres_put(struct dlm_lock_resource *res); +void __dlm_unhash_lockres(struct dlm_ctxt *dlm, struct dlm_lock_resource *res); +void __dlm_insert_lockres(struct dlm_ctxt *dlm, struct dlm_lock_resource *res); +struct dlm_lock_resource * __dlm_lookup_lockres_full(struct dlm_ctxt *dlm, + const char *name, + unsigned int len, + unsigned int hash); +struct dlm_lock_resource * __dlm_lookup_lockres(struct dlm_ctxt *dlm, + const char *name, + unsigned int len, + unsigned int hash); +struct dlm_lock_resource * dlm_lookup_lockres(struct dlm_ctxt *dlm, + const char *name, + unsigned int len); + +int dlm_is_host_down(int errno); + +struct dlm_lock_resource * dlm_get_lock_resource(struct dlm_ctxt *dlm, + const char *lockid, + int namelen, + int flags); +struct dlm_lock_resource *dlm_new_lockres(struct dlm_ctxt *dlm, + const char *name, + unsigned int namelen); + +void dlm_lockres_set_refmap_bit(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, int bit); +void dlm_lockres_clear_refmap_bit(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, int bit); + +void dlm_lockres_drop_inflight_ref(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res); +void dlm_lockres_grab_inflight_ref(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res); + +void __dlm_lockres_grab_inflight_worker(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res); + +void dlm_queue_ast(struct dlm_ctxt *dlm, struct dlm_lock *lock); +void __dlm_queue_ast(struct dlm_ctxt *dlm, struct dlm_lock *lock); +void __dlm_queue_bast(struct dlm_ctxt *dlm, struct dlm_lock *lock); +void dlm_do_local_ast(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock); +int dlm_do_remote_ast(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock); +void dlm_do_local_bast(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock, + int blocked_type); +int dlm_send_proxy_ast_msg(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock, + int msg_type, + int blocked_type, int flags); +static inline int dlm_send_proxy_bast(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock, + int blocked_type) +{ + return dlm_send_proxy_ast_msg(dlm, res, lock, DLM_BAST, + blocked_type, 0); +} + +static inline int dlm_send_proxy_ast(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock, + int flags) +{ + return dlm_send_proxy_ast_msg(dlm, res, lock, DLM_AST, + 0, flags); +} + +void dlm_print_one_lock_resource(struct dlm_lock_resource *res); +void __dlm_print_one_lock_resource(struct dlm_lock_resource *res); + +void dlm_kick_thread(struct dlm_ctxt *dlm, struct dlm_lock_resource *res); +void __dlm_dirty_lockres(struct dlm_ctxt *dlm, struct dlm_lock_resource *res); + + +void dlm_hb_node_down_cb(struct o2nm_node *node, int idx, void *data); +void dlm_hb_node_up_cb(struct o2nm_node *node, int idx, void *data); + +int dlm_empty_lockres(struct dlm_ctxt *dlm, struct dlm_lock_resource *res); +int dlm_finish_migration(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + u8 old_master); +void dlm_lockres_release_ast(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res); +void __dlm_lockres_reserve_ast(struct dlm_lock_resource *res); + +int dlm_master_request_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data); +int dlm_assert_master_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data); +void dlm_assert_master_post_handler(int status, void *data, void *ret_data); +int dlm_deref_lockres_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data); +int dlm_deref_lockres_done_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data); +int dlm_migrate_request_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data); +int dlm_mig_lockres_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data); +int dlm_master_requery_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data); +int dlm_request_all_locks_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data); +int dlm_reco_data_done_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data); +int dlm_begin_reco_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data); +int dlm_finalize_reco_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data); +int dlm_do_master_requery(struct dlm_ctxt *dlm, struct dlm_lock_resource *res, + u8 nodenum, u8 *real_master); + +void __dlm_do_purge_lockres(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res); + +int dlm_dispatch_assert_master(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + int ignore_higher, + u8 request_from, + u32 flags); + + +int dlm_send_one_lockres(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_migratable_lockres *mres, + u8 send_to, + u8 flags); +void dlm_move_lockres_to_recovery_list(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res); + +/* will exit holding res->spinlock, but may drop in function */ +void __dlm_wait_on_lockres_flags(struct dlm_lock_resource *res, int flags); + +/* will exit holding res->spinlock, but may drop in function */ +static inline void __dlm_wait_on_lockres(struct dlm_lock_resource *res) +{ + __dlm_wait_on_lockres_flags(res, (DLM_LOCK_RES_IN_PROGRESS| + DLM_LOCK_RES_RECOVERING| + DLM_LOCK_RES_RECOVERY_WAITING| + DLM_LOCK_RES_MIGRATING)); +} + +void __dlm_unlink_mle(struct dlm_ctxt *dlm, struct dlm_master_list_entry *mle); +void __dlm_insert_mle(struct dlm_ctxt *dlm, struct dlm_master_list_entry *mle); + +/* create/destroy slab caches */ +int dlm_init_master_caches(void); +void dlm_destroy_master_caches(void); + +int dlm_init_lock_cache(void); +void dlm_destroy_lock_cache(void); + +int dlm_init_mle_cache(void); +void dlm_destroy_mle_cache(void); + +void dlm_hb_event_notify_attached(struct dlm_ctxt *dlm, int idx, int node_up); +int dlm_drop_lockres_ref(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res); +void dlm_clean_master_list(struct dlm_ctxt *dlm, + u8 dead_node); +void dlm_force_free_mles(struct dlm_ctxt *dlm); +int dlm_lock_basts_flushed(struct dlm_ctxt *dlm, struct dlm_lock *lock); +int __dlm_lockres_has_locks(struct dlm_lock_resource *res); +int __dlm_lockres_unused(struct dlm_lock_resource *res); + +static inline const char * dlm_lock_mode_name(int mode) +{ + switch (mode) { + case LKM_EXMODE: + return "EX"; + case LKM_PRMODE: + return "PR"; + case LKM_NLMODE: + return "NL"; + } + return "UNKNOWN"; +} + + +static inline int dlm_lock_compatible(int existing, int request) +{ + /* NO_LOCK compatible with all */ + if (request == LKM_NLMODE || + existing == LKM_NLMODE) + return 1; + + /* EX incompatible with all non-NO_LOCK */ + if (request == LKM_EXMODE) + return 0; + + /* request must be PR, which is compatible with PR */ + if (existing == LKM_PRMODE) + return 1; + + return 0; +} + +static inline int dlm_lock_on_list(struct list_head *head, + struct dlm_lock *lock) +{ + struct dlm_lock *tmplock; + + list_for_each_entry(tmplock, head, list) { + if (tmplock == lock) + return 1; + } + return 0; +} + + +static inline enum dlm_status dlm_err_to_dlm_status(int err) +{ + enum dlm_status ret; + if (err == -ENOMEM) + ret = DLM_SYSERR; + else if (err == -ETIMEDOUT || o2net_link_down(err, NULL)) + ret = DLM_NOLOCKMGR; + else if (err == -EINVAL) + ret = DLM_BADPARAM; + else if (err == -ENAMETOOLONG) + ret = DLM_IVBUFLEN; + else + ret = DLM_BADARGS; + return ret; +} + + +static inline void dlm_node_iter_init(unsigned long *map, + struct dlm_node_iter *iter) +{ + bitmap_copy(iter->node_map, map, O2NM_MAX_NODES); + iter->curnode = -1; +} + +static inline int dlm_node_iter_next(struct dlm_node_iter *iter) +{ + int bit; + bit = find_next_bit(iter->node_map, O2NM_MAX_NODES, iter->curnode+1); + if (bit >= O2NM_MAX_NODES) { + iter->curnode = O2NM_MAX_NODES; + return -ENOENT; + } + iter->curnode = bit; + return bit; +} + +static inline void dlm_set_lockres_owner(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + u8 owner) +{ + assert_spin_locked(&res->spinlock); + + res->owner = owner; +} + +static inline void dlm_change_lockres_owner(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + u8 owner) +{ + assert_spin_locked(&res->spinlock); + + if (owner != res->owner) + dlm_set_lockres_owner(dlm, res, owner); +} + +#endif /* DLMCOMMON_H */ diff --git a/fs/ocfs2/dlm/dlmconvert.c b/fs/ocfs2/dlm/dlmconvert.c new file mode 100644 index 000000000..450d46eef --- /dev/null +++ b/fs/ocfs2/dlm/dlmconvert.c @@ -0,0 +1,555 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * dlmconvert.c + * + * underlying calls for lock conversion + * + * Copyright (C) 2004 Oracle. All rights reserved. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#include "../cluster/heartbeat.h" +#include "../cluster/nodemanager.h" +#include "../cluster/tcp.h" + +#include "dlmapi.h" +#include "dlmcommon.h" + +#include "dlmconvert.h" + +#define MLOG_MASK_PREFIX ML_DLM +#include "../cluster/masklog.h" + +/* NOTE: __dlmconvert_master is the only function in here that + * needs a spinlock held on entry (res->spinlock) and it is the + * only one that holds a lock on exit (res->spinlock). + * All other functions in here need no locks and drop all of + * the locks that they acquire. */ +static enum dlm_status __dlmconvert_master(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock, int flags, + int type, int *call_ast, + int *kick_thread); +static enum dlm_status dlm_send_remote_convert_request(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock, int flags, int type); + +/* + * this is only called directly by dlmlock(), and only when the + * local node is the owner of the lockres + * locking: + * caller needs: none + * taken: takes and drops res->spinlock + * held on exit: none + * returns: see __dlmconvert_master + */ +enum dlm_status dlmconvert_master(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock, int flags, int type) +{ + int call_ast = 0, kick_thread = 0; + enum dlm_status status; + + spin_lock(&res->spinlock); + /* we are not in a network handler, this is fine */ + __dlm_wait_on_lockres(res); + __dlm_lockres_reserve_ast(res); + res->state |= DLM_LOCK_RES_IN_PROGRESS; + + status = __dlmconvert_master(dlm, res, lock, flags, type, + &call_ast, &kick_thread); + + res->state &= ~DLM_LOCK_RES_IN_PROGRESS; + spin_unlock(&res->spinlock); + wake_up(&res->wq); + if (status != DLM_NORMAL && status != DLM_NOTQUEUED) + dlm_error(status); + + /* either queue the ast or release it */ + if (call_ast) + dlm_queue_ast(dlm, lock); + else + dlm_lockres_release_ast(dlm, res); + + if (kick_thread) + dlm_kick_thread(dlm, res); + + return status; +} + +/* performs lock conversion at the lockres master site + * locking: + * caller needs: res->spinlock + * taken: takes and drops lock->spinlock + * held on exit: res->spinlock + * returns: DLM_NORMAL, DLM_NOTQUEUED, DLM_DENIED + * call_ast: whether ast should be called for this lock + * kick_thread: whether dlm_kick_thread should be called + */ +static enum dlm_status __dlmconvert_master(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock, int flags, + int type, int *call_ast, + int *kick_thread) +{ + enum dlm_status status = DLM_NORMAL; + struct dlm_lock *tmplock=NULL; + + assert_spin_locked(&res->spinlock); + + mlog(0, "type=%d, convert_type=%d, new convert_type=%d\n", + lock->ml.type, lock->ml.convert_type, type); + + spin_lock(&lock->spinlock); + + /* already converting? */ + if (lock->ml.convert_type != LKM_IVMODE) { + mlog(ML_ERROR, "attempted to convert a lock with a lock " + "conversion pending\n"); + status = DLM_DENIED; + goto unlock_exit; + } + + /* must be on grant queue to convert */ + if (!dlm_lock_on_list(&res->granted, lock)) { + mlog(ML_ERROR, "attempted to convert a lock not on grant " + "queue\n"); + status = DLM_DENIED; + goto unlock_exit; + } + + if (flags & LKM_VALBLK) { + switch (lock->ml.type) { + case LKM_EXMODE: + /* EX + LKM_VALBLK + convert == set lvb */ + mlog(0, "will set lvb: converting %s->%s\n", + dlm_lock_mode_name(lock->ml.type), + dlm_lock_mode_name(type)); + lock->lksb->flags |= DLM_LKSB_PUT_LVB; + break; + case LKM_PRMODE: + case LKM_NLMODE: + /* refetch if new level is not NL */ + if (type > LKM_NLMODE) { + mlog(0, "will fetch new value into " + "lvb: converting %s->%s\n", + dlm_lock_mode_name(lock->ml.type), + dlm_lock_mode_name(type)); + lock->lksb->flags |= DLM_LKSB_GET_LVB; + } else { + mlog(0, "will NOT fetch new value " + "into lvb: converting %s->%s\n", + dlm_lock_mode_name(lock->ml.type), + dlm_lock_mode_name(type)); + flags &= ~(LKM_VALBLK); + } + break; + } + } + + + /* in-place downconvert? */ + if (type <= lock->ml.type) + goto grant; + + /* upconvert from here on */ + status = DLM_NORMAL; + list_for_each_entry(tmplock, &res->granted, list) { + if (tmplock == lock) + continue; + if (!dlm_lock_compatible(tmplock->ml.type, type)) + goto switch_queues; + } + + list_for_each_entry(tmplock, &res->converting, list) { + if (!dlm_lock_compatible(tmplock->ml.type, type)) + goto switch_queues; + /* existing conversion requests take precedence */ + if (!dlm_lock_compatible(tmplock->ml.convert_type, type)) + goto switch_queues; + } + + /* fall thru to grant */ + +grant: + mlog(0, "res %.*s, granting %s lock\n", res->lockname.len, + res->lockname.name, dlm_lock_mode_name(type)); + /* immediately grant the new lock type */ + lock->lksb->status = DLM_NORMAL; + if (lock->ml.node == dlm->node_num) + mlog(0, "doing in-place convert for nonlocal lock\n"); + lock->ml.type = type; + if (lock->lksb->flags & DLM_LKSB_PUT_LVB) + memcpy(res->lvb, lock->lksb->lvb, DLM_LVB_LEN); + + /* + * Move the lock to the tail because it may be the only lock which has + * an invalid lvb. + */ + list_move_tail(&lock->list, &res->granted); + + status = DLM_NORMAL; + *call_ast = 1; + goto unlock_exit; + +switch_queues: + if (flags & LKM_NOQUEUE) { + mlog(0, "failed to convert NOQUEUE lock %.*s from " + "%d to %d...\n", res->lockname.len, res->lockname.name, + lock->ml.type, type); + status = DLM_NOTQUEUED; + goto unlock_exit; + } + mlog(0, "res %.*s, queueing...\n", res->lockname.len, + res->lockname.name); + + lock->ml.convert_type = type; + /* do not alter lock refcount. switching lists. */ + list_move_tail(&lock->list, &res->converting); + +unlock_exit: + spin_unlock(&lock->spinlock); + if (status == DLM_DENIED) { + __dlm_print_one_lock_resource(res); + } + if (status == DLM_NORMAL) + *kick_thread = 1; + return status; +} + +void dlm_revert_pending_convert(struct dlm_lock_resource *res, + struct dlm_lock *lock) +{ + /* do not alter lock refcount. switching lists. */ + list_move_tail(&lock->list, &res->granted); + lock->ml.convert_type = LKM_IVMODE; + lock->lksb->flags &= ~(DLM_LKSB_GET_LVB|DLM_LKSB_PUT_LVB); +} + +/* messages the master site to do lock conversion + * locking: + * caller needs: none + * taken: takes and drops res->spinlock, uses DLM_LOCK_RES_IN_PROGRESS + * held on exit: none + * returns: DLM_NORMAL, DLM_RECOVERING, status from remote node + */ +enum dlm_status dlmconvert_remote(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock, int flags, int type) +{ + enum dlm_status status; + + mlog(0, "type=%d, convert_type=%d, busy=%d\n", lock->ml.type, + lock->ml.convert_type, res->state & DLM_LOCK_RES_IN_PROGRESS); + + spin_lock(&res->spinlock); + if (res->state & DLM_LOCK_RES_RECOVERING) { + mlog(0, "bailing out early since res is RECOVERING " + "on secondary queue\n"); + /* __dlm_print_one_lock_resource(res); */ + status = DLM_RECOVERING; + goto bail; + } + /* will exit this call with spinlock held */ + __dlm_wait_on_lockres(res); + + if (lock->ml.convert_type != LKM_IVMODE) { + __dlm_print_one_lock_resource(res); + mlog(ML_ERROR, "converting a remote lock that is already " + "converting! (cookie=%u:%llu, conv=%d)\n", + dlm_get_lock_cookie_node(be64_to_cpu(lock->ml.cookie)), + dlm_get_lock_cookie_seq(be64_to_cpu(lock->ml.cookie)), + lock->ml.convert_type); + status = DLM_DENIED; + goto bail; + } + + if (lock->ml.type == type && lock->ml.convert_type == LKM_IVMODE) { + mlog(0, "last convert request returned DLM_RECOVERING, but " + "owner has already queued and sent ast to me. res %.*s, " + "(cookie=%u:%llu, type=%d, conv=%d)\n", + res->lockname.len, res->lockname.name, + dlm_get_lock_cookie_node(be64_to_cpu(lock->ml.cookie)), + dlm_get_lock_cookie_seq(be64_to_cpu(lock->ml.cookie)), + lock->ml.type, lock->ml.convert_type); + status = DLM_NORMAL; + goto bail; + } + + res->state |= DLM_LOCK_RES_IN_PROGRESS; + /* move lock to local convert queue */ + /* do not alter lock refcount. switching lists. */ + list_move_tail(&lock->list, &res->converting); + lock->convert_pending = 1; + lock->ml.convert_type = type; + + if (flags & LKM_VALBLK) { + if (lock->ml.type == LKM_EXMODE) { + flags |= LKM_PUT_LVB; + lock->lksb->flags |= DLM_LKSB_PUT_LVB; + } else { + if (lock->ml.convert_type == LKM_NLMODE) + flags &= ~LKM_VALBLK; + else { + flags |= LKM_GET_LVB; + lock->lksb->flags |= DLM_LKSB_GET_LVB; + } + } + } + spin_unlock(&res->spinlock); + + /* no locks held here. + * need to wait for a reply as to whether it got queued or not. */ + status = dlm_send_remote_convert_request(dlm, res, lock, flags, type); + + spin_lock(&res->spinlock); + res->state &= ~DLM_LOCK_RES_IN_PROGRESS; + /* if it failed, move it back to granted queue. + * if master returns DLM_NORMAL and then down before sending ast, + * it may have already been moved to granted queue, reset to + * DLM_RECOVERING and retry convert */ + if (status != DLM_NORMAL) { + if (status != DLM_NOTQUEUED) + dlm_error(status); + dlm_revert_pending_convert(res, lock); + } else if (!lock->convert_pending) { + mlog(0, "%s: res %.*s, owner died and lock has been moved back " + "to granted list, retry convert.\n", + dlm->name, res->lockname.len, res->lockname.name); + status = DLM_RECOVERING; + } + + lock->convert_pending = 0; +bail: + spin_unlock(&res->spinlock); + + /* TODO: should this be a wake_one? */ + /* wake up any IN_PROGRESS waiters */ + wake_up(&res->wq); + + return status; +} + +/* sends DLM_CONVERT_LOCK_MSG to master site + * locking: + * caller needs: none + * taken: none + * held on exit: none + * returns: DLM_NOLOCKMGR, status from remote node + */ +static enum dlm_status dlm_send_remote_convert_request(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock, int flags, int type) +{ + struct dlm_convert_lock convert; + int tmpret; + enum dlm_status ret; + int status = 0; + struct kvec vec[2]; + size_t veclen = 1; + + mlog(0, "%.*s\n", res->lockname.len, res->lockname.name); + + memset(&convert, 0, sizeof(struct dlm_convert_lock)); + convert.node_idx = dlm->node_num; + convert.requested_type = type; + convert.cookie = lock->ml.cookie; + convert.namelen = res->lockname.len; + convert.flags = cpu_to_be32(flags); + memcpy(convert.name, res->lockname.name, convert.namelen); + + vec[0].iov_len = sizeof(struct dlm_convert_lock); + vec[0].iov_base = &convert; + + if (flags & LKM_PUT_LVB) { + /* extra data to send if we are updating lvb */ + vec[1].iov_len = DLM_LVB_LEN; + vec[1].iov_base = lock->lksb->lvb; + veclen++; + } + + tmpret = o2net_send_message_vec(DLM_CONVERT_LOCK_MSG, dlm->key, + vec, veclen, res->owner, &status); + if (tmpret >= 0) { + // successfully sent and received + ret = status; // this is already a dlm_status + if (ret == DLM_RECOVERING) { + mlog(0, "node %u returned DLM_RECOVERING from convert " + "message!\n", res->owner); + } else if (ret == DLM_MIGRATING) { + mlog(0, "node %u returned DLM_MIGRATING from convert " + "message!\n", res->owner); + } else if (ret == DLM_FORWARD) { + mlog(0, "node %u returned DLM_FORWARD from convert " + "message!\n", res->owner); + } else if (ret != DLM_NORMAL && ret != DLM_NOTQUEUED) + dlm_error(ret); + } else { + mlog(ML_ERROR, "Error %d when sending message %u (key 0x%x) to " + "node %u\n", tmpret, DLM_CONVERT_LOCK_MSG, dlm->key, + res->owner); + if (dlm_is_host_down(tmpret)) { + /* instead of logging the same network error over + * and over, sleep here and wait for the heartbeat + * to notice the node is dead. times out after 5s. */ + dlm_wait_for_node_death(dlm, res->owner, + DLM_NODE_DEATH_WAIT_MAX); + ret = DLM_RECOVERING; + mlog(0, "node %u died so returning DLM_RECOVERING " + "from convert message!\n", res->owner); + } else { + ret = dlm_err_to_dlm_status(tmpret); + } + } + + return ret; +} + +/* handler for DLM_CONVERT_LOCK_MSG on master site + * locking: + * caller needs: none + * taken: takes and drop res->spinlock + * held on exit: none + * returns: DLM_NORMAL, DLM_IVLOCKID, DLM_BADARGS, + * status from __dlmconvert_master + */ +int dlm_convert_lock_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data) +{ + struct dlm_ctxt *dlm = data; + struct dlm_convert_lock *cnv = (struct dlm_convert_lock *)msg->buf; + struct dlm_lock_resource *res = NULL; + struct dlm_lock *lock = NULL; + struct dlm_lock *tmp_lock; + struct dlm_lockstatus *lksb; + enum dlm_status status = DLM_NORMAL; + u32 flags; + int call_ast = 0, kick_thread = 0, ast_reserved = 0, wake = 0; + + if (!dlm_grab(dlm)) { + dlm_error(DLM_REJECTED); + return DLM_REJECTED; + } + + mlog_bug_on_msg(!dlm_domain_fully_joined(dlm), + "Domain %s not fully joined!\n", dlm->name); + + if (cnv->namelen > DLM_LOCKID_NAME_MAX) { + status = DLM_IVBUFLEN; + dlm_error(status); + goto leave; + } + + flags = be32_to_cpu(cnv->flags); + + if ((flags & (LKM_PUT_LVB|LKM_GET_LVB)) == + (LKM_PUT_LVB|LKM_GET_LVB)) { + mlog(ML_ERROR, "both PUT and GET lvb specified\n"); + status = DLM_BADARGS; + goto leave; + } + + mlog(0, "lvb: %s\n", flags & LKM_PUT_LVB ? "put lvb" : + (flags & LKM_GET_LVB ? "get lvb" : "none")); + + status = DLM_IVLOCKID; + res = dlm_lookup_lockres(dlm, cnv->name, cnv->namelen); + if (!res) { + dlm_error(status); + goto leave; + } + + spin_lock(&res->spinlock); + status = __dlm_lockres_state_to_status(res); + if (status != DLM_NORMAL) { + spin_unlock(&res->spinlock); + dlm_error(status); + goto leave; + } + list_for_each_entry(tmp_lock, &res->granted, list) { + if (tmp_lock->ml.cookie == cnv->cookie && + tmp_lock->ml.node == cnv->node_idx) { + lock = tmp_lock; + dlm_lock_get(lock); + break; + } + } + spin_unlock(&res->spinlock); + if (!lock) { + status = DLM_IVLOCKID; + mlog(ML_ERROR, "did not find lock to convert on grant queue! " + "cookie=%u:%llu\n", + dlm_get_lock_cookie_node(be64_to_cpu(cnv->cookie)), + dlm_get_lock_cookie_seq(be64_to_cpu(cnv->cookie))); + dlm_print_one_lock_resource(res); + goto leave; + } + + /* found the lock */ + lksb = lock->lksb; + + /* see if caller needed to get/put lvb */ + if (flags & LKM_PUT_LVB) { + BUG_ON(lksb->flags & (DLM_LKSB_PUT_LVB|DLM_LKSB_GET_LVB)); + lksb->flags |= DLM_LKSB_PUT_LVB; + memcpy(&lksb->lvb[0], &cnv->lvb[0], DLM_LVB_LEN); + } else if (flags & LKM_GET_LVB) { + BUG_ON(lksb->flags & (DLM_LKSB_PUT_LVB|DLM_LKSB_GET_LVB)); + lksb->flags |= DLM_LKSB_GET_LVB; + } + + spin_lock(&res->spinlock); + status = __dlm_lockres_state_to_status(res); + if (status == DLM_NORMAL) { + __dlm_lockres_reserve_ast(res); + ast_reserved = 1; + res->state |= DLM_LOCK_RES_IN_PROGRESS; + status = __dlmconvert_master(dlm, res, lock, flags, + cnv->requested_type, + &call_ast, &kick_thread); + res->state &= ~DLM_LOCK_RES_IN_PROGRESS; + wake = 1; + } + spin_unlock(&res->spinlock); + if (wake) + wake_up(&res->wq); + + if (status != DLM_NORMAL) { + if (status != DLM_NOTQUEUED) + dlm_error(status); + lksb->flags &= ~(DLM_LKSB_GET_LVB|DLM_LKSB_PUT_LVB); + } + +leave: + if (lock) + dlm_lock_put(lock); + + /* either queue the ast or release it, if reserved */ + if (call_ast) + dlm_queue_ast(dlm, lock); + else if (ast_reserved) + dlm_lockres_release_ast(dlm, res); + + if (kick_thread) + dlm_kick_thread(dlm, res); + + if (res) + dlm_lockres_put(res); + + dlm_put(dlm); + + return status; +} diff --git a/fs/ocfs2/dlm/dlmconvert.h b/fs/ocfs2/dlm/dlmconvert.h new file mode 100644 index 000000000..1f3717165 --- /dev/null +++ b/fs/ocfs2/dlm/dlmconvert.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * dlmconvert.h + * + * Copyright (C) 2004 Oracle. All rights reserved. + */ + +#ifndef DLMCONVERT_H +#define DLMCONVERT_H + +enum dlm_status dlmconvert_master(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock, int flags, int type); +enum dlm_status dlmconvert_remote(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock, int flags, int type); + +#endif diff --git a/fs/ocfs2/dlm/dlmdebug.c b/fs/ocfs2/dlm/dlmdebug.c new file mode 100644 index 000000000..ce6d301cb --- /dev/null +++ b/fs/ocfs2/dlm/dlmdebug.c @@ -0,0 +1,844 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * dlmdebug.c + * + * debug functionality for the dlm + * + * Copyright (C) 2004, 2008 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../cluster/heartbeat.h" +#include "../cluster/nodemanager.h" +#include "../cluster/tcp.h" + +#include "dlmapi.h" +#include "dlmcommon.h" +#include "dlmdomain.h" +#include "dlmdebug.h" + +#define MLOG_MASK_PREFIX ML_DLM +#include "../cluster/masklog.h" + +static int stringify_lockname(const char *lockname, int locklen, char *buf, + int len); + +void dlm_print_one_lock_resource(struct dlm_lock_resource *res) +{ + spin_lock(&res->spinlock); + __dlm_print_one_lock_resource(res); + spin_unlock(&res->spinlock); +} + +static void dlm_print_lockres_refmap(struct dlm_lock_resource *res) +{ + int bit; + assert_spin_locked(&res->spinlock); + + printk(" refmap nodes: [ "); + bit = 0; + while (1) { + bit = find_next_bit(res->refmap, O2NM_MAX_NODES, bit); + if (bit >= O2NM_MAX_NODES) + break; + printk("%u ", bit); + bit++; + } + printk("], inflight=%u\n", res->inflight_locks); +} + +static void __dlm_print_lock(struct dlm_lock *lock) +{ + spin_lock(&lock->spinlock); + + printk(" type=%d, conv=%d, node=%u, cookie=%u:%llu, " + "ref=%u, ast=(empty=%c,pend=%c), bast=(empty=%c,pend=%c), " + "pending=(conv=%c,lock=%c,cancel=%c,unlock=%c)\n", + lock->ml.type, lock->ml.convert_type, lock->ml.node, + dlm_get_lock_cookie_node(be64_to_cpu(lock->ml.cookie)), + dlm_get_lock_cookie_seq(be64_to_cpu(lock->ml.cookie)), + kref_read(&lock->lock_refs), + (list_empty(&lock->ast_list) ? 'y' : 'n'), + (lock->ast_pending ? 'y' : 'n'), + (list_empty(&lock->bast_list) ? 'y' : 'n'), + (lock->bast_pending ? 'y' : 'n'), + (lock->convert_pending ? 'y' : 'n'), + (lock->lock_pending ? 'y' : 'n'), + (lock->cancel_pending ? 'y' : 'n'), + (lock->unlock_pending ? 'y' : 'n')); + + spin_unlock(&lock->spinlock); +} + +void __dlm_print_one_lock_resource(struct dlm_lock_resource *res) +{ + struct dlm_lock *lock; + char buf[DLM_LOCKID_NAME_MAX]; + + assert_spin_locked(&res->spinlock); + + stringify_lockname(res->lockname.name, res->lockname.len, + buf, sizeof(buf)); + printk("lockres: %s, owner=%u, state=%u\n", + buf, res->owner, res->state); + printk(" last used: %lu, refcnt: %u, on purge list: %s\n", + res->last_used, kref_read(&res->refs), + str_no_yes(list_empty(&res->purge))); + printk(" on dirty list: %s, on reco list: %s, " + "migrating pending: %s\n", + str_no_yes(list_empty(&res->dirty)), + str_no_yes(list_empty(&res->recovering)), + str_yes_no(res->migration_pending)); + printk(" inflight locks: %d, asts reserved: %d\n", + res->inflight_locks, atomic_read(&res->asts_reserved)); + dlm_print_lockres_refmap(res); + printk(" granted queue:\n"); + list_for_each_entry(lock, &res->granted, list) { + __dlm_print_lock(lock); + } + printk(" converting queue:\n"); + list_for_each_entry(lock, &res->converting, list) { + __dlm_print_lock(lock); + } + printk(" blocked queue:\n"); + list_for_each_entry(lock, &res->blocked, list) { + __dlm_print_lock(lock); + } +} + +void dlm_print_one_lock(struct dlm_lock *lockid) +{ + dlm_print_one_lock_resource(lockid->lockres); +} +EXPORT_SYMBOL_GPL(dlm_print_one_lock); + +static const char *dlm_errnames[] = { + [DLM_NORMAL] = "DLM_NORMAL", + [DLM_GRANTED] = "DLM_GRANTED", + [DLM_DENIED] = "DLM_DENIED", + [DLM_DENIED_NOLOCKS] = "DLM_DENIED_NOLOCKS", + [DLM_WORKING] = "DLM_WORKING", + [DLM_BLOCKED] = "DLM_BLOCKED", + [DLM_BLOCKED_ORPHAN] = "DLM_BLOCKED_ORPHAN", + [DLM_DENIED_GRACE_PERIOD] = "DLM_DENIED_GRACE_PERIOD", + [DLM_SYSERR] = "DLM_SYSERR", + [DLM_NOSUPPORT] = "DLM_NOSUPPORT", + [DLM_CANCELGRANT] = "DLM_CANCELGRANT", + [DLM_IVLOCKID] = "DLM_IVLOCKID", + [DLM_SYNC] = "DLM_SYNC", + [DLM_BADTYPE] = "DLM_BADTYPE", + [DLM_BADRESOURCE] = "DLM_BADRESOURCE", + [DLM_MAXHANDLES] = "DLM_MAXHANDLES", + [DLM_NOCLINFO] = "DLM_NOCLINFO", + [DLM_NOLOCKMGR] = "DLM_NOLOCKMGR", + [DLM_NOPURGED] = "DLM_NOPURGED", + [DLM_BADARGS] = "DLM_BADARGS", + [DLM_VOID] = "DLM_VOID", + [DLM_NOTQUEUED] = "DLM_NOTQUEUED", + [DLM_IVBUFLEN] = "DLM_IVBUFLEN", + [DLM_CVTUNGRANT] = "DLM_CVTUNGRANT", + [DLM_BADPARAM] = "DLM_BADPARAM", + [DLM_VALNOTVALID] = "DLM_VALNOTVALID", + [DLM_REJECTED] = "DLM_REJECTED", + [DLM_ABORT] = "DLM_ABORT", + [DLM_CANCEL] = "DLM_CANCEL", + [DLM_IVRESHANDLE] = "DLM_IVRESHANDLE", + [DLM_DEADLOCK] = "DLM_DEADLOCK", + [DLM_DENIED_NOASTS] = "DLM_DENIED_NOASTS", + [DLM_FORWARD] = "DLM_FORWARD", + [DLM_TIMEOUT] = "DLM_TIMEOUT", + [DLM_IVGROUPID] = "DLM_IVGROUPID", + [DLM_VERS_CONFLICT] = "DLM_VERS_CONFLICT", + [DLM_BAD_DEVICE_PATH] = "DLM_BAD_DEVICE_PATH", + [DLM_NO_DEVICE_PERMISSION] = "DLM_NO_DEVICE_PERMISSION", + [DLM_NO_CONTROL_DEVICE ] = "DLM_NO_CONTROL_DEVICE ", + [DLM_RECOVERING] = "DLM_RECOVERING", + [DLM_MIGRATING] = "DLM_MIGRATING", + [DLM_MAXSTATS] = "DLM_MAXSTATS", +}; + +const char *dlm_errname(enum dlm_status err) +{ + if (err >= DLM_MAXSTATS || err < 0) + return dlm_errnames[DLM_MAXSTATS]; + return dlm_errnames[err]; +} +EXPORT_SYMBOL_GPL(dlm_errname); + +/* NOTE: This function converts a lockname into a string. It uses knowledge + * of the format of the lockname that should be outside the purview of the dlm. + * We are adding only to make dlm debugging slightly easier. + * + * For more on lockname formats, please refer to dlmglue.c and ocfs2_lockid.h. + */ +static int stringify_lockname(const char *lockname, int locklen, char *buf, + int len) +{ + int out = 0; + __be64 inode_blkno_be; + +#define OCFS2_DENTRY_LOCK_INO_START 18 + if (*lockname == 'N') { + memcpy((__be64 *)&inode_blkno_be, + (char *)&lockname[OCFS2_DENTRY_LOCK_INO_START], + sizeof(__be64)); + out += scnprintf(buf + out, len - out, "%.*s%08x", + OCFS2_DENTRY_LOCK_INO_START - 1, lockname, + (unsigned int)be64_to_cpu(inode_blkno_be)); + } else + out += scnprintf(buf + out, len - out, "%.*s", + locklen, lockname); + return out; +} + +static int stringify_nodemap(unsigned long *nodemap, int maxnodes, + char *buf, int len) +{ + int out = 0; + int i = -1; + + while ((i = find_next_bit(nodemap, maxnodes, i + 1)) < maxnodes) + out += scnprintf(buf + out, len - out, "%d ", i); + + return out; +} + +static int dump_mle(struct dlm_master_list_entry *mle, char *buf, int len) +{ + int out = 0; + char *mle_type; + + if (mle->type == DLM_MLE_BLOCK) + mle_type = "BLK"; + else if (mle->type == DLM_MLE_MASTER) + mle_type = "MAS"; + else + mle_type = "MIG"; + + out += stringify_lockname(mle->mname, mle->mnamelen, buf + out, len - out); + out += scnprintf(buf + out, len - out, + "\t%3s\tmas=%3u\tnew=%3u\tevt=%1d\tuse=%1d\tref=%3d\n", + mle_type, mle->master, mle->new_master, + !list_empty(&mle->hb_events), + !!mle->inuse, + kref_read(&mle->mle_refs)); + + out += scnprintf(buf + out, len - out, "Maybe="); + out += stringify_nodemap(mle->maybe_map, O2NM_MAX_NODES, + buf + out, len - out); + out += scnprintf(buf + out, len - out, "\n"); + + out += scnprintf(buf + out, len - out, "Vote="); + out += stringify_nodemap(mle->vote_map, O2NM_MAX_NODES, + buf + out, len - out); + out += scnprintf(buf + out, len - out, "\n"); + + out += scnprintf(buf + out, len - out, "Response="); + out += stringify_nodemap(mle->response_map, O2NM_MAX_NODES, + buf + out, len - out); + out += scnprintf(buf + out, len - out, "\n"); + + out += scnprintf(buf + out, len - out, "Node="); + out += stringify_nodemap(mle->node_map, O2NM_MAX_NODES, + buf + out, len - out); + out += scnprintf(buf + out, len - out, "\n"); + + out += scnprintf(buf + out, len - out, "\n"); + + return out; +} + +void dlm_print_one_mle(struct dlm_master_list_entry *mle) +{ + char *buf; + + buf = kzalloc(PAGE_SIZE, GFP_ATOMIC); + if (buf) { + dump_mle(mle, buf, PAGE_SIZE - 1); + kfree(buf); + } +} + +#ifdef CONFIG_DEBUG_FS + +static struct dentry *dlm_debugfs_root; + +#define DLM_DEBUGFS_DIR "o2dlm" +#define DLM_DEBUGFS_DLM_STATE "dlm_state" +#define DLM_DEBUGFS_LOCKING_STATE "locking_state" +#define DLM_DEBUGFS_MLE_STATE "mle_state" +#define DLM_DEBUGFS_PURGE_LIST "purge_list" + +/* begin - utils funcs */ +static int debug_release(struct inode *inode, struct file *file) +{ + kfree(file->private_data); + return 0; +} + +static ssize_t debug_read(struct file *file, char __user *buf, + size_t nbytes, loff_t *ppos) +{ + return simple_read_from_buffer(buf, nbytes, ppos, file->private_data, + i_size_read(file->f_mapping->host)); +} +/* end - util funcs */ + +/* begin - purge list funcs */ +static int debug_purgelist_print(struct dlm_ctxt *dlm, char *buf, int len) +{ + struct dlm_lock_resource *res; + int out = 0; + unsigned long total = 0; + + out += scnprintf(buf + out, len - out, + "Dumping Purgelist for Domain: %s\n", dlm->name); + + spin_lock(&dlm->spinlock); + list_for_each_entry(res, &dlm->purge_list, purge) { + ++total; + if (len - out < 100) + continue; + spin_lock(&res->spinlock); + out += stringify_lockname(res->lockname.name, + res->lockname.len, + buf + out, len - out); + out += scnprintf(buf + out, len - out, "\t%ld\n", + (jiffies - res->last_used)/HZ); + spin_unlock(&res->spinlock); + } + spin_unlock(&dlm->spinlock); + + out += scnprintf(buf + out, len - out, "Total on list: %lu\n", total); + + return out; +} + +static int debug_purgelist_open(struct inode *inode, struct file *file) +{ + struct dlm_ctxt *dlm = inode->i_private; + char *buf = NULL; + + buf = kzalloc(PAGE_SIZE, GFP_NOFS); + if (!buf) + return -ENOMEM; + + i_size_write(inode, debug_purgelist_print(dlm, buf, PAGE_SIZE - 1)); + + file->private_data = buf; + + return 0; +} + +static const struct file_operations debug_purgelist_fops = { + .open = debug_purgelist_open, + .release = debug_release, + .read = debug_read, + .llseek = generic_file_llseek, +}; +/* end - purge list funcs */ + +/* begin - debug mle funcs */ +static int debug_mle_print(struct dlm_ctxt *dlm, char *buf, int len) +{ + struct dlm_master_list_entry *mle; + struct hlist_head *bucket; + int i, out = 0; + unsigned long total = 0, longest = 0, bucket_count = 0; + + out += scnprintf(buf + out, len - out, + "Dumping MLEs for Domain: %s\n", dlm->name); + + spin_lock(&dlm->master_lock); + for (i = 0; i < DLM_HASH_BUCKETS; i++) { + bucket = dlm_master_hash(dlm, i); + hlist_for_each_entry(mle, bucket, master_hash_node) { + ++total; + ++bucket_count; + if (len - out < 200) + continue; + out += dump_mle(mle, buf + out, len - out); + } + longest = max(longest, bucket_count); + bucket_count = 0; + } + spin_unlock(&dlm->master_lock); + + out += scnprintf(buf + out, len - out, + "Total: %lu, Longest: %lu\n", total, longest); + return out; +} + +static int debug_mle_open(struct inode *inode, struct file *file) +{ + struct dlm_ctxt *dlm = inode->i_private; + char *buf = NULL; + + buf = kzalloc(PAGE_SIZE, GFP_NOFS); + if (!buf) + return -ENOMEM; + + i_size_write(inode, debug_mle_print(dlm, buf, PAGE_SIZE - 1)); + + file->private_data = buf; + + return 0; +} + +static const struct file_operations debug_mle_fops = { + .open = debug_mle_open, + .release = debug_release, + .read = debug_read, + .llseek = generic_file_llseek, +}; + +/* end - debug mle funcs */ + +/* begin - debug lockres funcs */ +static int dump_lock(struct dlm_lock *lock, int list_type, char *buf, int len) +{ + int out; + +#define DEBUG_LOCK_VERSION 1 + spin_lock(&lock->spinlock); + out = scnprintf(buf, len, "LOCK:%d,%d,%d,%d,%d,%d:%lld,%d,%d,%d,%d,%d," + "%d,%d,%d,%d\n", + DEBUG_LOCK_VERSION, + list_type, lock->ml.type, lock->ml.convert_type, + lock->ml.node, + dlm_get_lock_cookie_node(be64_to_cpu(lock->ml.cookie)), + dlm_get_lock_cookie_seq(be64_to_cpu(lock->ml.cookie)), + !list_empty(&lock->ast_list), + !list_empty(&lock->bast_list), + lock->ast_pending, lock->bast_pending, + lock->convert_pending, lock->lock_pending, + lock->cancel_pending, lock->unlock_pending, + kref_read(&lock->lock_refs)); + spin_unlock(&lock->spinlock); + + return out; +} + +static int dump_lockres(struct dlm_lock_resource *res, char *buf, int len) +{ + struct dlm_lock *lock; + int i; + int out = 0; + + out += scnprintf(buf + out, len - out, "NAME:"); + out += stringify_lockname(res->lockname.name, res->lockname.len, + buf + out, len - out); + out += scnprintf(buf + out, len - out, "\n"); + +#define DEBUG_LRES_VERSION 1 + out += scnprintf(buf + out, len - out, + "LRES:%d,%d,%d,%ld,%d,%d,%d,%d,%d,%d,%d\n", + DEBUG_LRES_VERSION, + res->owner, res->state, res->last_used, + !list_empty(&res->purge), + !list_empty(&res->dirty), + !list_empty(&res->recovering), + res->inflight_locks, res->migration_pending, + atomic_read(&res->asts_reserved), + kref_read(&res->refs)); + + /* refmap */ + out += scnprintf(buf + out, len - out, "RMAP:"); + out += stringify_nodemap(res->refmap, O2NM_MAX_NODES, + buf + out, len - out); + out += scnprintf(buf + out, len - out, "\n"); + + /* lvb */ + out += scnprintf(buf + out, len - out, "LVBX:"); + for (i = 0; i < DLM_LVB_LEN; i++) + out += scnprintf(buf + out, len - out, + "%02x", (unsigned char)res->lvb[i]); + out += scnprintf(buf + out, len - out, "\n"); + + /* granted */ + list_for_each_entry(lock, &res->granted, list) + out += dump_lock(lock, 0, buf + out, len - out); + + /* converting */ + list_for_each_entry(lock, &res->converting, list) + out += dump_lock(lock, 1, buf + out, len - out); + + /* blocked */ + list_for_each_entry(lock, &res->blocked, list) + out += dump_lock(lock, 2, buf + out, len - out); + + out += scnprintf(buf + out, len - out, "\n"); + + return out; +} + +static void *lockres_seq_start(struct seq_file *m, loff_t *pos) +{ + struct debug_lockres *dl = m->private; + struct dlm_ctxt *dlm = dl->dl_ctxt; + struct dlm_lock_resource *oldres = dl->dl_res; + struct dlm_lock_resource *res = NULL, *iter; + struct list_head *track_list; + + spin_lock(&dlm->track_lock); + if (oldres) + track_list = &oldres->tracking; + else { + track_list = &dlm->tracking_list; + if (list_empty(track_list)) { + dl = NULL; + spin_unlock(&dlm->track_lock); + goto bail; + } + } + + list_for_each_entry(iter, track_list, tracking) { + if (&iter->tracking != &dlm->tracking_list) { + dlm_lockres_get(iter); + res = iter; + } + break; + } + spin_unlock(&dlm->track_lock); + + if (oldres) + dlm_lockres_put(oldres); + + dl->dl_res = res; + + if (res) { + spin_lock(&res->spinlock); + dump_lockres(res, dl->dl_buf, dl->dl_len - 1); + spin_unlock(&res->spinlock); + } else + dl = NULL; + +bail: + /* passed to seq_show */ + return dl; +} + +static void lockres_seq_stop(struct seq_file *m, void *v) +{ +} + +static void *lockres_seq_next(struct seq_file *m, void *v, loff_t *pos) +{ + return NULL; +} + +static int lockres_seq_show(struct seq_file *s, void *v) +{ + struct debug_lockres *dl = (struct debug_lockres *)v; + + seq_printf(s, "%s", dl->dl_buf); + + return 0; +} + +static const struct seq_operations debug_lockres_ops = { + .start = lockres_seq_start, + .stop = lockres_seq_stop, + .next = lockres_seq_next, + .show = lockres_seq_show, +}; + +static int debug_lockres_open(struct inode *inode, struct file *file) +{ + struct dlm_ctxt *dlm = inode->i_private; + struct debug_lockres *dl; + void *buf; + int status = -ENOMEM; + + buf = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!buf) + goto bail; + + dl = __seq_open_private(file, &debug_lockres_ops, sizeof(*dl)); + if (!dl) + goto bailfree; + + dl->dl_len = PAGE_SIZE; + dl->dl_buf = buf; + + /* ->release uses dl_ctxt after open, so it needs a real pin. */ + dl->dl_ctxt = dlm_grab(dlm); + if (!dl->dl_ctxt) { + status = -ENOENT; + goto bailseq; + } + + return 0; + +bailseq: + seq_release_private(inode, file); +bailfree: + kfree(buf); +bail: + if (status != -ENOENT) + mlog_errno(status); + return status; +} + +static int debug_lockres_release(struct inode *inode, struct file *file) +{ + struct seq_file *seq = file->private_data; + struct debug_lockres *dl = (struct debug_lockres *)seq->private; + + if (dl->dl_res) + dlm_lockres_put(dl->dl_res); + dlm_put(dl->dl_ctxt); + kfree(dl->dl_buf); + return seq_release_private(inode, file); +} + +static const struct file_operations debug_lockres_fops = { + .open = debug_lockres_open, + .release = debug_lockres_release, + .read = seq_read, + .llseek = seq_lseek, +}; +/* end - debug lockres funcs */ + +/* begin - debug state funcs */ +static int debug_state_print(struct dlm_ctxt *dlm, char *buf, int len) +{ + int out = 0; + struct dlm_reco_node_data *node; + char *state; + int cur_mles = 0, tot_mles = 0; + int i; + + spin_lock(&dlm->spinlock); + + switch (dlm->dlm_state) { + case DLM_CTXT_NEW: + state = "NEW"; break; + case DLM_CTXT_JOINED: + state = "JOINED"; break; + case DLM_CTXT_IN_SHUTDOWN: + state = "SHUTDOWN"; break; + case DLM_CTXT_LEAVING: + state = "LEAVING"; break; + default: + state = "UNKNOWN"; break; + } + + /* Domain: xxxxxxxxxx Key: 0xdfbac769 */ + out += scnprintf(buf + out, len - out, + "Domain: %s Key: 0x%08x Protocol: %d.%d\n", + dlm->name, dlm->key, dlm->dlm_locking_proto.pv_major, + dlm->dlm_locking_proto.pv_minor); + + /* Thread Pid: xxx Node: xxx State: xxxxx */ + out += scnprintf(buf + out, len - out, + "Thread Pid: %d Node: %d State: %s\n", + task_pid_nr(dlm->dlm_thread_task), dlm->node_num, state); + + /* Number of Joins: xxx Joining Node: xxx */ + out += scnprintf(buf + out, len - out, + "Number of Joins: %d Joining Node: %d\n", + dlm->num_joins, dlm->joining_node); + + /* Domain Map: xx xx xx */ + out += scnprintf(buf + out, len - out, "Domain Map: "); + out += stringify_nodemap(dlm->domain_map, O2NM_MAX_NODES, + buf + out, len - out); + out += scnprintf(buf + out, len - out, "\n"); + + /* Exit Domain Map: xx xx xx */ + out += scnprintf(buf + out, len - out, "Exit Domain Map: "); + out += stringify_nodemap(dlm->exit_domain_map, O2NM_MAX_NODES, + buf + out, len - out); + out += scnprintf(buf + out, len - out, "\n"); + + /* Live Map: xx xx xx */ + out += scnprintf(buf + out, len - out, "Live Map: "); + out += stringify_nodemap(dlm->live_nodes_map, O2NM_MAX_NODES, + buf + out, len - out); + out += scnprintf(buf + out, len - out, "\n"); + + /* Lock Resources: xxx (xxx) */ + out += scnprintf(buf + out, len - out, + "Lock Resources: %d (%d)\n", + atomic_read(&dlm->res_cur_count), + atomic_read(&dlm->res_tot_count)); + + for (i = 0; i < DLM_MLE_NUM_TYPES; ++i) + tot_mles += atomic_read(&dlm->mle_tot_count[i]); + + for (i = 0; i < DLM_MLE_NUM_TYPES; ++i) + cur_mles += atomic_read(&dlm->mle_cur_count[i]); + + /* MLEs: xxx (xxx) */ + out += scnprintf(buf + out, len - out, + "MLEs: %d (%d)\n", cur_mles, tot_mles); + + /* Blocking: xxx (xxx) */ + out += scnprintf(buf + out, len - out, + " Blocking: %d (%d)\n", + atomic_read(&dlm->mle_cur_count[DLM_MLE_BLOCK]), + atomic_read(&dlm->mle_tot_count[DLM_MLE_BLOCK])); + + /* Mastery: xxx (xxx) */ + out += scnprintf(buf + out, len - out, + " Mastery: %d (%d)\n", + atomic_read(&dlm->mle_cur_count[DLM_MLE_MASTER]), + atomic_read(&dlm->mle_tot_count[DLM_MLE_MASTER])); + + /* Migration: xxx (xxx) */ + out += scnprintf(buf + out, len - out, + " Migration: %d (%d)\n", + atomic_read(&dlm->mle_cur_count[DLM_MLE_MIGRATION]), + atomic_read(&dlm->mle_tot_count[DLM_MLE_MIGRATION])); + + /* Lists: Dirty=Empty Purge=InUse PendingASTs=Empty ... */ + out += scnprintf(buf + out, len - out, + "Lists: Dirty=%s Purge=%s PendingASTs=%s " + "PendingBASTs=%s\n", + (list_empty(&dlm->dirty_list) ? "Empty" : "InUse"), + (list_empty(&dlm->purge_list) ? "Empty" : "InUse"), + (list_empty(&dlm->pending_asts) ? "Empty" : "InUse"), + (list_empty(&dlm->pending_basts) ? "Empty" : "InUse")); + + /* Purge Count: xxx Refs: xxx */ + out += scnprintf(buf + out, len - out, + "Purge Count: %d Refs: %d\n", dlm->purge_count, + kref_read(&dlm->dlm_refs)); + + /* Dead Node: xxx */ + out += scnprintf(buf + out, len - out, + "Dead Node: %d\n", dlm->reco.dead_node); + + /* What about DLM_RECO_STATE_FINALIZE? */ + if (dlm->reco.state == DLM_RECO_STATE_ACTIVE) + state = "ACTIVE"; + else + state = "INACTIVE"; + + /* Recovery Pid: xxxx Master: xxx State: xxxx */ + out += scnprintf(buf + out, len - out, + "Recovery Pid: %d Master: %d State: %s\n", + task_pid_nr(dlm->dlm_reco_thread_task), + dlm->reco.new_master, state); + + /* Recovery Map: xx xx */ + out += scnprintf(buf + out, len - out, "Recovery Map: "); + out += stringify_nodemap(dlm->recovery_map, O2NM_MAX_NODES, + buf + out, len - out); + out += scnprintf(buf + out, len - out, "\n"); + + /* Recovery Node State: */ + out += scnprintf(buf + out, len - out, "Recovery Node State:\n"); + list_for_each_entry(node, &dlm->reco.node_data, list) { + switch (node->state) { + case DLM_RECO_NODE_DATA_INIT: + state = "INIT"; + break; + case DLM_RECO_NODE_DATA_REQUESTING: + state = "REQUESTING"; + break; + case DLM_RECO_NODE_DATA_DEAD: + state = "DEAD"; + break; + case DLM_RECO_NODE_DATA_RECEIVING: + state = "RECEIVING"; + break; + case DLM_RECO_NODE_DATA_REQUESTED: + state = "REQUESTED"; + break; + case DLM_RECO_NODE_DATA_DONE: + state = "DONE"; + break; + case DLM_RECO_NODE_DATA_FINALIZE_SENT: + state = "FINALIZE-SENT"; + break; + default: + state = "BAD"; + break; + } + out += scnprintf(buf + out, len - out, "\t%u - %s\n", + node->node_num, state); + } + + spin_unlock(&dlm->spinlock); + + return out; +} + +static int debug_state_open(struct inode *inode, struct file *file) +{ + struct dlm_ctxt *dlm = inode->i_private; + char *buf = NULL; + + buf = kzalloc(PAGE_SIZE, GFP_NOFS); + if (!buf) + return -ENOMEM; + + i_size_write(inode, debug_state_print(dlm, buf, PAGE_SIZE - 1)); + + file->private_data = buf; + + return 0; +} + +static const struct file_operations debug_state_fops = { + .open = debug_state_open, + .release = debug_release, + .read = debug_read, + .llseek = generic_file_llseek, +}; +/* end - debug state funcs */ + +/* files in subroot */ +void dlm_debug_init(struct dlm_ctxt *dlm) +{ + /* for dumping dlm_ctxt */ + debugfs_create_file(DLM_DEBUGFS_DLM_STATE, S_IFREG|S_IRUSR, + dlm->dlm_debugfs_subroot, dlm, &debug_state_fops); + + /* for dumping lockres */ + debugfs_create_file(DLM_DEBUGFS_LOCKING_STATE, S_IFREG|S_IRUSR, + dlm->dlm_debugfs_subroot, dlm, &debug_lockres_fops); + + /* for dumping mles */ + debugfs_create_file(DLM_DEBUGFS_MLE_STATE, S_IFREG|S_IRUSR, + dlm->dlm_debugfs_subroot, dlm, &debug_mle_fops); + + /* for dumping lockres on the purge list */ + debugfs_create_file(DLM_DEBUGFS_PURGE_LIST, S_IFREG|S_IRUSR, + dlm->dlm_debugfs_subroot, dlm, + &debug_purgelist_fops); +} + +/* subroot - domain dir */ +void dlm_create_debugfs_subroot(struct dlm_ctxt *dlm) +{ + dlm->dlm_debugfs_subroot = debugfs_create_dir(dlm->name, + dlm_debugfs_root); +} + +void dlm_destroy_debugfs_subroot(struct dlm_ctxt *dlm) +{ + debugfs_remove_recursive(dlm->dlm_debugfs_subroot); +} + +/* debugfs root */ +void dlm_create_debugfs_root(void) +{ + dlm_debugfs_root = debugfs_create_dir(DLM_DEBUGFS_DIR, NULL); +} + +void dlm_destroy_debugfs_root(void) +{ + debugfs_remove(dlm_debugfs_root); +} +#endif /* CONFIG_DEBUG_FS */ diff --git a/fs/ocfs2/dlm/dlmdebug.h b/fs/ocfs2/dlm/dlmdebug.h new file mode 100644 index 000000000..e08f7357e --- /dev/null +++ b/fs/ocfs2/dlm/dlmdebug.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * dlmdebug.h + * + * Copyright (C) 2008 Oracle. All rights reserved. + */ + +#ifndef DLMDEBUG_H +#define DLMDEBUG_H + +void dlm_print_one_mle(struct dlm_master_list_entry *mle); + +#ifdef CONFIG_DEBUG_FS + +struct debug_lockres { + int dl_len; + char *dl_buf; + struct dlm_ctxt *dl_ctxt; + struct dlm_lock_resource *dl_res; +}; + +void dlm_debug_init(struct dlm_ctxt *dlm); + +void dlm_create_debugfs_subroot(struct dlm_ctxt *dlm); +void dlm_destroy_debugfs_subroot(struct dlm_ctxt *dlm); + +void dlm_create_debugfs_root(void); +void dlm_destroy_debugfs_root(void); + +#else + +static inline void dlm_debug_init(struct dlm_ctxt *dlm) +{ +} +static inline void dlm_create_debugfs_subroot(struct dlm_ctxt *dlm) +{ +} +static inline void dlm_destroy_debugfs_subroot(struct dlm_ctxt *dlm) +{ +} +static inline void dlm_create_debugfs_root(void) +{ +} +static inline void dlm_destroy_debugfs_root(void) +{ +} + +#endif /* CONFIG_DEBUG_FS */ +#endif /* DLMDEBUG_H */ diff --git a/fs/ocfs2/dlm/dlmdomain.c b/fs/ocfs2/dlm/dlmdomain.c new file mode 100644 index 000000000..97bb9400e --- /dev/null +++ b/fs/ocfs2/dlm/dlmdomain.c @@ -0,0 +1,2371 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * dlmdomain.c + * + * defines domain join / leave apis + * + * Copyright (C) 2004 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../cluster/heartbeat.h" +#include "../cluster/nodemanager.h" +#include "../cluster/tcp.h" + +#include "dlmapi.h" +#include "dlmcommon.h" +#include "dlmdomain.h" +#include "dlmdebug.h" + +#define MLOG_MASK_PREFIX (ML_DLM|ML_DLM_DOMAIN) +#include "../cluster/masklog.h" + +/* + * ocfs2 node maps are array of long int, which limits to send them freely + * across the wire due to endianness issues. To workaround this, we convert + * long ints to byte arrays. Following 3 routines are helper functions to + * set/test/copy bits within those array of bytes + */ +static inline void byte_set_bit(u8 nr, u8 map[]) +{ + map[nr >> 3] |= (1UL << (nr & 7)); +} + +static inline int byte_test_bit(u8 nr, u8 map[]) +{ + return ((1UL << (nr & 7)) & (map[nr >> 3])) != 0; +} + +static inline void byte_copymap(u8 dmap[], unsigned long smap[], + unsigned int sz) +{ + unsigned int nn; + + if (!sz) + return; + + memset(dmap, 0, ((sz + 7) >> 3)); + for (nn = 0 ; nn < sz; nn++) + if (test_bit(nn, smap)) + byte_set_bit(nn, dmap); +} + +static void dlm_free_pagevec(void **vec, int pages) +{ + while (pages--) + kfree(vec[pages]); + kfree(vec); +} + +static void **dlm_alloc_pagevec(int pages) +{ + void **vec = kmalloc_array(pages, sizeof(void *), GFP_KERNEL); + int i; + + if (!vec) + return NULL; + + for (i = 0; i < pages; i++) { + vec[i] = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!vec[i]) + goto out_free; + } + + mlog(0, "Allocated DLM hash pagevec; %d pages (%lu expected), %lu buckets per page\n", + pages, (unsigned long)DLM_HASH_PAGES, + (unsigned long)DLM_BUCKETS_PER_PAGE); + return vec; +out_free: + dlm_free_pagevec(vec, i); + return NULL; +} + +/* + * + * spinlock lock ordering: if multiple locks are needed, obey this ordering: + * dlm_domain_lock + * struct dlm_ctxt->spinlock + * struct dlm_lock_resource->spinlock + * struct dlm_ctxt->master_lock + * struct dlm_ctxt->ast_lock + * dlm_master_list_entry->spinlock + * dlm_lock->spinlock + * + */ + +DEFINE_SPINLOCK(dlm_domain_lock); +LIST_HEAD(dlm_domains); +static DECLARE_WAIT_QUEUE_HEAD(dlm_domain_events); + +/* + * The supported protocol version for DLM communication. Running domains + * will have a negotiated version with the same major number and a minor + * number equal or smaller. The dlm_ctxt->dlm_locking_proto field should + * be used to determine what a running domain is actually using. + * + * New in version 1.1: + * - Message DLM_QUERY_REGION added to support global heartbeat + * - Message DLM_QUERY_NODEINFO added to allow online node removes + * New in version 1.2: + * - Message DLM_BEGIN_EXIT_DOMAIN_MSG added to mark start of exit domain + * New in version 1.3: + * - Message DLM_DEREF_LOCKRES_DONE added to inform non-master that the + * refmap is cleared + */ +static const struct dlm_protocol_version dlm_protocol = { + .pv_major = 1, + .pv_minor = 3, +}; + +#define DLM_DOMAIN_BACKOFF_MS 200 + +static int dlm_query_join_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data); +static int dlm_assert_joined_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data); +static int dlm_cancel_join_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data); +static int dlm_query_region_handler(struct o2net_msg *msg, u32 len, + void *data, void **ret_data); +static int dlm_exit_domain_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data); +static int dlm_protocol_compare(struct dlm_protocol_version *existing, + struct dlm_protocol_version *request); + +static void dlm_unregister_domain_handlers(struct dlm_ctxt *dlm); + +void __dlm_unhash_lockres(struct dlm_ctxt *dlm, struct dlm_lock_resource *res) +{ + if (hlist_unhashed(&res->hash_node)) + return; + + mlog(0, "%s: Unhash res %.*s\n", dlm->name, res->lockname.len, + res->lockname.name); + hlist_del_init(&res->hash_node); + dlm_lockres_put(res); +} + +void __dlm_insert_lockres(struct dlm_ctxt *dlm, struct dlm_lock_resource *res) +{ + struct hlist_head *bucket; + + assert_spin_locked(&dlm->spinlock); + + bucket = dlm_lockres_hash(dlm, res->lockname.hash); + + /* get a reference for our hashtable */ + dlm_lockres_get(res); + + hlist_add_head(&res->hash_node, bucket); + + mlog(0, "%s: Hash res %.*s\n", dlm->name, res->lockname.len, + res->lockname.name); +} + +struct dlm_lock_resource * __dlm_lookup_lockres_full(struct dlm_ctxt *dlm, + const char *name, + unsigned int len, + unsigned int hash) +{ + struct hlist_head *bucket; + struct dlm_lock_resource *res; + + mlog(0, "%.*s\n", len, name); + + assert_spin_locked(&dlm->spinlock); + + bucket = dlm_lockres_hash(dlm, hash); + + hlist_for_each_entry(res, bucket, hash_node) { + if (res->lockname.name[0] != name[0]) + continue; + if (unlikely(res->lockname.len != len)) + continue; + if (memcmp(res->lockname.name + 1, name + 1, len - 1)) + continue; + dlm_lockres_get(res); + return res; + } + return NULL; +} + +/* intended to be called by functions which do not care about lock + * resources which are being purged (most net _handler functions). + * this will return NULL for any lock resource which is found but + * currently in the process of dropping its mastery reference. + * use __dlm_lookup_lockres_full when you need the lock resource + * regardless (e.g. dlm_get_lock_resource) */ +struct dlm_lock_resource * __dlm_lookup_lockres(struct dlm_ctxt *dlm, + const char *name, + unsigned int len, + unsigned int hash) +{ + struct dlm_lock_resource *res = NULL; + + mlog(0, "%.*s\n", len, name); + + assert_spin_locked(&dlm->spinlock); + + res = __dlm_lookup_lockres_full(dlm, name, len, hash); + if (res) { + spin_lock(&res->spinlock); + if (res->state & DLM_LOCK_RES_DROPPING_REF) { + spin_unlock(&res->spinlock); + dlm_lockres_put(res); + return NULL; + } + spin_unlock(&res->spinlock); + } + + return res; +} + +struct dlm_lock_resource * dlm_lookup_lockres(struct dlm_ctxt *dlm, + const char *name, + unsigned int len) +{ + struct dlm_lock_resource *res; + unsigned int hash = dlm_lockid_hash(name, len); + + spin_lock(&dlm->spinlock); + res = __dlm_lookup_lockres(dlm, name, len, hash); + spin_unlock(&dlm->spinlock); + return res; +} + +static struct dlm_ctxt * __dlm_lookup_domain_full(const char *domain, int len) +{ + struct dlm_ctxt *tmp; + + assert_spin_locked(&dlm_domain_lock); + + /* tmp->name here is always NULL terminated, + * but domain may not be! */ + list_for_each_entry(tmp, &dlm_domains, list) { + if (strlen(tmp->name) == len && + memcmp(tmp->name, domain, len)==0) + return tmp; + } + + return NULL; +} + +/* For null terminated domain strings ONLY */ +static struct dlm_ctxt * __dlm_lookup_domain(const char *domain) +{ + assert_spin_locked(&dlm_domain_lock); + + return __dlm_lookup_domain_full(domain, strlen(domain)); +} + + +/* returns true on one of two conditions: + * 1) the domain does not exist + * 2) the domain exists and it's state is "joined" */ +static int dlm_wait_on_domain_helper(const char *domain) +{ + int ret = 0; + struct dlm_ctxt *tmp = NULL; + + spin_lock(&dlm_domain_lock); + + tmp = __dlm_lookup_domain(domain); + if (!tmp) + ret = 1; + else if (tmp->dlm_state == DLM_CTXT_JOINED) + ret = 1; + + spin_unlock(&dlm_domain_lock); + return ret; +} + +static void dlm_free_ctxt_mem(struct dlm_ctxt *dlm) +{ + dlm_destroy_debugfs_subroot(dlm); + + if (dlm->lockres_hash) + dlm_free_pagevec((void **)dlm->lockres_hash, DLM_HASH_PAGES); + + if (dlm->master_hash) + dlm_free_pagevec((void **)dlm->master_hash, DLM_HASH_PAGES); + + kfree(dlm->name); + kfree(dlm); +} + +/* A little strange - this function will be called while holding + * dlm_domain_lock and is expected to be holding it on the way out. We + * will however drop and reacquire it multiple times */ +static void dlm_ctxt_release(struct kref *kref) +{ + struct dlm_ctxt *dlm; + + dlm = container_of(kref, struct dlm_ctxt, dlm_refs); + + BUG_ON(dlm->num_joins); + BUG_ON(dlm->dlm_state == DLM_CTXT_JOINED); + + /* we may still be in the list if we hit an error during join. */ + list_del_init(&dlm->list); + + spin_unlock(&dlm_domain_lock); + + mlog(0, "freeing memory from domain %s\n", dlm->name); + + wake_up(&dlm_domain_events); + + dlm_free_ctxt_mem(dlm); + + spin_lock(&dlm_domain_lock); +} + +void dlm_put(struct dlm_ctxt *dlm) +{ + spin_lock(&dlm_domain_lock); + kref_put(&dlm->dlm_refs, dlm_ctxt_release); + spin_unlock(&dlm_domain_lock); +} + +static void __dlm_get(struct dlm_ctxt *dlm) +{ + kref_get(&dlm->dlm_refs); +} + +/* given a questionable reference to a dlm object, gets a reference if + * it can find it in the list, otherwise returns NULL in which case + * you shouldn't trust your pointer. */ +struct dlm_ctxt *dlm_grab(struct dlm_ctxt *dlm) +{ + struct dlm_ctxt *target; + struct dlm_ctxt *ret = NULL; + + spin_lock(&dlm_domain_lock); + + list_for_each_entry(target, &dlm_domains, list) { + if (target == dlm) { + __dlm_get(target); + ret = target; + break; + } + } + + spin_unlock(&dlm_domain_lock); + + return ret; +} + +int dlm_domain_fully_joined(struct dlm_ctxt *dlm) +{ + int ret; + + spin_lock(&dlm_domain_lock); + ret = (dlm->dlm_state == DLM_CTXT_JOINED) || + (dlm->dlm_state == DLM_CTXT_IN_SHUTDOWN); + spin_unlock(&dlm_domain_lock); + + return ret; +} + +static void dlm_destroy_dlm_worker(struct dlm_ctxt *dlm) +{ + if (dlm->dlm_worker) { + destroy_workqueue(dlm->dlm_worker); + dlm->dlm_worker = NULL; + } +} + +static void dlm_complete_dlm_shutdown(struct dlm_ctxt *dlm) +{ + dlm_unregister_domain_handlers(dlm); + dlm_complete_thread(dlm); + dlm_complete_recovery_thread(dlm); + dlm_destroy_dlm_worker(dlm); + + /* We've left the domain. Now we can take ourselves out of the + * list and allow the kref stuff to help us free the + * memory. */ + spin_lock(&dlm_domain_lock); + list_del_init(&dlm->list); + spin_unlock(&dlm_domain_lock); + + /* Wake up anyone waiting for us to remove this domain */ + wake_up(&dlm_domain_events); +} + +static int dlm_migrate_all_locks(struct dlm_ctxt *dlm) +{ + int i, num, n, ret = 0; + struct dlm_lock_resource *res; + struct hlist_node *iter; + struct hlist_head *bucket; + int dropped; + + mlog(0, "Migrating locks from domain %s\n", dlm->name); + + num = 0; + spin_lock(&dlm->spinlock); + for (i = 0; i < DLM_HASH_BUCKETS; i++) { +redo_bucket: + n = 0; + bucket = dlm_lockres_hash(dlm, i); + iter = bucket->first; + while (iter) { + n++; + res = hlist_entry(iter, struct dlm_lock_resource, + hash_node); + dlm_lockres_get(res); + /* migrate, if necessary. this will drop the dlm + * spinlock and retake it if it does migration. */ + dropped = dlm_empty_lockres(dlm, res); + + spin_lock(&res->spinlock); + if (dropped) + __dlm_lockres_calc_usage(dlm, res); + else + iter = res->hash_node.next; + spin_unlock(&res->spinlock); + + dlm_lockres_put(res); + + if (dropped) { + cond_resched_lock(&dlm->spinlock); + goto redo_bucket; + } + } + cond_resched_lock(&dlm->spinlock); + num += n; + } + + if (!num) { + if (dlm->reco.state & DLM_RECO_STATE_ACTIVE) { + mlog(0, "%s: perhaps there are more lock resources " + "need to be migrated after dlm recovery\n", dlm->name); + ret = -EAGAIN; + } else { + mlog(0, "%s: we won't do dlm recovery after migrating " + "all lock resources\n", dlm->name); + dlm->migrate_done = 1; + } + } + + spin_unlock(&dlm->spinlock); + wake_up(&dlm->dlm_thread_wq); + + /* let the dlm thread take care of purging, keep scanning until + * nothing remains in the hash */ + if (num) { + mlog(0, "%s: %d lock resources in hash last pass\n", + dlm->name, num); + ret = -EAGAIN; + } + mlog(0, "DONE Migrating locks from domain %s\n", dlm->name); + return ret; +} + +static int dlm_no_joining_node(struct dlm_ctxt *dlm) +{ + int ret; + + spin_lock(&dlm->spinlock); + ret = dlm->joining_node == DLM_LOCK_RES_OWNER_UNKNOWN; + spin_unlock(&dlm->spinlock); + + return ret; +} + +static int dlm_begin_exit_domain_handler(struct o2net_msg *msg, u32 len, + void *data, void **ret_data) +{ + struct dlm_ctxt *dlm = data; + unsigned int node; + struct dlm_exit_domain *exit_msg = (struct dlm_exit_domain *) msg->buf; + + if (!dlm_grab(dlm)) + return 0; + + node = exit_msg->node_idx; + mlog(0, "%s: Node %u sent a begin exit domain message\n", dlm->name, node); + + spin_lock(&dlm->spinlock); + set_bit(node, dlm->exit_domain_map); + spin_unlock(&dlm->spinlock); + + dlm_put(dlm); + + return 0; +} + +static void dlm_mark_domain_leaving(struct dlm_ctxt *dlm) +{ + /* Yikes, a double spinlock! I need domain_lock for the dlm + * state and the dlm spinlock for join state... Sorry! */ +again: + spin_lock(&dlm_domain_lock); + spin_lock(&dlm->spinlock); + + if (dlm->joining_node != DLM_LOCK_RES_OWNER_UNKNOWN) { + mlog(0, "Node %d is joining, we wait on it.\n", + dlm->joining_node); + spin_unlock(&dlm->spinlock); + spin_unlock(&dlm_domain_lock); + + wait_event(dlm->dlm_join_events, dlm_no_joining_node(dlm)); + goto again; + } + + dlm->dlm_state = DLM_CTXT_LEAVING; + spin_unlock(&dlm->spinlock); + spin_unlock(&dlm_domain_lock); +} + +static void __dlm_print_nodes(struct dlm_ctxt *dlm) +{ + int node = -1, num = 0; + + assert_spin_locked(&dlm->spinlock); + + printk("( "); + while ((node = find_next_bit(dlm->domain_map, O2NM_MAX_NODES, + node + 1)) < O2NM_MAX_NODES) { + printk("%d ", node); + ++num; + } + printk(") %u nodes\n", num); +} + +static int dlm_exit_domain_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data) +{ + struct dlm_ctxt *dlm = data; + unsigned int node; + struct dlm_exit_domain *exit_msg = (struct dlm_exit_domain *) msg->buf; + + mlog(0, "%p %u %p", msg, len, data); + + if (!dlm_grab(dlm)) + return 0; + + node = exit_msg->node_idx; + + spin_lock(&dlm->spinlock); + clear_bit(node, dlm->domain_map); + clear_bit(node, dlm->exit_domain_map); + printk(KERN_NOTICE "o2dlm: Node %u leaves domain %s ", node, dlm->name); + __dlm_print_nodes(dlm); + + /* notify anything attached to the heartbeat events */ + dlm_hb_event_notify_attached(dlm, node, 0); + + spin_unlock(&dlm->spinlock); + + dlm_put(dlm); + + return 0; +} + +static int dlm_send_one_domain_exit(struct dlm_ctxt *dlm, u32 msg_type, + unsigned int node) +{ + int status; + struct dlm_exit_domain leave_msg; + + mlog(0, "%s: Sending domain exit message %u to node %u\n", dlm->name, + msg_type, node); + + memset(&leave_msg, 0, sizeof(leave_msg)); + leave_msg.node_idx = dlm->node_num; + + status = o2net_send_message(msg_type, dlm->key, &leave_msg, + sizeof(leave_msg), node, NULL); + if (status < 0) + mlog(ML_ERROR, "Error %d sending domain exit message %u " + "to node %u on domain %s\n", status, msg_type, node, + dlm->name); + + return status; +} + +static void dlm_begin_exit_domain(struct dlm_ctxt *dlm) +{ + int node = -1; + + /* Support for begin exit domain was added in 1.2 */ + if (dlm->dlm_locking_proto.pv_major == 1 && + dlm->dlm_locking_proto.pv_minor < 2) + return; + + /* + * Unlike DLM_EXIT_DOMAIN_MSG, DLM_BEGIN_EXIT_DOMAIN_MSG is purely + * informational. Meaning if a node does not receive the message, + * so be it. + */ + spin_lock(&dlm->spinlock); + while (1) { + node = find_next_bit(dlm->domain_map, O2NM_MAX_NODES, node + 1); + if (node >= O2NM_MAX_NODES) + break; + if (node == dlm->node_num) + continue; + + spin_unlock(&dlm->spinlock); + dlm_send_one_domain_exit(dlm, DLM_BEGIN_EXIT_DOMAIN_MSG, node); + spin_lock(&dlm->spinlock); + } + spin_unlock(&dlm->spinlock); +} + +static void dlm_leave_domain(struct dlm_ctxt *dlm) +{ + int node, clear_node, status; + + /* At this point we've migrated away all our locks and won't + * accept mastership of new ones. The dlm is responsible for + * almost nothing now. We make sure not to confuse any joining + * nodes and then commence shutdown procedure. */ + + spin_lock(&dlm->spinlock); + /* Clear ourselves from the domain map */ + clear_bit(dlm->node_num, dlm->domain_map); + while ((node = find_next_bit(dlm->domain_map, O2NM_MAX_NODES, + 0)) < O2NM_MAX_NODES) { + /* Drop the dlm spinlock. This is safe wrt the domain_map. + * -nodes cannot be added now as the + * query_join_handlers knows to respond with OK_NO_MAP + * -we catch the right network errors if a node is + * removed from the map while we're sending him the + * exit message. */ + spin_unlock(&dlm->spinlock); + + clear_node = 1; + + status = dlm_send_one_domain_exit(dlm, DLM_EXIT_DOMAIN_MSG, + node); + if (status < 0 && + status != -ENOPROTOOPT && + status != -ENOTCONN) { + mlog(ML_NOTICE, "Error %d sending domain exit message " + "to node %d\n", status, node); + + /* Not sure what to do here but lets sleep for + * a bit in case this was a transient + * error... */ + msleep(DLM_DOMAIN_BACKOFF_MS); + clear_node = 0; + } + + spin_lock(&dlm->spinlock); + /* If we're not clearing the node bit then we intend + * to loop back around to try again. */ + if (clear_node) + clear_bit(node, dlm->domain_map); + } + spin_unlock(&dlm->spinlock); +} + +void dlm_unregister_domain(struct dlm_ctxt *dlm) +{ + int leave = 0; + struct dlm_lock_resource *res; + + spin_lock(&dlm_domain_lock); + BUG_ON(dlm->dlm_state != DLM_CTXT_JOINED); + BUG_ON(!dlm->num_joins); + + dlm->num_joins--; + if (!dlm->num_joins) { + /* We mark it "in shutdown" now so new register + * requests wait until we've completely left the + * domain. Don't use DLM_CTXT_LEAVING yet as we still + * want new domain joins to communicate with us at + * least until we've completed migration of our + * resources. */ + dlm->dlm_state = DLM_CTXT_IN_SHUTDOWN; + leave = 1; + } + spin_unlock(&dlm_domain_lock); + + if (leave) { + mlog(0, "shutting down domain %s\n", dlm->name); + dlm_begin_exit_domain(dlm); + + /* We changed dlm state, notify the thread */ + dlm_kick_thread(dlm, NULL); + + while (dlm_migrate_all_locks(dlm)) { + /* Give dlm_thread time to purge the lockres' */ + msleep(500); + mlog(0, "%s: more migration to do\n", dlm->name); + } + + /* This list should be empty. If not, print remaining lockres */ + if (!list_empty(&dlm->tracking_list)) { + mlog(ML_ERROR, "Following lockres' are still on the " + "tracking list:\n"); + list_for_each_entry(res, &dlm->tracking_list, tracking) + dlm_print_one_lock_resource(res); + } + + dlm_mark_domain_leaving(dlm); + dlm_leave_domain(dlm); + printk(KERN_NOTICE "o2dlm: Leaving domain %s\n", dlm->name); + dlm_force_free_mles(dlm); + dlm_complete_dlm_shutdown(dlm); + } + dlm_put(dlm); +} +EXPORT_SYMBOL_GPL(dlm_unregister_domain); + +static int dlm_query_join_proto_check(char *proto_type, int node, + struct dlm_protocol_version *ours, + struct dlm_protocol_version *request) +{ + int rc; + struct dlm_protocol_version proto = *request; + + if (!dlm_protocol_compare(ours, &proto)) { + mlog(0, + "node %u wanted to join with %s locking protocol " + "%u.%u, we respond with %u.%u\n", + node, proto_type, + request->pv_major, + request->pv_minor, + proto.pv_major, proto.pv_minor); + request->pv_minor = proto.pv_minor; + rc = 0; + } else { + mlog(ML_NOTICE, + "Node %u wanted to join with %s locking " + "protocol %u.%u, but we have %u.%u, disallowing\n", + node, proto_type, + request->pv_major, + request->pv_minor, + ours->pv_major, + ours->pv_minor); + rc = 1; + } + + return rc; +} + +/* + * struct dlm_query_join_packet is made up of four one-byte fields. They + * are effectively in big-endian order already. However, little-endian + * machines swap them before putting the packet on the wire (because + * query_join's response is a status, and that status is treated as a u32 + * on the wire). Thus, a big-endian and little-endian machines will treat + * this structure differently. + * + * The solution is to have little-endian machines swap the structure when + * converting from the structure to the u32 representation. This will + * result in the structure having the correct format on the wire no matter + * the host endian format. + */ +static void dlm_query_join_packet_to_wire(struct dlm_query_join_packet *packet, + u32 *wire) +{ + union dlm_query_join_response response; + + response.packet = *packet; + *wire = be32_to_cpu(response.intval); +} + +static void dlm_query_join_wire_to_packet(u32 wire, + struct dlm_query_join_packet *packet) +{ + union dlm_query_join_response response; + + response.intval = cpu_to_be32(wire); + *packet = response.packet; +} + +static int dlm_query_join_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data) +{ + struct dlm_query_join_request *query; + struct dlm_query_join_packet packet = { + .code = JOIN_DISALLOW, + }; + struct dlm_ctxt *dlm = NULL; + u32 response; + u8 nodenum; + + query = (struct dlm_query_join_request *) msg->buf; + + mlog(0, "node %u wants to join domain %s\n", query->node_idx, + query->domain); + + /* + * If heartbeat doesn't consider the node live, tell it + * to back off and try again. This gives heartbeat a chance + * to catch up. + */ + if (!o2hb_check_node_heartbeating_no_sem(query->node_idx)) { + mlog(0, "node %u is not in our live map yet\n", + query->node_idx); + + packet.code = JOIN_DISALLOW; + goto respond; + } + + packet.code = JOIN_OK_NO_MAP; + + spin_lock(&dlm_domain_lock); + dlm = __dlm_lookup_domain_full(query->domain, query->name_len); + if (!dlm) + goto unlock_respond; + + /* + * There is a small window where the joining node may not see the + * node(s) that just left but still part of the cluster. DISALLOW + * join request if joining node has different node map. + */ + nodenum=0; + while (nodenum < O2NM_MAX_NODES) { + if (test_bit(nodenum, dlm->domain_map)) { + if (!byte_test_bit(nodenum, query->node_map)) { + mlog(0, "disallow join as node %u does not " + "have node %u in its nodemap\n", + query->node_idx, nodenum); + packet.code = JOIN_DISALLOW; + goto unlock_respond; + } + } + nodenum++; + } + + /* Once the dlm ctxt is marked as leaving then we don't want + * to be put in someone's domain map. + * Also, explicitly disallow joining at certain troublesome + * times (ie. during recovery). */ + if (dlm->dlm_state != DLM_CTXT_LEAVING) { + int bit = query->node_idx; + spin_lock(&dlm->spinlock); + + if (dlm->dlm_state == DLM_CTXT_NEW && + dlm->joining_node == DLM_LOCK_RES_OWNER_UNKNOWN) { + /*If this is a brand new context and we + * haven't started our join process yet, then + * the other node won the race. */ + packet.code = JOIN_OK_NO_MAP; + } else if (dlm->joining_node != DLM_LOCK_RES_OWNER_UNKNOWN) { + /* Disallow parallel joins. */ + packet.code = JOIN_DISALLOW; + } else if (dlm->reco.state & DLM_RECO_STATE_ACTIVE) { + mlog(0, "node %u trying to join, but recovery " + "is ongoing.\n", bit); + packet.code = JOIN_DISALLOW; + } else if (test_bit(bit, dlm->recovery_map)) { + mlog(0, "node %u trying to join, but it " + "still needs recovery.\n", bit); + packet.code = JOIN_DISALLOW; + } else if (test_bit(bit, dlm->domain_map)) { + mlog(0, "node %u trying to join, but it " + "is still in the domain! needs recovery?\n", + bit); + packet.code = JOIN_DISALLOW; + } else { + /* Alright we're fully a part of this domain + * so we keep some state as to who's joining + * and indicate to him that needs to be fixed + * up. */ + + /* Make sure we speak compatible locking protocols. */ + if (dlm_query_join_proto_check("DLM", bit, + &dlm->dlm_locking_proto, + &query->dlm_proto)) { + packet.code = JOIN_PROTOCOL_MISMATCH; + } else if (dlm_query_join_proto_check("fs", bit, + &dlm->fs_locking_proto, + &query->fs_proto)) { + packet.code = JOIN_PROTOCOL_MISMATCH; + } else { + packet.dlm_minor = query->dlm_proto.pv_minor; + packet.fs_minor = query->fs_proto.pv_minor; + packet.code = JOIN_OK; + __dlm_set_joining_node(dlm, query->node_idx); + } + } + + spin_unlock(&dlm->spinlock); + } +unlock_respond: + spin_unlock(&dlm_domain_lock); + +respond: + mlog(0, "We respond with %u\n", packet.code); + + dlm_query_join_packet_to_wire(&packet, &response); + return response; +} + +static int dlm_assert_joined_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data) +{ + struct dlm_assert_joined *assert; + struct dlm_ctxt *dlm = NULL; + + assert = (struct dlm_assert_joined *) msg->buf; + + mlog(0, "node %u asserts join on domain %s\n", assert->node_idx, + assert->domain); + + spin_lock(&dlm_domain_lock); + dlm = __dlm_lookup_domain_full(assert->domain, assert->name_len); + /* XXX should we consider no dlm ctxt an error? */ + if (dlm) { + spin_lock(&dlm->spinlock); + + /* Alright, this node has officially joined our + * domain. Set him in the map and clean up our + * leftover join state. */ + BUG_ON(dlm->joining_node != assert->node_idx); + + if (dlm->reco.state & DLM_RECO_STATE_ACTIVE) { + mlog(0, "dlm recovery is ongoing, disallow join\n"); + spin_unlock(&dlm->spinlock); + spin_unlock(&dlm_domain_lock); + return -EAGAIN; + } + + set_bit(assert->node_idx, dlm->domain_map); + clear_bit(assert->node_idx, dlm->exit_domain_map); + __dlm_set_joining_node(dlm, DLM_LOCK_RES_OWNER_UNKNOWN); + + printk(KERN_NOTICE "o2dlm: Node %u joins domain %s ", + assert->node_idx, dlm->name); + __dlm_print_nodes(dlm); + + /* notify anything attached to the heartbeat events */ + dlm_hb_event_notify_attached(dlm, assert->node_idx, 1); + + spin_unlock(&dlm->spinlock); + } + spin_unlock(&dlm_domain_lock); + + return 0; +} + +static int dlm_match_regions(struct dlm_ctxt *dlm, + struct dlm_query_region *qr, + char *local, int locallen) +{ + char *remote = qr->qr_regions; + char *l, *r; + int localnr, i, j, foundit; + int status = 0; + + if (!o2hb_global_heartbeat_active()) { + if (qr->qr_numregions) { + mlog(ML_ERROR, "Domain %s: Joining node %d has global " + "heartbeat enabled but local node %d does not\n", + qr->qr_domain, qr->qr_node, dlm->node_num); + status = -EINVAL; + } + goto bail; + } + + if (o2hb_global_heartbeat_active() && !qr->qr_numregions) { + mlog(ML_ERROR, "Domain %s: Local node %d has global " + "heartbeat enabled but joining node %d does not\n", + qr->qr_domain, dlm->node_num, qr->qr_node); + status = -EINVAL; + goto bail; + } + + if (qr->qr_numregions > O2NM_MAX_REGIONS) { + mlog(ML_ERROR, "Domain %s: Joining node %d has invalid " + "number of heartbeat regions %u\n", + qr->qr_domain, qr->qr_node, qr->qr_numregions); + status = -EINVAL; + goto bail; + } + + r = remote; + for (i = 0; i < qr->qr_numregions; ++i) { + mlog(0, "Region %.*s\n", O2HB_MAX_REGION_NAME_LEN, r); + r += O2HB_MAX_REGION_NAME_LEN; + } + + localnr = min(O2NM_MAX_REGIONS, locallen/O2HB_MAX_REGION_NAME_LEN); + localnr = o2hb_get_all_regions(local, (u8)localnr); + + /* compare local regions with remote */ + l = local; + for (i = 0; i < localnr; ++i) { + foundit = 0; + r = remote; + for (j = 0; j < qr->qr_numregions; ++j) { + if (!memcmp(l, r, O2HB_MAX_REGION_NAME_LEN)) { + foundit = 1; + break; + } + r += O2HB_MAX_REGION_NAME_LEN; + } + if (!foundit) { + status = -EINVAL; + mlog(ML_ERROR, "Domain %s: Region '%.*s' registered " + "in local node %d but not in joining node %d\n", + qr->qr_domain, O2HB_MAX_REGION_NAME_LEN, l, + dlm->node_num, qr->qr_node); + goto bail; + } + l += O2HB_MAX_REGION_NAME_LEN; + } + + /* compare remote with local regions */ + r = remote; + for (i = 0; i < qr->qr_numregions; ++i) { + foundit = 0; + l = local; + for (j = 0; j < localnr; ++j) { + if (!memcmp(r, l, O2HB_MAX_REGION_NAME_LEN)) { + foundit = 1; + break; + } + l += O2HB_MAX_REGION_NAME_LEN; + } + if (!foundit) { + status = -EINVAL; + mlog(ML_ERROR, "Domain %s: Region '%.*s' registered " + "in joining node %d but not in local node %d\n", + qr->qr_domain, O2HB_MAX_REGION_NAME_LEN, r, + qr->qr_node, dlm->node_num); + goto bail; + } + r += O2HB_MAX_REGION_NAME_LEN; + } + +bail: + return status; +} + +static int dlm_send_regions(struct dlm_ctxt *dlm, unsigned long *node_map) +{ + struct dlm_query_region *qr = NULL; + int status, ret = 0, i; + char *p; + + if (find_first_bit(node_map, O2NM_MAX_NODES) >= O2NM_MAX_NODES) + goto bail; + + qr = kzalloc_obj(struct dlm_query_region); + if (!qr) { + ret = -ENOMEM; + mlog_errno(ret); + goto bail; + } + + qr->qr_node = dlm->node_num; + qr->qr_namelen = strlen(dlm->name); + memcpy(qr->qr_domain, dlm->name, qr->qr_namelen); + /* if local hb, the numregions will be zero */ + if (o2hb_global_heartbeat_active()) + qr->qr_numregions = o2hb_get_all_regions(qr->qr_regions, + O2NM_MAX_REGIONS); + + p = qr->qr_regions; + for (i = 0; i < qr->qr_numregions; ++i, p += O2HB_MAX_REGION_NAME_LEN) + mlog(0, "Region %.*s\n", O2HB_MAX_REGION_NAME_LEN, p); + + i = -1; + while ((i = find_next_bit(node_map, O2NM_MAX_NODES, + i + 1)) < O2NM_MAX_NODES) { + if (i == dlm->node_num) + continue; + + mlog(0, "Sending regions to node %d\n", i); + + ret = o2net_send_message(DLM_QUERY_REGION, DLM_MOD_KEY, qr, + sizeof(struct dlm_query_region), + i, &status); + if (ret >= 0) + ret = status; + if (ret) { + mlog(ML_ERROR, "Region mismatch %d, node %d\n", + ret, i); + break; + } + } + +bail: + kfree(qr); + return ret; +} + +static int dlm_query_region_handler(struct o2net_msg *msg, u32 len, + void *data, void **ret_data) +{ + struct dlm_query_region *qr; + struct dlm_ctxt *dlm = NULL; + char *local = NULL; + int status = 0; + + qr = (struct dlm_query_region *) msg->buf; + + mlog(0, "Node %u queries hb regions on domain %s\n", qr->qr_node, + qr->qr_domain); + + /* buffer used in dlm_match_regions() */ + local = kmalloc(sizeof(qr->qr_regions), GFP_KERNEL); + if (!local) + return -ENOMEM; + + status = -EINVAL; + + spin_lock(&dlm_domain_lock); + dlm = __dlm_lookup_domain_full(qr->qr_domain, qr->qr_namelen); + if (!dlm) { + mlog(ML_ERROR, "Node %d queried hb regions on domain %s " + "before join domain\n", qr->qr_node, qr->qr_domain); + goto out_domain_lock; + } + + spin_lock(&dlm->spinlock); + if (dlm->joining_node != qr->qr_node) { + mlog(ML_ERROR, "Node %d queried hb regions on domain %s " + "but joining node is %d\n", qr->qr_node, qr->qr_domain, + dlm->joining_node); + goto out_dlm_lock; + } + + /* Support for global heartbeat was added in 1.1 */ + if (dlm->dlm_locking_proto.pv_major == 1 && + dlm->dlm_locking_proto.pv_minor == 0) { + mlog(ML_ERROR, "Node %d queried hb regions on domain %s " + "but active dlm protocol is %d.%d\n", qr->qr_node, + qr->qr_domain, dlm->dlm_locking_proto.pv_major, + dlm->dlm_locking_proto.pv_minor); + goto out_dlm_lock; + } + + status = dlm_match_regions(dlm, qr, local, sizeof(qr->qr_regions)); + +out_dlm_lock: + spin_unlock(&dlm->spinlock); + +out_domain_lock: + spin_unlock(&dlm_domain_lock); + + kfree(local); + + return status; +} + +static int dlm_match_nodes(struct dlm_ctxt *dlm, struct dlm_query_nodeinfo *qn) +{ + struct o2nm_node *local; + struct dlm_node_info *remote; + int i, j; + int status = 0; + + for (j = 0; j < qn->qn_numnodes; ++j) + mlog(0, "Node %3d, %pI4:%u\n", qn->qn_nodes[j].ni_nodenum, + &(qn->qn_nodes[j].ni_ipv4_address), + ntohs(qn->qn_nodes[j].ni_ipv4_port)); + + for (i = 0; i < O2NM_MAX_NODES && !status; ++i) { + local = o2nm_get_node_by_num(i); + remote = NULL; + for (j = 0; j < qn->qn_numnodes; ++j) { + if (qn->qn_nodes[j].ni_nodenum == i) { + remote = &(qn->qn_nodes[j]); + break; + } + } + + if (!local && !remote) + continue; + + if ((local && !remote) || (!local && remote)) + status = -EINVAL; + + if (!status && + ((remote->ni_nodenum != local->nd_num) || + (remote->ni_ipv4_port != local->nd_ipv4_port) || + (remote->ni_ipv4_address != local->nd_ipv4_address))) + status = -EINVAL; + + if (status) { + if (remote && !local) + mlog(ML_ERROR, "Domain %s: Node %d (%pI4:%u) " + "registered in joining node %d but not in " + "local node %d\n", qn->qn_domain, + remote->ni_nodenum, + &(remote->ni_ipv4_address), + ntohs(remote->ni_ipv4_port), + qn->qn_nodenum, dlm->node_num); + if (local && !remote) + mlog(ML_ERROR, "Domain %s: Node %d (%pI4:%u) " + "registered in local node %d but not in " + "joining node %d\n", qn->qn_domain, + local->nd_num, &(local->nd_ipv4_address), + ntohs(local->nd_ipv4_port), + dlm->node_num, qn->qn_nodenum); + BUG_ON((!local && !remote)); + } + + if (local) + o2nm_node_put(local); + } + + return status; +} + +static int dlm_send_nodeinfo(struct dlm_ctxt *dlm, unsigned long *node_map) +{ + struct dlm_query_nodeinfo *qn = NULL; + struct o2nm_node *node; + int ret = 0, status, count, i; + + if (find_first_bit(node_map, O2NM_MAX_NODES) >= O2NM_MAX_NODES) + goto bail; + + qn = kzalloc_obj(struct dlm_query_nodeinfo); + if (!qn) { + ret = -ENOMEM; + mlog_errno(ret); + goto bail; + } + + for (i = 0, count = 0; i < O2NM_MAX_NODES; ++i) { + node = o2nm_get_node_by_num(i); + if (!node) + continue; + qn->qn_nodes[count].ni_nodenum = node->nd_num; + qn->qn_nodes[count].ni_ipv4_port = node->nd_ipv4_port; + qn->qn_nodes[count].ni_ipv4_address = node->nd_ipv4_address; + mlog(0, "Node %3d, %pI4:%u\n", node->nd_num, + &(node->nd_ipv4_address), ntohs(node->nd_ipv4_port)); + ++count; + o2nm_node_put(node); + } + + qn->qn_nodenum = dlm->node_num; + qn->qn_numnodes = count; + qn->qn_namelen = strlen(dlm->name); + memcpy(qn->qn_domain, dlm->name, qn->qn_namelen); + + i = -1; + while ((i = find_next_bit(node_map, O2NM_MAX_NODES, + i + 1)) < O2NM_MAX_NODES) { + if (i == dlm->node_num) + continue; + + mlog(0, "Sending nodeinfo to node %d\n", i); + + ret = o2net_send_message(DLM_QUERY_NODEINFO, DLM_MOD_KEY, + qn, sizeof(struct dlm_query_nodeinfo), + i, &status); + if (ret >= 0) + ret = status; + if (ret) { + mlog(ML_ERROR, "node mismatch %d, node %d\n", ret, i); + break; + } + } + +bail: + kfree(qn); + return ret; +} + +static int dlm_query_nodeinfo_handler(struct o2net_msg *msg, u32 len, + void *data, void **ret_data) +{ + struct dlm_query_nodeinfo *qn; + struct dlm_ctxt *dlm = NULL; + int status = -EINVAL; + + qn = (struct dlm_query_nodeinfo *) msg->buf; + + mlog(0, "Node %u queries nodes on domain %s\n", qn->qn_nodenum, + qn->qn_domain); + + spin_lock(&dlm_domain_lock); + dlm = __dlm_lookup_domain_full(qn->qn_domain, qn->qn_namelen); + if (!dlm) { + mlog(ML_ERROR, "Node %d queried nodes on domain %s before " + "join domain\n", qn->qn_nodenum, qn->qn_domain); + goto bail; + } + + spin_lock(&dlm->spinlock); + if (dlm->joining_node != qn->qn_nodenum) { + mlog(ML_ERROR, "Node %d queried nodes on domain %s but " + "joining node is %d\n", qn->qn_nodenum, qn->qn_domain, + dlm->joining_node); + goto unlock; + } + + /* Support for node query was added in 1.1 */ + if (dlm->dlm_locking_proto.pv_major == 1 && + dlm->dlm_locking_proto.pv_minor == 0) { + mlog(ML_ERROR, "Node %d queried nodes on domain %s " + "but active dlm protocol is %d.%d\n", qn->qn_nodenum, + qn->qn_domain, dlm->dlm_locking_proto.pv_major, + dlm->dlm_locking_proto.pv_minor); + goto unlock; + } + + status = dlm_match_nodes(dlm, qn); + +unlock: + spin_unlock(&dlm->spinlock); +bail: + spin_unlock(&dlm_domain_lock); + + return status; +} + +static int dlm_cancel_join_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data) +{ + struct dlm_cancel_join *cancel; + struct dlm_ctxt *dlm = NULL; + + cancel = (struct dlm_cancel_join *) msg->buf; + + mlog(0, "node %u cancels join on domain %s\n", cancel->node_idx, + cancel->domain); + + spin_lock(&dlm_domain_lock); + dlm = __dlm_lookup_domain_full(cancel->domain, cancel->name_len); + + if (dlm) { + spin_lock(&dlm->spinlock); + + /* Yikes, this guy wants to cancel his join. No + * problem, we simply cleanup our join state. */ + BUG_ON(dlm->joining_node != cancel->node_idx); + __dlm_set_joining_node(dlm, DLM_LOCK_RES_OWNER_UNKNOWN); + + spin_unlock(&dlm->spinlock); + } + spin_unlock(&dlm_domain_lock); + + return 0; +} + +static int dlm_send_one_join_cancel(struct dlm_ctxt *dlm, + unsigned int node) +{ + int status; + struct dlm_cancel_join cancel_msg; + + memset(&cancel_msg, 0, sizeof(cancel_msg)); + cancel_msg.node_idx = dlm->node_num; + cancel_msg.name_len = strlen(dlm->name); + memcpy(cancel_msg.domain, dlm->name, cancel_msg.name_len); + + status = o2net_send_message(DLM_CANCEL_JOIN_MSG, DLM_MOD_KEY, + &cancel_msg, sizeof(cancel_msg), node, + NULL); + if (status < 0) { + mlog(ML_ERROR, "Error %d when sending message %u (key 0x%x) to " + "node %u\n", status, DLM_CANCEL_JOIN_MSG, DLM_MOD_KEY, + node); + goto bail; + } + +bail: + return status; +} + +/* map_size should be in bytes. */ +static int dlm_send_join_cancels(struct dlm_ctxt *dlm, + unsigned long *node_map, + unsigned int map_size) +{ + int status, tmpstat; + int node; + + if (map_size != (BITS_TO_LONGS(O2NM_MAX_NODES) * + sizeof(unsigned long))) { + mlog(ML_ERROR, + "map_size %u != BITS_TO_LONGS(O2NM_MAX_NODES) %u\n", + map_size, (unsigned)BITS_TO_LONGS(O2NM_MAX_NODES)); + return -EINVAL; + } + + status = 0; + node = -1; + while ((node = find_next_bit(node_map, O2NM_MAX_NODES, + node + 1)) < O2NM_MAX_NODES) { + if (node == dlm->node_num) + continue; + + tmpstat = dlm_send_one_join_cancel(dlm, node); + if (tmpstat) { + mlog(ML_ERROR, "Error return %d cancelling join on " + "node %d\n", tmpstat, node); + if (!status) + status = tmpstat; + } + } + + if (status) + mlog_errno(status); + return status; +} + +static int dlm_request_join(struct dlm_ctxt *dlm, + int node, + enum dlm_query_join_response_code *response) +{ + int status; + struct dlm_query_join_request join_msg; + struct dlm_query_join_packet packet; + u32 join_resp; + + mlog(0, "querying node %d\n", node); + + memset(&join_msg, 0, sizeof(join_msg)); + join_msg.node_idx = dlm->node_num; + join_msg.name_len = strlen(dlm->name); + memcpy(join_msg.domain, dlm->name, join_msg.name_len); + join_msg.dlm_proto = dlm->dlm_locking_proto; + join_msg.fs_proto = dlm->fs_locking_proto; + + /* copy live node map to join message */ + byte_copymap(join_msg.node_map, dlm->live_nodes_map, O2NM_MAX_NODES); + + status = o2net_send_message(DLM_QUERY_JOIN_MSG, DLM_MOD_KEY, &join_msg, + sizeof(join_msg), node, &join_resp); + if (status < 0 && status != -ENOPROTOOPT) { + mlog(ML_ERROR, "Error %d when sending message %u (key 0x%x) to " + "node %u\n", status, DLM_QUERY_JOIN_MSG, DLM_MOD_KEY, + node); + goto bail; + } + dlm_query_join_wire_to_packet(join_resp, &packet); + + /* -ENOPROTOOPT from the net code means the other side isn't + listening for our message type -- that's fine, it means + his dlm isn't up, so we can consider him a 'yes' but not + joined into the domain. */ + if (status == -ENOPROTOOPT) { + status = 0; + *response = JOIN_OK_NO_MAP; + } else { + *response = packet.code; + switch (packet.code) { + case JOIN_DISALLOW: + case JOIN_OK_NO_MAP: + break; + case JOIN_PROTOCOL_MISMATCH: + mlog(ML_NOTICE, + "This node requested DLM locking protocol %u.%u and " + "filesystem locking protocol %u.%u. At least one of " + "the protocol versions on node %d is not compatible, " + "disconnecting\n", + dlm->dlm_locking_proto.pv_major, + dlm->dlm_locking_proto.pv_minor, + dlm->fs_locking_proto.pv_major, + dlm->fs_locking_proto.pv_minor, + node); + status = -EPROTO; + break; + case JOIN_OK: + /* Use the same locking protocol as the remote node */ + dlm->dlm_locking_proto.pv_minor = packet.dlm_minor; + dlm->fs_locking_proto.pv_minor = packet.fs_minor; + mlog(0, + "Node %d responds JOIN_OK with DLM locking protocol " + "%u.%u and fs locking protocol %u.%u\n", + node, + dlm->dlm_locking_proto.pv_major, + dlm->dlm_locking_proto.pv_minor, + dlm->fs_locking_proto.pv_major, + dlm->fs_locking_proto.pv_minor); + break; + default: + status = -EINVAL; + mlog(ML_ERROR, "invalid response %d from node %u\n", + packet.code, node); + /* Reset response to JOIN_DISALLOW */ + *response = JOIN_DISALLOW; + break; + } + } + + mlog(0, "status %d, node %d response is %d\n", status, node, + *response); + +bail: + return status; +} + +static int dlm_send_one_join_assert(struct dlm_ctxt *dlm, + unsigned int node) +{ + int status; + int ret; + struct dlm_assert_joined assert_msg; + + mlog(0, "Sending join assert to node %u\n", node); + + memset(&assert_msg, 0, sizeof(assert_msg)); + assert_msg.node_idx = dlm->node_num; + assert_msg.name_len = strlen(dlm->name); + memcpy(assert_msg.domain, dlm->name, assert_msg.name_len); + + status = o2net_send_message(DLM_ASSERT_JOINED_MSG, DLM_MOD_KEY, + &assert_msg, sizeof(assert_msg), node, + &ret); + if (status < 0) + mlog(ML_ERROR, "Error %d when sending message %u (key 0x%x) to " + "node %u\n", status, DLM_ASSERT_JOINED_MSG, DLM_MOD_KEY, + node); + else + status = ret; + + return status; +} + +static void dlm_send_join_asserts(struct dlm_ctxt *dlm, + unsigned long *node_map) +{ + int status, node, live; + + node = -1; + while ((node = find_next_bit(node_map, O2NM_MAX_NODES, + node + 1)) < O2NM_MAX_NODES) { + if (node == dlm->node_num) + continue; + + do { + /* It is very important that this message be + * received so we spin until either the node + * has died or it gets the message. */ + status = dlm_send_one_join_assert(dlm, node); + + spin_lock(&dlm->spinlock); + live = test_bit(node, dlm->live_nodes_map); + spin_unlock(&dlm->spinlock); + + if (status) { + mlog(ML_ERROR, "Error return %d asserting " + "join on node %d\n", status, node); + + /* give us some time between errors... */ + if (live) + msleep(DLM_DOMAIN_BACKOFF_MS); + } + } while (status && live); + } +} + +struct domain_join_ctxt { + unsigned long live_map[BITS_TO_LONGS(O2NM_MAX_NODES)]; + unsigned long yes_resp_map[BITS_TO_LONGS(O2NM_MAX_NODES)]; +}; + +static int dlm_should_restart_join(struct dlm_ctxt *dlm, + struct domain_join_ctxt *ctxt, + enum dlm_query_join_response_code response) +{ + int ret; + + if (response == JOIN_DISALLOW) { + mlog(0, "Latest response of disallow -- should restart\n"); + return 1; + } + + spin_lock(&dlm->spinlock); + /* For now, we restart the process if the node maps have + * changed at all */ + ret = !bitmap_equal(ctxt->live_map, dlm->live_nodes_map, + O2NM_MAX_NODES); + spin_unlock(&dlm->spinlock); + + if (ret) + mlog(0, "Node maps changed -- should restart\n"); + + return ret; +} + +static int dlm_try_to_join_domain(struct dlm_ctxt *dlm) +{ + int status = 0, tmpstat, node; + struct domain_join_ctxt *ctxt; + enum dlm_query_join_response_code response = JOIN_DISALLOW; + + mlog(0, "%p", dlm); + + ctxt = kzalloc_obj(*ctxt); + if (!ctxt) { + status = -ENOMEM; + mlog_errno(status); + goto bail; + } + + /* group sem locking should work for us here -- we're already + * registered for heartbeat events so filling this should be + * atomic wrt getting those handlers called. */ + o2hb_fill_node_map(dlm->live_nodes_map, O2NM_MAX_NODES); + + spin_lock(&dlm->spinlock); + bitmap_copy(ctxt->live_map, dlm->live_nodes_map, O2NM_MAX_NODES); + __dlm_set_joining_node(dlm, dlm->node_num); + spin_unlock(&dlm->spinlock); + + node = -1; + while ((node = find_next_bit(ctxt->live_map, O2NM_MAX_NODES, + node + 1)) < O2NM_MAX_NODES) { + if (node == dlm->node_num) + continue; + + status = dlm_request_join(dlm, node, &response); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + /* Ok, either we got a response or the node doesn't have a + * dlm up. */ + if (response == JOIN_OK) + set_bit(node, ctxt->yes_resp_map); + + if (dlm_should_restart_join(dlm, ctxt, response)) { + status = -EAGAIN; + goto bail; + } + } + + mlog(0, "Yay, done querying nodes!\n"); + + /* Yay, everyone agree's we can join the domain. My domain is + * comprised of all nodes who were put in the + * yes_resp_map. Copy that into our domain map and send a join + * assert message to clean up everyone elses state. */ + spin_lock(&dlm->spinlock); + bitmap_copy(dlm->domain_map, ctxt->yes_resp_map, O2NM_MAX_NODES); + set_bit(dlm->node_num, dlm->domain_map); + spin_unlock(&dlm->spinlock); + + /* Support for global heartbeat and node info was added in 1.1 */ + if (dlm->dlm_locking_proto.pv_major > 1 || + dlm->dlm_locking_proto.pv_minor > 0) { + status = dlm_send_nodeinfo(dlm, ctxt->yes_resp_map); + if (status) { + mlog_errno(status); + goto bail; + } + status = dlm_send_regions(dlm, ctxt->yes_resp_map); + if (status) { + mlog_errno(status); + goto bail; + } + } + + dlm_send_join_asserts(dlm, ctxt->yes_resp_map); + + /* Joined state *must* be set before the joining node + * information, otherwise the query_join handler may read no + * current joiner but a state of NEW and tell joining nodes + * we're not in the domain. */ + spin_lock(&dlm_domain_lock); + dlm->dlm_state = DLM_CTXT_JOINED; + dlm->num_joins++; + spin_unlock(&dlm_domain_lock); + +bail: + spin_lock(&dlm->spinlock); + __dlm_set_joining_node(dlm, DLM_LOCK_RES_OWNER_UNKNOWN); + if (!status) { + printk(KERN_NOTICE "o2dlm: Joining domain %s ", dlm->name); + __dlm_print_nodes(dlm); + } + spin_unlock(&dlm->spinlock); + + if (ctxt) { + /* Do we need to send a cancel message to any nodes? */ + if (status < 0) { + tmpstat = dlm_send_join_cancels(dlm, + ctxt->yes_resp_map, + sizeof(ctxt->yes_resp_map)); + if (tmpstat < 0) + mlog_errno(tmpstat); + } + kfree(ctxt); + } + + mlog(0, "returning %d\n", status); + return status; +} + +static void dlm_unregister_domain_handlers(struct dlm_ctxt *dlm) +{ + o2hb_unregister_callback(dlm->name, &dlm->dlm_hb_up); + o2hb_unregister_callback(dlm->name, &dlm->dlm_hb_down); + o2net_unregister_handler_list(&dlm->dlm_domain_handlers); +} + +static int dlm_register_domain_handlers(struct dlm_ctxt *dlm) +{ + int status; + + mlog(0, "registering handlers.\n"); + + o2hb_setup_callback(&dlm->dlm_hb_down, O2HB_NODE_DOWN_CB, + dlm_hb_node_down_cb, dlm, DLM_HB_NODE_DOWN_PRI); + o2hb_setup_callback(&dlm->dlm_hb_up, O2HB_NODE_UP_CB, + dlm_hb_node_up_cb, dlm, DLM_HB_NODE_UP_PRI); + + status = o2hb_register_callback(dlm->name, &dlm->dlm_hb_down); + if (status) + goto bail; + + status = o2hb_register_callback(dlm->name, &dlm->dlm_hb_up); + if (status) + goto bail; + + status = o2net_register_handler(DLM_MASTER_REQUEST_MSG, dlm->key, + sizeof(struct dlm_master_request), + dlm_master_request_handler, + dlm, NULL, &dlm->dlm_domain_handlers); + if (status) + goto bail; + + status = o2net_register_handler(DLM_ASSERT_MASTER_MSG, dlm->key, + sizeof(struct dlm_assert_master), + dlm_assert_master_handler, + dlm, dlm_assert_master_post_handler, + &dlm->dlm_domain_handlers); + if (status) + goto bail; + + status = o2net_register_handler(DLM_CREATE_LOCK_MSG, dlm->key, + sizeof(struct dlm_create_lock), + dlm_create_lock_handler, + dlm, NULL, &dlm->dlm_domain_handlers); + if (status) + goto bail; + + status = o2net_register_handler(DLM_CONVERT_LOCK_MSG, dlm->key, + DLM_CONVERT_LOCK_MAX_LEN, + dlm_convert_lock_handler, + dlm, NULL, &dlm->dlm_domain_handlers); + if (status) + goto bail; + + status = o2net_register_handler(DLM_UNLOCK_LOCK_MSG, dlm->key, + DLM_UNLOCK_LOCK_MAX_LEN, + dlm_unlock_lock_handler, + dlm, NULL, &dlm->dlm_domain_handlers); + if (status) + goto bail; + + status = o2net_register_handler(DLM_PROXY_AST_MSG, dlm->key, + DLM_PROXY_AST_MAX_LEN, + dlm_proxy_ast_handler, + dlm, NULL, &dlm->dlm_domain_handlers); + if (status) + goto bail; + + status = o2net_register_handler(DLM_EXIT_DOMAIN_MSG, dlm->key, + sizeof(struct dlm_exit_domain), + dlm_exit_domain_handler, + dlm, NULL, &dlm->dlm_domain_handlers); + if (status) + goto bail; + + status = o2net_register_handler(DLM_DEREF_LOCKRES_MSG, dlm->key, + sizeof(struct dlm_deref_lockres), + dlm_deref_lockres_handler, + dlm, NULL, &dlm->dlm_domain_handlers); + if (status) + goto bail; + + status = o2net_register_handler(DLM_MIGRATE_REQUEST_MSG, dlm->key, + sizeof(struct dlm_migrate_request), + dlm_migrate_request_handler, + dlm, NULL, &dlm->dlm_domain_handlers); + if (status) + goto bail; + + status = o2net_register_handler(DLM_MIG_LOCKRES_MSG, dlm->key, + DLM_MIG_LOCKRES_MAX_LEN, + dlm_mig_lockres_handler, + dlm, NULL, &dlm->dlm_domain_handlers); + if (status) + goto bail; + + status = o2net_register_handler(DLM_MASTER_REQUERY_MSG, dlm->key, + sizeof(struct dlm_master_requery), + dlm_master_requery_handler, + dlm, NULL, &dlm->dlm_domain_handlers); + if (status) + goto bail; + + status = o2net_register_handler(DLM_LOCK_REQUEST_MSG, dlm->key, + sizeof(struct dlm_lock_request), + dlm_request_all_locks_handler, + dlm, NULL, &dlm->dlm_domain_handlers); + if (status) + goto bail; + + status = o2net_register_handler(DLM_RECO_DATA_DONE_MSG, dlm->key, + sizeof(struct dlm_reco_data_done), + dlm_reco_data_done_handler, + dlm, NULL, &dlm->dlm_domain_handlers); + if (status) + goto bail; + + status = o2net_register_handler(DLM_BEGIN_RECO_MSG, dlm->key, + sizeof(struct dlm_begin_reco), + dlm_begin_reco_handler, + dlm, NULL, &dlm->dlm_domain_handlers); + if (status) + goto bail; + + status = o2net_register_handler(DLM_FINALIZE_RECO_MSG, dlm->key, + sizeof(struct dlm_finalize_reco), + dlm_finalize_reco_handler, + dlm, NULL, &dlm->dlm_domain_handlers); + if (status) + goto bail; + + status = o2net_register_handler(DLM_BEGIN_EXIT_DOMAIN_MSG, dlm->key, + sizeof(struct dlm_exit_domain), + dlm_begin_exit_domain_handler, + dlm, NULL, &dlm->dlm_domain_handlers); + if (status) + goto bail; + + status = o2net_register_handler(DLM_DEREF_LOCKRES_DONE, dlm->key, + sizeof(struct dlm_deref_lockres_done), + dlm_deref_lockres_done_handler, + dlm, NULL, &dlm->dlm_domain_handlers); +bail: + if (status) + dlm_unregister_domain_handlers(dlm); + + return status; +} + +static int dlm_join_domain(struct dlm_ctxt *dlm) +{ + int status; + unsigned int backoff; + unsigned int total_backoff = 0; + char wq_name[O2NM_MAX_NAME_LEN]; + + BUG_ON(!dlm); + + mlog(0, "Join domain %s\n", dlm->name); + + status = dlm_register_domain_handlers(dlm); + if (status) { + mlog_errno(status); + goto bail; + } + + status = dlm_launch_thread(dlm); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + status = dlm_launch_recovery_thread(dlm); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + dlm_debug_init(dlm); + + snprintf(wq_name, O2NM_MAX_NAME_LEN, "dlm_wq-%s", dlm->name); + dlm->dlm_worker = alloc_workqueue(wq_name, WQ_MEM_RECLAIM | WQ_PERCPU, + 0); + if (!dlm->dlm_worker) { + status = -ENOMEM; + mlog_errno(status); + goto bail; + } + + do { + status = dlm_try_to_join_domain(dlm); + + /* If we're racing another node to the join, then we + * need to back off temporarily and let them + * complete. */ +#define DLM_JOIN_TIMEOUT_MSECS 90000 + if (status == -EAGAIN) { + if (signal_pending(current)) { + status = -ERESTARTSYS; + goto bail; + } + + if (total_backoff > DLM_JOIN_TIMEOUT_MSECS) { + status = -ERESTARTSYS; + mlog(ML_NOTICE, "Timed out joining dlm domain " + "%s after %u msecs\n", dlm->name, + total_backoff); + goto bail; + } + + /* + * After you! + * No, after you! + * I insist! + * But you first! + * ... + */ + backoff = (unsigned int)(jiffies & 0x3); + backoff *= DLM_DOMAIN_BACKOFF_MS; + total_backoff += backoff; + mlog(0, "backoff %d\n", backoff); + msleep(backoff); + } + } while (status == -EAGAIN); + + if (status < 0) { + mlog_errno(status); + goto bail; + } + + status = 0; +bail: + wake_up(&dlm_domain_events); + + if (status) { + dlm_unregister_domain_handlers(dlm); + dlm_complete_thread(dlm); + dlm_complete_recovery_thread(dlm); + dlm_destroy_dlm_worker(dlm); + } + + return status; +} + +static struct dlm_ctxt *dlm_alloc_ctxt(const char *domain, + u32 key) +{ + int i; + int ret; + struct dlm_ctxt *dlm = NULL; + + dlm = kzalloc_obj(*dlm); + if (!dlm) { + ret = -ENOMEM; + mlog_errno(ret); + goto leave; + } + + dlm->name = kstrdup(domain, GFP_KERNEL); + if (dlm->name == NULL) { + ret = -ENOMEM; + mlog_errno(ret); + goto leave; + } + + dlm->lockres_hash = (struct hlist_head **)dlm_alloc_pagevec(DLM_HASH_PAGES); + if (!dlm->lockres_hash) { + ret = -ENOMEM; + mlog_errno(ret); + goto leave; + } + + for (i = 0; i < DLM_HASH_BUCKETS; i++) + INIT_HLIST_HEAD(dlm_lockres_hash(dlm, i)); + + dlm->master_hash = (struct hlist_head **) + dlm_alloc_pagevec(DLM_HASH_PAGES); + if (!dlm->master_hash) { + ret = -ENOMEM; + mlog_errno(ret); + goto leave; + } + + for (i = 0; i < DLM_HASH_BUCKETS; i++) + INIT_HLIST_HEAD(dlm_master_hash(dlm, i)); + + dlm->key = key; + dlm->node_num = o2nm_this_node(); + + dlm_create_debugfs_subroot(dlm); + + spin_lock_init(&dlm->spinlock); + spin_lock_init(&dlm->master_lock); + spin_lock_init(&dlm->ast_lock); + spin_lock_init(&dlm->track_lock); + INIT_LIST_HEAD(&dlm->list); + INIT_LIST_HEAD(&dlm->dirty_list); + INIT_LIST_HEAD(&dlm->reco.resources); + INIT_LIST_HEAD(&dlm->reco.node_data); + INIT_LIST_HEAD(&dlm->purge_list); + INIT_LIST_HEAD(&dlm->dlm_domain_handlers); + INIT_LIST_HEAD(&dlm->tracking_list); + dlm->reco.state = 0; + + INIT_LIST_HEAD(&dlm->pending_asts); + INIT_LIST_HEAD(&dlm->pending_basts); + + mlog(0, "dlm->recovery_map=%p, &(dlm->recovery_map[0])=%p\n", + dlm->recovery_map, &(dlm->recovery_map[0])); + + bitmap_zero(dlm->recovery_map, O2NM_MAX_NODES); + bitmap_zero(dlm->live_nodes_map, O2NM_MAX_NODES); + bitmap_zero(dlm->domain_map, O2NM_MAX_NODES); + + dlm->dlm_thread_task = NULL; + dlm->dlm_reco_thread_task = NULL; + dlm->dlm_worker = NULL; + init_waitqueue_head(&dlm->dlm_thread_wq); + init_waitqueue_head(&dlm->dlm_reco_thread_wq); + init_waitqueue_head(&dlm->reco.event); + init_waitqueue_head(&dlm->ast_wq); + init_waitqueue_head(&dlm->migration_wq); + INIT_LIST_HEAD(&dlm->mle_hb_events); + + dlm->joining_node = DLM_LOCK_RES_OWNER_UNKNOWN; + init_waitqueue_head(&dlm->dlm_join_events); + + dlm->migrate_done = 0; + + dlm->reco.new_master = O2NM_INVALID_NODE_NUM; + dlm->reco.dead_node = O2NM_INVALID_NODE_NUM; + + atomic_set(&dlm->res_tot_count, 0); + atomic_set(&dlm->res_cur_count, 0); + for (i = 0; i < DLM_MLE_NUM_TYPES; ++i) { + atomic_set(&dlm->mle_tot_count[i], 0); + atomic_set(&dlm->mle_cur_count[i], 0); + } + + spin_lock_init(&dlm->work_lock); + INIT_LIST_HEAD(&dlm->work_list); + INIT_WORK(&dlm->dispatched_work, dlm_dispatch_work); + + kref_init(&dlm->dlm_refs); + dlm->dlm_state = DLM_CTXT_NEW; + + INIT_LIST_HEAD(&dlm->dlm_eviction_callbacks); + + mlog(0, "context init: refcount %u\n", + kref_read(&dlm->dlm_refs)); + + ret = 0; +leave: + if (ret < 0 && dlm) { + if (dlm->master_hash) + dlm_free_pagevec((void **)dlm->master_hash, + DLM_HASH_PAGES); + + if (dlm->lockres_hash) + dlm_free_pagevec((void **)dlm->lockres_hash, + DLM_HASH_PAGES); + + kfree(dlm->name); + kfree(dlm); + dlm = NULL; + } + return dlm; +} + +/* + * Compare a requested locking protocol version against the current one. + * + * If the major numbers are different, they are incompatible. + * If the current minor is greater than the request, they are incompatible. + * If the current minor is less than or equal to the request, they are + * compatible, and the requester should run at the current minor version. + */ +static int dlm_protocol_compare(struct dlm_protocol_version *existing, + struct dlm_protocol_version *request) +{ + if (existing->pv_major != request->pv_major) + return 1; + + if (existing->pv_minor > request->pv_minor) + return 1; + + if (existing->pv_minor < request->pv_minor) + request->pv_minor = existing->pv_minor; + + return 0; +} + +/* + * dlm_register_domain: one-time setup per "domain". + * + * The filesystem passes in the requested locking version via proto. + * If registration was successful, proto will contain the negotiated + * locking protocol. + */ +struct dlm_ctxt * dlm_register_domain(const char *domain, + u32 key, + struct dlm_protocol_version *fs_proto) +{ + int ret; + struct dlm_ctxt *dlm = NULL; + struct dlm_ctxt *new_ctxt = NULL; + + if (strlen(domain) >= O2NM_MAX_NAME_LEN) { + ret = -ENAMETOOLONG; + mlog(ML_ERROR, "domain name length too long\n"); + goto leave; + } + + mlog(0, "register called for domain \"%s\"\n", domain); + +retry: + dlm = NULL; + if (signal_pending(current)) { + ret = -ERESTARTSYS; + mlog_errno(ret); + goto leave; + } + + spin_lock(&dlm_domain_lock); + + dlm = __dlm_lookup_domain(domain); + if (dlm) { + if (dlm->dlm_state != DLM_CTXT_JOINED) { + spin_unlock(&dlm_domain_lock); + + mlog(0, "This ctxt is not joined yet!\n"); + wait_event_interruptible(dlm_domain_events, + dlm_wait_on_domain_helper( + domain)); + goto retry; + } + + if (dlm_protocol_compare(&dlm->fs_locking_proto, fs_proto)) { + spin_unlock(&dlm_domain_lock); + mlog(ML_ERROR, + "Requested locking protocol version is not " + "compatible with already registered domain " + "\"%s\"\n", domain); + ret = -EPROTO; + goto leave; + } + + __dlm_get(dlm); + dlm->num_joins++; + + spin_unlock(&dlm_domain_lock); + + ret = 0; + goto leave; + } + + /* doesn't exist */ + if (!new_ctxt) { + spin_unlock(&dlm_domain_lock); + + new_ctxt = dlm_alloc_ctxt(domain, key); + if (new_ctxt) + goto retry; + + ret = -ENOMEM; + mlog_errno(ret); + goto leave; + } + + /* a little variable switch-a-roo here... */ + dlm = new_ctxt; + new_ctxt = NULL; + + /* add the new domain */ + list_add_tail(&dlm->list, &dlm_domains); + spin_unlock(&dlm_domain_lock); + + /* + * Pass the locking protocol version into the join. If the join + * succeeds, it will have the negotiated protocol set. + */ + dlm->dlm_locking_proto = dlm_protocol; + dlm->fs_locking_proto = *fs_proto; + + ret = dlm_join_domain(dlm); + if (ret) { + mlog_errno(ret); + dlm_put(dlm); + goto leave; + } + + /* Tell the caller what locking protocol we negotiated */ + *fs_proto = dlm->fs_locking_proto; + + ret = 0; +leave: + if (new_ctxt) + dlm_free_ctxt_mem(new_ctxt); + + if (ret < 0) + dlm = ERR_PTR(ret); + + return dlm; +} +EXPORT_SYMBOL_GPL(dlm_register_domain); + +static LIST_HEAD(dlm_join_handlers); + +static void dlm_unregister_net_handlers(void) +{ + o2net_unregister_handler_list(&dlm_join_handlers); +} + +static int dlm_register_net_handlers(void) +{ + int status = 0; + + status = o2net_register_handler(DLM_QUERY_JOIN_MSG, DLM_MOD_KEY, + sizeof(struct dlm_query_join_request), + dlm_query_join_handler, + NULL, NULL, &dlm_join_handlers); + if (status) + goto bail; + + status = o2net_register_handler(DLM_ASSERT_JOINED_MSG, DLM_MOD_KEY, + sizeof(struct dlm_assert_joined), + dlm_assert_joined_handler, + NULL, NULL, &dlm_join_handlers); + if (status) + goto bail; + + status = o2net_register_handler(DLM_CANCEL_JOIN_MSG, DLM_MOD_KEY, + sizeof(struct dlm_cancel_join), + dlm_cancel_join_handler, + NULL, NULL, &dlm_join_handlers); + if (status) + goto bail; + + status = o2net_register_handler(DLM_QUERY_REGION, DLM_MOD_KEY, + sizeof(struct dlm_query_region), + dlm_query_region_handler, + NULL, NULL, &dlm_join_handlers); + + if (status) + goto bail; + + status = o2net_register_handler(DLM_QUERY_NODEINFO, DLM_MOD_KEY, + sizeof(struct dlm_query_nodeinfo), + dlm_query_nodeinfo_handler, + NULL, NULL, &dlm_join_handlers); +bail: + if (status < 0) + dlm_unregister_net_handlers(); + + return status; +} + +/* Domain eviction callback handling. + * + * The file system requires notification of node death *before* the + * dlm completes it's recovery work, otherwise it may be able to + * acquire locks on resources requiring recovery. Since the dlm can + * evict a node from it's domain *before* heartbeat fires, a similar + * mechanism is required. */ + +/* Eviction is not expected to happen often, so a per-domain lock is + * not necessary. Eviction callbacks are allowed to sleep for short + * periods of time. */ +static DECLARE_RWSEM(dlm_callback_sem); + +void dlm_fire_domain_eviction_callbacks(struct dlm_ctxt *dlm, + int node_num) +{ + struct dlm_eviction_cb *cb; + + down_read(&dlm_callback_sem); + list_for_each_entry(cb, &dlm->dlm_eviction_callbacks, ec_item) { + cb->ec_func(node_num, cb->ec_data); + } + up_read(&dlm_callback_sem); +} + +void dlm_setup_eviction_cb(struct dlm_eviction_cb *cb, + dlm_eviction_func *f, + void *data) +{ + INIT_LIST_HEAD(&cb->ec_item); + cb->ec_func = f; + cb->ec_data = data; +} +EXPORT_SYMBOL_GPL(dlm_setup_eviction_cb); + +void dlm_register_eviction_cb(struct dlm_ctxt *dlm, + struct dlm_eviction_cb *cb) +{ + down_write(&dlm_callback_sem); + list_add_tail(&cb->ec_item, &dlm->dlm_eviction_callbacks); + up_write(&dlm_callback_sem); +} +EXPORT_SYMBOL_GPL(dlm_register_eviction_cb); + +void dlm_unregister_eviction_cb(struct dlm_eviction_cb *cb) +{ + down_write(&dlm_callback_sem); + list_del_init(&cb->ec_item); + up_write(&dlm_callback_sem); +} +EXPORT_SYMBOL_GPL(dlm_unregister_eviction_cb); + +static int __init dlm_init(void) +{ + int status; + + status = dlm_init_mle_cache(); + if (status) { + mlog(ML_ERROR, "Could not create o2dlm_mle slabcache\n"); + goto error; + } + + status = dlm_init_master_caches(); + if (status) { + mlog(ML_ERROR, "Could not create o2dlm_lockres and " + "o2dlm_lockname slabcaches\n"); + goto error; + } + + status = dlm_init_lock_cache(); + if (status) { + mlog(ML_ERROR, "Count not create o2dlm_lock slabcache\n"); + goto error; + } + + status = dlm_register_net_handlers(); + if (status) { + mlog(ML_ERROR, "Unable to register network handlers\n"); + goto error; + } + + dlm_create_debugfs_root(); + + return 0; +error: + dlm_unregister_net_handlers(); + dlm_destroy_lock_cache(); + dlm_destroy_master_caches(); + dlm_destroy_mle_cache(); + return -1; +} + +static void __exit dlm_exit (void) +{ + dlm_destroy_debugfs_root(); + dlm_unregister_net_handlers(); + dlm_destroy_lock_cache(); + dlm_destroy_master_caches(); + dlm_destroy_mle_cache(); +} + +MODULE_AUTHOR("Oracle"); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("OCFS2 Distributed Lock Management"); + +module_init(dlm_init); +module_exit(dlm_exit); diff --git a/fs/ocfs2/dlm/dlmdomain.h b/fs/ocfs2/dlm/dlmdomain.h new file mode 100644 index 000000000..815abe30a --- /dev/null +++ b/fs/ocfs2/dlm/dlmdomain.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * dlmdomain.h + * + * Copyright (C) 2004 Oracle. All rights reserved. + */ + +#ifndef DLMDOMAIN_H +#define DLMDOMAIN_H + +extern spinlock_t dlm_domain_lock; +extern struct list_head dlm_domains; + +static inline int dlm_joined(struct dlm_ctxt *dlm) +{ + int ret = 0; + + spin_lock(&dlm_domain_lock); + if (dlm->dlm_state == DLM_CTXT_JOINED) + ret = 1; + spin_unlock(&dlm_domain_lock); + + return ret; +} + +static inline int dlm_shutting_down(struct dlm_ctxt *dlm) +{ + int ret = 0; + + spin_lock(&dlm_domain_lock); + if (dlm->dlm_state == DLM_CTXT_IN_SHUTDOWN) + ret = 1; + spin_unlock(&dlm_domain_lock); + + return ret; +} + +void dlm_fire_domain_eviction_callbacks(struct dlm_ctxt *dlm, + int node_num); + +#endif diff --git a/fs/ocfs2/dlm/dlmlock.c b/fs/ocfs2/dlm/dlmlock.c new file mode 100644 index 000000000..46120eb75 --- /dev/null +++ b/fs/ocfs2/dlm/dlmlock.c @@ -0,0 +1,743 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * dlmlock.c + * + * underlying calls for lock creation + * + * Copyright (C) 2004 Oracle. All rights reserved. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#include "../cluster/heartbeat.h" +#include "../cluster/nodemanager.h" +#include "../cluster/tcp.h" + +#include "dlmapi.h" +#include "dlmcommon.h" + +#include "dlmconvert.h" + +#define MLOG_MASK_PREFIX ML_DLM +#include "../cluster/masklog.h" + +static struct kmem_cache *dlm_lock_cache; + +static DEFINE_SPINLOCK(dlm_cookie_lock); +static u64 dlm_next_cookie = 1; + +static enum dlm_status dlm_send_remote_lock_request(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock, int flags); +static void dlm_init_lock(struct dlm_lock *newlock, int type, + u8 node, u64 cookie); +static void dlm_lock_release(struct kref *kref); +static void dlm_lock_detach_lockres(struct dlm_lock *lock); + +int dlm_init_lock_cache(void) +{ + dlm_lock_cache = kmem_cache_create("o2dlm_lock", + sizeof(struct dlm_lock), + 0, SLAB_HWCACHE_ALIGN, NULL); + if (dlm_lock_cache == NULL) + return -ENOMEM; + return 0; +} + +void dlm_destroy_lock_cache(void) +{ + kmem_cache_destroy(dlm_lock_cache); +} + +/* Tell us whether we can grant a new lock request. + * locking: + * caller needs: res->spinlock + * taken: none + * held on exit: none + * returns: 1 if the lock can be granted, 0 otherwise. + */ +static int dlm_can_grant_new_lock(struct dlm_lock_resource *res, + struct dlm_lock *lock) +{ + struct dlm_lock *tmplock; + + list_for_each_entry(tmplock, &res->granted, list) { + if (!dlm_lock_compatible(tmplock->ml.type, lock->ml.type)) + return 0; + } + + list_for_each_entry(tmplock, &res->converting, list) { + if (!dlm_lock_compatible(tmplock->ml.type, lock->ml.type)) + return 0; + if (!dlm_lock_compatible(tmplock->ml.convert_type, + lock->ml.type)) + return 0; + } + + return 1; +} + +/* performs lock creation at the lockres master site + * locking: + * caller needs: none + * taken: takes and drops res->spinlock + * held on exit: none + * returns: DLM_NORMAL, DLM_NOTQUEUED + */ +static enum dlm_status dlmlock_master(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock, int flags) +{ + int call_ast = 0, kick_thread = 0; + enum dlm_status status = DLM_NORMAL; + + mlog(0, "type=%d\n", lock->ml.type); + + spin_lock(&res->spinlock); + /* if called from dlm_create_lock_handler, need to + * ensure it will not sleep in dlm_wait_on_lockres */ + status = __dlm_lockres_state_to_status(res); + if (status != DLM_NORMAL && + lock->ml.node != dlm->node_num) { + /* erf. state changed after lock was dropped. */ + spin_unlock(&res->spinlock); + dlm_error(status); + return status; + } + __dlm_wait_on_lockres(res); + __dlm_lockres_reserve_ast(res); + + if (dlm_can_grant_new_lock(res, lock)) { + mlog(0, "I can grant this lock right away\n"); + /* got it right away */ + lock->lksb->status = DLM_NORMAL; + status = DLM_NORMAL; + dlm_lock_get(lock); + list_add_tail(&lock->list, &res->granted); + + /* for the recovery lock, we can't allow the ast + * to be queued since the dlmthread is already + * frozen. but the recovery lock is always locked + * with LKM_NOQUEUE so we do not need the ast in + * this special case */ + if (!dlm_is_recovery_lock(res->lockname.name, + res->lockname.len)) { + kick_thread = 1; + call_ast = 1; + } else { + mlog(0, "%s: returning DLM_NORMAL to " + "node %u for reco lock\n", dlm->name, + lock->ml.node); + } + } else { + /* for NOQUEUE request, unless we get the + * lock right away, return DLM_NOTQUEUED */ + if (flags & LKM_NOQUEUE) { + status = DLM_NOTQUEUED; + if (dlm_is_recovery_lock(res->lockname.name, + res->lockname.len)) { + mlog(0, "%s: returning NOTQUEUED to " + "node %u for reco lock\n", dlm->name, + lock->ml.node); + } + } else { + status = DLM_NORMAL; + dlm_lock_get(lock); + list_add_tail(&lock->list, &res->blocked); + kick_thread = 1; + } + } + + spin_unlock(&res->spinlock); + wake_up(&res->wq); + + /* either queue the ast or release it */ + if (call_ast) + dlm_queue_ast(dlm, lock); + else + dlm_lockres_release_ast(dlm, res); + + dlm_lockres_calc_usage(dlm, res); + if (kick_thread) + dlm_kick_thread(dlm, res); + + return status; +} + +void dlm_revert_pending_lock(struct dlm_lock_resource *res, + struct dlm_lock *lock) +{ + /* remove from local queue if it failed */ + list_del_init(&lock->list); + lock->lksb->flags &= ~DLM_LKSB_GET_LVB; +} + + +/* + * locking: + * caller needs: none + * taken: takes and drops res->spinlock + * held on exit: none + * returns: DLM_DENIED, DLM_RECOVERING, or net status + */ +static enum dlm_status dlmlock_remote(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock, int flags) +{ + enum dlm_status status = DLM_DENIED; + int lockres_changed = 1; + + mlog(0, "type=%d, lockres %.*s, flags = 0x%x\n", + lock->ml.type, res->lockname.len, + res->lockname.name, flags); + + /* + * Wait if resource is getting recovered, remastered, etc. + * If the resource was remastered and new owner is self, then exit. + */ + spin_lock(&res->spinlock); + __dlm_wait_on_lockres(res); + if (res->owner == dlm->node_num) { + spin_unlock(&res->spinlock); + return DLM_RECOVERING; + } + res->state |= DLM_LOCK_RES_IN_PROGRESS; + + /* add lock to local (secondary) queue */ + dlm_lock_get(lock); + list_add_tail(&lock->list, &res->blocked); + lock->lock_pending = 1; + spin_unlock(&res->spinlock); + + /* spec seems to say that you will get DLM_NORMAL when the lock + * has been queued, meaning we need to wait for a reply here. */ + status = dlm_send_remote_lock_request(dlm, res, lock, flags); + + spin_lock(&res->spinlock); + res->state &= ~DLM_LOCK_RES_IN_PROGRESS; + lock->lock_pending = 0; + if (status != DLM_NORMAL) { + if (status == DLM_RECOVERING && + dlm_is_recovery_lock(res->lockname.name, + res->lockname.len)) { + /* recovery lock was mastered by dead node. + * we need to have calc_usage shoot down this + * lockres and completely remaster it. */ + mlog(0, "%s: recovery lock was owned by " + "dead node %u, remaster it now.\n", + dlm->name, res->owner); + } else if (status != DLM_NOTQUEUED) { + /* + * DO NOT call calc_usage, as this would unhash + * the remote lockres before we ever get to use + * it. treat as if we never made any change to + * the lockres. + */ + lockres_changed = 0; + dlm_error(status); + } + dlm_revert_pending_lock(res, lock); + dlm_lock_put(lock); + } else if (dlm_is_recovery_lock(res->lockname.name, + res->lockname.len)) { + /* special case for the $RECOVERY lock. + * there will never be an AST delivered to put + * this lock on the proper secondary queue + * (granted), so do it manually. */ + mlog(0, "%s: $RECOVERY lock for this node (%u) is " + "mastered by %u; got lock, manually granting (no ast)\n", + dlm->name, dlm->node_num, res->owner); + list_move_tail(&lock->list, &res->granted); + } + spin_unlock(&res->spinlock); + + if (lockres_changed) + dlm_lockres_calc_usage(dlm, res); + + wake_up(&res->wq); + return status; +} + + +/* for remote lock creation. + * locking: + * caller needs: none, but need res->state & DLM_LOCK_RES_IN_PROGRESS + * taken: none + * held on exit: none + * returns: DLM_NOLOCKMGR, or net status + */ +static enum dlm_status dlm_send_remote_lock_request(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock, int flags) +{ + struct dlm_create_lock create; + int tmpret, status = 0; + enum dlm_status ret; + + memset(&create, 0, sizeof(create)); + create.node_idx = dlm->node_num; + create.requested_type = lock->ml.type; + create.cookie = lock->ml.cookie; + create.namelen = res->lockname.len; + create.flags = cpu_to_be32(flags); + memcpy(create.name, res->lockname.name, create.namelen); + + tmpret = o2net_send_message(DLM_CREATE_LOCK_MSG, dlm->key, &create, + sizeof(create), res->owner, &status); + if (tmpret >= 0) { + ret = status; + if (ret == DLM_REJECTED) { + mlog(ML_ERROR, "%s: res %.*s, Stale lockres no longer " + "owned by node %u. That node is coming back up " + "currently.\n", dlm->name, create.namelen, + create.name, res->owner); + dlm_print_one_lock_resource(res); + BUG(); + } + } else { + mlog(ML_ERROR, "%s: res %.*s, Error %d send CREATE LOCK to " + "node %u\n", dlm->name, create.namelen, create.name, + tmpret, res->owner); + if (dlm_is_host_down(tmpret)) + ret = DLM_RECOVERING; + else + ret = dlm_err_to_dlm_status(tmpret); + } + + return ret; +} + +void dlm_lock_get(struct dlm_lock *lock) +{ + kref_get(&lock->lock_refs); +} + +void dlm_lock_put(struct dlm_lock *lock) +{ + kref_put(&lock->lock_refs, dlm_lock_release); +} + +static void dlm_lock_release(struct kref *kref) +{ + struct dlm_lock *lock; + + lock = container_of(kref, struct dlm_lock, lock_refs); + + BUG_ON(!list_empty(&lock->list)); + BUG_ON(!list_empty(&lock->ast_list)); + BUG_ON(!list_empty(&lock->bast_list)); + BUG_ON(lock->ast_pending); + BUG_ON(lock->bast_pending); + + dlm_lock_detach_lockres(lock); + + if (lock->lksb_kernel_allocated) { + mlog(0, "freeing kernel-allocated lksb\n"); + kfree(lock->lksb); + } + kmem_cache_free(dlm_lock_cache, lock); +} + +/* associate a lock with it's lockres, getting a ref on the lockres */ +void dlm_lock_attach_lockres(struct dlm_lock *lock, + struct dlm_lock_resource *res) +{ + dlm_lockres_get(res); + lock->lockres = res; +} + +/* drop ref on lockres, if there is still one associated with lock */ +static void dlm_lock_detach_lockres(struct dlm_lock *lock) +{ + struct dlm_lock_resource *res; + + res = lock->lockres; + if (res) { + lock->lockres = NULL; + mlog(0, "removing lock's lockres reference\n"); + dlm_lockres_put(res); + } +} + +static void dlm_init_lock(struct dlm_lock *newlock, int type, + u8 node, u64 cookie) +{ + INIT_LIST_HEAD(&newlock->list); + INIT_LIST_HEAD(&newlock->ast_list); + INIT_LIST_HEAD(&newlock->bast_list); + spin_lock_init(&newlock->spinlock); + newlock->ml.type = type; + newlock->ml.convert_type = LKM_IVMODE; + newlock->ml.highest_blocked = LKM_IVMODE; + newlock->ml.node = node; + newlock->ml.pad1 = 0; + newlock->ml.list = 0; + newlock->ml.flags = 0; + newlock->ast = NULL; + newlock->bast = NULL; + newlock->astdata = NULL; + newlock->ml.cookie = cpu_to_be64(cookie); + newlock->ast_pending = 0; + newlock->bast_pending = 0; + newlock->convert_pending = 0; + newlock->lock_pending = 0; + newlock->unlock_pending = 0; + newlock->cancel_pending = 0; + newlock->lksb_kernel_allocated = 0; + + kref_init(&newlock->lock_refs); +} + +struct dlm_lock * dlm_new_lock(int type, u8 node, u64 cookie, + struct dlm_lockstatus *lksb) +{ + struct dlm_lock *lock; + int kernel_allocated = 0; + + lock = kmem_cache_zalloc(dlm_lock_cache, GFP_NOFS); + if (!lock) + return NULL; + + if (!lksb) { + /* zero memory only if kernel-allocated */ + lksb = kzalloc_obj(*lksb, GFP_NOFS); + if (!lksb) { + kmem_cache_free(dlm_lock_cache, lock); + return NULL; + } + kernel_allocated = 1; + } + + dlm_init_lock(lock, type, node, cookie); + if (kernel_allocated) + lock->lksb_kernel_allocated = 1; + lock->lksb = lksb; + lksb->lockid = lock; + return lock; +} + +/* handler for lock creation net message + * locking: + * caller needs: none + * taken: takes and drops res->spinlock + * held on exit: none + * returns: DLM_NORMAL, DLM_SYSERR, DLM_IVLOCKID, DLM_NOTQUEUED + */ +int dlm_create_lock_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data) +{ + struct dlm_ctxt *dlm = data; + struct dlm_create_lock *create = (struct dlm_create_lock *)msg->buf; + struct dlm_lock_resource *res = NULL; + struct dlm_lock *newlock = NULL; + struct dlm_lockstatus *lksb = NULL; + enum dlm_status status = DLM_NORMAL; + char *name; + unsigned int namelen; + + BUG_ON(!dlm); + + if (!dlm_grab(dlm)) + return DLM_REJECTED; + + name = create->name; + namelen = create->namelen; + status = DLM_REJECTED; + if (!dlm_domain_fully_joined(dlm)) { + mlog(ML_ERROR, "Domain %s not fully joined, but node %u is " + "sending a create_lock message for lock %.*s!\n", + dlm->name, create->node_idx, namelen, name); + dlm_error(status); + goto leave; + } + + status = DLM_IVBUFLEN; + if (namelen > DLM_LOCKID_NAME_MAX) { + dlm_error(status); + goto leave; + } + + status = DLM_SYSERR; + newlock = dlm_new_lock(create->requested_type, + create->node_idx, + be64_to_cpu(create->cookie), NULL); + if (!newlock) { + dlm_error(status); + goto leave; + } + + lksb = newlock->lksb; + + if (be32_to_cpu(create->flags) & LKM_GET_LVB) { + lksb->flags |= DLM_LKSB_GET_LVB; + mlog(0, "set DLM_LKSB_GET_LVB flag\n"); + } + + status = DLM_IVLOCKID; + res = dlm_lookup_lockres(dlm, name, namelen); + if (!res) { + dlm_error(status); + goto leave; + } + + spin_lock(&res->spinlock); + status = __dlm_lockres_state_to_status(res); + spin_unlock(&res->spinlock); + + if (status != DLM_NORMAL) { + mlog(0, "lockres recovering/migrating/in-progress\n"); + goto leave; + } + + dlm_lock_attach_lockres(newlock, res); + + status = dlmlock_master(dlm, res, newlock, be32_to_cpu(create->flags)); +leave: + if (status != DLM_NORMAL) + if (newlock) + dlm_lock_put(newlock); + + if (res) + dlm_lockres_put(res); + + dlm_put(dlm); + + return status; +} + + +/* fetch next node-local (u8 nodenum + u56 cookie) into u64 */ +static inline void dlm_get_next_cookie(u8 node_num, u64 *cookie) +{ + u64 tmpnode = node_num; + + /* shift single byte of node num into top 8 bits */ + tmpnode <<= 56; + + spin_lock(&dlm_cookie_lock); + *cookie = (dlm_next_cookie | tmpnode); + if (++dlm_next_cookie & 0xff00000000000000ull) { + mlog(0, "This node's cookie will now wrap!\n"); + dlm_next_cookie = 1; + } + spin_unlock(&dlm_cookie_lock); +} + +enum dlm_status dlmlock(struct dlm_ctxt *dlm, int mode, + struct dlm_lockstatus *lksb, int flags, + const char *name, int namelen, dlm_astlockfunc_t *ast, + void *data, dlm_bastlockfunc_t *bast) +{ + enum dlm_status status; + struct dlm_lock_resource *res = NULL; + struct dlm_lock *lock = NULL; + int convert = 0, recovery = 0; + + /* yes this function is a mess. + * TODO: clean this up. lots of common code in the + * lock and convert paths, especially in the retry blocks */ + if (!lksb) { + dlm_error(DLM_BADARGS); + return DLM_BADARGS; + } + + status = DLM_BADPARAM; + if (mode != LKM_EXMODE && mode != LKM_PRMODE && mode != LKM_NLMODE) { + dlm_error(status); + goto error; + } + + if (flags & ~LKM_VALID_FLAGS) { + dlm_error(status); + goto error; + } + + convert = (flags & LKM_CONVERT); + recovery = (flags & LKM_RECOVERY); + + if (recovery && + (!dlm_is_recovery_lock(name, namelen) || convert) ) { + dlm_error(status); + goto error; + } + if (convert && (flags & LKM_LOCAL)) { + mlog(ML_ERROR, "strange LOCAL convert request!\n"); + goto error; + } + + if (convert) { + /* CONVERT request */ + + /* if converting, must pass in a valid dlm_lock */ + lock = lksb->lockid; + if (!lock) { + mlog(ML_ERROR, "NULL lock pointer in convert " + "request\n"); + goto error; + } + + res = lock->lockres; + if (!res) { + mlog(ML_ERROR, "NULL lockres pointer in convert " + "request\n"); + goto error; + } + dlm_lockres_get(res); + + /* XXX: for ocfs2 purposes, the ast/bast/astdata/lksb are + * static after the original lock call. convert requests will + * ensure that everything is the same, or return DLM_BADARGS. + * this means that DLM_DENIED_NOASTS will never be returned. + */ + if (lock->lksb != lksb || lock->ast != ast || + lock->bast != bast || lock->astdata != data) { + status = DLM_BADARGS; + mlog(ML_ERROR, "new args: lksb=%p, ast=%p, bast=%p, " + "astdata=%p\n", lksb, ast, bast, data); + mlog(ML_ERROR, "orig args: lksb=%p, ast=%p, bast=%p, " + "astdata=%p\n", lock->lksb, lock->ast, + lock->bast, lock->astdata); + goto error; + } +retry_convert: + dlm_wait_for_recovery(dlm); + + if (res->owner == dlm->node_num) + status = dlmconvert_master(dlm, res, lock, flags, mode); + else + status = dlmconvert_remote(dlm, res, lock, flags, mode); + if (status == DLM_RECOVERING || status == DLM_MIGRATING || + status == DLM_FORWARD) { + /* for now, see how this works without sleeping + * and just retry right away. I suspect the reco + * or migration will complete fast enough that + * no waiting will be necessary */ + mlog(0, "retrying convert with migration/recovery/" + "in-progress\n"); + msleep(100); + goto retry_convert; + } + } else { + u64 tmpcookie; + + /* LOCK request */ + status = DLM_BADARGS; + if (!name) { + dlm_error(status); + goto error; + } + + status = DLM_IVBUFLEN; + if (namelen > DLM_LOCKID_NAME_MAX || namelen < 1) { + dlm_error(status); + goto error; + } + + dlm_get_next_cookie(dlm->node_num, &tmpcookie); + lock = dlm_new_lock(mode, dlm->node_num, tmpcookie, lksb); + if (!lock) { + dlm_error(status); + goto error; + } + + if (!recovery) + dlm_wait_for_recovery(dlm); + + /* find or create the lock resource */ + res = dlm_get_lock_resource(dlm, name, namelen, flags); + if (!res) { + status = DLM_IVLOCKID; + dlm_error(status); + goto error; + } + + mlog(0, "type=%d, flags = 0x%x\n", mode, flags); + mlog(0, "creating lock: lock=%p res=%p\n", lock, res); + + dlm_lock_attach_lockres(lock, res); + lock->ast = ast; + lock->bast = bast; + lock->astdata = data; + +retry_lock: + if (flags & LKM_VALBLK) { + mlog(0, "LKM_VALBLK passed by caller\n"); + + /* LVB requests for non PR, PW or EX locks are + * ignored. */ + if (mode < LKM_PRMODE) + flags &= ~LKM_VALBLK; + else { + flags |= LKM_GET_LVB; + lock->lksb->flags |= DLM_LKSB_GET_LVB; + } + } + + if (res->owner == dlm->node_num) + status = dlmlock_master(dlm, res, lock, flags); + else + status = dlmlock_remote(dlm, res, lock, flags); + + if (status == DLM_RECOVERING || status == DLM_MIGRATING || + status == DLM_FORWARD) { + msleep(100); + if (recovery) { + if (status != DLM_RECOVERING) + goto retry_lock; + /* wait to see the node go down, then + * drop down and allow the lockres to + * get cleaned up. need to remaster. */ + dlm_wait_for_node_death(dlm, res->owner, + DLM_NODE_DEATH_WAIT_MAX); + } else { + dlm_wait_for_recovery(dlm); + goto retry_lock; + } + } + + /* Inflight taken in dlm_get_lock_resource() is dropped here */ + spin_lock(&res->spinlock); + dlm_lockres_drop_inflight_ref(dlm, res); + spin_unlock(&res->spinlock); + + dlm_lockres_calc_usage(dlm, res); + dlm_kick_thread(dlm, res); + + if (status != DLM_NORMAL) { + lock->lksb->flags &= ~DLM_LKSB_GET_LVB; + if (status != DLM_NOTQUEUED) + dlm_error(status); + goto error; + } + } + +error: + if (status != DLM_NORMAL) { + if (lock && !convert) + dlm_lock_put(lock); + // this is kind of unnecessary + lksb->status = status; + } + + /* put lockres ref from the convert path + * or from dlm_get_lock_resource */ + if (res) + dlm_lockres_put(res); + + return status; +} +EXPORT_SYMBOL_GPL(dlmlock); diff --git a/fs/ocfs2/dlm/dlmmaster.c b/fs/ocfs2/dlm/dlmmaster.c new file mode 100644 index 000000000..612969867 --- /dev/null +++ b/fs/ocfs2/dlm/dlmmaster.c @@ -0,0 +1,3557 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * dlmmod.c + * + * standalone DLM module + * + * Copyright (C) 2004 Oracle. All rights reserved. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../cluster/heartbeat.h" +#include "../cluster/nodemanager.h" +#include "../cluster/tcp.h" + +#include "dlmapi.h" +#include "dlmcommon.h" +#include "dlmdomain.h" +#include "dlmdebug.h" + +#define MLOG_MASK_PREFIX (ML_DLM|ML_DLM_MASTER) +#include "../cluster/masklog.h" + +static void dlm_mle_node_down(struct dlm_ctxt *dlm, + struct dlm_master_list_entry *mle, + struct o2nm_node *node, + int idx); +static void dlm_mle_node_up(struct dlm_ctxt *dlm, + struct dlm_master_list_entry *mle, + struct o2nm_node *node, + int idx); + +static void dlm_assert_master_worker(struct dlm_work_item *item, void *data); +static int dlm_do_assert_master(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + void *nodemap, u32 flags); +static void dlm_deref_lockres_worker(struct dlm_work_item *item, void *data); + +static inline int dlm_mle_equal(struct dlm_ctxt *dlm, + struct dlm_master_list_entry *mle, + const char *name, + unsigned int namelen) +{ + if (dlm != mle->dlm) + return 0; + + if (namelen != mle->mnamelen || + memcmp(name, mle->mname, namelen) != 0) + return 0; + + return 1; +} + +static struct kmem_cache *dlm_lockres_cache; +static struct kmem_cache *dlm_lockname_cache; +static struct kmem_cache *dlm_mle_cache; + +static void dlm_mle_release(struct kref *kref); +static void dlm_init_mle(struct dlm_master_list_entry *mle, + enum dlm_mle_type type, + struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + const char *name, + unsigned int namelen); +static void dlm_put_mle(struct dlm_master_list_entry *mle); +static void __dlm_put_mle(struct dlm_master_list_entry *mle); +static int dlm_find_mle(struct dlm_ctxt *dlm, + struct dlm_master_list_entry **mle, + char *name, unsigned int namelen); + +static int dlm_do_master_request(struct dlm_lock_resource *res, + struct dlm_master_list_entry *mle, int to); + + +static int dlm_wait_for_lock_mastery(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_master_list_entry *mle, + int *blocked); +static int dlm_restart_lock_mastery(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_master_list_entry *mle, + int blocked); +static int dlm_add_migration_mle(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_master_list_entry *mle, + struct dlm_master_list_entry **oldmle, + const char *name, unsigned int namelen, + u8 new_master, u8 master); + +static u8 dlm_pick_migration_target(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res); +static void dlm_remove_nonlocal_locks(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res); +static int dlm_mark_lockres_migrating(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + u8 target); +static int dlm_pre_master_reco_lockres(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res); + + +int dlm_is_host_down(int errno) +{ + switch (errno) { + case -EBADF: + case -ECONNREFUSED: + case -ENOTCONN: + case -ECONNRESET: + case -EPIPE: + case -EHOSTDOWN: + case -EHOSTUNREACH: + case -ETIMEDOUT: + case -ECONNABORTED: + case -ENETDOWN: + case -ENETUNREACH: + case -ENETRESET: + case -ESHUTDOWN: + case -ENOPROTOOPT: + case -EINVAL: /* if returned from our tcp code, + this means there is no socket */ + return 1; + } + return 0; +} + + +/* + * MASTER LIST FUNCTIONS + */ + + +/* + * regarding master list entries and heartbeat callbacks: + * + * in order to avoid sleeping and allocation that occurs in + * heartbeat, master list entries are simply attached to the + * dlm's established heartbeat callbacks. the mle is attached + * when it is created, and since the dlm->spinlock is held at + * that time, any heartbeat event will be properly discovered + * by the mle. the mle needs to be detached from the + * dlm->mle_hb_events list as soon as heartbeat events are no + * longer useful to the mle, and before the mle is freed. + * + * as a general rule, heartbeat events are no longer needed by + * the mle once an "answer" regarding the lock master has been + * received. + */ +static inline void __dlm_mle_attach_hb_events(struct dlm_ctxt *dlm, + struct dlm_master_list_entry *mle) +{ + assert_spin_locked(&dlm->spinlock); + + list_add_tail(&mle->hb_events, &dlm->mle_hb_events); +} + + +static inline void __dlm_mle_detach_hb_events(struct dlm_ctxt *dlm, + struct dlm_master_list_entry *mle) +{ + if (!list_empty(&mle->hb_events)) + list_del_init(&mle->hb_events); +} + + +static inline void dlm_mle_detach_hb_events(struct dlm_ctxt *dlm, + struct dlm_master_list_entry *mle) +{ + spin_lock(&dlm->spinlock); + __dlm_mle_detach_hb_events(dlm, mle); + spin_unlock(&dlm->spinlock); +} + +static void dlm_get_mle_inuse(struct dlm_master_list_entry *mle) +{ + struct dlm_ctxt *dlm; + dlm = mle->dlm; + + assert_spin_locked(&dlm->spinlock); + assert_spin_locked(&dlm->master_lock); + mle->inuse++; + kref_get(&mle->mle_refs); +} + +static void dlm_put_mle_inuse(struct dlm_master_list_entry *mle) +{ + struct dlm_ctxt *dlm; + dlm = mle->dlm; + + spin_lock(&dlm->spinlock); + spin_lock(&dlm->master_lock); + mle->inuse--; + __dlm_put_mle(mle); + spin_unlock(&dlm->master_lock); + spin_unlock(&dlm->spinlock); + +} + +/* remove from list and free */ +static void __dlm_put_mle(struct dlm_master_list_entry *mle) +{ + struct dlm_ctxt *dlm; + dlm = mle->dlm; + + assert_spin_locked(&dlm->spinlock); + assert_spin_locked(&dlm->master_lock); + if (!kref_read(&mle->mle_refs)) { + /* this may or may not crash, but who cares. + * it's a BUG. */ + mlog(ML_ERROR, "bad mle: %p\n", mle); + dlm_print_one_mle(mle); + BUG(); + } else + kref_put(&mle->mle_refs, dlm_mle_release); +} + + +/* must not have any spinlocks coming in */ +static void dlm_put_mle(struct dlm_master_list_entry *mle) +{ + struct dlm_ctxt *dlm; + dlm = mle->dlm; + + spin_lock(&dlm->spinlock); + spin_lock(&dlm->master_lock); + __dlm_put_mle(mle); + spin_unlock(&dlm->master_lock); + spin_unlock(&dlm->spinlock); +} + +static inline void dlm_get_mle(struct dlm_master_list_entry *mle) +{ + kref_get(&mle->mle_refs); +} + +static void dlm_init_mle(struct dlm_master_list_entry *mle, + enum dlm_mle_type type, + struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + const char *name, + unsigned int namelen) +{ + assert_spin_locked(&dlm->spinlock); + + mle->dlm = dlm; + mle->type = type; + INIT_HLIST_NODE(&mle->master_hash_node); + INIT_LIST_HEAD(&mle->hb_events); + bitmap_zero(mle->maybe_map, O2NM_MAX_NODES); + spin_lock_init(&mle->spinlock); + init_waitqueue_head(&mle->wq); + atomic_set(&mle->woken, 0); + kref_init(&mle->mle_refs); + bitmap_zero(mle->response_map, O2NM_MAX_NODES); + mle->master = O2NM_MAX_NODES; + mle->new_master = O2NM_MAX_NODES; + mle->inuse = 0; + + BUG_ON(mle->type != DLM_MLE_BLOCK && + mle->type != DLM_MLE_MASTER && + mle->type != DLM_MLE_MIGRATION); + + if (mle->type == DLM_MLE_MASTER) { + BUG_ON(!res); + mle->mleres = res; + memcpy(mle->mname, res->lockname.name, res->lockname.len); + mle->mnamelen = res->lockname.len; + mle->mnamehash = res->lockname.hash; + } else { + BUG_ON(!name); + mle->mleres = NULL; + memcpy(mle->mname, name, namelen); + mle->mnamelen = namelen; + mle->mnamehash = dlm_lockid_hash(name, namelen); + } + + atomic_inc(&dlm->mle_tot_count[mle->type]); + atomic_inc(&dlm->mle_cur_count[mle->type]); + + /* copy off the node_map and register hb callbacks on our copy */ + bitmap_copy(mle->node_map, dlm->domain_map, O2NM_MAX_NODES); + bitmap_copy(mle->vote_map, dlm->domain_map, O2NM_MAX_NODES); + clear_bit(dlm->node_num, mle->vote_map); + clear_bit(dlm->node_num, mle->node_map); + + /* attach the mle to the domain node up/down events */ + __dlm_mle_attach_hb_events(dlm, mle); +} + +void __dlm_unlink_mle(struct dlm_ctxt *dlm, struct dlm_master_list_entry *mle) +{ + assert_spin_locked(&dlm->spinlock); + assert_spin_locked(&dlm->master_lock); + + if (!hlist_unhashed(&mle->master_hash_node)) + hlist_del_init(&mle->master_hash_node); +} + +void __dlm_insert_mle(struct dlm_ctxt *dlm, struct dlm_master_list_entry *mle) +{ + struct hlist_head *bucket; + + assert_spin_locked(&dlm->master_lock); + + bucket = dlm_master_hash(dlm, mle->mnamehash); + hlist_add_head(&mle->master_hash_node, bucket); +} + +/* returns 1 if found, 0 if not */ +static int dlm_find_mle(struct dlm_ctxt *dlm, + struct dlm_master_list_entry **mle, + char *name, unsigned int namelen) +{ + struct dlm_master_list_entry *tmpmle; + struct hlist_head *bucket; + unsigned int hash; + + assert_spin_locked(&dlm->master_lock); + + hash = dlm_lockid_hash(name, namelen); + bucket = dlm_master_hash(dlm, hash); + hlist_for_each_entry(tmpmle, bucket, master_hash_node) { + if (!dlm_mle_equal(dlm, tmpmle, name, namelen)) + continue; + dlm_get_mle(tmpmle); + *mle = tmpmle; + return 1; + } + return 0; +} + +void dlm_hb_event_notify_attached(struct dlm_ctxt *dlm, int idx, int node_up) +{ + struct dlm_master_list_entry *mle; + + assert_spin_locked(&dlm->spinlock); + + list_for_each_entry(mle, &dlm->mle_hb_events, hb_events) { + if (node_up) + dlm_mle_node_up(dlm, mle, NULL, idx); + else + dlm_mle_node_down(dlm, mle, NULL, idx); + } +} + +static void dlm_mle_node_down(struct dlm_ctxt *dlm, + struct dlm_master_list_entry *mle, + struct o2nm_node *node, int idx) +{ + spin_lock(&mle->spinlock); + + if (!test_bit(idx, mle->node_map)) + mlog(0, "node %u already removed from nodemap!\n", idx); + else + clear_bit(idx, mle->node_map); + + spin_unlock(&mle->spinlock); +} + +static void dlm_mle_node_up(struct dlm_ctxt *dlm, + struct dlm_master_list_entry *mle, + struct o2nm_node *node, int idx) +{ + spin_lock(&mle->spinlock); + + if (test_bit(idx, mle->node_map)) + mlog(0, "node %u already in node map!\n", idx); + else + set_bit(idx, mle->node_map); + + spin_unlock(&mle->spinlock); +} + + +int dlm_init_mle_cache(void) +{ + dlm_mle_cache = kmem_cache_create("o2dlm_mle", + sizeof(struct dlm_master_list_entry), + 0, SLAB_HWCACHE_ALIGN, + NULL); + if (dlm_mle_cache == NULL) + return -ENOMEM; + return 0; +} + +void dlm_destroy_mle_cache(void) +{ + kmem_cache_destroy(dlm_mle_cache); +} + +static void dlm_mle_release(struct kref *kref) +{ + struct dlm_master_list_entry *mle; + struct dlm_ctxt *dlm; + + mle = container_of(kref, struct dlm_master_list_entry, mle_refs); + dlm = mle->dlm; + + assert_spin_locked(&dlm->spinlock); + assert_spin_locked(&dlm->master_lock); + + mlog(0, "Releasing mle for %.*s, type %d\n", mle->mnamelen, mle->mname, + mle->type); + + /* remove from list if not already */ + __dlm_unlink_mle(dlm, mle); + + /* detach the mle from the domain node up/down events */ + __dlm_mle_detach_hb_events(dlm, mle); + + atomic_dec(&dlm->mle_cur_count[mle->type]); + + /* NOTE: kfree under spinlock here. + * if this is bad, we can move this to a freelist. */ + kmem_cache_free(dlm_mle_cache, mle); +} + + +/* + * LOCK RESOURCE FUNCTIONS + */ + +int dlm_init_master_caches(void) +{ + dlm_lockres_cache = kmem_cache_create("o2dlm_lockres", + sizeof(struct dlm_lock_resource), + 0, SLAB_HWCACHE_ALIGN, NULL); + if (!dlm_lockres_cache) + goto bail; + + dlm_lockname_cache = kmem_cache_create("o2dlm_lockname", + DLM_LOCKID_NAME_MAX, 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!dlm_lockname_cache) + goto bail; + + return 0; +bail: + dlm_destroy_master_caches(); + return -ENOMEM; +} + +void dlm_destroy_master_caches(void) +{ + kmem_cache_destroy(dlm_lockname_cache); + dlm_lockname_cache = NULL; + + kmem_cache_destroy(dlm_lockres_cache); + dlm_lockres_cache = NULL; +} + +static void dlm_lockres_release(struct kref *kref) +{ + struct dlm_lock_resource *res; + struct dlm_ctxt *dlm; + + res = container_of(kref, struct dlm_lock_resource, refs); + dlm = res->dlm; + + /* This should not happen -- all lockres' have a name + * associated with them at init time. */ + BUG_ON(!res->lockname.name); + + mlog(0, "destroying lockres %.*s\n", res->lockname.len, + res->lockname.name); + + atomic_dec(&dlm->res_cur_count); + + if (!hlist_unhashed(&res->hash_node) || + !list_empty(&res->granted) || + !list_empty(&res->converting) || + !list_empty(&res->blocked) || + !list_empty(&res->dirty) || + !list_empty(&res->recovering) || + !list_empty(&res->purge)) { + mlog(ML_ERROR, + "Going to BUG for resource %.*s." + " We're on a list! [%c%c%c%c%c%c%c]\n", + res->lockname.len, res->lockname.name, + !hlist_unhashed(&res->hash_node) ? 'H' : ' ', + !list_empty(&res->granted) ? 'G' : ' ', + !list_empty(&res->converting) ? 'C' : ' ', + !list_empty(&res->blocked) ? 'B' : ' ', + !list_empty(&res->dirty) ? 'D' : ' ', + !list_empty(&res->recovering) ? 'R' : ' ', + !list_empty(&res->purge) ? 'P' : ' '); + + dlm_print_one_lock_resource(res); + } + + /* By the time we're ready to blow this guy away, we shouldn't + * be on any lists. */ + BUG_ON(!hlist_unhashed(&res->hash_node)); + BUG_ON(!list_empty(&res->granted)); + BUG_ON(!list_empty(&res->converting)); + BUG_ON(!list_empty(&res->blocked)); + BUG_ON(!list_empty(&res->dirty)); + BUG_ON(!list_empty(&res->recovering)); + BUG_ON(!list_empty(&res->purge)); + + kmem_cache_free(dlm_lockname_cache, (void *)res->lockname.name); + + kmem_cache_free(dlm_lockres_cache, res); +} + +void dlm_lockres_put(struct dlm_lock_resource *res) +{ + kref_put(&res->refs, dlm_lockres_release); +} + +static void dlm_init_lockres(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + const char *name, unsigned int namelen) +{ + char *qname; + + /* If we memset here, we lose our reference to the kmalloc'd + * res->lockname.name, so be sure to init every field + * correctly! */ + + qname = (char *) res->lockname.name; + memcpy(qname, name, namelen); + + res->lockname.len = namelen; + res->lockname.hash = dlm_lockid_hash(name, namelen); + + init_waitqueue_head(&res->wq); + spin_lock_init(&res->spinlock); + INIT_HLIST_NODE(&res->hash_node); + INIT_LIST_HEAD(&res->granted); + INIT_LIST_HEAD(&res->converting); + INIT_LIST_HEAD(&res->blocked); + INIT_LIST_HEAD(&res->dirty); + INIT_LIST_HEAD(&res->recovering); + INIT_LIST_HEAD(&res->purge); + INIT_LIST_HEAD(&res->tracking); + atomic_set(&res->asts_reserved, 0); + res->migration_pending = 0; + res->inflight_locks = 0; + res->inflight_assert_workers = 0; + + res->dlm = dlm; + + kref_init(&res->refs); + + atomic_inc(&dlm->res_tot_count); + atomic_inc(&dlm->res_cur_count); + + /* just for consistency */ + spin_lock(&res->spinlock); + dlm_set_lockres_owner(dlm, res, DLM_LOCK_RES_OWNER_UNKNOWN); + spin_unlock(&res->spinlock); + + res->state = DLM_LOCK_RES_IN_PROGRESS; + + res->last_used = 0; + + spin_lock(&dlm->track_lock); + list_add_tail(&res->tracking, &dlm->tracking_list); + spin_unlock(&dlm->track_lock); + + memset(res->lvb, 0, DLM_LVB_LEN); + bitmap_zero(res->refmap, O2NM_MAX_NODES); +} + +struct dlm_lock_resource *dlm_new_lockres(struct dlm_ctxt *dlm, + const char *name, + unsigned int namelen) +{ + struct dlm_lock_resource *res = NULL; + + res = kmem_cache_zalloc(dlm_lockres_cache, GFP_NOFS); + if (!res) + goto error; + + res->lockname.name = kmem_cache_zalloc(dlm_lockname_cache, GFP_NOFS); + if (!res->lockname.name) + goto error; + + dlm_init_lockres(dlm, res, name, namelen); + return res; + +error: + if (res) + kmem_cache_free(dlm_lockres_cache, res); + return NULL; +} + +void dlm_lockres_set_refmap_bit(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, int bit) +{ + assert_spin_locked(&res->spinlock); + + mlog(0, "res %.*s, set node %u, %ps()\n", res->lockname.len, + res->lockname.name, bit, __builtin_return_address(0)); + + set_bit(bit, res->refmap); +} + +void dlm_lockres_clear_refmap_bit(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, int bit) +{ + assert_spin_locked(&res->spinlock); + + mlog(0, "res %.*s, clr node %u, %ps()\n", res->lockname.len, + res->lockname.name, bit, __builtin_return_address(0)); + + clear_bit(bit, res->refmap); +} + +static void __dlm_lockres_grab_inflight_ref(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res) +{ + res->inflight_locks++; + + mlog(0, "%s: res %.*s, inflight++: now %u, %ps()\n", dlm->name, + res->lockname.len, res->lockname.name, res->inflight_locks, + __builtin_return_address(0)); +} + +void dlm_lockres_grab_inflight_ref(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res) +{ + assert_spin_locked(&res->spinlock); + __dlm_lockres_grab_inflight_ref(dlm, res); +} + +void dlm_lockres_drop_inflight_ref(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res) +{ + assert_spin_locked(&res->spinlock); + + BUG_ON(res->inflight_locks == 0); + + res->inflight_locks--; + + mlog(0, "%s: res %.*s, inflight--: now %u, %ps()\n", dlm->name, + res->lockname.len, res->lockname.name, res->inflight_locks, + __builtin_return_address(0)); + + wake_up(&res->wq); +} + +void __dlm_lockres_grab_inflight_worker(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res) +{ + assert_spin_locked(&res->spinlock); + res->inflight_assert_workers++; + mlog(0, "%s:%.*s: inflight assert worker++: now %u\n", + dlm->name, res->lockname.len, res->lockname.name, + res->inflight_assert_workers); +} + +static void __dlm_lockres_drop_inflight_worker(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res) +{ + assert_spin_locked(&res->spinlock); + BUG_ON(res->inflight_assert_workers == 0); + res->inflight_assert_workers--; + mlog(0, "%s:%.*s: inflight assert worker--: now %u\n", + dlm->name, res->lockname.len, res->lockname.name, + res->inflight_assert_workers); +} + +static void dlm_lockres_drop_inflight_worker(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res) +{ + spin_lock(&res->spinlock); + __dlm_lockres_drop_inflight_worker(dlm, res); + spin_unlock(&res->spinlock); +} + +/* + * lookup a lock resource by name. + * may already exist in the hashtable. + * lockid is null terminated + * + * if not, allocate enough for the lockres and for + * the temporary structure used in doing the mastering. + * + * also, do a lookup in the dlm->master_list to see + * if another node has begun mastering the same lock. + * if so, there should be a block entry in there + * for this name, and we should *not* attempt to master + * the lock here. need to wait around for that node + * to assert_master (or die). + * + */ +struct dlm_lock_resource * dlm_get_lock_resource(struct dlm_ctxt *dlm, + const char *lockid, + int namelen, + int flags) +{ + struct dlm_lock_resource *tmpres=NULL, *res=NULL; + struct dlm_master_list_entry *mle = NULL; + struct dlm_master_list_entry *alloc_mle = NULL; + int blocked = 0; + int ret, nodenum; + struct dlm_node_iter iter; + unsigned int hash; + int tries = 0; + int bit, wait_on_recovery = 0; + + BUG_ON(!lockid); + + hash = dlm_lockid_hash(lockid, namelen); + + mlog(0, "get lockres %s (len %d)\n", lockid, namelen); + +lookup: + spin_lock(&dlm->spinlock); + tmpres = __dlm_lookup_lockres_full(dlm, lockid, namelen, hash); + if (tmpres) { + spin_unlock(&dlm->spinlock); + spin_lock(&tmpres->spinlock); + + /* + * Right after dlm spinlock was released, dlm_thread could have + * purged the lockres. Check if lockres got unhashed. If so + * start over. + */ + if (hlist_unhashed(&tmpres->hash_node)) { + spin_unlock(&tmpres->spinlock); + dlm_lockres_put(tmpres); + tmpres = NULL; + goto lookup; + } + + /* Wait on the thread that is mastering the resource */ + if (tmpres->owner == DLM_LOCK_RES_OWNER_UNKNOWN) { + __dlm_wait_on_lockres(tmpres); + BUG_ON(tmpres->owner == DLM_LOCK_RES_OWNER_UNKNOWN); + spin_unlock(&tmpres->spinlock); + dlm_lockres_put(tmpres); + tmpres = NULL; + goto lookup; + } + + /* Wait on the resource purge to complete before continuing */ + if (tmpres->state & DLM_LOCK_RES_DROPPING_REF) { + BUG_ON(tmpres->owner == dlm->node_num); + __dlm_wait_on_lockres_flags(tmpres, + DLM_LOCK_RES_DROPPING_REF); + spin_unlock(&tmpres->spinlock); + dlm_lockres_put(tmpres); + tmpres = NULL; + goto lookup; + } + + /* Grab inflight ref to pin the resource */ + dlm_lockres_grab_inflight_ref(dlm, tmpres); + + spin_unlock(&tmpres->spinlock); + if (res) { + spin_lock(&dlm->track_lock); + if (!list_empty(&res->tracking)) + list_del_init(&res->tracking); + else + mlog(ML_ERROR, "Resource %.*s not " + "on the Tracking list\n", + res->lockname.len, + res->lockname.name); + spin_unlock(&dlm->track_lock); + dlm_lockres_put(res); + } + res = tmpres; + goto leave; + } + + if (!res) { + spin_unlock(&dlm->spinlock); + mlog(0, "allocating a new resource\n"); + /* nothing found and we need to allocate one. */ + alloc_mle = kmem_cache_alloc(dlm_mle_cache, GFP_NOFS); + if (!alloc_mle) + goto leave; + res = dlm_new_lockres(dlm, lockid, namelen); + if (!res) + goto leave; + goto lookup; + } + + mlog(0, "no lockres found, allocated our own: %p\n", res); + + if (flags & LKM_LOCAL) { + /* caller knows it's safe to assume it's not mastered elsewhere + * DONE! return right away */ + spin_lock(&res->spinlock); + dlm_change_lockres_owner(dlm, res, dlm->node_num); + __dlm_insert_lockres(dlm, res); + dlm_lockres_grab_inflight_ref(dlm, res); + spin_unlock(&res->spinlock); + spin_unlock(&dlm->spinlock); + /* lockres still marked IN_PROGRESS */ + goto wake_waiters; + } + + /* check master list to see if another node has started mastering it */ + spin_lock(&dlm->master_lock); + + /* if we found a block, wait for lock to be mastered by another node */ + blocked = dlm_find_mle(dlm, &mle, (char *)lockid, namelen); + if (blocked) { + int mig; + if (mle->type == DLM_MLE_MASTER) { + mlog(ML_ERROR, "master entry for nonexistent lock!\n"); + BUG(); + } + mig = (mle->type == DLM_MLE_MIGRATION); + /* if there is a migration in progress, let the migration + * finish before continuing. we can wait for the absence + * of the MIGRATION mle: either the migrate finished or + * one of the nodes died and the mle was cleaned up. + * if there is a BLOCK here, but it already has a master + * set, we are too late. the master does not have a ref + * for us in the refmap. detach the mle and drop it. + * either way, go back to the top and start over. */ + if (mig || mle->master != O2NM_MAX_NODES) { + BUG_ON(mig && mle->master == dlm->node_num); + /* we arrived too late. the master does not + * have a ref for us. retry. */ + mlog(0, "%s:%.*s: late on %s\n", + dlm->name, namelen, lockid, + mig ? "MIGRATION" : "BLOCK"); + spin_unlock(&dlm->master_lock); + spin_unlock(&dlm->spinlock); + + /* master is known, detach */ + if (!mig) + dlm_mle_detach_hb_events(dlm, mle); + dlm_put_mle(mle); + mle = NULL; + /* this is lame, but we can't wait on either + * the mle or lockres waitqueue here */ + if (mig) + msleep(100); + goto lookup; + } + } else { + /* go ahead and try to master lock on this node */ + mle = alloc_mle; + /* make sure this does not get freed below */ + alloc_mle = NULL; + dlm_init_mle(mle, DLM_MLE_MASTER, dlm, res, NULL, 0); + set_bit(dlm->node_num, mle->maybe_map); + __dlm_insert_mle(dlm, mle); + + /* still holding the dlm spinlock, check the recovery map + * to see if there are any nodes that still need to be + * considered. these will not appear in the mle nodemap + * but they might own this lockres. wait on them. */ + bit = find_first_bit(dlm->recovery_map, O2NM_MAX_NODES); + if (bit < O2NM_MAX_NODES) { + mlog(0, "%s: res %.*s, At least one node (%d) " + "to recover before lock mastery can begin\n", + dlm->name, namelen, (char *)lockid, bit); + wait_on_recovery = 1; + } + } + + /* at this point there is either a DLM_MLE_BLOCK or a + * DLM_MLE_MASTER on the master list, so it's safe to add the + * lockres to the hashtable. anyone who finds the lock will + * still have to wait on the IN_PROGRESS. */ + + /* finally add the lockres to its hash bucket */ + __dlm_insert_lockres(dlm, res); + + /* since this lockres is new it doesn't not require the spinlock */ + __dlm_lockres_grab_inflight_ref(dlm, res); + + /* get an extra ref on the mle in case this is a BLOCK + * if so, the creator of the BLOCK may try to put the last + * ref at this time in the assert master handler, so we + * need an extra one to keep from a bad ptr deref. */ + dlm_get_mle_inuse(mle); + spin_unlock(&dlm->master_lock); + spin_unlock(&dlm->spinlock); + +redo_request: + while (wait_on_recovery) { + /* any cluster changes that occurred after dropping the + * dlm spinlock would be detectable be a change on the mle, + * so we only need to clear out the recovery map once. */ + if (dlm_is_recovery_lock(lockid, namelen)) { + mlog(0, "%s: Recovery map is not empty, but must " + "master $RECOVERY lock now\n", dlm->name); + if (!dlm_pre_master_reco_lockres(dlm, res)) + wait_on_recovery = 0; + else { + mlog(0, "%s: waiting 500ms for heartbeat state " + "change\n", dlm->name); + msleep(500); + } + continue; + } + + dlm_kick_recovery_thread(dlm); + msleep(1000); + dlm_wait_for_recovery(dlm); + + spin_lock(&dlm->spinlock); + bit = find_first_bit(dlm->recovery_map, O2NM_MAX_NODES); + if (bit < O2NM_MAX_NODES) { + mlog(0, "%s: res %.*s, At least one node (%d) " + "to recover before lock mastery can begin\n", + dlm->name, namelen, (char *)lockid, bit); + wait_on_recovery = 1; + } else + wait_on_recovery = 0; + spin_unlock(&dlm->spinlock); + + if (wait_on_recovery) + dlm_wait_for_node_recovery(dlm, bit, 10000); + } + + /* must wait for lock to be mastered elsewhere */ + if (blocked) + goto wait; + + dlm_node_iter_init(mle->vote_map, &iter); + while ((nodenum = dlm_node_iter_next(&iter)) >= 0) { + ret = dlm_do_master_request(res, mle, nodenum); + if (ret < 0) + mlog_errno(ret); + if (mle->master != O2NM_MAX_NODES) { + /* found a master ! */ + if (mle->master <= nodenum) + break; + /* if our master request has not reached the master + * yet, keep going until it does. this is how the + * master will know that asserts are needed back to + * the lower nodes. */ + mlog(0, "%s: res %.*s, Requests only up to %u but " + "master is %u, keep going\n", dlm->name, namelen, + lockid, nodenum, mle->master); + } + } + +wait: + /* keep going until the response map includes all nodes */ + ret = dlm_wait_for_lock_mastery(dlm, res, mle, &blocked); + if (ret < 0) { + wait_on_recovery = 1; + mlog(0, "%s: res %.*s, Node map changed, redo the master " + "request now, blocked=%d\n", dlm->name, res->lockname.len, + res->lockname.name, blocked); + if (++tries > 20) { + mlog(ML_ERROR, "%s: res %.*s, Spinning on " + "dlm_wait_for_lock_mastery, blocked = %d\n", + dlm->name, res->lockname.len, + res->lockname.name, blocked); + dlm_print_one_lock_resource(res); + dlm_print_one_mle(mle); + tries = 0; + } + goto redo_request; + } + + mlog(0, "%s: res %.*s, Mastered by %u\n", dlm->name, res->lockname.len, + res->lockname.name, res->owner); + /* make sure we never continue without this */ + BUG_ON(res->owner == O2NM_MAX_NODES); + + /* master is known, detach if not already detached */ + dlm_mle_detach_hb_events(dlm, mle); + dlm_put_mle(mle); + /* put the extra ref */ + dlm_put_mle_inuse(mle); + +wake_waiters: + spin_lock(&res->spinlock); + res->state &= ~DLM_LOCK_RES_IN_PROGRESS; + spin_unlock(&res->spinlock); + wake_up(&res->wq); + +leave: + /* need to free the unused mle */ + if (alloc_mle) + kmem_cache_free(dlm_mle_cache, alloc_mle); + + return res; +} + + +#define DLM_MASTERY_TIMEOUT_MS 5000 + +static int dlm_wait_for_lock_mastery(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_master_list_entry *mle, + int *blocked) +{ + u8 m; + int ret, bit; + int map_changed, voting_done; + int assert, sleep; + +recheck: + ret = 0; + assert = 0; + + /* check if another node has already become the owner */ + spin_lock(&res->spinlock); + if (res->owner != DLM_LOCK_RES_OWNER_UNKNOWN) { + mlog(0, "%s:%.*s: owner is suddenly %u\n", dlm->name, + res->lockname.len, res->lockname.name, res->owner); + spin_unlock(&res->spinlock); + /* this will cause the master to re-assert across + * the whole cluster, freeing up mles */ + if (res->owner != dlm->node_num) { + ret = dlm_do_master_request(res, mle, res->owner); + if (ret < 0) { + /* give recovery a chance to run */ + mlog(ML_ERROR, "link to %u went down?: %d\n", res->owner, ret); + msleep(500); + goto recheck; + } + } + ret = 0; + goto leave; + } + spin_unlock(&res->spinlock); + + spin_lock(&mle->spinlock); + m = mle->master; + map_changed = !bitmap_equal(mle->vote_map, mle->node_map, + O2NM_MAX_NODES); + voting_done = bitmap_equal(mle->vote_map, mle->response_map, + O2NM_MAX_NODES); + + /* restart if we hit any errors */ + if (map_changed) { + int b; + mlog(0, "%s: %.*s: node map changed, restarting\n", + dlm->name, res->lockname.len, res->lockname.name); + ret = dlm_restart_lock_mastery(dlm, res, mle, *blocked); + b = (mle->type == DLM_MLE_BLOCK); + if ((*blocked && !b) || (!*blocked && b)) { + mlog(0, "%s:%.*s: status change: old=%d new=%d\n", + dlm->name, res->lockname.len, res->lockname.name, + *blocked, b); + *blocked = b; + } + spin_unlock(&mle->spinlock); + if (ret < 0) { + mlog_errno(ret); + goto leave; + } + mlog(0, "%s:%.*s: restart lock mastery succeeded, " + "rechecking now\n", dlm->name, res->lockname.len, + res->lockname.name); + goto recheck; + } else { + if (!voting_done) { + mlog(0, "map not changed and voting not done " + "for %s:%.*s\n", dlm->name, res->lockname.len, + res->lockname.name); + } + } + + if (m != O2NM_MAX_NODES) { + /* another node has done an assert! + * all done! */ + sleep = 0; + } else { + sleep = 1; + /* have all nodes responded? */ + if (voting_done && !*blocked) { + bit = find_first_bit(mle->maybe_map, O2NM_MAX_NODES); + if (dlm->node_num <= bit) { + /* my node number is lowest. + * now tell other nodes that I am + * mastering this. */ + mle->master = dlm->node_num; + /* ref was grabbed in get_lock_resource + * will be dropped in dlmlock_master */ + assert = 1; + sleep = 0; + } + /* if voting is done, but we have not received + * an assert master yet, we must sleep */ + } + } + + spin_unlock(&mle->spinlock); + + /* sleep if we haven't finished voting yet */ + if (sleep) { + unsigned long timeo = msecs_to_jiffies(DLM_MASTERY_TIMEOUT_MS); + atomic_set(&mle->woken, 0); + (void)wait_event_timeout(mle->wq, + (atomic_read(&mle->woken) == 1), + timeo); + if (res->owner == O2NM_MAX_NODES) { + mlog(0, "%s:%.*s: waiting again\n", dlm->name, + res->lockname.len, res->lockname.name); + goto recheck; + } + mlog(0, "done waiting, master is %u\n", res->owner); + ret = 0; + goto leave; + } + + ret = 0; /* done */ + if (assert) { + m = dlm->node_num; + mlog(0, "about to master %.*s here, this=%u\n", + res->lockname.len, res->lockname.name, m); + ret = dlm_do_assert_master(dlm, res, mle->vote_map, 0); + if (ret) { + /* This is a failure in the network path, + * not in the response to the assert_master + * (any nonzero response is a BUG on this node). + * Most likely a socket just got disconnected + * due to node death. */ + mlog_errno(ret); + } + /* no longer need to restart lock mastery. + * all living nodes have been contacted. */ + ret = 0; + } + + /* set the lockres owner */ + spin_lock(&res->spinlock); + /* mastery reference obtained either during + * assert_master_handler or in get_lock_resource */ + dlm_change_lockres_owner(dlm, res, m); + spin_unlock(&res->spinlock); + +leave: + return ret; +} + +struct dlm_bitmap_diff_iter +{ + int curnode; + unsigned long *orig_bm; + unsigned long *cur_bm; + unsigned long diff_bm[BITS_TO_LONGS(O2NM_MAX_NODES)]; +}; + +enum dlm_node_state_change +{ + NODE_DOWN = -1, + NODE_NO_CHANGE = 0, + NODE_UP +}; + +static void dlm_bitmap_diff_iter_init(struct dlm_bitmap_diff_iter *iter, + unsigned long *orig_bm, + unsigned long *cur_bm) +{ + unsigned long p1, p2; + int i; + + iter->curnode = -1; + iter->orig_bm = orig_bm; + iter->cur_bm = cur_bm; + + for (i = 0; i < BITS_TO_LONGS(O2NM_MAX_NODES); i++) { + p1 = *(iter->orig_bm + i); + p2 = *(iter->cur_bm + i); + iter->diff_bm[i] = (p1 & ~p2) | (p2 & ~p1); + } +} + +static int dlm_bitmap_diff_iter_next(struct dlm_bitmap_diff_iter *iter, + enum dlm_node_state_change *state) +{ + int bit; + + if (iter->curnode >= O2NM_MAX_NODES) + return -ENOENT; + + bit = find_next_bit(iter->diff_bm, O2NM_MAX_NODES, + iter->curnode+1); + if (bit >= O2NM_MAX_NODES) { + iter->curnode = O2NM_MAX_NODES; + return -ENOENT; + } + + /* if it was there in the original then this node died */ + if (test_bit(bit, iter->orig_bm)) + *state = NODE_DOWN; + else + *state = NODE_UP; + + iter->curnode = bit; + return bit; +} + + +static int dlm_restart_lock_mastery(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_master_list_entry *mle, + int blocked) +{ + struct dlm_bitmap_diff_iter bdi; + enum dlm_node_state_change sc; + int node; + int ret = 0; + + mlog(0, "something happened such that the " + "master process may need to be restarted!\n"); + + assert_spin_locked(&mle->spinlock); + + dlm_bitmap_diff_iter_init(&bdi, mle->vote_map, mle->node_map); + node = dlm_bitmap_diff_iter_next(&bdi, &sc); + while (node >= 0) { + if (sc == NODE_UP) { + /* a node came up. clear any old vote from + * the response map and set it in the vote map + * then restart the mastery. */ + mlog(ML_NOTICE, "node %d up while restarting\n", node); + + /* redo the master request, but only for the new node */ + mlog(0, "sending request to new node\n"); + clear_bit(node, mle->response_map); + set_bit(node, mle->vote_map); + } else { + mlog(ML_ERROR, "node down! %d\n", node); + if (blocked) { + int lowest = find_first_bit(mle->maybe_map, + O2NM_MAX_NODES); + + /* act like it was never there */ + clear_bit(node, mle->maybe_map); + + if (node == lowest) { + mlog(0, "expected master %u died" + " while this node was blocked " + "waiting on it!\n", node); + lowest = find_next_bit(mle->maybe_map, + O2NM_MAX_NODES, + lowest+1); + if (lowest < O2NM_MAX_NODES) { + mlog(0, "%s:%.*s:still " + "blocked. waiting on %u " + "now\n", dlm->name, + res->lockname.len, + res->lockname.name, + lowest); + } else { + /* mle is an MLE_BLOCK, but + * there is now nothing left to + * block on. we need to return + * all the way back out and try + * again with an MLE_MASTER. + * dlm_do_local_recovery_cleanup + * has already run, so the mle + * refcount is ok */ + mlog(0, "%s:%.*s: no " + "longer blocking. try to " + "master this here\n", + dlm->name, + res->lockname.len, + res->lockname.name); + mle->type = DLM_MLE_MASTER; + mle->mleres = res; + } + } + } + + /* now blank out everything, as if we had never + * contacted anyone */ + bitmap_zero(mle->maybe_map, O2NM_MAX_NODES); + bitmap_zero(mle->response_map, O2NM_MAX_NODES); + /* reset the vote_map to the current node_map */ + bitmap_copy(mle->vote_map, mle->node_map, + O2NM_MAX_NODES); + /* put myself into the maybe map */ + if (mle->type != DLM_MLE_BLOCK) + set_bit(dlm->node_num, mle->maybe_map); + } + ret = -EAGAIN; + node = dlm_bitmap_diff_iter_next(&bdi, &sc); + } + return ret; +} + + +/* + * DLM_MASTER_REQUEST_MSG + * + * returns: 0 on success, + * -errno on a network error + * + * on error, the caller should assume the target node is "dead" + * + */ + +static int dlm_do_master_request(struct dlm_lock_resource *res, + struct dlm_master_list_entry *mle, int to) +{ + struct dlm_ctxt *dlm = mle->dlm; + struct dlm_master_request request; + int ret, response=0, resend; + + memset(&request, 0, sizeof(request)); + request.node_idx = dlm->node_num; + + BUG_ON(mle->type == DLM_MLE_MIGRATION); + + request.namelen = (u8)mle->mnamelen; + memcpy(request.name, mle->mname, request.namelen); + +again: + ret = o2net_send_message(DLM_MASTER_REQUEST_MSG, dlm->key, &request, + sizeof(request), to, &response); + if (ret < 0) { + if (ret == -ESRCH) { + /* should never happen */ + mlog(ML_ERROR, "TCP stack not ready!\n"); + BUG(); + } else if (ret == -EINVAL) { + mlog(ML_ERROR, "bad args passed to o2net!\n"); + BUG(); + } else if (ret == -ENOMEM) { + mlog(ML_ERROR, "out of memory while trying to send " + "network message! retrying\n"); + /* this is totally crude */ + msleep(50); + goto again; + } else if (!dlm_is_host_down(ret)) { + /* not a network error. bad. */ + mlog_errno(ret); + mlog(ML_ERROR, "unhandled error!"); + BUG(); + } + /* all other errors should be network errors, + * and likely indicate node death */ + mlog(ML_ERROR, "link to %d went down!\n", to); + goto out; + } + + ret = 0; + resend = 0; + spin_lock(&mle->spinlock); + switch (response) { + case DLM_MASTER_RESP_YES: + set_bit(to, mle->response_map); + mlog(0, "node %u is the master, response=YES\n", to); + mlog(0, "%s:%.*s: master node %u now knows I have a " + "reference\n", dlm->name, res->lockname.len, + res->lockname.name, to); + mle->master = to; + break; + case DLM_MASTER_RESP_NO: + mlog(0, "node %u not master, response=NO\n", to); + set_bit(to, mle->response_map); + break; + case DLM_MASTER_RESP_MAYBE: + mlog(0, "node %u not master, response=MAYBE\n", to); + set_bit(to, mle->response_map); + set_bit(to, mle->maybe_map); + break; + case DLM_MASTER_RESP_ERROR: + mlog(0, "node %u hit an error, resending\n", to); + resend = 1; + response = 0; + break; + default: + mlog(ML_ERROR, "bad response! %u\n", response); + BUG(); + } + spin_unlock(&mle->spinlock); + if (resend) { + /* this is also totally crude */ + msleep(50); + goto again; + } + +out: + return ret; +} + +/* + * locks that can be taken here: + * dlm->spinlock + * res->spinlock + * mle->spinlock + * dlm->master_list + * + * if possible, TRIM THIS DOWN!!! + */ +int dlm_master_request_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data) +{ + u8 response = DLM_MASTER_RESP_MAYBE; + struct dlm_ctxt *dlm = data; + struct dlm_lock_resource *res = NULL; + struct dlm_master_request *request = (struct dlm_master_request *) msg->buf; + struct dlm_master_list_entry *mle = NULL, *tmpmle = NULL; + char *name; + unsigned int namelen, hash; + int found, ret; + int set_maybe; + int dispatch_assert = 0; + int dispatched = 0; + + if (!dlm_grab(dlm)) + return DLM_MASTER_RESP_NO; + + if (!dlm_domain_fully_joined(dlm)) { + response = DLM_MASTER_RESP_NO; + goto send_response; + } + + name = request->name; + namelen = request->namelen; + hash = dlm_lockid_hash(name, namelen); + + if (namelen > DLM_LOCKID_NAME_MAX) { + response = DLM_IVBUFLEN; + goto send_response; + } + +way_up_top: + spin_lock(&dlm->spinlock); + res = __dlm_lookup_lockres(dlm, name, namelen, hash); + if (res) { + spin_unlock(&dlm->spinlock); + + /* take care of the easy cases up front */ + spin_lock(&res->spinlock); + + /* + * Right after dlm spinlock was released, dlm_thread could have + * purged the lockres. Check if lockres got unhashed. If so + * start over. + */ + if (hlist_unhashed(&res->hash_node)) { + spin_unlock(&res->spinlock); + dlm_lockres_put(res); + goto way_up_top; + } + + if (res->state & (DLM_LOCK_RES_RECOVERING| + DLM_LOCK_RES_MIGRATING)) { + spin_unlock(&res->spinlock); + mlog(0, "returning DLM_MASTER_RESP_ERROR since res is " + "being recovered/migrated\n"); + response = DLM_MASTER_RESP_ERROR; + if (mle) + kmem_cache_free(dlm_mle_cache, mle); + goto send_response; + } + + if (res->owner == dlm->node_num) { + dlm_lockres_set_refmap_bit(dlm, res, request->node_idx); + spin_unlock(&res->spinlock); + response = DLM_MASTER_RESP_YES; + if (mle) + kmem_cache_free(dlm_mle_cache, mle); + + /* this node is the owner. + * there is some extra work that needs to + * happen now. the requesting node has + * caused all nodes up to this one to + * create mles. this node now needs to + * go back and clean those up. */ + dispatch_assert = 1; + goto send_response; + } else if (res->owner != DLM_LOCK_RES_OWNER_UNKNOWN) { + spin_unlock(&res->spinlock); + response = DLM_MASTER_RESP_NO; + if (mle) + kmem_cache_free(dlm_mle_cache, mle); + goto send_response; + } + + /* ok, there is no owner. either this node is + * being blocked, or it is actively trying to + * master this lock. */ + if (!(res->state & DLM_LOCK_RES_IN_PROGRESS)) { + mlog(ML_ERROR, "lock with no owner should be " + "in-progress!\n"); + BUG(); + } + + spin_lock(&dlm->master_lock); + found = dlm_find_mle(dlm, &tmpmle, name, namelen); + if (!found) { + mlog(ML_ERROR, "no mle found for this lock!\n"); + BUG(); + } + set_maybe = 1; + spin_lock(&tmpmle->spinlock); + if (tmpmle->type == DLM_MLE_BLOCK) { + response = DLM_MASTER_RESP_NO; + } else if (tmpmle->type == DLM_MLE_MIGRATION) { + mlog(0, "node %u is master, but trying to migrate to " + "node %u.\n", tmpmle->master, tmpmle->new_master); + if (tmpmle->master == dlm->node_num) { + mlog(ML_ERROR, "no owner on lockres, but this " + "node is trying to migrate it to %u?!\n", + tmpmle->new_master); + BUG(); + } else { + /* the real master can respond on its own */ + response = DLM_MASTER_RESP_NO; + } + } else if (tmpmle->master != DLM_LOCK_RES_OWNER_UNKNOWN) { + set_maybe = 0; + if (tmpmle->master == dlm->node_num) { + response = DLM_MASTER_RESP_YES; + /* this node will be the owner. + * go back and clean the mles on any + * other nodes */ + dispatch_assert = 1; + dlm_lockres_set_refmap_bit(dlm, res, + request->node_idx); + } else + response = DLM_MASTER_RESP_NO; + } else { + response = DLM_MASTER_RESP_MAYBE; + } + if (set_maybe) + set_bit(request->node_idx, tmpmle->maybe_map); + spin_unlock(&tmpmle->spinlock); + + spin_unlock(&dlm->master_lock); + spin_unlock(&res->spinlock); + + /* keep the mle attached to heartbeat events */ + dlm_put_mle(tmpmle); + if (mle) + kmem_cache_free(dlm_mle_cache, mle); + goto send_response; + } + + /* + * lockres doesn't exist on this node + * if there is an MLE_BLOCK, return NO + * if there is an MLE_MASTER, return MAYBE + * otherwise, add an MLE_BLOCK, return NO + */ + spin_lock(&dlm->master_lock); + found = dlm_find_mle(dlm, &tmpmle, name, namelen); + if (!found) { + /* this lockid has never been seen on this node yet */ + if (!mle) { + spin_unlock(&dlm->master_lock); + spin_unlock(&dlm->spinlock); + + mle = kmem_cache_alloc(dlm_mle_cache, GFP_NOFS); + if (!mle) { + response = DLM_MASTER_RESP_ERROR; + mlog_errno(-ENOMEM); + goto send_response; + } + goto way_up_top; + } + + dlm_init_mle(mle, DLM_MLE_BLOCK, dlm, NULL, name, namelen); + set_bit(request->node_idx, mle->maybe_map); + __dlm_insert_mle(dlm, mle); + response = DLM_MASTER_RESP_NO; + } else { + spin_lock(&tmpmle->spinlock); + if (tmpmle->master == dlm->node_num) { + mlog(ML_ERROR, "no lockres, but an mle with this node as master!\n"); + BUG(); + } + if (tmpmle->type == DLM_MLE_BLOCK) + response = DLM_MASTER_RESP_NO; + else if (tmpmle->type == DLM_MLE_MIGRATION) { + mlog(0, "migration mle was found (%u->%u)\n", + tmpmle->master, tmpmle->new_master); + /* real master can respond on its own */ + response = DLM_MASTER_RESP_NO; + } else + response = DLM_MASTER_RESP_MAYBE; + set_bit(request->node_idx, tmpmle->maybe_map); + spin_unlock(&tmpmle->spinlock); + } + spin_unlock(&dlm->master_lock); + spin_unlock(&dlm->spinlock); + + if (found) { + /* keep the mle attached to heartbeat events */ + dlm_put_mle(tmpmle); + } +send_response: + /* + * __dlm_lookup_lockres() grabbed a reference to this lockres. + * The reference is released by dlm_assert_master_worker() under + * the call to dlm_dispatch_assert_master(). If + * dlm_assert_master_worker() isn't called, we drop it here. + */ + if (dispatch_assert) { + mlog(0, "%u is the owner of %.*s, cleaning everyone else\n", + dlm->node_num, res->lockname.len, res->lockname.name); + spin_lock(&res->spinlock); + ret = dlm_dispatch_assert_master(dlm, res, 0, request->node_idx, + DLM_ASSERT_MASTER_MLE_CLEANUP); + if (ret < 0) { + mlog(ML_ERROR, "failed to dispatch assert master work\n"); + response = DLM_MASTER_RESP_ERROR; + spin_unlock(&res->spinlock); + dlm_lockres_put(res); + } else { + dispatched = 1; + __dlm_lockres_grab_inflight_worker(dlm, res); + spin_unlock(&res->spinlock); + } + } else { + if (res) + dlm_lockres_put(res); + } + + if (!dispatched) + dlm_put(dlm); + return response; +} + +/* + * DLM_ASSERT_MASTER_MSG + */ + + +/* + * NOTE: this can be used for debugging + * can periodically run all locks owned by this node + * and re-assert across the cluster... + */ +static int dlm_do_assert_master(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + void *nodemap, u32 flags) +{ + struct dlm_assert_master assert; + int to, tmpret; + struct dlm_node_iter iter; + int ret = 0; + int reassert; + const char *lockname = res->lockname.name; + unsigned int namelen = res->lockname.len; + + BUG_ON(namelen > O2NM_MAX_NAME_LEN); + + spin_lock(&res->spinlock); + res->state |= DLM_LOCK_RES_SETREF_INPROG; + spin_unlock(&res->spinlock); + +again: + reassert = 0; + + /* note that if this nodemap is empty, it returns 0 */ + dlm_node_iter_init(nodemap, &iter); + while ((to = dlm_node_iter_next(&iter)) >= 0) { + int r = 0; + struct dlm_master_list_entry *mle = NULL; + + mlog(0, "sending assert master to %d (%.*s)\n", to, + namelen, lockname); + memset(&assert, 0, sizeof(assert)); + assert.node_idx = dlm->node_num; + assert.namelen = namelen; + memcpy(assert.name, lockname, namelen); + assert.flags = cpu_to_be32(flags); + + tmpret = o2net_send_message(DLM_ASSERT_MASTER_MSG, dlm->key, + &assert, sizeof(assert), to, &r); + if (tmpret < 0) { + mlog(ML_ERROR, "Error %d when sending message %u (key " + "0x%x) to node %u\n", tmpret, + DLM_ASSERT_MASTER_MSG, dlm->key, to); + if (!dlm_is_host_down(tmpret)) { + mlog(ML_ERROR, "unhandled error=%d!\n", tmpret); + BUG(); + } + /* a node died. finish out the rest of the nodes. */ + mlog(0, "link to %d went down!\n", to); + /* any nonzero status return will do */ + ret = tmpret; + r = 0; + } else if (r < 0) { + /* ok, something horribly messed. kill thyself. */ + mlog(ML_ERROR,"during assert master of %.*s to %u, " + "got %d.\n", namelen, lockname, to, r); + spin_lock(&dlm->spinlock); + spin_lock(&dlm->master_lock); + if (dlm_find_mle(dlm, &mle, (char *)lockname, + namelen)) { + dlm_print_one_mle(mle); + __dlm_put_mle(mle); + } + spin_unlock(&dlm->master_lock); + spin_unlock(&dlm->spinlock); + BUG(); + } + + if (r & DLM_ASSERT_RESPONSE_REASSERT && + !(r & DLM_ASSERT_RESPONSE_MASTERY_REF)) { + mlog(ML_ERROR, "%.*s: very strange, " + "master MLE but no lockres on %u\n", + namelen, lockname, to); + } + + if (r & DLM_ASSERT_RESPONSE_REASSERT) { + mlog(0, "%.*s: node %u create mles on other " + "nodes and requests a re-assert\n", + namelen, lockname, to); + reassert = 1; + } + if (r & DLM_ASSERT_RESPONSE_MASTERY_REF) { + mlog(0, "%.*s: node %u has a reference to this " + "lockres, set the bit in the refmap\n", + namelen, lockname, to); + spin_lock(&res->spinlock); + dlm_lockres_set_refmap_bit(dlm, res, to); + spin_unlock(&res->spinlock); + } + } + + if (reassert) + goto again; + + spin_lock(&res->spinlock); + res->state &= ~DLM_LOCK_RES_SETREF_INPROG; + spin_unlock(&res->spinlock); + wake_up(&res->wq); + + return ret; +} + +/* + * locks that can be taken here: + * dlm->spinlock + * res->spinlock + * mle->spinlock + * dlm->master_list + * + * if possible, TRIM THIS DOWN!!! + */ +int dlm_assert_master_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data) +{ + struct dlm_ctxt *dlm = data; + struct dlm_master_list_entry *mle = NULL; + struct dlm_assert_master *assert = (struct dlm_assert_master *)msg->buf; + struct dlm_lock_resource *res = NULL; + char *name; + unsigned int namelen, hash; + u32 flags; + int master_request = 0, have_lockres_ref = 0; + int ret = 0; + + if (!dlm_grab(dlm)) + return 0; + + name = assert->name; + namelen = assert->namelen; + hash = dlm_lockid_hash(name, namelen); + flags = be32_to_cpu(assert->flags); + + if (namelen > DLM_LOCKID_NAME_MAX) { + mlog(ML_ERROR, "Invalid name length!"); + goto done; + } + + spin_lock(&dlm->spinlock); + + if (flags) + mlog(0, "assert_master with flags: %u\n", flags); + + /* find the MLE */ + spin_lock(&dlm->master_lock); + if (!dlm_find_mle(dlm, &mle, name, namelen)) { + /* not an error, could be master just re-asserting */ + mlog(0, "just got an assert_master from %u, but no " + "MLE for it! (%.*s)\n", assert->node_idx, + namelen, name); + } else { + int bit = find_first_bit(mle->maybe_map, O2NM_MAX_NODES); + if (bit >= O2NM_MAX_NODES) { + /* not necessarily an error, though less likely. + * could be master just re-asserting. */ + mlog(0, "no bits set in the maybe_map, but %u " + "is asserting! (%.*s)\n", assert->node_idx, + namelen, name); + } else if (bit != assert->node_idx) { + if (flags & DLM_ASSERT_MASTER_MLE_CLEANUP) { + mlog(0, "master %u was found, %u should " + "back off\n", assert->node_idx, bit); + } else { + /* with the fix for bug 569, a higher node + * number winning the mastery will respond + * YES to mastery requests, but this node + * had no way of knowing. let it pass. */ + mlog(0, "%u is the lowest node, " + "%u is asserting. (%.*s) %u must " + "have begun after %u won.\n", bit, + assert->node_idx, namelen, name, bit, + assert->node_idx); + } + } + if (mle->type == DLM_MLE_MIGRATION) { + if (flags & DLM_ASSERT_MASTER_MLE_CLEANUP) { + mlog(0, "%s:%.*s: got cleanup assert" + " from %u for migration\n", + dlm->name, namelen, name, + assert->node_idx); + } else if (!(flags & DLM_ASSERT_MASTER_FINISH_MIGRATION)) { + mlog(0, "%s:%.*s: got unrelated assert" + " from %u for migration, ignoring\n", + dlm->name, namelen, name, + assert->node_idx); + __dlm_put_mle(mle); + spin_unlock(&dlm->master_lock); + spin_unlock(&dlm->spinlock); + goto done; + } + } + } + spin_unlock(&dlm->master_lock); + + /* ok everything checks out with the MLE + * now check to see if there is a lockres */ + res = __dlm_lookup_lockres(dlm, name, namelen, hash); + if (res) { + spin_lock(&res->spinlock); + if (res->state & DLM_LOCK_RES_RECOVERING) { + mlog(ML_ERROR, "%u asserting but %.*s is " + "RECOVERING!\n", assert->node_idx, namelen, name); + goto kill; + } + if (!mle) { + if (res->owner != DLM_LOCK_RES_OWNER_UNKNOWN && + res->owner != assert->node_idx) { + mlog(ML_ERROR, "DIE! Mastery assert from %u, " + "but current owner is %u! (%.*s)\n", + assert->node_idx, res->owner, namelen, + name); + __dlm_print_one_lock_resource(res); + BUG(); + } + } else if (mle->type != DLM_MLE_MIGRATION) { + if (res->owner != DLM_LOCK_RES_OWNER_UNKNOWN) { + /* owner is just re-asserting */ + if (res->owner == assert->node_idx) { + mlog(0, "owner %u re-asserting on " + "lock %.*s\n", assert->node_idx, + namelen, name); + goto ok; + } + mlog(ML_ERROR, "got assert_master from " + "node %u, but %u is the owner! " + "(%.*s)\n", assert->node_idx, + res->owner, namelen, name); + goto kill; + } + if (!(res->state & DLM_LOCK_RES_IN_PROGRESS)) { + mlog(ML_ERROR, "got assert from %u, but lock " + "with no owner should be " + "in-progress! (%.*s)\n", + assert->node_idx, + namelen, name); + goto kill; + } + } else /* mle->type == DLM_MLE_MIGRATION */ { + /* should only be getting an assert from new master */ + if (assert->node_idx != mle->new_master) { + mlog(ML_ERROR, "got assert from %u, but " + "new master is %u, and old master " + "was %u (%.*s)\n", + assert->node_idx, mle->new_master, + mle->master, namelen, name); + goto kill; + } + + } +ok: + spin_unlock(&res->spinlock); + } + + if (mle) { + int extra_ref = 0; + int nn = -1; + int rr, err = 0; + + spin_lock(&mle->spinlock); + if (mle->type == DLM_MLE_BLOCK || mle->type == DLM_MLE_MIGRATION) + extra_ref = 1; + else { + /* MASTER mle: if any bits set in the response map + * then the calling node needs to re-assert to clear + * up nodes that this node contacted */ + while ((nn = find_next_bit (mle->response_map, O2NM_MAX_NODES, + nn+1)) < O2NM_MAX_NODES) { + if (nn != dlm->node_num && nn != assert->node_idx) { + master_request = 1; + break; + } + } + } + mle->master = assert->node_idx; + atomic_set(&mle->woken, 1); + wake_up(&mle->wq); + spin_unlock(&mle->spinlock); + + if (res) { + int wake = 0; + spin_lock(&res->spinlock); + if (mle->type == DLM_MLE_MIGRATION) { + mlog(0, "finishing off migration of lockres %.*s, " + "from %u to %u\n", + res->lockname.len, res->lockname.name, + dlm->node_num, mle->new_master); + res->state &= ~DLM_LOCK_RES_MIGRATING; + wake = 1; + dlm_change_lockres_owner(dlm, res, mle->new_master); + BUG_ON(res->state & DLM_LOCK_RES_DIRTY); + } else { + dlm_change_lockres_owner(dlm, res, mle->master); + } + spin_unlock(&res->spinlock); + have_lockres_ref = 1; + if (wake) + wake_up(&res->wq); + } + + /* master is known, detach if not already detached. + * ensures that only one assert_master call will happen + * on this mle. */ + spin_lock(&dlm->master_lock); + + rr = kref_read(&mle->mle_refs); + if (mle->inuse > 0) { + if (extra_ref && rr < 3) + err = 1; + else if (!extra_ref && rr < 2) + err = 1; + } else { + if (extra_ref && rr < 2) + err = 1; + else if (!extra_ref && rr < 1) + err = 1; + } + if (err) { + mlog(ML_ERROR, "%s:%.*s: got assert master from %u " + "that will mess up this node, refs=%d, extra=%d, " + "inuse=%d\n", dlm->name, namelen, name, + assert->node_idx, rr, extra_ref, mle->inuse); + dlm_print_one_mle(mle); + } + __dlm_unlink_mle(dlm, mle); + __dlm_mle_detach_hb_events(dlm, mle); + __dlm_put_mle(mle); + if (extra_ref) { + /* the assert master message now balances the extra + * ref given by the master / migration request message. + * if this is the last put, it will be removed + * from the list. */ + __dlm_put_mle(mle); + } + spin_unlock(&dlm->master_lock); + } else if (res) { + if (res->owner != assert->node_idx) { + mlog(0, "assert_master from %u, but current " + "owner is %u (%.*s), no mle\n", assert->node_idx, + res->owner, namelen, name); + } + } + spin_unlock(&dlm->spinlock); + +done: + ret = 0; + if (res) { + spin_lock(&res->spinlock); + res->state |= DLM_LOCK_RES_SETREF_INPROG; + spin_unlock(&res->spinlock); + *ret_data = (void *)res; + } + dlm_put(dlm); + if (master_request) { + mlog(0, "need to tell master to reassert\n"); + /* positive. negative would shoot down the node. */ + ret |= DLM_ASSERT_RESPONSE_REASSERT; + if (!have_lockres_ref) { + mlog(ML_ERROR, "strange, got assert from %u, MASTER " + "mle present here for %s:%.*s, but no lockres!\n", + assert->node_idx, dlm->name, namelen, name); + } + } + if (have_lockres_ref) { + /* let the master know we have a reference to the lockres */ + ret |= DLM_ASSERT_RESPONSE_MASTERY_REF; + mlog(0, "%s:%.*s: got assert from %u, need a ref\n", + dlm->name, namelen, name, assert->node_idx); + } + return ret; + +kill: + /* kill the caller! */ + mlog(ML_ERROR, "Bad message received from another node. Dumping state " + "and killing the other node now! This node is OK and can continue.\n"); + __dlm_print_one_lock_resource(res); + spin_unlock(&res->spinlock); + spin_lock(&dlm->master_lock); + if (mle) + __dlm_put_mle(mle); + spin_unlock(&dlm->master_lock); + spin_unlock(&dlm->spinlock); + *ret_data = (void *)res; + dlm_put(dlm); + return -EINVAL; +} + +void dlm_assert_master_post_handler(int status, void *data, void *ret_data) +{ + struct dlm_lock_resource *res = (struct dlm_lock_resource *)ret_data; + + if (ret_data) { + spin_lock(&res->spinlock); + res->state &= ~DLM_LOCK_RES_SETREF_INPROG; + spin_unlock(&res->spinlock); + wake_up(&res->wq); + dlm_lockres_put(res); + } + return; +} + +int dlm_dispatch_assert_master(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + int ignore_higher, u8 request_from, u32 flags) +{ + struct dlm_work_item *item; + item = kzalloc_obj(*item, GFP_ATOMIC); + if (!item) + return -ENOMEM; + + + /* queue up work for dlm_assert_master_worker */ + dlm_init_work_item(dlm, item, dlm_assert_master_worker, NULL); + item->u.am.lockres = res; /* already have a ref */ + /* can optionally ignore node numbers higher than this node */ + item->u.am.ignore_higher = ignore_higher; + item->u.am.request_from = request_from; + item->u.am.flags = flags; + + if (ignore_higher) + mlog(0, "IGNORE HIGHER: %.*s\n", res->lockname.len, + res->lockname.name); + + spin_lock(&dlm->work_lock); + list_add_tail(&item->list, &dlm->work_list); + spin_unlock(&dlm->work_lock); + + queue_work(dlm->dlm_worker, &dlm->dispatched_work); + return 0; +} + +static void dlm_assert_master_worker(struct dlm_work_item *item, void *data) +{ + struct dlm_ctxt *dlm = data; + int ret = 0; + struct dlm_lock_resource *res; + unsigned long nodemap[BITS_TO_LONGS(O2NM_MAX_NODES)]; + int ignore_higher; + int bit; + u8 request_from; + u32 flags; + + dlm = item->dlm; + res = item->u.am.lockres; + ignore_higher = item->u.am.ignore_higher; + request_from = item->u.am.request_from; + flags = item->u.am.flags; + + spin_lock(&dlm->spinlock); + bitmap_copy(nodemap, dlm->domain_map, O2NM_MAX_NODES); + spin_unlock(&dlm->spinlock); + + clear_bit(dlm->node_num, nodemap); + if (ignore_higher) { + /* if is this just to clear up mles for nodes below + * this node, do not send the message to the original + * caller or any node number higher than this */ + clear_bit(request_from, nodemap); + bit = dlm->node_num; + while (1) { + bit = find_next_bit(nodemap, O2NM_MAX_NODES, + bit+1); + if (bit >= O2NM_MAX_NODES) + break; + clear_bit(bit, nodemap); + } + } + + /* + * If we're migrating this lock to someone else, we are no + * longer allowed to assert out own mastery. OTOH, we need to + * prevent migration from starting while we're still asserting + * our dominance. The reserved ast delays migration. + */ + spin_lock(&res->spinlock); + if (res->state & DLM_LOCK_RES_MIGRATING) { + mlog(0, "Someone asked us to assert mastery, but we're " + "in the middle of migration. Skipping assert, " + "the new master will handle that.\n"); + spin_unlock(&res->spinlock); + goto put; + } else + __dlm_lockres_reserve_ast(res); + spin_unlock(&res->spinlock); + + /* this call now finishes out the nodemap + * even if one or more nodes die */ + mlog(0, "worker about to master %.*s here, this=%u\n", + res->lockname.len, res->lockname.name, dlm->node_num); + ret = dlm_do_assert_master(dlm, res, nodemap, flags); + if (ret < 0) { + /* no need to restart, we are done */ + if (!dlm_is_host_down(ret)) + mlog_errno(ret); + } + + /* Ok, we've asserted ourselves. Let's let migration start. */ + dlm_lockres_release_ast(dlm, res); + +put: + dlm_lockres_drop_inflight_worker(dlm, res); + + dlm_lockres_put(res); + + mlog(0, "finished with dlm_assert_master_worker\n"); +} + +/* SPECIAL CASE for the $RECOVERY lock used by the recovery thread. + * We cannot wait for node recovery to complete to begin mastering this + * lockres because this lockres is used to kick off recovery! ;-) + * So, do a pre-check on all living nodes to see if any of those nodes + * think that $RECOVERY is currently mastered by a dead node. If so, + * we wait a short time to allow that node to get notified by its own + * heartbeat stack, then check again. All $RECOVERY lock resources + * mastered by dead nodes are purged when the heartbeat callback is + * fired, so we can know for sure that it is safe to continue once + * the node returns a live node or no node. */ +static int dlm_pre_master_reco_lockres(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res) +{ + struct dlm_node_iter iter; + int nodenum; + int ret = 0; + u8 master = DLM_LOCK_RES_OWNER_UNKNOWN; + + spin_lock(&dlm->spinlock); + dlm_node_iter_init(dlm->domain_map, &iter); + spin_unlock(&dlm->spinlock); + + while ((nodenum = dlm_node_iter_next(&iter)) >= 0) { + /* do not send to self */ + if (nodenum == dlm->node_num) + continue; + ret = dlm_do_master_requery(dlm, res, nodenum, &master); + if (ret < 0) { + mlog_errno(ret); + if (!dlm_is_host_down(ret)) + BUG(); + /* host is down, so answer for that node would be + * DLM_LOCK_RES_OWNER_UNKNOWN. continue. */ + ret = 0; + } + + if (master != DLM_LOCK_RES_OWNER_UNKNOWN) { + /* check to see if this master is in the recovery map */ + spin_lock(&dlm->spinlock); + if (test_bit(master, dlm->recovery_map)) { + mlog(ML_NOTICE, "%s: node %u has not seen " + "node %u go down yet, and thinks the " + "dead node is mastering the recovery " + "lock. must wait.\n", dlm->name, + nodenum, master); + ret = -EAGAIN; + } + spin_unlock(&dlm->spinlock); + mlog(0, "%s: reco lock master is %u\n", dlm->name, + master); + break; + } + } + return ret; +} + +/* + * DLM_DEREF_LOCKRES_MSG + */ + +int dlm_drop_lockres_ref(struct dlm_ctxt *dlm, struct dlm_lock_resource *res) +{ + struct dlm_deref_lockres deref; + int ret = 0, r; + const char *lockname; + unsigned int namelen; + + lockname = res->lockname.name; + namelen = res->lockname.len; + BUG_ON(namelen > O2NM_MAX_NAME_LEN); + + memset(&deref, 0, sizeof(deref)); + deref.node_idx = dlm->node_num; + deref.namelen = namelen; + memcpy(deref.name, lockname, namelen); + + ret = o2net_send_message(DLM_DEREF_LOCKRES_MSG, dlm->key, + &deref, sizeof(deref), res->owner, &r); + if (ret < 0) + mlog(ML_ERROR, "%s: res %.*s, error %d send DEREF to node %u\n", + dlm->name, namelen, lockname, ret, res->owner); + else if (r < 0) { + /* BAD. other node says I did not have a ref. */ + mlog(ML_ERROR, "%s: res %.*s, DEREF to node %u got %d\n", + dlm->name, namelen, lockname, res->owner, r); + dlm_print_one_lock_resource(res); + if (r == -ENOMEM) + BUG(); + } else + ret = r; + + return ret; +} + +int dlm_deref_lockres_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data) +{ + struct dlm_ctxt *dlm = data; + struct dlm_deref_lockres *deref = (struct dlm_deref_lockres *)msg->buf; + struct dlm_lock_resource *res = NULL; + char *name; + unsigned int namelen; + int ret = -EINVAL; + u8 node; + unsigned int hash; + struct dlm_work_item *item; + int cleared = 0; + int dispatch = 0; + + if (!dlm_grab(dlm)) + return 0; + + name = deref->name; + namelen = deref->namelen; + node = deref->node_idx; + + if (namelen > DLM_LOCKID_NAME_MAX) { + mlog(ML_ERROR, "Invalid name length!"); + goto done; + } + if (deref->node_idx >= O2NM_MAX_NODES) { + mlog(ML_ERROR, "Invalid node number: %u\n", node); + goto done; + } + + hash = dlm_lockid_hash(name, namelen); + + spin_lock(&dlm->spinlock); + res = __dlm_lookup_lockres_full(dlm, name, namelen, hash); + if (!res) { + spin_unlock(&dlm->spinlock); + mlog(ML_ERROR, "%s:%.*s: bad lockres name\n", + dlm->name, namelen, name); + goto done; + } + spin_unlock(&dlm->spinlock); + + spin_lock(&res->spinlock); + if (res->state & DLM_LOCK_RES_SETREF_INPROG) + dispatch = 1; + else { + BUG_ON(res->state & DLM_LOCK_RES_DROPPING_REF); + if (test_bit(node, res->refmap)) { + dlm_lockres_clear_refmap_bit(dlm, res, node); + cleared = 1; + } + } + spin_unlock(&res->spinlock); + + if (!dispatch) { + if (cleared) + dlm_lockres_calc_usage(dlm, res); + else { + mlog(ML_ERROR, "%s:%.*s: node %u trying to drop ref " + "but it is already dropped!\n", dlm->name, + res->lockname.len, res->lockname.name, node); + dlm_print_one_lock_resource(res); + } + ret = DLM_DEREF_RESPONSE_DONE; + goto done; + } + + item = kzalloc_obj(*item, GFP_NOFS); + if (!item) { + ret = -ENOMEM; + mlog_errno(ret); + goto done; + } + + dlm_init_work_item(dlm, item, dlm_deref_lockres_worker, NULL); + item->u.dl.deref_res = res; + item->u.dl.deref_node = node; + + spin_lock(&dlm->work_lock); + list_add_tail(&item->list, &dlm->work_list); + spin_unlock(&dlm->work_lock); + + queue_work(dlm->dlm_worker, &dlm->dispatched_work); + return DLM_DEREF_RESPONSE_INPROG; + +done: + if (res) + dlm_lockres_put(res); + dlm_put(dlm); + + return ret; +} + +int dlm_deref_lockres_done_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data) +{ + struct dlm_ctxt *dlm = data; + struct dlm_deref_lockres_done *deref + = (struct dlm_deref_lockres_done *)msg->buf; + struct dlm_lock_resource *res = NULL; + char *name; + unsigned int namelen; + int ret = -EINVAL; + u8 node; + unsigned int hash; + + if (!dlm_grab(dlm)) + return 0; + + name = deref->name; + namelen = deref->namelen; + node = deref->node_idx; + + if (namelen > DLM_LOCKID_NAME_MAX) { + mlog(ML_ERROR, "Invalid name length!"); + goto done; + } + if (deref->node_idx >= O2NM_MAX_NODES) { + mlog(ML_ERROR, "Invalid node number: %u\n", node); + goto done; + } + + hash = dlm_lockid_hash(name, namelen); + + spin_lock(&dlm->spinlock); + res = __dlm_lookup_lockres_full(dlm, name, namelen, hash); + if (!res) { + spin_unlock(&dlm->spinlock); + mlog(ML_ERROR, "%s:%.*s: bad lockres name\n", + dlm->name, namelen, name); + goto done; + } + + spin_lock(&res->spinlock); + if (!(res->state & DLM_LOCK_RES_DROPPING_REF)) { + spin_unlock(&res->spinlock); + spin_unlock(&dlm->spinlock); + mlog(ML_NOTICE, "%s:%.*s: node %u sends deref done " + "but it is already derefed!\n", dlm->name, + res->lockname.len, res->lockname.name, node); + ret = 0; + goto done; + } + + __dlm_do_purge_lockres(dlm, res); + spin_unlock(&res->spinlock); + wake_up(&res->wq); + + spin_unlock(&dlm->spinlock); + + ret = 0; +done: + if (res) + dlm_lockres_put(res); + dlm_put(dlm); + return ret; +} + +static void dlm_drop_lockres_ref_done(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, u8 node) +{ + struct dlm_deref_lockres_done deref; + int ret = 0, r; + const char *lockname; + unsigned int namelen; + + lockname = res->lockname.name; + namelen = res->lockname.len; + BUG_ON(namelen > O2NM_MAX_NAME_LEN); + + memset(&deref, 0, sizeof(deref)); + deref.node_idx = dlm->node_num; + deref.namelen = namelen; + memcpy(deref.name, lockname, namelen); + + ret = o2net_send_message(DLM_DEREF_LOCKRES_DONE, dlm->key, + &deref, sizeof(deref), node, &r); + if (ret < 0) { + mlog(ML_ERROR, "%s: res %.*s, error %d send DEREF DONE " + " to node %u\n", dlm->name, namelen, + lockname, ret, node); + } else if (r < 0) { + /* ignore the error */ + mlog(ML_ERROR, "%s: res %.*s, DEREF to node %u got %d\n", + dlm->name, namelen, lockname, node, r); + dlm_print_one_lock_resource(res); + } +} + +static void dlm_deref_lockres_worker(struct dlm_work_item *item, void *data) +{ + struct dlm_ctxt *dlm; + struct dlm_lock_resource *res; + u8 node; + u8 cleared = 0; + + dlm = item->dlm; + res = item->u.dl.deref_res; + node = item->u.dl.deref_node; + + spin_lock(&res->spinlock); + BUG_ON(res->state & DLM_LOCK_RES_DROPPING_REF); + __dlm_wait_on_lockres_flags(res, DLM_LOCK_RES_SETREF_INPROG); + if (test_bit(node, res->refmap)) { + dlm_lockres_clear_refmap_bit(dlm, res, node); + cleared = 1; + } + spin_unlock(&res->spinlock); + + dlm_drop_lockres_ref_done(dlm, res, node); + + if (cleared) { + mlog(0, "%s:%.*s node %u ref dropped in dispatch\n", + dlm->name, res->lockname.len, res->lockname.name, node); + dlm_lockres_calc_usage(dlm, res); + } else { + mlog(ML_ERROR, "%s:%.*s: node %u trying to drop ref " + "but it is already dropped!\n", dlm->name, + res->lockname.len, res->lockname.name, node); + dlm_print_one_lock_resource(res); + } + + dlm_lockres_put(res); +} + +/* + * A migratable resource is one that is : + * 1. locally mastered, and, + * 2. zero local locks, and, + * 3. one or more non-local locks, or, one or more references + * Returns 1 if yes, 0 if not. + */ +static int dlm_is_lockres_migratable(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res) +{ + enum dlm_lockres_list idx; + int nonlocal = 0, node_ref; + struct list_head *queue; + struct dlm_lock *lock; + u64 cookie; + + assert_spin_locked(&res->spinlock); + + /* delay migration when the lockres is in MIGRATING state */ + if (res->state & DLM_LOCK_RES_MIGRATING) + return 0; + + /* delay migration when the lockres is in RECOCERING state */ + if (res->state & (DLM_LOCK_RES_RECOVERING| + DLM_LOCK_RES_RECOVERY_WAITING)) + return 0; + + if (res->owner != dlm->node_num) + return 0; + + for (idx = DLM_GRANTED_LIST; idx <= DLM_BLOCKED_LIST; idx++) { + queue = dlm_list_idx_to_ptr(res, idx); + list_for_each_entry(lock, queue, list) { + if (lock->ml.node != dlm->node_num) { + nonlocal++; + continue; + } + cookie = be64_to_cpu(lock->ml.cookie); + mlog(0, "%s: Not migratable res %.*s, lock %u:%llu on " + "%s list\n", dlm->name, res->lockname.len, + res->lockname.name, + dlm_get_lock_cookie_node(cookie), + dlm_get_lock_cookie_seq(cookie), + dlm_list_in_text(idx)); + return 0; + } + } + + if (!nonlocal) { + node_ref = find_first_bit(res->refmap, O2NM_MAX_NODES); + if (node_ref >= O2NM_MAX_NODES) + return 0; + } + + mlog(0, "%s: res %.*s, Migratable\n", dlm->name, res->lockname.len, + res->lockname.name); + + return 1; +} + +/* + * DLM_MIGRATE_LOCKRES + */ + + +static int dlm_migrate_lockres(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, u8 target) +{ + struct dlm_master_list_entry *mle = NULL; + struct dlm_master_list_entry *oldmle = NULL; + struct dlm_migratable_lockres *mres = NULL; + int ret = 0; + const char *name; + unsigned int namelen; + int mle_added = 0; + int wake = 0; + + if (!dlm_grab(dlm)) + return -EINVAL; + + name = res->lockname.name; + namelen = res->lockname.len; + + mlog(0, "%s: Migrating %.*s to node %u\n", dlm->name, namelen, name, + target); + + /* preallocate up front. if this fails, abort */ + ret = -ENOMEM; + mres = kmalloc(PAGE_SIZE, GFP_NOFS); + if (!mres) { + mlog_errno(ret); + goto leave; + } + + mle = kmem_cache_alloc(dlm_mle_cache, GFP_NOFS); + if (!mle) { + mlog_errno(ret); + goto leave; + } + ret = 0; + + /* + * clear any existing master requests and + * add the migration mle to the list + */ + spin_lock(&dlm->spinlock); + spin_lock(&dlm->master_lock); + ret = dlm_add_migration_mle(dlm, res, mle, &oldmle, name, + namelen, target, dlm->node_num); + /* get an extra reference on the mle. + * otherwise the assert_master from the new + * master will destroy this. + */ + if (ret != -EEXIST) + dlm_get_mle_inuse(mle); + + spin_unlock(&dlm->master_lock); + spin_unlock(&dlm->spinlock); + + if (ret == -EEXIST) { + mlog(0, "another process is already migrating it\n"); + goto fail; + } + mle_added = 1; + + /* + * set the MIGRATING flag and flush asts + * if we fail after this we need to re-dirty the lockres + */ + if (dlm_mark_lockres_migrating(dlm, res, target) < 0) { + mlog(ML_ERROR, "tried to migrate %.*s to %u, but " + "the target went down.\n", res->lockname.len, + res->lockname.name, target); + spin_lock(&res->spinlock); + res->state &= ~DLM_LOCK_RES_MIGRATING; + wake = 1; + spin_unlock(&res->spinlock); + ret = -EINVAL; + } + +fail: + if (ret != -EEXIST && oldmle) { + /* master is known, detach if not already detached */ + dlm_mle_detach_hb_events(dlm, oldmle); + dlm_put_mle(oldmle); + } + + if (ret < 0) { + if (mle_added) { + dlm_mle_detach_hb_events(dlm, mle); + dlm_put_mle(mle); + dlm_put_mle_inuse(mle); + } else if (mle) { + kmem_cache_free(dlm_mle_cache, mle); + mle = NULL; + } + goto leave; + } + + /* + * at this point, we have a migration target, an mle + * in the master list, and the MIGRATING flag set on + * the lockres + */ + + /* now that remote nodes are spinning on the MIGRATING flag, + * ensure that all assert_master work is flushed. */ + flush_workqueue(dlm->dlm_worker); + + /* notify new node and send all lock state */ + /* call send_one_lockres with migration flag. + * this serves as notice to the target node that a + * migration is starting. */ + ret = dlm_send_one_lockres(dlm, res, mres, target, + DLM_MRES_MIGRATION); + + if (ret < 0) { + mlog(0, "migration to node %u failed with %d\n", + target, ret); + /* migration failed, detach and clean up mle */ + dlm_mle_detach_hb_events(dlm, mle); + dlm_put_mle(mle); + dlm_put_mle_inuse(mle); + spin_lock(&res->spinlock); + res->state &= ~DLM_LOCK_RES_MIGRATING; + wake = 1; + spin_unlock(&res->spinlock); + if (dlm_is_host_down(ret)) + dlm_wait_for_node_death(dlm, target, + DLM_NODE_DEATH_WAIT_MAX); + goto leave; + } + + /* at this point, the target sends a message to all nodes, + * (using dlm_do_migrate_request). this node is skipped since + * we had to put an mle in the list to begin the process. this + * node now waits for target to do an assert master. this node + * will be the last one notified, ensuring that the migration + * is complete everywhere. if the target dies while this is + * going on, some nodes could potentially see the target as the + * master, so it is important that my recovery finds the migration + * mle and sets the master to UNKNOWN. */ + + + /* wait for new node to assert master */ + while (1) { + ret = wait_event_interruptible_timeout(mle->wq, + (atomic_read(&mle->woken) == 1), + msecs_to_jiffies(5000)); + + if (ret >= 0) { + if (atomic_read(&mle->woken) == 1 || + res->owner == target) + break; + + mlog(0, "%s:%.*s: timed out during migration\n", + dlm->name, res->lockname.len, res->lockname.name); + /* avoid hang during shutdown when migrating lockres + * to a node which also goes down */ + if (dlm_is_node_dead(dlm, target)) { + mlog(0, "%s:%.*s: expected migration " + "target %u is no longer up, restarting\n", + dlm->name, res->lockname.len, + res->lockname.name, target); + ret = -EINVAL; + /* migration failed, detach and clean up mle */ + dlm_mle_detach_hb_events(dlm, mle); + dlm_put_mle(mle); + dlm_put_mle_inuse(mle); + spin_lock(&res->spinlock); + res->state &= ~DLM_LOCK_RES_MIGRATING; + wake = 1; + spin_unlock(&res->spinlock); + goto leave; + } + } else + mlog(0, "%s:%.*s: caught signal during migration\n", + dlm->name, res->lockname.len, res->lockname.name); + } + + /* all done, set the owner, clear the flag */ + spin_lock(&res->spinlock); + dlm_set_lockres_owner(dlm, res, target); + res->state &= ~DLM_LOCK_RES_MIGRATING; + dlm_remove_nonlocal_locks(dlm, res); + spin_unlock(&res->spinlock); + wake_up(&res->wq); + + /* master is known, detach if not already detached */ + dlm_mle_detach_hb_events(dlm, mle); + dlm_put_mle_inuse(mle); + ret = 0; + + dlm_lockres_calc_usage(dlm, res); + +leave: + /* re-dirty the lockres if we failed */ + if (ret < 0) + dlm_kick_thread(dlm, res); + + /* wake up waiters if the MIGRATING flag got set + * but migration failed */ + if (wake) + wake_up(&res->wq); + + kfree(mres); + + dlm_put(dlm); + + mlog(0, "%s: Migrating %.*s to %u, returns %d\n", dlm->name, namelen, + name, target, ret); + return ret; +} + +/* + * Should be called only after beginning the domain leave process. + * There should not be any remaining locks on nonlocal lock resources, + * and there should be no local locks left on locally mastered resources. + * + * Called with the dlm spinlock held, may drop it to do migration, but + * will re-acquire before exit. + * + * Returns: 1 if dlm->spinlock was dropped/retaken, 0 if never dropped + */ +int dlm_empty_lockres(struct dlm_ctxt *dlm, struct dlm_lock_resource *res) + __must_hold(&dlm->spinlock) +{ + int ret; + int lock_dropped = 0; + u8 target = O2NM_MAX_NODES; + + assert_spin_locked(&dlm->spinlock); + + spin_lock(&res->spinlock); + if (dlm_is_lockres_migratable(dlm, res)) + target = dlm_pick_migration_target(dlm, res); + spin_unlock(&res->spinlock); + + if (target == O2NM_MAX_NODES) + goto leave; + + /* Wheee! Migrate lockres here! Will sleep so drop spinlock. */ + spin_unlock(&dlm->spinlock); + lock_dropped = 1; + ret = dlm_migrate_lockres(dlm, res, target); + if (ret) + mlog(0, "%s: res %.*s, Migrate to node %u failed with %d\n", + dlm->name, res->lockname.len, res->lockname.name, + target, ret); + spin_lock(&dlm->spinlock); +leave: + return lock_dropped; +} + +int dlm_lock_basts_flushed(struct dlm_ctxt *dlm, struct dlm_lock *lock) +{ + int ret; + spin_lock(&dlm->ast_lock); + spin_lock(&lock->spinlock); + ret = (list_empty(&lock->bast_list) && !lock->bast_pending); + spin_unlock(&lock->spinlock); + spin_unlock(&dlm->ast_lock); + return ret; +} + +static int dlm_migration_can_proceed(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + u8 mig_target) +{ + int can_proceed; + spin_lock(&res->spinlock); + can_proceed = !!(res->state & DLM_LOCK_RES_MIGRATING); + spin_unlock(&res->spinlock); + + /* target has died, so make the caller break out of the + * wait_event, but caller must recheck the domain_map */ + spin_lock(&dlm->spinlock); + if (!test_bit(mig_target, dlm->domain_map)) + can_proceed = 1; + spin_unlock(&dlm->spinlock); + return can_proceed; +} + +static int dlm_lockres_is_dirty(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res) +{ + int ret; + spin_lock(&res->spinlock); + ret = !!(res->state & DLM_LOCK_RES_DIRTY); + spin_unlock(&res->spinlock); + return ret; +} + + +static int dlm_mark_lockres_migrating(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + u8 target) +{ + int ret = 0; + + mlog(0, "dlm_mark_lockres_migrating: %.*s, from %u to %u\n", + res->lockname.len, res->lockname.name, dlm->node_num, + target); + /* need to set MIGRATING flag on lockres. this is done by + * ensuring that all asts have been flushed for this lockres. */ + spin_lock(&res->spinlock); + BUG_ON(res->migration_pending); + res->migration_pending = 1; + /* strategy is to reserve an extra ast then release + * it below, letting the release do all of the work */ + __dlm_lockres_reserve_ast(res); + spin_unlock(&res->spinlock); + + /* now flush all the pending asts */ + dlm_kick_thread(dlm, res); + /* before waiting on DIRTY, block processes which may + * try to dirty the lockres before MIGRATING is set */ + spin_lock(&res->spinlock); + BUG_ON(res->state & DLM_LOCK_RES_BLOCK_DIRTY); + res->state |= DLM_LOCK_RES_BLOCK_DIRTY; + spin_unlock(&res->spinlock); + /* now wait on any pending asts and the DIRTY state */ + wait_event(dlm->ast_wq, !dlm_lockres_is_dirty(dlm, res)); + dlm_lockres_release_ast(dlm, res); + + mlog(0, "about to wait on migration_wq, dirty=%s\n", + str_yes_no(res->state & DLM_LOCK_RES_DIRTY)); + /* if the extra ref we just put was the final one, this + * will pass thru immediately. otherwise, we need to wait + * for the last ast to finish. */ +again: + ret = wait_event_interruptible_timeout(dlm->migration_wq, + dlm_migration_can_proceed(dlm, res, target), + msecs_to_jiffies(1000)); + if (ret < 0) { + mlog(0, "woken again: migrating? %s, dead? %s\n", + str_yes_no(res->state & DLM_LOCK_RES_MIGRATING), + str_no_yes(test_bit(target, dlm->domain_map))); + } else { + mlog(0, "all is well: migrating? %s, dead? %s\n", + str_yes_no(res->state & DLM_LOCK_RES_MIGRATING), + str_no_yes(test_bit(target, dlm->domain_map))); + } + if (!dlm_migration_can_proceed(dlm, res, target)) { + mlog(0, "trying again...\n"); + goto again; + } + + ret = 0; + /* did the target go down or die? */ + spin_lock(&dlm->spinlock); + if (!test_bit(target, dlm->domain_map)) { + mlog(ML_ERROR, "aha. migration target %u just went down\n", + target); + ret = -EHOSTDOWN; + } + spin_unlock(&dlm->spinlock); + + /* + * if target is down, we need to clear DLM_LOCK_RES_BLOCK_DIRTY for + * another try; otherwise, we are sure the MIGRATING state is there, + * drop the unneeded state which blocked threads trying to DIRTY + */ + spin_lock(&res->spinlock); + BUG_ON(!(res->state & DLM_LOCK_RES_BLOCK_DIRTY)); + res->state &= ~DLM_LOCK_RES_BLOCK_DIRTY; + if (!ret) + BUG_ON(!(res->state & DLM_LOCK_RES_MIGRATING)); + else + res->migration_pending = 0; + spin_unlock(&res->spinlock); + + /* + * at this point: + * + * o the DLM_LOCK_RES_MIGRATING flag is set if target not down + * o there are no pending asts on this lockres + * o all processes trying to reserve an ast on this + * lockres must wait for the MIGRATING flag to clear + */ + return ret; +} + +/* last step in the migration process. + * original master calls this to free all of the dlm_lock + * structures that used to be for other nodes. */ +static void dlm_remove_nonlocal_locks(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res) +{ + struct list_head *queue = &res->granted; + int i, bit; + struct dlm_lock *lock, *next; + + assert_spin_locked(&res->spinlock); + + BUG_ON(res->owner == dlm->node_num); + + for (i=0; i<3; i++) { + list_for_each_entry_safe(lock, next, queue, list) { + if (lock->ml.node != dlm->node_num) { + mlog(0, "putting lock for node %u\n", + lock->ml.node); + /* be extra careful */ + BUG_ON(!list_empty(&lock->ast_list)); + BUG_ON(!list_empty(&lock->bast_list)); + BUG_ON(lock->ast_pending); + BUG_ON(lock->bast_pending); + dlm_lockres_clear_refmap_bit(dlm, res, + lock->ml.node); + list_del_init(&lock->list); + dlm_lock_put(lock); + /* In a normal unlock, we would have added a + * DLM_UNLOCK_FREE_LOCK action. Force it. */ + dlm_lock_put(lock); + } + } + queue++; + } + bit = 0; + while (1) { + bit = find_next_bit(res->refmap, O2NM_MAX_NODES, bit); + if (bit >= O2NM_MAX_NODES) + break; + /* do not clear the local node reference, if there is a + * process holding this, let it drop the ref itself */ + if (bit != dlm->node_num) { + mlog(0, "%s:%.*s: node %u had a ref to this " + "migrating lockres, clearing\n", dlm->name, + res->lockname.len, res->lockname.name, bit); + dlm_lockres_clear_refmap_bit(dlm, res, bit); + } + bit++; + } +} + +/* + * Pick a node to migrate the lock resource to. This function selects a + * potential target based first on the locks and then on refmap. It skips + * nodes that are in the process of exiting the domain. + */ +static u8 dlm_pick_migration_target(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res) +{ + enum dlm_lockres_list idx; + struct list_head *queue; + struct dlm_lock *lock; + int noderef; + u8 nodenum = O2NM_MAX_NODES; + + assert_spin_locked(&dlm->spinlock); + assert_spin_locked(&res->spinlock); + + /* Go through all the locks */ + for (idx = DLM_GRANTED_LIST; idx <= DLM_BLOCKED_LIST; idx++) { + queue = dlm_list_idx_to_ptr(res, idx); + list_for_each_entry(lock, queue, list) { + if (lock->ml.node == dlm->node_num) + continue; + if (test_bit(lock->ml.node, dlm->exit_domain_map)) + continue; + nodenum = lock->ml.node; + goto bail; + } + } + + /* Go thru the refmap */ + noderef = -1; + while (1) { + noderef = find_next_bit(res->refmap, O2NM_MAX_NODES, + noderef + 1); + if (noderef >= O2NM_MAX_NODES) + break; + if (noderef == dlm->node_num) + continue; + if (test_bit(noderef, dlm->exit_domain_map)) + continue; + nodenum = noderef; + goto bail; + } + +bail: + return nodenum; +} + +/* this is called by the new master once all lockres + * data has been received */ +static int dlm_do_migrate_request(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + u8 master, u8 new_master, + struct dlm_node_iter *iter) +{ + struct dlm_migrate_request migrate; + int ret, skip, status = 0; + int nodenum; + + memset(&migrate, 0, sizeof(migrate)); + migrate.namelen = res->lockname.len; + memcpy(migrate.name, res->lockname.name, migrate.namelen); + migrate.new_master = new_master; + migrate.master = master; + + ret = 0; + + /* send message to all nodes, except the master and myself */ + while ((nodenum = dlm_node_iter_next(iter)) >= 0) { + if (nodenum == master || + nodenum == new_master) + continue; + + /* We could race exit domain. If exited, skip. */ + spin_lock(&dlm->spinlock); + skip = (!test_bit(nodenum, dlm->domain_map)); + spin_unlock(&dlm->spinlock); + if (skip) { + clear_bit(nodenum, iter->node_map); + continue; + } + + ret = o2net_send_message(DLM_MIGRATE_REQUEST_MSG, dlm->key, + &migrate, sizeof(migrate), nodenum, + &status); + if (ret < 0) { + mlog(ML_ERROR, "%s: res %.*s, Error %d send " + "MIGRATE_REQUEST to node %u\n", dlm->name, + migrate.namelen, migrate.name, ret, nodenum); + if (!dlm_is_host_down(ret)) { + mlog(ML_ERROR, "unhandled error=%d!\n", ret); + BUG(); + } + clear_bit(nodenum, iter->node_map); + ret = 0; + } else if (status < 0) { + mlog(0, "migrate request (node %u) returned %d!\n", + nodenum, status); + ret = status; + } else if (status == DLM_MIGRATE_RESPONSE_MASTERY_REF) { + /* during the migration request we short-circuited + * the mastery of the lockres. make sure we have + * a mastery ref for nodenum */ + mlog(0, "%s:%.*s: need ref for node %u\n", + dlm->name, res->lockname.len, res->lockname.name, + nodenum); + spin_lock(&res->spinlock); + dlm_lockres_set_refmap_bit(dlm, res, nodenum); + spin_unlock(&res->spinlock); + } + } + + if (ret < 0) + mlog_errno(ret); + + mlog(0, "returning ret=%d\n", ret); + return ret; +} + + +/* if there is an existing mle for this lockres, we now know who the master is. + * (the one who sent us *this* message) we can clear it up right away. + * since the process that put the mle on the list still has a reference to it, + * we can unhash it now, set the master and wake the process. as a result, + * we will have no mle in the list to start with. now we can add an mle for + * the migration and this should be the only one found for those scanning the + * list. */ +int dlm_migrate_request_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data) +{ + struct dlm_ctxt *dlm = data; + struct dlm_lock_resource *res = NULL; + struct dlm_migrate_request *migrate = (struct dlm_migrate_request *) msg->buf; + struct dlm_master_list_entry *mle = NULL, *oldmle = NULL; + const char *name; + unsigned int namelen, hash; + int ret = 0; + + if (!dlm_grab(dlm)) + return 0; + + name = migrate->name; + namelen = migrate->namelen; + if (namelen > DLM_LOCKID_NAME_MAX) { + mlog(ML_ERROR, "%s: invalid name length %u in migrate request\n", + dlm->name, namelen); + ret = -EINVAL; + goto leave; + } + hash = dlm_lockid_hash(name, namelen); + + /* preallocate.. if this fails, abort */ + mle = kmem_cache_alloc(dlm_mle_cache, GFP_NOFS); + + if (!mle) { + ret = -ENOMEM; + goto leave; + } + + /* check for pre-existing lock */ + spin_lock(&dlm->spinlock); + res = __dlm_lookup_lockres(dlm, name, namelen, hash); + if (res) { + spin_lock(&res->spinlock); + if (res->state & DLM_LOCK_RES_RECOVERING) { + /* if all is working ok, this can only mean that we got + * a migrate request from a node that we now see as + * dead. what can we do here? drop it to the floor? */ + spin_unlock(&res->spinlock); + mlog(ML_ERROR, "Got a migrate request, but the " + "lockres is marked as recovering!"); + kmem_cache_free(dlm_mle_cache, mle); + ret = -EINVAL; /* need a better solution */ + goto unlock; + } + res->state |= DLM_LOCK_RES_MIGRATING; + spin_unlock(&res->spinlock); + } + + spin_lock(&dlm->master_lock); + /* ignore status. only nonzero status would BUG. */ + ret = dlm_add_migration_mle(dlm, res, mle, &oldmle, + name, namelen, + migrate->new_master, + migrate->master); + + if (ret < 0) + kmem_cache_free(dlm_mle_cache, mle); + + spin_unlock(&dlm->master_lock); +unlock: + spin_unlock(&dlm->spinlock); + + if (oldmle) { + /* master is known, detach if not already detached */ + dlm_mle_detach_hb_events(dlm, oldmle); + dlm_put_mle(oldmle); + } + + if (res) + dlm_lockres_put(res); +leave: + dlm_put(dlm); + return ret; +} + +/* must be holding dlm->spinlock and dlm->master_lock + * when adding a migration mle, we can clear any other mles + * in the master list because we know with certainty that + * the master is "master". so we remove any old mle from + * the list after setting it's master field, and then add + * the new migration mle. this way we can hold with the rule + * of having only one mle for a given lock name at all times. */ +static int dlm_add_migration_mle(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_master_list_entry *mle, + struct dlm_master_list_entry **oldmle, + const char *name, unsigned int namelen, + u8 new_master, u8 master) +{ + int found; + int ret = 0; + + *oldmle = NULL; + + assert_spin_locked(&dlm->spinlock); + assert_spin_locked(&dlm->master_lock); + + /* caller is responsible for any ref taken here on oldmle */ + found = dlm_find_mle(dlm, oldmle, (char *)name, namelen); + if (found) { + struct dlm_master_list_entry *tmp = *oldmle; + spin_lock(&tmp->spinlock); + if (tmp->type == DLM_MLE_MIGRATION) { + if (master == dlm->node_num) { + /* ah another process raced me to it */ + mlog(0, "tried to migrate %.*s, but some " + "process beat me to it\n", + namelen, name); + spin_unlock(&tmp->spinlock); + return -EEXIST; + } else { + /* bad. 2 NODES are trying to migrate! */ + mlog(ML_ERROR, "migration error mle: " + "master=%u new_master=%u // request: " + "master=%u new_master=%u // " + "lockres=%.*s\n", + tmp->master, tmp->new_master, + master, new_master, + namelen, name); + BUG(); + } + } else { + /* this is essentially what assert_master does */ + tmp->master = master; + atomic_set(&tmp->woken, 1); + wake_up(&tmp->wq); + /* remove it so that only one mle will be found */ + __dlm_unlink_mle(dlm, tmp); + __dlm_mle_detach_hb_events(dlm, tmp); + if (tmp->type == DLM_MLE_MASTER) { + ret = DLM_MIGRATE_RESPONSE_MASTERY_REF; + mlog(0, "%s:%.*s: master=%u, newmaster=%u, " + "telling master to get ref " + "for cleared out mle during " + "migration\n", dlm->name, + namelen, name, master, + new_master); + } + } + spin_unlock(&tmp->spinlock); + } + + /* now add a migration mle to the tail of the list */ + dlm_init_mle(mle, DLM_MLE_MIGRATION, dlm, res, name, namelen); + mle->new_master = new_master; + /* the new master will be sending an assert master for this. + * at that point we will get the refmap reference */ + mle->master = master; + /* do this for consistency with other mle types */ + set_bit(new_master, mle->maybe_map); + __dlm_insert_mle(dlm, mle); + + return ret; +} + +/* + * Sets the owner of the lockres, associated to the mle, to UNKNOWN + */ +static struct dlm_lock_resource *dlm_reset_mleres_owner(struct dlm_ctxt *dlm, + struct dlm_master_list_entry *mle) +{ + struct dlm_lock_resource *res; + + /* Find the lockres associated to the mle and set its owner to UNK */ + res = __dlm_lookup_lockres(dlm, mle->mname, mle->mnamelen, + mle->mnamehash); + if (res) { + spin_unlock(&dlm->master_lock); + + /* move lockres onto recovery list */ + spin_lock(&res->spinlock); + dlm_set_lockres_owner(dlm, res, DLM_LOCK_RES_OWNER_UNKNOWN); + dlm_move_lockres_to_recovery_list(dlm, res); + spin_unlock(&res->spinlock); + dlm_lockres_put(res); + + /* about to get rid of mle, detach from heartbeat */ + __dlm_mle_detach_hb_events(dlm, mle); + + /* dump the mle */ + spin_lock(&dlm->master_lock); + __dlm_put_mle(mle); + spin_unlock(&dlm->master_lock); + } + + return res; +} + +static void dlm_clean_migration_mle(struct dlm_ctxt *dlm, + struct dlm_master_list_entry *mle) +{ + __dlm_mle_detach_hb_events(dlm, mle); + + spin_lock(&mle->spinlock); + __dlm_unlink_mle(dlm, mle); + atomic_set(&mle->woken, 1); + spin_unlock(&mle->spinlock); + + wake_up(&mle->wq); +} + +static void dlm_clean_block_mle(struct dlm_ctxt *dlm, + struct dlm_master_list_entry *mle, u8 dead_node) +{ + int bit; + + BUG_ON(mle->type != DLM_MLE_BLOCK); + + spin_lock(&mle->spinlock); + bit = find_first_bit(mle->maybe_map, O2NM_MAX_NODES); + if (bit != dead_node) { + mlog(0, "mle found, but dead node %u would not have been " + "master\n", dead_node); + spin_unlock(&mle->spinlock); + } else { + /* Must drop the refcount by one since the assert_master will + * never arrive. This may result in the mle being unlinked and + * freed, but there may still be a process waiting in the + * dlmlock path which is fine. */ + mlog(0, "node %u was expected master\n", dead_node); + atomic_set(&mle->woken, 1); + spin_unlock(&mle->spinlock); + wake_up(&mle->wq); + + /* Do not need events any longer, so detach from heartbeat */ + __dlm_mle_detach_hb_events(dlm, mle); + __dlm_put_mle(mle); + } +} + +void dlm_clean_master_list(struct dlm_ctxt *dlm, u8 dead_node) +{ + struct dlm_master_list_entry *mle; + struct dlm_lock_resource *res; + struct hlist_head *bucket; + struct hlist_node *tmp; + unsigned int i; + + mlog(0, "dlm=%s, dead node=%u\n", dlm->name, dead_node); +top: + assert_spin_locked(&dlm->spinlock); + + /* clean the master list */ + spin_lock(&dlm->master_lock); + for (i = 0; i < DLM_HASH_BUCKETS; i++) { + bucket = dlm_master_hash(dlm, i); + hlist_for_each_entry_safe(mle, tmp, bucket, master_hash_node) { + BUG_ON(mle->type != DLM_MLE_BLOCK && + mle->type != DLM_MLE_MASTER && + mle->type != DLM_MLE_MIGRATION); + + /* MASTER mles are initiated locally. The waiting + * process will notice the node map change shortly. + * Let that happen as normal. */ + if (mle->type == DLM_MLE_MASTER) + continue; + + /* BLOCK mles are initiated by other nodes. Need to + * clean up if the dead node would have been the + * master. */ + if (mle->type == DLM_MLE_BLOCK) { + dlm_clean_block_mle(dlm, mle, dead_node); + continue; + } + + /* Everything else is a MIGRATION mle */ + + /* The rule for MIGRATION mles is that the master + * becomes UNKNOWN if *either* the original or the new + * master dies. All UNKNOWN lockres' are sent to + * whichever node becomes the recovery master. The new + * master is responsible for determining if there is + * still a master for this lockres, or if he needs to + * take over mastery. Either way, this node should + * expect another message to resolve this. */ + + if (mle->master != dead_node && + mle->new_master != dead_node) + continue; + + if (mle->new_master == dead_node && mle->inuse) { + mlog(ML_NOTICE, "%s: target %u died during " + "migration from %u, the MLE is " + "still keep used, ignore it!\n", + dlm->name, dead_node, + mle->master); + continue; + } + + /* If we have reached this point, this mle needs to be + * removed from the list and freed. */ + dlm_clean_migration_mle(dlm, mle); + + mlog(0, "%s: node %u died during migration from " + "%u to %u!\n", dlm->name, dead_node, mle->master, + mle->new_master); + + /* If we find a lockres associated with the mle, we've + * hit this rare case that messes up our lock ordering. + * If so, we need to drop the master lock so that we can + * take the lockres lock, meaning that we will have to + * restart from the head of list. */ + res = dlm_reset_mleres_owner(dlm, mle); + if (res) + /* restart */ + goto top; + + /* This may be the last reference */ + __dlm_put_mle(mle); + } + } + spin_unlock(&dlm->master_lock); +} + +int dlm_finish_migration(struct dlm_ctxt *dlm, struct dlm_lock_resource *res, + u8 old_master) +{ + struct dlm_node_iter iter; + int ret = 0; + + spin_lock(&dlm->spinlock); + dlm_node_iter_init(dlm->domain_map, &iter); + clear_bit(old_master, iter.node_map); + clear_bit(dlm->node_num, iter.node_map); + spin_unlock(&dlm->spinlock); + + /* ownership of the lockres is changing. account for the + * mastery reference here since old_master will briefly have + * a reference after the migration completes */ + spin_lock(&res->spinlock); + dlm_lockres_set_refmap_bit(dlm, res, old_master); + spin_unlock(&res->spinlock); + + mlog(0, "now time to do a migrate request to other nodes\n"); + ret = dlm_do_migrate_request(dlm, res, old_master, + dlm->node_num, &iter); + if (ret < 0) { + mlog_errno(ret); + goto leave; + } + + mlog(0, "doing assert master of %.*s to all except the original node\n", + res->lockname.len, res->lockname.name); + /* this call now finishes out the nodemap + * even if one or more nodes die */ + ret = dlm_do_assert_master(dlm, res, iter.node_map, + DLM_ASSERT_MASTER_FINISH_MIGRATION); + if (ret < 0) { + /* no longer need to retry. all living nodes contacted. */ + mlog_errno(ret); + ret = 0; + } + + bitmap_zero(iter.node_map, O2NM_MAX_NODES); + set_bit(old_master, iter.node_map); + mlog(0, "doing assert master of %.*s back to %u\n", + res->lockname.len, res->lockname.name, old_master); + ret = dlm_do_assert_master(dlm, res, iter.node_map, + DLM_ASSERT_MASTER_FINISH_MIGRATION); + if (ret < 0) { + mlog(0, "assert master to original master failed " + "with %d.\n", ret); + /* the only nonzero status here would be because of + * a dead original node. we're done. */ + ret = 0; + } + + /* all done, set the owner, clear the flag */ + spin_lock(&res->spinlock); + dlm_set_lockres_owner(dlm, res, dlm->node_num); + res->state &= ~DLM_LOCK_RES_MIGRATING; + spin_unlock(&res->spinlock); + /* re-dirty it on the new master */ + dlm_kick_thread(dlm, res); + wake_up(&res->wq); +leave: + return ret; +} + +/* + * LOCKRES AST REFCOUNT + * this is integral to migration + */ + +/* for future intent to call an ast, reserve one ahead of time. + * this should be called only after waiting on the lockres + * with dlm_wait_on_lockres, and while still holding the + * spinlock after the call. */ +void __dlm_lockres_reserve_ast(struct dlm_lock_resource *res) +{ + assert_spin_locked(&res->spinlock); + if (res->state & DLM_LOCK_RES_MIGRATING) { + __dlm_print_one_lock_resource(res); + } + BUG_ON(res->state & DLM_LOCK_RES_MIGRATING); + + atomic_inc(&res->asts_reserved); +} + +/* + * used to drop the reserved ast, either because it went unused, + * or because the ast/bast was actually called. + * + * also, if there is a pending migration on this lockres, + * and this was the last pending ast on the lockres, + * atomically set the MIGRATING flag before we drop the lock. + * this is how we ensure that migration can proceed with no + * asts in progress. note that it is ok if the state of the + * queues is such that a lock should be granted in the future + * or that a bast should be fired, because the new master will + * shuffle the lists on this lockres as soon as it is migrated. + */ +void dlm_lockres_release_ast(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res) +{ + if (!atomic_dec_and_lock(&res->asts_reserved, &res->spinlock)) + return; + + if (!res->migration_pending) { + spin_unlock(&res->spinlock); + return; + } + + BUG_ON(res->state & DLM_LOCK_RES_MIGRATING); + res->migration_pending = 0; + res->state |= DLM_LOCK_RES_MIGRATING; + spin_unlock(&res->spinlock); + wake_up(&res->wq); + wake_up(&dlm->migration_wq); +} + +void dlm_force_free_mles(struct dlm_ctxt *dlm) +{ + int i; + struct hlist_head *bucket; + struct dlm_master_list_entry *mle; + struct hlist_node *tmp; + + /* + * We notified all other nodes that we are exiting the domain and + * marked the dlm state to DLM_CTXT_LEAVING. If any mles are still + * around we force free them and wake any processes that are waiting + * on the mles + */ + spin_lock(&dlm->spinlock); + spin_lock(&dlm->master_lock); + + BUG_ON(dlm->dlm_state != DLM_CTXT_LEAVING); + BUG_ON((find_first_bit(dlm->domain_map, O2NM_MAX_NODES) < O2NM_MAX_NODES)); + + for (i = 0; i < DLM_HASH_BUCKETS; i++) { + bucket = dlm_master_hash(dlm, i); + hlist_for_each_entry_safe(mle, tmp, bucket, master_hash_node) { + if (mle->type != DLM_MLE_BLOCK) { + mlog(ML_ERROR, "bad mle: %p\n", mle); + dlm_print_one_mle(mle); + } + atomic_set(&mle->woken, 1); + wake_up(&mle->wq); + + __dlm_unlink_mle(dlm, mle); + __dlm_mle_detach_hb_events(dlm, mle); + __dlm_put_mle(mle); + } + } + spin_unlock(&dlm->master_lock); + spin_unlock(&dlm->spinlock); +} diff --git a/fs/ocfs2/dlm/dlmrecovery.c b/fs/ocfs2/dlm/dlmrecovery.c new file mode 100644 index 000000000..9d4a2695b --- /dev/null +++ b/fs/ocfs2/dlm/dlmrecovery.c @@ -0,0 +1,2962 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * dlmrecovery.c + * + * recovery stuff + * + * Copyright (C) 2004 Oracle. All rights reserved. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../cluster/heartbeat.h" +#include "../cluster/nodemanager.h" +#include "../cluster/tcp.h" + +#include "dlmapi.h" +#include "dlmcommon.h" +#include "dlmdomain.h" + +#define MLOG_MASK_PREFIX (ML_DLM|ML_DLM_RECOVERY) +#include "../cluster/masklog.h" + +static void dlm_do_local_recovery_cleanup(struct dlm_ctxt *dlm, u8 dead_node); + +static int dlm_recovery_thread(void *data); +static int dlm_do_recovery(struct dlm_ctxt *dlm); + +static int dlm_pick_recovery_master(struct dlm_ctxt *dlm); +static int dlm_remaster_locks(struct dlm_ctxt *dlm, u8 dead_node); +static int dlm_init_recovery_area(struct dlm_ctxt *dlm, u8 dead_node); +static int dlm_request_all_locks(struct dlm_ctxt *dlm, + u8 request_from, u8 dead_node); +static void dlm_destroy_recovery_area(struct dlm_ctxt *dlm); + +static inline int dlm_num_locks_in_lockres(struct dlm_lock_resource *res); +static void dlm_init_migratable_lockres(struct dlm_migratable_lockres *mres, + const char *lockname, int namelen, + int total_locks, u64 cookie, + u8 flags, u8 master); +static int dlm_send_mig_lockres_msg(struct dlm_ctxt *dlm, + struct dlm_migratable_lockres *mres, + u8 send_to, + struct dlm_lock_resource *res, + int total_locks); +static int dlm_process_recovery_data(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_migratable_lockres *mres); +static int dlm_send_finalize_reco_message(struct dlm_ctxt *dlm); +static int dlm_send_all_done_msg(struct dlm_ctxt *dlm, + u8 dead_node, u8 send_to); +static int dlm_send_begin_reco_message(struct dlm_ctxt *dlm, u8 dead_node); +static void dlm_move_reco_locks_to_list(struct dlm_ctxt *dlm, + struct list_head *list, u8 dead_node); +static void dlm_finish_local_lockres_recovery(struct dlm_ctxt *dlm, + u8 dead_node, u8 new_master); +static void dlm_reco_ast(void *astdata); +static void dlm_reco_bast(void *astdata, int blocked_type); +static void dlm_reco_unlock_ast(void *astdata, enum dlm_status st); +static void dlm_request_all_locks_worker(struct dlm_work_item *item, + void *data); +static void dlm_mig_lockres_worker(struct dlm_work_item *item, void *data); +static int dlm_lockres_master_requery(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + u8 *real_master); + +static u64 dlm_get_next_mig_cookie(void); + +static DEFINE_SPINLOCK(dlm_reco_state_lock); +static DEFINE_SPINLOCK(dlm_mig_cookie_lock); +static u64 dlm_mig_cookie = 1; + +static u64 dlm_get_next_mig_cookie(void) +{ + u64 c; + spin_lock(&dlm_mig_cookie_lock); + c = dlm_mig_cookie; + if (dlm_mig_cookie == (~0ULL)) + dlm_mig_cookie = 1; + else + dlm_mig_cookie++; + spin_unlock(&dlm_mig_cookie_lock); + return c; +} + +static inline void dlm_set_reco_dead_node(struct dlm_ctxt *dlm, + u8 dead_node) +{ + assert_spin_locked(&dlm->spinlock); + if (dlm->reco.dead_node != dead_node) + mlog(0, "%s: changing dead_node from %u to %u\n", + dlm->name, dlm->reco.dead_node, dead_node); + dlm->reco.dead_node = dead_node; +} + +static inline void dlm_set_reco_master(struct dlm_ctxt *dlm, + u8 master) +{ + assert_spin_locked(&dlm->spinlock); + mlog(0, "%s: changing new_master from %u to %u\n", + dlm->name, dlm->reco.new_master, master); + dlm->reco.new_master = master; +} + +static inline void __dlm_reset_recovery(struct dlm_ctxt *dlm) +{ + assert_spin_locked(&dlm->spinlock); + clear_bit(dlm->reco.dead_node, dlm->recovery_map); + dlm_set_reco_dead_node(dlm, O2NM_INVALID_NODE_NUM); + dlm_set_reco_master(dlm, O2NM_INVALID_NODE_NUM); +} + +/* Worker function used during recovery. */ +void dlm_dispatch_work(struct work_struct *work) +{ + struct dlm_ctxt *dlm = + container_of(work, struct dlm_ctxt, dispatched_work); + LIST_HEAD(tmp_list); + struct dlm_work_item *item, *next; + dlm_workfunc_t *workfunc; + int tot=0; + + spin_lock(&dlm->work_lock); + list_splice_init(&dlm->work_list, &tmp_list); + spin_unlock(&dlm->work_lock); + + list_for_each_entry(item, &tmp_list, list) { + tot++; + } + mlog(0, "%s: work thread has %d work items\n", dlm->name, tot); + + list_for_each_entry_safe(item, next, &tmp_list, list) { + workfunc = item->func; + list_del_init(&item->list); + + /* already have ref on dlm to avoid having + * it disappear. just double-check. */ + BUG_ON(item->dlm != dlm); + + /* this is allowed to sleep and + * call network stuff */ + workfunc(item, item->data); + + dlm_put(dlm); + kfree(item); + } +} + +/* + * RECOVERY THREAD + */ + +void dlm_kick_recovery_thread(struct dlm_ctxt *dlm) +{ + /* wake the recovery thread + * this will wake the reco thread in one of three places + * 1) sleeping with no recovery happening + * 2) sleeping with recovery mastered elsewhere + * 3) recovery mastered here, waiting on reco data */ + + wake_up(&dlm->dlm_reco_thread_wq); +} + +/* Launch the recovery thread */ +int dlm_launch_recovery_thread(struct dlm_ctxt *dlm) +{ + mlog(0, "starting dlm recovery thread...\n"); + + dlm->dlm_reco_thread_task = kthread_run(dlm_recovery_thread, dlm, + "dlm_reco-%s", dlm->name); + if (IS_ERR(dlm->dlm_reco_thread_task)) { + mlog_errno(PTR_ERR(dlm->dlm_reco_thread_task)); + dlm->dlm_reco_thread_task = NULL; + return -EINVAL; + } + + return 0; +} + +void dlm_complete_recovery_thread(struct dlm_ctxt *dlm) +{ + if (dlm->dlm_reco_thread_task) { + mlog(0, "waiting for dlm recovery thread to exit\n"); + kthread_stop(dlm->dlm_reco_thread_task); + dlm->dlm_reco_thread_task = NULL; + } +} + + + +/* + * this is lame, but here's how recovery works... + * 1) all recovery threads cluster wide will work on recovering + * ONE node at a time + * 2) negotiate who will take over all the locks for the dead node. + * that's right... ALL the locks. + * 3) once a new master is chosen, everyone scans all locks + * and moves aside those mastered by the dead guy + * 4) each of these locks should be locked until recovery is done + * 5) the new master collects up all of secondary lock queue info + * one lock at a time, forcing each node to communicate back + * before continuing + * 6) each secondary lock queue responds with the full known lock info + * 7) once the new master has run all its locks, it sends a ALLDONE! + * message to everyone + * 8) upon receiving this message, the secondary queue node unlocks + * and responds to the ALLDONE + * 9) once the new master gets responses from everyone, he unlocks + * everything and recovery for this dead node is done + *10) go back to 2) while there are still dead nodes + * + */ + +static void dlm_print_reco_node_status(struct dlm_ctxt *dlm) +{ + struct dlm_reco_node_data *ndata; + struct dlm_lock_resource *res; + + mlog(ML_NOTICE, "%s(%d): recovery info, state=%s, dead=%u, master=%u\n", + dlm->name, task_pid_nr(dlm->dlm_reco_thread_task), + dlm->reco.state & DLM_RECO_STATE_ACTIVE ? "ACTIVE" : "inactive", + dlm->reco.dead_node, dlm->reco.new_master); + + list_for_each_entry(ndata, &dlm->reco.node_data, list) { + char *st = "unknown"; + switch (ndata->state) { + case DLM_RECO_NODE_DATA_INIT: + st = "init"; + break; + case DLM_RECO_NODE_DATA_REQUESTING: + st = "requesting"; + break; + case DLM_RECO_NODE_DATA_DEAD: + st = "dead"; + break; + case DLM_RECO_NODE_DATA_RECEIVING: + st = "receiving"; + break; + case DLM_RECO_NODE_DATA_REQUESTED: + st = "requested"; + break; + case DLM_RECO_NODE_DATA_DONE: + st = "done"; + break; + case DLM_RECO_NODE_DATA_FINALIZE_SENT: + st = "finalize-sent"; + break; + default: + st = "bad"; + break; + } + mlog(ML_NOTICE, "%s: reco state, node %u, state=%s\n", + dlm->name, ndata->node_num, st); + } + list_for_each_entry(res, &dlm->reco.resources, recovering) { + mlog(ML_NOTICE, "%s: lockres %.*s on recovering list\n", + dlm->name, res->lockname.len, res->lockname.name); + } +} + +#define DLM_RECO_THREAD_TIMEOUT_MS (5 * 1000) + +static int dlm_recovery_thread(void *data) +{ + int status; + struct dlm_ctxt *dlm = data; + unsigned long timeout = msecs_to_jiffies(DLM_RECO_THREAD_TIMEOUT_MS); + + mlog(0, "dlm thread running for %s...\n", dlm->name); + + while (!kthread_should_stop()) { + if (dlm_domain_fully_joined(dlm)) { + status = dlm_do_recovery(dlm); + if (status == -EAGAIN) { + /* do not sleep, recheck immediately. */ + continue; + } + if (status < 0) + mlog_errno(status); + } + + wait_event_interruptible_timeout(dlm->dlm_reco_thread_wq, + kthread_should_stop(), + timeout); + } + + mlog(0, "quitting DLM recovery thread\n"); + return 0; +} + +/* returns true when the recovery master has contacted us */ +static int dlm_reco_master_ready(struct dlm_ctxt *dlm) +{ + int ready; + spin_lock(&dlm->spinlock); + ready = (dlm->reco.new_master != O2NM_INVALID_NODE_NUM); + spin_unlock(&dlm->spinlock); + return ready; +} + +/* returns true if node is no longer in the domain + * could be dead or just not joined */ +int dlm_is_node_dead(struct dlm_ctxt *dlm, u8 node) +{ + int dead; + spin_lock(&dlm->spinlock); + dead = !test_bit(node, dlm->domain_map); + spin_unlock(&dlm->spinlock); + return dead; +} + +/* returns true if node is no longer in the domain + * could be dead or just not joined */ +static int dlm_is_node_recovered(struct dlm_ctxt *dlm, u8 node) +{ + int recovered; + spin_lock(&dlm->spinlock); + recovered = !test_bit(node, dlm->recovery_map); + spin_unlock(&dlm->spinlock); + return recovered; +} + + +void dlm_wait_for_node_death(struct dlm_ctxt *dlm, u8 node, int timeout) +{ + if (dlm_is_node_dead(dlm, node)) + return; + + printk(KERN_NOTICE "o2dlm: Waiting on the death of node %u in " + "domain %s\n", node, dlm->name); + + if (timeout) + wait_event_timeout(dlm->dlm_reco_thread_wq, + dlm_is_node_dead(dlm, node), + msecs_to_jiffies(timeout)); + else + wait_event(dlm->dlm_reco_thread_wq, + dlm_is_node_dead(dlm, node)); +} + +void dlm_wait_for_node_recovery(struct dlm_ctxt *dlm, u8 node, int timeout) +{ + if (dlm_is_node_recovered(dlm, node)) + return; + + printk(KERN_NOTICE "o2dlm: Waiting on the recovery of node %u in " + "domain %s\n", node, dlm->name); + + if (timeout) + wait_event_timeout(dlm->dlm_reco_thread_wq, + dlm_is_node_recovered(dlm, node), + msecs_to_jiffies(timeout)); + else + wait_event(dlm->dlm_reco_thread_wq, + dlm_is_node_recovered(dlm, node)); +} + +/* callers of the top-level api calls (dlmlock/dlmunlock) should + * block on the dlm->reco.event when recovery is in progress. + * the dlm recovery thread will set this state when it begins + * recovering a dead node (as the new master or not) and clear + * the state and wake as soon as all affected lock resources have + * been marked with the RECOVERY flag */ +static int dlm_in_recovery(struct dlm_ctxt *dlm) +{ + int in_recovery; + spin_lock(&dlm->spinlock); + in_recovery = !!(dlm->reco.state & DLM_RECO_STATE_ACTIVE); + spin_unlock(&dlm->spinlock); + return in_recovery; +} + + +void dlm_wait_for_recovery(struct dlm_ctxt *dlm) +{ + if (dlm_in_recovery(dlm)) { + mlog(0, "%s: reco thread %d in recovery: " + "state=%d, master=%u, dead=%u\n", + dlm->name, task_pid_nr(dlm->dlm_reco_thread_task), + dlm->reco.state, dlm->reco.new_master, + dlm->reco.dead_node); + } + wait_event(dlm->reco.event, !dlm_in_recovery(dlm)); +} + +static void dlm_begin_recovery(struct dlm_ctxt *dlm) +{ + assert_spin_locked(&dlm->spinlock); + BUG_ON(dlm->reco.state & DLM_RECO_STATE_ACTIVE); + printk(KERN_NOTICE "o2dlm: Begin recovery on domain %s for node %u\n", + dlm->name, dlm->reco.dead_node); + dlm->reco.state |= DLM_RECO_STATE_ACTIVE; +} + +static void dlm_end_recovery(struct dlm_ctxt *dlm) +{ + spin_lock(&dlm->spinlock); + BUG_ON(!(dlm->reco.state & DLM_RECO_STATE_ACTIVE)); + dlm->reco.state &= ~DLM_RECO_STATE_ACTIVE; + spin_unlock(&dlm->spinlock); + printk(KERN_NOTICE "o2dlm: End recovery on domain %s\n", dlm->name); + wake_up(&dlm->reco.event); +} + +static void dlm_print_recovery_master(struct dlm_ctxt *dlm) +{ + printk(KERN_NOTICE "o2dlm: Node %u (%s) is the Recovery Master for the " + "dead node %u in domain %s\n", dlm->reco.new_master, + (dlm->node_num == dlm->reco.new_master ? "me" : "he"), + dlm->reco.dead_node, dlm->name); +} + +static int dlm_do_recovery(struct dlm_ctxt *dlm) +{ + int status = 0; + int ret; + + spin_lock(&dlm->spinlock); + + if (dlm->migrate_done) { + mlog(0, "%s: no need do recovery after migrating all " + "lock resources\n", dlm->name); + spin_unlock(&dlm->spinlock); + return 0; + } + + /* check to see if the new master has died */ + if (dlm->reco.new_master != O2NM_INVALID_NODE_NUM && + test_bit(dlm->reco.new_master, dlm->recovery_map)) { + mlog(0, "new master %u died while recovering %u!\n", + dlm->reco.new_master, dlm->reco.dead_node); + /* unset the new_master, leave dead_node */ + dlm_set_reco_master(dlm, O2NM_INVALID_NODE_NUM); + } + + /* select a target to recover */ + if (dlm->reco.dead_node == O2NM_INVALID_NODE_NUM) { + int bit; + + bit = find_first_bit(dlm->recovery_map, O2NM_MAX_NODES); + if (bit >= O2NM_MAX_NODES || bit < 0) + dlm_set_reco_dead_node(dlm, O2NM_INVALID_NODE_NUM); + else + dlm_set_reco_dead_node(dlm, bit); + } else if (!test_bit(dlm->reco.dead_node, dlm->recovery_map)) { + /* BUG? */ + mlog(ML_ERROR, "dead_node %u no longer in recovery map!\n", + dlm->reco.dead_node); + dlm_set_reco_dead_node(dlm, O2NM_INVALID_NODE_NUM); + } + + if (dlm->reco.dead_node == O2NM_INVALID_NODE_NUM) { + spin_unlock(&dlm->spinlock); + /* return to main thread loop and sleep. */ + return 0; + } + mlog(0, "%s(%d):recovery thread found node %u in the recovery map!\n", + dlm->name, task_pid_nr(dlm->dlm_reco_thread_task), + dlm->reco.dead_node); + + /* take write barrier */ + /* (stops the list reshuffling thread, proxy ast handling) */ + dlm_begin_recovery(dlm); + + spin_unlock(&dlm->spinlock); + + if (dlm->reco.new_master == dlm->node_num) + goto master_here; + + if (dlm->reco.new_master == O2NM_INVALID_NODE_NUM) { + /* choose a new master, returns 0 if this node + * is the master, -EEXIST if it's another node. + * this does not return until a new master is chosen + * or recovery completes entirely. */ + ret = dlm_pick_recovery_master(dlm); + if (!ret) { + /* already notified everyone. go. */ + goto master_here; + } + mlog(0, "another node will master this recovery session.\n"); + } + + dlm_print_recovery_master(dlm); + + /* it is safe to start everything back up here + * because all of the dead node's lock resources + * have been marked as in-recovery */ + dlm_end_recovery(dlm); + + /* sleep out in main dlm_recovery_thread loop. */ + return 0; + +master_here: + dlm_print_recovery_master(dlm); + + status = dlm_remaster_locks(dlm, dlm->reco.dead_node); + if (status < 0) { + /* we should never hit this anymore */ + mlog(ML_ERROR, "%s: Error %d remastering locks for node %u, " + "retrying.\n", dlm->name, status, dlm->reco.dead_node); + /* yield a bit to allow any final network messages + * to get handled on remaining nodes */ + msleep(100); + } else { + /* success! see if any other nodes need recovery */ + mlog(0, "DONE mastering recovery of %s:%u here(this=%u)!\n", + dlm->name, dlm->reco.dead_node, dlm->node_num); + spin_lock(&dlm->spinlock); + __dlm_reset_recovery(dlm); + dlm->reco.state &= ~DLM_RECO_STATE_FINALIZE; + spin_unlock(&dlm->spinlock); + } + dlm_end_recovery(dlm); + + /* continue and look for another dead node */ + return -EAGAIN; +} + +static int dlm_remaster_locks(struct dlm_ctxt *dlm, u8 dead_node) +{ + int status = 0; + struct dlm_reco_node_data *ndata; + int all_nodes_done; + int destroy = 0; + int pass = 0; + + do { + /* we have become recovery master. there is no escaping + * this, so just keep trying until we get it. */ + status = dlm_init_recovery_area(dlm, dead_node); + if (status < 0) { + mlog(ML_ERROR, "%s: failed to alloc recovery area, " + "retrying\n", dlm->name); + msleep(1000); + } + } while (status != 0); + + /* safe to access the node data list without a lock, since this + * process is the only one to change the list */ + list_for_each_entry(ndata, &dlm->reco.node_data, list) { + BUG_ON(ndata->state != DLM_RECO_NODE_DATA_INIT); + ndata->state = DLM_RECO_NODE_DATA_REQUESTING; + + mlog(0, "%s: Requesting lock info from node %u\n", dlm->name, + ndata->node_num); + + if (ndata->node_num == dlm->node_num) { + ndata->state = DLM_RECO_NODE_DATA_DONE; + continue; + } + + do { + status = dlm_request_all_locks(dlm, ndata->node_num, + dead_node); + if (status < 0) { + mlog_errno(status); + if (dlm_is_host_down(status)) { + /* node died, ignore it for recovery */ + status = 0; + ndata->state = DLM_RECO_NODE_DATA_DEAD; + /* wait for the domain map to catch up + * with the network state. */ + wait_event_timeout(dlm->dlm_reco_thread_wq, + dlm_is_node_dead(dlm, + ndata->node_num), + msecs_to_jiffies(1000)); + mlog(0, "waited 1 sec for %u, " + "dead? %s\n", ndata->node_num, + str_yes_no(dlm_is_node_dead(dlm, ndata->node_num))); + } else { + /* -ENOMEM on the other node */ + mlog(0, "%s: node %u returned " + "%d during recovery, retrying " + "after a short wait\n", + dlm->name, ndata->node_num, + status); + msleep(100); + } + } + } while (status != 0); + + spin_lock(&dlm_reco_state_lock); + switch (ndata->state) { + case DLM_RECO_NODE_DATA_INIT: + case DLM_RECO_NODE_DATA_FINALIZE_SENT: + case DLM_RECO_NODE_DATA_REQUESTED: + BUG(); + break; + case DLM_RECO_NODE_DATA_DEAD: + mlog(0, "node %u died after requesting " + "recovery info for node %u\n", + ndata->node_num, dead_node); + /* fine. don't need this node's info. + * continue without it. */ + break; + case DLM_RECO_NODE_DATA_REQUESTING: + ndata->state = DLM_RECO_NODE_DATA_REQUESTED; + mlog(0, "now receiving recovery data from " + "node %u for dead node %u\n", + ndata->node_num, dead_node); + break; + case DLM_RECO_NODE_DATA_RECEIVING: + mlog(0, "already receiving recovery data from " + "node %u for dead node %u\n", + ndata->node_num, dead_node); + break; + case DLM_RECO_NODE_DATA_DONE: + mlog(0, "already DONE receiving recovery data " + "from node %u for dead node %u\n", + ndata->node_num, dead_node); + break; + } + spin_unlock(&dlm_reco_state_lock); + } + + mlog(0, "%s: Done requesting all lock info\n", dlm->name); + + /* nodes should be sending reco data now + * just need to wait */ + + while (1) { + /* check all the nodes now to see if we are + * done, or if anyone died */ + all_nodes_done = 1; + spin_lock(&dlm_reco_state_lock); + list_for_each_entry(ndata, &dlm->reco.node_data, list) { + mlog(0, "checking recovery state of node %u\n", + ndata->node_num); + switch (ndata->state) { + case DLM_RECO_NODE_DATA_INIT: + case DLM_RECO_NODE_DATA_REQUESTING: + mlog(ML_ERROR, "bad ndata state for " + "node %u: state=%d\n", + ndata->node_num, ndata->state); + BUG(); + break; + case DLM_RECO_NODE_DATA_DEAD: + mlog(0, "node %u died after " + "requesting recovery info for " + "node %u\n", ndata->node_num, + dead_node); + break; + case DLM_RECO_NODE_DATA_RECEIVING: + case DLM_RECO_NODE_DATA_REQUESTED: + mlog(0, "%s: node %u still in state %s\n", + dlm->name, ndata->node_num, + ndata->state==DLM_RECO_NODE_DATA_RECEIVING ? + "receiving" : "requested"); + all_nodes_done = 0; + break; + case DLM_RECO_NODE_DATA_DONE: + mlog(0, "%s: node %u state is done\n", + dlm->name, ndata->node_num); + break; + case DLM_RECO_NODE_DATA_FINALIZE_SENT: + mlog(0, "%s: node %u state is finalize\n", + dlm->name, ndata->node_num); + break; + } + } + spin_unlock(&dlm_reco_state_lock); + + mlog(0, "pass #%d, all_nodes_done?: %s\n", ++pass, + str_yes_no(all_nodes_done)); + if (all_nodes_done) { + int ret; + + /* Set this flag on recovery master to avoid + * a new recovery for another dead node start + * before the recovery is not done. That may + * cause recovery hung.*/ + spin_lock(&dlm->spinlock); + dlm->reco.state |= DLM_RECO_STATE_FINALIZE; + spin_unlock(&dlm->spinlock); + + /* all nodes are now in DLM_RECO_NODE_DATA_DONE state + * just send a finalize message to everyone and + * clean up */ + mlog(0, "all nodes are done! send finalize\n"); + ret = dlm_send_finalize_reco_message(dlm); + if (ret < 0) + mlog_errno(ret); + + spin_lock(&dlm->spinlock); + dlm_finish_local_lockres_recovery(dlm, dead_node, + dlm->node_num); + spin_unlock(&dlm->spinlock); + mlog(0, "should be done with recovery!\n"); + + mlog(0, "finishing recovery of %s at %lu, " + "dead=%u, this=%u, new=%u\n", dlm->name, + jiffies, dlm->reco.dead_node, + dlm->node_num, dlm->reco.new_master); + destroy = 1; + status = 0; + /* rescan everything marked dirty along the way */ + dlm_kick_thread(dlm, NULL); + break; + } + /* wait to be signalled, with periodic timeout + * to check for node death */ + wait_event_interruptible_timeout(dlm->dlm_reco_thread_wq, + kthread_should_stop(), + msecs_to_jiffies(DLM_RECO_THREAD_TIMEOUT_MS)); + + } + + if (destroy) + dlm_destroy_recovery_area(dlm); + + return status; +} + +static int dlm_init_recovery_area(struct dlm_ctxt *dlm, u8 dead_node) +{ + int num=0; + struct dlm_reco_node_data *ndata; + + spin_lock(&dlm->spinlock); + bitmap_copy(dlm->reco.node_map, dlm->domain_map, O2NM_MAX_NODES); + /* nodes can only be removed (by dying) after dropping + * this lock, and death will be trapped later, so this should do */ + spin_unlock(&dlm->spinlock); + + while (1) { + num = find_next_bit (dlm->reco.node_map, O2NM_MAX_NODES, num); + if (num >= O2NM_MAX_NODES) { + break; + } + BUG_ON(num == dead_node); + + ndata = kzalloc_obj(*ndata, GFP_NOFS); + if (!ndata) { + dlm_destroy_recovery_area(dlm); + return -ENOMEM; + } + ndata->node_num = num; + ndata->state = DLM_RECO_NODE_DATA_INIT; + spin_lock(&dlm_reco_state_lock); + list_add_tail(&ndata->list, &dlm->reco.node_data); + spin_unlock(&dlm_reco_state_lock); + num++; + } + + return 0; +} + +static void dlm_destroy_recovery_area(struct dlm_ctxt *dlm) +{ + struct dlm_reco_node_data *ndata, *next; + LIST_HEAD(tmplist); + + spin_lock(&dlm_reco_state_lock); + list_splice_init(&dlm->reco.node_data, &tmplist); + spin_unlock(&dlm_reco_state_lock); + + list_for_each_entry_safe(ndata, next, &tmplist, list) { + list_del_init(&ndata->list); + kfree(ndata); + } +} + +static int dlm_request_all_locks(struct dlm_ctxt *dlm, u8 request_from, + u8 dead_node) +{ + struct dlm_lock_request lr; + int ret; + int status; + + mlog(0, "\n"); + + + mlog(0, "dlm_request_all_locks: dead node is %u, sending request " + "to %u\n", dead_node, request_from); + + memset(&lr, 0, sizeof(lr)); + lr.node_idx = dlm->node_num; + lr.dead_node = dead_node; + + // send message + ret = o2net_send_message(DLM_LOCK_REQUEST_MSG, dlm->key, + &lr, sizeof(lr), request_from, &status); + + /* negative status is handled by caller */ + if (ret < 0) + mlog(ML_ERROR, "%s: Error %d send LOCK_REQUEST to node %u " + "to recover dead node %u\n", dlm->name, ret, + request_from, dead_node); + else + ret = status; + // return from here, then + // sleep until all received or error + return ret; + +} + +int dlm_request_all_locks_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data) +{ + struct dlm_ctxt *dlm = data; + struct dlm_lock_request *lr = (struct dlm_lock_request *)msg->buf; + char *buf = NULL; + struct dlm_work_item *item = NULL; + + if (!dlm_grab(dlm)) + return -EINVAL; + + if (lr->dead_node != dlm->reco.dead_node) { + mlog(ML_ERROR, "%s: node %u sent dead_node=%u, but local " + "dead_node is %u\n", dlm->name, lr->node_idx, + lr->dead_node, dlm->reco.dead_node); + dlm_print_reco_node_status(dlm); + /* this is a hack */ + dlm_put(dlm); + return -ENOMEM; + } + BUG_ON(lr->dead_node != dlm->reco.dead_node); + + item = kzalloc_obj(*item, GFP_NOFS); + if (!item) { + dlm_put(dlm); + return -ENOMEM; + } + + /* this will get freed by dlm_request_all_locks_worker */ + buf = kmalloc(PAGE_SIZE, GFP_NOFS); + if (!buf) { + kfree(item); + dlm_put(dlm); + return -ENOMEM; + } + + /* queue up work for dlm_request_all_locks_worker */ + dlm_grab(dlm); /* get an extra ref for the work item */ + dlm_init_work_item(dlm, item, dlm_request_all_locks_worker, buf); + item->u.ral.reco_master = lr->node_idx; + item->u.ral.dead_node = lr->dead_node; + spin_lock(&dlm->work_lock); + list_add_tail(&item->list, &dlm->work_list); + spin_unlock(&dlm->work_lock); + queue_work(dlm->dlm_worker, &dlm->dispatched_work); + + dlm_put(dlm); + return 0; +} + +static void dlm_request_all_locks_worker(struct dlm_work_item *item, void *data) +{ + struct dlm_migratable_lockres *mres; + struct dlm_lock_resource *res; + struct dlm_ctxt *dlm; + LIST_HEAD(resources); + int ret; + u8 dead_node, reco_master; + int skip_all_done = 0; + + dlm = item->dlm; + dead_node = item->u.ral.dead_node; + reco_master = item->u.ral.reco_master; + mres = (struct dlm_migratable_lockres *)data; + + mlog(0, "%s: recovery worker started, dead=%u, master=%u\n", + dlm->name, dead_node, reco_master); + + if (dead_node != dlm->reco.dead_node || + reco_master != dlm->reco.new_master) { + /* worker could have been created before the recovery master + * died. if so, do not continue, but do not error. */ + if (dlm->reco.new_master == O2NM_INVALID_NODE_NUM) { + mlog(ML_NOTICE, "%s: will not send recovery state, " + "recovery master %u died, thread=(dead=%u,mas=%u)" + " current=(dead=%u,mas=%u)\n", dlm->name, + reco_master, dead_node, reco_master, + dlm->reco.dead_node, dlm->reco.new_master); + } else { + mlog(ML_NOTICE, "%s: reco state invalid: reco(dead=%u, " + "master=%u), request(dead=%u, master=%u)\n", + dlm->name, dlm->reco.dead_node, + dlm->reco.new_master, dead_node, reco_master); + } + goto leave; + } + + /* lock resources should have already been moved to the + * dlm->reco.resources list. now move items from that list + * to a temp list if the dead owner matches. note that the + * whole cluster recovers only one node at a time, so we + * can safely move UNKNOWN lock resources for each recovery + * session. */ + dlm_move_reco_locks_to_list(dlm, &resources, dead_node); + + /* now we can begin blasting lockreses without the dlm lock */ + + /* any errors returned will be due to the new_master dying, + * the dlm_reco_thread should detect this */ + list_for_each_entry(res, &resources, recovering) { + ret = dlm_send_one_lockres(dlm, res, mres, reco_master, + DLM_MRES_RECOVERY); + if (ret < 0) { + mlog(ML_ERROR, "%s: node %u went down while sending " + "recovery state for dead node %u, ret=%d\n", dlm->name, + reco_master, dead_node, ret); + skip_all_done = 1; + break; + } + } + + /* move the resources back to the list */ + spin_lock(&dlm->spinlock); + list_splice_init(&resources, &dlm->reco.resources); + spin_unlock(&dlm->spinlock); + + if (!skip_all_done) { + ret = dlm_send_all_done_msg(dlm, dead_node, reco_master); + if (ret < 0) { + mlog(ML_ERROR, "%s: node %u went down while sending " + "recovery all-done for dead node %u, ret=%d\n", + dlm->name, reco_master, dead_node, ret); + } + } +leave: + kfree(data); +} + + +static int dlm_send_all_done_msg(struct dlm_ctxt *dlm, u8 dead_node, u8 send_to) +{ + int ret, tmpret; + struct dlm_reco_data_done done_msg; + + memset(&done_msg, 0, sizeof(done_msg)); + done_msg.node_idx = dlm->node_num; + done_msg.dead_node = dead_node; + mlog(0, "sending DATA DONE message to %u, " + "my node=%u, dead node=%u\n", send_to, done_msg.node_idx, + done_msg.dead_node); + + ret = o2net_send_message(DLM_RECO_DATA_DONE_MSG, dlm->key, &done_msg, + sizeof(done_msg), send_to, &tmpret); + if (ret < 0) { + mlog(ML_ERROR, "%s: Error %d send RECO_DATA_DONE to node %u " + "to recover dead node %u\n", dlm->name, ret, send_to, + dead_node); + if (!dlm_is_host_down(ret)) { + BUG(); + } + } else + ret = tmpret; + return ret; +} + + +int dlm_reco_data_done_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data) +{ + struct dlm_ctxt *dlm = data; + struct dlm_reco_data_done *done = (struct dlm_reco_data_done *)msg->buf; + struct dlm_reco_node_data *ndata = NULL; + int ret = -EINVAL; + + if (!dlm_grab(dlm)) + return -EINVAL; + + mlog(0, "got DATA DONE: dead_node=%u, reco.dead_node=%u, " + "node_idx=%u, this node=%u\n", done->dead_node, + dlm->reco.dead_node, done->node_idx, dlm->node_num); + + mlog_bug_on_msg((done->dead_node != dlm->reco.dead_node), + "Got DATA DONE: dead_node=%u, reco.dead_node=%u, " + "node_idx=%u, this node=%u\n", done->dead_node, + dlm->reco.dead_node, done->node_idx, dlm->node_num); + + spin_lock(&dlm_reco_state_lock); + list_for_each_entry(ndata, &dlm->reco.node_data, list) { + if (ndata->node_num != done->node_idx) + continue; + + switch (ndata->state) { + /* should have moved beyond INIT but not to FINALIZE yet */ + case DLM_RECO_NODE_DATA_INIT: + case DLM_RECO_NODE_DATA_DEAD: + case DLM_RECO_NODE_DATA_FINALIZE_SENT: + mlog(ML_ERROR, "bad ndata state for node %u:" + " state=%d\n", ndata->node_num, + ndata->state); + BUG(); + break; + /* these states are possible at this point, anywhere along + * the line of recovery */ + case DLM_RECO_NODE_DATA_DONE: + case DLM_RECO_NODE_DATA_RECEIVING: + case DLM_RECO_NODE_DATA_REQUESTED: + case DLM_RECO_NODE_DATA_REQUESTING: + mlog(0, "node %u is DONE sending " + "recovery data!\n", + ndata->node_num); + + ndata->state = DLM_RECO_NODE_DATA_DONE; + ret = 0; + break; + } + } + spin_unlock(&dlm_reco_state_lock); + + /* wake the recovery thread, some node is done */ + if (!ret) + dlm_kick_recovery_thread(dlm); + + if (ret < 0) + mlog(ML_ERROR, "failed to find recovery node data for node " + "%u\n", done->node_idx); + dlm_put(dlm); + + mlog(0, "leaving reco data done handler, ret=%d\n", ret); + return ret; +} + +static void dlm_move_reco_locks_to_list(struct dlm_ctxt *dlm, + struct list_head *list, + u8 dead_node) +{ + struct dlm_lock_resource *res, *next; + struct dlm_lock *lock; + + spin_lock(&dlm->spinlock); + list_for_each_entry_safe(res, next, &dlm->reco.resources, recovering) { + /* always prune any $RECOVERY entries for dead nodes, + * otherwise hangs can occur during later recovery */ + if (dlm_is_recovery_lock(res->lockname.name, + res->lockname.len)) { + spin_lock(&res->spinlock); + list_for_each_entry(lock, &res->granted, list) { + if (lock->ml.node == dead_node) { + mlog(0, "AHA! there was " + "a $RECOVERY lock for dead " + "node %u (%s)!\n", + dead_node, dlm->name); + list_del_init(&lock->list); + dlm_lock_put(lock); + /* Can't schedule DLM_UNLOCK_FREE_LOCK + * - do manually */ + dlm_lock_put(lock); + break; + } + } + spin_unlock(&res->spinlock); + continue; + } + + if (res->owner == dead_node) { + mlog(0, "found lockres owned by dead node while " + "doing recovery for node %u. sending it.\n", + dead_node); + list_move_tail(&res->recovering, list); + } else if (res->owner == DLM_LOCK_RES_OWNER_UNKNOWN) { + mlog(0, "found UNKNOWN owner while doing recovery " + "for node %u. sending it.\n", dead_node); + list_move_tail(&res->recovering, list); + } + } + spin_unlock(&dlm->spinlock); +} + +static inline int dlm_num_locks_in_lockres(struct dlm_lock_resource *res) +{ + int total_locks = 0; + struct list_head *iter, *queue = &res->granted; + int i; + + for (i=0; i<3; i++) { + list_for_each(iter, queue) + total_locks++; + queue++; + } + return total_locks; +} + + +static int dlm_send_mig_lockres_msg(struct dlm_ctxt *dlm, + struct dlm_migratable_lockres *mres, + u8 send_to, + struct dlm_lock_resource *res, + int total_locks) +{ + u64 mig_cookie = be64_to_cpu(mres->mig_cookie); + int mres_total_locks = be32_to_cpu(mres->total_locks); + int ret = 0, status = 0; + u8 orig_flags = mres->flags, + orig_master = mres->master; + + BUG_ON(mres->num_locks > DLM_MAX_MIGRATABLE_LOCKS); + if (!mres->num_locks) + return 0; + + /* add an all-done flag if we reached the last lock */ + orig_flags = mres->flags; + BUG_ON(total_locks > mres_total_locks); + if (total_locks == mres_total_locks) + mres->flags |= DLM_MRES_ALL_DONE; + + mlog(0, "%s:%.*s: sending mig lockres (%s) to %u\n", + dlm->name, res->lockname.len, res->lockname.name, + orig_flags & DLM_MRES_MIGRATION ? "migration" : "recovery", + send_to); + + /* send it */ + ret = o2net_send_message(DLM_MIG_LOCKRES_MSG, dlm->key, mres, + struct_size(mres, ml, mres->num_locks), + send_to, &status); + if (ret < 0) { + /* XXX: negative status is not handled. + * this will end up killing this node. */ + mlog(ML_ERROR, "%s: res %.*s, Error %d send MIG_LOCKRES to " + "node %u (%s)\n", dlm->name, mres->lockname_len, + mres->lockname, ret, send_to, + (orig_flags & DLM_MRES_MIGRATION ? + "migration" : "recovery")); + } else { + /* might get an -ENOMEM back here */ + ret = status; + if (ret < 0) { + mlog_errno(ret); + + if (ret == -EFAULT) { + mlog(ML_ERROR, "node %u told me to kill " + "myself!\n", send_to); + BUG(); + } + } + } + + /* zero and reinit the message buffer */ + dlm_init_migratable_lockres(mres, res->lockname.name, + res->lockname.len, mres_total_locks, + mig_cookie, orig_flags, orig_master); + return ret; +} + +static void dlm_init_migratable_lockres(struct dlm_migratable_lockres *mres, + const char *lockname, int namelen, + int total_locks, u64 cookie, + u8 flags, u8 master) +{ + /* mres here is one full page */ + clear_page(mres); + mres->lockname_len = namelen; + memcpy(mres->lockname, lockname, namelen); + mres->num_locks = 0; + mres->total_locks = cpu_to_be32(total_locks); + mres->mig_cookie = cpu_to_be64(cookie); + mres->flags = flags; + mres->master = master; +} + +static void dlm_prepare_lvb_for_migration(struct dlm_lock *lock, + struct dlm_migratable_lockres *mres, + int queue) +{ + if (!lock->lksb) + return; + + /* Ignore lvb in all locks in the blocked list */ + if (queue == DLM_BLOCKED_LIST) + return; + + /* Only consider lvbs in locks with granted EX or PR lock levels */ + if (lock->ml.type != LKM_EXMODE && lock->ml.type != LKM_PRMODE) + return; + + if (dlm_lvb_is_empty(mres->lvb)) { + memcpy(mres->lvb, lock->lksb->lvb, DLM_LVB_LEN); + return; + } + + /* Ensure the lvb copied for migration matches in other valid locks */ + if (!memcmp(mres->lvb, lock->lksb->lvb, DLM_LVB_LEN)) + return; + + mlog(ML_ERROR, "Mismatched lvb in lock cookie=%u:%llu, name=%.*s, " + "node=%u\n", + dlm_get_lock_cookie_node(be64_to_cpu(lock->ml.cookie)), + dlm_get_lock_cookie_seq(be64_to_cpu(lock->ml.cookie)), + lock->lockres->lockname.len, lock->lockres->lockname.name, + lock->ml.node); + dlm_print_one_lock_resource(lock->lockres); + BUG(); +} + +/* returns 1 if this lock fills the network structure, + * 0 otherwise */ +static int dlm_add_lock_to_array(struct dlm_lock *lock, + struct dlm_migratable_lockres *mres, int queue) +{ + struct dlm_migratable_lock *ml; + int lock_num = mres->num_locks; + + ml = &(mres->ml[lock_num]); + ml->cookie = lock->ml.cookie; + ml->type = lock->ml.type; + ml->convert_type = lock->ml.convert_type; + ml->highest_blocked = lock->ml.highest_blocked; + ml->list = queue; + if (lock->lksb) { + ml->flags = lock->lksb->flags; + dlm_prepare_lvb_for_migration(lock, mres, queue); + } + ml->node = lock->ml.node; + mres->num_locks++; + /* we reached the max, send this network message */ + if (mres->num_locks == DLM_MAX_MIGRATABLE_LOCKS) + return 1; + return 0; +} + +static void dlm_add_dummy_lock(struct dlm_ctxt *dlm, + struct dlm_migratable_lockres *mres) +{ + struct dlm_lock dummy; + memset(&dummy, 0, sizeof(dummy)); + dummy.ml.cookie = 0; + dummy.ml.type = LKM_IVMODE; + dummy.ml.convert_type = LKM_IVMODE; + dummy.ml.highest_blocked = LKM_IVMODE; + dummy.lksb = NULL; + dummy.ml.node = dlm->node_num; + dlm_add_lock_to_array(&dummy, mres, DLM_BLOCKED_LIST); +} + +static inline int dlm_is_dummy_lock(struct dlm_ctxt *dlm, + struct dlm_migratable_lock *ml, + u8 *nodenum) +{ + if (unlikely(ml->cookie == 0 && + ml->type == LKM_IVMODE && + ml->convert_type == LKM_IVMODE && + ml->highest_blocked == LKM_IVMODE && + ml->list == DLM_BLOCKED_LIST)) { + *nodenum = ml->node; + return 1; + } + return 0; +} + +int dlm_send_one_lockres(struct dlm_ctxt *dlm, struct dlm_lock_resource *res, + struct dlm_migratable_lockres *mres, + u8 send_to, u8 flags) +{ + struct list_head *queue; + int total_locks, i; + u64 mig_cookie = 0; + struct dlm_lock *lock; + int ret = 0; + + BUG_ON(!(flags & (DLM_MRES_RECOVERY|DLM_MRES_MIGRATION))); + + mlog(0, "sending to %u\n", send_to); + + total_locks = dlm_num_locks_in_lockres(res); + if (total_locks > DLM_MAX_MIGRATABLE_LOCKS) { + /* rare, but possible */ + mlog(0, "argh. lockres has %d locks. this will " + "require more than one network packet to " + "migrate\n", total_locks); + mig_cookie = dlm_get_next_mig_cookie(); + } + + dlm_init_migratable_lockres(mres, res->lockname.name, + res->lockname.len, total_locks, + mig_cookie, flags, res->owner); + + total_locks = 0; + for (i=DLM_GRANTED_LIST; i<=DLM_BLOCKED_LIST; i++) { + queue = dlm_list_idx_to_ptr(res, i); + list_for_each_entry(lock, queue, list) { + /* add another lock. */ + total_locks++; + if (!dlm_add_lock_to_array(lock, mres, i)) + continue; + + /* this filled the lock message, + * we must send it immediately. */ + ret = dlm_send_mig_lockres_msg(dlm, mres, send_to, + res, total_locks); + if (ret < 0) + goto error; + } + } + if (total_locks == 0) { + /* send a dummy lock to indicate a mastery reference only */ + mlog(0, "%s:%.*s: sending dummy lock to %u, %s\n", + dlm->name, res->lockname.len, res->lockname.name, + send_to, flags & DLM_MRES_RECOVERY ? "recovery" : + "migration"); + dlm_add_dummy_lock(dlm, mres); + } + /* flush any remaining locks */ + ret = dlm_send_mig_lockres_msg(dlm, mres, send_to, res, total_locks); + if (ret < 0) + goto error; + return ret; + +error: + mlog(ML_ERROR, "%s: dlm_send_mig_lockres_msg returned %d\n", + dlm->name, ret); + if (!dlm_is_host_down(ret)) + BUG(); + mlog(0, "%s: node %u went down while sending %s " + "lockres %.*s\n", dlm->name, send_to, + flags & DLM_MRES_RECOVERY ? "recovery" : "migration", + res->lockname.len, res->lockname.name); + return ret; +} + + + +/* + * this message will contain no more than one page worth of + * recovery data, and it will work on only one lockres. + * there may be many locks in this page, and we may need to wait + * for additional packets to complete all the locks (rare, but + * possible). + */ +/* + * NOTE: the allocation error cases here are scary + * we really cannot afford to fail an alloc in recovery + * do we spin? returning an error only delays the problem really + */ + +int dlm_mig_lockres_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data) +{ + struct dlm_ctxt *dlm = data; + struct dlm_migratable_lockres *mres = + (struct dlm_migratable_lockres *)msg->buf; + int ret = 0; + u8 real_master; + u8 extra_refs = 0; + char *buf = NULL; + struct dlm_work_item *item = NULL; + struct dlm_lock_resource *res = NULL; + unsigned int hash; + + if (!dlm_grab(dlm)) + return -EINVAL; + + if (mres->lockname_len > DLM_LOCKID_NAME_MAX || + mres->num_locks > DLM_MAX_MIGRATABLE_LOCKS || + be16_to_cpu(msg->data_len) < struct_size(mres, ml, mres->num_locks)) { + mlog(ML_ERROR, "%s: invalid lockres migration message from %u\n", + dlm->name, mres->master); + dlm_put(dlm); + return -EINVAL; + } + + if (!dlm_joined(dlm)) { + mlog(ML_ERROR, "Domain %s not joined! " + "lockres %.*s, master %u\n", + dlm->name, mres->lockname_len, + mres->lockname, mres->master); + dlm_put(dlm); + return -EINVAL; + } + + BUG_ON(!(mres->flags & (DLM_MRES_RECOVERY|DLM_MRES_MIGRATION))); + + real_master = mres->master; + if (real_master == DLM_LOCK_RES_OWNER_UNKNOWN) { + /* cannot migrate a lockres with no master */ + BUG_ON(!(mres->flags & DLM_MRES_RECOVERY)); + } + + mlog(0, "%s message received from node %u\n", + (mres->flags & DLM_MRES_RECOVERY) ? + "recovery" : "migration", mres->master); + if (mres->flags & DLM_MRES_ALL_DONE) + mlog(0, "all done flag. all lockres data received!\n"); + + ret = -ENOMEM; + buf = kmalloc(be16_to_cpu(msg->data_len), GFP_NOFS); + item = kzalloc_obj(*item, GFP_NOFS); + if (!buf || !item) + goto leave; + + /* lookup the lock to see if we have a secondary queue for this + * already... just add the locks in and this will have its owner + * and RECOVERY flag changed when it completes. */ + hash = dlm_lockid_hash(mres->lockname, mres->lockname_len); + spin_lock(&dlm->spinlock); + res = __dlm_lookup_lockres_full(dlm, mres->lockname, mres->lockname_len, + hash); + if (res) { + /* this will get a ref on res */ + /* mark it as recovering/migrating and hash it */ + spin_lock(&res->spinlock); + if (res->state & DLM_LOCK_RES_DROPPING_REF) { + mlog(0, "%s: node is attempting to migrate " + "lockres %.*s, but marked as dropping " + " ref!\n", dlm->name, + mres->lockname_len, mres->lockname); + ret = -EINVAL; + spin_unlock(&res->spinlock); + spin_unlock(&dlm->spinlock); + dlm_lockres_put(res); + goto leave; + } + + if (mres->flags & DLM_MRES_RECOVERY) { + res->state |= DLM_LOCK_RES_RECOVERING; + } else { + if (res->state & DLM_LOCK_RES_MIGRATING) { + /* this is at least the second + * lockres message */ + mlog(0, "lock %.*s is already migrating\n", + mres->lockname_len, + mres->lockname); + } else if (res->state & DLM_LOCK_RES_RECOVERING) { + /* caller should BUG */ + mlog(ML_ERROR, "node is attempting to migrate " + "lock %.*s, but marked as recovering!\n", + mres->lockname_len, mres->lockname); + ret = -EFAULT; + spin_unlock(&res->spinlock); + spin_unlock(&dlm->spinlock); + dlm_lockres_put(res); + goto leave; + } + res->state |= DLM_LOCK_RES_MIGRATING; + } + spin_unlock(&res->spinlock); + spin_unlock(&dlm->spinlock); + } else { + spin_unlock(&dlm->spinlock); + /* need to allocate, just like if it was + * mastered here normally */ + res = dlm_new_lockres(dlm, mres->lockname, mres->lockname_len); + if (!res) + goto leave; + + /* to match the ref that we would have gotten if + * dlm_lookup_lockres had succeeded */ + dlm_lockres_get(res); + + /* mark it as recovering/migrating and hash it */ + if (mres->flags & DLM_MRES_RECOVERY) + res->state |= DLM_LOCK_RES_RECOVERING; + else + res->state |= DLM_LOCK_RES_MIGRATING; + + spin_lock(&dlm->spinlock); + __dlm_insert_lockres(dlm, res); + spin_unlock(&dlm->spinlock); + + /* Add an extra ref for this lock-less lockres lest the + * dlm_thread purges it before we get the chance to add + * locks to it */ + dlm_lockres_get(res); + + /* There are three refs that need to be put. + * 1. Taken above. + * 2. kref_init in dlm_new_lockres()->dlm_init_lockres(). + * 3. dlm_lookup_lockres() + * The first one is handled at the end of this function. The + * other two are handled in the worker thread after locks have + * been attached. Yes, we don't wait for purge time to match + * kref_init. The lockres will still have at least one ref + * added because it is in the hash __dlm_insert_lockres() */ + extra_refs++; + + /* now that the new lockres is inserted, + * make it usable by other processes */ + spin_lock(&res->spinlock); + res->state &= ~DLM_LOCK_RES_IN_PROGRESS; + spin_unlock(&res->spinlock); + wake_up(&res->wq); + } + + /* at this point we have allocated everything we need, + * and we have a hashed lockres with an extra ref and + * the proper res->state flags. */ + ret = 0; + spin_lock(&res->spinlock); + /* drop this either when master requery finds a different master + * or when a lock is added by the recovery worker */ + dlm_lockres_grab_inflight_ref(dlm, res); + if (mres->master == DLM_LOCK_RES_OWNER_UNKNOWN) { + /* migration cannot have an unknown master */ + BUG_ON(!(mres->flags & DLM_MRES_RECOVERY)); + mlog(0, "recovery has passed me a lockres with an " + "unknown owner.. will need to requery: " + "%.*s\n", mres->lockname_len, mres->lockname); + } else { + /* take a reference now to pin the lockres, drop it + * when locks are added in the worker */ + dlm_change_lockres_owner(dlm, res, dlm->node_num); + } + spin_unlock(&res->spinlock); + + /* queue up work for dlm_mig_lockres_worker */ + dlm_grab(dlm); /* get an extra ref for the work item */ + memcpy(buf, msg->buf, be16_to_cpu(msg->data_len)); /* copy the whole message */ + dlm_init_work_item(dlm, item, dlm_mig_lockres_worker, buf); + item->u.ml.lockres = res; /* already have a ref */ + item->u.ml.real_master = real_master; + item->u.ml.extra_ref = extra_refs; + spin_lock(&dlm->work_lock); + list_add_tail(&item->list, &dlm->work_list); + spin_unlock(&dlm->work_lock); + queue_work(dlm->dlm_worker, &dlm->dispatched_work); + +leave: + /* One extra ref taken needs to be put here */ + if (extra_refs) + dlm_lockres_put(res); + + dlm_put(dlm); + if (ret < 0) { + kfree(buf); + kfree(item); + mlog_errno(ret); + } + + return ret; +} + + +static void dlm_mig_lockres_worker(struct dlm_work_item *item, void *data) +{ + struct dlm_ctxt *dlm; + struct dlm_migratable_lockres *mres; + int ret = 0; + struct dlm_lock_resource *res; + u8 real_master; + u8 extra_ref; + + dlm = item->dlm; + mres = (struct dlm_migratable_lockres *)data; + + res = item->u.ml.lockres; + real_master = item->u.ml.real_master; + extra_ref = item->u.ml.extra_ref; + + if (real_master == DLM_LOCK_RES_OWNER_UNKNOWN) { + /* this case is super-rare. only occurs if + * node death happens during migration. */ +again: + ret = dlm_lockres_master_requery(dlm, res, &real_master); + if (ret < 0) { + mlog(0, "dlm_lockres_master_requery ret=%d\n", + ret); + goto again; + } + if (real_master == DLM_LOCK_RES_OWNER_UNKNOWN) { + mlog(0, "lockres %.*s not claimed. " + "this node will take it.\n", + res->lockname.len, res->lockname.name); + } else { + spin_lock(&res->spinlock); + dlm_lockres_drop_inflight_ref(dlm, res); + spin_unlock(&res->spinlock); + mlog(0, "master needs to respond to sender " + "that node %u still owns %.*s\n", + real_master, res->lockname.len, + res->lockname.name); + /* cannot touch this lockres */ + goto leave; + } + } + + ret = dlm_process_recovery_data(dlm, res, mres); + if (ret < 0) + mlog(0, "dlm_process_recovery_data returned %d\n", ret); + else + mlog(0, "dlm_process_recovery_data succeeded\n"); + + if ((mres->flags & (DLM_MRES_MIGRATION|DLM_MRES_ALL_DONE)) == + (DLM_MRES_MIGRATION|DLM_MRES_ALL_DONE)) { + ret = dlm_finish_migration(dlm, res, mres->master); + if (ret < 0) + mlog_errno(ret); + } + +leave: + /* See comment in dlm_mig_lockres_handler() */ + if (res) { + if (extra_ref) + dlm_lockres_put(res); + dlm_lockres_put(res); + } + kfree(data); +} + + + +static int dlm_lockres_master_requery(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + u8 *real_master) +{ + struct dlm_node_iter iter; + int nodenum; + int ret = 0; + + *real_master = DLM_LOCK_RES_OWNER_UNKNOWN; + + /* we only reach here if one of the two nodes in a + * migration died while the migration was in progress. + * at this point we need to requery the master. we + * know that the new_master got as far as creating + * an mle on at least one node, but we do not know + * if any nodes had actually cleared the mle and set + * the master to the new_master. the old master + * is supposed to set the owner to UNKNOWN in the + * event of a new_master death, so the only possible + * responses that we can get from nodes here are + * that the master is new_master, or that the master + * is UNKNOWN. + * if all nodes come back with UNKNOWN then we know + * the lock needs remastering here. + * if any node comes back with a valid master, check + * to see if that master is the one that we are + * recovering. if so, then the new_master died and + * we need to remaster this lock. if not, then the + * new_master survived and that node will respond to + * other nodes about the owner. + * if there is an owner, this node needs to dump this + * lockres and alert the sender that this lockres + * was rejected. */ + spin_lock(&dlm->spinlock); + dlm_node_iter_init(dlm->domain_map, &iter); + spin_unlock(&dlm->spinlock); + + while ((nodenum = dlm_node_iter_next(&iter)) >= 0) { + /* do not send to self */ + if (nodenum == dlm->node_num) + continue; + ret = dlm_do_master_requery(dlm, res, nodenum, real_master); + if (ret < 0) { + mlog_errno(ret); + if (!dlm_is_host_down(ret)) + BUG(); + /* host is down, so answer for that node would be + * DLM_LOCK_RES_OWNER_UNKNOWN. continue. */ + } + if (*real_master != DLM_LOCK_RES_OWNER_UNKNOWN) { + mlog(0, "lock master is %u\n", *real_master); + break; + } + } + return ret; +} + + +int dlm_do_master_requery(struct dlm_ctxt *dlm, struct dlm_lock_resource *res, + u8 nodenum, u8 *real_master) +{ + int ret; + struct dlm_master_requery req; + int status = DLM_LOCK_RES_OWNER_UNKNOWN; + + memset(&req, 0, sizeof(req)); + req.node_idx = dlm->node_num; + req.namelen = res->lockname.len; + memcpy(req.name, res->lockname.name, res->lockname.len); + +resend: + ret = o2net_send_message(DLM_MASTER_REQUERY_MSG, dlm->key, + &req, sizeof(req), nodenum, &status); + if (ret < 0) + mlog(ML_ERROR, "Error %d when sending message %u (key " + "0x%x) to node %u\n", ret, DLM_MASTER_REQUERY_MSG, + dlm->key, nodenum); + else if (status == -ENOMEM) { + mlog_errno(status); + msleep(50); + goto resend; + } else { + BUG_ON(status < 0); + BUG_ON(status > DLM_LOCK_RES_OWNER_UNKNOWN); + *real_master = (u8) (status & 0xff); + mlog(0, "node %u responded to master requery with %u\n", + nodenum, *real_master); + ret = 0; + } + return ret; +} + + +/* this function cannot error, so unless the sending + * or receiving of the message failed, the owner can + * be trusted */ +int dlm_master_requery_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data) +{ + struct dlm_ctxt *dlm = data; + struct dlm_master_requery *req = (struct dlm_master_requery *)msg->buf; + struct dlm_lock_resource *res = NULL; + unsigned int hash; + int master = DLM_LOCK_RES_OWNER_UNKNOWN; + u32 flags = DLM_ASSERT_MASTER_REQUERY; + int dispatched = 0; + + if (!dlm_grab(dlm)) { + /* since the domain has gone away on this + * node, the proper response is UNKNOWN */ + return master; + } + + hash = dlm_lockid_hash(req->name, req->namelen); + + spin_lock(&dlm->spinlock); + res = __dlm_lookup_lockres(dlm, req->name, req->namelen, hash); + if (res) { + spin_lock(&res->spinlock); + master = res->owner; + if (master == dlm->node_num) { + int ret = dlm_dispatch_assert_master(dlm, res, + 0, 0, flags); + if (ret < 0) { + mlog_errno(ret); + spin_unlock(&res->spinlock); + dlm_lockres_put(res); + spin_unlock(&dlm->spinlock); + dlm_put(dlm); + /* sender will take care of this and retry */ + return ret; + } else { + dispatched = 1; + __dlm_lockres_grab_inflight_worker(dlm, res); + spin_unlock(&res->spinlock); + } + } else { + /* put.. in case we are not the master */ + spin_unlock(&res->spinlock); + dlm_lockres_put(res); + } + } + spin_unlock(&dlm->spinlock); + + if (!dispatched) + dlm_put(dlm); + return master; +} + +static inline struct list_head * +dlm_list_num_to_pointer(struct dlm_lock_resource *res, int list_num) +{ + struct list_head *ret; + BUG_ON(list_num < 0); + BUG_ON(list_num > 2); + ret = &(res->granted); + ret += list_num; + return ret; +} +/* TODO: do ast flush business + * TODO: do MIGRATING and RECOVERING spinning + */ + +/* +* NOTE about in-flight requests during migration: +* +* Before attempting the migrate, the master has marked the lockres as +* MIGRATING and then flushed all of its pending ASTS. So any in-flight +* requests either got queued before the MIGRATING flag got set, in which +* case the lock data will reflect the change and a return message is on +* the way, or the request failed to get in before MIGRATING got set. In +* this case, the caller will be told to spin and wait for the MIGRATING +* flag to be dropped, then recheck the master. +* This holds true for the convert, cancel and unlock cases, and since lvb +* updates are tied to these same messages, it applies to lvb updates as +* well. For the lock case, there is no way a lock can be on the master +* queue and not be on the secondary queue since the lock is always added +* locally first. This means that the new target node will never be sent +* a lock that he doesn't already have on the list. +* In total, this means that the local lock is correct and should not be +* updated to match the one sent by the master. Any messages sent back +* from the master before the MIGRATING flag will bring the lock properly +* up-to-date, and the change will be ordered properly for the waiter. +* We will *not* attempt to modify the lock underneath the waiter. +*/ + +static int dlm_process_recovery_data(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_migratable_lockres *mres) +{ + struct dlm_migratable_lock *ml; + struct list_head *queue, *iter; + struct list_head *tmpq = NULL; + struct dlm_lock *newlock = NULL; + struct dlm_lockstatus *lksb = NULL; + int ret = 0; + int i, j, bad; + struct dlm_lock *lock; + u8 from = O2NM_MAX_NODES; + __be64 c; + + mlog(0, "running %d locks for this lockres\n", mres->num_locks); + for (i=0; inum_locks; i++) { + ml = &(mres->ml[i]); + + if (dlm_is_dummy_lock(dlm, ml, &from)) { + /* placeholder, just need to set the refmap bit */ + BUG_ON(mres->num_locks != 1); + mlog(0, "%s:%.*s: dummy lock for %u\n", + dlm->name, mres->lockname_len, mres->lockname, + from); + spin_lock(&res->spinlock); + dlm_lockres_set_refmap_bit(dlm, res, from); + spin_unlock(&res->spinlock); + break; + } + BUG_ON(ml->highest_blocked != LKM_IVMODE); + newlock = NULL; + lksb = NULL; + + queue = dlm_list_num_to_pointer(res, ml->list); + tmpq = NULL; + + /* if the lock is for the local node it needs to + * be moved to the proper location within the queue. + * do not allocate a new lock structure. */ + if (ml->node == dlm->node_num) { + /* MIGRATION ONLY! */ + BUG_ON(!(mres->flags & DLM_MRES_MIGRATION)); + + lock = NULL; + spin_lock(&res->spinlock); + for (j = DLM_GRANTED_LIST; j <= DLM_BLOCKED_LIST; j++) { + tmpq = dlm_list_idx_to_ptr(res, j); + list_for_each(iter, tmpq) { + lock = list_entry(iter, + struct dlm_lock, list); + if (lock->ml.cookie == ml->cookie) + break; + lock = NULL; + } + if (lock) + break; + } + + /* lock is always created locally first, and + * destroyed locally last. it must be on the list */ + if (!lock) { + c = ml->cookie; + mlog(ML_ERROR, "Could not find local lock " + "with cookie %u:%llu, node %u, " + "list %u, flags 0x%x, type %d, " + "conv %d, highest blocked %d\n", + dlm_get_lock_cookie_node(be64_to_cpu(c)), + dlm_get_lock_cookie_seq(be64_to_cpu(c)), + ml->node, ml->list, ml->flags, ml->type, + ml->convert_type, ml->highest_blocked); + __dlm_print_one_lock_resource(res); + BUG(); + } + + if (lock->ml.node != ml->node) { + c = lock->ml.cookie; + mlog(ML_ERROR, "Mismatched node# in lock " + "cookie %u:%llu, name %.*s, node %u\n", + dlm_get_lock_cookie_node(be64_to_cpu(c)), + dlm_get_lock_cookie_seq(be64_to_cpu(c)), + res->lockname.len, res->lockname.name, + lock->ml.node); + c = ml->cookie; + mlog(ML_ERROR, "Migrate lock cookie %u:%llu, " + "node %u, list %u, flags 0x%x, type %d, " + "conv %d, highest blocked %d\n", + dlm_get_lock_cookie_node(be64_to_cpu(c)), + dlm_get_lock_cookie_seq(be64_to_cpu(c)), + ml->node, ml->list, ml->flags, ml->type, + ml->convert_type, ml->highest_blocked); + __dlm_print_one_lock_resource(res); + BUG(); + } + + if (tmpq != queue) { + c = ml->cookie; + mlog(0, "Lock cookie %u:%llu was on list %u " + "instead of list %u for %.*s\n", + dlm_get_lock_cookie_node(be64_to_cpu(c)), + dlm_get_lock_cookie_seq(be64_to_cpu(c)), + j, ml->list, res->lockname.len, + res->lockname.name); + __dlm_print_one_lock_resource(res); + spin_unlock(&res->spinlock); + continue; + } + + /* see NOTE above about why we do not update + * to match the master here */ + + /* move the lock to its proper place */ + /* do not alter lock refcount. switching lists. */ + list_move_tail(&lock->list, queue); + spin_unlock(&res->spinlock); + + mlog(0, "just reordered a local lock!\n"); + continue; + } + + /* lock is for another node. */ + newlock = dlm_new_lock(ml->type, ml->node, + be64_to_cpu(ml->cookie), NULL); + if (!newlock) { + ret = -ENOMEM; + goto leave; + } + lksb = newlock->lksb; + dlm_lock_attach_lockres(newlock, res); + + if (ml->convert_type != LKM_IVMODE) { + BUG_ON(queue != &res->converting); + newlock->ml.convert_type = ml->convert_type; + } + lksb->flags |= (ml->flags & + (DLM_LKSB_PUT_LVB|DLM_LKSB_GET_LVB)); + + if (ml->type == LKM_NLMODE) + goto skip_lvb; + + /* + * If the lock is in the blocked list it can't have a valid lvb, + * so skip it + */ + if (ml->list == DLM_BLOCKED_LIST) + goto skip_lvb; + + if (!dlm_lvb_is_empty(mres->lvb)) { + if (lksb->flags & DLM_LKSB_PUT_LVB) { + /* other node was trying to update + * lvb when node died. recreate the + * lksb with the updated lvb. */ + memcpy(lksb->lvb, mres->lvb, DLM_LVB_LEN); + /* the lock resource lvb update must happen + * NOW, before the spinlock is dropped. + * we no longer wait for the AST to update + * the lvb. */ + memcpy(res->lvb, mres->lvb, DLM_LVB_LEN); + } else { + /* otherwise, the node is sending its + * most recent valid lvb info */ + BUG_ON(ml->type != LKM_EXMODE && + ml->type != LKM_PRMODE); + if (!dlm_lvb_is_empty(res->lvb) && + (ml->type == LKM_EXMODE || + memcmp(res->lvb, mres->lvb, DLM_LVB_LEN))) { + int i; + mlog(ML_ERROR, "%s:%.*s: received bad " + "lvb! type=%d\n", dlm->name, + res->lockname.len, + res->lockname.name, ml->type); + printk("lockres lvb=["); + for (i=0; ilvb[i]); + printk("]\nmigrated lvb=["); + for (i=0; ilvb[i]); + printk("]\n"); + dlm_print_one_lock_resource(res); + BUG(); + } + memcpy(res->lvb, mres->lvb, DLM_LVB_LEN); + } + } +skip_lvb: + + /* NOTE: + * wrt lock queue ordering and recovery: + * 1. order of locks on granted queue is + * meaningless. + * 2. order of locks on converting queue is + * LOST with the node death. sorry charlie. + * 3. order of locks on the blocked queue is + * also LOST. + * order of locks does not affect integrity, it + * just means that a lock request may get pushed + * back in line as a result of the node death. + * also note that for a given node the lock order + * for its secondary queue locks is preserved + * relative to each other, but clearly *not* + * preserved relative to locks from other nodes. + */ + bad = 0; + spin_lock(&res->spinlock); + list_for_each_entry(lock, queue, list) { + if (lock->ml.cookie == ml->cookie) { + c = lock->ml.cookie; + mlog(ML_ERROR, "%s:%.*s: %u:%llu: lock already " + "exists on this lockres!\n", dlm->name, + res->lockname.len, res->lockname.name, + dlm_get_lock_cookie_node(be64_to_cpu(c)), + dlm_get_lock_cookie_seq(be64_to_cpu(c))); + + mlog(ML_NOTICE, "sent lock: type=%d, conv=%d, " + "node=%u, cookie=%u:%llu, queue=%d\n", + ml->type, ml->convert_type, ml->node, + dlm_get_lock_cookie_node(be64_to_cpu(ml->cookie)), + dlm_get_lock_cookie_seq(be64_to_cpu(ml->cookie)), + ml->list); + + __dlm_print_one_lock_resource(res); + bad = 1; + break; + } + } + if (!bad) { + dlm_lock_get(newlock); + if (mres->flags & DLM_MRES_RECOVERY && + ml->list == DLM_CONVERTING_LIST && + newlock->ml.type > + newlock->ml.convert_type) { + /* newlock is doing downconvert, add it to the + * head of converting list */ + list_add(&newlock->list, queue); + } else + list_add_tail(&newlock->list, queue); + mlog(0, "%s:%.*s: added lock for node %u, " + "setting refmap bit\n", dlm->name, + res->lockname.len, res->lockname.name, ml->node); + dlm_lockres_set_refmap_bit(dlm, res, ml->node); + } + spin_unlock(&res->spinlock); + } + mlog(0, "done running all the locks\n"); + +leave: + /* balance the ref taken when the work was queued */ + spin_lock(&res->spinlock); + dlm_lockres_drop_inflight_ref(dlm, res); + spin_unlock(&res->spinlock); + + if (ret < 0) + mlog_errno(ret); + + return ret; +} + +void dlm_move_lockres_to_recovery_list(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res) +{ + int i; + struct list_head *queue; + struct dlm_lock *lock, *next; + + assert_spin_locked(&dlm->spinlock); + assert_spin_locked(&res->spinlock); + res->state |= DLM_LOCK_RES_RECOVERING; + if (!list_empty(&res->recovering)) { + mlog(0, + "Recovering res %s:%.*s, is already on recovery list!\n", + dlm->name, res->lockname.len, res->lockname.name); + list_del_init(&res->recovering); + dlm_lockres_put(res); + } + /* We need to hold a reference while on the recovery list */ + dlm_lockres_get(res); + list_add_tail(&res->recovering, &dlm->reco.resources); + + /* find any pending locks and put them back on proper list */ + for (i=DLM_BLOCKED_LIST; i>=DLM_GRANTED_LIST; i--) { + queue = dlm_list_idx_to_ptr(res, i); + list_for_each_entry_safe(lock, next, queue, list) { + dlm_lock_get(lock); + if (lock->convert_pending) { + /* move converting lock back to granted */ + mlog(0, "node died with convert pending " + "on %.*s. move back to granted list.\n", + res->lockname.len, res->lockname.name); + dlm_revert_pending_convert(res, lock); + lock->convert_pending = 0; + } else if (lock->lock_pending) { + /* remove pending lock requests completely */ + BUG_ON(i != DLM_BLOCKED_LIST); + mlog(0, "node died with lock pending " + "on %.*s. remove from blocked list and skip.\n", + res->lockname.len, res->lockname.name); + /* lock will be floating until ref in + * dlmlock_remote is freed after the network + * call returns. ok for it to not be on any + * list since no ast can be called + * (the master is dead). */ + dlm_revert_pending_lock(res, lock); + lock->lock_pending = 0; + } else if (lock->unlock_pending) { + /* if an unlock was in progress, treat as + * if this had completed successfully + * before sending this lock state to the + * new master. note that the dlm_unlock + * call is still responsible for calling + * the unlockast. that will happen after + * the network call times out. for now, + * just move lists to prepare the new + * recovery master. */ + BUG_ON(i != DLM_GRANTED_LIST); + mlog(0, "node died with unlock pending " + "on %.*s. remove from blocked list and skip.\n", + res->lockname.len, res->lockname.name); + dlm_commit_pending_unlock(res, lock); + lock->unlock_pending = 0; + } else if (lock->cancel_pending) { + /* if a cancel was in progress, treat as + * if this had completed successfully + * before sending this lock state to the + * new master */ + BUG_ON(i != DLM_CONVERTING_LIST); + mlog(0, "node died with cancel pending " + "on %.*s. move back to granted list.\n", + res->lockname.len, res->lockname.name); + dlm_commit_pending_cancel(res, lock); + lock->cancel_pending = 0; + } + dlm_lock_put(lock); + } + } +} + + + +/* removes all recovered locks from the recovery list. + * sets the res->owner to the new master. + * unsets the RECOVERY flag and wakes waiters. */ +static void dlm_finish_local_lockres_recovery(struct dlm_ctxt *dlm, + u8 dead_node, u8 new_master) +{ + int i; + struct hlist_head *bucket; + struct dlm_lock_resource *res, *next; + + assert_spin_locked(&dlm->spinlock); + + list_for_each_entry_safe(res, next, &dlm->reco.resources, recovering) { + if (res->owner == dead_node) { + mlog(0, "%s: res %.*s, Changing owner from %u to %u\n", + dlm->name, res->lockname.len, res->lockname.name, + res->owner, new_master); + list_del_init(&res->recovering); + spin_lock(&res->spinlock); + /* new_master has our reference from + * the lock state sent during recovery */ + dlm_change_lockres_owner(dlm, res, new_master); + res->state &= ~DLM_LOCK_RES_RECOVERING; + if (__dlm_lockres_has_locks(res)) + __dlm_dirty_lockres(dlm, res); + spin_unlock(&res->spinlock); + wake_up(&res->wq); + dlm_lockres_put(res); + } + } + + /* this will become unnecessary eventually, but + * for now we need to run the whole hash, clear + * the RECOVERING state and set the owner + * if necessary */ + for (i = 0; i < DLM_HASH_BUCKETS; i++) { + bucket = dlm_lockres_hash(dlm, i); + hlist_for_each_entry(res, bucket, hash_node) { + if (res->state & DLM_LOCK_RES_RECOVERY_WAITING) { + spin_lock(&res->spinlock); + res->state &= ~DLM_LOCK_RES_RECOVERY_WAITING; + spin_unlock(&res->spinlock); + wake_up(&res->wq); + } + + if (!(res->state & DLM_LOCK_RES_RECOVERING)) + continue; + + if (res->owner != dead_node && + res->owner != dlm->node_num) + continue; + + if (!list_empty(&res->recovering)) { + list_del_init(&res->recovering); + dlm_lockres_put(res); + } + + /* new_master has our reference from + * the lock state sent during recovery */ + mlog(0, "%s: res %.*s, Changing owner from %u to %u\n", + dlm->name, res->lockname.len, res->lockname.name, + res->owner, new_master); + spin_lock(&res->spinlock); + dlm_change_lockres_owner(dlm, res, new_master); + res->state &= ~DLM_LOCK_RES_RECOVERING; + if (__dlm_lockres_has_locks(res)) + __dlm_dirty_lockres(dlm, res); + spin_unlock(&res->spinlock); + wake_up(&res->wq); + } + } +} + +static inline int dlm_lvb_needs_invalidation(struct dlm_lock *lock, int local) +{ + if (local) { + if (lock->ml.type != LKM_EXMODE && + lock->ml.type != LKM_PRMODE) + return 1; + } else if (lock->ml.type == LKM_EXMODE) + return 1; + return 0; +} + +static void dlm_revalidate_lvb(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, u8 dead_node) +{ + struct list_head *queue; + struct dlm_lock *lock; + int blank_lvb = 0, local = 0; + int i; + u8 search_node; + + assert_spin_locked(&dlm->spinlock); + assert_spin_locked(&res->spinlock); + + if (res->owner == dlm->node_num) + /* if this node owned the lockres, and if the dead node + * had an EX when he died, blank out the lvb */ + search_node = dead_node; + else { + /* if this is a secondary lockres, and we had no EX or PR + * locks granted, we can no longer trust the lvb */ + search_node = dlm->node_num; + local = 1; /* check local state for valid lvb */ + } + + for (i=DLM_GRANTED_LIST; i<=DLM_CONVERTING_LIST; i++) { + queue = dlm_list_idx_to_ptr(res, i); + list_for_each_entry(lock, queue, list) { + if (lock->ml.node == search_node) { + if (dlm_lvb_needs_invalidation(lock, local)) { + /* zero the lksb lvb and lockres lvb */ + blank_lvb = 1; + memset(lock->lksb->lvb, 0, DLM_LVB_LEN); + } + } + } + } + + if (blank_lvb) { + mlog(0, "clearing %.*s lvb, dead node %u had EX\n", + res->lockname.len, res->lockname.name, dead_node); + memset(res->lvb, 0, DLM_LVB_LEN); + } +} + +static void dlm_free_dead_locks(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, u8 dead_node) +{ + struct dlm_lock *lock, *next; + unsigned int freed = 0; + + /* this node is the lockres master: + * 1) remove any stale locks for the dead node + * 2) if the dead node had an EX when he died, blank out the lvb + */ + assert_spin_locked(&dlm->spinlock); + assert_spin_locked(&res->spinlock); + + /* We do two dlm_lock_put(). One for removing from list and the other is + * to force the DLM_UNLOCK_FREE_LOCK action so as to free the locks */ + + /* TODO: check pending_asts, pending_basts here */ + list_for_each_entry_safe(lock, next, &res->granted, list) { + if (lock->ml.node == dead_node) { + list_del_init(&lock->list); + dlm_lock_put(lock); + /* Can't schedule DLM_UNLOCK_FREE_LOCK - do manually */ + dlm_lock_put(lock); + freed++; + } + } + list_for_each_entry_safe(lock, next, &res->converting, list) { + if (lock->ml.node == dead_node) { + list_del_init(&lock->list); + dlm_lock_put(lock); + /* Can't schedule DLM_UNLOCK_FREE_LOCK - do manually */ + dlm_lock_put(lock); + freed++; + } + } + list_for_each_entry_safe(lock, next, &res->blocked, list) { + if (lock->ml.node == dead_node) { + list_del_init(&lock->list); + dlm_lock_put(lock); + /* Can't schedule DLM_UNLOCK_FREE_LOCK - do manually */ + dlm_lock_put(lock); + freed++; + } + } + + if (freed) { + mlog(0, "%s:%.*s: freed %u locks for dead node %u, " + "dropping ref from lockres\n", dlm->name, + res->lockname.len, res->lockname.name, freed, dead_node); + if(!test_bit(dead_node, res->refmap)) { + mlog(ML_ERROR, "%s:%.*s: freed %u locks for dead node %u, " + "but ref was not set\n", dlm->name, + res->lockname.len, res->lockname.name, freed, dead_node); + __dlm_print_one_lock_resource(res); + } + res->state |= DLM_LOCK_RES_RECOVERY_WAITING; + dlm_lockres_clear_refmap_bit(dlm, res, dead_node); + } else if (test_bit(dead_node, res->refmap)) { + mlog(0, "%s:%.*s: dead node %u had a ref, but had " + "no locks and had not purged before dying\n", dlm->name, + res->lockname.len, res->lockname.name, dead_node); + dlm_lockres_clear_refmap_bit(dlm, res, dead_node); + } + + /* do not kick thread yet */ + __dlm_dirty_lockres(dlm, res); +} + +static void dlm_do_local_recovery_cleanup(struct dlm_ctxt *dlm, u8 dead_node) +{ + struct dlm_lock_resource *res; + int i; + struct hlist_head *bucket; + struct hlist_node *tmp; + struct dlm_lock *lock; + + + /* purge any stale mles */ + dlm_clean_master_list(dlm, dead_node); + + /* + * now clean up all lock resources. there are two rules: + * + * 1) if the dead node was the master, move the lockres + * to the recovering list. set the RECOVERING flag. + * this lockres needs to be cleaned up before it can + * be used further. + * + * 2) if this node was the master, remove all locks from + * each of the lockres queues that were owned by the + * dead node. once recovery finishes, the dlm thread + * can be kicked again to see if any ASTs or BASTs + * need to be fired as a result. + */ + for (i = 0; i < DLM_HASH_BUCKETS; i++) { + bucket = dlm_lockres_hash(dlm, i); + hlist_for_each_entry_safe(res, tmp, bucket, hash_node) { + /* always prune any $RECOVERY entries for dead nodes, + * otherwise hangs can occur during later recovery */ + if (dlm_is_recovery_lock(res->lockname.name, + res->lockname.len)) { + spin_lock(&res->spinlock); + list_for_each_entry(lock, &res->granted, list) { + if (lock->ml.node == dead_node) { + mlog(0, "AHA! there was " + "a $RECOVERY lock for dead " + "node %u (%s)!\n", + dead_node, dlm->name); + list_del_init(&lock->list); + dlm_lock_put(lock); + /* Can't schedule + * DLM_UNLOCK_FREE_LOCK + * - do manually */ + dlm_lock_put(lock); + break; + } + } + + if ((res->owner == dead_node) && + (res->state & DLM_LOCK_RES_DROPPING_REF)) { + dlm_lockres_get(res); + __dlm_do_purge_lockres(dlm, res); + spin_unlock(&res->spinlock); + wake_up(&res->wq); + dlm_lockres_put(res); + continue; + } else if (res->owner == dlm->node_num) + dlm_lockres_clear_refmap_bit(dlm, res, dead_node); + spin_unlock(&res->spinlock); + continue; + } + spin_lock(&res->spinlock); + /* zero the lvb if necessary */ + dlm_revalidate_lvb(dlm, res, dead_node); + if (res->owner == dead_node) { + if (res->state & DLM_LOCK_RES_DROPPING_REF) { + mlog(0, "%s:%.*s: owned by " + "dead node %u, this node was " + "dropping its ref when master died. " + "continue, purging the lockres.\n", + dlm->name, res->lockname.len, + res->lockname.name, dead_node); + dlm_lockres_get(res); + __dlm_do_purge_lockres(dlm, res); + spin_unlock(&res->spinlock); + wake_up(&res->wq); + dlm_lockres_put(res); + continue; + } + dlm_move_lockres_to_recovery_list(dlm, res); + } else if (res->owner == dlm->node_num) { + dlm_free_dead_locks(dlm, res, dead_node); + __dlm_lockres_calc_usage(dlm, res); + } else if (res->owner == DLM_LOCK_RES_OWNER_UNKNOWN) { + if (test_bit(dead_node, res->refmap)) { + mlog(0, "%s:%.*s: dead node %u had a ref, but had " + "no locks and had not purged before dying\n", + dlm->name, res->lockname.len, + res->lockname.name, dead_node); + dlm_lockres_clear_refmap_bit(dlm, res, dead_node); + } + } + spin_unlock(&res->spinlock); + } + } + +} + +static void __dlm_hb_node_down(struct dlm_ctxt *dlm, int idx) +{ + assert_spin_locked(&dlm->spinlock); + + if (dlm->reco.new_master == idx) { + mlog(0, "%s: recovery master %d just died\n", + dlm->name, idx); + if (dlm->reco.state & DLM_RECO_STATE_FINALIZE) { + /* finalize1 was reached, so it is safe to clear + * the new_master and dead_node. that recovery + * is complete. */ + mlog(0, "%s: dead master %d had reached " + "finalize1 state, clearing\n", dlm->name, idx); + dlm->reco.state &= ~DLM_RECO_STATE_FINALIZE; + __dlm_reset_recovery(dlm); + } + } + + /* Clean up join state on node death. */ + if (dlm->joining_node == idx) { + mlog(0, "Clearing join state for node %u\n", idx); + __dlm_set_joining_node(dlm, DLM_LOCK_RES_OWNER_UNKNOWN); + } + + /* check to see if the node is already considered dead */ + if (!test_bit(idx, dlm->live_nodes_map)) { + mlog(0, "for domain %s, node %d is already dead. " + "another node likely did recovery already.\n", + dlm->name, idx); + return; + } + + /* check to see if we do not care about this node */ + if (!test_bit(idx, dlm->domain_map)) { + /* This also catches the case that we get a node down + * but haven't joined the domain yet. */ + mlog(0, "node %u already removed from domain!\n", idx); + return; + } + + clear_bit(idx, dlm->live_nodes_map); + + /* make sure local cleanup occurs before the heartbeat events */ + if (!test_bit(idx, dlm->recovery_map)) + dlm_do_local_recovery_cleanup(dlm, idx); + + /* notify anything attached to the heartbeat events */ + dlm_hb_event_notify_attached(dlm, idx, 0); + + mlog(0, "node %u being removed from domain map!\n", idx); + clear_bit(idx, dlm->domain_map); + clear_bit(idx, dlm->exit_domain_map); + /* wake up migration waiters if a node goes down. + * perhaps later we can genericize this for other waiters. */ + wake_up(&dlm->migration_wq); + + set_bit(idx, dlm->recovery_map); +} + +void dlm_hb_node_down_cb(struct o2nm_node *node, int idx, void *data) +{ + struct dlm_ctxt *dlm = data; + + if (!dlm_grab(dlm)) + return; + + /* + * This will notify any dlm users that a node in our domain + * went away without notifying us first. + */ + if (test_bit(idx, dlm->domain_map)) + dlm_fire_domain_eviction_callbacks(dlm, idx); + + spin_lock(&dlm->spinlock); + __dlm_hb_node_down(dlm, idx); + spin_unlock(&dlm->spinlock); + + dlm_put(dlm); +} + +void dlm_hb_node_up_cb(struct o2nm_node *node, int idx, void *data) +{ + struct dlm_ctxt *dlm = data; + + if (!dlm_grab(dlm)) + return; + + spin_lock(&dlm->spinlock); + set_bit(idx, dlm->live_nodes_map); + /* do NOT notify mle attached to the heartbeat events. + * new nodes are not interesting in mastery until joined. */ + spin_unlock(&dlm->spinlock); + + dlm_put(dlm); +} + +static void dlm_reco_ast(void *astdata) +{ + struct dlm_ctxt *dlm = astdata; + mlog(0, "ast for recovery lock fired!, this=%u, dlm=%s\n", + dlm->node_num, dlm->name); +} +static void dlm_reco_bast(void *astdata, int blocked_type) +{ + struct dlm_ctxt *dlm = astdata; + mlog(0, "bast for recovery lock fired!, this=%u, dlm=%s\n", + dlm->node_num, dlm->name); +} +static void dlm_reco_unlock_ast(void *astdata, enum dlm_status st) +{ + mlog(0, "unlockast for recovery lock fired!\n"); +} + +/* + * dlm_pick_recovery_master will continually attempt to use + * dlmlock() on the special "$RECOVERY" lockres with the + * LKM_NOQUEUE flag to get an EX. every thread that enters + * this function on each node racing to become the recovery + * master will not stop attempting this until either: + * a) this node gets the EX (and becomes the recovery master), + * or b) dlm->reco.new_master gets set to some nodenum + * != O2NM_INVALID_NODE_NUM (another node will do the reco). + * so each time a recovery master is needed, the entire cluster + * will sync at this point. if the new master dies, that will + * be detected in dlm_do_recovery */ +static int dlm_pick_recovery_master(struct dlm_ctxt *dlm) +{ + enum dlm_status ret; + struct dlm_lockstatus lksb; + int status = -EINVAL; + + mlog(0, "starting recovery of %s at %lu, dead=%u, this=%u\n", + dlm->name, jiffies, dlm->reco.dead_node, dlm->node_num); +again: + memset(&lksb, 0, sizeof(lksb)); + + ret = dlmlock(dlm, LKM_EXMODE, &lksb, LKM_NOQUEUE|LKM_RECOVERY, + DLM_RECOVERY_LOCK_NAME, DLM_RECOVERY_LOCK_NAME_LEN, + dlm_reco_ast, dlm, dlm_reco_bast); + + mlog(0, "%s: dlmlock($RECOVERY) returned %d, lksb=%d\n", + dlm->name, ret, lksb.status); + + if (ret == DLM_NORMAL) { + mlog(0, "dlm=%s dlmlock says I got it (this=%u)\n", + dlm->name, dlm->node_num); + + /* got the EX lock. check to see if another node + * just became the reco master */ + if (dlm_reco_master_ready(dlm)) { + mlog(0, "%s: got reco EX lock, but %u will " + "do the recovery\n", dlm->name, + dlm->reco.new_master); + status = -EEXIST; + } else { + status = 0; + + /* see if recovery was already finished elsewhere */ + spin_lock(&dlm->spinlock); + if (dlm->reco.dead_node == O2NM_INVALID_NODE_NUM) { + status = -EINVAL; + mlog(0, "%s: got reco EX lock, but " + "node got recovered already\n", dlm->name); + if (dlm->reco.new_master != O2NM_INVALID_NODE_NUM) { + mlog(ML_ERROR, "%s: new master is %u " + "but no dead node!\n", + dlm->name, dlm->reco.new_master); + BUG(); + } + } + spin_unlock(&dlm->spinlock); + } + + /* if this node has actually become the recovery master, + * set the master and send the messages to begin recovery */ + if (!status) { + mlog(0, "%s: dead=%u, this=%u, sending " + "begin_reco now\n", dlm->name, + dlm->reco.dead_node, dlm->node_num); + status = dlm_send_begin_reco_message(dlm, + dlm->reco.dead_node); + /* this always succeeds */ + BUG_ON(status); + + /* set the new_master to this node */ + spin_lock(&dlm->spinlock); + dlm_set_reco_master(dlm, dlm->node_num); + spin_unlock(&dlm->spinlock); + } + + /* recovery lock is a special case. ast will not get fired, + * so just go ahead and unlock it. */ + ret = dlmunlock(dlm, &lksb, 0, dlm_reco_unlock_ast, dlm); + if (ret == DLM_DENIED) { + mlog(0, "got DLM_DENIED, trying LKM_CANCEL\n"); + ret = dlmunlock(dlm, &lksb, LKM_CANCEL, dlm_reco_unlock_ast, dlm); + } + if (ret != DLM_NORMAL) { + /* this would really suck. this could only happen + * if there was a network error during the unlock + * because of node death. this means the unlock + * is actually "done" and the lock structure is + * even freed. we can continue, but only + * because this specific lock name is special. */ + mlog(ML_ERROR, "dlmunlock returned %d\n", ret); + } + } else if (ret == DLM_NOTQUEUED) { + mlog(0, "dlm=%s dlmlock says another node got it (this=%u)\n", + dlm->name, dlm->node_num); + /* another node is master. wait on + * reco.new_master != O2NM_INVALID_NODE_NUM + * for at most one second */ + wait_event_timeout(dlm->dlm_reco_thread_wq, + dlm_reco_master_ready(dlm), + msecs_to_jiffies(1000)); + if (!dlm_reco_master_ready(dlm)) { + mlog(0, "%s: reco master taking a while\n", + dlm->name); + goto again; + } + /* another node has informed this one that it is reco master */ + mlog(0, "%s: reco master %u is ready to recover %u\n", + dlm->name, dlm->reco.new_master, dlm->reco.dead_node); + status = -EEXIST; + } else if (ret == DLM_RECOVERING) { + mlog(0, "dlm=%s dlmlock says master node died (this=%u)\n", + dlm->name, dlm->node_num); + goto again; + } else { + struct dlm_lock_resource *res; + + /* dlmlock returned something other than NOTQUEUED or NORMAL */ + mlog(ML_ERROR, "%s: got %s from dlmlock($RECOVERY), " + "lksb.status=%s\n", dlm->name, dlm_errname(ret), + dlm_errname(lksb.status)); + res = dlm_lookup_lockres(dlm, DLM_RECOVERY_LOCK_NAME, + DLM_RECOVERY_LOCK_NAME_LEN); + if (res) { + dlm_print_one_lock_resource(res); + dlm_lockres_put(res); + } else { + mlog(ML_ERROR, "recovery lock not found\n"); + } + BUG(); + } + + return status; +} + +static int dlm_send_begin_reco_message(struct dlm_ctxt *dlm, u8 dead_node) +{ + struct dlm_begin_reco br; + int ret = 0; + struct dlm_node_iter iter; + int nodenum; + int status; + + mlog(0, "%s: dead node is %u\n", dlm->name, dead_node); + + spin_lock(&dlm->spinlock); + dlm_node_iter_init(dlm->domain_map, &iter); + spin_unlock(&dlm->spinlock); + + clear_bit(dead_node, iter.node_map); + + memset(&br, 0, sizeof(br)); + br.node_idx = dlm->node_num; + br.dead_node = dead_node; + + while ((nodenum = dlm_node_iter_next(&iter)) >= 0) { + ret = 0; + if (nodenum == dead_node) { + mlog(0, "not sending begin reco to dead node " + "%u\n", dead_node); + continue; + } + if (nodenum == dlm->node_num) { + mlog(0, "not sending begin reco to self\n"); + continue; + } +retry: + mlog(0, "attempting to send begin reco msg to %d\n", + nodenum); + ret = o2net_send_message(DLM_BEGIN_RECO_MSG, dlm->key, + &br, sizeof(br), nodenum, &status); + /* negative status is handled ok by caller here */ + if (ret >= 0) + ret = status; + if (dlm_is_host_down(ret)) { + /* node is down. not involved in recovery + * so just keep going */ + mlog(ML_NOTICE, "%s: node %u was down when sending " + "begin reco msg (%d)\n", dlm->name, nodenum, ret); + ret = 0; + } + + /* + * Prior to commit aad1b15310b9bcd59fa81ab8f2b1513b59553ea8, + * dlm_begin_reco_handler() returned EAGAIN and not -EAGAIN. + * We are handling both for compatibility reasons. + */ + if (ret == -EAGAIN || ret == EAGAIN) { + mlog(0, "%s: trying to start recovery of node " + "%u, but node %u is waiting for last recovery " + "to complete, backoff for a bit\n", dlm->name, + dead_node, nodenum); + msleep(100); + goto retry; + } + if (ret < 0) { + struct dlm_lock_resource *res; + + /* this is now a serious problem, possibly ENOMEM + * in the network stack. must retry */ + mlog_errno(ret); + mlog(ML_ERROR, "begin reco of dlm %s to node %u " + "returned %d\n", dlm->name, nodenum, ret); + res = dlm_lookup_lockres(dlm, DLM_RECOVERY_LOCK_NAME, + DLM_RECOVERY_LOCK_NAME_LEN); + if (res) { + dlm_print_one_lock_resource(res); + dlm_lockres_put(res); + } else { + mlog(ML_ERROR, "recovery lock not found\n"); + } + /* sleep for a bit in hopes that we can avoid + * another ENOMEM */ + msleep(100); + goto retry; + } + } + + return ret; +} + +int dlm_begin_reco_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data) +{ + struct dlm_ctxt *dlm = data; + struct dlm_begin_reco *br = (struct dlm_begin_reco *)msg->buf; + + /* ok to return 0, domain has gone away */ + if (!dlm_grab(dlm)) + return 0; + + spin_lock(&dlm->spinlock); + if (dlm->reco.state & DLM_RECO_STATE_FINALIZE) { + mlog(0, "%s: node %u wants to recover node %u (%u:%u) " + "but this node is in finalize state, waiting on finalize2\n", + dlm->name, br->node_idx, br->dead_node, + dlm->reco.dead_node, dlm->reco.new_master); + spin_unlock(&dlm->spinlock); + dlm_put(dlm); + return -EAGAIN; + } + spin_unlock(&dlm->spinlock); + + mlog(0, "%s: node %u wants to recover node %u (%u:%u)\n", + dlm->name, br->node_idx, br->dead_node, + dlm->reco.dead_node, dlm->reco.new_master); + + dlm_fire_domain_eviction_callbacks(dlm, br->dead_node); + + spin_lock(&dlm->spinlock); + if (dlm->reco.new_master != O2NM_INVALID_NODE_NUM) { + if (test_bit(dlm->reco.new_master, dlm->recovery_map)) { + mlog(0, "%s: new_master %u died, changing " + "to %u\n", dlm->name, dlm->reco.new_master, + br->node_idx); + } else { + mlog(0, "%s: new_master %u NOT DEAD, changing " + "to %u\n", dlm->name, dlm->reco.new_master, + br->node_idx); + /* may not have seen the new master as dead yet */ + } + } + if (dlm->reco.dead_node != O2NM_INVALID_NODE_NUM) { + mlog(ML_NOTICE, "%s: dead_node previously set to %u, " + "node %u changing it to %u\n", dlm->name, + dlm->reco.dead_node, br->node_idx, br->dead_node); + } + dlm_set_reco_master(dlm, br->node_idx); + dlm_set_reco_dead_node(dlm, br->dead_node); + if (!test_bit(br->dead_node, dlm->recovery_map)) { + mlog(0, "recovery master %u sees %u as dead, but this " + "node has not yet. marking %u as dead\n", + br->node_idx, br->dead_node, br->dead_node); + if (!test_bit(br->dead_node, dlm->domain_map) || + !test_bit(br->dead_node, dlm->live_nodes_map)) + mlog(0, "%u not in domain/live_nodes map " + "so setting it in reco map manually\n", + br->dead_node); + /* force the recovery cleanup in __dlm_hb_node_down + * both of these will be cleared in a moment */ + set_bit(br->dead_node, dlm->domain_map); + set_bit(br->dead_node, dlm->live_nodes_map); + __dlm_hb_node_down(dlm, br->dead_node); + } + spin_unlock(&dlm->spinlock); + + dlm_kick_recovery_thread(dlm); + + mlog(0, "%s: recovery started by node %u, for %u (%u:%u)\n", + dlm->name, br->node_idx, br->dead_node, + dlm->reco.dead_node, dlm->reco.new_master); + + dlm_put(dlm); + return 0; +} + +#define DLM_FINALIZE_STAGE2 0x01 +static int dlm_send_finalize_reco_message(struct dlm_ctxt *dlm) +{ + int ret = 0; + struct dlm_finalize_reco fr; + struct dlm_node_iter iter; + int nodenum; + int status; + int stage = 1; + + mlog(0, "finishing recovery for node %s:%u, " + "stage %d\n", dlm->name, dlm->reco.dead_node, stage); + + spin_lock(&dlm->spinlock); + dlm_node_iter_init(dlm->domain_map, &iter); + spin_unlock(&dlm->spinlock); + +stage2: + memset(&fr, 0, sizeof(fr)); + fr.node_idx = dlm->node_num; + fr.dead_node = dlm->reco.dead_node; + if (stage == 2) + fr.flags |= DLM_FINALIZE_STAGE2; + + while ((nodenum = dlm_node_iter_next(&iter)) >= 0) { + if (nodenum == dlm->node_num) + continue; + ret = o2net_send_message(DLM_FINALIZE_RECO_MSG, dlm->key, + &fr, sizeof(fr), nodenum, &status); + if (ret >= 0) + ret = status; + if (ret < 0) { + mlog(ML_ERROR, "Error %d when sending message %u (key " + "0x%x) to node %u\n", ret, DLM_FINALIZE_RECO_MSG, + dlm->key, nodenum); + if (dlm_is_host_down(ret)) { + /* this has no effect on this recovery + * session, so set the status to zero to + * finish out the last recovery */ + mlog(ML_ERROR, "node %u went down after this " + "node finished recovery.\n", nodenum); + ret = 0; + continue; + } + break; + } + } + if (stage == 1) { + /* reset the node_iter back to the top and send finalize2 */ + iter.curnode = -1; + stage = 2; + goto stage2; + } + + return ret; +} + +int dlm_finalize_reco_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data) +{ + struct dlm_ctxt *dlm = data; + struct dlm_finalize_reco *fr = (struct dlm_finalize_reco *)msg->buf; + int stage = 1; + + /* ok to return 0, domain has gone away */ + if (!dlm_grab(dlm)) + return 0; + + if (fr->flags & DLM_FINALIZE_STAGE2) + stage = 2; + + mlog(0, "%s: node %u finalizing recovery stage%d of " + "node %u (%u:%u)\n", dlm->name, fr->node_idx, stage, + fr->dead_node, dlm->reco.dead_node, dlm->reco.new_master); + + spin_lock(&dlm->spinlock); + + if (dlm->reco.new_master != fr->node_idx) { + mlog(ML_ERROR, "node %u sent recovery finalize msg, but node " + "%u is supposed to be the new master, dead=%u\n", + fr->node_idx, dlm->reco.new_master, fr->dead_node); + BUG(); + } + if (dlm->reco.dead_node != fr->dead_node) { + mlog(ML_ERROR, "node %u sent recovery finalize msg for dead " + "node %u, but node %u is supposed to be dead\n", + fr->node_idx, fr->dead_node, dlm->reco.dead_node); + BUG(); + } + + switch (stage) { + case 1: + dlm_finish_local_lockres_recovery(dlm, fr->dead_node, fr->node_idx); + if (dlm->reco.state & DLM_RECO_STATE_FINALIZE) { + mlog(ML_ERROR, "%s: received finalize1 from " + "new master %u for dead node %u, but " + "this node has already received it!\n", + dlm->name, fr->node_idx, fr->dead_node); + dlm_print_reco_node_status(dlm); + BUG(); + } + dlm->reco.state |= DLM_RECO_STATE_FINALIZE; + spin_unlock(&dlm->spinlock); + break; + case 2: + if (!(dlm->reco.state & DLM_RECO_STATE_FINALIZE)) { + mlog(ML_ERROR, "%s: received finalize2 from " + "new master %u for dead node %u, but " + "this node did not have finalize1!\n", + dlm->name, fr->node_idx, fr->dead_node); + dlm_print_reco_node_status(dlm); + BUG(); + } + dlm->reco.state &= ~DLM_RECO_STATE_FINALIZE; + __dlm_reset_recovery(dlm); + spin_unlock(&dlm->spinlock); + dlm_kick_recovery_thread(dlm); + break; + } + + mlog(0, "%s: recovery done, reco master was %u, dead now %u, master now %u\n", + dlm->name, fr->node_idx, dlm->reco.dead_node, dlm->reco.new_master); + + dlm_put(dlm); + return 0; +} diff --git a/fs/ocfs2/dlm/dlmthread.c b/fs/ocfs2/dlm/dlmthread.c new file mode 100644 index 000000000..eedf07ca2 --- /dev/null +++ b/fs/ocfs2/dlm/dlmthread.c @@ -0,0 +1,809 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * dlmthread.c + * + * standalone DLM module + * + * Copyright (C) 2004 Oracle. All rights reserved. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#include "../cluster/heartbeat.h" +#include "../cluster/nodemanager.h" +#include "../cluster/tcp.h" + +#include "dlmapi.h" +#include "dlmcommon.h" +#include "dlmdomain.h" + +#define MLOG_MASK_PREFIX (ML_DLM|ML_DLM_THREAD) +#include "../cluster/masklog.h" + +static int dlm_thread(void *data); +static void dlm_flush_asts(struct dlm_ctxt *dlm); + +/* will exit holding res->spinlock, but may drop in function */ +/* waits until flags are cleared on res->state */ +void __dlm_wait_on_lockres_flags(struct dlm_lock_resource *res, int flags) +{ + DECLARE_WAITQUEUE(wait, current); + + assert_spin_locked(&res->spinlock); + + add_wait_queue(&res->wq, &wait); +repeat: + set_current_state(TASK_UNINTERRUPTIBLE); + if (res->state & flags) { + spin_unlock(&res->spinlock); + schedule(); + spin_lock(&res->spinlock); + goto repeat; + } + remove_wait_queue(&res->wq, &wait); + __set_current_state(TASK_RUNNING); +} + +int __dlm_lockres_has_locks(struct dlm_lock_resource *res) +{ + if (list_empty(&res->granted) && + list_empty(&res->converting) && + list_empty(&res->blocked)) + return 0; + return 1; +} + +/* "unused": the lockres has no locks, is not on the dirty list, + * has no inflight locks (in the gap between mastery and acquiring + * the first lock), and has no bits in its refmap. + * truly ready to be freed. */ +int __dlm_lockres_unused(struct dlm_lock_resource *res) +{ + int bit; + + assert_spin_locked(&res->spinlock); + + if (__dlm_lockres_has_locks(res)) + return 0; + + /* Locks are in the process of being created */ + if (res->inflight_locks) + return 0; + + if (!list_empty(&res->dirty) || res->state & DLM_LOCK_RES_DIRTY) + return 0; + + if (res->state & (DLM_LOCK_RES_RECOVERING| + DLM_LOCK_RES_RECOVERY_WAITING)) + return 0; + + /* Another node has this resource with this node as the master */ + bit = find_first_bit(res->refmap, O2NM_MAX_NODES); + if (bit < O2NM_MAX_NODES) + return 0; + + return 1; +} + + +/* Call whenever you may have added or deleted something from one of + * the lockres queue's. This will figure out whether it belongs on the + * unused list or not and does the appropriate thing. */ +void __dlm_lockres_calc_usage(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res) +{ + assert_spin_locked(&dlm->spinlock); + assert_spin_locked(&res->spinlock); + + if (__dlm_lockres_unused(res)){ + if (list_empty(&res->purge)) { + mlog(0, "%s: Adding res %.*s to purge list\n", + dlm->name, res->lockname.len, res->lockname.name); + + res->last_used = jiffies; + dlm_lockres_get(res); + list_add_tail(&res->purge, &dlm->purge_list); + dlm->purge_count++; + } + } else if (!list_empty(&res->purge)) { + mlog(0, "%s: Removing res %.*s from purge list\n", + dlm->name, res->lockname.len, res->lockname.name); + + list_del_init(&res->purge); + dlm_lockres_put(res); + dlm->purge_count--; + } +} + +void dlm_lockres_calc_usage(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res) +{ + spin_lock(&dlm->spinlock); + spin_lock(&res->spinlock); + + __dlm_lockres_calc_usage(dlm, res); + + spin_unlock(&res->spinlock); + spin_unlock(&dlm->spinlock); +} + +/* + * Do the real purge work: + * unhash the lockres, and + * clear flag DLM_LOCK_RES_DROPPING_REF. + * It requires dlm and lockres spinlock to be taken. + */ +void __dlm_do_purge_lockres(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res) +{ + assert_spin_locked(&dlm->spinlock); + assert_spin_locked(&res->spinlock); + + if (!list_empty(&res->purge)) { + mlog(0, "%s: Removing res %.*s from purgelist\n", + dlm->name, res->lockname.len, res->lockname.name); + list_del_init(&res->purge); + dlm_lockres_put(res); + dlm->purge_count--; + } + + if (!__dlm_lockres_unused(res)) { + mlog(ML_ERROR, "%s: res %.*s in use after deref\n", + dlm->name, res->lockname.len, res->lockname.name); + __dlm_print_one_lock_resource(res); + BUG(); + } + + __dlm_unhash_lockres(dlm, res); + + spin_lock(&dlm->track_lock); + if (!list_empty(&res->tracking)) + list_del_init(&res->tracking); + else { + mlog(ML_ERROR, "%s: Resource %.*s not on the Tracking list\n", + dlm->name, res->lockname.len, res->lockname.name); + __dlm_print_one_lock_resource(res); + } + spin_unlock(&dlm->track_lock); + + /* + * lockres is not in the hash now. drop the flag and wake up + * any processes waiting in dlm_get_lock_resource. + */ + res->state &= ~DLM_LOCK_RES_DROPPING_REF; +} + +static void dlm_purge_lockres(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res) +{ + int master; + int ret = 0; + + assert_spin_locked(&dlm->spinlock); + assert_spin_locked(&res->spinlock); + + master = (res->owner == dlm->node_num); + + mlog(0, "%s: Purging res %.*s, master %d\n", dlm->name, + res->lockname.len, res->lockname.name, master); + + if (!master) { + if (res->state & DLM_LOCK_RES_DROPPING_REF) { + mlog(ML_NOTICE, "%s: res %.*s already in DLM_LOCK_RES_DROPPING_REF state\n", + dlm->name, res->lockname.len, res->lockname.name); + spin_unlock(&res->spinlock); + return; + } + + res->state |= DLM_LOCK_RES_DROPPING_REF; + /* drop spinlock... retake below */ + spin_unlock(&res->spinlock); + spin_unlock(&dlm->spinlock); + + spin_lock(&res->spinlock); + /* This ensures that clear refmap is sent after the set */ + __dlm_wait_on_lockres_flags(res, DLM_LOCK_RES_SETREF_INPROG); + spin_unlock(&res->spinlock); + + /* clear our bit from the master's refmap, ignore errors */ + ret = dlm_drop_lockres_ref(dlm, res); + if (ret < 0) { + if (!dlm_is_host_down(ret)) + BUG(); + } + spin_lock(&dlm->spinlock); + spin_lock(&res->spinlock); + } + + if (!list_empty(&res->purge)) { + mlog(0, "%s: Removing res %.*s from purgelist, master %d\n", + dlm->name, res->lockname.len, res->lockname.name, master); + list_del_init(&res->purge); + dlm_lockres_put(res); + dlm->purge_count--; + } + + if (!master && ret == DLM_DEREF_RESPONSE_INPROG) { + mlog(0, "%s: deref %.*s in progress\n", + dlm->name, res->lockname.len, res->lockname.name); + spin_unlock(&res->spinlock); + return; + } + + if (!__dlm_lockres_unused(res)) { + mlog(ML_ERROR, "%s: res %.*s in use after deref\n", + dlm->name, res->lockname.len, res->lockname.name); + __dlm_print_one_lock_resource(res); + BUG(); + } + + __dlm_unhash_lockres(dlm, res); + + spin_lock(&dlm->track_lock); + if (!list_empty(&res->tracking)) + list_del_init(&res->tracking); + else { + mlog(ML_ERROR, "Resource %.*s not on the Tracking list\n", + res->lockname.len, res->lockname.name); + __dlm_print_one_lock_resource(res); + } + spin_unlock(&dlm->track_lock); + + /* lockres is not in the hash now. drop the flag and wake up + * any processes waiting in dlm_get_lock_resource. */ + if (!master) { + res->state &= ~DLM_LOCK_RES_DROPPING_REF; + spin_unlock(&res->spinlock); + wake_up(&res->wq); + } else + spin_unlock(&res->spinlock); +} + +static void dlm_run_purge_list(struct dlm_ctxt *dlm, + int purge_now) +{ + unsigned int run_max, unused; + unsigned long purge_jiffies; + struct dlm_lock_resource *lockres; + + spin_lock(&dlm->spinlock); + run_max = dlm->purge_count; + + while(run_max && !list_empty(&dlm->purge_list)) { + run_max--; + + lockres = list_entry(dlm->purge_list.next, + struct dlm_lock_resource, purge); + + spin_lock(&lockres->spinlock); + + purge_jiffies = lockres->last_used + + msecs_to_jiffies(DLM_PURGE_INTERVAL_MS); + + /* Make sure that we want to be processing this guy at + * this time. */ + if (!purge_now && time_after(purge_jiffies, jiffies)) { + /* Since resources are added to the purge list + * in tail order, we can stop at the first + * unpurgable resource -- anyone added after + * him will have a greater last_used value */ + spin_unlock(&lockres->spinlock); + break; + } + + /* Status of the lockres *might* change so double + * check. If the lockres is unused, holding the dlm + * spinlock will prevent people from getting and more + * refs on it. */ + unused = __dlm_lockres_unused(lockres); + if (!unused || + (lockres->state & DLM_LOCK_RES_MIGRATING) || + (lockres->inflight_assert_workers != 0)) { + mlog(0, "%s: res %.*s is in use or being remastered, " + "used %d, state %d, assert master workers %u\n", + dlm->name, lockres->lockname.len, + lockres->lockname.name, + !unused, lockres->state, + lockres->inflight_assert_workers); + list_move_tail(&lockres->purge, &dlm->purge_list); + spin_unlock(&lockres->spinlock); + continue; + } + + dlm_lockres_get(lockres); + + dlm_purge_lockres(dlm, lockres); + + dlm_lockres_put(lockres); + + /* Avoid adding any scheduling latencies */ + cond_resched_lock(&dlm->spinlock); + } + + spin_unlock(&dlm->spinlock); +} + +static void dlm_shuffle_lists(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res) +{ + struct dlm_lock *lock, *target; + int can_grant = 1; + + /* + * Because this function is called with the lockres + * spinlock, and because we know that it is not migrating/ + * recovering/in-progress, it is fine to reserve asts and + * basts right before queueing them all throughout + */ + assert_spin_locked(&dlm->ast_lock); + assert_spin_locked(&res->spinlock); + BUG_ON((res->state & (DLM_LOCK_RES_MIGRATING| + DLM_LOCK_RES_RECOVERING| + DLM_LOCK_RES_IN_PROGRESS))); + +converting: + if (list_empty(&res->converting)) + goto blocked; + mlog(0, "%s: res %.*s has locks on the convert queue\n", dlm->name, + res->lockname.len, res->lockname.name); + + target = list_entry(res->converting.next, struct dlm_lock, list); + if (target->ml.convert_type == LKM_IVMODE) { + mlog(ML_ERROR, "%s: res %.*s converting lock to invalid mode\n", + dlm->name, res->lockname.len, res->lockname.name); + BUG(); + } + list_for_each_entry(lock, &res->granted, list) { + if (lock==target) + continue; + if (!dlm_lock_compatible(lock->ml.type, + target->ml.convert_type)) { + can_grant = 0; + /* queue the BAST if not already */ + if (lock->ml.highest_blocked == LKM_IVMODE) { + __dlm_lockres_reserve_ast(res); + __dlm_queue_bast(dlm, lock); + } + /* update the highest_blocked if needed */ + if (lock->ml.highest_blocked < target->ml.convert_type) + lock->ml.highest_blocked = + target->ml.convert_type; + } + } + + list_for_each_entry(lock, &res->converting, list) { + if (lock==target) + continue; + if (!dlm_lock_compatible(lock->ml.type, + target->ml.convert_type)) { + can_grant = 0; + if (lock->ml.highest_blocked == LKM_IVMODE) { + __dlm_lockres_reserve_ast(res); + __dlm_queue_bast(dlm, lock); + } + if (lock->ml.highest_blocked < target->ml.convert_type) + lock->ml.highest_blocked = + target->ml.convert_type; + } + } + + /* we can convert the lock */ + if (can_grant) { + spin_lock(&target->spinlock); + BUG_ON(target->ml.highest_blocked != LKM_IVMODE); + + mlog(0, "%s: res %.*s, AST for Converting lock %u:%llu, type " + "%d => %d, node %u\n", dlm->name, res->lockname.len, + res->lockname.name, + dlm_get_lock_cookie_node(be64_to_cpu(target->ml.cookie)), + dlm_get_lock_cookie_seq(be64_to_cpu(target->ml.cookie)), + target->ml.type, + target->ml.convert_type, target->ml.node); + + target->ml.type = target->ml.convert_type; + target->ml.convert_type = LKM_IVMODE; + list_move_tail(&target->list, &res->granted); + + BUG_ON(!target->lksb); + target->lksb->status = DLM_NORMAL; + + spin_unlock(&target->spinlock); + + __dlm_lockres_reserve_ast(res); + __dlm_queue_ast(dlm, target); + /* go back and check for more */ + goto converting; + } + +blocked: + if (list_empty(&res->blocked)) + goto leave; + target = list_entry(res->blocked.next, struct dlm_lock, list); + + list_for_each_entry(lock, &res->granted, list) { + if (lock==target) + continue; + if (!dlm_lock_compatible(lock->ml.type, target->ml.type)) { + can_grant = 0; + if (lock->ml.highest_blocked == LKM_IVMODE) { + __dlm_lockres_reserve_ast(res); + __dlm_queue_bast(dlm, lock); + } + if (lock->ml.highest_blocked < target->ml.type) + lock->ml.highest_blocked = target->ml.type; + } + } + + list_for_each_entry(lock, &res->converting, list) { + if (lock==target) + continue; + if (!dlm_lock_compatible(lock->ml.type, target->ml.type)) { + can_grant = 0; + if (lock->ml.highest_blocked == LKM_IVMODE) { + __dlm_lockres_reserve_ast(res); + __dlm_queue_bast(dlm, lock); + } + if (lock->ml.highest_blocked < target->ml.type) + lock->ml.highest_blocked = target->ml.type; + } + } + + /* we can grant the blocked lock (only + * possible if converting list empty) */ + if (can_grant) { + spin_lock(&target->spinlock); + BUG_ON(target->ml.highest_blocked != LKM_IVMODE); + + mlog(0, "%s: res %.*s, AST for Blocked lock %u:%llu, type %d, " + "node %u\n", dlm->name, res->lockname.len, + res->lockname.name, + dlm_get_lock_cookie_node(be64_to_cpu(target->ml.cookie)), + dlm_get_lock_cookie_seq(be64_to_cpu(target->ml.cookie)), + target->ml.type, target->ml.node); + + /* target->ml.type is already correct */ + list_move_tail(&target->list, &res->granted); + + BUG_ON(!target->lksb); + target->lksb->status = DLM_NORMAL; + + spin_unlock(&target->spinlock); + + __dlm_lockres_reserve_ast(res); + __dlm_queue_ast(dlm, target); + /* go back and check for more */ + goto converting; + } + +leave: + return; +} + +/* must have NO locks when calling this with res !=NULL * */ +void dlm_kick_thread(struct dlm_ctxt *dlm, struct dlm_lock_resource *res) +{ + if (res) { + spin_lock(&dlm->spinlock); + spin_lock(&res->spinlock); + __dlm_dirty_lockres(dlm, res); + spin_unlock(&res->spinlock); + spin_unlock(&dlm->spinlock); + } + wake_up(&dlm->dlm_thread_wq); +} + +void __dlm_dirty_lockres(struct dlm_ctxt *dlm, struct dlm_lock_resource *res) +{ + assert_spin_locked(&dlm->spinlock); + assert_spin_locked(&res->spinlock); + + /* don't shuffle secondary queues */ + if (res->owner == dlm->node_num) { + if (res->state & (DLM_LOCK_RES_MIGRATING | + DLM_LOCK_RES_BLOCK_DIRTY)) + return; + + if (list_empty(&res->dirty)) { + /* ref for dirty_list */ + dlm_lockres_get(res); + list_add_tail(&res->dirty, &dlm->dirty_list); + res->state |= DLM_LOCK_RES_DIRTY; + } + } + + mlog(0, "%s: res %.*s\n", dlm->name, res->lockname.len, + res->lockname.name); +} + + +/* Launch the NM thread for the mounted volume */ +int dlm_launch_thread(struct dlm_ctxt *dlm) +{ + mlog(0, "Starting dlm_thread...\n"); + + dlm->dlm_thread_task = kthread_run(dlm_thread, dlm, "dlm-%s", + dlm->name); + if (IS_ERR(dlm->dlm_thread_task)) { + mlog_errno(PTR_ERR(dlm->dlm_thread_task)); + dlm->dlm_thread_task = NULL; + return -EINVAL; + } + + return 0; +} + +void dlm_complete_thread(struct dlm_ctxt *dlm) +{ + if (dlm->dlm_thread_task) { + mlog(ML_KTHREAD, "Waiting for dlm thread to exit\n"); + kthread_stop(dlm->dlm_thread_task); + dlm->dlm_thread_task = NULL; + } +} + +static int dlm_dirty_list_empty(struct dlm_ctxt *dlm) +{ + int empty; + + spin_lock(&dlm->spinlock); + empty = list_empty(&dlm->dirty_list); + spin_unlock(&dlm->spinlock); + + return empty; +} + +static void dlm_flush_asts(struct dlm_ctxt *dlm) +{ + int ret; + struct dlm_lock *lock; + struct dlm_lock_resource *res; + u8 hi; + + spin_lock(&dlm->ast_lock); + while (!list_empty(&dlm->pending_asts)) { + lock = list_entry(dlm->pending_asts.next, + struct dlm_lock, ast_list); + /* get an extra ref on lock */ + dlm_lock_get(lock); + res = lock->lockres; + mlog(0, "%s: res %.*s, Flush AST for lock %u:%llu, type %d, " + "node %u\n", dlm->name, res->lockname.len, + res->lockname.name, + dlm_get_lock_cookie_node(be64_to_cpu(lock->ml.cookie)), + dlm_get_lock_cookie_seq(be64_to_cpu(lock->ml.cookie)), + lock->ml.type, lock->ml.node); + + BUG_ON(!lock->ast_pending); + + /* remove from list (including ref) */ + list_del_init(&lock->ast_list); + dlm_lock_put(lock); + spin_unlock(&dlm->ast_lock); + + if (lock->ml.node != dlm->node_num) { + ret = dlm_do_remote_ast(dlm, res, lock); + if (ret < 0) + mlog_errno(ret); + } else + dlm_do_local_ast(dlm, res, lock); + + spin_lock(&dlm->ast_lock); + + /* possible that another ast was queued while + * we were delivering the last one */ + if (!list_empty(&lock->ast_list)) { + mlog(0, "%s: res %.*s, AST queued while flushing last " + "one\n", dlm->name, res->lockname.len, + res->lockname.name); + } else + lock->ast_pending = 0; + + /* drop the extra ref. + * this may drop it completely. */ + dlm_lock_put(lock); + dlm_lockres_release_ast(dlm, res); + } + + while (!list_empty(&dlm->pending_basts)) { + lock = list_entry(dlm->pending_basts.next, + struct dlm_lock, bast_list); + /* get an extra ref on lock */ + dlm_lock_get(lock); + res = lock->lockres; + + BUG_ON(!lock->bast_pending); + + /* get the highest blocked lock, and reset */ + spin_lock(&lock->spinlock); + BUG_ON(lock->ml.highest_blocked <= LKM_IVMODE); + hi = lock->ml.highest_blocked; + lock->ml.highest_blocked = LKM_IVMODE; + spin_unlock(&lock->spinlock); + + /* remove from list (including ref) */ + list_del_init(&lock->bast_list); + dlm_lock_put(lock); + spin_unlock(&dlm->ast_lock); + + mlog(0, "%s: res %.*s, Flush BAST for lock %u:%llu, " + "blocked %d, node %u\n", + dlm->name, res->lockname.len, res->lockname.name, + dlm_get_lock_cookie_node(be64_to_cpu(lock->ml.cookie)), + dlm_get_lock_cookie_seq(be64_to_cpu(lock->ml.cookie)), + hi, lock->ml.node); + + if (lock->ml.node != dlm->node_num) { + ret = dlm_send_proxy_bast(dlm, res, lock, hi); + if (ret < 0) + mlog_errno(ret); + } else + dlm_do_local_bast(dlm, res, lock, hi); + + spin_lock(&dlm->ast_lock); + + /* possible that another bast was queued while + * we were delivering the last one */ + if (!list_empty(&lock->bast_list)) { + mlog(0, "%s: res %.*s, BAST queued while flushing last " + "one\n", dlm->name, res->lockname.len, + res->lockname.name); + } else + lock->bast_pending = 0; + + /* drop the extra ref. + * this may drop it completely. */ + dlm_lock_put(lock); + dlm_lockres_release_ast(dlm, res); + } + wake_up(&dlm->ast_wq); + spin_unlock(&dlm->ast_lock); +} + + +#define DLM_THREAD_TIMEOUT_MS (4 * 1000) +#define DLM_THREAD_MAX_DIRTY 100 + +static int dlm_thread(void *data) +{ + struct dlm_lock_resource *res; + struct dlm_ctxt *dlm = data; + unsigned long timeout = msecs_to_jiffies(DLM_THREAD_TIMEOUT_MS); + + mlog(0, "dlm thread running for %s...\n", dlm->name); + + while (!kthread_should_stop()) { + int n = DLM_THREAD_MAX_DIRTY; + + /* dlm_shutting_down is very point-in-time, but that + * doesn't matter as we'll just loop back around if we + * get false on the leading edge of a state + * transition. */ + dlm_run_purge_list(dlm, dlm_shutting_down(dlm)); + + /* We really don't want to hold dlm->spinlock while + * calling dlm_shuffle_lists on each lockres that + * needs to have its queues adjusted and AST/BASTs + * run. So let's pull each entry off the dirty_list + * and drop dlm->spinlock ASAP. Once off the list, + * res->spinlock needs to be taken again to protect + * the queues while calling dlm_shuffle_lists. */ + spin_lock(&dlm->spinlock); + while (!list_empty(&dlm->dirty_list)) { + int delay = 0; + res = list_entry(dlm->dirty_list.next, + struct dlm_lock_resource, dirty); + + /* peel a lockres off, remove it from the list, + * unset the dirty flag and drop the dlm lock */ + BUG_ON(!res); + dlm_lockres_get(res); + + spin_lock(&res->spinlock); + /* We clear the DLM_LOCK_RES_DIRTY state once we shuffle lists below */ + list_del_init(&res->dirty); + spin_unlock(&res->spinlock); + spin_unlock(&dlm->spinlock); + /* Drop dirty_list ref */ + dlm_lockres_put(res); + + /* lockres can be re-dirtied/re-added to the + * dirty_list in this gap, but that is ok */ + + spin_lock(&dlm->ast_lock); + spin_lock(&res->spinlock); + if (res->owner != dlm->node_num) { + __dlm_print_one_lock_resource(res); + mlog(ML_ERROR, "%s: inprog %d, mig %d, reco %d," + " dirty %d\n", dlm->name, + !!(res->state & DLM_LOCK_RES_IN_PROGRESS), + !!(res->state & DLM_LOCK_RES_MIGRATING), + !!(res->state & DLM_LOCK_RES_RECOVERING), + !!(res->state & DLM_LOCK_RES_DIRTY)); + } + BUG_ON(res->owner != dlm->node_num); + + /* it is now ok to move lockreses in these states + * to the dirty list, assuming that they will only be + * dirty for a short while. */ + BUG_ON(res->state & DLM_LOCK_RES_MIGRATING); + if (res->state & (DLM_LOCK_RES_IN_PROGRESS | + DLM_LOCK_RES_RECOVERING | + DLM_LOCK_RES_RECOVERY_WAITING)) { + /* move it to the tail and keep going */ + res->state &= ~DLM_LOCK_RES_DIRTY; + spin_unlock(&res->spinlock); + spin_unlock(&dlm->ast_lock); + mlog(0, "%s: res %.*s, inprogress, delay list " + "shuffle, state %d\n", dlm->name, + res->lockname.len, res->lockname.name, + res->state); + delay = 1; + goto in_progress; + } + + /* at this point the lockres is not migrating/ + * recovering/in-progress. we have the lockres + * spinlock and do NOT have the dlm lock. + * safe to reserve/queue asts and run the lists. */ + + /* called while holding lockres lock */ + dlm_shuffle_lists(dlm, res); + res->state &= ~DLM_LOCK_RES_DIRTY; + spin_unlock(&res->spinlock); + spin_unlock(&dlm->ast_lock); + + dlm_lockres_calc_usage(dlm, res); + +in_progress: + + spin_lock(&dlm->spinlock); + /* if the lock was in-progress, stick + * it on the back of the list */ + if (delay) { + spin_lock(&res->spinlock); + __dlm_dirty_lockres(dlm, res); + spin_unlock(&res->spinlock); + } + dlm_lockres_put(res); + + /* unlikely, but we may need to give time to + * other tasks */ + if (!--n) { + mlog(0, "%s: Throttling dlm thread\n", + dlm->name); + break; + } + } + + spin_unlock(&dlm->spinlock); + dlm_flush_asts(dlm); + + /* yield and continue right away if there is more work to do */ + if (!n) { + cond_resched(); + continue; + } + + wait_event_interruptible_timeout(dlm->dlm_thread_wq, + !dlm_dirty_list_empty(dlm) || + kthread_should_stop(), + timeout); + } + + mlog(0, "quitting DLM thread\n"); + return 0; +} diff --git a/fs/ocfs2/dlm/dlmunlock.c b/fs/ocfs2/dlm/dlmunlock.c new file mode 100644 index 000000000..7318e4794 --- /dev/null +++ b/fs/ocfs2/dlm/dlmunlock.c @@ -0,0 +1,695 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * dlmunlock.c + * + * underlying calls for unlocking locks + * + * Copyright (C) 2004 Oracle. All rights reserved. + */ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../cluster/heartbeat.h" +#include "../cluster/nodemanager.h" +#include "../cluster/tcp.h" + +#include "dlmapi.h" +#include "dlmcommon.h" + +#define MLOG_MASK_PREFIX ML_DLM +#include "../cluster/masklog.h" + +#define DLM_UNLOCK_FREE_LOCK 0x00000001 +#define DLM_UNLOCK_CALL_AST 0x00000002 +#define DLM_UNLOCK_REMOVE_LOCK 0x00000004 +#define DLM_UNLOCK_REGRANT_LOCK 0x00000008 +#define DLM_UNLOCK_CLEAR_CONVERT_TYPE 0x00000010 + + +static enum dlm_status dlm_get_cancel_actions(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock, + struct dlm_lockstatus *lksb, + int *actions); +static enum dlm_status dlm_get_unlock_actions(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock, + struct dlm_lockstatus *lksb, + int *actions); + +static enum dlm_status dlm_send_remote_unlock_request(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock, + struct dlm_lockstatus *lksb, + int flags, + u8 owner); + + +/* + * according to the spec: + * http://opendlm.sourceforge.net/cvsmirror/opendlm/docs/dlmbook_final.pdf + * + * flags & LKM_CANCEL != 0: must be converting or blocked + * flags & LKM_CANCEL == 0: must be granted + * + * So to unlock a converting lock, you must first cancel the + * convert (passing LKM_CANCEL in flags), then call the unlock + * again (with no LKM_CANCEL in flags). + */ + + +/* + * locking: + * caller needs: none + * taken: res->spinlock and lock->spinlock taken and dropped + * held on exit: none + * returns: DLM_NORMAL, DLM_NOLOCKMGR, status from network + * all callers should have taken an extra ref on lock coming in + */ +static enum dlm_status dlmunlock_common(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock, + struct dlm_lockstatus *lksb, + int flags, int *call_ast, + int master_node) +{ + enum dlm_status status; + int actions = 0; + int in_use; + u8 owner; + int recovery_wait = 0; + + mlog(0, "master_node = %d, valblk = %d\n", master_node, + flags & LKM_VALBLK); + + if (master_node) + BUG_ON(res->owner != dlm->node_num); + else + BUG_ON(res->owner == dlm->node_num); + + spin_lock(&dlm->ast_lock); + /* We want to be sure that we're not freeing a lock + * that still has AST's pending... */ + in_use = !list_empty(&lock->ast_list); + spin_unlock(&dlm->ast_lock); + if (in_use && !(flags & LKM_CANCEL)) { + mlog(ML_ERROR, "lockres %.*s: Someone is calling dlmunlock " + "while waiting for an ast!", res->lockname.len, + res->lockname.name); + return DLM_BADPARAM; + } + + spin_lock(&res->spinlock); + if (res->state & DLM_LOCK_RES_IN_PROGRESS) { + if (master_node && !(flags & LKM_CANCEL)) { + mlog(ML_ERROR, "lockres in progress!\n"); + spin_unlock(&res->spinlock); + return DLM_FORWARD; + } + /* ok for this to sleep if not in a network handler */ + __dlm_wait_on_lockres(res); + res->state |= DLM_LOCK_RES_IN_PROGRESS; + } + spin_lock(&lock->spinlock); + + if (res->state & DLM_LOCK_RES_RECOVERING) { + status = DLM_RECOVERING; + goto leave; + } + + if (res->state & DLM_LOCK_RES_MIGRATING) { + status = DLM_MIGRATING; + goto leave; + } + + /* see above for what the spec says about + * LKM_CANCEL and the lock queue state */ + if (flags & LKM_CANCEL) + status = dlm_get_cancel_actions(dlm, res, lock, lksb, &actions); + else + status = dlm_get_unlock_actions(dlm, res, lock, lksb, &actions); + + if (status != DLM_NORMAL && (status != DLM_CANCELGRANT || !master_node)) + goto leave; + + /* By now this has been masked out of cancel requests. */ + if (flags & LKM_VALBLK) { + /* make the final update to the lvb */ + if (master_node) + memcpy(res->lvb, lksb->lvb, DLM_LVB_LEN); + else + flags |= LKM_PUT_LVB; /* let the send function + * handle it. */ + } + + if (!master_node) { + owner = res->owner; + /* drop locks and send message */ + if (flags & LKM_CANCEL) + lock->cancel_pending = 1; + else + lock->unlock_pending = 1; + spin_unlock(&lock->spinlock); + spin_unlock(&res->spinlock); + status = dlm_send_remote_unlock_request(dlm, res, lock, lksb, + flags, owner); + spin_lock(&res->spinlock); + spin_lock(&lock->spinlock); + /* if the master told us the lock was already granted, + * let the ast handle all of these actions */ + if (status == DLM_CANCELGRANT) { + actions &= ~(DLM_UNLOCK_REMOVE_LOCK| + DLM_UNLOCK_REGRANT_LOCK| + DLM_UNLOCK_CLEAR_CONVERT_TYPE); + } else if (status == DLM_RECOVERING || + status == DLM_MIGRATING || + status == DLM_FORWARD || + status == DLM_NOLOCKMGR + ) { + /* must clear the actions because this unlock + * is about to be retried. cannot free or do + * any list manipulation. */ + mlog(0, "%s:%.*s: clearing actions, %s\n", + dlm->name, res->lockname.len, + res->lockname.name, + status==DLM_RECOVERING?"recovering": + (status==DLM_MIGRATING?"migrating": + (status == DLM_FORWARD ? "forward" : + "nolockmanager"))); + actions = 0; + } + if (flags & LKM_CANCEL) + lock->cancel_pending = 0; + else { + if (!lock->unlock_pending) + recovery_wait = 1; + else + lock->unlock_pending = 0; + } + } + + /* get an extra ref on lock. if we are just switching + * lists here, we dont want the lock to go away. */ + dlm_lock_get(lock); + + if (actions & DLM_UNLOCK_REMOVE_LOCK) { + list_del_init(&lock->list); + dlm_lock_put(lock); + } + if (actions & DLM_UNLOCK_REGRANT_LOCK) { + dlm_lock_get(lock); + list_add_tail(&lock->list, &res->granted); + } + if (actions & DLM_UNLOCK_CLEAR_CONVERT_TYPE) { + mlog(0, "clearing convert_type at %smaster node\n", + master_node ? "" : "non-"); + lock->ml.convert_type = LKM_IVMODE; + } + + /* remove the extra ref on lock */ + dlm_lock_put(lock); + +leave: + res->state &= ~DLM_LOCK_RES_IN_PROGRESS; + if (!dlm_lock_on_list(&res->converting, lock)) + BUG_ON(lock->ml.convert_type != LKM_IVMODE); + else + BUG_ON(lock->ml.convert_type == LKM_IVMODE); + spin_unlock(&lock->spinlock); + spin_unlock(&res->spinlock); + wake_up(&res->wq); + + if (recovery_wait) { + spin_lock(&res->spinlock); + /* Unlock request will directly succeed after owner dies, + * and the lock is already removed from grant list. We have to + * wait for RECOVERING done or we miss the chance to purge it + * since the removement is much faster than RECOVERING proc. + */ + __dlm_wait_on_lockres_flags(res, DLM_LOCK_RES_RECOVERING); + spin_unlock(&res->spinlock); + } + + /* let the caller's final dlm_lock_put handle the actual kfree */ + if (actions & DLM_UNLOCK_FREE_LOCK) { + /* this should always be coupled with list removal */ + BUG_ON(!(actions & DLM_UNLOCK_REMOVE_LOCK)); + mlog(0, "lock %u:%llu should be gone now! refs=%d\n", + dlm_get_lock_cookie_node(be64_to_cpu(lock->ml.cookie)), + dlm_get_lock_cookie_seq(be64_to_cpu(lock->ml.cookie)), + kref_read(&lock->lock_refs)-1); + dlm_lock_put(lock); + } + if (actions & DLM_UNLOCK_CALL_AST) + *call_ast = 1; + + /* if cancel or unlock succeeded, lvb work is done */ + if (status == DLM_NORMAL) + lksb->flags &= ~(DLM_LKSB_PUT_LVB|DLM_LKSB_GET_LVB); + + return status; +} + +void dlm_commit_pending_unlock(struct dlm_lock_resource *res, + struct dlm_lock *lock) +{ + /* leave DLM_LKSB_PUT_LVB on the lksb so any final + * update of the lvb will be sent to the new master */ + list_del_init(&lock->list); +} + +void dlm_commit_pending_cancel(struct dlm_lock_resource *res, + struct dlm_lock *lock) +{ + list_move_tail(&lock->list, &res->granted); + lock->ml.convert_type = LKM_IVMODE; +} + + +static inline enum dlm_status dlmunlock_master(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock, + struct dlm_lockstatus *lksb, + int flags, + int *call_ast) +{ + return dlmunlock_common(dlm, res, lock, lksb, flags, call_ast, 1); +} + +static inline enum dlm_status dlmunlock_remote(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock, + struct dlm_lockstatus *lksb, + int flags, int *call_ast) +{ + return dlmunlock_common(dlm, res, lock, lksb, flags, call_ast, 0); +} + +/* + * locking: + * caller needs: none + * taken: none + * held on exit: none + * returns: DLM_NORMAL, DLM_NOLOCKMGR, status from network + */ +static enum dlm_status dlm_send_remote_unlock_request(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock, + struct dlm_lockstatus *lksb, + int flags, + u8 owner) +{ + struct dlm_unlock_lock unlock; + int tmpret; + enum dlm_status ret; + int status = 0; + struct kvec vec[2]; + size_t veclen = 1; + + mlog(0, "%.*s\n", res->lockname.len, res->lockname.name); + + if (owner == dlm->node_num) { + /* ended up trying to contact ourself. this means + * that the lockres had been remote but became local + * via a migration. just retry it, now as local */ + mlog(0, "%s:%.*s: this node became the master due to a " + "migration, re-evaluate now\n", dlm->name, + res->lockname.len, res->lockname.name); + return DLM_FORWARD; + } + + memset(&unlock, 0, sizeof(unlock)); + unlock.node_idx = dlm->node_num; + unlock.flags = cpu_to_be32(flags); + unlock.cookie = lock->ml.cookie; + unlock.namelen = res->lockname.len; + memcpy(unlock.name, res->lockname.name, unlock.namelen); + + vec[0].iov_len = sizeof(struct dlm_unlock_lock); + vec[0].iov_base = &unlock; + + if (flags & LKM_PUT_LVB) { + /* extra data to send if we are updating lvb */ + vec[1].iov_len = DLM_LVB_LEN; + vec[1].iov_base = lock->lksb->lvb; + veclen++; + } + + tmpret = o2net_send_message_vec(DLM_UNLOCK_LOCK_MSG, dlm->key, + vec, veclen, owner, &status); + if (tmpret >= 0) { + // successfully sent and received + if (status == DLM_FORWARD) + mlog(0, "master was in-progress. retry\n"); + ret = status; + } else { + mlog(ML_ERROR, "Error %d when sending message %u (key 0x%x) to " + "node %u\n", tmpret, DLM_UNLOCK_LOCK_MSG, dlm->key, owner); + if (dlm_is_host_down(tmpret)) { + /* NOTE: this seems strange, but it is what we want. + * when the master goes down during a cancel or + * unlock, the recovery code completes the operation + * as if the master had not died, then passes the + * updated state to the recovery master. this thread + * just needs to finish out the operation and call + * the unlockast. */ + if (dlm_is_node_dead(dlm, owner)) + ret = DLM_NORMAL; + else + ret = DLM_NOLOCKMGR; + } else { + /* something bad. this will BUG in ocfs2 */ + ret = dlm_err_to_dlm_status(tmpret); + } + } + + return ret; +} + +/* + * locking: + * caller needs: none + * taken: takes and drops res->spinlock + * held on exit: none + * returns: DLM_NORMAL, DLM_BADARGS, DLM_IVLOCKID, + * return value from dlmunlock_master + */ +int dlm_unlock_lock_handler(struct o2net_msg *msg, u32 len, void *data, + void **ret_data) +{ + struct dlm_ctxt *dlm = data; + struct dlm_unlock_lock *unlock = (struct dlm_unlock_lock *)msg->buf; + struct dlm_lock_resource *res = NULL; + struct dlm_lock *lock = NULL, *iter; + enum dlm_status status = DLM_NORMAL; + int i; + struct dlm_lockstatus *lksb = NULL; + int ignore; + u32 flags; + struct list_head *queue; + + flags = be32_to_cpu(unlock->flags); + + if (flags & LKM_GET_LVB) { + mlog(ML_ERROR, "bad args! GET_LVB specified on unlock!\n"); + return DLM_BADARGS; + } + + if ((flags & (LKM_PUT_LVB|LKM_CANCEL)) == (LKM_PUT_LVB|LKM_CANCEL)) { + mlog(ML_ERROR, "bad args! cannot modify lvb on a CANCEL " + "request!\n"); + return DLM_BADARGS; + } + + if (unlock->namelen > DLM_LOCKID_NAME_MAX) { + mlog(ML_ERROR, "Invalid name length in unlock handler!\n"); + return DLM_IVBUFLEN; + } + + if (!dlm_grab(dlm)) + return DLM_FORWARD; + + mlog_bug_on_msg(!dlm_domain_fully_joined(dlm), + "Domain %s not fully joined!\n", dlm->name); + + mlog(0, "lvb: %s\n", flags & LKM_PUT_LVB ? "put lvb" : "none"); + + res = dlm_lookup_lockres(dlm, unlock->name, unlock->namelen); + if (!res) { + /* We assume here that a no lock resource simply means + * it was migrated away and destroyed before the other + * node could detect it. */ + mlog(0, "returning DLM_FORWARD -- res no longer exists\n"); + status = DLM_FORWARD; + goto not_found; + } + + queue=&res->granted; + spin_lock(&res->spinlock); + if (res->state & DLM_LOCK_RES_RECOVERING) { + spin_unlock(&res->spinlock); + mlog(0, "returning DLM_RECOVERING\n"); + status = DLM_RECOVERING; + goto leave; + } + + if (res->state & DLM_LOCK_RES_MIGRATING) { + spin_unlock(&res->spinlock); + mlog(0, "returning DLM_MIGRATING\n"); + status = DLM_MIGRATING; + goto leave; + } + + if (res->owner != dlm->node_num) { + spin_unlock(&res->spinlock); + mlog(0, "returning DLM_FORWARD -- not master\n"); + status = DLM_FORWARD; + goto leave; + } + + for (i=0; i<3; i++) { + list_for_each_entry(iter, queue, list) { + if (iter->ml.cookie == unlock->cookie && + iter->ml.node == unlock->node_idx) { + dlm_lock_get(iter); + lock = iter; + break; + } + } + if (lock) + break; + /* scan granted -> converting -> blocked queues */ + queue++; + } + spin_unlock(&res->spinlock); + if (!lock) { + status = DLM_IVLOCKID; + goto not_found; + } + + /* lock was found on queue */ + lksb = lock->lksb; + if (flags & (LKM_VALBLK|LKM_PUT_LVB) && + lock->ml.type != LKM_EXMODE) + flags &= ~(LKM_VALBLK|LKM_PUT_LVB); + + /* unlockast only called on originating node */ + if (flags & LKM_PUT_LVB) { + lksb->flags |= DLM_LKSB_PUT_LVB; + memcpy(&lksb->lvb[0], &unlock->lvb[0], DLM_LVB_LEN); + } + + /* if this is in-progress, propagate the DLM_FORWARD + * all the way back out */ + status = dlmunlock_master(dlm, res, lock, lksb, flags, &ignore); + if (status == DLM_FORWARD) + mlog(0, "lockres is in progress\n"); + + if (flags & LKM_PUT_LVB) + lksb->flags &= ~DLM_LKSB_PUT_LVB; + + dlm_lockres_calc_usage(dlm, res); + dlm_kick_thread(dlm, res); + +not_found: + if (!lock) + mlog(ML_ERROR, "failed to find lock to unlock! " + "cookie=%u:%llu\n", + dlm_get_lock_cookie_node(be64_to_cpu(unlock->cookie)), + dlm_get_lock_cookie_seq(be64_to_cpu(unlock->cookie))); + else + dlm_lock_put(lock); + +leave: + if (res) + dlm_lockres_put(res); + + dlm_put(dlm); + + return status; +} + + +static enum dlm_status dlm_get_cancel_actions(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock, + struct dlm_lockstatus *lksb, + int *actions) +{ + enum dlm_status status; + + if (dlm_lock_on_list(&res->blocked, lock)) { + /* cancel this outright */ + status = DLM_NORMAL; + *actions = (DLM_UNLOCK_CALL_AST | + DLM_UNLOCK_REMOVE_LOCK); + } else if (dlm_lock_on_list(&res->converting, lock)) { + /* cancel the request, put back on granted */ + status = DLM_NORMAL; + *actions = (DLM_UNLOCK_CALL_AST | + DLM_UNLOCK_REMOVE_LOCK | + DLM_UNLOCK_REGRANT_LOCK | + DLM_UNLOCK_CLEAR_CONVERT_TYPE); + } else if (dlm_lock_on_list(&res->granted, lock)) { + /* too late, already granted. */ + status = DLM_CANCELGRANT; + *actions = DLM_UNLOCK_CALL_AST; + } else { + mlog(ML_ERROR, "lock to cancel is not on any list!\n"); + status = DLM_IVLOCKID; + *actions = 0; + } + return status; +} + +static enum dlm_status dlm_get_unlock_actions(struct dlm_ctxt *dlm, + struct dlm_lock_resource *res, + struct dlm_lock *lock, + struct dlm_lockstatus *lksb, + int *actions) +{ + enum dlm_status status; + + /* unlock request */ + if (!dlm_lock_on_list(&res->granted, lock)) { + status = DLM_DENIED; + dlm_error(status); + *actions = 0; + } else { + /* unlock granted lock */ + status = DLM_NORMAL; + *actions = (DLM_UNLOCK_FREE_LOCK | + DLM_UNLOCK_CALL_AST | + DLM_UNLOCK_REMOVE_LOCK); + } + return status; +} + +/* there seems to be no point in doing this async + * since (even for the remote case) there is really + * no work to queue up... so just do it and fire the + * unlockast by hand when done... */ +enum dlm_status dlmunlock(struct dlm_ctxt *dlm, struct dlm_lockstatus *lksb, + int flags, dlm_astunlockfunc_t *unlockast, void *data) +{ + enum dlm_status status; + struct dlm_lock_resource *res; + struct dlm_lock *lock = NULL; + int call_ast, is_master; + + if (!lksb) { + dlm_error(DLM_BADARGS); + return DLM_BADARGS; + } + + if (flags & ~(LKM_CANCEL | LKM_VALBLK | LKM_INVVALBLK)) { + dlm_error(DLM_BADPARAM); + return DLM_BADPARAM; + } + + if ((flags & (LKM_VALBLK | LKM_CANCEL)) == (LKM_VALBLK | LKM_CANCEL)) { + mlog(0, "VALBLK given with CANCEL: ignoring VALBLK\n"); + flags &= ~LKM_VALBLK; + } + + if (!lksb->lockid || !lksb->lockid->lockres) { + dlm_error(DLM_BADPARAM); + return DLM_BADPARAM; + } + + lock = lksb->lockid; + BUG_ON(!lock); + dlm_lock_get(lock); + + res = lock->lockres; + BUG_ON(!res); + dlm_lockres_get(res); +retry: + call_ast = 0; + /* need to retry up here because owner may have changed */ + mlog(0, "lock=%p res=%p\n", lock, res); + + spin_lock(&res->spinlock); + is_master = (res->owner == dlm->node_num); + if (flags & LKM_VALBLK && lock->ml.type != LKM_EXMODE) + flags &= ~LKM_VALBLK; + spin_unlock(&res->spinlock); + + if (is_master) { + status = dlmunlock_master(dlm, res, lock, lksb, flags, + &call_ast); + mlog(0, "done calling dlmunlock_master: returned %d, " + "call_ast is %d\n", status, call_ast); + } else { + status = dlmunlock_remote(dlm, res, lock, lksb, flags, + &call_ast); + mlog(0, "done calling dlmunlock_remote: returned %d, " + "call_ast is %d\n", status, call_ast); + } + + if (status == DLM_RECOVERING || + status == DLM_MIGRATING || + status == DLM_FORWARD || + status == DLM_NOLOCKMGR) { + + /* We want to go away for a tiny bit to allow recovery + * / migration to complete on this resource. I don't + * know of any wait queue we could sleep on as this + * may be happening on another node. Perhaps the + * proper solution is to queue up requests on the + * other end? */ + + /* do we want to yield(); ?? */ + msleep(50); + + mlog(0, "retrying unlock due to pending recovery/" + "migration/in-progress/reconnect\n"); + goto retry; + } + + if (call_ast) { + mlog(0, "calling unlockast(%p, %d)\n", data, status); + if (is_master) { + /* it is possible that there is one last bast + * pending. make sure it is flushed, then + * call the unlockast. + * not an issue if this is a mastered remotely, + * since this lock has been removed from the + * lockres queues and cannot be found. */ + dlm_kick_thread(dlm, NULL); + wait_event(dlm->ast_wq, + dlm_lock_basts_flushed(dlm, lock)); + } + (*unlockast)(data, status); + } + + if (status == DLM_CANCELGRANT) + status = DLM_NORMAL; + + if (status == DLM_NORMAL) { + mlog(0, "kicking the thread\n"); + dlm_kick_thread(dlm, res); + } else + dlm_error(status); + + dlm_lockres_calc_usage(dlm, res); + dlm_lockres_put(res); + dlm_lock_put(lock); + + mlog(0, "returning status=%d!\n", status); + return status; +} +EXPORT_SYMBOL_GPL(dlmunlock); + diff --git a/fs/ocfs2/dlmfs/Makefile b/fs/ocfs2/dlmfs/Makefile new file mode 100644 index 000000000..c7895f65b --- /dev/null +++ b/fs/ocfs2/dlmfs/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_OCFS2_FS) += ocfs2_dlmfs.o + +ocfs2_dlmfs-objs := userdlm.o dlmfs.o diff --git a/fs/ocfs2/dlmfs/dlmfs.c b/fs/ocfs2/dlmfs/dlmfs.c new file mode 100644 index 000000000..53df5dd10 --- /dev/null +++ b/fs/ocfs2/dlmfs/dlmfs.c @@ -0,0 +1,636 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * dlmfs.c + * + * Code which implements the kernel side of a minimal userspace + * interface to our DLM. This file handles the virtual file system + * used for communication with userspace. Credit should go to ramfs, + * which was a template for the fs side of this module. + * + * Copyright (C) 2003, 2004 Oracle. All rights reserved. + */ + +/* Simple VFS hooks based on: */ +/* + * Resizable simple ram filesystem for Linux. + * + * Copyright (C) 2000 Linus Torvalds. + * 2000 Transmeta Corp. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "../stackglue.h" +#include "userdlm.h" + +#define MLOG_MASK_PREFIX ML_DLMFS +#include "../cluster/masklog.h" + + +static const struct super_operations dlmfs_ops; +static const struct file_operations dlmfs_file_operations; +static const struct inode_operations dlmfs_dir_inode_operations; +static const struct inode_operations dlmfs_root_inode_operations; +static const struct inode_operations dlmfs_file_inode_operations; +static struct kmem_cache *dlmfs_inode_cache; + +struct workqueue_struct *user_dlm_worker; + + + +/* + * These are the ABI capabilities of dlmfs. + * + * Over time, dlmfs has added some features that were not part of the + * initial ABI. Unfortunately, some of these features are not detectable + * via standard usage. For example, Linux's default poll always returns + * EPOLLIN, so there is no way for a caller of poll(2) to know when dlmfs + * added poll support. Instead, we provide this list of new capabilities. + * + * Capabilities is a read-only attribute. We do it as a module parameter + * so we can discover it whether dlmfs is built in, loaded, or even not + * loaded. + * + * The ABI features are local to this machine's dlmfs mount. This is + * distinct from the locking protocol, which is concerned with inter-node + * interaction. + * + * Capabilities: + * - bast : EPOLLIN against the file descriptor of a held lock + * signifies a bast fired on the lock. + */ +#define DLMFS_CAPABILITIES "bast stackglue" +static int param_set_dlmfs_capabilities(const char *val, + const struct kernel_param *kp) +{ + printk(KERN_ERR "%s: readonly parameter\n", kp->name); + return -EINVAL; +} +static int param_get_dlmfs_capabilities(char *buffer, + const struct kernel_param *kp) +{ + return sysfs_emit(buffer, DLMFS_CAPABILITIES); +} +module_param_call(capabilities, param_set_dlmfs_capabilities, + param_get_dlmfs_capabilities, NULL, 0444); +MODULE_PARM_DESC(capabilities, DLMFS_CAPABILITIES); + + +/* + * decodes a set of open flags into a valid lock level and a set of flags. + * returns < 0 if we have invalid flags + * flags which mean something to us: + * O_RDONLY -> PRMODE level + * O_WRONLY -> EXMODE level + * + * O_NONBLOCK -> NOQUEUE + */ +static int dlmfs_decode_open_flags(int open_flags, + int *level, + int *flags) +{ + if (open_flags & (O_WRONLY|O_RDWR)) + *level = DLM_LOCK_EX; + else + *level = DLM_LOCK_PR; + + *flags = 0; + if (open_flags & O_NONBLOCK) + *flags |= DLM_LKF_NOQUEUE; + + return 0; +} + +static int dlmfs_file_open(struct inode *inode, + struct file *file) +{ + int status, level, flags; + struct dlmfs_filp_private *fp = NULL; + struct dlmfs_inode_private *ip; + + if (S_ISDIR(inode->i_mode)) + BUG(); + + mlog(0, "open called on inode %llu, flags 0x%x\n", inode->i_ino, + file->f_flags); + + status = dlmfs_decode_open_flags(file->f_flags, &level, &flags); + if (status < 0) + goto bail; + + /* We don't want to honor O_APPEND at read/write time as it + * doesn't make sense for LVB writes. */ + file->f_flags &= ~O_APPEND; + + fp = kmalloc_obj(*fp, GFP_NOFS); + if (!fp) { + status = -ENOMEM; + goto bail; + } + fp->fp_lock_level = level; + + ip = DLMFS_I(inode); + + status = user_dlm_cluster_lock(&ip->ip_lockres, level, flags); + if (status < 0) { + /* this is a strange error to return here but I want + * to be able userspace to be able to distinguish a + * valid lock request from one that simply couldn't be + * granted. */ + if (flags & DLM_LKF_NOQUEUE && status == -EAGAIN) + status = -ETXTBSY; + kfree(fp); + goto bail; + } + + file->private_data = fp; +bail: + return status; +} + +static int dlmfs_file_release(struct inode *inode, + struct file *file) +{ + int level; + struct dlmfs_inode_private *ip = DLMFS_I(inode); + struct dlmfs_filp_private *fp = file->private_data; + + if (S_ISDIR(inode->i_mode)) + BUG(); + + mlog(0, "close called on inode %llu\n", inode->i_ino); + + if (fp) { + level = fp->fp_lock_level; + if (level != DLM_LOCK_IV) + user_dlm_cluster_unlock(&ip->ip_lockres, level); + + kfree(fp); + file->private_data = NULL; + } + + return 0; +} + +/* + * We do ->setattr() just to override size changes. Our size is the size + * of the LVB and nothing else. + */ +static int dlmfs_file_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, struct iattr *attr) +{ + int error; + struct inode *inode = d_inode(dentry); + + attr->ia_valid &= ~ATTR_SIZE; + error = setattr_prepare(&nop_mnt_idmap, dentry, attr); + if (error) + return error; + + setattr_copy(&nop_mnt_idmap, inode, attr); + mark_inode_dirty(inode); + return 0; +} + +static __poll_t dlmfs_file_poll(struct file *file, poll_table *wait) +{ + __poll_t event = 0; + struct inode *inode = file_inode(file); + struct dlmfs_inode_private *ip = DLMFS_I(inode); + + poll_wait(file, &ip->ip_lockres.l_event, wait); + + spin_lock(&ip->ip_lockres.l_lock); + if (ip->ip_lockres.l_flags & USER_LOCK_BLOCKED) + event = EPOLLIN | EPOLLRDNORM; + spin_unlock(&ip->ip_lockres.l_lock); + + return event; +} + +static ssize_t dlmfs_file_read(struct file *file, + char __user *buf, + size_t count, + loff_t *ppos) +{ + char lvb[DLM_LVB_LEN]; + + if (!user_dlm_read_lvb(file_inode(file), lvb)) + return 0; + + return simple_read_from_buffer(buf, count, ppos, lvb, sizeof(lvb)); +} + +static ssize_t dlmfs_file_write(struct file *filp, + const char __user *buf, + size_t count, + loff_t *ppos) +{ + char lvb_buf[DLM_LVB_LEN]; + int bytes_left; + struct inode *inode = file_inode(filp); + + mlog(0, "inode %llu, count = %zu, *ppos = %llu\n", + inode->i_ino, count, *ppos); + + if (*ppos >= DLM_LVB_LEN) + return -ENOSPC; + + /* don't write past the lvb */ + if (count > DLM_LVB_LEN - *ppos) + count = DLM_LVB_LEN - *ppos; + + if (!count) + return 0; + + bytes_left = copy_from_user(lvb_buf, buf, count); + count -= bytes_left; + if (count) + user_dlm_write_lvb(inode, lvb_buf, count); + + *ppos = *ppos + count; + mlog(0, "wrote %zu bytes\n", count); + return count; +} + +static void dlmfs_init_once(void *foo) +{ + struct dlmfs_inode_private *ip = + (struct dlmfs_inode_private *) foo; + + ip->ip_conn = NULL; + ip->ip_parent = NULL; + + inode_init_once(&ip->ip_vfs_inode); +} + +static struct inode *dlmfs_alloc_inode(struct super_block *sb) +{ + struct dlmfs_inode_private *ip; + + ip = alloc_inode_sb(sb, dlmfs_inode_cache, GFP_NOFS); + if (!ip) + return NULL; + + return &ip->ip_vfs_inode; +} + +static void dlmfs_free_inode(struct inode *inode) +{ + kmem_cache_free(dlmfs_inode_cache, DLMFS_I(inode)); +} + +static void dlmfs_evict_inode(struct inode *inode) +{ + int status; + struct dlmfs_inode_private *ip; + struct user_lock_res *lockres; + int teardown; + + clear_inode(inode); + + mlog(0, "inode %llu\n", inode->i_ino); + + ip = DLMFS_I(inode); + lockres = &ip->ip_lockres; + + if (S_ISREG(inode->i_mode)) { + spin_lock(&lockres->l_lock); + teardown = !!(lockres->l_flags & USER_LOCK_IN_TEARDOWN); + spin_unlock(&lockres->l_lock); + if (!teardown) { + status = user_dlm_destroy_lock(lockres); + if (status < 0) + mlog_errno(status); + } + iput(ip->ip_parent); + goto clear_fields; + } + + mlog(0, "we're a directory, ip->ip_conn = 0x%p\n", ip->ip_conn); + /* we must be a directory. If required, lets unregister the + * dlm context now. */ + if (ip->ip_conn) + user_dlm_unregister(ip->ip_conn); +clear_fields: + ip->ip_parent = NULL; + ip->ip_conn = NULL; +} + +static struct inode *dlmfs_get_root_inode(struct super_block *sb) +{ + struct inode *inode = new_inode(sb); + umode_t mode = S_IFDIR | 0755; + + if (inode) { + inode->i_ino = get_next_ino(); + inode_init_owner(&nop_mnt_idmap, inode, NULL, mode); + simple_inode_init_ts(inode); + inc_nlink(inode); + + inode->i_fop = &simple_dir_operations; + inode->i_op = &dlmfs_root_inode_operations; + } + + return inode; +} + +static struct inode *dlmfs_get_inode(struct inode *parent, + struct dentry *dentry, + umode_t mode) +{ + struct super_block *sb = parent->i_sb; + struct inode * inode = new_inode(sb); + struct dlmfs_inode_private *ip; + + if (!inode) + return NULL; + + inode->i_ino = get_next_ino(); + inode_init_owner(&nop_mnt_idmap, inode, parent, mode); + simple_inode_init_ts(inode); + + ip = DLMFS_I(inode); + ip->ip_conn = DLMFS_I(parent)->ip_conn; + + switch (mode & S_IFMT) { + default: + /* for now we don't support anything other than + * directories and regular files. */ + BUG(); + break; + case S_IFREG: + inode->i_op = &dlmfs_file_inode_operations; + inode->i_fop = &dlmfs_file_operations; + + i_size_write(inode, DLM_LVB_LEN); + + user_dlm_lock_res_init(&ip->ip_lockres, dentry); + + /* released at clear_inode time, this insures that we + * get to drop the dlm reference on each lock *before* + * we call the unregister code for releasing parent + * directories. */ + ip->ip_parent = igrab(parent); + BUG_ON(!ip->ip_parent); + break; + case S_IFDIR: + inode->i_op = &dlmfs_dir_inode_operations; + inode->i_fop = &simple_dir_operations; + + /* directory inodes start off with i_nlink == + * 2 (for "." entry) */ + inc_nlink(inode); + break; + } + return inode; +} + +/* + * File creation. Allocate an inode, and we're done.. + */ +/* SMP-safe */ +static struct dentry *dlmfs_mkdir(struct mnt_idmap * idmap, + struct inode * dir, + struct dentry * dentry, + umode_t mode) +{ + int status; + struct inode *inode = NULL; + const struct qstr *domain = &dentry->d_name; + struct dlmfs_inode_private *ip; + struct ocfs2_cluster_connection *conn; + + mlog(0, "mkdir %.*s\n", domain->len, domain->name); + + /* verify that we have a proper domain */ + if (domain->len >= GROUP_NAME_MAX) { + status = -EINVAL; + mlog(ML_ERROR, "invalid domain name for directory.\n"); + goto bail; + } + + inode = dlmfs_get_inode(dir, dentry, mode); + if (!inode) { + status = -ENOMEM; + mlog_errno(status); + goto bail; + } + + ip = DLMFS_I(inode); + + conn = user_dlm_register(domain); + if (IS_ERR(conn)) { + status = PTR_ERR(conn); + mlog(ML_ERROR, "Error %d could not register domain \"%.*s\"\n", + status, domain->len, domain->name); + goto bail; + } + ip->ip_conn = conn; + + inc_nlink(dir); + d_make_persistent(dentry, inode); + + status = 0; +bail: + if (status < 0) + iput(inode); + return ERR_PTR(status); +} + +static int dlmfs_create(struct mnt_idmap *idmap, + struct inode *dir, + struct dentry *dentry, + umode_t mode) +{ + int status = 0; + struct inode *inode; + const struct qstr *name = &dentry->d_name; + + mlog(0, "create %.*s\n", name->len, name->name); + + /* verify name is valid and doesn't contain any dlm reserved + * characters */ + if (name->len >= USER_DLM_LOCK_ID_MAX_LEN || + name->name[0] == '$') { + status = -EINVAL; + mlog(ML_ERROR, "invalid lock name, %.*s\n", name->len, + name->name); + goto bail; + } + + inode = dlmfs_get_inode(dir, dentry, mode | S_IFREG); + if (!inode) { + status = -ENOMEM; + mlog_errno(status); + goto bail; + } + + d_make_persistent(dentry, inode); +bail: + return status; +} + +static int dlmfs_unlink(struct inode *dir, + struct dentry *dentry) +{ + int status; + struct inode *inode = d_inode(dentry); + + mlog(0, "unlink inode %llu\n", inode->i_ino); + + /* if there are no current holders, or none that are waiting + * to acquire a lock, this basically destroys our lockres. */ + status = user_dlm_destroy_lock(&DLMFS_I(inode)->ip_lockres); + if (status < 0) { + mlog(ML_ERROR, "unlink %pd, error %d from destroy\n", + dentry, status); + goto bail; + } + status = simple_unlink(dir, dentry); +bail: + return status; +} + +static int dlmfs_fill_super(struct super_block *sb, struct fs_context *fc) +{ + sb->s_maxbytes = MAX_LFS_FILESIZE; + sb->s_blocksize = PAGE_SIZE; + sb->s_blocksize_bits = PAGE_SHIFT; + sb->s_magic = DLMFS_MAGIC; + sb->s_op = &dlmfs_ops; + sb->s_root = d_make_root(dlmfs_get_root_inode(sb)); + if (!sb->s_root) + return -ENOMEM; + return 0; +} + +static const struct file_operations dlmfs_file_operations = { + .open = dlmfs_file_open, + .release = dlmfs_file_release, + .poll = dlmfs_file_poll, + .read = dlmfs_file_read, + .write = dlmfs_file_write, + .llseek = default_llseek, +}; + +static const struct inode_operations dlmfs_dir_inode_operations = { + .create = dlmfs_create, + .lookup = simple_lookup, + .unlink = dlmfs_unlink, +}; + +/* this way we can restrict mkdir to only the toplevel of the fs. */ +static const struct inode_operations dlmfs_root_inode_operations = { + .lookup = simple_lookup, + .mkdir = dlmfs_mkdir, + .rmdir = simple_rmdir, +}; + +static const struct super_operations dlmfs_ops = { + .statfs = simple_statfs, + .alloc_inode = dlmfs_alloc_inode, + .free_inode = dlmfs_free_inode, + .evict_inode = dlmfs_evict_inode, + .drop_inode = inode_just_drop, +}; + +static const struct inode_operations dlmfs_file_inode_operations = { + .getattr = simple_getattr, + .setattr = dlmfs_file_setattr, +}; + +static int dlmfs_get_tree(struct fs_context *fc) +{ + return get_tree_nodev(fc, dlmfs_fill_super); +} + +static const struct fs_context_operations dlmfs_context_ops = { + .get_tree = dlmfs_get_tree, +}; + +static int dlmfs_init_fs_context(struct fs_context *fc) +{ + fc->ops = &dlmfs_context_ops; + + return 0; +} + +static struct file_system_type dlmfs_fs_type = { + .owner = THIS_MODULE, + .name = "ocfs2_dlmfs", + .kill_sb = kill_anon_super, + .init_fs_context = dlmfs_init_fs_context, +}; +MODULE_ALIAS_FS("ocfs2_dlmfs"); + +static int __init init_dlmfs_fs(void) +{ + int status; + int cleanup_inode = 0, cleanup_worker = 0; + + dlmfs_inode_cache = kmem_cache_create("dlmfs_inode_cache", + sizeof(struct dlmfs_inode_private), + 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| + SLAB_ACCOUNT), + dlmfs_init_once); + if (!dlmfs_inode_cache) { + status = -ENOMEM; + goto bail; + } + cleanup_inode = 1; + + user_dlm_worker = alloc_workqueue("user_dlm", + WQ_MEM_RECLAIM | WQ_PERCPU, 0); + if (!user_dlm_worker) { + status = -ENOMEM; + goto bail; + } + cleanup_worker = 1; + + user_dlm_set_locking_protocol(); + status = register_filesystem(&dlmfs_fs_type); +bail: + if (status) { + if (cleanup_inode) + kmem_cache_destroy(dlmfs_inode_cache); + if (cleanup_worker) + destroy_workqueue(user_dlm_worker); + } else + printk("OCFS2 User DLM kernel interface loaded\n"); + return status; +} + +static void __exit exit_dlmfs_fs(void) +{ + unregister_filesystem(&dlmfs_fs_type); + + destroy_workqueue(user_dlm_worker); + + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(dlmfs_inode_cache); + +} + +MODULE_AUTHOR("Oracle"); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("OCFS2 DLM-Filesystem"); + +module_init(init_dlmfs_fs) +module_exit(exit_dlmfs_fs) diff --git a/fs/ocfs2/dlmfs/userdlm.c b/fs/ocfs2/dlmfs/userdlm.c new file mode 100644 index 000000000..617c92e7b --- /dev/null +++ b/fs/ocfs2/dlmfs/userdlm.c @@ -0,0 +1,682 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * userdlm.c + * + * Code which implements the kernel side of a minimal userspace + * interface to our DLM. + * + * Many of the functions here are pared down versions of dlmglue.c + * functions. + * + * Copyright (C) 2003, 2004 Oracle. All rights reserved. + */ + +#include +#include + +#include +#include +#include +#include + +#include "../ocfs2_lockingver.h" +#include "../stackglue.h" +#include "userdlm.h" + +#define MLOG_MASK_PREFIX ML_DLMFS +#include "../cluster/masklog.h" + + +static inline struct user_lock_res *user_lksb_to_lock_res(struct ocfs2_dlm_lksb *lksb) +{ + return container_of(lksb, struct user_lock_res, l_lksb); +} + +static inline int user_check_wait_flag(struct user_lock_res *lockres, + int flag) +{ + int ret; + + spin_lock(&lockres->l_lock); + ret = lockres->l_flags & flag; + spin_unlock(&lockres->l_lock); + + return ret; +} + +static inline void user_wait_on_busy_lock(struct user_lock_res *lockres) + +{ + wait_event(lockres->l_event, + !user_check_wait_flag(lockres, USER_LOCK_BUSY)); +} + +static inline void user_wait_on_blocked_lock(struct user_lock_res *lockres) + +{ + wait_event(lockres->l_event, + !user_check_wait_flag(lockres, USER_LOCK_BLOCKED)); +} + +/* I heart container_of... */ +static inline struct ocfs2_cluster_connection * +cluster_connection_from_user_lockres(struct user_lock_res *lockres) +{ + struct dlmfs_inode_private *ip; + + ip = container_of(lockres, + struct dlmfs_inode_private, + ip_lockres); + return ip->ip_conn; +} + +static struct inode * +user_dlm_inode_from_user_lockres(struct user_lock_res *lockres) +{ + struct dlmfs_inode_private *ip; + + ip = container_of(lockres, + struct dlmfs_inode_private, + ip_lockres); + return &ip->ip_vfs_inode; +} + +static inline void user_recover_from_dlm_error(struct user_lock_res *lockres) +{ + spin_lock(&lockres->l_lock); + lockres->l_flags &= ~USER_LOCK_BUSY; + spin_unlock(&lockres->l_lock); +} + +#define user_log_dlm_error(_func, _stat, _lockres) do { \ + mlog(ML_ERROR, "Dlm error %d while calling %s on " \ + "resource %.*s\n", _stat, _func, \ + _lockres->l_namelen, _lockres->l_name); \ +} while (0) + +/* WARNING: This function lives in a world where the only three lock + * levels are EX, PR, and NL. It *will* have to be adjusted when more + * lock types are added. */ +static inline int user_highest_compat_lock_level(int level) +{ + int new_level = DLM_LOCK_EX; + + if (level == DLM_LOCK_EX) + new_level = DLM_LOCK_NL; + else if (level == DLM_LOCK_PR) + new_level = DLM_LOCK_PR; + return new_level; +} + +static void user_ast(struct ocfs2_dlm_lksb *lksb) +{ + struct user_lock_res *lockres = user_lksb_to_lock_res(lksb); + int status; + + mlog(ML_BASTS, "AST fired for lockres %.*s, level %d => %d\n", + lockres->l_namelen, lockres->l_name, lockres->l_level, + lockres->l_requested); + + spin_lock(&lockres->l_lock); + + status = ocfs2_dlm_lock_status(&lockres->l_lksb); + if (status) { + mlog(ML_ERROR, "lksb status value of %u on lockres %.*s\n", + status, lockres->l_namelen, lockres->l_name); + spin_unlock(&lockres->l_lock); + return; + } + + mlog_bug_on_msg(lockres->l_requested == DLM_LOCK_IV, + "Lockres %.*s, requested ivmode. flags 0x%x\n", + lockres->l_namelen, lockres->l_name, lockres->l_flags); + + /* we're downconverting. */ + if (lockres->l_requested < lockres->l_level) { + if (lockres->l_requested <= + user_highest_compat_lock_level(lockres->l_blocking)) { + lockres->l_blocking = DLM_LOCK_NL; + lockres->l_flags &= ~USER_LOCK_BLOCKED; + } + } + + lockres->l_level = lockres->l_requested; + lockres->l_requested = DLM_LOCK_IV; + lockres->l_flags |= USER_LOCK_ATTACHED; + lockres->l_flags &= ~USER_LOCK_BUSY; + + spin_unlock(&lockres->l_lock); + + wake_up(&lockres->l_event); +} + +static inline void user_dlm_grab_inode_ref(struct user_lock_res *lockres) +{ + struct inode *inode; + inode = user_dlm_inode_from_user_lockres(lockres); + if (!igrab(inode)) + BUG(); +} + +static void user_dlm_unblock_lock(struct work_struct *work); + +static void __user_dlm_queue_lockres(struct user_lock_res *lockres) +{ + if (!(lockres->l_flags & USER_LOCK_QUEUED)) { + user_dlm_grab_inode_ref(lockres); + + INIT_WORK(&lockres->l_work, user_dlm_unblock_lock); + + queue_work(user_dlm_worker, &lockres->l_work); + lockres->l_flags |= USER_LOCK_QUEUED; + } +} + +static void __user_dlm_cond_queue_lockres(struct user_lock_res *lockres) +{ + int queue = 0; + + if (!(lockres->l_flags & USER_LOCK_BLOCKED)) + return; + + switch (lockres->l_blocking) { + case DLM_LOCK_EX: + if (!lockres->l_ex_holders && !lockres->l_ro_holders) + queue = 1; + break; + case DLM_LOCK_PR: + if (!lockres->l_ex_holders) + queue = 1; + break; + default: + BUG(); + } + + if (queue) + __user_dlm_queue_lockres(lockres); +} + +static void user_bast(struct ocfs2_dlm_lksb *lksb, int level) +{ + struct user_lock_res *lockres = user_lksb_to_lock_res(lksb); + + mlog(ML_BASTS, "BAST fired for lockres %.*s, blocking %d, level %d\n", + lockres->l_namelen, lockres->l_name, level, lockres->l_level); + + spin_lock(&lockres->l_lock); + lockres->l_flags |= USER_LOCK_BLOCKED; + if (level > lockres->l_blocking) + lockres->l_blocking = level; + + __user_dlm_queue_lockres(lockres); + spin_unlock(&lockres->l_lock); + + wake_up(&lockres->l_event); +} + +static void user_unlock_ast(struct ocfs2_dlm_lksb *lksb, int status) +{ + struct user_lock_res *lockres = user_lksb_to_lock_res(lksb); + + mlog(ML_BASTS, "UNLOCK AST fired for lockres %.*s, flags 0x%x\n", + lockres->l_namelen, lockres->l_name, lockres->l_flags); + + if (status) + mlog(ML_ERROR, "dlm returns status %d\n", status); + + spin_lock(&lockres->l_lock); + /* The teardown flag gets set early during the unlock process, + * so test the cancel flag to make sure that this ast isn't + * for a concurrent cancel. */ + if (lockres->l_flags & USER_LOCK_IN_TEARDOWN + && !(lockres->l_flags & USER_LOCK_IN_CANCEL)) { + lockres->l_level = DLM_LOCK_IV; + } else if (status == DLM_CANCELGRANT) { + /* We tried to cancel a convert request, but it was + * already granted. Don't clear the busy flag - the + * ast should've done this already. */ + BUG_ON(!(lockres->l_flags & USER_LOCK_IN_CANCEL)); + lockres->l_flags &= ~USER_LOCK_IN_CANCEL; + goto out_noclear; + } else { + BUG_ON(!(lockres->l_flags & USER_LOCK_IN_CANCEL)); + /* Cancel succeeded, we want to re-queue */ + lockres->l_requested = DLM_LOCK_IV; /* cancel an + * upconvert + * request. */ + lockres->l_flags &= ~USER_LOCK_IN_CANCEL; + /* we want the unblock thread to look at it again + * now. */ + if (lockres->l_flags & USER_LOCK_BLOCKED) + __user_dlm_queue_lockres(lockres); + } + + lockres->l_flags &= ~USER_LOCK_BUSY; +out_noclear: + spin_unlock(&lockres->l_lock); + + wake_up(&lockres->l_event); +} + +/* + * This is the userdlmfs locking protocol version. + * + * See fs/ocfs2/dlmglue.c for more details on locking versions. + */ +static struct ocfs2_locking_protocol user_dlm_lproto = { + .lp_max_version = { + .pv_major = OCFS2_LOCKING_PROTOCOL_MAJOR, + .pv_minor = OCFS2_LOCKING_PROTOCOL_MINOR, + }, + .lp_lock_ast = user_ast, + .lp_blocking_ast = user_bast, + .lp_unlock_ast = user_unlock_ast, +}; + +static inline void user_dlm_drop_inode_ref(struct user_lock_res *lockres) +{ + struct inode *inode; + inode = user_dlm_inode_from_user_lockres(lockres); + iput(inode); +} + +static void user_dlm_unblock_lock(struct work_struct *work) +{ + int new_level, status; + struct user_lock_res *lockres = + container_of(work, struct user_lock_res, l_work); + struct ocfs2_cluster_connection *conn = + cluster_connection_from_user_lockres(lockres); + + mlog(0, "lockres %.*s\n", lockres->l_namelen, lockres->l_name); + + spin_lock(&lockres->l_lock); + + mlog_bug_on_msg(!(lockres->l_flags & USER_LOCK_QUEUED), + "Lockres %.*s, flags 0x%x\n", + lockres->l_namelen, lockres->l_name, lockres->l_flags); + + /* notice that we don't clear USER_LOCK_BLOCKED here. If it's + * set, we want user_ast clear it. */ + lockres->l_flags &= ~USER_LOCK_QUEUED; + + /* It's valid to get here and no longer be blocked - if we get + * several basts in a row, we might be queued by the first + * one, the unblock thread might run and clear the queued + * flag, and finally we might get another bast which re-queues + * us before our ast for the downconvert is called. */ + if (!(lockres->l_flags & USER_LOCK_BLOCKED)) { + mlog(ML_BASTS, "lockres %.*s USER_LOCK_BLOCKED\n", + lockres->l_namelen, lockres->l_name); + spin_unlock(&lockres->l_lock); + goto drop_ref; + } + + if (lockres->l_flags & USER_LOCK_IN_TEARDOWN) { + mlog(ML_BASTS, "lockres %.*s USER_LOCK_IN_TEARDOWN\n", + lockres->l_namelen, lockres->l_name); + spin_unlock(&lockres->l_lock); + goto drop_ref; + } + + if (lockres->l_flags & USER_LOCK_BUSY) { + if (lockres->l_flags & USER_LOCK_IN_CANCEL) { + mlog(ML_BASTS, "lockres %.*s USER_LOCK_IN_CANCEL\n", + lockres->l_namelen, lockres->l_name); + spin_unlock(&lockres->l_lock); + goto drop_ref; + } + + lockres->l_flags |= USER_LOCK_IN_CANCEL; + spin_unlock(&lockres->l_lock); + + status = ocfs2_dlm_unlock(conn, &lockres->l_lksb, + DLM_LKF_CANCEL); + if (status) + user_log_dlm_error("ocfs2_dlm_unlock", status, lockres); + goto drop_ref; + } + + /* If there are still incompat holders, we can exit safely + * without worrying about re-queueing this lock as that will + * happen on the last call to user_cluster_unlock. */ + if ((lockres->l_blocking == DLM_LOCK_EX) + && (lockres->l_ex_holders || lockres->l_ro_holders)) { + spin_unlock(&lockres->l_lock); + mlog(ML_BASTS, "lockres %.*s, EX/PR Holders %u,%u\n", + lockres->l_namelen, lockres->l_name, + lockres->l_ex_holders, lockres->l_ro_holders); + goto drop_ref; + } + + if ((lockres->l_blocking == DLM_LOCK_PR) + && lockres->l_ex_holders) { + spin_unlock(&lockres->l_lock); + mlog(ML_BASTS, "lockres %.*s, EX Holders %u\n", + lockres->l_namelen, lockres->l_name, + lockres->l_ex_holders); + goto drop_ref; + } + + /* yay, we can downconvert now. */ + new_level = user_highest_compat_lock_level(lockres->l_blocking); + lockres->l_requested = new_level; + lockres->l_flags |= USER_LOCK_BUSY; + mlog(ML_BASTS, "lockres %.*s, downconvert %d => %d\n", + lockres->l_namelen, lockres->l_name, lockres->l_level, new_level); + spin_unlock(&lockres->l_lock); + + /* need lock downconvert request now... */ + status = ocfs2_dlm_lock(conn, new_level, &lockres->l_lksb, + DLM_LKF_CONVERT|DLM_LKF_VALBLK, + lockres->l_name, + lockres->l_namelen); + if (status) { + user_log_dlm_error("ocfs2_dlm_lock", status, lockres); + user_recover_from_dlm_error(lockres); + } + +drop_ref: + user_dlm_drop_inode_ref(lockres); +} + +static inline void user_dlm_inc_holders(struct user_lock_res *lockres, + int level) +{ + switch(level) { + case DLM_LOCK_EX: + lockres->l_ex_holders++; + break; + case DLM_LOCK_PR: + lockres->l_ro_holders++; + break; + default: + BUG(); + } +} + +/* predict what lock level we'll be dropping down to on behalf + * of another node, and return true if the currently wanted + * level will be compatible with it. */ +static inline int +user_may_continue_on_blocked_lock(struct user_lock_res *lockres, + int wanted) +{ + BUG_ON(!(lockres->l_flags & USER_LOCK_BLOCKED)); + + return wanted <= user_highest_compat_lock_level(lockres->l_blocking); +} + +int user_dlm_cluster_lock(struct user_lock_res *lockres, + int level, + int lkm_flags) +{ + int status, local_flags; + struct ocfs2_cluster_connection *conn = + cluster_connection_from_user_lockres(lockres); + + if (level != DLM_LOCK_EX && + level != DLM_LOCK_PR) { + mlog(ML_ERROR, "lockres %.*s: invalid request!\n", + lockres->l_namelen, lockres->l_name); + status = -EINVAL; + goto bail; + } + + mlog(ML_BASTS, "lockres %.*s, level %d, flags = 0x%x\n", + lockres->l_namelen, lockres->l_name, level, lkm_flags); + +again: + if (signal_pending(current)) { + status = -ERESTARTSYS; + goto bail; + } + + spin_lock(&lockres->l_lock); + if (lockres->l_flags & USER_LOCK_IN_TEARDOWN) { + spin_unlock(&lockres->l_lock); + status = -EAGAIN; + goto bail; + } + + /* We only compare against the currently granted level + * here. If the lock is blocked waiting on a downconvert, + * we'll get caught below. */ + if ((lockres->l_flags & USER_LOCK_BUSY) && + (level > lockres->l_level)) { + /* is someone sitting in dlm_lock? If so, wait on + * them. */ + spin_unlock(&lockres->l_lock); + + user_wait_on_busy_lock(lockres); + goto again; + } + + if ((lockres->l_flags & USER_LOCK_BLOCKED) && + (!user_may_continue_on_blocked_lock(lockres, level))) { + /* is the lock is currently blocked on behalf of + * another node */ + spin_unlock(&lockres->l_lock); + + user_wait_on_blocked_lock(lockres); + goto again; + } + + if (level > lockres->l_level) { + local_flags = lkm_flags | DLM_LKF_VALBLK; + if (lockres->l_level != DLM_LOCK_IV) + local_flags |= DLM_LKF_CONVERT; + + lockres->l_requested = level; + lockres->l_flags |= USER_LOCK_BUSY; + spin_unlock(&lockres->l_lock); + + BUG_ON(level == DLM_LOCK_IV); + BUG_ON(level == DLM_LOCK_NL); + + /* call dlm_lock to upgrade lock now */ + status = ocfs2_dlm_lock(conn, level, &lockres->l_lksb, + local_flags, lockres->l_name, + lockres->l_namelen); + if (status) { + if ((lkm_flags & DLM_LKF_NOQUEUE) && + (status != -EAGAIN)) + user_log_dlm_error("ocfs2_dlm_lock", + status, lockres); + user_recover_from_dlm_error(lockres); + goto bail; + } + + user_wait_on_busy_lock(lockres); + goto again; + } + + user_dlm_inc_holders(lockres, level); + spin_unlock(&lockres->l_lock); + + status = 0; +bail: + return status; +} + +static inline void user_dlm_dec_holders(struct user_lock_res *lockres, + int level) +{ + switch(level) { + case DLM_LOCK_EX: + BUG_ON(!lockres->l_ex_holders); + lockres->l_ex_holders--; + break; + case DLM_LOCK_PR: + BUG_ON(!lockres->l_ro_holders); + lockres->l_ro_holders--; + break; + default: + BUG(); + } +} + +void user_dlm_cluster_unlock(struct user_lock_res *lockres, + int level) +{ + if (level != DLM_LOCK_EX && + level != DLM_LOCK_PR) { + mlog(ML_ERROR, "lockres %.*s: invalid request!\n", + lockres->l_namelen, lockres->l_name); + return; + } + + spin_lock(&lockres->l_lock); + user_dlm_dec_holders(lockres, level); + __user_dlm_cond_queue_lockres(lockres); + spin_unlock(&lockres->l_lock); +} + +void user_dlm_write_lvb(struct inode *inode, + const char *val, + unsigned int len) +{ + struct user_lock_res *lockres = &DLMFS_I(inode)->ip_lockres; + char *lvb; + + BUG_ON(len > DLM_LVB_LEN); + + spin_lock(&lockres->l_lock); + + BUG_ON(lockres->l_level < DLM_LOCK_EX); + lvb = ocfs2_dlm_lvb(&lockres->l_lksb); + memcpy(lvb, val, len); + + spin_unlock(&lockres->l_lock); +} + +bool user_dlm_read_lvb(struct inode *inode, char *val) +{ + struct user_lock_res *lockres = &DLMFS_I(inode)->ip_lockres; + char *lvb; + bool ret = true; + + spin_lock(&lockres->l_lock); + + BUG_ON(lockres->l_level < DLM_LOCK_PR); + if (ocfs2_dlm_lvb_valid(&lockres->l_lksb)) { + lvb = ocfs2_dlm_lvb(&lockres->l_lksb); + memcpy(val, lvb, DLM_LVB_LEN); + } else + ret = false; + + spin_unlock(&lockres->l_lock); + return ret; +} + +void user_dlm_lock_res_init(struct user_lock_res *lockres, + struct dentry *dentry) +{ + memset(lockres, 0, sizeof(*lockres)); + + spin_lock_init(&lockres->l_lock); + init_waitqueue_head(&lockres->l_event); + lockres->l_level = DLM_LOCK_IV; + lockres->l_requested = DLM_LOCK_IV; + lockres->l_blocking = DLM_LOCK_IV; + + /* should have been checked before getting here. */ + BUG_ON(dentry->d_name.len >= USER_DLM_LOCK_ID_MAX_LEN); + + memcpy(lockres->l_name, + dentry->d_name.name, + dentry->d_name.len); + lockres->l_namelen = dentry->d_name.len; +} + +int user_dlm_destroy_lock(struct user_lock_res *lockres) +{ + int status = -EBUSY; + struct ocfs2_cluster_connection *conn = + cluster_connection_from_user_lockres(lockres); + + mlog(ML_BASTS, "lockres %.*s\n", lockres->l_namelen, lockres->l_name); + + spin_lock(&lockres->l_lock); + if (lockres->l_flags & USER_LOCK_IN_TEARDOWN) { + spin_unlock(&lockres->l_lock); + goto bail; + } + + lockres->l_flags |= USER_LOCK_IN_TEARDOWN; + + while (lockres->l_flags & USER_LOCK_BUSY) { + spin_unlock(&lockres->l_lock); + + user_wait_on_busy_lock(lockres); + + spin_lock(&lockres->l_lock); + } + + if (lockres->l_ro_holders || lockres->l_ex_holders) { + lockres->l_flags &= ~USER_LOCK_IN_TEARDOWN; + spin_unlock(&lockres->l_lock); + goto bail; + } + + status = 0; + if (!(lockres->l_flags & USER_LOCK_ATTACHED)) { + /* + * lock is never requested, leave USER_LOCK_IN_TEARDOWN set + * to avoid new lock request coming in. + */ + spin_unlock(&lockres->l_lock); + goto bail; + } + + lockres->l_flags |= USER_LOCK_BUSY; + spin_unlock(&lockres->l_lock); + + status = ocfs2_dlm_unlock(conn, &lockres->l_lksb, DLM_LKF_VALBLK); + if (status) { + spin_lock(&lockres->l_lock); + lockres->l_flags &= ~USER_LOCK_IN_TEARDOWN; + lockres->l_flags &= ~USER_LOCK_BUSY; + spin_unlock(&lockres->l_lock); + user_log_dlm_error("ocfs2_dlm_unlock", status, lockres); + goto bail; + } + + user_wait_on_busy_lock(lockres); + + status = 0; +bail: + return status; +} + +static void user_dlm_recovery_handler_noop(int node_num, + void *recovery_data) +{ + /* We ignore recovery events */ + return; +} + +void user_dlm_set_locking_protocol(void) +{ + ocfs2_stack_glue_set_max_proto_version(&user_dlm_lproto.lp_max_version); +} + +struct ocfs2_cluster_connection *user_dlm_register(const struct qstr *name) +{ + int rc; + struct ocfs2_cluster_connection *conn; + + rc = ocfs2_cluster_connect_agnostic(name->name, name->len, + &user_dlm_lproto, + user_dlm_recovery_handler_noop, + NULL, &conn); + if (rc) + mlog_errno(rc); + + return rc ? ERR_PTR(rc) : conn; +} + +void user_dlm_unregister(struct ocfs2_cluster_connection *conn) +{ + ocfs2_cluster_disconnect(conn, 0); +} diff --git a/fs/ocfs2/dlmfs/userdlm.h b/fs/ocfs2/dlmfs/userdlm.h new file mode 100644 index 000000000..47ba18eac --- /dev/null +++ b/fs/ocfs2/dlmfs/userdlm.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * userdlm.h + * + * Userspace dlm defines + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + + +#ifndef USERDLM_H +#define USERDLM_H + +#include +#include +#include +#include + +/* user_lock_res->l_flags flags. */ +#define USER_LOCK_ATTACHED (0x00000001) /* we have initialized + * the lvb */ +#define USER_LOCK_BUSY (0x00000002) /* we are currently in + * dlm_lock */ +#define USER_LOCK_BLOCKED (0x00000004) /* blocked waiting to + * downconvert*/ +#define USER_LOCK_IN_TEARDOWN (0x00000008) /* we're currently + * destroying this + * lock. */ +#define USER_LOCK_QUEUED (0x00000010) /* lock is on the + * workqueue */ +#define USER_LOCK_IN_CANCEL (0x00000020) + +struct user_lock_res { + spinlock_t l_lock; + + int l_flags; + +#define USER_DLM_LOCK_ID_MAX_LEN 32 + char l_name[USER_DLM_LOCK_ID_MAX_LEN]; + int l_namelen; + int l_level; + unsigned int l_ro_holders; + unsigned int l_ex_holders; + struct ocfs2_dlm_lksb l_lksb; + + int l_requested; + int l_blocking; + + wait_queue_head_t l_event; + + struct work_struct l_work; +}; + +extern struct workqueue_struct *user_dlm_worker; + +void user_dlm_lock_res_init(struct user_lock_res *lockres, + struct dentry *dentry); +int user_dlm_destroy_lock(struct user_lock_res *lockres); +int user_dlm_cluster_lock(struct user_lock_res *lockres, + int level, + int lkm_flags); +void user_dlm_cluster_unlock(struct user_lock_res *lockres, + int level); +void user_dlm_write_lvb(struct inode *inode, + const char *val, + unsigned int len); +bool user_dlm_read_lvb(struct inode *inode, char *val); +struct ocfs2_cluster_connection *user_dlm_register(const struct qstr *name); +void user_dlm_unregister(struct ocfs2_cluster_connection *conn); +void user_dlm_set_locking_protocol(void); + +struct dlmfs_inode_private { + struct ocfs2_cluster_connection *ip_conn; + + struct user_lock_res ip_lockres; /* unused for directories. */ + struct inode *ip_parent; + + struct inode ip_vfs_inode; +}; + +static inline struct dlmfs_inode_private * +DLMFS_I(struct inode *inode) +{ + return container_of(inode, + struct dlmfs_inode_private, + ip_vfs_inode); +} + +struct dlmfs_filp_private { + int fp_lock_level; +}; + +#define DLMFS_MAGIC 0x76a9f425 + +#endif /* USERDLM_H */ diff --git a/fs/ocfs2/dlmglue.c b/fs/ocfs2/dlmglue.c new file mode 100644 index 000000000..a23dd8f86 --- /dev/null +++ b/fs/ocfs2/dlmglue.c @@ -0,0 +1,4485 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * dlmglue.c + * + * Code which implements an OCFS2 specific interface to our DLM. + * + * Copyright (C) 2003, 2004 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MLOG_MASK_PREFIX ML_DLM_GLUE +#include + +#include "ocfs2.h" +#include "ocfs2_lockingver.h" + +#include "alloc.h" +#include "dcache.h" +#include "dlmglue.h" +#include "extent_map.h" +#include "file.h" +#include "heartbeat.h" +#include "inode.h" +#include "journal.h" +#include "stackglue.h" +#include "slot_map.h" +#include "super.h" +#include "uptodate.h" +#include "quota.h" +#include "refcounttree.h" +#include "acl.h" + +#include "buffer_head_io.h" + +struct ocfs2_mask_waiter { + struct list_head mw_item; + int mw_status; + struct completion mw_complete; + unsigned long mw_mask; + unsigned long mw_goal; +#ifdef CONFIG_OCFS2_FS_STATS + ktime_t mw_lock_start; +#endif +}; + +static struct ocfs2_super *ocfs2_get_dentry_osb(struct ocfs2_lock_res *lockres); +static struct ocfs2_super *ocfs2_get_inode_osb(struct ocfs2_lock_res *lockres); +static struct ocfs2_super *ocfs2_get_file_osb(struct ocfs2_lock_res *lockres); +static struct ocfs2_super *ocfs2_get_qinfo_osb(struct ocfs2_lock_res *lockres); + +/* + * Return value from ->downconvert_worker functions. + * + * These control the precise actions of ocfs2_unblock_lock() + * and ocfs2_process_blocked_lock() + * + */ +enum ocfs2_unblock_action { + UNBLOCK_CONTINUE = 0, /* Continue downconvert */ + UNBLOCK_CONTINUE_POST = 1, /* Continue downconvert, fire + * ->post_unlock callback */ + UNBLOCK_STOP_POST = 2, /* Do not downconvert, fire + * ->post_unlock() callback. */ +}; + +struct ocfs2_unblock_ctl { + int requeue; + enum ocfs2_unblock_action unblock_action; +}; + +/* Lockdep class keys */ +#ifdef CONFIG_DEBUG_LOCK_ALLOC +static struct lock_class_key lockdep_keys[OCFS2_NUM_LOCK_TYPES]; +#endif + +static int ocfs2_check_meta_downconvert(struct ocfs2_lock_res *lockres, + int new_level); +static void ocfs2_set_meta_lvb(struct ocfs2_lock_res *lockres); + +static int ocfs2_data_convert_worker(struct ocfs2_lock_res *lockres, + int blocking); + +static int ocfs2_dentry_convert_worker(struct ocfs2_lock_res *lockres, + int blocking); + +static void ocfs2_dentry_post_unlock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres); + +static void ocfs2_set_qinfo_lvb(struct ocfs2_lock_res *lockres); + +static int ocfs2_check_refcount_downconvert(struct ocfs2_lock_res *lockres, + int new_level); +static int ocfs2_refcount_convert_worker(struct ocfs2_lock_res *lockres, + int blocking); + +#define mlog_meta_lvb(__level, __lockres) ocfs2_dump_meta_lvb_info(__level, __PRETTY_FUNCTION__, __LINE__, __lockres) + +/* This aids in debugging situations where a bad LVB might be involved. */ +static void ocfs2_dump_meta_lvb_info(u64 level, + const char *function, + unsigned int line, + struct ocfs2_lock_res *lockres) +{ + struct ocfs2_meta_lvb *lvb = ocfs2_dlm_lvb(&lockres->l_lksb); + + mlog(level, "LVB information for %s (called from %s:%u):\n", + lockres->l_name, function, line); + mlog(level, "version: %u, clusters: %u, generation: 0x%x\n", + lvb->lvb_version, be32_to_cpu(lvb->lvb_iclusters), + be32_to_cpu(lvb->lvb_igeneration)); + mlog(level, "size: %llu, uid %u, gid %u, mode 0x%x\n", + (unsigned long long)be64_to_cpu(lvb->lvb_isize), + be32_to_cpu(lvb->lvb_iuid), be32_to_cpu(lvb->lvb_igid), + be16_to_cpu(lvb->lvb_imode)); + mlog(level, "nlink %u, atime_packed 0x%llx, ctime_packed 0x%llx, " + "mtime_packed 0x%llx iattr 0x%x\n", be16_to_cpu(lvb->lvb_inlink), + (long long)be64_to_cpu(lvb->lvb_iatime_packed), + (long long)be64_to_cpu(lvb->lvb_ictime_packed), + (long long)be64_to_cpu(lvb->lvb_imtime_packed), + be32_to_cpu(lvb->lvb_iattr)); +} + + +/* + * OCFS2 Lock Resource Operations + * + * These fine tune the behavior of the generic dlmglue locking infrastructure. + * + * The most basic of lock types can point ->l_priv to their respective + * struct ocfs2_super and allow the default actions to manage things. + * + * Right now, each lock type also needs to implement an init function, + * and trivial lock/unlock wrappers. ocfs2_simple_drop_lockres() + * should be called when the lock is no longer needed (i.e., object + * destruction time). + */ +struct ocfs2_lock_res_ops { + /* + * Translate an ocfs2_lock_res * into an ocfs2_super *. Define + * this callback if ->l_priv is not an ocfs2_super pointer + */ + struct ocfs2_super * (*get_osb)(struct ocfs2_lock_res *); + + /* + * Optionally called in the downconvert thread after a + * successful downconvert. The lockres will not be referenced + * after this callback is called, so it is safe to free + * memory, etc. + * + * The exact semantics of when this is called are controlled + * by ->downconvert_worker() + */ + void (*post_unlock)(struct ocfs2_super *, struct ocfs2_lock_res *); + + /* + * Allow a lock type to add checks to determine whether it is + * safe to downconvert a lock. Return 0 to re-queue the + * downconvert at a later time, nonzero to continue. + * + * For most locks, the default checks that there are no + * incompatible holders are sufficient. + * + * Called with the lockres spinlock held. + */ + int (*check_downconvert)(struct ocfs2_lock_res *, int); + + /* + * Allows a lock type to populate the lock value block. This + * is called on downconvert, and when we drop a lock. + * + * Locks that want to use this should set LOCK_TYPE_USES_LVB + * in the flags field. + * + * Called with the lockres spinlock held. + */ + void (*set_lvb)(struct ocfs2_lock_res *); + + /* + * Called from the downconvert thread when it is determined + * that a lock will be downconverted. This is called without + * any locks held so the function can do work that might + * schedule (syncing out data, etc). + * + * This should return any one of the ocfs2_unblock_action + * values, depending on what it wants the thread to do. + */ + int (*downconvert_worker)(struct ocfs2_lock_res *, int); + + /* + * LOCK_TYPE_* flags which describe the specific requirements + * of a lock type. Descriptions of each individual flag follow. + */ + int flags; +}; + +/* + * Some locks want to "refresh" potentially stale data when a + * meaningful (PRMODE or EXMODE) lock level is first obtained. If this + * flag is set, the OCFS2_LOCK_NEEDS_REFRESH flag will be set on the + * individual lockres l_flags member from the ast function. It is + * expected that the locking wrapper will clear the + * OCFS2_LOCK_NEEDS_REFRESH flag when done. + */ +#define LOCK_TYPE_REQUIRES_REFRESH 0x1 + +/* + * Indicate that a lock type makes use of the lock value block. The + * ->set_lvb lock type callback must be defined. + */ +#define LOCK_TYPE_USES_LVB 0x2 + +static const struct ocfs2_lock_res_ops ocfs2_inode_rw_lops = { + .get_osb = ocfs2_get_inode_osb, + .flags = 0, +}; + +static const struct ocfs2_lock_res_ops ocfs2_inode_inode_lops = { + .get_osb = ocfs2_get_inode_osb, + .check_downconvert = ocfs2_check_meta_downconvert, + .set_lvb = ocfs2_set_meta_lvb, + .downconvert_worker = ocfs2_data_convert_worker, + .flags = LOCK_TYPE_REQUIRES_REFRESH|LOCK_TYPE_USES_LVB, +}; + +static const struct ocfs2_lock_res_ops ocfs2_super_lops = { + .flags = LOCK_TYPE_REQUIRES_REFRESH, +}; + +static const struct ocfs2_lock_res_ops ocfs2_rename_lops = { + .flags = 0, +}; + +static const struct ocfs2_lock_res_ops ocfs2_nfs_sync_lops = { + .flags = 0, +}; + +static const struct ocfs2_lock_res_ops ocfs2_trim_fs_lops = { + .flags = LOCK_TYPE_REQUIRES_REFRESH|LOCK_TYPE_USES_LVB, +}; + +static const struct ocfs2_lock_res_ops ocfs2_orphan_scan_lops = { + .flags = LOCK_TYPE_REQUIRES_REFRESH|LOCK_TYPE_USES_LVB, +}; + +static const struct ocfs2_lock_res_ops ocfs2_dentry_lops = { + .get_osb = ocfs2_get_dentry_osb, + .post_unlock = ocfs2_dentry_post_unlock, + .downconvert_worker = ocfs2_dentry_convert_worker, + .flags = 0, +}; + +static const struct ocfs2_lock_res_ops ocfs2_inode_open_lops = { + .get_osb = ocfs2_get_inode_osb, + .flags = 0, +}; + +static const struct ocfs2_lock_res_ops ocfs2_flock_lops = { + .get_osb = ocfs2_get_file_osb, + .flags = 0, +}; + +static const struct ocfs2_lock_res_ops ocfs2_qinfo_lops = { + .set_lvb = ocfs2_set_qinfo_lvb, + .get_osb = ocfs2_get_qinfo_osb, + .flags = LOCK_TYPE_REQUIRES_REFRESH | LOCK_TYPE_USES_LVB, +}; + +static const struct ocfs2_lock_res_ops ocfs2_refcount_block_lops = { + .check_downconvert = ocfs2_check_refcount_downconvert, + .downconvert_worker = ocfs2_refcount_convert_worker, + .flags = 0, +}; + +static inline int ocfs2_is_inode_lock(struct ocfs2_lock_res *lockres) +{ + return lockres->l_type == OCFS2_LOCK_TYPE_META || + lockres->l_type == OCFS2_LOCK_TYPE_RW || + lockres->l_type == OCFS2_LOCK_TYPE_OPEN; +} + +static inline struct ocfs2_lock_res *ocfs2_lksb_to_lock_res(struct ocfs2_dlm_lksb *lksb) +{ + return container_of(lksb, struct ocfs2_lock_res, l_lksb); +} + +static inline struct inode *ocfs2_lock_res_inode(struct ocfs2_lock_res *lockres) +{ + BUG_ON(!ocfs2_is_inode_lock(lockres)); + + return (struct inode *) lockres->l_priv; +} + +static inline struct ocfs2_dentry_lock *ocfs2_lock_res_dl(struct ocfs2_lock_res *lockres) +{ + BUG_ON(lockres->l_type != OCFS2_LOCK_TYPE_DENTRY); + + return (struct ocfs2_dentry_lock *)lockres->l_priv; +} + +static inline struct ocfs2_mem_dqinfo *ocfs2_lock_res_qinfo(struct ocfs2_lock_res *lockres) +{ + BUG_ON(lockres->l_type != OCFS2_LOCK_TYPE_QINFO); + + return (struct ocfs2_mem_dqinfo *)lockres->l_priv; +} + +static inline struct ocfs2_refcount_tree * +ocfs2_lock_res_refcount_tree(struct ocfs2_lock_res *res) +{ + return container_of(res, struct ocfs2_refcount_tree, rf_lockres); +} + +static inline struct ocfs2_super *ocfs2_get_lockres_osb(struct ocfs2_lock_res *lockres) +{ + if (lockres->l_ops->get_osb) + return lockres->l_ops->get_osb(lockres); + + return (struct ocfs2_super *)lockres->l_priv; +} + +static int ocfs2_lock_create(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres, + int level, + u32 dlm_flags); +static inline int ocfs2_may_continue_on_blocked_lock(struct ocfs2_lock_res *lockres, + int wanted); +static void __ocfs2_cluster_unlock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres, + int level, unsigned long caller_ip); +static inline void ocfs2_cluster_unlock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres, + int level) +{ + __ocfs2_cluster_unlock(osb, lockres, level, _RET_IP_); +} + +static inline void ocfs2_generic_handle_downconvert_action(struct ocfs2_lock_res *lockres); +static inline void ocfs2_generic_handle_convert_action(struct ocfs2_lock_res *lockres); +static inline void ocfs2_generic_handle_attach_action(struct ocfs2_lock_res *lockres); +static int ocfs2_generic_handle_bast(struct ocfs2_lock_res *lockres, int level); +static void ocfs2_schedule_blocked_lock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres); +static inline void ocfs2_recover_from_dlm_error(struct ocfs2_lock_res *lockres, + int convert); +#define ocfs2_log_dlm_error(_func, _err, _lockres) do { \ + if ((_lockres)->l_type != OCFS2_LOCK_TYPE_DENTRY) \ + mlog(ML_ERROR, "DLM error %d while calling %s on resource %s\n", \ + _err, _func, _lockres->l_name); \ + else \ + mlog(ML_ERROR, "DLM error %d while calling %s on resource %.*s%08x\n", \ + _err, _func, OCFS2_DENTRY_LOCK_INO_START - 1, (_lockres)->l_name, \ + (unsigned int)ocfs2_get_dentry_lock_ino(_lockres)); \ +} while (0) +static int ocfs2_downconvert_thread(void *arg); +static void ocfs2_downconvert_on_unlock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres); +static int ocfs2_inode_lock_update(struct inode *inode, + struct buffer_head **bh); +static void ocfs2_drop_osb_locks(struct ocfs2_super *osb); +static inline int ocfs2_highest_compat_lock_level(int level); +static unsigned int ocfs2_prepare_downconvert(struct ocfs2_lock_res *lockres, + int new_level); +static int ocfs2_downconvert_lock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres, + int new_level, + int lvb, + unsigned int generation); +static int ocfs2_prepare_cancel_convert(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres); +static int ocfs2_cancel_convert(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres); + + +static void ocfs2_build_lock_name(enum ocfs2_lock_type type, + u64 blkno, + u32 generation, + char *name) +{ + int len; + + BUG_ON(type >= OCFS2_NUM_LOCK_TYPES); + + len = snprintf(name, OCFS2_LOCK_ID_MAX_LEN, "%c%s%016llx%08x", + ocfs2_lock_type_char(type), OCFS2_LOCK_ID_PAD, + (long long)blkno, generation); + + BUG_ON(len != (OCFS2_LOCK_ID_MAX_LEN - 1)); + + mlog(0, "built lock resource with name: %s\n", name); +} + +static DEFINE_SPINLOCK(ocfs2_dlm_tracking_lock); + +static void ocfs2_add_lockres_tracking(struct ocfs2_lock_res *res, + struct ocfs2_dlm_debug *dlm_debug) +{ + mlog(0, "Add tracking for lockres %s\n", res->l_name); + + spin_lock(&ocfs2_dlm_tracking_lock); + list_add(&res->l_debug_list, &dlm_debug->d_lockres_tracking); + spin_unlock(&ocfs2_dlm_tracking_lock); +} + +static void ocfs2_remove_lockres_tracking(struct ocfs2_lock_res *res) +{ + spin_lock(&ocfs2_dlm_tracking_lock); + if (!list_empty(&res->l_debug_list)) + list_del_init(&res->l_debug_list); + spin_unlock(&ocfs2_dlm_tracking_lock); +} + +#ifdef CONFIG_OCFS2_FS_STATS +static void ocfs2_init_lock_stats(struct ocfs2_lock_res *res) +{ + res->l_lock_refresh = 0; + res->l_lock_wait = 0; + memset(&res->l_lock_prmode, 0, sizeof(struct ocfs2_lock_stats)); + memset(&res->l_lock_exmode, 0, sizeof(struct ocfs2_lock_stats)); +} + +static void ocfs2_update_lock_stats(struct ocfs2_lock_res *res, int level, + struct ocfs2_mask_waiter *mw, int ret) +{ + u32 usec; + ktime_t kt; + struct ocfs2_lock_stats *stats; + + if (level == LKM_PRMODE) + stats = &res->l_lock_prmode; + else if (level == LKM_EXMODE) + stats = &res->l_lock_exmode; + else + return; + + kt = ktime_sub(ktime_get(), mw->mw_lock_start); + usec = ktime_to_us(kt); + + stats->ls_gets++; + stats->ls_total += ktime_to_ns(kt); + /* overflow */ + if (unlikely(stats->ls_gets == 0)) { + stats->ls_gets++; + stats->ls_total = ktime_to_ns(kt); + } + + if (stats->ls_max < usec) + stats->ls_max = usec; + + if (ret) + stats->ls_fail++; + + stats->ls_last = ktime_to_us(ktime_get_real()); +} + +static inline void ocfs2_track_lock_refresh(struct ocfs2_lock_res *lockres) +{ + lockres->l_lock_refresh++; +} + +static inline void ocfs2_track_lock_wait(struct ocfs2_lock_res *lockres) +{ + struct ocfs2_mask_waiter *mw; + + if (list_empty(&lockres->l_mask_waiters)) { + lockres->l_lock_wait = 0; + return; + } + + mw = list_first_entry(&lockres->l_mask_waiters, + struct ocfs2_mask_waiter, mw_item); + lockres->l_lock_wait = + ktime_to_us(ktime_mono_to_real(mw->mw_lock_start)); +} + +static inline void ocfs2_init_start_time(struct ocfs2_mask_waiter *mw) +{ + mw->mw_lock_start = ktime_get(); +} +#else +static inline void ocfs2_init_lock_stats(struct ocfs2_lock_res *res) +{ +} +static inline void ocfs2_update_lock_stats(struct ocfs2_lock_res *res, + int level, struct ocfs2_mask_waiter *mw, int ret) +{ +} +static inline void ocfs2_track_lock_refresh(struct ocfs2_lock_res *lockres) +{ +} +static inline void ocfs2_track_lock_wait(struct ocfs2_lock_res *lockres) +{ +} +static inline void ocfs2_init_start_time(struct ocfs2_mask_waiter *mw) +{ +} +#endif + +static void ocfs2_lock_res_init_common(struct ocfs2_super *osb, + struct ocfs2_lock_res *res, + enum ocfs2_lock_type type, + const struct ocfs2_lock_res_ops *ops, + void *priv) +{ + res->l_type = type; + res->l_ops = ops; + res->l_priv = priv; + + res->l_level = DLM_LOCK_IV; + res->l_requested = DLM_LOCK_IV; + res->l_blocking = DLM_LOCK_IV; + res->l_action = OCFS2_AST_INVALID; + res->l_unlock_action = OCFS2_UNLOCK_INVALID; + + res->l_flags = OCFS2_LOCK_INITIALIZED; + + ocfs2_add_lockres_tracking(res, osb->osb_dlm_debug); + + ocfs2_init_lock_stats(res); +#ifdef CONFIG_DEBUG_LOCK_ALLOC + if (type != OCFS2_LOCK_TYPE_OPEN) + lockdep_init_map(&res->l_lockdep_map, ocfs2_lock_type_strings[type], + &lockdep_keys[type], 0); + else + res->l_lockdep_map.key = NULL; +#endif +} + +void ocfs2_lock_res_init_once(struct ocfs2_lock_res *res) +{ + /* This also clears out the lock status block */ + memset(res, 0, sizeof(struct ocfs2_lock_res)); + spin_lock_init(&res->l_lock); + init_waitqueue_head(&res->l_event); + INIT_LIST_HEAD(&res->l_blocked_list); + INIT_LIST_HEAD(&res->l_mask_waiters); + INIT_LIST_HEAD(&res->l_holders); +} + +void ocfs2_inode_lock_res_init(struct ocfs2_lock_res *res, + enum ocfs2_lock_type type, + unsigned int generation, + struct inode *inode) +{ + const struct ocfs2_lock_res_ops *ops; + + switch(type) { + case OCFS2_LOCK_TYPE_RW: + ops = &ocfs2_inode_rw_lops; + break; + case OCFS2_LOCK_TYPE_META: + ops = &ocfs2_inode_inode_lops; + break; + case OCFS2_LOCK_TYPE_OPEN: + ops = &ocfs2_inode_open_lops; + break; + default: + mlog_bug_on_msg(1, "type: %d\n", type); + ops = NULL; /* thanks, gcc */ + break; + } + + ocfs2_build_lock_name(type, OCFS2_I(inode)->ip_blkno, + generation, res->l_name); + ocfs2_lock_res_init_common(OCFS2_SB(inode->i_sb), res, type, ops, inode); +} + +static struct ocfs2_super *ocfs2_get_inode_osb(struct ocfs2_lock_res *lockres) +{ + struct inode *inode = ocfs2_lock_res_inode(lockres); + + return OCFS2_SB(inode->i_sb); +} + +static struct ocfs2_super *ocfs2_get_qinfo_osb(struct ocfs2_lock_res *lockres) +{ + struct ocfs2_mem_dqinfo *info = lockres->l_priv; + + return OCFS2_SB(info->dqi_gi.dqi_sb); +} + +static struct ocfs2_super *ocfs2_get_file_osb(struct ocfs2_lock_res *lockres) +{ + struct ocfs2_file_private *fp = lockres->l_priv; + + return OCFS2_SB(fp->fp_file->f_mapping->host->i_sb); +} + +static __u64 ocfs2_get_dentry_lock_ino(struct ocfs2_lock_res *lockres) +{ + __be64 inode_blkno_be; + + memcpy(&inode_blkno_be, &lockres->l_name[OCFS2_DENTRY_LOCK_INO_START], + sizeof(__be64)); + + return be64_to_cpu(inode_blkno_be); +} + +static struct ocfs2_super *ocfs2_get_dentry_osb(struct ocfs2_lock_res *lockres) +{ + struct ocfs2_dentry_lock *dl = lockres->l_priv; + + return OCFS2_SB(dl->dl_inode->i_sb); +} + +void ocfs2_dentry_lock_res_init(struct ocfs2_dentry_lock *dl, + u64 parent, struct inode *inode) +{ + int len; + u64 inode_blkno = OCFS2_I(inode)->ip_blkno; + __be64 inode_blkno_be = cpu_to_be64(inode_blkno); + struct ocfs2_lock_res *lockres = &dl->dl_lockres; + + ocfs2_lock_res_init_once(lockres); + + /* + * Unfortunately, the standard lock naming scheme won't work + * here because we have two 16 byte values to use. Instead, + * we'll stuff the inode number as a binary value. We still + * want error prints to show something without garbling the + * display, so drop a null byte in there before the inode + * number. A future version of OCFS2 will likely use all + * binary lock names. The stringified names have been a + * tremendous aid in debugging, but now that the debugfs + * interface exists, we can mangle things there if need be. + * + * NOTE: We also drop the standard "pad" value (the total lock + * name size stays the same though - the last part is all + * zeros due to the memset in ocfs2_lock_res_init_once() + */ + len = snprintf(lockres->l_name, OCFS2_DENTRY_LOCK_INO_START, + "%c%016llx", + ocfs2_lock_type_char(OCFS2_LOCK_TYPE_DENTRY), + (long long)parent); + + BUG_ON(len != (OCFS2_DENTRY_LOCK_INO_START - 1)); + + memcpy(&lockres->l_name[OCFS2_DENTRY_LOCK_INO_START], &inode_blkno_be, + sizeof(__be64)); + + ocfs2_lock_res_init_common(OCFS2_SB(inode->i_sb), lockres, + OCFS2_LOCK_TYPE_DENTRY, &ocfs2_dentry_lops, + dl); +} + +static void ocfs2_super_lock_res_init(struct ocfs2_lock_res *res, + struct ocfs2_super *osb) +{ + /* Superblock lockres doesn't come from a slab so we call init + * once on it manually. */ + ocfs2_lock_res_init_once(res); + ocfs2_build_lock_name(OCFS2_LOCK_TYPE_SUPER, OCFS2_SUPER_BLOCK_BLKNO, + 0, res->l_name); + ocfs2_lock_res_init_common(osb, res, OCFS2_LOCK_TYPE_SUPER, + &ocfs2_super_lops, osb); +} + +static void ocfs2_rename_lock_res_init(struct ocfs2_lock_res *res, + struct ocfs2_super *osb) +{ + /* Rename lockres doesn't come from a slab so we call init + * once on it manually. */ + ocfs2_lock_res_init_once(res); + ocfs2_build_lock_name(OCFS2_LOCK_TYPE_RENAME, 0, 0, res->l_name); + ocfs2_lock_res_init_common(osb, res, OCFS2_LOCK_TYPE_RENAME, + &ocfs2_rename_lops, osb); +} + +static void ocfs2_nfs_sync_lock_res_init(struct ocfs2_lock_res *res, + struct ocfs2_super *osb) +{ + /* nfs_sync lockres doesn't come from a slab so we call init + * once on it manually. */ + ocfs2_lock_res_init_once(res); + ocfs2_build_lock_name(OCFS2_LOCK_TYPE_NFS_SYNC, 0, 0, res->l_name); + ocfs2_lock_res_init_common(osb, res, OCFS2_LOCK_TYPE_NFS_SYNC, + &ocfs2_nfs_sync_lops, osb); +} + +static void ocfs2_nfs_sync_lock_init(struct ocfs2_super *osb) +{ + ocfs2_nfs_sync_lock_res_init(&osb->osb_nfs_sync_lockres, osb); + init_rwsem(&osb->nfs_sync_rwlock); +} + +void ocfs2_trim_fs_lock_res_init(struct ocfs2_super *osb) +{ + struct ocfs2_lock_res *lockres = &osb->osb_trim_fs_lockres; + + /* Only one trimfs thread are allowed to work at the same time. */ + mutex_lock(&osb->obs_trim_fs_mutex); + + ocfs2_lock_res_init_once(lockres); + ocfs2_build_lock_name(OCFS2_LOCK_TYPE_TRIM_FS, 0, 0, lockres->l_name); + ocfs2_lock_res_init_common(osb, lockres, OCFS2_LOCK_TYPE_TRIM_FS, + &ocfs2_trim_fs_lops, osb); +} + +void ocfs2_trim_fs_lock_res_uninit(struct ocfs2_super *osb) +{ + struct ocfs2_lock_res *lockres = &osb->osb_trim_fs_lockres; + + ocfs2_simple_drop_lockres(osb, lockres); + ocfs2_lock_res_free(lockres); + + mutex_unlock(&osb->obs_trim_fs_mutex); +} + +static void ocfs2_orphan_scan_lock_res_init(struct ocfs2_lock_res *res, + struct ocfs2_super *osb) +{ + ocfs2_lock_res_init_once(res); + ocfs2_build_lock_name(OCFS2_LOCK_TYPE_ORPHAN_SCAN, 0, 0, res->l_name); + ocfs2_lock_res_init_common(osb, res, OCFS2_LOCK_TYPE_ORPHAN_SCAN, + &ocfs2_orphan_scan_lops, osb); +} + +void ocfs2_file_lock_res_init(struct ocfs2_lock_res *lockres, + struct ocfs2_file_private *fp) +{ + struct inode *inode = fp->fp_file->f_mapping->host; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + + ocfs2_lock_res_init_once(lockres); + ocfs2_build_lock_name(OCFS2_LOCK_TYPE_FLOCK, oi->ip_blkno, + inode->i_generation, lockres->l_name); + ocfs2_lock_res_init_common(OCFS2_SB(inode->i_sb), lockres, + OCFS2_LOCK_TYPE_FLOCK, &ocfs2_flock_lops, + fp); + lockres->l_flags |= OCFS2_LOCK_NOCACHE; +} + +void ocfs2_qinfo_lock_res_init(struct ocfs2_lock_res *lockres, + struct ocfs2_mem_dqinfo *info) +{ + ocfs2_lock_res_init_once(lockres); + ocfs2_build_lock_name(OCFS2_LOCK_TYPE_QINFO, info->dqi_gi.dqi_type, + 0, lockres->l_name); + ocfs2_lock_res_init_common(OCFS2_SB(info->dqi_gi.dqi_sb), lockres, + OCFS2_LOCK_TYPE_QINFO, &ocfs2_qinfo_lops, + info); +} + +void ocfs2_refcount_lock_res_init(struct ocfs2_lock_res *lockres, + struct ocfs2_super *osb, u64 ref_blkno, + unsigned int generation) +{ + ocfs2_lock_res_init_once(lockres); + ocfs2_build_lock_name(OCFS2_LOCK_TYPE_REFCOUNT, ref_blkno, + generation, lockres->l_name); + ocfs2_lock_res_init_common(osb, lockres, OCFS2_LOCK_TYPE_REFCOUNT, + &ocfs2_refcount_block_lops, osb); +} + +void ocfs2_lock_res_free(struct ocfs2_lock_res *res) +{ + if (!(res->l_flags & OCFS2_LOCK_INITIALIZED)) + return; + + ocfs2_remove_lockres_tracking(res); + + mlog_bug_on_msg(!list_empty(&res->l_blocked_list), + "Lockres %s is on the blocked list\n", + res->l_name); + mlog_bug_on_msg(!list_empty(&res->l_mask_waiters), + "Lockres %s has mask waiters pending\n", + res->l_name); + mlog_bug_on_msg(spin_is_locked(&res->l_lock), + "Lockres %s is locked\n", + res->l_name); + mlog_bug_on_msg(res->l_ro_holders, + "Lockres %s has %u ro holders\n", + res->l_name, res->l_ro_holders); + mlog_bug_on_msg(res->l_ex_holders, + "Lockres %s has %u ex holders\n", + res->l_name, res->l_ex_holders); + + /* Need to clear out the lock status block for the dlm */ + memset(&res->l_lksb, 0, sizeof(res->l_lksb)); + + res->l_flags = 0UL; +} + +/* + * Keep a list of processes who have interest in a lockres. + * Note: this is now only used for check recursive cluster locking. + */ +static inline void ocfs2_add_holder(struct ocfs2_lock_res *lockres, + struct ocfs2_lock_holder *oh) +{ + INIT_LIST_HEAD(&oh->oh_list); + oh->oh_owner_pid = get_pid(task_pid(current)); + + spin_lock(&lockres->l_lock); + list_add_tail(&oh->oh_list, &lockres->l_holders); + spin_unlock(&lockres->l_lock); +} + +static struct ocfs2_lock_holder * +ocfs2_pid_holder(struct ocfs2_lock_res *lockres, + struct pid *pid) +{ + struct ocfs2_lock_holder *oh; + + spin_lock(&lockres->l_lock); + list_for_each_entry(oh, &lockres->l_holders, oh_list) { + if (oh->oh_owner_pid == pid) { + spin_unlock(&lockres->l_lock); + return oh; + } + } + spin_unlock(&lockres->l_lock); + return NULL; +} + +static inline void ocfs2_remove_holder(struct ocfs2_lock_res *lockres, + struct ocfs2_lock_holder *oh) +{ + spin_lock(&lockres->l_lock); + list_del(&oh->oh_list); + spin_unlock(&lockres->l_lock); + + put_pid(oh->oh_owner_pid); +} + + +static inline void ocfs2_inc_holders(struct ocfs2_lock_res *lockres, + int level) +{ + BUG_ON(!lockres); + + switch(level) { + case DLM_LOCK_EX: + lockres->l_ex_holders++; + break; + case DLM_LOCK_PR: + lockres->l_ro_holders++; + break; + default: + BUG(); + } +} + +static inline void ocfs2_dec_holders(struct ocfs2_lock_res *lockres, + int level) +{ + BUG_ON(!lockres); + + switch(level) { + case DLM_LOCK_EX: + BUG_ON(!lockres->l_ex_holders); + lockres->l_ex_holders--; + break; + case DLM_LOCK_PR: + BUG_ON(!lockres->l_ro_holders); + lockres->l_ro_holders--; + break; + default: + BUG(); + } +} + +/* WARNING: This function lives in a world where the only three lock + * levels are EX, PR, and NL. It *will* have to be adjusted when more + * lock types are added. */ +static inline int ocfs2_highest_compat_lock_level(int level) +{ + int new_level = DLM_LOCK_EX; + + if (level == DLM_LOCK_EX) + new_level = DLM_LOCK_NL; + else if (level == DLM_LOCK_PR) + new_level = DLM_LOCK_PR; + return new_level; +} + +static void lockres_set_flags(struct ocfs2_lock_res *lockres, + unsigned long newflags) +{ + struct ocfs2_mask_waiter *mw, *tmp; + + assert_spin_locked(&lockres->l_lock); + + lockres->l_flags = newflags; + + list_for_each_entry_safe(mw, tmp, &lockres->l_mask_waiters, mw_item) { + if ((lockres->l_flags & mw->mw_mask) != mw->mw_goal) + continue; + + list_del_init(&mw->mw_item); + mw->mw_status = 0; + complete(&mw->mw_complete); + ocfs2_track_lock_wait(lockres); + } +} +static void lockres_or_flags(struct ocfs2_lock_res *lockres, unsigned long or) +{ + lockres_set_flags(lockres, lockres->l_flags | or); +} +static void lockres_clear_flags(struct ocfs2_lock_res *lockres, + unsigned long clear) +{ + lockres_set_flags(lockres, lockres->l_flags & ~clear); +} + +static inline void ocfs2_generic_handle_downconvert_action(struct ocfs2_lock_res *lockres) +{ + BUG_ON(!(lockres->l_flags & OCFS2_LOCK_BUSY)); + BUG_ON(!(lockres->l_flags & OCFS2_LOCK_ATTACHED)); + BUG_ON(!(lockres->l_flags & OCFS2_LOCK_BLOCKED)); + BUG_ON(lockres->l_blocking <= DLM_LOCK_NL); + + lockres->l_level = lockres->l_requested; + if (lockres->l_level <= + ocfs2_highest_compat_lock_level(lockres->l_blocking)) { + lockres->l_blocking = DLM_LOCK_NL; + lockres_clear_flags(lockres, OCFS2_LOCK_BLOCKED); + } + lockres_clear_flags(lockres, OCFS2_LOCK_BUSY); +} + +static inline void ocfs2_generic_handle_convert_action(struct ocfs2_lock_res *lockres) +{ + BUG_ON(!(lockres->l_flags & OCFS2_LOCK_BUSY)); + BUG_ON(!(lockres->l_flags & OCFS2_LOCK_ATTACHED)); + + /* Convert from RO to EX doesn't really need anything as our + * information is already up to data. Convert from NL to + * *anything* however should mark ourselves as needing an + * update */ + if (lockres->l_level == DLM_LOCK_NL && + lockres->l_ops->flags & LOCK_TYPE_REQUIRES_REFRESH) + lockres_or_flags(lockres, OCFS2_LOCK_NEEDS_REFRESH); + + lockres->l_level = lockres->l_requested; + + /* + * We set the OCFS2_LOCK_UPCONVERT_FINISHING flag before clearing + * the OCFS2_LOCK_BUSY flag to prevent the dc thread from + * downconverting the lock before the upconvert has fully completed. + * Do not prevent the dc thread from downconverting if NONBLOCK lock + * had already returned. + */ + if (!(lockres->l_flags & OCFS2_LOCK_NONBLOCK_FINISHED)) + lockres_or_flags(lockres, OCFS2_LOCK_UPCONVERT_FINISHING); + else + lockres_clear_flags(lockres, OCFS2_LOCK_NONBLOCK_FINISHED); + + lockres_clear_flags(lockres, OCFS2_LOCK_BUSY); +} + +static inline void ocfs2_generic_handle_attach_action(struct ocfs2_lock_res *lockres) +{ + BUG_ON((!(lockres->l_flags & OCFS2_LOCK_BUSY))); + BUG_ON(lockres->l_flags & OCFS2_LOCK_ATTACHED); + + if (lockres->l_requested > DLM_LOCK_NL && + !(lockres->l_flags & OCFS2_LOCK_LOCAL) && + lockres->l_ops->flags & LOCK_TYPE_REQUIRES_REFRESH) + lockres_or_flags(lockres, OCFS2_LOCK_NEEDS_REFRESH); + + lockres->l_level = lockres->l_requested; + lockres_or_flags(lockres, OCFS2_LOCK_ATTACHED); + lockres_clear_flags(lockres, OCFS2_LOCK_BUSY); +} + +static int ocfs2_generic_handle_bast(struct ocfs2_lock_res *lockres, + int level) +{ + int needs_downconvert = 0; + + assert_spin_locked(&lockres->l_lock); + + if (level > lockres->l_blocking) { + /* only schedule a downconvert if we haven't already scheduled + * one that goes low enough to satisfy the level we're + * blocking. this also catches the case where we get + * duplicate BASTs */ + if (ocfs2_highest_compat_lock_level(level) < + ocfs2_highest_compat_lock_level(lockres->l_blocking)) + needs_downconvert = 1; + + lockres->l_blocking = level; + } + + mlog(ML_BASTS, "lockres %s, block %d, level %d, l_block %d, dwn %d\n", + lockres->l_name, level, lockres->l_level, lockres->l_blocking, + needs_downconvert); + + if (needs_downconvert) + lockres_or_flags(lockres, OCFS2_LOCK_BLOCKED); + mlog(0, "needs_downconvert = %d\n", needs_downconvert); + return needs_downconvert; +} + +/* + * OCFS2_LOCK_PENDING and l_pending_gen. + * + * Why does OCFS2_LOCK_PENDING exist? To close a race between setting + * OCFS2_LOCK_BUSY and calling ocfs2_dlm_lock(). See ocfs2_unblock_lock() + * for more details on the race. + * + * OCFS2_LOCK_PENDING closes the race quite nicely. However, it introduces + * a race on itself. In o2dlm, we can get the ast before ocfs2_dlm_lock() + * returns. The ast clears OCFS2_LOCK_BUSY, and must therefore clear + * OCFS2_LOCK_PENDING at the same time. When ocfs2_dlm_lock() returns, + * the caller is going to try to clear PENDING again. If nothing else is + * happening, __lockres_clear_pending() sees PENDING is unset and does + * nothing. + * + * But what if another path (eg downconvert thread) has just started a + * new locking action? The other path has re-set PENDING. Our path + * cannot clear PENDING, because that will re-open the original race + * window. + * + * [Example] + * + * ocfs2_meta_lock() + * ocfs2_cluster_lock() + * set BUSY + * set PENDING + * drop l_lock + * ocfs2_dlm_lock() + * ocfs2_locking_ast() ocfs2_downconvert_thread() + * clear PENDING ocfs2_unblock_lock() + * take_l_lock + * !BUSY + * ocfs2_prepare_downconvert() + * set BUSY + * set PENDING + * drop l_lock + * take l_lock + * clear PENDING + * drop l_lock + * + * ocfs2_dlm_lock() + * + * So as you can see, we now have a window where l_lock is not held, + * PENDING is not set, and ocfs2_dlm_lock() has not been called. + * + * The core problem is that ocfs2_cluster_lock() has cleared the PENDING + * set by ocfs2_prepare_downconvert(). That wasn't nice. + * + * To solve this we introduce l_pending_gen. A call to + * lockres_clear_pending() will only do so when it is passed a generation + * number that matches the lockres. lockres_set_pending() will return the + * current generation number. When ocfs2_cluster_lock() goes to clear + * PENDING, it passes the generation it got from set_pending(). In our + * example above, the generation numbers will *not* match. Thus, + * ocfs2_cluster_lock() will not clear the PENDING set by + * ocfs2_prepare_downconvert(). + */ + +/* Unlocked version for ocfs2_locking_ast() */ +static void __lockres_clear_pending(struct ocfs2_lock_res *lockres, + unsigned int generation, + struct ocfs2_super *osb) +{ + assert_spin_locked(&lockres->l_lock); + + /* + * The ast and locking functions can race us here. The winner + * will clear pending, the loser will not. + */ + if (!(lockres->l_flags & OCFS2_LOCK_PENDING) || + (lockres->l_pending_gen != generation)) + return; + + lockres_clear_flags(lockres, OCFS2_LOCK_PENDING); + lockres->l_pending_gen++; + + /* + * The downconvert thread may have skipped us because we + * were PENDING. Wake it up. + */ + if (lockres->l_flags & OCFS2_LOCK_BLOCKED) + ocfs2_wake_downconvert_thread(osb); +} + +/* Locked version for callers of ocfs2_dlm_lock() */ +static void lockres_clear_pending(struct ocfs2_lock_res *lockres, + unsigned int generation, + struct ocfs2_super *osb) +{ + unsigned long flags; + + spin_lock_irqsave(&lockres->l_lock, flags); + __lockres_clear_pending(lockres, generation, osb); + spin_unlock_irqrestore(&lockres->l_lock, flags); +} + +static unsigned int lockres_set_pending(struct ocfs2_lock_res *lockres) +{ + assert_spin_locked(&lockres->l_lock); + BUG_ON(!(lockres->l_flags & OCFS2_LOCK_BUSY)); + + lockres_or_flags(lockres, OCFS2_LOCK_PENDING); + + return lockres->l_pending_gen; +} + +static void ocfs2_blocking_ast(struct ocfs2_dlm_lksb *lksb, int level) +{ + struct ocfs2_lock_res *lockres = ocfs2_lksb_to_lock_res(lksb); + struct ocfs2_super *osb = ocfs2_get_lockres_osb(lockres); + int needs_downconvert; + unsigned long flags; + + BUG_ON(level <= DLM_LOCK_NL); + + mlog(ML_BASTS, "BAST fired for lockres %s, blocking %d, level %d, " + "type %s\n", lockres->l_name, level, lockres->l_level, + ocfs2_lock_type_string(lockres->l_type)); + + /* + * We can skip the bast for locks which don't enable caching - + * they'll be dropped at the earliest possible time anyway. + */ + if (lockres->l_flags & OCFS2_LOCK_NOCACHE) + return; + + spin_lock_irqsave(&lockres->l_lock, flags); + needs_downconvert = ocfs2_generic_handle_bast(lockres, level); + if (needs_downconvert) + ocfs2_schedule_blocked_lock(osb, lockres); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + wake_up(&lockres->l_event); + + ocfs2_wake_downconvert_thread(osb); +} + +static void ocfs2_locking_ast(struct ocfs2_dlm_lksb *lksb) +{ + struct ocfs2_lock_res *lockres = ocfs2_lksb_to_lock_res(lksb); + struct ocfs2_super *osb = ocfs2_get_lockres_osb(lockres); + unsigned long flags; + int status; + + spin_lock_irqsave(&lockres->l_lock, flags); + + status = ocfs2_dlm_lock_status(&lockres->l_lksb); + + if (status == -EAGAIN) { + lockres_clear_flags(lockres, OCFS2_LOCK_BUSY); + goto out; + } + + if (status) { + mlog(ML_ERROR, "lockres %s: lksb status value of %d!\n", + lockres->l_name, status); + spin_unlock_irqrestore(&lockres->l_lock, flags); + return; + } + + mlog(ML_BASTS, "AST fired for lockres %s, action %d, unlock %d, " + "level %d => %d\n", lockres->l_name, lockres->l_action, + lockres->l_unlock_action, lockres->l_level, lockres->l_requested); + + switch(lockres->l_action) { + case OCFS2_AST_ATTACH: + ocfs2_generic_handle_attach_action(lockres); + lockres_clear_flags(lockres, OCFS2_LOCK_LOCAL); + break; + case OCFS2_AST_CONVERT: + ocfs2_generic_handle_convert_action(lockres); + break; + case OCFS2_AST_DOWNCONVERT: + ocfs2_generic_handle_downconvert_action(lockres); + break; + default: + mlog(ML_ERROR, "lockres %s: AST fired with invalid action: %u, " + "flags 0x%lx, unlock: %u\n", + lockres->l_name, lockres->l_action, lockres->l_flags, + lockres->l_unlock_action); + BUG(); + } +out: + /* set it to something invalid so if we get called again we + * can catch it. */ + lockres->l_action = OCFS2_AST_INVALID; + + /* Did we try to cancel this lock? Clear that state */ + if (lockres->l_unlock_action == OCFS2_UNLOCK_CANCEL_CONVERT) + lockres->l_unlock_action = OCFS2_UNLOCK_INVALID; + + /* + * We may have beaten the locking functions here. We certainly + * know that dlm_lock() has been called :-) + * Because we can't have two lock calls in flight at once, we + * can use lockres->l_pending_gen. + */ + __lockres_clear_pending(lockres, lockres->l_pending_gen, osb); + + wake_up(&lockres->l_event); + spin_unlock_irqrestore(&lockres->l_lock, flags); +} + +static void ocfs2_unlock_ast(struct ocfs2_dlm_lksb *lksb, int error) +{ + struct ocfs2_lock_res *lockres = ocfs2_lksb_to_lock_res(lksb); + unsigned long flags; + + mlog(ML_BASTS, "UNLOCK AST fired for lockres %s, action = %d\n", + lockres->l_name, lockres->l_unlock_action); + + spin_lock_irqsave(&lockres->l_lock, flags); + if (error) { + mlog(ML_ERROR, "Dlm passes error %d for lock %s, " + "unlock_action %d\n", error, lockres->l_name, + lockres->l_unlock_action); + spin_unlock_irqrestore(&lockres->l_lock, flags); + return; + } + + switch(lockres->l_unlock_action) { + case OCFS2_UNLOCK_CANCEL_CONVERT: + mlog(0, "Cancel convert success for %s\n", lockres->l_name); + lockres->l_action = OCFS2_AST_INVALID; + /* Downconvert thread may have requeued this lock, we + * need to wake it. */ + if (lockres->l_flags & OCFS2_LOCK_BLOCKED) + ocfs2_wake_downconvert_thread(ocfs2_get_lockres_osb(lockres)); + break; + case OCFS2_UNLOCK_DROP_LOCK: + lockres->l_level = DLM_LOCK_IV; + break; + default: + BUG(); + } + + lockres_clear_flags(lockres, OCFS2_LOCK_BUSY); + lockres->l_unlock_action = OCFS2_UNLOCK_INVALID; + wake_up(&lockres->l_event); + spin_unlock_irqrestore(&lockres->l_lock, flags); +} + +/* + * This is the filesystem locking protocol. It provides the lock handling + * hooks for the underlying DLM. It has a maximum version number. + * The version number allows interoperability with systems running at + * the same major number and an equal or smaller minor number. + * + * Whenever the filesystem does new things with locks (adds or removes a + * lock, orders them differently, does different things underneath a lock), + * the version must be changed. The protocol is negotiated when joining + * the dlm domain. A node may join the domain if its major version is + * identical to all other nodes and its minor version is greater than + * or equal to all other nodes. When its minor version is greater than + * the other nodes, it will run at the minor version specified by the + * other nodes. + * + * If a locking change is made that will not be compatible with older + * versions, the major number must be increased and the minor version set + * to zero. If a change merely adds a behavior that can be disabled when + * speaking to older versions, the minor version must be increased. If a + * change adds a fully backwards compatible change (eg, LVB changes that + * are just ignored by older versions), the version does not need to be + * updated. + */ +static struct ocfs2_locking_protocol lproto = { + .lp_max_version = { + .pv_major = OCFS2_LOCKING_PROTOCOL_MAJOR, + .pv_minor = OCFS2_LOCKING_PROTOCOL_MINOR, + }, + .lp_lock_ast = ocfs2_locking_ast, + .lp_blocking_ast = ocfs2_blocking_ast, + .lp_unlock_ast = ocfs2_unlock_ast, +}; + +void ocfs2_set_locking_protocol(void) +{ + ocfs2_stack_glue_set_max_proto_version(&lproto.lp_max_version); +} + +static inline void ocfs2_recover_from_dlm_error(struct ocfs2_lock_res *lockres, + int convert) +{ + unsigned long flags; + + spin_lock_irqsave(&lockres->l_lock, flags); + lockres_clear_flags(lockres, OCFS2_LOCK_BUSY); + lockres_clear_flags(lockres, OCFS2_LOCK_UPCONVERT_FINISHING); + if (convert) + lockres->l_action = OCFS2_AST_INVALID; + else + lockres->l_unlock_action = OCFS2_UNLOCK_INVALID; + spin_unlock_irqrestore(&lockres->l_lock, flags); + + wake_up(&lockres->l_event); +} + +/* Note: If we detect another process working on the lock (i.e., + * OCFS2_LOCK_BUSY), we'll bail out returning 0. It's up to the caller + * to do the right thing in that case. + */ +static int ocfs2_lock_create(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres, + int level, + u32 dlm_flags) +{ + int ret = 0; + unsigned long flags; + unsigned int gen; + + mlog(0, "lock %s, level = %d, flags = %u\n", lockres->l_name, level, + dlm_flags); + + spin_lock_irqsave(&lockres->l_lock, flags); + if ((lockres->l_flags & OCFS2_LOCK_ATTACHED) || + (lockres->l_flags & OCFS2_LOCK_BUSY)) { + spin_unlock_irqrestore(&lockres->l_lock, flags); + goto bail; + } + + lockres->l_action = OCFS2_AST_ATTACH; + lockres->l_requested = level; + lockres_or_flags(lockres, OCFS2_LOCK_BUSY); + gen = lockres_set_pending(lockres); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + ret = ocfs2_dlm_lock(osb->cconn, + level, + &lockres->l_lksb, + dlm_flags, + lockres->l_name, + OCFS2_LOCK_ID_MAX_LEN - 1); + lockres_clear_pending(lockres, gen, osb); + if (ret) { + ocfs2_log_dlm_error("ocfs2_dlm_lock", ret, lockres); + ocfs2_recover_from_dlm_error(lockres, 1); + } + + mlog(0, "lock %s, return from ocfs2_dlm_lock\n", lockres->l_name); + +bail: + return ret; +} + +static inline int ocfs2_check_wait_flag(struct ocfs2_lock_res *lockres, + int flag) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&lockres->l_lock, flags); + ret = lockres->l_flags & flag; + spin_unlock_irqrestore(&lockres->l_lock, flags); + + return ret; +} + +static inline void ocfs2_wait_on_busy_lock(struct ocfs2_lock_res *lockres) + +{ + wait_event(lockres->l_event, + !ocfs2_check_wait_flag(lockres, OCFS2_LOCK_BUSY)); +} + +static inline void ocfs2_wait_on_refreshing_lock(struct ocfs2_lock_res *lockres) + +{ + wait_event(lockres->l_event, + !ocfs2_check_wait_flag(lockres, OCFS2_LOCK_REFRESHING)); +} + +/* predict what lock level we'll be dropping down to on behalf + * of another node, and return true if the currently wanted + * level will be compatible with it. */ +static inline int ocfs2_may_continue_on_blocked_lock(struct ocfs2_lock_res *lockres, + int wanted) +{ + BUG_ON(!(lockres->l_flags & OCFS2_LOCK_BLOCKED)); + + return wanted <= ocfs2_highest_compat_lock_level(lockres->l_blocking); +} + +static void ocfs2_init_mask_waiter(struct ocfs2_mask_waiter *mw) +{ + INIT_LIST_HEAD(&mw->mw_item); + init_completion(&mw->mw_complete); + ocfs2_init_start_time(mw); +} + +static int ocfs2_wait_for_mask(struct ocfs2_mask_waiter *mw) +{ + wait_for_completion(&mw->mw_complete); + /* Re-arm the completion in case we want to wait on it again */ + reinit_completion(&mw->mw_complete); + return mw->mw_status; +} + +static void lockres_add_mask_waiter(struct ocfs2_lock_res *lockres, + struct ocfs2_mask_waiter *mw, + unsigned long mask, + unsigned long goal) +{ + BUG_ON(!list_empty(&mw->mw_item)); + + assert_spin_locked(&lockres->l_lock); + + list_add_tail(&mw->mw_item, &lockres->l_mask_waiters); + mw->mw_mask = mask; + mw->mw_goal = goal; + ocfs2_track_lock_wait(lockres); +} + +/* returns 0 if the mw that was removed was already satisfied, -EBUSY + * if the mask still hadn't reached its goal */ +static int __lockres_remove_mask_waiter(struct ocfs2_lock_res *lockres, + struct ocfs2_mask_waiter *mw) +{ + int ret = 0; + + assert_spin_locked(&lockres->l_lock); + if (!list_empty(&mw->mw_item)) { + if ((lockres->l_flags & mw->mw_mask) != mw->mw_goal) + ret = -EBUSY; + + list_del_init(&mw->mw_item); + init_completion(&mw->mw_complete); + ocfs2_track_lock_wait(lockres); + } + + return ret; +} + +static int lockres_remove_mask_waiter(struct ocfs2_lock_res *lockres, + struct ocfs2_mask_waiter *mw) +{ + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(&lockres->l_lock, flags); + ret = __lockres_remove_mask_waiter(lockres, mw); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + return ret; + +} + +static int ocfs2_wait_for_mask_interruptible(struct ocfs2_mask_waiter *mw, + struct ocfs2_lock_res *lockres) +{ + int ret; + + ret = wait_for_completion_interruptible(&mw->mw_complete); + if (ret) + lockres_remove_mask_waiter(lockres, mw); + else + ret = mw->mw_status; + /* Re-arm the completion in case we want to wait on it again */ + reinit_completion(&mw->mw_complete); + return ret; +} + +static int __ocfs2_cluster_lock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres, + int level, + u32 lkm_flags, + int arg_flags, + int l_subclass, + unsigned long caller_ip) +{ + struct ocfs2_mask_waiter mw; + int wait, catch_signals = !(osb->s_mount_opt & OCFS2_MOUNT_NOINTR); + int ret = 0; /* gcc doesn't realize wait = 1 guarantees ret is set */ + unsigned long flags; + unsigned int gen; + int noqueue_attempted = 0; + int dlm_locked = 0; + int kick_dc = 0; + + if (!(lockres->l_flags & OCFS2_LOCK_INITIALIZED)) { + mlog_errno(-EINVAL); + return -EINVAL; + } + + ocfs2_init_mask_waiter(&mw); + + if (lockres->l_ops->flags & LOCK_TYPE_USES_LVB) + lkm_flags |= DLM_LKF_VALBLK; + +again: + wait = 0; + + spin_lock_irqsave(&lockres->l_lock, flags); + + if (catch_signals && signal_pending(current)) { + ret = -ERESTARTSYS; + goto unlock; + } + + mlog_bug_on_msg(lockres->l_flags & OCFS2_LOCK_FREEING, + "Cluster lock called on freeing lockres %s! flags " + "0x%lx\n", lockres->l_name, lockres->l_flags); + + /* We only compare against the currently granted level + * here. If the lock is blocked waiting on a downconvert, + * we'll get caught below. */ + if (lockres->l_flags & OCFS2_LOCK_BUSY && + level > lockres->l_level) { + /* is someone sitting in dlm_lock? If so, wait on + * them. */ + lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_BUSY, 0); + wait = 1; + goto unlock; + } + + if (lockres->l_flags & OCFS2_LOCK_UPCONVERT_FINISHING) { + /* + * We've upconverted. If the lock now has a level we can + * work with, we take it. If, however, the lock is not at the + * required level, we go thru the full cycle. One way this could + * happen is if a process requesting an upconvert to PR is + * closely followed by another requesting upconvert to an EX. + * If the process requesting EX lands here, we want it to + * continue attempting to upconvert and let the process + * requesting PR take the lock. + * If multiple processes request upconvert to PR, the first one + * here will take the lock. The others will have to go thru the + * OCFS2_LOCK_BLOCKED check to ensure that there is no pending + * downconvert request. + */ + if (level <= lockres->l_level) + goto update_holders; + } + + if (lockres->l_flags & OCFS2_LOCK_BLOCKED && + !ocfs2_may_continue_on_blocked_lock(lockres, level)) { + /* is the lock is currently blocked on behalf of + * another node */ + lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_BLOCKED, 0); + wait = 1; + goto unlock; + } + + if (level > lockres->l_level) { + if (noqueue_attempted > 0) { + ret = -EAGAIN; + goto unlock; + } + if (lkm_flags & DLM_LKF_NOQUEUE) + noqueue_attempted = 1; + + if (lockres->l_action != OCFS2_AST_INVALID) + mlog(ML_ERROR, "lockres %s has action %u pending\n", + lockres->l_name, lockres->l_action); + + if (!(lockres->l_flags & OCFS2_LOCK_ATTACHED)) { + lockres->l_action = OCFS2_AST_ATTACH; + lkm_flags &= ~DLM_LKF_CONVERT; + } else { + lockres->l_action = OCFS2_AST_CONVERT; + lkm_flags |= DLM_LKF_CONVERT; + } + + lockres->l_requested = level; + lockres_or_flags(lockres, OCFS2_LOCK_BUSY); + gen = lockres_set_pending(lockres); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + BUG_ON(level == DLM_LOCK_IV); + BUG_ON(level == DLM_LOCK_NL); + + mlog(ML_BASTS, "lockres %s, convert from %d to %d\n", + lockres->l_name, lockres->l_level, level); + + /* call dlm_lock to upgrade lock now */ + ret = ocfs2_dlm_lock(osb->cconn, + level, + &lockres->l_lksb, + lkm_flags, + lockres->l_name, + OCFS2_LOCK_ID_MAX_LEN - 1); + lockres_clear_pending(lockres, gen, osb); + if (ret) { + if (!(lkm_flags & DLM_LKF_NOQUEUE) || + (ret != -EAGAIN)) { + ocfs2_log_dlm_error("ocfs2_dlm_lock", + ret, lockres); + } + ocfs2_recover_from_dlm_error(lockres, 1); + goto out; + } + dlm_locked = 1; + + mlog(0, "lock %s, successful return from ocfs2_dlm_lock\n", + lockres->l_name); + + /* At this point we've gone inside the dlm and need to + * complete our work regardless. */ + catch_signals = 0; + + /* wait for busy to clear and carry on */ + goto again; + } + +update_holders: + /* Ok, if we get here then we're good to go. */ + ocfs2_inc_holders(lockres, level); + + ret = 0; +unlock: + lockres_clear_flags(lockres, OCFS2_LOCK_UPCONVERT_FINISHING); + + /* ocfs2_unblock_lock request on seeing OCFS2_LOCK_UPCONVERT_FINISHING */ + kick_dc = (lockres->l_flags & OCFS2_LOCK_BLOCKED); + + spin_unlock_irqrestore(&lockres->l_lock, flags); + if (kick_dc) + ocfs2_wake_downconvert_thread(osb); +out: + /* + * This is helping work around a lock inversion between the page lock + * and dlm locks. One path holds the page lock while calling aops + * which block acquiring dlm locks. The voting thread holds dlm + * locks while acquiring page locks while down converting data locks. + * This block is helping an aop path notice the inversion and back + * off to unlock its page lock before trying the dlm lock again. + */ + if (wait && arg_flags & OCFS2_LOCK_NONBLOCK && + mw.mw_mask & (OCFS2_LOCK_BUSY|OCFS2_LOCK_BLOCKED)) { + wait = 0; + spin_lock_irqsave(&lockres->l_lock, flags); + if (__lockres_remove_mask_waiter(lockres, &mw)) { + if (dlm_locked) + lockres_or_flags(lockres, + OCFS2_LOCK_NONBLOCK_FINISHED); + spin_unlock_irqrestore(&lockres->l_lock, flags); + ret = -EAGAIN; + } else { + spin_unlock_irqrestore(&lockres->l_lock, flags); + goto again; + } + } + if (wait) { + ret = ocfs2_wait_for_mask(&mw); + if (ret == 0) + goto again; + mlog_errno(ret); + } + ocfs2_update_lock_stats(lockres, level, &mw, ret); + +#ifdef CONFIG_DEBUG_LOCK_ALLOC + if (!ret && lockres->l_lockdep_map.key != NULL) { + if (level == DLM_LOCK_PR) + rwsem_acquire_read(&lockres->l_lockdep_map, l_subclass, + !!(arg_flags & OCFS2_META_LOCK_NOQUEUE), + caller_ip); + else + rwsem_acquire(&lockres->l_lockdep_map, l_subclass, + !!(arg_flags & OCFS2_META_LOCK_NOQUEUE), + caller_ip); + } +#endif + return ret; +} + +static inline int ocfs2_cluster_lock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres, + int level, + u32 lkm_flags, + int arg_flags) +{ + return __ocfs2_cluster_lock(osb, lockres, level, lkm_flags, arg_flags, + 0, _RET_IP_); +} + + +static void __ocfs2_cluster_unlock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres, + int level, + unsigned long caller_ip) +{ + unsigned long flags; + + spin_lock_irqsave(&lockres->l_lock, flags); + ocfs2_dec_holders(lockres, level); + ocfs2_downconvert_on_unlock(osb, lockres); + spin_unlock_irqrestore(&lockres->l_lock, flags); +#ifdef CONFIG_DEBUG_LOCK_ALLOC + if (lockres->l_lockdep_map.key != NULL) + rwsem_release(&lockres->l_lockdep_map, caller_ip); +#endif +} + +static int ocfs2_create_new_lock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres, + int ex, + int local) +{ + int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; + unsigned long flags; + u32 lkm_flags = local ? DLM_LKF_LOCAL : 0; + + spin_lock_irqsave(&lockres->l_lock, flags); + BUG_ON(lockres->l_flags & OCFS2_LOCK_ATTACHED); + lockres_or_flags(lockres, OCFS2_LOCK_LOCAL); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + return ocfs2_lock_create(osb, lockres, level, lkm_flags); +} + +/* Grants us an EX lock on the data and metadata resources, skipping + * the normal cluster directory lookup. Use this ONLY on newly created + * inodes which other nodes can't possibly see, and which haven't been + * hashed in the inode hash yet. This can give us a good performance + * increase as it'll skip the network broadcast normally associated + * with creating a new lock resource. */ +int ocfs2_create_new_inode_locks(struct inode *inode) +{ + int ret; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + BUG_ON(!ocfs2_inode_is_new(inode)); + + mlog(0, "Inode %llu\n", (unsigned long long)OCFS2_I(inode)->ip_blkno); + + /* NOTE: That we don't increment any of the holder counts, nor + * do we add anything to a journal handle. Since this is + * supposed to be a new inode which the cluster doesn't know + * about yet, there is no need to. As far as the LVB handling + * is concerned, this is basically like acquiring an EX lock + * on a resource which has an invalid one -- we'll set it + * valid when we release the EX. */ + + ret = ocfs2_create_new_lock(osb, &OCFS2_I(inode)->ip_rw_lockres, 1, 1); + if (ret) { + mlog_errno(ret); + goto bail; + } + + /* + * We don't want to use DLM_LKF_LOCAL on a meta data lock as they + * don't use a generation in their lock names. + */ + ret = ocfs2_create_new_lock(osb, &OCFS2_I(inode)->ip_inode_lockres, 1, 0); + if (ret) { + mlog_errno(ret); + goto bail; + } + + ret = ocfs2_create_new_lock(osb, &OCFS2_I(inode)->ip_open_lockres, 0, 0); + if (ret) + mlog_errno(ret); + +bail: + return ret; +} + +int ocfs2_rw_lock(struct inode *inode, int write) +{ + int status, level; + struct ocfs2_lock_res *lockres; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + mlog(0, "inode %llu take %s RW lock\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno, + write ? "EXMODE" : "PRMODE"); + + if (ocfs2_mount_local(osb)) + return 0; + + lockres = &OCFS2_I(inode)->ip_rw_lockres; + + level = write ? DLM_LOCK_EX : DLM_LOCK_PR; + + status = ocfs2_cluster_lock(osb, lockres, level, 0, 0); + if (status < 0) + mlog_errno(status); + + return status; +} + +int ocfs2_try_rw_lock(struct inode *inode, int write) +{ + int status, level; + struct ocfs2_lock_res *lockres; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + mlog(0, "inode %llu try to take %s RW lock\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno, + write ? "EXMODE" : "PRMODE"); + + if (ocfs2_mount_local(osb)) + return 0; + + lockres = &OCFS2_I(inode)->ip_rw_lockres; + + level = write ? DLM_LOCK_EX : DLM_LOCK_PR; + + status = ocfs2_cluster_lock(osb, lockres, level, DLM_LKF_NOQUEUE, 0); + return status; +} + +void ocfs2_rw_unlock(struct inode *inode, int write) +{ + int level = write ? DLM_LOCK_EX : DLM_LOCK_PR; + struct ocfs2_lock_res *lockres = &OCFS2_I(inode)->ip_rw_lockres; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + mlog(0, "inode %llu drop %s RW lock\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno, + write ? "EXMODE" : "PRMODE"); + + if (!ocfs2_mount_local(osb)) + ocfs2_cluster_unlock(osb, lockres, level); +} + +/* + * ocfs2_open_lock always get PR mode lock. + */ +int ocfs2_open_lock(struct inode *inode) +{ + int status = 0; + struct ocfs2_lock_res *lockres; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + mlog(0, "inode %llu take PRMODE open lock\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno); + + if (ocfs2_is_hard_readonly(osb) || ocfs2_mount_local(osb)) + goto out; + + lockres = &OCFS2_I(inode)->ip_open_lockres; + + status = ocfs2_cluster_lock(osb, lockres, DLM_LOCK_PR, 0, 0); + if (status < 0) + mlog_errno(status); + +out: + return status; +} + +int ocfs2_try_open_lock(struct inode *inode, int write) +{ + int status = 0, level; + struct ocfs2_lock_res *lockres; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + mlog(0, "inode %llu try to take %s open lock\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno, + write ? "EXMODE" : "PRMODE"); + + if (ocfs2_is_hard_readonly(osb)) { + if (write) + status = -EROFS; + goto out; + } + + if (ocfs2_mount_local(osb)) + goto out; + + lockres = &OCFS2_I(inode)->ip_open_lockres; + + level = write ? DLM_LOCK_EX : DLM_LOCK_PR; + + /* + * The file system may already holding a PRMODE/EXMODE open lock. + * Since we pass DLM_LKF_NOQUEUE, the request won't block waiting on + * other nodes and the -EAGAIN will indicate to the caller that + * this inode is still in use. + */ + status = ocfs2_cluster_lock(osb, lockres, level, DLM_LKF_NOQUEUE, 0); + +out: + return status; +} + +/* + * ocfs2_open_unlock unlock PR and EX mode open locks. + */ +void ocfs2_open_unlock(struct inode *inode) +{ + struct ocfs2_lock_res *lockres = &OCFS2_I(inode)->ip_open_lockres; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + mlog(0, "inode %llu drop open lock\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno); + + if (ocfs2_mount_local(osb)) + goto out; + + if(lockres->l_ro_holders) + ocfs2_cluster_unlock(osb, lockres, DLM_LOCK_PR); + if(lockres->l_ex_holders) + ocfs2_cluster_unlock(osb, lockres, DLM_LOCK_EX); + +out: + return; +} + +static int ocfs2_flock_handle_signal(struct ocfs2_lock_res *lockres, + int level) +{ + int ret; + struct ocfs2_super *osb = ocfs2_get_lockres_osb(lockres); + unsigned long flags; + struct ocfs2_mask_waiter mw; + + ocfs2_init_mask_waiter(&mw); + +retry_cancel: + spin_lock_irqsave(&lockres->l_lock, flags); + if (lockres->l_flags & OCFS2_LOCK_BUSY) { + ret = ocfs2_prepare_cancel_convert(osb, lockres); + if (ret) { + spin_unlock_irqrestore(&lockres->l_lock, flags); + ret = ocfs2_cancel_convert(osb, lockres); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + goto retry_cancel; + } + lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_BUSY, 0); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + ocfs2_wait_for_mask(&mw); + goto retry_cancel; + } + + ret = -ERESTARTSYS; + /* + * We may still have gotten the lock, in which case there's no + * point to restarting the syscall. + */ + if (lockres->l_level == level) + ret = 0; + + mlog(0, "Cancel returning %d. flags: 0x%lx, level: %d, act: %d\n", ret, + lockres->l_flags, lockres->l_level, lockres->l_action); + + spin_unlock_irqrestore(&lockres->l_lock, flags); + +out: + return ret; +} + +/* + * ocfs2_file_lock() and ocfs2_file_unlock() map to a single pair of + * flock() calls. The locking approach this requires is sufficiently + * different from all other cluster lock types that we implement a + * separate path to the "low-level" dlm calls. In particular: + * + * - No optimization of lock levels is done - we take at exactly + * what's been requested. + * + * - No lock caching is employed. We immediately downconvert to + * no-lock at unlock time. This also means flock locks never go on + * the blocking list). + * + * - Since userspace can trivially deadlock itself with flock, we make + * sure to allow cancellation of a misbehaving applications flock() + * request. + * + * - Access to any flock lockres doesn't require concurrency, so we + * can simplify the code by requiring the caller to guarantee + * serialization of dlmglue flock calls. + */ +int ocfs2_file_lock(struct file *file, int ex, int trylock) +{ + int ret, level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; + unsigned int lkm_flags = trylock ? DLM_LKF_NOQUEUE : 0; + unsigned long flags; + struct ocfs2_file_private *fp = file->private_data; + struct ocfs2_lock_res *lockres = &fp->fp_flock; + struct ocfs2_super *osb = OCFS2_SB(file->f_mapping->host->i_sb); + struct ocfs2_mask_waiter mw; + + ocfs2_init_mask_waiter(&mw); + + if ((lockres->l_flags & OCFS2_LOCK_BUSY) || + (lockres->l_level > DLM_LOCK_NL)) { + mlog(ML_ERROR, + "File lock \"%s\" has busy or locked state: flags: 0x%lx, " + "level: %u\n", lockres->l_name, lockres->l_flags, + lockres->l_level); + return -EINVAL; + } + + spin_lock_irqsave(&lockres->l_lock, flags); + if (!(lockres->l_flags & OCFS2_LOCK_ATTACHED)) { + lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_BUSY, 0); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + /* + * Get the lock at NLMODE to start - that way we + * can cancel the upconvert request if need be. + */ + ret = ocfs2_lock_create(osb, lockres, DLM_LOCK_NL, 0); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_wait_for_mask(&mw); + if (ret) { + mlog_errno(ret); + goto out; + } + spin_lock_irqsave(&lockres->l_lock, flags); + } + + lockres->l_action = OCFS2_AST_CONVERT; + lkm_flags |= DLM_LKF_CONVERT; + lockres->l_requested = level; + lockres_or_flags(lockres, OCFS2_LOCK_BUSY); + + lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_BUSY, 0); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + ret = ocfs2_dlm_lock(osb->cconn, level, &lockres->l_lksb, lkm_flags, + lockres->l_name, OCFS2_LOCK_ID_MAX_LEN - 1); + if (ret) { + if (!trylock || (ret != -EAGAIN)) { + ocfs2_log_dlm_error("ocfs2_dlm_lock", ret, lockres); + ret = -EINVAL; + } + + ocfs2_recover_from_dlm_error(lockres, 1); + lockres_remove_mask_waiter(lockres, &mw); + goto out; + } + + ret = ocfs2_wait_for_mask_interruptible(&mw, lockres); + if (ret == -ERESTARTSYS) { + /* + * Userspace can cause deadlock itself with + * flock(). Current behavior locally is to allow the + * deadlock, but abort the system call if a signal is + * received. We follow this example, otherwise a + * poorly written program could sit in kernel until + * reboot. + * + * Handling this is a bit more complicated for Ocfs2 + * though. We can't exit this function with an + * outstanding lock request, so a cancel convert is + * required. We intentionally overwrite 'ret' - if the + * cancel fails and the lock was granted, it's easier + * to just bubble success back up to the user. + */ + ret = ocfs2_flock_handle_signal(lockres, level); + } else if (!ret && (level > lockres->l_level)) { + /* Trylock failed asynchronously */ + BUG_ON(!trylock); + ret = -EAGAIN; + } + +out: + + mlog(0, "Lock: \"%s\" ex: %d, trylock: %d, returns: %d\n", + lockres->l_name, ex, trylock, ret); + return ret; +} + +void ocfs2_file_unlock(struct file *file) +{ + int ret; + unsigned int gen; + unsigned long flags; + struct ocfs2_file_private *fp = file->private_data; + struct ocfs2_lock_res *lockres = &fp->fp_flock; + struct ocfs2_super *osb = OCFS2_SB(file->f_mapping->host->i_sb); + struct ocfs2_mask_waiter mw; + + ocfs2_init_mask_waiter(&mw); + + if (!(lockres->l_flags & OCFS2_LOCK_ATTACHED)) + return; + + if (lockres->l_level == DLM_LOCK_NL) + return; + + mlog(0, "Unlock: \"%s\" flags: 0x%lx, level: %d, act: %d\n", + lockres->l_name, lockres->l_flags, lockres->l_level, + lockres->l_action); + + spin_lock_irqsave(&lockres->l_lock, flags); + /* + * Fake a blocking ast for the downconvert code. + */ + lockres_or_flags(lockres, OCFS2_LOCK_BLOCKED); + lockres->l_blocking = DLM_LOCK_EX; + + gen = ocfs2_prepare_downconvert(lockres, DLM_LOCK_NL); + lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_BUSY, 0); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + ret = ocfs2_downconvert_lock(osb, lockres, DLM_LOCK_NL, 0, gen); + if (ret) { + mlog_errno(ret); + return; + } + + ret = ocfs2_wait_for_mask(&mw); + if (ret) + mlog_errno(ret); +} + +static void ocfs2_downconvert_on_unlock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres) +{ + int kick = 0; + + /* If we know that another node is waiting on our lock, kick + * the downconvert thread * pre-emptively when we reach a release + * condition. */ + if (lockres->l_flags & OCFS2_LOCK_BLOCKED) { + switch(lockres->l_blocking) { + case DLM_LOCK_EX: + if (!lockres->l_ex_holders && !lockres->l_ro_holders) + kick = 1; + break; + case DLM_LOCK_PR: + if (!lockres->l_ex_holders) + kick = 1; + break; + default: + BUG(); + } + } + + if (kick) + ocfs2_wake_downconvert_thread(osb); +} + +#define OCFS2_SEC_BITS 34 +#define OCFS2_SEC_SHIFT (64 - OCFS2_SEC_BITS) +#define OCFS2_NSEC_MASK ((1ULL << OCFS2_SEC_SHIFT) - 1) + +/* LVB only has room for 64 bits of time here so we pack it for + * now. */ +static u64 ocfs2_pack_timespec(struct timespec64 *spec) +{ + u64 res; + u64 sec = clamp_t(time64_t, spec->tv_sec, 0, 0x3ffffffffull); + u32 nsec = spec->tv_nsec; + + res = (sec << OCFS2_SEC_SHIFT) | (nsec & OCFS2_NSEC_MASK); + + return res; +} + +/* Call this with the lockres locked. I am reasonably sure we don't + * need ip_lock in this function as anyone who would be changing those + * values is supposed to be blocked in ocfs2_inode_lock right now. */ +static void __ocfs2_stuff_meta_lvb(struct inode *inode) +{ + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_lock_res *lockres = &oi->ip_inode_lockres; + struct ocfs2_meta_lvb *lvb; + struct timespec64 ts; + + lvb = ocfs2_dlm_lvb(&lockres->l_lksb); + + /* + * Invalidate the LVB of a deleted inode - this way other + * nodes are forced to go to disk and discover the new inode + * status. + */ + if (oi->ip_flags & OCFS2_INODE_DELETED) { + lvb->lvb_version = 0; + goto out; + } + + lvb->lvb_version = OCFS2_LVB_VERSION; + lvb->lvb_isize = cpu_to_be64(i_size_read(inode)); + lvb->lvb_iclusters = cpu_to_be32(oi->ip_clusters); + lvb->lvb_iuid = cpu_to_be32(i_uid_read(inode)); + lvb->lvb_igid = cpu_to_be32(i_gid_read(inode)); + lvb->lvb_imode = cpu_to_be16(inode->i_mode); + lvb->lvb_inlink = cpu_to_be16(inode->i_nlink); + ts = inode_get_atime(inode); + lvb->lvb_iatime_packed = cpu_to_be64(ocfs2_pack_timespec(&ts)); + ts = inode_get_ctime(inode); + lvb->lvb_ictime_packed = cpu_to_be64(ocfs2_pack_timespec(&ts)); + ts = inode_get_mtime(inode); + lvb->lvb_imtime_packed = cpu_to_be64(ocfs2_pack_timespec(&ts)); + lvb->lvb_iattr = cpu_to_be32(oi->ip_attr); + lvb->lvb_idynfeatures = cpu_to_be16(oi->ip_dyn_features); + lvb->lvb_igeneration = cpu_to_be32(inode->i_generation); + +out: + mlog_meta_lvb(0, lockres); +} + +static void ocfs2_unpack_timespec(struct timespec64 *spec, + u64 packed_time) +{ + spec->tv_sec = packed_time >> OCFS2_SEC_SHIFT; + spec->tv_nsec = packed_time & OCFS2_NSEC_MASK; +} + +static int ocfs2_refresh_inode_from_lvb(struct inode *inode) +{ + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_lock_res *lockres = &oi->ip_inode_lockres; + struct ocfs2_meta_lvb *lvb; + struct timespec64 ts; + + mlog_meta_lvb(0, lockres); + + lvb = ocfs2_dlm_lvb(&lockres->l_lksb); + if (inode_wrong_type(inode, be16_to_cpu(lvb->lvb_imode))) + return -ESTALE; + + /* We're safe here without the lockres lock... */ + spin_lock(&oi->ip_lock); + oi->ip_clusters = be32_to_cpu(lvb->lvb_iclusters); + i_size_write(inode, be64_to_cpu(lvb->lvb_isize)); + + oi->ip_attr = be32_to_cpu(lvb->lvb_iattr); + oi->ip_dyn_features = be16_to_cpu(lvb->lvb_idynfeatures); + ocfs2_set_inode_flags(inode); + + /* fast-symlinks are a special case */ + if (S_ISLNK(inode->i_mode) && !oi->ip_clusters) + inode->i_blocks = 0; + else + inode->i_blocks = ocfs2_inode_sector_count(inode); + + i_uid_write(inode, be32_to_cpu(lvb->lvb_iuid)); + i_gid_write(inode, be32_to_cpu(lvb->lvb_igid)); + inode->i_mode = be16_to_cpu(lvb->lvb_imode); + set_nlink(inode, be16_to_cpu(lvb->lvb_inlink)); + ocfs2_unpack_timespec(&ts, be64_to_cpu(lvb->lvb_iatime_packed)); + inode_set_atime_to_ts(inode, ts); + ocfs2_unpack_timespec(&ts, be64_to_cpu(lvb->lvb_imtime_packed)); + inode_set_mtime_to_ts(inode, ts); + ocfs2_unpack_timespec(&ts, be64_to_cpu(lvb->lvb_ictime_packed)); + inode_set_ctime_to_ts(inode, ts); + spin_unlock(&oi->ip_lock); + return 0; +} + +static inline int ocfs2_meta_lvb_is_trustable(struct inode *inode, + struct ocfs2_lock_res *lockres) +{ + struct ocfs2_meta_lvb *lvb = ocfs2_dlm_lvb(&lockres->l_lksb); + + if (ocfs2_dlm_lvb_valid(&lockres->l_lksb) + && lvb->lvb_version == OCFS2_LVB_VERSION + && be32_to_cpu(lvb->lvb_igeneration) == inode->i_generation) + return 1; + return 0; +} + +/* Determine whether a lock resource needs to be refreshed, and + * arbitrate who gets to refresh it. + * + * 0 means no refresh needed. + * + * > 0 means you need to refresh this and you MUST call + * ocfs2_complete_lock_res_refresh afterwards. */ +static int ocfs2_should_refresh_lock_res(struct ocfs2_lock_res *lockres) +{ + unsigned long flags; + int status = 0; + +refresh_check: + spin_lock_irqsave(&lockres->l_lock, flags); + if (!(lockres->l_flags & OCFS2_LOCK_NEEDS_REFRESH)) { + spin_unlock_irqrestore(&lockres->l_lock, flags); + goto bail; + } + + if (lockres->l_flags & OCFS2_LOCK_REFRESHING) { + spin_unlock_irqrestore(&lockres->l_lock, flags); + + ocfs2_wait_on_refreshing_lock(lockres); + goto refresh_check; + } + + /* Ok, I'll be the one to refresh this lock. */ + lockres_or_flags(lockres, OCFS2_LOCK_REFRESHING); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + status = 1; +bail: + mlog(0, "status %d\n", status); + return status; +} + +/* If status is non zero, I'll mark it as not being in refresh + * anymroe, but i won't clear the needs refresh flag. */ +static inline void ocfs2_complete_lock_res_refresh(struct ocfs2_lock_res *lockres, + int status) +{ + unsigned long flags; + + spin_lock_irqsave(&lockres->l_lock, flags); + lockres_clear_flags(lockres, OCFS2_LOCK_REFRESHING); + if (!status) + lockres_clear_flags(lockres, OCFS2_LOCK_NEEDS_REFRESH); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + wake_up(&lockres->l_event); +} + +/* may or may not return a bh if it went to disk. */ +static int ocfs2_inode_lock_update(struct inode *inode, + struct buffer_head **bh) +{ + int status = 0; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_lock_res *lockres = &oi->ip_inode_lockres; + struct ocfs2_dinode *fe; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + if (ocfs2_mount_local(osb)) + goto bail; + + spin_lock(&oi->ip_lock); + if (oi->ip_flags & OCFS2_INODE_DELETED) { + mlog(0, "Orphaned inode %llu was deleted while we " + "were waiting on a lock. ip_flags = 0x%x\n", + (unsigned long long)oi->ip_blkno, oi->ip_flags); + spin_unlock(&oi->ip_lock); + status = -ENOENT; + goto bail; + } + spin_unlock(&oi->ip_lock); + + if (!ocfs2_should_refresh_lock_res(lockres)) + goto bail; + + /* This will discard any caching information we might have had + * for the inode metadata. */ + ocfs2_metadata_cache_purge(INODE_CACHE(inode)); + + ocfs2_extent_map_trunc(inode, 0); + + if (ocfs2_meta_lvb_is_trustable(inode, lockres)) { + mlog(0, "Trusting LVB on inode %llu\n", + (unsigned long long)oi->ip_blkno); + status = ocfs2_refresh_inode_from_lvb(inode); + goto bail_refresh; + } else { + /* Boo, we have to go to disk. */ + /* read bh, cast, ocfs2_refresh_inode */ + status = ocfs2_read_inode_block(inode, bh); + if (status < 0) { + mlog_errno(status); + goto bail_refresh; + } + fe = (struct ocfs2_dinode *) (*bh)->b_data; + if (inode_wrong_type(inode, le16_to_cpu(fe->i_mode))) { + status = -ESTALE; + goto bail_refresh; + } + + /* This is a good chance to make sure we're not + * locking an invalid object. ocfs2_read_inode_block() + * already checked that the inode block is sane. + * + * We bug on a stale inode here because we checked + * above whether it was wiped from disk. The wiping + * node provides a guarantee that we receive that + * message and can mark the inode before dropping any + * locks associated with it. */ + mlog_bug_on_msg(inode->i_generation != + le32_to_cpu(fe->i_generation), + "Invalid dinode %llu disk generation: %u " + "inode->i_generation: %u\n", + (unsigned long long)oi->ip_blkno, + le32_to_cpu(fe->i_generation), + inode->i_generation); + mlog_bug_on_msg(le64_to_cpu(fe->i_dtime) || + !(fe->i_flags & cpu_to_le32(OCFS2_VALID_FL)), + "Stale dinode %llu dtime: %llu flags: 0x%x\n", + (unsigned long long)oi->ip_blkno, + (unsigned long long)le64_to_cpu(fe->i_dtime), + le32_to_cpu(fe->i_flags)); + + ocfs2_refresh_inode(inode, fe); + ocfs2_track_lock_refresh(lockres); + } + + status = 0; +bail_refresh: + ocfs2_complete_lock_res_refresh(lockres, status); +bail: + return status; +} + +static int ocfs2_assign_bh(struct inode *inode, + struct buffer_head **ret_bh, + struct buffer_head *passed_bh) +{ + int status; + + if (passed_bh) { + /* Ok, the update went to disk for us, use the + * returned bh. */ + *ret_bh = passed_bh; + get_bh(*ret_bh); + + return 0; + } + + status = ocfs2_read_inode_block(inode, ret_bh); + if (status < 0) + mlog_errno(status); + + return status; +} + +/* + * returns < 0 error if the callback will never be called, otherwise + * the result of the lock will be communicated via the callback. + */ +int ocfs2_inode_lock_full_nested(struct inode *inode, + struct buffer_head **ret_bh, + int ex, + int arg_flags, + int subclass) +{ + int status, level, acquired; + u32 dlm_flags; + struct ocfs2_lock_res *lockres = NULL; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct buffer_head *local_bh = NULL; + + mlog(0, "inode %llu, take %s META lock\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno, + ex ? "EXMODE" : "PRMODE"); + + status = 0; + acquired = 0; + /* We'll allow faking a readonly metadata lock for + * rodevices. */ + if (ocfs2_is_hard_readonly(osb)) { + if (ex) + status = -EROFS; + goto getbh; + } + + if ((arg_flags & OCFS2_META_LOCK_GETBH) || + ocfs2_mount_local(osb)) + goto update; + + if (!(arg_flags & OCFS2_META_LOCK_RECOVERY)) + ocfs2_wait_for_recovery(osb); + + lockres = &OCFS2_I(inode)->ip_inode_lockres; + level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; + dlm_flags = 0; + if (arg_flags & OCFS2_META_LOCK_NOQUEUE) + dlm_flags |= DLM_LKF_NOQUEUE; + + status = __ocfs2_cluster_lock(osb, lockres, level, dlm_flags, + arg_flags, subclass, _RET_IP_); + if (status < 0) { + if (status != -EAGAIN) + mlog_errno(status); + goto bail; + } + + /* Notify the error cleanup path to drop the cluster lock. */ + acquired = 1; + + /* We wait twice because a node may have died while we were in + * the lower dlm layers. The second time though, we've + * committed to owning this lock so we don't allow signals to + * abort the operation. */ + if (!(arg_flags & OCFS2_META_LOCK_RECOVERY)) + ocfs2_wait_for_recovery(osb); + +update: + /* + * We only see this flag if we're being called from + * ocfs2_read_locked_inode(). It means we're locking an inode + * which hasn't been populated yet, so clear the refresh flag + * and let the caller handle it. + */ + if (inode_state_read_once(inode) & I_NEW) { + status = 0; + if (lockres) + ocfs2_complete_lock_res_refresh(lockres, 0); + goto bail; + } + + /* This is fun. The caller may want a bh back, or it may + * not. ocfs2_inode_lock_update definitely wants one in, but + * may or may not read one, depending on what's in the + * LVB. The result of all of this is that we've *only* gone to + * disk if we have to, so the complexity is worthwhile. */ + status = ocfs2_inode_lock_update(inode, &local_bh); + if (status < 0) { + if (status != -ENOENT) + mlog_errno(status); + goto bail; + } +getbh: + if (ret_bh) { + status = ocfs2_assign_bh(inode, ret_bh, local_bh); + if (status < 0) { + mlog_errno(status); + goto bail; + } + } + +bail: + if (status < 0) { + if (ret_bh && (*ret_bh)) { + brelse(*ret_bh); + *ret_bh = NULL; + } + if (acquired) + ocfs2_inode_unlock(inode, ex); + } + + brelse(local_bh); + return status; +} + +/* + * This is working around a lock inversion between tasks acquiring DLM + * locks while holding a folio lock and the downconvert thread which + * blocks dlm lock acquiry while acquiring folio locks. + * + * ** These _with_folio variants are only intended to be called from aop + * methods that hold folio locks and return a very specific *positive* error + * code that aop methods pass up to the VFS -- test for errors with != 0. ** + * + * The DLM is called such that it returns -EAGAIN if it would have + * blocked waiting for the downconvert thread. In that case we unlock + * our folio so the downconvert thread can make progress. Once we've + * done this we have to return AOP_TRUNCATED_PAGE so the aop method + * that called us can bubble that back up into the VFS who will then + * immediately retry the aop call. + */ +int ocfs2_inode_lock_with_folio(struct inode *inode, + struct buffer_head **ret_bh, int ex, struct folio *folio) +{ + int ret; + + ret = ocfs2_inode_lock_full(inode, ret_bh, ex, OCFS2_LOCK_NONBLOCK); + if (ret == -EAGAIN) { + folio_unlock(folio); + /* + * If we can't get inode lock immediately, we should not return + * directly here, since this will lead to a softlockup problem. + * The method is to get a blocking lock and immediately unlock + * before returning, this can avoid CPU resource waste due to + * lots of retries, and benefits fairness in getting lock. + */ + if (ocfs2_inode_lock(inode, ret_bh, ex) == 0) + ocfs2_inode_unlock(inode, ex); + ret = AOP_TRUNCATED_PAGE; + } + + return ret; +} + +int ocfs2_inode_lock_atime(struct inode *inode, + struct vfsmount *vfsmnt, + int *level, int wait) +{ + int ret; + + if (wait) + ret = ocfs2_inode_lock(inode, NULL, 0); + else + ret = ocfs2_try_inode_lock(inode, NULL, 0); + + if (ret < 0) { + if (ret != -EAGAIN) + mlog_errno(ret); + return ret; + } + + /* + * If we should update atime, we will get EX lock, + * otherwise we just get PR lock. + */ + if (ocfs2_should_update_atime(inode, vfsmnt)) { + struct buffer_head *bh = NULL; + + ocfs2_inode_unlock(inode, 0); + if (wait) + ret = ocfs2_inode_lock(inode, &bh, 1); + else + ret = ocfs2_try_inode_lock(inode, &bh, 1); + + if (ret < 0) { + if (ret != -EAGAIN) + mlog_errno(ret); + return ret; + } + *level = 1; + if (ocfs2_should_update_atime(inode, vfsmnt)) + ocfs2_update_inode_atime(inode, bh); + brelse(bh); + } else + *level = 0; + + return ret; +} + +void ocfs2_inode_unlock(struct inode *inode, + int ex) +{ + int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; + struct ocfs2_lock_res *lockres = &OCFS2_I(inode)->ip_inode_lockres; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + mlog(0, "inode %llu drop %s META lock\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno, + ex ? "EXMODE" : "PRMODE"); + + if (!ocfs2_is_hard_readonly(osb) && + !ocfs2_mount_local(osb)) + ocfs2_cluster_unlock(osb, lockres, level); +} + +/* + * This _tracker variants are introduced to deal with the recursive cluster + * locking issue. The idea is to keep track of a lock holder on the stack of + * the current process. If there's a lock holder on the stack, we know the + * task context is already protected by cluster locking. Currently, they're + * used in some VFS entry routines. + * + * return < 0 on error, return == 0 if there's no lock holder on the stack + * before this call, return == 1 if this call would be a recursive locking. + * return == -1 if this lock attempt will cause an upgrade which is forbidden. + * + * When taking lock levels into account,we face some different situations. + * + * 1. no lock is held + * In this case, just lock the inode as requested and return 0 + * + * 2. We are holding a lock + * For this situation, things diverges into several cases + * + * wanted holding what to do + * ex ex see 2.1 below + * ex pr see 2.2 below + * pr ex see 2.1 below + * pr pr see 2.1 below + * + * 2.1 lock level that is been held is compatible + * with the wanted level, so no lock action will be tacken. + * + * 2.2 Otherwise, an upgrade is needed, but it is forbidden. + * + * Reason why upgrade within a process is forbidden is that + * lock upgrade may cause dead lock. The following illustrates + * how it happens. + * + * thread on node1 thread on node2 + * ocfs2_inode_lock_tracker(ex=0) + * + * <====== ocfs2_inode_lock_tracker(ex=1) + * + * ocfs2_inode_lock_tracker(ex=1) + */ +int ocfs2_inode_lock_tracker(struct inode *inode, + struct buffer_head **ret_bh, + int ex, + struct ocfs2_lock_holder *oh) +{ + int status = 0; + struct ocfs2_lock_res *lockres; + struct ocfs2_lock_holder *tmp_oh; + struct pid *pid = task_pid(current); + + + lockres = &OCFS2_I(inode)->ip_inode_lockres; + tmp_oh = ocfs2_pid_holder(lockres, pid); + + if (!tmp_oh) { + /* + * This corresponds to the case 1. + * We haven't got any lock before. + */ + status = ocfs2_inode_lock_full(inode, ret_bh, ex, 0); + if (status < 0) { + if (status != -ENOENT) + mlog_errno(status); + return status; + } + + oh->oh_ex = ex; + ocfs2_add_holder(lockres, oh); + return 0; + } + + if (unlikely(ex && !tmp_oh->oh_ex)) { + /* + * case 2.2 upgrade may cause dead lock, forbid it. + */ + mlog(ML_ERROR, "Recursive locking is not permitted to " + "upgrade to EX level from PR level.\n"); + dump_stack(); + return -EINVAL; + } + + /* + * case 2.1 OCFS2_META_LOCK_GETBH flag make ocfs2_inode_lock_full. + * ignore the lock level and just update it. + */ + if (ret_bh) { + status = ocfs2_inode_lock_full(inode, ret_bh, ex, + OCFS2_META_LOCK_GETBH); + if (status < 0) { + if (status != -ENOENT) + mlog_errno(status); + return status; + } + } + return 1; +} + +void ocfs2_inode_unlock_tracker(struct inode *inode, + int ex, + struct ocfs2_lock_holder *oh, + int had_lock) +{ + struct ocfs2_lock_res *lockres; + + lockres = &OCFS2_I(inode)->ip_inode_lockres; + /* had_lock means that the current process already takes the cluster + * lock previously. + * If had_lock is 1, we have nothing to do here. + * If had_lock is 0, we will release the lock. + */ + if (!had_lock) { + ocfs2_inode_unlock(inode, oh->oh_ex); + ocfs2_remove_holder(lockres, oh); + } +} + +int ocfs2_orphan_scan_lock(struct ocfs2_super *osb, u32 *seqno) +{ + struct ocfs2_lock_res *lockres; + struct ocfs2_orphan_scan_lvb *lvb; + int status = 0; + + if (ocfs2_is_hard_readonly(osb)) + return -EROFS; + + if (ocfs2_mount_local(osb)) + return 0; + + lockres = &osb->osb_orphan_scan.os_lockres; + status = ocfs2_cluster_lock(osb, lockres, DLM_LOCK_EX, 0, 0); + if (status < 0) + return status; + + lvb = ocfs2_dlm_lvb(&lockres->l_lksb); + if (ocfs2_dlm_lvb_valid(&lockres->l_lksb) && + lvb->lvb_version == OCFS2_ORPHAN_LVB_VERSION) + *seqno = be32_to_cpu(lvb->lvb_os_seqno); + else + *seqno = osb->osb_orphan_scan.os_seqno + 1; + + return status; +} + +void ocfs2_orphan_scan_unlock(struct ocfs2_super *osb, u32 seqno) +{ + struct ocfs2_lock_res *lockres; + struct ocfs2_orphan_scan_lvb *lvb; + + if (!ocfs2_is_hard_readonly(osb) && !ocfs2_mount_local(osb)) { + lockres = &osb->osb_orphan_scan.os_lockres; + lvb = ocfs2_dlm_lvb(&lockres->l_lksb); + lvb->lvb_version = OCFS2_ORPHAN_LVB_VERSION; + lvb->lvb_os_seqno = cpu_to_be32(seqno); + ocfs2_cluster_unlock(osb, lockres, DLM_LOCK_EX); + } +} + +int ocfs2_super_lock(struct ocfs2_super *osb, + int ex) +{ + int status = 0; + int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; + struct ocfs2_lock_res *lockres = &osb->osb_super_lockres; + + if (ocfs2_is_hard_readonly(osb)) + return -EROFS; + + if (ocfs2_mount_local(osb)) + goto bail; + + status = ocfs2_cluster_lock(osb, lockres, level, 0, 0); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + /* The super block lock path is really in the best position to + * know when resources covered by the lock need to be + * refreshed, so we do it here. Of course, making sense of + * everything is up to the caller :) */ + status = ocfs2_should_refresh_lock_res(lockres); + if (status) { + status = ocfs2_refresh_slot_info(osb); + + ocfs2_complete_lock_res_refresh(lockres, status); + + if (status < 0) { + ocfs2_cluster_unlock(osb, lockres, level); + mlog_errno(status); + } + ocfs2_track_lock_refresh(lockres); + } +bail: + return status; +} + +void ocfs2_super_unlock(struct ocfs2_super *osb, + int ex) +{ + int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; + struct ocfs2_lock_res *lockres = &osb->osb_super_lockres; + + if (!ocfs2_mount_local(osb)) + ocfs2_cluster_unlock(osb, lockres, level); +} + +int ocfs2_rename_lock(struct ocfs2_super *osb) +{ + int status; + struct ocfs2_lock_res *lockres = &osb->osb_rename_lockres; + + if (ocfs2_is_hard_readonly(osb)) + return -EROFS; + + if (ocfs2_mount_local(osb)) + return 0; + + status = ocfs2_cluster_lock(osb, lockres, DLM_LOCK_EX, 0, 0); + if (status < 0) + mlog_errno(status); + + return status; +} + +void ocfs2_rename_unlock(struct ocfs2_super *osb) +{ + struct ocfs2_lock_res *lockres = &osb->osb_rename_lockres; + + if (!ocfs2_mount_local(osb)) + ocfs2_cluster_unlock(osb, lockres, DLM_LOCK_EX); +} + +int ocfs2_nfs_sync_lock(struct ocfs2_super *osb, int ex) +{ + int status; + struct ocfs2_lock_res *lockres = &osb->osb_nfs_sync_lockres; + + if (ocfs2_is_hard_readonly(osb)) + return -EROFS; + + if (ex) + down_write(&osb->nfs_sync_rwlock); + else + down_read(&osb->nfs_sync_rwlock); + + if (ocfs2_mount_local(osb)) + return 0; + + status = ocfs2_cluster_lock(osb, lockres, ex ? LKM_EXMODE : LKM_PRMODE, + 0, 0); + if (status < 0) { + mlog(ML_ERROR, "lock on nfs sync lock failed %d\n", status); + + if (ex) + up_write(&osb->nfs_sync_rwlock); + else + up_read(&osb->nfs_sync_rwlock); + } + + return status; +} + +void ocfs2_nfs_sync_unlock(struct ocfs2_super *osb, int ex) +{ + struct ocfs2_lock_res *lockres = &osb->osb_nfs_sync_lockres; + + if (!ocfs2_mount_local(osb)) + ocfs2_cluster_unlock(osb, lockres, + ex ? LKM_EXMODE : LKM_PRMODE); + if (ex) + up_write(&osb->nfs_sync_rwlock); + else + up_read(&osb->nfs_sync_rwlock); +} + +int ocfs2_trim_fs_lock(struct ocfs2_super *osb, + struct ocfs2_trim_fs_info *info, int trylock) +{ + int status; + struct ocfs2_trim_fs_lvb *lvb; + struct ocfs2_lock_res *lockres = &osb->osb_trim_fs_lockres; + + if (info) + info->tf_valid = 0; + + if (ocfs2_is_hard_readonly(osb)) + return -EROFS; + + if (ocfs2_mount_local(osb)) + return 0; + + status = ocfs2_cluster_lock(osb, lockres, DLM_LOCK_EX, + trylock ? DLM_LKF_NOQUEUE : 0, 0); + if (status < 0) { + if (status != -EAGAIN) + mlog_errno(status); + return status; + } + + if (info) { + lvb = ocfs2_dlm_lvb(&lockres->l_lksb); + if (ocfs2_dlm_lvb_valid(&lockres->l_lksb) && + lvb->lvb_version == OCFS2_TRIMFS_LVB_VERSION) { + info->tf_valid = 1; + info->tf_success = lvb->lvb_success; + info->tf_nodenum = be32_to_cpu(lvb->lvb_nodenum); + info->tf_start = be64_to_cpu(lvb->lvb_start); + info->tf_len = be64_to_cpu(lvb->lvb_len); + info->tf_minlen = be64_to_cpu(lvb->lvb_minlen); + info->tf_trimlen = be64_to_cpu(lvb->lvb_trimlen); + } + } + + return status; +} + +void ocfs2_trim_fs_unlock(struct ocfs2_super *osb, + struct ocfs2_trim_fs_info *info) +{ + struct ocfs2_trim_fs_lvb *lvb; + struct ocfs2_lock_res *lockres = &osb->osb_trim_fs_lockres; + + if (ocfs2_mount_local(osb)) + return; + + if (info) { + lvb = ocfs2_dlm_lvb(&lockres->l_lksb); + lvb->lvb_version = OCFS2_TRIMFS_LVB_VERSION; + lvb->lvb_success = info->tf_success; + lvb->lvb_nodenum = cpu_to_be32(info->tf_nodenum); + lvb->lvb_start = cpu_to_be64(info->tf_start); + lvb->lvb_len = cpu_to_be64(info->tf_len); + lvb->lvb_minlen = cpu_to_be64(info->tf_minlen); + lvb->lvb_trimlen = cpu_to_be64(info->tf_trimlen); + } + + ocfs2_cluster_unlock(osb, lockres, DLM_LOCK_EX); +} + +int ocfs2_dentry_lock(struct dentry *dentry, int ex) +{ + int ret; + int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; + struct ocfs2_dentry_lock *dl = dentry->d_fsdata; + struct ocfs2_super *osb = OCFS2_SB(dentry->d_sb); + + BUG_ON(!dl); + + if (ocfs2_is_hard_readonly(osb)) { + if (ex) + return -EROFS; + return 0; + } + + if (ocfs2_mount_local(osb)) + return 0; + + ret = ocfs2_cluster_lock(osb, &dl->dl_lockres, level, 0, 0); + if (ret < 0) + mlog_errno(ret); + + return ret; +} + +void ocfs2_dentry_unlock(struct dentry *dentry, int ex) +{ + int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; + struct ocfs2_dentry_lock *dl = dentry->d_fsdata; + struct ocfs2_super *osb = OCFS2_SB(dentry->d_sb); + + if (!ocfs2_is_hard_readonly(osb) && !ocfs2_mount_local(osb)) + ocfs2_cluster_unlock(osb, &dl->dl_lockres, level); +} + +/* Reference counting of the dlm debug structure. We want this because + * open references on the debug inodes can live on after a mount, so + * we can't rely on the ocfs2_super to always exist. */ +static void ocfs2_dlm_debug_free(struct kref *kref) +{ + struct ocfs2_dlm_debug *dlm_debug; + + dlm_debug = container_of(kref, struct ocfs2_dlm_debug, d_refcnt); + + kfree(dlm_debug); +} + +void ocfs2_put_dlm_debug(struct ocfs2_dlm_debug *dlm_debug) +{ + if (dlm_debug) + kref_put(&dlm_debug->d_refcnt, ocfs2_dlm_debug_free); +} + +static void ocfs2_get_dlm_debug(struct ocfs2_dlm_debug *debug) +{ + kref_get(&debug->d_refcnt); +} + +struct ocfs2_dlm_debug *ocfs2_new_dlm_debug(void) +{ + struct ocfs2_dlm_debug *dlm_debug; + + dlm_debug = kmalloc_obj(struct ocfs2_dlm_debug); + if (!dlm_debug) { + mlog_errno(-ENOMEM); + goto out; + } + + kref_init(&dlm_debug->d_refcnt); + INIT_LIST_HEAD(&dlm_debug->d_lockres_tracking); + dlm_debug->d_filter_secs = 0; +out: + return dlm_debug; +} + +/* Access to this is arbitrated for us via seq_file->sem. */ +struct ocfs2_dlm_seq_priv { + struct ocfs2_dlm_debug *p_dlm_debug; + struct ocfs2_lock_res p_iter_res; + struct ocfs2_lock_res p_tmp_res; +}; + +static struct ocfs2_lock_res *ocfs2_dlm_next_res(struct ocfs2_lock_res *start, + struct ocfs2_dlm_seq_priv *priv) +{ + struct ocfs2_lock_res *iter, *ret = NULL; + struct ocfs2_dlm_debug *dlm_debug = priv->p_dlm_debug; + + assert_spin_locked(&ocfs2_dlm_tracking_lock); + + list_for_each_entry(iter, &start->l_debug_list, l_debug_list) { + /* discover the head of the list */ + if (&iter->l_debug_list == &dlm_debug->d_lockres_tracking) { + mlog(0, "End of list found, %p\n", ret); + break; + } + + /* We track our "dummy" iteration lockres' by a NULL + * l_ops field. */ + if (iter->l_ops != NULL) { + ret = iter; + break; + } + } + + return ret; +} + +static void *ocfs2_dlm_seq_start(struct seq_file *m, loff_t *pos) +{ + struct ocfs2_dlm_seq_priv *priv = m->private; + struct ocfs2_lock_res *iter; + + spin_lock(&ocfs2_dlm_tracking_lock); + iter = ocfs2_dlm_next_res(&priv->p_iter_res, priv); + if (iter) { + /* Since lockres' have the lifetime of their container + * (which can be inodes, ocfs2_supers, etc) we want to + * copy this out to a temporary lockres while still + * under the spinlock. Obviously after this we can't + * trust any pointers on the copy returned, but that's + * ok as the information we want isn't typically held + * in them. */ + priv->p_tmp_res = *iter; + iter = &priv->p_tmp_res; + } + spin_unlock(&ocfs2_dlm_tracking_lock); + + return iter; +} + +static void ocfs2_dlm_seq_stop(struct seq_file *m, void *v) +{ +} + +static void *ocfs2_dlm_seq_next(struct seq_file *m, void *v, loff_t *pos) +{ + struct ocfs2_dlm_seq_priv *priv = m->private; + struct ocfs2_lock_res *iter = v; + struct ocfs2_lock_res *dummy = &priv->p_iter_res; + + (*pos)++; + spin_lock(&ocfs2_dlm_tracking_lock); + iter = ocfs2_dlm_next_res(iter, priv); + list_del_init(&dummy->l_debug_list); + if (iter) { + list_add(&dummy->l_debug_list, &iter->l_debug_list); + priv->p_tmp_res = *iter; + iter = &priv->p_tmp_res; + } + spin_unlock(&ocfs2_dlm_tracking_lock); + + return iter; +} + +/* + * Version is used by debugfs.ocfs2 to determine the format being used + * + * New in version 2 + * - Lock stats printed + * New in version 3 + * - Max time in lock stats is in usecs (instead of nsecs) + * New in version 4 + * - Add last pr/ex unlock times and first lock wait time in usecs + */ +#define OCFS2_DLM_DEBUG_STR_VERSION 4 + +/* + * The debug iterator snapshots lockres by value, so a userspace-stack LVB + * pointer copied from the original lockres must be rebased to the copied + * lksb before the dump walks the raw bytes. + */ +static void ocfs2_dlm_seq_rebase_lvb(struct ocfs2_lock_res *lockres) +{ + if (!ocfs2_stack_supports_plocks()) + return; + + if (lockres->l_lksb.lksb_fsdlm.sb_lvbptr) + lockres->l_lksb.lksb_fsdlm.sb_lvbptr = + (char *)&lockres->l_lksb + sizeof(struct dlm_lksb); +} + +static int ocfs2_dlm_seq_show(struct seq_file *m, void *v) +{ + int i; + char *lvb; + struct ocfs2_lock_res *lockres = v; +#ifdef CONFIG_OCFS2_FS_STATS + u64 now, last; + struct ocfs2_dlm_debug *dlm_debug = + ((struct ocfs2_dlm_seq_priv *)m->private)->p_dlm_debug; +#endif + + if (!lockres) + return -EINVAL; + +#ifdef CONFIG_OCFS2_FS_STATS + if (!lockres->l_lock_wait && dlm_debug->d_filter_secs) { + now = ktime_to_us(ktime_get_real()); + last = max(lockres->l_lock_prmode.ls_last, + lockres->l_lock_exmode.ls_last); + /* + * Use d_filter_secs field to filter lock resources dump, + * the default d_filter_secs(0) value filters nothing, + * otherwise, only dump the last N seconds active lock + * resources. + */ + if (div_u64(now - last, 1000000) > dlm_debug->d_filter_secs) + return 0; + } +#endif + + seq_printf(m, "0x%x\t", OCFS2_DLM_DEBUG_STR_VERSION); + + if (lockres->l_type == OCFS2_LOCK_TYPE_DENTRY) + seq_printf(m, "%.*s%08x\t", OCFS2_DENTRY_LOCK_INO_START - 1, + lockres->l_name, + (unsigned int)ocfs2_get_dentry_lock_ino(lockres)); + else + seq_printf(m, "%.*s\t", OCFS2_LOCK_ID_MAX_LEN, lockres->l_name); + + seq_printf(m, "%d\t" + "0x%lx\t" + "0x%x\t" + "0x%x\t" + "%u\t" + "%u\t" + "%d\t" + "%d\t", + lockres->l_level, + lockres->l_flags, + lockres->l_action, + lockres->l_unlock_action, + lockres->l_ro_holders, + lockres->l_ex_holders, + lockres->l_requested, + lockres->l_blocking); + + /* Dump the raw LVB */ + ocfs2_dlm_seq_rebase_lvb(lockres); + lvb = ocfs2_dlm_lvb(&lockres->l_lksb); + for(i = 0; i < DLM_LVB_LEN; i++) + seq_printf(m, "0x%x\t", lvb[i]); + +#ifdef CONFIG_OCFS2_FS_STATS +# define lock_num_prmode(_l) ((_l)->l_lock_prmode.ls_gets) +# define lock_num_exmode(_l) ((_l)->l_lock_exmode.ls_gets) +# define lock_num_prmode_failed(_l) ((_l)->l_lock_prmode.ls_fail) +# define lock_num_exmode_failed(_l) ((_l)->l_lock_exmode.ls_fail) +# define lock_total_prmode(_l) ((_l)->l_lock_prmode.ls_total) +# define lock_total_exmode(_l) ((_l)->l_lock_exmode.ls_total) +# define lock_max_prmode(_l) ((_l)->l_lock_prmode.ls_max) +# define lock_max_exmode(_l) ((_l)->l_lock_exmode.ls_max) +# define lock_refresh(_l) ((_l)->l_lock_refresh) +# define lock_last_prmode(_l) ((_l)->l_lock_prmode.ls_last) +# define lock_last_exmode(_l) ((_l)->l_lock_exmode.ls_last) +# define lock_wait(_l) ((_l)->l_lock_wait) +#else +# define lock_num_prmode(_l) (0) +# define lock_num_exmode(_l) (0) +# define lock_num_prmode_failed(_l) (0) +# define lock_num_exmode_failed(_l) (0) +# define lock_total_prmode(_l) (0ULL) +# define lock_total_exmode(_l) (0ULL) +# define lock_max_prmode(_l) (0) +# define lock_max_exmode(_l) (0) +# define lock_refresh(_l) (0) +# define lock_last_prmode(_l) (0ULL) +# define lock_last_exmode(_l) (0ULL) +# define lock_wait(_l) (0ULL) +#endif + /* The following seq_print was added in version 2 of this output */ + seq_printf(m, "%u\t" + "%u\t" + "%u\t" + "%u\t" + "%llu\t" + "%llu\t" + "%u\t" + "%u\t" + "%u\t" + "%llu\t" + "%llu\t" + "%llu\t", + lock_num_prmode(lockres), + lock_num_exmode(lockres), + lock_num_prmode_failed(lockres), + lock_num_exmode_failed(lockres), + lock_total_prmode(lockres), + lock_total_exmode(lockres), + lock_max_prmode(lockres), + lock_max_exmode(lockres), + lock_refresh(lockres), + lock_last_prmode(lockres), + lock_last_exmode(lockres), + lock_wait(lockres)); + + /* End the line */ + seq_printf(m, "\n"); + return 0; +} + +static const struct seq_operations ocfs2_dlm_seq_ops = { + .start = ocfs2_dlm_seq_start, + .stop = ocfs2_dlm_seq_stop, + .next = ocfs2_dlm_seq_next, + .show = ocfs2_dlm_seq_show, +}; + +static int ocfs2_dlm_debug_release(struct inode *inode, struct file *file) +{ + struct seq_file *seq = file->private_data; + struct ocfs2_dlm_seq_priv *priv = seq->private; + struct ocfs2_lock_res *res = &priv->p_iter_res; + + ocfs2_remove_lockres_tracking(res); + ocfs2_put_dlm_debug(priv->p_dlm_debug); + return seq_release_private(inode, file); +} + +static int ocfs2_dlm_debug_open(struct inode *inode, struct file *file) +{ + struct ocfs2_dlm_seq_priv *priv; + struct ocfs2_super *osb; + + priv = __seq_open_private(file, &ocfs2_dlm_seq_ops, sizeof(*priv)); + if (!priv) { + mlog_errno(-ENOMEM); + return -ENOMEM; + } + + osb = inode->i_private; + ocfs2_get_dlm_debug(osb->osb_dlm_debug); + priv->p_dlm_debug = osb->osb_dlm_debug; + INIT_LIST_HEAD(&priv->p_iter_res.l_debug_list); + + ocfs2_add_lockres_tracking(&priv->p_iter_res, + priv->p_dlm_debug); + + return 0; +} + +static const struct file_operations ocfs2_dlm_debug_fops = { + .open = ocfs2_dlm_debug_open, + .release = ocfs2_dlm_debug_release, + .read = seq_read, + .llseek = seq_lseek, +}; + +static void ocfs2_dlm_init_debug(struct ocfs2_super *osb) +{ + struct ocfs2_dlm_debug *dlm_debug = osb->osb_dlm_debug; + + debugfs_create_file("locking_state", S_IFREG|S_IRUSR, + osb->osb_debug_root, osb, &ocfs2_dlm_debug_fops); + + debugfs_create_u32("locking_filter", 0600, osb->osb_debug_root, + &dlm_debug->d_filter_secs); + ocfs2_get_dlm_debug(dlm_debug); +} + +static void ocfs2_dlm_shutdown_debug(struct ocfs2_super *osb) +{ + struct ocfs2_dlm_debug *dlm_debug = osb->osb_dlm_debug; + + if (dlm_debug) + ocfs2_put_dlm_debug(dlm_debug); +} + +int ocfs2_dlm_init(struct ocfs2_super *osb) +{ + int status = 0; + struct ocfs2_cluster_connection *conn = NULL; + + if (ocfs2_mount_local(osb)) { + osb->node_num = 0; + goto local; + } + + ocfs2_dlm_init_debug(osb); + + /* launch downconvert thread */ + osb->dc_task = kthread_run(ocfs2_downconvert_thread, osb, "ocfs2dc-%s", + osb->uuid_str); + if (IS_ERR(osb->dc_task)) { + status = PTR_ERR(osb->dc_task); + osb->dc_task = NULL; + mlog_errno(status); + goto bail; + } + + /* for now, uuid == domain */ + status = ocfs2_cluster_connect(osb->osb_cluster_stack, + osb->osb_cluster_name, + strlen(osb->osb_cluster_name), + osb->uuid_str, + strlen(osb->uuid_str), + &lproto, ocfs2_do_node_down, osb, + &conn); + if (status) { + mlog_errno(status); + goto bail; + } + + status = ocfs2_cluster_this_node(conn, &osb->node_num); + if (status < 0) { + mlog_errno(status); + mlog(ML_ERROR, + "could not find this host's node number\n"); + ocfs2_cluster_disconnect(conn, 0); + goto bail; + } + +local: + ocfs2_super_lock_res_init(&osb->osb_super_lockres, osb); + ocfs2_rename_lock_res_init(&osb->osb_rename_lockres, osb); + ocfs2_nfs_sync_lock_init(osb); + ocfs2_orphan_scan_lock_res_init(&osb->osb_orphan_scan.os_lockres, osb); + + osb->cconn = conn; +bail: + if (status < 0) { + ocfs2_dlm_shutdown_debug(osb); + if (osb->dc_task) + kthread_stop(osb->dc_task); + } + + return status; +} + +void ocfs2_dlm_shutdown(struct ocfs2_super *osb, + int hangup_pending) +{ + ocfs2_drop_osb_locks(osb); + + /* + * Now that we have dropped all locks and ocfs2_dismount_volume() + * has disabled recovery, the DLM won't be talking to us. It's + * safe to tear things down before disconnecting the cluster. + */ + + if (osb->dc_task) { + kthread_stop(osb->dc_task); + osb->dc_task = NULL; + } + + ocfs2_lock_res_free(&osb->osb_super_lockres); + ocfs2_lock_res_free(&osb->osb_rename_lockres); + ocfs2_lock_res_free(&osb->osb_nfs_sync_lockres); + ocfs2_lock_res_free(&osb->osb_orphan_scan.os_lockres); + + if (osb->cconn) { + ocfs2_cluster_disconnect(osb->cconn, hangup_pending); + osb->cconn = NULL; + + ocfs2_dlm_shutdown_debug(osb); + } +} + +static int ocfs2_drop_lock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres) +{ + int ret; + unsigned long flags; + u32 lkm_flags = 0; + + /* We didn't get anywhere near actually using this lockres. */ + if (!(lockres->l_flags & OCFS2_LOCK_INITIALIZED)) + goto out; + + if (lockres->l_ops->flags & LOCK_TYPE_USES_LVB) + lkm_flags |= DLM_LKF_VALBLK; + + spin_lock_irqsave(&lockres->l_lock, flags); + + mlog_bug_on_msg(!(lockres->l_flags & OCFS2_LOCK_FREEING), + "lockres %s, flags 0x%lx\n", + lockres->l_name, lockres->l_flags); + + while (lockres->l_flags & OCFS2_LOCK_BUSY) { + mlog(0, "waiting on busy lock \"%s\": flags = %lx, action = " + "%u, unlock_action = %u\n", + lockres->l_name, lockres->l_flags, lockres->l_action, + lockres->l_unlock_action); + + spin_unlock_irqrestore(&lockres->l_lock, flags); + + /* XXX: Today we just wait on any busy + * locks... Perhaps we need to cancel converts in the + * future? */ + ocfs2_wait_on_busy_lock(lockres); + + spin_lock_irqsave(&lockres->l_lock, flags); + } + + if (lockres->l_ops->flags & LOCK_TYPE_USES_LVB) { + if (lockres->l_flags & OCFS2_LOCK_ATTACHED && + lockres->l_level == DLM_LOCK_EX && + !(lockres->l_flags & OCFS2_LOCK_NEEDS_REFRESH)) + lockres->l_ops->set_lvb(lockres); + } + + if (lockres->l_flags & OCFS2_LOCK_BUSY) + mlog(ML_ERROR, "destroying busy lock: \"%s\"\n", + lockres->l_name); + if (lockres->l_flags & OCFS2_LOCK_BLOCKED) + mlog(0, "destroying blocked lock: \"%s\"\n", lockres->l_name); + + if (!(lockres->l_flags & OCFS2_LOCK_ATTACHED)) { + spin_unlock_irqrestore(&lockres->l_lock, flags); + goto out; + } + + lockres_clear_flags(lockres, OCFS2_LOCK_ATTACHED); + + /* make sure we never get here while waiting for an ast to + * fire. */ + BUG_ON(lockres->l_action != OCFS2_AST_INVALID); + + /* is this necessary? */ + lockres_or_flags(lockres, OCFS2_LOCK_BUSY); + lockres->l_unlock_action = OCFS2_UNLOCK_DROP_LOCK; + spin_unlock_irqrestore(&lockres->l_lock, flags); + + mlog(0, "lock %s\n", lockres->l_name); + + ret = ocfs2_dlm_unlock(osb->cconn, &lockres->l_lksb, lkm_flags); + if (ret) { + ocfs2_log_dlm_error("ocfs2_dlm_unlock", ret, lockres); + mlog(ML_ERROR, "lockres flags: %lu\n", lockres->l_flags); + ocfs2_dlm_dump_lksb(&lockres->l_lksb); + BUG(); + } + mlog(0, "lock %s, successful return from ocfs2_dlm_unlock\n", + lockres->l_name); + + ocfs2_wait_on_busy_lock(lockres); +out: + return 0; +} + +static void ocfs2_process_blocked_lock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres); + +/* Mark the lockres as being dropped. It will no longer be + * queued if blocking, but we still may have to wait on it + * being dequeued from the downconvert thread before we can consider + * it safe to drop. + * + * You can *not* attempt to call cluster_lock on this lockres anymore. */ +void ocfs2_mark_lockres_freeing(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres) +{ + int status; + struct ocfs2_mask_waiter mw; + unsigned long flags, flags2; + + ocfs2_init_mask_waiter(&mw); + + spin_lock_irqsave(&lockres->l_lock, flags); + lockres->l_flags |= OCFS2_LOCK_FREEING; + if (lockres->l_flags & OCFS2_LOCK_QUEUED && current == osb->dc_task) { + /* + * We know the downconvert is queued but not in progress + * because we are the downconvert thread and processing + * different lock. So we can just remove the lock from the + * queue. This is not only an optimization but also a way + * to avoid the following deadlock: + * ocfs2_dentry_post_unlock() + * ocfs2_dentry_lock_put() + * ocfs2_drop_dentry_lock() + * iput() + * ocfs2_evict_inode() + * ocfs2_clear_inode() + * ocfs2_mark_lockres_freeing() + * ... blocks waiting for OCFS2_LOCK_QUEUED + * since we are the downconvert thread which + * should clear the flag. + */ + spin_unlock_irqrestore(&lockres->l_lock, flags); + spin_lock_irqsave(&osb->dc_task_lock, flags2); + list_del_init(&lockres->l_blocked_list); + osb->blocked_lock_count--; + spin_unlock_irqrestore(&osb->dc_task_lock, flags2); + /* + * Warn if we recurse into another post_unlock call. Strictly + * speaking it isn't a problem but we need to be careful if + * that happens (stack overflow, deadlocks, ...) so warn if + * ocfs2 grows a path for which this can happen. + */ + WARN_ON_ONCE(lockres->l_ops->post_unlock); + /* Since the lock is freeing we don't do much in the fn below */ + ocfs2_process_blocked_lock(osb, lockres); + return; + } + while (lockres->l_flags & OCFS2_LOCK_QUEUED) { + lockres_add_mask_waiter(lockres, &mw, OCFS2_LOCK_QUEUED, 0); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + mlog(0, "Waiting on lockres %s\n", lockres->l_name); + + status = ocfs2_wait_for_mask(&mw); + if (status) + mlog_errno(status); + + spin_lock_irqsave(&lockres->l_lock, flags); + } + spin_unlock_irqrestore(&lockres->l_lock, flags); +} + +void ocfs2_simple_drop_lockres(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres) +{ + int ret; + + ocfs2_mark_lockres_freeing(osb, lockres); + ret = ocfs2_drop_lock(osb, lockres); + if (ret) + mlog_errno(ret); +} + +static void ocfs2_drop_osb_locks(struct ocfs2_super *osb) +{ + ocfs2_simple_drop_lockres(osb, &osb->osb_super_lockres); + ocfs2_simple_drop_lockres(osb, &osb->osb_rename_lockres); + ocfs2_simple_drop_lockres(osb, &osb->osb_nfs_sync_lockres); + ocfs2_simple_drop_lockres(osb, &osb->osb_orphan_scan.os_lockres); +} + +int ocfs2_drop_inode_locks(struct inode *inode) +{ + int status, err; + + /* No need to call ocfs2_mark_lockres_freeing here - + * ocfs2_clear_inode has done it for us. */ + + err = ocfs2_drop_lock(OCFS2_SB(inode->i_sb), + &OCFS2_I(inode)->ip_open_lockres); + if (err < 0) + mlog_errno(err); + + status = err; + + err = ocfs2_drop_lock(OCFS2_SB(inode->i_sb), + &OCFS2_I(inode)->ip_inode_lockres); + if (err < 0) + mlog_errno(err); + if (err < 0 && !status) + status = err; + + err = ocfs2_drop_lock(OCFS2_SB(inode->i_sb), + &OCFS2_I(inode)->ip_rw_lockres); + if (err < 0) + mlog_errno(err); + if (err < 0 && !status) + status = err; + + return status; +} + +static unsigned int ocfs2_prepare_downconvert(struct ocfs2_lock_res *lockres, + int new_level) +{ + assert_spin_locked(&lockres->l_lock); + + BUG_ON(lockres->l_blocking <= DLM_LOCK_NL); + + if (lockres->l_level <= new_level) { + mlog(ML_ERROR, "lockres %s, lvl %d <= %d, blcklst %d, mask %d, " + "type %d, flags 0x%lx, hold %d %d, act %d %d, req %d, " + "block %d, pgen %d\n", lockres->l_name, lockres->l_level, + new_level, list_empty(&lockres->l_blocked_list), + list_empty(&lockres->l_mask_waiters), lockres->l_type, + lockres->l_flags, lockres->l_ro_holders, + lockres->l_ex_holders, lockres->l_action, + lockres->l_unlock_action, lockres->l_requested, + lockres->l_blocking, lockres->l_pending_gen); + BUG(); + } + + mlog(ML_BASTS, "lockres %s, level %d => %d, blocking %d\n", + lockres->l_name, lockres->l_level, new_level, lockres->l_blocking); + + lockres->l_action = OCFS2_AST_DOWNCONVERT; + lockres->l_requested = new_level; + lockres_or_flags(lockres, OCFS2_LOCK_BUSY); + return lockres_set_pending(lockres); +} + +static int ocfs2_downconvert_lock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres, + int new_level, + int lvb, + unsigned int generation) +{ + int ret; + u32 dlm_flags = DLM_LKF_CONVERT; + + mlog(ML_BASTS, "lockres %s, level %d => %d\n", lockres->l_name, + lockres->l_level, new_level); + + /* + * On DLM_LKF_VALBLK, fsdlm behaves differently with o2cb. It always + * expects DLM_LKF_VALBLK being set if the LKB has LVB, so that + * we can recover correctly from node failure. Otherwise, we may get + * invalid LVB in LKB, but without DLM_SBF_VALNOTVALID being set. + */ + if (ocfs2_userspace_stack(osb) && + lockres->l_ops->flags & LOCK_TYPE_USES_LVB) + lvb = 1; + + if (lvb) + dlm_flags |= DLM_LKF_VALBLK; + + ret = ocfs2_dlm_lock(osb->cconn, + new_level, + &lockres->l_lksb, + dlm_flags, + lockres->l_name, + OCFS2_LOCK_ID_MAX_LEN - 1); + lockres_clear_pending(lockres, generation, osb); + if (ret) { + ocfs2_log_dlm_error("ocfs2_dlm_lock", ret, lockres); + ocfs2_recover_from_dlm_error(lockres, 1); + goto bail; + } + + ret = 0; +bail: + return ret; +} + +/* returns 1 when the caller should unlock and call ocfs2_dlm_unlock */ +static int ocfs2_prepare_cancel_convert(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres) +{ + assert_spin_locked(&lockres->l_lock); + + if (lockres->l_unlock_action == OCFS2_UNLOCK_CANCEL_CONVERT) { + /* If we're already trying to cancel a lock conversion + * then just drop the spinlock and allow the caller to + * requeue this lock. */ + mlog(ML_BASTS, "lockres %s, skip convert\n", lockres->l_name); + return 0; + } + + /* were we in a convert when we got the bast fire? */ + BUG_ON(lockres->l_action != OCFS2_AST_CONVERT && + lockres->l_action != OCFS2_AST_DOWNCONVERT); + /* set things up for the unlockast to know to just + * clear out the ast_action and unset busy, etc. */ + lockres->l_unlock_action = OCFS2_UNLOCK_CANCEL_CONVERT; + + mlog_bug_on_msg(!(lockres->l_flags & OCFS2_LOCK_BUSY), + "lock %s, invalid flags: 0x%lx\n", + lockres->l_name, lockres->l_flags); + + mlog(ML_BASTS, "lockres %s\n", lockres->l_name); + + return 1; +} + +static int ocfs2_cancel_convert(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres) +{ + int ret; + + ret = ocfs2_dlm_unlock(osb->cconn, &lockres->l_lksb, + DLM_LKF_CANCEL); + if (ret) { + ocfs2_log_dlm_error("ocfs2_dlm_unlock", ret, lockres); + ocfs2_recover_from_dlm_error(lockres, 0); + } + + mlog(ML_BASTS, "lockres %s\n", lockres->l_name); + + return ret; +} + +static int ocfs2_unblock_lock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres, + struct ocfs2_unblock_ctl *ctl) +{ + unsigned long flags; + int blocking; + int new_level; + int level; + int ret = 0; + int set_lvb = 0; + unsigned int gen; + + spin_lock_irqsave(&lockres->l_lock, flags); + +recheck: + /* + * Is it still blocking? If not, we have no more work to do. + */ + if (!(lockres->l_flags & OCFS2_LOCK_BLOCKED)) { + BUG_ON(lockres->l_blocking != DLM_LOCK_NL); + spin_unlock_irqrestore(&lockres->l_lock, flags); + ret = 0; + goto leave; + } + + if (lockres->l_flags & OCFS2_LOCK_BUSY) { + /* XXX + * This is a *big* race. The OCFS2_LOCK_PENDING flag + * exists entirely for one reason - another thread has set + * OCFS2_LOCK_BUSY, but has *NOT* yet called dlm_lock(). + * + * If we do ocfs2_cancel_convert() before the other thread + * calls dlm_lock(), our cancel will do nothing. We will + * get no ast, and we will have no way of knowing the + * cancel failed. Meanwhile, the other thread will call + * into dlm_lock() and wait...forever. + * + * Why forever? Because another node has asked for the + * lock first; that's why we're here in unblock_lock(). + * + * The solution is OCFS2_LOCK_PENDING. When PENDING is + * set, we just requeue the unblock. Only when the other + * thread has called dlm_lock() and cleared PENDING will + * we then cancel their request. + * + * All callers of dlm_lock() must set OCFS2_DLM_PENDING + * at the same time they set OCFS2_DLM_BUSY. They must + * clear OCFS2_DLM_PENDING after dlm_lock() returns. + */ + if (lockres->l_flags & OCFS2_LOCK_PENDING) { + mlog(ML_BASTS, "lockres %s, ReQ: Pending\n", + lockres->l_name); + goto leave_requeue; + } + + ctl->requeue = 1; + ret = ocfs2_prepare_cancel_convert(osb, lockres); + spin_unlock_irqrestore(&lockres->l_lock, flags); + if (ret) { + ret = ocfs2_cancel_convert(osb, lockres); + if (ret < 0) + mlog_errno(ret); + } + goto leave; + } + + /* + * This prevents livelocks. OCFS2_LOCK_UPCONVERT_FINISHING flag is + * set when the ast is received for an upconvert just before the + * OCFS2_LOCK_BUSY flag is cleared. Now if the fs received a bast + * on the heels of the ast, we want to delay the downconvert just + * enough to allow the up requester to do its task. Because this + * lock is in the blocked queue, the lock will be downconverted + * as soon as the requester is done with the lock. + */ + if (lockres->l_flags & OCFS2_LOCK_UPCONVERT_FINISHING) + goto leave_requeue; + + /* + * How can we block and yet be at NL? We were trying to upconvert + * from NL and got canceled. The code comes back here, and now + * we notice and clear BLOCKING. + */ + if (lockres->l_level == DLM_LOCK_NL) { + BUG_ON(lockres->l_ex_holders || lockres->l_ro_holders); + mlog(ML_BASTS, "lockres %s, Aborting dc\n", lockres->l_name); + lockres->l_blocking = DLM_LOCK_NL; + lockres_clear_flags(lockres, OCFS2_LOCK_BLOCKED); + spin_unlock_irqrestore(&lockres->l_lock, flags); + goto leave; + } + + /* if we're blocking an exclusive and we have *any* holders, + * then requeue. */ + if ((lockres->l_blocking == DLM_LOCK_EX) + && (lockres->l_ex_holders || lockres->l_ro_holders)) { + mlog(ML_BASTS, "lockres %s, ReQ: EX/PR Holders %u,%u\n", + lockres->l_name, lockres->l_ex_holders, + lockres->l_ro_holders); + goto leave_requeue; + } + + /* If it's a PR we're blocking, then only + * requeue if we've got any EX holders */ + if (lockres->l_blocking == DLM_LOCK_PR && + lockres->l_ex_holders) { + mlog(ML_BASTS, "lockres %s, ReQ: EX Holders %u\n", + lockres->l_name, lockres->l_ex_holders); + goto leave_requeue; + } + + /* + * Can we get a lock in this state if the holder counts are + * zero? The meta data unblock code used to check this. + */ + if ((lockres->l_ops->flags & LOCK_TYPE_REQUIRES_REFRESH) + && (lockres->l_flags & OCFS2_LOCK_REFRESHING)) { + mlog(ML_BASTS, "lockres %s, ReQ: Lock Refreshing\n", + lockres->l_name); + goto leave_requeue; + } + + new_level = ocfs2_highest_compat_lock_level(lockres->l_blocking); + + if (lockres->l_ops->check_downconvert + && !lockres->l_ops->check_downconvert(lockres, new_level)) { + mlog(ML_BASTS, "lockres %s, ReQ: Checkpointing\n", + lockres->l_name); + goto leave_requeue; + } + + /* If we get here, then we know that there are no more + * incompatible holders (and anyone asking for an incompatible + * lock is blocked). We can now downconvert the lock */ + if (!lockres->l_ops->downconvert_worker) + goto downconvert; + + /* Some lockres types want to do a bit of work before + * downconverting a lock. Allow that here. The worker function + * may sleep, so we save off a copy of what we're blocking as + * it may change while we're not holding the spin lock. */ + blocking = lockres->l_blocking; + level = lockres->l_level; + spin_unlock_irqrestore(&lockres->l_lock, flags); + + ctl->unblock_action = lockres->l_ops->downconvert_worker(lockres, blocking); + + if (ctl->unblock_action == UNBLOCK_STOP_POST) { + mlog(ML_BASTS, "lockres %s, UNBLOCK_STOP_POST\n", + lockres->l_name); + goto leave; + } + + spin_lock_irqsave(&lockres->l_lock, flags); + if ((blocking != lockres->l_blocking) || (level != lockres->l_level)) { + /* If this changed underneath us, then we can't drop + * it just yet. */ + mlog(ML_BASTS, "lockres %s, block=%d:%d, level=%d:%d, " + "Recheck\n", lockres->l_name, blocking, + lockres->l_blocking, level, lockres->l_level); + goto recheck; + } + +downconvert: + ctl->requeue = 0; + + if (lockres->l_ops->flags & LOCK_TYPE_USES_LVB) { + if (lockres->l_level == DLM_LOCK_EX) + set_lvb = 1; + + /* + * We only set the lvb if the lock has been fully + * refreshed - otherwise we risk setting stale + * data. Otherwise, there's no need to actually clear + * out the lvb here as it's value is still valid. + */ + if (set_lvb && !(lockres->l_flags & OCFS2_LOCK_NEEDS_REFRESH)) + lockres->l_ops->set_lvb(lockres); + } + + gen = ocfs2_prepare_downconvert(lockres, new_level); + spin_unlock_irqrestore(&lockres->l_lock, flags); + ret = ocfs2_downconvert_lock(osb, lockres, new_level, set_lvb, + gen); + /* The dlm lock convert is being cancelled in background, + * ocfs2_cancel_convert() is asynchronous in fs/dlm, + * requeue it, try again later. + */ + if (ret == -EBUSY) { + ctl->requeue = 1; + mlog(ML_BASTS, "lockres %s, ReQ: Downconvert busy\n", + lockres->l_name); + ret = 0; + msleep(20); + } + +leave: + if (ret) + mlog_errno(ret); + return ret; + +leave_requeue: + spin_unlock_irqrestore(&lockres->l_lock, flags); + ctl->requeue = 1; + + return 0; +} + +static int ocfs2_data_convert_worker(struct ocfs2_lock_res *lockres, + int blocking) +{ + struct inode *inode; + struct address_space *mapping; + struct ocfs2_inode_info *oi; + + inode = ocfs2_lock_res_inode(lockres); + mapping = inode->i_mapping; + + if (S_ISDIR(inode->i_mode)) { + oi = OCFS2_I(inode); + oi->ip_dir_lock_gen++; + mlog(0, "generation: %u\n", oi->ip_dir_lock_gen); + goto out_forget; + } + + if (!S_ISREG(inode->i_mode)) + goto out; + + /* + * We need this before the filemap_fdatawrite() so that it can + * transfer the dirty bit from the PTE to the + * page. Unfortunately this means that even for EX->PR + * downconverts, we'll lose our mappings and have to build + * them up again. + */ + unmap_mapping_range(mapping, 0, 0, 0); + + if (filemap_fdatawrite(mapping)) { + mlog(ML_ERROR, "Could not sync inode %llu for downconvert!", + (unsigned long long)OCFS2_I(inode)->ip_blkno); + } + if (blocking == DLM_LOCK_EX) { + truncate_inode_pages(mapping, 0); + } else { + /* We only need to wait on the I/O if we're not also + * truncating pages because truncate_inode_pages waits + * for us above. We don't truncate pages if we're + * blocking anything < EXMODE because we want to keep + * them around in that case. */ + filemap_fdatawait(mapping); + } + +out_forget: + forget_all_cached_acls(inode); + +out: + return UNBLOCK_CONTINUE; +} + +static int ocfs2_ci_checkpointed(struct ocfs2_caching_info *ci, + struct ocfs2_lock_res *lockres, + int new_level) +{ + int checkpointed = ocfs2_ci_fully_checkpointed(ci); + + BUG_ON(new_level != DLM_LOCK_NL && new_level != DLM_LOCK_PR); + BUG_ON(lockres->l_level != DLM_LOCK_EX && !checkpointed); + + if (checkpointed) + return 1; + + ocfs2_start_checkpoint(OCFS2_SB(ocfs2_metadata_cache_get_super(ci))); + return 0; +} + +static int ocfs2_check_meta_downconvert(struct ocfs2_lock_res *lockres, + int new_level) +{ + struct inode *inode = ocfs2_lock_res_inode(lockres); + + return ocfs2_ci_checkpointed(INODE_CACHE(inode), lockres, new_level); +} + +static void ocfs2_set_meta_lvb(struct ocfs2_lock_res *lockres) +{ + struct inode *inode = ocfs2_lock_res_inode(lockres); + + __ocfs2_stuff_meta_lvb(inode); +} + +/* + * Does the final reference drop on our dentry lock. Right now this + * happens in the downconvert thread, but we could choose to simplify the + * dlmglue API and push these off to the ocfs2_wq in the future. + */ +static void ocfs2_dentry_post_unlock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres) +{ + struct ocfs2_dentry_lock *dl = ocfs2_lock_res_dl(lockres); + ocfs2_dentry_lock_put(osb, dl); +} + +/* + * d_delete() matching dentries before the lock downconvert. + * + * At this point, any process waiting to destroy the + * dentry_lock due to last ref count is stopped by the + * OCFS2_LOCK_QUEUED flag. + * + * We have two potential problems + * + * 1) If we do the last reference drop on our dentry_lock (via dput) + * we'll wind up in ocfs2_release_dentry_lock(), waiting on + * the downconvert to finish. Instead we take an elevated + * reference and push the drop until after we've completed our + * unblock processing. + * + * 2) There might be another process with a final reference, + * waiting on us to finish processing. If this is the case, we + * detect it and exit out - there's no more dentries anyway. + */ +static int ocfs2_dentry_convert_worker(struct ocfs2_lock_res *lockres, + int blocking) +{ + struct ocfs2_dentry_lock *dl = ocfs2_lock_res_dl(lockres); + struct ocfs2_inode_info *oi = OCFS2_I(dl->dl_inode); + struct dentry *dentry; + unsigned long flags; + int extra_ref = 0; + + /* + * This node is blocking another node from getting a read + * lock. This happens when we've renamed within a + * directory. We've forced the other nodes to d_delete(), but + * we never actually dropped our lock because it's still + * valid. The downconvert code will retain a PR for this node, + * so there's no further work to do. + */ + if (blocking == DLM_LOCK_PR) + return UNBLOCK_CONTINUE; + + /* + * Mark this inode as potentially orphaned. The code in + * ocfs2_delete_inode() will figure out whether it actually + * needs to be freed or not. + */ + spin_lock(&oi->ip_lock); + oi->ip_flags |= OCFS2_INODE_MAYBE_ORPHANED; + spin_unlock(&oi->ip_lock); + + /* + * Yuck. We need to make sure however that the check of + * OCFS2_LOCK_FREEING and the extra reference are atomic with + * respect to a reference decrement or the setting of that + * flag. + */ + spin_lock_irqsave(&lockres->l_lock, flags); + spin_lock(&dentry_attach_lock); + if (!(lockres->l_flags & OCFS2_LOCK_FREEING) + && dl->dl_count) { + dl->dl_count++; + extra_ref = 1; + } + spin_unlock(&dentry_attach_lock); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + mlog(0, "extra_ref = %d\n", extra_ref); + + /* + * We have a process waiting on us in ocfs2_dentry_iput(), + * which means we can't have any more outstanding + * aliases. There's no need to do any more work. + */ + if (!extra_ref) + return UNBLOCK_CONTINUE; + + spin_lock(&dentry_attach_lock); + while (1) { + dentry = ocfs2_find_local_alias(dl->dl_inode, + dl->dl_parent_blkno, 1); + if (!dentry) + break; + spin_unlock(&dentry_attach_lock); + + if (S_ISDIR(dl->dl_inode->i_mode)) + shrink_dcache_parent(dentry); + + mlog(0, "d_delete(%pd);\n", dentry); + + /* + * The following dcache calls may do an + * iput(). Normally we don't want that from the + * downconverting thread, but in this case it's ok + * because the requesting node already has an + * exclusive lock on the inode, so it can't be queued + * for a downconvert. + */ + d_delete(dentry); + dput(dentry); + + spin_lock(&dentry_attach_lock); + } + spin_unlock(&dentry_attach_lock); + + /* + * If we are the last holder of this dentry lock, there is no + * reason to downconvert so skip straight to the unlock. + */ + if (dl->dl_count == 1) + return UNBLOCK_STOP_POST; + + return UNBLOCK_CONTINUE_POST; +} + +static int ocfs2_check_refcount_downconvert(struct ocfs2_lock_res *lockres, + int new_level) +{ + struct ocfs2_refcount_tree *tree = + ocfs2_lock_res_refcount_tree(lockres); + + return ocfs2_ci_checkpointed(&tree->rf_ci, lockres, new_level); +} + +static int ocfs2_refcount_convert_worker(struct ocfs2_lock_res *lockres, + int blocking) +{ + struct ocfs2_refcount_tree *tree = + ocfs2_lock_res_refcount_tree(lockres); + + ocfs2_metadata_cache_purge(&tree->rf_ci); + + return UNBLOCK_CONTINUE; +} + +static void ocfs2_set_qinfo_lvb(struct ocfs2_lock_res *lockres) +{ + struct ocfs2_qinfo_lvb *lvb; + struct ocfs2_mem_dqinfo *oinfo = ocfs2_lock_res_qinfo(lockres); + struct mem_dqinfo *info = sb_dqinfo(oinfo->dqi_gi.dqi_sb, + oinfo->dqi_gi.dqi_type); + + lvb = ocfs2_dlm_lvb(&lockres->l_lksb); + lvb->lvb_version = OCFS2_QINFO_LVB_VERSION; + lvb->lvb_bgrace = cpu_to_be32(info->dqi_bgrace); + lvb->lvb_igrace = cpu_to_be32(info->dqi_igrace); + lvb->lvb_syncms = cpu_to_be32(oinfo->dqi_syncms); + lvb->lvb_blocks = cpu_to_be32(oinfo->dqi_gi.dqi_blocks); + lvb->lvb_free_blk = cpu_to_be32(oinfo->dqi_gi.dqi_free_blk); + lvb->lvb_free_entry = cpu_to_be32(oinfo->dqi_gi.dqi_free_entry); +} + +void ocfs2_qinfo_unlock(struct ocfs2_mem_dqinfo *oinfo, int ex) +{ + struct ocfs2_lock_res *lockres = &oinfo->dqi_gqlock; + struct ocfs2_super *osb = OCFS2_SB(oinfo->dqi_gi.dqi_sb); + int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; + + if (!ocfs2_is_hard_readonly(osb) && !ocfs2_mount_local(osb)) + ocfs2_cluster_unlock(osb, lockres, level); +} + +static int ocfs2_refresh_qinfo(struct ocfs2_mem_dqinfo *oinfo) +{ + struct mem_dqinfo *info = sb_dqinfo(oinfo->dqi_gi.dqi_sb, + oinfo->dqi_gi.dqi_type); + struct ocfs2_lock_res *lockres = &oinfo->dqi_gqlock; + struct ocfs2_qinfo_lvb *lvb = ocfs2_dlm_lvb(&lockres->l_lksb); + struct buffer_head *bh = NULL; + struct ocfs2_global_disk_dqinfo *gdinfo; + int status = 0; + + if (ocfs2_dlm_lvb_valid(&lockres->l_lksb) && + lvb->lvb_version == OCFS2_QINFO_LVB_VERSION) { + info->dqi_bgrace = be32_to_cpu(lvb->lvb_bgrace); + info->dqi_igrace = be32_to_cpu(lvb->lvb_igrace); + oinfo->dqi_syncms = be32_to_cpu(lvb->lvb_syncms); + oinfo->dqi_gi.dqi_blocks = be32_to_cpu(lvb->lvb_blocks); + oinfo->dqi_gi.dqi_free_blk = be32_to_cpu(lvb->lvb_free_blk); + oinfo->dqi_gi.dqi_free_entry = + be32_to_cpu(lvb->lvb_free_entry); + } else { + status = ocfs2_read_quota_phys_block(oinfo->dqi_gqinode, + oinfo->dqi_giblk, &bh); + if (status) { + mlog_errno(status); + goto bail; + } + gdinfo = (struct ocfs2_global_disk_dqinfo *) + (bh->b_data + OCFS2_GLOBAL_INFO_OFF); + info->dqi_bgrace = le32_to_cpu(gdinfo->dqi_bgrace); + info->dqi_igrace = le32_to_cpu(gdinfo->dqi_igrace); + oinfo->dqi_syncms = le32_to_cpu(gdinfo->dqi_syncms); + oinfo->dqi_gi.dqi_blocks = le32_to_cpu(gdinfo->dqi_blocks); + oinfo->dqi_gi.dqi_free_blk = le32_to_cpu(gdinfo->dqi_free_blk); + oinfo->dqi_gi.dqi_free_entry = + le32_to_cpu(gdinfo->dqi_free_entry); + brelse(bh); + ocfs2_track_lock_refresh(lockres); + } + +bail: + return status; +} + +/* Lock quota info, this function expects at least shared lock on the quota file + * so that we can safely refresh quota info from disk. */ +int ocfs2_qinfo_lock(struct ocfs2_mem_dqinfo *oinfo, int ex) +{ + struct ocfs2_lock_res *lockres = &oinfo->dqi_gqlock; + struct ocfs2_super *osb = OCFS2_SB(oinfo->dqi_gi.dqi_sb); + int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; + int status = 0; + + /* On RO devices, locking really isn't needed... */ + if (ocfs2_is_hard_readonly(osb)) { + if (ex) + status = -EROFS; + goto bail; + } + if (ocfs2_mount_local(osb)) + goto bail; + + status = ocfs2_cluster_lock(osb, lockres, level, 0, 0); + if (status < 0) { + mlog_errno(status); + goto bail; + } + if (!ocfs2_should_refresh_lock_res(lockres)) + goto bail; + /* OK, we have the lock but we need to refresh the quota info */ + status = ocfs2_refresh_qinfo(oinfo); + if (status) + ocfs2_qinfo_unlock(oinfo, ex); + ocfs2_complete_lock_res_refresh(lockres, status); +bail: + return status; +} + +int ocfs2_refcount_lock(struct ocfs2_refcount_tree *ref_tree, int ex) +{ + int status; + int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; + struct ocfs2_lock_res *lockres = &ref_tree->rf_lockres; + struct ocfs2_super *osb = lockres->l_priv; + + + if (ocfs2_is_hard_readonly(osb)) + return -EROFS; + + if (ocfs2_mount_local(osb)) + return 0; + + status = ocfs2_cluster_lock(osb, lockres, level, 0, 0); + if (status < 0) + mlog_errno(status); + + return status; +} + +void ocfs2_refcount_unlock(struct ocfs2_refcount_tree *ref_tree, int ex) +{ + int level = ex ? DLM_LOCK_EX : DLM_LOCK_PR; + struct ocfs2_lock_res *lockres = &ref_tree->rf_lockres; + struct ocfs2_super *osb = lockres->l_priv; + + if (!ocfs2_mount_local(osb)) + ocfs2_cluster_unlock(osb, lockres, level); +} + +static void ocfs2_process_blocked_lock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres) +{ + int status; + struct ocfs2_unblock_ctl ctl = {0, 0,}; + unsigned long flags; + + /* Our reference to the lockres in this function can be + * considered valid until we remove the OCFS2_LOCK_QUEUED + * flag. */ + + BUG_ON(!lockres); + BUG_ON(!lockres->l_ops); + + mlog(ML_BASTS, "lockres %s blocked\n", lockres->l_name); + + /* Detect whether a lock has been marked as going away while + * the downconvert thread was processing other things. A lock can + * still be marked with OCFS2_LOCK_FREEING after this check, + * but short circuiting here will still save us some + * performance. */ + spin_lock_irqsave(&lockres->l_lock, flags); + if (lockres->l_flags & OCFS2_LOCK_FREEING) + goto unqueue; + spin_unlock_irqrestore(&lockres->l_lock, flags); + + status = ocfs2_unblock_lock(osb, lockres, &ctl); + if (status < 0) + mlog_errno(status); + + spin_lock_irqsave(&lockres->l_lock, flags); +unqueue: + if (lockres->l_flags & OCFS2_LOCK_FREEING || !ctl.requeue) { + lockres_clear_flags(lockres, OCFS2_LOCK_QUEUED); + } else + ocfs2_schedule_blocked_lock(osb, lockres); + + mlog(ML_BASTS, "lockres %s, requeue = %s.\n", lockres->l_name, + str_yes_no(ctl.requeue)); + spin_unlock_irqrestore(&lockres->l_lock, flags); + + if (ctl.unblock_action != UNBLOCK_CONTINUE + && lockres->l_ops->post_unlock) + lockres->l_ops->post_unlock(osb, lockres); +} + +static void ocfs2_schedule_blocked_lock(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres) +{ + unsigned long flags; + + assert_spin_locked(&lockres->l_lock); + + if (lockres->l_flags & OCFS2_LOCK_FREEING) { + /* Do not schedule a lock for downconvert when it's on + * the way to destruction - any nodes wanting access + * to the resource will get it soon. */ + mlog(ML_BASTS, "lockres %s won't be scheduled: flags 0x%lx\n", + lockres->l_name, lockres->l_flags); + return; + } + + lockres_or_flags(lockres, OCFS2_LOCK_QUEUED); + + spin_lock_irqsave(&osb->dc_task_lock, flags); + if (list_empty(&lockres->l_blocked_list)) { + list_add_tail(&lockres->l_blocked_list, + &osb->blocked_lock_list); + osb->blocked_lock_count++; + } + spin_unlock_irqrestore(&osb->dc_task_lock, flags); +} + +static void ocfs2_downconvert_thread_do_work(struct ocfs2_super *osb) +{ + unsigned long processed; + unsigned long flags; + struct ocfs2_lock_res *lockres; + + spin_lock_irqsave(&osb->dc_task_lock, flags); + /* grab this early so we know to try again if a state change and + * wake happens part-way through our work */ + osb->dc_work_sequence = osb->dc_wake_sequence; + + processed = osb->blocked_lock_count; + /* + * blocked lock processing in this loop might call iput which can + * remove items off osb->blocked_lock_list. Downconvert up to + * 'processed' number of locks, but stop short if we had some + * removed in ocfs2_mark_lockres_freeing when downconverting. + */ + while (processed && !list_empty(&osb->blocked_lock_list)) { + lockres = list_entry(osb->blocked_lock_list.next, + struct ocfs2_lock_res, l_blocked_list); + list_del_init(&lockres->l_blocked_list); + osb->blocked_lock_count--; + spin_unlock_irqrestore(&osb->dc_task_lock, flags); + + BUG_ON(!processed); + processed--; + + ocfs2_process_blocked_lock(osb, lockres); + + spin_lock_irqsave(&osb->dc_task_lock, flags); + } + spin_unlock_irqrestore(&osb->dc_task_lock, flags); +} + +static int ocfs2_downconvert_thread_lists_empty(struct ocfs2_super *osb) +{ + int empty = 0; + unsigned long flags; + + spin_lock_irqsave(&osb->dc_task_lock, flags); + if (list_empty(&osb->blocked_lock_list)) + empty = 1; + + spin_unlock_irqrestore(&osb->dc_task_lock, flags); + return empty; +} + +static int ocfs2_downconvert_thread_should_wake(struct ocfs2_super *osb) +{ + int should_wake = 0; + unsigned long flags; + + spin_lock_irqsave(&osb->dc_task_lock, flags); + if (osb->dc_work_sequence != osb->dc_wake_sequence) + should_wake = 1; + spin_unlock_irqrestore(&osb->dc_task_lock, flags); + + return should_wake; +} + +static int ocfs2_downconvert_thread(void *arg) +{ + struct ocfs2_super *osb = arg; + + /* only quit once we've been asked to stop and there is no more + * work available */ + while (!(kthread_should_stop() && + ocfs2_downconvert_thread_lists_empty(osb))) { + + wait_event_interruptible(osb->dc_event, + ocfs2_downconvert_thread_should_wake(osb) || + kthread_should_stop()); + + mlog(0, "downconvert_thread: awoken\n"); + + ocfs2_downconvert_thread_do_work(osb); + } + + osb->dc_task = NULL; + return 0; +} + +void ocfs2_wake_downconvert_thread(struct ocfs2_super *osb) +{ + unsigned long flags; + + spin_lock_irqsave(&osb->dc_task_lock, flags); + /* make sure the voting thread gets a swipe at whatever changes + * the caller may have made to the voting state */ + osb->dc_wake_sequence++; + spin_unlock_irqrestore(&osb->dc_task_lock, flags); + wake_up(&osb->dc_event); +} diff --git a/fs/ocfs2/dlmglue.h b/fs/ocfs2/dlmglue.h new file mode 100644 index 000000000..a3ebd7303 --- /dev/null +++ b/fs/ocfs2/dlmglue.h @@ -0,0 +1,207 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * dlmglue.h + * + * description here + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + + +#ifndef DLMGLUE_H +#define DLMGLUE_H + +#include "dcache.h" + +#define OCFS2_LVB_VERSION 5 + +struct ocfs2_meta_lvb { + __u8 lvb_version; + __u8 lvb_reserved0; + __be16 lvb_idynfeatures; + __be32 lvb_iclusters; + __be32 lvb_iuid; + __be32 lvb_igid; + __be64 lvb_iatime_packed; + __be64 lvb_ictime_packed; + __be64 lvb_imtime_packed; + __be64 lvb_isize; + __be16 lvb_imode; + __be16 lvb_inlink; + __be32 lvb_iattr; + __be32 lvb_igeneration; + __be32 lvb_reserved2; +}; + +#define OCFS2_QINFO_LVB_VERSION 1 + +struct ocfs2_qinfo_lvb { + __u8 lvb_version; + __u8 lvb_reserved[3]; + __be32 lvb_bgrace; + __be32 lvb_igrace; + __be32 lvb_syncms; + __be32 lvb_blocks; + __be32 lvb_free_blk; + __be32 lvb_free_entry; +}; + +#define OCFS2_ORPHAN_LVB_VERSION 1 + +struct ocfs2_orphan_scan_lvb { + __u8 lvb_version; + __u8 lvb_reserved[3]; + __be32 lvb_os_seqno; +}; + +#define OCFS2_TRIMFS_LVB_VERSION 1 + +struct ocfs2_trim_fs_lvb { + __u8 lvb_version; + __u8 lvb_success; + __u8 lvb_reserved[2]; + __be32 lvb_nodenum; + __be64 lvb_start; + __be64 lvb_len; + __be64 lvb_minlen; + __be64 lvb_trimlen; +}; + +struct ocfs2_trim_fs_info { + u8 tf_valid; /* lvb is valid, or not */ + u8 tf_success; /* trim is successful, or not */ + u32 tf_nodenum; /* osb node number */ + u64 tf_start; /* trim start offset in clusters */ + u64 tf_len; /* trim end offset in clusters */ + u64 tf_minlen; /* trim minimum contiguous free clusters */ + u64 tf_trimlen; /* trimmed length in bytes */ +}; + +struct ocfs2_lock_holder { + struct list_head oh_list; + struct pid *oh_owner_pid; + int oh_ex; +}; + +/* ocfs2_inode_lock_full() 'arg_flags' flags */ +/* don't wait on recovery. */ +#define OCFS2_META_LOCK_RECOVERY (0x01) +/* Instruct the dlm not to queue ourselves on the other node. */ +#define OCFS2_META_LOCK_NOQUEUE (0x02) +/* don't block waiting for the downconvert thread, instead return -EAGAIN */ +#define OCFS2_LOCK_NONBLOCK (0x04) +/* just get back disk inode bh if we've got cluster lock. */ +#define OCFS2_META_LOCK_GETBH (0x08) + +/* Locking subclasses of inode cluster lock */ +enum { + OI_LS_NORMAL = 0, + OI_LS_PARENT, + OI_LS_RENAME1, + OI_LS_RENAME2, + OI_LS_REFLINK_TARGET, +}; + +int ocfs2_dlm_init(struct ocfs2_super *osb); +void ocfs2_dlm_shutdown(struct ocfs2_super *osb, int hangup_pending); +void ocfs2_lock_res_init_once(struct ocfs2_lock_res *res); +void ocfs2_inode_lock_res_init(struct ocfs2_lock_res *res, + enum ocfs2_lock_type type, + unsigned int generation, + struct inode *inode); +void ocfs2_dentry_lock_res_init(struct ocfs2_dentry_lock *dl, + u64 parent, struct inode *inode); +struct ocfs2_file_private; +void ocfs2_file_lock_res_init(struct ocfs2_lock_res *lockres, + struct ocfs2_file_private *fp); +struct ocfs2_mem_dqinfo; +void ocfs2_qinfo_lock_res_init(struct ocfs2_lock_res *lockres, + struct ocfs2_mem_dqinfo *info); +void ocfs2_refcount_lock_res_init(struct ocfs2_lock_res *lockres, + struct ocfs2_super *osb, u64 ref_blkno, + unsigned int generation); +void ocfs2_lock_res_free(struct ocfs2_lock_res *res); +int ocfs2_create_new_inode_locks(struct inode *inode); +int ocfs2_drop_inode_locks(struct inode *inode); +int ocfs2_rw_lock(struct inode *inode, int write); +int ocfs2_try_rw_lock(struct inode *inode, int write); +void ocfs2_rw_unlock(struct inode *inode, int write); +int ocfs2_open_lock(struct inode *inode); +int ocfs2_try_open_lock(struct inode *inode, int write); +void ocfs2_open_unlock(struct inode *inode); +int ocfs2_inode_lock_atime(struct inode *inode, + struct vfsmount *vfsmnt, + int *level, int wait); +int ocfs2_inode_lock_full_nested(struct inode *inode, + struct buffer_head **ret_bh, + int ex, + int arg_flags, + int subclass); +int ocfs2_inode_lock_with_folio(struct inode *inode, + struct buffer_head **ret_bh, int ex, struct folio *folio); +/* Variants without special locking class or flags */ +#define ocfs2_inode_lock_full(i, r, e, f)\ + ocfs2_inode_lock_full_nested(i, r, e, f, OI_LS_NORMAL) +#define ocfs2_inode_lock_nested(i, b, e, s)\ + ocfs2_inode_lock_full_nested(i, b, e, 0, s) +/* 99% of the time we don't want to supply any additional flags -- + * those are for very specific cases only. */ +#define ocfs2_inode_lock(i, b, e) ocfs2_inode_lock_full_nested(i, b, e, 0, OI_LS_NORMAL) +#define ocfs2_try_inode_lock(i, b, e)\ + ocfs2_inode_lock_full_nested(i, b, e, OCFS2_META_LOCK_NOQUEUE,\ + OI_LS_NORMAL) +void ocfs2_inode_unlock(struct inode *inode, + int ex); +int ocfs2_super_lock(struct ocfs2_super *osb, + int ex); +void ocfs2_super_unlock(struct ocfs2_super *osb, + int ex); +int ocfs2_orphan_scan_lock(struct ocfs2_super *osb, u32 *seqno); +void ocfs2_orphan_scan_unlock(struct ocfs2_super *osb, u32 seqno); + +int ocfs2_rename_lock(struct ocfs2_super *osb); +void ocfs2_rename_unlock(struct ocfs2_super *osb); +int ocfs2_nfs_sync_lock(struct ocfs2_super *osb, int ex); +void ocfs2_nfs_sync_unlock(struct ocfs2_super *osb, int ex); +void ocfs2_trim_fs_lock_res_init(struct ocfs2_super *osb); +void ocfs2_trim_fs_lock_res_uninit(struct ocfs2_super *osb); +int ocfs2_trim_fs_lock(struct ocfs2_super *osb, + struct ocfs2_trim_fs_info *info, int trylock); +void ocfs2_trim_fs_unlock(struct ocfs2_super *osb, + struct ocfs2_trim_fs_info *info); +int ocfs2_dentry_lock(struct dentry *dentry, int ex); +void ocfs2_dentry_unlock(struct dentry *dentry, int ex); +int ocfs2_file_lock(struct file *file, int ex, int trylock); +void ocfs2_file_unlock(struct file *file); +int ocfs2_qinfo_lock(struct ocfs2_mem_dqinfo *oinfo, int ex); +void ocfs2_qinfo_unlock(struct ocfs2_mem_dqinfo *oinfo, int ex); +struct ocfs2_refcount_tree; +int ocfs2_refcount_lock(struct ocfs2_refcount_tree *ref_tree, int ex); +void ocfs2_refcount_unlock(struct ocfs2_refcount_tree *ref_tree, int ex); + + +void ocfs2_mark_lockres_freeing(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres); +void ocfs2_simple_drop_lockres(struct ocfs2_super *osb, + struct ocfs2_lock_res *lockres); + +/* for the downconvert thread */ +void ocfs2_wake_downconvert_thread(struct ocfs2_super *osb); + +struct ocfs2_dlm_debug *ocfs2_new_dlm_debug(void); +void ocfs2_put_dlm_debug(struct ocfs2_dlm_debug *dlm_debug); + +/* To set the locking protocol on module initialization */ +void ocfs2_set_locking_protocol(void); + +/* The _tracker pair is used to avoid cluster recursive locking */ +int ocfs2_inode_lock_tracker(struct inode *inode, + struct buffer_head **ret_bh, + int ex, + struct ocfs2_lock_holder *oh); +void ocfs2_inode_unlock_tracker(struct inode *inode, + int ex, + struct ocfs2_lock_holder *oh, + int had_lock); + +#endif /* DLMGLUE_H */ diff --git a/fs/ocfs2/export.c b/fs/ocfs2/export.c new file mode 100644 index 000000000..9c2665dd2 --- /dev/null +++ b/fs/ocfs2/export.c @@ -0,0 +1,285 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * export.c + * + * Functions to facilitate NFS exporting + * + * Copyright (C) 2002, 2005 Oracle. All rights reserved. + */ + +#include +#include + +#include + +#include "ocfs2.h" + +#include "alloc.h" +#include "dir.h" +#include "dlmglue.h" +#include "dcache.h" +#include "export.h" +#include "inode.h" + +#include "buffer_head_io.h" +#include "suballoc.h" +#include "ocfs2_trace.h" + +struct ocfs2_inode_handle +{ + u64 ih_blkno; + u32 ih_generation; +}; + +static struct dentry *ocfs2_get_dentry(struct super_block *sb, + struct ocfs2_inode_handle *handle) +{ + struct inode *inode; + struct ocfs2_super *osb = OCFS2_SB(sb); + u64 blkno = handle->ih_blkno; + int status, set; + struct dentry *result; + + trace_ocfs2_get_dentry_begin(sb, handle, (unsigned long long)blkno); + + if (blkno == 0) { + result = ERR_PTR(-ESTALE); + goto bail; + } + + inode = ocfs2_ilookup(sb, blkno); + /* + * If the inode exists in memory, we only need to check it's + * generation number + */ + if (inode) + goto check_gen; + + /* + * This will synchronize us against ocfs2_delete_inode() on + * all nodes + */ + status = ocfs2_nfs_sync_lock(osb, 1); + if (status < 0) { + mlog(ML_ERROR, "getting nfs sync lock(EX) failed %d\n", status); + goto check_err; + } + + status = ocfs2_test_inode_bit(osb, blkno, &set); + if (status < 0) { + if (status == -EINVAL) { + /* + * The blkno NFS gave us doesn't even show up + * as an inode, we return -ESTALE to be + * nice + */ + status = -ESTALE; + } else if (status != -ESTALE) { + mlog(ML_ERROR, "test inode bit failed %d\n", status); + } + goto unlock_nfs_sync; + } + + trace_ocfs2_get_dentry_test_bit(status, set); + /* If the inode allocator bit is clear, this inode must be stale */ + if (!set) { + status = -ESTALE; + goto unlock_nfs_sync; + } + + inode = ocfs2_iget(osb, blkno, 0, 0); + +unlock_nfs_sync: + ocfs2_nfs_sync_unlock(osb, 1); + +check_err: + if (status < 0) { + if (status == -ESTALE) { + trace_ocfs2_get_dentry_stale((unsigned long long)blkno, + handle->ih_generation); + } + result = ERR_PTR(status); + goto bail; + } + + if (IS_ERR(inode)) { + mlog_errno(PTR_ERR(inode)); + result = ERR_CAST(inode); + goto bail; + } + +check_gen: + if (handle->ih_generation != inode->i_generation) { + trace_ocfs2_get_dentry_generation((unsigned long long)blkno, + handle->ih_generation, + inode->i_generation); + iput(inode); + result = ERR_PTR(-ESTALE); + goto bail; + } + + result = d_obtain_alias(inode); + if (IS_ERR(result)) + mlog_errno(PTR_ERR(result)); + +bail: + trace_ocfs2_get_dentry_end(result); + return result; +} + +static struct dentry *ocfs2_get_parent(struct dentry *child) +{ + int status; + u64 blkno; + struct dentry *parent; + struct inode *dir = d_inode(child); + int set; + + trace_ocfs2_get_parent(child, child->d_name.len, child->d_name.name, + (unsigned long long)OCFS2_I(dir)->ip_blkno); + + status = ocfs2_nfs_sync_lock(OCFS2_SB(dir->i_sb), 1); + if (status < 0) { + mlog(ML_ERROR, "getting nfs sync lock(EX) failed %d\n", status); + parent = ERR_PTR(status); + goto bail; + } + + status = ocfs2_inode_lock(dir, NULL, 0); + if (status < 0) { + if (status != -ENOENT) + mlog_errno(status); + parent = ERR_PTR(status); + goto unlock_nfs_sync; + } + + status = ocfs2_lookup_ino_from_name(dir, "..", 2, &blkno); + if (status < 0) { + parent = ERR_PTR(-ENOENT); + goto bail_unlock; + } + + status = ocfs2_test_inode_bit(OCFS2_SB(dir->i_sb), blkno, &set); + if (status < 0) { + if (status == -EINVAL) { + status = -ESTALE; + } else if (status != -ESTALE) { + mlog(ML_ERROR, "test inode bit failed %d\n", status); + } + parent = ERR_PTR(status); + goto bail_unlock; + } + + trace_ocfs2_get_dentry_test_bit(status, set); + if (!set) { + status = -ESTALE; + parent = ERR_PTR(status); + goto bail_unlock; + } + + parent = d_obtain_alias(ocfs2_iget(OCFS2_SB(dir->i_sb), blkno, 0, 0)); + +bail_unlock: + ocfs2_inode_unlock(dir, 0); + +unlock_nfs_sync: + ocfs2_nfs_sync_unlock(OCFS2_SB(dir->i_sb), 1); + +bail: + trace_ocfs2_get_parent_end(parent); + + return parent; +} + +static int ocfs2_encode_fh(struct inode *inode, u32 *fh_in, int *max_len, + struct inode *parent) +{ + int len = *max_len; + int type = 1; + u64 blkno; + u32 generation; + __le32 *fh = (__force __le32 *) fh_in; + +#ifdef TRACE_HOOKS_ARE_NOT_BRAINDEAD_IN_YOUR_OPINION +#error "You go ahead and fix that mess, then. Somehow" + trace_ocfs2_encode_fh_begin(dentry, dentry->d_name.len, + dentry->d_name.name, + fh, len, connectable); +#endif + + if (parent && (len < 6)) { + *max_len = 6; + type = FILEID_INVALID; + goto bail; + } else if (len < 3) { + *max_len = 3; + type = FILEID_INVALID; + goto bail; + } + + blkno = OCFS2_I(inode)->ip_blkno; + generation = inode->i_generation; + + trace_ocfs2_encode_fh_self((unsigned long long)blkno, generation); + + len = 3; + fh[0] = cpu_to_le32((u32)(blkno >> 32)); + fh[1] = cpu_to_le32((u32)(blkno & 0xffffffff)); + fh[2] = cpu_to_le32(generation); + + if (parent) { + blkno = OCFS2_I(parent)->ip_blkno; + generation = parent->i_generation; + + fh[3] = cpu_to_le32((u32)(blkno >> 32)); + fh[4] = cpu_to_le32((u32)(blkno & 0xffffffff)); + fh[5] = cpu_to_le32(generation); + + len = 6; + type = 2; + + trace_ocfs2_encode_fh_parent((unsigned long long)blkno, + generation); + } + + *max_len = len; + +bail: + trace_ocfs2_encode_fh_type(type); + return type; +} + +static struct dentry *ocfs2_fh_to_dentry(struct super_block *sb, + struct fid *fid, int fh_len, int fh_type) +{ + struct ocfs2_inode_handle handle; + + if (fh_len < 3 || fh_type > 2) + return NULL; + + handle.ih_blkno = (u64)le32_to_cpu((__force __le32)fid->raw[0]) << 32; + handle.ih_blkno |= (u64)le32_to_cpu((__force __le32)fid->raw[1]); + handle.ih_generation = le32_to_cpu((__force __le32)fid->raw[2]); + return ocfs2_get_dentry(sb, &handle); +} + +static struct dentry *ocfs2_fh_to_parent(struct super_block *sb, + struct fid *fid, int fh_len, int fh_type) +{ + struct ocfs2_inode_handle parent; + + if (fh_type != 2 || fh_len < 6) + return NULL; + + parent.ih_blkno = (u64)le32_to_cpu((__force __le32)fid->raw[3]) << 32; + parent.ih_blkno |= (u64)le32_to_cpu((__force __le32)fid->raw[4]); + parent.ih_generation = le32_to_cpu((__force __le32)fid->raw[5]); + return ocfs2_get_dentry(sb, &parent); +} + +const struct export_operations ocfs2_export_ops = { + .encode_fh = ocfs2_encode_fh, + .fh_to_dentry = ocfs2_fh_to_dentry, + .fh_to_parent = ocfs2_fh_to_parent, + .get_parent = ocfs2_get_parent, +}; diff --git a/fs/ocfs2/export.h b/fs/ocfs2/export.h new file mode 100644 index 000000000..636357400 --- /dev/null +++ b/fs/ocfs2/export.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * export.h + * + * Function prototypes + * + * Copyright (C) 2002, 2005 Oracle. All rights reserved. + */ + +#ifndef OCFS2_EXPORT_H +#define OCFS2_EXPORT_H + +#include + +extern const struct export_operations ocfs2_export_ops; + +#endif /* OCFS2_EXPORT_H */ diff --git a/fs/ocfs2/extent_map.c b/fs/ocfs2/extent_map.c new file mode 100644 index 000000000..eb5dcd17d --- /dev/null +++ b/fs/ocfs2/extent_map.c @@ -0,0 +1,1047 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * extent_map.c + * + * Block/Cluster mapping functions + * + * Copyright (C) 2004 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include + +#include + +#include "ocfs2.h" + +#include "alloc.h" +#include "dlmglue.h" +#include "extent_map.h" +#include "inode.h" +#include "super.h" +#include "symlink.h" +#include "aops.h" +#include "ocfs2_trace.h" + +#include "buffer_head_io.h" + +/* + * The extent caching implementation is intentionally trivial. + * + * We only cache a small number of extents stored directly on the + * inode, so linear order operations are acceptable. If we ever want + * to increase the size of the extent map, then these algorithms must + * get smarter. + */ + +void ocfs2_extent_map_init(struct inode *inode) +{ + struct ocfs2_inode_info *oi = OCFS2_I(inode); + + oi->ip_extent_map.em_num_items = 0; + INIT_LIST_HEAD(&oi->ip_extent_map.em_list); +} + +static void __ocfs2_extent_map_lookup(struct ocfs2_extent_map *em, + unsigned int cpos, + struct ocfs2_extent_map_item **ret_emi) +{ + unsigned int range; + struct ocfs2_extent_map_item *emi; + + *ret_emi = NULL; + + list_for_each_entry(emi, &em->em_list, ei_list) { + range = emi->ei_cpos + emi->ei_clusters; + + if (cpos >= emi->ei_cpos && cpos < range) { + list_move(&emi->ei_list, &em->em_list); + + *ret_emi = emi; + break; + } + } +} + +static int ocfs2_extent_map_lookup(struct inode *inode, unsigned int cpos, + unsigned int *phys, unsigned int *len, + unsigned int *flags) +{ + unsigned int coff; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_extent_map_item *emi; + + spin_lock(&oi->ip_lock); + + __ocfs2_extent_map_lookup(&oi->ip_extent_map, cpos, &emi); + if (emi) { + coff = cpos - emi->ei_cpos; + *phys = emi->ei_phys + coff; + if (len) + *len = emi->ei_clusters - coff; + if (flags) + *flags = emi->ei_flags; + } + + spin_unlock(&oi->ip_lock); + + if (emi == NULL) + return -ENOENT; + + return 0; +} + +/* + * Forget about all clusters equal to or greater than cpos. + */ +void ocfs2_extent_map_trunc(struct inode *inode, unsigned int cpos) +{ + struct ocfs2_extent_map_item *emi, *n; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_extent_map *em = &oi->ip_extent_map; + LIST_HEAD(tmp_list); + unsigned int range; + + spin_lock(&oi->ip_lock); + list_for_each_entry_safe(emi, n, &em->em_list, ei_list) { + if (emi->ei_cpos >= cpos) { + /* Full truncate of this record. */ + list_move(&emi->ei_list, &tmp_list); + BUG_ON(em->em_num_items == 0); + em->em_num_items--; + continue; + } + + range = emi->ei_cpos + emi->ei_clusters; + if (range > cpos) { + /* Partial truncate */ + emi->ei_clusters = cpos - emi->ei_cpos; + } + } + spin_unlock(&oi->ip_lock); + + list_for_each_entry_safe(emi, n, &tmp_list, ei_list) { + list_del(&emi->ei_list); + kfree(emi); + } +} + +/* + * Is any part of emi2 contained within emi1 + */ +static int ocfs2_ei_is_contained(struct ocfs2_extent_map_item *emi1, + struct ocfs2_extent_map_item *emi2) +{ + unsigned int range1, range2; + + /* + * Check if logical start of emi2 is inside emi1 + */ + range1 = emi1->ei_cpos + emi1->ei_clusters; + if (emi2->ei_cpos >= emi1->ei_cpos && emi2->ei_cpos < range1) + return 1; + + /* + * Check if logical end of emi2 is inside emi1 + */ + range2 = emi2->ei_cpos + emi2->ei_clusters; + if (range2 > emi1->ei_cpos && range2 <= range1) + return 1; + + return 0; +} + +static void ocfs2_copy_emi_fields(struct ocfs2_extent_map_item *dest, + struct ocfs2_extent_map_item *src) +{ + dest->ei_cpos = src->ei_cpos; + dest->ei_phys = src->ei_phys; + dest->ei_clusters = src->ei_clusters; + dest->ei_flags = src->ei_flags; +} + +/* + * Try to merge emi with ins. Returns 1 if merge succeeds, zero + * otherwise. + */ +static int ocfs2_try_to_merge_extent_map(struct ocfs2_extent_map_item *emi, + struct ocfs2_extent_map_item *ins) +{ + /* + * Handle contiguousness + */ + if (ins->ei_phys == (emi->ei_phys + emi->ei_clusters) && + ins->ei_cpos == (emi->ei_cpos + emi->ei_clusters) && + ins->ei_flags == emi->ei_flags) { + emi->ei_clusters += ins->ei_clusters; + return 1; + } else if ((ins->ei_phys + ins->ei_clusters) == emi->ei_phys && + (ins->ei_cpos + ins->ei_clusters) == emi->ei_cpos && + ins->ei_flags == emi->ei_flags) { + emi->ei_phys = ins->ei_phys; + emi->ei_cpos = ins->ei_cpos; + emi->ei_clusters += ins->ei_clusters; + return 1; + } + + /* + * Overlapping extents - this shouldn't happen unless we've + * split an extent to change it's flags. That is exceedingly + * rare, so there's no sense in trying to optimize it yet. + */ + if (ocfs2_ei_is_contained(emi, ins) || + ocfs2_ei_is_contained(ins, emi)) { + ocfs2_copy_emi_fields(emi, ins); + return 1; + } + + /* No merge was possible. */ + return 0; +} + +/* + * In order to reduce complexity on the caller, this insert function + * is intentionally liberal in what it will accept. + * + * The only rule is that the truncate call *must* be used whenever + * records have been deleted. This avoids inserting overlapping + * records with different physical mappings. + */ +void ocfs2_extent_map_insert_rec(struct inode *inode, + struct ocfs2_extent_rec *rec) +{ + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_extent_map *em = &oi->ip_extent_map; + struct ocfs2_extent_map_item *emi, *new_emi = NULL; + struct ocfs2_extent_map_item ins; + + ins.ei_cpos = le32_to_cpu(rec->e_cpos); + ins.ei_phys = ocfs2_blocks_to_clusters(inode->i_sb, + le64_to_cpu(rec->e_blkno)); + ins.ei_clusters = le16_to_cpu(rec->e_leaf_clusters); + ins.ei_flags = rec->e_flags; + +search: + spin_lock(&oi->ip_lock); + + list_for_each_entry(emi, &em->em_list, ei_list) { + if (ocfs2_try_to_merge_extent_map(emi, &ins)) { + list_move(&emi->ei_list, &em->em_list); + spin_unlock(&oi->ip_lock); + goto out; + } + } + + /* + * No item could be merged. + * + * Either allocate and add a new item, or overwrite the last recently + * inserted. + */ + + if (em->em_num_items < OCFS2_MAX_EXTENT_MAP_ITEMS) { + if (new_emi == NULL) { + spin_unlock(&oi->ip_lock); + + new_emi = kmalloc_obj(*new_emi, GFP_NOFS); + if (new_emi == NULL) + goto out; + + goto search; + } + + ocfs2_copy_emi_fields(new_emi, &ins); + list_add(&new_emi->ei_list, &em->em_list); + em->em_num_items++; + new_emi = NULL; + } else { + BUG_ON(list_empty(&em->em_list) || em->em_num_items == 0); + emi = list_entry(em->em_list.prev, + struct ocfs2_extent_map_item, ei_list); + list_move(&emi->ei_list, &em->em_list); + ocfs2_copy_emi_fields(emi, &ins); + } + + spin_unlock(&oi->ip_lock); + +out: + kfree(new_emi); +} + +static int ocfs2_last_eb_is_empty(struct inode *inode, + struct ocfs2_dinode *di) +{ + int ret, next_free; + u64 last_eb_blk = le64_to_cpu(di->i_last_eb_blk); + struct buffer_head *eb_bh = NULL; + struct ocfs2_extent_block *eb; + struct ocfs2_extent_list *el; + + ret = ocfs2_read_extent_block(INODE_CACHE(inode), last_eb_blk, &eb_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + eb = (struct ocfs2_extent_block *) eb_bh->b_data; + el = &eb->h_list; + + if (el->l_tree_depth) { + ocfs2_error(inode->i_sb, + "Inode %llu has non zero tree depth in leaf block %llu\n", + inode->i_ino, + (unsigned long long)eb_bh->b_blocknr); + ret = -EROFS; + goto out; + } + + next_free = le16_to_cpu(el->l_next_free_rec); + + if (next_free == 0 || + (next_free == 1 && ocfs2_is_empty_extent(&el->l_recs[0]))) + ret = 1; + +out: + brelse(eb_bh); + return ret; +} + +/* + * Return the 1st index within el which contains an extent start + * larger than v_cluster. + */ +static int ocfs2_search_for_hole_index(struct ocfs2_extent_list *el, + u32 v_cluster) +{ + int i; + struct ocfs2_extent_rec *rec; + + for(i = 0; i < le16_to_cpu(el->l_next_free_rec); i++) { + rec = &el->l_recs[i]; + + if (v_cluster < le32_to_cpu(rec->e_cpos)) + break; + } + + return i; +} + +/* + * Figure out the size of a hole which starts at v_cluster within the given + * extent list. + * + * If there is no more allocation past v_cluster, we return the maximum + * cluster size minus v_cluster. + * + * If we have in-inode extents, then el points to the dinode list and + * eb_bh is NULL. Otherwise, eb_bh should point to the extent block + * containing el. + */ +int ocfs2_figure_hole_clusters(struct ocfs2_caching_info *ci, + struct ocfs2_extent_list *el, + struct buffer_head *eb_bh, + u32 v_cluster, + u32 *num_clusters) +{ + int ret, i; + struct buffer_head *next_eb_bh = NULL; + struct ocfs2_extent_block *eb, *next_eb; + + i = ocfs2_search_for_hole_index(el, v_cluster); + + if (i == le16_to_cpu(el->l_next_free_rec) && eb_bh) { + eb = (struct ocfs2_extent_block *)eb_bh->b_data; + + /* + * Check the next leaf for any extents. + */ + + if (le64_to_cpu(eb->h_next_leaf_blk) == 0ULL) + goto no_more_extents; + + ret = ocfs2_read_extent_block(ci, + le64_to_cpu(eb->h_next_leaf_blk), + &next_eb_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + next_eb = (struct ocfs2_extent_block *)next_eb_bh->b_data; + el = &next_eb->h_list; + i = ocfs2_search_for_hole_index(el, v_cluster); + } + +no_more_extents: + if (i == le16_to_cpu(el->l_next_free_rec)) { + /* + * We're at the end of our existing allocation. Just + * return the maximum number of clusters we could + * possibly allocate. + */ + *num_clusters = UINT_MAX - v_cluster; + } else { + *num_clusters = le32_to_cpu(el->l_recs[i].e_cpos) - v_cluster; + } + + ret = 0; +out: + brelse(next_eb_bh); + return ret; +} + +static int ocfs2_get_clusters_nocache(struct inode *inode, + struct buffer_head *di_bh, + u32 v_cluster, unsigned int *hole_len, + struct ocfs2_extent_rec *ret_rec, + unsigned int *is_last) +{ + int i, ret, tree_height, len; + struct ocfs2_dinode *di; + struct ocfs2_extent_block *eb; + struct ocfs2_extent_list *el; + struct ocfs2_extent_rec *rec; + struct buffer_head *eb_bh = NULL; + + memset(ret_rec, 0, sizeof(*ret_rec)); + if (is_last) + *is_last = 0; + + di = (struct ocfs2_dinode *) di_bh->b_data; + el = &di->id2.i_list; + tree_height = le16_to_cpu(el->l_tree_depth); + + if (tree_height > 0) { + ret = ocfs2_find_leaf(INODE_CACHE(inode), el, v_cluster, + &eb_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + eb = (struct ocfs2_extent_block *) eb_bh->b_data; + el = &eb->h_list; + + if (el->l_tree_depth) { + ocfs2_error(inode->i_sb, + "Inode %llu has non zero tree depth in leaf block %llu\n", + inode->i_ino, + (unsigned long long)eb_bh->b_blocknr); + ret = -EROFS; + goto out; + } + } + + if (le16_to_cpu(el->l_next_free_rec) > le16_to_cpu(el->l_count)) { + ocfs2_error(inode->i_sb, + "Inode %llu has an invalid extent (next_free_rec %u, count %u)\n", + inode->i_ino, + le16_to_cpu(el->l_next_free_rec), + le16_to_cpu(el->l_count)); + ret = -EROFS; + goto out; + } + + i = ocfs2_search_extent_list(el, v_cluster); + if (i == -1) { + /* + * Holes can be larger than the maximum size of an + * extent, so we return their lengths in a separate + * field. + */ + if (hole_len) { + ret = ocfs2_figure_hole_clusters(INODE_CACHE(inode), + el, eb_bh, + v_cluster, &len); + if (ret) { + mlog_errno(ret); + goto out; + } + + *hole_len = len; + } + goto out_hole; + } + + rec = &el->l_recs[i]; + + BUG_ON(v_cluster < le32_to_cpu(rec->e_cpos)); + + if (!rec->e_blkno) { + ocfs2_error(inode->i_sb, + "Inode %llu has bad extent record (%u, %u, 0)\n", + inode->i_ino, + le32_to_cpu(rec->e_cpos), + ocfs2_rec_clusters(el, rec)); + ret = -EROFS; + goto out; + } + + *ret_rec = *rec; + + /* + * Checking for last extent is potentially expensive - we + * might have to look at the next leaf over to see if it's + * empty. + * + * The first two checks are to see whether the caller even + * cares for this information, and if the extent is at least + * the last in it's list. + * + * If those hold true, then the extent is last if any of the + * additional conditions hold true: + * - Extent list is in-inode + * - Extent list is right-most + * - Extent list is 2nd to rightmost, with empty right-most + */ + if (is_last) { + if (i == (le16_to_cpu(el->l_next_free_rec) - 1)) { + if (tree_height == 0) + *is_last = 1; + else if (eb->h_blkno == di->i_last_eb_blk) + *is_last = 1; + else if (eb->h_next_leaf_blk == di->i_last_eb_blk) { + ret = ocfs2_last_eb_is_empty(inode, di); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + if (ret == 1) + *is_last = 1; + } + } + } + +out_hole: + ret = 0; +out: + brelse(eb_bh); + return ret; +} + +static void ocfs2_relative_extent_offsets(struct super_block *sb, + u32 v_cluster, + struct ocfs2_extent_rec *rec, + u32 *p_cluster, u32 *num_clusters) + +{ + u32 coff = v_cluster - le32_to_cpu(rec->e_cpos); + + *p_cluster = ocfs2_blocks_to_clusters(sb, le64_to_cpu(rec->e_blkno)); + *p_cluster = *p_cluster + coff; + + if (num_clusters) + *num_clusters = le16_to_cpu(rec->e_leaf_clusters) - coff; +} + +int ocfs2_xattr_get_clusters(struct inode *inode, u32 v_cluster, + u32 *p_cluster, u32 *num_clusters, + struct ocfs2_extent_list *el, + unsigned int *extent_flags) +{ + int ret = 0, i; + struct buffer_head *eb_bh = NULL; + struct ocfs2_extent_block *eb; + struct ocfs2_extent_rec *rec; + u32 coff; + + if (el->l_tree_depth) { + ret = ocfs2_find_leaf(INODE_CACHE(inode), el, v_cluster, + &eb_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + eb = (struct ocfs2_extent_block *) eb_bh->b_data; + el = &eb->h_list; + + if (el->l_tree_depth) { + ocfs2_error(inode->i_sb, + "Inode %llu has non zero tree depth in xattr leaf block %llu\n", + inode->i_ino, + (unsigned long long)eb_bh->b_blocknr); + ret = -EROFS; + goto out; + } + } + + i = ocfs2_search_extent_list(el, v_cluster); + if (i == -1) { + ret = -EROFS; + mlog_errno(ret); + goto out; + } else { + rec = &el->l_recs[i]; + BUG_ON(v_cluster < le32_to_cpu(rec->e_cpos)); + + if (!rec->e_blkno) { + ocfs2_error(inode->i_sb, + "Inode %llu has bad extent record (%u, %u, 0) in xattr\n", + inode->i_ino, + le32_to_cpu(rec->e_cpos), + ocfs2_rec_clusters(el, rec)); + ret = -EROFS; + goto out; + } + coff = v_cluster - le32_to_cpu(rec->e_cpos); + *p_cluster = ocfs2_blocks_to_clusters(inode->i_sb, + le64_to_cpu(rec->e_blkno)); + *p_cluster = *p_cluster + coff; + if (num_clusters) + *num_clusters = ocfs2_rec_clusters(el, rec) - coff; + + if (extent_flags) + *extent_flags = rec->e_flags; + } +out: + brelse(eb_bh); + return ret; +} + +int ocfs2_get_clusters(struct inode *inode, u32 v_cluster, + u32 *p_cluster, u32 *num_clusters, + unsigned int *extent_flags) +{ + int ret; + unsigned int hole_len, flags = 0; + struct buffer_head *di_bh = NULL; + struct ocfs2_extent_rec rec; + + if (OCFS2_I(inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL) { + ret = -ERANGE; + mlog_errno(ret); + goto out; + } + + ret = ocfs2_extent_map_lookup(inode, v_cluster, p_cluster, + num_clusters, extent_flags); + if (ret == 0) + goto out; + + ret = ocfs2_read_inode_block(inode, &di_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_get_clusters_nocache(inode, di_bh, v_cluster, &hole_len, + &rec, NULL); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (rec.e_blkno == 0ULL) { + /* + * A hole was found. Return some canned values that + * callers can key on. If asked for, num_clusters will + * be populated with the size of the hole. + */ + *p_cluster = 0; + if (num_clusters) { + *num_clusters = hole_len; + } + } else { + ocfs2_relative_extent_offsets(inode->i_sb, v_cluster, &rec, + p_cluster, num_clusters); + flags = rec.e_flags; + + ocfs2_extent_map_insert_rec(inode, &rec); + } + + if (extent_flags) + *extent_flags = flags; + +out: + brelse(di_bh); + return ret; +} + +/* + * This expects alloc_sem to be held. The allocation cannot change at + * all while the map is in the process of being updated. + */ +int ocfs2_extent_map_get_blocks(struct inode *inode, u64 v_blkno, u64 *p_blkno, + u64 *ret_count, unsigned int *extent_flags) +{ + int ret; + int bpc = ocfs2_clusters_to_blocks(inode->i_sb, 1); + u32 cpos, num_clusters, p_cluster; + u64 boff = 0; + + cpos = ocfs2_blocks_to_clusters(inode->i_sb, v_blkno); + + ret = ocfs2_get_clusters(inode, cpos, &p_cluster, &num_clusters, + extent_flags); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * p_cluster == 0 indicates a hole. + */ + if (p_cluster) { + boff = ocfs2_clusters_to_blocks(inode->i_sb, p_cluster); + boff += (v_blkno & (u64)(bpc - 1)); + } + + *p_blkno = boff; + + if (ret_count) { + *ret_count = ocfs2_clusters_to_blocks(inode->i_sb, num_clusters); + *ret_count -= v_blkno & (u64)(bpc - 1); + } + +out: + return ret; +} + +/* + * The ocfs2_fiemap_inline() may be a little bit misleading, since + * it not only handles the fiemap for inlined files, but also deals + * with the fast symlink, cause they have no difference for extent + * mapping per se. + * + * Must be called with ip_alloc_sem semaphore held. + */ +static int ocfs2_fiemap_inline(struct inode *inode, struct buffer_head *di_bh, + struct fiemap_extent_info *fieinfo, + u64 map_start) +{ + int ret; + unsigned int id_count; + struct ocfs2_dinode *di; + u64 phys; + u32 flags = FIEMAP_EXTENT_DATA_INLINE|FIEMAP_EXTENT_LAST; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + lockdep_assert_held_read(&oi->ip_alloc_sem); + + di = (struct ocfs2_dinode *)di_bh->b_data; + if (ocfs2_inode_is_fast_symlink(inode)) + id_count = ocfs2_fast_symlink_chars(inode->i_sb); + else + id_count = le16_to_cpu(di->id2.i_data.id_count); + + if (map_start < id_count) { + phys = oi->ip_blkno << inode->i_sb->s_blocksize_bits; + if (ocfs2_inode_is_fast_symlink(inode)) + phys += offsetof(struct ocfs2_dinode, id2.i_symlink); + else + phys += offsetof(struct ocfs2_dinode, + id2.i_data.id_data); + + /* Release the ip_alloc_sem to prevent deadlock on page fault */ + up_read(&OCFS2_I(inode)->ip_alloc_sem); + ret = fiemap_fill_next_extent(fieinfo, 0, phys, id_count, + flags); + down_read(&OCFS2_I(inode)->ip_alloc_sem); + if (ret < 0) + return ret; + } + + return 0; +} + +int ocfs2_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + u64 map_start, u64 map_len) +{ + int ret, is_last; + u32 mapping_end, cpos; + unsigned int hole_size; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + u64 len_bytes, phys_bytes, virt_bytes; + struct buffer_head *di_bh = NULL; + struct ocfs2_extent_rec rec; + + ret = fiemap_prep(inode, fieinfo, map_start, &map_len, 0); + if (ret) + return ret; + + ret = ocfs2_inode_lock(inode, &di_bh, 0); + if (ret) { + mlog_errno(ret); + goto out; + } + + down_read(&OCFS2_I(inode)->ip_alloc_sem); + + /* + * Handle inline-data and fast symlink separately. + */ + if ((OCFS2_I(inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL) || + ocfs2_inode_is_fast_symlink(inode)) { + ret = ocfs2_fiemap_inline(inode, di_bh, fieinfo, map_start); + goto out_unlock; + } + + cpos = map_start >> osb->s_clustersize_bits; + mapping_end = ocfs2_clusters_for_bytes(inode->i_sb, + map_start + map_len); + is_last = 0; + while (cpos < mapping_end && !is_last) { + u32 fe_flags; + + ret = ocfs2_get_clusters_nocache(inode, di_bh, cpos, + &hole_size, &rec, &is_last); + if (ret) { + mlog_errno(ret); + goto out_unlock; + } + + if (rec.e_blkno == 0ULL) { + cpos += hole_size; + continue; + } + + fe_flags = 0; + if (rec.e_flags & OCFS2_EXT_UNWRITTEN) + fe_flags |= FIEMAP_EXTENT_UNWRITTEN; + if (rec.e_flags & OCFS2_EXT_REFCOUNTED) + fe_flags |= FIEMAP_EXTENT_SHARED; + if (is_last) + fe_flags |= FIEMAP_EXTENT_LAST; + len_bytes = (u64)le16_to_cpu(rec.e_leaf_clusters) << osb->s_clustersize_bits; + phys_bytes = le64_to_cpu(rec.e_blkno) << osb->sb->s_blocksize_bits; + virt_bytes = (u64)le32_to_cpu(rec.e_cpos) << osb->s_clustersize_bits; + /* Release the ip_alloc_sem to prevent deadlock on page fault */ + up_read(&OCFS2_I(inode)->ip_alloc_sem); + ret = fiemap_fill_next_extent(fieinfo, virt_bytes, phys_bytes, + len_bytes, fe_flags); + down_read(&OCFS2_I(inode)->ip_alloc_sem); + if (ret) + break; + + cpos = le32_to_cpu(rec.e_cpos)+ le16_to_cpu(rec.e_leaf_clusters); + } + + if (ret > 0) + ret = 0; + +out_unlock: + brelse(di_bh); + + up_read(&OCFS2_I(inode)->ip_alloc_sem); + + ocfs2_inode_unlock(inode, 0); +out: + + return ret; +} + +/* Is IO overwriting allocated blocks? */ +int ocfs2_overwrite_io(struct inode *inode, struct buffer_head *di_bh, + u64 map_start, u64 map_len) +{ + int ret = 0, is_last; + u32 mapping_end, cpos; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct ocfs2_extent_rec rec; + + if (OCFS2_I(inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL) { + if (ocfs2_size_fits_inline_data(di_bh, map_start + map_len)) + return ret; + else + return -EAGAIN; + } + + cpos = map_start >> osb->s_clustersize_bits; + mapping_end = ocfs2_clusters_for_bytes(inode->i_sb, + map_start + map_len); + is_last = 0; + while (cpos < mapping_end && !is_last) { + ret = ocfs2_get_clusters_nocache(inode, di_bh, cpos, + NULL, &rec, &is_last); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (rec.e_blkno == 0ULL) + break; + + if (rec.e_flags & OCFS2_EXT_REFCOUNTED) + break; + + cpos = le32_to_cpu(rec.e_cpos) + + le16_to_cpu(rec.e_leaf_clusters); + } + + if (cpos < mapping_end) + ret = -EAGAIN; +out: + return ret; +} + +int ocfs2_seek_data_hole_offset(struct file *file, loff_t *offset, int whence) +{ + struct inode *inode = file->f_mapping->host; + int ret; + unsigned int is_last = 0, is_data = 0; + u16 cs_bits = OCFS2_SB(inode->i_sb)->s_clustersize_bits; + u32 cpos, cend, clen, hole_size; + u64 extoff, extlen; + struct buffer_head *di_bh = NULL; + struct ocfs2_extent_rec rec; + + BUG_ON(whence != SEEK_DATA && whence != SEEK_HOLE); + + ret = ocfs2_inode_lock(inode, &di_bh, 0); + if (ret) { + mlog_errno(ret); + goto out; + } + + down_read(&OCFS2_I(inode)->ip_alloc_sem); + + if (*offset >= i_size_read(inode)) { + ret = -ENXIO; + goto out_unlock; + } + + if (OCFS2_I(inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL) { + if (whence == SEEK_HOLE) + *offset = i_size_read(inode); + goto out_unlock; + } + + clen = 0; + cpos = *offset >> cs_bits; + cend = ocfs2_clusters_for_bytes(inode->i_sb, i_size_read(inode)); + + while (cpos < cend && !is_last) { + ret = ocfs2_get_clusters_nocache(inode, di_bh, cpos, &hole_size, + &rec, &is_last); + if (ret) { + mlog_errno(ret); + goto out_unlock; + } + + extoff = cpos; + extoff <<= cs_bits; + + if (rec.e_blkno == 0ULL) { + clen = hole_size; + is_data = 0; + } else { + clen = le16_to_cpu(rec.e_leaf_clusters) - + (cpos - le32_to_cpu(rec.e_cpos)); + is_data = (rec.e_flags & OCFS2_EXT_UNWRITTEN) ? 0 : 1; + } + + if ((!is_data && whence == SEEK_HOLE) || + (is_data && whence == SEEK_DATA)) { + if (extoff > *offset) + *offset = extoff; + goto out_unlock; + } + + if (!is_last) + cpos += clen; + } + + if (whence == SEEK_HOLE) { + extoff = cpos; + extoff <<= cs_bits; + extlen = clen; + extlen <<= cs_bits; + + if ((extoff + extlen) > i_size_read(inode)) + extlen = i_size_read(inode) - extoff; + extoff += extlen; + if (extoff > *offset) + *offset = extoff; + goto out_unlock; + } + + ret = -ENXIO; + +out_unlock: + + brelse(di_bh); + + up_read(&OCFS2_I(inode)->ip_alloc_sem); + + ocfs2_inode_unlock(inode, 0); +out: + return ret; +} + +int ocfs2_read_virt_blocks(struct inode *inode, u64 v_block, int nr, + struct buffer_head *bhs[], int flags, + int (*validate)(struct super_block *sb, + struct buffer_head *bh)) +{ + int rc = 0; + u64 p_block, p_count; + int i, count, done = 0; + + trace_ocfs2_read_virt_blocks( + inode, (unsigned long long)v_block, nr, bhs, flags, + validate); + + if (((v_block + nr - 1) << inode->i_sb->s_blocksize_bits) >= + i_size_read(inode)) { + BUG_ON(!(flags & OCFS2_BH_READAHEAD)); + goto out; + } + + while (done < nr) { + if (!down_read_trylock(&OCFS2_I(inode)->ip_alloc_sem)) { + rc = -EAGAIN; + mlog(ML_ERROR, + "Inode #%llu ip_alloc_sem is temporarily unavailable\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno); + break; + } + rc = ocfs2_extent_map_get_blocks(inode, v_block + done, + &p_block, &p_count, NULL); + up_read(&OCFS2_I(inode)->ip_alloc_sem); + if (rc) { + mlog_errno(rc); + break; + } + + if (!p_block) { + rc = -EIO; + mlog(ML_ERROR, + "Inode #%llu contains a hole at offset %llu\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno, + (unsigned long long)(v_block + done) << + inode->i_sb->s_blocksize_bits); + break; + } + + count = nr - done; + if (p_count < count) + count = p_count; + + /* + * If the caller passed us bhs, they should have come + * from a previous readahead call to this function. Thus, + * they should have the right b_blocknr. + */ + for (i = 0; i < count; i++) { + if (!bhs[done + i]) + continue; + BUG_ON(bhs[done + i]->b_blocknr != (p_block + i)); + } + + rc = ocfs2_read_blocks(INODE_CACHE(inode), p_block, count, + bhs + done, flags, validate); + if (rc) { + mlog_errno(rc); + break; + } + done += count; + } + +out: + return rc; +} + + diff --git a/fs/ocfs2/extent_map.h b/fs/ocfs2/extent_map.h new file mode 100644 index 000000000..bc4ed59fb --- /dev/null +++ b/fs/ocfs2/extent_map.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * extent_map.h + * + * In-memory file extent mappings for OCFS2. + * + * Copyright (C) 2004 Oracle. All rights reserved. + */ + +#ifndef _EXTENT_MAP_H +#define _EXTENT_MAP_H + +struct ocfs2_extent_map_item { + unsigned int ei_cpos; + unsigned int ei_phys; + unsigned int ei_clusters; + unsigned int ei_flags; + + struct list_head ei_list; +}; + +#define OCFS2_MAX_EXTENT_MAP_ITEMS 3 +struct ocfs2_extent_map { + unsigned int em_num_items; + struct list_head em_list; +}; + +void ocfs2_extent_map_init(struct inode *inode); +void ocfs2_extent_map_trunc(struct inode *inode, unsigned int cluster); +void ocfs2_extent_map_insert_rec(struct inode *inode, + struct ocfs2_extent_rec *rec); + +int ocfs2_get_clusters(struct inode *inode, u32 v_cluster, u32 *p_cluster, + u32 *num_clusters, unsigned int *extent_flags); +int ocfs2_extent_map_get_blocks(struct inode *inode, u64 v_blkno, u64 *p_blkno, + u64 *ret_count, unsigned int *extent_flags); + +int ocfs2_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + u64 map_start, u64 map_len); + +int ocfs2_overwrite_io(struct inode *inode, struct buffer_head *di_bh, + u64 map_start, u64 map_len); + +int ocfs2_seek_data_hole_offset(struct file *file, loff_t *offset, int origin); + +int ocfs2_xattr_get_clusters(struct inode *inode, u32 v_cluster, + u32 *p_cluster, u32 *num_clusters, + struct ocfs2_extent_list *el, + unsigned int *extent_flags); + +int ocfs2_read_virt_blocks(struct inode *inode, u64 v_block, int nr, + struct buffer_head *bhs[], int flags, + int (*validate)(struct super_block *sb, + struct buffer_head *bh)); +int ocfs2_figure_hole_clusters(struct ocfs2_caching_info *ci, + struct ocfs2_extent_list *el, + struct buffer_head *eb_bh, + u32 v_cluster, + u32 *num_clusters); +static inline int ocfs2_read_virt_block(struct inode *inode, u64 v_block, + struct buffer_head **bh, + int (*validate)(struct super_block *sb, + struct buffer_head *bh)) +{ + int status = 0; + + if (bh == NULL) { + printk("ocfs2: bh == NULL\n"); + status = -EINVAL; + goto bail; + } + + status = ocfs2_read_virt_blocks(inode, v_block, 1, bh, 0, validate); + +bail: + return status; +} + + +#endif /* _EXTENT_MAP_H */ diff --git a/fs/ocfs2/file.c b/fs/ocfs2/file.c new file mode 100644 index 000000000..d6e977ba6 --- /dev/null +++ b/fs/ocfs2/file.c @@ -0,0 +1,2898 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * file.c + * + * File open, close, extend, truncate + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ocfs2.h" + +#include "alloc.h" +#include "aops.h" +#include "dir.h" +#include "dlmglue.h" +#include "extent_map.h" +#include "file.h" +#include "sysfile.h" +#include "inode.h" +#include "ioctl.h" +#include "journal.h" +#include "locks.h" +#include "mmap.h" +#include "suballoc.h" +#include "super.h" +#include "xattr.h" +#include "acl.h" +#include "quota.h" +#include "refcounttree.h" +#include "ocfs2_trace.h" + +#include "buffer_head_io.h" + +static int ocfs2_init_file_private(struct inode *inode, struct file *file) +{ + struct ocfs2_file_private *fp; + + fp = kzalloc_obj(struct ocfs2_file_private); + if (!fp) + return -ENOMEM; + + fp->fp_file = file; + mutex_init(&fp->fp_mutex); + ocfs2_file_lock_res_init(&fp->fp_flock, fp); + file->private_data = fp; + + return 0; +} + +static void ocfs2_free_file_private(struct inode *inode, struct file *file) +{ + struct ocfs2_file_private *fp = file->private_data; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + if (fp) { + ocfs2_simple_drop_lockres(osb, &fp->fp_flock); + ocfs2_lock_res_free(&fp->fp_flock); + kfree(fp); + file->private_data = NULL; + } +} + +static int ocfs2_file_open(struct inode *inode, struct file *file) +{ + int status; + int mode = file->f_flags; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + + trace_ocfs2_file_open(inode, file, file->f_path.dentry, + (unsigned long long)oi->ip_blkno, + file->f_path.dentry->d_name.len, + file->f_path.dentry->d_name.name, mode); + + if (file->f_mode & FMODE_WRITE) { + status = dquot_initialize(inode); + if (status) + goto leave; + } + + spin_lock(&oi->ip_lock); + + /* Check that the inode hasn't been wiped from disk by another + * node. If it hasn't then we're safe as long as we hold the + * spin lock until our increment of open count. */ + if (oi->ip_flags & OCFS2_INODE_DELETED) { + spin_unlock(&oi->ip_lock); + + status = -ENOENT; + goto leave; + } + + if (mode & O_DIRECT) + oi->ip_flags |= OCFS2_INODE_OPEN_DIRECT; + + oi->ip_open_count++; + spin_unlock(&oi->ip_lock); + + status = ocfs2_init_file_private(inode, file); + if (status) { + /* + * We want to set open count back if we're failing the + * open. + */ + spin_lock(&oi->ip_lock); + oi->ip_open_count--; + spin_unlock(&oi->ip_lock); + } + + file->f_mode |= FMODE_NOWAIT; + +leave: + return status; +} + +static int ocfs2_file_release(struct inode *inode, struct file *file) +{ + struct ocfs2_inode_info *oi = OCFS2_I(inode); + + spin_lock(&oi->ip_lock); + if (!--oi->ip_open_count) + oi->ip_flags &= ~OCFS2_INODE_OPEN_DIRECT; + + trace_ocfs2_file_release(inode, file, file->f_path.dentry, + oi->ip_blkno, + file->f_path.dentry->d_name.len, + file->f_path.dentry->d_name.name, + oi->ip_open_count); + spin_unlock(&oi->ip_lock); + + ocfs2_free_file_private(inode, file); + + return 0; +} + +static int ocfs2_dir_open(struct inode *inode, struct file *file) +{ + return ocfs2_init_file_private(inode, file); +} + +static int ocfs2_dir_release(struct inode *inode, struct file *file) +{ + ocfs2_free_file_private(inode, file); + return 0; +} + +static int ocfs2_sync_file(struct file *file, loff_t start, loff_t end, + int datasync) +{ + int err = 0; + struct inode *inode = file->f_mapping->host; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct ocfs2_inode_info *oi = OCFS2_I(inode); + journal_t *journal = osb->journal->j_journal; + int ret; + tid_t commit_tid; + bool needs_barrier = false; + + trace_ocfs2_sync_file(inode, file, file->f_path.dentry, + oi->ip_blkno, + file->f_path.dentry->d_name.len, + file->f_path.dentry->d_name.name, + (unsigned long long)datasync); + + if (unlikely(ocfs2_emergency_state(osb))) + return -EROFS; + + err = file_write_and_wait_range(file, start, end); + if (err) + return err; + + commit_tid = datasync ? oi->i_datasync_tid : oi->i_sync_tid; + if (journal->j_flags & JBD2_BARRIER && + !jbd2_trans_will_send_data_barrier(journal, commit_tid)) + needs_barrier = true; + err = jbd2_complete_transaction(journal, commit_tid); + if (needs_barrier) { + ret = blkdev_issue_flush(inode->i_sb->s_bdev); + if (!err) + err = ret; + } + + if (err) + mlog_errno(err); + + return (err < 0) ? -EIO : 0; +} + +int ocfs2_should_update_atime(struct inode *inode, + struct vfsmount *vfsmnt) +{ + struct timespec64 now; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + if (unlikely(ocfs2_emergency_state(osb))) + return 0; + + if ((inode->i_flags & S_NOATIME) || + ((inode->i_sb->s_flags & SB_NODIRATIME) && S_ISDIR(inode->i_mode))) + return 0; + + /* + * We can be called with no vfsmnt structure - NFSD will + * sometimes do this. + * + * Note that our action here is different than touch_atime() - + * if we can't tell whether this is a noatime mount, then we + * don't know whether to trust the value of s_atime_quantum. + */ + if (vfsmnt == NULL) + return 0; + + if ((vfsmnt->mnt_flags & MNT_NOATIME) || + ((vfsmnt->mnt_flags & MNT_NODIRATIME) && S_ISDIR(inode->i_mode))) + return 0; + + if (vfsmnt->mnt_flags & MNT_RELATIME) { + struct timespec64 ctime = inode_get_ctime(inode); + struct timespec64 atime = inode_get_atime(inode); + struct timespec64 mtime = inode_get_mtime(inode); + + if ((timespec64_compare(&atime, &mtime) <= 0) || + (timespec64_compare(&atime, &ctime) <= 0)) + return 1; + + return 0; + } + + now = current_time(inode); + if ((now.tv_sec - inode_get_atime_sec(inode) <= osb->s_atime_quantum)) + return 0; + else + return 1; +} + +int ocfs2_update_inode_atime(struct inode *inode, + struct buffer_head *bh) +{ + int ret; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + handle_t *handle; + struct ocfs2_dinode *di = (struct ocfs2_dinode *) bh->b_data; + + handle = ocfs2_start_trans(osb, OCFS2_INODE_UPDATE_CREDITS); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out; + } + + ret = ocfs2_journal_access_di(handle, INODE_CACHE(inode), bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + /* + * Don't use ocfs2_mark_inode_dirty() here as we don't always + * have i_rwsem to guard against concurrent changes to other + * inode fields. + */ + inode_set_atime_to_ts(inode, current_time(inode)); + di->i_atime = cpu_to_le64(inode_get_atime_sec(inode)); + di->i_atime_nsec = cpu_to_le32(inode_get_atime_nsec(inode)); + ocfs2_update_inode_fsync_trans(handle, inode, 0); + ocfs2_journal_dirty(handle, bh); + +out_commit: + ocfs2_commit_trans(osb, handle); +out: + return ret; +} + +int ocfs2_set_inode_size(handle_t *handle, + struct inode *inode, + struct buffer_head *fe_bh, + u64 new_i_size) +{ + int status; + + i_size_write(inode, new_i_size); + inode->i_blocks = ocfs2_inode_sector_count(inode); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + + status = ocfs2_mark_inode_dirty(handle, inode, fe_bh); + if (status < 0) { + mlog_errno(status); + goto bail; + } + +bail: + return status; +} + +int ocfs2_simple_size_update(struct inode *inode, + struct buffer_head *di_bh, + u64 new_i_size) +{ + int ret; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + handle_t *handle = NULL; + + handle = ocfs2_start_trans(osb, OCFS2_INODE_UPDATE_CREDITS); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out; + } + + ret = ocfs2_set_inode_size(handle, inode, di_bh, + new_i_size); + if (ret < 0) + mlog_errno(ret); + + ocfs2_update_inode_fsync_trans(handle, inode, 0); + ocfs2_commit_trans(osb, handle); +out: + return ret; +} + +static int ocfs2_cow_file_pos(struct inode *inode, + struct buffer_head *fe_bh, + u64 offset) +{ + int status; + u32 phys, cpos = offset >> OCFS2_SB(inode->i_sb)->s_clustersize_bits; + unsigned int num_clusters = 0; + unsigned int ext_flags = 0; + + /* + * If the new offset is aligned to the range of the cluster, there is + * no space for ocfs2_zero_range_for_truncate to fill, so no need to + * CoW either. + */ + if ((offset & (OCFS2_SB(inode->i_sb)->s_clustersize - 1)) == 0) + return 0; + + status = ocfs2_get_clusters(inode, cpos, &phys, + &num_clusters, &ext_flags); + if (status) { + mlog_errno(status); + goto out; + } + + if (!(ext_flags & OCFS2_EXT_REFCOUNTED)) + goto out; + + return ocfs2_refcount_cow(inode, fe_bh, cpos, 1, cpos+1); + +out: + return status; +} + +static int ocfs2_orphan_for_truncate(struct ocfs2_super *osb, + struct inode *inode, + struct buffer_head *fe_bh, + u64 new_i_size) +{ + int status; + handle_t *handle; + struct ocfs2_dinode *di; + u64 cluster_bytes; + + /* + * We need to CoW the cluster contains the offset if it is reflinked + * since we will call ocfs2_zero_range_for_truncate later which will + * write "0" from offset to the end of the cluster. + */ + status = ocfs2_cow_file_pos(inode, fe_bh, new_i_size); + if (status) { + mlog_errno(status); + return status; + } + + /* TODO: This needs to actually orphan the inode in this + * transaction. */ + + handle = ocfs2_start_trans(osb, OCFS2_INODE_UPDATE_CREDITS); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + mlog_errno(status); + goto out; + } + + status = ocfs2_journal_access_di(handle, INODE_CACHE(inode), fe_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto out_commit; + } + + /* + * Do this before setting i_size. + */ + cluster_bytes = ocfs2_align_bytes_to_clusters(inode->i_sb, new_i_size); + status = ocfs2_zero_range_for_truncate(inode, handle, new_i_size, + cluster_bytes); + if (status) { + mlog_errno(status); + goto out_commit; + } + + i_size_write(inode, new_i_size); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + + di = (struct ocfs2_dinode *) fe_bh->b_data; + di->i_size = cpu_to_le64(new_i_size); + di->i_ctime = di->i_mtime = cpu_to_le64(inode_get_ctime_sec(inode)); + di->i_ctime_nsec = di->i_mtime_nsec = cpu_to_le32(inode_get_ctime_nsec(inode)); + ocfs2_update_inode_fsync_trans(handle, inode, 0); + + ocfs2_journal_dirty(handle, fe_bh); + +out_commit: + ocfs2_commit_trans(osb, handle); +out: + return status; +} + +int ocfs2_truncate_file(struct inode *inode, + struct buffer_head *di_bh, + u64 new_i_size) +{ + int status = 0; + struct ocfs2_dinode *fe = NULL; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + /* + * On local mounts ocfs2_inode_lock_update() skips the inode + * refresh path, so truncation still needs to reject an inode + * state that no longer matches di_bh. + */ + fe = (struct ocfs2_dinode *) di_bh->b_data; + + trace_ocfs2_truncate_file((unsigned long long)OCFS2_I(inode)->ip_blkno, + (unsigned long long)le64_to_cpu(fe->i_size), + (unsigned long long)new_i_size); + + if (unlikely(le64_to_cpu(fe->i_size) != i_size_read(inode))) { + status = ocfs2_error(inode->i_sb, + "Inode %llu has inconsistent i_size: inode = %lld, dinode = %llu, i_flags = 0x%x\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno, + i_size_read(inode), + (unsigned long long)le64_to_cpu(fe->i_size), + le32_to_cpu(fe->i_flags)); + goto bail; + } + + if (new_i_size > le64_to_cpu(fe->i_size)) { + trace_ocfs2_truncate_file_error( + (unsigned long long)le64_to_cpu(fe->i_size), + (unsigned long long)new_i_size); + status = -EINVAL; + mlog_errno(status); + goto bail; + } + + down_write(&OCFS2_I(inode)->ip_alloc_sem); + + ocfs2_resv_discard(&osb->osb_la_resmap, + &OCFS2_I(inode)->ip_la_data_resv); + + /* + * The inode lock forced other nodes to sync and drop their + * pages, which (correctly) happens even if we have a truncate + * without allocation change - ocfs2 cluster sizes can be much + * greater than page size, so we have to truncate them + * anyway. + */ + + if (OCFS2_I(inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL) { + unmap_mapping_range(inode->i_mapping, + new_i_size + PAGE_SIZE - 1, 0, 1); + truncate_inode_pages(inode->i_mapping, new_i_size); + status = ocfs2_truncate_inline(inode, di_bh, new_i_size, + i_size_read(inode), 1); + if (status) + mlog_errno(status); + + goto bail_unlock_sem; + } + + /* alright, we're going to need to do a full blown alloc size + * change. Orphan the inode so that recovery can complete the + * truncate if necessary. This does the task of marking + * i_size. */ + status = ocfs2_orphan_for_truncate(osb, inode, di_bh, new_i_size); + if (status < 0) { + mlog_errno(status); + goto bail_unlock_sem; + } + + unmap_mapping_range(inode->i_mapping, new_i_size + PAGE_SIZE - 1, 0, 1); + truncate_inode_pages(inode->i_mapping, new_i_size); + + status = ocfs2_commit_truncate(osb, inode, di_bh); + if (status < 0) { + mlog_errno(status); + goto bail_unlock_sem; + } + + /* TODO: orphan dir cleanup here. */ +bail_unlock_sem: + up_write(&OCFS2_I(inode)->ip_alloc_sem); + +bail: + if (!status && OCFS2_I(inode)->ip_clusters == 0) + status = ocfs2_try_remove_refcount_tree(inode, di_bh); + + return status; +} + +/* + * extend file allocation only here. + * we'll update all the disk stuff, and oip->alloc_size + * + * expect stuff to be locked, a transaction started and enough data / + * metadata reservations in the contexts. + * + * Will return -EAGAIN, and a reason if a restart is needed. + * If passed in, *reason will always be set, even in error. + */ +int ocfs2_add_inode_data(struct ocfs2_super *osb, + struct inode *inode, + u32 *logical_offset, + u32 clusters_to_add, + int mark_unwritten, + struct buffer_head *fe_bh, + handle_t *handle, + struct ocfs2_alloc_context *data_ac, + struct ocfs2_alloc_context *meta_ac, + enum ocfs2_alloc_restarted *reason_ret) +{ + struct ocfs2_extent_tree et; + + ocfs2_init_dinode_extent_tree(&et, INODE_CACHE(inode), fe_bh); + return ocfs2_add_clusters_in_btree(handle, &et, logical_offset, + clusters_to_add, mark_unwritten, + data_ac, meta_ac, reason_ret); +} + +static int ocfs2_extend_allocation(struct inode *inode, u32 logical_start, + u32 clusters_to_add, int mark_unwritten) +{ + int status = 0; + int restart_func = 0; + int credits; + u32 prev_clusters; + struct buffer_head *bh = NULL; + struct ocfs2_dinode *fe = NULL; + handle_t *handle = NULL; + struct ocfs2_alloc_context *data_ac = NULL; + struct ocfs2_alloc_context *meta_ac = NULL; + enum ocfs2_alloc_restarted why = RESTART_NONE; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct ocfs2_extent_tree et; + int did_quota = 0; + + /* + * Unwritten extent only exists for file systems which + * support holes. + */ + BUG_ON(mark_unwritten && !ocfs2_sparse_alloc(osb)); + + status = ocfs2_read_inode_block(inode, &bh); + if (status < 0) { + mlog_errno(status); + goto leave; + } + fe = (struct ocfs2_dinode *) bh->b_data; + +restart_all: + BUG_ON(le32_to_cpu(fe->i_clusters) != OCFS2_I(inode)->ip_clusters); + + ocfs2_init_dinode_extent_tree(&et, INODE_CACHE(inode), bh); + status = ocfs2_lock_allocators(inode, &et, clusters_to_add, 0, + &data_ac, &meta_ac); + if (status) { + mlog_errno(status); + goto leave; + } + + credits = ocfs2_calc_extend_credits(osb->sb, &fe->id2.i_list); + handle = ocfs2_start_trans(osb, credits); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + handle = NULL; + mlog_errno(status); + goto leave; + } + +restarted_transaction: + trace_ocfs2_extend_allocation( + (unsigned long long)OCFS2_I(inode)->ip_blkno, + (unsigned long long)i_size_read(inode), + le32_to_cpu(fe->i_clusters), clusters_to_add, + why, restart_func); + + status = dquot_alloc_space_nodirty(inode, + ocfs2_clusters_to_bytes(osb->sb, clusters_to_add)); + if (status) + goto leave; + did_quota = 1; + + /* reserve a write to the file entry early on - that we if we + * run out of credits in the allocation path, we can still + * update i_size. */ + status = ocfs2_journal_access_di(handle, INODE_CACHE(inode), bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto leave; + } + + prev_clusters = OCFS2_I(inode)->ip_clusters; + + status = ocfs2_add_inode_data(osb, + inode, + &logical_start, + clusters_to_add, + mark_unwritten, + bh, + handle, + data_ac, + meta_ac, + &why); + if ((status < 0) && (status != -EAGAIN)) { + if (status != -ENOSPC) + mlog_errno(status); + goto leave; + } + ocfs2_update_inode_fsync_trans(handle, inode, 1); + ocfs2_journal_dirty(handle, bh); + + spin_lock(&OCFS2_I(inode)->ip_lock); + clusters_to_add -= (OCFS2_I(inode)->ip_clusters - prev_clusters); + spin_unlock(&OCFS2_I(inode)->ip_lock); + /* Release unused quota reservation */ + dquot_free_space(inode, + ocfs2_clusters_to_bytes(osb->sb, clusters_to_add)); + did_quota = 0; + + if (why != RESTART_NONE && clusters_to_add) { + if (why == RESTART_META) { + restart_func = 1; + status = 0; + } else { + BUG_ON(why != RESTART_TRANS); + + status = ocfs2_allocate_extend_trans(handle, 1); + if (status < 0) { + /* handle still has to be committed at + * this point. */ + status = -ENOMEM; + mlog_errno(status); + goto leave; + } + goto restarted_transaction; + } + } + + trace_ocfs2_extend_allocation_end(OCFS2_I(inode)->ip_blkno, + le32_to_cpu(fe->i_clusters), + (unsigned long long)le64_to_cpu(fe->i_size), + OCFS2_I(inode)->ip_clusters, + (unsigned long long)i_size_read(inode)); + +leave: + if (status < 0 && did_quota) + dquot_free_space(inode, + ocfs2_clusters_to_bytes(osb->sb, clusters_to_add)); + if (handle) { + ocfs2_commit_trans(osb, handle); + handle = NULL; + } + if (data_ac) { + ocfs2_free_alloc_context(data_ac); + data_ac = NULL; + } + if (meta_ac) { + ocfs2_free_alloc_context(meta_ac); + meta_ac = NULL; + } + if ((!status) && restart_func) { + restart_func = 0; + goto restart_all; + } + brelse(bh); + bh = NULL; + + return status; +} + +/* + * While a write will already be ordering the data, a truncate will not. + * Thus, we need to explicitly order the zeroed pages. + */ +static handle_t *ocfs2_zero_start_ordered_transaction(struct inode *inode, + struct buffer_head *di_bh, + loff_t start_byte, + loff_t length) +{ + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + handle_t *handle = NULL; + int ret = 0; + + if (!ocfs2_should_order_data(inode)) + goto out; + + handle = ocfs2_start_trans(osb, OCFS2_INODE_UPDATE_CREDITS); + if (IS_ERR(handle)) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + ret = ocfs2_jbd2_inode_add_write(handle, inode, start_byte, length); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_journal_access_di(handle, INODE_CACHE(inode), di_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) + mlog_errno(ret); + ocfs2_update_inode_fsync_trans(handle, inode, 1); + +out: + if (ret) { + if (!IS_ERR(handle)) + ocfs2_commit_trans(osb, handle); + handle = ERR_PTR(ret); + } + return handle; +} + +/* Some parts of this taken from generic_cont_expand, which turned out + * to be too fragile to do exactly what we need without us having to + * worry about recursive locking in ->write_begin() and ->write_end(). */ +static int ocfs2_write_zero_page(struct inode *inode, u64 abs_from, + u64 abs_to, struct buffer_head *di_bh) +{ + struct address_space *mapping = inode->i_mapping; + struct folio *folio; + unsigned long index = abs_from >> PAGE_SHIFT; + handle_t *handle; + int ret = 0; + unsigned zero_from, zero_to, block_start, block_end; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data; + + BUG_ON(abs_from >= abs_to); + BUG_ON(abs_to > (((u64)index + 1) << PAGE_SHIFT)); + BUG_ON(abs_from & (inode->i_blkbits - 1)); + + handle = ocfs2_zero_start_ordered_transaction(inode, di_bh, + abs_from, + abs_to - abs_from); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + goto out; + } + + folio = __filemap_get_folio(mapping, index, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, GFP_NOFS); + if (IS_ERR(folio)) { + ret = PTR_ERR(folio); + mlog_errno(ret); + goto out_commit_trans; + } + + /* Get the offsets within the folio that we want to zero */ + zero_from = offset_in_folio(folio, abs_from); + zero_to = offset_in_folio(folio, abs_to); + if (!zero_to) + zero_to = folio_size(folio); + + trace_ocfs2_write_zero_page( + (unsigned long long)OCFS2_I(inode)->ip_blkno, + (unsigned long long)abs_from, + (unsigned long long)abs_to, + index, zero_from, zero_to); + + /* We know that zero_from is block aligned */ + for (block_start = zero_from; block_start < zero_to; + block_start = block_end) { + block_end = block_start + i_blocksize(inode); + + /* + * block_start is block-aligned. Bump it by one to force + * __block_write_begin and block_commit_write to zero the + * whole block. + */ + ret = __block_write_begin(folio, block_start + 1, 0, + ocfs2_get_block); + if (ret < 0) { + mlog_errno(ret); + goto out_unlock; + } + + + /* must not update i_size! */ + block_commit_write(folio, block_start + 1, block_start + 1); + } + + /* + * fs-writeback will release the dirty pages without page lock + * whose offset are over inode size, the release happens at + * block_write_full_folio(). + */ + i_size_write(inode, abs_to); + inode->i_blocks = ocfs2_inode_sector_count(inode); + di->i_size = cpu_to_le64((u64)i_size_read(inode)); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + di->i_mtime = di->i_ctime = cpu_to_le64(inode_get_mtime_sec(inode)); + di->i_ctime_nsec = cpu_to_le32(inode_get_mtime_nsec(inode)); + di->i_mtime_nsec = di->i_ctime_nsec; + if (handle) { + ocfs2_journal_dirty(handle, di_bh); + ocfs2_update_inode_fsync_trans(handle, inode, 1); + } + +out_unlock: + folio_unlock(folio); + folio_put(folio); +out_commit_trans: + if (handle) + ocfs2_commit_trans(OCFS2_SB(inode->i_sb), handle); +out: + return ret; +} + +/* + * Find the next range to zero. We do this in terms of bytes because + * that's what ocfs2_zero_extend() wants, and it is dealing with the + * pagecache. We may return multiple extents. + * + * zero_start and zero_end are ocfs2_zero_extend()s current idea of what + * needs to be zeroed. range_start and range_end return the next zeroing + * range. A subsequent call should pass the previous range_end as its + * zero_start. If range_end is 0, there's nothing to do. + * + * Unwritten extents are skipped over. Refcounted extents are CoWd. + */ +static int ocfs2_zero_extend_get_range(struct inode *inode, + struct buffer_head *di_bh, + u64 zero_start, u64 zero_end, + u64 *range_start, u64 *range_end) +{ + int rc = 0, needs_cow = 0; + u32 p_cpos, zero_clusters = 0; + u32 zero_cpos = + zero_start >> OCFS2_SB(inode->i_sb)->s_clustersize_bits; + u32 last_cpos = ocfs2_clusters_for_bytes(inode->i_sb, zero_end); + unsigned int num_clusters = 0; + unsigned int ext_flags = 0; + + while (zero_cpos < last_cpos) { + rc = ocfs2_get_clusters(inode, zero_cpos, &p_cpos, + &num_clusters, &ext_flags); + if (rc) { + mlog_errno(rc); + goto out; + } + + if (p_cpos && !(ext_flags & OCFS2_EXT_UNWRITTEN)) { + zero_clusters = num_clusters; + if (ext_flags & OCFS2_EXT_REFCOUNTED) + needs_cow = 1; + break; + } + + zero_cpos += num_clusters; + } + if (!zero_clusters) { + *range_end = 0; + goto out; + } + + while ((zero_cpos + zero_clusters) < last_cpos) { + rc = ocfs2_get_clusters(inode, zero_cpos + zero_clusters, + &p_cpos, &num_clusters, + &ext_flags); + if (rc) { + mlog_errno(rc); + goto out; + } + + if (!p_cpos || (ext_flags & OCFS2_EXT_UNWRITTEN)) + break; + if (ext_flags & OCFS2_EXT_REFCOUNTED) + needs_cow = 1; + zero_clusters += num_clusters; + } + if ((zero_cpos + zero_clusters) > last_cpos) + zero_clusters = last_cpos - zero_cpos; + + if (needs_cow) { + rc = ocfs2_refcount_cow(inode, di_bh, zero_cpos, + zero_clusters, UINT_MAX); + if (rc) { + mlog_errno(rc); + goto out; + } + } + + *range_start = ocfs2_clusters_to_bytes(inode->i_sb, zero_cpos); + *range_end = ocfs2_clusters_to_bytes(inode->i_sb, + zero_cpos + zero_clusters); + +out: + return rc; +} + +/* + * Zero one range returned from ocfs2_zero_extend_get_range(). The caller + * has made sure that the entire range needs zeroing. + */ +static int ocfs2_zero_extend_range(struct inode *inode, u64 range_start, + u64 range_end, struct buffer_head *di_bh) +{ + int rc = 0; + u64 next_pos; + u64 zero_pos = range_start; + + trace_ocfs2_zero_extend_range( + (unsigned long long)OCFS2_I(inode)->ip_blkno, + (unsigned long long)range_start, + (unsigned long long)range_end); + BUG_ON(range_start >= range_end); + + while (zero_pos < range_end) { + next_pos = (zero_pos & PAGE_MASK) + PAGE_SIZE; + if (next_pos > range_end) + next_pos = range_end; + rc = ocfs2_write_zero_page(inode, zero_pos, next_pos, di_bh); + if (rc < 0) { + mlog_errno(rc); + break; + } + zero_pos = next_pos; + + /* + * Very large extends have the potential to lock up + * the cpu for extended periods of time. + */ + cond_resched(); + } + + return rc; +} + +int ocfs2_zero_extend(struct inode *inode, struct buffer_head *di_bh, + loff_t zero_to_size) +{ + int ret = 0; + u64 zero_start, range_start = 0, range_end = 0; + struct super_block *sb = inode->i_sb; + + zero_start = ocfs2_align_bytes_to_blocks(sb, i_size_read(inode)); + trace_ocfs2_zero_extend((unsigned long long)OCFS2_I(inode)->ip_blkno, + (unsigned long long)zero_start, + (unsigned long long)i_size_read(inode)); + while (zero_start < zero_to_size) { + ret = ocfs2_zero_extend_get_range(inode, di_bh, zero_start, + zero_to_size, + &range_start, + &range_end); + if (ret) { + mlog_errno(ret); + break; + } + if (!range_end) + break; + /* Trim the ends */ + if (range_start < zero_start) + range_start = zero_start; + if (range_end > zero_to_size) + range_end = zero_to_size; + + ret = ocfs2_zero_extend_range(inode, range_start, + range_end, di_bh); + if (ret) { + mlog_errno(ret); + break; + } + zero_start = range_end; + } + + return ret; +} + +int ocfs2_extend_no_holes(struct inode *inode, struct buffer_head *di_bh, + u64 new_i_size, u64 zero_to) +{ + int ret; + u32 clusters_to_add; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + + /* + * Only quota files call this without a bh, and they can't be + * refcounted. + */ + BUG_ON(!di_bh && ocfs2_is_refcount_inode(inode)); + BUG_ON(!di_bh && !(oi->ip_flags & OCFS2_INODE_SYSTEM_FILE)); + + clusters_to_add = ocfs2_clusters_for_bytes(inode->i_sb, new_i_size); + if (clusters_to_add < oi->ip_clusters) + clusters_to_add = 0; + else + clusters_to_add -= oi->ip_clusters; + + if (clusters_to_add) { + ret = ocfs2_extend_allocation(inode, oi->ip_clusters, + clusters_to_add, 0); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + /* + * Call this even if we don't add any clusters to the tree. We + * still need to zero the area between the old i_size and the + * new i_size. + */ + ret = ocfs2_zero_extend(inode, di_bh, zero_to); + if (ret < 0) + mlog_errno(ret); + +out: + return ret; +} + +static int ocfs2_extend_file(struct inode *inode, + struct buffer_head *di_bh, + u64 new_i_size) +{ + int ret = 0; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + + BUG_ON(!di_bh); + + /* setattr sometimes calls us like this. */ + if (new_i_size == 0) + goto out; + + if (i_size_read(inode) == new_i_size) + goto out; + BUG_ON(new_i_size < i_size_read(inode)); + + /* + * The alloc sem blocks people in read/write from reading our + * allocation until we're done changing it. We depend on + * i_rwsem to block other extend/truncate calls while we're + * here. We even have to hold it for sparse files because there + * might be some tail zeroing. + */ + down_write(&oi->ip_alloc_sem); + + if (oi->ip_dyn_features & OCFS2_INLINE_DATA_FL) { + /* + * We can optimize small extends by keeping the inodes + * inline data. + */ + if (ocfs2_size_fits_inline_data(di_bh, new_i_size)) { + up_write(&oi->ip_alloc_sem); + goto out_update_size; + } + + ret = ocfs2_convert_inline_data_to_extents(inode, di_bh); + if (ret) { + up_write(&oi->ip_alloc_sem); + mlog_errno(ret); + goto out; + } + } + + if (ocfs2_sparse_alloc(OCFS2_SB(inode->i_sb))) + ret = ocfs2_zero_extend(inode, di_bh, new_i_size); + else + ret = ocfs2_extend_no_holes(inode, di_bh, new_i_size, + new_i_size); + + up_write(&oi->ip_alloc_sem); + + if (ret < 0) { + mlog_errno(ret); + goto out; + } + +out_update_size: + ret = ocfs2_simple_size_update(inode, di_bh, new_i_size); + if (ret < 0) + mlog_errno(ret); + +out: + return ret; +} + +int ocfs2_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +{ + int status = 0, size_change; + int inode_locked = 0; + struct inode *inode = d_inode(dentry); + struct super_block *sb = inode->i_sb; + struct ocfs2_super *osb = OCFS2_SB(sb); + struct buffer_head *bh = NULL; + handle_t *handle = NULL; + struct dquot *transfer_to[MAXQUOTAS] = { }; + int qtype; + int had_lock; + struct ocfs2_lock_holder oh; + + trace_ocfs2_setattr(inode, dentry, + (unsigned long long)OCFS2_I(inode)->ip_blkno, + dentry->d_name.len, dentry->d_name.name, + attr->ia_valid, + attr->ia_valid & ATTR_MODE ? attr->ia_mode : 0, + attr->ia_valid & ATTR_UID ? + from_kuid(&init_user_ns, attr->ia_uid) : 0, + attr->ia_valid & ATTR_GID ? + from_kgid(&init_user_ns, attr->ia_gid) : 0); + + status = ocfs2_emergency_state(osb); + if (unlikely(status)) { + mlog_errno(status); + goto bail; + } + + /* ensuring we don't even attempt to truncate a symlink */ + if (S_ISLNK(inode->i_mode)) + attr->ia_valid &= ~ATTR_SIZE; + +#define OCFS2_VALID_ATTRS (ATTR_ATIME | ATTR_MTIME | ATTR_CTIME | ATTR_SIZE \ + | ATTR_GID | ATTR_UID | ATTR_MODE) + if (!(attr->ia_valid & OCFS2_VALID_ATTRS)) + return 0; + + status = setattr_prepare(&nop_mnt_idmap, dentry, attr); + if (status) + return status; + + if (is_quota_modification(&nop_mnt_idmap, inode, attr)) { + status = dquot_initialize(inode); + if (status) + return status; + } + size_change = S_ISREG(inode->i_mode) && attr->ia_valid & ATTR_SIZE; + if (size_change) { + /* + * Here we should wait dio to finish before inode lock + * to avoid a deadlock between ocfs2_setattr() and + * ocfs2_dio_end_io_write() + */ + inode_dio_wait(inode); + + status = ocfs2_rw_lock(inode, 1); + if (status < 0) { + mlog_errno(status); + goto bail; + } + } + + had_lock = ocfs2_inode_lock_tracker(inode, &bh, 1, &oh); + if (had_lock < 0) { + status = had_lock; + goto bail_unlock_rw; + } else if (had_lock) { + /* + * As far as we know, ocfs2_setattr() could only be the first + * VFS entry point in the call chain of recursive cluster + * locking issue. + * + * For instance: + * chmod_common() + * notify_change() + * ocfs2_setattr() + * posix_acl_chmod() + * ocfs2_iop_get_acl() + * + * But, we're not 100% sure if it's always true, because the + * ordering of the VFS entry points in the call chain is out + * of our control. So, we'd better dump the stack here to + * catch the other cases of recursive locking. + */ + mlog(ML_ERROR, "Another case of recursive locking:\n"); + dump_stack(); + } + inode_locked = 1; + + if (size_change) { + status = inode_newsize_ok(inode, attr->ia_size); + if (status) + goto bail_unlock; + + if (i_size_read(inode) >= attr->ia_size) { + if (ocfs2_should_order_data(inode)) { + status = ocfs2_begin_ordered_truncate(inode, + attr->ia_size); + if (status) + goto bail_unlock; + } + status = ocfs2_truncate_file(inode, bh, attr->ia_size); + } else + status = ocfs2_extend_file(inode, bh, attr->ia_size); + if (status < 0) { + if (status != -ENOSPC) + mlog_errno(status); + status = -ENOSPC; + goto bail_unlock; + } + } + + if ((attr->ia_valid & ATTR_UID && !uid_eq(attr->ia_uid, inode->i_uid)) || + (attr->ia_valid & ATTR_GID && !gid_eq(attr->ia_gid, inode->i_gid))) { + /* + * Gather pointers to quota structures so that allocation / + * freeing of quota structures happens here and not inside + * dquot_transfer() where we have problems with lock ordering + */ + if (attr->ia_valid & ATTR_UID && !uid_eq(attr->ia_uid, inode->i_uid) + && OCFS2_HAS_RO_COMPAT_FEATURE(sb, + OCFS2_FEATURE_RO_COMPAT_USRQUOTA)) { + transfer_to[USRQUOTA] = dqget(sb, make_kqid_uid(attr->ia_uid)); + if (IS_ERR(transfer_to[USRQUOTA])) { + status = PTR_ERR(transfer_to[USRQUOTA]); + transfer_to[USRQUOTA] = NULL; + goto bail_unlock; + } + } + if (attr->ia_valid & ATTR_GID && !gid_eq(attr->ia_gid, inode->i_gid) + && OCFS2_HAS_RO_COMPAT_FEATURE(sb, + OCFS2_FEATURE_RO_COMPAT_GRPQUOTA)) { + transfer_to[GRPQUOTA] = dqget(sb, make_kqid_gid(attr->ia_gid)); + if (IS_ERR(transfer_to[GRPQUOTA])) { + status = PTR_ERR(transfer_to[GRPQUOTA]); + transfer_to[GRPQUOTA] = NULL; + goto bail_unlock; + } + } + down_write(&OCFS2_I(inode)->ip_alloc_sem); + handle = ocfs2_start_trans(osb, OCFS2_INODE_UPDATE_CREDITS + + 2 * ocfs2_quota_trans_credits(sb)); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + mlog_errno(status); + goto bail_unlock_alloc; + } + status = __dquot_transfer(inode, transfer_to); + if (status < 0) + goto bail_commit; + } else { + down_write(&OCFS2_I(inode)->ip_alloc_sem); + handle = ocfs2_start_trans(osb, OCFS2_INODE_UPDATE_CREDITS); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + mlog_errno(status); + goto bail_unlock_alloc; + } + } + + setattr_copy(&nop_mnt_idmap, inode, attr); + mark_inode_dirty(inode); + + status = ocfs2_mark_inode_dirty(handle, inode, bh); + if (status < 0) + mlog_errno(status); + +bail_commit: + ocfs2_commit_trans(osb, handle); +bail_unlock_alloc: + up_write(&OCFS2_I(inode)->ip_alloc_sem); +bail_unlock: + if (status && inode_locked) { + ocfs2_inode_unlock_tracker(inode, 1, &oh, had_lock); + inode_locked = 0; + } +bail_unlock_rw: + if (size_change) + ocfs2_rw_unlock(inode, 1); +bail: + + /* Release quota pointers in case we acquired them */ + for (qtype = 0; qtype < OCFS2_MAXQUOTAS; qtype++) + dqput(transfer_to[qtype]); + + if (!status && attr->ia_valid & ATTR_MODE) { + status = ocfs2_acl_chmod(inode, bh); + if (status < 0) + mlog_errno(status); + } + if (inode_locked) + ocfs2_inode_unlock_tracker(inode, 1, &oh, had_lock); + + brelse(bh); + return status; +} + +int ocfs2_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags) +{ + struct inode *inode = d_inode(path->dentry); + struct super_block *sb = path->dentry->d_sb; + struct ocfs2_super *osb = sb->s_fs_info; + int err; + + err = ocfs2_inode_revalidate(path->dentry); + if (err) { + if (err != -ENOENT) + mlog_errno(err); + goto bail; + } + + generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat); + /* + * If there is inline data in the inode, the inode will normally not + * have data blocks allocated (it may have an external xattr block). + * Report at least one sector for such files, so tools like tar, rsync, + * others don't incorrectly think the file is completely sparse. + */ + if (unlikely(OCFS2_I(inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL)) + stat->blocks += (stat->size + 511)>>9; + + /* We set the blksize from the cluster size for performance */ + stat->blksize = osb->s_clustersize; + +bail: + return err; +} + +int ocfs2_permission(struct mnt_idmap *idmap, struct inode *inode, + int mask) +{ + int ret, had_lock; + struct ocfs2_lock_holder oh; + + if (mask & MAY_NOT_BLOCK) + return -ECHILD; + + had_lock = ocfs2_inode_lock_tracker(inode, NULL, 0, &oh); + if (had_lock < 0) { + ret = had_lock; + goto out; + } else if (had_lock) { + /* See comments in ocfs2_setattr() for details. + * The call chain of this case could be: + * do_sys_open() + * may_open() + * inode_permission() + * ocfs2_permission() + * ocfs2_iop_get_acl() + */ + mlog(ML_ERROR, "Another case of recursive locking:\n"); + dump_stack(); + } + + ret = generic_permission(&nop_mnt_idmap, inode, mask); + + ocfs2_inode_unlock_tracker(inode, 0, &oh, had_lock); +out: + return ret; +} + +static int __ocfs2_write_remove_suid(struct inode *inode, + struct buffer_head *bh) +{ + int ret; + handle_t *handle; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct ocfs2_dinode *di; + + trace_ocfs2_write_remove_suid( + (unsigned long long)OCFS2_I(inode)->ip_blkno, + inode->i_mode); + + handle = ocfs2_start_trans(osb, OCFS2_INODE_UPDATE_CREDITS); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out; + } + + ret = ocfs2_journal_access_di(handle, INODE_CACHE(inode), bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret < 0) { + mlog_errno(ret); + goto out_trans; + } + + inode->i_mode &= ~S_ISUID; + if ((inode->i_mode & S_ISGID) && (inode->i_mode & S_IXGRP)) + inode->i_mode &= ~S_ISGID; + + di = (struct ocfs2_dinode *) bh->b_data; + di->i_mode = cpu_to_le16(inode->i_mode); + ocfs2_update_inode_fsync_trans(handle, inode, 0); + + ocfs2_journal_dirty(handle, bh); + +out_trans: + ocfs2_commit_trans(osb, handle); +out: + return ret; +} + +static int ocfs2_write_remove_suid(struct inode *inode) +{ + int ret; + struct buffer_head *bh = NULL; + + ret = ocfs2_read_inode_block(inode, &bh); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + ret = __ocfs2_write_remove_suid(inode, bh); +out: + brelse(bh); + return ret; +} + +/* + * Allocate enough extents to cover the region starting at byte offset + * start for len bytes. Existing extents are skipped, any extents + * added are marked as "unwritten". + */ +static int ocfs2_allocate_unwritten_extents(struct inode *inode, + u64 start, u64 len) +{ + int ret; + u32 cpos, phys_cpos, clusters, alloc_size; + u64 end = start + len; + struct buffer_head *di_bh = NULL; + + if (OCFS2_I(inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL) { + ret = ocfs2_read_inode_block(inode, &di_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * Nothing to do if the requested reservation range + * fits within the inode. + */ + if (ocfs2_size_fits_inline_data(di_bh, end)) + goto out; + + ret = ocfs2_convert_inline_data_to_extents(inode, di_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + /* + * We consider both start and len to be inclusive. + */ + cpos = start >> OCFS2_SB(inode->i_sb)->s_clustersize_bits; + clusters = ocfs2_clusters_for_bytes(inode->i_sb, start + len); + clusters -= cpos; + + while (clusters) { + ret = ocfs2_get_clusters(inode, cpos, &phys_cpos, + &alloc_size, NULL); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * Hole or existing extent len can be arbitrary, so + * cap it to our own allocation request. + */ + if (alloc_size > clusters) + alloc_size = clusters; + + if (phys_cpos) { + /* + * We already have an allocation at this + * region so we can safely skip it. + */ + goto next; + } + + ret = ocfs2_extend_allocation(inode, cpos, alloc_size, 1); + if (ret) { + if (ret != -ENOSPC) + mlog_errno(ret); + goto out; + } + +next: + cpos += alloc_size; + clusters -= alloc_size; + } + + ret = 0; +out: + + brelse(di_bh); + return ret; +} + +/* + * Truncate a byte range, avoiding pages within partial clusters. This + * preserves those pages for the zeroing code to write to. + */ +static void ocfs2_truncate_cluster_pages(struct inode *inode, u64 byte_start, + u64 byte_len) +{ + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + loff_t start, end; + struct address_space *mapping = inode->i_mapping; + + start = (loff_t)ocfs2_align_bytes_to_clusters(inode->i_sb, byte_start); + end = byte_start + byte_len; + end = end & ~(osb->s_clustersize - 1); + + if (start < end) { + unmap_mapping_range(mapping, start, end - start, 0); + truncate_inode_pages_range(mapping, start, end - 1); + } +} + +/* + * zero out partial blocks of one cluster. + * + * start: file offset where zero starts, will be made upper block aligned. + * len: it will be trimmed to the end of current cluster if "start + len" + * is bigger than it. + */ +static int ocfs2_zeroout_partial_cluster(struct inode *inode, + u64 start, u64 len) +{ + int ret; + u64 start_block, end_block, nr_blocks; + u64 p_block, offset; + u32 cluster, p_cluster, nr_clusters; + struct super_block *sb = inode->i_sb; + u64 end = ocfs2_align_bytes_to_clusters(sb, start); + + if (start + len < end) + end = start + len; + + start_block = ocfs2_blocks_for_bytes(sb, start); + end_block = ocfs2_blocks_for_bytes(sb, end); + nr_blocks = end_block - start_block; + if (!nr_blocks) + return 0; + + cluster = ocfs2_bytes_to_clusters(sb, start); + ret = ocfs2_get_clusters(inode, cluster, &p_cluster, + &nr_clusters, NULL); + if (ret) + return ret; + if (!p_cluster) + return 0; + + offset = start_block - ocfs2_clusters_to_blocks(sb, cluster); + p_block = ocfs2_clusters_to_blocks(sb, p_cluster) + offset; + return sb_issue_zeroout(sb, p_block, nr_blocks, GFP_NOFS); +} + +static int ocfs2_zero_partial_clusters(struct inode *inode, + u64 start, u64 len) +{ + int ret = 0; + u64 tmpend = 0; + u64 end = start + len; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + unsigned int csize = osb->s_clustersize; + handle_t *handle; + loff_t isize = i_size_read(inode); + + /* + * The "start" and "end" values are NOT necessarily part of + * the range whose allocation is being deleted. Rather, this + * is what the user passed in with the request. We must zero + * partial clusters here. There's no need to worry about + * physical allocation - the zeroing code knows to skip holes. + */ + trace_ocfs2_zero_partial_clusters( + (unsigned long long)OCFS2_I(inode)->ip_blkno, + (unsigned long long)start, (unsigned long long)end); + + /* + * If both edges are on a cluster boundary then there's no + * zeroing required as the region is part of the allocation to + * be truncated. + */ + if ((start & (csize - 1)) == 0 && (end & (csize - 1)) == 0) + goto out; + + /* No page cache for EOF blocks, issue zero out to disk. */ + if (end > isize) { + /* + * zeroout eof blocks in last cluster starting from + * "isize" even "start" > "isize" because it is + * complicated to zeroout just at "start" as "start" + * may be not aligned with block size, buffer write + * would be required to do that, but out of eof buffer + * write is not supported. + */ + ret = ocfs2_zeroout_partial_cluster(inode, isize, + end - isize); + if (ret) { + mlog_errno(ret); + goto out; + } + if (start >= isize) + goto out; + end = isize; + } + handle = ocfs2_start_trans(osb, OCFS2_INODE_UPDATE_CREDITS); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out; + } + + /* + * If start is on a cluster boundary and end is somewhere in another + * cluster, we have not COWed the cluster starting at start, unless + * end is also within the same cluster. So, in this case, we skip this + * first call to ocfs2_zero_range_for_truncate() truncate and move on + * to the next one. + */ + if ((start & (csize - 1)) != 0) { + /* + * We want to get the byte offset of the end of the 1st + * cluster. + */ + tmpend = (u64)osb->s_clustersize + + (start & ~(osb->s_clustersize - 1)); + if (tmpend > end) + tmpend = end; + + trace_ocfs2_zero_partial_clusters_range1( + (unsigned long long)start, + (unsigned long long)tmpend); + + ret = ocfs2_zero_range_for_truncate(inode, handle, start, + tmpend); + if (ret) + mlog_errno(ret); + } + + if (tmpend < end) { + /* + * This may make start and end equal, but the zeroing + * code will skip any work in that case so there's no + * need to catch it up here. + */ + start = end & ~(osb->s_clustersize - 1); + + trace_ocfs2_zero_partial_clusters_range2( + (unsigned long long)start, (unsigned long long)end); + + ret = ocfs2_zero_range_for_truncate(inode, handle, start, end); + if (ret) + mlog_errno(ret); + } + ocfs2_update_inode_fsync_trans(handle, inode, 1); + + ocfs2_commit_trans(osb, handle); +out: + return ret; +} + +static int ocfs2_find_rec(struct ocfs2_extent_list *el, u32 pos) +{ + int i; + struct ocfs2_extent_rec *rec = NULL; + + for (i = le16_to_cpu(el->l_next_free_rec) - 1; i >= 0; i--) { + + rec = &el->l_recs[i]; + + if (le32_to_cpu(rec->e_cpos) < pos) + break; + } + + return i; +} + +/* + * Helper to calculate the punching pos and length in one run, we handle the + * following three cases in order: + * + * - remove the entire record + * - remove a partial record + * - no record needs to be removed (hole-punching completed) +*/ +static void ocfs2_calc_trunc_pos(struct inode *inode, + struct ocfs2_extent_list *el, + struct ocfs2_extent_rec *rec, + u32 trunc_start, u32 *trunc_cpos, + u32 *trunc_len, u32 *trunc_end, + u64 *blkno, int *done) +{ + int ret = 0; + u32 coff, range; + + range = le32_to_cpu(rec->e_cpos) + ocfs2_rec_clusters(el, rec); + + if (le32_to_cpu(rec->e_cpos) >= trunc_start) { + /* + * remove an entire extent record. + */ + *trunc_cpos = le32_to_cpu(rec->e_cpos); + /* + * Skip holes if any. + */ + if (range < *trunc_end) + *trunc_end = range; + *trunc_len = *trunc_end - le32_to_cpu(rec->e_cpos); + *blkno = le64_to_cpu(rec->e_blkno); + *trunc_end = le32_to_cpu(rec->e_cpos); + } else if (range > trunc_start) { + /* + * remove a partial extent record, which means we're + * removing the last extent record. + */ + *trunc_cpos = trunc_start; + /* + * skip hole if any. + */ + if (range < *trunc_end) + *trunc_end = range; + *trunc_len = *trunc_end - trunc_start; + coff = trunc_start - le32_to_cpu(rec->e_cpos); + *blkno = le64_to_cpu(rec->e_blkno) + + ocfs2_clusters_to_blocks(inode->i_sb, coff); + *trunc_end = trunc_start; + } else { + /* + * It may have two following possibilities: + * + * - last record has been removed + * - trunc_start was within a hole + * + * both two cases mean the completion of hole punching. + */ + ret = 1; + } + + *done = ret; +} + +int ocfs2_remove_inode_range(struct inode *inode, + struct buffer_head *di_bh, u64 byte_start, + u64 byte_len) +{ + int ret = 0, flags = 0, done = 0, i; + u32 trunc_start, trunc_len, trunc_end, trunc_cpos, phys_cpos; + u32 cluster_in_el; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct ocfs2_cached_dealloc_ctxt dealloc; + struct address_space *mapping = inode->i_mapping; + struct ocfs2_extent_tree et; + struct ocfs2_path *path = NULL; + struct ocfs2_extent_list *el = NULL; + struct ocfs2_extent_rec *rec = NULL; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data; + u64 blkno, refcount_loc = le64_to_cpu(di->i_refcount_loc); + + ocfs2_init_dinode_extent_tree(&et, INODE_CACHE(inode), di_bh); + ocfs2_init_dealloc_ctxt(&dealloc); + + trace_ocfs2_remove_inode_range( + (unsigned long long)OCFS2_I(inode)->ip_blkno, + (unsigned long long)byte_start, + (unsigned long long)byte_len); + + if (byte_len == 0) + return 0; + + if (OCFS2_I(inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL) { + int id_count = ocfs2_max_inline_data_with_xattr(inode->i_sb, di); + + if (byte_start > id_count || byte_start + byte_len > id_count) { + ret = -EINVAL; + mlog_errno(ret); + goto out; + } + + ret = ocfs2_truncate_inline(inode, di_bh, byte_start, + byte_start + byte_len, 0); + if (ret) { + mlog_errno(ret); + goto out; + } + /* + * There's no need to get fancy with the page cache + * truncate of an inline-data inode. We're talking + * about less than a page here, which will be cached + * in the dinode buffer anyway. + */ + unmap_mapping_range(mapping, 0, 0, 0); + truncate_inode_pages(mapping, 0); + goto out; + } + + /* + * For reflinks, we may need to CoW 2 clusters which might be + * partially zero'd later, if hole's start and end offset were + * within one cluster(means is not exactly aligned to clustersize). + */ + + if (ocfs2_is_refcount_inode(inode)) { + ret = ocfs2_cow_file_pos(inode, di_bh, byte_start); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_cow_file_pos(inode, di_bh, byte_start + byte_len); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + trunc_start = ocfs2_clusters_for_bytes(osb->sb, byte_start); + trunc_end = (byte_start + byte_len) >> osb->s_clustersize_bits; + cluster_in_el = trunc_end; + + ret = ocfs2_zero_partial_clusters(inode, byte_start, byte_len); + if (ret) { + mlog_errno(ret); + goto out; + } + + path = ocfs2_new_path_from_et(&et); + if (!path) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + while (trunc_end > trunc_start) { + + ret = ocfs2_find_path(INODE_CACHE(inode), path, + cluster_in_el); + if (ret) { + mlog_errno(ret); + goto out; + } + + el = path_leaf_el(path); + + i = ocfs2_find_rec(el, trunc_end); + /* + * Need to go to previous extent block. + */ + if (i < 0) { + if (path->p_tree_depth == 0) + break; + + ret = ocfs2_find_cpos_for_left_leaf(inode->i_sb, + path, + &cluster_in_el); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * We've reached the leftmost extent block, + * it's safe to leave. + */ + if (cluster_in_el == 0) + break; + + /* + * The 'pos' searched for previous extent block is + * always one cluster less than actual trunc_end. + */ + trunc_end = cluster_in_el + 1; + + ocfs2_reinit_path(path, 1); + + continue; + + } else + rec = &el->l_recs[i]; + + ocfs2_calc_trunc_pos(inode, el, rec, trunc_start, &trunc_cpos, + &trunc_len, &trunc_end, &blkno, &done); + if (done) + break; + + flags = rec->e_flags; + phys_cpos = ocfs2_blocks_to_clusters(inode->i_sb, blkno); + + ret = ocfs2_remove_btree_range(inode, &et, trunc_cpos, + phys_cpos, trunc_len, flags, + &dealloc, refcount_loc, false); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + cluster_in_el = trunc_end; + + ocfs2_reinit_path(path, 1); + } + + ocfs2_truncate_cluster_pages(inode, byte_start, byte_len); + +out: + ocfs2_free_path(path); + ocfs2_schedule_truncate_log_flush(osb, 1); + ocfs2_run_deallocs(osb, &dealloc); + + return ret; +} + +/* + * Parts of this function taken from xfs_change_file_space() + */ +static int __ocfs2_change_file_space(struct file *file, struct inode *inode, + loff_t f_pos, unsigned int cmd, + struct ocfs2_space_resv *sr, + int change_size) +{ + int ret; + s64 llen; + loff_t size, orig_isize; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct buffer_head *di_bh = NULL; + handle_t *handle; + unsigned long long max_off = inode->i_sb->s_maxbytes; + + if (unlikely(ocfs2_emergency_state(osb))) + return -EROFS; + + inode_lock(inode); + + /* Wait all existing dio workers, newcomers will block on i_rwsem */ + inode_dio_wait(inode); + /* + * This prevents concurrent writes on other nodes + */ + ret = ocfs2_rw_lock(inode, 1); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_inode_lock(inode, &di_bh, 1); + if (ret) { + mlog_errno(ret); + goto out_rw_unlock; + } + + if (inode->i_flags & (S_IMMUTABLE|S_APPEND)) { + ret = -EPERM; + goto out_inode_unlock; + } + + switch (sr->l_whence) { + case 0: /*SEEK_SET*/ + break; + case 1: /*SEEK_CUR*/ + sr->l_start += f_pos; + break; + case 2: /*SEEK_END*/ + sr->l_start += i_size_read(inode); + break; + default: + ret = -EINVAL; + goto out_inode_unlock; + } + sr->l_whence = 0; + + llen = sr->l_len > 0 ? sr->l_len - 1 : sr->l_len; + + if (sr->l_start < 0 + || sr->l_start > max_off + || (sr->l_start + llen) < 0 + || (sr->l_start + llen) > max_off) { + ret = -EINVAL; + goto out_inode_unlock; + } + size = sr->l_start + sr->l_len; + + if (cmd == OCFS2_IOC_RESVSP || cmd == OCFS2_IOC_RESVSP64 || + cmd == OCFS2_IOC_UNRESVSP || cmd == OCFS2_IOC_UNRESVSP64) { + if (sr->l_len <= 0) { + ret = -EINVAL; + goto out_inode_unlock; + } + } + + if (file && setattr_should_drop_suidgid(&nop_mnt_idmap, file_inode(file))) { + ret = __ocfs2_write_remove_suid(inode, di_bh); + if (ret) { + mlog_errno(ret); + goto out_inode_unlock; + } + } + + down_write(&OCFS2_I(inode)->ip_alloc_sem); + switch (cmd) { + case OCFS2_IOC_RESVSP: + case OCFS2_IOC_RESVSP64: + /* + * This takes unsigned offsets, but the signed ones we + * pass have been checked against overflow above. + */ + ret = ocfs2_allocate_unwritten_extents(inode, sr->l_start, + sr->l_len); + break; + case OCFS2_IOC_UNRESVSP: + case OCFS2_IOC_UNRESVSP64: + ret = ocfs2_remove_inode_range(inode, di_bh, sr->l_start, + sr->l_len); + break; + default: + ret = -EINVAL; + } + + orig_isize = i_size_read(inode); + /* zeroout eof blocks in the cluster. */ + if (!ret && change_size && orig_isize < size) { + ret = ocfs2_zeroout_partial_cluster(inode, orig_isize, + size - orig_isize); + if (!ret) + i_size_write(inode, size); + } + up_write(&OCFS2_I(inode)->ip_alloc_sem); + if (ret) { + mlog_errno(ret); + goto out_inode_unlock; + } + + /* + * We update c/mtime for these changes + */ + handle = ocfs2_start_trans(osb, OCFS2_INODE_UPDATE_CREDITS); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out_inode_unlock; + } + + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + ret = ocfs2_mark_inode_dirty(handle, inode, di_bh); + if (ret < 0) + mlog_errno(ret); + + if (file && (file->f_flags & O_SYNC)) + handle->h_sync = 1; + + ocfs2_commit_trans(osb, handle); + +out_inode_unlock: + brelse(di_bh); + ocfs2_inode_unlock(inode, 1); +out_rw_unlock: + ocfs2_rw_unlock(inode, 1); + +out: + inode_unlock(inode); + return ret; +} + +int ocfs2_change_file_space(struct file *file, unsigned int cmd, + struct ocfs2_space_resv *sr) +{ + struct inode *inode = file_inode(file); + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + int ret; + + if ((cmd == OCFS2_IOC_RESVSP || cmd == OCFS2_IOC_RESVSP64) && + !ocfs2_writes_unwritten_extents(osb)) + return -ENOTTY; + else if ((cmd == OCFS2_IOC_UNRESVSP || cmd == OCFS2_IOC_UNRESVSP64) && + !ocfs2_sparse_alloc(osb)) + return -ENOTTY; + + if (!S_ISREG(inode->i_mode)) + return -EINVAL; + + if (!(file->f_mode & FMODE_WRITE)) + return -EBADF; + + ret = mnt_want_write_file(file); + if (ret) + return ret; + ret = __ocfs2_change_file_space(file, inode, file->f_pos, cmd, sr, 0); + mnt_drop_write_file(file); + return ret; +} + +static long ocfs2_fallocate(struct file *file, int mode, loff_t offset, + loff_t len) +{ + struct inode *inode = file_inode(file); + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct ocfs2_space_resv sr; + int change_size = 1; + int cmd = OCFS2_IOC_RESVSP64; + int ret = 0; + + if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)) + return -EOPNOTSUPP; + if (!ocfs2_writes_unwritten_extents(osb)) + return -EOPNOTSUPP; + + if (mode & FALLOC_FL_KEEP_SIZE) { + change_size = 0; + } else { + ret = inode_newsize_ok(inode, offset + len); + if (ret) + return ret; + } + + if (mode & FALLOC_FL_PUNCH_HOLE) + cmd = OCFS2_IOC_UNRESVSP64; + + sr.l_whence = 0; + sr.l_start = (s64)offset; + sr.l_len = (s64)len; + + return __ocfs2_change_file_space(NULL, inode, offset, cmd, &sr, + change_size); +} + +int ocfs2_check_range_for_refcount(struct inode *inode, loff_t pos, + size_t count) +{ + int ret = 0; + unsigned int extent_flags; + u32 cpos, clusters, extent_len, phys_cpos; + struct super_block *sb = inode->i_sb; + + if (!ocfs2_refcount_tree(OCFS2_SB(inode->i_sb)) || + !ocfs2_is_refcount_inode(inode) || + OCFS2_I(inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL) + return 0; + + cpos = pos >> OCFS2_SB(sb)->s_clustersize_bits; + clusters = ocfs2_clusters_for_bytes(sb, pos + count) - cpos; + + while (clusters) { + ret = ocfs2_get_clusters(inode, cpos, &phys_cpos, &extent_len, + &extent_flags); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + if (phys_cpos && (extent_flags & OCFS2_EXT_REFCOUNTED)) { + ret = 1; + break; + } + + if (extent_len > clusters) + extent_len = clusters; + + clusters -= extent_len; + cpos += extent_len; + } +out: + return ret; +} + +static int ocfs2_is_io_unaligned(struct inode *inode, size_t count, loff_t pos) +{ + int blockmask = inode->i_sb->s_blocksize - 1; + loff_t final_size = pos + count; + + if ((pos & blockmask) || (final_size & blockmask)) + return 1; + return 0; +} + +static int ocfs2_inode_lock_for_extent_tree(struct inode *inode, + struct buffer_head **di_bh, + int meta_level, + int write_sem, + int wait) +{ + int ret = 0; + + if (wait) + ret = ocfs2_inode_lock(inode, di_bh, meta_level); + else + ret = ocfs2_try_inode_lock(inode, di_bh, meta_level); + if (ret < 0) + goto out; + + if (wait) { + if (write_sem) + down_write(&OCFS2_I(inode)->ip_alloc_sem); + else + down_read(&OCFS2_I(inode)->ip_alloc_sem); + } else { + if (write_sem) + ret = down_write_trylock(&OCFS2_I(inode)->ip_alloc_sem); + else + ret = down_read_trylock(&OCFS2_I(inode)->ip_alloc_sem); + + if (!ret) { + ret = -EAGAIN; + goto out_unlock; + } + } + + return ret; + +out_unlock: + brelse(*di_bh); + *di_bh = NULL; + ocfs2_inode_unlock(inode, meta_level); +out: + return ret; +} + +static void ocfs2_inode_unlock_for_extent_tree(struct inode *inode, + struct buffer_head **di_bh, + int meta_level, + int write_sem) +{ + if (write_sem) + up_write(&OCFS2_I(inode)->ip_alloc_sem); + else + up_read(&OCFS2_I(inode)->ip_alloc_sem); + + brelse(*di_bh); + *di_bh = NULL; + + if (meta_level >= 0) + ocfs2_inode_unlock(inode, meta_level); +} + +static int ocfs2_prepare_inode_for_write(struct file *file, + loff_t pos, size_t count, int wait) +{ + int ret = 0, meta_level = 0, overwrite_io = 0; + int write_sem = 0; + struct dentry *dentry = file->f_path.dentry; + struct inode *inode = d_inode(dentry); + struct buffer_head *di_bh = NULL; + u32 cpos; + u32 clusters; + + /* + * We start with a read level meta lock and only jump to an ex + * if we need to make modifications here. + */ + for(;;) { + ret = ocfs2_inode_lock_for_extent_tree(inode, + &di_bh, + meta_level, + write_sem, + wait); + if (ret < 0) { + if (ret != -EAGAIN) + mlog_errno(ret); + goto out; + } + + /* + * Check if IO will overwrite allocated blocks in case + * IOCB_NOWAIT flag is set. + */ + if (!wait && !overwrite_io) { + overwrite_io = 1; + + ret = ocfs2_overwrite_io(inode, di_bh, pos, count); + if (ret < 0) { + if (ret != -EAGAIN) + mlog_errno(ret); + goto out_unlock; + } + } + + /* Clear suid / sgid if necessary. We do this here + * instead of later in the write path because + * remove_suid() calls ->setattr without any hint that + * we may have already done our cluster locking. Since + * ocfs2_setattr() *must* take cluster locks to + * proceed, this will lead us to recursively lock the + * inode. There's also the dinode i_size state which + * can be lost via setattr during extending writes (we + * set inode->i_size at the end of a write. */ + if (setattr_should_drop_suidgid(&nop_mnt_idmap, inode)) { + if (meta_level == 0) { + ocfs2_inode_unlock_for_extent_tree(inode, + &di_bh, + meta_level, + write_sem); + meta_level = 1; + continue; + } + + ret = ocfs2_write_remove_suid(inode); + if (ret < 0) { + mlog_errno(ret); + goto out_unlock; + } + } + + ret = ocfs2_check_range_for_refcount(inode, pos, count); + if (ret == 1) { + ocfs2_inode_unlock_for_extent_tree(inode, + &di_bh, + meta_level, + write_sem); + meta_level = 1; + write_sem = 1; + ret = ocfs2_inode_lock_for_extent_tree(inode, + &di_bh, + meta_level, + write_sem, + wait); + if (ret < 0) { + if (ret != -EAGAIN) + mlog_errno(ret); + goto out; + } + + cpos = pos >> OCFS2_SB(inode->i_sb)->s_clustersize_bits; + clusters = + ocfs2_clusters_for_bytes(inode->i_sb, pos + count) - cpos; + ret = ocfs2_refcount_cow(inode, di_bh, cpos, clusters, UINT_MAX); + } + + if (ret < 0) { + if (ret != -EAGAIN) + mlog_errno(ret); + goto out_unlock; + } + + break; + } + +out_unlock: + trace_ocfs2_prepare_inode_for_write(OCFS2_I(inode)->ip_blkno, + pos, count, wait); + + ocfs2_inode_unlock_for_extent_tree(inode, + &di_bh, + meta_level, + write_sem); + +out: + return ret; +} + +static ssize_t ocfs2_file_write_iter(struct kiocb *iocb, + struct iov_iter *from) +{ + int rw_level; + ssize_t written = 0; + ssize_t ret; + size_t count = iov_iter_count(from); + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + int full_coherency = !(osb->s_mount_opt & + OCFS2_MOUNT_COHERENCY_BUFFERED); + void *saved_ki_complete = NULL; + int append_write = ((iocb->ki_pos + count) >= + i_size_read(inode) ? 1 : 0); + int direct_io = iocb->ki_flags & IOCB_DIRECT ? 1 : 0; + int nowait = iocb->ki_flags & IOCB_NOWAIT ? 1 : 0; + + trace_ocfs2_file_write_iter(inode, file, file->f_path.dentry, + (unsigned long long)OCFS2_I(inode)->ip_blkno, + file->f_path.dentry->d_name.len, + file->f_path.dentry->d_name.name, + (unsigned int)from->nr_segs); /* GRRRRR */ + + if (!direct_io && nowait) + return -EOPNOTSUPP; + + if (count == 0) + return 0; + + if (nowait) { + if (!inode_trylock(inode)) + return -EAGAIN; + } else + inode_lock(inode); + + ocfs2_iocb_init_rw_locked(iocb); + + /* + * Concurrent O_DIRECT writes are allowed with + * mount_option "coherency=buffered". + * For append write, we must take rw EX. + */ + rw_level = (!direct_io || full_coherency || append_write); + + if (nowait) + ret = ocfs2_try_rw_lock(inode, rw_level); + else + ret = ocfs2_rw_lock(inode, rw_level); + if (ret < 0) { + if (ret != -EAGAIN) + mlog_errno(ret); + goto out_mutex; + } + + /* + * O_DIRECT writes with "coherency=full" need to take EX cluster + * inode_lock to guarantee coherency. + */ + if (direct_io && full_coherency) { + /* + * We need to take and drop the inode lock to force + * other nodes to drop their caches. Buffered I/O + * already does this in write_begin(). + */ + if (nowait) + ret = ocfs2_try_inode_lock(inode, NULL, 1); + else + ret = ocfs2_inode_lock(inode, NULL, 1); + if (ret < 0) { + if (ret != -EAGAIN) + mlog_errno(ret); + goto out; + } + + ocfs2_inode_unlock(inode, 1); + } + + ret = generic_write_checks(iocb, from); + if (ret <= 0) { + if (ret) + mlog_errno(ret); + goto out; + } + count = ret; + + ret = ocfs2_prepare_inode_for_write(file, iocb->ki_pos, count, !nowait); + if (ret < 0) { + if (ret != -EAGAIN) + mlog_errno(ret); + goto out; + } + + if (direct_io && !is_sync_kiocb(iocb) && + ocfs2_is_io_unaligned(inode, count, iocb->ki_pos)) { + /* + * Make it a sync io if it's an unaligned aio. + */ + saved_ki_complete = xchg(&iocb->ki_complete, NULL); + } + + /* communicate with ocfs2_dio_end_io */ + ocfs2_iocb_set_rw_locked(iocb, rw_level); + + written = __generic_file_write_iter(iocb, from); + /* buffered aio wouldn't have proper lock coverage today */ + BUG_ON(written == -EIOCBQUEUED && !direct_io); + + /* + * deep in g_f_a_w_n()->ocfs2_direct_IO we pass in a ocfs2_dio_end_io + * function pointer which is called when o_direct io completes so that + * it can unlock our rw lock. + * Unfortunately there are error cases which call end_io and others + * that don't. so we don't have to unlock the rw_lock if either an + * async dio is going to do it in the future or an end_io after an + * error has already done it. + */ + if ((written == -EIOCBQUEUED) || (!ocfs2_iocb_is_rw_locked(iocb))) { + rw_level = -1; + } + + if (unlikely(written <= 0)) + goto out; + + if (((file->f_flags & O_DSYNC) && !direct_io) || + IS_SYNC(inode)) { + ret = filemap_fdatawrite_range(file->f_mapping, + iocb->ki_pos - written, + iocb->ki_pos - 1); + if (ret < 0) + written = ret; + + if (!ret) { + ret = jbd2_journal_force_commit(osb->journal->j_journal); + if (ret < 0) + written = ret; + } + + if (!ret) + ret = filemap_fdatawait_range(file->f_mapping, + iocb->ki_pos - written, + iocb->ki_pos - 1); + } + +out: + if (saved_ki_complete) + xchg(&iocb->ki_complete, saved_ki_complete); + + if (rw_level != -1) + ocfs2_rw_unlock(inode, rw_level); + +out_mutex: + inode_unlock(inode); + + if (written) + ret = written; + return ret; +} + +static ssize_t ocfs2_file_read_iter(struct kiocb *iocb, + struct iov_iter *to) +{ + int ret = 0, rw_level = -1, lock_level = 0; + struct file *filp = iocb->ki_filp; + struct inode *inode = file_inode(filp); + int direct_io = iocb->ki_flags & IOCB_DIRECT ? 1 : 0; + int nowait = iocb->ki_flags & IOCB_NOWAIT ? 1 : 0; + + trace_ocfs2_file_read_iter(inode, filp, filp->f_path.dentry, + (unsigned long long)OCFS2_I(inode)->ip_blkno, + filp->f_path.dentry->d_name.len, + filp->f_path.dentry->d_name.name, + to->nr_segs); /* GRRRRR */ + + + if (!inode) { + ret = -EINVAL; + mlog_errno(ret); + goto bail; + } + + if (!direct_io && nowait) + return -EOPNOTSUPP; + + ocfs2_iocb_init_rw_locked(iocb); + + /* + * buffered reads protect themselves in ->read_folio(). O_DIRECT reads + * need locks to protect pending reads from racing with truncate. + */ + if (direct_io) { + if (nowait) + ret = ocfs2_try_rw_lock(inode, 0); + else + ret = ocfs2_rw_lock(inode, 0); + + if (ret < 0) { + if (ret != -EAGAIN) + mlog_errno(ret); + goto bail; + } + rw_level = 0; + /* communicate with ocfs2_dio_end_io */ + ocfs2_iocb_set_rw_locked(iocb, rw_level); + } + + /* + * We're fine letting folks race truncates and extending + * writes with read across the cluster, just like they can + * locally. Hence no rw_lock during read. + * + * Take and drop the meta data lock to update inode fields + * like i_size. This allows the checks down below + * copy_splice_read() a chance of actually working. + */ + ret = ocfs2_inode_lock_atime(inode, filp->f_path.mnt, &lock_level, + !nowait); + if (ret < 0) { + if (ret != -EAGAIN) + mlog_errno(ret); + goto bail; + } + ocfs2_inode_unlock(inode, lock_level); + + ret = generic_file_read_iter(iocb, to); + trace_generic_file_read_iter_ret(ret); + + /* buffered aio wouldn't have proper lock coverage today */ + BUG_ON(ret == -EIOCBQUEUED && !direct_io); + + /* see ocfs2_file_write_iter */ + if (ret == -EIOCBQUEUED || !ocfs2_iocb_is_rw_locked(iocb)) { + rw_level = -1; + } + +bail: + if (rw_level != -1) + ocfs2_rw_unlock(inode, rw_level); + + return ret; +} + +static ssize_t ocfs2_file_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, + size_t len, unsigned int flags) +{ + struct inode *inode = file_inode(in); + ssize_t ret = 0; + int lock_level = 0; + + trace_ocfs2_file_splice_read(inode, in, in->f_path.dentry, + (unsigned long long)OCFS2_I(inode)->ip_blkno, + in->f_path.dentry->d_name.len, + in->f_path.dentry->d_name.name, + flags); + + /* + * We're fine letting folks race truncates and extending writes with + * read across the cluster, just like they can locally. Hence no + * rw_lock during read. + * + * Take and drop the meta data lock to update inode fields like i_size. + * This allows the checks down below filemap_splice_read() a chance of + * actually working. + */ + ret = ocfs2_inode_lock_atime(inode, in->f_path.mnt, &lock_level, 1); + if (ret < 0) { + if (ret != -EAGAIN) + mlog_errno(ret); + goto bail; + } + ocfs2_inode_unlock(inode, lock_level); + + ret = filemap_splice_read(in, ppos, pipe, len, flags); + trace_filemap_splice_read_ret(ret); +bail: + return ret; +} + +/* Refer generic_file_llseek_unlocked() */ +static loff_t ocfs2_file_llseek(struct file *file, loff_t offset, int whence) +{ + struct inode *inode = file->f_mapping->host; + int ret = 0; + + inode_lock(inode); + + switch (whence) { + case SEEK_SET: + break; + case SEEK_END: + /* SEEK_END requires the OCFS2 inode lock for the file + * because it references the file's size. + */ + ret = ocfs2_inode_lock(inode, NULL, 0); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + offset += i_size_read(inode); + ocfs2_inode_unlock(inode, 0); + break; + case SEEK_CUR: + if (offset == 0) { + offset = file->f_pos; + goto out; + } + offset += file->f_pos; + break; + case SEEK_DATA: + case SEEK_HOLE: + ret = ocfs2_seek_data_hole_offset(file, &offset, whence); + if (ret) + goto out; + break; + default: + ret = -EINVAL; + goto out; + } + + offset = vfs_setpos(file, offset, inode->i_sb->s_maxbytes); + +out: + inode_unlock(inode); + if (ret) + return ret; + return offset; +} + +static loff_t ocfs2_remap_file_range(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + loff_t len, unsigned int remap_flags) +{ + struct inode *inode_in = file_inode(file_in); + struct inode *inode_out = file_inode(file_out); + struct ocfs2_super *osb = OCFS2_SB(inode_in->i_sb); + struct buffer_head *in_bh = NULL, *out_bh = NULL; + bool same_inode = (inode_in == inode_out); + loff_t remapped = 0; + ssize_t ret; + + if (remap_flags & ~(REMAP_FILE_DEDUP | REMAP_FILE_ADVISORY)) + return -EINVAL; + if (!ocfs2_refcount_tree(osb)) + return -EOPNOTSUPP; + if (unlikely(ocfs2_emergency_state(osb))) + return -EROFS; + + /* Lock both files against IO */ + ret = ocfs2_reflink_inodes_lock(inode_in, &in_bh, inode_out, &out_bh); + if (ret) + return ret; + + /* Check file eligibility and prepare for block sharing. */ + ret = -EINVAL; + if ((OCFS2_I(inode_in)->ip_flags & OCFS2_INODE_SYSTEM_FILE) || + (OCFS2_I(inode_out)->ip_flags & OCFS2_INODE_SYSTEM_FILE)) + goto out_unlock; + + ret = generic_remap_file_range_prep(file_in, pos_in, file_out, pos_out, + &len, remap_flags); + if (ret < 0 || len == 0) + goto out_unlock; + + /* Lock out changes to the allocation maps and remap. */ + down_write(&OCFS2_I(inode_in)->ip_alloc_sem); + if (!same_inode) + down_write_nested(&OCFS2_I(inode_out)->ip_alloc_sem, + SINGLE_DEPTH_NESTING); + + /* Zap any page cache for the destination file's range. */ + truncate_inode_pages_range(&inode_out->i_data, + round_down(pos_out, PAGE_SIZE), + round_up(pos_out + len, PAGE_SIZE) - 1); + + remapped = ocfs2_reflink_remap_blocks(inode_in, in_bh, pos_in, + inode_out, out_bh, pos_out, len); + up_write(&OCFS2_I(inode_in)->ip_alloc_sem); + if (!same_inode) + up_write(&OCFS2_I(inode_out)->ip_alloc_sem); + if (remapped < 0) { + ret = remapped; + mlog_errno(ret); + goto out_unlock; + } + + /* + * Empty the extent map so that we may get the right extent + * record from the disk. + */ + ocfs2_extent_map_trunc(inode_in, 0); + ocfs2_extent_map_trunc(inode_out, 0); + + ret = ocfs2_reflink_update_dest(inode_out, out_bh, pos_out + len); + if (ret) { + mlog_errno(ret); + goto out_unlock; + } + +out_unlock: + ocfs2_reflink_inodes_unlock(inode_in, in_bh, inode_out, out_bh); + return remapped > 0 ? remapped : ret; +} + +static loff_t ocfs2_dir_llseek(struct file *file, loff_t offset, int whence) +{ + struct ocfs2_file_private *fp = file->private_data; + + return generic_llseek_cookie(file, offset, whence, &fp->cookie); +} + +const struct inode_operations ocfs2_file_iops = { + .setattr = ocfs2_setattr, + .getattr = ocfs2_getattr, + .permission = ocfs2_permission, + .listxattr = ocfs2_listxattr, + .fiemap = ocfs2_fiemap, + .get_inode_acl = ocfs2_iop_get_acl, + .set_acl = ocfs2_iop_set_acl, + .fileattr_get = ocfs2_fileattr_get, + .fileattr_set = ocfs2_fileattr_set, +}; + +const struct inode_operations ocfs2_special_file_iops = { + .setattr = ocfs2_setattr, + .getattr = ocfs2_getattr, + .listxattr = ocfs2_listxattr, + .permission = ocfs2_permission, + .get_inode_acl = ocfs2_iop_get_acl, + .set_acl = ocfs2_iop_set_acl, +}; + +/* + * Other than ->lock, keep ocfs2_fops and ocfs2_dops in sync with + * ocfs2_fops_no_plocks and ocfs2_dops_no_plocks! + */ +const struct file_operations ocfs2_fops = { + .llseek = ocfs2_file_llseek, + .mmap_prepare = ocfs2_mmap_prepare, + .fsync = ocfs2_sync_file, + .release = ocfs2_file_release, + .open = ocfs2_file_open, + .read_iter = ocfs2_file_read_iter, + .write_iter = ocfs2_file_write_iter, + .unlocked_ioctl = ocfs2_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = ocfs2_compat_ioctl, +#endif + .lock = ocfs2_lock, + .flock = ocfs2_flock, + .splice_read = ocfs2_file_splice_read, + .splice_write = iter_file_splice_write, + .fallocate = ocfs2_fallocate, + .remap_file_range = ocfs2_remap_file_range, + .fop_flags = FOP_ASYNC_LOCK, + .setlease = generic_setlease, +}; + +WRAP_DIR_ITER(ocfs2_readdir) // FIXME! +const struct file_operations ocfs2_dops = { + .llseek = ocfs2_dir_llseek, + .read = generic_read_dir, + .iterate_shared = shared_ocfs2_readdir, + .fsync = ocfs2_sync_file, + .release = ocfs2_dir_release, + .open = ocfs2_dir_open, + .unlocked_ioctl = ocfs2_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = ocfs2_compat_ioctl, +#endif + .lock = ocfs2_lock, + .flock = ocfs2_flock, + .fop_flags = FOP_ASYNC_LOCK, + .setlease = generic_setlease, +}; + +/* + * POSIX-lockless variants of our file_operations. + * + * These will be used if the underlying cluster stack does not support + * posix file locking, if the user passes the "localflocks" mount + * option, or if we have a local-only fs. + * + * ocfs2_flock is in here because all stacks handle UNIX file locks, + * so we still want it in the case of no stack support for + * plocks. Internally, it will do the right thing when asked to ignore + * the cluster. + */ +const struct file_operations ocfs2_fops_no_plocks = { + .llseek = ocfs2_file_llseek, + .mmap_prepare = ocfs2_mmap_prepare, + .fsync = ocfs2_sync_file, + .release = ocfs2_file_release, + .open = ocfs2_file_open, + .read_iter = ocfs2_file_read_iter, + .write_iter = ocfs2_file_write_iter, + .unlocked_ioctl = ocfs2_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = ocfs2_compat_ioctl, +#endif + .flock = ocfs2_flock, + .splice_read = filemap_splice_read, + .splice_write = iter_file_splice_write, + .fallocate = ocfs2_fallocate, + .remap_file_range = ocfs2_remap_file_range, + .setlease = generic_setlease, +}; + +const struct file_operations ocfs2_dops_no_plocks = { + .llseek = ocfs2_dir_llseek, + .read = generic_read_dir, + .iterate_shared = shared_ocfs2_readdir, + .fsync = ocfs2_sync_file, + .release = ocfs2_dir_release, + .open = ocfs2_dir_open, + .unlocked_ioctl = ocfs2_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = ocfs2_compat_ioctl, +#endif + .flock = ocfs2_flock, + .setlease = generic_setlease, +}; diff --git a/fs/ocfs2/file.h b/fs/ocfs2/file.h new file mode 100644 index 000000000..41e65e45a --- /dev/null +++ b/fs/ocfs2/file.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * file.h + * + * Function prototypes + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#ifndef OCFS2_FILE_H +#define OCFS2_FILE_H + +extern const struct file_operations ocfs2_fops; +extern const struct file_operations ocfs2_dops; +extern const struct file_operations ocfs2_fops_no_plocks; +extern const struct file_operations ocfs2_dops_no_plocks; +extern const struct inode_operations ocfs2_file_iops; +extern const struct inode_operations ocfs2_special_file_iops; +struct ocfs2_alloc_context; +enum ocfs2_alloc_restarted; + +struct ocfs2_file_private { + u64 cookie; + struct file *fp_file; + struct mutex fp_mutex; + struct ocfs2_lock_res fp_flock; +}; + +int ocfs2_add_inode_data(struct ocfs2_super *osb, + struct inode *inode, + u32 *logical_offset, + u32 clusters_to_add, + int mark_unwritten, + struct buffer_head *fe_bh, + handle_t *handle, + struct ocfs2_alloc_context *data_ac, + struct ocfs2_alloc_context *meta_ac, + enum ocfs2_alloc_restarted *reason_ret); +int ocfs2_set_inode_size(handle_t *handle, + struct inode *inode, + struct buffer_head *fe_bh, + u64 new_i_size); +int ocfs2_simple_size_update(struct inode *inode, + struct buffer_head *di_bh, + u64 new_i_size); +int ocfs2_truncate_file(struct inode *inode, + struct buffer_head *di_bh, + u64 new_i_size); +int ocfs2_extend_no_holes(struct inode *inode, struct buffer_head *di_bh, + u64 new_i_size, u64 zero_to); +int ocfs2_zero_extend(struct inode *inode, struct buffer_head *di_bh, + loff_t zero_to); +int ocfs2_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr); +int ocfs2_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags); +int ocfs2_permission(struct mnt_idmap *idmap, + struct inode *inode, + int mask); + +int ocfs2_should_update_atime(struct inode *inode, + struct vfsmount *vfsmnt); +int ocfs2_update_inode_atime(struct inode *inode, + struct buffer_head *bh); + +int ocfs2_change_file_space(struct file *file, unsigned int cmd, + struct ocfs2_space_resv *sr); + +int ocfs2_check_range_for_refcount(struct inode *inode, loff_t pos, + size_t count); +int ocfs2_remove_inode_range(struct inode *inode, + struct buffer_head *di_bh, u64 byte_start, + u64 byte_len); +#endif /* OCFS2_FILE_H */ diff --git a/fs/ocfs2/filecheck.c b/fs/ocfs2/filecheck.c new file mode 100644 index 000000000..be713d7d2 --- /dev/null +++ b/fs/ocfs2/filecheck.c @@ -0,0 +1,509 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * filecheck.c + * + * Code which implements online file check. + * + * Copyright (C) 2016 SuSE. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ocfs2.h" +#include "ocfs2_fs.h" +#include "stackglue.h" +#include "inode.h" + +#include "filecheck.h" + + +/* File check error strings, + * must correspond with error number in header file. + */ +static const char * const ocfs2_filecheck_errs[] = { + "SUCCESS", + "FAILED", + "INPROGRESS", + "READONLY", + "INJBD", + "INVALIDINO", + "BLOCKECC", + "BLOCKNO", + "VALIDFLAG", + "GENERATION", + "UNSUPPORTED" +}; + +struct ocfs2_filecheck_entry { + struct list_head fe_list; + unsigned long fe_ino; + unsigned int fe_type; + unsigned int fe_done:1; + unsigned int fe_status:31; +}; + +struct ocfs2_filecheck_args { + unsigned int fa_type; + union { + unsigned long fa_ino; + unsigned int fa_len; + }; +}; + +static const char * +ocfs2_filecheck_error(int errno) +{ + if (!errno) + return ocfs2_filecheck_errs[errno]; + + BUG_ON(errno < OCFS2_FILECHECK_ERR_START || + errno > OCFS2_FILECHECK_ERR_END); + return ocfs2_filecheck_errs[errno - OCFS2_FILECHECK_ERR_START + 1]; +} + +static ssize_t ocfs2_filecheck_attr_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf); +static ssize_t ocfs2_filecheck_attr_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count); +static struct kobj_attribute ocfs2_filecheck_attr_chk = + __ATTR(check, S_IRUSR | S_IWUSR, + ocfs2_filecheck_attr_show, + ocfs2_filecheck_attr_store); +static struct kobj_attribute ocfs2_filecheck_attr_fix = + __ATTR(fix, S_IRUSR | S_IWUSR, + ocfs2_filecheck_attr_show, + ocfs2_filecheck_attr_store); +static struct kobj_attribute ocfs2_filecheck_attr_set = + __ATTR(set, S_IRUSR | S_IWUSR, + ocfs2_filecheck_attr_show, + ocfs2_filecheck_attr_store); +static struct attribute *ocfs2_filecheck_attrs[] = { + &ocfs2_filecheck_attr_chk.attr, + &ocfs2_filecheck_attr_fix.attr, + &ocfs2_filecheck_attr_set.attr, + NULL +}; +ATTRIBUTE_GROUPS(ocfs2_filecheck); + +static void ocfs2_filecheck_release(struct kobject *kobj) +{ + struct ocfs2_filecheck_sysfs_entry *entry = container_of(kobj, + struct ocfs2_filecheck_sysfs_entry, fs_kobj); + + complete(&entry->fs_kobj_unregister); +} + +static ssize_t +ocfs2_filecheck_show(struct kobject *kobj, struct attribute *attr, char *buf) +{ + ssize_t ret = -EIO; + struct kobj_attribute *kattr = container_of(attr, + struct kobj_attribute, attr); + + kobject_get(kobj); + if (kattr->show) + ret = kattr->show(kobj, kattr, buf); + kobject_put(kobj); + return ret; +} + +static ssize_t +ocfs2_filecheck_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t count) +{ + ssize_t ret = -EIO; + struct kobj_attribute *kattr = container_of(attr, + struct kobj_attribute, attr); + + kobject_get(kobj); + if (kattr->store) + ret = kattr->store(kobj, kattr, buf, count); + kobject_put(kobj); + return ret; +} + +static const struct sysfs_ops ocfs2_filecheck_ops = { + .show = ocfs2_filecheck_show, + .store = ocfs2_filecheck_store, +}; + +static struct kobj_type ocfs2_ktype_filecheck = { + .default_groups = ocfs2_filecheck_groups, + .sysfs_ops = &ocfs2_filecheck_ops, + .release = ocfs2_filecheck_release, +}; + +static void +ocfs2_filecheck_sysfs_free(struct ocfs2_filecheck_sysfs_entry *entry) +{ + struct ocfs2_filecheck_entry *p; + + spin_lock(&entry->fs_fcheck->fc_lock); + while (!list_empty(&entry->fs_fcheck->fc_head)) { + p = list_first_entry(&entry->fs_fcheck->fc_head, + struct ocfs2_filecheck_entry, fe_list); + list_del(&p->fe_list); + BUG_ON(!p->fe_done); /* To free a undone file check entry */ + kfree(p); + } + spin_unlock(&entry->fs_fcheck->fc_lock); + + kfree(entry->fs_fcheck); + entry->fs_fcheck = NULL; +} + +int ocfs2_filecheck_create_sysfs(struct ocfs2_super *osb) +{ + int ret; + struct ocfs2_filecheck *fcheck; + struct ocfs2_filecheck_sysfs_entry *entry = &osb->osb_fc_ent; + + fcheck = kmalloc_obj(struct ocfs2_filecheck, GFP_NOFS); + if (!fcheck) + return -ENOMEM; + + INIT_LIST_HEAD(&fcheck->fc_head); + spin_lock_init(&fcheck->fc_lock); + fcheck->fc_max = OCFS2_FILECHECK_MINSIZE; + fcheck->fc_size = 0; + fcheck->fc_done = 0; + + entry->fs_kobj.kset = osb->osb_dev_kset; + init_completion(&entry->fs_kobj_unregister); + ret = kobject_init_and_add(&entry->fs_kobj, &ocfs2_ktype_filecheck, + NULL, "filecheck"); + if (ret) { + kobject_put(&entry->fs_kobj); + kfree(fcheck); + return ret; + } + + entry->fs_fcheck = fcheck; + return 0; +} + +void ocfs2_filecheck_remove_sysfs(struct ocfs2_super *osb) +{ + if (!osb->osb_fc_ent.fs_fcheck) + return; + + kobject_del(&osb->osb_fc_ent.fs_kobj); + kobject_put(&osb->osb_fc_ent.fs_kobj); + wait_for_completion(&osb->osb_fc_ent.fs_kobj_unregister); + ocfs2_filecheck_sysfs_free(&osb->osb_fc_ent); +} + +static int +ocfs2_filecheck_erase_entries(struct ocfs2_filecheck_sysfs_entry *ent, + unsigned int count); +static int +ocfs2_filecheck_adjust_max(struct ocfs2_filecheck_sysfs_entry *ent, + unsigned int len) +{ + int ret; + + if ((len < OCFS2_FILECHECK_MINSIZE) || (len > OCFS2_FILECHECK_MAXSIZE)) + return -EINVAL; + + spin_lock(&ent->fs_fcheck->fc_lock); + if (len < (ent->fs_fcheck->fc_size - ent->fs_fcheck->fc_done)) { + mlog(ML_NOTICE, + "Cannot set online file check maximum entry number " + "to %u due to too many pending entries(%u)\n", + len, ent->fs_fcheck->fc_size - ent->fs_fcheck->fc_done); + ret = -EBUSY; + } else { + if (len < ent->fs_fcheck->fc_size) + BUG_ON(!ocfs2_filecheck_erase_entries(ent, + ent->fs_fcheck->fc_size - len)); + + ent->fs_fcheck->fc_max = len; + ret = 0; + } + spin_unlock(&ent->fs_fcheck->fc_lock); + + return ret; +} + +#define OCFS2_FILECHECK_ARGS_LEN 24 +static int +ocfs2_filecheck_args_get_long(const char *buf, size_t count, + unsigned long *val) +{ + char buffer[OCFS2_FILECHECK_ARGS_LEN]; + + memcpy(buffer, buf, count); + buffer[count] = '\0'; + + if (kstrtoul(buffer, 0, val)) + return 1; + + return 0; +} + +static int +ocfs2_filecheck_type_parse(const char *name, unsigned int *type) +{ + if (!strncmp(name, "fix", 4)) + *type = OCFS2_FILECHECK_TYPE_FIX; + else if (!strncmp(name, "check", 6)) + *type = OCFS2_FILECHECK_TYPE_CHK; + else if (!strncmp(name, "set", 4)) + *type = OCFS2_FILECHECK_TYPE_SET; + else + return 1; + + return 0; +} + +static int +ocfs2_filecheck_args_parse(const char *name, const char *buf, size_t count, + struct ocfs2_filecheck_args *args) +{ + unsigned long val = 0; + unsigned int type; + + /* too short/long args length */ + if ((count < 1) || (count >= OCFS2_FILECHECK_ARGS_LEN)) + return 1; + + if (ocfs2_filecheck_type_parse(name, &type)) + return 1; + if (ocfs2_filecheck_args_get_long(buf, count, &val)) + return 1; + + if (val <= 0) + return 1; + + args->fa_type = type; + if (type == OCFS2_FILECHECK_TYPE_SET) + args->fa_len = (unsigned int)val; + else + args->fa_ino = val; + + return 0; +} + +static ssize_t ocfs2_filecheck_attr_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + + ssize_t ret = 0, total = 0, remain = PAGE_SIZE; + unsigned int type; + struct ocfs2_filecheck_entry *p; + struct ocfs2_filecheck_sysfs_entry *ent = container_of(kobj, + struct ocfs2_filecheck_sysfs_entry, fs_kobj); + + if (ocfs2_filecheck_type_parse(attr->attr.name, &type)) + return -EINVAL; + + if (type == OCFS2_FILECHECK_TYPE_SET) { + spin_lock(&ent->fs_fcheck->fc_lock); + total = snprintf(buf, remain, "%u\n", ent->fs_fcheck->fc_max); + spin_unlock(&ent->fs_fcheck->fc_lock); + goto exit; + } + + ret = snprintf(buf, remain, "INO\t\tDONE\tERROR\n"); + total += ret; + remain -= ret; + spin_lock(&ent->fs_fcheck->fc_lock); + list_for_each_entry(p, &ent->fs_fcheck->fc_head, fe_list) { + if (p->fe_type != type) + continue; + + ret = snprintf(buf + total, remain, "%lu\t\t%u\t%s\n", + p->fe_ino, p->fe_done, + ocfs2_filecheck_error(p->fe_status)); + if (ret >= remain) { + /* snprintf() didn't fit */ + total = -E2BIG; + break; + } + total += ret; + remain -= ret; + } + spin_unlock(&ent->fs_fcheck->fc_lock); + +exit: + return total; +} + +static inline int +ocfs2_filecheck_is_dup_entry(struct ocfs2_filecheck_sysfs_entry *ent, + unsigned long ino) +{ + struct ocfs2_filecheck_entry *p; + + list_for_each_entry(p, &ent->fs_fcheck->fc_head, fe_list) { + if (!p->fe_done) { + if (p->fe_ino == ino) + return 1; + } + } + + return 0; +} + +static inline int +ocfs2_filecheck_erase_entry(struct ocfs2_filecheck_sysfs_entry *ent) +{ + struct ocfs2_filecheck_entry *p; + + list_for_each_entry(p, &ent->fs_fcheck->fc_head, fe_list) { + if (p->fe_done) { + list_del(&p->fe_list); + kfree(p); + ent->fs_fcheck->fc_size--; + ent->fs_fcheck->fc_done--; + return 1; + } + } + + return 0; +} + +static int +ocfs2_filecheck_erase_entries(struct ocfs2_filecheck_sysfs_entry *ent, + unsigned int count) +{ + unsigned int i = 0; + unsigned int ret = 0; + + while (i++ < count) { + if (ocfs2_filecheck_erase_entry(ent)) + ret++; + else + break; + } + + return (ret == count ? 1 : 0); +} + +static void +ocfs2_filecheck_done_entry(struct ocfs2_filecheck_sysfs_entry *ent, + struct ocfs2_filecheck_entry *entry) +{ + spin_lock(&ent->fs_fcheck->fc_lock); + entry->fe_done = 1; + ent->fs_fcheck->fc_done++; + spin_unlock(&ent->fs_fcheck->fc_lock); +} + +static unsigned int +ocfs2_filecheck_handle(struct ocfs2_super *osb, + unsigned long ino, unsigned int flags) +{ + unsigned int ret = OCFS2_FILECHECK_ERR_SUCCESS; + struct inode *inode = NULL; + int rc; + + inode = ocfs2_iget(osb, ino, flags, 0); + if (IS_ERR(inode)) { + rc = (int)(-(long)inode); + if (rc >= OCFS2_FILECHECK_ERR_START && + rc < OCFS2_FILECHECK_ERR_END) + ret = rc; + else + ret = OCFS2_FILECHECK_ERR_FAILED; + } else + iput(inode); + + return ret; +} + +static void +ocfs2_filecheck_handle_entry(struct ocfs2_filecheck_sysfs_entry *ent, + struct ocfs2_filecheck_entry *entry) +{ + struct ocfs2_super *osb = container_of(ent, struct ocfs2_super, + osb_fc_ent); + + if (entry->fe_type == OCFS2_FILECHECK_TYPE_CHK) + entry->fe_status = ocfs2_filecheck_handle(osb, + entry->fe_ino, OCFS2_FI_FLAG_FILECHECK_CHK); + else if (entry->fe_type == OCFS2_FILECHECK_TYPE_FIX) + entry->fe_status = ocfs2_filecheck_handle(osb, + entry->fe_ino, OCFS2_FI_FLAG_FILECHECK_FIX); + else + entry->fe_status = OCFS2_FILECHECK_ERR_UNSUPPORTED; + + ocfs2_filecheck_done_entry(ent, entry); +} + +static ssize_t ocfs2_filecheck_attr_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + ssize_t ret = 0; + struct ocfs2_filecheck_args args; + struct ocfs2_filecheck_entry *entry; + struct ocfs2_filecheck_sysfs_entry *ent = container_of(kobj, + struct ocfs2_filecheck_sysfs_entry, fs_kobj); + + if (count == 0) + return count; + + if (ocfs2_filecheck_args_parse(attr->attr.name, buf, count, &args)) + return -EINVAL; + + if (args.fa_type == OCFS2_FILECHECK_TYPE_SET) { + ret = ocfs2_filecheck_adjust_max(ent, args.fa_len); + goto exit; + } + + entry = kmalloc_obj(struct ocfs2_filecheck_entry, GFP_NOFS); + if (!entry) { + ret = -ENOMEM; + goto exit; + } + + spin_lock(&ent->fs_fcheck->fc_lock); + if (ocfs2_filecheck_is_dup_entry(ent, args.fa_ino)) { + ret = -EEXIST; + kfree(entry); + } else if ((ent->fs_fcheck->fc_size >= ent->fs_fcheck->fc_max) && + (ent->fs_fcheck->fc_done == 0)) { + mlog(ML_NOTICE, + "Cannot do more file check " + "since file check queue(%u) is full now\n", + ent->fs_fcheck->fc_max); + ret = -EAGAIN; + kfree(entry); + } else { + if ((ent->fs_fcheck->fc_size >= ent->fs_fcheck->fc_max) && + (ent->fs_fcheck->fc_done > 0)) { + /* Delete the oldest entry which was done, + * make sure the entry size in list does + * not exceed maximum value + */ + BUG_ON(!ocfs2_filecheck_erase_entry(ent)); + } + + entry->fe_ino = args.fa_ino; + entry->fe_type = args.fa_type; + entry->fe_done = 0; + entry->fe_status = OCFS2_FILECHECK_ERR_INPROGRESS; + list_add_tail(&entry->fe_list, &ent->fs_fcheck->fc_head); + ent->fs_fcheck->fc_size++; + } + spin_unlock(&ent->fs_fcheck->fc_lock); + + if (!ret) + ocfs2_filecheck_handle_entry(ent, entry); + +exit: + return ret ?: count; +} diff --git a/fs/ocfs2/filecheck.h b/fs/ocfs2/filecheck.h new file mode 100644 index 000000000..d3bcb8bcf --- /dev/null +++ b/fs/ocfs2/filecheck.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * filecheck.h + * + * Online file check. + * + * Copyright (C) 2016 SuSE. All rights reserved. + */ + + +#ifndef FILECHECK_H +#define FILECHECK_H + +#include +#include + + +/* File check errno */ +enum { + OCFS2_FILECHECK_ERR_SUCCESS = 0, /* Success */ + OCFS2_FILECHECK_ERR_FAILED = 1000, /* Other failure */ + OCFS2_FILECHECK_ERR_INPROGRESS, /* In progress */ + OCFS2_FILECHECK_ERR_READONLY, /* Read only */ + OCFS2_FILECHECK_ERR_INJBD, /* Buffer in jbd */ + OCFS2_FILECHECK_ERR_INVALIDINO, /* Invalid ino */ + OCFS2_FILECHECK_ERR_BLOCKECC, /* Block ecc */ + OCFS2_FILECHECK_ERR_BLOCKNO, /* Block number */ + OCFS2_FILECHECK_ERR_VALIDFLAG, /* Inode valid flag */ + OCFS2_FILECHECK_ERR_GENERATION, /* Inode generation */ + OCFS2_FILECHECK_ERR_UNSUPPORTED /* Unsupported */ +}; + +#define OCFS2_FILECHECK_ERR_START OCFS2_FILECHECK_ERR_FAILED +#define OCFS2_FILECHECK_ERR_END OCFS2_FILECHECK_ERR_UNSUPPORTED + +struct ocfs2_filecheck { + struct list_head fc_head; /* File check entry list head */ + spinlock_t fc_lock; + unsigned int fc_max; /* Maximum number of entry in list */ + unsigned int fc_size; /* Current entry count in list */ + unsigned int fc_done; /* Finished entry count in list */ +}; + +#define OCFS2_FILECHECK_MAXSIZE 100 +#define OCFS2_FILECHECK_MINSIZE 10 + +/* File check operation type */ +enum { + OCFS2_FILECHECK_TYPE_CHK = 0, /* Check a file(inode) */ + OCFS2_FILECHECK_TYPE_FIX, /* Fix a file(inode) */ + OCFS2_FILECHECK_TYPE_SET = 100 /* Set entry list maximum size */ +}; + +struct ocfs2_filecheck_sysfs_entry { /* sysfs entry per partition */ + struct kobject fs_kobj; + struct completion fs_kobj_unregister; + struct ocfs2_filecheck *fs_fcheck; +}; + + +int ocfs2_filecheck_create_sysfs(struct ocfs2_super *osb); +void ocfs2_filecheck_remove_sysfs(struct ocfs2_super *osb); + +#endif /* FILECHECK_H */ diff --git a/fs/ocfs2/heartbeat.c b/fs/ocfs2/heartbeat.c new file mode 100644 index 000000000..22da768e6 --- /dev/null +++ b/fs/ocfs2/heartbeat.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * heartbeat.c + * + * Register ourselves with the heartbeat service, keep our node maps + * up to date, and fire off recovery when needed. + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#include +#include +#include +#include + +#include + +#include "ocfs2.h" + +#include "alloc.h" +#include "heartbeat.h" +#include "inode.h" +#include "journal.h" +#include "ocfs2_trace.h" + +#include "buffer_head_io.h" + +/* special case -1 for now + * TODO: should *really* make sure the calling func never passes -1!! */ +static void ocfs2_node_map_init(struct ocfs2_node_map *map) +{ + map->num_nodes = OCFS2_NODE_MAP_MAX_NODES; + bitmap_zero(map->map, OCFS2_NODE_MAP_MAX_NODES); +} + +void ocfs2_init_node_maps(struct ocfs2_super *osb) +{ + spin_lock_init(&osb->node_map_lock); + ocfs2_node_map_init(&osb->osb_recovering_orphan_dirs); +} + +void ocfs2_do_node_down(int node_num, void *data) +{ + struct ocfs2_super *osb = data; + + BUG_ON(osb->node_num == node_num); + + trace_ocfs2_do_node_down(node_num); + + if (!osb->cconn) { + /* + * No cluster connection means we're not even ready to + * participate yet. We check the slots after the cluster + * comes up, so we will notice the node death then. We + * can safely ignore it here. + */ + return; + } + + ocfs2_recovery_thread(osb, node_num); +} + +void ocfs2_node_map_set_bit(struct ocfs2_super *osb, + struct ocfs2_node_map *map, + int bit) +{ + if (bit==-1) + return; + BUG_ON(bit >= map->num_nodes); + spin_lock(&osb->node_map_lock); + set_bit(bit, map->map); + spin_unlock(&osb->node_map_lock); +} + +void ocfs2_node_map_clear_bit(struct ocfs2_super *osb, + struct ocfs2_node_map *map, + int bit) +{ + if (bit==-1) + return; + BUG_ON(bit >= map->num_nodes); + spin_lock(&osb->node_map_lock); + clear_bit(bit, map->map); + spin_unlock(&osb->node_map_lock); +} + +int ocfs2_node_map_test_bit(struct ocfs2_super *osb, + struct ocfs2_node_map *map, + int bit) +{ + int ret; + if (bit >= map->num_nodes) { + mlog(ML_ERROR, "bit=%d map->num_nodes=%d\n", bit, map->num_nodes); + BUG(); + } + spin_lock(&osb->node_map_lock); + ret = test_bit(bit, map->map); + spin_unlock(&osb->node_map_lock); + return ret; +} + diff --git a/fs/ocfs2/heartbeat.h b/fs/ocfs2/heartbeat.h new file mode 100644 index 000000000..f1f8b1802 --- /dev/null +++ b/fs/ocfs2/heartbeat.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * heartbeat.h + * + * Function prototypes + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#ifndef OCFS2_HEARTBEAT_H +#define OCFS2_HEARTBEAT_H + +void ocfs2_init_node_maps(struct ocfs2_super *osb); + +void ocfs2_do_node_down(int node_num, void *data); + +/* node map functions - used to keep track of mounted and in-recovery + * nodes. */ +void ocfs2_node_map_set_bit(struct ocfs2_super *osb, + struct ocfs2_node_map *map, + int bit); +void ocfs2_node_map_clear_bit(struct ocfs2_super *osb, + struct ocfs2_node_map *map, + int bit); +int ocfs2_node_map_test_bit(struct ocfs2_super *osb, + struct ocfs2_node_map *map, + int bit); + +#endif /* OCFS2_HEARTBEAT_H */ diff --git a/fs/ocfs2/inode.c b/fs/ocfs2/inode.c new file mode 100644 index 000000000..180107a11 --- /dev/null +++ b/fs/ocfs2/inode.c @@ -0,0 +1,2035 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * inode.c + * + * vfs' aops, fops, dops and iops + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include "ocfs2.h" + +#include "alloc.h" +#include "dir.h" +#include "blockcheck.h" +#include "dlmglue.h" +#include "extent_map.h" +#include "file.h" +#include "heartbeat.h" +#include "inode.h" +#include "journal.h" +#include "namei.h" +#include "suballoc.h" +#include "super.h" +#include "symlink.h" +#include "sysfile.h" +#include "uptodate.h" +#include "xattr.h" +#include "refcounttree.h" +#include "ocfs2_trace.h" +#include "filecheck.h" + +#include "buffer_head_io.h" + +struct ocfs2_find_inode_args +{ + u64 fi_blkno; + unsigned long fi_ino; + unsigned int fi_flags; + unsigned int fi_sysfile_type; +}; + +static int ocfs2_read_locked_inode(struct inode *inode, + struct ocfs2_find_inode_args *args); +static int ocfs2_init_locked_inode(struct inode *inode, void *opaque); +static int ocfs2_find_actor(struct inode *inode, void *opaque); +static int ocfs2_truncate_for_delete(struct ocfs2_super *osb, + struct inode *inode, + struct buffer_head *fe_bh); + +static int ocfs2_filecheck_read_inode_block_full(struct inode *inode, + struct buffer_head **bh, + int flags, int type); +static int ocfs2_filecheck_validate_inode_block(struct super_block *sb, + struct buffer_head *bh); +static int ocfs2_filecheck_repair_inode_block(struct super_block *sb, + struct buffer_head *bh); + +static bool ocfs2_valid_inode_mode(umode_t mode) +{ + return fs_umode_to_ftype(mode) != FT_UNKNOWN; +} + +static bool ocfs2_dinode_has_unexpected_rdev(struct ocfs2_dinode *di) +{ + umode_t mode = le16_to_cpu(di->i_mode); + + if (le32_to_cpu(di->i_flags) & OCFS2_SYSTEM_FL) + return false; + + return !S_ISCHR(mode) && !S_ISBLK(mode) && di->id1.dev1.i_rdev != 0; +} + +static bool ocfs2_dinode_has_size_without_clusters(struct super_block *sb, + struct ocfs2_dinode *di) +{ + umode_t mode = le16_to_cpu(di->i_mode); + + if (le32_to_cpu(di->i_flags) & OCFS2_SYSTEM_FL) + return false; + if (le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_DATA_FL) + return false; + if (!le64_to_cpu(di->i_size) || le32_to_cpu(di->i_clusters)) + return false; + + if (S_ISDIR(mode)) + return true; + + return !ocfs2_sparse_alloc(OCFS2_SB(sb)) && S_ISREG(mode); +} + +void ocfs2_set_inode_flags(struct inode *inode) +{ + unsigned int flags = OCFS2_I(inode)->ip_attr; + + inode->i_flags &= ~(S_IMMUTABLE | + S_SYNC | S_APPEND | S_NOATIME | S_DIRSYNC); + + if (flags & OCFS2_IMMUTABLE_FL) + inode->i_flags |= S_IMMUTABLE; + + if (flags & OCFS2_SYNC_FL) + inode->i_flags |= S_SYNC; + if (flags & OCFS2_APPEND_FL) + inode->i_flags |= S_APPEND; + if (flags & OCFS2_NOATIME_FL) + inode->i_flags |= S_NOATIME; + if (flags & OCFS2_DIRSYNC_FL) + inode->i_flags |= S_DIRSYNC; +} + +/* Propagate flags from i_flags to OCFS2_I(inode)->ip_attr */ +void ocfs2_get_inode_flags(struct ocfs2_inode_info *oi) +{ + unsigned int flags = oi->vfs_inode.i_flags; + + oi->ip_attr &= ~(OCFS2_SYNC_FL|OCFS2_APPEND_FL| + OCFS2_IMMUTABLE_FL|OCFS2_NOATIME_FL|OCFS2_DIRSYNC_FL); + if (flags & S_SYNC) + oi->ip_attr |= OCFS2_SYNC_FL; + if (flags & S_APPEND) + oi->ip_attr |= OCFS2_APPEND_FL; + if (flags & S_IMMUTABLE) + oi->ip_attr |= OCFS2_IMMUTABLE_FL; + if (flags & S_NOATIME) + oi->ip_attr |= OCFS2_NOATIME_FL; + if (flags & S_DIRSYNC) + oi->ip_attr |= OCFS2_DIRSYNC_FL; +} + +struct inode *ocfs2_ilookup(struct super_block *sb, u64 blkno) +{ + struct ocfs2_find_inode_args args; + + args.fi_blkno = blkno; + args.fi_flags = 0; + args.fi_ino = ino_from_blkno(sb, blkno); + args.fi_sysfile_type = 0; + + return ilookup5(sb, blkno, ocfs2_find_actor, &args); +} +struct inode *ocfs2_iget(struct ocfs2_super *osb, u64 blkno, unsigned flags, + int sysfile_type) +{ + int rc = -ESTALE; + struct inode *inode = NULL; + struct super_block *sb = osb->sb; + struct ocfs2_find_inode_args args; + journal_t *journal = osb->journal->j_journal; + + trace_ocfs2_iget_begin((unsigned long long)blkno, flags, + sysfile_type); + + /* Ok. By now we've either got the offsets passed to us by the + * caller, or we just pulled them off the bh. Lets do some + * sanity checks to make sure they're OK. */ + if (blkno == 0) { + inode = ERR_PTR(-EINVAL); + mlog_errno(PTR_ERR(inode)); + goto bail; + } + + args.fi_blkno = blkno; + args.fi_flags = flags; + args.fi_ino = ino_from_blkno(sb, blkno); + args.fi_sysfile_type = sysfile_type; + + inode = iget5_locked(sb, args.fi_ino, ocfs2_find_actor, + ocfs2_init_locked_inode, &args); + /* inode was *not* in the inode cache. 2.6.x requires + * us to do our own read_inode call and unlock it + * afterwards. */ + if (inode == NULL) { + inode = ERR_PTR(-ENOMEM); + mlog_errno(PTR_ERR(inode)); + goto bail; + } + trace_ocfs2_iget5_locked(inode_state_read_once(inode)); + if (inode_state_read_once(inode) & I_NEW) { + rc = ocfs2_read_locked_inode(inode, &args); + unlock_new_inode(inode); + } + if (is_bad_inode(inode)) { + iput(inode); + inode = ERR_PTR(rc); + goto bail; + } + + /* + * Set transaction id's of transactions that have to be committed + * to finish f[data]sync. We set them to currently running transaction + * as we cannot be sure that the inode or some of its metadata isn't + * part of the transaction - the inode could have been reclaimed and + * now it is reread from disk. + */ + if (journal) { + transaction_t *transaction; + tid_t tid; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + + read_lock(&journal->j_state_lock); + if (journal->j_running_transaction) + transaction = journal->j_running_transaction; + else + transaction = journal->j_committing_transaction; + if (transaction) + tid = transaction->t_tid; + else + tid = journal->j_commit_sequence; + read_unlock(&journal->j_state_lock); + oi->i_sync_tid = tid; + oi->i_datasync_tid = tid; + } + +bail: + if (!IS_ERR(inode)) { + trace_ocfs2_iget_end(inode, + (unsigned long long)OCFS2_I(inode)->ip_blkno); + } + + return inode; +} + +static int ocfs2_dinode_has_extents(struct ocfs2_dinode *di) +{ + /* inodes flagged with other stuff in id2 */ + if (le32_to_cpu(di->i_flags) & + (OCFS2_SUPER_BLOCK_FL | OCFS2_LOCAL_ALLOC_FL | OCFS2_CHAIN_FL | + OCFS2_DEALLOC_FL)) + return 0; + /* i_flags doesn't indicate when id2 is a fast symlink */ + if (S_ISLNK(le16_to_cpu(di->i_mode)) && le64_to_cpu(di->i_size) && + !le32_to_cpu(di->i_clusters)) + return 0; + if (le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_DATA_FL) + return 0; + + return 1; +} + +/* + * here's how inodes get read from disk: + * iget5_locked -> find_actor -> OCFS2_FIND_ACTOR + * found? : return the in-memory inode + * not found? : get_new_inode -> OCFS2_INIT_LOCKED_INODE + */ + +static int ocfs2_find_actor(struct inode *inode, void *opaque) +{ + struct ocfs2_find_inode_args *args = NULL; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + int ret = 0; + + args = opaque; + + mlog_bug_on_msg(!inode, "No inode in find actor!\n"); + + trace_ocfs2_find_actor(inode, inode->i_ino, opaque, args->fi_blkno); + + if (oi->ip_blkno != args->fi_blkno) + goto bail; + + ret = 1; +bail: + return ret; +} + +/* + * initialize the new inode, but don't do anything that would cause + * us to sleep. + * return 0 on success, 1 on failure + */ +static int ocfs2_init_locked_inode(struct inode *inode, void *opaque) +{ + struct ocfs2_find_inode_args *args = opaque; +#ifdef CONFIG_LOCKDEP + static struct lock_class_key ocfs2_sysfile_lock_key[NUM_SYSTEM_INODES]; + static struct lock_class_key ocfs2_quota_ip_alloc_sem_key, + ocfs2_file_ip_alloc_sem_key; +#endif + + inode->i_ino = args->fi_ino; + OCFS2_I(inode)->ip_blkno = args->fi_blkno; +#ifdef CONFIG_LOCKDEP + switch (args->fi_sysfile_type) { + case BAD_BLOCK_SYSTEM_INODE: + break; + case GLOBAL_INODE_ALLOC_SYSTEM_INODE: + lockdep_set_class(&inode->i_rwsem, + &ocfs2_sysfile_lock_key[GLOBAL_INODE_ALLOC_SYSTEM_INODE]); + break; + case SLOT_MAP_SYSTEM_INODE: + lockdep_set_class(&inode->i_rwsem, + &ocfs2_sysfile_lock_key[SLOT_MAP_SYSTEM_INODE]); + break; + case HEARTBEAT_SYSTEM_INODE: + lockdep_set_class(&inode->i_rwsem, + &ocfs2_sysfile_lock_key[HEARTBEAT_SYSTEM_INODE]); + break; + case GLOBAL_BITMAP_SYSTEM_INODE: + lockdep_set_class(&inode->i_rwsem, + &ocfs2_sysfile_lock_key[GLOBAL_BITMAP_SYSTEM_INODE]); + break; + case USER_QUOTA_SYSTEM_INODE: + lockdep_set_class(&inode->i_rwsem, + &ocfs2_sysfile_lock_key[USER_QUOTA_SYSTEM_INODE]); + break; + case GROUP_QUOTA_SYSTEM_INODE: + lockdep_set_class(&inode->i_rwsem, + &ocfs2_sysfile_lock_key[GROUP_QUOTA_SYSTEM_INODE]); + break; + case ORPHAN_DIR_SYSTEM_INODE: + lockdep_set_class(&inode->i_rwsem, + &ocfs2_sysfile_lock_key[ORPHAN_DIR_SYSTEM_INODE]); + break; + case EXTENT_ALLOC_SYSTEM_INODE: + lockdep_set_class(&inode->i_rwsem, + &ocfs2_sysfile_lock_key[EXTENT_ALLOC_SYSTEM_INODE]); + break; + case INODE_ALLOC_SYSTEM_INODE: + lockdep_set_class(&inode->i_rwsem, + &ocfs2_sysfile_lock_key[INODE_ALLOC_SYSTEM_INODE]); + break; + case JOURNAL_SYSTEM_INODE: + lockdep_set_class(&inode->i_rwsem, + &ocfs2_sysfile_lock_key[JOURNAL_SYSTEM_INODE]); + break; + case LOCAL_ALLOC_SYSTEM_INODE: + lockdep_set_class(&inode->i_rwsem, + &ocfs2_sysfile_lock_key[LOCAL_ALLOC_SYSTEM_INODE]); + break; + case TRUNCATE_LOG_SYSTEM_INODE: + lockdep_set_class(&inode->i_rwsem, + &ocfs2_sysfile_lock_key[TRUNCATE_LOG_SYSTEM_INODE]); + break; + case LOCAL_USER_QUOTA_SYSTEM_INODE: + lockdep_set_class(&inode->i_rwsem, + &ocfs2_sysfile_lock_key[LOCAL_USER_QUOTA_SYSTEM_INODE]); + break; + case LOCAL_GROUP_QUOTA_SYSTEM_INODE: + lockdep_set_class(&inode->i_rwsem, + &ocfs2_sysfile_lock_key[LOCAL_GROUP_QUOTA_SYSTEM_INODE]); + break; + default: + WARN_ONCE(1, "Unknown sysfile type %d\n", args->fi_sysfile_type); + } + if (args->fi_sysfile_type == USER_QUOTA_SYSTEM_INODE || + args->fi_sysfile_type == GROUP_QUOTA_SYSTEM_INODE || + args->fi_sysfile_type == LOCAL_USER_QUOTA_SYSTEM_INODE || + args->fi_sysfile_type == LOCAL_GROUP_QUOTA_SYSTEM_INODE) + lockdep_set_class(&OCFS2_I(inode)->ip_alloc_sem, + &ocfs2_quota_ip_alloc_sem_key); + else + lockdep_set_class(&OCFS2_I(inode)->ip_alloc_sem, + &ocfs2_file_ip_alloc_sem_key); +#endif + + return 0; +} + +void ocfs2_populate_inode(struct inode *inode, struct ocfs2_dinode *fe, + int create_ino) +{ + struct super_block *sb; + struct ocfs2_super *osb; + int use_plocks = 1; + + sb = inode->i_sb; + osb = OCFS2_SB(sb); + + if ((osb->s_mount_opt & OCFS2_MOUNT_LOCALFLOCKS) || + ocfs2_mount_local(osb) || !ocfs2_stack_supports_plocks()) + use_plocks = 0; + + /* + * These have all been checked by ocfs2_read_inode_block() or set + * by ocfs2_mknod_locked(), so a failure is a code bug. + */ + BUG_ON(!OCFS2_IS_VALID_DINODE(fe)); /* This means that read_inode + cannot create a superblock + inode today. change if + that is needed. */ + BUG_ON(!(fe->i_flags & cpu_to_le32(OCFS2_VALID_FL))); + BUG_ON(le32_to_cpu(fe->i_fs_generation) != osb->fs_generation); + + + OCFS2_I(inode)->ip_clusters = le32_to_cpu(fe->i_clusters); + OCFS2_I(inode)->ip_attr = le32_to_cpu(fe->i_attr); + OCFS2_I(inode)->ip_dyn_features = le16_to_cpu(fe->i_dyn_features); + + inode_set_iversion(inode, 1); + inode->i_generation = le32_to_cpu(fe->i_generation); + inode->i_rdev = huge_decode_dev(le64_to_cpu(fe->id1.dev1.i_rdev)); + inode->i_mode = le16_to_cpu(fe->i_mode); + i_uid_write(inode, le32_to_cpu(fe->i_uid)); + i_gid_write(inode, le32_to_cpu(fe->i_gid)); + + /* Fast symlinks will have i_size but no allocated clusters. */ + if (S_ISLNK(inode->i_mode) && !fe->i_clusters) { + inode->i_blocks = 0; + inode->i_mapping->a_ops = &ocfs2_fast_symlink_aops; + } else { + inode->i_blocks = ocfs2_inode_sector_count(inode); + inode->i_mapping->a_ops = &ocfs2_aops; + } + inode_set_atime(inode, le64_to_cpu(fe->i_atime), + le32_to_cpu(fe->i_atime_nsec)); + inode_set_mtime(inode, le64_to_cpu(fe->i_mtime), + le32_to_cpu(fe->i_mtime_nsec)); + inode_set_ctime(inode, le64_to_cpu(fe->i_ctime), + le32_to_cpu(fe->i_ctime_nsec)); + + if (OCFS2_I(inode)->ip_blkno != le64_to_cpu(fe->i_blkno)) + mlog(ML_ERROR, + "ip_blkno %llu != i_blkno %llu!\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno, + (unsigned long long)le64_to_cpu(fe->i_blkno)); + + set_nlink(inode, ocfs2_read_links_count(fe)); + + trace_ocfs2_populate_inode(OCFS2_I(inode)->ip_blkno, + le32_to_cpu(fe->i_flags)); + if (fe->i_flags & cpu_to_le32(OCFS2_SYSTEM_FL)) { + OCFS2_I(inode)->ip_flags |= OCFS2_INODE_SYSTEM_FILE; + inode->i_flags |= S_NOQUOTA; + } + + if (fe->i_flags & cpu_to_le32(OCFS2_LOCAL_ALLOC_FL)) { + OCFS2_I(inode)->ip_flags |= OCFS2_INODE_BITMAP; + } else if (fe->i_flags & cpu_to_le32(OCFS2_BITMAP_FL)) { + OCFS2_I(inode)->ip_flags |= OCFS2_INODE_BITMAP; + } else if (fe->i_flags & cpu_to_le32(OCFS2_QUOTA_FL)) { + inode->i_flags |= S_NOQUOTA; + } else if (fe->i_flags & cpu_to_le32(OCFS2_SUPER_BLOCK_FL)) { + /* we can't actually hit this as read_inode can't + * handle superblocks today ;-) */ + BUG(); + } + + switch (inode->i_mode & S_IFMT) { + case S_IFREG: + if (use_plocks) + inode->i_fop = &ocfs2_fops; + else + inode->i_fop = &ocfs2_fops_no_plocks; + inode->i_op = &ocfs2_file_iops; + i_size_write(inode, le64_to_cpu(fe->i_size)); + break; + case S_IFDIR: + inode->i_op = &ocfs2_dir_iops; + if (use_plocks) + inode->i_fop = &ocfs2_dops; + else + inode->i_fop = &ocfs2_dops_no_plocks; + i_size_write(inode, le64_to_cpu(fe->i_size)); + OCFS2_I(inode)->ip_dir_lock_gen = 1; + break; + case S_IFLNK: + inode->i_op = &ocfs2_symlink_inode_operations; + inode_nohighmem(inode); + i_size_write(inode, le64_to_cpu(fe->i_size)); + break; + default: + inode->i_op = &ocfs2_special_file_iops; + init_special_inode(inode, inode->i_mode, + inode->i_rdev); + break; + } + + if (create_ino) { + inode->i_ino = ino_from_blkno(inode->i_sb, + le64_to_cpu(fe->i_blkno)); + + /* + * If we ever want to create system files from kernel, + * the generation argument to + * ocfs2_inode_lock_res_init() will have to change. + */ + BUG_ON(le32_to_cpu(fe->i_flags) & OCFS2_SYSTEM_FL); + + ocfs2_inode_lock_res_init(&OCFS2_I(inode)->ip_inode_lockres, + OCFS2_LOCK_TYPE_META, 0, inode); + + ocfs2_inode_lock_res_init(&OCFS2_I(inode)->ip_open_lockres, + OCFS2_LOCK_TYPE_OPEN, 0, inode); + } + + ocfs2_inode_lock_res_init(&OCFS2_I(inode)->ip_rw_lockres, + OCFS2_LOCK_TYPE_RW, inode->i_generation, + inode); + + ocfs2_set_inode_flags(inode); + + OCFS2_I(inode)->ip_last_used_slot = 0; + OCFS2_I(inode)->ip_last_used_group = 0; + + if (S_ISDIR(inode->i_mode)) + ocfs2_resv_set_type(&OCFS2_I(inode)->ip_la_data_resv, + OCFS2_RESV_FLAG_DIR); +} + +static int ocfs2_read_locked_inode(struct inode *inode, + struct ocfs2_find_inode_args *args) +{ + struct super_block *sb; + struct ocfs2_super *osb; + struct ocfs2_dinode *fe; + struct buffer_head *bh = NULL; + int status, can_lock, lock_level = 0; + u32 generation = 0; + + status = -EINVAL; + sb = inode->i_sb; + osb = OCFS2_SB(sb); + + /* + * To improve performance of cold-cache inode stats, we take + * the cluster lock here if possible. + * + * Generally, OCFS2 never trusts the contents of an inode + * unless it's holding a cluster lock, so taking it here isn't + * a correctness issue as much as it is a performance + * improvement. + * + * There are three times when taking the lock is not a good idea: + * + * 1) During startup, before we have initialized the DLM. + * + * 2) If we are reading certain system files which never get + * cluster locks (local alloc, truncate log). + * + * 3) If the process doing the iget() is responsible for + * orphan dir recovery. We're holding the orphan dir lock and + * can get into a deadlock with another process on another + * node in ->delete_inode(). + * + * #1 and #2 can be simply solved by never taking the lock + * here for system files (which are the only type we read + * during mount). It's a heavier approach, but our main + * concern is user-accessible files anyway. + * + * #3 works itself out because we'll eventually take the + * cluster lock before trusting anything anyway. + */ + can_lock = !(args->fi_flags & OCFS2_FI_FLAG_SYSFILE) + && !(args->fi_flags & OCFS2_FI_FLAG_ORPHAN_RECOVERY) + && !ocfs2_mount_local(osb); + + trace_ocfs2_read_locked_inode( + (unsigned long long)OCFS2_I(inode)->ip_blkno, can_lock); + + /* + * To maintain backwards compatibility with older versions of + * ocfs2-tools, we still store the generation value for system + * files. The only ones that actually matter to userspace are + * the journals, but it's easier and inexpensive to just flag + * all system files similarly. + */ + if (args->fi_flags & OCFS2_FI_FLAG_SYSFILE) + generation = osb->fs_generation; + + ocfs2_inode_lock_res_init(&OCFS2_I(inode)->ip_inode_lockres, + OCFS2_LOCK_TYPE_META, + generation, inode); + + ocfs2_inode_lock_res_init(&OCFS2_I(inode)->ip_open_lockres, + OCFS2_LOCK_TYPE_OPEN, + 0, inode); + + if (can_lock) { + status = ocfs2_open_lock(inode); + if (status) { + make_bad_inode(inode); + mlog_errno(status); + return status; + } + status = ocfs2_inode_lock(inode, NULL, lock_level); + if (status) { + make_bad_inode(inode); + mlog_errno(status); + return status; + } + } + + if (args->fi_flags & OCFS2_FI_FLAG_ORPHAN_RECOVERY) { + status = ocfs2_try_open_lock(inode, 0); + if (status) { + make_bad_inode(inode); + return status; + } + } + + if (can_lock) { + if (args->fi_flags & OCFS2_FI_FLAG_FILECHECK_CHK) + status = ocfs2_filecheck_read_inode_block_full(inode, + &bh, OCFS2_BH_IGNORE_CACHE, 0); + else if (args->fi_flags & OCFS2_FI_FLAG_FILECHECK_FIX) + status = ocfs2_filecheck_read_inode_block_full(inode, + &bh, OCFS2_BH_IGNORE_CACHE, 1); + else + status = ocfs2_read_inode_block_full(inode, + &bh, OCFS2_BH_IGNORE_CACHE); + } else { + status = ocfs2_read_blocks_sync(osb, args->fi_blkno, 1, &bh); + /* + * If buffer is in jbd, then its checksum may not have been + * computed as yet. + */ + if (!status && !buffer_jbd(bh)) { + if (args->fi_flags & OCFS2_FI_FLAG_FILECHECK_CHK) + status = ocfs2_filecheck_validate_inode_block( + osb->sb, bh); + else if (args->fi_flags & OCFS2_FI_FLAG_FILECHECK_FIX) + status = ocfs2_filecheck_repair_inode_block( + osb->sb, bh); + else + status = ocfs2_validate_inode_block( + osb->sb, bh); + } + } + if (status < 0) { + mlog_errno(status); + goto bail; + } + + status = -EINVAL; + fe = (struct ocfs2_dinode *) bh->b_data; + + /* + * This is a code bug. Right now the caller needs to + * understand whether it is asking for a system file inode or + * not so the proper lock names can be built. + */ + mlog_bug_on_msg(!!(fe->i_flags & cpu_to_le32(OCFS2_SYSTEM_FL)) != + !!(args->fi_flags & OCFS2_FI_FLAG_SYSFILE), + "Inode %llu: system file state is ambiguous\n", + (unsigned long long)args->fi_blkno); + + if (S_ISCHR(le16_to_cpu(fe->i_mode)) || + S_ISBLK(le16_to_cpu(fe->i_mode))) + inode->i_rdev = huge_decode_dev(le64_to_cpu(fe->id1.dev1.i_rdev)); + + ocfs2_populate_inode(inode, fe, 0); + + BUG_ON(args->fi_blkno != le64_to_cpu(fe->i_blkno)); + + if (buffer_dirty(bh) && !buffer_jbd(bh)) { + if (can_lock) { + ocfs2_inode_unlock(inode, lock_level); + lock_level = 1; + ocfs2_inode_lock(inode, NULL, lock_level); + } + status = ocfs2_write_block(osb, bh, INODE_CACHE(inode)); + if (status < 0) { + mlog_errno(status); + goto bail; + } + } + + status = 0; + +bail: + if (can_lock) + ocfs2_inode_unlock(inode, lock_level); + + if (status < 0) + make_bad_inode(inode); + + brelse(bh); + + return status; +} + +void ocfs2_sync_blockdev(struct super_block *sb) +{ + sync_blockdev(sb->s_bdev); +} + +static int ocfs2_truncate_for_delete(struct ocfs2_super *osb, + struct inode *inode, + struct buffer_head *fe_bh) +{ + int status = 0; + struct ocfs2_dinode *fe; + handle_t *handle = NULL; + + fe = (struct ocfs2_dinode *) fe_bh->b_data; + + /* + * This check will also skip truncate of inodes with inline + * data and fast symlinks. + */ + if (fe->i_clusters) { + if (ocfs2_should_order_data(inode)) + ocfs2_begin_ordered_truncate(inode, 0); + + handle = ocfs2_start_trans(osb, OCFS2_INODE_UPDATE_CREDITS); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + handle = NULL; + mlog_errno(status); + goto out; + } + + status = ocfs2_journal_access_di(handle, INODE_CACHE(inode), + fe_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto out; + } + + i_size_write(inode, 0); + + status = ocfs2_mark_inode_dirty(handle, inode, fe_bh); + if (status < 0) { + mlog_errno(status); + goto out; + } + + ocfs2_commit_trans(osb, handle); + handle = NULL; + + status = ocfs2_commit_truncate(osb, inode, fe_bh); + if (status < 0) + mlog_errno(status); + } + +out: + if (handle) + ocfs2_commit_trans(osb, handle); + return status; +} + +static int ocfs2_remove_inode(struct inode *inode, + struct buffer_head *di_bh, + struct inode *orphan_dir_inode, + struct buffer_head *orphan_dir_bh) +{ + int status; + struct inode *inode_alloc_inode = NULL; + struct buffer_head *inode_alloc_bh = NULL; + handle_t *handle; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct ocfs2_dinode *di = (struct ocfs2_dinode *) di_bh->b_data; + + inode_alloc_inode = + ocfs2_get_system_file_inode(osb, INODE_ALLOC_SYSTEM_INODE, + le16_to_cpu(di->i_suballoc_slot)); + if (!inode_alloc_inode) { + status = -ENOENT; + mlog_errno(status); + goto bail; + } + + inode_lock(inode_alloc_inode); + status = ocfs2_inode_lock(inode_alloc_inode, &inode_alloc_bh, 1); + if (status < 0) { + inode_unlock(inode_alloc_inode); + + mlog_errno(status); + goto bail; + } + + handle = ocfs2_start_trans(osb, OCFS2_DELETE_INODE_CREDITS + + ocfs2_quota_trans_credits(inode->i_sb)); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + mlog_errno(status); + goto bail_unlock; + } + + if (!(OCFS2_I(inode)->ip_flags & OCFS2_INODE_SKIP_ORPHAN_DIR)) { + status = ocfs2_orphan_del(osb, handle, orphan_dir_inode, inode, + orphan_dir_bh, false); + if (status < 0) { + mlog_errno(status); + goto bail_commit; + } + } + + /* set the inodes dtime */ + status = ocfs2_journal_access_di(handle, INODE_CACHE(inode), di_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto bail_commit; + } + + di->i_dtime = cpu_to_le64(ktime_get_real_seconds()); + di->i_flags &= cpu_to_le32(~(OCFS2_VALID_FL | OCFS2_ORPHANED_FL)); + ocfs2_journal_dirty(handle, di_bh); + + ocfs2_remove_from_cache(INODE_CACHE(inode), di_bh); + dquot_free_inode(inode); + + status = ocfs2_free_dinode(handle, inode_alloc_inode, + inode_alloc_bh, di); + if (status < 0) + mlog_errno(status); + +bail_commit: + ocfs2_commit_trans(osb, handle); +bail_unlock: + ocfs2_inode_unlock(inode_alloc_inode, 1); + inode_unlock(inode_alloc_inode); + brelse(inode_alloc_bh); +bail: + iput(inode_alloc_inode); + + return status; +} + +/* + * Serialize with orphan dir recovery. If the process doing + * recovery on this orphan dir does an iget() with the dir + * i_rwsem held, we'll deadlock here. Instead we detect this + * and exit early - recovery will wipe this inode for us. + */ +static int ocfs2_check_orphan_recovery_state(struct ocfs2_super *osb, + int slot) +{ + int ret = 0; + + spin_lock(&osb->osb_lock); + if (ocfs2_node_map_test_bit(osb, &osb->osb_recovering_orphan_dirs, slot)) { + ret = -EDEADLK; + goto out; + } + /* This signals to the orphan recovery process that it should + * wait for us to handle the wipe. */ + osb->osb_orphan_wipes[slot]++; +out: + spin_unlock(&osb->osb_lock); + trace_ocfs2_check_orphan_recovery_state(slot, ret); + return ret; +} + +static void ocfs2_signal_wipe_completion(struct ocfs2_super *osb, + int slot) +{ + spin_lock(&osb->osb_lock); + osb->osb_orphan_wipes[slot]--; + spin_unlock(&osb->osb_lock); + + wake_up(&osb->osb_wipe_event); +} + +static int ocfs2_wipe_inode(struct inode *inode, + struct buffer_head *di_bh) +{ + int status, orphaned_slot = -1; + struct inode *orphan_dir_inode = NULL; + struct buffer_head *orphan_dir_bh = NULL; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct ocfs2_dinode *di = (struct ocfs2_dinode *) di_bh->b_data; + + if (!(OCFS2_I(inode)->ip_flags & OCFS2_INODE_SKIP_ORPHAN_DIR)) { + orphaned_slot = le16_to_cpu(di->i_orphaned_slot); + + status = ocfs2_check_orphan_recovery_state(osb, orphaned_slot); + if (status) + return status; + + orphan_dir_inode = ocfs2_get_system_file_inode(osb, + ORPHAN_DIR_SYSTEM_INODE, + orphaned_slot); + if (!orphan_dir_inode) { + status = -ENOENT; + mlog_errno(status); + goto bail; + } + + /* Lock the orphan dir. The lock will be held for the entire + * delete_inode operation. We do this now to avoid races with + * recovery completion on other nodes. */ + inode_lock(orphan_dir_inode); + status = ocfs2_inode_lock(orphan_dir_inode, &orphan_dir_bh, 1); + if (status < 0) { + inode_unlock(orphan_dir_inode); + + mlog_errno(status); + goto bail; + } + } + + /* we do this while holding the orphan dir lock because we + * don't want recovery being run from another node to try an + * inode delete underneath us -- this will result in two nodes + * truncating the same file! */ + status = ocfs2_truncate_for_delete(osb, inode, di_bh); + if (status < 0) { + mlog_errno(status); + goto bail_unlock_dir; + } + + /* Remove any dir index tree */ + if (S_ISDIR(inode->i_mode)) { + status = ocfs2_dx_dir_truncate(inode, di_bh); + if (status) { + mlog_errno(status); + goto bail_unlock_dir; + } + } + + /*Free extended attribute resources associated with this inode.*/ + status = ocfs2_xattr_remove(inode, di_bh); + if (status < 0) { + mlog_errno(status); + goto bail_unlock_dir; + } + + status = ocfs2_remove_refcount_tree(inode, di_bh); + if (status < 0) { + mlog_errno(status); + goto bail_unlock_dir; + } + + status = ocfs2_remove_inode(inode, di_bh, orphan_dir_inode, + orphan_dir_bh); + if (status < 0) + mlog_errno(status); + +bail_unlock_dir: + if (OCFS2_I(inode)->ip_flags & OCFS2_INODE_SKIP_ORPHAN_DIR) + return status; + + ocfs2_inode_unlock(orphan_dir_inode, 1); + inode_unlock(orphan_dir_inode); + brelse(orphan_dir_bh); +bail: + iput(orphan_dir_inode); + ocfs2_signal_wipe_completion(osb, orphaned_slot); + + return status; +} + +/* There is a series of simple checks that should be done before a + * trylock is even considered. Encapsulate those in this function. */ +static int ocfs2_inode_is_valid_to_delete(struct inode *inode) +{ + int ret = 0; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + trace_ocfs2_inode_is_valid_to_delete(current, osb->dc_task, + (unsigned long long)oi->ip_blkno, + oi->ip_flags); + + /* We shouldn't be getting here for the root directory + * inode.. */ + if (inode == osb->root_inode) { + mlog(ML_ERROR, "Skipping delete of root inode.\n"); + goto bail; + } + + /* + * If we're coming from downconvert_thread we can't go into our own + * voting [hello, deadlock city!] so we cannot delete the inode. But + * since we dropped last inode ref when downconverting dentry lock, + * we cannot have the file open and thus the node doing unlink will + * take care of deleting the inode. + */ + if (current == osb->dc_task) + goto bail; + + spin_lock(&oi->ip_lock); + /* OCFS2 *never* deletes system files. This should technically + * never get here as system file inodes should always have a + * positive link count. */ + if (oi->ip_flags & OCFS2_INODE_SYSTEM_FILE) { + mlog(ML_ERROR, "Skipping delete of system file %llu\n", + (unsigned long long)oi->ip_blkno); + goto bail_unlock; + } + + ret = 1; +bail_unlock: + spin_unlock(&oi->ip_lock); +bail: + return ret; +} + +/* Query the cluster to determine whether we should wipe an inode from + * disk or not. + * + * Requires the inode to have the cluster lock. */ +static int ocfs2_query_inode_wipe(struct inode *inode, + struct buffer_head *di_bh, + int *wipe) +{ + int status = 0, reason = 0; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_dinode *di; + + *wipe = 0; + + trace_ocfs2_query_inode_wipe_begin((unsigned long long)oi->ip_blkno, + inode->i_nlink); + + /* While we were waiting for the cluster lock in + * ocfs2_delete_inode, another node might have asked to delete + * the inode. Recheck our flags to catch this. */ + if (!ocfs2_inode_is_valid_to_delete(inode)) { + reason = 1; + goto bail; + } + + /* Now that we have an up to date inode, we can double check + * the link count. */ + if (inode->i_nlink) + goto bail; + + /* Do some basic inode verification... */ + di = (struct ocfs2_dinode *) di_bh->b_data; + if (!(di->i_flags & cpu_to_le32(OCFS2_ORPHANED_FL)) && + !(oi->ip_flags & OCFS2_INODE_SKIP_ORPHAN_DIR)) { + /* + * Inodes in the orphan dir must have ORPHANED_FL. The only + * inodes that come back out of the orphan dir are reflink + * targets. A reflink target may be moved out of the orphan + * dir between the time we scan the directory and the time we + * process it. This would lead to HAS_REFCOUNT_FL being set but + * ORPHANED_FL not. + */ + if (di->i_dyn_features & cpu_to_le16(OCFS2_HAS_REFCOUNT_FL)) { + reason = 2; + goto bail; + } + + /* for lack of a better error? */ + status = -EEXIST; + mlog(ML_ERROR, + "Inode %llu (on-disk %llu) not orphaned! " + "Disk flags 0x%x, inode flags 0x%x\n", + (unsigned long long)oi->ip_blkno, + (unsigned long long)le64_to_cpu(di->i_blkno), + le32_to_cpu(di->i_flags), oi->ip_flags); + goto bail; + } + + /* has someone already deleted us?! baaad... */ + if (di->i_dtime) { + status = -EEXIST; + mlog_errno(status); + goto bail; + } + + /* + * This is how ocfs2 determines whether an inode is still live + * within the cluster. Every node takes a shared read lock on + * the inode open lock in ocfs2_read_locked_inode(). When we + * get to ->delete_inode(), each node tries to convert it's + * lock to an exclusive. Trylocks are serialized by the inode + * meta data lock. If the upconvert succeeds, we know the inode + * is no longer live and can be deleted. + * + * Though we call this with the meta data lock held, the + * trylock keeps us from ABBA deadlock. + */ + status = ocfs2_try_open_lock(inode, 1); + if (status == -EAGAIN) { + status = 0; + reason = 3; + goto bail; + } + if (status < 0) { + mlog_errno(status); + goto bail; + } + + *wipe = 1; + trace_ocfs2_query_inode_wipe_succ(le16_to_cpu(di->i_orphaned_slot)); + +bail: + trace_ocfs2_query_inode_wipe_end(status, reason); + return status; +} + +/* Support function for ocfs2_delete_inode. Will help us keep the + * inode data in a consistent state for clear_inode. Always truncates + * pages, optionally sync's them first. */ +static void ocfs2_cleanup_delete_inode(struct inode *inode, + int sync_data) +{ + trace_ocfs2_cleanup_delete_inode( + (unsigned long long)OCFS2_I(inode)->ip_blkno, sync_data); + if (sync_data) + filemap_write_and_wait(inode->i_mapping); + truncate_inode_pages_final(&inode->i_data); +} + +static void ocfs2_delete_inode(struct inode *inode) +{ + int wipe, status; + sigset_t oldset; + struct buffer_head *di_bh = NULL; + struct ocfs2_dinode *di = NULL; + + trace_ocfs2_delete_inode(inode->i_ino, + (unsigned long long)OCFS2_I(inode)->ip_blkno, + is_bad_inode(inode)); + + /* When we fail in read_inode() we mark inode as bad. The second test + * catches the case when inode allocation fails before allocating + * a block for inode. */ + if (is_bad_inode(inode) || !OCFS2_I(inode)->ip_blkno) + goto bail; + + if (!ocfs2_inode_is_valid_to_delete(inode)) { + /* It's probably not necessary to truncate_inode_pages + * here but we do it for safety anyway (it will most + * likely be a no-op anyway) */ + ocfs2_cleanup_delete_inode(inode, 0); + goto bail; + } + + dquot_initialize(inode); + + /* We want to block signals in delete_inode as the lock and + * messaging paths may return us -ERESTARTSYS. Which would + * cause us to exit early, resulting in inodes being orphaned + * forever. */ + ocfs2_block_signals(&oldset); + + /* + * Synchronize us against ocfs2_get_dentry. We take this in + * shared mode so that all nodes can still concurrently + * process deletes. + */ + status = ocfs2_nfs_sync_lock(OCFS2_SB(inode->i_sb), 0); + if (status < 0) { + mlog(ML_ERROR, "getting nfs sync lock(PR) failed %d\n", status); + ocfs2_cleanup_delete_inode(inode, 0); + goto bail_unblock; + } + /* Lock down the inode. This gives us an up to date view of + * it's metadata (for verification), and allows us to + * serialize delete_inode on multiple nodes. + * + * Even though we might be doing a truncate, we don't take the + * allocation lock here as it won't be needed - nobody will + * have the file open. + */ + status = ocfs2_inode_lock(inode, &di_bh, 1); + if (status < 0) { + if (status != -ENOENT) + mlog_errno(status); + ocfs2_cleanup_delete_inode(inode, 0); + goto bail_unlock_nfs_sync; + } + + di = (struct ocfs2_dinode *)di_bh->b_data; + /* Skip inode deletion and wait for dio orphan entry recovered + * first */ + if (unlikely(di->i_flags & cpu_to_le32(OCFS2_DIO_ORPHANED_FL))) { + ocfs2_cleanup_delete_inode(inode, 0); + goto bail_unlock_inode; + } + + /* Query the cluster. This will be the final decision made + * before we go ahead and wipe the inode. */ + status = ocfs2_query_inode_wipe(inode, di_bh, &wipe); + if (!wipe || status < 0) { + /* Error and remote inode busy both mean we won't be + * removing the inode, so they take almost the same + * path. */ + if (status < 0) + mlog_errno(status); + + /* Someone in the cluster has disallowed a wipe of + * this inode, or it was never completely + * orphaned. Write out the pages and exit now. */ + ocfs2_cleanup_delete_inode(inode, 1); + goto bail_unlock_inode; + } + + ocfs2_cleanup_delete_inode(inode, 0); + + status = ocfs2_wipe_inode(inode, di_bh); + if (status < 0) { + if (status != -EDEADLK) + mlog_errno(status); + goto bail_unlock_inode; + } + + /* + * Mark the inode as successfully deleted. + * + * This is important for ocfs2_clear_inode() as it will check + * this flag and skip any checkpointing work + * + * ocfs2_stuff_meta_lvb() also uses this flag to invalidate + * the LVB for other nodes. + */ + OCFS2_I(inode)->ip_flags |= OCFS2_INODE_DELETED; + +bail_unlock_inode: + ocfs2_inode_unlock(inode, 1); + brelse(di_bh); + +bail_unlock_nfs_sync: + ocfs2_nfs_sync_unlock(OCFS2_SB(inode->i_sb), 0); + +bail_unblock: + ocfs2_unblock_signals(&oldset); +bail: + return; +} + +static void ocfs2_clear_inode(struct inode *inode) +{ + int status; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + clear_inode(inode); + trace_ocfs2_clear_inode((unsigned long long)oi->ip_blkno, + inode->i_nlink); + + mlog_bug_on_msg(osb == NULL, + "Inode=%llu\n", inode->i_ino); + + dquot_drop(inode); + + /* To prevent remote deletes we hold open lock before, now it + * is time to unlock PR and EX open locks. */ + ocfs2_open_unlock(inode); + + /* Do these before all the other work so that we don't bounce + * the downconvert thread while waiting to destroy the locks. */ + ocfs2_mark_lockres_freeing(osb, &oi->ip_rw_lockres); + ocfs2_mark_lockres_freeing(osb, &oi->ip_inode_lockres); + ocfs2_mark_lockres_freeing(osb, &oi->ip_open_lockres); + + ocfs2_resv_discard(&osb->osb_la_resmap, + &oi->ip_la_data_resv); + ocfs2_resv_init_once(&oi->ip_la_data_resv); + + /* We very well may get a clear_inode before all an inodes + * metadata has hit disk. Of course, we can't drop any cluster + * locks until the journal has finished with it. The only + * exception here are successfully wiped inodes - their + * metadata can now be considered to be part of the system + * inodes from which it came. */ + if (!(oi->ip_flags & OCFS2_INODE_DELETED)) + ocfs2_checkpoint_inode(inode); + + mlog_bug_on_msg(!list_empty(&oi->ip_io_markers), + "Clear inode of %llu, inode has io markers\n", + (unsigned long long)oi->ip_blkno); + mlog_bug_on_msg(!list_empty(&oi->ip_unwritten_list), + "Clear inode of %llu, inode has unwritten extents\n", + (unsigned long long)oi->ip_blkno); + + ocfs2_extent_map_trunc(inode, 0); + + status = ocfs2_drop_inode_locks(inode); + if (status < 0) + mlog_errno(status); + + ocfs2_lock_res_free(&oi->ip_rw_lockres); + ocfs2_lock_res_free(&oi->ip_inode_lockres); + ocfs2_lock_res_free(&oi->ip_open_lockres); + + ocfs2_metadata_cache_exit(INODE_CACHE(inode)); + + mlog_bug_on_msg(INODE_CACHE(inode)->ci_num_cached, + "Clear inode of %llu, inode has %u cache items\n", + (unsigned long long)oi->ip_blkno, + INODE_CACHE(inode)->ci_num_cached); + + mlog_bug_on_msg(!(INODE_CACHE(inode)->ci_flags & OCFS2_CACHE_FL_INLINE), + "Clear inode of %llu, inode has a bad flag\n", + (unsigned long long)oi->ip_blkno); + + mlog_bug_on_msg(spin_is_locked(&oi->ip_lock), + "Clear inode of %llu, inode is locked\n", + (unsigned long long)oi->ip_blkno); + + mlog_bug_on_msg(!mutex_trylock(&oi->ip_io_mutex), + "Clear inode of %llu, io_mutex is locked\n", + (unsigned long long)oi->ip_blkno); + mutex_unlock(&oi->ip_io_mutex); + + /* + * down_trylock() returns 0, down_write_trylock() returns 1 + * kernel 1, world 0 + */ + mlog_bug_on_msg(!down_write_trylock(&oi->ip_alloc_sem), + "Clear inode of %llu, alloc_sem is locked\n", + (unsigned long long)oi->ip_blkno); + up_write(&oi->ip_alloc_sem); + + mlog_bug_on_msg(oi->ip_open_count, + "Clear inode of %llu has open count %d\n", + (unsigned long long)oi->ip_blkno, oi->ip_open_count); + + /* Clear all other flags. */ + oi->ip_flags = 0; + oi->ip_dir_start_lookup = 0; + oi->ip_blkno = 0ULL; + + /* + * ip_jinode is used to track txns against this inode. We ensure that + * the journal is flushed before journal shutdown. Thus it is safe to + * have inodes get cleaned up after journal shutdown. + */ + if (!osb->journal) + return; + + jbd2_journal_release_jbd_inode(osb->journal->j_journal, + &oi->ip_jinode); +} + +void ocfs2_evict_inode(struct inode *inode) +{ + write_inode_now(inode, 1); + + if (!inode->i_nlink || + (OCFS2_I(inode)->ip_flags & OCFS2_INODE_MAYBE_ORPHANED)) { + ocfs2_delete_inode(inode); + } else { + truncate_inode_pages_final(&inode->i_data); + } + ocfs2_clear_inode(inode); +} + +/* + * This is called from our getattr. + */ +int ocfs2_inode_revalidate(struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + int status = 0; + + trace_ocfs2_inode_revalidate(inode, + inode ? (unsigned long long)OCFS2_I(inode)->ip_blkno : 0ULL, + inode ? (unsigned long long)OCFS2_I(inode)->ip_flags : 0); + + if (!inode) { + status = -ENOENT; + goto bail; + } + + spin_lock(&OCFS2_I(inode)->ip_lock); + if (OCFS2_I(inode)->ip_flags & OCFS2_INODE_DELETED) { + spin_unlock(&OCFS2_I(inode)->ip_lock); + status = -ENOENT; + goto bail; + } + spin_unlock(&OCFS2_I(inode)->ip_lock); + + /* Let ocfs2_inode_lock do the work of updating our struct + * inode for us. */ + status = ocfs2_inode_lock(inode, NULL, 0); + if (status < 0) { + if (status != -ENOENT) + mlog_errno(status); + goto bail; + } + ocfs2_inode_unlock(inode, 0); +bail: + return status; +} + +/* + * Updates a disk inode from a + * struct inode. + * Only takes ip_lock. + */ +int ocfs2_mark_inode_dirty(handle_t *handle, + struct inode *inode, + struct buffer_head *bh) +{ + int status; + struct ocfs2_dinode *fe = (struct ocfs2_dinode *) bh->b_data; + + trace_ocfs2_mark_inode_dirty((unsigned long long)OCFS2_I(inode)->ip_blkno); + + status = ocfs2_journal_access_di(handle, INODE_CACHE(inode), bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto leave; + } + + spin_lock(&OCFS2_I(inode)->ip_lock); + fe->i_clusters = cpu_to_le32(OCFS2_I(inode)->ip_clusters); + ocfs2_get_inode_flags(OCFS2_I(inode)); + fe->i_attr = cpu_to_le32(OCFS2_I(inode)->ip_attr); + fe->i_dyn_features = cpu_to_le16(OCFS2_I(inode)->ip_dyn_features); + spin_unlock(&OCFS2_I(inode)->ip_lock); + + fe->i_size = cpu_to_le64(i_size_read(inode)); + ocfs2_set_links_count(fe, inode->i_nlink); + fe->i_uid = cpu_to_le32(i_uid_read(inode)); + fe->i_gid = cpu_to_le32(i_gid_read(inode)); + fe->i_mode = cpu_to_le16(inode->i_mode); + fe->i_atime = cpu_to_le64(inode_get_atime_sec(inode)); + fe->i_atime_nsec = cpu_to_le32(inode_get_atime_nsec(inode)); + fe->i_ctime = cpu_to_le64(inode_get_ctime_sec(inode)); + fe->i_ctime_nsec = cpu_to_le32(inode_get_ctime_nsec(inode)); + fe->i_mtime = cpu_to_le64(inode_get_mtime_sec(inode)); + fe->i_mtime_nsec = cpu_to_le32(inode_get_mtime_nsec(inode)); + + ocfs2_journal_dirty(handle, bh); + ocfs2_update_inode_fsync_trans(handle, inode, 1); +leave: + return status; +} + +/* + * + * Updates a struct inode from a disk inode. + * does no i/o, only takes ip_lock. + */ +void ocfs2_refresh_inode(struct inode *inode, + struct ocfs2_dinode *fe) +{ + spin_lock(&OCFS2_I(inode)->ip_lock); + + OCFS2_I(inode)->ip_clusters = le32_to_cpu(fe->i_clusters); + OCFS2_I(inode)->ip_attr = le32_to_cpu(fe->i_attr); + OCFS2_I(inode)->ip_dyn_features = le16_to_cpu(fe->i_dyn_features); + ocfs2_set_inode_flags(inode); + i_size_write(inode, le64_to_cpu(fe->i_size)); + set_nlink(inode, ocfs2_read_links_count(fe)); + i_uid_write(inode, le32_to_cpu(fe->i_uid)); + i_gid_write(inode, le32_to_cpu(fe->i_gid)); + inode->i_mode = le16_to_cpu(fe->i_mode); + if (S_ISLNK(inode->i_mode) && le32_to_cpu(fe->i_clusters) == 0) + inode->i_blocks = 0; + else + inode->i_blocks = ocfs2_inode_sector_count(inode); + inode_set_atime(inode, le64_to_cpu(fe->i_atime), + le32_to_cpu(fe->i_atime_nsec)); + inode_set_mtime(inode, le64_to_cpu(fe->i_mtime), + le32_to_cpu(fe->i_mtime_nsec)); + inode_set_ctime(inode, le64_to_cpu(fe->i_ctime), + le32_to_cpu(fe->i_ctime_nsec)); + + spin_unlock(&OCFS2_I(inode)->ip_lock); +} + +int ocfs2_validate_inode_block(struct super_block *sb, + struct buffer_head *bh) +{ + int rc; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)bh->b_data; + + trace_ocfs2_validate_inode_block((unsigned long long)bh->b_blocknr); + + BUG_ON(!buffer_uptodate(bh)); + + /* + * If the ecc fails, we return the error but otherwise + * leave the filesystem running. We know any error is + * local to this block. + */ + rc = ocfs2_validate_meta_ecc(sb, bh->b_data, &di->i_check); + if (rc) { + mlog(ML_ERROR, "Checksum failed for dinode %llu\n", + (unsigned long long)bh->b_blocknr); + goto bail; + } + + if ((!di->i_links_count && !di->i_links_count_hi) || !di->i_mode) { + mlog(ML_ERROR, "Invalid dinode #%llu: " + "Corrupt state (nlink = %u or mode = %u) detected!\n", + (unsigned long long)bh->b_blocknr, + ocfs2_read_links_count(di), le16_to_cpu(di->i_mode)); + rc = -EFSCORRUPTED; + goto bail; + } + /* + * Errors after here are fatal. + */ + + rc = -EINVAL; + + if (!OCFS2_IS_VALID_DINODE(di)) { + rc = ocfs2_error(sb, "Invalid dinode #%llu: signature = %.*s\n", + (unsigned long long)bh->b_blocknr, 7, + di->i_signature); + goto bail; + } + + if (le64_to_cpu(di->i_blkno) != bh->b_blocknr) { + rc = ocfs2_error(sb, "Invalid dinode #%llu: i_blkno is %llu\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(di->i_blkno)); + goto bail; + } + + if (!(le32_to_cpu(di->i_flags) & OCFS2_VALID_FL)) { + rc = ocfs2_error(sb, + "Invalid dinode #%llu: OCFS2_VALID_FL not set\n", + (unsigned long long)bh->b_blocknr); + goto bail; + } + + if (le32_to_cpu(di->i_fs_generation) != + OCFS2_SB(sb)->fs_generation) { + rc = ocfs2_error(sb, + "Invalid dinode #%llu: fs_generation is %u\n", + (unsigned long long)bh->b_blocknr, + le32_to_cpu(di->i_fs_generation)); + goto bail; + } + + if (le16_to_cpu(di->i_suballoc_slot) != (u16)OCFS2_INVALID_SLOT && + (u32)le16_to_cpu(di->i_suballoc_slot) > OCFS2_SB(sb)->max_slots - 1) { + rc = ocfs2_error(sb, "Invalid dinode %llu: suballoc slot %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(di->i_suballoc_slot)); + goto bail; + } + + if ((le32_to_cpu(di->i_flags) & OCFS2_ORPHANED_FL) && + le16_to_cpu(di->i_orphaned_slot) >= OCFS2_SB(sb)->max_slots) { + rc = ocfs2_error(sb, "Invalid dinode %llu: orphaned slot %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(di->i_orphaned_slot)); + goto bail; + } + + if ((le32_to_cpu(di->i_flags) & OCFS2_DIO_ORPHANED_FL) && + le16_to_cpu(di->i_dio_orphaned_slot) >= OCFS2_SB(sb)->max_slots) { + rc = ocfs2_error(sb, "Invalid dinode %llu: DIO orphaned slot %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(di->i_dio_orphaned_slot)); + goto bail; + } + + /* + * Reject dinodes whose i_mode does not name one of the seven + * canonical POSIX file types. ocfs2_populate_inode() copies + * i_mode verbatim into inode->i_mode and then dispatches via + * switch (mode & S_IFMT) to file/dir/symlink/special_file iops; + * an unrecognised type falls into ocfs2_special_file_iops with + * init_special_inode(), which interprets i_rdev. Constrain the + * type here so the dispatch only ever sees a value mkfs.ocfs2 / + * VFS can produce. + */ + if (!ocfs2_valid_inode_mode(le16_to_cpu(di->i_mode))) { + rc = ocfs2_error(sb, + "Invalid dinode #%llu: mode 0%o has unknown file type\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(di->i_mode)); + goto bail; + } + + /* + * id1.dev1.i_rdev is the device-number arm of the id1 union and + * is only meaningful for character and block device inodes. For + * any other regular user-visible file type the on-disk value + * must be zero. ocfs2_populate_inode() currently runs + * + * inode->i_rdev = huge_decode_dev(le64_to_cpu(fe->id1.dev1.i_rdev)); + * + * unconditionally, before the S_IFMT switch decides whether the + * inode is a special file. As a result, an i_rdev value present + * on a non-device inode is silently published into the in-core + * inode; a subsequent forced re-read or in-core mode mutation + * (cluster peer with raw write access to the shared LUN, + * on-disk corruption, or a separately forged dinode) can then + * expose the attacker-controlled device number to + * init_special_inode() without ever showing an unusual i_mode + * at validation time. + * + * System inodes (OCFS2_SYSTEM_FL) legitimately use the bitmap1 + * and journal1 arms of the same union (allocator i_used / + * i_total counters and the journal ij_flags / + * ij_recovery_generation pair); those bytes are not an i_rdev + * and must not be checked here. Restrict the cross-check to + * non-system inodes, which is the full attacker-controllable + * surface. + */ + if (ocfs2_dinode_has_unexpected_rdev(di)) { + rc = ocfs2_error(sb, + "Invalid dinode #%llu: non-device mode 0%o with i_rdev %llu\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(di->i_mode), + (unsigned long long)le64_to_cpu(di->id1.dev1.i_rdev)); + goto bail; + } + + /* + * Non-inline directories must not have i_size without allocated + * clusters: directory growth adds storage before advancing i_size, + * and readdir walks i_size block-by-block. A forged directory + * with zero clusters and a huge i_size would repeatedly fault on + * holes while advancing through the claimed size. + * + * Non-inline regular files have the same invariant on non-sparse + * volumes. Sparse regular files are different: truncate can + * legitimately grow i_size without allocating clusters, so keep + * the sparse-alloc carveout for S_IFREG only. System inodes and + * inline-data dinodes have their own storage rules. + */ + if (ocfs2_dinode_has_size_without_clusters(sb, di)) { + if (S_ISDIR(le16_to_cpu(di->i_mode))) + rc = ocfs2_error(sb, + "Invalid dinode #%llu: directory i_size %llu with i_clusters 0 and no inline-data flag\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(di->i_size)); + else + rc = ocfs2_error(sb, + "Invalid dinode #%llu: regular file i_size %llu with i_clusters 0 and no inline-data flag on non-sparse volume\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(di->i_size)); + goto bail; + } + + rc = ocfs2_validate_inode_xattr(sb, bh->b_blocknr, di); + if (rc) + goto bail; + + if (le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_DATA_FL) { + struct ocfs2_inline_data *data = &di->id2.i_data; + + if (le32_to_cpu(di->i_clusters)) { + rc = ocfs2_error(sb, + "Invalid dinode %llu: %u clusters\n", + (unsigned long long)bh->b_blocknr, + le32_to_cpu(di->i_clusters)); + goto bail; + } + + if (le16_to_cpu(data->id_count) > + ocfs2_max_inline_data_with_xattr(sb, di)) { + rc = ocfs2_error(sb, + "Invalid dinode #%llu: inline data id_count %u exceeds max %d\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(data->id_count), + ocfs2_max_inline_data_with_xattr(sb, di)); + goto bail; + } + + if (le64_to_cpu(di->i_size) > le16_to_cpu(data->id_count)) { + rc = ocfs2_error(sb, + "Invalid dinode #%llu: inline data i_size %llu exceeds id_count %u\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(di->i_size), + le16_to_cpu(data->id_count)); + goto bail; + } + } + + if (S_ISLNK(le16_to_cpu(di->i_mode)) && + !le32_to_cpu(di->i_clusters)) { + int max_inline = ocfs2_fast_symlink_chars(sb); + u64 i_size = le64_to_cpu(di->i_size); + + if (i_size >= max_inline) { + rc = ocfs2_error(sb, + "Invalid dinode #%llu: fast symlink i_size %llu exceeds max %d\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)i_size, + max_inline - 1); + goto bail; + } + + if (strnlen((char *)di->id2.i_symlink, i_size + 1) != i_size) { + rc = ocfs2_error(sb, + "Invalid dinode #%llu: fast symlink is not NUL-terminated at i_size %llu\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)i_size); + goto bail; + } + } + + if (le32_to_cpu(di->i_flags) & OCFS2_CHAIN_FL) { + struct ocfs2_chain_list *cl = &di->id2.i_chain; + u16 bpc = 1 << (OCFS2_SB(sb)->s_clustersize_bits - + sb->s_blocksize_bits); + + if (le16_to_cpu(cl->cl_count) != ocfs2_chain_recs_per_inode(sb)) { + rc = ocfs2_error(sb, "Invalid dinode %llu: chain list count %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(cl->cl_count)); + goto bail; + } + if (le16_to_cpu(cl->cl_next_free_rec) > le16_to_cpu(cl->cl_count)) { + rc = ocfs2_error(sb, "Invalid dinode %llu: chain list index %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(cl->cl_next_free_rec)); + goto bail; + } + if (OCFS2_SB(sb)->bitmap_blkno && + OCFS2_SB(sb)->bitmap_blkno != le64_to_cpu(di->i_blkno) && + le16_to_cpu(cl->cl_bpc) != bpc) { + rc = ocfs2_error(sb, "Invalid dinode %llu: bits per cluster %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(cl->cl_bpc)); + goto bail; + } + } + + if ((le16_to_cpu(di->i_dyn_features) & OCFS2_HAS_REFCOUNT_FL) && + !di->i_refcount_loc) { + rc = ocfs2_error(sb, "Inode #%llu has refcount flag but no i_refcount_loc\n", + (unsigned long long)bh->b_blocknr); + goto bail; + } + + if (ocfs2_dinode_has_extents(di)) { + struct ocfs2_extent_list *el = &di->id2.i_list; + u16 count = le16_to_cpu(el->l_count); + u16 next_free = le16_to_cpu(el->l_next_free_rec); + + if (count == 0) { + rc = ocfs2_error(sb, + "Invalid dinode %llu: extent list l_count is zero\n", + (unsigned long long)bh->b_blocknr); + goto bail; + } + /* + * The exact capacity depends on i_xattr_inline_size, another + * unvalidated on-disk field. Inline xattrs only shrink the + * list, so the no-xattr maximum is a safe upper bound that a + * valid l_count never exceeds. + */ + if (count > ocfs2_extent_recs_per_inode(sb)) { + rc = ocfs2_error(sb, + "Invalid dinode %llu: extent list l_count %u exceeds max %u\n", + (unsigned long long)bh->b_blocknr, count, + ocfs2_extent_recs_per_inode(sb)); + goto bail; + } + if (next_free > count) { + rc = ocfs2_error(sb, + "Invalid dinode %llu: extent list l_next_free_rec %u exceeds l_count %u\n", + (unsigned long long)bh->b_blocknr, next_free, count); + goto bail; + } + } + + rc = 0; + +bail: + return rc; +} + +static int ocfs2_filecheck_validate_inode_block(struct super_block *sb, + struct buffer_head *bh) +{ + int rc = 0; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)bh->b_data; + + trace_ocfs2_filecheck_validate_inode_block( + (unsigned long long)bh->b_blocknr); + + BUG_ON(!buffer_uptodate(bh)); + + /* + * Call ocfs2_validate_meta_ecc() first since it has ecc repair + * function, but we should not return error immediately when ecc + * validation fails, because the reason is quite likely the invalid + * inode number inputted. + */ + rc = ocfs2_validate_meta_ecc(sb, bh->b_data, &di->i_check); + if (rc) { + mlog(ML_ERROR, + "Filecheck: checksum failed for dinode %llu\n", + (unsigned long long)bh->b_blocknr); + rc = -OCFS2_FILECHECK_ERR_BLOCKECC; + } + + if (!OCFS2_IS_VALID_DINODE(di)) { + mlog(ML_ERROR, + "Filecheck: invalid dinode #%llu: signature = %.*s\n", + (unsigned long long)bh->b_blocknr, 7, di->i_signature); + rc = -OCFS2_FILECHECK_ERR_INVALIDINO; + goto bail; + } else if (rc) + goto bail; + + if (le64_to_cpu(di->i_blkno) != bh->b_blocknr) { + mlog(ML_ERROR, + "Filecheck: invalid dinode #%llu: i_blkno is %llu\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(di->i_blkno)); + rc = -OCFS2_FILECHECK_ERR_BLOCKNO; + goto bail; + } + + if (!(di->i_flags & cpu_to_le32(OCFS2_VALID_FL))) { + mlog(ML_ERROR, + "Filecheck: invalid dinode #%llu: OCFS2_VALID_FL " + "not set\n", + (unsigned long long)bh->b_blocknr); + rc = -OCFS2_FILECHECK_ERR_VALIDFLAG; + goto bail; + } + + if (le32_to_cpu(di->i_fs_generation) != + OCFS2_SB(sb)->fs_generation) { + mlog(ML_ERROR, + "Filecheck: invalid dinode #%llu: fs_generation is %u\n", + (unsigned long long)bh->b_blocknr, + le32_to_cpu(di->i_fs_generation)); + rc = -OCFS2_FILECHECK_ERR_GENERATION; + goto bail; + } + + if (!ocfs2_valid_inode_mode(le16_to_cpu(di->i_mode))) { + mlog(ML_ERROR, + "Filecheck: invalid dinode #%llu: mode 0%o has unknown file type\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(di->i_mode)); + rc = -OCFS2_FILECHECK_ERR_INVALIDINO; + goto bail; + } + + if (ocfs2_dinode_has_unexpected_rdev(di)) { + mlog(ML_ERROR, + "Filecheck: invalid dinode #%llu: non-device mode 0%o with i_rdev %llu\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(di->i_mode), + (unsigned long long)le64_to_cpu(di->id1.dev1.i_rdev)); + rc = -OCFS2_FILECHECK_ERR_INVALIDINO; + goto bail; + } + + if (ocfs2_dinode_has_size_without_clusters(sb, di)) { + if (S_ISDIR(le16_to_cpu(di->i_mode))) + mlog(ML_ERROR, + "Filecheck: invalid dinode #%llu: directory i_size %llu with i_clusters 0 and no inline-data flag\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(di->i_size)); + else + mlog(ML_ERROR, + "Filecheck: invalid dinode #%llu: regular file i_size %llu with i_clusters 0 and no inline-data flag on non-sparse volume\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(di->i_size)); + rc = -OCFS2_FILECHECK_ERR_INVALIDINO; + } + +bail: + return rc; +} + +static int ocfs2_filecheck_repair_inode_block(struct super_block *sb, + struct buffer_head *bh) +{ + int changed = 0; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)bh->b_data; + + if (!ocfs2_filecheck_validate_inode_block(sb, bh)) + return 0; + + trace_ocfs2_filecheck_repair_inode_block( + (unsigned long long)bh->b_blocknr); + + if (unlikely(ocfs2_emergency_state(OCFS2_SB(sb)))) { + mlog(ML_ERROR, + "Filecheck: cannot repair dinode #%llu " + "on readonly filesystem\n", + (unsigned long long)bh->b_blocknr); + return -OCFS2_FILECHECK_ERR_READONLY; + } + + if (buffer_jbd(bh)) { + mlog(ML_ERROR, + "Filecheck: cannot repair dinode #%llu, " + "its buffer is in jbd\n", + (unsigned long long)bh->b_blocknr); + return -OCFS2_FILECHECK_ERR_INJBD; + } + + if (!OCFS2_IS_VALID_DINODE(di)) { + /* Cannot fix invalid inode block */ + return -OCFS2_FILECHECK_ERR_INVALIDINO; + } + + if (!(di->i_flags & cpu_to_le32(OCFS2_VALID_FL))) { + /* Cannot just add VALID_FL flag back as a fix, + * need more things to check here. + */ + return -OCFS2_FILECHECK_ERR_VALIDFLAG; + } + + if (le64_to_cpu(di->i_blkno) != bh->b_blocknr) { + di->i_blkno = cpu_to_le64(bh->b_blocknr); + changed = 1; + mlog(ML_ERROR, + "Filecheck: reset dinode #%llu: i_blkno to %llu\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(di->i_blkno)); + } + + if (le32_to_cpu(di->i_fs_generation) != + OCFS2_SB(sb)->fs_generation) { + di->i_fs_generation = cpu_to_le32(OCFS2_SB(sb)->fs_generation); + changed = 1; + mlog(ML_ERROR, + "Filecheck: reset dinode #%llu: fs_generation to %u\n", + (unsigned long long)bh->b_blocknr, + le32_to_cpu(di->i_fs_generation)); + } + + if (ocfs2_dinode_has_extents(di) && + le16_to_cpu(di->id2.i_list.l_next_free_rec) > le16_to_cpu(di->id2.i_list.l_count)) { + di->id2.i_list.l_next_free_rec = di->id2.i_list.l_count; + changed = 1; + mlog(ML_ERROR, + "Filecheck: reset dinode #%llu: l_next_free_rec to %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(di->id2.i_list.l_next_free_rec)); + } + + if (changed || ocfs2_validate_meta_ecc(sb, bh->b_data, &di->i_check)) { + ocfs2_compute_meta_ecc(sb, bh->b_data, &di->i_check); + mark_buffer_dirty(bh); + mlog(ML_ERROR, + "Filecheck: reset dinode #%llu: compute meta ecc\n", + (unsigned long long)bh->b_blocknr); + } + + return 0; +} + +static int +ocfs2_filecheck_read_inode_block_full(struct inode *inode, + struct buffer_head **bh, + int flags, int type) +{ + int rc; + struct buffer_head *tmp = *bh; + + if (!type) /* Check inode block */ + rc = ocfs2_read_blocks(INODE_CACHE(inode), + OCFS2_I(inode)->ip_blkno, + 1, &tmp, flags, + ocfs2_filecheck_validate_inode_block); + else /* Repair inode block */ + rc = ocfs2_read_blocks(INODE_CACHE(inode), + OCFS2_I(inode)->ip_blkno, + 1, &tmp, flags, + ocfs2_filecheck_repair_inode_block); + + /* If ocfs2_read_blocks() got us a new bh, pass it up. */ + if (!rc && !*bh) + *bh = tmp; + + return rc; +} + +int ocfs2_read_inode_block_full(struct inode *inode, struct buffer_head **bh, + int flags) +{ + int rc; + struct buffer_head *tmp = *bh; + + rc = ocfs2_read_blocks(INODE_CACHE(inode), OCFS2_I(inode)->ip_blkno, + 1, &tmp, flags, ocfs2_validate_inode_block); + + /* If ocfs2_read_blocks() got us a new bh, pass it up. */ + if (!rc && !*bh) + *bh = tmp; + + return rc; +} + +int ocfs2_read_inode_block(struct inode *inode, struct buffer_head **bh) +{ + return ocfs2_read_inode_block_full(inode, bh, 0); +} + + +static u64 ocfs2_inode_cache_owner(struct ocfs2_caching_info *ci) +{ + struct ocfs2_inode_info *oi = cache_info_to_inode(ci); + + return oi->ip_blkno; +} + +static struct super_block *ocfs2_inode_cache_get_super(struct ocfs2_caching_info *ci) +{ + struct ocfs2_inode_info *oi = cache_info_to_inode(ci); + + return oi->vfs_inode.i_sb; +} + +static void ocfs2_inode_cache_lock(struct ocfs2_caching_info *ci) +__acquires(&oi->ip_lock) +{ + struct ocfs2_inode_info *oi = cache_info_to_inode(ci); + + spin_lock(&oi->ip_lock); +} + +static void ocfs2_inode_cache_unlock(struct ocfs2_caching_info *ci) +__releases(&oi->ip_lock) +{ + struct ocfs2_inode_info *oi = cache_info_to_inode(ci); + + spin_unlock(&oi->ip_lock); +} + +static void ocfs2_inode_cache_io_lock(struct ocfs2_caching_info *ci) +{ + struct ocfs2_inode_info *oi = cache_info_to_inode(ci); + + mutex_lock(&oi->ip_io_mutex); +} + +static void ocfs2_inode_cache_io_unlock(struct ocfs2_caching_info *ci) +{ + struct ocfs2_inode_info *oi = cache_info_to_inode(ci); + + mutex_unlock(&oi->ip_io_mutex); +} + +const struct ocfs2_caching_operations ocfs2_inode_caching_ops = { + .co_owner = ocfs2_inode_cache_owner, + .co_get_super = ocfs2_inode_cache_get_super, + .co_cache_lock = ocfs2_inode_cache_lock, + .co_cache_unlock = ocfs2_inode_cache_unlock, + .co_io_lock = ocfs2_inode_cache_io_lock, + .co_io_unlock = ocfs2_inode_cache_io_unlock, +}; diff --git a/fs/ocfs2/inode.h b/fs/ocfs2/inode.h new file mode 100644 index 000000000..07bd838e7 --- /dev/null +++ b/fs/ocfs2/inode.h @@ -0,0 +1,173 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * inode.h + * + * Function prototypes + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#ifndef OCFS2_INODE_H +#define OCFS2_INODE_H + +#include "extent_map.h" + +/* OCFS2 Inode Private Data */ +struct ocfs2_inode_info +{ + u64 ip_blkno; + + struct ocfs2_lock_res ip_rw_lockres; + struct ocfs2_lock_res ip_inode_lockres; + struct ocfs2_lock_res ip_open_lockres; + + /* protects allocation changes on this inode. */ + struct rw_semaphore ip_alloc_sem; + + /* protects extended attribute changes on this inode */ + struct rw_semaphore ip_xattr_sem; + + /* These fields are protected by ip_lock */ + spinlock_t ip_lock; + u32 ip_open_count; + struct list_head ip_io_markers; + u32 ip_clusters; + + u16 ip_dyn_features; + struct mutex ip_io_mutex; + u32 ip_flags; /* see below */ + u32 ip_attr; /* inode attributes */ + + /* Record unwritten extents during direct io. */ + struct list_head ip_unwritten_list; + + /* protected by recovery_lock. */ + struct inode *ip_next_orphan; + + struct ocfs2_caching_info ip_metadata_cache; + struct ocfs2_extent_map ip_extent_map; + struct inode vfs_inode; + struct jbd2_inode ip_jinode; + + u32 ip_dir_start_lookup; + + /* Only valid if the inode is the dir. */ + u32 ip_last_used_slot; + u64 ip_last_used_group; + u32 ip_dir_lock_gen; + + struct ocfs2_alloc_reservation ip_la_data_resv; + + /* + * Transactions that contain inode's metadata needed to complete + * fsync and fdatasync, respectively. + */ + tid_t i_sync_tid; + tid_t i_datasync_tid; + + struct dquot __rcu *i_dquot[MAXQUOTAS]; +}; + +/* + * Flags for the ip_flags field + */ +/* System file inodes */ +#define OCFS2_INODE_SYSTEM_FILE 0x00000001 +#define OCFS2_INODE_JOURNAL 0x00000002 +#define OCFS2_INODE_BITMAP 0x00000004 +/* This inode has been wiped from disk */ +#define OCFS2_INODE_DELETED 0x00000008 +/* Has the inode been orphaned on another node? + * + * This hints to ocfs2_drop_inode that it should clear i_nlink before + * continuing. + * + * We *only* set this on unlink vote from another node. If the inode + * was locally orphaned, then we're sure of the state and don't need + * to twiddle i_nlink later - it's either zero or not depending on + * whether our unlink succeeded. Otherwise we got this from a node + * whose intention was to orphan the inode, however he may have + * crashed, failed etc, so we let ocfs2_drop_inode zero the value and + * rely on ocfs2_delete_inode to sort things out under the proper + * cluster locks. + */ +#define OCFS2_INODE_MAYBE_ORPHANED 0x00000010 +/* Does someone have the file open O_DIRECT */ +#define OCFS2_INODE_OPEN_DIRECT 0x00000020 +/* Tell the inode wipe code it's not in orphan dir */ +#define OCFS2_INODE_SKIP_ORPHAN_DIR 0x00000040 +/* Entry in orphan dir with 'dio-' prefix */ +#define OCFS2_INODE_DIO_ORPHAN_ENTRY 0x00000080 + +static inline struct ocfs2_inode_info *OCFS2_I(struct inode *inode) +{ + return container_of(inode, struct ocfs2_inode_info, vfs_inode); +} + +#define INODE_JOURNAL(i) (OCFS2_I(i)->ip_flags & OCFS2_INODE_JOURNAL) +#define SET_INODE_JOURNAL(i) (OCFS2_I(i)->ip_flags |= OCFS2_INODE_JOURNAL) + +extern const struct address_space_operations ocfs2_aops; +extern const struct ocfs2_caching_operations ocfs2_inode_caching_ops; + +static inline struct ocfs2_caching_info *INODE_CACHE(struct inode *inode) +{ + return &OCFS2_I(inode)->ip_metadata_cache; +} + +void ocfs2_evict_inode(struct inode *inode); + +/* Flags for ocfs2_iget() */ +#define OCFS2_FI_FLAG_SYSFILE 0x1 +#define OCFS2_FI_FLAG_ORPHAN_RECOVERY 0x2 +#define OCFS2_FI_FLAG_FILECHECK_CHK 0x4 +#define OCFS2_FI_FLAG_FILECHECK_FIX 0x8 + +struct inode *ocfs2_ilookup(struct super_block *sb, u64 feoff); +struct inode *ocfs2_iget(struct ocfs2_super *osb, u64 feoff, unsigned flags, + int sysfile_type); +int ocfs2_inode_revalidate(struct dentry *dentry); +void ocfs2_populate_inode(struct inode *inode, struct ocfs2_dinode *fe, + int create_ino); +void ocfs2_sync_blockdev(struct super_block *sb); +void ocfs2_refresh_inode(struct inode *inode, + struct ocfs2_dinode *fe); +int ocfs2_mark_inode_dirty(handle_t *handle, + struct inode *inode, + struct buffer_head *bh); + +void ocfs2_set_inode_flags(struct inode *inode); +void ocfs2_get_inode_flags(struct ocfs2_inode_info *oi); + +static inline blkcnt_t ocfs2_inode_sector_count(struct inode *inode) +{ + int c_to_s_bits = OCFS2_SB(inode->i_sb)->s_clustersize_bits - 9; + + return (blkcnt_t)OCFS2_I(inode)->ip_clusters << c_to_s_bits; +} + +/* Validate that a bh contains a valid inode */ +int ocfs2_validate_inode_block(struct super_block *sb, + struct buffer_head *bh); +/* + * Read an inode block into *bh. If *bh is NULL, a bh will be allocated. + * This is a cached read. The inode will be validated with + * ocfs2_validate_inode_block(). + */ +int ocfs2_read_inode_block(struct inode *inode, struct buffer_head **bh); +/* The same, but can be passed OCFS2_BH_* flags */ +int ocfs2_read_inode_block_full(struct inode *inode, struct buffer_head **bh, + int flags); + +static inline struct ocfs2_inode_info *cache_info_to_inode(struct ocfs2_caching_info *ci) +{ + return container_of(ci, struct ocfs2_inode_info, ip_metadata_cache); +} + +/* Does this inode have the reflink flag set? */ +static inline bool ocfs2_is_refcount_inode(struct inode *inode) +{ + return (OCFS2_I(inode)->ip_dyn_features & OCFS2_HAS_REFCOUNT_FL); +} + +#endif /* OCFS2_INODE_H */ diff --git a/fs/ocfs2/ioctl.c b/fs/ocfs2/ioctl.c new file mode 100644 index 000000000..cbe59d231 --- /dev/null +++ b/fs/ocfs2/ioctl.c @@ -0,0 +1,1003 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ocfs2/ioctl.c + * + * Copyright (C) 2006 Herbert Poetzl + * adapted from Remy Card's ext2/ioctl.c + */ + +#include +#include +#include +#include +#include + +#include + +#include "ocfs2.h" +#include "alloc.h" +#include "dlmglue.h" +#include "file.h" +#include "inode.h" +#include "journal.h" + +#include "ocfs2_fs.h" +#include "ioctl.h" +#include "resize.h" +#include "refcounttree.h" +#include "sysfile.h" +#include "dir.h" +#include "buffer_head_io.h" +#include "suballoc.h" +#include "move_extents.h" + +#define o2info_from_user(a, b) \ + copy_from_user(&(a), (b), sizeof(a)) +#define o2info_to_user(a, b) \ + copy_to_user((typeof(a) __user *)b, &(a), sizeof(a)) + +/* + * This is just a best-effort to tell userspace that this request + * caused the error. + */ +static inline void o2info_set_request_error(struct ocfs2_info_request *kreq, + struct ocfs2_info_request __user *req) +{ + kreq->ir_flags |= OCFS2_INFO_FL_ERROR; + (void)put_user(kreq->ir_flags, (__u32 __user *)&(req->ir_flags)); +} + +static inline void o2info_set_request_filled(struct ocfs2_info_request *req) +{ + req->ir_flags |= OCFS2_INFO_FL_FILLED; +} + +static inline void o2info_clear_request_filled(struct ocfs2_info_request *req) +{ + req->ir_flags &= ~OCFS2_INFO_FL_FILLED; +} + +static inline int o2info_coherent(struct ocfs2_info_request *req) +{ + return (!(req->ir_flags & OCFS2_INFO_FL_NON_COHERENT)); +} + +int ocfs2_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + unsigned int flags; + int status; + + status = ocfs2_inode_lock(inode, NULL, 0); + if (status < 0) { + mlog_errno(status); + return status; + } + ocfs2_get_inode_flags(OCFS2_I(inode)); + flags = OCFS2_I(inode)->ip_attr; + ocfs2_inode_unlock(inode, 0); + + fileattr_fill_flags(fa, flags & OCFS2_FL_VISIBLE); + + return status; +} + +int ocfs2_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + unsigned int flags = fa->flags; + struct ocfs2_inode_info *ocfs2_inode = OCFS2_I(inode); + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + handle_t *handle = NULL; + struct buffer_head *bh = NULL; + unsigned oldflags; + int status; + + if (fileattr_has_fsx(fa)) + return -EOPNOTSUPP; + + status = ocfs2_inode_lock(inode, &bh, 1); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + if (!S_ISDIR(inode->i_mode)) + flags &= ~OCFS2_DIRSYNC_FL; + + oldflags = ocfs2_inode->ip_attr; + flags = flags & OCFS2_FL_MODIFIABLE; + flags |= oldflags & ~OCFS2_FL_MODIFIABLE; + + /* Check already done by VFS, but repeat with ocfs lock */ + status = -EPERM; + if ((flags ^ oldflags) & (FS_APPEND_FL | FS_IMMUTABLE_FL) && + !capable(CAP_LINUX_IMMUTABLE)) + goto bail_unlock; + + handle = ocfs2_start_trans(osb, OCFS2_INODE_UPDATE_CREDITS); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + mlog_errno(status); + goto bail_unlock; + } + + ocfs2_inode->ip_attr = flags; + ocfs2_set_inode_flags(inode); + inode_set_ctime_current(inode); + + status = ocfs2_mark_inode_dirty(handle, inode, bh); + if (status < 0) + mlog_errno(status); + + ocfs2_commit_trans(osb, handle); + +bail_unlock: + ocfs2_inode_unlock(inode, 1); +bail: + brelse(bh); + + return status; +} + +static int ocfs2_info_handle_blocksize(struct inode *inode, + struct ocfs2_info_request __user *req) +{ + struct ocfs2_info_blocksize oib; + + if (o2info_from_user(oib, req)) + return -EFAULT; + + oib.ib_blocksize = inode->i_sb->s_blocksize; + + o2info_set_request_filled(&oib.ib_req); + + if (o2info_to_user(oib, req)) + return -EFAULT; + + return 0; +} + +static int ocfs2_info_handle_clustersize(struct inode *inode, + struct ocfs2_info_request __user *req) +{ + struct ocfs2_info_clustersize oic; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + if (o2info_from_user(oic, req)) + return -EFAULT; + + oic.ic_clustersize = osb->s_clustersize; + + o2info_set_request_filled(&oic.ic_req); + + if (o2info_to_user(oic, req)) + return -EFAULT; + + return 0; +} + +static int ocfs2_info_handle_maxslots(struct inode *inode, + struct ocfs2_info_request __user *req) +{ + struct ocfs2_info_maxslots oim; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + if (o2info_from_user(oim, req)) + return -EFAULT; + + oim.im_max_slots = osb->max_slots; + + o2info_set_request_filled(&oim.im_req); + + if (o2info_to_user(oim, req)) + return -EFAULT; + + return 0; +} + +static int ocfs2_info_handle_label(struct inode *inode, + struct ocfs2_info_request __user *req) +{ + struct ocfs2_info_label oil; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + if (o2info_from_user(oil, req)) + return -EFAULT; + + memcpy(oil.il_label, osb->vol_label, OCFS2_MAX_VOL_LABEL_LEN); + + o2info_set_request_filled(&oil.il_req); + + if (o2info_to_user(oil, req)) + return -EFAULT; + + return 0; +} + +static int ocfs2_info_handle_uuid(struct inode *inode, + struct ocfs2_info_request __user *req) +{ + struct ocfs2_info_uuid oiu; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + if (o2info_from_user(oiu, req)) + return -EFAULT; + + memcpy(oiu.iu_uuid_str, osb->uuid_str, OCFS2_TEXT_UUID_LEN + 1); + + o2info_set_request_filled(&oiu.iu_req); + + if (o2info_to_user(oiu, req)) + return -EFAULT; + + return 0; +} + +static int ocfs2_info_handle_fs_features(struct inode *inode, + struct ocfs2_info_request __user *req) +{ + struct ocfs2_info_fs_features oif; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + if (o2info_from_user(oif, req)) + return -EFAULT; + + oif.if_compat_features = osb->s_feature_compat; + oif.if_incompat_features = osb->s_feature_incompat; + oif.if_ro_compat_features = osb->s_feature_ro_compat; + + o2info_set_request_filled(&oif.if_req); + + if (o2info_to_user(oif, req)) + return -EFAULT; + + return 0; +} + +static int ocfs2_info_handle_journal_size(struct inode *inode, + struct ocfs2_info_request __user *req) +{ + struct ocfs2_info_journal_size oij; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + if (o2info_from_user(oij, req)) + return -EFAULT; + + oij.ij_journal_size = i_size_read(osb->journal->j_inode); + + o2info_set_request_filled(&oij.ij_req); + + if (o2info_to_user(oij, req)) + return -EFAULT; + + return 0; +} + +static int ocfs2_info_scan_inode_alloc(struct ocfs2_super *osb, + struct inode *inode_alloc, u64 blkno, + struct ocfs2_info_freeinode *fi, + u32 slot) +{ + int status = 0, unlock = 0; + + struct buffer_head *bh = NULL; + struct ocfs2_dinode *dinode_alloc = NULL; + + if (inode_alloc) + inode_lock(inode_alloc); + + if (inode_alloc && o2info_coherent(&fi->ifi_req)) { + status = ocfs2_inode_lock(inode_alloc, &bh, 0); + if (status < 0) { + mlog_errno(status); + goto bail; + } + unlock = 1; + } else { + status = ocfs2_read_blocks_sync(osb, blkno, 1, &bh); + if (status < 0) { + mlog_errno(status); + goto bail; + } + } + + dinode_alloc = (struct ocfs2_dinode *)bh->b_data; + + fi->ifi_stat[slot].lfi_total = + le32_to_cpu(dinode_alloc->id1.bitmap1.i_total); + fi->ifi_stat[slot].lfi_free = + le32_to_cpu(dinode_alloc->id1.bitmap1.i_total) - + le32_to_cpu(dinode_alloc->id1.bitmap1.i_used); + +bail: + if (unlock) + ocfs2_inode_unlock(inode_alloc, 0); + + if (inode_alloc) + inode_unlock(inode_alloc); + + brelse(bh); + + return status; +} + +static int ocfs2_info_handle_freeinode(struct inode *inode, + struct ocfs2_info_request __user *req) +{ + u32 i; + u64 blkno = -1; + char namebuf[40]; + int status, type = INODE_ALLOC_SYSTEM_INODE; + struct ocfs2_info_freeinode *oifi = NULL; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct inode *inode_alloc = NULL; + + oifi = kzalloc_obj(struct ocfs2_info_freeinode); + if (!oifi) { + status = -ENOMEM; + mlog_errno(status); + goto out_err; + } + + if (o2info_from_user(*oifi, req)) { + status = -EFAULT; + goto out_free; + } + + oifi->ifi_slotnum = osb->max_slots; + + for (i = 0; i < oifi->ifi_slotnum; i++) { + if (o2info_coherent(&oifi->ifi_req)) { + inode_alloc = ocfs2_get_system_file_inode(osb, type, i); + if (!inode_alloc) { + mlog(ML_ERROR, "unable to get alloc inode in " + "slot %u\n", i); + status = -EIO; + goto bail; + } + } else { + int len = ocfs2_sprintf_system_inode_name(namebuf, + sizeof(namebuf), + type, i); + status = ocfs2_lookup_ino_from_name(osb->sys_root_inode, + namebuf, len, &blkno); + if (status < 0) { + status = -ENOENT; + goto bail; + } + } + + status = ocfs2_info_scan_inode_alloc(osb, inode_alloc, blkno, oifi, i); + + iput(inode_alloc); + inode_alloc = NULL; + + if (status < 0) + goto bail; + } + + o2info_set_request_filled(&oifi->ifi_req); + + if (o2info_to_user(*oifi, req)) { + status = -EFAULT; + goto out_free; + } + + status = 0; +bail: + if (status) + o2info_set_request_error(&oifi->ifi_req, req); +out_free: + kfree(oifi); +out_err: + return status; +} + +static void o2ffg_update_histogram(struct ocfs2_info_free_chunk_list *hist, + unsigned int chunksize) +{ + u32 index; + + index = __ilog2_u32(chunksize); + if (index >= OCFS2_INFO_MAX_HIST) + index = OCFS2_INFO_MAX_HIST - 1; + + hist->fc_chunks[index]++; + hist->fc_clusters[index] += chunksize; +} + +static void o2ffg_update_stats(struct ocfs2_info_freefrag_stats *stats, + unsigned int chunksize) +{ + if (chunksize > stats->ffs_max) + stats->ffs_max = chunksize; + + if (chunksize < stats->ffs_min) + stats->ffs_min = chunksize; + + stats->ffs_avg += chunksize; + stats->ffs_free_chunks_real++; +} + +static void ocfs2_info_update_ffg(struct ocfs2_info_freefrag *ffg, + unsigned int chunksize) +{ + o2ffg_update_histogram(&(ffg->iff_ffs.ffs_fc_hist), chunksize); + o2ffg_update_stats(&(ffg->iff_ffs), chunksize); +} + +static int ocfs2_info_freefrag_scan_chain(struct ocfs2_super *osb, + struct inode *gb_inode, + struct ocfs2_dinode *gb_dinode, + struct ocfs2_chain_rec *rec, + struct ocfs2_info_freefrag *ffg, + u32 chunks_in_group) +{ + int status = 0, used; + u64 blkno; + + struct buffer_head *bh = NULL; + struct ocfs2_group_desc *bg = NULL; + + unsigned int max_bits, max_bitmap_bits, num_clusters; + unsigned int offset = 0, cluster, chunk; + unsigned int chunk_free, last_chunksize = 0; + + if (!le32_to_cpu(rec->c_free)) + goto bail; + + max_bitmap_bits = 8 * ocfs2_group_bitmap_size(osb->sb, 0, + osb->s_feature_incompat); + + do { + if (!bg) + blkno = le64_to_cpu(rec->c_blkno); + else + blkno = le64_to_cpu(bg->bg_next_group); + + if (bh) { + brelse(bh); + bh = NULL; + } + + if (o2info_coherent(&ffg->iff_req)) + status = ocfs2_read_group_descriptor(gb_inode, + gb_dinode, + blkno, &bh); + else + status = ocfs2_read_blocks_sync(osb, blkno, 1, &bh); + + if (status < 0) { + mlog(ML_ERROR, "Can't read the group descriptor # " + "%llu from device.", (unsigned long long)blkno); + status = -EIO; + goto bail; + } + + bg = (struct ocfs2_group_desc *)bh->b_data; + + if (!le16_to_cpu(bg->bg_free_bits_count)) + continue; + + max_bits = le16_to_cpu(bg->bg_bits); + + /* + * Non-coherent scans read raw blocks and do not get the + * bg_bits validation from + * ocfs2_read_group_descriptor(). + */ + if (max_bits > max_bitmap_bits) { + mlog(ML_ERROR, + "Group desc #%llu has %u bits, max bitmap bits %u\n", + (unsigned long long)blkno, max_bits, max_bitmap_bits); + max_bits = max_bitmap_bits; + } + + offset = 0; + + for (chunk = 0; chunk < chunks_in_group; chunk++) { + /* + * last chunk may be not an entire one. + */ + if ((offset + ffg->iff_chunksize) > max_bits) + num_clusters = max_bits - offset; + else + num_clusters = ffg->iff_chunksize; + + chunk_free = 0; + for (cluster = 0; cluster < num_clusters; cluster++) { + used = ocfs2_test_bit(offset, + (unsigned long *)bg->bg_bitmap); + /* + * - chunk_free counts free clusters in #N chunk. + * - last_chunksize records the size(in) clusters + * for the last real free chunk being counted. + */ + if (!used) { + last_chunksize++; + chunk_free++; + } + + if (used && last_chunksize) { + ocfs2_info_update_ffg(ffg, + last_chunksize); + last_chunksize = 0; + } + + offset++; + } + + if (chunk_free == ffg->iff_chunksize) + ffg->iff_ffs.ffs_free_chunks++; + } + + /* + * need to update the info for last free chunk. + */ + if (last_chunksize) + ocfs2_info_update_ffg(ffg, last_chunksize); + + } while (le64_to_cpu(bg->bg_next_group)); + +bail: + brelse(bh); + + return status; +} + +static int ocfs2_info_freefrag_scan_bitmap(struct ocfs2_super *osb, + struct inode *gb_inode, u64 blkno, + struct ocfs2_info_freefrag *ffg) +{ + u32 chunks_in_group; + int status = 0, unlock = 0, i; + + struct buffer_head *bh = NULL; + struct ocfs2_chain_list *cl = NULL; + struct ocfs2_chain_rec *rec = NULL; + struct ocfs2_dinode *gb_dinode = NULL; + + if (gb_inode) + inode_lock(gb_inode); + + if (o2info_coherent(&ffg->iff_req)) { + status = ocfs2_inode_lock(gb_inode, &bh, 0); + if (status < 0) { + mlog_errno(status); + goto bail; + } + unlock = 1; + } else { + status = ocfs2_read_blocks_sync(osb, blkno, 1, &bh); + if (status < 0) { + mlog_errno(status); + goto bail; + } + } + + gb_dinode = (struct ocfs2_dinode *)bh->b_data; + cl = &(gb_dinode->id2.i_chain); + + /* + * Chunksize(in) clusters from userspace should be + * less than clusters in a group. + */ + if (ffg->iff_chunksize > le16_to_cpu(cl->cl_cpg)) { + status = -EINVAL; + goto bail; + } + + memset(&ffg->iff_ffs, 0, sizeof(struct ocfs2_info_freefrag_stats)); + + ffg->iff_ffs.ffs_min = ~0U; + ffg->iff_ffs.ffs_clusters = + le32_to_cpu(gb_dinode->id1.bitmap1.i_total); + ffg->iff_ffs.ffs_free_clusters = ffg->iff_ffs.ffs_clusters - + le32_to_cpu(gb_dinode->id1.bitmap1.i_used); + + chunks_in_group = le16_to_cpu(cl->cl_cpg) / ffg->iff_chunksize + 1; + + for (i = 0; i < le16_to_cpu(cl->cl_next_free_rec); i++) { + rec = &(cl->cl_recs[i]); + status = ocfs2_info_freefrag_scan_chain(osb, gb_inode, + gb_dinode, + rec, ffg, + chunks_in_group); + if (status) + goto bail; + } + + if (ffg->iff_ffs.ffs_free_chunks_real) + ffg->iff_ffs.ffs_avg = (ffg->iff_ffs.ffs_avg / + ffg->iff_ffs.ffs_free_chunks_real); +bail: + if (unlock) + ocfs2_inode_unlock(gb_inode, 0); + + if (gb_inode) + inode_unlock(gb_inode); + + iput(gb_inode); + brelse(bh); + + return status; +} + +static int ocfs2_info_handle_freefrag(struct inode *inode, + struct ocfs2_info_request __user *req) +{ + u64 blkno = -1; + char namebuf[40]; + int status, type = GLOBAL_BITMAP_SYSTEM_INODE; + + struct ocfs2_info_freefrag *oiff; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct inode *gb_inode = NULL; + + oiff = kzalloc_obj(struct ocfs2_info_freefrag); + if (!oiff) { + status = -ENOMEM; + mlog_errno(status); + goto out_err; + } + + if (o2info_from_user(*oiff, req)) { + status = -EFAULT; + goto out_free; + } + /* + * chunksize from userspace should be power of 2. + */ + if ((oiff->iff_chunksize & (oiff->iff_chunksize - 1)) || + (!oiff->iff_chunksize)) { + status = -EINVAL; + goto bail; + } + + if (o2info_coherent(&oiff->iff_req)) { + gb_inode = ocfs2_get_system_file_inode(osb, type, + OCFS2_INVALID_SLOT); + if (!gb_inode) { + mlog(ML_ERROR, "unable to get global_bitmap inode\n"); + status = -EIO; + goto bail; + } + } else { + int len = ocfs2_sprintf_system_inode_name(namebuf, sizeof(namebuf), + type, OCFS2_INVALID_SLOT); + status = ocfs2_lookup_ino_from_name(osb->sys_root_inode, + namebuf, len, &blkno); + if (status < 0) { + status = -ENOENT; + goto bail; + } + } + + status = ocfs2_info_freefrag_scan_bitmap(osb, gb_inode, blkno, oiff); + if (status < 0) + goto bail; + + o2info_set_request_filled(&oiff->iff_req); + + if (o2info_to_user(*oiff, req)) { + status = -EFAULT; + goto out_free; + } + + status = 0; +bail: + if (status) + o2info_set_request_error(&oiff->iff_req, req); +out_free: + kfree(oiff); +out_err: + return status; +} + +static int ocfs2_info_handle_unknown(struct inode *inode, + struct ocfs2_info_request __user *req) +{ + struct ocfs2_info_request oir; + + if (o2info_from_user(oir, req)) + return -EFAULT; + + o2info_clear_request_filled(&oir); + + if (o2info_to_user(oir, req)) + return -EFAULT; + + return 0; +} + +/* + * Validate and distinguish OCFS2_IOC_INFO requests. + * + * - validate the magic number. + * - distinguish different requests. + * - validate size of different requests. + */ +static int ocfs2_info_handle_request(struct inode *inode, + struct ocfs2_info_request __user *req) +{ + int status = -EFAULT; + struct ocfs2_info_request oir; + + if (o2info_from_user(oir, req)) + goto bail; + + status = -EINVAL; + if (oir.ir_magic != OCFS2_INFO_MAGIC) + goto bail; + + switch (oir.ir_code) { + case OCFS2_INFO_BLOCKSIZE: + if (oir.ir_size == sizeof(struct ocfs2_info_blocksize)) + status = ocfs2_info_handle_blocksize(inode, req); + break; + case OCFS2_INFO_CLUSTERSIZE: + if (oir.ir_size == sizeof(struct ocfs2_info_clustersize)) + status = ocfs2_info_handle_clustersize(inode, req); + break; + case OCFS2_INFO_MAXSLOTS: + if (oir.ir_size == sizeof(struct ocfs2_info_maxslots)) + status = ocfs2_info_handle_maxslots(inode, req); + break; + case OCFS2_INFO_LABEL: + if (oir.ir_size == sizeof(struct ocfs2_info_label)) + status = ocfs2_info_handle_label(inode, req); + break; + case OCFS2_INFO_UUID: + if (oir.ir_size == sizeof(struct ocfs2_info_uuid)) + status = ocfs2_info_handle_uuid(inode, req); + break; + case OCFS2_INFO_FS_FEATURES: + if (oir.ir_size == sizeof(struct ocfs2_info_fs_features)) + status = ocfs2_info_handle_fs_features(inode, req); + break; + case OCFS2_INFO_JOURNAL_SIZE: + if (oir.ir_size == sizeof(struct ocfs2_info_journal_size)) + status = ocfs2_info_handle_journal_size(inode, req); + break; + case OCFS2_INFO_FREEINODE: + if (oir.ir_size == sizeof(struct ocfs2_info_freeinode)) + status = ocfs2_info_handle_freeinode(inode, req); + break; + case OCFS2_INFO_FREEFRAG: + if (oir.ir_size == sizeof(struct ocfs2_info_freefrag)) + status = ocfs2_info_handle_freefrag(inode, req); + break; + default: + status = ocfs2_info_handle_unknown(inode, req); + break; + } + +bail: + return status; +} + +static int ocfs2_get_request_ptr(struct ocfs2_info *info, int idx, + u64 *req_addr, int compat_flag) +{ + int status = -EFAULT; + u64 __user *bp = NULL; + + if (compat_flag) { +#ifdef CONFIG_COMPAT + /* + * pointer bp stores the base address of a pointers array, + * which collects all addresses of separate request. + */ + bp = (u64 __user *)(unsigned long)compat_ptr(info->oi_requests); +#else + BUG(); +#endif + } else + bp = (u64 __user *)(unsigned long)(info->oi_requests); + + if (o2info_from_user(*req_addr, bp + idx)) + goto bail; + + status = 0; +bail: + return status; +} + +/* + * OCFS2_IOC_INFO handles an array of requests passed from userspace. + * + * ocfs2_info_handle() receives a large info aggregation, grab and + * validate the request count from header, then break it into small + * pieces, later specific handlers can handle them one by one. + * + * Idea here is to make each separate request small enough to ensure + * a better backward&forward compatibility, since a small piece of + * request will be less likely to be broken if disk layout get changed. + */ +static noinline_for_stack int +ocfs2_info_handle(struct inode *inode, struct ocfs2_info *info, int compat_flag) +{ + int i, status = 0; + u64 req_addr; + struct ocfs2_info_request __user *reqp; + + if ((info->oi_count > OCFS2_INFO_MAX_REQUEST) || + (!info->oi_requests)) { + status = -EINVAL; + goto bail; + } + + for (i = 0; i < info->oi_count; i++) { + + status = ocfs2_get_request_ptr(info, i, &req_addr, compat_flag); + if (status) + break; + + reqp = (struct ocfs2_info_request __user *)(unsigned long)req_addr; + if (!reqp) { + status = -EINVAL; + goto bail; + } + + status = ocfs2_info_handle_request(inode, reqp); + if (status) + break; + } + +bail: + return status; +} + +long ocfs2_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + void __user *argp = (void __user *)arg; + int status; + + switch (cmd) { + case OCFS2_IOC_RESVSP: + case OCFS2_IOC_RESVSP64: + case OCFS2_IOC_UNRESVSP: + case OCFS2_IOC_UNRESVSP64: + { + struct ocfs2_space_resv sr; + + if (copy_from_user(&sr, (int __user *) arg, sizeof(sr))) + return -EFAULT; + + return ocfs2_change_file_space(filp, cmd, &sr); + } + case OCFS2_IOC_GROUP_EXTEND: + { + int new_clusters; + + if (!capable(CAP_SYS_RESOURCE)) + return -EPERM; + + if (get_user(new_clusters, (int __user *)arg)) + return -EFAULT; + + status = mnt_want_write_file(filp); + if (status) + return status; + status = ocfs2_group_extend(inode, new_clusters); + mnt_drop_write_file(filp); + return status; + } + case OCFS2_IOC_GROUP_ADD: + case OCFS2_IOC_GROUP_ADD64: + { + struct ocfs2_new_group_input input; + + if (!capable(CAP_SYS_RESOURCE)) + return -EPERM; + + if (copy_from_user(&input, (int __user *) arg, sizeof(input))) + return -EFAULT; + + status = mnt_want_write_file(filp); + if (status) + return status; + status = ocfs2_group_add(inode, &input); + mnt_drop_write_file(filp); + return status; + } + case OCFS2_IOC_REFLINK: + { + struct reflink_arguments args; + const char __user *old_path; + const char __user *new_path; + bool preserve; + + if (copy_from_user(&args, argp, sizeof(args))) + return -EFAULT; + old_path = (const char __user *)(unsigned long)args.old_path; + new_path = (const char __user *)(unsigned long)args.new_path; + preserve = (args.preserve != 0); + + return ocfs2_reflink_ioctl(inode, old_path, new_path, preserve); + } + case OCFS2_IOC_INFO: + { + struct ocfs2_info info; + + if (copy_from_user(&info, argp, sizeof(struct ocfs2_info))) + return -EFAULT; + + return ocfs2_info_handle(inode, &info, 0); + } + case FITRIM: + { + struct super_block *sb = inode->i_sb; + struct fstrim_range range; + int ret = 0; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (!bdev_max_discard_sectors(sb->s_bdev)) + return -EOPNOTSUPP; + + if (copy_from_user(&range, argp, sizeof(range))) + return -EFAULT; + + range.minlen = max_t(u64, bdev_discard_granularity(sb->s_bdev), + range.minlen); + ret = ocfs2_trim_fs(sb, &range); + if (ret < 0) + return ret; + + if (copy_to_user(argp, &range, sizeof(range))) + return -EFAULT; + + return 0; + } + case OCFS2_IOC_MOVE_EXT: + return ocfs2_ioctl_move_extents(filp, argp); + default: + return -ENOTTY; + } +} + +#ifdef CONFIG_COMPAT +long ocfs2_compat_ioctl(struct file *file, unsigned cmd, unsigned long arg) +{ + bool preserve; + struct reflink_arguments args; + struct inode *inode = file_inode(file); + struct ocfs2_info info; + void __user *argp = (void __user *)arg; + + switch (cmd) { + case OCFS2_IOC_RESVSP: + case OCFS2_IOC_RESVSP64: + case OCFS2_IOC_UNRESVSP: + case OCFS2_IOC_UNRESVSP64: + case OCFS2_IOC_GROUP_EXTEND: + case OCFS2_IOC_GROUP_ADD: + case OCFS2_IOC_GROUP_ADD64: + break; + case OCFS2_IOC_REFLINK: + if (copy_from_user(&args, argp, sizeof(args))) + return -EFAULT; + preserve = (args.preserve != 0); + + return ocfs2_reflink_ioctl(inode, compat_ptr(args.old_path), + compat_ptr(args.new_path), preserve); + case OCFS2_IOC_INFO: + if (copy_from_user(&info, argp, sizeof(struct ocfs2_info))) + return -EFAULT; + + return ocfs2_info_handle(inode, &info, 1); + case FITRIM: + case OCFS2_IOC_MOVE_EXT: + break; + default: + return -ENOIOCTLCMD; + } + + return ocfs2_ioctl(file, cmd, arg); +} +#endif diff --git a/fs/ocfs2/ioctl.h b/fs/ocfs2/ioctl.h new file mode 100644 index 000000000..4a1c2313b --- /dev/null +++ b/fs/ocfs2/ioctl.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * ioctl.h + * + * Function prototypes + * + * Copyright (C) 2006 Herbert Poetzl + * + */ + +#ifndef OCFS2_IOCTL_PROTO_H +#define OCFS2_IOCTL_PROTO_H + +int ocfs2_fileattr_get(struct dentry *dentry, struct file_kattr *fa); +int ocfs2_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa); +long ocfs2_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); +long ocfs2_compat_ioctl(struct file *file, unsigned cmd, unsigned long arg); + +#endif /* OCFS2_IOCTL_PROTO_H */ diff --git a/fs/ocfs2/journal.c b/fs/ocfs2/journal.c new file mode 100644 index 000000000..d8afbc1a7 --- /dev/null +++ b/fs/ocfs2/journal.c @@ -0,0 +1,2480 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * journal.c + * + * Defines functions of journalling api + * + * Copyright (C) 2003, 2004 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ocfs2.h" + +#include "alloc.h" +#include "blockcheck.h" +#include "dir.h" +#include "dlmglue.h" +#include "extent_map.h" +#include "heartbeat.h" +#include "inode.h" +#include "journal.h" +#include "localalloc.h" +#include "slot_map.h" +#include "super.h" +#include "sysfile.h" +#include "uptodate.h" +#include "quota.h" +#include "file.h" +#include "namei.h" + +#include "buffer_head_io.h" +#include "ocfs2_trace.h" + +DEFINE_SPINLOCK(trans_inc_lock); + +#define ORPHAN_SCAN_SCHEDULE_TIMEOUT 300000 + +static int ocfs2_force_read_journal(struct inode *inode); +static int ocfs2_recover_node(struct ocfs2_super *osb, + int node_num, int slot_num); +static int __ocfs2_recovery_thread(void *arg); +static int ocfs2_commit_cache(struct ocfs2_super *osb); +static int __ocfs2_wait_on_mount(struct ocfs2_super *osb, int quota); +static int ocfs2_journal_toggle_dirty(struct ocfs2_super *osb, + int dirty, int replayed); +static int ocfs2_trylock_journal(struct ocfs2_super *osb, + int slot_num); +static int ocfs2_recover_orphans(struct ocfs2_super *osb, + int slot, + enum ocfs2_orphan_reco_type orphan_reco_type); +static int ocfs2_commit_thread(void *arg); +static void ocfs2_queue_recovery_completion(struct ocfs2_journal *journal, + int slot_num, + struct ocfs2_dinode *la_dinode, + struct ocfs2_dinode *tl_dinode, + struct ocfs2_quota_recovery *qrec, + enum ocfs2_orphan_reco_type orphan_reco_type); + +static inline int ocfs2_wait_on_mount(struct ocfs2_super *osb) +{ + return __ocfs2_wait_on_mount(osb, 0); +} + +static inline int ocfs2_wait_on_quotas(struct ocfs2_super *osb) +{ + return __ocfs2_wait_on_mount(osb, 1); +} + +/* + * This replay_map is to track online/offline slots, so we could recover + * offline slots during recovery and mount + */ + +enum ocfs2_replay_state { + REPLAY_UNNEEDED = 0, /* Replay is not needed, so ignore this map */ + REPLAY_NEEDED, /* Replay slots marked in rm_replay_slots */ + REPLAY_DONE /* Replay was already queued */ +}; + +struct ocfs2_replay_map { + unsigned int rm_slots; + enum ocfs2_replay_state rm_state; + unsigned char rm_replay_slots[] __counted_by(rm_slots); +}; + +static void ocfs2_replay_map_set_state(struct ocfs2_super *osb, int state) +{ + if (!osb->replay_map) + return; + + /* If we've already queued the replay, we don't have any more to do */ + if (osb->replay_map->rm_state == REPLAY_DONE) + return; + + osb->replay_map->rm_state = state; +} + +int ocfs2_compute_replay_slots(struct ocfs2_super *osb) +{ + struct ocfs2_replay_map *replay_map; + int i, node_num; + + /* If replay map is already set, we don't do it again */ + if (osb->replay_map) + return 0; + + replay_map = kzalloc_flex(*replay_map, rm_replay_slots, osb->max_slots); + if (!replay_map) { + mlog_errno(-ENOMEM); + return -ENOMEM; + } + + spin_lock(&osb->osb_lock); + + replay_map->rm_slots = osb->max_slots; + replay_map->rm_state = REPLAY_UNNEEDED; + + /* set rm_replay_slots for offline slot(s) */ + for (i = 0; i < replay_map->rm_slots; i++) { + if (ocfs2_slot_to_node_num_locked(osb, i, &node_num) == -ENOENT) + replay_map->rm_replay_slots[i] = 1; + } + + osb->replay_map = replay_map; + spin_unlock(&osb->osb_lock); + return 0; +} + +static void ocfs2_queue_replay_slots(struct ocfs2_super *osb, + enum ocfs2_orphan_reco_type orphan_reco_type) +{ + struct ocfs2_replay_map *replay_map = osb->replay_map; + int i; + + if (!replay_map) + return; + + if (replay_map->rm_state != REPLAY_NEEDED) + return; + + for (i = 0; i < replay_map->rm_slots; i++) + if (replay_map->rm_replay_slots[i]) + ocfs2_queue_recovery_completion(osb->journal, i, NULL, + NULL, NULL, + orphan_reco_type); + replay_map->rm_state = REPLAY_DONE; +} + +void ocfs2_free_replay_slots(struct ocfs2_super *osb) +{ + struct ocfs2_replay_map *replay_map = osb->replay_map; + + if (!osb->replay_map) + return; + + kfree(replay_map); + osb->replay_map = NULL; +} + +int ocfs2_recovery_init(struct ocfs2_super *osb) +{ + struct ocfs2_recovery_map *rm; + + mutex_init(&osb->recovery_lock); + osb->recovery_state = OCFS2_REC_ENABLED; + osb->recovery_thread_task = NULL; + init_waitqueue_head(&osb->recovery_event); + + rm = kzalloc_flex(*rm, rm_entries, osb->max_slots); + if (!rm) { + mlog_errno(-ENOMEM); + return -ENOMEM; + } + + osb->recovery_map = rm; + + return 0; +} + +static int ocfs2_recovery_thread_running(struct ocfs2_super *osb) +{ + return osb->recovery_thread_task != NULL; +} + +static void ocfs2_recovery_disable(struct ocfs2_super *osb, + enum ocfs2_recovery_state state) +{ + mutex_lock(&osb->recovery_lock); + /* + * If recovery thread is not running, we can directly transition to + * final state. + */ + if (!ocfs2_recovery_thread_running(osb)) { + osb->recovery_state = state + 1; + goto out_lock; + } + osb->recovery_state = state; + /* Wait for recovery thread to acknowledge state transition */ + wait_event_cmd(osb->recovery_event, + !ocfs2_recovery_thread_running(osb) || + osb->recovery_state >= state + 1, + mutex_unlock(&osb->recovery_lock), + mutex_lock(&osb->recovery_lock)); +out_lock: + mutex_unlock(&osb->recovery_lock); + + /* + * At this point we know that no more recovery work can be queued so + * wait for any recovery completion work to complete. + */ + if (osb->ocfs2_wq) + flush_workqueue(osb->ocfs2_wq); +} + +void ocfs2_recovery_disable_quota(struct ocfs2_super *osb) +{ + ocfs2_recovery_disable(osb, OCFS2_REC_QUOTA_WANT_DISABLE); +} + +void ocfs2_recovery_exit(struct ocfs2_super *osb) +{ + struct ocfs2_recovery_map *rm; + + /* disable any new recovery threads and wait for any currently + * running ones to exit. Do this before setting the vol_state. */ + ocfs2_recovery_disable(osb, OCFS2_REC_WANT_DISABLE); + + /* + * Now that recovery is shut down, and the osb is about to be + * freed, the osb_lock is not taken here. + */ + rm = osb->recovery_map; + /* XXX: Should we bug if there are dirty entries? */ + + kfree(rm); +} + +static int __ocfs2_recovery_map_test(struct ocfs2_super *osb, + unsigned int node_num) +{ + int i; + struct ocfs2_recovery_map *rm = osb->recovery_map; + + assert_spin_locked(&osb->osb_lock); + + for (i = 0; i < rm->rm_used; i++) { + if (rm->rm_entries[i] == node_num) + return 1; + } + + return 0; +} + +/* Behaves like test-and-set. Returns the previous value */ +static int ocfs2_recovery_map_set(struct ocfs2_super *osb, + unsigned int node_num) +{ + struct ocfs2_recovery_map *rm = osb->recovery_map; + + spin_lock(&osb->osb_lock); + if (__ocfs2_recovery_map_test(osb, node_num)) { + spin_unlock(&osb->osb_lock); + return 1; + } + + /* XXX: Can this be exploited? Not from o2dlm... */ + BUG_ON(rm->rm_used >= osb->max_slots); + + rm->rm_entries[rm->rm_used] = node_num; + rm->rm_used++; + spin_unlock(&osb->osb_lock); + + return 0; +} + +static void ocfs2_recovery_map_clear(struct ocfs2_super *osb, + unsigned int node_num) +{ + int i; + struct ocfs2_recovery_map *rm = osb->recovery_map; + + spin_lock(&osb->osb_lock); + + for (i = 0; i < rm->rm_used; i++) { + if (rm->rm_entries[i] == node_num) + break; + } + + if (i < rm->rm_used) { + /* XXX: be careful with the pointer math */ + memmove(&(rm->rm_entries[i]), &(rm->rm_entries[i + 1]), + (rm->rm_used - i - 1) * sizeof(unsigned int)); + rm->rm_used--; + } + + spin_unlock(&osb->osb_lock); +} + +static int ocfs2_commit_cache(struct ocfs2_super *osb) +{ + int status = 0; + unsigned int flushed; + struct ocfs2_journal *journal = NULL; + + journal = osb->journal; + + /* Flush all pending commits and checkpoint the journal. */ + down_write(&journal->j_trans_barrier); + + flushed = atomic_read(&journal->j_num_trans); + trace_ocfs2_commit_cache_begin(flushed); + if (flushed == 0) { + up_write(&journal->j_trans_barrier); + goto finally; + } + + jbd2_journal_lock_updates(journal->j_journal); + status = jbd2_journal_flush(journal->j_journal, 0); + jbd2_journal_unlock_updates(journal->j_journal); + if (status < 0) { + up_write(&journal->j_trans_barrier); + mlog_errno(status); + goto finally; + } + + ocfs2_inc_trans_id(journal); + + flushed = atomic_read(&journal->j_num_trans); + atomic_set(&journal->j_num_trans, 0); + up_write(&journal->j_trans_barrier); + + trace_ocfs2_commit_cache_end(journal->j_trans_id, flushed); + + ocfs2_wake_downconvert_thread(osb); + wake_up(&journal->j_checkpointed); +finally: + return status; +} + +handle_t *ocfs2_start_trans(struct ocfs2_super *osb, int max_buffs) +{ + journal_t *journal = osb->journal->j_journal; + handle_t *handle; + + BUG_ON(!osb || !osb->journal->j_journal); + + if (ocfs2_is_hard_readonly(osb)) + return ERR_PTR(-EROFS); + + BUG_ON(osb->journal->j_state == OCFS2_JOURNAL_FREE); + BUG_ON(max_buffs <= 0); + + /* Nested transaction? Just return the handle... */ + if (journal_current_handle()) + return jbd2_journal_start(journal, max_buffs); + + sb_start_intwrite(osb->sb); + + down_read(&osb->journal->j_trans_barrier); + + handle = jbd2_journal_start(journal, max_buffs); + if (IS_ERR(handle)) { + up_read(&osb->journal->j_trans_barrier); + sb_end_intwrite(osb->sb); + + mlog_errno(PTR_ERR(handle)); + + if (is_journal_aborted(journal)) { + ocfs2_abort(osb->sb, "Detected aborted journal\n"); + handle = ERR_PTR(-EROFS); + } + } else { + if (!ocfs2_mount_local(osb)) + atomic_inc(&(osb->journal->j_num_trans)); + } + + return handle; +} + +int ocfs2_commit_trans(struct ocfs2_super *osb, + handle_t *handle) +{ + int ret, nested; + struct ocfs2_journal *journal = osb->journal; + + BUG_ON(!handle); + + nested = handle->h_ref > 1; + ret = jbd2_journal_stop(handle); + if (ret < 0) + mlog_errno(ret); + + if (!nested) { + up_read(&journal->j_trans_barrier); + sb_end_intwrite(osb->sb); + } + + return ret; +} + +/* + * 'nblocks' is what you want to add to the current transaction. + * + * This might call jbd2_journal_restart() which will commit dirty buffers + * and then restart the transaction. Before calling + * ocfs2_extend_trans(), any changed blocks should have been + * dirtied. After calling it, all blocks which need to be changed must + * go through another set of journal_access/journal_dirty calls. + * + * WARNING: This will not release any semaphores or disk locks taken + * during the transaction, so make sure they were taken *before* + * start_trans or we'll have ordering deadlocks. + * + * WARNING2: Note that we do *not* drop j_trans_barrier here. This is + * good because transaction ids haven't yet been recorded on the + * cluster locks associated with this handle. + */ +int ocfs2_extend_trans(handle_t *handle, int nblocks) +{ + int status, old_nblocks; + + BUG_ON(!handle); + BUG_ON(nblocks < 0); + + if (!nblocks) + return 0; + + old_nblocks = jbd2_handle_buffer_credits(handle); + + trace_ocfs2_extend_trans(old_nblocks, nblocks); + +#ifdef CONFIG_OCFS2_DEBUG_FS + status = 1; +#else + status = jbd2_journal_extend(handle, nblocks, 0); + if (status < 0) { + mlog_errno(status); + goto bail; + } +#endif + + if (status > 0) { + trace_ocfs2_extend_trans_restart(old_nblocks + nblocks); + status = jbd2_journal_restart(handle, + old_nblocks + nblocks); + if (status < 0) { + mlog_errno(status); + goto bail; + } + } + + status = 0; +bail: + return status; +} + +/* + * Make sure handle has at least 'nblocks' credits available. If it does not + * have that many credits available, we will try to extend the handle to have + * enough credits. If that fails, we will restart transaction to have enough + * credits. Similar notes regarding data consistency and locking implications + * as for ocfs2_extend_trans() apply here. + */ +int ocfs2_assure_trans_credits(handle_t *handle, int nblocks) +{ + int old_nblks; + + if (is_handle_aborted(handle)) + return -EROFS; + + old_nblks = jbd2_handle_buffer_credits(handle); + trace_ocfs2_assure_trans_credits(old_nblks); + if (old_nblks >= nblocks) + return 0; + return ocfs2_extend_trans(handle, nblocks - old_nblks); +} + +/* + * If we have fewer than thresh credits, extend by OCFS2_MAX_TRANS_DATA. + * If that fails, restart the transaction & regain write access for the + * buffer head which is used for metadata modifications. + * Taken from Ext4: extend_or_restart_transaction() + */ +int ocfs2_allocate_extend_trans(handle_t *handle, int thresh) +{ + int status, old_nblks; + + BUG_ON(!handle); + + old_nblks = jbd2_handle_buffer_credits(handle); + trace_ocfs2_allocate_extend_trans(old_nblks, thresh); + + if (old_nblks < thresh) + return 0; + + status = jbd2_journal_extend(handle, OCFS2_MAX_TRANS_DATA, 0); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + if (status > 0) { + status = jbd2_journal_restart(handle, OCFS2_MAX_TRANS_DATA); + if (status < 0) + mlog_errno(status); + } + +bail: + return status; +} + +static inline struct ocfs2_triggers *to_ocfs2_trigger(struct jbd2_buffer_trigger_type *triggers) +{ + return container_of(triggers, struct ocfs2_triggers, ot_triggers); +} + +static void ocfs2_frozen_trigger(struct jbd2_buffer_trigger_type *triggers, + struct buffer_head *bh, + void *data, size_t size) +{ + struct ocfs2_triggers *ot = to_ocfs2_trigger(triggers); + + /* + * We aren't guaranteed to have the superblock here, so we + * must unconditionally compute the ecc data. + * __ocfs2_journal_access() will only set the triggers if + * metaecc is enabled. + */ + ocfs2_block_check_compute(data, size, data + ot->ot_offset); +} + +/* + * Quota blocks have their own trigger because the struct ocfs2_block_check + * offset depends on the blocksize. + */ +static void ocfs2_dq_frozen_trigger(struct jbd2_buffer_trigger_type *triggers, + struct buffer_head *bh, + void *data, size_t size) +{ + struct ocfs2_disk_dqtrailer *dqt = + ocfs2_block_dqtrailer(size, data); + + /* + * We aren't guaranteed to have the superblock here, so we + * must unconditionally compute the ecc data. + * __ocfs2_journal_access() will only set the triggers if + * metaecc is enabled. + */ + ocfs2_block_check_compute(data, size, &dqt->dq_check); +} + +/* + * Directory blocks also have their own trigger because the + * struct ocfs2_block_check offset depends on the blocksize. + */ +static void ocfs2_db_frozen_trigger(struct jbd2_buffer_trigger_type *triggers, + struct buffer_head *bh, + void *data, size_t size) +{ + struct ocfs2_dir_block_trailer *trailer = + ocfs2_dir_trailer_from_size(size, data); + + /* + * We aren't guaranteed to have the superblock here, so we + * must unconditionally compute the ecc data. + * __ocfs2_journal_access() will only set the triggers if + * metaecc is enabled. + */ + ocfs2_block_check_compute(data, size, &trailer->db_check); +} + +static void ocfs2_abort_trigger(struct jbd2_buffer_trigger_type *triggers, + struct buffer_head *bh) +{ + struct ocfs2_triggers *ot = to_ocfs2_trigger(triggers); + + mlog(ML_ERROR, + "ocfs2_abort_trigger called by JBD2. bh = 0x%lx, " + "bh->b_blocknr = %llu\n", + (unsigned long)bh, + (unsigned long long)bh->b_blocknr); + + ocfs2_error(ot->sb, + "JBD2 has aborted our journal, ocfs2 cannot continue\n"); +} + +static void ocfs2_setup_csum_triggers(struct super_block *sb, + enum ocfs2_journal_trigger_type type, + struct ocfs2_triggers *ot) +{ + BUG_ON(type >= OCFS2_JOURNAL_TRIGGER_COUNT); + + switch (type) { + case OCFS2_JTR_DI: + ot->ot_triggers.t_frozen = ocfs2_frozen_trigger; + ot->ot_offset = offsetof(struct ocfs2_dinode, i_check); + break; + case OCFS2_JTR_EB: + ot->ot_triggers.t_frozen = ocfs2_frozen_trigger; + ot->ot_offset = offsetof(struct ocfs2_extent_block, h_check); + break; + case OCFS2_JTR_RB: + ot->ot_triggers.t_frozen = ocfs2_frozen_trigger; + ot->ot_offset = offsetof(struct ocfs2_refcount_block, rf_check); + break; + case OCFS2_JTR_GD: + ot->ot_triggers.t_frozen = ocfs2_frozen_trigger; + ot->ot_offset = offsetof(struct ocfs2_group_desc, bg_check); + break; + case OCFS2_JTR_DB: + ot->ot_triggers.t_frozen = ocfs2_db_frozen_trigger; + break; + case OCFS2_JTR_XB: + ot->ot_triggers.t_frozen = ocfs2_frozen_trigger; + ot->ot_offset = offsetof(struct ocfs2_xattr_block, xb_check); + break; + case OCFS2_JTR_DQ: + ot->ot_triggers.t_frozen = ocfs2_dq_frozen_trigger; + break; + case OCFS2_JTR_DR: + ot->ot_triggers.t_frozen = ocfs2_frozen_trigger; + ot->ot_offset = offsetof(struct ocfs2_dx_root_block, dr_check); + break; + case OCFS2_JTR_DL: + ot->ot_triggers.t_frozen = ocfs2_frozen_trigger; + ot->ot_offset = offsetof(struct ocfs2_dx_leaf, dl_check); + break; + case OCFS2_JTR_NONE: + /* To make compiler happy... */ + return; + } + + ot->ot_triggers.t_abort = ocfs2_abort_trigger; + ot->sb = sb; +} + +void ocfs2_initialize_journal_triggers(struct super_block *sb, + struct ocfs2_triggers triggers[]) +{ + enum ocfs2_journal_trigger_type type; + + for (type = OCFS2_JTR_DI; type < OCFS2_JOURNAL_TRIGGER_COUNT; type++) + ocfs2_setup_csum_triggers(sb, type, &triggers[type]); +} + +static int __ocfs2_journal_access(handle_t *handle, + struct ocfs2_caching_info *ci, + struct buffer_head *bh, + struct ocfs2_triggers *triggers, + int type) +{ + int status; + struct ocfs2_super *osb = + OCFS2_SB(ocfs2_metadata_cache_get_super(ci)); + + BUG_ON(!ci || !ci->ci_ops); + BUG_ON(!handle); + BUG_ON(!bh); + + trace_ocfs2_journal_access( + (unsigned long long)ocfs2_metadata_cache_owner(ci), + (unsigned long long)bh->b_blocknr, type, bh->b_size); + + /* we can safely remove this assertion after testing. */ + if (!buffer_uptodate(bh)) { + mlog(ML_ERROR, "giving me a buffer that's not uptodate!\n"); + mlog(ML_ERROR, "b_blocknr=%llu, b_state=0x%lx\n", + (unsigned long long)bh->b_blocknr, bh->b_state); + + lock_buffer(bh); + /* + * A previous transaction with a couple of buffer heads fail + * to checkpoint, so all the bhs are marked as BH_Write_EIO. + * For current transaction, the bh is just among those error + * bhs which previous transaction handle. We can't just clear + * its BH_Write_EIO and reuse directly, since other bhs are + * not written to disk yet and that will cause metadata + * inconsistency. So we should set fs read-only to avoid + * further damage. + */ + if (buffer_write_io_error(bh) && !buffer_uptodate(bh)) { + unlock_buffer(bh); + return ocfs2_error(osb->sb, "A previous attempt to " + "write this buffer head failed\n"); + } + unlock_buffer(bh); + } + + /* Set the current transaction information on the ci so + * that the locking code knows whether it can drop it's locks + * on this ci or not. We're protected from the commit + * thread updating the current transaction id until + * ocfs2_commit_trans() because ocfs2_start_trans() took + * j_trans_barrier for us. */ + ocfs2_set_ci_lock_trans(osb->journal, ci); + + ocfs2_metadata_cache_io_lock(ci); + switch (type) { + case OCFS2_JOURNAL_ACCESS_CREATE: + case OCFS2_JOURNAL_ACCESS_WRITE: + status = jbd2_journal_get_write_access(handle, bh); + break; + + case OCFS2_JOURNAL_ACCESS_UNDO: + status = jbd2_journal_get_undo_access(handle, bh); + break; + + default: + status = -EINVAL; + mlog(ML_ERROR, "Unknown access type!\n"); + } + if (!status && ocfs2_meta_ecc(osb) && triggers) + jbd2_journal_set_triggers(bh, &triggers->ot_triggers); + ocfs2_metadata_cache_io_unlock(ci); + + if (status < 0) + mlog(ML_ERROR, "Error %d getting %d access to buffer!\n", + status, type); + + return status; +} + +int ocfs2_journal_access_di(handle_t *handle, struct ocfs2_caching_info *ci, + struct buffer_head *bh, int type) +{ + struct ocfs2_super *osb = OCFS2_SB(ocfs2_metadata_cache_get_super(ci)); + + return __ocfs2_journal_access(handle, ci, bh, + &osb->s_journal_triggers[OCFS2_JTR_DI], + type); +} + +int ocfs2_journal_access_eb(handle_t *handle, struct ocfs2_caching_info *ci, + struct buffer_head *bh, int type) +{ + struct ocfs2_super *osb = OCFS2_SB(ocfs2_metadata_cache_get_super(ci)); + + return __ocfs2_journal_access(handle, ci, bh, + &osb->s_journal_triggers[OCFS2_JTR_EB], + type); +} + +int ocfs2_journal_access_rb(handle_t *handle, struct ocfs2_caching_info *ci, + struct buffer_head *bh, int type) +{ + struct ocfs2_super *osb = OCFS2_SB(ocfs2_metadata_cache_get_super(ci)); + + return __ocfs2_journal_access(handle, ci, bh, + &osb->s_journal_triggers[OCFS2_JTR_RB], + type); +} + +int ocfs2_journal_access_gd(handle_t *handle, struct ocfs2_caching_info *ci, + struct buffer_head *bh, int type) +{ + struct ocfs2_super *osb = OCFS2_SB(ocfs2_metadata_cache_get_super(ci)); + + return __ocfs2_journal_access(handle, ci, bh, + &osb->s_journal_triggers[OCFS2_JTR_GD], + type); +} + +int ocfs2_journal_access_db(handle_t *handle, struct ocfs2_caching_info *ci, + struct buffer_head *bh, int type) +{ + struct ocfs2_super *osb = OCFS2_SB(ocfs2_metadata_cache_get_super(ci)); + + return __ocfs2_journal_access(handle, ci, bh, + &osb->s_journal_triggers[OCFS2_JTR_DB], + type); +} + +int ocfs2_journal_access_xb(handle_t *handle, struct ocfs2_caching_info *ci, + struct buffer_head *bh, int type) +{ + struct ocfs2_super *osb = OCFS2_SB(ocfs2_metadata_cache_get_super(ci)); + + return __ocfs2_journal_access(handle, ci, bh, + &osb->s_journal_triggers[OCFS2_JTR_XB], + type); +} + +int ocfs2_journal_access_dq(handle_t *handle, struct ocfs2_caching_info *ci, + struct buffer_head *bh, int type) +{ + struct ocfs2_super *osb = OCFS2_SB(ocfs2_metadata_cache_get_super(ci)); + + return __ocfs2_journal_access(handle, ci, bh, + &osb->s_journal_triggers[OCFS2_JTR_DQ], + type); +} + +int ocfs2_journal_access_dr(handle_t *handle, struct ocfs2_caching_info *ci, + struct buffer_head *bh, int type) +{ + struct ocfs2_super *osb = OCFS2_SB(ocfs2_metadata_cache_get_super(ci)); + + return __ocfs2_journal_access(handle, ci, bh, + &osb->s_journal_triggers[OCFS2_JTR_DR], + type); +} + +int ocfs2_journal_access_dl(handle_t *handle, struct ocfs2_caching_info *ci, + struct buffer_head *bh, int type) +{ + struct ocfs2_super *osb = OCFS2_SB(ocfs2_metadata_cache_get_super(ci)); + + return __ocfs2_journal_access(handle, ci, bh, + &osb->s_journal_triggers[OCFS2_JTR_DL], + type); +} + +int ocfs2_journal_access(handle_t *handle, struct ocfs2_caching_info *ci, + struct buffer_head *bh, int type) +{ + return __ocfs2_journal_access(handle, ci, bh, NULL, type); +} + +void ocfs2_journal_dirty(handle_t *handle, struct buffer_head *bh) +{ + int status; + + trace_ocfs2_journal_dirty((unsigned long long)bh->b_blocknr); + + status = jbd2_journal_dirty_metadata(handle, bh); + if (status) { + mlog_errno(status); + if (!is_handle_aborted(handle)) { + journal_t *journal = handle->h_transaction->t_journal; + + mlog(ML_ERROR, "jbd2_journal_dirty_metadata failed: " + "handle type %u started at line %u, credits %u/%u " + "errcode %d. Aborting transaction and journal.\n", + handle->h_type, handle->h_line_no, + handle->h_requested_credits, + jbd2_handle_buffer_credits(handle), status); + handle->h_err = status; + jbd2_journal_abort_handle(handle); + jbd2_journal_abort(journal, status); + } + } +} + +#define OCFS2_DEFAULT_COMMIT_INTERVAL (HZ * JBD2_DEFAULT_MAX_COMMIT_AGE) + +void ocfs2_set_journal_params(struct ocfs2_super *osb) +{ + journal_t *journal = osb->journal->j_journal; + unsigned long commit_interval = OCFS2_DEFAULT_COMMIT_INTERVAL; + + if (osb->osb_commit_interval) + commit_interval = osb->osb_commit_interval; + + write_lock(&journal->j_state_lock); + journal->j_commit_interval = commit_interval; + if (osb->s_mount_opt & OCFS2_MOUNT_BARRIER) + journal->j_flags |= JBD2_BARRIER; + else + journal->j_flags &= ~JBD2_BARRIER; + write_unlock(&journal->j_state_lock); +} + +/* + * alloc & initialize skeleton for journal structure. + * ocfs2_journal_init() will make fs have journal ability. + */ +int ocfs2_journal_alloc(struct ocfs2_super *osb) +{ + int status = 0; + struct ocfs2_journal *journal; + + journal = kzalloc_obj(struct ocfs2_journal); + if (!journal) { + mlog(ML_ERROR, "unable to alloc journal\n"); + status = -ENOMEM; + goto bail; + } + osb->journal = journal; + journal->j_osb = osb; + + atomic_set(&journal->j_num_trans, 0); + init_rwsem(&journal->j_trans_barrier); + init_waitqueue_head(&journal->j_checkpointed); + spin_lock_init(&journal->j_lock); + journal->j_trans_id = 1UL; + INIT_LIST_HEAD(&journal->j_la_cleanups); + INIT_WORK(&journal->j_recovery_work, ocfs2_complete_recovery); + journal->j_state = OCFS2_JOURNAL_FREE; + +bail: + return status; +} + +static int ocfs2_journal_submit_inode_data_buffers(struct jbd2_inode *jinode) +{ + struct address_space *mapping = jinode->i_vfs_inode->i_mapping; + loff_t range_start, range_end; + + if (!jbd2_jinode_get_dirty_range(jinode, &range_start, &range_end)) + return 0; + + return filemap_fdatawrite_range(mapping, range_start, range_end); +} + +int ocfs2_journal_init(struct ocfs2_super *osb, int *dirty) +{ + int status = -1; + struct inode *inode = NULL; /* the journal inode */ + journal_t *j_journal = NULL; + struct ocfs2_journal *journal = osb->journal; + struct ocfs2_dinode *di = NULL; + struct buffer_head *bh = NULL; + int inode_lock = 0; + + BUG_ON(!journal); + /* already have the inode for our journal */ + inode = ocfs2_get_system_file_inode(osb, JOURNAL_SYSTEM_INODE, + osb->slot_num); + if (inode == NULL) { + status = -EACCES; + mlog_errno(status); + goto done; + } + if (is_bad_inode(inode)) { + mlog(ML_ERROR, "access error (bad inode)\n"); + iput(inode); + inode = NULL; + status = -EACCES; + goto done; + } + + SET_INODE_JOURNAL(inode); + OCFS2_I(inode)->ip_open_count++; + + /* Skip recovery waits here - journal inode metadata never + * changes in a live cluster so it can be considered an + * exception to the rule. */ + status = ocfs2_inode_lock_full(inode, &bh, 1, OCFS2_META_LOCK_RECOVERY); + if (status < 0) { + if (status != -ERESTARTSYS) + mlog(ML_ERROR, "Could not get lock on journal!\n"); + goto done; + } + + inode_lock = 1; + di = (struct ocfs2_dinode *)bh->b_data; + + if (i_size_read(inode) < OCFS2_MIN_JOURNAL_SIZE) { + mlog(ML_ERROR, "Journal file size (%lld) is too small!\n", + i_size_read(inode)); + status = -EINVAL; + goto done; + } + + trace_ocfs2_journal_init(i_size_read(inode), + (unsigned long long)inode->i_blocks, + OCFS2_I(inode)->ip_clusters); + + /* call the kernels journal init function now */ + j_journal = jbd2_journal_init_inode(inode); + if (IS_ERR(j_journal)) { + mlog(ML_ERROR, "Linux journal layer error\n"); + status = PTR_ERR(j_journal); + goto done; + } + + trace_ocfs2_journal_init_maxlen(j_journal->j_total_len); + + *dirty = (le32_to_cpu(di->id1.journal1.ij_flags) & + OCFS2_JOURNAL_DIRTY_FL); + + journal->j_journal = j_journal; + journal->j_journal->j_submit_inode_data_buffers = + ocfs2_journal_submit_inode_data_buffers; + journal->j_journal->j_finish_inode_data_buffers = + jbd2_journal_finish_inode_data_buffers; + journal->j_inode = inode; + journal->j_bh = bh; + + ocfs2_set_journal_params(osb); + + journal->j_state = OCFS2_JOURNAL_LOADED; + + status = 0; +done: + if (status < 0) { + if (inode_lock) + ocfs2_inode_unlock(inode, 1); + brelse(bh); + if (inode) { + OCFS2_I(inode)->ip_open_count--; + iput(inode); + } + } + + return status; +} + +static void ocfs2_bump_recovery_generation(struct ocfs2_dinode *di) +{ + le32_add_cpu(&(di->id1.journal1.ij_recovery_generation), 1); +} + +static u32 ocfs2_get_recovery_generation(struct ocfs2_dinode *di) +{ + return le32_to_cpu(di->id1.journal1.ij_recovery_generation); +} + +static int ocfs2_journal_toggle_dirty(struct ocfs2_super *osb, + int dirty, int replayed) +{ + int status; + unsigned int flags; + struct ocfs2_journal *journal = osb->journal; + struct buffer_head *bh = journal->j_bh; + struct ocfs2_dinode *fe; + + fe = (struct ocfs2_dinode *)bh->b_data; + if (WARN_ON(!OCFS2_IS_VALID_DINODE(fe))) + return -EIO; + + flags = le32_to_cpu(fe->id1.journal1.ij_flags); + if (dirty) + flags |= OCFS2_JOURNAL_DIRTY_FL; + else + flags &= ~OCFS2_JOURNAL_DIRTY_FL; + fe->id1.journal1.ij_flags = cpu_to_le32(flags); + + if (replayed) + ocfs2_bump_recovery_generation(fe); + + ocfs2_compute_meta_ecc(osb->sb, bh->b_data, &fe->i_check); + status = ocfs2_write_block(osb, bh, INODE_CACHE(journal->j_inode)); + if (status < 0) + mlog_errno(status); + + return status; +} + +/* + * If the journal has been kmalloc'd it needs to be freed after this + * call. + */ +void ocfs2_journal_shutdown(struct ocfs2_super *osb) +{ + struct ocfs2_journal *journal = NULL; + int status = 0; + struct inode *inode = NULL; + int num_running_trans = 0; + + BUG_ON(!osb); + + journal = osb->journal; + if (!journal) + goto done; + + inode = journal->j_inode; + + if (journal->j_state != OCFS2_JOURNAL_LOADED) + goto done; + + /* need to inc inode use count - jbd2_journal_destroy will iput. */ + if (!igrab(inode)) + BUG(); + + num_running_trans = atomic_read(&(journal->j_num_trans)); + trace_ocfs2_journal_shutdown(num_running_trans); + + /* Do a commit_cache here. It will flush our journal, *and* + * release any locks that are still held. + * set the SHUTDOWN flag and release the trans lock. + * the commit thread will take the trans lock for us below. */ + journal->j_state = OCFS2_JOURNAL_IN_SHUTDOWN; + + /* The OCFS2_JOURNAL_IN_SHUTDOWN will signal to commit_cache to not + * drop the trans_lock (which we want to hold until we + * completely destroy the journal. */ + if (osb->commit_task) { + /* Wait for the commit thread */ + trace_ocfs2_journal_shutdown_wait(osb->commit_task); + kthread_stop(osb->commit_task); + osb->commit_task = NULL; + } + + BUG_ON(atomic_read(&(journal->j_num_trans)) != 0); + + if (ocfs2_mount_local(osb) && + (journal->j_journal->j_flags & JBD2_LOADED)) { + jbd2_journal_lock_updates(journal->j_journal); + status = jbd2_journal_flush(journal->j_journal, 0); + jbd2_journal_unlock_updates(journal->j_journal); + if (status < 0) + mlog_errno(status); + } + + /* Shutdown the kernel journal system */ + if (!jbd2_journal_destroy(journal->j_journal) && !status) { + /* + * Do not toggle if flush was unsuccessful otherwise + * will leave dirty metadata in a "clean" journal + */ + status = ocfs2_journal_toggle_dirty(osb, 0, 0); + if (status < 0) + mlog_errno(status); + } + journal->j_journal = NULL; + + OCFS2_I(inode)->ip_open_count--; + + /* unlock our journal */ + ocfs2_inode_unlock(inode, 1); + + brelse(journal->j_bh); + journal->j_bh = NULL; + + journal->j_state = OCFS2_JOURNAL_FREE; + +done: + iput(inode); + kfree(journal); + osb->journal = NULL; +} + +static void ocfs2_clear_journal_error(struct super_block *sb, + journal_t *journal, + int slot) +{ + int olderr; + + olderr = jbd2_journal_errno(journal); + if (olderr) { + mlog(ML_ERROR, "File system error %d recorded in " + "journal %u.\n", olderr, slot); + mlog(ML_ERROR, "File system on device %s needs checking.\n", + sb->s_id); + + jbd2_journal_ack_err(journal); + jbd2_journal_clear_err(journal); + } +} + +int ocfs2_journal_load(struct ocfs2_journal *journal, int local, int replayed) +{ + int status = 0; + struct ocfs2_super *osb; + + BUG_ON(!journal); + + osb = journal->j_osb; + + status = jbd2_journal_load(journal->j_journal); + if (status < 0) { + mlog(ML_ERROR, "Failed to load journal!\n"); + goto done; + } + + ocfs2_clear_journal_error(osb->sb, journal->j_journal, osb->slot_num); + + if (replayed) { + jbd2_journal_lock_updates(journal->j_journal); + status = jbd2_journal_flush(journal->j_journal, 0); + jbd2_journal_unlock_updates(journal->j_journal); + if (status < 0) + mlog_errno(status); + } + + status = ocfs2_journal_toggle_dirty(osb, 1, replayed); + if (status < 0) { + mlog_errno(status); + goto done; + } + + /* Launch the commit thread */ + if (!local) { + osb->commit_task = kthread_run(ocfs2_commit_thread, osb, + "ocfs2cmt-%s", osb->uuid_str); + if (IS_ERR(osb->commit_task)) { + status = PTR_ERR(osb->commit_task); + osb->commit_task = NULL; + mlog(ML_ERROR, "unable to launch ocfs2commit thread, " + "error=%d", status); + goto done; + } + } else + osb->commit_task = NULL; + +done: + return status; +} + + +/* 'full' flag tells us whether we clear out all blocks or if we just + * mark the journal clean */ +int ocfs2_journal_wipe(struct ocfs2_journal *journal, int full) +{ + int status; + + BUG_ON(!journal); + + status = jbd2_journal_wipe(journal->j_journal, full); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + status = ocfs2_journal_toggle_dirty(journal->j_osb, 0, 0); + if (status < 0) + mlog_errno(status); + +bail: + return status; +} + +static int ocfs2_recovery_completed(struct ocfs2_super *osb) +{ + int empty; + struct ocfs2_recovery_map *rm = osb->recovery_map; + + spin_lock(&osb->osb_lock); + empty = (rm->rm_used == 0); + spin_unlock(&osb->osb_lock); + + return empty; +} + +void ocfs2_wait_for_recovery(struct ocfs2_super *osb) +{ + wait_event(osb->recovery_event, ocfs2_recovery_completed(osb)); +} + +/* + * JBD Might read a cached version of another nodes journal file. We + * don't want this as this file changes often and we get no + * notification on those changes. The only way to be sure that we've + * got the most up to date version of those blocks then is to force + * read them off disk. Just searching through the buffer cache won't + * work as there may be pages backing this file which are still marked + * up to date. We know things can't change on this file underneath us + * as we have the lock by now :) + */ +static int ocfs2_force_read_journal(struct inode *inode) +{ + int status = 0; + int i; + u64 v_blkno, p_blkno, p_blocks, num_blocks; + struct buffer_head *bh = NULL; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + num_blocks = ocfs2_blocks_for_bytes(inode->i_sb, i_size_read(inode)); + v_blkno = 0; + while (v_blkno < num_blocks) { + status = ocfs2_extent_map_get_blocks(inode, v_blkno, + &p_blkno, &p_blocks, NULL); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + for (i = 0; i < p_blocks; i++, p_blkno++) { + bh = __find_get_block_nonatomic(osb->sb->s_bdev, p_blkno, + osb->sb->s_blocksize); + /* block not cached. */ + if (!bh) + continue; + + brelse(bh); + bh = NULL; + /* We are reading journal data which should not + * be put in the uptodate cache. + */ + status = ocfs2_read_blocks_sync(osb, p_blkno, 1, &bh); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + brelse(bh); + bh = NULL; + } + + v_blkno += p_blocks; + } + +bail: + return status; +} + +struct ocfs2_la_recovery_item { + struct list_head lri_list; + int lri_slot; + struct ocfs2_dinode *lri_la_dinode; + struct ocfs2_dinode *lri_tl_dinode; + struct ocfs2_quota_recovery *lri_qrec; + enum ocfs2_orphan_reco_type lri_orphan_reco_type; +}; + +/* Does the second half of the recovery process. By this point, the + * node is marked clean and can actually be considered recovered, + * hence it's no longer in the recovery map, but there's still some + * cleanup we can do which shouldn't happen within the recovery thread + * as locking in that context becomes very difficult if we are to take + * recovering nodes into account. + * + * NOTE: This function can and will sleep on recovery of other nodes + * during cluster locking, just like any other ocfs2 process. + */ +void ocfs2_complete_recovery(struct work_struct *work) +{ + int ret = 0; + struct ocfs2_journal *journal = + container_of(work, struct ocfs2_journal, j_recovery_work); + struct ocfs2_super *osb = journal->j_osb; + struct ocfs2_dinode *la_dinode, *tl_dinode; + struct ocfs2_la_recovery_item *item, *n; + struct ocfs2_quota_recovery *qrec; + enum ocfs2_orphan_reco_type orphan_reco_type; + LIST_HEAD(tmp_la_list); + + trace_ocfs2_complete_recovery( + (unsigned long long)OCFS2_I(journal->j_inode)->ip_blkno); + + spin_lock(&journal->j_lock); + list_splice_init(&journal->j_la_cleanups, &tmp_la_list); + spin_unlock(&journal->j_lock); + + list_for_each_entry_safe(item, n, &tmp_la_list, lri_list) { + list_del_init(&item->lri_list); + + ocfs2_wait_on_quotas(osb); + + la_dinode = item->lri_la_dinode; + tl_dinode = item->lri_tl_dinode; + qrec = item->lri_qrec; + orphan_reco_type = item->lri_orphan_reco_type; + + trace_ocfs2_complete_recovery_slot(item->lri_slot, + la_dinode ? le64_to_cpu(la_dinode->i_blkno) : 0, + tl_dinode ? le64_to_cpu(tl_dinode->i_blkno) : 0, + qrec); + + if (la_dinode) { + ret = ocfs2_complete_local_alloc_recovery(osb, + la_dinode); + if (ret < 0) + mlog_errno(ret); + + kfree(la_dinode); + } + + if (tl_dinode) { + ret = ocfs2_complete_truncate_log_recovery(osb, + tl_dinode); + if (ret < 0) + mlog_errno(ret); + + kfree(tl_dinode); + } + + ret = ocfs2_recover_orphans(osb, item->lri_slot, + orphan_reco_type); + if (ret < 0) + mlog_errno(ret); + + if (qrec) { + ret = ocfs2_finish_quota_recovery(osb, qrec, + item->lri_slot); + if (ret < 0) + mlog_errno(ret); + /* Recovery info is already freed now */ + } + + kfree(item); + } + + trace_ocfs2_complete_recovery_end(ret); +} + +/* NOTE: This function always eats your references to la_dinode and + * tl_dinode, either manually on error, or by passing them to + * ocfs2_complete_recovery */ +static void ocfs2_queue_recovery_completion(struct ocfs2_journal *journal, + int slot_num, + struct ocfs2_dinode *la_dinode, + struct ocfs2_dinode *tl_dinode, + struct ocfs2_quota_recovery *qrec, + enum ocfs2_orphan_reco_type orphan_reco_type) +{ + struct ocfs2_la_recovery_item *item; + + item = kmalloc_obj(struct ocfs2_la_recovery_item, GFP_NOFS); + if (!item) { + /* Though we wish to avoid it, we are in fact safe in + * skipping local alloc cleanup as fsck.ocfs2 is more + * than capable of reclaiming unused space. */ + kfree(la_dinode); + kfree(tl_dinode); + + if (qrec) + ocfs2_free_quota_recovery(qrec); + + mlog_errno(-ENOMEM); + return; + } + + INIT_LIST_HEAD(&item->lri_list); + item->lri_la_dinode = la_dinode; + item->lri_slot = slot_num; + item->lri_tl_dinode = tl_dinode; + item->lri_qrec = qrec; + item->lri_orphan_reco_type = orphan_reco_type; + + spin_lock(&journal->j_lock); + list_add_tail(&item->lri_list, &journal->j_la_cleanups); + queue_work(journal->j_osb->ocfs2_wq, &journal->j_recovery_work); + spin_unlock(&journal->j_lock); +} + +/* Called by the mount code to queue recovery the last part of + * recovery for it's own and offline slot(s). */ +void ocfs2_complete_mount_recovery(struct ocfs2_super *osb) +{ + struct ocfs2_journal *journal = osb->journal; + + if (ocfs2_is_hard_readonly(osb)) + return; + + /* No need to queue up our truncate_log as regular cleanup will catch + * that */ + ocfs2_queue_recovery_completion(journal, osb->slot_num, + osb->local_alloc_copy, NULL, NULL, + ORPHAN_NEED_TRUNCATE); + ocfs2_schedule_truncate_log_flush(osb, 0); + + osb->local_alloc_copy = NULL; + + /* queue to recover orphan slots for all offline slots */ + ocfs2_replay_map_set_state(osb, REPLAY_NEEDED); + ocfs2_queue_replay_slots(osb, ORPHAN_NEED_TRUNCATE); + ocfs2_free_replay_slots(osb); +} + +void ocfs2_complete_quota_recovery(struct ocfs2_super *osb) +{ + if (osb->quota_rec) { + ocfs2_queue_recovery_completion(osb->journal, + osb->slot_num, + NULL, + NULL, + osb->quota_rec, + ORPHAN_NEED_TRUNCATE); + osb->quota_rec = NULL; + } +} + +static int __ocfs2_recovery_thread(void *arg) +{ + int status, node_num, slot_num; + struct ocfs2_super *osb = arg; + struct ocfs2_recovery_map *rm = osb->recovery_map; + int *rm_quota = NULL; + int rm_quota_used = 0, i; + struct ocfs2_quota_recovery *qrec; + + /* Whether the quota supported. */ + int quota_enabled = OCFS2_HAS_RO_COMPAT_FEATURE(osb->sb, + OCFS2_FEATURE_RO_COMPAT_USRQUOTA) + || OCFS2_HAS_RO_COMPAT_FEATURE(osb->sb, + OCFS2_FEATURE_RO_COMPAT_GRPQUOTA); + + status = ocfs2_wait_on_mount(osb); + if (status < 0) { + goto bail; + } + + if (quota_enabled) { + rm_quota = kzalloc_objs(int, osb->max_slots, GFP_NOFS); + if (!rm_quota) { + status = -ENOMEM; + goto bail; + } + } +restart: + if (quota_enabled) { + mutex_lock(&osb->recovery_lock); + /* Confirm that recovery thread will no longer recover quotas */ + if (osb->recovery_state == OCFS2_REC_QUOTA_WANT_DISABLE) { + osb->recovery_state = OCFS2_REC_QUOTA_DISABLED; + wake_up(&osb->recovery_event); + } + if (osb->recovery_state >= OCFS2_REC_QUOTA_DISABLED) + quota_enabled = 0; + mutex_unlock(&osb->recovery_lock); + } + + status = ocfs2_super_lock(osb, 1); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + status = ocfs2_compute_replay_slots(osb); + if (status < 0) + mlog_errno(status); + + /* queue recovery for our own slot */ + ocfs2_queue_recovery_completion(osb->journal, osb->slot_num, NULL, + NULL, NULL, ORPHAN_NO_NEED_TRUNCATE); + + spin_lock(&osb->osb_lock); + while (rm->rm_used) { + /* It's always safe to remove entry zero, as we won't + * clear it until ocfs2_recover_node() has succeeded. */ + node_num = rm->rm_entries[0]; + spin_unlock(&osb->osb_lock); + slot_num = ocfs2_node_num_to_slot(osb, node_num); + trace_ocfs2_recovery_thread_node(node_num, slot_num); + if (slot_num == -ENOENT) { + status = 0; + goto skip_recovery; + } + + /* It is a bit subtle with quota recovery. We cannot do it + * immediately because we have to obtain cluster locks from + * quota files and we also don't want to just skip it because + * then quota usage would be out of sync until some node takes + * the slot. So we remember which nodes need quota recovery + * and when everything else is done, we recover quotas. */ + if (quota_enabled) { + for (i = 0; i < rm_quota_used + && rm_quota[i] != slot_num; i++) + ; + + if (i == rm_quota_used) + rm_quota[rm_quota_used++] = slot_num; + } + + status = ocfs2_recover_node(osb, node_num, slot_num); +skip_recovery: + if (!status) { + ocfs2_recovery_map_clear(osb, node_num); + } else { + mlog(ML_ERROR, + "Error %d recovering node %d on device (%u,%u)!\n", + status, node_num, + MAJOR(osb->sb->s_dev), MINOR(osb->sb->s_dev)); + mlog(ML_ERROR, "Volume requires unmount.\n"); + } + + spin_lock(&osb->osb_lock); + } + spin_unlock(&osb->osb_lock); + trace_ocfs2_recovery_thread_end(status); + + /* Refresh all journal recovery generations from disk */ + status = ocfs2_check_journals_nolocks(osb); + status = (status == -EROFS) ? 0 : status; + if (status < 0) + mlog_errno(status); + + /* Now it is right time to recover quotas... We have to do this under + * superblock lock so that no one can start using the slot (and crash) + * before we recover it */ + if (quota_enabled) { + for (i = 0; i < rm_quota_used; i++) { + qrec = ocfs2_begin_quota_recovery(osb, rm_quota[i]); + if (IS_ERR(qrec)) { + status = PTR_ERR(qrec); + mlog_errno(status); + continue; + } + ocfs2_queue_recovery_completion(osb->journal, + rm_quota[i], + NULL, NULL, qrec, + ORPHAN_NEED_TRUNCATE); + } + } + + ocfs2_super_unlock(osb, 1); + + /* queue recovery for offline slots */ + ocfs2_queue_replay_slots(osb, ORPHAN_NEED_TRUNCATE); + +bail: + mutex_lock(&osb->recovery_lock); + if (!status && !ocfs2_recovery_completed(osb)) { + mutex_unlock(&osb->recovery_lock); + goto restart; + } + + ocfs2_free_replay_slots(osb); + osb->recovery_thread_task = NULL; + if (osb->recovery_state == OCFS2_REC_WANT_DISABLE) + osb->recovery_state = OCFS2_REC_DISABLED; + wake_up(&osb->recovery_event); + + mutex_unlock(&osb->recovery_lock); + + kfree(rm_quota); + + return status; +} + +void ocfs2_recovery_thread(struct ocfs2_super *osb, int node_num) +{ + int was_set = -1; + + mutex_lock(&osb->recovery_lock); + if (osb->recovery_state < OCFS2_REC_WANT_DISABLE) + was_set = ocfs2_recovery_map_set(osb, node_num); + + trace_ocfs2_recovery_thread(node_num, osb->node_num, + osb->recovery_state, osb->recovery_thread_task, was_set); + + if (osb->recovery_state >= OCFS2_REC_WANT_DISABLE) + goto out; + + if (osb->recovery_thread_task) + goto out; + + osb->recovery_thread_task = kthread_run(__ocfs2_recovery_thread, osb, + "ocfs2rec-%s", osb->uuid_str); + if (IS_ERR(osb->recovery_thread_task)) { + mlog_errno((int)PTR_ERR(osb->recovery_thread_task)); + osb->recovery_thread_task = NULL; + } + +out: + mutex_unlock(&osb->recovery_lock); + wake_up(&osb->recovery_event); +} + +static int ocfs2_read_journal_inode(struct ocfs2_super *osb, + int slot_num, + struct buffer_head **bh, + struct inode **ret_inode) +{ + int status = -EACCES; + struct inode *inode = NULL; + + BUG_ON(slot_num >= osb->max_slots); + + inode = ocfs2_get_system_file_inode(osb, JOURNAL_SYSTEM_INODE, + slot_num); + if (!inode || is_bad_inode(inode)) { + mlog_errno(status); + goto bail; + } + SET_INODE_JOURNAL(inode); + + status = ocfs2_read_inode_block_full(inode, bh, OCFS2_BH_IGNORE_CACHE); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + status = 0; + +bail: + if (inode) { + if (status || !ret_inode) + iput(inode); + else + *ret_inode = inode; + } + return status; +} + +/* Does the actual journal replay and marks the journal inode as + * clean. Will only replay if the journal inode is marked dirty. */ +static int ocfs2_replay_journal(struct ocfs2_super *osb, + int node_num, + int slot_num) +{ + int status; + int got_lock = 0; + unsigned int flags; + struct inode *inode = NULL; + struct ocfs2_dinode *fe; + journal_t *journal = NULL; + struct buffer_head *bh = NULL; + u32 slot_reco_gen; + + status = ocfs2_read_journal_inode(osb, slot_num, &bh, &inode); + if (status) { + mlog_errno(status); + goto done; + } + + fe = (struct ocfs2_dinode *)bh->b_data; + slot_reco_gen = ocfs2_get_recovery_generation(fe); + brelse(bh); + bh = NULL; + + /* + * As the fs recovery is asynchronous, there is a small chance that + * another node mounted (and recovered) the slot before the recovery + * thread could get the lock. To handle that, we dirty read the journal + * inode for that slot to get the recovery generation. If it is + * different than what we expected, the slot has been recovered. + * If not, it needs recovery. + */ + if (osb->slot_recovery_generations[slot_num] != slot_reco_gen) { + trace_ocfs2_replay_journal_recovered(slot_num, + osb->slot_recovery_generations[slot_num], slot_reco_gen); + osb->slot_recovery_generations[slot_num] = slot_reco_gen; + status = -EBUSY; + goto done; + } + + /* Continue with recovery as the journal has not yet been recovered */ + + status = ocfs2_inode_lock_full(inode, &bh, 1, OCFS2_META_LOCK_RECOVERY); + if (status < 0) { + trace_ocfs2_replay_journal_lock_err(status); + if (status != -ERESTARTSYS) + mlog(ML_ERROR, "Could not lock journal!\n"); + goto done; + } + got_lock = 1; + + fe = (struct ocfs2_dinode *) bh->b_data; + + flags = le32_to_cpu(fe->id1.journal1.ij_flags); + slot_reco_gen = ocfs2_get_recovery_generation(fe); + + if (!(flags & OCFS2_JOURNAL_DIRTY_FL)) { + trace_ocfs2_replay_journal_skip(node_num); + /* Refresh recovery generation for the slot */ + osb->slot_recovery_generations[slot_num] = slot_reco_gen; + goto done; + } + + /* we need to run complete recovery for offline orphan slots */ + ocfs2_replay_map_set_state(osb, REPLAY_NEEDED); + + printk(KERN_NOTICE "ocfs2: Begin replay journal (node %d, slot %d) on "\ + "device (%u,%u)\n", node_num, slot_num, MAJOR(osb->sb->s_dev), + MINOR(osb->sb->s_dev)); + + OCFS2_I(inode)->ip_clusters = le32_to_cpu(fe->i_clusters); + + status = ocfs2_force_read_journal(inode); + if (status < 0) { + mlog_errno(status); + goto done; + } + + journal = jbd2_journal_init_inode(inode); + if (IS_ERR(journal)) { + mlog(ML_ERROR, "Linux journal layer error\n"); + status = PTR_ERR(journal); + goto done; + } + + status = jbd2_journal_load(journal); + if (status < 0) { + mlog_errno(status); + BUG_ON(!igrab(inode)); + jbd2_journal_destroy(journal); + goto done; + } + + ocfs2_clear_journal_error(osb->sb, journal, slot_num); + + /* wipe the journal */ + jbd2_journal_lock_updates(journal); + status = jbd2_journal_flush(journal, 0); + jbd2_journal_unlock_updates(journal); + if (status < 0) + mlog_errno(status); + + /* This will mark the node clean */ + flags = le32_to_cpu(fe->id1.journal1.ij_flags); + flags &= ~OCFS2_JOURNAL_DIRTY_FL; + fe->id1.journal1.ij_flags = cpu_to_le32(flags); + + /* Increment recovery generation to indicate successful recovery */ + ocfs2_bump_recovery_generation(fe); + osb->slot_recovery_generations[slot_num] = + ocfs2_get_recovery_generation(fe); + + ocfs2_compute_meta_ecc(osb->sb, bh->b_data, &fe->i_check); + status = ocfs2_write_block(osb, bh, INODE_CACHE(inode)); + if (status < 0) + mlog_errno(status); + + BUG_ON(!igrab(inode)); + + jbd2_journal_destroy(journal); + + printk(KERN_NOTICE "ocfs2: End replay journal (node %d, slot %d) on "\ + "device (%u,%u)\n", node_num, slot_num, MAJOR(osb->sb->s_dev), + MINOR(osb->sb->s_dev)); +done: + /* drop the lock on this nodes journal */ + if (got_lock) + ocfs2_inode_unlock(inode, 1); + + iput(inode); + brelse(bh); + + return status; +} + +/* + * Do the most important parts of node recovery: + * - Replay it's journal + * - Stamp a clean local allocator file + * - Stamp a clean truncate log + * - Mark the node clean + * + * If this function completes without error, a node in OCFS2 can be + * said to have been safely recovered. As a result, failure during the + * second part of a nodes recovery process (local alloc recovery) is + * far less concerning. + */ +static int ocfs2_recover_node(struct ocfs2_super *osb, + int node_num, int slot_num) +{ + int status = 0; + struct ocfs2_dinode *la_copy = NULL; + struct ocfs2_dinode *tl_copy = NULL; + + trace_ocfs2_recover_node(node_num, slot_num, osb->node_num); + + /* Should not ever be called to recover ourselves -- in that + * case we should've called ocfs2_journal_load instead. */ + BUG_ON(osb->node_num == node_num); + + status = ocfs2_replay_journal(osb, node_num, slot_num); + if (status < 0) { + if (status == -EBUSY) { + trace_ocfs2_recover_node_skip(slot_num, node_num); + status = 0; + goto done; + } + mlog_errno(status); + goto done; + } + + /* Stamp a clean local alloc file AFTER recovering the journal... */ + status = ocfs2_begin_local_alloc_recovery(osb, slot_num, &la_copy); + if (status < 0) { + mlog_errno(status); + goto done; + } + + /* An error from begin_truncate_log_recovery is not + * serious enough to warrant halting the rest of + * recovery. */ + status = ocfs2_begin_truncate_log_recovery(osb, slot_num, &tl_copy); + if (status < 0) + mlog_errno(status); + + /* Likewise, this would be a strange but ultimately not so + * harmful place to get an error... */ + status = ocfs2_clear_slot(osb, slot_num); + if (status < 0) + mlog_errno(status); + + /* This will kfree the memory pointed to by la_copy and tl_copy */ + ocfs2_queue_recovery_completion(osb->journal, slot_num, la_copy, + tl_copy, NULL, ORPHAN_NEED_TRUNCATE); + + status = 0; +done: + + return status; +} + +/* Test node liveness by trylocking his journal. If we get the lock, + * we drop it here. Return 0 if we got the lock, -EAGAIN if node is + * still alive (we couldn't get the lock) and < 0 on error. */ +static int ocfs2_trylock_journal(struct ocfs2_super *osb, + int slot_num) +{ + int status, flags; + struct inode *inode = NULL; + + inode = ocfs2_get_system_file_inode(osb, JOURNAL_SYSTEM_INODE, + slot_num); + if (inode == NULL) { + mlog(ML_ERROR, "access error\n"); + status = -EACCES; + goto bail; + } + if (is_bad_inode(inode)) { + mlog(ML_ERROR, "access error (bad inode)\n"); + iput(inode); + inode = NULL; + status = -EACCES; + goto bail; + } + SET_INODE_JOURNAL(inode); + + flags = OCFS2_META_LOCK_RECOVERY | OCFS2_META_LOCK_NOQUEUE; + status = ocfs2_inode_lock_full(inode, NULL, 1, flags); + if (status < 0) { + if (status != -EAGAIN) + mlog_errno(status); + goto bail; + } + + ocfs2_inode_unlock(inode, 1); +bail: + iput(inode); + + return status; +} + +/* Call this underneath ocfs2_super_lock. It also assumes that the + * slot info struct has been updated from disk. */ +int ocfs2_mark_dead_nodes(struct ocfs2_super *osb) +{ + unsigned int node_num; + int status, i; + u32 gen; + struct buffer_head *bh = NULL; + struct ocfs2_dinode *di; + + /* This is called with the super block cluster lock, so we + * know that the slot map can't change underneath us. */ + + for (i = 0; i < osb->max_slots; i++) { + /* Read journal inode to get the recovery generation */ + status = ocfs2_read_journal_inode(osb, i, &bh, NULL); + if (status) { + mlog_errno(status); + goto bail; + } + di = (struct ocfs2_dinode *)bh->b_data; + gen = ocfs2_get_recovery_generation(di); + brelse(bh); + bh = NULL; + + spin_lock(&osb->osb_lock); + osb->slot_recovery_generations[i] = gen; + + trace_ocfs2_mark_dead_nodes(i, + osb->slot_recovery_generations[i]); + + if (i == osb->slot_num) { + spin_unlock(&osb->osb_lock); + continue; + } + + status = ocfs2_slot_to_node_num_locked(osb, i, &node_num); + if (status == -ENOENT) { + spin_unlock(&osb->osb_lock); + continue; + } + + if (__ocfs2_recovery_map_test(osb, node_num)) { + spin_unlock(&osb->osb_lock); + continue; + } + spin_unlock(&osb->osb_lock); + + /* Ok, we have a slot occupied by another node which + * is not in the recovery map. We trylock his journal + * file here to test if he's alive. */ + status = ocfs2_trylock_journal(osb, i); + if (!status) { + /* Since we're called from mount, we know that + * the recovery thread can't race us on + * setting / checking the recovery bits. */ + ocfs2_recovery_thread(osb, node_num); + } else if ((status < 0) && (status != -EAGAIN)) { + mlog_errno(status); + goto bail; + } + } + + status = 0; +bail: + return status; +} + +/* + * Scan timer should get fired every ORPHAN_SCAN_SCHEDULE_TIMEOUT. Add some + * randomness to the timeout to minimize multiple nodes firing the timer at the + * same time. + */ +static inline unsigned long ocfs2_orphan_scan_timeout(void) +{ + unsigned long time; + + get_random_bytes(&time, sizeof(time)); + time = ORPHAN_SCAN_SCHEDULE_TIMEOUT + (time % 5000); + return msecs_to_jiffies(time); +} + +/* + * ocfs2_queue_orphan_scan calls ocfs2_queue_recovery_completion for + * every slot, queuing a recovery of the slot on the ocfs2_wq thread. This + * is done to catch any orphans that are left over in orphan directories. + * + * It scans all slots, even ones that are in use. It does so to handle the + * case described below: + * + * Node 1 has an inode it was using. The dentry went away due to memory + * pressure. Node 1 closes the inode, but it's on the free list. The node + * has the open lock. + * Node 2 unlinks the inode. It grabs the dentry lock to notify others, + * but node 1 has no dentry and doesn't get the message. It trylocks the + * open lock, sees that another node has a PR, and does nothing. + * Later node 2 runs its orphan dir. It igets the inode, trylocks the + * open lock, sees the PR still, and does nothing. + * Basically, we have to trigger an orphan iput on node 1. The only way + * for this to happen is if node 1 runs node 2's orphan dir. + * + * ocfs2_queue_orphan_scan gets called every ORPHAN_SCAN_SCHEDULE_TIMEOUT + * seconds. It gets an EX lock on os_lockres and checks sequence number + * stored in LVB. If the sequence number has changed, it means some other + * node has done the scan. This node skips the scan and tracks the + * sequence number. If the sequence number didn't change, it means a scan + * hasn't happened. The node queues a scan and increments the + * sequence number in the LVB. + */ +static void ocfs2_queue_orphan_scan(struct ocfs2_super *osb) +{ + struct ocfs2_orphan_scan *os; + int status, i; + u32 seqno = 0; + + os = &osb->osb_orphan_scan; + + if (atomic_read(&os->os_state) == ORPHAN_SCAN_INACTIVE) + goto out; + + trace_ocfs2_queue_orphan_scan_begin(os->os_count, os->os_seqno, + atomic_read(&os->os_state)); + + status = ocfs2_orphan_scan_lock(osb, &seqno); + if (status < 0) { + if (status != -EAGAIN) + mlog_errno(status); + goto out; + } + + /* Do no queue the tasks if the volume is being umounted */ + if (atomic_read(&os->os_state) == ORPHAN_SCAN_INACTIVE) + goto unlock; + + if (os->os_seqno != seqno) { + os->os_seqno = seqno; + goto unlock; + } + + for (i = 0; i < osb->max_slots; i++) + ocfs2_queue_recovery_completion(osb->journal, i, NULL, NULL, + NULL, ORPHAN_NO_NEED_TRUNCATE); + /* + * We queued a recovery on orphan slots, increment the sequence + * number and update LVB so other node will skip the scan for a while + */ + seqno++; + os->os_count++; + os->os_scantime = ktime_get_seconds(); +unlock: + ocfs2_orphan_scan_unlock(osb, seqno); +out: + trace_ocfs2_queue_orphan_scan_end(os->os_count, os->os_seqno, + atomic_read(&os->os_state)); + return; +} + +/* Worker task that gets fired every ORPHAN_SCAN_SCHEDULE_TIMEOUT millsec */ +static void ocfs2_orphan_scan_work(struct work_struct *work) +{ + struct ocfs2_orphan_scan *os; + struct ocfs2_super *osb; + + os = container_of(work, struct ocfs2_orphan_scan, + os_orphan_scan_work.work); + osb = os->os_osb; + + mutex_lock(&os->os_lock); + ocfs2_queue_orphan_scan(osb); + if (atomic_read(&os->os_state) == ORPHAN_SCAN_ACTIVE) + queue_delayed_work(osb->ocfs2_wq, &os->os_orphan_scan_work, + ocfs2_orphan_scan_timeout()); + mutex_unlock(&os->os_lock); +} + +void ocfs2_orphan_scan_stop(struct ocfs2_super *osb) +{ + struct ocfs2_orphan_scan *os; + + os = &osb->osb_orphan_scan; + if (atomic_read(&os->os_state) == ORPHAN_SCAN_ACTIVE) { + atomic_set(&os->os_state, ORPHAN_SCAN_INACTIVE); + mutex_lock(&os->os_lock); + cancel_delayed_work(&os->os_orphan_scan_work); + mutex_unlock(&os->os_lock); + } +} + +void ocfs2_orphan_scan_init(struct ocfs2_super *osb) +{ + struct ocfs2_orphan_scan *os; + + os = &osb->osb_orphan_scan; + os->os_osb = osb; + os->os_count = 0; + os->os_seqno = 0; + mutex_init(&os->os_lock); + INIT_DELAYED_WORK(&os->os_orphan_scan_work, ocfs2_orphan_scan_work); +} + +void ocfs2_orphan_scan_start(struct ocfs2_super *osb) +{ + struct ocfs2_orphan_scan *os; + + os = &osb->osb_orphan_scan; + os->os_scantime = ktime_get_seconds(); + if (ocfs2_is_hard_readonly(osb) || ocfs2_mount_local(osb)) + atomic_set(&os->os_state, ORPHAN_SCAN_INACTIVE); + else { + atomic_set(&os->os_state, ORPHAN_SCAN_ACTIVE); + queue_delayed_work(osb->ocfs2_wq, &os->os_orphan_scan_work, + ocfs2_orphan_scan_timeout()); + } +} + +struct ocfs2_orphan_filldir_priv { + struct dir_context ctx; + struct inode *head; + struct ocfs2_super *osb; + enum ocfs2_orphan_reco_type orphan_reco_type; +}; + +static bool ocfs2_orphan_filldir(struct dir_context *ctx, const char *name, + int name_len, loff_t pos, u64 ino, + unsigned type) +{ + struct ocfs2_orphan_filldir_priv *p = + container_of(ctx, struct ocfs2_orphan_filldir_priv, ctx); + struct inode *iter; + + if (name_len == 1 && !strncmp(".", name, 1)) + return true; + if (name_len == 2 && !strncmp("..", name, 2)) + return true; + + /* do not include dio entry in case of orphan scan */ + if ((p->orphan_reco_type == ORPHAN_NO_NEED_TRUNCATE) && + (!strncmp(name, OCFS2_DIO_ORPHAN_PREFIX, + OCFS2_DIO_ORPHAN_PREFIX_LEN))) + return true; + + /* Skip bad inodes so that recovery can continue */ + iter = ocfs2_iget(p->osb, ino, + OCFS2_FI_FLAG_ORPHAN_RECOVERY, 0); + if (IS_ERR(iter)) + return true; + + if (!strncmp(name, OCFS2_DIO_ORPHAN_PREFIX, + OCFS2_DIO_ORPHAN_PREFIX_LEN)) + OCFS2_I(iter)->ip_flags |= OCFS2_INODE_DIO_ORPHAN_ENTRY; + + /* Skip inodes which are already added to recover list, since dio may + * happen concurrently with unlink/rename */ + if (OCFS2_I(iter)->ip_next_orphan) { + iput(iter); + return true; + } + + trace_ocfs2_orphan_filldir((unsigned long long)OCFS2_I(iter)->ip_blkno); + /* No locking is required for the next_orphan queue as there + * is only ever a single process doing orphan recovery. */ + OCFS2_I(iter)->ip_next_orphan = p->head; + p->head = iter; + + return true; +} + +static int ocfs2_queue_orphans(struct ocfs2_super *osb, + int slot, + struct inode **head, + enum ocfs2_orphan_reco_type orphan_reco_type) +{ + int status; + struct inode *orphan_dir_inode = NULL; + struct ocfs2_orphan_filldir_priv priv = { + .ctx.actor = ocfs2_orphan_filldir, + .osb = osb, + .head = *head, + .orphan_reco_type = orphan_reco_type + }; + + orphan_dir_inode = ocfs2_get_system_file_inode(osb, + ORPHAN_DIR_SYSTEM_INODE, + slot); + if (!orphan_dir_inode) { + status = -ENOENT; + mlog_errno(status); + return status; + } + + inode_lock(orphan_dir_inode); + status = ocfs2_inode_lock(orphan_dir_inode, NULL, 0); + if (status < 0) { + mlog_errno(status); + goto out; + } + + status = ocfs2_dir_foreach(orphan_dir_inode, &priv.ctx); + if (status) { + mlog_errno(status); + goto out_cluster; + } + + *head = priv.head; + +out_cluster: + ocfs2_inode_unlock(orphan_dir_inode, 0); +out: + inode_unlock(orphan_dir_inode); + iput(orphan_dir_inode); + return status; +} + +static int ocfs2_orphan_recovery_can_continue(struct ocfs2_super *osb, + int slot) +{ + int ret; + + spin_lock(&osb->osb_lock); + ret = !osb->osb_orphan_wipes[slot]; + spin_unlock(&osb->osb_lock); + return ret; +} + +static void ocfs2_mark_recovering_orphan_dir(struct ocfs2_super *osb, + int slot) +{ + spin_lock(&osb->osb_lock); + /* Mark ourselves such that new processes in delete_inode() + * know to quit early. */ + ocfs2_node_map_set_bit(osb, &osb->osb_recovering_orphan_dirs, slot); + while (osb->osb_orphan_wipes[slot]) { + /* If any processes are already in the middle of an + * orphan wipe on this dir, then we need to wait for + * them. */ + spin_unlock(&osb->osb_lock); + wait_event_interruptible(osb->osb_wipe_event, + ocfs2_orphan_recovery_can_continue(osb, slot)); + spin_lock(&osb->osb_lock); + } + spin_unlock(&osb->osb_lock); +} + +static void ocfs2_clear_recovering_orphan_dir(struct ocfs2_super *osb, + int slot) +{ + ocfs2_node_map_clear_bit(osb, &osb->osb_recovering_orphan_dirs, slot); +} + +/* + * Orphan recovery. Each mounted node has it's own orphan dir which we + * must run during recovery. Our strategy here is to build a list of + * the inodes in the orphan dir and iget/iput them. The VFS does + * (most) of the rest of the work. + * + * Orphan recovery can happen at any time, not just mount so we have a + * couple of extra considerations. + * + * - We grab as many inodes as we can under the orphan dir lock - + * doing iget() outside the orphan dir risks getting a reference on + * an invalid inode. + * - We must be sure not to deadlock with other processes on the + * system wanting to run delete_inode(). This can happen when they go + * to lock the orphan dir and the orphan recovery process attempts to + * iget() inside the orphan dir lock. This can be avoided by + * advertising our state to ocfs2_delete_inode(). + */ +static int ocfs2_recover_orphans(struct ocfs2_super *osb, + int slot, + enum ocfs2_orphan_reco_type orphan_reco_type) +{ + int ret = 0; + struct inode *inode = NULL; + struct inode *iter; + struct ocfs2_inode_info *oi; + struct buffer_head *di_bh = NULL; + struct ocfs2_dinode *di = NULL; + + trace_ocfs2_recover_orphans(slot); + + ocfs2_mark_recovering_orphan_dir(osb, slot); + ret = ocfs2_queue_orphans(osb, slot, &inode, orphan_reco_type); + ocfs2_clear_recovering_orphan_dir(osb, slot); + + /* Error here should be noted, but we want to continue with as + * many queued inodes as we've got. */ + if (ret) + mlog_errno(ret); + + while (inode) { + oi = OCFS2_I(inode); + trace_ocfs2_recover_orphans_iput( + (unsigned long long)oi->ip_blkno); + + iter = oi->ip_next_orphan; + oi->ip_next_orphan = NULL; + + if (oi->ip_flags & OCFS2_INODE_DIO_ORPHAN_ENTRY) { + inode_lock(inode); + ret = ocfs2_rw_lock(inode, 1); + if (ret < 0) { + mlog_errno(ret); + goto unlock_mutex; + } + /* + * We need to take and drop the inode lock to + * force read inode from disk. + */ + ret = ocfs2_inode_lock(inode, &di_bh, 1); + if (ret) { + mlog_errno(ret); + goto unlock_rw; + } + + di = (struct ocfs2_dinode *)di_bh->b_data; + + if (di->i_flags & cpu_to_le32(OCFS2_DIO_ORPHANED_FL)) { + ret = ocfs2_truncate_file(inode, di_bh, + i_size_read(inode)); + if (ret < 0) { + if (ret != -ENOSPC) + mlog_errno(ret); + goto unlock_inode; + } + + ret = ocfs2_del_inode_from_orphan(osb, inode, + di_bh, 0, 0); + if (ret) + mlog_errno(ret); + } +unlock_inode: + ocfs2_inode_unlock(inode, 1); + brelse(di_bh); + di_bh = NULL; +unlock_rw: + ocfs2_rw_unlock(inode, 1); +unlock_mutex: + inode_unlock(inode); + + /* clear dio flag in ocfs2_inode_info */ + oi->ip_flags &= ~OCFS2_INODE_DIO_ORPHAN_ENTRY; + } else { + spin_lock(&oi->ip_lock); + /* Set the proper information to get us going into + * ocfs2_delete_inode. */ + oi->ip_flags |= OCFS2_INODE_MAYBE_ORPHANED; + spin_unlock(&oi->ip_lock); + } + + iput(inode); + inode = iter; + } + + return ret; +} + +static int __ocfs2_wait_on_mount(struct ocfs2_super *osb, int quota) +{ + /* This check is good because ocfs2 will wait on our recovery + * thread before changing it to something other than MOUNTED + * or DISABLED. */ + wait_event(osb->osb_mount_event, + (!quota && atomic_read(&osb->vol_state) == VOLUME_MOUNTED) || + atomic_read(&osb->vol_state) == VOLUME_MOUNTED_QUOTAS || + atomic_read(&osb->vol_state) == VOLUME_DISABLED); + + /* If there's an error on mount, then we may never get to the + * MOUNTED flag, but this is set right before + * dismount_volume() so we can trust it. */ + if (atomic_read(&osb->vol_state) == VOLUME_DISABLED) { + trace_ocfs2_wait_on_mount(VOLUME_DISABLED); + mlog(0, "mount error, exiting!\n"); + return -EBUSY; + } + + return 0; +} + +static int ocfs2_commit_thread(void *arg) +{ + int status; + struct ocfs2_super *osb = arg; + struct ocfs2_journal *journal = osb->journal; + + /* we can trust j_num_trans here because _should_stop() is only set in + * shutdown and nobody other than ourselves should be able to start + * transactions. committing on shutdown might take a few iterations + * as final transactions put deleted inodes on the list */ + while (!(kthread_should_stop() && + atomic_read(&journal->j_num_trans) == 0)) { + + wait_event_interruptible(osb->checkpoint_event, + atomic_read(&journal->j_num_trans) + || kthread_should_stop()); + + status = ocfs2_commit_cache(osb); + if (status < 0) { + static unsigned long abort_warn_time; + + /* Warn about this once per minute */ + if (printk_timed_ratelimit(&abort_warn_time, 60*HZ)) + mlog(ML_ERROR, "status = %d, journal is " + "already aborted.\n", status); + /* + * After ocfs2_commit_cache() fails, j_num_trans has a + * non-zero value. Sleep here to avoid a busy-wait + * loop. + */ + msleep_interruptible(1000); + } + + if (kthread_should_stop() && atomic_read(&journal->j_num_trans)){ + mlog(ML_KTHREAD, + "commit_thread: %u transactions pending on " + "shutdown\n", + atomic_read(&journal->j_num_trans)); + } + } + + return 0; +} + +/* Reads all the journal inodes without taking any cluster locks. Used + * for hard readonly access to determine whether any journal requires + * recovery. Also used to refresh the recovery generation numbers after + * a journal has been recovered by another node. + */ +int ocfs2_check_journals_nolocks(struct ocfs2_super *osb) +{ + int ret = 0; + unsigned int slot; + struct buffer_head *di_bh = NULL; + struct ocfs2_dinode *di; + int journal_dirty = 0; + + for(slot = 0; slot < osb->max_slots; slot++) { + ret = ocfs2_read_journal_inode(osb, slot, &di_bh, NULL); + if (ret) { + mlog_errno(ret); + goto out; + } + + di = (struct ocfs2_dinode *) di_bh->b_data; + + osb->slot_recovery_generations[slot] = + ocfs2_get_recovery_generation(di); + + if (le32_to_cpu(di->id1.journal1.ij_flags) & + OCFS2_JOURNAL_DIRTY_FL) + journal_dirty = 1; + + brelse(di_bh); + di_bh = NULL; + } + +out: + if (journal_dirty) + ret = -EROFS; + return ret; +} diff --git a/fs/ocfs2/journal.h b/fs/ocfs2/journal.h new file mode 100644 index 000000000..f8b3b2a3d --- /dev/null +++ b/fs/ocfs2/journal.h @@ -0,0 +1,613 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * journal.h + * + * Defines journalling api and structures. + * + * Copyright (C) 2003, 2005 Oracle. All rights reserved. + */ + +#ifndef OCFS2_JOURNAL_H +#define OCFS2_JOURNAL_H + +#include +#include + +enum ocfs2_journal_state { + OCFS2_JOURNAL_FREE = 0, + OCFS2_JOURNAL_LOADED, + OCFS2_JOURNAL_IN_SHUTDOWN, +}; + +struct ocfs2_super; +struct ocfs2_dinode; + +/* + * The recovery_list is a simple linked list of node numbers to recover. + * It is protected by the recovery_lock. + */ + +struct ocfs2_recovery_map { + unsigned int rm_used; + unsigned int rm_entries[]; +}; + + +struct ocfs2_journal { + enum ocfs2_journal_state j_state; /* Journals current state */ + + journal_t *j_journal; /* The kernels journal type */ + struct inode *j_inode; /* Kernel inode pointing to + * this journal */ + struct ocfs2_super *j_osb; /* pointer to the super + * block for the node + * we're currently + * running on -- not + * necessarily the super + * block from the node + * which we usually run + * from (recovery, + * etc) */ + struct buffer_head *j_bh; /* Journal disk inode block */ + atomic_t j_num_trans; /* Number of transactions + * currently in the system. */ + spinlock_t j_lock; + unsigned long j_trans_id; + struct rw_semaphore j_trans_barrier; + wait_queue_head_t j_checkpointed; + + /* both fields protected by j_lock*/ + struct list_head j_la_cleanups; + struct work_struct j_recovery_work; +}; + +extern spinlock_t trans_inc_lock; + +/* wrap j_trans_id so we never have it equal to zero. */ +static inline unsigned long ocfs2_inc_trans_id(struct ocfs2_journal *j) +{ + unsigned long old_id; + spin_lock(&trans_inc_lock); + old_id = j->j_trans_id++; + if (unlikely(!j->j_trans_id)) + j->j_trans_id = 1; + spin_unlock(&trans_inc_lock); + return old_id; +} + +static inline void ocfs2_set_ci_lock_trans(struct ocfs2_journal *journal, + struct ocfs2_caching_info *ci) +{ + spin_lock(&trans_inc_lock); + ci->ci_last_trans = journal->j_trans_id; + spin_unlock(&trans_inc_lock); +} + +/* Used to figure out whether it's safe to drop a metadata lock on an + * cached object. Returns true if all the object's changes have been + * checkpointed to disk. You should be holding the spinlock on the + * metadata lock while calling this to be sure that nobody can take + * the lock and put it on another transaction. */ +static inline int ocfs2_ci_fully_checkpointed(struct ocfs2_caching_info *ci) +{ + int ret; + struct ocfs2_journal *journal = + OCFS2_SB(ocfs2_metadata_cache_get_super(ci))->journal; + + spin_lock(&trans_inc_lock); + ret = time_after(journal->j_trans_id, ci->ci_last_trans); + spin_unlock(&trans_inc_lock); + return ret; +} + +/* convenience function to check if an object backed by struct + * ocfs2_caching_info is still new (has never hit disk) Will do you a + * favor and set created_trans = 0 when you've + * been checkpointed. returns '1' if the ci is still new. */ +static inline int ocfs2_ci_is_new(struct ocfs2_caching_info *ci) +{ + int ret; + struct ocfs2_journal *journal = + OCFS2_SB(ocfs2_metadata_cache_get_super(ci))->journal; + + spin_lock(&trans_inc_lock); + ret = !(time_after(journal->j_trans_id, ci->ci_created_trans)); + if (!ret) + ci->ci_created_trans = 0; + spin_unlock(&trans_inc_lock); + return ret; +} + +/* Wrapper for inodes so we can check system files */ +static inline int ocfs2_inode_is_new(struct inode *inode) +{ + /* System files are never "new" as they're written out by + * mkfs. This helps us early during mount, before we have the + * journal open and j_trans_id could be junk. */ + if (OCFS2_I(inode)->ip_flags & OCFS2_INODE_SYSTEM_FILE) + return 0; + + return ocfs2_ci_is_new(INODE_CACHE(inode)); +} + +static inline void ocfs2_ci_set_new(struct ocfs2_super *osb, + struct ocfs2_caching_info *ci) +{ + spin_lock(&trans_inc_lock); + ci->ci_created_trans = osb->journal->j_trans_id; + spin_unlock(&trans_inc_lock); +} + +/* Exported only for the journal struct init code in super.c. Do not call. */ +void ocfs2_orphan_scan_init(struct ocfs2_super *osb); +void ocfs2_orphan_scan_start(struct ocfs2_super *osb); +void ocfs2_orphan_scan_stop(struct ocfs2_super *osb); + +void ocfs2_complete_recovery(struct work_struct *work); +void ocfs2_wait_for_recovery(struct ocfs2_super *osb); + +int ocfs2_recovery_init(struct ocfs2_super *osb); +void ocfs2_recovery_exit(struct ocfs2_super *osb); +void ocfs2_recovery_disable_quota(struct ocfs2_super *osb); + +int ocfs2_compute_replay_slots(struct ocfs2_super *osb); +void ocfs2_free_replay_slots(struct ocfs2_super *osb); +/* + * Journal Control: + * Initialize, Load, Shutdown, Wipe a journal. + * + * ocfs2_journal_alloc - Initialize skeleton for journal structure. + * ocfs2_journal_init - Initialize journal structures in the OSB. + * ocfs2_journal_load - Load the given journal off disk. Replay it if + * there's transactions still in there. + * ocfs2_journal_shutdown - Shutdown a journal, this will flush all + * uncommitted, uncheckpointed transactions. + * ocfs2_journal_wipe - Wipe transactions from a journal. Optionally + * zero out each block. + * ocfs2_recovery_thread - Perform recovery on a node. osb is our own osb. + * ocfs2_mark_dead_nodes - Start recovery on nodes we won't get a heartbeat + * event on. + * ocfs2_start_checkpoint - Kick the commit thread to do a checkpoint. + */ +void ocfs2_set_journal_params(struct ocfs2_super *osb); +int ocfs2_journal_alloc(struct ocfs2_super *osb); +int ocfs2_journal_init(struct ocfs2_super *osb, int *dirty); +void ocfs2_journal_shutdown(struct ocfs2_super *osb); +int ocfs2_journal_wipe(struct ocfs2_journal *journal, + int full); +int ocfs2_journal_load(struct ocfs2_journal *journal, int local, + int replayed); +int ocfs2_check_journals_nolocks(struct ocfs2_super *osb); +void ocfs2_recovery_thread(struct ocfs2_super *osb, + int node_num); +int ocfs2_mark_dead_nodes(struct ocfs2_super *osb); +void ocfs2_complete_mount_recovery(struct ocfs2_super *osb); +void ocfs2_complete_quota_recovery(struct ocfs2_super *osb); + +static inline void ocfs2_start_checkpoint(struct ocfs2_super *osb) +{ + wake_up(&osb->checkpoint_event); +} + +static inline void ocfs2_checkpoint_inode(struct inode *inode) +{ + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + if (ocfs2_mount_local(osb)) + return; + + if (!osb->journal) + return; + + if (!ocfs2_ci_fully_checkpointed(INODE_CACHE(inode))) { + /* WARNING: This only kicks off a single + * checkpoint. If someone races you and adds more + * metadata to the journal, you won't know, and will + * wind up waiting *a lot* longer than necessary. Right + * now we only use this in clear_inode so that's + * OK. */ + ocfs2_start_checkpoint(osb); + + wait_event(osb->journal->j_checkpointed, + ocfs2_ci_fully_checkpointed(INODE_CACHE(inode))); + } +} + +/* + * Transaction Handling: + * Manage the lifetime of a transaction handle. + * + * ocfs2_start_trans - Begin a transaction. Give it an upper estimate of + * the number of blocks that will be changed during + * this handle. + * ocfs2_commit_trans - Complete a handle. It might return -EIO if + * the journal was aborted. The majority of paths don't + * check the return value as an error there comes too + * late to do anything (and will be picked up in a + * later transaction). + * ocfs2_extend_trans - Extend a handle by nblocks credits. This may + * commit the handle to disk in the process, but will + * not release any locks taken during the transaction. + * ocfs2_journal_access* - Notify the handle that we want to journal this + * buffer. Will have to call ocfs2_journal_dirty once + * we've actually dirtied it. Type is one of . or . + * Always call the specific flavor of + * ocfs2_journal_access_*() unless you intend to + * manage the checksum by hand. + * ocfs2_journal_dirty - Mark a journalled buffer as having dirty data. + * ocfs2_jbd2_inode_add_write - Mark an inode with range so that its data goes + * out before the current handle commits. + */ + +/* You must always start_trans with a number of buffs > 0, but it's + * perfectly legal to go through an entire transaction without having + * dirtied any buffers. */ +handle_t *ocfs2_start_trans(struct ocfs2_super *osb, + int max_buffs); +int ocfs2_commit_trans(struct ocfs2_super *osb, + handle_t *handle); +int ocfs2_extend_trans(handle_t *handle, int nblocks); +int ocfs2_assure_trans_credits(handle_t *handle, + int nblocks); +int ocfs2_allocate_extend_trans(handle_t *handle, + int thresh); + +/* + * Define an arbitrary limit for the amount of data we will anticipate + * writing to any given transaction. For unbounded transactions such as + * fallocate(2) we can write more than this, but we always + * start off at the maximum transaction size and grow the transaction + * optimistically as we go. + */ +#define OCFS2_MAX_TRANS_DATA 64U + +/* + * Create access is for when we get a newly created buffer and we're + * not gonna read it off disk, but rather fill it ourselves. Right + * now, we don't do anything special with this (it turns into a write + * request), but this is a good placeholder in case we do... + * + * Write access is for when we read a block off disk and are going to + * modify it. This way the journalling layer knows it may need to make + * a copy of that block (if it's part of another, uncommitted + * transaction) before we do so. + */ +#define OCFS2_JOURNAL_ACCESS_CREATE 0 +#define OCFS2_JOURNAL_ACCESS_WRITE 1 +#define OCFS2_JOURNAL_ACCESS_UNDO 2 + + +/* ocfs2_inode */ +int ocfs2_journal_access_di(handle_t *handle, struct ocfs2_caching_info *ci, + struct buffer_head *bh, int type); +/* ocfs2_extent_block */ +int ocfs2_journal_access_eb(handle_t *handle, struct ocfs2_caching_info *ci, + struct buffer_head *bh, int type); +/* ocfs2_refcount_block */ +int ocfs2_journal_access_rb(handle_t *handle, struct ocfs2_caching_info *ci, + struct buffer_head *bh, int type); +/* ocfs2_group_desc */ +int ocfs2_journal_access_gd(handle_t *handle, struct ocfs2_caching_info *ci, + struct buffer_head *bh, int type); +/* ocfs2_xattr_block */ +int ocfs2_journal_access_xb(handle_t *handle, struct ocfs2_caching_info *ci, + struct buffer_head *bh, int type); +/* quota blocks */ +int ocfs2_journal_access_dq(handle_t *handle, struct ocfs2_caching_info *ci, + struct buffer_head *bh, int type); +/* dirblock */ +int ocfs2_journal_access_db(handle_t *handle, struct ocfs2_caching_info *ci, + struct buffer_head *bh, int type); +/* ocfs2_dx_root_block */ +int ocfs2_journal_access_dr(handle_t *handle, struct ocfs2_caching_info *ci, + struct buffer_head *bh, int type); +/* ocfs2_dx_leaf */ +int ocfs2_journal_access_dl(handle_t *handle, struct ocfs2_caching_info *ci, + struct buffer_head *bh, int type); +/* Anything that has no ecc */ +int ocfs2_journal_access(handle_t *handle, struct ocfs2_caching_info *ci, + struct buffer_head *bh, int type); + +/* + * A word about the journal_access/journal_dirty "dance". It is + * entirely legal to journal_access a buffer more than once (as long + * as the access type is the same -- I'm not sure what will happen if + * access type is different but this should never happen anyway) It is + * also legal to journal_dirty a buffer more than once. In fact, you + * can even journal_access a buffer after you've done a + * journal_access/journal_dirty pair. The only thing you cannot do + * however, is journal_dirty a buffer which you haven't yet passed to + * journal_access at least once. + * + * That said, 99% of the time this doesn't matter and this is what the + * path looks like: + * + * + * ocfs2_journal_access(handle, bh, OCFS2_JOURNAL_ACCESS_WRITE); + * + * ocfs2_journal_dirty(handle, bh); + */ +void ocfs2_journal_dirty(handle_t *handle, struct buffer_head *bh); + +/* + * Credit Macros: + * Convenience macros to calculate number of credits needed. + * + * For convenience sake, I have a set of macros here which calculate + * the *maximum* number of sectors which will be changed for various + * metadata updates. + */ + +/* simple file updates like chmod, etc. */ +#define OCFS2_INODE_UPDATE_CREDITS 1 + +/* extended attribute block update */ +#define OCFS2_XATTR_BLOCK_UPDATE_CREDITS 1 + +/* Update of a single quota block */ +#define OCFS2_QUOTA_BLOCK_UPDATE_CREDITS 1 + +/* global quotafile inode update, data block */ +#define OCFS2_QINFO_WRITE_CREDITS (OCFS2_INODE_UPDATE_CREDITS + \ + OCFS2_QUOTA_BLOCK_UPDATE_CREDITS) + +#define OCFS2_LOCAL_QINFO_WRITE_CREDITS OCFS2_QUOTA_BLOCK_UPDATE_CREDITS +/* + * The two writes below can accidentally see global info dirty due + * to set_info() quotactl so make them prepared for the writes. + */ +/* quota data block, global info */ +/* Write to local quota file */ +#define OCFS2_QWRITE_CREDITS (OCFS2_QINFO_WRITE_CREDITS + \ + OCFS2_QUOTA_BLOCK_UPDATE_CREDITS) + +/* global quota data block, local quota data block, global quota inode, + * global quota info */ +#define OCFS2_QSYNC_CREDITS (OCFS2_QINFO_WRITE_CREDITS + \ + 2 * OCFS2_QUOTA_BLOCK_UPDATE_CREDITS) + +static inline int ocfs2_quota_trans_credits(struct super_block *sb) +{ + int credits = 0; + + if (OCFS2_HAS_RO_COMPAT_FEATURE(sb, OCFS2_FEATURE_RO_COMPAT_USRQUOTA)) + credits += OCFS2_QWRITE_CREDITS; + if (OCFS2_HAS_RO_COMPAT_FEATURE(sb, OCFS2_FEATURE_RO_COMPAT_GRPQUOTA)) + credits += OCFS2_QWRITE_CREDITS; + return credits; +} + +/* group extend. inode update and last group update. */ +#define OCFS2_GROUP_EXTEND_CREDITS (OCFS2_INODE_UPDATE_CREDITS + 1) + +/* group add. inode update and the new group update. */ +#define OCFS2_GROUP_ADD_CREDITS (OCFS2_INODE_UPDATE_CREDITS + 1) + +/* get one bit out of a suballocator: dinode + group descriptor + + * prev. group desc. if we relink. */ +#define OCFS2_SUBALLOC_ALLOC (3) + +static inline int ocfs2_inline_to_extents_credits(struct super_block *sb) +{ + return OCFS2_SUBALLOC_ALLOC + OCFS2_INODE_UPDATE_CREDITS + + ocfs2_quota_trans_credits(sb); +} + +/* dinode + group descriptor update. We don't relink on free yet. */ +#define OCFS2_SUBALLOC_FREE (2) + +#define OCFS2_TRUNCATE_LOG_UPDATE OCFS2_INODE_UPDATE_CREDITS +#define OCFS2_TRUNCATE_LOG_FLUSH_ONE_REC (OCFS2_SUBALLOC_FREE \ + + OCFS2_TRUNCATE_LOG_UPDATE) + +static inline int ocfs2_remove_extent_credits(struct super_block *sb) +{ + return OCFS2_TRUNCATE_LOG_UPDATE + OCFS2_INODE_UPDATE_CREDITS + + ocfs2_quota_trans_credits(sb); +} + +/* data block for new dir/symlink, allocation of directory block, dx_root + * update for free list */ +#define OCFS2_DIR_LINK_ADDITIONAL_CREDITS (1 + OCFS2_SUBALLOC_ALLOC + 1) + +static inline int ocfs2_add_dir_index_credits(struct super_block *sb) +{ + /* 1 block for index, 2 allocs (data, metadata), 1 clusters + * worth of blocks for initial extent. */ + return 1 + 2 * OCFS2_SUBALLOC_ALLOC + + ocfs2_clusters_to_blocks(sb, 1); +} + +/* parent fe, parent block, new file entry, index leaf, inode alloc fe, inode + * alloc group descriptor + mkdir/symlink blocks + dir blocks + xattr + * blocks + quota update */ +static inline int ocfs2_mknod_credits(struct super_block *sb, int is_dir, + int xattr_credits) +{ + int dir_credits = OCFS2_DIR_LINK_ADDITIONAL_CREDITS; + + if (is_dir) + dir_credits += ocfs2_add_dir_index_credits(sb); + + return 4 + OCFS2_SUBALLOC_ALLOC + dir_credits + xattr_credits + + ocfs2_quota_trans_credits(sb); +} + +/* local alloc metadata change + main bitmap updates */ +#define OCFS2_WINDOW_MOVE_CREDITS (OCFS2_INODE_UPDATE_CREDITS \ + + OCFS2_SUBALLOC_ALLOC + OCFS2_SUBALLOC_FREE) + +/* used when we don't need an allocation change for a dir extend. One + * for the dinode, one for the new block. */ +#define OCFS2_SIMPLE_DIR_EXTEND_CREDITS (2) + +/* file update (nlink, etc) + directory mtime/ctime + dir entry block + quota + * update on dir + index leaf + dx root update for free list + + * previous dirblock update in the free list */ +static inline int ocfs2_link_credits(struct super_block *sb) +{ + return 2 * OCFS2_INODE_UPDATE_CREDITS + 4 + + ocfs2_quota_trans_credits(sb); +} + +/* inode + dir inode (if we unlink a dir), + dir entry block + orphan + * dir inode link + dir inode index leaf + dir index root */ +static inline int ocfs2_unlink_credits(struct super_block *sb) +{ + /* The quota update from ocfs2_link_credits is unused here... */ + return 2 * OCFS2_INODE_UPDATE_CREDITS + 3 + ocfs2_link_credits(sb); +} + +/* dinode + orphan dir dinode + inode alloc dinode + orphan dir entry + + * inode alloc group descriptor + orphan dir index root + + * orphan dir index leaf */ +#define OCFS2_DELETE_INODE_CREDITS (3 * OCFS2_INODE_UPDATE_CREDITS + 4) + +/* dinode + orphan dir dinode + extent tree leaf block + orphan dir entry + + * orphan dir index root + orphan dir index leaf */ +#define OCFS2_INODE_ADD_TO_ORPHAN_CREDITS (2 * OCFS2_INODE_UPDATE_CREDITS + 4) +#define OCFS2_INODE_DEL_FROM_ORPHAN_CREDITS OCFS2_INODE_ADD_TO_ORPHAN_CREDITS + +/* dinode update, old dir dinode update, new dir dinode update, old + * dir dir entry, new dir dir entry, dir entry update for renaming + * directory + target unlink + 3 x dir index leaves */ +static inline int ocfs2_rename_credits(struct super_block *sb) +{ + return 3 * OCFS2_INODE_UPDATE_CREDITS + 6 + ocfs2_unlink_credits(sb); +} + +/* global bitmap dinode, group desc., relinked group, + * suballocator dinode, group desc., relinked group, + * dinode, xattr block */ +#define OCFS2_XATTR_BLOCK_CREATE_CREDITS (OCFS2_SUBALLOC_ALLOC * 2 + \ + + OCFS2_INODE_UPDATE_CREDITS \ + + OCFS2_XATTR_BLOCK_UPDATE_CREDITS) + +/* inode update, removal of dx root block from allocator */ +#define OCFS2_DX_ROOT_REMOVE_CREDITS (OCFS2_INODE_UPDATE_CREDITS + \ + OCFS2_SUBALLOC_FREE) + +static inline int ocfs2_calc_dxi_expand_credits(struct super_block *sb) +{ + int credits = 1 + OCFS2_SUBALLOC_ALLOC; + + credits += ocfs2_clusters_to_blocks(sb, 1); + credits += ocfs2_quota_trans_credits(sb); + + return credits; +} + +/* inode update, new refcount block and its allocation credits. */ +#define OCFS2_REFCOUNT_TREE_CREATE_CREDITS (OCFS2_INODE_UPDATE_CREDITS + 1 \ + + OCFS2_SUBALLOC_ALLOC) + +/* inode and the refcount block update. */ +#define OCFS2_REFCOUNT_TREE_SET_CREDITS (OCFS2_INODE_UPDATE_CREDITS + 1) + +/* + * inode and the refcount block update. + * It doesn't include the credits for sub alloc change. + * So if we need to free the bit, OCFS2_SUBALLOC_FREE needs to be added. + */ +#define OCFS2_REFCOUNT_TREE_REMOVE_CREDITS (OCFS2_INODE_UPDATE_CREDITS + 1) + +/* 2 metadata alloc, 2 new blocks and root refcount block */ +#define OCFS2_EXPAND_REFCOUNT_TREE_CREDITS (OCFS2_SUBALLOC_ALLOC * 2 + 3) + +/* + * Please note that the caller must make sure that root_el is the root + * of extent tree. So for an inode, it should be &fe->id2.i_list. Otherwise + * the result may be wrong. + */ +static inline int ocfs2_calc_extend_credits(struct super_block *sb, + struct ocfs2_extent_list *root_el) +{ + int bitmap_blocks, sysfile_bitmap_blocks, extent_blocks; + + /* bitmap dinode, group desc. + relinked group. */ + bitmap_blocks = OCFS2_SUBALLOC_ALLOC; + + /* we might need to shift tree depth so lets assume an + * absolute worst case of complete fragmentation. Even with + * that, we only need one update for the dinode, and then + * however many metadata chunks needed * a remaining suballoc + * alloc. */ + sysfile_bitmap_blocks = 1 + + (OCFS2_SUBALLOC_ALLOC - 1) * ocfs2_extend_meta_needed(root_el); + + /* this does not include *new* metadata blocks, which are + * accounted for in sysfile_bitmap_blocks. root_el + + * prev. last_eb_blk + blocks along edge of tree. + * calc_symlink_credits passes because we just need 1 + * credit for the dinode there. */ + extent_blocks = 1 + 1 + le16_to_cpu(root_el->l_tree_depth); + + return bitmap_blocks + sysfile_bitmap_blocks + extent_blocks + + ocfs2_quota_trans_credits(sb); +} + +static inline int ocfs2_calc_symlink_credits(struct super_block *sb) +{ + int blocks = ocfs2_mknod_credits(sb, 0, 0); + + /* links can be longer than one block so we may update many + * within our single allocated extent. */ + blocks += ocfs2_clusters_to_blocks(sb, 1); + + return blocks + ocfs2_quota_trans_credits(sb); +} + +static inline int ocfs2_calc_group_alloc_credits(struct super_block *sb, + unsigned int cpg) +{ + int blocks; + int bitmap_blocks = OCFS2_SUBALLOC_ALLOC + 1; + /* parent inode update + new block group header + bitmap inode update + + bitmap blocks affected */ + blocks = 1 + 1 + 1 + bitmap_blocks; + return blocks; +} + +/* + * Allocating a discontiguous block group requires the credits from + * ocfs2_calc_group_alloc_credits() as well as enough credits to fill + * the group descriptor's extent list. The caller already has started + * the transaction with ocfs2_calc_group_alloc_credits(). They extend + * it with these credits. + */ +static inline int ocfs2_calc_bg_discontig_credits(struct super_block *sb) +{ + return ocfs2_extent_recs_per_gd(sb); +} + +static inline int ocfs2_jbd2_inode_add_write(handle_t *handle, struct inode *inode, + loff_t start_byte, loff_t length) +{ + return jbd2_journal_inode_ranged_write(handle, + &OCFS2_I(inode)->ip_jinode, + start_byte, length); +} + +static inline int ocfs2_begin_ordered_truncate(struct inode *inode, + loff_t new_size) +{ + return jbd2_journal_begin_ordered_truncate( + OCFS2_SB(inode->i_sb)->journal->j_journal, + &OCFS2_I(inode)->ip_jinode, + new_size); +} + +static inline void ocfs2_update_inode_fsync_trans(handle_t *handle, + struct inode *inode, + int datasync) +{ + struct ocfs2_inode_info *oi = OCFS2_I(inode); + + if (!is_handle_aborted(handle)) { + oi->i_sync_tid = handle->h_transaction->t_tid; + if (datasync) + oi->i_datasync_tid = handle->h_transaction->t_tid; + } +} + +#endif /* OCFS2_JOURNAL_H */ diff --git a/fs/ocfs2/localalloc.c b/fs/ocfs2/localalloc.c new file mode 100644 index 000000000..82d290cf4 --- /dev/null +++ b/fs/ocfs2/localalloc.c @@ -0,0 +1,1318 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * localalloc.c + * + * Node local data allocation + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include + +#include + +#include "ocfs2.h" + +#include "alloc.h" +#include "blockcheck.h" +#include "dlmglue.h" +#include "inode.h" +#include "journal.h" +#include "localalloc.h" +#include "suballoc.h" +#include "super.h" +#include "sysfile.h" +#include "ocfs2_trace.h" + +#include "buffer_head_io.h" + +#define OCFS2_LOCAL_ALLOC(dinode) (&((dinode)->id2.i_lab)) + +static u32 ocfs2_local_alloc_count_bits(struct ocfs2_dinode *alloc); + +static int ocfs2_local_alloc_find_clear_bits(struct ocfs2_super *osb, + struct ocfs2_dinode *alloc, + u32 *numbits, + struct ocfs2_alloc_reservation *resv); + +static void ocfs2_clear_local_alloc(struct ocfs2_dinode *alloc); + +static int ocfs2_sync_local_to_main(struct ocfs2_super *osb, + handle_t *handle, + struct ocfs2_dinode *alloc, + struct inode *main_bm_inode, + struct buffer_head *main_bm_bh); + +static int ocfs2_local_alloc_reserve_for_window(struct ocfs2_super *osb, + struct ocfs2_alloc_context **ac, + struct inode **bitmap_inode, + struct buffer_head **bitmap_bh); + +static int ocfs2_local_alloc_new_window(struct ocfs2_super *osb, + handle_t *handle, + struct ocfs2_alloc_context *ac); + +static int ocfs2_local_alloc_slide_window(struct ocfs2_super *osb, + struct inode *local_alloc_inode); + +/* + * ocfs2_la_default_mb() - determine a default size, in megabytes of + * the local alloc. + * + * Generally, we'd like to pick as large a local alloc as + * possible. Performance on large workloads tends to scale + * proportionally to la size. In addition to that, the reservations + * code functions more efficiently as it can reserve more windows for + * write. + * + * Some things work against us when trying to choose a large local alloc: + * + * - We need to ensure our sizing is picked to leave enough space in + * group descriptors for other allocations (such as block groups, + * etc). Picking default sizes which are a multiple of 4 could help + * - block groups are allocated in 2mb and 4mb chunks. + * + * - Likewise, we don't want to starve other nodes of bits on small + * file systems. This can easily be taken care of by limiting our + * default to a reasonable size (256M) on larger cluster sizes. + * + * - Some file systems can't support very large sizes - 4k and 8k in + * particular are limited to less than 128 and 256 megabytes respectively. + * + * The following reference table shows group descriptor and local + * alloc maximums at various cluster sizes (4k blocksize) + * + * csize: 4K group: 126M la: 121M + * csize: 8K group: 252M la: 243M + * csize: 16K group: 504M la: 486M + * csize: 32K group: 1008M la: 972M + * csize: 64K group: 2016M la: 1944M + * csize: 128K group: 4032M la: 3888M + * csize: 256K group: 8064M la: 7776M + * csize: 512K group: 16128M la: 15552M + * csize: 1024K group: 32256M la: 31104M + */ +#define OCFS2_LA_MAX_DEFAULT_MB 256 +#define OCFS2_LA_OLD_DEFAULT 8 +unsigned int ocfs2_la_default_mb(struct ocfs2_super *osb) +{ + unsigned int la_mb; + unsigned int gd_mb; + unsigned int la_max_mb; + unsigned int megs_per_slot; + struct super_block *sb = osb->sb; + + gd_mb = ocfs2_clusters_to_megabytes(osb->sb, + 8 * ocfs2_group_bitmap_size(sb, 0, osb->s_feature_incompat)); + + /* + * This takes care of files systems with very small group + * descriptors - 512 byte blocksize at cluster sizes lower + * than 16K and also 1k blocksize with 4k cluster size. + */ + if ((sb->s_blocksize == 512 && osb->s_clustersize <= 8192) + || (sb->s_blocksize == 1024 && osb->s_clustersize == 4096)) + return OCFS2_LA_OLD_DEFAULT; + + /* + * Leave enough room for some block groups and make the final + * value we work from a multiple of 4. + */ + gd_mb -= 16; + gd_mb &= 0xFFFFFFFB; + + la_mb = gd_mb; + + /* + * Keep window sizes down to a reasonable default + */ + if (la_mb > OCFS2_LA_MAX_DEFAULT_MB) { + /* + * Some clustersize / blocksize combinations will have + * given us a larger than OCFS2_LA_MAX_DEFAULT_MB + * default size, but get poor distribution when + * limited to exactly 256 megabytes. + * + * As an example, 16K clustersize at 4K blocksize + * gives us a cluster group size of 504M. Paring the + * local alloc size down to 256 however, would give us + * only one window and around 200MB left in the + * cluster group. Instead, find the first size below + * 256 which would give us an even distribution. + * + * Larger cluster group sizes actually work out pretty + * well when pared to 256, so we don't have to do this + * for any group that fits more than two + * OCFS2_LA_MAX_DEFAULT_MB windows. + */ + if (gd_mb > (2 * OCFS2_LA_MAX_DEFAULT_MB)) + la_mb = 256; + else { + unsigned int gd_mult = gd_mb; + + while (gd_mult > 256) + gd_mult = gd_mult >> 1; + + la_mb = gd_mult; + } + } + + megs_per_slot = osb->osb_clusters_at_boot / osb->max_slots; + megs_per_slot = ocfs2_clusters_to_megabytes(osb->sb, megs_per_slot); + /* Too many nodes, too few disk clusters. */ + if (megs_per_slot < la_mb) + la_mb = megs_per_slot; + + /* We can't store more bits than we can in a block. */ + la_max_mb = ocfs2_clusters_to_megabytes(osb->sb, + ocfs2_local_alloc_size(sb) * 8); + if (la_mb > la_max_mb) + la_mb = la_max_mb; + + return la_mb; +} + +void ocfs2_la_set_sizes(struct ocfs2_super *osb, int requested_mb) +{ + struct super_block *sb = osb->sb; + unsigned int la_default_mb = ocfs2_la_default_mb(osb); + unsigned int la_max_mb; + + la_max_mb = ocfs2_clusters_to_megabytes(sb, + ocfs2_local_alloc_size(sb) * 8); + + trace_ocfs2_la_set_sizes(requested_mb, la_max_mb, la_default_mb); + + if (requested_mb == -1) { + /* No user request - use defaults */ + osb->local_alloc_default_bits = + ocfs2_megabytes_to_clusters(sb, la_default_mb); + } else if (requested_mb > la_max_mb) { + /* Request is too big, we give the maximum available */ + osb->local_alloc_default_bits = + ocfs2_megabytes_to_clusters(sb, la_max_mb); + } else { + osb->local_alloc_default_bits = + ocfs2_megabytes_to_clusters(sb, requested_mb); + } + + osb->local_alloc_bits = osb->local_alloc_default_bits; +} + +static inline int ocfs2_la_state_enabled(struct ocfs2_super *osb) +{ + return (osb->local_alloc_state == OCFS2_LA_THROTTLED || + osb->local_alloc_state == OCFS2_LA_ENABLED); +} + +void ocfs2_local_alloc_seen_free_bits(struct ocfs2_super *osb, + unsigned int num_clusters) +{ + if (num_clusters >= osb->local_alloc_default_bits) { + spin_lock(&osb->osb_lock); + if (osb->local_alloc_state == OCFS2_LA_DISABLED || + osb->local_alloc_state == OCFS2_LA_THROTTLED) { + cancel_delayed_work(&osb->la_enable_wq); + osb->local_alloc_state = OCFS2_LA_ENABLED; + } + spin_unlock(&osb->osb_lock); + } +} + +void ocfs2_la_enable_worker(struct work_struct *work) +{ + struct ocfs2_super *osb = + container_of(work, struct ocfs2_super, + la_enable_wq.work); + spin_lock(&osb->osb_lock); + osb->local_alloc_state = OCFS2_LA_ENABLED; + spin_unlock(&osb->osb_lock); +} + +/* + * Tell us whether a given allocation should use the local alloc + * file. Otherwise, it has to go to the main bitmap. + * + * This function does semi-dirty reads of local alloc size and state! + * This is ok however, as the values are re-checked once under mutex. + */ +int ocfs2_alloc_should_use_local(struct ocfs2_super *osb, u64 bits) +{ + int ret = 0; + int la_bits; + + spin_lock(&osb->osb_lock); + la_bits = osb->local_alloc_bits; + + if (!ocfs2_la_state_enabled(osb)) + goto bail; + + /* la_bits should be at least twice the size (in clusters) of + * a new block group. We want to be sure block group + * allocations go through the local alloc, so allow an + * allocation to take up to half the bitmap. */ + if (bits > (la_bits / 2)) + goto bail; + + ret = 1; +bail: + trace_ocfs2_alloc_should_use_local( + (unsigned long long)bits, osb->local_alloc_state, la_bits, ret); + spin_unlock(&osb->osb_lock); + return ret; +} + +int ocfs2_load_local_alloc(struct ocfs2_super *osb) +{ + int status = 0; + struct ocfs2_dinode *alloc = NULL; + struct buffer_head *alloc_bh = NULL; + u32 num_used; + struct inode *inode = NULL; + struct ocfs2_local_alloc *la; + + if (osb->local_alloc_bits == 0) + goto bail; + + if (osb->local_alloc_bits >= osb->bitmap_cpg) { + mlog(ML_NOTICE, "Requested local alloc window %d is larger " + "than max possible %u. Using defaults.\n", + osb->local_alloc_bits, (osb->bitmap_cpg - 1)); + osb->local_alloc_bits = + ocfs2_megabytes_to_clusters(osb->sb, + ocfs2_la_default_mb(osb)); + } + + /* read the alloc off disk */ + inode = ocfs2_get_system_file_inode(osb, LOCAL_ALLOC_SYSTEM_INODE, + osb->slot_num); + if (!inode) { + status = -EINVAL; + mlog_errno(status); + goto bail; + } + + status = ocfs2_read_inode_block_full(inode, &alloc_bh, + OCFS2_BH_IGNORE_CACHE); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + alloc = (struct ocfs2_dinode *) alloc_bh->b_data; + la = OCFS2_LOCAL_ALLOC(alloc); + + if (!(le32_to_cpu(alloc->i_flags) & + (OCFS2_LOCAL_ALLOC_FL|OCFS2_BITMAP_FL))) { + mlog(ML_ERROR, "Invalid local alloc inode, %llu\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno); + status = -EINVAL; + goto bail; + } + + if ((la->la_size == 0) || + (le16_to_cpu(la->la_size) > ocfs2_local_alloc_size(inode->i_sb))) { + mlog(ML_ERROR, "Local alloc size is invalid (la_size = %u)\n", + le16_to_cpu(la->la_size)); + status = -EINVAL; + goto bail; + } + + /* do a little verification. */ + num_used = ocfs2_local_alloc_count_bits(alloc); + + /* hopefully the local alloc has always been recovered before + * we load it. */ + if (num_used + || alloc->id1.bitmap1.i_used + || alloc->id1.bitmap1.i_total + || la->la_bm_off) { + mlog(ML_ERROR, "inconsistent detected, clean journal with" + " unrecovered local alloc, please run fsck.ocfs2!\n" + "found = %u, set = %u, taken = %u, off = %u\n", + num_used, le32_to_cpu(alloc->id1.bitmap1.i_used), + le32_to_cpu(alloc->id1.bitmap1.i_total), + le32_to_cpu(OCFS2_LOCAL_ALLOC(alloc)->la_bm_off)); + + status = -EINVAL; + goto bail; + } + + osb->local_alloc_bh = alloc_bh; + osb->local_alloc_state = OCFS2_LA_ENABLED; + +bail: + if (status < 0) + brelse(alloc_bh); + iput(inode); + + trace_ocfs2_load_local_alloc(osb->local_alloc_bits); + + if (status) + mlog_errno(status); + return status; +} + +/* + * return any unused bits to the bitmap and write out a clean + * local_alloc. + * + * local_alloc_bh is optional. If not passed, we will simply use the + * one off osb. If you do pass it however, be warned that it *will* be + * returned brelse'd and NULL'd out.*/ +void ocfs2_shutdown_local_alloc(struct ocfs2_super *osb) +{ + int status; + handle_t *handle; + struct inode *local_alloc_inode = NULL; + struct buffer_head *bh = NULL; + struct buffer_head *main_bm_bh = NULL; + struct inode *main_bm_inode = NULL; + struct ocfs2_dinode *alloc_copy = NULL; + struct ocfs2_dinode *alloc = NULL; + + cancel_delayed_work(&osb->la_enable_wq); + if (osb->ocfs2_wq) + flush_workqueue(osb->ocfs2_wq); + + if (osb->local_alloc_state == OCFS2_LA_UNUSED) + goto out; + + local_alloc_inode = + ocfs2_get_system_file_inode(osb, + LOCAL_ALLOC_SYSTEM_INODE, + osb->slot_num); + if (!local_alloc_inode) { + status = -ENOENT; + mlog_errno(status); + goto out; + } + + osb->local_alloc_state = OCFS2_LA_DISABLED; + + ocfs2_resmap_uninit(&osb->osb_la_resmap); + + main_bm_inode = ocfs2_get_system_file_inode(osb, + GLOBAL_BITMAP_SYSTEM_INODE, + OCFS2_INVALID_SLOT); + if (!main_bm_inode) { + status = -EINVAL; + mlog_errno(status); + goto out; + } + + inode_lock(main_bm_inode); + + status = ocfs2_inode_lock(main_bm_inode, &main_bm_bh, 1); + if (status < 0) { + mlog_errno(status); + goto out_mutex; + } + + /* WINDOW_MOVE_CREDITS is a bit heavy... */ + handle = ocfs2_start_trans(osb, OCFS2_WINDOW_MOVE_CREDITS); + if (IS_ERR(handle)) { + mlog_errno(PTR_ERR(handle)); + handle = NULL; + goto out_unlock; + } + + bh = osb->local_alloc_bh; + alloc = (struct ocfs2_dinode *) bh->b_data; + + alloc_copy = kmemdup(alloc, bh->b_size, GFP_NOFS); + if (!alloc_copy) { + status = -ENOMEM; + goto out_commit; + } + + status = ocfs2_journal_access_di(handle, INODE_CACHE(local_alloc_inode), + bh, OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto out_commit; + } + + ocfs2_clear_local_alloc(alloc); + ocfs2_journal_dirty(handle, bh); + + brelse(bh); + osb->local_alloc_bh = NULL; + osb->local_alloc_state = OCFS2_LA_UNUSED; + + status = ocfs2_sync_local_to_main(osb, handle, alloc_copy, + main_bm_inode, main_bm_bh); + if (status < 0) + mlog_errno(status); + +out_commit: + ocfs2_commit_trans(osb, handle); + +out_unlock: + brelse(main_bm_bh); + + ocfs2_inode_unlock(main_bm_inode, 1); + +out_mutex: + inode_unlock(main_bm_inode); + iput(main_bm_inode); + +out: + iput(local_alloc_inode); + + kfree(alloc_copy); +} + +/* + * We want to free the bitmap bits outside of any recovery context as + * we'll need a cluster lock to do so, but we must clear the local + * alloc before giving up the recovered nodes journal. To solve this, + * we kmalloc a copy of the local alloc before it's change for the + * caller to process with ocfs2_complete_local_alloc_recovery + */ +int ocfs2_begin_local_alloc_recovery(struct ocfs2_super *osb, + int slot_num, + struct ocfs2_dinode **alloc_copy) +{ + int status = 0; + struct buffer_head *alloc_bh = NULL; + struct inode *inode = NULL; + struct ocfs2_dinode *alloc; + + trace_ocfs2_begin_local_alloc_recovery(slot_num); + + *alloc_copy = NULL; + + inode = ocfs2_get_system_file_inode(osb, + LOCAL_ALLOC_SYSTEM_INODE, + slot_num); + if (!inode) { + status = -EINVAL; + mlog_errno(status); + goto bail; + } + + inode_lock(inode); + + status = ocfs2_read_inode_block_full(inode, &alloc_bh, + OCFS2_BH_IGNORE_CACHE); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + *alloc_copy = kmalloc(alloc_bh->b_size, GFP_KERNEL); + if (!(*alloc_copy)) { + status = -ENOMEM; + goto bail; + } + memcpy((*alloc_copy), alloc_bh->b_data, alloc_bh->b_size); + + alloc = (struct ocfs2_dinode *) alloc_bh->b_data; + ocfs2_clear_local_alloc(alloc); + + ocfs2_compute_meta_ecc(osb->sb, alloc_bh->b_data, &alloc->i_check); + status = ocfs2_write_block(osb, alloc_bh, INODE_CACHE(inode)); + if (status < 0) + mlog_errno(status); + +bail: + if (status < 0) { + kfree(*alloc_copy); + *alloc_copy = NULL; + } + + brelse(alloc_bh); + + if (inode) { + inode_unlock(inode); + iput(inode); + } + + if (status) + mlog_errno(status); + return status; +} + +/* + * Step 2: By now, we've completed the journal recovery, we've stamped + * a clean local alloc on disk and dropped the node out of the + * recovery map. Dlm locks will no longer stall, so lets clear out the + * main bitmap. + */ +int ocfs2_complete_local_alloc_recovery(struct ocfs2_super *osb, + struct ocfs2_dinode *alloc) +{ + int status; + handle_t *handle; + struct buffer_head *main_bm_bh = NULL; + struct inode *main_bm_inode; + + main_bm_inode = ocfs2_get_system_file_inode(osb, + GLOBAL_BITMAP_SYSTEM_INODE, + OCFS2_INVALID_SLOT); + if (!main_bm_inode) { + status = -EINVAL; + mlog_errno(status); + goto out; + } + + inode_lock(main_bm_inode); + + status = ocfs2_inode_lock(main_bm_inode, &main_bm_bh, 1); + if (status < 0) { + mlog_errno(status); + goto out_mutex; + } + + handle = ocfs2_start_trans(osb, OCFS2_WINDOW_MOVE_CREDITS); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + handle = NULL; + mlog_errno(status); + goto out_unlock; + } + + /* we want the bitmap change to be recorded on disk asap */ + handle->h_sync = 1; + + status = ocfs2_sync_local_to_main(osb, handle, alloc, + main_bm_inode, main_bm_bh); + if (status < 0) + mlog_errno(status); + + ocfs2_commit_trans(osb, handle); + +out_unlock: + ocfs2_inode_unlock(main_bm_inode, 1); + +out_mutex: + inode_unlock(main_bm_inode); + + brelse(main_bm_bh); + + iput(main_bm_inode); + +out: + if (!status) + ocfs2_init_steal_slots(osb); + if (status) + mlog_errno(status); + return status; +} + +/* + * make sure we've got at least bits_wanted contiguous bits in the + * local alloc. You lose them when you drop i_rwsem. + * + * We will add ourselves to the transaction passed in, but may start + * our own in order to shift windows. + */ +int ocfs2_reserve_local_alloc_bits(struct ocfs2_super *osb, + u32 bits_wanted, + struct ocfs2_alloc_context *ac) +{ + int status; + struct ocfs2_dinode *alloc; + struct inode *local_alloc_inode; + unsigned int free_bits; + + BUG_ON(!ac); + + local_alloc_inode = + ocfs2_get_system_file_inode(osb, + LOCAL_ALLOC_SYSTEM_INODE, + osb->slot_num); + if (!local_alloc_inode) { + status = -ENOENT; + mlog_errno(status); + goto bail; + } + + inode_lock(local_alloc_inode); + + /* + * We must double check state and allocator bits because + * another process may have changed them while holding i_rwsem. + */ + spin_lock(&osb->osb_lock); + if (!ocfs2_la_state_enabled(osb) || + (bits_wanted > osb->local_alloc_bits)) { + spin_unlock(&osb->osb_lock); + status = -ENOSPC; + goto bail; + } + spin_unlock(&osb->osb_lock); + + alloc = (struct ocfs2_dinode *) osb->local_alloc_bh->b_data; + +#ifdef CONFIG_OCFS2_DEBUG_FS + if (le32_to_cpu(alloc->id1.bitmap1.i_used) != + ocfs2_local_alloc_count_bits(alloc)) { + status = ocfs2_error(osb->sb, "local alloc inode %llu says it has %u used bits, but a count shows %u\n", + (unsigned long long)le64_to_cpu(alloc->i_blkno), + le32_to_cpu(alloc->id1.bitmap1.i_used), + ocfs2_local_alloc_count_bits(alloc)); + goto bail; + } +#endif + + free_bits = le32_to_cpu(alloc->id1.bitmap1.i_total) - + le32_to_cpu(alloc->id1.bitmap1.i_used); + if (bits_wanted > free_bits) { + /* uhoh, window change time. */ + status = + ocfs2_local_alloc_slide_window(osb, local_alloc_inode); + if (status < 0) { + if (status != -ENOSPC) + mlog_errno(status); + goto bail; + } + + /* + * Under certain conditions, the window slide code + * might have reduced the number of bits available or + * disabled the local alloc entirely. Re-check + * here and return -ENOSPC if necessary. + */ + status = -ENOSPC; + if (!ocfs2_la_state_enabled(osb)) + goto bail; + + free_bits = le32_to_cpu(alloc->id1.bitmap1.i_total) - + le32_to_cpu(alloc->id1.bitmap1.i_used); + if (bits_wanted > free_bits) + goto bail; + } + + ac->ac_inode = local_alloc_inode; + /* We should never use localalloc from another slot */ + ac->ac_alloc_slot = osb->slot_num; + ac->ac_which = OCFS2_AC_USE_LOCAL; + get_bh(osb->local_alloc_bh); + ac->ac_bh = osb->local_alloc_bh; + status = 0; +bail: + if (status < 0 && local_alloc_inode) { + inode_unlock(local_alloc_inode); + iput(local_alloc_inode); + } + + trace_ocfs2_reserve_local_alloc_bits( + (unsigned long long)ac->ac_max_block, + bits_wanted, osb->slot_num, status); + + if (status) + mlog_errno(status); + return status; +} + +int ocfs2_claim_local_alloc_bits(struct ocfs2_super *osb, + handle_t *handle, + struct ocfs2_alloc_context *ac, + u32 bits_wanted, + u32 *bit_off, + u32 *num_bits) +{ + int status, start; + struct inode *local_alloc_inode; + void *bitmap; + struct ocfs2_dinode *alloc; + struct ocfs2_local_alloc *la; + + BUG_ON(ac->ac_which != OCFS2_AC_USE_LOCAL); + + local_alloc_inode = ac->ac_inode; + alloc = (struct ocfs2_dinode *) osb->local_alloc_bh->b_data; + la = OCFS2_LOCAL_ALLOC(alloc); + + start = ocfs2_local_alloc_find_clear_bits(osb, alloc, &bits_wanted, + ac->ac_resv); + if (start == -1) { + /* TODO: Shouldn't we just BUG here? */ + status = -ENOSPC; + mlog_errno(status); + goto bail; + } + + bitmap = la->la_bitmap; + *bit_off = le32_to_cpu(la->la_bm_off) + start; + *num_bits = bits_wanted; + + status = ocfs2_journal_access_di(handle, + INODE_CACHE(local_alloc_inode), + osb->local_alloc_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + ocfs2_resmap_claimed_bits(&osb->osb_la_resmap, ac->ac_resv, start, + bits_wanted); + + while(bits_wanted--) + ocfs2_set_bit(start++, bitmap); + + le32_add_cpu(&alloc->id1.bitmap1.i_used, *num_bits); + ocfs2_journal_dirty(handle, osb->local_alloc_bh); + +bail: + if (status) + mlog_errno(status); + return status; +} + +int ocfs2_free_local_alloc_bits(struct ocfs2_super *osb, + handle_t *handle, + struct ocfs2_alloc_context *ac, + u32 bit_off, + u32 num_bits) +{ + int status, start; + u32 clear_bits; + struct inode *local_alloc_inode; + void *bitmap; + struct ocfs2_dinode *alloc; + struct ocfs2_local_alloc *la; + + BUG_ON(ac->ac_which != OCFS2_AC_USE_LOCAL); + + local_alloc_inode = ac->ac_inode; + alloc = (struct ocfs2_dinode *) osb->local_alloc_bh->b_data; + la = OCFS2_LOCAL_ALLOC(alloc); + + bitmap = la->la_bitmap; + start = bit_off - le32_to_cpu(la->la_bm_off); + clear_bits = num_bits; + + status = ocfs2_journal_access_di(handle, + INODE_CACHE(local_alloc_inode), + osb->local_alloc_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + while (clear_bits--) + ocfs2_clear_bit(start++, bitmap); + + le32_add_cpu(&alloc->id1.bitmap1.i_used, -num_bits); + ocfs2_journal_dirty(handle, osb->local_alloc_bh); + +bail: + return status; +} + +static u32 ocfs2_local_alloc_count_bits(struct ocfs2_dinode *alloc) +{ + u32 count; + struct ocfs2_local_alloc *la = OCFS2_LOCAL_ALLOC(alloc); + + count = memweight(la->la_bitmap, le16_to_cpu(la->la_size)); + + trace_ocfs2_local_alloc_count_bits(count); + return count; +} + +static int ocfs2_local_alloc_find_clear_bits(struct ocfs2_super *osb, + struct ocfs2_dinode *alloc, + u32 *numbits, + struct ocfs2_alloc_reservation *resv) +{ + int numfound = 0, bitoff, left, startoff; + int local_resv = 0; + struct ocfs2_alloc_reservation r; + void *bitmap = NULL; + struct ocfs2_reservation_map *resmap = &osb->osb_la_resmap; + + if (!alloc->id1.bitmap1.i_total) { + bitoff = -1; + goto bail; + } + + if (!resv) { + local_resv = 1; + ocfs2_resv_init_once(&r); + ocfs2_resv_set_type(&r, OCFS2_RESV_FLAG_TMP); + resv = &r; + } + + numfound = *numbits; + if (ocfs2_resmap_resv_bits(resmap, resv, &bitoff, &numfound) == 0) { + if (numfound < *numbits) + *numbits = numfound; + goto bail; + } + + /* + * Code error. While reservations are enabled, local + * allocation should _always_ go through them. + */ + BUG_ON(osb->osb_resv_level != 0); + + /* + * Reservations are disabled. Handle this the old way. + */ + + bitmap = OCFS2_LOCAL_ALLOC(alloc)->la_bitmap; + + numfound = bitoff = startoff = 0; + left = le32_to_cpu(alloc->id1.bitmap1.i_total); + while ((bitoff = ocfs2_find_next_zero_bit(bitmap, left, startoff)) < + left) { + /* Ok, we found a zero bit... is it contig. or do we + * start over?*/ + if (bitoff == startoff) { + /* we found a zero */ + numfound++; + startoff++; + } else { + /* got a zero after some ones */ + numfound = 1; + startoff = bitoff+1; + } + /* we got everything we needed */ + if (numfound == *numbits) { + /* mlog(0, "Found it all!\n"); */ + break; + } + } + + trace_ocfs2_local_alloc_find_clear_bits_search_bitmap(bitoff, numfound); + + if (numfound == *numbits) + bitoff = startoff - numfound; + else + bitoff = -1; + +bail: + if (local_resv) + ocfs2_resv_discard(resmap, resv); + + trace_ocfs2_local_alloc_find_clear_bits(*numbits, + le32_to_cpu(alloc->id1.bitmap1.i_total), + bitoff, numfound); + + return bitoff; +} + +static void ocfs2_clear_local_alloc(struct ocfs2_dinode *alloc) +{ + struct ocfs2_local_alloc *la = OCFS2_LOCAL_ALLOC(alloc); + + alloc->id1.bitmap1.i_total = 0; + alloc->id1.bitmap1.i_used = 0; + la->la_bm_off = 0; + memset(la->la_bitmap, 0, le16_to_cpu(la->la_size)); +} + +#if 0 +/* turn this on and uncomment below to aid debugging window shifts. */ +static void ocfs2_verify_zero_bits(unsigned long *bitmap, + unsigned int start, + unsigned int count) +{ + unsigned int tmp = count; + while(tmp--) { + if (ocfs2_test_bit(start + tmp, bitmap)) { + printk("ocfs2_verify_zero_bits: start = %u, count = " + "%u\n", start, count); + printk("ocfs2_verify_zero_bits: bit %u is set!", + start + tmp); + BUG(); + } + } +} +#endif + +/* + * sync the local alloc to main bitmap. + * + * assumes you've already locked the main bitmap -- the bitmap inode + * passed is used for caching. + */ +static int ocfs2_sync_local_to_main(struct ocfs2_super *osb, + handle_t *handle, + struct ocfs2_dinode *alloc, + struct inode *main_bm_inode, + struct buffer_head *main_bm_bh) +{ + int status = 0; + int bit_off, left, count, start; + u64 la_start_blk; + u64 blkno; + void *bitmap; + struct ocfs2_local_alloc *la = OCFS2_LOCAL_ALLOC(alloc); + + trace_ocfs2_sync_local_to_main( + le32_to_cpu(alloc->id1.bitmap1.i_total), + le32_to_cpu(alloc->id1.bitmap1.i_used)); + + if (!alloc->id1.bitmap1.i_total) { + goto bail; + } + + if (le32_to_cpu(alloc->id1.bitmap1.i_used) == + le32_to_cpu(alloc->id1.bitmap1.i_total)) { + goto bail; + } + + la_start_blk = ocfs2_clusters_to_blocks(osb->sb, + le32_to_cpu(la->la_bm_off)); + bitmap = la->la_bitmap; + start = count = 0; + left = le32_to_cpu(alloc->id1.bitmap1.i_total); + + while (1) { + bit_off = ocfs2_find_next_zero_bit(bitmap, left, start); + if ((bit_off < left) && (bit_off == start)) { + count++; + start++; + continue; + } + if (count) { + blkno = la_start_blk + + ocfs2_clusters_to_blocks(osb->sb, + start - count); + + trace_ocfs2_sync_local_to_main_free( + count, start - count, + (unsigned long long)la_start_blk, + (unsigned long long)blkno); + + status = ocfs2_release_clusters(handle, + main_bm_inode, + main_bm_bh, blkno, + count); + if (status < 0) { + mlog_errno(status); + goto bail; + } + } + + if (bit_off >= left) + break; + count = 1; + start = bit_off + 1; + } + +bail: + if (status) + mlog_errno(status); + return status; +} + +enum ocfs2_la_event { + OCFS2_LA_EVENT_SLIDE, /* Normal window slide. */ + OCFS2_LA_EVENT_FRAGMENTED, /* The global bitmap has + * enough bits theoretically + * free, but a contiguous + * allocation could not be + * found. */ + OCFS2_LA_EVENT_ENOSPC, /* Global bitmap doesn't have + * enough bits free to satisfy + * our request. */ +}; +#define OCFS2_LA_ENABLE_INTERVAL (30 * HZ) +/* + * Given an event, calculate the size of our next local alloc window. + * + * This should always be called under i_rwsem of the local alloc inode + * so that local alloc disabling doesn't race with processes trying to + * use the allocator. + * + * Returns the state which the local alloc was left in. This value can + * be ignored by some paths. + */ +static int ocfs2_recalc_la_window(struct ocfs2_super *osb, + enum ocfs2_la_event event) +{ + unsigned int bits; + int state; + + spin_lock(&osb->osb_lock); + if (osb->local_alloc_state == OCFS2_LA_DISABLED) { + WARN_ON_ONCE(osb->local_alloc_state == OCFS2_LA_DISABLED); + goto out_unlock; + } + + /* + * ENOSPC and fragmentation are treated similarly for now. + */ + if (event == OCFS2_LA_EVENT_ENOSPC || + event == OCFS2_LA_EVENT_FRAGMENTED) { + /* + * We ran out of contiguous space in the primary + * bitmap. Drastically reduce the number of bits used + * by local alloc until we have to disable it. + */ + bits = osb->local_alloc_bits >> 1; + if (bits > ocfs2_megabytes_to_clusters(osb->sb, 1)) { + /* + * By setting state to THROTTLED, we'll keep + * the number of local alloc bits used down + * until an event occurs which would give us + * reason to assume the bitmap situation might + * have changed. + */ + osb->local_alloc_state = OCFS2_LA_THROTTLED; + osb->local_alloc_bits = bits; + } else { + osb->local_alloc_state = OCFS2_LA_DISABLED; + } + queue_delayed_work(osb->ocfs2_wq, &osb->la_enable_wq, + OCFS2_LA_ENABLE_INTERVAL); + goto out_unlock; + } + + /* + * Don't increase the size of the local alloc window until we + * know we might be able to fulfill the request. Otherwise, we + * risk bouncing around the global bitmap during periods of + * low space. + */ + if (osb->local_alloc_state != OCFS2_LA_THROTTLED) + osb->local_alloc_bits = osb->local_alloc_default_bits; + +out_unlock: + state = osb->local_alloc_state; + spin_unlock(&osb->osb_lock); + + return state; +} + +static int ocfs2_local_alloc_reserve_for_window(struct ocfs2_super *osb, + struct ocfs2_alloc_context **ac, + struct inode **bitmap_inode, + struct buffer_head **bitmap_bh) +{ + int status; + + *ac = kzalloc_obj(struct ocfs2_alloc_context); + if (!(*ac)) { + status = -ENOMEM; + mlog_errno(status); + goto bail; + } + +retry_enospc: + (*ac)->ac_bits_wanted = osb->local_alloc_bits; + status = ocfs2_reserve_cluster_bitmap_bits(osb, *ac); + if (status == -ENOSPC) { + if (ocfs2_recalc_la_window(osb, OCFS2_LA_EVENT_ENOSPC) == + OCFS2_LA_DISABLED) + goto bail; + + ocfs2_free_ac_resource(*ac); + memset(*ac, 0, sizeof(struct ocfs2_alloc_context)); + goto retry_enospc; + } + if (status < 0) { + mlog_errno(status); + goto bail; + } + + *bitmap_inode = (*ac)->ac_inode; + igrab(*bitmap_inode); + *bitmap_bh = (*ac)->ac_bh; + get_bh(*bitmap_bh); + status = 0; +bail: + if ((status < 0) && *ac) { + ocfs2_free_alloc_context(*ac); + *ac = NULL; + } + + if (status) + mlog_errno(status); + return status; +} + +/* + * pass it the bitmap lock in lock_bh if you have it. + */ +static int ocfs2_local_alloc_new_window(struct ocfs2_super *osb, + handle_t *handle, + struct ocfs2_alloc_context *ac) +{ + int status = 0; + u32 cluster_off, cluster_count; + struct ocfs2_dinode *alloc = NULL; + struct ocfs2_local_alloc *la; + + alloc = (struct ocfs2_dinode *) osb->local_alloc_bh->b_data; + la = OCFS2_LOCAL_ALLOC(alloc); + + trace_ocfs2_local_alloc_new_window( + le32_to_cpu(alloc->id1.bitmap1.i_total), + osb->local_alloc_bits); + + /* Instruct the allocation code to try the most recently used + * cluster group. We'll re-record the group used this pass + * below. */ + ac->ac_last_group = osb->la_last_gd; + + /* we used the generic suballoc reserve function, but we set + * everything up nicely, so there's no reason why we can't use + * the more specific cluster api to claim bits. */ + status = ocfs2_claim_clusters(handle, ac, osb->local_alloc_bits, + &cluster_off, &cluster_count); + if (status == -ENOSPC) { +retry_enospc: + /* + * Note: We could also try syncing the journal here to + * allow use of any free bits which the current + * transaction can't give us access to. --Mark + */ + if (ocfs2_recalc_la_window(osb, OCFS2_LA_EVENT_FRAGMENTED) == + OCFS2_LA_DISABLED) + goto bail; + + ac->ac_bits_wanted = osb->local_alloc_bits; + status = ocfs2_claim_clusters(handle, ac, + osb->local_alloc_bits, + &cluster_off, + &cluster_count); + if (status == -ENOSPC) + goto retry_enospc; + /* + * We only shrunk the *minimum* number of in our + * request - it's entirely possible that the allocator + * might give us more than we asked for. + */ + if (status == 0) { + spin_lock(&osb->osb_lock); + osb->local_alloc_bits = cluster_count; + spin_unlock(&osb->osb_lock); + } + } + if (status < 0) { + if (status != -ENOSPC) + mlog_errno(status); + goto bail; + } + + osb->la_last_gd = ac->ac_last_group; + + la->la_bm_off = cpu_to_le32(cluster_off); + alloc->id1.bitmap1.i_total = cpu_to_le32(cluster_count); + /* just in case... In the future when we find space ourselves, + * we don't have to get all contiguous -- but we'll have to + * set all previously used bits in bitmap and update + * la_bits_set before setting the bits in the main bitmap. */ + alloc->id1.bitmap1.i_used = 0; + memset(OCFS2_LOCAL_ALLOC(alloc)->la_bitmap, 0, + le16_to_cpu(la->la_size)); + + ocfs2_resmap_restart(&osb->osb_la_resmap, cluster_count, + OCFS2_LOCAL_ALLOC(alloc)->la_bitmap); + + trace_ocfs2_local_alloc_new_window_result( + le32_to_cpu(OCFS2_LOCAL_ALLOC(alloc)->la_bm_off), + le32_to_cpu(alloc->id1.bitmap1.i_total)); + +bail: + if (status) + mlog_errno(status); + return status; +} + +/* Note that we do *NOT* lock the local alloc inode here as + * it's been locked already for us. */ +static int ocfs2_local_alloc_slide_window(struct ocfs2_super *osb, + struct inode *local_alloc_inode) +{ + int status = 0; + struct buffer_head *main_bm_bh = NULL; + struct inode *main_bm_inode = NULL; + handle_t *handle = NULL; + struct ocfs2_dinode *alloc; + struct ocfs2_dinode *alloc_copy = NULL; + struct ocfs2_alloc_context *ac = NULL; + + ocfs2_recalc_la_window(osb, OCFS2_LA_EVENT_SLIDE); + + /* This will lock the main bitmap for us. */ + status = ocfs2_local_alloc_reserve_for_window(osb, + &ac, + &main_bm_inode, + &main_bm_bh); + if (status < 0) { + if (status != -ENOSPC) + mlog_errno(status); + goto bail; + } + + handle = ocfs2_start_trans(osb, OCFS2_WINDOW_MOVE_CREDITS); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + handle = NULL; + mlog_errno(status); + goto bail; + } + + alloc = (struct ocfs2_dinode *) osb->local_alloc_bh->b_data; + + /* We want to clear the local alloc before doing anything + * else, so that if we error later during this operation, + * local alloc shutdown won't try to double free main bitmap + * bits. Make a copy so the sync function knows which bits to + * free. */ + alloc_copy = kmemdup(alloc, osb->local_alloc_bh->b_size, GFP_NOFS); + if (!alloc_copy) { + status = -ENOMEM; + mlog_errno(status); + goto bail; + } + + status = ocfs2_journal_access_di(handle, + INODE_CACHE(local_alloc_inode), + osb->local_alloc_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + ocfs2_clear_local_alloc(alloc); + ocfs2_journal_dirty(handle, osb->local_alloc_bh); + + status = ocfs2_sync_local_to_main(osb, handle, alloc_copy, + main_bm_inode, main_bm_bh); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + status = ocfs2_local_alloc_new_window(osb, handle, ac); + if (status < 0) { + if (status != -ENOSPC) + mlog_errno(status); + goto bail; + } + + atomic_inc(&osb->alloc_stats.moves); + +bail: + if (handle) + ocfs2_commit_trans(osb, handle); + + brelse(main_bm_bh); + + iput(main_bm_inode); + kfree(alloc_copy); + + if (ac) + ocfs2_free_alloc_context(ac); + + if (status) + mlog_errno(status); + return status; +} + diff --git a/fs/ocfs2/localalloc.h b/fs/ocfs2/localalloc.h new file mode 100644 index 000000000..08f925b7e --- /dev/null +++ b/fs/ocfs2/localalloc.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * localalloc.h + * + * Function prototypes + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#ifndef OCFS2_LOCALALLOC_H +#define OCFS2_LOCALALLOC_H + +int ocfs2_load_local_alloc(struct ocfs2_super *osb); + +void ocfs2_shutdown_local_alloc(struct ocfs2_super *osb); + +void ocfs2_la_set_sizes(struct ocfs2_super *osb, int requested_mb); +unsigned int ocfs2_la_default_mb(struct ocfs2_super *osb); + +int ocfs2_begin_local_alloc_recovery(struct ocfs2_super *osb, + int node_num, + struct ocfs2_dinode **alloc_copy); + +int ocfs2_complete_local_alloc_recovery(struct ocfs2_super *osb, + struct ocfs2_dinode *alloc); + +int ocfs2_alloc_should_use_local(struct ocfs2_super *osb, + u64 bits); + +struct ocfs2_alloc_context; +int ocfs2_reserve_local_alloc_bits(struct ocfs2_super *osb, + u32 bits_wanted, + struct ocfs2_alloc_context *ac); + +int ocfs2_claim_local_alloc_bits(struct ocfs2_super *osb, + handle_t *handle, + struct ocfs2_alloc_context *ac, + u32 bits_wanted, + u32 *bit_off, + u32 *num_bits); + +int ocfs2_free_local_alloc_bits(struct ocfs2_super *osb, + handle_t *handle, + struct ocfs2_alloc_context *ac, + u32 bit_off, + u32 num_bits); + +void ocfs2_local_alloc_seen_free_bits(struct ocfs2_super *osb, + unsigned int num_clusters); +void ocfs2_la_enable_worker(struct work_struct *work); + +#endif /* OCFS2_LOCALALLOC_H */ diff --git a/fs/ocfs2/locks.c b/fs/ocfs2/locks.c new file mode 100644 index 000000000..6de944818 --- /dev/null +++ b/fs/ocfs2/locks.c @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * locks.c + * + * Userspace file locking support + * + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#include +#include +#include + +#include + +#include "ocfs2.h" + +#include "dlmglue.h" +#include "file.h" +#include "inode.h" +#include "locks.h" + +static int ocfs2_do_flock(struct file *file, struct inode *inode, + int cmd, struct file_lock *fl) +{ + int ret = 0, level = 0, trylock = 0; + struct ocfs2_file_private *fp = file->private_data; + struct ocfs2_lock_res *lockres = &fp->fp_flock; + + if (lock_is_write(fl)) + level = 1; + if (!IS_SETLKW(cmd)) + trylock = 1; + + mutex_lock(&fp->fp_mutex); + + if (lockres->l_flags & OCFS2_LOCK_ATTACHED && + lockres->l_level > LKM_NLMODE) { + int old_level = 0; + struct file_lock request; + + if (lockres->l_level == LKM_EXMODE) + old_level = 1; + + if (level == old_level) + goto out; + + /* + * Converting an existing lock is not guaranteed to be + * atomic, so we can get away with simply unlocking + * here and allowing the lock code to try at the new + * level. + */ + + locks_init_lock(&request); + request.c.flc_type = F_UNLCK; + request.c.flc_flags = FL_FLOCK; + locks_lock_file_wait(file, &request); + + ocfs2_file_unlock(file); + } + + ret = ocfs2_file_lock(file, level, trylock); + if (ret) { + if (ret == -EAGAIN && trylock) + ret = -EWOULDBLOCK; + else + mlog_errno(ret); + goto out; + } + + ret = locks_lock_file_wait(file, fl); + if (ret) + ocfs2_file_unlock(file); + +out: + mutex_unlock(&fp->fp_mutex); + + return ret; +} + +static int ocfs2_do_funlock(struct file *file, int cmd, struct file_lock *fl) +{ + int ret; + struct ocfs2_file_private *fp = file->private_data; + + mutex_lock(&fp->fp_mutex); + ocfs2_file_unlock(file); + ret = locks_lock_file_wait(file, fl); + mutex_unlock(&fp->fp_mutex); + + return ret; +} + +/* + * Overall flow of ocfs2_flock() was influenced by gfs2_flock(). + */ +int ocfs2_flock(struct file *file, int cmd, struct file_lock *fl) +{ + struct inode *inode = file->f_mapping->host; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + if (!(fl->c.flc_flags & FL_FLOCK)) + return -ENOLCK; + + if ((osb->s_mount_opt & OCFS2_MOUNT_LOCALFLOCKS) || + ocfs2_mount_local(osb)) + return locks_lock_file_wait(file, fl); + + if (lock_is_unlock(fl)) + return ocfs2_do_funlock(file, cmd, fl); + else + return ocfs2_do_flock(file, inode, cmd, fl); +} + +int ocfs2_lock(struct file *file, int cmd, struct file_lock *fl) +{ + struct inode *inode = file->f_mapping->host; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + if (!(fl->c.flc_flags & FL_POSIX)) + return -ENOLCK; + + return ocfs2_plock(osb->cconn, OCFS2_I(inode)->ip_blkno, file, cmd, fl); +} diff --git a/fs/ocfs2/locks.h b/fs/ocfs2/locks.h new file mode 100644 index 000000000..b52de3947 --- /dev/null +++ b/fs/ocfs2/locks.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * locks.h + * + * Function prototypes for Userspace file locking support + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#ifndef OCFS2_LOCKS_H +#define OCFS2_LOCKS_H + +int ocfs2_flock(struct file *file, int cmd, struct file_lock *fl); +int ocfs2_lock(struct file *file, int cmd, struct file_lock *fl); + +#endif /* OCFS2_LOCKS_H */ diff --git a/fs/ocfs2/mmap.c b/fs/ocfs2/mmap.c new file mode 100644 index 000000000..6c570157c --- /dev/null +++ b/fs/ocfs2/mmap.c @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * mmap.c + * + * Code to deal with the mess that is clustered mmap. + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ocfs2.h" + +#include "aops.h" +#include "dlmglue.h" +#include "file.h" +#include "inode.h" +#include "mmap.h" +#include "super.h" +#include "ocfs2_trace.h" + + +static vm_fault_t ocfs2_fault(struct vm_fault *vmf) +{ + unsigned long long ip_blkno = + OCFS2_I(file_inode(vmf->vma->vm_file))->ip_blkno; + sigset_t oldset; + vm_fault_t ret; + + ocfs2_block_signals(&oldset); + ret = filemap_fault(vmf); + ocfs2_unblock_signals(&oldset); + + trace_ocfs2_fault(ip_blkno, vmf->page, vmf->pgoff); + return ret; +} +static vm_fault_t __ocfs2_page_mkwrite(struct file *file, + struct buffer_head *di_bh, struct folio *folio) +{ + int err; + vm_fault_t ret = VM_FAULT_NOPAGE; + struct inode *inode = file_inode(file); + struct address_space *mapping = inode->i_mapping; + loff_t pos = folio_pos(folio); + unsigned int len = PAGE_SIZE; + pgoff_t last_index; + struct folio *locked_folio = NULL; + void *fsdata; + loff_t size = i_size_read(inode); + + last_index = (size - 1) >> PAGE_SHIFT; + + /* + * There are cases that lead to the page no longer belonging to the + * mapping. + * 1) pagecache truncates locally due to memory pressure. + * 2) pagecache truncates when another is taking EX lock against + * inode lock. see ocfs2_data_convert_worker. + * + * The i_size check doesn't catch the case where nodes truncated and + * then re-extended the file. We'll re-check the page mapping after + * taking the page lock inside of ocfs2_write_begin_nolock(). + * + * Let VM retry with these cases. + */ + if ((folio->mapping != inode->i_mapping) || + !folio_test_uptodate(folio) || + (pos >= size)) + goto out; + + /* + * Call ocfs2_write_begin() and ocfs2_write_end() to take + * advantage of the allocation code there. We pass a write + * length of the whole page (chopped to i_size) to make sure + * the whole thing is allocated. + * + * Since we know the page is up to date, we don't have to + * worry about ocfs2_write_begin() skipping some buffer reads + * because the "write" would invalidate their data. + */ + if (folio->index == last_index) + len = ((size - 1) & ~PAGE_MASK) + 1; + + err = ocfs2_write_begin_nolock(mapping, pos, len, OCFS2_WRITE_MMAP, + &locked_folio, &fsdata, di_bh, folio); + if (err) { + if (err != -ENOSPC) + mlog_errno(err); + ret = vmf_error(err); + goto out; + } + + if (!locked_folio) { + ret = VM_FAULT_NOPAGE; + goto out; + } + err = ocfs2_write_end_nolock(mapping, pos, len, len, fsdata); + BUG_ON(err != len); + ret = VM_FAULT_LOCKED; +out: + return ret; +} + +static vm_fault_t ocfs2_page_mkwrite(struct vm_fault *vmf) +{ + struct folio *folio = page_folio(vmf->page); + struct inode *inode = file_inode(vmf->vma->vm_file); + struct buffer_head *di_bh = NULL; + sigset_t oldset; + int err; + vm_fault_t ret; + + sb_start_pagefault(inode->i_sb); + ocfs2_block_signals(&oldset); + + /* + * The cluster locks taken will block a truncate from another + * node. Taking the data lock will also ensure that we don't + * attempt page truncation as part of a downconvert. + */ + err = ocfs2_inode_lock(inode, &di_bh, 1); + if (err < 0) { + mlog_errno(err); + ret = vmf_error(err); + goto out; + } + + /* + * The alloc sem should be enough to serialize with + * ocfs2_truncate_file() changing i_size as well as any thread + * modifying the inode btree. + */ + down_write(&OCFS2_I(inode)->ip_alloc_sem); + + ret = __ocfs2_page_mkwrite(vmf->vma->vm_file, di_bh, folio); + + up_write(&OCFS2_I(inode)->ip_alloc_sem); + + brelse(di_bh); + ocfs2_inode_unlock(inode, 1); + +out: + ocfs2_unblock_signals(&oldset); + sb_end_pagefault(inode->i_sb); + return ret; +} + +static const struct vm_operations_struct ocfs2_file_vm_ops = { + .fault = ocfs2_fault, + .page_mkwrite = ocfs2_page_mkwrite, +}; + +int ocfs2_mmap_prepare(struct vm_area_desc *desc) +{ + struct file *file = desc->file; + int ret = 0, lock_level = 0; + + ret = ocfs2_inode_lock_atime(file_inode(file), + file->f_path.mnt, &lock_level, 1); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + ocfs2_inode_unlock(file_inode(file), lock_level); +out: + desc->vm_ops = &ocfs2_file_vm_ops; + return 0; +} + diff --git a/fs/ocfs2/mmap.h b/fs/ocfs2/mmap.h new file mode 100644 index 000000000..d21c30de6 --- /dev/null +++ b/fs/ocfs2/mmap.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef OCFS2_MMAP_H +#define OCFS2_MMAP_H + +int ocfs2_mmap_prepare(struct vm_area_desc *desc); + +#endif /* OCFS2_MMAP_H */ diff --git a/fs/ocfs2/move_extents.c b/fs/ocfs2/move_extents.c new file mode 100644 index 000000000..ad1678ee7 --- /dev/null +++ b/fs/ocfs2/move_extents.c @@ -0,0 +1,1094 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * move_extents.c + * + * Copyright (C) 2011 Oracle. All rights reserved. + */ +#include +#include +#include +#include + +#include + +#include "ocfs2.h" +#include "ocfs2_ioctl.h" + +#include "alloc.h" +#include "localalloc.h" +#include "aops.h" +#include "dlmglue.h" +#include "extent_map.h" +#include "inode.h" +#include "journal.h" +#include "suballoc.h" +#include "uptodate.h" +#include "super.h" +#include "dir.h" +#include "buffer_head_io.h" +#include "sysfile.h" +#include "refcounttree.h" +#include "move_extents.h" + +struct ocfs2_move_extents_context { + struct inode *inode; + struct file *file; + int auto_defrag; + int partial; + int credits; + u32 new_phys_cpos; + u32 clusters_moved; + u64 refcount_loc; + struct ocfs2_move_extents *range; + struct ocfs2_extent_tree et; + struct ocfs2_alloc_context *meta_ac; + struct ocfs2_alloc_context *data_ac; + struct ocfs2_cached_dealloc_ctxt dealloc; +}; + +static int __ocfs2_move_extent(handle_t *handle, + struct ocfs2_move_extents_context *context, + u32 cpos, u32 len, u32 p_cpos, u32 new_p_cpos, + int ext_flags) +{ + int ret = 0, index; + struct inode *inode = context->inode; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct ocfs2_extent_rec *rec, replace_rec; + struct ocfs2_path *path = NULL; + struct ocfs2_extent_list *el; + u64 ino = ocfs2_metadata_cache_owner(context->et.et_ci); + u64 old_blkno = ocfs2_clusters_to_blocks(inode->i_sb, p_cpos); + + ret = ocfs2_duplicate_clusters_by_page(handle, inode, cpos, + p_cpos, new_p_cpos, len); + if (ret) { + mlog_errno(ret); + goto out; + } + + memset(&replace_rec, 0, sizeof(replace_rec)); + replace_rec.e_cpos = cpu_to_le32(cpos); + replace_rec.e_leaf_clusters = cpu_to_le16(len); + replace_rec.e_blkno = cpu_to_le64(ocfs2_clusters_to_blocks(inode->i_sb, + new_p_cpos)); + + path = ocfs2_new_path_from_et(&context->et); + if (!path) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + ret = ocfs2_find_path(INODE_CACHE(inode), path, cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + + el = path_leaf_el(path); + + index = ocfs2_search_extent_list(el, cpos); + if (index == -1) { + ret = ocfs2_error(inode->i_sb, + "Inode %llu has an extent at cpos %u which can no longer be found\n", + (unsigned long long)ino, cpos); + goto out; + } + + rec = &el->l_recs[index]; + + if (ext_flags != rec->e_flags) { + ret = ocfs2_error(inode->i_sb, + "Inode %llu has corrupted extent %d with flags 0x%x at cpos %u\n", + (unsigned long long)ino, index, rec->e_flags, cpos); + goto out; + } + + /* + * after moving/defraging to new location, the extent is not going + * to be refcounted anymore. + */ + replace_rec.e_flags = ext_flags & ~OCFS2_EXT_REFCOUNTED; + + ret = ocfs2_split_extent(handle, &context->et, path, index, + &replace_rec, context->meta_ac, + &context->dealloc); + if (ret) { + mlog_errno(ret); + goto out; + } + + context->new_phys_cpos = new_p_cpos; + + /* + * need I to append truncate log for old clusters? + */ + if (old_blkno) { + if (ext_flags & OCFS2_EXT_REFCOUNTED) + ret = ocfs2_decrease_refcount(inode, handle, + ocfs2_blocks_to_clusters(osb->sb, + old_blkno), + len, context->meta_ac, + &context->dealloc, 1); + else + ret = ocfs2_truncate_log_append(osb, handle, + old_blkno, len); + } + + ocfs2_update_inode_fsync_trans(handle, inode, 0); +out: + ocfs2_free_path(path); + return ret; +} + +/* + * lock allocator, and reserve appropriate number of bits for + * meta blocks. + */ +static int ocfs2_lock_meta_allocator_move_extents(struct inode *inode, + struct ocfs2_extent_tree *et, + u32 clusters_to_move, + u32 extents_to_split, + struct ocfs2_alloc_context **meta_ac, + int extra_blocks, + int *credits) +{ + int ret, num_free_extents; + unsigned int max_recs_needed = 2 * extents_to_split + clusters_to_move; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + num_free_extents = ocfs2_num_free_extents(et); + if (num_free_extents < 0) { + ret = num_free_extents; + mlog_errno(ret); + goto out; + } + + if (!num_free_extents || + (ocfs2_sparse_alloc(osb) && num_free_extents < max_recs_needed)) + extra_blocks += ocfs2_extend_meta_needed(et->et_root_el); + + ret = ocfs2_reserve_new_metadata_blocks(osb, extra_blocks, meta_ac); + if (ret) { + mlog_errno(ret); + goto out; + } + + + *credits += ocfs2_calc_extend_credits(osb->sb, et->et_root_el); + + mlog(0, "reserve metadata_blocks: %d, data_clusters: %u, credits: %d\n", + extra_blocks, clusters_to_move, *credits); +out: + if (ret) { + if (*meta_ac) { + ocfs2_free_alloc_context(*meta_ac); + *meta_ac = NULL; + } + } + + return ret; +} + +/* + * Using one journal handle to guarantee the data consistency in case + * crash happens anywhere. + * + * XXX: defrag can end up with finishing partial extent as requested, + * due to not enough contiguous clusters can be found in allocator. + */ +static int ocfs2_defrag_extent(struct ocfs2_move_extents_context *context, + u32 cpos, u32 phys_cpos, u32 *len, int ext_flags) +{ + int ret, credits = 0, extra_blocks = 0, partial = context->partial; + handle_t *handle; + struct inode *inode = context->inode; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct inode *tl_inode = osb->osb_tl_inode; + struct ocfs2_refcount_tree *ref_tree = NULL; + u32 new_phys_cpos, new_len; + u64 phys_blkno = ocfs2_clusters_to_blocks(inode->i_sb, phys_cpos); + int need_free = 0; + + if ((ext_flags & OCFS2_EXT_REFCOUNTED) && *len) { + BUG_ON(!ocfs2_is_refcount_inode(inode)); + BUG_ON(!context->refcount_loc); + + ret = ocfs2_lock_refcount_tree(osb, context->refcount_loc, 1, + &ref_tree, NULL); + if (ret) { + mlog_errno(ret); + return ret; + } + + ret = ocfs2_prepare_refcount_change_for_del(inode, + context->refcount_loc, + phys_blkno, + *len, + &credits, + &extra_blocks); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + ret = ocfs2_lock_meta_allocator_move_extents(inode, &context->et, + *len, 1, + &context->meta_ac, + extra_blocks, &credits); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * should be using allocation reservation strategy there? + * + * if (context->data_ac) + * context->data_ac->ac_resv = &OCFS2_I(inode)->ip_la_data_resv; + */ + + inode_lock(tl_inode); + + if (ocfs2_truncate_log_needs_flush(osb)) { + ret = __ocfs2_flush_truncate_log(osb); + if (ret < 0) { + mlog_errno(ret); + goto out_unlock_mutex; + } + } + + /* + * Make sure ocfs2_reserve_cluster is called after + * __ocfs2_flush_truncate_log, otherwise, dead lock may happen. + * + * If ocfs2_reserve_cluster is called + * before __ocfs2_flush_truncate_log, dead lock on global bitmap + * may happen. + * + */ + ret = ocfs2_reserve_clusters(osb, *len, &context->data_ac); + if (ret) { + mlog_errno(ret); + goto out_unlock_mutex; + } + + handle = ocfs2_start_trans(osb, credits); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out_unlock_mutex; + } + + ret = __ocfs2_claim_clusters(handle, context->data_ac, 1, *len, + &new_phys_cpos, &new_len); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + /* + * allowing partial extent moving is kind of 'pros and cons', it makes + * whole defragmentation less likely to fail, on the contrary, the bad + * thing is it may make the fs even more fragmented after moving, let + * userspace make a good decision here. + */ + if (new_len != *len) { + mlog(0, "len_claimed: %u, len: %u\n", new_len, *len); + if (!partial) { + context->range->me_flags &= ~OCFS2_MOVE_EXT_FL_COMPLETE; + ret = -ENOSPC; + need_free = 1; + goto out_commit; + } + } + + mlog(0, "cpos: %u, phys_cpos: %u, new_phys_cpos: %u\n", cpos, + phys_cpos, new_phys_cpos); + + ret = __ocfs2_move_extent(handle, context, cpos, new_len, phys_cpos, + new_phys_cpos, ext_flags); + if (ret) + mlog_errno(ret); + + if (partial && (new_len != *len)) + *len = new_len; + + /* + * Here we should write the new page out first if we are + * in write-back mode. + */ + ret = ocfs2_cow_sync_writeback(inode->i_sb, context->inode, cpos, *len); + if (ret) + mlog_errno(ret); + +out_commit: + if (need_free && context->data_ac) { + struct ocfs2_alloc_context *data_ac = context->data_ac; + + if (context->data_ac->ac_which == OCFS2_AC_USE_LOCAL) + ocfs2_free_local_alloc_bits(osb, handle, data_ac, + new_phys_cpos, new_len); + else + ocfs2_free_clusters(handle, + data_ac->ac_inode, + data_ac->ac_bh, + ocfs2_clusters_to_blocks(osb->sb, new_phys_cpos), + new_len); + } + + ocfs2_commit_trans(osb, handle); + +out_unlock_mutex: + inode_unlock(tl_inode); + + if (context->data_ac) { + ocfs2_free_alloc_context(context->data_ac); + context->data_ac = NULL; + } + + if (context->meta_ac) { + ocfs2_free_alloc_context(context->meta_ac); + context->meta_ac = NULL; + } + +out: + if (ref_tree) + ocfs2_unlock_refcount_tree(osb, ref_tree, 1); + + return ret; +} + +/* + * find the victim alloc group, where #blkno fits. + */ +static int ocfs2_find_victim_alloc_group(struct inode *inode, + u64 vict_blkno, + int type, int slot, + int *vict_bit, + struct buffer_head **ret_bh) +{ + int ret, i, len, bits_per_unit = 0; + u64 blkno; + char namebuf[40]; + + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct buffer_head *ac_bh = NULL, *gd_bh = NULL; + struct ocfs2_chain_list *cl; + struct ocfs2_chain_rec *rec; + struct ocfs2_dinode *ac_dinode; + struct ocfs2_group_desc *bg; + + len = ocfs2_sprintf_system_inode_name(namebuf, sizeof(namebuf), type, slot); + ret = ocfs2_lookup_ino_from_name(osb->sys_root_inode, namebuf, len, &blkno); + + if (ret) { + ret = -ENOENT; + goto out; + } + + ret = ocfs2_read_blocks_sync(osb, blkno, 1, &ac_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + ac_dinode = (struct ocfs2_dinode *)ac_bh->b_data; + cl = &(ac_dinode->id2.i_chain); + rec = &(cl->cl_recs[0]); + + if (type == GLOBAL_BITMAP_SYSTEM_INODE) + bits_per_unit = osb->s_clustersize_bits - + inode->i_sb->s_blocksize_bits; + /* + * 'vict_blkno' was out of the valid range. + */ + if ((vict_blkno < le64_to_cpu(rec->c_blkno)) || + (vict_blkno >= ((u64)le32_to_cpu(ac_dinode->id1.bitmap1.i_total) << + bits_per_unit))) { + ret = -EINVAL; + goto out; + } + + for (i = 0; i < le16_to_cpu(cl->cl_next_free_rec); i++) { + + rec = &(cl->cl_recs[i]); + if (!rec) + continue; + + bg = NULL; + + do { + if (!bg) + blkno = le64_to_cpu(rec->c_blkno); + else + blkno = le64_to_cpu(bg->bg_next_group); + + if (gd_bh) { + brelse(gd_bh); + gd_bh = NULL; + } + + ret = ocfs2_read_blocks_sync(osb, blkno, 1, &gd_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + bg = (struct ocfs2_group_desc *)gd_bh->b_data; + + if (vict_blkno < (le64_to_cpu(bg->bg_blkno) + + (le16_to_cpu(bg->bg_bits) << bits_per_unit))) { + + *ret_bh = gd_bh; + *vict_bit = (vict_blkno - blkno) >> + bits_per_unit; + mlog(0, "find the victim group: #%llu, " + "total_bits: %u, vict_bit: %u\n", + blkno, le16_to_cpu(bg->bg_bits), + *vict_bit); + goto out; + } + + } while (le64_to_cpu(bg->bg_next_group)); + } + + ret = -EINVAL; +out: + brelse(ac_bh); + + /* + * caller has to release the gd_bh properly. + */ + return ret; +} + +/* + * XXX: helper to validate and adjust moving goal. + */ +static int ocfs2_validate_and_adjust_move_goal(struct inode *inode, + struct ocfs2_move_extents *range) +{ + int ret, goal_bit = 0; + + struct buffer_head *gd_bh = NULL; + struct ocfs2_group_desc *bg; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + int c_to_b = 1 << (osb->s_clustersize_bits - + inode->i_sb->s_blocksize_bits); + + /* + * make goal become cluster aligned. + */ + range->me_goal = ocfs2_block_to_cluster_start(inode->i_sb, + range->me_goal); + /* + * validate goal sits within global_bitmap, and return the victim + * group desc + */ + ret = ocfs2_find_victim_alloc_group(inode, range->me_goal, + GLOBAL_BITMAP_SYSTEM_INODE, + OCFS2_INVALID_SLOT, + &goal_bit, &gd_bh); + if (ret) + goto out; + + bg = (struct ocfs2_group_desc *)gd_bh->b_data; + + /* + * moving goal is not allowed to start with a group desc blok(#0 blk) + * let's compromise to the latter cluster. + */ + if (range->me_goal == le64_to_cpu(bg->bg_blkno)) + range->me_goal += c_to_b; + + /* + * movement is not gonna cross two groups. + */ + if ((le16_to_cpu(bg->bg_bits) - goal_bit) * osb->s_clustersize < + range->me_len) { + ret = -EINVAL; + goto out; + } + /* + * more exact validations/adjustments will be performed later during + * moving operation for each extent range. + */ + mlog(0, "extents get ready to be moved to #%llu block\n", + range->me_goal); + +out: + brelse(gd_bh); + + return ret; +} + +static void ocfs2_probe_alloc_group(struct inode *inode, struct buffer_head *bh, + int *goal_bit, u32 move_len, u32 max_hop, + u32 *phys_cpos) +{ + int i, used, last_free_bits = 0, base_bit = *goal_bit; + struct ocfs2_group_desc *gd = (struct ocfs2_group_desc *)bh->b_data; + u32 base_cpos = ocfs2_blocks_to_clusters(inode->i_sb, + le64_to_cpu(gd->bg_blkno)); + + *phys_cpos = 0; + + for (i = base_bit; i < le16_to_cpu(gd->bg_bits); i++) { + + used = ocfs2_test_bit(i, (unsigned long *)gd->bg_bitmap); + if (used) { + /* + * we even tried searching the free chunk by jumping + * a 'max_hop' distance, but still failed. + */ + if ((i - base_bit) > max_hop) { + *phys_cpos = 0; + break; + } + + if (last_free_bits) + last_free_bits = 0; + + continue; + } else + last_free_bits++; + + if (last_free_bits == move_len) { + i = i - move_len + 1; + *goal_bit = i; + *phys_cpos = base_cpos + i; + break; + } + } + + mlog(0, "found phys_cpos: %u to fit the wanted moving.\n", *phys_cpos); +} + +static int ocfs2_move_extent(struct ocfs2_move_extents_context *context, + u32 cpos, u32 phys_cpos, u32 *new_phys_cpos, + u32 len, int ext_flags) +{ + int ret, credits = 0, extra_blocks = 0, goal_bit = 0; + handle_t *handle; + struct inode *inode = context->inode; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct inode *tl_inode = osb->osb_tl_inode; + struct inode *gb_inode = NULL; + struct buffer_head *gb_bh = NULL; + struct buffer_head *gd_bh = NULL; + struct ocfs2_group_desc *gd; + struct ocfs2_refcount_tree *ref_tree = NULL; + u32 move_max_hop = ocfs2_blocks_to_clusters(inode->i_sb, + context->range->me_threshold); + u64 phys_blkno, new_phys_blkno; + + phys_blkno = ocfs2_clusters_to_blocks(inode->i_sb, phys_cpos); + + if ((ext_flags & OCFS2_EXT_REFCOUNTED) && len) { + BUG_ON(!ocfs2_is_refcount_inode(inode)); + BUG_ON(!context->refcount_loc); + + ret = ocfs2_lock_refcount_tree(osb, context->refcount_loc, 1, + &ref_tree, NULL); + if (ret) { + mlog_errno(ret); + return ret; + } + + ret = ocfs2_prepare_refcount_change_for_del(inode, + context->refcount_loc, + phys_blkno, + len, + &credits, + &extra_blocks); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + ret = ocfs2_lock_meta_allocator_move_extents(inode, &context->et, + len, 1, + &context->meta_ac, + extra_blocks, &credits); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * need to count 2 extra credits for global_bitmap inode and + * group descriptor. + */ + credits += OCFS2_INODE_UPDATE_CREDITS + 1; + + inode_lock(tl_inode); + + /* + * ocfs2_move_extent() didn't reserve any clusters in lock_allocators() + * logic, while we still need to lock the global_bitmap. + */ + gb_inode = ocfs2_get_system_file_inode(osb, GLOBAL_BITMAP_SYSTEM_INODE, + OCFS2_INVALID_SLOT); + if (!gb_inode) { + mlog(ML_ERROR, "unable to get global_bitmap inode\n"); + ret = -EIO; + goto out_unlock_tl_inode; + } + + inode_lock(gb_inode); + + ret = ocfs2_inode_lock(gb_inode, &gb_bh, 1); + if (ret) { + mlog_errno(ret); + goto out_unlock_gb_inode; + } + + handle = ocfs2_start_trans(osb, credits); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out_unlock; + } + + new_phys_blkno = ocfs2_clusters_to_blocks(inode->i_sb, *new_phys_cpos); + ret = ocfs2_find_victim_alloc_group(inode, new_phys_blkno, + GLOBAL_BITMAP_SYSTEM_INODE, + OCFS2_INVALID_SLOT, + &goal_bit, &gd_bh); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + gd = (struct ocfs2_group_desc *)gd_bh->b_data; + if (le16_to_cpu(gd->bg_free_bits_count) < len) { + ret = -ENOSPC; + goto out_commit; + } + + /* + * probe the victim cluster group to find a proper + * region to fit wanted movement, it even will perform + * a best-effort attempt by compromising to a threshold + * around the goal. + */ + ocfs2_probe_alloc_group(inode, gd_bh, &goal_bit, len, move_max_hop, + new_phys_cpos); + if (!*new_phys_cpos) { + ret = -ENOSPC; + goto out_commit; + } + + ret = __ocfs2_move_extent(handle, context, cpos, len, phys_cpos, + *new_phys_cpos, ext_flags); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + ret = ocfs2_alloc_dinode_update_counts(gb_inode, handle, gb_bh, len, + le16_to_cpu(gd->bg_chain)); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + ret = ocfs2_block_group_set_bits(handle, gb_inode, gd, gd_bh, + goal_bit, len, 0, 0); + if (ret) { + ocfs2_rollback_alloc_dinode_counts(gb_inode, gb_bh, len, + le16_to_cpu(gd->bg_chain)); + mlog_errno(ret); + } + + /* + * Here we should write the new page out first if we are + * in write-back mode. + */ + ret = ocfs2_cow_sync_writeback(inode->i_sb, context->inode, cpos, len); + if (ret) + mlog_errno(ret); + +out_commit: + ocfs2_commit_trans(osb, handle); + brelse(gd_bh); +out_unlock: + ocfs2_inode_unlock(gb_inode, 1); +out_unlock_gb_inode: + inode_unlock(gb_inode); + brelse(gb_bh); + iput(gb_inode); +out_unlock_tl_inode: + inode_unlock(tl_inode); + +out: + if (context->meta_ac) { + ocfs2_free_alloc_context(context->meta_ac); + context->meta_ac = NULL; + } + + if (ref_tree) + ocfs2_unlock_refcount_tree(osb, ref_tree, 1); + + return ret; +} + +/* + * Helper to calculate the defraging length in one run according to threshold. + */ +static void ocfs2_calc_extent_defrag_len(u32 *alloc_size, u32 *len_defraged, + u32 threshold, int *skip) +{ + if ((*alloc_size + *len_defraged) < threshold) { + /* + * proceed defragmentation until we meet the thresh + */ + *len_defraged += *alloc_size; + } else if (*len_defraged == 0) { + /* + * XXX: skip a large extent. + */ + *skip = 1; + } else { + /* + * split this extent to coalesce with former pieces as + * to reach the threshold. + * + * we're done here with one cycle of defragmentation + * in a size of 'thresh', resetting 'len_defraged' + * forces a new defragmentation. + */ + *alloc_size = threshold - *len_defraged; + *len_defraged = 0; + } +} + +static int __ocfs2_move_extents_range(struct buffer_head *di_bh, + struct ocfs2_move_extents_context *context) +{ + int ret = 0, flags, do_defrag, skip = 0; + u32 cpos, phys_cpos, move_start, len_to_move, alloc_size; + u32 len_defraged = 0, defrag_thresh = 0, new_phys_cpos = 0; + + struct inode *inode = context->inode; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data; + struct ocfs2_move_extents *range = context->range; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + if ((i_size_read(inode) == 0) || (range->me_len == 0)) + return 0; + + if (OCFS2_I(inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL) + return 0; + + context->refcount_loc = le64_to_cpu(di->i_refcount_loc); + + ocfs2_init_dinode_extent_tree(&context->et, INODE_CACHE(inode), di_bh); + ocfs2_init_dealloc_ctxt(&context->dealloc); + + /* + * TO-DO XXX: + * + * - xattr extents. + */ + + do_defrag = context->auto_defrag; + + /* + * extents moving happens in unit of clusters, for the sake + * of simplicity, we may ignore two clusters where 'byte_start' + * and 'byte_start + len' were within. + */ + move_start = ocfs2_clusters_for_bytes(osb->sb, range->me_start); + len_to_move = (range->me_start + range->me_len) >> + osb->s_clustersize_bits; + if (len_to_move >= move_start) + len_to_move -= move_start; + else + len_to_move = 0; + + if (do_defrag) { + defrag_thresh = range->me_threshold >> osb->s_clustersize_bits; + if (defrag_thresh <= 1) + goto done; + } else + new_phys_cpos = ocfs2_blocks_to_clusters(inode->i_sb, + range->me_goal); + + mlog(0, "Inode: %llu, start: %llu, len: %llu, cstart: %u, clen: %u, " + "thresh: %u\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno, + (unsigned long long)range->me_start, + (unsigned long long)range->me_len, + move_start, len_to_move, defrag_thresh); + + cpos = move_start; + while (len_to_move) { + ret = ocfs2_get_clusters(inode, cpos, &phys_cpos, &alloc_size, + &flags); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (alloc_size > len_to_move) + alloc_size = len_to_move; + + /* + * XXX: how to deal with a hole: + * + * - skip the hole of course + * - force a new defragmentation + */ + if (!phys_cpos) { + if (do_defrag) + len_defraged = 0; + + goto next; + } + + if (do_defrag) { + ocfs2_calc_extent_defrag_len(&alloc_size, &len_defraged, + defrag_thresh, &skip); + /* + * skip large extents + */ + if (skip) { + skip = 0; + goto next; + } + + mlog(0, "#Defrag: cpos: %u, phys_cpos: %u, " + "alloc_size: %u, len_defraged: %u\n", + cpos, phys_cpos, alloc_size, len_defraged); + + ret = ocfs2_defrag_extent(context, cpos, phys_cpos, + &alloc_size, flags); + } else { + ret = ocfs2_move_extent(context, cpos, phys_cpos, + &new_phys_cpos, alloc_size, + flags); + + new_phys_cpos += alloc_size; + } + + if (ret < 0) { + mlog_errno(ret); + goto out; + } + /* + * Invalidate extent cache after moving/defragging to prevent + * stale cached data with outdated extent flags. + */ + ocfs2_extent_map_trunc(inode, cpos); + + context->clusters_moved += alloc_size; +next: + cpos += alloc_size; + len_to_move -= alloc_size; + } + +done: + range->me_flags |= OCFS2_MOVE_EXT_FL_COMPLETE; + +out: + range->me_moved_len = ocfs2_clusters_to_bytes(osb->sb, + context->clusters_moved); + range->me_new_offset = ocfs2_clusters_to_bytes(osb->sb, + context->new_phys_cpos); + + ocfs2_schedule_truncate_log_flush(osb, 1); + ocfs2_run_deallocs(osb, &context->dealloc); + + return ret; +} + +static int ocfs2_move_extents(struct ocfs2_move_extents_context *context) +{ + int status; + handle_t *handle; + struct inode *inode = context->inode; + struct ocfs2_dinode *di; + struct buffer_head *di_bh = NULL; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + if (unlikely(ocfs2_emergency_state(osb))) + return -EROFS; + + inode_lock(inode); + + /* + * This prevents concurrent writes from other nodes + */ + status = ocfs2_rw_lock(inode, 1); + if (status) { + mlog_errno(status); + goto out; + } + + status = ocfs2_inode_lock(inode, &di_bh, 1); + if (status) { + mlog_errno(status); + goto out_rw_unlock; + } + + /* + * remember ip_xattr_sem also needs to be held if necessary + */ + down_write(&OCFS2_I(inode)->ip_alloc_sem); + + status = __ocfs2_move_extents_range(di_bh, context); + + up_write(&OCFS2_I(inode)->ip_alloc_sem); + if (status) { + mlog_errno(status); + goto out_inode_unlock; + } + + /* + * We update ctime for these changes + */ + handle = ocfs2_start_trans(osb, OCFS2_INODE_UPDATE_CREDITS); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + mlog_errno(status); + goto out_inode_unlock; + } + + status = ocfs2_journal_access_di(handle, INODE_CACHE(inode), di_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status) { + mlog_errno(status); + goto out_commit; + } + + di = (struct ocfs2_dinode *)di_bh->b_data; + inode_set_ctime_current(inode); + di->i_ctime = cpu_to_le64(inode_get_ctime_sec(inode)); + di->i_ctime_nsec = cpu_to_le32(inode_get_ctime_nsec(inode)); + ocfs2_update_inode_fsync_trans(handle, inode, 0); + + ocfs2_journal_dirty(handle, di_bh); + +out_commit: + ocfs2_commit_trans(osb, handle); + +out_inode_unlock: + brelse(di_bh); + ocfs2_inode_unlock(inode, 1); +out_rw_unlock: + ocfs2_rw_unlock(inode, 1); +out: + inode_unlock(inode); + + return status; +} + +int ocfs2_ioctl_move_extents(struct file *filp, void __user *argp) +{ + int status; + + struct inode *inode = file_inode(filp); + struct ocfs2_move_extents range; + struct ocfs2_move_extents_context *context; + + if (!argp) + return -EINVAL; + + status = mnt_want_write_file(filp); + if (status) + return status; + + if ((!S_ISREG(inode->i_mode)) || !(filp->f_mode & FMODE_WRITE)) { + status = -EPERM; + goto out_drop; + } + + if (inode->i_flags & (S_IMMUTABLE|S_APPEND)) { + status = -EPERM; + goto out_drop; + } + + context = kzalloc_obj(struct ocfs2_move_extents_context, GFP_NOFS); + if (!context) { + status = -ENOMEM; + mlog_errno(status); + goto out_drop; + } + + context->inode = inode; + context->file = filp; + + if (copy_from_user(&range, argp, sizeof(range))) { + status = -EFAULT; + goto out_free; + } + + if (range.me_start > i_size_read(inode)) { + status = -EINVAL; + goto out_free; + } + + if (range.me_start + range.me_len > i_size_read(inode)) + range.me_len = i_size_read(inode) - range.me_start; + + context->range = ⦥ + + /* + * ok, the default threshold for the defragmentation + * is 1M, since our maximum clustersize was 1M also. + * any thought? + */ + if (!range.me_threshold) + range.me_threshold = 1024 * 1024; + + if (range.me_threshold > i_size_read(inode)) + range.me_threshold = i_size_read(inode); + + if (range.me_flags & ~(OCFS2_MOVE_EXT_FL_AUTO_DEFRAG | + OCFS2_MOVE_EXT_FL_PART_DEFRAG)) { + status = -EINVAL; + goto out_free; + } + + if (range.me_flags & OCFS2_MOVE_EXT_FL_AUTO_DEFRAG) { + context->auto_defrag = 1; + + if (range.me_flags & OCFS2_MOVE_EXT_FL_PART_DEFRAG) + context->partial = 1; + } else { + /* + * first best-effort attempt to validate and adjust the goal + * (physical address in block), while it can't guarantee later + * operation can succeed all the time since global_bitmap may + * change a bit over time. + */ + + status = ocfs2_validate_and_adjust_move_goal(inode, &range); + if (status) + goto out_copy; + } + + status = ocfs2_move_extents(context); + if (status) + mlog_errno(status); +out_copy: + /* + * movement/defragmentation may end up being partially completed, + * that's the reason why we need to return userspace the finished + * length and new_offset even if failure happens somewhere. + */ + if (copy_to_user(argp, &range, sizeof(range))) + status = -EFAULT; + +out_free: + kfree(context); +out_drop: + mnt_drop_write_file(filp); + + return status; +} diff --git a/fs/ocfs2/move_extents.h b/fs/ocfs2/move_extents.h new file mode 100644 index 000000000..987f9e559 --- /dev/null +++ b/fs/ocfs2/move_extents.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * move_extents.h + * + * Copyright (C) 2011 Oracle. All rights reserved. + */ +#ifndef OCFS2_MOVE_EXTENTS_H +#define OCFS2_MOVE_EXTENTS_H + +int ocfs2_ioctl_move_extents(struct file *filp, void __user *argp); + +#endif /* OCFS2_MOVE_EXTENTS_H */ diff --git a/fs/ocfs2/namei.c b/fs/ocfs2/namei.c new file mode 100644 index 000000000..e9c7774cc --- /dev/null +++ b/fs/ocfs2/namei.c @@ -0,0 +1,2956 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * namei.c + * + * Create and rename file, directory, symlinks + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + * + * Portions of this code from linux/fs/ext3/dir.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * from + * + * linux/fs/minix/dir.c + * + * Copyright (C) 1991, 1992 Linux Torvalds + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ocfs2.h" + +#include "alloc.h" +#include "dcache.h" +#include "dir.h" +#include "dlmglue.h" +#include "extent_map.h" +#include "file.h" +#include "inode.h" +#include "journal.h" +#include "namei.h" +#include "suballoc.h" +#include "super.h" +#include "symlink.h" +#include "sysfile.h" +#include "uptodate.h" +#include "xattr.h" +#include "acl.h" +#include "ocfs2_trace.h" +#include "ioctl.h" + +#include "buffer_head_io.h" + +static int ocfs2_mknod_locked(struct ocfs2_super *osb, + struct inode *dir, + struct inode *inode, + dev_t dev, + struct buffer_head **new_fe_bh, + struct buffer_head *parent_fe_bh, + handle_t *handle, + struct ocfs2_alloc_context *inode_ac); + +static int ocfs2_prepare_orphan_dir(struct ocfs2_super *osb, + struct inode **ret_orphan_dir, + u64 blkno, + char *name, + struct ocfs2_dir_lookup_result *lookup, + bool dio); + +static int ocfs2_orphan_add(struct ocfs2_super *osb, + handle_t *handle, + struct inode *inode, + struct buffer_head *fe_bh, + char *name, + struct ocfs2_dir_lookup_result *lookup, + struct inode *orphan_dir_inode, + bool dio); + +static int ocfs2_create_symlink_data(struct ocfs2_super *osb, + handle_t *handle, + struct inode *inode, + const char *symname); + +static int ocfs2_double_lock(struct ocfs2_super *osb, + struct buffer_head **bh1, + struct inode *inode1, + struct buffer_head **bh2, + struct inode *inode2, + int rename); + +static void ocfs2_double_unlock(struct inode *inode1, struct inode *inode2); +/* An orphan dir name is an 8 byte value, printed as a hex string */ +#define OCFS2_ORPHAN_NAMELEN ((int)(2 * sizeof(u64))) + +static struct dentry *ocfs2_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + int status; + u64 blkno; + struct inode *inode = NULL; + struct dentry *ret; + struct ocfs2_inode_info *oi; + + trace_ocfs2_lookup(dir, dentry, dentry->d_name.len, + dentry->d_name.name, + (unsigned long long)OCFS2_I(dir)->ip_blkno, 0); + + if (dentry->d_name.len > OCFS2_MAX_FILENAME_LEN) { + ret = ERR_PTR(-ENAMETOOLONG); + goto bail; + } + + status = ocfs2_inode_lock_nested(dir, NULL, 0, OI_LS_PARENT); + if (status < 0) { + if (status != -ENOENT) + mlog_errno(status); + ret = ERR_PTR(status); + goto bail; + } + + status = ocfs2_lookup_ino_from_name(dir, dentry->d_name.name, + dentry->d_name.len, &blkno); + if (status < 0) + goto bail_add; + + inode = ocfs2_iget(OCFS2_SB(dir->i_sb), blkno, 0, 0); + if (IS_ERR(inode)) { + ret = ERR_PTR(-EACCES); + goto bail_unlock; + } + + oi = OCFS2_I(inode); + /* Clear any orphaned state... If we were able to look up the + * inode from a directory, it certainly can't be orphaned. We + * might have the bad state from a node which intended to + * orphan this inode but crashed before it could commit the + * unlink. */ + spin_lock(&oi->ip_lock); + oi->ip_flags &= ~OCFS2_INODE_MAYBE_ORPHANED; + spin_unlock(&oi->ip_lock); + +bail_add: + ret = d_splice_alias(inode, dentry); + if (IS_ERR(ret)) + goto bail_unlock; + + if (inode) { + /* + * If d_splice_alias() finds a DCACHE_DISCONNECTED + * dentry, it will d_move() it on top of ourse. The + * return value will indicate this however, so in + * those cases, we switch them around for the locking + * code. + * + * NOTE: This dentry already has ->d_op set from + * ocfs2_get_parent() and ocfs2_get_dentry() + */ + if (ret) + dentry = ret; + + status = ocfs2_dentry_attach_lock(dentry, inode, + OCFS2_I(dir)->ip_blkno); + if (status) { + mlog_errno(status); + if (ret) + dput(ret); + ret = ERR_PTR(status); + } + } else + ocfs2_dentry_attach_gen(dentry); + +bail_unlock: + /* Don't drop the cluster lock until *after* the d_add -- + * unlink on another node will message us to remove that + * dentry under this lock so otherwise we can race this with + * the downconvert thread and have a stale dentry. */ + ocfs2_inode_unlock(dir, 0); + +bail: + + trace_ocfs2_lookup_ret(ret); + + return ret; +} + +static struct inode *ocfs2_get_init_inode(struct inode *dir, umode_t mode) +{ + struct inode *inode; + int status; + + inode = new_inode(dir->i_sb); + if (!inode) { + mlog(ML_ERROR, "new_inode failed!\n"); + return ERR_PTR(-ENOMEM); + } + + /* populate as many fields early on as possible - many of + * these are used by the support functions here and in + * callers. */ + if (S_ISDIR(mode)) + set_nlink(inode, 2); + mode = mode_strip_sgid(&nop_mnt_idmap, dir, mode); + inode_init_owner(&nop_mnt_idmap, inode, dir, mode); + status = dquot_initialize(inode); + if (status) { + iput(inode); + return ERR_PTR(status); + } + + return inode; +} + +static void ocfs2_cleanup_add_entry_failure(struct ocfs2_super *osb, + struct dentry *dentry, struct inode *inode) +{ + struct ocfs2_dentry_lock *dl = dentry->d_fsdata; + + ocfs2_simple_drop_lockres(osb, &dl->dl_lockres); + ocfs2_lock_res_free(&dl->dl_lockres); + BUG_ON(dl->dl_count != 1); + spin_lock(&dentry_attach_lock); + dentry->d_fsdata = NULL; + spin_unlock(&dentry_attach_lock); + kfree(dl); + iput(inode); +} + +static int ocfs2_mknod(struct mnt_idmap *idmap, + struct inode *dir, + struct dentry *dentry, + umode_t mode, + dev_t dev) +{ + int status = 0; + struct buffer_head *parent_fe_bh = NULL; + handle_t *handle = NULL; + struct ocfs2_super *osb; + struct ocfs2_dinode *dirfe; + struct ocfs2_dinode *fe = NULL; + struct buffer_head *new_fe_bh = NULL; + struct inode *inode = NULL; + struct ocfs2_alloc_context *inode_ac = NULL; + struct ocfs2_alloc_context *data_ac = NULL; + struct ocfs2_alloc_context *meta_ac = NULL; + int want_clusters = 0; + int want_meta = 0; + int xattr_credits = 0; + struct ocfs2_security_xattr_info si = { + .name = NULL, + .enable = 1, + }; + int did_quota_inode = 0; + struct ocfs2_dir_lookup_result lookup = { NULL, }; + sigset_t oldset; + int did_block_signals = 0; + struct ocfs2_dentry_lock *dl = NULL; + struct ocfs2_acl_state acl_state = { 0 }; + + trace_ocfs2_mknod(dir, dentry, dentry->d_name.len, dentry->d_name.name, + (unsigned long long)OCFS2_I(dir)->ip_blkno, + (unsigned long)dev, mode); + + status = dquot_initialize(dir); + if (status) { + mlog_errno(status); + return status; + } + + /* get our super block */ + osb = OCFS2_SB(dir->i_sb); + + status = ocfs2_inode_lock(dir, &parent_fe_bh, 1); + if (status < 0) { + if (status != -ENOENT) + mlog_errno(status); + return status; + } + + if (S_ISDIR(mode) && (dir->i_nlink >= ocfs2_link_max(osb))) { + status = -EMLINK; + goto leave; + } + + dirfe = (struct ocfs2_dinode *) parent_fe_bh->b_data; + if (!ocfs2_read_links_count(dirfe)) { + /* can't make a file in a deleted directory. */ + status = -ENOENT; + goto leave; + } + + status = ocfs2_check_dir_for_entry(dir, dentry->d_name.name, + dentry->d_name.len); + if (status) + goto leave; + + /* get a spot inside the dir. */ + status = ocfs2_prepare_dir_for_insert(osb, dir, parent_fe_bh, + dentry->d_name.name, + dentry->d_name.len, &lookup); + if (status < 0) { + mlog_errno(status); + goto leave; + } + + /* reserve an inode spot */ + status = ocfs2_reserve_new_inode(osb, &inode_ac); + if (status < 0) { + if (status != -ENOSPC) + mlog_errno(status); + goto leave; + } + + inode = ocfs2_get_init_inode(dir, mode); + if (IS_ERR(inode)) { + status = PTR_ERR(inode); + inode = NULL; + mlog_errno(status); + goto leave; + } + + /* get security xattr */ + status = ocfs2_init_security_get(inode, dir, &dentry->d_name, &si); + if (status) { + if (status == -EOPNOTSUPP) + si.enable = 0; + else { + mlog_errno(status); + goto leave; + } + } + + status = ocfs2_acl_init_prepare(inode, dir, parent_fe_bh, &acl_state); + if (status < 0) + goto leave; + + /* calculate meta data/clusters for setting security and acl xattr */ + status = ocfs2_calc_xattr_init(dir, mode, &si, &want_clusters, + &xattr_credits, &want_meta, + &acl_state); + if (status < 0) { + mlog_errno(status); + goto leave; + } + + /* Reserve a cluster if creating an extent based directory. */ + if (S_ISDIR(mode) && !ocfs2_supports_inline_data(osb)) { + want_clusters += 1; + + /* Dir indexing requires extra space as well */ + if (ocfs2_supports_indexed_dirs(osb)) + want_meta++; + } + + status = ocfs2_reserve_new_metadata_blocks(osb, want_meta, &meta_ac); + if (status < 0) { + if (status != -ENOSPC) + mlog_errno(status); + goto leave; + } + + status = ocfs2_reserve_clusters(osb, want_clusters, &data_ac); + if (status < 0) { + if (status != -ENOSPC) + mlog_errno(status); + goto leave; + } + + handle = ocfs2_start_trans(osb, ocfs2_mknod_credits(osb->sb, + S_ISDIR(mode), + xattr_credits)); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + handle = NULL; + mlog_errno(status); + goto leave; + } + + /* Starting to change things, restart is no longer possible. */ + ocfs2_block_signals(&oldset); + did_block_signals = 1; + + status = dquot_alloc_inode(inode); + if (status) + goto leave; + did_quota_inode = 1; + + /* do the real work now. */ + status = ocfs2_mknod_locked(osb, dir, inode, dev, + &new_fe_bh, parent_fe_bh, handle, + inode_ac); + if (status < 0) { + mlog_errno(status); + goto leave; + } + + fe = (struct ocfs2_dinode *) new_fe_bh->b_data; + if (S_ISDIR(mode)) { + status = ocfs2_fill_new_dir(osb, handle, dir, inode, + new_fe_bh, data_ac, meta_ac); + if (status < 0) { + mlog_errno(status); + goto leave; + } + + status = ocfs2_journal_access_di(handle, INODE_CACHE(dir), + parent_fe_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto leave; + } + ocfs2_add_links_count(dirfe, 1); + ocfs2_journal_dirty(handle, parent_fe_bh); + inc_nlink(dir); + } + + status = ocfs2_init_acl(handle, inode, new_fe_bh, meta_ac, data_ac, + &acl_state); + + if (status < 0) { + mlog_errno(status); + goto roll_back; + } + + if (si.enable) { + status = ocfs2_init_security_set(handle, inode, new_fe_bh, &si, + meta_ac, data_ac); + if (status < 0) { + mlog_errno(status); + goto roll_back; + } + } + + /* + * Do this before adding the entry to the directory. We add + * also set d_op after success so that ->d_iput() will cleanup + * the dentry lock even if ocfs2_add_entry() fails below. + */ + status = ocfs2_dentry_attach_lock(dentry, inode, + OCFS2_I(dir)->ip_blkno); + if (status) { + mlog_errno(status); + goto roll_back; + } + + dl = dentry->d_fsdata; + + status = ocfs2_add_entry(handle, dentry, inode, + OCFS2_I(inode)->ip_blkno, parent_fe_bh, + &lookup); + if (status < 0) { + mlog_errno(status); + goto roll_back; + } + + insert_inode_hash(inode); + d_instantiate(dentry, inode); + status = 0; + +roll_back: + if (status < 0 && S_ISDIR(mode)) { + ocfs2_add_links_count(dirfe, -1); + drop_nlink(dir); + } + +leave: + if (status < 0 && did_quota_inode) + dquot_free_inode(inode); + if (handle) { + if (status < 0 && fe) + ocfs2_set_links_count(fe, 0); + ocfs2_commit_trans(osb, handle); + } + + ocfs2_inode_unlock(dir, 1); + if (did_block_signals) + ocfs2_unblock_signals(&oldset); + + brelse(new_fe_bh); + brelse(parent_fe_bh); + kfree(si.value); + + ocfs2_acl_init_release(&acl_state); + + ocfs2_free_dir_lookup_result(&lookup); + + if (inode_ac) + ocfs2_free_alloc_context(inode_ac); + + if (data_ac) + ocfs2_free_alloc_context(data_ac); + + if (meta_ac) + ocfs2_free_alloc_context(meta_ac); + + /* + * We should call iput after the i_rwsem of the bitmap been + * unlocked in ocfs2_free_alloc_context, or the + * ocfs2_delete_inode will mutex_lock again. + */ + if ((status < 0) && inode) { + if (dl) + ocfs2_cleanup_add_entry_failure(osb, dentry, inode); + + OCFS2_I(inode)->ip_flags |= OCFS2_INODE_SKIP_ORPHAN_DIR; + clear_nlink(inode); + iput(inode); + } + + if (status) + mlog_errno(status); + + return status; +} + +static int __ocfs2_mknod_locked(struct inode *dir, + struct inode *inode, + dev_t dev, + struct buffer_head **new_fe_bh, + handle_t *handle, + struct ocfs2_alloc_context *inode_ac, + u64 fe_blkno, u64 suballoc_loc, u16 suballoc_bit) +{ + int status = 0; + struct ocfs2_super *osb = OCFS2_SB(dir->i_sb); + struct ocfs2_dinode *fe = NULL; + struct ocfs2_extent_list *fel; + u16 feat; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct timespec64 ts; + + *new_fe_bh = NULL; + + /* populate as many fields early on as possible - many of + * these are used by the support functions here and in + * callers. */ + inode->i_ino = ino_from_blkno(osb->sb, fe_blkno); + oi->ip_blkno = fe_blkno; + spin_lock(&osb->osb_lock); + inode->i_generation = osb->s_next_generation++; + spin_unlock(&osb->osb_lock); + + *new_fe_bh = sb_getblk(osb->sb, fe_blkno); + if (!*new_fe_bh) { + status = -ENOMEM; + mlog_errno(status); + goto leave; + } + ocfs2_set_new_buffer_uptodate(INODE_CACHE(inode), *new_fe_bh); + + status = ocfs2_journal_access_di(handle, INODE_CACHE(inode), + *new_fe_bh, + OCFS2_JOURNAL_ACCESS_CREATE); + if (status < 0) { + mlog_errno(status); + goto leave; + } + + fe = (struct ocfs2_dinode *) (*new_fe_bh)->b_data; + memset(fe, 0, osb->sb->s_blocksize); + + fe->i_generation = cpu_to_le32(inode->i_generation); + fe->i_fs_generation = cpu_to_le32(osb->fs_generation); + fe->i_blkno = cpu_to_le64(fe_blkno); + fe->i_suballoc_loc = cpu_to_le64(suballoc_loc); + fe->i_suballoc_bit = cpu_to_le16(suballoc_bit); + fe->i_suballoc_slot = cpu_to_le16(inode_ac->ac_alloc_slot); + fe->i_uid = cpu_to_le32(i_uid_read(inode)); + fe->i_gid = cpu_to_le32(i_gid_read(inode)); + fe->i_mode = cpu_to_le16(inode->i_mode); + if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) + fe->id1.dev1.i_rdev = cpu_to_le64(huge_encode_dev(dev)); + + ocfs2_set_links_count(fe, inode->i_nlink); + + fe->i_last_eb_blk = 0; + strscpy(fe->i_signature, OCFS2_INODE_SIGNATURE); + fe->i_flags |= cpu_to_le32(OCFS2_VALID_FL); + ktime_get_coarse_real_ts64(&ts); + fe->i_atime = fe->i_ctime = fe->i_mtime = + cpu_to_le64(ts.tv_sec); + fe->i_mtime_nsec = fe->i_ctime_nsec = fe->i_atime_nsec = + cpu_to_le32(ts.tv_nsec); + fe->i_dtime = 0; + + /* + * If supported, directories start with inline data. If inline + * isn't supported, but indexing is, we start them as indexed. + */ + feat = le16_to_cpu(fe->i_dyn_features); + if (S_ISDIR(inode->i_mode) && ocfs2_supports_inline_data(osb)) { + fe->i_dyn_features = cpu_to_le16(feat | OCFS2_INLINE_DATA_FL); + + fe->id2.i_data.id_count = cpu_to_le16( + ocfs2_max_inline_data_with_xattr(osb->sb, fe)); + } else { + fel = &fe->id2.i_list; + fel->l_tree_depth = 0; + fel->l_next_free_rec = 0; + fel->l_count = cpu_to_le16(ocfs2_extent_recs_per_inode(osb->sb)); + } + + ocfs2_journal_dirty(handle, *new_fe_bh); + + ocfs2_populate_inode(inode, fe, 1); + ocfs2_ci_set_new(osb, INODE_CACHE(inode)); + if (!ocfs2_mount_local(osb)) { + status = ocfs2_create_new_inode_locks(inode); + if (status < 0) + mlog_errno(status); + } + + ocfs2_update_inode_fsync_trans(handle, inode, 1); + +leave: + if (status < 0) { + if (*new_fe_bh) { + brelse(*new_fe_bh); + *new_fe_bh = NULL; + } + } + + if (status) + mlog_errno(status); + return status; +} + +static int ocfs2_mknod_locked(struct ocfs2_super *osb, + struct inode *dir, + struct inode *inode, + dev_t dev, + struct buffer_head **new_fe_bh, + struct buffer_head *parent_fe_bh, + handle_t *handle, + struct ocfs2_alloc_context *inode_ac) +{ + int status = 0; + u64 suballoc_loc, fe_blkno = 0; + u16 suballoc_bit; + + *new_fe_bh = NULL; + + status = ocfs2_claim_new_inode(handle, dir, parent_fe_bh, + inode_ac, &suballoc_loc, + &suballoc_bit, &fe_blkno); + if (status < 0) { + mlog_errno(status); + return status; + } + + return __ocfs2_mknod_locked(dir, inode, dev, new_fe_bh, + handle, inode_ac, fe_blkno, + suballoc_loc, suballoc_bit); +} + +static struct dentry *ocfs2_mkdir(struct mnt_idmap *idmap, + struct inode *dir, + struct dentry *dentry, + umode_t mode) +{ + int ret; + + trace_ocfs2_mkdir(dir, dentry, dentry->d_name.len, dentry->d_name.name, + OCFS2_I(dir)->ip_blkno, mode); + ret = ocfs2_mknod(&nop_mnt_idmap, dir, dentry, mode, 0); + if (ret) + mlog_errno(ret); + + return ERR_PTR(ret); +} + +static int ocfs2_create(struct mnt_idmap *idmap, + struct inode *dir, + struct dentry *dentry, + umode_t mode) +{ + int ret; + + trace_ocfs2_create(dir, dentry, dentry->d_name.len, dentry->d_name.name, + (unsigned long long)OCFS2_I(dir)->ip_blkno, mode); + ret = ocfs2_mknod(&nop_mnt_idmap, dir, dentry, mode | S_IFREG, 0); + if (ret) + mlog_errno(ret); + + return ret; +} + +static int ocfs2_link(struct dentry *old_dentry, + struct inode *dir, + struct dentry *dentry) +{ + handle_t *handle; + struct inode *inode = d_inode(old_dentry); + struct inode *old_dir = d_inode(old_dentry->d_parent); + int err; + struct buffer_head *fe_bh = NULL; + struct buffer_head *old_dir_bh = NULL; + struct buffer_head *parent_fe_bh = NULL; + struct ocfs2_dinode *fe = NULL; + struct ocfs2_super *osb = OCFS2_SB(dir->i_sb); + struct ocfs2_dir_lookup_result lookup = { NULL, }; + sigset_t oldset; + u64 old_de_ino; + + trace_ocfs2_link((unsigned long long)OCFS2_I(inode)->ip_blkno, + old_dentry->d_name.len, old_dentry->d_name.name, + dentry->d_name.len, dentry->d_name.name); + + if (S_ISDIR(inode->i_mode)) + return -EPERM; + + err = dquot_initialize(dir); + if (err) { + mlog_errno(err); + return err; + } + + err = ocfs2_double_lock(osb, &old_dir_bh, old_dir, + &parent_fe_bh, dir, 0); + if (err < 0) { + if (err != -ENOENT) + mlog_errno(err); + return err; + } + + /* make sure both dirs have bhs + * get an extra ref on old_dir_bh if old==new */ + if (!parent_fe_bh) { + if (old_dir_bh) { + parent_fe_bh = old_dir_bh; + get_bh(parent_fe_bh); + } else { + mlog(ML_ERROR, "%s: no old_dir_bh!\n", osb->uuid_str); + err = -EIO; + goto out; + } + } + + if (!dir->i_nlink) { + err = -ENOENT; + goto out; + } + + err = ocfs2_lookup_ino_from_name(old_dir, old_dentry->d_name.name, + old_dentry->d_name.len, &old_de_ino); + if (err) { + err = -ENOENT; + goto out; + } + + /* + * Check whether another node removed the source inode while we + * were in the vfs. + */ + if (old_de_ino != OCFS2_I(inode)->ip_blkno) { + err = -ENOENT; + goto out; + } + + err = ocfs2_check_dir_for_entry(dir, dentry->d_name.name, + dentry->d_name.len); + if (err) + goto out; + + err = ocfs2_prepare_dir_for_insert(osb, dir, parent_fe_bh, + dentry->d_name.name, + dentry->d_name.len, &lookup); + if (err < 0) { + mlog_errno(err); + goto out; + } + + err = ocfs2_inode_lock(inode, &fe_bh, 1); + if (err < 0) { + if (err != -ENOENT) + mlog_errno(err); + goto out; + } + + fe = (struct ocfs2_dinode *) fe_bh->b_data; + if (ocfs2_read_links_count(fe) >= ocfs2_link_max(osb)) { + err = -EMLINK; + goto out_unlock_inode; + } + + handle = ocfs2_start_trans(osb, ocfs2_link_credits(osb->sb)); + if (IS_ERR(handle)) { + err = PTR_ERR(handle); + handle = NULL; + mlog_errno(err); + goto out_unlock_inode; + } + + /* Starting to change things, restart is no longer possible. */ + ocfs2_block_signals(&oldset); + + err = ocfs2_journal_access_di(handle, INODE_CACHE(inode), fe_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (err < 0) { + mlog_errno(err); + goto out_commit; + } + + inc_nlink(inode); + inode_set_ctime_current(inode); + ocfs2_set_links_count(fe, inode->i_nlink); + fe->i_ctime = cpu_to_le64(inode_get_ctime_sec(inode)); + fe->i_ctime_nsec = cpu_to_le32(inode_get_ctime_nsec(inode)); + ocfs2_update_inode_fsync_trans(handle, inode, 0); + ocfs2_journal_dirty(handle, fe_bh); + + err = ocfs2_add_entry(handle, dentry, inode, + OCFS2_I(inode)->ip_blkno, + parent_fe_bh, &lookup); + if (err) { + ocfs2_add_links_count(fe, -1); + drop_nlink(inode); + mlog_errno(err); + goto out_commit; + } + + err = ocfs2_dentry_attach_lock(dentry, inode, OCFS2_I(dir)->ip_blkno); + if (err) { + mlog_errno(err); + goto out_commit; + } + + ihold(inode); + d_instantiate(dentry, inode); + +out_commit: + ocfs2_commit_trans(osb, handle); + ocfs2_unblock_signals(&oldset); +out_unlock_inode: + ocfs2_inode_unlock(inode, 1); + +out: + ocfs2_double_unlock(old_dir, dir); + + brelse(fe_bh); + brelse(parent_fe_bh); + brelse(old_dir_bh); + + ocfs2_free_dir_lookup_result(&lookup); + + if (err) + mlog_errno(err); + + return err; +} + +/* + * Takes and drops an exclusive lock on the given dentry. This will + * force other nodes to drop it. + */ +static int ocfs2_remote_dentry_delete(struct dentry *dentry) +{ + int ret; + + ret = ocfs2_dentry_lock(dentry, 1); + if (ret) + mlog_errno(ret); + else + ocfs2_dentry_unlock(dentry, 1); + + return ret; +} + +static inline int ocfs2_inode_is_unlinkable(struct inode *inode) +{ + if (S_ISDIR(inode->i_mode)) { + if (inode->i_nlink == 2) + return 1; + return 0; + } + + if (inode->i_nlink == 1) + return 1; + return 0; +} + +static int ocfs2_unlink(struct inode *dir, + struct dentry *dentry) +{ + int status; + int child_locked = 0; + bool is_unlinkable = false; + struct inode *inode = d_inode(dentry); + struct inode *orphan_dir = NULL; + struct ocfs2_super *osb = OCFS2_SB(dir->i_sb); + u64 blkno; + struct ocfs2_dinode *fe = NULL; + struct buffer_head *fe_bh = NULL; + struct buffer_head *parent_node_bh = NULL; + handle_t *handle = NULL; + char orphan_name[OCFS2_ORPHAN_NAMELEN + 1]; + struct ocfs2_dir_lookup_result lookup = { NULL, }; + struct ocfs2_dir_lookup_result orphan_insert = { NULL, }; + + trace_ocfs2_unlink(dir, dentry, dentry->d_name.len, + dentry->d_name.name, + (unsigned long long)OCFS2_I(dir)->ip_blkno, + (unsigned long long)OCFS2_I(inode)->ip_blkno); + + status = dquot_initialize(dir); + if (status) { + mlog_errno(status); + return status; + } + + BUG_ON(d_inode(dentry->d_parent) != dir); + + if (inode == osb->root_inode) + return -EPERM; + + status = ocfs2_inode_lock_nested(dir, &parent_node_bh, 1, + OI_LS_PARENT); + if (status < 0) { + if (status != -ENOENT) + mlog_errno(status); + return status; + } + + status = ocfs2_find_files_on_disk(dentry->d_name.name, + dentry->d_name.len, &blkno, dir, + &lookup); + if (status < 0) { + if (status != -ENOENT) + mlog_errno(status); + goto leave; + } + + if (OCFS2_I(inode)->ip_blkno != blkno) { + status = -ENOENT; + + trace_ocfs2_unlink_noent( + (unsigned long long)OCFS2_I(inode)->ip_blkno, + (unsigned long long)blkno, + OCFS2_I(inode)->ip_flags); + goto leave; + } + + status = ocfs2_inode_lock(inode, &fe_bh, 1); + if (status < 0) { + if (status != -ENOENT) + mlog_errno(status); + goto leave; + } + child_locked = 1; + + if (S_ISDIR(inode->i_mode)) { + status = ocfs2_empty_dir(inode); + if (status < 0) + goto leave; + if (inode->i_nlink != 2 || !status) { + status = -ENOTEMPTY; + goto leave; + } + } + + status = ocfs2_remote_dentry_delete(dentry); + if (status < 0) { + /* This remote delete should succeed under all normal + * circumstances. */ + mlog_errno(status); + goto leave; + } + + if (ocfs2_inode_is_unlinkable(inode)) { + status = ocfs2_prepare_orphan_dir(osb, &orphan_dir, + OCFS2_I(inode)->ip_blkno, + orphan_name, &orphan_insert, + false); + if (status < 0) { + mlog_errno(status); + goto leave; + } + is_unlinkable = true; + } + + handle = ocfs2_start_trans(osb, ocfs2_unlink_credits(osb->sb)); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + handle = NULL; + mlog_errno(status); + goto leave; + } + + status = ocfs2_journal_access_di(handle, INODE_CACHE(inode), fe_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto leave; + } + + fe = (struct ocfs2_dinode *) fe_bh->b_data; + + /* delete the name from the parent dir */ + status = ocfs2_delete_entry(handle, dir, &lookup); + if (status < 0) { + mlog_errno(status); + goto leave; + } + + if (S_ISDIR(inode->i_mode)) + drop_nlink(inode); + drop_nlink(inode); + ocfs2_set_links_count(fe, inode->i_nlink); + ocfs2_update_inode_fsync_trans(handle, inode, 0); + ocfs2_journal_dirty(handle, fe_bh); + + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + if (S_ISDIR(inode->i_mode)) + drop_nlink(dir); + + status = ocfs2_mark_inode_dirty(handle, dir, parent_node_bh); + if (status < 0) { + mlog_errno(status); + if (S_ISDIR(inode->i_mode)) + inc_nlink(dir); + goto leave; + } + + if (is_unlinkable) { + status = ocfs2_orphan_add(osb, handle, inode, fe_bh, + orphan_name, &orphan_insert, orphan_dir, false); + if (status < 0) + mlog_errno(status); + } + +leave: + if (handle) + ocfs2_commit_trans(osb, handle); + + if (orphan_dir) { + /* This was locked for us in ocfs2_prepare_orphan_dir() */ + ocfs2_inode_unlock(orphan_dir, 1); + inode_unlock(orphan_dir); + iput(orphan_dir); + } + + if (child_locked) + ocfs2_inode_unlock(inode, 1); + + ocfs2_inode_unlock(dir, 1); + + brelse(fe_bh); + brelse(parent_node_bh); + + ocfs2_free_dir_lookup_result(&orphan_insert); + ocfs2_free_dir_lookup_result(&lookup); + + if (status && (status != -ENOTEMPTY) && (status != -ENOENT)) + mlog_errno(status); + + return status; +} + +static int ocfs2_check_if_ancestor(struct ocfs2_super *osb, + u64 src_inode_no, u64 dest_inode_no) +{ + int ret = 0, i = 0; + u64 parent_inode_no = 0; + u64 child_inode_no = src_inode_no; + struct inode *child_inode; + +#define MAX_LOOKUP_TIMES 32 + while (1) { + child_inode = ocfs2_iget(osb, child_inode_no, 0, 0); + if (IS_ERR(child_inode)) { + ret = PTR_ERR(child_inode); + break; + } + + ret = ocfs2_inode_lock(child_inode, NULL, 0); + if (ret < 0) { + iput(child_inode); + if (ret != -ENOENT) + mlog_errno(ret); + break; + } + + ret = ocfs2_lookup_ino_from_name(child_inode, "..", 2, + &parent_inode_no); + ocfs2_inode_unlock(child_inode, 0); + iput(child_inode); + if (ret < 0) { + ret = -ENOENT; + break; + } + + if (parent_inode_no == dest_inode_no) { + ret = 1; + break; + } + + if (parent_inode_no == osb->root_inode->i_ino) { + ret = 0; + break; + } + + child_inode_no = parent_inode_no; + + if (++i >= MAX_LOOKUP_TIMES) { + mlog_ratelimited(ML_NOTICE, "max lookup times reached, " + "filesystem may have nested directories, " + "src inode: %llu, dest inode: %llu.\n", + (unsigned long long)src_inode_no, + (unsigned long long)dest_inode_no); + ret = 0; + break; + } + } + + return ret; +} + +/* + * The only place this should be used is rename and link! + * if they have the same id, then the 1st one is the only one locked. + */ +static int ocfs2_double_lock(struct ocfs2_super *osb, + struct buffer_head **bh1, + struct inode *inode1, + struct buffer_head **bh2, + struct inode *inode2, + int rename) +{ + int status; + int inode1_is_ancestor, inode2_is_ancestor; + struct ocfs2_inode_info *oi1 = OCFS2_I(inode1); + struct ocfs2_inode_info *oi2 = OCFS2_I(inode2); + + trace_ocfs2_double_lock((unsigned long long)oi1->ip_blkno, + (unsigned long long)oi2->ip_blkno); + + if (*bh1) + *bh1 = NULL; + if (*bh2) + *bh2 = NULL; + + /* we always want to lock the one with the lower lockid first. + * and if they are nested, we lock ancestor first */ + if (oi1->ip_blkno != oi2->ip_blkno) { + inode1_is_ancestor = ocfs2_check_if_ancestor(osb, oi2->ip_blkno, + oi1->ip_blkno); + if (inode1_is_ancestor < 0) { + status = inode1_is_ancestor; + goto bail; + } + + inode2_is_ancestor = ocfs2_check_if_ancestor(osb, oi1->ip_blkno, + oi2->ip_blkno); + if (inode2_is_ancestor < 0) { + status = inode2_is_ancestor; + goto bail; + } + + if ((inode1_is_ancestor == 1) || + (oi1->ip_blkno < oi2->ip_blkno && + inode2_is_ancestor == 0)) { + /* switch id1 and id2 around */ + swap(bh2, bh1); + swap(inode2, inode1); + } + /* lock id2 */ + status = ocfs2_inode_lock_nested(inode2, bh2, 1, + rename == 1 ? OI_LS_RENAME1 : OI_LS_PARENT); + if (status < 0) { + if (status != -ENOENT) + mlog_errno(status); + goto bail; + } + } + + /* lock id1 */ + status = ocfs2_inode_lock_nested(inode1, bh1, 1, + rename == 1 ? OI_LS_RENAME2 : OI_LS_PARENT); + if (status < 0) { + /* + * An error return must mean that no cluster locks + * were held on function exit. + */ + if (oi1->ip_blkno != oi2->ip_blkno) { + ocfs2_inode_unlock(inode2, 1); + brelse(*bh2); + *bh2 = NULL; + } + + if (status != -ENOENT) + mlog_errno(status); + } + + trace_ocfs2_double_lock_end( + (unsigned long long)oi1->ip_blkno, + (unsigned long long)oi2->ip_blkno); + +bail: + if (status) + mlog_errno(status); + return status; +} + +static void ocfs2_double_unlock(struct inode *inode1, struct inode *inode2) +{ + ocfs2_inode_unlock(inode1, 1); + + if (inode1 != inode2) + ocfs2_inode_unlock(inode2, 1); +} + +static int ocfs2_rename(struct mnt_idmap *idmap, + struct inode *old_dir, + struct dentry *old_dentry, + struct inode *new_dir, + struct dentry *new_dentry, + unsigned int flags) +{ + int status = 0, rename_lock = 0, parents_locked = 0, target_exists = 0; + int old_child_locked = 0, new_child_locked = 0, update_dot_dot = 0; + struct inode *old_inode = d_inode(old_dentry); + struct inode *new_inode = d_inode(new_dentry); + struct inode *orphan_dir = NULL; + struct ocfs2_dinode *newfe = NULL; + char orphan_name[OCFS2_ORPHAN_NAMELEN + 1]; + struct buffer_head *newfe_bh = NULL; + struct buffer_head *old_inode_bh = NULL; + struct ocfs2_super *osb = NULL; + u64 newfe_blkno, old_de_ino; + handle_t *handle = NULL; + struct buffer_head *old_dir_bh = NULL; + struct buffer_head *new_dir_bh = NULL; + u32 old_dir_nlink = old_dir->i_nlink; + struct ocfs2_dinode *old_di; + struct ocfs2_dir_lookup_result old_inode_dot_dot_res = { NULL, }; + struct ocfs2_dir_lookup_result target_lookup_res = { NULL, }; + struct ocfs2_dir_lookup_result old_entry_lookup = { NULL, }; + struct ocfs2_dir_lookup_result orphan_insert = { NULL, }; + struct ocfs2_dir_lookup_result target_insert = { NULL, }; + bool should_add_orphan = false; + + if (flags) + return -EINVAL; + + /* At some point it might be nice to break this function up a + * bit. */ + + trace_ocfs2_rename(old_dir, old_dentry, new_dir, new_dentry, + old_dentry->d_name.len, old_dentry->d_name.name, + new_dentry->d_name.len, new_dentry->d_name.name); + + status = dquot_initialize(old_dir); + if (status) { + mlog_errno(status); + goto bail; + } + status = dquot_initialize(new_dir); + if (status) { + mlog_errno(status); + goto bail; + } + + osb = OCFS2_SB(old_dir->i_sb); + + if (new_inode) { + if (!igrab(new_inode)) + BUG(); + } + + /* Assume a directory hierarchy thusly: + * a/b/c + * a/d + * a,b,c, and d are all directories. + * + * from cwd of 'a' on both nodes: + * node1: mv b/c d + * node2: mv d b/c + * + * And that's why, just like the VFS, we need a file system + * rename lock. */ + if (old_dir != new_dir && S_ISDIR(old_inode->i_mode)) { + status = ocfs2_rename_lock(osb); + if (status < 0) { + mlog_errno(status); + goto bail; + } + rename_lock = 1; + + /* here we cannot guarantee the inodes haven't just been + * changed, so check if they are nested again */ + status = ocfs2_check_if_ancestor(osb, new_dir->i_ino, + old_inode->i_ino); + if (status < 0) { + mlog_errno(status); + goto bail; + } else if (status == 1) { + status = -EPERM; + trace_ocfs2_rename_not_permitted( + (unsigned long long)old_inode->i_ino, + (unsigned long long)new_dir->i_ino); + goto bail; + } + } + + /* if old and new are the same, this'll just do one lock. */ + status = ocfs2_double_lock(osb, &old_dir_bh, old_dir, + &new_dir_bh, new_dir, 1); + if (status < 0) { + mlog_errno(status); + goto bail; + } + parents_locked = 1; + + if (!new_dir->i_nlink) { + status = -EACCES; + goto bail; + } + + /* make sure both dirs have bhs + * get an extra ref on old_dir_bh if old==new */ + if (!new_dir_bh) { + if (old_dir_bh) { + new_dir_bh = old_dir_bh; + get_bh(new_dir_bh); + } else { + mlog(ML_ERROR, "no old_dir_bh!\n"); + status = -EIO; + goto bail; + } + } + + /* + * Aside from allowing a meta data update, the locking here + * also ensures that the downconvert thread on other nodes + * won't have to concurrently downconvert the inode and the + * dentry locks. + */ + status = ocfs2_inode_lock_nested(old_inode, &old_inode_bh, 1, + OI_LS_PARENT); + if (status < 0) { + if (status != -ENOENT) + mlog_errno(status); + goto bail; + } + old_child_locked = 1; + + status = ocfs2_remote_dentry_delete(old_dentry); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + if (S_ISDIR(old_inode->i_mode) && new_dir != old_dir) { + u64 old_inode_parent; + + update_dot_dot = 1; + status = ocfs2_find_files_on_disk("..", 2, &old_inode_parent, + old_inode, + &old_inode_dot_dot_res); + if (status) { + status = -EIO; + goto bail; + } + + if (old_inode_parent != OCFS2_I(old_dir)->ip_blkno) { + status = -EIO; + goto bail; + } + + if (!new_inode && new_dir->i_nlink >= ocfs2_link_max(osb)) { + status = -EMLINK; + goto bail; + } + } + + status = ocfs2_lookup_ino_from_name(old_dir, old_dentry->d_name.name, + old_dentry->d_name.len, + &old_de_ino); + if (status) { + status = -ENOENT; + goto bail; + } + + /* + * Check for inode number is _not_ due to possible IO errors. + * We might rmdir the source, keep it as pwd of some process + * and merrily kill the link to whatever was created under the + * same name. Goodbye sticky bit ;-< + */ + if (old_de_ino != OCFS2_I(old_inode)->ip_blkno) { + status = -ENOENT; + goto bail; + } + + /* check if the target already exists (in which case we need + * to delete it */ + status = ocfs2_find_files_on_disk(new_dentry->d_name.name, + new_dentry->d_name.len, + &newfe_blkno, new_dir, + &target_lookup_res); + /* The only error we allow here is -ENOENT because the new + * file not existing is perfectly valid. */ + if ((status < 0) && (status != -ENOENT)) { + /* If we cannot find the file specified we should just */ + /* return the error... */ + mlog_errno(status); + goto bail; + } + if (status == 0) + target_exists = 1; + + if (!target_exists && new_inode) { + /* + * Target was unlinked by another node while we were + * waiting to get to ocfs2_rename(). There isn't + * anything we can do here to help the situation, so + * bubble up the appropriate error. + */ + status = -ENOENT; + goto bail; + } + + /* In case we need to overwrite an existing file, we blow it + * away first */ + if (target_exists) { + /* VFS didn't think there existed an inode here, but + * someone else in the cluster must have raced our + * rename to create one. Today we error cleanly, in + * the future we should consider calling iget to build + * a new struct inode for this entry. */ + if (!new_inode) { + status = -EACCES; + + trace_ocfs2_rename_target_exists(new_dentry->d_name.len, + new_dentry->d_name.name); + goto bail; + } + + if (OCFS2_I(new_inode)->ip_blkno != newfe_blkno) { + status = -EACCES; + + trace_ocfs2_rename_disagree( + (unsigned long long)OCFS2_I(new_inode)->ip_blkno, + (unsigned long long)newfe_blkno, + OCFS2_I(new_inode)->ip_flags); + goto bail; + } + + status = ocfs2_inode_lock(new_inode, &newfe_bh, 1); + if (status < 0) { + if (status != -ENOENT) + mlog_errno(status); + goto bail; + } + new_child_locked = 1; + + status = ocfs2_remote_dentry_delete(new_dentry); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + newfe = (struct ocfs2_dinode *) newfe_bh->b_data; + + trace_ocfs2_rename_over_existing( + (unsigned long long)newfe_blkno, newfe_bh, + (unsigned long long)newfe_bh->b_blocknr); + + if (S_ISDIR(new_inode->i_mode) || (new_inode->i_nlink == 1)) { + status = ocfs2_prepare_orphan_dir(osb, &orphan_dir, + OCFS2_I(new_inode)->ip_blkno, + orphan_name, &orphan_insert, + false); + if (status < 0) { + mlog_errno(status); + goto bail; + } + should_add_orphan = true; + } + } else { + BUG_ON(d_inode(new_dentry->d_parent) != new_dir); + + status = ocfs2_check_dir_for_entry(new_dir, + new_dentry->d_name.name, + new_dentry->d_name.len); + if (status) + goto bail; + + status = ocfs2_prepare_dir_for_insert(osb, new_dir, new_dir_bh, + new_dentry->d_name.name, + new_dentry->d_name.len, + &target_insert); + if (status < 0) { + mlog_errno(status); + goto bail; + } + } + + handle = ocfs2_start_trans(osb, ocfs2_rename_credits(osb->sb)); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + handle = NULL; + mlog_errno(status); + goto bail; + } + + if (target_exists) { + if (S_ISDIR(new_inode->i_mode)) { + status = ocfs2_empty_dir(new_inode); + if (status < 0) + goto bail; + if (new_inode->i_nlink != 2 || !status) { + status = -ENOTEMPTY; + goto bail; + } + } + status = ocfs2_journal_access_di(handle, INODE_CACHE(new_inode), + newfe_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + /* change the dirent to point to the correct inode */ + status = ocfs2_update_entry(new_dir, handle, &target_lookup_res, + old_inode); + if (status < 0) { + mlog_errno(status); + goto bail; + } + inode_inc_iversion(new_dir); + + if (S_ISDIR(new_inode->i_mode)) + ocfs2_set_links_count(newfe, 0); + else + ocfs2_add_links_count(newfe, -1); + ocfs2_journal_dirty(handle, newfe_bh); + if (should_add_orphan) { + status = ocfs2_orphan_add(osb, handle, new_inode, + newfe_bh, orphan_name, + &orphan_insert, orphan_dir, false); + if (status < 0) { + mlog_errno(status); + goto bail; + } + } + } else { + /* if the name was not found in new_dir, add it now */ + status = ocfs2_add_entry(handle, new_dentry, old_inode, + OCFS2_I(old_inode)->ip_blkno, + new_dir_bh, &target_insert); + if (status < 0) { + mlog_errno(status); + goto bail; + } + } + + inode_set_ctime_current(old_inode); + mark_inode_dirty(old_inode); + + status = ocfs2_journal_access_di(handle, INODE_CACHE(old_inode), + old_inode_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status >= 0) { + old_di = (struct ocfs2_dinode *) old_inode_bh->b_data; + + old_di->i_ctime = cpu_to_le64(inode_get_ctime_sec(old_inode)); + old_di->i_ctime_nsec = cpu_to_le32(inode_get_ctime_nsec(old_inode)); + ocfs2_journal_dirty(handle, old_inode_bh); + } else + mlog_errno(status); + + /* + * Now that the name has been added to new_dir, remove the old name. + * + * We don't keep any directory entry context around until now + * because the insert might have changed the type of directory + * we're dealing with. + */ + status = ocfs2_find_entry(old_dentry->d_name.name, + old_dentry->d_name.len, old_dir, + &old_entry_lookup); + if (status) { + if (!is_journal_aborted(osb->journal->j_journal)) { + ocfs2_error(osb->sb, "new entry %.*s is added, but old entry %.*s " + "is not deleted.", + new_dentry->d_name.len, new_dentry->d_name.name, + old_dentry->d_name.len, old_dentry->d_name.name); + } + goto bail; + } + + status = ocfs2_delete_entry(handle, old_dir, &old_entry_lookup); + if (status < 0) { + mlog_errno(status); + if (!is_journal_aborted(osb->journal->j_journal)) { + ocfs2_error(osb->sb, "new entry %.*s is added, but old entry %.*s " + "is not deleted.", + new_dentry->d_name.len, new_dentry->d_name.name, + old_dentry->d_name.len, old_dentry->d_name.name); + } + goto bail; + } + + if (new_inode) { + drop_nlink(new_inode); + inode_set_ctime_current(new_inode); + } + inode_set_mtime_to_ts(old_dir, inode_set_ctime_current(old_dir)); + + if (update_dot_dot) { + status = ocfs2_update_entry(old_inode, handle, + &old_inode_dot_dot_res, new_dir); + if (status < 0) { + mlog_errno(status); + goto bail; + } + } + + if (S_ISDIR(old_inode->i_mode)) { + drop_nlink(old_dir); + if (new_inode) { + drop_nlink(new_inode); + } else { + inc_nlink(new_dir); + mark_inode_dirty(new_dir); + } + } + mark_inode_dirty(old_dir); + ocfs2_mark_inode_dirty(handle, old_dir, old_dir_bh); + if (new_inode) { + mark_inode_dirty(new_inode); + ocfs2_mark_inode_dirty(handle, new_inode, newfe_bh); + } + + if (old_dir != new_dir) { + /* Keep the same times on both directories.*/ + inode_set_mtime_to_ts(new_dir, + inode_set_ctime_to_ts(new_dir, inode_get_ctime(old_dir))); + + /* + * This will also pick up the i_nlink change from the + * block above. + */ + ocfs2_mark_inode_dirty(handle, new_dir, new_dir_bh); + } + + if (old_dir_nlink != old_dir->i_nlink) { + if (!old_dir_bh) { + mlog(ML_ERROR, "need to change nlink for old dir " + "%llu from %d to %d but bh is NULL!\n", + (unsigned long long)OCFS2_I(old_dir)->ip_blkno, + (int)old_dir_nlink, old_dir->i_nlink); + } else { + struct ocfs2_dinode *fe; + status = ocfs2_journal_access_di(handle, + INODE_CACHE(old_dir), + old_dir_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto bail; + } + fe = (struct ocfs2_dinode *) old_dir_bh->b_data; + ocfs2_set_links_count(fe, old_dir->i_nlink); + ocfs2_journal_dirty(handle, old_dir_bh); + } + } + ocfs2_dentry_move(old_dentry, new_dentry, old_dir, new_dir); + status = 0; +bail: + if (handle) + ocfs2_commit_trans(osb, handle); + + if (orphan_dir) { + /* This was locked for us in ocfs2_prepare_orphan_dir() */ + ocfs2_inode_unlock(orphan_dir, 1); + inode_unlock(orphan_dir); + iput(orphan_dir); + } + + if (new_child_locked) + ocfs2_inode_unlock(new_inode, 1); + + if (old_child_locked) + ocfs2_inode_unlock(old_inode, 1); + + if (parents_locked) + ocfs2_double_unlock(old_dir, new_dir); + + if (rename_lock) + ocfs2_rename_unlock(osb); + + iput(new_inode); + + ocfs2_free_dir_lookup_result(&target_lookup_res); + ocfs2_free_dir_lookup_result(&old_entry_lookup); + ocfs2_free_dir_lookup_result(&old_inode_dot_dot_res); + ocfs2_free_dir_lookup_result(&orphan_insert); + ocfs2_free_dir_lookup_result(&target_insert); + + brelse(newfe_bh); + brelse(old_inode_bh); + brelse(old_dir_bh); + brelse(new_dir_bh); + + if (status) + mlog_errno(status); + + return status; +} + +/* + * we expect i_size = strlen(symname). Copy symname into the file + * data, including the null terminator. + */ +static int ocfs2_create_symlink_data(struct ocfs2_super *osb, + handle_t *handle, + struct inode *inode, + const char *symname) +{ + struct buffer_head **bhs = NULL; + const char *c; + struct super_block *sb = osb->sb; + u64 p_blkno, p_blocks; + int virtual, blocks, status, i, bytes_left; + + bytes_left = i_size_read(inode) + 1; + /* we can't trust i_blocks because we're actually going to + * write i_size + 1 bytes. */ + blocks = (bytes_left + sb->s_blocksize - 1) >> sb->s_blocksize_bits; + + trace_ocfs2_create_symlink_data((unsigned long long)inode->i_blocks, + i_size_read(inode), blocks); + + /* Sanity check -- make sure we're going to fit. */ + if (bytes_left > + ocfs2_clusters_to_bytes(sb, OCFS2_I(inode)->ip_clusters)) { + status = -EIO; + mlog_errno(status); + goto bail; + } + + bhs = kzalloc_objs(struct buffer_head *, blocks); + if (!bhs) { + status = -ENOMEM; + mlog_errno(status); + goto bail; + } + + status = ocfs2_extent_map_get_blocks(inode, 0, &p_blkno, &p_blocks, + NULL); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + /* links can never be larger than one cluster so we know this + * is all going to be contiguous, but do a sanity check + * anyway. */ + if ((p_blocks << sb->s_blocksize_bits) < bytes_left) { + status = -EIO; + mlog_errno(status); + goto bail; + } + + virtual = 0; + while(bytes_left > 0) { + c = &symname[virtual * sb->s_blocksize]; + + bhs[virtual] = sb_getblk(sb, p_blkno); + if (!bhs[virtual]) { + status = -ENOMEM; + mlog_errno(status); + goto bail; + } + ocfs2_set_new_buffer_uptodate(INODE_CACHE(inode), + bhs[virtual]); + + status = ocfs2_journal_access(handle, INODE_CACHE(inode), + bhs[virtual], + OCFS2_JOURNAL_ACCESS_CREATE); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + memset(bhs[virtual]->b_data, 0, sb->s_blocksize); + + memcpy(bhs[virtual]->b_data, c, + (bytes_left > sb->s_blocksize) ? sb->s_blocksize : + bytes_left); + + ocfs2_journal_dirty(handle, bhs[virtual]); + + virtual++; + p_blkno++; + bytes_left -= sb->s_blocksize; + } + + status = 0; +bail: + + if (bhs) { + for(i = 0; i < blocks; i++) + brelse(bhs[i]); + kfree(bhs); + } + + if (status) + mlog_errno(status); + return status; +} + +static int ocfs2_symlink(struct mnt_idmap *idmap, + struct inode *dir, + struct dentry *dentry, + const char *symname) +{ + int status, l, credits; + u64 newsize; + struct ocfs2_super *osb = NULL; + struct inode *inode = NULL; + struct super_block *sb; + struct buffer_head *new_fe_bh = NULL; + struct buffer_head *parent_fe_bh = NULL; + struct ocfs2_dinode *fe = NULL; + struct ocfs2_dinode *dirfe; + handle_t *handle = NULL; + struct ocfs2_alloc_context *inode_ac = NULL; + struct ocfs2_alloc_context *data_ac = NULL; + struct ocfs2_alloc_context *xattr_ac = NULL; + int want_clusters = 0; + int xattr_credits = 0; + struct ocfs2_security_xattr_info si = { + .name = NULL, + .enable = 1, + }; + int did_quota = 0, did_quota_inode = 0; + struct ocfs2_dir_lookup_result lookup = { NULL, }; + sigset_t oldset; + int did_block_signals = 0; + struct ocfs2_dentry_lock *dl = NULL; + + trace_ocfs2_symlink_begin(dir, dentry, symname, + dentry->d_name.len, dentry->d_name.name); + + status = dquot_initialize(dir); + if (status) { + mlog_errno(status); + goto bail; + } + + sb = dir->i_sb; + osb = OCFS2_SB(sb); + + l = strlen(symname) + 1; + + credits = ocfs2_calc_symlink_credits(sb); + + /* lock the parent directory */ + status = ocfs2_inode_lock(dir, &parent_fe_bh, 1); + if (status < 0) { + if (status != -ENOENT) + mlog_errno(status); + return status; + } + + dirfe = (struct ocfs2_dinode *) parent_fe_bh->b_data; + if (!ocfs2_read_links_count(dirfe)) { + /* can't make a file in a deleted directory. */ + status = -ENOENT; + goto bail; + } + + status = ocfs2_check_dir_for_entry(dir, dentry->d_name.name, + dentry->d_name.len); + if (status) + goto bail; + + status = ocfs2_prepare_dir_for_insert(osb, dir, parent_fe_bh, + dentry->d_name.name, + dentry->d_name.len, &lookup); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + status = ocfs2_reserve_new_inode(osb, &inode_ac); + if (status < 0) { + if (status != -ENOSPC) + mlog_errno(status); + goto bail; + } + + inode = ocfs2_get_init_inode(dir, S_IFLNK | S_IRWXUGO); + if (IS_ERR(inode)) { + status = PTR_ERR(inode); + inode = NULL; + mlog_errno(status); + goto bail; + } + + /* get security xattr */ + status = ocfs2_init_security_get(inode, dir, &dentry->d_name, &si); + if (status) { + if (status == -EOPNOTSUPP) + si.enable = 0; + else { + mlog_errno(status); + goto bail; + } + } + + /* calculate meta data/clusters for setting security xattr */ + if (si.enable) { + status = ocfs2_calc_security_init(dir, &si, &want_clusters, + &xattr_credits, &xattr_ac); + if (status < 0) { + mlog_errno(status); + goto bail; + } + } + + /* don't reserve bitmap space for fast symlinks. */ + if (l > ocfs2_fast_symlink_chars(sb)) + want_clusters += 1; + + status = ocfs2_reserve_clusters(osb, want_clusters, &data_ac); + if (status < 0) { + if (status != -ENOSPC) + mlog_errno(status); + goto bail; + } + + handle = ocfs2_start_trans(osb, credits + xattr_credits); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + handle = NULL; + mlog_errno(status); + goto bail; + } + + /* Starting to change things, restart is no longer possible. */ + ocfs2_block_signals(&oldset); + did_block_signals = 1; + + status = dquot_alloc_inode(inode); + if (status) + goto bail; + did_quota_inode = 1; + + trace_ocfs2_symlink_create(dir, dentry, dentry->d_name.len, + dentry->d_name.name, + (unsigned long long)OCFS2_I(dir)->ip_blkno, + inode->i_mode); + + status = ocfs2_mknod_locked(osb, dir, inode, + 0, &new_fe_bh, parent_fe_bh, handle, + inode_ac); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + fe = (struct ocfs2_dinode *) new_fe_bh->b_data; + inode->i_rdev = 0; + newsize = l - 1; + inode->i_op = &ocfs2_symlink_inode_operations; + inode_nohighmem(inode); + if (l > ocfs2_fast_symlink_chars(sb)) { + u32 offset = 0; + + status = dquot_alloc_space_nodirty(inode, + ocfs2_clusters_to_bytes(osb->sb, 1)); + if (status) + goto bail; + did_quota = 1; + inode->i_mapping->a_ops = &ocfs2_aops; + status = ocfs2_add_inode_data(osb, inode, &offset, 1, 0, + new_fe_bh, + handle, data_ac, NULL, + NULL); + if (status < 0) { + if (status != -ENOSPC && status != -EINTR) { + mlog(ML_ERROR, + "Failed to extend file to %llu\n", + (unsigned long long)newsize); + mlog_errno(status); + status = -ENOSPC; + } + goto bail; + } + i_size_write(inode, newsize); + inode->i_blocks = ocfs2_inode_sector_count(inode); + } else { + inode->i_mapping->a_ops = &ocfs2_fast_symlink_aops; + memcpy((char *) fe->id2.i_symlink, symname, l); + i_size_write(inode, newsize); + inode->i_blocks = 0; + } + + status = ocfs2_mark_inode_dirty(handle, inode, new_fe_bh); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + if (!ocfs2_inode_is_fast_symlink(inode)) { + status = ocfs2_create_symlink_data(osb, handle, inode, + symname); + if (status < 0) { + mlog_errno(status); + goto bail; + } + } + + if (si.enable) { + status = ocfs2_init_security_set(handle, inode, new_fe_bh, &si, + xattr_ac, data_ac); + if (status < 0) { + mlog_errno(status); + goto bail; + } + } + + /* + * Do this before adding the entry to the directory. We add + * also set d_op after success so that ->d_iput() will cleanup + * the dentry lock even if ocfs2_add_entry() fails below. + */ + status = ocfs2_dentry_attach_lock(dentry, inode, OCFS2_I(dir)->ip_blkno); + if (status) { + mlog_errno(status); + goto bail; + } + + dl = dentry->d_fsdata; + + status = ocfs2_add_entry(handle, dentry, inode, + le64_to_cpu(fe->i_blkno), parent_fe_bh, + &lookup); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + insert_inode_hash(inode); + d_instantiate(dentry, inode); +bail: + if (status < 0 && did_quota) + dquot_free_space_nodirty(inode, + ocfs2_clusters_to_bytes(osb->sb, 1)); + if (status < 0 && did_quota_inode) + dquot_free_inode(inode); + if (handle) { + if (status < 0 && fe) + ocfs2_set_links_count(fe, 0); + ocfs2_commit_trans(osb, handle); + } + + ocfs2_inode_unlock(dir, 1); + if (did_block_signals) + ocfs2_unblock_signals(&oldset); + + brelse(new_fe_bh); + brelse(parent_fe_bh); + kfree(si.value); + ocfs2_free_dir_lookup_result(&lookup); + if (inode_ac) + ocfs2_free_alloc_context(inode_ac); + if (data_ac) + ocfs2_free_alloc_context(data_ac); + if (xattr_ac) + ocfs2_free_alloc_context(xattr_ac); + if ((status < 0) && inode) { + if (dl) + ocfs2_cleanup_add_entry_failure(osb, dentry, inode); + + OCFS2_I(inode)->ip_flags |= OCFS2_INODE_SKIP_ORPHAN_DIR; + clear_nlink(inode); + iput(inode); + } + + if (status) + mlog_errno(status); + + return status; +} + +static int ocfs2_blkno_stringify(u64 blkno, char *name) +{ + int status, namelen; + + namelen = snprintf(name, OCFS2_ORPHAN_NAMELEN + 1, "%016llx", + (long long)blkno); + if (namelen <= 0) { + if (namelen) + status = namelen; + else + status = -EINVAL; + mlog_errno(status); + goto bail; + } + if (namelen != OCFS2_ORPHAN_NAMELEN) { + status = -EINVAL; + mlog_errno(status); + goto bail; + } + + trace_ocfs2_blkno_stringify(blkno, name, namelen); + + status = 0; +bail: + if (status < 0) + mlog_errno(status); + return status; +} + +static int ocfs2_lookup_lock_orphan_dir(struct ocfs2_super *osb, + struct inode **ret_orphan_dir, + struct buffer_head **ret_orphan_dir_bh) +{ + struct inode *orphan_dir_inode; + struct buffer_head *orphan_dir_bh = NULL; + int ret = 0; + + orphan_dir_inode = ocfs2_get_system_file_inode(osb, + ORPHAN_DIR_SYSTEM_INODE, + osb->slot_num); + if (!orphan_dir_inode) { + ret = -ENOENT; + mlog_errno(ret); + return ret; + } + + inode_lock_nested(orphan_dir_inode, I_MUTEX_NONDIR2); + + ret = ocfs2_inode_lock(orphan_dir_inode, &orphan_dir_bh, 1); + if (ret < 0) { + inode_unlock(orphan_dir_inode); + iput(orphan_dir_inode); + + mlog_errno(ret); + return ret; + } + + *ret_orphan_dir = orphan_dir_inode; + *ret_orphan_dir_bh = orphan_dir_bh; + + return 0; +} + +static int __ocfs2_prepare_orphan_dir(struct inode *orphan_dir_inode, + struct buffer_head *orphan_dir_bh, + u64 blkno, + char *name, + struct ocfs2_dir_lookup_result *lookup, + bool dio) +{ + int ret; + struct ocfs2_super *osb = OCFS2_SB(orphan_dir_inode->i_sb); + int namelen = dio ? + (OCFS2_DIO_ORPHAN_PREFIX_LEN + OCFS2_ORPHAN_NAMELEN) : + OCFS2_ORPHAN_NAMELEN; + + if (dio) { + ret = snprintf(name, OCFS2_DIO_ORPHAN_PREFIX_LEN + 1, "%s", + OCFS2_DIO_ORPHAN_PREFIX); + if (ret != OCFS2_DIO_ORPHAN_PREFIX_LEN) { + ret = -EINVAL; + mlog_errno(ret); + return ret; + } + + ret = ocfs2_blkno_stringify(blkno, + name + OCFS2_DIO_ORPHAN_PREFIX_LEN); + } else + ret = ocfs2_blkno_stringify(blkno, name); + if (ret < 0) { + mlog_errno(ret); + return ret; + } + + ret = ocfs2_prepare_dir_for_insert(osb, orphan_dir_inode, + orphan_dir_bh, name, + namelen, lookup); + if (ret < 0) { + mlog_errno(ret); + return ret; + } + + return 0; +} + +/** + * ocfs2_prepare_orphan_dir() - Prepare an orphan directory for + * insertion of an orphan. + * @osb: ocfs2 file system + * @ret_orphan_dir: Orphan dir inode - returned locked! + * @blkno: Actual block number of the inode to be inserted into orphan dir. + * @name: Buffer to store the name of the orphan. + * @lookup: dir lookup result, to be passed back into functions like + * ocfs2_orphan_add + * @dio: Flag indicating if direct IO is being used or not. + * + * Returns zero on success and the ret_orphan_dir, name and lookup + * fields will be populated. + * + * Returns non-zero on failure. + */ +static int ocfs2_prepare_orphan_dir(struct ocfs2_super *osb, + struct inode **ret_orphan_dir, + u64 blkno, + char *name, + struct ocfs2_dir_lookup_result *lookup, + bool dio) +{ + struct inode *orphan_dir_inode = NULL; + struct buffer_head *orphan_dir_bh = NULL; + int ret = 0; + + ret = ocfs2_lookup_lock_orphan_dir(osb, &orphan_dir_inode, + &orphan_dir_bh); + if (ret < 0) { + mlog_errno(ret); + return ret; + } + + ret = __ocfs2_prepare_orphan_dir(orphan_dir_inode, orphan_dir_bh, + blkno, name, lookup, dio); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + *ret_orphan_dir = orphan_dir_inode; + +out: + brelse(orphan_dir_bh); + + if (ret) { + ocfs2_inode_unlock(orphan_dir_inode, 1); + inode_unlock(orphan_dir_inode); + iput(orphan_dir_inode); + } + + if (ret) + mlog_errno(ret); + return ret; +} + +static int ocfs2_orphan_add(struct ocfs2_super *osb, + handle_t *handle, + struct inode *inode, + struct buffer_head *fe_bh, + char *name, + struct ocfs2_dir_lookup_result *lookup, + struct inode *orphan_dir_inode, + bool dio) +{ + struct buffer_head *orphan_dir_bh = NULL; + int status = 0; + struct ocfs2_dinode *orphan_fe; + struct ocfs2_dinode *fe = (struct ocfs2_dinode *) fe_bh->b_data; + int namelen = dio ? + (OCFS2_DIO_ORPHAN_PREFIX_LEN + OCFS2_ORPHAN_NAMELEN) : + OCFS2_ORPHAN_NAMELEN; + + trace_ocfs2_orphan_add_begin( + (unsigned long long)OCFS2_I(inode)->ip_blkno); + + status = ocfs2_read_inode_block(orphan_dir_inode, &orphan_dir_bh); + if (status < 0) { + mlog_errno(status); + goto leave; + } + + status = ocfs2_journal_access_di(handle, + INODE_CACHE(orphan_dir_inode), + orphan_dir_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto leave; + } + + /* + * We're going to journal the change of i_flags and i_orphaned_slot. + * It's safe anyway, though some callers may duplicate the journaling. + * Journaling within the func just make the logic look more + * straightforward. + */ + status = ocfs2_journal_access_di(handle, + INODE_CACHE(inode), + fe_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto leave; + } + + /* we're a cluster, and nlink can change on disk from + * underneath us... */ + orphan_fe = (struct ocfs2_dinode *) orphan_dir_bh->b_data; + if (S_ISDIR(inode->i_mode)) + ocfs2_add_links_count(orphan_fe, 1); + set_nlink(orphan_dir_inode, ocfs2_read_links_count(orphan_fe)); + ocfs2_journal_dirty(handle, orphan_dir_bh); + + status = __ocfs2_add_entry(handle, orphan_dir_inode, name, + namelen, inode, + OCFS2_I(inode)->ip_blkno, + orphan_dir_bh, lookup); + if (status < 0) { + mlog_errno(status); + goto rollback; + } + + if (dio) { + /* Update flag OCFS2_DIO_ORPHANED_FL and record the orphan + * slot. + */ + fe->i_flags |= cpu_to_le32(OCFS2_DIO_ORPHANED_FL); + fe->i_dio_orphaned_slot = cpu_to_le16(osb->slot_num); + } else { + fe->i_flags |= cpu_to_le32(OCFS2_ORPHANED_FL); + OCFS2_I(inode)->ip_flags &= ~OCFS2_INODE_SKIP_ORPHAN_DIR; + + /* Record which orphan dir our inode now resides + * in. delete_inode will use this to determine which orphan + * dir to lock. */ + fe->i_orphaned_slot = cpu_to_le16(osb->slot_num); + } + + ocfs2_journal_dirty(handle, fe_bh); + + trace_ocfs2_orphan_add_end((unsigned long long)OCFS2_I(inode)->ip_blkno, + osb->slot_num); + +rollback: + if (status < 0) { + if (S_ISDIR(inode->i_mode)) + ocfs2_add_links_count(orphan_fe, -1); + set_nlink(orphan_dir_inode, ocfs2_read_links_count(orphan_fe)); + } + +leave: + brelse(orphan_dir_bh); + + return status; +} + +/* unlike orphan_add, we expect the orphan dir to already be locked here. */ +int ocfs2_orphan_del(struct ocfs2_super *osb, + handle_t *handle, + struct inode *orphan_dir_inode, + struct inode *inode, + struct buffer_head *orphan_dir_bh, + bool dio) +{ + char name[OCFS2_DIO_ORPHAN_PREFIX_LEN + OCFS2_ORPHAN_NAMELEN + 1]; + struct ocfs2_dinode *orphan_fe; + int status = 0; + struct ocfs2_dir_lookup_result lookup = { NULL, }; + + if (dio) { + status = snprintf(name, OCFS2_DIO_ORPHAN_PREFIX_LEN + 1, "%s", + OCFS2_DIO_ORPHAN_PREFIX); + if (status != OCFS2_DIO_ORPHAN_PREFIX_LEN) { + status = -EINVAL; + mlog_errno(status); + return status; + } + + status = ocfs2_blkno_stringify(OCFS2_I(inode)->ip_blkno, + name + OCFS2_DIO_ORPHAN_PREFIX_LEN); + } else + status = ocfs2_blkno_stringify(OCFS2_I(inode)->ip_blkno, name); + if (status < 0) { + mlog_errno(status); + goto leave; + } + + trace_ocfs2_orphan_del( + (unsigned long long)OCFS2_I(orphan_dir_inode)->ip_blkno, + name, strlen(name)); + + status = ocfs2_journal_access_di(handle, + INODE_CACHE(orphan_dir_inode), + orphan_dir_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto leave; + } + + /* find it's spot in the orphan directory */ + status = ocfs2_find_entry(name, strlen(name), orphan_dir_inode, + &lookup); + if (status) { + mlog_errno(status); + goto leave; + } + + /* remove it from the orphan directory */ + status = ocfs2_delete_entry(handle, orphan_dir_inode, &lookup); + if (status < 0) { + mlog_errno(status); + goto leave; + } + + /* do the i_nlink dance! :) */ + orphan_fe = (struct ocfs2_dinode *) orphan_dir_bh->b_data; + if (S_ISDIR(inode->i_mode)) + ocfs2_add_links_count(orphan_fe, -1); + set_nlink(orphan_dir_inode, ocfs2_read_links_count(orphan_fe)); + ocfs2_journal_dirty(handle, orphan_dir_bh); + +leave: + ocfs2_free_dir_lookup_result(&lookup); + + if (status) + mlog_errno(status); + return status; +} + +/** + * ocfs2_prep_new_orphaned_file() - Prepare the orphan dir to receive a newly + * allocated file. This is different from the typical 'add to orphan dir' + * operation in that the inode does not yet exist. This is a problem because + * the orphan dir stringifies the inode block number to come up with it's + * dirent. Obviously if the inode does not yet exist we have a chicken and egg + * problem. This function works around it by calling deeper into the orphan + * and suballoc code than other callers. Use this only by necessity. + * @dir: The directory which this inode will ultimately wind up under - not the + * orphan dir! + * @dir_bh: buffer_head the @dir inode block + * @orphan_name: string of length (CFS2_ORPHAN_NAMELEN + 1). Will be filled + * with the string to be used for orphan dirent. Pass back to the orphan dir + * code. + * @ret_orphan_dir: orphan dir inode returned to be passed back into orphan + * dir code. + * @ret_di_blkno: block number where the new inode will be allocated. + * @orphan_insert: Dir insert context to be passed back into orphan dir code. + * @ret_inode_ac: Inode alloc context to be passed back to the allocator. + * + * Returns zero on success and the ret_orphan_dir, name and lookup + * fields will be populated. + * + * Returns non-zero on failure. + */ +static int ocfs2_prep_new_orphaned_file(struct inode *dir, + struct buffer_head *dir_bh, + char *orphan_name, + struct inode **ret_orphan_dir, + u64 *ret_di_blkno, + struct ocfs2_dir_lookup_result *orphan_insert, + struct ocfs2_alloc_context **ret_inode_ac) +{ + int ret; + u64 di_blkno; + struct ocfs2_super *osb = OCFS2_SB(dir->i_sb); + struct inode *orphan_dir = NULL; + struct buffer_head *orphan_dir_bh = NULL; + struct ocfs2_alloc_context *inode_ac = NULL; + + ret = ocfs2_lookup_lock_orphan_dir(osb, &orphan_dir, &orphan_dir_bh); + if (ret < 0) { + mlog_errno(ret); + return ret; + } + + /* reserve an inode spot */ + ret = ocfs2_reserve_new_inode(osb, &inode_ac); + if (ret < 0) { + if (ret != -ENOSPC) + mlog_errno(ret); + goto out; + } + + ret = ocfs2_find_new_inode_loc(dir, dir_bh, inode_ac, + &di_blkno); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = __ocfs2_prepare_orphan_dir(orphan_dir, orphan_dir_bh, + di_blkno, orphan_name, orphan_insert, + false); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + +out: + if (ret == 0) { + *ret_orphan_dir = orphan_dir; + *ret_di_blkno = di_blkno; + *ret_inode_ac = inode_ac; + /* + * orphan_name and orphan_insert are already up to + * date via prepare_orphan_dir + */ + } else { + /* Unroll reserve_new_inode* */ + if (inode_ac) + ocfs2_free_alloc_context(inode_ac); + + /* Unroll orphan dir locking */ + inode_unlock(orphan_dir); + ocfs2_inode_unlock(orphan_dir, 1); + iput(orphan_dir); + } + + brelse(orphan_dir_bh); + + return ret; +} + +int ocfs2_create_inode_in_orphan(struct inode *dir, + int mode, + struct inode **new_inode) +{ + int status, did_quota_inode = 0; + struct inode *inode = NULL; + struct inode *orphan_dir = NULL; + struct ocfs2_super *osb = OCFS2_SB(dir->i_sb); + handle_t *handle = NULL; + char orphan_name[OCFS2_ORPHAN_NAMELEN + 1]; + struct buffer_head *parent_di_bh = NULL; + struct buffer_head *new_di_bh = NULL; + struct ocfs2_alloc_context *inode_ac = NULL; + struct ocfs2_dir_lookup_result orphan_insert = { NULL, }; + u64 di_blkno, suballoc_loc; + u16 suballoc_bit; + + status = ocfs2_inode_lock(dir, &parent_di_bh, 1); + if (status < 0) { + if (status != -ENOENT) + mlog_errno(status); + return status; + } + + status = ocfs2_prep_new_orphaned_file(dir, parent_di_bh, + orphan_name, &orphan_dir, + &di_blkno, &orphan_insert, &inode_ac); + if (status < 0) { + if (status != -ENOSPC) + mlog_errno(status); + goto leave; + } + + inode = ocfs2_get_init_inode(dir, mode); + if (IS_ERR(inode)) { + status = PTR_ERR(inode); + inode = NULL; + mlog_errno(status); + goto leave; + } + + handle = ocfs2_start_trans(osb, ocfs2_mknod_credits(osb->sb, 0, 0)); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + handle = NULL; + mlog_errno(status); + goto leave; + } + + status = dquot_alloc_inode(inode); + if (status) + goto leave; + did_quota_inode = 1; + + status = ocfs2_claim_new_inode_at_loc(handle, dir, inode_ac, + &suballoc_loc, + &suballoc_bit, di_blkno); + if (status < 0) { + mlog_errno(status); + goto leave; + } + + clear_nlink(inode); + /* do the real work now. */ + status = __ocfs2_mknod_locked(dir, inode, + 0, &new_di_bh, handle, + inode_ac, di_blkno, suballoc_loc, + suballoc_bit); + if (status < 0) { + mlog_errno(status); + goto leave; + } + + status = ocfs2_orphan_add(osb, handle, inode, new_di_bh, orphan_name, + &orphan_insert, orphan_dir, false); + if (status < 0) { + mlog_errno(status); + goto leave; + } + + /* get open lock so that only nodes can't remove it from orphan dir. */ + status = ocfs2_open_lock(inode); + if (status < 0) + mlog_errno(status); + + insert_inode_hash(inode); +leave: + if (status < 0 && did_quota_inode) + dquot_free_inode(inode); + if (handle) + ocfs2_commit_trans(osb, handle); + + if (orphan_dir) { + /* This was locked for us in ocfs2_prepare_orphan_dir() */ + ocfs2_inode_unlock(orphan_dir, 1); + inode_unlock(orphan_dir); + iput(orphan_dir); + } + + if ((status < 0) && inode) { + clear_nlink(inode); + iput(inode); + } + + if (inode_ac) + ocfs2_free_alloc_context(inode_ac); + + brelse(new_di_bh); + + if (!status) + *new_inode = inode; + + ocfs2_free_dir_lookup_result(&orphan_insert); + + ocfs2_inode_unlock(dir, 1); + brelse(parent_di_bh); + return status; +} + +int ocfs2_add_inode_to_orphan(struct ocfs2_super *osb, + struct inode *inode) +{ + char orphan_name[OCFS2_DIO_ORPHAN_PREFIX_LEN + OCFS2_ORPHAN_NAMELEN + 1]; + struct inode *orphan_dir_inode = NULL; + struct ocfs2_dir_lookup_result orphan_insert = { NULL, }; + struct buffer_head *di_bh = NULL; + int status = 0; + handle_t *handle = NULL; + struct ocfs2_dinode *di = NULL; + + status = ocfs2_inode_lock(inode, &di_bh, 1); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + di = (struct ocfs2_dinode *) di_bh->b_data; + /* + * Another append dio crashed? + * If so, manually recover it first. + */ + if (unlikely(di->i_flags & cpu_to_le32(OCFS2_DIO_ORPHANED_FL))) { + status = ocfs2_truncate_file(inode, di_bh, i_size_read(inode)); + if (status < 0) { + if (status != -ENOSPC) + mlog_errno(status); + goto bail_unlock_inode; + } + + status = ocfs2_del_inode_from_orphan(osb, inode, di_bh, 0, 0); + if (status < 0) { + mlog_errno(status); + goto bail_unlock_inode; + } + } + + status = ocfs2_prepare_orphan_dir(osb, &orphan_dir_inode, + OCFS2_I(inode)->ip_blkno, + orphan_name, + &orphan_insert, + true); + if (status < 0) { + mlog_errno(status); + goto bail_unlock_inode; + } + + handle = ocfs2_start_trans(osb, + OCFS2_INODE_ADD_TO_ORPHAN_CREDITS); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + goto bail_unlock_orphan; + } + + status = ocfs2_orphan_add(osb, handle, inode, di_bh, orphan_name, + &orphan_insert, orphan_dir_inode, true); + if (status) + mlog_errno(status); + + ocfs2_commit_trans(osb, handle); + +bail_unlock_orphan: + ocfs2_inode_unlock(orphan_dir_inode, 1); + inode_unlock(orphan_dir_inode); + iput(orphan_dir_inode); + + ocfs2_free_dir_lookup_result(&orphan_insert); + +bail_unlock_inode: + ocfs2_inode_unlock(inode, 1); + brelse(di_bh); + +bail: + return status; +} + +int ocfs2_del_inode_from_orphan(struct ocfs2_super *osb, + struct inode *inode, struct buffer_head *di_bh, + int update_isize, loff_t end) +{ + struct inode *orphan_dir_inode = NULL; + struct buffer_head *orphan_dir_bh = NULL; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data; + handle_t *handle = NULL; + int status = 0; + + orphan_dir_inode = ocfs2_get_system_file_inode(osb, + ORPHAN_DIR_SYSTEM_INODE, + le16_to_cpu(di->i_dio_orphaned_slot)); + if (!orphan_dir_inode) { + status = -ENOENT; + mlog_errno(status); + goto bail; + } + + inode_lock_nested(orphan_dir_inode, I_MUTEX_NONDIR2); + status = ocfs2_inode_lock(orphan_dir_inode, &orphan_dir_bh, 1); + if (status < 0) { + inode_unlock(orphan_dir_inode); + iput(orphan_dir_inode); + mlog_errno(status); + goto bail; + } + + handle = ocfs2_start_trans(osb, + OCFS2_INODE_DEL_FROM_ORPHAN_CREDITS); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + goto bail_unlock_orphan; + } + + BUG_ON(!(di->i_flags & cpu_to_le32(OCFS2_DIO_ORPHANED_FL))); + + status = ocfs2_orphan_del(osb, handle, orphan_dir_inode, + inode, orphan_dir_bh, true); + if (status < 0) { + mlog_errno(status); + goto bail_commit; + } + + status = ocfs2_journal_access_di(handle, + INODE_CACHE(inode), + di_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto bail_commit; + } + + di->i_flags &= ~cpu_to_le32(OCFS2_DIO_ORPHANED_FL); + di->i_dio_orphaned_slot = 0; + + if (update_isize) { + status = ocfs2_set_inode_size(handle, inode, di_bh, end); + if (status) + mlog_errno(status); + } else + ocfs2_journal_dirty(handle, di_bh); + +bail_commit: + ocfs2_commit_trans(osb, handle); + +bail_unlock_orphan: + ocfs2_inode_unlock(orphan_dir_inode, 1); + inode_unlock(orphan_dir_inode); + brelse(orphan_dir_bh); + iput(orphan_dir_inode); + +bail: + return status; +} + +int ocfs2_mv_orphaned_inode_to_new(struct inode *dir, + struct inode *inode, + struct dentry *dentry) +{ + int status = 0; + struct buffer_head *parent_di_bh = NULL; + handle_t *handle = NULL; + struct ocfs2_super *osb = OCFS2_SB(dir->i_sb); + struct ocfs2_dinode *dir_di, *di; + struct inode *orphan_dir_inode = NULL; + struct buffer_head *orphan_dir_bh = NULL; + struct buffer_head *di_bh = NULL; + struct ocfs2_dir_lookup_result lookup = { NULL, }; + + trace_ocfs2_mv_orphaned_inode_to_new(dir, dentry, + dentry->d_name.len, dentry->d_name.name, + (unsigned long long)OCFS2_I(dir)->ip_blkno, + (unsigned long long)OCFS2_I(inode)->ip_blkno); + + status = ocfs2_inode_lock(dir, &parent_di_bh, 1); + if (status < 0) { + if (status != -ENOENT) + mlog_errno(status); + return status; + } + + dir_di = (struct ocfs2_dinode *) parent_di_bh->b_data; + if (!dir_di->i_links_count) { + /* can't make a file in a deleted directory. */ + status = -ENOENT; + goto leave; + } + + status = ocfs2_check_dir_for_entry(dir, dentry->d_name.name, + dentry->d_name.len); + if (status) + goto leave; + + /* get a spot inside the dir. */ + status = ocfs2_prepare_dir_for_insert(osb, dir, parent_di_bh, + dentry->d_name.name, + dentry->d_name.len, &lookup); + if (status < 0) { + mlog_errno(status); + goto leave; + } + + orphan_dir_inode = ocfs2_get_system_file_inode(osb, + ORPHAN_DIR_SYSTEM_INODE, + osb->slot_num); + if (!orphan_dir_inode) { + status = -ENOENT; + mlog_errno(status); + goto leave; + } + + inode_lock_nested(orphan_dir_inode, I_MUTEX_NONDIR2); + + status = ocfs2_inode_lock(orphan_dir_inode, &orphan_dir_bh, 1); + if (status < 0) { + mlog_errno(status); + inode_unlock(orphan_dir_inode); + iput(orphan_dir_inode); + goto leave; + } + + status = ocfs2_read_inode_block(inode, &di_bh); + if (status < 0) { + mlog_errno(status); + goto orphan_unlock; + } + + handle = ocfs2_start_trans(osb, ocfs2_rename_credits(osb->sb)); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + handle = NULL; + mlog_errno(status); + goto orphan_unlock; + } + + status = ocfs2_journal_access_di(handle, INODE_CACHE(inode), + di_bh, OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto out_commit; + } + + status = ocfs2_orphan_del(osb, handle, orphan_dir_inode, inode, + orphan_dir_bh, false); + if (status < 0) { + mlog_errno(status); + goto out_commit; + } + + di = (struct ocfs2_dinode *)di_bh->b_data; + di->i_flags &= ~cpu_to_le32(OCFS2_ORPHANED_FL); + di->i_orphaned_slot = 0; + set_nlink(inode, 1); + ocfs2_set_links_count(di, inode->i_nlink); + ocfs2_update_inode_fsync_trans(handle, inode, 1); + ocfs2_journal_dirty(handle, di_bh); + + status = ocfs2_add_entry(handle, dentry, inode, + OCFS2_I(inode)->ip_blkno, parent_di_bh, + &lookup); + if (status < 0) { + mlog_errno(status); + goto out_commit; + } + + status = ocfs2_dentry_attach_lock(dentry, inode, + OCFS2_I(dir)->ip_blkno); + if (status) { + mlog_errno(status); + goto out_commit; + } + + d_instantiate(dentry, inode); + status = 0; +out_commit: + ocfs2_commit_trans(osb, handle); +orphan_unlock: + ocfs2_inode_unlock(orphan_dir_inode, 1); + inode_unlock(orphan_dir_inode); + iput(orphan_dir_inode); +leave: + + ocfs2_inode_unlock(dir, 1); + + brelse(di_bh); + brelse(parent_di_bh); + brelse(orphan_dir_bh); + + ocfs2_free_dir_lookup_result(&lookup); + + if (status) + mlog_errno(status); + + return status; +} + +const struct inode_operations ocfs2_dir_iops = { + .create = ocfs2_create, + .lookup = ocfs2_lookup, + .link = ocfs2_link, + .unlink = ocfs2_unlink, + .rmdir = ocfs2_unlink, + .symlink = ocfs2_symlink, + .mkdir = ocfs2_mkdir, + .mknod = ocfs2_mknod, + .rename = ocfs2_rename, + .setattr = ocfs2_setattr, + .getattr = ocfs2_getattr, + .permission = ocfs2_permission, + .listxattr = ocfs2_listxattr, + .fiemap = ocfs2_fiemap, + .get_inode_acl = ocfs2_iop_get_acl, + .set_acl = ocfs2_iop_set_acl, + .fileattr_get = ocfs2_fileattr_get, + .fileattr_set = ocfs2_fileattr_set, +}; diff --git a/fs/ocfs2/namei.h b/fs/ocfs2/namei.h new file mode 100644 index 000000000..9cc891eb8 --- /dev/null +++ b/fs/ocfs2/namei.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * namei.h + * + * Function prototypes + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#ifndef OCFS2_NAMEI_H +#define OCFS2_NAMEI_H + +#define OCFS2_DIO_ORPHAN_PREFIX "dio-" +#define OCFS2_DIO_ORPHAN_PREFIX_LEN 4 + +extern const struct inode_operations ocfs2_dir_iops; + +struct dentry *ocfs2_get_parent(struct dentry *child); + +int ocfs2_orphan_del(struct ocfs2_super *osb, + handle_t *handle, + struct inode *orphan_dir_inode, + struct inode *inode, + struct buffer_head *orphan_dir_bh, + bool dio); +int ocfs2_create_inode_in_orphan(struct inode *dir, + int mode, + struct inode **new_inode); +int ocfs2_add_inode_to_orphan(struct ocfs2_super *osb, + struct inode *inode); +int ocfs2_del_inode_from_orphan(struct ocfs2_super *osb, + struct inode *inode, struct buffer_head *di_bh, + int update_isize, loff_t end); +int ocfs2_mv_orphaned_inode_to_new(struct inode *dir, + struct inode *new_inode, + struct dentry *new_dentry); + +#endif /* OCFS2_NAMEI_H */ diff --git a/fs/ocfs2/ocfs1_fs_compat.h b/fs/ocfs2/ocfs1_fs_compat.h new file mode 100644 index 000000000..6dbcf3d46 --- /dev/null +++ b/fs/ocfs2/ocfs1_fs_compat.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * ocfs1_fs_compat.h + * + * OCFS1 volume header definitions. OCFS2 creates valid but unmountable + * OCFS1 volume headers on the first two sectors of an OCFS2 volume. + * This allows an OCFS1 volume to see the partition and cleanly fail to + * mount it. + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#ifndef _OCFS1_FS_COMPAT_H +#define _OCFS1_FS_COMPAT_H + +#define OCFS1_MAX_VOL_SIGNATURE_LEN 128 +#define OCFS1_MAX_MOUNT_POINT_LEN 128 +#define OCFS1_MAX_VOL_ID_LENGTH 16 +#define OCFS1_MAX_VOL_LABEL_LEN 64 +#define OCFS1_MAX_CLUSTER_NAME_LEN 64 + +#define OCFS1_MAJOR_VERSION (2) +#define OCFS1_MINOR_VERSION (0) +#define OCFS1_VOLUME_SIGNATURE "OracleCFS" + +/* + * OCFS1 superblock. Lives at sector 0. + */ +struct ocfs1_vol_disk_hdr +{ +/*00*/ __u32 minor_version; + __u32 major_version; +/*08*/ __u8 signature[OCFS1_MAX_VOL_SIGNATURE_LEN]; +/*88*/ __u8 mount_point[OCFS1_MAX_MOUNT_POINT_LEN]; +/*108*/ __u64 serial_num; +/*110*/ __u64 device_size; + __u64 start_off; +/*120*/ __u64 bitmap_off; + __u64 publ_off; +/*130*/ __u64 vote_off; + __u64 root_bitmap_off; +/*140*/ __u64 data_start_off; + __u64 root_bitmap_size; +/*150*/ __u64 root_off; + __u64 root_size; +/*160*/ __u64 cluster_size; + __u64 num_nodes; +/*170*/ __u64 num_clusters; + __u64 dir_node_size; +/*180*/ __u64 file_node_size; + __u64 internal_off; +/*190*/ __u64 node_cfg_off; + __u64 node_cfg_size; +/*1A0*/ __u64 new_cfg_off; + __u32 prot_bits; + __s32 excl_mount; +/*1B0*/ +}; + + +struct ocfs1_disk_lock +{ +/*00*/ __u32 curr_master; + __u8 file_lock; + __u8 compat_pad[3]; /* Not in original definition. Used to + make the already existing alignment + explicit */ + __u64 last_write_time; +/*10*/ __u64 last_read_time; + __u32 writer_node_num; + __u32 reader_node_num; +/*20*/ __u64 oin_node_map; + __u64 dlock_seq_num; +/*30*/ +}; + +/* + * OCFS1 volume label. Lives at sector 1. + */ +struct ocfs1_vol_label +{ +/*00*/ struct ocfs1_disk_lock disk_lock; +/*30*/ __u8 label[OCFS1_MAX_VOL_LABEL_LEN]; +/*70*/ __u16 label_len; +/*72*/ __u8 vol_id[OCFS1_MAX_VOL_ID_LENGTH]; +/*82*/ __u16 vol_id_len; +/*84*/ __u8 cluster_name[OCFS1_MAX_CLUSTER_NAME_LEN]; +/*A4*/ __u16 cluster_name_len; +/*A6*/ +}; + + +#endif /* _OCFS1_FS_COMPAT_H */ + diff --git a/fs/ocfs2/ocfs2.h b/fs/ocfs2/ocfs2.h new file mode 100644 index 000000000..62cad6522 --- /dev/null +++ b/fs/ocfs2/ocfs2.h @@ -0,0 +1,991 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * ocfs2.h + * + * Defines macros and structures used in OCFS2 + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#ifndef OCFS2_H +#define OCFS2_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* For union ocfs2_dlm_lksb */ +#include "stackglue.h" + +#include "ocfs2_fs.h" +#include "ocfs2_lockid.h" +#include "ocfs2_ioctl.h" + +/* For struct ocfs2_blockcheck_stats */ +#include "blockcheck.h" + +#include "reservations.h" + +#include "filecheck.h" + +/* Caching of metadata buffers */ + +/* Most user visible OCFS2 inodes will have very few pieces of + * metadata, but larger files (including bitmaps, etc) must be taken + * into account when designing an access scheme. We allow a small + * amount of inlined blocks to be stored on an array and grow the + * structure into a rb tree when necessary. */ +#define OCFS2_CACHE_INFO_MAX_ARRAY 2 + +/* Flags for ocfs2_caching_info */ + +enum ocfs2_caching_info_flags { + /* Indicates that the metadata cache is using the inline array */ + OCFS2_CACHE_FL_INLINE = 1<<1, +}; + +struct ocfs2_caching_operations; +struct ocfs2_caching_info { + /* + * The parent structure provides the locks, but because the + * parent structure can differ, it provides locking operations + * to struct ocfs2_caching_info. + */ + const struct ocfs2_caching_operations *ci_ops; + + /* next two are protected by trans_inc_lock */ + /* which transaction were we created on? Zero if none. */ + unsigned long ci_created_trans; + /* last transaction we were a part of. */ + unsigned long ci_last_trans; + + /* Cache structures */ + unsigned int ci_flags; + unsigned int ci_num_cached; + union { + sector_t ci_array[OCFS2_CACHE_INFO_MAX_ARRAY]; + struct rb_root ci_tree; + } ci_cache; +}; +/* + * Need this prototype here instead of in uptodate.h because journal.h + * uses it. + */ +struct super_block *ocfs2_metadata_cache_get_super(struct ocfs2_caching_info *ci); + +/* this limits us to 256 nodes + * if we need more, we can do a kmalloc for the map */ +#define OCFS2_NODE_MAP_MAX_NODES 256 +struct ocfs2_node_map { + u16 num_nodes; + unsigned long map[BITS_TO_LONGS(OCFS2_NODE_MAP_MAX_NODES)]; +}; + +enum ocfs2_ast_action { + OCFS2_AST_INVALID = 0, + OCFS2_AST_ATTACH, + OCFS2_AST_CONVERT, + OCFS2_AST_DOWNCONVERT, +}; + +/* actions for an unlockast function to take. */ +enum ocfs2_unlock_action { + OCFS2_UNLOCK_INVALID = 0, + OCFS2_UNLOCK_CANCEL_CONVERT, + OCFS2_UNLOCK_DROP_LOCK, +}; + +/* ocfs2_lock_res->l_flags flags. */ +#define OCFS2_LOCK_ATTACHED (0x00000001) /* we have initialized + * the lvb */ +#define OCFS2_LOCK_BUSY (0x00000002) /* we are currently in + * dlm_lock */ +#define OCFS2_LOCK_BLOCKED (0x00000004) /* blocked waiting to + * downconvert*/ +#define OCFS2_LOCK_LOCAL (0x00000008) /* newly created inode */ +#define OCFS2_LOCK_NEEDS_REFRESH (0x00000010) +#define OCFS2_LOCK_REFRESHING (0x00000020) +#define OCFS2_LOCK_INITIALIZED (0x00000040) /* track initialization + * for shutdown paths */ +#define OCFS2_LOCK_FREEING (0x00000080) /* help dlmglue track + * when to skip queueing + * a lock because it's + * about to be + * dropped. */ +#define OCFS2_LOCK_QUEUED (0x00000100) /* queued for downconvert */ +#define OCFS2_LOCK_NOCACHE (0x00000200) /* don't use a holder count */ +#define OCFS2_LOCK_PENDING (0x00000400) /* This lockres is pending a + call to dlm_lock. Only + exists with BUSY set. */ +#define OCFS2_LOCK_UPCONVERT_FINISHING (0x00000800) /* blocks the dc thread + * from downconverting + * before the upconvert + * has completed */ + +#define OCFS2_LOCK_NONBLOCK_FINISHED (0x00001000) /* NONBLOCK cluster + * lock has already + * returned, do not block + * dc thread from + * downconverting */ + +struct ocfs2_lock_res_ops; + +typedef void (*ocfs2_lock_callback)(int status, unsigned long data); + +#ifdef CONFIG_OCFS2_FS_STATS +struct ocfs2_lock_stats { + u64 ls_total; /* Total wait in NSEC */ + u32 ls_gets; /* Num acquires */ + u32 ls_fail; /* Num failed acquires */ + + /* Storing max wait in usecs saves 24 bytes per inode */ + u32 ls_max; /* Max wait in USEC */ + u64 ls_last; /* Last unlock time in USEC */ +}; +#endif + +struct ocfs2_lock_res { + void *l_priv; + const struct ocfs2_lock_res_ops *l_ops; + + + struct list_head l_blocked_list; + struct list_head l_mask_waiters; + struct list_head l_holders; + + unsigned long l_flags; + char l_name[OCFS2_LOCK_ID_MAX_LEN]; + unsigned int l_ro_holders; + unsigned int l_ex_holders; + signed char l_level; + signed char l_requested; + signed char l_blocking; + + /* Data packed - type enum ocfs2_lock_type */ + unsigned char l_type; + + /* used from AST/BAST funcs. */ + /* Data packed - enum type ocfs2_ast_action */ + unsigned char l_action; + /* Data packed - enum type ocfs2_unlock_action */ + unsigned char l_unlock_action; + unsigned int l_pending_gen; + + spinlock_t l_lock; + + struct ocfs2_dlm_lksb l_lksb; + + wait_queue_head_t l_event; + + struct list_head l_debug_list; + +#ifdef CONFIG_OCFS2_FS_STATS + struct ocfs2_lock_stats l_lock_prmode; /* PR mode stats */ + u32 l_lock_refresh; /* Disk refreshes */ + u64 l_lock_wait; /* First lock wait time */ + struct ocfs2_lock_stats l_lock_exmode; /* EX mode stats */ +#endif +#ifdef CONFIG_DEBUG_LOCK_ALLOC + struct lockdep_map l_lockdep_map; +#endif +}; + +enum ocfs2_orphan_reco_type { + ORPHAN_NO_NEED_TRUNCATE = 0, + ORPHAN_NEED_TRUNCATE, +}; + +enum ocfs2_orphan_scan_state { + ORPHAN_SCAN_ACTIVE, + ORPHAN_SCAN_INACTIVE +}; + +struct ocfs2_orphan_scan { + struct mutex os_lock; + struct ocfs2_super *os_osb; + struct ocfs2_lock_res os_lockres; /* lock to synchronize scans */ + struct delayed_work os_orphan_scan_work; + time64_t os_scantime; /* time this node ran the scan */ + u32 os_count; /* tracks node specific scans */ + u32 os_seqno; /* tracks cluster wide scans */ + atomic_t os_state; /* ACTIVE or INACTIVE */ +}; + +struct ocfs2_dlm_debug { + struct kref d_refcnt; + u32 d_filter_secs; + struct list_head d_lockres_tracking; +}; + +enum ocfs2_vol_state +{ + VOLUME_INIT = 0, + VOLUME_MOUNTED, + VOLUME_MOUNTED_QUOTAS, + VOLUME_DISMOUNTED, + VOLUME_DISABLED +}; + +struct ocfs2_alloc_stats +{ + atomic_t moves; + atomic_t local_data; + atomic_t bitmap_data; + atomic_t bg_allocs; + atomic_t bg_extends; +}; + +enum ocfs2_local_alloc_state +{ + OCFS2_LA_UNUSED = 0, /* Local alloc will never be used for + * this mountpoint. */ + OCFS2_LA_ENABLED, /* Local alloc is in use. */ + OCFS2_LA_THROTTLED, /* Local alloc is in use, but number + * of bits has been reduced. */ + OCFS2_LA_DISABLED /* Local alloc has temporarily been + * disabled. */ +}; + +enum ocfs2_mount_options +{ + OCFS2_MOUNT_HB_LOCAL = 1 << 0, /* Local heartbeat */ + OCFS2_MOUNT_BARRIER = 1 << 1, /* Use block barriers */ + OCFS2_MOUNT_NOINTR = 1 << 2, /* Don't catch signals */ + OCFS2_MOUNT_ERRORS_PANIC = 1 << 3, /* Panic on errors */ + OCFS2_MOUNT_DATA_WRITEBACK = 1 << 4, /* No data ordering */ + OCFS2_MOUNT_LOCALFLOCKS = 1 << 5, /* No cluster aware user file locks */ + OCFS2_MOUNT_NOUSERXATTR = 1 << 6, /* No user xattr */ + OCFS2_MOUNT_INODE64 = 1 << 7, /* Allow inode numbers > 2^32 */ + OCFS2_MOUNT_POSIX_ACL = 1 << 8, /* Force POSIX access control lists */ + OCFS2_MOUNT_NO_POSIX_ACL = 1 << 9, /* Disable POSIX access + control lists */ + OCFS2_MOUNT_USRQUOTA = 1 << 10, /* We support user quotas */ + OCFS2_MOUNT_GRPQUOTA = 1 << 11, /* We support group quotas */ + OCFS2_MOUNT_COHERENCY_BUFFERED = 1 << 12, /* Allow concurrent O_DIRECT + writes */ + OCFS2_MOUNT_HB_NONE = 1 << 13, /* No heartbeat */ + OCFS2_MOUNT_HB_GLOBAL = 1 << 14, /* Global heartbeat */ + + OCFS2_MOUNT_JOURNAL_ASYNC_COMMIT = 1 << 15, /* Journal Async Commit */ + OCFS2_MOUNT_ERRORS_CONT = 1 << 16, /* Return EIO to the calling process on error */ + OCFS2_MOUNT_ERRORS_ROFS = 1 << 17, /* Change filesystem to read-only on error */ +}; + +#define OCFS2_OSB_SOFT_RO 0x0001 +#define OCFS2_OSB_HARD_RO 0x0002 +#define OCFS2_OSB_ERROR_FS 0x0004 +#define OCFS2_DEFAULT_ATIME_QUANTUM 60 + +struct ocfs2_triggers { + struct jbd2_buffer_trigger_type ot_triggers; + int ot_offset; + struct super_block *sb; +}; + +enum ocfs2_journal_trigger_type { + OCFS2_JTR_DI, + OCFS2_JTR_EB, + OCFS2_JTR_RB, + OCFS2_JTR_GD, + OCFS2_JTR_DB, + OCFS2_JTR_XB, + OCFS2_JTR_DQ, + OCFS2_JTR_DR, + OCFS2_JTR_DL, + OCFS2_JTR_NONE /* This must be the last entry */ +}; + +#define OCFS2_JOURNAL_TRIGGER_COUNT OCFS2_JTR_NONE + +void ocfs2_initialize_journal_triggers(struct super_block *sb, + struct ocfs2_triggers triggers[]); + +enum ocfs2_recovery_state { + OCFS2_REC_ENABLED = 0, + OCFS2_REC_QUOTA_WANT_DISABLE, + /* + * Must be OCFS2_REC_QUOTA_WANT_DISABLE + 1 for + * ocfs2_recovery_disable_quota() to work. + */ + OCFS2_REC_QUOTA_DISABLED, + OCFS2_REC_WANT_DISABLE, + /* + * Must be OCFS2_REC_WANT_DISABLE + 1 for ocfs2_recovery_exit() to work + */ + OCFS2_REC_DISABLED, +}; + +struct ocfs2_journal; +struct ocfs2_slot_info; +struct ocfs2_recovery_map; +struct ocfs2_replay_map; +struct ocfs2_quota_recovery; +struct ocfs2_super +{ + struct task_struct *commit_task; + struct super_block *sb; + struct inode *root_inode; + struct inode *sys_root_inode; + struct inode *global_system_inodes[NUM_GLOBAL_SYSTEM_INODES]; + struct inode **local_system_inodes; + + struct ocfs2_slot_info *slot_info; + + u32 *slot_recovery_generations; + + spinlock_t node_map_lock; + + u64 root_blkno; + u64 system_dir_blkno; + u64 bitmap_blkno; + u32 bitmap_cpg; + char *uuid_str; + u32 uuid_hash; + u8 *vol_label; + u64 first_cluster_group_blkno; + u32 fs_generation; + + u32 s_feature_compat; + u32 s_feature_incompat; + u32 s_feature_ro_compat; + + /* Protects s_next_generation, osb_flags and s_inode_steal_slot. + * Could protect more on osb as it's very short lived. + */ + spinlock_t osb_lock; + u32 s_next_generation; + unsigned long osb_flags; + u16 s_inode_steal_slot; + u16 s_meta_steal_slot; + atomic_t s_num_inodes_stolen; + atomic_t s_num_meta_stolen; + + unsigned long s_mount_opt; + unsigned int s_atime_quantum; + + unsigned int max_slots; + unsigned int node_num; + int slot_num; + int preferred_slot; + int s_sectsize_bits; + int s_clustersize; + int s_clustersize_bits; + unsigned int s_xattr_inline_size; + + atomic_t vol_state; + struct mutex recovery_lock; + struct ocfs2_recovery_map *recovery_map; + struct ocfs2_replay_map *replay_map; + struct task_struct *recovery_thread_task; + enum ocfs2_recovery_state recovery_state; + wait_queue_head_t checkpoint_event; + struct ocfs2_journal *journal; + unsigned long osb_commit_interval; + + /* Journal triggers for checksum */ + struct ocfs2_triggers s_journal_triggers[OCFS2_JOURNAL_TRIGGER_COUNT]; + + struct delayed_work la_enable_wq; + + /* + * Must hold local alloc i_rwsem and osb->osb_lock to change + * local_alloc_bits. Reads can be done under either lock. + */ + unsigned int local_alloc_bits; + unsigned int local_alloc_default_bits; + /* osb_clusters_at_boot can become stale! Do not trust it to + * be up to date. */ + unsigned int osb_clusters_at_boot; + + enum ocfs2_local_alloc_state local_alloc_state; /* protected + * by osb_lock */ + + struct buffer_head *local_alloc_bh; + + u64 la_last_gd; + + struct ocfs2_reservation_map osb_la_resmap; + + unsigned int osb_resv_level; + unsigned int osb_dir_resv_level; + + /* Next two fields are for local node slot recovery during + * mount. */ + struct ocfs2_dinode *local_alloc_copy; + struct ocfs2_quota_recovery *quota_rec; + + struct ocfs2_blockcheck_stats osb_ecc_stats; + struct ocfs2_alloc_stats alloc_stats; + char dev_str[20]; /* "major,minor" of the device */ + + u8 osb_stackflags; + + char osb_cluster_stack[OCFS2_STACK_LABEL_LEN + 1]; + char osb_cluster_name[OCFS2_CLUSTER_NAME_LEN + 1]; + struct ocfs2_cluster_connection *cconn; + struct ocfs2_lock_res osb_super_lockres; + struct ocfs2_lock_res osb_rename_lockres; + struct ocfs2_lock_res osb_nfs_sync_lockres; + struct rw_semaphore nfs_sync_rwlock; + struct ocfs2_lock_res osb_trim_fs_lockres; + struct mutex obs_trim_fs_mutex; + struct ocfs2_dlm_debug *osb_dlm_debug; + + struct dentry *osb_debug_root; + + wait_queue_head_t recovery_event; + + spinlock_t dc_task_lock; + struct task_struct *dc_task; + wait_queue_head_t dc_event; + unsigned long dc_wake_sequence; + unsigned long dc_work_sequence; + + /* + * Any thread can add locks to the list, but the downconvert + * thread is the only one allowed to remove locks. Any change + * to this rule requires updating + * ocfs2_downconvert_thread_do_work(). + */ + struct list_head blocked_lock_list; + unsigned long blocked_lock_count; + + /* List of dquot structures to drop last reference to */ + struct llist_head dquot_drop_list; + struct work_struct dquot_drop_work; + + wait_queue_head_t osb_mount_event; + + /* Truncate log info */ + struct inode *osb_tl_inode; + struct buffer_head *osb_tl_bh; + struct delayed_work osb_truncate_log_wq; + atomic_t osb_tl_disable; + /* + * How many clusters in our truncate log. + * It must be protected by osb_tl_inode->i_rwsem. + */ + unsigned int truncated_clusters; + + struct ocfs2_node_map osb_recovering_orphan_dirs; + unsigned int *osb_orphan_wipes; + wait_queue_head_t osb_wipe_event; + + struct ocfs2_orphan_scan osb_orphan_scan; + + /* used to protect metaecc calculation check of xattr. */ + spinlock_t osb_xattr_lock; + + unsigned int osb_dx_mask; + u32 osb_dx_seed[4]; + + /* the group we used to allocate inodes. */ + u64 osb_inode_alloc_group; + + /* rb tree root for refcount lock. */ + struct rb_root osb_rf_lock_tree; + struct ocfs2_refcount_tree *osb_ref_tree_lru; + + /* + * OCFS2 needs to schedule several different types of work which + * require cluster locking, disk I/O, recovery waits, etc. Since these + * types of work tend to be heavy we avoid using the kernel events + * workqueue and schedule on our own. + */ + struct workqueue_struct *ocfs2_wq; + + /* sysfs directory per partition */ + struct kset *osb_dev_kset; + + /* file check related stuff */ + struct ocfs2_filecheck_sysfs_entry osb_fc_ent; +}; + +#define OCFS2_SB(sb) ((struct ocfs2_super *)(sb)->s_fs_info) + +/* Useful typedef for passing around journal access functions */ +typedef int (*ocfs2_journal_access_func)(handle_t *handle, + struct ocfs2_caching_info *ci, + struct buffer_head *bh, int type); + +static inline int ocfs2_should_order_data(struct inode *inode) +{ + if (!S_ISREG(inode->i_mode)) + return 0; + if (OCFS2_SB(inode->i_sb)->s_mount_opt & OCFS2_MOUNT_DATA_WRITEBACK) + return 0; + return 1; +} + +static inline int ocfs2_sparse_alloc(struct ocfs2_super *osb) +{ + if (osb->s_feature_incompat & OCFS2_FEATURE_INCOMPAT_SPARSE_ALLOC) + return 1; + return 0; +} + +static inline int ocfs2_writes_unwritten_extents(struct ocfs2_super *osb) +{ + /* + * Support for sparse files is a pre-requisite + */ + if (!ocfs2_sparse_alloc(osb)) + return 0; + + if (osb->s_feature_ro_compat & OCFS2_FEATURE_RO_COMPAT_UNWRITTEN) + return 1; + return 0; +} + +static inline int ocfs2_supports_append_dio(struct ocfs2_super *osb) +{ + if (osb->s_feature_incompat & OCFS2_FEATURE_INCOMPAT_APPEND_DIO) + return 1; + return 0; +} + + +static inline int ocfs2_supports_inline_data(struct ocfs2_super *osb) +{ + if (osb->s_feature_incompat & OCFS2_FEATURE_INCOMPAT_INLINE_DATA) + return 1; + return 0; +} + +static inline int ocfs2_supports_xattr(struct ocfs2_super *osb) +{ + if (osb->s_feature_incompat & OCFS2_FEATURE_INCOMPAT_XATTR) + return 1; + return 0; +} + +static inline int ocfs2_meta_ecc(struct ocfs2_super *osb) +{ + if (osb->s_feature_incompat & OCFS2_FEATURE_INCOMPAT_META_ECC) + return 1; + return 0; +} + +static inline int ocfs2_supports_indexed_dirs(struct ocfs2_super *osb) +{ + if (osb->s_feature_incompat & OCFS2_FEATURE_INCOMPAT_INDEXED_DIRS) + return 1; + return 0; +} + +static inline int ocfs2_supports_discontig_bg(struct ocfs2_super *osb) +{ + if (osb->s_feature_incompat & OCFS2_FEATURE_INCOMPAT_DISCONTIG_BG) + return 1; + return 0; +} + +static inline unsigned int ocfs2_link_max(struct ocfs2_super *osb) +{ + if (ocfs2_supports_indexed_dirs(osb)) + return OCFS2_DX_LINK_MAX; + return OCFS2_LINK_MAX; +} + +static inline unsigned int ocfs2_read_links_count(struct ocfs2_dinode *di) +{ + u32 nlink = le16_to_cpu(di->i_links_count); + u32 hi = le16_to_cpu(di->i_links_count_hi); + + nlink |= (hi << OCFS2_LINKS_HI_SHIFT); + + return nlink; +} + +static inline void ocfs2_set_links_count(struct ocfs2_dinode *di, u32 nlink) +{ + u16 lo, hi; + + lo = nlink; + hi = nlink >> OCFS2_LINKS_HI_SHIFT; + + di->i_links_count = cpu_to_le16(lo); + di->i_links_count_hi = cpu_to_le16(hi); +} + +static inline void ocfs2_add_links_count(struct ocfs2_dinode *di, int n) +{ + u32 links = ocfs2_read_links_count(di); + + links += n; + + ocfs2_set_links_count(di, links); +} + +static inline int ocfs2_refcount_tree(struct ocfs2_super *osb) +{ + if (osb->s_feature_incompat & OCFS2_FEATURE_INCOMPAT_REFCOUNT_TREE) + return 1; + return 0; +} + +/* set / clear functions because cluster events can make these happen + * in parallel so we want the transitions to be atomic. this also + * means that any future flags osb_flags must be protected by spinlock + * too! */ +static inline void ocfs2_set_osb_flag(struct ocfs2_super *osb, + unsigned long flag) +{ + spin_lock(&osb->osb_lock); + osb->osb_flags |= flag; + spin_unlock(&osb->osb_lock); +} + +static inline void ocfs2_set_ro_flag(struct ocfs2_super *osb, + int hard) +{ + spin_lock(&osb->osb_lock); + osb->osb_flags &= ~(OCFS2_OSB_SOFT_RO|OCFS2_OSB_HARD_RO); + if (hard) + osb->osb_flags |= OCFS2_OSB_HARD_RO; + else + osb->osb_flags |= OCFS2_OSB_SOFT_RO; + spin_unlock(&osb->osb_lock); +} + +static inline int ocfs2_is_hard_readonly(struct ocfs2_super *osb) +{ + int ret; + + spin_lock(&osb->osb_lock); + ret = osb->osb_flags & OCFS2_OSB_HARD_RO; + spin_unlock(&osb->osb_lock); + + return ret; +} + +static inline int ocfs2_is_soft_readonly(struct ocfs2_super *osb) +{ + int ret; + + spin_lock(&osb->osb_lock); + ret = osb->osb_flags & OCFS2_OSB_SOFT_RO; + spin_unlock(&osb->osb_lock); + + return ret; +} + +static inline int ocfs2_is_readonly(struct ocfs2_super *osb) +{ + int ret; + spin_lock(&osb->osb_lock); + ret = osb->osb_flags & (OCFS2_OSB_SOFT_RO | OCFS2_OSB_HARD_RO); + spin_unlock(&osb->osb_lock); + + return ret; +} + +static inline int ocfs2_emergency_state(struct ocfs2_super *osb) +{ + if (ocfs2_is_readonly(osb)) + return -EROFS; + + return 0; +} + +static inline int ocfs2_clusterinfo_valid(struct ocfs2_super *osb) +{ + return (osb->s_feature_incompat & + (OCFS2_FEATURE_INCOMPAT_USERSPACE_STACK | + OCFS2_FEATURE_INCOMPAT_CLUSTERINFO)); +} + +static inline int ocfs2_userspace_stack(struct ocfs2_super *osb) +{ + if (ocfs2_clusterinfo_valid(osb) && + memcmp(osb->osb_cluster_stack, OCFS2_CLASSIC_CLUSTER_STACK, + OCFS2_STACK_LABEL_LEN)) + return 1; + return 0; +} + +static inline int ocfs2_o2cb_stack(struct ocfs2_super *osb) +{ + if (ocfs2_clusterinfo_valid(osb) && + !memcmp(osb->osb_cluster_stack, OCFS2_CLASSIC_CLUSTER_STACK, + OCFS2_STACK_LABEL_LEN)) + return 1; + return 0; +} + +static inline int ocfs2_cluster_o2cb_global_heartbeat(struct ocfs2_super *osb) +{ + return ocfs2_o2cb_stack(osb) && + (osb->osb_stackflags & OCFS2_CLUSTER_O2CB_GLOBAL_HEARTBEAT); +} + +static inline int ocfs2_mount_local(struct ocfs2_super *osb) +{ + return (osb->s_feature_incompat & OCFS2_FEATURE_INCOMPAT_LOCAL_MOUNT); +} + +static inline int ocfs2_uses_extended_slot_map(struct ocfs2_super *osb) +{ + return (osb->s_feature_incompat & + OCFS2_FEATURE_INCOMPAT_EXTENDED_SLOT_MAP); +} + + +#define OCFS2_IS_VALID_DINODE(ptr) \ + (!strcmp((ptr)->i_signature, OCFS2_INODE_SIGNATURE)) + +#define OCFS2_IS_VALID_EXTENT_BLOCK(ptr) \ + (!strcmp((ptr)->h_signature, OCFS2_EXTENT_BLOCK_SIGNATURE)) + +#define OCFS2_IS_VALID_GROUP_DESC(ptr) \ + (!strcmp((ptr)->bg_signature, OCFS2_GROUP_DESC_SIGNATURE)) + + +#define OCFS2_IS_VALID_XATTR_BLOCK(ptr) \ + (!strcmp((ptr)->xb_signature, OCFS2_XATTR_BLOCK_SIGNATURE)) + +#define OCFS2_IS_VALID_DIR_TRAILER(ptr) \ + (!strcmp((ptr)->db_signature, OCFS2_DIR_TRAILER_SIGNATURE)) + +#define OCFS2_IS_VALID_DX_ROOT(ptr) \ + (!strcmp((ptr)->dr_signature, OCFS2_DX_ROOT_SIGNATURE)) + +#define OCFS2_IS_VALID_DX_LEAF(ptr) \ + (!strcmp((ptr)->dl_signature, OCFS2_DX_LEAF_SIGNATURE)) + +#define OCFS2_IS_VALID_REFCOUNT_BLOCK(ptr) \ + (!strcmp((ptr)->rf_signature, OCFS2_REFCOUNT_BLOCK_SIGNATURE)) + +static inline unsigned long ino_from_blkno(struct super_block *sb, + u64 blkno) +{ + return (unsigned long)(blkno & (u64)ULONG_MAX); +} + +static inline u64 ocfs2_clusters_to_blocks(struct super_block *sb, + u32 clusters) +{ + int c_to_b_bits = OCFS2_SB(sb)->s_clustersize_bits - + sb->s_blocksize_bits; + + return (u64)clusters << c_to_b_bits; +} + +static inline u32 ocfs2_clusters_for_blocks(struct super_block *sb, + u64 blocks) +{ + int b_to_c_bits = OCFS2_SB(sb)->s_clustersize_bits - + sb->s_blocksize_bits; + + blocks += (1 << b_to_c_bits) - 1; + return (u32)(blocks >> b_to_c_bits); +} + +static inline u32 ocfs2_blocks_to_clusters(struct super_block *sb, + u64 blocks) +{ + int b_to_c_bits = OCFS2_SB(sb)->s_clustersize_bits - + sb->s_blocksize_bits; + + return (u32)(blocks >> b_to_c_bits); +} + +static inline unsigned int ocfs2_clusters_for_bytes(struct super_block *sb, + u64 bytes) +{ + int cl_bits = OCFS2_SB(sb)->s_clustersize_bits; + unsigned int clusters; + + bytes += OCFS2_SB(sb)->s_clustersize - 1; + /* OCFS2 just cannot have enough clusters to overflow this */ + clusters = (unsigned int)(bytes >> cl_bits); + + return clusters; +} + +static inline unsigned int ocfs2_bytes_to_clusters(struct super_block *sb, + u64 bytes) +{ + int cl_bits = OCFS2_SB(sb)->s_clustersize_bits; + unsigned int clusters; + + clusters = (unsigned int)(bytes >> cl_bits); + return clusters; +} + +static inline u64 ocfs2_blocks_for_bytes(struct super_block *sb, + u64 bytes) +{ + bytes += sb->s_blocksize - 1; + return bytes >> sb->s_blocksize_bits; +} + +static inline u64 ocfs2_clusters_to_bytes(struct super_block *sb, + u32 clusters) +{ + return (u64)clusters << OCFS2_SB(sb)->s_clustersize_bits; +} + +static inline u64 ocfs2_block_to_cluster_start(struct super_block *sb, + u64 blocks) +{ + int bits = OCFS2_SB(sb)->s_clustersize_bits - sb->s_blocksize_bits; + unsigned int clusters; + + clusters = ocfs2_blocks_to_clusters(sb, blocks); + return (u64)clusters << bits; +} + +static inline u64 ocfs2_align_bytes_to_clusters(struct super_block *sb, + u64 bytes) +{ + int cl_bits = OCFS2_SB(sb)->s_clustersize_bits; + unsigned int clusters; + + clusters = ocfs2_clusters_for_bytes(sb, bytes); + return (u64)clusters << cl_bits; +} + +static inline u64 ocfs2_align_bytes_to_blocks(struct super_block *sb, + u64 bytes) +{ + u64 blocks; + + blocks = ocfs2_blocks_for_bytes(sb, bytes); + return blocks << sb->s_blocksize_bits; +} + +static inline unsigned long ocfs2_align_bytes_to_sectors(u64 bytes) +{ + return (unsigned long)((bytes + 511) >> 9); +} + +static inline unsigned int ocfs2_page_index_to_clusters(struct super_block *sb, + unsigned long pg_index) +{ + u32 clusters = pg_index; + unsigned int cbits = OCFS2_SB(sb)->s_clustersize_bits; + + if (unlikely(PAGE_SHIFT > cbits)) + clusters = pg_index << (PAGE_SHIFT - cbits); + else if (PAGE_SHIFT < cbits) + clusters = pg_index >> (cbits - PAGE_SHIFT); + + return clusters; +} + +/* + * Find the 1st page index which covers the given clusters. + */ +static inline pgoff_t ocfs2_align_clusters_to_page_index(struct super_block *sb, + u32 clusters) +{ + unsigned int cbits = OCFS2_SB(sb)->s_clustersize_bits; + pgoff_t index = clusters; + + if (PAGE_SHIFT > cbits) { + index = (pgoff_t)clusters >> (PAGE_SHIFT - cbits); + } else if (PAGE_SHIFT < cbits) { + index = (pgoff_t)clusters << (cbits - PAGE_SHIFT); + } + + return index; +} + +static inline unsigned int ocfs2_pages_per_cluster(struct super_block *sb) +{ + unsigned int cbits = OCFS2_SB(sb)->s_clustersize_bits; + unsigned int pages_per_cluster = 1; + + if (PAGE_SHIFT < cbits) + pages_per_cluster = 1 << (cbits - PAGE_SHIFT); + + return pages_per_cluster; +} + +static inline unsigned int ocfs2_megabytes_to_clusters(struct super_block *sb, + unsigned int megs) +{ + BUILD_BUG_ON(OCFS2_MAX_CLUSTERSIZE > 1048576); + + return megs << (20 - OCFS2_SB(sb)->s_clustersize_bits); +} + +static inline unsigned int ocfs2_clusters_to_megabytes(struct super_block *sb, + unsigned int clusters) +{ + return clusters >> (20 - OCFS2_SB(sb)->s_clustersize_bits); +} + +static inline void _ocfs2_set_bit(unsigned int bit, unsigned long *bitmap) +{ + __set_bit_le(bit, bitmap); +} +#define ocfs2_set_bit(bit, addr) _ocfs2_set_bit((bit), (unsigned long *)(addr)) + +static inline void _ocfs2_clear_bit(unsigned int bit, unsigned long *bitmap) +{ + __clear_bit_le(bit, bitmap); +} +#define ocfs2_clear_bit(bit, addr) _ocfs2_clear_bit((bit), (unsigned long *)(addr)) + +#define ocfs2_test_bit test_bit_le +#define ocfs2_find_next_zero_bit find_next_zero_bit_le +#define ocfs2_find_next_bit find_next_bit_le + +static inline void *correct_addr_and_bit_unaligned(int *bit, void *addr) +{ +#if BITS_PER_LONG == 64 + *bit += ((unsigned long) addr & 7UL) << 3; + addr = (void *) ((unsigned long) addr & ~7UL); +#elif BITS_PER_LONG == 32 + *bit += ((unsigned long) addr & 3UL) << 3; + addr = (void *) ((unsigned long) addr & ~3UL); +#else +#error "how many bits you are?!" +#endif + return addr; +} + +static inline void ocfs2_set_bit_unaligned(int bit, void *bitmap) +{ + bitmap = correct_addr_and_bit_unaligned(&bit, bitmap); + ocfs2_set_bit(bit, bitmap); +} + +static inline void ocfs2_clear_bit_unaligned(int bit, void *bitmap) +{ + bitmap = correct_addr_and_bit_unaligned(&bit, bitmap); + ocfs2_clear_bit(bit, bitmap); +} + +static inline int ocfs2_test_bit_unaligned(int bit, void *bitmap) +{ + bitmap = correct_addr_and_bit_unaligned(&bit, bitmap); + return ocfs2_test_bit(bit, bitmap); +} + +static inline int ocfs2_find_next_zero_bit_unaligned(void *bitmap, int max, + int start) +{ + int fix = 0, ret, tmpmax; + bitmap = correct_addr_and_bit_unaligned(&fix, bitmap); + tmpmax = max + fix; + start += fix; + + ret = ocfs2_find_next_zero_bit(bitmap, tmpmax, start) - fix; + if (ret > max) + return max; + return ret; +} + +#endif /* OCFS2_H */ + diff --git a/fs/ocfs2/ocfs2_fs.h b/fs/ocfs2/ocfs2_fs.h new file mode 100644 index 000000000..c501eb3cd --- /dev/null +++ b/fs/ocfs2/ocfs2_fs.h @@ -0,0 +1,1616 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * ocfs2_fs.h + * + * On-disk structures for OCFS2. + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#ifndef _OCFS2_FS_H +#define _OCFS2_FS_H + +#include + +/* Version */ +#define OCFS2_MAJOR_REV_LEVEL 0 +#define OCFS2_MINOR_REV_LEVEL 90 + +/* + * An OCFS2 volume starts this way: + * Sector 0: Valid ocfs1_vol_disk_hdr that cleanly fails to mount OCFS. + * Sector 1: Valid ocfs1_vol_label that cleanly fails to mount OCFS. + * Block OCFS2_SUPER_BLOCK_BLKNO: OCFS2 superblock. + * + * All other structures are found from the superblock information. + * + * OCFS2_SUPER_BLOCK_BLKNO is in blocks, not sectors. eg, for a + * blocksize of 2K, it is 4096 bytes into disk. + */ +#define OCFS2_SUPER_BLOCK_BLKNO 2 + +/* + * Cluster size limits. The maximum is kept arbitrarily at 1 MB, and could + * grow if needed. + */ +#define OCFS2_MIN_CLUSTERSIZE 4096 +#define OCFS2_MAX_CLUSTERSIZE 1048576 + +/* + * Blocks cannot be bigger than clusters, so the maximum blocksize is the + * minimum cluster size. + */ +#define OCFS2_MIN_BLOCKSIZE 512 +#define OCFS2_MAX_BLOCKSIZE OCFS2_MIN_CLUSTERSIZE + +/* Object signatures */ +#define OCFS2_SUPER_BLOCK_SIGNATURE "OCFSV2" +#define OCFS2_INODE_SIGNATURE "INODE01" +#define OCFS2_EXTENT_BLOCK_SIGNATURE "EXBLK01" +#define OCFS2_GROUP_DESC_SIGNATURE "GROUP01" +#define OCFS2_XATTR_BLOCK_SIGNATURE "XATTR01" +#define OCFS2_DIR_TRAILER_SIGNATURE "DIRTRL1" +#define OCFS2_DX_ROOT_SIGNATURE "DXDIR01" +#define OCFS2_DX_LEAF_SIGNATURE "DXLEAF1" +#define OCFS2_REFCOUNT_BLOCK_SIGNATURE "REFCNT1" + +/* Compatibility flags */ +#define OCFS2_HAS_COMPAT_FEATURE(sb,mask) \ + ( OCFS2_SB(sb)->s_feature_compat & (mask) ) +#define OCFS2_HAS_RO_COMPAT_FEATURE(sb,mask) \ + ( OCFS2_SB(sb)->s_feature_ro_compat & (mask) ) +#define OCFS2_HAS_INCOMPAT_FEATURE(sb,mask) \ + ( OCFS2_SB(sb)->s_feature_incompat & (mask) ) +#define OCFS2_SET_COMPAT_FEATURE(sb,mask) \ + OCFS2_SB(sb)->s_feature_compat |= (mask) +#define OCFS2_SET_RO_COMPAT_FEATURE(sb,mask) \ + OCFS2_SB(sb)->s_feature_ro_compat |= (mask) +#define OCFS2_SET_INCOMPAT_FEATURE(sb,mask) \ + OCFS2_SB(sb)->s_feature_incompat |= (mask) +#define OCFS2_CLEAR_COMPAT_FEATURE(sb,mask) \ + OCFS2_SB(sb)->s_feature_compat &= ~(mask) +#define OCFS2_CLEAR_RO_COMPAT_FEATURE(sb,mask) \ + OCFS2_SB(sb)->s_feature_ro_compat &= ~(mask) +#define OCFS2_CLEAR_INCOMPAT_FEATURE(sb,mask) \ + OCFS2_SB(sb)->s_feature_incompat &= ~(mask) + +#define OCFS2_FEATURE_COMPAT_SUPP (OCFS2_FEATURE_COMPAT_BACKUP_SB \ + | OCFS2_FEATURE_COMPAT_JBD2_SB) +#define OCFS2_FEATURE_INCOMPAT_SUPP (OCFS2_FEATURE_INCOMPAT_LOCAL_MOUNT \ + | OCFS2_FEATURE_INCOMPAT_SPARSE_ALLOC \ + | OCFS2_FEATURE_INCOMPAT_INLINE_DATA \ + | OCFS2_FEATURE_INCOMPAT_EXTENDED_SLOT_MAP \ + | OCFS2_FEATURE_INCOMPAT_USERSPACE_STACK \ + | OCFS2_FEATURE_INCOMPAT_XATTR \ + | OCFS2_FEATURE_INCOMPAT_META_ECC \ + | OCFS2_FEATURE_INCOMPAT_INDEXED_DIRS \ + | OCFS2_FEATURE_INCOMPAT_REFCOUNT_TREE \ + | OCFS2_FEATURE_INCOMPAT_DISCONTIG_BG \ + | OCFS2_FEATURE_INCOMPAT_CLUSTERINFO \ + | OCFS2_FEATURE_INCOMPAT_APPEND_DIO) +#define OCFS2_FEATURE_RO_COMPAT_SUPP (OCFS2_FEATURE_RO_COMPAT_UNWRITTEN \ + | OCFS2_FEATURE_RO_COMPAT_USRQUOTA \ + | OCFS2_FEATURE_RO_COMPAT_GRPQUOTA) + +/* + * Heartbeat-only devices are missing journals and other files. The + * filesystem driver can't load them, but the library can. Never put + * this in OCFS2_FEATURE_INCOMPAT_SUPP, *ever*. + */ +#define OCFS2_FEATURE_INCOMPAT_HEARTBEAT_DEV 0x0002 + +/* + * tunefs sets this incompat flag before starting the resize and clears it + * at the end. This flag protects users from inadvertently mounting the fs + * after an aborted run without fsck-ing. + */ +#define OCFS2_FEATURE_INCOMPAT_RESIZE_INPROG 0x0004 + +/* Used to denote a non-clustered volume */ +#define OCFS2_FEATURE_INCOMPAT_LOCAL_MOUNT 0x0008 + +/* Support for sparse allocation in b-trees */ +#define OCFS2_FEATURE_INCOMPAT_SPARSE_ALLOC 0x0010 + +/* + * Tunefs sets this incompat flag before starting an operation which + * would require cleanup on abort. This is done to protect users from + * inadvertently mounting the fs after an aborted run without + * fsck-ing. + * + * s_tunefs_flags on the super block describes precisely which + * operations were in progress. + */ +#define OCFS2_FEATURE_INCOMPAT_TUNEFS_INPROG 0x0020 + +/* Support for data packed into inode blocks */ +#define OCFS2_FEATURE_INCOMPAT_INLINE_DATA 0x0040 + +/* + * Support for alternate, userspace cluster stacks. If set, the superblock + * field s_cluster_info contains a tag for the alternate stack in use as + * well as the name of the cluster being joined. + * mount.ocfs2 must pass in a matching stack name. + * + * If not set, the classic stack will be used. This is compatible with + * all older versions. + */ +#define OCFS2_FEATURE_INCOMPAT_USERSPACE_STACK 0x0080 + +/* Support for the extended slot map */ +#define OCFS2_FEATURE_INCOMPAT_EXTENDED_SLOT_MAP 0x100 + +/* Support for extended attributes */ +#define OCFS2_FEATURE_INCOMPAT_XATTR 0x0200 + +/* Support for indexed directories */ +#define OCFS2_FEATURE_INCOMPAT_INDEXED_DIRS 0x0400 + +/* Metadata checksum and error correction */ +#define OCFS2_FEATURE_INCOMPAT_META_ECC 0x0800 + +/* Refcount tree support */ +#define OCFS2_FEATURE_INCOMPAT_REFCOUNT_TREE 0x1000 + +/* Discontiguous block groups */ +#define OCFS2_FEATURE_INCOMPAT_DISCONTIG_BG 0x2000 + +/* + * Incompat bit to indicate usable clusterinfo with stackflags for all + * cluster stacks (userspace adnd o2cb). If this bit is set, + * INCOMPAT_USERSPACE_STACK becomes superfluous and thus should not be set. + */ +#define OCFS2_FEATURE_INCOMPAT_CLUSTERINFO 0x4000 + +/* + * Append Direct IO support + */ +#define OCFS2_FEATURE_INCOMPAT_APPEND_DIO 0x8000 + +/* + * backup superblock flag is used to indicate that this volume + * has backup superblocks. + */ +#define OCFS2_FEATURE_COMPAT_BACKUP_SB 0x0001 + +/* + * The filesystem will correctly handle journal feature bits. + */ +#define OCFS2_FEATURE_COMPAT_JBD2_SB 0x0002 + +/* + * Unwritten extents support. + */ +#define OCFS2_FEATURE_RO_COMPAT_UNWRITTEN 0x0001 + +/* + * Maintain quota information for this filesystem + */ +#define OCFS2_FEATURE_RO_COMPAT_USRQUOTA 0x0002 +#define OCFS2_FEATURE_RO_COMPAT_GRPQUOTA 0x0004 + + +/* The byte offset of the first backup block will be 1G. + * The following will be 4G, 16G, 64G, 256G and 1T. + */ +#define OCFS2_BACKUP_SB_START 1 << 30 + +/* the max backup superblock nums */ +#define OCFS2_MAX_BACKUP_SUPERBLOCKS 6 + +/* + * Flags on ocfs2_super_block.s_tunefs_flags + */ +#define OCFS2_TUNEFS_INPROG_REMOVE_SLOT 0x0001 /* Removing slots */ + +/* + * Flags on ocfs2_dinode.i_flags + */ +#define OCFS2_VALID_FL (0x00000001) /* Inode is valid */ +#define OCFS2_UNUSED2_FL (0x00000002) +#define OCFS2_ORPHANED_FL (0x00000004) /* On the orphan list */ +#define OCFS2_UNUSED3_FL (0x00000008) +/* System inode flags */ +#define OCFS2_SYSTEM_FL (0x00000010) /* System inode */ +#define OCFS2_SUPER_BLOCK_FL (0x00000020) /* Super block */ +#define OCFS2_LOCAL_ALLOC_FL (0x00000040) /* Slot local alloc bitmap */ +#define OCFS2_BITMAP_FL (0x00000080) /* Allocation bitmap */ +#define OCFS2_JOURNAL_FL (0x00000100) /* Slot local journal */ +#define OCFS2_HEARTBEAT_FL (0x00000200) /* Heartbeat area */ +#define OCFS2_CHAIN_FL (0x00000400) /* Chain allocator */ +#define OCFS2_DEALLOC_FL (0x00000800) /* Truncate log */ +#define OCFS2_QUOTA_FL (0x00001000) /* Quota file */ +#define OCFS2_DIO_ORPHANED_FL (0X00002000) /* On the orphan list especially + * for dio */ + +/* + * Flags on ocfs2_dinode.i_dyn_features + * + * These can change much more often than i_flags. When adding flags, + * keep in mind that i_dyn_features is only 16 bits wide. + */ +#define OCFS2_INLINE_DATA_FL (0x0001) /* Data stored in inode block */ +#define OCFS2_HAS_XATTR_FL (0x0002) +#define OCFS2_INLINE_XATTR_FL (0x0004) +#define OCFS2_INDEXED_DIR_FL (0x0008) +#define OCFS2_HAS_REFCOUNT_FL (0x0010) + +/* Inode attributes, keep in sync with EXT2 */ +#define OCFS2_SECRM_FL FS_SECRM_FL /* Secure deletion */ +#define OCFS2_UNRM_FL FS_UNRM_FL /* Undelete */ +#define OCFS2_COMPR_FL FS_COMPR_FL /* Compress file */ +#define OCFS2_SYNC_FL FS_SYNC_FL /* Synchronous updates */ +#define OCFS2_IMMUTABLE_FL FS_IMMUTABLE_FL /* Immutable file */ +#define OCFS2_APPEND_FL FS_APPEND_FL /* writes to file may only append */ +#define OCFS2_NODUMP_FL FS_NODUMP_FL /* do not dump file */ +#define OCFS2_NOATIME_FL FS_NOATIME_FL /* do not update atime */ +/* Reserved for compression usage... */ +#define OCFS2_DIRTY_FL FS_DIRTY_FL +#define OCFS2_COMPRBLK_FL FS_COMPRBLK_FL /* One or more compressed clusters */ +#define OCFS2_NOCOMP_FL FS_NOCOMP_FL /* Don't compress */ +#define OCFS2_ECOMPR_FL FS_ECOMPR_FL /* Compression error */ +/* End compression flags --- maybe not all used */ +#define OCFS2_BTREE_FL FS_BTREE_FL /* btree format dir */ +#define OCFS2_INDEX_FL FS_INDEX_FL /* hash-indexed directory */ +#define OCFS2_IMAGIC_FL FS_IMAGIC_FL /* AFS directory */ +#define OCFS2_JOURNAL_DATA_FL FS_JOURNAL_DATA_FL /* Reserved for ext3 */ +#define OCFS2_NOTAIL_FL FS_NOTAIL_FL /* file tail should not be merged */ +#define OCFS2_DIRSYNC_FL FS_DIRSYNC_FL /* dirsync behaviour (directories only) */ +#define OCFS2_TOPDIR_FL FS_TOPDIR_FL /* Top of directory hierarchies*/ +#define OCFS2_RESERVED_FL FS_RESERVED_FL /* reserved for ext2 lib */ + +#define OCFS2_FL_VISIBLE FS_FL_USER_VISIBLE /* User visible flags */ +#define OCFS2_FL_MODIFIABLE FS_FL_USER_MODIFIABLE /* User modifiable flags */ + +/* + * Extent record flags (e_node.leaf.flags) + */ +#define OCFS2_EXT_UNWRITTEN (0x01) /* Extent is allocated but + * unwritten */ +#define OCFS2_EXT_REFCOUNTED (0x02) /* Extent is reference + * counted in an associated + * refcount tree */ + +/* + * Journal Flags (ocfs2_dinode.id1.journal1.i_flags) + */ +#define OCFS2_JOURNAL_DIRTY_FL (0x00000001) /* Journal needs recovery */ + +/* + * superblock s_state flags + */ +#define OCFS2_ERROR_FS (0x00000001) /* FS saw errors */ + +/* Limit of space in ocfs2_dir_entry */ +#define OCFS2_MAX_FILENAME_LEN 255 + +/* Maximum slots on an ocfs2 file system */ +#define OCFS2_MAX_SLOTS 255 + +/* Slot map indicator for an empty slot */ +#define OCFS2_INVALID_SLOT ((u16)-1) + +#define OCFS2_VOL_UUID_LEN 16 +#define OCFS2_MAX_VOL_LABEL_LEN 64 + +/* The cluster stack fields */ +#define OCFS2_STACK_LABEL_LEN 4 +#define OCFS2_CLUSTER_NAME_LEN 16 + +/* Classic (historically speaking) cluster stack */ +#define OCFS2_CLASSIC_CLUSTER_STACK "o2cb" + +/* Journal limits (in bytes) */ +#define OCFS2_MIN_JOURNAL_SIZE (4 * 1024 * 1024) + +/* + * Inline extended attribute size (in bytes) + * The value chosen should be aligned to 16 byte boundaries. + */ +#define OCFS2_MIN_XATTR_INLINE_SIZE 256 + +/* + * Cluster info flags (ocfs2_cluster_info.ci_stackflags) + */ +#define OCFS2_CLUSTER_O2CB_GLOBAL_HEARTBEAT (0x01) + +struct ocfs2_system_inode_info { + char *si_name; + int si_iflags; + int si_mode; +}; + +/* System file index */ +enum { + BAD_BLOCK_SYSTEM_INODE = 0, + GLOBAL_INODE_ALLOC_SYSTEM_INODE, +#define OCFS2_FIRST_ONLINE_SYSTEM_INODE GLOBAL_INODE_ALLOC_SYSTEM_INODE + SLOT_MAP_SYSTEM_INODE, + HEARTBEAT_SYSTEM_INODE, + GLOBAL_BITMAP_SYSTEM_INODE, + USER_QUOTA_SYSTEM_INODE, + GROUP_QUOTA_SYSTEM_INODE, +#define OCFS2_LAST_GLOBAL_SYSTEM_INODE GROUP_QUOTA_SYSTEM_INODE +#define OCFS2_FIRST_LOCAL_SYSTEM_INODE ORPHAN_DIR_SYSTEM_INODE + ORPHAN_DIR_SYSTEM_INODE, + EXTENT_ALLOC_SYSTEM_INODE, + INODE_ALLOC_SYSTEM_INODE, + JOURNAL_SYSTEM_INODE, + LOCAL_ALLOC_SYSTEM_INODE, + TRUNCATE_LOG_SYSTEM_INODE, + LOCAL_USER_QUOTA_SYSTEM_INODE, + LOCAL_GROUP_QUOTA_SYSTEM_INODE, +#define OCFS2_LAST_LOCAL_SYSTEM_INODE LOCAL_GROUP_QUOTA_SYSTEM_INODE + NUM_SYSTEM_INODES +}; +#define NUM_GLOBAL_SYSTEM_INODES OCFS2_FIRST_LOCAL_SYSTEM_INODE +#define NUM_LOCAL_SYSTEM_INODES \ + (NUM_SYSTEM_INODES - OCFS2_FIRST_LOCAL_SYSTEM_INODE) + +static struct ocfs2_system_inode_info ocfs2_system_inodes[NUM_SYSTEM_INODES] = { + /* Global system inodes (single copy) */ + /* The first two are only used from userspace mfks/tunefs */ + [BAD_BLOCK_SYSTEM_INODE] = { "bad_blocks", 0, S_IFREG | 0644 }, + [GLOBAL_INODE_ALLOC_SYSTEM_INODE] = { "global_inode_alloc", OCFS2_BITMAP_FL | OCFS2_CHAIN_FL, S_IFREG | 0644 }, + + /* These are used by the running filesystem */ + [SLOT_MAP_SYSTEM_INODE] = { "slot_map", 0, S_IFREG | 0644 }, + [HEARTBEAT_SYSTEM_INODE] = { "heartbeat", OCFS2_HEARTBEAT_FL, S_IFREG | 0644 }, + [GLOBAL_BITMAP_SYSTEM_INODE] = { "global_bitmap", 0, S_IFREG | 0644 }, + [USER_QUOTA_SYSTEM_INODE] = { "aquota.user", OCFS2_QUOTA_FL, S_IFREG | 0644 }, + [GROUP_QUOTA_SYSTEM_INODE] = { "aquota.group", OCFS2_QUOTA_FL, S_IFREG | 0644 }, + + /* Slot-specific system inodes (one copy per slot) */ + [ORPHAN_DIR_SYSTEM_INODE] = { "orphan_dir:%04d", 0, S_IFDIR | 0755 }, + [EXTENT_ALLOC_SYSTEM_INODE] = { "extent_alloc:%04d", OCFS2_BITMAP_FL | OCFS2_CHAIN_FL, S_IFREG | 0644 }, + [INODE_ALLOC_SYSTEM_INODE] = { "inode_alloc:%04d", OCFS2_BITMAP_FL | OCFS2_CHAIN_FL, S_IFREG | 0644 }, + [JOURNAL_SYSTEM_INODE] = { "journal:%04d", OCFS2_JOURNAL_FL, S_IFREG | 0644 }, + [LOCAL_ALLOC_SYSTEM_INODE] = { "local_alloc:%04d", OCFS2_BITMAP_FL | OCFS2_LOCAL_ALLOC_FL, S_IFREG | 0644 }, + [TRUNCATE_LOG_SYSTEM_INODE] = { "truncate_log:%04d", OCFS2_DEALLOC_FL, S_IFREG | 0644 }, + [LOCAL_USER_QUOTA_SYSTEM_INODE] = { "aquota.user:%04d", OCFS2_QUOTA_FL, S_IFREG | 0644 }, + [LOCAL_GROUP_QUOTA_SYSTEM_INODE] = { "aquota.group:%04d", OCFS2_QUOTA_FL, S_IFREG | 0644 }, +}; + +/* Parameter passed from mount.ocfs2 to module */ +#define OCFS2_HB_NONE "heartbeat=none" +#define OCFS2_HB_LOCAL "heartbeat=local" +#define OCFS2_HB_GLOBAL "heartbeat=global" + +/* + * OCFS2_DIR_PAD defines the directory entries boundaries + * + * NOTE: It must be a multiple of 4 + */ +#define OCFS2_DIR_PAD 4 +#define OCFS2_DIR_ROUND (OCFS2_DIR_PAD - 1) +#define OCFS2_DIR_MEMBER_LEN offsetof(struct ocfs2_dir_entry, name) +#define OCFS2_DIR_REC_LEN(name_len) (((name_len) + OCFS2_DIR_MEMBER_LEN + \ + OCFS2_DIR_ROUND) & \ + ~OCFS2_DIR_ROUND) +#define OCFS2_DIR_MIN_REC_LEN OCFS2_DIR_REC_LEN(1) + +#define OCFS2_LINK_MAX 32000 +#define OCFS2_DX_LINK_MAX ((1U << 31) - 1U) +#define OCFS2_LINKS_HI_SHIFT 16 +#define OCFS2_DX_ENTRIES_MAX (0xffffffffU) + + +/* + * Convenience casts + */ +#define OCFS2_RAW_SB(dinode) (&((dinode)->id2.i_super)) + +/* + * Block checking structure. This is used in metadata to validate the + * contents. If OCFS2_FEATURE_INCOMPAT_META_ECC is not set, it is all + * zeros. + */ +struct ocfs2_block_check { +/*00*/ __le32 bc_crc32e; /* 802.3 Ethernet II CRC32 */ + __le16 bc_ecc; /* Single-error-correction parity vector. + This is a simple Hamming code dependent + on the blocksize. OCFS2's maximum + blocksize, 4K, requires 16 parity bits, + so we fit in __le16. */ + __le16 bc_reserved1; +/*08*/ +}; + +/* + * On disk extent record for OCFS2 + * It describes a range of clusters on disk. + * + * Length fields are divided into interior and leaf node versions. + * This leaves room for a flags field (OCFS2_EXT_*) in the leaf nodes. + */ +struct ocfs2_extent_rec { +/*00*/ __le32 e_cpos; /* Offset into the file, in clusters */ + union { + __le32 e_int_clusters; /* Clusters covered by all children */ + struct { + __le16 e_leaf_clusters; /* Clusters covered by this + extent */ + __u8 e_reserved1; + __u8 e_flags; /* Extent flags */ + }; + }; + __le64 e_blkno; /* Physical disk offset, in blocks */ +/*10*/ +}; + +struct ocfs2_chain_rec { + __le32 c_free; /* Number of free bits in this chain. */ + __le32 c_total; /* Number of total bits in this chain */ + __le64 c_blkno; /* Physical disk offset (blocks) of 1st group */ +}; + +struct ocfs2_truncate_rec { + __le32 t_start; /* 1st cluster in this log */ + __le32 t_clusters; /* Number of total clusters covered */ +}; + +/* + * On disk extent list for OCFS2 (node in the tree). Note that this + * is contained inside ocfs2_dinode or ocfs2_extent_block, so the + * offsets are relative to ocfs2_dinode.id2.i_list or + * ocfs2_extent_block.h_list, respectively. + */ +struct ocfs2_extent_list { +/*00*/ __le16 l_tree_depth; /* Extent tree depth from this + point. 0 means data extents + hang directly off this + header (a leaf) + NOTE: The high 8 bits cannot be + used - tree_depth is never that big. + */ + __le16 l_count; /* Number of extent records */ + __le16 l_next_free_rec; /* Next unused extent slot */ + __le16 l_reserved1; + __le64 l_reserved2; /* Pad to + sizeof(ocfs2_extent_rec) */ + /* Extent records */ +/*10*/ struct ocfs2_extent_rec l_recs[] __counted_by_le(l_count); +}; + +/* + * On disk allocation chain list for OCFS2. Note that this is + * contained inside ocfs2_dinode, so the offsets are relative to + * ocfs2_dinode.id2.i_chain. + */ +struct ocfs2_chain_list { +/*00*/ __le16 cl_cpg; /* Clusters per Block Group */ + __le16 cl_bpc; /* Bits per cluster */ + __le16 cl_count; /* Total chains in this list */ + __le16 cl_next_free_rec; /* Next unused chain slot */ + __le64 cl_reserved1; + /* Chain records */ +/*10*/ struct ocfs2_chain_rec cl_recs[] __counted_by_le(cl_count); +}; + +/* + * On disk deallocation log for OCFS2. Note that this is + * contained inside ocfs2_dinode, so the offsets are relative to + * ocfs2_dinode.id2.i_dealloc. + */ +struct ocfs2_truncate_log { +/*00*/ __le16 tl_count; /* Total records in this log */ + __le16 tl_used; /* Number of records in use */ + __le32 tl_reserved1; + /* Truncate records */ +/*08*/ struct ocfs2_truncate_rec tl_recs[] __counted_by_le(tl_count); +}; + +/* + * On disk extent block (indirect block) for OCFS2 + */ +struct ocfs2_extent_block +{ +/*00*/ __u8 h_signature[8]; /* Signature for verification */ + struct ocfs2_block_check h_check; /* Error checking */ +/*10*/ __le16 h_suballoc_slot; /* Slot suballocator this + extent_header belongs to */ + __le16 h_suballoc_bit; /* Bit offset in suballocator + block group */ + __le32 h_fs_generation; /* Must match super block */ + __le64 h_blkno; /* Offset on disk, in blocks */ +/*20*/ __le64 h_suballoc_loc; /* Suballocator block group this + eb belongs to. Only valid + if allocated from a + discontiguous block group */ + __le64 h_next_leaf_blk; /* Offset on disk, in blocks, + of next leaf header pointing + to data */ +/*30*/ struct ocfs2_extent_list h_list; /* Extent record list */ +/* Actual on-disk size is one block */ +}; + +/* + * On disk slot map for OCFS2. This defines the contents of the "slot_map" + * system file. A slot is valid if it contains a node number >= 0. The + * value -1 (0xFFFF) is OCFS2_INVALID_SLOT. This marks a slot empty. + */ +struct ocfs2_slot_map { +/*00*/ DECLARE_FLEX_ARRAY(__le16, sm_slots); +/* + * Actual on-disk size is one block. OCFS2_MAX_SLOTS is 255, + * 255 * sizeof(__le16) == 512B, within the 512B block minimum blocksize. + */ +}; + +struct ocfs2_extended_slot { +/*00*/ __u8 es_valid; + __u8 es_reserved1[3]; + __le32 es_node_num; +/*08*/ +}; + +/* + * The extended slot map, used when OCFS2_FEATURE_INCOMPAT_EXTENDED_SLOT_MAP + * is set. It separates out the valid marker from the node number, and + * has room to grow. Unlike the old slot map, this format is defined by + * i_size. + */ +struct ocfs2_slot_map_extended { +/*00*/ DECLARE_FLEX_ARRAY(struct ocfs2_extended_slot, se_slots); +/* + * Actual size is i_size of the slot_map system file. It should + * match s_max_slots * sizeof(struct ocfs2_extended_slot) + */ +}; + +/* + * ci_stackflags is only valid if the incompat bit + * OCFS2_FEATURE_INCOMPAT_CLUSTERINFO is set. + */ +struct ocfs2_cluster_info { +/*00*/ __u8 ci_stack[OCFS2_STACK_LABEL_LEN]; + union { + __le32 ci_reserved; + struct { + __u8 ci_stackflags; + __u8 ci_reserved1; + __u8 ci_reserved2; + __u8 ci_reserved3; + }; + }; +/*08*/ __u8 ci_cluster[OCFS2_CLUSTER_NAME_LEN]; +/*18*/ +}; + +/* + * On disk superblock for OCFS2 + * Note that it is contained inside an ocfs2_dinode, so all offsets + * are relative to the start of ocfs2_dinode.id2. + */ +struct ocfs2_super_block { +/*00*/ __le16 s_major_rev_level; + __le16 s_minor_rev_level; + __le16 s_mnt_count; + __le16 s_max_mnt_count; + __le16 s_state; /* File system state */ + __le16 s_errors; /* Behaviour when detecting errors */ + __le32 s_checkinterval; /* Max time between checks */ +/*10*/ __le64 s_lastcheck; /* Time of last check */ + __le32 s_creator_os; /* OS */ + __le32 s_feature_compat; /* Compatible feature set */ +/*20*/ __le32 s_feature_incompat; /* Incompatible feature set */ + __le32 s_feature_ro_compat; /* Readonly-compatible feature set */ + __le64 s_root_blkno; /* Offset, in blocks, of root directory + dinode */ +/*30*/ __le64 s_system_dir_blkno; /* Offset, in blocks, of system + directory dinode */ + __le32 s_blocksize_bits; /* Blocksize for this fs */ + __le32 s_clustersize_bits; /* Clustersize for this fs */ +/*40*/ __le16 s_max_slots; /* Max number of simultaneous mounts + before tunefs required */ + __le16 s_tunefs_flag; + __le32 s_uuid_hash; /* hash value of uuid */ + __le64 s_first_cluster_group; /* Block offset of 1st cluster + * group header */ +/*50*/ __u8 s_label[OCFS2_MAX_VOL_LABEL_LEN]; /* Label for mounting, etc. */ +/*90*/ __u8 s_uuid[OCFS2_VOL_UUID_LEN]; /* 128-bit uuid */ +/*A0*/ struct ocfs2_cluster_info s_cluster_info; /* Only valid if either + userspace or clusterinfo + INCOMPAT flag set. */ +/*B8*/ __le16 s_xattr_inline_size; /* extended attribute inline size + for this fs*/ + __le16 s_reserved0; + __le32 s_dx_seed[3]; /* seed[0-2] for dx dir hash. + * s_uuid_hash serves as seed[3]. */ +/*C8*/ __le64 s_reserved2[15]; /* Fill out superblock */ +/*140*/ + + /* + * NOTE: As stated above, all offsets are relative to + * ocfs2_dinode.id2, which is at 0xC0 in the inode. + * 0xC0 + 0x140 = 0x200 or 512 bytes. A superblock must fit within + * our smallest blocksize, which is 512 bytes. To ensure this, + * we reserve the space in s_reserved2. Anything past s_reserved2 + * will not be available on the smallest blocksize. + */ +}; + +/* + * Local allocation bitmap for OCFS2 slots + * Note that it exists inside an ocfs2_dinode, so all offsets are + * relative to the start of ocfs2_dinode.id2. + */ +struct ocfs2_local_alloc +{ +/*00*/ __le32 la_bm_off; /* Starting bit offset in main bitmap */ + __le16 la_size; /* Size of included bitmap, in bytes */ + __le16 la_reserved1; + __le64 la_reserved2; +/*10*/ __u8 la_bitmap[] __counted_by_le(la_size); +}; + +/* + * Data-in-inode header. This is only used if i_dyn_features has + * OCFS2_INLINE_DATA_FL set. + */ +struct ocfs2_inline_data +{ +/*00*/ __le16 id_count; /* Number of bytes that can be used + * for data, starting at id_data */ + __le16 id_reserved0; + __le32 id_reserved1; + __u8 id_data[] __counted_by_le(id_count); /* Start of user data */ +}; + +/* + * On disk inode for OCFS2 + */ +struct ocfs2_dinode { +/*00*/ __u8 i_signature[8]; /* Signature for validation */ + __le32 i_generation; /* Generation number */ + __le16 i_suballoc_slot; /* Slot suballocator this inode + belongs to */ + __le16 i_suballoc_bit; /* Bit offset in suballocator + block group */ +/*10*/ __le16 i_links_count_hi; /* High 16 bits of links count */ + __le16 i_xattr_inline_size; + __le32 i_clusters; /* Cluster count */ + __le32 i_uid; /* Owner UID */ + __le32 i_gid; /* Owning GID */ +/*20*/ __le64 i_size; /* Size in bytes */ + __le16 i_mode; /* File mode */ + __le16 i_links_count; /* Links count */ + __le32 i_flags; /* File flags */ +/*30*/ __le64 i_atime; /* Access time */ + __le64 i_ctime; /* Creation time */ +/*40*/ __le64 i_mtime; /* Modification time */ + __le64 i_dtime; /* Deletion time */ +/*50*/ __le64 i_blkno; /* Offset on disk, in blocks */ + __le64 i_last_eb_blk; /* Pointer to last extent + block */ +/*60*/ __le32 i_fs_generation; /* Generation per fs-instance */ + __le32 i_atime_nsec; + __le32 i_ctime_nsec; + __le32 i_mtime_nsec; +/*70*/ __le32 i_attr; + __le16 i_orphaned_slot; /* Only valid when OCFS2_ORPHANED_FL + was set in i_flags */ + __le16 i_dyn_features; + __le64 i_xattr_loc; +/*80*/ struct ocfs2_block_check i_check; /* Error checking */ +/*88*/ __le64 i_dx_root; /* Pointer to dir index root block */ +/*90*/ __le64 i_refcount_loc; + __le64 i_suballoc_loc; /* Suballocator block group this + inode belongs to. Only valid + if allocated from a + discontiguous block group */ +/*A0*/ __le16 i_dio_orphaned_slot; /* only used for append dio write */ + __le16 i_reserved1[3]; + __le64 i_reserved2[2]; +/*B8*/ union { + __le64 i_pad1; /* Generic way to refer to this + 64bit union */ + struct { + __le64 i_rdev; /* Device number */ + } dev1; + struct { /* Info for bitmap system + inodes */ + __le32 i_used; /* Bits (ie, clusters) used */ + __le32 i_total; /* Total bits (clusters) + available */ + } bitmap1; + struct { /* Info for journal system + inodes */ + __le32 ij_flags; /* Mounted, version, etc. */ + __le32 ij_recovery_generation; /* Incremented when the + journal is recovered + after an unclean + shutdown */ + } journal1; + } id1; /* Inode type dependent 1 */ +/*C0*/ union { + struct ocfs2_super_block i_super; + struct ocfs2_local_alloc i_lab; + struct ocfs2_chain_list i_chain; + struct ocfs2_extent_list i_list; + struct ocfs2_truncate_log i_dealloc; + struct ocfs2_inline_data i_data; + DECLARE_FLEX_ARRAY(__u8, i_symlink); + } id2; +/* Actual on-disk size is one block */ +}; + +/* + * On-disk directory entry structure for OCFS2 + * + * Packed as this structure could be accessed unaligned on 64-bit platforms + */ +struct ocfs2_dir_entry { +/*00*/ __le64 inode; /* Inode number */ + __le16 rec_len; /* Directory entry length */ + __u8 name_len; /* Name length */ + __u8 file_type; +/*0C*/ char name[OCFS2_MAX_FILENAME_LEN]; /* File name */ +/* Actual on-disk length specified by rec_len */ +} __attribute__ ((packed)); + +/* + * Per-block record for the unindexed directory btree. This is carefully + * crafted so that the rec_len and name_len records of an ocfs2_dir_entry are + * mirrored. That way, the directory manipulation code needs a minimal amount + * of update. + * + * NOTE: Keep this structure aligned to a multiple of 4 bytes. + */ +struct ocfs2_dir_block_trailer { +/*00*/ __le64 db_compat_inode; /* Always zero. Was inode */ + + __le16 db_compat_rec_len; /* Backwards compatible with + * ocfs2_dir_entry. */ + __u8 db_compat_name_len; /* Always zero. Was name_len */ + __u8 db_reserved0; + __le16 db_reserved1; + __le16 db_free_rec_len; /* Size of largest empty hole + * in this block. (unused) */ +/*10*/ __u8 db_signature[8]; /* Signature for verification */ + __le64 db_reserved2; +/*20*/ __le64 db_free_next; /* Next block in list (unused) */ + __le64 db_blkno; /* Offset on disk, in blocks */ +/*30*/ __le64 db_parent_dinode; /* dinode which owns me, in + blocks */ + struct ocfs2_block_check db_check; /* Error checking */ +/*40*/ +}; + + /* + * A directory entry in the indexed tree. We don't store the full name here, + * but instead provide a pointer to the full dirent in the unindexed tree. + * + * We also store name_len here so as to reduce the number of leaf blocks we + * need to search in case of collisions. + */ +struct ocfs2_dx_entry { + __le32 dx_major_hash; /* Used to find logical + * cluster in index */ + __le32 dx_minor_hash; /* Lower bits used to find + * block in cluster */ + __le64 dx_dirent_blk; /* Physical block in unindexed + * tree holding this dirent. */ +}; + +struct ocfs2_dx_entry_list { + __le32 de_reserved; + __le16 de_count; /* Maximum number of entries + * possible in de_entries */ + __le16 de_num_used; /* Current number of + * de_entries entries */ + /* Indexed dir entries in a packed + * array of length de_num_used. + */ + struct ocfs2_dx_entry de_entries[] __counted_by_le(de_count); +}; + +#define OCFS2_DX_FLAG_INLINE 0x01 + +/* + * A directory indexing block. Each indexed directory has one of these, + * pointed to by ocfs2_dinode. + * + * This block stores an indexed btree root, and a set of free space + * start-of-list pointers. + */ +struct ocfs2_dx_root_block { + __u8 dr_signature[8]; /* Signature for verification */ + struct ocfs2_block_check dr_check; /* Error checking */ + __le16 dr_suballoc_slot; /* Slot suballocator this + * block belongs to. */ + __le16 dr_suballoc_bit; /* Bit offset in suballocator + * block group */ + __le32 dr_fs_generation; /* Must match super block */ + __le64 dr_blkno; /* Offset on disk, in blocks */ + __le64 dr_last_eb_blk; /* Pointer to last + * extent block */ + __le32 dr_clusters; /* Clusters allocated + * to the indexed tree. */ + __u8 dr_flags; /* OCFS2_DX_FLAG_* flags */ + __u8 dr_reserved0; + __le16 dr_reserved1; + __le64 dr_dir_blkno; /* Pointer to parent inode */ + __le32 dr_num_entries; /* Total number of + * names stored in + * this directory.*/ + __le32 dr_reserved2; + __le64 dr_free_blk; /* Pointer to head of free + * unindexed block list. */ + __le64 dr_suballoc_loc; /* Suballocator block group + this root belongs to. + Only valid if allocated + from a discontiguous + block group */ + __le64 dr_reserved3[14]; + union { + struct ocfs2_extent_list dr_list; /* Keep this aligned to 128 + * bits for maximum space + * efficiency. */ + struct ocfs2_dx_entry_list dr_entries; /* In-root-block list of + * entries. We grow out + * to extents if this + * gets too big. */ + }; +}; + +/* + * The header of a leaf block in the indexed tree. + */ +struct ocfs2_dx_leaf { + __u8 dl_signature[8];/* Signature for verification */ + struct ocfs2_block_check dl_check; /* Error checking */ + __le64 dl_blkno; /* Offset on disk, in blocks */ + __le32 dl_fs_generation;/* Must match super block */ + __le32 dl_reserved0; + __le64 dl_reserved1; + struct ocfs2_dx_entry_list dl_list; +}; + +/* + * Largest bitmap for a block (suballocator) group in bytes. This limit + * does not affect cluster groups (global allocator). Cluster group + * bitmaps run to the end of the block. + */ +#define OCFS2_MAX_BG_BITMAP_SIZE 256 + +/* + * On disk allocator group structure for OCFS2 + */ +struct ocfs2_group_desc +{ +/*00*/ __u8 bg_signature[8]; /* Signature for validation */ + __le16 bg_size; /* Size of included bitmap in + bytes. */ + __le16 bg_bits; /* Bits represented by this + group. */ + __le16 bg_free_bits_count; /* Free bits count */ + __le16 bg_chain; /* What chain I am in. */ +/*10*/ __le32 bg_generation; + __le16 bg_contig_free_bits; /* max contig free bits length */ + __le16 bg_reserved1; + __le64 bg_next_group; /* Next group in my list, in + blocks */ +/*20*/ __le64 bg_parent_dinode; /* dinode which owns me, in + blocks */ + __le64 bg_blkno; /* Offset on disk, in blocks */ +/*30*/ struct ocfs2_block_check bg_check; /* Error checking */ + __le64 bg_reserved2; +/*40*/ union { + DECLARE_FLEX_ARRAY(__u8, bg_bitmap); + struct { + /* + * Block groups may be discontiguous when + * OCFS2_FEATURE_INCOMPAT_DISCONTIG_BG is set. + * The extents of a discontiguous block group are + * stored in bg_list. It is a flat list. + * l_tree_depth must always be zero. A + * discontiguous group is signified by a non-zero + * bg_list->l_next_free_rec. Only block groups + * can be discontiguous; Cluster groups cannot. + * We've never made a block group with more than + * 2048 blocks (256 bytes of bg_bitmap). This + * codifies that limit so that we can fit bg_list. + * bg_size of a discontiguous block group will + * be 256 to match bg_bitmap_filler. + */ + __u8 bg_bitmap_filler[OCFS2_MAX_BG_BITMAP_SIZE]; +/*140*/ struct ocfs2_extent_list bg_list; + }; + }; +/* Actual on-disk size is one block */ +}; + +struct ocfs2_refcount_rec { +/*00*/ __le64 r_cpos; /* Physical offset, in clusters */ + __le32 r_clusters; /* Clusters covered by this extent */ + __le32 r_refcount; /* Reference count of this extent */ +/*10*/ +}; +#define OCFS2_32BIT_POS_MASK (0xffffffffULL) + +#define OCFS2_REFCOUNT_LEAF_FL (0x00000001) +#define OCFS2_REFCOUNT_TREE_FL (0x00000002) + +struct ocfs2_refcount_list { +/*00*/ __le16 rl_count; /* Maximum number of entries possible + in rl_records */ + __le16 rl_used; /* Current number of used records */ + __le32 rl_reserved2; + __le64 rl_reserved1; /* Pad to sizeof(ocfs2_refcount_record) */ + /* Refcount records */ +/*10*/ struct ocfs2_refcount_rec rl_recs[] __counted_by_le(rl_count); +}; + + +struct ocfs2_refcount_block { +/*00*/ __u8 rf_signature[8]; /* Signature for verification */ + __le16 rf_suballoc_slot; /* Slot suballocator this block + belongs to */ + __le16 rf_suballoc_bit; /* Bit offset in suballocator + block group */ + __le32 rf_fs_generation; /* Must match superblock */ +/*10*/ __le64 rf_blkno; /* Offset on disk, in blocks */ + __le64 rf_parent; /* Parent block, only valid if + OCFS2_REFCOUNT_LEAF_FL is set in + rf_flags */ +/*20*/ struct ocfs2_block_check rf_check; /* Error checking */ + __le64 rf_last_eb_blk; /* Pointer to last extent block */ +/*30*/ __le32 rf_count; /* Number of inodes sharing this + refcount tree */ + __le32 rf_flags; /* See the flags above */ + __le32 rf_clusters; /* clusters covered by refcount tree. */ + __le32 rf_cpos; /* cluster offset in refcount tree.*/ +/*40*/ __le32 rf_generation; /* generation number. all be the same + * for the same refcount tree. */ + __le32 rf_reserved0; + __le64 rf_suballoc_loc; /* Suballocator block group this + refcount block belongs to. Only + valid if allocated from a + discontiguous block group */ +/*50*/ __le64 rf_reserved1[6]; +/*80*/ union { + struct ocfs2_refcount_list rf_records; /* List of refcount + records */ + struct ocfs2_extent_list rf_list; /* Extent record list, + only valid if + OCFS2_REFCOUNT_TREE_FL + is set in rf_flags */ + }; +/* Actual on-disk size is one block */ +}; + +/* + * On disk extended attribute structure for OCFS2. + */ + +/* + * ocfs2_xattr_entry indicates one extend attribute. + * + * Note that it can be stored in inode, one block or one xattr bucket. + */ +struct ocfs2_xattr_entry { + __le32 xe_name_hash; /* hash value of xattr prefix+suffix. */ + __le16 xe_name_offset; /* byte offset from the 1st entry in the + local xattr storage(inode, xattr block or + xattr bucket). */ + __u8 xe_name_len; /* xattr name len, doesn't include prefix. */ + __u8 xe_type; /* the low 7 bits indicate the name prefix + * type and the highest bit indicates whether + * the EA is stored in the local storage. */ + __le64 xe_value_size; /* real xattr value length. */ +}; + +/* + * On disk structure for xattr header. + * + * One ocfs2_xattr_header describes how many ocfs2_xattr_entry records in + * the local xattr storage. + */ +struct ocfs2_xattr_header { + __le16 xh_count; /* contains the count of how + many records are in the + local xattr storage. */ + __le16 xh_free_start; /* current offset for storing + xattr. */ + __le16 xh_name_value_len; /* total length of name/value + length in this bucket. */ + __le16 xh_num_buckets; /* Number of xattr buckets + in this extent record, + only valid in the first + bucket. */ + struct ocfs2_block_check xh_check; /* Error checking + (Note, this is only + used for xattr + buckets. A block uses + xb_check and sets + this field to zero.) */ + /* xattr entry list. */ + struct ocfs2_xattr_entry xh_entries[] __counted_by_le(xh_count); +}; + +/* + * On disk structure for xattr value root. + * + * When an xattr's value is large enough, it is stored in an external + * b-tree like file data. The xattr value root points to this structure. + */ +struct ocfs2_xattr_value_root { +/*00*/ __le32 xr_clusters; /* clusters covered by xattr value. */ + __le32 xr_reserved0; + __le64 xr_last_eb_blk; /* Pointer to last extent block */ +/*10*/ struct ocfs2_extent_list xr_list; /* Extent record list */ +}; + +/* + * On disk structure for xattr tree root. + * + * It is used when there are too many extended attributes for one file. These + * attributes will be organized and stored in an indexed-btree. + */ +struct ocfs2_xattr_tree_root { +/*00*/ __le32 xt_clusters; /* clusters covered by xattr. */ + __le32 xt_reserved0; + __le64 xt_last_eb_blk; /* Pointer to last extent block */ +/*10*/ struct ocfs2_extent_list xt_list; /* Extent record list */ +}; + +#define OCFS2_XATTR_INDEXED 0x1 +#define OCFS2_HASH_SHIFT 5 +#define OCFS2_XATTR_ROUND 3 +#define OCFS2_XATTR_SIZE(size) (((size) + OCFS2_XATTR_ROUND) & \ + ~(OCFS2_XATTR_ROUND)) + +#define OCFS2_XATTR_BUCKET_SIZE 4096 +#define OCFS2_XATTR_MAX_BLOCKS_PER_BUCKET (OCFS2_XATTR_BUCKET_SIZE \ + / OCFS2_MIN_BLOCKSIZE) + +/* + * On disk structure for xattr block. + */ +struct ocfs2_xattr_block { +/*00*/ __u8 xb_signature[8]; /* Signature for verification */ + __le16 xb_suballoc_slot; /* Slot suballocator this + block belongs to. */ + __le16 xb_suballoc_bit; /* Bit offset in suballocator + block group */ + __le32 xb_fs_generation; /* Must match super block */ +/*10*/ __le64 xb_blkno; /* Offset on disk, in blocks */ + struct ocfs2_block_check xb_check; /* Error checking */ +/*20*/ __le16 xb_flags; /* Indicates whether this block contains + real xattr or a xattr tree. */ + __le16 xb_reserved0; + __le32 xb_reserved1; + __le64 xb_suballoc_loc; /* Suballocator block group this + xattr block belongs to. Only + valid if allocated from a + discontiguous block group */ +/*30*/ union { + struct ocfs2_xattr_header xb_header; /* xattr header if this + block contains xattr */ + struct ocfs2_xattr_tree_root xb_root;/* xattr tree root if this + block contains xattr + tree. */ + } xb_attrs; +}; + +#define OCFS2_XATTR_ENTRY_LOCAL 0x80 +#define OCFS2_XATTR_TYPE_MASK 0x7F +static inline void ocfs2_xattr_set_local(struct ocfs2_xattr_entry *xe, + int local) +{ + if (local) + xe->xe_type |= OCFS2_XATTR_ENTRY_LOCAL; + else + xe->xe_type &= ~OCFS2_XATTR_ENTRY_LOCAL; +} + +static inline int ocfs2_xattr_is_local(struct ocfs2_xattr_entry *xe) +{ + return xe->xe_type & OCFS2_XATTR_ENTRY_LOCAL; +} + +static inline void ocfs2_xattr_set_type(struct ocfs2_xattr_entry *xe, int type) +{ + xe->xe_type |= type & OCFS2_XATTR_TYPE_MASK; +} + +static inline int ocfs2_xattr_get_type(struct ocfs2_xattr_entry *xe) +{ + return xe->xe_type & OCFS2_XATTR_TYPE_MASK; +} + +/* + * On disk structures for global quota file + */ + +/* Magic numbers and known versions for global quota files */ +#define OCFS2_GLOBAL_QMAGICS {\ + 0x0cf52470, /* USRQUOTA */ \ + 0x0cf52471 /* GRPQUOTA */ \ +} + +#define OCFS2_GLOBAL_QVERSIONS {\ + 0, \ + 0, \ +} + + +/* Each block of each quota file has a certain fixed number of bytes reserved + * for OCFS2 internal use at its end. OCFS2 can use it for things like + * checksums, etc. */ +#define OCFS2_QBLK_RESERVED_SPACE 8 + +/* Generic header of all quota files */ +struct ocfs2_disk_dqheader { + __le32 dqh_magic; /* Magic number identifying file */ + __le32 dqh_version; /* Quota format version */ +}; + +#define OCFS2_GLOBAL_INFO_OFF (sizeof(struct ocfs2_disk_dqheader)) + +/* Information header of global quota file (immediately follows the generic + * header) */ +struct ocfs2_global_disk_dqinfo { +/*00*/ __le32 dqi_bgrace; /* Grace time for space softlimit excess */ + __le32 dqi_igrace; /* Grace time for inode softlimit excess */ + __le32 dqi_syncms; /* Time after which we sync local changes to + * global quota file */ + __le32 dqi_blocks; /* Number of blocks in quota file */ +/*10*/ __le32 dqi_free_blk; /* First free block in quota file */ + __le32 dqi_free_entry; /* First block with free dquot entry in quota + * file */ +}; + +/* Structure with global user / group information. We reserve some space + * for future use. */ +struct ocfs2_global_disk_dqblk { +/*00*/ __le32 dqb_id; /* ID the structure belongs to */ + __le32 dqb_use_count; /* Number of nodes having reference to this structure */ + __le64 dqb_ihardlimit; /* absolute limit on allocated inodes */ +/*10*/ __le64 dqb_isoftlimit; /* preferred inode limit */ + __le64 dqb_curinodes; /* current # allocated inodes */ +/*20*/ __le64 dqb_bhardlimit; /* absolute limit on disk space */ + __le64 dqb_bsoftlimit; /* preferred limit on disk space */ +/*30*/ __le64 dqb_curspace; /* current space occupied */ + __le64 dqb_btime; /* time limit for excessive disk use */ +/*40*/ __le64 dqb_itime; /* time limit for excessive inode use */ + __le64 dqb_pad1; +/*50*/ __le64 dqb_pad2; +}; + +/* + * On-disk structures for local quota file + */ + +/* Magic numbers and known versions for local quota files */ +#define OCFS2_LOCAL_QMAGICS {\ + 0x0cf524c0, /* USRQUOTA */ \ + 0x0cf524c1 /* GRPQUOTA */ \ +} + +#define OCFS2_LOCAL_QVERSIONS {\ + 0, \ + 0, \ +} + +/* Quota flags in dqinfo header */ +#define OLQF_CLEAN 0x0001 /* Quota file is empty (this should be after\ + * quota has been cleanly turned off) */ + +#define OCFS2_LOCAL_INFO_OFF (sizeof(struct ocfs2_disk_dqheader)) + +/* Information header of local quota file (immediately follows the generic + * header) */ +struct ocfs2_local_disk_dqinfo { + __le32 dqi_flags; /* Flags for quota file */ + __le32 dqi_chunks; /* Number of chunks of quota structures + * with a bitmap */ + __le32 dqi_blocks; /* Number of blocks allocated for quota file */ +}; + +/* Header of one chunk of a quota file */ +struct ocfs2_local_disk_chunk { + __le32 dqc_free; /* Number of free entries in the bitmap */ + __u8 dqc_bitmap[]; /* Bitmap of entries in the corresponding + * chunk of quota file */ +}; + +/* One entry in local quota file */ +struct ocfs2_local_disk_dqblk { +/*00*/ __le64 dqb_id; /* id this quota applies to */ + __le64 dqb_spacemod; /* Change in the amount of used space */ +/*10*/ __le64 dqb_inodemod; /* Change in the amount of used inodes */ +}; + + +/* + * The quota trailer lives at the end of each quota block. + */ + +struct ocfs2_disk_dqtrailer { +/*00*/ struct ocfs2_block_check dq_check; /* Error checking */ +/*08*/ /* Cannot be larger than OCFS2_QBLK_RESERVED_SPACE */ +}; + +static inline struct ocfs2_disk_dqtrailer *ocfs2_block_dqtrailer(int blocksize, + void *buf) +{ + char *ptr = buf; + ptr += blocksize - OCFS2_QBLK_RESERVED_SPACE; + + return (struct ocfs2_disk_dqtrailer *)ptr; +} + +#ifdef __KERNEL__ +static inline int ocfs2_fast_symlink_chars(struct super_block *sb) +{ + return sb->s_blocksize - + offsetof(struct ocfs2_dinode, id2.i_symlink); +} + +static inline int ocfs2_max_inline_data_with_xattr(struct super_block *sb, + struct ocfs2_dinode *di) +{ + unsigned int xattrsize = le16_to_cpu(di->i_xattr_inline_size); + + if (le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_XATTR_FL) + return sb->s_blocksize - + offsetof(struct ocfs2_dinode, id2.i_data.id_data) - + xattrsize; + else + return sb->s_blocksize - + offsetof(struct ocfs2_dinode, id2.i_data.id_data); +} + +static inline int ocfs2_extent_recs_per_inode(struct super_block *sb) +{ + int size; + + size = sb->s_blocksize - + offsetof(struct ocfs2_dinode, id2.i_list.l_recs); + + return size / sizeof(struct ocfs2_extent_rec); +} + +static inline int ocfs2_extent_recs_per_inode_with_xattr( + struct super_block *sb, + struct ocfs2_dinode *di) +{ + int size; + unsigned int xattrsize = le16_to_cpu(di->i_xattr_inline_size); + + if (le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_XATTR_FL) + size = sb->s_blocksize - + offsetof(struct ocfs2_dinode, id2.i_list.l_recs) - + xattrsize; + else + size = sb->s_blocksize - + offsetof(struct ocfs2_dinode, id2.i_list.l_recs); + + return size / sizeof(struct ocfs2_extent_rec); +} + +static inline int ocfs2_extent_recs_per_dx_root(struct super_block *sb) +{ + int size; + + size = sb->s_blocksize - + offsetof(struct ocfs2_dx_root_block, dr_list.l_recs); + + return size / sizeof(struct ocfs2_extent_rec); +} + +static inline int ocfs2_chain_recs_per_inode(struct super_block *sb) +{ + int size; + + size = sb->s_blocksize - + offsetof(struct ocfs2_dinode, id2.i_chain.cl_recs); + + return size / sizeof(struct ocfs2_chain_rec); +} + +static inline u16 ocfs2_extent_recs_per_eb(struct super_block *sb) +{ + int size; + + size = sb->s_blocksize - + offsetof(struct ocfs2_extent_block, h_list.l_recs); + + return size / sizeof(struct ocfs2_extent_rec); +} + +static inline u16 ocfs2_extent_recs_per_gd(struct super_block *sb) +{ + int size; + + size = sb->s_blocksize - + offsetof(struct ocfs2_group_desc, bg_list.l_recs); + + return size / sizeof(struct ocfs2_extent_rec); +} + +static inline int ocfs2_dx_entries_per_leaf(struct super_block *sb) +{ + int size; + + size = sb->s_blocksize - + offsetof(struct ocfs2_dx_leaf, dl_list.de_entries); + + return size / sizeof(struct ocfs2_dx_entry); +} + +static inline int ocfs2_dx_entries_per_root(struct super_block *sb) +{ + int size; + + size = sb->s_blocksize - + offsetof(struct ocfs2_dx_root_block, dr_entries.de_entries); + + return size / sizeof(struct ocfs2_dx_entry); +} + +static inline u16 ocfs2_local_alloc_size(struct super_block *sb) +{ + u16 size; + + size = sb->s_blocksize - + offsetof(struct ocfs2_dinode, id2.i_lab.la_bitmap); + + return size; +} + +static inline int ocfs2_group_bitmap_size(struct super_block *sb, + int suballocator, + u32 feature_incompat) +{ + int size = sb->s_blocksize - + offsetof(struct ocfs2_group_desc, bg_bitmap); + + /* + * The cluster allocator uses the entire block. Suballocators have + * never used more than OCFS2_MAX_BG_BITMAP_SIZE. Unfortunately, older + * code expects bg_size set to the maximum. Thus we must keep + * bg_size as-is unless discontig_bg is enabled. + */ + if (suballocator && + (feature_incompat & OCFS2_FEATURE_INCOMPAT_DISCONTIG_BG)) + size = OCFS2_MAX_BG_BITMAP_SIZE; + + return size; +} + +static inline int ocfs2_truncate_recs_per_inode(struct super_block *sb) +{ + int size; + + size = sb->s_blocksize - + offsetof(struct ocfs2_dinode, id2.i_dealloc.tl_recs); + + return size / sizeof(struct ocfs2_truncate_rec); +} + +static inline u64 ocfs2_backup_super_blkno(struct super_block *sb, int index) +{ + u64 offset = OCFS2_BACKUP_SB_START; + + if (index >= 0 && index < OCFS2_MAX_BACKUP_SUPERBLOCKS) { + offset <<= (2 * index); + offset >>= sb->s_blocksize_bits; + return offset; + } + + return 0; + +} + +static inline u16 ocfs2_xattr_recs_per_xb(struct super_block *sb) +{ + int size; + + size = sb->s_blocksize - + offsetof(struct ocfs2_xattr_block, + xb_attrs.xb_root.xt_list.l_recs); + + return size / sizeof(struct ocfs2_extent_rec); +} + +static inline u16 ocfs2_extent_recs_per_rb(struct super_block *sb) +{ + int size; + + size = sb->s_blocksize - + offsetof(struct ocfs2_refcount_block, rf_list.l_recs); + + return size / sizeof(struct ocfs2_extent_rec); +} + +static inline u16 ocfs2_refcount_recs_per_rb(struct super_block *sb) +{ + int size; + + size = sb->s_blocksize - + offsetof(struct ocfs2_refcount_block, rf_records.rl_recs); + + return size / sizeof(struct ocfs2_refcount_rec); +} + +static inline u32 +ocfs2_get_ref_rec_low_cpos(const struct ocfs2_refcount_rec *rec) +{ + return le64_to_cpu(rec->r_cpos) & OCFS2_32BIT_POS_MASK; +} +#else +static inline int ocfs2_fast_symlink_chars(int blocksize) +{ + return blocksize - offsetof(struct ocfs2_dinode, id2.i_symlink); +} + +static inline int ocfs2_max_inline_data_with_xattr(int blocksize, + struct ocfs2_dinode *di) +{ + if (di && (di->i_dyn_features & OCFS2_INLINE_XATTR_FL)) + return blocksize - + offsetof(struct ocfs2_dinode, id2.i_data.id_data) - + di->i_xattr_inline_size; + else + return blocksize - + offsetof(struct ocfs2_dinode, id2.i_data.id_data); +} + +static inline int ocfs2_extent_recs_per_inode(int blocksize) +{ + int size; + + size = blocksize - + offsetof(struct ocfs2_dinode, id2.i_list.l_recs); + + return size / sizeof(struct ocfs2_extent_rec); +} + +static inline int ocfs2_chain_recs_per_inode(int blocksize) +{ + int size; + + size = blocksize - + offsetof(struct ocfs2_dinode, id2.i_chain.cl_recs); + + return size / sizeof(struct ocfs2_chain_rec); +} + +static inline int ocfs2_extent_recs_per_eb(int blocksize) +{ + int size; + + size = blocksize - + offsetof(struct ocfs2_extent_block, h_list.l_recs); + + return size / sizeof(struct ocfs2_extent_rec); +} + +static inline int ocfs2_extent_recs_per_gd(int blocksize) +{ + int size; + + size = blocksize - + offsetof(struct ocfs2_group_desc, bg_list.l_recs); + + return size / sizeof(struct ocfs2_extent_rec); +} + +static inline int ocfs2_local_alloc_size(int blocksize) +{ + int size; + + size = blocksize - + offsetof(struct ocfs2_dinode, id2.i_lab.la_bitmap); + + return size; +} + +static inline int ocfs2_group_bitmap_size(int blocksize, + int suballocator, + uint32_t feature_incompat) +{ + int size = sb->s_blocksize - + offsetof(struct ocfs2_group_desc, bg_bitmap); + + /* + * The cluster allocator uses the entire block. Suballocators have + * never used more than OCFS2_MAX_BG_BITMAP_SIZE. Unfortunately, older + * code expects bg_size set to the maximum. Thus we must keep + * bg_size as-is unless discontig_bg is enabled. + */ + if (suballocator && + (feature_incompat & OCFS2_FEATURE_INCOMPAT_DISCONTIG_BG)) + size = OCFS2_MAX_BG_BITMAP_SIZE; + + return size; +} + +static inline int ocfs2_truncate_recs_per_inode(int blocksize) +{ + int size; + + size = blocksize - + offsetof(struct ocfs2_dinode, id2.i_dealloc.tl_recs); + + return size / sizeof(struct ocfs2_truncate_rec); +} + +static inline uint64_t ocfs2_backup_super_blkno(int blocksize, int index) +{ + uint64_t offset = OCFS2_BACKUP_SB_START; + + if (index >= 0 && index < OCFS2_MAX_BACKUP_SUPERBLOCKS) { + offset <<= (2 * index); + offset /= blocksize; + return offset; + } + + return 0; +} + +static inline int ocfs2_xattr_recs_per_xb(int blocksize) +{ + int size; + + size = blocksize - + offsetof(struct ocfs2_xattr_block, + xb_attrs.xb_root.xt_list.l_recs); + + return size / sizeof(struct ocfs2_extent_rec); +} +#endif /* __KERNEL__ */ + + +static inline int ocfs2_system_inode_is_global(int type) +{ + return ((type >= 0) && + (type <= OCFS2_LAST_GLOBAL_SYSTEM_INODE)); +} + +static inline int ocfs2_sprintf_system_inode_name(char *buf, int len, + int type, int slot) +{ + int chars; + + /* + * Global system inodes can only have one copy. Everything + * after OCFS2_LAST_GLOBAL_SYSTEM_INODE in the system inode + * list has a copy per slot. + */ + if (type <= OCFS2_LAST_GLOBAL_SYSTEM_INODE) + chars = snprintf(buf, len, "%s", + ocfs2_system_inodes[type].si_name); + else + chars = snprintf(buf, len, + ocfs2_system_inodes[type].si_name, + slot); + + return chars; +} + +static inline void ocfs2_set_de_type(struct ocfs2_dir_entry *de, + umode_t mode) +{ + de->file_type = fs_umode_to_ftype(mode); +} + +static inline int ocfs2_gd_is_discontig(struct ocfs2_group_desc *gd) +{ + if ((offsetof(struct ocfs2_group_desc, bg_bitmap) + + le16_to_cpu(gd->bg_size)) != + offsetof(struct ocfs2_group_desc, bg_list)) + return 0; + /* + * Only valid to check l_next_free_rec if + * bg_bitmap + bg_size == bg_list. + */ + if (!gd->bg_list.l_next_free_rec) + return 0; + return 1; +} +#endif /* _OCFS2_FS_H */ + diff --git a/fs/ocfs2/ocfs2_ioctl.h b/fs/ocfs2/ocfs2_ioctl.h new file mode 100644 index 000000000..2de2f8733 --- /dev/null +++ b/fs/ocfs2/ocfs2_ioctl.h @@ -0,0 +1,224 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * ocfs2_ioctl.h + * + * Defines OCFS2 ioctls. + * + * Copyright (C) 2010 Oracle. All rights reserved. + */ + +#ifndef OCFS2_IOCTL_H +#define OCFS2_IOCTL_H + +/* + * Space reservation / allocation / free ioctls and argument structure + * are designed to be compatible with XFS. + * + * ALLOCSP* and FREESP* are not and will never be supported, but are + * included here for completeness. + */ +struct ocfs2_space_resv { + __s16 l_type; + __s16 l_whence; + __s64 l_start; + __s64 l_len; /* len == 0 means until end of file */ + __s32 l_sysid; + __u32 l_pid; + __s32 l_pad[4]; /* reserve area */ +}; + +#define OCFS2_IOC_ALLOCSP _IOW ('X', 10, struct ocfs2_space_resv) +#define OCFS2_IOC_FREESP _IOW ('X', 11, struct ocfs2_space_resv) +#define OCFS2_IOC_RESVSP _IOW ('X', 40, struct ocfs2_space_resv) +#define OCFS2_IOC_UNRESVSP _IOW ('X', 41, struct ocfs2_space_resv) +#define OCFS2_IOC_ALLOCSP64 _IOW ('X', 36, struct ocfs2_space_resv) +#define OCFS2_IOC_FREESP64 _IOW ('X', 37, struct ocfs2_space_resv) +#define OCFS2_IOC_RESVSP64 _IOW ('X', 42, struct ocfs2_space_resv) +#define OCFS2_IOC_UNRESVSP64 _IOW ('X', 43, struct ocfs2_space_resv) + +/* Used to pass group descriptor data when online resize is done */ +struct ocfs2_new_group_input { + __u64 group; /* Group descriptor's blkno. */ + __u32 clusters; /* Total number of clusters in this group */ + __u32 frees; /* Total free clusters in this group */ + __u16 chain; /* Chain for this group */ + __u16 reserved1; + __u32 reserved2; +}; + +#define OCFS2_IOC_GROUP_EXTEND _IOW('o', 1, int) +#define OCFS2_IOC_GROUP_ADD _IOW('o', 2,struct ocfs2_new_group_input) +#define OCFS2_IOC_GROUP_ADD64 _IOW('o', 3,struct ocfs2_new_group_input) + +/* Used to pass 2 file names to reflink. */ +struct reflink_arguments { + __u64 old_path; + __u64 new_path; + __u64 preserve; +}; +#define OCFS2_IOC_REFLINK _IOW('o', 4, struct reflink_arguments) + +/* Following definitions dedicated for ocfs2_info_request ioctls. */ +#define OCFS2_INFO_MAX_REQUEST (50) +#define OCFS2_TEXT_UUID_LEN (OCFS2_VOL_UUID_LEN * 2) + +/* Magic number of all requests */ +#define OCFS2_INFO_MAGIC (0x4F32494E) + +/* + * Always try to separate info request into small pieces to + * guarantee the backward&forward compatibility. + */ +struct ocfs2_info { + __u64 oi_requests; /* Array of __u64 pointers to requests */ + __u32 oi_count; /* Number of requests in info_requests */ + __u32 oi_pad; +}; + +struct ocfs2_info_request { +/*00*/ __u32 ir_magic; /* Magic number */ + __u32 ir_code; /* Info request code */ + __u32 ir_size; /* Size of request */ + __u32 ir_flags; /* Request flags */ +/*10*/ /* Request specific fields */ +}; + +struct ocfs2_info_clustersize { + struct ocfs2_info_request ic_req; + __u32 ic_clustersize; + __u32 ic_pad; +}; + +struct ocfs2_info_blocksize { + struct ocfs2_info_request ib_req; + __u32 ib_blocksize; + __u32 ib_pad; +}; + +struct ocfs2_info_maxslots { + struct ocfs2_info_request im_req; + __u32 im_max_slots; + __u32 im_pad; +}; + +struct ocfs2_info_label { + struct ocfs2_info_request il_req; + __u8 il_label[OCFS2_MAX_VOL_LABEL_LEN]; +} __attribute__ ((packed)); + +struct ocfs2_info_uuid { + struct ocfs2_info_request iu_req; + __u8 iu_uuid_str[OCFS2_TEXT_UUID_LEN + 1]; +} __attribute__ ((packed)); + +struct ocfs2_info_fs_features { + struct ocfs2_info_request if_req; + __u32 if_compat_features; + __u32 if_incompat_features; + __u32 if_ro_compat_features; + __u32 if_pad; +}; + +struct ocfs2_info_journal_size { + struct ocfs2_info_request ij_req; + __u64 ij_journal_size; +}; + +struct ocfs2_info_freeinode { + struct ocfs2_info_request ifi_req; + struct ocfs2_info_local_freeinode { + __u64 lfi_total; + __u64 lfi_free; + } ifi_stat[OCFS2_MAX_SLOTS]; + __u32 ifi_slotnum; /* out */ + __u32 ifi_pad; +}; + +#define OCFS2_INFO_MAX_HIST (32) + +struct ocfs2_info_freefrag { + struct ocfs2_info_request iff_req; + struct ocfs2_info_freefrag_stats { /* (out) */ + struct ocfs2_info_free_chunk_list { + __u32 fc_chunks[OCFS2_INFO_MAX_HIST]; + __u32 fc_clusters[OCFS2_INFO_MAX_HIST]; + } ffs_fc_hist; + __u32 ffs_clusters; + __u32 ffs_free_clusters; + __u32 ffs_free_chunks; + __u32 ffs_free_chunks_real; + __u32 ffs_min; /* Minimum free chunksize in clusters */ + __u32 ffs_max; + __u32 ffs_avg; + __u32 ffs_pad; + } iff_ffs; + __u32 iff_chunksize; /* chunksize in clusters(in) */ + __u32 iff_pad; +}; + +/* Codes for ocfs2_info_request */ +enum ocfs2_info_type { + OCFS2_INFO_CLUSTERSIZE = 1, + OCFS2_INFO_BLOCKSIZE, + OCFS2_INFO_MAXSLOTS, + OCFS2_INFO_LABEL, + OCFS2_INFO_UUID, + OCFS2_INFO_FS_FEATURES, + OCFS2_INFO_JOURNAL_SIZE, + OCFS2_INFO_FREEINODE, + OCFS2_INFO_FREEFRAG, + OCFS2_INFO_NUM_TYPES +}; + +/* Flags for struct ocfs2_info_request */ +/* Filled by the caller */ +#define OCFS2_INFO_FL_NON_COHERENT (0x00000001) /* Cluster coherency not + required. This is a hint. + It is up to ocfs2 whether + the request can be fulfilled + without locking. */ +/* Filled by ocfs2 */ +#define OCFS2_INFO_FL_FILLED (0x40000000) /* Filesystem understood + this request and + filled in the answer */ + +#define OCFS2_INFO_FL_ERROR (0x80000000) /* Error happened during + request handling. */ + +#define OCFS2_IOC_INFO _IOR('o', 5, struct ocfs2_info) + +struct ocfs2_move_extents { +/* All values are in bytes */ + /* in */ + __u64 me_start; /* Virtual start in the file to move */ + __u64 me_len; /* Length of the extents to be moved */ + __u64 me_goal; /* Physical offset of the goal, + it's in block unit */ + __u64 me_threshold; /* Maximum distance from goal or threshold + for auto defragmentation */ + __u64 me_flags; /* Flags for the operation: + * - auto defragmentation. + * - refcount,xattr cases. + */ + /* out */ + __u64 me_moved_len; /* Moved/defraged length */ + __u64 me_new_offset; /* Resulting physical location */ + __u32 me_reserved[2]; /* Reserved for futhure */ +}; + +#define OCFS2_MOVE_EXT_FL_AUTO_DEFRAG (0x00000001) /* Kernel manages to + claim new clusters + as the goal place + for extents moving */ +#define OCFS2_MOVE_EXT_FL_PART_DEFRAG (0x00000002) /* Allow partial extent + moving, is to make + movement less likely + to fail, may make fs + even more fragmented */ +#define OCFS2_MOVE_EXT_FL_COMPLETE (0x00000004) /* Move or defragmentation + completely gets done. + */ + +#define OCFS2_IOC_MOVE_EXT _IOW('o', 6, struct ocfs2_move_extents) + +#endif /* OCFS2_IOCTL_H */ diff --git a/fs/ocfs2/ocfs2_lockid.h b/fs/ocfs2/ocfs2_lockid.h new file mode 100644 index 000000000..9b234c03d --- /dev/null +++ b/fs/ocfs2/ocfs2_lockid.h @@ -0,0 +1,117 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * ocfs2_lockid.h + * + * Defines OCFS2 lockid bits. + * + * Copyright (C) 2002, 2005 Oracle. All rights reserved. + */ + +#ifndef OCFS2_LOCKID_H +#define OCFS2_LOCKID_H + +/* lock ids are made up in the following manner: + * name[0] --> type + * name[1-6] --> 6 pad characters, reserved for now + * name[7-22] --> block number, expressed in hex as 16 chars + * name[23-30] --> i_generation, expressed in hex 8 chars + * name[31] --> '\0' */ +#define OCFS2_LOCK_ID_MAX_LEN 32 +#define OCFS2_LOCK_ID_PAD "000000" + +#define OCFS2_DENTRY_LOCK_INO_START 18 + +enum ocfs2_lock_type { + OCFS2_LOCK_TYPE_META = 0, + OCFS2_LOCK_TYPE_DATA, + OCFS2_LOCK_TYPE_SUPER, + OCFS2_LOCK_TYPE_RENAME, + OCFS2_LOCK_TYPE_RW, + OCFS2_LOCK_TYPE_DENTRY, + OCFS2_LOCK_TYPE_OPEN, + OCFS2_LOCK_TYPE_FLOCK, + OCFS2_LOCK_TYPE_QINFO, + OCFS2_LOCK_TYPE_NFS_SYNC, + OCFS2_LOCK_TYPE_ORPHAN_SCAN, + OCFS2_LOCK_TYPE_REFCOUNT, + OCFS2_LOCK_TYPE_TRIM_FS, + OCFS2_NUM_LOCK_TYPES +}; + +static inline char ocfs2_lock_type_char(enum ocfs2_lock_type type) +{ + char c; + switch (type) { + case OCFS2_LOCK_TYPE_META: + c = 'M'; + break; + case OCFS2_LOCK_TYPE_DATA: + c = 'D'; + break; + case OCFS2_LOCK_TYPE_SUPER: + c = 'S'; + break; + case OCFS2_LOCK_TYPE_RENAME: + c = 'R'; + break; + case OCFS2_LOCK_TYPE_RW: + c = 'W'; + break; + case OCFS2_LOCK_TYPE_DENTRY: + c = 'N'; + break; + case OCFS2_LOCK_TYPE_OPEN: + c = 'O'; + break; + case OCFS2_LOCK_TYPE_FLOCK: + c = 'F'; + break; + case OCFS2_LOCK_TYPE_QINFO: + c = 'Q'; + break; + case OCFS2_LOCK_TYPE_NFS_SYNC: + c = 'Y'; + break; + case OCFS2_LOCK_TYPE_ORPHAN_SCAN: + c = 'P'; + break; + case OCFS2_LOCK_TYPE_REFCOUNT: + c = 'T'; + break; + case OCFS2_LOCK_TYPE_TRIM_FS: + c = 'I'; + break; + default: + c = '\0'; + } + + return c; +} + +static char *ocfs2_lock_type_strings[] = { + [OCFS2_LOCK_TYPE_META] = "Meta", + [OCFS2_LOCK_TYPE_DATA] = "Data", + [OCFS2_LOCK_TYPE_SUPER] = "Super", + [OCFS2_LOCK_TYPE_RENAME] = "Rename", + /* Need to differentiate from [R]ename.. serializing writes is the + * important job it does, anyway. */ + [OCFS2_LOCK_TYPE_RW] = "Write/Read", + [OCFS2_LOCK_TYPE_DENTRY] = "Dentry", + [OCFS2_LOCK_TYPE_OPEN] = "Open", + [OCFS2_LOCK_TYPE_FLOCK] = "Flock", + [OCFS2_LOCK_TYPE_QINFO] = "Quota", + [OCFS2_LOCK_TYPE_NFS_SYNC] = "NFSSync", + [OCFS2_LOCK_TYPE_ORPHAN_SCAN] = "OrphanScan", + [OCFS2_LOCK_TYPE_REFCOUNT] = "Refcount", + [OCFS2_LOCK_TYPE_TRIM_FS] = "TrimFs", +}; + +static inline const char *ocfs2_lock_type_string(enum ocfs2_lock_type type) +{ +#ifdef __KERNEL__ + BUG_ON(type >= OCFS2_NUM_LOCK_TYPES); +#endif + return ocfs2_lock_type_strings[type]; +} + +#endif /* OCFS2_LOCKID_H */ diff --git a/fs/ocfs2/ocfs2_lockingver.h b/fs/ocfs2/ocfs2_lockingver.h new file mode 100644 index 000000000..31a5e1619 --- /dev/null +++ b/fs/ocfs2/ocfs2_lockingver.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * ocfs2_lockingver.h + * + * Defines OCFS2 Locking version values. + * + * Copyright (C) 2008 Oracle. All rights reserved. + */ + +#ifndef OCFS2_LOCKINGVER_H +#define OCFS2_LOCKINGVER_H + +/* + * The protocol version for ocfs2 cluster locking. See dlmglue.c for + * more details. + * + * 1.0 - Initial locking version from ocfs2 1.4. + */ +#define OCFS2_LOCKING_PROTOCOL_MAJOR 1 +#define OCFS2_LOCKING_PROTOCOL_MINOR 0 + +#endif /* OCFS2_LOCKINGVER_H */ diff --git a/fs/ocfs2/ocfs2_trace.h b/fs/ocfs2/ocfs2_trace.h new file mode 100644 index 000000000..6c2c97a98 --- /dev/null +++ b/fs/ocfs2/ocfs2_trace.h @@ -0,0 +1,2764 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM ocfs2 + +#if !defined(_TRACE_OCFS2_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_OCFS2_H + +#include + +DECLARE_EVENT_CLASS(ocfs2__int, + TP_PROTO(int num), + TP_ARGS(num), + TP_STRUCT__entry( + __field(int, num) + ), + TP_fast_assign( + __entry->num = num; + ), + TP_printk("%d", __entry->num) +); + +#define DEFINE_OCFS2_INT_EVENT(name) \ +DEFINE_EVENT(ocfs2__int, name, \ + TP_PROTO(int num), \ + TP_ARGS(num)) + +DECLARE_EVENT_CLASS(ocfs2__uint, + TP_PROTO(unsigned int num), + TP_ARGS(num), + TP_STRUCT__entry( + __field( unsigned int, num ) + ), + TP_fast_assign( + __entry->num = num; + ), + TP_printk("%u", __entry->num) +); + +#define DEFINE_OCFS2_UINT_EVENT(name) \ +DEFINE_EVENT(ocfs2__uint, name, \ + TP_PROTO(unsigned int num), \ + TP_ARGS(num)) + +DECLARE_EVENT_CLASS(ocfs2__ull, + TP_PROTO(unsigned long long blkno), + TP_ARGS(blkno), + TP_STRUCT__entry( + __field(unsigned long long, blkno) + ), + TP_fast_assign( + __entry->blkno = blkno; + ), + TP_printk("%llu", __entry->blkno) +); + +#define DEFINE_OCFS2_ULL_EVENT(name) \ +DEFINE_EVENT(ocfs2__ull, name, \ + TP_PROTO(unsigned long long num), \ + TP_ARGS(num)) + +DECLARE_EVENT_CLASS(ocfs2__pointer, + TP_PROTO(void *pointer), + TP_ARGS(pointer), + TP_STRUCT__entry( + __field(void *, pointer) + ), + TP_fast_assign( + __entry->pointer = pointer; + ), + TP_printk("%p", __entry->pointer) +); + +#define DEFINE_OCFS2_POINTER_EVENT(name) \ +DEFINE_EVENT(ocfs2__pointer, name, \ + TP_PROTO(void *pointer), \ + TP_ARGS(pointer)) + +DECLARE_EVENT_CLASS(ocfs2__string, + TP_PROTO(const char *name), + TP_ARGS(name), + TP_STRUCT__entry( + __string(name,name) + ), + TP_fast_assign( + __assign_str(name); + ), + TP_printk("%s", __get_str(name)) +); + +#define DEFINE_OCFS2_STRING_EVENT(name) \ +DEFINE_EVENT(ocfs2__string, name, \ + TP_PROTO(const char *name), \ + TP_ARGS(name)) + +DECLARE_EVENT_CLASS(ocfs2__int_int, + TP_PROTO(int value1, int value2), + TP_ARGS(value1, value2), + TP_STRUCT__entry( + __field(int, value1) + __field(int, value2) + ), + TP_fast_assign( + __entry->value1 = value1; + __entry->value2 = value2; + ), + TP_printk("%d %d", __entry->value1, __entry->value2) +); + +#define DEFINE_OCFS2_INT_INT_EVENT(name) \ +DEFINE_EVENT(ocfs2__int_int, name, \ + TP_PROTO(int val1, int val2), \ + TP_ARGS(val1, val2)) + +DECLARE_EVENT_CLASS(ocfs2__uint_int, + TP_PROTO(unsigned int value1, int value2), + TP_ARGS(value1, value2), + TP_STRUCT__entry( + __field(unsigned int, value1) + __field(int, value2) + ), + TP_fast_assign( + __entry->value1 = value1; + __entry->value2 = value2; + ), + TP_printk("%u %d", __entry->value1, __entry->value2) +); + +#define DEFINE_OCFS2_UINT_INT_EVENT(name) \ +DEFINE_EVENT(ocfs2__uint_int, name, \ + TP_PROTO(unsigned int val1, int val2), \ + TP_ARGS(val1, val2)) + +DECLARE_EVENT_CLASS(ocfs2__uint_uint, + TP_PROTO(unsigned int value1, unsigned int value2), + TP_ARGS(value1, value2), + TP_STRUCT__entry( + __field(unsigned int, value1) + __field(unsigned int, value2) + ), + TP_fast_assign( + __entry->value1 = value1; + __entry->value2 = value2; + ), + TP_printk("%u %u", __entry->value1, __entry->value2) +); + +#define DEFINE_OCFS2_UINT_UINT_EVENT(name) \ +DEFINE_EVENT(ocfs2__uint_uint, name, \ + TP_PROTO(unsigned int val1, unsigned int val2), \ + TP_ARGS(val1, val2)) + +DECLARE_EVENT_CLASS(ocfs2__ull_uint, + TP_PROTO(unsigned long long value1, unsigned int value2), + TP_ARGS(value1, value2), + TP_STRUCT__entry( + __field(unsigned long long, value1) + __field(unsigned int, value2) + ), + TP_fast_assign( + __entry->value1 = value1; + __entry->value2 = value2; + ), + TP_printk("%llu %u", __entry->value1, __entry->value2) +); + +#define DEFINE_OCFS2_ULL_UINT_EVENT(name) \ +DEFINE_EVENT(ocfs2__ull_uint, name, \ + TP_PROTO(unsigned long long val1, unsigned int val2), \ + TP_ARGS(val1, val2)) + +DECLARE_EVENT_CLASS(ocfs2__ull_int, + TP_PROTO(unsigned long long value1, int value2), + TP_ARGS(value1, value2), + TP_STRUCT__entry( + __field(unsigned long long, value1) + __field(int, value2) + ), + TP_fast_assign( + __entry->value1 = value1; + __entry->value2 = value2; + ), + TP_printk("%llu %d", __entry->value1, __entry->value2) +); + +#define DEFINE_OCFS2_ULL_INT_EVENT(name) \ +DEFINE_EVENT(ocfs2__ull_int, name, \ + TP_PROTO(unsigned long long val1, int val2), \ + TP_ARGS(val1, val2)) + +DECLARE_EVENT_CLASS(ocfs2__ull_ull, + TP_PROTO(unsigned long long value1, unsigned long long value2), + TP_ARGS(value1, value2), + TP_STRUCT__entry( + __field(unsigned long long, value1) + __field(unsigned long long, value2) + ), + TP_fast_assign( + __entry->value1 = value1; + __entry->value2 = value2; + ), + TP_printk("%llu %llu", __entry->value1, __entry->value2) +); + +#define DEFINE_OCFS2_ULL_ULL_EVENT(name) \ +DEFINE_EVENT(ocfs2__ull_ull, name, \ + TP_PROTO(unsigned long long val1, unsigned long long val2), \ + TP_ARGS(val1, val2)) + +DECLARE_EVENT_CLASS(ocfs2__ull_ull_uint, + TP_PROTO(unsigned long long value1, + unsigned long long value2, unsigned int value3), + TP_ARGS(value1, value2, value3), + TP_STRUCT__entry( + __field(unsigned long long, value1) + __field(unsigned long long, value2) + __field(unsigned int, value3) + ), + TP_fast_assign( + __entry->value1 = value1; + __entry->value2 = value2; + __entry->value3 = value3; + ), + TP_printk("%llu %llu %u", + __entry->value1, __entry->value2, __entry->value3) +); + +#define DEFINE_OCFS2_ULL_ULL_UINT_EVENT(name) \ +DEFINE_EVENT(ocfs2__ull_ull_uint, name, \ + TP_PROTO(unsigned long long val1, \ + unsigned long long val2, unsigned int val3), \ + TP_ARGS(val1, val2, val3)) + +DECLARE_EVENT_CLASS(ocfs2__ull_uint_uint, + TP_PROTO(unsigned long long value1, + unsigned int value2, unsigned int value3), + TP_ARGS(value1, value2, value3), + TP_STRUCT__entry( + __field(unsigned long long, value1) + __field(unsigned int, value2) + __field(unsigned int, value3) + ), + TP_fast_assign( + __entry->value1 = value1; + __entry->value2 = value2; + __entry->value3 = value3; + ), + TP_printk("%llu %u %u", __entry->value1, + __entry->value2, __entry->value3) +); + +#define DEFINE_OCFS2_ULL_UINT_UINT_EVENT(name) \ +DEFINE_EVENT(ocfs2__ull_uint_uint, name, \ + TP_PROTO(unsigned long long val1, \ + unsigned int val2, unsigned int val3), \ + TP_ARGS(val1, val2, val3)) + +DECLARE_EVENT_CLASS(ocfs2__uint_uint_uint, + TP_PROTO(unsigned int value1, unsigned int value2, + unsigned int value3), + TP_ARGS(value1, value2, value3), + TP_STRUCT__entry( + __field( unsigned int, value1 ) + __field( unsigned int, value2 ) + __field( unsigned int, value3 ) + ), + TP_fast_assign( + __entry->value1 = value1; + __entry->value2 = value2; + __entry->value3 = value3; + ), + TP_printk("%u %u %u", __entry->value1, __entry->value2, __entry->value3) +); + +#define DEFINE_OCFS2_UINT_UINT_UINT_EVENT(name) \ +DEFINE_EVENT(ocfs2__uint_uint_uint, name, \ + TP_PROTO(unsigned int value1, unsigned int value2, \ + unsigned int value3), \ + TP_ARGS(value1, value2, value3)) + +DECLARE_EVENT_CLASS(ocfs2__ull_ull_ull, + TP_PROTO(unsigned long long value1, + unsigned long long value2, unsigned long long value3), + TP_ARGS(value1, value2, value3), + TP_STRUCT__entry( + __field(unsigned long long, value1) + __field(unsigned long long, value2) + __field(unsigned long long, value3) + ), + TP_fast_assign( + __entry->value1 = value1; + __entry->value2 = value2; + __entry->value3 = value3; + ), + TP_printk("%llu %llu %llu", + __entry->value1, __entry->value2, __entry->value3) +); + +#define DEFINE_OCFS2_ULL_ULL_ULL_EVENT(name) \ +DEFINE_EVENT(ocfs2__ull_ull_ull, name, \ + TP_PROTO(unsigned long long value1, unsigned long long value2, \ + unsigned long long value3), \ + TP_ARGS(value1, value2, value3)) + +DECLARE_EVENT_CLASS(ocfs2__ull_int_int_int, + TP_PROTO(unsigned long long ull, int value1, int value2, int value3), + TP_ARGS(ull, value1, value2, value3), + TP_STRUCT__entry( + __field( unsigned long long, ull ) + __field( int, value1 ) + __field( int, value2 ) + __field( int, value3 ) + ), + TP_fast_assign( + __entry->ull = ull; + __entry->value1 = value1; + __entry->value2 = value2; + __entry->value3 = value3; + ), + TP_printk("%llu %d %d %d", + __entry->ull, __entry->value1, + __entry->value2, __entry->value3) +); + +#define DEFINE_OCFS2_ULL_INT_INT_INT_EVENT(name) \ +DEFINE_EVENT(ocfs2__ull_int_int_int, name, \ + TP_PROTO(unsigned long long ull, int value1, \ + int value2, int value3), \ + TP_ARGS(ull, value1, value2, value3)) + +DECLARE_EVENT_CLASS(ocfs2__ull_uint_uint_uint, + TP_PROTO(unsigned long long ull, unsigned int value1, + unsigned int value2, unsigned int value3), + TP_ARGS(ull, value1, value2, value3), + TP_STRUCT__entry( + __field(unsigned long long, ull) + __field(unsigned int, value1) + __field(unsigned int, value2) + __field(unsigned int, value3) + ), + TP_fast_assign( + __entry->ull = ull; + __entry->value1 = value1; + __entry->value2 = value2; + __entry->value3 = value3; + ), + TP_printk("%llu %u %u %u", + __entry->ull, __entry->value1, + __entry->value2, __entry->value3) +); + +#define DEFINE_OCFS2_ULL_UINT_UINT_UINT_EVENT(name) \ +DEFINE_EVENT(ocfs2__ull_uint_uint_uint, name, \ + TP_PROTO(unsigned long long ull, unsigned int value1, \ + unsigned int value2, unsigned int value3), \ + TP_ARGS(ull, value1, value2, value3)) + +DECLARE_EVENT_CLASS(ocfs2__ull_ull_uint_uint, + TP_PROTO(unsigned long long value1, unsigned long long value2, + unsigned int value3, unsigned int value4), + TP_ARGS(value1, value2, value3, value4), + TP_STRUCT__entry( + __field(unsigned long long, value1) + __field(unsigned long long, value2) + __field(unsigned int, value3) + __field(unsigned int, value4) + ), + TP_fast_assign( + __entry->value1 = value1; + __entry->value2 = value2; + __entry->value3 = value3; + __entry->value4 = value4; + ), + TP_printk("%llu %llu %u %u", + __entry->value1, __entry->value2, + __entry->value3, __entry->value4) +); + +#define DEFINE_OCFS2_ULL_ULL_UINT_UINT_EVENT(name) \ +DEFINE_EVENT(ocfs2__ull_ull_uint_uint, name, \ + TP_PROTO(unsigned long long ull, unsigned long long ull1, \ + unsigned int value2, unsigned int value3), \ + TP_ARGS(ull, ull1, value2, value3)) + +/* Trace events for fs/ocfs2/alloc.c. */ +DECLARE_EVENT_CLASS(ocfs2__btree_ops, + TP_PROTO(unsigned long long owner,\ + unsigned int value1, unsigned int value2), + TP_ARGS(owner, value1, value2), + TP_STRUCT__entry( + __field(unsigned long long, owner) + __field(unsigned int, value1) + __field(unsigned int, value2) + ), + TP_fast_assign( + __entry->owner = owner; + __entry->value1 = value1; + __entry->value2 = value2; + ), + TP_printk("%llu %u %u", + __entry->owner, __entry->value1, __entry->value2) +); + +#define DEFINE_OCFS2_BTREE_EVENT(name) \ +DEFINE_EVENT(ocfs2__btree_ops, name, \ + TP_PROTO(unsigned long long owner, \ + unsigned int value1, unsigned int value2), \ + TP_ARGS(owner, value1, value2)) + +DEFINE_OCFS2_BTREE_EVENT(ocfs2_adjust_rightmost_branch); + +DEFINE_OCFS2_BTREE_EVENT(ocfs2_rotate_tree_right); + +DEFINE_OCFS2_BTREE_EVENT(ocfs2_append_rec_to_path); + +DEFINE_OCFS2_BTREE_EVENT(ocfs2_insert_extent_start); + +DEFINE_OCFS2_BTREE_EVENT(ocfs2_add_clusters_in_btree); + +DEFINE_OCFS2_INT_EVENT(ocfs2_num_free_extents); + +DEFINE_OCFS2_INT_EVENT(ocfs2_complete_edge_insert); + +TRACE_EVENT(ocfs2_grow_tree, + TP_PROTO(unsigned long long owner, int depth), + TP_ARGS(owner, depth), + TP_STRUCT__entry( + __field(unsigned long long, owner) + __field(int, depth) + ), + TP_fast_assign( + __entry->owner = owner; + __entry->depth = depth; + ), + TP_printk("%llu %d", __entry->owner, __entry->depth) +); + +TRACE_EVENT(ocfs2_rotate_subtree, + TP_PROTO(int subtree_root, unsigned long long blkno, + int depth), + TP_ARGS(subtree_root, blkno, depth), + TP_STRUCT__entry( + __field(int, subtree_root) + __field(unsigned long long, blkno) + __field(int, depth) + ), + TP_fast_assign( + __entry->subtree_root = subtree_root; + __entry->blkno = blkno; + __entry->depth = depth; + ), + TP_printk("%d %llu %d", __entry->subtree_root, + __entry->blkno, __entry->depth) +); + +TRACE_EVENT(ocfs2_insert_extent, + TP_PROTO(unsigned int ins_appending, unsigned int ins_contig, + int ins_contig_index, int free_records, int ins_tree_depth), + TP_ARGS(ins_appending, ins_contig, ins_contig_index, free_records, + ins_tree_depth), + TP_STRUCT__entry( + __field(unsigned int, ins_appending) + __field(unsigned int, ins_contig) + __field(int, ins_contig_index) + __field(int, free_records) + __field(int, ins_tree_depth) + ), + TP_fast_assign( + __entry->ins_appending = ins_appending; + __entry->ins_contig = ins_contig; + __entry->ins_contig_index = ins_contig_index; + __entry->free_records = free_records; + __entry->ins_tree_depth = ins_tree_depth; + ), + TP_printk("%u %u %d %d %d", + __entry->ins_appending, __entry->ins_contig, + __entry->ins_contig_index, __entry->free_records, + __entry->ins_tree_depth) +); + +TRACE_EVENT(ocfs2_split_extent, + TP_PROTO(int split_index, unsigned int c_contig_type, + unsigned int c_has_empty_extent, + unsigned int c_split_covers_rec), + TP_ARGS(split_index, c_contig_type, + c_has_empty_extent, c_split_covers_rec), + TP_STRUCT__entry( + __field(int, split_index) + __field(unsigned int, c_contig_type) + __field(unsigned int, c_has_empty_extent) + __field(unsigned int, c_split_covers_rec) + ), + TP_fast_assign( + __entry->split_index = split_index; + __entry->c_contig_type = c_contig_type; + __entry->c_has_empty_extent = c_has_empty_extent; + __entry->c_split_covers_rec = c_split_covers_rec; + ), + TP_printk("%d %u %u %u", __entry->split_index, __entry->c_contig_type, + __entry->c_has_empty_extent, __entry->c_split_covers_rec) +); + +TRACE_EVENT(ocfs2_remove_extent, + TP_PROTO(unsigned long long owner, unsigned int cpos, + unsigned int len, int index, + unsigned int e_cpos, unsigned int clusters), + TP_ARGS(owner, cpos, len, index, e_cpos, clusters), + TP_STRUCT__entry( + __field(unsigned long long, owner) + __field(unsigned int, cpos) + __field(unsigned int, len) + __field(int, index) + __field(unsigned int, e_cpos) + __field(unsigned int, clusters) + ), + TP_fast_assign( + __entry->owner = owner; + __entry->cpos = cpos; + __entry->len = len; + __entry->index = index; + __entry->e_cpos = e_cpos; + __entry->clusters = clusters; + ), + TP_printk("%llu %u %u %d %u %u", + __entry->owner, __entry->cpos, __entry->len, __entry->index, + __entry->e_cpos, __entry->clusters) +); + +TRACE_EVENT(ocfs2_commit_truncate, + TP_PROTO(unsigned long long ino, unsigned int new_cpos, + unsigned int clusters, unsigned int depth), + TP_ARGS(ino, new_cpos, clusters, depth), + TP_STRUCT__entry( + __field(unsigned long long, ino) + __field(unsigned int, new_cpos) + __field(unsigned int, clusters) + __field(unsigned int, depth) + ), + TP_fast_assign( + __entry->ino = ino; + __entry->new_cpos = new_cpos; + __entry->clusters = clusters; + __entry->depth = depth; + ), + TP_printk("%llu %u %u %u", + __entry->ino, __entry->new_cpos, + __entry->clusters, __entry->depth) +); + +TRACE_EVENT(ocfs2_validate_extent_block, + TP_PROTO(unsigned long long blkno), + TP_ARGS(blkno), + TP_STRUCT__entry( + __field(unsigned long long, blkno) + ), + TP_fast_assign( + __entry->blkno = blkno; + ), + TP_printk("%llu ", __entry->blkno) +); + +TRACE_EVENT(ocfs2_rotate_leaf, + TP_PROTO(unsigned int insert_cpos, int insert_index, + int has_empty, int next_free, + unsigned int l_count), + TP_ARGS(insert_cpos, insert_index, has_empty, + next_free, l_count), + TP_STRUCT__entry( + __field(unsigned int, insert_cpos) + __field(int, insert_index) + __field(int, has_empty) + __field(int, next_free) + __field(unsigned int, l_count) + ), + TP_fast_assign( + __entry->insert_cpos = insert_cpos; + __entry->insert_index = insert_index; + __entry->has_empty = has_empty; + __entry->next_free = next_free; + __entry->l_count = l_count; + ), + TP_printk("%u %d %d %d %u", __entry->insert_cpos, + __entry->insert_index, __entry->has_empty, + __entry->next_free, __entry->l_count) +); + +TRACE_EVENT(ocfs2_add_clusters_in_btree_ret, + TP_PROTO(int status, int reason, int err), + TP_ARGS(status, reason, err), + TP_STRUCT__entry( + __field(int, status) + __field(int, reason) + __field(int, err) + ), + TP_fast_assign( + __entry->status = status; + __entry->reason = reason; + __entry->err = err; + ), + TP_printk("%d %d %d", __entry->status, + __entry->reason, __entry->err) +); + +TRACE_EVENT(ocfs2_mark_extent_written, + TP_PROTO(unsigned long long owner, unsigned int cpos, + unsigned int len, unsigned int phys), + TP_ARGS(owner, cpos, len, phys), + TP_STRUCT__entry( + __field(unsigned long long, owner) + __field(unsigned int, cpos) + __field(unsigned int, len) + __field(unsigned int, phys) + ), + TP_fast_assign( + __entry->owner = owner; + __entry->cpos = cpos; + __entry->len = len; + __entry->phys = phys; + ), + TP_printk("%llu %u %u %u", + __entry->owner, __entry->cpos, + __entry->len, __entry->phys) +); + +DECLARE_EVENT_CLASS(ocfs2__truncate_log_ops, + TP_PROTO(unsigned long long blkno, int index, + unsigned int start, unsigned int num), + TP_ARGS(blkno, index, start, num), + TP_STRUCT__entry( + __field(unsigned long long, blkno) + __field(int, index) + __field(unsigned int, start) + __field(unsigned int, num) + ), + TP_fast_assign( + __entry->blkno = blkno; + __entry->index = index; + __entry->start = start; + __entry->num = num; + ), + TP_printk("%llu %d %u %u", + __entry->blkno, __entry->index, + __entry->start, __entry->num) +); + +#define DEFINE_OCFS2_TRUNCATE_LOG_OPS_EVENT(name) \ +DEFINE_EVENT(ocfs2__truncate_log_ops, name, \ + TP_PROTO(unsigned long long blkno, int index, \ + unsigned int start, unsigned int num), \ + TP_ARGS(blkno, index, start, num)) + +DEFINE_OCFS2_TRUNCATE_LOG_OPS_EVENT(ocfs2_truncate_log_append); + +DEFINE_OCFS2_TRUNCATE_LOG_OPS_EVENT(ocfs2_replay_truncate_records); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_flush_truncate_log); + +DEFINE_OCFS2_INT_EVENT(ocfs2_begin_truncate_log_recovery); + +DEFINE_OCFS2_INT_EVENT(ocfs2_truncate_log_recovery_num); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_complete_truncate_log_recovery); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_free_cached_blocks); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_cache_cluster_dealloc); + +DEFINE_OCFS2_INT_INT_EVENT(ocfs2_run_deallocs); + +TRACE_EVENT(ocfs2_cache_block_dealloc, + TP_PROTO(int type, int slot, unsigned long long suballoc, + unsigned long long blkno, unsigned int bit), + TP_ARGS(type, slot, suballoc, blkno, bit), + TP_STRUCT__entry( + __field(int, type) + __field(int, slot) + __field(unsigned long long, suballoc) + __field(unsigned long long, blkno) + __field(unsigned int, bit) + ), + TP_fast_assign( + __entry->type = type; + __entry->slot = slot; + __entry->suballoc = suballoc; + __entry->blkno = blkno; + __entry->bit = bit; + ), + TP_printk("%d %d %llu %llu %u", + __entry->type, __entry->slot, __entry->suballoc, + __entry->blkno, __entry->bit) +); + +TRACE_EVENT(ocfs2_trim_extent, + TP_PROTO(struct super_block *sb, unsigned long long blk, + unsigned long long count), + TP_ARGS(sb, blk, count), + TP_STRUCT__entry( + __field(int, dev_major) + __field(int, dev_minor) + __field(unsigned long long, blk) + __field(__u64, count) + ), + TP_fast_assign( + __entry->dev_major = MAJOR(sb->s_dev); + __entry->dev_minor = MINOR(sb->s_dev); + __entry->blk = blk; + __entry->count = count; + ), + TP_printk("%d %d %llu %llu", + __entry->dev_major, __entry->dev_minor, + __entry->blk, __entry->count) +); + +DEFINE_OCFS2_ULL_UINT_UINT_UINT_EVENT(ocfs2_trim_group); + +DEFINE_OCFS2_ULL_ULL_ULL_EVENT(ocfs2_trim_mainbm); + +DEFINE_OCFS2_ULL_ULL_ULL_EVENT(ocfs2_trim_fs); + +/* End of trace events for fs/ocfs2/alloc.c. */ + +/* Trace events for fs/ocfs2/localalloc.c. */ + +DEFINE_OCFS2_UINT_UINT_UINT_EVENT(ocfs2_la_set_sizes); + +DEFINE_OCFS2_ULL_INT_INT_INT_EVENT(ocfs2_alloc_should_use_local); + +DEFINE_OCFS2_INT_EVENT(ocfs2_load_local_alloc); + +DEFINE_OCFS2_INT_EVENT(ocfs2_begin_local_alloc_recovery); + +DEFINE_OCFS2_ULL_INT_INT_INT_EVENT(ocfs2_reserve_local_alloc_bits); + +DEFINE_OCFS2_UINT_EVENT(ocfs2_local_alloc_count_bits); + +DEFINE_OCFS2_INT_INT_EVENT(ocfs2_local_alloc_find_clear_bits_search_bitmap); + +DEFINE_OCFS2_ULL_INT_INT_INT_EVENT(ocfs2_local_alloc_find_clear_bits); + +DEFINE_OCFS2_INT_INT_EVENT(ocfs2_sync_local_to_main); + +TRACE_EVENT(ocfs2_sync_local_to_main_free, + TP_PROTO(int count, int bit, unsigned long long start_blk, + unsigned long long blkno), + TP_ARGS(count, bit, start_blk, blkno), + TP_STRUCT__entry( + __field(int, count) + __field(int, bit) + __field(unsigned long long, start_blk) + __field(unsigned long long, blkno) + ), + TP_fast_assign( + __entry->count = count; + __entry->bit = bit; + __entry->start_blk = start_blk; + __entry->blkno = blkno; + ), + TP_printk("%d %d %llu %llu", + __entry->count, __entry->bit, __entry->start_blk, + __entry->blkno) +); + +DEFINE_OCFS2_INT_INT_EVENT(ocfs2_local_alloc_new_window); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_local_alloc_new_window_result); + +/* End of trace events for fs/ocfs2/localalloc.c. */ + +/* Trace events for fs/ocfs2/resize.c. */ + +DEFINE_OCFS2_UINT_UINT_EVENT(ocfs2_update_last_group_and_inode); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_group_extend); + +DEFINE_OCFS2_ULL_UINT_UINT_UINT_EVENT(ocfs2_group_add); + +/* End of trace events for fs/ocfs2/resize.c. */ + +/* Trace events for fs/ocfs2/suballoc.c. */ + +DEFINE_OCFS2_ULL_EVENT(ocfs2_validate_group_descriptor); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_block_group_alloc_contig); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_block_group_alloc_discontig); + +DEFINE_OCFS2_ULL_EVENT(ocfs2_block_group_alloc); + +DEFINE_OCFS2_UINT_UINT_EVENT(ocfs2_reserve_suballoc_bits_nospc); + +DEFINE_OCFS2_UINT_UINT_UINT_EVENT(ocfs2_reserve_suballoc_bits_no_new_group); + +DEFINE_OCFS2_ULL_EVENT(ocfs2_reserve_new_inode_new_group); + +DEFINE_OCFS2_UINT_UINT_EVENT(ocfs2_block_group_set_bits); + +TRACE_EVENT(ocfs2_relink_block_group, + TP_PROTO(unsigned long long i_blkno, unsigned int chain, + unsigned long long bg_blkno, + unsigned long long prev_blkno), + TP_ARGS(i_blkno, chain, bg_blkno, prev_blkno), + TP_STRUCT__entry( + __field(unsigned long long, i_blkno) + __field(unsigned int, chain) + __field(unsigned long long, bg_blkno) + __field(unsigned long long, prev_blkno) + ), + TP_fast_assign( + __entry->i_blkno = i_blkno; + __entry->chain = chain; + __entry->bg_blkno = bg_blkno; + __entry->prev_blkno = prev_blkno; + ), + TP_printk("%llu %u %llu %llu", + __entry->i_blkno, __entry->chain, __entry->bg_blkno, + __entry->prev_blkno) +); + +DEFINE_OCFS2_ULL_UINT_UINT_UINT_EVENT(ocfs2_cluster_group_search_wrong_max_bits); + +DEFINE_OCFS2_ULL_ULL_EVENT(ocfs2_cluster_group_search_max_block); + +DEFINE_OCFS2_ULL_ULL_EVENT(ocfs2_block_group_search_max_block); + +DEFINE_OCFS2_ULL_UINT_UINT_EVENT(ocfs2_search_chain_begin); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_search_chain_succ); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_search_chain_end); + +DEFINE_OCFS2_UINT_EVENT(ocfs2_claim_suballoc_bits); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_claim_new_inode_at_loc); + +DEFINE_OCFS2_UINT_UINT_EVENT(ocfs2_block_group_clear_bits); + +TRACE_EVENT(ocfs2_free_suballoc_bits, + TP_PROTO(unsigned long long inode, unsigned long long group, + unsigned int start_bit, unsigned int count), + TP_ARGS(inode, group, start_bit, count), + TP_STRUCT__entry( + __field(unsigned long long, inode) + __field(unsigned long long, group) + __field(unsigned int, start_bit) + __field(unsigned int, count) + ), + TP_fast_assign( + __entry->inode = inode; + __entry->group = group; + __entry->start_bit = start_bit; + __entry->count = count; + ), + TP_printk("%llu %llu %u %u", __entry->inode, __entry->group, + __entry->start_bit, __entry->count) +); + +TRACE_EVENT(ocfs2_free_clusters, + TP_PROTO(unsigned long long bg_blkno, unsigned long long start_blk, + unsigned int start_bit, unsigned int count), + TP_ARGS(bg_blkno, start_blk, start_bit, count), + TP_STRUCT__entry( + __field(unsigned long long, bg_blkno) + __field(unsigned long long, start_blk) + __field(unsigned int, start_bit) + __field(unsigned int, count) + ), + TP_fast_assign( + __entry->bg_blkno = bg_blkno; + __entry->start_blk = start_blk; + __entry->start_bit = start_bit; + __entry->count = count; + ), + TP_printk("%llu %llu %u %u", __entry->bg_blkno, __entry->start_blk, + __entry->start_bit, __entry->count) +); + +DEFINE_OCFS2_ULL_EVENT(ocfs2_get_suballoc_slot_bit); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_test_suballoc_bit); + +DEFINE_OCFS2_ULL_EVENT(ocfs2_test_inode_bit); + +/* End of trace events for fs/ocfs2/suballoc.c. */ + +/* Trace events for fs/ocfs2/refcounttree.c. */ + +DEFINE_OCFS2_ULL_EVENT(ocfs2_validate_refcount_block); + +DEFINE_OCFS2_ULL_EVENT(ocfs2_purge_refcount_trees); + +DEFINE_OCFS2_ULL_EVENT(ocfs2_create_refcount_tree); + +DEFINE_OCFS2_ULL_EVENT(ocfs2_create_refcount_tree_blkno); + +DEFINE_OCFS2_ULL_INT_INT_INT_EVENT(ocfs2_change_refcount_rec); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_expand_inline_ref_root); + +DEFINE_OCFS2_ULL_UINT_UINT_EVENT(ocfs2_divide_leaf_refcount_block); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_new_leaf_refcount_block); + +DECLARE_EVENT_CLASS(ocfs2__refcount_tree_ops, + TP_PROTO(unsigned long long blkno, int index, + unsigned long long cpos, + unsigned int clusters, unsigned int refcount), + TP_ARGS(blkno, index, cpos, clusters, refcount), + TP_STRUCT__entry( + __field(unsigned long long, blkno) + __field(int, index) + __field(unsigned long long, cpos) + __field(unsigned int, clusters) + __field(unsigned int, refcount) + ), + TP_fast_assign( + __entry->blkno = blkno; + __entry->index = index; + __entry->cpos = cpos; + __entry->clusters = clusters; + __entry->refcount = refcount; + ), + TP_printk("%llu %d %llu %u %u", __entry->blkno, __entry->index, + __entry->cpos, __entry->clusters, __entry->refcount) +); + +#define DEFINE_OCFS2_REFCOUNT_TREE_OPS_EVENT(name) \ +DEFINE_EVENT(ocfs2__refcount_tree_ops, name, \ + TP_PROTO(unsigned long long blkno, int index, \ + unsigned long long cpos, \ + unsigned int count, unsigned int refcount), \ + TP_ARGS(blkno, index, cpos, count, refcount)) + +DEFINE_OCFS2_REFCOUNT_TREE_OPS_EVENT(ocfs2_insert_refcount_rec); + +TRACE_EVENT(ocfs2_split_refcount_rec, + TP_PROTO(unsigned long long cpos, + unsigned int clusters, unsigned int refcount, + unsigned long long split_cpos, + unsigned int split_clusters, unsigned int split_refcount), + TP_ARGS(cpos, clusters, refcount, + split_cpos, split_clusters, split_refcount), + TP_STRUCT__entry( + __field(unsigned long long, cpos) + __field(unsigned int, clusters) + __field(unsigned int, refcount) + __field(unsigned long long, split_cpos) + __field(unsigned int, split_clusters) + __field(unsigned int, split_refcount) + ), + TP_fast_assign( + __entry->cpos = cpos; + __entry->clusters = clusters; + __entry->refcount = refcount; + __entry->split_cpos = split_cpos; + __entry->split_clusters = split_clusters; + __entry->split_refcount = split_refcount; + ), + TP_printk("%llu %u %u %llu %u %u", + __entry->cpos, __entry->clusters, __entry->refcount, + __entry->split_cpos, __entry->split_clusters, + __entry->split_refcount) +); + +DEFINE_OCFS2_REFCOUNT_TREE_OPS_EVENT(ocfs2_split_refcount_rec_insert); + +DEFINE_OCFS2_ULL_ULL_UINT_EVENT(ocfs2_increase_refcount_begin); + +DEFINE_OCFS2_ULL_UINT_UINT_EVENT(ocfs2_increase_refcount_change); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_increase_refcount_insert); + +DEFINE_OCFS2_ULL_UINT_UINT_EVENT(ocfs2_increase_refcount_split); + +DEFINE_OCFS2_ULL_ULL_UINT_EVENT(ocfs2_remove_refcount_extent); + +DEFINE_OCFS2_ULL_EVENT(ocfs2_restore_refcount_block); + +DEFINE_OCFS2_ULL_ULL_UINT_EVENT(ocfs2_decrease_refcount_rec); + +TRACE_EVENT(ocfs2_decrease_refcount, + TP_PROTO(unsigned long long owner, + unsigned long long cpos, + unsigned int len, int delete), + TP_ARGS(owner, cpos, len, delete), + TP_STRUCT__entry( + __field(unsigned long long, owner) + __field(unsigned long long, cpos) + __field(unsigned int, len) + __field(int, delete) + ), + TP_fast_assign( + __entry->owner = owner; + __entry->cpos = cpos; + __entry->len = len; + __entry->delete = delete; + ), + TP_printk("%llu %llu %u %d", + __entry->owner, __entry->cpos, __entry->len, __entry->delete) +); + +DEFINE_OCFS2_ULL_UINT_UINT_UINT_EVENT(ocfs2_mark_extent_refcounted); + +DEFINE_OCFS2_ULL_UINT_UINT_UINT_EVENT(ocfs2_calc_refcount_meta_credits); + +TRACE_EVENT(ocfs2_calc_refcount_meta_credits_iterate, + TP_PROTO(int recs_add, unsigned long long cpos, + unsigned int clusters, unsigned long long r_cpos, + unsigned int r_clusters, unsigned int refcount, int index), + TP_ARGS(recs_add, cpos, clusters, r_cpos, r_clusters, refcount, index), + TP_STRUCT__entry( + __field(int, recs_add) + __field(unsigned long long, cpos) + __field(unsigned int, clusters) + __field(unsigned long long, r_cpos) + __field(unsigned int, r_clusters) + __field(unsigned int, refcount) + __field(int, index) + ), + TP_fast_assign( + __entry->recs_add = recs_add; + __entry->cpos = cpos; + __entry->clusters = clusters; + __entry->r_cpos = r_cpos; + __entry->r_clusters = r_clusters; + __entry->refcount = refcount; + __entry->index = index; + ), + TP_printk("%d %llu %u %llu %u %u %d", + __entry->recs_add, __entry->cpos, __entry->clusters, + __entry->r_cpos, __entry->r_clusters, + __entry->refcount, __entry->index) +); + +DEFINE_OCFS2_INT_INT_EVENT(ocfs2_add_refcount_flag); + +DEFINE_OCFS2_INT_INT_EVENT(ocfs2_prepare_refcount_change_for_del); + +DEFINE_OCFS2_INT_INT_EVENT(ocfs2_lock_refcount_allocators); + +DEFINE_OCFS2_ULL_UINT_UINT_UINT_EVENT(ocfs2_duplicate_clusters_by_page); + +DEFINE_OCFS2_ULL_UINT_UINT_UINT_EVENT(ocfs2_duplicate_clusters_by_jbd); + +TRACE_EVENT(ocfs2_clear_ext_refcount, + TP_PROTO(unsigned long long ino, unsigned int cpos, + unsigned int len, unsigned int p_cluster, + unsigned int ext_flags), + TP_ARGS(ino, cpos, len, p_cluster, ext_flags), + TP_STRUCT__entry( + __field(unsigned long long, ino) + __field(unsigned int, cpos) + __field(unsigned int, len) + __field(unsigned int, p_cluster) + __field(unsigned int, ext_flags) + ), + TP_fast_assign( + __entry->ino = ino; + __entry->cpos = cpos; + __entry->len = len; + __entry->p_cluster = p_cluster; + __entry->ext_flags = ext_flags; + ), + TP_printk("%llu %u %u %u %u", + __entry->ino, __entry->cpos, __entry->len, + __entry->p_cluster, __entry->ext_flags) +); + +TRACE_EVENT(ocfs2_replace_clusters, + TP_PROTO(unsigned long long ino, unsigned int cpos, + unsigned int old, unsigned int new, unsigned int len, + unsigned int ext_flags), + TP_ARGS(ino, cpos, old, new, len, ext_flags), + TP_STRUCT__entry( + __field(unsigned long long, ino) + __field(unsigned int, cpos) + __field(unsigned int, old) + __field(unsigned int, new) + __field(unsigned int, len) + __field(unsigned int, ext_flags) + ), + TP_fast_assign( + __entry->ino = ino; + __entry->cpos = cpos; + __entry->old = old; + __entry->new = new; + __entry->len = len; + __entry->ext_flags = ext_flags; + ), + TP_printk("%llu %u %u %u %u %u", + __entry->ino, __entry->cpos, __entry->old, __entry->new, + __entry->len, __entry->ext_flags) +); + +DEFINE_OCFS2_ULL_UINT_UINT_UINT_EVENT(ocfs2_make_clusters_writable); + +TRACE_EVENT(ocfs2_refcount_cow_hunk, + TP_PROTO(unsigned long long ino, unsigned int cpos, + unsigned int write_len, unsigned int max_cpos, + unsigned int cow_start, unsigned int cow_len), + TP_ARGS(ino, cpos, write_len, max_cpos, cow_start, cow_len), + TP_STRUCT__entry( + __field(unsigned long long, ino) + __field(unsigned int, cpos) + __field(unsigned int, write_len) + __field(unsigned int, max_cpos) + __field(unsigned int, cow_start) + __field(unsigned int, cow_len) + ), + TP_fast_assign( + __entry->ino = ino; + __entry->cpos = cpos; + __entry->write_len = write_len; + __entry->max_cpos = max_cpos; + __entry->cow_start = cow_start; + __entry->cow_len = cow_len; + ), + TP_printk("%llu %u %u %u %u %u", + __entry->ino, __entry->cpos, __entry->write_len, + __entry->max_cpos, __entry->cow_start, __entry->cow_len) +); + +/* End of trace events for fs/ocfs2/refcounttree.c. */ + +/* Trace events for fs/ocfs2/aops.c. */ + +DECLARE_EVENT_CLASS(ocfs2__get_block, + TP_PROTO(unsigned long long ino, unsigned long long iblock, + void *bh_result, int create), + TP_ARGS(ino, iblock, bh_result, create), + TP_STRUCT__entry( + __field(unsigned long long, ino) + __field(unsigned long long, iblock) + __field(void *, bh_result) + __field(int, create) + ), + TP_fast_assign( + __entry->ino = ino; + __entry->iblock = iblock; + __entry->bh_result = bh_result; + __entry->create = create; + ), + TP_printk("%llu %llu %p %d", + __entry->ino, __entry->iblock, + __entry->bh_result, __entry->create) +); + +#define DEFINE_OCFS2_GET_BLOCK_EVENT(name) \ +DEFINE_EVENT(ocfs2__get_block, name, \ + TP_PROTO(unsigned long long ino, unsigned long long iblock, \ + void *bh_result, int create), \ + TP_ARGS(ino, iblock, bh_result, create)) + +DEFINE_OCFS2_GET_BLOCK_EVENT(ocfs2_symlink_get_block); + +DEFINE_OCFS2_GET_BLOCK_EVENT(ocfs2_get_block); + +DEFINE_OCFS2_ULL_ULL_EVENT(ocfs2_get_block_end); + +DEFINE_OCFS2_ULL_ULL_EVENT(ocfs2_readpage); + +DEFINE_OCFS2_ULL_ULL_EVENT(ocfs2_bmap); + +TRACE_EVENT(ocfs2_try_to_write_inline_data, + TP_PROTO(unsigned long long ino, unsigned int len, + unsigned long long pos, unsigned int flags), + TP_ARGS(ino, len, pos, flags), + TP_STRUCT__entry( + __field(unsigned long long, ino) + __field(unsigned int, len) + __field(unsigned long long, pos) + __field(unsigned int, flags) + ), + TP_fast_assign( + __entry->ino = ino; + __entry->len = len; + __entry->pos = pos; + __entry->flags = flags; + ), + TP_printk("%llu %u %llu 0x%x", + __entry->ino, __entry->len, __entry->pos, __entry->flags) +); + +TRACE_EVENT(ocfs2_write_begin_nolock, + TP_PROTO(unsigned long long ino, + long long i_size, unsigned int i_clusters, + unsigned long long pos, unsigned int len, + unsigned int flags, void *page, + unsigned int clusters, unsigned int extents_to_split), + TP_ARGS(ino, i_size, i_clusters, pos, len, flags, + page, clusters, extents_to_split), + TP_STRUCT__entry( + __field(unsigned long long, ino) + __field(long long, i_size) + __field(unsigned int, i_clusters) + __field(unsigned long long, pos) + __field(unsigned int, len) + __field(unsigned int, flags) + __field(void *, page) + __field(unsigned int, clusters) + __field(unsigned int, extents_to_split) + ), + TP_fast_assign( + __entry->ino = ino; + __entry->i_size = i_size; + __entry->i_clusters = i_clusters; + __entry->pos = pos; + __entry->len = len; + __entry->flags = flags; + __entry->page = page; + __entry->clusters = clusters; + __entry->extents_to_split = extents_to_split; + ), + TP_printk("%llu %lld %u %llu %u %u %p %u %u", + __entry->ino, __entry->i_size, __entry->i_clusters, + __entry->pos, __entry->len, + __entry->flags, __entry->page, __entry->clusters, + __entry->extents_to_split) +); + +TRACE_EVENT(ocfs2_write_end_inline, + TP_PROTO(unsigned long long ino, + unsigned long long pos, unsigned int copied, + unsigned int id_count, unsigned int features), + TP_ARGS(ino, pos, copied, id_count, features), + TP_STRUCT__entry( + __field(unsigned long long, ino) + __field(unsigned long long, pos) + __field(unsigned int, copied) + __field(unsigned int, id_count) + __field(unsigned int, features) + ), + TP_fast_assign( + __entry->ino = ino; + __entry->pos = pos; + __entry->copied = copied; + __entry->id_count = id_count; + __entry->features = features; + ), + TP_printk("%llu %llu %u %u %u", + __entry->ino, __entry->pos, __entry->copied, + __entry->id_count, __entry->features) +); + +/* End of trace events for fs/ocfs2/aops.c. */ + +/* Trace events for fs/ocfs2/mmap.c. */ + +TRACE_EVENT(ocfs2_fault, + TP_PROTO(unsigned long long ino, + void *page, unsigned long pgoff), + TP_ARGS(ino, page, pgoff), + TP_STRUCT__entry( + __field(unsigned long long, ino) + __field(void *, page) + __field(unsigned long, pgoff) + ), + TP_fast_assign( + __entry->ino = ino; + __entry->page = page; + __entry->pgoff = pgoff; + ), + TP_printk("%llu %p %lu", + __entry->ino, __entry->page, __entry->pgoff) +); + +/* End of trace events for fs/ocfs2/mmap.c. */ + +/* Trace events for fs/ocfs2/file.c. */ + +DECLARE_EVENT_CLASS(ocfs2__file_ops, + TP_PROTO(void *inode, void *file, void *dentry, + unsigned long long ino, + unsigned int d_len, const unsigned char *d_name, + unsigned long long para), + TP_ARGS(inode, file, dentry, ino, d_len, d_name, para), + TP_STRUCT__entry( + __field(void *, inode) + __field(void *, file) + __field(void *, dentry) + __field(unsigned long long, ino) + __field(unsigned int, d_len) + __string(d_name, d_name) + __field(unsigned long long, para) + ), + TP_fast_assign( + __entry->inode = inode; + __entry->file = file; + __entry->dentry = dentry; + __entry->ino = ino; + __entry->d_len = d_len; + __assign_str(d_name); + __entry->para = para; + ), + TP_printk("%p %p %p %llu %llu %.*s", __entry->inode, __entry->file, + __entry->dentry, __entry->ino, __entry->para, + __entry->d_len, __get_str(d_name)) +); + +#define DEFINE_OCFS2_FILE_OPS(name) \ +DEFINE_EVENT(ocfs2__file_ops, name, \ +TP_PROTO(void *inode, void *file, void *dentry, \ + unsigned long long ino, \ + unsigned int d_len, const unsigned char *d_name, \ + unsigned long long mode), \ + TP_ARGS(inode, file, dentry, ino, d_len, d_name, mode)) + +DEFINE_OCFS2_FILE_OPS(ocfs2_file_open); + +DEFINE_OCFS2_FILE_OPS(ocfs2_file_release); + +DEFINE_OCFS2_FILE_OPS(ocfs2_sync_file); + +DEFINE_OCFS2_FILE_OPS(ocfs2_file_write_iter); + +DEFINE_OCFS2_FILE_OPS(ocfs2_file_read_iter); + +DEFINE_OCFS2_FILE_OPS(ocfs2_file_splice_read); + +DEFINE_OCFS2_ULL_ULL_ULL_EVENT(ocfs2_truncate_file); + +DEFINE_OCFS2_ULL_ULL_EVENT(ocfs2_truncate_file_error); + +TRACE_EVENT(ocfs2_extend_allocation, + TP_PROTO(unsigned long long ip_blkno, unsigned long long size, + unsigned int clusters, unsigned int clusters_to_add, + int why, int restart_func), + TP_ARGS(ip_blkno, size, clusters, clusters_to_add, why, restart_func), + TP_STRUCT__entry( + __field(unsigned long long, ip_blkno) + __field(unsigned long long, size) + __field(unsigned int, clusters) + __field(unsigned int, clusters_to_add) + __field(int, why) + __field(int, restart_func) + ), + TP_fast_assign( + __entry->ip_blkno = ip_blkno; + __entry->size = size; + __entry->clusters = clusters; + __entry->clusters_to_add = clusters_to_add; + __entry->why = why; + __entry->restart_func = restart_func; + ), + TP_printk("%llu %llu %u %u %d %d", + __entry->ip_blkno, __entry->size, __entry->clusters, + __entry->clusters_to_add, __entry->why, __entry->restart_func) +); + +TRACE_EVENT(ocfs2_extend_allocation_end, + TP_PROTO(unsigned long long ino, + unsigned int di_clusters, unsigned long long di_size, + unsigned int ip_clusters, unsigned long long i_size), + TP_ARGS(ino, di_clusters, di_size, ip_clusters, i_size), + TP_STRUCT__entry( + __field(unsigned long long, ino) + __field(unsigned int, di_clusters) + __field(unsigned long long, di_size) + __field(unsigned int, ip_clusters) + __field(unsigned long long, i_size) + ), + TP_fast_assign( + __entry->ino = ino; + __entry->di_clusters = di_clusters; + __entry->di_size = di_size; + __entry->ip_clusters = ip_clusters; + __entry->i_size = i_size; + ), + TP_printk("%llu %u %llu %u %llu", __entry->ino, __entry->di_clusters, + __entry->di_size, __entry->ip_clusters, __entry->i_size) +); + +TRACE_EVENT(ocfs2_write_zero_page, + TP_PROTO(unsigned long long ino, + unsigned long long abs_from, unsigned long long abs_to, + unsigned long index, unsigned int zero_from, + unsigned int zero_to), + TP_ARGS(ino, abs_from, abs_to, index, zero_from, zero_to), + TP_STRUCT__entry( + __field(unsigned long long, ino) + __field(unsigned long long, abs_from) + __field(unsigned long long, abs_to) + __field(unsigned long, index) + __field(unsigned int, zero_from) + __field(unsigned int, zero_to) + ), + TP_fast_assign( + __entry->ino = ino; + __entry->abs_from = abs_from; + __entry->abs_to = abs_to; + __entry->index = index; + __entry->zero_from = zero_from; + __entry->zero_to = zero_to; + ), + TP_printk("%llu %llu %llu %lu %u %u", __entry->ino, + __entry->abs_from, __entry->abs_to, + __entry->index, __entry->zero_from, __entry->zero_to) +); + +DEFINE_OCFS2_ULL_ULL_ULL_EVENT(ocfs2_zero_extend_range); + +DEFINE_OCFS2_ULL_ULL_ULL_EVENT(ocfs2_zero_extend); + +TRACE_EVENT(ocfs2_setattr, + TP_PROTO(void *inode, void *dentry, + unsigned long long ino, + unsigned int d_len, const unsigned char *d_name, + unsigned int ia_valid, unsigned int ia_mode, + unsigned int ia_uid, unsigned int ia_gid), + TP_ARGS(inode, dentry, ino, d_len, d_name, + ia_valid, ia_mode, ia_uid, ia_gid), + TP_STRUCT__entry( + __field(void *, inode) + __field(void *, dentry) + __field(unsigned long long, ino) + __field(unsigned int, d_len) + __string(d_name, d_name) + __field(unsigned int, ia_valid) + __field(unsigned int, ia_mode) + __field(unsigned int, ia_uid) + __field(unsigned int, ia_gid) + ), + TP_fast_assign( + __entry->inode = inode; + __entry->dentry = dentry; + __entry->ino = ino; + __entry->d_len = d_len; + __assign_str(d_name); + __entry->ia_valid = ia_valid; + __entry->ia_mode = ia_mode; + __entry->ia_uid = ia_uid; + __entry->ia_gid = ia_gid; + ), + TP_printk("%p %p %llu %.*s %u %u %u %u", __entry->inode, + __entry->dentry, __entry->ino, __entry->d_len, + __get_str(d_name), __entry->ia_valid, __entry->ia_mode, + __entry->ia_uid, __entry->ia_gid) +); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_write_remove_suid); + +DEFINE_OCFS2_ULL_ULL_ULL_EVENT(ocfs2_zero_partial_clusters); + +DEFINE_OCFS2_ULL_ULL_EVENT(ocfs2_zero_partial_clusters_range1); + +DEFINE_OCFS2_ULL_ULL_EVENT(ocfs2_zero_partial_clusters_range2); + +DEFINE_OCFS2_ULL_ULL_ULL_EVENT(ocfs2_remove_inode_range); + +TRACE_EVENT(ocfs2_prepare_inode_for_write, + TP_PROTO(unsigned long long ino, unsigned long long saved_pos, + unsigned long count, int wait), + TP_ARGS(ino, saved_pos, count, wait), + TP_STRUCT__entry( + __field(unsigned long long, ino) + __field(unsigned long long, saved_pos) + __field(unsigned long, count) + __field(int, wait) + ), + TP_fast_assign( + __entry->ino = ino; + __entry->saved_pos = saved_pos; + __entry->count = count; + __entry->wait = wait; + ), + TP_printk("%llu %llu %lu %d", __entry->ino, + __entry->saved_pos, __entry->count, __entry->wait) +); + +DEFINE_OCFS2_INT_EVENT(generic_file_read_iter_ret); +DEFINE_OCFS2_INT_EVENT(filemap_splice_read_ret); + +/* End of trace events for fs/ocfs2/file.c. */ + +/* Trace events for fs/ocfs2/inode.c. */ + +TRACE_EVENT(ocfs2_iget_begin, + TP_PROTO(unsigned long long ino, unsigned int flags, int sysfile_type), + TP_ARGS(ino, flags, sysfile_type), + TP_STRUCT__entry( + __field(unsigned long long, ino) + __field(unsigned int, flags) + __field(int, sysfile_type) + ), + TP_fast_assign( + __entry->ino = ino; + __entry->flags = flags; + __entry->sysfile_type = sysfile_type; + ), + TP_printk("%llu %u %d", __entry->ino, + __entry->flags, __entry->sysfile_type) +); + +DEFINE_OCFS2_ULL_EVENT(ocfs2_iget5_locked); + +TRACE_EVENT(ocfs2_iget_end, + TP_PROTO(void *inode, unsigned long long ino), + TP_ARGS(inode, ino), + TP_STRUCT__entry( + __field(void *, inode) + __field(unsigned long long, ino) + ), + TP_fast_assign( + __entry->inode = inode; + __entry->ino = ino; + ), + TP_printk("%p %llu", __entry->inode, __entry->ino) +); + +TRACE_EVENT(ocfs2_find_actor, + TP_PROTO(void *inode, unsigned long long ino, + void *args, unsigned long long fi_blkno), + TP_ARGS(inode, ino, args, fi_blkno), + TP_STRUCT__entry( + __field(void *, inode) + __field(unsigned long long, ino) + __field(void *, args) + __field(unsigned long long, fi_blkno) + ), + TP_fast_assign( + __entry->inode = inode; + __entry->ino = ino; + __entry->args = args; + __entry->fi_blkno = fi_blkno; + ), + TP_printk("%p %llu %p %llu", __entry->inode, __entry->ino, + __entry->args, __entry->fi_blkno) +); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_populate_inode); + +DEFINE_OCFS2_ULL_INT_EVENT(ocfs2_read_locked_inode); + +DEFINE_OCFS2_INT_INT_EVENT(ocfs2_check_orphan_recovery_state); + +DEFINE_OCFS2_ULL_EVENT(ocfs2_validate_inode_block); +DEFINE_OCFS2_ULL_EVENT(ocfs2_filecheck_validate_inode_block); +DEFINE_OCFS2_ULL_EVENT(ocfs2_filecheck_repair_inode_block); + +TRACE_EVENT(ocfs2_inode_is_valid_to_delete, + TP_PROTO(void *task, void *dc_task, unsigned long long ino, + unsigned int flags), + TP_ARGS(task, dc_task, ino, flags), + TP_STRUCT__entry( + __field(void *, task) + __field(void *, dc_task) + __field(unsigned long long, ino) + __field(unsigned int, flags) + ), + TP_fast_assign( + __entry->task = task; + __entry->dc_task = dc_task; + __entry->ino = ino; + __entry->flags = flags; + ), + TP_printk("%p %p %llu %u", __entry->task, __entry->dc_task, + __entry->ino, __entry->flags) +); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_query_inode_wipe_begin); + +DEFINE_OCFS2_UINT_EVENT(ocfs2_query_inode_wipe_succ); + +DEFINE_OCFS2_INT_INT_EVENT(ocfs2_query_inode_wipe_end); + +DEFINE_OCFS2_ULL_INT_EVENT(ocfs2_cleanup_delete_inode); + +DEFINE_OCFS2_ULL_ULL_UINT_EVENT(ocfs2_delete_inode); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_clear_inode); + +TRACE_EVENT(ocfs2_inode_revalidate, + TP_PROTO(void *inode, unsigned long long ino, + unsigned int flags), + TP_ARGS(inode, ino, flags), + TP_STRUCT__entry( + __field(void *, inode) + __field(unsigned long long, ino) + __field(unsigned int, flags) + ), + TP_fast_assign( + __entry->inode = inode; + __entry->ino = ino; + __entry->flags = flags; + ), + TP_printk("%p %llu %u", __entry->inode, __entry->ino, __entry->flags) +); + +DEFINE_OCFS2_ULL_EVENT(ocfs2_mark_inode_dirty); + +/* End of trace events for fs/ocfs2/inode.c. */ + +/* Trace events for fs/ocfs2/extent_map.c. */ + +TRACE_EVENT(ocfs2_read_virt_blocks, + TP_PROTO(void *inode, unsigned long long vblock, int nr, + void *bhs, unsigned int flags, void *validate), + TP_ARGS(inode, vblock, nr, bhs, flags, validate), + TP_STRUCT__entry( + __field(void *, inode) + __field(unsigned long long, vblock) + __field(int, nr) + __field(void *, bhs) + __field(unsigned int, flags) + __field(void *, validate) + ), + TP_fast_assign( + __entry->inode = inode; + __entry->vblock = vblock; + __entry->nr = nr; + __entry->bhs = bhs; + __entry->flags = flags; + __entry->validate = validate; + ), + TP_printk("%p %llu %d %p %x %p", __entry->inode, __entry->vblock, + __entry->nr, __entry->bhs, __entry->flags, __entry->validate) +); + +/* End of trace events for fs/ocfs2/extent_map.c. */ + +/* Trace events for fs/ocfs2/slot_map.c. */ + +DEFINE_OCFS2_UINT_EVENT(ocfs2_refresh_slot_info); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_map_slot_buffers); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_map_slot_buffers_block); + +DEFINE_OCFS2_INT_EVENT(ocfs2_find_slot); + +/* End of trace events for fs/ocfs2/slot_map.c. */ + +/* Trace events for fs/ocfs2/heartbeat.c. */ + +DEFINE_OCFS2_INT_EVENT(ocfs2_do_node_down); + +/* End of trace events for fs/ocfs2/heartbeat.c. */ + +/* Trace events for fs/ocfs2/super.c. */ + +TRACE_EVENT(ocfs2_remount, + TP_PROTO(unsigned long s_flags, unsigned long osb_flags, int flags), + TP_ARGS(s_flags, osb_flags, flags), + TP_STRUCT__entry( + __field(unsigned long, s_flags) + __field(unsigned long, osb_flags) + __field(int, flags) + ), + TP_fast_assign( + __entry->s_flags = s_flags; + __entry->osb_flags = osb_flags; + __entry->flags = flags; + ), + TP_printk("%lu %lu %d", __entry->s_flags, + __entry->osb_flags, __entry->flags) +); + +TRACE_EVENT(ocfs2_fill_super, + TP_PROTO(void *sb, void *fc, int silent), + TP_ARGS(sb, fc, silent), + TP_STRUCT__entry( + __field(void *, sb) + __field(void *, fc) + __field(int, silent) + ), + TP_fast_assign( + __entry->sb = sb; + __entry->fc = fc; + __entry->silent = silent; + ), + TP_printk("%p %p %d", __entry->sb, + __entry->fc, __entry->silent) +); + +TRACE_EVENT(ocfs2_parse_options, + TP_PROTO(int is_remount, const char *option), + TP_ARGS(is_remount, option), + TP_STRUCT__entry( + __field(int, is_remount) + __string(option, option) + ), + TP_fast_assign( + __entry->is_remount = is_remount; + __assign_str(option); + ), + TP_printk("%d %s", __entry->is_remount, __get_str(option)) +); + +DEFINE_OCFS2_POINTER_EVENT(ocfs2_put_super); + +TRACE_EVENT(ocfs2_statfs, + TP_PROTO(void *sb, void *buf), + TP_ARGS(sb, buf), + TP_STRUCT__entry( + __field(void *, sb) + __field(void *, buf) + ), + TP_fast_assign( + __entry->sb = sb; + __entry->buf = buf; + ), + TP_printk("%p %p", __entry->sb, __entry->buf) +); + +DEFINE_OCFS2_POINTER_EVENT(ocfs2_dismount_volume); + +TRACE_EVENT(ocfs2_initialize_super, + TP_PROTO(char *label, char *uuid_str, unsigned long long root_dir, + unsigned long long system_dir, int cluster_bits), + TP_ARGS(label, uuid_str, root_dir, system_dir, cluster_bits), + TP_STRUCT__entry( + __string(label, label) + __string(uuid_str, uuid_str) + __field(unsigned long long, root_dir) + __field(unsigned long long, system_dir) + __field(int, cluster_bits) + ), + TP_fast_assign( + __assign_str(label); + __assign_str(uuid_str); + __entry->root_dir = root_dir; + __entry->system_dir = system_dir; + __entry->cluster_bits = cluster_bits; + ), + TP_printk("%s %s %llu %llu %d", __get_str(label), __get_str(uuid_str), + __entry->root_dir, __entry->system_dir, __entry->cluster_bits) +); + +/* End of trace events for fs/ocfs2/super.c. */ + +/* Trace events for fs/ocfs2/xattr.c. */ + +DEFINE_OCFS2_ULL_EVENT(ocfs2_validate_xattr_block); + +DEFINE_OCFS2_UINT_EVENT(ocfs2_xattr_extend_allocation); + +TRACE_EVENT(ocfs2_init_xattr_set_ctxt, + TP_PROTO(const char *name, int meta, int clusters, int credits), + TP_ARGS(name, meta, clusters, credits), + TP_STRUCT__entry( + __string(name, name) + __field(int, meta) + __field(int, clusters) + __field(int, credits) + ), + TP_fast_assign( + __assign_str(name); + __entry->meta = meta; + __entry->clusters = clusters; + __entry->credits = credits; + ), + TP_printk("%s %d %d %d", __get_str(name), __entry->meta, + __entry->clusters, __entry->credits) +); + +DECLARE_EVENT_CLASS(ocfs2__xattr_find, + TP_PROTO(unsigned long long ino, const char *name, int name_index, + unsigned int hash, unsigned long long location, + int xe_index), + TP_ARGS(ino, name, name_index, hash, location, xe_index), + TP_STRUCT__entry( + __field(unsigned long long, ino) + __string(name, name) + __field(int, name_index) + __field(unsigned int, hash) + __field(unsigned long long, location) + __field(int, xe_index) + ), + TP_fast_assign( + __entry->ino = ino; + __assign_str(name); + __entry->name_index = name_index; + __entry->hash = hash; + __entry->location = location; + __entry->xe_index = xe_index; + ), + TP_printk("%llu %s %d %u %llu %d", __entry->ino, __get_str(name), + __entry->name_index, __entry->hash, __entry->location, + __entry->xe_index) +); + +#define DEFINE_OCFS2_XATTR_FIND_EVENT(name) \ +DEFINE_EVENT(ocfs2__xattr_find, name, \ +TP_PROTO(unsigned long long ino, const char *name, int name_index, \ + unsigned int hash, unsigned long long bucket, \ + int xe_index), \ + TP_ARGS(ino, name, name_index, hash, bucket, xe_index)) + +DEFINE_OCFS2_XATTR_FIND_EVENT(ocfs2_xattr_bucket_find); + +DEFINE_OCFS2_XATTR_FIND_EVENT(ocfs2_xattr_index_block_find); + +DEFINE_OCFS2_XATTR_FIND_EVENT(ocfs2_xattr_index_block_find_rec); + +DEFINE_OCFS2_ULL_ULL_UINT_EVENT(ocfs2_iterate_xattr_buckets); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_iterate_xattr_bucket); + +DEFINE_OCFS2_ULL_ULL_EVENT(ocfs2_cp_xattr_block_to_bucket_begin); + +DEFINE_OCFS2_UINT_UINT_UINT_EVENT(ocfs2_cp_xattr_block_to_bucket_end); + +DEFINE_OCFS2_ULL_EVENT(ocfs2_xattr_create_index_block_begin); + +DEFINE_OCFS2_ULL_EVENT(ocfs2_xattr_create_index_block); + +DEFINE_OCFS2_ULL_UINT_UINT_UINT_EVENT(ocfs2_defrag_xattr_bucket); + +DEFINE_OCFS2_ULL_ULL_EVENT(ocfs2_mv_xattr_bucket_cross_cluster); + +DEFINE_OCFS2_ULL_ULL_EVENT(ocfs2_divide_xattr_bucket_begin); + +DEFINE_OCFS2_UINT_UINT_UINT_EVENT(ocfs2_divide_xattr_bucket_move); + +DEFINE_OCFS2_ULL_ULL_UINT_EVENT(ocfs2_cp_xattr_bucket); + +DEFINE_OCFS2_ULL_ULL_EVENT(ocfs2_mv_xattr_buckets); + +DEFINE_OCFS2_ULL_ULL_UINT_EVENT(ocfs2_adjust_xattr_cross_cluster); + +DEFINE_OCFS2_ULL_ULL_UINT_UINT_EVENT(ocfs2_add_new_xattr_cluster_begin); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_add_new_xattr_cluster); + +DEFINE_OCFS2_ULL_UINT_UINT_EVENT(ocfs2_add_new_xattr_cluster_insert); + +DEFINE_OCFS2_ULL_ULL_UINT_UINT_EVENT(ocfs2_extend_xattr_bucket); + +DEFINE_OCFS2_ULL_EVENT(ocfs2_add_new_xattr_bucket); + +DEFINE_OCFS2_ULL_UINT_UINT_EVENT(ocfs2_xattr_bucket_value_truncate); + +DEFINE_OCFS2_ULL_ULL_UINT_UINT_EVENT(ocfs2_rm_xattr_cluster); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_reflink_xattr_header); + +DEFINE_OCFS2_ULL_INT_EVENT(ocfs2_create_empty_xattr_block); + +DEFINE_OCFS2_STRING_EVENT(ocfs2_xattr_set_entry_bucket); + +DEFINE_OCFS2_STRING_EVENT(ocfs2_xattr_set_entry_index_block); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_xattr_bucket_value_refcount); + +DEFINE_OCFS2_ULL_UINT_UINT_EVENT(ocfs2_reflink_xattr_buckets); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_reflink_xattr_rec); + +/* End of trace events for fs/ocfs2/xattr.c. */ + +/* Trace events for fs/ocfs2/reservations.c. */ + +DEFINE_OCFS2_UINT_UINT_EVENT(ocfs2_resv_insert); + +DEFINE_OCFS2_ULL_UINT_UINT_UINT_EVENT(ocfs2_resmap_find_free_bits_begin); + +DEFINE_OCFS2_UINT_UINT_EVENT(ocfs2_resmap_find_free_bits_end); + +TRACE_EVENT(ocfs2_resv_find_window_begin, + TP_PROTO(unsigned int r_start, unsigned int r_end, unsigned int goal, + unsigned int wanted, int empty_root), + TP_ARGS(r_start, r_end, goal, wanted, empty_root), + TP_STRUCT__entry( + __field(unsigned int, r_start) + __field(unsigned int, r_end) + __field(unsigned int, goal) + __field(unsigned int, wanted) + __field(int, empty_root) + ), + TP_fast_assign( + __entry->r_start = r_start; + __entry->r_end = r_end; + __entry->goal = goal; + __entry->wanted = wanted; + __entry->empty_root = empty_root; + ), + TP_printk("%u %u %u %u %d", __entry->r_start, __entry->r_end, + __entry->goal, __entry->wanted, __entry->empty_root) +); + +DEFINE_OCFS2_UINT_UINT_EVENT(ocfs2_resv_find_window_prev); + +DEFINE_OCFS2_INT_INT_EVENT(ocfs2_resv_find_window_next); + +DEFINE_OCFS2_UINT_UINT_UINT_EVENT(ocfs2_cannibalize_resv_begin); + +TRACE_EVENT(ocfs2_cannibalize_resv_end, + TP_PROTO(unsigned int start, unsigned int end, unsigned int len, + unsigned int last_start, unsigned int last_len), + TP_ARGS(start, end, len, last_start, last_len), + TP_STRUCT__entry( + __field(unsigned int, start) + __field(unsigned int, end) + __field(unsigned int, len) + __field(unsigned int, last_start) + __field(unsigned int, last_len) + ), + TP_fast_assign( + __entry->start = start; + __entry->end = end; + __entry->len = len; + __entry->last_start = last_start; + __entry->last_len = last_len; + ), + TP_printk("%u %u %u %u %u", __entry->start, __entry->end, + __entry->len, __entry->last_start, __entry->last_len) +); + +DEFINE_OCFS2_UINT_UINT_EVENT(ocfs2_resmap_resv_bits); + +TRACE_EVENT(ocfs2_resmap_claimed_bits_begin, + TP_PROTO(unsigned int cstart, unsigned int cend, unsigned int clen, + unsigned int r_start, unsigned int r_end, unsigned int r_len, + unsigned int last_start, unsigned int last_len), + TP_ARGS(cstart, cend, clen, r_start, r_end, + r_len, last_start, last_len), + TP_STRUCT__entry( + __field(unsigned int, cstart) + __field(unsigned int, cend) + __field(unsigned int, clen) + __field(unsigned int, r_start) + __field(unsigned int, r_end) + __field(unsigned int, r_len) + __field(unsigned int, last_start) + __field(unsigned int, last_len) + ), + TP_fast_assign( + __entry->cstart = cstart; + __entry->cend = cend; + __entry->clen = clen; + __entry->r_start = r_start; + __entry->r_end = r_end; + __entry->r_len = r_len; + __entry->last_start = last_start; + __entry->last_len = last_len; + ), + TP_printk("%u %u %u %u %u %u %u %u", + __entry->cstart, __entry->cend, __entry->clen, + __entry->r_start, __entry->r_end, __entry->r_len, + __entry->last_start, __entry->last_len) +); + +TRACE_EVENT(ocfs2_resmap_claimed_bits_end, + TP_PROTO(unsigned int start, unsigned int end, unsigned int len, + unsigned int last_start, unsigned int last_len), + TP_ARGS(start, end, len, last_start, last_len), + TP_STRUCT__entry( + __field(unsigned int, start) + __field(unsigned int, end) + __field(unsigned int, len) + __field(unsigned int, last_start) + __field(unsigned int, last_len) + ), + TP_fast_assign( + __entry->start = start; + __entry->end = end; + __entry->len = len; + __entry->last_start = last_start; + __entry->last_len = last_len; + ), + TP_printk("%u %u %u %u %u", __entry->start, __entry->end, + __entry->len, __entry->last_start, __entry->last_len) +); + +/* End of trace events for fs/ocfs2/reservations.c. */ + +/* Trace events for fs/ocfs2/quota_local.c. */ + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_recover_local_quota_file); + +DEFINE_OCFS2_INT_EVENT(ocfs2_finish_quota_recovery); + +DEFINE_OCFS2_ULL_ULL_UINT_EVENT(olq_set_dquot); + +/* End of trace events for fs/ocfs2/quota_local.c. */ + +/* Trace events for fs/ocfs2/quota_global.c. */ + +DEFINE_OCFS2_ULL_EVENT(ocfs2_validate_quota_block); + +TRACE_EVENT(ocfs2_sync_dquot, + TP_PROTO(unsigned int dq_id, long long dqb_curspace, + long long spacechange, long long curinodes, + long long inodechange), + TP_ARGS(dq_id, dqb_curspace, spacechange, curinodes, inodechange), + TP_STRUCT__entry( + __field(unsigned int, dq_id) + __field(long long, dqb_curspace) + __field(long long, spacechange) + __field(long long, curinodes) + __field(long long, inodechange) + ), + TP_fast_assign( + __entry->dq_id = dq_id; + __entry->dqb_curspace = dqb_curspace; + __entry->spacechange = spacechange; + __entry->curinodes = curinodes; + __entry->inodechange = inodechange; + ), + TP_printk("%u %lld %lld %lld %lld", __entry->dq_id, + __entry->dqb_curspace, __entry->spacechange, + __entry->curinodes, __entry->inodechange) +); + +TRACE_EVENT(ocfs2_sync_dquot_helper, + TP_PROTO(unsigned int dq_id, unsigned int dq_type, unsigned long type, + const char *s_id), + TP_ARGS(dq_id, dq_type, type, s_id), + + TP_STRUCT__entry( + __field(unsigned int, dq_id) + __field(unsigned int, dq_type) + __field(unsigned long, type) + __string(s_id, s_id) + ), + TP_fast_assign( + __entry->dq_id = dq_id; + __entry->dq_type = dq_type; + __entry->type = type; + __assign_str(s_id); + ), + TP_printk("%u %u %lu %s", __entry->dq_id, __entry->dq_type, + __entry->type, __get_str(s_id)) +); + +DEFINE_OCFS2_UINT_INT_EVENT(ocfs2_write_dquot); + +DEFINE_OCFS2_UINT_INT_EVENT(ocfs2_release_dquot); + +DEFINE_OCFS2_UINT_INT_EVENT(ocfs2_acquire_dquot); + +DEFINE_OCFS2_UINT_INT_EVENT(ocfs2_get_next_id); + +DEFINE_OCFS2_UINT_INT_EVENT(ocfs2_mark_dquot_dirty); + +/* End of trace events for fs/ocfs2/quota_global.c. */ + +/* Trace events for fs/ocfs2/dir.c. */ +DEFINE_OCFS2_INT_EVENT(ocfs2_search_dirblock); + +DEFINE_OCFS2_ULL_EVENT(ocfs2_validate_dir_block); + +DEFINE_OCFS2_POINTER_EVENT(ocfs2_find_entry_el); + +TRACE_EVENT(ocfs2_dx_dir_search, + TP_PROTO(unsigned long long ino, int namelen, const char *name, + unsigned int major_hash, unsigned int minor_hash, + unsigned long long blkno), + TP_ARGS(ino, namelen, name, major_hash, minor_hash, blkno), + TP_STRUCT__entry( + __field(unsigned long long, ino) + __field(int, namelen) + __string(name, name) + __field(unsigned int, major_hash) + __field(unsigned int,minor_hash) + __field(unsigned long long, blkno) + ), + TP_fast_assign( + __entry->ino = ino; + __entry->namelen = namelen; + __assign_str(name); + __entry->major_hash = major_hash; + __entry->minor_hash = minor_hash; + __entry->blkno = blkno; + ), + TP_printk("%llu %.*s %u %u %llu", __entry->ino, + __entry->namelen, __get_str(name), + __entry->major_hash, __entry->minor_hash, __entry->blkno) +); + +DEFINE_OCFS2_UINT_UINT_EVENT(ocfs2_dx_dir_search_leaf_info); + +DEFINE_OCFS2_ULL_INT_EVENT(ocfs2_delete_entry_dx); + +DEFINE_OCFS2_ULL_EVENT(ocfs2_readdir); + +TRACE_EVENT(ocfs2_find_files_on_disk, + TP_PROTO(int namelen, const char *name, void *blkno, + unsigned long long dir), + TP_ARGS(namelen, name, blkno, dir), + TP_STRUCT__entry( + __field(int, namelen) + __string(name, name) + __field(void *, blkno) + __field(unsigned long long, dir) + ), + TP_fast_assign( + __entry->namelen = namelen; + __assign_str(name); + __entry->blkno = blkno; + __entry->dir = dir; + ), + TP_printk("%.*s %p %llu", __entry->namelen, __get_str(name), + __entry->blkno, __entry->dir) +); + +TRACE_EVENT(ocfs2_check_dir_for_entry, + TP_PROTO(unsigned long long dir, int namelen, const char *name), + TP_ARGS(dir, namelen, name), + TP_STRUCT__entry( + __field(unsigned long long, dir) + __field(int, namelen) + __string(name, name) + ), + TP_fast_assign( + __entry->dir = dir; + __entry->namelen = namelen; + __assign_str(name); + ), + TP_printk("%llu %.*s", __entry->dir, + __entry->namelen, __get_str(name)) +); + +DEFINE_OCFS2_ULL_ULL_EVENT(ocfs2_dx_dir_attach_index); + +DEFINE_OCFS2_ULL_ULL_UINT_EVENT(ocfs2_dx_dir_format_cluster); + +TRACE_EVENT(ocfs2_dx_dir_index_root_block, + TP_PROTO(unsigned long long dir, + unsigned int major_hash, unsigned int minor_hash, + int namelen, const char *name, unsigned int num_used), + TP_ARGS(dir, major_hash, minor_hash, namelen, name, num_used), + TP_STRUCT__entry( + __field(unsigned long long, dir) + __field(unsigned int, major_hash) + __field(unsigned int, minor_hash) + __field(int, namelen) + __string(name, name) + __field(unsigned int, num_used) + ), + TP_fast_assign( + __entry->dir = dir; + __entry->major_hash = major_hash; + __entry->minor_hash = minor_hash; + __entry->namelen = namelen; + __assign_str(name); + __entry->num_used = num_used; + ), + TP_printk("%llu %x %x %.*s %u", __entry->dir, + __entry->major_hash, __entry->minor_hash, + __entry->namelen, __get_str(name), __entry->num_used) +); + +DEFINE_OCFS2_ULL_ULL_EVENT(ocfs2_extend_dir); + +DEFINE_OCFS2_ULL_ULL_UINT_EVENT(ocfs2_dx_dir_rebalance); + +DEFINE_OCFS2_UINT_UINT_UINT_EVENT(ocfs2_dx_dir_rebalance_split); + +DEFINE_OCFS2_ULL_INT_EVENT(ocfs2_prepare_dir_for_insert); + +/* End of trace events for fs/ocfs2/dir.c. */ + +/* Trace events for fs/ocfs2/namei.c. */ + +DECLARE_EVENT_CLASS(ocfs2__dentry_ops, + TP_PROTO(void *dir, void *dentry, int name_len, const char *name, + unsigned long long dir_blkno, unsigned long long extra), + TP_ARGS(dir, dentry, name_len, name, dir_blkno, extra), + TP_STRUCT__entry( + __field(void *, dir) + __field(void *, dentry) + __field(int, name_len) + __string(name, name) + __field(unsigned long long, dir_blkno) + __field(unsigned long long, extra) + ), + TP_fast_assign( + __entry->dir = dir; + __entry->dentry = dentry; + __entry->name_len = name_len; + __assign_str(name); + __entry->dir_blkno = dir_blkno; + __entry->extra = extra; + ), + TP_printk("%p %p %.*s %llu %llu", __entry->dir, __entry->dentry, + __entry->name_len, __get_str(name), + __entry->dir_blkno, __entry->extra) +); + +#define DEFINE_OCFS2_DENTRY_OPS(name) \ +DEFINE_EVENT(ocfs2__dentry_ops, name, \ +TP_PROTO(void *dir, void *dentry, int name_len, const char *name, \ + unsigned long long dir_blkno, unsigned long long extra), \ + TP_ARGS(dir, dentry, name_len, name, dir_blkno, extra)) + +DEFINE_OCFS2_DENTRY_OPS(ocfs2_lookup); + +DEFINE_OCFS2_DENTRY_OPS(ocfs2_mkdir); + +DEFINE_OCFS2_DENTRY_OPS(ocfs2_create); + +DEFINE_OCFS2_DENTRY_OPS(ocfs2_unlink); + +DEFINE_OCFS2_DENTRY_OPS(ocfs2_symlink_create); + +DEFINE_OCFS2_DENTRY_OPS(ocfs2_mv_orphaned_inode_to_new); + +DEFINE_OCFS2_POINTER_EVENT(ocfs2_lookup_ret); + +TRACE_EVENT(ocfs2_mknod, + TP_PROTO(void *dir, void *dentry, int name_len, const char *name, + unsigned long long dir_blkno, unsigned long dev, int mode), + TP_ARGS(dir, dentry, name_len, name, dir_blkno, dev, mode), + TP_STRUCT__entry( + __field(void *, dir) + __field(void *, dentry) + __field(int, name_len) + __string(name, name) + __field(unsigned long long, dir_blkno) + __field(unsigned long, dev) + __field(int, mode) + ), + TP_fast_assign( + __entry->dir = dir; + __entry->dentry = dentry; + __entry->name_len = name_len; + __assign_str(name); + __entry->dir_blkno = dir_blkno; + __entry->dev = dev; + __entry->mode = mode; + ), + TP_printk("%p %p %.*s %llu %lu %d", __entry->dir, __entry->dentry, + __entry->name_len, __get_str(name), + __entry->dir_blkno, __entry->dev, __entry->mode) +); + +TRACE_EVENT(ocfs2_link, + TP_PROTO(unsigned long long ino, int old_len, const char *old_name, + int name_len, const char *name), + TP_ARGS(ino, old_len, old_name, name_len, name), + TP_STRUCT__entry( + __field(unsigned long long, ino) + __field(int, old_len) + __string(old_name, old_name) + __field(int, name_len) + __string(name, name) + ), + TP_fast_assign( + __entry->ino = ino; + __entry->old_len = old_len; + __assign_str(old_name); + __entry->name_len = name_len; + __assign_str(name); + ), + TP_printk("%llu %.*s %.*s", __entry->ino, + __entry->old_len, __get_str(old_name), + __entry->name_len, __get_str(name)) +); + +DEFINE_OCFS2_ULL_ULL_UINT_EVENT(ocfs2_unlink_noent); + +DEFINE_OCFS2_ULL_ULL_EVENT(ocfs2_double_lock); + +DEFINE_OCFS2_ULL_ULL_EVENT(ocfs2_double_lock_end); + +TRACE_EVENT(ocfs2_rename, + TP_PROTO(void *old_dir, void *old_dentry, + void *new_dir, void *new_dentry, + int old_len, const char *old_name, + int new_len, const char *new_name), + TP_ARGS(old_dir, old_dentry, new_dir, new_dentry, + old_len, old_name, new_len, new_name), + TP_STRUCT__entry( + __field(void *, old_dir) + __field(void *, old_dentry) + __field(void *, new_dir) + __field(void *, new_dentry) + __field(int, old_len) + __string(old_name, old_name) + __field(int, new_len) + __string(new_name, new_name) + ), + TP_fast_assign( + __entry->old_dir = old_dir; + __entry->old_dentry = old_dentry; + __entry->new_dir = new_dir; + __entry->new_dentry = new_dentry; + __entry->old_len = old_len; + __assign_str(old_name); + __entry->new_len = new_len; + __assign_str(new_name); + ), + TP_printk("%p %p %p %p %.*s %.*s", + __entry->old_dir, __entry->old_dentry, + __entry->new_dir, __entry->new_dentry, + __entry->old_len, __get_str(old_name), + __entry->new_len, __get_str(new_name)) +); + +DEFINE_OCFS2_ULL_ULL_EVENT(ocfs2_rename_not_permitted); + +TRACE_EVENT(ocfs2_rename_target_exists, + TP_PROTO(int new_len, const char *new_name), + TP_ARGS(new_len, new_name), + TP_STRUCT__entry( + __field(int, new_len) + __string(new_name, new_name) + ), + TP_fast_assign( + __entry->new_len = new_len; + __assign_str(new_name); + ), + TP_printk("%.*s", __entry->new_len, __get_str(new_name)) +); + +DEFINE_OCFS2_ULL_ULL_UINT_EVENT(ocfs2_rename_disagree); + +TRACE_EVENT(ocfs2_rename_over_existing, + TP_PROTO(unsigned long long new_blkno, void *new_bh, + unsigned long long newdi_blkno), + TP_ARGS(new_blkno, new_bh, newdi_blkno), + TP_STRUCT__entry( + __field(unsigned long long, new_blkno) + __field(void *, new_bh) + __field(unsigned long long, newdi_blkno) + ), + TP_fast_assign( + __entry->new_blkno = new_blkno; + __entry->new_bh = new_bh; + __entry->newdi_blkno = newdi_blkno; + ), + TP_printk("%llu %p %llu", __entry->new_blkno, __entry->new_bh, + __entry->newdi_blkno) +); + +DEFINE_OCFS2_ULL_ULL_UINT_EVENT(ocfs2_create_symlink_data); + +TRACE_EVENT(ocfs2_symlink_begin, + TP_PROTO(void *dir, void *dentry, const char *symname, + int len, const char *name), + TP_ARGS(dir, dentry, symname, len, name), + TP_STRUCT__entry( + __field(void *, dir) + __field(void *, dentry) + __field(const char *, symname) + __field(int, len) + __string(name, name) + ), + TP_fast_assign( + __entry->dir = dir; + __entry->dentry = dentry; + __entry->symname = symname; + __entry->len = len; + __assign_str(name); + ), + TP_printk("%p %p %s %.*s", __entry->dir, __entry->dentry, + __entry->symname, __entry->len, __get_str(name)) +); + +TRACE_EVENT(ocfs2_blkno_stringify, + TP_PROTO(unsigned long long blkno, const char *name, int namelen), + TP_ARGS(blkno, name, namelen), + TP_STRUCT__entry( + __field(unsigned long long, blkno) + __string(name, name) + __field(int, namelen) + ), + TP_fast_assign( + __entry->blkno = blkno; + __assign_str(name); + __entry->namelen = namelen; + ), + TP_printk("%llu %s %d", __entry->blkno, __get_str(name), + __entry->namelen) +); + +DEFINE_OCFS2_ULL_EVENT(ocfs2_orphan_add_begin); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_orphan_add_end); + +TRACE_EVENT(ocfs2_orphan_del, + TP_PROTO(unsigned long long dir, const char *name, int namelen), + TP_ARGS(dir, name, namelen), + TP_STRUCT__entry( + __field(unsigned long long, dir) + __string(name, name) + __field(int, namelen) + ), + TP_fast_assign( + __entry->dir = dir; + __assign_str(name); + __entry->namelen = namelen; + ), + TP_printk("%llu %s %d", __entry->dir, __get_str(name), + __entry->namelen) +); + +/* End of trace events for fs/ocfs2/namei.c. */ + +/* Trace events for fs/ocfs2/dcache.c. */ + +TRACE_EVENT(ocfs2_dentry_revalidate, + TP_PROTO(void *dentry, int len, const char *name), + TP_ARGS(dentry, len, name), + TP_STRUCT__entry( + __field(void *, dentry) + __field(int, len) + __string(name, name) + ), + TP_fast_assign( + __entry->dentry = dentry; + __entry->len = len; + __assign_str(name); + ), + TP_printk("%p %.*s", __entry->dentry, __entry->len, __get_str(name)) +); + +TRACE_EVENT(ocfs2_dentry_revalidate_negative, + TP_PROTO(int len, const char *name, unsigned long pgen, + unsigned long gen), + TP_ARGS(len, name, pgen, gen), + TP_STRUCT__entry( + __field(int, len) + __string(name, name) + __field(unsigned long, pgen) + __field(unsigned long, gen) + ), + TP_fast_assign( + __entry->len = len; + __assign_str(name); + __entry->pgen = pgen; + __entry->gen = gen; + ), + TP_printk("%.*s %lu %lu", __entry->len, __get_str(name), + __entry->pgen, __entry->gen) +); + +DEFINE_OCFS2_ULL_EVENT(ocfs2_dentry_revalidate_delete); + +DEFINE_OCFS2_ULL_INT_EVENT(ocfs2_dentry_revalidate_orphaned); + +DEFINE_OCFS2_ULL_EVENT(ocfs2_dentry_revalidate_nofsdata); + +DEFINE_OCFS2_INT_EVENT(ocfs2_dentry_revalidate_ret); + +TRACE_EVENT(ocfs2_find_local_alias, + TP_PROTO(int len, const char *name), + TP_ARGS(len, name), + TP_STRUCT__entry( + __field(int, len) + __string(name, name) + ), + TP_fast_assign( + __entry->len = len; + __assign_str(name); + ), + TP_printk("%.*s", __entry->len, __get_str(name)) +); + +TRACE_EVENT(ocfs2_dentry_attach_lock, + TP_PROTO(int len, const char *name, + unsigned long long parent, void *fsdata), + TP_ARGS(len, name, parent, fsdata), + TP_STRUCT__entry( + __field(int, len) + __string(name, name) + __field(unsigned long long, parent) + __field(void *, fsdata) + ), + TP_fast_assign( + __entry->len = len; + __assign_str(name); + __entry->parent = parent; + __entry->fsdata = fsdata; + ), + TP_printk("%.*s %llu %p", __entry->len, __get_str(name), + __entry->parent, __entry->fsdata) +); + +TRACE_EVENT(ocfs2_dentry_attach_lock_found, + TP_PROTO(const char *name, unsigned long long parent, + unsigned long long ino), + TP_ARGS(name, parent, ino), + TP_STRUCT__entry( + __string(name, name) + __field(unsigned long long, parent) + __field(unsigned long long, ino) + ), + TP_fast_assign( + __assign_str(name); + __entry->parent = parent; + __entry->ino = ino; + ), + TP_printk("%s %llu %llu", __get_str(name), __entry->parent, __entry->ino) +); +/* End of trace events for fs/ocfs2/dcache.c. */ + +/* Trace events for fs/ocfs2/export.c. */ + +TRACE_EVENT(ocfs2_get_dentry_begin, + TP_PROTO(void *sb, void *handle, unsigned long long blkno), + TP_ARGS(sb, handle, blkno), + TP_STRUCT__entry( + __field(void *, sb) + __field(void *, handle) + __field(unsigned long long, blkno) + ), + TP_fast_assign( + __entry->sb = sb; + __entry->handle = handle; + __entry->blkno = blkno; + ), + TP_printk("%p %p %llu", __entry->sb, __entry->handle, __entry->blkno) +); + +DEFINE_OCFS2_INT_INT_EVENT(ocfs2_get_dentry_test_bit); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_get_dentry_stale); + +DEFINE_OCFS2_ULL_UINT_UINT_EVENT(ocfs2_get_dentry_generation); + +DEFINE_OCFS2_POINTER_EVENT(ocfs2_get_dentry_end); + +TRACE_EVENT(ocfs2_get_parent, + TP_PROTO(void *child, int len, const char *name, + unsigned long long ino), + TP_ARGS(child, len, name, ino), + TP_STRUCT__entry( + __field(void *, child) + __field(int, len) + __string(name, name) + __field(unsigned long long, ino) + ), + TP_fast_assign( + __entry->child = child; + __entry->len = len; + __assign_str(name); + __entry->ino = ino; + ), + TP_printk("%p %.*s %llu", __entry->child, __entry->len, + __get_str(name), __entry->ino) +); + +DEFINE_OCFS2_POINTER_EVENT(ocfs2_get_parent_end); + +TRACE_EVENT(ocfs2_encode_fh_begin, + TP_PROTO(void *dentry, int name_len, const char *name, + void *fh, int len, int connectable), + TP_ARGS(dentry, name_len, name, fh, len, connectable), + TP_STRUCT__entry( + __field(void *, dentry) + __field(int, name_len) + __string(name, name) + __field(void *, fh) + __field(int, len) + __field(int, connectable) + ), + TP_fast_assign( + __entry->dentry = dentry; + __entry->name_len = name_len; + __assign_str(name); + __entry->fh = fh; + __entry->len = len; + __entry->connectable = connectable; + ), + TP_printk("%p %.*s %p %d %d", __entry->dentry, __entry->name_len, + __get_str(name), __entry->fh, __entry->len, + __entry->connectable) +); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_encode_fh_self); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_encode_fh_parent); + +DEFINE_OCFS2_INT_EVENT(ocfs2_encode_fh_type); + +/* End of trace events for fs/ocfs2/export.c. */ + +/* Trace events for fs/ocfs2/journal.c. */ + +DEFINE_OCFS2_UINT_EVENT(ocfs2_commit_cache_begin); + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_commit_cache_end); + +DEFINE_OCFS2_INT_INT_EVENT(ocfs2_extend_trans); + +DEFINE_OCFS2_INT_EVENT(ocfs2_assure_trans_credits); + +DEFINE_OCFS2_INT_EVENT(ocfs2_extend_trans_restart); + +DEFINE_OCFS2_INT_INT_EVENT(ocfs2_allocate_extend_trans); + +DEFINE_OCFS2_ULL_ULL_UINT_UINT_EVENT(ocfs2_journal_access); + +DEFINE_OCFS2_ULL_EVENT(ocfs2_journal_dirty); + +DEFINE_OCFS2_ULL_ULL_UINT_EVENT(ocfs2_journal_init); + +DEFINE_OCFS2_UINT_EVENT(ocfs2_journal_init_maxlen); + +DEFINE_OCFS2_INT_EVENT(ocfs2_journal_shutdown); + +DEFINE_OCFS2_POINTER_EVENT(ocfs2_journal_shutdown_wait); + +DEFINE_OCFS2_ULL_EVENT(ocfs2_complete_recovery); + +DEFINE_OCFS2_INT_EVENT(ocfs2_complete_recovery_end); + +TRACE_EVENT(ocfs2_complete_recovery_slot, + TP_PROTO(int slot, unsigned long long la_ino, + unsigned long long tl_ino, void *qrec), + TP_ARGS(slot, la_ino, tl_ino, qrec), + TP_STRUCT__entry( + __field(int, slot) + __field(unsigned long long, la_ino) + __field(unsigned long long, tl_ino) + __field(void *, qrec) + ), + TP_fast_assign( + __entry->slot = slot; + __entry->la_ino = la_ino; + __entry->tl_ino = tl_ino; + __entry->qrec = qrec; + ), + TP_printk("%d %llu %llu %p", __entry->slot, __entry->la_ino, + __entry->tl_ino, __entry->qrec) +); + +DEFINE_OCFS2_INT_INT_EVENT(ocfs2_recovery_thread_node); + +DEFINE_OCFS2_INT_EVENT(ocfs2_recovery_thread_end); + +TRACE_EVENT(ocfs2_recovery_thread, + TP_PROTO(int node_num, int osb_node_num, int disable, + void *recovery_thread, int map_set), + TP_ARGS(node_num, osb_node_num, disable, recovery_thread, map_set), + TP_STRUCT__entry( + __field(int, node_num) + __field(int, osb_node_num) + __field(int,disable) + __field(void *, recovery_thread) + __field(int,map_set) + ), + TP_fast_assign( + __entry->node_num = node_num; + __entry->osb_node_num = osb_node_num; + __entry->disable = disable; + __entry->recovery_thread = recovery_thread; + __entry->map_set = map_set; + ), + TP_printk("%d %d %d %p %d", __entry->node_num, + __entry->osb_node_num, __entry->disable, + __entry->recovery_thread, __entry->map_set) +); + +DEFINE_OCFS2_UINT_UINT_UINT_EVENT(ocfs2_replay_journal_recovered); + +DEFINE_OCFS2_INT_EVENT(ocfs2_replay_journal_lock_err); + +DEFINE_OCFS2_INT_EVENT(ocfs2_replay_journal_skip); + +DEFINE_OCFS2_UINT_UINT_UINT_EVENT(ocfs2_recover_node); + +DEFINE_OCFS2_UINT_UINT_EVENT(ocfs2_recover_node_skip); + +DEFINE_OCFS2_UINT_UINT_EVENT(ocfs2_mark_dead_nodes); + +DEFINE_OCFS2_UINT_UINT_UINT_EVENT(ocfs2_queue_orphan_scan_begin); + +DEFINE_OCFS2_UINT_UINT_UINT_EVENT(ocfs2_queue_orphan_scan_end); + +DEFINE_OCFS2_ULL_EVENT(ocfs2_orphan_filldir); + +DEFINE_OCFS2_INT_EVENT(ocfs2_recover_orphans); + +DEFINE_OCFS2_ULL_EVENT(ocfs2_recover_orphans_iput); + +DEFINE_OCFS2_INT_EVENT(ocfs2_wait_on_mount); + +/* End of trace events for fs/ocfs2/journal.c. */ + +/* Trace events for fs/ocfs2/buffer_head_io.c. */ + +DEFINE_OCFS2_ULL_UINT_EVENT(ocfs2_read_blocks_sync); + +DEFINE_OCFS2_ULL_EVENT(ocfs2_read_blocks_sync_jbd); + +DEFINE_OCFS2_ULL_ULL_EVENT(ocfs2_read_blocks_from_disk); + +DEFINE_OCFS2_ULL_INT_INT_INT_EVENT(ocfs2_read_blocks_bh); + +DEFINE_OCFS2_ULL_INT_INT_INT_EVENT(ocfs2_read_blocks_end); + +TRACE_EVENT(ocfs2_write_block, + TP_PROTO(unsigned long long block, void *ci), + TP_ARGS(block, ci), + TP_STRUCT__entry( + __field(unsigned long long, block) + __field(void *, ci) + ), + TP_fast_assign( + __entry->block = block; + __entry->ci = ci; + ), + TP_printk("%llu %p", __entry->block, __entry->ci) +); + +TRACE_EVENT(ocfs2_read_blocks_begin, + TP_PROTO(void *ci, unsigned long long block, + unsigned int nr, int flags), + TP_ARGS(ci, block, nr, flags), + TP_STRUCT__entry( + __field(void *, ci) + __field(unsigned long long, block) + __field(unsigned int, nr) + __field(int, flags) + ), + TP_fast_assign( + __entry->ci = ci; + __entry->block = block; + __entry->nr = nr; + __entry->flags = flags; + ), + TP_printk("%p %llu %u %d", __entry->ci, __entry->block, + __entry->nr, __entry->flags) +); + +/* End of trace events for fs/ocfs2/buffer_head_io.c. */ + +/* Trace events for fs/ocfs2/uptodate.c. */ + +DEFINE_OCFS2_ULL_EVENT(ocfs2_purge_copied_metadata_tree); + +DEFINE_OCFS2_ULL_UINT_UINT_EVENT(ocfs2_metadata_cache_purge); + +DEFINE_OCFS2_ULL_ULL_UINT_EVENT(ocfs2_buffer_cached_begin); + +TRACE_EVENT(ocfs2_buffer_cached_end, + TP_PROTO(int index, void *item), + TP_ARGS(index, item), + TP_STRUCT__entry( + __field(int, index) + __field(void *, item) + ), + TP_fast_assign( + __entry->index = index; + __entry->item = item; + ), + TP_printk("%d %p", __entry->index, __entry->item) +); + +DEFINE_OCFS2_ULL_ULL_UINT_EVENT(ocfs2_append_cache_array); + +DEFINE_OCFS2_ULL_ULL_UINT_EVENT(ocfs2_insert_cache_tree); + +DEFINE_OCFS2_ULL_UINT_UINT_EVENT(ocfs2_expand_cache); + +DEFINE_OCFS2_ULL_UINT_UINT_EVENT(ocfs2_set_buffer_uptodate); + +DEFINE_OCFS2_ULL_ULL_EVENT(ocfs2_set_buffer_uptodate_begin); + +DEFINE_OCFS2_ULL_UINT_UINT_EVENT(ocfs2_remove_metadata_array); + +DEFINE_OCFS2_ULL_ULL_EVENT(ocfs2_remove_metadata_tree); + +DEFINE_OCFS2_ULL_ULL_UINT_UINT_EVENT(ocfs2_remove_block_from_cache); + +/* End of trace events for fs/ocfs2/uptodate.c. */ +#endif /* _TRACE_OCFS2_H */ + +/* This part must be outside protection */ +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE ocfs2_trace +#include diff --git a/fs/ocfs2/quota.h b/fs/ocfs2/quota.h new file mode 100644 index 000000000..788a8de92 --- /dev/null +++ b/fs/ocfs2/quota.h @@ -0,0 +1,123 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * quota.h for OCFS2 + * + * On disk quota structures for local and global quota file, in-memory + * structures. + * + */ + +#ifndef _OCFS2_QUOTA_H +#define _OCFS2_QUOTA_H + +#include +#include +#include +#include +#include + +#include "ocfs2.h" + +/* Number of quota types we support */ +#define OCFS2_MAXQUOTAS 2 + +/* + * In-memory structures + */ +struct ocfs2_dquot { + struct dquot dq_dquot; /* Generic VFS dquot */ + loff_t dq_local_off; /* Offset in the local quota file */ + u64 dq_local_phys_blk; /* Physical block carrying quota structure */ + struct ocfs2_quota_chunk *dq_chunk; /* Chunk dquot is in */ + unsigned int dq_use_count; /* Number of nodes having reference to this entry in global quota file */ + s64 dq_origspace; /* Last globally synced space usage */ + s64 dq_originodes; /* Last globally synced inode usage */ + struct llist_node list; /* Member of list of dquots to drop */ +}; + +/* Description of one chunk to recover in memory */ +struct ocfs2_recovery_chunk { + struct list_head rc_list; /* List of chunks */ + int rc_chunk; /* Chunk number */ + unsigned long *rc_bitmap; /* Bitmap of entries to recover */ +}; + +struct ocfs2_quota_recovery { + struct list_head r_list[OCFS2_MAXQUOTAS]; /* List of chunks to recover */ +}; + +/* In-memory structure with quota header information */ +struct ocfs2_mem_dqinfo { + unsigned int dqi_type; /* Quota type this structure describes */ + unsigned int dqi_flags; /* Flags OLQF_* */ + unsigned int dqi_chunks; /* Number of chunks in local quota file */ + unsigned int dqi_blocks; /* Number of blocks allocated for local quota file */ + unsigned int dqi_syncms; /* How often should we sync with other nodes */ + struct list_head dqi_chunk; /* List of chunks */ + struct inode *dqi_gqinode; /* Global quota file inode */ + struct ocfs2_lock_res dqi_gqlock; /* Lock protecting quota information structure */ + struct buffer_head *dqi_gqi_bh; /* Buffer head with global quota file inode - set only if inode lock is obtained */ + int dqi_gqi_count; /* Number of holders of dqi_gqi_bh */ + u64 dqi_giblk; /* Number of block with global information header */ + struct buffer_head *dqi_lqi_bh; /* Buffer head with local quota file inode */ + struct buffer_head *dqi_libh; /* Buffer with local information header */ + struct qtree_mem_dqinfo dqi_gi; /* Info about global file */ + struct delayed_work dqi_sync_work; /* Work for syncing dquots */ + struct ocfs2_quota_recovery *dqi_rec; /* Pointer to recovery + * information, in case we + * enable quotas on file + * needing it */ +}; + +static inline struct ocfs2_dquot *OCFS2_DQUOT(struct dquot *dquot) +{ + return container_of(dquot, struct ocfs2_dquot, dq_dquot); +} + +struct ocfs2_quota_chunk { + struct list_head qc_chunk; /* List of quotafile chunks */ + int qc_num; /* Number of quota chunk */ + struct buffer_head *qc_headerbh; /* Buffer head with chunk header */ +}; + +extern struct kmem_cache *ocfs2_dquot_cachep; +extern struct kmem_cache *ocfs2_qf_chunk_cachep; + +extern const struct qtree_fmt_operations ocfs2_global_ops; + +struct ocfs2_quota_recovery *ocfs2_begin_quota_recovery( + struct ocfs2_super *osb, int slot_num); +int ocfs2_finish_quota_recovery(struct ocfs2_super *osb, + struct ocfs2_quota_recovery *rec, + int slot_num); +void ocfs2_free_quota_recovery(struct ocfs2_quota_recovery *rec); +ssize_t ocfs2_quota_read(struct super_block *sb, int type, char *data, + size_t len, loff_t off); +ssize_t ocfs2_quota_write(struct super_block *sb, int type, + const char *data, size_t len, loff_t off); +int ocfs2_global_read_info(struct super_block *sb, int type); +int ocfs2_global_write_info(struct super_block *sb, int type); +int __ocfs2_sync_dquot(struct dquot *dquot, int freeing); +static inline int ocfs2_sync_dquot(struct dquot *dquot) +{ + return __ocfs2_sync_dquot(dquot, 0); +} +static inline int ocfs2_global_release_dquot(struct dquot *dquot) +{ + return __ocfs2_sync_dquot(dquot, 1); +} + +int ocfs2_lock_global_qf(struct ocfs2_mem_dqinfo *oinfo, int ex); +void ocfs2_unlock_global_qf(struct ocfs2_mem_dqinfo *oinfo, int ex); +int ocfs2_validate_quota_block(struct super_block *sb, struct buffer_head *bh); +int ocfs2_read_quota_phys_block(struct inode *inode, u64 p_block, + struct buffer_head **bh); +int ocfs2_create_local_dquot(struct dquot *dquot); +int ocfs2_local_release_dquot(handle_t *handle, struct dquot *dquot); +int ocfs2_local_write_dquot(struct dquot *dquot); +void ocfs2_drop_dquot_refs(struct work_struct *work); + +extern const struct dquot_operations ocfs2_quota_operations; +extern struct quota_format_type ocfs2_quota_format; + +#endif /* _OCFS2_QUOTA_H */ diff --git a/fs/ocfs2/quota_global.c b/fs/ocfs2/quota_global.c new file mode 100644 index 000000000..77b8f0363 --- /dev/null +++ b/fs/ocfs2/quota_global.c @@ -0,0 +1,1051 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Implementation of operations over global quota file + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ocfs2_fs.h" +#include "ocfs2.h" +#include "alloc.h" +#include "blockcheck.h" +#include "inode.h" +#include "journal.h" +#include "file.h" +#include "sysfile.h" +#include "dlmglue.h" +#include "uptodate.h" +#include "super.h" +#include "buffer_head_io.h" +#include "quota.h" +#include "ocfs2_trace.h" + +/* + * Locking of quotas with OCFS2 is rather complex. Here are rules that + * should be obeyed by all the functions: + * - any write of quota structure (either to local or global file) is protected + * by dqio_sem or dquot->dq_lock. + * - any modification of global quota file holds inode cluster lock, i_rwsem, + * and ip_alloc_sem of the global quota file (achieved by + * ocfs2_lock_global_qf). It also has to hold qinfo_lock. + * - an allocation of new blocks for local quota file is protected by + * its ip_alloc_sem + * + * A rough sketch of locking dependencies (lf = local file, gf = global file): + * Normal filesystem operation: + * start_trans -> dqio_sem -> write to lf + * Syncing of local and global file: + * ocfs2_lock_global_qf -> start_trans -> dqio_sem -> qinfo_lock -> + * write to gf + * -> write to lf + * Acquire dquot for the first time: + * dq_lock -> ocfs2_lock_global_qf -> qinfo_lock -> read from gf + * -> alloc space for gf + * -> start_trans -> qinfo_lock -> write to gf + * -> ip_alloc_sem of lf -> alloc space for lf + * -> write to lf + * Release last reference to dquot: + * dq_lock -> ocfs2_lock_global_qf -> start_trans -> qinfo_lock -> write to gf + * -> write to lf + * Note that all the above operations also hold the inode cluster lock of lf. + * Recovery: + * inode cluster lock of recovered lf + * -> read bitmaps -> ip_alloc_sem of lf + * -> ocfs2_lock_global_qf -> start_trans -> dqio_sem -> qinfo_lock -> + * write to gf + */ + +static void qsync_work_fn(struct work_struct *work); + +static void ocfs2_global_disk2memdqb(struct dquot *dquot, void *dp) +{ + struct ocfs2_global_disk_dqblk *d = dp; + struct mem_dqblk *m = &dquot->dq_dqb; + + /* Update from disk only entries not set by the admin */ + if (!test_bit(DQ_LASTSET_B + QIF_ILIMITS_B, &dquot->dq_flags)) { + m->dqb_ihardlimit = le64_to_cpu(d->dqb_ihardlimit); + m->dqb_isoftlimit = le64_to_cpu(d->dqb_isoftlimit); + } + if (!test_bit(DQ_LASTSET_B + QIF_INODES_B, &dquot->dq_flags)) + m->dqb_curinodes = le64_to_cpu(d->dqb_curinodes); + if (!test_bit(DQ_LASTSET_B + QIF_BLIMITS_B, &dquot->dq_flags)) { + m->dqb_bhardlimit = le64_to_cpu(d->dqb_bhardlimit); + m->dqb_bsoftlimit = le64_to_cpu(d->dqb_bsoftlimit); + } + if (!test_bit(DQ_LASTSET_B + QIF_SPACE_B, &dquot->dq_flags)) + m->dqb_curspace = le64_to_cpu(d->dqb_curspace); + if (!test_bit(DQ_LASTSET_B + QIF_BTIME_B, &dquot->dq_flags)) + m->dqb_btime = le64_to_cpu(d->dqb_btime); + if (!test_bit(DQ_LASTSET_B + QIF_ITIME_B, &dquot->dq_flags)) + m->dqb_itime = le64_to_cpu(d->dqb_itime); + OCFS2_DQUOT(dquot)->dq_use_count = le32_to_cpu(d->dqb_use_count); +} + +static void ocfs2_global_mem2diskdqb(void *dp, struct dquot *dquot) +{ + struct ocfs2_global_disk_dqblk *d = dp; + struct mem_dqblk *m = &dquot->dq_dqb; + + d->dqb_id = cpu_to_le32(from_kqid(&init_user_ns, dquot->dq_id)); + d->dqb_use_count = cpu_to_le32(OCFS2_DQUOT(dquot)->dq_use_count); + d->dqb_ihardlimit = cpu_to_le64(m->dqb_ihardlimit); + d->dqb_isoftlimit = cpu_to_le64(m->dqb_isoftlimit); + d->dqb_curinodes = cpu_to_le64(m->dqb_curinodes); + d->dqb_bhardlimit = cpu_to_le64(m->dqb_bhardlimit); + d->dqb_bsoftlimit = cpu_to_le64(m->dqb_bsoftlimit); + d->dqb_curspace = cpu_to_le64(m->dqb_curspace); + d->dqb_btime = cpu_to_le64(m->dqb_btime); + d->dqb_itime = cpu_to_le64(m->dqb_itime); + d->dqb_pad1 = d->dqb_pad2 = 0; +} + +static int ocfs2_global_is_id(void *dp, struct dquot *dquot) +{ + struct ocfs2_global_disk_dqblk *d = dp; + struct ocfs2_mem_dqinfo *oinfo = + sb_dqinfo(dquot->dq_sb, dquot->dq_id.type)->dqi_priv; + + if (qtree_entry_unused(&oinfo->dqi_gi, dp)) + return 0; + + return qid_eq(make_kqid(&init_user_ns, dquot->dq_id.type, + le32_to_cpu(d->dqb_id)), + dquot->dq_id); +} + +const struct qtree_fmt_operations ocfs2_global_ops = { + .mem2disk_dqblk = ocfs2_global_mem2diskdqb, + .disk2mem_dqblk = ocfs2_global_disk2memdqb, + .is_id = ocfs2_global_is_id, +}; + +int ocfs2_validate_quota_block(struct super_block *sb, struct buffer_head *bh) +{ + struct ocfs2_disk_dqtrailer *dqt = + ocfs2_block_dqtrailer(sb->s_blocksize, bh->b_data); + + trace_ocfs2_validate_quota_block((unsigned long long)bh->b_blocknr); + + BUG_ON(!buffer_uptodate(bh)); + + /* + * If the ecc fails, we return the error but otherwise + * leave the filesystem running. We know any error is + * local to this block. + */ + return ocfs2_validate_meta_ecc(sb, bh->b_data, &dqt->dq_check); +} + +int ocfs2_read_quota_phys_block(struct inode *inode, u64 p_block, + struct buffer_head **bhp) +{ + int rc; + + *bhp = NULL; + rc = ocfs2_read_blocks(INODE_CACHE(inode), p_block, 1, bhp, 0, + ocfs2_validate_quota_block); + if (rc) + mlog_errno(rc); + return rc; +} + +/* Read data from global quotafile - avoid pagecache and such because we cannot + * afford acquiring the locks... We use quota cluster lock to serialize + * operations. Caller is responsible for acquiring it. */ +ssize_t ocfs2_quota_read(struct super_block *sb, int type, char *data, + size_t len, loff_t off) +{ + struct ocfs2_mem_dqinfo *oinfo = sb_dqinfo(sb, type)->dqi_priv; + struct inode *gqinode = oinfo->dqi_gqinode; + loff_t i_size = i_size_read(gqinode); + int offset = off & (sb->s_blocksize - 1); + sector_t blk = off >> sb->s_blocksize_bits; + int err = 0; + struct buffer_head *bh; + size_t toread, tocopy; + u64 pblock = 0, pcount = 0; + + if (off > i_size) + return 0; + if (off + len > i_size) + len = i_size - off; + toread = len; + while (toread > 0) { + tocopy = min_t(size_t, (sb->s_blocksize - offset), toread); + if (!pcount) { + err = ocfs2_extent_map_get_blocks(gqinode, blk, &pblock, + &pcount, NULL); + if (err) { + mlog_errno(err); + return err; + } + } else { + pcount--; + pblock++; + } + bh = NULL; + err = ocfs2_read_quota_phys_block(gqinode, pblock, &bh); + if (err) { + mlog_errno(err); + return err; + } + memcpy(data, bh->b_data + offset, tocopy); + brelse(bh); + offset = 0; + toread -= tocopy; + data += tocopy; + blk++; + } + return len; +} + +/* Write to quotafile (we know the transaction is already started and has + * enough credits) */ +ssize_t ocfs2_quota_write(struct super_block *sb, int type, + const char *data, size_t len, loff_t off) +{ + struct mem_dqinfo *info = sb_dqinfo(sb, type); + struct ocfs2_mem_dqinfo *oinfo = info->dqi_priv; + struct inode *gqinode = oinfo->dqi_gqinode; + int offset = off & (sb->s_blocksize - 1); + sector_t blk = off >> sb->s_blocksize_bits; + int err = 0, new = 0, ja_type; + struct buffer_head *bh = NULL; + handle_t *handle = journal_current_handle(); + u64 pblock, pcount; + + if (!handle) { + mlog(ML_ERROR, "Quota write (off=%llu, len=%llu) cancelled " + "because transaction was not started.\n", + (unsigned long long)off, (unsigned long long)len); + return -EIO; + } + if (len > sb->s_blocksize - OCFS2_QBLK_RESERVED_SPACE - offset) { + WARN_ON(1); + len = sb->s_blocksize - OCFS2_QBLK_RESERVED_SPACE - offset; + } + + if (i_size_read(gqinode) < off + len) { + loff_t rounded_end = + ocfs2_align_bytes_to_blocks(sb, off + len); + + /* Space is already allocated in ocfs2_acquire_dquot() */ + err = ocfs2_simple_size_update(gqinode, + oinfo->dqi_gqi_bh, + rounded_end); + if (err < 0) + goto out; + new = 1; + } + err = ocfs2_extent_map_get_blocks(gqinode, blk, &pblock, &pcount, NULL); + if (err) { + mlog_errno(err); + goto out; + } + /* Not rewriting whole block? */ + if ((offset || len < sb->s_blocksize - OCFS2_QBLK_RESERVED_SPACE) && + !new) { + err = ocfs2_read_quota_phys_block(gqinode, pblock, &bh); + ja_type = OCFS2_JOURNAL_ACCESS_WRITE; + } else { + bh = sb_getblk(sb, pblock); + if (!bh) + err = -ENOMEM; + ja_type = OCFS2_JOURNAL_ACCESS_CREATE; + } + if (err) { + mlog_errno(err); + goto out; + } + lock_buffer(bh); + if (new) + memset(bh->b_data, 0, sb->s_blocksize); + memcpy(bh->b_data + offset, data, len); + flush_dcache_folio(bh->b_folio); + set_buffer_uptodate(bh); + unlock_buffer(bh); + ocfs2_set_buffer_uptodate(INODE_CACHE(gqinode), bh); + err = ocfs2_journal_access_dq(handle, INODE_CACHE(gqinode), bh, + ja_type); + if (err < 0) { + brelse(bh); + goto out; + } + ocfs2_journal_dirty(handle, bh); + brelse(bh); +out: + if (err) { + mlog_errno(err); + return err; + } + inode_inc_iversion(gqinode); + ocfs2_mark_inode_dirty(handle, gqinode, oinfo->dqi_gqi_bh); + return len; +} + +int ocfs2_lock_global_qf(struct ocfs2_mem_dqinfo *oinfo, int ex) +{ + int status; + struct buffer_head *bh = NULL; + + status = ocfs2_inode_lock(oinfo->dqi_gqinode, &bh, ex); + if (status < 0) + return status; + spin_lock(&dq_data_lock); + if (!oinfo->dqi_gqi_count++) + oinfo->dqi_gqi_bh = bh; + else + WARN_ON(bh != oinfo->dqi_gqi_bh); + spin_unlock(&dq_data_lock); + if (ex) { + inode_lock(oinfo->dqi_gqinode); + if (!down_write_trylock(&OCFS2_I(oinfo->dqi_gqinode)->ip_alloc_sem)) { + inode_unlock(oinfo->dqi_gqinode); + status = -EBUSY; + goto bail; + } + } else { + down_read(&OCFS2_I(oinfo->dqi_gqinode)->ip_alloc_sem); + } + return 0; + +bail: + /* does a similar job as ocfs2_unlock_global_qf */ + ocfs2_inode_unlock(oinfo->dqi_gqinode, ex); + brelse(oinfo->dqi_gqi_bh); + spin_lock(&dq_data_lock); + if (!--oinfo->dqi_gqi_count) + oinfo->dqi_gqi_bh = NULL; + spin_unlock(&dq_data_lock); + return status; +} + +void ocfs2_unlock_global_qf(struct ocfs2_mem_dqinfo *oinfo, int ex) +{ + if (ex) { + up_write(&OCFS2_I(oinfo->dqi_gqinode)->ip_alloc_sem); + inode_unlock(oinfo->dqi_gqinode); + } else { + up_read(&OCFS2_I(oinfo->dqi_gqinode)->ip_alloc_sem); + } + ocfs2_inode_unlock(oinfo->dqi_gqinode, ex); + brelse(oinfo->dqi_gqi_bh); + spin_lock(&dq_data_lock); + if (!--oinfo->dqi_gqi_count) + oinfo->dqi_gqi_bh = NULL; + spin_unlock(&dq_data_lock); +} + +/* Read information header from global quota file */ +int ocfs2_global_read_info(struct super_block *sb, int type) +{ + unsigned int ino[OCFS2_MAXQUOTAS] = { USER_QUOTA_SYSTEM_INODE, + GROUP_QUOTA_SYSTEM_INODE }; + struct ocfs2_global_disk_dqinfo dinfo; + struct mem_dqinfo *info = sb_dqinfo(sb, type); + struct ocfs2_mem_dqinfo *oinfo = info->dqi_priv; + u64 pcount; + int status; + + oinfo->dqi_gi.dqi_sb = sb; + oinfo->dqi_gi.dqi_type = type; + ocfs2_qinfo_lock_res_init(&oinfo->dqi_gqlock, oinfo); + oinfo->dqi_gi.dqi_entry_size = sizeof(struct ocfs2_global_disk_dqblk); + oinfo->dqi_gi.dqi_ops = &ocfs2_global_ops; + oinfo->dqi_gqi_bh = NULL; + oinfo->dqi_gqi_count = 0; + + /* Read global header */ + oinfo->dqi_gqinode = ocfs2_get_system_file_inode(OCFS2_SB(sb), ino[type], + OCFS2_INVALID_SLOT); + if (!oinfo->dqi_gqinode) { + mlog(ML_ERROR, "failed to get global quota inode (type=%d)\n", + type); + status = -EINVAL; + goto out_err; + } + + status = ocfs2_lock_global_qf(oinfo, 0); + if (status < 0) { + mlog_errno(status); + goto out_err; + } + + status = ocfs2_extent_map_get_blocks(oinfo->dqi_gqinode, 0, &oinfo->dqi_giblk, + &pcount, NULL); + if (status < 0) { + mlog_errno(status); + goto out_unlock; + } + + status = ocfs2_qinfo_lock(oinfo, 0); + if (status < 0) { + mlog_errno(status); + goto out_unlock; + } + status = sb->s_op->quota_read(sb, type, (char *)&dinfo, + sizeof(struct ocfs2_global_disk_dqinfo), + OCFS2_GLOBAL_INFO_OFF); + ocfs2_qinfo_unlock(oinfo, 0); + ocfs2_unlock_global_qf(oinfo, 0); + if (status != sizeof(struct ocfs2_global_disk_dqinfo)) { + mlog(ML_ERROR, "Cannot read global quota info (%d).\n", + status); + if (status >= 0) + status = -EIO; + mlog_errno(status); + goto out_err; + } + info->dqi_bgrace = le32_to_cpu(dinfo.dqi_bgrace); + info->dqi_igrace = le32_to_cpu(dinfo.dqi_igrace); + oinfo->dqi_syncms = le32_to_cpu(dinfo.dqi_syncms); + oinfo->dqi_gi.dqi_blocks = le32_to_cpu(dinfo.dqi_blocks); + oinfo->dqi_gi.dqi_free_blk = le32_to_cpu(dinfo.dqi_free_blk); + oinfo->dqi_gi.dqi_free_entry = le32_to_cpu(dinfo.dqi_free_entry); + oinfo->dqi_gi.dqi_blocksize_bits = sb->s_blocksize_bits; + oinfo->dqi_gi.dqi_usable_bs = sb->s_blocksize - + OCFS2_QBLK_RESERVED_SPACE; + oinfo->dqi_gi.dqi_qtree_depth = qtree_depth(&oinfo->dqi_gi); + INIT_DELAYED_WORK(&oinfo->dqi_sync_work, qsync_work_fn); + schedule_delayed_work(&oinfo->dqi_sync_work, + msecs_to_jiffies(oinfo->dqi_syncms)); + + return 0; +out_unlock: + ocfs2_unlock_global_qf(oinfo, 0); +out_err: + return status; +} + +/* Write information to global quota file. Expects exclusive lock on quota + * file inode and quota info */ +static int __ocfs2_global_write_info(struct super_block *sb, int type) +{ + struct mem_dqinfo *info = sb_dqinfo(sb, type); + struct ocfs2_mem_dqinfo *oinfo = info->dqi_priv; + struct ocfs2_global_disk_dqinfo dinfo; + ssize_t size; + + spin_lock(&dq_data_lock); + info->dqi_flags &= ~DQF_INFO_DIRTY; + dinfo.dqi_bgrace = cpu_to_le32(info->dqi_bgrace); + dinfo.dqi_igrace = cpu_to_le32(info->dqi_igrace); + spin_unlock(&dq_data_lock); + dinfo.dqi_syncms = cpu_to_le32(oinfo->dqi_syncms); + dinfo.dqi_blocks = cpu_to_le32(oinfo->dqi_gi.dqi_blocks); + dinfo.dqi_free_blk = cpu_to_le32(oinfo->dqi_gi.dqi_free_blk); + dinfo.dqi_free_entry = cpu_to_le32(oinfo->dqi_gi.dqi_free_entry); + size = sb->s_op->quota_write(sb, type, (char *)&dinfo, + sizeof(struct ocfs2_global_disk_dqinfo), + OCFS2_GLOBAL_INFO_OFF); + if (size != sizeof(struct ocfs2_global_disk_dqinfo)) { + mlog(ML_ERROR, "Cannot write global quota info structure\n"); + if (size >= 0) + size = -EIO; + return size; + } + return 0; +} + +int ocfs2_global_write_info(struct super_block *sb, int type) +{ + int err; + struct quota_info *dqopt = sb_dqopt(sb); + struct ocfs2_mem_dqinfo *info = dqopt->info[type].dqi_priv; + unsigned int memalloc; + + down_write(&dqopt->dqio_sem); + memalloc = memalloc_nofs_save(); + err = ocfs2_qinfo_lock(info, 1); + if (err < 0) + goto out_sem; + err = __ocfs2_global_write_info(sb, type); + ocfs2_qinfo_unlock(info, 1); +out_sem: + memalloc_nofs_restore(memalloc); + up_write(&dqopt->dqio_sem); + return err; +} + +static int ocfs2_global_qinit_alloc(struct super_block *sb, int type) +{ + struct ocfs2_mem_dqinfo *oinfo = sb_dqinfo(sb, type)->dqi_priv; + + /* + * We may need to allocate tree blocks and a leaf block but not the + * root block + */ + return oinfo->dqi_gi.dqi_qtree_depth; +} + +static int ocfs2_calc_global_qinit_credits(struct super_block *sb, int type) +{ + /* We modify all the allocated blocks, tree root, info block and + * the inode */ + return (ocfs2_global_qinit_alloc(sb, type) + 2) * + OCFS2_QUOTA_BLOCK_UPDATE_CREDITS + 1; +} + +/* Sync local information about quota modifications with global quota file. + * Caller must have started the transaction and obtained exclusive lock for + * global quota file inode */ +int __ocfs2_sync_dquot(struct dquot *dquot, int freeing) +{ + int err, err2; + struct super_block *sb = dquot->dq_sb; + int type = dquot->dq_id.type; + struct ocfs2_mem_dqinfo *info = sb_dqinfo(sb, type)->dqi_priv; + struct ocfs2_global_disk_dqblk dqblk; + s64 spacechange, inodechange; + time64_t olditime, oldbtime; + + err = sb->s_op->quota_read(sb, type, (char *)&dqblk, + sizeof(struct ocfs2_global_disk_dqblk), + dquot->dq_off); + if (err != sizeof(struct ocfs2_global_disk_dqblk)) { + if (err >= 0) { + mlog(ML_ERROR, "Short read from global quota file " + "(%u read)\n", err); + err = -EIO; + } + goto out; + } + + /* Update space and inode usage. Get also other information from + * global quota file so that we don't overwrite any changes there. + * We are */ + spin_lock(&dquot->dq_dqb_lock); + spacechange = dquot->dq_dqb.dqb_curspace - + OCFS2_DQUOT(dquot)->dq_origspace; + inodechange = dquot->dq_dqb.dqb_curinodes - + OCFS2_DQUOT(dquot)->dq_originodes; + olditime = dquot->dq_dqb.dqb_itime; + oldbtime = dquot->dq_dqb.dqb_btime; + ocfs2_global_disk2memdqb(dquot, &dqblk); + trace_ocfs2_sync_dquot(from_kqid(&init_user_ns, dquot->dq_id), + dquot->dq_dqb.dqb_curspace, + (long long)spacechange, + dquot->dq_dqb.dqb_curinodes, + (long long)inodechange); + if (!test_bit(DQ_LASTSET_B + QIF_SPACE_B, &dquot->dq_flags)) + dquot->dq_dqb.dqb_curspace += spacechange; + if (!test_bit(DQ_LASTSET_B + QIF_INODES_B, &dquot->dq_flags)) + dquot->dq_dqb.dqb_curinodes += inodechange; + /* Set properly space grace time... */ + if (dquot->dq_dqb.dqb_bsoftlimit && + dquot->dq_dqb.dqb_curspace > dquot->dq_dqb.dqb_bsoftlimit) { + if (!test_bit(DQ_LASTSET_B + QIF_BTIME_B, &dquot->dq_flags) && + oldbtime > 0) { + if (dquot->dq_dqb.dqb_btime > 0) + dquot->dq_dqb.dqb_btime = + min(dquot->dq_dqb.dqb_btime, oldbtime); + else + dquot->dq_dqb.dqb_btime = oldbtime; + } + } else { + dquot->dq_dqb.dqb_btime = 0; + clear_bit(DQ_BLKS_B, &dquot->dq_flags); + } + /* Set properly inode grace time... */ + if (dquot->dq_dqb.dqb_isoftlimit && + dquot->dq_dqb.dqb_curinodes > dquot->dq_dqb.dqb_isoftlimit) { + if (!test_bit(DQ_LASTSET_B + QIF_ITIME_B, &dquot->dq_flags) && + olditime > 0) { + if (dquot->dq_dqb.dqb_itime > 0) + dquot->dq_dqb.dqb_itime = + min(dquot->dq_dqb.dqb_itime, olditime); + else + dquot->dq_dqb.dqb_itime = olditime; + } + } else { + dquot->dq_dqb.dqb_itime = 0; + clear_bit(DQ_INODES_B, &dquot->dq_flags); + } + /* All information is properly updated, clear the flags */ + __clear_bit(DQ_LASTSET_B + QIF_SPACE_B, &dquot->dq_flags); + __clear_bit(DQ_LASTSET_B + QIF_INODES_B, &dquot->dq_flags); + __clear_bit(DQ_LASTSET_B + QIF_BLIMITS_B, &dquot->dq_flags); + __clear_bit(DQ_LASTSET_B + QIF_ILIMITS_B, &dquot->dq_flags); + __clear_bit(DQ_LASTSET_B + QIF_BTIME_B, &dquot->dq_flags); + __clear_bit(DQ_LASTSET_B + QIF_ITIME_B, &dquot->dq_flags); + OCFS2_DQUOT(dquot)->dq_origspace = dquot->dq_dqb.dqb_curspace; + OCFS2_DQUOT(dquot)->dq_originodes = dquot->dq_dqb.dqb_curinodes; + spin_unlock(&dquot->dq_dqb_lock); + err = ocfs2_qinfo_lock(info, freeing); + if (err < 0) { + mlog(ML_ERROR, "Failed to lock quota info, losing quota write" + " (type=%d, id=%u)\n", dquot->dq_id.type, + (unsigned)from_kqid(&init_user_ns, dquot->dq_id)); + goto out; + } + if (freeing) + OCFS2_DQUOT(dquot)->dq_use_count--; + err = qtree_write_dquot(&info->dqi_gi, dquot); + if (err < 0) + goto out_qlock; + if (freeing && !OCFS2_DQUOT(dquot)->dq_use_count) { + err = qtree_release_dquot(&info->dqi_gi, dquot); + if (info_dirty(sb_dqinfo(sb, type))) { + err2 = __ocfs2_global_write_info(sb, type); + if (!err) + err = err2; + } + } +out_qlock: + ocfs2_qinfo_unlock(info, freeing); +out: + if (err < 0) + mlog_errno(err); + return err; +} + +/* + * Functions for periodic syncing of dquots with global file + */ +static int ocfs2_sync_dquot_helper(struct dquot *dquot, unsigned long type) +{ + handle_t *handle; + struct super_block *sb = dquot->dq_sb; + struct ocfs2_mem_dqinfo *oinfo = sb_dqinfo(sb, type)->dqi_priv; + struct ocfs2_super *osb = OCFS2_SB(sb); + int status = 0; + unsigned int memalloc; + + trace_ocfs2_sync_dquot_helper(from_kqid(&init_user_ns, dquot->dq_id), + dquot->dq_id.type, + type, sb->s_id); + if (type != dquot->dq_id.type) + goto out; + status = ocfs2_lock_global_qf(oinfo, 1); + if (status < 0) + goto out; + + handle = ocfs2_start_trans(osb, OCFS2_QSYNC_CREDITS); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + mlog_errno(status); + goto out_ilock; + } + down_write(&sb_dqopt(sb)->dqio_sem); + memalloc = memalloc_nofs_save(); + status = ocfs2_sync_dquot(dquot); + if (status < 0) + mlog_errno(status); + /* We have to write local structure as well... */ + status = ocfs2_local_write_dquot(dquot); + if (status < 0) + mlog_errno(status); + memalloc_nofs_restore(memalloc); + up_write(&sb_dqopt(sb)->dqio_sem); + ocfs2_commit_trans(osb, handle); +out_ilock: + ocfs2_unlock_global_qf(oinfo, 1); +out: + return status; +} + +static void qsync_work_fn(struct work_struct *work) +{ + struct ocfs2_mem_dqinfo *oinfo = container_of(work, + struct ocfs2_mem_dqinfo, + dqi_sync_work.work); + struct super_block *sb = oinfo->dqi_gqinode->i_sb; + + /* + * We have to be careful here not to deadlock on s_umount as umount + * disabling quotas may be in progress and it waits for this work to + * complete. If trylock fails, we'll do the sync next time... + */ + if (down_read_trylock(&sb->s_umount)) { + dquot_scan_active(sb, ocfs2_sync_dquot_helper, oinfo->dqi_type); + up_read(&sb->s_umount); + } + schedule_delayed_work(&oinfo->dqi_sync_work, + msecs_to_jiffies(oinfo->dqi_syncms)); +} + +/* + * Wrappers for generic quota functions + */ + +static int ocfs2_write_dquot(struct dquot *dquot) +{ + handle_t *handle; + struct ocfs2_super *osb = OCFS2_SB(dquot->dq_sb); + int status = 0; + unsigned int memalloc; + + trace_ocfs2_write_dquot(from_kqid(&init_user_ns, dquot->dq_id), + dquot->dq_id.type); + + handle = ocfs2_start_trans(osb, OCFS2_QWRITE_CREDITS); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + mlog_errno(status); + goto out; + } + down_write(&sb_dqopt(dquot->dq_sb)->dqio_sem); + memalloc = memalloc_nofs_save(); + status = ocfs2_local_write_dquot(dquot); + memalloc_nofs_restore(memalloc); + up_write(&sb_dqopt(dquot->dq_sb)->dqio_sem); + ocfs2_commit_trans(osb, handle); +out: + return status; +} + +static int ocfs2_calc_qdel_credits(struct super_block *sb, int type) +{ + struct ocfs2_mem_dqinfo *oinfo = sb_dqinfo(sb, type)->dqi_priv; + /* + * We modify tree, leaf block, global info, local chunk header, + * global and local inode; OCFS2_QINFO_WRITE_CREDITS already + * accounts for inode update + */ + return (oinfo->dqi_gi.dqi_qtree_depth + 2) * + OCFS2_QUOTA_BLOCK_UPDATE_CREDITS + + OCFS2_QINFO_WRITE_CREDITS + + OCFS2_INODE_UPDATE_CREDITS; +} + +void ocfs2_drop_dquot_refs(struct work_struct *work) +{ + struct ocfs2_super *osb = container_of(work, struct ocfs2_super, + dquot_drop_work); + struct llist_node *list; + struct ocfs2_dquot *odquot, *next_odquot; + + list = llist_del_all(&osb->dquot_drop_list); + llist_for_each_entry_safe(odquot, next_odquot, list, list) { + /* Drop the reference we acquired in ocfs2_dquot_release() */ + dqput(&odquot->dq_dquot); + } +} + +/* + * Called when the last reference to dquot is dropped. If we are called from + * downconvert thread, we cannot do all the handling here because grabbing + * quota lock could deadlock (the node holding the quota lock could need some + * other cluster lock to proceed but with blocked downconvert thread we cannot + * release any lock). + */ +static int ocfs2_release_dquot(struct dquot *dquot) +{ + handle_t *handle; + struct ocfs2_mem_dqinfo *oinfo = + sb_dqinfo(dquot->dq_sb, dquot->dq_id.type)->dqi_priv; + struct ocfs2_super *osb = OCFS2_SB(dquot->dq_sb); + int status = 0; + + trace_ocfs2_release_dquot(from_kqid(&init_user_ns, dquot->dq_id), + dquot->dq_id.type); + + mutex_lock(&dquot->dq_lock); + /* Check whether we are not racing with some other dqget() */ + if (dquot_is_busy(dquot)) + goto out; + /* Running from downconvert thread? Postpone quota processing to wq */ + if (current == osb->dc_task) { + /* + * Grab our own reference to dquot and queue it for delayed + * dropping. Quota code rechecks after calling + * ->release_dquot() and won't free dquot structure. + */ + dqgrab(dquot); + /* First entry on list -> queue work */ + if (llist_add(&OCFS2_DQUOT(dquot)->list, &osb->dquot_drop_list)) + queue_work(osb->ocfs2_wq, &osb->dquot_drop_work); + goto out; + } + status = ocfs2_lock_global_qf(oinfo, 1); + if (status < 0) + goto out; + handle = ocfs2_start_trans(osb, + ocfs2_calc_qdel_credits(dquot->dq_sb, dquot->dq_id.type)); + if (IS_ERR(handle)) { + /* + * Mark dquot as inactive to avoid endless cycle in + * quota_release_workfn(). + */ + clear_bit(DQ_ACTIVE_B, &dquot->dq_flags); + status = PTR_ERR(handle); + mlog_errno(status); + goto out_ilock; + } + + status = ocfs2_global_release_dquot(dquot); + if (status < 0) { + mlog_errno(status); + goto out_trans; + } + status = ocfs2_local_release_dquot(handle, dquot); + /* + * If we fail here, we cannot do much as global structure is + * already released. So just complain... + */ + if (status < 0) + mlog_errno(status); + /* + * Clear dq_off so that we search for the structure in quota file next + * time we acquire it. The structure might be deleted and reallocated + * elsewhere by another node while our dquot structure is on freelist. + */ + dquot->dq_off = 0; + clear_bit(DQ_ACTIVE_B, &dquot->dq_flags); +out_trans: + ocfs2_commit_trans(osb, handle); +out_ilock: + ocfs2_unlock_global_qf(oinfo, 1); +out: + mutex_unlock(&dquot->dq_lock); + if (status) + mlog_errno(status); + return status; +} + +/* + * Read global dquot structure from disk or create it if it does + * not exist. Also update use count of the global structure and + * create structure in node-local quota file. + */ +static int ocfs2_acquire_dquot(struct dquot *dquot) +{ + int status = 0, err; + int ex = 0; + struct super_block *sb = dquot->dq_sb; + struct ocfs2_super *osb = OCFS2_SB(sb); + int type = dquot->dq_id.type; + struct ocfs2_mem_dqinfo *info = sb_dqinfo(sb, type)->dqi_priv; + struct inode *gqinode = info->dqi_gqinode; + int need_alloc = ocfs2_global_qinit_alloc(sb, type); + handle_t *handle; + + trace_ocfs2_acquire_dquot(from_kqid(&init_user_ns, dquot->dq_id), + type); + mutex_lock(&dquot->dq_lock); + /* + * We need an exclusive lock, because we're going to update use count + * and instantiate possibly new dquot structure + */ + status = ocfs2_lock_global_qf(info, 1); + if (status < 0) + goto out; + status = ocfs2_qinfo_lock(info, 0); + if (status < 0) + goto out_dq; + /* + * We always want to read dquot structure from disk because we don't + * know what happened with it while it was on freelist. + */ + status = qtree_read_dquot(&info->dqi_gi, dquot); + ocfs2_qinfo_unlock(info, 0); + if (status < 0) + goto out_dq; + + OCFS2_DQUOT(dquot)->dq_use_count++; + OCFS2_DQUOT(dquot)->dq_origspace = dquot->dq_dqb.dqb_curspace; + OCFS2_DQUOT(dquot)->dq_originodes = dquot->dq_dqb.dqb_curinodes; + if (!dquot->dq_off) { /* No real quota entry? */ + ex = 1; + /* + * Add blocks to quota file before we start a transaction since + * locking allocators ranks above a transaction start + */ + WARN_ON(journal_current_handle()); + status = ocfs2_extend_no_holes(gqinode, NULL, + i_size_read(gqinode) + (need_alloc << sb->s_blocksize_bits), + i_size_read(gqinode)); + if (status < 0) + goto out_dq; + } + + handle = ocfs2_start_trans(osb, + ocfs2_calc_global_qinit_credits(sb, type)); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + goto out_dq; + } + status = ocfs2_qinfo_lock(info, ex); + if (status < 0) + goto out_trans; + status = qtree_write_dquot(&info->dqi_gi, dquot); + if (ex && info_dirty(sb_dqinfo(sb, type))) { + err = __ocfs2_global_write_info(sb, type); + if (!status) + status = err; + } + ocfs2_qinfo_unlock(info, ex); +out_trans: + ocfs2_commit_trans(osb, handle); +out_dq: + ocfs2_unlock_global_qf(info, 1); + if (status < 0) + goto out; + + status = ocfs2_create_local_dquot(dquot); + if (status < 0) + goto out; + set_bit(DQ_ACTIVE_B, &dquot->dq_flags); +out: + mutex_unlock(&dquot->dq_lock); + if (status) + mlog_errno(status); + return status; +} + +static int ocfs2_get_next_id(struct super_block *sb, struct kqid *qid) +{ + int type = qid->type; + struct ocfs2_mem_dqinfo *info = sb_dqinfo(sb, type)->dqi_priv; + int status = 0; + + trace_ocfs2_get_next_id(from_kqid(&init_user_ns, *qid), type); + if (!sb_has_quota_active(sb, type)) { + status = -ESRCH; + goto out; + } + status = ocfs2_lock_global_qf(info, 0); + if (status < 0) + goto out; + status = ocfs2_qinfo_lock(info, 0); + if (status < 0) + goto out_global; + status = qtree_get_next_id(&info->dqi_gi, qid); + ocfs2_qinfo_unlock(info, 0); +out_global: + ocfs2_unlock_global_qf(info, 0); +out: + /* + * Avoid logging ENOENT since it just means there isn't next ID and + * ESRCH which means quota isn't enabled for the filesystem. + */ + if (status && status != -ENOENT && status != -ESRCH) + mlog_errno(status); + return status; +} + +static int ocfs2_mark_dquot_dirty(struct dquot *dquot) +{ + unsigned long mask = (1 << (DQ_LASTSET_B + QIF_ILIMITS_B)) | + (1 << (DQ_LASTSET_B + QIF_BLIMITS_B)) | + (1 << (DQ_LASTSET_B + QIF_INODES_B)) | + (1 << (DQ_LASTSET_B + QIF_SPACE_B)) | + (1 << (DQ_LASTSET_B + QIF_BTIME_B)) | + (1 << (DQ_LASTSET_B + QIF_ITIME_B)); + int sync = 0; + int status; + struct super_block *sb = dquot->dq_sb; + int type = dquot->dq_id.type; + struct ocfs2_mem_dqinfo *oinfo = sb_dqinfo(sb, type)->dqi_priv; + handle_t *handle; + struct ocfs2_super *osb = OCFS2_SB(sb); + unsigned int memalloc; + + trace_ocfs2_mark_dquot_dirty(from_kqid(&init_user_ns, dquot->dq_id), + type); + + /* In case user set some limits, sync dquot immediately to global + * quota file so that information propagates quicker */ + spin_lock(&dquot->dq_dqb_lock); + if (dquot->dq_flags & mask) + sync = 1; + spin_unlock(&dquot->dq_dqb_lock); + /* This is a slight hack but we can't afford getting global quota + * lock if we already have a transaction started. */ + if (!sync || journal_current_handle()) { + status = ocfs2_write_dquot(dquot); + goto out; + } + status = ocfs2_lock_global_qf(oinfo, 1); + if (status < 0) + goto out; + handle = ocfs2_start_trans(osb, OCFS2_QSYNC_CREDITS); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + mlog_errno(status); + goto out_ilock; + } + down_write(&sb_dqopt(sb)->dqio_sem); + memalloc = memalloc_nofs_save(); + status = ocfs2_sync_dquot(dquot); + if (status < 0) { + mlog_errno(status); + goto out_dlock; + } + /* Now write updated local dquot structure */ + status = ocfs2_local_write_dquot(dquot); +out_dlock: + memalloc_nofs_restore(memalloc); + up_write(&sb_dqopt(sb)->dqio_sem); + ocfs2_commit_trans(osb, handle); +out_ilock: + ocfs2_unlock_global_qf(oinfo, 1); +out: + if (status) + mlog_errno(status); + return status; +} + +/* This should happen only after set_dqinfo(). */ +static int ocfs2_write_info(struct super_block *sb, int type) +{ + handle_t *handle; + int status = 0; + struct ocfs2_mem_dqinfo *oinfo = sb_dqinfo(sb, type)->dqi_priv; + + status = ocfs2_lock_global_qf(oinfo, 1); + if (status < 0) + goto out; + handle = ocfs2_start_trans(OCFS2_SB(sb), OCFS2_QINFO_WRITE_CREDITS); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + mlog_errno(status); + goto out_ilock; + } + status = dquot_commit_info(sb, type); + ocfs2_commit_trans(OCFS2_SB(sb), handle); +out_ilock: + ocfs2_unlock_global_qf(oinfo, 1); +out: + if (status) + mlog_errno(status); + return status; +} + +static struct dquot *ocfs2_alloc_dquot(struct super_block *sb, int type) +{ + struct ocfs2_dquot *dquot = + kmem_cache_zalloc(ocfs2_dquot_cachep, GFP_NOFS); + + if (!dquot) + return NULL; + return &dquot->dq_dquot; +} + +static void ocfs2_destroy_dquot(struct dquot *dquot) +{ + kmem_cache_free(ocfs2_dquot_cachep, dquot); +} + +const struct dquot_operations ocfs2_quota_operations = { + /* We never make dquot dirty so .write_dquot is never called */ + .acquire_dquot = ocfs2_acquire_dquot, + .release_dquot = ocfs2_release_dquot, + .mark_dirty = ocfs2_mark_dquot_dirty, + .write_info = ocfs2_write_info, + .alloc_dquot = ocfs2_alloc_dquot, + .destroy_dquot = ocfs2_destroy_dquot, + .get_next_id = ocfs2_get_next_id, +}; diff --git a/fs/ocfs2/quota_local.c b/fs/ocfs2/quota_local.c new file mode 100644 index 000000000..f55810c59 --- /dev/null +++ b/fs/ocfs2/quota_local.c @@ -0,0 +1,1318 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Implementation of operations over local quota file + */ + +#include +#include +#include +#include +#include + +#include + +#include "ocfs2_fs.h" +#include "ocfs2.h" +#include "inode.h" +#include "alloc.h" +#include "file.h" +#include "buffer_head_io.h" +#include "journal.h" +#include "sysfile.h" +#include "dlmglue.h" +#include "quota.h" +#include "uptodate.h" +#include "super.h" +#include "ocfs2_trace.h" + +/* Number of local quota structures per block */ +static inline unsigned int ol_quota_entries_per_block(struct super_block *sb) +{ + return ((sb->s_blocksize - OCFS2_QBLK_RESERVED_SPACE) / + sizeof(struct ocfs2_local_disk_dqblk)); +} + +/* Number of blocks with entries in one chunk */ +static inline unsigned int ol_chunk_blocks(struct super_block *sb) +{ + return ((sb->s_blocksize - sizeof(struct ocfs2_local_disk_chunk) - + OCFS2_QBLK_RESERVED_SPACE) << 3) / + ol_quota_entries_per_block(sb); +} + +/* Number of entries in a chunk bitmap */ +static unsigned int ol_chunk_entries(struct super_block *sb) +{ + return ol_chunk_blocks(sb) * ol_quota_entries_per_block(sb); +} + +/* Offset of the chunk in quota file */ +static unsigned int ol_quota_chunk_block(struct super_block *sb, int c) +{ + /* 1 block for local quota file info, 1 block per chunk for chunk info */ + return 1 + (ol_chunk_blocks(sb) + 1) * c; +} + +static unsigned int ol_dqblk_block(struct super_block *sb, int c, int off) +{ + int epb = ol_quota_entries_per_block(sb); + + return ol_quota_chunk_block(sb, c) + 1 + off / epb; +} + +static unsigned int ol_dqblk_block_off(struct super_block *sb, int c, int off) +{ + int epb = ol_quota_entries_per_block(sb); + + return (off % epb) * sizeof(struct ocfs2_local_disk_dqblk); +} + +/* Offset of the dquot structure in the quota file */ +static loff_t ol_dqblk_off(struct super_block *sb, int c, int off) +{ + return (ol_dqblk_block(sb, c, off) << sb->s_blocksize_bits) + + ol_dqblk_block_off(sb, c, off); +} + +static inline unsigned int ol_dqblk_block_offset(struct super_block *sb, loff_t off) +{ + return off & ((1 << sb->s_blocksize_bits) - 1); +} + +/* Compute offset in the chunk of a structure with the given offset */ +static int ol_dqblk_chunk_off(struct super_block *sb, int c, loff_t off) +{ + int epb = ol_quota_entries_per_block(sb); + + return ((off >> sb->s_blocksize_bits) - + ol_quota_chunk_block(sb, c) - 1) * epb + + ((unsigned int)(off & ((1 << sb->s_blocksize_bits) - 1))) / + sizeof(struct ocfs2_local_disk_dqblk); +} + +/* Write bufferhead into the fs */ +static int ocfs2_modify_bh(struct inode *inode, struct buffer_head *bh, + void (*modify)(struct buffer_head *, void *), void *private) +{ + struct super_block *sb = inode->i_sb; + handle_t *handle; + int status; + + handle = ocfs2_start_trans(OCFS2_SB(sb), + OCFS2_QUOTA_BLOCK_UPDATE_CREDITS); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + mlog_errno(status); + return status; + } + status = ocfs2_journal_access_dq(handle, INODE_CACHE(inode), bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + ocfs2_commit_trans(OCFS2_SB(sb), handle); + return status; + } + lock_buffer(bh); + modify(bh, private); + unlock_buffer(bh); + ocfs2_journal_dirty(handle, bh); + + status = ocfs2_commit_trans(OCFS2_SB(sb), handle); + if (status < 0) { + mlog_errno(status); + return status; + } + return 0; +} + +/* + * Read quota block from a given logical offset. + * + * This function acquires ip_alloc_sem and thus it must not be called with a + * transaction started. + */ +static int ocfs2_read_quota_block(struct inode *inode, u64 v_block, + struct buffer_head **bh) +{ + int rc = 0; + struct buffer_head *tmp = *bh; + + if (i_size_read(inode) >> inode->i_sb->s_blocksize_bits <= v_block) + return ocfs2_error(inode->i_sb, + "Quota file %llu is probably corrupted! Requested to read block %Lu but file has size only %Lu\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno, + (unsigned long long)v_block, + (unsigned long long)i_size_read(inode)); + + rc = ocfs2_read_virt_blocks(inode, v_block, 1, &tmp, 0, + ocfs2_validate_quota_block); + if (rc) + mlog_errno(rc); + + /* If ocfs2_read_virt_blocks() got us a new bh, pass it up. */ + if (!rc && !*bh) + *bh = tmp; + + return rc; +} + +/* Check whether we understand format of quota files */ +static int ocfs2_local_check_quota_file(struct super_block *sb, int type) +{ + unsigned int lmagics[OCFS2_MAXQUOTAS] = OCFS2_LOCAL_QMAGICS; + unsigned int lversions[OCFS2_MAXQUOTAS] = OCFS2_LOCAL_QVERSIONS; + unsigned int gmagics[OCFS2_MAXQUOTAS] = OCFS2_GLOBAL_QMAGICS; + unsigned int gversions[OCFS2_MAXQUOTAS] = OCFS2_GLOBAL_QVERSIONS; + unsigned int ino[OCFS2_MAXQUOTAS] = { USER_QUOTA_SYSTEM_INODE, + GROUP_QUOTA_SYSTEM_INODE }; + struct buffer_head *bh = NULL; + struct inode *linode = sb_dqopt(sb)->files[type]; + struct inode *ginode = NULL; + struct ocfs2_disk_dqheader *dqhead; + int status, ret = 0; + + /* First check whether we understand local quota file */ + status = ocfs2_read_quota_block(linode, 0, &bh); + if (status) { + mlog_errno(status); + mlog(ML_ERROR, "failed to read quota file header (type=%d)\n", + type); + goto out_err; + } + dqhead = (struct ocfs2_disk_dqheader *)(bh->b_data); + if (le32_to_cpu(dqhead->dqh_magic) != lmagics[type]) { + mlog(ML_ERROR, "quota file magic does not match (%u != %u)," + " type=%d\n", le32_to_cpu(dqhead->dqh_magic), + lmagics[type], type); + goto out_err; + } + if (le32_to_cpu(dqhead->dqh_version) != lversions[type]) { + mlog(ML_ERROR, "quota file version does not match (%u != %u)," + " type=%d\n", le32_to_cpu(dqhead->dqh_version), + lversions[type], type); + goto out_err; + } + brelse(bh); + bh = NULL; + + /* Next check whether we understand global quota file */ + ginode = ocfs2_get_system_file_inode(OCFS2_SB(sb), ino[type], + OCFS2_INVALID_SLOT); + if (!ginode) { + mlog(ML_ERROR, "cannot get global quota file inode " + "(type=%d)\n", type); + goto out_err; + } + /* Since the header is read only, we don't care about locking */ + status = ocfs2_read_quota_block(ginode, 0, &bh); + if (status) { + mlog_errno(status); + mlog(ML_ERROR, "failed to read global quota file header " + "(type=%d)\n", type); + goto out_err; + } + dqhead = (struct ocfs2_disk_dqheader *)(bh->b_data); + if (le32_to_cpu(dqhead->dqh_magic) != gmagics[type]) { + mlog(ML_ERROR, "global quota file magic does not match " + "(%u != %u), type=%d\n", + le32_to_cpu(dqhead->dqh_magic), gmagics[type], type); + goto out_err; + } + if (le32_to_cpu(dqhead->dqh_version) != gversions[type]) { + mlog(ML_ERROR, "global quota file version does not match " + "(%u != %u), type=%d\n", + le32_to_cpu(dqhead->dqh_version), gversions[type], + type); + goto out_err; + } + + ret = 1; +out_err: + brelse(bh); + iput(ginode); + return ret; +} + +/* Release given list of quota file chunks */ +static void ocfs2_release_local_quota_bitmaps(struct list_head *head) +{ + struct ocfs2_quota_chunk *pos, *next; + + list_for_each_entry_safe(pos, next, head, qc_chunk) { + list_del(&pos->qc_chunk); + brelse(pos->qc_headerbh); + kmem_cache_free(ocfs2_qf_chunk_cachep, pos); + } +} + +/* Load quota bitmaps into memory */ +static int ocfs2_load_local_quota_bitmaps(struct inode *inode, + struct ocfs2_local_disk_dqinfo *ldinfo, + struct list_head *head) +{ + struct ocfs2_quota_chunk *newchunk; + int i, status; + + INIT_LIST_HEAD(head); + for (i = 0; i < le32_to_cpu(ldinfo->dqi_chunks); i++) { + newchunk = kmem_cache_alloc(ocfs2_qf_chunk_cachep, GFP_NOFS); + if (!newchunk) { + ocfs2_release_local_quota_bitmaps(head); + return -ENOMEM; + } + newchunk->qc_num = i; + newchunk->qc_headerbh = NULL; + status = ocfs2_read_quota_block(inode, + ol_quota_chunk_block(inode->i_sb, i), + &newchunk->qc_headerbh); + if (status) { + mlog_errno(status); + kmem_cache_free(ocfs2_qf_chunk_cachep, newchunk); + ocfs2_release_local_quota_bitmaps(head); + return status; + } + list_add_tail(&newchunk->qc_chunk, head); + } + return 0; +} + +static void olq_update_info(struct buffer_head *bh, void *private) +{ + struct mem_dqinfo *info = private; + struct ocfs2_mem_dqinfo *oinfo = info->dqi_priv; + struct ocfs2_local_disk_dqinfo *ldinfo; + + ldinfo = (struct ocfs2_local_disk_dqinfo *)(bh->b_data + + OCFS2_LOCAL_INFO_OFF); + spin_lock(&dq_data_lock); + ldinfo->dqi_flags = cpu_to_le32(oinfo->dqi_flags); + ldinfo->dqi_chunks = cpu_to_le32(oinfo->dqi_chunks); + ldinfo->dqi_blocks = cpu_to_le32(oinfo->dqi_blocks); + spin_unlock(&dq_data_lock); +} + +static int ocfs2_add_recovery_chunk(struct super_block *sb, + struct ocfs2_local_disk_chunk *dchunk, + int chunk, + struct list_head *head) +{ + struct ocfs2_recovery_chunk *rc; + + rc = kmalloc_obj(struct ocfs2_recovery_chunk, GFP_NOFS); + if (!rc) + return -ENOMEM; + rc->rc_chunk = chunk; + rc->rc_bitmap = kzalloc(sb->s_blocksize, GFP_NOFS); + if (!rc->rc_bitmap) { + kfree(rc); + return -ENOMEM; + } + memcpy(rc->rc_bitmap, dchunk->dqc_bitmap, + (ol_chunk_entries(sb) + 7) >> 3); + list_add_tail(&rc->rc_list, head); + return 0; +} + +static void free_recovery_list(struct list_head *head) +{ + struct ocfs2_recovery_chunk *next; + struct ocfs2_recovery_chunk *rchunk; + + list_for_each_entry_safe(rchunk, next, head, rc_list) { + list_del(&rchunk->rc_list); + kfree(rchunk->rc_bitmap); + kfree(rchunk); + } +} + +void ocfs2_free_quota_recovery(struct ocfs2_quota_recovery *rec) +{ + int type; + + for (type = 0; type < OCFS2_MAXQUOTAS; type++) + free_recovery_list(&(rec->r_list[type])); + kfree(rec); +} + +/* Load entries in our quota file we have to recover*/ +static int ocfs2_recovery_load_quota(struct inode *lqinode, + struct ocfs2_local_disk_dqinfo *ldinfo, + int type, + struct list_head *head) +{ + struct super_block *sb = lqinode->i_sb; + struct buffer_head *hbh; + struct ocfs2_local_disk_chunk *dchunk; + int i, chunks = le32_to_cpu(ldinfo->dqi_chunks); + int status = 0; + + for (i = 0; i < chunks; i++) { + hbh = NULL; + status = ocfs2_read_quota_block(lqinode, + ol_quota_chunk_block(sb, i), + &hbh); + if (status) { + mlog_errno(status); + break; + } + dchunk = (struct ocfs2_local_disk_chunk *)hbh->b_data; + if (le32_to_cpu(dchunk->dqc_free) < ol_chunk_entries(sb)) + status = ocfs2_add_recovery_chunk(sb, dchunk, i, head); + brelse(hbh); + if (status < 0) + break; + } + if (status < 0) + free_recovery_list(head); + return status; +} + +static struct ocfs2_quota_recovery *ocfs2_alloc_quota_recovery(void) +{ + int type; + struct ocfs2_quota_recovery *rec; + + rec = kmalloc_obj(struct ocfs2_quota_recovery, GFP_NOFS); + if (!rec) + return NULL; + for (type = 0; type < OCFS2_MAXQUOTAS; type++) + INIT_LIST_HEAD(&(rec->r_list[type])); + return rec; +} + +/* Load information we need for quota recovery into memory */ +struct ocfs2_quota_recovery *ocfs2_begin_quota_recovery( + struct ocfs2_super *osb, + int slot_num) +{ + unsigned int feature[OCFS2_MAXQUOTAS] = { + OCFS2_FEATURE_RO_COMPAT_USRQUOTA, + OCFS2_FEATURE_RO_COMPAT_GRPQUOTA}; + unsigned int ino[OCFS2_MAXQUOTAS] = { LOCAL_USER_QUOTA_SYSTEM_INODE, + LOCAL_GROUP_QUOTA_SYSTEM_INODE }; + struct super_block *sb = osb->sb; + struct ocfs2_local_disk_dqinfo *ldinfo; + struct inode *lqinode; + struct buffer_head *bh; + int type; + int status = 0; + struct ocfs2_quota_recovery *rec; + + printk(KERN_NOTICE "ocfs2: Beginning quota recovery on device (%s) for " + "slot %u\n", osb->dev_str, slot_num); + + rec = ocfs2_alloc_quota_recovery(); + if (!rec) + return ERR_PTR(-ENOMEM); + /* First init... */ + + for (type = 0; type < OCFS2_MAXQUOTAS; type++) { + if (!OCFS2_HAS_RO_COMPAT_FEATURE(sb, feature[type])) + continue; + /* At this point, journal of the slot is already replayed so + * we can trust metadata and data of the quota file */ + lqinode = ocfs2_get_system_file_inode(osb, ino[type], slot_num); + if (!lqinode) { + status = -ENOENT; + goto out; + } + status = ocfs2_inode_lock_full(lqinode, NULL, 1, + OCFS2_META_LOCK_RECOVERY); + if (status < 0) { + mlog_errno(status); + goto out_put; + } + /* Now read local header */ + bh = NULL; + status = ocfs2_read_quota_block(lqinode, 0, &bh); + if (status) { + mlog_errno(status); + mlog(ML_ERROR, "failed to read quota file info header " + "(slot=%d type=%d)\n", slot_num, type); + goto out_lock; + } + ldinfo = (struct ocfs2_local_disk_dqinfo *)(bh->b_data + + OCFS2_LOCAL_INFO_OFF); + status = ocfs2_recovery_load_quota(lqinode, ldinfo, type, + &rec->r_list[type]); + brelse(bh); +out_lock: + ocfs2_inode_unlock(lqinode, 1); +out_put: + iput(lqinode); + if (status < 0) + break; + } +out: + if (status < 0) { + ocfs2_free_quota_recovery(rec); + rec = ERR_PTR(status); + } + return rec; +} + +/* Sync changes in local quota file into global quota file and + * reinitialize local quota file. + * The function expects local quota file to be already locked. */ +static int ocfs2_recover_local_quota_file(struct inode *lqinode, + int type, + struct ocfs2_quota_recovery *rec) +{ + struct super_block *sb = lqinode->i_sb; + struct ocfs2_mem_dqinfo *oinfo = sb_dqinfo(sb, type)->dqi_priv; + struct ocfs2_local_disk_chunk *dchunk; + struct ocfs2_local_disk_dqblk *dqblk; + struct dquot *dquot; + handle_t *handle; + struct buffer_head *hbh = NULL, *qbh = NULL; + int status = 0; + int bit, chunk; + struct ocfs2_recovery_chunk *rchunk, *next; + qsize_t spacechange, inodechange; + unsigned int memalloc; + + trace_ocfs2_recover_local_quota_file(lqinode->i_ino, type); + + list_for_each_entry_safe(rchunk, next, &(rec->r_list[type]), rc_list) { + chunk = rchunk->rc_chunk; + hbh = NULL; + status = ocfs2_read_quota_block(lqinode, + ol_quota_chunk_block(sb, chunk), + &hbh); + if (status) { + mlog_errno(status); + break; + } + dchunk = (struct ocfs2_local_disk_chunk *)hbh->b_data; + for_each_set_bit(bit, rchunk->rc_bitmap, ol_chunk_entries(sb)) { + qbh = NULL; + status = ocfs2_read_quota_block(lqinode, + ol_dqblk_block(sb, chunk, bit), + &qbh); + if (status) { + mlog_errno(status); + break; + } + dqblk = (struct ocfs2_local_disk_dqblk *)(qbh->b_data + + ol_dqblk_block_off(sb, chunk, bit)); + dquot = dqget(sb, + make_kqid(&init_user_ns, type, + le64_to_cpu(dqblk->dqb_id))); + if (IS_ERR(dquot)) { + status = PTR_ERR(dquot); + mlog(ML_ERROR, "Failed to get quota structure " + "for id %u, type %d. Cannot finish quota " + "file recovery.\n", + (unsigned)le64_to_cpu(dqblk->dqb_id), + type); + goto out_put_bh; + } + status = ocfs2_lock_global_qf(oinfo, 1); + if (status < 0) { + mlog_errno(status); + goto out_put_dquot; + } + + handle = ocfs2_start_trans(OCFS2_SB(sb), + OCFS2_QSYNC_CREDITS); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + mlog_errno(status); + goto out_drop_lock; + } + down_write(&sb_dqopt(sb)->dqio_sem); + memalloc = memalloc_nofs_save(); + spin_lock(&dquot->dq_dqb_lock); + /* Add usage from quota entry into quota changes + * of our node. Auxiliary variables are important + * due to signedness */ + spacechange = le64_to_cpu(dqblk->dqb_spacemod); + inodechange = le64_to_cpu(dqblk->dqb_inodemod); + dquot->dq_dqb.dqb_curspace += spacechange; + dquot->dq_dqb.dqb_curinodes += inodechange; + spin_unlock(&dquot->dq_dqb_lock); + /* We want to drop reference held by the crashed + * node. Since we have our own reference we know + * global structure actually won't be freed. */ + status = ocfs2_global_release_dquot(dquot); + if (status < 0) { + mlog_errno(status); + goto out_commit; + } + /* Release local quota file entry */ + status = ocfs2_journal_access_dq(handle, + INODE_CACHE(lqinode), + qbh, OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto out_commit; + } + lock_buffer(qbh); + WARN_ON(!ocfs2_test_bit_unaligned(bit, dchunk->dqc_bitmap)); + ocfs2_clear_bit_unaligned(bit, dchunk->dqc_bitmap); + le32_add_cpu(&dchunk->dqc_free, 1); + unlock_buffer(qbh); + ocfs2_journal_dirty(handle, qbh); +out_commit: + memalloc_nofs_restore(memalloc); + up_write(&sb_dqopt(sb)->dqio_sem); + ocfs2_commit_trans(OCFS2_SB(sb), handle); +out_drop_lock: + ocfs2_unlock_global_qf(oinfo, 1); +out_put_dquot: + dqput(dquot); +out_put_bh: + brelse(qbh); + if (status < 0) + break; + } + brelse(hbh); + list_del(&rchunk->rc_list); + kfree(rchunk->rc_bitmap); + kfree(rchunk); + if (status < 0) + break; + } + if (status < 0) + free_recovery_list(&(rec->r_list[type])); + if (status) + mlog_errno(status); + return status; +} + +/* Recover local quota files for given node different from us */ +int ocfs2_finish_quota_recovery(struct ocfs2_super *osb, + struct ocfs2_quota_recovery *rec, + int slot_num) +{ + unsigned int ino[OCFS2_MAXQUOTAS] = { LOCAL_USER_QUOTA_SYSTEM_INODE, + LOCAL_GROUP_QUOTA_SYSTEM_INODE }; + struct ocfs2_local_disk_dqinfo *ldinfo; + struct buffer_head *bh; + handle_t *handle; + int type; + int status = 0; + struct inode *lqinode; + unsigned int flags; + + printk(KERN_NOTICE "ocfs2: Finishing quota recovery on device (%s) for " + "slot %u\n", osb->dev_str, slot_num); + + for (type = 0; type < OCFS2_MAXQUOTAS; type++) { + if (list_empty(&(rec->r_list[type]))) + continue; + trace_ocfs2_finish_quota_recovery(slot_num); + lqinode = ocfs2_get_system_file_inode(osb, ino[type], slot_num); + if (!lqinode) { + status = -ENOENT; + goto out; + } + status = ocfs2_inode_lock_full(lqinode, NULL, 1, + OCFS2_META_LOCK_NOQUEUE); + /* Someone else is holding the lock? Then he must be + * doing the recovery. Just skip the file... */ + if (status == -EAGAIN) { + printk(KERN_NOTICE "ocfs2: Skipping quota recovery on " + "device (%s) for slot %d because quota file is " + "locked.\n", osb->dev_str, slot_num); + status = 0; + goto out_put; + } else if (status < 0) { + mlog_errno(status); + goto out_put; + } + /* Now read local header */ + bh = NULL; + status = ocfs2_read_quota_block(lqinode, 0, &bh); + if (status) { + mlog_errno(status); + mlog(ML_ERROR, "failed to read quota file info header " + "(slot=%d type=%d)\n", slot_num, type); + goto out_lock; + } + ldinfo = (struct ocfs2_local_disk_dqinfo *)(bh->b_data + + OCFS2_LOCAL_INFO_OFF); + /* Is recovery still needed? */ + flags = le32_to_cpu(ldinfo->dqi_flags); + if (!(flags & OLQF_CLEAN)) + status = ocfs2_recover_local_quota_file(lqinode, + type, + rec); + /* We don't want to mark file as clean when it is actually + * active */ + if (slot_num == osb->slot_num) + goto out_bh; + /* Mark quota file as clean if we are recovering quota file of + * some other node. */ + handle = ocfs2_start_trans(osb, + OCFS2_LOCAL_QINFO_WRITE_CREDITS); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + mlog_errno(status); + goto out_bh; + } + status = ocfs2_journal_access_dq(handle, INODE_CACHE(lqinode), + bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto out_trans; + } + lock_buffer(bh); + ldinfo->dqi_flags = cpu_to_le32(flags | OLQF_CLEAN); + unlock_buffer(bh); + ocfs2_journal_dirty(handle, bh); +out_trans: + ocfs2_commit_trans(osb, handle); +out_bh: + brelse(bh); +out_lock: + ocfs2_inode_unlock(lqinode, 1); +out_put: + iput(lqinode); + if (status < 0) + break; + } +out: + ocfs2_free_quota_recovery(rec); + return status; +} + +/* Read information header from quota file */ +static int ocfs2_local_read_info(struct super_block *sb, int type) +{ + struct ocfs2_local_disk_dqinfo *ldinfo; + struct mem_dqinfo *info = sb_dqinfo(sb, type); + struct ocfs2_mem_dqinfo *oinfo; + struct inode *lqinode = sb_dqopt(sb)->files[type]; + int status; + struct buffer_head *bh = NULL; + struct ocfs2_quota_recovery *rec; + int locked = 0, global_read = 0; + + info->dqi_max_spc_limit = 0x7fffffffffffffffLL; + info->dqi_max_ino_limit = 0x7fffffffffffffffLL; + oinfo = kmalloc_obj(struct ocfs2_mem_dqinfo, GFP_NOFS); + if (!oinfo) { + mlog(ML_ERROR, "failed to allocate memory for ocfs2 quota" + " info."); + status = -ENOMEM; + goto out_err; + } + info->dqi_priv = oinfo; + oinfo->dqi_type = type; + INIT_LIST_HEAD(&oinfo->dqi_chunk); + oinfo->dqi_rec = NULL; + oinfo->dqi_lqi_bh = NULL; + oinfo->dqi_libh = NULL; + + status = ocfs2_global_read_info(sb, type); + if (status < 0) + goto out_err; + global_read = 1; + + status = ocfs2_inode_lock(lqinode, &oinfo->dqi_lqi_bh, 1); + if (status < 0) { + mlog_errno(status); + goto out_err; + } + locked = 1; + + /* Now read local header */ + status = ocfs2_read_quota_block(lqinode, 0, &bh); + if (status) { + mlog_errno(status); + mlog(ML_ERROR, "failed to read quota file info header " + "(type=%d)\n", type); + goto out_err; + } + ldinfo = (struct ocfs2_local_disk_dqinfo *)(bh->b_data + + OCFS2_LOCAL_INFO_OFF); + oinfo->dqi_flags = le32_to_cpu(ldinfo->dqi_flags); + oinfo->dqi_chunks = le32_to_cpu(ldinfo->dqi_chunks); + oinfo->dqi_blocks = le32_to_cpu(ldinfo->dqi_blocks); + oinfo->dqi_libh = bh; + + /* We crashed when using local quota file? */ + if (!(oinfo->dqi_flags & OLQF_CLEAN)) { + rec = OCFS2_SB(sb)->quota_rec; + if (!rec) { + rec = ocfs2_alloc_quota_recovery(); + if (!rec) { + status = -ENOMEM; + mlog_errno(status); + goto out_err; + } + OCFS2_SB(sb)->quota_rec = rec; + } + + status = ocfs2_recovery_load_quota(lqinode, ldinfo, type, + &rec->r_list[type]); + if (status < 0) { + mlog_errno(status); + goto out_err; + } + } + + status = ocfs2_load_local_quota_bitmaps(lqinode, + ldinfo, + &oinfo->dqi_chunk); + if (status < 0) { + mlog_errno(status); + goto out_err; + } + + /* Now mark quota file as used */ + oinfo->dqi_flags &= ~OLQF_CLEAN; + status = ocfs2_modify_bh(lqinode, bh, olq_update_info, info); + if (status < 0) { + mlog_errno(status); + goto out_err; + } + + return 0; +out_err: + if (oinfo) { + iput(oinfo->dqi_gqinode); + ocfs2_simple_drop_lockres(OCFS2_SB(sb), &oinfo->dqi_gqlock); + ocfs2_lock_res_free(&oinfo->dqi_gqlock); + brelse(oinfo->dqi_lqi_bh); + if (locked) + ocfs2_inode_unlock(lqinode, 1); + ocfs2_release_local_quota_bitmaps(&oinfo->dqi_chunk); + if (global_read) + cancel_delayed_work_sync(&oinfo->dqi_sync_work); + kfree(oinfo); + } + brelse(bh); + return status; +} + +/* Write local info to quota file */ +static int ocfs2_local_write_info(struct super_block *sb, int type) +{ + struct mem_dqinfo *info = sb_dqinfo(sb, type); + struct buffer_head *bh = ((struct ocfs2_mem_dqinfo *)info->dqi_priv) + ->dqi_libh; + int status; + + status = ocfs2_modify_bh(sb_dqopt(sb)->files[type], bh, olq_update_info, + info); + if (status < 0) { + mlog_errno(status); + return -1; + } + + return 0; +} + +/* Release info from memory */ +static int ocfs2_local_free_info(struct super_block *sb, int type) +{ + struct mem_dqinfo *info = sb_dqinfo(sb, type); + struct ocfs2_mem_dqinfo *oinfo = info->dqi_priv; + struct ocfs2_quota_chunk *chunk; + struct ocfs2_local_disk_chunk *dchunk; + int mark_clean = 1, len; + int status = 0; + + iput(oinfo->dqi_gqinode); + ocfs2_simple_drop_lockres(OCFS2_SB(sb), &oinfo->dqi_gqlock); + ocfs2_lock_res_free(&oinfo->dqi_gqlock); + list_for_each_entry(chunk, &oinfo->dqi_chunk, qc_chunk) { + dchunk = (struct ocfs2_local_disk_chunk *) + (chunk->qc_headerbh->b_data); + if (chunk->qc_num < oinfo->dqi_chunks - 1) { + len = ol_chunk_entries(sb); + } else { + len = (oinfo->dqi_blocks - + ol_quota_chunk_block(sb, chunk->qc_num) - 1) + * ol_quota_entries_per_block(sb); + } + /* Not all entries free? Bug! */ + if (le32_to_cpu(dchunk->dqc_free) != len) { + mlog(ML_ERROR, "releasing quota file with used " + "entries (type=%d)\n", type); + mark_clean = 0; + } + } + ocfs2_release_local_quota_bitmaps(&oinfo->dqi_chunk); + + /* + * ocfs2_dismount_volume() has already aborted quota recovery... + */ + if (oinfo->dqi_rec) { + ocfs2_free_quota_recovery(oinfo->dqi_rec); + mark_clean = 0; + } + + if (!mark_clean) + goto out; + + /* Mark local file as clean */ + oinfo->dqi_flags |= OLQF_CLEAN; + status = ocfs2_modify_bh(sb_dqopt(sb)->files[type], + oinfo->dqi_libh, + olq_update_info, + info); + if (status < 0) + mlog_errno(status); +out: + ocfs2_inode_unlock(sb_dqopt(sb)->files[type], 1); + brelse(oinfo->dqi_libh); + brelse(oinfo->dqi_lqi_bh); + kfree(oinfo); + info->dqi_priv = NULL; + return status; +} + +static void olq_set_dquot(struct buffer_head *bh, void *private) +{ + struct ocfs2_dquot *od = private; + struct ocfs2_local_disk_dqblk *dqblk; + struct super_block *sb = od->dq_dquot.dq_sb; + + dqblk = (struct ocfs2_local_disk_dqblk *)(bh->b_data + + ol_dqblk_block_offset(sb, od->dq_local_off)); + + dqblk->dqb_id = cpu_to_le64(from_kqid(&init_user_ns, + od->dq_dquot.dq_id)); + spin_lock(&od->dq_dquot.dq_dqb_lock); + dqblk->dqb_spacemod = cpu_to_le64(od->dq_dquot.dq_dqb.dqb_curspace - + od->dq_origspace); + dqblk->dqb_inodemod = cpu_to_le64(od->dq_dquot.dq_dqb.dqb_curinodes - + od->dq_originodes); + spin_unlock(&od->dq_dquot.dq_dqb_lock); + trace_olq_set_dquot( + (unsigned long long)le64_to_cpu(dqblk->dqb_spacemod), + (unsigned long long)le64_to_cpu(dqblk->dqb_inodemod), + from_kqid(&init_user_ns, od->dq_dquot.dq_id)); +} + +/* Write dquot to local quota file */ +int ocfs2_local_write_dquot(struct dquot *dquot) +{ + struct super_block *sb = dquot->dq_sb; + struct ocfs2_dquot *od = OCFS2_DQUOT(dquot); + struct buffer_head *bh; + struct inode *lqinode = sb_dqopt(sb)->files[dquot->dq_id.type]; + int status; + + status = ocfs2_read_quota_phys_block(lqinode, od->dq_local_phys_blk, + &bh); + if (status) { + mlog_errno(status); + goto out; + } + status = ocfs2_modify_bh(lqinode, bh, olq_set_dquot, od); + if (status < 0) { + mlog_errno(status); + goto out; + } +out: + brelse(bh); + return status; +} + +/* Find free entry in local quota file */ +static struct ocfs2_quota_chunk *ocfs2_find_free_entry(struct super_block *sb, + int type, + int *offset) +{ + struct mem_dqinfo *info = sb_dqinfo(sb, type); + struct ocfs2_mem_dqinfo *oinfo = info->dqi_priv; + struct ocfs2_quota_chunk *chunk = NULL, *iter; + struct ocfs2_local_disk_chunk *dchunk; + int found = 0, len; + + list_for_each_entry(iter, &oinfo->dqi_chunk, qc_chunk) { + dchunk = (struct ocfs2_local_disk_chunk *) + iter->qc_headerbh->b_data; + if (le32_to_cpu(dchunk->dqc_free) > 0) { + chunk = iter; + break; + } + } + if (!chunk) + return NULL; + + if (chunk->qc_num < oinfo->dqi_chunks - 1) { + len = ol_chunk_entries(sb); + } else { + len = (oinfo->dqi_blocks - + ol_quota_chunk_block(sb, chunk->qc_num) - 1) + * ol_quota_entries_per_block(sb); + } + + found = ocfs2_find_next_zero_bit_unaligned(dchunk->dqc_bitmap, len, 0); + /* We failed? */ + if (found == len) { + mlog(ML_ERROR, "Did not find empty entry in chunk %d with %u" + " entries free (type=%d)\n", chunk->qc_num, + le32_to_cpu(dchunk->dqc_free), type); + return ERR_PTR(-EIO); + } + *offset = found; + return chunk; +} + +/* Add new chunk to the local quota file */ +static struct ocfs2_quota_chunk *ocfs2_local_quota_add_chunk( + struct super_block *sb, + int type, + int *offset) +{ + struct mem_dqinfo *info = sb_dqinfo(sb, type); + struct ocfs2_mem_dqinfo *oinfo = info->dqi_priv; + struct inode *lqinode = sb_dqopt(sb)->files[type]; + struct ocfs2_quota_chunk *chunk = NULL; + struct ocfs2_local_disk_chunk *dchunk; + int status; + handle_t *handle; + struct buffer_head *bh = NULL, *dbh = NULL; + u64 p_blkno; + + /* We are protected by dqio_sem so no locking needed */ + status = ocfs2_extend_no_holes(lqinode, NULL, + i_size_read(lqinode) + 2 * sb->s_blocksize, + i_size_read(lqinode)); + if (status < 0) { + mlog_errno(status); + goto out; + } + status = ocfs2_simple_size_update(lqinode, oinfo->dqi_lqi_bh, + i_size_read(lqinode) + 2 * sb->s_blocksize); + if (status < 0) { + mlog_errno(status); + goto out; + } + + chunk = kmem_cache_alloc(ocfs2_qf_chunk_cachep, GFP_NOFS); + if (!chunk) { + status = -ENOMEM; + mlog_errno(status); + goto out; + } + /* Local quota info and two new blocks we initialize */ + handle = ocfs2_start_trans(OCFS2_SB(sb), + OCFS2_LOCAL_QINFO_WRITE_CREDITS + + 2 * OCFS2_QUOTA_BLOCK_UPDATE_CREDITS); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + mlog_errno(status); + goto out; + } + + /* Initialize chunk header */ + status = ocfs2_extent_map_get_blocks(lqinode, oinfo->dqi_blocks, + &p_blkno, NULL, NULL); + if (status < 0) { + mlog_errno(status); + goto out_trans; + } + bh = sb_getblk(sb, p_blkno); + if (!bh) { + status = -ENOMEM; + mlog_errno(status); + goto out_trans; + } + dchunk = (struct ocfs2_local_disk_chunk *)bh->b_data; + ocfs2_set_new_buffer_uptodate(INODE_CACHE(lqinode), bh); + status = ocfs2_journal_access_dq(handle, INODE_CACHE(lqinode), bh, + OCFS2_JOURNAL_ACCESS_CREATE); + if (status < 0) { + mlog_errno(status); + goto out_trans; + } + lock_buffer(bh); + dchunk->dqc_free = cpu_to_le32(ol_quota_entries_per_block(sb)); + memset(dchunk->dqc_bitmap, 0, + sb->s_blocksize - sizeof(struct ocfs2_local_disk_chunk) - + OCFS2_QBLK_RESERVED_SPACE); + unlock_buffer(bh); + ocfs2_journal_dirty(handle, bh); + + /* Initialize new block with structures */ + status = ocfs2_extent_map_get_blocks(lqinode, oinfo->dqi_blocks + 1, + &p_blkno, NULL, NULL); + if (status < 0) { + mlog_errno(status); + goto out_trans; + } + dbh = sb_getblk(sb, p_blkno); + if (!dbh) { + status = -ENOMEM; + mlog_errno(status); + goto out_trans; + } + ocfs2_set_new_buffer_uptodate(INODE_CACHE(lqinode), dbh); + status = ocfs2_journal_access_dq(handle, INODE_CACHE(lqinode), dbh, + OCFS2_JOURNAL_ACCESS_CREATE); + if (status < 0) { + mlog_errno(status); + goto out_trans; + } + lock_buffer(dbh); + memset(dbh->b_data, 0, sb->s_blocksize - OCFS2_QBLK_RESERVED_SPACE); + unlock_buffer(dbh); + ocfs2_journal_dirty(handle, dbh); + + /* Update local quotafile info */ + oinfo->dqi_blocks += 2; + oinfo->dqi_chunks++; + status = ocfs2_local_write_info(sb, type); + if (status < 0) { + mlog_errno(status); + goto out_trans; + } + status = ocfs2_commit_trans(OCFS2_SB(sb), handle); + if (status < 0) { + mlog_errno(status); + goto out; + } + + list_add_tail(&chunk->qc_chunk, &oinfo->dqi_chunk); + chunk->qc_num = list_entry(chunk->qc_chunk.prev, + struct ocfs2_quota_chunk, + qc_chunk)->qc_num + 1; + chunk->qc_headerbh = bh; + *offset = 0; + return chunk; +out_trans: + ocfs2_commit_trans(OCFS2_SB(sb), handle); +out: + brelse(bh); + brelse(dbh); + kmem_cache_free(ocfs2_qf_chunk_cachep, chunk); + return ERR_PTR(status); +} + +/* Find free entry in local quota file */ +static struct ocfs2_quota_chunk *ocfs2_extend_local_quota_file( + struct super_block *sb, + int type, + int *offset) +{ + struct mem_dqinfo *info = sb_dqinfo(sb, type); + struct ocfs2_mem_dqinfo *oinfo = info->dqi_priv; + struct ocfs2_quota_chunk *chunk; + struct inode *lqinode = sb_dqopt(sb)->files[type]; + struct ocfs2_local_disk_chunk *dchunk; + int epb = ol_quota_entries_per_block(sb); + unsigned int chunk_blocks; + struct buffer_head *bh; + u64 p_blkno; + int status; + handle_t *handle; + + if (list_empty(&oinfo->dqi_chunk)) + return ocfs2_local_quota_add_chunk(sb, type, offset); + /* Is the last chunk full? */ + chunk = list_entry(oinfo->dqi_chunk.prev, + struct ocfs2_quota_chunk, qc_chunk); + chunk_blocks = oinfo->dqi_blocks - + ol_quota_chunk_block(sb, chunk->qc_num) - 1; + if (ol_chunk_blocks(sb) == chunk_blocks) + return ocfs2_local_quota_add_chunk(sb, type, offset); + + /* We are protected by dqio_sem so no locking needed */ + status = ocfs2_extend_no_holes(lqinode, NULL, + i_size_read(lqinode) + sb->s_blocksize, + i_size_read(lqinode)); + if (status < 0) { + mlog_errno(status); + goto out; + } + status = ocfs2_simple_size_update(lqinode, oinfo->dqi_lqi_bh, + i_size_read(lqinode) + sb->s_blocksize); + if (status < 0) { + mlog_errno(status); + goto out; + } + + /* Get buffer from the just added block */ + status = ocfs2_extent_map_get_blocks(lqinode, oinfo->dqi_blocks, + &p_blkno, NULL, NULL); + if (status < 0) { + mlog_errno(status); + goto out; + } + bh = sb_getblk(sb, p_blkno); + if (!bh) { + status = -ENOMEM; + mlog_errno(status); + goto out; + } + ocfs2_set_new_buffer_uptodate(INODE_CACHE(lqinode), bh); + + /* Local quota info, chunk header and the new block we initialize */ + handle = ocfs2_start_trans(OCFS2_SB(sb), + OCFS2_LOCAL_QINFO_WRITE_CREDITS + + 2 * OCFS2_QUOTA_BLOCK_UPDATE_CREDITS); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + mlog_errno(status); + goto out; + } + /* Zero created block */ + status = ocfs2_journal_access_dq(handle, INODE_CACHE(lqinode), bh, + OCFS2_JOURNAL_ACCESS_CREATE); + if (status < 0) { + mlog_errno(status); + goto out_trans; + } + lock_buffer(bh); + memset(bh->b_data, 0, sb->s_blocksize); + unlock_buffer(bh); + ocfs2_journal_dirty(handle, bh); + + /* Update chunk header */ + status = ocfs2_journal_access_dq(handle, INODE_CACHE(lqinode), + chunk->qc_headerbh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto out_trans; + } + + dchunk = (struct ocfs2_local_disk_chunk *)chunk->qc_headerbh->b_data; + lock_buffer(chunk->qc_headerbh); + le32_add_cpu(&dchunk->dqc_free, ol_quota_entries_per_block(sb)); + unlock_buffer(chunk->qc_headerbh); + ocfs2_journal_dirty(handle, chunk->qc_headerbh); + + /* Update file header */ + oinfo->dqi_blocks++; + status = ocfs2_local_write_info(sb, type); + if (status < 0) { + mlog_errno(status); + goto out_trans; + } + + status = ocfs2_commit_trans(OCFS2_SB(sb), handle); + if (status < 0) { + mlog_errno(status); + goto out; + } + *offset = chunk_blocks * epb; + return chunk; +out_trans: + ocfs2_commit_trans(OCFS2_SB(sb), handle); +out: + return ERR_PTR(status); +} + +static void olq_alloc_dquot(struct buffer_head *bh, void *private) +{ + int *offset = private; + struct ocfs2_local_disk_chunk *dchunk; + + dchunk = (struct ocfs2_local_disk_chunk *)bh->b_data; + ocfs2_set_bit_unaligned(*offset, dchunk->dqc_bitmap); + le32_add_cpu(&dchunk->dqc_free, -1); +} + +/* Create dquot in the local file for given id */ +int ocfs2_create_local_dquot(struct dquot *dquot) +{ + struct super_block *sb = dquot->dq_sb; + int type = dquot->dq_id.type; + struct inode *lqinode = sb_dqopt(sb)->files[type]; + struct ocfs2_quota_chunk *chunk; + struct ocfs2_dquot *od = OCFS2_DQUOT(dquot); + int offset; + int status; + u64 pcount; + + if (!down_write_trylock(&OCFS2_I(lqinode)->ip_alloc_sem)) + return -EBUSY; + + chunk = ocfs2_find_free_entry(sb, type, &offset); + if (!chunk) { + chunk = ocfs2_extend_local_quota_file(sb, type, &offset); + if (IS_ERR(chunk)) { + status = PTR_ERR(chunk); + goto out; + } + } else if (IS_ERR(chunk)) { + status = PTR_ERR(chunk); + goto out; + } + od->dq_local_off = ol_dqblk_off(sb, chunk->qc_num, offset); + od->dq_chunk = chunk; + status = ocfs2_extent_map_get_blocks(lqinode, + ol_dqblk_block(sb, chunk->qc_num, offset), + &od->dq_local_phys_blk, + &pcount, + NULL); + if (status < 0) { + mlog_errno(status); + goto out; + } + + /* Initialize dquot structure on disk */ + status = ocfs2_local_write_dquot(dquot); + if (status < 0) { + mlog_errno(status); + goto out; + } + + /* Mark structure as allocated */ + status = ocfs2_modify_bh(lqinode, chunk->qc_headerbh, olq_alloc_dquot, + &offset); + if (status < 0) { + mlog_errno(status); + goto out; + } +out: + up_write(&OCFS2_I(lqinode)->ip_alloc_sem); + return status; +} + +/* + * Release dquot structure from local quota file. ocfs2_release_dquot() has + * already started a transaction and written all changes to global quota file + */ +int ocfs2_local_release_dquot(handle_t *handle, struct dquot *dquot) +{ + int status; + int type = dquot->dq_id.type; + struct ocfs2_dquot *od = OCFS2_DQUOT(dquot); + struct super_block *sb = dquot->dq_sb; + struct ocfs2_local_disk_chunk *dchunk; + int offset; + + status = ocfs2_journal_access_dq(handle, + INODE_CACHE(sb_dqopt(sb)->files[type]), + od->dq_chunk->qc_headerbh, OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto out; + } + offset = ol_dqblk_chunk_off(sb, od->dq_chunk->qc_num, + od->dq_local_off); + dchunk = (struct ocfs2_local_disk_chunk *) + (od->dq_chunk->qc_headerbh->b_data); + /* Mark structure as freed */ + lock_buffer(od->dq_chunk->qc_headerbh); + ocfs2_clear_bit_unaligned(offset, dchunk->dqc_bitmap); + le32_add_cpu(&dchunk->dqc_free, 1); + unlock_buffer(od->dq_chunk->qc_headerbh); + ocfs2_journal_dirty(handle, od->dq_chunk->qc_headerbh); + +out: + return status; +} + +static const struct quota_format_ops ocfs2_format_ops = { + .check_quota_file = ocfs2_local_check_quota_file, + .read_file_info = ocfs2_local_read_info, + .write_file_info = ocfs2_global_write_info, + .free_file_info = ocfs2_local_free_info, +}; + +struct quota_format_type ocfs2_quota_format = { + .qf_fmt_id = QFMT_OCFS2, + .qf_ops = &ocfs2_format_ops, + .qf_owner = THIS_MODULE +}; diff --git a/fs/ocfs2/refcounttree.c b/fs/ocfs2/refcounttree.c new file mode 100644 index 000000000..d9f22b4a2 --- /dev/null +++ b/fs/ocfs2/refcounttree.c @@ -0,0 +1,4834 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * refcounttree.c + * + * Copyright (C) 2009 Oracle. All rights reserved. + */ + +#include +#include +#include "ocfs2.h" +#include "inode.h" +#include "alloc.h" +#include "suballoc.h" +#include "journal.h" +#include "uptodate.h" +#include "super.h" +#include "buffer_head_io.h" +#include "blockcheck.h" +#include "refcounttree.h" +#include "sysfile.h" +#include "dlmglue.h" +#include "extent_map.h" +#include "aops.h" +#include "xattr.h" +#include "namei.h" +#include "ocfs2_trace.h" +#include "file.h" +#include "symlink.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct ocfs2_cow_context { + struct inode *inode; + u32 cow_start; + u32 cow_len; + struct ocfs2_extent_tree data_et; + struct ocfs2_refcount_tree *ref_tree; + struct buffer_head *ref_root_bh; + struct ocfs2_alloc_context *meta_ac; + struct ocfs2_alloc_context *data_ac; + struct ocfs2_cached_dealloc_ctxt dealloc; + void *cow_object; + struct ocfs2_post_refcount *post_refcount; + int extra_credits; + int (*get_clusters)(struct ocfs2_cow_context *context, + u32 v_cluster, u32 *p_cluster, + u32 *num_clusters, + unsigned int *extent_flags); + int (*cow_duplicate_clusters)(handle_t *handle, + struct inode *inode, + u32 cpos, u32 old_cluster, + u32 new_cluster, u32 new_len); +}; + +static inline struct ocfs2_refcount_tree * +cache_info_to_refcount(struct ocfs2_caching_info *ci) +{ + return container_of(ci, struct ocfs2_refcount_tree, rf_ci); +} + +static int ocfs2_validate_refcount_block(struct super_block *sb, + struct buffer_head *bh) +{ + int rc; + struct ocfs2_refcount_block *rb = + (struct ocfs2_refcount_block *)bh->b_data; + + trace_ocfs2_validate_refcount_block((unsigned long long)bh->b_blocknr); + + BUG_ON(!buffer_uptodate(bh)); + + /* + * If the ecc fails, we return the error but otherwise + * leave the filesystem running. We know any error is + * local to this block. + */ + rc = ocfs2_validate_meta_ecc(sb, bh->b_data, &rb->rf_check); + if (rc) { + mlog(ML_ERROR, "Checksum failed for refcount block %llu\n", + (unsigned long long)bh->b_blocknr); + return rc; + } + + + if (!OCFS2_IS_VALID_REFCOUNT_BLOCK(rb)) { + rc = ocfs2_error(sb, + "Refcount block #%llu has bad signature %.*s\n", + (unsigned long long)bh->b_blocknr, 7, + rb->rf_signature); + goto out; + } + + if (le64_to_cpu(rb->rf_blkno) != bh->b_blocknr) { + rc = ocfs2_error(sb, + "Refcount block #%llu has an invalid rf_blkno of %llu\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(rb->rf_blkno)); + goto out; + } + + if (le32_to_cpu(rb->rf_fs_generation) != OCFS2_SB(sb)->fs_generation) { + rc = ocfs2_error(sb, + "Refcount block #%llu has an invalid rf_fs_generation of #%u\n", + (unsigned long long)bh->b_blocknr, + le32_to_cpu(rb->rf_fs_generation)); + goto out; + } + + /* + * rf_records (rl_count/rl_used/rl_recs[]) is only meaningful when + * this block is not an interior tree block (OCFS2_REFCOUNT_TREE_FL); + * in that case the same union bytes hold an extent list (rf_list) + * instead, which is validated by ocfs2_validate_extent_block(). + */ + if (!(le32_to_cpu(rb->rf_flags) & OCFS2_REFCOUNT_TREE_FL)) { + if (le16_to_cpu(rb->rf_records.rl_count) != + ocfs2_refcount_recs_per_rb(sb)) { + rc = ocfs2_error(sb, + "Refcount block #%llu has an invalid rl_count of %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(rb->rf_records.rl_count)); + goto out; + } + + if (le16_to_cpu(rb->rf_records.rl_used) > + le16_to_cpu(rb->rf_records.rl_count)) { + rc = ocfs2_error(sb, + "Refcount block #%llu has an invalid rl_used of %u (rl_count %u)\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(rb->rf_records.rl_used), + le16_to_cpu(rb->rf_records.rl_count)); + goto out; + } + } +out: + return rc; +} + +static int ocfs2_read_refcount_block(struct ocfs2_caching_info *ci, + u64 rb_blkno, + struct buffer_head **bh) +{ + int rc; + struct buffer_head *tmp = *bh; + + rc = ocfs2_read_block(ci, rb_blkno, &tmp, + ocfs2_validate_refcount_block); + + /* If ocfs2_read_block() got us a new bh, pass it up. */ + if (!rc && !*bh) + *bh = tmp; + + return rc; +} + +static u64 ocfs2_refcount_cache_owner(struct ocfs2_caching_info *ci) +{ + struct ocfs2_refcount_tree *rf = cache_info_to_refcount(ci); + + return rf->rf_blkno; +} + +static struct super_block * +ocfs2_refcount_cache_get_super(struct ocfs2_caching_info *ci) +{ + struct ocfs2_refcount_tree *rf = cache_info_to_refcount(ci); + + return rf->rf_sb; +} + +static void ocfs2_refcount_cache_lock(struct ocfs2_caching_info *ci) +__acquires(&rf->rf_lock) +{ + struct ocfs2_refcount_tree *rf = cache_info_to_refcount(ci); + + spin_lock(&rf->rf_lock); +} + +static void ocfs2_refcount_cache_unlock(struct ocfs2_caching_info *ci) +__releases(&rf->rf_lock) +{ + struct ocfs2_refcount_tree *rf = cache_info_to_refcount(ci); + + spin_unlock(&rf->rf_lock); +} + +static void ocfs2_refcount_cache_io_lock(struct ocfs2_caching_info *ci) +{ + struct ocfs2_refcount_tree *rf = cache_info_to_refcount(ci); + + mutex_lock(&rf->rf_io_mutex); +} + +static void ocfs2_refcount_cache_io_unlock(struct ocfs2_caching_info *ci) +{ + struct ocfs2_refcount_tree *rf = cache_info_to_refcount(ci); + + mutex_unlock(&rf->rf_io_mutex); +} + +static const struct ocfs2_caching_operations ocfs2_refcount_caching_ops = { + .co_owner = ocfs2_refcount_cache_owner, + .co_get_super = ocfs2_refcount_cache_get_super, + .co_cache_lock = ocfs2_refcount_cache_lock, + .co_cache_unlock = ocfs2_refcount_cache_unlock, + .co_io_lock = ocfs2_refcount_cache_io_lock, + .co_io_unlock = ocfs2_refcount_cache_io_unlock, +}; + +static struct ocfs2_refcount_tree * +ocfs2_find_refcount_tree(struct ocfs2_super *osb, u64 blkno) +{ + struct rb_node *n = osb->osb_rf_lock_tree.rb_node; + struct ocfs2_refcount_tree *tree = NULL; + + while (n) { + tree = rb_entry(n, struct ocfs2_refcount_tree, rf_node); + + if (blkno < tree->rf_blkno) + n = n->rb_left; + else if (blkno > tree->rf_blkno) + n = n->rb_right; + else + return tree; + } + + return NULL; +} + +/* osb_lock is already locked. */ +static void ocfs2_insert_refcount_tree(struct ocfs2_super *osb, + struct ocfs2_refcount_tree *new) +{ + u64 rf_blkno = new->rf_blkno; + struct rb_node *parent = NULL; + struct rb_node **p = &osb->osb_rf_lock_tree.rb_node; + struct ocfs2_refcount_tree *tmp; + + while (*p) { + parent = *p; + + tmp = rb_entry(parent, struct ocfs2_refcount_tree, + rf_node); + + if (rf_blkno < tmp->rf_blkno) + p = &(*p)->rb_left; + else if (rf_blkno > tmp->rf_blkno) + p = &(*p)->rb_right; + else { + /* This should never happen! */ + mlog(ML_ERROR, "Duplicate refcount block %llu found!\n", + (unsigned long long)rf_blkno); + BUG(); + } + } + + rb_link_node(&new->rf_node, parent, p); + rb_insert_color(&new->rf_node, &osb->osb_rf_lock_tree); +} + +static void ocfs2_free_refcount_tree(struct ocfs2_refcount_tree *tree) +{ + ocfs2_metadata_cache_exit(&tree->rf_ci); + ocfs2_simple_drop_lockres(OCFS2_SB(tree->rf_sb), &tree->rf_lockres); + ocfs2_lock_res_free(&tree->rf_lockres); + kfree(tree); +} + +static inline void +ocfs2_erase_refcount_tree_from_list_no_lock(struct ocfs2_super *osb, + struct ocfs2_refcount_tree *tree) +{ + rb_erase(&tree->rf_node, &osb->osb_rf_lock_tree); + if (osb->osb_ref_tree_lru && osb->osb_ref_tree_lru == tree) + osb->osb_ref_tree_lru = NULL; +} + +static void ocfs2_erase_refcount_tree_from_list(struct ocfs2_super *osb, + struct ocfs2_refcount_tree *tree) +{ + spin_lock(&osb->osb_lock); + ocfs2_erase_refcount_tree_from_list_no_lock(osb, tree); + spin_unlock(&osb->osb_lock); +} + +static void ocfs2_kref_remove_refcount_tree(struct kref *kref) +{ + struct ocfs2_refcount_tree *tree = + container_of(kref, struct ocfs2_refcount_tree, rf_getcnt); + + ocfs2_free_refcount_tree(tree); +} + +static inline void +ocfs2_refcount_tree_get(struct ocfs2_refcount_tree *tree) +{ + kref_get(&tree->rf_getcnt); +} + +static inline void +ocfs2_refcount_tree_put(struct ocfs2_refcount_tree *tree) +{ + kref_put(&tree->rf_getcnt, ocfs2_kref_remove_refcount_tree); +} + +static inline void ocfs2_init_refcount_tree_ci(struct ocfs2_refcount_tree *new, + struct super_block *sb) +{ + ocfs2_metadata_cache_init(&new->rf_ci, &ocfs2_refcount_caching_ops); + mutex_init(&new->rf_io_mutex); + new->rf_sb = sb; + spin_lock_init(&new->rf_lock); +} + +static inline void ocfs2_init_refcount_tree_lock(struct ocfs2_super *osb, + struct ocfs2_refcount_tree *new, + u64 rf_blkno, u32 generation) +{ + init_rwsem(&new->rf_sem); + ocfs2_refcount_lock_res_init(&new->rf_lockres, osb, + rf_blkno, generation); +} + +static struct ocfs2_refcount_tree* +ocfs2_allocate_refcount_tree(struct ocfs2_super *osb, u64 rf_blkno) +{ + struct ocfs2_refcount_tree *new; + + new = kzalloc_obj(struct ocfs2_refcount_tree, GFP_NOFS); + if (!new) + return NULL; + + new->rf_blkno = rf_blkno; + kref_init(&new->rf_getcnt); + ocfs2_init_refcount_tree_ci(new, osb->sb); + + return new; +} + +static int ocfs2_get_refcount_tree(struct ocfs2_super *osb, u64 rf_blkno, + struct ocfs2_refcount_tree **ret_tree) +{ + int ret = 0; + struct ocfs2_refcount_tree *tree, *new = NULL; + struct buffer_head *ref_root_bh = NULL; + struct ocfs2_refcount_block *ref_rb; + + spin_lock(&osb->osb_lock); + if (osb->osb_ref_tree_lru && + osb->osb_ref_tree_lru->rf_blkno == rf_blkno) + tree = osb->osb_ref_tree_lru; + else + tree = ocfs2_find_refcount_tree(osb, rf_blkno); + if (tree) + goto out; + + spin_unlock(&osb->osb_lock); + + new = ocfs2_allocate_refcount_tree(osb, rf_blkno); + if (!new) { + ret = -ENOMEM; + mlog_errno(ret); + return ret; + } + /* + * We need the generation to create the refcount tree lock and since + * it isn't changed during the tree modification, we are safe here to + * read without protection. + * We also have to purge the cache after we create the lock since the + * refcount block may have the stale data. It can only be trusted when + * we hold the refcount lock. + */ + ret = ocfs2_read_refcount_block(&new->rf_ci, rf_blkno, &ref_root_bh); + if (ret) { + mlog_errno(ret); + ocfs2_metadata_cache_exit(&new->rf_ci); + kfree(new); + return ret; + } + + ref_rb = (struct ocfs2_refcount_block *)ref_root_bh->b_data; + new->rf_generation = le32_to_cpu(ref_rb->rf_generation); + ocfs2_init_refcount_tree_lock(osb, new, rf_blkno, + new->rf_generation); + ocfs2_metadata_cache_purge(&new->rf_ci); + + spin_lock(&osb->osb_lock); + tree = ocfs2_find_refcount_tree(osb, rf_blkno); + if (tree) + goto out; + + ocfs2_insert_refcount_tree(osb, new); + + tree = new; + new = NULL; + +out: + *ret_tree = tree; + + osb->osb_ref_tree_lru = tree; + + spin_unlock(&osb->osb_lock); + + if (new) + ocfs2_free_refcount_tree(new); + + brelse(ref_root_bh); + return ret; +} + +static int ocfs2_get_refcount_block(struct inode *inode, u64 *ref_blkno) +{ + int ret; + struct buffer_head *di_bh = NULL; + struct ocfs2_dinode *di; + + ret = ocfs2_read_inode_block(inode, &di_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + BUG_ON(!ocfs2_is_refcount_inode(inode)); + + di = (struct ocfs2_dinode *)di_bh->b_data; + *ref_blkno = le64_to_cpu(di->i_refcount_loc); + brelse(di_bh); +out: + return ret; +} + +static int __ocfs2_lock_refcount_tree(struct ocfs2_super *osb, + struct ocfs2_refcount_tree *tree, int rw) +{ + int ret; + + ret = ocfs2_refcount_lock(tree, rw); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (rw) + down_write(&tree->rf_sem); + else + down_read(&tree->rf_sem); + +out: + return ret; +} + +/* + * Lock the refcount tree pointed by ref_blkno and return the tree. + * In most case, we lock the tree and read the refcount block. + * So read it here if the caller really needs it. + * + * If the tree has been re-created by other node, it will free the + * old one and re-create it. + */ +int ocfs2_lock_refcount_tree(struct ocfs2_super *osb, + u64 ref_blkno, int rw, + struct ocfs2_refcount_tree **ret_tree, + struct buffer_head **ref_bh) +{ + int ret, delete_tree = 0; + struct ocfs2_refcount_tree *tree = NULL; + struct buffer_head *ref_root_bh = NULL; + struct ocfs2_refcount_block *rb; + +again: + ret = ocfs2_get_refcount_tree(osb, ref_blkno, &tree); + if (ret) { + mlog_errno(ret); + return ret; + } + + ocfs2_refcount_tree_get(tree); + + ret = __ocfs2_lock_refcount_tree(osb, tree, rw); + if (ret) { + mlog_errno(ret); + ocfs2_refcount_tree_put(tree); + goto out; + } + + ret = ocfs2_read_refcount_block(&tree->rf_ci, tree->rf_blkno, + &ref_root_bh); + if (ret) { + mlog_errno(ret); + ocfs2_unlock_refcount_tree(osb, tree, rw); + goto out; + } + + rb = (struct ocfs2_refcount_block *)ref_root_bh->b_data; + /* + * If the refcount block has been freed and re-created, we may need + * to recreate the refcount tree also. + * + * Here we just remove the tree from the rb-tree, and the last + * kref holder will unlock and delete this refcount_tree. + * Then we goto "again" and ocfs2_get_refcount_tree will create + * the new refcount tree for us. + */ + if (tree->rf_generation != le32_to_cpu(rb->rf_generation)) { + if (!tree->rf_removed) { + ocfs2_erase_refcount_tree_from_list(osb, tree); + tree->rf_removed = 1; + delete_tree = 1; + } + + ocfs2_unlock_refcount_tree(osb, tree, rw); + /* + * We get an extra reference when we create the refcount + * tree, so another put will destroy it. + */ + if (delete_tree) + ocfs2_refcount_tree_put(tree); + brelse(ref_root_bh); + ref_root_bh = NULL; + goto again; + } + + *ret_tree = tree; + if (ref_bh) { + *ref_bh = ref_root_bh; + ref_root_bh = NULL; + } +out: + brelse(ref_root_bh); + return ret; +} + +void ocfs2_unlock_refcount_tree(struct ocfs2_super *osb, + struct ocfs2_refcount_tree *tree, int rw) +{ + if (rw) + up_write(&tree->rf_sem); + else + up_read(&tree->rf_sem); + + ocfs2_refcount_unlock(tree, rw); + ocfs2_refcount_tree_put(tree); +} + +void ocfs2_purge_refcount_trees(struct ocfs2_super *osb) +{ + struct rb_node *node; + struct ocfs2_refcount_tree *tree; + struct rb_root *root = &osb->osb_rf_lock_tree; + + while ((node = rb_last(root)) != NULL) { + tree = rb_entry(node, struct ocfs2_refcount_tree, rf_node); + + trace_ocfs2_purge_refcount_trees( + (unsigned long long) tree->rf_blkno); + + rb_erase(&tree->rf_node, root); + ocfs2_free_refcount_tree(tree); + } +} + +/* + * Create a refcount tree for an inode. + * We take for granted that the inode is already locked. + */ +static int ocfs2_create_refcount_tree(struct inode *inode, + struct buffer_head *di_bh) +{ + int ret; + handle_t *handle = NULL; + struct ocfs2_alloc_context *meta_ac = NULL; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct buffer_head *new_bh = NULL; + struct ocfs2_refcount_block *rb; + struct ocfs2_refcount_tree *new_tree = NULL, *tree = NULL; + u16 suballoc_bit_start; + u32 num_got; + u64 suballoc_loc, first_blkno; + + BUG_ON(ocfs2_is_refcount_inode(inode)); + + trace_ocfs2_create_refcount_tree( + (unsigned long long)oi->ip_blkno); + + ret = ocfs2_reserve_new_metadata_blocks(osb, 1, &meta_ac); + if (ret) { + mlog_errno(ret); + goto out; + } + + handle = ocfs2_start_trans(osb, OCFS2_REFCOUNT_TREE_CREATE_CREDITS); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out; + } + + ret = ocfs2_journal_access_di(handle, INODE_CACHE(inode), di_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + ret = ocfs2_claim_metadata(handle, meta_ac, 1, &suballoc_loc, + &suballoc_bit_start, &num_got, + &first_blkno); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + new_tree = ocfs2_allocate_refcount_tree(osb, first_blkno); + if (!new_tree) { + ret = -ENOMEM; + mlog_errno(ret); + goto out_commit; + } + + new_bh = sb_getblk(inode->i_sb, first_blkno); + if (!new_bh) { + ret = -ENOMEM; + mlog_errno(ret); + goto out_commit; + } + ocfs2_set_new_buffer_uptodate(&new_tree->rf_ci, new_bh); + + ret = ocfs2_journal_access_rb(handle, &new_tree->rf_ci, new_bh, + OCFS2_JOURNAL_ACCESS_CREATE); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + /* Initialize ocfs2_refcount_block. */ + rb = (struct ocfs2_refcount_block *)new_bh->b_data; + memset(rb, 0, inode->i_sb->s_blocksize); + strscpy(rb->rf_signature, OCFS2_REFCOUNT_BLOCK_SIGNATURE); + rb->rf_suballoc_slot = cpu_to_le16(meta_ac->ac_alloc_slot); + rb->rf_suballoc_loc = cpu_to_le64(suballoc_loc); + rb->rf_suballoc_bit = cpu_to_le16(suballoc_bit_start); + rb->rf_fs_generation = cpu_to_le32(osb->fs_generation); + rb->rf_blkno = cpu_to_le64(first_blkno); + rb->rf_count = cpu_to_le32(1); + rb->rf_records.rl_count = + cpu_to_le16(ocfs2_refcount_recs_per_rb(osb->sb)); + spin_lock(&osb->osb_lock); + rb->rf_generation = cpu_to_le32(osb->s_next_generation++); + spin_unlock(&osb->osb_lock); + + ocfs2_journal_dirty(handle, new_bh); + + spin_lock(&oi->ip_lock); + oi->ip_dyn_features |= OCFS2_HAS_REFCOUNT_FL; + di->i_dyn_features = cpu_to_le16(oi->ip_dyn_features); + di->i_refcount_loc = cpu_to_le64(first_blkno); + spin_unlock(&oi->ip_lock); + + trace_ocfs2_create_refcount_tree_blkno((unsigned long long)first_blkno); + + ocfs2_journal_dirty(handle, di_bh); + + /* + * We have to init the tree lock here since it will use + * the generation number to create it. + */ + new_tree->rf_generation = le32_to_cpu(rb->rf_generation); + ocfs2_init_refcount_tree_lock(osb, new_tree, first_blkno, + new_tree->rf_generation); + + spin_lock(&osb->osb_lock); + tree = ocfs2_find_refcount_tree(osb, first_blkno); + + /* + * We've just created a new refcount tree in this block. If + * we found a refcount tree on the ocfs2_super, it must be + * one we just deleted. We free the old tree before + * inserting the new tree. + */ + BUG_ON(tree && tree->rf_generation == new_tree->rf_generation); + if (tree) + ocfs2_erase_refcount_tree_from_list_no_lock(osb, tree); + ocfs2_insert_refcount_tree(osb, new_tree); + spin_unlock(&osb->osb_lock); + new_tree = NULL; + if (tree) + ocfs2_refcount_tree_put(tree); + +out_commit: + ocfs2_commit_trans(osb, handle); + +out: + if (new_tree) { + ocfs2_metadata_cache_exit(&new_tree->rf_ci); + kfree(new_tree); + } + + brelse(new_bh); + if (meta_ac) + ocfs2_free_alloc_context(meta_ac); + + return ret; +} + +static int ocfs2_set_refcount_tree(struct inode *inode, + struct buffer_head *di_bh, + u64 refcount_loc) +{ + int ret; + handle_t *handle = NULL; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct buffer_head *ref_root_bh = NULL; + struct ocfs2_refcount_block *rb; + struct ocfs2_refcount_tree *ref_tree; + + BUG_ON(ocfs2_is_refcount_inode(inode)); + + ret = ocfs2_lock_refcount_tree(osb, refcount_loc, 1, + &ref_tree, &ref_root_bh); + if (ret) { + mlog_errno(ret); + return ret; + } + + handle = ocfs2_start_trans(osb, OCFS2_REFCOUNT_TREE_SET_CREDITS); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out; + } + + ret = ocfs2_journal_access_di(handle, INODE_CACHE(inode), di_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + ret = ocfs2_journal_access_rb(handle, &ref_tree->rf_ci, ref_root_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + rb = (struct ocfs2_refcount_block *)ref_root_bh->b_data; + le32_add_cpu(&rb->rf_count, 1); + + ocfs2_journal_dirty(handle, ref_root_bh); + + spin_lock(&oi->ip_lock); + oi->ip_dyn_features |= OCFS2_HAS_REFCOUNT_FL; + di->i_dyn_features = cpu_to_le16(oi->ip_dyn_features); + di->i_refcount_loc = cpu_to_le64(refcount_loc); + spin_unlock(&oi->ip_lock); + ocfs2_journal_dirty(handle, di_bh); + +out_commit: + ocfs2_commit_trans(osb, handle); +out: + ocfs2_unlock_refcount_tree(osb, ref_tree, 1); + brelse(ref_root_bh); + + return ret; +} + +int ocfs2_remove_refcount_tree(struct inode *inode, struct buffer_head *di_bh) +{ + int ret, delete_tree = 0; + handle_t *handle = NULL; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct ocfs2_refcount_block *rb; + struct inode *alloc_inode = NULL; + struct buffer_head *alloc_bh = NULL; + struct buffer_head *blk_bh = NULL; + struct ocfs2_refcount_tree *ref_tree; + int credits = OCFS2_REFCOUNT_TREE_REMOVE_CREDITS; + u64 blk = 0, bg_blkno = 0, ref_blkno = le64_to_cpu(di->i_refcount_loc); + u16 bit = 0; + + if (!ocfs2_is_refcount_inode(inode)) + return 0; + + BUG_ON(!ref_blkno); + ret = ocfs2_lock_refcount_tree(osb, ref_blkno, 1, &ref_tree, &blk_bh); + if (ret) { + mlog_errno(ret); + return ret; + } + + rb = (struct ocfs2_refcount_block *)blk_bh->b_data; + + /* + * If we are the last user, we need to free the block. + * So lock the allocator ahead. + */ + if (le32_to_cpu(rb->rf_count) == 1) { + blk = le64_to_cpu(rb->rf_blkno); + bit = le16_to_cpu(rb->rf_suballoc_bit); + if (rb->rf_suballoc_loc) + bg_blkno = le64_to_cpu(rb->rf_suballoc_loc); + else + bg_blkno = ocfs2_which_suballoc_group(blk, bit); + + alloc_inode = ocfs2_get_system_file_inode(osb, + EXTENT_ALLOC_SYSTEM_INODE, + le16_to_cpu(rb->rf_suballoc_slot)); + if (!alloc_inode) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + inode_lock(alloc_inode); + + ret = ocfs2_inode_lock(alloc_inode, &alloc_bh, 1); + if (ret) { + mlog_errno(ret); + goto out_mutex; + } + + credits += OCFS2_SUBALLOC_FREE; + } + + handle = ocfs2_start_trans(osb, credits); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out_unlock; + } + + ret = ocfs2_journal_access_di(handle, INODE_CACHE(inode), di_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + ret = ocfs2_journal_access_rb(handle, &ref_tree->rf_ci, blk_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + spin_lock(&oi->ip_lock); + oi->ip_dyn_features &= ~OCFS2_HAS_REFCOUNT_FL; + di->i_dyn_features = cpu_to_le16(oi->ip_dyn_features); + di->i_refcount_loc = 0; + spin_unlock(&oi->ip_lock); + ocfs2_journal_dirty(handle, di_bh); + + le32_add_cpu(&rb->rf_count , -1); + ocfs2_journal_dirty(handle, blk_bh); + + if (!rb->rf_count) { + delete_tree = 1; + ocfs2_erase_refcount_tree_from_list(osb, ref_tree); + ret = ocfs2_free_suballoc_bits(handle, alloc_inode, + alloc_bh, bit, bg_blkno, 1); + if (ret) + mlog_errno(ret); + } + +out_commit: + ocfs2_commit_trans(osb, handle); +out_unlock: + if (alloc_inode) { + ocfs2_inode_unlock(alloc_inode, 1); + brelse(alloc_bh); + } +out_mutex: + if (alloc_inode) { + inode_unlock(alloc_inode); + iput(alloc_inode); + } +out: + ocfs2_unlock_refcount_tree(osb, ref_tree, 1); + if (delete_tree) + ocfs2_refcount_tree_put(ref_tree); + brelse(blk_bh); + + return ret; +} + +static void ocfs2_find_refcount_rec_in_rl(struct ocfs2_caching_info *ci, + struct buffer_head *ref_leaf_bh, + u64 cpos, unsigned int len, + struct ocfs2_refcount_rec *ret_rec, + int *index) +{ + int i = 0; + struct ocfs2_refcount_block *rb = + (struct ocfs2_refcount_block *)ref_leaf_bh->b_data; + struct ocfs2_refcount_rec *rec = NULL; + + for (; i < le16_to_cpu(rb->rf_records.rl_used); i++) { + rec = &rb->rf_records.rl_recs[i]; + + if (le64_to_cpu(rec->r_cpos) + + le32_to_cpu(rec->r_clusters) <= cpos) + continue; + else if (le64_to_cpu(rec->r_cpos) > cpos) + break; + + /* ok, cpos fail in this rec. Just return. */ + if (ret_rec) + *ret_rec = *rec; + goto out; + } + + if (ret_rec) { + /* We meet with a hole here, so fake the rec. */ + ret_rec->r_cpos = cpu_to_le64(cpos); + ret_rec->r_refcount = 0; + if (i < le16_to_cpu(rb->rf_records.rl_used) && + le64_to_cpu(rec->r_cpos) < cpos + len) + ret_rec->r_clusters = + cpu_to_le32(le64_to_cpu(rec->r_cpos) - cpos); + else + ret_rec->r_clusters = cpu_to_le32(len); + } + +out: + *index = i; +} + +/* + * Try to remove refcount tree. The mechanism is: + * 1) Check whether i_clusters == 0, if no, exit. + * 2) check whether we have i_xattr_loc in dinode. if yes, exit. + * 3) Check whether we have inline xattr stored outside, if yes, exit. + * 4) Remove the tree. + */ +int ocfs2_try_remove_refcount_tree(struct inode *inode, + struct buffer_head *di_bh) +{ + int ret; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data; + + down_write(&oi->ip_xattr_sem); + down_write(&oi->ip_alloc_sem); + + if (oi->ip_clusters) + goto out; + + if ((oi->ip_dyn_features & OCFS2_HAS_XATTR_FL) && di->i_xattr_loc) + goto out; + + if (oi->ip_dyn_features & OCFS2_INLINE_XATTR_FL && + ocfs2_has_inline_xattr_value_outside(inode, di)) + goto out; + + ret = ocfs2_remove_refcount_tree(inode, di_bh); + if (ret) + mlog_errno(ret); +out: + up_write(&oi->ip_alloc_sem); + up_write(&oi->ip_xattr_sem); + return 0; +} + +/* + * Find the end range for a leaf refcount block indicated by + * el->l_recs[index].e_blkno. + */ +static int ocfs2_get_refcount_cpos_end(struct ocfs2_caching_info *ci, + struct buffer_head *ref_root_bh, + struct ocfs2_extent_block *eb, + struct ocfs2_extent_list *el, + int index, u32 *cpos_end) +{ + int ret, i, subtree_root; + u32 cpos; + u64 blkno; + struct super_block *sb = ocfs2_metadata_cache_get_super(ci); + struct ocfs2_path *left_path = NULL, *right_path = NULL; + struct ocfs2_extent_tree et; + struct ocfs2_extent_list *tmp_el; + + if (index < le16_to_cpu(el->l_next_free_rec) - 1) { + /* + * We have a extent rec after index, so just use the e_cpos + * of the next extent rec. + */ + *cpos_end = le32_to_cpu(el->l_recs[index+1].e_cpos); + return 0; + } + + if (!eb || !eb->h_next_leaf_blk) { + /* + * We are the last extent rec, so any high cpos should + * be stored in this leaf refcount block. + */ + *cpos_end = UINT_MAX; + return 0; + } + + /* + * If the extent block isn't the last one, we have to find + * the subtree root between this extent block and the next + * leaf extent block and get the corresponding e_cpos from + * the subroot. Otherwise we may corrupt the b-tree. + */ + ocfs2_init_refcount_extent_tree(&et, ci, ref_root_bh); + + left_path = ocfs2_new_path_from_et(&et); + if (!left_path) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + cpos = le32_to_cpu(eb->h_list.l_recs[index].e_cpos); + ret = ocfs2_find_path(ci, left_path, cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + + right_path = ocfs2_new_path_from_path(left_path); + if (!right_path) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + ret = ocfs2_find_cpos_for_right_leaf(sb, left_path, &cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_find_path(ci, right_path, cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + + subtree_root = ocfs2_find_subtree_root(&et, left_path, + right_path); + + tmp_el = left_path->p_node[subtree_root].el; + blkno = left_path->p_node[subtree_root+1].bh->b_blocknr; + for (i = 0; i < le16_to_cpu(tmp_el->l_next_free_rec); i++) { + if (le64_to_cpu(tmp_el->l_recs[i].e_blkno) == blkno) { + *cpos_end = le32_to_cpu(tmp_el->l_recs[i+1].e_cpos); + break; + } + } + + BUG_ON(i == le16_to_cpu(tmp_el->l_next_free_rec)); + +out: + ocfs2_free_path(left_path); + ocfs2_free_path(right_path); + return ret; +} + +/* + * Given a cpos and len, try to find the refcount record which contains cpos. + * 1. If cpos can be found in one refcount record, return the record. + * 2. If cpos can't be found, return a fake record which start from cpos + * and end at a small value between cpos+len and start of the next record. + * This fake record has r_refcount = 0. + */ +static int ocfs2_get_refcount_rec(struct ocfs2_caching_info *ci, + struct buffer_head *ref_root_bh, + u64 cpos, unsigned int len, + struct ocfs2_refcount_rec *ret_rec, + int *index, + struct buffer_head **ret_bh) +{ + int ret = 0, i, found; + u32 low_cpos, cpos_end; + struct ocfs2_extent_list *el; + struct ocfs2_extent_rec *rec = NULL; + struct ocfs2_extent_block *eb = NULL; + struct buffer_head *eb_bh = NULL, *ref_leaf_bh = NULL; + struct super_block *sb = ocfs2_metadata_cache_get_super(ci); + struct ocfs2_refcount_block *rb = + (struct ocfs2_refcount_block *)ref_root_bh->b_data; + + if (!(le32_to_cpu(rb->rf_flags) & OCFS2_REFCOUNT_TREE_FL)) { + ocfs2_find_refcount_rec_in_rl(ci, ref_root_bh, cpos, len, + ret_rec, index); + *ret_bh = ref_root_bh; + get_bh(ref_root_bh); + return 0; + } + + el = &rb->rf_list; + low_cpos = cpos & OCFS2_32BIT_POS_MASK; + + if (el->l_tree_depth) { + ret = ocfs2_find_leaf(ci, el, low_cpos, &eb_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + eb = (struct ocfs2_extent_block *) eb_bh->b_data; + el = &eb->h_list; + + if (el->l_tree_depth) { + ret = ocfs2_error(sb, + "refcount tree %llu has non zero tree depth in leaf btree tree block %llu\n", + (unsigned long long)ocfs2_metadata_cache_owner(ci), + (unsigned long long)eb_bh->b_blocknr); + goto out; + } + } + + found = 0; + for (i = le16_to_cpu(el->l_next_free_rec) - 1; i >= 0; i--) { + rec = &el->l_recs[i]; + + if (le32_to_cpu(rec->e_cpos) <= low_cpos) { + found = 1; + break; + } + } + + if (found) { + ret = ocfs2_get_refcount_cpos_end(ci, ref_root_bh, + eb, el, i, &cpos_end); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (cpos_end < low_cpos + len) + len = cpos_end - low_cpos; + } + + ret = ocfs2_read_refcount_block(ci, le64_to_cpu(rec->e_blkno), + &ref_leaf_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + ocfs2_find_refcount_rec_in_rl(ci, ref_leaf_bh, cpos, len, + ret_rec, index); + *ret_bh = ref_leaf_bh; +out: + brelse(eb_bh); + return ret; +} + +enum ocfs2_ref_rec_contig { + REF_CONTIG_NONE = 0, + REF_CONTIG_LEFT, + REF_CONTIG_RIGHT, + REF_CONTIG_LEFTRIGHT, +}; + +static enum ocfs2_ref_rec_contig + ocfs2_refcount_rec_adjacent(struct ocfs2_refcount_block *rb, + int index) +{ + if ((rb->rf_records.rl_recs[index].r_refcount == + rb->rf_records.rl_recs[index + 1].r_refcount) && + (le64_to_cpu(rb->rf_records.rl_recs[index].r_cpos) + + le32_to_cpu(rb->rf_records.rl_recs[index].r_clusters) == + le64_to_cpu(rb->rf_records.rl_recs[index + 1].r_cpos))) + return REF_CONTIG_RIGHT; + + return REF_CONTIG_NONE; +} + +static enum ocfs2_ref_rec_contig + ocfs2_refcount_rec_contig(struct ocfs2_refcount_block *rb, + int index) +{ + enum ocfs2_ref_rec_contig ret = REF_CONTIG_NONE; + + if (index < le16_to_cpu(rb->rf_records.rl_used) - 1) + ret = ocfs2_refcount_rec_adjacent(rb, index); + + if (index > 0) { + enum ocfs2_ref_rec_contig tmp; + + tmp = ocfs2_refcount_rec_adjacent(rb, index - 1); + + if (tmp == REF_CONTIG_RIGHT) { + if (ret == REF_CONTIG_RIGHT) + ret = REF_CONTIG_LEFTRIGHT; + else + ret = REF_CONTIG_LEFT; + } + } + + return ret; +} + +static void ocfs2_rotate_refcount_rec_left(struct ocfs2_refcount_block *rb, + int index) +{ + BUG_ON(rb->rf_records.rl_recs[index].r_refcount != + rb->rf_records.rl_recs[index+1].r_refcount); + + le32_add_cpu(&rb->rf_records.rl_recs[index].r_clusters, + le32_to_cpu(rb->rf_records.rl_recs[index+1].r_clusters)); + + if (index < le16_to_cpu(rb->rf_records.rl_used) - 2) + memmove(&rb->rf_records.rl_recs[index + 1], + &rb->rf_records.rl_recs[index + 2], + sizeof(struct ocfs2_refcount_rec) * + (le16_to_cpu(rb->rf_records.rl_used) - index - 2)); + + memset(&rb->rf_records.rl_recs[le16_to_cpu(rb->rf_records.rl_used) - 1], + 0, sizeof(struct ocfs2_refcount_rec)); + le16_add_cpu(&rb->rf_records.rl_used, -1); +} + +/* + * Merge the refcount rec if we are contiguous with the adjacent recs. + */ +static void ocfs2_refcount_rec_merge(struct ocfs2_refcount_block *rb, + int index) +{ + enum ocfs2_ref_rec_contig contig = + ocfs2_refcount_rec_contig(rb, index); + + if (contig == REF_CONTIG_NONE) + return; + + if (contig == REF_CONTIG_LEFT || contig == REF_CONTIG_LEFTRIGHT) { + BUG_ON(index == 0); + index--; + } + + ocfs2_rotate_refcount_rec_left(rb, index); + + if (contig == REF_CONTIG_LEFTRIGHT) + ocfs2_rotate_refcount_rec_left(rb, index); +} + +/* + * Change the refcount indexed by "index" in ref_bh. + * If refcount reaches 0, remove it. + */ +static int ocfs2_change_refcount_rec(handle_t *handle, + struct ocfs2_caching_info *ci, + struct buffer_head *ref_leaf_bh, + int index, int merge, int change) +{ + int ret; + struct ocfs2_refcount_block *rb = + (struct ocfs2_refcount_block *)ref_leaf_bh->b_data; + struct ocfs2_refcount_list *rl = &rb->rf_records; + struct ocfs2_refcount_rec *rec = &rl->rl_recs[index]; + + ret = ocfs2_journal_access_rb(handle, ci, ref_leaf_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + + trace_ocfs2_change_refcount_rec( + (unsigned long long)ocfs2_metadata_cache_owner(ci), + index, le32_to_cpu(rec->r_refcount), change); + le32_add_cpu(&rec->r_refcount, change); + + if (!rec->r_refcount) { + if (index != le16_to_cpu(rl->rl_used) - 1) { + memmove(rec, rec + 1, + (le16_to_cpu(rl->rl_used) - index - 1) * + sizeof(struct ocfs2_refcount_rec)); + memset(&rl->rl_recs[le16_to_cpu(rl->rl_used) - 1], + 0, sizeof(struct ocfs2_refcount_rec)); + } + + le16_add_cpu(&rl->rl_used, -1); + } else if (merge) + ocfs2_refcount_rec_merge(rb, index); + + ocfs2_journal_dirty(handle, ref_leaf_bh); +out: + return ret; +} + +static int ocfs2_expand_inline_ref_root(handle_t *handle, + struct ocfs2_caching_info *ci, + struct buffer_head *ref_root_bh, + struct buffer_head **ref_leaf_bh, + struct ocfs2_alloc_context *meta_ac) +{ + int ret; + u16 suballoc_bit_start; + u32 num_got; + u64 suballoc_loc, blkno; + struct super_block *sb = ocfs2_metadata_cache_get_super(ci); + struct buffer_head *new_bh = NULL; + struct ocfs2_refcount_block *new_rb; + struct ocfs2_refcount_block *root_rb = + (struct ocfs2_refcount_block *)ref_root_bh->b_data; + + ret = ocfs2_journal_access_rb(handle, ci, ref_root_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_claim_metadata(handle, meta_ac, 1, &suballoc_loc, + &suballoc_bit_start, &num_got, + &blkno); + if (ret) { + mlog_errno(ret); + goto out; + } + + new_bh = sb_getblk(sb, blkno); + if (new_bh == NULL) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + ocfs2_set_new_buffer_uptodate(ci, new_bh); + + ret = ocfs2_journal_access_rb(handle, ci, new_bh, + OCFS2_JOURNAL_ACCESS_CREATE); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * Initialize ocfs2_refcount_block. + * It should contain the same information as the old root. + * so just memcpy it and change the corresponding field. + */ + memcpy(new_bh->b_data, ref_root_bh->b_data, sb->s_blocksize); + + new_rb = (struct ocfs2_refcount_block *)new_bh->b_data; + new_rb->rf_suballoc_slot = cpu_to_le16(meta_ac->ac_alloc_slot); + new_rb->rf_suballoc_loc = cpu_to_le64(suballoc_loc); + new_rb->rf_suballoc_bit = cpu_to_le16(suballoc_bit_start); + new_rb->rf_blkno = cpu_to_le64(blkno); + new_rb->rf_cpos = cpu_to_le32(0); + new_rb->rf_parent = cpu_to_le64(ref_root_bh->b_blocknr); + new_rb->rf_flags = cpu_to_le32(OCFS2_REFCOUNT_LEAF_FL); + ocfs2_journal_dirty(handle, new_bh); + + /* Now change the root. */ + memset(&root_rb->rf_list, 0, sb->s_blocksize - + offsetof(struct ocfs2_refcount_block, rf_list)); + root_rb->rf_list.l_count = cpu_to_le16(ocfs2_extent_recs_per_rb(sb)); + root_rb->rf_clusters = cpu_to_le32(1); + root_rb->rf_list.l_next_free_rec = cpu_to_le16(1); + root_rb->rf_list.l_recs[0].e_blkno = cpu_to_le64(blkno); + root_rb->rf_list.l_recs[0].e_leaf_clusters = cpu_to_le16(1); + root_rb->rf_flags = cpu_to_le32(OCFS2_REFCOUNT_TREE_FL); + + ocfs2_journal_dirty(handle, ref_root_bh); + + trace_ocfs2_expand_inline_ref_root((unsigned long long)blkno, + le16_to_cpu(new_rb->rf_records.rl_used)); + + *ref_leaf_bh = new_bh; + new_bh = NULL; +out: + brelse(new_bh); + return ret; +} + +static int ocfs2_refcount_rec_no_intersect(struct ocfs2_refcount_rec *prev, + struct ocfs2_refcount_rec *next) +{ + if (ocfs2_get_ref_rec_low_cpos(prev) + le32_to_cpu(prev->r_clusters) <= + ocfs2_get_ref_rec_low_cpos(next)) + return 1; + + return 0; +} + +static int cmp_refcount_rec_by_low_cpos(const void *a, const void *b) +{ + const struct ocfs2_refcount_rec *l = a, *r = b; + u32 l_cpos = ocfs2_get_ref_rec_low_cpos(l); + u32 r_cpos = ocfs2_get_ref_rec_low_cpos(r); + + if (l_cpos > r_cpos) + return 1; + if (l_cpos < r_cpos) + return -1; + return 0; +} + +static int cmp_refcount_rec_by_cpos(const void *a, const void *b) +{ + const struct ocfs2_refcount_rec *l = a, *r = b; + u64 l_cpos = le64_to_cpu(l->r_cpos); + u64 r_cpos = le64_to_cpu(r->r_cpos); + + if (l_cpos > r_cpos) + return 1; + if (l_cpos < r_cpos) + return -1; + return 0; +} + +/* + * The refcount cpos are ordered by their 64bit cpos, + * But we will use the low 32 bit to be the e_cpos in the b-tree. + * So we need to make sure that this pos isn't intersected with others. + * + * Note: The refcount block is already sorted by their low 32 bit cpos, + * So just try the middle pos first, and we will exit when we find + * the good position. + */ +static int ocfs2_find_refcount_split_pos(struct ocfs2_refcount_list *rl, + u32 *split_pos, int *split_index) +{ + int num_used = le16_to_cpu(rl->rl_used); + int delta, middle = num_used / 2; + + for (delta = 0; delta < middle; delta++) { + /* Let's check delta earlier than middle */ + if (ocfs2_refcount_rec_no_intersect( + &rl->rl_recs[middle - delta - 1], + &rl->rl_recs[middle - delta])) { + *split_index = middle - delta; + break; + } + + /* For even counts, don't walk off the end */ + if ((middle + delta + 1) == num_used) + continue; + + /* Now try delta past middle */ + if (ocfs2_refcount_rec_no_intersect( + &rl->rl_recs[middle + delta], + &rl->rl_recs[middle + delta + 1])) { + *split_index = middle + delta + 1; + break; + } + } + + if (delta >= middle) + return -ENOSPC; + + *split_pos = ocfs2_get_ref_rec_low_cpos(&rl->rl_recs[*split_index]); + return 0; +} + +static int ocfs2_divide_leaf_refcount_block(struct buffer_head *ref_leaf_bh, + struct buffer_head *new_bh, + u32 *split_cpos) +{ + int split_index = 0, num_moved, ret; + u32 cpos = 0; + struct ocfs2_refcount_block *rb = + (struct ocfs2_refcount_block *)ref_leaf_bh->b_data; + struct ocfs2_refcount_list *rl = &rb->rf_records; + struct ocfs2_refcount_block *new_rb = + (struct ocfs2_refcount_block *)new_bh->b_data; + struct ocfs2_refcount_list *new_rl = &new_rb->rf_records; + + trace_ocfs2_divide_leaf_refcount_block( + (unsigned long long)ref_leaf_bh->b_blocknr, + le16_to_cpu(rl->rl_count), le16_to_cpu(rl->rl_used)); + + /* + * XXX: Improvement later. + * If we know all the high 32 bit cpos is the same, no need to sort. + * + * In order to make the whole process safe, we do: + * 1. sort the entries by their low 32 bit cpos first so that we can + * find the split cpos easily. + * 2. call ocfs2_insert_extent to insert the new refcount block. + * 3. move the refcount rec to the new block. + * 4. sort the entries by their 64 bit cpos. + * 5. dirty the new_rb and rb. + */ + sort(&rl->rl_recs, le16_to_cpu(rl->rl_used), + sizeof(struct ocfs2_refcount_rec), + cmp_refcount_rec_by_low_cpos, NULL); + + ret = ocfs2_find_refcount_split_pos(rl, &cpos, &split_index); + if (ret) { + mlog_errno(ret); + return ret; + } + + new_rb->rf_cpos = cpu_to_le32(cpos); + + /* move refcount records starting from split_index to the new block. */ + num_moved = le16_to_cpu(rl->rl_used) - split_index; + memcpy(new_rl->rl_recs, &rl->rl_recs[split_index], + num_moved * sizeof(struct ocfs2_refcount_rec)); + + /*ok, remove the entries we just moved over to the other block. */ + memset(&rl->rl_recs[split_index], 0, + num_moved * sizeof(struct ocfs2_refcount_rec)); + + /* change old and new rl_used accordingly. */ + le16_add_cpu(&rl->rl_used, -num_moved); + new_rl->rl_used = cpu_to_le16(num_moved); + + sort(&rl->rl_recs, le16_to_cpu(rl->rl_used), + sizeof(struct ocfs2_refcount_rec), + cmp_refcount_rec_by_cpos, NULL); + + sort(&new_rl->rl_recs, le16_to_cpu(new_rl->rl_used), + sizeof(struct ocfs2_refcount_rec), + cmp_refcount_rec_by_cpos, NULL); + + *split_cpos = cpos; + return 0; +} + +static int ocfs2_new_leaf_refcount_block(handle_t *handle, + struct ocfs2_caching_info *ci, + struct buffer_head *ref_root_bh, + struct buffer_head *ref_leaf_bh, + struct ocfs2_alloc_context *meta_ac) +{ + int ret; + u16 suballoc_bit_start; + u32 num_got, new_cpos; + u64 suballoc_loc, blkno; + struct super_block *sb = ocfs2_metadata_cache_get_super(ci); + struct ocfs2_refcount_block *root_rb = + (struct ocfs2_refcount_block *)ref_root_bh->b_data; + struct buffer_head *new_bh = NULL; + struct ocfs2_refcount_block *new_rb; + struct ocfs2_extent_tree ref_et; + + BUG_ON(!(le32_to_cpu(root_rb->rf_flags) & OCFS2_REFCOUNT_TREE_FL)); + + ret = ocfs2_journal_access_rb(handle, ci, ref_root_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_journal_access_rb(handle, ci, ref_leaf_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_claim_metadata(handle, meta_ac, 1, &suballoc_loc, + &suballoc_bit_start, &num_got, + &blkno); + if (ret) { + mlog_errno(ret); + goto out; + } + + new_bh = sb_getblk(sb, blkno); + if (new_bh == NULL) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + ocfs2_set_new_buffer_uptodate(ci, new_bh); + + ret = ocfs2_journal_access_rb(handle, ci, new_bh, + OCFS2_JOURNAL_ACCESS_CREATE); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* Initialize ocfs2_refcount_block. */ + new_rb = (struct ocfs2_refcount_block *)new_bh->b_data; + memset(new_rb, 0, sb->s_blocksize); + strscpy(new_rb->rf_signature, OCFS2_REFCOUNT_BLOCK_SIGNATURE); + new_rb->rf_suballoc_slot = cpu_to_le16(meta_ac->ac_alloc_slot); + new_rb->rf_suballoc_loc = cpu_to_le64(suballoc_loc); + new_rb->rf_suballoc_bit = cpu_to_le16(suballoc_bit_start); + new_rb->rf_fs_generation = cpu_to_le32(OCFS2_SB(sb)->fs_generation); + new_rb->rf_blkno = cpu_to_le64(blkno); + new_rb->rf_parent = cpu_to_le64(ref_root_bh->b_blocknr); + new_rb->rf_flags = cpu_to_le32(OCFS2_REFCOUNT_LEAF_FL); + new_rb->rf_records.rl_count = + cpu_to_le16(ocfs2_refcount_recs_per_rb(sb)); + new_rb->rf_generation = root_rb->rf_generation; + + ret = ocfs2_divide_leaf_refcount_block(ref_leaf_bh, new_bh, &new_cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + + ocfs2_journal_dirty(handle, ref_leaf_bh); + ocfs2_journal_dirty(handle, new_bh); + + ocfs2_init_refcount_extent_tree(&ref_et, ci, ref_root_bh); + + trace_ocfs2_new_leaf_refcount_block( + (unsigned long long)new_bh->b_blocknr, new_cpos); + + /* Insert the new leaf block with the specific offset cpos. */ + ret = ocfs2_insert_extent(handle, &ref_et, new_cpos, new_bh->b_blocknr, + 1, 0, meta_ac); + if (ret) + mlog_errno(ret); + +out: + brelse(new_bh); + return ret; +} + +static int ocfs2_expand_refcount_tree(handle_t *handle, + struct ocfs2_caching_info *ci, + struct buffer_head *ref_root_bh, + struct buffer_head *ref_leaf_bh, + struct ocfs2_alloc_context *meta_ac) +{ + int ret; + struct buffer_head *expand_bh = NULL; + + if (ref_root_bh == ref_leaf_bh) { + /* + * the old root bh hasn't been expanded to a b-tree, + * so expand it first. + */ + ret = ocfs2_expand_inline_ref_root(handle, ci, ref_root_bh, + &expand_bh, meta_ac); + if (ret) { + mlog_errno(ret); + goto out; + } + } else { + expand_bh = ref_leaf_bh; + get_bh(expand_bh); + } + + + /* Now add a new refcount block into the tree.*/ + ret = ocfs2_new_leaf_refcount_block(handle, ci, ref_root_bh, + expand_bh, meta_ac); + if (ret) + mlog_errno(ret); +out: + brelse(expand_bh); + return ret; +} + +/* + * Adjust the extent rec in b-tree representing ref_leaf_bh. + * + * Only called when we have inserted a new refcount rec at index 0 + * which means ocfs2_extent_rec.e_cpos may need some change. + */ +static int ocfs2_adjust_refcount_rec(handle_t *handle, + struct ocfs2_caching_info *ci, + struct buffer_head *ref_root_bh, + struct buffer_head *ref_leaf_bh, + struct ocfs2_refcount_rec *rec) +{ + int ret = 0, i; + u32 new_cpos, old_cpos; + struct ocfs2_path *path = NULL; + struct ocfs2_extent_tree et; + struct ocfs2_refcount_block *rb = + (struct ocfs2_refcount_block *)ref_root_bh->b_data; + struct ocfs2_extent_list *el; + + if (!(le32_to_cpu(rb->rf_flags) & OCFS2_REFCOUNT_TREE_FL)) + goto out; + + rb = (struct ocfs2_refcount_block *)ref_leaf_bh->b_data; + old_cpos = le32_to_cpu(rb->rf_cpos); + new_cpos = le64_to_cpu(rec->r_cpos) & OCFS2_32BIT_POS_MASK; + if (old_cpos <= new_cpos) + goto out; + + ocfs2_init_refcount_extent_tree(&et, ci, ref_root_bh); + + path = ocfs2_new_path_from_et(&et); + if (!path) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + ret = ocfs2_find_path(ci, path, old_cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * 2 more credits, one for the leaf refcount block, one for + * the extent block contains the extent rec. + */ + ret = ocfs2_extend_trans(handle, 2); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_journal_access_rb(handle, ci, ref_leaf_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_journal_access_eb(handle, ci, path_leaf_bh(path), + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + /* change the leaf extent block first. */ + el = path_leaf_el(path); + + for (i = 0; i < le16_to_cpu(el->l_next_free_rec); i++) + if (le32_to_cpu(el->l_recs[i].e_cpos) == old_cpos) + break; + + BUG_ON(i == le16_to_cpu(el->l_next_free_rec)); + + el->l_recs[i].e_cpos = cpu_to_le32(new_cpos); + + /* change the r_cpos in the leaf block. */ + rb->rf_cpos = cpu_to_le32(new_cpos); + + ocfs2_journal_dirty(handle, path_leaf_bh(path)); + ocfs2_journal_dirty(handle, ref_leaf_bh); + +out: + ocfs2_free_path(path); + return ret; +} + +static int ocfs2_insert_refcount_rec(handle_t *handle, + struct ocfs2_caching_info *ci, + struct buffer_head *ref_root_bh, + struct buffer_head *ref_leaf_bh, + struct ocfs2_refcount_rec *rec, + int index, int merge, + struct ocfs2_alloc_context *meta_ac) +{ + int ret; + struct ocfs2_refcount_block *rb = + (struct ocfs2_refcount_block *)ref_leaf_bh->b_data; + struct ocfs2_refcount_list *rf_list = &rb->rf_records; + struct buffer_head *new_bh = NULL; + + BUG_ON(le32_to_cpu(rb->rf_flags) & OCFS2_REFCOUNT_TREE_FL); + + if (rf_list->rl_used == rf_list->rl_count) { + u64 cpos = le64_to_cpu(rec->r_cpos); + u32 len = le32_to_cpu(rec->r_clusters); + + ret = ocfs2_expand_refcount_tree(handle, ci, ref_root_bh, + ref_leaf_bh, meta_ac); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_get_refcount_rec(ci, ref_root_bh, + cpos, len, NULL, &index, + &new_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + ref_leaf_bh = new_bh; + rb = (struct ocfs2_refcount_block *)ref_leaf_bh->b_data; + rf_list = &rb->rf_records; + } + + ret = ocfs2_journal_access_rb(handle, ci, ref_leaf_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (index < le16_to_cpu(rf_list->rl_used)) + memmove(&rf_list->rl_recs[index + 1], + &rf_list->rl_recs[index], + (le16_to_cpu(rf_list->rl_used) - index) * + sizeof(struct ocfs2_refcount_rec)); + + trace_ocfs2_insert_refcount_rec( + (unsigned long long)ref_leaf_bh->b_blocknr, index, + (unsigned long long)le64_to_cpu(rec->r_cpos), + le32_to_cpu(rec->r_clusters), le32_to_cpu(rec->r_refcount)); + + rf_list->rl_recs[index] = *rec; + + le16_add_cpu(&rf_list->rl_used, 1); + + if (merge) + ocfs2_refcount_rec_merge(rb, index); + + ocfs2_journal_dirty(handle, ref_leaf_bh); + + if (index == 0) { + ret = ocfs2_adjust_refcount_rec(handle, ci, + ref_root_bh, + ref_leaf_bh, rec); + if (ret) + mlog_errno(ret); + } +out: + brelse(new_bh); + return ret; +} + +/* + * Split the refcount_rec indexed by "index" in ref_leaf_bh. + * This is much simple than our b-tree code. + * split_rec is the new refcount rec we want to insert. + * If split_rec->r_refcount > 0, we are changing the refcount(in case we + * increase refcount or decrease a refcount to non-zero). + * If split_rec->r_refcount == 0, we are punching a hole in current refcount + * rec( in case we decrease a refcount to zero). + */ +static int ocfs2_split_refcount_rec(handle_t *handle, + struct ocfs2_caching_info *ci, + struct buffer_head *ref_root_bh, + struct buffer_head *ref_leaf_bh, + struct ocfs2_refcount_rec *split_rec, + int index, int merge, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_cached_dealloc_ctxt *dealloc) +{ + int ret, recs_need; + u32 len; + struct ocfs2_refcount_block *rb = + (struct ocfs2_refcount_block *)ref_leaf_bh->b_data; + struct ocfs2_refcount_list *rf_list = &rb->rf_records; + struct ocfs2_refcount_rec *orig_rec = &rf_list->rl_recs[index]; + struct ocfs2_refcount_rec *tail_rec = NULL; + struct buffer_head *new_bh = NULL; + + BUG_ON(le32_to_cpu(rb->rf_flags) & OCFS2_REFCOUNT_TREE_FL); + + trace_ocfs2_split_refcount_rec(le64_to_cpu(orig_rec->r_cpos), + le32_to_cpu(orig_rec->r_clusters), + le32_to_cpu(orig_rec->r_refcount), + le64_to_cpu(split_rec->r_cpos), + le32_to_cpu(split_rec->r_clusters), + le32_to_cpu(split_rec->r_refcount)); + + /* + * If we just need to split the header or tail clusters, + * no more recs are needed, just split is OK. + * Otherwise we at least need one new recs. + */ + if (!split_rec->r_refcount && + (split_rec->r_cpos == orig_rec->r_cpos || + le64_to_cpu(split_rec->r_cpos) + + le32_to_cpu(split_rec->r_clusters) == + le64_to_cpu(orig_rec->r_cpos) + le32_to_cpu(orig_rec->r_clusters))) + recs_need = 0; + else + recs_need = 1; + + /* + * We need one more rec if we split in the middle and the new rec have + * some refcount in it. + */ + if (split_rec->r_refcount && + (split_rec->r_cpos != orig_rec->r_cpos && + le64_to_cpu(split_rec->r_cpos) + + le32_to_cpu(split_rec->r_clusters) != + le64_to_cpu(orig_rec->r_cpos) + le32_to_cpu(orig_rec->r_clusters))) + recs_need++; + + /* If the leaf block don't have enough record, expand it. */ + if (le16_to_cpu(rf_list->rl_used) + recs_need > + le16_to_cpu(rf_list->rl_count)) { + struct ocfs2_refcount_rec tmp_rec; + u64 cpos = le64_to_cpu(orig_rec->r_cpos); + len = le32_to_cpu(orig_rec->r_clusters); + ret = ocfs2_expand_refcount_tree(handle, ci, ref_root_bh, + ref_leaf_bh, meta_ac); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * We have to re-get it since now cpos may be moved to + * another leaf block. + */ + ret = ocfs2_get_refcount_rec(ci, ref_root_bh, + cpos, len, &tmp_rec, &index, + &new_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + ref_leaf_bh = new_bh; + rb = (struct ocfs2_refcount_block *)ref_leaf_bh->b_data; + rf_list = &rb->rf_records; + orig_rec = &rf_list->rl_recs[index]; + } + + ret = ocfs2_journal_access_rb(handle, ci, ref_leaf_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * We have calculated out how many new records we need and store + * in recs_need, so spare enough space first by moving the records + * after "index" to the end. + */ + if (index != le16_to_cpu(rf_list->rl_used) - 1) + memmove(&rf_list->rl_recs[index + 1 + recs_need], + &rf_list->rl_recs[index + 1], + (le16_to_cpu(rf_list->rl_used) - index - 1) * + sizeof(struct ocfs2_refcount_rec)); + + len = (le64_to_cpu(orig_rec->r_cpos) + + le32_to_cpu(orig_rec->r_clusters)) - + (le64_to_cpu(split_rec->r_cpos) + + le32_to_cpu(split_rec->r_clusters)); + + /* + * If we have "len", the we will split in the tail and move it + * to the end of the space we have just spared. + */ + if (len) { + tail_rec = &rf_list->rl_recs[index + recs_need]; + + memcpy(tail_rec, orig_rec, sizeof(struct ocfs2_refcount_rec)); + le64_add_cpu(&tail_rec->r_cpos, + le32_to_cpu(tail_rec->r_clusters) - len); + tail_rec->r_clusters = cpu_to_le32(len); + } + + /* + * If the split pos isn't the same as the original one, we need to + * split in the head. + * + * Note: We have the chance that split_rec.r_refcount = 0, + * recs_need = 0 and len > 0, which means we just cut the head from + * the orig_rec and in that case we have done some modification in + * orig_rec above, so the check for r_cpos is faked. + */ + if (split_rec->r_cpos != orig_rec->r_cpos && tail_rec != orig_rec) { + len = le64_to_cpu(split_rec->r_cpos) - + le64_to_cpu(orig_rec->r_cpos); + orig_rec->r_clusters = cpu_to_le32(len); + index++; + } + + le16_add_cpu(&rf_list->rl_used, recs_need); + + if (split_rec->r_refcount) { + rf_list->rl_recs[index] = *split_rec; + trace_ocfs2_split_refcount_rec_insert( + (unsigned long long)ref_leaf_bh->b_blocknr, index, + (unsigned long long)le64_to_cpu(split_rec->r_cpos), + le32_to_cpu(split_rec->r_clusters), + le32_to_cpu(split_rec->r_refcount)); + + if (merge) + ocfs2_refcount_rec_merge(rb, index); + } + + ocfs2_journal_dirty(handle, ref_leaf_bh); + +out: + brelse(new_bh); + return ret; +} + +static int __ocfs2_increase_refcount(handle_t *handle, + struct ocfs2_caching_info *ci, + struct buffer_head *ref_root_bh, + u64 cpos, u32 len, int merge, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_cached_dealloc_ctxt *dealloc) +{ + int ret = 0, index; + struct buffer_head *ref_leaf_bh = NULL; + struct ocfs2_refcount_rec rec; + unsigned int set_len = 0; + + trace_ocfs2_increase_refcount_begin( + (unsigned long long)ocfs2_metadata_cache_owner(ci), + (unsigned long long)cpos, len); + + while (len) { + ret = ocfs2_get_refcount_rec(ci, ref_root_bh, + cpos, len, &rec, &index, + &ref_leaf_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + set_len = le32_to_cpu(rec.r_clusters); + + /* + * Here we may meet with 3 situations: + * + * 1. If we find an already existing record, and the length + * is the same, cool, we just need to increase the r_refcount + * and it is OK. + * 2. If we find a hole, just insert it with r_refcount = 1. + * 3. If we are in the middle of one extent record, split + * it. + */ + if (rec.r_refcount && le64_to_cpu(rec.r_cpos) == cpos && + set_len <= len) { + trace_ocfs2_increase_refcount_change( + (unsigned long long)cpos, set_len, + le32_to_cpu(rec.r_refcount)); + ret = ocfs2_change_refcount_rec(handle, ci, + ref_leaf_bh, index, + merge, 1); + if (ret) { + mlog_errno(ret); + goto out; + } + } else if (!rec.r_refcount) { + rec.r_refcount = cpu_to_le32(1); + + trace_ocfs2_increase_refcount_insert( + (unsigned long long)le64_to_cpu(rec.r_cpos), + set_len); + ret = ocfs2_insert_refcount_rec(handle, ci, ref_root_bh, + ref_leaf_bh, + &rec, index, + merge, meta_ac); + if (ret) { + mlog_errno(ret); + goto out; + } + } else { + set_len = min((u64)(cpos + len), + le64_to_cpu(rec.r_cpos) + set_len) - cpos; + rec.r_cpos = cpu_to_le64(cpos); + rec.r_clusters = cpu_to_le32(set_len); + le32_add_cpu(&rec.r_refcount, 1); + + trace_ocfs2_increase_refcount_split( + (unsigned long long)le64_to_cpu(rec.r_cpos), + set_len, le32_to_cpu(rec.r_refcount)); + ret = ocfs2_split_refcount_rec(handle, ci, + ref_root_bh, ref_leaf_bh, + &rec, index, merge, + meta_ac, dealloc); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + cpos += set_len; + len -= set_len; + brelse(ref_leaf_bh); + ref_leaf_bh = NULL; + } + +out: + brelse(ref_leaf_bh); + return ret; +} + +static int ocfs2_remove_refcount_extent(handle_t *handle, + struct ocfs2_caching_info *ci, + struct buffer_head *ref_root_bh, + struct buffer_head *ref_leaf_bh, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_cached_dealloc_ctxt *dealloc) +{ + int ret; + struct super_block *sb = ocfs2_metadata_cache_get_super(ci); + struct ocfs2_refcount_block *rb = + (struct ocfs2_refcount_block *)ref_leaf_bh->b_data; + struct ocfs2_extent_tree et; + + BUG_ON(rb->rf_records.rl_used); + + trace_ocfs2_remove_refcount_extent( + (unsigned long long)ocfs2_metadata_cache_owner(ci), + (unsigned long long)ref_leaf_bh->b_blocknr, + le32_to_cpu(rb->rf_cpos)); + + ocfs2_init_refcount_extent_tree(&et, ci, ref_root_bh); + ret = ocfs2_remove_extent(handle, &et, le32_to_cpu(rb->rf_cpos), + 1, meta_ac, dealloc); + if (ret) { + mlog_errno(ret); + goto out; + } + + ocfs2_remove_from_cache(ci, ref_leaf_bh); + + /* + * add the freed block to the dealloc so that it will be freed + * when we run dealloc. + */ + ret = ocfs2_cache_block_dealloc(dealloc, EXTENT_ALLOC_SYSTEM_INODE, + le16_to_cpu(rb->rf_suballoc_slot), + le64_to_cpu(rb->rf_suballoc_loc), + le64_to_cpu(rb->rf_blkno), + le16_to_cpu(rb->rf_suballoc_bit)); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_journal_access_rb(handle, ci, ref_root_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + + rb = (struct ocfs2_refcount_block *)ref_root_bh->b_data; + + le32_add_cpu(&rb->rf_clusters, -1); + + /* + * check whether we need to restore the root refcount block if + * there is no leaf extent block at atll. + */ + if (!rb->rf_list.l_next_free_rec) { + BUG_ON(rb->rf_clusters); + + trace_ocfs2_restore_refcount_block( + (unsigned long long)ref_root_bh->b_blocknr); + + rb->rf_flags = 0; + rb->rf_parent = 0; + rb->rf_cpos = 0; + rb->rf_records.rl_used = 0; + rb->rf_records.rl_reserved2 = 0; + rb->rf_records.rl_reserved1 = 0; + /* rl_count determines the memset size and fortify object size. */ + rb->rf_records.rl_count = + cpu_to_le16(ocfs2_refcount_recs_per_rb(sb)); + memset(rb->rf_records.rl_recs, 0, + le16_to_cpu(rb->rf_records.rl_count) * + sizeof(*rb->rf_records.rl_recs)); + } + + ocfs2_journal_dirty(handle, ref_root_bh); + +out: + return ret; +} + +int ocfs2_increase_refcount(handle_t *handle, + struct ocfs2_caching_info *ci, + struct buffer_head *ref_root_bh, + u64 cpos, u32 len, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_cached_dealloc_ctxt *dealloc) +{ + return __ocfs2_increase_refcount(handle, ci, ref_root_bh, + cpos, len, 1, + meta_ac, dealloc); +} + +static int ocfs2_decrease_refcount_rec(handle_t *handle, + struct ocfs2_caching_info *ci, + struct buffer_head *ref_root_bh, + struct buffer_head *ref_leaf_bh, + int index, u64 cpos, unsigned int len, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_cached_dealloc_ctxt *dealloc) +{ + int ret; + struct ocfs2_refcount_block *rb = + (struct ocfs2_refcount_block *)ref_leaf_bh->b_data; + struct ocfs2_refcount_rec *rec = &rb->rf_records.rl_recs[index]; + + BUG_ON(cpos < le64_to_cpu(rec->r_cpos)); + BUG_ON(cpos + len > + le64_to_cpu(rec->r_cpos) + le32_to_cpu(rec->r_clusters)); + + trace_ocfs2_decrease_refcount_rec( + (unsigned long long)ocfs2_metadata_cache_owner(ci), + (unsigned long long)cpos, len); + + if (cpos == le64_to_cpu(rec->r_cpos) && + len == le32_to_cpu(rec->r_clusters)) + ret = ocfs2_change_refcount_rec(handle, ci, + ref_leaf_bh, index, 1, -1); + else { + struct ocfs2_refcount_rec split = *rec; + split.r_cpos = cpu_to_le64(cpos); + split.r_clusters = cpu_to_le32(len); + + le32_add_cpu(&split.r_refcount, -1); + + ret = ocfs2_split_refcount_rec(handle, ci, + ref_root_bh, ref_leaf_bh, + &split, index, 1, + meta_ac, dealloc); + } + + if (ret) { + mlog_errno(ret); + goto out; + } + + /* Remove the leaf refcount block if it contains no refcount record. */ + if (!rb->rf_records.rl_used && ref_leaf_bh != ref_root_bh) { + ret = ocfs2_remove_refcount_extent(handle, ci, ref_root_bh, + ref_leaf_bh, meta_ac, + dealloc); + if (ret) + mlog_errno(ret); + } + +out: + return ret; +} + +static int __ocfs2_decrease_refcount(handle_t *handle, + struct ocfs2_caching_info *ci, + struct buffer_head *ref_root_bh, + u64 cpos, u32 len, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_cached_dealloc_ctxt *dealloc, + int delete) +{ + int ret = 0, index = 0; + struct ocfs2_refcount_rec rec; + unsigned int r_count = 0, r_len; + struct super_block *sb = ocfs2_metadata_cache_get_super(ci); + struct buffer_head *ref_leaf_bh = NULL; + + trace_ocfs2_decrease_refcount( + (unsigned long long)ocfs2_metadata_cache_owner(ci), + (unsigned long long)cpos, len, delete); + + while (len) { + ret = ocfs2_get_refcount_rec(ci, ref_root_bh, + cpos, len, &rec, &index, + &ref_leaf_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + r_count = le32_to_cpu(rec.r_refcount); + BUG_ON(r_count == 0); + if (!delete) + BUG_ON(r_count > 1); + + r_len = min((u64)(cpos + len), le64_to_cpu(rec.r_cpos) + + le32_to_cpu(rec.r_clusters)) - cpos; + + ret = ocfs2_decrease_refcount_rec(handle, ci, ref_root_bh, + ref_leaf_bh, index, + cpos, r_len, + meta_ac, dealloc); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (le32_to_cpu(rec.r_refcount) == 1 && delete) { + ret = ocfs2_cache_cluster_dealloc(dealloc, + ocfs2_clusters_to_blocks(sb, cpos), + r_len); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + cpos += r_len; + len -= r_len; + brelse(ref_leaf_bh); + ref_leaf_bh = NULL; + } + +out: + brelse(ref_leaf_bh); + return ret; +} + +/* Caller must hold refcount tree lock. */ +int ocfs2_decrease_refcount(struct inode *inode, + handle_t *handle, u32 cpos, u32 len, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_cached_dealloc_ctxt *dealloc, + int delete) +{ + int ret; + u64 ref_blkno; + struct buffer_head *ref_root_bh = NULL; + struct ocfs2_refcount_tree *tree; + + BUG_ON(!ocfs2_is_refcount_inode(inode)); + + ret = ocfs2_get_refcount_block(inode, &ref_blkno); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_get_refcount_tree(OCFS2_SB(inode->i_sb), ref_blkno, &tree); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_read_refcount_block(&tree->rf_ci, tree->rf_blkno, + &ref_root_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = __ocfs2_decrease_refcount(handle, &tree->rf_ci, ref_root_bh, + cpos, len, meta_ac, dealloc, delete); + if (ret) + mlog_errno(ret); +out: + brelse(ref_root_bh); + return ret; +} + +/* + * Mark the already-existing extent at cpos as refcounted for len clusters. + * This adds the refcount extent flag. + * + * If the existing extent is larger than the request, initiate a + * split. An attempt will be made at merging with adjacent extents. + * + * The caller is responsible for passing down meta_ac if we'll need it. + */ +static int ocfs2_mark_extent_refcounted(struct inode *inode, + struct ocfs2_extent_tree *et, + handle_t *handle, u32 cpos, + u32 len, u32 phys, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_cached_dealloc_ctxt *dealloc) +{ + int ret; + + trace_ocfs2_mark_extent_refcounted(OCFS2_I(inode)->ip_blkno, + cpos, len, phys); + + if (!ocfs2_refcount_tree(OCFS2_SB(inode->i_sb))) { + ret = ocfs2_error(inode->i_sb, "Inode %llu want to use refcount tree, but the feature bit is not set in the super block\n", + inode->i_ino); + goto out; + } + + ret = ocfs2_change_extent_flag(handle, et, cpos, + len, phys, meta_ac, dealloc, + OCFS2_EXT_REFCOUNTED, 0); + if (ret) + mlog_errno(ret); + +out: + return ret; +} + +/* + * Given some contiguous physical clusters, calculate what we need + * for modifying their refcount. + */ +static int ocfs2_calc_refcount_meta_credits(struct super_block *sb, + struct ocfs2_caching_info *ci, + struct buffer_head *ref_root_bh, + u64 start_cpos, + u32 clusters, + int *meta_add, + int *credits) +{ + int ret = 0, index, ref_blocks = 0, recs_add = 0; + u64 cpos = start_cpos; + struct ocfs2_refcount_block *rb; + struct ocfs2_refcount_rec rec; + struct buffer_head *ref_leaf_bh = NULL, *prev_bh = NULL; + u32 len; + + while (clusters) { + ret = ocfs2_get_refcount_rec(ci, ref_root_bh, + cpos, clusters, &rec, + &index, &ref_leaf_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (ref_leaf_bh != prev_bh) { + /* + * Now we encounter a new leaf block, so calculate + * whether we need to extend the old leaf. + */ + if (prev_bh) { + rb = (struct ocfs2_refcount_block *) + prev_bh->b_data; + + if (le16_to_cpu(rb->rf_records.rl_used) + + recs_add > + le16_to_cpu(rb->rf_records.rl_count)) + ref_blocks++; + } + + recs_add = 0; + *credits += 1; + brelse(prev_bh); + prev_bh = ref_leaf_bh; + get_bh(prev_bh); + } + + trace_ocfs2_calc_refcount_meta_credits_iterate( + recs_add, (unsigned long long)cpos, clusters, + (unsigned long long)le64_to_cpu(rec.r_cpos), + le32_to_cpu(rec.r_clusters), + le32_to_cpu(rec.r_refcount), index); + + len = min((u64)cpos + clusters, le64_to_cpu(rec.r_cpos) + + le32_to_cpu(rec.r_clusters)) - cpos; + /* + * We record all the records which will be inserted to the + * same refcount block, so that we can tell exactly whether + * we need a new refcount block or not. + * + * If we will insert a new one, this is easy and only happens + * during adding refcounted flag to the extent, so we don't + * have a chance of splitting. We just need one record. + * + * If the refcount rec already exists, that would be a little + * complicated. we may have to: + * 1) split at the beginning if the start pos isn't aligned. + * we need 1 more record in this case. + * 2) split int the end if the end pos isn't aligned. + * we need 1 more record in this case. + * 3) split in the middle because of file system fragmentation. + * we need 2 more records in this case(we can't detect this + * beforehand, so always think of the worst case). + */ + if (rec.r_refcount) { + recs_add += 2; + /* Check whether we need a split at the beginning. */ + if (cpos == start_cpos && + cpos != le64_to_cpu(rec.r_cpos)) + recs_add++; + + /* Check whether we need a split in the end. */ + if (cpos + clusters < le64_to_cpu(rec.r_cpos) + + le32_to_cpu(rec.r_clusters)) + recs_add++; + } else + recs_add++; + + brelse(ref_leaf_bh); + ref_leaf_bh = NULL; + clusters -= len; + cpos += len; + } + + if (prev_bh) { + rb = (struct ocfs2_refcount_block *)prev_bh->b_data; + + if (le16_to_cpu(rb->rf_records.rl_used) + recs_add > + le16_to_cpu(rb->rf_records.rl_count)) + ref_blocks++; + + *credits += 1; + } + + if (!ref_blocks) + goto out; + + *meta_add += ref_blocks; + *credits += ref_blocks; + + /* + * So we may need ref_blocks to insert into the tree. + * That also means we need to change the b-tree and add that number + * of records since we never merge them. + * We need one more block for expansion since the new created leaf + * block is also full and needs split. + */ + rb = (struct ocfs2_refcount_block *)ref_root_bh->b_data; + if (le32_to_cpu(rb->rf_flags) & OCFS2_REFCOUNT_TREE_FL) { + struct ocfs2_extent_tree et; + + ocfs2_init_refcount_extent_tree(&et, ci, ref_root_bh); + *meta_add += ocfs2_extend_meta_needed(et.et_root_el); + *credits += ocfs2_calc_extend_credits(sb, + et.et_root_el); + } else { + *credits += OCFS2_EXPAND_REFCOUNT_TREE_CREDITS; + *meta_add += 1; + } + +out: + + trace_ocfs2_calc_refcount_meta_credits( + (unsigned long long)start_cpos, clusters, + *meta_add, *credits); + brelse(ref_leaf_bh); + brelse(prev_bh); + return ret; +} + +/* + * For refcount tree, we will decrease some contiguous clusters + * refcount count, so just go through it to see how many blocks + * we gonna touch and whether we need to create new blocks. + * + * Normally the refcount blocks store these refcount should be + * contiguous also, so that we can get the number easily. + * We will at most add split 2 refcount records and 2 more + * refcount blocks, so just check it in a rough way. + * + * Caller must hold refcount tree lock. + */ +int ocfs2_prepare_refcount_change_for_del(struct inode *inode, + u64 refcount_loc, + u64 phys_blkno, + u32 clusters, + int *credits, + int *ref_blocks) +{ + int ret; + struct buffer_head *ref_root_bh = NULL; + struct ocfs2_refcount_tree *tree; + u64 start_cpos = ocfs2_blocks_to_clusters(inode->i_sb, phys_blkno); + + if (!ocfs2_refcount_tree(OCFS2_SB(inode->i_sb))) { + ret = ocfs2_error(inode->i_sb, "Inode %llu want to use refcount tree, but the feature bit is not set in the super block\n", + inode->i_ino); + goto out; + } + + BUG_ON(!ocfs2_is_refcount_inode(inode)); + + ret = ocfs2_get_refcount_tree(OCFS2_SB(inode->i_sb), + refcount_loc, &tree); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_read_refcount_block(&tree->rf_ci, refcount_loc, + &ref_root_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_calc_refcount_meta_credits(inode->i_sb, + &tree->rf_ci, + ref_root_bh, + start_cpos, clusters, + ref_blocks, credits); + if (ret) { + mlog_errno(ret); + goto out; + } + + trace_ocfs2_prepare_refcount_change_for_del(*ref_blocks, *credits); + +out: + brelse(ref_root_bh); + return ret; +} + +#define MAX_CONTIG_BYTES 1048576 + +static inline unsigned int ocfs2_cow_contig_clusters(struct super_block *sb) +{ + return ocfs2_clusters_for_bytes(sb, MAX_CONTIG_BYTES); +} + +static inline unsigned int ocfs2_cow_contig_mask(struct super_block *sb) +{ + return ~(ocfs2_cow_contig_clusters(sb) - 1); +} + +/* + * Given an extent that starts at 'start' and an I/O that starts at 'cpos', + * find an offset (start + (n * contig_clusters)) that is closest to cpos + * while still being less than or equal to it. + * + * The goal is to break the extent at a multiple of contig_clusters. + */ +static inline unsigned int ocfs2_cow_align_start(struct super_block *sb, + unsigned int start, + unsigned int cpos) +{ + BUG_ON(start > cpos); + + return start + ((cpos - start) & ocfs2_cow_contig_mask(sb)); +} + +/* + * Given a cluster count of len, pad it out so that it is a multiple + * of contig_clusters. + */ +static inline unsigned int ocfs2_cow_align_length(struct super_block *sb, + unsigned int len) +{ + unsigned int padded = + (len + (ocfs2_cow_contig_clusters(sb) - 1)) & + ocfs2_cow_contig_mask(sb); + + /* Did we wrap? */ + if (padded < len) + padded = UINT_MAX; + + return padded; +} + +/* + * Calculate out the start and number of virtual clusters we need to CoW. + * + * cpos is virtual start cluster position we want to do CoW in a + * file and write_len is the cluster length. + * max_cpos is the place where we want to stop CoW intentionally. + * + * Normal we will start CoW from the beginning of extent record containing cpos. + * We try to break up extents on boundaries of MAX_CONTIG_BYTES so that we + * get good I/O from the resulting extent tree. + */ +static int ocfs2_refcount_cal_cow_clusters(struct inode *inode, + struct ocfs2_extent_list *el, + u32 cpos, + u32 write_len, + u32 max_cpos, + u32 *cow_start, + u32 *cow_len) +{ + int ret = 0; + int tree_height = le16_to_cpu(el->l_tree_depth), i; + struct buffer_head *eb_bh = NULL; + struct ocfs2_extent_block *eb = NULL; + struct ocfs2_extent_rec *rec; + unsigned int want_clusters, rec_end = 0; + int contig_clusters = ocfs2_cow_contig_clusters(inode->i_sb); + int leaf_clusters; + + BUG_ON(cpos + write_len > max_cpos); + + if (tree_height > 0) { + ret = ocfs2_find_leaf(INODE_CACHE(inode), el, cpos, &eb_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + eb = (struct ocfs2_extent_block *) eb_bh->b_data; + el = &eb->h_list; + + if (el->l_tree_depth) { + ret = ocfs2_error(inode->i_sb, + "Inode %llu has non zero tree depth in leaf block %llu\n", + inode->i_ino, + (unsigned long long)eb_bh->b_blocknr); + goto out; + } + } + + *cow_len = 0; + for (i = 0; i < le16_to_cpu(el->l_next_free_rec); i++) { + rec = &el->l_recs[i]; + + if (ocfs2_is_empty_extent(rec)) { + mlog_bug_on_msg(i != 0, "Inode %llu has empty record in " + "index %d\n", inode->i_ino, i); + continue; + } + + if (le32_to_cpu(rec->e_cpos) + + le16_to_cpu(rec->e_leaf_clusters) <= cpos) + continue; + + if (*cow_len == 0) { + /* + * We should find a refcounted record in the + * first pass. + */ + BUG_ON(!(rec->e_flags & OCFS2_EXT_REFCOUNTED)); + *cow_start = le32_to_cpu(rec->e_cpos); + } + + /* + * If we encounter a hole, a non-refcounted record or + * pass the max_cpos, stop the search. + */ + if ((!(rec->e_flags & OCFS2_EXT_REFCOUNTED)) || + (*cow_len && rec_end != le32_to_cpu(rec->e_cpos)) || + (max_cpos <= le32_to_cpu(rec->e_cpos))) + break; + + leaf_clusters = le16_to_cpu(rec->e_leaf_clusters); + rec_end = le32_to_cpu(rec->e_cpos) + leaf_clusters; + if (rec_end > max_cpos) { + rec_end = max_cpos; + leaf_clusters = rec_end - le32_to_cpu(rec->e_cpos); + } + + /* + * How many clusters do we actually need from + * this extent? First we see how many we actually + * need to complete the write. If that's smaller + * than contig_clusters, we try for contig_clusters. + */ + if (!*cow_len) + want_clusters = write_len; + else + want_clusters = (cpos + write_len) - + (*cow_start + *cow_len); + if (want_clusters < contig_clusters) + want_clusters = contig_clusters; + + /* + * If the write does not cover the whole extent, we + * need to calculate how we're going to split the extent. + * We try to do it on contig_clusters boundaries. + * + * Any extent smaller than contig_clusters will be + * CoWed in its entirety. + */ + if (leaf_clusters <= contig_clusters) + *cow_len += leaf_clusters; + else if (*cow_len || (*cow_start == cpos)) { + /* + * This extent needs to be CoW'd from its + * beginning, so all we have to do is compute + * how many clusters to grab. We align + * want_clusters to the edge of contig_clusters + * to get better I/O. + */ + want_clusters = ocfs2_cow_align_length(inode->i_sb, + want_clusters); + + if (leaf_clusters < want_clusters) + *cow_len += leaf_clusters; + else + *cow_len += want_clusters; + } else if ((*cow_start + contig_clusters) >= + (cpos + write_len)) { + /* + * Breaking off contig_clusters at the front + * of the extent will cover our write. That's + * easy. + */ + *cow_len = contig_clusters; + } else if ((rec_end - cpos) <= contig_clusters) { + /* + * Breaking off contig_clusters at the tail of + * this extent will cover cpos. + */ + *cow_start = rec_end - contig_clusters; + *cow_len = contig_clusters; + } else if ((rec_end - cpos) <= want_clusters) { + /* + * While we can't fit the entire write in this + * extent, we know that the write goes from cpos + * to the end of the extent. Break that off. + * We try to break it at some multiple of + * contig_clusters from the front of the extent. + * Failing that (ie, cpos is within + * contig_clusters of the front), we'll CoW the + * entire extent. + */ + *cow_start = ocfs2_cow_align_start(inode->i_sb, + *cow_start, cpos); + *cow_len = rec_end - *cow_start; + } else { + /* + * Ok, the entire write lives in the middle of + * this extent. Let's try to slice the extent up + * nicely. Optimally, our CoW region starts at + * m*contig_clusters from the beginning of the + * extent and goes for n*contig_clusters, + * covering the entire write. + */ + *cow_start = ocfs2_cow_align_start(inode->i_sb, + *cow_start, cpos); + + want_clusters = (cpos + write_len) - *cow_start; + want_clusters = ocfs2_cow_align_length(inode->i_sb, + want_clusters); + if (*cow_start + want_clusters <= rec_end) + *cow_len = want_clusters; + else + *cow_len = rec_end - *cow_start; + } + + /* Have we covered our entire write yet? */ + if ((*cow_start + *cow_len) >= (cpos + write_len)) + break; + + /* + * If we reach the end of the extent block and don't get enough + * clusters, continue with the next extent block if possible. + */ + if (i + 1 == le16_to_cpu(el->l_next_free_rec) && + eb && eb->h_next_leaf_blk) { + brelse(eb_bh); + eb_bh = NULL; + + ret = ocfs2_read_extent_block(INODE_CACHE(inode), + le64_to_cpu(eb->h_next_leaf_blk), + &eb_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + eb = (struct ocfs2_extent_block *) eb_bh->b_data; + el = &eb->h_list; + i = -1; + } + } + +out: + brelse(eb_bh); + return ret; +} + +/* + * Prepare meta_ac, data_ac and calculate credits when we want to add some + * num_clusters in data_tree "et" and change the refcount for the old + * clusters(starting form p_cluster) in the refcount tree. + * + * Note: + * 1. since we may split the old tree, so we at most will need num_clusters + 2 + * more new leaf records. + * 2. In some case, we may not need to reserve new clusters(e.g, reflink), so + * just give data_ac = NULL. + */ +static int ocfs2_lock_refcount_allocators(struct super_block *sb, + u32 p_cluster, u32 num_clusters, + struct ocfs2_extent_tree *et, + struct ocfs2_caching_info *ref_ci, + struct buffer_head *ref_root_bh, + struct ocfs2_alloc_context **meta_ac, + struct ocfs2_alloc_context **data_ac, + int *credits) +{ + int ret = 0, meta_add = 0; + int num_free_extents = ocfs2_num_free_extents(et); + + if (num_free_extents < 0) { + ret = num_free_extents; + mlog_errno(ret); + goto out; + } + + if (num_free_extents < num_clusters + 2) + meta_add = + ocfs2_extend_meta_needed(et->et_root_el); + + *credits += ocfs2_calc_extend_credits(sb, et->et_root_el); + + ret = ocfs2_calc_refcount_meta_credits(sb, ref_ci, ref_root_bh, + p_cluster, num_clusters, + &meta_add, credits); + if (ret) { + mlog_errno(ret); + goto out; + } + + trace_ocfs2_lock_refcount_allocators(meta_add, *credits); + ret = ocfs2_reserve_new_metadata_blocks(OCFS2_SB(sb), meta_add, + meta_ac); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (data_ac) { + ret = ocfs2_reserve_clusters(OCFS2_SB(sb), num_clusters, + data_ac); + if (ret) + mlog_errno(ret); + } + +out: + if (ret) { + if (*meta_ac) { + ocfs2_free_alloc_context(*meta_ac); + *meta_ac = NULL; + } + } + + return ret; +} + +static int ocfs2_clear_cow_buffer(handle_t *handle, struct buffer_head *bh) +{ + BUG_ON(buffer_dirty(bh)); + + clear_buffer_mapped(bh); + + return 0; +} + +int ocfs2_duplicate_clusters_by_page(handle_t *handle, + struct inode *inode, + u32 cpos, u32 old_cluster, + u32 new_cluster, u32 new_len) +{ + int ret = 0, partial; + struct super_block *sb = inode->i_sb; + u64 new_block = ocfs2_clusters_to_blocks(sb, new_cluster); + pgoff_t page_index; + unsigned int from, to; + loff_t offset, end, map_end; + struct address_space *mapping = inode->i_mapping; + + trace_ocfs2_duplicate_clusters_by_page(cpos, old_cluster, + new_cluster, new_len); + + offset = ((loff_t)cpos) << OCFS2_SB(sb)->s_clustersize_bits; + end = offset + (new_len << OCFS2_SB(sb)->s_clustersize_bits); + /* + * We only duplicate pages until we reach the page contains i_size - 1. + * So trim 'end' to i_size. + */ + if (end > i_size_read(inode)) + end = i_size_read(inode); + + while (offset < end) { + struct folio *folio; + page_index = offset >> PAGE_SHIFT; + map_end = ((loff_t)page_index + 1) << PAGE_SHIFT; + if (map_end > end) + map_end = end; + + /* from, to is the offset within the page. */ + from = offset & (PAGE_SIZE - 1); + to = PAGE_SIZE; + if (map_end & (PAGE_SIZE - 1)) + to = map_end & (PAGE_SIZE - 1); + +retry: + folio = __filemap_get_folio(mapping, page_index, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, GFP_NOFS); + if (IS_ERR(folio)) { + ret = PTR_ERR(folio); + mlog_errno(ret); + break; + } + + /* + * In case PAGE_SIZE <= CLUSTER_SIZE, we do not expect a dirty + * page, so write it back. + */ + if (PAGE_SIZE <= OCFS2_SB(sb)->s_clustersize) { + if (folio_test_dirty(folio)) { + folio_unlock(folio); + folio_put(folio); + + ret = filemap_write_and_wait_range(mapping, + offset, map_end - 1); + goto retry; + } + } + + if (!folio_test_uptodate(folio)) { + ret = block_read_full_folio(folio, ocfs2_get_block); + if (ret) { + mlog_errno(ret); + goto unlock; + } + folio_lock(folio); + } + + if (folio_buffers(folio)) { + ret = walk_page_buffers(handle, folio_buffers(folio), + from, to, &partial, + ocfs2_clear_cow_buffer); + if (ret) { + mlog_errno(ret); + goto unlock; + } + } + + ocfs2_map_and_dirty_folio(inode, handle, from, to, + folio, 0, &new_block); + folio_mark_accessed(folio); +unlock: + folio_unlock(folio); + folio_put(folio); + offset = map_end; + if (ret) + break; + } + + return ret; +} + +int ocfs2_duplicate_clusters_by_jbd(handle_t *handle, + struct inode *inode, + u32 cpos, u32 old_cluster, + u32 new_cluster, u32 new_len) +{ + int ret = 0; + struct super_block *sb = inode->i_sb; + struct ocfs2_caching_info *ci = INODE_CACHE(inode); + int i, blocks = ocfs2_clusters_to_blocks(sb, new_len); + u64 old_block = ocfs2_clusters_to_blocks(sb, old_cluster); + u64 new_block = ocfs2_clusters_to_blocks(sb, new_cluster); + struct ocfs2_super *osb = OCFS2_SB(sb); + struct buffer_head *old_bh = NULL; + struct buffer_head *new_bh = NULL; + + trace_ocfs2_duplicate_clusters_by_page(cpos, old_cluster, + new_cluster, new_len); + + for (i = 0; i < blocks; i++, old_block++, new_block++) { + new_bh = sb_getblk(osb->sb, new_block); + if (new_bh == NULL) { + ret = -ENOMEM; + mlog_errno(ret); + break; + } + + ocfs2_set_new_buffer_uptodate(ci, new_bh); + + ret = ocfs2_read_block(ci, old_block, &old_bh, NULL); + if (ret) { + mlog_errno(ret); + break; + } + + ret = ocfs2_journal_access(handle, ci, new_bh, + OCFS2_JOURNAL_ACCESS_CREATE); + if (ret) { + mlog_errno(ret); + break; + } + + memcpy(new_bh->b_data, old_bh->b_data, sb->s_blocksize); + ocfs2_journal_dirty(handle, new_bh); + + brelse(new_bh); + brelse(old_bh); + new_bh = NULL; + old_bh = NULL; + } + + brelse(new_bh); + brelse(old_bh); + return ret; +} + +static int ocfs2_clear_ext_refcount(handle_t *handle, + struct ocfs2_extent_tree *et, + u32 cpos, u32 p_cluster, u32 len, + unsigned int ext_flags, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_cached_dealloc_ctxt *dealloc) +{ + int ret, index; + struct ocfs2_extent_rec replace_rec; + struct ocfs2_path *path = NULL; + struct ocfs2_extent_list *el; + struct super_block *sb = ocfs2_metadata_cache_get_super(et->et_ci); + u64 ino = ocfs2_metadata_cache_owner(et->et_ci); + + trace_ocfs2_clear_ext_refcount((unsigned long long)ino, + cpos, len, p_cluster, ext_flags); + + memset(&replace_rec, 0, sizeof(replace_rec)); + replace_rec.e_cpos = cpu_to_le32(cpos); + replace_rec.e_leaf_clusters = cpu_to_le16(len); + replace_rec.e_blkno = cpu_to_le64(ocfs2_clusters_to_blocks(sb, + p_cluster)); + replace_rec.e_flags = ext_flags; + replace_rec.e_flags &= ~OCFS2_EXT_REFCOUNTED; + + path = ocfs2_new_path_from_et(et); + if (!path) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + ret = ocfs2_find_path(et->et_ci, path, cpos); + if (ret) { + mlog_errno(ret); + goto out; + } + + el = path_leaf_el(path); + + index = ocfs2_search_extent_list(el, cpos); + if (index == -1) { + ret = ocfs2_error(sb, + "Inode %llu has an extent at cpos %u which can no longer be found\n", + (unsigned long long)ino, cpos); + goto out; + } + + ret = ocfs2_split_extent(handle, et, path, index, + &replace_rec, meta_ac, dealloc); + if (ret) + mlog_errno(ret); + +out: + ocfs2_free_path(path); + return ret; +} + +static int ocfs2_replace_clusters(handle_t *handle, + struct ocfs2_cow_context *context, + u32 cpos, u32 old, + u32 new, u32 len, + unsigned int ext_flags) +{ + int ret; + struct ocfs2_caching_info *ci = context->data_et.et_ci; + u64 ino = ocfs2_metadata_cache_owner(ci); + + trace_ocfs2_replace_clusters((unsigned long long)ino, + cpos, old, new, len, ext_flags); + + /*If the old clusters is unwritten, no need to duplicate. */ + if (!(ext_flags & OCFS2_EXT_UNWRITTEN)) { + ret = context->cow_duplicate_clusters(handle, context->inode, + cpos, old, new, len); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + ret = ocfs2_clear_ext_refcount(handle, &context->data_et, + cpos, new, len, ext_flags, + context->meta_ac, &context->dealloc); + if (ret) + mlog_errno(ret); +out: + return ret; +} + +int ocfs2_cow_sync_writeback(struct super_block *sb, + struct inode *inode, + u32 cpos, u32 num_clusters) +{ + int ret; + loff_t start, end; + + if (ocfs2_should_order_data(inode)) + return 0; + + start = ((loff_t)cpos) << OCFS2_SB(sb)->s_clustersize_bits; + end = start + (num_clusters << OCFS2_SB(sb)->s_clustersize_bits) - 1; + + ret = filemap_write_and_wait_range(inode->i_mapping, start, end); + if (ret < 0) + mlog_errno(ret); + + return ret; +} + +static int ocfs2_di_get_clusters(struct ocfs2_cow_context *context, + u32 v_cluster, u32 *p_cluster, + u32 *num_clusters, + unsigned int *extent_flags) +{ + return ocfs2_get_clusters(context->inode, v_cluster, p_cluster, + num_clusters, extent_flags); +} + +static int ocfs2_make_clusters_writable(struct super_block *sb, + struct ocfs2_cow_context *context, + u32 cpos, u32 p_cluster, + u32 num_clusters, unsigned int e_flags) +{ + int ret, delete, index, credits = 0; + u32 new_bit, new_len, orig_num_clusters; + unsigned int set_len; + struct ocfs2_super *osb = OCFS2_SB(sb); + handle_t *handle; + struct buffer_head *ref_leaf_bh = NULL; + struct ocfs2_caching_info *ref_ci = &context->ref_tree->rf_ci; + struct ocfs2_refcount_rec rec; + + trace_ocfs2_make_clusters_writable(cpos, p_cluster, + num_clusters, e_flags); + + ret = ocfs2_lock_refcount_allocators(sb, p_cluster, num_clusters, + &context->data_et, + ref_ci, + context->ref_root_bh, + &context->meta_ac, + &context->data_ac, &credits); + if (ret) { + mlog_errno(ret); + return ret; + } + + if (context->post_refcount) + credits += context->post_refcount->credits; + + credits += context->extra_credits; + handle = ocfs2_start_trans(osb, credits); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out; + } + + orig_num_clusters = num_clusters; + + while (num_clusters) { + ret = ocfs2_get_refcount_rec(ref_ci, context->ref_root_bh, + p_cluster, num_clusters, + &rec, &index, &ref_leaf_bh); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + BUG_ON(!rec.r_refcount); + set_len = min((u64)p_cluster + num_clusters, + le64_to_cpu(rec.r_cpos) + + le32_to_cpu(rec.r_clusters)) - p_cluster; + + /* + * There are many different situation here. + * 1. If refcount == 1, remove the flag and don't COW. + * 2. If refcount > 1, allocate clusters. + * Here we may not allocate r_len once at a time, so continue + * until we reach num_clusters. + */ + if (le32_to_cpu(rec.r_refcount) == 1) { + delete = 0; + ret = ocfs2_clear_ext_refcount(handle, + &context->data_et, + cpos, p_cluster, + set_len, e_flags, + context->meta_ac, + &context->dealloc); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + } else { + delete = 1; + + ret = __ocfs2_claim_clusters(handle, + context->data_ac, + 1, set_len, + &new_bit, &new_len); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + ret = ocfs2_replace_clusters(handle, context, + cpos, p_cluster, new_bit, + new_len, e_flags); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + set_len = new_len; + } + + ret = __ocfs2_decrease_refcount(handle, ref_ci, + context->ref_root_bh, + p_cluster, set_len, + context->meta_ac, + &context->dealloc, delete); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + cpos += set_len; + p_cluster += set_len; + num_clusters -= set_len; + brelse(ref_leaf_bh); + ref_leaf_bh = NULL; + } + + /* handle any post_cow action. */ + if (context->post_refcount && context->post_refcount->func) { + ret = context->post_refcount->func(context->inode, handle, + context->post_refcount->para); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + } + + /* + * Here we should write the new page out first if we are + * in write-back mode. + */ + if (context->get_clusters == ocfs2_di_get_clusters) { + ret = ocfs2_cow_sync_writeback(sb, context->inode, cpos, + orig_num_clusters); + if (ret) + mlog_errno(ret); + } + +out_commit: + ocfs2_commit_trans(osb, handle); + +out: + if (context->data_ac) { + ocfs2_free_alloc_context(context->data_ac); + context->data_ac = NULL; + } + if (context->meta_ac) { + ocfs2_free_alloc_context(context->meta_ac); + context->meta_ac = NULL; + } + brelse(ref_leaf_bh); + + return ret; +} + +static int ocfs2_replace_cow(struct ocfs2_cow_context *context) +{ + int ret = 0; + struct inode *inode = context->inode; + u32 cow_start = context->cow_start, cow_len = context->cow_len; + u32 p_cluster, num_clusters; + unsigned int ext_flags; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + if (!ocfs2_refcount_tree(osb)) { + return ocfs2_error(inode->i_sb, "Inode %llu want to use refcount tree, but the feature bit is not set in the super block\n", + inode->i_ino); + } + + ocfs2_init_dealloc_ctxt(&context->dealloc); + + while (cow_len) { + ret = context->get_clusters(context, cow_start, &p_cluster, + &num_clusters, &ext_flags); + if (ret) { + mlog_errno(ret); + break; + } + + BUG_ON(!(ext_flags & OCFS2_EXT_REFCOUNTED)); + + if (cow_len < num_clusters) + num_clusters = cow_len; + + ret = ocfs2_make_clusters_writable(inode->i_sb, context, + cow_start, p_cluster, + num_clusters, ext_flags); + if (ret) { + mlog_errno(ret); + break; + } + + cow_len -= num_clusters; + cow_start += num_clusters; + } + + if (ocfs2_dealloc_has_cluster(&context->dealloc)) + ocfs2_schedule_truncate_log_flush(osb, 1); + ocfs2_run_deallocs(osb, &context->dealloc); + + return ret; +} + +/* + * Starting at cpos, try to CoW write_len clusters. Don't CoW + * past max_cpos. This will stop when it runs into a hole or an + * unrefcounted extent. + */ +static int ocfs2_refcount_cow_hunk(struct inode *inode, + struct buffer_head *di_bh, + u32 cpos, u32 write_len, u32 max_cpos) +{ + int ret; + u32 cow_start = 0, cow_len = 0; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data; + struct buffer_head *ref_root_bh = NULL; + struct ocfs2_refcount_tree *ref_tree; + struct ocfs2_cow_context *context = NULL; + + BUG_ON(!ocfs2_is_refcount_inode(inode)); + + ret = ocfs2_refcount_cal_cow_clusters(inode, &di->id2.i_list, + cpos, write_len, max_cpos, + &cow_start, &cow_len); + if (ret) { + mlog_errno(ret); + goto out; + } + + trace_ocfs2_refcount_cow_hunk(OCFS2_I(inode)->ip_blkno, + cpos, write_len, max_cpos, + cow_start, cow_len); + + BUG_ON(cow_len == 0); + + context = kzalloc_obj(struct ocfs2_cow_context, GFP_NOFS); + if (!context) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + ret = ocfs2_lock_refcount_tree(osb, le64_to_cpu(di->i_refcount_loc), + 1, &ref_tree, &ref_root_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + context->inode = inode; + context->cow_start = cow_start; + context->cow_len = cow_len; + context->ref_tree = ref_tree; + context->ref_root_bh = ref_root_bh; + context->cow_duplicate_clusters = ocfs2_duplicate_clusters_by_page; + context->get_clusters = ocfs2_di_get_clusters; + + ocfs2_init_dinode_extent_tree(&context->data_et, + INODE_CACHE(inode), di_bh); + + ret = ocfs2_replace_cow(context); + if (ret) + mlog_errno(ret); + + /* + * truncate the extent map here since no matter whether we meet with + * any error during the action, we shouldn't trust cached extent map + * any more. + */ + ocfs2_extent_map_trunc(inode, cow_start); + + ocfs2_unlock_refcount_tree(osb, ref_tree, 1); + brelse(ref_root_bh); +out: + kfree(context); + return ret; +} + +/* + * CoW any and all clusters between cpos and cpos+write_len. + * Don't CoW past max_cpos. If this returns successfully, all + * clusters between cpos and cpos+write_len are safe to modify. + */ +int ocfs2_refcount_cow(struct inode *inode, + struct buffer_head *di_bh, + u32 cpos, u32 write_len, u32 max_cpos) +{ + int ret = 0; + u32 p_cluster, num_clusters; + unsigned int ext_flags; + + while (write_len) { + ret = ocfs2_get_clusters(inode, cpos, &p_cluster, + &num_clusters, &ext_flags); + if (ret) { + mlog_errno(ret); + break; + } + + if (write_len < num_clusters) + num_clusters = write_len; + + if (ext_flags & OCFS2_EXT_REFCOUNTED) { + ret = ocfs2_refcount_cow_hunk(inode, di_bh, cpos, + num_clusters, max_cpos); + if (ret) { + mlog_errno(ret); + break; + } + } + + write_len -= num_clusters; + cpos += num_clusters; + } + + return ret; +} + +static int ocfs2_xattr_value_get_clusters(struct ocfs2_cow_context *context, + u32 v_cluster, u32 *p_cluster, + u32 *num_clusters, + unsigned int *extent_flags) +{ + struct inode *inode = context->inode; + struct ocfs2_xattr_value_root *xv = context->cow_object; + + return ocfs2_xattr_get_clusters(inode, v_cluster, p_cluster, + num_clusters, &xv->xr_list, + extent_flags); +} + +/* + * Given a xattr value root, calculate the most meta/credits we need for + * refcount tree change if we truncate it to 0. + */ +int ocfs2_refcounted_xattr_delete_need(struct inode *inode, + struct ocfs2_caching_info *ref_ci, + struct buffer_head *ref_root_bh, + struct ocfs2_xattr_value_root *xv, + int *meta_add, int *credits) +{ + int ret = 0, index, ref_blocks = 0; + u32 p_cluster, num_clusters; + u32 cpos = 0, clusters = le32_to_cpu(xv->xr_clusters); + struct ocfs2_refcount_block *rb; + struct ocfs2_refcount_rec rec; + struct buffer_head *ref_leaf_bh = NULL; + + while (cpos < clusters) { + ret = ocfs2_xattr_get_clusters(inode, cpos, &p_cluster, + &num_clusters, &xv->xr_list, + NULL); + if (ret) { + mlog_errno(ret); + goto out; + } + + cpos += num_clusters; + + while (num_clusters) { + ret = ocfs2_get_refcount_rec(ref_ci, ref_root_bh, + p_cluster, num_clusters, + &rec, &index, + &ref_leaf_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + BUG_ON(!rec.r_refcount); + + rb = (struct ocfs2_refcount_block *)ref_leaf_bh->b_data; + + /* + * We really don't know whether the other clusters is in + * this refcount block or not, so just take the worst + * case that all the clusters are in this block and each + * one will split a refcount rec, so totally we need + * clusters * 2 new refcount rec. + */ + if (le16_to_cpu(rb->rf_records.rl_used) + clusters * 2 > + le16_to_cpu(rb->rf_records.rl_count)) + ref_blocks++; + + *credits += 1; + brelse(ref_leaf_bh); + ref_leaf_bh = NULL; + + if (num_clusters <= le32_to_cpu(rec.r_clusters)) + break; + else + num_clusters -= le32_to_cpu(rec.r_clusters); + p_cluster += num_clusters; + } + } + + *meta_add += ref_blocks; + if (!ref_blocks) + goto out; + + rb = (struct ocfs2_refcount_block *)ref_root_bh->b_data; + if (le32_to_cpu(rb->rf_flags) & OCFS2_REFCOUNT_TREE_FL) + *credits += OCFS2_EXPAND_REFCOUNT_TREE_CREDITS; + else { + struct ocfs2_extent_tree et; + + ocfs2_init_refcount_extent_tree(&et, ref_ci, ref_root_bh); + *credits += ocfs2_calc_extend_credits(inode->i_sb, + et.et_root_el); + } + +out: + brelse(ref_leaf_bh); + return ret; +} + +/* + * Do CoW for xattr. + */ +int ocfs2_refcount_cow_xattr(struct inode *inode, + struct ocfs2_dinode *di, + struct ocfs2_xattr_value_buf *vb, + struct ocfs2_refcount_tree *ref_tree, + struct buffer_head *ref_root_bh, + u32 cpos, u32 write_len, + struct ocfs2_post_refcount *post) +{ + int ret; + struct ocfs2_xattr_value_root *xv = vb->vb_xv; + struct ocfs2_cow_context *context = NULL; + u32 cow_start, cow_len; + + BUG_ON(!ocfs2_is_refcount_inode(inode)); + + ret = ocfs2_refcount_cal_cow_clusters(inode, &xv->xr_list, + cpos, write_len, UINT_MAX, + &cow_start, &cow_len); + if (ret) { + mlog_errno(ret); + goto out; + } + + BUG_ON(cow_len == 0); + + context = kzalloc_obj(struct ocfs2_cow_context, GFP_NOFS); + if (!context) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + context->inode = inode; + context->cow_start = cow_start; + context->cow_len = cow_len; + context->ref_tree = ref_tree; + context->ref_root_bh = ref_root_bh; + context->cow_object = xv; + + context->cow_duplicate_clusters = ocfs2_duplicate_clusters_by_jbd; + /* We need the extra credits for duplicate_clusters by jbd. */ + context->extra_credits = + ocfs2_clusters_to_blocks(inode->i_sb, 1) * cow_len; + context->get_clusters = ocfs2_xattr_value_get_clusters; + context->post_refcount = post; + + ocfs2_init_xattr_value_extent_tree(&context->data_et, + INODE_CACHE(inode), vb); + + ret = ocfs2_replace_cow(context); + if (ret) + mlog_errno(ret); + +out: + kfree(context); + return ret; +} + +/* + * Insert a new extent into refcount tree and mark a extent rec + * as refcounted in the dinode tree. + */ +int ocfs2_add_refcount_flag(struct inode *inode, + struct ocfs2_extent_tree *data_et, + struct ocfs2_caching_info *ref_ci, + struct buffer_head *ref_root_bh, + u32 cpos, u32 p_cluster, u32 num_clusters, + struct ocfs2_cached_dealloc_ctxt *dealloc, + struct ocfs2_post_refcount *post) +{ + int ret; + handle_t *handle; + int credits = 1, ref_blocks = 0; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct ocfs2_alloc_context *meta_ac = NULL; + + /* We need to be able to handle at least an extent tree split. */ + ref_blocks = ocfs2_extend_meta_needed(data_et->et_root_el); + + ret = ocfs2_calc_refcount_meta_credits(inode->i_sb, + ref_ci, ref_root_bh, + p_cluster, num_clusters, + &ref_blocks, &credits); + if (ret) { + mlog_errno(ret); + goto out; + } + + trace_ocfs2_add_refcount_flag(ref_blocks, credits); + + if (ref_blocks) { + ret = ocfs2_reserve_new_metadata_blocks(osb, + ref_blocks, &meta_ac); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + if (post) + credits += post->credits; + + handle = ocfs2_start_trans(osb, credits); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out; + } + + ret = ocfs2_mark_extent_refcounted(inode, data_et, handle, + cpos, num_clusters, p_cluster, + meta_ac, dealloc); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + ret = __ocfs2_increase_refcount(handle, ref_ci, ref_root_bh, + p_cluster, num_clusters, 0, + meta_ac, dealloc); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + if (post && post->func) { + ret = post->func(inode, handle, post->para); + if (ret) + mlog_errno(ret); + } + +out_commit: + ocfs2_commit_trans(osb, handle); +out: + if (meta_ac) + ocfs2_free_alloc_context(meta_ac); + return ret; +} + +static int ocfs2_change_ctime(struct inode *inode, + struct buffer_head *di_bh) +{ + int ret; + handle_t *handle; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data; + + handle = ocfs2_start_trans(OCFS2_SB(inode->i_sb), + OCFS2_INODE_UPDATE_CREDITS); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out; + } + + ret = ocfs2_journal_access_di(handle, INODE_CACHE(inode), di_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + inode_set_ctime_current(inode); + di->i_ctime = cpu_to_le64(inode_get_ctime_sec(inode)); + di->i_ctime_nsec = cpu_to_le32(inode_get_ctime_nsec(inode)); + + ocfs2_journal_dirty(handle, di_bh); + +out_commit: + ocfs2_commit_trans(OCFS2_SB(inode->i_sb), handle); +out: + return ret; +} + +static int ocfs2_attach_refcount_tree(struct inode *inode, + struct buffer_head *di_bh) +{ + int ret, data_changed = 0; + struct buffer_head *ref_root_bh = NULL; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct ocfs2_refcount_tree *ref_tree; + unsigned int ext_flags; + loff_t size; + u32 cpos, num_clusters, clusters, p_cluster; + struct ocfs2_cached_dealloc_ctxt dealloc; + struct ocfs2_extent_tree di_et; + + ocfs2_init_dealloc_ctxt(&dealloc); + + if (!ocfs2_is_refcount_inode(inode)) { + ret = ocfs2_create_refcount_tree(inode, di_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + BUG_ON(!di->i_refcount_loc); + ret = ocfs2_lock_refcount_tree(osb, + le64_to_cpu(di->i_refcount_loc), 1, + &ref_tree, &ref_root_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (oi->ip_dyn_features & OCFS2_INLINE_DATA_FL) + goto attach_xattr; + + ocfs2_init_dinode_extent_tree(&di_et, INODE_CACHE(inode), di_bh); + + size = i_size_read(inode); + clusters = ocfs2_clusters_for_bytes(inode->i_sb, size); + + cpos = 0; + while (cpos < clusters) { + ret = ocfs2_get_clusters(inode, cpos, &p_cluster, + &num_clusters, &ext_flags); + if (ret) { + mlog_errno(ret); + goto unlock; + } + if (p_cluster && !(ext_flags & OCFS2_EXT_REFCOUNTED)) { + ret = ocfs2_add_refcount_flag(inode, &di_et, + &ref_tree->rf_ci, + ref_root_bh, cpos, + p_cluster, num_clusters, + &dealloc, NULL); + if (ret) { + mlog_errno(ret); + goto unlock; + } + + data_changed = 1; + } + cpos += num_clusters; + } + +attach_xattr: + if (oi->ip_dyn_features & OCFS2_HAS_XATTR_FL) { + ret = ocfs2_xattr_attach_refcount_tree(inode, di_bh, + &ref_tree->rf_ci, + ref_root_bh, + &dealloc); + if (ret) { + mlog_errno(ret); + goto unlock; + } + } + + if (data_changed) { + ret = ocfs2_change_ctime(inode, di_bh); + if (ret) + mlog_errno(ret); + } + +unlock: + ocfs2_unlock_refcount_tree(osb, ref_tree, 1); + brelse(ref_root_bh); + + if (!ret && ocfs2_dealloc_has_cluster(&dealloc)) + ocfs2_schedule_truncate_log_flush(osb, 1); + ocfs2_run_deallocs(osb, &dealloc); +out: + /* + * Empty the extent map so that we may get the right extent + * record from the disk. + */ + ocfs2_extent_map_trunc(inode, 0); + + return ret; +} + +static int ocfs2_add_refcounted_extent(struct inode *inode, + struct ocfs2_extent_tree *et, + struct ocfs2_caching_info *ref_ci, + struct buffer_head *ref_root_bh, + u32 cpos, u32 p_cluster, u32 num_clusters, + unsigned int ext_flags, + struct ocfs2_cached_dealloc_ctxt *dealloc) +{ + int ret; + handle_t *handle; + int credits = 0; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct ocfs2_alloc_context *meta_ac = NULL; + + ret = ocfs2_lock_refcount_allocators(inode->i_sb, + p_cluster, num_clusters, + et, ref_ci, + ref_root_bh, &meta_ac, + NULL, &credits); + if (ret) { + mlog_errno(ret); + goto out; + } + + handle = ocfs2_start_trans(osb, credits); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out; + } + + ret = ocfs2_insert_extent(handle, et, cpos, + ocfs2_clusters_to_blocks(inode->i_sb, p_cluster), + num_clusters, ext_flags, meta_ac); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + ret = ocfs2_increase_refcount(handle, ref_ci, ref_root_bh, + p_cluster, num_clusters, + meta_ac, dealloc); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + ret = dquot_alloc_space_nodirty(inode, + ocfs2_clusters_to_bytes(osb->sb, num_clusters)); + if (ret) + mlog_errno(ret); + +out_commit: + ocfs2_commit_trans(osb, handle); +out: + if (meta_ac) + ocfs2_free_alloc_context(meta_ac); + return ret; +} + +static int ocfs2_duplicate_inline_data(struct inode *s_inode, + struct buffer_head *s_bh, + struct inode *t_inode, + struct buffer_head *t_bh) +{ + int ret; + handle_t *handle; + struct ocfs2_super *osb = OCFS2_SB(s_inode->i_sb); + struct ocfs2_dinode *s_di = (struct ocfs2_dinode *)s_bh->b_data; + struct ocfs2_dinode *t_di = (struct ocfs2_dinode *)t_bh->b_data; + + BUG_ON(!(OCFS2_I(s_inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL)); + + handle = ocfs2_start_trans(osb, OCFS2_INODE_UPDATE_CREDITS); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out; + } + + ret = ocfs2_journal_access_di(handle, INODE_CACHE(t_inode), t_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + t_di->id2.i_data.id_count = s_di->id2.i_data.id_count; + memcpy(t_di->id2.i_data.id_data, s_di->id2.i_data.id_data, + le16_to_cpu(s_di->id2.i_data.id_count)); + spin_lock(&OCFS2_I(t_inode)->ip_lock); + OCFS2_I(t_inode)->ip_dyn_features |= OCFS2_INLINE_DATA_FL; + t_di->i_dyn_features = cpu_to_le16(OCFS2_I(t_inode)->ip_dyn_features); + spin_unlock(&OCFS2_I(t_inode)->ip_lock); + + ocfs2_journal_dirty(handle, t_bh); + +out_commit: + ocfs2_commit_trans(osb, handle); +out: + return ret; +} + +static int ocfs2_duplicate_extent_list(struct inode *s_inode, + struct inode *t_inode, + struct buffer_head *t_bh, + struct ocfs2_caching_info *ref_ci, + struct buffer_head *ref_root_bh, + struct ocfs2_cached_dealloc_ctxt *dealloc) +{ + int ret = 0; + u32 p_cluster, num_clusters, clusters, cpos; + loff_t size; + unsigned int ext_flags; + struct ocfs2_extent_tree et; + + ocfs2_init_dinode_extent_tree(&et, INODE_CACHE(t_inode), t_bh); + + size = i_size_read(s_inode); + clusters = ocfs2_clusters_for_bytes(s_inode->i_sb, size); + + cpos = 0; + while (cpos < clusters) { + ret = ocfs2_get_clusters(s_inode, cpos, &p_cluster, + &num_clusters, &ext_flags); + if (ret) { + mlog_errno(ret); + goto out; + } + if (p_cluster) { + ret = ocfs2_add_refcounted_extent(t_inode, &et, + ref_ci, ref_root_bh, + cpos, p_cluster, + num_clusters, + ext_flags, + dealloc); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + cpos += num_clusters; + } + +out: + return ret; +} + +/* + * change the new file's attributes to the src. + * + * reflink creates a snapshot of a file, that means the attributes + * must be identical except for three exceptions - nlink, ino, and ctime. + */ +static int ocfs2_complete_reflink(struct inode *s_inode, + struct buffer_head *s_bh, + struct inode *t_inode, + struct buffer_head *t_bh, + bool preserve) +{ + int ret; + handle_t *handle; + struct ocfs2_dinode *s_di = (struct ocfs2_dinode *)s_bh->b_data; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)t_bh->b_data; + loff_t size = i_size_read(s_inode); + + handle = ocfs2_start_trans(OCFS2_SB(t_inode->i_sb), + OCFS2_INODE_UPDATE_CREDITS); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + return ret; + } + + ret = ocfs2_journal_access_di(handle, INODE_CACHE(t_inode), t_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + spin_lock(&OCFS2_I(t_inode)->ip_lock); + OCFS2_I(t_inode)->ip_clusters = OCFS2_I(s_inode)->ip_clusters; + OCFS2_I(t_inode)->ip_attr = OCFS2_I(s_inode)->ip_attr; + OCFS2_I(t_inode)->ip_dyn_features = OCFS2_I(s_inode)->ip_dyn_features; + spin_unlock(&OCFS2_I(t_inode)->ip_lock); + i_size_write(t_inode, size); + t_inode->i_blocks = s_inode->i_blocks; + + di->i_xattr_inline_size = s_di->i_xattr_inline_size; + di->i_clusters = s_di->i_clusters; + di->i_size = s_di->i_size; + di->i_dyn_features = s_di->i_dyn_features; + di->i_attr = s_di->i_attr; + + if (preserve) { + t_inode->i_uid = s_inode->i_uid; + t_inode->i_gid = s_inode->i_gid; + t_inode->i_mode = s_inode->i_mode; + di->i_uid = s_di->i_uid; + di->i_gid = s_di->i_gid; + di->i_mode = s_di->i_mode; + + /* + * update time. + * we want mtime to appear identical to the source and + * update ctime. + */ + inode_set_ctime_current(t_inode); + + di->i_ctime = cpu_to_le64(inode_get_ctime_sec(t_inode)); + di->i_ctime_nsec = cpu_to_le32(inode_get_ctime_nsec(t_inode)); + + inode_set_mtime_to_ts(t_inode, inode_get_mtime(s_inode)); + di->i_mtime = s_di->i_mtime; + di->i_mtime_nsec = s_di->i_mtime_nsec; + } + + ocfs2_journal_dirty(handle, t_bh); + +out_commit: + ocfs2_commit_trans(OCFS2_SB(t_inode->i_sb), handle); + return ret; +} + +static int ocfs2_create_reflink_node(struct inode *s_inode, + struct buffer_head *s_bh, + struct inode *t_inode, + struct buffer_head *t_bh, + bool preserve) +{ + int ret; + struct buffer_head *ref_root_bh = NULL; + struct ocfs2_cached_dealloc_ctxt dealloc; + struct ocfs2_super *osb = OCFS2_SB(s_inode->i_sb); + struct ocfs2_dinode *di = (struct ocfs2_dinode *)s_bh->b_data; + struct ocfs2_refcount_tree *ref_tree; + + ocfs2_init_dealloc_ctxt(&dealloc); + + ret = ocfs2_set_refcount_tree(t_inode, t_bh, + le64_to_cpu(di->i_refcount_loc)); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (OCFS2_I(s_inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL) { + ret = ocfs2_duplicate_inline_data(s_inode, s_bh, + t_inode, t_bh); + if (ret) + mlog_errno(ret); + goto out; + } + + ret = ocfs2_lock_refcount_tree(osb, le64_to_cpu(di->i_refcount_loc), + 1, &ref_tree, &ref_root_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_duplicate_extent_list(s_inode, t_inode, t_bh, + &ref_tree->rf_ci, ref_root_bh, + &dealloc); + if (ret) { + mlog_errno(ret); + goto out_unlock_refcount; + } + +out_unlock_refcount: + ocfs2_unlock_refcount_tree(osb, ref_tree, 1); + brelse(ref_root_bh); +out: + if (ocfs2_dealloc_has_cluster(&dealloc)) + ocfs2_schedule_truncate_log_flush(osb, 1); + ocfs2_run_deallocs(osb, &dealloc); + + return ret; +} + +static int __ocfs2_reflink(struct dentry *old_dentry, + struct buffer_head *old_bh, + struct inode *new_inode, + bool preserve) +{ + int ret; + struct inode *inode = d_inode(old_dentry); + struct buffer_head *new_bh = NULL; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + + if (oi->ip_flags & OCFS2_INODE_SYSTEM_FILE) { + ret = -EINVAL; + mlog_errno(ret); + goto out; + } + + ret = filemap_fdatawrite(inode->i_mapping); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_attach_refcount_tree(inode, old_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + inode_lock_nested(new_inode, I_MUTEX_CHILD); + ret = ocfs2_inode_lock_nested(new_inode, &new_bh, 1, + OI_LS_REFLINK_TARGET); + if (ret) { + mlog_errno(ret); + goto out_unlock; + } + + if ((oi->ip_dyn_features & OCFS2_HAS_XATTR_FL) && + (oi->ip_dyn_features & OCFS2_INLINE_XATTR_FL)) { + /* + * Adjust extent record count to reserve space for extended attribute. + * Inline data count had been adjusted in ocfs2_duplicate_inline_data(). + */ + struct ocfs2_inode_info *new_oi = OCFS2_I(new_inode); + + if (!(new_oi->ip_dyn_features & OCFS2_INLINE_DATA_FL) && + !(ocfs2_inode_is_fast_symlink(new_inode))) { + struct ocfs2_dinode *new_di = (struct ocfs2_dinode *)new_bh->b_data; + struct ocfs2_dinode *old_di = (struct ocfs2_dinode *)old_bh->b_data; + struct ocfs2_extent_list *el = &new_di->id2.i_list; + int inline_size = le16_to_cpu(old_di->i_xattr_inline_size); + + le16_add_cpu(&el->l_count, -(inline_size / + sizeof(struct ocfs2_extent_rec))); + } + } + + ret = ocfs2_create_reflink_node(inode, old_bh, + new_inode, new_bh, preserve); + if (ret) { + mlog_errno(ret); + goto inode_unlock; + } + + if (oi->ip_dyn_features & OCFS2_HAS_XATTR_FL) { + ret = ocfs2_reflink_xattrs(inode, old_bh, + new_inode, new_bh, + preserve); + if (ret) { + mlog_errno(ret); + goto inode_unlock; + } + } + + ret = ocfs2_complete_reflink(inode, old_bh, + new_inode, new_bh, preserve); + if (ret) + mlog_errno(ret); + +inode_unlock: + ocfs2_inode_unlock(new_inode, 1); + brelse(new_bh); +out_unlock: + inode_unlock(new_inode); +out: + if (!ret) { + ret = filemap_fdatawait(inode->i_mapping); + if (ret) + mlog_errno(ret); + } + return ret; +} + +static int ocfs2_reflink(struct dentry *old_dentry, struct inode *dir, + struct dentry *new_dentry, bool preserve) +{ + int error, had_lock; + struct inode *inode = d_inode(old_dentry); + struct buffer_head *old_bh = NULL; + struct inode *new_orphan_inode = NULL; + struct ocfs2_lock_holder oh; + + if (!ocfs2_refcount_tree(OCFS2_SB(inode->i_sb))) + return -EOPNOTSUPP; + + + error = ocfs2_create_inode_in_orphan(dir, inode->i_mode, + &new_orphan_inode); + if (error) { + mlog_errno(error); + goto out; + } + + error = ocfs2_rw_lock(inode, 1); + if (error) { + mlog_errno(error); + goto out; + } + + error = ocfs2_inode_lock(inode, &old_bh, 1); + if (error) { + mlog_errno(error); + ocfs2_rw_unlock(inode, 1); + goto out; + } + + down_write(&OCFS2_I(inode)->ip_xattr_sem); + down_write(&OCFS2_I(inode)->ip_alloc_sem); + error = __ocfs2_reflink(old_dentry, old_bh, + new_orphan_inode, preserve); + up_write(&OCFS2_I(inode)->ip_alloc_sem); + up_write(&OCFS2_I(inode)->ip_xattr_sem); + + ocfs2_inode_unlock(inode, 1); + ocfs2_rw_unlock(inode, 1); + brelse(old_bh); + + if (error) { + mlog_errno(error); + goto out; + } + + had_lock = ocfs2_inode_lock_tracker(new_orphan_inode, NULL, 1, + &oh); + if (had_lock < 0) { + error = had_lock; + mlog_errno(error); + goto out; + } + + /* If the security isn't preserved, we need to re-initialize them. */ + if (!preserve) { + error = ocfs2_init_security_and_acl(dir, new_orphan_inode, + &new_dentry->d_name); + if (error) + mlog_errno(error); + } + if (!error) { + error = ocfs2_mv_orphaned_inode_to_new(dir, new_orphan_inode, + new_dentry); + if (error) + mlog_errno(error); + } + ocfs2_inode_unlock_tracker(new_orphan_inode, 1, &oh, had_lock); + +out: + if (new_orphan_inode) { + /* + * We need to open_unlock the inode no matter whether we + * succeed or not, so that other nodes can delete it later. + */ + ocfs2_open_unlock(new_orphan_inode); + if (error) + iput(new_orphan_inode); + } + + return error; +} + +/* + * Below here are the bits used by OCFS2_IOC_REFLINK() to fake + * sys_reflink(). This will go away when vfs_reflink() exists in + * fs/namei.c. + */ + +/* copied from may_create in VFS. */ +static inline int ocfs2_may_create(struct inode *dir, struct dentry *child) +{ + if (d_really_is_positive(child)) + return -EEXIST; + if (IS_DEADDIR(dir)) + return -ENOENT; + return inode_permission(&nop_mnt_idmap, dir, MAY_WRITE | MAY_EXEC); +} + +/** + * ocfs2_vfs_reflink - Create a reference-counted link + * + * @old_dentry: source dentry + inode + * @dir: directory to create the target + * @new_dentry: target dentry + * @preserve: if true, preserve all file attributes + */ +static int ocfs2_vfs_reflink(struct dentry *old_dentry, struct inode *dir, + struct dentry *new_dentry, bool preserve) +{ + struct inode *inode = d_inode(old_dentry); + int error; + + if (!inode) + return -ENOENT; + + error = ocfs2_may_create(dir, new_dentry); + if (error) + return error; + + if (dir->i_sb != inode->i_sb) + return -EXDEV; + + /* + * A reflink to an append-only or immutable file cannot be created. + */ + if (IS_APPEND(inode) || IS_IMMUTABLE(inode)) + return -EPERM; + + /* Only regular files can be reflinked. */ + if (!S_ISREG(inode->i_mode)) + return -EPERM; + + /* + * If the caller wants to preserve ownership, they require the + * rights to do so. + */ + if (preserve) { + if (!uid_eq(current_fsuid(), inode->i_uid) && !capable(CAP_CHOWN)) + return -EPERM; + if (!in_group_p(inode->i_gid) && !capable(CAP_CHOWN)) + return -EPERM; + } + + /* + * If the caller is modifying any aspect of the attributes, they + * are not creating a snapshot. They need read permission on the + * file. + */ + if (!preserve) { + error = inode_permission(&nop_mnt_idmap, inode, MAY_READ); + if (error) + return error; + } + + inode_lock(inode); + error = dquot_initialize(dir); + if (!error) + error = ocfs2_reflink(old_dentry, dir, new_dentry, preserve); + inode_unlock(inode); + if (!error) + fsnotify_create(dir, new_dentry); + return error; +} +/* + * Most codes are copied from sys_linkat. + */ +int ocfs2_reflink_ioctl(struct inode *inode, + const char __user *oldname, + const char __user *newname, + bool preserve) +{ + struct dentry *new_dentry; + struct path old_path, new_path; + int error; + + if (!ocfs2_refcount_tree(OCFS2_SB(inode->i_sb))) + return -EOPNOTSUPP; + + error = user_path_at(AT_FDCWD, oldname, 0, &old_path); + if (error) { + mlog_errno(error); + return error; + } + + new_dentry = start_creating_user_path(AT_FDCWD, newname, &new_path, 0); + error = PTR_ERR(new_dentry); + if (IS_ERR(new_dentry)) { + mlog_errno(error); + goto out; + } + + error = -EXDEV; + if (old_path.mnt != new_path.mnt) { + mlog_errno(error); + goto out_dput; + } + + error = ocfs2_vfs_reflink(old_path.dentry, + d_inode(new_path.dentry), + new_dentry, preserve); +out_dput: + end_creating_path(&new_path, new_dentry); +out: + path_put(&old_path); + + return error; +} + +/* Update destination inode size, if necessary. */ +int ocfs2_reflink_update_dest(struct inode *dest, + struct buffer_head *d_bh, + loff_t newlen) +{ + handle_t *handle; + int ret; + + dest->i_blocks = ocfs2_inode_sector_count(dest); + + if (newlen <= i_size_read(dest)) + return 0; + + handle = ocfs2_start_trans(OCFS2_SB(dest->i_sb), + OCFS2_INODE_UPDATE_CREDITS); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + return ret; + } + + /* Extend i_size if needed. */ + spin_lock(&OCFS2_I(dest)->ip_lock); + if (newlen > i_size_read(dest)) + i_size_write(dest, newlen); + spin_unlock(&OCFS2_I(dest)->ip_lock); + inode_set_mtime_to_ts(dest, inode_set_ctime_current(dest)); + + ret = ocfs2_mark_inode_dirty(handle, dest, d_bh); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + +out_commit: + ocfs2_commit_trans(OCFS2_SB(dest->i_sb), handle); + return ret; +} + +/* Remap the range pos_in:len in s_inode to pos_out:len in t_inode. */ +static loff_t ocfs2_reflink_remap_extent(struct inode *s_inode, + struct buffer_head *s_bh, + loff_t pos_in, + struct inode *t_inode, + struct buffer_head *t_bh, + loff_t pos_out, + loff_t len, + struct ocfs2_cached_dealloc_ctxt *dealloc) +{ + struct ocfs2_extent_tree s_et; + struct ocfs2_extent_tree t_et; + struct ocfs2_dinode *dis; + struct buffer_head *ref_root_bh = NULL; + struct ocfs2_refcount_tree *ref_tree; + struct ocfs2_super *osb; + loff_t remapped_bytes = 0; + loff_t pstart, plen; + u32 p_cluster, num_clusters, slast, spos, tpos, remapped_clus = 0; + unsigned int ext_flags; + int ret = 0; + + osb = OCFS2_SB(s_inode->i_sb); + dis = (struct ocfs2_dinode *)s_bh->b_data; + ocfs2_init_dinode_extent_tree(&s_et, INODE_CACHE(s_inode), s_bh); + ocfs2_init_dinode_extent_tree(&t_et, INODE_CACHE(t_inode), t_bh); + + spos = ocfs2_bytes_to_clusters(s_inode->i_sb, pos_in); + tpos = ocfs2_bytes_to_clusters(t_inode->i_sb, pos_out); + slast = ocfs2_clusters_for_bytes(s_inode->i_sb, pos_in + len); + + while (spos < slast) { + if (fatal_signal_pending(current)) { + ret = -EINTR; + goto out; + } + + /* Look up the extent. */ + ret = ocfs2_get_clusters(s_inode, spos, &p_cluster, + &num_clusters, &ext_flags); + if (ret) { + mlog_errno(ret); + goto out; + } + + num_clusters = min_t(u32, num_clusters, slast - spos); + + /* Punch out the dest range. */ + pstart = ocfs2_clusters_to_bytes(t_inode->i_sb, tpos); + plen = ocfs2_clusters_to_bytes(t_inode->i_sb, num_clusters); + ret = ocfs2_remove_inode_range(t_inode, t_bh, pstart, plen); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (p_cluster == 0) + goto next_loop; + + /* Lock the refcount btree... */ + ret = ocfs2_lock_refcount_tree(osb, + le64_to_cpu(dis->i_refcount_loc), + 1, &ref_tree, &ref_root_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* Mark s_inode's extent as refcounted. */ + if (!(ext_flags & OCFS2_EXT_REFCOUNTED)) { + ret = ocfs2_add_refcount_flag(s_inode, &s_et, + &ref_tree->rf_ci, + ref_root_bh, spos, + p_cluster, num_clusters, + dealloc, NULL); + if (ret) { + mlog_errno(ret); + goto out_unlock_refcount; + } + } + + /* Map in the new extent. */ + ext_flags |= OCFS2_EXT_REFCOUNTED; + ret = ocfs2_add_refcounted_extent(t_inode, &t_et, + &ref_tree->rf_ci, + ref_root_bh, + tpos, p_cluster, + num_clusters, + ext_flags, + dealloc); + if (ret) { + mlog_errno(ret); + goto out_unlock_refcount; + } + + ocfs2_unlock_refcount_tree(osb, ref_tree, 1); + brelse(ref_root_bh); +next_loop: + spos += num_clusters; + tpos += num_clusters; + remapped_clus += num_clusters; + } + + goto out; +out_unlock_refcount: + ocfs2_unlock_refcount_tree(osb, ref_tree, 1); + brelse(ref_root_bh); +out: + remapped_bytes = ocfs2_clusters_to_bytes(t_inode->i_sb, remapped_clus); + remapped_bytes = min_t(loff_t, len, remapped_bytes); + + return remapped_bytes > 0 ? remapped_bytes : ret; +} + +/* Set up refcount tree and remap s_inode to t_inode. */ +loff_t ocfs2_reflink_remap_blocks(struct inode *s_inode, + struct buffer_head *s_bh, + loff_t pos_in, + struct inode *t_inode, + struct buffer_head *t_bh, + loff_t pos_out, + loff_t len) +{ + struct ocfs2_cached_dealloc_ctxt dealloc; + struct ocfs2_super *osb; + struct ocfs2_dinode *dis; + struct ocfs2_dinode *dit; + loff_t ret; + + osb = OCFS2_SB(s_inode->i_sb); + dis = (struct ocfs2_dinode *)s_bh->b_data; + dit = (struct ocfs2_dinode *)t_bh->b_data; + ocfs2_init_dealloc_ctxt(&dealloc); + + /* + * If we're reflinking the entire file and the source is inline + * data, just copy the contents. + */ + if (pos_in == pos_out && pos_in == 0 && len == i_size_read(s_inode) && + i_size_read(t_inode) <= len && + (OCFS2_I(s_inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL)) { + ret = ocfs2_duplicate_inline_data(s_inode, s_bh, t_inode, t_bh); + if (ret) + mlog_errno(ret); + goto out; + } + + /* + * If both inodes belong to two different refcount groups then + * forget it because we don't know how (or want) to go merging + * refcount trees. + */ + ret = -EOPNOTSUPP; + if (ocfs2_is_refcount_inode(s_inode) && + ocfs2_is_refcount_inode(t_inode) && + le64_to_cpu(dis->i_refcount_loc) != + le64_to_cpu(dit->i_refcount_loc)) + goto out; + + /* Neither inode has a refcount tree. Add one to s_inode. */ + if (!ocfs2_is_refcount_inode(s_inode) && + !ocfs2_is_refcount_inode(t_inode)) { + ret = ocfs2_create_refcount_tree(s_inode, s_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + /* Ensure that both inodes end up with the same refcount tree. */ + if (!ocfs2_is_refcount_inode(s_inode)) { + ret = ocfs2_set_refcount_tree(s_inode, s_bh, + le64_to_cpu(dit->i_refcount_loc)); + if (ret) { + mlog_errno(ret); + goto out; + } + } + if (!ocfs2_is_refcount_inode(t_inode)) { + ret = ocfs2_set_refcount_tree(t_inode, t_bh, + le64_to_cpu(dis->i_refcount_loc)); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + /* Turn off inline data in the dest file. */ + if (OCFS2_I(t_inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL) { + ret = ocfs2_convert_inline_data_to_extents(t_inode, t_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + /* Actually remap extents now. */ + ret = ocfs2_reflink_remap_extent(s_inode, s_bh, pos_in, t_inode, t_bh, + pos_out, len, &dealloc); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + +out: + if (ocfs2_dealloc_has_cluster(&dealloc)) + ocfs2_schedule_truncate_log_flush(osb, 1); + ocfs2_run_deallocs(osb, &dealloc); + + return ret; +} + +/* Lock an inode and grab a bh pointing to the inode. */ +int ocfs2_reflink_inodes_lock(struct inode *s_inode, + struct buffer_head **bh_s, + struct inode *t_inode, + struct buffer_head **bh_t) +{ + struct inode *inode1 = s_inode; + struct inode *inode2 = t_inode; + struct ocfs2_inode_info *oi1; + struct ocfs2_inode_info *oi2; + struct buffer_head *bh1 = NULL; + struct buffer_head *bh2 = NULL; + bool same_inode = (s_inode == t_inode); + bool need_swap = (inode1->i_ino > inode2->i_ino); + int status; + + /* First grab the VFS and rw locks. */ + lock_two_nondirectories(s_inode, t_inode); + if (need_swap) + swap(inode1, inode2); + + status = ocfs2_rw_lock(inode1, 1); + if (status) { + mlog_errno(status); + goto out_i1; + } + if (!same_inode) { + status = ocfs2_rw_lock(inode2, 1); + if (status) { + mlog_errno(status); + goto out_i2; + } + } + + /* Now go for the cluster locks */ + oi1 = OCFS2_I(inode1); + oi2 = OCFS2_I(inode2); + + trace_ocfs2_double_lock((unsigned long long)oi1->ip_blkno, + (unsigned long long)oi2->ip_blkno); + + /* We always want to lock the one with the lower lockid first. */ + if (oi1->ip_blkno > oi2->ip_blkno) + mlog_errno(-ENOLCK); + + /* lock id1 */ + status = ocfs2_inode_lock_nested(inode1, &bh1, 1, + OI_LS_REFLINK_TARGET); + if (status < 0) { + if (status != -ENOENT) + mlog_errno(status); + goto out_rw2; + } + + /* lock id2 */ + if (!same_inode) { + status = ocfs2_inode_lock_nested(inode2, &bh2, 1, + OI_LS_REFLINK_TARGET); + if (status < 0) { + if (status != -ENOENT) + mlog_errno(status); + goto out_cl1; + } + } else { + bh2 = bh1; + } + + /* + * If we swapped inode order above, we have to swap the buffer heads + * before passing them back to the caller. + */ + if (need_swap) + swap(bh1, bh2); + *bh_s = bh1; + *bh_t = bh2; + + trace_ocfs2_double_lock_end( + (unsigned long long)oi1->ip_blkno, + (unsigned long long)oi2->ip_blkno); + + return 0; + +out_cl1: + ocfs2_inode_unlock(inode1, 1); + brelse(bh1); +out_rw2: + ocfs2_rw_unlock(inode2, 1); +out_i2: + ocfs2_rw_unlock(inode1, 1); +out_i1: + unlock_two_nondirectories(s_inode, t_inode); + return status; +} + +/* Unlock both inodes and release buffers. */ +void ocfs2_reflink_inodes_unlock(struct inode *s_inode, + struct buffer_head *s_bh, + struct inode *t_inode, + struct buffer_head *t_bh) +{ + ocfs2_inode_unlock(s_inode, 1); + ocfs2_rw_unlock(s_inode, 1); + brelse(s_bh); + if (s_inode != t_inode) { + ocfs2_inode_unlock(t_inode, 1); + ocfs2_rw_unlock(t_inode, 1); + brelse(t_bh); + } + unlock_two_nondirectories(s_inode, t_inode); +} diff --git a/fs/ocfs2/refcounttree.h b/fs/ocfs2/refcounttree.h new file mode 100644 index 000000000..8197a94fe --- /dev/null +++ b/fs/ocfs2/refcounttree.h @@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * refcounttree.h + * + * Copyright (C) 2009 Oracle. All rights reserved. + */ +#ifndef OCFS2_REFCOUNTTREE_H +#define OCFS2_REFCOUNTTREE_H + +struct ocfs2_refcount_tree { + struct rb_node rf_node; + u64 rf_blkno; + u32 rf_generation; + struct kref rf_getcnt; + struct rw_semaphore rf_sem; + struct ocfs2_lock_res rf_lockres; + int rf_removed; + + /* the following 4 fields are used by caching_info. */ + spinlock_t rf_lock; + struct ocfs2_caching_info rf_ci; + struct mutex rf_io_mutex; + struct super_block *rf_sb; +}; + +void ocfs2_purge_refcount_trees(struct ocfs2_super *osb); +int ocfs2_lock_refcount_tree(struct ocfs2_super *osb, u64 ref_blkno, int rw, + struct ocfs2_refcount_tree **tree, + struct buffer_head **ref_bh); +void ocfs2_unlock_refcount_tree(struct ocfs2_super *osb, + struct ocfs2_refcount_tree *tree, + int rw); + +int ocfs2_decrease_refcount(struct inode *inode, + handle_t *handle, u32 cpos, u32 len, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_cached_dealloc_ctxt *dealloc, + int delete); +int ocfs2_prepare_refcount_change_for_del(struct inode *inode, + u64 refcount_loc, + u64 phys_blkno, + u32 clusters, + int *credits, + int *ref_blocks); +int ocfs2_refcount_cow(struct inode *inode, + struct buffer_head *di_bh, + u32 cpos, u32 write_len, u32 max_cpos); + +typedef int (ocfs2_post_refcount_func)(struct inode *inode, + handle_t *handle, + void *para); +/* + * Some refcount caller need to do more work after we modify the data b-tree + * during refcount operation(including CoW and add refcount flag), and make the + * transaction complete. So it must give us this structure so that we can do it + * within our transaction. + * + */ +struct ocfs2_post_refcount { + int credits; /* credits it need for journal. */ + ocfs2_post_refcount_func *func; /* real function. */ + void *para; +}; + +int ocfs2_refcounted_xattr_delete_need(struct inode *inode, + struct ocfs2_caching_info *ref_ci, + struct buffer_head *ref_root_bh, + struct ocfs2_xattr_value_root *xv, + int *meta_add, int *credits); +int ocfs2_refcount_cow_xattr(struct inode *inode, + struct ocfs2_dinode *di, + struct ocfs2_xattr_value_buf *vb, + struct ocfs2_refcount_tree *ref_tree, + struct buffer_head *ref_root_bh, + u32 cpos, u32 write_len, + struct ocfs2_post_refcount *post); +int ocfs2_duplicate_clusters_by_page(handle_t *handle, + struct inode *inode, + u32 cpos, u32 old_cluster, + u32 new_cluster, u32 new_len); +int ocfs2_duplicate_clusters_by_jbd(handle_t *handle, + struct inode *inode, + u32 cpos, u32 old_cluster, + u32 new_cluster, u32 new_len); +int ocfs2_cow_sync_writeback(struct super_block *sb, + struct inode *inode, + u32 cpos, u32 num_clusters); +int ocfs2_add_refcount_flag(struct inode *inode, + struct ocfs2_extent_tree *data_et, + struct ocfs2_caching_info *ref_ci, + struct buffer_head *ref_root_bh, + u32 cpos, u32 p_cluster, u32 num_clusters, + struct ocfs2_cached_dealloc_ctxt *dealloc, + struct ocfs2_post_refcount *post); +int ocfs2_remove_refcount_tree(struct inode *inode, struct buffer_head *di_bh); +int ocfs2_try_remove_refcount_tree(struct inode *inode, + struct buffer_head *di_bh); +int ocfs2_increase_refcount(handle_t *handle, + struct ocfs2_caching_info *ci, + struct buffer_head *ref_root_bh, + u64 cpos, u32 len, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_cached_dealloc_ctxt *dealloc); +int ocfs2_reflink_ioctl(struct inode *inode, + const char __user *oldname, + const char __user *newname, + bool preserve); +loff_t ocfs2_reflink_remap_blocks(struct inode *s_inode, + struct buffer_head *s_bh, + loff_t pos_in, + struct inode *t_inode, + struct buffer_head *t_bh, + loff_t pos_out, + loff_t len); +int ocfs2_reflink_inodes_lock(struct inode *s_inode, + struct buffer_head **bh1, + struct inode *t_inode, + struct buffer_head **bh2); +void ocfs2_reflink_inodes_unlock(struct inode *s_inode, + struct buffer_head *s_bh, + struct inode *t_inode, + struct buffer_head *t_bh); +int ocfs2_reflink_update_dest(struct inode *dest, + struct buffer_head *d_bh, + loff_t newlen); + +#endif /* OCFS2_REFCOUNTTREE_H */ diff --git a/fs/ocfs2/reservations.c b/fs/ocfs2/reservations.c new file mode 100644 index 000000000..1fe61974d --- /dev/null +++ b/fs/ocfs2/reservations.c @@ -0,0 +1,824 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * reservations.c + * + * Allocation reservations implementation + * + * Some code borrowed from fs/ext3/balloc.c and is: + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * The rest is copyright (C) 2010 Novell. All rights reserved. + */ + +#include +#include +#include +#include +#include + +#include + +#include "ocfs2.h" +#include "ocfs2_trace.h" + +#ifdef CONFIG_OCFS2_DEBUG_FS +#define OCFS2_CHECK_RESERVATIONS +#endif + +static DEFINE_SPINLOCK(resv_lock); + +int ocfs2_dir_resv_allowed(struct ocfs2_super *osb) +{ + return (osb->osb_resv_level && osb->osb_dir_resv_level); +} + +static unsigned int ocfs2_resv_window_bits(struct ocfs2_reservation_map *resmap, + struct ocfs2_alloc_reservation *resv) +{ + struct ocfs2_super *osb = resmap->m_osb; + unsigned int bits; + + if (!(resv->r_flags & OCFS2_RESV_FLAG_DIR)) { + /* 8, 16, 32, 64, 128, 256, 512, 1024 */ + bits = 4 << osb->osb_resv_level; + } else { + bits = 4 << osb->osb_dir_resv_level; + } + return bits; +} + +static inline unsigned int ocfs2_resv_end(struct ocfs2_alloc_reservation *resv) +{ + if (resv->r_len) + return resv->r_start + resv->r_len - 1; + return resv->r_start; +} + +static inline int ocfs2_resv_empty(struct ocfs2_alloc_reservation *resv) +{ + return !!(resv->r_len == 0); +} + +static inline int ocfs2_resmap_disabled(struct ocfs2_reservation_map *resmap) +{ + if (resmap->m_osb->osb_resv_level == 0) + return 1; + return 0; +} + +static void ocfs2_dump_resv(struct ocfs2_reservation_map *resmap) +{ + struct ocfs2_super *osb = resmap->m_osb; + struct rb_node *node; + struct ocfs2_alloc_reservation *resv; + int i = 0; + + mlog(ML_NOTICE, "Dumping resmap for device %s. Bitmap length: %u\n", + osb->dev_str, resmap->m_bitmap_len); + + node = rb_first(&resmap->m_reservations); + while (node) { + resv = rb_entry(node, struct ocfs2_alloc_reservation, r_node); + + mlog(ML_NOTICE, "start: %u\tend: %u\tlen: %u\tlast_start: %u" + "\tlast_len: %u\n", resv->r_start, + ocfs2_resv_end(resv), resv->r_len, resv->r_last_start, + resv->r_last_len); + + node = rb_next(node); + i++; + } + + mlog(ML_NOTICE, "%d reservations found. LRU follows\n", i); + + i = 0; + list_for_each_entry(resv, &resmap->m_lru, r_lru) { + mlog(ML_NOTICE, "LRU(%d) start: %u\tend: %u\tlen: %u\t" + "last_start: %u\tlast_len: %u\n", i, resv->r_start, + ocfs2_resv_end(resv), resv->r_len, resv->r_last_start, + resv->r_last_len); + + i++; + } +} + +#ifdef OCFS2_CHECK_RESERVATIONS +static int ocfs2_validate_resmap_bits(struct ocfs2_reservation_map *resmap, + int i, + struct ocfs2_alloc_reservation *resv) +{ + char *disk_bitmap = resmap->m_disk_bitmap; + unsigned int start = resv->r_start; + unsigned int end = ocfs2_resv_end(resv); + + while (start <= end) { + if (ocfs2_test_bit(start, disk_bitmap)) { + mlog(ML_ERROR, + "reservation %d covers an allocated area " + "starting at bit %u!\n", i, start); + return 1; + } + + start++; + } + return 0; +} + +static void ocfs2_check_resmap(struct ocfs2_reservation_map *resmap) +{ + unsigned int off = 0; + int i = 0; + struct rb_node *node; + struct ocfs2_alloc_reservation *resv; + + node = rb_first(&resmap->m_reservations); + while (node) { + resv = rb_entry(node, struct ocfs2_alloc_reservation, r_node); + + if (i > 0 && resv->r_start <= off) { + mlog(ML_ERROR, "reservation %d has bad start off!\n", + i); + goto bad; + } + + if (resv->r_len == 0) { + mlog(ML_ERROR, "reservation %d has no length!\n", + i); + goto bad; + } + + if (resv->r_start > ocfs2_resv_end(resv)) { + mlog(ML_ERROR, "reservation %d has invalid range!\n", + i); + goto bad; + } + + if (ocfs2_resv_end(resv) >= resmap->m_bitmap_len) { + mlog(ML_ERROR, "reservation %d extends past bitmap!\n", + i); + goto bad; + } + + if (ocfs2_validate_resmap_bits(resmap, i, resv)) + goto bad; + + off = ocfs2_resv_end(resv); + node = rb_next(node); + + i++; + } + return; + +bad: + ocfs2_dump_resv(resmap); + BUG(); +} +#else +static inline void ocfs2_check_resmap(struct ocfs2_reservation_map *resmap) +{ + +} +#endif + +void ocfs2_resv_init_once(struct ocfs2_alloc_reservation *resv) +{ + memset(resv, 0, sizeof(*resv)); + INIT_LIST_HEAD(&resv->r_lru); +} + +void ocfs2_resv_set_type(struct ocfs2_alloc_reservation *resv, + unsigned int flags) +{ + BUG_ON(flags & ~OCFS2_RESV_TYPES); + + resv->r_flags |= flags; +} + +void ocfs2_resmap_init(struct ocfs2_super *osb, + struct ocfs2_reservation_map *resmap) +{ + memset(resmap, 0, sizeof(*resmap)); + + resmap->m_osb = osb; + resmap->m_reservations = RB_ROOT; + /* m_bitmap_len is initialized to zero by the above memset. */ + INIT_LIST_HEAD(&resmap->m_lru); +} + +static void ocfs2_resv_mark_lru(struct ocfs2_reservation_map *resmap, + struct ocfs2_alloc_reservation *resv) +{ + assert_spin_locked(&resv_lock); + + if (!list_empty(&resv->r_lru)) + list_del_init(&resv->r_lru); + + list_add_tail(&resv->r_lru, &resmap->m_lru); +} + +static void __ocfs2_resv_trunc(struct ocfs2_alloc_reservation *resv) +{ + resv->r_len = 0; + resv->r_start = 0; +} + +static void ocfs2_resv_remove(struct ocfs2_reservation_map *resmap, + struct ocfs2_alloc_reservation *resv) +{ + if (resv->r_flags & OCFS2_RESV_FLAG_INUSE) { + list_del_init(&resv->r_lru); + rb_erase(&resv->r_node, &resmap->m_reservations); + resv->r_flags &= ~OCFS2_RESV_FLAG_INUSE; + } +} + +static void __ocfs2_resv_discard(struct ocfs2_reservation_map *resmap, + struct ocfs2_alloc_reservation *resv) +{ + assert_spin_locked(&resv_lock); + + __ocfs2_resv_trunc(resv); + /* + * last_len and last_start no longer make sense if + * we're changing the range of our allocations. + */ + resv->r_last_len = resv->r_last_start = 0; + + ocfs2_resv_remove(resmap, resv); +} + +/* does nothing if 'resv' is null */ +void ocfs2_resv_discard(struct ocfs2_reservation_map *resmap, + struct ocfs2_alloc_reservation *resv) +{ + if (resv) { + spin_lock(&resv_lock); + __ocfs2_resv_discard(resmap, resv); + spin_unlock(&resv_lock); + } +} + +static void ocfs2_resmap_clear_all_resv(struct ocfs2_reservation_map *resmap) +{ + struct rb_node *node; + struct ocfs2_alloc_reservation *resv; + + assert_spin_locked(&resv_lock); + + while ((node = rb_last(&resmap->m_reservations)) != NULL) { + resv = rb_entry(node, struct ocfs2_alloc_reservation, r_node); + + __ocfs2_resv_discard(resmap, resv); + } +} + +void ocfs2_resmap_restart(struct ocfs2_reservation_map *resmap, + unsigned int clen, char *disk_bitmap) +{ + if (ocfs2_resmap_disabled(resmap)) + return; + + spin_lock(&resv_lock); + + ocfs2_resmap_clear_all_resv(resmap); + resmap->m_bitmap_len = clen; + resmap->m_disk_bitmap = disk_bitmap; + + spin_unlock(&resv_lock); +} + +void ocfs2_resmap_uninit(struct ocfs2_reservation_map *resmap) +{ + /* Does nothing for now. Keep this around for API symmetry */ +} + +static void ocfs2_resv_insert(struct ocfs2_reservation_map *resmap, + struct ocfs2_alloc_reservation *new) +{ + struct rb_root *root = &resmap->m_reservations; + struct rb_node *parent = NULL; + struct rb_node **p = &root->rb_node; + struct ocfs2_alloc_reservation *tmp; + + assert_spin_locked(&resv_lock); + + trace_ocfs2_resv_insert(new->r_start, new->r_len); + + while (*p) { + parent = *p; + + tmp = rb_entry(parent, struct ocfs2_alloc_reservation, r_node); + + if (new->r_start < tmp->r_start) { + p = &(*p)->rb_left; + + /* + * This is a good place to check for + * overlapping reservations. + */ + BUG_ON(ocfs2_resv_end(new) >= tmp->r_start); + } else if (new->r_start > ocfs2_resv_end(tmp)) { + p = &(*p)->rb_right; + } else { + /* This should never happen! */ + mlog(ML_ERROR, "Duplicate reservation window!\n"); + BUG(); + } + } + + rb_link_node(&new->r_node, parent, p); + rb_insert_color(&new->r_node, root); + new->r_flags |= OCFS2_RESV_FLAG_INUSE; + + ocfs2_resv_mark_lru(resmap, new); + + ocfs2_check_resmap(resmap); +} + +/** + * ocfs2_find_resv_lhs() - find the window which contains goal + * @resmap: reservation map to search + * @goal: which bit to search for + * + * If a window containing that goal is not found, we return the window + * which comes before goal. Returns NULL on empty rbtree or no window + * before goal. + */ +static struct ocfs2_alloc_reservation * +ocfs2_find_resv_lhs(struct ocfs2_reservation_map *resmap, unsigned int goal) +{ + struct ocfs2_alloc_reservation *resv = NULL; + struct ocfs2_alloc_reservation *prev_resv = NULL; + struct rb_node *node = resmap->m_reservations.rb_node; + + assert_spin_locked(&resv_lock); + + if (!node) + return NULL; + + node = rb_first(&resmap->m_reservations); + while (node) { + resv = rb_entry(node, struct ocfs2_alloc_reservation, r_node); + + if (resv->r_start <= goal && ocfs2_resv_end(resv) >= goal) + break; + + /* Check if we overshot the reservation just before goal? */ + if (resv->r_start > goal) { + resv = prev_resv; + break; + } + + prev_resv = resv; + node = rb_next(node); + } + + return resv; +} + +/* + * We are given a range within the bitmap, which corresponds to a gap + * inside the reservations tree (search_start, search_len). The range + * can be anything from the whole bitmap, to a gap between + * reservations. + * + * The start value of *rstart is insignificant. + * + * This function searches the bitmap range starting at search_start + * with length search_len for a set of contiguous free bits. We try + * to find up to 'wanted' bits, but can sometimes return less. + * + * Returns the length of allocation, 0 if no free bits are found. + * + * *cstart and *clen will also be populated with the result. + */ +static int ocfs2_resmap_find_free_bits(struct ocfs2_reservation_map *resmap, + unsigned int wanted, + unsigned int search_start, + unsigned int search_len, + unsigned int *rstart, + unsigned int *rlen) +{ + void *bitmap = resmap->m_disk_bitmap; + unsigned int best_start, best_len = 0; + int offset, start, found; + + trace_ocfs2_resmap_find_free_bits_begin(search_start, search_len, + wanted, resmap->m_bitmap_len); + + found = best_start = best_len = 0; + + start = search_start; + while ((offset = ocfs2_find_next_zero_bit(bitmap, resmap->m_bitmap_len, + start)) < resmap->m_bitmap_len) { + /* Search reached end of the region */ + if (offset >= (search_start + search_len)) + break; + + if (offset == start) { + /* we found a zero */ + found++; + /* move start to the next bit to test */ + start++; + } else { + /* got a zero after some ones */ + found = 1; + start = offset + 1; + } + if (found > best_len) { + best_len = found; + best_start = start - found; + } + + if (found >= wanted) + break; + } + + if (best_len == 0) + return 0; + + if (best_len >= wanted) + best_len = wanted; + + *rlen = best_len; + *rstart = best_start; + + trace_ocfs2_resmap_find_free_bits_end(best_start, best_len); + + return *rlen; +} + +static void __ocfs2_resv_find_window(struct ocfs2_reservation_map *resmap, + struct ocfs2_alloc_reservation *resv, + unsigned int goal, unsigned int wanted) +{ + struct rb_root *root = &resmap->m_reservations; + unsigned int gap_start, gap_end, gap_len; + struct ocfs2_alloc_reservation *prev_resv, *next_resv; + struct rb_node *prev, *next; + unsigned int cstart, clen; + unsigned int best_start = 0, best_len = 0; + + /* + * Nasty cases to consider: + * + * - rbtree is empty + * - our window should be first in all reservations + * - our window should be last in all reservations + * - need to make sure we don't go past end of bitmap + */ + trace_ocfs2_resv_find_window_begin(resv->r_start, ocfs2_resv_end(resv), + goal, wanted, RB_EMPTY_ROOT(root)); + + assert_spin_locked(&resv_lock); + + if (RB_EMPTY_ROOT(root)) { + /* + * Easiest case - empty tree. We can just take + * whatever window of free bits we want. + */ + clen = ocfs2_resmap_find_free_bits(resmap, wanted, goal, + resmap->m_bitmap_len - goal, + &cstart, &clen); + + /* + * This should never happen - the local alloc window + * will always have free bits when we're called. + */ + BUG_ON(goal == 0 && clen == 0); + + if (clen == 0) + return; + + resv->r_start = cstart; + resv->r_len = clen; + + ocfs2_resv_insert(resmap, resv); + return; + } + + prev_resv = ocfs2_find_resv_lhs(resmap, goal); + + if (prev_resv == NULL) { + /* + * A NULL here means that the search code couldn't + * find a window that starts before goal. + * + * However, we can take the first window after goal, + * which is also by definition, the leftmost window in + * the entire tree. If we can find free bits in the + * gap between goal and the LHS window, then the + * reservation can safely be placed there. + * + * Otherwise we fall back to a linear search, checking + * the gaps in between windows for a place to + * allocate. + */ + + next = rb_first(root); + next_resv = rb_entry(next, struct ocfs2_alloc_reservation, + r_node); + + /* + * The search should never return such a window. (see + * comment above + */ + if (next_resv->r_start <= goal) { + mlog(ML_ERROR, "goal: %u next_resv: start %u len %u\n", + goal, next_resv->r_start, next_resv->r_len); + ocfs2_dump_resv(resmap); + BUG(); + } + + clen = ocfs2_resmap_find_free_bits(resmap, wanted, goal, + next_resv->r_start - goal, + &cstart, &clen); + if (clen) { + best_len = clen; + best_start = cstart; + if (best_len == wanted) + goto out_insert; + } + + prev_resv = next_resv; + next_resv = NULL; + } + + trace_ocfs2_resv_find_window_prev(prev_resv->r_start, + ocfs2_resv_end(prev_resv)); + + prev = &prev_resv->r_node; + + /* Now we do a linear search for a window, starting at 'prev_rsv' */ + while (1) { + next = rb_next(prev); + if (next) { + next_resv = rb_entry(next, + struct ocfs2_alloc_reservation, + r_node); + + gap_start = ocfs2_resv_end(prev_resv) + 1; + gap_end = next_resv->r_start - 1; + gap_len = gap_end - gap_start + 1; + } else { + /* + * We're at the rightmost edge of the + * tree. See if a reservation between this + * window and the end of the bitmap will work. + */ + gap_start = ocfs2_resv_end(prev_resv) + 1; + gap_len = resmap->m_bitmap_len - gap_start; + gap_end = resmap->m_bitmap_len - 1; + } + + trace_ocfs2_resv_find_window_next(next ? next_resv->r_start: -1, + next ? ocfs2_resv_end(next_resv) : -1); + /* + * No need to check this gap if we have already found + * a larger region of free bits. + */ + if (gap_len <= best_len) + goto next_resv; + + clen = ocfs2_resmap_find_free_bits(resmap, wanted, gap_start, + gap_len, &cstart, &clen); + if (clen == wanted) { + best_len = clen; + best_start = cstart; + goto out_insert; + } else if (clen > best_len) { + best_len = clen; + best_start = cstart; + } + +next_resv: + if (!next) + break; + + prev = next; + prev_resv = rb_entry(prev, struct ocfs2_alloc_reservation, + r_node); + } + +out_insert: + if (best_len) { + resv->r_start = best_start; + resv->r_len = best_len; + ocfs2_resv_insert(resmap, resv); + } +} + +static void ocfs2_cannibalize_resv(struct ocfs2_reservation_map *resmap, + struct ocfs2_alloc_reservation *resv, + unsigned int wanted) +{ + struct ocfs2_alloc_reservation *lru_resv; + int tmpwindow = !!(resv->r_flags & OCFS2_RESV_FLAG_TMP); + unsigned int min_bits; + + if (!tmpwindow) + min_bits = ocfs2_resv_window_bits(resmap, resv) >> 1; + else + min_bits = wanted; /* We at know the temp window will use all + * of these bits */ + + /* + * Take the first reservation off the LRU as our 'target'. We + * don't try to be smart about it. There might be a case for + * searching based on size but I don't have enough data to be + * sure. --Mark (3/16/2010) + */ + lru_resv = list_first_entry(&resmap->m_lru, + struct ocfs2_alloc_reservation, r_lru); + + trace_ocfs2_cannibalize_resv_begin(lru_resv->r_start, + lru_resv->r_len, + ocfs2_resv_end(lru_resv)); + + /* + * Cannibalize (some or all) of the target reservation and + * feed it to the current window. + */ + if (lru_resv->r_len <= min_bits) { + /* + * Discard completely if size is less than or equal to a + * reasonable threshold - 50% of window bits for non temporary + * windows. + */ + resv->r_start = lru_resv->r_start; + resv->r_len = lru_resv->r_len; + + __ocfs2_resv_discard(resmap, lru_resv); + } else { + unsigned int shrink; + if (tmpwindow) + shrink = min_bits; + else + shrink = lru_resv->r_len / 2; + + lru_resv->r_len -= shrink; + + resv->r_start = ocfs2_resv_end(lru_resv) + 1; + resv->r_len = shrink; + } + + trace_ocfs2_cannibalize_resv_end(resv->r_start, ocfs2_resv_end(resv), + resv->r_len, resv->r_last_start, + resv->r_last_len); + + ocfs2_resv_insert(resmap, resv); +} + +static void ocfs2_resv_find_window(struct ocfs2_reservation_map *resmap, + struct ocfs2_alloc_reservation *resv, + unsigned int wanted) +{ + unsigned int goal = 0; + + BUG_ON(!ocfs2_resv_empty(resv)); + + /* + * Begin by trying to get a window as close to the previous + * one as possible. Using the most recent allocation as a + * start goal makes sense. + */ + if (resv->r_last_len) { + goal = resv->r_last_start + resv->r_last_len; + if (goal >= resmap->m_bitmap_len) + goal = 0; + } + + __ocfs2_resv_find_window(resmap, resv, goal, wanted); + + /* Search from last alloc didn't work, try once more from beginning. */ + if (ocfs2_resv_empty(resv) && goal != 0) + __ocfs2_resv_find_window(resmap, resv, 0, wanted); + + if (ocfs2_resv_empty(resv)) { + /* + * Still empty? Pull oldest one off the LRU, remove it from + * tree, put this one in it's place. + */ + ocfs2_cannibalize_resv(resmap, resv, wanted); + } + + BUG_ON(ocfs2_resv_empty(resv)); +} + +int ocfs2_resmap_resv_bits(struct ocfs2_reservation_map *resmap, + struct ocfs2_alloc_reservation *resv, + int *cstart, int *clen) +{ + if (resv == NULL || ocfs2_resmap_disabled(resmap)) + return -ENOSPC; + + spin_lock(&resv_lock); + + if (ocfs2_resv_empty(resv)) { + /* + * We don't want to over-allocate for temporary + * windows. Otherwise, we run the risk of fragmenting the + * allocation space. + */ + unsigned int wanted = ocfs2_resv_window_bits(resmap, resv); + + if ((resv->r_flags & OCFS2_RESV_FLAG_TMP) || wanted < *clen) + wanted = *clen; + + /* + * Try to get a window here. If it works, we must fall + * through and test the bitmap . This avoids some + * ping-ponging of windows due to non-reserved space + * being allocation before we initialize a window for + * that inode. + */ + ocfs2_resv_find_window(resmap, resv, wanted); + trace_ocfs2_resmap_resv_bits(resv->r_start, resv->r_len); + } + + BUG_ON(ocfs2_resv_empty(resv)); + + *cstart = resv->r_start; + *clen = resv->r_len; + + spin_unlock(&resv_lock); + return 0; +} + +static void + ocfs2_adjust_resv_from_alloc(struct ocfs2_reservation_map *resmap, + struct ocfs2_alloc_reservation *resv, + unsigned int start, unsigned int end) +{ + unsigned int rhs = 0; + unsigned int old_end = ocfs2_resv_end(resv); + + BUG_ON(start != resv->r_start || old_end < end); + + /* + * Completely used? We can remove it then. + */ + if (old_end == end) { + __ocfs2_resv_discard(resmap, resv); + return; + } + + rhs = old_end - end; + + /* + * This should have been trapped above. + */ + BUG_ON(rhs == 0); + + resv->r_start = end + 1; + resv->r_len = old_end - resv->r_start + 1; +} + +void ocfs2_resmap_claimed_bits(struct ocfs2_reservation_map *resmap, + struct ocfs2_alloc_reservation *resv, + u32 cstart, u32 clen) +{ + unsigned int cend = cstart + clen - 1; + + if (resmap == NULL || ocfs2_resmap_disabled(resmap)) + return; + + if (resv == NULL) + return; + + BUG_ON(cstart != resv->r_start); + + spin_lock(&resv_lock); + + trace_ocfs2_resmap_claimed_bits_begin(cstart, cend, clen, resv->r_start, + ocfs2_resv_end(resv), resv->r_len, + resv->r_last_start, + resv->r_last_len); + + BUG_ON(cstart < resv->r_start); + BUG_ON(cstart > ocfs2_resv_end(resv)); + BUG_ON(cend > ocfs2_resv_end(resv)); + + ocfs2_adjust_resv_from_alloc(resmap, resv, cstart, cend); + resv->r_last_start = cstart; + resv->r_last_len = clen; + + /* + * May have been discarded above from + * ocfs2_adjust_resv_from_alloc(). + */ + if (!ocfs2_resv_empty(resv)) + ocfs2_resv_mark_lru(resmap, resv); + + trace_ocfs2_resmap_claimed_bits_end(resv->r_start, ocfs2_resv_end(resv), + resv->r_len, resv->r_last_start, + resv->r_last_len); + + ocfs2_check_resmap(resmap); + + spin_unlock(&resv_lock); +} diff --git a/fs/ocfs2/reservations.h b/fs/ocfs2/reservations.h new file mode 100644 index 000000000..4fce17180 --- /dev/null +++ b/fs/ocfs2/reservations.h @@ -0,0 +1,144 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * reservations.h + * + * Allocation reservations function prototypes and structures. + * + * Copyright (C) 2010 Novell. All rights reserved. + */ + +#ifndef OCFS2_RESERVATIONS_H +#define OCFS2_RESERVATIONS_H + +#include + +#define OCFS2_DEFAULT_RESV_LEVEL 2 +#define OCFS2_MAX_RESV_LEVEL 9 +#define OCFS2_MIN_RESV_LEVEL 0 + +struct ocfs2_alloc_reservation { + struct rb_node r_node; + + unsigned int r_start; /* Beginning of current window */ + unsigned int r_len; /* Length of the window */ + + unsigned int r_last_len; /* Length of most recent alloc */ + unsigned int r_last_start; /* Start of most recent alloc */ + struct list_head r_lru; /* LRU list head */ + + unsigned int r_flags; +}; + +#define OCFS2_RESV_FLAG_INUSE 0x01 /* Set when r_node is part of a btree */ +#define OCFS2_RESV_FLAG_TMP 0x02 /* Temporary reservation, will be + * destroyed immediately after use */ +#define OCFS2_RESV_FLAG_DIR 0x04 /* Reservation is for an unindexed + * directory btree */ + +struct ocfs2_reservation_map { + struct rb_root m_reservations; + char *m_disk_bitmap; + + struct ocfs2_super *m_osb; + + /* The following are not initialized to meaningful values until a disk + * bitmap is provided. */ + u32 m_bitmap_len; /* Number of valid + * bits available */ + + struct list_head m_lru; /* LRU of reservations + * structures. */ + +}; + +void ocfs2_resv_init_once(struct ocfs2_alloc_reservation *resv); + +#define OCFS2_RESV_TYPES (OCFS2_RESV_FLAG_TMP|OCFS2_RESV_FLAG_DIR) +void ocfs2_resv_set_type(struct ocfs2_alloc_reservation *resv, + unsigned int flags); + +int ocfs2_dir_resv_allowed(struct ocfs2_super *osb); + +/** + * ocfs2_resv_discard() - truncate a reservation + * @resmap: + * @resv: the reservation to truncate. + * + * After this function is called, the reservation will be empty, and + * unlinked from the rbtree. + */ +void ocfs2_resv_discard(struct ocfs2_reservation_map *resmap, + struct ocfs2_alloc_reservation *resv); + + +/** + * ocfs2_resmap_init() - Initialize fields of a reservations bitmap + * @osb: struct ocfs2_super to be saved in resmap + * @resmap: struct ocfs2_reservation_map to initialize + */ +void ocfs2_resmap_init(struct ocfs2_super *osb, + struct ocfs2_reservation_map *resmap); + +/** + * ocfs2_resmap_restart() - "restart" a reservation bitmap + * @resmap: reservations bitmap + * @clen: Number of valid bits in the bitmap + * @disk_bitmap: the disk bitmap this resmap should refer to. + * + * Re-initialize the parameters of a reservation bitmap. This is + * useful for local alloc window slides. + * + * This function will call ocfs2_trunc_resv against all existing + * reservations. A future version will recalculate existing + * reservations based on the new bitmap. + */ +void ocfs2_resmap_restart(struct ocfs2_reservation_map *resmap, + unsigned int clen, char *disk_bitmap); + +/** + * ocfs2_resmap_uninit() - uninitialize a reservation bitmap structure + * @resmap: the struct ocfs2_reservation_map to uninitialize + */ +void ocfs2_resmap_uninit(struct ocfs2_reservation_map *resmap); + +/** + * ocfs2_resmap_resv_bits() - Return still-valid reservation bits + * @resmap: reservations bitmap + * @resv: reservation to base search from + * @cstart: start of proposed allocation + * @clen: length (in clusters) of proposed allocation + * + * Using the reservation data from resv, this function will compare + * resmap and resmap->m_disk_bitmap to determine what part (if any) of + * the reservation window is still clear to use. If resv is empty, + * this function will try to allocate a window for it. + * + * On success, zero is returned and the valid allocation area is set in cstart + * and clen. + * + * Returns -ENOSPC if reservations are disabled. + */ +int ocfs2_resmap_resv_bits(struct ocfs2_reservation_map *resmap, + struct ocfs2_alloc_reservation *resv, + int *cstart, int *clen); + +/** + * ocfs2_resmap_claimed_bits() - Tell the reservation code that bits were used. + * @resmap: reservations bitmap + * @resv: optional reservation to recalculate based on new bitmap + * @cstart: start of allocation in clusters + * @clen: end of allocation in clusters. + * + * Tell the reservation code that bits were used to fulfill allocation in + * resmap. The bits don't have to have been part of any existing + * reservation. But we must always call this function when bits are claimed. + * Internally, the reservations code will use this information to mark the + * reservations bitmap. If resv is passed, it's next allocation window will be + * calculated. It also expects that 'cstart' is the same as we passed back + * from ocfs2_resmap_resv_bits(). + */ +void ocfs2_resmap_claimed_bits(struct ocfs2_reservation_map *resmap, + struct ocfs2_alloc_reservation *resv, + u32 cstart, u32 clen); + +#endif /* OCFS2_RESERVATIONS_H */ diff --git a/fs/ocfs2/resize.c b/fs/ocfs2/resize.c new file mode 100644 index 000000000..6375d5035 --- /dev/null +++ b/fs/ocfs2/resize.c @@ -0,0 +1,598 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * resize.c + * + * volume resize. + * Inspired by ext3/resize.c. + * + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#include +#include + +#include + +#include "ocfs2.h" + +#include "alloc.h" +#include "dlmglue.h" +#include "inode.h" +#include "journal.h" +#include "super.h" +#include "sysfile.h" +#include "uptodate.h" +#include "ocfs2_trace.h" + +#include "buffer_head_io.h" +#include "suballoc.h" +#include "resize.h" + +/* + * Check whether there are new backup superblocks exist + * in the last group. If there are some, mark them or clear + * them in the bitmap. + * + * Return how many backups we find in the last group. + */ +static u16 ocfs2_calc_new_backup_super(struct inode *inode, + struct ocfs2_group_desc *gd, + u16 cl_cpg, + u16 old_bg_clusters, + int set) +{ + int i; + u16 backups = 0; + u32 cluster, lgd_cluster; + u64 blkno, gd_blkno, lgd_blkno = le64_to_cpu(gd->bg_blkno); + + for (i = 0; i < OCFS2_MAX_BACKUP_SUPERBLOCKS; i++) { + blkno = ocfs2_backup_super_blkno(inode->i_sb, i); + cluster = ocfs2_blocks_to_clusters(inode->i_sb, blkno); + + gd_blkno = ocfs2_which_cluster_group(inode, cluster); + if (gd_blkno < lgd_blkno) + continue; + else if (gd_blkno > lgd_blkno) + break; + + /* check if already done backup super */ + lgd_cluster = ocfs2_blocks_to_clusters(inode->i_sb, lgd_blkno); + lgd_cluster += old_bg_clusters; + if (lgd_cluster >= cluster) + continue; + + if (set) + ocfs2_set_bit(cluster % cl_cpg, + (unsigned long *)gd->bg_bitmap); + else + ocfs2_clear_bit(cluster % cl_cpg, + (unsigned long *)gd->bg_bitmap); + backups++; + } + + return backups; +} + +static int ocfs2_update_last_group_and_inode(handle_t *handle, + struct inode *bm_inode, + struct buffer_head *bm_bh, + struct buffer_head *group_bh, + u32 first_new_cluster, + int new_clusters) +{ + int ret = 0; + struct ocfs2_super *osb = OCFS2_SB(bm_inode->i_sb); + struct ocfs2_dinode *fe = (struct ocfs2_dinode *) bm_bh->b_data; + struct ocfs2_chain_list *cl = &fe->id2.i_chain; + struct ocfs2_chain_rec *cr; + struct ocfs2_group_desc *group; + u16 chain, num_bits, backups = 0; + u16 cl_bpc = le16_to_cpu(cl->cl_bpc); + u16 cl_cpg = le16_to_cpu(cl->cl_cpg); + u16 old_bg_clusters; + u16 contig_bits; + __le16 old_bg_contig_free_bits; + + trace_ocfs2_update_last_group_and_inode(new_clusters, + first_new_cluster); + + ret = ocfs2_journal_access_gd(handle, INODE_CACHE(bm_inode), + group_bh, OCFS2_JOURNAL_ACCESS_WRITE); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + group = (struct ocfs2_group_desc *)group_bh->b_data; + + old_bg_clusters = le16_to_cpu(group->bg_bits) / cl_bpc; + /* update the group first. */ + num_bits = new_clusters * cl_bpc; + le16_add_cpu(&group->bg_bits, num_bits); + le16_add_cpu(&group->bg_free_bits_count, num_bits); + + /* + * check whether there are some new backup superblocks exist in + * this group and update the group bitmap accordingly. + */ + if (OCFS2_HAS_COMPAT_FEATURE(osb->sb, + OCFS2_FEATURE_COMPAT_BACKUP_SB)) { + backups = ocfs2_calc_new_backup_super(bm_inode, + group, + cl_cpg, old_bg_clusters, 1); + le16_add_cpu(&group->bg_free_bits_count, -1 * backups); + } + + contig_bits = ocfs2_find_max_contig_free_bits(group->bg_bitmap, + le16_to_cpu(group->bg_bits), 0); + old_bg_contig_free_bits = group->bg_contig_free_bits; + group->bg_contig_free_bits = cpu_to_le16(contig_bits); + + ocfs2_journal_dirty(handle, group_bh); + + /* update the inode accordingly. */ + ret = ocfs2_journal_access_di(handle, INODE_CACHE(bm_inode), bm_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret < 0) { + mlog_errno(ret); + goto out_rollback; + } + + chain = le16_to_cpu(group->bg_chain); + cr = (&cl->cl_recs[chain]); + le32_add_cpu(&cr->c_total, num_bits); + le32_add_cpu(&cr->c_free, num_bits); + le32_add_cpu(&fe->id1.bitmap1.i_total, num_bits); + le32_add_cpu(&fe->i_clusters, new_clusters); + + if (backups) { + le32_add_cpu(&cr->c_free, -1 * backups); + le32_add_cpu(&fe->id1.bitmap1.i_used, backups); + } + + spin_lock(&OCFS2_I(bm_inode)->ip_lock); + OCFS2_I(bm_inode)->ip_clusters = le32_to_cpu(fe->i_clusters); + le64_add_cpu(&fe->i_size, (u64)new_clusters << osb->s_clustersize_bits); + spin_unlock(&OCFS2_I(bm_inode)->ip_lock); + i_size_write(bm_inode, le64_to_cpu(fe->i_size)); + + ocfs2_journal_dirty(handle, bm_bh); + +out_rollback: + if (ret < 0) { + ocfs2_calc_new_backup_super(bm_inode, + group, + cl_cpg, old_bg_clusters, 0); + le16_add_cpu(&group->bg_free_bits_count, backups); + le16_add_cpu(&group->bg_bits, -1 * num_bits); + le16_add_cpu(&group->bg_free_bits_count, -1 * num_bits); + group->bg_contig_free_bits = old_bg_contig_free_bits; + } +out: + if (ret) + mlog_errno(ret); + return ret; +} + +static int update_backups(struct inode * inode, u32 clusters, char *data) +{ + int i, ret = 0; + u32 cluster; + u64 blkno; + struct buffer_head *backup = NULL; + struct ocfs2_dinode *backup_di = NULL; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + /* calculate the real backups we need to update. */ + for (i = 0; i < OCFS2_MAX_BACKUP_SUPERBLOCKS; i++) { + blkno = ocfs2_backup_super_blkno(inode->i_sb, i); + cluster = ocfs2_blocks_to_clusters(inode->i_sb, blkno); + if (cluster >= clusters) + break; + + ret = ocfs2_read_blocks_sync(osb, blkno, 1, &backup); + if (ret < 0) { + mlog_errno(ret); + break; + } + + memcpy(backup->b_data, data, inode->i_sb->s_blocksize); + + backup_di = (struct ocfs2_dinode *)backup->b_data; + backup_di->i_blkno = cpu_to_le64(blkno); + + ret = ocfs2_write_super_or_backup(osb, backup); + brelse(backup); + backup = NULL; + if (ret < 0) { + mlog_errno(ret); + break; + } + } + + return ret; +} + +static void ocfs2_update_super_and_backups(struct inode *inode, + int new_clusters) +{ + int ret; + u32 clusters = 0; + struct buffer_head *super_bh = NULL; + struct ocfs2_dinode *super_di = NULL; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + /* + * update the superblock last. + * It doesn't matter if the write failed. + */ + ret = ocfs2_read_blocks_sync(osb, OCFS2_SUPER_BLOCK_BLKNO, 1, + &super_bh); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + super_di = (struct ocfs2_dinode *)super_bh->b_data; + le32_add_cpu(&super_di->i_clusters, new_clusters); + clusters = le32_to_cpu(super_di->i_clusters); + + ret = ocfs2_write_super_or_backup(osb, super_bh); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + if (OCFS2_HAS_COMPAT_FEATURE(osb->sb, OCFS2_FEATURE_COMPAT_BACKUP_SB)) + ret = update_backups(inode, clusters, super_bh->b_data); + +out: + brelse(super_bh); + if (ret) + printk(KERN_WARNING "ocfs2: Failed to update super blocks on %s" + " during fs resize. This condition is not fatal," + " but fsck.ocfs2 should be run to fix it\n", + osb->dev_str); + return; +} + +/* + * Extend the filesystem to the new number of clusters specified. This entry + * point is only used to extend the current filesystem to the end of the last + * existing group. + */ +int ocfs2_group_extend(struct inode * inode, int new_clusters) +{ + int ret; + handle_t *handle; + struct buffer_head *main_bm_bh = NULL; + struct buffer_head *group_bh = NULL; + struct inode *main_bm_inode = NULL; + struct ocfs2_dinode *fe = NULL; + struct ocfs2_group_desc *group = NULL; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + u16 cl_bpc; + u32 first_new_cluster; + u64 lgd_blkno; + + if (unlikely(ocfs2_emergency_state(osb))) + return -EROFS; + + if (new_clusters < 0) + return -EINVAL; + else if (new_clusters == 0) + return 0; + + main_bm_inode = ocfs2_get_system_file_inode(osb, + GLOBAL_BITMAP_SYSTEM_INODE, + OCFS2_INVALID_SLOT); + if (!main_bm_inode) { + ret = -EINVAL; + mlog_errno(ret); + goto out; + } + + inode_lock(main_bm_inode); + + ret = ocfs2_inode_lock(main_bm_inode, &main_bm_bh, 1); + if (ret < 0) { + mlog_errno(ret); + goto out_mutex; + } + + fe = (struct ocfs2_dinode *)main_bm_bh->b_data; + + /* JBD-managed buffers can bypass validation, so treat this as corruption. */ + if (!OCFS2_IS_VALID_DINODE(fe)) { + ret = ocfs2_error(main_bm_inode->i_sb, + "Invalid dinode #%llu\n", + (unsigned long long)OCFS2_I(main_bm_inode)->ip_blkno); + goto out_unlock; + } + + if (le16_to_cpu(fe->id2.i_chain.cl_cpg) != + ocfs2_group_bitmap_size(osb->sb, 0, + osb->s_feature_incompat) * 8) { + mlog(ML_ERROR, "The disk is too old and small. " + "Force to do offline resize."); + ret = -EINVAL; + goto out_unlock; + } + + first_new_cluster = le32_to_cpu(fe->i_clusters); + lgd_blkno = ocfs2_which_cluster_group(main_bm_inode, + first_new_cluster - 1); + + ret = ocfs2_read_group_descriptor(main_bm_inode, fe, lgd_blkno, + &group_bh); + if (ret < 0) { + mlog_errno(ret); + goto out_unlock; + } + group = (struct ocfs2_group_desc *)group_bh->b_data; + + cl_bpc = le16_to_cpu(fe->id2.i_chain.cl_bpc); + if (le16_to_cpu(group->bg_bits) / cl_bpc + new_clusters > + le16_to_cpu(fe->id2.i_chain.cl_cpg)) { + ret = -EINVAL; + goto out_unlock; + } + + + trace_ocfs2_group_extend( + (unsigned long long)le64_to_cpu(group->bg_blkno), new_clusters); + + handle = ocfs2_start_trans(osb, OCFS2_GROUP_EXTEND_CREDITS); + if (IS_ERR(handle)) { + mlog_errno(PTR_ERR(handle)); + ret = -EINVAL; + goto out_unlock; + } + + /* update the last group descriptor and inode. */ + ret = ocfs2_update_last_group_and_inode(handle, main_bm_inode, + main_bm_bh, group_bh, + first_new_cluster, + new_clusters); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + ocfs2_update_super_and_backups(main_bm_inode, new_clusters); + +out_commit: + ocfs2_commit_trans(osb, handle); +out_unlock: + brelse(group_bh); + brelse(main_bm_bh); + + ocfs2_inode_unlock(main_bm_inode, 1); + +out_mutex: + inode_unlock(main_bm_inode); + iput(main_bm_inode); + +out: + return ret; +} + +static int ocfs2_check_new_group(struct inode *inode, + struct ocfs2_dinode *di, + struct ocfs2_new_group_input *input, + struct buffer_head *group_bh) +{ + int ret; + struct ocfs2_group_desc *gd = + (struct ocfs2_group_desc *)group_bh->b_data; + u16 cl_bpc = le16_to_cpu(di->id2.i_chain.cl_bpc); + + ret = ocfs2_check_group_descriptor(inode->i_sb, di, group_bh); + if (ret) + goto out; + + ret = -EINVAL; + if (le16_to_cpu(gd->bg_chain) != input->chain) + mlog(ML_ERROR, "Group descriptor # %llu has bad chain %u " + "while input has %u set.\n", + (unsigned long long)le64_to_cpu(gd->bg_blkno), + le16_to_cpu(gd->bg_chain), input->chain); + else if (le16_to_cpu(gd->bg_bits) != input->clusters * cl_bpc) + mlog(ML_ERROR, "Group descriptor # %llu has bit count %u but " + "input has %u clusters set\n", + (unsigned long long)le64_to_cpu(gd->bg_blkno), + le16_to_cpu(gd->bg_bits), input->clusters); + else if (le16_to_cpu(gd->bg_free_bits_count) != input->frees * cl_bpc) + mlog(ML_ERROR, "Group descriptor # %llu has free bit count %u " + "but it should have %u set\n", + (unsigned long long)le64_to_cpu(gd->bg_blkno), + le16_to_cpu(gd->bg_bits), + input->frees * cl_bpc); + else + ret = 0; + +out: + return ret; +} + +static int ocfs2_verify_group_and_input(struct inode *inode, + struct ocfs2_dinode *di, + struct ocfs2_new_group_input *input, + struct buffer_head *group_bh) +{ + u16 cl_count = le16_to_cpu(di->id2.i_chain.cl_count); + u16 cl_cpg = le16_to_cpu(di->id2.i_chain.cl_cpg); + u16 next_free = le16_to_cpu(di->id2.i_chain.cl_next_free_rec); + u32 cluster = ocfs2_blocks_to_clusters(inode->i_sb, input->group); + u32 total_clusters = le32_to_cpu(di->i_clusters); + int ret = -EINVAL; + + if (cluster < total_clusters) + mlog(ML_ERROR, "add a group which is in the current volume.\n"); + else if (input->chain >= cl_count) + mlog(ML_ERROR, "input chain exceeds the limit.\n"); + else if (next_free != cl_count && next_free != input->chain) + mlog(ML_ERROR, + "the add group should be in chain %u\n", next_free); + else if (total_clusters + input->clusters < total_clusters) + mlog(ML_ERROR, "add group's clusters overflow.\n"); + else if (input->clusters > cl_cpg) + mlog(ML_ERROR, "the cluster exceeds the maximum of a group\n"); + else if (input->frees > input->clusters) + mlog(ML_ERROR, "the free cluster exceeds the total clusters\n"); + else if (total_clusters % cl_cpg != 0) + mlog(ML_ERROR, + "the last group isn't full. Use group extend first.\n"); + else if (input->group != ocfs2_which_cluster_group(inode, cluster)) + mlog(ML_ERROR, "group blkno is invalid\n"); + else if ((ret = ocfs2_check_new_group(inode, di, input, group_bh))) + mlog(ML_ERROR, "group descriptor check failed.\n"); + else + ret = 0; + + return ret; +} + +/* Add a new group descriptor to global_bitmap. */ +int ocfs2_group_add(struct inode *inode, struct ocfs2_new_group_input *input) +{ + int ret; + handle_t *handle; + struct buffer_head *main_bm_bh = NULL; + struct inode *main_bm_inode = NULL; + struct ocfs2_dinode *fe = NULL; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct buffer_head *group_bh = NULL; + struct ocfs2_group_desc *group = NULL; + struct ocfs2_chain_list *cl; + struct ocfs2_chain_rec *cr; + u16 cl_bpc; + u64 bg_ptr; + + if (unlikely(ocfs2_emergency_state(osb))) + return -EROFS; + + main_bm_inode = ocfs2_get_system_file_inode(osb, + GLOBAL_BITMAP_SYSTEM_INODE, + OCFS2_INVALID_SLOT); + if (!main_bm_inode) { + ret = -EINVAL; + mlog_errno(ret); + goto out; + } + + inode_lock(main_bm_inode); + + ret = ocfs2_inode_lock(main_bm_inode, &main_bm_bh, 1); + if (ret < 0) { + mlog_errno(ret); + goto out_mutex; + } + + fe = (struct ocfs2_dinode *)main_bm_bh->b_data; + + if (le16_to_cpu(fe->id2.i_chain.cl_cpg) != + ocfs2_group_bitmap_size(osb->sb, 0, + osb->s_feature_incompat) * 8) { + mlog(ML_ERROR, "The disk is too old and small." + " Force to do offline resize."); + ret = -EINVAL; + goto out_unlock; + } + + ret = ocfs2_read_blocks_sync(osb, input->group, 1, &group_bh); + if (ret < 0) { + mlog(ML_ERROR, "Can't read the group descriptor # %llu " + "from the device.", (unsigned long long)input->group); + goto out_unlock; + } + + ret = ocfs2_verify_group_and_input(main_bm_inode, fe, input, group_bh); + if (ret) { + mlog_errno(ret); + goto out_free_group_bh; + } + + ocfs2_set_new_buffer_uptodate(INODE_CACHE(main_bm_inode), group_bh); + + trace_ocfs2_group_add((unsigned long long)input->group, + input->chain, input->clusters, input->frees); + + handle = ocfs2_start_trans(osb, OCFS2_GROUP_ADD_CREDITS); + if (IS_ERR(handle)) { + mlog_errno(PTR_ERR(handle)); + ret = -EINVAL; + goto out_remove_cache; + } + + cl_bpc = le16_to_cpu(fe->id2.i_chain.cl_bpc); + cl = &fe->id2.i_chain; + cr = &cl->cl_recs[input->chain]; + + ret = ocfs2_journal_access_gd(handle, INODE_CACHE(main_bm_inode), + group_bh, OCFS2_JOURNAL_ACCESS_WRITE); + if (ret < 0) { + mlog_errno(ret); + goto out_commit; + } + + group = (struct ocfs2_group_desc *)group_bh->b_data; + bg_ptr = le64_to_cpu(group->bg_next_group); + group->bg_next_group = cr->c_blkno; + ocfs2_journal_dirty(handle, group_bh); + + ret = ocfs2_journal_access_di(handle, INODE_CACHE(main_bm_inode), + main_bm_bh, OCFS2_JOURNAL_ACCESS_WRITE); + if (ret < 0) { + group->bg_next_group = cpu_to_le64(bg_ptr); + mlog_errno(ret); + goto out_commit; + } + + if (input->chain == le16_to_cpu(cl->cl_next_free_rec)) { + le16_add_cpu(&cl->cl_next_free_rec, 1); + memset(cr, 0, sizeof(struct ocfs2_chain_rec)); + } + + cr->c_blkno = cpu_to_le64(input->group); + le32_add_cpu(&cr->c_total, input->clusters * cl_bpc); + le32_add_cpu(&cr->c_free, input->frees * cl_bpc); + + le32_add_cpu(&fe->id1.bitmap1.i_total, input->clusters *cl_bpc); + le32_add_cpu(&fe->id1.bitmap1.i_used, + (input->clusters - input->frees) * cl_bpc); + le32_add_cpu(&fe->i_clusters, input->clusters); + + ocfs2_journal_dirty(handle, main_bm_bh); + + spin_lock(&OCFS2_I(main_bm_inode)->ip_lock); + OCFS2_I(main_bm_inode)->ip_clusters = le32_to_cpu(fe->i_clusters); + le64_add_cpu(&fe->i_size, (u64)input->clusters << osb->s_clustersize_bits); + spin_unlock(&OCFS2_I(main_bm_inode)->ip_lock); + i_size_write(main_bm_inode, le64_to_cpu(fe->i_size)); + + ocfs2_update_super_and_backups(main_bm_inode, input->clusters); + +out_commit: + ocfs2_commit_trans(osb, handle); + +out_remove_cache: + if (ret < 0) + ocfs2_remove_from_cache(INODE_CACHE(main_bm_inode), group_bh); + +out_free_group_bh: + brelse(group_bh); + +out_unlock: + brelse(main_bm_bh); + + ocfs2_inode_unlock(main_bm_inode, 1); + +out_mutex: + inode_unlock(main_bm_inode); + iput(main_bm_inode); + +out: + return ret; +} diff --git a/fs/ocfs2/resize.h b/fs/ocfs2/resize.h new file mode 100644 index 000000000..499063721 --- /dev/null +++ b/fs/ocfs2/resize.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * resize.h + * + * Function prototypes + * + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#ifndef OCFS2_RESIZE_H +#define OCFS2_RESIZE_H + +int ocfs2_group_extend(struct inode * inode, int new_clusters); +int ocfs2_group_add(struct inode *inode, struct ocfs2_new_group_input *input); + +#endif /* OCFS2_RESIZE_H */ diff --git a/fs/ocfs2/slot_map.c b/fs/ocfs2/slot_map.c new file mode 100644 index 000000000..3aa4a082e --- /dev/null +++ b/fs/ocfs2/slot_map.c @@ -0,0 +1,542 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * slot_map.c + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#include +#include +#include + +#include + +#include "ocfs2.h" + +#include "dlmglue.h" +#include "extent_map.h" +#include "heartbeat.h" +#include "inode.h" +#include "slot_map.h" +#include "super.h" +#include "sysfile.h" +#include "ocfs2_trace.h" + +#include "buffer_head_io.h" + + +struct ocfs2_slot { + int sl_valid; + unsigned int sl_node_num; +}; + +struct ocfs2_slot_info { + int si_extended; + int si_slots_per_block; + struct inode *si_inode; + unsigned int si_blocks; + struct buffer_head **si_bh; + unsigned int si_num_slots; + struct ocfs2_slot si_slots[] __counted_by(si_num_slots); +}; + + +static int __ocfs2_node_num_to_slot(struct ocfs2_slot_info *si, + unsigned int node_num); + +static int ocfs2_validate_slot_map_block(struct super_block *sb, + struct buffer_head *bh); + +static void ocfs2_invalidate_slot(struct ocfs2_slot_info *si, + int slot_num) +{ + BUG_ON((slot_num < 0) || (slot_num >= si->si_num_slots)); + si->si_slots[slot_num].sl_valid = 0; +} + +static void ocfs2_set_slot(struct ocfs2_slot_info *si, + int slot_num, unsigned int node_num) +{ + BUG_ON((slot_num < 0) || (slot_num >= si->si_num_slots)); + + si->si_slots[slot_num].sl_valid = 1; + si->si_slots[slot_num].sl_node_num = node_num; +} + +/* This version is for the extended slot map */ +static void ocfs2_update_slot_info_extended(struct ocfs2_slot_info *si) +{ + int b, i, slotno; + struct ocfs2_slot_map_extended *se; + + slotno = 0; + for (b = 0; b < si->si_blocks; b++) { + se = (struct ocfs2_slot_map_extended *)si->si_bh[b]->b_data; + for (i = 0; + (i < si->si_slots_per_block) && + (slotno < si->si_num_slots); + i++, slotno++) { + if (se->se_slots[i].es_valid) + ocfs2_set_slot(si, slotno, + le32_to_cpu(se->se_slots[i].es_node_num)); + else + ocfs2_invalidate_slot(si, slotno); + } + } +} + +/* + * Post the slot information on disk into our slot_info struct. + * Must be protected by osb_lock. + */ +static void ocfs2_update_slot_info_old(struct ocfs2_slot_info *si) +{ + int i; + struct ocfs2_slot_map *sm; + + sm = (struct ocfs2_slot_map *)si->si_bh[0]->b_data; + + for (i = 0; i < si->si_num_slots; i++) { + if (le16_to_cpu(sm->sm_slots[i]) == (u16)OCFS2_INVALID_SLOT) + ocfs2_invalidate_slot(si, i); + else + ocfs2_set_slot(si, i, le16_to_cpu(sm->sm_slots[i])); + } +} + +static void ocfs2_update_slot_info(struct ocfs2_slot_info *si) +{ + /* + * The slot data will have been refreshed when ocfs2_super_lock + * was taken. + */ + if (si->si_extended) + ocfs2_update_slot_info_extended(si); + else + ocfs2_update_slot_info_old(si); +} + +int ocfs2_refresh_slot_info(struct ocfs2_super *osb) +{ + int ret; + struct ocfs2_slot_info *si = osb->slot_info; + + if (si == NULL) + return 0; + + BUG_ON(si->si_blocks == 0); + BUG_ON(si->si_bh == NULL); + + trace_ocfs2_refresh_slot_info(si->si_blocks); + + /* + * We pass -1 as blocknr because we expect all of si->si_bh to + * be !NULL. Thus, ocfs2_read_blocks() will ignore blocknr. If + * this is not true, the read of -1 (UINT64_MAX) will fail. + */ + ret = ocfs2_read_blocks(INODE_CACHE(si->si_inode), -1, si->si_blocks, + si->si_bh, OCFS2_BH_IGNORE_CACHE, + ocfs2_validate_slot_map_block); + if (ret == 0) { + spin_lock(&osb->osb_lock); + ocfs2_update_slot_info(si); + spin_unlock(&osb->osb_lock); + } + + return ret; +} + +/* post the our slot info stuff into it's destination bh and write it + * out. */ +static void ocfs2_update_disk_slot_extended(struct ocfs2_slot_info *si, + int slot_num, + struct buffer_head **bh) +{ + int blkind = slot_num / si->si_slots_per_block; + int slotno = slot_num % si->si_slots_per_block; + struct ocfs2_slot_map_extended *se; + + BUG_ON(blkind >= si->si_blocks); + + se = (struct ocfs2_slot_map_extended *)si->si_bh[blkind]->b_data; + se->se_slots[slotno].es_valid = si->si_slots[slot_num].sl_valid; + if (si->si_slots[slot_num].sl_valid) + se->se_slots[slotno].es_node_num = + cpu_to_le32(si->si_slots[slot_num].sl_node_num); + *bh = si->si_bh[blkind]; +} + +static void ocfs2_update_disk_slot_old(struct ocfs2_slot_info *si, + int slot_num, + struct buffer_head **bh) +{ + int i; + struct ocfs2_slot_map *sm; + + sm = (struct ocfs2_slot_map *)si->si_bh[0]->b_data; + for (i = 0; i < si->si_num_slots; i++) { + if (si->si_slots[i].sl_valid) + sm->sm_slots[i] = + cpu_to_le16(si->si_slots[i].sl_node_num); + else + sm->sm_slots[i] = cpu_to_le16(OCFS2_INVALID_SLOT); + } + *bh = si->si_bh[0]; +} + +static int ocfs2_update_disk_slot(struct ocfs2_super *osb, + struct ocfs2_slot_info *si, + int slot_num) +{ + int status; + struct buffer_head *bh; + + spin_lock(&osb->osb_lock); + if (si->si_extended) + ocfs2_update_disk_slot_extended(si, slot_num, &bh); + else + ocfs2_update_disk_slot_old(si, slot_num, &bh); + spin_unlock(&osb->osb_lock); + + status = ocfs2_write_block(osb, bh, INODE_CACHE(si->si_inode)); + if (status < 0) + mlog_errno(status); + + return status; +} + +/* + * Calculate how many bytes are needed by the slot map. Returns + * an error if the slot map file is too small. + */ +static int ocfs2_slot_map_physical_size(struct ocfs2_super *osb, + struct inode *inode, + unsigned long long *bytes) +{ + unsigned long long bytes_needed; + + if (ocfs2_uses_extended_slot_map(osb)) { + bytes_needed = osb->max_slots * + sizeof(struct ocfs2_extended_slot); + } else { + bytes_needed = osb->max_slots * sizeof(__le16); + } + if (bytes_needed > i_size_read(inode)) { + mlog(ML_ERROR, + "Slot map file is too small! (size %llu, needed %llu)\n", + i_size_read(inode), bytes_needed); + return -ENOSPC; + } + + *bytes = bytes_needed; + return 0; +} + +/* try to find global node in the slot info. Returns -ENOENT + * if nothing is found. */ +static int __ocfs2_node_num_to_slot(struct ocfs2_slot_info *si, + unsigned int node_num) +{ + int i, ret = -ENOENT; + + for(i = 0; i < si->si_num_slots; i++) { + if (si->si_slots[i].sl_valid && + (node_num == si->si_slots[i].sl_node_num)) { + ret = i; + break; + } + } + + return ret; +} + +static int __ocfs2_find_empty_slot(struct ocfs2_slot_info *si, + int preferred) +{ + int i, ret = -ENOSPC; + + if ((preferred >= 0) && (preferred < si->si_num_slots)) { + if (!si->si_slots[preferred].sl_valid) { + ret = preferred; + goto out; + } + } + + for(i = 0; i < si->si_num_slots; i++) { + if (!si->si_slots[i].sl_valid) { + ret = i; + break; + } + } +out: + return ret; +} + +int ocfs2_node_num_to_slot(struct ocfs2_super *osb, unsigned int node_num) +{ + int slot; + struct ocfs2_slot_info *si = osb->slot_info; + + spin_lock(&osb->osb_lock); + slot = __ocfs2_node_num_to_slot(si, node_num); + spin_unlock(&osb->osb_lock); + + return slot; +} + +int ocfs2_slot_to_node_num_locked(struct ocfs2_super *osb, int slot_num, + unsigned int *node_num) +{ + struct ocfs2_slot_info *si = osb->slot_info; + + assert_spin_locked(&osb->osb_lock); + + BUG_ON(slot_num < 0); + BUG_ON(slot_num >= osb->max_slots); + + if (!si->si_slots[slot_num].sl_valid) + return -ENOENT; + + *node_num = si->si_slots[slot_num].sl_node_num; + return 0; +} + +static void __ocfs2_free_slot_info(struct ocfs2_slot_info *si) +{ + unsigned int i; + + if (si == NULL) + return; + + iput(si->si_inode); + if (si->si_bh) { + for (i = 0; i < si->si_blocks; i++) { + if (si->si_bh[i]) { + brelse(si->si_bh[i]); + si->si_bh[i] = NULL; + } + } + kfree(si->si_bh); + } + + kfree(si); +} + +int ocfs2_clear_slot(struct ocfs2_super *osb, int slot_num) +{ + struct ocfs2_slot_info *si = osb->slot_info; + + if (si == NULL) + return 0; + + spin_lock(&osb->osb_lock); + ocfs2_invalidate_slot(si, slot_num); + spin_unlock(&osb->osb_lock); + + return ocfs2_update_disk_slot(osb, osb->slot_info, slot_num); +} + +static int ocfs2_validate_slot_map_block(struct super_block *sb, + struct buffer_head *bh) +{ + int rc; + + BUG_ON(!buffer_uptodate(bh)); + + if (bh->b_blocknr < OCFS2_SUPER_BLOCK_BLKNO) { + rc = ocfs2_error(sb, + "Invalid Slot Map Buffer Head " + "Block Number : %llu, Should be >= %d", + (unsigned long long)bh->b_blocknr, + OCFS2_SUPER_BLOCK_BLKNO); + return rc; + } + return 0; +} + +static int ocfs2_map_slot_buffers(struct ocfs2_super *osb, + struct ocfs2_slot_info *si) +{ + int status = 0; + u64 blkno; + unsigned long long blocks, bytes = 0; + unsigned int i; + struct buffer_head *bh; + + status = ocfs2_slot_map_physical_size(osb, si->si_inode, &bytes); + if (status) + goto bail; + + blocks = ocfs2_blocks_for_bytes(si->si_inode->i_sb, bytes); + BUG_ON(blocks > UINT_MAX); + si->si_blocks = blocks; + if (!si->si_blocks) + goto bail; + + if (si->si_extended) + si->si_slots_per_block = + (osb->sb->s_blocksize / + sizeof(struct ocfs2_extended_slot)); + else + si->si_slots_per_block = osb->sb->s_blocksize / sizeof(__le16); + + /* The size checks above should ensure this */ + BUG_ON((osb->max_slots / si->si_slots_per_block) > blocks); + + trace_ocfs2_map_slot_buffers(bytes, si->si_blocks); + + si->si_bh = kzalloc_objs(struct buffer_head *, si->si_blocks); + if (!si->si_bh) { + status = -ENOMEM; + mlog_errno(status); + goto bail; + } + + for (i = 0; i < si->si_blocks; i++) { + status = ocfs2_extent_map_get_blocks(si->si_inode, i, + &blkno, NULL, NULL); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + trace_ocfs2_map_slot_buffers_block((unsigned long long)blkno, i); + + bh = NULL; /* Acquire a fresh bh */ + status = ocfs2_read_blocks(INODE_CACHE(si->si_inode), blkno, + 1, &bh, OCFS2_BH_IGNORE_CACHE, + ocfs2_validate_slot_map_block); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + si->si_bh[i] = bh; + } + +bail: + return status; +} + +int ocfs2_init_slot_info(struct ocfs2_super *osb) +{ + int status; + struct inode *inode = NULL; + struct ocfs2_slot_info *si; + + si = kzalloc_flex(*si, si_slots, osb->max_slots); + if (!si) { + status = -ENOMEM; + mlog_errno(status); + return status; + } + + si->si_extended = ocfs2_uses_extended_slot_map(osb); + si->si_num_slots = osb->max_slots; + + inode = ocfs2_get_system_file_inode(osb, SLOT_MAP_SYSTEM_INODE, + OCFS2_INVALID_SLOT); + if (!inode) { + status = -EINVAL; + mlog_errno(status); + goto bail; + } + + si->si_inode = inode; + status = ocfs2_map_slot_buffers(osb, si); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + osb->slot_info = (struct ocfs2_slot_info *)si; +bail: + if (status < 0) + __ocfs2_free_slot_info(si); + + return status; +} + +void ocfs2_free_slot_info(struct ocfs2_super *osb) +{ + struct ocfs2_slot_info *si = osb->slot_info; + + osb->slot_info = NULL; + __ocfs2_free_slot_info(si); +} + +int ocfs2_find_slot(struct ocfs2_super *osb) +{ + int status; + int slot; + struct ocfs2_slot_info *si; + + si = osb->slot_info; + + spin_lock(&osb->osb_lock); + ocfs2_update_slot_info(si); + + /* search for ourselves first and take the slot if it already + * exists. Perhaps we need to mark this in a variable for our + * own journal recovery? Possibly not, though we certainly + * need to warn to the user */ + slot = __ocfs2_node_num_to_slot(si, osb->node_num); + if (slot < 0) { + /* if no slot yet, then just take 1st available + * one. */ + slot = __ocfs2_find_empty_slot(si, osb->preferred_slot); + if (slot < 0) { + spin_unlock(&osb->osb_lock); + mlog(ML_ERROR, "no free slots available!\n"); + status = -EINVAL; + goto bail; + } + } else + printk(KERN_INFO "ocfs2: Slot %d on device (%s) was already " + "allocated to this node!\n", slot, osb->dev_str); + + ocfs2_set_slot(si, slot, osb->node_num); + osb->slot_num = slot; + spin_unlock(&osb->osb_lock); + + trace_ocfs2_find_slot(osb->slot_num); + + status = ocfs2_update_disk_slot(osb, si, osb->slot_num); + if (status < 0) { + mlog_errno(status); + /* + * if write block failed, invalidate slot to avoid overwrite + * slot during dismount in case another node rightly has mounted + */ + spin_lock(&osb->osb_lock); + ocfs2_invalidate_slot(si, osb->slot_num); + osb->slot_num = OCFS2_INVALID_SLOT; + spin_unlock(&osb->osb_lock); + } + +bail: + return status; +} + +void ocfs2_put_slot(struct ocfs2_super *osb) +{ + int status, slot_num; + struct ocfs2_slot_info *si = osb->slot_info; + + if (!si) + return; + + spin_lock(&osb->osb_lock); + ocfs2_update_slot_info(si); + + slot_num = osb->slot_num; + ocfs2_invalidate_slot(si, osb->slot_num); + osb->slot_num = OCFS2_INVALID_SLOT; + spin_unlock(&osb->osb_lock); + + status = ocfs2_update_disk_slot(osb, si, slot_num); + if (status < 0) + mlog_errno(status); + + ocfs2_free_slot_info(osb); +} diff --git a/fs/ocfs2/slot_map.h b/fs/ocfs2/slot_map.h new file mode 100644 index 000000000..a43644570 --- /dev/null +++ b/fs/ocfs2/slot_map.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * slotmap.h + * + * description here + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + + +#ifndef SLOTMAP_H +#define SLOTMAP_H + +int ocfs2_init_slot_info(struct ocfs2_super *osb); +void ocfs2_free_slot_info(struct ocfs2_super *osb); + +int ocfs2_find_slot(struct ocfs2_super *osb); +void ocfs2_put_slot(struct ocfs2_super *osb); + +int ocfs2_refresh_slot_info(struct ocfs2_super *osb); + +int ocfs2_node_num_to_slot(struct ocfs2_super *osb, unsigned int node_num); +int ocfs2_slot_to_node_num_locked(struct ocfs2_super *osb, int slot_num, + unsigned int *node_num); + +int ocfs2_clear_slot(struct ocfs2_super *osb, int slot_num); + +#endif diff --git a/fs/ocfs2/stack_o2cb.c b/fs/ocfs2/stack_o2cb.c new file mode 100644 index 000000000..7a55b1a07 --- /dev/null +++ b/fs/ocfs2/stack_o2cb.c @@ -0,0 +1,439 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * stack_o2cb.c + * + * Code which interfaces ocfs2 with the o2cb stack. + * + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#include +#include +#include +#include + +/* Needed for AOP_TRUNCATED_PAGE in mlog_errno() */ +#include + +#include "cluster/masklog.h" +#include "cluster/nodemanager.h" +#include "cluster/heartbeat.h" +#include "cluster/tcp.h" + +#include "stackglue.h" + +struct o2dlm_private { + struct dlm_eviction_cb op_eviction_cb; +}; + +static struct ocfs2_stack_plugin o2cb_stack; + +/* These should be identical */ +#if (DLM_LOCK_IV != LKM_IVMODE) +# error Lock modes do not match +#endif +#if (DLM_LOCK_NL != LKM_NLMODE) +# error Lock modes do not match +#endif +#if (DLM_LOCK_CR != LKM_CRMODE) +# error Lock modes do not match +#endif +#if (DLM_LOCK_CW != LKM_CWMODE) +# error Lock modes do not match +#endif +#if (DLM_LOCK_PR != LKM_PRMODE) +# error Lock modes do not match +#endif +#if (DLM_LOCK_PW != LKM_PWMODE) +# error Lock modes do not match +#endif +#if (DLM_LOCK_EX != LKM_EXMODE) +# error Lock modes do not match +#endif +static inline int mode_to_o2dlm(int mode) +{ + BUG_ON(mode > LKM_MAXMODE); + + return mode; +} + +static int flags_to_o2dlm(u32 flags) +{ + int o2dlm_flags = 0; + + if (flags & DLM_LKF_NOQUEUE) + o2dlm_flags |= LKM_NOQUEUE; + if (flags & DLM_LKF_CANCEL) + o2dlm_flags |= LKM_CANCEL; + if (flags & DLM_LKF_CONVERT) + o2dlm_flags |= LKM_CONVERT; + if (flags & DLM_LKF_VALBLK) + o2dlm_flags |= LKM_VALBLK; + if (flags & DLM_LKF_IVVALBLK) + o2dlm_flags |= LKM_INVVALBLK; + if (flags & DLM_LKF_ORPHAN) + o2dlm_flags |= LKM_ORPHAN; + if (flags & DLM_LKF_FORCEUNLOCK) + o2dlm_flags |= LKM_FORCE; + if (flags & DLM_LKF_TIMEOUT) + o2dlm_flags |= LKM_TIMEOUT; + if (flags & DLM_LKF_LOCAL) + o2dlm_flags |= LKM_LOCAL; + + return o2dlm_flags; +} + +/* + * Map an o2dlm status to standard errno values. + * + * o2dlm only uses a handful of these, and returns even fewer to the + * caller. Still, we try to assign sane values to each error. + * + * The following value pairs have special meanings to dlmglue, thus + * the right hand side needs to stay unique - never duplicate the + * mapping elsewhere in the table! + * + * DLM_NORMAL: 0 + * DLM_NOTQUEUED: -EAGAIN + * DLM_CANCELGRANT: -EBUSY + * DLM_CANCEL: -DLM_ECANCEL + */ +/* Keep in sync with dlmapi.h */ +static int status_map[] = { + [DLM_NORMAL] = 0, /* Success */ + [DLM_GRANTED] = -EINVAL, + [DLM_DENIED] = -EACCES, + [DLM_DENIED_NOLOCKS] = -EACCES, + [DLM_WORKING] = -EACCES, + [DLM_BLOCKED] = -EINVAL, + [DLM_BLOCKED_ORPHAN] = -EINVAL, + [DLM_DENIED_GRACE_PERIOD] = -EACCES, + [DLM_SYSERR] = -ENOMEM, /* It is what it is */ + [DLM_NOSUPPORT] = -EPROTO, + [DLM_CANCELGRANT] = -EBUSY, /* Cancel after grant */ + [DLM_IVLOCKID] = -EINVAL, + [DLM_SYNC] = -EINVAL, + [DLM_BADTYPE] = -EINVAL, + [DLM_BADRESOURCE] = -EINVAL, + [DLM_MAXHANDLES] = -ENOMEM, + [DLM_NOCLINFO] = -EINVAL, + [DLM_NOLOCKMGR] = -EINVAL, + [DLM_NOPURGED] = -EINVAL, + [DLM_BADARGS] = -EINVAL, + [DLM_VOID] = -EINVAL, + [DLM_NOTQUEUED] = -EAGAIN, /* Trylock failed */ + [DLM_IVBUFLEN] = -EINVAL, + [DLM_CVTUNGRANT] = -EPERM, + [DLM_BADPARAM] = -EINVAL, + [DLM_VALNOTVALID] = -EINVAL, + [DLM_REJECTED] = -EPERM, + [DLM_ABORT] = -EINVAL, + [DLM_CANCEL] = -DLM_ECANCEL, /* Successful cancel */ + [DLM_IVRESHANDLE] = -EINVAL, + [DLM_DEADLOCK] = -EDEADLK, + [DLM_DENIED_NOASTS] = -EINVAL, + [DLM_FORWARD] = -EINVAL, + [DLM_TIMEOUT] = -ETIMEDOUT, + [DLM_IVGROUPID] = -EINVAL, + [DLM_VERS_CONFLICT] = -EOPNOTSUPP, + [DLM_BAD_DEVICE_PATH] = -ENOENT, + [DLM_NO_DEVICE_PERMISSION] = -EPERM, + [DLM_NO_CONTROL_DEVICE] = -ENOENT, + [DLM_RECOVERING] = -ENOTCONN, + [DLM_MIGRATING] = -ERESTART, + [DLM_MAXSTATS] = -EINVAL, +}; + +static int dlm_status_to_errno(enum dlm_status status) +{ + BUG_ON(status < 0 || status >= ARRAY_SIZE(status_map)); + + return status_map[status]; +} + +static void o2dlm_lock_ast_wrapper(void *astarg) +{ + struct ocfs2_dlm_lksb *lksb = astarg; + + lksb->lksb_conn->cc_proto->lp_lock_ast(lksb); +} + +static void o2dlm_blocking_ast_wrapper(void *astarg, int level) +{ + struct ocfs2_dlm_lksb *lksb = astarg; + + lksb->lksb_conn->cc_proto->lp_blocking_ast(lksb, level); +} + +static void o2dlm_unlock_ast_wrapper(void *astarg, enum dlm_status status) +{ + struct ocfs2_dlm_lksb *lksb = astarg; + int error = dlm_status_to_errno(status); + + /* + * In o2dlm, you can get both the lock_ast() for the lock being + * granted and the unlock_ast() for the CANCEL failing. A + * successful cancel sends DLM_NORMAL here. If the + * lock grant happened before the cancel arrived, you get + * DLM_CANCELGRANT. + * + * There's no need for the double-ast. If we see DLM_CANCELGRANT, + * we just ignore it. We expect the lock_ast() to handle the + * granted lock. + */ + if (status == DLM_CANCELGRANT) + return; + + lksb->lksb_conn->cc_proto->lp_unlock_ast(lksb, error); +} + +static int o2cb_dlm_lock(struct ocfs2_cluster_connection *conn, + int mode, + struct ocfs2_dlm_lksb *lksb, + u32 flags, + void *name, + unsigned int namelen) +{ + enum dlm_status status; + int o2dlm_mode = mode_to_o2dlm(mode); + int o2dlm_flags = flags_to_o2dlm(flags); + int ret; + + status = dlmlock(conn->cc_lockspace, o2dlm_mode, &lksb->lksb_o2dlm, + o2dlm_flags, name, namelen, + o2dlm_lock_ast_wrapper, lksb, + o2dlm_blocking_ast_wrapper); + ret = dlm_status_to_errno(status); + return ret; +} + +static int o2cb_dlm_unlock(struct ocfs2_cluster_connection *conn, + struct ocfs2_dlm_lksb *lksb, + u32 flags) +{ + enum dlm_status status; + int o2dlm_flags = flags_to_o2dlm(flags); + int ret; + + status = dlmunlock(conn->cc_lockspace, &lksb->lksb_o2dlm, + o2dlm_flags, o2dlm_unlock_ast_wrapper, lksb); + ret = dlm_status_to_errno(status); + return ret; +} + +static int o2cb_dlm_lock_status(struct ocfs2_dlm_lksb *lksb) +{ + return dlm_status_to_errno(lksb->lksb_o2dlm.status); +} + +/* + * o2dlm always has a "valid" LVB. If the dlm loses track of the LVB + * contents, it will zero out the LVB. Thus the caller can always trust + * the contents. + */ +static int o2cb_dlm_lvb_valid(struct ocfs2_dlm_lksb *lksb) +{ + return 1; +} + +static void *o2cb_dlm_lvb(struct ocfs2_dlm_lksb *lksb) +{ + return (void *)(lksb->lksb_o2dlm.lvb); +} + +static void o2cb_dump_lksb(struct ocfs2_dlm_lksb *lksb) +{ + dlm_print_one_lock(lksb->lksb_o2dlm.lockid); +} + +/* + * Check if this node is heartbeating and is connected to all other + * heartbeating nodes. + */ +static int o2cb_cluster_check(void) +{ + u8 node_num; + int i; + unsigned long hbmap[BITS_TO_LONGS(O2NM_MAX_NODES)]; + unsigned long netmap[BITS_TO_LONGS(O2NM_MAX_NODES)]; + + node_num = o2nm_this_node(); + if (node_num == O2NM_MAX_NODES) { + printk(KERN_ERR "o2cb: This node has not been configured.\n"); + return -EINVAL; + } + + /* + * o2dlm expects o2net sockets to be created. If not, then + * dlm_join_domain() fails with a stack of errors which are both cryptic + * and incomplete. The idea here is to detect upfront whether we have + * managed to connect to all nodes or not. If not, then list the nodes + * to allow the user to check the configuration (incorrect IP, firewall, + * etc.) Yes, this is racy. But its not the end of the world. + */ +#define O2CB_MAP_STABILIZE_COUNT 60 + for (i = 0; i < O2CB_MAP_STABILIZE_COUNT; ++i) { + o2hb_fill_node_map(hbmap, O2NM_MAX_NODES); + if (!test_bit(node_num, hbmap)) { + printk(KERN_ERR "o2cb: %s heartbeat has not been " + "started.\n", (o2hb_global_heartbeat_active() ? + "Global" : "Local")); + return -EINVAL; + } + o2net_fill_node_map(netmap, O2NM_MAX_NODES); + /* Force set the current node to allow easy compare */ + set_bit(node_num, netmap); + if (bitmap_equal(hbmap, netmap, O2NM_MAX_NODES)) + return 0; + if (i < O2CB_MAP_STABILIZE_COUNT - 1) + msleep(1000); + } + + printk(KERN_ERR "o2cb: This node could not connect to nodes:"); + i = -1; + while ((i = find_next_bit(hbmap, O2NM_MAX_NODES, + i + 1)) < O2NM_MAX_NODES) { + if (!test_bit(i, netmap)) + printk(" %u", i); + } + printk(".\n"); + + return -ENOTCONN; +} + +/* + * Called from the dlm when it's about to evict a node. This is how the + * classic stack signals node death. + */ +static void o2dlm_eviction_cb(int node_num, void *data) +{ + struct ocfs2_cluster_connection *conn = data; + + printk(KERN_NOTICE "o2cb: o2dlm has evicted node %d from domain %.*s\n", + node_num, conn->cc_namelen, conn->cc_name); + + conn->cc_recovery_handler(node_num, conn->cc_recovery_data); +} + +static int o2cb_cluster_connect(struct ocfs2_cluster_connection *conn) +{ + int rc = 0; + u32 dlm_key; + struct dlm_ctxt *dlm; + struct o2dlm_private *priv; + struct dlm_protocol_version fs_version; + + BUG_ON(conn == NULL); + BUG_ON(conn->cc_proto == NULL); + + /* Ensure cluster stack is up and all nodes are connected */ + rc = o2cb_cluster_check(); + if (rc) { + printk(KERN_ERR "o2cb: Cluster check failed. Fix errors " + "before retrying.\n"); + goto out; + } + + priv = kzalloc_obj(struct o2dlm_private); + if (!priv) { + rc = -ENOMEM; + goto out_free; + } + + /* This just fills the structure in. It is safe to pass conn. */ + dlm_setup_eviction_cb(&priv->op_eviction_cb, o2dlm_eviction_cb, + conn); + + conn->cc_private = priv; + + /* used by the dlm code to make message headers unique, each + * node in this domain must agree on this. */ + dlm_key = crc32_le(0, conn->cc_name, conn->cc_namelen); + fs_version.pv_major = conn->cc_version.pv_major; + fs_version.pv_minor = conn->cc_version.pv_minor; + + dlm = dlm_register_domain(conn->cc_name, dlm_key, &fs_version); + if (IS_ERR(dlm)) { + rc = PTR_ERR(dlm); + mlog_errno(rc); + goto out_free; + } + + conn->cc_version.pv_major = fs_version.pv_major; + conn->cc_version.pv_minor = fs_version.pv_minor; + conn->cc_lockspace = dlm; + + dlm_register_eviction_cb(dlm, &priv->op_eviction_cb); + +out_free: + if (rc) + kfree(conn->cc_private); + +out: + return rc; +} + +static int o2cb_cluster_disconnect(struct ocfs2_cluster_connection *conn) +{ + struct dlm_ctxt *dlm = conn->cc_lockspace; + struct o2dlm_private *priv = conn->cc_private; + + dlm_unregister_eviction_cb(&priv->op_eviction_cb); + conn->cc_private = NULL; + kfree(priv); + + dlm_unregister_domain(dlm); + conn->cc_lockspace = NULL; + + return 0; +} + +static int o2cb_cluster_this_node(struct ocfs2_cluster_connection *conn, + unsigned int *node) +{ + int node_num; + + node_num = o2nm_this_node(); + if (node_num == O2NM_INVALID_NODE_NUM) + return -ENOENT; + + if (node_num >= O2NM_MAX_NODES) + return -EOVERFLOW; + + *node = node_num; + return 0; +} + +static const struct ocfs2_stack_operations o2cb_stack_ops = { + .connect = o2cb_cluster_connect, + .disconnect = o2cb_cluster_disconnect, + .this_node = o2cb_cluster_this_node, + .dlm_lock = o2cb_dlm_lock, + .dlm_unlock = o2cb_dlm_unlock, + .lock_status = o2cb_dlm_lock_status, + .lvb_valid = o2cb_dlm_lvb_valid, + .lock_lvb = o2cb_dlm_lvb, + .dump_lksb = o2cb_dump_lksb, +}; + +static struct ocfs2_stack_plugin o2cb_stack = { + .sp_name = "o2cb", + .sp_ops = &o2cb_stack_ops, + .sp_owner = THIS_MODULE, +}; + +static int __init o2cb_stack_init(void) +{ + return ocfs2_stack_glue_register(&o2cb_stack); +} + +static void __exit o2cb_stack_exit(void) +{ + ocfs2_stack_glue_unregister(&o2cb_stack); +} + +MODULE_AUTHOR("Oracle"); +MODULE_DESCRIPTION("ocfs2 driver for the classic o2cb stack"); +MODULE_LICENSE("GPL"); +module_init(o2cb_stack_init); +module_exit(o2cb_stack_exit); diff --git a/fs/ocfs2/stack_user.c b/fs/ocfs2/stack_user.c new file mode 100644 index 000000000..91e19d338 --- /dev/null +++ b/fs/ocfs2/stack_user.c @@ -0,0 +1,1102 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * stack_user.c + * + * Code which interfaces ocfs2 with fs/dlm and a userspace stack. + * + * Copyright (C) 2007 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "stackglue.h" + +#include + +/* + * The control protocol starts with a handshake. Until the handshake + * is complete, the control device will fail all write(2)s. + * + * The handshake is simple. First, the client reads until EOF. Each line + * of output is a supported protocol tag. All protocol tags are a single + * character followed by a two hex digit version number. Currently the + * only things supported is T01, for "Text-base version 0x01". Next, the + * client writes the version they would like to use, including the newline. + * Thus, the protocol tag is 'T01\n'. If the version tag written is + * unknown, -EINVAL is returned. Once the negotiation is complete, the + * client can start sending messages. + * + * The T01 protocol has three messages. First is the "SETN" message. + * It has the following syntax: + * + * SETN<8-char-hex-nodenum> + * + * This is 14 characters. + * + * The "SETN" message must be the first message following the protocol. + * It tells ocfs2_control the local node number. + * + * Next comes the "SETV" message. It has the following syntax: + * + * SETV<2-char-hex-major><2-char-hex-minor> + * + * This is 11 characters. + * + * The "SETV" message sets the filesystem locking protocol version as + * negotiated by the client. The client negotiates based on the maximum + * version advertised in /sys/fs/ocfs2/max_locking_protocol. The major + * number from the "SETV" message must match + * ocfs2_user_plugin.sp_max_proto.pv_major, and the minor number + * must be less than or equal to ...sp_max_version.pv_minor. + * + * Once this information has been set, mounts will be allowed. From this + * point on, the "DOWN" message can be sent for node down notification. + * It has the following syntax: + * + * DOWN<32-char-cap-hex-uuid><8-char-hex-nodenum> + * + * eg: + * + * DOWN 632A924FDD844190BDA93C0DF6B94899 00000001\n + * + * This is 47 characters. + */ + +/* + * Whether or not the client has done the handshake. + * For now, we have just one protocol version. + */ +#define OCFS2_CONTROL_PROTO "T01\n" +#define OCFS2_CONTROL_PROTO_LEN 4 + +/* Handshake states */ +#define OCFS2_CONTROL_HANDSHAKE_INVALID (0) +#define OCFS2_CONTROL_HANDSHAKE_READ (1) +#define OCFS2_CONTROL_HANDSHAKE_PROTOCOL (2) +#define OCFS2_CONTROL_HANDSHAKE_VALID (3) + +/* Messages */ +#define OCFS2_CONTROL_MESSAGE_OP_LEN 4 +#define OCFS2_CONTROL_MESSAGE_SETNODE_OP "SETN" +#define OCFS2_CONTROL_MESSAGE_SETNODE_TOTAL_LEN 14 +#define OCFS2_CONTROL_MESSAGE_SETVERSION_OP "SETV" +#define OCFS2_CONTROL_MESSAGE_SETVERSION_TOTAL_LEN 11 +#define OCFS2_CONTROL_MESSAGE_DOWN_OP "DOWN" +#define OCFS2_CONTROL_MESSAGE_DOWN_TOTAL_LEN 47 +#define OCFS2_TEXT_UUID_LEN 32 +#define OCFS2_CONTROL_MESSAGE_VERNUM_LEN 2 +#define OCFS2_CONTROL_MESSAGE_NODENUM_LEN 8 +#define VERSION_LOCK "version_lock" + +enum ocfs2_connection_type { + WITH_CONTROLD, + NO_CONTROLD +}; + +/* + * ocfs2_live_connection is refcounted because the filesystem and + * miscdevice sides can detach in different order. Let's just be safe. + */ +struct ocfs2_live_connection { + struct list_head oc_list; + struct ocfs2_cluster_connection *oc_conn; + enum ocfs2_connection_type oc_type; + atomic_t oc_this_node; + int oc_our_slot; + struct dlm_lksb oc_version_lksb; + char oc_lvb[DLM_LVB_LEN]; + struct completion oc_sync_wait; + wait_queue_head_t oc_wait; +}; + +struct ocfs2_control_private { + struct list_head op_list; + int op_state; + int op_this_node; + struct ocfs2_protocol_version op_proto; +}; + +/* SETN<8-char-hex-nodenum> */ +struct ocfs2_control_message_setn { + char tag[OCFS2_CONTROL_MESSAGE_OP_LEN]; + char space; + char nodestr[OCFS2_CONTROL_MESSAGE_NODENUM_LEN]; + char newline; +}; + +/* SETV<2-char-hex-major><2-char-hex-minor> */ +struct ocfs2_control_message_setv { + char tag[OCFS2_CONTROL_MESSAGE_OP_LEN]; + char space1; + char major[OCFS2_CONTROL_MESSAGE_VERNUM_LEN]; + char space2; + char minor[OCFS2_CONTROL_MESSAGE_VERNUM_LEN]; + char newline; +}; + +/* DOWN<32-char-cap-hex-uuid><8-char-hex-nodenum> */ +struct ocfs2_control_message_down { + char tag[OCFS2_CONTROL_MESSAGE_OP_LEN]; + char space1; + char uuid[OCFS2_TEXT_UUID_LEN]; + char space2; + char nodestr[OCFS2_CONTROL_MESSAGE_NODENUM_LEN]; + char newline; +}; + +union ocfs2_control_message { + char tag[OCFS2_CONTROL_MESSAGE_OP_LEN]; + struct ocfs2_control_message_setn u_setn; + struct ocfs2_control_message_setv u_setv; + struct ocfs2_control_message_down u_down; +}; + +static struct ocfs2_stack_plugin ocfs2_user_plugin; + +static atomic_t ocfs2_control_opened; +static int ocfs2_control_this_node = -1; +static struct ocfs2_protocol_version running_proto; + +static LIST_HEAD(ocfs2_live_connection_list); +static LIST_HEAD(ocfs2_control_private_list); +static DEFINE_MUTEX(ocfs2_control_lock); + +static inline void ocfs2_control_set_handshake_state(struct file *file, + int state) +{ + struct ocfs2_control_private *p = file->private_data; + p->op_state = state; +} + +static inline int ocfs2_control_get_handshake_state(struct file *file) +{ + struct ocfs2_control_private *p = file->private_data; + return p->op_state; +} + +static struct ocfs2_live_connection *ocfs2_connection_find(const char *name) +{ + size_t len = strlen(name); + struct ocfs2_live_connection *c; + + BUG_ON(!mutex_is_locked(&ocfs2_control_lock)); + + list_for_each_entry(c, &ocfs2_live_connection_list, oc_list) { + if ((c->oc_conn->cc_namelen == len) && + !strncmp(c->oc_conn->cc_name, name, len)) + return c; + } + + return NULL; +} + +/* + * ocfs2_live_connection structures are created underneath the ocfs2 + * mount path. Since the VFS prevents multiple calls to + * fill_super(), we can't get dupes here. + */ +static int ocfs2_live_connection_attach(struct ocfs2_cluster_connection *conn, + struct ocfs2_live_connection *c) +{ + int rc = 0; + + mutex_lock(&ocfs2_control_lock); + c->oc_conn = conn; + + if ((c->oc_type == NO_CONTROLD) || atomic_read(&ocfs2_control_opened)) + list_add(&c->oc_list, &ocfs2_live_connection_list); + else { + printk(KERN_ERR + "ocfs2: Userspace control daemon is not present\n"); + rc = -ESRCH; + } + + mutex_unlock(&ocfs2_control_lock); + return rc; +} + +/* + * This function disconnects the cluster connection from ocfs2_control. + * Afterwards, userspace can't affect the cluster connection. + */ +static void ocfs2_live_connection_drop(struct ocfs2_live_connection *c) +{ + mutex_lock(&ocfs2_control_lock); + list_del_init(&c->oc_list); + c->oc_conn = NULL; + mutex_unlock(&ocfs2_control_lock); + + kfree(c); +} + +static int ocfs2_control_cfu(void *target, size_t target_len, + const char __user *buf, size_t count) +{ + /* The T01 expects write(2) calls to have exactly one command */ + if ((count != target_len) || + (count > sizeof(union ocfs2_control_message))) + return -EINVAL; + + if (copy_from_user(target, buf, target_len)) + return -EFAULT; + + return 0; +} + +static ssize_t ocfs2_control_validate_protocol(struct file *file, + const char __user *buf, + size_t count) +{ + ssize_t ret; + char kbuf[OCFS2_CONTROL_PROTO_LEN]; + + ret = ocfs2_control_cfu(kbuf, OCFS2_CONTROL_PROTO_LEN, + buf, count); + if (ret) + return ret; + + if (strncmp(kbuf, OCFS2_CONTROL_PROTO, OCFS2_CONTROL_PROTO_LEN)) + return -EINVAL; + + ocfs2_control_set_handshake_state(file, + OCFS2_CONTROL_HANDSHAKE_PROTOCOL); + + return count; +} + +static void ocfs2_control_send_down(const char *uuid, + int nodenum) +{ + struct ocfs2_live_connection *c; + + mutex_lock(&ocfs2_control_lock); + + c = ocfs2_connection_find(uuid); + if (c) { + BUG_ON(c->oc_conn == NULL); + c->oc_conn->cc_recovery_handler(nodenum, + c->oc_conn->cc_recovery_data); + } + + mutex_unlock(&ocfs2_control_lock); +} + +/* + * Called whenever configuration elements are sent to /dev/ocfs2_control. + * If all configuration elements are present, try to set the global + * values. If there is a problem, return an error. Skip any missing + * elements, and only bump ocfs2_control_opened when we have all elements + * and are successful. + */ +static int ocfs2_control_install_private(struct file *file) +{ + int rc = 0; + int set_p = 1; + struct ocfs2_control_private *p = file->private_data; + + BUG_ON(p->op_state != OCFS2_CONTROL_HANDSHAKE_PROTOCOL); + + mutex_lock(&ocfs2_control_lock); + + if (p->op_this_node < 0) { + set_p = 0; + } else if ((ocfs2_control_this_node >= 0) && + (ocfs2_control_this_node != p->op_this_node)) { + rc = -EINVAL; + goto out_unlock; + } + + if (!p->op_proto.pv_major) { + set_p = 0; + } else if (!list_empty(&ocfs2_live_connection_list) && + ((running_proto.pv_major != p->op_proto.pv_major) || + (running_proto.pv_minor != p->op_proto.pv_minor))) { + rc = -EINVAL; + goto out_unlock; + } + + if (set_p) { + ocfs2_control_this_node = p->op_this_node; + running_proto.pv_major = p->op_proto.pv_major; + running_proto.pv_minor = p->op_proto.pv_minor; + atomic_inc(&ocfs2_control_opened); + ocfs2_control_set_handshake_state(file, + OCFS2_CONTROL_HANDSHAKE_VALID); + } + +out_unlock: + mutex_unlock(&ocfs2_control_lock); + + return rc; +} + +static int ocfs2_control_get_this_node(void) +{ + int rc; + + mutex_lock(&ocfs2_control_lock); + if (ocfs2_control_this_node < 0) + rc = -EINVAL; + else + rc = ocfs2_control_this_node; + mutex_unlock(&ocfs2_control_lock); + + return rc; +} + +static int ocfs2_control_do_setnode_msg(struct file *file, + struct ocfs2_control_message_setn *msg) +{ + long nodenum; + struct ocfs2_control_private *p = file->private_data; + + if (ocfs2_control_get_handshake_state(file) != + OCFS2_CONTROL_HANDSHAKE_PROTOCOL) + return -EINVAL; + + if (strncmp(msg->tag, OCFS2_CONTROL_MESSAGE_SETNODE_OP, + OCFS2_CONTROL_MESSAGE_OP_LEN)) + return -EINVAL; + + if ((msg->space != ' ') || (msg->newline != '\n')) + return -EINVAL; + msg->space = msg->newline = '\0'; + + if (kstrtol(msg->nodestr, 16, &nodenum)) + return -EINVAL; + + if ((nodenum == LONG_MIN) || (nodenum == LONG_MAX) || + (nodenum > INT_MAX) || (nodenum < 0)) + return -ERANGE; + p->op_this_node = nodenum; + + return ocfs2_control_install_private(file); +} + +static int ocfs2_control_do_setversion_msg(struct file *file, + struct ocfs2_control_message_setv *msg) +{ + long major, minor; + struct ocfs2_control_private *p = file->private_data; + struct ocfs2_protocol_version *max = + &ocfs2_user_plugin.sp_max_proto; + + if (ocfs2_control_get_handshake_state(file) != + OCFS2_CONTROL_HANDSHAKE_PROTOCOL) + return -EINVAL; + + if (strncmp(msg->tag, OCFS2_CONTROL_MESSAGE_SETVERSION_OP, + OCFS2_CONTROL_MESSAGE_OP_LEN)) + return -EINVAL; + + if ((msg->space1 != ' ') || (msg->space2 != ' ') || + (msg->newline != '\n')) + return -EINVAL; + msg->space1 = msg->space2 = msg->newline = '\0'; + + if (kstrtol(msg->major, 16, &major)) + return -EINVAL; + if (kstrtol(msg->minor, 16, &minor)) + return -EINVAL; + + /* + * The major must be between 1 and 255, inclusive. The minor + * must be between 0 and 255, inclusive. The version passed in + * must be within the maximum version supported by the filesystem. + */ + if ((major == LONG_MIN) || (major == LONG_MAX) || + (major > (u8)-1) || (major < 1)) + return -ERANGE; + if ((minor == LONG_MIN) || (minor == LONG_MAX) || + (minor > (u8)-1) || (minor < 0)) + return -ERANGE; + if ((major != max->pv_major) || + (minor > max->pv_minor)) + return -EINVAL; + + p->op_proto.pv_major = major; + p->op_proto.pv_minor = minor; + + return ocfs2_control_install_private(file); +} + +static int ocfs2_control_do_down_msg(struct file *file, + struct ocfs2_control_message_down *msg) +{ + long nodenum; + + if (ocfs2_control_get_handshake_state(file) != + OCFS2_CONTROL_HANDSHAKE_VALID) + return -EINVAL; + + if (strncmp(msg->tag, OCFS2_CONTROL_MESSAGE_DOWN_OP, + OCFS2_CONTROL_MESSAGE_OP_LEN)) + return -EINVAL; + + if ((msg->space1 != ' ') || (msg->space2 != ' ') || + (msg->newline != '\n')) + return -EINVAL; + msg->space1 = msg->space2 = msg->newline = '\0'; + + if (kstrtol(msg->nodestr, 16, &nodenum)) + return -EINVAL; + + if ((nodenum == LONG_MIN) || (nodenum == LONG_MAX) || + (nodenum > INT_MAX) || (nodenum < 0)) + return -ERANGE; + + ocfs2_control_send_down(msg->uuid, nodenum); + + return 0; +} + +static ssize_t ocfs2_control_message(struct file *file, + const char __user *buf, + size_t count) +{ + ssize_t ret; + union ocfs2_control_message msg; + + /* Try to catch padding issues */ + WARN_ON(offsetof(struct ocfs2_control_message_down, uuid) != + (sizeof(msg.u_down.tag) + sizeof(msg.u_down.space1))); + + memset(&msg, 0, sizeof(union ocfs2_control_message)); + ret = ocfs2_control_cfu(&msg, count, buf, count); + if (ret) + goto out; + + if ((count == OCFS2_CONTROL_MESSAGE_SETNODE_TOTAL_LEN) && + !strncmp(msg.tag, OCFS2_CONTROL_MESSAGE_SETNODE_OP, + OCFS2_CONTROL_MESSAGE_OP_LEN)) + ret = ocfs2_control_do_setnode_msg(file, &msg.u_setn); + else if ((count == OCFS2_CONTROL_MESSAGE_SETVERSION_TOTAL_LEN) && + !strncmp(msg.tag, OCFS2_CONTROL_MESSAGE_SETVERSION_OP, + OCFS2_CONTROL_MESSAGE_OP_LEN)) + ret = ocfs2_control_do_setversion_msg(file, &msg.u_setv); + else if ((count == OCFS2_CONTROL_MESSAGE_DOWN_TOTAL_LEN) && + !strncmp(msg.tag, OCFS2_CONTROL_MESSAGE_DOWN_OP, + OCFS2_CONTROL_MESSAGE_OP_LEN)) + ret = ocfs2_control_do_down_msg(file, &msg.u_down); + else + ret = -EINVAL; + +out: + return ret ? ret : count; +} + +static ssize_t ocfs2_control_write(struct file *file, + const char __user *buf, + size_t count, + loff_t *ppos) +{ + ssize_t ret; + + switch (ocfs2_control_get_handshake_state(file)) { + case OCFS2_CONTROL_HANDSHAKE_INVALID: + ret = -EINVAL; + break; + + case OCFS2_CONTROL_HANDSHAKE_READ: + ret = ocfs2_control_validate_protocol(file, buf, + count); + break; + + case OCFS2_CONTROL_HANDSHAKE_PROTOCOL: + case OCFS2_CONTROL_HANDSHAKE_VALID: + ret = ocfs2_control_message(file, buf, count); + break; + + default: + BUG(); + ret = -EIO; + break; + } + + return ret; +} + +/* + * This is a naive version. If we ever have a new protocol, we'll expand + * it. Probably using seq_file. + */ +static ssize_t ocfs2_control_read(struct file *file, + char __user *buf, + size_t count, + loff_t *ppos) +{ + ssize_t ret; + + ret = simple_read_from_buffer(buf, count, ppos, + OCFS2_CONTROL_PROTO, OCFS2_CONTROL_PROTO_LEN); + + /* Have we read the whole protocol list? */ + if (ret > 0 && *ppos >= OCFS2_CONTROL_PROTO_LEN) + ocfs2_control_set_handshake_state(file, + OCFS2_CONTROL_HANDSHAKE_READ); + + return ret; +} + +static int ocfs2_control_release(struct inode *inode, struct file *file) +{ + struct ocfs2_control_private *p = file->private_data; + + mutex_lock(&ocfs2_control_lock); + + if (ocfs2_control_get_handshake_state(file) != + OCFS2_CONTROL_HANDSHAKE_VALID) + goto out; + + if (atomic_dec_and_test(&ocfs2_control_opened)) { + if (!list_empty(&ocfs2_live_connection_list)) { + /* XXX: Do bad things! */ + printk(KERN_ERR + "ocfs2: Unexpected release of ocfs2_control!\n" + " Loss of cluster connection requires " + "an emergency restart!\n"); + emergency_restart(); + } + /* + * Last valid close clears the node number and resets + * the locking protocol version + */ + ocfs2_control_this_node = -1; + running_proto.pv_major = 0; + running_proto.pv_minor = 0; + } + +out: + list_del_init(&p->op_list); + file->private_data = NULL; + + mutex_unlock(&ocfs2_control_lock); + + kfree(p); + + return 0; +} + +static int ocfs2_control_open(struct inode *inode, struct file *file) +{ + struct ocfs2_control_private *p; + + p = kzalloc_obj(struct ocfs2_control_private); + if (!p) + return -ENOMEM; + p->op_this_node = -1; + + mutex_lock(&ocfs2_control_lock); + file->private_data = p; + list_add(&p->op_list, &ocfs2_control_private_list); + mutex_unlock(&ocfs2_control_lock); + + return 0; +} + +static const struct file_operations ocfs2_control_fops = { + .open = ocfs2_control_open, + .release = ocfs2_control_release, + .read = ocfs2_control_read, + .write = ocfs2_control_write, + .owner = THIS_MODULE, + .llseek = default_llseek, +}; + +static struct miscdevice ocfs2_control_device = { + .minor = MISC_DYNAMIC_MINOR, + .name = "ocfs2_control", + .fops = &ocfs2_control_fops, +}; + +static int ocfs2_control_init(void) +{ + int rc; + + atomic_set(&ocfs2_control_opened, 0); + + rc = misc_register(&ocfs2_control_device); + if (rc) + printk(KERN_ERR + "ocfs2: Unable to register ocfs2_control device " + "(errno %d)\n", + -rc); + + return rc; +} + +static void ocfs2_control_exit(void) +{ + misc_deregister(&ocfs2_control_device); +} + +static void fsdlm_lock_ast_wrapper(void *astarg) +{ + struct ocfs2_dlm_lksb *lksb = astarg; + int status = lksb->lksb_fsdlm.sb_status; + + /* + * For now we're punting on the issue of other non-standard errors + * where we can't tell if the unlock_ast or lock_ast should be called. + * The main "other error" that's possible is EINVAL which means the + * function was called with invalid args, which shouldn't be possible + * since the caller here is under our control. Other non-standard + * errors probably fall into the same category, or otherwise are fatal + * which means we can't carry on anyway. + */ + + if (status == -DLM_EUNLOCK || status == -DLM_ECANCEL) + lksb->lksb_conn->cc_proto->lp_unlock_ast(lksb, 0); + else + lksb->lksb_conn->cc_proto->lp_lock_ast(lksb); +} + +static void fsdlm_blocking_ast_wrapper(void *astarg, int level) +{ + struct ocfs2_dlm_lksb *lksb = astarg; + + lksb->lksb_conn->cc_proto->lp_blocking_ast(lksb, level); +} + +static int user_dlm_lock(struct ocfs2_cluster_connection *conn, + int mode, + struct ocfs2_dlm_lksb *lksb, + u32 flags, + void *name, + unsigned int namelen) +{ + if (!lksb->lksb_fsdlm.sb_lvbptr) + lksb->lksb_fsdlm.sb_lvbptr = (char *)lksb + + sizeof(struct dlm_lksb); + + return dlm_lock(conn->cc_lockspace, mode, &lksb->lksb_fsdlm, + flags|DLM_LKF_NODLCKWT, name, namelen, 0, + fsdlm_lock_ast_wrapper, lksb, + fsdlm_blocking_ast_wrapper); +} + +static int user_dlm_unlock(struct ocfs2_cluster_connection *conn, + struct ocfs2_dlm_lksb *lksb, + u32 flags) +{ + return dlm_unlock(conn->cc_lockspace, lksb->lksb_fsdlm.sb_lkid, + flags, &lksb->lksb_fsdlm, lksb); +} + +static int user_dlm_lock_status(struct ocfs2_dlm_lksb *lksb) +{ + return lksb->lksb_fsdlm.sb_status; +} + +static int user_dlm_lvb_valid(struct ocfs2_dlm_lksb *lksb) +{ + int invalid = lksb->lksb_fsdlm.sb_flags & DLM_SBF_VALNOTVALID; + + return !invalid; +} + +static void *user_dlm_lvb(struct ocfs2_dlm_lksb *lksb) +{ + if (!lksb->lksb_fsdlm.sb_lvbptr) + lksb->lksb_fsdlm.sb_lvbptr = (char *)lksb + + sizeof(struct dlm_lksb); + return (void *)(lksb->lksb_fsdlm.sb_lvbptr); +} + +static void user_dlm_dump_lksb(struct ocfs2_dlm_lksb *lksb) +{ +} + +static int user_plock(struct ocfs2_cluster_connection *conn, + u64 ino, + struct file *file, + int cmd, + struct file_lock *fl) +{ + /* + * This more or less just demuxes the plock request into any + * one of three dlm calls. + * + * Internally, fs/dlm will pass these to a misc device, which + * a userspace daemon will read and write to. + */ + + if (cmd == F_CANCELLK) + return dlm_posix_cancel(conn->cc_lockspace, ino, file, fl); + else if (IS_GETLK(cmd)) + return dlm_posix_get(conn->cc_lockspace, ino, file, fl); + else if (lock_is_unlock(fl)) + return dlm_posix_unlock(conn->cc_lockspace, ino, file, fl); + else + return dlm_posix_lock(conn->cc_lockspace, ino, file, cmd, fl); +} + +/* + * Compare a requested locking protocol version against the current one. + * + * If the major numbers are different, they are incompatible. + * If the current minor is greater than the request, they are incompatible. + * If the current minor is less than or equal to the request, they are + * compatible, and the requester should run at the current minor version. + */ +static int fs_protocol_compare(struct ocfs2_protocol_version *existing, + struct ocfs2_protocol_version *request) +{ + if (existing->pv_major != request->pv_major) + return 1; + + if (existing->pv_minor > request->pv_minor) + return 1; + + if (existing->pv_minor < request->pv_minor) + request->pv_minor = existing->pv_minor; + + return 0; +} + +static void lvb_to_version(char *lvb, struct ocfs2_protocol_version *ver) +{ + struct ocfs2_protocol_version *pv = + (struct ocfs2_protocol_version *)lvb; + /* + * ocfs2_protocol_version has two u8 variables, so we don't + * need any endian conversion. + */ + ver->pv_major = pv->pv_major; + ver->pv_minor = pv->pv_minor; +} + +static void version_to_lvb(struct ocfs2_protocol_version *ver, char *lvb) +{ + struct ocfs2_protocol_version *pv = + (struct ocfs2_protocol_version *)lvb; + /* + * ocfs2_protocol_version has two u8 variables, so we don't + * need any endian conversion. + */ + pv->pv_major = ver->pv_major; + pv->pv_minor = ver->pv_minor; +} + +static void sync_wait_cb(void *arg) +{ + struct ocfs2_cluster_connection *conn = arg; + struct ocfs2_live_connection *lc = conn->cc_private; + complete(&lc->oc_sync_wait); +} + +static int sync_unlock(struct ocfs2_cluster_connection *conn, + struct dlm_lksb *lksb, char *name) +{ + int error; + struct ocfs2_live_connection *lc = conn->cc_private; + + error = dlm_unlock(conn->cc_lockspace, lksb->sb_lkid, 0, lksb, conn); + if (error) { + printk(KERN_ERR "%s lkid %x error %d\n", + name, lksb->sb_lkid, error); + return error; + } + + wait_for_completion(&lc->oc_sync_wait); + + if (lksb->sb_status != -DLM_EUNLOCK) { + printk(KERN_ERR "%s lkid %x status %d\n", + name, lksb->sb_lkid, lksb->sb_status); + return -1; + } + return 0; +} + +static int sync_lock(struct ocfs2_cluster_connection *conn, + int mode, uint32_t flags, + struct dlm_lksb *lksb, char *name) +{ + int error, status; + struct ocfs2_live_connection *lc = conn->cc_private; + + error = dlm_lock(conn->cc_lockspace, mode, lksb, flags, + name, strlen(name), + 0, sync_wait_cb, conn, NULL); + if (error) { + printk(KERN_ERR "%s lkid %x flags %x mode %d error %d\n", + name, lksb->sb_lkid, flags, mode, error); + return error; + } + + wait_for_completion(&lc->oc_sync_wait); + + status = lksb->sb_status; + + if (status && status != -EAGAIN) { + printk(KERN_ERR "%s lkid %x flags %x mode %d status %d\n", + name, lksb->sb_lkid, flags, mode, status); + } + + return status; +} + + +static int version_lock(struct ocfs2_cluster_connection *conn, int mode, + int flags) +{ + struct ocfs2_live_connection *lc = conn->cc_private; + return sync_lock(conn, mode, flags, + &lc->oc_version_lksb, VERSION_LOCK); +} + +static int version_unlock(struct ocfs2_cluster_connection *conn) +{ + struct ocfs2_live_connection *lc = conn->cc_private; + return sync_unlock(conn, &lc->oc_version_lksb, VERSION_LOCK); +} + +/* get_protocol_version() + * + * To exchange ocfs2 versioning, we use the LVB of the version dlm lock. + * The algorithm is: + * 1. Attempt to take the lock in EX mode (non-blocking). + * 2. If successful (which means it is the first mount), write the + * version number and downconvert to PR lock. + * 3. If unsuccessful (returns -EAGAIN), read the version from the LVB after + * taking the PR lock. + */ + +static int get_protocol_version(struct ocfs2_cluster_connection *conn) +{ + int ret; + struct ocfs2_live_connection *lc = conn->cc_private; + struct ocfs2_protocol_version pv; + + running_proto.pv_major = + ocfs2_user_plugin.sp_max_proto.pv_major; + running_proto.pv_minor = + ocfs2_user_plugin.sp_max_proto.pv_minor; + + lc->oc_version_lksb.sb_lvbptr = lc->oc_lvb; + ret = version_lock(conn, DLM_LOCK_EX, + DLM_LKF_VALBLK|DLM_LKF_NOQUEUE); + if (!ret) { + conn->cc_version.pv_major = running_proto.pv_major; + conn->cc_version.pv_minor = running_proto.pv_minor; + version_to_lvb(&running_proto, lc->oc_lvb); + version_lock(conn, DLM_LOCK_PR, DLM_LKF_CONVERT|DLM_LKF_VALBLK); + } else if (ret == -EAGAIN) { + ret = version_lock(conn, DLM_LOCK_PR, DLM_LKF_VALBLK); + if (ret) + goto out; + lvb_to_version(lc->oc_lvb, &pv); + + if ((pv.pv_major != running_proto.pv_major) || + (pv.pv_minor > running_proto.pv_minor)) { + ret = -EINVAL; + goto out; + } + + conn->cc_version.pv_major = pv.pv_major; + conn->cc_version.pv_minor = pv.pv_minor; + } +out: + return ret; +} + +static void user_recover_prep(void *arg) +{ +} + +static void user_recover_slot(void *arg, struct dlm_slot *slot) +{ + struct ocfs2_cluster_connection *conn = arg; + printk(KERN_INFO "ocfs2: Node %d/%d down. Initiating recovery.\n", + slot->nodeid, slot->slot); + conn->cc_recovery_handler(slot->nodeid, conn->cc_recovery_data); + +} + +static void user_recover_done(void *arg, struct dlm_slot *slots, + int num_slots, int our_slot, + uint32_t generation) +{ + struct ocfs2_cluster_connection *conn = arg; + struct ocfs2_live_connection *lc = conn->cc_private; + int i; + + for (i = 0; i < num_slots; i++) + if (slots[i].slot == our_slot) { + atomic_set(&lc->oc_this_node, slots[i].nodeid); + break; + } + + lc->oc_our_slot = our_slot; + wake_up(&lc->oc_wait); +} + +static const struct dlm_lockspace_ops ocfs2_ls_ops = { + .recover_prep = user_recover_prep, + .recover_slot = user_recover_slot, + .recover_done = user_recover_done, +}; + +static int user_cluster_disconnect(struct ocfs2_cluster_connection *conn) +{ + version_unlock(conn); + dlm_release_lockspace(conn->cc_lockspace, DLM_RELEASE_NORMAL); + conn->cc_lockspace = NULL; + ocfs2_live_connection_drop(conn->cc_private); + conn->cc_private = NULL; + return 0; +} + +static int user_cluster_connect(struct ocfs2_cluster_connection *conn) +{ + dlm_lockspace_t *fsdlm; + struct ocfs2_live_connection *lc; + int rc, ops_rv; + + BUG_ON(conn == NULL); + + lc = kzalloc_obj(struct ocfs2_live_connection); + if (!lc) + return -ENOMEM; + + init_waitqueue_head(&lc->oc_wait); + init_completion(&lc->oc_sync_wait); + atomic_set(&lc->oc_this_node, 0); + conn->cc_private = lc; + lc->oc_type = NO_CONTROLD; + + rc = dlm_new_lockspace(conn->cc_name, conn->cc_cluster_name, + DLM_LSFL_NEWEXCL, DLM_LVB_LEN, + &ocfs2_ls_ops, conn, &ops_rv, &fsdlm); + if (rc) { + if (rc == -EEXIST || rc == -EPROTO) + printk(KERN_ERR "ocfs2: Unable to create the " + "lockspace %s (%d), because a ocfs2-tools " + "program is running on this file system " + "with the same name lockspace\n", + conn->cc_name, rc); + goto out; + } + + if (ops_rv == -EOPNOTSUPP) { + lc->oc_type = WITH_CONTROLD; + printk(KERN_NOTICE "ocfs2: You seem to be using an older " + "version of dlm_controld and/or ocfs2-tools." + " Please consider upgrading.\n"); + } else if (ops_rv) { + rc = ops_rv; + goto out; + } + conn->cc_lockspace = fsdlm; + + rc = ocfs2_live_connection_attach(conn, lc); + if (rc) + goto out; + + if (lc->oc_type == NO_CONTROLD) { + rc = get_protocol_version(conn); + if (rc) { + printk(KERN_ERR "ocfs2: Could not determine" + " locking version\n"); + user_cluster_disconnect(conn); + lc = NULL; + goto out; + } + wait_event(lc->oc_wait, (atomic_read(&lc->oc_this_node) > 0)); + } + + /* + * running_proto must have been set before we allowed any mounts + * to proceed. + */ + if (fs_protocol_compare(&running_proto, &conn->cc_version)) { + printk(KERN_ERR + "Unable to mount with fs locking protocol version " + "%u.%u because negotiated protocol is %u.%u\n", + conn->cc_version.pv_major, conn->cc_version.pv_minor, + running_proto.pv_major, running_proto.pv_minor); + rc = -EPROTO; + ocfs2_live_connection_drop(lc); + lc = NULL; + } + +out: + if (rc) + kfree(lc); + return rc; +} + + +static int user_cluster_this_node(struct ocfs2_cluster_connection *conn, + unsigned int *this_node) +{ + int rc; + struct ocfs2_live_connection *lc = conn->cc_private; + + if (lc->oc_type == WITH_CONTROLD) + rc = ocfs2_control_get_this_node(); + else if (lc->oc_type == NO_CONTROLD) + rc = atomic_read(&lc->oc_this_node); + else + rc = -EINVAL; + + if (rc < 0) + return rc; + + *this_node = rc; + return 0; +} + +static const struct ocfs2_stack_operations ocfs2_user_plugin_ops = { + .connect = user_cluster_connect, + .disconnect = user_cluster_disconnect, + .this_node = user_cluster_this_node, + .dlm_lock = user_dlm_lock, + .dlm_unlock = user_dlm_unlock, + .lock_status = user_dlm_lock_status, + .lvb_valid = user_dlm_lvb_valid, + .lock_lvb = user_dlm_lvb, + .plock = user_plock, + .dump_lksb = user_dlm_dump_lksb, +}; + +static struct ocfs2_stack_plugin ocfs2_user_plugin = { + .sp_name = "user", + .sp_ops = &ocfs2_user_plugin_ops, + .sp_owner = THIS_MODULE, +}; + + +static int __init ocfs2_user_plugin_init(void) +{ + int rc; + + rc = ocfs2_control_init(); + if (!rc) { + rc = ocfs2_stack_glue_register(&ocfs2_user_plugin); + if (rc) + ocfs2_control_exit(); + } + + return rc; +} + +static void __exit ocfs2_user_plugin_exit(void) +{ + ocfs2_stack_glue_unregister(&ocfs2_user_plugin); + ocfs2_control_exit(); +} + +MODULE_AUTHOR("Oracle"); +MODULE_DESCRIPTION("ocfs2 driver for userspace cluster stacks"); +MODULE_LICENSE("GPL"); +module_init(ocfs2_user_plugin_init); +module_exit(ocfs2_user_plugin_exit); diff --git a/fs/ocfs2/stackglue.c b/fs/ocfs2/stackglue.c new file mode 100644 index 000000000..741d6191d --- /dev/null +++ b/fs/ocfs2/stackglue.c @@ -0,0 +1,701 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * stackglue.c + * + * Code which implements an OCFS2 specific interface to underlying + * cluster stacks. + * + * Copyright (C) 2007, 2009 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ocfs2_fs.h" + +#include "stackglue.h" + +#define OCFS2_STACK_PLUGIN_O2CB "o2cb" +#define OCFS2_STACK_PLUGIN_USER "user" +#define OCFS2_MAX_HB_CTL_PATH 256 + +static struct ocfs2_protocol_version locking_max_version; +static DEFINE_SPINLOCK(ocfs2_stack_lock); +static LIST_HEAD(ocfs2_stack_list); +static char cluster_stack_name[OCFS2_STACK_LABEL_LEN + 1]; +static char ocfs2_hb_ctl_path[OCFS2_MAX_HB_CTL_PATH] = "/sbin/ocfs2_hb_ctl"; + +/* + * The stack currently in use. If not null, active_stack->sp_count > 0, + * the module is pinned, and the locking protocol cannot be changed. + */ +static struct ocfs2_stack_plugin *active_stack; + +static struct ocfs2_stack_plugin *ocfs2_stack_lookup(const char *name) +{ + struct ocfs2_stack_plugin *p; + + assert_spin_locked(&ocfs2_stack_lock); + + list_for_each_entry(p, &ocfs2_stack_list, sp_list) { + if (!strcmp(p->sp_name, name)) + return p; + } + + return NULL; +} + +static int ocfs2_stack_driver_request(const char *stack_name, + const char *plugin_name) +{ + int rc; + struct ocfs2_stack_plugin *p; + + spin_lock(&ocfs2_stack_lock); + + /* + * If the stack passed by the filesystem isn't the selected one, + * we can't continue. + */ + if (strcmp(stack_name, cluster_stack_name)) { + rc = -EBUSY; + goto out; + } + + if (active_stack) { + /* + * If the active stack isn't the one we want, it cannot + * be selected right now. + */ + if (!strcmp(active_stack->sp_name, plugin_name)) + rc = 0; + else + rc = -EBUSY; + goto out; + } + + p = ocfs2_stack_lookup(plugin_name); + if (!p || !try_module_get(p->sp_owner)) { + rc = -ENOENT; + goto out; + } + + active_stack = p; + rc = 0; + +out: + /* If we found it, pin it */ + if (!rc) + active_stack->sp_count++; + + spin_unlock(&ocfs2_stack_lock); + return rc; +} + +/* + * This function looks up the appropriate stack and makes it active. If + * there is no stack, it tries to load it. It will fail if the stack still + * cannot be found. It will also fail if a different stack is in use. + */ +static int ocfs2_stack_driver_get(const char *stack_name) +{ + int rc; + char *plugin_name = OCFS2_STACK_PLUGIN_O2CB; + + /* + * Classic stack does not pass in a stack name. This is + * compatible with older tools as well. + */ + if (!stack_name || !*stack_name) + stack_name = OCFS2_STACK_PLUGIN_O2CB; + + if (strlen(stack_name) != OCFS2_STACK_LABEL_LEN) { + printk(KERN_ERR + "ocfs2 passed an invalid cluster stack label: \"%s\"\n", + stack_name); + return -EINVAL; + } + + /* Anything that isn't the classic stack is a user stack */ + if (strcmp(stack_name, OCFS2_STACK_PLUGIN_O2CB)) + plugin_name = OCFS2_STACK_PLUGIN_USER; + + rc = ocfs2_stack_driver_request(stack_name, plugin_name); + if (rc == -ENOENT) { + request_module("ocfs2_stack_%s", plugin_name); + rc = ocfs2_stack_driver_request(stack_name, plugin_name); + } + + if (rc == -ENOENT) { + printk(KERN_ERR + "ocfs2: Cluster stack driver \"%s\" cannot be found\n", + plugin_name); + } else if (rc == -EBUSY) { + printk(KERN_ERR + "ocfs2: A different cluster stack is in use\n"); + } + + return rc; +} + +static void ocfs2_stack_driver_put(void) +{ + spin_lock(&ocfs2_stack_lock); + BUG_ON(active_stack == NULL); + BUG_ON(active_stack->sp_count == 0); + + active_stack->sp_count--; + if (!active_stack->sp_count) { + module_put(active_stack->sp_owner); + active_stack = NULL; + } + spin_unlock(&ocfs2_stack_lock); +} + +int ocfs2_stack_glue_register(struct ocfs2_stack_plugin *plugin) +{ + int rc; + + spin_lock(&ocfs2_stack_lock); + if (!ocfs2_stack_lookup(plugin->sp_name)) { + plugin->sp_count = 0; + plugin->sp_max_proto = locking_max_version; + list_add(&plugin->sp_list, &ocfs2_stack_list); + printk(KERN_INFO "ocfs2: Registered cluster interface %s\n", + plugin->sp_name); + rc = 0; + } else { + printk(KERN_ERR "ocfs2: Stack \"%s\" already registered\n", + plugin->sp_name); + rc = -EEXIST; + } + spin_unlock(&ocfs2_stack_lock); + + return rc; +} +EXPORT_SYMBOL_GPL(ocfs2_stack_glue_register); + +void ocfs2_stack_glue_unregister(struct ocfs2_stack_plugin *plugin) +{ + struct ocfs2_stack_plugin *p; + + spin_lock(&ocfs2_stack_lock); + p = ocfs2_stack_lookup(plugin->sp_name); + if (p) { + BUG_ON(p != plugin); + BUG_ON(plugin == active_stack); + BUG_ON(plugin->sp_count != 0); + list_del_init(&plugin->sp_list); + printk(KERN_INFO "ocfs2: Unregistered cluster interface %s\n", + plugin->sp_name); + } else { + printk(KERN_ERR "Stack \"%s\" is not registered\n", + plugin->sp_name); + } + spin_unlock(&ocfs2_stack_lock); +} +EXPORT_SYMBOL_GPL(ocfs2_stack_glue_unregister); + +void ocfs2_stack_glue_set_max_proto_version(struct ocfs2_protocol_version *max_proto) +{ + struct ocfs2_stack_plugin *p; + + spin_lock(&ocfs2_stack_lock); + if (memcmp(max_proto, &locking_max_version, + sizeof(struct ocfs2_protocol_version))) { + BUG_ON(locking_max_version.pv_major != 0); + + locking_max_version = *max_proto; + list_for_each_entry(p, &ocfs2_stack_list, sp_list) { + p->sp_max_proto = locking_max_version; + } + } + spin_unlock(&ocfs2_stack_lock); +} +EXPORT_SYMBOL_GPL(ocfs2_stack_glue_set_max_proto_version); + + +/* + * The ocfs2_dlm_lock() and ocfs2_dlm_unlock() functions take no argument + * for the ast and bast functions. They will pass the lksb to the ast + * and bast. The caller can wrap the lksb with their own structure to + * get more information. + */ +int ocfs2_dlm_lock(struct ocfs2_cluster_connection *conn, + int mode, + struct ocfs2_dlm_lksb *lksb, + u32 flags, + void *name, + unsigned int namelen) +{ + if (!lksb->lksb_conn) + lksb->lksb_conn = conn; + else + BUG_ON(lksb->lksb_conn != conn); + return active_stack->sp_ops->dlm_lock(conn, mode, lksb, flags, + name, namelen); +} +EXPORT_SYMBOL_GPL(ocfs2_dlm_lock); + +int ocfs2_dlm_unlock(struct ocfs2_cluster_connection *conn, + struct ocfs2_dlm_lksb *lksb, + u32 flags) +{ + BUG_ON(lksb->lksb_conn == NULL); + + return active_stack->sp_ops->dlm_unlock(conn, lksb, flags); +} +EXPORT_SYMBOL_GPL(ocfs2_dlm_unlock); + +int ocfs2_dlm_lock_status(struct ocfs2_dlm_lksb *lksb) +{ + return active_stack->sp_ops->lock_status(lksb); +} +EXPORT_SYMBOL_GPL(ocfs2_dlm_lock_status); + +int ocfs2_dlm_lvb_valid(struct ocfs2_dlm_lksb *lksb) +{ + return active_stack->sp_ops->lvb_valid(lksb); +} +EXPORT_SYMBOL_GPL(ocfs2_dlm_lvb_valid); + +void *ocfs2_dlm_lvb(struct ocfs2_dlm_lksb *lksb) +{ + return active_stack->sp_ops->lock_lvb(lksb); +} +EXPORT_SYMBOL_GPL(ocfs2_dlm_lvb); + +void ocfs2_dlm_dump_lksb(struct ocfs2_dlm_lksb *lksb) +{ + active_stack->sp_ops->dump_lksb(lksb); +} +EXPORT_SYMBOL_GPL(ocfs2_dlm_dump_lksb); + +int ocfs2_stack_supports_plocks(void) +{ + return active_stack && active_stack->sp_ops->plock; +} +EXPORT_SYMBOL_GPL(ocfs2_stack_supports_plocks); + +/* + * ocfs2_plock() can only be safely called if + * ocfs2_stack_supports_plocks() returned true + */ +int ocfs2_plock(struct ocfs2_cluster_connection *conn, u64 ino, + struct file *file, int cmd, struct file_lock *fl) +{ + WARN_ON_ONCE(active_stack->sp_ops->plock == NULL); + if (active_stack->sp_ops->plock) + return active_stack->sp_ops->plock(conn, ino, file, cmd, fl); + return -EOPNOTSUPP; +} +EXPORT_SYMBOL_GPL(ocfs2_plock); + +int ocfs2_cluster_connect(const char *stack_name, + const char *cluster_name, + int cluster_name_len, + const char *group, + int grouplen, + struct ocfs2_locking_protocol *lproto, + void (*recovery_handler)(int node_num, + void *recovery_data), + void *recovery_data, + struct ocfs2_cluster_connection **conn) +{ + int rc = 0; + struct ocfs2_cluster_connection *new_conn; + + BUG_ON(group == NULL); + BUG_ON(conn == NULL); + BUG_ON(recovery_handler == NULL); + + if (grouplen > GROUP_NAME_MAX) { + rc = -EINVAL; + goto out; + } + + if (memcmp(&lproto->lp_max_version, &locking_max_version, + sizeof(struct ocfs2_protocol_version))) { + rc = -EINVAL; + goto out; + } + + new_conn = kzalloc_obj(struct ocfs2_cluster_connection); + if (!new_conn) { + rc = -ENOMEM; + goto out; + } + + strscpy(new_conn->cc_name, group, GROUP_NAME_MAX + 1); + new_conn->cc_namelen = grouplen; + if (cluster_name_len) + strscpy(new_conn->cc_cluster_name, cluster_name, + CLUSTER_NAME_MAX + 1); + new_conn->cc_cluster_name_len = cluster_name_len; + new_conn->cc_recovery_handler = recovery_handler; + new_conn->cc_recovery_data = recovery_data; + + new_conn->cc_proto = lproto; + /* Start the new connection at our maximum compatibility level */ + new_conn->cc_version = lproto->lp_max_version; + + /* This will pin the stack driver if successful */ + rc = ocfs2_stack_driver_get(stack_name); + if (rc) + goto out_free; + + rc = active_stack->sp_ops->connect(new_conn); + if (rc) { + ocfs2_stack_driver_put(); + goto out_free; + } + + *conn = new_conn; + +out_free: + if (rc) + kfree(new_conn); + +out: + return rc; +} +EXPORT_SYMBOL_GPL(ocfs2_cluster_connect); + +/* The caller will ensure all nodes have the same cluster stack */ +int ocfs2_cluster_connect_agnostic(const char *group, + int grouplen, + struct ocfs2_locking_protocol *lproto, + void (*recovery_handler)(int node_num, + void *recovery_data), + void *recovery_data, + struct ocfs2_cluster_connection **conn) +{ + char *stack_name = NULL; + + if (cluster_stack_name[0]) + stack_name = cluster_stack_name; + return ocfs2_cluster_connect(stack_name, NULL, 0, group, grouplen, + lproto, recovery_handler, recovery_data, + conn); +} +EXPORT_SYMBOL_GPL(ocfs2_cluster_connect_agnostic); + +/* If hangup_pending is 0, the stack driver will be dropped */ +int ocfs2_cluster_disconnect(struct ocfs2_cluster_connection *conn, + int hangup_pending) +{ + int ret; + + BUG_ON(conn == NULL); + + ret = active_stack->sp_ops->disconnect(conn); + + /* XXX Should we free it anyway? */ + if (!ret) { + kfree(conn); + if (!hangup_pending) + ocfs2_stack_driver_put(); + } + + return ret; +} +EXPORT_SYMBOL_GPL(ocfs2_cluster_disconnect); + +/* + * Leave the group for this filesystem. This is executed by a userspace + * program (stored in ocfs2_hb_ctl_path). + */ +static void ocfs2_leave_group(const char *group) +{ + int ret; + char *argv[5], *envp[3]; + + argv[0] = ocfs2_hb_ctl_path; + argv[1] = "-K"; + argv[2] = "-u"; + argv[3] = (char *)group; + argv[4] = NULL; + + /* minimal command environment taken from cpu_run_sbin_hotplug */ + envp[0] = "HOME=/"; + envp[1] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin"; + envp[2] = NULL; + + ret = call_usermodehelper(argv[0], argv, envp, UMH_WAIT_PROC); + if (ret < 0) { + printk(KERN_ERR + "ocfs2: Error %d running user helper " + "\"%s %s %s %s\"\n", + ret, argv[0], argv[1], argv[2], argv[3]); + } +} + +/* + * Hangup is a required post-umount. ocfs2-tools software expects the + * filesystem to call "ocfs2_hb_ctl" during unmount. This happens + * regardless of whether the DLM got started, so we can't do it + * in ocfs2_cluster_disconnect(). The ocfs2_leave_group() function does + * the actual work. + */ +void ocfs2_cluster_hangup(const char *group, int grouplen) +{ + BUG_ON(group == NULL); + BUG_ON(group[grouplen] != '\0'); + + ocfs2_leave_group(group); + + /* cluster_disconnect() was called with hangup_pending==1 */ + ocfs2_stack_driver_put(); +} +EXPORT_SYMBOL_GPL(ocfs2_cluster_hangup); + +int ocfs2_cluster_this_node(struct ocfs2_cluster_connection *conn, + unsigned int *node) +{ + return active_stack->sp_ops->this_node(conn, node); +} +EXPORT_SYMBOL_GPL(ocfs2_cluster_this_node); + + +/* + * Sysfs bits + */ + +static ssize_t ocfs2_max_locking_protocol_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + ssize_t ret = 0; + + spin_lock(&ocfs2_stack_lock); + if (locking_max_version.pv_major) + ret = snprintf(buf, PAGE_SIZE, "%u.%u\n", + locking_max_version.pv_major, + locking_max_version.pv_minor); + spin_unlock(&ocfs2_stack_lock); + + return ret; +} + +static struct kobj_attribute ocfs2_attr_max_locking_protocol = + __ATTR(max_locking_protocol, S_IRUGO, + ocfs2_max_locking_protocol_show, NULL); + +static ssize_t ocfs2_loaded_cluster_plugins_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + ssize_t ret = 0, total = 0, remain = PAGE_SIZE; + struct ocfs2_stack_plugin *p; + + spin_lock(&ocfs2_stack_lock); + list_for_each_entry(p, &ocfs2_stack_list, sp_list) { + ret = snprintf(buf, remain, "%s\n", + p->sp_name); + if (ret >= remain) { + /* snprintf() didn't fit */ + total = -E2BIG; + break; + } + total += ret; + remain -= ret; + } + spin_unlock(&ocfs2_stack_lock); + + return total; +} + +static struct kobj_attribute ocfs2_attr_loaded_cluster_plugins = + __ATTR(loaded_cluster_plugins, S_IRUGO, + ocfs2_loaded_cluster_plugins_show, NULL); + +static ssize_t ocfs2_active_cluster_plugin_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + ssize_t ret = 0; + + spin_lock(&ocfs2_stack_lock); + if (active_stack) { + ret = snprintf(buf, PAGE_SIZE, "%s\n", + active_stack->sp_name); + if (ret >= PAGE_SIZE) + ret = -E2BIG; + } + spin_unlock(&ocfs2_stack_lock); + + return ret; +} + +static struct kobj_attribute ocfs2_attr_active_cluster_plugin = + __ATTR(active_cluster_plugin, S_IRUGO, + ocfs2_active_cluster_plugin_show, NULL); + +static ssize_t ocfs2_cluster_stack_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + ssize_t ret; + spin_lock(&ocfs2_stack_lock); + ret = snprintf(buf, PAGE_SIZE, "%s\n", cluster_stack_name); + spin_unlock(&ocfs2_stack_lock); + + return ret; +} + +static ssize_t ocfs2_cluster_stack_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + size_t len = count; + ssize_t ret; + + if (len == 0) + return len; + + if (buf[len - 1] == '\n') + len--; + + if ((len != OCFS2_STACK_LABEL_LEN) || + (strnlen(buf, len) != len)) + return -EINVAL; + + spin_lock(&ocfs2_stack_lock); + if (active_stack) { + if (!strncmp(buf, cluster_stack_name, len)) + ret = count; + else + ret = -EBUSY; + } else { + memcpy(cluster_stack_name, buf, len); + ret = count; + } + spin_unlock(&ocfs2_stack_lock); + + return ret; +} + + +static struct kobj_attribute ocfs2_attr_cluster_stack = + __ATTR(cluster_stack, S_IRUGO | S_IWUSR, + ocfs2_cluster_stack_show, + ocfs2_cluster_stack_store); + + + +static ssize_t ocfs2_dlm_recover_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + return snprintf(buf, PAGE_SIZE, "1\n"); +} + +static struct kobj_attribute ocfs2_attr_dlm_recover_support = + __ATTR(dlm_recover_callback_support, S_IRUGO, + ocfs2_dlm_recover_show, NULL); + +static struct attribute *ocfs2_attrs[] = { + &ocfs2_attr_max_locking_protocol.attr, + &ocfs2_attr_loaded_cluster_plugins.attr, + &ocfs2_attr_active_cluster_plugin.attr, + &ocfs2_attr_cluster_stack.attr, + &ocfs2_attr_dlm_recover_support.attr, + NULL, +}; + +static const struct attribute_group ocfs2_attr_group = { + .attrs = ocfs2_attrs, +}; + +struct kset *ocfs2_kset; +EXPORT_SYMBOL_GPL(ocfs2_kset); + +static void ocfs2_sysfs_exit(void) +{ + kset_unregister(ocfs2_kset); +} + +static int ocfs2_sysfs_init(void) +{ + int ret; + + ocfs2_kset = kset_create_and_add("ocfs2", NULL, fs_kobj); + if (!ocfs2_kset) + return -ENOMEM; + + ret = sysfs_create_group(&ocfs2_kset->kobj, &ocfs2_attr_group); + if (ret) + goto error; + + return 0; + +error: + kset_unregister(ocfs2_kset); + return ret; +} + +/* + * Sysctl bits + * + * The sysctl lives at /proc/sys/fs/ocfs2/nm/hb_ctl_path. The 'nm' doesn't + * make as much sense in a multiple cluster stack world, but it's safer + * and easier to preserve the name. + */ + +static const struct ctl_table ocfs2_nm_table[] = { + { + .procname = "hb_ctl_path", + .data = ocfs2_hb_ctl_path, + .maxlen = OCFS2_MAX_HB_CTL_PATH, + .mode = 0644, + .proc_handler = proc_dostring, + }, +}; + +static struct ctl_table_header *ocfs2_table_header; + +/* + * Initialization + */ + +static int __init ocfs2_stack_glue_init(void) +{ + int ret; + + strscpy(cluster_stack_name, OCFS2_STACK_PLUGIN_O2CB); + + ocfs2_table_header = register_sysctl("fs/ocfs2/nm", ocfs2_nm_table); + if (!ocfs2_table_header) { + printk(KERN_ERR + "ocfs2 stack glue: unable to register sysctl\n"); + return -ENOMEM; /* or something. */ + } + + ret = ocfs2_sysfs_init(); + if (ret) + unregister_sysctl_table(ocfs2_table_header); + + return ret; +} + +static void __exit ocfs2_stack_glue_exit(void) +{ + memset(&locking_max_version, 0, + sizeof(struct ocfs2_protocol_version)); + ocfs2_sysfs_exit(); + unregister_sysctl_table(ocfs2_table_header); +} + +MODULE_AUTHOR("Oracle"); +MODULE_DESCRIPTION("ocfs2 cluster stack glue layer"); +MODULE_LICENSE("GPL"); +module_init(ocfs2_stack_glue_init); +module_exit(ocfs2_stack_glue_exit); diff --git a/fs/ocfs2/stackglue.h b/fs/ocfs2/stackglue.h new file mode 100644 index 000000000..5486a6dce --- /dev/null +++ b/fs/ocfs2/stackglue.h @@ -0,0 +1,293 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * stackglue.h + * + * Glue to the underlying cluster stack. + * + * Copyright (C) 2007 Oracle. All rights reserved. + */ + + +#ifndef STACKGLUE_H +#define STACKGLUE_H + +#include +#include +#include + +#include "dlm/dlmapi.h" +#include + +/* Needed for plock-related prototypes */ +struct file; +struct file_lock; + +/* + * dlmconstants.h does not have a LOCAL flag. We hope to remove it + * some day, but right now we need it. Let's fake it. This value is larger + * than any flag in dlmconstants.h. + */ +#define DLM_LKF_LOCAL 0x00100000 + +/* + * This shadows DLM_LOCKSPACE_LEN in fs/dlm/dlm_internal.h. That probably + * wants to be in a public header. + */ +#define GROUP_NAME_MAX 64 + +/* This shadows OCFS2_CLUSTER_NAME_LEN */ +#define CLUSTER_NAME_MAX 16 + + +/* + * ocfs2_protocol_version changes when ocfs2 does something different in + * its inter-node behavior. See dlmglue.c for more information. + */ +struct ocfs2_protocol_version { + u8 pv_major; + u8 pv_minor; +}; + +/* + * The dlm_lockstatus struct includes lvb space, but the dlm_lksb struct only + * has a pointer to separately allocated lvb space. This struct exists only to + * include in the lksb union to make space for a combined dlm_lksb and lvb. + */ +struct fsdlm_lksb_plus_lvb { + struct dlm_lksb lksb; + char lvb[DLM_LVB_LEN]; +}; + +/* + * A union of all lock status structures. We define it here so that the + * size of the union is known. Lock status structures are embedded in + * ocfs2 inodes. + */ +struct ocfs2_cluster_connection; +struct ocfs2_dlm_lksb { + union { + struct dlm_lockstatus lksb_o2dlm; + struct dlm_lksb lksb_fsdlm; + struct fsdlm_lksb_plus_lvb padding; + }; + struct ocfs2_cluster_connection *lksb_conn; +}; + +/* + * The ocfs2_locking_protocol defines the handlers called on ocfs2's behalf. + */ +struct ocfs2_locking_protocol { + struct ocfs2_protocol_version lp_max_version; + void (*lp_lock_ast)(struct ocfs2_dlm_lksb *lksb); + void (*lp_blocking_ast)(struct ocfs2_dlm_lksb *lksb, int level); + void (*lp_unlock_ast)(struct ocfs2_dlm_lksb *lksb, int error); +}; + + +/* + * A cluster connection. Mostly opaque to ocfs2, the connection holds + * state for the underlying stack. ocfs2 does use cc_version to determine + * locking compatibility. + */ +struct ocfs2_cluster_connection { + char cc_name[GROUP_NAME_MAX + 1]; + int cc_namelen; + char cc_cluster_name[CLUSTER_NAME_MAX + 1]; + int cc_cluster_name_len; + struct ocfs2_protocol_version cc_version; + struct ocfs2_locking_protocol *cc_proto; + void (*cc_recovery_handler)(int node_num, void *recovery_data); + void *cc_recovery_data; + void *cc_lockspace; + void *cc_private; +}; + +/* + * Each cluster stack implements the stack operations structure. Not used + * in the ocfs2 code, the stackglue code translates generic cluster calls + * into stack operations. + */ +struct ocfs2_stack_operations { + /* + * The fs code calls ocfs2_cluster_connect() to attach a new + * filesystem to the cluster stack. The ->connect() op is passed + * an ocfs2_cluster_connection with the name and recovery field + * filled in. + * + * The stack must set up any notification mechanisms and create + * the filesystem lockspace in the DLM. The lockspace should be + * stored on cc_lockspace. Any other information can be stored on + * cc_private. + * + * ->connect() must not return until it is guaranteed that + * + * - Node down notifications for the filesystem will be received + * and passed to conn->cc_recovery_handler(). + * - Locking requests for the filesystem will be processed. + */ + int (*connect)(struct ocfs2_cluster_connection *conn); + + /* + * The fs code calls ocfs2_cluster_disconnect() when a filesystem + * no longer needs cluster services. All DLM locks have been + * dropped, and recovery notification is being ignored by the + * fs code. The stack must disengage from the DLM and discontinue + * recovery notification. + * + * Once ->disconnect() has returned, the connection structure will + * be freed. Thus, a stack must not return from ->disconnect() + * until it will no longer reference the conn pointer. + * + * Once this call returns, the stack glue will be dropping this + * connection's reference on the module. + */ + int (*disconnect)(struct ocfs2_cluster_connection *conn); + + /* + * ->this_node() returns the cluster's unique identifier for the + * local node. + */ + int (*this_node)(struct ocfs2_cluster_connection *conn, + unsigned int *node); + + /* + * Call the underlying dlm lock function. The ->dlm_lock() + * callback should convert the flags and mode as appropriate. + * + * ast and bast functions are not part of the call because the + * stack will likely want to wrap ast and bast calls before passing + * them to stack->sp_proto. There is no astarg. The lksb will + * be passed back to the ast and bast functions. The caller can + * use this to find their object. + */ + int (*dlm_lock)(struct ocfs2_cluster_connection *conn, + int mode, + struct ocfs2_dlm_lksb *lksb, + u32 flags, + void *name, + unsigned int namelen); + + /* + * Call the underlying dlm unlock function. The ->dlm_unlock() + * function should convert the flags as appropriate. + * + * The unlock ast is not passed, as the stack will want to wrap + * it before calling stack->sp_proto->lp_unlock_ast(). There is + * no astarg. The lksb will be passed back to the unlock ast + * function. The caller can use this to find their object. + */ + int (*dlm_unlock)(struct ocfs2_cluster_connection *conn, + struct ocfs2_dlm_lksb *lksb, + u32 flags); + + /* + * Return the status of the current lock status block. The fs + * code should never dereference the union. The ->lock_status() + * callback pulls out the stack-specific lksb, converts the status + * to a proper errno, and returns it. + */ + int (*lock_status)(struct ocfs2_dlm_lksb *lksb); + + /* + * Return non-zero if the LVB is valid. + */ + int (*lvb_valid)(struct ocfs2_dlm_lksb *lksb); + + /* + * Pull the lvb pointer off of the stack-specific lksb. + */ + void *(*lock_lvb)(struct ocfs2_dlm_lksb *lksb); + + /* + * Cluster-aware posix locks + * + * This is NULL for stacks which do not support posix locks. + */ + int (*plock)(struct ocfs2_cluster_connection *conn, + u64 ino, + struct file *file, + int cmd, + struct file_lock *fl); + + /* + * This is an optional debugging hook. If provided, the + * stack can dump debugging information about this lock. + */ + void (*dump_lksb)(struct ocfs2_dlm_lksb *lksb); +}; + +/* + * Each stack plugin must describe itself by registering a + * ocfs2_stack_plugin structure. This is only seen by stackglue and the + * stack driver. + */ +struct ocfs2_stack_plugin { + char *sp_name; + const struct ocfs2_stack_operations *sp_ops; + struct module *sp_owner; + + /* These are managed by the stackglue code. */ + struct list_head sp_list; + unsigned int sp_count; + struct ocfs2_protocol_version sp_max_proto; +}; + + +/* Used by the filesystem */ +int ocfs2_cluster_connect(const char *stack_name, + const char *cluster_name, + int cluster_name_len, + const char *group, + int grouplen, + struct ocfs2_locking_protocol *lproto, + void (*recovery_handler)(int node_num, + void *recovery_data), + void *recovery_data, + struct ocfs2_cluster_connection **conn); +/* + * Used by callers that don't store their stack name. They must ensure + * all nodes have the same stack. + */ +int ocfs2_cluster_connect_agnostic(const char *group, + int grouplen, + struct ocfs2_locking_protocol *lproto, + void (*recovery_handler)(int node_num, + void *recovery_data), + void *recovery_data, + struct ocfs2_cluster_connection **conn); +int ocfs2_cluster_disconnect(struct ocfs2_cluster_connection *conn, + int hangup_pending); +void ocfs2_cluster_hangup(const char *group, int grouplen); +int ocfs2_cluster_this_node(struct ocfs2_cluster_connection *conn, + unsigned int *node); + +struct ocfs2_lock_res; +int ocfs2_dlm_lock(struct ocfs2_cluster_connection *conn, + int mode, + struct ocfs2_dlm_lksb *lksb, + u32 flags, + void *name, + unsigned int namelen); +int ocfs2_dlm_unlock(struct ocfs2_cluster_connection *conn, + struct ocfs2_dlm_lksb *lksb, + u32 flags); + +int ocfs2_dlm_lock_status(struct ocfs2_dlm_lksb *lksb); +int ocfs2_dlm_lvb_valid(struct ocfs2_dlm_lksb *lksb); +void *ocfs2_dlm_lvb(struct ocfs2_dlm_lksb *lksb); +void ocfs2_dlm_dump_lksb(struct ocfs2_dlm_lksb *lksb); + +int ocfs2_stack_supports_plocks(void); +int ocfs2_plock(struct ocfs2_cluster_connection *conn, u64 ino, + struct file *file, int cmd, struct file_lock *fl); + +void ocfs2_stack_glue_set_max_proto_version(struct ocfs2_protocol_version *max_proto); + + +/* Used by stack plugins */ +int ocfs2_stack_glue_register(struct ocfs2_stack_plugin *plugin); +void ocfs2_stack_glue_unregister(struct ocfs2_stack_plugin *plugin); + +extern struct kset *ocfs2_kset; + +#endif /* STACKGLUE_H */ diff --git a/fs/ocfs2/suballoc.c b/fs/ocfs2/suballoc.c new file mode 100644 index 000000000..20c3aec6b --- /dev/null +++ b/fs/ocfs2/suballoc.c @@ -0,0 +1,3325 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * suballoc.c + * + * metadata alloc and free + * Inspired by ext3 block groups. + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include + +#include + +#include "ocfs2.h" + +#include "alloc.h" +#include "blockcheck.h" +#include "dlmglue.h" +#include "inode.h" +#include "journal.h" +#include "localalloc.h" +#include "suballoc.h" +#include "super.h" +#include "sysfile.h" +#include "uptodate.h" +#include "ocfs2_trace.h" + +#include "buffer_head_io.h" + +#define NOT_ALLOC_NEW_GROUP 0 +#define ALLOC_NEW_GROUP 0x1 +#define ALLOC_GROUPS_FROM_GLOBAL 0x2 + +#define OCFS2_MAX_TO_STEAL 1024 + +struct ocfs2_suballoc_result { + u64 sr_bg_blkno; /* The bg we allocated from. Set + to 0 when a block group is + contiguous. */ + u64 sr_bg_stable_blkno; /* + * Doesn't change, always + * set to target block + * group descriptor + * block. + */ + u64 sr_blkno; /* The first allocated block */ + unsigned int sr_bit_offset; /* The bit in the bg */ + unsigned int sr_bits; /* How many bits we claimed */ + unsigned int sr_max_contig_bits; /* The length for contiguous + * free bits, only available + * for cluster group + */ +}; + +static u64 ocfs2_group_from_res(struct ocfs2_suballoc_result *res) +{ + if (res->sr_blkno == 0) + return 0; + + if (res->sr_bg_blkno) + return res->sr_bg_blkno; + + return ocfs2_which_suballoc_group(res->sr_blkno, res->sr_bit_offset); +} + +static inline u16 ocfs2_find_victim_chain(struct ocfs2_chain_list *cl); +static int ocfs2_block_group_fill(handle_t *handle, + struct inode *alloc_inode, + struct buffer_head *bg_bh, + u64 group_blkno, + unsigned int group_clusters, + u16 my_chain, + struct ocfs2_chain_list *cl); +static int ocfs2_block_group_alloc(struct ocfs2_super *osb, + struct inode *alloc_inode, + struct buffer_head *bh, + u64 max_block, + u64 *last_alloc_group, + int flags); + +static int ocfs2_cluster_group_search(struct inode *inode, + struct buffer_head *group_bh, + u32 bits_wanted, u32 min_bits, + u64 max_block, + struct ocfs2_suballoc_result *res); +static int ocfs2_block_group_search(struct inode *inode, + struct buffer_head *group_bh, + u32 bits_wanted, u32 min_bits, + u64 max_block, + struct ocfs2_suballoc_result *res); +static int ocfs2_claim_suballoc_bits(struct ocfs2_alloc_context *ac, + handle_t *handle, + u32 bits_wanted, + u32 min_bits, + struct ocfs2_suballoc_result *res); +static int ocfs2_test_bg_bit_allocatable(struct buffer_head *bg_bh, + int nr); +static int ocfs2_relink_block_group(handle_t *handle, + struct inode *alloc_inode, + struct buffer_head *fe_bh, + struct buffer_head *bg_bh, + struct buffer_head *prev_bg_bh, + u16 chain); +static inline int ocfs2_block_group_reasonably_empty(struct ocfs2_group_desc *bg, + u32 wanted); +static inline u32 ocfs2_desc_bitmap_to_cluster_off(struct inode *inode, + u64 bg_blkno, + u16 bg_bit_off); +static inline void ocfs2_block_to_cluster_group(struct inode *inode, + u64 data_blkno, + u64 *bg_blkno, + u16 *bg_bit_off); +static int ocfs2_reserve_clusters_with_limit(struct ocfs2_super *osb, + u32 bits_wanted, u64 max_block, + int flags, + struct ocfs2_alloc_context **ac); + +void ocfs2_free_ac_resource(struct ocfs2_alloc_context *ac) +{ + struct inode *inode = ac->ac_inode; + + if (inode) { + if (ac->ac_which != OCFS2_AC_USE_LOCAL) + ocfs2_inode_unlock(inode, 1); + + inode_unlock(inode); + + iput(inode); + ac->ac_inode = NULL; + } + brelse(ac->ac_bh); + ac->ac_bh = NULL; + ac->ac_resv = NULL; + kfree(ac->ac_find_loc_priv); + ac->ac_find_loc_priv = NULL; +} + +void ocfs2_free_alloc_context(struct ocfs2_alloc_context *ac) +{ + ocfs2_free_ac_resource(ac); + kfree(ac); +} + +static u32 ocfs2_bits_per_group(struct ocfs2_chain_list *cl) +{ + return (u32)le16_to_cpu(cl->cl_cpg) * (u32)le16_to_cpu(cl->cl_bpc); +} + +#define do_error(fmt, ...) \ +do { \ + if (resize) \ + mlog(ML_ERROR, fmt, ##__VA_ARGS__); \ + else \ + return ocfs2_error(sb, fmt, ##__VA_ARGS__); \ +} while (0) + +static int ocfs2_validate_gd_self(struct super_block *sb, + struct buffer_head *bh, + int resize) +{ + struct ocfs2_group_desc *gd = (struct ocfs2_group_desc *)bh->b_data; + + if (!OCFS2_IS_VALID_GROUP_DESC(gd)) { + do_error("Group descriptor #%llu has bad signature %.*s\n", + (unsigned long long)bh->b_blocknr, 7, + gd->bg_signature); + } + + if (le64_to_cpu(gd->bg_blkno) != bh->b_blocknr) { + do_error("Group descriptor #%llu has an invalid bg_blkno of %llu\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(gd->bg_blkno)); + } + + if (le32_to_cpu(gd->bg_generation) != OCFS2_SB(sb)->fs_generation) { + do_error("Group descriptor #%llu has an invalid fs_generation of #%u\n", + (unsigned long long)bh->b_blocknr, + le32_to_cpu(gd->bg_generation)); + } + + if (le16_to_cpu(gd->bg_free_bits_count) > le16_to_cpu(gd->bg_bits)) { + do_error("Group descriptor #%llu has bit count %u but claims that %u are free\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(gd->bg_bits), + le16_to_cpu(gd->bg_free_bits_count)); + } + + if (le16_to_cpu(gd->bg_bits) > (8 * le16_to_cpu(gd->bg_size))) { + do_error("Group descriptor #%llu has bit count %u but max bitmap bits of %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(gd->bg_bits), + 8 * le16_to_cpu(gd->bg_size)); + } + + /* + * For discontiguous block groups, validate the on-disk extent list + * against the maximum number of extent records that can physically + * fit in a single block. + */ + if (ocfs2_gd_is_discontig(gd)) { + u16 max_recs = ocfs2_extent_recs_per_gd(sb); + u16 l_count = le16_to_cpu(gd->bg_list.l_count); + u16 l_next_free_rec = le16_to_cpu(gd->bg_list.l_next_free_rec); + + if (l_count != max_recs) { + do_error("Group descriptor #%llu bad discontig l_count %u expected %u\n", + (unsigned long long)bh->b_blocknr, + l_count, + max_recs); + } + + if (l_next_free_rec > l_count) { + do_error("Group descriptor #%llu bad discontig l_next_free_rec %u max %u\n", + (unsigned long long)bh->b_blocknr, + l_next_free_rec, + l_count); + } + } + + return 0; +} + +static int ocfs2_validate_gd_parent(struct super_block *sb, + struct ocfs2_dinode *di, + struct buffer_head *bh, + int resize) +{ + unsigned int max_bits; + unsigned int max_bitmap_bits; + unsigned int max_bitmap_size; + int suballocator; + struct ocfs2_group_desc *gd = (struct ocfs2_group_desc *)bh->b_data; + + suballocator = le64_to_cpu(di->i_blkno) != OCFS2_SB(sb)->bitmap_blkno; + max_bitmap_size = ocfs2_group_bitmap_size(sb, suballocator, + OCFS2_SB(sb)->s_feature_incompat); + max_bitmap_bits = max_bitmap_size * 8; + + if (di->i_blkno != gd->bg_parent_dinode) { + do_error("Group descriptor #%llu has bad parent pointer (%llu, expected %llu)\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(gd->bg_parent_dinode), + (unsigned long long)le64_to_cpu(di->i_blkno)); + } + + if (le16_to_cpu(gd->bg_size) > max_bitmap_size) { + do_error("Group descriptor #%llu has bitmap size %u but physical max of %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(gd->bg_size), + max_bitmap_size); + } + + if (le16_to_cpu(gd->bg_bits) > max_bitmap_bits) { + do_error("Group descriptor #%llu has bit count %u but physical max of %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(gd->bg_bits), + max_bitmap_bits); + } + + max_bits = le16_to_cpu(di->id2.i_chain.cl_cpg) * le16_to_cpu(di->id2.i_chain.cl_bpc); + if (le16_to_cpu(gd->bg_bits) > max_bits) { + do_error("Group descriptor #%llu has bit count of %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(gd->bg_bits)); + } + + /* In resize, we may meet the case bg_chain == cl_next_free_rec. */ + if ((le16_to_cpu(gd->bg_chain) > + le16_to_cpu(di->id2.i_chain.cl_next_free_rec)) || + ((le16_to_cpu(gd->bg_chain) == + le16_to_cpu(di->id2.i_chain.cl_next_free_rec)) && !resize)) { + do_error("Group descriptor #%llu has bad chain %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(gd->bg_chain)); + } + + return 0; +} + +#undef do_error + +/* + * This version only prints errors. It does not fail the filesystem, and + * exists only for resize. + */ +int ocfs2_check_group_descriptor(struct super_block *sb, + struct ocfs2_dinode *di, + struct buffer_head *bh) +{ + int rc; + struct ocfs2_group_desc *gd = (struct ocfs2_group_desc *)bh->b_data; + + BUG_ON(!buffer_uptodate(bh)); + + /* + * If the ecc fails, we return the error but otherwise + * leave the filesystem running. We know any error is + * local to this block. + */ + rc = ocfs2_validate_meta_ecc(sb, bh->b_data, &gd->bg_check); + if (rc) { + mlog(ML_ERROR, + "Checksum failed for group descriptor %llu\n", + (unsigned long long)bh->b_blocknr); + } else + rc = ocfs2_validate_gd_self(sb, bh, 1); + if (!rc) + rc = ocfs2_validate_gd_parent(sb, di, bh, 1); + + return rc; +} + +static int ocfs2_validate_group_descriptor(struct super_block *sb, + struct buffer_head *bh) +{ + int rc; + struct ocfs2_group_desc *gd = (struct ocfs2_group_desc *)bh->b_data; + + trace_ocfs2_validate_group_descriptor( + (unsigned long long)bh->b_blocknr); + + BUG_ON(!buffer_uptodate(bh)); + + /* + * If the ecc fails, we return the error but otherwise + * leave the filesystem running. We know any error is + * local to this block. + */ + rc = ocfs2_validate_meta_ecc(sb, bh->b_data, &gd->bg_check); + if (rc) + return rc; + + /* + * Errors after here are fatal. + */ + + return ocfs2_validate_gd_self(sb, bh, 0); +} + +/* + * The hint group descriptor (gd) may already have been released + * in _ocfs2_free_suballoc_bits(). We first check the gd signature, + * then perform the standard ocfs2_read_group_descriptor() jobs. + * + * If the gd signature is invalid, we return 'rc=0' and set + * '*released=1'. The caller is expected to handle this specific case. + * Otherwise, we return the actual error code. + * + * We treat gd signature corruption case as a release case. The + * caller ocfs2_claim_suballoc_bits() will use ocfs2_search_chain() + * to search each gd block. The code will eventually find this + * corrupted gd block - Late, but not missed. + * + * Note: + * The caller is responsible for initializing the '*released' status. + */ +static int ocfs2_read_hint_group_descriptor(struct inode *inode, + struct ocfs2_dinode *di, u64 gd_blkno, + struct buffer_head **bh, int *released) +{ + int rc; + struct buffer_head *tmp = *bh; + struct ocfs2_group_desc *gd; + + rc = ocfs2_read_block(INODE_CACHE(inode), gd_blkno, &tmp, NULL); + if (rc) + goto out; + + gd = (struct ocfs2_group_desc *) tmp->b_data; + if (!OCFS2_IS_VALID_GROUP_DESC(gd)) { + /* + * Invalid gd cache was set in ocfs2_read_block(), + * which will affect block_group allocation. + * Path: + * ocfs2_reserve_suballoc_bits + * ocfs2_block_group_alloc + * ocfs2_block_group_alloc_contig + * ocfs2_set_new_buffer_uptodate + */ + ocfs2_remove_from_cache(INODE_CACHE(inode), tmp); + *released = 1; /* we return 'rc=0' for this case */ + goto free_bh; + } + + /* below jobs same with ocfs2_read_group_descriptor() */ + if (!buffer_jbd(tmp)) { + rc = ocfs2_validate_group_descriptor(inode->i_sb, tmp); + if (rc) + goto free_bh; + } + + rc = ocfs2_validate_gd_parent(inode->i_sb, di, tmp, 0); + if (rc) + goto free_bh; + + /* If ocfs2_read_block() got us a new bh, pass it up. */ + if (!*bh) + *bh = tmp; + + return rc; + +free_bh: + brelse(tmp); +out: + return rc; +} + +int ocfs2_read_group_descriptor(struct inode *inode, struct ocfs2_dinode *di, + u64 gd_blkno, struct buffer_head **bh) +{ + int rc; + struct buffer_head *tmp = *bh; + + rc = ocfs2_read_block(INODE_CACHE(inode), gd_blkno, &tmp, + ocfs2_validate_group_descriptor); + if (rc) + goto out; + + rc = ocfs2_validate_gd_parent(inode->i_sb, di, tmp, 0); + if (rc) { + brelse(tmp); + goto out; + } + + /* If ocfs2_read_block() got us a new bh, pass it up. */ + if (!*bh) + *bh = tmp; + +out: + return rc; +} + +static void ocfs2_bg_discontig_add_extent(struct ocfs2_super *osb, + struct ocfs2_group_desc *bg, + struct ocfs2_chain_list *cl, + u64 p_blkno, unsigned int clusters) +{ + struct ocfs2_extent_list *el = &bg->bg_list; + struct ocfs2_extent_rec *rec; + + BUG_ON(!ocfs2_supports_discontig_bg(osb)); + if (!el->l_next_free_rec) + el->l_count = cpu_to_le16(ocfs2_extent_recs_per_gd(osb->sb)); + rec = &el->l_recs[le16_to_cpu(el->l_next_free_rec)]; + rec->e_blkno = cpu_to_le64(p_blkno); + rec->e_cpos = cpu_to_le32(le16_to_cpu(bg->bg_bits) / + le16_to_cpu(cl->cl_bpc)); + rec->e_leaf_clusters = cpu_to_le16(clusters); + le16_add_cpu(&bg->bg_bits, clusters * le16_to_cpu(cl->cl_bpc)); + le16_add_cpu(&bg->bg_free_bits_count, + clusters * le16_to_cpu(cl->cl_bpc)); + le16_add_cpu(&el->l_next_free_rec, 1); +} + +static int ocfs2_block_group_fill(handle_t *handle, + struct inode *alloc_inode, + struct buffer_head *bg_bh, + u64 group_blkno, + unsigned int group_clusters, + u16 my_chain, + struct ocfs2_chain_list *cl) +{ + int status = 0; + struct ocfs2_super *osb = OCFS2_SB(alloc_inode->i_sb); + struct ocfs2_group_desc *bg = (struct ocfs2_group_desc *) bg_bh->b_data; + struct super_block * sb = alloc_inode->i_sb; + + if (((unsigned long long) bg_bh->b_blocknr) != group_blkno) { + status = ocfs2_error(alloc_inode->i_sb, + "group block (%llu) != b_blocknr (%llu)\n", + (unsigned long long)group_blkno, + (unsigned long long) bg_bh->b_blocknr); + goto bail; + } + + status = ocfs2_journal_access_gd(handle, + INODE_CACHE(alloc_inode), + bg_bh, + OCFS2_JOURNAL_ACCESS_CREATE); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + memset(bg, 0, sb->s_blocksize); + strscpy(bg->bg_signature, OCFS2_GROUP_DESC_SIGNATURE); + bg->bg_generation = cpu_to_le32(osb->fs_generation); + bg->bg_size = cpu_to_le16(ocfs2_group_bitmap_size(sb, 1, + osb->s_feature_incompat)); + bg->bg_chain = cpu_to_le16(my_chain); + bg->bg_next_group = cl->cl_recs[my_chain].c_blkno; + bg->bg_parent_dinode = cpu_to_le64(OCFS2_I(alloc_inode)->ip_blkno); + bg->bg_blkno = cpu_to_le64(group_blkno); + if (group_clusters == le16_to_cpu(cl->cl_cpg)) + bg->bg_bits = cpu_to_le16(ocfs2_bits_per_group(cl)); + else + ocfs2_bg_discontig_add_extent(osb, bg, cl, group_blkno, + group_clusters); + + /* set the 1st bit in the bitmap to account for the descriptor block */ + ocfs2_set_bit(0, (unsigned long *)bg->bg_bitmap); + bg->bg_free_bits_count = cpu_to_le16(le16_to_cpu(bg->bg_bits) - 1); + + ocfs2_journal_dirty(handle, bg_bh); + + /* There is no need to zero out or otherwise initialize the + * other blocks in a group - All valid FS metadata in a block + * group stores the superblock fs_generation value at + * allocation time. */ + +bail: + if (status) + mlog_errno(status); + return status; +} + +static inline u16 ocfs2_find_smallest_chain(struct ocfs2_chain_list *cl) +{ + u16 curr, best; + + best = curr = 0; + while (curr < le16_to_cpu(cl->cl_count)) { + if (le32_to_cpu(cl->cl_recs[best].c_total) > + le32_to_cpu(cl->cl_recs[curr].c_total)) + best = curr; + curr++; + } + return best; +} + +static struct buffer_head * +ocfs2_block_group_alloc_contig(struct ocfs2_super *osb, handle_t *handle, + struct inode *alloc_inode, + struct ocfs2_alloc_context *ac, + struct ocfs2_chain_list *cl) +{ + int status; + u32 bit_off, num_bits; + u64 bg_blkno; + struct buffer_head *bg_bh; + unsigned int alloc_rec = ocfs2_find_smallest_chain(cl); + + status = ocfs2_claim_clusters(handle, ac, + le16_to_cpu(cl->cl_cpg), &bit_off, + &num_bits); + if (status < 0) { + if (status != -ENOSPC) + mlog_errno(status); + goto bail; + } + + /* setup the group */ + bg_blkno = ocfs2_clusters_to_blocks(osb->sb, bit_off); + trace_ocfs2_block_group_alloc_contig( + (unsigned long long)bg_blkno, alloc_rec); + + bg_bh = sb_getblk(osb->sb, bg_blkno); + if (!bg_bh) { + status = -ENOMEM; + mlog_errno(status); + goto bail; + } + ocfs2_set_new_buffer_uptodate(INODE_CACHE(alloc_inode), bg_bh); + + status = ocfs2_block_group_fill(handle, alloc_inode, bg_bh, + bg_blkno, num_bits, alloc_rec, cl); + if (status < 0) { + brelse(bg_bh); + mlog_errno(status); + } + +bail: + return status ? ERR_PTR(status) : bg_bh; +} + +static int ocfs2_block_group_claim_bits(struct ocfs2_super *osb, + handle_t *handle, + struct ocfs2_alloc_context *ac, + unsigned int min_bits, + u32 *bit_off, u32 *num_bits) +{ + int status = 0; + + while (min_bits) { + status = ocfs2_claim_clusters(handle, ac, min_bits, + bit_off, num_bits); + if (status != -ENOSPC) + break; + + min_bits >>= 1; + } + + return status; +} + +static int ocfs2_block_group_grow_discontig(handle_t *handle, + struct inode *alloc_inode, + struct buffer_head *bg_bh, + struct ocfs2_alloc_context *ac, + struct ocfs2_chain_list *cl, + unsigned int min_bits) +{ + int status; + struct ocfs2_super *osb = OCFS2_SB(alloc_inode->i_sb); + struct ocfs2_group_desc *bg = + (struct ocfs2_group_desc *)bg_bh->b_data; + unsigned int needed = le16_to_cpu(cl->cl_cpg) - + le16_to_cpu(bg->bg_bits) / le16_to_cpu(cl->cl_bpc); + u32 p_cpos, clusters; + u64 p_blkno; + struct ocfs2_extent_list *el = &bg->bg_list; + + status = ocfs2_journal_access_gd(handle, + INODE_CACHE(alloc_inode), + bg_bh, + OCFS2_JOURNAL_ACCESS_CREATE); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + while ((needed > 0) && (le16_to_cpu(el->l_next_free_rec) < + le16_to_cpu(el->l_count))) { + if (min_bits > needed) + min_bits = needed; + status = ocfs2_block_group_claim_bits(osb, handle, ac, + min_bits, &p_cpos, + &clusters); + if (status < 0) { + if (status != -ENOSPC) + mlog_errno(status); + goto bail; + } + p_blkno = ocfs2_clusters_to_blocks(osb->sb, p_cpos); + ocfs2_bg_discontig_add_extent(osb, bg, cl, p_blkno, + clusters); + + min_bits = clusters; + needed = le16_to_cpu(cl->cl_cpg) - + le16_to_cpu(bg->bg_bits) / le16_to_cpu(cl->cl_bpc); + } + + if (needed > 0) { + /* + * We have used up all the extent rec but can't fill up + * the cpg. So bail out. + */ + status = -ENOSPC; + goto bail; + } + + ocfs2_journal_dirty(handle, bg_bh); + +bail: + return status; +} + +static void ocfs2_bg_alloc_cleanup(handle_t *handle, + struct ocfs2_alloc_context *cluster_ac, + struct inode *alloc_inode, + struct buffer_head *bg_bh) +{ + int i, ret; + struct ocfs2_group_desc *bg; + struct ocfs2_extent_list *el; + struct ocfs2_extent_rec *rec; + + if (!bg_bh) + return; + + bg = (struct ocfs2_group_desc *)bg_bh->b_data; + el = &bg->bg_list; + for (i = 0; i < le16_to_cpu(el->l_next_free_rec); i++) { + rec = &el->l_recs[i]; + ret = ocfs2_free_clusters(handle, cluster_ac->ac_inode, + cluster_ac->ac_bh, + le64_to_cpu(rec->e_blkno), + le16_to_cpu(rec->e_leaf_clusters)); + if (ret) + mlog_errno(ret); + /* Try all the clusters to free */ + } + + ocfs2_remove_from_cache(INODE_CACHE(alloc_inode), bg_bh); + brelse(bg_bh); +} + +static struct buffer_head * +ocfs2_block_group_alloc_discontig(handle_t *handle, + struct inode *alloc_inode, + struct ocfs2_alloc_context *ac, + struct ocfs2_chain_list *cl) +{ + int status; + u32 bit_off, num_bits; + u64 bg_blkno; + unsigned int min_bits = le16_to_cpu(cl->cl_cpg) >> 1; + struct buffer_head *bg_bh = NULL; + unsigned int alloc_rec = ocfs2_find_smallest_chain(cl); + struct ocfs2_super *osb = OCFS2_SB(alloc_inode->i_sb); + + if (!ocfs2_supports_discontig_bg(osb)) { + status = -ENOSPC; + goto bail; + } + + status = ocfs2_extend_trans(handle, + ocfs2_calc_bg_discontig_credits(osb->sb)); + if (status) { + mlog_errno(status); + goto bail; + } + + /* + * We're going to be grabbing from multiple cluster groups. + * We don't have enough credits to relink them all, and the + * cluster groups will be staying in cache for the duration of + * this operation. + */ + ac->ac_disable_chain_relink = 1; + + /* Claim the first region */ + status = ocfs2_block_group_claim_bits(osb, handle, ac, min_bits, + &bit_off, &num_bits); + if (status < 0) { + if (status != -ENOSPC) + mlog_errno(status); + goto bail; + } + min_bits = num_bits; + + /* setup the group */ + bg_blkno = ocfs2_clusters_to_blocks(osb->sb, bit_off); + trace_ocfs2_block_group_alloc_discontig( + (unsigned long long)bg_blkno, alloc_rec); + + bg_bh = sb_getblk(osb->sb, bg_blkno); + if (!bg_bh) { + status = -ENOMEM; + mlog_errno(status); + goto bail; + } + ocfs2_set_new_buffer_uptodate(INODE_CACHE(alloc_inode), bg_bh); + + status = ocfs2_block_group_fill(handle, alloc_inode, bg_bh, + bg_blkno, num_bits, alloc_rec, cl); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + status = ocfs2_block_group_grow_discontig(handle, alloc_inode, + bg_bh, ac, cl, min_bits); + if (status) + mlog_errno(status); + +bail: + if (status) + ocfs2_bg_alloc_cleanup(handle, ac, alloc_inode, bg_bh); + return status ? ERR_PTR(status) : bg_bh; +} + +/* + * We expect the block group allocator to already be locked. + */ +static int ocfs2_block_group_alloc(struct ocfs2_super *osb, + struct inode *alloc_inode, + struct buffer_head *bh, + u64 max_block, + u64 *last_alloc_group, + int flags) +{ + int status, credits; + struct ocfs2_dinode *fe = (struct ocfs2_dinode *) bh->b_data; + struct ocfs2_chain_list *cl; + struct ocfs2_alloc_context *ac = NULL; + handle_t *handle = NULL; + u16 alloc_rec; + struct buffer_head *bg_bh = NULL; + struct ocfs2_group_desc *bg; + + BUG_ON(ocfs2_is_cluster_bitmap(alloc_inode)); + + cl = &fe->id2.i_chain; + status = ocfs2_reserve_clusters_with_limit(osb, + le16_to_cpu(cl->cl_cpg), + max_block, flags, &ac); + if (status < 0) { + if (status != -ENOSPC) + mlog_errno(status); + goto bail; + } + + credits = ocfs2_calc_group_alloc_credits(osb->sb, + le16_to_cpu(cl->cl_cpg)); + handle = ocfs2_start_trans(osb, credits); + if (IS_ERR(handle)) { + status = PTR_ERR(handle); + handle = NULL; + mlog_errno(status); + goto bail; + } + + if (last_alloc_group && *last_alloc_group != 0) { + trace_ocfs2_block_group_alloc( + (unsigned long long)*last_alloc_group); + ac->ac_last_group = *last_alloc_group; + } + + bg_bh = ocfs2_block_group_alloc_contig(osb, handle, alloc_inode, + ac, cl); + if (PTR_ERR(bg_bh) == -ENOSPC) { + ac->ac_which = OCFS2_AC_USE_MAIN_DISCONTIG; + bg_bh = ocfs2_block_group_alloc_discontig(handle, + alloc_inode, + ac, cl); + } + if (IS_ERR(bg_bh)) { + status = PTR_ERR(bg_bh); + bg_bh = NULL; + if (status != -ENOSPC) + mlog_errno(status); + goto bail; + } + bg = (struct ocfs2_group_desc *) bg_bh->b_data; + + status = ocfs2_journal_access_di(handle, INODE_CACHE(alloc_inode), + bh, OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + alloc_rec = le16_to_cpu(bg->bg_chain); + le32_add_cpu(&cl->cl_recs[alloc_rec].c_free, + le16_to_cpu(bg->bg_free_bits_count)); + le32_add_cpu(&cl->cl_recs[alloc_rec].c_total, + le16_to_cpu(bg->bg_bits)); + cl->cl_recs[alloc_rec].c_blkno = bg->bg_blkno; + if (le16_to_cpu(cl->cl_next_free_rec) < le16_to_cpu(cl->cl_count)) + le16_add_cpu(&cl->cl_next_free_rec, 1); + + le32_add_cpu(&fe->id1.bitmap1.i_used, le16_to_cpu(bg->bg_bits) - + le16_to_cpu(bg->bg_free_bits_count)); + le32_add_cpu(&fe->id1.bitmap1.i_total, le16_to_cpu(bg->bg_bits)); + le32_add_cpu(&fe->i_clusters, le16_to_cpu(cl->cl_cpg)); + + ocfs2_journal_dirty(handle, bh); + + spin_lock(&OCFS2_I(alloc_inode)->ip_lock); + OCFS2_I(alloc_inode)->ip_clusters = le32_to_cpu(fe->i_clusters); + fe->i_size = cpu_to_le64(ocfs2_clusters_to_bytes(alloc_inode->i_sb, + le32_to_cpu(fe->i_clusters))); + spin_unlock(&OCFS2_I(alloc_inode)->ip_lock); + i_size_write(alloc_inode, le64_to_cpu(fe->i_size)); + alloc_inode->i_blocks = ocfs2_inode_sector_count(alloc_inode); + ocfs2_update_inode_fsync_trans(handle, alloc_inode, 0); + + status = 0; + + /* save the new last alloc group so that the caller can cache it. */ + if (last_alloc_group) + *last_alloc_group = ac->ac_last_group; + +bail: + if (handle) + ocfs2_commit_trans(osb, handle); + + if (ac) + ocfs2_free_alloc_context(ac); + + brelse(bg_bh); + + if (status) + mlog_errno(status); + return status; +} + +static int ocfs2_reserve_suballoc_bits(struct ocfs2_super *osb, + struct ocfs2_alloc_context *ac, + int type, + u32 slot, + u64 *last_alloc_group, + int flags) +{ + int status; + u32 bits_wanted = ac->ac_bits_wanted; + struct inode *alloc_inode; + struct buffer_head *bh = NULL; + struct ocfs2_dinode *fe; + u32 free_bits; + + alloc_inode = ocfs2_get_system_file_inode(osb, type, slot); + if (!alloc_inode) { + mlog_errno(-EINVAL); + return -EINVAL; + } + + inode_lock(alloc_inode); + + status = ocfs2_inode_lock(alloc_inode, &bh, 1); + if (status < 0) { + inode_unlock(alloc_inode); + iput(alloc_inode); + + mlog_errno(status); + return status; + } + + ac->ac_inode = alloc_inode; + ac->ac_alloc_slot = slot; + + fe = (struct ocfs2_dinode *) bh->b_data; + + /* The bh was validated by the inode read inside + * ocfs2_inode_lock(). Any corruption is a code bug. */ + BUG_ON(!OCFS2_IS_VALID_DINODE(fe)); + + if (!(fe->i_flags & cpu_to_le32(OCFS2_CHAIN_FL))) { + status = ocfs2_error(alloc_inode->i_sb, + "Invalid chain allocator %llu\n", + (unsigned long long)le64_to_cpu(fe->i_blkno)); + goto bail; + } + + free_bits = le32_to_cpu(fe->id1.bitmap1.i_total) - + le32_to_cpu(fe->id1.bitmap1.i_used); + + if (bits_wanted > free_bits) { + /* cluster bitmap never grows */ + if (ocfs2_is_cluster_bitmap(alloc_inode)) { + trace_ocfs2_reserve_suballoc_bits_nospc(bits_wanted, + free_bits); + status = -ENOSPC; + goto bail; + } + + if (!(flags & ALLOC_NEW_GROUP)) { + trace_ocfs2_reserve_suballoc_bits_no_new_group( + slot, bits_wanted, free_bits); + status = -ENOSPC; + goto bail; + } + + status = ocfs2_block_group_alloc(osb, alloc_inode, bh, + ac->ac_max_block, + last_alloc_group, flags); + if (status < 0) { + if (status != -ENOSPC) + mlog_errno(status); + goto bail; + } + atomic_inc(&osb->alloc_stats.bg_extends); + + /* You should never ask for this much metadata */ + BUG_ON(bits_wanted > + (le32_to_cpu(fe->id1.bitmap1.i_total) + - le32_to_cpu(fe->id1.bitmap1.i_used))); + } + + get_bh(bh); + ac->ac_bh = bh; +bail: + brelse(bh); + + if (status) + mlog_errno(status); + return status; +} + +static void ocfs2_init_inode_steal_slot(struct ocfs2_super *osb) +{ + spin_lock(&osb->osb_lock); + osb->s_inode_steal_slot = OCFS2_INVALID_SLOT; + spin_unlock(&osb->osb_lock); + atomic_set(&osb->s_num_inodes_stolen, 0); +} + +static void ocfs2_init_meta_steal_slot(struct ocfs2_super *osb) +{ + spin_lock(&osb->osb_lock); + osb->s_meta_steal_slot = OCFS2_INVALID_SLOT; + spin_unlock(&osb->osb_lock); + atomic_set(&osb->s_num_meta_stolen, 0); +} + +void ocfs2_init_steal_slots(struct ocfs2_super *osb) +{ + ocfs2_init_inode_steal_slot(osb); + ocfs2_init_meta_steal_slot(osb); +} + +static void __ocfs2_set_steal_slot(struct ocfs2_super *osb, int slot, int type) +{ + spin_lock(&osb->osb_lock); + if (type == INODE_ALLOC_SYSTEM_INODE) + osb->s_inode_steal_slot = (u16)slot; + else if (type == EXTENT_ALLOC_SYSTEM_INODE) + osb->s_meta_steal_slot = (u16)slot; + spin_unlock(&osb->osb_lock); +} + +static int __ocfs2_get_steal_slot(struct ocfs2_super *osb, int type) +{ + int slot = OCFS2_INVALID_SLOT; + + spin_lock(&osb->osb_lock); + if (type == INODE_ALLOC_SYSTEM_INODE) + slot = osb->s_inode_steal_slot; + else if (type == EXTENT_ALLOC_SYSTEM_INODE) + slot = osb->s_meta_steal_slot; + spin_unlock(&osb->osb_lock); + + return slot; +} + +static int ocfs2_get_inode_steal_slot(struct ocfs2_super *osb) +{ + return __ocfs2_get_steal_slot(osb, INODE_ALLOC_SYSTEM_INODE); +} + +static int ocfs2_get_meta_steal_slot(struct ocfs2_super *osb) +{ + return __ocfs2_get_steal_slot(osb, EXTENT_ALLOC_SYSTEM_INODE); +} + +static int ocfs2_steal_resource(struct ocfs2_super *osb, + struct ocfs2_alloc_context *ac, + int type) +{ + int i, status = -ENOSPC; + int slot = __ocfs2_get_steal_slot(osb, type); + + /* Start to steal resource from the first slot after ours. */ + if (slot == OCFS2_INVALID_SLOT) + slot = osb->slot_num + 1; + + for (i = 0; i < osb->max_slots; i++, slot++) { + if (slot == osb->max_slots) + slot = 0; + + if (slot == osb->slot_num) + continue; + + status = ocfs2_reserve_suballoc_bits(osb, ac, + type, + (u32)slot, NULL, + NOT_ALLOC_NEW_GROUP); + if (status >= 0) { + __ocfs2_set_steal_slot(osb, slot, type); + break; + } + + ocfs2_free_ac_resource(ac); + } + + return status; +} + +static int ocfs2_steal_inode(struct ocfs2_super *osb, + struct ocfs2_alloc_context *ac) +{ + return ocfs2_steal_resource(osb, ac, INODE_ALLOC_SYSTEM_INODE); +} + +static int ocfs2_steal_meta(struct ocfs2_super *osb, + struct ocfs2_alloc_context *ac) +{ + return ocfs2_steal_resource(osb, ac, EXTENT_ALLOC_SYSTEM_INODE); +} + +int ocfs2_reserve_new_metadata_blocks(struct ocfs2_super *osb, + int blocks, + struct ocfs2_alloc_context **ac) +{ + int status; + int slot = ocfs2_get_meta_steal_slot(osb); + + *ac = kzalloc_obj(struct ocfs2_alloc_context); + if (!(*ac)) { + status = -ENOMEM; + mlog_errno(status); + goto bail; + } + + (*ac)->ac_bits_wanted = blocks; + (*ac)->ac_which = OCFS2_AC_USE_META; + (*ac)->ac_group_search = ocfs2_block_group_search; + + if (slot != OCFS2_INVALID_SLOT && + atomic_read(&osb->s_num_meta_stolen) < OCFS2_MAX_TO_STEAL) + goto extent_steal; + + atomic_set(&osb->s_num_meta_stolen, 0); + status = ocfs2_reserve_suballoc_bits(osb, (*ac), + EXTENT_ALLOC_SYSTEM_INODE, + (u32)osb->slot_num, NULL, + ALLOC_GROUPS_FROM_GLOBAL|ALLOC_NEW_GROUP); + + + if (status >= 0) { + status = 0; + if (slot != OCFS2_INVALID_SLOT) + ocfs2_init_meta_steal_slot(osb); + goto bail; + } else if (status < 0 && status != -ENOSPC) { + mlog_errno(status); + goto bail; + } + + ocfs2_free_ac_resource(*ac); + +extent_steal: + status = ocfs2_steal_meta(osb, *ac); + atomic_inc(&osb->s_num_meta_stolen); + if (status < 0) { + if (status != -ENOSPC) + mlog_errno(status); + goto bail; + } + + status = 0; +bail: + if ((status < 0) && *ac) { + ocfs2_free_alloc_context(*ac); + *ac = NULL; + } + + if (status) + mlog_errno(status); + return status; +} + +int ocfs2_reserve_new_metadata(struct ocfs2_super *osb, + struct ocfs2_extent_list *root_el, + struct ocfs2_alloc_context **ac) +{ + return ocfs2_reserve_new_metadata_blocks(osb, + ocfs2_extend_meta_needed(root_el), + ac); +} + +int ocfs2_reserve_new_inode(struct ocfs2_super *osb, + struct ocfs2_alloc_context **ac) +{ + int status; + int slot = ocfs2_get_inode_steal_slot(osb); + u64 alloc_group; + + *ac = kzalloc_obj(struct ocfs2_alloc_context); + if (!(*ac)) { + status = -ENOMEM; + mlog_errno(status); + goto bail; + } + + (*ac)->ac_bits_wanted = 1; + (*ac)->ac_which = OCFS2_AC_USE_INODE; + + (*ac)->ac_group_search = ocfs2_block_group_search; + + /* + * stat(2) can't handle i_ino > 32bits, so we tell the + * lower levels not to allocate us a block group past that + * limit. The 'inode64' mount option avoids this behavior. + */ + if (!(osb->s_mount_opt & OCFS2_MOUNT_INODE64)) + (*ac)->ac_max_block = (u32)~0U; + + /* + * slot is set when we successfully steal inode from other nodes. + * It is reset in 3 places: + * 1. when we flush the truncate log + * 2. when we complete local alloc recovery. + * 3. when we successfully allocate from our own slot. + * After it is set, we will go on stealing inodes until we find the + * need to check our slots to see whether there is some space for us. + */ + if (slot != OCFS2_INVALID_SLOT && + atomic_read(&osb->s_num_inodes_stolen) < OCFS2_MAX_TO_STEAL) + goto inode_steal; + + atomic_set(&osb->s_num_inodes_stolen, 0); + alloc_group = osb->osb_inode_alloc_group; + status = ocfs2_reserve_suballoc_bits(osb, *ac, + INODE_ALLOC_SYSTEM_INODE, + (u32)osb->slot_num, + &alloc_group, + ALLOC_NEW_GROUP | + ALLOC_GROUPS_FROM_GLOBAL); + if (status >= 0) { + status = 0; + + spin_lock(&osb->osb_lock); + osb->osb_inode_alloc_group = alloc_group; + spin_unlock(&osb->osb_lock); + trace_ocfs2_reserve_new_inode_new_group( + (unsigned long long)alloc_group); + + /* + * Some inodes must be freed by us, so try to allocate + * from our own next time. + */ + if (slot != OCFS2_INVALID_SLOT) + ocfs2_init_inode_steal_slot(osb); + goto bail; + } else if (status < 0 && status != -ENOSPC) { + mlog_errno(status); + goto bail; + } + + ocfs2_free_ac_resource(*ac); + +inode_steal: + status = ocfs2_steal_inode(osb, *ac); + atomic_inc(&osb->s_num_inodes_stolen); + if (status < 0) { + if (status != -ENOSPC) + mlog_errno(status); + goto bail; + } + + status = 0; +bail: + if ((status < 0) && *ac) { + ocfs2_free_alloc_context(*ac); + *ac = NULL; + } + + if (status) + mlog_errno(status); + return status; +} + +/* local alloc code has to do the same thing, so rather than do this + * twice.. */ +int ocfs2_reserve_cluster_bitmap_bits(struct ocfs2_super *osb, + struct ocfs2_alloc_context *ac) +{ + int status; + + ac->ac_which = OCFS2_AC_USE_MAIN; + ac->ac_group_search = ocfs2_cluster_group_search; + + status = ocfs2_reserve_suballoc_bits(osb, ac, + GLOBAL_BITMAP_SYSTEM_INODE, + OCFS2_INVALID_SLOT, NULL, + ALLOC_NEW_GROUP); + if (status < 0 && status != -ENOSPC) + mlog_errno(status); + + return status; +} + +/* Callers don't need to care which bitmap (local alloc or main) to + * use so we figure it out for them, but unfortunately this clutters + * things a bit. */ +static int ocfs2_reserve_clusters_with_limit(struct ocfs2_super *osb, + u32 bits_wanted, u64 max_block, + int flags, + struct ocfs2_alloc_context **ac) +{ + int status, ret = 0; + int retried = 0; + + *ac = kzalloc_obj(struct ocfs2_alloc_context); + if (!(*ac)) { + status = -ENOMEM; + mlog_errno(status); + goto bail; + } + + (*ac)->ac_bits_wanted = bits_wanted; + (*ac)->ac_max_block = max_block; + + status = -ENOSPC; + if (!(flags & ALLOC_GROUPS_FROM_GLOBAL) && + ocfs2_alloc_should_use_local(osb, bits_wanted)) { + status = ocfs2_reserve_local_alloc_bits(osb, + bits_wanted, + *ac); + if ((status < 0) && (status != -ENOSPC)) { + mlog_errno(status); + goto bail; + } + } + + if (status == -ENOSPC) { +retry: + status = ocfs2_reserve_cluster_bitmap_bits(osb, *ac); + /* Retry if there is sufficient space cached in truncate log */ + if (status == -ENOSPC && !retried) { + retried = 1; + ocfs2_inode_unlock((*ac)->ac_inode, 1); + inode_unlock((*ac)->ac_inode); + + ret = ocfs2_try_to_free_truncate_log(osb, bits_wanted); + if (ret == 1) { + iput((*ac)->ac_inode); + (*ac)->ac_inode = NULL; + goto retry; + } + + if (ret < 0) + mlog_errno(ret); + + inode_lock((*ac)->ac_inode); + ret = ocfs2_inode_lock((*ac)->ac_inode, NULL, 1); + if (ret < 0) { + mlog_errno(ret); + inode_unlock((*ac)->ac_inode); + iput((*ac)->ac_inode); + (*ac)->ac_inode = NULL; + goto bail; + } + } + if (status < 0) { + if (status != -ENOSPC) + mlog_errno(status); + goto bail; + } + } + + status = 0; +bail: + if ((status < 0) && *ac) { + ocfs2_free_alloc_context(*ac); + *ac = NULL; + } + + if (status) + mlog_errno(status); + return status; +} + +int ocfs2_reserve_clusters(struct ocfs2_super *osb, + u32 bits_wanted, + struct ocfs2_alloc_context **ac) +{ + return ocfs2_reserve_clusters_with_limit(osb, bits_wanted, 0, + ALLOC_NEW_GROUP, ac); +} + +/* + * More or less lifted from ext3. I'll leave their description below: + * + * "For ext3 allocations, we must not reuse any blocks which are + * allocated in the bitmap buffer's "last committed data" copy. This + * prevents deletes from freeing up the page for reuse until we have + * committed the delete transaction. + * + * If we didn't do this, then deleting something and reallocating it as + * data would allow the old block to be overwritten before the + * transaction committed (because we force data to disk before commit). + * This would lead to corruption if we crashed between overwriting the + * data and committing the delete. + * + * @@@ We may want to make this allocation behaviour conditional on + * data-writes at some point, and disable it for metadata allocations or + * sync-data inodes." + * + * Note: OCFS2 already does this differently for metadata vs data + * allocations, as those bitmaps are separate and undo access is never + * called on a metadata group descriptor. + */ +static int ocfs2_test_bg_bit_allocatable(struct buffer_head *bg_bh, + int nr) +{ + struct ocfs2_group_desc *bg = (struct ocfs2_group_desc *) bg_bh->b_data; + struct journal_head *jh; + int ret; + + if (ocfs2_test_bit(nr, (unsigned long *)bg->bg_bitmap)) + return 0; + + jh = jbd2_journal_grab_journal_head(bg_bh); + if (!jh) + return 1; + + spin_lock(&jh->b_state_lock); + bg = (struct ocfs2_group_desc *) jh->b_committed_data; + if (bg) + ret = !ocfs2_test_bit(nr, (unsigned long *)bg->bg_bitmap); + else + ret = 1; + spin_unlock(&jh->b_state_lock); + jbd2_journal_put_journal_head(jh); + + return ret; +} + +u16 ocfs2_find_max_contig_free_bits(void *bitmap, + u16 total_bits, u16 start) +{ + u16 offset, free_bits; + u16 contig_bits = 0; + + while (start < total_bits) { + offset = ocfs2_find_next_zero_bit(bitmap, total_bits, start); + if (offset == total_bits) + break; + + start = ocfs2_find_next_bit(bitmap, total_bits, offset); + free_bits = start - offset; + if (contig_bits < free_bits) + contig_bits = free_bits; + } + + return contig_bits; +} + +static int ocfs2_block_group_find_clear_bits(struct ocfs2_super *osb, + struct buffer_head *bg_bh, + unsigned int bits_wanted, + unsigned int total_bits, + struct ocfs2_suballoc_result *res) +{ + void *bitmap; + u16 best_offset, best_size; + u16 prev_best_size = 0; + int offset, start, found, status = 0; + struct ocfs2_group_desc *bg = (struct ocfs2_group_desc *) bg_bh->b_data; + + /* Callers got this descriptor from + * ocfs2_read_group_descriptor(). Any corruption is a code bug. */ + BUG_ON(!OCFS2_IS_VALID_GROUP_DESC(bg)); + + found = start = best_offset = best_size = 0; + bitmap = bg->bg_bitmap; + + while ((offset = ocfs2_find_next_zero_bit(bitmap, total_bits, start)) < + total_bits) { + if (!ocfs2_test_bg_bit_allocatable(bg_bh, offset)) { + /* We found a zero, but we can't use it as it + * hasn't been put to disk yet! */ + found = 0; + start = offset + 1; + } else if (offset == start) { + /* we found a zero */ + found++; + /* move start to the next bit to test */ + start++; + } else { + /* got a zero after some ones */ + found = 1; + start = offset + 1; + prev_best_size = best_size; + } + if (found > best_size) { + best_size = found; + best_offset = start - found; + } + /* we got everything we needed */ + if (found == bits_wanted) { + /* mlog(0, "Found it all!\n"); */ + break; + } + } + + /* best_size will be allocated, we save prev_best_size */ + res->sr_max_contig_bits = prev_best_size; + if (best_size) { + res->sr_bit_offset = best_offset; + res->sr_bits = best_size; + } else { + status = -ENOSPC; + /* No error log here -- see the comment above + * ocfs2_test_bg_bit_allocatable */ + } + + return status; +} + +int ocfs2_block_group_set_bits(handle_t *handle, + struct inode *alloc_inode, + struct ocfs2_group_desc *bg, + struct buffer_head *group_bh, + unsigned int bit_off, + unsigned int num_bits, + unsigned int max_contig_bits, + int fastpath) +{ + int status; + void *bitmap = bg->bg_bitmap; + int journal_type = OCFS2_JOURNAL_ACCESS_WRITE; + unsigned int start = bit_off + num_bits; + u16 contig_bits; + struct ocfs2_super *osb = OCFS2_SB(alloc_inode->i_sb); + + /* All callers get the descriptor via + * ocfs2_read_group_descriptor(). Any corruption is a code bug. */ + BUG_ON(!OCFS2_IS_VALID_GROUP_DESC(bg)); + BUG_ON(le16_to_cpu(bg->bg_free_bits_count) < num_bits); + + trace_ocfs2_block_group_set_bits(bit_off, num_bits); + + if (ocfs2_is_cluster_bitmap(alloc_inode)) + journal_type = OCFS2_JOURNAL_ACCESS_UNDO; + + status = ocfs2_journal_access_gd(handle, + INODE_CACHE(alloc_inode), + group_bh, + journal_type); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + le16_add_cpu(&bg->bg_free_bits_count, -num_bits); + if (le16_to_cpu(bg->bg_free_bits_count) > le16_to_cpu(bg->bg_bits)) { + return ocfs2_error(alloc_inode->i_sb, "Group descriptor # %llu has bit count %u but claims %u are freed. num_bits %d\n", + (unsigned long long)le64_to_cpu(bg->bg_blkno), + le16_to_cpu(bg->bg_bits), + le16_to_cpu(bg->bg_free_bits_count), + num_bits); + } + while(num_bits--) + ocfs2_set_bit(bit_off++, bitmap); + + /* + * this is optimize path, caller set old contig value + * in max_contig_bits to bypass finding action. + */ + if (fastpath) { + bg->bg_contig_free_bits = cpu_to_le16(max_contig_bits); + } else if (ocfs2_is_cluster_bitmap(alloc_inode)) { + /* + * Usually, the block group bitmap allocates only 1 bit + * at a time, while the cluster group allocates n bits + * each time. Therefore, we only save the contig bits for + * the cluster group. + */ + contig_bits = ocfs2_find_max_contig_free_bits(bitmap, + le16_to_cpu(bg->bg_bits), start); + if (contig_bits > max_contig_bits) + max_contig_bits = contig_bits; + bg->bg_contig_free_bits = cpu_to_le16(max_contig_bits); + ocfs2_local_alloc_seen_free_bits(osb, max_contig_bits); + } else { + bg->bg_contig_free_bits = 0; + } + + ocfs2_journal_dirty(handle, group_bh); + +bail: + return status; +} + +/* find the one with the most empty bits */ +static inline u16 ocfs2_find_victim_chain(struct ocfs2_chain_list *cl) +{ + u16 curr, best; + + BUG_ON(!cl->cl_next_free_rec); + + best = curr = 0; + while (curr < le16_to_cpu(cl->cl_next_free_rec)) { + if (le32_to_cpu(cl->cl_recs[curr].c_free) > + le32_to_cpu(cl->cl_recs[best].c_free)) + best = curr; + curr++; + } + + BUG_ON(best >= le16_to_cpu(cl->cl_next_free_rec)); + return best; +} + +static int ocfs2_relink_block_group(handle_t *handle, + struct inode *alloc_inode, + struct buffer_head *fe_bh, + struct buffer_head *bg_bh, + struct buffer_head *prev_bg_bh, + u16 chain) +{ + int status; + /* there is a really tiny chance the journal calls could fail, + * but we wouldn't want inconsistent blocks in *any* case. */ + u64 bg_ptr, prev_bg_ptr; + struct ocfs2_dinode *fe = (struct ocfs2_dinode *) fe_bh->b_data; + struct ocfs2_group_desc *bg = (struct ocfs2_group_desc *) bg_bh->b_data; + struct ocfs2_group_desc *prev_bg = (struct ocfs2_group_desc *) prev_bg_bh->b_data; + + /* The caller got these descriptors from + * ocfs2_read_group_descriptor(). Any corruption is a code bug. */ + BUG_ON(!OCFS2_IS_VALID_GROUP_DESC(bg)); + BUG_ON(!OCFS2_IS_VALID_GROUP_DESC(prev_bg)); + + trace_ocfs2_relink_block_group( + (unsigned long long)le64_to_cpu(fe->i_blkno), chain, + (unsigned long long)le64_to_cpu(bg->bg_blkno), + (unsigned long long)le64_to_cpu(prev_bg->bg_blkno)); + + bg_ptr = le64_to_cpu(bg->bg_next_group); + prev_bg_ptr = le64_to_cpu(prev_bg->bg_next_group); + + status = ocfs2_journal_access_gd(handle, INODE_CACHE(alloc_inode), + prev_bg_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) + goto out; + + prev_bg->bg_next_group = bg->bg_next_group; + ocfs2_journal_dirty(handle, prev_bg_bh); + + status = ocfs2_journal_access_gd(handle, INODE_CACHE(alloc_inode), + bg_bh, OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) + goto out_rollback_prev_bg; + + bg->bg_next_group = fe->id2.i_chain.cl_recs[chain].c_blkno; + ocfs2_journal_dirty(handle, bg_bh); + + status = ocfs2_journal_access_di(handle, INODE_CACHE(alloc_inode), + fe_bh, OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) + goto out_rollback_bg; + + fe->id2.i_chain.cl_recs[chain].c_blkno = bg->bg_blkno; + ocfs2_journal_dirty(handle, fe_bh); + +out: + if (status < 0) + mlog_errno(status); + return status; + +out_rollback_bg: + bg->bg_next_group = cpu_to_le64(bg_ptr); +out_rollback_prev_bg: + prev_bg->bg_next_group = cpu_to_le64(prev_bg_ptr); + goto out; +} + +static inline int ocfs2_block_group_reasonably_empty(struct ocfs2_group_desc *bg, + u32 wanted) +{ + return le16_to_cpu(bg->bg_free_bits_count) > wanted; +} + +/* return 0 on success, -ENOSPC to keep searching and any other < 0 + * value on error. */ +static int ocfs2_cluster_group_search(struct inode *inode, + struct buffer_head *group_bh, + u32 bits_wanted, u32 min_bits, + u64 max_block, + struct ocfs2_suballoc_result *res) +{ + int search = -ENOSPC; + int ret; + u64 blkoff; + struct ocfs2_group_desc *gd = (struct ocfs2_group_desc *) group_bh->b_data; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + unsigned int max_bits, gd_cluster_off; + + BUG_ON(!ocfs2_is_cluster_bitmap(inode)); + + if (le16_to_cpu(gd->bg_contig_free_bits) && + le16_to_cpu(gd->bg_contig_free_bits) < bits_wanted) + return -ENOSPC; + + /* ->bg_contig_free_bits may un-initialized, so compare again */ + if (le16_to_cpu(gd->bg_free_bits_count) >= bits_wanted) { + max_bits = le16_to_cpu(gd->bg_bits); + + /* Tail groups in cluster bitmaps which aren't cpg + * aligned are prone to partial extension by a failed + * fs resize. If the file system resize never got to + * update the dinode cluster count, then we don't want + * to trust any clusters past it, regardless of what + * the group descriptor says. */ + gd_cluster_off = ocfs2_blocks_to_clusters(inode->i_sb, + le64_to_cpu(gd->bg_blkno)); + if ((gd_cluster_off + max_bits) > + OCFS2_I(inode)->ip_clusters) { + max_bits = OCFS2_I(inode)->ip_clusters - gd_cluster_off; + trace_ocfs2_cluster_group_search_wrong_max_bits( + (unsigned long long)le64_to_cpu(gd->bg_blkno), + le16_to_cpu(gd->bg_bits), + OCFS2_I(inode)->ip_clusters, max_bits); + } + + ret = ocfs2_block_group_find_clear_bits(osb, + group_bh, bits_wanted, + max_bits, res); + if (ret) + return ret; + + if (max_block) { + blkoff = ocfs2_clusters_to_blocks(inode->i_sb, + gd_cluster_off + + res->sr_bit_offset + + res->sr_bits); + trace_ocfs2_cluster_group_search_max_block( + (unsigned long long)blkoff, + (unsigned long long)max_block); + if (blkoff > max_block) + return -ENOSPC; + } + + /* ocfs2_block_group_find_clear_bits() might + * return success, but we still want to return + * -ENOSPC unless it found the minimum number + * of bits. */ + if (min_bits <= res->sr_bits) + search = 0; /* success */ + } + + return search; +} + +static int ocfs2_block_group_search(struct inode *inode, + struct buffer_head *group_bh, + u32 bits_wanted, u32 min_bits, + u64 max_block, + struct ocfs2_suballoc_result *res) +{ + int ret = -ENOSPC; + u64 blkoff; + struct ocfs2_group_desc *bg = (struct ocfs2_group_desc *) group_bh->b_data; + + BUG_ON(min_bits != 1); + BUG_ON(ocfs2_is_cluster_bitmap(inode)); + + if (le16_to_cpu(bg->bg_free_bits_count) >= bits_wanted) { + ret = ocfs2_block_group_find_clear_bits(OCFS2_SB(inode->i_sb), + group_bh, bits_wanted, + le16_to_cpu(bg->bg_bits), + res); + if (!ret && max_block) { + blkoff = le64_to_cpu(bg->bg_blkno) + + res->sr_bit_offset + res->sr_bits; + trace_ocfs2_block_group_search_max_block( + (unsigned long long)blkoff, + (unsigned long long)max_block); + if (blkoff > max_block) + ret = -ENOSPC; + } + } + + return ret; +} + +int ocfs2_alloc_dinode_update_counts(struct inode *inode, + handle_t *handle, + struct buffer_head *di_bh, + u32 num_bits, + u16 chain) +{ + int ret; + u32 tmp_used; + struct ocfs2_dinode *di = (struct ocfs2_dinode *) di_bh->b_data; + struct ocfs2_chain_list *cl = (struct ocfs2_chain_list *) &di->id2.i_chain; + + ret = ocfs2_journal_access_di(handle, INODE_CACHE(inode), di_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + tmp_used = le32_to_cpu(di->id1.bitmap1.i_used); + di->id1.bitmap1.i_used = cpu_to_le32(num_bits + tmp_used); + le32_add_cpu(&cl->cl_recs[chain].c_free, -num_bits); + ocfs2_journal_dirty(handle, di_bh); + +out: + return ret; +} + +void ocfs2_rollback_alloc_dinode_counts(struct inode *inode, + struct buffer_head *di_bh, + u32 num_bits, + u16 chain) +{ + u32 tmp_used; + struct ocfs2_dinode *di = (struct ocfs2_dinode *) di_bh->b_data; + struct ocfs2_chain_list *cl; + + cl = (struct ocfs2_chain_list *)&di->id2.i_chain; + tmp_used = le32_to_cpu(di->id1.bitmap1.i_used); + di->id1.bitmap1.i_used = cpu_to_le32(tmp_used - num_bits); + le32_add_cpu(&cl->cl_recs[chain].c_free, num_bits); +} + +static int ocfs2_bg_discontig_fix_by_rec(struct ocfs2_suballoc_result *res, + struct ocfs2_extent_rec *rec, + struct ocfs2_chain_list *cl) +{ + unsigned int bpc = le16_to_cpu(cl->cl_bpc); + unsigned int bitoff = le32_to_cpu(rec->e_cpos) * bpc; + unsigned int bitcount = le16_to_cpu(rec->e_leaf_clusters) * bpc; + + if (res->sr_bit_offset < bitoff) + return 0; + if (res->sr_bit_offset >= (bitoff + bitcount)) + return 0; + res->sr_blkno = le64_to_cpu(rec->e_blkno) + + (res->sr_bit_offset - bitoff); + if ((res->sr_bit_offset + res->sr_bits) > (bitoff + bitcount)) + res->sr_bits = (bitoff + bitcount) - res->sr_bit_offset; + return 1; +} + +static void ocfs2_bg_discontig_fix_result(struct ocfs2_alloc_context *ac, + struct ocfs2_group_desc *bg, + struct ocfs2_suballoc_result *res) +{ + int i; + u64 bg_blkno = res->sr_bg_blkno; /* Save off */ + struct ocfs2_extent_rec *rec; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)ac->ac_bh->b_data; + struct ocfs2_chain_list *cl = &di->id2.i_chain; + + if (ocfs2_is_cluster_bitmap(ac->ac_inode)) { + res->sr_blkno = 0; + return; + } + + res->sr_blkno = res->sr_bg_blkno + res->sr_bit_offset; + res->sr_bg_blkno = 0; /* Clear it for contig block groups */ + if (!ocfs2_supports_discontig_bg(OCFS2_SB(ac->ac_inode->i_sb)) || + !bg->bg_list.l_next_free_rec) + return; + + for (i = 0; i < le16_to_cpu(bg->bg_list.l_next_free_rec); i++) { + rec = &bg->bg_list.l_recs[i]; + if (ocfs2_bg_discontig_fix_by_rec(res, rec, cl)) { + res->sr_bg_blkno = bg_blkno; /* Restore */ + break; + } + } +} + +static int ocfs2_search_one_group(struct ocfs2_alloc_context *ac, + handle_t *handle, + u32 bits_wanted, + u32 min_bits, + struct ocfs2_suballoc_result *res, + u16 *bits_left, int *released) +{ + int ret; + struct buffer_head *group_bh = NULL; + struct ocfs2_group_desc *gd; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)ac->ac_bh->b_data; + struct inode *alloc_inode = ac->ac_inode; + + ret = ocfs2_read_hint_group_descriptor(alloc_inode, di, + res->sr_bg_blkno, &group_bh, released); + if (*released) { + return 0; + } else if (ret < 0) { + mlog_errno(ret); + return ret; + } + + gd = (struct ocfs2_group_desc *) group_bh->b_data; + ret = ac->ac_group_search(alloc_inode, group_bh, bits_wanted, min_bits, + ac->ac_max_block, res); + if (ret < 0) { + if (ret != -ENOSPC) + mlog_errno(ret); + goto out; + } + + if (!ret) + ocfs2_bg_discontig_fix_result(ac, gd, res); + + /* + * sr_bg_blkno might have been changed by + * ocfs2_bg_discontig_fix_result + */ + res->sr_bg_stable_blkno = group_bh->b_blocknr; + + if (ac->ac_find_loc_only) + goto out_loc_only; + + ret = ocfs2_alloc_dinode_update_counts(alloc_inode, handle, ac->ac_bh, + res->sr_bits, + le16_to_cpu(gd->bg_chain)); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_block_group_set_bits(handle, alloc_inode, gd, group_bh, + res->sr_bit_offset, res->sr_bits, + res->sr_max_contig_bits, 0); + if (ret < 0) { + ocfs2_rollback_alloc_dinode_counts(alloc_inode, ac->ac_bh, + res->sr_bits, + le16_to_cpu(gd->bg_chain)); + mlog_errno(ret); + } + +out_loc_only: + *bits_left = le16_to_cpu(gd->bg_free_bits_count); + +out: + brelse(group_bh); + + return ret; +} + +static int ocfs2_search_chain(struct ocfs2_alloc_context *ac, + handle_t *handle, + u32 bits_wanted, + u32 min_bits, + struct ocfs2_suballoc_result *res, + u16 *bits_left) +{ + int status; + u16 chain; + u32 contig_bits; + u64 next_group; + struct inode *alloc_inode = ac->ac_inode; + struct buffer_head *group_bh = NULL; + struct buffer_head *prev_group_bh = NULL; + struct ocfs2_dinode *fe = (struct ocfs2_dinode *) ac->ac_bh->b_data; + struct ocfs2_chain_list *cl = (struct ocfs2_chain_list *) &fe->id2.i_chain; + struct ocfs2_group_desc *bg; + + chain = ac->ac_chain; + trace_ocfs2_search_chain_begin( + (unsigned long long)OCFS2_I(alloc_inode)->ip_blkno, + bits_wanted, chain); + + status = ocfs2_read_group_descriptor(alloc_inode, fe, + le64_to_cpu(cl->cl_recs[chain].c_blkno), + &group_bh); + if (status < 0) { + mlog_errno(status); + goto bail; + } + bg = (struct ocfs2_group_desc *) group_bh->b_data; + + status = -ENOSPC; + /* for now, the chain search is a bit simplistic. We just use + * the 1st group with any empty bits. */ + while (1) { + if (ac->ac_which == OCFS2_AC_USE_MAIN_DISCONTIG) { + contig_bits = le16_to_cpu(bg->bg_contig_free_bits); + if (!contig_bits) + contig_bits = ocfs2_find_max_contig_free_bits(bg->bg_bitmap, + le16_to_cpu(bg->bg_bits), 0); + if (bits_wanted > contig_bits && contig_bits >= min_bits) + bits_wanted = contig_bits; + } + + status = ac->ac_group_search(alloc_inode, group_bh, + bits_wanted, min_bits, + ac->ac_max_block, res); + if (status != -ENOSPC) + break; + if (!bg->bg_next_group) + break; + + brelse(prev_group_bh); + prev_group_bh = NULL; + + next_group = le64_to_cpu(bg->bg_next_group); + prev_group_bh = group_bh; + group_bh = NULL; + status = ocfs2_read_group_descriptor(alloc_inode, fe, + next_group, &group_bh); + if (status < 0) { + mlog_errno(status); + goto bail; + } + bg = (struct ocfs2_group_desc *) group_bh->b_data; + } + if (status < 0) { + if (status != -ENOSPC) + mlog_errno(status); + goto bail; + } + + trace_ocfs2_search_chain_succ( + (unsigned long long)le64_to_cpu(bg->bg_blkno), res->sr_bits); + + res->sr_bg_blkno = le64_to_cpu(bg->bg_blkno); + + BUG_ON(res->sr_bits == 0); + if (!status) + ocfs2_bg_discontig_fix_result(ac, bg, res); + + /* + * sr_bg_blkno might have been changed by + * ocfs2_bg_discontig_fix_result + */ + res->sr_bg_stable_blkno = group_bh->b_blocknr; + + /* + * Keep track of previous block descriptor read. When + * we find a target, if we have read more than X + * number of descriptors, and the target is reasonably + * empty, relink him to top of his chain. + * + * We've read 0 extra blocks and only send one more to + * the transaction, yet the next guy to search has a + * much easier time. + * + * Do this *after* figuring out how many bits we're taking out + * of our target group. + */ + if (!ac->ac_disable_chain_relink && + (prev_group_bh) && + (ocfs2_block_group_reasonably_empty(bg, res->sr_bits))) { + status = ocfs2_relink_block_group(handle, alloc_inode, + ac->ac_bh, group_bh, + prev_group_bh, chain); + if (status < 0) { + mlog_errno(status); + goto bail; + } + } + + if (ac->ac_find_loc_only) + goto out_loc_only; + + status = ocfs2_alloc_dinode_update_counts(alloc_inode, handle, + ac->ac_bh, res->sr_bits, + chain); + if (status) { + mlog_errno(status); + goto bail; + } + + status = ocfs2_block_group_set_bits(handle, + alloc_inode, + bg, + group_bh, + res->sr_bit_offset, + res->sr_bits, + res->sr_max_contig_bits, + 0); + if (status < 0) { + ocfs2_rollback_alloc_dinode_counts(alloc_inode, + ac->ac_bh, res->sr_bits, chain); + mlog_errno(status); + goto bail; + } + + trace_ocfs2_search_chain_end( + (unsigned long long)le64_to_cpu(fe->i_blkno), + res->sr_bits); + +out_loc_only: + *bits_left = le16_to_cpu(bg->bg_free_bits_count); +bail: + brelse(group_bh); + brelse(prev_group_bh); + + if (status) + mlog_errno(status); + return status; +} + +/* will give out up to bits_wanted contiguous bits. */ +static int ocfs2_claim_suballoc_bits(struct ocfs2_alloc_context *ac, + handle_t *handle, + u32 bits_wanted, + u32 min_bits, + struct ocfs2_suballoc_result *res) +{ + int status; + int released = 0; + u16 victim, i; + u16 bits_left = 0; + u64 hint = ac->ac_last_group; + struct ocfs2_chain_list *cl; + struct ocfs2_dinode *fe; + + BUG_ON(ac->ac_bits_given >= ac->ac_bits_wanted); + BUG_ON(bits_wanted > (ac->ac_bits_wanted - ac->ac_bits_given)); + BUG_ON(!ac->ac_bh); + + fe = (struct ocfs2_dinode *) ac->ac_bh->b_data; + + /* The bh was validated by the inode read during + * ocfs2_reserve_suballoc_bits(). Any corruption is a code bug. */ + BUG_ON(!OCFS2_IS_VALID_DINODE(fe)); + + if (le32_to_cpu(fe->id1.bitmap1.i_used) >= + le32_to_cpu(fe->id1.bitmap1.i_total)) { + status = ocfs2_error(ac->ac_inode->i_sb, + "Chain allocator dinode %llu has %u used bits but only %u total\n", + (unsigned long long)le64_to_cpu(fe->i_blkno), + le32_to_cpu(fe->id1.bitmap1.i_used), + le32_to_cpu(fe->id1.bitmap1.i_total)); + goto bail; + } + + /* the hint bg may already be released, we quiet search this group. */ + res->sr_bg_blkno = hint; + if (res->sr_bg_blkno) { + /* Attempt to short-circuit the usual search mechanism + * by jumping straight to the most recently used + * allocation group. This helps us maintain some + * contiguousness across allocations. */ + status = ocfs2_search_one_group(ac, handle, bits_wanted, + min_bits, res, &bits_left, + &released); + if (released) { + res->sr_bg_blkno = 0; + goto chain_search; + } + if (!status) + goto set_hint; + if (status < 0 && status != -ENOSPC) { + mlog_errno(status); + goto bail; + } + } +chain_search: + cl = (struct ocfs2_chain_list *) &fe->id2.i_chain; + if (!le16_to_cpu(cl->cl_next_free_rec) || + le16_to_cpu(cl->cl_next_free_rec) > le16_to_cpu(cl->cl_count)) { + status = ocfs2_error(ac->ac_inode->i_sb, + "Chain allocator dinode %llu has invalid next " + "free chain record %u, but only %u total\n", + (unsigned long long)le64_to_cpu(fe->i_blkno), + le16_to_cpu(cl->cl_next_free_rec), + le16_to_cpu(cl->cl_count)); + goto bail; + } + + victim = ocfs2_find_victim_chain(cl); + ac->ac_chain = victim; + +search: + status = ocfs2_search_chain(ac, handle, bits_wanted, min_bits, + res, &bits_left); + if (!status) { + if (ocfs2_is_cluster_bitmap(ac->ac_inode)) + hint = res->sr_bg_blkno; + else + hint = ocfs2_group_from_res(res); + goto set_hint; + } + if (status < 0 && status != -ENOSPC) { + mlog_errno(status); + goto bail; + } + + trace_ocfs2_claim_suballoc_bits(victim); + + /* If we didn't pick a good victim, then just default to + * searching each chain in order. Don't allow chain relinking + * because we only calculate enough journal credits for one + * relink per alloc. */ + ac->ac_disable_chain_relink = 1; + for (i = 0; i < le16_to_cpu(cl->cl_next_free_rec); i ++) { + if (i == victim) + continue; + if (le32_to_cpu(cl->cl_recs[i].c_free) < bits_wanted) + continue; + + ac->ac_chain = i; + status = ocfs2_search_chain(ac, handle, bits_wanted, min_bits, + res, &bits_left); + if (!status) { + hint = ocfs2_group_from_res(res); + break; + } + if (status < 0 && status != -ENOSPC) { + mlog_errno(status); + goto bail; + } + } + + /* Chains can't supply the bits_wanted contiguous space. + * We should switch to using every single bit when allocating + * from the global bitmap. */ + if (i == le16_to_cpu(cl->cl_next_free_rec) && + status == -ENOSPC && ac->ac_which == OCFS2_AC_USE_MAIN) { + ac->ac_which = OCFS2_AC_USE_MAIN_DISCONTIG; + ac->ac_chain = victim; + goto search; + } + +set_hint: + if (status != -ENOSPC) { + /* If the next search of this group is not likely to + * yield a suitable extent, then we reset the last + * group hint so as to not waste a disk read */ + if (bits_left < min_bits) + ac->ac_last_group = 0; + else + ac->ac_last_group = hint; + } + +bail: + if (status) + mlog_errno(status); + return status; +} + +int ocfs2_claim_metadata(handle_t *handle, + struct ocfs2_alloc_context *ac, + u32 bits_wanted, + u64 *suballoc_loc, + u16 *suballoc_bit_start, + unsigned int *num_bits, + u64 *blkno_start) +{ + int status; + struct ocfs2_suballoc_result res = { .sr_blkno = 0, }; + + BUG_ON(!ac); + BUG_ON(ac->ac_bits_wanted < (ac->ac_bits_given + bits_wanted)); + BUG_ON(ac->ac_which != OCFS2_AC_USE_META); + + status = ocfs2_claim_suballoc_bits(ac, + handle, + bits_wanted, + 1, + &res); + if (status < 0) { + mlog_errno(status); + goto bail; + } + atomic_inc(&OCFS2_SB(ac->ac_inode->i_sb)->alloc_stats.bg_allocs); + + *suballoc_loc = res.sr_bg_blkno; + *suballoc_bit_start = res.sr_bit_offset; + *blkno_start = res.sr_blkno; + ac->ac_bits_given += res.sr_bits; + *num_bits = res.sr_bits; + status = 0; +bail: + if (status) + mlog_errno(status); + return status; +} + +/* + * after ocfs2 has the ability to release block group unused space, + * the ->ip_last_used_group may be invalid. so this function returns + * ac->ac_last_group need to verify. + * refer the 'hint' in ocfs2_claim_suballoc_bits() for more details. + */ +static void ocfs2_init_inode_ac_group(struct inode *dir, + struct buffer_head *parent_di_bh, + struct ocfs2_alloc_context *ac) +{ + struct ocfs2_dinode *di = (struct ocfs2_dinode *)parent_di_bh->b_data; + /* + * Try to allocate inodes from some specific group. + * + * If the parent dir has recorded the last group used in allocation, + * cool, use it. Otherwise if we try to allocate new inode from the + * same slot the parent dir belongs to, use the same chunk. + * + * We are very careful here to avoid the mistake of setting + * ac_last_group to a group descriptor from a different (unlocked) slot. + */ + if (OCFS2_I(dir)->ip_last_used_group && + OCFS2_I(dir)->ip_last_used_slot == ac->ac_alloc_slot) + ac->ac_last_group = OCFS2_I(dir)->ip_last_used_group; + else if (le16_to_cpu(di->i_suballoc_slot) == ac->ac_alloc_slot) { + if (di->i_suballoc_loc) + ac->ac_last_group = le64_to_cpu(di->i_suballoc_loc); + else + ac->ac_last_group = ocfs2_which_suballoc_group( + le64_to_cpu(di->i_blkno), + le16_to_cpu(di->i_suballoc_bit)); + } +} + +static inline void ocfs2_save_inode_ac_group(struct inode *dir, + struct ocfs2_alloc_context *ac) +{ + OCFS2_I(dir)->ip_last_used_group = ac->ac_last_group; + OCFS2_I(dir)->ip_last_used_slot = ac->ac_alloc_slot; +} + +int ocfs2_find_new_inode_loc(struct inode *dir, + struct buffer_head *parent_fe_bh, + struct ocfs2_alloc_context *ac, + u64 *fe_blkno) +{ + int ret; + handle_t *handle = NULL; + struct ocfs2_suballoc_result *res; + + BUG_ON(!ac); + BUG_ON(ac->ac_bits_given != 0); + BUG_ON(ac->ac_bits_wanted != 1); + BUG_ON(ac->ac_which != OCFS2_AC_USE_INODE); + + res = kzalloc_obj(*res, GFP_NOFS); + if (res == NULL) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + ocfs2_init_inode_ac_group(dir, parent_fe_bh, ac); + + /* + * The handle started here is for chain relink. Alternatively, + * we could just disable relink for these calls. + */ + handle = ocfs2_start_trans(OCFS2_SB(dir->i_sb), OCFS2_SUBALLOC_ALLOC); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + handle = NULL; + mlog_errno(ret); + goto out; + } + + /* + * This will instruct ocfs2_claim_suballoc_bits and + * ocfs2_search_one_group to search but save actual allocation + * for later. + */ + ac->ac_find_loc_only = 1; + + ret = ocfs2_claim_suballoc_bits(ac, handle, 1, 1, res); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + ac->ac_find_loc_priv = res; + *fe_blkno = res->sr_blkno; + ocfs2_update_inode_fsync_trans(handle, dir, 0); +out: + if (handle) + ocfs2_commit_trans(OCFS2_SB(dir->i_sb), handle); + + if (ret) + kfree(res); + + return ret; +} + +int ocfs2_claim_new_inode_at_loc(handle_t *handle, + struct inode *dir, + struct ocfs2_alloc_context *ac, + u64 *suballoc_loc, + u16 *suballoc_bit, + u64 di_blkno) +{ + int ret; + u16 chain; + struct ocfs2_suballoc_result *res = ac->ac_find_loc_priv; + struct buffer_head *bg_bh = NULL; + struct ocfs2_group_desc *bg; + struct ocfs2_dinode *di = (struct ocfs2_dinode *) ac->ac_bh->b_data; + + /* + * Since di_blkno is being passed back in, we check for any + * inconsistencies which may have happened between + * calls. These are code bugs as di_blkno is not expected to + * change once returned from ocfs2_find_new_inode_loc() + */ + BUG_ON(res->sr_blkno != di_blkno); + + ret = ocfs2_read_group_descriptor(ac->ac_inode, di, + res->sr_bg_stable_blkno, &bg_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + bg = (struct ocfs2_group_desc *) bg_bh->b_data; + chain = le16_to_cpu(bg->bg_chain); + + ret = ocfs2_alloc_dinode_update_counts(ac->ac_inode, handle, + ac->ac_bh, res->sr_bits, + chain); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_block_group_set_bits(handle, + ac->ac_inode, + bg, + bg_bh, + res->sr_bit_offset, + res->sr_bits, + res->sr_max_contig_bits, + 0); + if (ret < 0) { + ocfs2_rollback_alloc_dinode_counts(ac->ac_inode, + ac->ac_bh, res->sr_bits, chain); + mlog_errno(ret); + goto out; + } + + trace_ocfs2_claim_new_inode_at_loc((unsigned long long)di_blkno, + res->sr_bits); + + atomic_inc(&OCFS2_SB(ac->ac_inode->i_sb)->alloc_stats.bg_allocs); + + BUG_ON(res->sr_bits != 1); + + *suballoc_loc = res->sr_bg_blkno; + *suballoc_bit = res->sr_bit_offset; + ac->ac_bits_given++; + ocfs2_save_inode_ac_group(dir, ac); + +out: + brelse(bg_bh); + + return ret; +} + +int ocfs2_claim_new_inode(handle_t *handle, + struct inode *dir, + struct buffer_head *parent_fe_bh, + struct ocfs2_alloc_context *ac, + u64 *suballoc_loc, + u16 *suballoc_bit, + u64 *fe_blkno) +{ + int status; + struct ocfs2_suballoc_result res; + + BUG_ON(!ac); + BUG_ON(ac->ac_bits_given != 0); + BUG_ON(ac->ac_bits_wanted != 1); + BUG_ON(ac->ac_which != OCFS2_AC_USE_INODE); + + ocfs2_init_inode_ac_group(dir, parent_fe_bh, ac); + + status = ocfs2_claim_suballoc_bits(ac, + handle, + 1, + 1, + &res); + if (status < 0) { + mlog_errno(status); + goto bail; + } + atomic_inc(&OCFS2_SB(ac->ac_inode->i_sb)->alloc_stats.bg_allocs); + + BUG_ON(res.sr_bits != 1); + + *suballoc_loc = res.sr_bg_blkno; + *suballoc_bit = res.sr_bit_offset; + *fe_blkno = res.sr_blkno; + ac->ac_bits_given++; + ocfs2_save_inode_ac_group(dir, ac); + status = 0; +bail: + if (status) + mlog_errno(status); + return status; +} + +/* translate a group desc. blkno and it's bitmap offset into + * disk cluster offset. */ +static inline u32 ocfs2_desc_bitmap_to_cluster_off(struct inode *inode, + u64 bg_blkno, + u16 bg_bit_off) +{ + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + u32 cluster = 0; + + BUG_ON(!ocfs2_is_cluster_bitmap(inode)); + + if (bg_blkno != osb->first_cluster_group_blkno) + cluster = ocfs2_blocks_to_clusters(inode->i_sb, bg_blkno); + cluster += (u32) bg_bit_off; + return cluster; +} + +/* given a cluster offset, calculate which block group it belongs to + * and return that block offset. */ +u64 ocfs2_which_cluster_group(struct inode *inode, u32 cluster) +{ + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + u32 group_no; + + BUG_ON(!ocfs2_is_cluster_bitmap(inode)); + + group_no = cluster / osb->bitmap_cpg; + if (!group_no) + return osb->first_cluster_group_blkno; + return ocfs2_clusters_to_blocks(inode->i_sb, + group_no * osb->bitmap_cpg); +} + +/* given the block number of a cluster start, calculate which cluster + * group and descriptor bitmap offset that corresponds to. */ +static inline void ocfs2_block_to_cluster_group(struct inode *inode, + u64 data_blkno, + u64 *bg_blkno, + u16 *bg_bit_off) +{ + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + u32 data_cluster = ocfs2_blocks_to_clusters(osb->sb, data_blkno); + + BUG_ON(!ocfs2_is_cluster_bitmap(inode)); + + *bg_blkno = ocfs2_which_cluster_group(inode, + data_cluster); + + if (*bg_blkno == osb->first_cluster_group_blkno) + *bg_bit_off = (u16) data_cluster; + else + *bg_bit_off = (u16) ocfs2_blocks_to_clusters(osb->sb, + data_blkno - *bg_blkno); +} + +/* + * min_bits - minimum contiguous chunk from this total allocation we + * can handle. set to what we asked for originally for a full + * contig. allocation, set to '1' to indicate we can deal with extents + * of any size. + */ +int __ocfs2_claim_clusters(handle_t *handle, + struct ocfs2_alloc_context *ac, + u32 min_clusters, + u32 max_clusters, + u32 *cluster_start, + u32 *num_clusters) +{ + int status; + unsigned int bits_wanted = max_clusters; + struct ocfs2_suballoc_result res = { .sr_blkno = 0, }; + struct ocfs2_super *osb = OCFS2_SB(ac->ac_inode->i_sb); + + BUG_ON(ac->ac_bits_given >= ac->ac_bits_wanted); + + BUG_ON(ac->ac_which != OCFS2_AC_USE_LOCAL + && ac->ac_which != OCFS2_AC_USE_MAIN + && ac->ac_which != OCFS2_AC_USE_MAIN_DISCONTIG); + + if (ac->ac_which == OCFS2_AC_USE_LOCAL) { + WARN_ON(min_clusters > 1); + + status = ocfs2_claim_local_alloc_bits(osb, + handle, + ac, + bits_wanted, + cluster_start, + num_clusters); + if (!status) + atomic_inc(&osb->alloc_stats.local_data); + } else { + if (min_clusters > (osb->bitmap_cpg - 1)) { + /* The only paths asking for contiguousness + * should know about this already. */ + mlog(ML_ERROR, "minimum allocation requested %u exceeds " + "group bitmap size %u!\n", min_clusters, + osb->bitmap_cpg); + status = -ENOSPC; + goto bail; + } + /* clamp the current request down to a realistic size. */ + if (bits_wanted > (osb->bitmap_cpg - 1)) + bits_wanted = osb->bitmap_cpg - 1; + + status = ocfs2_claim_suballoc_bits(ac, + handle, + bits_wanted, + min_clusters, + &res); + if (!status) { + BUG_ON(res.sr_blkno); /* cluster alloc can't set */ + *cluster_start = + ocfs2_desc_bitmap_to_cluster_off(ac->ac_inode, + res.sr_bg_blkno, + res.sr_bit_offset); + atomic_inc(&osb->alloc_stats.bitmap_data); + *num_clusters = res.sr_bits; + } + } + if (status < 0) { + if (status != -ENOSPC) + mlog_errno(status); + goto bail; + } + + ac->ac_bits_given += *num_clusters; + +bail: + if (status) + mlog_errno(status); + return status; +} + +int ocfs2_claim_clusters(handle_t *handle, + struct ocfs2_alloc_context *ac, + u32 min_clusters, + u32 *cluster_start, + u32 *num_clusters) +{ + unsigned int bits_wanted = ac->ac_bits_wanted - ac->ac_bits_given; + + return __ocfs2_claim_clusters(handle, ac, min_clusters, + bits_wanted, cluster_start, num_clusters); +} + +static int ocfs2_block_group_clear_bits(handle_t *handle, + struct inode *alloc_inode, + struct ocfs2_group_desc *bg, + struct buffer_head *group_bh, + unsigned int bit_off, + unsigned int num_bits, + unsigned int max_contig_bits, + void (*undo_fn)(unsigned int bit, + unsigned long *bmap)) +{ + int status; + unsigned int tmp; + u16 contig_bits; + struct ocfs2_group_desc *undo_bg = NULL; + struct journal_head *jh; + + /* The caller got this descriptor from + * ocfs2_read_group_descriptor(). Any corruption is a code bug. */ + BUG_ON(!OCFS2_IS_VALID_GROUP_DESC(bg)); + + trace_ocfs2_block_group_clear_bits(bit_off, num_bits); + + BUG_ON(undo_fn && !ocfs2_is_cluster_bitmap(alloc_inode)); + status = ocfs2_journal_access_gd(handle, INODE_CACHE(alloc_inode), + group_bh, + undo_fn ? + OCFS2_JOURNAL_ACCESS_UNDO : + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + jh = bh2jh(group_bh); + if (undo_fn) { + spin_lock(&jh->b_state_lock); + undo_bg = (struct ocfs2_group_desc *) jh->b_committed_data; + BUG_ON(!undo_bg); + } + + tmp = num_bits; + while(tmp--) { + ocfs2_clear_bit((bit_off + tmp), + (unsigned long *) bg->bg_bitmap); + if (undo_fn) + undo_fn(bit_off + tmp, + (unsigned long *) undo_bg->bg_bitmap); + } + le16_add_cpu(&bg->bg_free_bits_count, num_bits); + if (le16_to_cpu(bg->bg_free_bits_count) > le16_to_cpu(bg->bg_bits)) { + if (undo_fn) + spin_unlock(&jh->b_state_lock); + return ocfs2_error(alloc_inode->i_sb, "Group descriptor # %llu has bit count %u but claims %u are freed. num_bits %d\n", + (unsigned long long)le64_to_cpu(bg->bg_blkno), + le16_to_cpu(bg->bg_bits), + le16_to_cpu(bg->bg_free_bits_count), + num_bits); + } + + /* + * TODO: even 'num_bits == 1' (the worst case, release 1 cluster), + * we still need to rescan whole bitmap. + */ + if (ocfs2_is_cluster_bitmap(alloc_inode)) { + contig_bits = ocfs2_find_max_contig_free_bits(bg->bg_bitmap, + le16_to_cpu(bg->bg_bits), 0); + if (contig_bits > max_contig_bits) + max_contig_bits = contig_bits; + bg->bg_contig_free_bits = cpu_to_le16(max_contig_bits); + } else { + bg->bg_contig_free_bits = 0; + } + + if (undo_fn) + spin_unlock(&jh->b_state_lock); + + ocfs2_journal_dirty(handle, group_bh); +bail: + return status; +} + +/* + * Reclaim the suballocator managed space to main bitmap. + * This function first works on the suballocator to perform the + * cleanup rec/alloc_inode job, then switches to the main bitmap + * to reclaim released space. + * + * handle: The transaction handle + * alloc_inode: The suballoc inode + * alloc_bh: The buffer_head of suballoc inode + * group_bh: The group descriptor buffer_head of suballocator managed. + * Caller should release the input group_bh. + */ +static int _ocfs2_reclaim_suballoc_to_main(handle_t *handle, + struct inode *alloc_inode, + struct buffer_head *alloc_bh, + struct buffer_head *group_bh) +{ + int idx, status = 0; + int i, next_free_rec, len = 0; + __le16 old_bg_contig_free_bits = 0; + u16 start_bit; + u32 tmp_used; + u64 bg_blkno, start_blk; + unsigned int count; + struct ocfs2_chain_rec *rec; + struct buffer_head *main_bm_bh = NULL; + struct inode *main_bm_inode = NULL; + struct ocfs2_super *osb = OCFS2_SB(alloc_inode->i_sb); + struct ocfs2_dinode *fe = (struct ocfs2_dinode *) alloc_bh->b_data; + struct ocfs2_chain_list *cl = &fe->id2.i_chain; + struct ocfs2_group_desc *group = (struct ocfs2_group_desc *) group_bh->b_data; + + idx = le16_to_cpu(group->bg_chain); + rec = &(cl->cl_recs[idx]); + + status = ocfs2_extend_trans(handle, + ocfs2_calc_group_alloc_credits(osb->sb, + le16_to_cpu(cl->cl_cpg))); + if (status) { + mlog_errno(status); + goto bail; + } + status = ocfs2_journal_access_di(handle, INODE_CACHE(alloc_inode), + alloc_bh, OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + /* + * Only clear the suballocator rec item in-place. + * + * If idx is not the last, we don't compress (remove the empty item) + * the cl_recs[]. If not, we need to do lots jobs. + * + * Compress cl_recs[] code example: + * if (idx != cl->cl_next_free_rec - 1) + * memmove(&cl->cl_recs[idx], &cl->cl_recs[idx + 1], + * sizeof(struct ocfs2_chain_rec) * + * (cl->cl_next_free_rec - idx - 1)); + * for(i = idx; i < cl->cl_next_free_rec-1; i++) { + * group->bg_chain = "later group->bg_chain"; + * group->bg_blkno = xxx; + * ... ... + * } + */ + + tmp_used = le32_to_cpu(fe->id1.bitmap1.i_total); + fe->id1.bitmap1.i_total = cpu_to_le32(tmp_used - le32_to_cpu(rec->c_total)); + + /* Substraction 1 for the block group itself */ + tmp_used = le32_to_cpu(fe->id1.bitmap1.i_used); + fe->id1.bitmap1.i_used = cpu_to_le32(tmp_used - 1); + + tmp_used = le32_to_cpu(fe->i_clusters); + fe->i_clusters = cpu_to_le32(tmp_used - le16_to_cpu(cl->cl_cpg)); + + spin_lock(&OCFS2_I(alloc_inode)->ip_lock); + OCFS2_I(alloc_inode)->ip_clusters = le32_to_cpu(fe->i_clusters); + fe->i_size = cpu_to_le64(ocfs2_clusters_to_bytes(alloc_inode->i_sb, + le32_to_cpu(fe->i_clusters))); + spin_unlock(&OCFS2_I(alloc_inode)->ip_lock); + i_size_write(alloc_inode, le64_to_cpu(fe->i_size)); + alloc_inode->i_blocks = ocfs2_inode_sector_count(alloc_inode); + + ocfs2_journal_dirty(handle, alloc_bh); + ocfs2_update_inode_fsync_trans(handle, alloc_inode, 0); + + start_blk = le64_to_cpu(rec->c_blkno); + count = le32_to_cpu(rec->c_total) / le16_to_cpu(cl->cl_bpc); + + /* + * If the rec is the last one, let's compress the chain list by + * removing the empty cl_recs[] at the end. + */ + next_free_rec = le16_to_cpu(cl->cl_next_free_rec); + if (idx == (next_free_rec - 1)) { + len++; /* the last item should be counted first */ + for (i = (next_free_rec - 2); i > 0; i--) { + if (cl->cl_recs[i].c_free == cl->cl_recs[i].c_total) + len++; + else + break; + } + } + le16_add_cpu(&cl->cl_next_free_rec, -len); + + rec->c_free = 0; + rec->c_total = 0; + rec->c_blkno = 0; + ocfs2_remove_from_cache(INODE_CACHE(alloc_inode), group_bh); + memset(group, 0, sizeof(struct ocfs2_group_desc)); + + /* prepare job for reclaim clusters */ + main_bm_inode = ocfs2_get_system_file_inode(osb, + GLOBAL_BITMAP_SYSTEM_INODE, + OCFS2_INVALID_SLOT); + if (!main_bm_inode) + goto bail; /* ignore the error in reclaim path */ + + inode_lock(main_bm_inode); + + status = ocfs2_inode_lock(main_bm_inode, &main_bm_bh, 1); + if (status < 0) + goto free_bm_inode; /* ignore the error in reclaim path */ + + ocfs2_block_to_cluster_group(main_bm_inode, start_blk, &bg_blkno, + &start_bit); + fe = (struct ocfs2_dinode *) main_bm_bh->b_data; + cl = &fe->id2.i_chain; + /* reuse group_bh, caller will release the input group_bh */ + group_bh = NULL; + + /* reclaim clusters to global_bitmap */ + status = ocfs2_read_group_descriptor(main_bm_inode, fe, bg_blkno, + &group_bh); + if (status < 0) { + mlog_errno(status); + goto free_bm_bh; + } + group = (struct ocfs2_group_desc *) group_bh->b_data; + + if ((count + start_bit) > le16_to_cpu(group->bg_bits)) { + ocfs2_error(alloc_inode->i_sb, + "reclaim length (%d) beyands block group length (%d)", + count + start_bit, le16_to_cpu(group->bg_bits)); + goto free_group_bh; + } + + old_bg_contig_free_bits = group->bg_contig_free_bits; + status = ocfs2_block_group_clear_bits(handle, main_bm_inode, + group, group_bh, + start_bit, count, 0, + _ocfs2_clear_bit); + if (status < 0) { + mlog_errno(status); + goto free_group_bh; + } + + status = ocfs2_journal_access_di(handle, INODE_CACHE(main_bm_inode), + main_bm_bh, OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + ocfs2_block_group_set_bits(handle, main_bm_inode, group, group_bh, + start_bit, count, + le16_to_cpu(old_bg_contig_free_bits), 1); + goto free_group_bh; + } + + idx = le16_to_cpu(group->bg_chain); + rec = &(cl->cl_recs[idx]); + + le32_add_cpu(&rec->c_free, count); + tmp_used = le32_to_cpu(fe->id1.bitmap1.i_used); + fe->id1.bitmap1.i_used = cpu_to_le32(tmp_used - count); + ocfs2_journal_dirty(handle, main_bm_bh); + +free_group_bh: + brelse(group_bh); + +free_bm_bh: + ocfs2_inode_unlock(main_bm_inode, 1); + brelse(main_bm_bh); + +free_bm_inode: + inode_unlock(main_bm_inode); + iput(main_bm_inode); + +bail: + return status; +} + +/* + * expects the suballoc inode to already be locked. + */ +static int _ocfs2_free_suballoc_bits(handle_t *handle, + struct inode *alloc_inode, + struct buffer_head *alloc_bh, + unsigned int start_bit, + u64 bg_blkno, + unsigned int count, + void (*undo_fn)(unsigned int bit, + unsigned long *bitmap)) +{ + int idx, status = 0; + u32 tmp_used; + struct ocfs2_dinode *fe = (struct ocfs2_dinode *) alloc_bh->b_data; + struct ocfs2_chain_list *cl = &fe->id2.i_chain; + struct buffer_head *group_bh = NULL; + struct ocfs2_group_desc *group; + struct ocfs2_chain_rec *rec; + __le16 old_bg_contig_free_bits = 0; + + /* The alloc_bh comes from ocfs2_free_dinode() or + * ocfs2_free_clusters(). The callers have all locked the + * allocator and gotten alloc_bh from the lock call. This + * validates the dinode buffer. Any corruption that has happened + * is a code bug. */ + BUG_ON(!OCFS2_IS_VALID_DINODE(fe)); + BUG_ON((count + start_bit) > ocfs2_bits_per_group(cl)); + + trace_ocfs2_free_suballoc_bits( + (unsigned long long)OCFS2_I(alloc_inode)->ip_blkno, + (unsigned long long)bg_blkno, + start_bit, count); + + status = ocfs2_read_group_descriptor(alloc_inode, fe, bg_blkno, + &group_bh); + if (status < 0) { + mlog_errno(status); + goto bail; + } + group = (struct ocfs2_group_desc *) group_bh->b_data; + + BUG_ON((count + start_bit) > le16_to_cpu(group->bg_bits)); + + if (ocfs2_is_cluster_bitmap(alloc_inode)) + old_bg_contig_free_bits = group->bg_contig_free_bits; + status = ocfs2_block_group_clear_bits(handle, alloc_inode, + group, group_bh, + start_bit, count, 0, undo_fn); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + status = ocfs2_journal_access_di(handle, INODE_CACHE(alloc_inode), + alloc_bh, OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + ocfs2_block_group_set_bits(handle, alloc_inode, group, group_bh, + start_bit, count, + le16_to_cpu(old_bg_contig_free_bits), 1); + goto bail; + } + + idx = le16_to_cpu(group->bg_chain); + rec = &(cl->cl_recs[idx]); + + le32_add_cpu(&rec->c_free, count); + tmp_used = le32_to_cpu(fe->id1.bitmap1.i_used); + fe->id1.bitmap1.i_used = cpu_to_le32(tmp_used - count); + ocfs2_journal_dirty(handle, alloc_bh); + + /* + * Reclaim suballocator free space. + * Bypass: global_bitmap, non empty rec, first rec in cl_recs[] + */ + if (ocfs2_is_cluster_bitmap(alloc_inode) || + (le32_to_cpu(rec->c_free) != (le32_to_cpu(rec->c_total) - 1)) || + (le16_to_cpu(cl->cl_next_free_rec) == 1)) { + goto bail; + } + + _ocfs2_reclaim_suballoc_to_main(handle, alloc_inode, alloc_bh, group_bh); + +bail: + brelse(group_bh); + return status; +} + +int ocfs2_free_suballoc_bits(handle_t *handle, + struct inode *alloc_inode, + struct buffer_head *alloc_bh, + unsigned int start_bit, + u64 bg_blkno, + unsigned int count) +{ + return _ocfs2_free_suballoc_bits(handle, alloc_inode, alloc_bh, + start_bit, bg_blkno, count, NULL); +} + +int ocfs2_free_dinode(handle_t *handle, + struct inode *inode_alloc_inode, + struct buffer_head *inode_alloc_bh, + struct ocfs2_dinode *di) +{ + u64 blk = le64_to_cpu(di->i_blkno); + u16 bit = le16_to_cpu(di->i_suballoc_bit); + u64 bg_blkno = ocfs2_which_suballoc_group(blk, bit); + + if (di->i_suballoc_loc) + bg_blkno = le64_to_cpu(di->i_suballoc_loc); + return ocfs2_free_suballoc_bits(handle, inode_alloc_inode, + inode_alloc_bh, bit, bg_blkno, 1); +} + +static int _ocfs2_free_clusters(handle_t *handle, + struct inode *bitmap_inode, + struct buffer_head *bitmap_bh, + u64 start_blk, + unsigned int num_clusters, + void (*undo_fn)(unsigned int bit, + unsigned long *bitmap)) +{ + int status; + u16 bg_start_bit; + u64 bg_blkno; + + /* You can't ever have a contiguous set of clusters + * bigger than a block group bitmap so we never have to worry + * about looping on them. + * This is expensive. We can safely remove once this stuff has + * gotten tested really well. */ + BUG_ON(start_blk != ocfs2_clusters_to_blocks(bitmap_inode->i_sb, + ocfs2_blocks_to_clusters(bitmap_inode->i_sb, + start_blk))); + + + ocfs2_block_to_cluster_group(bitmap_inode, start_blk, &bg_blkno, + &bg_start_bit); + + trace_ocfs2_free_clusters((unsigned long long)bg_blkno, + (unsigned long long)start_blk, + bg_start_bit, num_clusters); + + status = _ocfs2_free_suballoc_bits(handle, bitmap_inode, bitmap_bh, + bg_start_bit, bg_blkno, + num_clusters, undo_fn); + if (status < 0) { + mlog_errno(status); + goto out; + } + + ocfs2_local_alloc_seen_free_bits(OCFS2_SB(bitmap_inode->i_sb), + num_clusters); + +out: + return status; +} + +int ocfs2_free_clusters(handle_t *handle, + struct inode *bitmap_inode, + struct buffer_head *bitmap_bh, + u64 start_blk, + unsigned int num_clusters) +{ + return _ocfs2_free_clusters(handle, bitmap_inode, bitmap_bh, + start_blk, num_clusters, + _ocfs2_set_bit); +} + +/* + * Give never-used clusters back to the global bitmap. We don't need + * to protect these bits in the undo buffer. + */ +int ocfs2_release_clusters(handle_t *handle, + struct inode *bitmap_inode, + struct buffer_head *bitmap_bh, + u64 start_blk, + unsigned int num_clusters) +{ + return _ocfs2_free_clusters(handle, bitmap_inode, bitmap_bh, + start_blk, num_clusters, + _ocfs2_clear_bit); +} + +/* + * For a given allocation, determine which allocators will need to be + * accessed, and lock them, reserving the appropriate number of bits. + * + * Sparse file systems call this from ocfs2_write_begin_nolock() + * and ocfs2_allocate_unwritten_extents(). + * + * File systems which don't support holes call this from + * ocfs2_extend_allocation(). + */ +int ocfs2_lock_allocators(struct inode *inode, + struct ocfs2_extent_tree *et, + u32 clusters_to_add, u32 extents_to_split, + struct ocfs2_alloc_context **data_ac, + struct ocfs2_alloc_context **meta_ac) +{ + int ret = 0, num_free_extents; + unsigned int max_recs_needed = clusters_to_add + 2 * extents_to_split; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + *meta_ac = NULL; + if (data_ac) + *data_ac = NULL; + + BUG_ON(clusters_to_add != 0 && data_ac == NULL); + + num_free_extents = ocfs2_num_free_extents(et); + if (num_free_extents < 0) { + ret = num_free_extents; + mlog_errno(ret); + goto out; + } + + /* + * Sparse allocation file systems need to be more conservative + * with reserving room for expansion - the actual allocation + * happens while we've got a journal handle open so re-taking + * a cluster lock (because we ran out of room for another + * extent) will violate ordering rules. + * + * Most of the time we'll only be seeing this 1 cluster at a time + * anyway. + * + * Always lock for any unwritten extents - we might want to + * add blocks during a split. + */ + if (!num_free_extents || + (ocfs2_sparse_alloc(osb) && num_free_extents < max_recs_needed)) { + ret = ocfs2_reserve_new_metadata(osb, et->et_root_el, meta_ac); + if (ret < 0) { + if (ret != -ENOSPC) + mlog_errno(ret); + goto out; + } + } + + if (clusters_to_add == 0) + goto out; + + ret = ocfs2_reserve_clusters(osb, clusters_to_add, data_ac); + if (ret < 0) { + if (ret != -ENOSPC) + mlog_errno(ret); + goto out; + } + +out: + if (ret) { + if (*meta_ac) { + ocfs2_free_alloc_context(*meta_ac); + *meta_ac = NULL; + } + + /* + * We cannot have an error and a non null *data_ac. + */ + } + + return ret; +} + +/* + * Read the inode specified by blkno to get suballoc_slot and + * suballoc_bit. + */ +static int ocfs2_get_suballoc_slot_bit(struct ocfs2_super *osb, u64 blkno, + u16 *suballoc_slot, u64 *group_blkno, + u16 *suballoc_bit) +{ + int status; + struct buffer_head *inode_bh = NULL; + struct ocfs2_dinode *inode_fe; + + trace_ocfs2_get_suballoc_slot_bit((unsigned long long)blkno); + + /* dirty read disk */ + status = ocfs2_read_blocks_sync(osb, blkno, 1, &inode_bh); + if (status < 0) { + mlog(ML_ERROR, "read block %llu failed %d\n", + (unsigned long long)blkno, status); + goto bail; + } + + inode_fe = (struct ocfs2_dinode *) inode_bh->b_data; + if (!OCFS2_IS_VALID_DINODE(inode_fe)) { + mlog(ML_ERROR, "invalid inode %llu requested\n", + (unsigned long long)blkno); + status = -EINVAL; + goto bail; + } + + if (le16_to_cpu(inode_fe->i_suballoc_slot) != (u16)OCFS2_INVALID_SLOT && + (u32)le16_to_cpu(inode_fe->i_suballoc_slot) > osb->max_slots - 1) { + mlog(ML_ERROR, "inode %llu has invalid suballoc slot %u\n", + (unsigned long long)blkno, + (u32)le16_to_cpu(inode_fe->i_suballoc_slot)); + status = -EINVAL; + goto bail; + } + + if (suballoc_slot) + *suballoc_slot = le16_to_cpu(inode_fe->i_suballoc_slot); + if (suballoc_bit) + *suballoc_bit = le16_to_cpu(inode_fe->i_suballoc_bit); + if (group_blkno) + *group_blkno = le64_to_cpu(inode_fe->i_suballoc_loc); + +bail: + brelse(inode_bh); + + if (status) + mlog_errno(status); + return status; +} + +/* + * test whether bit is SET in allocator bitmap or not. on success, 0 + * is returned and *res is 1 for SET; 0 otherwise. when fails, errno + * is returned and *res is meaningless. Call this after you have + * cluster locked against suballoc, or you may get a result based on + * non-up2date contents + */ +static int ocfs2_test_suballoc_bit(struct ocfs2_super *osb, + struct inode *suballoc, + struct buffer_head *alloc_bh, + u64 group_blkno, u64 blkno, + u16 bit, int *res) +{ + struct ocfs2_dinode *alloc_di; + struct ocfs2_group_desc *group; + struct buffer_head *group_bh = NULL; + u64 bg_blkno; + int status, quiet = 0, released = 0; + + trace_ocfs2_test_suballoc_bit((unsigned long long)blkno, + (unsigned int)bit); + + alloc_di = (struct ocfs2_dinode *)alloc_bh->b_data; + if ((bit + 1) > ocfs2_bits_per_group(&alloc_di->id2.i_chain)) { + mlog(ML_ERROR, "suballoc bit %u out of range of %u\n", + (unsigned int)bit, + ocfs2_bits_per_group(&alloc_di->id2.i_chain)); + status = -EINVAL; + goto bail; + } + + bg_blkno = group_blkno ? group_blkno : + ocfs2_which_suballoc_group(blkno, bit); + status = ocfs2_read_hint_group_descriptor(suballoc, alloc_di, bg_blkno, + &group_bh, &released); + if (released) { + quiet = 1; + status = -ESTALE; + goto bail; + } else if (status < 0) { + mlog(ML_ERROR, "read group %llu failed %d\n", + (unsigned long long)bg_blkno, status); + goto bail; + } + + group = (struct ocfs2_group_desc *) group_bh->b_data; + *res = ocfs2_test_bit(bit, (unsigned long *)group->bg_bitmap); + +bail: + brelse(group_bh); + + if (status && !quiet) + mlog_errno(status); + return status; +} + +/* + * Test if the bit representing this inode (blkno) is set in the + * suballocator. + * + * On success, 0 is returned and *res is 1 for SET; 0 otherwise. + * + * In the event of failure, a negative value is returned and *res is + * meaningless. + * + * Callers must make sure to hold nfs_sync_lock to prevent + * ocfs2_delete_inode() on another node from accessing the same + * suballocator concurrently. + */ +int ocfs2_test_inode_bit(struct ocfs2_super *osb, u64 blkno, int *res) +{ + int status, quiet = 0; + u64 group_blkno = 0; + u16 suballoc_bit = 0, suballoc_slot = 0; + struct inode *inode_alloc_inode; + struct buffer_head *alloc_bh = NULL; + + trace_ocfs2_test_inode_bit((unsigned long long)blkno); + + status = ocfs2_get_suballoc_slot_bit(osb, blkno, &suballoc_slot, + &group_blkno, &suballoc_bit); + if (status < 0) { + mlog(ML_ERROR, "get alloc slot and bit failed %d\n", status); + goto bail; + } + + if (suballoc_slot == (u16)OCFS2_INVALID_SLOT) + inode_alloc_inode = ocfs2_get_system_file_inode(osb, + GLOBAL_INODE_ALLOC_SYSTEM_INODE, suballoc_slot); + else + inode_alloc_inode = ocfs2_get_system_file_inode(osb, + INODE_ALLOC_SYSTEM_INODE, suballoc_slot); + if (!inode_alloc_inode) { + /* the error code could be inaccurate, but we are not able to + * get the correct one. */ + status = -EINVAL; + mlog(ML_ERROR, "unable to get alloc inode in slot %u\n", + (u32)suballoc_slot); + goto bail; + } + + inode_lock(inode_alloc_inode); + status = ocfs2_inode_lock(inode_alloc_inode, &alloc_bh, 0); + if (status < 0) { + inode_unlock(inode_alloc_inode); + iput(inode_alloc_inode); + mlog(ML_ERROR, "lock on alloc inode on slot %u failed %d\n", + (u32)suballoc_slot, status); + goto bail; + } + + status = ocfs2_test_suballoc_bit(osb, inode_alloc_inode, alloc_bh, + group_blkno, blkno, suballoc_bit, res); + if (status < 0) { + if (status == -ESTALE) + quiet = 1; + else + mlog(ML_ERROR, "test suballoc bit failed %d\n", status); + } + + ocfs2_inode_unlock(inode_alloc_inode, 0); + inode_unlock(inode_alloc_inode); + + iput(inode_alloc_inode); + brelse(alloc_bh); +bail: + if (status && !quiet) + mlog_errno(status); + return status; +} diff --git a/fs/ocfs2/suballoc.h b/fs/ocfs2/suballoc.h new file mode 100644 index 000000000..bcf2ed4a8 --- /dev/null +++ b/fs/ocfs2/suballoc.h @@ -0,0 +1,226 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * suballoc.h + * + * Defines sub allocator api + * + * Copyright (C) 2003, 2004 Oracle. All rights reserved. + */ + +#ifndef _CHAINALLOC_H_ +#define _CHAINALLOC_H_ + +struct ocfs2_suballoc_result; +typedef int (group_search_t)(struct inode *, + struct buffer_head *, + u32, /* bits_wanted */ + u32, /* min_bits */ + u64, /* max_block */ + struct ocfs2_suballoc_result *); + /* found bits */ + +struct ocfs2_alloc_context { + struct inode *ac_inode; /* which bitmap are we allocating from? */ + struct buffer_head *ac_bh; /* file entry bh */ + u32 ac_alloc_slot; /* which slot are we allocating from? */ + u32 ac_bits_wanted; + u32 ac_bits_given; +#define OCFS2_AC_USE_LOCAL 1 +#define OCFS2_AC_USE_MAIN 2 +#define OCFS2_AC_USE_INODE 3 +#define OCFS2_AC_USE_META 4 +#define OCFS2_AC_USE_MAIN_DISCONTIG 5 + u32 ac_which; + + /* these are used by the chain search */ + u16 ac_chain; + int ac_disable_chain_relink; + group_search_t *ac_group_search; + + u64 ac_last_group; + u64 ac_max_block; /* Highest block number to allocate. 0 is + the same as ~0 - unlimited */ + + int ac_find_loc_only; /* hack for reflink operation ordering */ + struct ocfs2_suballoc_result *ac_find_loc_priv; /* */ + + struct ocfs2_alloc_reservation *ac_resv; +}; + +void ocfs2_init_steal_slots(struct ocfs2_super *osb); +void ocfs2_free_alloc_context(struct ocfs2_alloc_context *ac); +static inline int ocfs2_alloc_context_bits_left(struct ocfs2_alloc_context *ac) +{ + return ac->ac_bits_wanted - ac->ac_bits_given; +} + +/* + * Please note that the caller must make sure that root_el is the root + * of extent tree. So for an inode, it should be &fe->id2.i_list. Otherwise + * the result may be wrong. + */ +int ocfs2_reserve_new_metadata(struct ocfs2_super *osb, + struct ocfs2_extent_list *root_el, + struct ocfs2_alloc_context **ac); +int ocfs2_reserve_new_metadata_blocks(struct ocfs2_super *osb, + int blocks, + struct ocfs2_alloc_context **ac); +int ocfs2_reserve_new_inode(struct ocfs2_super *osb, + struct ocfs2_alloc_context **ac); +int ocfs2_reserve_clusters(struct ocfs2_super *osb, + u32 bits_wanted, + struct ocfs2_alloc_context **ac); + +int ocfs2_alloc_dinode_update_counts(struct inode *inode, + handle_t *handle, + struct buffer_head *di_bh, + u32 num_bits, + u16 chain); +void ocfs2_rollback_alloc_dinode_counts(struct inode *inode, + struct buffer_head *di_bh, + u32 num_bits, + u16 chain); +u16 ocfs2_find_max_contig_free_bits(void *bitmap, + u16 total_bits, u16 start); +int ocfs2_block_group_set_bits(handle_t *handle, + struct inode *alloc_inode, + struct ocfs2_group_desc *bg, + struct buffer_head *group_bh, + unsigned int bit_off, + unsigned int num_bits, + unsigned int max_contig_bits, + int fastpath); + +int ocfs2_claim_metadata(handle_t *handle, + struct ocfs2_alloc_context *ac, + u32 bits_wanted, + u64 *suballoc_loc, + u16 *suballoc_bit_start, + u32 *num_bits, + u64 *blkno_start); +int ocfs2_claim_new_inode(handle_t *handle, + struct inode *dir, + struct buffer_head *parent_fe_bh, + struct ocfs2_alloc_context *ac, + u64 *suballoc_loc, + u16 *suballoc_bit, + u64 *fe_blkno); +int ocfs2_claim_clusters(handle_t *handle, + struct ocfs2_alloc_context *ac, + u32 min_clusters, + u32 *cluster_start, + u32 *num_clusters); +/* + * Use this variant of ocfs2_claim_clusters to specify a maximum + * number of clusters smaller than the allocation reserved. + */ +int __ocfs2_claim_clusters(handle_t *handle, + struct ocfs2_alloc_context *ac, + u32 min_clusters, + u32 max_clusters, + u32 *cluster_start, + u32 *num_clusters); + +int ocfs2_free_suballoc_bits(handle_t *handle, + struct inode *alloc_inode, + struct buffer_head *alloc_bh, + unsigned int start_bit, + u64 bg_blkno, + unsigned int count); +int ocfs2_free_dinode(handle_t *handle, + struct inode *inode_alloc_inode, + struct buffer_head *inode_alloc_bh, + struct ocfs2_dinode *di); +int ocfs2_free_clusters(handle_t *handle, + struct inode *bitmap_inode, + struct buffer_head *bitmap_bh, + u64 start_blk, + unsigned int num_clusters); +int ocfs2_release_clusters(handle_t *handle, + struct inode *bitmap_inode, + struct buffer_head *bitmap_bh, + u64 start_blk, + unsigned int num_clusters); + +static inline u64 ocfs2_which_suballoc_group(u64 block, unsigned int bit) +{ + u64 group = block - (u64) bit; + + return group; +} + +static inline u32 ocfs2_cluster_from_desc(struct ocfs2_super *osb, + u64 bg_blkno) +{ + /* This should work for all block group descriptors as only + * the 1st group descriptor of the cluster bitmap is + * different. */ + + if (bg_blkno == osb->first_cluster_group_blkno) + return 0; + + /* the rest of the block groups are located at the beginning + * of their 1st cluster, so a direct translation just + * works. */ + return ocfs2_blocks_to_clusters(osb->sb, bg_blkno); +} + +static inline int ocfs2_is_cluster_bitmap(struct inode *inode) +{ + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + return osb->bitmap_blkno == OCFS2_I(inode)->ip_blkno; +} + +/* This is for local alloc ONLY. Others should use the task-specific + * apis above. */ +int ocfs2_reserve_cluster_bitmap_bits(struct ocfs2_super *osb, + struct ocfs2_alloc_context *ac); +void ocfs2_free_ac_resource(struct ocfs2_alloc_context *ac); + +/* given a cluster offset, calculate which block group it belongs to + * and return that block offset. */ +u64 ocfs2_which_cluster_group(struct inode *inode, u32 cluster); + +/* + * By default, ocfs2_read_group_descriptor() calls ocfs2_error() when it + * finds a problem. A caller that wants to check a group descriptor + * without going readonly should read the block with ocfs2_read_block[s]() + * and then checking it with this function. This is only resize, really. + * Everyone else should be using ocfs2_read_group_descriptor(). + */ +int ocfs2_check_group_descriptor(struct super_block *sb, + struct ocfs2_dinode *di, + struct buffer_head *bh); +/* + * Read a group descriptor block into *bh. If *bh is NULL, a bh will be + * allocated. This is a cached read. The descriptor will be validated with + * ocfs2_validate_group_descriptor(). + */ +int ocfs2_read_group_descriptor(struct inode *inode, struct ocfs2_dinode *di, + u64 gd_blkno, struct buffer_head **bh); + +int ocfs2_lock_allocators(struct inode *inode, struct ocfs2_extent_tree *et, + u32 clusters_to_add, u32 extents_to_split, + struct ocfs2_alloc_context **data_ac, + struct ocfs2_alloc_context **meta_ac); + +int ocfs2_test_inode_bit(struct ocfs2_super *osb, u64 blkno, int *res); + + + +/* + * The following two interfaces are for ocfs2_create_inode_in_orphan(). + */ +int ocfs2_find_new_inode_loc(struct inode *dir, + struct buffer_head *parent_fe_bh, + struct ocfs2_alloc_context *ac, + u64 *fe_blkno); + +int ocfs2_claim_new_inode_at_loc(handle_t *handle, + struct inode *dir, + struct ocfs2_alloc_context *ac, + u64 *suballoc_loc, + u16 *suballoc_bit, + u64 di_blkno); + +#endif /* _CHAINALLOC_H_ */ diff --git a/fs/ocfs2/super.c b/fs/ocfs2/super.c new file mode 100644 index 000000000..c62e389d4 --- /dev/null +++ b/fs/ocfs2/super.c @@ -0,0 +1,2573 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * super.c + * + * load/unload driver, mount/dismount volumes + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CREATE_TRACE_POINTS +#include "ocfs2_trace.h" + +#include + +#include "ocfs2.h" + +/* this should be the only file to include a version 1 header */ +#include "ocfs1_fs_compat.h" + +#include "alloc.h" +#include "aops.h" +#include "blockcheck.h" +#include "dlmglue.h" +#include "export.h" +#include "extent_map.h" +#include "heartbeat.h" +#include "inode.h" +#include "journal.h" +#include "localalloc.h" +#include "namei.h" +#include "slot_map.h" +#include "super.h" +#include "sysfile.h" +#include "uptodate.h" +#include "xattr.h" +#include "quota.h" +#include "refcounttree.h" +#include "suballoc.h" + +#include "buffer_head_io.h" +#include "filecheck.h" + +static struct kmem_cache *ocfs2_inode_cachep; +struct kmem_cache *ocfs2_dquot_cachep; +struct kmem_cache *ocfs2_qf_chunk_cachep; + +static struct dentry *ocfs2_debugfs_root; + +MODULE_AUTHOR("Oracle"); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("OCFS2 cluster file system"); + +struct mount_options +{ + unsigned long commit_interval; + unsigned long mount_opt; + unsigned int atime_quantum; + unsigned short slot; + int localalloc_opt; + unsigned int resv_level; + int dir_resv_level; + char cluster_stack[OCFS2_STACK_LABEL_LEN + 1]; + bool user_stack; +}; + +static int ocfs2_parse_param(struct fs_context *fc, struct fs_parameter *param); +static int ocfs2_check_set_options(struct super_block *sb, + struct mount_options *options); +static int ocfs2_show_options(struct seq_file *s, struct dentry *root); +static void ocfs2_put_super(struct super_block *sb); +static int ocfs2_mount_volume(struct super_block *sb); +static void ocfs2_dismount_volume(struct super_block *sb, int mnt_err); +static int ocfs2_initialize_mem_caches(void); +static void ocfs2_free_mem_caches(void); +static void ocfs2_delete_osb(struct ocfs2_super *osb); + +static int ocfs2_statfs(struct dentry *dentry, struct kstatfs *buf); + +static int ocfs2_sync_fs(struct super_block *sb, int wait); + +static int ocfs2_init_global_system_inodes(struct ocfs2_super *osb); +static int ocfs2_init_local_system_inodes(struct ocfs2_super *osb); +static void ocfs2_release_system_inodes(struct ocfs2_super *osb); +static int ocfs2_check_volume(struct ocfs2_super *osb); +static int ocfs2_verify_volume(struct ocfs2_dinode *di, + struct buffer_head *bh, + u32 sectsize, + struct ocfs2_blockcheck_stats *stats); +static int ocfs2_initialize_super(struct super_block *sb, + struct buffer_head *bh, + int sector_size, + struct ocfs2_blockcheck_stats *stats); +static int ocfs2_get_sector(struct super_block *sb, + struct buffer_head **bh, + int block, + int sect_size); +static struct inode *ocfs2_alloc_inode(struct super_block *sb); +static void ocfs2_free_inode(struct inode *inode); +static int ocfs2_susp_quotas(struct ocfs2_super *osb, int unsuspend); +static int ocfs2_enable_quotas(struct ocfs2_super *osb); +static void ocfs2_disable_quotas(struct ocfs2_super *osb); + +static struct dquot __rcu **ocfs2_get_dquots(struct inode *inode) +{ + return OCFS2_I(inode)->i_dquot; +} + +static const struct super_operations ocfs2_sops = { + .statfs = ocfs2_statfs, + .alloc_inode = ocfs2_alloc_inode, + .free_inode = ocfs2_free_inode, + .drop_inode = inode_just_drop, + .evict_inode = ocfs2_evict_inode, + .sync_fs = ocfs2_sync_fs, + .put_super = ocfs2_put_super, + .show_options = ocfs2_show_options, + .quota_read = ocfs2_quota_read, + .quota_write = ocfs2_quota_write, + .get_dquots = ocfs2_get_dquots, +}; + +enum { + Opt_barrier, + Opt_errors, + Opt_intr, + Opt_heartbeat, + Opt_data, + Opt_atime_quantum, + Opt_slot, + Opt_commit, + Opt_localalloc, + Opt_localflocks, + Opt_stack, + Opt_user_xattr, + Opt_inode64, + Opt_acl, + Opt_usrquota, + Opt_grpquota, + Opt_coherency, + Opt_resv_level, + Opt_dir_resv_level, + Opt_journal_async_commit, +}; + +static const struct constant_table ocfs2_param_errors[] = { + {"panic", OCFS2_MOUNT_ERRORS_PANIC}, + {"remount-ro", OCFS2_MOUNT_ERRORS_ROFS}, + {"continue", OCFS2_MOUNT_ERRORS_CONT}, + {} +}; + +static const struct constant_table ocfs2_param_heartbeat[] = { + {"local", OCFS2_MOUNT_HB_LOCAL}, + {"none", OCFS2_MOUNT_HB_NONE}, + {"global", OCFS2_MOUNT_HB_GLOBAL}, + {} +}; + +static const struct constant_table ocfs2_param_data[] = { + {"writeback", OCFS2_MOUNT_DATA_WRITEBACK}, + {"ordered", 0}, + {} +}; + +static const struct constant_table ocfs2_param_coherency[] = { + {"buffered", OCFS2_MOUNT_COHERENCY_BUFFERED}, + {"full", 0}, + {} +}; + +static const struct fs_parameter_spec ocfs2_param_spec[] = { + fsparam_u32 ("barrier", Opt_barrier), + fsparam_enum ("errors", Opt_errors, ocfs2_param_errors), + fsparam_flag_no ("intr", Opt_intr), + fsparam_enum ("heartbeat", Opt_heartbeat, ocfs2_param_heartbeat), + fsparam_enum ("data", Opt_data, ocfs2_param_data), + fsparam_u32 ("atime_quantum", Opt_atime_quantum), + fsparam_u32 ("preferred_slot", Opt_slot), + fsparam_u32 ("commit", Opt_commit), + fsparam_s32 ("localalloc", Opt_localalloc), + fsparam_flag ("localflocks", Opt_localflocks), + fsparam_string ("cluster_stack", Opt_stack), + fsparam_flag_no ("user_xattr", Opt_user_xattr), + fsparam_flag ("inode64", Opt_inode64), + fsparam_flag_no ("acl", Opt_acl), + fsparam_flag ("usrquota", Opt_usrquota), + fsparam_flag ("grpquota", Opt_grpquota), + fsparam_enum ("coherency", Opt_coherency, ocfs2_param_coherency), + fsparam_u32 ("resv_level", Opt_resv_level), + fsparam_u32 ("dir_resv_level", Opt_dir_resv_level), + fsparam_flag ("journal_async_commit", Opt_journal_async_commit), + {} +}; + +#ifdef CONFIG_DEBUG_FS +static int ocfs2_osb_dump(struct ocfs2_super *osb, char *buf, int len) +{ + struct ocfs2_cluster_connection *cconn = osb->cconn; + struct ocfs2_recovery_map *rm = osb->recovery_map; + struct ocfs2_orphan_scan *os = &osb->osb_orphan_scan; + int i, out = 0; + unsigned long flags; + + out += scnprintf(buf + out, len - out, + "%10s => Id: %-s Uuid: %-s Gen: 0x%X Label: %-s\n", + "Device", osb->dev_str, osb->uuid_str, + osb->fs_generation, osb->vol_label); + + out += scnprintf(buf + out, len - out, + "%10s => State: %d Flags: 0x%lX\n", "Volume", + atomic_read(&osb->vol_state), osb->osb_flags); + + out += scnprintf(buf + out, len - out, + "%10s => Block: %lu Cluster: %d\n", "Sizes", + osb->sb->s_blocksize, osb->s_clustersize); + + out += scnprintf(buf + out, len - out, + "%10s => Compat: 0x%X Incompat: 0x%X " + "ROcompat: 0x%X\n", + "Features", osb->s_feature_compat, + osb->s_feature_incompat, osb->s_feature_ro_compat); + + out += scnprintf(buf + out, len - out, + "%10s => Opts: 0x%lX AtimeQuanta: %u\n", "Mount", + osb->s_mount_opt, osb->s_atime_quantum); + + if (cconn) { + out += scnprintf(buf + out, len - out, + "%10s => Stack: %s Name: %*s " + "Version: %d.%d\n", "Cluster", + (*osb->osb_cluster_stack == '\0' ? + "o2cb" : osb->osb_cluster_stack), + cconn->cc_namelen, cconn->cc_name, + cconn->cc_version.pv_major, + cconn->cc_version.pv_minor); + } + + spin_lock_irqsave(&osb->dc_task_lock, flags); + out += scnprintf(buf + out, len - out, + "%10s => Pid: %d Count: %lu WakeSeq: %lu " + "WorkSeq: %lu\n", "DownCnvt", + (osb->dc_task ? task_pid_nr(osb->dc_task) : -1), + osb->blocked_lock_count, osb->dc_wake_sequence, + osb->dc_work_sequence); + spin_unlock_irqrestore(&osb->dc_task_lock, flags); + + spin_lock(&osb->osb_lock); + out += scnprintf(buf + out, len - out, "%10s => Pid: %d Nodes:", + "Recovery", + (osb->recovery_thread_task ? + task_pid_nr(osb->recovery_thread_task) : -1)); + if (rm->rm_used == 0) + out += scnprintf(buf + out, len - out, " None\n"); + else { + for (i = 0; i < rm->rm_used; i++) + out += scnprintf(buf + out, len - out, " %d", + rm->rm_entries[i]); + out += scnprintf(buf + out, len - out, "\n"); + } + spin_unlock(&osb->osb_lock); + + out += scnprintf(buf + out, len - out, + "%10s => Pid: %d Interval: %lu\n", "Commit", + (osb->commit_task ? task_pid_nr(osb->commit_task) : -1), + osb->osb_commit_interval); + + out += scnprintf(buf + out, len - out, + "%10s => State: %d TxnId: %lu NumTxns: %d\n", + "Journal", osb->journal->j_state, + osb->journal->j_trans_id, + atomic_read(&osb->journal->j_num_trans)); + + out += scnprintf(buf + out, len - out, + "%10s => GlobalAllocs: %d LocalAllocs: %d " + "SubAllocs: %d LAWinMoves: %d SAExtends: %d\n", + "Stats", + atomic_read(&osb->alloc_stats.bitmap_data), + atomic_read(&osb->alloc_stats.local_data), + atomic_read(&osb->alloc_stats.bg_allocs), + atomic_read(&osb->alloc_stats.moves), + atomic_read(&osb->alloc_stats.bg_extends)); + + out += scnprintf(buf + out, len - out, + "%10s => State: %u Descriptor: %llu Size: %u bits " + "Default: %u bits\n", + "LocalAlloc", osb->local_alloc_state, + (unsigned long long)osb->la_last_gd, + osb->local_alloc_bits, osb->local_alloc_default_bits); + + spin_lock(&osb->osb_lock); + out += scnprintf(buf + out, len - out, + "%10s => InodeSlot: %d StolenInodes: %d, " + "MetaSlot: %d StolenMeta: %d\n", "Steal", + osb->s_inode_steal_slot, + atomic_read(&osb->s_num_inodes_stolen), + osb->s_meta_steal_slot, + atomic_read(&osb->s_num_meta_stolen)); + spin_unlock(&osb->osb_lock); + + out += scnprintf(buf + out, len - out, "OrphanScan => "); + out += scnprintf(buf + out, len - out, "Local: %u Global: %u ", + os->os_count, os->os_seqno); + out += scnprintf(buf + out, len - out, " Last Scan: "); + if (atomic_read(&os->os_state) == ORPHAN_SCAN_INACTIVE) + out += scnprintf(buf + out, len - out, "Disabled\n"); + else + out += scnprintf(buf + out, len - out, "%lu seconds ago\n", + (unsigned long)(ktime_get_seconds() - os->os_scantime)); + + out += scnprintf(buf + out, len - out, "%10s => %3s %10s\n", + "Slots", "Num", "RecoGen"); + for (i = 0; i < osb->max_slots; ++i) { + out += scnprintf(buf + out, len - out, + "%10s %c %3d %10d\n", + " ", + (i == osb->slot_num ? '*' : ' '), + i, osb->slot_recovery_generations[i]); + } + + return out; +} + +static int ocfs2_osb_debug_open(struct inode *inode, struct file *file) +{ + struct ocfs2_super *osb = inode->i_private; + char *buf = NULL; + + buf = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!buf) + goto bail; + + i_size_write(inode, ocfs2_osb_dump(osb, buf, PAGE_SIZE)); + + file->private_data = buf; + + return 0; +bail: + return -ENOMEM; +} + +static int ocfs2_debug_release(struct inode *inode, struct file *file) +{ + kfree(file->private_data); + return 0; +} + +static ssize_t ocfs2_debug_read(struct file *file, char __user *buf, + size_t nbytes, loff_t *ppos) +{ + return simple_read_from_buffer(buf, nbytes, ppos, file->private_data, + i_size_read(file->f_mapping->host)); +} +#else +static int ocfs2_osb_debug_open(struct inode *inode, struct file *file) +{ + return 0; +} +static int ocfs2_debug_release(struct inode *inode, struct file *file) +{ + return 0; +} +static ssize_t ocfs2_debug_read(struct file *file, char __user *buf, + size_t nbytes, loff_t *ppos) +{ + return 0; +} +#endif /* CONFIG_DEBUG_FS */ + +static const struct file_operations ocfs2_osb_debug_fops = { + .open = ocfs2_osb_debug_open, + .release = ocfs2_debug_release, + .read = ocfs2_debug_read, + .llseek = generic_file_llseek, +}; + +static int ocfs2_sync_fs(struct super_block *sb, int wait) +{ + int status; + tid_t target; + struct ocfs2_super *osb = OCFS2_SB(sb); + + if (ocfs2_is_hard_readonly(osb)) + return -EROFS; + + if (wait) { + status = ocfs2_flush_truncate_log(osb); + if (status < 0) + mlog_errno(status); + } else { + ocfs2_schedule_truncate_log_flush(osb, 0); + } + + if (jbd2_journal_start_commit(osb->journal->j_journal, + &target)) { + if (wait) + jbd2_log_wait_commit(osb->journal->j_journal, + target); + } + return 0; +} + +static int ocfs2_need_system_inode(struct ocfs2_super *osb, int ino) +{ + if (!OCFS2_HAS_RO_COMPAT_FEATURE(osb->sb, OCFS2_FEATURE_RO_COMPAT_USRQUOTA) + && (ino == USER_QUOTA_SYSTEM_INODE + || ino == LOCAL_USER_QUOTA_SYSTEM_INODE)) + return 0; + if (!OCFS2_HAS_RO_COMPAT_FEATURE(osb->sb, OCFS2_FEATURE_RO_COMPAT_GRPQUOTA) + && (ino == GROUP_QUOTA_SYSTEM_INODE + || ino == LOCAL_GROUP_QUOTA_SYSTEM_INODE)) + return 0; + return 1; +} + +static int ocfs2_init_global_system_inodes(struct ocfs2_super *osb) +{ + struct inode *new = NULL; + int status = 0; + int i; + + new = ocfs2_iget(osb, osb->root_blkno, OCFS2_FI_FLAG_SYSFILE, 0); + if (IS_ERR(new)) { + status = PTR_ERR(new); + mlog_errno(status); + goto bail; + } + osb->root_inode = new; + + new = ocfs2_iget(osb, osb->system_dir_blkno, OCFS2_FI_FLAG_SYSFILE, 0); + if (IS_ERR(new)) { + status = PTR_ERR(new); + mlog_errno(status); + goto bail; + } + osb->sys_root_inode = new; + + for (i = OCFS2_FIRST_ONLINE_SYSTEM_INODE; + i <= OCFS2_LAST_GLOBAL_SYSTEM_INODE; i++) { + if (!ocfs2_need_system_inode(osb, i)) + continue; + new = ocfs2_get_system_file_inode(osb, i, osb->slot_num); + if (!new) { + ocfs2_release_system_inodes(osb); + status = ocfs2_is_soft_readonly(osb) ? -EROFS : -EINVAL; + mlog_errno(status); + mlog(ML_ERROR, "Unable to load system inode %d, " + "possibly corrupt fs?", i); + goto bail; + } + // the array now has one ref, so drop this one + iput(new); + } + +bail: + if (status) + mlog_errno(status); + return status; +} + +static int ocfs2_init_local_system_inodes(struct ocfs2_super *osb) +{ + struct inode *new = NULL; + int status = 0; + int i; + + for (i = OCFS2_LAST_GLOBAL_SYSTEM_INODE + 1; + i < NUM_SYSTEM_INODES; + i++) { + if (!ocfs2_need_system_inode(osb, i)) + continue; + new = ocfs2_get_system_file_inode(osb, i, osb->slot_num); + if (!new) { + ocfs2_release_system_inodes(osb); + status = ocfs2_is_soft_readonly(osb) ? -EROFS : -EINVAL; + mlog(ML_ERROR, "status=%d, sysfile=%d, slot=%d\n", + status, i, osb->slot_num); + goto bail; + } + /* the array now has one ref, so drop this one */ + iput(new); + } + +bail: + if (status) + mlog_errno(status); + return status; +} + +static void ocfs2_release_system_inodes(struct ocfs2_super *osb) +{ + int i; + struct inode *inode; + + for (i = 0; i < NUM_GLOBAL_SYSTEM_INODES; i++) { + inode = osb->global_system_inodes[i]; + if (inode) { + iput(inode); + osb->global_system_inodes[i] = NULL; + } + } + + inode = osb->sys_root_inode; + if (inode) { + iput(inode); + osb->sys_root_inode = NULL; + } + + inode = osb->root_inode; + if (inode) { + iput(inode); + osb->root_inode = NULL; + } + + if (!osb->local_system_inodes) + return; + + for (i = 0; i < NUM_LOCAL_SYSTEM_INODES * osb->max_slots; i++) { + if (osb->local_system_inodes[i]) { + iput(osb->local_system_inodes[i]); + osb->local_system_inodes[i] = NULL; + } + } + + kfree(osb->local_system_inodes); + osb->local_system_inodes = NULL; +} + +/* We're allocating fs objects, use GFP_NOFS */ +static struct inode *ocfs2_alloc_inode(struct super_block *sb) +{ + struct ocfs2_inode_info *oi; + + oi = alloc_inode_sb(sb, ocfs2_inode_cachep, GFP_NOFS); + if (!oi) + return NULL; + + oi->i_sync_tid = 0; + oi->i_datasync_tid = 0; + memset(&oi->i_dquot, 0, sizeof(oi->i_dquot)); + + jbd2_journal_init_jbd_inode(&oi->ip_jinode, &oi->vfs_inode); + return &oi->vfs_inode; +} + +static void ocfs2_free_inode(struct inode *inode) +{ + kmem_cache_free(ocfs2_inode_cachep, OCFS2_I(inode)); +} + +static unsigned long long ocfs2_max_file_offset(unsigned int bbits, + unsigned int cbits) +{ + unsigned int bytes = 1 << cbits; + unsigned int trim = bytes; + unsigned int bitshift = 32; + + /* + * i_size and all block offsets in ocfs2 are always 64 bits + * wide. i_clusters is 32 bits, in cluster-sized units. So on + * 64 bit platforms, cluster size will be the limiting factor. + */ + +#if BITS_PER_LONG == 32 + BUILD_BUG_ON(sizeof(sector_t) != 8); + /* + * We might be limited by page cache size. + */ + if (bytes > PAGE_SIZE) { + bytes = PAGE_SIZE; + trim = 1; + /* + * Shift by 31 here so that we don't get larger than + * MAX_LFS_FILESIZE + */ + bitshift = 31; + } +#endif + + /* + * Trim by a whole cluster when we can actually approach the + * on-disk limits. Otherwise we can overflow i_clusters when + * an extent start is at the max offset. + */ + return (((unsigned long long)bytes) << bitshift) - trim; +} + +static int ocfs2_reconfigure(struct fs_context *fc) +{ + int incompat_features; + int ret = 0; + struct mount_options *parsed_options = fc->fs_private; + struct super_block *sb = fc->root->d_sb; + struct ocfs2_super *osb = OCFS2_SB(sb); + u32 tmp; + + sync_filesystem(sb); + + if (!ocfs2_check_set_options(sb, parsed_options)) { + ret = -EINVAL; + goto out; + } + + tmp = OCFS2_MOUNT_HB_LOCAL | OCFS2_MOUNT_HB_GLOBAL | + OCFS2_MOUNT_HB_NONE; + if ((osb->s_mount_opt & tmp) != (parsed_options->mount_opt & tmp)) { + ret = -EINVAL; + mlog(ML_ERROR, "Cannot change heartbeat mode on remount\n"); + goto out; + } + + if ((osb->s_mount_opt & OCFS2_MOUNT_DATA_WRITEBACK) != + (parsed_options->mount_opt & OCFS2_MOUNT_DATA_WRITEBACK)) { + ret = -EINVAL; + mlog(ML_ERROR, "Cannot change data mode on remount\n"); + goto out; + } + + /* Probably don't want this on remount; it might + * mess with other nodes */ + if (!(osb->s_mount_opt & OCFS2_MOUNT_INODE64) && + (parsed_options->mount_opt & OCFS2_MOUNT_INODE64)) { + ret = -EINVAL; + mlog(ML_ERROR, "Cannot enable inode64 on remount\n"); + goto out; + } + + /* We're going to/from readonly mode. */ + if ((bool)(fc->sb_flags & SB_RDONLY) != sb_rdonly(sb)) { + /* Disable quota accounting before remounting RO */ + if (fc->sb_flags & SB_RDONLY) { + ret = ocfs2_susp_quotas(osb, 0); + if (ret < 0) + goto out; + } + /* Lock here so the check of HARD_RO and the potential + * setting of SOFT_RO is atomic. */ + spin_lock(&osb->osb_lock); + if (osb->osb_flags & OCFS2_OSB_HARD_RO) { + mlog(ML_ERROR, "Remount on readonly device is forbidden.\n"); + ret = -EROFS; + goto unlock_osb; + } + + if (fc->sb_flags & SB_RDONLY) { + sb->s_flags |= SB_RDONLY; + osb->osb_flags |= OCFS2_OSB_SOFT_RO; + } else { + if (osb->osb_flags & OCFS2_OSB_ERROR_FS) { + mlog(ML_ERROR, "Cannot remount RDWR " + "filesystem due to previous errors.\n"); + ret = -EROFS; + goto unlock_osb; + } + incompat_features = OCFS2_HAS_RO_COMPAT_FEATURE(sb, ~OCFS2_FEATURE_RO_COMPAT_SUPP); + if (incompat_features) { + mlog(ML_ERROR, "Cannot remount RDWR because " + "of unsupported optional features " + "(%x).\n", incompat_features); + ret = -EINVAL; + goto unlock_osb; + } + sb->s_flags &= ~SB_RDONLY; + osb->osb_flags &= ~OCFS2_OSB_SOFT_RO; + } + trace_ocfs2_remount(sb->s_flags, osb->osb_flags, fc->sb_flags); +unlock_osb: + spin_unlock(&osb->osb_lock); + /* Enable quota accounting after remounting RW */ + if (!ret && !(fc->sb_flags & SB_RDONLY)) { + if (sb_any_quota_suspended(sb)) + ret = ocfs2_susp_quotas(osb, 1); + else + ret = ocfs2_enable_quotas(osb); + if (ret < 0) { + /* Return back changes... */ + spin_lock(&osb->osb_lock); + sb->s_flags |= SB_RDONLY; + osb->osb_flags |= OCFS2_OSB_SOFT_RO; + spin_unlock(&osb->osb_lock); + goto out; + } + } + } + + if (!ret) { + /* Only save off the new mount options in case of a successful + * remount. */ + osb->s_mount_opt = parsed_options->mount_opt; + osb->s_atime_quantum = parsed_options->atime_quantum; + osb->preferred_slot = parsed_options->slot; + if (parsed_options->commit_interval) + osb->osb_commit_interval = parsed_options->commit_interval; + + if (!ocfs2_is_hard_readonly(osb)) + ocfs2_set_journal_params(osb); + + sb->s_flags = (sb->s_flags & ~SB_POSIXACL) | + ((osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) ? + SB_POSIXACL : 0); + } +out: + return ret; +} + +static int ocfs2_sb_probe(struct super_block *sb, + struct buffer_head **bh, + int *sector_size, + struct ocfs2_blockcheck_stats *stats) +{ + int status, tmpstat; + struct ocfs1_vol_disk_hdr *hdr; + struct ocfs2_dinode *di; + int blksize; + + *bh = NULL; + + /* may be > 512 */ + *sector_size = bdev_logical_block_size(sb->s_bdev); + if (*sector_size > OCFS2_MAX_BLOCKSIZE) { + mlog(ML_ERROR, "Hardware sector size too large: %d (max=%d)\n", + *sector_size, OCFS2_MAX_BLOCKSIZE); + status = -EINVAL; + goto bail; + } + + /* Can this really happen? */ + if (*sector_size < OCFS2_MIN_BLOCKSIZE) + *sector_size = OCFS2_MIN_BLOCKSIZE; + + /* check block zero for old format */ + status = ocfs2_get_sector(sb, bh, 0, *sector_size); + if (status < 0) { + mlog_errno(status); + goto bail; + } + hdr = (struct ocfs1_vol_disk_hdr *) (*bh)->b_data; + if (hdr->major_version == OCFS1_MAJOR_VERSION) { + mlog(ML_ERROR, "incompatible version: %u.%u\n", + hdr->major_version, hdr->minor_version); + status = -EINVAL; + } + if (memcmp(hdr->signature, OCFS1_VOLUME_SIGNATURE, + strlen(OCFS1_VOLUME_SIGNATURE)) == 0) { + mlog(ML_ERROR, "incompatible volume signature: %8s\n", + hdr->signature); + status = -EINVAL; + } + brelse(*bh); + *bh = NULL; + if (status < 0) { + mlog(ML_ERROR, "This is an ocfs v1 filesystem which must be " + "upgraded before mounting with ocfs v2\n"); + goto bail; + } + + /* + * Now check at magic offset for 512, 1024, 2048, 4096 + * blocksizes. 4096 is the maximum blocksize because it is + * the minimum clustersize. + */ + status = -EINVAL; + for (blksize = *sector_size; + blksize <= OCFS2_MAX_BLOCKSIZE; + blksize <<= 1) { + tmpstat = ocfs2_get_sector(sb, bh, + OCFS2_SUPER_BLOCK_BLKNO, + blksize); + if (tmpstat < 0) { + status = tmpstat; + mlog_errno(status); + break; + } + di = (struct ocfs2_dinode *) (*bh)->b_data; + memset(stats, 0, sizeof(struct ocfs2_blockcheck_stats)); + spin_lock_init(&stats->b_lock); + tmpstat = ocfs2_verify_volume(di, *bh, blksize, stats); + if (tmpstat < 0) { + brelse(*bh); + *bh = NULL; + } + if (tmpstat != -EAGAIN) { + status = tmpstat; + break; + } + } + +bail: + return status; +} + +static int ocfs2_verify_heartbeat(struct ocfs2_super *osb) +{ + u32 hb_enabled = OCFS2_MOUNT_HB_LOCAL | OCFS2_MOUNT_HB_GLOBAL; + + if (osb->s_mount_opt & hb_enabled) { + if (ocfs2_mount_local(osb)) { + mlog(ML_ERROR, "Cannot heartbeat on a locally " + "mounted device.\n"); + return -EINVAL; + } + if (ocfs2_userspace_stack(osb)) { + mlog(ML_ERROR, "Userspace stack expected, but " + "o2cb heartbeat arguments passed to mount\n"); + return -EINVAL; + } + if (((osb->s_mount_opt & OCFS2_MOUNT_HB_GLOBAL) && + !ocfs2_cluster_o2cb_global_heartbeat(osb)) || + ((osb->s_mount_opt & OCFS2_MOUNT_HB_LOCAL) && + ocfs2_cluster_o2cb_global_heartbeat(osb))) { + mlog(ML_ERROR, "Mismatching o2cb heartbeat modes\n"); + return -EINVAL; + } + } + + if (!(osb->s_mount_opt & hb_enabled)) { + if (!ocfs2_mount_local(osb) && !ocfs2_is_hard_readonly(osb) && + !ocfs2_userspace_stack(osb)) { + mlog(ML_ERROR, "Heartbeat has to be started to mount " + "a read-write clustered device.\n"); + return -EINVAL; + } + } + + return 0; +} + +/* + * If we're using a userspace stack, mount should have passed + * a name that matches the disk. If not, mount should not + * have passed a stack. + */ +static int ocfs2_verify_userspace_stack(struct ocfs2_super *osb, + struct mount_options *mopt) +{ + if (!ocfs2_userspace_stack(osb) && mopt->cluster_stack[0]) { + mlog(ML_ERROR, + "cluster stack passed to mount, but this filesystem " + "does not support it\n"); + return -EINVAL; + } + + if (ocfs2_userspace_stack(osb) && + strncmp(osb->osb_cluster_stack, mopt->cluster_stack, + OCFS2_STACK_LABEL_LEN)) { + mlog(ML_ERROR, + "cluster stack passed to mount (\"%s\") does not " + "match the filesystem (\"%s\")\n", + mopt->cluster_stack, + osb->osb_cluster_stack); + return -EINVAL; + } + + return 0; +} + +static int ocfs2_susp_quotas(struct ocfs2_super *osb, int unsuspend) +{ + int type; + struct super_block *sb = osb->sb; + unsigned int feature[OCFS2_MAXQUOTAS] = { + OCFS2_FEATURE_RO_COMPAT_USRQUOTA, + OCFS2_FEATURE_RO_COMPAT_GRPQUOTA}; + int status = 0; + + for (type = 0; type < OCFS2_MAXQUOTAS; type++) { + if (!OCFS2_HAS_RO_COMPAT_FEATURE(sb, feature[type])) + continue; + if (unsuspend) + status = dquot_resume(sb, type); + else { + struct ocfs2_mem_dqinfo *oinfo; + + /* Cancel periodic syncing before suspending */ + oinfo = sb_dqinfo(sb, type)->dqi_priv; + cancel_delayed_work_sync(&oinfo->dqi_sync_work); + status = dquot_suspend(sb, type); + } + if (status < 0) + break; + } + if (status < 0) + mlog(ML_ERROR, "Failed to suspend/unsuspend quotas on " + "remount (error = %d).\n", status); + return status; +} + +static int ocfs2_enable_quotas(struct ocfs2_super *osb) +{ + struct inode *inode[OCFS2_MAXQUOTAS] = { NULL, NULL }; + struct super_block *sb = osb->sb; + unsigned int feature[OCFS2_MAXQUOTAS] = { + OCFS2_FEATURE_RO_COMPAT_USRQUOTA, + OCFS2_FEATURE_RO_COMPAT_GRPQUOTA}; + unsigned int ino[OCFS2_MAXQUOTAS] = { + LOCAL_USER_QUOTA_SYSTEM_INODE, + LOCAL_GROUP_QUOTA_SYSTEM_INODE }; + int status; + int type; + + sb_dqopt(sb)->flags |= DQUOT_QUOTA_SYS_FILE | DQUOT_NEGATIVE_USAGE; + for (type = 0; type < OCFS2_MAXQUOTAS; type++) { + if (!OCFS2_HAS_RO_COMPAT_FEATURE(sb, feature[type])) + continue; + inode[type] = ocfs2_get_system_file_inode(osb, ino[type], + osb->slot_num); + if (!inode[type]) { + status = -ENOENT; + goto out_quota_off; + } + status = dquot_load_quota_inode(inode[type], type, QFMT_OCFS2, + DQUOT_USAGE_ENABLED); + if (status < 0) + goto out_quota_off; + } + + for (type = 0; type < OCFS2_MAXQUOTAS; type++) + iput(inode[type]); + return 0; +out_quota_off: + ocfs2_disable_quotas(osb); + for (type = 0; type < OCFS2_MAXQUOTAS; type++) + iput(inode[type]); + mlog_errno(status); + return status; +} + +static void ocfs2_disable_quotas(struct ocfs2_super *osb) +{ + int type; + struct inode *inode; + struct super_block *sb = osb->sb; + struct ocfs2_mem_dqinfo *oinfo; + + /* We mostly ignore errors in this function because there's not much + * we can do when we see them */ + for (type = 0; type < OCFS2_MAXQUOTAS; type++) { + if (!sb_has_quota_loaded(sb, type)) + continue; + if (!sb_has_quota_suspended(sb, type)) { + oinfo = sb_dqinfo(sb, type)->dqi_priv; + cancel_delayed_work_sync(&oinfo->dqi_sync_work); + } + inode = igrab(sb->s_dquot.files[type]); + /* Turn off quotas. This will remove all dquot structures from + * memory and so they will be automatically synced to global + * quota files */ + dquot_disable(sb, type, DQUOT_USAGE_ENABLED | + DQUOT_LIMITS_ENABLED); + iput(inode); + } +} + +static int ocfs2_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct dentry *root; + int status, sector_size; + struct mount_options *parsed_options = fc->fs_private; + struct inode *inode = NULL; + struct ocfs2_super *osb = NULL; + struct buffer_head *bh = NULL; + char nodestr[12]; + struct ocfs2_blockcheck_stats stats; + + trace_ocfs2_fill_super(sb, fc, fc->sb_flags & SB_SILENT); + + /* probe for superblock */ + status = ocfs2_sb_probe(sb, &bh, §or_size, &stats); + if (status < 0) { + mlog(ML_ERROR, "superblock probe failed!\n"); + goto out; + } + + status = ocfs2_initialize_super(sb, bh, sector_size, &stats); + brelse(bh); + bh = NULL; + if (status < 0) + goto out; + + osb = OCFS2_SB(sb); + + if (!ocfs2_check_set_options(sb, parsed_options)) { + status = -EINVAL; + goto out_super; + } + osb->s_mount_opt = parsed_options->mount_opt; + osb->s_atime_quantum = parsed_options->atime_quantum; + osb->preferred_slot = parsed_options->slot; + osb->osb_commit_interval = parsed_options->commit_interval; + + ocfs2_la_set_sizes(osb, parsed_options->localalloc_opt); + osb->osb_resv_level = parsed_options->resv_level; + osb->osb_dir_resv_level = parsed_options->resv_level; + if (parsed_options->dir_resv_level == -1) + osb->osb_dir_resv_level = parsed_options->resv_level; + else + osb->osb_dir_resv_level = parsed_options->dir_resv_level; + + status = ocfs2_verify_userspace_stack(osb, parsed_options); + if (status) + goto out_super; + + sb->s_magic = OCFS2_SUPER_MAGIC; + + sb->s_flags = (sb->s_flags & ~(SB_POSIXACL | SB_NOSEC)) | + ((osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) ? SB_POSIXACL : 0); + + /* Hard readonly mode only if: bdev_read_only, SB_RDONLY, + * heartbeat=none */ + if (bdev_read_only(sb->s_bdev)) { + if (!sb_rdonly(sb)) { + status = -EACCES; + mlog(ML_ERROR, "Readonly device detected but readonly " + "mount was not specified.\n"); + goto out_super; + } + + /* You should not be able to start a local heartbeat + * on a readonly device. */ + if (osb->s_mount_opt & OCFS2_MOUNT_HB_LOCAL) { + status = -EROFS; + mlog(ML_ERROR, "Local heartbeat specified on readonly " + "device.\n"); + goto out_super; + } + + status = ocfs2_check_journals_nolocks(osb); + if (status < 0) { + if (status == -EROFS) + mlog(ML_ERROR, "Recovery required on readonly " + "file system, but write access is " + "unavailable.\n"); + goto out_super; + } + + ocfs2_set_ro_flag(osb, 1); + + printk(KERN_NOTICE "ocfs2: Readonly device (%s) detected. " + "Cluster services will not be used for this mount. " + "Recovery will be skipped.\n", osb->dev_str); + } + + if (!ocfs2_is_hard_readonly(osb)) { + if (sb_rdonly(sb)) + ocfs2_set_ro_flag(osb, 0); + } + + status = ocfs2_verify_heartbeat(osb); + if (status < 0) + goto out_super; + + osb->osb_debug_root = debugfs_create_dir(osb->uuid_str, + ocfs2_debugfs_root); + + debugfs_create_file("fs_state", S_IFREG|S_IRUSR, osb->osb_debug_root, + osb, &ocfs2_osb_debug_fops); + + if (ocfs2_meta_ecc(osb)) { + ocfs2_initialize_journal_triggers(sb, osb->s_journal_triggers); + ocfs2_blockcheck_stats_debugfs_install( &osb->osb_ecc_stats, + osb->osb_debug_root); + } + + status = ocfs2_mount_volume(sb); + if (status < 0) + goto out_debugfs; + + if (osb->root_inode) + inode = igrab(osb->root_inode); + + if (!inode) { + status = -EIO; + goto out_dismount; + } + + osb->osb_dev_kset = kset_create_and_add(sb->s_id, NULL, + &ocfs2_kset->kobj); + if (!osb->osb_dev_kset) { + status = -ENOMEM; + mlog(ML_ERROR, "Unable to create device kset %s.\n", sb->s_id); + goto out_dismount; + } + + /* Create filecheck sysfs related directories/files at + * /sys/fs/ocfs2//filecheck */ + if (ocfs2_filecheck_create_sysfs(osb)) { + status = -ENOMEM; + mlog(ML_ERROR, "Unable to create filecheck sysfs directory at " + "/sys/fs/ocfs2/%s/filecheck.\n", sb->s_id); + goto out_dismount; + } + + root = d_make_root(inode); + if (!root) { + status = -ENOMEM; + goto out_dismount; + } + + sb->s_root = root; + + ocfs2_complete_mount_recovery(osb); + + if (ocfs2_mount_local(osb)) + snprintf(nodestr, sizeof(nodestr), "local"); + else + snprintf(nodestr, sizeof(nodestr), "%u", osb->node_num); + + printk(KERN_INFO "ocfs2: Mounting device (%s) on (node %s, slot %d) " + "with %s data mode.\n", + osb->dev_str, nodestr, osb->slot_num, + osb->s_mount_opt & OCFS2_MOUNT_DATA_WRITEBACK ? "writeback" : + "ordered"); + + atomic_set(&osb->vol_state, VOLUME_MOUNTED); + wake_up(&osb->osb_mount_event); + + /* Now we can initialize quotas because we can afford to wait + * for cluster locks recovery now. That also means that truncation + * log recovery can happen but that waits for proper quota setup */ + if (!sb_rdonly(sb)) { + status = ocfs2_enable_quotas(osb); + if (status < 0) { + /* We have to err-out specially here because + * s_root is already set */ + mlog_errno(status); + atomic_set(&osb->vol_state, VOLUME_DISABLED); + wake_up(&osb->osb_mount_event); + return status; + } + } + + ocfs2_complete_quota_recovery(osb); + + /* Now we wake up again for processes waiting for quotas */ + atomic_set(&osb->vol_state, VOLUME_MOUNTED_QUOTAS); + wake_up(&osb->osb_mount_event); + + /* Start this when the mount is almost sure of being successful */ + ocfs2_orphan_scan_start(osb); + + return status; + +out_dismount: + atomic_set(&osb->vol_state, VOLUME_DISABLED); + wake_up(&osb->osb_mount_event); + ocfs2_free_replay_slots(osb); + ocfs2_dismount_volume(sb, 1); + goto out; + +out_debugfs: + debugfs_remove_recursive(osb->osb_debug_root); +out_super: + ocfs2_release_system_inodes(osb); + kfree(osb->recovery_map); + ocfs2_delete_osb(osb); + kfree(osb); +out: + mlog_errno(status); + + return status; +} + +static int ocfs2_get_tree(struct fs_context *fc) +{ + return get_tree_bdev(fc, ocfs2_fill_super); +} + +static void ocfs2_free_fc(struct fs_context *fc) +{ + kfree(fc->fs_private); +} + +static const struct fs_context_operations ocfs2_context_ops = { + .parse_param = ocfs2_parse_param, + .get_tree = ocfs2_get_tree, + .reconfigure = ocfs2_reconfigure, + .free = ocfs2_free_fc, +}; + +static int ocfs2_init_fs_context(struct fs_context *fc) +{ + struct mount_options *mopt; + + mopt = kzalloc_obj(struct mount_options); + if (!mopt) + return -EINVAL; + + mopt->commit_interval = 0; + mopt->mount_opt = OCFS2_MOUNT_NOINTR; + mopt->atime_quantum = OCFS2_DEFAULT_ATIME_QUANTUM; + mopt->slot = OCFS2_INVALID_SLOT; + mopt->localalloc_opt = -1; + mopt->cluster_stack[0] = '\0'; + mopt->resv_level = OCFS2_DEFAULT_RESV_LEVEL; + mopt->dir_resv_level = -1; + + fc->fs_private = mopt; + fc->ops = &ocfs2_context_ops; + + return 0; +} + +static struct file_system_type ocfs2_fs_type = { + .owner = THIS_MODULE, + .name = "ocfs2", + .kill_sb = kill_block_super, + .fs_flags = FS_REQUIRES_DEV|FS_RENAME_DOES_D_MOVE, + .init_fs_context = ocfs2_init_fs_context, + .parameters = ocfs2_param_spec, +}; +MODULE_ALIAS_FS("ocfs2"); + +static int ocfs2_check_set_options(struct super_block *sb, + struct mount_options *options) +{ + if (options->user_stack == 0) { + u32 tmp; + + /* Ensure only one heartbeat mode */ + tmp = options->mount_opt & (OCFS2_MOUNT_HB_LOCAL | + OCFS2_MOUNT_HB_GLOBAL | + OCFS2_MOUNT_HB_NONE); + if (hweight32(tmp) != 1) { + mlog(ML_ERROR, "Invalid heartbeat mount options\n"); + return 0; + } + } + if (options->mount_opt & OCFS2_MOUNT_USRQUOTA && + !OCFS2_HAS_RO_COMPAT_FEATURE(sb, + OCFS2_FEATURE_RO_COMPAT_USRQUOTA)) { + mlog(ML_ERROR, "User quotas were requested, but this " + "filesystem does not have the feature enabled.\n"); + return 0; + } + if (options->mount_opt & OCFS2_MOUNT_GRPQUOTA && + !OCFS2_HAS_RO_COMPAT_FEATURE(sb, + OCFS2_FEATURE_RO_COMPAT_GRPQUOTA)) { + mlog(ML_ERROR, "Group quotas were requested, but this " + "filesystem does not have the feature enabled.\n"); + return 0; + } + if (options->mount_opt & OCFS2_MOUNT_POSIX_ACL && + !OCFS2_HAS_INCOMPAT_FEATURE(sb, OCFS2_FEATURE_INCOMPAT_XATTR)) { + mlog(ML_ERROR, "ACL support requested but extended attributes " + "feature is not enabled\n"); + return 0; + } + /* No ACL setting specified? Use XATTR feature... */ + if (!(options->mount_opt & (OCFS2_MOUNT_POSIX_ACL | + OCFS2_MOUNT_NO_POSIX_ACL))) { + if (OCFS2_HAS_INCOMPAT_FEATURE(sb, OCFS2_FEATURE_INCOMPAT_XATTR)) + options->mount_opt |= OCFS2_MOUNT_POSIX_ACL; + else + options->mount_opt |= OCFS2_MOUNT_NO_POSIX_ACL; + } + return 1; +} + +static int ocfs2_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct fs_parse_result result; + int opt; + struct mount_options *mopt = fc->fs_private; + bool is_remount = (fc->purpose & FS_CONTEXT_FOR_RECONFIGURE); + + trace_ocfs2_parse_options(is_remount, param->key); + + opt = fs_parse(fc, ocfs2_param_spec, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_heartbeat: + mopt->mount_opt |= result.uint_32; + break; + case Opt_barrier: + if (result.uint_32) + mopt->mount_opt |= OCFS2_MOUNT_BARRIER; + else + mopt->mount_opt &= ~OCFS2_MOUNT_BARRIER; + break; + case Opt_intr: + if (result.negated) + mopt->mount_opt |= OCFS2_MOUNT_NOINTR; + else + mopt->mount_opt &= ~OCFS2_MOUNT_NOINTR; + break; + case Opt_errors: + mopt->mount_opt &= ~(OCFS2_MOUNT_ERRORS_CONT | + OCFS2_MOUNT_ERRORS_ROFS | + OCFS2_MOUNT_ERRORS_PANIC); + mopt->mount_opt |= result.uint_32; + break; + case Opt_data: + mopt->mount_opt &= ~OCFS2_MOUNT_DATA_WRITEBACK; + mopt->mount_opt |= result.uint_32; + break; + case Opt_user_xattr: + if (result.negated) + mopt->mount_opt |= OCFS2_MOUNT_NOUSERXATTR; + else + mopt->mount_opt &= ~OCFS2_MOUNT_NOUSERXATTR; + break; + case Opt_atime_quantum: + mopt->atime_quantum = result.uint_32; + break; + case Opt_slot: + if (result.uint_32) + mopt->slot = (u16)result.uint_32; + break; + case Opt_commit: + if (result.uint_32 == 0) + mopt->commit_interval = HZ * JBD2_DEFAULT_MAX_COMMIT_AGE; + else + mopt->commit_interval = HZ * result.uint_32; + break; + case Opt_localalloc: + if (result.int_32 >= 0) + mopt->localalloc_opt = result.int_32; + break; + case Opt_localflocks: + /* + * Changing this during remount could race flock() requests, or + * "unbalance" existing ones (e.g., a lock is taken in one mode + * but dropped in the other). If users care enough to flip + * locking modes during remount, we could add a "local" flag to + * individual flock structures for proper tracking of state. + */ + if (!is_remount) + mopt->mount_opt |= OCFS2_MOUNT_LOCALFLOCKS; + break; + case Opt_stack: + /* Check both that the option we were passed is of the right + * length and that it is a proper string of the right length. + */ + if (strlen(param->string) != OCFS2_STACK_LABEL_LEN) { + mlog(ML_ERROR, "Invalid cluster_stack option\n"); + return -EINVAL; + } + memcpy(mopt->cluster_stack, param->string, OCFS2_STACK_LABEL_LEN); + mopt->cluster_stack[OCFS2_STACK_LABEL_LEN] = '\0'; + /* + * Open code the memcmp here as we don't have an osb to pass + * to ocfs2_userspace_stack(). + */ + if (memcmp(mopt->cluster_stack, + OCFS2_CLASSIC_CLUSTER_STACK, + OCFS2_STACK_LABEL_LEN)) + mopt->user_stack = 1; + break; + case Opt_inode64: + mopt->mount_opt |= OCFS2_MOUNT_INODE64; + break; + case Opt_usrquota: + mopt->mount_opt |= OCFS2_MOUNT_USRQUOTA; + break; + case Opt_grpquota: + mopt->mount_opt |= OCFS2_MOUNT_GRPQUOTA; + break; + case Opt_coherency: + mopt->mount_opt &= ~OCFS2_MOUNT_COHERENCY_BUFFERED; + mopt->mount_opt |= result.uint_32; + break; + case Opt_acl: + if (result.negated) { + mopt->mount_opt |= OCFS2_MOUNT_NO_POSIX_ACL; + mopt->mount_opt &= ~OCFS2_MOUNT_POSIX_ACL; + } else { + mopt->mount_opt |= OCFS2_MOUNT_POSIX_ACL; + mopt->mount_opt &= ~OCFS2_MOUNT_NO_POSIX_ACL; + } + break; + case Opt_resv_level: + if (is_remount) + break; + if (result.uint_32 >= OCFS2_MIN_RESV_LEVEL && + result.uint_32 < OCFS2_MAX_RESV_LEVEL) + mopt->resv_level = result.uint_32; + break; + case Opt_dir_resv_level: + if (is_remount) + break; + if (result.uint_32 >= OCFS2_MIN_RESV_LEVEL && + result.uint_32 < OCFS2_MAX_RESV_LEVEL) + mopt->dir_resv_level = result.uint_32; + break; + case Opt_journal_async_commit: + mopt->mount_opt |= OCFS2_MOUNT_JOURNAL_ASYNC_COMMIT; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int ocfs2_show_options(struct seq_file *s, struct dentry *root) +{ + struct ocfs2_super *osb = OCFS2_SB(root->d_sb); + unsigned long opts = osb->s_mount_opt; + unsigned int local_alloc_megs; + + if (opts & (OCFS2_MOUNT_HB_LOCAL | OCFS2_MOUNT_HB_GLOBAL)) { + seq_printf(s, ",_netdev"); + if (opts & OCFS2_MOUNT_HB_LOCAL) + seq_printf(s, ",%s", OCFS2_HB_LOCAL); + else + seq_printf(s, ",%s", OCFS2_HB_GLOBAL); + } else + seq_printf(s, ",%s", OCFS2_HB_NONE); + + if (opts & OCFS2_MOUNT_NOINTR) + seq_printf(s, ",nointr"); + + if (opts & OCFS2_MOUNT_DATA_WRITEBACK) + seq_printf(s, ",data=writeback"); + else + seq_printf(s, ",data=ordered"); + + if (opts & OCFS2_MOUNT_BARRIER) + seq_printf(s, ",barrier=1"); + + if (opts & OCFS2_MOUNT_ERRORS_PANIC) + seq_printf(s, ",errors=panic"); + else if (opts & OCFS2_MOUNT_ERRORS_CONT) + seq_printf(s, ",errors=continue"); + else + seq_printf(s, ",errors=remount-ro"); + + if (osb->preferred_slot != OCFS2_INVALID_SLOT) + seq_printf(s, ",preferred_slot=%d", osb->preferred_slot); + + seq_printf(s, ",atime_quantum=%u", osb->s_atime_quantum); + + if (osb->osb_commit_interval) + seq_printf(s, ",commit=%u", + (unsigned) (osb->osb_commit_interval / HZ)); + + local_alloc_megs = osb->local_alloc_bits >> (20 - osb->s_clustersize_bits); + if (local_alloc_megs != ocfs2_la_default_mb(osb)) + seq_printf(s, ",localalloc=%d", local_alloc_megs); + + if (opts & OCFS2_MOUNT_LOCALFLOCKS) + seq_printf(s, ",localflocks,"); + + if (osb->osb_cluster_stack[0]) + seq_show_option(s, "cluster_stack", osb->osb_cluster_stack); + if (opts & OCFS2_MOUNT_USRQUOTA) + seq_printf(s, ",usrquota"); + if (opts & OCFS2_MOUNT_GRPQUOTA) + seq_printf(s, ",grpquota"); + + if (opts & OCFS2_MOUNT_COHERENCY_BUFFERED) + seq_printf(s, ",coherency=buffered"); + else + seq_printf(s, ",coherency=full"); + + if (opts & OCFS2_MOUNT_NOUSERXATTR) + seq_printf(s, ",nouser_xattr"); + else + seq_printf(s, ",user_xattr"); + + if (opts & OCFS2_MOUNT_INODE64) + seq_printf(s, ",inode64"); + + if (opts & OCFS2_MOUNT_POSIX_ACL) + seq_printf(s, ",acl"); + else + seq_printf(s, ",noacl"); + + if (osb->osb_resv_level != OCFS2_DEFAULT_RESV_LEVEL) + seq_printf(s, ",resv_level=%d", osb->osb_resv_level); + + if (osb->osb_dir_resv_level != osb->osb_resv_level) + seq_printf(s, ",dir_resv_level=%d", osb->osb_resv_level); + + if (opts & OCFS2_MOUNT_JOURNAL_ASYNC_COMMIT) + seq_printf(s, ",journal_async_commit"); + + return 0; +} + +static int __init ocfs2_init(void) +{ + int status; + + status = init_ocfs2_uptodate_cache(); + if (status < 0) + goto out1; + + status = ocfs2_initialize_mem_caches(); + if (status < 0) + goto out2; + + ocfs2_debugfs_root = debugfs_create_dir("ocfs2", NULL); + + ocfs2_set_locking_protocol(); + + register_quota_format(&ocfs2_quota_format); + + status = register_filesystem(&ocfs2_fs_type); + if (!status) + return 0; + + unregister_quota_format(&ocfs2_quota_format); + debugfs_remove(ocfs2_debugfs_root); + ocfs2_free_mem_caches(); +out2: + exit_ocfs2_uptodate_cache(); +out1: + mlog_errno(status); + return status; +} + +static void __exit ocfs2_exit(void) +{ + unregister_quota_format(&ocfs2_quota_format); + + debugfs_remove(ocfs2_debugfs_root); + + ocfs2_free_mem_caches(); + + unregister_filesystem(&ocfs2_fs_type); + + exit_ocfs2_uptodate_cache(); +} + +static void ocfs2_put_super(struct super_block *sb) +{ + trace_ocfs2_put_super(sb); + + ocfs2_sync_blockdev(sb); + ocfs2_dismount_volume(sb, 0); +} + +static int ocfs2_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct ocfs2_super *osb; + u32 numbits, freebits; + int status; + struct ocfs2_dinode *bm_lock; + struct buffer_head *bh = NULL; + struct inode *inode = NULL; + + trace_ocfs2_statfs(dentry->d_sb, buf); + + osb = OCFS2_SB(dentry->d_sb); + + inode = ocfs2_get_system_file_inode(osb, + GLOBAL_BITMAP_SYSTEM_INODE, + OCFS2_INVALID_SLOT); + if (!inode) { + mlog(ML_ERROR, "failed to get bitmap inode\n"); + status = -EIO; + goto bail; + } + + status = ocfs2_inode_lock(inode, &bh, 0); + if (status < 0) { + mlog_errno(status); + goto bail; + } + + bm_lock = (struct ocfs2_dinode *) bh->b_data; + + numbits = le32_to_cpu(bm_lock->id1.bitmap1.i_total); + freebits = numbits - le32_to_cpu(bm_lock->id1.bitmap1.i_used); + + buf->f_type = OCFS2_SUPER_MAGIC; + buf->f_bsize = dentry->d_sb->s_blocksize; + buf->f_namelen = OCFS2_MAX_FILENAME_LEN; + buf->f_blocks = ((sector_t) numbits) * + (osb->s_clustersize >> osb->sb->s_blocksize_bits); + buf->f_bfree = ((sector_t) freebits) * + (osb->s_clustersize >> osb->sb->s_blocksize_bits); + buf->f_bavail = buf->f_bfree; + buf->f_files = numbits; + buf->f_ffree = freebits; + buf->f_fsid.val[0] = crc32_le(0, osb->uuid_str, OCFS2_VOL_UUID_LEN) + & 0xFFFFFFFFUL; + buf->f_fsid.val[1] = crc32_le(0, osb->uuid_str + OCFS2_VOL_UUID_LEN, + OCFS2_VOL_UUID_LEN) & 0xFFFFFFFFUL; + + brelse(bh); + + ocfs2_inode_unlock(inode, 0); + status = 0; +bail: + iput(inode); + + if (status) + mlog_errno(status); + + return status; +} + +static void ocfs2_inode_init_once(void *data) +{ + struct ocfs2_inode_info *oi = data; + + oi->ip_flags = 0; + oi->ip_open_count = 0; + spin_lock_init(&oi->ip_lock); + ocfs2_extent_map_init(&oi->vfs_inode); + INIT_LIST_HEAD(&oi->ip_io_markers); + INIT_LIST_HEAD(&oi->ip_unwritten_list); + oi->ip_dir_start_lookup = 0; + init_rwsem(&oi->ip_alloc_sem); + init_rwsem(&oi->ip_xattr_sem); + mutex_init(&oi->ip_io_mutex); + + oi->ip_blkno = 0ULL; + oi->ip_clusters = 0; + oi->ip_next_orphan = NULL; + + ocfs2_resv_init_once(&oi->ip_la_data_resv); + + ocfs2_lock_res_init_once(&oi->ip_rw_lockres); + ocfs2_lock_res_init_once(&oi->ip_inode_lockres); + ocfs2_lock_res_init_once(&oi->ip_open_lockres); + + ocfs2_metadata_cache_init(INODE_CACHE(&oi->vfs_inode), + &ocfs2_inode_caching_ops); + + inode_init_once(&oi->vfs_inode); +} + +static int ocfs2_initialize_mem_caches(void) +{ + ocfs2_inode_cachep = kmem_cache_create("ocfs2_inode_cache", + sizeof(struct ocfs2_inode_info), + 0, + (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| + SLAB_ACCOUNT), + ocfs2_inode_init_once); + ocfs2_dquot_cachep = kmem_cache_create("ocfs2_dquot_cache", + sizeof(struct ocfs2_dquot), + 0, + SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT, + NULL); + ocfs2_qf_chunk_cachep = kmem_cache_create("ocfs2_qf_chunk_cache", + sizeof(struct ocfs2_quota_chunk), + 0, + SLAB_RECLAIM_ACCOUNT, + NULL); + if (!ocfs2_inode_cachep || !ocfs2_dquot_cachep || + !ocfs2_qf_chunk_cachep) { + kmem_cache_destroy(ocfs2_inode_cachep); + kmem_cache_destroy(ocfs2_dquot_cachep); + kmem_cache_destroy(ocfs2_qf_chunk_cachep); + return -ENOMEM; + } + + return 0; +} + +static void ocfs2_free_mem_caches(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(ocfs2_inode_cachep); + ocfs2_inode_cachep = NULL; + + kmem_cache_destroy(ocfs2_dquot_cachep); + ocfs2_dquot_cachep = NULL; + + kmem_cache_destroy(ocfs2_qf_chunk_cachep); + ocfs2_qf_chunk_cachep = NULL; +} + +static int ocfs2_get_sector(struct super_block *sb, + struct buffer_head **bh, + int block, + int sect_size) +{ + if (!sb_set_blocksize(sb, sect_size)) { + mlog(ML_ERROR, "unable to set blocksize\n"); + return -EIO; + } + + *bh = sb_getblk(sb, block); + if (!*bh) { + mlog_errno(-ENOMEM); + return -ENOMEM; + } + lock_buffer(*bh); + if (!buffer_dirty(*bh)) + clear_buffer_uptodate(*bh); + unlock_buffer(*bh); + if (bh_read(*bh, 0) < 0) { + mlog_errno(-EIO); + brelse(*bh); + *bh = NULL; + return -EIO; + } + + return 0; +} + +static int ocfs2_mount_volume(struct super_block *sb) +{ + int status = 0; + struct ocfs2_super *osb = OCFS2_SB(sb); + + if (ocfs2_is_hard_readonly(osb)) + goto out; + + mutex_init(&osb->obs_trim_fs_mutex); + + status = ocfs2_dlm_init(osb); + if (status < 0) { + mlog_errno(status); + if (status == -EBADR && ocfs2_userspace_stack(osb)) + mlog(ML_ERROR, "couldn't mount because cluster name on" + " disk does not match the running cluster name.\n"); + goto out; + } + + status = ocfs2_super_lock(osb, 1); + if (status < 0) { + mlog_errno(status); + goto out_dlm; + } + + /* This will load up the node map and add ourselves to it. */ + status = ocfs2_find_slot(osb); + if (status < 0) { + mlog_errno(status); + goto out_super_lock; + } + + /* load all node-local system inodes */ + status = ocfs2_init_local_system_inodes(osb); + if (status < 0) { + mlog_errno(status); + goto out_super_lock; + } + + status = ocfs2_check_volume(osb); + if (status < 0) { + mlog_errno(status); + goto out_system_inodes; + } + + status = ocfs2_truncate_log_init(osb); + if (status < 0) { + mlog_errno(status); + goto out_check_volume; + } + + ocfs2_super_unlock(osb, 1); + return 0; + +out_check_volume: + ocfs2_free_replay_slots(osb); +out_system_inodes: + if (osb->local_alloc_state == OCFS2_LA_ENABLED) + ocfs2_shutdown_local_alloc(osb); + ocfs2_release_system_inodes(osb); + /* before journal shutdown, we should release slot_info */ + ocfs2_free_slot_info(osb); + ocfs2_journal_shutdown(osb); +out_super_lock: + ocfs2_super_unlock(osb, 1); +out_dlm: + ocfs2_dlm_shutdown(osb, 0); +out: + return status; +} + +static void ocfs2_dismount_volume(struct super_block *sb, int mnt_err) +{ + int tmp, hangup_needed = 0; + struct ocfs2_super *osb = NULL; + char nodestr[12]; + + trace_ocfs2_dismount_volume(sb); + + BUG_ON(!sb); + osb = OCFS2_SB(sb); + BUG_ON(!osb); + + /* Remove file check sysfs related directories/files, + * and wait for the pending file check operations */ + ocfs2_filecheck_remove_sysfs(osb); + + kset_unregister(osb->osb_dev_kset); + + /* Orphan scan should be stopped as early as possible */ + ocfs2_orphan_scan_stop(osb); + + /* Stop quota recovery so that we can disable quotas */ + ocfs2_recovery_disable_quota(osb); + + ocfs2_disable_quotas(osb); + + /* All dquots should be freed by now */ + WARN_ON(!llist_empty(&osb->dquot_drop_list)); + /* Wait for worker to be done with the work structure in osb */ + cancel_work_sync(&osb->dquot_drop_work); + + ocfs2_shutdown_local_alloc(osb); + + ocfs2_truncate_log_shutdown(osb); + + /* This will disable recovery and flush any recovery work. */ + ocfs2_recovery_exit(osb); + + ocfs2_sync_blockdev(sb); + + ocfs2_purge_refcount_trees(osb); + + /* No cluster connection means we've failed during mount, so skip + * all the steps which depended on that to complete. */ + if (osb->cconn) { + tmp = ocfs2_super_lock(osb, 1); + if (tmp < 0) { + mlog_errno(tmp); + return; + } + } + + if (osb->slot_num != OCFS2_INVALID_SLOT) + ocfs2_put_slot(osb); + + if (osb->cconn) + ocfs2_super_unlock(osb, 1); + + ocfs2_release_system_inodes(osb); + + ocfs2_journal_shutdown(osb); + + /* + * If we're dismounting due to mount error, mount.ocfs2 will clean + * up heartbeat. If we're a local mount, there is no heartbeat. + * If we failed before we got a uuid_str yet, we can't stop + * heartbeat. Otherwise, do it. + */ + if (!mnt_err && !ocfs2_mount_local(osb) && osb->uuid_str && + !ocfs2_is_hard_readonly(osb)) + hangup_needed = 1; + + ocfs2_dlm_shutdown(osb, hangup_needed); + + ocfs2_blockcheck_stats_debugfs_remove(&osb->osb_ecc_stats); + debugfs_remove_recursive(osb->osb_debug_root); + + if (hangup_needed) + ocfs2_cluster_hangup(osb->uuid_str, strlen(osb->uuid_str)); + + atomic_set(&osb->vol_state, VOLUME_DISMOUNTED); + + if (ocfs2_mount_local(osb)) + snprintf(nodestr, sizeof(nodestr), "local"); + else + snprintf(nodestr, sizeof(nodestr), "%u", osb->node_num); + + printk(KERN_INFO "ocfs2: Unmounting device (%s) on (node %s)\n", + osb->dev_str, nodestr); + + ocfs2_delete_osb(osb); + kfree(osb); + sb->s_fs_info = NULL; +} + +static int ocfs2_setup_osb_uuid(struct ocfs2_super *osb, const unsigned char *uuid, + unsigned uuid_bytes) +{ + int i, ret; + char *ptr; + + BUG_ON(uuid_bytes != OCFS2_VOL_UUID_LEN); + + osb->uuid_str = kzalloc(OCFS2_VOL_UUID_LEN * 2 + 1, GFP_KERNEL); + if (osb->uuid_str == NULL) + return -ENOMEM; + + for (i = 0, ptr = osb->uuid_str; i < OCFS2_VOL_UUID_LEN; i++) { + /* print with null */ + ret = snprintf(ptr, 3, "%02X", uuid[i]); + if (ret != 2) /* drop super cleans up */ + return -EINVAL; + /* then only advance past the last char */ + ptr += 2; + } + + return 0; +} + +/* Make sure entire volume is addressable by our journal. Requires + osb_clusters_at_boot to be valid and for the journal to have been + initialized by ocfs2_journal_init(). */ +static int ocfs2_journal_addressable(struct ocfs2_super *osb) +{ + int status = 0; + u64 max_block = + ocfs2_clusters_to_blocks(osb->sb, + osb->osb_clusters_at_boot) - 1; + + /* 32-bit block number is always OK. */ + if (max_block <= (u32)~0ULL) + goto out; + + /* Volume is "huge", so see if our journal is new enough to + support it. */ + if (!(OCFS2_HAS_COMPAT_FEATURE(osb->sb, + OCFS2_FEATURE_COMPAT_JBD2_SB) && + jbd2_journal_check_used_features(osb->journal->j_journal, 0, 0, + JBD2_FEATURE_INCOMPAT_64BIT))) { + mlog(ML_ERROR, "The journal cannot address the entire volume. " + "Enable the 'block64' journal option with tunefs.ocfs2"); + status = -EFBIG; + goto out; + } + + out: + return status; +} + +static int ocfs2_initialize_super(struct super_block *sb, + struct buffer_head *bh, + int sector_size, + struct ocfs2_blockcheck_stats *stats) +{ + int status; + int i, cbits, bbits; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)bh->b_data; + struct inode *inode = NULL; + struct ocfs2_super *osb; + u64 total_blocks; + + osb = kzalloc_obj(struct ocfs2_super); + if (!osb) { + status = -ENOMEM; + mlog_errno(status); + goto out; + } + + sb->s_fs_info = osb; + sb->s_op = &ocfs2_sops; + set_default_d_op(sb, &ocfs2_dentry_ops); + sb->s_export_op = &ocfs2_export_ops; + sb->s_qcop = &dquot_quotactl_sysfile_ops; + sb->dq_op = &ocfs2_quota_operations; + sb->s_quota_types = QTYPE_MASK_USR | QTYPE_MASK_GRP; + sb->s_xattr = ocfs2_xattr_handlers; + sb->s_time_gran = 1; + sb->s_flags |= SB_NOATIME; + /* this is needed to support O_LARGEFILE */ + cbits = le32_to_cpu(di->id2.i_super.s_clustersize_bits); + bbits = le32_to_cpu(di->id2.i_super.s_blocksize_bits); + sb->s_maxbytes = ocfs2_max_file_offset(bbits, cbits); + super_set_uuid(sb, di->id2.i_super.s_uuid, + sizeof(di->id2.i_super.s_uuid)); + + osb->osb_dx_mask = (1 << (cbits - bbits)) - 1; + + for (i = 0; i < 3; i++) + osb->osb_dx_seed[i] = le32_to_cpu(di->id2.i_super.s_dx_seed[i]); + osb->osb_dx_seed[3] = le32_to_cpu(di->id2.i_super.s_uuid_hash); + + osb->sb = sb; + osb->s_sectsize_bits = blksize_bits(sector_size); + BUG_ON(!osb->s_sectsize_bits); + + spin_lock_init(&osb->dc_task_lock); + init_waitqueue_head(&osb->dc_event); + osb->dc_work_sequence = 0; + osb->dc_wake_sequence = 0; + INIT_LIST_HEAD(&osb->blocked_lock_list); + osb->blocked_lock_count = 0; + spin_lock_init(&osb->osb_lock); + spin_lock_init(&osb->osb_xattr_lock); + ocfs2_init_steal_slots(osb); + + atomic_set(&osb->alloc_stats.moves, 0); + atomic_set(&osb->alloc_stats.local_data, 0); + atomic_set(&osb->alloc_stats.bitmap_data, 0); + atomic_set(&osb->alloc_stats.bg_allocs, 0); + atomic_set(&osb->alloc_stats.bg_extends, 0); + + /* Copy the blockcheck stats from the superblock probe */ + osb->osb_ecc_stats = *stats; + + ocfs2_init_node_maps(osb); + + snprintf(osb->dev_str, sizeof(osb->dev_str), "%u,%u", + MAJOR(osb->sb->s_dev), MINOR(osb->sb->s_dev)); + + osb->max_slots = le16_to_cpu(di->id2.i_super.s_max_slots); + if (osb->max_slots > OCFS2_MAX_SLOTS || osb->max_slots == 0) { + mlog(ML_ERROR, "Invalid number of node slots (%u)\n", + osb->max_slots); + status = -EINVAL; + goto out; + } + + ocfs2_orphan_scan_init(osb); + + status = ocfs2_recovery_init(osb); + if (status) { + mlog(ML_ERROR, "Unable to initialize recovery state\n"); + mlog_errno(status); + goto out; + } + + init_waitqueue_head(&osb->checkpoint_event); + + osb->s_atime_quantum = OCFS2_DEFAULT_ATIME_QUANTUM; + + osb->slot_num = OCFS2_INVALID_SLOT; + + osb->s_xattr_inline_size = le16_to_cpu( + di->id2.i_super.s_xattr_inline_size); + + osb->local_alloc_state = OCFS2_LA_UNUSED; + osb->local_alloc_bh = NULL; + INIT_DELAYED_WORK(&osb->la_enable_wq, ocfs2_la_enable_worker); + + init_waitqueue_head(&osb->osb_mount_event); + + ocfs2_resmap_init(osb, &osb->osb_la_resmap); + + osb->vol_label = kmalloc(OCFS2_MAX_VOL_LABEL_LEN, GFP_KERNEL); + if (!osb->vol_label) { + mlog(ML_ERROR, "unable to alloc vol label\n"); + status = -ENOMEM; + goto out_recovery_map; + } + + osb->slot_recovery_generations = + kcalloc(osb->max_slots, sizeof(*osb->slot_recovery_generations), + GFP_KERNEL); + if (!osb->slot_recovery_generations) { + status = -ENOMEM; + mlog_errno(status); + goto out_vol_label; + } + + init_waitqueue_head(&osb->osb_wipe_event); + osb->osb_orphan_wipes = kcalloc(osb->max_slots, + sizeof(*osb->osb_orphan_wipes), + GFP_KERNEL); + if (!osb->osb_orphan_wipes) { + status = -ENOMEM; + mlog_errno(status); + goto out_slot_recovery_gen; + } + + osb->osb_rf_lock_tree = RB_ROOT; + + osb->s_feature_compat = + le32_to_cpu(OCFS2_RAW_SB(di)->s_feature_compat); + osb->s_feature_ro_compat = + le32_to_cpu(OCFS2_RAW_SB(di)->s_feature_ro_compat); + osb->s_feature_incompat = + le32_to_cpu(OCFS2_RAW_SB(di)->s_feature_incompat); + + if ((i = OCFS2_HAS_INCOMPAT_FEATURE(osb->sb, ~OCFS2_FEATURE_INCOMPAT_SUPP))) { + mlog(ML_ERROR, "couldn't mount because of unsupported " + "optional features (%x).\n", i); + status = -EINVAL; + goto out_orphan_wipes; + } + if (!sb_rdonly(osb->sb) && (i = OCFS2_HAS_RO_COMPAT_FEATURE(osb->sb, ~OCFS2_FEATURE_RO_COMPAT_SUPP))) { + mlog(ML_ERROR, "couldn't mount RDWR because of " + "unsupported optional features (%x).\n", i); + status = -EINVAL; + goto out_orphan_wipes; + } + + if (ocfs2_clusterinfo_valid(osb)) { + /* + * ci_stack and ci_cluster in ocfs2_cluster_info may not be null + * terminated, so make sure no overflow happens here by using + * memcpy. Destination strings will always be null terminated + * because osb is allocated using kzalloc. + */ + osb->osb_stackflags = + OCFS2_RAW_SB(di)->s_cluster_info.ci_stackflags; + memcpy(osb->osb_cluster_stack, + OCFS2_RAW_SB(di)->s_cluster_info.ci_stack, + OCFS2_STACK_LABEL_LEN); + if (strlen(osb->osb_cluster_stack) != OCFS2_STACK_LABEL_LEN) { + mlog(ML_ERROR, + "couldn't mount because of an invalid " + "cluster stack label (%s) \n", + osb->osb_cluster_stack); + status = -EINVAL; + goto out_orphan_wipes; + } + memcpy(osb->osb_cluster_name, + OCFS2_RAW_SB(di)->s_cluster_info.ci_cluster, + OCFS2_CLUSTER_NAME_LEN); + } else { + /* The empty string is identical with classic tools that + * don't know about s_cluster_info. */ + osb->osb_cluster_stack[0] = '\0'; + } + + osb->s_next_generation = get_random_u32(); + + /* + * FIXME + * This should be done in ocfs2_journal_init(), but any inode + * writes back operation will cause the filesystem to crash. + */ + status = ocfs2_journal_alloc(osb); + if (status < 0) + goto out_orphan_wipes; + + INIT_WORK(&osb->dquot_drop_work, ocfs2_drop_dquot_refs); + init_llist_head(&osb->dquot_drop_list); + + /* get some pseudo constants for clustersize bits */ + osb->s_clustersize_bits = + le32_to_cpu(di->id2.i_super.s_clustersize_bits); + osb->s_clustersize = 1 << osb->s_clustersize_bits; + + if (osb->s_clustersize < OCFS2_MIN_CLUSTERSIZE || + osb->s_clustersize > OCFS2_MAX_CLUSTERSIZE) { + mlog(ML_ERROR, "Volume has invalid cluster size (%d)\n", + osb->s_clustersize); + status = -EINVAL; + goto out_journal; + } + + total_blocks = ocfs2_clusters_to_blocks(osb->sb, + le32_to_cpu(di->i_clusters)); + + status = generic_check_addressable(osb->sb->s_blocksize_bits, + total_blocks); + if (status) { + mlog(ML_ERROR, "Volume too large " + "to mount safely on this system"); + status = -EFBIG; + goto out_journal; + } + + if (ocfs2_setup_osb_uuid(osb, di->id2.i_super.s_uuid, + sizeof(di->id2.i_super.s_uuid))) { + mlog(ML_ERROR, "Out of memory trying to setup our uuid.\n"); + status = -ENOMEM; + goto out_journal; + } + + strscpy(osb->vol_label, di->id2.i_super.s_label, + OCFS2_MAX_VOL_LABEL_LEN); + osb->root_blkno = le64_to_cpu(di->id2.i_super.s_root_blkno); + osb->system_dir_blkno = le64_to_cpu(di->id2.i_super.s_system_dir_blkno); + osb->first_cluster_group_blkno = + le64_to_cpu(di->id2.i_super.s_first_cluster_group); + osb->fs_generation = le32_to_cpu(di->i_fs_generation); + osb->uuid_hash = le32_to_cpu(di->id2.i_super.s_uuid_hash); + trace_ocfs2_initialize_super(osb->vol_label, osb->uuid_str, + (unsigned long long)osb->root_blkno, + (unsigned long long)osb->system_dir_blkno, + osb->s_clustersize_bits); + + osb->osb_dlm_debug = ocfs2_new_dlm_debug(); + if (!osb->osb_dlm_debug) { + status = -ENOMEM; + mlog_errno(status); + goto out_uuid_str; + } + + atomic_set(&osb->vol_state, VOLUME_INIT); + + /* load root, system_dir, and all global system inodes */ + status = ocfs2_init_global_system_inodes(osb); + if (status < 0) { + mlog_errno(status); + goto out_dlm_out; + } + + /* + * global bitmap + */ + inode = ocfs2_get_system_file_inode(osb, GLOBAL_BITMAP_SYSTEM_INODE, + OCFS2_INVALID_SLOT); + if (!inode) { + status = -EINVAL; + mlog_errno(status); + goto out_system_inodes; + } + + osb->bitmap_blkno = OCFS2_I(inode)->ip_blkno; + osb->osb_clusters_at_boot = OCFS2_I(inode)->ip_clusters; + iput(inode); + + osb->bitmap_cpg = ocfs2_group_bitmap_size(sb, 0, + osb->s_feature_incompat) * 8; + + status = ocfs2_init_slot_info(osb); + if (status < 0) { + mlog_errno(status); + goto out_system_inodes; + } + + osb->ocfs2_wq = alloc_ordered_workqueue("ocfs2_wq", WQ_MEM_RECLAIM); + if (!osb->ocfs2_wq) { + status = -ENOMEM; + mlog_errno(status); + goto out_slot_info; + } + + return status; + +out_slot_info: + ocfs2_free_slot_info(osb); +out_system_inodes: + ocfs2_release_system_inodes(osb); +out_dlm_out: + ocfs2_put_dlm_debug(osb->osb_dlm_debug); +out_uuid_str: + kfree(osb->uuid_str); +out_journal: + kfree(osb->journal); +out_orphan_wipes: + kfree(osb->osb_orphan_wipes); +out_slot_recovery_gen: + kfree(osb->slot_recovery_generations); +out_vol_label: + kfree(osb->vol_label); +out_recovery_map: + kfree(osb->recovery_map); +out: + kfree(osb); + sb->s_fs_info = NULL; + return status; +} + +/* + * will return: -EAGAIN if it is ok to keep searching for superblocks + * -EINVAL if there is a bad superblock + * 0 on success + */ +static int ocfs2_verify_volume(struct ocfs2_dinode *di, + struct buffer_head *bh, + u32 blksz, + struct ocfs2_blockcheck_stats *stats) +{ + int status = -EAGAIN; + u32 blksz_bits; + + if (memcmp(di->i_signature, OCFS2_SUPER_BLOCK_SIGNATURE, + strlen(OCFS2_SUPER_BLOCK_SIGNATURE)) == 0) { + /* We have to do a raw check of the feature here */ + if (le32_to_cpu(di->id2.i_super.s_feature_incompat) & + OCFS2_FEATURE_INCOMPAT_META_ECC) { + status = ocfs2_block_check_validate(bh->b_data, + bh->b_size, + &di->i_check, + stats); + if (status) + goto out; + } + status = -EINVAL; + /* Acceptable block sizes are 512 bytes, 1K, 2K and 4K. */ + blksz_bits = le32_to_cpu(di->id2.i_super.s_blocksize_bits); + if (blksz_bits < 9 || blksz_bits > 12) { + mlog(ML_ERROR, "found superblock with incorrect block " + "size bits: found %u, should be 9, 10, 11, or 12\n", + blksz_bits); + } else if ((1 << blksz_bits) != blksz) { + mlog(ML_ERROR, "found superblock with incorrect block " + "size: found %u, should be %u\n", 1 << blksz_bits, blksz); + } else if (le16_to_cpu(di->id2.i_super.s_major_rev_level) != + OCFS2_MAJOR_REV_LEVEL || + le16_to_cpu(di->id2.i_super.s_minor_rev_level) != + OCFS2_MINOR_REV_LEVEL) { + mlog(ML_ERROR, "found superblock with bad version: " + "found %u.%u, should be %u.%u\n", + le16_to_cpu(di->id2.i_super.s_major_rev_level), + le16_to_cpu(di->id2.i_super.s_minor_rev_level), + OCFS2_MAJOR_REV_LEVEL, + OCFS2_MINOR_REV_LEVEL); + } else if (bh->b_blocknr != le64_to_cpu(di->i_blkno)) { + mlog(ML_ERROR, "bad block number on superblock: " + "found %llu, should be %llu\n", + (unsigned long long)le64_to_cpu(di->i_blkno), + (unsigned long long)bh->b_blocknr); + } else if (le32_to_cpu(di->id2.i_super.s_clustersize_bits) < 12 || + le32_to_cpu(di->id2.i_super.s_clustersize_bits) > 20) { + mlog(ML_ERROR, "bad cluster size bit found: %u\n", + le32_to_cpu(di->id2.i_super.s_clustersize_bits)); + } else if (!le64_to_cpu(di->id2.i_super.s_root_blkno)) { + mlog(ML_ERROR, "bad root_blkno: 0\n"); + } else if (!le64_to_cpu(di->id2.i_super.s_system_dir_blkno)) { + mlog(ML_ERROR, "bad system_dir_blkno: 0\n"); + } else if (le16_to_cpu(di->id2.i_super.s_max_slots) > OCFS2_MAX_SLOTS) { + mlog(ML_ERROR, + "Superblock slots found greater than file system " + "maximum: found %u, max %u\n", + le16_to_cpu(di->id2.i_super.s_max_slots), + OCFS2_MAX_SLOTS); + } else { + /* found it! */ + status = 0; + } + } + +out: + if (status && status != -EAGAIN) + mlog_errno(status); + return status; +} + +static int ocfs2_check_volume(struct ocfs2_super *osb) +{ + int status; + int dirty; + int local; + struct ocfs2_dinode *local_alloc = NULL; /* only used if we + * recover + * ourselves. */ + + /* Init our journal object. */ + status = ocfs2_journal_init(osb, &dirty); + if (status < 0) { + mlog(ML_ERROR, "Could not initialize journal!\n"); + goto finally; + } + + /* Now that journal has been initialized, check to make sure + entire volume is addressable. */ + status = ocfs2_journal_addressable(osb); + if (status) + goto finally; + + /* If the journal was unmounted cleanly then we don't want to + * recover anything. Otherwise, journal_load will do that + * dirty work for us :) */ + if (!dirty) { + status = ocfs2_journal_wipe(osb->journal, 0); + if (status < 0) { + mlog_errno(status); + goto finally; + } + } else { + printk(KERN_NOTICE "ocfs2: File system on device (%s) was not " + "unmounted cleanly, recovering it.\n", osb->dev_str); + } + + local = ocfs2_mount_local(osb); + + /* will play back anything left in the journal. */ + status = ocfs2_journal_load(osb->journal, local, dirty); + if (status < 0) { + mlog(ML_ERROR, "ocfs2 journal load failed! %d\n", status); + goto finally; + } + + if (osb->s_mount_opt & OCFS2_MOUNT_JOURNAL_ASYNC_COMMIT) + jbd2_journal_set_features(osb->journal->j_journal, + JBD2_FEATURE_COMPAT_CHECKSUM, 0, + JBD2_FEATURE_INCOMPAT_ASYNC_COMMIT); + else + jbd2_journal_clear_features(osb->journal->j_journal, + JBD2_FEATURE_COMPAT_CHECKSUM, 0, + JBD2_FEATURE_INCOMPAT_ASYNC_COMMIT); + + if (dirty) { + /* recover my local alloc if we didn't unmount cleanly. */ + status = ocfs2_begin_local_alloc_recovery(osb, + osb->slot_num, + &local_alloc); + if (status < 0) { + mlog_errno(status); + goto finally; + } + /* we complete the recovery process after we've marked + * ourselves as mounted. */ + } + + status = ocfs2_load_local_alloc(osb); + if (status < 0) { + mlog_errno(status); + goto finally; + } + + if (dirty) { + /* Recovery will be completed after we've mounted the + * rest of the volume. */ + osb->local_alloc_copy = local_alloc; + local_alloc = NULL; + } + + /* go through each journal, trylock it and if you get the + * lock, and it's marked as dirty, set the bit in the recover + * map and launch a recovery thread for it. */ + status = ocfs2_mark_dead_nodes(osb); + if (status < 0) { + mlog_errno(status); + goto finally; + } + + status = ocfs2_compute_replay_slots(osb); + if (status < 0) + mlog_errno(status); + +finally: + kfree(local_alloc); + + if (status) + mlog_errno(status); + return status; +} + +/* + * The routine gets called from dismount or close whenever a dismount on + * volume is requested and the osb open count becomes 1. + * It will remove the osb from the global list and also free up all the + * initialized resources and fileobject. + */ +static void ocfs2_delete_osb(struct ocfs2_super *osb) +{ + /* This function assumes that the caller has the main osb resource */ + + /* ocfs2_initializer_super have already created this workqueue */ + if (osb->ocfs2_wq) + destroy_workqueue(osb->ocfs2_wq); + + ocfs2_free_slot_info(osb); + + kfree(osb->osb_orphan_wipes); + kfree(osb->slot_recovery_generations); + /* FIXME + * This belongs in journal shutdown, but because we have to + * allocate osb->journal at the middle of ocfs2_initialize_super(), + * we free it here. + */ + kfree(osb->journal); + kfree(osb->local_alloc_copy); + kfree(osb->uuid_str); + kfree(osb->vol_label); + ocfs2_put_dlm_debug(osb->osb_dlm_debug); + memset(osb, 0, sizeof(struct ocfs2_super)); +} + +/* Depending on the mount option passed, perform one of the following: + * Put OCFS2 into a readonly state (default) + * Return EIO so that only the process errs + * Fix the error as if fsck.ocfs2 -y + * panic + */ +static int ocfs2_handle_error(struct super_block *sb) +{ + struct ocfs2_super *osb = OCFS2_SB(sb); + int rv = 0; + + ocfs2_set_osb_flag(osb, OCFS2_OSB_ERROR_FS); + pr_crit("On-disk corruption discovered. " + "Please run fsck.ocfs2 once the filesystem is unmounted.\n"); + + if (osb->s_mount_opt & OCFS2_MOUNT_ERRORS_PANIC) { + panic("OCFS2: (device %s): panic forced after error\n", + sb->s_id); + } else if (osb->s_mount_opt & OCFS2_MOUNT_ERRORS_CONT) { + pr_crit("OCFS2: Returning error to the calling process.\n"); + rv = -EIO; + } else { /* default option */ + rv = -EROFS; + if (sb_rdonly(sb) && ocfs2_emergency_state(osb)) + return rv; + + pr_crit("OCFS2: File system is now read-only.\n"); + sb->s_flags |= SB_RDONLY; + ocfs2_set_ro_flag(osb, 0); + } + + return rv; +} + +int __ocfs2_error(struct super_block *sb, const char *function, + const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + + /* Not using mlog here because we want to show the actual + * function the error came from. */ + printk(KERN_CRIT "OCFS2: ERROR (device %s): %s: %pV", + sb->s_id, function, &vaf); + + va_end(args); + + return ocfs2_handle_error(sb); +} + +/* Handle critical errors. This is intentionally more drastic than + * ocfs2_handle_error, so we only use for things like journal errors, + * etc. */ +void __ocfs2_abort(struct super_block *sb, const char *function, + const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + printk(KERN_CRIT "OCFS2: abort (device %s): %s: %pV", + sb->s_id, function, &vaf); + + va_end(args); + + /* We don't have the cluster support yet to go straight to + * hard readonly in here. Until then, we want to keep + * ocfs2_abort() so that we can at least mark critical + * errors. + * + * TODO: This should abort the journal and alert other nodes + * that our slot needs recovery. */ + + /* Force a panic(). This stinks, but it's better than letting + * things continue without having a proper hard readonly + * here. */ + if (!ocfs2_mount_local(OCFS2_SB(sb))) + OCFS2_SB(sb)->s_mount_opt |= OCFS2_MOUNT_ERRORS_PANIC; + ocfs2_handle_error(sb); +} + +/* + * Void signal blockers, because in-kernel sigprocmask() only fails + * when SIG_* is wrong. + */ +void ocfs2_block_signals(sigset_t *oldset) +{ + int rc; + sigset_t blocked; + + sigfillset(&blocked); + rc = sigprocmask(SIG_BLOCK, &blocked, oldset); + BUG_ON(rc); +} + +void ocfs2_unblock_signals(sigset_t *oldset) +{ + int rc = sigprocmask(SIG_SETMASK, oldset, NULL); + BUG_ON(rc); +} + +module_init(ocfs2_init); +module_exit(ocfs2_exit); diff --git a/fs/ocfs2/super.h b/fs/ocfs2/super.h new file mode 100644 index 000000000..831265113 --- /dev/null +++ b/fs/ocfs2/super.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * super.h + * + * Function prototypes + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#ifndef OCFS2_SUPER_H +#define OCFS2_SUPER_H + +__printf(3, 4) +int __ocfs2_error(struct super_block *sb, const char *function, + const char *fmt, ...); + +#define ocfs2_error(sb, fmt, ...) \ + __ocfs2_error(sb, __PRETTY_FUNCTION__, fmt, ##__VA_ARGS__) + +__printf(3, 4) +void __ocfs2_abort(struct super_block *sb, const char *function, + const char *fmt, ...); + +#define ocfs2_abort(sb, fmt, ...) \ + __ocfs2_abort(sb, __PRETTY_FUNCTION__, fmt, ##__VA_ARGS__) + +/* + * Void signal blockers, because in-kernel sigprocmask() only fails + * when SIG_* is wrong. + */ +void ocfs2_block_signals(sigset_t *oldset); +void ocfs2_unblock_signals(sigset_t *oldset); + +#endif /* OCFS2_SUPER_H */ diff --git a/fs/ocfs2/symlink.c b/fs/ocfs2/symlink.c new file mode 100644 index 000000000..ad8be3300 --- /dev/null +++ b/fs/ocfs2/symlink.c @@ -0,0 +1,90 @@ +/* + * linux/cluster/ssi/cfs/symlink.c + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE + * or NON INFRINGEMENT. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * + * Questions/Comments/Bugfixes to ssic-linux-devel@lists.sourceforge.net + * + * Copyright (C) 1992 Rick Sladkey + * + * Optimization changes Copyright (C) 1994 Florian La Roche + * + * Jun 7 1999, cache symlink lookups in the page cache. -DaveM + * + * Portions Copyright (C) 2001 Compaq Computer Corporation + * + * ocfs2 symlink handling code. + * + * Copyright (C) 2004, 2005 Oracle. + * + */ + +#include +#include +#include +#include +#include + +#include + +#include "ocfs2.h" + +#include "alloc.h" +#include "file.h" +#include "inode.h" +#include "journal.h" +#include "symlink.h" +#include "xattr.h" + +#include "buffer_head_io.h" + + +static int ocfs2_fast_symlink_read_folio(struct file *f, struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + struct buffer_head *bh = NULL; + int status = ocfs2_read_inode_block(inode, &bh); + struct ocfs2_dinode *fe; + const char *link; + size_t len; + + if (status < 0) { + mlog_errno(status); + goto out; + } + + fe = (struct ocfs2_dinode *) bh->b_data; + link = (char *) fe->id2.i_symlink; + /* will be less than a page size */ + len = strnlen(link, ocfs2_fast_symlink_chars(inode->i_sb)); + memcpy_to_folio(folio, 0, link, len + 1); +out: + folio_end_read(folio, status == 0); + brelse(bh); + return status; +} + +const struct address_space_operations ocfs2_fast_symlink_aops = { + .read_folio = ocfs2_fast_symlink_read_folio, +}; + +const struct inode_operations ocfs2_symlink_inode_operations = { + .get_link = page_get_link, + .getattr = ocfs2_getattr, + .setattr = ocfs2_setattr, + .listxattr = ocfs2_listxattr, + .fiemap = ocfs2_fiemap, +}; diff --git a/fs/ocfs2/symlink.h b/fs/ocfs2/symlink.h new file mode 100644 index 000000000..ffcf02105 --- /dev/null +++ b/fs/ocfs2/symlink.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * symlink.h + * + * Function prototypes + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#ifndef OCFS2_SYMLINK_H +#define OCFS2_SYMLINK_H + +extern const struct inode_operations ocfs2_symlink_inode_operations; +extern const struct address_space_operations ocfs2_fast_symlink_aops; + +/* + * Test whether an inode is a fast symlink. + */ +static inline int ocfs2_inode_is_fast_symlink(struct inode *inode) +{ + return (S_ISLNK(inode->i_mode) && + inode->i_blocks == 0); +} + + +#endif /* OCFS2_SYMLINK_H */ diff --git a/fs/ocfs2/sysfile.c b/fs/ocfs2/sysfile.c new file mode 100644 index 000000000..67e492f4b --- /dev/null +++ b/fs/ocfs2/sysfile.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * sysfile.c + * + * Initialize, read, write, etc. system files. + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#include +#include +#include + +#include + +#include "ocfs2.h" + +#include "alloc.h" +#include "dir.h" +#include "inode.h" +#include "journal.h" +#include "sysfile.h" + +#include "buffer_head_io.h" + +static struct inode * _ocfs2_get_system_file_inode(struct ocfs2_super *osb, + int type, + u32 slot); + +#ifdef CONFIG_DEBUG_LOCK_ALLOC +static struct lock_class_key ocfs2_sysfile_cluster_lock_key[NUM_SYSTEM_INODES]; +#endif + +static inline int is_global_system_inode(int type) +{ + return type >= OCFS2_FIRST_ONLINE_SYSTEM_INODE && + type <= OCFS2_LAST_GLOBAL_SYSTEM_INODE; +} + +static struct inode **get_local_system_inode(struct ocfs2_super *osb, + int type, + u32 slot) +{ + int index; + struct inode **local_system_inodes, **free = NULL; + + BUG_ON(slot == OCFS2_INVALID_SLOT); + BUG_ON(type < OCFS2_FIRST_LOCAL_SYSTEM_INODE || + type > OCFS2_LAST_LOCAL_SYSTEM_INODE); + + spin_lock(&osb->osb_lock); + local_system_inodes = osb->local_system_inodes; + spin_unlock(&osb->osb_lock); + + if (unlikely(!local_system_inodes)) { + local_system_inodes = + kzalloc(array3_size(sizeof(struct inode *), + NUM_LOCAL_SYSTEM_INODES, + osb->max_slots), + GFP_NOFS); + if (!local_system_inodes) { + mlog_errno(-ENOMEM); + /* + * return NULL here so that ocfs2_get_sytem_file_inodes + * will try to create an inode and use it. We will try + * to initialize local_system_inodes next time. + */ + return NULL; + } + + spin_lock(&osb->osb_lock); + if (osb->local_system_inodes) { + /* Someone has initialized it for us. */ + free = local_system_inodes; + local_system_inodes = osb->local_system_inodes; + } else + osb->local_system_inodes = local_system_inodes; + spin_unlock(&osb->osb_lock); + kfree(free); + } + + index = (slot * NUM_LOCAL_SYSTEM_INODES) + + (type - OCFS2_FIRST_LOCAL_SYSTEM_INODE); + + return &local_system_inodes[index]; +} + +struct inode *ocfs2_get_system_file_inode(struct ocfs2_super *osb, + int type, + u32 slot) +{ + struct inode *inode = NULL; + struct inode **arr = NULL; + + /* avoid the lookup if cached in local system file array */ + if (is_global_system_inode(type)) { + arr = &(osb->global_system_inodes[type]); + } else + arr = get_local_system_inode(osb, type, slot); + + if (arr && ((inode = *arr) != NULL)) { + /* get a ref in addition to the array ref */ + inode = igrab(inode); + BUG_ON(!inode); + + return inode; + } + + /* this gets one ref thru iget */ + inode = _ocfs2_get_system_file_inode(osb, type, slot); + + /* add one more if putting into array for first time */ + if (inode && arr && !*arr && !cmpxchg(&(*arr), NULL, inode)) { + inode = igrab(inode); + BUG_ON(!inode); + } + return inode; +} + +static struct inode * _ocfs2_get_system_file_inode(struct ocfs2_super *osb, + int type, + u32 slot) +{ + char namebuf[40]; + struct inode *inode = NULL; + u64 blkno; + int len, status = 0; + + len = ocfs2_sprintf_system_inode_name(namebuf, + sizeof(namebuf), + type, slot); + + status = ocfs2_lookup_ino_from_name(osb->sys_root_inode, + namebuf, len, &blkno); + if (status < 0) { + goto bail; + } + + inode = ocfs2_iget(osb, blkno, OCFS2_FI_FLAG_SYSFILE, type); + if (IS_ERR(inode)) { + mlog_errno(PTR_ERR(inode)); + inode = NULL; + goto bail; + } +#ifdef CONFIG_DEBUG_LOCK_ALLOC + if (type == LOCAL_USER_QUOTA_SYSTEM_INODE || + type == LOCAL_GROUP_QUOTA_SYSTEM_INODE || + type == JOURNAL_SYSTEM_INODE) { + /* Ignore inode lock on these inodes as the lock does not + * really belong to any process and lockdep cannot handle + * that */ + OCFS2_I(inode)->ip_inode_lockres.l_lockdep_map.key = NULL; + } else { + lockdep_init_map(&OCFS2_I(inode)->ip_inode_lockres. + l_lockdep_map, + ocfs2_system_inodes[type].si_name, + &ocfs2_sysfile_cluster_lock_key[type], 0); + } +#endif +bail: + + return inode; +} + diff --git a/fs/ocfs2/sysfile.h b/fs/ocfs2/sysfile.h new file mode 100644 index 000000000..2b38c7599 --- /dev/null +++ b/fs/ocfs2/sysfile.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * sysfile.h + * + * Function prototypes + * + * Copyright (C) 2002, 2004 Oracle. All rights reserved. + */ + +#ifndef OCFS2_SYSFILE_H +#define OCFS2_SYSFILE_H + +struct inode * ocfs2_get_system_file_inode(struct ocfs2_super *osb, + int type, + u32 slot); + +#endif /* OCFS2_SYSFILE_H */ diff --git a/fs/ocfs2/uptodate.c b/fs/ocfs2/uptodate.c new file mode 100644 index 000000000..09854925f --- /dev/null +++ b/fs/ocfs2/uptodate.c @@ -0,0 +1,621 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * uptodate.c + * + * Tracking the up-to-date-ness of a local buffer_head with respect to + * the cluster. + * + * Copyright (C) 2002, 2004, 2005 Oracle. All rights reserved. + * + * Standard buffer head caching flags (uptodate, etc) are insufficient + * in a clustered environment - a buffer may be marked up to date on + * our local node but could have been modified by another cluster + * member. As a result an additional (and performant) caching scheme + * is required. A further requirement is that we consume as little + * memory as possible - we never pin buffer_head structures in order + * to cache them. + * + * We track the existence of up to date buffers on the inodes which + * are associated with them. Because we don't want to pin + * buffer_heads, this is only a (strong) hint and several other checks + * are made in the I/O path to ensure that we don't use a stale or + * invalid buffer without going to disk: + * - buffer_jbd is used liberally - if a bh is in the journal on + * this node then it *must* be up to date. + * - the standard buffer_uptodate() macro is used to detect buffers + * which may be invalid (even if we have an up to date tracking + * item for them) + * + * For a full understanding of how this code works together, one + * should read the callers in dlmglue.c, the I/O functions in + * buffer_head_io.c and ocfs2_journal_access in journal.c + */ + +#include +#include +#include +#include +#include +#include + +#include + +#include "ocfs2.h" + +#include "inode.h" +#include "uptodate.h" +#include "ocfs2_trace.h" + +struct ocfs2_meta_cache_item { + struct rb_node c_node; + sector_t c_block; +}; + +static struct kmem_cache *ocfs2_uptodate_cachep; + +u64 ocfs2_metadata_cache_owner(struct ocfs2_caching_info *ci) +{ + BUG_ON(!ci || !ci->ci_ops); + + return ci->ci_ops->co_owner(ci); +} + +struct super_block *ocfs2_metadata_cache_get_super(struct ocfs2_caching_info *ci) +{ + BUG_ON(!ci || !ci->ci_ops); + + return ci->ci_ops->co_get_super(ci); +} + +static void ocfs2_metadata_cache_lock(struct ocfs2_caching_info *ci) +{ + BUG_ON(!ci || !ci->ci_ops); + + ci->ci_ops->co_cache_lock(ci); +} + +static void ocfs2_metadata_cache_unlock(struct ocfs2_caching_info *ci) +{ + BUG_ON(!ci || !ci->ci_ops); + + ci->ci_ops->co_cache_unlock(ci); +} + +void ocfs2_metadata_cache_io_lock(struct ocfs2_caching_info *ci) +{ + BUG_ON(!ci || !ci->ci_ops); + + ci->ci_ops->co_io_lock(ci); +} + +void ocfs2_metadata_cache_io_unlock(struct ocfs2_caching_info *ci) +{ + BUG_ON(!ci || !ci->ci_ops); + + ci->ci_ops->co_io_unlock(ci); +} + + +static void ocfs2_metadata_cache_reset(struct ocfs2_caching_info *ci, + int clear) +{ + ci->ci_flags |= OCFS2_CACHE_FL_INLINE; + ci->ci_num_cached = 0; + + if (clear) { + ci->ci_created_trans = 0; + ci->ci_last_trans = 0; + } +} + +void ocfs2_metadata_cache_init(struct ocfs2_caching_info *ci, + const struct ocfs2_caching_operations *ops) +{ + BUG_ON(!ops); + + ci->ci_ops = ops; + ocfs2_metadata_cache_reset(ci, 1); +} + +void ocfs2_metadata_cache_exit(struct ocfs2_caching_info *ci) +{ + ocfs2_metadata_cache_purge(ci); + ocfs2_metadata_cache_reset(ci, 1); +} + + +/* No lock taken here as 'root' is not expected to be visible to other + * processes. */ +static unsigned int ocfs2_purge_copied_metadata_tree(struct rb_root *root) +{ + unsigned int purged = 0; + struct rb_node *node; + struct ocfs2_meta_cache_item *item; + + while ((node = rb_last(root)) != NULL) { + item = rb_entry(node, struct ocfs2_meta_cache_item, c_node); + + trace_ocfs2_purge_copied_metadata_tree( + (unsigned long long) item->c_block); + + rb_erase(&item->c_node, root); + kmem_cache_free(ocfs2_uptodate_cachep, item); + + purged++; + } + return purged; +} + +/* Called from locking and called from ocfs2_clear_inode. Dump the + * cache for a given inode. + * + * This function is a few more lines longer than necessary due to some + * accounting done here, but I think it's worth tracking down those + * bugs sooner -- Mark */ +void ocfs2_metadata_cache_purge(struct ocfs2_caching_info *ci) +{ + unsigned int tree, to_purge, purged; + struct rb_root root = RB_ROOT; + + BUG_ON(!ci || !ci->ci_ops); + + ocfs2_metadata_cache_lock(ci); + tree = !(ci->ci_flags & OCFS2_CACHE_FL_INLINE); + to_purge = ci->ci_num_cached; + + trace_ocfs2_metadata_cache_purge( + (unsigned long long)ocfs2_metadata_cache_owner(ci), + to_purge, tree); + + /* If we're a tree, save off the root so that we can safely + * initialize the cache. We do the work to free tree members + * without the spinlock. */ + if (tree) + root = ci->ci_cache.ci_tree; + + ocfs2_metadata_cache_reset(ci, 0); + ocfs2_metadata_cache_unlock(ci); + + purged = ocfs2_purge_copied_metadata_tree(&root); + /* If possible, track the number wiped so that we can more + * easily detect counting errors. Unfortunately, this is only + * meaningful for trees. */ + if (tree && purged != to_purge) + mlog(ML_ERROR, "Owner %llu, count = %u, purged = %u\n", + (unsigned long long)ocfs2_metadata_cache_owner(ci), + to_purge, purged); +} + +/* Returns the index in the cache array, -1 if not found. + * Requires ip_lock. */ +static int ocfs2_search_cache_array(struct ocfs2_caching_info *ci, + sector_t item) +{ + int i; + + for (i = 0; i < ci->ci_num_cached; i++) { + if (item == ci->ci_cache.ci_array[i]) + return i; + } + + return -1; +} + +/* Returns the cache item if found, otherwise NULL. + * Requires ip_lock. */ +static struct ocfs2_meta_cache_item * +ocfs2_search_cache_tree(struct ocfs2_caching_info *ci, + sector_t block) +{ + struct rb_node * n = ci->ci_cache.ci_tree.rb_node; + struct ocfs2_meta_cache_item *item = NULL; + + while (n) { + item = rb_entry(n, struct ocfs2_meta_cache_item, c_node); + + if (block < item->c_block) + n = n->rb_left; + else if (block > item->c_block) + n = n->rb_right; + else + return item; + } + + return NULL; +} + +static int ocfs2_buffer_cached(struct ocfs2_caching_info *ci, + struct buffer_head *bh) +{ + int index = -1; + struct ocfs2_meta_cache_item *item = NULL; + + ocfs2_metadata_cache_lock(ci); + + trace_ocfs2_buffer_cached_begin( + (unsigned long long)ocfs2_metadata_cache_owner(ci), + (unsigned long long) bh->b_blocknr, + !!(ci->ci_flags & OCFS2_CACHE_FL_INLINE)); + + if (ci->ci_flags & OCFS2_CACHE_FL_INLINE) + index = ocfs2_search_cache_array(ci, bh->b_blocknr); + else + item = ocfs2_search_cache_tree(ci, bh->b_blocknr); + + ocfs2_metadata_cache_unlock(ci); + + trace_ocfs2_buffer_cached_end(index, item); + + return (index != -1) || (item != NULL); +} + +/* Warning: even if it returns true, this does *not* guarantee that + * the block is stored in our inode metadata cache. + * + * This can be called under lock_buffer() + */ +int ocfs2_buffer_uptodate(struct ocfs2_caching_info *ci, + struct buffer_head *bh) +{ + /* Doesn't matter if the bh is in our cache or not -- if it's + * not marked uptodate then we know it can't have correct + * data. */ + if (!buffer_uptodate(bh)) + return 0; + + /* OCFS2 does not allow multiple nodes to be changing the same + * block at the same time. */ + if (buffer_jbd(bh)) + return 1; + + /* Ok, locally the buffer is marked as up to date, now search + * our cache to see if we can trust that. */ + return ocfs2_buffer_cached(ci, bh); +} + +/* + * Determine whether a buffer is currently out on a read-ahead request. + * ci_io_sem should be held to serialize submitters with the logic here. + */ +int ocfs2_buffer_read_ahead(struct ocfs2_caching_info *ci, + struct buffer_head *bh) +{ + return buffer_locked(bh) && ocfs2_buffer_cached(ci, bh); +} + +/* Requires ip_lock */ +static void ocfs2_append_cache_array(struct ocfs2_caching_info *ci, + sector_t block) +{ + BUG_ON(ci->ci_num_cached >= OCFS2_CACHE_INFO_MAX_ARRAY); + + trace_ocfs2_append_cache_array( + (unsigned long long)ocfs2_metadata_cache_owner(ci), + (unsigned long long)block, ci->ci_num_cached); + + ci->ci_cache.ci_array[ci->ci_num_cached] = block; + ci->ci_num_cached++; +} + +/* By now the caller should have checked that the item does *not* + * exist in the tree. + * Requires ip_lock. */ +static void __ocfs2_insert_cache_tree(struct ocfs2_caching_info *ci, + struct ocfs2_meta_cache_item *new) +{ + sector_t block = new->c_block; + struct rb_node *parent = NULL; + struct rb_node **p = &ci->ci_cache.ci_tree.rb_node; + struct ocfs2_meta_cache_item *tmp; + + trace_ocfs2_insert_cache_tree( + (unsigned long long)ocfs2_metadata_cache_owner(ci), + (unsigned long long)block, ci->ci_num_cached); + + while(*p) { + parent = *p; + + tmp = rb_entry(parent, struct ocfs2_meta_cache_item, c_node); + + if (block < tmp->c_block) + p = &(*p)->rb_left; + else if (block > tmp->c_block) + p = &(*p)->rb_right; + else { + /* This should never happen! */ + mlog(ML_ERROR, "Duplicate block %llu cached!\n", + (unsigned long long) block); + BUG(); + } + } + + rb_link_node(&new->c_node, parent, p); + rb_insert_color(&new->c_node, &ci->ci_cache.ci_tree); + ci->ci_num_cached++; +} + +/* co_cache_lock() must be held */ +static inline int ocfs2_insert_can_use_array(struct ocfs2_caching_info *ci) +{ + return (ci->ci_flags & OCFS2_CACHE_FL_INLINE) && + (ci->ci_num_cached < OCFS2_CACHE_INFO_MAX_ARRAY); +} + +/* tree should be exactly OCFS2_CACHE_INFO_MAX_ARRAY wide. NULL the + * pointers in tree after we use them - this allows caller to detect + * when to free in case of error. + * + * The co_cache_lock() must be held. */ +static void ocfs2_expand_cache(struct ocfs2_caching_info *ci, + struct ocfs2_meta_cache_item **tree) +{ + int i; + + mlog_bug_on_msg(ci->ci_num_cached != OCFS2_CACHE_INFO_MAX_ARRAY, + "Owner %llu, num cached = %u, should be %u\n", + (unsigned long long)ocfs2_metadata_cache_owner(ci), + ci->ci_num_cached, OCFS2_CACHE_INFO_MAX_ARRAY); + mlog_bug_on_msg(!(ci->ci_flags & OCFS2_CACHE_FL_INLINE), + "Owner %llu not marked as inline anymore!\n", + (unsigned long long)ocfs2_metadata_cache_owner(ci)); + + /* Be careful to initialize the tree members *first* because + * once the ci_tree is used, the array is junk... */ + for (i = 0; i < OCFS2_CACHE_INFO_MAX_ARRAY; i++) + tree[i]->c_block = ci->ci_cache.ci_array[i]; + + ci->ci_flags &= ~OCFS2_CACHE_FL_INLINE; + ci->ci_cache.ci_tree = RB_ROOT; + /* this will be set again by __ocfs2_insert_cache_tree */ + ci->ci_num_cached = 0; + + for (i = 0; i < OCFS2_CACHE_INFO_MAX_ARRAY; i++) { + __ocfs2_insert_cache_tree(ci, tree[i]); + tree[i] = NULL; + } + + trace_ocfs2_expand_cache( + (unsigned long long)ocfs2_metadata_cache_owner(ci), + ci->ci_flags, ci->ci_num_cached); +} + +/* Slow path function - memory allocation is necessary. See the + * comment above ocfs2_set_buffer_uptodate for more information. */ +static void __ocfs2_set_buffer_uptodate(struct ocfs2_caching_info *ci, + sector_t block, + int expand_tree) +{ + int i; + struct ocfs2_meta_cache_item *new = NULL; + struct ocfs2_meta_cache_item *tree[OCFS2_CACHE_INFO_MAX_ARRAY] = + { NULL, }; + + trace_ocfs2_set_buffer_uptodate( + (unsigned long long)ocfs2_metadata_cache_owner(ci), + (unsigned long long)block, expand_tree); + + new = kmem_cache_alloc(ocfs2_uptodate_cachep, GFP_NOFS); + if (!new) { + mlog_errno(-ENOMEM); + return; + } + new->c_block = block; + + if (expand_tree) { + /* Do *not* allocate an array here - the removal code + * has no way of tracking that. */ + for (i = 0; i < OCFS2_CACHE_INFO_MAX_ARRAY; i++) { + tree[i] = kmem_cache_alloc(ocfs2_uptodate_cachep, + GFP_NOFS); + if (!tree[i]) { + mlog_errno(-ENOMEM); + goto out_free; + } + + /* These are initialized in ocfs2_expand_cache! */ + } + } + + ocfs2_metadata_cache_lock(ci); + if (ocfs2_insert_can_use_array(ci)) { + /* Ok, items were removed from the cache in between + * locks. Detect this and revert back to the fast path */ + ocfs2_append_cache_array(ci, block); + ocfs2_metadata_cache_unlock(ci); + goto out_free; + } + + if (expand_tree) + ocfs2_expand_cache(ci, tree); + + __ocfs2_insert_cache_tree(ci, new); + ocfs2_metadata_cache_unlock(ci); + + new = NULL; +out_free: + if (new) + kmem_cache_free(ocfs2_uptodate_cachep, new); + + /* If these were used, then ocfs2_expand_cache re-set them to + * NULL for us. */ + if (tree[0]) { + for (i = 0; i < OCFS2_CACHE_INFO_MAX_ARRAY; i++) + if (tree[i]) + kmem_cache_free(ocfs2_uptodate_cachep, + tree[i]); + } +} + +/* Item insertion is guarded by co_io_lock(), so the insertion path takes + * advantage of this by not rechecking for a duplicate insert during + * the slow case. Additionally, if the cache needs to be bumped up to + * a tree, the code will not recheck after acquiring the lock -- + * multiple paths cannot be expanding to a tree at the same time. + * + * The slow path takes into account that items can be removed + * (including the whole tree wiped and reset) when this process it out + * allocating memory. In those cases, it reverts back to the fast + * path. + * + * Note that this function may actually fail to insert the block if + * memory cannot be allocated. This is not fatal however (but may + * result in a performance penalty) + * + * Readahead buffers can be passed in here before the I/O request is + * completed. + */ +void ocfs2_set_buffer_uptodate(struct ocfs2_caching_info *ci, + struct buffer_head *bh) +{ + int expand; + + /* The block may very well exist in our cache already, so avoid + * doing any more work in that case. */ + if (ocfs2_buffer_cached(ci, bh)) + return; + + trace_ocfs2_set_buffer_uptodate_begin( + (unsigned long long)ocfs2_metadata_cache_owner(ci), + (unsigned long long)bh->b_blocknr); + + /* No need to recheck under spinlock - insertion is guarded by + * co_io_lock() */ + ocfs2_metadata_cache_lock(ci); + if (ocfs2_insert_can_use_array(ci)) { + /* Fast case - it's an array and there's a free + * spot. */ + ocfs2_append_cache_array(ci, bh->b_blocknr); + ocfs2_metadata_cache_unlock(ci); + return; + } + + expand = 0; + if (ci->ci_flags & OCFS2_CACHE_FL_INLINE) { + /* We need to bump things up to a tree. */ + expand = 1; + } + ocfs2_metadata_cache_unlock(ci); + + __ocfs2_set_buffer_uptodate(ci, bh->b_blocknr, expand); +} + +/* Called against a newly allocated buffer. Most likely nobody should + * be able to read this sort of metadata while it's still being + * allocated, but this is careful to take co_io_lock() anyway. */ +void ocfs2_set_new_buffer_uptodate(struct ocfs2_caching_info *ci, + struct buffer_head *bh) +{ + /* This should definitely *not* exist in our cache */ + BUG_ON(ocfs2_buffer_cached(ci, bh)); + + set_buffer_uptodate(bh); + + ocfs2_metadata_cache_io_lock(ci); + ocfs2_set_buffer_uptodate(ci, bh); + ocfs2_metadata_cache_io_unlock(ci); +} + +/* Requires ip_lock. */ +static void ocfs2_remove_metadata_array(struct ocfs2_caching_info *ci, + int index) +{ + sector_t *array = ci->ci_cache.ci_array; + int bytes; + + BUG_ON(index < 0 || index >= OCFS2_CACHE_INFO_MAX_ARRAY); + BUG_ON(index >= ci->ci_num_cached); + BUG_ON(!ci->ci_num_cached); + + trace_ocfs2_remove_metadata_array( + (unsigned long long)ocfs2_metadata_cache_owner(ci), + index, ci->ci_num_cached); + + ci->ci_num_cached--; + + /* don't need to copy if the array is now empty, or if we + * removed at the tail */ + if (ci->ci_num_cached && index < ci->ci_num_cached) { + bytes = sizeof(sector_t) * (ci->ci_num_cached - index); + memmove(&array[index], &array[index + 1], bytes); + } +} + +/* Requires ip_lock. */ +static void ocfs2_remove_metadata_tree(struct ocfs2_caching_info *ci, + struct ocfs2_meta_cache_item *item) +{ + trace_ocfs2_remove_metadata_tree( + (unsigned long long)ocfs2_metadata_cache_owner(ci), + (unsigned long long)item->c_block); + + rb_erase(&item->c_node, &ci->ci_cache.ci_tree); + ci->ci_num_cached--; +} + +static void ocfs2_remove_block_from_cache(struct ocfs2_caching_info *ci, + sector_t block) +{ + int index; + struct ocfs2_meta_cache_item *item = NULL; + + ocfs2_metadata_cache_lock(ci); + trace_ocfs2_remove_block_from_cache( + (unsigned long long)ocfs2_metadata_cache_owner(ci), + (unsigned long long) block, ci->ci_num_cached, + ci->ci_flags); + + if (ci->ci_flags & OCFS2_CACHE_FL_INLINE) { + index = ocfs2_search_cache_array(ci, block); + if (index != -1) + ocfs2_remove_metadata_array(ci, index); + } else { + item = ocfs2_search_cache_tree(ci, block); + if (item) + ocfs2_remove_metadata_tree(ci, item); + } + ocfs2_metadata_cache_unlock(ci); + + if (item) + kmem_cache_free(ocfs2_uptodate_cachep, item); +} + +/* + * Called when we remove a chunk of metadata from an inode. We don't + * bother reverting things to an inlined array in the case of a remove + * which moves us back under the limit. + */ +void ocfs2_remove_from_cache(struct ocfs2_caching_info *ci, + struct buffer_head *bh) +{ + sector_t block = bh->b_blocknr; + + ocfs2_remove_block_from_cache(ci, block); +} + +/* Called when we remove xattr clusters from an inode. */ +void ocfs2_remove_xattr_clusters_from_cache(struct ocfs2_caching_info *ci, + sector_t block, + u32 c_len) +{ + struct super_block *sb = ocfs2_metadata_cache_get_super(ci); + unsigned int i, b_len = ocfs2_clusters_to_blocks(sb, 1) * c_len; + + for (i = 0; i < b_len; i++, block++) + ocfs2_remove_block_from_cache(ci, block); +} + +int __init init_ocfs2_uptodate_cache(void) +{ + ocfs2_uptodate_cachep = kmem_cache_create("ocfs2_uptodate", + sizeof(struct ocfs2_meta_cache_item), + 0, SLAB_HWCACHE_ALIGN, NULL); + if (!ocfs2_uptodate_cachep) + return -ENOMEM; + + return 0; +} + +void exit_ocfs2_uptodate_cache(void) +{ + kmem_cache_destroy(ocfs2_uptodate_cachep); +} diff --git a/fs/ocfs2/uptodate.h b/fs/ocfs2/uptodate.h new file mode 100644 index 000000000..85d941340 --- /dev/null +++ b/fs/ocfs2/uptodate.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * uptodate.h + * + * Cluster uptodate tracking + * + * Copyright (C) 2002, 2004, 2005 Oracle. All rights reserved. + */ + +#ifndef OCFS2_UPTODATE_H +#define OCFS2_UPTODATE_H + +/* + * The caching code relies on locking provided by the user of + * struct ocfs2_caching_info. These operations connect that up. + */ +struct ocfs2_caching_operations { + /* + * A u64 representing the owning structure. Usually this + * is the block number (i_blkno or whatnot). This is used so + * that caching log messages can identify the owning structure. + */ + u64 (*co_owner)(struct ocfs2_caching_info *ci); + + /* The superblock is needed during I/O. */ + struct super_block *(*co_get_super)(struct ocfs2_caching_info *ci); + /* + * Lock and unlock the caching data. These will not sleep, and + * should probably be spinlocks. + */ + void (*co_cache_lock)(struct ocfs2_caching_info *ci); + void (*co_cache_unlock)(struct ocfs2_caching_info *ci); + + /* + * Lock and unlock for disk I/O. These will sleep, and should + * be mutexes. + */ + void (*co_io_lock)(struct ocfs2_caching_info *ci); + void (*co_io_unlock)(struct ocfs2_caching_info *ci); +}; + +int __init init_ocfs2_uptodate_cache(void); +void exit_ocfs2_uptodate_cache(void); + +void ocfs2_metadata_cache_init(struct ocfs2_caching_info *ci, + const struct ocfs2_caching_operations *ops); +void ocfs2_metadata_cache_purge(struct ocfs2_caching_info *ci); +void ocfs2_metadata_cache_exit(struct ocfs2_caching_info *ci); + +u64 ocfs2_metadata_cache_owner(struct ocfs2_caching_info *ci); +void ocfs2_metadata_cache_io_lock(struct ocfs2_caching_info *ci); +void ocfs2_metadata_cache_io_unlock(struct ocfs2_caching_info *ci); + +int ocfs2_buffer_uptodate(struct ocfs2_caching_info *ci, + struct buffer_head *bh); +void ocfs2_set_buffer_uptodate(struct ocfs2_caching_info *ci, + struct buffer_head *bh); +void ocfs2_set_new_buffer_uptodate(struct ocfs2_caching_info *ci, + struct buffer_head *bh); +void ocfs2_remove_from_cache(struct ocfs2_caching_info *ci, + struct buffer_head *bh); +void ocfs2_remove_xattr_clusters_from_cache(struct ocfs2_caching_info *ci, + sector_t block, + u32 c_len); +int ocfs2_buffer_read_ahead(struct ocfs2_caching_info *ci, + struct buffer_head *bh); + +#endif /* OCFS2_UPTODATE_H */ diff --git a/fs/ocfs2/xattr.c b/fs/ocfs2/xattr.c new file mode 100644 index 000000000..35bcbb0ff --- /dev/null +++ b/fs/ocfs2/xattr.c @@ -0,0 +1,7650 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * xattr.c + * + * Copyright (C) 2004, 2008 Oracle. All rights reserved. + * + * CREDITS: + * Lots of code in this file is copy from linux/fs/ext3/xattr.c. + * Copyright (C) 2001-2003 Andreas Gruenbacher, + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ocfs2.h" +#include "alloc.h" +#include "blockcheck.h" +#include "dlmglue.h" +#include "file.h" +#include "symlink.h" +#include "sysfile.h" +#include "inode.h" +#include "journal.h" +#include "ocfs2_fs.h" +#include "suballoc.h" +#include "uptodate.h" +#include "buffer_head_io.h" +#include "super.h" +#include "xattr.h" +#include "refcounttree.h" +#include "acl.h" +#include "ocfs2_trace.h" + +struct ocfs2_xattr_def_value_root { + /* Must be last as it ends in a flexible-array member. */ + TRAILING_OVERLAP(struct ocfs2_xattr_value_root, xv, xr_list.l_recs, + struct ocfs2_extent_rec er; + ); +}; +static_assert(offsetof(struct ocfs2_xattr_def_value_root, xv.xr_list.l_recs) == + offsetof(struct ocfs2_xattr_def_value_root, er)); + +struct ocfs2_xattr_bucket { + /* The inode these xattrs are associated with */ + struct inode *bu_inode; + + /* The actual buffers that make up the bucket */ + struct buffer_head *bu_bhs[OCFS2_XATTR_MAX_BLOCKS_PER_BUCKET]; + + /* How many blocks make up one bucket for this filesystem */ + int bu_blocks; +}; + +struct ocfs2_xattr_set_ctxt { + handle_t *handle; + struct ocfs2_alloc_context *meta_ac; + struct ocfs2_alloc_context *data_ac; + struct ocfs2_cached_dealloc_ctxt dealloc; + int set_abort; +}; + +#define OCFS2_XATTR_ROOT_SIZE (sizeof(struct ocfs2_xattr_def_value_root)) +#define OCFS2_XATTR_INLINE_SIZE 80 +#define OCFS2_XATTR_HEADER_GAP 4 +#define OCFS2_XATTR_FREE_IN_IBODY (OCFS2_MIN_XATTR_INLINE_SIZE \ + - sizeof(struct ocfs2_xattr_header) \ + - OCFS2_XATTR_HEADER_GAP) +#define OCFS2_XATTR_FREE_IN_BLOCK(ptr) ((ptr)->i_sb->s_blocksize \ + - sizeof(struct ocfs2_xattr_block) \ + - sizeof(struct ocfs2_xattr_header) \ + - OCFS2_XATTR_HEADER_GAP) + +static struct ocfs2_xattr_def_value_root def_xv = { + .xv.xr_list.l_count = cpu_to_le16(1), +}; + +const struct xattr_handler * const ocfs2_xattr_handlers[] = { + &ocfs2_xattr_user_handler, + &ocfs2_xattr_trusted_handler, + &ocfs2_xattr_security_handler, + NULL +}; + +static const struct xattr_handler * const ocfs2_xattr_handler_map[OCFS2_XATTR_MAX] = { + [OCFS2_XATTR_INDEX_USER] = &ocfs2_xattr_user_handler, + [OCFS2_XATTR_INDEX_POSIX_ACL_ACCESS] = &nop_posix_acl_access, + [OCFS2_XATTR_INDEX_POSIX_ACL_DEFAULT] = &nop_posix_acl_default, + [OCFS2_XATTR_INDEX_TRUSTED] = &ocfs2_xattr_trusted_handler, + [OCFS2_XATTR_INDEX_SECURITY] = &ocfs2_xattr_security_handler, +}; + +struct ocfs2_xattr_info { + int xi_name_index; + const char *xi_name; + int xi_name_len; + const void *xi_value; + size_t xi_value_len; +}; + +struct ocfs2_xattr_search { + struct buffer_head *inode_bh; + /* + * xattr_bh point to the block buffer head which has extended attribute + * when extended attribute in inode, xattr_bh is equal to inode_bh. + */ + struct buffer_head *xattr_bh; + struct ocfs2_xattr_header *header; + struct ocfs2_xattr_bucket *bucket; + void *base; + void *end; + struct ocfs2_xattr_entry *here; + int not_found; +}; + +/* Operations on struct ocfs2_xa_entry */ +struct ocfs2_xa_loc; +struct ocfs2_xa_loc_operations { + /* + * Journal functions + */ + int (*xlo_journal_access)(handle_t *handle, struct ocfs2_xa_loc *loc, + int type); + void (*xlo_journal_dirty)(handle_t *handle, struct ocfs2_xa_loc *loc); + + /* + * Return a pointer to the appropriate buffer in loc->xl_storage + * at the given offset from loc->xl_header. + */ + void *(*xlo_offset_pointer)(struct ocfs2_xa_loc *loc, int offset); + + /* Can we reuse the existing entry for the new value? */ + int (*xlo_can_reuse)(struct ocfs2_xa_loc *loc, + struct ocfs2_xattr_info *xi); + + /* How much space is needed for the new value? */ + int (*xlo_check_space)(struct ocfs2_xa_loc *loc, + struct ocfs2_xattr_info *xi); + + /* + * Return the offset of the first name+value pair. This is + * the start of our downward-filling free space. + */ + int (*xlo_get_free_start)(struct ocfs2_xa_loc *loc); + + /* + * Remove the name+value at this location. Do whatever is + * appropriate with the remaining name+value pairs. + */ + void (*xlo_wipe_namevalue)(struct ocfs2_xa_loc *loc); + + /* Fill xl_entry with a new entry */ + void (*xlo_add_entry)(struct ocfs2_xa_loc *loc, u32 name_hash); + + /* Add name+value storage to an entry */ + void (*xlo_add_namevalue)(struct ocfs2_xa_loc *loc, int size); + + /* + * Initialize the value buf's access and bh fields for this entry. + * ocfs2_xa_fill_value_buf() will handle the xv pointer. + */ + void (*xlo_fill_value_buf)(struct ocfs2_xa_loc *loc, + struct ocfs2_xattr_value_buf *vb); +}; + +/* + * Describes an xattr entry location. This is a memory structure + * tracking the on-disk structure. + */ +struct ocfs2_xa_loc { + /* This xattr belongs to this inode */ + struct inode *xl_inode; + + /* The ocfs2_xattr_header inside the on-disk storage. Not NULL. */ + struct ocfs2_xattr_header *xl_header; + + /* Bytes from xl_header to the end of the storage */ + int xl_size; + + /* + * The ocfs2_xattr_entry this location describes. If this is + * NULL, this location describes the on-disk structure where it + * would have been. + */ + struct ocfs2_xattr_entry *xl_entry; + + /* + * Internal housekeeping + */ + + /* Buffer(s) containing this entry */ + void *xl_storage; + + /* Operations on the storage backing this location */ + const struct ocfs2_xa_loc_operations *xl_ops; +}; + +/* + * Convenience functions to calculate how much space is needed for a + * given name+value pair + */ +static int namevalue_size(int name_len, uint64_t value_len) +{ + if (value_len > OCFS2_XATTR_INLINE_SIZE) + return OCFS2_XATTR_SIZE(name_len) + OCFS2_XATTR_ROOT_SIZE; + else + return OCFS2_XATTR_SIZE(name_len) + OCFS2_XATTR_SIZE(value_len); +} + +static int namevalue_size_xi(struct ocfs2_xattr_info *xi) +{ + return namevalue_size(xi->xi_name_len, xi->xi_value_len); +} + +static int namevalue_size_xe(struct ocfs2_xattr_entry *xe) +{ + u64 value_len = le64_to_cpu(xe->xe_value_size); + + BUG_ON((value_len > OCFS2_XATTR_INLINE_SIZE) && + ocfs2_xattr_is_local(xe)); + return namevalue_size(xe->xe_name_len, value_len); +} + + +static int ocfs2_xattr_bucket_get_name_value(struct super_block *sb, + struct ocfs2_xattr_header *xh, + int index, + int *block_off, + int *new_offset); + +static int ocfs2_xattr_block_find(struct inode *inode, + int name_index, + const char *name, + struct ocfs2_xattr_search *xs); +static int ocfs2_xattr_index_block_find(struct inode *inode, + struct buffer_head *root_bh, + int name_index, + const char *name, + struct ocfs2_xattr_search *xs); + +static int ocfs2_xattr_tree_list_index_block(struct inode *inode, + struct buffer_head *blk_bh, + char *buffer, + size_t buffer_size); + +static int ocfs2_xattr_create_index_block(struct inode *inode, + struct ocfs2_xattr_search *xs, + struct ocfs2_xattr_set_ctxt *ctxt); + +static int ocfs2_xattr_set_entry_index_block(struct inode *inode, + struct ocfs2_xattr_info *xi, + struct ocfs2_xattr_search *xs, + struct ocfs2_xattr_set_ctxt *ctxt); + +typedef int (xattr_tree_rec_func)(struct inode *inode, + struct buffer_head *root_bh, + u64 blkno, u32 cpos, u32 len, void *para); +static int ocfs2_iterate_xattr_index_block(struct inode *inode, + struct buffer_head *root_bh, + xattr_tree_rec_func *rec_func, + void *para); +static int ocfs2_delete_xattr_in_bucket(struct inode *inode, + struct ocfs2_xattr_bucket *bucket, + void *para); +static int ocfs2_rm_xattr_cluster(struct inode *inode, + struct buffer_head *root_bh, + u64 blkno, + u32 cpos, + u32 len, + void *para); + +static int ocfs2_mv_xattr_buckets(struct inode *inode, handle_t *handle, + u64 src_blk, u64 last_blk, u64 to_blk, + unsigned int start_bucket, + u32 *first_hash); +static int ocfs2_prepare_refcount_xattr(struct inode *inode, + struct ocfs2_dinode *di, + struct ocfs2_xattr_info *xi, + struct ocfs2_xattr_search *xis, + struct ocfs2_xattr_search *xbs, + struct ocfs2_refcount_tree **ref_tree, + int *meta_need, + int *credits); +static int ocfs2_get_xattr_tree_value_root(struct super_block *sb, + struct ocfs2_xattr_bucket *bucket, + int offset, + struct ocfs2_xattr_value_root **xv, + struct buffer_head **bh); + +static inline u16 ocfs2_xattr_buckets_per_cluster(struct ocfs2_super *osb) +{ + return (1 << osb->s_clustersize_bits) / OCFS2_XATTR_BUCKET_SIZE; +} + +static inline u16 ocfs2_blocks_per_xattr_bucket(struct super_block *sb) +{ + return OCFS2_XATTR_BUCKET_SIZE / (1 << sb->s_blocksize_bits); +} + +#define bucket_blkno(_b) ((_b)->bu_bhs[0]->b_blocknr) +#define bucket_block(_b, _n) ((_b)->bu_bhs[(_n)]->b_data) +#define bucket_xh(_b) ((struct ocfs2_xattr_header *)bucket_block((_b), 0)) + +static struct ocfs2_xattr_bucket *ocfs2_xattr_bucket_new(struct inode *inode) +{ + struct ocfs2_xattr_bucket *bucket; + int blks = ocfs2_blocks_per_xattr_bucket(inode->i_sb); + + BUG_ON(blks > OCFS2_XATTR_MAX_BLOCKS_PER_BUCKET); + + bucket = kzalloc_obj(struct ocfs2_xattr_bucket, GFP_NOFS); + if (bucket) { + bucket->bu_inode = inode; + bucket->bu_blocks = blks; + } + + return bucket; +} + +static void ocfs2_xattr_bucket_relse(struct ocfs2_xattr_bucket *bucket) +{ + int i; + + for (i = 0; i < bucket->bu_blocks; i++) { + brelse(bucket->bu_bhs[i]); + bucket->bu_bhs[i] = NULL; + } +} + +static void ocfs2_xattr_bucket_free(struct ocfs2_xattr_bucket *bucket) +{ + if (bucket) { + ocfs2_xattr_bucket_relse(bucket); + bucket->bu_inode = NULL; + kfree(bucket); + } +} + +/* + * A bucket that has never been written to disk doesn't need to be + * read. We just need the buffer_heads. Don't call this for + * buckets that are already on disk. ocfs2_read_xattr_bucket() initializes + * them fully. + */ +static int ocfs2_init_xattr_bucket(struct ocfs2_xattr_bucket *bucket, + u64 xb_blkno, int new) +{ + int i, rc = 0; + + for (i = 0; i < bucket->bu_blocks; i++) { + bucket->bu_bhs[i] = sb_getblk(bucket->bu_inode->i_sb, + xb_blkno + i); + if (!bucket->bu_bhs[i]) { + rc = -ENOMEM; + mlog_errno(rc); + break; + } + + if (!ocfs2_buffer_uptodate(INODE_CACHE(bucket->bu_inode), + bucket->bu_bhs[i])) { + if (new) + ocfs2_set_new_buffer_uptodate(INODE_CACHE(bucket->bu_inode), + bucket->bu_bhs[i]); + else { + set_buffer_uptodate(bucket->bu_bhs[i]); + ocfs2_set_buffer_uptodate(INODE_CACHE(bucket->bu_inode), + bucket->bu_bhs[i]); + } + } + } + + if (rc) + ocfs2_xattr_bucket_relse(bucket); + return rc; +} + +static int ocfs2_validate_xattr_entries_flat(struct super_block *sb, u64 blkno, + struct ocfs2_xattr_header *xh, + size_t region_size); +static int ocfs2_validate_xattr_bucket(struct ocfs2_xattr_bucket *bucket, + u64 blkno); + +/* Read the xattr bucket at xb_blkno */ +static int ocfs2_read_xattr_bucket(struct ocfs2_xattr_bucket *bucket, + u64 xb_blkno) +{ + int rc; + + rc = ocfs2_read_blocks(INODE_CACHE(bucket->bu_inode), xb_blkno, + bucket->bu_blocks, bucket->bu_bhs, 0, + NULL); + if (!rc) { + spin_lock(&OCFS2_SB(bucket->bu_inode->i_sb)->osb_xattr_lock); + rc = ocfs2_validate_meta_ecc_bhs(bucket->bu_inode->i_sb, + bucket->bu_bhs, + bucket->bu_blocks, + &bucket_xh(bucket)->xh_check); + spin_unlock(&OCFS2_SB(bucket->bu_inode->i_sb)->osb_xattr_lock); + if (rc) + mlog_errno(rc); + else + rc = ocfs2_validate_xattr_bucket(bucket, xb_blkno); + } + + if (rc) + ocfs2_xattr_bucket_relse(bucket); + return rc; +} + +static int ocfs2_xattr_bucket_journal_access(handle_t *handle, + struct ocfs2_xattr_bucket *bucket, + int type) +{ + int i, rc = 0; + + for (i = 0; i < bucket->bu_blocks; i++) { + rc = ocfs2_journal_access(handle, + INODE_CACHE(bucket->bu_inode), + bucket->bu_bhs[i], type); + if (rc) { + mlog_errno(rc); + break; + } + } + + return rc; +} + +static void ocfs2_xattr_bucket_journal_dirty(handle_t *handle, + struct ocfs2_xattr_bucket *bucket) +{ + int i; + + spin_lock(&OCFS2_SB(bucket->bu_inode->i_sb)->osb_xattr_lock); + ocfs2_compute_meta_ecc_bhs(bucket->bu_inode->i_sb, + bucket->bu_bhs, bucket->bu_blocks, + &bucket_xh(bucket)->xh_check); + spin_unlock(&OCFS2_SB(bucket->bu_inode->i_sb)->osb_xattr_lock); + + for (i = 0; i < bucket->bu_blocks; i++) + ocfs2_journal_dirty(handle, bucket->bu_bhs[i]); +} + +static void ocfs2_xattr_bucket_copy_data(struct ocfs2_xattr_bucket *dest, + struct ocfs2_xattr_bucket *src) +{ + int i; + int blocksize = src->bu_inode->i_sb->s_blocksize; + + BUG_ON(dest->bu_blocks != src->bu_blocks); + BUG_ON(dest->bu_inode != src->bu_inode); + + for (i = 0; i < src->bu_blocks; i++) { + memcpy(bucket_block(dest, i), bucket_block(src, i), + blocksize); + } +} + +static int ocfs2_validate_xattr_block(struct super_block *sb, + struct buffer_head *bh) +{ + int rc; + struct ocfs2_xattr_block *xb = + (struct ocfs2_xattr_block *)bh->b_data; + + trace_ocfs2_validate_xattr_block((unsigned long long)bh->b_blocknr); + + BUG_ON(!buffer_uptodate(bh)); + + /* + * If the ecc fails, we return the error but otherwise + * leave the filesystem running. We know any error is + * local to this block. + */ + rc = ocfs2_validate_meta_ecc(sb, bh->b_data, &xb->xb_check); + if (rc) + return rc; + + /* + * Errors after here are fatal + */ + + if (!OCFS2_IS_VALID_XATTR_BLOCK(xb)) { + return ocfs2_error(sb, + "Extended attribute block #%llu has bad signature %.*s\n", + (unsigned long long)bh->b_blocknr, 7, + xb->xb_signature); + } + + if (le64_to_cpu(xb->xb_blkno) != bh->b_blocknr) { + return ocfs2_error(sb, + "Extended attribute block #%llu has an invalid xb_blkno of %llu\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(xb->xb_blkno)); + } + + if (le32_to_cpu(xb->xb_fs_generation) != OCFS2_SB(sb)->fs_generation) { + return ocfs2_error(sb, + "Extended attribute block #%llu has an invalid xb_fs_generation of #%u\n", + (unsigned long long)bh->b_blocknr, + le32_to_cpu(xb->xb_fs_generation)); + } + + if (!(le16_to_cpu(xb->xb_flags) & OCFS2_XATTR_INDEXED)) { + size_t region_offset = + offsetof(struct ocfs2_xattr_block, xb_attrs.xb_header); + + if (bh->b_size < region_offset) + return ocfs2_error(sb, + "Invalid xattr block %llu: block size %zu is too small\n", + (unsigned long long)bh->b_blocknr, + bh->b_size); + + return ocfs2_validate_xattr_entries_flat(sb, bh->b_blocknr, + &xb->xb_attrs.xb_header, + bh->b_size - + region_offset); + } + + return 0; +} + +static int ocfs2_read_xattr_block(struct inode *inode, u64 xb_blkno, + struct buffer_head **bh) +{ + int rc; + struct buffer_head *tmp = *bh; + + rc = ocfs2_read_block(INODE_CACHE(inode), xb_blkno, &tmp, + ocfs2_validate_xattr_block); + + /* If ocfs2_read_block() got us a new bh, pass it up. */ + if (!rc && !*bh) + *bh = tmp; + + return rc; +} + +static inline const char *ocfs2_xattr_prefix(int name_index) +{ + const struct xattr_handler *handler = NULL; + + if (name_index > 0 && name_index < OCFS2_XATTR_MAX) + handler = ocfs2_xattr_handler_map[name_index]; + return handler ? xattr_prefix(handler) : NULL; +} + +static u32 ocfs2_xattr_name_hash(struct inode *inode, + const char *name, + int name_len) +{ + /* Get hash value of uuid from super block */ + u32 hash = OCFS2_SB(inode->i_sb)->uuid_hash; + int i; + + /* hash extended attribute name */ + for (i = 0; i < name_len; i++) { + hash = (hash << OCFS2_HASH_SHIFT) ^ + (hash >> (8*sizeof(hash) - OCFS2_HASH_SHIFT)) ^ + *name++; + } + + return hash; +} + +static int ocfs2_xattr_entry_real_size(int name_len, size_t value_len) +{ + return namevalue_size(name_len, value_len) + + sizeof(struct ocfs2_xattr_entry); +} + +static int ocfs2_xi_entry_usage(struct ocfs2_xattr_info *xi) +{ + return namevalue_size_xi(xi) + + sizeof(struct ocfs2_xattr_entry); +} + +static int ocfs2_xe_entry_usage(struct ocfs2_xattr_entry *xe) +{ + return namevalue_size_xe(xe) + + sizeof(struct ocfs2_xattr_entry); +} + +int ocfs2_calc_security_init(struct inode *dir, + struct ocfs2_security_xattr_info *si, + int *want_clusters, + int *xattr_credits, + struct ocfs2_alloc_context **xattr_ac) +{ + int ret = 0; + struct ocfs2_super *osb = OCFS2_SB(dir->i_sb); + int s_size = ocfs2_xattr_entry_real_size(strlen(si->name), + si->value_len); + + /* + * The max space of security xattr taken inline is + * 256(name) + 80(value) + 16(entry) = 352 bytes, + * So reserve one metadata block for it is ok. + */ + if (dir->i_sb->s_blocksize == OCFS2_MIN_BLOCKSIZE || + s_size > OCFS2_XATTR_FREE_IN_IBODY) { + ret = ocfs2_reserve_new_metadata_blocks(osb, 1, xattr_ac); + if (ret) { + mlog_errno(ret); + return ret; + } + *xattr_credits += OCFS2_XATTR_BLOCK_CREATE_CREDITS; + } + + /* reserve clusters for xattr value which will be set in B tree*/ + if (si->value_len > OCFS2_XATTR_INLINE_SIZE) { + int new_clusters = ocfs2_clusters_for_bytes(dir->i_sb, + si->value_len); + + *xattr_credits += ocfs2_clusters_to_blocks(dir->i_sb, + new_clusters); + *want_clusters += new_clusters; + } + return ret; +} + +int ocfs2_calc_xattr_init(struct inode *dir, umode_t mode, + struct ocfs2_security_xattr_info *si, + int *want_clusters, int *xattr_credits, + int *want_meta, struct ocfs2_acl_state *acl_state) +{ + int ret = 0; + struct ocfs2_super *osb = OCFS2_SB(dir->i_sb); + int s_size = 0, a_size = 0, acl_len = 0, new_clusters; + + if (si->enable) + s_size = ocfs2_xattr_entry_real_size(strlen(si->name), + si->value_len); + + if (osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) { + if (acl_state->default_acl && S_ISDIR(mode)) { + acl_len = acl_state->default_acl->a_count * + sizeof(struct ocfs2_acl_entry); + a_size += ocfs2_xattr_entry_real_size(0, acl_len); + } + if (acl_state->acl) { + acl_len = acl_state->acl->a_count * + sizeof(struct ocfs2_acl_entry); + a_size += ocfs2_xattr_entry_real_size(0, acl_len); + } + } + + if (!(s_size + a_size)) + return ret; + + /* + * The max space of security xattr taken inline is + * 256(name) + 80(value) + 16(entry) = 352 bytes, + * The max space of acl xattr taken inline is + * 80(value) + 16(entry) * 2(if directory) = 192 bytes, + * when blocksize = 512, may reserve one more cluster for + * xattr bucket, otherwise reserve one metadata block + * for them is ok. + * If this is a new directory with inline data, + * we choose to reserve the entire inline area for + * directory contents and force an external xattr block. + */ + if (dir->i_sb->s_blocksize == OCFS2_MIN_BLOCKSIZE || + (S_ISDIR(mode) && ocfs2_supports_inline_data(osb)) || + (s_size + a_size) > OCFS2_XATTR_FREE_IN_IBODY) { + *want_meta = *want_meta + 1; + *xattr_credits += OCFS2_XATTR_BLOCK_CREATE_CREDITS; + } + + if (dir->i_sb->s_blocksize == OCFS2_MIN_BLOCKSIZE && + (s_size + a_size) > OCFS2_XATTR_FREE_IN_BLOCK(dir)) { + *want_clusters += 1; + *xattr_credits += ocfs2_blocks_per_xattr_bucket(dir->i_sb); + } + + /* + * reserve credits and clusters for xattrs which has large value + * and have to be set outside + */ + if (si->enable && si->value_len > OCFS2_XATTR_INLINE_SIZE) { + new_clusters = ocfs2_clusters_for_bytes(dir->i_sb, + si->value_len); + *xattr_credits += ocfs2_clusters_to_blocks(dir->i_sb, + new_clusters); + *want_clusters += new_clusters; + } + if (osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) { + if (acl_state->default_acl && S_ISDIR(mode)) { + acl_len = acl_state->default_acl->a_count * + sizeof(struct ocfs2_acl_entry); + if (acl_len > OCFS2_XATTR_INLINE_SIZE) { + new_clusters = + ocfs2_clusters_for_bytes(dir->i_sb, + acl_len); + *xattr_credits += + ocfs2_clusters_to_blocks(dir->i_sb, + new_clusters); + *want_clusters += new_clusters; + } + } + if (acl_state->acl) { + acl_len = acl_state->acl->a_count * + sizeof(struct ocfs2_acl_entry); + if (acl_len > OCFS2_XATTR_INLINE_SIZE) { + new_clusters = + ocfs2_clusters_for_bytes(dir->i_sb, + acl_len); + *xattr_credits += + ocfs2_clusters_to_blocks(dir->i_sb, + new_clusters); + *want_clusters += new_clusters; + } + } + } + + return ret; +} + +static int ocfs2_xattr_extend_allocation(struct inode *inode, + u32 clusters_to_add, + struct ocfs2_xattr_value_buf *vb, + struct ocfs2_xattr_set_ctxt *ctxt) +{ + int status = 0, credits; + handle_t *handle = ctxt->handle; + enum ocfs2_alloc_restarted why; + u32 prev_clusters, logical_start = le32_to_cpu(vb->vb_xv->xr_clusters); + struct ocfs2_extent_tree et; + + ocfs2_init_xattr_value_extent_tree(&et, INODE_CACHE(inode), vb); + + while (clusters_to_add) { + trace_ocfs2_xattr_extend_allocation(clusters_to_add); + + status = vb->vb_access(handle, INODE_CACHE(inode), vb->vb_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (status < 0) { + mlog_errno(status); + break; + } + + prev_clusters = le32_to_cpu(vb->vb_xv->xr_clusters); + status = ocfs2_add_clusters_in_btree(handle, + &et, + &logical_start, + clusters_to_add, + 0, + ctxt->data_ac, + ctxt->meta_ac, + &why); + if ((status < 0) && (status != -EAGAIN)) { + if (status != -ENOSPC) + mlog_errno(status); + break; + } + + ocfs2_journal_dirty(handle, vb->vb_bh); + + clusters_to_add -= le32_to_cpu(vb->vb_xv->xr_clusters) - + prev_clusters; + + if (why != RESTART_NONE && clusters_to_add) { + if (why == RESTART_META) { + status = -ENOSPC; + break; + } + credits = ocfs2_calc_extend_credits(inode->i_sb, + &vb->vb_xv->xr_list); + status = ocfs2_extend_trans(handle, credits); + if (status < 0) { + status = -ENOMEM; + mlog_errno(status); + break; + } + } + } + + return status; +} + +static int __ocfs2_remove_xattr_range(struct inode *inode, + struct ocfs2_xattr_value_buf *vb, + u32 cpos, u32 phys_cpos, u32 len, + unsigned int ext_flags, + struct ocfs2_xattr_set_ctxt *ctxt) +{ + int ret; + u64 phys_blkno = ocfs2_clusters_to_blocks(inode->i_sb, phys_cpos); + handle_t *handle = ctxt->handle; + struct ocfs2_extent_tree et; + + ocfs2_init_xattr_value_extent_tree(&et, INODE_CACHE(inode), vb); + + ret = vb->vb_access(handle, INODE_CACHE(inode), vb->vb_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_remove_extent(handle, &et, cpos, len, ctxt->meta_ac, + &ctxt->dealloc); + if (ret) { + mlog_errno(ret); + goto out; + } + + le32_add_cpu(&vb->vb_xv->xr_clusters, -len); + ocfs2_journal_dirty(handle, vb->vb_bh); + + if (ext_flags & OCFS2_EXT_REFCOUNTED) + ret = ocfs2_decrease_refcount(inode, handle, + ocfs2_blocks_to_clusters(inode->i_sb, + phys_blkno), + len, ctxt->meta_ac, &ctxt->dealloc, 1); + else + ret = ocfs2_cache_cluster_dealloc(&ctxt->dealloc, + phys_blkno, len); + if (ret) + mlog_errno(ret); + +out: + return ret; +} + +static int ocfs2_xattr_shrink_size(struct inode *inode, + u32 old_clusters, + u32 new_clusters, + struct ocfs2_xattr_value_buf *vb, + struct ocfs2_xattr_set_ctxt *ctxt) +{ + int ret = 0; + unsigned int ext_flags; + u32 trunc_len, cpos, phys_cpos, alloc_size; + u64 block; + + if (old_clusters <= new_clusters) + return 0; + + cpos = new_clusters; + trunc_len = old_clusters - new_clusters; + while (trunc_len) { + ret = ocfs2_xattr_get_clusters(inode, cpos, &phys_cpos, + &alloc_size, + &vb->vb_xv->xr_list, &ext_flags); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (alloc_size > trunc_len) + alloc_size = trunc_len; + + ret = __ocfs2_remove_xattr_range(inode, vb, cpos, + phys_cpos, alloc_size, + ext_flags, ctxt); + if (ret) { + mlog_errno(ret); + goto out; + } + + block = ocfs2_clusters_to_blocks(inode->i_sb, phys_cpos); + ocfs2_remove_xattr_clusters_from_cache(INODE_CACHE(inode), + block, alloc_size); + cpos += alloc_size; + trunc_len -= alloc_size; + } + +out: + return ret; +} + +static int ocfs2_xattr_value_truncate(struct inode *inode, + struct ocfs2_xattr_value_buf *vb, + int len, + struct ocfs2_xattr_set_ctxt *ctxt) +{ + int ret; + u32 new_clusters = ocfs2_clusters_for_bytes(inode->i_sb, len); + u32 old_clusters = le32_to_cpu(vb->vb_xv->xr_clusters); + + if (new_clusters == old_clusters) + return 0; + + if (new_clusters > old_clusters) + ret = ocfs2_xattr_extend_allocation(inode, + new_clusters - old_clusters, + vb, ctxt); + else + ret = ocfs2_xattr_shrink_size(inode, + old_clusters, new_clusters, + vb, ctxt); + + return ret; +} + +static int ocfs2_xattr_list_entry(struct super_block *sb, + char *buffer, size_t size, + size_t *result, int type, + const char *name, int name_len) +{ + char *p = buffer + *result; + const char *prefix; + int prefix_len; + int total_len; + + switch(type) { + case OCFS2_XATTR_INDEX_USER: + if (OCFS2_SB(sb)->s_mount_opt & OCFS2_MOUNT_NOUSERXATTR) + return 0; + break; + + case OCFS2_XATTR_INDEX_POSIX_ACL_ACCESS: + case OCFS2_XATTR_INDEX_POSIX_ACL_DEFAULT: + if (!(sb->s_flags & SB_POSIXACL)) + return 0; + break; + + case OCFS2_XATTR_INDEX_TRUSTED: + if (!capable(CAP_SYS_ADMIN)) + return 0; + break; + } + + prefix = ocfs2_xattr_prefix(type); + if (!prefix) + return 0; + prefix_len = strlen(prefix); + total_len = prefix_len + name_len + 1; + *result += total_len; + + /* No buffer means we are only looking for the required size. */ + if (!buffer) + return 0; + + if (*result > size) + return -ERANGE; + + memcpy(p, prefix, prefix_len); + memcpy(p + prefix_len, name, name_len); + p[prefix_len + name_len] = '\0'; + + return 0; +} + +static int ocfs2_xattr_list_entries(struct inode *inode, + struct ocfs2_xattr_header *header, + char *buffer, size_t buffer_size) +{ + size_t result = 0; + int i, type, ret; + const char *name; + + for (i = 0 ; i < le16_to_cpu(header->xh_count); i++) { + struct ocfs2_xattr_entry *entry = &header->xh_entries[i]; + type = ocfs2_xattr_get_type(entry); + name = (const char *)header + + le16_to_cpu(entry->xe_name_offset); + + ret = ocfs2_xattr_list_entry(inode->i_sb, + buffer, buffer_size, + &result, type, name, + entry->xe_name_len); + if (ret) + return ret; + } + + return result; +} + +static int ocfs2_validate_xattr_entries_flat(struct super_block *sb, u64 blkno, + struct ocfs2_xattr_header *xh, + size_t region_size) +{ + u16 xattr_count = le16_to_cpu(xh->xh_count); + size_t entries_limit = region_size; + size_t nv_limit = region_size; + size_t max_entries; + int i; + + if (region_size < sizeof(*xh)) + return ocfs2_error(sb, + "Invalid xattr in block %llu: region size %zu is too small\n", + (unsigned long long)blkno, region_size); + + max_entries = (entries_limit - sizeof(*xh)) / + sizeof(struct ocfs2_xattr_entry); + + if (xattr_count > max_entries) + return ocfs2_error(sb, + "Invalid xattr in block %llu: entry count %u exceeds maximum %zu\n", + (unsigned long long)blkno, + xattr_count, max_entries); + + for (i = 0; i < xattr_count; i++) { + struct ocfs2_xattr_entry *xe = &xh->xh_entries[i]; + size_t name_offset = le16_to_cpu(xe->xe_name_offset); + size_t value_offset; + + if (name_offset > nv_limit || + xe->xe_name_len > nv_limit - name_offset) + return ocfs2_error(sb, + "Invalid xattr in block %llu: entry %d name is out of bounds\n", + (unsigned long long)blkno, i); + + value_offset = name_offset + OCFS2_XATTR_SIZE(xe->xe_name_len); + if (value_offset > nv_limit) + return ocfs2_error(sb, + "Invalid xattr in block %llu: entry %d value starts out of bounds\n", + (unsigned long long)blkno, i); + + if (ocfs2_xattr_is_local(xe)) { + if (le64_to_cpu(xe->xe_value_size) > + nv_limit - value_offset) + return ocfs2_error(sb, + "Invalid xattr in block %llu: entry %d value is out of bounds\n", + (unsigned long long)blkno, + i); + } else if (sizeof(struct ocfs2_xattr_value_root) > + nv_limit - value_offset) { + return ocfs2_error(sb, + "Invalid xattr in block %llu: entry %d value root is out of bounds\n", + (unsigned long long)blkno, i); + } + } + + return 0; +} + +static int ocfs2_xattr_ibody_lookup_header_raw(struct super_block *sb, + u64 blkno, + struct ocfs2_dinode *di, + struct ocfs2_xattr_header **header, + u16 *inline_size_ret) +{ + struct ocfs2_xattr_header *xh; + u16 xattr_count; + size_t max_entries; + u16 inline_size = le16_to_cpu(di->i_xattr_inline_size); + + if (inline_size > sb->s_blocksize || + inline_size < sizeof(struct ocfs2_xattr_header)) { + ocfs2_error(sb, + "Invalid inode %llu: xattr inline size %u\n", + (unsigned long long)blkno, inline_size); + return -EFSCORRUPTED; + } + + xh = (struct ocfs2_xattr_header *) + ((void *)di + sb->s_blocksize - inline_size); + + xattr_count = le16_to_cpu(xh->xh_count); + max_entries = (inline_size - sizeof(struct ocfs2_xattr_header)) / + sizeof(struct ocfs2_xattr_entry); + + if (xattr_count > max_entries) { + ocfs2_error(sb, + "xattr entry count %u exceeds maximum %zu in inode %llu\n", + xattr_count, max_entries, + (unsigned long long)blkno); + return -EFSCORRUPTED; + } + + *header = xh; + if (inline_size_ret) + *inline_size_ret = inline_size; + + return 0; +} + +int ocfs2_validate_inode_xattr(struct super_block *sb, u64 blkno, + struct ocfs2_dinode *di) +{ + struct ocfs2_xattr_header *xh; + u16 inline_size; + int ret; + + if (!(le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_XATTR_FL)) + return 0; + + ret = ocfs2_xattr_ibody_lookup_header_raw(sb, blkno, di, &xh, + &inline_size); + if (ret) + return ret; + + return ocfs2_validate_xattr_entries_flat(sb, blkno, xh, inline_size); +} + +static int ocfs2_validate_xattr_bucket(struct ocfs2_xattr_bucket *bucket, + u64 blkno) +{ + struct super_block *sb = bucket->bu_inode->i_sb; + struct ocfs2_xattr_header *xh = bucket_xh(bucket); + u16 xattr_count = le16_to_cpu(xh->xh_count); + size_t region_size = (size_t)sb->s_blocksize * bucket->bu_blocks; + size_t entries_limit = sb->s_blocksize; + size_t nv_limit = sb->s_blocksize; + size_t max_entries; + int i; + + if (region_size < sizeof(*xh)) + return ocfs2_error(sb, + "Invalid xattr bucket %llu: region size %zu is too small\n", + (unsigned long long)blkno, region_size); + + if (entries_limit < sizeof(*xh)) + return ocfs2_error(sb, + "Invalid xattr bucket %llu: entries limit %zu is too small\n", + (unsigned long long)blkno, + entries_limit); + + max_entries = (entries_limit - sizeof(*xh)) / + sizeof(struct ocfs2_xattr_entry); + + if (xattr_count > max_entries) + return ocfs2_error(sb, + "Invalid xattr bucket %llu: entry count %u exceeds maximum %zu\n", + (unsigned long long)blkno, + xattr_count, max_entries); + + for (i = 0; i < xattr_count; i++) { + struct ocfs2_xattr_entry *xe = &xh->xh_entries[i]; + size_t name_offset = le16_to_cpu(xe->xe_name_offset); + size_t block_off = name_offset >> sb->s_blocksize_bits; + size_t block_offset = name_offset % nv_limit; + size_t value_offset; + + if (name_offset >= region_size || block_off >= bucket->bu_blocks) + return ocfs2_error(sb, + "Invalid xattr bucket %llu: entry %d name is out of bounds\n", + (unsigned long long)blkno, i); + + if (xe->xe_name_len > nv_limit - block_offset) + return ocfs2_error(sb, + "Invalid xattr bucket %llu: entry %d name crosses block boundary\n", + (unsigned long long)blkno, i); + + value_offset = block_offset + OCFS2_XATTR_SIZE(xe->xe_name_len); + if (value_offset > nv_limit) + return ocfs2_error(sb, + "Invalid xattr bucket %llu: entry %d value starts out of bounds\n", + (unsigned long long)blkno, i); + + if (ocfs2_xattr_is_local(xe)) { + if (le64_to_cpu(xe->xe_value_size) > + nv_limit - value_offset) + return ocfs2_error(sb, + "Invalid xattr bucket %llu: entry %d value is out of bounds\n", + (unsigned long long)blkno, + i); + } else if (sizeof(struct ocfs2_xattr_value_root) > + nv_limit - value_offset) { + return ocfs2_error(sb, + "Invalid xattr bucket %llu: entry %d value root is out of bounds\n", + (unsigned long long)blkno, i); + } + } + + return 0; +} + +static int ocfs2_xattr_ibody_lookup_header(struct inode *inode, + struct ocfs2_dinode *di, + struct ocfs2_xattr_header **header) +{ + return ocfs2_xattr_ibody_lookup_header_raw(inode->i_sb, + OCFS2_I(inode)->ip_blkno, + di, header, NULL); +} + +int ocfs2_has_inline_xattr_value_outside(struct inode *inode, + struct ocfs2_dinode *di) +{ + struct ocfs2_xattr_header *xh; + int ret; + int i; + + ret = ocfs2_xattr_ibody_lookup_header(inode, di, &xh); + if (ret) + return 1; + + for (i = 0; i < le16_to_cpu(xh->xh_count); i++) + if (!ocfs2_xattr_is_local(&xh->xh_entries[i])) + return 1; + + return 0; +} + +static int ocfs2_xattr_ibody_list(struct inode *inode, + struct ocfs2_dinode *di, + char *buffer, + size_t buffer_size) +{ + struct ocfs2_xattr_header *header = NULL; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + int ret = 0; + + if (!(oi->ip_dyn_features & OCFS2_INLINE_XATTR_FL)) + return ret; + + ret = ocfs2_xattr_ibody_lookup_header(inode, di, &header); + if (ret) + return ret; + + ret = ocfs2_xattr_list_entries(inode, header, buffer, buffer_size); + + return ret; +} + +static int ocfs2_xattr_block_list(struct inode *inode, + struct ocfs2_dinode *di, + char *buffer, + size_t buffer_size) +{ + struct buffer_head *blk_bh = NULL; + struct ocfs2_xattr_block *xb; + int ret = 0; + + if (!di->i_xattr_loc) + return ret; + + ret = ocfs2_read_xattr_block(inode, le64_to_cpu(di->i_xattr_loc), + &blk_bh); + if (ret < 0) { + mlog_errno(ret); + return ret; + } + + xb = (struct ocfs2_xattr_block *)blk_bh->b_data; + if (!(le16_to_cpu(xb->xb_flags) & OCFS2_XATTR_INDEXED)) { + struct ocfs2_xattr_header *header = &xb->xb_attrs.xb_header; + ret = ocfs2_xattr_list_entries(inode, header, + buffer, buffer_size); + } else + ret = ocfs2_xattr_tree_list_index_block(inode, blk_bh, + buffer, buffer_size); + + brelse(blk_bh); + + return ret; +} + +ssize_t ocfs2_listxattr(struct dentry *dentry, + char *buffer, + size_t size) +{ + int ret = 0, i_ret = 0, b_ret = 0; + struct buffer_head *di_bh = NULL; + struct ocfs2_dinode *di = NULL; + struct ocfs2_inode_info *oi = OCFS2_I(d_inode(dentry)); + + if (!ocfs2_supports_xattr(OCFS2_SB(dentry->d_sb))) + return -EOPNOTSUPP; + + if (!(oi->ip_dyn_features & OCFS2_HAS_XATTR_FL)) + return ret; + + ret = ocfs2_inode_lock(d_inode(dentry), &di_bh, 0); + if (ret < 0) { + mlog_errno(ret); + return ret; + } + + di = (struct ocfs2_dinode *)di_bh->b_data; + + down_read(&oi->ip_xattr_sem); + i_ret = ocfs2_xattr_ibody_list(d_inode(dentry), di, buffer, size); + if (i_ret < 0) + b_ret = 0; + else { + if (buffer) { + buffer += i_ret; + size -= i_ret; + } + b_ret = ocfs2_xattr_block_list(d_inode(dentry), di, + buffer, size); + if (b_ret < 0) + i_ret = 0; + } + up_read(&oi->ip_xattr_sem); + ocfs2_inode_unlock(d_inode(dentry), 0); + + brelse(di_bh); + + return i_ret + b_ret; +} + +static int ocfs2_xattr_find_entry(struct inode *inode, int name_index, + const char *name, + struct ocfs2_xattr_search *xs) +{ + struct ocfs2_xattr_entry *entry; + size_t name_len; + int i, name_offset, cmp = 1; + + if (name == NULL) + return -EINVAL; + + name_len = strlen(name); + entry = xs->here; + for (i = 0; i < le16_to_cpu(xs->header->xh_count); i++) { + if ((void *)entry >= xs->end) { + ocfs2_error(inode->i_sb, "corrupted xattr entries"); + return -EFSCORRUPTED; + } + cmp = name_index - ocfs2_xattr_get_type(entry); + if (!cmp) + cmp = name_len - entry->xe_name_len; + if (!cmp) { + name_offset = le16_to_cpu(entry->xe_name_offset); + if ((xs->base + name_offset + name_len) > xs->end) { + ocfs2_error(inode->i_sb, + "corrupted xattr entries"); + return -EFSCORRUPTED; + } + cmp = memcmp(name, (xs->base + name_offset), name_len); + } + if (cmp == 0) + break; + entry += 1; + } + xs->here = entry; + + return cmp ? -ENODATA : 0; +} + +static int ocfs2_xattr_get_value_outside(struct inode *inode, + struct ocfs2_xattr_value_root *xv, + void *buffer, + size_t len) +{ + u32 cpos, p_cluster, num_clusters, bpc, clusters; + u64 blkno; + int i, ret = 0; + size_t cplen, blocksize; + struct buffer_head *bh = NULL; + struct ocfs2_extent_list *el; + + el = &xv->xr_list; + clusters = le32_to_cpu(xv->xr_clusters); + bpc = ocfs2_clusters_to_blocks(inode->i_sb, 1); + blocksize = inode->i_sb->s_blocksize; + + cpos = 0; + while (cpos < clusters) { + ret = ocfs2_xattr_get_clusters(inode, cpos, &p_cluster, + &num_clusters, el, NULL); + if (ret) { + mlog_errno(ret); + goto out; + } + + blkno = ocfs2_clusters_to_blocks(inode->i_sb, p_cluster); + /* Copy ocfs2_xattr_value */ + for (i = 0; i < num_clusters * bpc; i++, blkno++) { + ret = ocfs2_read_block(INODE_CACHE(inode), blkno, + &bh, NULL); + if (ret) { + mlog_errno(ret); + goto out; + } + + cplen = len >= blocksize ? blocksize : len; + memcpy(buffer, bh->b_data, cplen); + len -= cplen; + buffer += cplen; + + brelse(bh); + bh = NULL; + if (len == 0) + break; + } + cpos += num_clusters; + } +out: + return ret; +} + +static int ocfs2_xattr_ibody_get(struct inode *inode, + int name_index, + const char *name, + void *buffer, + size_t buffer_size, + struct ocfs2_xattr_search *xs) +{ + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_dinode *di = (struct ocfs2_dinode *)xs->inode_bh->b_data; + struct ocfs2_xattr_value_root *xv; + size_t size; + int ret = 0; + + if (!(oi->ip_dyn_features & OCFS2_INLINE_XATTR_FL)) + return -ENODATA; + + xs->end = (void *)di + inode->i_sb->s_blocksize; + ret = ocfs2_xattr_ibody_lookup_header(inode, di, &xs->header); + if (ret) + return ret; + xs->base = (void *)xs->header; + xs->here = xs->header->xh_entries; + + ret = ocfs2_xattr_find_entry(inode, name_index, name, xs); + if (ret) + return ret; + size = le64_to_cpu(xs->here->xe_value_size); + if (buffer) { + if (size > buffer_size) + return -ERANGE; + if (ocfs2_xattr_is_local(xs->here)) { + memcpy(buffer, (void *)xs->base + + le16_to_cpu(xs->here->xe_name_offset) + + OCFS2_XATTR_SIZE(xs->here->xe_name_len), size); + } else { + xv = (struct ocfs2_xattr_value_root *) + (xs->base + le16_to_cpu( + xs->here->xe_name_offset) + + OCFS2_XATTR_SIZE(xs->here->xe_name_len)); + ret = ocfs2_xattr_get_value_outside(inode, xv, + buffer, size); + if (ret < 0) { + mlog_errno(ret); + return ret; + } + } + } + + return size; +} + +static int ocfs2_xattr_block_get(struct inode *inode, + int name_index, + const char *name, + void *buffer, + size_t buffer_size, + struct ocfs2_xattr_search *xs) +{ + struct ocfs2_xattr_block *xb; + struct ocfs2_xattr_value_root *xv; + size_t size; + int ret = -ENODATA, name_offset, name_len, i; + int block_off; + + xs->bucket = ocfs2_xattr_bucket_new(inode); + if (!xs->bucket) { + ret = -ENOMEM; + mlog_errno(ret); + goto cleanup; + } + + ret = ocfs2_xattr_block_find(inode, name_index, name, xs); + if (ret) { + mlog_errno(ret); + goto cleanup; + } + + if (xs->not_found) { + ret = -ENODATA; + goto cleanup; + } + + xb = (struct ocfs2_xattr_block *)xs->xattr_bh->b_data; + size = le64_to_cpu(xs->here->xe_value_size); + if (buffer) { + ret = -ERANGE; + if (size > buffer_size) + goto cleanup; + + name_offset = le16_to_cpu(xs->here->xe_name_offset); + name_len = OCFS2_XATTR_SIZE(xs->here->xe_name_len); + i = xs->here - xs->header->xh_entries; + + if (le16_to_cpu(xb->xb_flags) & OCFS2_XATTR_INDEXED) { + ret = ocfs2_xattr_bucket_get_name_value(inode->i_sb, + bucket_xh(xs->bucket), + i, + &block_off, + &name_offset); + if (ret) { + mlog_errno(ret); + goto cleanup; + } + xs->base = bucket_block(xs->bucket, block_off); + } + if (ocfs2_xattr_is_local(xs->here)) { + memcpy(buffer, (void *)xs->base + + name_offset + name_len, size); + } else { + xv = (struct ocfs2_xattr_value_root *) + (xs->base + name_offset + name_len); + ret = ocfs2_xattr_get_value_outside(inode, xv, + buffer, size); + if (ret < 0) { + mlog_errno(ret); + goto cleanup; + } + } + } + ret = size; +cleanup: + ocfs2_xattr_bucket_free(xs->bucket); + + brelse(xs->xattr_bh); + xs->xattr_bh = NULL; + return ret; +} + +int ocfs2_xattr_get_nolock(struct inode *inode, + struct buffer_head *di_bh, + int name_index, + const char *name, + void *buffer, + size_t buffer_size) +{ + int ret; + struct ocfs2_dinode *di = NULL; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_xattr_search xis = { + .not_found = -ENODATA, + }; + struct ocfs2_xattr_search xbs = { + .not_found = -ENODATA, + }; + + if (!ocfs2_supports_xattr(OCFS2_SB(inode->i_sb))) + return -EOPNOTSUPP; + + if (!(oi->ip_dyn_features & OCFS2_HAS_XATTR_FL)) + return -ENODATA; + + xis.inode_bh = xbs.inode_bh = di_bh; + di = (struct ocfs2_dinode *)di_bh->b_data; + + ret = ocfs2_xattr_ibody_get(inode, name_index, name, buffer, + buffer_size, &xis); + if (ret == -ENODATA && di->i_xattr_loc) + ret = ocfs2_xattr_block_get(inode, name_index, name, buffer, + buffer_size, &xbs); + + return ret; +} + +/* ocfs2_xattr_get() + * + * Copy an extended attribute into the buffer provided. + * Buffer is NULL to compute the size of buffer required. + */ +static int ocfs2_xattr_get(struct inode *inode, + int name_index, + const char *name, + void *buffer, + size_t buffer_size) +{ + int ret, had_lock; + struct buffer_head *di_bh = NULL; + struct ocfs2_lock_holder oh; + + had_lock = ocfs2_inode_lock_tracker(inode, &di_bh, 0, &oh); + if (had_lock < 0) { + mlog_errno(had_lock); + return had_lock; + } + down_read(&OCFS2_I(inode)->ip_xattr_sem); + ret = ocfs2_xattr_get_nolock(inode, di_bh, name_index, + name, buffer, buffer_size); + up_read(&OCFS2_I(inode)->ip_xattr_sem); + + ocfs2_inode_unlock_tracker(inode, 0, &oh, had_lock); + + brelse(di_bh); + + return ret; +} + +static int __ocfs2_xattr_set_value_outside(struct inode *inode, + handle_t *handle, + struct ocfs2_xattr_value_buf *vb, + const void *value, + int value_len) +{ + int ret = 0, i, cp_len; + u16 blocksize = inode->i_sb->s_blocksize; + u32 p_cluster, num_clusters; + u32 cpos = 0, bpc = ocfs2_clusters_to_blocks(inode->i_sb, 1); + u32 clusters = ocfs2_clusters_for_bytes(inode->i_sb, value_len); + u64 blkno; + struct buffer_head *bh = NULL; + unsigned int ext_flags; + struct ocfs2_xattr_value_root *xv = vb->vb_xv; + + BUG_ON(clusters > le32_to_cpu(xv->xr_clusters)); + + while (cpos < clusters) { + ret = ocfs2_xattr_get_clusters(inode, cpos, &p_cluster, + &num_clusters, &xv->xr_list, + &ext_flags); + if (ret) { + mlog_errno(ret); + goto out; + } + + BUG_ON(ext_flags & OCFS2_EXT_REFCOUNTED); + + blkno = ocfs2_clusters_to_blocks(inode->i_sb, p_cluster); + + for (i = 0; i < num_clusters * bpc; i++, blkno++) { + ret = ocfs2_read_block(INODE_CACHE(inode), blkno, + &bh, NULL); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_journal_access(handle, + INODE_CACHE(inode), + bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + cp_len = value_len > blocksize ? blocksize : value_len; + memcpy(bh->b_data, value, cp_len); + value_len -= cp_len; + value += cp_len; + if (cp_len < blocksize) + memset(bh->b_data + cp_len, 0, + blocksize - cp_len); + + ocfs2_journal_dirty(handle, bh); + brelse(bh); + bh = NULL; + + /* + * XXX: do we need to empty all the following + * blocks in this cluster? + */ + if (!value_len) + break; + } + cpos += num_clusters; + } +out: + brelse(bh); + + return ret; +} + +static int ocfs2_xa_check_space_helper(int needed_space, int free_start, + int num_entries) +{ + int free_space; + + if (!needed_space) + return 0; + + free_space = free_start - + sizeof(struct ocfs2_xattr_header) - + (num_entries * sizeof(struct ocfs2_xattr_entry)) - + OCFS2_XATTR_HEADER_GAP; + if (free_space < 0) + return -EIO; + if (free_space < needed_space) + return -ENOSPC; + + return 0; +} + +static int ocfs2_xa_journal_access(handle_t *handle, struct ocfs2_xa_loc *loc, + int type) +{ + return loc->xl_ops->xlo_journal_access(handle, loc, type); +} + +static void ocfs2_xa_journal_dirty(handle_t *handle, struct ocfs2_xa_loc *loc) +{ + loc->xl_ops->xlo_journal_dirty(handle, loc); +} + +/* Give a pointer into the storage for the given offset */ +static void *ocfs2_xa_offset_pointer(struct ocfs2_xa_loc *loc, int offset) +{ + BUG_ON(offset >= loc->xl_size); + return loc->xl_ops->xlo_offset_pointer(loc, offset); +} + +/* + * Wipe the name+value pair and allow the storage to reclaim it. This + * must be followed by either removal of the entry or a call to + * ocfs2_xa_add_namevalue(). + */ +static void ocfs2_xa_wipe_namevalue(struct ocfs2_xa_loc *loc) +{ + loc->xl_ops->xlo_wipe_namevalue(loc); +} + +/* + * Find lowest offset to a name+value pair. This is the start of our + * downward-growing free space. + */ +static int ocfs2_xa_get_free_start(struct ocfs2_xa_loc *loc) +{ + return loc->xl_ops->xlo_get_free_start(loc); +} + +/* Can we reuse loc->xl_entry for xi? */ +static int ocfs2_xa_can_reuse_entry(struct ocfs2_xa_loc *loc, + struct ocfs2_xattr_info *xi) +{ + return loc->xl_ops->xlo_can_reuse(loc, xi); +} + +/* How much free space is needed to set the new value */ +static int ocfs2_xa_check_space(struct ocfs2_xa_loc *loc, + struct ocfs2_xattr_info *xi) +{ + return loc->xl_ops->xlo_check_space(loc, xi); +} + +static void ocfs2_xa_add_entry(struct ocfs2_xa_loc *loc, u32 name_hash) +{ + loc->xl_ops->xlo_add_entry(loc, name_hash); + loc->xl_entry->xe_name_hash = cpu_to_le32(name_hash); + /* + * We can't leave the new entry's xe_name_offset at zero or + * add_namevalue() will go nuts. We set it to the size of our + * storage so that it can never be less than any other entry. + */ + loc->xl_entry->xe_name_offset = cpu_to_le16(loc->xl_size); +} + +static void ocfs2_xa_add_namevalue(struct ocfs2_xa_loc *loc, + struct ocfs2_xattr_info *xi) +{ + int size = namevalue_size_xi(xi); + int nameval_offset; + char *nameval_buf; + + loc->xl_ops->xlo_add_namevalue(loc, size); + loc->xl_entry->xe_value_size = cpu_to_le64(xi->xi_value_len); + loc->xl_entry->xe_name_len = xi->xi_name_len; + ocfs2_xattr_set_type(loc->xl_entry, xi->xi_name_index); + ocfs2_xattr_set_local(loc->xl_entry, + xi->xi_value_len <= OCFS2_XATTR_INLINE_SIZE); + + nameval_offset = le16_to_cpu(loc->xl_entry->xe_name_offset); + nameval_buf = ocfs2_xa_offset_pointer(loc, nameval_offset); + memset(nameval_buf, 0, size); + memcpy(nameval_buf, xi->xi_name, xi->xi_name_len); +} + +static void ocfs2_xa_fill_value_buf(struct ocfs2_xa_loc *loc, + struct ocfs2_xattr_value_buf *vb) +{ + int nameval_offset = le16_to_cpu(loc->xl_entry->xe_name_offset); + int name_size = OCFS2_XATTR_SIZE(loc->xl_entry->xe_name_len); + + /* Value bufs are for value trees */ + BUG_ON(ocfs2_xattr_is_local(loc->xl_entry)); + BUG_ON(namevalue_size_xe(loc->xl_entry) != + (name_size + OCFS2_XATTR_ROOT_SIZE)); + + loc->xl_ops->xlo_fill_value_buf(loc, vb); + vb->vb_xv = + (struct ocfs2_xattr_value_root *)ocfs2_xa_offset_pointer(loc, + nameval_offset + + name_size); +} + +static int ocfs2_xa_block_journal_access(handle_t *handle, + struct ocfs2_xa_loc *loc, int type) +{ + struct buffer_head *bh = loc->xl_storage; + ocfs2_journal_access_func access; + + if (loc->xl_size == (bh->b_size - + offsetof(struct ocfs2_xattr_block, + xb_attrs.xb_header))) + access = ocfs2_journal_access_xb; + else + access = ocfs2_journal_access_di; + return access(handle, INODE_CACHE(loc->xl_inode), bh, type); +} + +static void ocfs2_xa_block_journal_dirty(handle_t *handle, + struct ocfs2_xa_loc *loc) +{ + struct buffer_head *bh = loc->xl_storage; + + ocfs2_journal_dirty(handle, bh); +} + +static void *ocfs2_xa_block_offset_pointer(struct ocfs2_xa_loc *loc, + int offset) +{ + return (char *)loc->xl_header + offset; +} + +static int ocfs2_xa_block_can_reuse(struct ocfs2_xa_loc *loc, + struct ocfs2_xattr_info *xi) +{ + /* + * Block storage is strict. If the sizes aren't exact, we will + * remove the old one and reinsert the new. + */ + return namevalue_size_xe(loc->xl_entry) == + namevalue_size_xi(xi); +} + +static int ocfs2_xa_block_get_free_start(struct ocfs2_xa_loc *loc) +{ + struct ocfs2_xattr_header *xh = loc->xl_header; + int i, count = le16_to_cpu(xh->xh_count); + int offset, free_start = loc->xl_size; + + for (i = 0; i < count; i++) { + offset = le16_to_cpu(xh->xh_entries[i].xe_name_offset); + if (offset < free_start) + free_start = offset; + } + + return free_start; +} + +static int ocfs2_xa_block_check_space(struct ocfs2_xa_loc *loc, + struct ocfs2_xattr_info *xi) +{ + int count = le16_to_cpu(loc->xl_header->xh_count); + int free_start = ocfs2_xa_get_free_start(loc); + int needed_space = ocfs2_xi_entry_usage(xi); + + /* + * Block storage will reclaim the original entry before inserting + * the new value, so we only need the difference. If the new + * entry is smaller than the old one, we don't need anything. + */ + if (loc->xl_entry) { + /* Don't need space if we're reusing! */ + if (ocfs2_xa_can_reuse_entry(loc, xi)) + needed_space = 0; + else + needed_space -= ocfs2_xe_entry_usage(loc->xl_entry); + } + if (needed_space < 0) + needed_space = 0; + return ocfs2_xa_check_space_helper(needed_space, free_start, count); +} + +/* + * Block storage for xattrs keeps the name+value pairs compacted. When + * we remove one, we have to shift any that preceded it towards the end. + */ +static void ocfs2_xa_block_wipe_namevalue(struct ocfs2_xa_loc *loc) +{ + int i, offset; + int namevalue_offset, first_namevalue_offset, namevalue_size; + struct ocfs2_xattr_entry *entry = loc->xl_entry; + struct ocfs2_xattr_header *xh = loc->xl_header; + int count = le16_to_cpu(xh->xh_count); + + namevalue_offset = le16_to_cpu(entry->xe_name_offset); + namevalue_size = namevalue_size_xe(entry); + first_namevalue_offset = ocfs2_xa_get_free_start(loc); + + /* Shift the name+value pairs */ + memmove((char *)xh + first_namevalue_offset + namevalue_size, + (char *)xh + first_namevalue_offset, + namevalue_offset - first_namevalue_offset); + memset((char *)xh + first_namevalue_offset, 0, namevalue_size); + + /* Now tell xh->xh_entries about it */ + for (i = 0; i < count; i++) { + offset = le16_to_cpu(xh->xh_entries[i].xe_name_offset); + if (offset <= namevalue_offset) + le16_add_cpu(&xh->xh_entries[i].xe_name_offset, + namevalue_size); + } + + /* + * Note that we don't update xh_free_start or xh_name_value_len + * because they're not used in block-stored xattrs. + */ +} + +static void ocfs2_xa_block_add_entry(struct ocfs2_xa_loc *loc, u32 name_hash) +{ + int count = le16_to_cpu(loc->xl_header->xh_count); + loc->xl_entry = &(loc->xl_header->xh_entries[count]); + le16_add_cpu(&loc->xl_header->xh_count, 1); + memset(loc->xl_entry, 0, sizeof(struct ocfs2_xattr_entry)); +} + +static void ocfs2_xa_block_add_namevalue(struct ocfs2_xa_loc *loc, int size) +{ + int free_start = ocfs2_xa_get_free_start(loc); + + loc->xl_entry->xe_name_offset = cpu_to_le16(free_start - size); +} + +static void ocfs2_xa_block_fill_value_buf(struct ocfs2_xa_loc *loc, + struct ocfs2_xattr_value_buf *vb) +{ + struct buffer_head *bh = loc->xl_storage; + + if (loc->xl_size == (bh->b_size - + offsetof(struct ocfs2_xattr_block, + xb_attrs.xb_header))) + vb->vb_access = ocfs2_journal_access_xb; + else + vb->vb_access = ocfs2_journal_access_di; + vb->vb_bh = bh; +} + +/* + * Operations for xattrs stored in blocks. This includes inline inode + * storage and unindexed ocfs2_xattr_blocks. + */ +static const struct ocfs2_xa_loc_operations ocfs2_xa_block_loc_ops = { + .xlo_journal_access = ocfs2_xa_block_journal_access, + .xlo_journal_dirty = ocfs2_xa_block_journal_dirty, + .xlo_offset_pointer = ocfs2_xa_block_offset_pointer, + .xlo_check_space = ocfs2_xa_block_check_space, + .xlo_can_reuse = ocfs2_xa_block_can_reuse, + .xlo_get_free_start = ocfs2_xa_block_get_free_start, + .xlo_wipe_namevalue = ocfs2_xa_block_wipe_namevalue, + .xlo_add_entry = ocfs2_xa_block_add_entry, + .xlo_add_namevalue = ocfs2_xa_block_add_namevalue, + .xlo_fill_value_buf = ocfs2_xa_block_fill_value_buf, +}; + +static int ocfs2_xa_bucket_journal_access(handle_t *handle, + struct ocfs2_xa_loc *loc, int type) +{ + struct ocfs2_xattr_bucket *bucket = loc->xl_storage; + + return ocfs2_xattr_bucket_journal_access(handle, bucket, type); +} + +static void ocfs2_xa_bucket_journal_dirty(handle_t *handle, + struct ocfs2_xa_loc *loc) +{ + struct ocfs2_xattr_bucket *bucket = loc->xl_storage; + + ocfs2_xattr_bucket_journal_dirty(handle, bucket); +} + +static void *ocfs2_xa_bucket_offset_pointer(struct ocfs2_xa_loc *loc, + int offset) +{ + struct ocfs2_xattr_bucket *bucket = loc->xl_storage; + int block, block_offset; + + /* The header is at the front of the bucket */ + block = offset >> loc->xl_inode->i_sb->s_blocksize_bits; + block_offset = offset % loc->xl_inode->i_sb->s_blocksize; + + return bucket_block(bucket, block) + block_offset; +} + +static int ocfs2_xa_bucket_can_reuse(struct ocfs2_xa_loc *loc, + struct ocfs2_xattr_info *xi) +{ + return namevalue_size_xe(loc->xl_entry) >= + namevalue_size_xi(xi); +} + +static int ocfs2_xa_bucket_get_free_start(struct ocfs2_xa_loc *loc) +{ + struct ocfs2_xattr_bucket *bucket = loc->xl_storage; + return le16_to_cpu(bucket_xh(bucket)->xh_free_start); +} + +static int ocfs2_bucket_align_free_start(struct super_block *sb, + int free_start, int size) +{ + /* + * We need to make sure that the name+value pair fits within + * one block. + */ + if (((free_start - size) >> sb->s_blocksize_bits) != + ((free_start - 1) >> sb->s_blocksize_bits)) + free_start -= free_start % sb->s_blocksize; + + return free_start; +} + +static int ocfs2_xa_bucket_check_space(struct ocfs2_xa_loc *loc, + struct ocfs2_xattr_info *xi) +{ + int rc; + int count = le16_to_cpu(loc->xl_header->xh_count); + int free_start = ocfs2_xa_get_free_start(loc); + int needed_space = ocfs2_xi_entry_usage(xi); + int size = namevalue_size_xi(xi); + struct super_block *sb = loc->xl_inode->i_sb; + + /* + * Bucket storage does not reclaim name+value pairs it cannot + * reuse. They live as holes until the bucket fills, and then + * the bucket is defragmented. However, the bucket can reclaim + * the ocfs2_xattr_entry. + */ + if (loc->xl_entry) { + /* Don't need space if we're reusing! */ + if (ocfs2_xa_can_reuse_entry(loc, xi)) + needed_space = 0; + else + needed_space -= sizeof(struct ocfs2_xattr_entry); + } + BUG_ON(needed_space < 0); + + if (free_start < size) { + if (needed_space) + return -ENOSPC; + } else { + /* + * First we check if it would fit in the first place. + * Below, we align the free start to a block. This may + * slide us below the minimum gap. By checking unaligned + * first, we avoid that error. + */ + rc = ocfs2_xa_check_space_helper(needed_space, free_start, + count); + if (rc) + return rc; + free_start = ocfs2_bucket_align_free_start(sb, free_start, + size); + } + return ocfs2_xa_check_space_helper(needed_space, free_start, count); +} + +static void ocfs2_xa_bucket_wipe_namevalue(struct ocfs2_xa_loc *loc) +{ + le16_add_cpu(&loc->xl_header->xh_name_value_len, + -namevalue_size_xe(loc->xl_entry)); +} + +static void ocfs2_xa_bucket_add_entry(struct ocfs2_xa_loc *loc, u32 name_hash) +{ + struct ocfs2_xattr_header *xh = loc->xl_header; + int count = le16_to_cpu(xh->xh_count); + int low = 0, high = count - 1, tmp; + struct ocfs2_xattr_entry *tmp_xe; + + /* + * We keep buckets sorted by name_hash, so we need to find + * our insert place. + */ + while (low <= high && count) { + tmp = (low + high) / 2; + tmp_xe = &xh->xh_entries[tmp]; + + if (name_hash > le32_to_cpu(tmp_xe->xe_name_hash)) + low = tmp + 1; + else if (name_hash < le32_to_cpu(tmp_xe->xe_name_hash)) + high = tmp - 1; + else { + low = tmp; + break; + } + } + + if (low != count) + memmove(&xh->xh_entries[low + 1], + &xh->xh_entries[low], + ((count - low) * sizeof(struct ocfs2_xattr_entry))); + + le16_add_cpu(&xh->xh_count, 1); + loc->xl_entry = &xh->xh_entries[low]; + memset(loc->xl_entry, 0, sizeof(struct ocfs2_xattr_entry)); +} + +static void ocfs2_xa_bucket_add_namevalue(struct ocfs2_xa_loc *loc, int size) +{ + int free_start = ocfs2_xa_get_free_start(loc); + struct ocfs2_xattr_header *xh = loc->xl_header; + struct super_block *sb = loc->xl_inode->i_sb; + int nameval_offset; + + free_start = ocfs2_bucket_align_free_start(sb, free_start, size); + nameval_offset = free_start - size; + loc->xl_entry->xe_name_offset = cpu_to_le16(nameval_offset); + xh->xh_free_start = cpu_to_le16(nameval_offset); + le16_add_cpu(&xh->xh_name_value_len, size); + +} + +static void ocfs2_xa_bucket_fill_value_buf(struct ocfs2_xa_loc *loc, + struct ocfs2_xattr_value_buf *vb) +{ + struct ocfs2_xattr_bucket *bucket = loc->xl_storage; + struct super_block *sb = loc->xl_inode->i_sb; + int nameval_offset = le16_to_cpu(loc->xl_entry->xe_name_offset); + int size = namevalue_size_xe(loc->xl_entry); + int block_offset = nameval_offset >> sb->s_blocksize_bits; + + /* Values are not allowed to straddle block boundaries */ + BUG_ON(block_offset != + ((nameval_offset + size - 1) >> sb->s_blocksize_bits)); + /* We expect the bucket to be filled in */ + BUG_ON(!bucket->bu_bhs[block_offset]); + + vb->vb_access = ocfs2_journal_access; + vb->vb_bh = bucket->bu_bhs[block_offset]; +} + +/* Operations for xattrs stored in buckets. */ +static const struct ocfs2_xa_loc_operations ocfs2_xa_bucket_loc_ops = { + .xlo_journal_access = ocfs2_xa_bucket_journal_access, + .xlo_journal_dirty = ocfs2_xa_bucket_journal_dirty, + .xlo_offset_pointer = ocfs2_xa_bucket_offset_pointer, + .xlo_check_space = ocfs2_xa_bucket_check_space, + .xlo_can_reuse = ocfs2_xa_bucket_can_reuse, + .xlo_get_free_start = ocfs2_xa_bucket_get_free_start, + .xlo_wipe_namevalue = ocfs2_xa_bucket_wipe_namevalue, + .xlo_add_entry = ocfs2_xa_bucket_add_entry, + .xlo_add_namevalue = ocfs2_xa_bucket_add_namevalue, + .xlo_fill_value_buf = ocfs2_xa_bucket_fill_value_buf, +}; + +static unsigned int ocfs2_xa_value_clusters(struct ocfs2_xa_loc *loc) +{ + struct ocfs2_xattr_value_buf vb; + + if (ocfs2_xattr_is_local(loc->xl_entry)) + return 0; + + ocfs2_xa_fill_value_buf(loc, &vb); + return le32_to_cpu(vb.vb_xv->xr_clusters); +} + +static int ocfs2_xa_value_truncate(struct ocfs2_xa_loc *loc, u64 bytes, + struct ocfs2_xattr_set_ctxt *ctxt) +{ + int trunc_rc, access_rc; + struct ocfs2_xattr_value_buf vb; + + ocfs2_xa_fill_value_buf(loc, &vb); + trunc_rc = ocfs2_xattr_value_truncate(loc->xl_inode, &vb, bytes, + ctxt); + + /* + * The caller of ocfs2_xa_value_truncate() has already called + * ocfs2_xa_journal_access on the loc. However, The truncate code + * calls ocfs2_extend_trans(). This may commit the previous + * transaction and open a new one. If this is a bucket, truncate + * could leave only vb->vb_bh set up for journaling. Meanwhile, + * the caller is expecting to dirty the entire bucket. So we must + * reset the journal work. We do this even if truncate has failed, + * as it could have failed after committing the extend. + */ + access_rc = ocfs2_xa_journal_access(ctxt->handle, loc, + OCFS2_JOURNAL_ACCESS_WRITE); + + /* Errors in truncate take precedence */ + return trunc_rc ? trunc_rc : access_rc; +} + +static void ocfs2_xa_remove_entry(struct ocfs2_xa_loc *loc) +{ + int index, count; + struct ocfs2_xattr_header *xh = loc->xl_header; + struct ocfs2_xattr_entry *entry = loc->xl_entry; + + ocfs2_xa_wipe_namevalue(loc); + loc->xl_entry = NULL; + + count = le16_to_cpu(xh->xh_count) - 1; + + /* + * Only zero out the entry if there are more remaining. This is + * important for an empty bucket, as it keeps track of the + * bucket's hash value. It doesn't hurt empty block storage. + */ + if (count) { + index = ((char *)entry - (char *)&xh->xh_entries) / + sizeof(struct ocfs2_xattr_entry); + memmove(&xh->xh_entries[index], &xh->xh_entries[index + 1], + (count - index) * sizeof(struct ocfs2_xattr_entry)); + memset(&xh->xh_entries[count], 0, + sizeof(struct ocfs2_xattr_entry)); + } + + xh->xh_count = cpu_to_le16(count); +} + +/* + * If we have a problem adjusting the size of an external value during + * ocfs2_xa_prepare_entry() or ocfs2_xa_remove(), we may have an xattr + * in an intermediate state. For example, the value may be partially + * truncated. + * + * If the value tree hasn't changed, the extend/truncate went nowhere. + * We have nothing to do. The caller can treat it as a straight error. + * + * If the value tree got partially truncated, we now have a corrupted + * extended attribute. We're going to wipe its entry and leak the + * clusters. Better to leak some storage than leave a corrupt entry. + * + * If the value tree grew, it obviously didn't grow enough for the + * new entry. We're not going to try and reclaim those clusters either. + * If there was already an external value there (orig_clusters != 0), + * the new clusters are attached safely and we can just leave the old + * value in place. If there was no external value there, we remove + * the entry. + * + * This way, the xattr block we store in the journal will be consistent. + * If the size change broke because of the journal, no changes will hit + * disk anyway. + */ +static void ocfs2_xa_cleanup_value_truncate(struct ocfs2_xa_loc *loc, + const char *what, + unsigned int orig_clusters) +{ + unsigned int new_clusters = ocfs2_xa_value_clusters(loc); + char *nameval_buf = ocfs2_xa_offset_pointer(loc, + le16_to_cpu(loc->xl_entry->xe_name_offset)); + + if (new_clusters < orig_clusters) { + mlog(ML_ERROR, + "Partial truncate while %s xattr %.*s. Leaking " + "%u clusters and removing the entry\n", + what, loc->xl_entry->xe_name_len, nameval_buf, + orig_clusters - new_clusters); + ocfs2_xa_remove_entry(loc); + } else if (!orig_clusters) { + mlog(ML_ERROR, + "Unable to allocate an external value for xattr " + "%.*s safely. Leaking %u clusters and removing the " + "entry\n", + loc->xl_entry->xe_name_len, nameval_buf, + new_clusters - orig_clusters); + ocfs2_xa_remove_entry(loc); + } else if (new_clusters > orig_clusters) + mlog(ML_ERROR, + "Unable to grow xattr %.*s safely. %u new clusters " + "have been added, but the value will not be " + "modified\n", + loc->xl_entry->xe_name_len, nameval_buf, + new_clusters - orig_clusters); +} + +static int ocfs2_xa_remove(struct ocfs2_xa_loc *loc, + struct ocfs2_xattr_set_ctxt *ctxt) +{ + int rc = 0; + unsigned int orig_clusters; + + if (!ocfs2_xattr_is_local(loc->xl_entry)) { + orig_clusters = ocfs2_xa_value_clusters(loc); + rc = ocfs2_xa_value_truncate(loc, 0, ctxt); + if (rc) { + mlog_errno(rc); + /* + * Since this is remove, we can return 0 if + * ocfs2_xa_cleanup_value_truncate() is going to + * wipe the entry anyway. So we check the + * cluster count as well. + */ + if (orig_clusters != ocfs2_xa_value_clusters(loc)) + rc = 0; + ocfs2_xa_cleanup_value_truncate(loc, "removing", + orig_clusters); + goto out; + } + } + + ocfs2_xa_remove_entry(loc); + +out: + return rc; +} + +static void ocfs2_xa_install_value_root(struct ocfs2_xa_loc *loc) +{ + int name_size = OCFS2_XATTR_SIZE(loc->xl_entry->xe_name_len); + char *nameval_buf; + + nameval_buf = ocfs2_xa_offset_pointer(loc, + le16_to_cpu(loc->xl_entry->xe_name_offset)); + memcpy(nameval_buf + name_size, &def_xv, OCFS2_XATTR_ROOT_SIZE); +} + +/* + * Take an existing entry and make it ready for the new value. This + * won't allocate space, but it may free space. It should be ready for + * ocfs2_xa_prepare_entry() to finish the work. + */ +static int ocfs2_xa_reuse_entry(struct ocfs2_xa_loc *loc, + struct ocfs2_xattr_info *xi, + struct ocfs2_xattr_set_ctxt *ctxt) +{ + int rc = 0; + int name_size = OCFS2_XATTR_SIZE(xi->xi_name_len); + unsigned int orig_clusters; + char *nameval_buf; + int xe_local = ocfs2_xattr_is_local(loc->xl_entry); + int xi_local = xi->xi_value_len <= OCFS2_XATTR_INLINE_SIZE; + + BUG_ON(OCFS2_XATTR_SIZE(loc->xl_entry->xe_name_len) != + name_size); + + nameval_buf = ocfs2_xa_offset_pointer(loc, + le16_to_cpu(loc->xl_entry->xe_name_offset)); + if (xe_local) { + memset(nameval_buf + name_size, 0, + namevalue_size_xe(loc->xl_entry) - name_size); + if (!xi_local) + ocfs2_xa_install_value_root(loc); + } else { + orig_clusters = ocfs2_xa_value_clusters(loc); + if (xi_local) { + rc = ocfs2_xa_value_truncate(loc, 0, ctxt); + if (rc < 0) + mlog_errno(rc); + else + memset(nameval_buf + name_size, 0, + namevalue_size_xe(loc->xl_entry) - + name_size); + } else if (le64_to_cpu(loc->xl_entry->xe_value_size) > + xi->xi_value_len) { + rc = ocfs2_xa_value_truncate(loc, xi->xi_value_len, + ctxt); + if (rc < 0) + mlog_errno(rc); + } + + if (rc) { + ocfs2_xa_cleanup_value_truncate(loc, "reusing", + orig_clusters); + goto out; + } + } + + loc->xl_entry->xe_value_size = cpu_to_le64(xi->xi_value_len); + ocfs2_xattr_set_local(loc->xl_entry, xi_local); + +out: + return rc; +} + +/* + * Prepares loc->xl_entry to receive the new xattr. This includes + * properly setting up the name+value pair region. If loc->xl_entry + * already exists, it will take care of modifying it appropriately. + * + * Note that this modifies the data. You did journal_access already, + * right? + */ +static int ocfs2_xa_prepare_entry(struct ocfs2_xa_loc *loc, + struct ocfs2_xattr_info *xi, + u32 name_hash, + struct ocfs2_xattr_set_ctxt *ctxt) +{ + int rc = 0; + unsigned int orig_clusters; + __le64 orig_value_size = 0; + + rc = ocfs2_xa_check_space(loc, xi); + if (rc) + goto out; + + if (loc->xl_entry) { + if (ocfs2_xa_can_reuse_entry(loc, xi)) { + orig_value_size = loc->xl_entry->xe_value_size; + rc = ocfs2_xa_reuse_entry(loc, xi, ctxt); + if (rc) + goto out; + goto alloc_value; + } + + if (!ocfs2_xattr_is_local(loc->xl_entry)) { + orig_clusters = ocfs2_xa_value_clusters(loc); + rc = ocfs2_xa_value_truncate(loc, 0, ctxt); + if (rc) { + mlog_errno(rc); + ocfs2_xa_cleanup_value_truncate(loc, + "overwriting", + orig_clusters); + goto out; + } + } + ocfs2_xa_wipe_namevalue(loc); + } else + ocfs2_xa_add_entry(loc, name_hash); + + /* + * If we get here, we have a blank entry. Fill it. We grow our + * name+value pair back from the end. + */ + ocfs2_xa_add_namevalue(loc, xi); + if (xi->xi_value_len > OCFS2_XATTR_INLINE_SIZE) + ocfs2_xa_install_value_root(loc); + +alloc_value: + if (xi->xi_value_len > OCFS2_XATTR_INLINE_SIZE) { + orig_clusters = ocfs2_xa_value_clusters(loc); + rc = ocfs2_xa_value_truncate(loc, xi->xi_value_len, ctxt); + if (rc < 0) { + ctxt->set_abort = 1; + ocfs2_xa_cleanup_value_truncate(loc, "growing", + orig_clusters); + /* + * If we were growing an existing value, + * ocfs2_xa_cleanup_value_truncate() won't remove + * the entry. We need to restore the original value + * size. + */ + if (loc->xl_entry) { + BUG_ON(!orig_value_size); + loc->xl_entry->xe_value_size = orig_value_size; + } + mlog_errno(rc); + } + } + +out: + return rc; +} + +/* + * Store the value portion of the name+value pair. This will skip + * values that are stored externally. Their tree roots were set up + * by ocfs2_xa_prepare_entry(). + */ +static int ocfs2_xa_store_value(struct ocfs2_xa_loc *loc, + struct ocfs2_xattr_info *xi, + struct ocfs2_xattr_set_ctxt *ctxt) +{ + int rc = 0; + int nameval_offset = le16_to_cpu(loc->xl_entry->xe_name_offset); + int name_size = OCFS2_XATTR_SIZE(xi->xi_name_len); + char *nameval_buf; + struct ocfs2_xattr_value_buf vb; + + nameval_buf = ocfs2_xa_offset_pointer(loc, nameval_offset); + if (xi->xi_value_len > OCFS2_XATTR_INLINE_SIZE) { + ocfs2_xa_fill_value_buf(loc, &vb); + rc = __ocfs2_xattr_set_value_outside(loc->xl_inode, + ctxt->handle, &vb, + xi->xi_value, + xi->xi_value_len); + } else + memcpy(nameval_buf + name_size, xi->xi_value, xi->xi_value_len); + + return rc; +} + +static int ocfs2_xa_set(struct ocfs2_xa_loc *loc, + struct ocfs2_xattr_info *xi, + struct ocfs2_xattr_set_ctxt *ctxt) +{ + int ret; + u32 name_hash = ocfs2_xattr_name_hash(loc->xl_inode, xi->xi_name, + xi->xi_name_len); + + ret = ocfs2_xa_journal_access(ctxt->handle, loc, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * From here on out, everything is going to modify the buffer a + * little. Errors are going to leave the xattr header in a + * sane state. Thus, even with errors we dirty the sucker. + */ + + /* Don't worry, we are never called with !xi_value and !xl_entry */ + if (!xi->xi_value) { + ret = ocfs2_xa_remove(loc, ctxt); + goto out_dirty; + } + + ret = ocfs2_xa_prepare_entry(loc, xi, name_hash, ctxt); + if (ret) { + if (ret != -ENOSPC) + mlog_errno(ret); + goto out_dirty; + } + + ret = ocfs2_xa_store_value(loc, xi, ctxt); + if (ret) + mlog_errno(ret); + +out_dirty: + ocfs2_xa_journal_dirty(ctxt->handle, loc); + +out: + return ret; +} + +static void ocfs2_init_dinode_xa_loc(struct ocfs2_xa_loc *loc, + struct inode *inode, + struct buffer_head *bh, + struct ocfs2_xattr_entry *entry) +{ + struct ocfs2_dinode *di = (struct ocfs2_dinode *)bh->b_data; + + BUG_ON(!(OCFS2_I(inode)->ip_dyn_features & OCFS2_INLINE_XATTR_FL)); + + loc->xl_inode = inode; + loc->xl_ops = &ocfs2_xa_block_loc_ops; + loc->xl_storage = bh; + loc->xl_entry = entry; + loc->xl_size = le16_to_cpu(di->i_xattr_inline_size); + loc->xl_header = + (struct ocfs2_xattr_header *)(bh->b_data + bh->b_size - + loc->xl_size); +} + +static void ocfs2_init_xattr_block_xa_loc(struct ocfs2_xa_loc *loc, + struct inode *inode, + struct buffer_head *bh, + struct ocfs2_xattr_entry *entry) +{ + struct ocfs2_xattr_block *xb = + (struct ocfs2_xattr_block *)bh->b_data; + + BUG_ON(le16_to_cpu(xb->xb_flags) & OCFS2_XATTR_INDEXED); + + loc->xl_inode = inode; + loc->xl_ops = &ocfs2_xa_block_loc_ops; + loc->xl_storage = bh; + loc->xl_header = &(xb->xb_attrs.xb_header); + loc->xl_entry = entry; + loc->xl_size = bh->b_size - offsetof(struct ocfs2_xattr_block, + xb_attrs.xb_header); +} + +static void ocfs2_init_xattr_bucket_xa_loc(struct ocfs2_xa_loc *loc, + struct ocfs2_xattr_bucket *bucket, + struct ocfs2_xattr_entry *entry) +{ + loc->xl_inode = bucket->bu_inode; + loc->xl_ops = &ocfs2_xa_bucket_loc_ops; + loc->xl_storage = bucket; + loc->xl_header = bucket_xh(bucket); + loc->xl_entry = entry; + loc->xl_size = OCFS2_XATTR_BUCKET_SIZE; +} + +/* + * In xattr remove, if it is stored outside and refcounted, we may have + * the chance to split the refcount tree. So need the allocators. + */ +static int ocfs2_lock_xattr_remove_allocators(struct inode *inode, + struct ocfs2_xattr_value_root *xv, + struct ocfs2_caching_info *ref_ci, + struct buffer_head *ref_root_bh, + struct ocfs2_alloc_context **meta_ac, + int *ref_credits) +{ + int ret, meta_add = 0; + u32 p_cluster, num_clusters; + unsigned int ext_flags; + + *ref_credits = 0; + ret = ocfs2_xattr_get_clusters(inode, 0, &p_cluster, + &num_clusters, + &xv->xr_list, + &ext_flags); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (!(ext_flags & OCFS2_EXT_REFCOUNTED)) + goto out; + + ret = ocfs2_refcounted_xattr_delete_need(inode, ref_ci, + ref_root_bh, xv, + &meta_add, ref_credits); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_reserve_new_metadata_blocks(OCFS2_SB(inode->i_sb), + meta_add, meta_ac); + if (ret) + mlog_errno(ret); + +out: + return ret; +} + +static int ocfs2_remove_value_outside(struct inode*inode, + struct ocfs2_xattr_value_buf *vb, + struct ocfs2_xattr_header *header, + struct ocfs2_caching_info *ref_ci, + struct buffer_head *ref_root_bh) +{ + int ret = 0, i, ref_credits; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct ocfs2_xattr_set_ctxt ctxt = { NULL, NULL, }; + void *val; + + ocfs2_init_dealloc_ctxt(&ctxt.dealloc); + + for (i = 0; i < le16_to_cpu(header->xh_count); i++) { + struct ocfs2_xattr_entry *entry = &header->xh_entries[i]; + + if (ocfs2_xattr_is_local(entry)) + continue; + + val = (void *)header + + le16_to_cpu(entry->xe_name_offset); + vb->vb_xv = (struct ocfs2_xattr_value_root *) + (val + OCFS2_XATTR_SIZE(entry->xe_name_len)); + + ret = ocfs2_lock_xattr_remove_allocators(inode, vb->vb_xv, + ref_ci, ref_root_bh, + &ctxt.meta_ac, + &ref_credits); + + ctxt.handle = ocfs2_start_trans(osb, ref_credits + + ocfs2_remove_extent_credits(osb->sb)); + if (IS_ERR(ctxt.handle)) { + ret = PTR_ERR(ctxt.handle); + mlog_errno(ret); + break; + } + + ret = ocfs2_xattr_value_truncate(inode, vb, 0, &ctxt); + + ocfs2_commit_trans(osb, ctxt.handle); + if (ctxt.meta_ac) { + ocfs2_free_alloc_context(ctxt.meta_ac); + ctxt.meta_ac = NULL; + } + + if (ret < 0) { + mlog_errno(ret); + break; + } + + } + + if (ctxt.meta_ac) + ocfs2_free_alloc_context(ctxt.meta_ac); + ocfs2_schedule_truncate_log_flush(osb, 1); + ocfs2_run_deallocs(osb, &ctxt.dealloc); + return ret; +} + +static int ocfs2_xattr_ibody_remove(struct inode *inode, + struct buffer_head *di_bh, + struct ocfs2_caching_info *ref_ci, + struct buffer_head *ref_root_bh) +{ + + struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data; + struct ocfs2_xattr_header *header; + int ret; + struct ocfs2_xattr_value_buf vb = { + .vb_bh = di_bh, + .vb_access = ocfs2_journal_access_di, + }; + + ret = ocfs2_xattr_ibody_lookup_header(inode, di, &header); + if (ret) + return ret; + + ret = ocfs2_remove_value_outside(inode, &vb, header, + ref_ci, ref_root_bh); + + return ret; +} + +struct ocfs2_rm_xattr_bucket_para { + struct ocfs2_caching_info *ref_ci; + struct buffer_head *ref_root_bh; +}; + +static int ocfs2_xattr_block_remove(struct inode *inode, + struct buffer_head *blk_bh, + struct ocfs2_caching_info *ref_ci, + struct buffer_head *ref_root_bh) +{ + struct ocfs2_xattr_block *xb; + int ret = 0; + struct ocfs2_xattr_value_buf vb = { + .vb_bh = blk_bh, + .vb_access = ocfs2_journal_access_xb, + }; + struct ocfs2_rm_xattr_bucket_para args = { + .ref_ci = ref_ci, + .ref_root_bh = ref_root_bh, + }; + + xb = (struct ocfs2_xattr_block *)blk_bh->b_data; + if (!(le16_to_cpu(xb->xb_flags) & OCFS2_XATTR_INDEXED)) { + struct ocfs2_xattr_header *header = &(xb->xb_attrs.xb_header); + ret = ocfs2_remove_value_outside(inode, &vb, header, + ref_ci, ref_root_bh); + } else + ret = ocfs2_iterate_xattr_index_block(inode, + blk_bh, + ocfs2_rm_xattr_cluster, + &args); + + return ret; +} + +static int ocfs2_xattr_free_block(struct inode *inode, + u64 block, + struct ocfs2_caching_info *ref_ci, + struct buffer_head *ref_root_bh) +{ + struct inode *xb_alloc_inode; + struct buffer_head *xb_alloc_bh = NULL; + struct buffer_head *blk_bh = NULL; + struct ocfs2_xattr_block *xb; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + handle_t *handle; + int ret = 0; + u64 blk, bg_blkno; + u16 bit; + + ret = ocfs2_read_xattr_block(inode, block, &blk_bh); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_xattr_block_remove(inode, blk_bh, ref_ci, ref_root_bh); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + xb = (struct ocfs2_xattr_block *)blk_bh->b_data; + blk = le64_to_cpu(xb->xb_blkno); + bit = le16_to_cpu(xb->xb_suballoc_bit); + if (xb->xb_suballoc_loc) + bg_blkno = le64_to_cpu(xb->xb_suballoc_loc); + else + bg_blkno = ocfs2_which_suballoc_group(blk, bit); + + xb_alloc_inode = ocfs2_get_system_file_inode(osb, + EXTENT_ALLOC_SYSTEM_INODE, + le16_to_cpu(xb->xb_suballoc_slot)); + if (!xb_alloc_inode) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + inode_lock(xb_alloc_inode); + + ret = ocfs2_inode_lock(xb_alloc_inode, &xb_alloc_bh, 1); + if (ret < 0) { + mlog_errno(ret); + goto out_mutex; + } + + handle = ocfs2_start_trans(osb, OCFS2_SUBALLOC_FREE); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out_unlock; + } + + ret = ocfs2_free_suballoc_bits(handle, xb_alloc_inode, xb_alloc_bh, + bit, bg_blkno, 1); + if (ret < 0) + mlog_errno(ret); + + ocfs2_commit_trans(osb, handle); +out_unlock: + ocfs2_inode_unlock(xb_alloc_inode, 1); + brelse(xb_alloc_bh); +out_mutex: + inode_unlock(xb_alloc_inode); + iput(xb_alloc_inode); +out: + brelse(blk_bh); + return ret; +} + +/* + * ocfs2_xattr_remove() + * + * Free extended attribute resources associated with this inode. + */ +int ocfs2_xattr_remove(struct inode *inode, struct buffer_head *di_bh) +{ + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data; + struct ocfs2_refcount_tree *ref_tree = NULL; + struct buffer_head *ref_root_bh = NULL; + struct ocfs2_caching_info *ref_ci = NULL; + handle_t *handle; + int ret; + + if (!ocfs2_supports_xattr(OCFS2_SB(inode->i_sb))) + return 0; + + if (!(oi->ip_dyn_features & OCFS2_HAS_XATTR_FL)) + return 0; + + if (ocfs2_is_refcount_inode(inode)) { + ret = ocfs2_lock_refcount_tree(OCFS2_SB(inode->i_sb), + le64_to_cpu(di->i_refcount_loc), + 1, &ref_tree, &ref_root_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + ref_ci = &ref_tree->rf_ci; + + } + + if (oi->ip_dyn_features & OCFS2_INLINE_XATTR_FL) { + ret = ocfs2_xattr_ibody_remove(inode, di_bh, + ref_ci, ref_root_bh); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + } + + if (di->i_xattr_loc) { + ret = ocfs2_xattr_free_block(inode, + le64_to_cpu(di->i_xattr_loc), + ref_ci, ref_root_bh); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + } + + handle = ocfs2_start_trans((OCFS2_SB(inode->i_sb)), + OCFS2_INODE_UPDATE_CREDITS); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out; + } + ret = ocfs2_journal_access_di(handle, INODE_CACHE(inode), di_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + di->i_xattr_loc = 0; + + spin_lock(&oi->ip_lock); + oi->ip_dyn_features &= ~(OCFS2_INLINE_XATTR_FL | OCFS2_HAS_XATTR_FL); + di->i_dyn_features = cpu_to_le16(oi->ip_dyn_features); + spin_unlock(&oi->ip_lock); + ocfs2_update_inode_fsync_trans(handle, inode, 0); + + ocfs2_journal_dirty(handle, di_bh); +out_commit: + ocfs2_commit_trans(OCFS2_SB(inode->i_sb), handle); +out: + if (ref_tree) + ocfs2_unlock_refcount_tree(OCFS2_SB(inode->i_sb), ref_tree, 1); + brelse(ref_root_bh); + return ret; +} + +static int ocfs2_xattr_has_space_inline(struct inode *inode, + struct ocfs2_dinode *di) +{ + struct ocfs2_inode_info *oi = OCFS2_I(inode); + unsigned int xattrsize = OCFS2_SB(inode->i_sb)->s_xattr_inline_size; + int free; + + if (xattrsize < OCFS2_MIN_XATTR_INLINE_SIZE) + return 0; + + if (oi->ip_dyn_features & OCFS2_INLINE_DATA_FL) { + struct ocfs2_inline_data *idata = &di->id2.i_data; + free = le16_to_cpu(idata->id_count) - le64_to_cpu(di->i_size); + } else if (ocfs2_inode_is_fast_symlink(inode)) { + free = ocfs2_fast_symlink_chars(inode->i_sb) - + le64_to_cpu(di->i_size); + } else { + struct ocfs2_extent_list *el = &di->id2.i_list; + free = (le16_to_cpu(el->l_count) - + le16_to_cpu(el->l_next_free_rec)) * + sizeof(struct ocfs2_extent_rec); + } + if (free >= xattrsize) + return 1; + + return 0; +} + +/* + * ocfs2_xattr_ibody_find() + * + * Find extended attribute in inode block and + * fill search info into struct ocfs2_xattr_search. + */ +static int ocfs2_xattr_ibody_find(struct inode *inode, + int name_index, + const char *name, + struct ocfs2_xattr_search *xs) +{ + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_dinode *di = (struct ocfs2_dinode *)xs->inode_bh->b_data; + int ret; + int has_space = 0; + + if (inode->i_sb->s_blocksize == OCFS2_MIN_BLOCKSIZE) + return 0; + + if (!(oi->ip_dyn_features & OCFS2_INLINE_XATTR_FL)) { + down_read(&oi->ip_alloc_sem); + has_space = ocfs2_xattr_has_space_inline(inode, di); + up_read(&oi->ip_alloc_sem); + if (!has_space) + return 0; + } + + xs->xattr_bh = xs->inode_bh; + xs->end = (void *)di + inode->i_sb->s_blocksize; + if (oi->ip_dyn_features & OCFS2_INLINE_XATTR_FL) { + ret = ocfs2_xattr_ibody_lookup_header(inode, di, &xs->header); + if (ret) + return ret; + } else { + xs->header = (struct ocfs2_xattr_header *) + (xs->end - OCFS2_SB(inode->i_sb)->s_xattr_inline_size); + } + xs->base = (void *)xs->header; + xs->here = xs->header->xh_entries; + + /* Find the named attribute. */ + if (oi->ip_dyn_features & OCFS2_INLINE_XATTR_FL) { + ret = ocfs2_xattr_find_entry(inode, name_index, name, xs); + if (ret && ret != -ENODATA) + return ret; + xs->not_found = ret; + } + + return 0; +} + +static int ocfs2_xattr_ibody_init(struct inode *inode, + struct buffer_head *di_bh, + struct ocfs2_xattr_set_ctxt *ctxt) +{ + int ret; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_dinode *di = (struct ocfs2_dinode *)di_bh->b_data; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + unsigned int xattrsize = osb->s_xattr_inline_size; + + if (!ocfs2_xattr_has_space_inline(inode, di)) { + ret = -ENOSPC; + goto out; + } + + ret = ocfs2_journal_access_di(ctxt->handle, INODE_CACHE(inode), di_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * Adjust extent record count or inline data size + * to reserve space for extended attribute. + */ + if (oi->ip_dyn_features & OCFS2_INLINE_DATA_FL) { + struct ocfs2_inline_data *idata = &di->id2.i_data; + le16_add_cpu(&idata->id_count, -xattrsize); + } else if (!(ocfs2_inode_is_fast_symlink(inode))) { + struct ocfs2_extent_list *el = &di->id2.i_list; + le16_add_cpu(&el->l_count, -(xattrsize / + sizeof(struct ocfs2_extent_rec))); + } + di->i_xattr_inline_size = cpu_to_le16(xattrsize); + + spin_lock(&oi->ip_lock); + oi->ip_dyn_features |= OCFS2_INLINE_XATTR_FL|OCFS2_HAS_XATTR_FL; + di->i_dyn_features = cpu_to_le16(oi->ip_dyn_features); + spin_unlock(&oi->ip_lock); + + ocfs2_journal_dirty(ctxt->handle, di_bh); + +out: + return ret; +} + +/* + * ocfs2_xattr_ibody_set() + * + * Set, replace or remove an extended attribute into inode block. + * + */ +static int ocfs2_xattr_ibody_set(struct inode *inode, + struct ocfs2_xattr_info *xi, + struct ocfs2_xattr_search *xs, + struct ocfs2_xattr_set_ctxt *ctxt) +{ + int ret; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_xa_loc loc; + + if (inode->i_sb->s_blocksize == OCFS2_MIN_BLOCKSIZE) + return -ENOSPC; + + down_write(&oi->ip_alloc_sem); + if (!(oi->ip_dyn_features & OCFS2_INLINE_XATTR_FL)) { + ret = ocfs2_xattr_ibody_init(inode, xs->inode_bh, ctxt); + if (ret) { + if (ret != -ENOSPC) + mlog_errno(ret); + goto out; + } + } + + ocfs2_init_dinode_xa_loc(&loc, inode, xs->inode_bh, + xs->not_found ? NULL : xs->here); + ret = ocfs2_xa_set(&loc, xi, ctxt); + if (ret) { + if (ret != -ENOSPC) + mlog_errno(ret); + goto out; + } + xs->here = loc.xl_entry; + +out: + up_write(&oi->ip_alloc_sem); + + return ret; +} + +/* + * ocfs2_xattr_block_find() + * + * Find extended attribute in external block and + * fill search info into struct ocfs2_xattr_search. + */ +static int ocfs2_xattr_block_find(struct inode *inode, + int name_index, + const char *name, + struct ocfs2_xattr_search *xs) +{ + struct ocfs2_dinode *di = (struct ocfs2_dinode *)xs->inode_bh->b_data; + struct buffer_head *blk_bh = NULL; + struct ocfs2_xattr_block *xb; + int ret = 0; + + if (!di->i_xattr_loc) + return ret; + + ret = ocfs2_read_xattr_block(inode, le64_to_cpu(di->i_xattr_loc), + &blk_bh); + if (ret < 0) { + mlog_errno(ret); + return ret; + } + + xs->xattr_bh = blk_bh; + xb = (struct ocfs2_xattr_block *)blk_bh->b_data; + + if (!(le16_to_cpu(xb->xb_flags) & OCFS2_XATTR_INDEXED)) { + xs->header = &xb->xb_attrs.xb_header; + xs->base = (void *)xs->header; + xs->end = (void *)(blk_bh->b_data) + blk_bh->b_size; + xs->here = xs->header->xh_entries; + + ret = ocfs2_xattr_find_entry(inode, name_index, name, xs); + } else + ret = ocfs2_xattr_index_block_find(inode, blk_bh, + name_index, + name, xs); + + if (ret && ret != -ENODATA) { + xs->xattr_bh = NULL; + goto cleanup; + } + xs->not_found = ret; + return 0; +cleanup: + brelse(blk_bh); + + return ret; +} + +static int ocfs2_create_xattr_block(struct inode *inode, + struct buffer_head *inode_bh, + struct ocfs2_xattr_set_ctxt *ctxt, + int indexed, + struct buffer_head **ret_bh) +{ + int ret; + u16 suballoc_bit_start; + u32 num_got; + u64 suballoc_loc, first_blkno; + struct ocfs2_dinode *di = (struct ocfs2_dinode *)inode_bh->b_data; + struct buffer_head *new_bh = NULL; + struct ocfs2_xattr_block *xblk; + + ret = ocfs2_journal_access_di(ctxt->handle, INODE_CACHE(inode), + inode_bh, OCFS2_JOURNAL_ACCESS_CREATE); + if (ret < 0) { + mlog_errno(ret); + goto end; + } + + ret = ocfs2_claim_metadata(ctxt->handle, ctxt->meta_ac, 1, + &suballoc_loc, &suballoc_bit_start, + &num_got, &first_blkno); + if (ret < 0) { + mlog_errno(ret); + goto end; + } + + new_bh = sb_getblk(inode->i_sb, first_blkno); + if (!new_bh) { + ret = -ENOMEM; + mlog_errno(ret); + goto end; + } + + ocfs2_set_new_buffer_uptodate(INODE_CACHE(inode), new_bh); + + ret = ocfs2_journal_access_xb(ctxt->handle, INODE_CACHE(inode), + new_bh, + OCFS2_JOURNAL_ACCESS_CREATE); + if (ret < 0) { + mlog_errno(ret); + goto end; + } + + /* Initialize ocfs2_xattr_block */ + xblk = (struct ocfs2_xattr_block *)new_bh->b_data; + memset(xblk, 0, inode->i_sb->s_blocksize); + strscpy(xblk->xb_signature, OCFS2_XATTR_BLOCK_SIGNATURE); + xblk->xb_suballoc_slot = cpu_to_le16(ctxt->meta_ac->ac_alloc_slot); + xblk->xb_suballoc_loc = cpu_to_le64(suballoc_loc); + xblk->xb_suballoc_bit = cpu_to_le16(suballoc_bit_start); + xblk->xb_fs_generation = + cpu_to_le32(OCFS2_SB(inode->i_sb)->fs_generation); + xblk->xb_blkno = cpu_to_le64(first_blkno); + if (indexed) { + struct ocfs2_xattr_tree_root *xr = &xblk->xb_attrs.xb_root; + xr->xt_clusters = cpu_to_le32(1); + xr->xt_last_eb_blk = 0; + xr->xt_list.l_tree_depth = 0; + xr->xt_list.l_count = cpu_to_le16( + ocfs2_xattr_recs_per_xb(inode->i_sb)); + xr->xt_list.l_next_free_rec = cpu_to_le16(1); + xblk->xb_flags = cpu_to_le16(OCFS2_XATTR_INDEXED); + } + ocfs2_journal_dirty(ctxt->handle, new_bh); + + /* Add it to the inode */ + di->i_xattr_loc = cpu_to_le64(first_blkno); + + spin_lock(&OCFS2_I(inode)->ip_lock); + OCFS2_I(inode)->ip_dyn_features |= OCFS2_HAS_XATTR_FL; + di->i_dyn_features = cpu_to_le16(OCFS2_I(inode)->ip_dyn_features); + spin_unlock(&OCFS2_I(inode)->ip_lock); + + ocfs2_journal_dirty(ctxt->handle, inode_bh); + + *ret_bh = new_bh; + new_bh = NULL; + +end: + brelse(new_bh); + return ret; +} + +/* + * ocfs2_xattr_block_set() + * + * Set, replace or remove an extended attribute into external block. + * + */ +static int ocfs2_xattr_block_set(struct inode *inode, + struct ocfs2_xattr_info *xi, + struct ocfs2_xattr_search *xs, + struct ocfs2_xattr_set_ctxt *ctxt) +{ + struct buffer_head *new_bh = NULL; + struct ocfs2_xattr_block *xblk = NULL; + int ret; + struct ocfs2_xa_loc loc; + + if (!xs->xattr_bh) { + ret = ocfs2_create_xattr_block(inode, xs->inode_bh, ctxt, + 0, &new_bh); + if (ret) { + mlog_errno(ret); + goto end; + } + + xs->xattr_bh = new_bh; + xblk = (struct ocfs2_xattr_block *)xs->xattr_bh->b_data; + xs->header = &xblk->xb_attrs.xb_header; + xs->base = (void *)xs->header; + xs->end = (void *)xblk + inode->i_sb->s_blocksize; + xs->here = xs->header->xh_entries; + } else + xblk = (struct ocfs2_xattr_block *)xs->xattr_bh->b_data; + + if (!(le16_to_cpu(xblk->xb_flags) & OCFS2_XATTR_INDEXED)) { + ocfs2_init_xattr_block_xa_loc(&loc, inode, xs->xattr_bh, + xs->not_found ? NULL : xs->here); + + ret = ocfs2_xa_set(&loc, xi, ctxt); + if (!ret) + xs->here = loc.xl_entry; + else if ((ret != -ENOSPC) || ctxt->set_abort) + goto end; + else { + ret = ocfs2_xattr_create_index_block(inode, xs, ctxt); + if (ret) + goto end; + } + } + + if (le16_to_cpu(xblk->xb_flags) & OCFS2_XATTR_INDEXED) + ret = ocfs2_xattr_set_entry_index_block(inode, xi, xs, ctxt); + +end: + return ret; +} + +/* Check whether the new xattr can be inserted into the inode. */ +static int ocfs2_xattr_can_be_in_inode(struct inode *inode, + struct ocfs2_xattr_info *xi, + struct ocfs2_xattr_search *xs) +{ + struct ocfs2_xattr_entry *last; + int free, i; + size_t min_offs = xs->end - xs->base; + + if (!xs->header) + return 0; + + last = xs->header->xh_entries; + + for (i = 0; i < le16_to_cpu(xs->header->xh_count); i++) { + size_t offs = le16_to_cpu(last->xe_name_offset); + if (offs < min_offs) + min_offs = offs; + last += 1; + } + + free = min_offs - ((void *)last - xs->base) - OCFS2_XATTR_HEADER_GAP; + if (free < 0) + return 0; + + BUG_ON(!xs->not_found); + + if (free >= (sizeof(struct ocfs2_xattr_entry) + namevalue_size_xi(xi))) + return 1; + + return 0; +} + +static int ocfs2_calc_xattr_set_need(struct inode *inode, + struct ocfs2_dinode *di, + struct ocfs2_xattr_info *xi, + struct ocfs2_xattr_search *xis, + struct ocfs2_xattr_search *xbs, + int *clusters_need, + int *meta_need, + int *credits_need) +{ + int ret = 0, old_in_xb = 0; + int clusters_add = 0, meta_add = 0, credits = 0; + struct buffer_head *bh = NULL; + struct ocfs2_xattr_block *xb = NULL; + struct ocfs2_xattr_entry *xe = NULL; + struct ocfs2_xattr_value_root *xv = NULL; + char *base = NULL; + int name_offset, name_len = 0; + u32 new_clusters = ocfs2_clusters_for_bytes(inode->i_sb, + xi->xi_value_len); + u64 value_size; + + /* + * Calculate the clusters we need to write. + * No matter whether we replace an old one or add a new one, + * we need this for writing. + */ + if (xi->xi_value_len > OCFS2_XATTR_INLINE_SIZE) + credits += new_clusters * + ocfs2_clusters_to_blocks(inode->i_sb, 1); + + if (xis->not_found && xbs->not_found) { + credits += ocfs2_blocks_per_xattr_bucket(inode->i_sb); + + if (xi->xi_value_len > OCFS2_XATTR_INLINE_SIZE) { + clusters_add += new_clusters; + credits += ocfs2_calc_extend_credits(inode->i_sb, + &def_xv.xv.xr_list); + } + + goto meta_guess; + } + + if (!xis->not_found) { + xe = xis->here; + name_offset = le16_to_cpu(xe->xe_name_offset); + name_len = OCFS2_XATTR_SIZE(xe->xe_name_len); + base = xis->base; + credits += OCFS2_INODE_UPDATE_CREDITS; + } else { + int i, block_off = 0; + xb = (struct ocfs2_xattr_block *)xbs->xattr_bh->b_data; + xe = xbs->here; + name_offset = le16_to_cpu(xe->xe_name_offset); + name_len = OCFS2_XATTR_SIZE(xe->xe_name_len); + i = xbs->here - xbs->header->xh_entries; + old_in_xb = 1; + + if (le16_to_cpu(xb->xb_flags) & OCFS2_XATTR_INDEXED) { + ret = ocfs2_xattr_bucket_get_name_value(inode->i_sb, + bucket_xh(xbs->bucket), + i, &block_off, + &name_offset); + base = bucket_block(xbs->bucket, block_off); + credits += ocfs2_blocks_per_xattr_bucket(inode->i_sb); + } else { + base = xbs->base; + credits += OCFS2_XATTR_BLOCK_UPDATE_CREDITS; + } + } + + /* + * delete a xattr doesn't need metadata and cluster allocation. + * so just calculate the credits and return. + * + * The credits for removing the value tree will be extended + * by ocfs2_remove_extent itself. + */ + if (!xi->xi_value) { + if (!ocfs2_xattr_is_local(xe)) + credits += ocfs2_remove_extent_credits(inode->i_sb); + + goto out; + } + + /* do cluster allocation guess first. */ + value_size = le64_to_cpu(xe->xe_value_size); + + if (old_in_xb) { + /* + * In xattr set, we always try to set the xe in inode first, + * so if it can be inserted into inode successfully, the old + * one will be removed from the xattr block, and this xattr + * will be inserted into inode as a new xattr in inode. + */ + if (ocfs2_xattr_can_be_in_inode(inode, xi, xis)) { + clusters_add += new_clusters; + credits += ocfs2_remove_extent_credits(inode->i_sb) + + OCFS2_INODE_UPDATE_CREDITS; + if (!ocfs2_xattr_is_local(xe)) + credits += ocfs2_calc_extend_credits( + inode->i_sb, + &def_xv.xv.xr_list); + goto out; + } + } + + if (xi->xi_value_len > OCFS2_XATTR_INLINE_SIZE) { + /* the new values will be stored outside. */ + u32 old_clusters = 0; + + if (!ocfs2_xattr_is_local(xe)) { + old_clusters = ocfs2_clusters_for_bytes(inode->i_sb, + value_size); + xv = (struct ocfs2_xattr_value_root *) + (base + name_offset + name_len); + value_size = OCFS2_XATTR_ROOT_SIZE; + } else + xv = &def_xv.xv; + + if (old_clusters >= new_clusters) { + credits += ocfs2_remove_extent_credits(inode->i_sb); + goto out; + } else { + meta_add += ocfs2_extend_meta_needed(&xv->xr_list); + clusters_add += new_clusters - old_clusters; + credits += ocfs2_calc_extend_credits(inode->i_sb, + &xv->xr_list); + if (value_size >= OCFS2_XATTR_ROOT_SIZE) + goto out; + } + } else { + /* + * Now the new value will be stored inside. So if the new + * value is smaller than the size of value root or the old + * value, we don't need any allocation, otherwise we have + * to guess metadata allocation. + */ + if ((ocfs2_xattr_is_local(xe) && + (value_size >= xi->xi_value_len)) || + (!ocfs2_xattr_is_local(xe) && + OCFS2_XATTR_ROOT_SIZE >= xi->xi_value_len)) + goto out; + } + +meta_guess: + /* calculate metadata allocation. */ + if (di->i_xattr_loc) { + if (!xbs->xattr_bh) { + ret = ocfs2_read_xattr_block(inode, + le64_to_cpu(di->i_xattr_loc), + &bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + xb = (struct ocfs2_xattr_block *)bh->b_data; + } else + xb = (struct ocfs2_xattr_block *)xbs->xattr_bh->b_data; + + /* + * If there is already an xattr tree, good, we can calculate + * like other b-trees. Otherwise we may have the chance of + * create a tree, the credit calculation is borrowed from + * ocfs2_calc_extend_credits with root_el = NULL. And the + * new tree will be cluster based, so no meta is needed. + */ + if (le16_to_cpu(xb->xb_flags) & OCFS2_XATTR_INDEXED) { + struct ocfs2_extent_list *el = + &xb->xb_attrs.xb_root.xt_list; + meta_add += ocfs2_extend_meta_needed(el); + credits += ocfs2_calc_extend_credits(inode->i_sb, + el); + } else + credits += OCFS2_SUBALLOC_ALLOC + 1; + + /* + * Reserve metadata for the new xattr's value extent tree. + * The not_found path above adds credits for this tree but + * omits meta_add, leaving meta_ac NULL for large values. + */ + if (xi->xi_value_len > OCFS2_XATTR_INLINE_SIZE) + meta_add += ocfs2_extend_meta_needed(&def_xv.xv.xr_list); + + /* + * This cluster will be used either for new bucket or for + * new xattr block. + * If the cluster size is the same as the bucket size, one + * more is needed since we may need to extend the bucket + * also. + */ + clusters_add += 1; + credits += ocfs2_blocks_per_xattr_bucket(inode->i_sb); + if (OCFS2_XATTR_BUCKET_SIZE == + OCFS2_SB(inode->i_sb)->s_clustersize) { + credits += ocfs2_blocks_per_xattr_bucket(inode->i_sb); + clusters_add += 1; + } + } else { + credits += OCFS2_XATTR_BLOCK_CREATE_CREDITS; + if (xi->xi_value_len > OCFS2_XATTR_INLINE_SIZE) { + struct ocfs2_extent_list *el = &def_xv.xv.xr_list; + meta_add += ocfs2_extend_meta_needed(el); + credits += ocfs2_calc_extend_credits(inode->i_sb, + el); + } else { + meta_add += 1; + } + } +out: + if (clusters_need) + *clusters_need = clusters_add; + if (meta_need) + *meta_need = meta_add; + if (credits_need) + *credits_need = credits; + brelse(bh); + return ret; +} + +static int ocfs2_init_xattr_set_ctxt(struct inode *inode, + struct ocfs2_dinode *di, + struct ocfs2_xattr_info *xi, + struct ocfs2_xattr_search *xis, + struct ocfs2_xattr_search *xbs, + struct ocfs2_xattr_set_ctxt *ctxt, + int extra_meta, + int *credits) +{ + int clusters_add, meta_add, ret; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + memset(ctxt, 0, sizeof(struct ocfs2_xattr_set_ctxt)); + + ocfs2_init_dealloc_ctxt(&ctxt->dealloc); + + ret = ocfs2_calc_xattr_set_need(inode, di, xi, xis, xbs, + &clusters_add, &meta_add, credits); + if (ret) { + mlog_errno(ret); + return ret; + } + + meta_add += extra_meta; + trace_ocfs2_init_xattr_set_ctxt(xi->xi_name, meta_add, + clusters_add, *credits); + + if (meta_add) { + ret = ocfs2_reserve_new_metadata_blocks(osb, meta_add, + &ctxt->meta_ac); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + if (clusters_add) { + ret = ocfs2_reserve_clusters(osb, clusters_add, &ctxt->data_ac); + if (ret) + mlog_errno(ret); + } +out: + if (ret) { + if (ctxt->meta_ac) { + ocfs2_free_alloc_context(ctxt->meta_ac); + ctxt->meta_ac = NULL; + } + + /* + * We cannot have an error and a non null ctxt->data_ac. + */ + } + + return ret; +} + +static int __ocfs2_xattr_set_handle(struct inode *inode, + struct ocfs2_dinode *di, + struct ocfs2_xattr_info *xi, + struct ocfs2_xattr_search *xis, + struct ocfs2_xattr_search *xbs, + struct ocfs2_xattr_set_ctxt *ctxt) +{ + int ret = 0, credits, old_found; + + if (!xi->xi_value) { + /* Remove existing extended attribute */ + if (!xis->not_found) + ret = ocfs2_xattr_ibody_set(inode, xi, xis, ctxt); + else if (!xbs->not_found) + ret = ocfs2_xattr_block_set(inode, xi, xbs, ctxt); + } else { + /* We always try to set extended attribute into inode first*/ + ret = ocfs2_xattr_ibody_set(inode, xi, xis, ctxt); + if (!ret && !xbs->not_found) { + /* + * If succeed and that extended attribute existing in + * external block, then we will remove it. + */ + xi->xi_value = NULL; + xi->xi_value_len = 0; + + old_found = xis->not_found; + xis->not_found = -ENODATA; + ret = ocfs2_calc_xattr_set_need(inode, + di, + xi, + xis, + xbs, + NULL, + NULL, + &credits); + xis->not_found = old_found; + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_extend_trans(ctxt->handle, credits); + if (ret) { + mlog_errno(ret); + goto out; + } + ret = ocfs2_xattr_block_set(inode, xi, xbs, ctxt); + } else if ((ret == -ENOSPC) && !ctxt->set_abort) { + if (di->i_xattr_loc && !xbs->xattr_bh) { + ret = ocfs2_xattr_block_find(inode, + xi->xi_name_index, + xi->xi_name, xbs); + if (ret) + goto out; + + old_found = xis->not_found; + xis->not_found = -ENODATA; + ret = ocfs2_calc_xattr_set_need(inode, + di, + xi, + xis, + xbs, + NULL, + NULL, + &credits); + xis->not_found = old_found; + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_extend_trans(ctxt->handle, credits); + if (ret) { + mlog_errno(ret); + goto out; + } + } + /* + * If no space in inode, we will set extended attribute + * into external block. + */ + ret = ocfs2_xattr_block_set(inode, xi, xbs, ctxt); + if (ret) + goto out; + if (!xis->not_found) { + /* + * If succeed and that extended attribute + * existing in inode, we will remove it. + */ + xi->xi_value = NULL; + xi->xi_value_len = 0; + xbs->not_found = -ENODATA; + ret = ocfs2_calc_xattr_set_need(inode, + di, + xi, + xis, + xbs, + NULL, + NULL, + &credits); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_extend_trans(ctxt->handle, credits); + if (ret) { + mlog_errno(ret); + goto out; + } + ret = ocfs2_xattr_ibody_set(inode, xi, + xis, ctxt); + } + } + } + + if (!ret) { + /* Update inode ctime. */ + ret = ocfs2_journal_access_di(ctxt->handle, INODE_CACHE(inode), + xis->inode_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + + inode_set_ctime_current(inode); + di->i_ctime = cpu_to_le64(inode_get_ctime_sec(inode)); + di->i_ctime_nsec = cpu_to_le32(inode_get_ctime_nsec(inode)); + ocfs2_journal_dirty(ctxt->handle, xis->inode_bh); + } +out: + return ret; +} + +/* + * This helper is only for setting initial ACL or security xattrs on an inode + * that is still unpublished, unhashed, and unattached to a dentry. + * Ordinary xattr updates must use ocfs2_xattr_set(). + * All transaction credits have been reserved in mknod or symlink callers. + */ +int ocfs2_xattr_set_handle(handle_t *handle, + struct inode *inode, + struct buffer_head *di_bh, + int name_index, + const char *name, + const void *value, + size_t value_len, + int flags, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_alloc_context *data_ac) +{ + struct ocfs2_dinode *di; + int ret; + + struct ocfs2_xattr_info xi = { + .xi_name_index = name_index, + .xi_name = name, + .xi_name_len = strlen(name), + .xi_value = value, + .xi_value_len = value_len, + }; + + struct ocfs2_xattr_search xis = { + .not_found = -ENODATA, + }; + + struct ocfs2_xattr_search xbs = { + .not_found = -ENODATA, + }; + + struct ocfs2_xattr_set_ctxt ctxt = { + .handle = handle, + .meta_ac = meta_ac, + .data_ac = data_ac, + }; + + if (!ocfs2_supports_xattr(OCFS2_SB(inode->i_sb))) + return -EOPNOTSUPP; + + /* + * In extreme situation, may need xattr bucket when + * block size is too small. And we have already reserved + * the credits for bucket in mknod. + */ + if (inode->i_sb->s_blocksize == OCFS2_MIN_BLOCKSIZE) { + xbs.bucket = ocfs2_xattr_bucket_new(inode); + if (!xbs.bucket) { + mlog_errno(-ENOMEM); + return -ENOMEM; + } + } + + xis.inode_bh = xbs.inode_bh = di_bh; + di = (struct ocfs2_dinode *)di_bh->b_data; + + ret = ocfs2_xattr_ibody_find(inode, name_index, name, &xis); + if (ret) + goto cleanup; + if (xis.not_found) { + ret = ocfs2_xattr_block_find(inode, name_index, name, &xbs); + if (ret) + goto cleanup; + } + + ret = __ocfs2_xattr_set_handle(inode, di, &xi, &xis, &xbs, &ctxt); + +cleanup: + brelse(xbs.xattr_bh); + ocfs2_xattr_bucket_free(xbs.bucket); + + return ret; +} + +/* + * ocfs2_xattr_set() + * + * Set, replace or remove an extended attribute for this inode. + * value is NULL to remove an existing extended attribute, else either + * create or replace an extended attribute. + */ +int ocfs2_xattr_set(struct inode *inode, + int name_index, + const char *name, + const void *value, + size_t value_len, + int flags) +{ + struct buffer_head *di_bh = NULL; + struct ocfs2_dinode *di; + int ret, credits, had_lock, ref_meta = 0, ref_credits = 0; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct inode *tl_inode = osb->osb_tl_inode; + struct ocfs2_xattr_set_ctxt ctxt = { NULL, NULL, NULL, }; + struct ocfs2_refcount_tree *ref_tree = NULL; + struct ocfs2_lock_holder oh; + + struct ocfs2_xattr_info xi = { + .xi_name_index = name_index, + .xi_name = name, + .xi_name_len = strlen(name), + .xi_value = value, + .xi_value_len = value_len, + }; + + struct ocfs2_xattr_search xis = { + .not_found = -ENODATA, + }; + + struct ocfs2_xattr_search xbs = { + .not_found = -ENODATA, + }; + + if (!ocfs2_supports_xattr(osb)) + return -EOPNOTSUPP; + + /* + * Only xbs will be used on indexed trees. xis doesn't need a + * bucket. + */ + xbs.bucket = ocfs2_xattr_bucket_new(inode); + if (!xbs.bucket) { + mlog_errno(-ENOMEM); + return -ENOMEM; + } + + had_lock = ocfs2_inode_lock_tracker(inode, &di_bh, 1, &oh); + if (had_lock < 0) { + ret = had_lock; + mlog_errno(ret); + goto cleanup_nolock; + } + xis.inode_bh = xbs.inode_bh = di_bh; + di = (struct ocfs2_dinode *)di_bh->b_data; + + down_write(&OCFS2_I(inode)->ip_xattr_sem); + /* + * Scan inode and external block to find the same name + * extended attribute and collect search information. + */ + ret = ocfs2_xattr_ibody_find(inode, name_index, name, &xis); + if (ret) + goto cleanup; + if (xis.not_found) { + ret = ocfs2_xattr_block_find(inode, name_index, name, &xbs); + if (ret) + goto cleanup; + } + + if (xis.not_found && xbs.not_found) { + ret = -ENODATA; + if (flags & XATTR_REPLACE) + goto cleanup; + ret = 0; + if (!value) + goto cleanup; + } else { + ret = -EEXIST; + if (flags & XATTR_CREATE) + goto cleanup; + } + + /* Check whether the value is refcounted and do some preparation. */ + if (ocfs2_is_refcount_inode(inode) && + (!xis.not_found || !xbs.not_found)) { + ret = ocfs2_prepare_refcount_xattr(inode, di, &xi, + &xis, &xbs, &ref_tree, + &ref_meta, &ref_credits); + if (ret) { + mlog_errno(ret); + goto cleanup; + } + } + + inode_lock(tl_inode); + + if (ocfs2_truncate_log_needs_flush(osb)) { + ret = __ocfs2_flush_truncate_log(osb); + if (ret < 0) { + inode_unlock(tl_inode); + mlog_errno(ret); + goto cleanup; + } + } + inode_unlock(tl_inode); + + ret = ocfs2_init_xattr_set_ctxt(inode, di, &xi, &xis, + &xbs, &ctxt, ref_meta, &credits); + if (ret) { + mlog_errno(ret); + goto cleanup; + } + + /* we need to update inode's ctime field, so add credit for it. */ + credits += OCFS2_INODE_UPDATE_CREDITS; + ctxt.handle = ocfs2_start_trans(osb, credits + ref_credits); + if (IS_ERR(ctxt.handle)) { + ret = PTR_ERR(ctxt.handle); + mlog_errno(ret); + goto out_free_ac; + } + + ret = __ocfs2_xattr_set_handle(inode, di, &xi, &xis, &xbs, &ctxt); + ocfs2_update_inode_fsync_trans(ctxt.handle, inode, 0); + + ocfs2_commit_trans(osb, ctxt.handle); + +out_free_ac: + if (ctxt.data_ac) + ocfs2_free_alloc_context(ctxt.data_ac); + if (ctxt.meta_ac) + ocfs2_free_alloc_context(ctxt.meta_ac); + if (ocfs2_dealloc_has_cluster(&ctxt.dealloc)) + ocfs2_schedule_truncate_log_flush(osb, 1); + ocfs2_run_deallocs(osb, &ctxt.dealloc); + +cleanup: + if (ref_tree) + ocfs2_unlock_refcount_tree(osb, ref_tree, 1); + up_write(&OCFS2_I(inode)->ip_xattr_sem); + if (!value && !ret) { + ret = ocfs2_try_remove_refcount_tree(inode, di_bh); + if (ret) + mlog_errno(ret); + } + ocfs2_inode_unlock_tracker(inode, 1, &oh, had_lock); +cleanup_nolock: + brelse(di_bh); + brelse(xbs.xattr_bh); + ocfs2_xattr_bucket_free(xbs.bucket); + + return ret; +} + +/* + * Find the xattr extent rec which may contains name_hash. + * e_cpos will be the first name hash of the xattr rec. + * el must be the ocfs2_xattr_header.xb_attrs.xb_root.xt_list. + */ +static int ocfs2_xattr_get_rec(struct inode *inode, + u32 name_hash, + u64 *p_blkno, + u32 *e_cpos, + u32 *num_clusters, + struct ocfs2_extent_list *el) +{ + int ret = 0, i; + struct buffer_head *eb_bh = NULL; + struct ocfs2_extent_block *eb; + struct ocfs2_extent_rec *rec = NULL; + u64 e_blkno = 0; + + if (el->l_tree_depth) { + ret = ocfs2_find_leaf(INODE_CACHE(inode), el, name_hash, + &eb_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + eb = (struct ocfs2_extent_block *) eb_bh->b_data; + el = &eb->h_list; + + if (el->l_tree_depth) { + ret = ocfs2_error(inode->i_sb, + "Inode %llu has non zero tree depth in xattr tree block %llu\n", + inode->i_ino, + (unsigned long long)eb_bh->b_blocknr); + goto out; + } + } + + for (i = le16_to_cpu(el->l_next_free_rec) - 1; i >= 0; i--) { + rec = &el->l_recs[i]; + + if (le32_to_cpu(rec->e_cpos) <= name_hash) { + e_blkno = le64_to_cpu(rec->e_blkno); + break; + } + } + + if (!e_blkno) { + ret = ocfs2_error(inode->i_sb, "Inode %llu has bad extent record (%u, %u, 0) in xattr\n", + inode->i_ino, + le32_to_cpu(rec->e_cpos), + ocfs2_rec_clusters(el, rec)); + goto out; + } + + *p_blkno = le64_to_cpu(rec->e_blkno); + *num_clusters = le16_to_cpu(rec->e_leaf_clusters); + if (e_cpos) + *e_cpos = le32_to_cpu(rec->e_cpos); +out: + brelse(eb_bh); + return ret; +} + +typedef int (xattr_bucket_func)(struct inode *inode, + struct ocfs2_xattr_bucket *bucket, + void *para); + +static int ocfs2_find_xe_in_bucket(struct inode *inode, + struct ocfs2_xattr_bucket *bucket, + int name_index, + const char *name, + u32 name_hash, + u16 *xe_index, + int *found) +{ + int i, ret = 0, cmp = 1, block_off, new_offset; + struct ocfs2_xattr_header *xh = bucket_xh(bucket); + size_t name_len = strlen(name); + struct ocfs2_xattr_entry *xe = NULL; + char *xe_name; + + /* + * We don't use binary search in the bucket because there + * may be multiple entries with the same name hash. + */ + for (i = 0; i < le16_to_cpu(xh->xh_count); i++) { + xe = &xh->xh_entries[i]; + + if (name_hash > le32_to_cpu(xe->xe_name_hash)) + continue; + else if (name_hash < le32_to_cpu(xe->xe_name_hash)) + break; + + cmp = name_index - ocfs2_xattr_get_type(xe); + if (!cmp) + cmp = name_len - xe->xe_name_len; + if (cmp) + continue; + + ret = ocfs2_xattr_bucket_get_name_value(inode->i_sb, + xh, + i, + &block_off, + &new_offset); + if (ret) { + mlog_errno(ret); + break; + } + + + xe_name = bucket_block(bucket, block_off) + new_offset; + if (!memcmp(name, xe_name, name_len)) { + *xe_index = i; + *found = 1; + ret = 0; + break; + } + } + + return ret; +} + +/* + * Find the specified xattr entry in a series of buckets. + * This series start from p_blkno and last for num_clusters. + * The ocfs2_xattr_header.xh_num_buckets of the first bucket contains + * the num of the valid buckets. + * + * Return the buffer_head this xattr should reside in. And if the xattr's + * hash is in the gap of 2 buckets, return the lower bucket. + */ +static int ocfs2_xattr_bucket_find(struct inode *inode, + int name_index, + const char *name, + u32 name_hash, + u64 p_blkno, + u32 first_hash, + u32 num_clusters, + struct ocfs2_xattr_search *xs) +{ + int ret, found = 0; + struct ocfs2_xattr_header *xh = NULL; + struct ocfs2_xattr_entry *xe = NULL; + u16 index = 0; + u16 blk_per_bucket = ocfs2_blocks_per_xattr_bucket(inode->i_sb); + int low_bucket = 0, bucket, high_bucket; + struct ocfs2_xattr_bucket *search; + u64 blkno, lower_blkno = 0; + + search = ocfs2_xattr_bucket_new(inode); + if (!search) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + ret = ocfs2_read_xattr_bucket(search, p_blkno); + if (ret) { + mlog_errno(ret); + goto out; + } + + xh = bucket_xh(search); + high_bucket = le16_to_cpu(xh->xh_num_buckets) - 1; + while (low_bucket <= high_bucket) { + ocfs2_xattr_bucket_relse(search); + + bucket = (low_bucket + high_bucket) / 2; + blkno = p_blkno + bucket * blk_per_bucket; + ret = ocfs2_read_xattr_bucket(search, blkno); + if (ret) { + mlog_errno(ret); + goto out; + } + + xh = bucket_xh(search); + xe = &xh->xh_entries[0]; + if (name_hash < le32_to_cpu(xe->xe_name_hash)) { + high_bucket = bucket - 1; + continue; + } + + /* + * Check whether the hash of the last entry in our + * bucket is larger than the search one. for an empty + * bucket, the last one is also the first one. + */ + if (xh->xh_count) + xe = &xh->xh_entries[le16_to_cpu(xh->xh_count) - 1]; + + /* record lower_blkno which may be the insert place. */ + lower_blkno = blkno; + + if (name_hash > le32_to_cpu(xe->xe_name_hash)) { + low_bucket = bucket + 1; + continue; + } + + /* the searched xattr should reside in this bucket if exists. */ + ret = ocfs2_find_xe_in_bucket(inode, search, + name_index, name, name_hash, + &index, &found); + if (ret) { + mlog_errno(ret); + goto out; + } + break; + } + + /* + * Record the bucket we have found. + * When the xattr's hash value is in the gap of 2 buckets, we will + * always set it to the previous bucket. + */ + if (!lower_blkno) + lower_blkno = p_blkno; + + /* This should be in cache - we just read it during the search */ + ret = ocfs2_read_xattr_bucket(xs->bucket, lower_blkno); + if (ret) { + mlog_errno(ret); + goto out; + } + + xs->header = bucket_xh(xs->bucket); + xs->base = bucket_block(xs->bucket, 0); + xs->end = xs->base + inode->i_sb->s_blocksize; + + if (found) { + xs->here = &xs->header->xh_entries[index]; + trace_ocfs2_xattr_bucket_find(OCFS2_I(inode)->ip_blkno, + name, name_index, name_hash, + (unsigned long long)bucket_blkno(xs->bucket), + index); + } else + ret = -ENODATA; + +out: + ocfs2_xattr_bucket_free(search); + return ret; +} + +static int ocfs2_xattr_index_block_find(struct inode *inode, + struct buffer_head *root_bh, + int name_index, + const char *name, + struct ocfs2_xattr_search *xs) +{ + int ret; + struct ocfs2_xattr_block *xb = + (struct ocfs2_xattr_block *)root_bh->b_data; + struct ocfs2_xattr_tree_root *xb_root = &xb->xb_attrs.xb_root; + struct ocfs2_extent_list *el = &xb_root->xt_list; + u64 p_blkno = 0; + u32 first_hash, num_clusters = 0; + u32 name_hash = ocfs2_xattr_name_hash(inode, name, strlen(name)); + + if (le16_to_cpu(el->l_next_free_rec) == 0) + return -ENODATA; + + trace_ocfs2_xattr_index_block_find(OCFS2_I(inode)->ip_blkno, + name, name_index, name_hash, + (unsigned long long)root_bh->b_blocknr, + -1); + + ret = ocfs2_xattr_get_rec(inode, name_hash, &p_blkno, &first_hash, + &num_clusters, el); + if (ret) { + mlog_errno(ret); + goto out; + } + + BUG_ON(p_blkno == 0 || num_clusters == 0 || first_hash > name_hash); + + trace_ocfs2_xattr_index_block_find_rec(OCFS2_I(inode)->ip_blkno, + name, name_index, first_hash, + (unsigned long long)p_blkno, + num_clusters); + + ret = ocfs2_xattr_bucket_find(inode, name_index, name, name_hash, + p_blkno, first_hash, num_clusters, xs); + +out: + return ret; +} + +static int ocfs2_iterate_xattr_buckets(struct inode *inode, + u64 blkno, + u32 clusters, + xattr_bucket_func *func, + void *para) +{ + int i, ret = 0; + u32 bpc = ocfs2_xattr_buckets_per_cluster(OCFS2_SB(inode->i_sb)); + u32 num_buckets = clusters * bpc; + struct ocfs2_xattr_bucket *bucket; + + bucket = ocfs2_xattr_bucket_new(inode); + if (!bucket) { + mlog_errno(-ENOMEM); + return -ENOMEM; + } + + trace_ocfs2_iterate_xattr_buckets( + (unsigned long long)OCFS2_I(inode)->ip_blkno, + (unsigned long long)blkno, clusters); + + for (i = 0; i < num_buckets; i++, blkno += bucket->bu_blocks) { + ret = ocfs2_read_xattr_bucket(bucket, blkno); + if (ret) { + mlog_errno(ret); + break; + } + + /* + * The real bucket num in this series of blocks is stored + * in the 1st bucket. + */ + if (i == 0) + num_buckets = le16_to_cpu(bucket_xh(bucket)->xh_num_buckets); + + trace_ocfs2_iterate_xattr_bucket((unsigned long long)blkno, + le32_to_cpu(bucket_xh(bucket)->xh_entries[0].xe_name_hash)); + if (func) { + ret = func(inode, bucket, para); + if (ret && ret != -ERANGE) + mlog_errno(ret); + /* Fall through to bucket_relse() */ + } + + ocfs2_xattr_bucket_relse(bucket); + if (ret) + break; + } + + ocfs2_xattr_bucket_free(bucket); + return ret; +} + +struct ocfs2_xattr_tree_list { + char *buffer; + size_t buffer_size; + size_t result; +}; + +static int ocfs2_xattr_bucket_get_name_value(struct super_block *sb, + struct ocfs2_xattr_header *xh, + int index, + int *block_off, + int *new_offset) +{ + u16 name_offset; + + if (index < 0 || index >= le16_to_cpu(xh->xh_count)) + return -EINVAL; + + name_offset = le16_to_cpu(xh->xh_entries[index].xe_name_offset); + + *block_off = name_offset >> sb->s_blocksize_bits; + *new_offset = name_offset % sb->s_blocksize; + + return 0; +} + +static int ocfs2_list_xattr_bucket(struct inode *inode, + struct ocfs2_xattr_bucket *bucket, + void *para) +{ + int ret = 0, type; + struct ocfs2_xattr_tree_list *xl = (struct ocfs2_xattr_tree_list *)para; + int i, block_off, new_offset; + const char *name; + + for (i = 0 ; i < le16_to_cpu(bucket_xh(bucket)->xh_count); i++) { + struct ocfs2_xattr_entry *entry = &bucket_xh(bucket)->xh_entries[i]; + type = ocfs2_xattr_get_type(entry); + + ret = ocfs2_xattr_bucket_get_name_value(inode->i_sb, + bucket_xh(bucket), + i, + &block_off, + &new_offset); + if (ret) + break; + + name = (const char *)bucket_block(bucket, block_off) + + new_offset; + ret = ocfs2_xattr_list_entry(inode->i_sb, + xl->buffer, + xl->buffer_size, + &xl->result, + type, name, + entry->xe_name_len); + if (ret) + break; + } + + return ret; +} + +static int ocfs2_iterate_xattr_index_block(struct inode *inode, + struct buffer_head *blk_bh, + xattr_tree_rec_func *rec_func, + void *para) +{ + struct ocfs2_xattr_block *xb = + (struct ocfs2_xattr_block *)blk_bh->b_data; + struct ocfs2_extent_list *el = &xb->xb_attrs.xb_root.xt_list; + int ret = 0; + u32 name_hash = UINT_MAX, e_cpos = 0, num_clusters = 0; + u64 p_blkno = 0; + + if (!el->l_next_free_rec || !rec_func) + return 0; + + while (name_hash > 0) { + ret = ocfs2_xattr_get_rec(inode, name_hash, &p_blkno, + &e_cpos, &num_clusters, el); + if (ret) { + mlog_errno(ret); + break; + } + + ret = rec_func(inode, blk_bh, p_blkno, e_cpos, + num_clusters, para); + if (ret) { + if (ret != -ERANGE) + mlog_errno(ret); + break; + } + + if (e_cpos == 0) + break; + + name_hash = e_cpos - 1; + } + + return ret; + +} + +static int ocfs2_list_xattr_tree_rec(struct inode *inode, + struct buffer_head *root_bh, + u64 blkno, u32 cpos, u32 len, void *para) +{ + return ocfs2_iterate_xattr_buckets(inode, blkno, len, + ocfs2_list_xattr_bucket, para); +} + +static int ocfs2_xattr_tree_list_index_block(struct inode *inode, + struct buffer_head *blk_bh, + char *buffer, + size_t buffer_size) +{ + int ret; + struct ocfs2_xattr_tree_list xl = { + .buffer = buffer, + .buffer_size = buffer_size, + .result = 0, + }; + + ret = ocfs2_iterate_xattr_index_block(inode, blk_bh, + ocfs2_list_xattr_tree_rec, &xl); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = xl.result; +out: + return ret; +} + +static int cmp_xe(const void *a, const void *b) +{ + const struct ocfs2_xattr_entry *l = a, *r = b; + u32 l_hash = le32_to_cpu(l->xe_name_hash); + u32 r_hash = le32_to_cpu(r->xe_name_hash); + + if (l_hash > r_hash) + return 1; + if (l_hash < r_hash) + return -1; + return 0; +} + +/* + * When the ocfs2_xattr_block is filled up, new bucket will be created + * and all the xattr entries will be moved to the new bucket. + * The header goes at the start of the bucket, and the names+values are + * filled from the end. This is why *target starts as the last buffer. + * Note: we need to sort the entries since they are not saved in order + * in the ocfs2_xattr_block. + */ +static void ocfs2_cp_xattr_block_to_bucket(struct inode *inode, + struct buffer_head *xb_bh, + struct ocfs2_xattr_bucket *bucket) +{ + int i, blocksize = inode->i_sb->s_blocksize; + int blks = ocfs2_blocks_per_xattr_bucket(inode->i_sb); + u16 offset, size, off_change; + struct ocfs2_xattr_entry *xe; + struct ocfs2_xattr_block *xb = + (struct ocfs2_xattr_block *)xb_bh->b_data; + struct ocfs2_xattr_header *xb_xh = &xb->xb_attrs.xb_header; + struct ocfs2_xattr_header *xh = bucket_xh(bucket); + u16 count = le16_to_cpu(xb_xh->xh_count); + char *src = xb_bh->b_data; + char *target = bucket_block(bucket, blks - 1); + + trace_ocfs2_cp_xattr_block_to_bucket_begin( + (unsigned long long)xb_bh->b_blocknr, + (unsigned long long)bucket_blkno(bucket)); + + for (i = 0; i < blks; i++) + memset(bucket_block(bucket, i), 0, blocksize); + + /* + * Since the xe_name_offset is based on ocfs2_xattr_header, + * there is a offset change corresponding to the change of + * ocfs2_xattr_header's position. + */ + off_change = offsetof(struct ocfs2_xattr_block, xb_attrs.xb_header); + xe = &xb_xh->xh_entries[count - 1]; + offset = le16_to_cpu(xe->xe_name_offset) + off_change; + size = blocksize - offset; + + /* copy all the names and values. */ + memcpy(target + offset, src + offset, size); + + /* Init new header now. */ + xh->xh_count = xb_xh->xh_count; + xh->xh_num_buckets = cpu_to_le16(1); + xh->xh_name_value_len = cpu_to_le16(size); + xh->xh_free_start = cpu_to_le16(OCFS2_XATTR_BUCKET_SIZE - size); + + /* copy all the entries. */ + target = bucket_block(bucket, 0); + offset = offsetof(struct ocfs2_xattr_header, xh_entries); + size = count * sizeof(struct ocfs2_xattr_entry); + memcpy(target + offset, (char *)xb_xh + offset, size); + + /* Change the xe offset for all the xe because of the move. */ + off_change = OCFS2_XATTR_BUCKET_SIZE - blocksize + + offsetof(struct ocfs2_xattr_block, xb_attrs.xb_header); + for (i = 0; i < count; i++) + le16_add_cpu(&xh->xh_entries[i].xe_name_offset, off_change); + + trace_ocfs2_cp_xattr_block_to_bucket_end(offset, size, off_change); + + sort(target + offset, count, sizeof(struct ocfs2_xattr_entry), + cmp_xe, NULL); +} + +/* + * After we move xattr from block to index btree, we have to + * update ocfs2_xattr_search to the new xe and base. + * + * When the entry is in xattr block, xattr_bh indicates the storage place. + * While if the entry is in index b-tree, "bucket" indicates the + * real place of the xattr. + */ +static void ocfs2_xattr_update_xattr_search(struct inode *inode, + struct ocfs2_xattr_search *xs, + struct buffer_head *old_bh) +{ + char *buf = old_bh->b_data; + struct ocfs2_xattr_block *old_xb = (struct ocfs2_xattr_block *)buf; + struct ocfs2_xattr_header *old_xh = &old_xb->xb_attrs.xb_header; + int i; + + xs->header = bucket_xh(xs->bucket); + xs->base = bucket_block(xs->bucket, 0); + xs->end = xs->base + inode->i_sb->s_blocksize; + + if (xs->not_found) + return; + + i = xs->here - old_xh->xh_entries; + xs->here = &xs->header->xh_entries[i]; +} + +static int ocfs2_xattr_create_index_block(struct inode *inode, + struct ocfs2_xattr_search *xs, + struct ocfs2_xattr_set_ctxt *ctxt) +{ + int ret; + u32 bit_off, len; + u64 blkno; + handle_t *handle = ctxt->handle; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct buffer_head *xb_bh = xs->xattr_bh; + struct ocfs2_xattr_block *xb = + (struct ocfs2_xattr_block *)xb_bh->b_data; + struct ocfs2_xattr_tree_root *xr; + u16 xb_flags = le16_to_cpu(xb->xb_flags); + + trace_ocfs2_xattr_create_index_block_begin( + (unsigned long long)xb_bh->b_blocknr); + + BUG_ON(xb_flags & OCFS2_XATTR_INDEXED); + BUG_ON(!xs->bucket); + + /* + * XXX: + * We can use this lock for now, and maybe move to a dedicated mutex + * if performance becomes a problem later. + */ + down_write(&oi->ip_alloc_sem); + + ret = ocfs2_journal_access_xb(handle, INODE_CACHE(inode), xb_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = __ocfs2_claim_clusters(handle, ctxt->data_ac, + 1, 1, &bit_off, &len); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * The bucket may spread in many blocks, and + * we will only touch the 1st block and the last block + * in the whole bucket(one for entry and one for data). + */ + blkno = ocfs2_clusters_to_blocks(inode->i_sb, bit_off); + + trace_ocfs2_xattr_create_index_block((unsigned long long)blkno); + + ret = ocfs2_init_xattr_bucket(xs->bucket, blkno, 1); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_xattr_bucket_journal_access(handle, xs->bucket, + OCFS2_JOURNAL_ACCESS_CREATE); + if (ret) { + mlog_errno(ret); + goto out; + } + + ocfs2_cp_xattr_block_to_bucket(inode, xb_bh, xs->bucket); + ocfs2_xattr_bucket_journal_dirty(handle, xs->bucket); + + ocfs2_xattr_update_xattr_search(inode, xs, xb_bh); + + /* Change from ocfs2_xattr_header to ocfs2_xattr_tree_root */ + memset(&xb->xb_attrs, 0, inode->i_sb->s_blocksize - + offsetof(struct ocfs2_xattr_block, xb_attrs)); + + xr = &xb->xb_attrs.xb_root; + xr->xt_clusters = cpu_to_le32(1); + xr->xt_last_eb_blk = 0; + xr->xt_list.l_tree_depth = 0; + xr->xt_list.l_count = cpu_to_le16(ocfs2_xattr_recs_per_xb(inode->i_sb)); + xr->xt_list.l_next_free_rec = cpu_to_le16(1); + + xr->xt_list.l_recs[0].e_cpos = 0; + xr->xt_list.l_recs[0].e_blkno = cpu_to_le64(blkno); + xr->xt_list.l_recs[0].e_leaf_clusters = cpu_to_le16(1); + + xb->xb_flags = cpu_to_le16(xb_flags | OCFS2_XATTR_INDEXED); + + ocfs2_journal_dirty(handle, xb_bh); + +out: + up_write(&oi->ip_alloc_sem); + + return ret; +} + +static int cmp_xe_offset(const void *a, const void *b) +{ + const struct ocfs2_xattr_entry *l = a, *r = b; + u32 l_name_offset = le16_to_cpu(l->xe_name_offset); + u32 r_name_offset = le16_to_cpu(r->xe_name_offset); + + if (l_name_offset < r_name_offset) + return 1; + if (l_name_offset > r_name_offset) + return -1; + return 0; +} + +/* + * defrag a xattr bucket if we find that the bucket has some + * holes between name/value pairs. + * We will move all the name/value pairs to the end of the bucket + * so that we can spare some space for insertion. + */ +static int ocfs2_defrag_xattr_bucket(struct inode *inode, + handle_t *handle, + struct ocfs2_xattr_bucket *bucket) +{ + int ret, i; + size_t end, offset, len; + struct ocfs2_xattr_header *xh; + char *entries, *buf, *bucket_buf = NULL; + u64 blkno = bucket_blkno(bucket); + u16 xh_free_start; + size_t blocksize = inode->i_sb->s_blocksize; + struct ocfs2_xattr_entry *xe; + + /* + * In order to make the operation more efficient and generic, + * we copy all the blocks into a contiguous memory and do the + * defragment there, so if anything is error, we will not touch + * the real block. + */ + bucket_buf = kmalloc(OCFS2_XATTR_BUCKET_SIZE, GFP_NOFS); + if (!bucket_buf) { + ret = -EIO; + goto out; + } + + buf = bucket_buf; + for (i = 0; i < bucket->bu_blocks; i++, buf += blocksize) + memcpy(buf, bucket_block(bucket, i), blocksize); + + ret = ocfs2_xattr_bucket_journal_access(handle, bucket, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + xh = (struct ocfs2_xattr_header *)bucket_buf; + entries = (char *)xh->xh_entries; + xh_free_start = le16_to_cpu(xh->xh_free_start); + + trace_ocfs2_defrag_xattr_bucket( + (unsigned long long)blkno, le16_to_cpu(xh->xh_count), + xh_free_start, le16_to_cpu(xh->xh_name_value_len)); + + /* + * sort all the entries by their offset. + * the largest will be the first, so that we can + * move them to the end one by one. + */ + sort(entries, le16_to_cpu(xh->xh_count), + sizeof(struct ocfs2_xattr_entry), + cmp_xe_offset, NULL); + + /* Move all name/values to the end of the bucket. */ + xe = xh->xh_entries; + end = OCFS2_XATTR_BUCKET_SIZE; + for (i = 0; i < le16_to_cpu(xh->xh_count); i++, xe++) { + offset = le16_to_cpu(xe->xe_name_offset); + len = namevalue_size_xe(xe); + + /* + * We must make sure that the name/value pair + * exist in the same block. So adjust end to + * the previous block end if needed. + */ + if (((end - len) / blocksize != + (end - 1) / blocksize)) + end = end - end % blocksize; + + if (end > offset + len) { + memmove(bucket_buf + end - len, + bucket_buf + offset, len); + xe->xe_name_offset = cpu_to_le16(end - len); + } + + mlog_bug_on_msg(end < offset + len, "Defrag check failed for " + "bucket %llu\n", (unsigned long long)blkno); + + end -= len; + } + + mlog_bug_on_msg(xh_free_start > end, "Defrag check failed for " + "bucket %llu\n", (unsigned long long)blkno); + + if (xh_free_start == end) + goto out; + + memset(bucket_buf + xh_free_start, 0, end - xh_free_start); + xh->xh_free_start = cpu_to_le16(end); + + /* sort the entries by their name_hash. */ + sort(entries, le16_to_cpu(xh->xh_count), + sizeof(struct ocfs2_xattr_entry), + cmp_xe, NULL); + + buf = bucket_buf; + for (i = 0; i < bucket->bu_blocks; i++, buf += blocksize) + memcpy(bucket_block(bucket, i), buf, blocksize); + ocfs2_xattr_bucket_journal_dirty(handle, bucket); + +out: + kfree(bucket_buf); + return ret; +} + +/* + * prev_blkno points to the start of an existing extent. new_blkno + * points to a newly allocated extent. Because we know each of our + * clusters contains more than bucket, we can easily split one cluster + * at a bucket boundary. So we take the last cluster of the existing + * extent and split it down the middle. We move the last half of the + * buckets in the last cluster of the existing extent over to the new + * extent. + * + * first_bh is the buffer at prev_blkno so we can update the existing + * extent's bucket count. header_bh is the bucket were we were hoping + * to insert our xattr. If the bucket move places the target in the new + * extent, we'll update first_bh and header_bh after modifying the old + * extent. + * + * first_hash will be set as the 1st xe's name_hash in the new extent. + */ +static int ocfs2_mv_xattr_bucket_cross_cluster(struct inode *inode, + handle_t *handle, + struct ocfs2_xattr_bucket *first, + struct ocfs2_xattr_bucket *target, + u64 new_blkno, + u32 num_clusters, + u32 *first_hash) +{ + int ret; + struct super_block *sb = inode->i_sb; + int blks_per_bucket = ocfs2_blocks_per_xattr_bucket(sb); + int num_buckets = ocfs2_xattr_buckets_per_cluster(OCFS2_SB(sb)); + int to_move = num_buckets / 2; + u64 src_blkno; + u64 last_cluster_blkno = bucket_blkno(first) + + ((num_clusters - 1) * ocfs2_clusters_to_blocks(sb, 1)); + + BUG_ON(le16_to_cpu(bucket_xh(first)->xh_num_buckets) < num_buckets); + BUG_ON(OCFS2_XATTR_BUCKET_SIZE == OCFS2_SB(sb)->s_clustersize); + + trace_ocfs2_mv_xattr_bucket_cross_cluster( + (unsigned long long)last_cluster_blkno, + (unsigned long long)new_blkno); + + ret = ocfs2_mv_xattr_buckets(inode, handle, bucket_blkno(first), + last_cluster_blkno, new_blkno, + to_move, first_hash); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* This is the first bucket that got moved */ + src_blkno = last_cluster_blkno + (to_move * blks_per_bucket); + + /* + * If the target bucket was part of the moved buckets, we need to + * update first and target. + */ + if (bucket_blkno(target) >= src_blkno) { + /* Find the block for the new target bucket */ + src_blkno = new_blkno + + (bucket_blkno(target) - src_blkno); + + ocfs2_xattr_bucket_relse(first); + ocfs2_xattr_bucket_relse(target); + + /* + * These shouldn't fail - the buffers are in the + * journal from ocfs2_cp_xattr_bucket(). + */ + ret = ocfs2_read_xattr_bucket(first, new_blkno); + if (ret) { + mlog_errno(ret); + goto out; + } + ret = ocfs2_read_xattr_bucket(target, src_blkno); + if (ret) + mlog_errno(ret); + + } + +out: + return ret; +} + +/* + * Find the suitable pos when we divide a bucket into 2. + * We have to make sure the xattrs with the same hash value exist + * in the same bucket. + * + * If this ocfs2_xattr_header covers more than one hash value, find a + * place where the hash value changes. Try to find the most even split. + * The most common case is that all entries have different hash values, + * and the first check we make will find a place to split. + */ +static int ocfs2_xattr_find_divide_pos(struct ocfs2_xattr_header *xh) +{ + struct ocfs2_xattr_entry *entries = xh->xh_entries; + int count = le16_to_cpu(xh->xh_count); + int delta, middle = count / 2; + + /* + * We start at the middle. Each step gets farther away in both + * directions. We therefore hit the change in hash value + * nearest to the middle. Note that this loop does not execute for + * count < 2. + */ + for (delta = 0; delta < middle; delta++) { + /* Let's check delta earlier than middle */ + if (cmp_xe(&entries[middle - delta - 1], + &entries[middle - delta])) + return middle - delta; + + /* For even counts, don't walk off the end */ + if ((middle + delta + 1) == count) + continue; + + /* Now try delta past middle */ + if (cmp_xe(&entries[middle + delta], + &entries[middle + delta + 1])) + return middle + delta + 1; + } + + /* Every entry had the same hash */ + return count; +} + +/* + * Move some xattrs in old bucket(blk) to new bucket(new_blk). + * first_hash will record the 1st hash of the new bucket. + * + * Normally half of the xattrs will be moved. But we have to make + * sure that the xattrs with the same hash value are stored in the + * same bucket. If all the xattrs in this bucket have the same hash + * value, the new bucket will be initialized as an empty one and the + * first_hash will be initialized as (hash_value+1). + */ +static int ocfs2_divide_xattr_bucket(struct inode *inode, + handle_t *handle, + u64 blk, + u64 new_blk, + u32 *first_hash, + int new_bucket_head) +{ + int ret, i; + int count, start, len, name_value_len = 0, name_offset = 0; + struct ocfs2_xattr_bucket *s_bucket = NULL, *t_bucket = NULL; + struct ocfs2_xattr_header *xh; + struct ocfs2_xattr_entry *xe; + int blocksize = inode->i_sb->s_blocksize; + + trace_ocfs2_divide_xattr_bucket_begin((unsigned long long)blk, + (unsigned long long)new_blk); + + s_bucket = ocfs2_xattr_bucket_new(inode); + t_bucket = ocfs2_xattr_bucket_new(inode); + if (!s_bucket || !t_bucket) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + ret = ocfs2_read_xattr_bucket(s_bucket, blk); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_xattr_bucket_journal_access(handle, s_bucket, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * Even if !new_bucket_head, we're overwriting t_bucket. Thus, + * there's no need to read it. + */ + ret = ocfs2_init_xattr_bucket(t_bucket, new_blk, new_bucket_head); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * Hey, if we're overwriting t_bucket, what difference does + * ACCESS_CREATE vs ACCESS_WRITE make? See the comment in the + * same part of ocfs2_cp_xattr_bucket(). + */ + ret = ocfs2_xattr_bucket_journal_access(handle, t_bucket, + new_bucket_head ? + OCFS2_JOURNAL_ACCESS_CREATE : + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + + xh = bucket_xh(s_bucket); + count = le16_to_cpu(xh->xh_count); + start = ocfs2_xattr_find_divide_pos(xh); + + if (start == count) { + xe = &xh->xh_entries[start-1]; + + /* + * initialized a new empty bucket here. + * The hash value is set as one larger than + * that of the last entry in the previous bucket. + */ + for (i = 0; i < t_bucket->bu_blocks; i++) + memset(bucket_block(t_bucket, i), 0, blocksize); + + xh = bucket_xh(t_bucket); + xh->xh_free_start = cpu_to_le16(blocksize); + xh->xh_entries[0].xe_name_hash = xe->xe_name_hash; + le32_add_cpu(&xh->xh_entries[0].xe_name_hash, 1); + + goto set_num_buckets; + } + + /* copy the whole bucket to the new first. */ + ocfs2_xattr_bucket_copy_data(t_bucket, s_bucket); + + /* update the new bucket. */ + xh = bucket_xh(t_bucket); + + /* + * Calculate the total name/value len and xh_free_start for + * the old bucket first. + */ + name_offset = OCFS2_XATTR_BUCKET_SIZE; + name_value_len = 0; + for (i = 0; i < start; i++) { + xe = &xh->xh_entries[i]; + name_value_len += namevalue_size_xe(xe); + if (le16_to_cpu(xe->xe_name_offset) < name_offset) + name_offset = le16_to_cpu(xe->xe_name_offset); + } + + /* + * Now begin the modification to the new bucket. + * + * In the new bucket, We just move the xattr entry to the beginning + * and don't touch the name/value. So there will be some holes in the + * bucket, and they will be removed when ocfs2_defrag_xattr_bucket is + * called. + */ + xe = &xh->xh_entries[start]; + len = sizeof(struct ocfs2_xattr_entry) * (count - start); + trace_ocfs2_divide_xattr_bucket_move(len, + (int)((char *)xe - (char *)xh), + (int)((char *)xh->xh_entries - (char *)xh)); + memmove((char *)xh->xh_entries, (char *)xe, len); + xe = &xh->xh_entries[count - start]; + len = sizeof(struct ocfs2_xattr_entry) * start; + memset((char *)xe, 0, len); + + le16_add_cpu(&xh->xh_count, -start); + le16_add_cpu(&xh->xh_name_value_len, -name_value_len); + + /* Calculate xh_free_start for the new bucket. */ + xh->xh_free_start = cpu_to_le16(OCFS2_XATTR_BUCKET_SIZE); + for (i = 0; i < le16_to_cpu(xh->xh_count); i++) { + xe = &xh->xh_entries[i]; + if (le16_to_cpu(xe->xe_name_offset) < + le16_to_cpu(xh->xh_free_start)) + xh->xh_free_start = xe->xe_name_offset; + } + +set_num_buckets: + /* set xh->xh_num_buckets for the new xh. */ + if (new_bucket_head) + xh->xh_num_buckets = cpu_to_le16(1); + else + xh->xh_num_buckets = 0; + + ocfs2_xattr_bucket_journal_dirty(handle, t_bucket); + + /* store the first_hash of the new bucket. */ + if (first_hash) + *first_hash = le32_to_cpu(xh->xh_entries[0].xe_name_hash); + + /* + * Now only update the 1st block of the old bucket. If we + * just added a new empty bucket, there is no need to modify + * it. + */ + if (start == count) + goto out; + + xh = bucket_xh(s_bucket); + memset(&xh->xh_entries[start], 0, + sizeof(struct ocfs2_xattr_entry) * (count - start)); + xh->xh_count = cpu_to_le16(start); + xh->xh_free_start = cpu_to_le16(name_offset); + xh->xh_name_value_len = cpu_to_le16(name_value_len); + + ocfs2_xattr_bucket_journal_dirty(handle, s_bucket); + +out: + ocfs2_xattr_bucket_free(s_bucket); + ocfs2_xattr_bucket_free(t_bucket); + + return ret; +} + +/* + * Copy xattr from one bucket to another bucket. + * + * The caller must make sure that the journal transaction + * has enough space for journaling. + */ +static int ocfs2_cp_xattr_bucket(struct inode *inode, + handle_t *handle, + u64 s_blkno, + u64 t_blkno, + int t_is_new) +{ + int ret; + struct ocfs2_xattr_bucket *s_bucket = NULL, *t_bucket = NULL; + + BUG_ON(s_blkno == t_blkno); + + trace_ocfs2_cp_xattr_bucket((unsigned long long)s_blkno, + (unsigned long long)t_blkno, + t_is_new); + + s_bucket = ocfs2_xattr_bucket_new(inode); + t_bucket = ocfs2_xattr_bucket_new(inode); + if (!s_bucket || !t_bucket) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + ret = ocfs2_read_xattr_bucket(s_bucket, s_blkno); + if (ret) + goto out; + + /* + * Even if !t_is_new, we're overwriting t_bucket. Thus, + * there's no need to read it. + */ + ret = ocfs2_init_xattr_bucket(t_bucket, t_blkno, t_is_new); + if (ret) + goto out; + + /* + * Hey, if we're overwriting t_bucket, what difference does + * ACCESS_CREATE vs ACCESS_WRITE make? Well, if we allocated a new + * cluster to fill, we came here from + * ocfs2_mv_xattr_buckets(), and it is really new - + * ACCESS_CREATE is required. But we also might have moved data + * out of t_bucket before extending back into it. + * ocfs2_add_new_xattr_bucket() can do this - its call to + * ocfs2_add_new_xattr_cluster() may have created a new extent + * and copied out the end of the old extent. Then it re-extends + * the old extent back to create space for new xattrs. That's + * how we get here, and the bucket isn't really new. + */ + ret = ocfs2_xattr_bucket_journal_access(handle, t_bucket, + t_is_new ? + OCFS2_JOURNAL_ACCESS_CREATE : + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) + goto out; + + ocfs2_xattr_bucket_copy_data(t_bucket, s_bucket); + ocfs2_xattr_bucket_journal_dirty(handle, t_bucket); + +out: + ocfs2_xattr_bucket_free(t_bucket); + ocfs2_xattr_bucket_free(s_bucket); + + return ret; +} + +/* + * src_blk points to the start of an existing extent. last_blk points to + * last cluster in that extent. to_blk points to a newly allocated + * extent. We copy the buckets from the cluster at last_blk to the new + * extent. If start_bucket is non-zero, we skip that many buckets before + * we start copying. The new extent's xh_num_buckets gets set to the + * number of buckets we copied. The old extent's xh_num_buckets shrinks + * by the same amount. + */ +static int ocfs2_mv_xattr_buckets(struct inode *inode, handle_t *handle, + u64 src_blk, u64 last_blk, u64 to_blk, + unsigned int start_bucket, + u32 *first_hash) +{ + int i, ret, credits; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + int blks_per_bucket = ocfs2_blocks_per_xattr_bucket(inode->i_sb); + int num_buckets = ocfs2_xattr_buckets_per_cluster(osb); + struct ocfs2_xattr_bucket *old_first, *new_first; + + trace_ocfs2_mv_xattr_buckets((unsigned long long)last_blk, + (unsigned long long)to_blk); + + BUG_ON(start_bucket >= num_buckets); + if (start_bucket) { + num_buckets -= start_bucket; + last_blk += (start_bucket * blks_per_bucket); + } + + /* The first bucket of the original extent */ + old_first = ocfs2_xattr_bucket_new(inode); + /* The first bucket of the new extent */ + new_first = ocfs2_xattr_bucket_new(inode); + if (!old_first || !new_first) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + ret = ocfs2_read_xattr_bucket(old_first, src_blk); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * We need to update the first bucket of the old extent and all + * the buckets going to the new extent. + */ + credits = ((num_buckets + 1) * blks_per_bucket); + ret = ocfs2_extend_trans(handle, credits); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_xattr_bucket_journal_access(handle, old_first, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + + for (i = 0; i < num_buckets; i++) { + ret = ocfs2_cp_xattr_bucket(inode, handle, + last_blk + (i * blks_per_bucket), + to_blk + (i * blks_per_bucket), + 1); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + /* + * Get the new bucket ready before we dirty anything + * (This actually shouldn't fail, because we already dirtied + * it once in ocfs2_cp_xattr_bucket()). + */ + ret = ocfs2_read_xattr_bucket(new_first, to_blk); + if (ret) { + mlog_errno(ret); + goto out; + } + ret = ocfs2_xattr_bucket_journal_access(handle, new_first, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* Now update the headers */ + le16_add_cpu(&bucket_xh(old_first)->xh_num_buckets, -num_buckets); + ocfs2_xattr_bucket_journal_dirty(handle, old_first); + + bucket_xh(new_first)->xh_num_buckets = cpu_to_le16(num_buckets); + ocfs2_xattr_bucket_journal_dirty(handle, new_first); + + if (first_hash) + *first_hash = le32_to_cpu(bucket_xh(new_first)->xh_entries[0].xe_name_hash); + +out: + ocfs2_xattr_bucket_free(new_first); + ocfs2_xattr_bucket_free(old_first); + return ret; +} + +/* + * Move some xattrs in this cluster to the new cluster. + * This function should only be called when bucket size == cluster size. + * Otherwise ocfs2_mv_xattr_bucket_cross_cluster should be used instead. + */ +static int ocfs2_divide_xattr_cluster(struct inode *inode, + handle_t *handle, + u64 prev_blk, + u64 new_blk, + u32 *first_hash) +{ + u16 blk_per_bucket = ocfs2_blocks_per_xattr_bucket(inode->i_sb); + int ret, credits = 2 * blk_per_bucket; + + BUG_ON(OCFS2_XATTR_BUCKET_SIZE < OCFS2_SB(inode->i_sb)->s_clustersize); + + ret = ocfs2_extend_trans(handle, credits); + if (ret) { + mlog_errno(ret); + return ret; + } + + /* Move half of the xattr in start_blk to the next bucket. */ + return ocfs2_divide_xattr_bucket(inode, handle, prev_blk, + new_blk, first_hash, 1); +} + +/* + * Move some xattrs from the old cluster to the new one since they are not + * contiguous in ocfs2 xattr tree. + * + * new_blk starts a new separate cluster, and we will move some xattrs from + * prev_blk to it. v_start will be set as the first name hash value in this + * new cluster so that it can be used as e_cpos during tree insertion and + * don't collide with our original b-tree operations. first_bh and header_bh + * will also be updated since they will be used in ocfs2_extend_xattr_bucket + * to extend the insert bucket. + * + * The problem is how much xattr should we move to the new one and when should + * we update first_bh and header_bh? + * 1. If cluster size > bucket size, that means the previous cluster has more + * than 1 bucket, so just move half nums of bucket into the new cluster and + * update the first_bh and header_bh if the insert bucket has been moved + * to the new cluster. + * 2. If cluster_size == bucket_size: + * a) If the previous extent rec has more than one cluster and the insert + * place isn't in the last cluster, copy the entire last cluster to the + * new one. This time, we don't need to update the first_bh and header_bh + * since they will not be moved into the new cluster. + * b) Otherwise, move the bottom half of the xattrs in the last cluster into + * the new one. And we set the extend flag to zero if the insert place is + * moved into the new allocated cluster since no extend is needed. + */ +static int ocfs2_adjust_xattr_cross_cluster(struct inode *inode, + handle_t *handle, + struct ocfs2_xattr_bucket *first, + struct ocfs2_xattr_bucket *target, + u64 new_blk, + u32 prev_clusters, + u32 *v_start, + int *extend) +{ + int ret; + + trace_ocfs2_adjust_xattr_cross_cluster( + (unsigned long long)bucket_blkno(first), + (unsigned long long)new_blk, prev_clusters); + + if (ocfs2_xattr_buckets_per_cluster(OCFS2_SB(inode->i_sb)) > 1) { + ret = ocfs2_mv_xattr_bucket_cross_cluster(inode, + handle, + first, target, + new_blk, + prev_clusters, + v_start); + if (ret) + mlog_errno(ret); + } else { + /* The start of the last cluster in the first extent */ + u64 last_blk = bucket_blkno(first) + + ((prev_clusters - 1) * + ocfs2_clusters_to_blocks(inode->i_sb, 1)); + + if (prev_clusters > 1 && bucket_blkno(target) != last_blk) { + ret = ocfs2_mv_xattr_buckets(inode, handle, + bucket_blkno(first), + last_blk, new_blk, 0, + v_start); + if (ret) + mlog_errno(ret); + } else { + ret = ocfs2_divide_xattr_cluster(inode, handle, + last_blk, new_blk, + v_start); + if (ret) + mlog_errno(ret); + + if ((bucket_blkno(target) == last_blk) && extend) + *extend = 0; + } + } + + return ret; +} + +/* + * Add a new cluster for xattr storage. + * + * If the new cluster is contiguous with the previous one, it will be + * appended to the same extent record, and num_clusters will be updated. + * If not, we will insert a new extent for it and move some xattrs in + * the last cluster into the new allocated one. + * We also need to limit the maximum size of a btree leaf, otherwise we'll + * lose the benefits of hashing because we'll have to search large leaves. + * So now the maximum size is OCFS2_MAX_XATTR_TREE_LEAF_SIZE(or clustersize, + * if it's bigger). + * + * first_bh is the first block of the previous extent rec and header_bh + * indicates the bucket we will insert the new xattrs. They will be updated + * when the header_bh is moved into the new cluster. + */ +static int ocfs2_add_new_xattr_cluster(struct inode *inode, + struct buffer_head *root_bh, + struct ocfs2_xattr_bucket *first, + struct ocfs2_xattr_bucket *target, + u32 *num_clusters, + u32 prev_cpos, + int *extend, + struct ocfs2_xattr_set_ctxt *ctxt) +{ + int ret; + u16 bpc = ocfs2_clusters_to_blocks(inode->i_sb, 1); + u32 prev_clusters = *num_clusters; + u32 clusters_to_add = 1, bit_off, num_bits, v_start = 0; + u64 block; + handle_t *handle = ctxt->handle; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct ocfs2_extent_tree et; + + trace_ocfs2_add_new_xattr_cluster_begin( + (unsigned long long)OCFS2_I(inode)->ip_blkno, + (unsigned long long)bucket_blkno(first), + prev_cpos, prev_clusters); + + ocfs2_init_xattr_tree_extent_tree(&et, INODE_CACHE(inode), root_bh); + + ret = ocfs2_journal_access_xb(handle, INODE_CACHE(inode), root_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret < 0) { + mlog_errno(ret); + goto leave; + } + + ret = __ocfs2_claim_clusters(handle, ctxt->data_ac, 1, + clusters_to_add, &bit_off, &num_bits); + if (ret < 0) { + if (ret != -ENOSPC) + mlog_errno(ret); + goto leave; + } + + BUG_ON(num_bits > clusters_to_add); + + block = ocfs2_clusters_to_blocks(osb->sb, bit_off); + trace_ocfs2_add_new_xattr_cluster((unsigned long long)block, num_bits); + + if (bucket_blkno(first) + (prev_clusters * bpc) == block && + (prev_clusters + num_bits) << osb->s_clustersize_bits <= + OCFS2_MAX_XATTR_TREE_LEAF_SIZE) { + /* + * If this cluster is contiguous with the old one and + * adding this new cluster, we don't surpass the limit of + * OCFS2_MAX_XATTR_TREE_LEAF_SIZE, cool. We will let it be + * initialized and used like other buckets in the previous + * cluster. + * So add it as a contiguous one. The caller will handle + * its init process. + */ + v_start = prev_cpos + prev_clusters; + *num_clusters = prev_clusters + num_bits; + } else { + ret = ocfs2_adjust_xattr_cross_cluster(inode, + handle, + first, + target, + block, + prev_clusters, + &v_start, + extend); + if (ret) { + mlog_errno(ret); + goto leave; + } + } + + trace_ocfs2_add_new_xattr_cluster_insert((unsigned long long)block, + v_start, num_bits); + ret = ocfs2_insert_extent(handle, &et, v_start, block, + num_bits, 0, ctxt->meta_ac); + if (ret < 0) { + mlog_errno(ret); + goto leave; + } + + ocfs2_journal_dirty(handle, root_bh); + +leave: + return ret; +} + +/* + * We are given an extent. 'first' is the bucket at the very front of + * the extent. The extent has space for an additional bucket past + * bucket_xh(first)->xh_num_buckets. 'target_blkno' is the block number + * of the target bucket. We wish to shift every bucket past the target + * down one, filling in that additional space. When we get back to the + * target, we split the target between itself and the now-empty bucket + * at target+1 (aka, target_blkno + blks_per_bucket). + */ +static int ocfs2_extend_xattr_bucket(struct inode *inode, + handle_t *handle, + struct ocfs2_xattr_bucket *first, + u64 target_blk, + u32 num_clusters) +{ + int ret, credits; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + u16 blk_per_bucket = ocfs2_blocks_per_xattr_bucket(inode->i_sb); + u64 end_blk; + u16 new_bucket = le16_to_cpu(bucket_xh(first)->xh_num_buckets); + + trace_ocfs2_extend_xattr_bucket((unsigned long long)target_blk, + (unsigned long long)bucket_blkno(first), + num_clusters, new_bucket); + + /* The extent must have room for an additional bucket */ + BUG_ON(new_bucket >= + (num_clusters * ocfs2_xattr_buckets_per_cluster(osb))); + + /* end_blk points to the last existing bucket */ + end_blk = bucket_blkno(first) + ((new_bucket - 1) * blk_per_bucket); + + /* + * end_blk is the start of the last existing bucket. + * Thus, (end_blk - target_blk) covers the target bucket and + * every bucket after it up to, but not including, the last + * existing bucket. Then we add the last existing bucket, the + * new bucket, and the first bucket (3 * blk_per_bucket). + */ + credits = (end_blk - target_blk) + (3 * blk_per_bucket); + ret = ocfs2_extend_trans(handle, credits); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_xattr_bucket_journal_access(handle, first, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + + while (end_blk != target_blk) { + ret = ocfs2_cp_xattr_bucket(inode, handle, end_blk, + end_blk + blk_per_bucket, 0); + if (ret) + goto out; + end_blk -= blk_per_bucket; + } + + /* Move half of the xattr in target_blkno to the next bucket. */ + ret = ocfs2_divide_xattr_bucket(inode, handle, target_blk, + target_blk + blk_per_bucket, NULL, 0); + + le16_add_cpu(&bucket_xh(first)->xh_num_buckets, 1); + ocfs2_xattr_bucket_journal_dirty(handle, first); + +out: + return ret; +} + +/* + * Add new xattr bucket in an extent record and adjust the buckets + * accordingly. xb_bh is the ocfs2_xattr_block, and target is the + * bucket we want to insert into. + * + * In the easy case, we will move all the buckets after target down by + * one. Half of target's xattrs will be moved to the next bucket. + * + * If current cluster is full, we'll allocate a new one. This may not + * be contiguous. The underlying calls will make sure that there is + * space for the insert, shifting buckets around if necessary. + * 'target' may be moved by those calls. + */ +static int ocfs2_add_new_xattr_bucket(struct inode *inode, + struct buffer_head *xb_bh, + struct ocfs2_xattr_bucket *target, + struct ocfs2_xattr_set_ctxt *ctxt) +{ + struct ocfs2_xattr_block *xb = + (struct ocfs2_xattr_block *)xb_bh->b_data; + struct ocfs2_xattr_tree_root *xb_root = &xb->xb_attrs.xb_root; + struct ocfs2_extent_list *el = &xb_root->xt_list; + u32 name_hash = + le32_to_cpu(bucket_xh(target)->xh_entries[0].xe_name_hash); + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + int ret, num_buckets, extend = 1; + u64 p_blkno; + u32 e_cpos, num_clusters; + /* The bucket at the front of the extent */ + struct ocfs2_xattr_bucket *first; + + trace_ocfs2_add_new_xattr_bucket( + (unsigned long long)bucket_blkno(target)); + + /* The first bucket of the original extent */ + first = ocfs2_xattr_bucket_new(inode); + if (!first) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + ret = ocfs2_xattr_get_rec(inode, name_hash, &p_blkno, &e_cpos, + &num_clusters, el); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_read_xattr_bucket(first, p_blkno); + if (ret) { + mlog_errno(ret); + goto out; + } + + num_buckets = ocfs2_xattr_buckets_per_cluster(osb) * num_clusters; + if (num_buckets == le16_to_cpu(bucket_xh(first)->xh_num_buckets)) { + /* + * This can move first+target if the target bucket moves + * to the new extent. + */ + ret = ocfs2_add_new_xattr_cluster(inode, + xb_bh, + first, + target, + &num_clusters, + e_cpos, + &extend, + ctxt); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + if (extend) { + ret = ocfs2_extend_xattr_bucket(inode, + ctxt->handle, + first, + bucket_blkno(target), + num_clusters); + if (ret) + mlog_errno(ret); + } + +out: + ocfs2_xattr_bucket_free(first); + + return ret; +} + +/* + * Truncate the specified xe_off entry in xattr bucket. + * bucket is indicated by header_bh and len is the new length. + * Both the ocfs2_xattr_value_root and the entry will be updated here. + * + * Copy the new updated xe and xe_value_root to new_xe and new_xv if needed. + */ +static int ocfs2_xattr_bucket_value_truncate(struct inode *inode, + struct ocfs2_xattr_bucket *bucket, + int xe_off, + int len, + struct ocfs2_xattr_set_ctxt *ctxt) +{ + int ret, offset; + u64 value_blk; + struct ocfs2_xattr_entry *xe; + struct ocfs2_xattr_header *xh = bucket_xh(bucket); + size_t blocksize = inode->i_sb->s_blocksize; + struct ocfs2_xattr_value_buf vb = { + .vb_access = ocfs2_journal_access, + }; + + xe = &xh->xh_entries[xe_off]; + + BUG_ON(!xe || ocfs2_xattr_is_local(xe)); + + offset = le16_to_cpu(xe->xe_name_offset) + + OCFS2_XATTR_SIZE(xe->xe_name_len); + + value_blk = offset / blocksize; + + /* We don't allow ocfs2_xattr_value to be stored in different block. */ + BUG_ON(value_blk != (offset + OCFS2_XATTR_ROOT_SIZE - 1) / blocksize); + + vb.vb_bh = bucket->bu_bhs[value_blk]; + BUG_ON(!vb.vb_bh); + + vb.vb_xv = (struct ocfs2_xattr_value_root *) + (vb.vb_bh->b_data + offset % blocksize); + + /* + * From here on out we have to dirty the bucket. The generic + * value calls only modify one of the bucket's bhs, but we need + * to send the bucket at once. So if they error, they *could* have + * modified something. We have to assume they did, and dirty + * the whole bucket. This leaves us in a consistent state. + */ + trace_ocfs2_xattr_bucket_value_truncate( + (unsigned long long)bucket_blkno(bucket), xe_off, len); + ret = ocfs2_xattr_value_truncate(inode, &vb, len, ctxt); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_xattr_bucket_journal_access(ctxt->handle, bucket, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out; + } + + xe->xe_value_size = cpu_to_le64(len); + + ocfs2_xattr_bucket_journal_dirty(ctxt->handle, bucket); + +out: + return ret; +} + +static int ocfs2_rm_xattr_cluster(struct inode *inode, + struct buffer_head *root_bh, + u64 blkno, + u32 cpos, + u32 len, + void *para) +{ + int ret; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct inode *tl_inode = osb->osb_tl_inode; + handle_t *handle; + struct ocfs2_xattr_block *xb = + (struct ocfs2_xattr_block *)root_bh->b_data; + struct ocfs2_alloc_context *meta_ac = NULL; + struct ocfs2_cached_dealloc_ctxt dealloc; + struct ocfs2_extent_tree et; + + ret = ocfs2_iterate_xattr_buckets(inode, blkno, len, + ocfs2_delete_xattr_in_bucket, para); + if (ret) { + mlog_errno(ret); + return ret; + } + + ocfs2_init_xattr_tree_extent_tree(&et, INODE_CACHE(inode), root_bh); + + ocfs2_init_dealloc_ctxt(&dealloc); + + trace_ocfs2_rm_xattr_cluster( + (unsigned long long)OCFS2_I(inode)->ip_blkno, + (unsigned long long)blkno, cpos, len); + + ocfs2_remove_xattr_clusters_from_cache(INODE_CACHE(inode), blkno, + len); + + ret = ocfs2_lock_allocators(inode, &et, 0, 1, NULL, &meta_ac); + if (ret) { + mlog_errno(ret); + return ret; + } + + inode_lock(tl_inode); + + if (ocfs2_truncate_log_needs_flush(osb)) { + ret = __ocfs2_flush_truncate_log(osb); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + } + + handle = ocfs2_start_trans(osb, ocfs2_remove_extent_credits(osb->sb)); + if (IS_ERR(handle)) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + ret = ocfs2_journal_access_xb(handle, INODE_CACHE(inode), root_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + ret = ocfs2_remove_extent(handle, &et, cpos, len, meta_ac, + &dealloc); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + le32_add_cpu(&xb->xb_attrs.xb_root.xt_clusters, -len); + ocfs2_journal_dirty(handle, root_bh); + + ret = ocfs2_truncate_log_append(osb, handle, blkno, len); + if (ret) + mlog_errno(ret); + ocfs2_update_inode_fsync_trans(handle, inode, 0); + +out_commit: + ocfs2_commit_trans(osb, handle); +out: + ocfs2_schedule_truncate_log_flush(osb, 1); + + inode_unlock(tl_inode); + + if (meta_ac) + ocfs2_free_alloc_context(meta_ac); + + ocfs2_run_deallocs(osb, &dealloc); + + return ret; +} + +/* + * check whether the xattr bucket is filled up with the same hash value. + * If we want to insert the xattr with the same hash, return -ENOSPC. + * If we want to insert a xattr with different hash value, go ahead + * and ocfs2_divide_xattr_bucket will handle this. + */ +static int ocfs2_check_xattr_bucket_collision(struct inode *inode, + struct ocfs2_xattr_bucket *bucket, + const char *name) +{ + struct ocfs2_xattr_header *xh = bucket_xh(bucket); + u32 name_hash = ocfs2_xattr_name_hash(inode, name, strlen(name)); + + if (name_hash != le32_to_cpu(xh->xh_entries[0].xe_name_hash)) + return 0; + + if (xh->xh_entries[le16_to_cpu(xh->xh_count) - 1].xe_name_hash == + xh->xh_entries[0].xe_name_hash) { + mlog(ML_ERROR, "Too much hash collision in xattr bucket %llu, " + "hash = %u\n", + (unsigned long long)bucket_blkno(bucket), + le32_to_cpu(xh->xh_entries[0].xe_name_hash)); + return -ENOSPC; + } + + return 0; +} + +/* + * Try to set the entry in the current bucket. If we fail, the caller + * will handle getting us another bucket. + */ +static int ocfs2_xattr_set_entry_bucket(struct inode *inode, + struct ocfs2_xattr_info *xi, + struct ocfs2_xattr_search *xs, + struct ocfs2_xattr_set_ctxt *ctxt) +{ + int ret; + struct ocfs2_xa_loc loc; + + trace_ocfs2_xattr_set_entry_bucket(xi->xi_name); + + ocfs2_init_xattr_bucket_xa_loc(&loc, xs->bucket, + xs->not_found ? NULL : xs->here); + ret = ocfs2_xa_set(&loc, xi, ctxt); + if (!ret) { + xs->here = loc.xl_entry; + goto out; + } + if (ret != -ENOSPC) { + mlog_errno(ret); + goto out; + } + + /* Ok, we need space. Let's try defragmenting the bucket. */ + ret = ocfs2_defrag_xattr_bucket(inode, ctxt->handle, + xs->bucket); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_xa_set(&loc, xi, ctxt); + if (!ret) { + xs->here = loc.xl_entry; + goto out; + } + if (ret != -ENOSPC) + mlog_errno(ret); + + +out: + return ret; +} + +static int ocfs2_xattr_set_entry_index_block(struct inode *inode, + struct ocfs2_xattr_info *xi, + struct ocfs2_xattr_search *xs, + struct ocfs2_xattr_set_ctxt *ctxt) +{ + int ret; + + trace_ocfs2_xattr_set_entry_index_block(xi->xi_name); + + ret = ocfs2_xattr_set_entry_bucket(inode, xi, xs, ctxt); + if (!ret) + goto out; + if (ret != -ENOSPC) { + mlog_errno(ret); + goto out; + } + + /* Ack, need more space. Let's try to get another bucket! */ + + /* + * We do not allow for overlapping ranges between buckets. And + * the maximum number of collisions we will allow for then is + * one bucket's worth, so check it here whether we need to + * add a new bucket for the insert. + */ + ret = ocfs2_check_xattr_bucket_collision(inode, + xs->bucket, + xi->xi_name); + if (ret) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_add_new_xattr_bucket(inode, + xs->xattr_bh, + xs->bucket, + ctxt); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * ocfs2_add_new_xattr_bucket() will have updated + * xs->bucket if it moved, but it will not have updated + * any of the other search fields. Thus, we drop it and + * re-search. Everything should be cached, so it'll be + * quick. + */ + ocfs2_xattr_bucket_relse(xs->bucket); + ret = ocfs2_xattr_index_block_find(inode, xs->xattr_bh, + xi->xi_name_index, + xi->xi_name, xs); + if (ret && ret != -ENODATA) + goto out; + xs->not_found = ret; + + /* Ok, we have a new bucket, let's try again */ + ret = ocfs2_xattr_set_entry_bucket(inode, xi, xs, ctxt); + if (ret && (ret != -ENOSPC)) + mlog_errno(ret); + +out: + return ret; +} + +static int ocfs2_delete_xattr_in_bucket(struct inode *inode, + struct ocfs2_xattr_bucket *bucket, + void *para) +{ + int ret = 0, ref_credits; + struct ocfs2_xattr_header *xh = bucket_xh(bucket); + u16 i; + struct ocfs2_xattr_entry *xe; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct ocfs2_xattr_set_ctxt ctxt = {NULL, NULL,}; + int credits = ocfs2_remove_extent_credits(osb->sb) + + ocfs2_blocks_per_xattr_bucket(inode->i_sb); + struct ocfs2_xattr_value_root *xv; + struct ocfs2_rm_xattr_bucket_para *args = + (struct ocfs2_rm_xattr_bucket_para *)para; + + ocfs2_init_dealloc_ctxt(&ctxt.dealloc); + + for (i = 0; i < le16_to_cpu(xh->xh_count); i++) { + xe = &xh->xh_entries[i]; + if (ocfs2_xattr_is_local(xe)) + continue; + + ret = ocfs2_get_xattr_tree_value_root(inode->i_sb, bucket, + i, &xv, NULL); + if (ret) { + mlog_errno(ret); + break; + } + + ret = ocfs2_lock_xattr_remove_allocators(inode, xv, + args->ref_ci, + args->ref_root_bh, + &ctxt.meta_ac, + &ref_credits); + + ctxt.handle = ocfs2_start_trans(osb, credits + ref_credits); + if (IS_ERR(ctxt.handle)) { + ret = PTR_ERR(ctxt.handle); + mlog_errno(ret); + break; + } + + ret = ocfs2_xattr_bucket_value_truncate(inode, bucket, + i, 0, &ctxt); + + ocfs2_commit_trans(osb, ctxt.handle); + if (ctxt.meta_ac) { + ocfs2_free_alloc_context(ctxt.meta_ac); + ctxt.meta_ac = NULL; + } + if (ret) { + mlog_errno(ret); + break; + } + } + + if (ctxt.meta_ac) + ocfs2_free_alloc_context(ctxt.meta_ac); + ocfs2_schedule_truncate_log_flush(osb, 1); + ocfs2_run_deallocs(osb, &ctxt.dealloc); + return ret; +} + +/* + * Whenever we modify a xattr value root in the bucket(e.g, CoW + * or change the extent record flag), we need to recalculate + * the metaecc for the whole bucket. So it is done here. + * + * Note: + * We have to give the extra credits for the caller. + */ +static int ocfs2_xattr_bucket_post_refcount(struct inode *inode, + handle_t *handle, + void *para) +{ + int ret; + struct ocfs2_xattr_bucket *bucket = + (struct ocfs2_xattr_bucket *)para; + + ret = ocfs2_xattr_bucket_journal_access(handle, bucket, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + return ret; + } + + ocfs2_xattr_bucket_journal_dirty(handle, bucket); + + return 0; +} + +/* + * Special action we need if the xattr value is refcounted. + * + * 1. If the xattr is refcounted, lock the tree. + * 2. CoW the xattr if we are setting the new value and the value + * will be stored outside. + * 3. In other case, decrease_refcount will work for us, so just + * lock the refcount tree, calculate the meta and credits is OK. + * + * We have to do CoW before ocfs2_init_xattr_set_ctxt since + * currently CoW is a completed transaction, while this function + * will also lock the allocators and let us deadlock. So we will + * CoW the whole xattr value. + */ +static int ocfs2_prepare_refcount_xattr(struct inode *inode, + struct ocfs2_dinode *di, + struct ocfs2_xattr_info *xi, + struct ocfs2_xattr_search *xis, + struct ocfs2_xattr_search *xbs, + struct ocfs2_refcount_tree **ref_tree, + int *meta_add, + int *credits) +{ + int ret = 0; + struct ocfs2_xattr_block *xb; + struct ocfs2_xattr_entry *xe; + char *base; + u32 p_cluster, num_clusters; + unsigned int ext_flags; + int name_offset, name_len; + struct ocfs2_xattr_value_buf vb; + struct ocfs2_xattr_bucket *bucket = NULL; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct ocfs2_post_refcount refcount; + struct ocfs2_post_refcount *p = NULL; + struct buffer_head *ref_root_bh = NULL; + + if (!xis->not_found) { + xe = xis->here; + name_offset = le16_to_cpu(xe->xe_name_offset); + name_len = OCFS2_XATTR_SIZE(xe->xe_name_len); + base = xis->base; + vb.vb_bh = xis->inode_bh; + vb.vb_access = ocfs2_journal_access_di; + } else { + int i, block_off = 0; + xb = (struct ocfs2_xattr_block *)xbs->xattr_bh->b_data; + xe = xbs->here; + name_offset = le16_to_cpu(xe->xe_name_offset); + name_len = OCFS2_XATTR_SIZE(xe->xe_name_len); + i = xbs->here - xbs->header->xh_entries; + + if (le16_to_cpu(xb->xb_flags) & OCFS2_XATTR_INDEXED) { + ret = ocfs2_xattr_bucket_get_name_value(inode->i_sb, + bucket_xh(xbs->bucket), + i, &block_off, + &name_offset); + if (ret) { + mlog_errno(ret); + goto out; + } + base = bucket_block(xbs->bucket, block_off); + vb.vb_bh = xbs->bucket->bu_bhs[block_off]; + vb.vb_access = ocfs2_journal_access; + + if (ocfs2_meta_ecc(osb)) { + /*create parameters for ocfs2_post_refcount. */ + bucket = xbs->bucket; + refcount.credits = bucket->bu_blocks; + refcount.para = bucket; + refcount.func = + ocfs2_xattr_bucket_post_refcount; + p = &refcount; + } + } else { + base = xbs->base; + vb.vb_bh = xbs->xattr_bh; + vb.vb_access = ocfs2_journal_access_xb; + } + } + + if (ocfs2_xattr_is_local(xe)) + goto out; + + vb.vb_xv = (struct ocfs2_xattr_value_root *) + (base + name_offset + name_len); + + ret = ocfs2_xattr_get_clusters(inode, 0, &p_cluster, + &num_clusters, &vb.vb_xv->xr_list, + &ext_flags); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * We just need to check the 1st extent record, since we always + * CoW the whole xattr. So there shouldn't be a xattr with + * some REFCOUNT extent recs after the 1st one. + */ + if (!(ext_flags & OCFS2_EXT_REFCOUNTED)) + goto out; + + ret = ocfs2_lock_refcount_tree(osb, le64_to_cpu(di->i_refcount_loc), + 1, ref_tree, &ref_root_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * If we are deleting the xattr or the new size will be stored inside, + * cool, leave it there, the xattr truncate process will remove them + * for us(it still needs the refcount tree lock and the meta, credits). + * And the worse case is that every cluster truncate will split the + * refcount tree, and make the original extent become 3. So we will need + * 2 * cluster more extent recs at most. + */ + if (!xi->xi_value || xi->xi_value_len <= OCFS2_XATTR_INLINE_SIZE) { + + ret = ocfs2_refcounted_xattr_delete_need(inode, + &(*ref_tree)->rf_ci, + ref_root_bh, vb.vb_xv, + meta_add, credits); + if (ret) + mlog_errno(ret); + goto out; + } + + ret = ocfs2_refcount_cow_xattr(inode, di, &vb, + *ref_tree, ref_root_bh, 0, + le32_to_cpu(vb.vb_xv->xr_clusters), p); + if (ret) + mlog_errno(ret); + +out: + brelse(ref_root_bh); + return ret; +} + +/* + * Add the REFCOUNTED flags for all the extent rec in ocfs2_xattr_value_root. + * The physical clusters will be added to refcount tree. + */ +static int ocfs2_xattr_value_attach_refcount(struct inode *inode, + struct ocfs2_xattr_value_root *xv, + struct ocfs2_extent_tree *value_et, + struct ocfs2_caching_info *ref_ci, + struct buffer_head *ref_root_bh, + struct ocfs2_cached_dealloc_ctxt *dealloc, + struct ocfs2_post_refcount *refcount) +{ + int ret = 0; + u32 clusters = le32_to_cpu(xv->xr_clusters); + u32 cpos, p_cluster, num_clusters; + struct ocfs2_extent_list *el = &xv->xr_list; + unsigned int ext_flags; + + cpos = 0; + while (cpos < clusters) { + ret = ocfs2_xattr_get_clusters(inode, cpos, &p_cluster, + &num_clusters, el, &ext_flags); + if (ret) { + mlog_errno(ret); + break; + } + + cpos += num_clusters; + if ((ext_flags & OCFS2_EXT_REFCOUNTED)) + continue; + + BUG_ON(!p_cluster); + + ret = ocfs2_add_refcount_flag(inode, value_et, + ref_ci, ref_root_bh, + cpos - num_clusters, + p_cluster, num_clusters, + dealloc, refcount); + if (ret) { + mlog_errno(ret); + break; + } + } + + return ret; +} + +/* + * Given a normal ocfs2_xattr_header, refcount all the entries which + * have value stored outside. + * Used for xattrs stored in inode and ocfs2_xattr_block. + */ +static int ocfs2_xattr_attach_refcount_normal(struct inode *inode, + struct ocfs2_xattr_value_buf *vb, + struct ocfs2_xattr_header *header, + struct ocfs2_caching_info *ref_ci, + struct buffer_head *ref_root_bh, + struct ocfs2_cached_dealloc_ctxt *dealloc) +{ + + struct ocfs2_xattr_entry *xe; + struct ocfs2_xattr_value_root *xv; + struct ocfs2_extent_tree et; + int i, ret = 0; + + for (i = 0; i < le16_to_cpu(header->xh_count); i++) { + xe = &header->xh_entries[i]; + + if (ocfs2_xattr_is_local(xe)) + continue; + + xv = (struct ocfs2_xattr_value_root *)((void *)header + + le16_to_cpu(xe->xe_name_offset) + + OCFS2_XATTR_SIZE(xe->xe_name_len)); + + vb->vb_xv = xv; + ocfs2_init_xattr_value_extent_tree(&et, INODE_CACHE(inode), vb); + + ret = ocfs2_xattr_value_attach_refcount(inode, xv, &et, + ref_ci, ref_root_bh, + dealloc, NULL); + if (ret) { + mlog_errno(ret); + break; + } + } + + return ret; +} + +static int ocfs2_xattr_inline_attach_refcount(struct inode *inode, + struct buffer_head *fe_bh, + struct ocfs2_caching_info *ref_ci, + struct buffer_head *ref_root_bh, + struct ocfs2_cached_dealloc_ctxt *dealloc) +{ + struct ocfs2_dinode *di = (struct ocfs2_dinode *)fe_bh->b_data; + struct ocfs2_xattr_header *header; + int ret; + struct ocfs2_xattr_value_buf vb = { + .vb_bh = fe_bh, + .vb_access = ocfs2_journal_access_di, + }; + + ret = ocfs2_xattr_ibody_lookup_header(inode, di, &header); + if (ret) + return ret; + + return ocfs2_xattr_attach_refcount_normal(inode, &vb, header, + ref_ci, ref_root_bh, dealloc); +} + +struct ocfs2_xattr_tree_value_refcount_para { + struct ocfs2_caching_info *ref_ci; + struct buffer_head *ref_root_bh; + struct ocfs2_cached_dealloc_ctxt *dealloc; +}; + +static int ocfs2_get_xattr_tree_value_root(struct super_block *sb, + struct ocfs2_xattr_bucket *bucket, + int offset, + struct ocfs2_xattr_value_root **xv, + struct buffer_head **bh) +{ + int ret, block_off, name_offset; + struct ocfs2_xattr_header *xh = bucket_xh(bucket); + struct ocfs2_xattr_entry *xe = &xh->xh_entries[offset]; + void *base; + + ret = ocfs2_xattr_bucket_get_name_value(sb, + bucket_xh(bucket), + offset, + &block_off, + &name_offset); + if (ret) { + mlog_errno(ret); + goto out; + } + + base = bucket_block(bucket, block_off); + + *xv = (struct ocfs2_xattr_value_root *)(base + name_offset + + OCFS2_XATTR_SIZE(xe->xe_name_len)); + + if (bh) + *bh = bucket->bu_bhs[block_off]; +out: + return ret; +} + +/* + * For a given xattr bucket, refcount all the entries which + * have value stored outside. + */ +static int ocfs2_xattr_bucket_value_refcount(struct inode *inode, + struct ocfs2_xattr_bucket *bucket, + void *para) +{ + int i, ret = 0; + struct ocfs2_extent_tree et; + struct ocfs2_xattr_tree_value_refcount_para *ref = + (struct ocfs2_xattr_tree_value_refcount_para *)para; + struct ocfs2_xattr_header *xh = + (struct ocfs2_xattr_header *)bucket->bu_bhs[0]->b_data; + struct ocfs2_xattr_entry *xe; + struct ocfs2_xattr_value_buf vb = { + .vb_access = ocfs2_journal_access, + }; + struct ocfs2_post_refcount refcount = { + .credits = bucket->bu_blocks, + .para = bucket, + .func = ocfs2_xattr_bucket_post_refcount, + }; + struct ocfs2_post_refcount *p = NULL; + + /* We only need post_refcount if we support metaecc. */ + if (ocfs2_meta_ecc(OCFS2_SB(inode->i_sb))) + p = &refcount; + + trace_ocfs2_xattr_bucket_value_refcount( + (unsigned long long)bucket_blkno(bucket), + le16_to_cpu(xh->xh_count)); + for (i = 0; i < le16_to_cpu(xh->xh_count); i++) { + xe = &xh->xh_entries[i]; + + if (ocfs2_xattr_is_local(xe)) + continue; + + ret = ocfs2_get_xattr_tree_value_root(inode->i_sb, bucket, i, + &vb.vb_xv, &vb.vb_bh); + if (ret) { + mlog_errno(ret); + break; + } + + ocfs2_init_xattr_value_extent_tree(&et, + INODE_CACHE(inode), &vb); + + ret = ocfs2_xattr_value_attach_refcount(inode, vb.vb_xv, + &et, ref->ref_ci, + ref->ref_root_bh, + ref->dealloc, p); + if (ret) { + mlog_errno(ret); + break; + } + } + + return ret; + +} + +static int ocfs2_refcount_xattr_tree_rec(struct inode *inode, + struct buffer_head *root_bh, + u64 blkno, u32 cpos, u32 len, void *para) +{ + return ocfs2_iterate_xattr_buckets(inode, blkno, len, + ocfs2_xattr_bucket_value_refcount, + para); +} + +static int ocfs2_xattr_block_attach_refcount(struct inode *inode, + struct buffer_head *blk_bh, + struct ocfs2_caching_info *ref_ci, + struct buffer_head *ref_root_bh, + struct ocfs2_cached_dealloc_ctxt *dealloc) +{ + int ret = 0; + struct ocfs2_xattr_block *xb = + (struct ocfs2_xattr_block *)blk_bh->b_data; + + if (!(le16_to_cpu(xb->xb_flags) & OCFS2_XATTR_INDEXED)) { + struct ocfs2_xattr_header *header = &xb->xb_attrs.xb_header; + struct ocfs2_xattr_value_buf vb = { + .vb_bh = blk_bh, + .vb_access = ocfs2_journal_access_xb, + }; + + ret = ocfs2_xattr_attach_refcount_normal(inode, &vb, header, + ref_ci, ref_root_bh, + dealloc); + } else { + struct ocfs2_xattr_tree_value_refcount_para para = { + .ref_ci = ref_ci, + .ref_root_bh = ref_root_bh, + .dealloc = dealloc, + }; + + ret = ocfs2_iterate_xattr_index_block(inode, blk_bh, + ocfs2_refcount_xattr_tree_rec, + ¶); + } + + return ret; +} + +int ocfs2_xattr_attach_refcount_tree(struct inode *inode, + struct buffer_head *fe_bh, + struct ocfs2_caching_info *ref_ci, + struct buffer_head *ref_root_bh, + struct ocfs2_cached_dealloc_ctxt *dealloc) +{ + int ret = 0; + struct ocfs2_inode_info *oi = OCFS2_I(inode); + struct ocfs2_dinode *di = (struct ocfs2_dinode *)fe_bh->b_data; + struct buffer_head *blk_bh = NULL; + + if (oi->ip_dyn_features & OCFS2_INLINE_XATTR_FL) { + ret = ocfs2_xattr_inline_attach_refcount(inode, fe_bh, + ref_ci, ref_root_bh, + dealloc); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + if (!di->i_xattr_loc) + goto out; + + ret = ocfs2_read_xattr_block(inode, le64_to_cpu(di->i_xattr_loc), + &blk_bh); + if (ret < 0) { + mlog_errno(ret); + goto out; + } + + ret = ocfs2_xattr_block_attach_refcount(inode, blk_bh, ref_ci, + ref_root_bh, dealloc); + if (ret) + mlog_errno(ret); + + brelse(blk_bh); +out: + + return ret; +} + +typedef int (should_xattr_reflinked)(struct ocfs2_xattr_entry *xe); +/* + * Store the information we need in xattr reflink. + * old_bh and new_bh are inode bh for the old and new inode. + */ +struct ocfs2_xattr_reflink { + struct inode *old_inode; + struct inode *new_inode; + struct buffer_head *old_bh; + struct buffer_head *new_bh; + struct ocfs2_caching_info *ref_ci; + struct buffer_head *ref_root_bh; + struct ocfs2_cached_dealloc_ctxt *dealloc; + should_xattr_reflinked *xattr_reflinked; +}; + +/* + * Given a xattr header and xe offset, + * return the proper xv and the corresponding bh. + * xattr in inode, block and xattr tree have different implementations. + */ +typedef int (get_xattr_value_root)(struct super_block *sb, + struct buffer_head *bh, + struct ocfs2_xattr_header *xh, + int offset, + struct ocfs2_xattr_value_root **xv, + struct buffer_head **ret_bh, + void *para); + +/* + * Calculate all the xattr value root metadata stored in this xattr header and + * credits we need if we create them from the scratch. + * We use get_xattr_value_root so that all types of xattr container can use it. + */ +static int ocfs2_value_metas_in_xattr_header(struct super_block *sb, + struct buffer_head *bh, + struct ocfs2_xattr_header *xh, + int *metas, int *credits, + int *num_recs, + get_xattr_value_root *func, + void *para) +{ + int i, ret = 0; + struct ocfs2_xattr_value_root *xv; + struct ocfs2_xattr_entry *xe; + + for (i = 0; i < le16_to_cpu(xh->xh_count); i++) { + xe = &xh->xh_entries[i]; + if (ocfs2_xattr_is_local(xe)) + continue; + + ret = func(sb, bh, xh, i, &xv, NULL, para); + if (ret) { + mlog_errno(ret); + break; + } + + *metas += le16_to_cpu(xv->xr_list.l_tree_depth) * + le16_to_cpu(xv->xr_list.l_next_free_rec); + + *credits += ocfs2_calc_extend_credits(sb, + &def_xv.xv.xr_list); + + /* + * If the value is a tree with depth > 1, We don't go deep + * to the extent block, so just calculate a maximum record num. + */ + if (!xv->xr_list.l_tree_depth) + *num_recs += le16_to_cpu(xv->xr_list.l_next_free_rec); + else + *num_recs += ocfs2_clusters_for_bytes(sb, + XATTR_SIZE_MAX); + } + + return ret; +} + +/* Used by xattr inode and block to return the right xv and buffer_head. */ +static int ocfs2_get_xattr_value_root(struct super_block *sb, + struct buffer_head *bh, + struct ocfs2_xattr_header *xh, + int offset, + struct ocfs2_xattr_value_root **xv, + struct buffer_head **ret_bh, + void *para) +{ + struct ocfs2_xattr_entry *xe = &xh->xh_entries[offset]; + + *xv = (struct ocfs2_xattr_value_root *)((void *)xh + + le16_to_cpu(xe->xe_name_offset) + + OCFS2_XATTR_SIZE(xe->xe_name_len)); + + if (ret_bh) + *ret_bh = bh; + + return 0; +} + +/* + * Lock the meta_ac and calculate how much credits we need for reflink xattrs. + * It is only used for inline xattr and xattr block. + */ +static int ocfs2_reflink_lock_xattr_allocators(struct ocfs2_super *osb, + struct ocfs2_xattr_header *xh, + struct buffer_head *ref_root_bh, + int *credits, + struct ocfs2_alloc_context **meta_ac) +{ + int ret, meta_add = 0, num_recs = 0; + struct ocfs2_refcount_block *rb = + (struct ocfs2_refcount_block *)ref_root_bh->b_data; + + *credits = 0; + + ret = ocfs2_value_metas_in_xattr_header(osb->sb, NULL, xh, + &meta_add, credits, &num_recs, + ocfs2_get_xattr_value_root, + NULL); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * We need to add/modify num_recs in refcount tree, so just calculate + * an approximate number we need for refcount tree change. + * Sometimes we need to split the tree, and after split, half recs + * will be moved to the new block, and a new block can only provide + * half number of recs. So we multiple new blocks by 2. + */ + num_recs = num_recs / ocfs2_refcount_recs_per_rb(osb->sb) * 2; + meta_add += num_recs; + *credits += num_recs + num_recs * OCFS2_EXPAND_REFCOUNT_TREE_CREDITS; + if (le32_to_cpu(rb->rf_flags) & OCFS2_REFCOUNT_TREE_FL) + *credits += le16_to_cpu(rb->rf_list.l_tree_depth) * + le16_to_cpu(rb->rf_list.l_next_free_rec) + 1; + else + *credits += 1; + + ret = ocfs2_reserve_new_metadata_blocks(osb, meta_add, meta_ac); + if (ret) + mlog_errno(ret); + +out: + return ret; +} + +/* + * Given a xattr header, reflink all the xattrs in this container. + * It can be used for inode, block and bucket. + * + * NOTE: + * Before we call this function, the caller has memcpy the xattr in + * old_xh to the new_xh. + * + * If args.xattr_reflinked is set, call it to decide whether the xe should + * be reflinked or not. If not, remove it from the new xattr header. + */ +static int ocfs2_reflink_xattr_header(handle_t *handle, + struct ocfs2_xattr_reflink *args, + struct buffer_head *old_bh, + struct ocfs2_xattr_header *xh, + struct buffer_head *new_bh, + struct ocfs2_xattr_header *new_xh, + struct ocfs2_xattr_value_buf *vb, + struct ocfs2_alloc_context *meta_ac, + get_xattr_value_root *func, + void *para) +{ + int ret = 0, i, j; + struct super_block *sb = args->old_inode->i_sb; + struct buffer_head *value_bh; + struct ocfs2_xattr_entry *xe, *last; + struct ocfs2_xattr_value_root *xv, *new_xv; + struct ocfs2_extent_tree data_et; + u32 clusters, cpos, p_cluster, num_clusters; + unsigned int ext_flags = 0; + + trace_ocfs2_reflink_xattr_header((unsigned long long)old_bh->b_blocknr, + le16_to_cpu(xh->xh_count)); + + last = &new_xh->xh_entries[le16_to_cpu(new_xh->xh_count)] - 1; + for (i = 0, j = 0; i < le16_to_cpu(xh->xh_count); i++, j++) { + xe = &xh->xh_entries[i]; + + if (args->xattr_reflinked && !args->xattr_reflinked(xe)) { + xe = &new_xh->xh_entries[j]; + + le16_add_cpu(&new_xh->xh_count, -1); + if (new_xh->xh_count) { + memmove(xe, xe + 1, + (void *)last - (void *)xe); + memset(last, 0, + sizeof(struct ocfs2_xattr_entry)); + last = &new_xh->xh_entries[le16_to_cpu(new_xh->xh_count)] - 1; + } else { + memset(xe, 0, sizeof(struct ocfs2_xattr_entry)); + last = NULL; + } + + /* + * We don't want j to increase in the next round since + * it is already moved ahead. + */ + j--; + continue; + } + + if (ocfs2_xattr_is_local(xe)) + continue; + + ret = func(sb, old_bh, xh, i, &xv, NULL, para); + if (ret) { + mlog_errno(ret); + break; + } + + ret = func(sb, new_bh, new_xh, j, &new_xv, &value_bh, para); + if (ret) { + mlog_errno(ret); + break; + } + + /* + * For the xattr which has l_tree_depth = 0, all the extent + * recs have already be copied to the new xh with the + * propriate OCFS2_EXT_REFCOUNTED flag we just need to + * increase the refount count int the refcount tree. + * + * For the xattr which has l_tree_depth > 0, we need + * to initialize it to the empty default value root, + * and then insert the extents one by one. + */ + if (xv->xr_list.l_tree_depth) { + memcpy(new_xv, &def_xv, OCFS2_XATTR_ROOT_SIZE); + vb->vb_xv = new_xv; + vb->vb_bh = value_bh; + ocfs2_init_xattr_value_extent_tree(&data_et, + INODE_CACHE(args->new_inode), vb); + } + + clusters = le32_to_cpu(xv->xr_clusters); + cpos = 0; + while (cpos < clusters) { + ret = ocfs2_xattr_get_clusters(args->old_inode, + cpos, + &p_cluster, + &num_clusters, + &xv->xr_list, + &ext_flags); + if (ret) { + mlog_errno(ret); + goto out; + } + + BUG_ON(!p_cluster); + + if (xv->xr_list.l_tree_depth) { + ret = ocfs2_insert_extent(handle, + &data_et, cpos, + ocfs2_clusters_to_blocks( + args->old_inode->i_sb, + p_cluster), + num_clusters, ext_flags, + meta_ac); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + ret = ocfs2_increase_refcount(handle, args->ref_ci, + args->ref_root_bh, + p_cluster, num_clusters, + meta_ac, args->dealloc); + if (ret) { + mlog_errno(ret); + goto out; + } + + cpos += num_clusters; + } + } + +out: + return ret; +} + +static int ocfs2_reflink_xattr_inline(struct ocfs2_xattr_reflink *args) +{ + int ret = 0, credits = 0; + handle_t *handle; + struct ocfs2_super *osb = OCFS2_SB(args->old_inode->i_sb); + struct ocfs2_dinode *di = (struct ocfs2_dinode *)args->old_bh->b_data; + int inline_size; + int header_off; + struct ocfs2_xattr_header *xh; + struct ocfs2_xattr_header *new_xh; + struct ocfs2_alloc_context *meta_ac = NULL; + struct ocfs2_inode_info *new_oi; + struct ocfs2_dinode *new_di; + struct ocfs2_xattr_value_buf vb = { + .vb_bh = args->new_bh, + .vb_access = ocfs2_journal_access_di, + }; + + ret = ocfs2_xattr_ibody_lookup_header(args->old_inode, di, &xh); + if (ret) + goto out; + + inline_size = le16_to_cpu(di->i_xattr_inline_size); + header_off = osb->sb->s_blocksize - inline_size; + new_xh = (struct ocfs2_xattr_header *) + (args->new_bh->b_data + header_off); + + ret = ocfs2_reflink_lock_xattr_allocators(osb, xh, args->ref_root_bh, + &credits, &meta_ac); + if (ret) { + mlog_errno(ret); + goto out; + } + + handle = ocfs2_start_trans(osb, credits); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out; + } + + ret = ocfs2_journal_access_di(handle, INODE_CACHE(args->new_inode), + args->new_bh, OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + memcpy(args->new_bh->b_data + header_off, + args->old_bh->b_data + header_off, inline_size); + + new_di = (struct ocfs2_dinode *)args->new_bh->b_data; + new_di->i_xattr_inline_size = cpu_to_le16(inline_size); + + ret = ocfs2_reflink_xattr_header(handle, args, args->old_bh, xh, + args->new_bh, new_xh, &vb, meta_ac, + ocfs2_get_xattr_value_root, NULL); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + new_oi = OCFS2_I(args->new_inode); + + spin_lock(&new_oi->ip_lock); + new_oi->ip_dyn_features |= OCFS2_HAS_XATTR_FL | OCFS2_INLINE_XATTR_FL; + new_di->i_dyn_features = cpu_to_le16(new_oi->ip_dyn_features); + spin_unlock(&new_oi->ip_lock); + + ocfs2_journal_dirty(handle, args->new_bh); + +out_commit: + ocfs2_commit_trans(osb, handle); + +out: + if (meta_ac) + ocfs2_free_alloc_context(meta_ac); + return ret; +} + +static int ocfs2_create_empty_xattr_block(struct inode *inode, + struct buffer_head *fe_bh, + struct buffer_head **ret_bh, + int indexed) +{ + int ret; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct ocfs2_xattr_set_ctxt ctxt; + + memset(&ctxt, 0, sizeof(ctxt)); + ret = ocfs2_reserve_new_metadata_blocks(osb, 1, &ctxt.meta_ac); + if (ret < 0) { + mlog_errno(ret); + return ret; + } + + ctxt.handle = ocfs2_start_trans(osb, OCFS2_XATTR_BLOCK_CREATE_CREDITS); + if (IS_ERR(ctxt.handle)) { + ret = PTR_ERR(ctxt.handle); + mlog_errno(ret); + goto out; + } + + trace_ocfs2_create_empty_xattr_block( + (unsigned long long)fe_bh->b_blocknr, indexed); + ret = ocfs2_create_xattr_block(inode, fe_bh, &ctxt, indexed, + ret_bh); + if (ret) + mlog_errno(ret); + + ocfs2_commit_trans(osb, ctxt.handle); +out: + ocfs2_free_alloc_context(ctxt.meta_ac); + return ret; +} + +static int ocfs2_reflink_xattr_block(struct ocfs2_xattr_reflink *args, + struct buffer_head *blk_bh, + struct buffer_head *new_blk_bh) +{ + int ret = 0, credits = 0; + handle_t *handle; + struct ocfs2_inode_info *new_oi = OCFS2_I(args->new_inode); + struct ocfs2_dinode *new_di; + struct ocfs2_super *osb = OCFS2_SB(args->new_inode->i_sb); + int header_off = offsetof(struct ocfs2_xattr_block, xb_attrs.xb_header); + struct ocfs2_xattr_block *xb = + (struct ocfs2_xattr_block *)blk_bh->b_data; + struct ocfs2_xattr_header *xh = &xb->xb_attrs.xb_header; + struct ocfs2_xattr_block *new_xb = + (struct ocfs2_xattr_block *)new_blk_bh->b_data; + struct ocfs2_xattr_header *new_xh = &new_xb->xb_attrs.xb_header; + struct ocfs2_alloc_context *meta_ac; + struct ocfs2_xattr_value_buf vb = { + .vb_bh = new_blk_bh, + .vb_access = ocfs2_journal_access_xb, + }; + + ret = ocfs2_reflink_lock_xattr_allocators(osb, xh, args->ref_root_bh, + &credits, &meta_ac); + if (ret) { + mlog_errno(ret); + return ret; + } + + /* One more credits in case we need to add xattr flags in new inode. */ + handle = ocfs2_start_trans(osb, credits + 1); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out; + } + + if (!(new_oi->ip_dyn_features & OCFS2_HAS_XATTR_FL)) { + ret = ocfs2_journal_access_di(handle, + INODE_CACHE(args->new_inode), + args->new_bh, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + } + + ret = ocfs2_journal_access_xb(handle, INODE_CACHE(args->new_inode), + new_blk_bh, OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + memcpy(new_blk_bh->b_data + header_off, blk_bh->b_data + header_off, + osb->sb->s_blocksize - header_off); + + ret = ocfs2_reflink_xattr_header(handle, args, blk_bh, xh, + new_blk_bh, new_xh, &vb, meta_ac, + ocfs2_get_xattr_value_root, NULL); + if (ret) { + mlog_errno(ret); + goto out_commit; + } + + ocfs2_journal_dirty(handle, new_blk_bh); + + if (!(new_oi->ip_dyn_features & OCFS2_HAS_XATTR_FL)) { + new_di = (struct ocfs2_dinode *)args->new_bh->b_data; + spin_lock(&new_oi->ip_lock); + new_oi->ip_dyn_features |= OCFS2_HAS_XATTR_FL; + new_di->i_dyn_features = cpu_to_le16(new_oi->ip_dyn_features); + spin_unlock(&new_oi->ip_lock); + + ocfs2_journal_dirty(handle, args->new_bh); + } + +out_commit: + ocfs2_commit_trans(osb, handle); + +out: + ocfs2_free_alloc_context(meta_ac); + return ret; +} + +struct ocfs2_reflink_xattr_tree_args { + struct ocfs2_xattr_reflink *reflink; + struct buffer_head *old_blk_bh; + struct buffer_head *new_blk_bh; + struct ocfs2_xattr_bucket *old_bucket; + struct ocfs2_xattr_bucket *new_bucket; +}; + +/* + * NOTE: + * We have to handle the case that both old bucket and new bucket + * will call this function to get the right ret_bh. + * So The caller must give us the right bh. + */ +static int ocfs2_get_reflink_xattr_value_root(struct super_block *sb, + struct buffer_head *bh, + struct ocfs2_xattr_header *xh, + int offset, + struct ocfs2_xattr_value_root **xv, + struct buffer_head **ret_bh, + void *para) +{ + struct ocfs2_reflink_xattr_tree_args *args = + (struct ocfs2_reflink_xattr_tree_args *)para; + struct ocfs2_xattr_bucket *bucket; + + if (bh == args->old_bucket->bu_bhs[0]) + bucket = args->old_bucket; + else + bucket = args->new_bucket; + + return ocfs2_get_xattr_tree_value_root(sb, bucket, offset, + xv, ret_bh); +} + +struct ocfs2_value_tree_metas { + int num_metas; + int credits; + int num_recs; +}; + +static int ocfs2_value_tree_metas_in_bucket(struct super_block *sb, + struct buffer_head *bh, + struct ocfs2_xattr_header *xh, + int offset, + struct ocfs2_xattr_value_root **xv, + struct buffer_head **ret_bh, + void *para) +{ + struct ocfs2_xattr_bucket *bucket = + (struct ocfs2_xattr_bucket *)para; + + return ocfs2_get_xattr_tree_value_root(sb, bucket, offset, + xv, ret_bh); +} + +static int ocfs2_calc_value_tree_metas(struct inode *inode, + struct ocfs2_xattr_bucket *bucket, + void *para) +{ + struct ocfs2_value_tree_metas *metas = + (struct ocfs2_value_tree_metas *)para; + struct ocfs2_xattr_header *xh = + (struct ocfs2_xattr_header *)bucket->bu_bhs[0]->b_data; + + /* Add the credits for this bucket first. */ + metas->credits += bucket->bu_blocks; + return ocfs2_value_metas_in_xattr_header(inode->i_sb, bucket->bu_bhs[0], + xh, &metas->num_metas, + &metas->credits, &metas->num_recs, + ocfs2_value_tree_metas_in_bucket, + bucket); +} + +/* + * Given a xattr extent rec starting from blkno and having len clusters, + * iterate all the buckets calculate how much metadata we need for reflinking + * all the ocfs2_xattr_value_root and lock the allocators accordingly. + */ +static int ocfs2_lock_reflink_xattr_rec_allocators( + struct ocfs2_reflink_xattr_tree_args *args, + struct ocfs2_extent_tree *xt_et, + u64 blkno, u32 len, int *credits, + struct ocfs2_alloc_context **meta_ac, + struct ocfs2_alloc_context **data_ac) +{ + int ret, num_free_extents; + struct ocfs2_value_tree_metas metas; + struct ocfs2_super *osb = OCFS2_SB(args->reflink->old_inode->i_sb); + struct ocfs2_refcount_block *rb; + + memset(&metas, 0, sizeof(metas)); + + ret = ocfs2_iterate_xattr_buckets(args->reflink->old_inode, blkno, len, + ocfs2_calc_value_tree_metas, &metas); + if (ret) { + mlog_errno(ret); + goto out; + } + + *credits = metas.credits; + + /* + * Calculate we need for refcount tree change. + * + * We need to add/modify num_recs in refcount tree, so just calculate + * an approximate number we need for refcount tree change. + * Sometimes we need to split the tree, and after split, half recs + * will be moved to the new block, and a new block can only provide + * half number of recs. So we multiple new blocks by 2. + * In the end, we have to add credits for modifying the already + * existed refcount block. + */ + rb = (struct ocfs2_refcount_block *)args->reflink->ref_root_bh->b_data; + metas.num_recs = + (metas.num_recs + ocfs2_refcount_recs_per_rb(osb->sb) - 1) / + ocfs2_refcount_recs_per_rb(osb->sb) * 2; + metas.num_metas += metas.num_recs; + *credits += metas.num_recs + + metas.num_recs * OCFS2_EXPAND_REFCOUNT_TREE_CREDITS; + if (le32_to_cpu(rb->rf_flags) & OCFS2_REFCOUNT_TREE_FL) + *credits += le16_to_cpu(rb->rf_list.l_tree_depth) * + le16_to_cpu(rb->rf_list.l_next_free_rec) + 1; + else + *credits += 1; + + /* count in the xattr tree change. */ + num_free_extents = ocfs2_num_free_extents(xt_et); + if (num_free_extents < 0) { + ret = num_free_extents; + mlog_errno(ret); + goto out; + } + + if (num_free_extents < len) + metas.num_metas += ocfs2_extend_meta_needed(xt_et->et_root_el); + + *credits += ocfs2_calc_extend_credits(osb->sb, + xt_et->et_root_el); + + if (metas.num_metas) { + ret = ocfs2_reserve_new_metadata_blocks(osb, metas.num_metas, + meta_ac); + if (ret) { + mlog_errno(ret); + goto out; + } + } + + if (len) { + ret = ocfs2_reserve_clusters(osb, len, data_ac); + if (ret) + mlog_errno(ret); + } +out: + if (ret) { + if (*meta_ac) { + ocfs2_free_alloc_context(*meta_ac); + *meta_ac = NULL; + } + } + + return ret; +} + +static int ocfs2_reflink_xattr_bucket(handle_t *handle, + u64 blkno, u64 new_blkno, u32 clusters, + u32 *cpos, int num_buckets, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_alloc_context *data_ac, + struct ocfs2_reflink_xattr_tree_args *args) +{ + int i, j, ret = 0; + struct super_block *sb = args->reflink->old_inode->i_sb; + int bpb = args->old_bucket->bu_blocks; + struct ocfs2_xattr_value_buf vb = { + .vb_access = ocfs2_journal_access, + }; + + for (i = 0; i < num_buckets; i++, blkno += bpb, new_blkno += bpb) { + ret = ocfs2_read_xattr_bucket(args->old_bucket, blkno); + if (ret) { + mlog_errno(ret); + break; + } + + ret = ocfs2_init_xattr_bucket(args->new_bucket, new_blkno, 1); + if (ret) { + mlog_errno(ret); + break; + } + + ret = ocfs2_xattr_bucket_journal_access(handle, + args->new_bucket, + OCFS2_JOURNAL_ACCESS_CREATE); + if (ret) { + mlog_errno(ret); + break; + } + + for (j = 0; j < bpb; j++) + memcpy(bucket_block(args->new_bucket, j), + bucket_block(args->old_bucket, j), + sb->s_blocksize); + + /* + * Record the start cpos so that we can use it to initialize + * our xattr tree we also set the xh_num_bucket for the new + * bucket. + */ + if (i == 0) { + *cpos = le32_to_cpu(bucket_xh(args->new_bucket)-> + xh_entries[0].xe_name_hash); + bucket_xh(args->new_bucket)->xh_num_buckets = + cpu_to_le16(num_buckets); + } + + ocfs2_xattr_bucket_journal_dirty(handle, args->new_bucket); + + ret = ocfs2_reflink_xattr_header(handle, args->reflink, + args->old_bucket->bu_bhs[0], + bucket_xh(args->old_bucket), + args->new_bucket->bu_bhs[0], + bucket_xh(args->new_bucket), + &vb, meta_ac, + ocfs2_get_reflink_xattr_value_root, + args); + if (ret) { + mlog_errno(ret); + break; + } + + /* + * Re-access and dirty the bucket to calculate metaecc. + * Because we may extend the transaction in reflink_xattr_header + * which will let the already accessed block gone. + */ + ret = ocfs2_xattr_bucket_journal_access(handle, + args->new_bucket, + OCFS2_JOURNAL_ACCESS_WRITE); + if (ret) { + mlog_errno(ret); + break; + } + + ocfs2_xattr_bucket_journal_dirty(handle, args->new_bucket); + + ocfs2_xattr_bucket_relse(args->old_bucket); + ocfs2_xattr_bucket_relse(args->new_bucket); + } + + ocfs2_xattr_bucket_relse(args->old_bucket); + ocfs2_xattr_bucket_relse(args->new_bucket); + return ret; +} + +static int ocfs2_reflink_xattr_buckets(handle_t *handle, + struct inode *inode, + struct ocfs2_reflink_xattr_tree_args *args, + struct ocfs2_extent_tree *et, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_alloc_context *data_ac, + u64 blkno, u32 cpos, u32 len) +{ + int ret, first_inserted = 0; + u32 p_cluster, num_clusters, reflink_cpos = 0; + u64 new_blkno; + unsigned int num_buckets, reflink_buckets; + unsigned int bpc = + ocfs2_xattr_buckets_per_cluster(OCFS2_SB(inode->i_sb)); + + ret = ocfs2_read_xattr_bucket(args->old_bucket, blkno); + if (ret) { + mlog_errno(ret); + goto out; + } + num_buckets = le16_to_cpu(bucket_xh(args->old_bucket)->xh_num_buckets); + ocfs2_xattr_bucket_relse(args->old_bucket); + + while (len && num_buckets) { + ret = ocfs2_claim_clusters(handle, data_ac, + 1, &p_cluster, &num_clusters); + if (ret) { + mlog_errno(ret); + goto out; + } + + new_blkno = ocfs2_clusters_to_blocks(inode->i_sb, p_cluster); + reflink_buckets = min(num_buckets, bpc * num_clusters); + + ret = ocfs2_reflink_xattr_bucket(handle, blkno, + new_blkno, num_clusters, + &reflink_cpos, reflink_buckets, + meta_ac, data_ac, args); + if (ret) { + mlog_errno(ret); + goto out; + } + + /* + * For the 1st allocated cluster, we make it use the same cpos + * so that the xattr tree looks the same as the original one + * in the most case. + */ + if (!first_inserted) { + reflink_cpos = cpos; + first_inserted = 1; + } + ret = ocfs2_insert_extent(handle, et, reflink_cpos, new_blkno, + num_clusters, 0, meta_ac); + if (ret) + mlog_errno(ret); + + trace_ocfs2_reflink_xattr_buckets((unsigned long long)new_blkno, + num_clusters, reflink_cpos); + + len -= num_clusters; + blkno += ocfs2_clusters_to_blocks(inode->i_sb, num_clusters); + num_buckets -= reflink_buckets; + } +out: + return ret; +} + +/* + * Create the same xattr extent record in the new inode's xattr tree. + */ +static int ocfs2_reflink_xattr_rec(struct inode *inode, + struct buffer_head *root_bh, + u64 blkno, + u32 cpos, + u32 len, + void *para) +{ + int ret, credits = 0; + handle_t *handle; + struct ocfs2_reflink_xattr_tree_args *args = + (struct ocfs2_reflink_xattr_tree_args *)para; + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + struct ocfs2_alloc_context *meta_ac = NULL; + struct ocfs2_alloc_context *data_ac = NULL; + struct ocfs2_extent_tree et; + + trace_ocfs2_reflink_xattr_rec((unsigned long long)blkno, len); + + ocfs2_init_xattr_tree_extent_tree(&et, + INODE_CACHE(args->reflink->new_inode), + args->new_blk_bh); + + ret = ocfs2_lock_reflink_xattr_rec_allocators(args, &et, blkno, + len, &credits, + &meta_ac, &data_ac); + if (ret) { + mlog_errno(ret); + goto out; + } + + handle = ocfs2_start_trans(osb, credits); + if (IS_ERR(handle)) { + ret = PTR_ERR(handle); + mlog_errno(ret); + goto out; + } + + ret = ocfs2_reflink_xattr_buckets(handle, inode, args, &et, + meta_ac, data_ac, + blkno, cpos, len); + if (ret) + mlog_errno(ret); + + ocfs2_commit_trans(osb, handle); + +out: + if (meta_ac) + ocfs2_free_alloc_context(meta_ac); + if (data_ac) + ocfs2_free_alloc_context(data_ac); + return ret; +} + +/* + * Create reflinked xattr buckets. + * We will add bucket one by one, and refcount all the xattrs in the bucket + * if they are stored outside. + */ +static int ocfs2_reflink_xattr_tree(struct ocfs2_xattr_reflink *args, + struct buffer_head *blk_bh, + struct buffer_head *new_blk_bh) +{ + int ret; + struct ocfs2_reflink_xattr_tree_args para; + + memset(¶, 0, sizeof(para)); + para.reflink = args; + para.old_blk_bh = blk_bh; + para.new_blk_bh = new_blk_bh; + + para.old_bucket = ocfs2_xattr_bucket_new(args->old_inode); + if (!para.old_bucket) { + mlog_errno(-ENOMEM); + return -ENOMEM; + } + + para.new_bucket = ocfs2_xattr_bucket_new(args->new_inode); + if (!para.new_bucket) { + ret = -ENOMEM; + mlog_errno(ret); + goto out; + } + + ret = ocfs2_iterate_xattr_index_block(args->old_inode, blk_bh, + ocfs2_reflink_xattr_rec, + ¶); + if (ret) + mlog_errno(ret); + +out: + ocfs2_xattr_bucket_free(para.old_bucket); + ocfs2_xattr_bucket_free(para.new_bucket); + return ret; +} + +static int ocfs2_reflink_xattr_in_block(struct ocfs2_xattr_reflink *args, + struct buffer_head *blk_bh) +{ + int ret, indexed = 0; + struct buffer_head *new_blk_bh = NULL; + struct ocfs2_xattr_block *xb = + (struct ocfs2_xattr_block *)blk_bh->b_data; + + + if (le16_to_cpu(xb->xb_flags) & OCFS2_XATTR_INDEXED) + indexed = 1; + + ret = ocfs2_create_empty_xattr_block(args->new_inode, args->new_bh, + &new_blk_bh, indexed); + if (ret) { + mlog_errno(ret); + goto out; + } + + if (!indexed) + ret = ocfs2_reflink_xattr_block(args, blk_bh, new_blk_bh); + else + ret = ocfs2_reflink_xattr_tree(args, blk_bh, new_blk_bh); + if (ret) + mlog_errno(ret); + +out: + brelse(new_blk_bh); + return ret; +} + +static int ocfs2_reflink_xattr_no_security(struct ocfs2_xattr_entry *xe) +{ + int type = ocfs2_xattr_get_type(xe); + + return type != OCFS2_XATTR_INDEX_SECURITY && + type != OCFS2_XATTR_INDEX_POSIX_ACL_ACCESS && + type != OCFS2_XATTR_INDEX_POSIX_ACL_DEFAULT; +} + +int ocfs2_reflink_xattrs(struct inode *old_inode, + struct buffer_head *old_bh, + struct inode *new_inode, + struct buffer_head *new_bh, + bool preserve_security) +{ + int ret; + struct ocfs2_xattr_reflink args; + struct ocfs2_inode_info *oi = OCFS2_I(old_inode); + struct ocfs2_dinode *di = (struct ocfs2_dinode *)old_bh->b_data; + struct buffer_head *blk_bh = NULL; + struct ocfs2_cached_dealloc_ctxt dealloc; + struct ocfs2_refcount_tree *ref_tree; + struct buffer_head *ref_root_bh = NULL; + + ret = ocfs2_lock_refcount_tree(OCFS2_SB(old_inode->i_sb), + le64_to_cpu(di->i_refcount_loc), + 1, &ref_tree, &ref_root_bh); + if (ret) { + mlog_errno(ret); + goto out; + } + + ocfs2_init_dealloc_ctxt(&dealloc); + + args.old_inode = old_inode; + args.new_inode = new_inode; + args.old_bh = old_bh; + args.new_bh = new_bh; + args.ref_ci = &ref_tree->rf_ci; + args.ref_root_bh = ref_root_bh; + args.dealloc = &dealloc; + if (preserve_security) + args.xattr_reflinked = NULL; + else + args.xattr_reflinked = ocfs2_reflink_xattr_no_security; + + if (oi->ip_dyn_features & OCFS2_INLINE_XATTR_FL) { + ret = ocfs2_reflink_xattr_inline(&args); + if (ret) { + mlog_errno(ret); + goto out_unlock; + } + } + + if (!di->i_xattr_loc) + goto out_unlock; + + ret = ocfs2_read_xattr_block(old_inode, le64_to_cpu(di->i_xattr_loc), + &blk_bh); + if (ret < 0) { + mlog_errno(ret); + goto out_unlock; + } + + ret = ocfs2_reflink_xattr_in_block(&args, blk_bh); + if (ret) + mlog_errno(ret); + + brelse(blk_bh); + +out_unlock: + ocfs2_unlock_refcount_tree(OCFS2_SB(old_inode->i_sb), + ref_tree, 1); + brelse(ref_root_bh); + + if (ocfs2_dealloc_has_cluster(&dealloc)) + ocfs2_schedule_truncate_log_flush(OCFS2_SB(old_inode->i_sb), 1); + ocfs2_run_deallocs(OCFS2_SB(old_inode->i_sb), &dealloc); + +out: + return ret; +} + +/* + * Initialize security and acl for a already created inode. + * Used for reflink a non-preserve-security file. + * + * It uses common api like ocfs2_xattr_set, so the caller + * must not hold any lock expect i_rwsem. + */ +int ocfs2_init_security_and_acl(struct inode *dir, + struct inode *inode, + const struct qstr *qstr) +{ + int ret = 0; + struct buffer_head *dir_bh = NULL; + struct ocfs2_acl_state acl_state = { 0 }; + + ret = ocfs2_init_security_get(inode, dir, qstr, NULL); + if (ret) { + mlog_errno(ret); + goto leave; + } + + ret = ocfs2_inode_lock(dir, &dir_bh, 0); + if (ret) { + mlog_errno(ret); + goto leave; + } + + ret = ocfs2_acl_init_prepare(inode, dir, dir_bh, &acl_state); + if (ret) + goto unlock; + + ret = ocfs2_init_acl(NULL, inode, NULL, NULL, NULL, &acl_state); + if (ret) + mlog_errno(ret); + +unlock: + ocfs2_acl_init_release(&acl_state); + ocfs2_inode_unlock(dir, 0); + brelse(dir_bh); +leave: + return ret; +} + +/* + * 'security' attributes support + */ +static int ocfs2_xattr_security_get(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + return ocfs2_xattr_get(inode, OCFS2_XATTR_INDEX_SECURITY, + name, buffer, size); +} + +static int ocfs2_xattr_security_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +{ + return ocfs2_xattr_set(inode, OCFS2_XATTR_INDEX_SECURITY, + name, value, size, flags); +} + +static int ocfs2_initxattrs(struct inode *inode, const struct xattr *xattr_array, + void *fs_info) +{ + struct ocfs2_security_xattr_info *si = fs_info; + const struct xattr *xattr; + int err = 0; + + if (si) { + si->value = kmemdup(xattr_array->value, xattr_array->value_len, + GFP_KERNEL); + if (!si->value) + return -ENOMEM; + + si->name = xattr_array->name; + si->value_len = xattr_array->value_len; + return 0; + } + + for (xattr = xattr_array; xattr->name != NULL; xattr++) { + err = ocfs2_xattr_set(inode, OCFS2_XATTR_INDEX_SECURITY, + xattr->name, xattr->value, + xattr->value_len, XATTR_CREATE); + if (err) + break; + } + return err; +} + +int ocfs2_init_security_get(struct inode *inode, + struct inode *dir, + const struct qstr *qstr, + struct ocfs2_security_xattr_info *si) +{ + int ret; + + /* check whether ocfs2 support feature xattr */ + if (!ocfs2_supports_xattr(OCFS2_SB(dir->i_sb))) + return -EOPNOTSUPP; + if (si) { + ret = security_inode_init_security(inode, dir, qstr, + &ocfs2_initxattrs, si); + /* + * security_inode_init_security() does not return -EOPNOTSUPP, + * we have to check the xattr ourselves. + */ + if (!ret && !si->name) + si->enable = 0; + + return ret; + } + + return security_inode_init_security(inode, dir, qstr, + &ocfs2_initxattrs, NULL); +} + +int ocfs2_init_security_set(handle_t *handle, + struct inode *inode, + struct buffer_head *di_bh, + struct ocfs2_security_xattr_info *si, + struct ocfs2_alloc_context *xattr_ac, + struct ocfs2_alloc_context *data_ac) +{ + return ocfs2_xattr_set_handle(handle, inode, di_bh, + OCFS2_XATTR_INDEX_SECURITY, + si->name, si->value, si->value_len, 0, + xattr_ac, data_ac); +} + +const struct xattr_handler ocfs2_xattr_security_handler = { + .prefix = XATTR_SECURITY_PREFIX, + .get = ocfs2_xattr_security_get, + .set = ocfs2_xattr_security_set, +}; + +/* + * 'trusted' attributes support + */ +static int ocfs2_xattr_trusted_get(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + return ocfs2_xattr_get(inode, OCFS2_XATTR_INDEX_TRUSTED, + name, buffer, size); +} + +static int ocfs2_xattr_trusted_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +{ + return ocfs2_xattr_set(inode, OCFS2_XATTR_INDEX_TRUSTED, + name, value, size, flags); +} + +const struct xattr_handler ocfs2_xattr_trusted_handler = { + .prefix = XATTR_TRUSTED_PREFIX, + .get = ocfs2_xattr_trusted_get, + .set = ocfs2_xattr_trusted_set, +}; + +/* + * 'user' attributes support + */ +static int ocfs2_xattr_user_get(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + if (osb->s_mount_opt & OCFS2_MOUNT_NOUSERXATTR) + return -EOPNOTSUPP; + return ocfs2_xattr_get(inode, OCFS2_XATTR_INDEX_USER, name, + buffer, size); +} + +static int ocfs2_xattr_user_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +{ + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + + if (osb->s_mount_opt & OCFS2_MOUNT_NOUSERXATTR) + return -EOPNOTSUPP; + + return ocfs2_xattr_set(inode, OCFS2_XATTR_INDEX_USER, + name, value, size, flags); +} + +const struct xattr_handler ocfs2_xattr_user_handler = { + .prefix = XATTR_USER_PREFIX, + .get = ocfs2_xattr_user_get, + .set = ocfs2_xattr_user_set, +}; diff --git a/fs/ocfs2/xattr.h b/fs/ocfs2/xattr.h new file mode 100644 index 000000000..5e1851327 --- /dev/null +++ b/fs/ocfs2/xattr.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * xattr.h + * + * Copyright (C) 2004, 2008 Oracle. All rights reserved. + */ + +#ifndef OCFS2_XATTR_H +#define OCFS2_XATTR_H + +#include +#include + +enum ocfs2_xattr_type { + OCFS2_XATTR_INDEX_USER = 1, + OCFS2_XATTR_INDEX_POSIX_ACL_ACCESS, + OCFS2_XATTR_INDEX_POSIX_ACL_DEFAULT, + OCFS2_XATTR_INDEX_TRUSTED, + OCFS2_XATTR_INDEX_SECURITY, + OCFS2_XATTR_MAX +}; + +struct ocfs2_security_xattr_info { + int enable; + const char *name; + void *value; + size_t value_len; +}; + +extern const struct xattr_handler ocfs2_xattr_user_handler; +extern const struct xattr_handler ocfs2_xattr_trusted_handler; +extern const struct xattr_handler ocfs2_xattr_security_handler; +extern const struct xattr_handler * const ocfs2_xattr_handlers[]; + +ssize_t ocfs2_listxattr(struct dentry *, char *, size_t); +int ocfs2_xattr_get_nolock(struct inode *, struct buffer_head *, int, + const char *, void *, size_t); +int ocfs2_xattr_set(struct inode *, int, const char *, const void *, + size_t, int); +int ocfs2_xattr_set_handle(handle_t *, struct inode *, struct buffer_head *, + int, const char *, const void *, size_t, int, + struct ocfs2_alloc_context *, + struct ocfs2_alloc_context *); +int ocfs2_has_inline_xattr_value_outside(struct inode *inode, + struct ocfs2_dinode *di); +int ocfs2_validate_inode_xattr(struct super_block *sb, u64 blkno, + struct ocfs2_dinode *di); +int ocfs2_xattr_remove(struct inode *, struct buffer_head *); +int ocfs2_init_security_get(struct inode *, struct inode *, + const struct qstr *, + struct ocfs2_security_xattr_info *); +int ocfs2_init_security_set(handle_t *, struct inode *, + struct buffer_head *, + struct ocfs2_security_xattr_info *, + struct ocfs2_alloc_context *, + struct ocfs2_alloc_context *); +int ocfs2_calc_security_init(struct inode *, + struct ocfs2_security_xattr_info *, + int *, int *, struct ocfs2_alloc_context **); + +struct ocfs2_acl_state; +int ocfs2_calc_xattr_init(struct inode *dir, umode_t mode, + struct ocfs2_security_xattr_info *si, + int *want_clusters, int *xattr_credits, + int *want_meta, struct ocfs2_acl_state *acl_state); + +/* + * xattrs can live inside an inode, as part of an external xattr block, + * or inside an xattr bucket, which is the leaf of a tree rooted in an + * xattr block. Some of the xattr calls, especially the value setting + * functions, want to treat each of these locations as equal. Let's wrap + * them in a structure that we can pass around instead of raw buffer_heads. + */ +struct ocfs2_xattr_value_buf { + struct buffer_head *vb_bh; + ocfs2_journal_access_func vb_access; + struct ocfs2_xattr_value_root *vb_xv; +}; + +int ocfs2_xattr_attach_refcount_tree(struct inode *inode, + struct buffer_head *fe_bh, + struct ocfs2_caching_info *ref_ci, + struct buffer_head *ref_root_bh, + struct ocfs2_cached_dealloc_ctxt *dealloc); +int ocfs2_reflink_xattrs(struct inode *old_inode, + struct buffer_head *old_bh, + struct inode *new_inode, + struct buffer_head *new_bh, + bool preserve_security); +int ocfs2_init_security_and_acl(struct inode *dir, + struct inode *inode, + const struct qstr *qstr); +#endif /* OCFS2_XATTR_H */ diff --git a/fs/omfs/Kconfig b/fs/omfs/Kconfig new file mode 100644 index 000000000..8470f6c3e --- /dev/null +++ b/fs/omfs/Kconfig @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0-only +config OMFS_FS + tristate "SonicBlue Optimized MPEG File System support" + depends on BLOCK + select BUFFER_HEAD + select CRC_ITU_T + help + This is the proprietary file system used by the Rio Karma music + player and ReplayTV DVR. Despite the name, this filesystem is not + more efficient than a standard FS for MPEG files, in fact likely + the opposite is true. Say Y if you have either of these devices + and wish to mount its disk. + + To compile this file system support as a module, choose M here: the + module will be called omfs. If unsure, say N. diff --git a/fs/omfs/Makefile b/fs/omfs/Makefile new file mode 100644 index 000000000..ae16fc3d3 --- /dev/null +++ b/fs/omfs/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-$(CONFIG_OMFS_FS) += omfs.o + +omfs-y := bitmap.o dir.o file.o inode.o diff --git a/fs/omfs/bitmap.c b/fs/omfs/bitmap.c new file mode 100644 index 000000000..7147ba6a6 --- /dev/null +++ b/fs/omfs/bitmap.c @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include "omfs.h" + +unsigned long omfs_count_free(struct super_block *sb) +{ + unsigned int i; + unsigned long sum = 0; + struct omfs_sb_info *sbi = OMFS_SB(sb); + int nbits = sb->s_blocksize * 8; + + for (i = 0; i < sbi->s_imap_size; i++) + sum += nbits - bitmap_weight(sbi->s_imap[i], nbits); + + return sum; +} + +/* + * Counts the run of zero bits starting at bit up to max. + * It handles the case where a run might spill over a buffer. + * Called with bitmap lock. + */ +static int count_run(unsigned long **addr, int nbits, + int addrlen, int bit, int max) +{ + int count = 0; + int x; + + for (; addrlen > 0; addrlen--, addr++) { + x = find_next_bit(*addr, nbits, bit); + count += x - bit; + + if (x < nbits || count > max) + return min(count, max); + + bit = 0; + } + return min(count, max); +} + +/* + * Sets or clears the run of count bits starting with bit. + * Called with bitmap lock. + */ +static int set_run(struct super_block *sb, int map, + int nbits, int bit, int count, int set) +{ + int i; + int err; + struct buffer_head *bh; + struct omfs_sb_info *sbi = OMFS_SB(sb); + + err = -ENOMEM; + bh = sb_bread(sb, clus_to_blk(sbi, sbi->s_bitmap_ino) + map); + if (!bh) + goto out; + + for (i = 0; i < count; i++, bit++) { + if (bit >= nbits) { + bit = 0; + map++; + + mark_buffer_dirty(bh); + brelse(bh); + bh = sb_bread(sb, + clus_to_blk(sbi, sbi->s_bitmap_ino) + map); + if (!bh) + goto out; + } + if (set) { + set_bit(bit, sbi->s_imap[map]); + set_bit(bit, (unsigned long *)bh->b_data); + } else { + clear_bit(bit, sbi->s_imap[map]); + clear_bit(bit, (unsigned long *)bh->b_data); + } + } + mark_buffer_dirty(bh); + brelse(bh); + err = 0; +out: + return err; +} + +/* + * Tries to allocate exactly one block. Returns true if successful. + */ +int omfs_allocate_block(struct super_block *sb, u64 block) +{ + struct buffer_head *bh; + struct omfs_sb_info *sbi = OMFS_SB(sb); + int bits_per_entry = 8 * sb->s_blocksize; + unsigned int map, bit; + int ret = 0; + u64 tmp; + + tmp = block; + bit = do_div(tmp, bits_per_entry); + map = tmp; + + mutex_lock(&sbi->s_bitmap_lock); + if (map >= sbi->s_imap_size || test_and_set_bit(bit, sbi->s_imap[map])) + goto out; + + if (sbi->s_bitmap_ino > 0) { + bh = sb_bread(sb, clus_to_blk(sbi, sbi->s_bitmap_ino) + map); + if (!bh) + goto out; + + set_bit(bit, (unsigned long *)bh->b_data); + mark_buffer_dirty(bh); + brelse(bh); + } + ret = 1; +out: + mutex_unlock(&sbi->s_bitmap_lock); + return ret; +} + + +/* + * Tries to allocate a set of blocks. The request size depends on the + * type: for inodes, we must allocate sbi->s_mirrors blocks, and for file + * blocks, we try to allocate sbi->s_clustersize, but can always get away + * with just one block. + */ +int omfs_allocate_range(struct super_block *sb, + int min_request, + int max_request, + u64 *return_block, + int *return_size) +{ + struct omfs_sb_info *sbi = OMFS_SB(sb); + int bits_per_entry = 8 * sb->s_blocksize; + int ret = 0; + int i, run, bit; + + mutex_lock(&sbi->s_bitmap_lock); + for (i = 0; i < sbi->s_imap_size; i++) { + bit = 0; + while (bit < bits_per_entry) { + bit = find_next_zero_bit(sbi->s_imap[i], bits_per_entry, + bit); + + if (bit == bits_per_entry) + break; + + run = count_run(&sbi->s_imap[i], bits_per_entry, + sbi->s_imap_size-i, bit, max_request); + + if (run >= min_request) + goto found; + bit += run; + } + } + ret = -ENOSPC; + goto out; + +found: + *return_block = (u64) i * bits_per_entry + bit; + *return_size = run; + ret = set_run(sb, i, bits_per_entry, bit, run, 1); + +out: + mutex_unlock(&sbi->s_bitmap_lock); + return ret; +} + +/* + * Clears count bits starting at a given block. + */ +int omfs_clear_range(struct super_block *sb, u64 block, int count) +{ + struct omfs_sb_info *sbi = OMFS_SB(sb); + int bits_per_entry = 8 * sb->s_blocksize; + u64 tmp; + unsigned int map, bit; + int ret; + + tmp = block; + bit = do_div(tmp, bits_per_entry); + map = tmp; + + if (map >= sbi->s_imap_size) + return 0; + + mutex_lock(&sbi->s_bitmap_lock); + ret = set_run(sb, map, bits_per_entry, bit, count, 0); + mutex_unlock(&sbi->s_bitmap_lock); + return ret; +} diff --git a/fs/omfs/dir.c b/fs/omfs/dir.c new file mode 100644 index 000000000..692297cf8 --- /dev/null +++ b/fs/omfs/dir.c @@ -0,0 +1,461 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * OMFS (as used by RIO Karma) directory operations. + * Copyright (C) 2005 Bob Copeland + */ + +#include +#include +#include +#include "omfs.h" + +static int omfs_hash(const char *name, int namelen, int mod) +{ + int i, hash = 0; + for (i = 0; i < namelen; i++) + hash ^= tolower(name[i]) << (i % 24); + return hash % mod; +} + +/* + * Finds the bucket for a given name and reads the containing block; + * *ofs is set to the offset of the first list entry. + */ +static struct buffer_head *omfs_get_bucket(struct inode *dir, + const char *name, int namelen, int *ofs) +{ + int nbuckets = (dir->i_size - OMFS_DIR_START)/8; + int bucket = omfs_hash(name, namelen, nbuckets); + + *ofs = OMFS_DIR_START + bucket * 8; + return omfs_bread(dir->i_sb, dir->i_ino); +} + +static struct buffer_head *omfs_scan_list(struct inode *dir, u64 block, + const char *name, int namelen, + u64 *prev_block) +{ + struct buffer_head *bh; + struct omfs_inode *oi; + int err = -ENOENT; + *prev_block = ~0; + + while (block != ~0) { + bh = omfs_bread(dir->i_sb, block); + if (!bh) { + err = -EIO; + goto err; + } + + oi = (struct omfs_inode *) bh->b_data; + if (omfs_is_bad(OMFS_SB(dir->i_sb), &oi->i_head, block)) { + brelse(bh); + goto err; + } + + if (strncmp(oi->i_name, name, namelen) == 0) + return bh; + + *prev_block = block; + block = be64_to_cpu(oi->i_sibling); + brelse(bh); + } +err: + return ERR_PTR(err); +} + +static struct buffer_head *omfs_find_entry(struct inode *dir, + const char *name, int namelen) +{ + struct buffer_head *bh; + int ofs; + u64 block, dummy; + + bh = omfs_get_bucket(dir, name, namelen, &ofs); + if (!bh) + return ERR_PTR(-EIO); + + block = be64_to_cpu(*((__be64 *) &bh->b_data[ofs])); + brelse(bh); + + return omfs_scan_list(dir, block, name, namelen, &dummy); +} + +int omfs_make_empty(struct inode *inode, struct super_block *sb) +{ + struct omfs_sb_info *sbi = OMFS_SB(sb); + struct buffer_head *bh; + struct omfs_inode *oi; + + bh = omfs_bread(sb, inode->i_ino); + if (!bh) + return -ENOMEM; + + memset(bh->b_data, 0, sizeof(struct omfs_inode)); + + if (S_ISDIR(inode->i_mode)) { + memset(&bh->b_data[OMFS_DIR_START], 0xff, + sbi->s_sys_blocksize - OMFS_DIR_START); + } else + omfs_make_empty_table(bh, OMFS_EXTENT_START); + + oi = (struct omfs_inode *) bh->b_data; + oi->i_head.h_self = cpu_to_be64(inode->i_ino); + oi->i_sibling = ~cpu_to_be64(0ULL); + + mark_buffer_dirty(bh); + brelse(bh); + return 0; +} + +static int omfs_add_link(struct dentry *dentry, struct inode *inode) +{ + struct inode *dir = d_inode(dentry->d_parent); + const char *name = dentry->d_name.name; + int namelen = dentry->d_name.len; + struct omfs_inode *oi; + struct buffer_head *bh; + u64 block; + __be64 *entry; + int ofs; + + /* just prepend to head of queue in proper bucket */ + bh = omfs_get_bucket(dir, name, namelen, &ofs); + if (!bh) + goto out; + + entry = (__be64 *) &bh->b_data[ofs]; + block = be64_to_cpu(*entry); + *entry = cpu_to_be64(inode->i_ino); + mark_buffer_dirty(bh); + brelse(bh); + + /* now set the sibling and parent pointers on the new inode */ + bh = omfs_bread(dir->i_sb, inode->i_ino); + if (!bh) + goto out; + + oi = (struct omfs_inode *) bh->b_data; + memcpy(oi->i_name, name, namelen); + memset(oi->i_name + namelen, 0, OMFS_NAMELEN - namelen); + oi->i_sibling = cpu_to_be64(block); + oi->i_parent = cpu_to_be64(dir->i_ino); + mark_buffer_dirty(bh); + brelse(bh); + + inode_set_ctime_current(dir); + + /* mark affected inodes dirty to rebuild checksums */ + mark_inode_dirty(dir); + mark_inode_dirty(inode); + return 0; +out: + return -ENOMEM; +} + +static int omfs_delete_entry(struct dentry *dentry) +{ + struct inode *dir = d_inode(dentry->d_parent); + struct inode *dirty; + const char *name = dentry->d_name.name; + int namelen = dentry->d_name.len; + struct omfs_inode *oi; + struct buffer_head *bh, *bh2; + __be64 *entry, next; + u64 block, prev; + int ofs; + int err = -ENOMEM; + + /* delete the proper node in the bucket's linked list */ + bh = omfs_get_bucket(dir, name, namelen, &ofs); + if (!bh) + goto out; + + entry = (__be64 *) &bh->b_data[ofs]; + block = be64_to_cpu(*entry); + + bh2 = omfs_scan_list(dir, block, name, namelen, &prev); + if (IS_ERR(bh2)) { + err = PTR_ERR(bh2); + goto out_free_bh; + } + + oi = (struct omfs_inode *) bh2->b_data; + next = oi->i_sibling; + brelse(bh2); + + if (prev != ~0) { + /* found in middle of list, get list ptr */ + brelse(bh); + bh = omfs_bread(dir->i_sb, prev); + if (!bh) + goto out; + + oi = (struct omfs_inode *) bh->b_data; + entry = &oi->i_sibling; + } + + *entry = next; + mark_buffer_dirty(bh); + + if (prev != ~0) { + dirty = omfs_iget(dir->i_sb, prev); + if (!IS_ERR(dirty)) { + mark_inode_dirty(dirty); + iput(dirty); + } + } + + err = 0; +out_free_bh: + brelse(bh); +out: + return err; +} + +static int omfs_dir_is_empty(struct inode *inode) +{ + int nbuckets = (inode->i_size - OMFS_DIR_START) / 8; + struct buffer_head *bh; + u64 *ptr; + int i; + + bh = omfs_bread(inode->i_sb, inode->i_ino); + + if (!bh) + return 0; + + ptr = (u64 *) &bh->b_data[OMFS_DIR_START]; + + for (i = 0; i < nbuckets; i++, ptr++) + if (*ptr != ~0) + break; + + brelse(bh); + return *ptr != ~0; +} + +static int omfs_remove(struct inode *dir, struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + int ret; + + + if (S_ISDIR(inode->i_mode) && + !omfs_dir_is_empty(inode)) + return -ENOTEMPTY; + + ret = omfs_delete_entry(dentry); + if (ret) + return ret; + + clear_nlink(inode); + mark_inode_dirty(inode); + mark_inode_dirty(dir); + return 0; +} + +static int omfs_add_node(struct inode *dir, struct dentry *dentry, umode_t mode) +{ + int err; + struct inode *inode = omfs_new_inode(dir, mode); + + if (IS_ERR(inode)) + return PTR_ERR(inode); + + err = omfs_make_empty(inode, dir->i_sb); + if (err) + goto out_free_inode; + + err = omfs_add_link(dentry, inode); + if (err) + goto out_free_inode; + + d_instantiate(dentry, inode); + return 0; + +out_free_inode: + iput(inode); + return err; +} + +static struct dentry *omfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + return ERR_PTR(omfs_add_node(dir, dentry, mode)); +} + +static int omfs_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + return omfs_add_node(dir, dentry, mode | S_IFREG); +} + +static struct dentry *omfs_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + struct buffer_head *bh; + struct inode *inode = NULL; + + if (dentry->d_name.len > OMFS_NAMELEN) + return ERR_PTR(-ENAMETOOLONG); + + bh = omfs_find_entry(dir, dentry->d_name.name, dentry->d_name.len); + if (!IS_ERR(bh)) { + struct omfs_inode *oi = (struct omfs_inode *)bh->b_data; + ino_t ino = be64_to_cpu(oi->i_head.h_self); + brelse(bh); + inode = omfs_iget(dir->i_sb, ino); + } else if (bh != ERR_PTR(-ENOENT)) { + inode = ERR_CAST(bh); + } + return d_splice_alias(inode, dentry); +} + +/* sanity check block's self pointer */ +int omfs_is_bad(struct omfs_sb_info *sbi, struct omfs_header *header, + u64 fsblock) +{ + int is_bad; + u64 ino = be64_to_cpu(header->h_self); + is_bad = ((ino != fsblock) || (ino < sbi->s_root_ino) || + (ino > sbi->s_num_blocks)); + + if (is_bad) + printk(KERN_WARNING "omfs: bad hash chain detected\n"); + + return is_bad; +} + +static bool omfs_fill_chain(struct inode *dir, struct dir_context *ctx, + u64 fsblock, int hindex) +{ + /* follow chain in this bucket */ + while (fsblock != ~0) { + struct buffer_head *bh = omfs_bread(dir->i_sb, fsblock); + struct omfs_inode *oi; + u64 self; + unsigned char d_type; + + if (!bh) + return true; + + oi = (struct omfs_inode *) bh->b_data; + if (omfs_is_bad(OMFS_SB(dir->i_sb), &oi->i_head, fsblock)) { + brelse(bh); + return true; + } + + self = fsblock; + fsblock = be64_to_cpu(oi->i_sibling); + + /* skip visited nodes */ + if (hindex) { + hindex--; + brelse(bh); + continue; + } + + d_type = (oi->i_type == OMFS_DIR) ? DT_DIR : DT_REG; + + if (!dir_emit(ctx, oi->i_name, + strnlen(oi->i_name, OMFS_NAMELEN), + self, d_type)) { + brelse(bh); + return false; + } + brelse(bh); + ctx->pos++; + } + return true; +} + +static int omfs_rename(struct mnt_idmap *idmap, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) +{ + struct inode *new_inode = d_inode(new_dentry); + struct inode *old_inode = d_inode(old_dentry); + int err; + + if (flags & ~RENAME_NOREPLACE) + return -EINVAL; + + if (new_inode) { + /* overwriting existing file/dir */ + err = omfs_remove(new_dir, new_dentry); + if (err) + goto out; + } + + /* since omfs locates files by name, we need to unlink _before_ + * adding the new link or we won't find the old one */ + err = omfs_delete_entry(old_dentry); + if (err) + goto out; + + mark_inode_dirty(old_dir); + err = omfs_add_link(new_dentry, old_inode); + if (err) + goto out; + + inode_set_ctime_current(old_inode); + mark_inode_dirty(old_inode); +out: + return err; +} + +static int omfs_readdir(struct file *file, struct dir_context *ctx) +{ + struct inode *dir = file_inode(file); + struct buffer_head *bh; + __be64 *p; + unsigned int hchain, hindex; + int nbuckets; + + if (ctx->pos >> 32) + return -EINVAL; + + if (ctx->pos < 1 << 20) { + if (!dir_emit_dots(file, ctx)) + return 0; + ctx->pos = 1 << 20; + } + + nbuckets = (dir->i_size - OMFS_DIR_START) / 8; + + /* high 12 bits store bucket + 1 and low 20 bits store hash index */ + hchain = (ctx->pos >> 20) - 1; + hindex = ctx->pos & 0xfffff; + + bh = omfs_bread(dir->i_sb, dir->i_ino); + if (!bh) + return -EINVAL; + + p = (__be64 *)(bh->b_data + OMFS_DIR_START) + hchain; + + for (; hchain < nbuckets; hchain++) { + __u64 fsblock = be64_to_cpu(*p++); + if (!omfs_fill_chain(dir, ctx, fsblock, hindex)) + break; + hindex = 0; + ctx->pos = (hchain+2) << 20; + } + brelse(bh); + return 0; +} + +const struct inode_operations omfs_dir_inops = { + .lookup = omfs_lookup, + .mkdir = omfs_mkdir, + .rename = omfs_rename, + .create = omfs_create, + .unlink = omfs_remove, + .rmdir = omfs_remove, +}; + +const struct file_operations omfs_dir_operations = { + .read = generic_read_dir, + .iterate_shared = omfs_readdir, + .llseek = generic_file_llseek, +}; diff --git a/fs/omfs/file.c b/fs/omfs/file.c new file mode 100644 index 000000000..28f3b1133 --- /dev/null +++ b/fs/omfs/file.c @@ -0,0 +1,380 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * OMFS (as used by RIO Karma) file operations. + * Copyright (C) 2005 Bob Copeland + */ + +#include +#include +#include +#include +#include "omfs.h" + +static u32 omfs_max_extents(struct omfs_sb_info *sbi, int offset) +{ + return (sbi->s_sys_blocksize - offset - + sizeof(struct omfs_extent)) / + sizeof(struct omfs_extent_entry); +} + +void omfs_make_empty_table(struct buffer_head *bh, int offset) +{ + struct omfs_extent *oe = (struct omfs_extent *) &bh->b_data[offset]; + + oe->e_next = ~cpu_to_be64(0ULL); + oe->e_extent_count = cpu_to_be32(1), + oe->e_fill = cpu_to_be32(0x22), + oe->e_entry[0].e_cluster = ~cpu_to_be64(0ULL); + oe->e_entry[0].e_blocks = ~cpu_to_be64(0ULL); +} + +int omfs_shrink_inode(struct inode *inode) +{ + struct omfs_sb_info *sbi = OMFS_SB(inode->i_sb); + struct omfs_extent *oe; + struct omfs_extent_entry *entry; + struct buffer_head *bh; + u64 next, last; + u32 extent_count; + u32 max_extents; + int ret; + + /* traverse extent table, freeing each entry that is greater + * than inode->i_size; + */ + next = inode->i_ino; + + /* only support truncate -> 0 for now */ + ret = -EIO; + if (inode->i_size != 0) + goto out; + + bh = omfs_bread(inode->i_sb, next); + if (!bh) + goto out; + + oe = (struct omfs_extent *)(&bh->b_data[OMFS_EXTENT_START]); + max_extents = omfs_max_extents(sbi, OMFS_EXTENT_START); + + for (;;) { + + if (omfs_is_bad(sbi, (struct omfs_header *) bh->b_data, next)) + goto out_brelse; + + extent_count = be32_to_cpu(oe->e_extent_count); + + if (extent_count > max_extents) + goto out_brelse; + + last = next; + next = be64_to_cpu(oe->e_next); + entry = oe->e_entry; + + /* ignore last entry as it is the terminator */ + for (; extent_count > 1; extent_count--) { + u64 start, count; + start = be64_to_cpu(entry->e_cluster); + count = be64_to_cpu(entry->e_blocks); + + omfs_clear_range(inode->i_sb, start, (int) count); + entry++; + } + omfs_make_empty_table(bh, (char *) oe - bh->b_data); + mark_buffer_dirty(bh); + brelse(bh); + + if (last != inode->i_ino) + omfs_clear_range(inode->i_sb, last, sbi->s_mirrors); + + if (next == ~0) + break; + + bh = omfs_bread(inode->i_sb, next); + if (!bh) + goto out; + oe = (struct omfs_extent *) (&bh->b_data[OMFS_EXTENT_CONT]); + max_extents = omfs_max_extents(sbi, OMFS_EXTENT_CONT); + } + ret = 0; +out: + return ret; +out_brelse: + brelse(bh); + return ret; +} + +static void omfs_truncate(struct inode *inode) +{ + omfs_shrink_inode(inode); + mark_inode_dirty(inode); +} + +/* + * Add new blocks to the current extent, or create new entries/continuations + * as necessary. + */ +static int omfs_grow_extent(struct inode *inode, struct omfs_extent *oe, + u64 *ret_block) +{ + struct omfs_extent_entry *terminator; + struct omfs_extent_entry *entry = oe->e_entry; + struct omfs_sb_info *sbi = OMFS_SB(inode->i_sb); + u32 extent_count = be32_to_cpu(oe->e_extent_count); + u64 new_block = 0; + u32 max_count; + int new_count; + int ret = 0; + + /* reached the end of the extent table with no blocks mapped. + * there are three possibilities for adding: grow last extent, + * add a new extent to the current extent table, and add a + * continuation inode. in last two cases need an allocator for + * sbi->s_cluster_size + */ + + /* TODO: handle holes */ + + /* should always have a terminator */ + if (extent_count < 1) + return -EIO; + + /* trivially grow current extent, if next block is not taken */ + terminator = entry + extent_count - 1; + if (extent_count > 1) { + entry = terminator-1; + new_block = be64_to_cpu(entry->e_cluster) + + be64_to_cpu(entry->e_blocks); + + if (omfs_allocate_block(inode->i_sb, new_block)) { + be64_add_cpu(&entry->e_blocks, 1); + terminator->e_blocks = ~(cpu_to_be64( + be64_to_cpu(~terminator->e_blocks) + 1)); + goto out; + } + } + max_count = omfs_max_extents(sbi, OMFS_EXTENT_START); + + /* TODO: add a continuation block here */ + if (be32_to_cpu(oe->e_extent_count) > max_count-1) + return -EIO; + + /* try to allocate a new cluster */ + ret = omfs_allocate_range(inode->i_sb, 1, sbi->s_clustersize, + &new_block, &new_count); + if (ret) + goto out_fail; + + /* copy terminator down an entry */ + entry = terminator; + terminator++; + memcpy(terminator, entry, sizeof(struct omfs_extent_entry)); + + entry->e_cluster = cpu_to_be64(new_block); + entry->e_blocks = cpu_to_be64((u64) new_count); + + terminator->e_blocks = ~(cpu_to_be64( + be64_to_cpu(~terminator->e_blocks) + (u64) new_count)); + + /* write in new entry */ + be32_add_cpu(&oe->e_extent_count, 1); + +out: + *ret_block = new_block; +out_fail: + return ret; +} + +/* + * Scans across the directory table for a given file block number. + * If block not found, return 0. + */ +static sector_t find_block(struct inode *inode, struct omfs_extent_entry *ent, + sector_t block, int count, int *left) +{ + /* count > 1 because of terminator */ + sector_t searched = 0; + for (; count > 1; count--) { + int numblocks = clus_to_blk(OMFS_SB(inode->i_sb), + be64_to_cpu(ent->e_blocks)); + + if (block >= searched && + block < searched + numblocks) { + /* + * found it at cluster + (block - searched) + * numblocks - (block - searched) is remainder + */ + *left = numblocks - (block - searched); + return clus_to_blk(OMFS_SB(inode->i_sb), + be64_to_cpu(ent->e_cluster)) + + block - searched; + } + searched += numblocks; + ent++; + } + return 0; +} + +static int omfs_get_block(struct inode *inode, sector_t block, + struct buffer_head *bh_result, int create) +{ + struct buffer_head *bh; + sector_t next, offset; + int ret; + u64 new_block; + u32 max_extents; + int extent_count; + struct omfs_extent *oe; + struct omfs_extent_entry *entry; + struct omfs_sb_info *sbi = OMFS_SB(inode->i_sb); + int max_blocks = bh_result->b_size >> inode->i_blkbits; + int remain; + + ret = -EIO; + bh = omfs_bread(inode->i_sb, inode->i_ino); + if (!bh) + goto out; + + oe = (struct omfs_extent *)(&bh->b_data[OMFS_EXTENT_START]); + max_extents = omfs_max_extents(sbi, OMFS_EXTENT_START); + next = inode->i_ino; + + for (;;) { + + if (omfs_is_bad(sbi, (struct omfs_header *) bh->b_data, next)) + goto out_brelse; + + extent_count = be32_to_cpu(oe->e_extent_count); + next = be64_to_cpu(oe->e_next); + entry = oe->e_entry; + + if (extent_count > max_extents) + goto out_brelse; + + offset = find_block(inode, entry, block, extent_count, &remain); + if (offset > 0) { + ret = 0; + map_bh(bh_result, inode->i_sb, offset); + if (remain > max_blocks) + remain = max_blocks; + bh_result->b_size = (remain << inode->i_blkbits); + goto out_brelse; + } + if (next == ~0) + break; + + brelse(bh); + bh = omfs_bread(inode->i_sb, next); + if (!bh) + goto out; + oe = (struct omfs_extent *) (&bh->b_data[OMFS_EXTENT_CONT]); + max_extents = omfs_max_extents(sbi, OMFS_EXTENT_CONT); + } + if (create) { + ret = omfs_grow_extent(inode, oe, &new_block); + if (ret == 0) { + mark_buffer_dirty(bh); + mark_inode_dirty(inode); + map_bh(bh_result, inode->i_sb, + clus_to_blk(sbi, new_block)); + } + } +out_brelse: + brelse(bh); +out: + return ret; +} + +static int omfs_read_folio(struct file *file, struct folio *folio) +{ + return block_read_full_folio(folio, omfs_get_block); +} + +static void omfs_readahead(struct readahead_control *rac) +{ + mpage_readahead(rac, omfs_get_block); +} + +static int +omfs_writepages(struct address_space *mapping, struct writeback_control *wbc) +{ + return mpage_writepages(mapping, wbc, omfs_get_block); +} + +static void omfs_write_failed(struct address_space *mapping, loff_t to) +{ + struct inode *inode = mapping->host; + + if (to > inode->i_size) { + truncate_pagecache(inode, inode->i_size); + omfs_truncate(inode); + } +} + +static int omfs_write_begin(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, + struct folio **foliop, void **fsdata) +{ + int ret; + + ret = block_write_begin(mapping, pos, len, foliop, omfs_get_block); + if (unlikely(ret)) + omfs_write_failed(mapping, pos + len); + + return ret; +} + +static sector_t omfs_bmap(struct address_space *mapping, sector_t block) +{ + return generic_block_bmap(mapping, block, omfs_get_block); +} + +const struct file_operations omfs_file_operations = { + .llseek = generic_file_llseek, + .read_iter = generic_file_read_iter, + .write_iter = generic_file_write_iter, + .mmap_prepare = generic_file_mmap_prepare, + .fsync = simple_fsync, + .splice_read = filemap_splice_read, +}; + +static int omfs_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, struct iattr *attr) +{ + struct inode *inode = d_inode(dentry); + int error; + + error = setattr_prepare(&nop_mnt_idmap, dentry, attr); + if (error) + return error; + + if ((attr->ia_valid & ATTR_SIZE) && + attr->ia_size != i_size_read(inode)) { + error = inode_newsize_ok(inode, attr->ia_size); + if (error) + return error; + truncate_setsize(inode, attr->ia_size); + omfs_truncate(inode); + } + + setattr_copy(&nop_mnt_idmap, inode, attr); + mark_inode_dirty(inode); + return 0; +} + +const struct inode_operations omfs_file_inops = { + .setattr = omfs_setattr, +}; + +const struct address_space_operations omfs_aops = { + .dirty_folio = block_dirty_folio, + .invalidate_folio = block_invalidate_folio, + .read_folio = omfs_read_folio, + .readahead = omfs_readahead, + .writepages = omfs_writepages, + .write_begin = omfs_write_begin, + .write_end = generic_write_end, + .bmap = omfs_bmap, + .migrate_folio = buffer_migrate_folio, +}; + diff --git a/fs/omfs/inode.c b/fs/omfs/inode.c new file mode 100644 index 000000000..1d915ef72 --- /dev/null +++ b/fs/omfs/inode.c @@ -0,0 +1,664 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Optimized MPEG FS - inode and super operations. + * Copyright (C) 2006 Bob Copeland + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "omfs.h" + +MODULE_AUTHOR("Bob Copeland "); +MODULE_DESCRIPTION("OMFS (ReplayTV/Karma) Filesystem for Linux"); +MODULE_LICENSE("GPL"); + +struct buffer_head *omfs_bread(struct super_block *sb, sector_t block) +{ + struct omfs_sb_info *sbi = OMFS_SB(sb); + if (block >= sbi->s_num_blocks) + return NULL; + + return sb_bread(sb, clus_to_blk(sbi, block)); +} + +struct inode *omfs_new_inode(struct inode *dir, umode_t mode) +{ + struct inode *inode; + u64 new_block; + int err; + int len; + struct omfs_sb_info *sbi = OMFS_SB(dir->i_sb); + + inode = new_inode(dir->i_sb); + if (!inode) + return ERR_PTR(-ENOMEM); + + err = omfs_allocate_range(dir->i_sb, sbi->s_mirrors, sbi->s_mirrors, + &new_block, &len); + if (err) + goto fail; + + inode->i_ino = new_block; + inode_init_owner(&nop_mnt_idmap, inode, NULL, mode); + inode->i_mapping->a_ops = &omfs_aops; + + simple_inode_init_ts(inode); + switch (mode & S_IFMT) { + case S_IFDIR: + inode->i_op = &omfs_dir_inops; + inode->i_fop = &omfs_dir_operations; + inode->i_size = sbi->s_sys_blocksize; + inc_nlink(inode); + break; + case S_IFREG: + inode->i_op = &omfs_file_inops; + inode->i_fop = &omfs_file_operations; + inode->i_size = 0; + break; + } + + insert_inode_hash(inode); + mark_inode_dirty(inode); + return inode; +fail: + make_bad_inode(inode); + iput(inode); + return ERR_PTR(err); +} + +/* + * Update the header checksums for a dirty inode based on its contents. + * Caller is expected to hold the buffer head underlying oi and mark it + * dirty. + */ +static void omfs_update_checksums(struct omfs_inode *oi) +{ + int xor, i, ofs = 0, count; + u16 crc = 0; + unsigned char *ptr = (unsigned char *) oi; + + count = be32_to_cpu(oi->i_head.h_body_size); + ofs = sizeof(struct omfs_header); + + crc = crc_itu_t(crc, ptr + ofs, count); + oi->i_head.h_crc = cpu_to_be16(crc); + + xor = ptr[0]; + for (i = 1; i < OMFS_XOR_COUNT; i++) + xor ^= ptr[i]; + + oi->i_head.h_check_xor = xor; +} + +static int __omfs_write_inode(struct inode *inode, int wait) +{ + struct omfs_inode *oi; + struct omfs_sb_info *sbi = OMFS_SB(inode->i_sb); + struct buffer_head *bh, *bh2; + u64 ctime; + int i; + int ret = -EIO; + int sync_failed = 0; + + /* get current inode since we may have written sibling ptrs etc. */ + bh = omfs_bread(inode->i_sb, inode->i_ino); + if (!bh) + goto out; + + oi = (struct omfs_inode *) bh->b_data; + + oi->i_head.h_self = cpu_to_be64(inode->i_ino); + if (S_ISDIR(inode->i_mode)) + oi->i_type = OMFS_DIR; + else if (S_ISREG(inode->i_mode)) + oi->i_type = OMFS_FILE; + else { + printk(KERN_WARNING "omfs: unknown file type: %d\n", + inode->i_mode); + goto out_brelse; + } + + oi->i_head.h_body_size = cpu_to_be32(sbi->s_sys_blocksize - + sizeof(struct omfs_header)); + oi->i_head.h_version = 1; + oi->i_head.h_type = OMFS_INODE_NORMAL; + oi->i_head.h_magic = OMFS_IMAGIC; + oi->i_size = cpu_to_be64(inode->i_size); + + ctime = inode_get_ctime_sec(inode) * 1000LL + + ((inode_get_ctime_nsec(inode) + 999)/1000); + oi->i_ctime = cpu_to_be64(ctime); + + omfs_update_checksums(oi); + + mark_buffer_dirty(bh); + if (wait) { + sync_dirty_buffer(bh); + if (buffer_req(bh) && !buffer_uptodate(bh)) + sync_failed = 1; + } + + /* if mirroring writes, copy to next fsblock */ + for (i = 1; i < sbi->s_mirrors; i++) { + bh2 = omfs_bread(inode->i_sb, inode->i_ino + i); + if (!bh2) + goto out_brelse; + + memcpy(bh2->b_data, bh->b_data, bh->b_size); + mark_buffer_dirty(bh2); + if (wait) { + sync_dirty_buffer(bh2); + if (buffer_req(bh2) && !buffer_uptodate(bh2)) + sync_failed = 1; + } + brelse(bh2); + } + ret = (sync_failed) ? -EIO : 0; +out_brelse: + brelse(bh); +out: + return ret; +} + +static int omfs_write_inode(struct inode *inode, struct writeback_control *wbc) +{ + return __omfs_write_inode(inode, wbc->sync_mode == WB_SYNC_ALL); +} + +int omfs_sync_inode(struct inode *inode) +{ + return __omfs_write_inode(inode, 1); +} + +/* + * called when an entry is deleted, need to clear the bits in the + * bitmaps. + */ +static void omfs_evict_inode(struct inode *inode) +{ + truncate_inode_pages_final(&inode->i_data); + clear_inode(inode); + + if (inode->i_nlink) + return; + + if (S_ISREG(inode->i_mode)) { + inode->i_size = 0; + omfs_shrink_inode(inode); + } + + omfs_clear_range(inode->i_sb, inode->i_ino, 2); +} + +struct inode *omfs_iget(struct super_block *sb, ino_t ino) +{ + struct omfs_sb_info *sbi = OMFS_SB(sb); + struct omfs_inode *oi; + struct buffer_head *bh; + u64 ctime; + unsigned long nsecs; + struct inode *inode; + + inode = iget_locked(sb, ino); + if (!inode) + return ERR_PTR(-ENOMEM); + if (!(inode_state_read_once(inode) & I_NEW)) + return inode; + + bh = omfs_bread(inode->i_sb, ino); + if (!bh) + goto iget_failed; + + oi = (struct omfs_inode *)bh->b_data; + + /* check self */ + if (ino != be64_to_cpu(oi->i_head.h_self)) + goto fail_bh; + + inode->i_uid = sbi->s_uid; + inode->i_gid = sbi->s_gid; + + ctime = be64_to_cpu(oi->i_ctime); + nsecs = do_div(ctime, 1000) * 1000L; + + inode_set_atime(inode, ctime, nsecs); + inode_set_mtime(inode, ctime, nsecs); + inode_set_ctime(inode, ctime, nsecs); + + inode->i_mapping->a_ops = &omfs_aops; + + switch (oi->i_type) { + case OMFS_DIR: + inode->i_mode = S_IFDIR | (S_IRWXUGO & ~sbi->s_dmask); + inode->i_op = &omfs_dir_inops; + inode->i_fop = &omfs_dir_operations; + inode->i_size = sbi->s_sys_blocksize; + inc_nlink(inode); + break; + case OMFS_FILE: + inode->i_mode = S_IFREG | (S_IRWXUGO & ~sbi->s_fmask); + inode->i_fop = &omfs_file_operations; + inode->i_size = be64_to_cpu(oi->i_size); + break; + } + brelse(bh); + unlock_new_inode(inode); + return inode; +fail_bh: + brelse(bh); +iget_failed: + iget_failed(inode); + return ERR_PTR(-EIO); +} + +static void omfs_put_super(struct super_block *sb) +{ + struct omfs_sb_info *sbi = OMFS_SB(sb); + kfree(sbi->s_imap); + kfree(sbi); + sb->s_fs_info = NULL; +} + +static int omfs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *s = dentry->d_sb; + struct omfs_sb_info *sbi = OMFS_SB(s); + u64 id = huge_encode_dev(s->s_bdev->bd_dev); + + buf->f_type = OMFS_MAGIC; + buf->f_bsize = sbi->s_blocksize; + buf->f_blocks = sbi->s_num_blocks; + buf->f_files = sbi->s_num_blocks; + buf->f_namelen = OMFS_NAMELEN; + buf->f_fsid = u64_to_fsid(id); + + buf->f_bfree = buf->f_bavail = buf->f_ffree = + omfs_count_free(s); + + return 0; +} + +/* + * Display the mount options in /proc/mounts. + */ +static int omfs_show_options(struct seq_file *m, struct dentry *root) +{ + struct omfs_sb_info *sbi = OMFS_SB(root->d_sb); + umode_t cur_umask = current_umask(); + + if (!uid_eq(sbi->s_uid, current_uid())) + seq_printf(m, ",uid=%u", + from_kuid_munged(&init_user_ns, sbi->s_uid)); + if (!gid_eq(sbi->s_gid, current_gid())) + seq_printf(m, ",gid=%u", + from_kgid_munged(&init_user_ns, sbi->s_gid)); + + if (sbi->s_dmask == sbi->s_fmask) { + if (sbi->s_fmask != cur_umask) + seq_printf(m, ",umask=%o", sbi->s_fmask); + } else { + if (sbi->s_dmask != cur_umask) + seq_printf(m, ",dmask=%o", sbi->s_dmask); + if (sbi->s_fmask != cur_umask) + seq_printf(m, ",fmask=%o", sbi->s_fmask); + } + + return 0; +} + +static const struct super_operations omfs_sops = { + .write_inode = omfs_write_inode, + .evict_inode = omfs_evict_inode, + .put_super = omfs_put_super, + .statfs = omfs_statfs, + .show_options = omfs_show_options, +}; + +/* + * For Rio Karma, there is an on-disk free bitmap whose location is + * stored in the root block. For ReplayTV, there is no such free bitmap + * so we have to walk the tree. Both inodes and file data are allocated + * from the same map. This array can be big (300k) so we allocate + * in units of the blocksize. + */ +static int omfs_get_imap(struct super_block *sb) +{ + unsigned int bitmap_size, array_size; + int count; + struct omfs_sb_info *sbi = OMFS_SB(sb); + struct buffer_head *bh; + unsigned long **ptr; + sector_t block; + + bitmap_size = DIV_ROUND_UP(sbi->s_num_blocks, 8); + array_size = DIV_ROUND_UP(bitmap_size, sb->s_blocksize); + + if (sbi->s_bitmap_ino == ~0ULL) + goto out; + + sbi->s_imap_size = array_size; + sbi->s_imap = kcalloc(array_size, sizeof(unsigned long *), GFP_KERNEL); + if (!sbi->s_imap) + goto nomem; + + block = clus_to_blk(sbi, sbi->s_bitmap_ino); + if (block >= sbi->s_num_blocks) + goto nomem; + + ptr = sbi->s_imap; + for (count = bitmap_size; count > 0; count -= sb->s_blocksize) { + bh = sb_bread(sb, block++); + if (!bh) + goto nomem_free; + *ptr = kmemdup(bh->b_data, sb->s_blocksize, GFP_KERNEL); + if (!*ptr) { + brelse(bh); + goto nomem_free; + } + if (count < sb->s_blocksize) + memset((void *)*ptr + count, 0xff, + sb->s_blocksize - count); + brelse(bh); + ptr++; + } +out: + return 0; + +nomem_free: + for (count = 0; count < array_size; count++) + kfree(sbi->s_imap[count]); + + kfree(sbi->s_imap); +nomem: + sbi->s_imap = NULL; + sbi->s_imap_size = 0; + return -ENOMEM; +} + +struct omfs_mount_options { + kuid_t s_uid; + kgid_t s_gid; + int s_dmask; + int s_fmask; +}; + +enum { + Opt_uid, Opt_gid, Opt_umask, Opt_dmask, Opt_fmask, +}; + +static const struct fs_parameter_spec omfs_param_spec[] = { + fsparam_uid ("uid", Opt_uid), + fsparam_gid ("gid", Opt_gid), + fsparam_u32oct ("umask", Opt_umask), + fsparam_u32oct ("dmask", Opt_dmask), + fsparam_u32oct ("fmask", Opt_fmask), + {} +}; + +static int +omfs_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct omfs_mount_options *opts = fc->fs_private; + int token; + struct fs_parse_result result; + + /* All options are ignored on remount */ + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE) + return 0; + + token = fs_parse(fc, omfs_param_spec, param, &result); + if (token < 0) + return token; + + switch (token) { + case Opt_uid: + opts->s_uid = result.uid; + break; + case Opt_gid: + opts->s_gid = result.gid; + break; + case Opt_umask: + opts->s_fmask = opts->s_dmask = result.uint_32; + break; + case Opt_dmask: + opts->s_dmask = result.uint_32; + break; + case Opt_fmask: + opts->s_fmask = result.uint_32; + break; + default: + return -EINVAL; + } + + return 0; +} + +static void +omfs_set_options(struct omfs_sb_info *sbi, struct omfs_mount_options *opts) +{ + sbi->s_uid = opts->s_uid; + sbi->s_gid = opts->s_gid; + sbi->s_dmask = opts->s_dmask; + sbi->s_fmask = opts->s_fmask; +} + +static int omfs_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct buffer_head *bh, *bh2; + struct omfs_super_block *omfs_sb; + struct omfs_root_block *omfs_rb; + struct omfs_sb_info *sbi; + struct inode *root; + struct omfs_mount_options *parsed_opts = fc->fs_private; + int ret = -EINVAL; + int silent = fc->sb_flags & SB_SILENT; + + sbi = kzalloc_obj(struct omfs_sb_info); + if (!sbi) + return -ENOMEM; + + sb->s_fs_info = sbi; + + omfs_set_options(sbi, parsed_opts); + + sb->s_maxbytes = 0xffffffff; + + sb->s_time_gran = NSEC_PER_MSEC; + sb->s_time_min = 0; + sb->s_time_max = U64_MAX / MSEC_PER_SEC; + + if (!sb_set_blocksize(sb, 0x200)) + goto end; + + bh = sb_bread(sb, 0); + if (!bh) + goto end; + + omfs_sb = (struct omfs_super_block *)bh->b_data; + + if (omfs_sb->s_magic != cpu_to_be32(OMFS_MAGIC)) { + if (!silent) + printk(KERN_ERR "omfs: Invalid superblock (%x)\n", + omfs_sb->s_magic); + goto out_brelse_bh; + } + sb->s_magic = OMFS_MAGIC; + + sbi->s_num_blocks = be64_to_cpu(omfs_sb->s_num_blocks); + sbi->s_blocksize = be32_to_cpu(omfs_sb->s_blocksize); + sbi->s_mirrors = be32_to_cpu(omfs_sb->s_mirrors); + sbi->s_root_ino = be64_to_cpu(omfs_sb->s_root_block); + sbi->s_sys_blocksize = be32_to_cpu(omfs_sb->s_sys_blocksize); + mutex_init(&sbi->s_bitmap_lock); + + if (sbi->s_num_blocks > OMFS_MAX_BLOCKS) { + printk(KERN_ERR "omfs: sysblock number (%llx) is out of range\n", + (unsigned long long)sbi->s_num_blocks); + goto out_brelse_bh; + } + + if (sbi->s_sys_blocksize > PAGE_SIZE) { + printk(KERN_ERR "omfs: sysblock size (%d) is out of range\n", + sbi->s_sys_blocksize); + goto out_brelse_bh; + } + + if (sbi->s_sys_blocksize < OMFS_DIR_START) { + printk(KERN_ERR "omfs: sysblock size (%d) is too small\n", + sbi->s_sys_blocksize); + goto out_brelse_bh; + } + + if (sbi->s_blocksize < sbi->s_sys_blocksize || + sbi->s_blocksize > OMFS_MAX_BLOCK_SIZE) { + printk(KERN_ERR "omfs: block size (%d) is out of range\n", + sbi->s_blocksize); + goto out_brelse_bh; + } + + /* + * Use sys_blocksize as the fs block since it is smaller than a + * page while the fs blocksize can be larger. + */ + if (!sb_set_blocksize(sb, sbi->s_sys_blocksize)) + goto out_brelse_bh; + + /* + * ...and the difference goes into a shift. sys_blocksize is always + * a power of two factor of blocksize. + */ + sbi->s_block_shift = get_bitmask_order(sbi->s_blocksize) - + get_bitmask_order(sbi->s_sys_blocksize); + + bh2 = omfs_bread(sb, be64_to_cpu(omfs_sb->s_root_block)); + if (!bh2) + goto out_brelse_bh; + + omfs_rb = (struct omfs_root_block *)bh2->b_data; + + sbi->s_bitmap_ino = be64_to_cpu(omfs_rb->r_bitmap); + sbi->s_clustersize = be32_to_cpu(omfs_rb->r_clustersize); + + if (sbi->s_num_blocks != be64_to_cpu(omfs_rb->r_num_blocks)) { + printk(KERN_ERR "omfs: block count discrepancy between " + "super and root blocks (%llx, %llx)\n", + (unsigned long long)sbi->s_num_blocks, + (unsigned long long)be64_to_cpu(omfs_rb->r_num_blocks)); + goto out_brelse_bh2; + } + + if (sbi->s_bitmap_ino != ~0ULL && + sbi->s_bitmap_ino > sbi->s_num_blocks) { + printk(KERN_ERR "omfs: free space bitmap location is corrupt " + "(%llx, total blocks %llx)\n", + (unsigned long long) sbi->s_bitmap_ino, + (unsigned long long) sbi->s_num_blocks); + goto out_brelse_bh2; + } + if (sbi->s_clustersize < 1 || + sbi->s_clustersize > OMFS_MAX_CLUSTER_SIZE) { + printk(KERN_ERR "omfs: cluster size out of range (%d)", + sbi->s_clustersize); + goto out_brelse_bh2; + } + + ret = omfs_get_imap(sb); + if (ret) + goto out_brelse_bh2; + + sb->s_op = &omfs_sops; + + root = omfs_iget(sb, be64_to_cpu(omfs_rb->r_root_dir)); + if (IS_ERR(root)) { + ret = PTR_ERR(root); + goto out_brelse_bh2; + } + + sb->s_root = d_make_root(root); + if (!sb->s_root) { + ret = -ENOMEM; + goto out_brelse_bh2; + } + printk(KERN_DEBUG "omfs: Mounted volume %s\n", omfs_rb->r_name); + + ret = 0; +out_brelse_bh2: + brelse(bh2); +out_brelse_bh: + brelse(bh); +end: + if (ret) + kfree(sbi); + return ret; +} + +static int omfs_get_tree(struct fs_context *fc) +{ + return get_tree_bdev(fc, omfs_fill_super); +} + +static void omfs_free_fc(struct fs_context *fc); + +static const struct fs_context_operations omfs_context_ops = { + .parse_param = omfs_parse_param, + .get_tree = omfs_get_tree, + .free = omfs_free_fc, +}; + +static int omfs_init_fs_context(struct fs_context *fc) +{ + struct omfs_mount_options *opts; + + opts = kzalloc_obj(*opts); + if (!opts) + return -ENOMEM; + + /* Set mount options defaults */ + opts->s_uid = current_uid(); + opts->s_gid = current_gid(); + opts->s_dmask = opts->s_fmask = current_umask(); + + fc->fs_private = opts; + fc->ops = &omfs_context_ops; + + return 0; +} + +static void omfs_free_fc(struct fs_context *fc) +{ + kfree(fc->fs_private); +} + +static struct file_system_type omfs_fs_type = { + .owner = THIS_MODULE, + .name = "omfs", + .kill_sb = kill_block_super, + .fs_flags = FS_REQUIRES_DEV, + .init_fs_context = omfs_init_fs_context, + .parameters = omfs_param_spec, +}; +MODULE_ALIAS_FS("omfs"); + +static int __init init_omfs_fs(void) +{ + return register_filesystem(&omfs_fs_type); +} + +static void __exit exit_omfs_fs(void) +{ + unregister_filesystem(&omfs_fs_type); +} + +module_init(init_omfs_fs); +module_exit(exit_omfs_fs); diff --git a/fs/omfs/omfs.h b/fs/omfs/omfs.h new file mode 100644 index 000000000..4008be73d --- /dev/null +++ b/fs/omfs/omfs.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _OMFS_H +#define _OMFS_H + +#include +#include + +#include "omfs_fs.h" + +/* In-memory structures */ +struct omfs_sb_info { + u64 s_num_blocks; + u64 s_bitmap_ino; + u64 s_root_ino; + u32 s_blocksize; + u32 s_mirrors; + u32 s_sys_blocksize; + u32 s_clustersize; + int s_block_shift; + unsigned long **s_imap; + int s_imap_size; + struct mutex s_bitmap_lock; + kuid_t s_uid; + kgid_t s_gid; + int s_dmask; + int s_fmask; +}; + +/* convert a cluster number to a scaled block number */ +static inline sector_t clus_to_blk(struct omfs_sb_info *sbi, sector_t block) +{ + return block << sbi->s_block_shift; +} + +static inline struct omfs_sb_info *OMFS_SB(struct super_block *sb) +{ + return sb->s_fs_info; +} + +/* bitmap.c */ +extern unsigned long omfs_count_free(struct super_block *sb); +extern int omfs_allocate_block(struct super_block *sb, u64 block); +extern int omfs_allocate_range(struct super_block *sb, int min_request, + int max_request, u64 *return_block, int *return_size); +extern int omfs_clear_range(struct super_block *sb, u64 block, int count); + +/* dir.c */ +extern const struct file_operations omfs_dir_operations; +extern const struct inode_operations omfs_dir_inops; +extern int omfs_make_empty(struct inode *inode, struct super_block *sb); +extern int omfs_is_bad(struct omfs_sb_info *sbi, struct omfs_header *header, + u64 fsblock); + +/* file.c */ +extern const struct file_operations omfs_file_operations; +extern const struct inode_operations omfs_file_inops; +extern const struct address_space_operations omfs_aops; +extern void omfs_make_empty_table(struct buffer_head *bh, int offset); +extern int omfs_shrink_inode(struct inode *inode); + +/* inode.c */ +extern struct buffer_head *omfs_bread(struct super_block *sb, sector_t block); +extern struct inode *omfs_iget(struct super_block *sb, ino_t inode); +extern struct inode *omfs_new_inode(struct inode *dir, umode_t mode); +extern int omfs_reserve_block(struct super_block *sb, sector_t block); +extern int omfs_find_empty_block(struct super_block *sb, int mode, ino_t *ino); +extern int omfs_sync_inode(struct inode *inode); + +#endif diff --git a/fs/omfs/omfs_fs.h b/fs/omfs/omfs_fs.h new file mode 100644 index 000000000..1ff6b9e41 --- /dev/null +++ b/fs/omfs/omfs_fs.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _OMFS_FS_H +#define _OMFS_FS_H + +/* OMFS On-disk structures */ + +#define OMFS_MAGIC 0xC2993D87 +#define OMFS_IMAGIC 0xD2 + +#define OMFS_DIR 'D' +#define OMFS_FILE 'F' +#define OMFS_INODE_NORMAL 'e' +#define OMFS_INODE_CONTINUATION 'c' +#define OMFS_INODE_SYSTEM 's' +#define OMFS_NAMELEN 256 +#define OMFS_DIR_START 0x1b8 +#define OMFS_EXTENT_START 0x1d0 +#define OMFS_EXTENT_CONT 0x40 +#define OMFS_XOR_COUNT 19 +#define OMFS_MAX_BLOCK_SIZE 8192 +#define OMFS_MAX_CLUSTER_SIZE 8 +#define OMFS_MAX_BLOCKS (1ul << 31) + +struct omfs_super_block { + char s_fill1[256]; + __be64 s_root_block; /* block number of omfs_root_block */ + __be64 s_num_blocks; /* total number of FS blocks */ + __be32 s_magic; /* OMFS_MAGIC */ + __be32 s_blocksize; /* size of a block */ + __be32 s_mirrors; /* # of mirrors of system blocks */ + __be32 s_sys_blocksize; /* size of non-data blocks */ +}; + +struct omfs_header { + __be64 h_self; /* FS block where this is located */ + __be32 h_body_size; /* size of useful data after header */ + __be16 h_crc; /* crc-ccitt of body_size bytes */ + char h_fill1[2]; + u8 h_version; /* version, always 1 */ + char h_type; /* OMFS_INODE_X */ + u8 h_magic; /* OMFS_IMAGIC */ + u8 h_check_xor; /* XOR of header bytes before this */ + __be32 h_fill2; +}; + +struct omfs_root_block { + struct omfs_header r_head; /* header */ + __be64 r_fill1; + __be64 r_num_blocks; /* total number of FS blocks */ + __be64 r_root_dir; /* block # of root directory */ + __be64 r_bitmap; /* block # of free space bitmap */ + __be32 r_blocksize; /* size of a block */ + __be32 r_clustersize; /* size allocated for data blocks */ + __be64 r_mirrors; /* # of mirrors of system blocks */ + char r_name[OMFS_NAMELEN]; /* partition label */ +}; + +struct omfs_inode { + struct omfs_header i_head; /* header */ + __be64 i_parent; /* parent containing this inode */ + __be64 i_sibling; /* next inode in hash bucket */ + __be64 i_ctime; /* ctime, in milliseconds */ + char i_fill1[35]; + char i_type; /* OMFS_[DIR,FILE] */ + __be32 i_fill2; + char i_fill3[64]; + char i_name[OMFS_NAMELEN]; /* filename */ + __be64 i_size; /* size of file, in bytes */ +}; + +struct omfs_extent_entry { + __be64 e_cluster; /* start location of a set of blocks */ + __be64 e_blocks; /* number of blocks after e_cluster */ +}; + +struct omfs_extent { + __be64 e_next; /* next extent table location */ + __be32 e_extent_count; /* total # extents in this table */ + __be32 e_fill; + struct omfs_extent_entry e_entry[]; /* start of extent entries */ +}; + +#endif diff --git a/fs/open.c b/fs/open.c new file mode 100644 index 000000000..6b1c14e68 --- /dev/null +++ b/fs/open.c @@ -0,0 +1,1634 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/open.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +int do_truncate(struct mnt_idmap *idmap, struct dentry *dentry, + loff_t length, unsigned int time_attrs, struct file *filp) +{ + int ret; + struct iattr newattrs; + + /* Not pretty: "inode->i_size" shouldn't really be signed. But it is. */ + if (length < 0) + return -EINVAL; + + newattrs.ia_size = length; + newattrs.ia_valid = ATTR_SIZE | time_attrs; + if (filp) { + newattrs.ia_file = filp; + newattrs.ia_valid |= ATTR_FILE; + } + + /* Remove suid, sgid, and file capabilities on truncate too */ + ret = dentry_needs_remove_privs(idmap, dentry); + if (ret < 0) + return ret; + if (ret) + newattrs.ia_valid |= ret | ATTR_FORCE; + + ret = inode_lock_killable(dentry->d_inode); + if (ret) + return ret; + + /* Note any delegations or leases have already been broken: */ + ret = notify_change(idmap, dentry, &newattrs, NULL); + inode_unlock(dentry->d_inode); + return ret; +} + +int vfs_truncate(const struct path *path, loff_t length) +{ + struct mnt_idmap *idmap; + struct inode *inode; + int error; + + inode = path->dentry->d_inode; + + /* For directories it's -EISDIR, for other non-regulars - -EINVAL */ + if (S_ISDIR(inode->i_mode)) + return -EISDIR; + if (!S_ISREG(inode->i_mode)) + return -EINVAL; + + idmap = mnt_idmap(path->mnt); + error = inode_permission(idmap, inode, MAY_WRITE); + if (error) + return error; + + error = fsnotify_truncate_perm(path, length); + if (error) + return error; + + error = mnt_want_write(path->mnt); + if (error) + return error; + + error = -EPERM; + if (IS_APPEND(inode)) + goto mnt_drop_write_and_out; + + error = get_write_access(inode); + if (error) + goto mnt_drop_write_and_out; + + /* + * Make sure that there are no leases. get_write_access() protects + * against the truncate racing with a lease-granting setlease(). + */ + error = break_lease(inode, O_WRONLY); + if (error) + goto put_write_and_out; + + error = security_path_truncate(path); + if (!error) + error = do_truncate(idmap, path->dentry, length, 0, NULL); + +put_write_and_out: + put_write_access(inode); +mnt_drop_write_and_out: + mnt_drop_write(path->mnt); + + return error; +} +EXPORT_SYMBOL_GPL(vfs_truncate); + +int ksys_truncate(const char __user *pathname, loff_t length) +{ + unsigned int lookup_flags = LOOKUP_FOLLOW; + struct path path; + int error; + + if (length < 0) /* sorry, but loff_t says... */ + return -EINVAL; + + CLASS(filename, name)(pathname); +retry: + error = filename_lookup(AT_FDCWD, name, lookup_flags, &path, NULL); + if (!error) { + error = vfs_truncate(&path, length); + path_put(&path); + if (retry_estale(error, lookup_flags)) { + lookup_flags |= LOOKUP_REVAL; + goto retry; + } + } + return error; +} + +SYSCALL_DEFINE2(truncate, const char __user *, path, long, length) +{ + return ksys_truncate(path, length); +} + +#ifdef CONFIG_COMPAT +COMPAT_SYSCALL_DEFINE2(truncate, const char __user *, path, compat_off_t, length) +{ + return ksys_truncate(path, length); +} +#endif + +int do_ftruncate(struct file *file, loff_t length, unsigned int flags) +{ + struct dentry *dentry = file->f_path.dentry; + struct inode *inode = dentry->d_inode; + int error; + + if (!S_ISREG(inode->i_mode) || !(file->f_mode & FMODE_WRITE)) + return -EINVAL; + + /* + * Cannot ftruncate over 2^31 bytes without large file support, either + * through opening with O_LARGEFILE or by using ftruncate64(). + */ + if (length > MAX_NON_LFS && + !(file->f_flags & O_LARGEFILE) && !(flags & FTRUNCATE_LFS)) + return -EINVAL; + + /* Check IS_APPEND on real upper inode */ + if (IS_APPEND(file_inode(file))) + return -EPERM; + + error = security_file_truncate(file); + if (error) + return error; + + error = fsnotify_truncate_perm(&file->f_path, length); + if (error) + return error; + + scoped_guard(super_write, inode->i_sb) + return do_truncate(file_mnt_idmap(file), dentry, length, + ATTR_MTIME | ATTR_CTIME, file); +} + +int ksys_ftruncate(unsigned int fd, loff_t length, unsigned int flags) +{ + if (length < 0) + return -EINVAL; + CLASS(fd, f)(fd); + if (fd_empty(f)) + return -EBADF; + + return do_ftruncate(fd_file(f), length, flags); +} + +SYSCALL_DEFINE2(ftruncate, unsigned int, fd, off_t, length) +{ + return ksys_ftruncate(fd, length, 0); +} + +#ifdef CONFIG_COMPAT +COMPAT_SYSCALL_DEFINE2(ftruncate, unsigned int, fd, compat_off_t, length) +{ + return ksys_ftruncate(fd, length, 0); +} +#endif + +/* LFS versions of truncate are only needed on 32 bit machines */ +#if BITS_PER_LONG == 32 +SYSCALL_DEFINE2(truncate64, const char __user *, path, loff_t, length) +{ + return ksys_truncate(path, length); +} + +SYSCALL_DEFINE2(ftruncate64, unsigned int, fd, loff_t, length) +{ + return ksys_ftruncate(fd, length, FTRUNCATE_LFS); +} +#endif /* BITS_PER_LONG == 32 */ + +#if defined(CONFIG_COMPAT) && defined(__ARCH_WANT_COMPAT_TRUNCATE64) +COMPAT_SYSCALL_DEFINE3(truncate64, const char __user *, pathname, + compat_arg_u64_dual(length)) +{ + return ksys_truncate(pathname, compat_arg_u64_glue(length)); +} +#endif + +#if defined(CONFIG_COMPAT) && defined(__ARCH_WANT_COMPAT_FTRUNCATE64) +COMPAT_SYSCALL_DEFINE3(ftruncate64, unsigned int, fd, + compat_arg_u64_dual(length)) +{ + return ksys_ftruncate(fd, compat_arg_u64_glue(length), FTRUNCATE_LFS); +} +#endif + +int vfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len) +{ + struct inode *inode = file_inode(file); + int ret; + loff_t sum; + + if (offset < 0 || len <= 0) + return -EINVAL; + + if (mode & ~(FALLOC_FL_MODE_MASK | FALLOC_FL_KEEP_SIZE)) + return -EOPNOTSUPP; + + /* + * Modes are exclusive, even if that is not obvious from the encoding + * as bit masks and the mix with the flag in the same namespace. + * + * To make things even more complicated, FALLOC_FL_ALLOCATE_RANGE is + * encoded as no bit set. + */ + switch (mode & FALLOC_FL_MODE_MASK) { + case FALLOC_FL_ALLOCATE_RANGE: + case FALLOC_FL_UNSHARE_RANGE: + case FALLOC_FL_ZERO_RANGE: + break; + case FALLOC_FL_PUNCH_HOLE: + if (!(mode & FALLOC_FL_KEEP_SIZE)) + return -EOPNOTSUPP; + break; + case FALLOC_FL_COLLAPSE_RANGE: + case FALLOC_FL_INSERT_RANGE: + case FALLOC_FL_WRITE_ZEROES: + if (mode & FALLOC_FL_KEEP_SIZE) + return -EOPNOTSUPP; + break; + default: + return -EOPNOTSUPP; + } + + if (!(file->f_mode & FMODE_WRITE)) + return -EBADF; + + /* + * On append-only files only space preallocation is supported. + */ + if ((mode & ~FALLOC_FL_KEEP_SIZE) && IS_APPEND(inode)) + return -EPERM; + + if (IS_IMMUTABLE(inode)) + return -EPERM; + + /* + * We cannot allow any fallocate operation on an active swapfile + */ + if (IS_SWAPFILE(inode)) + return -ETXTBSY; + + /* + * Revalidate the write permissions, in case security policy has + * changed since the files were opened. + */ + ret = security_file_permission(file, MAY_WRITE); + if (ret) + return ret; + + ret = fsnotify_file_area_perm(file, MAY_WRITE, &offset, len); + if (ret) + return ret; + + if (S_ISFIFO(inode->i_mode)) + return -ESPIPE; + + if (S_ISDIR(inode->i_mode)) + return -EISDIR; + + if (!S_ISREG(inode->i_mode) && !S_ISBLK(inode->i_mode)) + return -ENODEV; + + /* Check for wraparound */ + if (check_add_overflow(offset, len, &sum)) + return -EFBIG; + + if (sum > inode->i_sb->s_maxbytes) + return -EFBIG; + + if (!file->f_op->fallocate) + return -EOPNOTSUPP; + + file_start_write(file); + ret = file->f_op->fallocate(file, mode, offset, len); + + /* + * Create inotify and fanotify events. + * + * To keep the logic simple always create events if fallocate succeeds. + * This implies that events are even created if the file size remains + * unchanged, e.g. when using flag FALLOC_FL_KEEP_SIZE. + */ + if (ret == 0) + fsnotify_modify(file); + + file_end_write(file); + return ret; +} +EXPORT_SYMBOL_GPL(vfs_fallocate); + +int ksys_fallocate(int fd, int mode, loff_t offset, loff_t len) +{ + CLASS(fd, f)(fd); + + if (fd_empty(f)) + return -EBADF; + + return vfs_fallocate(fd_file(f), mode, offset, len); +} + +SYSCALL_DEFINE4(fallocate, int, fd, int, mode, loff_t, offset, loff_t, len) +{ + return ksys_fallocate(fd, mode, offset, len); +} + +#if defined(CONFIG_COMPAT) && defined(__ARCH_WANT_COMPAT_FALLOCATE) +COMPAT_SYSCALL_DEFINE6(fallocate, int, fd, int, mode, compat_arg_u64_dual(offset), + compat_arg_u64_dual(len)) +{ + return ksys_fallocate(fd, mode, compat_arg_u64_glue(offset), + compat_arg_u64_glue(len)); +} +#endif + +/* + * access() needs to use the real uid/gid, not the effective uid/gid. + * We do this by temporarily clearing all FS-related capabilities and + * switching the fsuid/fsgid around to the real ones. + * + * Creating new credentials is expensive, so we try to skip doing it, + * which we can if the result would match what we already got. + */ +static bool access_need_override_creds(int flags) +{ + const struct cred *cred; + + if (flags & AT_EACCESS) + return false; + + cred = current_cred(); + if (!uid_eq(cred->fsuid, cred->uid) || + !gid_eq(cred->fsgid, cred->gid)) + return true; + + if (!issecure(SECURE_NO_SETUID_FIXUP)) { + kuid_t root_uid = make_kuid(cred->user_ns, 0); + if (!uid_eq(cred->uid, root_uid)) { + if (!cap_isclear(cred->cap_effective)) + return true; + } else { + if (!cap_isidentical(cred->cap_effective, + cred->cap_permitted)) + return true; + } + } + + return false; +} + +static const struct cred *access_override_creds(void) +{ + struct cred *override_cred; + + override_cred = prepare_creds(); + if (!override_cred) + return NULL; + + /* + * XXX access_need_override_creds performs checks in hopes of skipping + * this work. Make sure it stays in sync if making any changes in this + * routine. + */ + + override_cred->fsuid = override_cred->uid; + override_cred->fsgid = override_cred->gid; + + if (!issecure(SECURE_NO_SETUID_FIXUP)) { + /* Clear the capabilities if we switch to a non-root user */ + kuid_t root_uid = make_kuid(override_cred->user_ns, 0); + if (!uid_eq(override_cred->uid, root_uid)) + cap_clear(override_cred->cap_effective); + else + override_cred->cap_effective = + override_cred->cap_permitted; + } + + /* + * The new set of credentials can *only* be used in + * task-synchronous circumstances, and does not need + * RCU freeing, unless somebody then takes a separate + * reference to it. + * + * NOTE! This is _only_ true because this credential + * is used purely for override_creds() that installs + * it as the subjective cred. Other threads will be + * accessing ->real_cred, not the subjective cred. + * + * If somebody _does_ make a copy of this (using the + * 'get_current_cred()' function), that will clear the + * non_rcu field, because now that other user may be + * expecting RCU freeing. But normal thread-synchronous + * cred accesses will keep things non-racy to avoid RCU + * freeing. + */ + override_cred->non_rcu = 1; + return override_creds(override_cred); +} + +static int do_faccessat(int dfd, const char __user *filename, int mode, int flags) +{ + struct path path; + struct inode *inode; + int res; + unsigned int lookup_flags = LOOKUP_FOLLOW; + const struct cred *old_cred = NULL; + + if (mode & ~S_IRWXO) /* where's F_OK, X_OK, W_OK, R_OK? */ + return -EINVAL; + + if (flags & ~(AT_EACCESS | AT_SYMLINK_NOFOLLOW | AT_EMPTY_PATH)) + return -EINVAL; + + if (flags & AT_SYMLINK_NOFOLLOW) + lookup_flags &= ~LOOKUP_FOLLOW; + + if (access_need_override_creds(flags)) { + old_cred = access_override_creds(); + if (!old_cred) + return -ENOMEM; + } + + CLASS(filename_uflags, name)(filename, flags); +retry: + res = filename_lookup(dfd, name, lookup_flags, &path, NULL); + if (res) + goto out; + + inode = d_backing_inode(path.dentry); + + if ((mode & MAY_EXEC) && S_ISREG(inode->i_mode)) { + /* + * MAY_EXEC on regular files is denied if the fs is mounted + * with the "noexec" flag. + */ + res = -EACCES; + if (path_noexec(&path)) + goto out_path_release; + } + + res = inode_permission(mnt_idmap(path.mnt), inode, mode | MAY_ACCESS); + /* SuS v2 requires we report a read only fs too */ + if (res || !(mode & S_IWOTH) || special_file(inode->i_mode)) + goto out_path_release; + /* + * This is a rare case where using __mnt_is_readonly() + * is OK without a mnt_want/drop_write() pair. Since + * no actual write to the fs is performed here, we do + * not need to telegraph to that to anyone. + * + * By doing this, we accept that this access is + * inherently racy and know that the fs may change + * state before we even see this result. + */ + if (__mnt_is_readonly(path.mnt)) + res = -EROFS; + +out_path_release: + path_put(&path); + if (retry_estale(res, lookup_flags)) { + lookup_flags |= LOOKUP_REVAL; + goto retry; + } +out: + if (old_cred) + put_cred(revert_creds(old_cred)); + + return res; +} + +SYSCALL_DEFINE3(faccessat, int, dfd, const char __user *, filename, int, mode) +{ + return do_faccessat(dfd, filename, mode, 0); +} + +SYSCALL_DEFINE4(faccessat2, int, dfd, const char __user *, filename, int, mode, + int, flags) +{ + return do_faccessat(dfd, filename, mode, flags); +} + +SYSCALL_DEFINE2(access, const char __user *, filename, int, mode) +{ + return do_faccessat(AT_FDCWD, filename, mode, 0); +} + +SYSCALL_DEFINE1(chdir, const char __user *, filename) +{ + struct path path; + int error; + unsigned int lookup_flags = LOOKUP_FOLLOW | LOOKUP_DIRECTORY; + CLASS(filename, name)(filename); +retry: + error = filename_lookup(AT_FDCWD, name, lookup_flags, &path, NULL); + if (!error) { + error = path_permission(&path, MAY_EXEC | MAY_CHDIR); + if (!error) + set_fs_pwd(current->fs, &path); + path_put(&path); + if (retry_estale(error, lookup_flags)) { + lookup_flags |= LOOKUP_REVAL; + goto retry; + } + } + return error; +} + +SYSCALL_DEFINE1(fchdir, unsigned int, fd) +{ + int error; + + if ((int)fd == FD_FAILFS_ROOT) + return failfs_current_chdir(); + + CLASS(fd_raw, f)(fd); + if (fd_empty(f)) + return -EBADF; + + if (!d_can_lookup(fd_file(f)->f_path.dentry)) + return -ENOTDIR; + + error = file_permission(fd_file(f), MAY_EXEC | MAY_CHDIR); + if (!error) + set_fs_pwd(current->fs, &fd_file(f)->f_path); + return error; +} + +SYSCALL_DEFINE1(chroot, const char __user *, filename) +{ + struct path path; + int error; + unsigned int lookup_flags = LOOKUP_FOLLOW | LOOKUP_DIRECTORY; + CLASS(filename, name)(filename); +retry: + error = filename_lookup(AT_FDCWD, name, lookup_flags, &path, NULL); + if (error) + return error; + + error = path_permission(&path, MAY_EXEC | MAY_CHDIR); + if (error) + goto dput_and_out; + + error = -EPERM; + if (!ns_capable(current_user_ns(), CAP_SYS_CHROOT)) + goto dput_and_out; + error = security_path_chroot(&path); + if (!error) + set_fs_root(current->fs, &path); +dput_and_out: + path_put(&path); + if (retry_estale(error, lookup_flags)) { + lookup_flags |= LOOKUP_REVAL; + goto retry; + } + return error; +} + +SYSCALL_DEFINE2(fchroot, int, fd, unsigned int, flags) +{ + struct path path; + int error; + + if (flags) + return -EINVAL; + + if (fd == FD_FAILFS_ROOT) { + if (!ns_capable(current_user_ns(), CAP_SYS_CHROOT)) { + if (!task_no_new_privs(current)) + return -EPERM; + /* A shared fs_struct lets a sibling exec setuid past the check above. */ + if (current->fs->users != 1) + return -EINVAL; + /* Moving the root to failfs lifts the old root's ".." barrier. */ + if (current_chrooted()) + return -EPERM; + } + failfs_get_root(&path); + } else { + CLASS(fd_raw, f)(fd); + if (fd_empty(f)) + return -EBADF; + + if (!d_can_lookup(fd_file(f)->f_path.dentry)) + return -ENOTDIR; + + error = file_permission(fd_file(f), MAY_EXEC | MAY_CHDIR); + if (error) + return error; + + if (!ns_capable(current_user_ns(), CAP_SYS_CHROOT)) + return -EPERM; + + path = fd_file(f)->f_path; + path_get(&path); + } + + error = security_path_chroot(&path); + if (!error) + set_fs_root(current->fs, &path); + path_put(&path); + return error; +} + +int chmod_common(const struct path *path, umode_t mode) +{ + struct inode *inode = path->dentry->d_inode; + struct delegated_inode delegated_inode = { }; + struct iattr newattrs; + int error; + + error = mnt_want_write(path->mnt); + if (error) + return error; +retry_deleg: + error = inode_lock_killable(inode); + if (error) + goto out_mnt_unlock; + error = security_path_chmod(path, mode); + if (error) + goto out_unlock; + newattrs.ia_mode = (mode & S_IALLUGO) | (inode->i_mode & ~S_IALLUGO); + newattrs.ia_valid = ATTR_MODE | ATTR_CTIME; + error = notify_change(mnt_idmap(path->mnt), path->dentry, + &newattrs, &delegated_inode); +out_unlock: + inode_unlock(inode); + if (is_delegated(&delegated_inode)) { + error = break_deleg_wait(&delegated_inode); + if (!error) + goto retry_deleg; + } +out_mnt_unlock: + mnt_drop_write(path->mnt); + return error; +} + +int vfs_fchmod(struct file *file, umode_t mode) +{ + audit_file(file); + return chmod_common(&file->f_path, mode); +} + +SYSCALL_DEFINE2(fchmod, unsigned int, fd, umode_t, mode) +{ + CLASS(fd, f)(fd); + + if (fd_empty(f)) + return -EBADF; + + return vfs_fchmod(fd_file(f), mode); +} + +static int do_fchmodat(int dfd, const char __user *filename, umode_t mode, + unsigned int flags) +{ + struct path path; + int error; + unsigned int lookup_flags; + + if (unlikely(flags & ~(AT_SYMLINK_NOFOLLOW | AT_EMPTY_PATH))) + return -EINVAL; + + lookup_flags = (flags & AT_SYMLINK_NOFOLLOW) ? 0 : LOOKUP_FOLLOW; + CLASS(filename_uflags, name)(filename, flags); +retry: + error = filename_lookup(dfd, name, lookup_flags, &path, NULL); + if (!error) { + error = chmod_common(&path, mode); + path_put(&path); + if (retry_estale(error, lookup_flags)) { + lookup_flags |= LOOKUP_REVAL; + goto retry; + } + } + return error; +} + +SYSCALL_DEFINE4(fchmodat2, int, dfd, const char __user *, filename, + umode_t, mode, unsigned int, flags) +{ + return do_fchmodat(dfd, filename, mode, flags); +} + +SYSCALL_DEFINE3(fchmodat, int, dfd, const char __user *, filename, + umode_t, mode) +{ + return do_fchmodat(dfd, filename, mode, 0); +} + +SYSCALL_DEFINE2(chmod, const char __user *, filename, umode_t, mode) +{ + return do_fchmodat(AT_FDCWD, filename, mode, 0); +} + +/* + * Check whether @kuid is valid and if so generate and set vfsuid_t in + * ia_vfsuid. + * + * Return: true if @kuid is valid, false if not. + */ +static inline bool setattr_vfsuid(struct iattr *attr, kuid_t kuid) +{ + if (!uid_valid(kuid)) + return false; + attr->ia_valid |= ATTR_UID; + attr->ia_vfsuid = VFSUIDT_INIT(kuid); + return true; +} + +/* + * Check whether @kgid is valid and if so generate and set vfsgid_t in + * ia_vfsgid. + * + * Return: true if @kgid is valid, false if not. + */ +static inline bool setattr_vfsgid(struct iattr *attr, kgid_t kgid) +{ + if (!gid_valid(kgid)) + return false; + attr->ia_valid |= ATTR_GID; + attr->ia_vfsgid = VFSGIDT_INIT(kgid); + return true; +} + +int chown_common(const struct path *path, uid_t user, gid_t group) +{ + struct mnt_idmap *idmap; + struct user_namespace *fs_userns; + struct inode *inode = path->dentry->d_inode; + struct delegated_inode delegated_inode = { }; + int error; + struct iattr newattrs; + kuid_t uid; + kgid_t gid; + + uid = make_kuid(current_user_ns(), user); + gid = make_kgid(current_user_ns(), group); + + idmap = mnt_idmap(path->mnt); + fs_userns = i_user_ns(inode); + +retry_deleg: + newattrs.ia_vfsuid = INVALID_VFSUID; + newattrs.ia_vfsgid = INVALID_VFSGID; + newattrs.ia_valid = ATTR_CTIME; + if ((user != (uid_t)-1) && !setattr_vfsuid(&newattrs, uid)) + return -EINVAL; + if ((group != (gid_t)-1) && !setattr_vfsgid(&newattrs, gid)) + return -EINVAL; + error = inode_lock_killable(inode); + if (error) + return error; + if (!S_ISDIR(inode->i_mode)) + newattrs.ia_valid |= ATTR_KILL_SUID | ATTR_KILL_PRIV | + setattr_should_drop_sgid(idmap, inode); + /* Continue to send actual fs values, not the mount values. */ + error = security_path_chown( + path, + from_vfsuid(idmap, fs_userns, newattrs.ia_vfsuid), + from_vfsgid(idmap, fs_userns, newattrs.ia_vfsgid)); + if (!error) + error = notify_change(idmap, path->dentry, &newattrs, + &delegated_inode); + inode_unlock(inode); + if (is_delegated(&delegated_inode)) { + error = break_deleg_wait(&delegated_inode); + if (!error) + goto retry_deleg; + } + return error; +} + +int do_fchownat(int dfd, const char __user *filename, uid_t user, gid_t group, + int flag) +{ + struct path path; + int error; + int lookup_flags; + + if ((flag & ~(AT_SYMLINK_NOFOLLOW | AT_EMPTY_PATH)) != 0) + return -EINVAL; + + lookup_flags = (flag & AT_SYMLINK_NOFOLLOW) ? 0 : LOOKUP_FOLLOW; + CLASS(filename_uflags, name)(filename, flag); +retry: + error = filename_lookup(dfd, name, lookup_flags, &path, NULL); + if (!error) { + error = mnt_want_write(path.mnt); + if (!error) { + error = chown_common(&path, user, group); + mnt_drop_write(path.mnt); + } + path_put(&path); + if (retry_estale(error, lookup_flags)) { + lookup_flags |= LOOKUP_REVAL; + goto retry; + } + } + return error; +} + +SYSCALL_DEFINE5(fchownat, int, dfd, const char __user *, filename, uid_t, user, + gid_t, group, int, flag) +{ + return do_fchownat(dfd, filename, user, group, flag); +} + +SYSCALL_DEFINE3(chown, const char __user *, filename, uid_t, user, gid_t, group) +{ + return do_fchownat(AT_FDCWD, filename, user, group, 0); +} + +SYSCALL_DEFINE3(lchown, const char __user *, filename, uid_t, user, gid_t, group) +{ + return do_fchownat(AT_FDCWD, filename, user, group, + AT_SYMLINK_NOFOLLOW); +} + +int vfs_fchown(struct file *file, uid_t user, gid_t group) +{ + int error; + + error = mnt_want_write_file(file); + if (error) + return error; + audit_file(file); + error = chown_common(&file->f_path, user, group); + mnt_drop_write_file(file); + return error; +} + +int ksys_fchown(unsigned int fd, uid_t user, gid_t group) +{ + CLASS(fd, f)(fd); + + if (fd_empty(f)) + return -EBADF; + + return vfs_fchown(fd_file(f), user, group); +} + +SYSCALL_DEFINE3(fchown, unsigned int, fd, uid_t, user, gid_t, group) +{ + return ksys_fchown(fd, user, group); +} + +static inline int file_get_write_access(struct file *f) +{ + int error; + + error = get_write_access(f->f_inode); + if (unlikely(error)) + return error; + error = mnt_get_write_access(f->f_path.mnt); + if (unlikely(error)) + goto cleanup_inode; + if (unlikely(f->f_mode & FMODE_BACKING)) { + error = mnt_get_write_access(backing_file_user_path(f)->mnt); + if (unlikely(error)) + goto cleanup_mnt; + } + return 0; + +cleanup_mnt: + mnt_put_write_access(f->f_path.mnt); +cleanup_inode: + put_write_access(f->f_inode); + return error; +} + +static int do_dentry_open(struct file *f, + int (*open)(struct inode *, struct file *)) +{ + static const struct file_operations empty_fops = {}; + struct inode *inode = f->f_path.dentry->d_inode; + int error; + + path_get(&f->f_path); + f->f_inode = inode; + f->f_mapping = inode->i_mapping; + f->f_wb_err = filemap_sample_wb_err(f->f_mapping); + f->f_sb_err = file_sample_sb_err(f); + + if (unlikely(f->f_flags & O_PATH)) { + f->f_mode = FMODE_PATH | FMODE_OPENED; + file_set_fsnotify_mode(f, FMODE_NONOTIFY); + f->f_op = &empty_fops; + return 0; + } + + if ((f->f_mode & (FMODE_READ | FMODE_WRITE)) == FMODE_READ) { + i_readcount_inc(inode); + } else if (f->f_mode & FMODE_WRITE && !special_file(inode->i_mode)) { + error = file_get_write_access(f); + if (unlikely(error)) + goto cleanup_file; + f->f_mode |= FMODE_WRITER; + } + + /* POSIX.1-2008/SUSv4 Section XSI 2.9.7 */ + if (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode)) + f->f_mode |= FMODE_ATOMIC_POS; + + f->f_op = fops_get(inode->i_fop); + if (WARN_ON(!f->f_op)) { + error = -ENODEV; + goto cleanup_all; + } + + error = security_file_open(f); + if (unlikely(error)) + goto cleanup_all; + + /* + * Call fsnotify open permission hook and set FMODE_NONOTIFY_* bits + * according to existing permission watches. + * If FMODE_NONOTIFY mode was already set for an fanotify fd or for a + * pseudo file, this call will not change the mode. + */ + error = fsnotify_open_perm_and_set_mode(f); + if (unlikely(error)) + goto cleanup_all; + + error = break_lease(file_inode(f), f->f_flags); + if (unlikely(error)) + goto cleanup_all; + + /* normally all 3 are set; ->open() can clear them if needed */ + f->f_mode |= FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE; + if (!open) + open = f->f_op->open; + if (open) { + error = open(inode, f); + if (error) + goto cleanup_all; + } + f->f_mode |= FMODE_OPENED; + if ((f->f_mode & FMODE_READ) && + likely(f->f_op->read || f->f_op->read_iter)) + f->f_mode |= FMODE_CAN_READ; + if ((f->f_mode & FMODE_WRITE) && + likely(f->f_op->write || f->f_op->write_iter)) + f->f_mode |= FMODE_CAN_WRITE; + if ((f->f_mode & FMODE_LSEEK) && !f->f_op->llseek) + f->f_mode &= ~FMODE_LSEEK; + if (f->f_mapping->a_ops && f->f_mapping->a_ops->direct_IO) + f->f_mode |= FMODE_CAN_ODIRECT; + + f->f_flags &= ~(O_CREAT | O_EXCL | O_NOCTTY | O_TRUNC | __O_REGULAR); + f->f_iocb_flags = iocb_flags(f); + + file_ra_state_init(&f->f_ra, f->f_mapping->host->i_mapping); + + if ((f->f_flags & O_DIRECT) && !(f->f_mode & FMODE_CAN_ODIRECT)) + return -EINVAL; + + return 0; + +cleanup_all: + if (WARN_ON_ONCE(error > 0)) + error = -EINVAL; + fops_put(f->f_op); + put_file_access(f); +cleanup_file: + path_put(&f->f_path); + f->__f_path.mnt = NULL; + f->__f_path.dentry = NULL; + f->f_inode = NULL; + return error; +} + +/** + * finish_open - finish opening a file + * @file: file pointer + * @dentry: pointer to dentry + * @open: open callback + * + * This can be used to finish opening a file passed to i_op->atomic_open(). + * + * If the open callback is set to NULL, then the standard f_op->open() + * filesystem callback is substituted. + * + * NB: the dentry reference is _not_ consumed. If, for example, the dentry is + * the return value of d_splice_alias(), then the caller needs to perform dput() + * on it after finish_open(). + * + * Returns zero on success or -errno if the open failed. + */ +int finish_open(struct file *file, struct dentry *dentry, + int (*open)(struct inode *, struct file *)) +{ + BUG_ON(file->f_mode & FMODE_OPENED); /* once it's opened, it's opened */ + + file->__f_path.dentry = dentry; + return do_dentry_open(file, open); +} +EXPORT_SYMBOL(finish_open); + +/** + * finish_no_open - finish ->atomic_open() without opening the file + * + * @file: file pointer + * @dentry: dentry, ERR_PTR(-E...) or NULL (as returned from ->lookup()) + * + * This can be used to set the result of a lookup in ->atomic_open(). + * + * NB: unlike finish_open() this function does consume the dentry reference and + * the caller need not dput() it. + * + * Returns 0 or -E..., which must be the return value of ->atomic_open() after + * having called this function. + */ +int finish_no_open(struct file *file, struct dentry *dentry) +{ + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + file->__f_path.dentry = dentry; + return 0; +} +EXPORT_SYMBOL(finish_no_open); + +char *file_path(struct file *filp, char *buf, int buflen) +{ + return d_path(&filp->f_path, buf, buflen); +} +EXPORT_SYMBOL(file_path); + +/** + * vfs_open - open the file at the given path + * @path: path to open + * @file: newly allocated file with f_flag initialized + */ +int vfs_open(const struct path *path, struct file *file) +{ + int ret; + + file->__f_path = *path; + ret = do_dentry_open(file, NULL); + if (!ret) { + /* + * Once we return a file with FMODE_OPENED, __fput() will call + * fsnotify_close(), so we need fsnotify_open() here for + * symmetry. + */ + fsnotify_open(file); + } + return ret; +} + +struct file *dentry_open(const struct path *path, int flags, + const struct cred *cred) +{ + int error; + struct file *f; + + /* We must always pass in a valid mount pointer. */ + BUG_ON(!path->mnt); + + f = alloc_empty_file(flags, cred); + if (!IS_ERR(f)) { + error = vfs_open(path, f); + if (error) { + fput(f); + f = ERR_PTR(error); + } + } + return f; +} +EXPORT_SYMBOL(dentry_open); + +struct file *dentry_open_nonotify(const struct path *path, int flags, + const struct cred *cred) +{ + struct file *f = alloc_empty_file(flags, cred); + if (!IS_ERR(f)) { + int error; + + file_set_fsnotify_mode(f, FMODE_NONOTIFY); + error = vfs_open(path, f); + if (error) { + fput(f); + f = ERR_PTR(error); + } + } + return f; +} + +/** + * kernel_file_open - open a file for kernel internal use + * @path: path of the file to open + * @flags: open flags + * @cred: credentials for open + * + * Open a file for use by in-kernel consumers. The file is not accounted + * against nr_files and must not be installed into the file descriptor + * table. + * + * Return: Opened file on success, an error pointer on failure. + */ +struct file *kernel_file_open(const struct path *path, int flags, + const struct cred *cred) +{ + struct file *f; + int error; + + f = alloc_empty_file_noaccount(flags, cred); + if (IS_ERR(f)) + return f; + + error = vfs_open(path, f); + if (error) { + fput(f); + return ERR_PTR(error); + } + return f; +} +EXPORT_SYMBOL_GPL(kernel_file_open); + +#define WILL_CREATE(flags) (flags & (O_CREAT | __O_TMPFILE)) +#define O_PATH_FLAGS (O_DIRECTORY | O_NOFOLLOW | O_PATH | O_CLOEXEC | O_EMPTYPATH) + +inline struct open_how build_open_how(int flags, umode_t mode) +{ + struct open_how how = { + .flags = flags & VALID_OPEN_FLAGS, + .mode = mode & S_IALLUGO, + }; + + /* O_PATH beats everything else. */ + if (how.flags & O_PATH) + how.flags &= O_PATH_FLAGS; + /* Modes should only be set for create-like flags. */ + if (!WILL_CREATE(how.flags)) + how.mode = 0; + return how; +} + +inline int build_open_flags(const struct open_how *how, struct open_flags *op) +{ + u64 flags = how->flags; + u64 strip = O_CLOEXEC; + int lookup_flags = 0; + int acc_mode = ACC_MODE(flags); + + BUILD_BUG_ON_MSG(upper_32_bits(VALID_OPEN_FLAGS), + "VALID_OPEN_FLAGS must fit in 32 bits"); + /* The whole point: OPENAT2_REGULAR must be unrepresentable in int. */ + BUILD_BUG_ON_MSG(!upper_32_bits(OPENAT2_REGULAR), + "OPENAT2_REGULAR must live in the upper 32 bits of open_how::flags"); + /* Prevent a future bit collision between UAPI and internal carrier. */ + BUILD_BUG_ON_MSG(OPENAT2_REGULAR & VALID_OPEN_FLAGS, + "OPENAT2_REGULAR must not alias any open()/openat() flag"); + BUILD_BUG_ON_MSG(__O_REGULAR & VALID_OPENAT2_FLAGS, + "__O_REGULAR must not alias any user-visible flag"); + + /* + * Strip flags that aren't relevant in determining struct open_flags. + */ + flags &= ~strip; + + /* + * Older syscalls implicitly clear all of the invalid flags or argument + * values before calling build_open_flags(), but openat2(2) checks all + * of its arguments. + */ + if (flags & ~VALID_OPENAT2_FLAGS) + return -EINVAL; + if (how->resolve & ~VALID_RESOLVE_FLAGS) + return -EINVAL; + + /* Scoping flags are mutually exclusive. */ + if ((how->resolve & RESOLVE_BENEATH) && (how->resolve & RESOLVE_IN_ROOT)) + return -EINVAL; + + /* Deal with the mode. */ + if (WILL_CREATE(flags)) { + if (how->mode & ~S_IALLUGO) + return -EINVAL; + op->mode = how->mode | S_IFREG; + } else { + if (how->mode != 0) + return -EINVAL; + op->mode = 0; + } + + /* + * Block bugs where O_DIRECTORY | O_CREAT created regular files. + * Note, that blocking O_DIRECTORY | O_CREAT here also protects + * O_TMPFILE below which requires O_DIRECTORY being raised. + */ + if ((flags & (O_DIRECTORY | O_CREAT)) == (O_DIRECTORY | O_CREAT)) + return -EINVAL; + + /* Now handle the creative implementation of O_TMPFILE. */ + if (flags & __O_TMPFILE) { + /* + * In order to ensure programs get explicit errors when trying + * to use O_TMPFILE on old kernels we enforce that O_DIRECTORY + * is raised alongside __O_TMPFILE. + */ + if (!(flags & O_DIRECTORY)) + return -EINVAL; + if (!(acc_mode & MAY_WRITE)) + return -EINVAL; + } + /* + * Asking to open a directory and a regular file at the same time is + * contradictory. + */ + if ((flags & (O_DIRECTORY | OPENAT2_REGULAR)) == + (O_DIRECTORY | OPENAT2_REGULAR)) + return -EINVAL; + + if (flags & O_PATH) { + /* O_PATH only permits certain other flags to be set. */ + if (flags & ~O_PATH_FLAGS) + return -EINVAL; + acc_mode = 0; + } + + /* + * O_SYNC is implemented as __O_SYNC|O_DSYNC. As many places only + * check for O_DSYNC if the need any syncing at all we enforce it's + * always set instead of having to deal with possibly weird behaviour + * for malicious applications setting only __O_SYNC. + */ + if (flags & __O_SYNC) + flags |= O_DSYNC; + + /* + * Translate the upper-32-bit UAPI bit OPENAT2_REGULAR into the + * kernel-internal lower-32-bit __O_REGULAR carrier so the bit + * survives the assignment to op->open_flag (an int) below and the + * subsequent flow through f->f_flags (unsigned int) and the + * i_op->atomic_open() callback (unsigned). do_dentry_open() strips + * __O_REGULAR before the file becomes visible to userspace. + */ + if (flags & OPENAT2_REGULAR) { + flags &= ~OPENAT2_REGULAR; + flags |= __O_REGULAR; + } + + op->open_flag = flags; + + /* O_TRUNC implies we need access checks for write permissions */ + if (flags & O_TRUNC) + acc_mode |= MAY_WRITE; + + /* Allow the LSM permission hook to distinguish append + access from general write access. */ + if (flags & O_APPEND) + acc_mode |= MAY_APPEND; + + op->acc_mode = acc_mode; + + op->intent = flags & O_PATH ? 0 : LOOKUP_OPEN; + + if (flags & O_CREAT) { + op->intent |= LOOKUP_CREATE; + if (flags & O_EXCL) { + op->intent |= LOOKUP_EXCL; + flags |= O_NOFOLLOW; + } + } + + if (flags & O_DIRECTORY) + lookup_flags |= LOOKUP_DIRECTORY; + if (!(flags & O_NOFOLLOW)) + lookup_flags |= LOOKUP_FOLLOW; + if (flags & O_EMPTYPATH) + lookup_flags |= LOOKUP_EMPTY; + + if (how->resolve & RESOLVE_NO_XDEV) + lookup_flags |= LOOKUP_NO_XDEV; + if (how->resolve & RESOLVE_NO_MAGICLINKS) + lookup_flags |= LOOKUP_NO_MAGICLINKS; + if (how->resolve & RESOLVE_NO_SYMLINKS) + lookup_flags |= LOOKUP_NO_SYMLINKS; + if (how->resolve & RESOLVE_BENEATH) + lookup_flags |= LOOKUP_BENEATH; + if (how->resolve & RESOLVE_IN_ROOT) + lookup_flags |= LOOKUP_IN_ROOT; + if (how->resolve & RESOLVE_CACHED) { + /* Don't bother even trying for create/truncate/tmpfile open */ + if (flags & (O_TRUNC | O_CREAT | __O_TMPFILE)) + return -EAGAIN; + lookup_flags |= LOOKUP_CACHED; + } + + op->lookup_flags = lookup_flags; + return 0; +} + +/** + * file_open_name - open file and return file pointer + * + * @name: struct filename containing path to open + * @flags: open flags as per the open(2) second argument + * @mode: mode for the new file if O_CREAT is set, else ignored + * + * This is the helper to open a file from kernelspace if you really + * have to. But in generally you should not do this, so please move + * along, nothing to see here.. + */ +struct file *file_open_name(struct filename *name, int flags, umode_t mode) +{ + struct open_flags op; + struct open_how how = build_open_how(flags, mode); + int err = build_open_flags(&how, &op); + if (err) + return ERR_PTR(err); + return do_file_open(AT_FDCWD, name, &op); +} + +/** + * filp_open - open file and return file pointer + * + * @filename: path to open + * @flags: open flags as per the open(2) second argument + * @mode: mode for the new file if O_CREAT is set, else ignored + * + * This is the helper to open a file from kernelspace if you really + * have to. But in generally you should not do this, so please move + * along, nothing to see here.. + */ +struct file *filp_open(const char *filename, int flags, umode_t mode) +{ + CLASS(filename_kernel, name)(filename); + return file_open_name(name, flags, mode); +} +EXPORT_SYMBOL(filp_open); + +struct file *file_open_root(const struct path *root, + const char *filename, int flags, umode_t mode) +{ + struct open_flags op; + struct open_how how = build_open_how(flags, mode); + int err = build_open_flags(&how, &op); + if (err) + return ERR_PTR(err); + return do_file_open_root(root, filename, &op); +} +EXPORT_SYMBOL(file_open_root); + +static int do_sys_openat2(int dfd, const char __user *filename, + struct open_how *how) +{ + struct open_flags op; + int err = build_open_flags(how, &op); + if (unlikely(err)) + return err; + + CLASS(filename_flags, name)(filename, op.lookup_flags); + return FD_ADD(how->flags, do_file_open(dfd, name, &op)); +} + +int do_sys_open(int dfd, const char __user *filename, int flags, umode_t mode) +{ + struct open_how how = build_open_how(flags, mode); + return do_sys_openat2(dfd, filename, &how); +} + + +SYSCALL_DEFINE3(open, const char __user *, filename, int, flags, umode_t, mode) +{ + if (force_o_largefile()) + flags |= O_LARGEFILE; + return do_sys_open(AT_FDCWD, filename, flags, mode); +} + +SYSCALL_DEFINE4(openat, int, dfd, const char __user *, filename, int, flags, + umode_t, mode) +{ + if (force_o_largefile()) + flags |= O_LARGEFILE; + return do_sys_open(dfd, filename, flags, mode); +} + +SYSCALL_DEFINE4(openat2, int, dfd, const char __user *, filename, + struct open_how __user *, how, size_t, usize) +{ + int err; + struct open_how tmp; + + BUILD_BUG_ON(sizeof(struct open_how) < OPEN_HOW_SIZE_VER0); + BUILD_BUG_ON(sizeof(struct open_how) != OPEN_HOW_SIZE_LATEST); + + if (unlikely(usize < OPEN_HOW_SIZE_VER0)) + return -EINVAL; + if (unlikely(usize > PAGE_SIZE)) + return -E2BIG; + + err = copy_struct_from_user(&tmp, sizeof(tmp), how, usize); + if (err) + return err; + + audit_openat2_how(&tmp); + + /* O_LARGEFILE is only allowed for non-O_PATH. */ + if (!(tmp.flags & O_PATH) && force_o_largefile()) + tmp.flags |= O_LARGEFILE; + + return do_sys_openat2(dfd, filename, &tmp); +} + +#ifdef CONFIG_COMPAT +/* + * Exactly like sys_open(), except that it doesn't set the + * O_LARGEFILE flag. + */ +COMPAT_SYSCALL_DEFINE3(open, const char __user *, filename, int, flags, umode_t, mode) +{ + return do_sys_open(AT_FDCWD, filename, flags, mode); +} + +/* + * Exactly like sys_openat(), except that it doesn't set the + * O_LARGEFILE flag. + */ +COMPAT_SYSCALL_DEFINE4(openat, int, dfd, const char __user *, filename, int, flags, umode_t, mode) +{ + return do_sys_open(dfd, filename, flags, mode); +} +#endif + +#ifndef __alpha__ + +/* + * For backward compatibility? Maybe this should be moved + * into arch/i386 instead? + */ +SYSCALL_DEFINE2(creat, const char __user *, pathname, umode_t, mode) +{ + int flags = O_CREAT | O_WRONLY | O_TRUNC; + + if (force_o_largefile()) + flags |= O_LARGEFILE; + return do_sys_open(AT_FDCWD, pathname, flags, mode); +} +#endif + +/* + * "id" is the POSIX thread ID. We use the + * files pointer for this.. + */ +static int filp_flush(struct file *filp, fl_owner_t id) +{ + int retval = 0; + + if (CHECK_DATA_CORRUPTION(file_count(filp) == 0, filp, + "VFS: Close: file count is 0 (f_op=%ps)", + filp->f_op)) { + return 0; + } + + if (filp->f_op->flush) + retval = filp->f_op->flush(filp, id); + + if (likely(!(filp->f_mode & FMODE_PATH))) { + dnotify_flush(filp, id); + locks_remove_posix(filp, id); + } + return retval; +} + +int filp_close(struct file *filp, fl_owner_t id) +{ + int retval; + + retval = filp_flush(filp, id); + fput_close(filp); + + return retval; +} +EXPORT_SYMBOL(filp_close); + +/* + * Careful here! We test whether the file pointer is NULL before + * releasing the fd. This ensures that one clone task can't release + * an fd while another clone is opening it. + */ +SYSCALL_DEFINE1(close, unsigned int, fd) +{ + int retval; + struct file *file; + + file = file_close_fd(fd); + if (!file) + return -EBADF; + + retval = filp_flush(file, current->files); + + /* + * We're returning to user space. Don't bother + * with any delayed fput() cases. + */ + fput_close_sync(file); + + if (likely(retval == 0)) + return 0; + + /* can't restart close syscall because file table entry was cleared */ + if (retval == -ERESTARTSYS || + retval == -ERESTARTNOINTR || + retval == -ERESTARTNOHAND || + retval == -ERESTART_RESTARTBLOCK) + retval = -EINTR; + + return retval; +} + +/* + * This routine simulates a hangup on the tty, to arrange that users + * are given clean terminals at login time. + */ +SYSCALL_DEFINE0(vhangup) +{ + if (capable(CAP_SYS_TTY_CONFIG)) { + tty_vhangup_self(); + return 0; + } + return -EPERM; +} + +/* + * Called when an inode is about to be open. + * We use this to disallow opening large files on 32bit systems if + * the caller didn't specify O_LARGEFILE. On 64bit systems we force + * on this flag in sys_open. + */ +int generic_file_open(struct inode * inode, struct file * filp) +{ + if (!(filp->f_flags & O_LARGEFILE) && i_size_read(inode) > MAX_NON_LFS) + return -EOVERFLOW; + return 0; +} + +EXPORT_SYMBOL(generic_file_open); + +/* + * This is used by subsystems that don't want seekable + * file descriptors. The function is not supposed to ever fail, the only + * reason it returns an 'int' and not 'void' is so that it can be plugged + * directly into file_operations structure. + */ +int nonseekable_open(struct inode *inode, struct file *filp) +{ + filp->f_mode &= ~(FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE); + return 0; +} + +EXPORT_SYMBOL(nonseekable_open); + +/* + * stream_open is used by subsystems that want stream-like file descriptors. + * Such file descriptors are not seekable and don't have notion of position + * (file.f_pos is always 0 and ppos passed to .read()/.write() is always NULL). + * Contrary to file descriptors of other regular files, .read() and .write() + * can run simultaneously. + * + * stream_open never fails and is marked to return int so that it could be + * directly used as file_operations.open . + */ +int stream_open(struct inode *inode, struct file *filp) +{ + filp->f_mode &= ~(FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE | FMODE_ATOMIC_POS); + filp->f_mode |= FMODE_STREAM; + return 0; +} + +EXPORT_SYMBOL(stream_open); diff --git a/fs/openpromfs/Makefile b/fs/openpromfs/Makefile new file mode 100644 index 000000000..843c270cd --- /dev/null +++ b/fs/openpromfs/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the Linux Sun Openprom filesystem routines. +# + +obj-$(CONFIG_SUN_OPENPROMFS) += openpromfs.o + +openpromfs-objs := inode.o diff --git a/fs/openpromfs/inode.c b/fs/openpromfs/inode.c new file mode 100644 index 000000000..fb8d84bde --- /dev/null +++ b/fs/openpromfs/inode.c @@ -0,0 +1,475 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* inode.c: /proc/openprom handling routines + * + * Copyright (C) 1996-1999 Jakub Jelinek (jakub@redhat.com) + * Copyright (C) 1998 Eddie C. Dost (ecd@skynet.be) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +static DEFINE_MUTEX(op_mutex); + +#define OPENPROM_ROOT_INO 0 + +enum op_inode_type { + op_inode_node, + op_inode_prop, +}; + +union op_inode_data { + struct device_node *node; + struct property *prop; +}; + +struct op_inode_info { + struct inode vfs_inode; + enum op_inode_type type; + union op_inode_data u; +}; + +static struct inode *openprom_iget(struct super_block *sb, ino_t ino); + +static inline struct op_inode_info *OP_I(struct inode *inode) +{ + return container_of(inode, struct op_inode_info, vfs_inode); +} + +static int is_string(unsigned char *p, int len) +{ + int i; + + for (i = 0; i < len; i++) { + unsigned char val = p[i]; + + if ((i && !val) || + (val >= ' ' && val <= '~')) + continue; + + return 0; + } + + return 1; +} + +static int property_show(struct seq_file *f, void *v) +{ + struct property *prop = f->private; + void *pval; + int len; + + len = prop->length; + pval = prop->value; + + if (is_string(pval, len)) { + while (len > 0) { + int n = strlen(pval); + + seq_printf(f, "%s", (char *) pval); + + /* Skip over the NULL byte too. */ + pval += n + 1; + len -= n + 1; + + if (len > 0) + seq_printf(f, " + "); + } + } else { + if (len & 3) { + while (len) { + len--; + if (len) + seq_printf(f, "%02x.", + *(unsigned char *) pval); + else + seq_printf(f, "%02x", + *(unsigned char *) pval); + pval++; + } + } else { + while (len >= 4) { + len -= 4; + + if (len) + seq_printf(f, "%08x.", + *(unsigned int *) pval); + else + seq_printf(f, "%08x", + *(unsigned int *) pval); + pval += 4; + } + } + } + seq_printf(f, "\n"); + + return 0; +} + +static void *property_start(struct seq_file *f, loff_t *pos) +{ + if (*pos == 0) + return pos; + return NULL; +} + +static void *property_next(struct seq_file *f, void *v, loff_t *pos) +{ + (*pos)++; + return NULL; +} + +static void property_stop(struct seq_file *f, void *v) +{ + /* Nothing to do */ +} + +static const struct seq_operations property_op = { + .start = property_start, + .next = property_next, + .stop = property_stop, + .show = property_show +}; + +static int property_open(struct inode *inode, struct file *file) +{ + struct op_inode_info *oi = OP_I(inode); + int ret; + + BUG_ON(oi->type != op_inode_prop); + + ret = seq_open(file, &property_op); + if (!ret) { + struct seq_file *m = file->private_data; + m->private = oi->u.prop; + } + return ret; +} + +static const struct file_operations openpromfs_prop_ops = { + .open = property_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; + +static int openpromfs_readdir(struct file *, struct dir_context *); + +static const struct file_operations openprom_operations = { + .read = generic_read_dir, + .iterate_shared = openpromfs_readdir, + .llseek = generic_file_llseek, +}; + +static struct dentry *openpromfs_lookup(struct inode *, struct dentry *, unsigned int); + +static const struct inode_operations openprom_inode_operations = { + .lookup = openpromfs_lookup, +}; + +static struct dentry *openpromfs_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) +{ + struct op_inode_info *ent_oi, *oi = OP_I(dir); + struct device_node *dp, *child; + struct property *prop; + enum op_inode_type ent_type; + union op_inode_data ent_data; + const char *name; + struct inode *inode; + unsigned int ino; + int len; + + BUG_ON(oi->type != op_inode_node); + + dp = oi->u.node; + + name = dentry->d_name.name; + len = dentry->d_name.len; + + mutex_lock(&op_mutex); + + child = dp->child; + while (child) { + const char *node_name = kbasename(child->full_name); + int n = strlen(node_name); + + if (len == n && + !strncmp(node_name, name, len)) { + ent_type = op_inode_node; + ent_data.node = child; + ino = child->unique_id; + goto found; + } + child = child->sibling; + } + + prop = dp->properties; + while (prop) { + int n = strlen(prop->name); + + if (len == n && !strncmp(prop->name, name, len)) { + ent_type = op_inode_prop; + ent_data.prop = prop; + ino = prop->unique_id; + goto found; + } + + prop = prop->next; + } + + mutex_unlock(&op_mutex); + return ERR_PTR(-ENOENT); + +found: + inode = openprom_iget(dir->i_sb, ino); + mutex_unlock(&op_mutex); + if (IS_ERR(inode)) + return ERR_CAST(inode); + if (inode_state_read_once(inode) & I_NEW) { + simple_inode_init_ts(inode); + ent_oi = OP_I(inode); + ent_oi->type = ent_type; + ent_oi->u = ent_data; + + switch (ent_type) { + case op_inode_node: + inode->i_mode = S_IFDIR | S_IRUGO | S_IXUGO; + inode->i_op = &openprom_inode_operations; + inode->i_fop = &openprom_operations; + set_nlink(inode, 2); + break; + case op_inode_prop: + if (of_node_name_eq(dp, "options") && (len == 17) && + !strncmp (name, "security-password", 17)) + inode->i_mode = S_IFREG | S_IRUSR | S_IWUSR; + else + inode->i_mode = S_IFREG | S_IRUGO; + inode->i_fop = &openpromfs_prop_ops; + set_nlink(inode, 1); + inode->i_size = ent_oi->u.prop->length; + break; + } + unlock_new_inode(inode); + } + + return d_splice_alias(inode, dentry); +} + +static int openpromfs_readdir(struct file *file, struct dir_context *ctx) +{ + struct inode *inode = file_inode(file); + struct op_inode_info *oi = OP_I(inode); + struct device_node *dp = oi->u.node; + struct device_node *child; + struct property *prop; + int i; + + mutex_lock(&op_mutex); + + if (ctx->pos == 0) { + if (!dir_emit(ctx, ".", 1, inode->i_ino, DT_DIR)) + goto out; + ctx->pos = 1; + } + if (ctx->pos == 1) { + if (!dir_emit(ctx, "..", 2, + (dp->parent == NULL ? + OPENPROM_ROOT_INO : + dp->parent->unique_id), DT_DIR)) + goto out; + ctx->pos = 2; + } + i = ctx->pos - 2; + + /* First, the children nodes as directories. */ + child = dp->child; + while (i && child) { + child = child->sibling; + i--; + } + while (child) { + if (!dir_emit(ctx, + kbasename(child->full_name), + strlen(kbasename(child->full_name)), + child->unique_id, DT_DIR)) + goto out; + + ctx->pos++; + child = child->sibling; + } + + /* Next, the properties as files. */ + prop = dp->properties; + while (i && prop) { + prop = prop->next; + i--; + } + while (prop) { + if (!dir_emit(ctx, prop->name, strlen(prop->name), + prop->unique_id, DT_REG)) + goto out; + + ctx->pos++; + prop = prop->next; + } + +out: + mutex_unlock(&op_mutex); + return 0; +} + +static struct kmem_cache *op_inode_cachep; + +static struct inode *openprom_alloc_inode(struct super_block *sb) +{ + struct op_inode_info *oi; + + oi = alloc_inode_sb(sb, op_inode_cachep, GFP_KERNEL); + if (!oi) + return NULL; + + return &oi->vfs_inode; +} + +static void openprom_free_inode(struct inode *inode) +{ + kmem_cache_free(op_inode_cachep, OP_I(inode)); +} + +static struct inode *openprom_iget(struct super_block *sb, ino_t ino) +{ + struct inode *inode = iget_locked(sb, ino); + if (!inode) + inode = ERR_PTR(-ENOMEM); + return inode; +} + +static int openpromfs_reconfigure(struct fs_context *fc) +{ + sync_filesystem(fc->root->d_sb); + fc->sb_flags |= SB_NOATIME; + return 0; +} + +static const struct super_operations openprom_sops = { + .alloc_inode = openprom_alloc_inode, + .free_inode = openprom_free_inode, + .statfs = simple_statfs, +}; + +static int openprom_fill_super(struct super_block *s, struct fs_context *fc) +{ + struct inode *root_inode; + struct op_inode_info *oi; + int ret; + + s->s_flags |= SB_NOATIME; + s->s_blocksize = 1024; + s->s_blocksize_bits = 10; + s->s_magic = OPENPROM_SUPER_MAGIC; + s->s_op = &openprom_sops; + s->s_time_gran = 1; + root_inode = openprom_iget(s, OPENPROM_ROOT_INO); + if (IS_ERR(root_inode)) { + ret = PTR_ERR(root_inode); + goto out_no_root; + } + + simple_inode_init_ts(root_inode); + root_inode->i_op = &openprom_inode_operations; + root_inode->i_fop = &openprom_operations; + root_inode->i_mode = S_IFDIR | S_IRUGO | S_IXUGO; + oi = OP_I(root_inode); + oi->type = op_inode_node; + oi->u.node = of_find_node_by_path("/"); + unlock_new_inode(root_inode); + + s->s_root = d_make_root(root_inode); + if (!s->s_root) + goto out_no_root_dentry; + return 0; + +out_no_root_dentry: + ret = -ENOMEM; +out_no_root: + printk("openprom_fill_super: get root inode failed\n"); + return ret; +} + +static int openpromfs_get_tree(struct fs_context *fc) +{ + return get_tree_single(fc, openprom_fill_super); +} + +static const struct fs_context_operations openpromfs_context_ops = { + .get_tree = openpromfs_get_tree, + .reconfigure = openpromfs_reconfigure, +}; + +static int openpromfs_init_fs_context(struct fs_context *fc) +{ + fc->ops = &openpromfs_context_ops; + return 0; +} + +static struct file_system_type openprom_fs_type = { + .owner = THIS_MODULE, + .name = "openpromfs", + .init_fs_context = openpromfs_init_fs_context, + .kill_sb = kill_anon_super, +}; +MODULE_ALIAS_FS("openpromfs"); + +static void op_inode_init_once(void *data) +{ + struct op_inode_info *oi = (struct op_inode_info *) data; + + inode_init_once(&oi->vfs_inode); +} + +static int __init init_openprom_fs(void) +{ + int err; + + op_inode_cachep = kmem_cache_create("op_inode_cache", + sizeof(struct op_inode_info), + 0, + (SLAB_RECLAIM_ACCOUNT | + SLAB_ACCOUNT), + op_inode_init_once); + if (!op_inode_cachep) + return -ENOMEM; + + err = register_filesystem(&openprom_fs_type); + if (err) + kmem_cache_destroy(op_inode_cachep); + + return err; +} + +static void __exit exit_openprom_fs(void) +{ + unregister_filesystem(&openprom_fs_type); + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(op_inode_cachep); +} + +module_init(init_openprom_fs) +module_exit(exit_openprom_fs) +MODULE_DESCRIPTION("OpenPROM filesystem support"); +MODULE_LICENSE("GPL"); diff --git a/fs/orangefs/Kconfig b/fs/orangefs/Kconfig new file mode 100644 index 000000000..890c0ae9a --- /dev/null +++ b/fs/orangefs/Kconfig @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0-only +config ORANGEFS_FS + tristate "ORANGEFS (Powered by PVFS) support" + select FS_POSIX_ACL + help + Orange is a parallel file system designed for use on high end + computing (HEC) systems. diff --git a/fs/orangefs/Makefile b/fs/orangefs/Makefile new file mode 100644 index 000000000..9b6c50bb1 --- /dev/null +++ b/fs/orangefs/Makefile @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the ORANGEFS filesystem. +# + +obj-$(CONFIG_ORANGEFS_FS) += orangefs.o + +orangefs-objs := acl.o file.o orangefs-cache.o orangefs-utils.o xattr.o \ + dcache.o inode.o orangefs-sysfs.o orangefs-mod.o super.o \ + devorangefs-req.o namei.o symlink.o dir.o orangefs-bufmap.o \ + orangefs-debugfs.o waitqueue.o diff --git a/fs/orangefs/acl.c b/fs/orangefs/acl.c new file mode 100644 index 000000000..a01ef0c1b --- /dev/null +++ b/fs/orangefs/acl.c @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * (C) 2001 Clemson University and The University of Chicago + * + * See COPYING in top-level directory. + */ + +#include "protocol.h" +#include "orangefs-kernel.h" +#include "orangefs-bufmap.h" +#include + +struct posix_acl *orangefs_get_acl(struct inode *inode, int type, bool rcu) +{ + struct posix_acl *acl; + int ret; + char *key = NULL, *value = NULL; + + if (rcu) + return ERR_PTR(-ECHILD); + + switch (type) { + case ACL_TYPE_ACCESS: + key = XATTR_NAME_POSIX_ACL_ACCESS; + break; + case ACL_TYPE_DEFAULT: + key = XATTR_NAME_POSIX_ACL_DEFAULT; + break; + default: + gossip_err("orangefs_get_acl: bogus value of type %d\n", type); + return ERR_PTR(-EINVAL); + } + /* + * Rather than incurring a network call just to determine the exact + * length of the attribute, I just allocate a max length to save on + * the network call. Conceivably, we could pass NULL to + * orangefs_inode_getxattr() to probe the length of the value, but + * I don't do that for now. + */ + value = kmalloc(ORANGEFS_MAX_XATTR_VALUELEN, GFP_KERNEL); + if (!value) + return ERR_PTR(-ENOMEM); + + gossip_debug(GOSSIP_ACL_DEBUG, + "inode %pU, key %s, type %d\n", + get_khandle_from_ino(inode), + key, + type); + ret = orangefs_inode_getxattr(inode, key, value, + ORANGEFS_MAX_XATTR_VALUELEN); + /* if the key exists, convert it to an in-memory rep */ + if (ret > 0) { + acl = posix_acl_from_xattr(&init_user_ns, value, ret); + } else if (ret == -ENODATA || ret == -ENOSYS) { + acl = NULL; + } else { + gossip_err("inode %pU retrieving acl's failed with error %d\n", + get_khandle_from_ino(inode), + ret); + acl = ERR_PTR(ret); + } + /* kfree(NULL) is safe, so don't worry if value ever got used */ + kfree(value); + return acl; +} + +int __orangefs_set_acl(struct inode *inode, struct posix_acl *acl, int type) +{ + int error = 0; + void *value = NULL; + size_t size = 0; + const char *name = NULL; + + switch (type) { + case ACL_TYPE_ACCESS: + name = XATTR_NAME_POSIX_ACL_ACCESS; + break; + case ACL_TYPE_DEFAULT: + name = XATTR_NAME_POSIX_ACL_DEFAULT; + break; + default: + gossip_err("%s: invalid type %d!\n", __func__, type); + return -EINVAL; + } + + gossip_debug(GOSSIP_ACL_DEBUG, + "%s: inode %pU, key %s type %d\n", + __func__, get_khandle_from_ino(inode), + name, + type); + + if (acl) { + value = posix_acl_to_xattr(&init_user_ns, acl, &size, GFP_KERNEL); + if (!value) + return -ENOMEM; + } + + gossip_debug(GOSSIP_ACL_DEBUG, + "%s: name %s, value %p, size %zd, acl %p\n", + __func__, name, value, size, acl); + /* + * Go ahead and set the extended attribute now. NOTE: Suppose acl + * was NULL, then value will be NULL and size will be 0 and that + * will xlate to a removexattr. However, we don't want removexattr + * complain if attributes does not exist. + */ + error = orangefs_inode_setxattr(inode, name, value, size, 0); + + kfree(value); + if (!error) + set_cached_acl(inode, type, acl); + return error; +} + +int orangefs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type) +{ + int error; + struct iattr iattr; + int rc; + struct inode *inode = d_inode(dentry); + + memset(&iattr, 0, sizeof iattr); + + if (type == ACL_TYPE_ACCESS && acl) { + /* + * posix_acl_update_mode checks to see if the permissions + * described by the ACL can be encoded into the + * object's mode. If so, it sets "acl" to NULL + * and "mode" to the new desired value. It is up to + * us to propagate the new mode back to the server... + */ + error = posix_acl_update_mode(&nop_mnt_idmap, inode, + &iattr.ia_mode, &acl); + if (error) { + gossip_err("%s: posix_acl_update_mode err: %d\n", + __func__, + error); + return error; + } + + if (inode->i_mode != iattr.ia_mode) + iattr.ia_valid = ATTR_MODE; + + } + + rc = __orangefs_set_acl(inode, acl, type); + + if (!rc && (iattr.ia_valid == ATTR_MODE)) + rc = __orangefs_setattr_mode(dentry, &iattr); + + return rc; +} diff --git a/fs/orangefs/dcache.c b/fs/orangefs/dcache.c new file mode 100644 index 000000000..a19d1ad70 --- /dev/null +++ b/fs/orangefs/dcache.c @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * (C) 2001 Clemson University and The University of Chicago + * + * See COPYING in top-level directory. + */ + +/* + * Implementation of dentry (directory cache) functions. + */ + +#include "protocol.h" +#include "orangefs-kernel.h" + +/* Returns 1 if dentry can still be trusted, else 0. */ +static int orangefs_revalidate_lookup(struct inode *parent_inode, const struct qstr *name, + struct dentry *dentry) +{ + struct orangefs_inode_s *parent = ORANGEFS_I(parent_inode); + struct inode *inode = dentry->d_inode; + struct orangefs_kernel_op_s *new_op; + int ret = 0; + int err = 0; + + gossip_debug(GOSSIP_DCACHE_DEBUG, "%s: attempting lookup.\n", __func__); + + new_op = op_alloc(ORANGEFS_VFS_OP_LOOKUP); + if (!new_op) + return -ENOMEM; + + new_op->upcall.req.lookup.sym_follow = ORANGEFS_LOOKUP_LINK_NO_FOLLOW; + new_op->upcall.req.lookup.parent_refn = parent->refn; + /* op_alloc() leaves ->upcall zeroed */ + memcpy(new_op->upcall.req.lookup.d_name, name->name, + min(name->len, ORANGEFS_NAME_MAX - 1)); + + gossip_debug(GOSSIP_DCACHE_DEBUG, + "%s:%s:%d interrupt flag [%d]\n", + __FILE__, + __func__, + __LINE__, + get_interruptible_flag(parent_inode)); + + err = service_operation(new_op, "orangefs_lookup", + get_interruptible_flag(parent_inode)); + + /* Positive dentry: reject if error or not the same inode. */ + if (inode) { + if (err) { + gossip_debug(GOSSIP_DCACHE_DEBUG, + "%s:%s:%d lookup failure.\n", + __FILE__, __func__, __LINE__); + goto out_drop; + } + if (!match_handle(new_op->downcall.resp.lookup.refn.khandle, + inode)) { + gossip_debug(GOSSIP_DCACHE_DEBUG, + "%s:%s:%d no match.\n", + __FILE__, __func__, __LINE__); + goto out_drop; + } + + /* Negative dentry: reject if success or error other than ENOENT. */ + } else { + gossip_debug(GOSSIP_DCACHE_DEBUG, "%s: negative dentry.\n", + __func__); + if (!err || err != -ENOENT) { + if (new_op->downcall.status != 0) + gossip_debug(GOSSIP_DCACHE_DEBUG, + "%s:%s:%d lookup failure.\n", + __FILE__, __func__, __LINE__); + goto out_drop; + } + } + + orangefs_set_timeout(dentry); + ret = 1; +out_release_op: + op_release(new_op); + return ret; +out_drop: + gossip_debug(GOSSIP_DCACHE_DEBUG, "%s:%s:%d revalidate failed\n", + __FILE__, __func__, __LINE__); + goto out_release_op; +} + +/* + * Verify that dentry is valid. + * + * Should return 1 if dentry can still be trusted, else 0. + */ +static int orangefs_d_revalidate(struct inode *dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + int ret; + unsigned long time = (unsigned long) dentry->d_fsdata; + + if (time_before(jiffies, time)) + return 1; + + if (flags & LOOKUP_RCU) + return -ECHILD; + + gossip_debug(GOSSIP_DCACHE_DEBUG, "%s: called on dentry %p.\n", + __func__, dentry); + + /* skip root handle lookups. */ + if (dentry->d_inode && is_root_handle(dentry->d_inode)) + return 1; + + /* + * If this passes, the positive dentry still exists or the negative + * dentry still does not exist. + */ + if (!orangefs_revalidate_lookup(dir, name, dentry)) + return 0; + + /* We do not need to continue with negative dentries. */ + if (!dentry->d_inode) { + gossip_debug(GOSSIP_DCACHE_DEBUG, + "%s: negative dentry or positive dentry and inode valid.\n", + __func__); + return 1; + } + + /* Now we must perform a getattr to validate the inode contents. */ + + ret = orangefs_inode_check_changed(dentry->d_inode); + if (ret < 0) { + gossip_debug(GOSSIP_DCACHE_DEBUG, "%s:%s:%d getattr failure.\n", + __FILE__, __func__, __LINE__); + return 0; + } + return !ret; +} + +const struct dentry_operations orangefs_dentry_operations = { + .d_revalidate = orangefs_d_revalidate, +}; diff --git a/fs/orangefs/devorangefs-req.c b/fs/orangefs/devorangefs-req.c new file mode 100644 index 000000000..e5c60da7e --- /dev/null +++ b/fs/orangefs/devorangefs-req.c @@ -0,0 +1,815 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * (C) 2001 Clemson University and The University of Chicago + * + * Changes by Acxiom Corporation to add protocol version to kernel + * communication, Copyright Acxiom Corporation, 2005. + * + * See COPYING in top-level directory. + */ + +#include "protocol.h" +#include "orangefs-kernel.h" +#include "orangefs-dev-proto.h" +#include "orangefs-bufmap.h" +#include "orangefs-debugfs.h" + +#include +#include + +/* this file implements the /dev/pvfs2-req device node */ + +uint32_t orangefs_userspace_version; + +static int open_access_count; + +static DEFINE_MUTEX(devreq_mutex); + +#define DUMP_DEVICE_ERROR() \ +do { \ + gossip_err("*****************************************************\n");\ + gossip_err("ORANGEFS Device Error: You cannot open the device file "); \ + gossip_err("\n/dev/%s more than once. Please make sure that\nthere " \ + "are no ", ORANGEFS_REQDEVICE_NAME); \ + gossip_err("instances of a program using this device\ncurrently " \ + "running. (You must verify this!)\n"); \ + gossip_err("For example, you can use the lsof program as follows:\n");\ + gossip_err("'lsof | grep %s' (run this as root)\n", \ + ORANGEFS_REQDEVICE_NAME); \ + gossip_err(" open_access_count = %d\n", open_access_count); \ + gossip_err("*****************************************************\n");\ +} while (0) + +static int hash_func(__u64 tag, int table_size) +{ + return do_div(tag, (unsigned int)table_size); +} + +static void orangefs_devreq_add_op(struct orangefs_kernel_op_s *op) +{ + int index = hash_func(op->tag, hash_table_size); + + list_add_tail(&op->list, &orangefs_htable_ops_in_progress[index]); +} + +/* + * find the op with this tag and remove it from the in progress + * hash table. + */ +static struct orangefs_kernel_op_s *orangefs_devreq_remove_op(__u64 tag) +{ + struct orangefs_kernel_op_s *op, *next; + int index; + + index = hash_func(tag, hash_table_size); + + spin_lock(&orangefs_htable_ops_in_progress_lock); + list_for_each_entry_safe(op, + next, + &orangefs_htable_ops_in_progress[index], + list) { + if (op->tag == tag && !op_state_purged(op) && + !op_state_given_up(op)) { + list_del_init(&op->list); + spin_unlock(&orangefs_htable_ops_in_progress_lock); + return op; + } + } + + spin_unlock(&orangefs_htable_ops_in_progress_lock); + return NULL; +} + +/* Returns whether any FS are still pending remounted */ +static int mark_all_pending_mounts(void) +{ + int unmounted = 1; + struct orangefs_sb_info_s *orangefs_sb = NULL; + + spin_lock(&orangefs_superblocks_lock); + list_for_each_entry(orangefs_sb, &orangefs_superblocks, list) { + /* All of these file system require a remount */ + orangefs_sb->mount_pending = 1; + unmounted = 0; + } + spin_unlock(&orangefs_superblocks_lock); + return unmounted; +} + +/* + * Determine if a given file system needs to be remounted or not + * Returns -1 on error + * 0 if already mounted + * 1 if needs remount + */ +static int fs_mount_pending(__s32 fsid) +{ + int mount_pending = -1; + struct orangefs_sb_info_s *orangefs_sb = NULL; + + spin_lock(&orangefs_superblocks_lock); + list_for_each_entry(orangefs_sb, &orangefs_superblocks, list) { + if (orangefs_sb->fs_id == fsid) { + mount_pending = orangefs_sb->mount_pending; + break; + } + } + spin_unlock(&orangefs_superblocks_lock); + return mount_pending; +} + +static int orangefs_devreq_open(struct inode *inode, struct file *file) +{ + int ret = -EINVAL; + + /* in order to ensure that the filesystem driver sees correct UIDs */ + if (file->f_cred->user_ns != &init_user_ns) { + gossip_err("%s: device cannot be opened outside init_user_ns\n", + __func__); + goto out; + } + + if (!(file->f_flags & O_NONBLOCK)) { + gossip_err("%s: device cannot be opened in blocking mode\n", + __func__); + goto out; + } + ret = -EACCES; + gossip_debug(GOSSIP_DEV_DEBUG, "client-core: opening device\n"); + mutex_lock(&devreq_mutex); + + if (open_access_count == 0) { + open_access_count = 1; + ret = 0; + } else { + DUMP_DEVICE_ERROR(); + } + mutex_unlock(&devreq_mutex); + +out: + + gossip_debug(GOSSIP_DEV_DEBUG, + "pvfs2-client-core: open device complete (ret = %d)\n", + ret); + return ret; +} + +/* Function for read() callers into the device */ +static ssize_t orangefs_devreq_read(struct file *file, + char __user *buf, + size_t count, loff_t *offset) +{ + struct orangefs_kernel_op_s *op, *temp; + __s32 proto_ver = ORANGEFS_KERNEL_PROTO_VERSION; + static __s32 magic = ORANGEFS_DEVREQ_MAGIC; + struct orangefs_kernel_op_s *cur_op; + unsigned long ret; + + /* We do not support blocking IO. */ + if (!(file->f_flags & O_NONBLOCK)) { + gossip_err("%s: blocking read from client-core.\n", + __func__); + return -EINVAL; + } + + /* + * The client will do an ioctl to find MAX_DEV_REQ_UPSIZE, then + * always read with that size buffer. + */ + if (count != MAX_DEV_REQ_UPSIZE) { + gossip_err("orangefs: client-core tried to read wrong size\n"); + return -EINVAL; + } + + /* Check for an empty list before locking. */ + if (list_empty(&orangefs_request_list)) + return -EAGAIN; + +restart: + cur_op = NULL; + /* Get next op (if any) from top of list. */ + spin_lock(&orangefs_request_list_lock); + list_for_each_entry_safe(op, temp, &orangefs_request_list, list) { + __s32 fsid; + /* This lock is held past the end of the loop when we break. */ + spin_lock(&op->lock); + if (unlikely(op_state_purged(op) || op_state_given_up(op))) { + spin_unlock(&op->lock); + continue; + } + + fsid = fsid_of_op(op); + if (fsid != ORANGEFS_FS_ID_NULL) { + int ret; + /* Skip ops whose filesystem needs to be mounted. */ + ret = fs_mount_pending(fsid); + if (ret == 1) { + gossip_debug(GOSSIP_DEV_DEBUG, + "%s: mount pending, skipping op tag " + "%llu %s\n", + __func__, + llu(op->tag), + get_opname_string(op)); + spin_unlock(&op->lock); + continue; + /* + * Skip ops whose filesystem we don't know about unless + * it is being mounted or unmounted. It is possible for + * a filesystem we don't know about to be unmounted if + * it fails to mount in the kernel after userspace has + * been sent the mount request. + */ + /* XXX: is there a better way to detect this? */ + } else if (ret == -1 && + !(op->upcall.type == + ORANGEFS_VFS_OP_FS_MOUNT || + op->upcall.type == + ORANGEFS_VFS_OP_GETATTR || + op->upcall.type == + ORANGEFS_VFS_OP_FS_UMOUNT)) { + gossip_debug(GOSSIP_DEV_DEBUG, + "orangefs: skipping op tag %llu %s\n", + llu(op->tag), get_opname_string(op)); + gossip_err( + "orangefs: ERROR: fs_mount_pending %d\n", + fsid); + spin_unlock(&op->lock); + continue; + } + } + /* + * Either this op does not pertain to a filesystem, is mounting + * a filesystem, or pertains to a mounted filesystem. Let it + * through. + */ + cur_op = op; + break; + } + + /* + * At this point we either have a valid op and can continue or have not + * found an op and must ask the client to try again later. + */ + if (!cur_op) { + spin_unlock(&orangefs_request_list_lock); + return -EAGAIN; + } + + gossip_debug(GOSSIP_DEV_DEBUG, "%s: reading op tag %llu %s\n", + __func__, + llu(cur_op->tag), + get_opname_string(cur_op)); + + /* + * Such an op should never be on the list in the first place. If so, we + * will abort. + */ + if (op_state_in_progress(cur_op) || op_state_serviced(cur_op)) { + gossip_err("orangefs: ERROR: Current op already queued.\n"); + list_del_init(&cur_op->list); + spin_unlock(&cur_op->lock); + spin_unlock(&orangefs_request_list_lock); + return -EAGAIN; + } + + list_del_init(&cur_op->list); + spin_unlock(&orangefs_request_list_lock); + + spin_unlock(&cur_op->lock); + + /* Push the upcall out. */ + ret = copy_to_user(buf, &proto_ver, sizeof(__s32)); + if (ret != 0) + goto error; + ret = copy_to_user(buf + sizeof(__s32), &magic, sizeof(__s32)); + if (ret != 0) + goto error; + ret = copy_to_user(buf + 2 * sizeof(__s32), + &cur_op->tag, + sizeof(__u64)); + if (ret != 0) + goto error; + ret = copy_to_user(buf + 2 * sizeof(__s32) + sizeof(__u64), + &cur_op->upcall, + sizeof(struct orangefs_upcall_s)); + if (ret != 0) + goto error; + + spin_lock(&orangefs_htable_ops_in_progress_lock); + spin_lock(&cur_op->lock); + if (unlikely(op_state_given_up(cur_op))) { + spin_unlock(&cur_op->lock); + spin_unlock(&orangefs_htable_ops_in_progress_lock); + complete(&cur_op->waitq); + goto restart; + } + + /* + * Set the operation to be in progress and move it between lists since + * it has been sent to the client. + */ + set_op_state_inprogress(cur_op); + gossip_debug(GOSSIP_DEV_DEBUG, + "%s: 1 op:%s: op_state:%d: process:%s:\n", + __func__, + get_opname_string(cur_op), + cur_op->op_state, + current->comm); + orangefs_devreq_add_op(cur_op); + spin_unlock(&cur_op->lock); + spin_unlock(&orangefs_htable_ops_in_progress_lock); + + /* The client only asks to read one size buffer. */ + return MAX_DEV_REQ_UPSIZE; +error: + /* + * We were unable to copy the op data to the client. Put the op back in + * list. If client has crashed, the op will be purged later when the + * device is released. + */ + gossip_err("orangefs: Failed to copy data to user space\n"); + spin_lock(&orangefs_request_list_lock); + spin_lock(&cur_op->lock); + if (likely(!op_state_given_up(cur_op))) { + set_op_state_waiting(cur_op); + gossip_debug(GOSSIP_DEV_DEBUG, + "%s: 2 op:%s: op_state:%d: process:%s:\n", + __func__, + get_opname_string(cur_op), + cur_op->op_state, + current->comm); + list_add(&cur_op->list, &orangefs_request_list); + spin_unlock(&cur_op->lock); + } else { + spin_unlock(&cur_op->lock); + complete(&cur_op->waitq); + } + spin_unlock(&orangefs_request_list_lock); + return -EFAULT; +} + +/* + * Function for writev() callers into the device. + * + * Userspace should have written: + * - __u32 version + * - __u32 magic + * - __u64 tag + * - struct orangefs_downcall_s + * - trailer buffer (in the case of READDIR operations) + */ +static ssize_t orangefs_devreq_write_iter(struct kiocb *iocb, + struct iov_iter *iter) +{ + ssize_t ret; + struct orangefs_kernel_op_s *op = NULL; + struct { + __u32 version; + __u32 magic; + __u64 tag; + } head; + int total = ret = iov_iter_count(iter); + int downcall_size = sizeof(struct orangefs_downcall_s); + int head_size = sizeof(head); + + gossip_debug(GOSSIP_DEV_DEBUG, "%s: total:%d: ret:%zd:\n", + __func__, + total, + ret); + + if (total < MAX_DEV_REQ_DOWNSIZE) { + gossip_err("%s: total:%d: must be at least:%u:\n", + __func__, + total, + (unsigned int) MAX_DEV_REQ_DOWNSIZE); + return -EFAULT; + } + + if (!copy_from_iter_full(&head, head_size, iter)) { + gossip_err("%s: failed to copy head.\n", __func__); + return -EFAULT; + } + + if (head.version < ORANGEFS_MINIMUM_USERSPACE_VERSION) { + gossip_err("%s: userspace claims version" + "%d, minimum version required: %d.\n", + __func__, + head.version, + ORANGEFS_MINIMUM_USERSPACE_VERSION); + return -EPROTO; + } + + if (head.magic != ORANGEFS_DEVREQ_MAGIC) { + gossip_err("Error: Device magic number does not match.\n"); + return -EPROTO; + } + + if (!orangefs_userspace_version) { + orangefs_userspace_version = head.version; + } else if (orangefs_userspace_version != head.version) { + gossip_err("Error: userspace version changes\n"); + return -EPROTO; + } + + /* remove the op from the in progress hash table */ + op = orangefs_devreq_remove_op(head.tag); + if (!op) { + gossip_debug(GOSSIP_DEV_DEBUG, + "%s: No one's waiting for tag %llu\n", + __func__, llu(head.tag)); + return ret; + } + + if (!copy_from_iter_full(&op->downcall, downcall_size, iter)) { + gossip_err("%s: failed to copy downcall.\n", __func__); + goto Efault; + } + + if (op->downcall.status) + goto wakeup; + + /* + * We've successfully peeled off the head and the downcall. + * Something has gone awry if total doesn't equal the + * sum of head_size, downcall_size and trailer_size. + */ + if ((head_size + downcall_size + op->downcall.trailer_size) != total) { + gossip_err("%s: funky write, head_size:%d" + ": downcall_size:%d: trailer_size:%lld" + ": total size:%d:\n", + __func__, + head_size, + downcall_size, + op->downcall.trailer_size, + total); + goto Efault; + } + + /* Only READDIR operations should have trailers. */ + if ((op->downcall.type != ORANGEFS_VFS_OP_READDIR) && + (op->downcall.trailer_size != 0)) { + gossip_err("%s: %x operation with trailer.", + __func__, + op->downcall.type); + goto Efault; + } + + /* READDIR operations should always have trailers. */ + if ((op->downcall.type == ORANGEFS_VFS_OP_READDIR) && + (op->downcall.trailer_size == 0)) { + gossip_err("%s: %x operation with no trailer.", + __func__, + op->downcall.type); + goto Efault; + } + + if (op->downcall.type != ORANGEFS_VFS_OP_READDIR) + goto wakeup; + + op->downcall.trailer_buf = vzalloc(op->downcall.trailer_size); + if (!op->downcall.trailer_buf) + goto Enomem; + + if (!copy_from_iter_full(op->downcall.trailer_buf, + op->downcall.trailer_size, iter)) { + gossip_err("%s: failed to copy trailer.\n", __func__); + vfree(op->downcall.trailer_buf); + op->downcall.trailer_buf = NULL; + goto Efault; + } + +wakeup: + /* + * Return to vfs waitqueue, and back to service_operation + * through wait_for_matching_downcall. + */ + spin_lock(&op->lock); + if (unlikely(op_is_cancel(op))) { + spin_unlock(&op->lock); + put_cancel(op); + } else if (unlikely(op_state_given_up(op))) { + spin_unlock(&op->lock); + complete(&op->waitq); + } else { + set_op_state_serviced(op); + gossip_debug(GOSSIP_DEV_DEBUG, + "%s: op:%s: op_state:%d: process:%s:\n", + __func__, + get_opname_string(op), + op->op_state, + current->comm); + spin_unlock(&op->lock); + } + return ret; + +Efault: + op->downcall.status = -(ORANGEFS_ERROR_BIT | 9); + ret = -EFAULT; + goto wakeup; + +Enomem: + op->downcall.status = -(ORANGEFS_ERROR_BIT | 8); + ret = -ENOMEM; + goto wakeup; +} + +/* + * NOTE: gets called when the last reference to this device is dropped. + * Using the open_access_count variable, we enforce a reference count + * on this file so that it can be opened by only one process at a time. + * the devreq_mutex is used to make sure all i/o has completed + * before we call orangefs_bufmap_finalize, and similar such tricky + * situations + */ +static int orangefs_devreq_release(struct inode *inode, struct file *file) +{ + int unmounted = 0; + + gossip_debug(GOSSIP_DEV_DEBUG, + "%s:pvfs2-client-core: exiting, closing device\n", + __func__); + + mutex_lock(&devreq_mutex); + orangefs_bufmap_finalize(); + + open_access_count = -1; + + unmounted = mark_all_pending_mounts(); + gossip_debug(GOSSIP_DEV_DEBUG, "ORANGEFS Device Close: Filesystem(s) %s\n", + (unmounted ? "UNMOUNTED" : "MOUNTED")); + + purge_waiting_ops(); + purge_inprogress_ops(); + + orangefs_bufmap_run_down(); + + gossip_debug(GOSSIP_DEV_DEBUG, + "pvfs2-client-core: device close complete\n"); + open_access_count = 0; + orangefs_userspace_version = 0; + mutex_unlock(&devreq_mutex); + return 0; +} + +int is_daemon_in_service(void) +{ + int in_service; + + /* + * What this function does is checks if client-core is alive + * based on the access count we maintain on the device. + */ + mutex_lock(&devreq_mutex); + in_service = open_access_count == 1 ? 0 : -EIO; + mutex_unlock(&devreq_mutex); + return in_service; +} + +bool __is_daemon_in_service(void) +{ + return open_access_count == 1; +} + +static inline long check_ioctl_command(unsigned int command) +{ + /* Check for valid ioctl codes */ + if (_IOC_TYPE(command) != ORANGEFS_DEV_MAGIC) { + gossip_err("device ioctl magic numbers don't match! Did you rebuild pvfs2-client-core/libpvfs2? [cmd %x, magic %x != %x]\n", + command, + _IOC_TYPE(command), + ORANGEFS_DEV_MAGIC); + return -EINVAL; + } + /* and valid ioctl commands */ + if (_IOC_NR(command) >= ORANGEFS_DEV_MAXNR || _IOC_NR(command) <= 0) { + gossip_err("Invalid ioctl command number [%d >= %d]\n", + _IOC_NR(command), ORANGEFS_DEV_MAXNR); + return -ENOIOCTLCMD; + } + return 0; +} + +static long dispatch_ioctl_command(unsigned int command, unsigned long arg) +{ + static __s32 magic = ORANGEFS_DEVREQ_MAGIC; + static __s32 max_up_size = MAX_DEV_REQ_UPSIZE; + static __s32 max_down_size = MAX_DEV_REQ_DOWNSIZE; + struct ORANGEFS_dev_map_desc user_desc; + int ret = 0; + int upstream_kmod = 1; + struct orangefs_sb_info_s *orangefs_sb; + + /* mtmoore: add locking here */ + + switch (command) { + case ORANGEFS_DEV_GET_MAGIC: + return ((put_user(magic, (__s32 __user *) arg) == -EFAULT) ? + -EIO : + 0); + case ORANGEFS_DEV_GET_MAX_UPSIZE: + return ((put_user(max_up_size, + (__s32 __user *) arg) == -EFAULT) ? + -EIO : + 0); + case ORANGEFS_DEV_GET_MAX_DOWNSIZE: + return ((put_user(max_down_size, + (__s32 __user *) arg) == -EFAULT) ? + -EIO : + 0); + case ORANGEFS_DEV_MAP: + ret = copy_from_user(&user_desc, + (struct ORANGEFS_dev_map_desc __user *) + arg, + sizeof(struct ORANGEFS_dev_map_desc)); + /* WTF -EIO and not -EFAULT? */ + return ret ? -EIO : orangefs_bufmap_initialize(&user_desc); + case ORANGEFS_DEV_REMOUNT_ALL: + gossip_debug(GOSSIP_DEV_DEBUG, + "%s: got ORANGEFS_DEV_REMOUNT_ALL\n", + __func__); + + /* + * remount all mounted orangefs volumes to regain the lost + * dynamic mount tables (if any) -- NOTE: this is done + * without keeping the superblock list locked due to the + * upcall/downcall waiting. also, the request mutex is + * used to ensure that no operations will be serviced until + * all of the remounts are serviced (to avoid ops between + * mounts to fail) + */ + ret = mutex_lock_interruptible(&orangefs_request_mutex); + if (ret < 0) + return ret; + gossip_debug(GOSSIP_DEV_DEBUG, + "%s: priority remount in progress\n", + __func__); + spin_lock(&orangefs_superblocks_lock); + list_for_each_entry(orangefs_sb, &orangefs_superblocks, list) { + /* + * We have to drop the spinlock, so entries can be + * removed. They can't be freed, though, so we just + * keep the forward pointers and zero the back ones - + * that way we can get to the rest of the list. + */ + if (!orangefs_sb->list.prev) + continue; + gossip_debug(GOSSIP_DEV_DEBUG, + "%s: Remounting SB %p\n", + __func__, + orangefs_sb); + + spin_unlock(&orangefs_superblocks_lock); + ret = orangefs_remount(orangefs_sb); + spin_lock(&orangefs_superblocks_lock); + if (ret) { + gossip_debug(GOSSIP_DEV_DEBUG, + "SB %p remount failed\n", + orangefs_sb); + break; + } + } + spin_unlock(&orangefs_superblocks_lock); + gossip_debug(GOSSIP_DEV_DEBUG, + "%s: priority remount complete\n", + __func__); + mutex_unlock(&orangefs_request_mutex); + return ret; + + case ORANGEFS_DEV_UPSTREAM: + ret = copy_to_user((void __user *)arg, + &upstream_kmod, + sizeof(upstream_kmod)); + + if (ret != 0) + return -EIO; + else + return ret; + + case ORANGEFS_DEV_CLIENT_MASK: + return orangefs_debugfs_new_client_mask((void __user *)arg); + case ORANGEFS_DEV_CLIENT_STRING: + return orangefs_debugfs_new_client_string((void __user *)arg); + case ORANGEFS_DEV_DEBUG: + return orangefs_debugfs_new_debug((void __user *)arg); + default: + return -ENOIOCTLCMD; + } + return -ENOIOCTLCMD; +} + +static long orangefs_devreq_ioctl(struct file *file, + unsigned int command, unsigned long arg) +{ + long ret; + + /* Check for properly constructed commands */ + ret = check_ioctl_command(command); + if (ret < 0) + return (int)ret; + + return (int)dispatch_ioctl_command(command, arg); +} + +#ifdef CONFIG_COMPAT /* CONFIG_COMPAT is in .config */ + +/* Compat structure for the ORANGEFS_DEV_MAP ioctl */ +struct ORANGEFS_dev_map_desc32 { + compat_uptr_t ptr; + __s32 total_size; + __s32 size; + __s32 count; +}; + +/* + * 32 bit user-space apps' ioctl handlers when kernel modules + * is compiled as a 64 bit one + */ +static long orangefs_devreq_compat_ioctl(struct file *filp, unsigned int cmd, + unsigned long args) +{ + long ret; + + /* Check for properly constructed commands */ + ret = check_ioctl_command(cmd); + if (ret < 0) + return ret; + if (cmd == ORANGEFS_DEV_MAP) { + struct ORANGEFS_dev_map_desc desc; + struct ORANGEFS_dev_map_desc32 d32; + + if (copy_from_user(&d32, (void __user *)args, sizeof(d32))) + return -EFAULT; + + desc.ptr = compat_ptr(d32.ptr); + desc.total_size = d32.total_size; + desc.size = d32.size; + desc.count = d32.count; + return orangefs_bufmap_initialize(&desc); + } + /* no other ioctl requires translation */ + return dispatch_ioctl_command(cmd, args); +} + +#endif /* CONFIG_COMPAT is in .config */ + +static __poll_t orangefs_devreq_poll(struct file *file, + struct poll_table_struct *poll_table) +{ + __poll_t poll_revent_mask = 0; + + poll_wait(file, &orangefs_request_list_waitq, poll_table); + + if (!list_empty(&orangefs_request_list)) + poll_revent_mask |= EPOLLIN; + return poll_revent_mask; +} + +/* the assigned character device major number */ +static int orangefs_dev_major; + +static const struct file_operations orangefs_devreq_file_operations = { + .owner = THIS_MODULE, + .read = orangefs_devreq_read, + .write_iter = orangefs_devreq_write_iter, + .open = orangefs_devreq_open, + .release = orangefs_devreq_release, + .unlocked_ioctl = orangefs_devreq_ioctl, + +#ifdef CONFIG_COMPAT /* CONFIG_COMPAT is in .config */ + .compat_ioctl = orangefs_devreq_compat_ioctl, +#endif + .poll = orangefs_devreq_poll +}; + +/* + * Initialize orangefs device specific state: + * Must be called at module load time only + */ +int orangefs_dev_init(void) +{ + /* register orangefs-req device */ + orangefs_dev_major = register_chrdev(0, + ORANGEFS_REQDEVICE_NAME, + &orangefs_devreq_file_operations); + if (orangefs_dev_major < 0) { + gossip_debug(GOSSIP_DEV_DEBUG, + "Failed to register /dev/%s (error %d)\n", + ORANGEFS_REQDEVICE_NAME, orangefs_dev_major); + return orangefs_dev_major; + } + + gossip_debug(GOSSIP_DEV_DEBUG, + "*** /dev/%s character device registered ***\n", + ORANGEFS_REQDEVICE_NAME); + gossip_debug(GOSSIP_DEV_DEBUG, "'mknod /dev/%s c %d 0'.\n", + ORANGEFS_REQDEVICE_NAME, orangefs_dev_major); + return 0; +} + +void orangefs_dev_cleanup(void) +{ + unregister_chrdev(orangefs_dev_major, ORANGEFS_REQDEVICE_NAME); + gossip_debug(GOSSIP_DEV_DEBUG, + "*** /dev/%s character device unregistered ***\n", + ORANGEFS_REQDEVICE_NAME); +} diff --git a/fs/orangefs/dir.c b/fs/orangefs/dir.c new file mode 100644 index 000000000..115b2c2f5 --- /dev/null +++ b/fs/orangefs/dir.c @@ -0,0 +1,398 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2017 Omnibond Systems, L.L.C. + */ + +#include +#include "protocol.h" +#include "orangefs-kernel.h" +#include "orangefs-bufmap.h" + +struct orangefs_dir_part { + struct orangefs_dir_part *next; + size_t len; +}; + +struct orangefs_dir { + __u64 token; + struct orangefs_dir_part *part; + loff_t end; + int error; +}; + +#define PART_SHIFT (24) +#define PART_SIZE (1<<24) +#define PART_MASK (~(PART_SIZE - 1)) + +/* + * There can be up to 512 directory entries. Each entry is encoded as + * follows: + * 4 bytes: string size (n) + * n bytes: string + * 1 byte: trailing zero + * padding to 8 bytes + * 16 bytes: khandle + * padding to 8 bytes + * + * The trailer_buf starts with a struct orangefs_readdir_response_s + * which must be skipped to get to the directory data. + * + * The data which is received from the userspace daemon is termed a + * part and is stored in a linked list in case more than one part is + * needed for a large directory. + * + * The position pointer (ctx->pos) encodes the part and offset on which + * to begin reading at. Bits above PART_SHIFT encode the part and bits + * below PART_SHIFT encode the offset. Parts are stored in a linked + * list which grows as data is received from the server. The overhead + * associated with managing the list is presumed to be small compared to + * the overhead of communicating with the server. + * + * As data is received from the server, it is placed at the end of the + * part list. Data is parsed from the current position as it is needed. + * When data is determined to be corrupt, it is either because the + * userspace component has sent back corrupt data or because the file + * pointer has been moved to an invalid location. Since the two cannot + * be differentiated, return EIO. + * + * Part zero is synthesized to contains `.' and `..'. Part one is the + * first part of the part list. + */ + +static int do_readdir(struct orangefs_dir *od, struct inode *inode, + struct orangefs_kernel_op_s *op) +{ + struct orangefs_inode_s *oi = ORANGEFS_I(inode); + struct orangefs_readdir_response_s *resp; + int bufi, r; + + /* + * Despite the badly named field, readdir does not use shared + * memory. However, there are a limited number of readdir + * slots, which must be allocated here. This flag simply tells + * the op scheduler to return the op here for retry. + */ + op->uses_shared_memory = 1; + op->upcall.req.readdir.refn = oi->refn; + op->upcall.req.readdir.token = od->token; + op->upcall.req.readdir.max_dirent_count = + ORANGEFS_MAX_DIRENT_COUNT_READDIR; + +again: + bufi = orangefs_readdir_index_get(); + if (bufi < 0) { + od->error = bufi; + return bufi; + } + + op->upcall.req.readdir.buf_index = bufi; + + r = service_operation(op, "orangefs_readdir", + get_interruptible_flag(inode)); + + orangefs_readdir_index_put(bufi); + + if (op_state_purged(op)) { + if (r == -EAGAIN) { + vfree(op->downcall.trailer_buf); + goto again; + } else if (r == -EIO) { + vfree(op->downcall.trailer_buf); + od->error = r; + return r; + } + } + + if (r < 0) { + vfree(op->downcall.trailer_buf); + od->error = r; + return r; + } else if (op->downcall.status) { + vfree(op->downcall.trailer_buf); + od->error = op->downcall.status; + return op->downcall.status; + } + + /* + * The maximum size is size per entry times the 512 entries plus + * the header. This is well under the limit. + */ + if (op->downcall.trailer_size > PART_SIZE) { + vfree(op->downcall.trailer_buf); + od->error = -EIO; + return -EIO; + } + + resp = (struct orangefs_readdir_response_s *) + op->downcall.trailer_buf; + od->token = resp->token; + return 0; +} + +static int parse_readdir(struct orangefs_dir *od, + struct orangefs_kernel_op_s *op) +{ + struct orangefs_dir_part *part, *new; + size_t count; + + count = 1; + part = od->part; + while (part) { + count++; + if (part->next) + part = part->next; + else + break; + } + + new = (void *)op->downcall.trailer_buf; + new->next = NULL; + new->len = op->downcall.trailer_size - + sizeof(struct orangefs_readdir_response_s); + if (!od->part) + od->part = new; + else + part->next = new; + count++; + od->end = count << PART_SHIFT; + + return 0; +} + +static int orangefs_dir_more(struct orangefs_dir *od, struct inode *inode) +{ + struct orangefs_kernel_op_s *op; + int r; + + op = op_alloc(ORANGEFS_VFS_OP_READDIR); + if (!op) { + od->error = -ENOMEM; + return -ENOMEM; + } + r = do_readdir(od, inode, op); + if (r) { + od->error = r; + goto out; + } + r = parse_readdir(od, op); + if (r) { + od->error = r; + goto out; + } + + od->error = 0; +out: + op_release(op); + return od->error; +} + +static int fill_from_part(struct orangefs_dir_part *part, + struct dir_context *ctx) +{ + const int offset = sizeof(struct orangefs_readdir_response_s); + struct orangefs_khandle *khandle; + __u32 *len; + u64 padlen; + loff_t i; + char *s; + i = ctx->pos & ~PART_MASK; + + /* The file offset from userspace is too large. */ + if (i > part->len) + return 1; + + /* + * If the seek pointer is positioned just before an entry it + * should find the next entry. + */ + if (i % 8) + i = i + (8 - i%8)%8; + + while (i < part->len) { + if (part->len < i + sizeof *len) + break; + len = (void *)part + offset + i; + /* + * len is the size of the string itself. padlen is the + * total size of the encoded string. + */ + padlen = (u64)sizeof *len + *len + 1; + padlen += (8 - padlen % 8) % 8; + if (part->len < i + padlen + sizeof *khandle) + goto next; + s = (void *)part + offset + i + sizeof *len; + if (s[*len] != 0) + goto next; + khandle = (void *)part + offset + i + padlen; + if (!dir_emit(ctx, s, *len, + orangefs_khandle_to_ino(khandle), + DT_UNKNOWN)) + return 0; + i += padlen + sizeof *khandle; + i = i + (8 - i%8)%8; + BUG_ON(i > part->len); + ctx->pos = (ctx->pos & PART_MASK) | i; + continue; +next: + i += 8; + } + return 1; +} + +static int orangefs_dir_fill(struct orangefs_dir *od, struct dir_context *ctx) +{ + struct orangefs_dir_part *part; + size_t count; + + count = ((ctx->pos & PART_MASK) >> PART_SHIFT) - 1; + + part = od->part; + while (part->next && count) { + count--; + part = part->next; + } + /* This means the userspace file offset is invalid. */ + if (count) { + od->error = -EIO; + return -EIO; + } + + while (part && part->len) { + int r; + r = fill_from_part(part, ctx); + if (r < 0) { + od->error = r; + return r; + } else if (r == 0) { + /* Userspace buffer is full. */ + break; + } else { + /* + * The part ran out of data. Move to the next + * part. */ + ctx->pos = (ctx->pos & PART_MASK) + + (1 << PART_SHIFT); + part = part->next; + } + } + return 0; +} + +static loff_t orangefs_dir_llseek(struct file *file, loff_t offset, + int whence) +{ + struct orangefs_dir *od = file->private_data; + /* + * Delete the stored data so userspace sees new directory + * entries. + */ + if (!whence && offset < od->end) { + struct orangefs_dir_part *part = od->part; + while (part) { + struct orangefs_dir_part *next = part->next; + vfree(part); + part = next; + } + od->token = ORANGEFS_ITERATE_START; + od->part = NULL; + od->end = 1 << PART_SHIFT; + } + return default_llseek(file, offset, whence); +} + +static int orangefs_dir_iterate(struct file *file, + struct dir_context *ctx) +{ + struct orangefs_dir *od = file->private_data; + struct inode *inode = file_inode(file); + int r; + + if (od->error) + return od->error; + + if (ctx->pos == 0) { + if (!dir_emit_dot(file, ctx)) + return 0; + ctx->pos++; + } + if (ctx->pos == 1) { + if (!dir_emit_dotdot(file, ctx)) + return 0; + ctx->pos = 1 << PART_SHIFT; + } + + /* + * The seek position is in the first synthesized part but is not + * valid. + */ + if ((ctx->pos & PART_MASK) == 0) + return -EIO; + + r = 0; + + /* + * Must read more if the user has sought past what has been read + * so far. Stop a user who has sought past the end. + */ + while (od->token != ORANGEFS_ITERATE_END && + ctx->pos > od->end) { + r = orangefs_dir_more(od, inode); + if (r) + return r; + } + if (od->token == ORANGEFS_ITERATE_END && ctx->pos > od->end) + return -EIO; + + /* Then try to fill if there's any left in the buffer. */ + if (ctx->pos < od->end) { + r = orangefs_dir_fill(od, ctx); + if (r) + return r; + } + + /* Finally get some more and try to fill. */ + if (od->token != ORANGEFS_ITERATE_END) { + r = orangefs_dir_more(od, inode); + if (r) + return r; + r = orangefs_dir_fill(od, ctx); + } + + return r; +} + +static int orangefs_dir_open(struct inode *inode, struct file *file) +{ + struct orangefs_dir *od; + file->private_data = kmalloc_obj(struct orangefs_dir); + if (!file->private_data) + return -ENOMEM; + od = file->private_data; + od->token = ORANGEFS_ITERATE_START; + od->part = NULL; + od->end = 1 << PART_SHIFT; + od->error = 0; + return 0; +} + +static int orangefs_dir_release(struct inode *inode, struct file *file) +{ + struct orangefs_dir *od = file->private_data; + struct orangefs_dir_part *part = od->part; + while (part) { + struct orangefs_dir_part *next = part->next; + vfree(part); + part = next; + } + kfree(od); + return 0; +} + +const struct file_operations orangefs_dir_operations = { + .llseek = orangefs_dir_llseek, + .read = generic_read_dir, + .iterate_shared = orangefs_dir_iterate, + .open = orangefs_dir_open, + .release = orangefs_dir_release, + .setlease = generic_setlease, +}; diff --git a/fs/orangefs/downcall.h b/fs/orangefs/downcall.h new file mode 100644 index 000000000..ea2332e16 --- /dev/null +++ b/fs/orangefs/downcall.h @@ -0,0 +1,137 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * (C) 2001 Clemson University and The University of Chicago + * + * See COPYING in top-level directory. + */ + +/* + * Definitions of downcalls used in Linux kernel module. + */ + +#ifndef __DOWNCALL_H +#define __DOWNCALL_H + +/* + * Sanitized the device-client core interaction + * for clean 32-64 bit usage + */ +struct orangefs_io_response { + __s64 amt_complete; +}; + +struct orangefs_lookup_response { + struct orangefs_object_kref refn; +}; + +struct orangefs_create_response { + struct orangefs_object_kref refn; +}; + +struct orangefs_symlink_response { + struct orangefs_object_kref refn; +}; + +struct orangefs_getattr_response { + struct ORANGEFS_sys_attr_s attributes; + char link_target[ORANGEFS_NAME_MAX]; +}; + +struct orangefs_mkdir_response { + struct orangefs_object_kref refn; +}; + +struct orangefs_statfs_response { + __s64 block_size; + __s64 blocks_total; + __s64 blocks_avail; + __s64 files_total; + __s64 files_avail; +}; + +struct orangefs_fs_mount_response { + __s32 fs_id; + __s32 id; + struct orangefs_khandle root_khandle; +}; + +/* the getxattr response is the attribute value */ +struct orangefs_getxattr_response { + __s32 val_sz; + __s32 __pad1; + char val[ORANGEFS_MAX_XATTR_VALUELEN]; +}; + +/* the listxattr response is an array of attribute names */ +struct orangefs_listxattr_response { + __s32 returned_count; + __s32 __pad1; + __u64 token; + char key[ORANGEFS_MAX_XATTR_LISTLEN * ORANGEFS_MAX_XATTR_NAMELEN]; + __s32 keylen; + __s32 __pad2; + __s32 lengths[ORANGEFS_MAX_XATTR_LISTLEN]; +}; + +struct orangefs_param_response { + union { + __s64 value64; + __s32 value32[2]; + } u; +}; + +#define PERF_COUNT_BUF_SIZE 4096 +struct orangefs_perf_count_response { + char buffer[PERF_COUNT_BUF_SIZE]; +}; + +#define FS_KEY_BUF_SIZE 4096 +struct orangefs_fs_key_response { + __s32 fs_keylen; + __s32 __pad1; + char fs_key[FS_KEY_BUF_SIZE]; +}; + +/* 2.9.6 */ +struct orangefs_features_response { + __u64 features; +}; + +struct orangefs_downcall_s { + __s32 type; + __s32 status; + /* currently trailer is used only by readdir */ + __s64 trailer_size; + char *trailer_buf; + + union { + struct orangefs_io_response io; + struct orangefs_lookup_response lookup; + struct orangefs_create_response create; + struct orangefs_symlink_response sym; + struct orangefs_getattr_response getattr; + struct orangefs_mkdir_response mkdir; + struct orangefs_statfs_response statfs; + struct orangefs_fs_mount_response fs_mount; + struct orangefs_getxattr_response getxattr; + struct orangefs_listxattr_response listxattr; + struct orangefs_param_response param; + struct orangefs_perf_count_response perf_count; + struct orangefs_fs_key_response fs_key; + struct orangefs_features_response features; + } resp; +}; + +/* + * The readdir response comes in the trailer. It is followed by the + * directory entries as described in dir.c. + */ + +struct orangefs_readdir_response_s { + __u64 token; + __u64 directory_version; + __u32 __pad2; + __u32 orangefs_dirent_outcount; +}; + +#endif /* __DOWNCALL_H */ diff --git a/fs/orangefs/file.c b/fs/orangefs/file.c new file mode 100644 index 000000000..42591252e --- /dev/null +++ b/fs/orangefs/file.c @@ -0,0 +1,587 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * (C) 2001 Clemson University and The University of Chicago + * Copyright 2018 Omnibond Systems, L.L.C. + * + * See COPYING in top-level directory. + */ + +/* + * Linux VFS file operations. + */ + +#include "protocol.h" +#include "orangefs-kernel.h" +#include "orangefs-bufmap.h" +#include +#include +#include + +static int flush_racache(struct inode *inode) +{ + struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode); + struct orangefs_kernel_op_s *new_op; + int ret; + + gossip_debug(GOSSIP_UTILS_DEBUG, + "%s: %pU: Handle is %pU | fs_id %d\n", __func__, + get_khandle_from_ino(inode), &orangefs_inode->refn.khandle, + orangefs_inode->refn.fs_id); + + new_op = op_alloc(ORANGEFS_VFS_OP_RA_FLUSH); + if (!new_op) + return -ENOMEM; + new_op->upcall.req.ra_cache_flush.refn = orangefs_inode->refn; + + ret = service_operation(new_op, "orangefs_flush_racache", + get_interruptible_flag(inode)); + + gossip_debug(GOSSIP_UTILS_DEBUG, "%s: got return value of %d\n", + __func__, ret); + + op_release(new_op); + return ret; +} + +/* + * Post and wait for the I/O upcall to finish + */ +ssize_t wait_for_direct_io(enum ORANGEFS_io_type type, struct inode *inode, + loff_t *offset, struct iov_iter *iter, size_t total_size, + loff_t readahead_size, struct orangefs_write_range *wr, + int *index_return, struct file *file) +{ + struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode); + struct orangefs_khandle *handle = &orangefs_inode->refn.khandle; + struct orangefs_kernel_op_s *new_op = NULL; + int buffer_index; + ssize_t ret; + size_t copy_amount; + bool open_for_read; + bool open_for_write; + + new_op = op_alloc(ORANGEFS_VFS_OP_FILE_IO); + if (!new_op) + return -ENOMEM; + + /* synchronous I/O */ + new_op->upcall.req.io.readahead_size = readahead_size; + new_op->upcall.req.io.io_type = type; + new_op->upcall.req.io.refn = orangefs_inode->refn; + +populate_shared_memory: + /* get a shared buffer index */ + buffer_index = orangefs_bufmap_get(); + if (buffer_index < 0) { + ret = buffer_index; + gossip_debug(GOSSIP_FILE_DEBUG, + "%s: orangefs_bufmap_get failure (%zd)\n", + __func__, ret); + goto out; + } + gossip_debug(GOSSIP_FILE_DEBUG, + "%s(%pU): GET op %p -> buffer_index %d\n", + __func__, + handle, + new_op, + buffer_index); + + new_op->uses_shared_memory = 1; + new_op->upcall.req.io.buf_index = buffer_index; + new_op->upcall.req.io.count = total_size; + new_op->upcall.req.io.offset = *offset; + if (type == ORANGEFS_IO_WRITE && wr) { + new_op->upcall.uid = from_kuid(&init_user_ns, wr->uid); + new_op->upcall.gid = from_kgid(&init_user_ns, wr->gid); + } + /* + * Orangefs has no open, and orangefs checks file permissions + * on each file access. Posix requires that file permissions + * be checked on open and nowhere else. Orangefs-through-the-kernel + * needs to seem posix compliant. + * + * The VFS opens files, even if the filesystem provides no + * method. We can see if a file was successfully opened for + * read and or for write by looking at file->f_mode. + * + * When writes are flowing from the page cache, file is no + * longer available. We can trust the VFS to have checked + * file->f_mode before writing to the page cache. + * + * The mode of a file might change between when it is opened + * and IO commences, or it might be created with an arbitrary mode. + * + * We'll make sure we don't hit EACCES during the IO stage by + * using UID 0. Some of the time we have access without changing + * to UID 0 - how to check? + */ + if (file) { + open_for_write = file->f_mode & FMODE_WRITE; + open_for_read = file->f_mode & FMODE_READ; + } else { + open_for_write = 1; + open_for_read = 0; /* not relevant? */ + } + if ((type == ORANGEFS_IO_WRITE) && open_for_write) + new_op->upcall.uid = 0; + if ((type == ORANGEFS_IO_READ) && open_for_read) + new_op->upcall.uid = 0; + + gossip_debug(GOSSIP_FILE_DEBUG, + "%s(%pU): offset: %llu total_size: %zd\n", + __func__, + handle, + llu(*offset), + total_size); + /* + * Stage 1: copy the buffers into client-core's address space + */ + if (type == ORANGEFS_IO_WRITE && total_size) { + ret = orangefs_bufmap_copy_from_iovec(iter, buffer_index, + total_size); + if (ret < 0) { + gossip_err("%s: Failed to copy-in buffers. Please make sure that the pvfs2-client is running. %ld\n", + __func__, (long)ret); + goto out; + } + } + + gossip_debug(GOSSIP_FILE_DEBUG, + "%s(%pU): Calling post_io_request with tag (%llu)\n", + __func__, + handle, + llu(new_op->tag)); + + /* Stage 2: Service the I/O operation */ + ret = service_operation(new_op, + type == ORANGEFS_IO_WRITE ? + "file_write" : + "file_read", + get_interruptible_flag(inode)); + + /* + * If service_operation() returns -EAGAIN #and# the operation was + * purged from orangefs_request_list or htable_ops_in_progress, then + * we know that the client was restarted, causing the shared memory + * area to be wiped clean. To restart a write operation in this + * case, we must re-copy the data from the user's iovec to a NEW + * shared memory location. To restart a read operation, we must get + * a new shared memory location. + */ + if (ret == -EAGAIN && op_state_purged(new_op)) { + orangefs_bufmap_put(buffer_index); + if (type == ORANGEFS_IO_WRITE) + iov_iter_revert(iter, total_size); + gossip_debug(GOSSIP_FILE_DEBUG, + "%s:going to repopulate_shared_memory.\n", + __func__); + goto populate_shared_memory; + } + + if (ret < 0) { + if (ret == -EINTR) { + /* + * We can't return EINTR if any data was written, + * it's not POSIX. It is minimally acceptable + * to give a partial write, the way NFS does. + * + * It would be optimal to return all or nothing, + * but if a userspace write is bigger than + * an IO buffer, and the interrupt occurs + * between buffer writes, that would not be + * possible. + */ + switch (new_op->op_state - OP_VFS_STATE_GIVEN_UP) { + /* + * If the op was waiting when the interrupt + * occurred, then the client-core did not + * trigger the write. + */ + case OP_VFS_STATE_WAITING: + if (*offset == 0) + ret = -EINTR; + else + ret = 0; + break; + /* + * If the op was in progress when the interrupt + * occurred, then the client-core was able to + * trigger the write. + */ + case OP_VFS_STATE_INPROGR: + if (type == ORANGEFS_IO_READ) + ret = -EINTR; + else + ret = total_size; + break; + default: + gossip_err("%s: unexpected op state :%d:.\n", + __func__, + new_op->op_state); + ret = 0; + break; + } + gossip_debug(GOSSIP_FILE_DEBUG, + "%s: got EINTR, state:%d: %p\n", + __func__, + new_op->op_state, + new_op); + } else { + gossip_err("%s: error in %s handle %pU, returning %zd\n", + __func__, + type == ORANGEFS_IO_READ ? + "read from" : "write to", + handle, ret); + } + if (orangefs_cancel_op_in_progress(new_op)) + return ret; + + goto out; + } + + /* + * Stage 3: Post copy buffers from client-core's address space + */ + if (type == ORANGEFS_IO_READ && new_op->downcall.resp.io.amt_complete) { + /* + * NOTE: the iovector can either contain addresses which + * can futher be kernel-space or user-space addresses. + * or it can pointers to struct page's + */ + + copy_amount = new_op->downcall.resp.io.amt_complete; + + ret = orangefs_bufmap_copy_to_iovec(iter, buffer_index, + copy_amount); + if (ret < 0) { + gossip_err("%s: Failed to copy-out buffers. Please make sure that the pvfs2-client is running (%ld)\n", + __func__, (long)ret); + goto out; + } + } + gossip_debug(GOSSIP_FILE_DEBUG, + "%s(%pU): Amount %s, returned by the sys-io call:%d\n", + __func__, + handle, + type == ORANGEFS_IO_READ ? "read" : "written", + (int)new_op->downcall.resp.io.amt_complete); + + ret = new_op->downcall.resp.io.amt_complete; + +out: + if (buffer_index >= 0) { + orangefs_bufmap_put(buffer_index); + gossip_debug(GOSSIP_FILE_DEBUG, + "%s(%pU): PUT buffer_index %d\n", + __func__, handle, buffer_index); + } + op_release(new_op); + return ret; +} + +int orangefs_revalidate_mapping(struct inode *inode) +{ + struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode); + struct address_space *mapping = inode->i_mapping; + unsigned long *bitlock = &orangefs_inode->bitlock; + int ret; + + while (1) { + ret = wait_on_bit(bitlock, 1, TASK_KILLABLE); + if (ret) + return ret; + spin_lock(&inode->i_lock); + if (test_bit(1, bitlock)) { + spin_unlock(&inode->i_lock); + continue; + } + if (!time_before(jiffies, orangefs_inode->mapping_time)) + break; + spin_unlock(&inode->i_lock); + return 0; + } + + set_bit(1, bitlock); + smp_wmb(); + spin_unlock(&inode->i_lock); + + unmap_mapping_range(mapping, 0, 0, 0); + ret = filemap_write_and_wait(mapping); + if (!ret) + ret = invalidate_inode_pages2(mapping); + + orangefs_inode->mapping_time = jiffies + + orangefs_cache_timeout_msecs*HZ/1000; + + clear_bit(1, bitlock); + smp_mb__after_atomic(); + wake_up_bit(bitlock, 1); + + return ret; +} + +static ssize_t orangefs_file_read_iter(struct kiocb *iocb, + struct iov_iter *iter) +{ + int ret; + orangefs_stats.reads++; + + down_read(&file_inode(iocb->ki_filp)->i_rwsem); + ret = orangefs_revalidate_mapping(file_inode(iocb->ki_filp)); + if (ret) + goto out; + + ret = generic_file_read_iter(iocb, iter); +out: + up_read(&file_inode(iocb->ki_filp)->i_rwsem); + return ret; +} + +static ssize_t orangefs_file_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, + size_t len, unsigned int flags) +{ + struct inode *inode = file_inode(in); + ssize_t ret; + + orangefs_stats.reads++; + + down_read(&inode->i_rwsem); + ret = orangefs_revalidate_mapping(inode); + if (ret) + goto out; + + ret = filemap_splice_read(in, ppos, pipe, len, flags); +out: + up_read(&inode->i_rwsem); + return ret; +} + +static ssize_t orangefs_file_write_iter(struct kiocb *iocb, + struct iov_iter *iter) +{ + int ret; + orangefs_stats.writes++; + + if (iocb->ki_pos > i_size_read(file_inode(iocb->ki_filp))) { + ret = orangefs_revalidate_mapping(file_inode(iocb->ki_filp)); + if (ret) + return ret; + } + + ret = generic_file_write_iter(iocb, iter); + return ret; +} + +static vm_fault_t orangefs_fault(struct vm_fault *vmf) +{ + struct file *file = vmf->vma->vm_file; + int ret; + ret = orangefs_inode_getattr(file->f_mapping->host, + ORANGEFS_GETATTR_SIZE); + if (ret == -ESTALE) + ret = -EIO; + if (ret) { + gossip_err("%s: orangefs_inode_getattr failed, " + "ret:%d:.\n", __func__, ret); + return VM_FAULT_SIGBUS; + } + return filemap_fault(vmf); +} + +static const struct vm_operations_struct orangefs_file_vm_ops = { + .fault = orangefs_fault, + .map_pages = filemap_map_pages, + .page_mkwrite = orangefs_page_mkwrite, +}; + +/* + * Memory map a region of a file. + */ +static int orangefs_file_mmap_prepare(struct vm_area_desc *desc) +{ + struct file *file = desc->file; + int ret; + + ret = orangefs_revalidate_mapping(file_inode(file)); + if (ret) + return ret; + + gossip_debug(GOSSIP_FILE_DEBUG, + "orangefs_file_mmap: called on %pD\n", file); + + /* set the sequential readahead hint */ + vma_desc_set_flags(desc, VMA_SEQ_READ_BIT); + vma_desc_clear_flags(desc, VMA_RAND_READ_BIT); + + file_accessed(file); + desc->vm_ops = &orangefs_file_vm_ops; + return 0; +} + +#define mapping_nrpages(idata) ((idata)->nrpages) + +/* + * Called to notify the module that there are no more references to + * this file (i.e. no processes have it open). + * + * \note Not called when each file is closed. + */ +static int orangefs_file_release(struct inode *inode, struct file *file) +{ + gossip_debug(GOSSIP_FILE_DEBUG, + "orangefs_file_release: called on %pD\n", + file); + + /* + * remove all associated inode pages from the page cache and + * readahead cache (if any); this forces an expensive refresh of + * data for the next caller of mmap (or 'get_block' accesses) + */ + if (mapping_nrpages(file->f_mapping)) { + if (orangefs_features & ORANGEFS_FEATURE_READAHEAD) { + gossip_debug(GOSSIP_INODE_DEBUG, + "calling flush_racache on %pU\n", + get_khandle_from_ino(inode)); + flush_racache(inode); + gossip_debug(GOSSIP_INODE_DEBUG, + "flush_racache finished\n"); + } + + } + return 0; +} + +/* + * Push all data for a specific file onto permanent storage. + */ +static int orangefs_fsync(struct file *file, + loff_t start, + loff_t end, + int datasync) +{ + int ret; + struct orangefs_inode_s *orangefs_inode = + ORANGEFS_I(file_inode(file)); + struct orangefs_kernel_op_s *new_op = NULL; + + ret = filemap_write_and_wait_range(file_inode(file)->i_mapping, + start, end); + if (ret < 0) + return ret; + + new_op = op_alloc(ORANGEFS_VFS_OP_FSYNC); + if (!new_op) + return -ENOMEM; + new_op->upcall.req.fsync.refn = orangefs_inode->refn; + + ret = service_operation(new_op, + "orangefs_fsync", + get_interruptible_flag(file_inode(file))); + + gossip_debug(GOSSIP_FILE_DEBUG, + "orangefs_fsync got return value of %d\n", + ret); + + op_release(new_op); + return ret; +} + +/* + * Change the file pointer position for an instance of an open file. + * + * \note If .llseek is overriden, we must acquire lock as described in + * Documentation/filesystems/locking.rst. + * + * Future upgrade could support SEEK_DATA and SEEK_HOLE but would + * require much changes to the FS + */ +static loff_t orangefs_file_llseek(struct file *file, loff_t offset, int origin) +{ + int ret = -EINVAL; + struct inode *inode = file_inode(file); + + if (origin == SEEK_END) { + /* + * revalidate the inode's file size. + * NOTE: We are only interested in file size here, + * so we set mask accordingly. + */ + ret = orangefs_inode_getattr(file->f_mapping->host, + ORANGEFS_GETATTR_SIZE); + if (ret == -ESTALE) + ret = -EIO; + if (ret) { + gossip_debug(GOSSIP_FILE_DEBUG, + "%s:%s:%d calling make bad inode\n", + __FILE__, + __func__, + __LINE__); + return ret; + } + } + + gossip_debug(GOSSIP_FILE_DEBUG, + "orangefs_file_llseek: offset is %ld | origin is %d" + " | inode size is %lu\n", + (long)offset, + origin, + (unsigned long)i_size_read(inode)); + + return generic_file_llseek(file, offset, origin); +} + +/* + * Support local locks (locks that only this kernel knows about) + * if Orangefs was mounted -o local_lock. + */ +static int orangefs_lock(struct file *filp, int cmd, struct file_lock *fl) +{ + int rc = -EINVAL; + + if (ORANGEFS_SB(file_inode(filp)->i_sb)->flags & ORANGEFS_OPT_LOCAL_LOCK) { + if (cmd == F_GETLK) { + rc = 0; + posix_test_lock(filp, fl); + } else { + rc = posix_lock_file(filp, fl, NULL); + } + } + + return rc; +} + +static int orangefs_flush(struct file *file, fl_owner_t id) +{ + /* + * This is vfs_fsync_range(file, 0, LLONG_MAX, 0) without the + * service_operation in orangefs_fsync. + * + * Do not send fsync to OrangeFS server on a close. Do send fsync + * on an explicit fsync call. This duplicates historical OrangeFS + * behavior. + */ + int r; + + r = filemap_write_and_wait_range(file->f_mapping, 0, LLONG_MAX); + if (r > 0) + return 0; + else + return r; +} + +/** ORANGEFS implementation of VFS file operations */ +const struct file_operations orangefs_file_operations = { + .llseek = orangefs_file_llseek, + .read_iter = orangefs_file_read_iter, + .write_iter = orangefs_file_write_iter, + .lock = orangefs_lock, + .mmap_prepare = orangefs_file_mmap_prepare, + .open = generic_file_open, + .splice_read = orangefs_file_splice_read, + .splice_write = iter_file_splice_write, + .flush = orangefs_flush, + .release = orangefs_file_release, + .fsync = orangefs_fsync, + .setlease = generic_setlease, +}; diff --git a/fs/orangefs/inode.c b/fs/orangefs/inode.c new file mode 100644 index 000000000..cd3273c88 --- /dev/null +++ b/fs/orangefs/inode.c @@ -0,0 +1,1167 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * (C) 2001 Clemson University and The University of Chicago + * Copyright 2018 Omnibond Systems, L.L.C. + * + * See COPYING in top-level directory. + */ + +/* + * Linux VFS inode operations. + */ + +#include +#include +#include "protocol.h" +#include "orangefs-kernel.h" +#include "orangefs-bufmap.h" + +static int orangefs_writepage_locked(struct folio *folio, + struct writeback_control *wbc) +{ + struct inode *inode = folio->mapping->host; + struct orangefs_write_range *wr = NULL; + struct iov_iter iter; + struct bio_vec bv; + size_t wlen; + ssize_t ret; + loff_t len, off; + + folio_start_writeback(folio); + + len = i_size_read(inode); + if (folio->private) { + wr = folio->private; + off = wr->pos; + if ((off + wr->len > len) && (off <= len)) + wlen = len - off; + else + wlen = wr->len; + if (wlen == 0) + wlen = wr->len; + } else { + WARN_ON(1); + off = folio_pos(folio); + wlen = folio_size(folio); + + if (wlen > len - off) + wlen = len - off; + } + + WARN_ON(wlen == 0); + bvec_set_folio(&bv, folio, wlen, offset_in_folio(folio, off)); + iov_iter_bvec(&iter, ITER_SOURCE, &bv, 1, wlen); + + ret = wait_for_direct_io(ORANGEFS_IO_WRITE, inode, &off, &iter, wlen, + len, wr, NULL, NULL); + if (ret < 0) { + mapping_set_error(folio->mapping, ret); + } else { + ret = 0; + } + kfree(folio_detach_private(folio)); + return ret; +} + +struct orangefs_writepages { + loff_t off; + size_t len; + kuid_t uid; + kgid_t gid; + int maxpages; + int nfolios; + struct address_space *mapping; + struct folio **folios; + struct bio_vec *bv; +}; + +static int orangefs_writepages_work(struct orangefs_writepages *ow, + struct writeback_control *wbc) +{ + struct inode *inode = ow->mapping->host; + struct orangefs_write_range *wrp, wr; + struct iov_iter iter; + ssize_t ret; + size_t start; + loff_t len, off; + int i; + + len = i_size_read(inode); + + start = offset_in_folio(ow->folios[0], ow->off); + for (i = 0; i < ow->nfolios; i++) { + folio_start_writeback(ow->folios[i]); + bvec_set_folio(&ow->bv[i], ow->folios[i], + folio_size(ow->folios[i]) - start, start); + start = 0; + } + iov_iter_bvec(&iter, ITER_SOURCE, ow->bv, ow->nfolios, ow->len); + + WARN_ON(ow->off >= len); + if (ow->off + ow->len > len) + ow->len = len - ow->off; + + off = ow->off; + wr.uid = ow->uid; + wr.gid = ow->gid; + ret = wait_for_direct_io(ORANGEFS_IO_WRITE, inode, &off, &iter, ow->len, + 0, &wr, NULL, NULL); + if (ret < 0) + mapping_set_error(ow->mapping, ret); + else + ret = 0; + + for (i = 0; i < ow->nfolios; i++) { + wrp = folio_detach_private(ow->folios[i]); + kfree(wrp); + folio_end_writeback(ow->folios[i]); + folio_unlock(ow->folios[i]); + } + + return ret; +} + +static int orangefs_writepages_callback(struct folio *folio, + struct writeback_control *wbc, struct orangefs_writepages *ow) +{ + struct orangefs_write_range *wr = folio->private; + int ret; + + if (!wr) { + folio_unlock(folio); + /* It's not private so there's nothing to write, right? */ + printk("writepages_callback not private!\n"); + BUG(); + return 0; + } + + ret = -1; + if (ow->nfolios == 0) { + ow->off = wr->pos; + ow->len = wr->len; + ow->uid = wr->uid; + ow->gid = wr->gid; + ow->folios[ow->nfolios++] = folio; + ret = 0; + goto done; + } + if (!uid_eq(ow->uid, wr->uid) || !gid_eq(ow->gid, wr->gid)) { + orangefs_writepages_work(ow, wbc); + ow->nfolios = 0; + ret = -1; + goto done; + } + if (ow->off + ow->len == wr->pos) { + ow->len += wr->len; + ow->folios[ow->nfolios++] = folio; + ret = 0; + goto done; + } +done: + if (ret == -1) { + if (ow->nfolios) { + orangefs_writepages_work(ow, wbc); + ow->nfolios = 0; + } + ret = orangefs_writepage_locked(folio, wbc); + mapping_set_error(folio->mapping, ret); + folio_unlock(folio); + folio_end_writeback(folio); + } else { + if (ow->nfolios == ow->maxpages) { + orangefs_writepages_work(ow, wbc); + ow->nfolios = 0; + } + } + return ret; +} + +static int orangefs_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct orangefs_writepages *ow; + struct blk_plug plug; + int error; + struct folio *folio = NULL; + + ow = kzalloc_obj(struct orangefs_writepages); + if (!ow) + return -ENOMEM; + ow->maxpages = orangefs_bufmap_size_query()/PAGE_SIZE; + ow->folios = kzalloc_objs(struct folio *, ow->maxpages); + if (!ow->folios) { + kfree(ow); + return -ENOMEM; + } + ow->bv = kzalloc_objs(struct bio_vec, ow->maxpages); + if (!ow->bv) { + kfree(ow->folios); + kfree(ow); + return -ENOMEM; + } + ow->mapping = mapping; + blk_start_plug(&plug); + while ((folio = writeback_iter(mapping, wbc, folio, &error))) + error = orangefs_writepages_callback(folio, wbc, ow); + if (ow->nfolios) + error = orangefs_writepages_work(ow, wbc); + blk_finish_plug(&plug); + kfree(ow->folios); + kfree(ow->bv); + kfree(ow); + return error; +} + +static int orangefs_launder_folio(struct folio *); + +static void orangefs_readahead(struct readahead_control *rac) +{ + loff_t offset; + struct iov_iter iter; + struct inode *inode = rac->mapping->host; + struct xarray *i_pages; + struct folio *folio; + loff_t new_start = readahead_pos(rac); + int ret; + size_t new_len = 0; + size_t this_size; + size_t remaining; + + loff_t bytes_remaining = inode->i_size - readahead_pos(rac); + loff_t pages_remaining = bytes_remaining / PAGE_SIZE; + + if (pages_remaining >= 1024) + new_len = 4194304; + else if (pages_remaining > readahead_count(rac)) + new_len = bytes_remaining; + + if (new_len) + readahead_expand(rac, new_start, new_len); + + offset = readahead_pos(rac); + i_pages = &rac->mapping->i_pages; + + iov_iter_xarray(&iter, ITER_DEST, i_pages, + offset, readahead_length(rac)); + + remaining = readahead_length(rac); + while (remaining) { + if (remaining > 4194304) + this_size = 4194304; + else + this_size = remaining; + + /* read in the pages. */ + if ((ret = wait_for_direct_io(ORANGEFS_IO_READ, inode, + &offset, &iter, this_size, + inode->i_size, NULL, NULL, rac->file)) < 0) { + gossip_debug(GOSSIP_FILE_DEBUG, + "%s: wait_for_direct_io failed. :%d: \n", + __func__, ret); + goto cleanup; + } else { + ret = 0; + } + + remaining -= this_size; + offset += this_size; + } + +cleanup: + /* clean up. */ + while ((folio = readahead_folio(rac))) { + if (!ret) + folio_mark_uptodate(folio); + folio_unlock(folio); + } +} + +static int orangefs_read_folio(struct file *file, struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + struct iov_iter iter; + struct bio_vec bv; + ssize_t ret; + loff_t off; /* offset of this folio in the file */ + + if (folio_test_dirty(folio)) + orangefs_launder_folio(folio); + + off = folio_pos(folio); + bvec_set_folio(&bv, folio, folio_size(folio), 0); + iov_iter_bvec(&iter, ITER_DEST, &bv, 1, folio_size(folio)); + + ret = wait_for_direct_io(ORANGEFS_IO_READ, inode, &off, &iter, + folio_size(folio), inode->i_size, NULL, NULL, file); + /* this will only zero remaining unread portions of the folio data */ + iov_iter_zero(~0U, &iter); + /* takes care of potential aliasing */ + flush_dcache_folio(folio); + if (ret > 0) + ret = 0; + folio_end_read(folio, ret == 0); + return ret; +} + +static int orangefs_write_begin(const struct kiocb *iocb, + struct address_space *mapping, loff_t pos, + unsigned len, struct folio **foliop, + void **fsdata) +{ + struct orangefs_write_range *wr; + struct folio *folio; + int ret; + + folio = __filemap_get_folio(mapping, pos / PAGE_SIZE, FGP_WRITEBEGIN, + mapping_gfp_mask(mapping)); + if (IS_ERR(folio)) + return PTR_ERR(folio); + + *foliop = folio; + + if (folio_test_dirty(folio) && !folio_test_private(folio)) { + /* + * Should be impossible. If it happens, launder the page + * since we don't know what's dirty. This will WARN in + * orangefs_writepage_locked. + */ + ret = orangefs_launder_folio(folio); + if (ret) + return ret; + } + if (folio_test_private(folio)) { + struct orangefs_write_range *wr; + wr = folio_get_private(folio); + if (wr->pos + wr->len == pos && + uid_eq(wr->uid, current_fsuid()) && + gid_eq(wr->gid, current_fsgid())) { + wr->len += len; + goto okay; + } else { + wr->pos = pos; + wr->len = len; + ret = orangefs_launder_folio(folio); + if (ret) + return ret; + } + } + + wr = kmalloc_obj(*wr); + if (!wr) + return -ENOMEM; + + wr->pos = pos; + wr->len = len; + wr->uid = current_fsuid(); + wr->gid = current_fsgid(); + folio_attach_private(folio, wr); +okay: + return 0; +} + +static int orangefs_write_end(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct folio *folio, void *fsdata) +{ + struct inode *inode = folio->mapping->host; + loff_t last_pos = pos + copied; + + /* + * No need to use i_size_read() here, the i_size + * cannot change under us because we hold the i_mutex. + */ + if (last_pos > inode->i_size) + i_size_write(inode, last_pos); + + /* zero the stale part of the folio if we did a short copy */ + if (!folio_test_uptodate(folio)) { + unsigned from = pos & (PAGE_SIZE - 1); + if (copied < len) { + folio_zero_range(folio, from + copied, len - copied); + } + /* Set fully written pages uptodate. */ + if (pos == folio_pos(folio) && + (len == PAGE_SIZE || pos + len == inode->i_size)) { + folio_zero_segment(folio, from + copied, PAGE_SIZE); + folio_mark_uptodate(folio); + } + } + + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + + mark_inode_dirty_sync(file_inode(iocb->ki_filp)); + return copied; +} + +static void orangefs_invalidate_folio(struct folio *folio, + size_t offset, size_t length) +{ + struct orangefs_write_range *wr = folio_get_private(folio); + + if (offset == 0 && length == PAGE_SIZE) { + kfree(folio_detach_private(folio)); + return; + /* write range entirely within invalidate range (or equal) */ + } else if (folio_pos(folio) + offset <= wr->pos && + wr->pos + wr->len <= folio_pos(folio) + offset + length) { + kfree(folio_detach_private(folio)); + /* XXX is this right? only caller in fs */ + folio_cancel_dirty(folio); + return; + /* invalidate range chops off end of write range */ + } else if (wr->pos < folio_pos(folio) + offset && + wr->pos + wr->len <= folio_pos(folio) + offset + length && + folio_pos(folio) + offset < wr->pos + wr->len) { + size_t x; + x = wr->pos + wr->len - (folio_pos(folio) + offset); + WARN_ON(x > wr->len); + wr->len -= x; + wr->uid = current_fsuid(); + wr->gid = current_fsgid(); + /* invalidate range chops off beginning of write range */ + } else if (folio_pos(folio) + offset <= wr->pos && + folio_pos(folio) + offset + length < wr->pos + wr->len && + wr->pos < folio_pos(folio) + offset + length) { + size_t x; + x = folio_pos(folio) + offset + length - wr->pos; + WARN_ON(x > wr->len); + wr->pos += x; + wr->len -= x; + wr->uid = current_fsuid(); + wr->gid = current_fsgid(); + /* invalidate range entirely within write range (punch hole) */ + } else if (wr->pos < folio_pos(folio) + offset && + folio_pos(folio) + offset + length < wr->pos + wr->len) { + /* XXX what do we do here... should not WARN_ON */ + WARN_ON(1); + /* punch hole */ + /* + * should we just ignore this and write it out anyway? + * it hardly makes sense + */ + return; + /* non-overlapping ranges */ + } else { + /* WARN if they do overlap */ + if (!((folio_pos(folio) + offset + length <= wr->pos) ^ + (wr->pos + wr->len <= folio_pos(folio) + offset))) { + WARN_ON(1); + printk("invalidate range offset %llu length %zu\n", + folio_pos(folio) + offset, length); + printk("write range offset %llu length %zu\n", + wr->pos, wr->len); + } + return; + } + + /* + * Above there are returns where wr is freed or where we WARN. + * Thus the following runs if wr was modified above. + */ + + orangefs_launder_folio(folio); +} + +static bool orangefs_release_folio(struct folio *folio, gfp_t foo) +{ + return !folio_test_private(folio); +} + +static void orangefs_free_folio(struct folio *folio) +{ + kfree(folio_detach_private(folio)); +} + +static int orangefs_launder_folio(struct folio *folio) +{ + int r = 0; + struct writeback_control wbc = { + .sync_mode = WB_SYNC_ALL, + .nr_to_write = 0, + }; + folio_wait_writeback(folio); + if (folio_clear_dirty_for_io(folio)) { + r = orangefs_writepage_locked(folio, &wbc); + folio_end_writeback(folio); + } + return r; +} + +static ssize_t orangefs_direct_IO(struct kiocb *iocb, + struct iov_iter *iter) +{ + /* + * Comment from original do_readv_writev: + * Common entry point for read/write/readv/writev + * This function will dispatch it to either the direct I/O + * or buffered I/O path depending on the mount options and/or + * augmented/extended metadata attached to the file. + * Note: File extended attributes override any mount options. + */ + struct file *file = iocb->ki_filp; + loff_t pos = iocb->ki_pos; + enum ORANGEFS_io_type type = iov_iter_rw(iter) == WRITE ? + ORANGEFS_IO_WRITE : ORANGEFS_IO_READ; + loff_t *offset = &pos; + struct inode *inode = file->f_mapping->host; + struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode); + struct orangefs_khandle *handle = &orangefs_inode->refn.khandle; + size_t count = iov_iter_count(iter); + ssize_t total_count = 0; + ssize_t ret = -EINVAL; + + gossip_debug(GOSSIP_FILE_DEBUG, + "%s-BEGIN(%pU): count(%d) after estimate_max_iovecs.\n", + __func__, + handle, + (int)count); + + if (type == ORANGEFS_IO_WRITE) { + gossip_debug(GOSSIP_FILE_DEBUG, + "%s(%pU): proceeding with offset : %llu, " + "size %d\n", + __func__, + handle, + llu(*offset), + (int)count); + } + + if (count == 0) { + ret = 0; + goto out; + } + + while (iov_iter_count(iter)) { + size_t each_count = iov_iter_count(iter); + size_t amt_complete; + + /* how much to transfer in this loop iteration */ + if (each_count > orangefs_bufmap_size_query()) + each_count = orangefs_bufmap_size_query(); + + gossip_debug(GOSSIP_FILE_DEBUG, + "%s(%pU): size of each_count(%d)\n", + __func__, + handle, + (int)each_count); + gossip_debug(GOSSIP_FILE_DEBUG, + "%s(%pU): BEFORE wait_for_io: offset is %d\n", + __func__, + handle, + (int)*offset); + + ret = wait_for_direct_io(type, inode, offset, iter, + each_count, 0, NULL, NULL, file); + gossip_debug(GOSSIP_FILE_DEBUG, + "%s(%pU): return from wait_for_io:%d\n", + __func__, + handle, + (int)ret); + + if (ret < 0) + goto out; + + *offset += ret; + total_count += ret; + amt_complete = ret; + + gossip_debug(GOSSIP_FILE_DEBUG, + "%s(%pU): AFTER wait_for_io: offset is %d\n", + __func__, + handle, + (int)*offset); + + /* + * if we got a short I/O operations, + * fall out and return what we got so far + */ + if (amt_complete < each_count) + break; + } /*end while */ + +out: + if (total_count > 0) + ret = total_count; + if (ret > 0) { + if (type == ORANGEFS_IO_READ) { + file_accessed(file); + } else { + file_update_time(file); + if (*offset > i_size_read(inode)) + i_size_write(inode, *offset); + } + } + + gossip_debug(GOSSIP_FILE_DEBUG, + "%s(%pU): Value(%d) returned.\n", + __func__, + handle, + (int)ret); + + return ret; +} + +/** ORANGEFS2 implementation of address space operations */ +static const struct address_space_operations orangefs_address_operations = { + .readahead = orangefs_readahead, + .read_folio = orangefs_read_folio, + .writepages = orangefs_writepages, + .dirty_folio = filemap_dirty_folio, + .write_begin = orangefs_write_begin, + .write_end = orangefs_write_end, + .invalidate_folio = orangefs_invalidate_folio, + .release_folio = orangefs_release_folio, + .free_folio = orangefs_free_folio, + .migrate_folio = filemap_migrate_folio, + .launder_folio = orangefs_launder_folio, + .direct_IO = orangefs_direct_IO, +}; + +vm_fault_t orangefs_page_mkwrite(struct vm_fault *vmf) +{ + struct folio *folio = page_folio(vmf->page); + struct inode *inode = file_inode(vmf->vma->vm_file); + struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode); + unsigned long *bitlock = &orangefs_inode->bitlock; + vm_fault_t ret; + struct orangefs_write_range *wr; + + sb_start_pagefault(inode->i_sb); + + if (wait_on_bit(bitlock, 1, TASK_KILLABLE)) { + ret = VM_FAULT_RETRY; + goto out; + } + + folio_lock(folio); + if (folio_test_dirty(folio) && !folio_test_private(folio)) { + /* + * Should be impossible. If it happens, launder the folio + * since we don't know what's dirty. This will WARN in + * orangefs_writepage_locked. + */ + if (orangefs_launder_folio(folio)) { + ret = VM_FAULT_LOCKED|VM_FAULT_RETRY; + goto out; + } + } + if (folio_test_private(folio)) { + wr = folio_get_private(folio); + if (uid_eq(wr->uid, current_fsuid()) && + gid_eq(wr->gid, current_fsgid())) { + wr->pos = folio_pos(folio); + wr->len = folio_size(folio); + goto okay; + } else { + if (orangefs_launder_folio(folio)) { + ret = VM_FAULT_LOCKED|VM_FAULT_RETRY; + goto out; + } + } + } + wr = kmalloc_obj(*wr); + if (!wr) { + ret = VM_FAULT_LOCKED|VM_FAULT_RETRY; + goto out; + } + wr->pos = folio_pos(folio); + wr->len = folio_size(folio); + wr->uid = current_fsuid(); + wr->gid = current_fsgid(); + folio_attach_private(folio, wr); +okay: + + file_update_time(vmf->vma->vm_file); + if (folio->mapping != inode->i_mapping) { + folio_unlock(folio); + ret = VM_FAULT_LOCKED|VM_FAULT_NOPAGE; + goto out; + } + + /* + * We mark the folio dirty already here so that when freeze is in + * progress, we are guaranteed that writeback during freezing will + * see the dirty folio and writeprotect it again. + */ + folio_mark_dirty(folio); + folio_wait_stable(folio); + ret = VM_FAULT_LOCKED; +out: + sb_end_pagefault(inode->i_sb); + return ret; +} + +static int orangefs_setattr_size(struct inode *inode, struct iattr *iattr) +{ + struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode); + struct orangefs_kernel_op_s *new_op; + loff_t orig_size; + int ret = -EINVAL; + + gossip_debug(GOSSIP_INODE_DEBUG, + "%s: %pU: Handle is %pU | fs_id %d | size is %llu\n", + __func__, + get_khandle_from_ino(inode), + &orangefs_inode->refn.khandle, + orangefs_inode->refn.fs_id, + iattr->ia_size); + + /* Ensure that we have a up to date size, so we know if it changed. */ + ret = orangefs_inode_getattr(inode, ORANGEFS_GETATTR_SIZE); + if (ret == -ESTALE) + ret = -EIO; + if (ret) { + gossip_err("%s: orangefs_inode_getattr failed, ret:%d:.\n", + __func__, ret); + return ret; + } + orig_size = i_size_read(inode); + + /* This is truncate_setsize in a different order. */ + truncate_pagecache(inode, iattr->ia_size); + i_size_write(inode, iattr->ia_size); + if (iattr->ia_size > orig_size) + pagecache_isize_extended(inode, orig_size, iattr->ia_size); + + new_op = op_alloc(ORANGEFS_VFS_OP_TRUNCATE); + if (!new_op) + return -ENOMEM; + + new_op->upcall.req.truncate.refn = orangefs_inode->refn; + new_op->upcall.req.truncate.size = (__s64) iattr->ia_size; + + ret = service_operation(new_op, + __func__, + get_interruptible_flag(inode)); + + /* + * the truncate has no downcall members to retrieve, but + * the status value tells us if it went through ok or not + */ + gossip_debug(GOSSIP_INODE_DEBUG, "%s: ret:%d:\n", __func__, ret); + + op_release(new_op); + + if (ret != 0) + return ret; + + if (orig_size != i_size_read(inode)) + iattr->ia_valid |= ATTR_CTIME | ATTR_MTIME; + + return ret; +} + +int __orangefs_setattr(struct inode *inode, struct iattr *iattr) +{ + int ret; + + if (iattr->ia_valid & ATTR_MODE) { + if (iattr->ia_mode & (S_ISVTX)) { + if (is_root_handle(inode)) { + /* + * allow sticky bit to be set on root (since + * it shows up that way by default anyhow), + * but don't show it to the server + */ + iattr->ia_mode -= S_ISVTX; + } else { + gossip_debug(GOSSIP_UTILS_DEBUG, + "User attempted to set sticky bit on non-root directory; returning EINVAL.\n"); + ret = -EINVAL; + goto out; + } + } + if (iattr->ia_mode & (S_ISUID)) { + gossip_debug(GOSSIP_UTILS_DEBUG, + "Attempting to set setuid bit (not supported); returning EINVAL.\n"); + ret = -EINVAL; + goto out; + } + } + + if (iattr->ia_valid & ATTR_SIZE) { + ret = orangefs_setattr_size(inode, iattr); + if (ret) + goto out; + } + +again: + spin_lock(&inode->i_lock); + if (ORANGEFS_I(inode)->attr_valid) { + if (uid_eq(ORANGEFS_I(inode)->attr_uid, current_fsuid()) && + gid_eq(ORANGEFS_I(inode)->attr_gid, current_fsgid())) { + ORANGEFS_I(inode)->attr_valid = iattr->ia_valid; + } else { + spin_unlock(&inode->i_lock); + write_inode_now(inode, 1); + goto again; + } + } else { + ORANGEFS_I(inode)->attr_valid = iattr->ia_valid; + ORANGEFS_I(inode)->attr_uid = current_fsuid(); + ORANGEFS_I(inode)->attr_gid = current_fsgid(); + } + setattr_copy(&nop_mnt_idmap, inode, iattr); + spin_unlock(&inode->i_lock); + mark_inode_dirty(inode); + + ret = 0; +out: + return ret; +} + +int __orangefs_setattr_mode(struct dentry *dentry, struct iattr *iattr) +{ + int ret; + struct inode *inode = d_inode(dentry); + + ret = __orangefs_setattr(inode, iattr); + /* change mode on a file that has ACLs */ + if (!ret && (iattr->ia_valid & ATTR_MODE)) + ret = posix_acl_chmod(&nop_mnt_idmap, dentry, inode->i_mode); + return ret; +} + +/* + * Change attributes of an object referenced by dentry. + */ +int orangefs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *iattr) +{ + int ret; + gossip_debug(GOSSIP_INODE_DEBUG, "__orangefs_setattr: called on %pd\n", + dentry); + ret = setattr_prepare(&nop_mnt_idmap, dentry, iattr); + if (ret) + goto out; + ret = __orangefs_setattr_mode(dentry, iattr); + sync_inode_metadata(d_inode(dentry), 1); +out: + gossip_debug(GOSSIP_INODE_DEBUG, "orangefs_setattr: returning %d\n", + ret); + return ret; +} + +/* + * Obtain attributes of an object given a dentry + */ +int orangefs_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags) +{ + int ret; + struct inode *inode = path->dentry->d_inode; + + gossip_debug(GOSSIP_INODE_DEBUG, + "orangefs_getattr: called on %pd mask %u\n", + path->dentry, request_mask); + + ret = orangefs_inode_getattr(inode, + request_mask & STATX_SIZE ? ORANGEFS_GETATTR_SIZE : 0); + if (ret == 0) { + generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat); + + /* override block size reported to stat */ + if (!(request_mask & STATX_SIZE)) + stat->result_mask &= ~STATX_SIZE; + + generic_fill_statx_attr(inode, stat); + } + return ret; +} + +int orangefs_permission(struct mnt_idmap *idmap, + struct inode *inode, int mask) +{ + int ret; + + if (mask & MAY_NOT_BLOCK) + return -ECHILD; + + gossip_debug(GOSSIP_INODE_DEBUG, "%s: refreshing\n", __func__); + + /* Make sure the permission (and other common attrs) are up to date. */ + ret = orangefs_inode_getattr(inode, 0); + if (ret < 0) + return ret; + + return generic_permission(&nop_mnt_idmap, inode, mask); +} + +int orangefs_update_time(struct inode *inode, enum fs_update_time type, + unsigned int flags) +{ + struct iattr iattr = { }; + int dirty; + + if (flags & IOCB_NOWAIT) + return -EAGAIN; + + switch (type) { + case FS_UPD_ATIME: + iattr.ia_valid = ATTR_ATIME; + break; + case FS_UPD_CMTIME: + iattr.ia_valid = ATTR_CTIME | ATTR_MTIME; + break; + } + + dirty = inode_update_time(inode, type, flags); + if (dirty <= 0) + return dirty; + return __orangefs_setattr(inode, &iattr); +} + +static int orangefs_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + u64 val = 0; + int ret; + + gossip_debug(GOSSIP_FILE_DEBUG, "%s: called on %pd\n", __func__, + dentry); + + ret = orangefs_inode_getxattr(d_inode(dentry), + "user.pvfs2.meta_hint", + &val, sizeof(val)); + if (ret < 0 && ret != -ENODATA) + return ret; + + gossip_debug(GOSSIP_FILE_DEBUG, "%s: flags=%u\n", __func__, (u32) val); + + fileattr_fill_flags(fa, val); + return 0; +} + +static int orangefs_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa) +{ + u64 val = 0; + + gossip_debug(GOSSIP_FILE_DEBUG, "%s: called on %pd\n", __func__, + dentry); + /* + * ORANGEFS_MIRROR_FL is set internally when the mirroring mode is + * turned on for a file. The user is not allowed to turn on this bit, + * but the bit is present if the user first gets the flags and then + * updates the flags with some new settings. So, we ignore it in the + * following edit. bligon. + */ + if (fileattr_has_fsx(fa) || + (fa->flags & ~(FS_IMMUTABLE_FL | FS_APPEND_FL | FS_NOATIME_FL | ORANGEFS_MIRROR_FL))) { + gossip_err("%s: only supports setting one of FS_IMMUTABLE_FL|FS_APPEND_FL|FS_NOATIME_FL\n", + __func__); + return -EOPNOTSUPP; + } + val = fa->flags; + gossip_debug(GOSSIP_FILE_DEBUG, "%s: flags=%u\n", __func__, (u32) val); + return orangefs_inode_setxattr(d_inode(dentry), + "user.pvfs2.meta_hint", + &val, sizeof(val), 0); +} + +/* ORANGEFS2 implementation of VFS inode operations for files */ +static const struct inode_operations orangefs_file_inode_operations = { + .get_inode_acl = orangefs_get_acl, + .set_acl = orangefs_set_acl, + .setattr = orangefs_setattr, + .getattr = orangefs_getattr, + .listxattr = orangefs_listxattr, + .permission = orangefs_permission, + .update_time = orangefs_update_time, + .fileattr_get = orangefs_fileattr_get, + .fileattr_set = orangefs_fileattr_set, +}; + +static int orangefs_init_iops(struct inode *inode) +{ + inode->i_mapping->a_ops = &orangefs_address_operations; + + switch (inode->i_mode & S_IFMT) { + case S_IFREG: + inode->i_op = &orangefs_file_inode_operations; + inode->i_fop = &orangefs_file_operations; + break; + case S_IFLNK: + inode->i_op = &orangefs_symlink_inode_operations; + break; + case S_IFDIR: + inode->i_op = &orangefs_dir_inode_operations; + inode->i_fop = &orangefs_dir_operations; + break; + default: + gossip_debug(GOSSIP_INODE_DEBUG, + "%s: unsupported mode\n", + __func__); + return -EINVAL; + } + + return 0; +} + +/* + * Given an ORANGEFS object identifier (fsid, handle), convert it into + * a ino_t type that will be used as a hash-index from where the handle will + * be searched for in the VFS hash table of inodes. + */ +static inline ino_t orangefs_handle_hash(struct orangefs_object_kref *ref) +{ + if (!ref) + return 0; + return orangefs_khandle_to_ino(&(ref->khandle)); +} + +/* + * Called to set up an inode from iget5_locked. + */ +static int orangefs_set_inode(struct inode *inode, void *data) +{ + struct orangefs_object_kref *ref = (struct orangefs_object_kref *) data; + ORANGEFS_I(inode)->refn.fs_id = ref->fs_id; + ORANGEFS_I(inode)->refn.khandle = ref->khandle; + ORANGEFS_I(inode)->attr_valid = 0; + hash_init(ORANGEFS_I(inode)->xattr_cache); + ORANGEFS_I(inode)->mapping_time = jiffies - 1; + ORANGEFS_I(inode)->bitlock = 0; + return 0; +} + +/* + * Called to determine if handles match. + */ +static int orangefs_test_inode(struct inode *inode, void *data) +{ + struct orangefs_object_kref *ref = (struct orangefs_object_kref *) data; + struct orangefs_inode_s *orangefs_inode = NULL; + + orangefs_inode = ORANGEFS_I(inode); + /* test handles and fs_ids... */ + return (!ORANGEFS_khandle_cmp(&(orangefs_inode->refn.khandle), + &(ref->khandle)) && + orangefs_inode->refn.fs_id == ref->fs_id); +} + +/* + * Front-end to lookup the inode-cache maintained by the VFS using the ORANGEFS + * file handle. + * + * @sb: the file system super block instance. + * @ref: The ORANGEFS object for which we are trying to locate an inode. + */ +struct inode *orangefs_iget(struct super_block *sb, + struct orangefs_object_kref *ref) +{ + struct inode *inode = NULL; + unsigned long hash; + int error; + + hash = orangefs_handle_hash(ref); + inode = iget5_locked(sb, + hash, + orangefs_test_inode, + orangefs_set_inode, + ref); + + if (!inode) + return ERR_PTR(-ENOMEM); + + if (!(inode_state_read_once(inode) & I_NEW)) + return inode; + + error = orangefs_inode_getattr(inode, ORANGEFS_GETATTR_NEW); + if (error) { + iget_failed(inode); + return ERR_PTR(error); + } + + inode->i_ino = hash; /* needed for stat etc */ + orangefs_init_iops(inode); + unlock_new_inode(inode); + + gossip_debug(GOSSIP_INODE_DEBUG, + "iget handle %pU, fsid %d hash %ld i_ino %llu\n", + &ref->khandle, + ref->fs_id, + hash, + inode->i_ino); + + return inode; +} + +/* + * Allocate an inode for a newly created file and insert it into the inode hash. + */ +struct inode *orangefs_new_inode(struct super_block *sb, struct inode *dir, + umode_t mode, dev_t dev, struct orangefs_object_kref *ref) +{ + struct posix_acl *acl = NULL, *default_acl = NULL; + unsigned long hash = orangefs_handle_hash(ref); + struct inode *inode; + int error; + + gossip_debug(GOSSIP_INODE_DEBUG, + "%s:(sb is %p | MAJOR(dev)=%u | MINOR(dev)=%u mode=%o)\n", + __func__, + sb, + MAJOR(dev), + MINOR(dev), + mode); + + inode = new_inode(sb); + if (!inode) + return ERR_PTR(-ENOMEM); + + error = posix_acl_create(dir, &mode, &default_acl, &acl); + if (error) + goto out_iput; + + orangefs_set_inode(inode, ref); + inode->i_ino = hash; /* needed for stat etc */ + + error = orangefs_inode_getattr(inode, ORANGEFS_GETATTR_NEW); + if (error) + goto out_iput; + + orangefs_init_iops(inode); + inode->i_rdev = dev; + + if (default_acl) { + error = __orangefs_set_acl(inode, default_acl, + ACL_TYPE_DEFAULT); + if (error) + goto out_iput; + } + + if (acl) { + error = __orangefs_set_acl(inode, acl, ACL_TYPE_ACCESS); + if (error) + goto out_iput; + } + + error = insert_inode_locked4(inode, hash, orangefs_test_inode, ref); + if (error < 0) + goto out_iput; + + gossip_debug(GOSSIP_INODE_DEBUG, + "Initializing ACL's for inode %pU\n", + get_khandle_from_ino(inode)); + if (mode != inode->i_mode) { + struct iattr iattr = { + .ia_mode = mode, + .ia_valid = ATTR_MODE, + }; + inode->i_mode = mode; + __orangefs_setattr(inode, &iattr); + __posix_acl_chmod(&acl, GFP_KERNEL, inode->i_mode); + } + posix_acl_release(acl); + posix_acl_release(default_acl); + return inode; + +out_iput: + iput(inode); + posix_acl_release(acl); + posix_acl_release(default_acl); + return ERR_PTR(error); +} diff --git a/fs/orangefs/namei.c b/fs/orangefs/namei.c new file mode 100644 index 000000000..8ebc34e11 --- /dev/null +++ b/fs/orangefs/namei.c @@ -0,0 +1,431 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * (C) 2001 Clemson University and The University of Chicago + * + * See COPYING in top-level directory. + */ + +/* + * Linux VFS namei operations. + */ + +#include "protocol.h" +#include "orangefs-kernel.h" + +/* + * Get a newly allocated inode to go with a negative dentry. + */ +static int orangefs_create(struct mnt_idmap *idmap, + struct inode *dir, + struct dentry *dentry, + umode_t mode) +{ + struct orangefs_inode_s *parent = ORANGEFS_I(dir); + struct orangefs_kernel_op_s *new_op; + struct orangefs_object_kref ref; + struct inode *inode; + struct iattr iattr; + int ret; + + gossip_debug(GOSSIP_NAME_DEBUG, "%s: %pd\n", + __func__, + dentry); + + new_op = op_alloc(ORANGEFS_VFS_OP_CREATE); + if (!new_op) + return -ENOMEM; + + new_op->upcall.req.create.parent_refn = parent->refn; + + fill_default_sys_attrs(new_op->upcall.req.create.attributes, mode); + + strscpy(new_op->upcall.req.create.d_name, dentry->d_name.name); + + ret = service_operation(new_op, __func__, get_interruptible_flag(dir)); + + gossip_debug(GOSSIP_NAME_DEBUG, + "%s: %pd: handle:%pU: fsid:%d: new_op:%p: ret:%d:\n", + __func__, + dentry, + &new_op->downcall.resp.create.refn.khandle, + new_op->downcall.resp.create.refn.fs_id, + new_op, + ret); + + if (ret < 0) + goto out; + + ref = new_op->downcall.resp.create.refn; + + inode = orangefs_new_inode(dir->i_sb, dir, S_IFREG | mode, 0, &ref); + if (IS_ERR(inode)) { + gossip_err("%s: Failed to allocate inode for file :%pd:\n", + __func__, + dentry); + ret = PTR_ERR(inode); + goto out; + } + + gossip_debug(GOSSIP_NAME_DEBUG, + "%s: Assigned inode :%pU: for file :%pd:\n", + __func__, + get_khandle_from_ino(inode), + dentry); + + d_instantiate_new(dentry, inode); + orangefs_set_timeout(dentry); + + gossip_debug(GOSSIP_NAME_DEBUG, + "%s: dentry instantiated for %pd\n", + __func__, + dentry); + + memset(&iattr, 0, sizeof iattr); + iattr.ia_valid |= ATTR_MTIME | ATTR_CTIME; + iattr.ia_mtime = iattr.ia_ctime = current_time(dir); + __orangefs_setattr(dir, &iattr); + ret = 0; +out: + op_release(new_op); + gossip_debug(GOSSIP_NAME_DEBUG, + "%s: %pd: returning %d\n", + __func__, + dentry, + ret); + return ret; +} + +/* + * Attempt to resolve an object name (dentry->d_name), parent handle, and + * fsid into a handle for the object. + */ +static struct dentry *orangefs_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + struct orangefs_inode_s *parent = ORANGEFS_I(dir); + struct orangefs_kernel_op_s *new_op; + struct inode *inode; + int ret = -EINVAL; + + /* + * in theory we could skip a lookup here (if the intent is to + * create) in order to avoid a potentially failed lookup, but + * leaving it in can skip a valid lookup and try to create a file + * that already exists (e.g. the vfs already handles checking for + * -EEXIST on O_EXCL opens, which is broken if we skip this lookup + * in the create path) + */ + gossip_debug(GOSSIP_NAME_DEBUG, "%s called on %pd\n", + __func__, dentry); + + if (dentry->d_name.len > (ORANGEFS_NAME_MAX - 1)) + return ERR_PTR(-ENAMETOOLONG); + + new_op = op_alloc(ORANGEFS_VFS_OP_LOOKUP); + if (!new_op) + return ERR_PTR(-ENOMEM); + + new_op->upcall.req.lookup.sym_follow = ORANGEFS_LOOKUP_LINK_NO_FOLLOW; + + gossip_debug(GOSSIP_NAME_DEBUG, "%s:%s:%d using parent %pU\n", + __FILE__, + __func__, + __LINE__, + &parent->refn.khandle); + new_op->upcall.req.lookup.parent_refn = parent->refn; + + strscpy(new_op->upcall.req.lookup.d_name, dentry->d_name.name); + + gossip_debug(GOSSIP_NAME_DEBUG, + "%s: doing lookup on %s under %pU,%d\n", + __func__, + new_op->upcall.req.lookup.d_name, + &new_op->upcall.req.lookup.parent_refn.khandle, + new_op->upcall.req.lookup.parent_refn.fs_id); + + ret = service_operation(new_op, __func__, get_interruptible_flag(dir)); + + gossip_debug(GOSSIP_NAME_DEBUG, + "Lookup Got %pU, fsid %d (ret=%d)\n", + &new_op->downcall.resp.lookup.refn.khandle, + new_op->downcall.resp.lookup.refn.fs_id, + ret); + + if (ret == 0) { + orangefs_set_timeout(dentry); + inode = orangefs_iget(dir->i_sb, &new_op->downcall.resp.lookup.refn); + } else if (ret == -ENOENT) { + inode = NULL; + } else { + /* must be a non-recoverable error */ + inode = ERR_PTR(ret); + } + + op_release(new_op); + return d_splice_alias(inode, dentry); +} + +/* return 0 on success; non-zero otherwise */ +static int orangefs_unlink(struct inode *dir, struct dentry *dentry) +{ + struct inode *inode = dentry->d_inode; + struct orangefs_inode_s *parent = ORANGEFS_I(dir); + struct orangefs_kernel_op_s *new_op; + struct iattr iattr; + int ret; + + gossip_debug(GOSSIP_NAME_DEBUG, + "%s: called on %pd\n" + " (inode %pU): Parent is %pU | fs_id %d\n", + __func__, + dentry, + get_khandle_from_ino(inode), + &parent->refn.khandle, + parent->refn.fs_id); + + new_op = op_alloc(ORANGEFS_VFS_OP_REMOVE); + if (!new_op) + return -ENOMEM; + + new_op->upcall.req.remove.parent_refn = parent->refn; + strscpy(new_op->upcall.req.remove.d_name, dentry->d_name.name); + + ret = service_operation(new_op, "orangefs_unlink", + get_interruptible_flag(inode)); + + gossip_debug(GOSSIP_NAME_DEBUG, + "%s: service_operation returned:%d:\n", + __func__, + ret); + + op_release(new_op); + + if (!ret) { + drop_nlink(inode); + + memset(&iattr, 0, sizeof iattr); + iattr.ia_valid |= ATTR_MTIME | ATTR_CTIME; + iattr.ia_mtime = iattr.ia_ctime = current_time(dir); + __orangefs_setattr(dir, &iattr); + } + return ret; +} + +static int orangefs_symlink(struct mnt_idmap *idmap, + struct inode *dir, + struct dentry *dentry, + const char *symname) +{ + struct orangefs_inode_s *parent = ORANGEFS_I(dir); + struct orangefs_kernel_op_s *new_op; + struct orangefs_object_kref ref; + struct inode *inode; + struct iattr iattr; + int mode = 0755; + int ret; + + gossip_debug(GOSSIP_NAME_DEBUG, "%s: called\n", __func__); + + if (!symname) + return -EINVAL; + + if (strlen(symname)+1 > ORANGEFS_NAME_MAX) + return -ENAMETOOLONG; + + new_op = op_alloc(ORANGEFS_VFS_OP_SYMLINK); + if (!new_op) + return -ENOMEM; + + new_op->upcall.req.sym.parent_refn = parent->refn; + + fill_default_sys_attrs(new_op->upcall.req.sym.attributes, mode); + + strscpy(new_op->upcall.req.sym.entry_name, dentry->d_name.name); + strscpy(new_op->upcall.req.sym.target, symname); + + ret = service_operation(new_op, __func__, get_interruptible_flag(dir)); + + gossip_debug(GOSSIP_NAME_DEBUG, + "Symlink Got ORANGEFS handle %pU on fsid %d (ret=%d)\n", + &new_op->downcall.resp.sym.refn.khandle, + new_op->downcall.resp.sym.refn.fs_id, ret); + + if (ret < 0) { + gossip_debug(GOSSIP_NAME_DEBUG, + "%s: failed with error code %d\n", + __func__, ret); + goto out; + } + + ref = new_op->downcall.resp.sym.refn; + + inode = orangefs_new_inode(dir->i_sb, dir, S_IFLNK | mode, 0, &ref); + if (IS_ERR(inode)) { + gossip_err + ("*** Failed to allocate orangefs symlink inode\n"); + ret = PTR_ERR(inode); + goto out; + } + /* + * This is necessary because orangefs_inode_getattr will not + * re-read symlink size as it is impossible for it to change. + * Invalidating the cache does not help. orangefs_new_inode + * does not set the correct size (it does not know symname). + */ + inode->i_size = strlen(symname); + + gossip_debug(GOSSIP_NAME_DEBUG, + "Assigned symlink inode new number of %pU\n", + get_khandle_from_ino(inode)); + + d_instantiate_new(dentry, inode); + orangefs_set_timeout(dentry); + + gossip_debug(GOSSIP_NAME_DEBUG, + "Inode (Symlink) %pU -> %pd\n", + get_khandle_from_ino(inode), + dentry); + + memset(&iattr, 0, sizeof iattr); + iattr.ia_valid |= ATTR_MTIME | ATTR_CTIME; + iattr.ia_mtime = iattr.ia_ctime = current_time(dir); + __orangefs_setattr(dir, &iattr); + ret = 0; +out: + op_release(new_op); + return ret; +} + +static struct dentry *orangefs_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct orangefs_inode_s *parent = ORANGEFS_I(dir); + struct orangefs_kernel_op_s *new_op; + struct orangefs_object_kref ref; + struct inode *inode; + struct iattr iattr; + int ret; + + new_op = op_alloc(ORANGEFS_VFS_OP_MKDIR); + if (!new_op) + return ERR_PTR(-ENOMEM); + + new_op->upcall.req.mkdir.parent_refn = parent->refn; + + fill_default_sys_attrs(new_op->upcall.req.mkdir.attributes, mode); + + strscpy(new_op->upcall.req.mkdir.d_name, dentry->d_name.name); + + ret = service_operation(new_op, __func__, get_interruptible_flag(dir)); + + gossip_debug(GOSSIP_NAME_DEBUG, + "Mkdir Got ORANGEFS handle %pU on fsid %d\n", + &new_op->downcall.resp.mkdir.refn.khandle, + new_op->downcall.resp.mkdir.refn.fs_id); + + if (ret < 0) { + gossip_debug(GOSSIP_NAME_DEBUG, + "%s: failed with error code %d\n", + __func__, ret); + goto out; + } + + ref = new_op->downcall.resp.mkdir.refn; + + inode = orangefs_new_inode(dir->i_sb, dir, mode, 0, &ref); + if (IS_ERR(inode)) { + gossip_err("*** Failed to allocate orangefs dir inode\n"); + ret = PTR_ERR(inode); + goto out; + } + + gossip_debug(GOSSIP_NAME_DEBUG, + "Assigned dir inode new number of %pU\n", + get_khandle_from_ino(inode)); + + d_instantiate_new(dentry, inode); + orangefs_set_timeout(dentry); + + gossip_debug(GOSSIP_NAME_DEBUG, + "Inode (Directory) %pU -> %pd\n", + get_khandle_from_ino(inode), + dentry); + + /* + * NOTE: we have no good way to keep nlink consistent for directories + * across clients; keep constant at 1. + */ + memset(&iattr, 0, sizeof iattr); + iattr.ia_valid |= ATTR_MTIME | ATTR_CTIME; + iattr.ia_mtime = iattr.ia_ctime = current_time(dir); + __orangefs_setattr(dir, &iattr); +out: + op_release(new_op); + return ret ? ERR_PTR(ret) : NULL; +} + +static int orangefs_rename(struct mnt_idmap *idmap, + struct inode *old_dir, + struct dentry *old_dentry, + struct inode *new_dir, + struct dentry *new_dentry, + unsigned int flags) +{ + struct orangefs_kernel_op_s *new_op; + struct iattr iattr; + int ret; + + if (flags) + return -EINVAL; + + gossip_debug(GOSSIP_NAME_DEBUG, + "orangefs_rename: called (%pd2 => %pd2) ct=%d\n", + old_dentry, new_dentry, d_count(new_dentry)); + + memset(&iattr, 0, sizeof iattr); + iattr.ia_valid |= ATTR_MTIME | ATTR_CTIME; + iattr.ia_mtime = iattr.ia_ctime = current_time(new_dir); + __orangefs_setattr(new_dir, &iattr); + + new_op = op_alloc(ORANGEFS_VFS_OP_RENAME); + if (!new_op) + return -EINVAL; + + new_op->upcall.req.rename.old_parent_refn = ORANGEFS_I(old_dir)->refn; + new_op->upcall.req.rename.new_parent_refn = ORANGEFS_I(new_dir)->refn; + + strscpy(new_op->upcall.req.rename.d_old_name, old_dentry->d_name.name); + strscpy(new_op->upcall.req.rename.d_new_name, new_dentry->d_name.name); + + ret = service_operation(new_op, + "orangefs_rename", + get_interruptible_flag(old_dentry->d_inode)); + + gossip_debug(GOSSIP_NAME_DEBUG, + "orangefs_rename: got downcall status %d\n", + ret); + + if (new_dentry->d_inode) + inode_set_ctime_current(d_inode(new_dentry)); + + op_release(new_op); + return ret; +} + +/* ORANGEFS implementation of VFS inode operations for directories */ +const struct inode_operations orangefs_dir_inode_operations = { + .lookup = orangefs_lookup, + .get_inode_acl = orangefs_get_acl, + .set_acl = orangefs_set_acl, + .create = orangefs_create, + .unlink = orangefs_unlink, + .symlink = orangefs_symlink, + .mkdir = orangefs_mkdir, + .rmdir = orangefs_unlink, + .rename = orangefs_rename, + .setattr = orangefs_setattr, + .getattr = orangefs_getattr, + .listxattr = orangefs_listxattr, + .permission = orangefs_permission, + .update_time = orangefs_update_time, +}; diff --git a/fs/orangefs/orangefs-bufmap.c b/fs/orangefs/orangefs-bufmap.c new file mode 100644 index 000000000..a557e1cf7 --- /dev/null +++ b/fs/orangefs/orangefs-bufmap.c @@ -0,0 +1,836 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * (C) 2001 Clemson University and The University of Chicago + * + * See COPYING in top-level directory. + */ +#include "protocol.h" +#include "orangefs-kernel.h" +#include "orangefs-bufmap.h" + +struct slot_map { + int c; + wait_queue_head_t q; + int count; + unsigned long *map; +}; + +static struct slot_map rw_map = { + .c = -1, + .q = __WAIT_QUEUE_HEAD_INITIALIZER(rw_map.q) +}; +static struct slot_map readdir_map = { + .c = -1, + .q = __WAIT_QUEUE_HEAD_INITIALIZER(readdir_map.q) +}; + + +static void install(struct slot_map *m, int count, unsigned long *map) +{ + spin_lock(&m->q.lock); + m->c = m->count = count; + m->map = map; + wake_up_all_locked(&m->q); + spin_unlock(&m->q.lock); +} + +static void mark_killed(struct slot_map *m) +{ + spin_lock(&m->q.lock); + m->c -= m->count + 1; + spin_unlock(&m->q.lock); +} + +static void run_down(struct slot_map *m) +{ + DEFINE_WAIT(wait); + spin_lock(&m->q.lock); + if (m->c != -1) { + for (;;) { + if (likely(list_empty(&wait.entry))) + __add_wait_queue_entry_tail(&m->q, &wait); + set_current_state(TASK_UNINTERRUPTIBLE); + + if (m->c == -1) + break; + + spin_unlock(&m->q.lock); + schedule(); + spin_lock(&m->q.lock); + } + __remove_wait_queue(&m->q, &wait); + __set_current_state(TASK_RUNNING); + } + m->map = NULL; + spin_unlock(&m->q.lock); +} + +static void put(struct slot_map *m, int slot) +{ + int v; + spin_lock(&m->q.lock); + __clear_bit(slot, m->map); + v = ++m->c; + if (v > 0) + wake_up_locked(&m->q); + if (unlikely(v == -1)) /* finished dying */ + wake_up_all_locked(&m->q); + spin_unlock(&m->q.lock); +} + +static int wait_for_free(struct slot_map *m) +{ + long left = slot_timeout_secs * HZ; + DEFINE_WAIT(wait); + + do { + long n = left, t; + if (likely(list_empty(&wait.entry))) + __add_wait_queue_entry_tail_exclusive(&m->q, &wait); + set_current_state(TASK_INTERRUPTIBLE); + + if (m->c > 0) + break; + + if (m->c < 0) { + /* we are waiting for map to be installed */ + /* it would better be there soon, or we go away */ + if (n > ORANGEFS_BUFMAP_WAIT_TIMEOUT_SECS * HZ) + n = ORANGEFS_BUFMAP_WAIT_TIMEOUT_SECS * HZ; + } + spin_unlock(&m->q.lock); + t = schedule_timeout(n); + spin_lock(&m->q.lock); + if (unlikely(!t) && n != left && m->c < 0) + left = t; + else + left = t + (left - n); + if (signal_pending(current)) + left = -EINTR; + } while (left > 0); + + if (!list_empty(&wait.entry)) + list_del(&wait.entry); + else if (left <= 0 && waitqueue_active(&m->q)) + __wake_up_locked_key(&m->q, TASK_INTERRUPTIBLE, NULL); + __set_current_state(TASK_RUNNING); + + if (likely(left > 0)) + return 0; + + return left < 0 ? -EINTR : -ETIMEDOUT; +} + +static int get(struct slot_map *m) +{ + int res = 0; + spin_lock(&m->q.lock); + if (unlikely(m->c <= 0)) + res = wait_for_free(m); + if (likely(!res)) { + m->c--; + res = find_first_zero_bit(m->map, m->count); + __set_bit(res, m->map); + } + spin_unlock(&m->q.lock); + return res; +} + +/* used to describe mapped buffers */ +struct orangefs_bufmap_desc { + void __user *uaddr; /* user space address pointer */ + struct folio **folio_array; + /* + * folio_offsets could be needed when userspace sets custom + * sizes in user_desc, or when folios aren't all backed by + * 2MB THPs. + */ + size_t *folio_offsets; + int folio_count; + bool is_two_2mib_chunks; +}; + +static struct orangefs_bufmap { + int desc_size; + int desc_shift; + int desc_count; + int total_size; + int page_count; + int folio_count; + + struct page **page_array; + struct folio **folio_array; + struct orangefs_bufmap_desc *desc_array; + + /* array to track usage of buffer descriptors */ + unsigned long *buffer_index_array; + + /* array to track usage of buffer descriptors for readdir */ +#define N DIV_ROUND_UP(ORANGEFS_READDIR_DEFAULT_DESC_COUNT, BITS_PER_LONG) + unsigned long readdir_index_array[N]; +#undef N +} *__orangefs_bufmap; + +static DEFINE_SPINLOCK(orangefs_bufmap_lock); + +static void +orangefs_bufmap_unmap(struct orangefs_bufmap *bufmap) +{ + unpin_user_pages(bufmap->page_array, bufmap->page_count); +} + +static void +orangefs_bufmap_free(struct orangefs_bufmap *bufmap) +{ + int i; + + if (!bufmap) + return; + + for (i = 0; i < bufmap->desc_count; i++) { + kfree(bufmap->desc_array[i].folio_array); + kfree(bufmap->desc_array[i].folio_offsets); + bufmap->desc_array[i].folio_array = NULL; + bufmap->desc_array[i].folio_offsets = NULL; + } + kfree(bufmap->page_array); + kfree(bufmap->desc_array); + bitmap_free(bufmap->buffer_index_array); + kfree(bufmap); +} + +/* + * XXX: Can the size and shift change while the caller gives up the + * XXX: lock between calling this and doing something useful? + */ + +int orangefs_bufmap_size_query(void) +{ + struct orangefs_bufmap *bufmap; + int size = 0; + spin_lock(&orangefs_bufmap_lock); + bufmap = __orangefs_bufmap; + if (bufmap) + size = bufmap->desc_size; + spin_unlock(&orangefs_bufmap_lock); + return size; +} + +static DECLARE_WAIT_QUEUE_HEAD(bufmap_waitq); +static DECLARE_WAIT_QUEUE_HEAD(readdir_waitq); + +static struct orangefs_bufmap * +orangefs_bufmap_alloc(struct ORANGEFS_dev_map_desc *user_desc) +{ + struct orangefs_bufmap *bufmap; + + bufmap = kzalloc_obj(*bufmap); + if (!bufmap) + goto out; + + bufmap->total_size = user_desc->total_size; + bufmap->desc_count = user_desc->count; + bufmap->desc_size = user_desc->size; + bufmap->desc_shift = ilog2(bufmap->desc_size); + bufmap->page_count = bufmap->total_size / PAGE_SIZE; + + bufmap->buffer_index_array = + bitmap_zalloc(bufmap->desc_count, GFP_KERNEL); + if (!bufmap->buffer_index_array) + goto out_free_bufmap; + + bufmap->desc_array = + kzalloc_objs(struct orangefs_bufmap_desc, bufmap->desc_count); + if (!bufmap->desc_array) + goto out_free_index_array; + + /* allocate storage to track our page mappings */ + bufmap->page_array = + kzalloc_objs(struct page *, bufmap->page_count); + if (!bufmap->page_array) + goto out_free_desc_array; + + /* allocate folio array. */ + bufmap->folio_array = kzalloc_objs(struct folio *, bufmap->page_count); + if (!bufmap->folio_array) + goto out_free_page_array; + + return bufmap; + +out_free_page_array: + kfree(bufmap->page_array); +out_free_desc_array: + kfree(bufmap->desc_array); +out_free_index_array: + bitmap_free(bufmap->buffer_index_array); +out_free_bufmap: + kfree(bufmap); +out: + return NULL; +} + +static int orangefs_bufmap_group_folios(struct orangefs_bufmap *bufmap) +{ + int i = 0; + int f = 0; + int k; + int num_pages; + struct page *page; + struct folio *folio; + + while (i < bufmap->page_count) { + page = bufmap->page_array[i]; + folio = page_folio(page); + num_pages = folio_nr_pages(folio); + gossip_debug(GOSSIP_BUFMAP_DEBUG, + "%s: i:%d: num_pages:%d: \n", __func__, i, num_pages); + + for (k = 1; k < num_pages; k++) { + if (bufmap->page_array[i + k] != folio_page(folio, k)) { + gossip_err("%s: bad match, i:%d: k:%d:\n", + __func__, i, k); + return -EINVAL; + } + } + + bufmap->folio_array[f++] = folio; + i += num_pages; + } + + bufmap->folio_count = f; + pr_info("%s: Grouped %d folios from %d pages.\n", + __func__, + bufmap->folio_count, + bufmap->page_count); + return 0; +} + +static int orangefs_bufmap_map(struct orangefs_bufmap *bufmap, + struct ORANGEFS_dev_map_desc *user_desc) +{ + int pages_per_desc = bufmap->desc_size / PAGE_SIZE; + int ret; + int i; + int j; + int current_folio; + int desc_pages_needed; + int desc_folio_count; + int remaining_pages; + int need_avail_min; + int pages_assigned_to_this_desc; + int allocated_descs = 0; + size_t current_offset; + size_t adjust_offset; + struct folio *folio; + + /* map the pages */ + ret = pin_user_pages_fast((unsigned long)user_desc->ptr, + bufmap->page_count, + FOLL_WRITE, + bufmap->page_array); + + if (ret < 0) + return ret; + + if (ret != bufmap->page_count) { + gossip_err("orangefs error: asked for %d pages, only got %d.\n", + bufmap->page_count, ret); + for (i = 0; i < ret; i++) + unpin_user_page(bufmap->page_array[i]); + return -ENOMEM; + } + + /* + * ideally we want to get kernel space pointers for each page, but + * we can't kmap that many pages at once if highmem is being used. + * so instead, we just kmap/kunmap the page address each time the + * kaddr is needed. + */ + for (i = 0; i < bufmap->page_count; i++) + flush_dcache_page(bufmap->page_array[i]); + + /* + * Group pages into folios. + */ + ret = orangefs_bufmap_group_folios(bufmap); + if (ret) + goto unpin; + + pr_info("%s: desc_size=%d bytes (%d pages per desc), total folios=%d\n", + __func__, bufmap->desc_size, pages_per_desc, + bufmap->folio_count); + + current_folio = 0; + remaining_pages = 0; + current_offset = 0; + for (i = 0; i < bufmap->desc_count; i++) { + desc_pages_needed = pages_per_desc; + desc_folio_count = 0; + pages_assigned_to_this_desc = 0; + bufmap->desc_array[i].is_two_2mib_chunks = false; + + /* + * We hope there was enough memory that each desc is + * covered by two THPs/folios, if not we want to keep on + * working even if there's only one page per folio. + */ + bufmap->desc_array[i].folio_array = + kzalloc_objs(struct folio *, pages_per_desc); + if (!bufmap->desc_array[i].folio_array) { + ret = -ENOMEM; + goto unpin; + } + + bufmap->desc_array[i].folio_offsets = + kzalloc_objs(size_t, pages_per_desc); + if (!bufmap->desc_array[i].folio_offsets) { + ret = -ENOMEM; + kfree(bufmap->desc_array[i].folio_array); + bufmap->desc_array[i].folio_array = NULL; + goto unpin; + } + + bufmap->desc_array[i].uaddr = + user_desc->ptr + (size_t)i * bufmap->desc_size; + + /* + * Accumulate folios until desc is full. + */ + while (desc_pages_needed > 0) { + if (remaining_pages == 0) { + /* shouldn't happen. */ + if (current_folio >= bufmap->folio_count) { + ret = -EINVAL; + goto unpin; + } + folio = bufmap->folio_array[current_folio++]; + remaining_pages = folio_nr_pages(folio); + current_offset = 0; + } else { + folio = bufmap->folio_array[current_folio - 1]; + } + + need_avail_min = + min(desc_pages_needed, remaining_pages); + adjust_offset = need_avail_min * PAGE_SIZE; + + bufmap->desc_array[i].folio_array[desc_folio_count] = + folio; + bufmap->desc_array[i].folio_offsets[desc_folio_count] = + current_offset; + desc_folio_count++; + pages_assigned_to_this_desc += need_avail_min; + desc_pages_needed -= need_avail_min; + remaining_pages -= need_avail_min; + current_offset += adjust_offset; + } + + /* Detect optimal case: two 2MiB folios per 4MiB slot. */ + if (desc_folio_count == 2 && + folio_nr_pages(bufmap->desc_array[i].folio_array[0]) == 512 && + folio_nr_pages(bufmap->desc_array[i].folio_array[1]) == 512) { + bufmap->desc_array[i].is_two_2mib_chunks = true; + gossip_debug(GOSSIP_BUFMAP_DEBUG, "%s: descriptor :%d: " + "optimal folio/page ratio.\n", __func__, i); + } + + bufmap->desc_array[i].folio_count = desc_folio_count; + gossip_debug(GOSSIP_BUFMAP_DEBUG, + " descriptor %d: folio_count=%d, " + "pages_assigned=%d (should be %d)\n", + i, desc_folio_count, pages_assigned_to_this_desc, + pages_per_desc); + + allocated_descs = i + 1; + } + + return 0; +unpin: + /* + * rollback any allocations we got so far... + * Memory pressure, like in generic/340, led me + * to write the rollback this way. + */ + for (j = 0; j < allocated_descs; j++) { + if (bufmap->desc_array[j].folio_array) { + kfree(bufmap->desc_array[j].folio_array); + bufmap->desc_array[j].folio_array = NULL; + } + if (bufmap->desc_array[j].folio_offsets) { + kfree(bufmap->desc_array[j].folio_offsets); + bufmap->desc_array[j].folio_offsets = NULL; + } + } + unpin_user_pages(bufmap->page_array, bufmap->page_count); + return ret; +} + +/* + * orangefs_bufmap_initialize() + * + * initializes the mapped buffer interface + * + * user_desc is the parameters provided by userspace for the bufmap. + * + * returns 0 on success, -errno on failure + */ +int orangefs_bufmap_initialize(struct ORANGEFS_dev_map_desc *user_desc) +{ + struct orangefs_bufmap *bufmap; + int ret = -EINVAL; + + gossip_debug(GOSSIP_BUFMAP_DEBUG, + "%s: called (ptr (" "%p) sz (%d) cnt(%d).\n", + __func__, + user_desc->ptr, + user_desc->size, + user_desc->count); + + if (user_desc->total_size < 0 || + user_desc->size < 0 || + user_desc->count < 0) + goto out; + + /* + * sanity check alignment and size of buffer that caller wants to + * work with + */ + if (PAGE_ALIGN((unsigned long)user_desc->ptr) != + (unsigned long)user_desc->ptr) { + gossip_err("orangefs error: memory alignment (front). %p\n", + user_desc->ptr); + goto out; + } + + if (PAGE_ALIGN(((unsigned long)user_desc->ptr + user_desc->total_size)) + != (unsigned long)(user_desc->ptr + user_desc->total_size)) { + gossip_err("orangefs error: memory alignment (back).(%p + %d)\n", + user_desc->ptr, + user_desc->total_size); + goto out; + } + + if (user_desc->total_size != (user_desc->size * user_desc->count)) { + gossip_err("orangefs error: user provided an oddly sized buffer: (%d, %d, %d)\n", + user_desc->total_size, + user_desc->size, + user_desc->count); + goto out; + } + + if ((user_desc->size % PAGE_SIZE) != 0) { + gossip_err("orangefs error: bufmap size not page size divisible (%d).\n", + user_desc->size); + goto out; + } + + ret = -ENOMEM; + bufmap = orangefs_bufmap_alloc(user_desc); + if (!bufmap) + goto out; + + ret = orangefs_bufmap_map(bufmap, user_desc); + if (ret) + goto out_free_bufmap; + + + spin_lock(&orangefs_bufmap_lock); + if (__orangefs_bufmap) { + spin_unlock(&orangefs_bufmap_lock); + gossip_err("orangefs: error: bufmap already initialized.\n"); + ret = -EINVAL; + goto out_unmap_bufmap; + } + __orangefs_bufmap = bufmap; + install(&rw_map, + bufmap->desc_count, + bufmap->buffer_index_array); + install(&readdir_map, + ORANGEFS_READDIR_DEFAULT_DESC_COUNT, + bufmap->readdir_index_array); + spin_unlock(&orangefs_bufmap_lock); + + gossip_debug(GOSSIP_BUFMAP_DEBUG, + "%s: exiting normally\n", __func__); + return 0; + +out_unmap_bufmap: + orangefs_bufmap_unmap(bufmap); +out_free_bufmap: + orangefs_bufmap_free(bufmap); +out: + return ret; +} + +/* + * orangefs_bufmap_finalize() + * + * shuts down the mapped buffer interface and releases any resources + * associated with it + * + * no return value + */ +void orangefs_bufmap_finalize(void) +{ + struct orangefs_bufmap *bufmap = __orangefs_bufmap; + if (!bufmap) + return; + gossip_debug(GOSSIP_BUFMAP_DEBUG, "orangefs_bufmap_finalize: called\n"); + mark_killed(&rw_map); + mark_killed(&readdir_map); + gossip_debug(GOSSIP_BUFMAP_DEBUG, + "orangefs_bufmap_finalize: exiting normally\n"); +} + +void orangefs_bufmap_run_down(void) +{ + struct orangefs_bufmap *bufmap = __orangefs_bufmap; + if (!bufmap) + return; + run_down(&rw_map); + run_down(&readdir_map); + spin_lock(&orangefs_bufmap_lock); + __orangefs_bufmap = NULL; + spin_unlock(&orangefs_bufmap_lock); + orangefs_bufmap_unmap(bufmap); + orangefs_bufmap_free(bufmap); +} + +/* + * orangefs_bufmap_get() + * + * gets a free mapped buffer descriptor, will sleep until one becomes + * available if necessary + * + * returns slot on success, -errno on failure + */ +int orangefs_bufmap_get(void) +{ + return get(&rw_map); +} + +/* + * orangefs_bufmap_put() + * + * returns a mapped buffer descriptor to the collection + * + * no return value + */ +void orangefs_bufmap_put(int buffer_index) +{ + put(&rw_map, buffer_index); +} + +/* + * orangefs_readdir_index_get() + * + * gets a free descriptor, will sleep until one becomes + * available if necessary. + * Although the readdir buffers are not mapped into kernel space + * we could do that at a later point of time. Regardless, these + * indices are used by the client-core. + * + * returns slot on success, -errno on failure + */ +int orangefs_readdir_index_get(void) +{ + return get(&readdir_map); +} + +void orangefs_readdir_index_put(int buffer_index) +{ + put(&readdir_map, buffer_index); +} + +/* + * we've been handed an iovec, we need to copy it to + * the shared memory descriptor at "buffer_index". + */ +int orangefs_bufmap_copy_from_iovec(struct iov_iter *iter, + int buffer_index, + size_t size) +{ + struct orangefs_bufmap_desc *to; + size_t remaining = size; + int folio_index = 0; + struct folio *folio; + size_t folio_offset; + size_t folio_avail; + size_t copy_amount; + size_t copied; + void *kaddr; + size_t half; + size_t first; + size_t second; + + to = &__orangefs_bufmap->desc_array[buffer_index]; + + /* shouldn't happen... */ + if (size > 4194304) + pr_info("%s: size:%zu\n", __func__, size); + + gossip_debug(GOSSIP_BUFMAP_DEBUG, + "%s: buffer_index:%d size:%zu folio_count:%d\n", + __func__, + buffer_index, + size, + to->folio_count); + + /* Fast path: exactly two 2 MiB folios */ + if (to->is_two_2mib_chunks && size <= 4194304) { + gossip_debug(GOSSIP_BUFMAP_DEBUG, + "%s: fastpath hit.\n", __func__); + half = 2097152; /* 2 MiB */ + first = min(size, half); + second = (size > half) ? size - half : 0; + + /* First 2 MiB chunk */ + kaddr = kmap_local_folio(to->folio_array[0], 0); + copied = copy_from_iter(kaddr, first, iter); + kunmap_local(kaddr); + if (copied != first) + return -EFAULT; + + if (second == 0) + return 0; + + /* Second 2 MiB chunk */ + kaddr = kmap_local_folio(to->folio_array[1], 0); + copied = copy_from_iter(kaddr, second, iter); + kunmap_local(kaddr); + if (copied != second) + return -EFAULT; + + return 0; + } + + while (remaining > 0) { + + if (unlikely(folio_index >= to->folio_count || + to->folio_array[folio_index] == NULL)) { + gossip_err("%s: " + "folio_index:%d: >= folio_count:%d: " + "(size %zu, buffer %d)\n", + __func__, + folio_index, + to->folio_count, + size, + buffer_index); + return -EFAULT; + } + + folio = to->folio_array[folio_index]; + folio_offset = to->folio_offsets[folio_index]; + folio_avail = folio_nr_pages(folio) * PAGE_SIZE - folio_offset; + copy_amount = min(remaining, folio_avail); + kaddr = kmap_local_folio(folio, folio_offset); + copied = copy_from_iter(kaddr, copy_amount, iter); + kunmap_local(kaddr); + + if (copied != copy_amount) + return -EFAULT; + + remaining -= copied; + folio_index++; + } + + return 0; +} + +/* + * we've been handed an iovec, we need to fill it from + * the shared memory descriptor at "buffer_index". + */ +int orangefs_bufmap_copy_to_iovec(struct iov_iter *iter, + int buffer_index, + size_t size) +{ + struct orangefs_bufmap_desc *from; + size_t remaining = size; + int folio_index = 0; + struct folio *folio; + size_t folio_offset; + size_t folio_avail; + size_t copy_amount; + size_t copied; + void *kaddr; + size_t half; + size_t first; + size_t second; + + from = &__orangefs_bufmap->desc_array[buffer_index]; + + /* shouldn't happen... */ + if (size > 4194304) + pr_info("%s: size:%zu\n", __func__, size); + + gossip_debug(GOSSIP_BUFMAP_DEBUG, + "%s: buffer_index:%d size:%zu folio_count:%d\n", + __func__, + buffer_index, + size, + from->folio_count); + + /* Fast path: exactly two 2 MiB folios */ + if (from->is_two_2mib_chunks && size <= 4194304) { + gossip_debug(GOSSIP_BUFMAP_DEBUG, + "%s: fastpath hit.\n", __func__); + half = 2097152; /* 2 MiB */ + first = min(size, half); + second = (size > half) ? size - half : 0; + void *kaddr; + size_t copied; + + /* First 2 MiB chunk */ + kaddr = kmap_local_folio(from->folio_array[0], 0); + copied = copy_to_iter(kaddr, first, iter); + kunmap_local(kaddr); + if (copied != first) + return -EFAULT; + + if (second == 0) + return 0; + + /* Second 2 MiB chunk */ + kaddr = kmap_local_folio(from->folio_array[1], 0); + copied = copy_to_iter(kaddr, second, iter); + kunmap_local(kaddr); + if (copied != second) + return -EFAULT; + + return 0; + } + + while (remaining > 0) { + + if (unlikely(folio_index >= from->folio_count || + from->folio_array[folio_index] == NULL)) { + gossip_err("%s: " + "folio_index:%d: >= folio_count:%d: " + "(size %zu, buffer %d)\n", + __func__, + folio_index, + from->folio_count, + size, + buffer_index); + return -EFAULT; + } + + folio = from->folio_array[folio_index]; + folio_offset = from->folio_offsets[folio_index]; + folio_avail = folio_nr_pages(folio) * PAGE_SIZE - folio_offset; + copy_amount = min(remaining, folio_avail); + + kaddr = kmap_local_folio(folio, folio_offset); + copied = copy_to_iter(kaddr, copy_amount, iter); + kunmap_local(kaddr); + + if (copied != copy_amount) + return -EFAULT; + + remaining -= copied; + folio_index++; + } + + return 0; +} diff --git a/fs/orangefs/orangefs-bufmap.h b/fs/orangefs/orangefs-bufmap.h new file mode 100644 index 000000000..4231175cc --- /dev/null +++ b/fs/orangefs/orangefs-bufmap.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * (C) 2001 Clemson University and The University of Chicago + * + * See COPYING in top-level directory. + */ + +#ifndef __ORANGEFS_BUFMAP_H +#define __ORANGEFS_BUFMAP_H + +int orangefs_bufmap_size_query(void); + +int orangefs_bufmap_initialize(struct ORANGEFS_dev_map_desc *user_desc); + +void orangefs_bufmap_finalize(void); + +void orangefs_bufmap_run_down(void); + +int orangefs_bufmap_get(void); + +void orangefs_bufmap_put(int buffer_index); + +int orangefs_readdir_index_get(void); + +void orangefs_readdir_index_put(int buffer_index); + +int orangefs_bufmap_copy_from_iovec(struct iov_iter *iter, + int buffer_index, + size_t size); + +int orangefs_bufmap_copy_to_iovec(struct iov_iter *iter, + int buffer_index, + size_t size); + + +#endif /* __ORANGEFS_BUFMAP_H */ diff --git a/fs/orangefs/orangefs-cache.c b/fs/orangefs/orangefs-cache.c new file mode 100644 index 000000000..0bdb99e89 --- /dev/null +++ b/fs/orangefs/orangefs-cache.c @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * (C) 2001 Clemson University and The University of Chicago + * + * See COPYING in top-level directory. + */ + +#include "protocol.h" +#include "orangefs-kernel.h" + +/* tags assigned to kernel upcall operations */ +static __u64 next_tag_value; +static DEFINE_SPINLOCK(next_tag_value_lock); + +/* the orangefs memory caches */ + +/* a cache for orangefs upcall/downcall operations */ +static struct kmem_cache *op_cache; + +int op_cache_initialize(void) +{ + op_cache = kmem_cache_create_usercopy("orangefs_op_cache", + sizeof(struct orangefs_kernel_op_s), + 0, + 0, + offsetof(struct orangefs_kernel_op_s, tag), + offsetof(struct orangefs_kernel_op_s, upcall) + + sizeof(struct orangefs_upcall_s) - + offsetof(struct orangefs_kernel_op_s, tag), + NULL); + + if (!op_cache) { + gossip_err("Cannot create orangefs_op_cache\n"); + return -ENOMEM; + } + + /* initialize our atomic tag counter */ + spin_lock(&next_tag_value_lock); + next_tag_value = 100; + spin_unlock(&next_tag_value_lock); + return 0; +} + +int op_cache_finalize(void) +{ + kmem_cache_destroy(op_cache); + return 0; +} + +char *get_opname_string(struct orangefs_kernel_op_s *new_op) +{ + if (new_op) { + __s32 type = new_op->upcall.type; + + if (type == ORANGEFS_VFS_OP_FILE_IO) + return "OP_FILE_IO"; + else if (type == ORANGEFS_VFS_OP_LOOKUP) + return "OP_LOOKUP"; + else if (type == ORANGEFS_VFS_OP_CREATE) + return "OP_CREATE"; + else if (type == ORANGEFS_VFS_OP_GETATTR) + return "OP_GETATTR"; + else if (type == ORANGEFS_VFS_OP_REMOVE) + return "OP_REMOVE"; + else if (type == ORANGEFS_VFS_OP_MKDIR) + return "OP_MKDIR"; + else if (type == ORANGEFS_VFS_OP_READDIR) + return "OP_READDIR"; + else if (type == ORANGEFS_VFS_OP_READDIRPLUS) + return "OP_READDIRPLUS"; + else if (type == ORANGEFS_VFS_OP_SETATTR) + return "OP_SETATTR"; + else if (type == ORANGEFS_VFS_OP_SYMLINK) + return "OP_SYMLINK"; + else if (type == ORANGEFS_VFS_OP_RENAME) + return "OP_RENAME"; + else if (type == ORANGEFS_VFS_OP_STATFS) + return "OP_STATFS"; + else if (type == ORANGEFS_VFS_OP_TRUNCATE) + return "OP_TRUNCATE"; + else if (type == ORANGEFS_VFS_OP_RA_FLUSH) + return "OP_RA_FLUSH"; + else if (type == ORANGEFS_VFS_OP_FS_MOUNT) + return "OP_FS_MOUNT"; + else if (type == ORANGEFS_VFS_OP_FS_UMOUNT) + return "OP_FS_UMOUNT"; + else if (type == ORANGEFS_VFS_OP_GETXATTR) + return "OP_GETXATTR"; + else if (type == ORANGEFS_VFS_OP_SETXATTR) + return "OP_SETXATTR"; + else if (type == ORANGEFS_VFS_OP_LISTXATTR) + return "OP_LISTXATTR"; + else if (type == ORANGEFS_VFS_OP_REMOVEXATTR) + return "OP_REMOVEXATTR"; + else if (type == ORANGEFS_VFS_OP_PARAM) + return "OP_PARAM"; + else if (type == ORANGEFS_VFS_OP_PERF_COUNT) + return "OP_PERF_COUNT"; + else if (type == ORANGEFS_VFS_OP_CANCEL) + return "OP_CANCEL"; + else if (type == ORANGEFS_VFS_OP_FSYNC) + return "OP_FSYNC"; + else if (type == ORANGEFS_VFS_OP_FSKEY) + return "OP_FSKEY"; + else if (type == ORANGEFS_VFS_OP_FEATURES) + return "OP_FEATURES"; + } + return "OP_UNKNOWN?"; +} + +void orangefs_new_tag(struct orangefs_kernel_op_s *op) +{ + spin_lock(&next_tag_value_lock); + op->tag = next_tag_value++; + if (next_tag_value == 0) + next_tag_value = 100; + spin_unlock(&next_tag_value_lock); +} + +struct orangefs_kernel_op_s *op_alloc(__s32 type) +{ + struct orangefs_kernel_op_s *new_op = NULL; + + new_op = kmem_cache_zalloc(op_cache, GFP_KERNEL); + if (new_op) { + INIT_LIST_HEAD(&new_op->list); + spin_lock_init(&new_op->lock); + init_completion(&new_op->waitq); + + new_op->upcall.type = ORANGEFS_VFS_OP_INVALID; + new_op->downcall.type = ORANGEFS_VFS_OP_INVALID; + new_op->downcall.status = -1; + + new_op->op_state = OP_VFS_STATE_UNKNOWN; + + /* initialize the op specific tag and upcall credentials */ + orangefs_new_tag(new_op); + new_op->upcall.type = type; + new_op->attempts = 0; + gossip_debug(GOSSIP_CACHE_DEBUG, + "Alloced OP (%p: %llu %s)\n", + new_op, + llu(new_op->tag), + get_opname_string(new_op)); + + new_op->upcall.uid = from_kuid(&init_user_ns, + current_fsuid()); + + new_op->upcall.gid = from_kgid(&init_user_ns, + current_fsgid()); + } else { + gossip_err("op_alloc: kmem_cache_zalloc failed!\n"); + } + return new_op; +} + +void op_release(struct orangefs_kernel_op_s *orangefs_op) +{ + if (orangefs_op) { + gossip_debug(GOSSIP_CACHE_DEBUG, + "Releasing OP (%p: %llu)\n", + orangefs_op, + llu(orangefs_op->tag)); + kmem_cache_free(op_cache, orangefs_op); + } else { + gossip_err("NULL pointer in op_release\n"); + } +} diff --git a/fs/orangefs/orangefs-debug.h b/fs/orangefs/orangefs-debug.h new file mode 100644 index 000000000..d4463534c --- /dev/null +++ b/fs/orangefs/orangefs-debug.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * (C) 2001 Clemson University and The University of Chicago + * + * See COPYING in top-level directory. + */ + +/* This file just defines debugging masks to be used with the gossip + * logging utility. All debugging masks for ORANGEFS are kept here to make + * sure we don't have collisions. + */ + +#ifndef __ORANGEFS_DEBUG_H +#define __ORANGEFS_DEBUG_H + +#ifdef __KERNEL__ +#include +#include +#else +#include +#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) +#endif + +#define GOSSIP_NO_DEBUG (__u64)0 + +#define GOSSIP_SUPER_DEBUG ((__u64)1 << 0) +#define GOSSIP_INODE_DEBUG ((__u64)1 << 1) +#define GOSSIP_FILE_DEBUG ((__u64)1 << 2) +#define GOSSIP_DIR_DEBUG ((__u64)1 << 3) +#define GOSSIP_UTILS_DEBUG ((__u64)1 << 4) +#define GOSSIP_WAIT_DEBUG ((__u64)1 << 5) +#define GOSSIP_ACL_DEBUG ((__u64)1 << 6) +#define GOSSIP_DCACHE_DEBUG ((__u64)1 << 7) +#define GOSSIP_DEV_DEBUG ((__u64)1 << 8) +#define GOSSIP_NAME_DEBUG ((__u64)1 << 9) +#define GOSSIP_BUFMAP_DEBUG ((__u64)1 << 10) +#define GOSSIP_CACHE_DEBUG ((__u64)1 << 11) +#define GOSSIP_DEBUGFS_DEBUG ((__u64)1 << 12) +#define GOSSIP_XATTR_DEBUG ((__u64)1 << 13) +#define GOSSIP_INIT_DEBUG ((__u64)1 << 14) +#define GOSSIP_SYSFS_DEBUG ((__u64)1 << 15) + +#define GOSSIP_MAX_NR 16 +#define GOSSIP_MAX_DEBUG (((__u64)1 << GOSSIP_MAX_NR) - 1) + +#endif /* __ORANGEFS_DEBUG_H */ diff --git a/fs/orangefs/orangefs-debugfs.c b/fs/orangefs/orangefs-debugfs.c new file mode 100644 index 000000000..9f94919a6 --- /dev/null +++ b/fs/orangefs/orangefs-debugfs.c @@ -0,0 +1,1050 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * What: /sys/kernel/debug/orangefs/debug-help + * Date: June 2015 + * Contact: Mike Marshall + * Description: + * List of client and kernel debug keywords. + * + * + * What: /sys/kernel/debug/orangefs/client-debug + * Date: June 2015 + * Contact: Mike Marshall + * Description: + * Debug setting for "the client", the userspace + * helper for the kernel module. + * + * + * What: /sys/kernel/debug/orangefs/kernel-debug + * Date: June 2015 + * Contact: Mike Marshall + * Description: + * Debug setting for the orangefs kernel module. + * + * Any of the keywords, or comma-separated lists + * of keywords, from debug-help can be catted to + * client-debug or kernel-debug. + * + * "none", "all" and "verbose" are special keywords + * for client-debug. Setting client-debug to "all" + * is kind of like trying to drink water from a + * fire hose, "verbose" triggers most of the same + * output except for the constant flow of output + * from the main wait loop. + * + * "none" and "all" are similar settings for kernel-debug + * no need for a "verbose". + */ +#include +#include + +#include + +#include "orangefs-debugfs.h" +#include "protocol.h" +#include "orangefs-kernel.h" + +/* a private internal type */ +struct __keyword_mask_s { + const char *keyword; + __u64 mask_val; +}; + +/* + * Map all kmod keywords to kmod debug masks here. Keep this + * structure "packed": + * + * "all" is always last... + * + * keyword mask_val index + * foo 1 0 + * bar 2 1 + * baz 4 2 + * qux 8 3 + * . . . + */ +static struct __keyword_mask_s s_kmod_keyword_mask_map[] = { + {"super", GOSSIP_SUPER_DEBUG}, + {"inode", GOSSIP_INODE_DEBUG}, + {"file", GOSSIP_FILE_DEBUG}, + {"dir", GOSSIP_DIR_DEBUG}, + {"utils", GOSSIP_UTILS_DEBUG}, + {"wait", GOSSIP_WAIT_DEBUG}, + {"acl", GOSSIP_ACL_DEBUG}, + {"dcache", GOSSIP_DCACHE_DEBUG}, + {"dev", GOSSIP_DEV_DEBUG}, + {"name", GOSSIP_NAME_DEBUG}, + {"bufmap", GOSSIP_BUFMAP_DEBUG}, + {"cache", GOSSIP_CACHE_DEBUG}, + {"debugfs", GOSSIP_DEBUGFS_DEBUG}, + {"xattr", GOSSIP_XATTR_DEBUG}, + {"init", GOSSIP_INIT_DEBUG}, + {"sysfs", GOSSIP_SYSFS_DEBUG}, + {"none", GOSSIP_NO_DEBUG}, + {"all", GOSSIP_MAX_DEBUG} +}; + +static const int num_kmod_keyword_mask_map = (int) + (ARRAY_SIZE(s_kmod_keyword_mask_map)); + +#define DEBUG_HELP_STRING_SIZE 4096 +#define HELP_STRING_UNINITIALIZED \ + "Client Debug Keywords are unknown until the first time\n" \ + "the client is started after boot.\n" +#define ORANGEFS_KMOD_DEBUG_HELP_FILE "debug-help" +#define ORANGEFS_KMOD_DEBUG_FILE "kernel-debug" +#define ORANGEFS_CLIENT_DEBUG_FILE "client-debug" +#define ORANGEFS_VERBOSE "verbose" +#define ORANGEFS_ALL "all" + +/* + * An array of client_debug_mask will be built to hold debug keyword/mask + * values fetched from userspace. + */ +struct client_debug_mask { + char *keyword; + __u64 mask1; + __u64 mask2; +}; + +static void orangefs_kernel_debug_init(void); + +static int orangefs_debug_help_open(struct inode *, struct file *); +static void *help_start(struct seq_file *, loff_t *); +static void *help_next(struct seq_file *, void *, loff_t *); +static void help_stop(struct seq_file *, void *); +static int help_show(struct seq_file *, void *); + +static int orangefs_debug_open(struct inode *, struct file *); + +static ssize_t orangefs_debug_read(struct file *, + char __user *, + size_t, + loff_t *); + +static ssize_t orangefs_debug_write(struct file *, + const char __user *, + size_t, + loff_t *); + +static int orangefs_prepare_cdm_array(char *); +static void debug_mask_to_string(void *, int); +static void do_k_string(void *, int); +static void do_c_string(void *, int); +static int keyword_is_amalgam(char *); +static int check_amalgam_keyword(void *, int); +static void debug_string_to_mask(char *, void *, int); +static void do_c_mask(int, char *, struct client_debug_mask **); +static void do_k_mask(int, char *, __u64 **); + +static char kernel_debug_string[ORANGEFS_MAX_DEBUG_STRING_LEN] = "none"; +static char *debug_help_string; +static char client_debug_string[ORANGEFS_MAX_DEBUG_STRING_LEN]; +static char client_debug_array_string[ORANGEFS_MAX_DEBUG_STRING_LEN]; + +static struct dentry *client_debug_dentry; +static struct dentry *debug_dir; + +static unsigned int kernel_mask_set_mod_init; +static int orangefs_debug_disabled = 1; +static int help_string_initialized; + +static const struct seq_operations help_debug_ops = { + .start = help_start, + .next = help_next, + .stop = help_stop, + .show = help_show, +}; + +static const struct file_operations debug_help_fops = { + .owner = THIS_MODULE, + .open = orangefs_debug_help_open, + .read = seq_read, + .release = seq_release, + .llseek = seq_lseek, +}; + +static const struct file_operations kernel_debug_fops = { + .owner = THIS_MODULE, + .open = orangefs_debug_open, + .read = orangefs_debug_read, + .write = orangefs_debug_write, + .llseek = generic_file_llseek, +}; + +static int client_all_index; +static int client_verbose_index; + +static struct client_debug_mask *cdm_array; +static int cdm_element_count; + +static struct client_debug_mask client_debug_mask; + +/* + * Used to protect data in ORANGEFS_KMOD_DEBUG_FILE and + * ORANGEFS_KMOD_DEBUG_FILE. + */ +static DEFINE_MUTEX(orangefs_debug_lock); + +/* Used to protect data in ORANGEFS_KMOD_DEBUG_HELP_FILE */ +static DEFINE_MUTEX(orangefs_help_file_lock); + +/* + * initialize kmod debug operations, create orangefs debugfs dir and + * ORANGEFS_KMOD_DEBUG_HELP_FILE. + */ +void orangefs_debugfs_init(int debug_mask) +{ + /* convert input debug mask to a 64-bit unsigned integer */ + orangefs_gossip_debug_mask = (unsigned long long)debug_mask; + + /* + * set the kernel's gossip debug string; invalid mask values will + * be ignored. + */ + debug_mask_to_string(&orangefs_gossip_debug_mask, 0); + + /* remove any invalid values from the mask */ + debug_string_to_mask(kernel_debug_string, &orangefs_gossip_debug_mask, + 0); + + /* + * if the mask has a non-zero value, then indicate that the mask + * was set when the kernel module was loaded. The orangefs dev ioctl + * command will look at this boolean to determine if the kernel's + * debug mask should be overwritten when the client-core is started. + */ + if (orangefs_gossip_debug_mask != 0) + kernel_mask_set_mod_init = true; + + pr_info("%s: called with debug mask: :%s: :%llx:\n", + __func__, + kernel_debug_string, + (unsigned long long)orangefs_gossip_debug_mask); + + debug_dir = debugfs_create_dir("orangefs", NULL); + + debugfs_create_file(ORANGEFS_KMOD_DEBUG_HELP_FILE, 0444, debug_dir, + debug_help_string, &debug_help_fops); + + orangefs_debug_disabled = 0; + + orangefs_kernel_debug_init(); +} + +/* + * initialize the kernel-debug file. + */ +static void orangefs_kernel_debug_init(void) +{ + static char k_buffer[ORANGEFS_MAX_DEBUG_STRING_LEN] = { }; + size_t len = + strscpy(k_buffer, kernel_debug_string, sizeof(k_buffer) - 1); + + if (len > 0) { + k_buffer[len] = '\n'; + k_buffer[len + 1] = '\0'; + } else { + strscpy(k_buffer, "none\n"); + pr_info("%s: overflow 1!\n", __func__); + } + + debugfs_create_file_aux_num(ORANGEFS_KMOD_DEBUG_FILE, 0444, debug_dir, k_buffer, + 0, &kernel_debug_fops); +} + + +void orangefs_debugfs_cleanup(void) +{ + debugfs_remove_recursive(debug_dir); + kfree(debug_help_string); + debug_help_string = NULL; +} + +/* open ORANGEFS_KMOD_DEBUG_HELP_FILE */ +static int orangefs_debug_help_open(struct inode *inode, struct file *file) +{ + int rc = -ENODEV; + int ret; + + gossip_debug(GOSSIP_DEBUGFS_DEBUG, + "orangefs_debug_help_open: start\n"); + + if (orangefs_debug_disabled) + goto out; + + ret = seq_open(file, &help_debug_ops); + if (ret) + goto out; + + ((struct seq_file *)(file->private_data))->private = inode->i_private; + + rc = 0; + +out: + gossip_debug(GOSSIP_DEBUGFS_DEBUG, + "orangefs_debug_help_open: rc:%d:\n", + rc); + return rc; +} + +/* + * I think start always gets called again after stop. Start + * needs to return NULL when it is done. The whole "payload" + * in this case is a single (long) string, so by the second + * time we get to start (pos = 1), we're done. + */ +static void *help_start(struct seq_file *m, loff_t *pos) +{ + void *payload = NULL; + + gossip_debug(GOSSIP_DEBUGFS_DEBUG, "help_start: start\n"); + + mutex_lock(&orangefs_help_file_lock); + + if (*pos == 0) + payload = m->private; + + return payload; +} + +static void *help_next(struct seq_file *m, void *v, loff_t *pos) +{ + (*pos)++; + gossip_debug(GOSSIP_DEBUGFS_DEBUG, "help_next: start\n"); + + return NULL; +} + +static void help_stop(struct seq_file *m, void *p) +{ + gossip_debug(GOSSIP_DEBUGFS_DEBUG, "help_stop: start\n"); + mutex_unlock(&orangefs_help_file_lock); +} + +static int help_show(struct seq_file *m, void *v) +{ + gossip_debug(GOSSIP_DEBUGFS_DEBUG, "help_show: start\n"); + + seq_puts(m, v); + + return 0; +} + +/* + * initialize the client-debug file. + */ +static void orangefs_client_debug_init(void) +{ + static char c_buffer[ORANGEFS_MAX_DEBUG_STRING_LEN] = { }; + size_t len = + strscpy(c_buffer, client_debug_string, sizeof(c_buffer) - 1); + + if (len > 0) { + c_buffer[len] = '\n'; + c_buffer[len + 1] = '\0'; + } else { + strscpy(c_buffer, "none\n"); + pr_info("%s: overflow! 2\n", __func__); + } + + client_debug_dentry = debugfs_create_file_aux_num( + ORANGEFS_CLIENT_DEBUG_FILE, + 0444, debug_dir, c_buffer, 1, + &kernel_debug_fops); +} + +/* open ORANGEFS_KMOD_DEBUG_FILE or ORANGEFS_CLIENT_DEBUG_FILE.*/ +static int orangefs_debug_open(struct inode *inode, struct file *file) +{ + int rc = -ENODEV; + + gossip_debug(GOSSIP_DEBUGFS_DEBUG, + "%s: orangefs_debug_disabled: %d\n", + __func__, + orangefs_debug_disabled); + + if (orangefs_debug_disabled) + goto out; + + rc = 0; + mutex_lock(&orangefs_debug_lock); + file->private_data = inode->i_private; + mutex_unlock(&orangefs_debug_lock); + +out: + gossip_debug(GOSSIP_DEBUGFS_DEBUG, + "orangefs_debug_open: rc: %d\n", + rc); + return rc; +} + +static ssize_t orangefs_debug_read(struct file *file, + char __user *ubuf, + size_t count, + loff_t *ppos) +{ + char *buf; + int sprintf_ret; + ssize_t read_ret = -ENOMEM; + + gossip_debug(GOSSIP_DEBUGFS_DEBUG, "orangefs_debug_read: start\n"); + + buf = kmalloc(ORANGEFS_MAX_DEBUG_STRING_LEN, GFP_KERNEL); + if (!buf) + goto out; + + mutex_lock(&orangefs_debug_lock); + sprintf_ret = scnprintf(buf, ORANGEFS_MAX_DEBUG_STRING_LEN, "%s", (char *)file->private_data); + mutex_unlock(&orangefs_debug_lock); + + read_ret = simple_read_from_buffer(ubuf, count, ppos, buf, sprintf_ret); + + kfree(buf); + +out: + gossip_debug(GOSSIP_DEBUGFS_DEBUG, + "orangefs_debug_read: ret: %zu\n", + read_ret); + + return read_ret; +} + +static ssize_t orangefs_debug_write(struct file *file, + const char __user *ubuf, + size_t count, + loff_t *ppos) +{ + char *buf; + int rc = -EFAULT; + size_t silly = 0; + char *debug_string; + struct orangefs_kernel_op_s *new_op = NULL; + struct client_debug_mask c_mask = { NULL, 0, 0 }; + char *s; + + gossip_debug(GOSSIP_DEBUGFS_DEBUG, + "orangefs_debug_write: %pD\n", + file); + + if (count == 0) + return 0; + + /* + * Thwart users who try to jamb a ridiculous number + * of bytes into the debug file... + */ + if (count > ORANGEFS_MAX_DEBUG_STRING_LEN) { + silly = count; + count = ORANGEFS_MAX_DEBUG_STRING_LEN; + } + + buf = memdup_user_nul(ubuf, count); + if (IS_ERR(buf)) { + gossip_debug(GOSSIP_DEBUGFS_DEBUG, + "%s: memdup_user_nul failed!\n", + __func__); + rc = PTR_ERR(buf); + buf = NULL; + goto out; + } + strim(buf); + + /* + * Map the keyword string from userspace into a valid debug mask. + * The mapping process involves mapping the human-inputted string + * into a valid mask, and then rebuilding the string from the + * verified valid mask. + * + * A service operation is required to set a new client-side + * debug mask. + */ + if (!debugfs_get_aux_num(file)) { // kernel-debug + debug_string_to_mask(buf, &orangefs_gossip_debug_mask, 0); + debug_mask_to_string(&orangefs_gossip_debug_mask, 0); + debug_string = kernel_debug_string; + gossip_debug(GOSSIP_DEBUGFS_DEBUG, + "New kernel debug string is %s\n", + kernel_debug_string); + } else { + /* Can't reset client debug mask if client is not running. */ + if (is_daemon_in_service()) { + pr_info("%s: Client not running :%d:\n", + __func__, + is_daemon_in_service()); + goto out; + } + + debug_string_to_mask(buf, &c_mask, 1); + debug_mask_to_string(&c_mask, 1); + debug_string = client_debug_string; + + new_op = op_alloc(ORANGEFS_VFS_OP_PARAM); + if (!new_op) { + pr_info("%s: op_alloc failed!\n", __func__); + goto out; + } + + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_TWO_MASK_VALUES; + new_op->upcall.req.param.type = ORANGEFS_PARAM_REQUEST_SET; + memset(new_op->upcall.req.param.s_value, + 0, + ORANGEFS_MAX_DEBUG_STRING_LEN); + sprintf(new_op->upcall.req.param.s_value, + "%llx %llx\n", + c_mask.mask1, + c_mask.mask2); + + /* service_operation returns 0 on success... */ + rc = service_operation(new_op, + "orangefs_param", + ORANGEFS_OP_INTERRUPTIBLE); + + if (rc) + gossip_debug(GOSSIP_DEBUGFS_DEBUG, + "%s: service_operation failed! rc:%d:\n", + __func__, + rc); + + op_release(new_op); + } + + mutex_lock(&orangefs_debug_lock); + s = file_inode(file)->i_private; + memset(s, 0, ORANGEFS_MAX_DEBUG_STRING_LEN); + sprintf(s, "%s\n", debug_string); + mutex_unlock(&orangefs_debug_lock); + + *ppos += count; + if (silly) + rc = silly; + else + rc = count; + +out: + gossip_debug(GOSSIP_DEBUGFS_DEBUG, + "orangefs_debug_write: rc: %d\n", + rc); + kfree(buf); + return rc; +} + +/* + * After obtaining a string representation of the client's debug + * keywords and their associated masks, this function is called to build an + * array of these values. + */ +static int orangefs_prepare_cdm_array(char *debug_array_string) +{ + int i; + int rc = -EINVAL; + char *cds_head = NULL; + char *cds_delimiter = NULL; + int keyword_len = 0; + + gossip_debug(GOSSIP_UTILS_DEBUG, "%s: start\n", __func__); + + /* + * figure out how many elements the cdm_array needs. + */ + for (i = 0; i < strlen(debug_array_string); i++) + if (debug_array_string[i] == '\n') + cdm_element_count++; + + if (!cdm_element_count) { + pr_info("No elements in client debug array string!\n"); + goto out; + } + + cdm_array = kzalloc_objs(*cdm_array, cdm_element_count); + if (!cdm_array) { + rc = -ENOMEM; + goto out; + } + + cds_head = debug_array_string; + + for (i = 0; i < cdm_element_count; i++) { + cds_delimiter = strchr(cds_head, '\n'); + *cds_delimiter = '\0'; + + cds_head = skip_spaces(cds_head); + keyword_len = strcspn(cds_head, " "); + + cdm_array[i].keyword = kzalloc(keyword_len + 1, GFP_KERNEL); + if (!cdm_array[i].keyword) { + rc = -ENOMEM; + goto out; + } + + sscanf(cds_head, + "%s %llx %llx", + cdm_array[i].keyword, + (unsigned long long *)&(cdm_array[i].mask1), + (unsigned long long *)&(cdm_array[i].mask2)); + + if (!strcmp(cdm_array[i].keyword, ORANGEFS_VERBOSE)) + client_verbose_index = i; + + if (!strcmp(cdm_array[i].keyword, ORANGEFS_ALL)) + client_all_index = i; + + cds_head = cds_delimiter + 1; + } + + rc = cdm_element_count; + + gossip_debug(GOSSIP_UTILS_DEBUG, "%s: rc:%d:\n", __func__, rc); + +out: + + return rc; + +} + +/* + * /sys/kernel/debug/orangefs/debug-help can be catted to + * see all the available kernel and client debug keywords. + * + * When orangefs.ko initializes, we have no idea what keywords the + * client supports, nor their associated masks. + * + * We pass through this function once at module-load and stamp a + * boilerplate "we don't know" message for the client in the + * debug-help file. We pass through here again when the client + * starts and then we can fill out the debug-help file fully. + * + * The client might be restarted any number of times between + * module reloads, we only build the debug-help file the first time. + */ +int orangefs_prepare_debugfs_help_string(int at_boot) +{ + char *client_title = "Client Debug Keywords:\n"; + char *kernel_title = "Kernel Debug Keywords:\n"; + size_t string_size = DEBUG_HELP_STRING_SIZE; + size_t result_size; + size_t i; + char *new; + int rc = -EINVAL; + + gossip_debug(GOSSIP_UTILS_DEBUG, "%s: start\n", __func__); + + if (at_boot) + client_title = HELP_STRING_UNINITIALIZED; + + /* build a new debug_help_string. */ + new = kzalloc(DEBUG_HELP_STRING_SIZE, GFP_KERNEL); + if (!new) { + rc = -ENOMEM; + goto out; + } + + /* + * strlcat(dst, src, size) will append at most + * "size - strlen(dst) - 1" bytes of src onto dst, + * null terminating the result, and return the total + * length of the string it tried to create. + * + * We'll just plow through here building our new debug + * help string and let strlcat take care of assuring that + * dst doesn't overflow. + */ + strlcat(new, client_title, string_size); + + if (!at_boot) { + + /* + * fill the client keyword/mask array and remember + * how many elements there were. + */ + cdm_element_count = + orangefs_prepare_cdm_array(client_debug_array_string); + if (cdm_element_count <= 0) { + kfree(new); + goto out; + } + + for (i = 0; i < cdm_element_count; i++) { + strlcat(new, "\t", string_size); + strlcat(new, cdm_array[i].keyword, string_size); + strlcat(new, "\n", string_size); + } + } + + strlcat(new, "\n", string_size); + strlcat(new, kernel_title, string_size); + + for (i = 0; i < num_kmod_keyword_mask_map; i++) { + strlcat(new, "\t", string_size); + strlcat(new, s_kmod_keyword_mask_map[i].keyword, string_size); + result_size = strlcat(new, "\n", string_size); + } + + /* See if we tried to put too many bytes into "new"... */ + if (result_size >= string_size) { + kfree(new); + goto out; + } + + if (at_boot) { + debug_help_string = new; + } else { + mutex_lock(&orangefs_help_file_lock); + memset(debug_help_string, 0, DEBUG_HELP_STRING_SIZE); + strlcat(debug_help_string, new, string_size); + mutex_unlock(&orangefs_help_file_lock); + kfree(new); + } + + rc = 0; + +out: return rc; + +} + +/* + * kernel = type 0 + * client = type 1 + */ +static void debug_mask_to_string(void *mask, int type) +{ + int i; + int len = 0; + char *debug_string; + int element_count = 0; + + gossip_debug(GOSSIP_UTILS_DEBUG, "%s: start\n", __func__); + + if (type) { + debug_string = client_debug_string; + element_count = cdm_element_count; + } else { + debug_string = kernel_debug_string; + element_count = num_kmod_keyword_mask_map; + } + + memset(debug_string, 0, ORANGEFS_MAX_DEBUG_STRING_LEN); + + /* + * Some keywords, like "all" or "verbose", are amalgams of + * numerous other keywords. Make a special check for those + * before grinding through the whole mask only to find out + * later... + */ + if (check_amalgam_keyword(mask, type)) + goto out; + + /* Build the debug string. */ + for (i = 0; i < element_count; i++) + if (type) + do_c_string(mask, i); + else + do_k_string(mask, i); + + len = strlen(debug_string); + + if ((len) && (type)) + client_debug_string[len - 1] = '\0'; + else if (len) + kernel_debug_string[len - 1] = '\0'; + else if (type) + strscpy(client_debug_string, "none"); + else + strscpy(kernel_debug_string, "none"); + +out: +gossip_debug(GOSSIP_UTILS_DEBUG, "%s: string:%s:\n", __func__, debug_string); + + return; +} + +static void do_k_string(void *k_mask, int index) +{ + __u64 *mask = (__u64 *) k_mask; + + if (keyword_is_amalgam((char *) s_kmod_keyword_mask_map[index].keyword)) + goto out; + + if (*mask & s_kmod_keyword_mask_map[index].mask_val) { + if ((strlen(kernel_debug_string) + + strlen(s_kmod_keyword_mask_map[index].keyword) + 1) + < ORANGEFS_MAX_DEBUG_STRING_LEN) { + strcat(kernel_debug_string, + s_kmod_keyword_mask_map[index].keyword); + strcat(kernel_debug_string, ","); + } else { + gossip_err("%s: overflow!\n", __func__); + strscpy(kernel_debug_string, ORANGEFS_ALL); + goto out; + } + } + +out: + + return; +} + +static void do_c_string(void *c_mask, int index) +{ + struct client_debug_mask *mask = (struct client_debug_mask *) c_mask; + + if (keyword_is_amalgam(cdm_array[index].keyword)) + goto out; + + if ((mask->mask1 & cdm_array[index].mask1) || + (mask->mask2 & cdm_array[index].mask2)) { + if ((strlen(client_debug_string) + + strlen(cdm_array[index].keyword) + 1) + < ORANGEFS_MAX_DEBUG_STRING_LEN) { + strcat(client_debug_string, + cdm_array[index].keyword); + strcat(client_debug_string, ","); + } else { + gossip_err("%s: overflow!\n", __func__); + strscpy(client_debug_string, ORANGEFS_ALL); + goto out; + } + } +out: + return; +} + +static int keyword_is_amalgam(char *keyword) +{ + int rc = 0; + + if ((!strcmp(keyword, ORANGEFS_ALL)) || (!strcmp(keyword, ORANGEFS_VERBOSE))) + rc = 1; + + return rc; +} + +/* + * kernel = type 0 + * client = type 1 + * + * return 1 if we found an amalgam. + */ +static int check_amalgam_keyword(void *mask, int type) +{ + __u64 *k_mask; + struct client_debug_mask *c_mask; + int k_all_index = num_kmod_keyword_mask_map - 1; + int rc = 0; + + if (type) { + c_mask = (struct client_debug_mask *) mask; + + if ((c_mask->mask1 == cdm_array[client_all_index].mask1) && + (c_mask->mask2 == cdm_array[client_all_index].mask2)) { + strscpy(client_debug_string, ORANGEFS_ALL); + rc = 1; + goto out; + } + + if ((c_mask->mask1 == cdm_array[client_verbose_index].mask1) && + (c_mask->mask2 == cdm_array[client_verbose_index].mask2)) { + strscpy(client_debug_string, ORANGEFS_VERBOSE); + rc = 1; + goto out; + } + + } else { + k_mask = (__u64 *) mask; + + if (*k_mask >= s_kmod_keyword_mask_map[k_all_index].mask_val) { + strscpy(kernel_debug_string, ORANGEFS_ALL); + rc = 1; + goto out; + } + } + +out: + + return rc; +} + +/* + * kernel = type 0 + * client = type 1 + */ +static void debug_string_to_mask(char *debug_string, void *mask, int type) +{ + int i; + char *strsep_fodder = kstrdup(debug_string, GFP_KERNEL); + char *trimmed; + char *token; + char *original_pointer; + int element_count = 0; + struct client_debug_mask *c_mask = NULL; + __u64 *k_mask = NULL; + + gossip_debug(GOSSIP_UTILS_DEBUG, "%s: start\n", __func__); + + if (type) { + c_mask = (struct client_debug_mask *)mask; + element_count = cdm_element_count; + } else { + k_mask = (__u64 *)mask; + *k_mask = 0; + element_count = num_kmod_keyword_mask_map; + } + + original_pointer = strsep_fodder; + while ((token = strsep(&strsep_fodder, ",")) != NULL) { + trimmed = strim(token); + if (*trimmed) { + for (i = 0; i < element_count; i++) + if (type) + do_c_mask(i, trimmed, &c_mask); + else + do_k_mask(i, trimmed, &k_mask); + + } + } + + kfree(original_pointer); +} + +static void do_c_mask(int i, char *unchecked_keyword, + struct client_debug_mask **sane_mask) +{ + + if (!strcmp(cdm_array[i].keyword, unchecked_keyword)) { + (**sane_mask).mask1 = (**sane_mask).mask1 | cdm_array[i].mask1; + (**sane_mask).mask2 = (**sane_mask).mask2 | cdm_array[i].mask2; + } +} + +static void do_k_mask(int i, char *unchecked_keyword, __u64 **sane_mask) +{ + + if (!strcmp(s_kmod_keyword_mask_map[i].keyword, unchecked_keyword)) + **sane_mask = (**sane_mask) | + s_kmod_keyword_mask_map[i].mask_val; +} + +int orangefs_debugfs_new_client_mask(void __user *arg) +{ + struct dev_mask2_info_s mask2_info = {0}; + int ret; + + ret = copy_from_user(&mask2_info, + (void __user *)arg, + sizeof(struct dev_mask2_info_s)); + + if (ret != 0) + return -EIO; + + client_debug_mask.mask1 = mask2_info.mask1_value; + client_debug_mask.mask2 = mask2_info.mask2_value; + + pr_info("%s: client debug mask has been been received " + ":%llx: :%llx:\n", + __func__, + (unsigned long long)client_debug_mask.mask1, + (unsigned long long)client_debug_mask.mask2); + + return ret; +} + +int orangefs_debugfs_new_client_string(void __user *arg) +{ + int ret; + + ret = copy_from_user(&client_debug_array_string, + (void __user *)arg, + ORANGEFS_MAX_DEBUG_STRING_LEN); + + if (ret != 0) { + pr_info("%s: CLIENT_STRING: copy_from_user failed\n", + __func__); + return -EFAULT; + } + + /* + * The real client-core makes an effort to ensure + * that actual strings that aren't too long to fit in + * this buffer is what we get here. We're going to use + * string functions on the stuff we got, so we'll make + * this extra effort to try and keep from + * flowing out of this buffer when we use the string + * functions, even if somehow the stuff we end up + * with here is garbage. + */ + client_debug_array_string[ORANGEFS_MAX_DEBUG_STRING_LEN - 1] = + '\0'; + + pr_info("%s: client debug array string has been received.\n", + __func__); + + if (!help_string_initialized) { + + /* Build a proper debug help string. */ + ret = orangefs_prepare_debugfs_help_string(0); + if (ret) { + gossip_err("%s: no debug help string \n", + __func__); + return ret; + } + + } + + debug_mask_to_string(&client_debug_mask, 1); + + debugfs_remove(client_debug_dentry); + + orangefs_client_debug_init(); + + help_string_initialized++; + + return 0; +} + +int orangefs_debugfs_new_debug(void __user *arg) +{ + struct dev_mask_info_s mask_info = {0}; + int ret; + + ret = copy_from_user(&mask_info, + (void __user *)arg, + sizeof(mask_info)); + + if (ret != 0) + return -EIO; + + if (mask_info.mask_type == KERNEL_MASK) { + if ((mask_info.mask_value == 0) + && (kernel_mask_set_mod_init)) { + /* + * the kernel debug mask was set when the + * kernel module was loaded; don't override + * it if the client-core was started without + * a value for ORANGEFS_KMODMASK. + */ + return 0; + } + debug_mask_to_string(&mask_info.mask_value, + mask_info.mask_type); + orangefs_gossip_debug_mask = mask_info.mask_value; + pr_info("%s: kernel debug mask has been modified to " + ":%s: :%llx:\n", + __func__, + kernel_debug_string, + (unsigned long long)orangefs_gossip_debug_mask); + } else if (mask_info.mask_type == CLIENT_MASK) { + debug_mask_to_string(&mask_info.mask_value, + mask_info.mask_type); + pr_info("%s: client debug mask has been modified to" + ":%s: :%llx:\n", + __func__, + client_debug_string, + llu(mask_info.mask_value)); + } else { + gossip_err("Invalid mask type....\n"); + return -EINVAL; + } + + return ret; +} diff --git a/fs/orangefs/orangefs-debugfs.h b/fs/orangefs/orangefs-debugfs.h new file mode 100644 index 000000000..502f6dedc --- /dev/null +++ b/fs/orangefs/orangefs-debugfs.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +void orangefs_debugfs_init(int); +void orangefs_debugfs_cleanup(void); +int orangefs_prepare_debugfs_help_string(int); +int orangefs_debugfs_new_client_mask(void __user *); +int orangefs_debugfs_new_client_string(void __user *); +int orangefs_debugfs_new_debug(void __user *); diff --git a/fs/orangefs/orangefs-dev-proto.h b/fs/orangefs/orangefs-dev-proto.h new file mode 100644 index 000000000..dc6609824 --- /dev/null +++ b/fs/orangefs/orangefs-dev-proto.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * (C) 2001 Clemson University and The University of Chicago + * + * See COPYING in top-level directory. + */ + +#ifndef _ORANGEFS_DEV_PROTO_H +#define _ORANGEFS_DEV_PROTO_H + +/* + * types and constants shared between user space and kernel space for + * device interaction using a common protocol + */ + +/* + * valid orangefs kernel operation types + */ +#define ORANGEFS_VFS_OP_INVALID 0xFF000000 +#define ORANGEFS_VFS_OP_FILE_IO 0xFF000001 +#define ORANGEFS_VFS_OP_LOOKUP 0xFF000002 +#define ORANGEFS_VFS_OP_CREATE 0xFF000003 +#define ORANGEFS_VFS_OP_GETATTR 0xFF000004 +#define ORANGEFS_VFS_OP_REMOVE 0xFF000005 +#define ORANGEFS_VFS_OP_MKDIR 0xFF000006 +#define ORANGEFS_VFS_OP_READDIR 0xFF000007 +#define ORANGEFS_VFS_OP_SETATTR 0xFF000008 +#define ORANGEFS_VFS_OP_SYMLINK 0xFF000009 +#define ORANGEFS_VFS_OP_RENAME 0xFF00000A +#define ORANGEFS_VFS_OP_STATFS 0xFF00000B +#define ORANGEFS_VFS_OP_TRUNCATE 0xFF00000C +#define ORANGEFS_VFS_OP_RA_FLUSH 0xFF00000D +#define ORANGEFS_VFS_OP_FS_MOUNT 0xFF00000E +#define ORANGEFS_VFS_OP_FS_UMOUNT 0xFF00000F +#define ORANGEFS_VFS_OP_GETXATTR 0xFF000010 +#define ORANGEFS_VFS_OP_SETXATTR 0xFF000011 +#define ORANGEFS_VFS_OP_LISTXATTR 0xFF000012 +#define ORANGEFS_VFS_OP_REMOVEXATTR 0xFF000013 +#define ORANGEFS_VFS_OP_PARAM 0xFF000014 +#define ORANGEFS_VFS_OP_PERF_COUNT 0xFF000015 +#define ORANGEFS_VFS_OP_CANCEL 0xFF00EE00 +#define ORANGEFS_VFS_OP_FSYNC 0xFF00EE01 +#define ORANGEFS_VFS_OP_FSKEY 0xFF00EE02 +#define ORANGEFS_VFS_OP_READDIRPLUS 0xFF00EE03 +#define ORANGEFS_VFS_OP_FEATURES 0xFF00EE05 /* 2.9.6 */ + +/* features is a 64-bit unsigned bitmask */ +#define ORANGEFS_FEATURE_READAHEAD 1 + +/* + * Misc constants. Please retain them as multiples of 8! + * Otherwise 32-64 bit interactions will be messed up :) + */ +#define ORANGEFS_MAX_DEBUG_STRING_LEN 0x00000800 + +#define ORANGEFS_MAX_DIRENT_COUNT_READDIR 512 + +#include "upcall.h" +#include "downcall.h" + +#endif diff --git a/fs/orangefs/orangefs-kernel.h b/fs/orangefs/orangefs-kernel.h new file mode 100644 index 000000000..1451fc2c1 --- /dev/null +++ b/fs/orangefs/orangefs-kernel.h @@ -0,0 +1,484 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * (C) 2001 Clemson University and The University of Chicago + * + * See COPYING in top-level directory. + */ + +/* + * The ORANGEFS Linux kernel support allows ORANGEFS volumes to be mounted and + * accessed through the Linux VFS (i.e. using standard I/O system calls). + * This support is only needed on clients that wish to mount the file system. + * + */ + +/* + * Declarations and macros for the ORANGEFS Linux kernel support. + */ + +#ifndef __ORANGEFSKERNEL_H +#define __ORANGEFSKERNEL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "orangefs-dev-proto.h" + +#define ORANGEFS_DEFAULT_OP_TIMEOUT_SECS 20 + +#define ORANGEFS_BUFMAP_WAIT_TIMEOUT_SECS 30 + +#define ORANGEFS_DEFAULT_SLOT_TIMEOUT_SECS 900 /* 15 minutes */ + +#define ORANGEFS_REQDEVICE_NAME "pvfs2-req" + +#define ORANGEFS_DEVREQ_MAGIC 0x20030529 +#define ORANGEFS_PURGE_RETRY_COUNT 0x00000005 + +#define MAX_DEV_REQ_UPSIZE (2 * sizeof(__s32) + \ +sizeof(__u64) + sizeof(struct orangefs_upcall_s)) +#define MAX_DEV_REQ_DOWNSIZE (2 * sizeof(__s32) + \ +sizeof(__u64) + sizeof(struct orangefs_downcall_s)) + +/* + * valid orangefs kernel operation states + * + * unknown - op was just initialized + * waiting - op is on request_list (upward bound) + * inprogr - op is in progress (waiting for downcall) + * serviced - op has matching downcall; ok + * purged - op has to start a timer since client-core + * exited uncleanly before servicing op + * given up - submitter has given up waiting for it + */ +enum orangefs_vfs_op_states { + OP_VFS_STATE_UNKNOWN = 0, + OP_VFS_STATE_WAITING = 1, + OP_VFS_STATE_INPROGR = 2, + OP_VFS_STATE_SERVICED = 4, + OP_VFS_STATE_PURGED = 8, + OP_VFS_STATE_GIVEN_UP = 16, +}; + +extern const struct xattr_handler * const orangefs_xattr_handlers[]; + +extern struct posix_acl *orangefs_get_acl(struct inode *inode, int type, bool rcu); +extern int orangefs_set_acl(struct mnt_idmap *idmap, + struct dentry *dentry, struct posix_acl *acl, + int type); +int __orangefs_set_acl(struct inode *inode, struct posix_acl *acl, int type); + +/* + * orangefs data structures + */ +struct orangefs_kernel_op_s { + enum orangefs_vfs_op_states op_state; + __u64 tag; + + /* + * Set uses_shared_memory to non zero if this operation uses + * shared memory. If true, then a retry on the op must also + * get a new shared memory buffer and re-populate it. + * Cancels don't care - it only matters for service_operation() + * retry logics and cancels don't go through it anymore. It + * safely stays non-zero when we use it as slot_to_free. + */ + union { + int uses_shared_memory; + int slot_to_free; + }; + + struct orangefs_upcall_s upcall; + struct orangefs_downcall_s downcall; + + struct completion waitq; + spinlock_t lock; + + int attempts; + + struct list_head list; +}; + +#define set_op_state_waiting(op) ((op)->op_state = OP_VFS_STATE_WAITING) +#define set_op_state_inprogress(op) ((op)->op_state = OP_VFS_STATE_INPROGR) +#define set_op_state_given_up(op) ((op)->op_state = OP_VFS_STATE_GIVEN_UP) +static inline void set_op_state_serviced(struct orangefs_kernel_op_s *op) +{ + op->op_state = OP_VFS_STATE_SERVICED; + complete(&op->waitq); +} + +#define op_state_waiting(op) ((op)->op_state & OP_VFS_STATE_WAITING) +#define op_state_in_progress(op) ((op)->op_state & OP_VFS_STATE_INPROGR) +#define op_state_serviced(op) ((op)->op_state & OP_VFS_STATE_SERVICED) +#define op_state_purged(op) ((op)->op_state & OP_VFS_STATE_PURGED) +#define op_state_given_up(op) ((op)->op_state & OP_VFS_STATE_GIVEN_UP) +#define op_is_cancel(op) ((op)->upcall.type == ORANGEFS_VFS_OP_CANCEL) + +void op_release(struct orangefs_kernel_op_s *op); + +extern void orangefs_bufmap_put(int); +static inline void put_cancel(struct orangefs_kernel_op_s *op) +{ + orangefs_bufmap_put(op->slot_to_free); + op_release(op); +} + +static inline void set_op_state_purged(struct orangefs_kernel_op_s *op) +{ + spin_lock(&op->lock); + if (unlikely(op_is_cancel(op))) { + list_del_init(&op->list); + spin_unlock(&op->lock); + put_cancel(op); + } else { + op->op_state |= OP_VFS_STATE_PURGED; + complete(&op->waitq); + spin_unlock(&op->lock); + } +} + +/* per inode private orangefs info */ +struct orangefs_inode_s { + struct orangefs_object_kref refn; + char link_target[ORANGEFS_NAME_MAX]; + /* + * Reading/Writing Extended attributes need to acquire the appropriate + * reader/writer semaphore on the orangefs_inode_s structure. + */ + struct rw_semaphore xattr_sem; + + struct inode vfs_inode; + sector_t last_failed_block_index_read; + + unsigned long getattr_time; + unsigned long mapping_time; + int attr_valid; + kuid_t attr_uid; + kgid_t attr_gid; + unsigned long bitlock; + + DECLARE_HASHTABLE(xattr_cache, 4); +}; + +/* per superblock private orangefs info */ +struct orangefs_sb_info_s { + struct orangefs_khandle root_khandle; + __s32 fs_id; + int id; + int flags; +#define ORANGEFS_OPT_INTR 0x01 +#define ORANGEFS_OPT_LOCAL_LOCK 0x02 + char devname[ORANGEFS_MAX_SERVER_ADDR_LEN]; + struct super_block *sb; + int mount_pending; + int no_list; + struct list_head list; +}; + +struct orangefs_stats { + unsigned long cache_hits; + unsigned long cache_misses; + unsigned long reads; + unsigned long writes; +}; + +struct orangefs_cached_xattr { + struct hlist_node node; + char key[ORANGEFS_MAX_XATTR_NAMELEN]; + char val[ORANGEFS_MAX_XATTR_VALUELEN]; + ssize_t length; + unsigned long timeout; +}; + +struct orangefs_write_range { + loff_t pos; + size_t len; + kuid_t uid; + kgid_t gid; +}; + +extern struct orangefs_stats orangefs_stats; + +/* + * NOTE: See Documentation/filesystems/porting.rst for information + * on implementing FOO_I and properly accessing fs private data + */ +static inline struct orangefs_inode_s *ORANGEFS_I(struct inode *inode) +{ + return container_of(inode, struct orangefs_inode_s, vfs_inode); +} + +static inline struct orangefs_sb_info_s *ORANGEFS_SB(struct super_block *sb) +{ + return (struct orangefs_sb_info_s *) sb->s_fs_info; +} + +/* ino_t descends from "unsigned long", 8 bytes, 64 bits. */ +static inline ino_t orangefs_khandle_to_ino(struct orangefs_khandle *khandle) +{ + union { + unsigned char u[8]; + __u64 ino; + } ihandle; + + ihandle.u[0] = khandle->u[0] ^ khandle->u[4]; + ihandle.u[1] = khandle->u[1] ^ khandle->u[5]; + ihandle.u[2] = khandle->u[2] ^ khandle->u[6]; + ihandle.u[3] = khandle->u[3] ^ khandle->u[7]; + ihandle.u[4] = khandle->u[12] ^ khandle->u[8]; + ihandle.u[5] = khandle->u[13] ^ khandle->u[9]; + ihandle.u[6] = khandle->u[14] ^ khandle->u[10]; + ihandle.u[7] = khandle->u[15] ^ khandle->u[11]; + + return ihandle.ino; +} + +static inline struct orangefs_khandle *get_khandle_from_ino(struct inode *inode) +{ + return &(ORANGEFS_I(inode)->refn.khandle); +} + +static inline int is_root_handle(struct inode *inode) +{ + gossip_debug(GOSSIP_DCACHE_DEBUG, + "%s: root handle: %pU, this handle: %pU:\n", + __func__, + &ORANGEFS_SB(inode->i_sb)->root_khandle, + get_khandle_from_ino(inode)); + + if (ORANGEFS_khandle_cmp(&(ORANGEFS_SB(inode->i_sb)->root_khandle), + get_khandle_from_ino(inode))) + return 0; + else + return 1; +} + +static inline int match_handle(struct orangefs_khandle resp_handle, + struct inode *inode) +{ + gossip_debug(GOSSIP_DCACHE_DEBUG, + "%s: one handle: %pU, another handle:%pU:\n", + __func__, + &resp_handle, + get_khandle_from_ino(inode)); + + if (ORANGEFS_khandle_cmp(&resp_handle, get_khandle_from_ino(inode))) + return 0; + else + return 1; +} + +/* + * defined in orangefs-cache.c + */ +int op_cache_initialize(void); +int op_cache_finalize(void); +struct orangefs_kernel_op_s *op_alloc(__s32 type); +void orangefs_new_tag(struct orangefs_kernel_op_s *op); +char *get_opname_string(struct orangefs_kernel_op_s *new_op); + +int orangefs_inode_cache_initialize(void); +int orangefs_inode_cache_finalize(void); + +/* + * defined in orangefs-mod.c + */ +void purge_inprogress_ops(void); + +/* + * defined in waitqueue.c + */ +void purge_waiting_ops(void); + +/* + * defined in super.c + */ +extern uint64_t orangefs_features; +extern const struct fs_parameter_spec orangefs_fs_param_spec[]; + +int orangefs_init_fs_context(struct fs_context *fc); + +void orangefs_kill_sb(struct super_block *sb); +int orangefs_remount(struct orangefs_sb_info_s *); + +int fsid_key_table_initialize(void); +void fsid_key_table_finalize(void); + +/* + * defined in inode.c + */ +vm_fault_t orangefs_page_mkwrite(struct vm_fault *); +struct inode *orangefs_new_inode(struct super_block *sb, + struct inode *dir, + umode_t mode, + dev_t dev, + struct orangefs_object_kref *ref); + +int __orangefs_setattr(struct inode *, struct iattr *); +int __orangefs_setattr_mode(struct dentry *dentry, struct iattr *iattr); +int orangefs_setattr(struct mnt_idmap *, struct dentry *, struct iattr *); + +int orangefs_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags); + +int orangefs_permission(struct mnt_idmap *idmap, + struct inode *inode, int mask); + +int orangefs_update_time(struct inode *inode, enum fs_update_time type, + unsigned int flags); + +/* + * defined in xattr.c + */ +ssize_t orangefs_listxattr(struct dentry *dentry, char *buffer, size_t size); + +/* + * defined in namei.c + */ +struct inode *orangefs_iget(struct super_block *sb, + struct orangefs_object_kref *ref); + +/* + * defined in devorangefs-req.c + */ +extern uint32_t orangefs_userspace_version; + +int orangefs_dev_init(void); +void orangefs_dev_cleanup(void); +int is_daemon_in_service(void); +bool __is_daemon_in_service(void); + +/* + * defined in file.c + */ +int orangefs_revalidate_mapping(struct inode *); +ssize_t wait_for_direct_io(enum ORANGEFS_io_type, struct inode *, loff_t *, + struct iov_iter *, size_t, loff_t, struct orangefs_write_range *, int *, + struct file *); +ssize_t do_readv_writev(enum ORANGEFS_io_type, struct file *, loff_t *, + struct iov_iter *); + +/* + * defined in orangefs-utils.c + */ +__s32 fsid_of_op(struct orangefs_kernel_op_s *op); + +ssize_t orangefs_inode_getxattr(struct inode *inode, + const char *name, + void *buffer, + size_t size); + +int orangefs_inode_setxattr(struct inode *inode, + const char *name, + const void *value, + size_t size, + int flags); + +#define ORANGEFS_GETATTR_NEW 1 +#define ORANGEFS_GETATTR_SIZE 2 + +int orangefs_inode_getattr(struct inode *, int); + +int orangefs_inode_check_changed(struct inode *inode); + +int orangefs_inode_setattr(struct inode *inode); + +bool orangefs_cancel_op_in_progress(struct orangefs_kernel_op_s *op); + +int orangefs_normalize_to_errno(__s32 error_code); + +extern struct mutex orangefs_request_mutex; +extern int op_timeout_secs; +extern int slot_timeout_secs; +extern int orangefs_cache_timeout_msecs; +extern int orangefs_dcache_timeout_msecs; +extern int orangefs_getattr_timeout_msecs; +extern struct list_head orangefs_superblocks; +extern spinlock_t orangefs_superblocks_lock; +extern struct list_head orangefs_request_list; +extern spinlock_t orangefs_request_list_lock; +extern wait_queue_head_t orangefs_request_list_waitq; +extern struct list_head *orangefs_htable_ops_in_progress; +extern spinlock_t orangefs_htable_ops_in_progress_lock; +extern int hash_table_size; + +extern const struct file_operations orangefs_file_operations; +extern const struct inode_operations orangefs_symlink_inode_operations; +extern const struct inode_operations orangefs_dir_inode_operations; +extern const struct file_operations orangefs_dir_operations; +extern const struct dentry_operations orangefs_dentry_operations; + +/* + * misc convenience macros + */ + +#define ORANGEFS_OP_INTERRUPTIBLE 1 /* service_operation() is interruptible */ +#define ORANGEFS_OP_PRIORITY 2 /* service_operation() is high priority */ +#define ORANGEFS_OP_CANCELLATION 4 /* this is a cancellation */ +#define ORANGEFS_OP_NO_MUTEX 8 /* don't acquire request_mutex */ +#define ORANGEFS_OP_ASYNC 16 /* Queue it, but don't wait */ +#define ORANGEFS_OP_WRITEBACK 32 + +int service_operation(struct orangefs_kernel_op_s *op, + const char *op_name, + int flags); + +#define get_interruptible_flag(inode) \ + ((ORANGEFS_SB(inode->i_sb)->flags & ORANGEFS_OPT_INTR) ? \ + ORANGEFS_OP_INTERRUPTIBLE : 0) + +#define fill_default_sys_attrs(sys_attr, mode) \ +do { \ + sys_attr.owner = from_kuid(&init_user_ns, current_fsuid()); \ + sys_attr.group = from_kgid(&init_user_ns, current_fsgid()); \ + sys_attr.perms = ORANGEFS_util_translate_mode(mode); \ + sys_attr.mtime = 0; \ + sys_attr.atime = 0; \ + sys_attr.ctime = 0; \ + sys_attr.mask = ORANGEFS_ATTR_SYS_ALL_SETABLE; \ +} while (0) + +static inline void orangefs_set_timeout(struct dentry *dentry) +{ + unsigned long time = jiffies + orangefs_dcache_timeout_msecs*HZ/1000; + + dentry->d_fsdata = (void *) time; +} + +#endif /* __ORANGEFSKERNEL_H */ diff --git a/fs/orangefs/orangefs-mod.c b/fs/orangefs/orangefs-mod.c new file mode 100644 index 000000000..f591f0953 --- /dev/null +++ b/fs/orangefs/orangefs-mod.c @@ -0,0 +1,232 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * (C) 2001 Clemson University and The University of Chicago + * + * Changes by Acxiom Corporation to add proc file handler for pvfs2 client + * parameters, Copyright Acxiom Corporation, 2005. + * + * See COPYING in top-level directory. + */ + +#include "protocol.h" +#include "orangefs-kernel.h" +#include "orangefs-debugfs.h" +#include "orangefs-sysfs.h" + +/* ORANGEFS_VERSION is a ./configure define */ +#ifndef ORANGEFS_VERSION +#define ORANGEFS_VERSION "upstream" +#endif + +/* + * global variables declared here + */ + +struct orangefs_stats orangefs_stats; + +/* the size of the hash tables for ops in progress */ +int hash_table_size = 509; + +static ulong module_parm_debug_mask; +__u64 orangefs_gossip_debug_mask; +int op_timeout_secs = ORANGEFS_DEFAULT_OP_TIMEOUT_SECS; +int slot_timeout_secs = ORANGEFS_DEFAULT_SLOT_TIMEOUT_SECS; +int orangefs_cache_timeout_msecs = 500; +int orangefs_dcache_timeout_msecs = 50; +int orangefs_getattr_timeout_msecs = 50; + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("ORANGEFS Development Team"); +MODULE_DESCRIPTION("The Linux Kernel VFS interface to ORANGEFS"); +MODULE_PARM_DESC(module_parm_debug_mask, "debugging level (see orangefs-debug.h for values)"); +MODULE_PARM_DESC(op_timeout_secs, "Operation timeout in seconds"); +MODULE_PARM_DESC(slot_timeout_secs, "Slot timeout in seconds"); +MODULE_PARM_DESC(hash_table_size, + "size of hash table for operations in progress"); + +static struct file_system_type orangefs_fs_type = { + .name = "pvfs2", + .init_fs_context = orangefs_init_fs_context, + .parameters = orangefs_fs_param_spec, + .kill_sb = orangefs_kill_sb, + .owner = THIS_MODULE, +}; + +module_param(hash_table_size, int, 0); +module_param(module_parm_debug_mask, ulong, 0644); +module_param(op_timeout_secs, int, 0); +module_param(slot_timeout_secs, int, 0); + +/* + * Blocks non-priority requests from being queued for servicing. This + * could be used for protecting the request list data structure, but + * for now it's only being used to stall the op addition to the request + * list + */ +DEFINE_MUTEX(orangefs_request_mutex); + +/* hash table for storing operations waiting for matching downcall */ +struct list_head *orangefs_htable_ops_in_progress; +DEFINE_SPINLOCK(orangefs_htable_ops_in_progress_lock); + +/* list for queueing upcall operations */ +LIST_HEAD(orangefs_request_list); + +/* used to protect the above orangefs_request_list */ +DEFINE_SPINLOCK(orangefs_request_list_lock); + +/* used for incoming request notification */ +DECLARE_WAIT_QUEUE_HEAD(orangefs_request_list_waitq); + +static int __init orangefs_init(void) +{ + int ret; + __u32 i = 0; + + if (op_timeout_secs < 0) + op_timeout_secs = 0; + + if (slot_timeout_secs < 0) + slot_timeout_secs = 0; + + /* initialize global book keeping data structures */ + ret = op_cache_initialize(); + if (ret < 0) + goto out; + + ret = orangefs_inode_cache_initialize(); + if (ret < 0) + goto cleanup_op; + + orangefs_htable_ops_in_progress = + kzalloc_objs(struct list_head, hash_table_size); + if (!orangefs_htable_ops_in_progress) { + ret = -ENOMEM; + goto cleanup_inode; + } + + /* initialize a doubly linked at each hash table index */ + for (i = 0; i < hash_table_size; i++) + INIT_LIST_HEAD(&orangefs_htable_ops_in_progress[i]); + + ret = fsid_key_table_initialize(); + if (ret < 0) + goto cleanup_progress_table; + + /* + * Build the contents of /sys/kernel/debug/orangefs/debug-help + * from the keywords in the kernel keyword/mask array. + * + * The keywords in the client keyword/mask array are + * unknown at boot time. + * + * orangefs_prepare_debugfs_help_string will be used again + * later to rebuild the debug-help-string after the client starts + * and passes along the needed info. The argument signifies + * which time orangefs_prepare_debugfs_help_string is being + * called. + */ + ret = orangefs_prepare_debugfs_help_string(1); + if (ret) + goto cleanup_key_table; + + orangefs_debugfs_init(module_parm_debug_mask); + + ret = orangefs_sysfs_init(); + if (ret) + goto sysfs_init_failed; + + /* Initialize the orangefsdev subsystem. */ + ret = orangefs_dev_init(); + if (ret < 0) { + gossip_err("%s: could not initialize device subsystem %d!\n", + __func__, + ret); + goto cleanup_sysfs; + } + + ret = register_filesystem(&orangefs_fs_type); + if (ret == 0) { + pr_info("%s: module version %s loaded\n", + __func__, + ORANGEFS_VERSION); + goto out; + } + + orangefs_dev_cleanup(); + +cleanup_sysfs: + orangefs_sysfs_exit(); + +sysfs_init_failed: + orangefs_debugfs_cleanup(); + +cleanup_key_table: + fsid_key_table_finalize(); + +cleanup_progress_table: + kfree(orangefs_htable_ops_in_progress); + +cleanup_inode: + orangefs_inode_cache_finalize(); + +cleanup_op: + op_cache_finalize(); + +out: + return ret; +} + +static void __exit orangefs_exit(void) +{ + int i = 0; + gossip_debug(GOSSIP_INIT_DEBUG, "orangefs: orangefs_exit called\n"); + + unregister_filesystem(&orangefs_fs_type); + orangefs_debugfs_cleanup(); + orangefs_sysfs_exit(); + fsid_key_table_finalize(); + orangefs_dev_cleanup(); + BUG_ON(!list_empty(&orangefs_request_list)); + for (i = 0; i < hash_table_size; i++) + BUG_ON(!list_empty(&orangefs_htable_ops_in_progress[i])); + + orangefs_inode_cache_finalize(); + op_cache_finalize(); + + kfree(orangefs_htable_ops_in_progress); + + pr_info("orangefs: module version %s unloaded\n", ORANGEFS_VERSION); +} + +/* + * What we do in this function is to walk the list of operations + * that are in progress in the hash table and mark them as purged as well. + */ +void purge_inprogress_ops(void) +{ + int i; + + for (i = 0; i < hash_table_size; i++) { + struct orangefs_kernel_op_s *op; + struct orangefs_kernel_op_s *next; + + spin_lock(&orangefs_htable_ops_in_progress_lock); + list_for_each_entry_safe(op, + next, + &orangefs_htable_ops_in_progress[i], + list) { + set_op_state_purged(op); + gossip_debug(GOSSIP_DEV_DEBUG, + "%s: op:%s: op_state:%d: process:%s:\n", + __func__, + get_opname_string(op), + op->op_state, + current->comm); + } + spin_unlock(&orangefs_htable_ops_in_progress_lock); + } +} + +module_init(orangefs_init); +module_exit(orangefs_exit); diff --git a/fs/orangefs/orangefs-sysfs.c b/fs/orangefs/orangefs-sysfs.c new file mode 100644 index 000000000..a02bfc7b3 --- /dev/null +++ b/fs/orangefs/orangefs-sysfs.c @@ -0,0 +1,1316 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Documentation/ABI/stable/sysfs-fs-orangefs: + * + * What: /sys/fs/orangefs/perf_counter_reset + * Date: June 2015 + * Contact: Mike Marshall + * Description: + * echo a 0 or a 1 into perf_counter_reset to + * reset all the counters in + * /sys/fs/orangefs/perf_counters + * except ones with PINT_PERF_PRESERVE set. + * + * + * What: /sys/fs/orangefs/perf_counters/... + * Date: Jun 2015 + * Contact: Mike Marshall + * Description: + * Counters and settings for various caches. + * Read only. + * + * + * What: /sys/fs/orangefs/perf_time_interval_secs + * Date: Jun 2015 + * Contact: Mike Marshall + * Description: + * Length of perf counter intervals in + * seconds. + * + * + * What: /sys/fs/orangefs/perf_history_size + * Date: Jun 2015 + * Contact: Mike Marshall + * Description: + * The perf_counters cache statistics have N, or + * perf_history_size, samples. The default is + * one. + * + * Every perf_time_interval_secs the (first) + * samples are reset. + * + * If N is greater than one, the "current" set + * of samples is reset, and the samples from the + * other N-1 intervals remain available. + * + * + * What: /sys/fs/orangefs/op_timeout_secs + * Date: Jun 2015 + * Contact: Mike Marshall + * Description: + * Service operation timeout in seconds. + * + * + * What: /sys/fs/orangefs/slot_timeout_secs + * Date: Jun 2015 + * Contact: Mike Marshall + * Description: + * "Slot" timeout in seconds. A "slot" + * is an indexed buffer in the shared + * memory segment used for communication + * between the kernel module and userspace. + * Slots are requested and waited for, + * the wait times out after slot_timeout_secs. + * + * What: /sys/fs/orangefs/cache_timeout_msecs + * Date: Mar 2018 + * Contact: Martin Brandenburg + * Description: + * Time in milliseconds between which + * orangefs_revalidate_mapping will invalidate the page + * cache. + * + * What: /sys/fs/orangefs/dcache_timeout_msecs + * Date: Jul 2016 + * Contact: Martin Brandenburg + * Description: + * Time lookup is valid in milliseconds. + * + * What: /sys/fs/orangefs/getattr_timeout_msecs + * Date: Jul 2016 + * Contact: Martin Brandenburg + * Description: + * Time getattr is valid in milliseconds. + * + * What: /sys/fs/orangefs/readahead_count + * Date: Aug 2016 + * Contact: Martin Brandenburg + * Description: + * Readahead cache buffer count. + * + * What: /sys/fs/orangefs/readahead_size + * Date: Aug 2016 + * Contact: Martin Brandenburg + * Description: + * Readahead cache buffer size. + * + * What: /sys/fs/orangefs/readahead_count_size + * Date: Aug 2016 + * Contact: Martin Brandenburg + * Description: + * Readahead cache buffer count and size. + * + * What: /sys/fs/orangefs/readahead_readcnt + * Date: Jan 2017 + * Contact: Martin Brandenburg + * Description: + * Number of buffers (in multiples of readahead_size) + * which can be read ahead for a single file at once. + * + * What: /sys/fs/orangefs/acache/... + * Date: Jun 2015 + * Contact: Martin Brandenburg + * Description: + * Attribute cache configurable settings. + * + * + * What: /sys/fs/orangefs/ncache/... + * Date: Jun 2015 + * Contact: Mike Marshall + * Description: + * Name cache configurable settings. + * + * + * What: /sys/fs/orangefs/capcache/... + * Date: Jun 2015 + * Contact: Mike Marshall + * Description: + * Capability cache configurable settings. + * + * + * What: /sys/fs/orangefs/ccache/... + * Date: Jun 2015 + * Contact: Mike Marshall + * Description: + * Credential cache configurable settings. + * + */ + +#include +#include +#include +#include +#include +#include + +#include "protocol.h" +#include "orangefs-kernel.h" +#include "orangefs-sysfs.h" + +#define ORANGEFS_KOBJ_ID "orangefs" +#define ACACHE_KOBJ_ID "acache" +#define CAPCACHE_KOBJ_ID "capcache" +#define CCACHE_KOBJ_ID "ccache" +#define NCACHE_KOBJ_ID "ncache" +#define PC_KOBJ_ID "pc" +#define STATS_KOBJ_ID "stats" + +/* + * Every item calls orangefs_attr_show and orangefs_attr_store through + * orangefs_sysfs_ops. They look at the orangefs_attributes further below to + * call one of sysfs_int_show, sysfs_int_store, sysfs_service_op_show, or + * sysfs_service_op_store. + */ + +struct orangefs_attribute { + struct attribute attr; + ssize_t (*show)(struct kobject *kobj, + struct orangefs_attribute *attr, + char *buf); + ssize_t (*store)(struct kobject *kobj, + struct orangefs_attribute *attr, + const char *buf, + size_t count); +}; + +static ssize_t orangefs_attr_show(struct kobject *kobj, + struct attribute *attr, + char *buf) +{ + struct orangefs_attribute *attribute; + + attribute = container_of(attr, struct orangefs_attribute, attr); + if (!attribute->show) + return -EIO; + return attribute->show(kobj, attribute, buf); +} + +static ssize_t orangefs_attr_store(struct kobject *kobj, + struct attribute *attr, + const char *buf, + size_t len) +{ + struct orangefs_attribute *attribute; + + if (!strcmp(kobj->name, PC_KOBJ_ID) || + !strcmp(kobj->name, STATS_KOBJ_ID)) + return -EPERM; + + attribute = container_of(attr, struct orangefs_attribute, attr); + if (!attribute->store) + return -EIO; + return attribute->store(kobj, attribute, buf, len); +} + +static const struct sysfs_ops orangefs_sysfs_ops = { + .show = orangefs_attr_show, + .store = orangefs_attr_store, +}; + +static ssize_t sysfs_int_show(struct kobject *kobj, + struct orangefs_attribute *attr, char *buf) +{ + int rc = -EIO; + + gossip_debug(GOSSIP_SYSFS_DEBUG, "sysfs_int_show: id:%s:\n", + kobj->name); + + if (!strcmp(kobj->name, ORANGEFS_KOBJ_ID)) { + if (!strcmp(attr->attr.name, "op_timeout_secs")) { + rc = sysfs_emit(buf, + "%d\n", + op_timeout_secs); + goto out; + } else if (!strcmp(attr->attr.name, + "slot_timeout_secs")) { + rc = sysfs_emit(buf, + "%d\n", + slot_timeout_secs); + goto out; + } else if (!strcmp(attr->attr.name, + "cache_timeout_msecs")) { + rc = sysfs_emit(buf, + "%d\n", + orangefs_cache_timeout_msecs); + goto out; + } else if (!strcmp(attr->attr.name, + "dcache_timeout_msecs")) { + rc = sysfs_emit(buf, + "%d\n", + orangefs_dcache_timeout_msecs); + goto out; + } else if (!strcmp(attr->attr.name, + "getattr_timeout_msecs")) { + rc = sysfs_emit(buf, + "%d\n", + orangefs_getattr_timeout_msecs); + goto out; + } else { + goto out; + } + + } else if (!strcmp(kobj->name, STATS_KOBJ_ID)) { + if (!strcmp(attr->attr.name, "reads")) { + rc = sysfs_emit(buf, + "%lu\n", + orangefs_stats.reads); + goto out; + } else if (!strcmp(attr->attr.name, "writes")) { + rc = sysfs_emit(buf, + "%lu\n", + orangefs_stats.writes); + goto out; + } else { + goto out; + } + } + +out: + + return rc; +} + +static ssize_t sysfs_int_store(struct kobject *kobj, + struct orangefs_attribute *attr, const char *buf, size_t count) +{ + int rc = 0; + + gossip_debug(GOSSIP_SYSFS_DEBUG, + "sysfs_int_store: start attr->attr.name:%s: buf:%s:\n", + attr->attr.name, buf); + + if (!strcmp(attr->attr.name, "op_timeout_secs")) { + rc = kstrtoint(buf, 0, &op_timeout_secs); + goto out; + } else if (!strcmp(attr->attr.name, "slot_timeout_secs")) { + rc = kstrtoint(buf, 0, &slot_timeout_secs); + goto out; + } else if (!strcmp(attr->attr.name, "cache_timeout_msecs")) { + rc = kstrtoint(buf, 0, &orangefs_cache_timeout_msecs); + goto out; + } else if (!strcmp(attr->attr.name, "dcache_timeout_msecs")) { + rc = kstrtoint(buf, 0, &orangefs_dcache_timeout_msecs); + goto out; + } else if (!strcmp(attr->attr.name, "getattr_timeout_msecs")) { + rc = kstrtoint(buf, 0, &orangefs_getattr_timeout_msecs); + goto out; + } else { + goto out; + } + +out: + if (rc) + rc = -EINVAL; + else + rc = count; + + return rc; +} + +/* + * obtain attribute values from userspace with a service operation. + */ +static ssize_t sysfs_service_op_show(struct kobject *kobj, + struct orangefs_attribute *attr, char *buf) +{ + struct orangefs_kernel_op_s *new_op = NULL; + int rc = 0; + char *ser_op_type = NULL; + __u32 op_alloc_type; + + gossip_debug(GOSSIP_SYSFS_DEBUG, + "sysfs_service_op_show: id:%s:\n", + kobj->name); + + if (strcmp(kobj->name, PC_KOBJ_ID)) + op_alloc_type = ORANGEFS_VFS_OP_PARAM; + else + op_alloc_type = ORANGEFS_VFS_OP_PERF_COUNT; + + new_op = op_alloc(op_alloc_type); + if (!new_op) + return -ENOMEM; + + /* Can't do a service_operation if the client is not running... */ + rc = is_daemon_in_service(); + if (rc) { + pr_info_ratelimited("%s: Client not running :%d:\n", + __func__, + is_daemon_in_service()); + goto out; + } + + if (strcmp(kobj->name, PC_KOBJ_ID)) + new_op->upcall.req.param.type = ORANGEFS_PARAM_REQUEST_GET; + + if (!strcmp(kobj->name, ORANGEFS_KOBJ_ID)) { + /* Drop unsupported requests first. */ + if (!(orangefs_features & ORANGEFS_FEATURE_READAHEAD) && + (!strcmp(attr->attr.name, "readahead_count") || + !strcmp(attr->attr.name, "readahead_size") || + !strcmp(attr->attr.name, "readahead_count_size") || + !strcmp(attr->attr.name, "readahead_readcnt"))) { + rc = -EINVAL; + goto out; + } + + if (!strcmp(attr->attr.name, "perf_history_size")) + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_PERF_HISTORY_SIZE; + else if (!strcmp(attr->attr.name, + "perf_time_interval_secs")) + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_PERF_TIME_INTERVAL_SECS; + else if (!strcmp(attr->attr.name, + "perf_counter_reset")) + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_PERF_RESET; + + else if (!strcmp(attr->attr.name, + "readahead_count")) + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_READAHEAD_COUNT; + + else if (!strcmp(attr->attr.name, + "readahead_size")) + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_READAHEAD_SIZE; + + else if (!strcmp(attr->attr.name, + "readahead_count_size")) + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_READAHEAD_COUNT_SIZE; + + else if (!strcmp(attr->attr.name, + "readahead_readcnt")) + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_READAHEAD_READCNT; + } else if (!strcmp(kobj->name, ACACHE_KOBJ_ID)) { + if (!strcmp(attr->attr.name, "timeout_msecs")) + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_ACACHE_TIMEOUT_MSECS; + + if (!strcmp(attr->attr.name, "hard_limit")) + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_ACACHE_HARD_LIMIT; + + if (!strcmp(attr->attr.name, "soft_limit")) + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_ACACHE_SOFT_LIMIT; + + if (!strcmp(attr->attr.name, "reclaim_percentage")) + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_ACACHE_RECLAIM_PERCENTAGE; + + } else if (!strcmp(kobj->name, CAPCACHE_KOBJ_ID)) { + if (!strcmp(attr->attr.name, "timeout_secs")) + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_CAPCACHE_TIMEOUT_SECS; + + if (!strcmp(attr->attr.name, "hard_limit")) + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_CAPCACHE_HARD_LIMIT; + + if (!strcmp(attr->attr.name, "soft_limit")) + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_CAPCACHE_SOFT_LIMIT; + + if (!strcmp(attr->attr.name, "reclaim_percentage")) + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_CAPCACHE_RECLAIM_PERCENTAGE; + + } else if (!strcmp(kobj->name, CCACHE_KOBJ_ID)) { + if (!strcmp(attr->attr.name, "timeout_secs")) + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_CCACHE_TIMEOUT_SECS; + + if (!strcmp(attr->attr.name, "hard_limit")) + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_CCACHE_HARD_LIMIT; + + if (!strcmp(attr->attr.name, "soft_limit")) + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_CCACHE_SOFT_LIMIT; + + if (!strcmp(attr->attr.name, "reclaim_percentage")) + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_CCACHE_RECLAIM_PERCENTAGE; + + } else if (!strcmp(kobj->name, NCACHE_KOBJ_ID)) { + if (!strcmp(attr->attr.name, "timeout_msecs")) + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_NCACHE_TIMEOUT_MSECS; + + if (!strcmp(attr->attr.name, "hard_limit")) + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_NCACHE_HARD_LIMIT; + + if (!strcmp(attr->attr.name, "soft_limit")) + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_NCACHE_SOFT_LIMIT; + + if (!strcmp(attr->attr.name, "reclaim_percentage")) + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_NCACHE_RECLAIM_PERCENTAGE; + + } else if (!strcmp(kobj->name, PC_KOBJ_ID)) { + if (!strcmp(attr->attr.name, ACACHE_KOBJ_ID)) + new_op->upcall.req.perf_count.type = + ORANGEFS_PERF_COUNT_REQUEST_ACACHE; + + if (!strcmp(attr->attr.name, CAPCACHE_KOBJ_ID)) + new_op->upcall.req.perf_count.type = + ORANGEFS_PERF_COUNT_REQUEST_CAPCACHE; + + if (!strcmp(attr->attr.name, NCACHE_KOBJ_ID)) + new_op->upcall.req.perf_count.type = + ORANGEFS_PERF_COUNT_REQUEST_NCACHE; + + } else { + gossip_err("sysfs_service_op_show: unknown kobj_id:%s:\n", + kobj->name); + rc = -EINVAL; + goto out; + } + + + if (strcmp(kobj->name, PC_KOBJ_ID)) + ser_op_type = "orangefs_param"; + else + ser_op_type = "orangefs_perf_count"; + + /* + * The service_operation will return an errno return code on + * error, and zero on success. + */ + rc = service_operation(new_op, ser_op_type, ORANGEFS_OP_INTERRUPTIBLE); + +out: + if (!rc) { + if (strcmp(kobj->name, PC_KOBJ_ID)) { + if (new_op->upcall.req.param.op == + ORANGEFS_PARAM_REQUEST_OP_READAHEAD_COUNT_SIZE) { + rc = sysfs_emit(buf, "%d %d\n", + (int)new_op->downcall.resp.param.u. + value32[0], + (int)new_op->downcall.resp.param.u. + value32[1]); + } else { + rc = sysfs_emit(buf, "%d\n", + (int)new_op->downcall.resp.param.u.value64); + } + } else { + rc = sysfs_emit( + buf, + "%s", + new_op->downcall.resp.perf_count.buffer); + } + } + + op_release(new_op); + + return rc; + +} + +/* + * pass attribute values back to userspace with a service operation. + * + * We have to do a memory allocation, an sscanf and a service operation. + * And we have to evaluate what the user entered, to make sure the + * value is within the range supported by the attribute. So, there's + * a lot of return code checking and mapping going on here. + * + * We want to return 1 if we think everything went OK, and + * EINVAL if not. + */ +static ssize_t sysfs_service_op_store(struct kobject *kobj, + struct orangefs_attribute *attr, const char *buf, size_t count) +{ + struct orangefs_kernel_op_s *new_op = NULL; + int val = 0; + int rc = 0; + + gossip_debug(GOSSIP_SYSFS_DEBUG, + "sysfs_service_op_store: id:%s:\n", + kobj->name); + + new_op = op_alloc(ORANGEFS_VFS_OP_PARAM); + if (!new_op) + return -EINVAL; /* sic */ + + /* Can't do a service_operation if the client is not running... */ + rc = is_daemon_in_service(); + if (rc) { + pr_info("%s: Client not running :%d:\n", + __func__, + is_daemon_in_service()); + goto out; + } + + /* + * The value we want to send back to userspace is in buf, unless this + * there are two parameters, which is specially handled below. + */ + if (strcmp(kobj->name, ORANGEFS_KOBJ_ID) || + strcmp(attr->attr.name, "readahead_count_size")) { + rc = kstrtoint(buf, 0, &val); + if (rc) + goto out; + } + + new_op->upcall.req.param.type = ORANGEFS_PARAM_REQUEST_SET; + + if (!strcmp(kobj->name, ORANGEFS_KOBJ_ID)) { + /* Drop unsupported requests first. */ + if (!(orangefs_features & ORANGEFS_FEATURE_READAHEAD) && + (!strcmp(attr->attr.name, "readahead_count") || + !strcmp(attr->attr.name, "readahead_size") || + !strcmp(attr->attr.name, "readahead_count_size") || + !strcmp(attr->attr.name, "readahead_readcnt"))) { + rc = -EINVAL; + goto out; + } + + if (!strcmp(attr->attr.name, "perf_history_size")) { + if (val > 0) { + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_PERF_HISTORY_SIZE; + } else { + rc = 0; + goto out; + } + } else if (!strcmp(attr->attr.name, + "perf_time_interval_secs")) { + if (val > 0) { + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_PERF_TIME_INTERVAL_SECS; + } else { + rc = 0; + goto out; + } + } else if (!strcmp(attr->attr.name, + "perf_counter_reset")) { + if ((val == 0) || (val == 1)) { + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_PERF_RESET; + } else { + rc = 0; + goto out; + } + } else if (!strcmp(attr->attr.name, + "readahead_count")) { + if ((val >= 0)) { + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_READAHEAD_COUNT; + } else { + rc = 0; + goto out; + } + } else if (!strcmp(attr->attr.name, + "readahead_size")) { + if ((val >= 0)) { + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_READAHEAD_SIZE; + } else { + rc = 0; + goto out; + } + } else if (!strcmp(attr->attr.name, + "readahead_count_size")) { + int val1, val2; + rc = sscanf(buf, "%d %d", &val1, &val2); + if (rc < 2) { + rc = 0; + goto out; + } + if ((val1 >= 0) && (val2 >= 0)) { + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_READAHEAD_COUNT_SIZE; + } else { + rc = 0; + goto out; + } + new_op->upcall.req.param.u.value32[0] = val1; + new_op->upcall.req.param.u.value32[1] = val2; + goto value_set; + } else if (!strcmp(attr->attr.name, + "readahead_readcnt")) { + if ((val >= 0)) { + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_READAHEAD_READCNT; + } else { + rc = 0; + goto out; + } + } + + } else if (!strcmp(kobj->name, ACACHE_KOBJ_ID)) { + if (!strcmp(attr->attr.name, "hard_limit")) { + if (val > -1) { + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_ACACHE_HARD_LIMIT; + } else { + rc = 0; + goto out; + } + } else if (!strcmp(attr->attr.name, "soft_limit")) { + if (val > -1) { + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_ACACHE_SOFT_LIMIT; + } else { + rc = 0; + goto out; + } + } else if (!strcmp(attr->attr.name, + "reclaim_percentage")) { + if ((val > -1) && (val < 101)) { + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_ACACHE_RECLAIM_PERCENTAGE; + } else { + rc = 0; + goto out; + } + } else if (!strcmp(attr->attr.name, "timeout_msecs")) { + if (val > -1) { + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_ACACHE_TIMEOUT_MSECS; + } else { + rc = 0; + goto out; + } + } + + } else if (!strcmp(kobj->name, CAPCACHE_KOBJ_ID)) { + if (!strcmp(attr->attr.name, "hard_limit")) { + if (val > -1) { + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_CAPCACHE_HARD_LIMIT; + } else { + rc = 0; + goto out; + } + } else if (!strcmp(attr->attr.name, "soft_limit")) { + if (val > -1) { + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_CAPCACHE_SOFT_LIMIT; + } else { + rc = 0; + goto out; + } + } else if (!strcmp(attr->attr.name, + "reclaim_percentage")) { + if ((val > -1) && (val < 101)) { + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_CAPCACHE_RECLAIM_PERCENTAGE; + } else { + rc = 0; + goto out; + } + } else if (!strcmp(attr->attr.name, "timeout_secs")) { + if (val > -1) { + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_CAPCACHE_TIMEOUT_SECS; + } else { + rc = 0; + goto out; + } + } + + } else if (!strcmp(kobj->name, CCACHE_KOBJ_ID)) { + if (!strcmp(attr->attr.name, "hard_limit")) { + if (val > -1) { + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_CCACHE_HARD_LIMIT; + } else { + rc = 0; + goto out; + } + } else if (!strcmp(attr->attr.name, "soft_limit")) { + if (val > -1) { + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_CCACHE_SOFT_LIMIT; + } else { + rc = 0; + goto out; + } + } else if (!strcmp(attr->attr.name, + "reclaim_percentage")) { + if ((val > -1) && (val < 101)) { + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_CCACHE_RECLAIM_PERCENTAGE; + } else { + rc = 0; + goto out; + } + } else if (!strcmp(attr->attr.name, "timeout_secs")) { + if (val > -1) { + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_CCACHE_TIMEOUT_SECS; + } else { + rc = 0; + goto out; + } + } + + } else if (!strcmp(kobj->name, NCACHE_KOBJ_ID)) { + if (!strcmp(attr->attr.name, "hard_limit")) { + if (val > -1) { + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_NCACHE_HARD_LIMIT; + } else { + rc = 0; + goto out; + } + } else if (!strcmp(attr->attr.name, "soft_limit")) { + if (val > -1) { + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_NCACHE_SOFT_LIMIT; + } else { + rc = 0; + goto out; + } + } else if (!strcmp(attr->attr.name, + "reclaim_percentage")) { + if ((val > -1) && (val < 101)) { + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_NCACHE_RECLAIM_PERCENTAGE; + } else { + rc = 0; + goto out; + } + } else if (!strcmp(attr->attr.name, "timeout_msecs")) { + if (val > -1) { + new_op->upcall.req.param.op = + ORANGEFS_PARAM_REQUEST_OP_NCACHE_TIMEOUT_MSECS; + } else { + rc = 0; + goto out; + } + } + + } else { + gossip_err("sysfs_service_op_store: unknown kobj_id:%s:\n", + kobj->name); + rc = -EINVAL; + goto out; + } + + new_op->upcall.req.param.u.value64 = val; +value_set: + + /* + * The service_operation will return a errno return code on + * error, and zero on success. + */ + rc = service_operation(new_op, "orangefs_param", ORANGEFS_OP_INTERRUPTIBLE); + + if (rc < 0) { + gossip_err("sysfs_service_op_store: service op returned:%d:\n", + rc); + rc = 0; + } else { + rc = count; + } + +out: + op_release(new_op); + + if (rc == -ENOMEM || rc == 0) + rc = -EINVAL; + + return rc; +} + +static struct orangefs_attribute op_timeout_secs_attribute = + __ATTR(op_timeout_secs, 0664, sysfs_int_show, sysfs_int_store); + +static struct orangefs_attribute slot_timeout_secs_attribute = + __ATTR(slot_timeout_secs, 0664, sysfs_int_show, sysfs_int_store); + +static struct orangefs_attribute cache_timeout_msecs_attribute = + __ATTR(cache_timeout_msecs, 0664, sysfs_int_show, sysfs_int_store); + +static struct orangefs_attribute dcache_timeout_msecs_attribute = + __ATTR(dcache_timeout_msecs, 0664, sysfs_int_show, sysfs_int_store); + +static struct orangefs_attribute getattr_timeout_msecs_attribute = + __ATTR(getattr_timeout_msecs, 0664, sysfs_int_show, sysfs_int_store); + +static struct orangefs_attribute readahead_count_attribute = + __ATTR(readahead_count, 0664, sysfs_service_op_show, + sysfs_service_op_store); + +static struct orangefs_attribute readahead_size_attribute = + __ATTR(readahead_size, 0664, sysfs_service_op_show, + sysfs_service_op_store); + +static struct orangefs_attribute readahead_count_size_attribute = + __ATTR(readahead_count_size, 0664, sysfs_service_op_show, + sysfs_service_op_store); + +static struct orangefs_attribute readahead_readcnt_attribute = + __ATTR(readahead_readcnt, 0664, sysfs_service_op_show, + sysfs_service_op_store); + +static struct orangefs_attribute perf_counter_reset_attribute = + __ATTR(perf_counter_reset, + 0664, + sysfs_service_op_show, + sysfs_service_op_store); + +static struct orangefs_attribute perf_history_size_attribute = + __ATTR(perf_history_size, + 0664, + sysfs_service_op_show, + sysfs_service_op_store); + +static struct orangefs_attribute perf_time_interval_secs_attribute = + __ATTR(perf_time_interval_secs, + 0664, + sysfs_service_op_show, + sysfs_service_op_store); + +static struct attribute *orangefs_default_attrs[] = { + &op_timeout_secs_attribute.attr, + &slot_timeout_secs_attribute.attr, + &cache_timeout_msecs_attribute.attr, + &dcache_timeout_msecs_attribute.attr, + &getattr_timeout_msecs_attribute.attr, + &readahead_count_attribute.attr, + &readahead_size_attribute.attr, + &readahead_count_size_attribute.attr, + &readahead_readcnt_attribute.attr, + &perf_counter_reset_attribute.attr, + &perf_history_size_attribute.attr, + &perf_time_interval_secs_attribute.attr, + NULL, +}; +ATTRIBUTE_GROUPS(orangefs_default); + +static struct kobject *orangefs_obj; + +static void orangefs_obj_release(struct kobject *kobj) +{ + kfree(orangefs_obj); + orangefs_obj = NULL; +} + +static const struct kobj_type orangefs_ktype = { + .sysfs_ops = &orangefs_sysfs_ops, + .default_groups = orangefs_default_groups, + .release = orangefs_obj_release, +}; + +static struct orangefs_attribute acache_hard_limit_attribute = + __ATTR(hard_limit, + 0664, + sysfs_service_op_show, + sysfs_service_op_store); + +static struct orangefs_attribute acache_reclaim_percent_attribute = + __ATTR(reclaim_percentage, + 0664, + sysfs_service_op_show, + sysfs_service_op_store); + +static struct orangefs_attribute acache_soft_limit_attribute = + __ATTR(soft_limit, + 0664, + sysfs_service_op_show, + sysfs_service_op_store); + +static struct orangefs_attribute acache_timeout_msecs_attribute = + __ATTR(timeout_msecs, + 0664, + sysfs_service_op_show, + sysfs_service_op_store); + +static struct attribute *acache_orangefs_default_attrs[] = { + &acache_hard_limit_attribute.attr, + &acache_reclaim_percent_attribute.attr, + &acache_soft_limit_attribute.attr, + &acache_timeout_msecs_attribute.attr, + NULL, +}; +ATTRIBUTE_GROUPS(acache_orangefs_default); + +static struct kobject *acache_orangefs_obj; + +static void acache_orangefs_obj_release(struct kobject *kobj) +{ + kfree(acache_orangefs_obj); + acache_orangefs_obj = NULL; +} + +static const struct kobj_type acache_orangefs_ktype = { + .sysfs_ops = &orangefs_sysfs_ops, + .default_groups = acache_orangefs_default_groups, + .release = acache_orangefs_obj_release, +}; + +static struct orangefs_attribute capcache_hard_limit_attribute = + __ATTR(hard_limit, + 0664, + sysfs_service_op_show, + sysfs_service_op_store); + +static struct orangefs_attribute capcache_reclaim_percent_attribute = + __ATTR(reclaim_percentage, + 0664, + sysfs_service_op_show, + sysfs_service_op_store); + +static struct orangefs_attribute capcache_soft_limit_attribute = + __ATTR(soft_limit, + 0664, + sysfs_service_op_show, + sysfs_service_op_store); + +static struct orangefs_attribute capcache_timeout_secs_attribute = + __ATTR(timeout_secs, + 0664, + sysfs_service_op_show, + sysfs_service_op_store); + +static struct attribute *capcache_orangefs_default_attrs[] = { + &capcache_hard_limit_attribute.attr, + &capcache_reclaim_percent_attribute.attr, + &capcache_soft_limit_attribute.attr, + &capcache_timeout_secs_attribute.attr, + NULL, +}; +ATTRIBUTE_GROUPS(capcache_orangefs_default); + +static struct kobject *capcache_orangefs_obj; + +static void capcache_orangefs_obj_release(struct kobject *kobj) +{ + kfree(capcache_orangefs_obj); + capcache_orangefs_obj = NULL; +} + +static const struct kobj_type capcache_orangefs_ktype = { + .sysfs_ops = &orangefs_sysfs_ops, + .default_groups = capcache_orangefs_default_groups, + .release = capcache_orangefs_obj_release, +}; + +static struct orangefs_attribute ccache_hard_limit_attribute = + __ATTR(hard_limit, + 0664, + sysfs_service_op_show, + sysfs_service_op_store); + +static struct orangefs_attribute ccache_reclaim_percent_attribute = + __ATTR(reclaim_percentage, + 0664, + sysfs_service_op_show, + sysfs_service_op_store); + +static struct orangefs_attribute ccache_soft_limit_attribute = + __ATTR(soft_limit, + 0664, + sysfs_service_op_show, + sysfs_service_op_store); + +static struct orangefs_attribute ccache_timeout_secs_attribute = + __ATTR(timeout_secs, + 0664, + sysfs_service_op_show, + sysfs_service_op_store); + +static struct attribute *ccache_orangefs_default_attrs[] = { + &ccache_hard_limit_attribute.attr, + &ccache_reclaim_percent_attribute.attr, + &ccache_soft_limit_attribute.attr, + &ccache_timeout_secs_attribute.attr, + NULL, +}; +ATTRIBUTE_GROUPS(ccache_orangefs_default); + +static struct kobject *ccache_orangefs_obj; + +static void ccache_orangefs_obj_release(struct kobject *kobj) +{ + kfree(ccache_orangefs_obj); + ccache_orangefs_obj = NULL; +} + +static const struct kobj_type ccache_orangefs_ktype = { + .sysfs_ops = &orangefs_sysfs_ops, + .default_groups = ccache_orangefs_default_groups, + .release = ccache_orangefs_obj_release, +}; + +static struct orangefs_attribute ncache_hard_limit_attribute = + __ATTR(hard_limit, + 0664, + sysfs_service_op_show, + sysfs_service_op_store); + +static struct orangefs_attribute ncache_reclaim_percent_attribute = + __ATTR(reclaim_percentage, + 0664, + sysfs_service_op_show, + sysfs_service_op_store); + +static struct orangefs_attribute ncache_soft_limit_attribute = + __ATTR(soft_limit, + 0664, + sysfs_service_op_show, + sysfs_service_op_store); + +static struct orangefs_attribute ncache_timeout_msecs_attribute = + __ATTR(timeout_msecs, + 0664, + sysfs_service_op_show, + sysfs_service_op_store); + +static struct attribute *ncache_orangefs_default_attrs[] = { + &ncache_hard_limit_attribute.attr, + &ncache_reclaim_percent_attribute.attr, + &ncache_soft_limit_attribute.attr, + &ncache_timeout_msecs_attribute.attr, + NULL, +}; +ATTRIBUTE_GROUPS(ncache_orangefs_default); + +static struct kobject *ncache_orangefs_obj; + +static void ncache_orangefs_obj_release(struct kobject *kobj) +{ + kfree(ncache_orangefs_obj); + ncache_orangefs_obj = NULL; +} + +static const struct kobj_type ncache_orangefs_ktype = { + .sysfs_ops = &orangefs_sysfs_ops, + .default_groups = ncache_orangefs_default_groups, + .release = ncache_orangefs_obj_release, +}; + +static struct orangefs_attribute pc_acache_attribute = + __ATTR(acache, + 0664, + sysfs_service_op_show, + NULL); + +static struct orangefs_attribute pc_capcache_attribute = + __ATTR(capcache, + 0664, + sysfs_service_op_show, + NULL); + +static struct orangefs_attribute pc_ncache_attribute = + __ATTR(ncache, + 0664, + sysfs_service_op_show, + NULL); + +static struct attribute *pc_orangefs_default_attrs[] = { + &pc_acache_attribute.attr, + &pc_capcache_attribute.attr, + &pc_ncache_attribute.attr, + NULL, +}; +ATTRIBUTE_GROUPS(pc_orangefs_default); + +static struct kobject *pc_orangefs_obj; + +static void pc_orangefs_obj_release(struct kobject *kobj) +{ + kfree(pc_orangefs_obj); + pc_orangefs_obj = NULL; +} + +static const struct kobj_type pc_orangefs_ktype = { + .sysfs_ops = &orangefs_sysfs_ops, + .default_groups = pc_orangefs_default_groups, + .release = pc_orangefs_obj_release, +}; + +static struct orangefs_attribute stats_reads_attribute = + __ATTR(reads, + 0664, + sysfs_int_show, + NULL); + +static struct orangefs_attribute stats_writes_attribute = + __ATTR(writes, + 0664, + sysfs_int_show, + NULL); + +static struct attribute *stats_orangefs_default_attrs[] = { + &stats_reads_attribute.attr, + &stats_writes_attribute.attr, + NULL, +}; +ATTRIBUTE_GROUPS(stats_orangefs_default); + +static struct kobject *stats_orangefs_obj; + +static void stats_orangefs_obj_release(struct kobject *kobj) +{ + kfree(stats_orangefs_obj); + stats_orangefs_obj = NULL; +} + +static const struct kobj_type stats_orangefs_ktype = { + .sysfs_ops = &orangefs_sysfs_ops, + .default_groups = stats_orangefs_default_groups, + .release = stats_orangefs_obj_release, +}; + +int orangefs_sysfs_init(void) +{ + int rc = -EINVAL; + + gossip_debug(GOSSIP_SYSFS_DEBUG, "orangefs_sysfs_init: start\n"); + + /* create /sys/fs/orangefs. */ + orangefs_obj = kzalloc_obj(*orangefs_obj); + if (!orangefs_obj) + goto out; + + rc = kobject_init_and_add(orangefs_obj, + &orangefs_ktype, + fs_kobj, + ORANGEFS_KOBJ_ID); + + if (rc) + goto ofs_obj_bail; + + kobject_uevent(orangefs_obj, KOBJ_ADD); + + /* create /sys/fs/orangefs/acache. */ + acache_orangefs_obj = kzalloc_obj(*acache_orangefs_obj); + if (!acache_orangefs_obj) { + rc = -EINVAL; + goto ofs_obj_bail; + } + + rc = kobject_init_and_add(acache_orangefs_obj, + &acache_orangefs_ktype, + orangefs_obj, + ACACHE_KOBJ_ID); + + if (rc) + goto acache_obj_bail; + + kobject_uevent(acache_orangefs_obj, KOBJ_ADD); + + /* create /sys/fs/orangefs/capcache. */ + capcache_orangefs_obj = kzalloc_obj(*capcache_orangefs_obj); + if (!capcache_orangefs_obj) { + rc = -EINVAL; + goto acache_obj_bail; + } + + rc = kobject_init_and_add(capcache_orangefs_obj, + &capcache_orangefs_ktype, + orangefs_obj, + CAPCACHE_KOBJ_ID); + if (rc) + goto capcache_obj_bail; + + kobject_uevent(capcache_orangefs_obj, KOBJ_ADD); + + /* create /sys/fs/orangefs/ccache. */ + ccache_orangefs_obj = kzalloc_obj(*ccache_orangefs_obj); + if (!ccache_orangefs_obj) { + rc = -EINVAL; + goto capcache_obj_bail; + } + + rc = kobject_init_and_add(ccache_orangefs_obj, + &ccache_orangefs_ktype, + orangefs_obj, + CCACHE_KOBJ_ID); + if (rc) + goto ccache_obj_bail; + + kobject_uevent(ccache_orangefs_obj, KOBJ_ADD); + + /* create /sys/fs/orangefs/ncache. */ + ncache_orangefs_obj = kzalloc_obj(*ncache_orangefs_obj); + if (!ncache_orangefs_obj) { + rc = -EINVAL; + goto ccache_obj_bail; + } + + rc = kobject_init_and_add(ncache_orangefs_obj, + &ncache_orangefs_ktype, + orangefs_obj, + NCACHE_KOBJ_ID); + + if (rc) + goto ncache_obj_bail; + + kobject_uevent(ncache_orangefs_obj, KOBJ_ADD); + + /* create /sys/fs/orangefs/perf_counters. */ + pc_orangefs_obj = kzalloc_obj(*pc_orangefs_obj); + if (!pc_orangefs_obj) { + rc = -EINVAL; + goto ncache_obj_bail; + } + + rc = kobject_init_and_add(pc_orangefs_obj, + &pc_orangefs_ktype, + orangefs_obj, + "perf_counters"); + + if (rc) + goto pc_obj_bail; + + kobject_uevent(pc_orangefs_obj, KOBJ_ADD); + + /* create /sys/fs/orangefs/stats. */ + stats_orangefs_obj = kzalloc_obj(*stats_orangefs_obj); + if (!stats_orangefs_obj) { + rc = -EINVAL; + goto pc_obj_bail; + } + + rc = kobject_init_and_add(stats_orangefs_obj, + &stats_orangefs_ktype, + orangefs_obj, + STATS_KOBJ_ID); + + if (rc) + goto stats_obj_bail; + + kobject_uevent(stats_orangefs_obj, KOBJ_ADD); + goto out; + +stats_obj_bail: + kobject_put(stats_orangefs_obj); +pc_obj_bail: + kobject_put(pc_orangefs_obj); +ncache_obj_bail: + kobject_put(ncache_orangefs_obj); +ccache_obj_bail: + kobject_put(ccache_orangefs_obj); +capcache_obj_bail: + kobject_put(capcache_orangefs_obj); +acache_obj_bail: + kobject_put(acache_orangefs_obj); +ofs_obj_bail: + kobject_put(orangefs_obj); +out: + return rc; +} + +void orangefs_sysfs_exit(void) +{ + gossip_debug(GOSSIP_SYSFS_DEBUG, "orangefs_sysfs_exit: start\n"); + kobject_put(acache_orangefs_obj); + kobject_put(capcache_orangefs_obj); + kobject_put(ccache_orangefs_obj); + kobject_put(ncache_orangefs_obj); + kobject_put(pc_orangefs_obj); + kobject_put(stats_orangefs_obj); + kobject_put(orangefs_obj); +} diff --git a/fs/orangefs/orangefs-sysfs.h b/fs/orangefs/orangefs-sysfs.h new file mode 100644 index 000000000..f0b76382d --- /dev/null +++ b/fs/orangefs/orangefs-sysfs.h @@ -0,0 +1,2 @@ +extern int orangefs_sysfs_init(void); +extern void orangefs_sysfs_exit(void); diff --git a/fs/orangefs/orangefs-utils.c b/fs/orangefs/orangefs-utils.c new file mode 100644 index 000000000..9ab1119eb --- /dev/null +++ b/fs/orangefs/orangefs-utils.c @@ -0,0 +1,560 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * (C) 2001 Clemson University and The University of Chicago + * Copyright 2018 Omnibond Systems, L.L.C. + * + * See COPYING in top-level directory. + */ +#include +#include "protocol.h" +#include "orangefs-kernel.h" +#include "orangefs-dev-proto.h" +#include "orangefs-bufmap.h" + +__s32 fsid_of_op(struct orangefs_kernel_op_s *op) +{ + __s32 fsid = ORANGEFS_FS_ID_NULL; + + if (op) { + switch (op->upcall.type) { + case ORANGEFS_VFS_OP_FILE_IO: + fsid = op->upcall.req.io.refn.fs_id; + break; + case ORANGEFS_VFS_OP_LOOKUP: + fsid = op->upcall.req.lookup.parent_refn.fs_id; + break; + case ORANGEFS_VFS_OP_CREATE: + fsid = op->upcall.req.create.parent_refn.fs_id; + break; + case ORANGEFS_VFS_OP_GETATTR: + fsid = op->upcall.req.getattr.refn.fs_id; + break; + case ORANGEFS_VFS_OP_REMOVE: + fsid = op->upcall.req.remove.parent_refn.fs_id; + break; + case ORANGEFS_VFS_OP_MKDIR: + fsid = op->upcall.req.mkdir.parent_refn.fs_id; + break; + case ORANGEFS_VFS_OP_READDIR: + fsid = op->upcall.req.readdir.refn.fs_id; + break; + case ORANGEFS_VFS_OP_SETATTR: + fsid = op->upcall.req.setattr.refn.fs_id; + break; + case ORANGEFS_VFS_OP_SYMLINK: + fsid = op->upcall.req.sym.parent_refn.fs_id; + break; + case ORANGEFS_VFS_OP_RENAME: + fsid = op->upcall.req.rename.old_parent_refn.fs_id; + break; + case ORANGEFS_VFS_OP_STATFS: + fsid = op->upcall.req.statfs.fs_id; + break; + case ORANGEFS_VFS_OP_TRUNCATE: + fsid = op->upcall.req.truncate.refn.fs_id; + break; + case ORANGEFS_VFS_OP_RA_FLUSH: + fsid = op->upcall.req.ra_cache_flush.refn.fs_id; + break; + case ORANGEFS_VFS_OP_FS_UMOUNT: + fsid = op->upcall.req.fs_umount.fs_id; + break; + case ORANGEFS_VFS_OP_GETXATTR: + fsid = op->upcall.req.getxattr.refn.fs_id; + break; + case ORANGEFS_VFS_OP_SETXATTR: + fsid = op->upcall.req.setxattr.refn.fs_id; + break; + case ORANGEFS_VFS_OP_LISTXATTR: + fsid = op->upcall.req.listxattr.refn.fs_id; + break; + case ORANGEFS_VFS_OP_REMOVEXATTR: + fsid = op->upcall.req.removexattr.refn.fs_id; + break; + case ORANGEFS_VFS_OP_FSYNC: + fsid = op->upcall.req.fsync.refn.fs_id; + break; + default: + break; + } + } + return fsid; +} + +static int orangefs_inode_flags(struct ORANGEFS_sys_attr_s *attrs) +{ + int flags = 0; + if (attrs->flags & ORANGEFS_IMMUTABLE_FL) + flags |= S_IMMUTABLE; + else + flags &= ~S_IMMUTABLE; + if (attrs->flags & ORANGEFS_APPEND_FL) + flags |= S_APPEND; + else + flags &= ~S_APPEND; + if (attrs->flags & ORANGEFS_NOATIME_FL) + flags |= S_NOATIME; + else + flags &= ~S_NOATIME; + return flags; +} + +static int orangefs_inode_perms(struct ORANGEFS_sys_attr_s *attrs) +{ + int perm_mode = 0; + + if (attrs->perms & ORANGEFS_O_EXECUTE) + perm_mode |= S_IXOTH; + if (attrs->perms & ORANGEFS_O_WRITE) + perm_mode |= S_IWOTH; + if (attrs->perms & ORANGEFS_O_READ) + perm_mode |= S_IROTH; + + if (attrs->perms & ORANGEFS_G_EXECUTE) + perm_mode |= S_IXGRP; + if (attrs->perms & ORANGEFS_G_WRITE) + perm_mode |= S_IWGRP; + if (attrs->perms & ORANGEFS_G_READ) + perm_mode |= S_IRGRP; + + if (attrs->perms & ORANGEFS_U_EXECUTE) + perm_mode |= S_IXUSR; + if (attrs->perms & ORANGEFS_U_WRITE) + perm_mode |= S_IWUSR; + if (attrs->perms & ORANGEFS_U_READ) + perm_mode |= S_IRUSR; + + if (attrs->perms & ORANGEFS_G_SGID) + perm_mode |= S_ISGID; + if (attrs->perms & ORANGEFS_U_SUID) + perm_mode |= S_ISUID; + + return perm_mode; +} + +/* + * NOTE: in kernel land, we never use the sys_attr->link_target for + * anything, so don't bother copying it into the sys_attr object here. + */ +static inline void copy_attributes_from_inode(struct inode *inode, + struct ORANGEFS_sys_attr_s *attrs) +{ + struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode); + attrs->mask = 0; + if (orangefs_inode->attr_valid & ATTR_UID) { + attrs->owner = from_kuid(&init_user_ns, inode->i_uid); + attrs->mask |= ORANGEFS_ATTR_SYS_UID; + gossip_debug(GOSSIP_UTILS_DEBUG, "(UID) %d\n", attrs->owner); + } + if (orangefs_inode->attr_valid & ATTR_GID) { + attrs->group = from_kgid(&init_user_ns, inode->i_gid); + attrs->mask |= ORANGEFS_ATTR_SYS_GID; + gossip_debug(GOSSIP_UTILS_DEBUG, "(GID) %d\n", attrs->group); + } + + if (orangefs_inode->attr_valid & ATTR_ATIME) { + attrs->mask |= ORANGEFS_ATTR_SYS_ATIME; + if (orangefs_inode->attr_valid & ATTR_ATIME_SET) { + attrs->atime = (time64_t) inode_get_atime_sec(inode); + attrs->mask |= ORANGEFS_ATTR_SYS_ATIME_SET; + } + } + if (orangefs_inode->attr_valid & ATTR_MTIME) { + attrs->mask |= ORANGEFS_ATTR_SYS_MTIME; + if (orangefs_inode->attr_valid & ATTR_MTIME_SET) { + attrs->mtime = (time64_t) inode_get_mtime_sec(inode); + attrs->mask |= ORANGEFS_ATTR_SYS_MTIME_SET; + } + } + if (orangefs_inode->attr_valid & ATTR_CTIME) + attrs->mask |= ORANGEFS_ATTR_SYS_CTIME; + + /* + * ORANGEFS cannot set size with a setattr operation. Probably not + * likely to be requested through the VFS, but just in case, don't + * worry about ATTR_SIZE + */ + + if (orangefs_inode->attr_valid & ATTR_MODE) { + attrs->perms = ORANGEFS_util_translate_mode(inode->i_mode); + attrs->mask |= ORANGEFS_ATTR_SYS_PERM; + } +} + +static int orangefs_inode_type(enum orangefs_ds_type objtype) +{ + if (objtype == ORANGEFS_TYPE_METAFILE) + return S_IFREG; + else if (objtype == ORANGEFS_TYPE_DIRECTORY) + return S_IFDIR; + else if (objtype == ORANGEFS_TYPE_SYMLINK) + return S_IFLNK; + else + return -1; +} + +static void orangefs_make_bad_inode(struct inode *inode) +{ + if (is_root_handle(inode)) { + /* + * if this occurs, the pvfs2-client-core was killed but we + * can't afford to lose the inode operations and such + * associated with the root handle in any case. + */ + gossip_debug(GOSSIP_UTILS_DEBUG, + "*** NOT making bad root inode %pU\n", + get_khandle_from_ino(inode)); + } else { + gossip_debug(GOSSIP_UTILS_DEBUG, + "*** making bad inode %pU\n", + get_khandle_from_ino(inode)); + make_bad_inode(inode); + } +} + +static int orangefs_inode_is_stale(struct inode *inode, + struct ORANGEFS_sys_attr_s *attrs, char *link_target) +{ + struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode); + int type = orangefs_inode_type(attrs->objtype); + /* + * If the inode type or symlink target have changed then this + * inode is stale. + */ + if (type == -1 || inode_wrong_type(inode, type)) { + orangefs_make_bad_inode(inode); + return 1; + } + if (type == S_IFLNK && strncmp(orangefs_inode->link_target, + link_target, ORANGEFS_NAME_MAX)) { + orangefs_make_bad_inode(inode); + return 1; + } + return 0; +} + +int orangefs_inode_getattr(struct inode *inode, int flags) +{ + struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode); + struct orangefs_kernel_op_s *new_op; + loff_t inode_size; + int ret, type; + + gossip_debug(GOSSIP_UTILS_DEBUG, "%s: called on inode %pU flags %d\n", + __func__, get_khandle_from_ino(inode), flags); + +again: + spin_lock(&inode->i_lock); + /* Must have all the attributes in the mask and be within cache time. */ + if ((!flags && time_before(jiffies, orangefs_inode->getattr_time)) || + orangefs_inode->attr_valid || inode_state_read(inode) & I_DIRTY_PAGES) { + if (orangefs_inode->attr_valid) { + spin_unlock(&inode->i_lock); + write_inode_now(inode, 1); + goto again; + } + spin_unlock(&inode->i_lock); + return 0; + } + spin_unlock(&inode->i_lock); + + new_op = op_alloc(ORANGEFS_VFS_OP_GETATTR); + if (!new_op) + return -ENOMEM; + new_op->upcall.req.getattr.refn = orangefs_inode->refn; + /* + * Size is the hardest attribute to get. The incremental cost of any + * other attribute is essentially zero. + */ + if (flags) + new_op->upcall.req.getattr.mask = ORANGEFS_ATTR_SYS_ALL_NOHINT; + else + new_op->upcall.req.getattr.mask = + ORANGEFS_ATTR_SYS_ALL_NOHINT & ~ORANGEFS_ATTR_SYS_SIZE; + + ret = service_operation(new_op, __func__, + get_interruptible_flag(inode)); + if (ret != 0) + goto out; + +again2: + spin_lock(&inode->i_lock); + /* Must have all the attributes in the mask and be within cache time. */ + if ((!flags && time_before(jiffies, orangefs_inode->getattr_time)) || + orangefs_inode->attr_valid || inode_state_read(inode) & I_DIRTY_PAGES) { + if (orangefs_inode->attr_valid) { + spin_unlock(&inode->i_lock); + write_inode_now(inode, 1); + goto again2; + } + if (inode_state_read(inode) & I_DIRTY_PAGES) { + ret = 0; + goto out_unlock; + } + gossip_debug(GOSSIP_UTILS_DEBUG, "%s: in cache or dirty\n", + __func__); + ret = 0; + goto out_unlock; + } + + if (!(flags & ORANGEFS_GETATTR_NEW)) { + ret = orangefs_inode_is_stale(inode, + &new_op->downcall.resp.getattr.attributes, + new_op->downcall.resp.getattr.link_target); + if (ret) { + ret = -ESTALE; + goto out_unlock; + } + } + + type = orangefs_inode_type(new_op-> + downcall.resp.getattr.attributes.objtype); + switch (type) { + case S_IFREG: + inode->i_flags = orangefs_inode_flags(&new_op-> + downcall.resp.getattr.attributes); + if (flags) { + inode_size = (loff_t)new_op-> + downcall.resp.getattr.attributes.size; + inode->i_size = inode_size; + inode->i_blkbits = ffs(new_op->downcall.resp.getattr. + attributes.blksize); + inode->i_bytes = inode_size; + inode->i_blocks = + (inode_size + 512 - inode_size % 512)/512; + } + break; + case S_IFDIR: + if (flags) { + inode->i_size = PAGE_SIZE; + inode_set_bytes(inode, inode->i_size); + } + set_nlink(inode, 1); + break; + case S_IFLNK: + if (flags & ORANGEFS_GETATTR_NEW) { + inode->i_size = (loff_t)strlen(new_op-> + downcall.resp.getattr.link_target); + ret = strscpy(orangefs_inode->link_target, + new_op->downcall.resp.getattr.link_target, + ORANGEFS_NAME_MAX); + if (ret == -E2BIG) { + ret = -EIO; + goto out_unlock; + } + inode->i_link = orangefs_inode->link_target; + } + break; + /* i.e. -1 */ + default: + /* XXX: ESTALE? This is what is done if it is not new. */ + orangefs_make_bad_inode(inode); + ret = -ESTALE; + goto out_unlock; + } + + inode->i_uid = make_kuid(&init_user_ns, new_op-> + downcall.resp.getattr.attributes.owner); + inode->i_gid = make_kgid(&init_user_ns, new_op-> + downcall.resp.getattr.attributes.group); + inode_set_atime(inode, + (time64_t)new_op->downcall.resp.getattr.attributes.atime, + 0); + inode_set_mtime(inode, + (time64_t)new_op->downcall.resp.getattr.attributes.mtime, + 0); + inode_set_ctime(inode, + (time64_t)new_op->downcall.resp.getattr.attributes.ctime, + 0); + + /* special case: mark the root inode as sticky */ + inode->i_mode = type | (is_root_handle(inode) ? S_ISVTX : 0) | + orangefs_inode_perms(&new_op->downcall.resp.getattr.attributes); + + orangefs_inode->getattr_time = jiffies + + orangefs_getattr_timeout_msecs*HZ/1000; + ret = 0; +out_unlock: + spin_unlock(&inode->i_lock); +out: + op_release(new_op); + return ret; +} + +int orangefs_inode_check_changed(struct inode *inode) +{ + struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode); + struct orangefs_kernel_op_s *new_op; + int ret; + + gossip_debug(GOSSIP_UTILS_DEBUG, "%s: called on inode %pU\n", __func__, + get_khandle_from_ino(inode)); + + new_op = op_alloc(ORANGEFS_VFS_OP_GETATTR); + if (!new_op) + return -ENOMEM; + new_op->upcall.req.getattr.refn = orangefs_inode->refn; + new_op->upcall.req.getattr.mask = ORANGEFS_ATTR_SYS_TYPE | + ORANGEFS_ATTR_SYS_LNK_TARGET; + + ret = service_operation(new_op, __func__, + get_interruptible_flag(inode)); + if (ret != 0) + goto out; + + ret = orangefs_inode_is_stale(inode, + &new_op->downcall.resp.getattr.attributes, + new_op->downcall.resp.getattr.link_target); +out: + op_release(new_op); + return ret; +} + +/* + * issues a orangefs setattr request to make sure the new attribute values + * take effect if successful. returns 0 on success; -errno otherwise + */ +int orangefs_inode_setattr(struct inode *inode) +{ + struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode); + struct orangefs_kernel_op_s *new_op; + int ret; + + new_op = op_alloc(ORANGEFS_VFS_OP_SETATTR); + if (!new_op) + return -ENOMEM; + + spin_lock(&inode->i_lock); + new_op->upcall.uid = from_kuid(&init_user_ns, orangefs_inode->attr_uid); + new_op->upcall.gid = from_kgid(&init_user_ns, orangefs_inode->attr_gid); + new_op->upcall.req.setattr.refn = orangefs_inode->refn; + copy_attributes_from_inode(inode, + &new_op->upcall.req.setattr.attributes); + orangefs_inode->attr_valid = 0; + if (!new_op->upcall.req.setattr.attributes.mask) { + spin_unlock(&inode->i_lock); + op_release(new_op); + return 0; + } + spin_unlock(&inode->i_lock); + + ret = service_operation(new_op, __func__, + get_interruptible_flag(inode) | ORANGEFS_OP_WRITEBACK); + gossip_debug(GOSSIP_UTILS_DEBUG, + "orangefs_inode_setattr: returning %d\n", ret); + if (ret) + orangefs_make_bad_inode(inode); + + op_release(new_op); + + if (ret == 0) + orangefs_inode->getattr_time = jiffies - 1; + return ret; +} + +/* + * The following is a very dirty hack that is now a permanent part of the + * ORANGEFS protocol. See protocol.h for more error definitions. + */ + +/* The order matches include/orangefs-types.h in the OrangeFS source. */ +static int PINT_errno_mapping[] = { + 0, EPERM, ENOENT, EINTR, EIO, ENXIO, EBADF, EAGAIN, ENOMEM, + EFAULT, EBUSY, EEXIST, ENODEV, ENOTDIR, EISDIR, EINVAL, EMFILE, + EFBIG, ENOSPC, EROFS, EMLINK, EPIPE, EDEADLK, ENAMETOOLONG, + ENOLCK, ENOSYS, ENOTEMPTY, ELOOP, EWOULDBLOCK, ENOMSG, EUNATCH, + EBADR, EDEADLOCK, ENODATA, ETIME, ENONET, EREMOTE, ECOMM, + EPROTO, EBADMSG, EOVERFLOW, ERESTART, EMSGSIZE, EPROTOTYPE, + ENOPROTOOPT, EPROTONOSUPPORT, EOPNOTSUPP, EADDRINUSE, + EADDRNOTAVAIL, ENETDOWN, ENETUNREACH, ENETRESET, ENOBUFS, + ETIMEDOUT, ECONNREFUSED, EHOSTDOWN, EHOSTUNREACH, EALREADY, + EACCES, ECONNRESET, ERANGE +}; + +int orangefs_normalize_to_errno(__s32 error_code) +{ + __u32 i; + + /* Success */ + if (error_code == 0) { + return 0; + /* + * This shouldn't ever happen. If it does it should be fixed on the + * server. + */ + } else if (error_code > 0) { + gossip_err("orangefs: error status received.\n"); + gossip_err("orangefs: assuming error code is inverted.\n"); + error_code = -error_code; + } + + /* + * XXX: This is very bad since error codes from ORANGEFS may not be + * suitable for return into userspace. + */ + + /* + * Convert ORANGEFS error values into errno values suitable for return + * from the kernel. + */ + if ((-error_code) & ORANGEFS_NON_ERRNO_ERROR_BIT) { + if (((-error_code) & + (ORANGEFS_ERROR_NUMBER_BITS|ORANGEFS_NON_ERRNO_ERROR_BIT| + ORANGEFS_ERROR_BIT)) == ORANGEFS_ECANCEL) { + /* + * cancellation error codes generally correspond to + * a timeout from the client's perspective + */ + error_code = -ETIMEDOUT; + } else { + /* assume a default error code */ + gossip_err("%s: bad error code :%d:.\n", + __func__, + error_code); + error_code = -EINVAL; + } + + /* Convert ORANGEFS encoded errno values into regular errno values. */ + } else if ((-error_code) & ORANGEFS_ERROR_BIT) { + i = (-error_code) & ~(ORANGEFS_ERROR_BIT|ORANGEFS_ERROR_CLASS_BITS); + if (i < ARRAY_SIZE(PINT_errno_mapping)) + error_code = -PINT_errno_mapping[i]; + else + error_code = -EINVAL; + + /* + * Only ORANGEFS protocol error codes should ever come here. Otherwise + * there is a bug somewhere. + */ + } else { + gossip_err("%s: unknown error code.\n", __func__); + error_code = -EINVAL; + } + return error_code; +} + +#define NUM_MODES 11 +__s32 ORANGEFS_util_translate_mode(int mode) +{ + int ret = 0; + int i = 0; + static int modes[NUM_MODES] = { + S_IXOTH, S_IWOTH, S_IROTH, + S_IXGRP, S_IWGRP, S_IRGRP, + S_IXUSR, S_IWUSR, S_IRUSR, + S_ISGID, S_ISUID + }; + static int orangefs_modes[NUM_MODES] = { + ORANGEFS_O_EXECUTE, ORANGEFS_O_WRITE, ORANGEFS_O_READ, + ORANGEFS_G_EXECUTE, ORANGEFS_G_WRITE, ORANGEFS_G_READ, + ORANGEFS_U_EXECUTE, ORANGEFS_U_WRITE, ORANGEFS_U_READ, + ORANGEFS_G_SGID, ORANGEFS_U_SUID + }; + + for (i = 0; i < NUM_MODES; i++) + if (mode & modes[i]) + ret |= orangefs_modes[i]; + + return ret; +} +#undef NUM_MODES diff --git a/fs/orangefs/protocol.h b/fs/orangefs/protocol.h new file mode 100644 index 000000000..d403cf29a --- /dev/null +++ b/fs/orangefs/protocol.h @@ -0,0 +1,362 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include +#include +#include +#include + +/* khandle stuff ***********************************************************/ + +/* + * The 2.9 core will put 64 bit handles in here like this: + * 1234 0000 0000 5678 + * The 3.0 and beyond cores will put 128 bit handles in here like this: + * 1234 5678 90AB CDEF + * The kernel module will always use the first four bytes and + * the last four bytes as an inum. + */ +struct orangefs_khandle { + unsigned char u[16]; +} __aligned(8); + +/* + * kernel version of an object ref. + */ +struct orangefs_object_kref { + struct orangefs_khandle khandle; + __s32 fs_id; + __s32 __pad1; +}; + +/* + * compare 2 khandles assumes little endian thus from large address to + * small address + */ +static inline int ORANGEFS_khandle_cmp(const struct orangefs_khandle *kh1, + const struct orangefs_khandle *kh2) +{ + int i; + + for (i = 15; i >= 0; i--) { + if (kh1->u[i] > kh2->u[i]) + return 1; + if (kh1->u[i] < kh2->u[i]) + return -1; + } + + return 0; +} + +static inline void ORANGEFS_khandle_to(const struct orangefs_khandle *kh, + void *p, int size) +{ + + memcpy(p, kh->u, 16); + memset(p + 16, 0, size - 16); + +} + +static inline void ORANGEFS_khandle_from(struct orangefs_khandle *kh, + void *p, int size) +{ + memset(kh, 0, 16); + memcpy(kh->u, p, 16); + +} + +/* pvfs2-types.h ************************************************************/ + +#define ORANGEFS_SUPER_MAGIC 0x20030528 + +/* + * ORANGEFS error codes are a signed 32-bit integer. Error codes are negative, but + * the sign is stripped before decoding. + */ + +/* Bit 31 is not used since it is the sign. */ + +/* + * Bit 30 specifies that this is a ORANGEFS error. A ORANGEFS error is either an + * encoded errno value or a ORANGEFS protocol error. + */ +#define ORANGEFS_ERROR_BIT (1 << 30) + +/* + * Bit 29 specifies that this is a ORANGEFS protocol error and not an encoded + * errno value. + */ +#define ORANGEFS_NON_ERRNO_ERROR_BIT (1 << 29) + +/* + * Bits 9, 8, and 7 specify the error class, which encodes the section of + * server code the error originated in for logging purposes. It is not used + * in the kernel except to be masked out. + */ +#define ORANGEFS_ERROR_CLASS_BITS 0x380 + +/* Bits 6 - 0 are reserved for the actual error code. */ +#define ORANGEFS_ERROR_NUMBER_BITS 0x7f + +/* Encoded errno values decoded by PINT_errno_mapping in orangefs-utils.c. */ + +/* Our own ORANGEFS protocol error codes. */ +#define ORANGEFS_ECANCEL (1|ORANGEFS_NON_ERRNO_ERROR_BIT|ORANGEFS_ERROR_BIT) +#define ORANGEFS_EDEVINIT (2|ORANGEFS_NON_ERRNO_ERROR_BIT|ORANGEFS_ERROR_BIT) +#define ORANGEFS_EDETAIL (3|ORANGEFS_NON_ERRNO_ERROR_BIT|ORANGEFS_ERROR_BIT) +#define ORANGEFS_EHOSTNTFD (4|ORANGEFS_NON_ERRNO_ERROR_BIT|ORANGEFS_ERROR_BIT) +#define ORANGEFS_EADDRNTFD (5|ORANGEFS_NON_ERRNO_ERROR_BIT|ORANGEFS_ERROR_BIT) +#define ORANGEFS_ENORECVR (6|ORANGEFS_NON_ERRNO_ERROR_BIT|ORANGEFS_ERROR_BIT) +#define ORANGEFS_ETRYAGAIN (7|ORANGEFS_NON_ERRNO_ERROR_BIT|ORANGEFS_ERROR_BIT) +#define ORANGEFS_ENOTPVFS (8|ORANGEFS_NON_ERRNO_ERROR_BIT|ORANGEFS_ERROR_BIT) +#define ORANGEFS_ESECURITY (9|ORANGEFS_NON_ERRNO_ERROR_BIT|ORANGEFS_ERROR_BIT) + +/* permission bits */ +#define ORANGEFS_O_EXECUTE (1 << 0) +#define ORANGEFS_O_WRITE (1 << 1) +#define ORANGEFS_O_READ (1 << 2) +#define ORANGEFS_G_EXECUTE (1 << 3) +#define ORANGEFS_G_WRITE (1 << 4) +#define ORANGEFS_G_READ (1 << 5) +#define ORANGEFS_U_EXECUTE (1 << 6) +#define ORANGEFS_U_WRITE (1 << 7) +#define ORANGEFS_U_READ (1 << 8) +/* no ORANGEFS_U_VTX (sticky bit) */ +#define ORANGEFS_G_SGID (1 << 10) +#define ORANGEFS_U_SUID (1 << 11) + +#define ORANGEFS_ITERATE_START 2147483646 +#define ORANGEFS_ITERATE_END 2147483645 +#define ORANGEFS_IMMUTABLE_FL FS_IMMUTABLE_FL +#define ORANGEFS_APPEND_FL FS_APPEND_FL +#define ORANGEFS_NOATIME_FL FS_NOATIME_FL +#define ORANGEFS_MIRROR_FL 0x01000000ULL +#define ORANGEFS_FS_ID_NULL ((__s32)0) +#define ORANGEFS_ATTR_SYS_UID (1 << 0) +#define ORANGEFS_ATTR_SYS_GID (1 << 1) +#define ORANGEFS_ATTR_SYS_PERM (1 << 2) +#define ORANGEFS_ATTR_SYS_ATIME (1 << 3) +#define ORANGEFS_ATTR_SYS_CTIME (1 << 4) +#define ORANGEFS_ATTR_SYS_MTIME (1 << 5) +#define ORANGEFS_ATTR_SYS_TYPE (1 << 6) +#define ORANGEFS_ATTR_SYS_ATIME_SET (1 << 7) +#define ORANGEFS_ATTR_SYS_MTIME_SET (1 << 8) +#define ORANGEFS_ATTR_SYS_SIZE (1 << 20) +#define ORANGEFS_ATTR_SYS_LNK_TARGET (1 << 24) +#define ORANGEFS_ATTR_SYS_DFILE_COUNT (1 << 25) +#define ORANGEFS_ATTR_SYS_DIRENT_COUNT (1 << 26) +#define ORANGEFS_ATTR_SYS_BLKSIZE (1 << 28) +#define ORANGEFS_ATTR_SYS_MIRROR_COPIES_COUNT (1 << 29) +#define ORANGEFS_ATTR_SYS_COMMON_ALL \ + (ORANGEFS_ATTR_SYS_UID | \ + ORANGEFS_ATTR_SYS_GID | \ + ORANGEFS_ATTR_SYS_PERM | \ + ORANGEFS_ATTR_SYS_ATIME | \ + ORANGEFS_ATTR_SYS_CTIME | \ + ORANGEFS_ATTR_SYS_MTIME | \ + ORANGEFS_ATTR_SYS_TYPE) + +#define ORANGEFS_ATTR_SYS_ALL_SETABLE \ +(ORANGEFS_ATTR_SYS_COMMON_ALL-ORANGEFS_ATTR_SYS_TYPE) + +#define ORANGEFS_ATTR_SYS_ALL_NOHINT \ + (ORANGEFS_ATTR_SYS_COMMON_ALL | \ + ORANGEFS_ATTR_SYS_SIZE | \ + ORANGEFS_ATTR_SYS_LNK_TARGET | \ + ORANGEFS_ATTR_SYS_DFILE_COUNT | \ + ORANGEFS_ATTR_SYS_MIRROR_COPIES_COUNT | \ + ORANGEFS_ATTR_SYS_DIRENT_COUNT | \ + ORANGEFS_ATTR_SYS_BLKSIZE) + +#define ORANGEFS_XATTR_REPLACE 0x2 +#define ORANGEFS_XATTR_CREATE 0x1 +#define ORANGEFS_MAX_SERVER_ADDR_LEN 256 +#define ORANGEFS_NAME_MAX 256 +/* + * max extended attribute name len as imposed by the VFS and exploited for the + * upcall request types. + * NOTE: Please retain them as multiples of 8 even if you wish to change them + * This is *NECESSARY* for supporting 32 bit user-space binaries on a 64-bit + * kernel. Due to implementation within DBPF, this really needs to be + * ORANGEFS_NAME_MAX, which it was the same value as, but no reason to let it + * break if that changes in the future. + */ +#define ORANGEFS_MAX_XATTR_NAMELEN ORANGEFS_NAME_MAX /* Not the same as + * XATTR_NAME_MAX defined + * by + */ +#define ORANGEFS_MAX_XATTR_VALUELEN 8192 /* Not the same as XATTR_SIZE_MAX + * defined by + */ +#define ORANGEFS_MAX_XATTR_LISTLEN 16 /* Not the same as XATTR_LIST_MAX + * defined by + */ +/* + * ORANGEFS I/O operation types, used in both system and server interfaces. + */ +enum ORANGEFS_io_type { + ORANGEFS_IO_READ = 1, + ORANGEFS_IO_WRITE = 2 +}; + +/* + * If this enum is modified the server parameters related to the precreate pool + * batch and low threshold sizes may need to be modified to reflect this + * change. + */ +enum orangefs_ds_type { + ORANGEFS_TYPE_NONE = 0, + ORANGEFS_TYPE_METAFILE = (1 << 0), + ORANGEFS_TYPE_DATAFILE = (1 << 1), + ORANGEFS_TYPE_DIRECTORY = (1 << 2), + ORANGEFS_TYPE_SYMLINK = (1 << 3), + ORANGEFS_TYPE_DIRDATA = (1 << 4), + ORANGEFS_TYPE_INTERNAL = (1 << 5) /* for the server's private use */ +}; + +/* This structure is used by the VFS-client interaction alone */ +struct ORANGEFS_keyval_pair { + char key[ORANGEFS_MAX_XATTR_NAMELEN]; + __s32 key_sz; /* __s32 for portable, fixed-size structures */ + __s32 val_sz; + char val[ORANGEFS_MAX_XATTR_VALUELEN]; +}; + +/* pvfs2-sysint.h ***********************************************************/ +/* Describes attributes for a file, directory, or symlink. */ +struct ORANGEFS_sys_attr_s { + __u32 owner; + __u32 group; + __u32 perms; + __u64 atime; + __u64 mtime; + __u64 ctime; + __s64 size; + + /* NOTE: caller must free if valid */ + char *link_target; + + /* Changed to __s32 so that size of structure does not change */ + __s32 dfile_count; + + /* Changed to __s32 so that size of structure does not change */ + __s32 distr_dir_servers_initial; + + /* Changed to __s32 so that size of structure does not change */ + __s32 distr_dir_servers_max; + + /* Changed to __s32 so that size of structure does not change */ + __s32 distr_dir_split_size; + + __u32 mirror_copies_count; + + /* NOTE: caller must free if valid */ + char *dist_name; + + /* NOTE: caller must free if valid */ + char *dist_params; + + __s64 dirent_count; + enum orangefs_ds_type objtype; + __u64 flags; + __u32 mask; + __s64 blksize; +}; + +#define ORANGEFS_LOOKUP_LINK_NO_FOLLOW 0 + +/* pint-dev.h ***************************************************************/ + +/* parameter structure used in ORANGEFS_DEV_DEBUG ioctl command */ +struct dev_mask_info_s { + enum { + KERNEL_MASK, + CLIENT_MASK, + } mask_type; + __u64 mask_value; +}; + +struct dev_mask2_info_s { + __u64 mask1_value; + __u64 mask2_value; +}; + +/* pvfs2-util.h *************************************************************/ +__s32 ORANGEFS_util_translate_mode(int mode); + +/* pvfs2-debug.h ************************************************************/ +#include "orangefs-debug.h" + +/* pvfs2-internal.h *********************************************************/ +#define llu(x) (unsigned long long)(x) +#define lld(x) (long long)(x) + +/* pint-dev-shared.h ********************************************************/ +#define ORANGEFS_DEV_MAGIC 'k' + +#define ORANGEFS_READDIR_DEFAULT_DESC_COUNT 5 + +#define DEV_GET_MAGIC 0x1 +#define DEV_GET_MAX_UPSIZE 0x2 +#define DEV_GET_MAX_DOWNSIZE 0x3 +#define DEV_MAP 0x4 +#define DEV_REMOUNT_ALL 0x5 +#define DEV_DEBUG 0x6 +#define DEV_UPSTREAM 0x7 +#define DEV_CLIENT_MASK 0x8 +#define DEV_CLIENT_STRING 0x9 +#define DEV_MAX_NR 0xa + +/* supported ioctls, codes are with respect to user-space */ +enum { + ORANGEFS_DEV_GET_MAGIC = _IOW(ORANGEFS_DEV_MAGIC, DEV_GET_MAGIC, __s32), + ORANGEFS_DEV_GET_MAX_UPSIZE = + _IOW(ORANGEFS_DEV_MAGIC, DEV_GET_MAX_UPSIZE, __s32), + ORANGEFS_DEV_GET_MAX_DOWNSIZE = + _IOW(ORANGEFS_DEV_MAGIC, DEV_GET_MAX_DOWNSIZE, __s32), + ORANGEFS_DEV_MAP = _IO(ORANGEFS_DEV_MAGIC, DEV_MAP), + ORANGEFS_DEV_REMOUNT_ALL = _IO(ORANGEFS_DEV_MAGIC, DEV_REMOUNT_ALL), + ORANGEFS_DEV_DEBUG = _IOR(ORANGEFS_DEV_MAGIC, DEV_DEBUG, __s32), + ORANGEFS_DEV_UPSTREAM = _IOW(ORANGEFS_DEV_MAGIC, DEV_UPSTREAM, int), + ORANGEFS_DEV_CLIENT_MASK = _IOW(ORANGEFS_DEV_MAGIC, + DEV_CLIENT_MASK, + struct dev_mask2_info_s), + ORANGEFS_DEV_CLIENT_STRING = _IOW(ORANGEFS_DEV_MAGIC, + DEV_CLIENT_STRING, + char *), + ORANGEFS_DEV_MAXNR = DEV_MAX_NR, +}; + +/* + * version number for use in communicating between kernel space and user + * space. Zero signifies the upstream version of the kernel module. + */ +#define ORANGEFS_KERNEL_PROTO_VERSION 0 +#define ORANGEFS_MINIMUM_USERSPACE_VERSION 20903 + +/* + * describes memory regions to map in the ORANGEFS_DEV_MAP ioctl. + * NOTE: See devorangefs-req.c for 32 bit compat structure. + * Since this structure has a variable-sized layout that is different + * on 32 and 64 bit platforms, we need to normalize to a 64 bit layout + * on such systems before servicing ioctl calls from user-space binaries + * that may be 32 bit! + */ +struct ORANGEFS_dev_map_desc { + void __user *ptr; + __s32 total_size; + __s32 size; + __s32 count; +}; + +/* gossip.h *****************************************************************/ + +extern __u64 orangefs_gossip_debug_mask; + +/* try to avoid function call overhead by checking masks in macro */ +#define gossip_debug(mask, fmt, ...) \ +do { \ + if (orangefs_gossip_debug_mask & (mask)) \ + printk(KERN_DEBUG fmt, ##__VA_ARGS__); \ +} while (0) + +#define gossip_err pr_err diff --git a/fs/orangefs/super.c b/fs/orangefs/super.c new file mode 100644 index 000000000..4ec7329b4 --- /dev/null +++ b/fs/orangefs/super.c @@ -0,0 +1,664 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * (C) 2001 Clemson University and The University of Chicago + * + * See COPYING in top-level directory. + */ + +#include "protocol.h" +#include "orangefs-kernel.h" +#include "orangefs-bufmap.h" + +#include +#include + +/* a cache for orangefs-inode objects (i.e. orangefs inode private data) */ +static struct kmem_cache *orangefs_inode_cache; + +/* list for storing orangefs specific superblocks in use */ +LIST_HEAD(orangefs_superblocks); + +DEFINE_SPINLOCK(orangefs_superblocks_lock); + +enum { + Opt_acl, + Opt_intr, + Opt_local_lock, +}; + +const struct fs_parameter_spec orangefs_fs_param_spec[] = { + fsparam_flag ("acl", Opt_acl), + fsparam_flag ("intr", Opt_intr), + fsparam_flag ("local_lock", Opt_local_lock), + {} +}; + +uint64_t orangefs_features; + +static int orangefs_show_options(struct seq_file *m, struct dentry *root) +{ + struct orangefs_sb_info_s *orangefs_sb = ORANGEFS_SB(root->d_sb); + + if (root->d_sb->s_flags & SB_POSIXACL) + seq_puts(m, ",acl"); + if (orangefs_sb->flags & ORANGEFS_OPT_INTR) + seq_puts(m, ",intr"); + if (orangefs_sb->flags & ORANGEFS_OPT_LOCAL_LOCK) + seq_puts(m, ",local_lock"); + return 0; +} + +static int orangefs_parse_param(struct fs_context *fc, + struct fs_parameter *param) +{ + struct orangefs_sb_info_s *orangefs_sb = fc->s_fs_info; + struct fs_parse_result result; + int opt; + + opt = fs_parse(fc, orangefs_fs_param_spec, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_acl: + fc->sb_flags |= SB_POSIXACL; + break; + case Opt_intr: + orangefs_sb->flags |= ORANGEFS_OPT_INTR; + break; + case Opt_local_lock: + orangefs_sb->flags |= ORANGEFS_OPT_LOCAL_LOCK; + break; + } + + return 0; +} + +static void orangefs_inode_cache_ctor(void *req) +{ + struct orangefs_inode_s *orangefs_inode = req; + + inode_init_once(&orangefs_inode->vfs_inode); + init_rwsem(&orangefs_inode->xattr_sem); +} + +static struct inode *orangefs_alloc_inode(struct super_block *sb) +{ + struct orangefs_inode_s *orangefs_inode; + + orangefs_inode = alloc_inode_sb(sb, orangefs_inode_cache, GFP_KERNEL); + if (!orangefs_inode) + return NULL; + + /* + * We want to clear everything except for rw_semaphore and the + * vfs_inode. + */ + memset(&orangefs_inode->refn.khandle, 0, 16); + orangefs_inode->refn.fs_id = ORANGEFS_FS_ID_NULL; + orangefs_inode->last_failed_block_index_read = 0; + memset(orangefs_inode->link_target, 0, sizeof(orangefs_inode->link_target)); + + gossip_debug(GOSSIP_SUPER_DEBUG, + "orangefs_alloc_inode: allocated %p\n", + &orangefs_inode->vfs_inode); + return &orangefs_inode->vfs_inode; +} + +static void orangefs_free_inode(struct inode *inode) +{ + struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode); + struct orangefs_cached_xattr *cx; + struct hlist_node *tmp; + int i; + + hash_for_each_safe(orangefs_inode->xattr_cache, i, tmp, cx, node) { + hlist_del(&cx->node); + kfree(cx); + } + + kmem_cache_free(orangefs_inode_cache, orangefs_inode); +} + +static void orangefs_destroy_inode(struct inode *inode) +{ + struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode); + + gossip_debug(GOSSIP_SUPER_DEBUG, + "%s: deallocated %p destroying inode %pU\n", + __func__, orangefs_inode, get_khandle_from_ino(inode)); +} + +static int orangefs_write_inode(struct inode *inode, + struct writeback_control *wbc) +{ + gossip_debug(GOSSIP_SUPER_DEBUG, "orangefs_write_inode\n"); + return orangefs_inode_setattr(inode); +} + +/* + * NOTE: information filled in here is typically reflected in the + * output of the system command 'df' +*/ +static int orangefs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + int ret = -ENOMEM; + struct orangefs_kernel_op_s *new_op = NULL; + int flags = 0; + struct super_block *sb = NULL; + + sb = dentry->d_sb; + + gossip_debug(GOSSIP_SUPER_DEBUG, + "%s: called on sb %p (fs_id is %d)\n", + __func__, + sb, + (int)(ORANGEFS_SB(sb)->fs_id)); + + new_op = op_alloc(ORANGEFS_VFS_OP_STATFS); + if (!new_op) + return ret; + new_op->upcall.req.statfs.fs_id = ORANGEFS_SB(sb)->fs_id; + + if (ORANGEFS_SB(sb)->flags & ORANGEFS_OPT_INTR) + flags = ORANGEFS_OP_INTERRUPTIBLE; + + ret = service_operation(new_op, "orangefs_statfs", flags); + + if (new_op->downcall.status < 0) + goto out_op_release; + + gossip_debug(GOSSIP_SUPER_DEBUG, + "%s: got %ld blocks available | " + "%ld blocks total | %ld block size | " + "%ld files total | %ld files avail\n", + __func__, + (long)new_op->downcall.resp.statfs.blocks_avail, + (long)new_op->downcall.resp.statfs.blocks_total, + (long)new_op->downcall.resp.statfs.block_size, + (long)new_op->downcall.resp.statfs.files_total, + (long)new_op->downcall.resp.statfs.files_avail); + + buf->f_type = sb->s_magic; + buf->f_fsid.val[0] = ORANGEFS_SB(sb)->fs_id; + buf->f_fsid.val[1] = ORANGEFS_SB(sb)->id; + buf->f_bsize = new_op->downcall.resp.statfs.block_size; + buf->f_namelen = ORANGEFS_NAME_MAX; + + buf->f_blocks = (sector_t) new_op->downcall.resp.statfs.blocks_total; + buf->f_bfree = (sector_t) new_op->downcall.resp.statfs.blocks_avail; + buf->f_bavail = (sector_t) new_op->downcall.resp.statfs.blocks_avail; + buf->f_files = (sector_t) new_op->downcall.resp.statfs.files_total; + buf->f_ffree = (sector_t) new_op->downcall.resp.statfs.files_avail; + buf->f_frsize = 0; + +out_op_release: + op_release(new_op); + gossip_debug(GOSSIP_SUPER_DEBUG, "%s: returning %d\n", __func__, ret); + return ret; +} + +/* + * Remount as initiated by VFS layer. We just need to reparse the mount + * options, no need to signal pvfs2-client-core about it. + */ +static int orangefs_reconfigure(struct fs_context *fc) +{ + struct super_block *sb = fc->root->d_sb; + struct orangefs_sb_info_s *orangefs_sb = ORANGEFS_SB(sb); + struct orangefs_sb_info_s *revised = fc->s_fs_info; + unsigned int flags; + + flags = orangefs_sb->flags; + flags &= ~(ORANGEFS_OPT_INTR | ORANGEFS_OPT_LOCAL_LOCK); + flags |= revised->flags; + WRITE_ONCE(orangefs_sb->flags, flags); + + gossip_debug(GOSSIP_SUPER_DEBUG, "orangefs_reconfigure: called\n"); + return 0; +} + +/* + * Remount as initiated by pvfs2-client-core on restart. This is used to + * repopulate mount information left from previous pvfs2-client-core. + * + * the idea here is that given a valid superblock, we're + * re-initializing the user space client with the initial mount + * information specified when the super block was first initialized. + * this is very different than the first initialization/creation of a + * superblock. we use the special service_priority_operation to make + * sure that the mount gets ahead of any other pending operation that + * is waiting for servicing. this means that the pvfs2-client won't + * fail to start several times for all other pending operations before + * the client regains all of the mount information from us. + * NOTE: this function assumes that the request_mutex is already acquired! + */ +int orangefs_remount(struct orangefs_sb_info_s *orangefs_sb) +{ + struct orangefs_kernel_op_s *new_op; + int ret = -EINVAL; + + gossip_debug(GOSSIP_SUPER_DEBUG, "orangefs_remount: called\n"); + + new_op = op_alloc(ORANGEFS_VFS_OP_FS_MOUNT); + if (!new_op) + return -ENOMEM; + strscpy(new_op->upcall.req.fs_mount.orangefs_config_server, + orangefs_sb->devname); + + gossip_debug(GOSSIP_SUPER_DEBUG, + "Attempting ORANGEFS Remount via host %s\n", + new_op->upcall.req.fs_mount.orangefs_config_server); + + /* + * we assume that the calling function has already acquired the + * request_mutex to prevent other operations from bypassing + * this one + */ + ret = service_operation(new_op, "orangefs_remount", + ORANGEFS_OP_PRIORITY | ORANGEFS_OP_NO_MUTEX); + gossip_debug(GOSSIP_SUPER_DEBUG, + "orangefs_remount: mount got return value of %d\n", + ret); + if (ret == 0) { + /* + * store the id assigned to this sb -- it's just a + * short-lived mapping that the system interface uses + * to map this superblock to a particular mount entry + */ + orangefs_sb->id = new_op->downcall.resp.fs_mount.id; + orangefs_sb->mount_pending = 0; + } + + op_release(new_op); + + if (orangefs_userspace_version >= 20906) { + new_op = op_alloc(ORANGEFS_VFS_OP_FEATURES); + if (!new_op) + return -ENOMEM; + new_op->upcall.req.features.features = 0; + ret = service_operation(new_op, "orangefs_features", + ORANGEFS_OP_PRIORITY | ORANGEFS_OP_NO_MUTEX); + if (!ret) + orangefs_features = + new_op->downcall.resp.features.features; + else + orangefs_features = 0; + op_release(new_op); + } else { + orangefs_features = 0; + } + + return ret; +} + +int fsid_key_table_initialize(void) +{ + return 0; +} + +void fsid_key_table_finalize(void) +{ +} + +static const struct super_operations orangefs_s_ops = { + .alloc_inode = orangefs_alloc_inode, + .free_inode = orangefs_free_inode, + .destroy_inode = orangefs_destroy_inode, + .write_inode = orangefs_write_inode, + .drop_inode = inode_just_drop, + .statfs = orangefs_statfs, + .show_options = orangefs_show_options, +}; + +static struct dentry *orangefs_fh_to_dentry(struct super_block *sb, + struct fid *fid, + int fh_len, + int fh_type) +{ + struct orangefs_object_kref refn; + + if (fh_len < 5 || fh_type > 2) + return NULL; + + ORANGEFS_khandle_from(&(refn.khandle), fid->raw, 16); + refn.fs_id = (u32) fid->raw[4]; + gossip_debug(GOSSIP_SUPER_DEBUG, + "fh_to_dentry: handle %pU, fs_id %d\n", + &refn.khandle, + refn.fs_id); + + return d_obtain_alias(orangefs_iget(sb, &refn)); +} + +static int orangefs_encode_fh(struct inode *inode, + __u32 *fh, + int *max_len, + struct inode *parent) +{ + int len = parent ? 10 : 5; + int type = 1; + struct orangefs_object_kref refn; + + if (*max_len < len) { + gossip_err("fh buffer is too small for encoding\n"); + *max_len = len; + type = 255; + goto out; + } + + refn = ORANGEFS_I(inode)->refn; + ORANGEFS_khandle_to(&refn.khandle, fh, 16); + fh[4] = refn.fs_id; + + gossip_debug(GOSSIP_SUPER_DEBUG, + "Encoding fh: handle %pU, fsid %u\n", + &refn.khandle, + refn.fs_id); + + + if (parent) { + refn = ORANGEFS_I(parent)->refn; + ORANGEFS_khandle_to(&refn.khandle, (char *) fh + 20, 16); + fh[9] = refn.fs_id; + + type = 2; + gossip_debug(GOSSIP_SUPER_DEBUG, + "Encoding parent: handle %pU, fsid %u\n", + &refn.khandle, + refn.fs_id); + } + *max_len = len; + +out: + return type; +} + +static const struct export_operations orangefs_export_ops = { + .encode_fh = orangefs_encode_fh, + .fh_to_dentry = orangefs_fh_to_dentry, +}; + +static int orangefs_unmount(int id, __s32 fs_id, const char *devname) +{ + struct orangefs_kernel_op_s *op; + int r; + op = op_alloc(ORANGEFS_VFS_OP_FS_UMOUNT); + if (!op) + return -ENOMEM; + op->upcall.req.fs_umount.id = id; + op->upcall.req.fs_umount.fs_id = fs_id; + strscpy(op->upcall.req.fs_umount.orangefs_config_server, devname); + r = service_operation(op, "orangefs_fs_umount", 0); + /* Not much to do about an error here. */ + if (r) + gossip_err("orangefs_unmount: service_operation %d\n", r); + op_release(op); + return r; +} + +static int orangefs_fill_sb(struct super_block *sb, + struct fs_context *fc, + struct orangefs_fs_mount_response *fs_mount) +{ + int ret; + struct inode *root; + struct dentry *root_dentry; + struct orangefs_object_kref root_object; + + ORANGEFS_SB(sb)->sb = sb; + + ORANGEFS_SB(sb)->root_khandle = fs_mount->root_khandle; + ORANGEFS_SB(sb)->fs_id = fs_mount->fs_id; + ORANGEFS_SB(sb)->id = fs_mount->id; + + /* Hang the xattr handlers off the superblock */ + sb->s_xattr = orangefs_xattr_handlers; + sb->s_magic = ORANGEFS_SUPER_MAGIC; + sb->s_op = &orangefs_s_ops; + set_default_d_op(sb, &orangefs_dentry_operations); + + sb->s_blocksize = PAGE_SIZE; + sb->s_blocksize_bits = PAGE_SHIFT; + sb->s_maxbytes = MAX_LFS_FILESIZE; + + ret = super_setup_bdi(sb); + if (ret) + return ret; + + root_object.khandle = ORANGEFS_SB(sb)->root_khandle; + root_object.fs_id = ORANGEFS_SB(sb)->fs_id; + gossip_debug(GOSSIP_SUPER_DEBUG, + "get inode %pU, fsid %d\n", + &root_object.khandle, + root_object.fs_id); + + root = orangefs_iget(sb, &root_object); + if (IS_ERR(root)) + return PTR_ERR(root); + + gossip_debug(GOSSIP_SUPER_DEBUG, + "Allocated root inode [%p] with mode %x\n", + root, + root->i_mode); + + /* allocates and places root dentry in dcache */ + root_dentry = d_make_root(root); + if (!root_dentry) + return -ENOMEM; + + sb->s_export_op = &orangefs_export_ops; + sb->s_root = root_dentry; + return 0; +} + +static int orangefs_get_tree(struct fs_context *fc) +{ + int ret; + struct super_block *sb = ERR_PTR(-EINVAL); + struct orangefs_kernel_op_s *new_op; + + if (!fc->source) + return invalf(fc, "Device name not specified.\n"); + + gossip_debug(GOSSIP_SUPER_DEBUG, + "orangefs_mount: called with devname %s\n", + fc->source); + + new_op = op_alloc(ORANGEFS_VFS_OP_FS_MOUNT); + if (!new_op) + return -ENOMEM; + + strscpy(new_op->upcall.req.fs_mount.orangefs_config_server, fc->source); + + gossip_debug(GOSSIP_SUPER_DEBUG, + "Attempting ORANGEFS Mount via host %s\n", + new_op->upcall.req.fs_mount.orangefs_config_server); + + ret = service_operation(new_op, "orangefs_mount", 0); + gossip_debug(GOSSIP_SUPER_DEBUG, + "orangefs_mount: mount got return value of %d\n", ret); + if (ret) + goto free_op; + + if (new_op->downcall.resp.fs_mount.fs_id == ORANGEFS_FS_ID_NULL) { + gossip_err("ERROR: Retrieved null fs_id\n"); + ret = -EINVAL; + goto free_op; + } + + sb = sget_fc(fc, NULL, set_anon_super_fc); + + if (IS_ERR(sb)) { + ret = PTR_ERR(sb); + orangefs_unmount(new_op->downcall.resp.fs_mount.id, + new_op->downcall.resp.fs_mount.fs_id, + fc->source); + goto free_op; + } + + /* init our private orangefs sb info */ + ret = orangefs_fill_sb(sb, fc, &new_op->downcall.resp.fs_mount); + + if (ret) + goto free_sb_and_op; + + /* + * on successful mount, store the devname and data + * used + */ + strscpy(ORANGEFS_SB(sb)->devname, fc->source); + + /* mount_pending must be cleared */ + ORANGEFS_SB(sb)->mount_pending = 0; + + /* + * finally, add this sb to our list of known orangefs + * sb's + */ + gossip_debug(GOSSIP_SUPER_DEBUG, + "Adding SB %p to orangefs superblocks\n", + ORANGEFS_SB(sb)); + spin_lock(&orangefs_superblocks_lock); + list_add_tail(&ORANGEFS_SB(sb)->list, &orangefs_superblocks); + spin_unlock(&orangefs_superblocks_lock); + op_release(new_op); + + /* Must be removed from the list now. */ + ORANGEFS_SB(sb)->no_list = 0; + + if (orangefs_userspace_version >= 20906) { + new_op = op_alloc(ORANGEFS_VFS_OP_FEATURES); + if (!new_op) + return -ENOMEM; + new_op->upcall.req.features.features = 0; + ret = service_operation(new_op, "orangefs_features", 0); + orangefs_features = new_op->downcall.resp.features.features; + op_release(new_op); + } else { + orangefs_features = 0; + } + + fc->root = dget(sb->s_root); + return 0; + +free_sb_and_op: + /* Will call orangefs_kill_sb with sb not in list. */ + ORANGEFS_SB(sb)->no_list = 1; + /* ORANGEFS_VFS_OP_FS_UMOUNT is done by orangefs_kill_sb. */ + deactivate_locked_super(sb); +free_op: + gossip_err("orangefs_mount: mount request failed with %d\n", ret); + if (ret == -EINVAL) { + gossip_err("Ensure that all orangefs-servers have the same FS configuration files\n"); + gossip_err("Look at pvfs2-client-core log file (typically /tmp/pvfs2-client.log) for more details\n"); + } + + op_release(new_op); + + return ret; +} + +static void orangefs_free_fc(struct fs_context *fc) +{ + kfree(fc->s_fs_info); +} + +static const struct fs_context_operations orangefs_context_ops = { + .free = orangefs_free_fc, + .parse_param = orangefs_parse_param, + .get_tree = orangefs_get_tree, + .reconfigure = orangefs_reconfigure, +}; + +/* + * Set up the filesystem mount context. + */ +int orangefs_init_fs_context(struct fs_context *fc) +{ + struct orangefs_sb_info_s *osi; + + osi = kzalloc_obj(struct orangefs_sb_info_s); + if (!osi) + return -ENOMEM; + + /* + * Force any potential flags that might be set from the mount + * to zero, ie, initialize to unset. + */ + fc->sb_flags_mask &= ~SB_POSIXACL; + osi->flags &= ~ORANGEFS_OPT_INTR; + osi->flags &= ~ORANGEFS_OPT_LOCAL_LOCK; + + fc->s_fs_info = osi; + fc->ops = &orangefs_context_ops; + return 0; +} + +void orangefs_kill_sb(struct super_block *sb) +{ + int r; + gossip_debug(GOSSIP_SUPER_DEBUG, "orangefs_kill_sb: called\n"); + + /* provided sb cleanup */ + kill_anon_super(sb); + + if (!ORANGEFS_SB(sb)) { + mutex_lock(&orangefs_request_mutex); + mutex_unlock(&orangefs_request_mutex); + return; + } + /* + * issue the unmount to userspace to tell it to remove the + * dynamic mount info it has for this superblock + */ + r = orangefs_unmount(ORANGEFS_SB(sb)->id, ORANGEFS_SB(sb)->fs_id, + ORANGEFS_SB(sb)->devname); + if (!r) + ORANGEFS_SB(sb)->mount_pending = 1; + + if (!ORANGEFS_SB(sb)->no_list) { + /* remove the sb from our list of orangefs specific sb's */ + spin_lock(&orangefs_superblocks_lock); + /* not list_del_init */ + __list_del_entry(&ORANGEFS_SB(sb)->list); + ORANGEFS_SB(sb)->list.prev = NULL; + spin_unlock(&orangefs_superblocks_lock); + } + + /* + * make sure that ORANGEFS_DEV_REMOUNT_ALL loop that might've seen us + * gets completed before we free the dang thing. + */ + mutex_lock(&orangefs_request_mutex); + mutex_unlock(&orangefs_request_mutex); + + /* free the orangefs superblock private data */ + kfree(ORANGEFS_SB(sb)); +} + +int orangefs_inode_cache_initialize(void) +{ + orangefs_inode_cache = kmem_cache_create_usercopy( + "orangefs_inode_cache", + sizeof(struct orangefs_inode_s), + 0, + 0, + offsetof(struct orangefs_inode_s, + link_target), + sizeof_field(struct orangefs_inode_s, + link_target), + orangefs_inode_cache_ctor); + + if (!orangefs_inode_cache) { + gossip_err("Cannot create orangefs_inode_cache\n"); + return -ENOMEM; + } + return 0; +} + +int orangefs_inode_cache_finalize(void) +{ + kmem_cache_destroy(orangefs_inode_cache); + return 0; +} diff --git a/fs/orangefs/symlink.c b/fs/orangefs/symlink.c new file mode 100644 index 000000000..db107fe91 --- /dev/null +++ b/fs/orangefs/symlink.c @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * (C) 2001 Clemson University and The University of Chicago + * + * See COPYING in top-level directory. + */ + +#include "protocol.h" +#include "orangefs-kernel.h" +#include "orangefs-bufmap.h" + +const struct inode_operations orangefs_symlink_inode_operations = { + .get_link = simple_get_link, + .setattr = orangefs_setattr, + .getattr = orangefs_getattr, + .listxattr = orangefs_listxattr, + .permission = orangefs_permission, + .update_time = orangefs_update_time, +}; diff --git a/fs/orangefs/upcall.h b/fs/orangefs/upcall.h new file mode 100644 index 000000000..16118452a --- /dev/null +++ b/fs/orangefs/upcall.h @@ -0,0 +1,260 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * (C) 2001 Clemson University and The University of Chicago + * + * See COPYING in top-level directory. + */ + +#ifndef __UPCALL_H +#define __UPCALL_H + +/* + * Sanitized this header file to fix + * 32-64 bit interaction issues between + * client-core and device + */ +struct orangefs_io_request_s { + __s32 __pad1; + __s32 buf_index; + __s32 count; + __s32 __pad2; + __s64 offset; + struct orangefs_object_kref refn; + enum ORANGEFS_io_type io_type; + __s32 readahead_size; +}; + +struct orangefs_lookup_request_s { + __s32 sym_follow; + __s32 __pad1; + struct orangefs_object_kref parent_refn; + char d_name[ORANGEFS_NAME_MAX]; +}; + +struct orangefs_create_request_s { + struct orangefs_object_kref parent_refn; + struct ORANGEFS_sys_attr_s attributes; + char d_name[ORANGEFS_NAME_MAX]; +}; + +struct orangefs_symlink_request_s { + struct orangefs_object_kref parent_refn; + struct ORANGEFS_sys_attr_s attributes; + char entry_name[ORANGEFS_NAME_MAX]; + char target[ORANGEFS_NAME_MAX]; +}; + +struct orangefs_getattr_request_s { + struct orangefs_object_kref refn; + __u32 mask; + __u32 __pad1; +}; + +struct orangefs_setattr_request_s { + struct orangefs_object_kref refn; + struct ORANGEFS_sys_attr_s attributes; +}; + +struct orangefs_remove_request_s { + struct orangefs_object_kref parent_refn; + char d_name[ORANGEFS_NAME_MAX]; +}; + +struct orangefs_mkdir_request_s { + struct orangefs_object_kref parent_refn; + struct ORANGEFS_sys_attr_s attributes; + char d_name[ORANGEFS_NAME_MAX]; +}; + +struct orangefs_readdir_request_s { + struct orangefs_object_kref refn; + __u64 token; + __s32 max_dirent_count; + __s32 buf_index; +}; + +struct orangefs_readdirplus_request_s { + struct orangefs_object_kref refn; + __u64 token; + __s32 max_dirent_count; + __u32 mask; + __s32 buf_index; + __s32 __pad1; +}; + +struct orangefs_rename_request_s { + struct orangefs_object_kref old_parent_refn; + struct orangefs_object_kref new_parent_refn; + char d_old_name[ORANGEFS_NAME_MAX]; + char d_new_name[ORANGEFS_NAME_MAX]; +}; + +struct orangefs_statfs_request_s { + __s32 fs_id; + __s32 __pad1; +}; + +struct orangefs_truncate_request_s { + struct orangefs_object_kref refn; + __s64 size; +}; + +struct orangefs_ra_cache_flush_request_s { + struct orangefs_object_kref refn; +}; + +struct orangefs_fs_mount_request_s { + char orangefs_config_server[ORANGEFS_MAX_SERVER_ADDR_LEN]; +}; + +struct orangefs_fs_umount_request_s { + __s32 id; + __s32 fs_id; + char orangefs_config_server[ORANGEFS_MAX_SERVER_ADDR_LEN]; +}; + +struct orangefs_getxattr_request_s { + struct orangefs_object_kref refn; + __s32 key_sz; + __s32 __pad1; + char key[ORANGEFS_MAX_XATTR_NAMELEN]; +}; + +struct orangefs_setxattr_request_s { + struct orangefs_object_kref refn; + struct ORANGEFS_keyval_pair keyval; + __s32 flags; + __s32 __pad1; +}; + +struct orangefs_listxattr_request_s { + struct orangefs_object_kref refn; + __s32 requested_count; + __s32 __pad1; + __u64 token; +}; + +struct orangefs_removexattr_request_s { + struct orangefs_object_kref refn; + __s32 key_sz; + __s32 __pad1; + char key[ORANGEFS_MAX_XATTR_NAMELEN]; +}; + +struct orangefs_op_cancel_s { + __u64 op_tag; +}; + +struct orangefs_fsync_request_s { + struct orangefs_object_kref refn; +}; + +enum orangefs_param_request_type { + ORANGEFS_PARAM_REQUEST_SET = 1, + ORANGEFS_PARAM_REQUEST_GET = 2 +}; + +enum orangefs_param_request_op { + ORANGEFS_PARAM_REQUEST_OP_ACACHE_TIMEOUT_MSECS = 1, + ORANGEFS_PARAM_REQUEST_OP_ACACHE_HARD_LIMIT = 2, + ORANGEFS_PARAM_REQUEST_OP_ACACHE_SOFT_LIMIT = 3, + ORANGEFS_PARAM_REQUEST_OP_ACACHE_RECLAIM_PERCENTAGE = 4, + ORANGEFS_PARAM_REQUEST_OP_PERF_TIME_INTERVAL_SECS = 5, + ORANGEFS_PARAM_REQUEST_OP_PERF_HISTORY_SIZE = 6, + ORANGEFS_PARAM_REQUEST_OP_PERF_RESET = 7, + ORANGEFS_PARAM_REQUEST_OP_NCACHE_TIMEOUT_MSECS = 8, + ORANGEFS_PARAM_REQUEST_OP_NCACHE_HARD_LIMIT = 9, + ORANGEFS_PARAM_REQUEST_OP_NCACHE_SOFT_LIMIT = 10, + ORANGEFS_PARAM_REQUEST_OP_NCACHE_RECLAIM_PERCENTAGE = 11, + ORANGEFS_PARAM_REQUEST_OP_STATIC_ACACHE_TIMEOUT_MSECS = 12, + ORANGEFS_PARAM_REQUEST_OP_STATIC_ACACHE_HARD_LIMIT = 13, + ORANGEFS_PARAM_REQUEST_OP_STATIC_ACACHE_SOFT_LIMIT = 14, + ORANGEFS_PARAM_REQUEST_OP_STATIC_ACACHE_RECLAIM_PERCENTAGE = 15, + ORANGEFS_PARAM_REQUEST_OP_CLIENT_DEBUG = 16, + ORANGEFS_PARAM_REQUEST_OP_CCACHE_TIMEOUT_SECS = 17, + ORANGEFS_PARAM_REQUEST_OP_CCACHE_HARD_LIMIT = 18, + ORANGEFS_PARAM_REQUEST_OP_CCACHE_SOFT_LIMIT = 19, + ORANGEFS_PARAM_REQUEST_OP_CCACHE_RECLAIM_PERCENTAGE = 20, + ORANGEFS_PARAM_REQUEST_OP_CAPCACHE_TIMEOUT_SECS = 21, + ORANGEFS_PARAM_REQUEST_OP_CAPCACHE_HARD_LIMIT = 22, + ORANGEFS_PARAM_REQUEST_OP_CAPCACHE_SOFT_LIMIT = 23, + ORANGEFS_PARAM_REQUEST_OP_CAPCACHE_RECLAIM_PERCENTAGE = 24, + ORANGEFS_PARAM_REQUEST_OP_TWO_MASK_VALUES = 25, + ORANGEFS_PARAM_REQUEST_OP_READAHEAD_SIZE = 26, + ORANGEFS_PARAM_REQUEST_OP_READAHEAD_COUNT = 27, + ORANGEFS_PARAM_REQUEST_OP_READAHEAD_COUNT_SIZE = 28, + ORANGEFS_PARAM_REQUEST_OP_READAHEAD_READCNT = 29, +}; + +struct orangefs_param_request_s { + enum orangefs_param_request_type type; + enum orangefs_param_request_op op; + union { + __s64 value64; + __s32 value32[2]; + } u; + char s_value[ORANGEFS_MAX_DEBUG_STRING_LEN]; +}; + +enum orangefs_perf_count_request_type { + ORANGEFS_PERF_COUNT_REQUEST_ACACHE = 1, + ORANGEFS_PERF_COUNT_REQUEST_NCACHE = 2, + ORANGEFS_PERF_COUNT_REQUEST_CAPCACHE = 3, +}; + +struct orangefs_perf_count_request_s { + enum orangefs_perf_count_request_type type; + __s32 __pad1; +}; + +struct orangefs_fs_key_request_s { + __s32 fsid; + __s32 __pad1; +}; + +/* 2.9.6 */ +struct orangefs_features_request_s { + __u64 features; +}; + +struct orangefs_upcall_s { + __s32 type; + __u32 uid; + __u32 gid; + int pid; + int tgid; + /* Trailers unused but must be retained for protocol compatibility. */ + __s64 trailer_size; + char *trailer_buf; + + union { + struct orangefs_io_request_s io; + struct orangefs_lookup_request_s lookup; + struct orangefs_create_request_s create; + struct orangefs_symlink_request_s sym; + struct orangefs_getattr_request_s getattr; + struct orangefs_setattr_request_s setattr; + struct orangefs_remove_request_s remove; + struct orangefs_mkdir_request_s mkdir; + struct orangefs_readdir_request_s readdir; + struct orangefs_readdirplus_request_s readdirplus; + struct orangefs_rename_request_s rename; + struct orangefs_statfs_request_s statfs; + struct orangefs_truncate_request_s truncate; + struct orangefs_ra_cache_flush_request_s ra_cache_flush; + struct orangefs_fs_mount_request_s fs_mount; + struct orangefs_fs_umount_request_s fs_umount; + struct orangefs_getxattr_request_s getxattr; + struct orangefs_setxattr_request_s setxattr; + struct orangefs_listxattr_request_s listxattr; + struct orangefs_removexattr_request_s removexattr; + struct orangefs_op_cancel_s cancel; + struct orangefs_fsync_request_s fsync; + struct orangefs_param_request_s param; + struct orangefs_perf_count_request_s perf_count; + struct orangefs_fs_key_request_s fs_key; + struct orangefs_features_request_s features; + } req; +}; + +#endif /* __UPCALL_H */ diff --git a/fs/orangefs/waitqueue.c b/fs/orangefs/waitqueue.c new file mode 100644 index 000000000..beafc33d5 --- /dev/null +++ b/fs/orangefs/waitqueue.c @@ -0,0 +1,373 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * (C) 2001 Clemson University and The University of Chicago + * (C) 2011 Omnibond Systems + * + * Changes by Acxiom Corporation to implement generic service_operation() + * function, Copyright Acxiom Corporation, 2005. + * + * See COPYING in top-level directory. + */ + +/* + * In-kernel waitqueue operations. + */ + +#include "protocol.h" +#include "orangefs-kernel.h" +#include "orangefs-bufmap.h" + +static int wait_for_matching_downcall(struct orangefs_kernel_op_s *op, + long timeout, + int flags) + __acquires(op->lock); +static void orangefs_clean_up_interrupted_operation(struct orangefs_kernel_op_s *op) + __releases(op->lock); + +/* + * What we do in this function is to walk the list of operations that are + * present in the request queue and mark them as purged. + * NOTE: This is called from the device close after client-core has + * guaranteed that no new operations could appear on the list since the + * client-core is anyway going to exit. + */ +void purge_waiting_ops(void) +{ + struct orangefs_kernel_op_s *op, *tmp; + + spin_lock(&orangefs_request_list_lock); + list_for_each_entry_safe(op, tmp, &orangefs_request_list, list) { + gossip_debug(GOSSIP_WAIT_DEBUG, + "pvfs2-client-core: purging op tag %llu %s\n", + llu(op->tag), + get_opname_string(op)); + set_op_state_purged(op); + gossip_debug(GOSSIP_DEV_DEBUG, + "%s: op:%s: op_state:%d: process:%s:\n", + __func__, + get_opname_string(op), + op->op_state, + current->comm); + } + spin_unlock(&orangefs_request_list_lock); +} + +/* + * submits a ORANGEFS operation and waits for it to complete + * + * Note op->downcall.status will contain the status of the operation (in + * errno format), whether provided by pvfs2-client or a result of failure to + * service the operation. If the caller wishes to distinguish, then + * op->state can be checked to see if it was serviced or not. + * + * Returns contents of op->downcall.status for convenience + */ +int service_operation(struct orangefs_kernel_op_s *op, + const char *op_name, + int flags) +{ + long timeout = MAX_SCHEDULE_TIMEOUT; + int ret = 0; + + DEFINE_WAIT(wait_entry); + + op->upcall.tgid = current->tgid; + op->upcall.pid = current->pid; + +retry_servicing: + op->downcall.status = 0; + gossip_debug(GOSSIP_WAIT_DEBUG, + "%s: %s op:%p: process:%s: pid:%d:\n", + __func__, + op_name, + op, + current->comm, + current->pid); + + /* + * If ORANGEFS_OP_NO_MUTEX was set in flags, we need to avoid + * acquiring the request_mutex because we're servicing a + * high priority remount operation and the request_mutex is + * already taken. + */ + if (!(flags & ORANGEFS_OP_NO_MUTEX)) { + if (flags & ORANGEFS_OP_INTERRUPTIBLE) + ret = mutex_lock_interruptible(&orangefs_request_mutex); + else + ret = mutex_lock_killable(&orangefs_request_mutex); + /* + * check to see if we were interrupted while waiting for + * mutex + */ + if (ret < 0) { + op->downcall.status = ret; + gossip_debug(GOSSIP_WAIT_DEBUG, + "%s: service_operation interrupted.\n", + __func__); + return ret; + } + } + + /* queue up the operation */ + spin_lock(&orangefs_request_list_lock); + spin_lock(&op->lock); + set_op_state_waiting(op); + gossip_debug(GOSSIP_DEV_DEBUG, + "%s: op:%s: op_state:%d: process:%s:\n", + __func__, + get_opname_string(op), + op->op_state, + current->comm); + /* add high priority remount op to the front of the line. */ + if (flags & ORANGEFS_OP_PRIORITY) + list_add(&op->list, &orangefs_request_list); + else + list_add_tail(&op->list, &orangefs_request_list); + spin_unlock(&op->lock); + wake_up_interruptible(&orangefs_request_list_waitq); + if (!__is_daemon_in_service()) { + gossip_debug(GOSSIP_WAIT_DEBUG, + "%s:client core is NOT in service.\n", + __func__); + /* + * Don't wait for the userspace component to return if + * the filesystem is being umounted anyway. + */ + if (op->upcall.type == ORANGEFS_VFS_OP_FS_UMOUNT) + timeout = 0; + else + timeout = op_timeout_secs * HZ; + } + spin_unlock(&orangefs_request_list_lock); + + if (!(flags & ORANGEFS_OP_NO_MUTEX)) + mutex_unlock(&orangefs_request_mutex); + + ret = wait_for_matching_downcall(op, timeout, flags); + gossip_debug(GOSSIP_WAIT_DEBUG, + "%s: wait_for_matching_downcall returned %d for %p\n", + __func__, + ret, + op); + + /* got matching downcall; make sure status is in errno format */ + if (!ret) { + spin_unlock(&op->lock); + op->downcall.status = + orangefs_normalize_to_errno(op->downcall.status); + ret = op->downcall.status; + goto out; + } + + /* failed to get matching downcall */ + if (ret == -ETIMEDOUT) { + gossip_err("%s: %s -- wait timed out; aborting attempt.\n", + __func__, + op_name); + } + + /* + * remove a waiting op from the request list or + * remove an in-progress op from the in-progress list. + */ + orangefs_clean_up_interrupted_operation(op); + + op->downcall.status = ret; + /* retry if operation has not been serviced and if requested */ + if (ret == -EAGAIN) { + op->attempts++; + timeout = op_timeout_secs * HZ; + gossip_debug(GOSSIP_WAIT_DEBUG, + "orangefs: tag %llu (%s)" + " -- operation to be retried (%d attempt)\n", + llu(op->tag), + op_name, + op->attempts); + + /* + * io ops (ops that use the shared memory buffer) have + * to be returned to their caller for a retry. Other ops + * can just be recycled here. + */ + if (!op->uses_shared_memory) + goto retry_servicing; + } + +out: + gossip_debug(GOSSIP_WAIT_DEBUG, + "%s: %s returning: %d for %p.\n", + __func__, + op_name, + ret, + op); + return ret; +} + +/* This can get called on an I/O op if it had a bad service_operation. */ +bool orangefs_cancel_op_in_progress(struct orangefs_kernel_op_s *op) +{ + u64 tag = op->tag; + if (!op_state_in_progress(op)) + return false; + + op->slot_to_free = op->upcall.req.io.buf_index; + memset(&op->upcall, 0, sizeof(op->upcall)); + memset(&op->downcall, 0, sizeof(op->downcall)); + op->upcall.type = ORANGEFS_VFS_OP_CANCEL; + op->upcall.req.cancel.op_tag = tag; + op->downcall.type = ORANGEFS_VFS_OP_INVALID; + op->downcall.status = -1; + orangefs_new_tag(op); + + spin_lock(&orangefs_request_list_lock); + /* orangefs_request_list_lock is enough of a barrier here */ + if (!__is_daemon_in_service()) { + spin_unlock(&orangefs_request_list_lock); + return false; + } + spin_lock(&op->lock); + set_op_state_waiting(op); + gossip_debug(GOSSIP_DEV_DEBUG, + "%s: op:%s: op_state:%d: process:%s:\n", + __func__, + get_opname_string(op), + op->op_state, + current->comm); + list_add(&op->list, &orangefs_request_list); + spin_unlock(&op->lock); + spin_unlock(&orangefs_request_list_lock); + + gossip_debug(GOSSIP_WAIT_DEBUG, + "Attempting ORANGEFS operation cancellation of tag %llu\n", + llu(tag)); + return true; +} + +/* + * Change an op to the "given up" state and remove it from its list. + */ +static void + orangefs_clean_up_interrupted_operation(struct orangefs_kernel_op_s *op) + __releases(op->lock) +{ + /* + * handle interrupted cases depending on what state we were in when + * the interruption is detected. + * + * Called with op->lock held. + */ + + /* + * List manipulation code elsewhere will ignore ops that + * have been given up upon. + */ + op->op_state |= OP_VFS_STATE_GIVEN_UP; + + if (list_empty(&op->list)) { + /* caught copying to/from daemon */ + BUG_ON(op_state_serviced(op)); + spin_unlock(&op->lock); + wait_for_completion(&op->waitq); + } else if (op_state_waiting(op)) { + /* + * upcall hasn't been read; remove op from upcall request + * list. + */ + spin_unlock(&op->lock); + spin_lock(&orangefs_request_list_lock); + list_del_init(&op->list); + spin_unlock(&orangefs_request_list_lock); + gossip_debug(GOSSIP_WAIT_DEBUG, + "Interrupted: Removed op %p from request_list\n", + op); + } else if (op_state_in_progress(op)) { + /* op must be removed from the in progress htable */ + spin_unlock(&op->lock); + spin_lock(&orangefs_htable_ops_in_progress_lock); + list_del_init(&op->list); + spin_unlock(&orangefs_htable_ops_in_progress_lock); + gossip_debug(GOSSIP_WAIT_DEBUG, + "Interrupted: Removed op %p" + " from htable_ops_in_progress\n", + op); + } else { + spin_unlock(&op->lock); + gossip_err("interrupted operation is in a weird state 0x%x\n", + op->op_state); + } + reinit_completion(&op->waitq); +} + +/* + * Sleeps on waitqueue waiting for matching downcall. + * If client-core finishes servicing, then we are good to go. + * else if client-core exits, we get woken up here, and retry with a timeout + * + * When this call returns to the caller, the specified op will no + * longer be in either the in_progress hash table or on the request list. + * + * Returns 0 on success and -errno on failure + * Errors are: + * EAGAIN in case we want the caller to requeue and try again.. + * EINTR/EIO/ETIMEDOUT indicating we are done trying to service this + * operation since client-core seems to be exiting too often + * or if we were interrupted. + * + * Returns with op->lock taken. + */ +static int wait_for_matching_downcall(struct orangefs_kernel_op_s *op, + long timeout, + int flags) + __acquires(op->lock) +{ + long n; + int writeback = flags & ORANGEFS_OP_WRITEBACK, + interruptible = flags & ORANGEFS_OP_INTERRUPTIBLE; + + /* + * There's a "schedule_timeout" inside of these wait + * primitives, during which the op is out of the hands of the + * user process that needs something done and is being + * manipulated by the client-core process. + */ + if (writeback) + n = wait_for_completion_io_timeout(&op->waitq, timeout); + else if (!writeback && interruptible) + n = wait_for_completion_interruptible_timeout(&op->waitq, + timeout); + else /* !writeback && !interruptible but compiler complains */ + n = wait_for_completion_killable_timeout(&op->waitq, timeout); + + spin_lock(&op->lock); + + if (op_state_serviced(op)) + return 0; + + if (unlikely(n < 0)) { + gossip_debug(GOSSIP_WAIT_DEBUG, + "%s: operation interrupted, tag %llu, %p\n", + __func__, + llu(op->tag), + op); + return -EINTR; + } + if (op_state_purged(op)) { + gossip_debug(GOSSIP_WAIT_DEBUG, + "%s: operation purged, tag %llu, %p, %d\n", + __func__, + llu(op->tag), + op, + op->attempts); + return (op->attempts < ORANGEFS_PURGE_RETRY_COUNT) ? + -EAGAIN : + -EIO; + } + /* must have timed out, then... */ + gossip_debug(GOSSIP_WAIT_DEBUG, + "%s: operation timed out, tag %llu, %p, %d)\n", + __func__, + llu(op->tag), + op, + op->attempts); + return -ETIMEDOUT; +} diff --git a/fs/orangefs/xattr.c b/fs/orangefs/xattr.c new file mode 100644 index 000000000..885fd3bd5 --- /dev/null +++ b/fs/orangefs/xattr.c @@ -0,0 +1,561 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * (C) 2001 Clemson University and The University of Chicago + * Copyright 2018 Omnibond Systems, L.L.C. + * + * See COPYING in top-level directory. + */ + +/* + * Linux VFS extended attribute operations. + */ + +#include "protocol.h" +#include "orangefs-kernel.h" +#include "orangefs-bufmap.h" +#include +#include +#include + +#define SYSTEM_ORANGEFS_KEY "system.pvfs2." +#define SYSTEM_ORANGEFS_KEY_LEN 13 + +/* + * this function returns + * 0 if the key corresponding to name is not meant to be printed as part + * of a listxattr. + * 1 if the key corresponding to name is meant to be returned as part of + * a listxattr. + * The ones that start SYSTEM_ORANGEFS_KEY are the ones to avoid printing. + */ +static int is_reserved_key(const char *key, size_t size) +{ + + if (size < SYSTEM_ORANGEFS_KEY_LEN) + return 1; + + return strncmp(key, SYSTEM_ORANGEFS_KEY, SYSTEM_ORANGEFS_KEY_LEN) ? 1 : 0; +} + +static inline int convert_to_internal_xattr_flags(int setxattr_flags) +{ + int internal_flag = 0; + + if (setxattr_flags & XATTR_REPLACE) { + /* Attribute must exist! */ + internal_flag = ORANGEFS_XATTR_REPLACE; + } else if (setxattr_flags & XATTR_CREATE) { + /* Attribute must not exist */ + internal_flag = ORANGEFS_XATTR_CREATE; + } + return internal_flag; +} + +static unsigned int xattr_key(const char *key) +{ + unsigned int i = 0; + if (!key) + return 0; + while (*key) + i += *key++; + return i % 16; +} + +static struct orangefs_cached_xattr *find_cached_xattr(struct inode *inode, + const char *key) +{ + struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode); + struct orangefs_cached_xattr *cx; + struct hlist_head *h; + struct hlist_node *tmp; + h = &orangefs_inode->xattr_cache[xattr_key(key)]; + if (hlist_empty(h)) + return NULL; + hlist_for_each_entry_safe(cx, tmp, h, node) { + if (!strcmp(cx->key, key)) + return cx; + } + return NULL; +} + +/* + * Tries to get a specified key's attributes of a given + * file into a user-specified buffer. Note that the getxattr + * interface allows for the users to probe the size of an + * extended attribute by passing in a value of 0 to size. + * Thus our return value is always the size of the attribute + * unless the key does not exist for the file and/or if + * there were errors in fetching the attribute value. + */ +ssize_t orangefs_inode_getxattr(struct inode *inode, const char *name, + void *buffer, size_t size) +{ + struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode); + struct orangefs_kernel_op_s *new_op = NULL; + struct orangefs_cached_xattr *cx; + ssize_t ret = -ENOMEM; + ssize_t length = 0; + int fsuid; + int fsgid; + + gossip_debug(GOSSIP_XATTR_DEBUG, + "%s: name %s, buffer_size %zd\n", + __func__, name, size); + + if (S_ISLNK(inode->i_mode)) + return -EOPNOTSUPP; + + if (strlen(name) >= ORANGEFS_MAX_XATTR_NAMELEN) + return -EINVAL; + + fsuid = from_kuid(&init_user_ns, current_fsuid()); + fsgid = from_kgid(&init_user_ns, current_fsgid()); + + gossip_debug(GOSSIP_XATTR_DEBUG, + "getxattr on inode %pU, name %s " + "(uid %o, gid %o)\n", + get_khandle_from_ino(inode), + name, + fsuid, + fsgid); + + down_read(&orangefs_inode->xattr_sem); + + cx = find_cached_xattr(inode, name); + if (cx && time_before(jiffies, cx->timeout)) { + if (cx->length == -1) { + ret = -ENODATA; + goto out_unlock; + } else { + if (size == 0) { + ret = cx->length; + goto out_unlock; + } + if (cx->length > size) { + ret = -ERANGE; + goto out_unlock; + } + memcpy(buffer, cx->val, cx->length); + memset(buffer + cx->length, 0, size - cx->length); + ret = cx->length; + goto out_unlock; + } + } + + new_op = op_alloc(ORANGEFS_VFS_OP_GETXATTR); + if (!new_op) + goto out_unlock; + + new_op->upcall.req.getxattr.refn = orangefs_inode->refn; + strscpy(new_op->upcall.req.getxattr.key, name); + + /* + * NOTE: Although keys are meant to be NULL terminated textual + * strings, I am going to explicitly pass the length just in case + * we change this later on... + */ + new_op->upcall.req.getxattr.key_sz = strlen(name) + 1; + + ret = service_operation(new_op, "orangefs_inode_getxattr", + get_interruptible_flag(inode)); + if (ret != 0) { + if (ret == -ENOENT) { + ret = -ENODATA; + gossip_debug(GOSSIP_XATTR_DEBUG, + "orangefs_inode_getxattr: inode %pU key %s" + " does not exist!\n", + get_khandle_from_ino(inode), + (char *)new_op->upcall.req.getxattr.key); + cx = kmalloc_obj(*cx); + if (cx) { + strscpy(cx->key, name); + cx->length = -1; + cx->timeout = jiffies + + orangefs_getattr_timeout_msecs*HZ/1000; + hlist_add_head( &cx->node, + &orangefs_inode->xattr_cache[xattr_key(cx->key)]); + } + } + goto out_release_op; + } + + /* + * Length returned includes null terminator. + */ + length = new_op->downcall.resp.getxattr.val_sz; + if (length < 0 || length > ORANGEFS_MAX_XATTR_VALUELEN) { + ret = -EIO; + goto out_release_op; + } + + /* + * Just return the length of the queried attribute. + */ + if (size == 0) { + ret = length; + goto out_release_op; + } + + /* + * Check to see if key length is > provided buffer size. + */ + if (length > size) { + ret = -ERANGE; + goto out_release_op; + } + + memcpy(buffer, new_op->downcall.resp.getxattr.val, length); + memset(buffer + length, 0, size - length); + gossip_debug(GOSSIP_XATTR_DEBUG, + "orangefs_inode_getxattr: inode %pU " + "key %s key_sz %d, val_len %d\n", + get_khandle_from_ino(inode), + (char *)new_op-> + upcall.req.getxattr.key, + (int)new_op-> + upcall.req.getxattr.key_sz, + (int)ret); + + ret = length; + + if (cx) { + strscpy(cx->key, name); + memcpy(cx->val, buffer, length); + cx->length = length; + cx->timeout = jiffies + HZ; + } else { + cx = kmalloc_obj(*cx); + if (cx) { + strscpy(cx->key, name); + memcpy(cx->val, buffer, length); + cx->length = length; + cx->timeout = jiffies + HZ; + hlist_add_head(&cx->node, + &orangefs_inode->xattr_cache[xattr_key(cx->key)]); + } + } + +out_release_op: + op_release(new_op); +out_unlock: + up_read(&orangefs_inode->xattr_sem); + return ret; +} + +static int orangefs_inode_removexattr(struct inode *inode, const char *name, + int flags) +{ + struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode); + struct orangefs_kernel_op_s *new_op = NULL; + struct orangefs_cached_xattr *cx; + struct hlist_head *h; + struct hlist_node *tmp; + int ret = -ENOMEM; + + if (strlen(name) >= ORANGEFS_MAX_XATTR_NAMELEN) + return -EINVAL; + + down_write(&orangefs_inode->xattr_sem); + new_op = op_alloc(ORANGEFS_VFS_OP_REMOVEXATTR); + if (!new_op) + goto out_unlock; + + new_op->upcall.req.removexattr.refn = orangefs_inode->refn; + /* + * NOTE: Although keys are meant to be NULL terminated + * textual strings, I am going to explicitly pass the + * length just in case we change this later on... + */ + strscpy(new_op->upcall.req.removexattr.key, name); + new_op->upcall.req.removexattr.key_sz = strlen(name) + 1; + + gossip_debug(GOSSIP_XATTR_DEBUG, + "orangefs_inode_removexattr: key %s, key_sz %d\n", + (char *)new_op->upcall.req.removexattr.key, + (int)new_op->upcall.req.removexattr.key_sz); + + ret = service_operation(new_op, + "orangefs_inode_removexattr", + get_interruptible_flag(inode)); + if (ret == -ENOENT) { + /* + * Request to replace a non-existent attribute is an error. + */ + if (flags & XATTR_REPLACE) + ret = -ENODATA; + else + ret = 0; + } + + gossip_debug(GOSSIP_XATTR_DEBUG, + "orangefs_inode_removexattr: returning %d\n", ret); + + op_release(new_op); + + h = &orangefs_inode->xattr_cache[xattr_key(name)]; + hlist_for_each_entry_safe(cx, tmp, h, node) { + if (!strcmp(cx->key, name)) { + hlist_del(&cx->node); + kfree(cx); + break; + } + } + +out_unlock: + up_write(&orangefs_inode->xattr_sem); + return ret; +} + +/* + * Tries to set an attribute for a given key on a file. + * + * Returns a -ve number on error and 0 on success. Key is text, but value + * can be binary! + */ +int orangefs_inode_setxattr(struct inode *inode, const char *name, + const void *value, size_t size, int flags) +{ + struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode); + struct orangefs_kernel_op_s *new_op; + int internal_flag = 0; + struct orangefs_cached_xattr *cx; + struct hlist_head *h; + struct hlist_node *tmp; + int ret = -ENOMEM; + + gossip_debug(GOSSIP_XATTR_DEBUG, + "%s: name %s, buffer_size %zd\n", + __func__, name, size); + + if (size > ORANGEFS_MAX_XATTR_VALUELEN) + return -EINVAL; + if (strlen(name) >= ORANGEFS_MAX_XATTR_NAMELEN) + return -EINVAL; + + internal_flag = convert_to_internal_xattr_flags(flags); + + /* This is equivalent to a removexattr */ + if (size == 0 && !value) { + gossip_debug(GOSSIP_XATTR_DEBUG, + "removing xattr (%s)\n", + name); + return orangefs_inode_removexattr(inode, name, flags); + } + + gossip_debug(GOSSIP_XATTR_DEBUG, + "setxattr on inode %pU, name %s\n", + get_khandle_from_ino(inode), + name); + + down_write(&orangefs_inode->xattr_sem); + new_op = op_alloc(ORANGEFS_VFS_OP_SETXATTR); + if (!new_op) + goto out_unlock; + + + new_op->upcall.req.setxattr.refn = orangefs_inode->refn; + new_op->upcall.req.setxattr.flags = internal_flag; + /* + * NOTE: Although keys are meant to be NULL terminated textual + * strings, I am going to explicitly pass the length just in + * case we change this later on... + */ + strscpy(new_op->upcall.req.setxattr.keyval.key, name); + new_op->upcall.req.setxattr.keyval.key_sz = strlen(name) + 1; + memcpy(new_op->upcall.req.setxattr.keyval.val, value, size); + new_op->upcall.req.setxattr.keyval.val_sz = size; + + gossip_debug(GOSSIP_XATTR_DEBUG, + "orangefs_inode_setxattr: key %s, key_sz %d " + " value size %zd\n", + (char *)new_op->upcall.req.setxattr.keyval.key, + (int)new_op->upcall.req.setxattr.keyval.key_sz, + size); + + ret = service_operation(new_op, + "orangefs_inode_setxattr", + get_interruptible_flag(inode)); + + gossip_debug(GOSSIP_XATTR_DEBUG, + "orangefs_inode_setxattr: returning %d\n", + ret); + + /* when request is serviced properly, free req op struct */ + op_release(new_op); + + h = &orangefs_inode->xattr_cache[xattr_key(name)]; + hlist_for_each_entry_safe(cx, tmp, h, node) { + if (!strcmp(cx->key, name)) { + hlist_del(&cx->node); + kfree(cx); + break; + } + } + +out_unlock: + up_write(&orangefs_inode->xattr_sem); + return ret; +} + +/* + * Tries to get a specified object's keys into a user-specified buffer of a + * given size. Note that like the previous instances of xattr routines, this + * also allows you to pass in a NULL pointer and 0 size to probe the size for + * subsequent memory allocations. Thus our return value is always the size of + * all the keys unless there were errors in fetching the keys! + */ +ssize_t orangefs_listxattr(struct dentry *dentry, char *buffer, size_t size) +{ + struct inode *inode = dentry->d_inode; + struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode); + struct orangefs_kernel_op_s *new_op; + __u64 token = ORANGEFS_ITERATE_START; + ssize_t ret = -ENOMEM; + ssize_t total = 0; + int count_keys = 0; + int key_size; + int i = 0; + int returned_count = 0; + + if (size > 0 && !buffer) { + gossip_err("%s: bogus NULL pointers\n", __func__); + return -EINVAL; + } + + down_read(&orangefs_inode->xattr_sem); + new_op = op_alloc(ORANGEFS_VFS_OP_LISTXATTR); + if (!new_op) + goto out_unlock; + + if (buffer && size > 0) + memset(buffer, 0, size); + +try_again: + key_size = 0; + new_op->upcall.req.listxattr.refn = orangefs_inode->refn; + new_op->upcall.req.listxattr.token = token; + new_op->upcall.req.listxattr.requested_count = + (size == 0) ? 0 : ORANGEFS_MAX_XATTR_LISTLEN; + ret = service_operation(new_op, __func__, + get_interruptible_flag(inode)); + if (ret != 0) + goto done; + + if (size == 0) { + /* + * This is a bit of a big upper limit, but I did not want to + * spend too much time getting this correct, since users end + * up allocating memory rather than us... + */ + total = new_op->downcall.resp.listxattr.returned_count * + ORANGEFS_MAX_XATTR_NAMELEN; + goto done; + } + + returned_count = new_op->downcall.resp.listxattr.returned_count; + if (returned_count < 0 || + returned_count > ORANGEFS_MAX_XATTR_LISTLEN) { + gossip_err("%s: impossible value for returned_count:%d:\n", + __func__, + returned_count); + ret = -EIO; + goto done; + } + + /* + * Check to see how much can be fit in the buffer. Fit only whole keys. + */ + for (i = 0; i < returned_count; i++) { + if (new_op->downcall.resp.listxattr.lengths[i] < 0 || + new_op->downcall.resp.listxattr.lengths[i] > + ORANGEFS_MAX_XATTR_NAMELEN) { + gossip_err("%s: impossible value for lengths[%d]\n", + __func__, + new_op->downcall.resp.listxattr.lengths[i]); + ret = -EIO; + goto done; + } + if (total + new_op->downcall.resp.listxattr.lengths[i] > size) + goto done; + + /* + * Since many dumb programs try to setxattr() on our reserved + * xattrs this is a feeble attempt at defeating those by not + * listing them in the output of listxattr.. sigh + */ + if (is_reserved_key(new_op->downcall.resp.listxattr.key + + key_size, + new_op->downcall.resp. + listxattr.lengths[i])) { + gossip_debug(GOSSIP_XATTR_DEBUG, "Copying key %d -> %s\n", + i, new_op->downcall.resp.listxattr.key + + key_size); + memcpy(buffer + total, + new_op->downcall.resp.listxattr.key + key_size, + new_op->downcall.resp.listxattr.lengths[i]); + total += new_op->downcall.resp.listxattr.lengths[i]; + count_keys++; + } else { + gossip_debug(GOSSIP_XATTR_DEBUG, "[RESERVED] key %d -> %s\n", + i, new_op->downcall.resp.listxattr.key + + key_size); + } + key_size += new_op->downcall.resp.listxattr.lengths[i]; + } + + /* + * Since the buffer was large enough, we might have to continue + * fetching more keys! + */ + token = new_op->downcall.resp.listxattr.token; + if (token != ORANGEFS_ITERATE_END) + goto try_again; + +done: + gossip_debug(GOSSIP_XATTR_DEBUG, "%s: returning %d" + " [size of buffer %ld] (filled in %d keys)\n", + __func__, + ret ? (int)ret : (int)total, + (long)size, + count_keys); + op_release(new_op); + if (ret == 0) + ret = total; +out_unlock: + up_read(&orangefs_inode->xattr_sem); + return ret; +} + +static int orangefs_xattr_set_default(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *unused, + struct inode *inode, + const char *name, + const void *buffer, + size_t size, + int flags) +{ + return orangefs_inode_setxattr(inode, name, buffer, size, flags); +} + +static int orangefs_xattr_get_default(const struct xattr_handler *handler, + struct dentry *unused, + struct inode *inode, + const char *name, + void *buffer, + size_t size) +{ + return orangefs_inode_getxattr(inode, name, buffer, size); + +} + +static const struct xattr_handler orangefs_xattr_default_handler = { + .prefix = "", /* match any name => handlers called with full name */ + .get = orangefs_xattr_get_default, + .set = orangefs_xattr_set_default, +}; + +const struct xattr_handler * const orangefs_xattr_handlers[] = { + &orangefs_xattr_default_handler, + NULL +}; diff --git a/fs/overlayfs/Kconfig b/fs/overlayfs/Kconfig new file mode 100644 index 000000000..2ac67e04a --- /dev/null +++ b/fs/overlayfs/Kconfig @@ -0,0 +1,136 @@ +# SPDX-License-Identifier: GPL-2.0-only +config OVERLAY_FS + tristate "Overlay filesystem support" + select FS_STACK + select EXPORTFS + help + An overlay filesystem combines two filesystems - an 'upper' filesystem + and a 'lower' filesystem. When a name exists in both filesystems, the + object in the 'upper' filesystem is visible while the object in the + 'lower' filesystem is either hidden or, in the case of directories, + merged with the 'upper' object. + + For more information see Documentation/filesystems/overlayfs.rst + +config OVERLAY_FS_REDIRECT_DIR + bool "Overlayfs: turn on redirect directory feature by default" + depends on OVERLAY_FS + help + If this config option is enabled then overlay filesystems will use + redirects when renaming directories by default. In this case it is + still possible to turn off redirects globally with the + "redirect_dir=off" module option or on a filesystem instance basis + with the "redirect_dir=off" mount option. + + Note, that redirects are not backward compatible. That is, mounting + an overlay which has redirects on a kernel that doesn't support this + feature will have unexpected results. + + If unsure, say N. + +config OVERLAY_FS_REDIRECT_ALWAYS_FOLLOW + bool "Overlayfs: follow redirects even if redirects are turned off" + default y + depends on OVERLAY_FS + help + Disable this to get a possibly more secure configuration, but that + might not be backward compatible with previous kernels. + + If backward compatibility is not an issue, then it is safe and + recommended to say N here. + + For more information, see Documentation/filesystems/overlayfs.rst + + If unsure, say Y. + +config OVERLAY_FS_INDEX + bool "Overlayfs: turn on inodes index feature by default" + depends on OVERLAY_FS + help + If this config option is enabled then overlay filesystems will use + the index directory to map lower inodes to upper inodes by default. + In this case it is still possible to turn off index globally with the + "index=off" module option or on a filesystem instance basis with the + "index=off" mount option. + + The inodes index feature prevents breaking of lower hardlinks on copy + up. + + Note, that the inodes index feature is not backward compatible. + That is, mounting an overlay which has an inodes index on a kernel + that doesn't support this feature will have unexpected results. + + If unsure, say N. + +config OVERLAY_FS_NFS_EXPORT + bool "Overlayfs: turn on NFS export feature by default" + depends on OVERLAY_FS + depends on OVERLAY_FS_INDEX + depends on !OVERLAY_FS_METACOPY + help + If this config option is enabled then overlay filesystems will use + the index directory to decode overlay NFS file handles by default. + In this case, it is still possible to turn off NFS export support + globally with the "nfs_export=off" module option or on a filesystem + instance basis with the "nfs_export=off" mount option. + + The NFS export feature creates an index on copy up of every file and + directory. This full index is used to detect overlay filesystems + inconsistencies on lookup, like redirect from multiple upper dirs to + the same lower dir. The full index may incur some overhead on mount + time, especially when verifying that directory file handles are not + stale. + + Note, that the NFS export feature is not backward compatible. + That is, mounting an overlay which has a full index on a kernel + that doesn't support this feature will have unexpected results. + + Most users should say N here and enable this feature on a case-by- + case basis with the "nfs_export=on" mount option. + + Say N unless you fully understand the consequences. + +config OVERLAY_FS_XINO_AUTO + bool "Overlayfs: auto enable inode number mapping" + default n + depends on OVERLAY_FS + depends on 64BIT + help + If this config option is enabled then overlay filesystems will use + unused high bits in underlying filesystem inode numbers to map all + inodes to a unified address space. The mapped 64bit inode numbers + might not be compatible with applications that expect 32bit inodes. + + If compatibility with applications that expect 32bit inodes is not an + issue, then it is safe and recommended to say Y here. + + For more information, see Documentation/filesystems/overlayfs.rst + + If unsure, say N. + +config OVERLAY_FS_METACOPY + bool "Overlayfs: turn on metadata only copy up feature by default" + depends on OVERLAY_FS + select OVERLAY_FS_REDIRECT_DIR + help + If this config option is enabled then overlay filesystems will + copy up only metadata where appropriate and data copy up will + happen when a file is opened for WRITE operation. It is still + possible to turn off this feature globally with the "metacopy=off" + module option or on a filesystem instance basis with the + "metacopy=off" mount option. + + Note, that this feature is not backward compatible. That is, + mounting an overlay which has metacopy only inodes on a kernel + that doesn't support this feature will have unexpected results. + + If unsure, say N. + +config OVERLAY_FS_DEBUG + bool "Overlayfs: turn on extra debugging checks" + default n + depends on OVERLAY_FS + help + Say Y here to enable extra debugging checks in overlayfs. + + If unsure, say N. diff --git a/fs/overlayfs/Makefile b/fs/overlayfs/Makefile new file mode 100644 index 000000000..5648954f8 --- /dev/null +++ b/fs/overlayfs/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the overlay filesystem. +# + +obj-$(CONFIG_OVERLAY_FS) += overlay.o + +overlay-objs := super.o namei.o util.o inode.o file.o dir.o readdir.o \ + copy_up.o export.o params.o xattrs.o diff --git a/fs/overlayfs/copy_up.c b/fs/overlayfs/copy_up.c new file mode 100644 index 000000000..e963701b4 --- /dev/null +++ b/fs/overlayfs/copy_up.c @@ -0,0 +1,1291 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * + * Copyright (C) 2011 Novell Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "overlayfs.h" + +#define OVL_COPY_UP_CHUNK_SIZE (1 << 20) + +static int ovl_ccup_set(const char *buf, const struct kernel_param *param) +{ + pr_warn("\"check_copy_up\" module option is obsolete\n"); + return 0; +} + +static int ovl_ccup_get(char *buf, const struct kernel_param *param) +{ + return sprintf(buf, "N\n"); +} + +module_param_call(check_copy_up, ovl_ccup_set, ovl_ccup_get, NULL, 0644); +MODULE_PARM_DESC(check_copy_up, "Obsolete; does nothing"); + +static bool ovl_must_copy_xattr(const char *name) +{ + return !strcmp(name, XATTR_POSIX_ACL_ACCESS) || + !strcmp(name, XATTR_POSIX_ACL_DEFAULT) || + !strncmp(name, XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN); +} + +static int ovl_copy_acl(struct ovl_fs *ofs, const struct path *path, + struct dentry *dentry, const char *acl_name) +{ + int err; + struct posix_acl *clone, *real_acl = NULL; + + real_acl = ovl_get_acl_path(path, acl_name, false); + if (!real_acl) + return 0; + + if (IS_ERR(real_acl)) { + err = PTR_ERR(real_acl); + if (err == -ENODATA || err == -EOPNOTSUPP) + return 0; + return err; + } + + clone = posix_acl_clone(real_acl, GFP_KERNEL); + posix_acl_release(real_acl); /* release original acl */ + if (!clone) + return -ENOMEM; + + err = ovl_do_set_acl(ofs, dentry, acl_name, clone); + + /* release cloned acl */ + posix_acl_release(clone); + return err; +} + +int ovl_copy_xattr(struct super_block *sb, const struct path *oldpath, struct dentry *new) +{ + struct dentry *old = oldpath->dentry; + ssize_t list_size, size, value_size = 0; + char *buf, *name, *value = NULL; + int error = 0; + size_t slen; + + if (!old->d_inode->i_op->listxattr || !new->d_inode->i_op->listxattr) + return 0; + + list_size = vfs_listxattr(old, NULL, 0); + if (list_size <= 0) { + if (list_size == -EOPNOTSUPP) + return 0; + return list_size; + } + + buf = kvzalloc(list_size, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + list_size = vfs_listxattr(old, buf, list_size); + if (list_size <= 0) { + error = list_size; + goto out; + } + + for (name = buf; list_size; name += slen) { + slen = strnlen(name, list_size) + 1; + + /* underlying fs providing us with an broken xattr list? */ + if (WARN_ON(slen > list_size)) { + error = -EIO; + break; + } + list_size -= slen; + + if (ovl_is_private_xattr(sb, name)) + continue; + + error = security_inode_copy_up_xattr(old, name); + if (error == -ECANCELED) { + error = 0; + continue; /* Discard */ + } + if (error < 0 && error != -EOPNOTSUPP) + break; + + if (is_posix_acl_xattr(name)) { + error = ovl_copy_acl(OVL_FS(sb), oldpath, new, name); + if (!error) + continue; + /* POSIX ACLs must be copied. */ + break; + } + +retry: + size = ovl_do_getxattr(oldpath, name, value, value_size); + if (size == -ERANGE) + size = ovl_do_getxattr(oldpath, name, NULL, 0); + + if (size < 0) { + error = size; + break; + } + + if (size > value_size) { + void *new; + + new = kvmalloc(size, GFP_KERNEL); + if (!new) { + error = -ENOMEM; + break; + } + kvfree(value); + value = new; + value_size = size; + goto retry; + } + + error = ovl_do_setxattr(OVL_FS(sb), new, name, value, size, 0); + if (error) { + if (error != -EOPNOTSUPP || ovl_must_copy_xattr(name)) + break; + + /* Ignore failure to copy unknown xattrs */ + error = 0; + } + } + kvfree(value); +out: + kvfree(buf); + return error; +} + +static int ovl_copy_fileattr(struct inode *inode, const struct path *old, + const struct path *new) +{ + struct file_kattr oldfa = { .flags_valid = true }; + struct file_kattr newfa = { .flags_valid = true }; + int err; + + err = ovl_real_fileattr_get(old, &oldfa); + if (err) { + /* Ntfs-3g returns -EINVAL for "no fileattr support" */ + if (err == -ENOTTY || err == -EINVAL) + return 0; + pr_warn("failed to retrieve lower fileattr (%pd2, err=%i)\n", + old->dentry, err); + return err; + } + + /* + * We cannot set immutable and append-only flags on upper inode, + * because we would not be able to link upper inode to upper dir + * not set overlay private xattr on upper inode. + * Store these flags in overlay.protattr xattr instead. + */ + if (oldfa.flags & OVL_PROT_FS_FLAGS_MASK) { + err = ovl_set_protattr(inode, new->dentry, &oldfa); + if (err == -EPERM) + pr_warn_once("copying fileattr: no xattr on upper\n"); + else if (err) + return err; + } + + /* Don't bother copying flags if none are set */ + if (!(oldfa.flags & OVL_COPY_FS_FLAGS_MASK)) + return 0; + + err = ovl_real_fileattr_get(new, &newfa); + if (err) { + /* + * Returning an error if upper doesn't support fileattr will + * result in a regression, so revert to the old behavior. + */ + if (err == -ENOTTY || err == -EINVAL) { + pr_warn_once("copying fileattr: no support on upper\n"); + return 0; + } + pr_warn("failed to retrieve upper fileattr (%pd2, err=%i)\n", + new->dentry, err); + return err; + } + + BUILD_BUG_ON(OVL_COPY_FS_FLAGS_MASK & ~FS_COMMON_FL); + newfa.flags &= ~OVL_COPY_FS_FLAGS_MASK; + newfa.flags |= (oldfa.flags & OVL_COPY_FS_FLAGS_MASK); + + BUILD_BUG_ON(OVL_COPY_FSX_FLAGS_MASK & ~FS_XFLAG_COMMON); + newfa.fsx_xflags &= ~OVL_COPY_FSX_FLAGS_MASK; + newfa.fsx_xflags |= (oldfa.fsx_xflags & OVL_COPY_FSX_FLAGS_MASK); + + return ovl_real_fileattr_set(new, &newfa); +} + +static int ovl_verify_area(loff_t pos, loff_t pos2, loff_t len, loff_t totlen) +{ + loff_t tmp; + + if (pos != pos2) + return -EIO; + if (pos < 0 || len < 0 || totlen < 0) + return -EIO; + if (check_add_overflow(pos, len, &tmp)) + return -EIO; + return 0; +} + +static int ovl_sync_file(const struct path *path) +{ + struct file *new_file; + int err; + + new_file = ovl_path_open(path, O_LARGEFILE | O_RDONLY); + if (IS_ERR(new_file)) + return PTR_ERR(new_file); + + err = vfs_fsync(new_file, 0); + fput(new_file); + + return err; +} + +static int ovl_copy_up_file(struct ovl_fs *ofs, struct dentry *dentry, + struct file *new_file, loff_t len, + bool datasync) +{ + struct path datapath; + struct file *old_file; + loff_t old_pos = 0; + loff_t new_pos = 0; + loff_t cloned; + loff_t data_pos = -1; + loff_t hole_len; + bool skip_hole = false; + int error = 0; + + ovl_path_lowerdata(dentry, &datapath); + if (WARN_ON_ONCE(datapath.dentry == NULL) || + WARN_ON_ONCE(len < 0)) + return -EIO; + + old_file = ovl_path_open(&datapath, O_LARGEFILE | O_RDONLY); + if (IS_ERR(old_file)) + return PTR_ERR(old_file); + + /* Try to use clone_file_range to clone up within the same fs */ + cloned = vfs_clone_file_range(old_file, 0, new_file, 0, len, 0); + if (cloned == len) + goto out_fput; + + /* Couldn't clone, so now we try to copy the data */ + error = rw_verify_area(READ, old_file, &old_pos, len); + if (!error) + error = rw_verify_area(WRITE, new_file, &new_pos, len); + if (error) + goto out_fput; + + /* Check if lower fs supports seek operation */ + if (old_file->f_mode & FMODE_LSEEK) + skip_hole = true; + + while (len) { + size_t this_len = OVL_COPY_UP_CHUNK_SIZE; + ssize_t bytes; + + if (len < this_len) + this_len = len; + + if (signal_pending_state(TASK_KILLABLE, current)) { + error = -EINTR; + break; + } + + /* + * Fill zero for hole will cost unnecessary disk space + * and meanwhile slow down the copy-up speed, so we do + * an optimization for hole during copy-up, it relies + * on SEEK_DATA implementation in lower fs so if lower + * fs does not support it, copy-up will behave as before. + * + * Detail logic of hole detection as below: + * When we detect next data position is larger than current + * position we will skip that hole, otherwise we copy + * data in the size of OVL_COPY_UP_CHUNK_SIZE. Actually, + * it may not recognize all kind of holes and sometimes + * only skips partial of hole area. However, it will be + * enough for most of the use cases. + * + * We do not hold upper sb_writers throughout the loop to avert + * lockdep warning with llseek of lower file in nested overlay: + * - upper sb_writers + * -- lower ovl_inode_lock (ovl_llseek) + */ + if (skip_hole && data_pos < old_pos) { + data_pos = vfs_llseek(old_file, old_pos, SEEK_DATA); + if (data_pos > old_pos) { + hole_len = data_pos - old_pos; + len -= hole_len; + old_pos = new_pos = data_pos; + continue; + } else if (data_pos == -ENXIO) { + break; + } else if (data_pos < 0) { + skip_hole = false; + } + } + + error = ovl_verify_area(old_pos, new_pos, this_len, len); + if (error) + break; + + bytes = do_splice_direct(old_file, &old_pos, + new_file, &new_pos, + this_len, SPLICE_F_MOVE); + if (bytes <= 0) { + error = bytes; + break; + } + WARN_ON(old_pos != new_pos); + + len -= bytes; + } + /* call fsync once, either now or later along with metadata */ + if (!error && ovl_should_sync(ofs) && datasync) + error = vfs_fsync(new_file, 0); +out_fput: + fput(old_file); + return error; +} + +static int ovl_set_size(struct ovl_fs *ofs, + struct dentry *upperdentry, struct kstat *stat) +{ + struct iattr attr = { + .ia_valid = ATTR_SIZE, + .ia_size = stat->size, + }; + + return ovl_do_notify_change(ofs, upperdentry, &attr); +} + +static int ovl_set_timestamps(struct ovl_fs *ofs, struct dentry *upperdentry, + struct kstat *stat) +{ + struct iattr attr = { + .ia_valid = + ATTR_ATIME | ATTR_MTIME | ATTR_ATIME_SET | ATTR_MTIME_SET | ATTR_CTIME, + .ia_atime = stat->atime, + .ia_mtime = stat->mtime, + }; + + return ovl_do_notify_change(ofs, upperdentry, &attr); +} + +int ovl_set_attr(struct ovl_fs *ofs, struct dentry *upperdentry, + struct kstat *stat) +{ + int err = 0; + + if (!S_ISLNK(stat->mode)) { + struct iattr attr = { + .ia_valid = ATTR_MODE, + .ia_mode = stat->mode, + }; + err = ovl_do_notify_change(ofs, upperdentry, &attr); + } + if (!err) { + struct iattr attr = { + .ia_valid = ATTR_UID | ATTR_GID, + .ia_vfsuid = VFSUIDT_INIT(stat->uid), + .ia_vfsgid = VFSGIDT_INIT(stat->gid), + }; + err = ovl_do_notify_change(ofs, upperdentry, &attr); + } + if (!err) + ovl_set_timestamps(ofs, upperdentry, stat); + + return err; +} + +struct ovl_fh *ovl_encode_real_fh(struct ovl_fs *ofs, struct inode *realinode, + bool is_upper) +{ + struct ovl_fh *fh; + int fh_type, dwords; + int buflen = MAX_HANDLE_SZ; + uuid_t *uuid = &realinode->i_sb->s_uuid; + int err; + + /* Make sure the real fid stays 32bit aligned */ + BUILD_BUG_ON(OVL_FH_FID_OFFSET % 4); + BUILD_BUG_ON(MAX_HANDLE_SZ + OVL_FH_FID_OFFSET > 255); + + fh = kzalloc(buflen + OVL_FH_FID_OFFSET, GFP_KERNEL); + if (!fh) + return ERR_PTR(-ENOMEM); + + /* + * We encode a non-connectable file handle for non-dir, because we + * only need to find the lower inode number and we don't want to pay + * the price or reconnecting the dentry. + */ + dwords = buflen >> 2; + fh_type = exportfs_encode_inode_fh(realinode, (void *)fh->fb.fid, + &dwords, NULL, 0); + buflen = (dwords << 2); + + err = -EIO; + if (fh_type < 0 || fh_type == FILEID_INVALID || + WARN_ON(buflen > MAX_HANDLE_SZ)) + goto out_err; + + fh->fb.version = OVL_FH_VERSION; + fh->fb.magic = OVL_FH_MAGIC; + fh->fb.type = fh_type; + fh->fb.flags = OVL_FH_FLAG_CPU_ENDIAN; + /* + * When we will want to decode an overlay dentry from this handle + * and all layers are on the same fs, if we get a disconncted real + * dentry when we decode fid, the only way to tell if we should assign + * it to upperdentry or to lowerstack is by checking this flag. + */ + if (is_upper) + fh->fb.flags |= OVL_FH_FLAG_PATH_UPPER; + fh->fb.len = sizeof(fh->fb) + buflen; + if (ovl_origin_uuid(ofs)) + fh->fb.uuid = *uuid; + + return fh; + +out_err: + kfree(fh); + return ERR_PTR(err); +} + +struct ovl_fh *ovl_get_origin_fh(struct ovl_fs *ofs, struct dentry *origin) +{ + /* + * When lower layer doesn't support export operations store a 'null' fh, + * so we can use the overlay.origin xattr to distignuish between a copy + * up and a pure upper inode. + */ + if (!ovl_can_decode_fh(origin->d_sb)) + return NULL; + + return ovl_encode_real_fh(ofs, d_inode(origin), false); +} + +int ovl_set_origin_fh(struct ovl_fs *ofs, const struct ovl_fh *fh, + struct dentry *upper) +{ + int err; + + /* + * Do not fail when upper doesn't support xattrs. + */ + err = ovl_check_setxattr(ofs, upper, OVL_XATTR_ORIGIN, fh->buf, + fh ? fh->fb.len : 0, 0); + + /* Ignore -EPERM from setting "user.*" on symlink/special */ + return err == -EPERM ? 0 : err; +} + +/* Store file handle of @upper dir in @index dir entry */ +static int ovl_set_upper_fh(struct ovl_fs *ofs, struct dentry *upper, + struct dentry *index) +{ + const struct ovl_fh *fh; + int err; + + fh = ovl_encode_real_fh(ofs, d_inode(upper), true); + if (IS_ERR(fh)) + return PTR_ERR(fh); + + err = ovl_setxattr(ofs, index, OVL_XATTR_UPPER, fh->buf, fh->fb.len); + + kfree(fh); + return err; +} + +/* + * Create and install index entry. + */ +static int ovl_create_index(struct dentry *dentry, const struct ovl_fh *fh, + struct dentry *upper) +{ + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + struct dentry *indexdir = ovl_indexdir(dentry->d_sb); + struct dentry *temp = NULL; + struct renamedata rd = {}; + struct qstr name = { }; + int err; + + /* + * For now this is only used for creating index entry for directories, + * because non-dir are copied up directly to index and then hardlinked + * to upper dir. + * + * TODO: implement create index for non-dir, so we can call it when + * encoding file handle for non-dir in case index does not exist. + */ + if (WARN_ON(!d_is_dir(dentry))) + return -EIO; + + /* Directory not expected to be indexed before copy up */ + if (WARN_ON(ovl_test_flag(OVL_INDEX, d_inode(dentry)))) + return -EIO; + + err = ovl_get_index_name_fh(fh, &name); + if (err) + return err; + + temp = ovl_create_temp(ofs, indexdir, OVL_CATTR(S_IFDIR | 0)); + err = PTR_ERR(temp); + if (IS_ERR(temp)) + goto free_name; + + err = ovl_set_upper_fh(ofs, upper, temp); + if (err) + goto out; + + rd.mnt_idmap = ovl_upper_mnt_idmap(ofs); + rd.old_parent = indexdir; + rd.new_parent = indexdir; + err = start_renaming_dentry(&rd, 0, temp, &name); + if (err) + goto out; + + err = ovl_do_rename_rd(&rd); + end_renaming(&rd); +out: + if (err) + ovl_cleanup(ofs, indexdir, temp); + dput(temp); +free_name: + kfree(name.name); + return err; +} + +struct ovl_copy_up_ctx { + struct dentry *parent; + struct dentry *dentry; + struct path lowerpath; + struct kstat stat; + struct kstat pstat; + const char *link; + struct dentry *destdir; + struct qstr destname; + struct dentry *workdir; + const struct ovl_fh *origin_fh; + bool origin; + bool indexed; + bool metacopy; + bool metacopy_digest; + bool metadata_fsync; +}; + +static int ovl_link_up(struct ovl_copy_up_ctx *c) +{ + int err; + struct dentry *upper; + struct dentry *upperdir = ovl_dentry_upper(c->parent); + struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb); + struct inode *udir = d_inode(upperdir); + + ovl_start_write(c->dentry); + + /* Mark parent "impure" because it may now contain non-pure upper */ + err = ovl_set_impure(c->parent, upperdir); + if (err) + goto out; + + err = ovl_set_nlink_lower(c->dentry); + if (err) + goto out; + + upper = ovl_start_creating_upper(ofs, upperdir, + &QSTR_LEN(c->dentry->d_name.name, + c->dentry->d_name.len)); + err = PTR_ERR(upper); + if (!IS_ERR(upper)) { + err = ovl_do_link(ofs, ovl_dentry_upper(c->dentry), udir, upper); + + if (!err) { + /* Restore timestamps on parent (best effort) */ + ovl_set_timestamps(ofs, upperdir, &c->pstat); + ovl_dentry_set_upper_alias(c->dentry); + ovl_dentry_update_reval(c->dentry, upper); + } + end_creating(upper); + } + if (err) + goto out; + + err = ovl_set_nlink_upper(c->dentry); + +out: + ovl_end_write(c->dentry); + return err; +} + +static int ovl_copy_up_data(struct ovl_copy_up_ctx *c, const struct path *temp) +{ + struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb); + struct file *new_file; + int err; + + if (!S_ISREG(c->stat.mode) || c->metacopy || !c->stat.size) + return 0; + + new_file = ovl_path_open(temp, O_LARGEFILE | O_WRONLY); + if (IS_ERR(new_file)) + return PTR_ERR(new_file); + + err = ovl_copy_up_file(ofs, c->dentry, new_file, c->stat.size, + !c->metadata_fsync); + fput(new_file); + + return err; +} + +static int ovl_copy_up_metadata(struct ovl_copy_up_ctx *c, struct dentry *temp) +{ + struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb); + struct inode *inode = d_inode(c->dentry); + struct path upperpath = { .mnt = ovl_upper_mnt(ofs), .dentry = temp }; + int err; + + err = ovl_copy_xattr(c->dentry->d_sb, &c->lowerpath, temp); + if (err) + return err; + + if (inode->i_flags & OVL_FATTR_I_FLAGS_MASK && + (S_ISREG(c->stat.mode) || S_ISDIR(c->stat.mode))) { + /* + * Copy the fileattr inode flags that are the source of already + * copied i_flags + */ + err = ovl_copy_fileattr(inode, &c->lowerpath, &upperpath); + if (err) + return err; + } + + /* + * Store identifier of lower inode in upper inode xattr to + * allow lookup of the copy up origin inode. + * + * Don't set origin when we are breaking the association with a lower + * hard link. + */ + if (c->origin) { + err = ovl_set_origin_fh(ofs, c->origin_fh, temp); + if (err) + return err; + } + + if (c->metacopy) { + struct path lowerdatapath; + struct ovl_metacopy metacopy_data = OVL_METACOPY_INIT; + + ovl_path_lowerdata(c->dentry, &lowerdatapath); + if (WARN_ON_ONCE(lowerdatapath.dentry == NULL)) + return -EIO; + err = ovl_get_verity_digest(ofs, &lowerdatapath, &metacopy_data); + if (err) + return err; + + if (metacopy_data.digest_algo) + c->metacopy_digest = true; + + err = ovl_set_metacopy_xattr(ofs, temp, &metacopy_data); + if (err) + return err; + } + + inode_lock(temp->d_inode); + if (S_ISREG(c->stat.mode)) + err = ovl_set_size(ofs, temp, &c->stat); + if (!err) + err = ovl_set_attr(ofs, temp, &c->stat); + inode_unlock(temp->d_inode); + + /* fsync metadata before moving it into upper dir */ + if (!err && ovl_should_sync(ofs) && c->metadata_fsync) + err = ovl_sync_file(&upperpath); + + return err; +} + +static const struct cred *ovl_prepare_copy_up_creds(struct dentry *dentry) +{ + struct cred *copy_up_cred = NULL; + int err; + + err = security_inode_copy_up(dentry, ©_up_cred); + if (err < 0) + return ERR_PTR(err); + + if (!copy_up_cred) + return NULL; + + return override_creds(copy_up_cred); +} + +static void ovl_revert_copy_up_creds(const struct cred *orig_cred) +{ + const struct cred *copy_up_cred; + + copy_up_cred = revert_creds(orig_cred); + put_cred(copy_up_cred); +} + +DEFINE_CLASS(copy_up_creds, const struct cred *, + if (!IS_ERR_OR_NULL(_T)) ovl_revert_copy_up_creds(_T), + ovl_prepare_copy_up_creds(dentry), struct dentry *dentry) + +/* + * Copyup using workdir to prepare temp file. Used when copying up directories, + * special files or when upper fs doesn't support O_TMPFILE. + */ +static int ovl_copy_up_workdir(struct ovl_copy_up_ctx *c) +{ + struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb); + struct inode *inode; + struct path path = { .mnt = ovl_upper_mnt(ofs) }; + struct renamedata rd = {}; + struct dentry *temp; + int err; + struct ovl_cattr cattr = { + /* Can't properly set mode on creation because of the umask */ + .mode = c->stat.mode & S_IFMT, + .rdev = c->stat.rdev, + .link = c->link + }; + + scoped_class(copy_up_creds, copy_up_creds, c->dentry) { + if (IS_ERR(copy_up_creds)) + return PTR_ERR(copy_up_creds); + + ovl_start_write(c->dentry); + temp = ovl_create_temp(ofs, c->workdir, &cattr); + ovl_end_write(c->dentry); + } + + if (IS_ERR(temp)) + return PTR_ERR(temp); + + /* + * Copy up data first and then xattrs. Writing data after + * xattrs will remove security.capability xattr automatically. + */ + path.dentry = temp; + err = ovl_copy_up_data(c, &path); + ovl_start_write(c->dentry); + if (err) + goto cleanup_unlocked; + + if (S_ISDIR(c->stat.mode) && c->indexed) { + err = ovl_create_index(c->dentry, c->origin_fh, temp); + if (err) + goto cleanup_unlocked; + } + + /* + * We cannot hold lock_rename() throughout this helper, because of + * lock ordering with sb_writers, which shouldn't be held when calling + * ovl_copy_up_data(), so lock workdir and destdir and make sure that + * temp wasn't moved before copy up completion or cleanup. + */ + rd.mnt_idmap = ovl_upper_mnt_idmap(ofs); + rd.old_parent = c->workdir; + rd.new_parent = c->destdir; + rd.flags = 0; + err = start_renaming_dentry(&rd, 0, temp, + &QSTR_LEN(c->destname.name, c->destname.len)); + if (err) { + /* temp or workdir moved underneath us? map to -EIO */ + err = -EIO; + } + if (err) + goto cleanup_unlocked; + + err = ovl_copy_up_metadata(c, temp); + if (!err) + err = ovl_do_rename_rd(&rd); + end_renaming(&rd); + + if (err) + goto cleanup_unlocked; + + inode = d_inode(c->dentry); + if (c->metacopy_digest) + ovl_set_flag(OVL_HAS_DIGEST, inode); + else + ovl_clear_flag(OVL_HAS_DIGEST, inode); + ovl_clear_flag(OVL_VERIFIED_DIGEST, inode); + + if (!c->metacopy) + ovl_set_upperdata(inode); + ovl_inode_update(inode, temp); + if (S_ISDIR(inode->i_mode)) + ovl_set_flag(OVL_WHITEOUTS, inode); +out: + ovl_end_write(c->dentry); + + return err; + +cleanup_unlocked: + ovl_cleanup(ofs, c->workdir, temp); + dput(temp); + goto out; +} + +/* Copyup using O_TMPFILE which does not require cross dir locking */ +static int ovl_copy_up_tmpfile(struct ovl_copy_up_ctx *c) +{ + struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb); + struct inode *udir = d_inode(c->destdir); + struct dentry *temp, *upper, *newdentry = NULL; + struct file *tmpfile; + int err; + + scoped_class(copy_up_creds, copy_up_creds, c->dentry) { + if (IS_ERR(copy_up_creds)) + return PTR_ERR(copy_up_creds); + + ovl_start_write(c->dentry); + tmpfile = ovl_do_tmpfile(ofs, c->workdir, c->stat.mode); + ovl_end_write(c->dentry); + } + + if (IS_ERR(tmpfile)) + return PTR_ERR(tmpfile); + + temp = tmpfile->f_path.dentry; + if (!c->metacopy && c->stat.size) { + err = ovl_copy_up_file(ofs, c->dentry, tmpfile, c->stat.size, + !c->metadata_fsync); + if (err) + goto out_fput; + } + + ovl_start_write(c->dentry); + + err = ovl_copy_up_metadata(c, temp); + if (err) + goto out; + + upper = ovl_start_creating_upper(ofs, c->destdir, + &QSTR_LEN(c->destname.name, + c->destname.len)); + err = PTR_ERR(upper); + if (!IS_ERR(upper)) { + err = ovl_do_link(ofs, temp, udir, upper); + if (!err) { + /* + * Record the linked dentry -- not the disconnected + * O_TMPFILE dentry -- so that ->d_revalidate() on + * the upper fs sees the real parent/name. + */ + newdentry = dget(upper); + } + end_creating(upper); + } + + if (err) + goto out; + + if (c->metacopy_digest) + ovl_set_flag(OVL_HAS_DIGEST, d_inode(c->dentry)); + else + ovl_clear_flag(OVL_HAS_DIGEST, d_inode(c->dentry)); + ovl_clear_flag(OVL_VERIFIED_DIGEST, d_inode(c->dentry)); + + if (!c->metacopy) + ovl_set_upperdata(d_inode(c->dentry)); + ovl_inode_update(d_inode(c->dentry), newdentry); + +out: + ovl_end_write(c->dentry); +out_fput: + fput(tmpfile); + return err; +} + +/* + * Copy up a single dentry + * + * All renames start with copy up of source if necessary. The actual + * rename will only proceed once the copy up was successful. Copy up uses + * upper parent i_mutex for exclusion. Since rename can change d_parent it + * is possible that the copy up will lock the old parent. At that point + * the file will have already been copied up anyway. + */ +static int ovl_do_copy_up(struct ovl_copy_up_ctx *c) +{ + int err; + struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb); + struct dentry *origin = c->lowerpath.dentry; + struct ovl_fh *fh = NULL; + bool to_index = false; + + /* + * Indexed non-dir is copied up directly to the index entry and then + * hardlinked to upper dir. Indexed dir is copied up to indexdir, + * then index entry is created and then copied up dir installed. + * Copying dir up to indexdir instead of workdir simplifies locking. + */ + if (ovl_need_index(c->dentry)) { + c->indexed = true; + if (S_ISDIR(c->stat.mode)) + c->workdir = ovl_indexdir(c->dentry->d_sb); + else + to_index = true; + } + + if (S_ISDIR(c->stat.mode) || c->stat.nlink == 1 || to_index) { + fh = ovl_get_origin_fh(ofs, origin); + if (IS_ERR(fh)) + return PTR_ERR(fh); + + /* origin_fh may be NULL */ + c->origin_fh = fh; + c->origin = true; + } + + if (to_index) { + c->destdir = ovl_indexdir(c->dentry->d_sb); + err = ovl_get_index_name(ofs, origin, &c->destname); + if (err) + goto out_free_fh; + } else if (WARN_ON(!c->parent)) { + /* Disconnected dentry must be copied up to index dir */ + err = -EIO; + goto out_free_fh; + } else { + /* + * c->dentry->d_name is stabilzed by ovl_copy_up_start(), + * because if we got here, it means that c->dentry has no upper + * alias and changing ->d_name means going through ovl_rename() + * that will call ovl_copy_up() on source and target dentry. + */ + c->destname = c->dentry->d_name; + /* + * Mark parent "impure" because it may now contain non-pure + * upper + */ + ovl_start_write(c->dentry); + err = ovl_set_impure(c->parent, c->destdir); + ovl_end_write(c->dentry); + if (err) + goto out_free_fh; + } + + /* Should we copyup with O_TMPFILE or with workdir? */ + if (S_ISREG(c->stat.mode) && ofs->tmpfile) + err = ovl_copy_up_tmpfile(c); + else + err = ovl_copy_up_workdir(c); + if (err) + goto out; + + if (c->indexed) + ovl_set_flag(OVL_INDEX, d_inode(c->dentry)); + + ovl_start_write(c->dentry); + if (to_index) { + /* Initialize nlink for copy up of disconnected dentry */ + err = ovl_set_nlink_upper(c->dentry); + } else { + struct inode *udir = d_inode(c->destdir); + + /* Restore timestamps on parent (best effort) */ + inode_lock(udir); + ovl_set_timestamps(ofs, c->destdir, &c->pstat); + inode_unlock(udir); + + ovl_dentry_set_upper_alias(c->dentry); + ovl_dentry_update_reval(c->dentry, ovl_dentry_upper(c->dentry)); + } + ovl_end_write(c->dentry); + +out: + if (to_index) + kfree(c->destname.name); +out_free_fh: + kfree(fh); + return err; +} + +static bool ovl_need_meta_copy_up(struct dentry *dentry, umode_t mode, + int flags) +{ + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + + if (!ofs->config.metacopy) + return false; + + if (!S_ISREG(mode)) + return false; + + if (flags && ((OPEN_FMODE(flags) & FMODE_WRITE) || (flags & O_TRUNC))) + return false; + + /* Fall back to full copy if no fsverity on source data and we require verity */ + if (ofs->config.verity_mode == OVL_VERITY_REQUIRE) { + struct path lowerdata; + + ovl_path_lowerdata(dentry, &lowerdata); + + if (WARN_ON_ONCE(lowerdata.dentry == NULL) || + ovl_ensure_verity_loaded(&lowerdata) || + !fsverity_active(d_inode(lowerdata.dentry))) { + return false; + } + } + + return true; +} + +static ssize_t ovl_getxattr_value(const struct path *path, char *name, char **value) +{ + ssize_t res; + char *buf; + + res = ovl_do_getxattr(path, name, NULL, 0); + if (res == -ENODATA || res == -EOPNOTSUPP) + res = 0; + + if (res > 0) { + buf = kzalloc(res, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + res = ovl_do_getxattr(path, name, buf, res); + if (res < 0) + kfree(buf); + else + *value = buf; + } + return res; +} + +/* Copy up data of an inode which was copied up metadata only in the past. */ +static int ovl_copy_up_meta_inode_data(struct ovl_copy_up_ctx *c) +{ + struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb); + struct path upperpath; + int err; + char *capability = NULL; + ssize_t cap_size; + + ovl_path_upper(c->dentry, &upperpath); + if (WARN_ON(upperpath.dentry == NULL)) + return -EIO; + + if (c->stat.size) { + err = cap_size = ovl_getxattr_value(&upperpath, XATTR_NAME_CAPS, + &capability); + if (cap_size < 0) + goto out; + } + + err = ovl_copy_up_data(c, &upperpath); + if (err) + goto out_free; + + /* + * Writing to upper file will clear security.capability xattr. We + * don't want that to happen for normal copy-up operation. + */ + ovl_start_write(c->dentry); + if (capability) { + err = ovl_do_setxattr(ofs, upperpath.dentry, XATTR_NAME_CAPS, + capability, cap_size, 0); + } + if (!err) { + err = ovl_removexattr(ofs, upperpath.dentry, + OVL_XATTR_METACOPY); + } + ovl_end_write(c->dentry); + if (err) + goto out_free; + + ovl_clear_flag(OVL_HAS_DIGEST, d_inode(c->dentry)); + ovl_clear_flag(OVL_VERIFIED_DIGEST, d_inode(c->dentry)); + ovl_set_upperdata(d_inode(c->dentry)); +out_free: + kfree(capability); +out: + return err; +} + +static int ovl_copy_up_one(struct dentry *parent, struct dentry *dentry, + int flags) +{ + int err; + DEFINE_DELAYED_CALL(done); + struct path parentpath; + struct ovl_copy_up_ctx ctx = { + .parent = parent, + .dentry = dentry, + .workdir = ovl_workdir(dentry), + }; + + if (WARN_ON(!ctx.workdir)) + return -EROFS; + + ovl_path_lower(dentry, &ctx.lowerpath); + err = vfs_getattr(&ctx.lowerpath, &ctx.stat, + STATX_BASIC_STATS, AT_STATX_SYNC_AS_STAT); + if (err) + return err; + + if (!kuid_has_mapping(current_user_ns(), ctx.stat.uid) || + !kgid_has_mapping(current_user_ns(), ctx.stat.gid)) + return -EOVERFLOW; + + /* + * With "fsync=strict", we fsync after final metadata copyup, for + * both regular files and directories to get atomic copyup semantics + * on filesystems that do not use strict metadata ordering (e.g. ubifs). + * + * By default, we want to avoid fsync on all meta copyup, because + * that will hurt performance of workloads such as chown -R, so we + * only fsync on data copyup as legacy behavior. + */ + ctx.metadata_fsync = ovl_should_sync_metadata(OVL_FS(dentry->d_sb)) && + (S_ISREG(ctx.stat.mode) || S_ISDIR(ctx.stat.mode)); + ctx.metacopy = ovl_need_meta_copy_up(dentry, ctx.stat.mode, flags); + + if (parent) { + ovl_path_upper(parent, &parentpath); + ctx.destdir = parentpath.dentry; + + err = vfs_getattr(&parentpath, &ctx.pstat, + STATX_ATIME | STATX_MTIME, + AT_STATX_SYNC_AS_STAT); + if (err) + return err; + } + + /* maybe truncate regular file. this has no effect on dirs */ + if (flags & O_TRUNC) + ctx.stat.size = 0; + + if (S_ISLNK(ctx.stat.mode)) { + ctx.link = vfs_get_link(ctx.lowerpath.dentry, &done); + if (IS_ERR(ctx.link)) + return PTR_ERR(ctx.link); + } + + err = ovl_copy_up_start(dentry, flags); + /* err < 0: interrupted, err > 0: raced with another copy-up */ + if (unlikely(err)) { + if (err > 0) + err = 0; + } else { + if (!ovl_dentry_upper(dentry)) + err = ovl_do_copy_up(&ctx); + if (!err && parent && !ovl_dentry_has_upper_alias(dentry)) + err = ovl_link_up(&ctx); + if (!err && ovl_dentry_needs_data_copy_up_locked(dentry, flags)) + err = ovl_copy_up_meta_inode_data(&ctx); + ovl_copy_up_end(dentry); + } + do_delayed_call(&done); + + return err; +} + +static int ovl_copy_up_flags(struct dentry *dentry, int flags) +{ + int err = 0; + bool disconnected = (dentry->d_flags & DCACHE_DISCONNECTED); + + /* + * With NFS export, copy up can get called for a disconnected non-dir. + * In this case, we will copy up lower inode to index dir without + * linking it to upper dir. + */ + if (WARN_ON(disconnected && d_is_dir(dentry))) + return -EIO; + + /* + * We may not need lowerdata if we are only doing metacopy up, but it is + * not very important to optimize this case, so do lazy lowerdata lookup + * before any copy up, so we can do it before taking ovl_inode_lock(). + */ + err = ovl_verify_lowerdata(dentry); + if (err) + return err; + + while (!err) { + struct dentry *next; + struct dentry *parent = NULL; + + if (ovl_already_copied_up(dentry, flags)) + break; + + next = dget(dentry); + /* find the topmost dentry not yet copied up */ + for (; !disconnected;) { + parent = dget_parent(next); + + if (ovl_dentry_upper(parent)) + break; + + dput(next); + next = parent; + } + + with_ovl_creds(dentry->d_sb) + err = ovl_copy_up_one(parent, next, flags); + + dput(parent); + dput(next); + } + + return err; +} + +static bool ovl_open_need_copy_up(struct dentry *dentry, int flags) +{ + /* Copy up of disconnected dentry does not set upper alias */ + if (ovl_already_copied_up(dentry, flags)) + return false; + + if (special_file(d_inode(dentry)->i_mode)) + return false; + + if (!ovl_open_flags_need_copy_up(flags)) + return false; + + return true; +} + +int ovl_maybe_copy_up(struct dentry *dentry, int flags) +{ + if (!ovl_open_need_copy_up(dentry, flags)) + return 0; + + return ovl_copy_up_flags(dentry, flags); +} + +int ovl_copy_up_with_data(struct dentry *dentry) +{ + return ovl_copy_up_flags(dentry, O_WRONLY); +} + +int ovl_copy_up(struct dentry *dentry) +{ + return ovl_copy_up_flags(dentry, 0); +} diff --git a/fs/overlayfs/dir.c b/fs/overlayfs/dir.c new file mode 100644 index 000000000..7beb0af26 --- /dev/null +++ b/fs/overlayfs/dir.c @@ -0,0 +1,1495 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * + * Copyright (C) 2011 Novell Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "overlayfs.h" + +static unsigned short ovl_redirect_max = 256; +module_param_named(redirect_max, ovl_redirect_max, ushort, 0644); +MODULE_PARM_DESC(redirect_max, + "Maximum length of absolute redirect xattr value"); + +static int ovl_set_redirect(struct dentry *dentry, bool samedir); + +static int ovl_cleanup_locked(struct ovl_fs *ofs, struct inode *wdir, + struct dentry *wdentry) +{ + int err; + + dget(wdentry); + if (d_is_dir(wdentry)) + err = ovl_do_rmdir(ofs, wdir, wdentry); + else + err = ovl_do_unlink(ofs, wdir, wdentry); + dput(wdentry); + + if (err) { + pr_err("cleanup of '%pd2' failed (%i)\n", + wdentry, err); + } + + return err; +} + +int ovl_cleanup(struct ovl_fs *ofs, struct dentry *workdir, + struct dentry *wdentry) +{ + wdentry = start_removing_dentry(workdir, wdentry); + if (IS_ERR(wdentry)) + return PTR_ERR(wdentry); + + ovl_cleanup_locked(ofs, workdir->d_inode, wdentry); + end_removing(wdentry); + + return 0; +} + +void ovl_tempname(char name[OVL_TEMPNAME_SIZE]) +{ + static atomic_t temp_id = ATOMIC_INIT(0); + + /* counter is allowed to wrap, since temp dentries are ephemeral */ + snprintf(name, OVL_TEMPNAME_SIZE, "#%x", atomic_inc_return(&temp_id)); +} + +static struct dentry *ovl_start_creating_temp(struct ovl_fs *ofs, + struct dentry *workdir, + char name[OVL_TEMPNAME_SIZE]) +{ + ovl_tempname(name); + return start_creating(ovl_upper_mnt_idmap(ofs), workdir, + &QSTR(name)); +} + +static struct dentry *ovl_whiteout(struct ovl_fs *ofs) +{ + int err; + struct dentry *whiteout, *link; + struct dentry *workdir = ofs->workdir; + struct inode *wdir = workdir->d_inode; + char name[OVL_TEMPNAME_SIZE]; + + guard(mutex)(&ofs->whiteout_lock); + + if (!ofs->whiteout) { + whiteout = ovl_start_creating_temp(ofs, workdir, name); + if (IS_ERR(whiteout)) + return whiteout; + err = ovl_do_whiteout(ofs, wdir, whiteout); + if (!err) + ofs->whiteout = dget(whiteout); + end_creating(whiteout); + if (err) + return ERR_PTR(err); + } + + if (!ofs->no_shared_whiteout) { + link = ovl_start_creating_temp(ofs, workdir, name); + if (IS_ERR(link)) + return link; + err = ovl_do_link(ofs, ofs->whiteout, wdir, link); + if (!err) + whiteout = dget(link); + end_creating(link); + if (!err) + return whiteout; + + if (err != -EMLINK) { + pr_warn("Failed to link whiteout - disabling whiteout inode sharing(nlink=%u, err=%u)\n", + ofs->whiteout->d_inode->i_nlink, + err); + ofs->no_shared_whiteout = true; + } + } + whiteout = ofs->whiteout; + ofs->whiteout = NULL; + return whiteout; +} + +int ovl_cleanup_and_whiteout(struct ovl_fs *ofs, struct dentry *dir, + struct dentry *dentry) +{ + struct dentry *whiteout; + struct renamedata rd = {}; + int err; + int flags = 0; + + whiteout = ovl_whiteout(ofs); + err = PTR_ERR(whiteout); + if (IS_ERR(whiteout)) + return err; + + if (d_is_dir(dentry)) + flags = RENAME_EXCHANGE; + + rd.mnt_idmap = ovl_upper_mnt_idmap(ofs); + rd.old_parent = ofs->workdir; + rd.new_parent = dir; + rd.flags = flags; + err = start_renaming_two_dentries(&rd, whiteout, dentry); + if (!err) { + err = ovl_do_rename_rd(&rd); + end_renaming(&rd); + } + if (err) + goto kill_whiteout; + if (flags) + ovl_cleanup(ofs, ofs->workdir, dentry); + +out: + dput(whiteout); + return err; + +kill_whiteout: + ovl_cleanup(ofs, ofs->workdir, whiteout); + goto out; +} + +struct dentry *ovl_create_real(struct ovl_fs *ofs, struct dentry *parent, + struct dentry *newdentry, struct qstr *qname, + struct ovl_cattr *attr) +{ + struct inode *dir = parent->d_inode; + int err; + + if (IS_ERR(newdentry)) + return newdentry; + + err = -ESTALE; + if (newdentry->d_inode) + goto out; + + if (attr->hardlink) { + err = ovl_do_link(ofs, attr->hardlink, dir, newdentry); + } else { + switch (attr->mode & S_IFMT) { + case S_IFREG: + err = ovl_do_create(ofs, dir, newdentry, attr->mode); + break; + + case S_IFDIR: + /* mkdir is special... */ + newdentry = ovl_do_mkdir(ofs, dir, newdentry, attr->mode); + err = PTR_ERR_OR_ZERO(newdentry); + /* expect to inherit casefolding from workdir/upperdir */ + if (!err && ofs->casefold != ovl_dentry_casefolded(newdentry)) { + pr_warn_ratelimited("wrong inherited casefold (%pd2)\n", + newdentry); + err = -EINVAL; + } + break; + + case S_IFCHR: + case S_IFBLK: + case S_IFIFO: + case S_IFSOCK: + err = ovl_do_mknod(ofs, dir, newdentry, attr->mode, + attr->rdev); + break; + + case S_IFLNK: + err = ovl_do_symlink(ofs, dir, newdentry, attr->link); + break; + + default: + err = -EPERM; + } + } + if (err) + goto out; + + if (WARN_ON(!newdentry->d_inode)) { + /* + * Not quite sure if non-instantiated dentry is legal or not. + * VFS doesn't seem to care so check and warn here. + */ + err = -EIO; + } else if (d_unhashed(newdentry)) { + struct dentry *d; + /* + * Some filesystems (i.e. casefolded) may return an unhashed + * negative dentry from the ovl_start_creating_upper() call before + * ovl_create_real(). + * In that case, lookup again after making the newdentry + * positive, so ovl_create_upper() always returns a hashed + * positive dentry. We lookup using qname which should be + * the same name as newentry, but is certain not to change. + * As we have to drop the lock before the lookup a race + * could result in a lookup failure. In that case we return + * an error. + */ + end_creating_keep(newdentry); + d = ovl_start_creating_upper(ofs, parent, qname); + + if (IS_ERR_OR_NULL(d)) { + err = d ? PTR_ERR(d) : -ENOENT; + } else if (d->d_inode != newdentry->d_inode) { + err = -EIO; + } else { + dput(newdentry); + return d; + } + end_creating(d); + dput(newdentry); + return ERR_PTR(err); + } +out: + if (err) { + end_creating(newdentry); + return ERR_PTR(err); + } + return newdentry; +} + +struct dentry *ovl_create_temp(struct ovl_fs *ofs, struct dentry *workdir, + struct ovl_cattr *attr) +{ + struct dentry *ret; + char name[OVL_TEMPNAME_SIZE]; + + ret = ovl_start_creating_temp(ofs, workdir, name); + if (IS_ERR(ret)) + return ret; + ret = ovl_create_real(ofs, workdir, ret, &QSTR(name), attr); + return end_creating_keep(ret); +} + +static int ovl_set_opaque_xerr(struct dentry *dentry, struct dentry *upper, + int xerr) +{ + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + int err; + + err = ovl_check_setxattr(ofs, upper, OVL_XATTR_OPAQUE, "y", 1, xerr); + if (!err) + ovl_dentry_set_opaque(dentry); + + return err; +} + +static int ovl_set_opaque(struct dentry *dentry, struct dentry *upperdentry) +{ + /* + * Fail with -EIO when trying to create opaque dir and upper doesn't + * support xattrs. ovl_rename() calls ovl_set_opaque_xerr(-EXDEV) to + * return a specific error for noxattr case. + */ + return ovl_set_opaque_xerr(dentry, upperdentry, -EIO); +} + +/* + * Common operations required to be done after creation of file on upper. + * If @hardlink is false, then @inode is a pre-allocated inode, we may or + * may not use to instantiate the new dentry. + */ +static int ovl_instantiate(struct dentry *dentry, struct inode *inode, + struct dentry *newdentry, bool hardlink, struct file *tmpfile) +{ + struct ovl_inode_params oip = { + .upperdentry = newdentry, + .newinode = inode, + }; + + ovl_dentry_set_upper_alias(dentry); + ovl_dentry_init_reval(dentry, newdentry, NULL); + + if (!hardlink) { + /* + * ovl_obtain_alias() can be called after ovl_create_real() + * and before we get here, so we may get an inode from cache + * with the same real upperdentry that is not the inode we + * pre-allocated. In this case we will use the cached inode + * to instantiate the new dentry. + * + * XXX: if we ever use ovl_obtain_alias() to decode directory + * file handles, need to use ovl_get_inode_locked() and + * d_instantiate_new() here to prevent from creating two + * hashed directory inode aliases. We then need to return + * the obtained alias to ovl_mkdir(). + */ + inode = ovl_get_inode(dentry->d_sb, &oip); + if (IS_ERR(inode)) + return PTR_ERR(inode); + if (inode == oip.newinode) + ovl_set_flag(OVL_UPPERDATA, inode); + } else { + WARN_ON(ovl_inode_real(inode) != d_inode(newdentry)); + dput(newdentry); + inc_nlink(inode); + } + + if (tmpfile) + d_mark_tmpfile(tmpfile, inode); + + d_instantiate(dentry, inode); + if (inode != oip.newinode) { + pr_warn_ratelimited("newly created inode found in cache (%pd2)\n", + dentry); + } + + /* Force lookup of new upper hardlink to find its lower */ + if (hardlink) + d_drop(dentry); + + return 0; +} + +static bool ovl_type_merge(struct dentry *dentry) +{ + return OVL_TYPE_MERGE(ovl_path_type(dentry)); +} + +static bool ovl_type_origin(struct dentry *dentry) +{ + return OVL_TYPE_ORIGIN(ovl_path_type(dentry)); +} + +static int ovl_create_upper(struct dentry *dentry, struct inode *inode, + struct ovl_cattr *attr) +{ + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + struct dentry *upperdir = ovl_dentry_upper(dentry->d_parent); + struct dentry *newdentry; + struct qstr qname = QSTR_LEN(dentry->d_name.name, + dentry->d_name.len); + int err; + + newdentry = ovl_start_creating_upper(ofs, upperdir, + &qname); + if (IS_ERR(newdentry)) + return PTR_ERR(newdentry); + newdentry = ovl_create_real(ofs, upperdir, newdentry, &qname, attr); + if (IS_ERR(newdentry)) + return PTR_ERR(newdentry); + + end_creating_keep(newdentry); + + if (ovl_type_merge(dentry->d_parent) && d_is_dir(newdentry) && + !ovl_allow_offline_changes(ofs)) { + /* Setting opaque here is just an optimization, allow to fail */ + ovl_set_opaque(dentry, newdentry); + } + + ovl_dir_modified(dentry->d_parent, false); + err = ovl_instantiate(dentry, inode, newdentry, !!attr->hardlink, NULL); + if (err) + goto out_cleanup; + return 0; + +out_cleanup: + ovl_cleanup(ofs, upperdir, newdentry); + dput(newdentry); + return err; +} + +static struct dentry *ovl_clear_empty(struct dentry *dentry, + struct list_head *list) +{ + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + struct dentry *workdir = ovl_workdir(dentry); + struct dentry *upperdir = ovl_dentry_upper(dentry->d_parent); + struct renamedata rd = {}; + struct path upperpath; + struct dentry *upper; + struct dentry *opaquedir; + struct kstat stat; + int err; + + if (WARN_ON(!workdir)) + return ERR_PTR(-EROFS); + + ovl_path_upper(dentry, &upperpath); + err = vfs_getattr(&upperpath, &stat, + STATX_BASIC_STATS, AT_STATX_SYNC_AS_STAT); + if (err) + goto out; + + err = -ESTALE; + if (!S_ISDIR(stat.mode)) + goto out; + upper = upperpath.dentry; + + opaquedir = ovl_create_temp(ofs, workdir, OVL_CATTR(stat.mode)); + err = PTR_ERR(opaquedir); + if (IS_ERR(opaquedir)) + goto out; + + rd.mnt_idmap = ovl_upper_mnt_idmap(ofs); + rd.old_parent = workdir; + rd.new_parent = upperdir; + rd.flags = RENAME_EXCHANGE; + err = start_renaming_two_dentries(&rd, opaquedir, upper); + if (err) + goto out_cleanup_unlocked; + + err = ovl_copy_xattr(dentry->d_sb, &upperpath, opaquedir); + if (err) + goto out_cleanup; + + err = ovl_set_opaque(dentry, opaquedir); + if (err) + goto out_cleanup; + + inode_lock(opaquedir->d_inode); + err = ovl_set_attr(ofs, opaquedir, &stat); + inode_unlock(opaquedir->d_inode); + if (err) + goto out_cleanup; + + err = ovl_do_rename_rd(&rd); + end_renaming(&rd); + if (err) + goto out_cleanup_unlocked; + + ovl_cleanup_whiteouts(ofs, upper, list); + ovl_cleanup(ofs, workdir, upper); + + /* dentry's upper doesn't match now, get rid of it */ + d_drop(dentry); + + return opaquedir; + +out_cleanup: + end_renaming(&rd); +out_cleanup_unlocked: + ovl_cleanup(ofs, workdir, opaquedir); + dput(opaquedir); +out: + return ERR_PTR(err); +} + +static int ovl_set_upper_acl(struct ovl_fs *ofs, struct dentry *upperdentry, + const char *acl_name, struct posix_acl *acl) +{ + if (!IS_ENABLED(CONFIG_FS_POSIX_ACL) || !acl) + return 0; + + return ovl_do_set_acl(ofs, upperdentry, acl_name, acl); +} + +static int ovl_create_over_whiteout(struct dentry *dentry, struct inode *inode, + struct ovl_cattr *cattr) +{ + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + struct dentry *workdir = ovl_workdir(dentry); + struct dentry *upperdir = ovl_dentry_upper(dentry->d_parent); + struct renamedata rd = {}; + struct dentry *upper; + struct dentry *newdentry; + int err; + struct posix_acl *acl, *default_acl; + bool hardlink = !!cattr->hardlink; + + if (WARN_ON(!workdir)) + return -EROFS; + + if (!hardlink) { + err = posix_acl_create(dentry->d_parent->d_inode, + &cattr->mode, &default_acl, &acl); + if (err) + return err; + } + + upper = ovl_lookup_upper_unlocked(ofs, dentry->d_name.name, upperdir, + dentry->d_name.len); + err = PTR_ERR(upper); + if (IS_ERR(upper)) + goto out; + + err = -ESTALE; + if (d_is_negative(upper) || !ovl_upper_is_whiteout(ofs, upper)) + goto out_dput; + + newdentry = ovl_create_temp(ofs, workdir, cattr); + err = PTR_ERR(newdentry); + if (IS_ERR(newdentry)) + goto out_dput; + + rd.mnt_idmap = ovl_upper_mnt_idmap(ofs); + rd.old_parent = workdir; + rd.new_parent = upperdir; + rd.flags = 0; + err = start_renaming_two_dentries(&rd, newdentry, upper); + if (err) + goto out_cleanup_unlocked; + + /* + * mode could have been mutilated due to umask (e.g. sgid directory) + */ + if (!hardlink && + !S_ISLNK(cattr->mode) && + newdentry->d_inode->i_mode != cattr->mode) { + struct iattr attr = { + .ia_valid = ATTR_MODE, + .ia_mode = cattr->mode, + }; + inode_lock(newdentry->d_inode); + err = ovl_do_notify_change(ofs, newdentry, &attr); + inode_unlock(newdentry->d_inode); + if (err) + goto out_cleanup; + } + if (!hardlink) { + err = ovl_set_upper_acl(ofs, newdentry, + XATTR_NAME_POSIX_ACL_ACCESS, acl); + if (err) + goto out_cleanup; + + err = ovl_set_upper_acl(ofs, newdentry, + XATTR_NAME_POSIX_ACL_DEFAULT, default_acl); + if (err) + goto out_cleanup; + } + + if (!hardlink && S_ISDIR(cattr->mode)) { + err = ovl_set_opaque(dentry, newdentry); + if (err) + goto out_cleanup; + + rd.flags = RENAME_EXCHANGE; + err = ovl_do_rename_rd(&rd); + end_renaming(&rd); + if (err) + goto out_cleanup_unlocked; + + ovl_cleanup(ofs, workdir, upper); + } else { + err = ovl_do_rename_rd(&rd); + end_renaming(&rd); + if (err) + goto out_cleanup_unlocked; + } + ovl_dir_modified(dentry->d_parent, false); + err = ovl_instantiate(dentry, inode, newdentry, hardlink, NULL); + if (err) { + ovl_cleanup(ofs, upperdir, newdentry); + dput(newdentry); + } +out_dput: + dput(upper); +out: + if (!hardlink) { + posix_acl_release(acl); + posix_acl_release(default_acl); + } + return err; + +out_cleanup: + end_renaming(&rd); +out_cleanup_unlocked: + ovl_cleanup(ofs, workdir, newdentry); + dput(newdentry); + goto out_dput; +} + +static const struct cred *ovl_override_creator_creds(const struct cred *original_creds, + struct dentry *dentry, struct inode *inode, umode_t mode) +{ + int err; + + if (WARN_ON_ONCE(current->cred != ovl_creds(dentry->d_sb))) + return ERR_PTR(-EINVAL); + + CLASS(prepare_creds, override_cred)(); + if (!override_cred) + return ERR_PTR(-ENOMEM); + + override_cred->fsuid = inode->i_uid; + override_cred->fsgid = inode->i_gid; + + err = security_dentry_create_files_as(dentry, mode, &dentry->d_name, + original_creds, override_cred); + if (err) + return ERR_PTR(err); + + return override_creds(no_free_ptr(override_cred)); +} + +static void ovl_revert_creator_creds(const struct cred *old_cred) +{ + const struct cred *override_cred; + + override_cred = revert_creds(old_cred); + put_cred(override_cred); +} + +DEFINE_CLASS(ovl_override_creator_creds, + const struct cred *, + if (!IS_ERR_OR_NULL(_T)) ovl_revert_creator_creds(_T), + ovl_override_creator_creds(original_creds, dentry, inode, mode), + const struct cred *original_creds, + struct dentry *dentry, + struct inode *inode, + umode_t mode) + +static int ovl_create_handle_whiteouts(struct dentry *dentry, + struct inode *inode, + struct ovl_cattr *attr) +{ + if (!ovl_dentry_is_whiteout(dentry)) + return ovl_create_upper(dentry, inode, attr); + + return ovl_create_over_whiteout(dentry, inode, attr); +} + +static int ovl_create_or_link(struct dentry *dentry, struct inode *inode, + struct ovl_cattr *attr, bool origin) +{ + int err; + struct dentry *parent = dentry->d_parent; + + scoped_class(override_creds_ovl, original_creds, dentry->d_sb) { + /* + * When linking a file with copy up origin into a new parent, mark the + * new parent dir "impure". + */ + if (origin) { + err = ovl_set_impure(parent, ovl_dentry_upper(parent)); + if (err) + return err; + } + + /* + * In the creation cases(create, mkdir, mknod, symlink), + * ovl should transfer current's fs{u,g}id to underlying + * fs. Because underlying fs want to initialize its new + * inode owner using current's fs{u,g}id. And in this + * case, the @inode is a new inode that is initialized + * in inode_init_owner() to current's fs{u,g}id. So use + * the inode's i_{u,g}id to override the cred's fs{u,g}id. + * + * But in the other hardlink case, ovl_link() does not + * create a new inode, so just use the ovl mounter's + * fs{u,g}id. + */ + + if (attr->hardlink) + return ovl_create_handle_whiteouts(dentry, inode, attr); + + scoped_class(ovl_override_creator_creds, cred, original_creds, dentry, inode, attr->mode) { + if (IS_ERR(cred)) + return PTR_ERR(cred); + return ovl_create_handle_whiteouts(dentry, inode, attr); + } + } + return err; +} + +static int ovl_create_object(struct mnt_idmap *idmap, struct dentry *dentry, + int mode, dev_t rdev, const char *link) +{ + int err; + struct inode *inode; + struct ovl_cattr attr = { + .rdev = rdev, + .link = link, + }; + + err = ovl_copy_up(dentry->d_parent); + if (err) + return err; + + err = ovl_want_write(dentry); + if (err) + goto out; + + /* Preallocate inode to be used by ovl_get_inode() */ + err = -ENOMEM; + inode = ovl_new_inode(dentry->d_sb, mode, rdev); + if (!inode) + goto out_drop_write; + + spin_lock(&inode->i_lock); + inode_state_set(inode, I_CREATING); + spin_unlock(&inode->i_lock); + + inode_init_owner(idmap, inode, dentry->d_parent->d_inode, mode); + attr.mode = inode->i_mode; + + err = ovl_create_or_link(dentry, inode, &attr, false); + /* Did we end up using the preallocated inode? */ + if (inode != d_inode(dentry)) + iput(inode); + +out_drop_write: + ovl_drop_write(dentry); +out: + return err; +} + +static int ovl_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + return ovl_create_object(idmap, dentry, (mode & 07777) | S_IFREG, 0, NULL); +} + +static struct dentry *ovl_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + return ERR_PTR(ovl_create_object(idmap, dentry, (mode & 07777) | S_IFDIR, 0, NULL)); +} + +static int ovl_mknod(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) +{ + /* Don't allow creation of "whiteout" on overlay */ + if (S_ISCHR(mode) && rdev == WHITEOUT_DEV) + return -EPERM; + + return ovl_create_object(idmap, dentry, mode, rdev, NULL); +} + +static int ovl_symlink(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const char *link) +{ + return ovl_create_object(idmap, dentry, S_IFLNK, 0, link); +} + +static int ovl_set_link_redirect(struct dentry *dentry) +{ + with_ovl_creds(dentry->d_sb) + return ovl_set_redirect(dentry, false); +} + +static int ovl_link(struct dentry *old, struct inode *newdir, + struct dentry *new) +{ + int err; + struct inode *inode; + + err = ovl_copy_up(old); + if (err) + goto out; + + err = ovl_copy_up(new->d_parent); + if (err) + goto out; + + err = ovl_nlink_start(old); + if (err) + goto out; + + if (ovl_is_metacopy_dentry(old)) { + err = ovl_set_link_redirect(old); + if (err) + goto out_nlink_end; + } + + inode = d_inode(old); + ihold(inode); + + err = ovl_create_or_link(new, inode, + &(struct ovl_cattr) {.hardlink = ovl_dentry_upper(old)}, + ovl_type_origin(old)); + if (err) + iput(inode); + +out_nlink_end: + ovl_nlink_end(old); +out: + return err; +} + +static bool ovl_matches_upper(struct dentry *dentry, struct dentry *upper) +{ + return d_inode(ovl_dentry_upper(dentry)) == d_inode(upper); +} + +static int ovl_remove_and_whiteout(struct dentry *dentry, + struct list_head *list) +{ + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + struct dentry *workdir = ovl_workdir(dentry); + struct dentry *upperdir = ovl_dentry_upper(dentry->d_parent); + struct dentry *upper; + struct dentry *opaquedir = NULL; + int err; + + if (WARN_ON(!workdir)) + return -EROFS; + + if (!list_empty(list)) { + opaquedir = ovl_clear_empty(dentry, list); + err = PTR_ERR(opaquedir); + if (IS_ERR(opaquedir)) + goto out; + } + + upper = ovl_lookup_upper_unlocked(ofs, dentry->d_name.name, upperdir, + dentry->d_name.len); + err = PTR_ERR(upper); + if (IS_ERR(upper)) + goto out_dput; + + err = -ESTALE; + if ((opaquedir && upper != opaquedir) || + (!opaquedir && ovl_dentry_upper(dentry) && + !ovl_matches_upper(dentry, upper))) { + goto out_dput_upper; + } + + err = ovl_cleanup_and_whiteout(ofs, upperdir, upper); + if (!err) + ovl_dir_modified(dentry->d_parent, true); + + d_drop(dentry); +out_dput_upper: + dput(upper); +out_dput: + dput(opaquedir); +out: + return err; +} + +static int ovl_remove_upper(struct dentry *dentry, bool is_dir, + struct list_head *list) +{ + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + struct dentry *upperdir = ovl_dentry_upper(dentry->d_parent); + struct inode *dir = upperdir->d_inode; + struct dentry *upper; + struct dentry *opaquedir = NULL; + int err; + + if (!list_empty(list)) { + opaquedir = ovl_clear_empty(dentry, list); + err = PTR_ERR(opaquedir); + if (IS_ERR(opaquedir)) + goto out; + } + + upper = ovl_start_removing_upper(ofs, upperdir, + &QSTR_LEN(dentry->d_name.name, + dentry->d_name.len)); + err = PTR_ERR(upper); + if (IS_ERR(upper)) + goto out_dput; + + err = -ESTALE; + if ((opaquedir && upper != opaquedir) || + (!opaquedir && !ovl_matches_upper(dentry, upper))) + goto out_unlock; + + if (is_dir) + err = ovl_do_rmdir(ofs, dir, upper); + else + err = ovl_do_unlink(ofs, dir, upper); + ovl_dir_modified(dentry->d_parent, ovl_type_origin(dentry)); + + /* + * Keeping this dentry hashed would mean having to release + * upperpath/lowerpath, which could only be done if we are the + * sole user of this dentry. Too tricky... Just unhash for + * now. + */ + if (!err) + d_drop(dentry); +out_unlock: + end_removing(upper); +out_dput: + dput(opaquedir); +out: + return err; +} + +static bool ovl_pure_upper(struct dentry *dentry) +{ + return !ovl_dentry_lower(dentry) && + !ovl_test_flag(OVL_WHITEOUTS, d_inode(dentry)); +} + +static void ovl_drop_nlink(struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + struct dentry *alias; + + /* Try to find another, hashed alias */ + spin_lock(&inode->i_lock); + for_each_alias(alias, inode) { + if (alias != dentry && !d_unhashed(alias)) + break; + } + spin_unlock(&inode->i_lock); + + /* + * Changes to underlying layers may cause i_nlink to lose sync with + * reality. In this case prevent the link count from going to zero + * prematurely. + */ + if (inode->i_nlink > !!alias) + drop_nlink(inode); +} + +static int ovl_do_remove(struct dentry *dentry, bool is_dir) +{ + int err; + bool lower_positive = ovl_lower_positive(dentry); + LIST_HEAD(list); + + /* No need to clean pure upper removed by vfs_rmdir() */ + if (is_dir && (lower_positive || !ovl_pure_upper(dentry))) { + err = ovl_check_empty_dir(dentry, &list); + if (err) + goto out; + } + + err = ovl_copy_up(dentry->d_parent); + if (err) + goto out; + + err = ovl_nlink_start(dentry); + if (err) + goto out; + + with_ovl_creds(dentry->d_sb) { + if (!lower_positive) + err = ovl_remove_upper(dentry, is_dir, &list); + else + err = ovl_remove_and_whiteout(dentry, &list); + } + if (!err) { + if (is_dir) + clear_nlink(dentry->d_inode); + else + ovl_drop_nlink(dentry); + } + ovl_nlink_end(dentry); + + /* + * Copy ctime + * + * Note: we fail to update ctime if there was no copy-up, only a + * whiteout + */ + if (ovl_dentry_upper(dentry)) + ovl_copyattr(d_inode(dentry)); + +out: + ovl_cache_free(&list); + return err; +} + +static int ovl_unlink(struct inode *dir, struct dentry *dentry) +{ + return ovl_do_remove(dentry, false); +} + +static int ovl_rmdir(struct inode *dir, struct dentry *dentry) +{ + return ovl_do_remove(dentry, true); +} + +static bool ovl_type_merge_or_lower(struct dentry *dentry) +{ + enum ovl_path_type type = ovl_path_type(dentry); + + return OVL_TYPE_MERGE(type) || !OVL_TYPE_UPPER(type); +} + +static bool ovl_can_move(struct dentry *dentry) +{ + return ovl_redirect_dir(OVL_FS(dentry->d_sb)) || + !d_is_dir(dentry) || !ovl_type_merge_or_lower(dentry); +} + +static char *ovl_get_redirect(struct dentry *dentry, bool abs_redirect) +{ + char *buf, *ret; + struct dentry *d, *tmp; + int buflen = ovl_redirect_max + 1; + + if (!abs_redirect) { + ret = kstrndup(dentry->d_name.name, dentry->d_name.len, + GFP_KERNEL); + goto out; + } + + buf = ret = kmalloc(buflen, GFP_KERNEL); + if (!buf) + goto out; + + buflen--; + buf[buflen] = '\0'; + for (d = dget(dentry); !IS_ROOT(d);) { + const char *name; + int thislen; + + spin_lock(&d->d_lock); + name = ovl_dentry_get_redirect(d); + if (name) { + thislen = strlen(name); + } else { + name = d->d_name.name; + thislen = d->d_name.len; + } + + /* If path is too long, fall back to userspace move */ + if (thislen + (name[0] != '/') > buflen) { + ret = ERR_PTR(-EXDEV); + spin_unlock(&d->d_lock); + goto out_put; + } + + buflen -= thislen; + memcpy(&buf[buflen], name, thislen); + spin_unlock(&d->d_lock); + tmp = dget_parent(d); + + dput(d); + d = tmp; + + /* Absolute redirect: finished */ + if (buf[buflen] == '/') + break; + buflen--; + buf[buflen] = '/'; + } + ret = kstrdup(&buf[buflen], GFP_KERNEL); +out_put: + dput(d); + kfree(buf); +out: + return ret ? ret : ERR_PTR(-ENOMEM); +} + +static bool ovl_need_absolute_redirect(struct dentry *dentry, bool samedir) +{ + struct dentry *lowerdentry; + + if (!samedir) + return true; + + if (d_is_dir(dentry)) + return false; + + /* + * For non-dir hardlinked files, we need absolute redirects + * in general as two upper hardlinks could be in different + * dirs. We could put a relative redirect now and convert + * it to absolute redirect later. But when nlink > 1 and + * indexing is on, that means relative redirect needs to be + * converted to absolute during copy up of another lower + * hardllink as well. + * + * So without optimizing too much, just check if lower is + * a hard link or not. If lower is hard link, put absolute + * redirect. + */ + lowerdentry = ovl_dentry_lower(dentry); + return (d_inode(lowerdentry)->i_nlink > 1); +} + +static int ovl_set_redirect(struct dentry *dentry, bool samedir) +{ + int err; + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + const char *redirect = ovl_dentry_get_redirect(dentry); + bool absolute_redirect = ovl_need_absolute_redirect(dentry, samedir); + + if (redirect && (!absolute_redirect || redirect[0] == '/')) + return 0; + + redirect = ovl_get_redirect(dentry, absolute_redirect); + if (IS_ERR(redirect)) + return PTR_ERR(redirect); + + err = ovl_check_setxattr(ofs, ovl_dentry_upper(dentry), + OVL_XATTR_REDIRECT, + redirect, strlen(redirect), -EXDEV); + if (!err) { + spin_lock(&dentry->d_lock); + ovl_dentry_set_redirect(dentry, redirect); + spin_unlock(&dentry->d_lock); + } else { + kfree(redirect); + pr_warn_ratelimited("failed to set redirect (%i)\n", + err); + /* Fall back to userspace copy-up */ + err = -EXDEV; + } + return err; +} + +struct ovl_renamedata { + struct renamedata; + struct dentry *opaquedir; + bool cleanup_whiteout; + bool update_nlink; + bool overwrite; +}; + +static int ovl_rename_start(struct ovl_renamedata *ovlrd, struct list_head *list) +{ + struct dentry *old = ovlrd->old_dentry; + struct dentry *new = ovlrd->new_dentry; + bool is_dir = d_is_dir(old); + bool new_is_dir = d_is_dir(new); + int err; + + if (ovlrd->flags & ~(RENAME_EXCHANGE | RENAME_NOREPLACE)) + return -EINVAL; + + ovlrd->flags &= ~RENAME_NOREPLACE; + + /* Don't copy up directory trees */ + err = -EXDEV; + if (!ovl_can_move(old)) + return err; + if (!ovlrd->overwrite && !ovl_can_move(new)) + return err; + + if (ovlrd->overwrite && new_is_dir && !ovl_pure_upper(new)) { + err = ovl_check_empty_dir(new, list); + if (err) + return err; + } + + if (ovlrd->overwrite) { + if (ovl_lower_positive(old)) { + if (!ovl_dentry_is_whiteout(new)) { + /* Whiteout source */ + ovlrd->flags |= RENAME_WHITEOUT; + } else { + /* Switch whiteouts */ + ovlrd->flags |= RENAME_EXCHANGE; + } + } else if (is_dir && ovl_dentry_is_whiteout(new)) { + ovlrd->flags |= RENAME_EXCHANGE; + ovlrd->cleanup_whiteout = true; + } + } + + err = ovl_copy_up(old); + if (err) + return err; + + err = ovl_copy_up(new->d_parent); + if (err) + return err; + + if (!ovlrd->overwrite) { + err = ovl_copy_up(new); + if (err) + return err; + } else if (d_inode(new)) { + err = ovl_nlink_start(new); + if (err) + return err; + + ovlrd->update_nlink = true; + } + + if (!ovlrd->update_nlink) { + /* ovl_nlink_start() took ovl_want_write() */ + err = ovl_want_write(old); + if (err) + return err; + } + + return 0; +} + +static int ovl_rename_upper(struct ovl_renamedata *ovlrd, struct list_head *list) +{ + struct dentry *old = ovlrd->old_dentry; + struct dentry *new = ovlrd->new_dentry; + struct ovl_fs *ofs = OVL_FS(old->d_sb); + struct dentry *old_upperdir = ovl_dentry_upper(old->d_parent); + struct dentry *new_upperdir = ovl_dentry_upper(new->d_parent); + bool is_dir = d_is_dir(old); + bool new_is_dir = d_is_dir(new); + bool samedir = old->d_parent == new->d_parent; + struct renamedata rd = {}; + struct dentry *de; + struct dentry *whiteout = NULL; + bool old_opaque, new_opaque; + int err; + + if (!list_empty(list)) { + de = ovl_clear_empty(new, list); + if (IS_ERR(de)) + return PTR_ERR(de); + ovlrd->opaquedir = de; + } + + if (!samedir) { + /* + * When moving a merge dir or non-dir with copy up origin into + * a new parent, we are marking the new parent dir "impure". + * When ovl_iterate() iterates an "impure" upper dir, it will + * lookup the origin inodes of the entries to fill d_ino. + */ + if (ovl_type_origin(old)) { + err = ovl_set_impure(new->d_parent, new_upperdir); + if (err) + return err; + } + if (!ovlrd->overwrite && ovl_type_origin(new)) { + err = ovl_set_impure(old->d_parent, old_upperdir); + if (err) + return err; + } + } + + rd.mnt_idmap = ovl_upper_mnt_idmap(ofs); + rd.old_parent = old_upperdir; + rd.new_parent = new_upperdir; + rd.flags = ovlrd->flags; + + err = start_renaming(&rd, 0, + &QSTR_LEN(old->d_name.name, old->d_name.len), + &QSTR_LEN(new->d_name.name, new->d_name.len)); + if (err) + return err; + + err = -ESTALE; + if (!ovl_matches_upper(old, rd.old_dentry)) + goto out_unlock; + + old_opaque = ovl_dentry_is_opaque(old); + new_opaque = ovl_dentry_is_opaque(new); + + err = -ESTALE; + if (d_inode(new) && ovl_dentry_upper(new)) { + if (ovlrd->opaquedir) { + if (rd.new_dentry != ovlrd->opaquedir) + goto out_unlock; + } else { + if (!ovl_matches_upper(new, rd.new_dentry)) + goto out_unlock; + } + } else { + if (!d_is_negative(rd.new_dentry)) { + if (!new_opaque || !ovl_upper_is_whiteout(ofs, rd.new_dentry)) + goto out_unlock; + } else { + if (ovlrd->flags & RENAME_EXCHANGE) + goto out_unlock; + } + } + + if (rd.old_dentry->d_inode == rd.new_dentry->d_inode) + goto out_unlock; + + err = 0; + if (ovl_type_merge_or_lower(old)) + err = ovl_set_redirect(old, samedir); + else if (is_dir && !old_opaque && ovl_type_merge(new->d_parent)) + err = ovl_set_opaque_xerr(old, rd.old_dentry, -EXDEV); + if (err) + goto out_unlock; + + if (!ovlrd->overwrite && ovl_type_merge_or_lower(new)) + err = ovl_set_redirect(new, samedir); + else if (!ovlrd->overwrite && new_is_dir && !new_opaque && + ovl_type_merge(old->d_parent)) + err = ovl_set_opaque_xerr(new, rd.new_dentry, -EXDEV); + if (err) + goto out_unlock; + + err = ovl_do_rename_rd(&rd); + + if (!err && ovlrd->cleanup_whiteout) + whiteout = dget(rd.new_dentry); + +out_unlock: + end_renaming(&rd); + + if (err) + return err; + + if (whiteout) { + ovl_cleanup(ofs, old_upperdir, whiteout); + dput(whiteout); + } + + if (ovlrd->overwrite && d_inode(new)) { + if (new_is_dir) + clear_nlink(d_inode(new)); + else + ovl_drop_nlink(new); + } + + ovl_dir_modified(old->d_parent, ovl_type_origin(old) || + (!ovlrd->overwrite && ovl_type_origin(new))); + ovl_dir_modified(new->d_parent, ovl_type_origin(old) || + (d_inode(new) && ovl_type_origin(new))); + + /* copy ctime: */ + ovl_copyattr(d_inode(old)); + if (d_inode(new) && ovl_dentry_upper(new)) + ovl_copyattr(d_inode(new)); + + return err; +} + +static void ovl_rename_end(struct ovl_renamedata *ovlrd) +{ + if (ovlrd->update_nlink) + ovl_nlink_end(ovlrd->new_dentry); + else + ovl_drop_write(ovlrd->old_dentry); +} + +static int ovl_rename(struct mnt_idmap *idmap, struct inode *olddir, + struct dentry *old, struct inode *newdir, + struct dentry *new, unsigned int flags) +{ + struct ovl_renamedata ovlrd = { + .old_parent = old->d_parent, + .old_dentry = old, + .new_parent = new->d_parent, + .new_dentry = new, + .flags = flags, + .overwrite = !(flags & RENAME_EXCHANGE), + }; + LIST_HEAD(list); + int err; + + err = ovl_rename_start(&ovlrd, &list); + if (!err) { + with_ovl_creds(old->d_sb) + err = ovl_rename_upper(&ovlrd, &list); + ovl_rename_end(&ovlrd); + } + + dput(ovlrd.opaquedir); + ovl_cache_free(&list); + return err; +} + +static int ovl_create_tmpfile(struct file *file, struct dentry *dentry, + struct inode *inode, umode_t mode) +{ + struct path realparentpath; + struct file *realfile; + struct ovl_file *of; + struct dentry *newdentry; + /* It's okay to set O_NOATIME, since the owner will be current fsuid */ + int flags = file->f_flags | OVL_OPEN_FLAGS; + int err; + + scoped_class(override_creds_ovl, original_creds, dentry->d_sb) { + scoped_class(ovl_override_creator_creds, cred, original_creds, dentry, inode, mode) { + if (IS_ERR(cred)) + return PTR_ERR(cred); + + ovl_path_upper(dentry->d_parent, &realparentpath); + realfile = backing_tmpfile_open(file, flags, &realparentpath, + mode, current_cred()); + err = PTR_ERR_OR_ZERO(realfile); + pr_debug("tmpfile/open(%pd2, 0%o) = %i\n", realparentpath.dentry, mode, err); + if (err) + return err; + + of = ovl_file_alloc(realfile); + if (!of) { + fput(realfile); + return -ENOMEM; + } + + /* ovl_instantiate() consumes the newdentry reference on success */ + newdentry = dget(realfile->f_path.dentry); + err = ovl_instantiate(dentry, inode, newdentry, false, file); + if (!err) { + file->private_data = of; + } else { + dput(newdentry); + ovl_file_free(of); + } + } + } + return err; +} + +static int ovl_dummy_open(struct inode *inode, struct file *file) +{ + return 0; +} + +static int ovl_tmpfile(struct mnt_idmap *idmap, struct inode *dir, + struct file *file, umode_t mode) +{ + int err; + struct dentry *dentry = file->f_path.dentry; + struct inode *inode; + + if (!OVL_FS(dentry->d_sb)->tmpfile) + return -EOPNOTSUPP; + + err = ovl_copy_up(dentry->d_parent); + if (err) + return err; + + err = ovl_want_write(dentry); + if (err) + return err; + + err = -ENOMEM; + inode = ovl_new_inode(dentry->d_sb, mode, 0); + if (!inode) + goto drop_write; + + inode_init_owner(idmap, inode, dir, mode); + err = ovl_create_tmpfile(file, dentry, inode, inode->i_mode); + if (err) + goto put_inode; + + /* + * Check if the preallocated inode was actually used. Having something + * else assigned to the dentry shouldn't happen as that would indicate + * that the backing tmpfile "leaked" out of overlayfs. + */ + err = -EIO; + if (WARN_ON(inode != d_inode(dentry))) + goto put_realfile; + + /* inode reference was transferred to dentry */ + inode = NULL; + err = finish_open(file, dentry, ovl_dummy_open); +put_realfile: + /* Without FMODE_OPENED ->release() won't be called on @file */ + if (!(file->f_mode & FMODE_OPENED)) + ovl_file_free(file->private_data); +put_inode: + iput(inode); +drop_write: + ovl_drop_write(dentry); + return err; +} + +const struct inode_operations ovl_dir_inode_operations = { + .lookup = ovl_lookup, + .mkdir = ovl_mkdir, + .symlink = ovl_symlink, + .unlink = ovl_unlink, + .rmdir = ovl_rmdir, + .rename = ovl_rename, + .link = ovl_link, + .setattr = ovl_setattr, + .create = ovl_create, + .mknod = ovl_mknod, + .permission = ovl_permission, + .getattr = ovl_getattr, + .listxattr = ovl_listxattr, + .get_inode_acl = ovl_get_inode_acl, + .get_acl = ovl_get_acl, + .set_acl = ovl_set_acl, + .update_time = ovl_update_time, + .fileattr_get = ovl_fileattr_get, + .fileattr_set = ovl_fileattr_set, + .tmpfile = ovl_tmpfile, +}; diff --git a/fs/overlayfs/export.c b/fs/overlayfs/export.c new file mode 100644 index 000000000..9aae20c59 --- /dev/null +++ b/fs/overlayfs/export.c @@ -0,0 +1,868 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Overlayfs NFS export support. + * + * Amir Goldstein + * + * Copyright (C) 2017-2018 CTERA Networks. All Rights Reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "overlayfs.h" + +static int ovl_encode_maybe_copy_up(struct dentry *dentry) +{ + int err; + + if (ovl_dentry_upper(dentry)) + return 0; + + err = ovl_copy_up(dentry); + if (err) { + pr_warn_ratelimited("failed to copy up on encode (%pd2, err=%i)\n", + dentry, err); + } + + return err; +} + +/* + * Before encoding a non-upper directory file handle from real layer N, we need + * to check if it will be possible to reconnect an overlay dentry from the real + * lower decoded dentry. This is done by following the overlay ancestry up to a + * "layer N connected" ancestor and verifying that all parents along the way are + * "layer N connectable". If an ancestor that is NOT "layer N connectable" is + * found, we need to copy up an ancestor, which is "layer N connectable", thus + * making that ancestor "layer N connected". For example: + * + * layer 1: /a + * layer 2: /a/b/c + * + * The overlay dentry /a is NOT "layer 2 connectable", because if dir /a is + * copied up and renamed, upper dir /a will be indexed by lower dir /a from + * layer 1. The dir /a from layer 2 will never be indexed, so the algorithm (*) + * in ovl_lookup_real_ancestor() will not be able to lookup a connected overlay + * dentry from the connected lower dentry /a/b/c. + * + * To avoid this problem on decode time, we need to copy up an ancestor of + * /a/b/c, which is "layer 2 connectable", on encode time. That ancestor is + * /a/b. After copy up (and index) of /a/b, it will become "layer 2 connected" + * and when the time comes to decode the file handle from lower dentry /a/b/c, + * ovl_lookup_real_ancestor() will find the indexed ancestor /a/b and decoding + * a connected overlay dentry will be accomplished. + * + * (*) the algorithm in ovl_lookup_real_ancestor() can be improved to lookup an + * entry /a in the lower layers above layer N and find the indexed dir /a from + * layer 1. If that improvement is made, then the check for "layer N connected" + * will need to verify there are no redirects in lower layers above N. In the + * example above, /a will be "layer 2 connectable". However, if layer 2 dir /a + * is a target of a layer 1 redirect, then /a will NOT be "layer 2 connectable": + * + * layer 1: /A (redirect = /a) + * layer 2: /a/b/c + */ + +/* Return the lowest layer for encoding a connectable file handle */ +static int ovl_connectable_layer(struct dentry *dentry) +{ + struct ovl_entry *oe = OVL_E(dentry); + + /* We can get overlay root from root of any layer */ + if (dentry == dentry->d_sb->s_root) + return ovl_numlower(oe); + + /* + * If it's an unindexed merge dir, then it's not connectable with any + * lower layer + */ + if (ovl_dentry_upper(dentry) && + !ovl_test_flag(OVL_INDEX, d_inode(dentry))) + return 0; + + /* We can get upper/overlay path from indexed/lower dentry */ + return ovl_lowerstack(oe)->layer->idx; +} + +/* + * @dentry is "connected" if all ancestors up to root or a "connected" ancestor + * have the same uppermost lower layer as the origin's layer. We may need to + * copy up a "connectable" ancestor to make it "connected". A "connected" dentry + * cannot become non "connected", so cache positive result in dentry flags. + * + * Return the connected origin layer or < 0 on error. + */ +static int ovl_connect_layer(struct dentry *dentry) +{ + struct dentry *next, *parent = NULL; + struct ovl_entry *oe = OVL_E(dentry); + int origin_layer; + int err = 0; + + if (WARN_ON(dentry == dentry->d_sb->s_root) || + WARN_ON(!ovl_dentry_lower(dentry))) + return -EIO; + + origin_layer = ovl_lowerstack(oe)->layer->idx; + if (ovl_dentry_test_flag(OVL_E_CONNECTED, dentry)) + return origin_layer; + + /* Find the topmost origin layer connectable ancestor of @dentry */ + next = dget(dentry); + for (;;) { + parent = dget_parent(next); + if (WARN_ON(parent == next)) { + err = -EIO; + break; + } + + /* + * If @parent is not origin layer connectable, then copy up + * @next which is origin layer connectable and we are done. + */ + if (ovl_connectable_layer(parent) < origin_layer) { + err = ovl_encode_maybe_copy_up(next); + break; + } + + /* If @parent is connected or indexed we are done */ + if (ovl_dentry_test_flag(OVL_E_CONNECTED, parent) || + ovl_test_flag(OVL_INDEX, d_inode(parent))) + break; + + dput(next); + next = parent; + } + + dput(parent); + dput(next); + + if (!err) + ovl_dentry_set_flag(OVL_E_CONNECTED, dentry); + + return err ?: origin_layer; +} + +/* + * We only need to encode origin if there is a chance that the same object was + * encoded pre copy up and then we need to stay consistent with the same + * encoding also after copy up. If non-pure upper is not indexed, then it was + * copied up before NFS export was enabled. In that case we don't need to worry + * about staying consistent with pre copy up encoding and we encode an upper + * file handle. Overlay root dentry is a private case of non-indexed upper. + * + * The following table summarizes the different file handle encodings used for + * different overlay object types: + * + * Object type | Encoding + * -------------------------------- + * Pure upper | U + * Non-indexed upper | U + * Indexed upper | L (*) + * Non-upper | L (*) + * + * U = upper file handle + * L = lower file handle + * + * (*) Decoding a connected overlay dir from real lower dentry is not always + * possible when there are redirects in lower layers and non-indexed merge dirs. + * To mitigate those case, we may copy up the lower dir ancestor before encode + * of a decodable file handle for non-upper dir. + * + * Return 0 for upper file handle, > 0 for lower file handle or < 0 on error. + */ +static int ovl_check_encode_origin(struct inode *inode) +{ + struct ovl_fs *ofs = OVL_FS(inode->i_sb); + bool decodable = ofs->config.nfs_export; + struct dentry *dentry; + int err; + + /* No upper layer? */ + if (!ovl_upper_mnt(ofs)) + return 1; + + /* Lower file handle for non-upper non-decodable */ + if (!ovl_inode_upper(inode) && !decodable) + return 1; + + /* Upper file handle for pure upper */ + if (!ovl_inode_lower(inode)) + return 0; + + /* + * Root is never indexed, so if there's an upper layer, encode upper for + * root. + */ + if (inode == d_inode(inode->i_sb->s_root)) + return 0; + + /* + * Upper decodable file handle for non-indexed upper. + */ + if (ovl_inode_upper(inode) && decodable && + !ovl_test_flag(OVL_INDEX, inode)) + return 0; + + /* + * Decoding a merge dir, whose origin's ancestor is under a redirected + * lower dir or under a non-indexed upper is not always possible. + * ovl_connect_layer() will try to make origin's layer "connected" by + * copying up a "connectable" ancestor. + */ + if (!decodable || !S_ISDIR(inode->i_mode)) + return 1; + + dentry = d_find_any_alias(inode); + if (!dentry) + return -ENOENT; + + err = ovl_connect_layer(dentry); + dput(dentry); + if (err < 0) + return err; + + /* Lower file handle for indexed and non-upper dir/non-dir */ + return 1; +} + +static int ovl_dentry_to_fid(struct ovl_fs *ofs, struct inode *inode, + u32 *fid, int buflen) +{ + struct ovl_fh *fh = NULL; + int err, enc_lower; + int len; + + /* + * Check if we should encode a lower or upper file handle and maybe + * copy up an ancestor to make lower file handle connectable. + */ + err = enc_lower = ovl_check_encode_origin(inode); + if (enc_lower < 0) + goto fail; + + /* Encode an upper or lower file handle */ + fh = ovl_encode_real_fh(ofs, enc_lower ? ovl_inode_lower(inode) : + ovl_inode_upper(inode), !enc_lower); + if (IS_ERR(fh)) + return PTR_ERR(fh); + + len = OVL_FH_LEN(fh); + if (len <= buflen) + memcpy(fid, fh, len); + err = len; + +out: + kfree(fh); + return err; + +fail: + pr_warn_ratelimited("failed to encode file handle (ino=%llu, err=%i)\n", + inode->i_ino, err); + goto out; +} + +static int ovl_encode_fh(struct inode *inode, u32 *fid, int *max_len, + struct inode *parent) +{ + struct ovl_fs *ofs = OVL_FS(inode->i_sb); + int bytes, buflen = *max_len << 2; + + /* TODO: encode connectable file handles */ + if (parent) + return FILEID_INVALID; + + bytes = ovl_dentry_to_fid(ofs, inode, fid, buflen); + if (bytes <= 0) + return FILEID_INVALID; + + *max_len = bytes >> 2; + if (bytes > buflen) + return FILEID_INVALID; + + return OVL_FILEID_V1; +} + +/* + * Find or instantiate an overlay dentry from real dentries and index. + */ +static struct dentry *ovl_obtain_alias(struct super_block *sb, + struct dentry *upper_alias, + struct ovl_path *lowerpath, + struct dentry *index) +{ + struct dentry *lower = lowerpath ? lowerpath->dentry : NULL; + struct dentry *upper = upper_alias ?: index; + struct inode *inode = NULL; + struct ovl_entry *oe; + struct ovl_inode_params oip = { + .index = index, + }; + + /* We get overlay directory dentries with ovl_lookup_real() */ + if (d_is_dir(upper ?: lower)) + return ERR_PTR(-EIO); + + oe = ovl_alloc_entry(!!lower); + if (!oe) + return ERR_PTR(-ENOMEM); + + oip.upperdentry = dget(upper); + if (lower) { + ovl_lowerstack(oe)->dentry = dget(lower); + ovl_lowerstack(oe)->layer = lowerpath->layer; + } + oip.oe = oe; + inode = ovl_get_inode(sb, &oip); + if (IS_ERR(inode)) { + ovl_free_entry(oe); + dput(upper); + return ERR_CAST(inode); + } + + if (upper) + ovl_set_flag(OVL_UPPERDATA, inode); + + return d_obtain_alias(inode); +} + +/* Get the upper or lower dentry in stack whose on layer @idx */ +static struct dentry *ovl_dentry_real_at(struct dentry *dentry, int idx) +{ + struct ovl_entry *oe = OVL_E(dentry); + struct ovl_path *lowerstack = ovl_lowerstack(oe); + int i; + + if (!idx) + return ovl_dentry_upper(dentry); + + for (i = 0; i < ovl_numlower(oe); i++) { + if (lowerstack[i].layer->idx == idx) + return lowerstack[i].dentry; + } + + return NULL; +} + +/** + * ovl_lookup_real_one - Lookup a child overlay dentry to get an overlay dentry whose real dentry is given + * @connected: parent overlay dentry + * @real: given child real dentry + * @layer: layer in which @real exists + * + * + * Lookup a child overlay dentry in @connected with the same name as the @real + * dentry. Then check that the parent of the result is the real dentry for + * @connected, and @real is the real dentry for the result. + * + * Returns: + * %-ECHILD if the parent of @real is no longer the real dentry for @connected. + * %-ESTALE if @real is not the real dentry of the found dentry. + * Otherwise the found dentry is returned. + */ +static struct dentry *ovl_lookup_real_one(struct dentry *connected, + struct dentry *real, + const struct ovl_layer *layer) +{ + struct dentry *this; + struct name_snapshot name; + int err; + + /* + * We need to take a snapshot of real dentry name to protect us + * from racing with underlying layer rename. In this case, we don't + * care about returning ESTALE, only from dereferencing a free name + * pointer because we hold no lock on the real dentry. + */ + take_dentry_name_snapshot(&name, real); + this = lookup_noperm_unlocked(&name.name, connected); + release_dentry_name_snapshot(&name); + + err = -ECHILD; + if (ovl_dentry_real_at(connected, layer->idx) != real->d_parent) + goto fail; + + err = PTR_ERR(this); + if (IS_ERR(this)) + goto fail; + + err = -ENOENT; + if (!this || !this->d_inode) + goto fail; + + err = -ESTALE; + if (ovl_dentry_real_at(this, layer->idx) != real) + goto fail; + + return this; + +fail: + pr_warn_ratelimited("failed to lookup one by real (%pd2, layer=%d, connected=%pd2, err=%i)\n", + real, layer->idx, connected, err); + if (!IS_ERR(this)) + dput(this); + return ERR_PTR(err); +} + +static struct dentry *ovl_lookup_real(struct super_block *sb, + struct dentry *real, + const struct ovl_layer *layer); + +/* + * Lookup an indexed or hashed overlay dentry by real inode. + */ +static struct dentry *ovl_lookup_real_inode(struct super_block *sb, + struct dentry *real, + const struct ovl_layer *layer) +{ + struct ovl_fs *ofs = OVL_FS(sb); + struct dentry *index = NULL; + struct dentry *this = NULL; + struct inode *inode; + + /* + * Decoding upper dir from index is expensive, so first try to lookup + * overlay dentry in inode/dcache. + */ + inode = ovl_lookup_inode(sb, real, !layer->idx); + if (IS_ERR(inode)) + return ERR_CAST(inode); + if (inode) { + this = d_find_any_alias(inode); + iput(inode); + } + + /* + * For decoded lower dir file handle, lookup index by origin to check + * if lower dir was copied up and and/or removed. + */ + if (!this && layer->idx && ovl_indexdir(sb) && !WARN_ON(!d_is_dir(real))) { + index = ovl_lookup_index(ofs, NULL, real, false); + if (IS_ERR(index)) + return index; + } + + /* Get connected upper overlay dir from index */ + if (index) { + struct dentry *upper = ovl_index_upper(ofs, index, true); + + dput(index); + if (IS_ERR_OR_NULL(upper)) + return upper; + + /* + * ovl_lookup_real() in lower layer may call recursively once to + * ovl_lookup_real() in upper layer. The first level call walks + * back lower parents to the topmost indexed parent. The second + * recursive call walks back from indexed upper to the topmost + * connected/hashed upper parent (or up to root). + */ + this = ovl_lookup_real(sb, upper, &ofs->layers[0]); + dput(upper); + } + + if (IS_ERR_OR_NULL(this)) + return this; + + if (ovl_dentry_real_at(this, layer->idx) != real) { + dput(this); + this = ERR_PTR(-EIO); + } + + return this; +} + +/* + * Lookup an indexed or hashed overlay dentry, whose real dentry is an + * ancestor of @real. + */ +static struct dentry *ovl_lookup_real_ancestor(struct super_block *sb, + struct dentry *real, + const struct ovl_layer *layer) +{ + struct dentry *next, *parent = NULL; + struct dentry *ancestor = ERR_PTR(-EIO); + + if (real == layer->mnt->mnt_root) + return dget(sb->s_root); + + /* Find the topmost indexed or hashed ancestor */ + next = dget(real); + for (;;) { + parent = dget_parent(next); + + /* + * Lookup a matching overlay dentry in inode/dentry + * cache or in index by real inode. + */ + ancestor = ovl_lookup_real_inode(sb, next, layer); + if (ancestor) + break; + + if (parent == layer->mnt->mnt_root) { + ancestor = dget(sb->s_root); + break; + } + + /* + * If @real has been moved out of the layer root directory, + * we will eventully hit the real fs root. This cannot happen + * by legit overlay rename, so we return error in that case. + */ + if (parent == next) { + ancestor = ERR_PTR(-EXDEV); + break; + } + + dput(next); + next = parent; + } + + dput(parent); + dput(next); + + return ancestor; +} + +/* + * Lookup a connected overlay dentry whose real dentry is @real. + * If @real is on upper layer, we lookup a child overlay dentry with the same + * path the real dentry. Otherwise, we need to consult index for lookup. + */ +static struct dentry *ovl_lookup_real(struct super_block *sb, + struct dentry *real, + const struct ovl_layer *layer) +{ + struct dentry *connected; + int err = 0; + + connected = ovl_lookup_real_ancestor(sb, real, layer); + if (IS_ERR(connected)) + return connected; + + while (!err) { + struct dentry *next, *this; + struct dentry *parent = NULL; + struct dentry *real_connected = ovl_dentry_real_at(connected, + layer->idx); + + if (real_connected == real) + break; + + /* Find the topmost dentry not yet connected */ + next = dget(real); + for (;;) { + parent = dget_parent(next); + + if (parent == real_connected) + break; + + /* + * If real has been moved out of 'real_connected', + * we will not find 'real_connected' and hit the layer + * root. In that case, we need to restart connecting. + * This game can go on forever in the worst case. We + * may want to consider taking s_vfs_rename_mutex if + * this happens more than once. + */ + if (parent == layer->mnt->mnt_root) { + dput(connected); + connected = dget(sb->s_root); + break; + } + + /* + * If real file has been moved out of the layer root + * directory, we will eventully hit the real fs root. + * This cannot happen by legit overlay rename, so we + * return error in that case. + */ + if (parent == next) { + err = -EXDEV; + break; + } + + dput(next); + next = parent; + } + + if (!err) { + this = ovl_lookup_real_one(connected, next, layer); + if (IS_ERR(this)) + err = PTR_ERR(this); + + /* + * Lookup of child in overlay can fail when racing with + * overlay rename of child away from 'connected' parent. + * In this case, we need to restart the lookup from the + * top, because we cannot trust that 'real_connected' is + * still an ancestor of 'real'. There is a good chance + * that the renamed overlay ancestor is now in cache, so + * ovl_lookup_real_ancestor() will find it and we can + * continue to connect exactly from where lookup failed. + */ + if (err == -ECHILD) { + this = ovl_lookup_real_ancestor(sb, real, + layer); + err = PTR_ERR_OR_ZERO(this); + } + if (!err) { + dput(connected); + connected = this; + } + } + + dput(parent); + dput(next); + } + + if (err) + goto fail; + + return connected; + +fail: + pr_warn_ratelimited("failed to lookup by real (%pd2, layer=%d, connected=%pd2, err=%i)\n", + real, layer->idx, connected, err); + dput(connected); + return ERR_PTR(err); +} + +/* + * Get an overlay dentry from upper/lower real dentries and index. + */ +static struct dentry *ovl_get_dentry(struct super_block *sb, + struct dentry *upper, + struct ovl_path *lowerpath, + struct dentry *index) +{ + struct ovl_fs *ofs = OVL_FS(sb); + const struct ovl_layer *layer = upper ? &ofs->layers[0] : lowerpath->layer; + struct dentry *real = upper ?: (index ?: lowerpath->dentry); + + /* + * Obtain a disconnected overlay dentry from a non-dir real dentry + * and index. + */ + if (!d_is_dir(real)) + return ovl_obtain_alias(sb, upper, lowerpath, index); + + /* Removed empty directory? */ + if ((real->d_flags & DCACHE_DISCONNECTED) || d_unhashed(real)) + return ERR_PTR(-ENOENT); + + /* + * If real dentry is connected and hashed, get a connected overlay + * dentry whose real dentry is @real. + */ + return ovl_lookup_real(sb, real, layer); +} + +static struct dentry *ovl_upper_fh_to_d(struct super_block *sb, + struct ovl_fh *fh) +{ + struct ovl_fs *ofs = OVL_FS(sb); + struct dentry *dentry; + struct dentry *upper; + + if (!ovl_upper_mnt(ofs)) + return ERR_PTR(-EACCES); + + upper = ovl_decode_real_fh(ofs, fh, ovl_upper_mnt(ofs), true); + if (IS_ERR_OR_NULL(upper)) + return upper; + + dentry = ovl_get_dentry(sb, upper, NULL, NULL); + dput(upper); + + return dentry; +} + +static struct dentry *ovl_lower_fh_to_d(struct super_block *sb, + struct ovl_fh *fh) +{ + struct ovl_fs *ofs = OVL_FS(sb); + struct ovl_path origin = { }; + struct ovl_path *stack = &origin; + struct dentry *dentry = NULL; + struct dentry *index = NULL; + struct inode *inode; + int err; + + /* First lookup overlay inode in inode cache by origin fh */ + err = ovl_check_origin_fh(ofs, fh, false, NULL, &stack); + if (err) + return ERR_PTR(err); + + if (!d_is_dir(origin.dentry) || + !(origin.dentry->d_flags & DCACHE_DISCONNECTED)) { + inode = ovl_lookup_inode(sb, origin.dentry, false); + err = PTR_ERR(inode); + if (IS_ERR(inode)) + goto out_err; + if (inode) { + dentry = d_find_any_alias(inode); + iput(inode); + if (dentry) + goto out; + } + } + + /* Then lookup indexed upper/whiteout by origin fh */ + if (ovl_indexdir(sb)) { + index = ovl_get_index_fh(ofs, fh); + err = PTR_ERR(index); + if (IS_ERR(index)) { + index = NULL; + goto out_err; + } + } + + /* Then try to get a connected upper dir by index */ + if (index && d_is_dir(index)) { + struct dentry *upper = ovl_index_upper(ofs, index, true); + + err = PTR_ERR(upper); + if (IS_ERR_OR_NULL(upper)) + goto out_err; + + dentry = ovl_get_dentry(sb, upper, NULL, NULL); + dput(upper); + goto out; + } + + /* Find origin.dentry again with ovl_acceptable() layer check */ + if (d_is_dir(origin.dentry)) { + dput(origin.dentry); + origin.dentry = NULL; + err = ovl_check_origin_fh(ofs, fh, true, NULL, &stack); + if (err) + goto out_err; + } + if (index) { + err = ovl_verify_origin(ofs, index, origin.dentry, false); + if (err) + goto out_err; + } + + /* Get a connected non-upper dir or disconnected non-dir */ + dentry = ovl_get_dentry(sb, NULL, &origin, index); + +out: + dput(origin.dentry); + dput(index); + return dentry; + +out_err: + dentry = ERR_PTR(err); + goto out; +} + +static struct ovl_fh *ovl_fid_to_fh(struct fid *fid, int buflen, int fh_type) +{ + struct ovl_fh *fh; + + /* If on-wire inner fid is aligned - nothing to do */ + if (fh_type == OVL_FILEID_V1) + return (struct ovl_fh *)fid; + + if (fh_type != OVL_FILEID_V0) + return ERR_PTR(-EINVAL); + + if (buflen <= OVL_FH_WIRE_OFFSET) + return ERR_PTR(-EINVAL); + + fh = kzalloc(buflen, GFP_KERNEL); + if (!fh) + return ERR_PTR(-ENOMEM); + + /* Copy unaligned inner fh into aligned buffer */ + memcpy(fh->buf, fid, buflen - OVL_FH_WIRE_OFFSET); + return fh; +} + +static struct dentry *ovl_fh_to_dentry(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + struct dentry *dentry = NULL; + struct ovl_fh *fh = NULL; + int len = fh_len << 2; + unsigned int flags = 0; + int err; + + fh = ovl_fid_to_fh(fid, len, fh_type); + err = PTR_ERR(fh); + if (IS_ERR(fh)) + goto out_err; + + err = ovl_check_fh_len(fh, len); + if (err) + goto out_err; + + flags = fh->fb.flags; + dentry = (flags & OVL_FH_FLAG_PATH_UPPER) ? + ovl_upper_fh_to_d(sb, fh) : + ovl_lower_fh_to_d(sb, fh); + err = PTR_ERR(dentry); + if (IS_ERR(dentry) && err != -ESTALE) + goto out_err; + +out: + /* We may have needed to re-align OVL_FILEID_V0 */ + if (!IS_ERR_OR_NULL(fh) && fh != (void *)fid) + kfree(fh); + + return dentry; + +out_err: + pr_warn_ratelimited("failed to decode file handle (len=%d, type=%d, flags=%x, err=%i)\n", + fh_len, fh_type, flags, err); + dentry = ERR_PTR(err); + goto out; +} + +static struct dentry *ovl_fh_to_parent(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + pr_warn_ratelimited("connectable file handles not supported; use 'no_subtree_check' exportfs option.\n"); + return ERR_PTR(-EACCES); +} + +static int ovl_get_name(struct dentry *parent, char *name, + struct dentry *child) +{ + /* + * ovl_fh_to_dentry() returns connected dir overlay dentries and + * ovl_fh_to_parent() is not implemented, so we should not get here. + */ + WARN_ON_ONCE(1); + return -EIO; +} + +static struct dentry *ovl_get_parent(struct dentry *dentry) +{ + /* + * ovl_fh_to_dentry() returns connected dir overlay dentries, so we + * should not get here. + */ + WARN_ON_ONCE(1); + return ERR_PTR(-EIO); +} + +const struct export_operations ovl_export_operations = { + .encode_fh = ovl_encode_fh, + .fh_to_dentry = ovl_fh_to_dentry, + .fh_to_parent = ovl_fh_to_parent, + .get_name = ovl_get_name, + .get_parent = ovl_get_parent, +}; + +/* encode_fh() encodes non-decodable file handles with nfs_export=off */ +const struct export_operations ovl_export_fid_operations = { + .encode_fh = ovl_encode_fh, +}; diff --git a/fs/overlayfs/file.c b/fs/overlayfs/file.c new file mode 100644 index 000000000..f3d97eb14 --- /dev/null +++ b/fs/overlayfs/file.c @@ -0,0 +1,666 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017 Red Hat, Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "overlayfs.h" + +static char ovl_whatisit(struct inode *inode, struct inode *realinode) +{ + if (realinode != ovl_inode_upper(inode)) + return 'l'; + if (ovl_has_upperdata(inode)) + return 'u'; + else + return 'm'; +} + +static struct file *ovl_open_realfile(const struct file *file, + const struct path *realpath) +{ + struct inode *realinode = d_inode(realpath->dentry); + struct inode *inode = file_inode(file); + struct mnt_idmap *real_idmap; + struct file *realfile; + int flags = file->f_flags | OVL_OPEN_FLAGS; + int acc_mode = ACC_MODE(flags); + int err; + + if (flags & O_APPEND) + acc_mode |= MAY_APPEND; + + with_ovl_creds(inode->i_sb) { + real_idmap = mnt_idmap(realpath->mnt); + err = inode_permission(real_idmap, realinode, MAY_OPEN | acc_mode); + if (err) { + realfile = ERR_PTR(err); + } else { + if (!inode_owner_or_capable(real_idmap, realinode)) + flags &= ~O_NOATIME; + + realfile = backing_file_open(file, + flags, realpath, current_cred()); + } + } + + pr_debug("open(%p[%pD2/%c], 0%o) -> (%p, 0%o)\n", + file, file, ovl_whatisit(inode, realinode), file->f_flags, + realfile, IS_ERR(realfile) ? 0 : realfile->f_flags); + + return realfile; +} + +#define OVL_SETFL_MASK (O_APPEND | O_NONBLOCK | O_NDELAY | O_DIRECT) + +static int ovl_change_flags(struct file *file, unsigned int flags) +{ + struct inode *inode = file_inode(file); + int err; + + flags &= OVL_SETFL_MASK; + + if (((flags ^ file->f_flags) & O_APPEND) && IS_APPEND(inode)) + return -EPERM; + + if ((flags & O_DIRECT) && !(file->f_mode & FMODE_CAN_ODIRECT)) + return -EINVAL; + + if (file->f_op->check_flags) { + err = file->f_op->check_flags(flags); + if (err) + return err; + } + + spin_lock(&file->f_lock); + file->f_flags = (file->f_flags & ~OVL_SETFL_MASK) | flags; + file->f_iocb_flags = iocb_flags(file); + spin_unlock(&file->f_lock); + + return 0; +} + +struct ovl_file { + struct file *realfile; + struct file *upperfile; +}; + +struct ovl_file *ovl_file_alloc(struct file *realfile) +{ + struct ovl_file *of = kzalloc_obj(struct ovl_file); + + if (unlikely(!of)) + return NULL; + + of->realfile = realfile; + return of; +} + +void ovl_file_free(struct ovl_file *of) +{ + fput(of->realfile); + if (of->upperfile) + fput(of->upperfile); + kfree(of); +} + +static bool ovl_is_real_file(const struct file *realfile, + const struct path *realpath) +{ + return file_inode(realfile) == d_inode(realpath->dentry); +} + +static struct file *ovl_real_file_path(const struct file *file, + const struct path *realpath) +{ + struct ovl_file *of = file->private_data; + struct file *realfile = of->realfile; + + if (WARN_ON_ONCE(!realpath->dentry)) + return ERR_PTR(-EIO); + + /* + * If the realfile that we want is not where the data used to be at + * open time, either we'd been copied up, or it's an fsync of a + * metacopied file. We need the upperfile either way, so see if it + * is already opened and if it is not then open and store it. + */ + if (unlikely(!ovl_is_real_file(realfile, realpath))) { + struct file *upperfile = READ_ONCE(of->upperfile); + struct file *old; + + if (!upperfile) { /* Nobody opened upperfile yet */ + upperfile = ovl_open_realfile(file, realpath); + if (IS_ERR(upperfile)) + return upperfile; + + /* Store the upperfile for later */ + old = cmpxchg_release(&of->upperfile, NULL, upperfile); + if (old) { /* Someone opened upperfile before us */ + fput(upperfile); + upperfile = old; + } + } + /* + * Stored file must be from the right inode, unless someone's + * been corrupting the upper layer. + */ + if (WARN_ON_ONCE(!ovl_is_real_file(upperfile, realpath))) + return ERR_PTR(-EIO); + + realfile = upperfile; + } + + /* Did the flags change since open? */ + if (unlikely((file->f_flags ^ realfile->f_flags) & ~OVL_OPEN_FLAGS)) { + int err = ovl_change_flags(realfile, file->f_flags); + + if (err) + return ERR_PTR(err); + } + + return realfile; +} + +static struct file *ovl_real_file(const struct file *file) +{ + struct dentry *dentry = file_dentry(file); + struct path realpath; + int err; + + if (d_is_dir(dentry)) { + struct file *f = ovl_dir_real_file(file, false); + + if (WARN_ON_ONCE(!f)) + return ERR_PTR(-EIO); + return f; + } + + /* lazy lookup and verify of lowerdata */ + err = ovl_verify_lowerdata(dentry); + if (err) + return ERR_PTR(err); + + ovl_path_realdata(dentry, &realpath); + + return ovl_real_file_path(file, &realpath); +} + +static int ovl_open(struct inode *inode, struct file *file) +{ + struct dentry *dentry = file_dentry(file); + struct file *realfile; + struct path realpath; + struct ovl_file *of; + int err; + + /* lazy lookup and verify lowerdata */ + err = ovl_verify_lowerdata(dentry); + if (err) + return err; + + err = ovl_maybe_copy_up(dentry, file->f_flags); + if (err) + return err; + + /* No longer need these flags, so don't pass them on to underlying fs */ + file->f_flags &= ~(O_CREAT | O_EXCL | O_NOCTTY | O_TRUNC); + + ovl_path_realdata(dentry, &realpath); + if (!realpath.dentry) + return -EIO; + + realfile = ovl_open_realfile(file, &realpath); + if (IS_ERR(realfile)) + return PTR_ERR(realfile); + + of = ovl_file_alloc(realfile); + if (!of) { + fput(realfile); + return -ENOMEM; + } + + file->private_data = of; + + return 0; +} + +static int ovl_release(struct inode *inode, struct file *file) +{ + ovl_file_free(file->private_data); + return 0; +} + +static loff_t ovl_llseek(struct file *file, loff_t offset, int whence) +{ + struct inode *inode = file_inode(file); + struct file *realfile; + loff_t ret; + + /* + * The two special cases below do not need to involve real fs, + * so we can optimizing concurrent callers. + */ + if (offset == 0) { + if (whence == SEEK_CUR) + return file->f_pos; + + if (whence == SEEK_SET) + return vfs_setpos(file, 0, 0); + } + + realfile = ovl_real_file(file); + if (IS_ERR(realfile)) + return PTR_ERR(realfile); + + /* + * Overlay file f_pos is the master copy that is preserved + * through copy up and modified on read/write, but only real + * fs knows how to SEEK_HOLE/SEEK_DATA and real fs may impose + * limitations that are more strict than ->s_maxbytes for specific + * files, so we use the real file to perform seeks. + */ + ovl_inode_lock(inode); + realfile->f_pos = file->f_pos; + + with_ovl_creds(inode->i_sb) + ret = vfs_llseek(realfile, offset, whence); + + file->f_pos = realfile->f_pos; + ovl_inode_unlock(inode); + + return ret; +} + +static void ovl_file_modified(struct file *file) +{ + /* Update size/mtime */ + ovl_copyattr(file_inode(file)); +} + +static void ovl_file_end_write(struct kiocb *iocb, ssize_t ret) +{ + ovl_file_modified(iocb->ki_filp); +} + +static void ovl_file_accessed(struct file *file) +{ + struct inode *inode, *upperinode; + struct timespec64 ctime, uctime; + struct timespec64 mtime, umtime; + + if (file->f_flags & O_NOATIME) + return; + + inode = file_inode(file); + upperinode = ovl_inode_upper(inode); + + if (!upperinode) + return; + + ctime = inode_get_ctime(inode); + uctime = inode_get_ctime(upperinode); + mtime = inode_get_mtime(inode); + umtime = inode_get_mtime(upperinode); + if ((!timespec64_equal(&mtime, &umtime)) || + !timespec64_equal(&ctime, &uctime)) { + inode_set_mtime_to_ts(inode, inode_get_mtime(upperinode)); + inode_set_ctime_to_ts(inode, uctime); + } + + touch_atime(&file->f_path); +} + +static ssize_t ovl_read_iter(struct kiocb *iocb, struct iov_iter *iter) +{ + struct file *file = iocb->ki_filp; + struct file *realfile; + struct backing_file_ctx ctx = { + .cred = ovl_creds(file_inode(file)->i_sb), + .accessed = ovl_file_accessed, + }; + + if (!iov_iter_count(iter)) + return 0; + + realfile = ovl_real_file(file); + if (IS_ERR(realfile)) + return PTR_ERR(realfile); + + return backing_file_read_iter(realfile, iter, iocb, iocb->ki_flags, + &ctx); +} + +static ssize_t ovl_write_iter(struct kiocb *iocb, struct iov_iter *iter) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct file *realfile; + ssize_t ret; + int ifl = iocb->ki_flags; + struct backing_file_ctx ctx = { + .cred = ovl_creds(inode->i_sb), + .end_write = ovl_file_end_write, + }; + + if (!iov_iter_count(iter)) + return 0; + + inode_lock(inode); + /* Update mode */ + ovl_copyattr(inode); + + realfile = ovl_real_file(file); + ret = PTR_ERR(realfile); + if (IS_ERR(realfile)) + goto out_unlock; + + if (!ovl_should_sync(OVL_FS(inode->i_sb))) + ifl &= ~(IOCB_DSYNC | IOCB_SYNC); + + ret = backing_file_write_iter(realfile, iter, iocb, ifl, &ctx); + +out_unlock: + inode_unlock(inode); + + return ret; +} + +static ssize_t ovl_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, + unsigned int flags) +{ + struct file *realfile; + ssize_t ret; + struct backing_file_ctx ctx = { + .cred = ovl_creds(file_inode(in)->i_sb), + .accessed = ovl_file_accessed, + }; + struct kiocb iocb; + + realfile = ovl_real_file(in); + if (IS_ERR(realfile)) + return PTR_ERR(realfile); + + init_sync_kiocb(&iocb, in); + iocb.ki_pos = *ppos; + ret = backing_file_splice_read(realfile, &iocb, pipe, len, flags, &ctx); + *ppos = iocb.ki_pos; + + return ret; +} + +/* + * Calling iter_file_splice_write() directly from overlay's f_op may deadlock + * due to lock order inversion between pipe->mutex in iter_file_splice_write() + * and file_start_write(realfile) in ovl_write_iter(). + * + * So do everything ovl_write_iter() does and call iter_file_splice_write() on + * the real file. + */ +static ssize_t ovl_splice_write(struct pipe_inode_info *pipe, struct file *out, + loff_t *ppos, size_t len, unsigned int flags) +{ + struct file *realfile; + struct inode *inode = file_inode(out); + ssize_t ret; + struct backing_file_ctx ctx = { + .cred = ovl_creds(inode->i_sb), + .end_write = ovl_file_end_write, + }; + struct kiocb iocb; + + inode_lock(inode); + /* Update mode */ + ovl_copyattr(inode); + + realfile = ovl_real_file(out); + ret = PTR_ERR(realfile); + if (IS_ERR(realfile)) + goto out_unlock; + + init_sync_kiocb(&iocb, out); + iocb.ki_pos = *ppos; + ret = backing_file_splice_write(pipe, realfile, &iocb, len, flags, &ctx); + *ppos = iocb.ki_pos; + +out_unlock: + inode_unlock(inode); + + return ret; +} + +static int ovl_fsync(struct file *file, loff_t start, loff_t end, int datasync) +{ + struct dentry *dentry = file_dentry(file); + enum ovl_path_type type; + struct path upperpath; + struct file *upperfile; + int ret; + + ret = ovl_sync_status(OVL_FS(file_inode(file)->i_sb)); + if (ret <= 0) + return ret; + + /* Don't sync lower file for fear of receiving EROFS error */ + type = ovl_path_type(dentry); + if (!OVL_TYPE_UPPER(type) || (datasync && OVL_TYPE_MERGE(type))) + return 0; + + ovl_path_upper(dentry, &upperpath); + upperfile = ovl_real_file_path(file, &upperpath); + if (IS_ERR(upperfile)) + return PTR_ERR(upperfile); + + with_ovl_creds(file_inode(file)->i_sb) + return vfs_fsync_range(upperfile, start, end, datasync); +} + +static int ovl_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct ovl_file *of = file->private_data; + struct backing_file_ctx ctx = { + .cred = ovl_creds(file_inode(file)->i_sb), + .accessed = ovl_file_accessed, + }; + + return backing_file_mmap(of->realfile, vma, &ctx); +} + +static long ovl_fallocate(struct file *file, int mode, loff_t offset, loff_t len) +{ + struct inode *inode = file_inode(file); + struct file *realfile; + int ret; + + inode_lock(inode); + /* Update mode */ + ovl_copyattr(inode); + ret = file_remove_privs(file); + if (ret) + goto out_unlock; + + realfile = ovl_real_file(file); + ret = PTR_ERR(realfile); + if (IS_ERR(realfile)) + goto out_unlock; + + with_ovl_creds(inode->i_sb) + ret = vfs_fallocate(realfile, mode, offset, len); + + /* Update size */ + ovl_file_modified(file); + +out_unlock: + inode_unlock(inode); + + return ret; +} + +static int ovl_fadvise(struct file *file, loff_t offset, loff_t len, int advice) +{ + struct file *realfile; + + realfile = ovl_real_file(file); + if (IS_ERR(realfile)) + return PTR_ERR(realfile); + + with_ovl_creds(file_inode(file)->i_sb) + return vfs_fadvise(realfile, offset, len, advice); +} + +enum ovl_copyop { + OVL_COPY, + OVL_CLONE, + OVL_DEDUPE, +}; + +static loff_t ovl_copyfile(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + loff_t len, unsigned int flags, enum ovl_copyop op) +{ + struct inode *inode_in = file_inode(file_in); + struct inode *inode_out = file_inode(file_out); + struct file *realfile_in, *realfile_out; + loff_t ret; + + inode_lock(inode_out); + if (op != OVL_DEDUPE) { + /* Update mode */ + ovl_copyattr(inode_out); + ret = file_remove_privs(file_out); + if (ret) + goto out_unlock; + } + + realfile_out = ovl_real_file(file_out); + ret = PTR_ERR(realfile_out); + if (IS_ERR(realfile_out)) + goto out_unlock; + + realfile_in = ovl_real_file(file_in); + ret = PTR_ERR(realfile_in); + if (IS_ERR(realfile_in)) + goto out_unlock; + + /* + * For cross-sb copy, vfs_copy_file_range() will verify read access with + * the mounter creds of the dest fs mounter, so we need to explicitly + * verify read access with the source mounter creds. + */ + if (unlikely(inode_in->i_sb != inode_out->i_sb)) { + with_ovl_creds(inode_in->i_sb) { + ret = rw_verify_area(READ, realfile_in, &pos_in, len); + if (unlikely(ret)) + goto out_unlock; + } + } + + with_ovl_creds(inode_out->i_sb) { + switch (op) { + case OVL_COPY: + ret = vfs_copy_file_range(realfile_in, pos_in, + realfile_out, pos_out, len, flags); + break; + + case OVL_CLONE: + ret = vfs_clone_file_range(realfile_in, pos_in, + realfile_out, pos_out, len, flags); + break; + + case OVL_DEDUPE: + ret = vfs_dedupe_file_range_one(realfile_in, pos_in, + realfile_out, pos_out, len, + flags); + break; + } + } + + /* Update size */ + ovl_file_modified(file_out); + +out_unlock: + inode_unlock(inode_out); + + return ret; +} + +static ssize_t ovl_copy_file_range(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + size_t len, unsigned int flags) +{ + return ovl_copyfile(file_in, pos_in, file_out, pos_out, len, flags, + OVL_COPY); +} + +static loff_t ovl_remap_file_range(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + loff_t len, unsigned int remap_flags) +{ + enum ovl_copyop op; + + if (remap_flags & ~(REMAP_FILE_DEDUP | REMAP_FILE_ADVISORY)) + return -EINVAL; + + if (remap_flags & REMAP_FILE_DEDUP) + op = OVL_DEDUPE; + else + op = OVL_CLONE; + + /* + * Don't copy up because of a dedupe request, this wouldn't make sense + * most of the time (data would be duplicated instead of deduplicated). + */ + if (op == OVL_DEDUPE && + (!ovl_inode_upper(file_inode(file_in)) || + !ovl_inode_upper(file_inode(file_out)))) + return -EPERM; + + return ovl_copyfile(file_in, pos_in, file_out, pos_out, len, + remap_flags, op); +} + +static int ovl_flush(struct file *file, fl_owner_t id) +{ + struct file *realfile; + int err = 0; + + realfile = ovl_real_file(file); + if (IS_ERR(realfile)) + return PTR_ERR(realfile); + + if (realfile->f_op->flush) { + with_ovl_creds(file_inode(file)->i_sb) + err = realfile->f_op->flush(realfile, id); + } + + return err; +} + +const struct file_operations ovl_file_operations = { + .open = ovl_open, + .release = ovl_release, + .llseek = ovl_llseek, + .read_iter = ovl_read_iter, + .write_iter = ovl_write_iter, + .fsync = ovl_fsync, + .mmap = ovl_mmap, + .fallocate = ovl_fallocate, + .fadvise = ovl_fadvise, + .flush = ovl_flush, + .splice_read = ovl_splice_read, + .splice_write = ovl_splice_write, + + .copy_file_range = ovl_copy_file_range, + .remap_file_range = ovl_remap_file_range, + .setlease = generic_setlease, +}; diff --git a/fs/overlayfs/inode.c b/fs/overlayfs/inode.c new file mode 100644 index 000000000..401cb8c75 --- /dev/null +++ b/fs/overlayfs/inode.c @@ -0,0 +1,1314 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * + * Copyright (C) 2011 Novell Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "overlayfs.h" + + +int ovl_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +{ + int err; + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + bool full_copy_up = false; + struct dentry *upperdentry; + + err = setattr_prepare(idmap, dentry, attr); + if (err) + return err; + + /* Rebase ownership from the mount idmap into overlay id space. */ + if (attr->ia_valid & ATTR_UID) + attr->ia_vfsuid = VFSUIDT_INIT(from_vfsuid(idmap, + i_user_ns(d_inode(dentry)), attr->ia_vfsuid)); + if (attr->ia_valid & ATTR_GID) + attr->ia_vfsgid = VFSGIDT_INIT(from_vfsgid(idmap, + i_user_ns(d_inode(dentry)), attr->ia_vfsgid)); + + if (attr->ia_valid & ATTR_SIZE) { + /* Truncate should trigger data copy up as well */ + full_copy_up = true; + } + + if (!full_copy_up) + err = ovl_copy_up(dentry); + else + err = ovl_copy_up_with_data(dentry); + if (!err) { + struct inode *winode = NULL; + + upperdentry = ovl_dentry_upper(dentry); + + if (attr->ia_valid & ATTR_SIZE) { + winode = d_inode(upperdentry); + err = get_write_access(winode); + if (err) + goto out; + } + + if (attr->ia_valid & (ATTR_KILL_SUID|ATTR_KILL_SGID)) + attr->ia_valid &= ~ATTR_MODE; + + /* + * We might have to translate ovl file into real file object + * once use cases emerge. For now, simply don't let underlying + * filesystem rely on attr->ia_file + */ + attr->ia_valid &= ~ATTR_FILE; + + /* + * If open(O_TRUNC) is done, VFS calls ->setattr with ATTR_OPEN + * set. Overlayfs does not pass O_TRUNC flag to underlying + * filesystem during open -> do not pass ATTR_OPEN. This + * disables optimization in fuse which assumes open(O_TRUNC) + * already set file size to 0. But we never passed O_TRUNC to + * fuse. So by clearing ATTR_OPEN, fuse will be forced to send + * setattr request to server. + */ + attr->ia_valid &= ~ATTR_OPEN; + + err = ovl_want_write(dentry); + if (err) + goto out_put_write; + + inode_lock(upperdentry->d_inode); + with_ovl_creds(dentry->d_sb) + err = ovl_do_notify_change(ofs, upperdentry, attr); + if (!err) + ovl_copyattr(dentry->d_inode); + inode_unlock(upperdentry->d_inode); + ovl_drop_write(dentry); + +out_put_write: + if (winode) + put_write_access(winode); + } +out: + return err; +} + +static void ovl_map_dev_ino(struct dentry *dentry, struct kstat *stat, int fsid) +{ + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + bool samefs = ovl_same_fs(ofs); + unsigned int xinobits = ovl_xino_bits(ofs); + unsigned int xinoshift = 64 - xinobits; + + if (samefs) { + /* + * When all layers are on the same fs, all real inode + * number are unique, so we use the overlay st_dev, + * which is friendly to du -x. + */ + stat->dev = dentry->d_sb->s_dev; + return; + } else if (xinobits) { + /* + * All inode numbers of underlying fs should not be using the + * high xinobits, so we use high xinobits to partition the + * overlay st_ino address space. The high bits holds the fsid + * (upper fsid is 0). The lowest xinobit is reserved for mapping + * the non-persistent inode numbers range in case of overflow. + * This way all overlay inode numbers are unique and use the + * overlay st_dev. + */ + if (likely(!(stat->ino >> xinoshift))) { + stat->ino |= ((u64)fsid) << (xinoshift + 1); + stat->dev = dentry->d_sb->s_dev; + return; + } else if (ovl_xino_warn(ofs)) { + pr_warn_ratelimited("inode number too big (%pd2, ino=%llu, xinobits=%d)\n", + dentry, stat->ino, xinobits); + } + } + + /* The inode could not be mapped to a unified st_ino address space */ + if (S_ISDIR(dentry->d_inode->i_mode)) { + /* + * Always use the overlay st_dev for directories, so 'find + * -xdev' will scan the entire overlay mount and won't cross the + * overlay mount boundaries. + * + * If not all layers are on the same fs the pair {real st_ino; + * overlay st_dev} is not unique, so use the non persistent + * overlay st_ino for directories. + */ + stat->dev = dentry->d_sb->s_dev; + stat->ino = dentry->d_inode->i_ino; + } else { + /* + * For non-samefs setup, if we cannot map all layers st_ino + * to a unified address space, we need to make sure that st_dev + * is unique per underlying fs, so we use the unique anonymous + * bdev assigned to the underlying fs. + */ + stat->dev = ofs->fs[fsid].pseudo_dev; + } +} + +static inline int ovl_real_getattr_nosec(struct super_block *sb, + const struct path *path, + struct kstat *stat, u32 request_mask, + unsigned int flags) +{ + with_ovl_creds(sb) + return vfs_getattr_nosec(path, stat, request_mask, flags); +} + +int ovl_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags) +{ + struct dentry *dentry = path->dentry; + struct super_block *sb = dentry->d_sb; + enum ovl_path_type type; + struct path realpath; + struct inode *inode = d_inode(dentry); + bool is_dir = S_ISDIR(inode->i_mode); + int fsid = 0; + int err; + bool metacopy_blocks = false; + vfsuid_t vfsuid; + vfsgid_t vfsgid; + + metacopy_blocks = ovl_is_metacopy_dentry(dentry); + + type = ovl_path_real(dentry, &realpath); + err = ovl_real_getattr_nosec(sb, &realpath, stat, request_mask, flags); + if (err) + return err; + + /* Report the effective immutable/append-only STATX flags */ + generic_fill_statx_attr(inode, stat); + + /* + * For non-dir or same fs, we use st_ino of the copy up origin. + * This guaranties constant st_dev/st_ino across copy up. + * With xino feature and non-samefs, we use st_ino of the copy up + * origin masked with high bits that represent the layer id. + * + * If lower filesystem supports NFS file handles, this also guaranties + * persistent st_ino across mount cycle. + */ + if (!is_dir || ovl_same_dev(OVL_FS(dentry->d_sb))) { + if (!OVL_TYPE_UPPER(type)) { + fsid = ovl_layer_lower(dentry)->fsid; + } else if (OVL_TYPE_ORIGIN(type)) { + struct kstat lowerstat; + u32 lowermask = STATX_INO | STATX_BLOCKS | + (!is_dir ? STATX_NLINK : 0); + + ovl_path_lower(dentry, &realpath); + err = ovl_real_getattr_nosec(sb, &realpath, &lowerstat, lowermask, flags); + if (err) + return err; + + /* + * Lower hardlinks may be broken on copy up to different + * upper files, so we cannot use the lower origin st_ino + * for those different files, even for the same fs case. + * + * Similarly, several redirected dirs can point to the + * same dir on a lower layer. With the "verify_lower" + * feature, we do not use the lower origin st_ino, if + * we haven't verified that this redirect is unique. + * + * With inodes index enabled, it is safe to use st_ino + * of an indexed origin. The index validates that the + * upper hardlink is not broken and that a redirected + * dir is the only redirect to that origin. + */ + if (ovl_test_flag(OVL_INDEX, d_inode(dentry)) || + (!ovl_verify_lower(dentry->d_sb) && + (is_dir || lowerstat.nlink == 1))) { + fsid = ovl_layer_lower(dentry)->fsid; + stat->ino = lowerstat.ino; + } + + /* + * If we are querying a metacopy dentry and lower + * dentry is data dentry, then use the blocks we + * queried just now. We don't have to do additional + * vfs_getattr(). If lower itself is metacopy, then + * additional vfs_getattr() is unavoidable. + */ + if (metacopy_blocks && + realpath.dentry == ovl_dentry_lowerdata(dentry)) { + stat->blocks = lowerstat.blocks; + metacopy_blocks = false; + } + } + + if (metacopy_blocks) { + /* + * If lower is not same as lowerdata or if there was + * no origin on upper, we can end up here. + * With lazy lowerdata lookup, guess lowerdata blocks + * from size to avoid lowerdata lookup on stat(2). + */ + struct kstat lowerdatastat; + u32 lowermask = STATX_BLOCKS; + + ovl_path_lowerdata(dentry, &realpath); + if (realpath.dentry) { + err = ovl_real_getattr_nosec(sb, &realpath, &lowerdatastat, + lowermask, flags); + if (err) + return err; + } else { + lowerdatastat.blocks = + round_up(stat->size, stat->blksize) >> 9; + } + stat->blocks = lowerdatastat.blocks; + } + } + + ovl_map_dev_ino(dentry, stat, fsid); + + /* + * It's probably not worth it to count subdirs to get the + * correct link count. nlink=1 seems to pacify 'find' and + * other utilities. + */ + if (is_dir && OVL_TYPE_MERGE(type)) + stat->nlink = 1; + + /* + * Return the overlay inode nlinks for indexed upper inodes. + * Overlay inode nlink counts the union of the upper hardlinks + * and non-covered lower hardlinks. It does not include the upper + * index hardlink. + */ + if (!is_dir && ovl_test_flag(OVL_INDEX, d_inode(dentry))) + stat->nlink = dentry->d_inode->i_nlink; + + /* Map ownership of the real inode through the overlay mount idmap. */ + vfsuid = make_vfsuid(idmap, i_user_ns(inode), stat->uid); + vfsgid = make_vfsgid(idmap, i_user_ns(inode), stat->gid); + stat->uid = vfsuid_into_kuid(vfsuid); + stat->gid = vfsgid_into_kgid(vfsgid); + + return err; +} + +int ovl_permission(struct mnt_idmap *idmap, + struct inode *inode, int mask) +{ + struct inode *upperinode = ovl_inode_upper(inode); + struct inode *realinode; + struct path realpath; + int err; + + /* Careful in RCU walk mode */ + realinode = ovl_i_path_real(inode, &realpath); + if (!realinode) { + WARN_ON(!(mask & MAY_NOT_BLOCK)); + return -ECHILD; + } + + /* + * Check overlay inode with the creds of task and underlying inode + * with creds of mounter + */ + err = generic_permission(idmap, inode, mask); + if (err) + return err; + + if (!upperinode && + !special_file(realinode->i_mode) && mask & MAY_WRITE) { + mask &= ~(MAY_WRITE | MAY_APPEND); + /* Make sure mounter can read file for copy up later */ + mask |= MAY_READ; + } + + with_ovl_creds(inode->i_sb) + return inode_permission(mnt_idmap(realpath.mnt), realinode, mask); +} + +static const char *ovl_get_link(struct dentry *dentry, + struct inode *inode, + struct delayed_call *done) +{ + if (!dentry) + return ERR_PTR(-ECHILD); + + with_ovl_creds(dentry->d_sb) + return vfs_get_link(ovl_dentry_real(dentry), done); +} + +#ifdef CONFIG_FS_POSIX_ACL +/* + * Apply the idmapping of the layer to POSIX ACLs. The caller must pass a clone + * of the POSIX ACLs retrieved from the lower layer to this function to not + * alter the POSIX ACLs for the underlying filesystem. + */ +static void ovl_idmap_posix_acl(const struct inode *realinode, + struct mnt_idmap *idmap, + struct posix_acl *acl) +{ + struct user_namespace *fs_userns = i_user_ns(realinode); + + for (unsigned int i = 0; i < acl->a_count; i++) { + vfsuid_t vfsuid; + vfsgid_t vfsgid; + + struct posix_acl_entry *e = &acl->a_entries[i]; + switch (e->e_tag) { + case ACL_USER: + vfsuid = make_vfsuid(idmap, fs_userns, e->e_uid); + e->e_uid = vfsuid_into_kuid(vfsuid); + break; + case ACL_GROUP: + vfsgid = make_vfsgid(idmap, fs_userns, e->e_gid); + e->e_gid = vfsgid_into_kgid(vfsgid); + break; + } + } +} + +/* + * The @noperm argument is used to skip permission checking and is a temporary + * measure. Quoting Miklos from an earlier discussion: + * + * > So there are two paths to getting an acl: + * > 1) permission checking and 2) retrieving the value via getxattr(2). + * > This is a similar situation as reading a symlink vs. following it. + * > When following a symlink overlayfs always reads the link on the + * > underlying fs just as if it was a readlink(2) call, calling + * > security_inode_readlink() instead of security_inode_follow_link(). + * > This is logical: we are reading the link from the underlying storage, + * > and following it on overlayfs. + * > + * > Applying the same logic to acl: we do need to call the + * > security_inode_getxattr() on the underlying fs, even if just want to + * > check permissions on overlay. This is currently not done, which is an + * > inconsistency. + * > + * > Maybe adding the check to ovl_get_acl() is the right way to go, but + * > I'm a little afraid of a performance regression. Will look into that. + * + * Until we have made a decision allow this helper to take the @noperm + * argument. We should hopefully be able to remove it soon. + */ +struct posix_acl *ovl_get_acl_path(const struct path *path, + const char *acl_name, bool noperm) +{ + struct posix_acl *real_acl, *clone; + struct mnt_idmap *idmap; + struct inode *realinode = d_inode(path->dentry); + + idmap = mnt_idmap(path->mnt); + + if (noperm) + real_acl = get_inode_acl(realinode, posix_acl_type(acl_name)); + else + real_acl = vfs_get_acl(idmap, path->dentry, acl_name); + if (IS_ERR_OR_NULL(real_acl)) + return real_acl; + + if (!is_idmapped_mnt(path->mnt)) + return real_acl; + + /* + * We cannot alter the ACLs returned from the relevant layer as that + * would alter the cached values filesystem wide for the lower + * filesystem. Instead we can clone the ACLs and then apply the + * relevant idmapping of the layer. + */ + clone = posix_acl_clone(real_acl, GFP_KERNEL); + posix_acl_release(real_acl); /* release original acl */ + if (!clone) + return ERR_PTR(-ENOMEM); + + ovl_idmap_posix_acl(realinode, idmap, clone); + return clone; +} + +/* + * When the relevant layer is an idmapped mount we need to take the idmapping + * of the layer into account and translate any ACL_{GROUP,USER} values + * according to the idmapped mount. + * + * We cannot alter the ACLs returned from the relevant layer as that would + * alter the cached values filesystem wide for the lower filesystem. Instead we + * can clone the ACLs and then apply the relevant idmapping of the layer. + * + * This is obviously only relevant when idmapped layers are used. + */ +struct posix_acl *do_ovl_get_acl(struct mnt_idmap *idmap, + struct inode *inode, int type, + bool rcu, bool noperm) +{ + struct inode *realinode; + struct posix_acl *acl; + struct path realpath; + + /* Careful in RCU walk mode */ + realinode = ovl_i_path_real(inode, &realpath); + if (!realinode) { + WARN_ON(!rcu); + return ERR_PTR(-ECHILD); + } + + if (!IS_POSIXACL(realinode)) + return NULL; + + if (rcu) { + /* + * If the layer is idmapped drop out of RCU path walk + * so we can clone the ACLs. + */ + if (is_idmapped_mnt(realpath.mnt)) + return ERR_PTR(-ECHILD); + + acl = get_cached_acl_rcu(realinode, type); + } else { + with_ovl_creds(inode->i_sb) + acl = ovl_get_acl_path(&realpath, posix_acl_xattr_name(type), noperm); + } + + return acl; +} + +static int ovl_set_or_remove_acl(struct dentry *dentry, struct inode *inode, + struct posix_acl *acl, int type) +{ + int err; + struct path realpath; + const char *acl_name; + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + struct dentry *upperdentry = ovl_dentry_upper(dentry); + struct dentry *realdentry = upperdentry ?: ovl_dentry_lower(dentry); + + /* + * If ACL is to be removed from a lower file, check if it exists in + * the first place before copying it up. + */ + acl_name = posix_acl_xattr_name(type); + if (!acl && !upperdentry) { + struct posix_acl *real_acl; + + ovl_path_lower(dentry, &realpath); + with_ovl_creds(dentry->d_sb) + real_acl = vfs_get_acl(mnt_idmap(realpath.mnt), realdentry, acl_name); + if (IS_ERR(real_acl)) { + err = PTR_ERR(real_acl); + goto out; + } + posix_acl_release(real_acl); + } + + if (!upperdentry) { + err = ovl_copy_up(dentry); + if (err) + goto out; + + realdentry = ovl_dentry_upper(dentry); + } + + err = ovl_want_write(dentry); + if (err) + goto out; + + with_ovl_creds(dentry->d_sb) { + if (acl) + err = ovl_do_set_acl(ofs, realdentry, acl_name, acl); + else + err = ovl_do_remove_acl(ofs, realdentry, acl_name); + } + ovl_drop_write(dentry); + + /* copy c/mtime */ + ovl_copyattr(inode); +out: + return err; +} + +int ovl_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type) +{ + int err; + struct inode *inode = d_inode(dentry); + struct dentry *workdir = ovl_workdir(dentry); + struct inode *realinode = ovl_inode_real(inode); + + if (!IS_POSIXACL(d_inode(workdir))) + return -EOPNOTSUPP; + if (!realinode->i_op->set_acl) + return -EOPNOTSUPP; + if (type == ACL_TYPE_DEFAULT && !S_ISDIR(inode->i_mode)) + return acl ? -EACCES : 0; + if (!inode_owner_or_capable(idmap, inode)) + return -EPERM; + + /* + * Check if sgid bit needs to be cleared (actual setacl operation will + * be done with mounter's capabilities and so that won't do it for us). + */ + if (unlikely(inode->i_mode & S_ISGID) && type == ACL_TYPE_ACCESS && + !in_group_or_capable(idmap, inode, + i_gid_into_vfsgid(idmap, inode))) { + struct iattr iattr = { .ia_valid = ATTR_KILL_SGID }; + + err = ovl_setattr(&nop_mnt_idmap, dentry, &iattr); + if (err) + return err; + } + + return ovl_set_or_remove_acl(dentry, inode, acl, type); +} +#endif + +int ovl_update_time(struct inode *inode, enum fs_update_time type, + unsigned int flags) +{ + if (type == FS_UPD_ATIME) { + struct ovl_fs *ofs = OVL_FS(inode->i_sb); + struct path upperpath = { + .mnt = ovl_upper_mnt(ofs), + .dentry = ovl_upperdentry_dereference(OVL_I(inode)), + }; + + if (upperpath.dentry) { + if (flags & IOCB_NOWAIT) + return -EAGAIN; + touch_atime(&upperpath); + inode_set_atime_to_ts(inode, + inode_get_atime(d_inode(upperpath.dentry))); + } + } + return 0; +} + +static int ovl_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, + u64 start, u64 len) +{ + struct inode *realinode = ovl_inode_realdata(inode); + + if (!realinode) + return -EIO; + + if (!realinode->i_op->fiemap) + return -EOPNOTSUPP; + + with_ovl_creds(inode->i_sb) + return realinode->i_op->fiemap(realinode, fieinfo, start, len); +} + +/* + * Work around the fact that security_file_ioctl() takes a file argument. + * Introducing security_inode_fileattr_get/set() hooks would solve this issue + * properly. + */ +static int ovl_security_fileattr(const struct path *realpath, struct file_kattr *fa, + bool set) +{ + struct file *file; + unsigned int cmd; + int err; + unsigned int flags; + + flags = O_RDONLY; + if (force_o_largefile()) + flags |= O_LARGEFILE; + + file = dentry_open(realpath, flags, current_cred()); + if (IS_ERR(file)) + return PTR_ERR(file); + + if (set) + cmd = fa->fsx_valid ? FS_IOC_FSSETXATTR : FS_IOC_SETFLAGS; + else + cmd = fa->fsx_valid ? FS_IOC_FSGETXATTR : FS_IOC_GETFLAGS; + + err = security_file_ioctl(file, cmd, 0); + fput(file); + + return err; +} + +int ovl_real_fileattr_set(const struct path *realpath, struct file_kattr *fa) +{ + int err; + + err = ovl_security_fileattr(realpath, fa, true); + if (err) + return err; + + return vfs_fileattr_set(mnt_idmap(realpath->mnt), realpath->dentry, fa); +} + +int ovl_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + struct path upperpath; + unsigned int flags; + int err; + + err = ovl_copy_up(dentry); + if (!err) { + ovl_path_real(dentry, &upperpath); + + err = ovl_want_write(dentry); + if (err) + goto out; + + with_ovl_creds(inode->i_sb) { + /* + * Store immutable/append-only flags in xattr and clear them + * in upper fileattr (in case they were set by older kernel) + * so children of "ovl-immutable" directories lower aliases of + * "ovl-immutable" hardlinks could be copied up. + * Clear xattr when flags are cleared. + */ + err = ovl_set_protattr(inode, upperpath.dentry, fa); + if (!err) + err = ovl_real_fileattr_set(&upperpath, fa); + } + ovl_drop_write(dentry); + + /* + * Merge real inode flags with inode flags read from + * overlay.protattr xattr + */ + flags = ovl_inode_real(inode)->i_flags & OVL_COPY_I_FLAGS_MASK; + + BUILD_BUG_ON(OVL_PROT_I_FLAGS_MASK & ~OVL_COPY_I_FLAGS_MASK); + flags |= inode->i_flags & OVL_PROT_I_FLAGS_MASK; + inode_set_flags(inode, flags, OVL_COPY_I_FLAGS_MASK); + + /* Update ctime */ + ovl_copyattr(inode); + } +out: + return err; +} + +/* Convert inode protection flags to fileattr flags */ +static void ovl_fileattr_prot_flags(struct inode *inode, struct file_kattr *fa) +{ + BUILD_BUG_ON(OVL_PROT_FS_FLAGS_MASK & ~FS_COMMON_FL); + BUILD_BUG_ON(OVL_PROT_FSX_FLAGS_MASK & ~FS_XFLAG_COMMON); + + if (inode->i_flags & S_APPEND) { + fa->flags |= FS_APPEND_FL; + fa->fsx_xflags |= FS_XFLAG_APPEND; + } + if (inode->i_flags & S_IMMUTABLE) { + fa->flags |= FS_IMMUTABLE_FL; + fa->fsx_xflags |= FS_XFLAG_IMMUTABLE; + } +} + +int ovl_real_fileattr_get(const struct path *realpath, struct file_kattr *fa) +{ + int err; + + err = ovl_security_fileattr(realpath, fa, false); + if (err) + return err; + + err = vfs_fileattr_get(realpath->dentry, fa); + if (err == -ENOIOCTLCMD) + err = -ENOTTY; + return err; +} + +int ovl_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + struct path realpath; + int err; + + ovl_path_real(dentry, &realpath); + + with_ovl_creds(inode->i_sb) + err = ovl_real_fileattr_get(&realpath, fa); + ovl_fileattr_prot_flags(inode, fa); + + return err; +} + +static const struct inode_operations ovl_file_inode_operations = { + .setattr = ovl_setattr, + .permission = ovl_permission, + .getattr = ovl_getattr, + .listxattr = ovl_listxattr, + .get_inode_acl = ovl_get_inode_acl, + .get_acl = ovl_get_acl, + .set_acl = ovl_set_acl, + .update_time = ovl_update_time, + .fiemap = ovl_fiemap, + .fileattr_get = ovl_fileattr_get, + .fileattr_set = ovl_fileattr_set, +}; + +static const struct inode_operations ovl_symlink_inode_operations = { + .setattr = ovl_setattr, + .get_link = ovl_get_link, + .getattr = ovl_getattr, + .listxattr = ovl_listxattr, + .update_time = ovl_update_time, +}; + +static const struct inode_operations ovl_special_inode_operations = { + .setattr = ovl_setattr, + .permission = ovl_permission, + .getattr = ovl_getattr, + .listxattr = ovl_listxattr, + .get_inode_acl = ovl_get_inode_acl, + .get_acl = ovl_get_acl, + .set_acl = ovl_set_acl, + .update_time = ovl_update_time, +}; + +static const struct address_space_operations ovl_aops = { + /* For O_DIRECT dentry_open() checks f_mapping->a_ops->direct_IO */ + .direct_IO = noop_direct_IO, +}; + +/* + * It is possible to stack overlayfs instance on top of another + * overlayfs instance as lower layer. We need to annotate the + * stackable i_mutex locks according to stack level of the super + * block instance. An overlayfs instance can never be in stack + * depth 0 (there is always a real fs below it). An overlayfs + * inode lock will use the lockdep annotation ovl_i_mutex_key[depth]. + * + * For example, here is a snip from /proc/lockdep_chains after + * dir_iterate of nested overlayfs: + * + * [...] &ovl_i_mutex_dir_key[depth] (stack_depth=2) + * [...] &ovl_i_mutex_dir_key[depth]#2 (stack_depth=1) + * [...] &type->i_mutex_dir_key (stack_depth=0) + * + * Locking order w.r.t ovl_want_write() is important for nested overlayfs. + * + * This chain is valid: + * - inode->i_rwsem (inode_lock[2]) + * - OVL_I(inode)->lock (ovl_inode_lock[2]) + * - upper_mnt->mnt_sb->s_writers (ovl_want_write[0]) + * - OVL_I(lowerinode)->lock (ovl_inode_lock[1]) + * + * And this chain is valid: + * - inode->i_rwsem (inode_lock[2]) + * - OVL_I(inode)->lock (ovl_inode_lock[2]) + * - lowerinode->i_rwsem (inode_lock[1]) + * - OVL_I(lowerinode)->lock (ovl_inode_lock[1]) + * + * But lowerinode->i_rwsem SHOULD NOT be acquired while ovl_want_write() is + * held, because it is in reverse order of the non-nested case using the same + * upper fs: + * - inode->i_rwsem (inode_lock[1]) + * - OVL_I(inode)->lock (ovl_inode_lock[1]) + * - upper_mnt->mnt_sb->s_writers (ovl_want_write[0]) + */ +#define OVL_MAX_NESTING FILESYSTEM_MAX_STACK_DEPTH + +static inline void ovl_lockdep_annotate_inode_mutex_key(struct inode *inode) +{ +#ifdef CONFIG_LOCKDEP + static struct lock_class_key ovl_i_mutex_key[OVL_MAX_NESTING]; + static struct lock_class_key ovl_i_mutex_dir_key[OVL_MAX_NESTING]; + static struct lock_class_key ovl_i_lock_key[OVL_MAX_NESTING]; + + int depth = inode->i_sb->s_stack_depth - 1; + + if (WARN_ON_ONCE(depth < 0 || depth >= OVL_MAX_NESTING)) + depth = 0; + + if (S_ISDIR(inode->i_mode)) + lockdep_set_class(&inode->i_rwsem, &ovl_i_mutex_dir_key[depth]); + else + lockdep_set_class(&inode->i_rwsem, &ovl_i_mutex_key[depth]); + + lockdep_set_class(&OVL_I(inode)->lock, &ovl_i_lock_key[depth]); +#endif +} + +static void ovl_next_ino(struct inode *inode) +{ + struct ovl_fs *ofs = OVL_FS(inode->i_sb); + + inode->i_ino = atomic_long_inc_return(&ofs->last_ino); + if (unlikely(!inode->i_ino)) + inode->i_ino = atomic_long_inc_return(&ofs->last_ino); +} + +static void ovl_map_ino(struct inode *inode, unsigned long ino, int fsid) +{ + struct ovl_fs *ofs = OVL_FS(inode->i_sb); + int xinobits = ovl_xino_bits(ofs); + unsigned int xinoshift = 64 - xinobits; + + /* + * When d_ino is consistent with st_ino (samefs or i_ino has enough + * bits to encode layer), set the same value used for st_ino to i_ino, + * so inode number exposed via /proc/locks and a like will be + * consistent with d_ino and st_ino values. An i_ino value inconsistent + * with d_ino also causes nfsd readdirplus to fail. + */ + inode->i_ino = ino; + if (ovl_same_fs(ofs)) { + return; + } else if (xinobits && likely(!(ino >> xinoshift))) { + inode->i_ino |= (unsigned long)fsid << (xinoshift + 1); + return; + } + + /* + * For directory inodes on non-samefs with xino disabled or xino + * overflow, we allocate a non-persistent inode number, to be used for + * resolving st_ino collisions in ovl_map_dev_ino(). + * + * To avoid ino collision with legitimate xino values from upper + * layer (fsid 0), use the lowest xinobit to map the non + * persistent inode numbers to the unified st_ino address space. + */ + if (S_ISDIR(inode->i_mode)) { + ovl_next_ino(inode); + if (xinobits) { + inode->i_ino &= ~0UL >> xinobits; + inode->i_ino |= 1UL << xinoshift; + } + } +} + +void ovl_inode_init(struct inode *inode, struct ovl_inode_params *oip, + unsigned long ino, int fsid) +{ + struct inode *realinode; + struct ovl_inode *oi = OVL_I(inode); + + oi->__upperdentry = oip->upperdentry; + oi->oe = oip->oe; + oi->redirect = oip->redirect; + oi->lowerdata_redirect = oip->lowerdata_redirect; + + realinode = ovl_inode_real(inode); + ovl_copyattr(inode); + ovl_copyflags(realinode, inode); + ovl_map_ino(inode, ino, fsid); +} + +static void ovl_fill_inode(struct inode *inode, umode_t mode, dev_t rdev) +{ + inode->i_mode = mode; + inode->i_flags |= S_NOCMTIME; +#ifdef CONFIG_FS_POSIX_ACL + inode->i_acl = inode->i_default_acl = ACL_DONT_CACHE; +#endif + + ovl_lockdep_annotate_inode_mutex_key(inode); + + switch (mode & S_IFMT) { + case S_IFREG: + inode->i_op = &ovl_file_inode_operations; + inode->i_fop = &ovl_file_operations; + inode->i_mapping->a_ops = &ovl_aops; + break; + + case S_IFDIR: + inode->i_op = &ovl_dir_inode_operations; + inode->i_fop = &ovl_dir_operations; + break; + + case S_IFLNK: + inode->i_op = &ovl_symlink_inode_operations; + break; + + default: + inode->i_op = &ovl_special_inode_operations; + init_special_inode(inode, mode, rdev); + break; + } +} + +/* + * With inodes index enabled, an overlay inode nlink counts the union of upper + * hardlinks and non-covered lower hardlinks. During the lifetime of a non-pure + * upper inode, the following nlink modifying operations can happen: + * + * 1. Lower hardlink copy up + * 2. Upper hardlink created, unlinked or renamed over + * 3. Lower hardlink whiteout or renamed over + * + * For the first, copy up case, the union nlink does not change, whether the + * operation succeeds or fails, but the upper inode nlink may change. + * Therefore, before copy up, we store the union nlink value relative to the + * lower inode nlink in the index inode xattr .overlay.nlink. + * + * For the second, upper hardlink case, the union nlink should be incremented + * or decremented IFF the operation succeeds, aligned with nlink change of the + * upper inode. Therefore, before link/unlink/rename, we store the union nlink + * value relative to the upper inode nlink in the index inode. + * + * For the last, lower cover up case, we simplify things by preceding the + * whiteout or cover up with copy up. This makes sure that there is an index + * upper inode where the nlink xattr can be stored before the copied up upper + * entry is unlink. + */ +#define OVL_NLINK_ADD_UPPER (1 << 0) + +/* + * On-disk format for indexed nlink: + * + * nlink relative to the upper inode - "U[+-]NUM" + * nlink relative to the lower inode - "L[+-]NUM" + */ + +static int ovl_set_nlink_common(struct dentry *dentry, + struct dentry *realdentry, const char *format) +{ + struct inode *inode = d_inode(dentry); + struct inode *realinode = d_inode(realdentry); + char buf[13]; + int len; + + len = snprintf(buf, sizeof(buf), format, + (int) (inode->i_nlink - realinode->i_nlink)); + + if (WARN_ON(len >= sizeof(buf))) + return -EIO; + + return ovl_setxattr(OVL_FS(inode->i_sb), ovl_dentry_upper(dentry), + OVL_XATTR_NLINK, buf, len); +} + +int ovl_set_nlink_upper(struct dentry *dentry) +{ + return ovl_set_nlink_common(dentry, ovl_dentry_upper(dentry), "U%+i"); +} + +int ovl_set_nlink_lower(struct dentry *dentry) +{ + return ovl_set_nlink_common(dentry, ovl_dentry_lower(dentry), "L%+i"); +} + +unsigned int ovl_get_nlink(struct ovl_fs *ofs, struct dentry *lowerdentry, + struct dentry *upperdentry, + unsigned int fallback) +{ + int nlink_diff; + int nlink; + char buf[13]; + int err; + + if (!lowerdentry || !upperdentry || d_inode(lowerdentry)->i_nlink == 1) + return fallback; + + err = ovl_getxattr_upper(ofs, upperdentry, OVL_XATTR_NLINK, + &buf, sizeof(buf) - 1); + if (err < 0) + goto fail; + + buf[err] = '\0'; + if ((buf[0] != 'L' && buf[0] != 'U') || + (buf[1] != '+' && buf[1] != '-')) + goto fail; + + err = kstrtoint(buf + 1, 10, &nlink_diff); + if (err < 0) + goto fail; + + nlink = d_inode(buf[0] == 'L' ? lowerdentry : upperdentry)->i_nlink; + nlink += nlink_diff; + + if (nlink <= 0) + goto fail; + + return nlink; + +fail: + pr_warn_ratelimited("failed to get index nlink (%pd2, err=%i)\n", + upperdentry, err); + return fallback; +} + +struct inode *ovl_new_inode(struct super_block *sb, umode_t mode, dev_t rdev) +{ + struct inode *inode; + + inode = new_inode(sb); + if (inode) + ovl_fill_inode(inode, mode, rdev); + + return inode; +} + +static int ovl_inode_test(struct inode *inode, void *data) +{ + return inode->i_private == data; +} + +static int ovl_inode_set(struct inode *inode, void *data) +{ + inode->i_private = data; + return 0; +} + +static bool ovl_verify_inode(struct inode *inode, struct dentry *lowerdentry, + struct dentry *upperdentry, bool strict) +{ + /* + * For directories, @strict verify from lookup path performs consistency + * checks, so NULL lower/upper in dentry must match NULL lower/upper in + * inode. Non @strict verify from NFS handle decode path passes NULL for + * 'unknown' lower/upper. + */ + if (S_ISDIR(inode->i_mode) && strict) { + /* Real lower dir moved to upper layer under us? */ + if (!lowerdentry && ovl_inode_lower(inode)) + return false; + + /* Lookup of an uncovered redirect origin? */ + if (!upperdentry && ovl_inode_upper(inode)) + return false; + } + + /* + * Allow non-NULL lower inode in ovl_inode even if lowerdentry is NULL. + * This happens when finding a copied up overlay inode for a renamed + * or hardlinked overlay dentry and lower dentry cannot be followed + * by origin because lower fs does not support file handles. + */ + if (lowerdentry && ovl_inode_lower(inode) != d_inode(lowerdentry)) + return false; + + /* + * Allow non-NULL __upperdentry in inode even if upperdentry is NULL. + * This happens when finding a lower alias for a copied up hard link. + */ + if (upperdentry && ovl_inode_upper(inode) != d_inode(upperdentry)) + return false; + + return true; +} + +struct inode *ovl_lookup_inode(struct super_block *sb, struct dentry *real, + bool is_upper) +{ + struct inode *inode, *key = d_inode(real); + + inode = ilookup5(sb, (unsigned long) key, ovl_inode_test, key); + if (!inode) + return NULL; + + if (!ovl_verify_inode(inode, is_upper ? NULL : real, + is_upper ? real : NULL, false)) { + iput(inode); + return ERR_PTR(-ESTALE); + } + + return inode; +} + +bool ovl_lookup_trap_inode(struct super_block *sb, struct dentry *dir) +{ + struct inode *key = d_inode(dir); + struct inode *trap; + bool res; + + trap = ilookup5(sb, (unsigned long) key, ovl_inode_test, key); + if (!trap) + return false; + + res = IS_DEADDIR(trap) && !ovl_inode_upper(trap) && + !ovl_inode_lower(trap); + + iput(trap); + return res; +} + +/* + * Create an inode cache entry for layer root dir, that will intentionally + * fail ovl_verify_inode(), so any lookup that will find some layer root + * will fail. + */ +struct inode *ovl_get_trap_inode(struct super_block *sb, struct dentry *dir) +{ + struct inode *key = d_inode(dir); + struct inode *trap; + + if (!d_is_dir(dir)) + return ERR_PTR(-ENOTDIR); + + trap = iget5_locked(sb, (unsigned long) key, ovl_inode_test, + ovl_inode_set, key); + if (!trap) + return ERR_PTR(-ENOMEM); + + if (!(inode_state_read_once(trap) & I_NEW)) { + /* Conflicting layer roots? */ + iput(trap); + return ERR_PTR(-ELOOP); + } + + trap->i_mode = S_IFDIR; + trap->i_flags = S_DEAD; + unlock_new_inode(trap); + + return trap; +} + +/* + * Does overlay inode need to be hashed by lower inode? + */ +static bool ovl_hash_bylower(struct super_block *sb, struct dentry *upper, + struct dentry *lower, bool index) +{ + struct ovl_fs *ofs = OVL_FS(sb); + + /* No, if pure upper */ + if (!lower) + return false; + + /* Yes, if already indexed */ + if (index) + return true; + + /* Yes, if won't be copied up */ + if (!ovl_upper_mnt(ofs)) + return true; + + /* No, if lower hardlink is or will be broken on copy up */ + if ((upper || !ovl_indexdir(sb)) && + !d_is_dir(lower) && d_inode(lower)->i_nlink > 1) + return false; + + /* No, if non-indexed upper with NFS export */ + if (ofs->config.nfs_export && upper) + return false; + + /* Otherwise, hash by lower inode for fsnotify */ + return true; +} + +static struct inode *ovl_iget5(struct super_block *sb, struct inode *newinode, + struct inode *key) +{ + return newinode ? inode_insert5(newinode, (unsigned long) key, + ovl_inode_test, ovl_inode_set, key) : + iget5_locked(sb, (unsigned long) key, + ovl_inode_test, ovl_inode_set, key); +} + +struct inode *ovl_get_inode(struct super_block *sb, + struct ovl_inode_params *oip) +{ + struct ovl_fs *ofs = OVL_FS(sb); + struct dentry *upperdentry = oip->upperdentry; + struct ovl_path *lowerpath = ovl_lowerpath(oip->oe); + struct inode *realinode = upperdentry ? d_inode(upperdentry) : NULL; + struct inode *inode; + struct dentry *lowerdentry = lowerpath ? lowerpath->dentry : NULL; + struct path realpath = { + .dentry = upperdentry ?: lowerdentry, + .mnt = upperdentry ? ovl_upper_mnt(ofs) : lowerpath->layer->mnt, + }; + bool bylower = ovl_hash_bylower(sb, upperdentry, lowerdentry, + oip->index); + int fsid = bylower ? lowerpath->layer->fsid : 0; + bool is_dir; + unsigned long ino = 0; + int err = oip->newinode ? -EEXIST : -ENOMEM; + + if (!realinode) + realinode = d_inode(lowerdentry); + + /* + * Copy up origin (lower) may exist for non-indexed upper, but we must + * not use lower as hash key if this is a broken hardlink. + */ + is_dir = S_ISDIR(realinode->i_mode); + if (upperdentry || bylower) { + struct inode *key = d_inode(bylower ? lowerdentry : + upperdentry); + unsigned int nlink = is_dir ? 1 : realinode->i_nlink; + + inode = ovl_iget5(sb, oip->newinode, key); + if (!inode) + goto out_err; + if (!(inode_state_read_once(inode) & I_NEW)) { + /* + * Verify that the underlying files stored in the inode + * match those in the dentry. + */ + if (!ovl_verify_inode(inode, lowerdentry, upperdentry, + true)) { + iput(inode); + err = -ESTALE; + goto out_err; + } + + dput(upperdentry); + ovl_free_entry(oip->oe); + kfree(oip->redirect); + kfree(oip->lowerdata_redirect); + goto out; + } + + /* Recalculate nlink for non-dir due to indexing */ + if (!is_dir) + nlink = ovl_get_nlink(ofs, lowerdentry, upperdentry, + nlink); + set_nlink(inode, nlink); + ino = key->i_ino; + } else { + /* Lower hardlink that will be broken on copy up */ + inode = new_inode(sb); + if (!inode) { + err = -ENOMEM; + goto out_err; + } + ino = realinode->i_ino; + fsid = lowerpath->layer->fsid; + } + ovl_fill_inode(inode, realinode->i_mode, realinode->i_rdev); + ovl_inode_init(inode, oip, ino, fsid); + WARN_ON_ONCE(!!IS_CASEFOLDED(inode) != ofs->casefold); + + if (upperdentry && ovl_is_impuredir(sb, upperdentry)) + ovl_set_flag(OVL_IMPURE, inode); + + if (oip->index) + ovl_set_flag(OVL_INDEX, inode); + + if (bylower) + ovl_set_flag(OVL_CONST_INO, inode); + + /* Check for non-merge dir that may have whiteouts */ + if (is_dir) { + if (((upperdentry && lowerdentry) || ovl_numlower(oip->oe) > 1) || + ovl_path_check_origin_xattr(ofs, &realpath)) { + ovl_set_flag(OVL_WHITEOUTS, inode); + } + } + + /* Check for immutable/append-only inode flags in xattr */ + if (upperdentry) + ovl_check_protattr(inode, upperdentry); + + if (inode_state_read_once(inode) & I_NEW) + unlock_new_inode(inode); +out: + return inode; + +out_err: + pr_warn_ratelimited("failed to get inode (%i)\n", err); + inode = ERR_PTR(err); + goto out; +} diff --git a/fs/overlayfs/namei.c b/fs/overlayfs/namei.c new file mode 100644 index 000000000..ca899fdfa --- /dev/null +++ b/fs/overlayfs/namei.c @@ -0,0 +1,1483 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2011 Novell Inc. + * Copyright (C) 2016 Red Hat, Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "overlayfs.h" + +struct ovl_lookup_data { + struct super_block *sb; + struct dentry *dentry; + const struct ovl_layer *layer; + struct qstr name; + bool is_dir; + bool opaque; + bool xwhiteouts; + bool stop; + bool last; + char *redirect; + char *upperredirect; + int metacopy; + /* Referring to last redirect xattr */ + bool absolute_redirect; +}; + +static int ovl_check_redirect(const struct path *path, struct ovl_lookup_data *d, + size_t prelen, const char *post) +{ + int res; + char *buf; + struct ovl_fs *ofs = OVL_FS(d->sb); + + d->absolute_redirect = false; + buf = ovl_get_redirect_xattr(ofs, path, prelen + strlen(post)); + if (IS_ERR_OR_NULL(buf)) + return PTR_ERR(buf); + + if (buf[0] == '/') { + d->absolute_redirect = true; + /* + * One of the ancestor path elements in an absolute path + * lookup in ovl_lookup_layer() could have been opaque and + * that will stop further lookup in lower layers (d->stop=true) + * But we have found an absolute redirect in descendant path + * element and that should force continue lookup in lower + * layers (reset d->stop). + */ + d->stop = false; + } else { + res = strlen(buf) + 1; + memmove(buf + prelen, buf, res); + memcpy(buf, d->name.name, prelen); + } + + strcat(buf, post); + kfree(d->redirect); + d->redirect = buf; + d->name.name = d->redirect; + d->name.len = strlen(d->redirect); + + return 0; +} + +static int ovl_acceptable(void *ctx, struct dentry *dentry) +{ + /* + * A non-dir origin may be disconnected, which is fine, because + * we only need it for its unique inode number. + */ + if (!d_is_dir(dentry)) + return 1; + + /* Don't decode a deleted empty directory */ + if (d_unhashed(dentry)) + return 0; + + /* Check if directory belongs to the layer we are decoding from */ + return is_subdir(dentry, ((struct vfsmount *)ctx)->mnt_root); +} + +/* + * Check validity of an overlay file handle buffer. + * + * Return 0 for a valid file handle. + * Return -ENODATA for "origin unknown". + * Return <0 for an invalid file handle. + */ +int ovl_check_fb_len(struct ovl_fb *fb, int fb_len) +{ + if (fb_len < sizeof(struct ovl_fb) || fb_len < fb->len) + return -EINVAL; + + if (fb->magic != OVL_FH_MAGIC) + return -EINVAL; + + /* Treat larger version and unknown flags as "origin unknown" */ + if (fb->version > OVL_FH_VERSION || fb->flags & ~OVL_FH_FLAG_ALL) + return -ENODATA; + + /* Treat endianness mismatch as "origin unknown" */ + if (!(fb->flags & OVL_FH_FLAG_ANY_ENDIAN) && + (fb->flags & OVL_FH_FLAG_BIG_ENDIAN) != OVL_FH_FLAG_CPU_ENDIAN) + return -ENODATA; + + return 0; +} + +static struct ovl_fh *ovl_get_fh(struct ovl_fs *ofs, struct dentry *upperdentry, + enum ovl_xattr ox) +{ + int res, err; + struct ovl_fh *fh = NULL; + + res = ovl_getxattr_upper(ofs, upperdentry, ox, NULL, 0); + if (res < 0) { + if (res == -ENODATA || res == -EOPNOTSUPP) + return NULL; + goto fail; + } + /* Zero size value means "copied up but origin unknown" */ + if (res == 0) + return NULL; + + fh = kzalloc(res + OVL_FH_WIRE_OFFSET, GFP_KERNEL); + if (!fh) + return ERR_PTR(-ENOMEM); + + res = ovl_getxattr_upper(ofs, upperdentry, ox, fh->buf, res); + if (res < 0) + goto fail; + + err = ovl_check_fb_len(&fh->fb, res); + if (err < 0) { + if (err == -ENODATA) + goto out; + goto invalid; + } + + return fh; + +out: + kfree(fh); + return NULL; + +fail: + pr_warn_ratelimited("failed to get origin (%i)\n", res); + goto out; +invalid: + pr_warn_ratelimited("invalid origin (%*phN)\n", res, fh); + goto out; +} + +bool ovl_uuid_match(struct ovl_fs *ofs, const struct super_block *sb, + const uuid_t *uuid) +{ + /* + * Make sure that the stored uuid matches the uuid of the lower + * layer where file handle will be decoded. + * In case of uuid=off option just make sure that stored uuid is null. + */ + return ovl_origin_uuid(ofs) ? uuid_equal(uuid, &sb->s_uuid) : + uuid_is_null(uuid); +} + +struct dentry *ovl_decode_real_fh(struct ovl_fs *ofs, struct ovl_fh *fh, + struct vfsmount *mnt, bool connected) +{ + struct dentry *real; + int bytes; + + if (!capable(CAP_DAC_READ_SEARCH)) + return NULL; + + if (!ovl_uuid_match(ofs, mnt->mnt_sb, &fh->fb.uuid)) + return NULL; + + bytes = (fh->fb.len - offsetof(struct ovl_fb, fid)); + real = exportfs_decode_fh(mnt, (struct fid *)fh->fb.fid, + bytes >> 2, (int)fh->fb.type, + connected ? ovl_acceptable : NULL, mnt); + if (IS_ERR(real)) { + /* + * Treat stale file handle to lower file as "origin unknown". + * upper file handle could become stale when upper file is + * unlinked and this information is needed to handle stale + * index entries correctly. + */ + if (real == ERR_PTR(-ESTALE) && + !(fh->fb.flags & OVL_FH_FLAG_PATH_UPPER)) + real = NULL; + return real; + } + + if (ovl_dentry_weird(real)) { + dput(real); + return NULL; + } + + return real; +} + +static struct dentry *ovl_lookup_positive_unlocked(struct ovl_lookup_data *d, + const char *name, + struct dentry *base, int len, + bool drop_negative) +{ + struct dentry *ret = lookup_one_unlocked(mnt_idmap(d->layer->mnt), + &QSTR_LEN(name, len), base); + + if (!IS_ERR(ret) && d_flags_negative(smp_load_acquire(&ret->d_flags))) { + if (drop_negative && ret->d_lockref.count == 1) { + spin_lock(&ret->d_lock); + /* Recheck condition under lock */ + if (d_is_negative(ret) && ret->d_lockref.count == 1) + __d_drop(ret); + spin_unlock(&ret->d_lock); + } + dput(ret); + ret = ERR_PTR(-ENOENT); + } + return ret; +} + +static int ovl_lookup_single(struct dentry *base, struct ovl_lookup_data *d, + const char *name, unsigned int namelen, + size_t prelen, const char *post, + struct dentry **ret, bool drop_negative) +{ + struct ovl_fs *ofs = OVL_FS(d->sb); + struct dentry *this = NULL; + const char *warn; + struct path path; + int err; + bool last_element = !post[0]; + bool is_upper = d->layer->idx == 0; + char val; + + /* + * We allow filesystems that are case-folding capable as long as the + * layers are consistently enabled in the stack, enabled for every dir + * or disabled in all dirs. If someone has modified case folding on a + * directory on underlying layer, the warranty of the ovl stack is + * voided. + */ + if (ofs->casefold != ovl_dentry_casefolded(base)) { + warn = "parent wrong casefold"; + err = -ESTALE; + goto out_warn; + } + + this = ovl_lookup_positive_unlocked(d, name, base, namelen, drop_negative); + if (IS_ERR(this)) { + err = PTR_ERR(this); + this = NULL; + if (err == -ENOENT || err == -ENAMETOOLONG) + goto out; + goto out_err; + } + + if (ofs->casefold != ovl_dentry_casefolded(this)) { + warn = "child wrong casefold"; + err = -EREMOTE; + goto out_warn; + } + + if (ovl_dentry_weird(this)) { + /* Don't support traversing automounts and other weirdness */ + warn = "unsupported object type"; + err = -EREMOTE; + goto out_warn; + } + + path.dentry = this; + path.mnt = d->layer->mnt; + if (ovl_path_is_whiteout(ofs, &path)) { + d->stop = d->opaque = true; + goto put_and_out; + } + /* + * This dentry should be a regular file if previous layer lookup + * found a metacopy dentry. + */ + if (last_element && d->metacopy && !d_is_reg(this)) { + d->stop = true; + goto put_and_out; + } + + if (!d_can_lookup(this)) { + if (d->is_dir || !last_element) { + d->stop = true; + goto put_and_out; + } + err = ovl_check_metacopy_xattr(ofs, &path, NULL); + if (err < 0) + goto out_err; + + d->metacopy = err; + d->stop = !d->metacopy; + if (!d->metacopy || d->last) + goto out; + } else { + if (ovl_lookup_trap_inode(d->sb, this)) { + /* Caught in a trap of overlapping layers */ + warn = "overlapping layers"; + err = -ELOOP; + goto out_warn; + } + + if (last_element) + d->is_dir = true; + if (d->last) + goto out; + + /* overlay.opaque=x means xwhiteouts directory */ + val = ovl_get_opaquedir_val(ofs, &path); + if (last_element && !is_upper && val == 'x') { + d->xwhiteouts = true; + ovl_layer_set_xwhiteouts(ofs, d->layer); + } else if (val == 'y') { + d->stop = true; + if (last_element) + d->opaque = true; + goto out; + } + } + err = ovl_check_redirect(&path, d, prelen, post); + if (err) + goto out_err; +out: + *ret = this; + return 0; + +put_and_out: + dput(this); + this = NULL; + goto out; + +out_warn: + pr_warn_ratelimited("failed lookup in %s (%pd2, name='%.*s', err=%i): %s\n", + is_upper ? "upper" : "lower", base, + namelen, name, err, warn); +out_err: + dput(this); + return err; +} + +static int ovl_lookup_layer(struct dentry *base, struct ovl_lookup_data *d, + struct dentry **ret, bool drop_negative) +{ + /* Counting down from the end, since the prefix can change */ + size_t rem = d->name.len - 1; + struct dentry *dentry = NULL; + int err; + + if (d->name.name[0] != '/') + return ovl_lookup_single(base, d, d->name.name, d->name.len, + 0, "", ret, drop_negative); + + while (!IS_ERR_OR_NULL(base) && d_can_lookup(base)) { + const char *s = d->name.name + d->name.len - rem; + const char *next = strchrnul(s, '/'); + size_t thislen = next - s; + bool end = !next[0]; + + /* Verify we did not go off the rails */ + if (WARN_ON(s[-1] != '/')) + return -EIO; + + err = ovl_lookup_single(base, d, s, thislen, + d->name.len - rem, next, &base, + drop_negative); + dput(dentry); + if (err) + return err; + dentry = base; + if (end) + break; + + rem -= thislen + 1; + + if (WARN_ON(rem >= d->name.len)) + return -EIO; + } + *ret = dentry; + return 0; +} + +static int ovl_lookup_data_layer(struct dentry *dentry, const char *redirect, + const struct ovl_layer *layer, + struct path *datapath) +{ + int err; + + err = vfs_path_lookup(layer->mnt->mnt_root, layer->mnt, redirect, + LOOKUP_BENEATH | LOOKUP_NO_SYMLINKS | LOOKUP_NO_XDEV, + datapath); + pr_debug("lookup lowerdata (%pd2, redirect=\"%s\", layer=%d, err=%i)\n", + dentry, redirect, layer->idx, err); + + if (err) + return err; + + err = -EREMOTE; + if (ovl_dentry_weird(datapath->dentry)) + goto out_path_put; + + err = -ENOENT; + /* Only regular file is acceptable as lower data */ + if (!d_is_reg(datapath->dentry)) + goto out_path_put; + + return 0; + +out_path_put: + path_put(datapath); + + return err; +} + +/* Lookup in data-only layers by absolute redirect to layer root */ +static int ovl_lookup_data_layers(struct dentry *dentry, const char *redirect, + struct ovl_path *lowerdata) +{ + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + const struct ovl_layer *layer; + struct path datapath; + int err = -ENOENT; + int i; + + layer = &ofs->layers[ofs->numlayer - ofs->numdatalayer]; + for (i = 0; i < ofs->numdatalayer; i++, layer++) { + err = ovl_lookup_data_layer(dentry, redirect, layer, &datapath); + if (!err) { + mntput(datapath.mnt); + lowerdata->dentry = datapath.dentry; + lowerdata->layer = layer; + return 0; + } + } + + return err; +} + +int ovl_check_origin_fh(struct ovl_fs *ofs, struct ovl_fh *fh, bool connected, + struct dentry *upperdentry, struct ovl_path **stackp) +{ + struct dentry *origin = NULL; + int i; + + for (i = 1; i <= ovl_numlowerlayer(ofs); i++) { + /* + * If lower fs uuid is not unique among lower fs we cannot match + * fh->uuid to layer. + */ + if (ofs->layers[i].fsid && + ofs->layers[i].fs->bad_uuid) + continue; + + origin = ovl_decode_real_fh(ofs, fh, ofs->layers[i].mnt, + connected); + if (origin) + break; + } + + if (!origin) + return -ESTALE; + else if (IS_ERR(origin)) + return PTR_ERR(origin); + + if (upperdentry && !ovl_upper_is_whiteout(ofs, upperdentry) && + inode_wrong_type(d_inode(upperdentry), d_inode(origin)->i_mode)) + goto invalid; + + if (!*stackp) + *stackp = kmalloc_obj(struct ovl_path); + if (!*stackp) { + dput(origin); + return -ENOMEM; + } + **stackp = (struct ovl_path){ + .dentry = origin, + .layer = &ofs->layers[i] + }; + + return 0; + +invalid: + pr_warn_ratelimited("invalid origin (%pd2, ftype=%x, origin ftype=%x).\n", + upperdentry, d_inode(upperdentry)->i_mode & S_IFMT, + d_inode(origin)->i_mode & S_IFMT); + dput(origin); + return -ESTALE; +} + +static int ovl_check_origin(struct ovl_fs *ofs, struct dentry *upperdentry, + struct ovl_path **stackp) +{ + struct ovl_fh *fh = ovl_get_fh(ofs, upperdentry, OVL_XATTR_ORIGIN); + int err; + + if (IS_ERR_OR_NULL(fh)) + return PTR_ERR(fh); + + err = ovl_check_origin_fh(ofs, fh, false, upperdentry, stackp); + kfree(fh); + + if (err) { + if (err == -ESTALE) + return 0; + return err; + } + + return 0; +} + +/* + * Verify that @fh matches the file handle stored in xattr @name. + * Return 0 on match, -ESTALE on mismatch, < 0 on error. + */ +static int ovl_verify_fh(struct ovl_fs *ofs, struct dentry *dentry, + enum ovl_xattr ox, const struct ovl_fh *fh) +{ + struct ovl_fh *ofh = ovl_get_fh(ofs, dentry, ox); + int err = 0; + + if (!ofh) + return -ENODATA; + + if (IS_ERR(ofh)) + return PTR_ERR(ofh); + + if (fh->fb.len != ofh->fb.len || memcmp(&fh->fb, &ofh->fb, fh->fb.len)) + err = -ESTALE; + + kfree(ofh); + return err; +} + +int ovl_verify_set_fh(struct ovl_fs *ofs, struct dentry *dentry, + enum ovl_xattr ox, const struct ovl_fh *fh, + bool is_upper, bool set) +{ + int err; + + err = ovl_verify_fh(ofs, dentry, ox, fh); + if (set && err == -ENODATA) + err = ovl_setxattr(ofs, dentry, ox, fh->buf, fh->fb.len); + + return err; +} + +/* + * Verify that @real dentry matches the file handle stored in xattr @name. + * + * If @set is true and there is no stored file handle, encode @real and store + * file handle in xattr @name. + * + * Return 0 on match, -ESTALE on mismatch, -ENODATA on no xattr, < 0 on error. + */ +int ovl_verify_origin_xattr(struct ovl_fs *ofs, struct dentry *dentry, + enum ovl_xattr ox, struct dentry *real, + bool is_upper, bool set) +{ + struct inode *inode; + struct ovl_fh *fh; + int err; + + fh = ovl_encode_real_fh(ofs, d_inode(real), is_upper); + err = PTR_ERR(fh); + if (IS_ERR(fh)) { + fh = NULL; + goto fail; + } + + err = ovl_verify_set_fh(ofs, dentry, ox, fh, is_upper, set); + if (err) + goto fail; + +out: + kfree(fh); + return err; + +fail: + inode = d_inode(real); + pr_warn_ratelimited("failed to verify %s (%pd2, ino=%llu, err=%i)\n", + is_upper ? "upper" : "origin", real, + inode ? inode->i_ino : 0, err); + goto out; +} + + +/* Get upper dentry from index */ +struct dentry *ovl_index_upper(struct ovl_fs *ofs, struct dentry *index, + bool connected) +{ + struct ovl_fh *fh; + struct dentry *upper; + + if (!d_is_dir(index)) + return dget(index); + + fh = ovl_get_fh(ofs, index, OVL_XATTR_UPPER); + if (IS_ERR_OR_NULL(fh)) + return ERR_CAST(fh); + + upper = ovl_decode_real_fh(ofs, fh, ovl_upper_mnt(ofs), connected); + kfree(fh); + + if (IS_ERR_OR_NULL(upper)) + return upper ?: ERR_PTR(-ESTALE); + + if (!d_is_dir(upper)) { + pr_warn_ratelimited("invalid index upper (%pd2, upper=%pd2).\n", + index, upper); + dput(upper); + return ERR_PTR(-EIO); + } + + return upper; +} + +/* + * Verify that an index entry name matches the origin file handle stored in + * OVL_XATTR_ORIGIN and that origin file handle can be decoded to lower path. + * Return 0 on match, -ESTALE on mismatch or stale origin, < 0 on error. + */ +int ovl_verify_index(struct ovl_fs *ofs, struct dentry *index) +{ + struct ovl_fh *fh = NULL; + size_t len; + struct ovl_path origin = { }; + struct ovl_path *stack = &origin; + struct dentry *upper = NULL; + int err; + + if (!d_inode(index)) + return 0; + + err = -EINVAL; + if (index->d_name.len < sizeof(struct ovl_fb)*2) + goto fail; + + err = -ENOMEM; + len = index->d_name.len / 2; + fh = kzalloc(len + OVL_FH_WIRE_OFFSET, GFP_KERNEL); + if (!fh) + goto fail; + + err = -EINVAL; + if (hex2bin(fh->buf, index->d_name.name, len)) + goto fail; + + err = ovl_check_fb_len(&fh->fb, len); + if (err) + goto fail; + + /* + * Whiteout index entries are used as an indication that an exported + * overlay file handle should be treated as stale (i.e. after unlink + * of the overlay inode). These entries contain no origin xattr. + */ + if (ovl_is_whiteout(index)) + goto out; + + /* + * Verifying directory index entries are not stale is expensive, so + * only verify stale dir index if NFS export is enabled. + */ + if (d_is_dir(index) && !ofs->config.nfs_export) + goto out; + + /* + * Directory index entries should have 'upper' xattr pointing to the + * real upper dir. Non-dir index entries are hardlinks to the upper + * real inode. For non-dir index, we can read the copy up origin xattr + * directly from the index dentry, but for dir index we first need to + * decode the upper directory. + */ + upper = ovl_index_upper(ofs, index, false); + if (IS_ERR_OR_NULL(upper)) { + err = PTR_ERR(upper); + /* + * Directory index entries with no 'upper' xattr need to be + * removed. When dir index entry has a stale 'upper' xattr, + * we assume that upper dir was removed and we treat the dir + * index as orphan entry that needs to be whited out. + */ + if (err == -ESTALE) + goto orphan; + else if (!err) + err = -ESTALE; + goto fail; + } + + err = ovl_verify_fh(ofs, upper, OVL_XATTR_ORIGIN, fh); + dput(upper); + if (err) + goto fail; + + /* Check if non-dir index is orphan and don't warn before cleaning it */ + if (!d_is_dir(index) && d_inode(index)->i_nlink == 1) { + err = ovl_check_origin_fh(ofs, fh, false, index, &stack); + if (err) + goto fail; + + if (ovl_get_nlink(ofs, origin.dentry, index, 0) == 0) + goto orphan; + } + +out: + dput(origin.dentry); + kfree(fh); + return err; + +fail: + pr_warn_ratelimited("failed to verify index (%pd2, ftype=%x, err=%i)\n", + index, d_inode(index)->i_mode & S_IFMT, err); + goto out; + +orphan: + pr_warn_ratelimited("orphan index entry (%pd2, ftype=%x, nlink=%u)\n", + index, d_inode(index)->i_mode & S_IFMT, + d_inode(index)->i_nlink); + err = -ENOENT; + goto out; +} + +int ovl_get_index_name_fh(const struct ovl_fh *fh, struct qstr *name) +{ + char *n, *s; + + n = kcalloc(fh->fb.len, 2, GFP_KERNEL); + if (!n) + return -ENOMEM; + + s = bin2hex(n, fh->buf, fh->fb.len); + *name = (struct qstr) QSTR_INIT(n, s - n); + + return 0; + +} + +/* + * Lookup in indexdir for the index entry of a lower real inode or a copy up + * origin inode. The index entry name is the hex representation of the lower + * inode file handle. + * + * If the index dentry in negative, then either no lower aliases have been + * copied up yet, or aliases have been copied up in older kernels and are + * not indexed. + * + * If the index dentry for a copy up origin inode is positive, but points + * to an inode different than the upper inode, then either the upper inode + * has been copied up and not indexed or it was indexed, but since then + * index dir was cleared. Either way, that index cannot be used to identify + * the overlay inode. + */ +int ovl_get_index_name(struct ovl_fs *ofs, struct dentry *origin, + struct qstr *name) +{ + struct ovl_fh *fh; + int err; + + fh = ovl_encode_real_fh(ofs, d_inode(origin), false); + if (IS_ERR(fh)) + return PTR_ERR(fh); + + err = ovl_get_index_name_fh(fh, name); + + kfree(fh); + return err; +} + +/* Lookup index by file handle for NFS export */ +struct dentry *ovl_get_index_fh(struct ovl_fs *ofs, struct ovl_fh *fh) +{ + struct dentry *index; + struct qstr name; + int err; + + err = ovl_get_index_name_fh(fh, &name); + if (err) + return ERR_PTR(err); + + index = lookup_noperm_positive_unlocked(&name, ofs->workdir); + kfree(name.name); + if (IS_ERR(index)) { + if (PTR_ERR(index) == -ENOENT) + index = NULL; + return index; + } + + if (ovl_is_whiteout(index)) + err = -ESTALE; + else if (ovl_dentry_weird(index)) + err = -EIO; + else + return index; + + dput(index); + return ERR_PTR(err); +} + +struct dentry *ovl_lookup_index(struct ovl_fs *ofs, struct dentry *upper, + struct dentry *origin, bool verify) +{ + struct dentry *index; + struct inode *inode; + struct qstr name; + bool is_dir = d_is_dir(origin); + int err; + + err = ovl_get_index_name(ofs, origin, &name); + if (err) + return ERR_PTR(err); + + index = lookup_one_positive_unlocked(ovl_upper_mnt_idmap(ofs), &name, + ofs->workdir); + if (IS_ERR(index)) { + err = PTR_ERR(index); + if (err == -ENOENT) { + index = NULL; + goto out; + } + pr_warn_ratelimited("failed inode index lookup (ino=%llu, key=%.*s, err=%i);\n" + "overlayfs: mount with '-o index=off' to disable inodes index.\n", + d_inode(origin)->i_ino, name.len, name.name, + err); + goto out; + } + + inode = d_inode(index); + if (ovl_is_whiteout(index) && !verify) { + /* + * When index lookup is called with !verify for decoding an + * overlay file handle, a whiteout index implies that decode + * should treat file handle as stale and no need to print a + * warning about it. + */ + dput(index); + index = ERR_PTR(-ESTALE); + goto out; + } else if (ovl_dentry_weird(index) || ovl_is_whiteout(index) || + inode_wrong_type(inode, d_inode(origin)->i_mode)) { + /* + * Index should always be of the same file type as origin + * except for the case of a whiteout index. A whiteout + * index should only exist if all lower aliases have been + * unlinked, which means that finding a lower origin on lookup + * whose index is a whiteout should be treated as an error. + */ + pr_warn_ratelimited("bad index found (index=%pd2, ftype=%x, origin ftype=%x).\n", + index, d_inode(index)->i_mode & S_IFMT, + d_inode(origin)->i_mode & S_IFMT); + goto fail; + } else if (is_dir && verify) { + if (!upper) { + pr_warn_ratelimited("suspected uncovered redirected dir found (origin=%pd2, index=%pd2).\n", + origin, index); + goto fail; + } + + /* Verify that dir index 'upper' xattr points to upper dir */ + err = ovl_verify_upper(ofs, index, upper, false); + if (err) { + if (err == -ESTALE) { + pr_warn_ratelimited("suspected multiply redirected dir found (upper=%pd2, origin=%pd2, index=%pd2).\n", + upper, origin, index); + } + goto fail; + } + } else if (upper && d_inode(upper) != inode) { + goto out_dput; + } +out: + kfree(name.name); + return index; + +out_dput: + dput(index); + index = NULL; + goto out; + +fail: + dput(index); + index = ERR_PTR(-EIO); + goto out; +} + +/* + * Returns next layer in stack starting from top. + * Returns -1 if this is the last layer. + */ +int ovl_path_next(int idx, struct dentry *dentry, struct path *path, + const struct ovl_layer **layer) +{ + struct ovl_entry *oe = OVL_E(dentry); + struct ovl_path *lowerstack = ovl_lowerstack(oe); + + BUG_ON(idx < 0); + if (idx == 0) { + ovl_path_upper(dentry, path); + if (path->dentry) { + *layer = &OVL_FS(dentry->d_sb)->layers[0]; + return ovl_numlower(oe) ? 1 : -1; + } + idx++; + } + BUG_ON(idx > ovl_numlower(oe)); + path->dentry = lowerstack[idx - 1].dentry; + *layer = lowerstack[idx - 1].layer; + path->mnt = (*layer)->mnt; + + return (idx < ovl_numlower(oe)) ? idx + 1 : -1; +} + +/* Fix missing 'origin' xattr */ +static int ovl_fix_origin(struct ovl_fs *ofs, struct dentry *dentry, + struct dentry *lower, struct dentry *upper) +{ + const struct ovl_fh *fh; + int err; + + if (ovl_check_origin_xattr(ofs, upper)) + return 0; + + fh = ovl_get_origin_fh(ofs, lower); + if (IS_ERR(fh)) + return PTR_ERR(fh); + + err = ovl_want_write(dentry); + if (err) + goto out; + + err = ovl_set_origin_fh(ofs, fh, upper); + if (!err) + err = ovl_set_impure(dentry->d_parent, upper->d_parent); + + ovl_drop_write(dentry); +out: + kfree(fh); + return err; +} + +static int ovl_maybe_validate_verity(struct dentry *dentry) +{ + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + struct inode *inode = d_inode(dentry); + struct path datapath, metapath; + int err; + + if (!ofs->config.verity_mode || + !ovl_is_metacopy_dentry(dentry) || + ovl_test_flag(OVL_VERIFIED_DIGEST, inode)) + return 0; + + if (!ovl_test_flag(OVL_HAS_DIGEST, inode)) { + if (ofs->config.verity_mode == OVL_VERITY_REQUIRE) { + pr_warn_ratelimited("metacopy file '%pd' has no digest specified\n", + dentry); + return -EIO; + } + return 0; + } + + ovl_path_lowerdata(dentry, &datapath); + if (!datapath.dentry) + return -EIO; + + ovl_path_real(dentry, &metapath); + if (!metapath.dentry) + return -EIO; + + err = ovl_inode_lock_interruptible(inode); + if (err) + return err; + + if (!ovl_test_flag(OVL_VERIFIED_DIGEST, inode)) { + with_ovl_creds(dentry->d_sb) + err = ovl_validate_verity(ofs, &metapath, &datapath); + if (err == 0) + ovl_set_flag(OVL_VERIFIED_DIGEST, inode); + } + + ovl_inode_unlock(inode); + + return err; +} + +/* Lazy lookup of lowerdata */ +static int ovl_maybe_lookup_lowerdata(struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + const char *redirect = ovl_lowerdata_redirect(inode); + struct ovl_path datapath = {}; + int err; + + if (!redirect || ovl_dentry_lowerdata(dentry)) + return 0; + + if (redirect[0] != '/') + return -EIO; + + err = ovl_inode_lock_interruptible(inode); + if (err) + return err; + + err = 0; + /* Someone got here before us? */ + if (ovl_dentry_lowerdata(dentry)) + goto out; + + with_ovl_creds(dentry->d_sb) + err = ovl_lookup_data_layers(dentry, redirect, &datapath); + if (err) + goto out_err; + + err = ovl_dentry_set_lowerdata(dentry, &datapath); + if (err) + goto out_err; + +out: + ovl_inode_unlock(inode); + dput(datapath.dentry); + + return err; + +out_err: + pr_warn_ratelimited("lazy lowerdata lookup failed (%pd2, err=%i)\n", + dentry, err); + goto out; +} + +int ovl_verify_lowerdata(struct dentry *dentry) +{ + int err; + + err = ovl_maybe_lookup_lowerdata(dentry); + if (err) + return err; + + return ovl_maybe_validate_verity(dentry); +} + +/* + * Following redirects/metacopy can have security consequences: it's like a + * symlink into the lower layer without the permission checks. + * + * This is only a problem if the upper layer is untrusted (e.g comes from an USB + * drive). This can allow a non-readable file or directory to become readable. + * + * Only following redirects when redirects are enabled disables this attack + * vector when not necessary. + */ +static bool ovl_check_follow_redirect(struct ovl_lookup_data *d) +{ + struct ovl_fs *ofs = OVL_FS(d->sb); + + if (d->metacopy && !ofs->config.metacopy) { + pr_warn_ratelimited("refusing to follow metacopy origin for (%pd2)\n", d->dentry); + return false; + } + if ((d->redirect || d->upperredirect) && !ovl_redirect_follow(ofs)) { + pr_warn_ratelimited("refusing to follow redirect for (%pd2)\n", d->dentry); + return false; + } + return true; +} + +struct ovl_lookup_ctx { + struct dentry *dentry; + struct ovl_entry *oe; + struct ovl_path *stack; + struct ovl_path *origin_path; + struct dentry *upperdentry; + struct dentry *index; + struct inode *inode; + unsigned int ctr; +}; + +static int ovl_lookup_layers(struct ovl_lookup_ctx *ctx, struct ovl_lookup_data *d) +{ + struct dentry *dentry = ctx->dentry; + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + struct ovl_entry *poe = OVL_E(dentry->d_parent); + struct ovl_entry *roe = OVL_E(dentry->d_sb->s_root); + bool check_redirect = (ovl_redirect_follow(ofs) || ofs->numdatalayer); + struct dentry *upperdir; + struct dentry *this; + struct dentry *origin = NULL; + bool upperopaque = false; + bool uppermetacopy = false; + int metacopy_size = 0; + unsigned int i; + int err; + + upperdir = ovl_dentry_upper(dentry->d_parent); + if (upperdir) { + d->layer = &ofs->layers[0]; + err = ovl_lookup_layer(upperdir, d, &ctx->upperdentry, true); + if (err) + return err; + + if (ctx->upperdentry && ctx->upperdentry->d_flags & DCACHE_OP_REAL) + return -EREMOTE; + + if (ctx->upperdentry && !d->is_dir) { + /* + * Lookup copy up origin by decoding origin file handle. + * We may get a disconnected dentry, which is fine, + * because we only need to hold the origin inode in + * cache and use its inode number. We may even get a + * connected dentry, that is not under any of the lower + * layers root. That is also fine for using it's inode + * number - it's the same as if we held a reference + * to a dentry in lower layer that was moved under us. + */ + err = ovl_check_origin(ofs, ctx->upperdentry, &ctx->origin_path); + if (err) + return err; + + if (d->metacopy) + uppermetacopy = true; + metacopy_size = d->metacopy; + } + + if (d->redirect) { + err = -ENOMEM; + d->upperredirect = kstrdup(d->redirect, GFP_KERNEL); + if (!d->upperredirect) + return err; + if (d->redirect[0] == '/') + poe = roe; + } + upperopaque = d->opaque; + } + + if (!d->stop && ovl_numlower(poe)) { + err = -ENOMEM; + ctx->stack = ovl_stack_alloc(ofs->numlayer - 1); + if (!ctx->stack) + return err; + } + + for (i = 0; !d->stop && i < ovl_numlower(poe); i++) { + struct ovl_path lower = ovl_lowerstack(poe)[i]; + + if (!ovl_check_follow_redirect(d)) { + err = -EPERM; + return err; + } + + if (!check_redirect) + d->last = i == ovl_numlower(poe) - 1; + else if (d->is_dir || !ofs->numdatalayer) + d->last = lower.layer->idx == ovl_numlower(roe); + + d->layer = lower.layer; + err = ovl_lookup_layer(lower.dentry, d, &this, false); + if (err) + return err; + + if (!this) + continue; + + /* + * If no origin fh is stored in upper of a merge dir, store fh + * of lower dir and set upper parent "impure". + */ + if (ctx->upperdentry && !ctx->ctr && !ofs->noxattr && d->is_dir) { + err = ovl_fix_origin(ofs, dentry, this, ctx->upperdentry); + if (err) { + dput(this); + return err; + } + } + + /* + * When "verify_lower" feature is enabled, do not merge with a + * lower dir that does not match a stored origin xattr. In any + * case, only verified origin is used for index lookup. + * + * For non-dir dentry, if index=on, then ensure origin + * matches the dentry found using path based lookup, + * otherwise error out. + */ + if (ctx->upperdentry && !ctx->ctr && + ((d->is_dir && ovl_verify_lower(dentry->d_sb)) || + (!d->is_dir && ofs->config.index && ctx->origin_path))) { + err = ovl_verify_origin(ofs, ctx->upperdentry, this, false); + if (err) { + dput(this); + if (d->is_dir) + break; + return err; + } + origin = this; + } + + if (!ctx->upperdentry && !d->is_dir && !ctx->ctr && d->metacopy) + metacopy_size = d->metacopy; + + if (d->metacopy && ctx->ctr) { + /* + * Do not store intermediate metacopy dentries in + * lower chain, except top most lower metacopy dentry. + * Continue the loop so that if there is an absolute + * redirect on this dentry, poe can be reset to roe. + */ + dput(this); + this = NULL; + } else { + ctx->stack[ctx->ctr].dentry = this; + ctx->stack[ctx->ctr].layer = lower.layer; + ctx->ctr++; + } + + if (d->stop) + break; + + if (d->redirect && d->redirect[0] == '/' && poe != roe) { + poe = roe; + /* Find the current layer on the root dentry */ + i = lower.layer->idx - 1; + } + } + + /* + * Defer lookup of lowerdata in data-only layers to first access. + * Don't require redirect=follow and metacopy=on in this case. + */ + if (d->metacopy && ctx->ctr && ofs->numdatalayer && d->absolute_redirect) { + d->metacopy = 0; + ctx->ctr++; + } else if (!ovl_check_follow_redirect(d)) { + err = -EPERM; + return err; + } + + /* + * For regular non-metacopy upper dentries, there is no lower + * path based lookup, hence ctr will be zero. If a dentry is found + * using ORIGIN xattr on upper, install it in stack. + * + * For metacopy dentry, path based lookup will find lower dentries. + * Just make sure a corresponding data dentry has been found. + */ + if (d->metacopy || (uppermetacopy && !ctx->ctr)) { + pr_warn_ratelimited("metacopy with no lower data found - abort lookup (%pd2)\n", + dentry); + err = -EIO; + return err; + } else if (!d->is_dir && ctx->upperdentry && !ctx->ctr && ctx->origin_path) { + if (WARN_ON(ctx->stack != NULL)) { + err = -EIO; + return err; + } + ctx->stack = ctx->origin_path; + ctx->ctr = 1; + origin = ctx->origin_path->dentry; + ctx->origin_path = NULL; + } + + /* + * Always lookup index if there is no-upperdentry. + * + * For the case of upperdentry, we have set origin by now if it + * needed to be set. There are basically three cases. + * + * For directories, lookup index by lower inode and verify it matches + * upper inode. We only trust dir index if we verified that lower dir + * matches origin, otherwise dir index entries may be inconsistent + * and we ignore them. + * + * For regular upper, we already set origin if upper had ORIGIN + * xattr. There is no verification though as there is no path + * based dentry lookup in lower in this case. + * + * For metacopy upper, we set a verified origin already if index + * is enabled and if upper had an ORIGIN xattr. + * + */ + if (!ctx->upperdentry && ctx->ctr) + origin = ctx->stack[0].dentry; + + if (origin && ovl_indexdir(dentry->d_sb) && + (!d->is_dir || ovl_index_all(dentry->d_sb))) { + ctx->index = ovl_lookup_index(ofs, ctx->upperdentry, origin, true); + if (IS_ERR(ctx->index)) { + err = PTR_ERR(ctx->index); + ctx->index = NULL; + return err; + } + } + + if (ctx->ctr) { + ctx->oe = ovl_alloc_entry(ctx->ctr); + err = -ENOMEM; + if (!ctx->oe) + return err; + + ovl_stack_cpy(ovl_lowerstack(ctx->oe), ctx->stack, ctx->ctr); + } + + if (upperopaque) + ovl_dentry_set_opaque(dentry); + if (d->xwhiteouts) + ovl_dentry_set_xwhiteouts(dentry); + + if (ctx->upperdentry) + ovl_dentry_set_upper_alias(dentry); + else if (ctx->index) { + char *upperredirect; + struct path upperpath = { + .dentry = ctx->upperdentry = dget(ctx->index), + .mnt = ovl_upper_mnt(ofs), + }; + + /* + * It's safe to assign upperredirect here: the previous + * assignment happens only if upperdentry is non-NULL, and + * this one only if upperdentry is NULL. + */ + upperredirect = ovl_get_redirect_xattr(ofs, &upperpath, 0); + if (IS_ERR(upperredirect)) + return PTR_ERR(upperredirect); + d->upperredirect = upperredirect; + + err = ovl_check_metacopy_xattr(ofs, &upperpath, NULL); + if (err < 0) + return err; + d->metacopy = uppermetacopy = err; + metacopy_size = err; + + if (!ovl_check_follow_redirect(d)) { + err = -EPERM; + return err; + } + } + + if (ctx->upperdentry || ctx->ctr) { + struct inode *inode; + struct ovl_inode_params oip = { + .upperdentry = ctx->upperdentry, + .oe = ctx->oe, + .index = ctx->index, + .redirect = d->upperredirect, + }; + + /* Store lowerdata redirect for lazy lookup */ + if (ctx->ctr > 1 && !d->is_dir && !ctx->stack[ctx->ctr - 1].dentry) { + oip.lowerdata_redirect = d->redirect; + d->redirect = NULL; + } + + inode = ovl_get_inode(dentry->d_sb, &oip); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + ctx->inode = inode; + if (ctx->upperdentry && !uppermetacopy) + ovl_set_flag(OVL_UPPERDATA, ctx->inode); + + if (metacopy_size > OVL_METACOPY_MIN_SIZE) + ovl_set_flag(OVL_HAS_DIGEST, ctx->inode); + } + + ovl_dentry_init_reval(dentry, ctx->upperdentry, OVL_I_E(ctx->inode)); + + return 0; +} + +struct dentry *ovl_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + struct ovl_entry *poe = OVL_E(dentry->d_parent); + bool check_redirect = (ovl_redirect_follow(ofs) || ofs->numdatalayer); + int err; + struct ovl_lookup_ctx ctx = { + .dentry = dentry, + }; + struct ovl_lookup_data d = { + .sb = dentry->d_sb, + .dentry = dentry, + .name = dentry->d_name, + .last = check_redirect ? false : !ovl_numlower(poe), + }; + + if (dentry->d_name.len > ofs->namelen) + return ERR_PTR(-ENAMETOOLONG); + + with_ovl_creds(dentry->d_sb) + err = ovl_lookup_layers(&ctx, &d); + + if (ctx.origin_path) { + dput(ctx.origin_path->dentry); + kfree(ctx.origin_path); + } + dput(ctx.index); + ovl_stack_free(ctx.stack, ctx.ctr); + kfree(d.redirect); + + if (err) { + ovl_free_entry(ctx.oe); + dput(ctx.upperdentry); + kfree(d.upperredirect); + return ERR_PTR(err); + } + + return d_splice_alias(ctx.inode, dentry); +} + +bool ovl_lower_positive(struct dentry *dentry) +{ + struct ovl_entry *poe = OVL_E(dentry->d_parent); + const struct qstr *name = &dentry->d_name; + unsigned int i; + bool positive = false; + bool done = false; + + /* + * If dentry is negative, then lower is positive iff this is a + * whiteout. + */ + if (!dentry->d_inode) + return ovl_dentry_is_opaque(dentry); + + /* Negative upper -> positive lower */ + if (!ovl_dentry_upper(dentry)) + return true; + + with_ovl_creds(dentry->d_sb) { + /* Positive upper -> have to look up lower to see whether it exists */ + for (i = 0; !done && !positive && i < ovl_numlower(poe); i++) { + struct dentry *this; + struct ovl_path *parentpath = &ovl_lowerstack(poe)[i]; + + /* + * We need to make a non-const copy of dentry->d_name, + * because lookup_one_positive_unlocked() will hash name + * with parentpath base, which is on another (lower fs). + */ + this = lookup_one_positive_unlocked(mnt_idmap(parentpath->layer->mnt), + &QSTR_LEN(name->name, name->len), + parentpath->dentry); + if (IS_ERR(this)) { + switch (PTR_ERR(this)) { + case -ENOENT: + case -ENAMETOOLONG: + break; + + default: + /* + * Assume something is there, we just couldn't + * access it. + */ + positive = true; + break; + } + } else { + struct path path = { + .dentry = this, + .mnt = parentpath->layer->mnt, + }; + positive = !ovl_path_is_whiteout(OVL_FS(dentry->d_sb), &path); + done = true; + dput(this); + } + } + } + + return positive; +} diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h new file mode 100644 index 000000000..e0d8c6152 --- /dev/null +++ b/fs/overlayfs/overlayfs.h @@ -0,0 +1,955 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * + * Copyright (C) 2011 Novell Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "ovl_entry.h" + +#undef pr_fmt +#define pr_fmt(fmt) "overlayfs: " fmt + +enum ovl_path_type { + __OVL_PATH_UPPER = (1 << 0), + __OVL_PATH_MERGE = (1 << 1), + __OVL_PATH_ORIGIN = (1 << 2), +}; + +#define OVL_TYPE_UPPER(type) ((type) & __OVL_PATH_UPPER) +#define OVL_TYPE_MERGE(type) ((type) & __OVL_PATH_MERGE) +#define OVL_TYPE_ORIGIN(type) ((type) & __OVL_PATH_ORIGIN) + +#define OVL_XATTR_NAMESPACE "overlay." +#define OVL_XATTR_TRUSTED_PREFIX XATTR_TRUSTED_PREFIX OVL_XATTR_NAMESPACE +#define OVL_XATTR_TRUSTED_PREFIX_LEN (sizeof(OVL_XATTR_TRUSTED_PREFIX) - 1) +#define OVL_XATTR_USER_PREFIX XATTR_USER_PREFIX OVL_XATTR_NAMESPACE +#define OVL_XATTR_USER_PREFIX_LEN (sizeof(OVL_XATTR_USER_PREFIX) - 1) + +#define OVL_XATTR_ESCAPE_PREFIX OVL_XATTR_NAMESPACE +#define OVL_XATTR_ESCAPE_PREFIX_LEN (sizeof(OVL_XATTR_ESCAPE_PREFIX) - 1) +#define OVL_XATTR_ESCAPE_TRUSTED_PREFIX OVL_XATTR_TRUSTED_PREFIX OVL_XATTR_ESCAPE_PREFIX +#define OVL_XATTR_ESCAPE_TRUSTED_PREFIX_LEN (sizeof(OVL_XATTR_ESCAPE_TRUSTED_PREFIX) - 1) +#define OVL_XATTR_ESCAPE_USER_PREFIX OVL_XATTR_USER_PREFIX OVL_XATTR_ESCAPE_PREFIX +#define OVL_XATTR_ESCAPE_USER_PREFIX_LEN (sizeof(OVL_XATTR_ESCAPE_USER_PREFIX) - 1) + +enum ovl_xattr { + OVL_XATTR_OPAQUE, + OVL_XATTR_REDIRECT, + OVL_XATTR_ORIGIN, + OVL_XATTR_IMPURE, + OVL_XATTR_NLINK, + OVL_XATTR_UPPER, + OVL_XATTR_UUID, + OVL_XATTR_METACOPY, + OVL_XATTR_PROTATTR, + OVL_XATTR_XWHITEOUT, +}; + +enum ovl_inode_flag { + /* Pure upper dir that may contain non pure upper entries */ + OVL_IMPURE, + /* Non-merge dir that may contain whiteout entries */ + OVL_WHITEOUTS, + OVL_INDEX, + OVL_UPPERDATA, + /* Inode number will remain constant over copy up. */ + OVL_CONST_INO, + OVL_HAS_DIGEST, + OVL_VERIFIED_DIGEST, +}; + +enum ovl_entry_flag { + OVL_E_UPPER_ALIAS, + OVL_E_OPAQUE, + OVL_E_CONNECTED, + /* Lower stack may contain xwhiteout entries */ + OVL_E_XWHITEOUTS, +}; + +enum { + OVL_REDIRECT_OFF, /* "off" mode is never used. In effect */ + OVL_REDIRECT_FOLLOW, /* ...it translates to either "follow" */ + OVL_REDIRECT_NOFOLLOW, /* ...or "nofollow". */ + OVL_REDIRECT_ON, +}; + +enum { + OVL_UUID_OFF, + OVL_UUID_NULL, + OVL_UUID_AUTO, + OVL_UUID_ON, +}; + +enum { + OVL_XINO_OFF, + OVL_XINO_AUTO, + OVL_XINO_ON, +}; + +enum { + OVL_VERITY_OFF, + OVL_VERITY_ON, + OVL_VERITY_REQUIRE, +}; + +enum { + OVL_FSYNC_VOLATILE, + OVL_FSYNC_AUTO, + OVL_FSYNC_STRICT, +}; + +/* + * The tuple (fh,uuid) is a universal unique identifier for a copy up origin, + * where: + * origin.fh - exported file handle of the lower file + * origin.uuid - uuid of the lower filesystem + */ +#define OVL_FH_VERSION 0 +#define OVL_FH_MAGIC 0xfb + +/* CPU byte order required for fid decoding: */ +#define OVL_FH_FLAG_BIG_ENDIAN (1 << 0) +#define OVL_FH_FLAG_ANY_ENDIAN (1 << 1) +/* Is the real inode encoded in fid an upper inode? */ +#define OVL_FH_FLAG_PATH_UPPER (1 << 2) + +#define OVL_FH_FLAG_ALL (OVL_FH_FLAG_BIG_ENDIAN | OVL_FH_FLAG_ANY_ENDIAN | \ + OVL_FH_FLAG_PATH_UPPER) + +#if defined(__LITTLE_ENDIAN) +#define OVL_FH_FLAG_CPU_ENDIAN 0 +#elif defined(__BIG_ENDIAN) +#define OVL_FH_FLAG_CPU_ENDIAN OVL_FH_FLAG_BIG_ENDIAN +#else +#error Endianness not defined +#endif + +/* The type used to be returned by overlay exportfs for misaligned fid */ +#define OVL_FILEID_V0 0xfb +/* The type returned by overlay exportfs for 32bit aligned fid */ +#define OVL_FILEID_V1 0xf8 + +/* On-disk format for "origin" file handle */ +struct ovl_fb { + u8 version; /* 0 */ + u8 magic; /* 0xfb */ + u8 len; /* size of this header + size of fid */ + u8 flags; /* OVL_FH_FLAG_* */ + u8 type; /* fid_type of fid */ + uuid_t uuid; /* uuid of filesystem */ + u32 fid[]; /* file identifier should be 32bit aligned in-memory */ +} __packed; + +/* In-memory and on-wire format for overlay file handle */ +struct ovl_fh { + u8 padding[3]; /* make sure fb.fid is 32bit aligned */ + union { + struct ovl_fb fb; + DECLARE_FLEX_ARRAY(u8, buf); + }; +} __packed; + +#define OVL_FH_WIRE_OFFSET offsetof(struct ovl_fh, fb) +#define OVL_FH_LEN(fh) (OVL_FH_WIRE_OFFSET + (fh)->fb.len) +#define OVL_FH_FID_OFFSET (OVL_FH_WIRE_OFFSET + \ + offsetof(struct ovl_fb, fid)) + +/* On-disk format for "metacopy" xattr (if non-zero size) */ +struct ovl_metacopy { + u8 version; /* 0 */ + u8 len; /* size of this header + used digest bytes */ + u8 flags; + u8 digest_algo; /* FS_VERITY_HASH_ALG_* constant, 0 for no digest */ + u8 digest[FS_VERITY_MAX_DIGEST_SIZE]; /* Only the used part on disk */ +} __packed; + +#define OVL_METACOPY_MAX_SIZE (sizeof(struct ovl_metacopy)) +#define OVL_METACOPY_MIN_SIZE (OVL_METACOPY_MAX_SIZE - FS_VERITY_MAX_DIGEST_SIZE) +#define OVL_METACOPY_INIT { 0, OVL_METACOPY_MIN_SIZE } + +static inline int ovl_metadata_digest_size(const struct ovl_metacopy *metacopy) +{ + if (metacopy->len < OVL_METACOPY_MIN_SIZE) + return 0; + return (int)metacopy->len - OVL_METACOPY_MIN_SIZE; +} + +/* No atime modification on underlying */ +#define OVL_OPEN_FLAGS (O_NOATIME) + +extern const char *const ovl_xattr_table[][2]; +static inline const char *ovl_xattr(struct ovl_fs *ofs, enum ovl_xattr ox) +{ + return ovl_xattr_table[ox][ofs->config.userxattr]; +} + +/* + * When changing ownership of an upper object map the intended ownership + * according to the upper layer's idmapping. When an upper mount idmaps files + * that are stored on-disk as owned by id 1001 to id 1000 this means stat on + * this object will report it as being owned by id 1000 when calling stat via + * the upper mount. + * In order to change ownership of an object so stat reports id 1000 when + * called on an idmapped upper mount the value written to disk - i.e., the + * value stored in ia_*id - must 1001. The mount mapping helper will thus take + * care to map 1000 to 1001. + * The mnt idmapping helpers are nops if the upper layer isn't idmapped. + */ +static inline int ovl_do_notify_change(struct ovl_fs *ofs, + struct dentry *upperdentry, + struct iattr *attr) +{ + return notify_change(ovl_upper_mnt_idmap(ofs), upperdentry, attr, NULL); +} + +static inline int ovl_do_rmdir(struct ovl_fs *ofs, + struct inode *dir, struct dentry *dentry) +{ + int err = vfs_rmdir(ovl_upper_mnt_idmap(ofs), dir, dentry, NULL); + + pr_debug("rmdir(%pd2) = %i\n", dentry, err); + return err; +} + +static inline int ovl_do_unlink(struct ovl_fs *ofs, struct inode *dir, + struct dentry *dentry) +{ + int err = vfs_unlink(ovl_upper_mnt_idmap(ofs), dir, dentry, NULL); + + pr_debug("unlink(%pd2) = %i\n", dentry, err); + return err; +} + +static inline int ovl_do_link(struct ovl_fs *ofs, struct dentry *old_dentry, + struct inode *dir, struct dentry *new_dentry) +{ + int err = vfs_link(old_dentry, ovl_upper_mnt_idmap(ofs), dir, + new_dentry, NULL); + + pr_debug("link(%pd2, %pd2) = %i\n", old_dentry, new_dentry, err); + return err; +} + +static inline int ovl_do_create(struct ovl_fs *ofs, + struct inode *dir, struct dentry *dentry, + umode_t mode) +{ + int err = vfs_create(ovl_upper_mnt_idmap(ofs), dentry, mode, NULL); + + pr_debug("create(%pd2, 0%o) = %i\n", dentry, mode, err); + return err; +} + +static inline struct dentry *ovl_do_mkdir(struct ovl_fs *ofs, + struct inode *dir, + struct dentry *dentry, + umode_t mode) +{ + struct dentry *ret; + + ret = vfs_mkdir(ovl_upper_mnt_idmap(ofs), dir, dentry, mode, NULL); + pr_debug("mkdir(%pd2, 0%o) = %i\n", dentry, mode, PTR_ERR_OR_ZERO(ret)); + return ret; +} + +static inline int ovl_do_mknod(struct ovl_fs *ofs, + struct inode *dir, struct dentry *dentry, + umode_t mode, dev_t dev) +{ + int err = vfs_mknod(ovl_upper_mnt_idmap(ofs), dir, dentry, mode, dev, NULL); + + pr_debug("mknod(%pd2, 0%o, 0%o) = %i\n", dentry, mode, dev, err); + return err; +} + +static inline int ovl_do_symlink(struct ovl_fs *ofs, + struct inode *dir, struct dentry *dentry, + const char *oldname) +{ + int err = vfs_symlink(ovl_upper_mnt_idmap(ofs), dir, dentry, oldname, NULL); + + pr_debug("symlink(\"%s\", %pd2) = %i\n", oldname, dentry, err); + return err; +} + +static inline ssize_t ovl_do_getxattr(const struct path *path, const char *name, + void *value, size_t size) +{ + int err, len; + + WARN_ON(path->dentry->d_sb != path->mnt->mnt_sb); + + err = vfs_getxattr(mnt_idmap(path->mnt), path->dentry, + name, value, size); + len = (value && err > 0) ? err : 0; + + pr_debug("getxattr(%pd2, \"%s\", \"%*pE\", %zu, 0) = %i\n", + path->dentry, name, min(len, 48), value, size, err); + return err; +} + +static inline ssize_t ovl_getxattr_upper(struct ovl_fs *ofs, + struct dentry *upperdentry, + enum ovl_xattr ox, void *value, + size_t size) +{ + struct path upperpath = { + .dentry = upperdentry, + .mnt = ovl_upper_mnt(ofs), + }; + + return ovl_do_getxattr(&upperpath, ovl_xattr(ofs, ox), value, size); +} + +static inline ssize_t ovl_path_getxattr(struct ovl_fs *ofs, + const struct path *path, + enum ovl_xattr ox, void *value, + size_t size) +{ + return ovl_do_getxattr(path, ovl_xattr(ofs, ox), value, size); +} + +static inline int ovl_do_setxattr(struct ovl_fs *ofs, struct dentry *dentry, + const char *name, const void *value, + size_t size, int flags) +{ + /* Use vfs_setxattr(), not __vfs_setxattr(): it idmaps the security.capability rootid. */ + int err = vfs_setxattr(ovl_upper_mnt_idmap(ofs), dentry, name, + value, size, flags); + + pr_debug("setxattr(%pd2, \"%s\", \"%*pE\", %zu, %d) = %i\n", + dentry, name, min((int)size, 48), value, size, flags, err); + return err; +} + +static inline int ovl_setxattr(struct ovl_fs *ofs, struct dentry *dentry, + enum ovl_xattr ox, const void *value, + size_t size) +{ + return ovl_do_setxattr(ofs, dentry, ovl_xattr(ofs, ox), value, size, 0); +} + +static inline int ovl_do_removexattr(struct ovl_fs *ofs, struct dentry *dentry, + const char *name) +{ + int err = vfs_removexattr(ovl_upper_mnt_idmap(ofs), dentry, name); + pr_debug("removexattr(%pd2, \"%s\") = %i\n", dentry, name, err); + return err; +} + +static inline int ovl_removexattr(struct ovl_fs *ofs, struct dentry *dentry, + enum ovl_xattr ox) +{ + return ovl_do_removexattr(ofs, dentry, ovl_xattr(ofs, ox)); +} + +static inline int ovl_do_set_acl(struct ovl_fs *ofs, struct dentry *dentry, + const char *acl_name, struct posix_acl *acl) +{ + return vfs_set_acl(ovl_upper_mnt_idmap(ofs), dentry, acl_name, acl); +} + +static inline int ovl_do_remove_acl(struct ovl_fs *ofs, struct dentry *dentry, + const char *acl_name) +{ + return vfs_remove_acl(ovl_upper_mnt_idmap(ofs), dentry, acl_name); +} + +static inline int ovl_do_rename_rd(struct renamedata *rd) +{ + int err; + + pr_debug("rename(%pd2, %pd2, 0x%x)\n", rd->old_dentry, rd->new_dentry, + rd->flags); + err = vfs_rename(rd); + if (err) { + pr_debug("...rename(%pd2, %pd2, ...) = %i\n", + rd->old_dentry, rd->new_dentry, err); + } + return err; +} + +static inline int ovl_do_rename(struct ovl_fs *ofs, struct dentry *olddir, + struct dentry *olddentry, struct dentry *newdir, + struct dentry *newdentry, unsigned int flags) +{ + struct renamedata rd = { + .mnt_idmap = ovl_upper_mnt_idmap(ofs), + .old_parent = olddir, + .old_dentry = olddentry, + .new_parent = newdir, + .new_dentry = newdentry, + .flags = flags, + }; + + return ovl_do_rename_rd(&rd); +} + +static inline int ovl_do_whiteout(struct ovl_fs *ofs, + struct inode *dir, struct dentry *dentry) +{ + int err = vfs_whiteout(ovl_upper_mnt_idmap(ofs), dir, dentry); + pr_debug("whiteout(%pd2) = %i\n", dentry, err); + return err; +} + +static inline struct file *ovl_do_tmpfile(struct ovl_fs *ofs, + struct dentry *dentry, umode_t mode) +{ + struct path path = { .mnt = ovl_upper_mnt(ofs), .dentry = dentry }; + struct file *file = kernel_tmpfile_open(ovl_upper_mnt_idmap(ofs), &path, + mode, O_LARGEFILE | O_WRONLY, + current_cred()); + int err = PTR_ERR_OR_ZERO(file); + + pr_debug("tmpfile(%pd2, 0%o) = %i\n", dentry, mode, err); + return file; +} + +static inline struct dentry *ovl_lookup_upper_unlocked(struct ovl_fs *ofs, + const char *name, + struct dentry *base, + int len) +{ + return lookup_one_unlocked(ovl_upper_mnt_idmap(ofs), + &QSTR_LEN(name, len), base); +} + +static inline struct dentry *ovl_start_creating_upper(struct ovl_fs *ofs, + struct dentry *parent, + struct qstr *name) +{ + return start_creating(ovl_upper_mnt_idmap(ofs), + parent, name); +} + +static inline struct dentry *ovl_start_removing_upper(struct ovl_fs *ofs, + struct dentry *parent, + struct qstr *name) +{ + return start_removing(ovl_upper_mnt_idmap(ofs), + parent, name); +} + +static inline bool ovl_open_flags_need_copy_up(int flags) +{ + if (!flags) + return false; + + return ((OPEN_FMODE(flags) & FMODE_WRITE) || (flags & O_TRUNC)); +} + +/* util.c */ +int ovl_get_write_access(struct dentry *dentry); +void ovl_put_write_access(struct dentry *dentry); +void ovl_start_write(struct dentry *dentry); +void ovl_end_write(struct dentry *dentry); +int ovl_want_write(struct dentry *dentry); +void ovl_drop_write(struct dentry *dentry); +struct dentry *ovl_workdir(struct dentry *dentry); +const struct cred *ovl_override_creds(struct super_block *sb); + +EXTEND_CLASS(override_creds, _ovl, ovl_override_creds(sb), struct super_block *sb) + +#define with_ovl_creds(sb) \ + scoped_class(override_creds_ovl, __UNIQUE_ID(label), sb) + +static inline const struct cred *ovl_creds(struct super_block *sb) +{ + return OVL_FS(sb)->creator_cred; +} + +int ovl_can_decode_fh(struct super_block *sb); +struct dentry *ovl_indexdir(struct super_block *sb); +bool ovl_index_all(struct super_block *sb); +bool ovl_verify_lower(struct super_block *sb); +struct ovl_path *ovl_stack_alloc(unsigned int n); +void ovl_stack_cpy(struct ovl_path *dst, struct ovl_path *src, unsigned int n); +void ovl_stack_put(struct ovl_path *stack, unsigned int n); +void ovl_stack_free(struct ovl_path *stack, unsigned int n); +struct ovl_entry *ovl_alloc_entry(unsigned int numlower); +void ovl_free_entry(struct ovl_entry *oe); +bool ovl_dentry_remote(struct dentry *dentry); +void ovl_dentry_update_reval(struct dentry *dentry, struct dentry *realdentry); +void ovl_dentry_init_reval(struct dentry *dentry, struct dentry *upperdentry, + struct ovl_entry *oe); +void ovl_dentry_init_flags(struct dentry *dentry, struct dentry *upperdentry, + struct ovl_entry *oe, unsigned int mask); +bool ovl_dentry_weird(struct dentry *dentry); + +static inline bool ovl_dentry_casefolded(struct dentry *dentry) +{ + return sb_has_encoding(dentry->d_sb) && IS_CASEFOLDED(d_inode(dentry)); +} + +enum ovl_path_type ovl_path_type(struct dentry *dentry); +void ovl_path_upper(struct dentry *dentry, struct path *path); +void ovl_path_lower(struct dentry *dentry, struct path *path); +void ovl_path_lowerdata(struct dentry *dentry, struct path *path); +struct inode *ovl_i_path_real(struct inode *inode, struct path *path); +enum ovl_path_type ovl_path_real(struct dentry *dentry, struct path *path); +enum ovl_path_type ovl_path_realdata(struct dentry *dentry, struct path *path); +struct dentry *ovl_dentry_upper(struct dentry *dentry); +struct dentry *ovl_dentry_lower(struct dentry *dentry); +struct dentry *ovl_dentry_lowerdata(struct dentry *dentry); +int ovl_dentry_set_lowerdata(struct dentry *dentry, struct ovl_path *datapath); +const struct ovl_layer *ovl_i_layer_lower(struct inode *inode); +const struct ovl_layer *ovl_layer_lower(struct dentry *dentry); +struct dentry *ovl_dentry_real(struct dentry *dentry); +struct dentry *ovl_i_dentry_upper(struct inode *inode); +struct inode *ovl_inode_upper(struct inode *inode); +struct inode *ovl_inode_lower(struct inode *inode); +struct inode *ovl_inode_lowerdata(struct inode *inode); +struct inode *ovl_inode_real(struct inode *inode); +struct inode *ovl_inode_realdata(struct inode *inode); +const char *ovl_lowerdata_redirect(struct inode *inode); +struct ovl_dir_cache *ovl_dir_cache(struct inode *inode); +void ovl_set_dir_cache(struct inode *inode, struct ovl_dir_cache *cache); +void ovl_dentry_set_flag(unsigned long flag, struct dentry *dentry); +void ovl_dentry_clear_flag(unsigned long flag, struct dentry *dentry); +bool ovl_dentry_test_flag(unsigned long flag, struct dentry *dentry); +bool ovl_dentry_is_opaque(struct dentry *dentry); +bool ovl_dentry_is_whiteout(struct dentry *dentry); +void ovl_dentry_set_opaque(struct dentry *dentry); +bool ovl_dentry_has_xwhiteouts(struct dentry *dentry); +void ovl_dentry_set_xwhiteouts(struct dentry *dentry); +void ovl_layer_set_xwhiteouts(struct ovl_fs *ofs, + const struct ovl_layer *layer); +bool ovl_dentry_has_upper_alias(struct dentry *dentry); +void ovl_dentry_set_upper_alias(struct dentry *dentry); +bool ovl_dentry_needs_data_copy_up(struct dentry *dentry, int flags); +bool ovl_dentry_needs_data_copy_up_locked(struct dentry *dentry, int flags); +bool ovl_has_upperdata(struct inode *inode); +void ovl_set_upperdata(struct inode *inode); +const char *ovl_dentry_get_redirect(struct dentry *dentry); +void ovl_dentry_set_redirect(struct dentry *dentry, const char *redirect); +void ovl_inode_update(struct inode *inode, struct dentry *upperdentry); +void ovl_dir_modified(struct dentry *dentry, bool impurity); +u64 ovl_inode_version_get(struct inode *inode); +bool ovl_is_whiteout(struct dentry *dentry); +bool ovl_path_is_whiteout(struct ovl_fs *ofs, const struct path *path); +struct file *ovl_path_open(const struct path *path, int flags); +int ovl_copy_up_start(struct dentry *dentry, int flags); +void ovl_copy_up_end(struct dentry *dentry); +bool ovl_already_copied_up(struct dentry *dentry, int flags); +char ovl_get_dir_xattr_val(struct ovl_fs *ofs, const struct path *path, + enum ovl_xattr ox); +bool ovl_path_check_origin_xattr(struct ovl_fs *ofs, const struct path *path); +bool ovl_path_check_xwhiteout_xattr(struct ovl_fs *ofs, const struct path *path); +bool ovl_init_uuid_xattr(struct super_block *sb, struct ovl_fs *ofs, + const struct path *upperpath); + +static inline bool ovl_upper_is_whiteout(struct ovl_fs *ofs, + struct dentry *upperdentry) +{ + struct path upperpath = { + .dentry = upperdentry, + .mnt = ovl_upper_mnt(ofs), + }; + return ovl_path_is_whiteout(ofs, &upperpath); +} + +static inline bool ovl_check_origin_xattr(struct ovl_fs *ofs, + struct dentry *upperdentry) +{ + struct path upperpath = { + .dentry = upperdentry, + .mnt = ovl_upper_mnt(ofs), + }; + return ovl_path_check_origin_xattr(ofs, &upperpath); +} + +int ovl_check_setxattr(struct ovl_fs *ofs, struct dentry *upperdentry, + enum ovl_xattr ox, const void *value, size_t size, + int xerr); +int ovl_set_impure(struct dentry *dentry, struct dentry *upperdentry); +bool ovl_inuse_trylock(struct dentry *dentry); +void ovl_inuse_unlock(struct dentry *dentry); +bool ovl_is_inuse(struct dentry *dentry); +bool ovl_need_index(struct dentry *dentry); +int ovl_nlink_start(struct dentry *dentry); +void ovl_nlink_end(struct dentry *dentry); +int ovl_check_metacopy_xattr(struct ovl_fs *ofs, const struct path *path, + struct ovl_metacopy *data); +int ovl_set_metacopy_xattr(struct ovl_fs *ofs, struct dentry *d, + struct ovl_metacopy *metacopy); +bool ovl_is_metacopy_dentry(struct dentry *dentry); +char *ovl_get_redirect_xattr(struct ovl_fs *ofs, const struct path *path, int padding); +int ovl_ensure_verity_loaded(const struct path *path); +int ovl_validate_verity(struct ovl_fs *ofs, + const struct path *metapath, + const struct path *datapath); +int ovl_get_verity_digest(struct ovl_fs *ofs, const struct path *src, + struct ovl_metacopy *metacopy); +int ovl_sync_status(struct ovl_fs *ofs); + +static inline void ovl_set_flag(unsigned long flag, struct inode *inode) +{ + set_bit(flag, &OVL_I(inode)->flags); +} + +static inline void ovl_clear_flag(unsigned long flag, struct inode *inode) +{ + clear_bit(flag, &OVL_I(inode)->flags); +} + +static inline bool ovl_test_flag(unsigned long flag, struct inode *inode) +{ + return test_bit(flag, &OVL_I(inode)->flags); +} + +static inline bool ovl_is_impuredir(struct super_block *sb, + struct dentry *upperdentry) +{ + struct ovl_fs *ofs = OVL_FS(sb); + struct path upperpath = { + .dentry = upperdentry, + .mnt = ovl_upper_mnt(ofs), + }; + + return ovl_get_dir_xattr_val(ofs, &upperpath, OVL_XATTR_IMPURE) == 'y'; +} + +static inline char ovl_get_opaquedir_val(struct ovl_fs *ofs, + const struct path *path) +{ + return ovl_get_dir_xattr_val(ofs, path, OVL_XATTR_OPAQUE); +} + +static inline bool ovl_redirect_follow(struct ovl_fs *ofs) +{ + return ofs->config.redirect_mode != OVL_REDIRECT_NOFOLLOW; +} + +static inline bool ovl_redirect_dir(struct ovl_fs *ofs) +{ + return ofs->config.redirect_mode == OVL_REDIRECT_ON; +} + +static inline bool ovl_origin_uuid(struct ovl_fs *ofs) +{ + return ofs->config.uuid != OVL_UUID_OFF; +} + +static inline bool ovl_has_fsid(struct ovl_fs *ofs) +{ + return ofs->config.uuid == OVL_UUID_ON || + ofs->config.uuid == OVL_UUID_AUTO; +} + +/* + * With xino=auto, we do best effort to keep all inodes on same st_dev and + * d_ino consistent with st_ino. + * With xino=on, we do the same effort but we warn if we failed. + */ +static inline bool ovl_xino_warn(struct ovl_fs *ofs) +{ + return ofs->config.xino == OVL_XINO_ON; +} + +static inline bool ovl_should_sync(struct ovl_fs *ofs) +{ + return ofs->config.fsync_mode != OVL_FSYNC_VOLATILE; +} + +static inline bool ovl_should_sync_metadata(struct ovl_fs *ofs) +{ + return ofs->config.fsync_mode == OVL_FSYNC_STRICT; +} + +static inline bool ovl_is_volatile(struct ovl_config *config) +{ + return config->fsync_mode == OVL_FSYNC_VOLATILE; +} + +/* + * To avoid regressions in existing setups with overlay lower offline changes, + * we allow lower changes only if none of the new features are used. + */ +static inline bool ovl_allow_offline_changes(struct ovl_fs *ofs) +{ + return (!ofs->config.index && !ofs->config.metacopy && + !ovl_redirect_dir(ofs) && !ovl_xino_warn(ofs)); +} + +/* All layers on same fs? */ +static inline bool ovl_same_fs(struct ovl_fs *ofs) +{ + return ofs->xino_mode == 0; +} + +/* All overlay inodes have same st_dev? */ +static inline bool ovl_same_dev(struct ovl_fs *ofs) +{ + return ofs->xino_mode >= 0; +} + +static inline unsigned int ovl_xino_bits(struct ovl_fs *ofs) +{ + return ovl_same_dev(ofs) ? ofs->xino_mode : 0; +} + +static inline void ovl_inode_lock(struct inode *inode) +{ + mutex_lock(&OVL_I(inode)->lock); +} + +static inline int ovl_inode_lock_interruptible(struct inode *inode) +{ + return mutex_lock_interruptible(&OVL_I(inode)->lock); +} + +static inline void ovl_inode_unlock(struct inode *inode) +{ + mutex_unlock(&OVL_I(inode)->lock); +} + + +/* namei.c */ +int ovl_check_fb_len(struct ovl_fb *fb, int fb_len); + +static inline int ovl_check_fh_len(struct ovl_fh *fh, int fh_len) +{ + if (fh_len < sizeof(struct ovl_fh)) + return -EINVAL; + + return ovl_check_fb_len(&fh->fb, fh_len - OVL_FH_WIRE_OFFSET); +} + +bool ovl_uuid_match(struct ovl_fs *ofs, const struct super_block *sb, + const uuid_t *uuid); +struct dentry *ovl_decode_real_fh(struct ovl_fs *ofs, struct ovl_fh *fh, + struct vfsmount *mnt, bool connected); +int ovl_check_origin_fh(struct ovl_fs *ofs, struct ovl_fh *fh, bool connected, + struct dentry *upperdentry, struct ovl_path **stackp); +int ovl_verify_set_fh(struct ovl_fs *ofs, struct dentry *dentry, + enum ovl_xattr ox, const struct ovl_fh *fh, + bool is_upper, bool set); +int ovl_verify_origin_xattr(struct ovl_fs *ofs, struct dentry *dentry, + enum ovl_xattr ox, struct dentry *real, + bool is_upper, bool set); +struct dentry *ovl_index_upper(struct ovl_fs *ofs, struct dentry *index, + bool connected); +int ovl_verify_index(struct ovl_fs *ofs, struct dentry *index); +int ovl_get_index_name_fh(const struct ovl_fh *fh, struct qstr *name); +int ovl_get_index_name(struct ovl_fs *ofs, struct dentry *origin, + struct qstr *name); +struct dentry *ovl_get_index_fh(struct ovl_fs *ofs, struct ovl_fh *fh); +struct dentry *ovl_lookup_index(struct ovl_fs *ofs, struct dentry *upper, + struct dentry *origin, bool verify); +int ovl_path_next(int idx, struct dentry *dentry, struct path *path, + const struct ovl_layer **layer); +int ovl_verify_lowerdata(struct dentry *dentry); +struct dentry *ovl_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags); +bool ovl_lower_positive(struct dentry *dentry); + +static inline int ovl_verify_origin_fh(struct ovl_fs *ofs, struct dentry *upper, + const struct ovl_fh *fh, bool set) +{ + return ovl_verify_set_fh(ofs, upper, OVL_XATTR_ORIGIN, fh, false, set); +} + +static inline int ovl_verify_origin(struct ovl_fs *ofs, struct dentry *upper, + struct dentry *origin, bool set) +{ + return ovl_verify_origin_xattr(ofs, upper, OVL_XATTR_ORIGIN, origin, + false, set); +} + +static inline int ovl_verify_upper(struct ovl_fs *ofs, struct dentry *index, + struct dentry *upper, bool set) +{ + return ovl_verify_origin_xattr(ofs, index, OVL_XATTR_UPPER, upper, + true, set); +} + +/* readdir.c */ +extern const struct file_operations ovl_dir_operations; +struct file *ovl_dir_real_file(const struct file *file, bool want_upper); +int ovl_check_empty_dir(struct dentry *dentry, struct list_head *list); +void ovl_cleanup_whiteouts(struct ovl_fs *ofs, struct dentry *upper, + struct list_head *list); +void ovl_cache_free(struct list_head *list); +void ovl_dir_cache_free(struct inode *inode); +int ovl_check_d_type_supported(const struct path *realpath); +int ovl_workdir_cleanup(struct ovl_fs *ofs, struct dentry *parent, + struct vfsmount *mnt, struct dentry *dentry, int level); +int ovl_indexdir_cleanup(struct ovl_fs *ofs); + +/* + * Can we iterate real dir directly? + * + * Non-merge dir may contain whiteouts from a time it was a merge upper, before + * lower dir was removed under it and possibly before it was rotated from upper + * to lower layer. + */ +static inline bool ovl_dir_is_real(struct inode *dir) +{ + return !ovl_test_flag(OVL_WHITEOUTS, dir); +} + +/* inode.c */ +int ovl_set_nlink_upper(struct dentry *dentry); +int ovl_set_nlink_lower(struct dentry *dentry); +unsigned int ovl_get_nlink(struct ovl_fs *ofs, struct dentry *lowerdentry, + struct dentry *upperdentry, + unsigned int fallback); +int ovl_permission(struct mnt_idmap *idmap, struct inode *inode, + int mask); + +#ifdef CONFIG_FS_POSIX_ACL +struct posix_acl *do_ovl_get_acl(struct mnt_idmap *idmap, + struct inode *inode, int type, + bool rcu, bool noperm); +static inline struct posix_acl *ovl_get_inode_acl(struct inode *inode, int type, + bool rcu) +{ + return do_ovl_get_acl(&nop_mnt_idmap, inode, type, rcu, true); +} +static inline struct posix_acl *ovl_get_acl(struct mnt_idmap *idmap, + struct dentry *dentry, int type) +{ + return do_ovl_get_acl(idmap, d_inode(dentry), type, false, false); +} +int ovl_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type); +struct posix_acl *ovl_get_acl_path(const struct path *path, + const char *acl_name, bool noperm); +#else +#define ovl_get_inode_acl NULL +#define ovl_get_acl NULL +#define ovl_set_acl NULL +static inline struct posix_acl *ovl_get_acl_path(const struct path *path, + const char *acl_name, + bool noperm) +{ + return NULL; +} +#endif + +int ovl_update_time(struct inode *inode, enum fs_update_time type, + unsigned int flags); +bool ovl_is_private_xattr(struct super_block *sb, const char *name); + +struct ovl_inode_params { + struct inode *newinode; + struct dentry *upperdentry; + struct ovl_entry *oe; + bool index; + char *redirect; + char *lowerdata_redirect; +}; +void ovl_inode_init(struct inode *inode, struct ovl_inode_params *oip, + unsigned long ino, int fsid); +struct inode *ovl_new_inode(struct super_block *sb, umode_t mode, dev_t rdev); +struct inode *ovl_lookup_inode(struct super_block *sb, struct dentry *real, + bool is_upper); +bool ovl_lookup_trap_inode(struct super_block *sb, struct dentry *dir); +struct inode *ovl_get_trap_inode(struct super_block *sb, struct dentry *dir); +struct inode *ovl_get_inode(struct super_block *sb, + struct ovl_inode_params *oip); +void ovl_copyattr(struct inode *to); + +/* vfs fileattr flags read from overlay.protattr xattr to ovl inode */ +#define OVL_PROT_I_FLAGS_MASK (S_APPEND | S_IMMUTABLE) +/* vfs fileattr flags copied from real to ovl inode */ +#define OVL_FATTR_I_FLAGS_MASK (OVL_PROT_I_FLAGS_MASK | S_SYNC | S_NOATIME) +/* vfs inode flags copied from real to ovl inode */ +#define OVL_COPY_I_FLAGS_MASK (OVL_FATTR_I_FLAGS_MASK | S_CASEFOLD) + +/* + * fileattr flags copied from lower to upper inode on copy up. + * We cannot copy up immutable/append-only flags, because that would prevent + * linking temp inode to upper dir, so we store them in xattr instead. + */ +#define OVL_COPY_FS_FLAGS_MASK (FS_SYNC_FL | FS_NOATIME_FL) +#define OVL_COPY_FSX_FLAGS_MASK (FS_XFLAG_SYNC | FS_XFLAG_NOATIME) +#define OVL_PROT_FS_FLAGS_MASK (FS_APPEND_FL | FS_IMMUTABLE_FL) +#define OVL_PROT_FSX_FLAGS_MASK (FS_XFLAG_APPEND | FS_XFLAG_IMMUTABLE) + +void ovl_check_protattr(struct inode *inode, struct dentry *upper); +int ovl_set_protattr(struct inode *inode, struct dentry *upper, + struct file_kattr *fa); + +static inline void ovl_copyflags(struct inode *from, struct inode *to) +{ + unsigned int mask = OVL_COPY_I_FLAGS_MASK; + + inode_set_flags(to, from->i_flags & mask, mask); +} + +/* dir.c */ +extern const struct inode_operations ovl_dir_inode_operations; +int ovl_cleanup_and_whiteout(struct ovl_fs *ofs, struct dentry *dir, + struct dentry *dentry); +struct ovl_cattr { + dev_t rdev; + umode_t mode; + const char *link; + struct dentry *hardlink; +}; + +#define OVL_CATTR(m) (&(struct ovl_cattr) { .mode = (m) }) + +struct dentry *ovl_create_real(struct ovl_fs *ofs, + struct dentry *parent, struct dentry *newdentry, + struct qstr *qname, + struct ovl_cattr *attr); +int ovl_cleanup(struct ovl_fs *ofs, struct dentry *workdir, struct dentry *dentry); +#define OVL_TEMPNAME_SIZE 20 +void ovl_tempname(char name[OVL_TEMPNAME_SIZE]); +struct dentry *ovl_create_temp(struct ovl_fs *ofs, struct dentry *workdir, + struct ovl_cattr *attr); + +/* file.c */ +extern const struct file_operations ovl_file_operations; +int ovl_real_fileattr_get(const struct path *realpath, struct file_kattr *fa); +int ovl_real_fileattr_set(const struct path *realpath, struct file_kattr *fa); +int ovl_fileattr_get(struct dentry *dentry, struct file_kattr *fa); +int ovl_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa); +struct ovl_file; +struct ovl_file *ovl_file_alloc(struct file *realfile); +void ovl_file_free(struct ovl_file *of); + +/* copy_up.c */ +int ovl_copy_up(struct dentry *dentry); +int ovl_copy_up_with_data(struct dentry *dentry); +int ovl_maybe_copy_up(struct dentry *dentry, int flags); +int ovl_copy_xattr(struct super_block *sb, const struct path *path, struct dentry *new); +int ovl_set_attr(struct ovl_fs *ofs, struct dentry *upper, struct kstat *stat); +struct ovl_fh *ovl_encode_real_fh(struct ovl_fs *ofs, struct inode *realinode, + bool is_upper); +struct ovl_fh *ovl_get_origin_fh(struct ovl_fs *ofs, struct dentry *origin); +int ovl_set_origin_fh(struct ovl_fs *ofs, const struct ovl_fh *fh, + struct dentry *upper); + +/* export.c */ +extern const struct export_operations ovl_export_operations; +extern const struct export_operations ovl_export_fid_operations; + +/* super.c */ +int ovl_fill_super(struct super_block *sb, struct fs_context *fc); + +/* Will this overlay be forced to mount/remount ro? */ +static inline bool ovl_force_readonly(struct ovl_fs *ofs) +{ + return (!ovl_upper_mnt(ofs) || !ofs->workdir); +} + +/* xattr.c */ + +const struct xattr_handler * const *ovl_xattr_handlers(struct ovl_fs *ofs); +int ovl_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr); +int ovl_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags); +ssize_t ovl_listxattr(struct dentry *dentry, char *list, size_t size); diff --git a/fs/overlayfs/ovl_entry.h b/fs/overlayfs/ovl_entry.h new file mode 100644 index 000000000..80cad4ea9 --- /dev/null +++ b/fs/overlayfs/ovl_entry.h @@ -0,0 +1,193 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * + * Copyright (C) 2011 Novell Inc. + * Copyright (C) 2016 Red Hat, Inc. + */ + +struct ovl_config { + char *upperdir; + char *workdir; + char **lowerdirs; + bool default_permissions; + int redirect_mode; + int verity_mode; + bool index; + int uuid; + bool nfs_export; + int xino; + bool metacopy; + bool userxattr; + int fsync_mode; +}; + +struct ovl_sb { + struct super_block *sb; + dev_t pseudo_dev; + /* Unusable (conflicting) uuid */ + bool bad_uuid; + /* Used as a lower layer (but maybe also as upper) */ + bool is_lower; +}; + +struct ovl_layer { + /* ovl_free_fs() relies on @mnt being the first member! */ + struct vfsmount *mnt; + /* Trap in ovl inode cache */ + struct inode *trap; + struct ovl_sb *fs; + /* Index of this layer in fs root (upper idx == 0) */ + int idx; + /* One fsid per unique underlying sb (upper fsid == 0) */ + int fsid; + /* xwhiteouts were found on this layer */ + bool has_xwhiteouts; +}; + +struct ovl_path { + const struct ovl_layer *layer; + struct dentry *dentry; +}; + +struct ovl_entry { + unsigned int __numlower; + struct ovl_path __lowerstack[] __counted_by(__numlower); +}; + +/* private information held for overlayfs's superblock */ +struct ovl_fs { + unsigned int numlayer; + /* Number of unique fs among layers including upper fs */ + unsigned int numfs; + /* Number of data-only lower layers */ + unsigned int numdatalayer; + struct ovl_layer *layers; + struct ovl_sb *fs; + /* workbasedir is the path at workdir= mount option */ + struct dentry *workbasedir; + /* workdir is the 'work' or 'index' directory under workbasedir */ + struct dentry *workdir; + long namelen; + /* pathnames of lower and upper dirs, for show_options */ + struct ovl_config config; + /* creds of process who forced instantiation of super block */ + const struct cred *creator_cred; + bool tmpfile; + bool noxattr; + bool nofh; + /* Did we take the inuse lock? */ + bool upperdir_locked; + bool workdir_locked; + /* Traps in ovl inode cache */ + struct inode *workbasedir_trap; + struct inode *workdir_trap; + /* -1: disabled, 0: same fs, 1..32: number of unused ino bits */ + int xino_mode; + /* For allocation of non-persistent inode numbers */ + atomic_long_t last_ino; + /* Shared whiteout cache */ + struct dentry *whiteout; + bool no_shared_whiteout; + struct mutex whiteout_lock; + /* r/o snapshot of upperdir sb's only taken on volatile mounts */ + errseq_t errseq; + bool casefold; +}; + +/* Number of lower layers, not including data-only layers */ +static inline unsigned int ovl_numlowerlayer(struct ovl_fs *ofs) +{ + return ofs->numlayer - ofs->numdatalayer - 1; +} + +static inline struct vfsmount *ovl_upper_mnt(struct ovl_fs *ofs) +{ + return ofs->layers[0].mnt; +} + +static inline struct mnt_idmap *ovl_upper_mnt_idmap(struct ovl_fs *ofs) +{ + return mnt_idmap(ovl_upper_mnt(ofs)); +} + +extern struct file_system_type ovl_fs_type; + +static inline struct ovl_fs *OVL_FS(struct super_block *sb) +{ + if (IS_ENABLED(CONFIG_OVERLAY_FS_DEBUG)) + WARN_ON_ONCE(sb->s_type != &ovl_fs_type); + + return (struct ovl_fs *)sb->s_fs_info; +} + +static inline unsigned int ovl_numlower(struct ovl_entry *oe) +{ + return oe ? oe->__numlower : 0; +} + +static inline struct ovl_path *ovl_lowerstack(struct ovl_entry *oe) +{ + return ovl_numlower(oe) ? oe->__lowerstack : NULL; +} + +static inline struct ovl_path *ovl_lowerpath(struct ovl_entry *oe) +{ + return ovl_lowerstack(oe); +} + +static inline struct ovl_path *ovl_lowerdata(struct ovl_entry *oe) +{ + struct ovl_path *lowerstack = ovl_lowerstack(oe); + + return lowerstack ? &lowerstack[oe->__numlower - 1] : NULL; +} + +/* May return NULL if lazy lookup of lowerdata is needed */ +static inline struct dentry *ovl_lowerdata_dentry(struct ovl_entry *oe) +{ + struct ovl_path *lowerdata = ovl_lowerdata(oe); + + return lowerdata ? READ_ONCE(lowerdata->dentry) : NULL; +} + +/* private information held for every overlayfs dentry */ +static inline unsigned long *OVL_E_FLAGS(struct dentry *dentry) +{ + return (unsigned long *) &dentry->d_fsdata; +} + +struct ovl_inode { + union { + struct ovl_dir_cache *cache; /* directory */ + const char *lowerdata_redirect; /* regular file */ + }; + const char *redirect; + u64 version; + unsigned long flags; + struct inode vfs_inode; + struct dentry *__upperdentry; + struct ovl_entry *oe; + + /* synchronize copy up and more */ + struct mutex lock; +}; + +static inline struct ovl_inode *OVL_I(struct inode *inode) +{ + return container_of(inode, struct ovl_inode, vfs_inode); +} + +static inline struct ovl_entry *OVL_I_E(struct inode *inode) +{ + return inode ? OVL_I(inode)->oe : NULL; +} + +static inline struct ovl_entry *OVL_E(struct dentry *dentry) +{ + return OVL_I_E(d_inode(dentry)); +} + +static inline struct dentry *ovl_upperdentry_dereference(struct ovl_inode *oi) +{ + return READ_ONCE(oi->__upperdentry); +} diff --git a/fs/overlayfs/params.c b/fs/overlayfs/params.c new file mode 100644 index 000000000..c93fcaa45 --- /dev/null +++ b/fs/overlayfs/params.c @@ -0,0 +1,1104 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include +#include +#include +#include "overlayfs.h" +#include "params.h" + +static bool ovl_redirect_dir_def = IS_ENABLED(CONFIG_OVERLAY_FS_REDIRECT_DIR); +module_param_named(redirect_dir, ovl_redirect_dir_def, bool, 0644); +MODULE_PARM_DESC(redirect_dir, + "Default to on or off for the redirect_dir feature"); + +static bool ovl_redirect_always_follow = + IS_ENABLED(CONFIG_OVERLAY_FS_REDIRECT_ALWAYS_FOLLOW); +module_param_named(redirect_always_follow, ovl_redirect_always_follow, + bool, 0644); +MODULE_PARM_DESC(redirect_always_follow, + "Follow redirects even if redirect_dir feature is turned off"); + +static bool ovl_xino_auto_def = IS_ENABLED(CONFIG_OVERLAY_FS_XINO_AUTO); +module_param_named(xino_auto, ovl_xino_auto_def, bool, 0644); +MODULE_PARM_DESC(xino_auto, + "Auto enable xino feature"); + +static bool ovl_index_def = IS_ENABLED(CONFIG_OVERLAY_FS_INDEX); +module_param_named(index, ovl_index_def, bool, 0644); +MODULE_PARM_DESC(index, + "Default to on or off for the inodes index feature"); + +static bool ovl_nfs_export_def = IS_ENABLED(CONFIG_OVERLAY_FS_NFS_EXPORT); +module_param_named(nfs_export, ovl_nfs_export_def, bool, 0644); +MODULE_PARM_DESC(nfs_export, + "Default to on or off for the NFS export feature"); + +static bool ovl_metacopy_def = IS_ENABLED(CONFIG_OVERLAY_FS_METACOPY); +module_param_named(metacopy, ovl_metacopy_def, bool, 0644); +MODULE_PARM_DESC(metacopy, + "Default to on or off for the metadata only copy up feature"); + +enum ovl_opt { + Opt_lowerdir, + Opt_lowerdir_add, + Opt_datadir_add, + Opt_upperdir, + Opt_workdir, + Opt_default_permissions, + Opt_redirect_dir, + Opt_index, + Opt_uuid, + Opt_nfs_export, + Opt_userxattr, + Opt_xino, + Opt_metacopy, + Opt_verity, + Opt_fsync, + Opt_volatile, + Opt_override_creds, +}; + +static const struct constant_table ovl_parameter_bool[] = { + { "on", true }, + { "off", false }, + {} +}; + +static const struct constant_table ovl_parameter_uuid[] = { + { "off", OVL_UUID_OFF }, + { "null", OVL_UUID_NULL }, + { "auto", OVL_UUID_AUTO }, + { "on", OVL_UUID_ON }, + {} +}; + +static const char *ovl_uuid_mode(struct ovl_config *config) +{ + return ovl_parameter_uuid[config->uuid].name; +} + +static int ovl_uuid_def(void) +{ + return OVL_UUID_AUTO; +} + +static const struct constant_table ovl_parameter_xino[] = { + { "off", OVL_XINO_OFF }, + { "auto", OVL_XINO_AUTO }, + { "on", OVL_XINO_ON }, + {} +}; + +const char *ovl_xino_mode(struct ovl_config *config) +{ + return ovl_parameter_xino[config->xino].name; +} + +static int ovl_xino_def(void) +{ + return ovl_xino_auto_def ? OVL_XINO_AUTO : OVL_XINO_OFF; +} + +const struct constant_table ovl_parameter_redirect_dir[] = { + { "off", OVL_REDIRECT_OFF }, + { "follow", OVL_REDIRECT_FOLLOW }, + { "nofollow", OVL_REDIRECT_NOFOLLOW }, + { "on", OVL_REDIRECT_ON }, + {} +}; + +static const char *ovl_redirect_mode(struct ovl_config *config) +{ + return ovl_parameter_redirect_dir[config->redirect_mode].name; +} + +static int ovl_redirect_mode_def(void) +{ + return ovl_redirect_dir_def ? OVL_REDIRECT_ON : + ovl_redirect_always_follow ? OVL_REDIRECT_FOLLOW : + OVL_REDIRECT_NOFOLLOW; +} + +static const struct constant_table ovl_parameter_verity[] = { + { "off", OVL_VERITY_OFF }, + { "on", OVL_VERITY_ON }, + { "require", OVL_VERITY_REQUIRE }, + {} +}; + +static const char *ovl_verity_mode(struct ovl_config *config) +{ + return ovl_parameter_verity[config->verity_mode].name; +} + +static int ovl_verity_mode_def(void) +{ + return OVL_VERITY_OFF; +} + +static const struct constant_table ovl_parameter_fsync[] = { + { "volatile", OVL_FSYNC_VOLATILE }, + { "auto", OVL_FSYNC_AUTO }, + { "strict", OVL_FSYNC_STRICT }, + {} +}; + +static const char *ovl_fsync_mode(struct ovl_config *config) +{ + return ovl_parameter_fsync[config->fsync_mode].name; +} + +static int ovl_fsync_mode_def(void) +{ + return OVL_FSYNC_AUTO; +} + +const struct fs_parameter_spec ovl_parameter_spec[] = { + fsparam_string_empty("lowerdir", Opt_lowerdir), + fsparam_file_or_string("lowerdir+", Opt_lowerdir_add), + fsparam_file_or_string("datadir+", Opt_datadir_add), + fsparam_file_or_string("upperdir", Opt_upperdir), + fsparam_file_or_string("workdir", Opt_workdir), + fsparam_flag("default_permissions", Opt_default_permissions), + fsparam_enum("redirect_dir", Opt_redirect_dir, ovl_parameter_redirect_dir), + fsparam_enum("index", Opt_index, ovl_parameter_bool), + fsparam_enum("uuid", Opt_uuid, ovl_parameter_uuid), + fsparam_enum("nfs_export", Opt_nfs_export, ovl_parameter_bool), + fsparam_flag("userxattr", Opt_userxattr), + fsparam_enum("xino", Opt_xino, ovl_parameter_xino), + fsparam_enum("metacopy", Opt_metacopy, ovl_parameter_bool), + fsparam_enum("verity", Opt_verity, ovl_parameter_verity), + fsparam_enum("fsync", Opt_fsync, ovl_parameter_fsync), + fsparam_flag("volatile", Opt_volatile), + fsparam_flag_no("override_creds", Opt_override_creds), + {} +}; + +static char *ovl_next_opt(char **s) +{ + char *sbegin = *s; + char *p; + + if (sbegin == NULL) + return NULL; + + for (p = sbegin; *p; p++) { + if (*p == '\\') { + p++; + if (!*p) + break; + } else if (*p == ',') { + *p = '\0'; + *s = p + 1; + return sbegin; + } + } + *s = NULL; + return sbegin; +} + +static int ovl_parse_monolithic(struct fs_context *fc, void *data) +{ + return vfs_parse_monolithic_sep(fc, data, ovl_next_opt); +} + +static ssize_t ovl_parse_param_split_lowerdirs(char *str) +{ + ssize_t nr_layers = 1, nr_colons = 0; + char *s, *d; + + for (s = d = str;; s++, d++) { + if (*s == '\\') { + /* keep esc chars in split lowerdir */ + *d++ = *s++; + } else if (*s == ':') { + bool next_colon = (*(s + 1) == ':'); + + nr_colons++; + if (nr_colons == 2 && next_colon) { + pr_err("only single ':' or double '::' sequences of unescaped colons in lowerdir mount option allowed.\n"); + return -EINVAL; + } + /* count layers, not colons */ + if (!next_colon) + nr_layers++; + + *d = '\0'; + continue; + } + + *d = *s; + if (!*s) { + /* trailing colons */ + if (nr_colons) { + pr_err("unescaped trailing colons in lowerdir mount option.\n"); + return -EINVAL; + } + break; + } + nr_colons = 0; + } + + return nr_layers; +} + +static int ovl_mount_dir_noesc(const char *name, struct path *path) +{ + int err = -EINVAL; + + if (!*name) { + pr_err("empty lowerdir\n"); + goto out; + } + err = kern_path(name, LOOKUP_FOLLOW, path); + if (err) { + pr_err("failed to resolve '%s': %i\n", name, err); + goto out; + } + return 0; + +out: + return err; +} + +static void ovl_unescape(char *s) +{ + char *d = s; + + for (;; s++, d++) { + if (*s == '\\') + s++; + *d = *s; + if (!*s) + break; + } +} + +static int ovl_mount_dir(const char *name, struct path *path) +{ + int err = -ENOMEM; + char *tmp = kstrdup(name, GFP_KERNEL); + + if (tmp) { + ovl_unescape(tmp); + err = ovl_mount_dir_noesc(tmp, path); + kfree(tmp); + } + return err; +} + +static int ovl_mount_dir_check(struct fs_context *fc, const struct path *path, + enum ovl_opt layer, const char *name, bool upper) +{ + bool is_casefolded = ovl_dentry_casefolded(path->dentry); + struct ovl_fs_context *ctx = fc->fs_private; + struct ovl_fs *ofs = fc->s_fs_info; + + if (!d_is_dir(path->dentry)) + return invalfc(fc, "%s is not a directory", name); + + /* + * Allow filesystems that are case-folding capable but deny composing + * ovl stack from inconsistent case-folded directories. + */ + if (!ctx->casefold_set) { + ofs->casefold = is_casefolded; + ctx->casefold_set = true; + } + + if (ofs->casefold != is_casefolded) { + return invalfc(fc, "case-%ssensitive directory on %s is inconsistent", + is_casefolded ? "in" : "", name); + } + + if (ovl_dentry_weird(path->dentry)) + return invalfc(fc, "filesystem on %s not supported", name); + + /* + * Check whether upper path is read-only here to report failures + * early. Don't forget to recheck when the superblock is created + * as the mount attributes could change. + */ + if (upper) { + if (path->dentry->d_flags & DCACHE_OP_REAL) + return invalfc(fc, "filesystem on %s not supported as upperdir", name); + if (__mnt_is_readonly(path->mnt)) + return invalfc(fc, "filesystem on %s is read-only", name); + } else { + if (ctx->lowerdir_all && layer != Opt_lowerdir) + return invalfc(fc, "lowerdir+ and datadir+ cannot follow lowerdir"); + if (ctx->nr_data && layer == Opt_lowerdir_add) + return invalfc(fc, "regular lower layers cannot follow data layers"); + if (ctx->nr == OVL_MAX_STACK) + return invalfc(fc, "too many lower directories, limit is %d", + OVL_MAX_STACK); + } + return 0; +} + +static int ovl_ctx_realloc_lower(struct fs_context *fc) +{ + struct ovl_fs_context *ctx = fc->fs_private; + struct ovl_fs_context_layer *l; + size_t nr; + + if (ctx->nr < ctx->capacity) + return 0; + + nr = min_t(size_t, max(4096 / sizeof(*l), ctx->capacity * 2), + OVL_MAX_STACK); + l = krealloc_array(ctx->lower, nr, sizeof(*l), GFP_KERNEL_ACCOUNT); + if (!l) + return -ENOMEM; + + ctx->lower = l; + ctx->capacity = nr; + return 0; +} + +static void ovl_add_layer(struct fs_context *fc, enum ovl_opt layer, + struct path *path, char **pname) +{ + struct ovl_fs *ofs = fc->s_fs_info; + struct ovl_config *config = &ofs->config; + struct ovl_fs_context *ctx = fc->fs_private; + struct ovl_fs_context_layer *l; + + switch (layer) { + case Opt_workdir: + swap(config->workdir, *pname); + swap(ctx->work, *path); + break; + case Opt_upperdir: + swap(config->upperdir, *pname); + swap(ctx->upper, *path); + break; + case Opt_datadir_add: + ctx->nr_data++; + fallthrough; + case Opt_lowerdir: + fallthrough; + case Opt_lowerdir_add: + WARN_ON(ctx->nr >= ctx->capacity); + l = &ctx->lower[ctx->nr++]; + memset(l, 0, sizeof(*l)); + swap(l->name, *pname); + swap(l->path, *path); + break; + default: + WARN_ON(1); + } +} + +static inline bool is_upper_layer(enum ovl_opt layer) +{ + return layer == Opt_upperdir || layer == Opt_workdir; +} + +/* Handle non-file descriptor-based layer options that require path lookup. */ +static inline int ovl_kern_path(const char *layer_name, struct path *layer_path, + enum ovl_opt layer) +{ + int err; + + switch (layer) { + case Opt_upperdir: + fallthrough; + case Opt_workdir: + fallthrough; + case Opt_lowerdir: + err = ovl_mount_dir(layer_name, layer_path); + break; + case Opt_lowerdir_add: + fallthrough; + case Opt_datadir_add: + err = ovl_mount_dir_noesc(layer_name, layer_path); + break; + default: + WARN_ON_ONCE(true); + err = -EINVAL; + } + + return err; +} + +static int ovl_do_parse_layer(struct fs_context *fc, const char *layer_name, + struct path *layer_path, enum ovl_opt layer) +{ + char *name __free(kfree) = kstrdup(layer_name, GFP_KERNEL); + bool upper; + int err = 0; + + if (!name) + return -ENOMEM; + + upper = is_upper_layer(layer); + err = ovl_mount_dir_check(fc, layer_path, layer, name, upper); + if (err) + return err; + + if (!upper) { + err = ovl_ctx_realloc_lower(fc); + if (err) + return err; + } + + /* Store the user provided path string in ctx to show in mountinfo */ + ovl_add_layer(fc, layer, layer_path, &name); + return err; +} + +static int ovl_parse_layer(struct fs_context *fc, struct fs_parameter *param, + enum ovl_opt layer) +{ + struct path layer_path __free(path_put) = {}; + int err = 0; + + switch (param->type) { + case fs_value_is_string: + err = ovl_kern_path(param->string, &layer_path, layer); + if (err) + return err; + err = ovl_do_parse_layer(fc, param->string, &layer_path, layer); + break; + case fs_value_is_file: { + char *buf __free(kfree); + char *layer_name; + + buf = kmalloc(PATH_MAX, GFP_KERNEL_ACCOUNT); + if (!buf) + return -ENOMEM; + + layer_path = param->file->f_path; + path_get(&layer_path); + + layer_name = d_path(&layer_path, buf, PATH_MAX); + if (IS_ERR(layer_name)) + return PTR_ERR(layer_name); + + err = ovl_do_parse_layer(fc, layer_name, &layer_path, layer); + break; + } + default: + WARN_ON_ONCE(true); + err = -EINVAL; + } + + return err; +} + +static void ovl_reset_lowerdirs(struct ovl_fs_context *ctx) +{ + struct ovl_fs_context_layer *l = ctx->lower; + + // Reset old user provided lowerdir string + kfree(ctx->lowerdir_all); + ctx->lowerdir_all = NULL; + + for (size_t nr = 0; nr < ctx->nr; nr++, l++) { + path_put(&l->path); + kfree(l->name); + l->name = NULL; + } + ctx->nr = 0; + ctx->nr_data = 0; +} + +/* + * Parse lowerdir= mount option: + * + * e.g.: lowerdir=/lower1:/lower2:/lower3::/data1::/data2 + * Set "/lower1", "/lower2", and "/lower3" as lower layers and + * "/data1" and "/data2" as data lower layers. Any existing lower + * layers are replaced. + */ +static int ovl_parse_param_lowerdir(const char *name, struct fs_context *fc) +{ + int err; + struct ovl_fs_context *ctx = fc->fs_private; + char *dup = NULL, *iter; + ssize_t nr_lower, nr; + bool data_layer = false; + + /* + * Ensure we're backwards compatible with mount(2) + * by allowing relative paths. + */ + + /* drop all existing lower layers */ + ovl_reset_lowerdirs(ctx); + + if (!*name) + return 0; + + if (*name == ':') { + pr_err("cannot append lower layer\n"); + return -EINVAL; + } + + // Store user provided lowerdir string to show in mount options + ctx->lowerdir_all = kstrdup(name, GFP_KERNEL); + if (!ctx->lowerdir_all) + return -ENOMEM; + + dup = kstrdup(name, GFP_KERNEL); + if (!dup) + return -ENOMEM; + + err = -EINVAL; + nr_lower = ovl_parse_param_split_lowerdirs(dup); + if (nr_lower < 0) + goto out_err; + + if (nr_lower > OVL_MAX_STACK) { + pr_err("too many lower directories, limit is %d\n", OVL_MAX_STACK); + goto out_err; + } + + iter = dup; + for (nr = 0; nr < nr_lower; nr++) { + struct path path __free(path_put) = {}; + + err = ovl_kern_path(iter, &path, Opt_lowerdir); + if (err) + goto out_err; + + err = ovl_do_parse_layer(fc, iter, &path, Opt_lowerdir); + if (err) + goto out_err; + + if (data_layer) + ctx->nr_data++; + + /* Calling strchr() again would overrun. */ + if (ctx->nr == nr_lower) + break; + + err = -EINVAL; + iter = strchr(iter, '\0') + 1; + if (*iter) { + /* + * This is a regular layer so we require that + * there are no data layers. + */ + if (ctx->nr_data > 0) { + pr_err("regular lower layers cannot follow data lower layers\n"); + goto out_err; + } + + data_layer = false; + continue; + } + + /* This is a data lower layer. */ + data_layer = true; + iter++; + } + kfree(dup); + return 0; + +out_err: + kfree(dup); + + /* Intentionally don't realloc to a smaller size. */ + return err; +} + +static int ovl_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + int err = 0; + struct fs_parse_result result; + struct ovl_fs *ofs = fc->s_fs_info; + struct ovl_config *config = &ofs->config; + struct ovl_fs_context *ctx = fc->fs_private; + int opt; + + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE) { + /* + * On remount overlayfs has always ignored all mount + * options no matter if malformed or not so for + * backwards compatibility we do the same here. + */ + if (fc->oldapi) + return 0; + + /* + * Give us the freedom to allow changing mount options + * with the new mount api in the future. So instead of + * silently ignoring everything we report a proper + * error. This is only visible for users of the new + * mount api. + */ + return invalfc(fc, "No changes allowed in reconfigure"); + } + + opt = fs_parse(fc, ovl_parameter_spec, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_lowerdir: + err = ovl_parse_param_lowerdir(param->string, fc); + break; + case Opt_lowerdir_add: + case Opt_datadir_add: + case Opt_upperdir: + case Opt_workdir: + err = ovl_parse_layer(fc, param, opt); + break; + case Opt_default_permissions: + config->default_permissions = true; + break; + case Opt_redirect_dir: + config->redirect_mode = result.uint_32; + if (config->redirect_mode == OVL_REDIRECT_OFF) { + config->redirect_mode = ovl_redirect_always_follow ? + OVL_REDIRECT_FOLLOW : + OVL_REDIRECT_NOFOLLOW; + } + ctx->set.redirect = true; + break; + case Opt_index: + config->index = result.uint_32; + ctx->set.index = true; + break; + case Opt_uuid: + config->uuid = result.uint_32; + break; + case Opt_nfs_export: + config->nfs_export = result.uint_32; + ctx->set.nfs_export = true; + break; + case Opt_xino: + config->xino = result.uint_32; + break; + case Opt_metacopy: + config->metacopy = result.uint_32; + ctx->set.metacopy = true; + break; + case Opt_verity: + config->verity_mode = result.uint_32; + break; + case Opt_fsync: + config->fsync_mode = result.uint_32; + break; + case Opt_volatile: + config->fsync_mode = OVL_FSYNC_VOLATILE; + break; + case Opt_userxattr: + config->userxattr = true; + break; + case Opt_override_creds: { + const struct cred *cred = NULL; + + if (result.negated) { + swap(cred, ofs->creator_cred); + put_cred(cred); + break; + } + + if (!current_in_userns(fc->user_ns)) { + err = -EINVAL; + break; + } + + cred = prepare_creds(); + if (cred) + swap(cred, ofs->creator_cred); + else + err = -ENOMEM; + + put_cred(cred); + break; + } + default: + pr_err("unrecognized mount option \"%s\" or missing value\n", + param->key); + return -EINVAL; + } + + return err; +} + +static int ovl_get_tree(struct fs_context *fc) +{ + return get_tree_nodev(fc, ovl_fill_super); +} + +static inline void ovl_fs_context_free(struct ovl_fs_context *ctx) +{ + ovl_reset_lowerdirs(ctx); + path_put(&ctx->upper); + path_put(&ctx->work); + kfree(ctx->lower); + kfree(ctx); +} + +static void ovl_free(struct fs_context *fc) +{ + struct ovl_fs *ofs = fc->s_fs_info; + struct ovl_fs_context *ctx = fc->fs_private; + + /* + * ofs is stored in the fs_context when it is initialized. + * ofs is transferred to the superblock on a successful mount, + * but if an error occurs before the transfer we have to free + * it here. + */ + if (ofs) + ovl_free_fs(ofs); + + if (ctx) + ovl_fs_context_free(ctx); +} + +static int ovl_reconfigure(struct fs_context *fc) +{ + struct super_block *sb = fc->root->d_sb; + struct ovl_fs *ofs = OVL_FS(sb); + struct super_block *upper_sb; + int ret = 0; + + if (!(fc->sb_flags & SB_RDONLY) && ovl_force_readonly(ofs)) + return -EROFS; + + if (fc->sb_flags & SB_RDONLY && !sb_rdonly(sb)) { + upper_sb = ovl_upper_mnt(ofs)->mnt_sb; + if (ovl_should_sync(ofs)) { + down_read(&upper_sb->s_umount); + ret = sync_filesystem(upper_sb); + up_read(&upper_sb->s_umount); + } + } + + return ret; +} + +static const struct fs_context_operations ovl_context_ops = { + .parse_monolithic = ovl_parse_monolithic, + .parse_param = ovl_parse_param, + .get_tree = ovl_get_tree, + .reconfigure = ovl_reconfigure, + .free = ovl_free, +}; + +/* + * This is called during fsopen() and will record the user namespace of + * the caller in fc->user_ns since we've raised FS_USERNS_MOUNT. We'll + * need it when we actually create the superblock to verify that the + * process creating the superblock is in the same user namespace as + * process that called fsopen(). + */ +int ovl_init_fs_context(struct fs_context *fc) +{ + struct ovl_fs_context *ctx; + struct ovl_fs *ofs; + + ctx = kzalloc_obj(*ctx, GFP_KERNEL_ACCOUNT); + if (!ctx) + return -ENOMEM; + + /* + * By default we allocate for three lower layers. It's likely + * that it'll cover most users. + */ + ctx->lower = kmalloc_objs(*ctx->lower, 3, GFP_KERNEL_ACCOUNT); + if (!ctx->lower) + goto out_err; + ctx->capacity = 3; + + ofs = kzalloc_obj(struct ovl_fs); + if (!ofs) + goto out_err; + + ofs->config.redirect_mode = ovl_redirect_mode_def(); + ofs->config.index = ovl_index_def; + ofs->config.uuid = ovl_uuid_def(); + ofs->config.nfs_export = ovl_nfs_export_def; + ofs->config.xino = ovl_xino_def(); + ofs->config.metacopy = ovl_metacopy_def; + ofs->config.fsync_mode = ovl_fsync_mode_def(); + + fc->s_fs_info = ofs; + fc->fs_private = ctx; + fc->ops = &ovl_context_ops; + + mutex_init(&ofs->whiteout_lock); + return 0; + +out_err: + ovl_fs_context_free(ctx); + return -ENOMEM; + +} + +void ovl_free_fs(struct ovl_fs *ofs) +{ + struct vfsmount **mounts; + unsigned i; + + iput(ofs->workbasedir_trap); + iput(ofs->workdir_trap); + dput(ofs->whiteout); + dput(ofs->workdir); + if (ofs->workdir_locked) + ovl_inuse_unlock(ofs->workbasedir); + dput(ofs->workbasedir); + if (ofs->upperdir_locked) + ovl_inuse_unlock(ovl_upper_mnt(ofs)->mnt_root); + + /* Reuse ofs->config.lowerdirs as a vfsmount array before freeing it */ + mounts = (struct vfsmount **) ofs->config.lowerdirs; + for (i = 0; i < ofs->numlayer; i++) { + iput(ofs->layers[i].trap); + kfree(ofs->config.lowerdirs[i]); + mounts[i] = ofs->layers[i].mnt; + } + kern_unmount_array(mounts, ofs->numlayer); + kfree(ofs->layers); + for (i = 0; i < ofs->numfs; i++) + free_anon_bdev(ofs->fs[i].pseudo_dev); + kfree(ofs->fs); + + kfree(ofs->config.lowerdirs); + kfree(ofs->config.upperdir); + kfree(ofs->config.workdir); + if (ofs->creator_cred) + put_cred(ofs->creator_cred); + kfree(ofs); +} + +int ovl_fs_params_verify(const struct ovl_fs_context *ctx, + struct ovl_config *config) +{ + struct ovl_opt_set set = ctx->set; + + /* Workdir/index are useless in non-upper mount */ + if (!config->upperdir) { + if (config->workdir) { + pr_info("option \"workdir=%s\" is useless in a non-upper mount, ignore\n", + config->workdir); + kfree(config->workdir); + config->workdir = NULL; + } + if (config->index && set.index) { + pr_info("option \"index=on\" is useless in a non-upper mount, ignore\n"); + set.index = false; + } + config->index = false; + } + + if (!config->upperdir && ovl_is_volatile(config)) { + pr_info("option \"volatile\" is meaningless in a non-upper mount, ignoring it.\n"); + config->fsync_mode = ovl_fsync_mode_def(); + } + + if (!config->upperdir && config->uuid == OVL_UUID_ON) { + pr_info("option \"uuid=on\" requires an upper fs, falling back to uuid=null.\n"); + config->uuid = OVL_UUID_NULL; + } + + /* + * This is to make the logic below simpler. It doesn't make any other + * difference, since redirect_dir=on is only used for upper. + */ + if (!config->upperdir && config->redirect_mode == OVL_REDIRECT_FOLLOW) + config->redirect_mode = OVL_REDIRECT_ON; + + /* metacopy -> redirect_dir dependency */ + if (config->metacopy && config->redirect_mode != OVL_REDIRECT_ON) { + if (set.metacopy && set.redirect) { + pr_err("conflicting options: metacopy=on,redirect_dir=%s\n", + ovl_redirect_mode(config)); + return -EINVAL; + } + if (set.redirect) { + /* + * There was an explicit redirect_dir=... that resulted + * in this conflict. + */ + pr_info("disabling metacopy due to redirect_dir=%s\n", + ovl_redirect_mode(config)); + config->metacopy = false; + } else { + /* Automatically enable redirect otherwise. */ + config->redirect_mode = OVL_REDIRECT_ON; + } + } + + /* Resolve nfs_export -> index dependency */ + if (config->nfs_export && !config->index) { + if (!config->upperdir && + config->redirect_mode != OVL_REDIRECT_NOFOLLOW) { + pr_info("NFS export requires \"redirect_dir=nofollow\" on non-upper mount, falling back to nfs_export=off.\n"); + config->nfs_export = false; + } else if (set.nfs_export && set.index) { + pr_err("conflicting options: nfs_export=on,index=off\n"); + return -EINVAL; + } else if (set.index) { + /* + * There was an explicit index=off that resulted + * in this conflict. + */ + pr_info("disabling nfs_export due to index=off\n"); + config->nfs_export = false; + } else { + /* Automatically enable index otherwise. */ + config->index = true; + } + } + + /* Resolve nfs_export -> !metacopy && !verity dependency */ + if (config->nfs_export && config->metacopy) { + if (set.nfs_export && set.metacopy) { + pr_err("conflicting options: nfs_export=on,metacopy=on\n"); + return -EINVAL; + } + if (set.metacopy) { + /* + * There was an explicit metacopy=on that resulted + * in this conflict. + */ + pr_info("disabling nfs_export due to metacopy=on\n"); + config->nfs_export = false; + } else if (config->verity_mode) { + /* + * There was an explicit verity=.. that resulted + * in this conflict. + */ + pr_info("disabling nfs_export due to verity=%s\n", + ovl_verity_mode(config)); + config->nfs_export = false; + } else { + /* + * There was an explicit nfs_export=on that resulted + * in this conflict. + */ + pr_info("disabling metacopy due to nfs_export=on\n"); + config->metacopy = false; + } + } + + + /* Resolve userxattr -> !redirect && !metacopy dependency */ + if (config->userxattr) { + if (set.redirect && + config->redirect_mode != OVL_REDIRECT_NOFOLLOW) { + pr_err("conflicting options: userxattr,redirect_dir=%s\n", + ovl_redirect_mode(config)); + return -EINVAL; + } + if (config->metacopy && set.metacopy) { + pr_err("conflicting options: userxattr,metacopy=on\n"); + return -EINVAL; + } + /* + * Silently disable default setting of redirect and metacopy. + * This shall be the default in the future as well: these + * options must be explicitly enabled if used together with + * userxattr. + */ + config->redirect_mode = OVL_REDIRECT_NOFOLLOW; + config->metacopy = false; + } + + /* + * Fail if we don't have trusted xattr capability and a feature was + * explicitly requested that requires them. + */ + if (!config->userxattr && !capable(CAP_SYS_ADMIN)) { + if (set.redirect && + config->redirect_mode != OVL_REDIRECT_NOFOLLOW) { + pr_err("redirect_dir requires permission to access trusted xattrs\n"); + return -EPERM; + } + if (config->metacopy && set.metacopy) { + pr_err("metacopy requires permission to access trusted xattrs\n"); + return -EPERM; + } + if (config->verity_mode) { + pr_err("verity requires permission to access trusted xattrs\n"); + return -EPERM; + } + if (ctx->nr_data > 0) { + pr_err("lower data-only dirs require permission to access trusted xattrs\n"); + return -EPERM; + } + /* + * Other xattr-dependent features should be disabled without + * great disturbance to the user in ovl_make_workdir(). + */ + } + + return 0; +} + +/** + * ovl_show_options + * @m: the seq_file handle + * @dentry: The dentry to query + * + * Prints the mount options for a given superblock. + * Returns zero; does not fail. + */ +int ovl_show_options(struct seq_file *m, struct dentry *dentry) +{ + struct super_block *sb = dentry->d_sb; + struct ovl_fs *ofs = OVL_FS(sb); + size_t nr, nr_merged_lower, nr_lower = 0; + char **lowerdirs = ofs->config.lowerdirs; + + /* + * lowerdirs[0] holds the colon separated list that user provided + * with lowerdir mount option. + * lowerdirs[1..numlayer] hold the lowerdir paths that were added + * using the lowerdir+ and datadir+ mount options. + * For now, we do not allow mixing the legacy lowerdir mount option + * with the new lowerdir+ and datadir+ mount options. + */ + if (lowerdirs[0]) { + seq_show_option(m, "lowerdir", lowerdirs[0]); + } else { + nr_lower = ofs->numlayer; + nr_merged_lower = nr_lower - ofs->numdatalayer; + } + for (nr = 1; nr < nr_lower; nr++) { + if (nr < nr_merged_lower) + seq_show_option(m, "lowerdir+", lowerdirs[nr]); + else + seq_show_option(m, "datadir+", lowerdirs[nr]); + } + if (ofs->config.upperdir) { + seq_show_option(m, "upperdir", ofs->config.upperdir); + seq_show_option(m, "workdir", ofs->config.workdir); + } + if (ofs->config.default_permissions) + seq_puts(m, ",default_permissions"); + if (ofs->config.redirect_mode != ovl_redirect_mode_def()) + seq_printf(m, ",redirect_dir=%s", + ovl_redirect_mode(&ofs->config)); + if (ofs->config.index != ovl_index_def) + seq_printf(m, ",index=%s", str_on_off(ofs->config.index)); + if (ofs->config.uuid != ovl_uuid_def()) + seq_printf(m, ",uuid=%s", ovl_uuid_mode(&ofs->config)); + if (ofs->config.nfs_export != ovl_nfs_export_def) + seq_printf(m, ",nfs_export=%s", + str_on_off(ofs->config.nfs_export)); + if (ofs->config.xino != ovl_xino_def() && !ovl_same_fs(ofs)) + seq_printf(m, ",xino=%s", ovl_xino_mode(&ofs->config)); + if (ofs->config.metacopy != ovl_metacopy_def) + seq_printf(m, ",metacopy=%s", str_on_off(ofs->config.metacopy)); + if (ofs->config.fsync_mode != ovl_fsync_mode_def()) + seq_printf(m, ",fsync=%s", ovl_fsync_mode(&ofs->config)); + if (ofs->config.userxattr) + seq_puts(m, ",userxattr"); + if (ofs->config.verity_mode != ovl_verity_mode_def()) + seq_printf(m, ",verity=%s", + ovl_verity_mode(&ofs->config)); + return 0; +} diff --git a/fs/overlayfs/params.h b/fs/overlayfs/params.h new file mode 100644 index 000000000..ffd53cdd8 --- /dev/null +++ b/fs/overlayfs/params.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include +#include + +struct ovl_fs; +struct ovl_config; + +extern const struct fs_parameter_spec ovl_parameter_spec[]; +extern const struct constant_table ovl_parameter_redirect_dir[]; + +/* The set of options that user requested explicitly via mount options */ +struct ovl_opt_set { + bool metacopy; + bool redirect; + bool nfs_export; + bool index; +}; + +#define OVL_MAX_STACK 500 + +struct ovl_fs_context_layer { + char *name; + struct path path; +}; + +struct ovl_fs_context { + struct path upper; + struct path work; + size_t capacity; + size_t nr; /* includes nr_data */ + size_t nr_data; + struct ovl_opt_set set; + struct ovl_fs_context_layer *lower; + char *lowerdir_all; /* user provided lowerdir string */ + bool casefold_set; +}; + +int ovl_init_fs_context(struct fs_context *fc); +void ovl_free_fs(struct ovl_fs *ofs); +int ovl_fs_params_verify(const struct ovl_fs_context *ctx, + struct ovl_config *config); +int ovl_show_options(struct seq_file *m, struct dentry *dentry); +const char *ovl_xino_mode(struct ovl_config *config); diff --git a/fs/overlayfs/readdir.c b/fs/overlayfs/readdir.c new file mode 100644 index 000000000..e7fe29cb6 --- /dev/null +++ b/fs/overlayfs/readdir.c @@ -0,0 +1,1326 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * + * Copyright (C) 2011 Novell Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "overlayfs.h" + +struct ovl_cache_entry { + unsigned int len; + unsigned int type; + u64 real_ino; + u64 ino; + struct list_head l_node; + struct rb_node node; + struct ovl_cache_entry *next_maybe_whiteout; + bool is_upper; + bool is_whiteout; + bool check_xwhiteout; + const char *c_name; + int c_len; + char name[]; +}; + +struct ovl_dir_cache { + long refcount; + u64 version; + struct list_head entries; + struct rb_root root; +}; + +struct ovl_readdir_data { + struct dir_context ctx; + struct dentry *dentry; + bool is_lowest; + struct rb_root *root; + struct list_head *list; + struct list_head middle; + struct ovl_cache_entry *first_maybe_whiteout; + struct unicode_map *map; + int count; + int err; + bool is_upper; + bool d_type_supported; + bool in_xwhiteouts_dir; +}; + +struct ovl_dir_file { + bool is_real; + bool is_upper; + struct ovl_dir_cache *cache; + struct list_head *cursor; + struct file *realfile; + struct file *upperfile; +}; + +static struct ovl_cache_entry *ovl_cache_entry_from_node(struct rb_node *n) +{ + return rb_entry(n, struct ovl_cache_entry, node); +} + +static int ovl_casefold(struct ovl_readdir_data *rdd, const char *str, int len, + char **dst) +{ + const struct qstr qstr = { .name = str, .len = len }; + char *cf_name; + int cf_len; + + if (!IS_ENABLED(CONFIG_UNICODE) || !rdd->map || + name_is_dot_dotdot(str, len)) + return 0; + + cf_name = kmalloc(NAME_MAX, GFP_KERNEL); + if (!cf_name) { + rdd->err = -ENOMEM; + return -ENOMEM; + } + + cf_len = utf8_casefold(rdd->map, &qstr, cf_name, NAME_MAX); + if (cf_len > 0) + *dst = cf_name; + else + kfree(cf_name); + + return cf_len; +} + +static bool ovl_cache_entry_find_link(const char *name, int len, + struct rb_node ***link, + struct rb_node **parent) +{ + bool found = false; + struct rb_node **newp = *link; + + while (!found && *newp) { + int cmp; + struct ovl_cache_entry *tmp; + + *parent = *newp; + tmp = ovl_cache_entry_from_node(*newp); + cmp = strncmp(name, tmp->c_name, len); + if (cmp > 0) + newp = &tmp->node.rb_right; + else if (cmp < 0 || len < tmp->c_len) + newp = &tmp->node.rb_left; + else + found = true; + } + *link = newp; + + return found; +} + +static struct ovl_cache_entry *ovl_cache_entry_find(struct rb_root *root, + const char *name, int len) +{ + struct rb_node *node = root->rb_node; + int cmp; + + while (node) { + struct ovl_cache_entry *p = ovl_cache_entry_from_node(node); + + cmp = strncmp(name, p->c_name, len); + if (cmp > 0) + node = p->node.rb_right; + else if (cmp < 0 || len < p->c_len) + node = p->node.rb_left; + else + return p; + } + + return NULL; +} + +static bool ovl_calc_d_ino(struct ovl_readdir_data *rdd, + struct ovl_cache_entry *p) +{ + /* Don't care if not doing ovl_iter() */ + if (!rdd->dentry) + return false; + + /* Always recalc d_ino when remapping lower inode numbers */ + if (ovl_xino_bits(OVL_FS(rdd->dentry->d_sb))) + return true; + + /* Always recalc d_ino for parent */ + if (name_is_dotdot(p->name, p->len)) + return true; + + /* If this is lower, then native d_ino will do */ + if (!rdd->is_upper) + return false; + + /* + * Recalc d_ino for '.' and for all entries if dir is impure (contains + * copied up entries) + */ + if (name_is_dot(p->name, p->len) || + ovl_test_flag(OVL_IMPURE, d_inode(rdd->dentry))) + return true; + + return false; +} + +static struct ovl_cache_entry *ovl_cache_entry_new(struct ovl_readdir_data *rdd, + const char *name, int len, + const char *c_name, int c_len, + u64 ino, unsigned int d_type) +{ + struct ovl_cache_entry *p; + + p = kmalloc_flex(*p, name, len + 1); + if (!p) + return NULL; + + memcpy(p->name, name, len); + p->name[len] = '\0'; + p->len = len; + p->type = d_type; + p->real_ino = ino; + p->ino = ino; + /* Defer setting d_ino for upper entry to ovl_iterate() */ + if (ovl_calc_d_ino(rdd, p)) + p->ino = 0; + p->is_upper = rdd->is_upper; + p->is_whiteout = false; + /* Defer check for overlay.whiteout to ovl_iterate() */ + p->check_xwhiteout = rdd->in_xwhiteouts_dir && d_type == DT_REG; + + if (c_name && c_name != name) { + p->c_name = c_name; + p->c_len = c_len; + } else { + p->c_name = p->name; + p->c_len = len; + } + + if (d_type == DT_CHR) { + p->next_maybe_whiteout = rdd->first_maybe_whiteout; + rdd->first_maybe_whiteout = p; + } + return p; +} + +/* Return 0 for found, 1 for added, <0 for error */ +static int ovl_cache_entry_add_rb(struct ovl_readdir_data *rdd, + const char *name, int len, + const char *c_name, int c_len, + u64 ino, + unsigned int d_type) +{ + struct rb_node **newp = &rdd->root->rb_node; + struct rb_node *parent = NULL; + struct ovl_cache_entry *p; + + if (ovl_cache_entry_find_link(c_name, c_len, &newp, &parent)) + return 0; + + p = ovl_cache_entry_new(rdd, name, len, c_name, c_len, ino, d_type); + if (p == NULL) { + rdd->err = -ENOMEM; + return -ENOMEM; + } + + list_add_tail(&p->l_node, rdd->list); + rb_link_node(&p->node, parent, newp); + rb_insert_color(&p->node, rdd->root); + + return 1; +} + +/* Return 0 for found, 1 for added, <0 for error */ +static int ovl_fill_lowest(struct ovl_readdir_data *rdd, + const char *name, int namelen, + const char *c_name, int c_len, + loff_t offset, u64 ino, unsigned int d_type) +{ + struct ovl_cache_entry *p; + + p = ovl_cache_entry_find(rdd->root, c_name, c_len); + if (p) { + list_move_tail(&p->l_node, &rdd->middle); + return 0; + } else { + p = ovl_cache_entry_new(rdd, name, namelen, c_name, c_len, + ino, d_type); + if (p == NULL) + rdd->err = -ENOMEM; + else + list_add_tail(&p->l_node, &rdd->middle); + } + + return rdd->err ?: 1; +} + +static void ovl_cache_entry_free(struct ovl_cache_entry *p) +{ + if (p->c_name != p->name) + kfree(p->c_name); + kfree(p); +} + +void ovl_cache_free(struct list_head *list) +{ + struct ovl_cache_entry *p; + struct ovl_cache_entry *n; + + list_for_each_entry_safe(p, n, list, l_node) + ovl_cache_entry_free(p); + + INIT_LIST_HEAD(list); +} + +void ovl_dir_cache_free(struct inode *inode) +{ + struct ovl_dir_cache *cache = ovl_dir_cache(inode); + + if (cache) { + ovl_cache_free(&cache->entries); + kfree(cache); + } +} + +static void ovl_cache_put(struct ovl_dir_file *od, struct inode *inode) +{ + struct ovl_dir_cache *cache = od->cache; + + WARN_ON(cache->refcount <= 0); + cache->refcount--; + if (!cache->refcount) { + if (ovl_dir_cache(inode) == cache) + ovl_set_dir_cache(inode, NULL); + + ovl_cache_free(&cache->entries); + kfree(cache); + } +} + +static bool ovl_fill_merge(struct dir_context *ctx, const char *name, + int namelen, loff_t offset, u64 ino, + unsigned int d_type) +{ + struct ovl_readdir_data *rdd = + container_of(ctx, struct ovl_readdir_data, ctx); + struct ovl_fs *ofs = OVL_FS(rdd->dentry->d_sb); + const char *c_name = NULL; + char *cf_name = NULL; + int c_len = 0, ret; + + if (ofs->casefold) + c_len = ovl_casefold(rdd, name, namelen, &cf_name); + + if (rdd->err) + return false; + + if (c_len <= 0) { + c_name = name; + c_len = namelen; + } else { + c_name = cf_name; + } + + rdd->count++; + if (!rdd->is_lowest) + ret = ovl_cache_entry_add_rb(rdd, name, namelen, c_name, c_len, ino, d_type); + else + ret = ovl_fill_lowest(rdd, name, namelen, c_name, c_len, offset, ino, d_type); + + /* + * If ret == 1, that means that c_name is being used as part of struct + * ovl_cache_entry and will be freed at ovl_cache_free(). Otherwise, + * c_name was found in the rb-tree so we can free it here. + */ + if (ret != 1 && c_name != name) + kfree(c_name); + + return ret >= 0; +} + +static int ovl_check_whiteouts(const struct path *path, struct ovl_readdir_data *rdd) +{ + struct dentry *dentry, *dir = path->dentry; + + while (rdd->first_maybe_whiteout) { + struct ovl_cache_entry *p = + rdd->first_maybe_whiteout; + rdd->first_maybe_whiteout = p->next_maybe_whiteout; + dentry = lookup_one_positive_killable(mnt_idmap(path->mnt), + &QSTR_LEN(p->name, p->len), + dir); + if (!IS_ERR(dentry)) { + p->is_whiteout = ovl_is_whiteout(dentry); + dput(dentry); + } else if (PTR_ERR(dentry) == -EINTR) { + return -EINTR; + } + } + + return 0; +} + +static inline int ovl_dir_read(const struct path *realpath, + struct ovl_readdir_data *rdd) +{ + struct file *realfile; + int err; + + realfile = ovl_path_open(realpath, O_RDONLY | O_LARGEFILE); + if (IS_ERR(realfile)) + return PTR_ERR(realfile); + + rdd->first_maybe_whiteout = NULL; + rdd->ctx.pos = 0; + do { + rdd->count = 0; + rdd->err = 0; + err = iterate_dir(realfile, &rdd->ctx); + if (err >= 0) + err = rdd->err; + } while (!err && rdd->count); + + if (!err && rdd->first_maybe_whiteout && rdd->dentry) + err = ovl_check_whiteouts(realpath, rdd); + + fput(realfile); + + return err; +} + +static void ovl_dir_reset(struct file *file) +{ + struct ovl_dir_file *od = file->private_data; + struct ovl_dir_cache *cache = od->cache; + struct inode *inode = file_inode(file); + bool is_real; + + if (cache && ovl_inode_version_get(inode) != cache->version) { + ovl_cache_put(od, inode); + od->cache = NULL; + od->cursor = NULL; + } + is_real = ovl_dir_is_real(inode); + if (od->is_real != is_real) { + /* is_real can only become false when dir is copied up */ + if (WARN_ON(is_real)) + return; + od->is_real = false; + } +} + +static int ovl_dir_read_merged(struct dentry *dentry, struct list_head *list, + struct rb_root *root) +{ + int err; + struct path realpath; + struct ovl_readdir_data rdd = { + .ctx.actor = ovl_fill_merge, + .ctx.count = INT_MAX, + .dentry = dentry, + .list = list, + .root = root, + .is_lowest = false, + .map = NULL, + }; + int idx, next; + const struct ovl_layer *layer; + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + + for (idx = 0; idx != -1; idx = next) { + next = ovl_path_next(idx, dentry, &realpath, &layer); + + if (ofs->casefold) + rdd.map = sb_encoding(realpath.dentry->d_sb); + + rdd.is_upper = ovl_dentry_upper(dentry) == realpath.dentry; + rdd.in_xwhiteouts_dir = layer->has_xwhiteouts && + ovl_dentry_has_xwhiteouts(dentry); + + if (next != -1) { + err = ovl_dir_read(&realpath, &rdd); + if (err) + break; + } else { + /* + * Insert lowest layer entries before upper ones, this + * allows offsets to be reasonably constant + */ + list_add(&rdd.middle, rdd.list); + rdd.is_lowest = true; + err = ovl_dir_read(&realpath, &rdd); + list_del(&rdd.middle); + } + } + return err; +} + +static void ovl_seek_cursor(struct ovl_dir_file *od, loff_t pos) +{ + struct list_head *p; + loff_t off = 0; + + list_for_each(p, &od->cache->entries) { + if (off >= pos) + break; + off++; + } + /* Cursor is safe since the cache is stable */ + od->cursor = p; +} + +static struct ovl_dir_cache *ovl_cache_get(struct dentry *dentry) +{ + int res; + struct ovl_dir_cache *cache; + struct inode *inode = d_inode(dentry); + + cache = ovl_dir_cache(inode); + if (cache && ovl_inode_version_get(inode) == cache->version) { + WARN_ON(!cache->refcount); + cache->refcount++; + return cache; + } + ovl_set_dir_cache(d_inode(dentry), NULL); + + cache = kzalloc_obj(struct ovl_dir_cache); + if (!cache) + return ERR_PTR(-ENOMEM); + + cache->refcount = 1; + INIT_LIST_HEAD(&cache->entries); + cache->root = RB_ROOT; + + res = ovl_dir_read_merged(dentry, &cache->entries, &cache->root); + if (res) { + ovl_cache_free(&cache->entries); + kfree(cache); + return ERR_PTR(res); + } + + cache->version = ovl_inode_version_get(inode); + ovl_set_dir_cache(inode, cache); + + return cache; +} + +/* Map inode number to lower fs unique range */ +static u64 ovl_remap_lower_ino(u64 ino, int xinobits, int fsid, + const char *name, int namelen, bool warn) +{ + unsigned int xinoshift = 64 - xinobits; + + if (unlikely(ino >> xinoshift)) { + if (warn) { + pr_warn_ratelimited("d_ino too big (%.*s, ino=%llu, xinobits=%d)\n", + namelen, name, ino, xinobits); + } + return ino; + } + + /* + * The lowest xinobit is reserved for mapping the non-peresistent inode + * numbers range, but this range is only exposed via st_ino, not here. + */ + return ino | ((u64)fsid) << (xinoshift + 1); +} + +/* + * Set d_ino for upper entries if needed. Non-upper entries should always report + * the uppermost real inode ino and should not call this function. + * + * When not all layer are on same fs, report real ino also for upper. + * + * When all layers are on the same fs, and upper has a reference to + * copy up origin, call vfs_getattr() on the overlay entry to make + * sure that d_ino will be consistent with st_ino from stat(2). + * + * Also checks the overlay.whiteout xattr by doing a full lookup which will return + * negative in this case. + */ +static int ovl_cache_update(const struct path *path, struct ovl_cache_entry *p, bool update_ino) + +{ + struct dentry *dir = path->dentry; + struct ovl_fs *ofs = OVL_FS(dir->d_sb); + struct dentry *this = NULL; + enum ovl_path_type type; + u64 ino = p->real_ino; + int xinobits = ovl_xino_bits(ofs); + int err = 0; + + if (!ovl_same_dev(ofs) && !p->check_xwhiteout) + goto out; + + if (name_is_dot_dotdot(p->name, p->len)) { + if (p->len == 1) { + this = dget(dir); + goto get; + } + if (p->len == 2) { + /* we shall not be moved */ + this = dget(dir->d_parent); + goto get; + } + } + /* This checks also for xwhiteouts */ + this = lookup_one(mnt_idmap(path->mnt), &QSTR_LEN(p->name, p->len), dir); + if (IS_ERR_OR_NULL(this) || !this->d_inode) { + /* Mark a stale entry */ + p->is_whiteout = true; + if (IS_ERR(this)) { + err = PTR_ERR(this); + this = NULL; + goto fail; + } + goto out; + } + +get: + if (!ovl_same_dev(ofs) || !update_ino) + goto out; + + type = ovl_path_type(this); + if (OVL_TYPE_ORIGIN(type)) { + struct kstat stat; + struct path statpath = *path; + + statpath.dentry = this; + err = vfs_getattr(&statpath, &stat, STATX_INO, 0); + if (err) + goto fail; + + /* + * Directory inode is always on overlay st_dev. + * Non-dir with ovl_same_dev() could be on pseudo st_dev in case + * of xino bits overflow. + */ + WARN_ON_ONCE(S_ISDIR(stat.mode) && + dir->d_sb->s_dev != stat.dev); + ino = stat.ino; + } else if (xinobits && !OVL_TYPE_UPPER(type)) { + ino = ovl_remap_lower_ino(ino, xinobits, + ovl_layer_lower(this)->fsid, + p->name, p->len, + ovl_xino_warn(ofs)); + } + +out: + p->ino = ino; + dput(this); + return err; + +fail: + pr_warn_ratelimited("failed to look up (%s) for ino (%i)\n", + p->name, err); + goto out; +} + +static bool ovl_fill_plain(struct dir_context *ctx, const char *name, + int namelen, loff_t offset, u64 ino, + unsigned int d_type) +{ + struct ovl_cache_entry *p; + struct ovl_readdir_data *rdd = + container_of(ctx, struct ovl_readdir_data, ctx); + + rdd->count++; + p = ovl_cache_entry_new(rdd, name, namelen, NULL, 0, ino, d_type); + if (p == NULL) { + rdd->err = -ENOMEM; + return false; + } + list_add_tail(&p->l_node, rdd->list); + + return true; +} + +static int ovl_dir_read_impure(const struct path *path, struct list_head *list, + struct rb_root *root) +{ + int err; + struct path realpath; + struct ovl_cache_entry *p, *n; + struct ovl_readdir_data rdd = { + .ctx.actor = ovl_fill_plain, + .ctx.count = INT_MAX, + .list = list, + .root = root, + }; + + INIT_LIST_HEAD(list); + *root = RB_ROOT; + ovl_path_upper(path->dentry, &realpath); + + err = ovl_dir_read(&realpath, &rdd); + if (err) + return err; + + list_for_each_entry_safe(p, n, list, l_node) { + if (!name_is_dot_dotdot(p->name, p->len)) { + err = ovl_cache_update(path, p, true); + if (err) + return err; + } + if (p->ino == p->real_ino) { + list_del(&p->l_node); + ovl_cache_entry_free(p); + } else { + struct rb_node **newp = &root->rb_node; + struct rb_node *parent = NULL; + + if (WARN_ON(ovl_cache_entry_find_link(p->name, p->len, + &newp, &parent))) + return -EIO; + + rb_link_node(&p->node, parent, newp); + rb_insert_color(&p->node, root); + } + } + return 0; +} + +static struct ovl_dir_cache *ovl_cache_get_impure(const struct path *path) +{ + int res; + struct dentry *dentry = path->dentry; + struct inode *inode = d_inode(dentry); + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + struct ovl_dir_cache *cache; + + cache = ovl_dir_cache(inode); + if (cache && ovl_inode_version_get(inode) == cache->version) + return cache; + + /* Impure cache is not refcounted, free it here */ + ovl_dir_cache_free(inode); + ovl_set_dir_cache(inode, NULL); + + cache = kzalloc_obj(struct ovl_dir_cache); + if (!cache) + return ERR_PTR(-ENOMEM); + + res = ovl_dir_read_impure(path, &cache->entries, &cache->root); + if (res) { + ovl_cache_free(&cache->entries); + kfree(cache); + return ERR_PTR(res); + } + if (list_empty(&cache->entries)) { + /* + * A good opportunity to get rid of an unneeded "impure" flag. + * Removing the "impure" xattr is best effort. + */ + if (!ovl_want_write(dentry)) { + ovl_removexattr(ofs, ovl_dentry_upper(dentry), + OVL_XATTR_IMPURE); + ovl_drop_write(dentry); + } + ovl_clear_flag(OVL_IMPURE, inode); + kfree(cache); + return NULL; + } + + cache->version = ovl_inode_version_get(inode); + ovl_set_dir_cache(inode, cache); + + return cache; +} + +struct ovl_readdir_translate { + struct dir_context *orig_ctx; + struct ovl_dir_cache *cache; + struct dir_context ctx; + u64 parent_ino; + int fsid; + int xinobits; + bool xinowarn; +}; + +static bool ovl_fill_real(struct dir_context *ctx, const char *name, + int namelen, loff_t offset, u64 ino, + unsigned int d_type) +{ + struct ovl_readdir_translate *rdt = + container_of(ctx, struct ovl_readdir_translate, ctx); + struct dir_context *orig_ctx = rdt->orig_ctx; + bool res; + + if (rdt->parent_ino && name_is_dotdot(name, namelen)) { + ino = rdt->parent_ino; + } else if (rdt->cache) { + struct ovl_cache_entry *p; + + p = ovl_cache_entry_find(&rdt->cache->root, name, namelen); + if (p) + ino = p->ino; + } else if (rdt->xinobits) { + ino = ovl_remap_lower_ino(ino, rdt->xinobits, rdt->fsid, + name, namelen, rdt->xinowarn); + } + + res = orig_ctx->actor(orig_ctx, name, namelen, offset, ino, d_type); + ctx->count = orig_ctx->count; + + return res; +} + +static bool ovl_is_impure_dir(struct file *file) +{ + struct ovl_dir_file *od = file->private_data; + struct inode *dir = file_inode(file); + + /* + * Only upper dir can be impure, but if we are in the middle of + * iterating a lower real dir, dir could be copied up and marked + * impure. We only want the impure cache if we started iterating + * a real upper dir to begin with. + */ + return od->is_upper && ovl_test_flag(OVL_IMPURE, dir); + +} + +static int ovl_iterate_real(struct file *file, struct dir_context *ctx) +{ + int err; + struct ovl_dir_file *od = file->private_data; + struct dentry *dir = file->f_path.dentry; + struct ovl_fs *ofs = OVL_FS(dir->d_sb); + const struct ovl_layer *lower_layer = ovl_layer_lower(dir); + struct ovl_readdir_translate rdt = { + .ctx.actor = ovl_fill_real, + .ctx.count = ctx->count, + .orig_ctx = ctx, + .xinobits = ovl_xino_bits(ofs), + .xinowarn = ovl_xino_warn(ofs), + }; + + if (rdt.xinobits && lower_layer) + rdt.fsid = lower_layer->fsid; + + if (OVL_TYPE_MERGE(ovl_path_type(dir->d_parent))) { + struct kstat stat; + struct path statpath = file->f_path; + + statpath.dentry = dir->d_parent; + err = vfs_getattr(&statpath, &stat, STATX_INO, 0); + if (err) + return err; + + WARN_ON_ONCE(dir->d_sb->s_dev != stat.dev); + rdt.parent_ino = stat.ino; + } + + if (ovl_is_impure_dir(file)) { + rdt.cache = ovl_cache_get_impure(&file->f_path); + if (IS_ERR(rdt.cache)) + return PTR_ERR(rdt.cache); + } + + err = iterate_dir(od->realfile, &rdt.ctx); + ctx->pos = rdt.ctx.pos; + + return err; +} + +static int ovl_iterate_merged(struct file *file, struct dir_context *ctx) +{ + struct ovl_dir_file *od = file->private_data; + struct dentry *dentry = file->f_path.dentry; + struct ovl_cache_entry *p; + int err; + + if (!od->cache) { + struct ovl_dir_cache *cache; + + cache = ovl_cache_get(dentry); + if (IS_ERR(cache)) + return PTR_ERR(cache); + + od->cache = cache; + ovl_seek_cursor(od, ctx->pos); + } + + while (od->cursor != &od->cache->entries) { + p = list_entry(od->cursor, struct ovl_cache_entry, l_node); + if (!p->is_whiteout) { + if (!p->ino || p->check_xwhiteout) { + err = ovl_cache_update(&file->f_path, p, !p->ino); + if (err) + return err; + } + } + /* ovl_cache_update() sets is_whiteout on stale entry */ + if (!p->is_whiteout) { + if (!dir_emit(ctx, p->name, p->len, p->ino, p->type)) + break; + } + od->cursor = p->l_node.next; + ctx->pos++; + } + return 0; +} + +static bool ovl_need_adjust_d_ino(struct file *file) +{ + struct dentry *dentry = file->f_path.dentry; + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + + /* If parent is merge, then need to adjust d_ino for '..' */ + if (ovl_xino_bits(ofs)) + return true; + + /* Can't do consistent inode numbering */ + if (!ovl_same_fs(ofs)) + return false; + + /* If dir is impure then need to adjust d_ino for copied up entries */ + if (ovl_is_impure_dir(file) || + OVL_TYPE_MERGE(ovl_path_type(dentry->d_parent))) + return true; + + /* Pure: no need to adjust d_ino */ + return false; +} + + +static int ovl_iterate(struct file *file, struct dir_context *ctx) +{ + struct ovl_dir_file *od = file->private_data; + + if (!ctx->pos) + ovl_dir_reset(file); + + with_ovl_creds(file_dentry(file)->d_sb) { + if (!od->is_real) + return ovl_iterate_merged(file, ctx); + + if (ovl_need_adjust_d_ino(file)) + return ovl_iterate_real(file, ctx); + + return iterate_dir(od->realfile, ctx); + } +} + +static loff_t ovl_dir_llseek(struct file *file, loff_t offset, int origin) +{ + loff_t res; + struct ovl_dir_file *od = file->private_data; + + inode_lock(file_inode(file)); + if (!file->f_pos) + ovl_dir_reset(file); + + if (od->is_real) { + res = vfs_llseek(od->realfile, offset, origin); + file->f_pos = od->realfile->f_pos; + } else { + res = -EINVAL; + + switch (origin) { + case SEEK_CUR: + offset += file->f_pos; + break; + case SEEK_SET: + break; + default: + goto out_unlock; + } + if (offset < 0) + goto out_unlock; + + if (offset != file->f_pos) { + file->f_pos = offset; + if (od->cache) + ovl_seek_cursor(od, offset); + } + res = offset; + } +out_unlock: + inode_unlock(file_inode(file)); + + return res; +} + +static struct file *ovl_dir_open_realfile(const struct file *file, + const struct path *realpath) +{ + with_ovl_creds(file_inode(file)->i_sb) + return ovl_path_open(realpath, O_RDONLY | (file->f_flags & O_LARGEFILE)); +} + +/* + * Like ovl_real_fdget(), returns upperfile if dir was copied up since open. + * Unlike ovl_real_fdget(), this caches upperfile in file->private_data. + * + * TODO: use same abstract type for file->private_data of dir and file so + * upperfile could also be cached for files as well. + */ +struct file *ovl_dir_real_file(const struct file *file, bool want_upper) +{ + + struct ovl_dir_file *od = file->private_data; + struct dentry *dentry = file->f_path.dentry; + struct file *old, *realfile = od->realfile; + + if (!OVL_TYPE_UPPER(ovl_path_type(dentry))) + return want_upper ? NULL : realfile; + + /* + * Need to check if we started out being a lower dir, but got copied up + */ + if (!od->is_upper) { + realfile = READ_ONCE(od->upperfile); + if (!realfile) { + struct path upperpath; + + ovl_path_upper(dentry, &upperpath); + realfile = ovl_dir_open_realfile(file, &upperpath); + if (IS_ERR(realfile)) + return realfile; + + old = cmpxchg_release(&od->upperfile, NULL, realfile); + if (old) { + fput(realfile); + realfile = old; + } + } + } + + return realfile; +} + +static int ovl_dir_fsync(struct file *file, loff_t start, loff_t end, + int datasync) +{ + struct file *realfile; + int err; + + err = ovl_sync_status(OVL_FS(file_inode(file)->i_sb)); + if (err <= 0) + return err; + + realfile = ovl_dir_real_file(file, true); + err = PTR_ERR_OR_ZERO(realfile); + + /* Nothing to sync for lower */ + if (!realfile || err) + return err; + + return vfs_fsync_range(realfile, start, end, datasync); +} + +static int ovl_dir_release(struct inode *inode, struct file *file) +{ + struct ovl_dir_file *od = file->private_data; + + if (od->cache) { + inode_lock(inode); + ovl_cache_put(od, inode); + inode_unlock(inode); + } + fput(od->realfile); + if (od->upperfile) + fput(od->upperfile); + kfree(od); + + return 0; +} + +static int ovl_dir_open(struct inode *inode, struct file *file) +{ + struct path realpath; + struct file *realfile; + struct ovl_dir_file *od; + enum ovl_path_type type; + + od = kzalloc(sizeof(struct ovl_dir_file), GFP_KERNEL); + if (!od) + return -ENOMEM; + + type = ovl_path_real(file->f_path.dentry, &realpath); + realfile = ovl_dir_open_realfile(file, &realpath); + if (IS_ERR(realfile)) { + kfree(od); + return PTR_ERR(realfile); + } + od->realfile = realfile; + od->is_real = ovl_dir_is_real(inode); + od->is_upper = OVL_TYPE_UPPER(type); + file->private_data = od; + + return 0; +} + +WRAP_DIR_ITER(ovl_iterate) // FIXME! +const struct file_operations ovl_dir_operations = { + .read = generic_read_dir, + .open = ovl_dir_open, + .iterate_shared = shared_ovl_iterate, + .llseek = ovl_dir_llseek, + .fsync = ovl_dir_fsync, + .release = ovl_dir_release, + .setlease = generic_setlease, +}; + +int ovl_check_empty_dir(struct dentry *dentry, struct list_head *list) +{ + int err; + struct ovl_cache_entry *p, *n; + struct rb_root root = RB_ROOT; + + with_ovl_creds(dentry->d_sb) + err = ovl_dir_read_merged(dentry, list, &root); + if (err) + return err; + + err = 0; + + list_for_each_entry_safe(p, n, list, l_node) { + /* + * Select whiteouts in upperdir, they should + * be cleared when deleting this directory. + */ + if (p->is_whiteout) { + if (p->is_upper) + continue; + goto del_entry; + } + + if (name_is_dot_dotdot(p->name, p->len)) + goto del_entry; + err = -ENOTEMPTY; + break; + +del_entry: + list_del(&p->l_node); + ovl_cache_entry_free(p); + } + + return err; +} + +void ovl_cleanup_whiteouts(struct ovl_fs *ofs, struct dentry *upper, + struct list_head *list) +{ + struct ovl_cache_entry *p; + + list_for_each_entry(p, list, l_node) { + struct dentry *dentry; + + if (WARN_ON(!p->is_whiteout || !p->is_upper)) + continue; + + dentry = ovl_lookup_upper_unlocked(ofs, p->name, upper, p->len); + if (IS_ERR(dentry)) { + pr_err("lookup '%s/%.*s' failed (%i)\n", + upper->d_name.name, p->len, p->name, + (int) PTR_ERR(dentry)); + continue; + } + if (dentry->d_inode) + ovl_cleanup(ofs, upper, dentry); + dput(dentry); + } +} + +static bool ovl_check_d_type(struct dir_context *ctx, const char *name, + int namelen, loff_t offset, u64 ino, + unsigned int d_type) +{ + struct ovl_readdir_data *rdd = + container_of(ctx, struct ovl_readdir_data, ctx); + + /* Even if d_type is not supported, DT_DIR is returned for . and .. */ + if (name_is_dot_dotdot(name, namelen)) + return true; + + if (d_type != DT_UNKNOWN) + rdd->d_type_supported = true; + + return true; +} + +/* + * Returns 1 if d_type is supported, 0 not supported/unknown. Negative values + * if error is encountered. + */ +int ovl_check_d_type_supported(const struct path *realpath) +{ + int err; + struct ovl_readdir_data rdd = { + .ctx.actor = ovl_check_d_type, + .ctx.count = INT_MAX, + .d_type_supported = false, + }; + + err = ovl_dir_read(realpath, &rdd); + if (err) + return err; + + return rdd.d_type_supported; +} + +#define OVL_INCOMPATDIR_NAME "incompat" + +static int ovl_workdir_cleanup_recurse(struct ovl_fs *ofs, const struct path *path, + int level) +{ + int err; + LIST_HEAD(list); + struct ovl_cache_entry *p; + struct ovl_readdir_data rdd = { + .ctx.actor = ovl_fill_plain, + .ctx.count = INT_MAX, + .list = &list, + }; + bool incompat = false; + + /* + * The "work/incompat" directory is treated specially - if it is not + * empty, instead of printing a generic error and mounting read-only, + * we will error about incompat features and fail the mount. + * + * When called from ovl_indexdir_cleanup(), path->dentry->d_name.name + * starts with '#'. + */ + if (level == 2 && + !strcmp(path->dentry->d_name.name, OVL_INCOMPATDIR_NAME)) + incompat = true; + + err = ovl_dir_read(path, &rdd); + if (err) + goto out; + + list_for_each_entry(p, &list, l_node) { + struct dentry *dentry; + + if (name_is_dot_dotdot(p->name, p->len)) { + continue; + } else if (incompat) { + pr_err("overlay with incompat feature '%s' cannot be mounted\n", + p->name); + err = -EINVAL; + break; + } + dentry = ovl_lookup_upper_unlocked(ofs, p->name, path->dentry, p->len); + if (IS_ERR(dentry)) + continue; + if (dentry->d_inode) + err = ovl_workdir_cleanup(ofs, path->dentry, path->mnt, + dentry, level); + dput(dentry); + if (err) + break; + } +out: + ovl_cache_free(&list); + return err; +} + +int ovl_workdir_cleanup(struct ovl_fs *ofs, struct dentry *parent, + struct vfsmount *mnt, struct dentry *dentry, int level) +{ + int err; + + if (!d_is_dir(dentry) || level > 1) + return ovl_cleanup(ofs, parent, dentry); + + dentry = start_removing_dentry(parent, dentry); + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + err = ovl_do_rmdir(ofs, parent->d_inode, dentry); + end_removing(dentry); + if (err) { + struct path path = { .mnt = mnt, .dentry = dentry }; + + err = ovl_workdir_cleanup_recurse(ofs, &path, level + 1); + if (!err) + err = ovl_cleanup(ofs, parent, dentry); + } + + return err; +} + +int ovl_indexdir_cleanup(struct ovl_fs *ofs) +{ + int err; + struct dentry *indexdir = ofs->workdir; + struct dentry *index = NULL; + struct path path = { .mnt = ovl_upper_mnt(ofs), .dentry = indexdir }; + LIST_HEAD(list); + struct ovl_cache_entry *p; + struct ovl_readdir_data rdd = { + .ctx.actor = ovl_fill_plain, + .ctx.count = INT_MAX, + .list = &list, + }; + + err = ovl_dir_read(&path, &rdd); + if (err) + goto out; + + list_for_each_entry(p, &list, l_node) { + if (name_is_dot_dotdot(p->name, p->len)) + continue; + index = ovl_lookup_upper_unlocked(ofs, p->name, indexdir, p->len); + if (IS_ERR(index)) { + err = PTR_ERR(index); + index = NULL; + break; + } + /* Cleanup leftover from index create/cleanup attempt */ + if (index->d_name.name[0] == '#') { + err = ovl_workdir_cleanup(ofs, indexdir, path.mnt, index, 1); + if (err) + break; + goto next; + } + err = ovl_verify_index(ofs, index); + if (!err) { + goto next; + } else if (err == -ESTALE) { + /* Cleanup stale index entries */ + err = ovl_cleanup(ofs, indexdir, index); + } else if (err != -ENOENT) { + /* + * Abort mount to avoid corrupting the index if + * an incompatible index entry was found or on out + * of memory. + */ + break; + } else if (ofs->config.nfs_export) { + /* + * Whiteout orphan index to block future open by + * handle after overlay nlink dropped to zero. + */ + err = ovl_cleanup_and_whiteout(ofs, indexdir, index); + } else { + /* Cleanup orphan index entries */ + err = ovl_cleanup(ofs, indexdir, index); + } + + if (err) + break; + +next: + dput(index); + index = NULL; + } + dput(index); +out: + ovl_cache_free(&list); + if (err) + pr_err("failed index dir cleanup (%i)\n", err); + return err; +} diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c new file mode 100644 index 000000000..e48759733 --- /dev/null +++ b/fs/overlayfs/super.c @@ -0,0 +1,1623 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * + * Copyright (C) 2011 Novell Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "overlayfs.h" +#include "params.h" + +MODULE_AUTHOR("Miklos Szeredi "); +MODULE_DESCRIPTION("Overlay filesystem"); +MODULE_LICENSE("GPL"); + + +struct ovl_dir_cache; + +static struct dentry *ovl_d_real(struct dentry *dentry, enum d_real_type type) +{ + struct dentry *upper, *lower; + int err; + + switch (type) { + case D_REAL_DATA: + case D_REAL_METADATA: + break; + default: + goto bug; + } + + if (!d_is_reg(dentry)) { + /* d_real_inode() is only relevant for regular files */ + return dentry; + } + + upper = ovl_dentry_upper(dentry); + if (upper && (type == D_REAL_METADATA || + ovl_has_upperdata(d_inode(dentry)))) + return upper; + + if (type == D_REAL_METADATA) { + lower = ovl_dentry_lower(dentry); + goto real_lower; + } + + /* + * Best effort lazy lookup of lowerdata for D_REAL_DATA case to return + * the real lowerdata dentry. The only current caller of d_real() with + * D_REAL_DATA is d_real_inode() from trace_uprobe and this caller is + * likely going to be followed reading from the file, before placing + * uprobes on offset within the file, so lowerdata should be available + * when setting the uprobe. + */ + err = ovl_verify_lowerdata(dentry); + if (err) + goto bug; + lower = ovl_dentry_lowerdata(dentry); + if (!lower) + goto bug; + +real_lower: + /* Handle recursion into stacked lower fs */ + return d_real(lower, type); + +bug: + WARN(1, "%s(%pd4, %d): real dentry not found\n", __func__, dentry, type); + return dentry; +} + +static int ovl_revalidate_real(struct dentry *d, unsigned int flags, bool weak) +{ + int ret = 1; + + if (!d) + return 1; + + if (weak) { + if (d->d_flags & DCACHE_OP_WEAK_REVALIDATE) + ret = d->d_op->d_weak_revalidate(d, flags); + } else if (d->d_flags & DCACHE_OP_REVALIDATE) { + struct dentry *parent; + struct inode *dir; + struct name_snapshot n; + + if (flags & LOOKUP_RCU) { + parent = READ_ONCE(d->d_parent); + dir = d_inode_rcu(parent); + if (!dir) + return -ECHILD; + } else { + parent = dget_parent(d); + dir = d_inode(parent); + } + take_dentry_name_snapshot(&n, d); + ret = d->d_op->d_revalidate(dir, &n.name, d, flags); + release_dentry_name_snapshot(&n); + if (!(flags & LOOKUP_RCU)) + dput(parent); + if (!ret) { + if (!(flags & LOOKUP_RCU)) + d_invalidate(d); + ret = -ESTALE; + } + } + return ret; +} + +static int ovl_dentry_revalidate_common(struct dentry *dentry, + unsigned int flags, bool weak) +{ + struct ovl_entry *oe; + struct ovl_path *lowerstack; + struct inode *inode = d_inode_rcu(dentry); + struct dentry *upper; + unsigned int i; + int ret = 1; + + if (!inode) { + /* + * Lookup of negative dentries will call ovl_dentry_init_flags() + * with NULL upperdentry and NULL oe, resulting in the + * DCACHE_OP*_REVALIDATE flags being cleared. Hence the only + * way to get a negative inode is due to a race with dentry + * destruction. + */ + WARN_ON(!(flags & LOOKUP_RCU)); + return -ECHILD; + } + + oe = OVL_I_E(inode); + lowerstack = ovl_lowerstack(oe); + upper = ovl_i_dentry_upper(inode); + if (upper) + ret = ovl_revalidate_real(upper, flags, weak); + + for (i = 0; ret > 0 && i < ovl_numlower(oe); i++) + ret = ovl_revalidate_real(lowerstack[i].dentry, flags, weak); + + return ret; +} + +static int ovl_dentry_revalidate(struct inode *dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + return ovl_dentry_revalidate_common(dentry, flags, false); +} + +static int ovl_dentry_weak_revalidate(struct dentry *dentry, unsigned int flags) +{ + return ovl_dentry_revalidate_common(dentry, flags, true); +} + +static const struct dentry_operations ovl_dentry_operations = { + .d_real = ovl_d_real, + .d_revalidate = ovl_dentry_revalidate, + .d_weak_revalidate = ovl_dentry_weak_revalidate, +}; + +#if IS_ENABLED(CONFIG_UNICODE) +static const struct dentry_operations ovl_dentry_ci_operations = { + .d_real = ovl_d_real, + .d_revalidate = ovl_dentry_revalidate, + .d_weak_revalidate = ovl_dentry_weak_revalidate, + .d_hash = generic_ci_d_hash, + .d_compare = generic_ci_d_compare, +}; +#endif + +static struct kmem_cache *ovl_inode_cachep; + +static struct inode *ovl_alloc_inode(struct super_block *sb) +{ + struct ovl_inode *oi = alloc_inode_sb(sb, ovl_inode_cachep, GFP_KERNEL); + + if (!oi) + return NULL; + + oi->cache = NULL; + oi->redirect = NULL; + oi->version = 0; + oi->flags = 0; + oi->__upperdentry = NULL; + oi->lowerdata_redirect = NULL; + oi->oe = NULL; + mutex_init(&oi->lock); + + return &oi->vfs_inode; +} + +static void ovl_free_inode(struct inode *inode) +{ + struct ovl_inode *oi = OVL_I(inode); + + kfree(oi->redirect); + kfree(oi->oe); + mutex_destroy(&oi->lock); + kmem_cache_free(ovl_inode_cachep, oi); +} + +static void ovl_destroy_inode(struct inode *inode) +{ + struct ovl_inode *oi = OVL_I(inode); + + dput(oi->__upperdentry); + ovl_stack_put(ovl_lowerstack(oi->oe), ovl_numlower(oi->oe)); + if (S_ISDIR(inode->i_mode)) + ovl_dir_cache_free(inode); + else + kfree(oi->lowerdata_redirect); +} + +static void ovl_put_super(struct super_block *sb) +{ + struct ovl_fs *ofs = OVL_FS(sb); + + if (ofs) + ovl_free_fs(ofs); +} + +/* Sync real dirty inodes in upper filesystem (if it exists) */ +static int ovl_sync_fs(struct super_block *sb, int wait) +{ + struct ovl_fs *ofs = OVL_FS(sb); + struct super_block *upper_sb; + int ret; + + ret = ovl_sync_status(ofs); + + if (ret < 0) + return -EIO; + + if (!ret) + return ret; + + /* + * Not called for sync(2) call or an emergency sync (SB_I_SKIP_SYNC). + * All the super blocks will be iterated, including upper_sb. + * + * If this is a syncfs(2) call, then we do need to call + * sync_filesystem() on upper_sb, but enough if we do it when being + * called with wait == 1. + */ + if (!wait) + return 0; + + upper_sb = ovl_upper_mnt(ofs)->mnt_sb; + + down_read(&upper_sb->s_umount); + ret = sync_filesystem(upper_sb); + up_read(&upper_sb->s_umount); + + return ret; +} + +/** + * ovl_statfs + * @dentry: The dentry to query + * @buf: The struct kstatfs to fill in with stats + * + * Get the filesystem statistics. As writes always target the upper layer + * filesystem pass the statfs to the upper filesystem (if it exists) + */ +static int ovl_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + struct ovl_fs *ofs = OVL_FS(sb); + struct dentry *root_dentry = sb->s_root; + struct path path; + int err; + + ovl_path_real(root_dentry, &path); + + err = vfs_statfs(&path, buf); + if (!err) { + buf->f_namelen = ofs->namelen; + buf->f_type = OVERLAYFS_SUPER_MAGIC; + if (ovl_has_fsid(ofs)) + buf->f_fsid = uuid_to_fsid(sb->s_uuid.b); + } + + return err; +} + +static const struct super_operations ovl_super_operations = { + .alloc_inode = ovl_alloc_inode, + .free_inode = ovl_free_inode, + .destroy_inode = ovl_destroy_inode, + .drop_inode = inode_just_drop, + .put_super = ovl_put_super, + .sync_fs = ovl_sync_fs, + .statfs = ovl_statfs, + .show_options = ovl_show_options, +}; + +#define OVL_WORKDIR_NAME "work" +#define OVL_INDEXDIR_NAME "index" + +static struct dentry *ovl_workdir_create(struct ovl_fs *ofs, + const char *name, bool persist) +{ + struct inode *dir = ofs->workbasedir->d_inode; + struct vfsmount *mnt = ovl_upper_mnt(ofs); + struct dentry *work; + int err; + bool retried = false; + +retry: + work = ovl_start_creating_upper(ofs, ofs->workbasedir, &QSTR(name)); + + if (!IS_ERR(work)) { + struct iattr attr = { + .ia_valid = ATTR_MODE, + .ia_mode = S_IFDIR | 0, + }; + + if (work->d_inode) { + end_creating_keep(work); + if (persist) + return work; + err = -EEXIST; + if (retried) + goto out_dput; + retried = true; + err = ovl_workdir_cleanup(ofs, ofs->workbasedir, mnt, work, 0); + dput(work); + if (err == -EINVAL) + return ERR_PTR(err); + + goto retry; + } + + work = ovl_do_mkdir(ofs, dir, work, attr.ia_mode); + end_creating_keep(work); + err = PTR_ERR(work); + if (IS_ERR(work)) + goto out_err; + + /* Weird filesystem returning with hashed negative (kernfs)? */ + err = -EINVAL; + if (d_really_is_negative(work)) + goto out_dput; + + /* + * Try to remove POSIX ACL xattrs from workdir. We are good if: + * + * a) success (there was a POSIX ACL xattr and was removed) + * b) -ENODATA (there was no POSIX ACL xattr) + * c) -EOPNOTSUPP (POSIX ACL xattrs are not supported) + * + * There are various other error values that could effectively + * mean that the xattr doesn't exist (e.g. -ERANGE is returned + * if the xattr name is too long), but the set of filesystems + * allowed as upper are limited to "normal" ones, where checking + * for the above two errors is sufficient. + */ + err = ovl_do_remove_acl(ofs, work, XATTR_NAME_POSIX_ACL_DEFAULT); + if (err && err != -ENODATA && err != -EOPNOTSUPP) + goto out_dput; + + err = ovl_do_remove_acl(ofs, work, XATTR_NAME_POSIX_ACL_ACCESS); + if (err && err != -ENODATA && err != -EOPNOTSUPP) + goto out_dput; + + /* Clear any inherited mode bits */ + inode_lock(work->d_inode); + err = ovl_do_notify_change(ofs, work, &attr); + inode_unlock(work->d_inode); + if (err) + goto out_dput; + } else { + err = PTR_ERR(work); + goto out_err; + } + return work; + +out_dput: + dput(work); +out_err: + pr_warn("failed to create directory %s/%s (errno: %i); mounting read-only\n", + ofs->config.workdir, name, -err); + return NULL; +} + +static int ovl_check_namelen(const struct path *path, struct ovl_fs *ofs, + const char *name) +{ + struct kstatfs statfs; + int err = vfs_statfs(path, &statfs); + + if (err) + pr_err("statfs failed on '%s'\n", name); + else + ofs->namelen = max(ofs->namelen, statfs.f_namelen); + + return err; +} + +static int ovl_lower_dir(const char *name, const struct path *path, + struct ovl_fs *ofs, int *stack_depth) +{ + int fh_type; + int err; + + err = ovl_check_namelen(path, ofs, name); + if (err) + return err; + + *stack_depth = max(*stack_depth, path->mnt->mnt_sb->s_stack_depth); + + /* + * The inodes index feature and NFS export need to encode and decode + * file handles, so they require that all layers support them. + */ + fh_type = ovl_can_decode_fh(path->dentry->d_sb); + if ((ofs->config.nfs_export || + (ofs->config.index && ofs->config.upperdir)) && !fh_type) { + ofs->config.index = false; + ofs->config.nfs_export = false; + pr_warn("fs on '%s' does not support file handles, falling back to index=off,nfs_export=off.\n", + name); + } + ofs->nofh |= !fh_type; + /* + * Decoding origin file handle is required for persistent st_ino. + * Without persistent st_ino, xino=auto falls back to xino=off. + */ + if (ofs->config.xino == OVL_XINO_AUTO && + ofs->config.upperdir && !fh_type) { + ofs->config.xino = OVL_XINO_OFF; + pr_warn("fs on '%s' does not support file handles, falling back to xino=off.\n", + name); + } + + /* Check if lower fs has 32bit inode numbers */ + if (fh_type != FILEID_INO32_GEN) + ofs->xino_mode = -1; + + return 0; +} + +/* + * Workdir should not be subdir of upperdir and vice versa, and + * they should not be the same. + */ +static bool ovl_workdir_ok(struct dentry *workdir, struct dentry *upperdir) +{ + return !is_subdir(workdir, upperdir) && !is_subdir(upperdir, workdir); +} + +static int ovl_setup_trap(struct super_block *sb, struct dentry *dir, + struct inode **ptrap, const char *name) +{ + struct inode *trap; + int err; + + trap = ovl_get_trap_inode(sb, dir); + err = PTR_ERR_OR_ZERO(trap); + if (err) { + if (err == -ELOOP) + pr_err("conflicting %s path\n", name); + return err; + } + + *ptrap = trap; + return 0; +} + +/* + * Determine how we treat concurrent use of upperdir/workdir based on the + * index feature. This is papering over mount leaks of container runtimes, + * for example, an old overlay mount is leaked and now its upperdir is + * attempted to be used as a lower layer in a new overlay mount. + */ +static int ovl_report_in_use(struct ovl_fs *ofs, const char *name) +{ + if (ofs->config.index) { + pr_err("%s is in-use as upperdir/workdir of another mount, mount with '-o index=off' to override exclusive upperdir protection.\n", + name); + return -EBUSY; + } else { + pr_warn("%s is in-use as upperdir/workdir of another mount, accessing files from both mounts will result in undefined behavior.\n", + name); + return 0; + } +} + +static int ovl_get_upper(struct super_block *sb, struct ovl_fs *ofs, + struct ovl_layer *upper_layer, + const struct path *upperpath) +{ + struct vfsmount *upper_mnt; + int err; + + /* Upperdir path should not be r/o */ + if (__mnt_is_readonly(upperpath->mnt)) { + pr_err("upper fs is r/o, try multi-lower layers mount\n"); + err = -EINVAL; + goto out; + } + + err = ovl_check_namelen(upperpath, ofs, ofs->config.upperdir); + if (err) + goto out; + + err = ovl_setup_trap(sb, upperpath->dentry, &upper_layer->trap, + "upperdir"); + if (err) + goto out; + + upper_mnt = clone_private_mount(upperpath); + err = PTR_ERR(upper_mnt); + if (IS_ERR(upper_mnt)) { + pr_err("failed to clone upperpath\n"); + goto out; + } + + /* Don't inherit atime flags */ + upper_mnt->mnt_flags &= ~(MNT_NOATIME | MNT_NODIRATIME | MNT_RELATIME); + upper_layer->mnt = upper_mnt; + upper_layer->idx = 0; + upper_layer->fsid = 0; + + /* + * Inherit SB_NOSEC flag from upperdir. + * + * This optimization changes behavior when a security related attribute + * (suid/sgid/security.*) is changed on an underlying layer. This is + * okay because we don't yet have guarantees in that case, but it will + * need careful treatment once we want to honour changes to underlying + * filesystems. + */ + if (upper_mnt->mnt_sb->s_flags & SB_NOSEC) + sb->s_flags |= SB_NOSEC; + + if (ovl_inuse_trylock(ovl_upper_mnt(ofs)->mnt_root)) { + ofs->upperdir_locked = true; + } else { + err = ovl_report_in_use(ofs, "upperdir"); + if (err) + goto out; + } + + err = 0; +out: + return err; +} + +/* + * Returns 1 if RENAME_WHITEOUT is supported, 0 if not supported and + * negative values if error is encountered. + */ +static int ovl_check_rename_whiteout(struct ovl_fs *ofs) +{ + struct dentry *workdir = ofs->workdir; + struct dentry *temp; + struct dentry *whiteout; + struct name_snapshot name; + struct renamedata rd = {}; + char name2[OVL_TEMPNAME_SIZE]; + int err; + + temp = ovl_create_temp(ofs, workdir, OVL_CATTR(S_IFREG | 0)); + err = PTR_ERR(temp); + if (IS_ERR(temp)) + return err; + + rd.mnt_idmap = ovl_upper_mnt_idmap(ofs); + rd.old_parent = workdir; + rd.new_parent = workdir; + rd.flags = RENAME_WHITEOUT; + ovl_tempname(name2); + err = start_renaming_dentry(&rd, 0, temp, &QSTR(name2)); + if (err) { + dput(temp); + return err; + } + + /* Name is inline and stable - using snapshot as a copy helper */ + take_dentry_name_snapshot(&name, temp); + err = ovl_do_rename_rd(&rd); + end_renaming(&rd); + if (err) { + if (err == -EINVAL) + err = 0; + goto cleanup_temp; + } + + whiteout = ovl_lookup_upper_unlocked(ofs, name.name.name, + workdir, name.name.len); + err = PTR_ERR(whiteout); + if (IS_ERR(whiteout)) + goto cleanup_temp; + + err = ovl_upper_is_whiteout(ofs, whiteout); + + /* Best effort cleanup of whiteout and temp file */ + if (err) + ovl_cleanup(ofs, workdir, whiteout); + dput(whiteout); + +cleanup_temp: + ovl_cleanup(ofs, workdir, temp); + release_dentry_name_snapshot(&name); + dput(temp); + + return err; +} + +static struct dentry *ovl_lookup_or_create(struct ovl_fs *ofs, + struct dentry *parent, + const char *name, umode_t mode) +{ + struct dentry *child; + + child = ovl_start_creating_upper(ofs, parent, &QSTR(name)); + if (!IS_ERR(child)) { + if (!child->d_inode) + child = ovl_create_real(ofs, parent, child, + &QSTR(name), + OVL_CATTR(mode)); + end_creating_keep(child); + } + dput(parent); + + return child; +} + +/* + * Creates $workdir/work/incompat/volatile/dirty file if it is not already + * present. + */ +static int ovl_create_volatile_dirty(struct ovl_fs *ofs) +{ + unsigned int ctr; + struct dentry *d = dget(ofs->workbasedir); + static const char *const volatile_path[] = { + OVL_WORKDIR_NAME, "incompat", "volatile", "dirty" + }; + const char *const *name = volatile_path; + + for (ctr = ARRAY_SIZE(volatile_path); ctr; ctr--, name++) { + d = ovl_lookup_or_create(ofs, d, *name, ctr > 1 ? S_IFDIR : S_IFREG); + if (IS_ERR(d)) + return PTR_ERR(d); + } + dput(d); + return 0; +} + +static int ovl_make_workdir(struct super_block *sb, struct ovl_fs *ofs, + const struct path *workpath) +{ + struct vfsmount *mnt = ovl_upper_mnt(ofs); + struct dentry *workdir; + struct file *tmpfile; + bool rename_whiteout; + bool d_type; + int fh_type; + int err; + + err = mnt_want_write(mnt); + if (err) + return err; + + workdir = ovl_workdir_create(ofs, OVL_WORKDIR_NAME, false); + err = PTR_ERR(workdir); + if (IS_ERR_OR_NULL(workdir)) + goto out; + + ofs->workdir = workdir; + + err = ovl_setup_trap(sb, ofs->workdir, &ofs->workdir_trap, "workdir"); + if (err) + goto out; + + /* + * Upper should support d_type, else whiteouts are visible. Given + * workdir and upper are on same fs, we can do iterate_dir() on + * workdir. This check requires successful creation of workdir in + * previous step. + */ + err = ovl_check_d_type_supported(workpath); + if (err < 0) + goto out; + + d_type = err; + if (!d_type) + pr_warn("upper fs needs to support d_type.\n"); + + /* Check if upper/work fs supports O_TMPFILE */ + tmpfile = ovl_do_tmpfile(ofs, ofs->workdir, S_IFREG | 0); + ofs->tmpfile = !IS_ERR(tmpfile); + if (ofs->tmpfile) + fput(tmpfile); + else + pr_warn("upper fs does not support tmpfile.\n"); + + + /* Check if upper/work fs supports RENAME_WHITEOUT */ + err = ovl_check_rename_whiteout(ofs); + if (err < 0) + goto out; + + rename_whiteout = err; + if (!rename_whiteout) + pr_warn("upper fs does not support RENAME_WHITEOUT.\n"); + + /* + * Check if upper/work fs supports (trusted|user).overlay.* xattr + */ + err = ovl_setxattr(ofs, ofs->workdir, OVL_XATTR_OPAQUE, "0", 1); + if (err) { + pr_warn("failed to set xattr on upper\n"); + ofs->noxattr = true; + if (ovl_redirect_follow(ofs)) { + ofs->config.redirect_mode = OVL_REDIRECT_NOFOLLOW; + pr_warn("...falling back to redirect_dir=nofollow.\n"); + } + if (ofs->config.metacopy) { + ofs->config.metacopy = false; + pr_warn("...falling back to metacopy=off.\n"); + } + if (ofs->config.index) { + ofs->config.index = false; + pr_warn("...falling back to index=off.\n"); + } + if (ovl_has_fsid(ofs)) { + ofs->config.uuid = OVL_UUID_NULL; + pr_warn("...falling back to uuid=null.\n"); + } + /* + * xattr support is required for persistent st_ino. + * Without persistent st_ino, xino=auto falls back to xino=off. + */ + if (ofs->config.xino == OVL_XINO_AUTO) { + ofs->config.xino = OVL_XINO_OFF; + pr_warn("...falling back to xino=off.\n"); + } + if (err == -EPERM && !ofs->config.userxattr) + pr_info("try mounting with 'userxattr' option\n"); + err = 0; + } else { + ovl_removexattr(ofs, ofs->workdir, OVL_XATTR_OPAQUE); + } + + /* + * We allowed sub-optimal upper fs configuration and don't want to break + * users over kernel upgrade, but we never allowed remote upper fs, so + * we can enforce strict requirements for remote upper fs. + */ + if (ovl_dentry_remote(ofs->workdir) && + (!d_type || !rename_whiteout || ofs->noxattr)) { + pr_err("upper fs missing required features.\n"); + err = -EINVAL; + goto out; + } + + /* + * For volatile mount, create a incompat/volatile/dirty file to keep + * track of it. + */ + if (ovl_is_volatile(&ofs->config)) { + err = ovl_create_volatile_dirty(ofs); + if (err < 0) { + pr_err("Failed to create volatile/dirty file.\n"); + goto out; + } + } + + /* Check if upper/work fs supports file handles */ + fh_type = ovl_can_decode_fh(ofs->workdir->d_sb); + if (ofs->config.index && !fh_type) { + ofs->config.index = false; + pr_warn("upper fs does not support file handles, falling back to index=off.\n"); + } + ofs->nofh |= !fh_type; + + /* Check if upper fs has 32bit inode numbers */ + if (fh_type != FILEID_INO32_GEN) + ofs->xino_mode = -1; + + /* NFS export of r/w mount depends on index */ + if (ofs->config.nfs_export && !ofs->config.index) { + pr_warn("NFS export requires \"index=on\", falling back to nfs_export=off.\n"); + ofs->config.nfs_export = false; + } +out: + mnt_drop_write(mnt); + return err; +} + +static int ovl_get_workdir(struct super_block *sb, struct ovl_fs *ofs, + const struct path *upperpath, + const struct path *workpath) +{ + int err; + + err = -EINVAL; + if (upperpath->mnt != workpath->mnt) { + pr_err("workdir and upperdir must reside under the same mount\n"); + return err; + } + if (!ovl_workdir_ok(workpath->dentry, upperpath->dentry)) { + pr_err("workdir and upperdir must be separate subtrees\n"); + return err; + } + + ofs->workbasedir = dget(workpath->dentry); + + if (ovl_inuse_trylock(ofs->workbasedir)) { + ofs->workdir_locked = true; + } else { + err = ovl_report_in_use(ofs, "workdir"); + if (err) + return err; + } + + err = ovl_setup_trap(sb, ofs->workbasedir, &ofs->workbasedir_trap, + "workdir"); + if (err) + return err; + + return ovl_make_workdir(sb, ofs, workpath); +} + +static int ovl_get_indexdir(struct super_block *sb, struct ovl_fs *ofs, + struct ovl_entry *oe, const struct path *upperpath) +{ + struct vfsmount *mnt = ovl_upper_mnt(ofs); + struct dentry *indexdir; + struct dentry *origin = ovl_lowerstack(oe)->dentry; + const struct ovl_fh *fh; + int err; + + fh = ovl_get_origin_fh(ofs, origin); + if (IS_ERR(fh)) + return PTR_ERR(fh); + + err = mnt_want_write(mnt); + if (err) + goto out_free_fh; + + /* Verify lower root is upper root origin */ + err = ovl_verify_origin_fh(ofs, upperpath->dentry, fh, true); + if (err) { + pr_err("failed to verify upper root origin\n"); + goto out; + } + + /* index dir will act also as workdir */ + iput(ofs->workdir_trap); + ofs->workdir_trap = NULL; + dput(ofs->workdir); + ofs->workdir = NULL; + indexdir = ovl_workdir_create(ofs, OVL_INDEXDIR_NAME, true); + if (IS_ERR(indexdir)) { + err = PTR_ERR(indexdir); + } else if (indexdir) { + ofs->workdir = indexdir; + err = ovl_setup_trap(sb, indexdir, &ofs->workdir_trap, + "indexdir"); + if (err) + goto out; + + /* + * Verify upper root is exclusively associated with index dir. + * Older kernels stored upper fh in ".overlay.origin" + * xattr. If that xattr exists, verify that it is a match to + * upper dir file handle. In any case, verify or set xattr + * ".overlay.upper" to indicate that index may have + * directory entries. + */ + if (ovl_check_origin_xattr(ofs, indexdir)) { + err = ovl_verify_origin_xattr(ofs, indexdir, + OVL_XATTR_ORIGIN, + upperpath->dentry, true, + false); + if (err) + pr_err("failed to verify index dir 'origin' xattr\n"); + } + err = ovl_verify_upper(ofs, indexdir, upperpath->dentry, true); + if (err) + pr_err("failed to verify index dir 'upper' xattr\n"); + + /* Cleanup bad/stale/orphan index entries */ + if (!err) + err = ovl_indexdir_cleanup(ofs); + } + if (err || !indexdir) + pr_warn("try deleting index dir or mounting with '-o index=off' to disable inodes index.\n"); + +out: + mnt_drop_write(mnt); +out_free_fh: + kfree(fh); + return err; +} + +static bool ovl_lower_uuid_ok(struct ovl_fs *ofs, const uuid_t *uuid) +{ + unsigned int i; + + if (!ofs->config.nfs_export && !ovl_upper_mnt(ofs)) + return true; + + /* + * We allow using single lower with null uuid for index and nfs_export + * for example to support those features with single lower squashfs. + * To avoid regressions in setups of overlay with re-formatted lower + * squashfs, do not allow decoding origin with lower null uuid unless + * user opted-in to one of the new features that require following the + * lower inode of non-dir upper. + */ + if (ovl_allow_offline_changes(ofs) && uuid_is_null(uuid)) + return false; + + for (i = 0; i < ofs->numfs; i++) { + /* + * We use uuid to associate an overlay lower file handle with a + * lower layer, so we can accept lower fs with null uuid as long + * as all lower layers with null uuid are on the same fs. + * if we detect multiple lower fs with the same uuid, we + * disable lower file handle decoding on all of them. + */ + if (ofs->fs[i].is_lower && + ovl_uuid_match(ofs, ofs->fs[i].sb, uuid)) { + ofs->fs[i].bad_uuid = true; + return false; + } + } + return true; +} + +/* Get a unique fsid for the layer */ +static int ovl_get_fsid(struct ovl_fs *ofs, const struct path *path) +{ + struct super_block *sb = path->mnt->mnt_sb; + const uuid_t *uuid = ovl_origin_uuid(ofs) ? &sb->s_uuid : &uuid_null; + unsigned int i; + dev_t dev; + int err; + bool bad_uuid = false; + bool warn = false; + + for (i = 0; i < ofs->numfs; i++) { + if (ofs->fs[i].sb == sb) + return i; + } + + if (!ovl_lower_uuid_ok(ofs, uuid)) { + bad_uuid = true; + if (ofs->config.xino == OVL_XINO_AUTO) { + ofs->config.xino = OVL_XINO_OFF; + warn = true; + } + if (ofs->config.index || ofs->config.nfs_export) { + ofs->config.index = false; + ofs->config.nfs_export = false; + warn = true; + } + if (warn) { + pr_warn("%s uuid in non-single lower fs '%pd2', falling back to xino=%s,index=off,nfs_export=off.\n", + uuid_is_null(uuid) ? "null" : "conflicting", + path->dentry, ovl_xino_mode(&ofs->config)); + } + } + + err = get_anon_bdev(&dev); + if (err) { + pr_err("failed to get anonymous bdev for lowerpath\n"); + return err; + } + + ofs->fs[ofs->numfs].sb = sb; + ofs->fs[ofs->numfs].pseudo_dev = dev; + ofs->fs[ofs->numfs].bad_uuid = bad_uuid; + + return ofs->numfs++; +} + +/* + * The fsid after the last lower fsid is used for the data layers. + * It is a "null fs" with a null sb, null uuid, and no pseudo dev. + */ +static int ovl_get_data_fsid(struct ovl_fs *ofs) +{ + return ofs->numfs; +} + +/* + * Set the ovl sb encoding as the same one used by the first layer + */ +static int ovl_set_encoding(struct super_block *sb, struct super_block *fs_sb) +{ + if (!sb_has_encoding(fs_sb)) + return 0; + +#if IS_ENABLED(CONFIG_UNICODE) + if (sb_has_strict_encoding(fs_sb)) { + pr_err("strict encoding not supported\n"); + return -EINVAL; + } + + sb->s_encoding = fs_sb->s_encoding; + sb->s_encoding_flags = fs_sb->s_encoding_flags; +#endif + return 0; +} + +static int ovl_get_layers(struct super_block *sb, struct ovl_fs *ofs, + struct ovl_fs_context *ctx, struct ovl_layer *layers) +{ + int err; + unsigned int i; + size_t nr_merged_lower; + + ofs->fs = kzalloc_objs(struct ovl_sb, ctx->nr + 2); + if (ofs->fs == NULL) + return -ENOMEM; + + /* + * idx/fsid 0 are reserved for upper fs even with lower only overlay + * and the last fsid is reserved for "null fs" of the data layers. + */ + ofs->numfs++; + + /* + * All lower layers that share the same fs as upper layer, use the same + * pseudo_dev as upper layer. Allocate fs[0].pseudo_dev even for lower + * only overlay to simplify ovl_fs_free(). + * is_lower will be set if upper fs is shared with a lower layer. + */ + err = get_anon_bdev(&ofs->fs[0].pseudo_dev); + if (err) { + pr_err("failed to get anonymous bdev for upper fs\n"); + return err; + } + + if (ovl_upper_mnt(ofs)) { + ofs->fs[0].sb = ovl_upper_mnt(ofs)->mnt_sb; + ofs->fs[0].is_lower = false; + + if (ofs->casefold) { + err = ovl_set_encoding(sb, ofs->fs[0].sb); + if (err) + return err; + } + } + + nr_merged_lower = ctx->nr - ctx->nr_data; + for (i = 0; i < ctx->nr; i++) { + struct ovl_fs_context_layer *l = &ctx->lower[i]; + struct vfsmount *mnt; + struct inode *trap; + int fsid; + + if (i < nr_merged_lower) + fsid = ovl_get_fsid(ofs, &l->path); + else + fsid = ovl_get_data_fsid(ofs); + if (fsid < 0) + return fsid; + + /* + * Check if lower root conflicts with this overlay layers before + * checking if it is in-use as upperdir/workdir of "another" + * mount, because we do not bother to check in ovl_is_inuse() if + * the upperdir/workdir is in fact in-use by our + * upperdir/workdir. + */ + err = ovl_setup_trap(sb, l->path.dentry, &trap, "lowerdir"); + if (err) + return err; + + if (ovl_is_inuse(l->path.dentry)) { + err = ovl_report_in_use(ofs, "lowerdir"); + if (err) { + iput(trap); + return err; + } + } + + mnt = clone_private_mount(&l->path); + err = PTR_ERR(mnt); + if (IS_ERR(mnt)) { + pr_err("failed to clone lowerpath\n"); + iput(trap); + return err; + } + + /* + * Make lower layers R/O. That way fchmod/fchown on lower file + * will fail instead of modifying lower fs. + */ + mnt->mnt_flags |= MNT_READONLY | MNT_NOATIME; + + layers[ofs->numlayer].trap = trap; + layers[ofs->numlayer].mnt = mnt; + layers[ofs->numlayer].idx = ofs->numlayer; + layers[ofs->numlayer].fsid = fsid; + layers[ofs->numlayer].fs = &ofs->fs[fsid]; + /* Store for printing lowerdir=... in ovl_show_options() */ + ofs->config.lowerdirs[ofs->numlayer] = l->name; + l->name = NULL; + ofs->numlayer++; + ofs->fs[fsid].is_lower = true; + + if (ofs->casefold) { + if (!ovl_upper_mnt(ofs) && !sb_has_encoding(sb)) { + err = ovl_set_encoding(sb, ofs->fs[fsid].sb); + if (err) + return err; + } + + if (!sb_same_encoding(sb, mnt->mnt_sb)) { + pr_err("all layers must have the same encoding\n"); + return -EINVAL; + } + } + } + + /* + * When all layers on same fs, overlay can use real inode numbers. + * With mount option "xino=", mounter declares that there are + * enough free high bits in underlying fs to hold the unique fsid. + * If overlayfs does encounter underlying inodes using the high xino + * bits reserved for fsid, it emits a warning and uses the original + * inode number or a non persistent inode number allocated from a + * dedicated range. + */ + if (ofs->numfs - !ovl_upper_mnt(ofs) == 1) { + if (ofs->config.xino == OVL_XINO_ON) + pr_info("\"xino=on\" is useless with all layers on same fs, ignore.\n"); + ofs->xino_mode = 0; + } else if (ofs->config.xino == OVL_XINO_OFF) { + ofs->xino_mode = -1; + } else if (ofs->xino_mode < 0) { + /* + * This is a roundup of number of bits needed for encoding + * fsid, where fsid 0 is reserved for upper fs (even with + * lower only overlay) +1 extra bit is reserved for the non + * persistent inode number range that is used for resolving + * xino lower bits overflow. + */ + BUILD_BUG_ON(ilog2(OVL_MAX_STACK) > 30); + ofs->xino_mode = ilog2(ofs->numfs - 1) + 2; + } + + if (ofs->xino_mode > 0) { + pr_info("\"xino\" feature enabled using %d upper inode bits.\n", + ofs->xino_mode); + } + + return 0; +} + +static struct ovl_entry *ovl_get_lowerstack(struct super_block *sb, + struct ovl_fs_context *ctx, + struct ovl_fs *ofs, + struct ovl_layer *layers) +{ + int err; + unsigned int i; + size_t nr_merged_lower; + struct ovl_entry *oe; + struct ovl_path *lowerstack; + + struct ovl_fs_context_layer *l; + + if (!ofs->config.upperdir && ctx->nr == 1) { + pr_err("at least 2 lowerdir are needed while upperdir nonexistent\n"); + return ERR_PTR(-EINVAL); + } + + if (ctx->nr == ctx->nr_data) { + pr_err("at least one non-data lowerdir is required\n"); + return ERR_PTR(-EINVAL); + } + + err = -EINVAL; + for (i = 0; i < ctx->nr; i++) { + l = &ctx->lower[i]; + + err = ovl_lower_dir(l->name, &l->path, ofs, &sb->s_stack_depth); + if (err) + return ERR_PTR(err); + } + + err = -EINVAL; + sb->s_stack_depth++; + if (sb->s_stack_depth > FILESYSTEM_MAX_STACK_DEPTH) { + pr_err("maximum fs stacking depth exceeded\n"); + return ERR_PTR(err); + } + + err = ovl_get_layers(sb, ofs, ctx, layers); + if (err) + return ERR_PTR(err); + + err = -ENOMEM; + /* Data-only layers are not merged in root directory */ + nr_merged_lower = ctx->nr - ctx->nr_data; + oe = ovl_alloc_entry(nr_merged_lower); + if (!oe) + return ERR_PTR(err); + + lowerstack = ovl_lowerstack(oe); + for (i = 0; i < nr_merged_lower; i++) { + l = &ctx->lower[i]; + lowerstack[i].dentry = dget(l->path.dentry); + lowerstack[i].layer = &ofs->layers[i + 1]; + } + ofs->numdatalayer = ctx->nr_data; + + return oe; +} + +/* + * Check if this layer root is a descendant of: + * - another layer of this overlayfs instance + * - upper/work dir of any overlayfs instance + */ +static int ovl_check_layer(struct super_block *sb, struct ovl_fs *ofs, + struct dentry *dentry, const char *name, + bool is_lower) +{ + struct dentry *next = dentry, *parent; + int err = 0; + + if (!dentry) + return 0; + + parent = dget_parent(next); + + /* Walk back ancestors to root (inclusive) looking for traps */ + while (!err && parent != next) { + if (is_lower && ovl_lookup_trap_inode(sb, parent)) { + err = -ELOOP; + pr_err("overlapping %s path\n", name); + } else if (ovl_is_inuse(parent)) { + err = ovl_report_in_use(ofs, name); + } + next = parent; + parent = dget_parent(next); + dput(next); + } + + dput(parent); + + return err; +} + +/* + * Check if any of the layers or work dirs overlap. + */ +static int ovl_check_overlapping_layers(struct super_block *sb, + struct ovl_fs *ofs) +{ + int i, err; + + if (ovl_upper_mnt(ofs)) { + err = ovl_check_layer(sb, ofs, ovl_upper_mnt(ofs)->mnt_root, + "upperdir", false); + if (err) + return err; + + /* + * Checking workbasedir avoids hitting ovl_is_inuse(parent) of + * this instance and covers overlapping work and index dirs, + * unless work or index dir have been moved since created inside + * workbasedir. In that case, we already have their traps in + * inode cache and we will catch that case on lookup. + */ + err = ovl_check_layer(sb, ofs, ofs->workbasedir, "workdir", + false); + if (err) + return err; + } + + for (i = 1; i < ofs->numlayer; i++) { + err = ovl_check_layer(sb, ofs, + ofs->layers[i].mnt->mnt_root, + "lowerdir", true); + if (err) + return err; + } + + return 0; +} + +static struct dentry *ovl_get_root(struct super_block *sb, + struct dentry *upperdentry, + struct ovl_entry *oe) +{ + struct dentry *root; + struct ovl_fs *ofs = OVL_FS(sb); + struct ovl_path *lowerpath = ovl_lowerstack(oe); + unsigned long ino = d_inode(lowerpath->dentry)->i_ino; + int fsid = lowerpath->layer->fsid; + struct ovl_inode_params oip = { + .upperdentry = upperdentry, + .oe = oe, + }; + + root = d_make_root(ovl_new_inode(sb, S_IFDIR, 0)); + if (!root) + return NULL; + + if (upperdentry) { + /* Root inode uses upper st_ino/i_ino */ + ino = d_inode(upperdentry)->i_ino; + fsid = 0; + ovl_dentry_set_upper_alias(root); + if (ovl_is_impuredir(sb, upperdentry)) + ovl_set_flag(OVL_IMPURE, d_inode(root)); + } + + /* Look for xwhiteouts marker except in the lowermost layer */ + for (int i = 0; i < ovl_numlower(oe) - 1; i++, lowerpath++) { + struct path path = { + .mnt = lowerpath->layer->mnt, + .dentry = lowerpath->dentry, + }; + + /* overlay.opaque=x means xwhiteouts directory */ + if (ovl_get_opaquedir_val(ofs, &path) == 'x') { + ovl_layer_set_xwhiteouts(ofs, lowerpath->layer); + ovl_dentry_set_xwhiteouts(root); + } + } + + /* Root is always merge -> can have whiteouts */ + ovl_set_flag(OVL_WHITEOUTS, d_inode(root)); + ovl_dentry_set_flag(OVL_E_CONNECTED, root); + ovl_set_upperdata(d_inode(root)); + ovl_inode_init(d_inode(root), &oip, ino, fsid); + WARN_ON(!!IS_CASEFOLDED(d_inode(root)) != ofs->casefold); + ovl_dentry_init_flags(root, upperdentry, oe, DCACHE_OP_WEAK_REVALIDATE); + /* root keeps a reference of upperdentry */ + dget(upperdentry); + + return root; +} + +static void ovl_set_d_op(struct super_block *sb) +{ +#if IS_ENABLED(CONFIG_UNICODE) + struct ovl_fs *ofs = sb->s_fs_info; + + if (ofs->casefold) { + set_default_d_op(sb, &ovl_dentry_ci_operations); + return; + } +#endif + set_default_d_op(sb, &ovl_dentry_operations); +} + +static int ovl_fill_super_creds(struct fs_context *fc, struct super_block *sb) +{ + struct ovl_fs *ofs = sb->s_fs_info; + struct cred *creator_cred = (struct cred *)ofs->creator_cred; + struct ovl_fs_context *ctx = fc->fs_private; + struct ovl_layer *layers; + struct ovl_entry *oe = NULL; + int err; + + err = ovl_fs_params_verify(ctx, &ofs->config); + if (err) + return err; + + err = -EINVAL; + if (ctx->nr == 0) { + if (!(fc->sb_flags & SB_SILENT)) + pr_err("missing 'lowerdir'\n"); + return err; + } + + err = -ENOMEM; + layers = kzalloc_objs(struct ovl_layer, ctx->nr + 1); + if (!layers) + return err; + + ofs->config.lowerdirs = kcalloc(ctx->nr + 1, sizeof(char *), GFP_KERNEL); + if (!ofs->config.lowerdirs) { + kfree(layers); + return err; + } + ofs->layers = layers; + /* + * Layer 0 is reserved for upper even if there's no upper. + * config.lowerdirs[0] is used for storing the user provided colon + * separated lowerdir string. + */ + ofs->config.lowerdirs[0] = ctx->lowerdir_all; + ctx->lowerdir_all = NULL; + ofs->numlayer = 1; + + sb->s_stack_depth = 0; + sb->s_maxbytes = MAX_LFS_FILESIZE; + atomic_long_set(&ofs->last_ino, 1); + /* Assume underlying fs uses 32bit inodes unless proven otherwise */ + if (ofs->config.xino != OVL_XINO_OFF) { + ofs->xino_mode = BITS_PER_LONG - 32; + if (!ofs->xino_mode) { + pr_warn("xino not supported on 32bit kernel, falling back to xino=off.\n"); + ofs->config.xino = OVL_XINO_OFF; + } + } + + /* alloc/destroy_inode needed for setting up traps in inode cache */ + sb->s_op = &ovl_super_operations; + + if (ofs->config.upperdir) { + struct super_block *upper_sb; + + err = -EINVAL; + if (!ofs->config.workdir) { + pr_err("missing 'workdir'\n"); + return err; + } + + err = ovl_get_upper(sb, ofs, &layers[0], &ctx->upper); + if (err) + return err; + + upper_sb = ovl_upper_mnt(ofs)->mnt_sb; + if (!ovl_should_sync(ofs)) { + ofs->errseq = errseq_sample(&upper_sb->s_wb_err); + if (errseq_check(&upper_sb->s_wb_err, ofs->errseq)) { + err = -EIO; + pr_err("Cannot mount volatile when upperdir has an unseen error. Sync upperdir fs to clear state.\n"); + return err; + } + } + + err = ovl_get_workdir(sb, ofs, &ctx->upper, &ctx->work); + if (err) + return err; + + if (!ofs->workdir) + sb->s_flags |= SB_RDONLY; + + sb->s_stack_depth = upper_sb->s_stack_depth; + sb->s_time_gran = upper_sb->s_time_gran; + } + oe = ovl_get_lowerstack(sb, ctx, ofs, layers); + err = PTR_ERR(oe); + if (IS_ERR(oe)) + return err; + + /* If the upper fs is nonexistent, we mark overlayfs r/o too */ + if (!ovl_upper_mnt(ofs)) + sb->s_flags |= SB_RDONLY; + + if (ovl_has_fsid(ofs) && ovl_upper_mnt(ofs)) { + /* Use per instance persistent uuid/fsid */ + ovl_init_uuid_xattr(sb, ofs, &ctx->upper); + } + + if (!ovl_force_readonly(ofs) && ofs->config.index) { + err = ovl_get_indexdir(sb, ofs, oe, &ctx->upper); + if (err) + goto out_free_oe; + + /* Force r/o mount with no index dir */ + if (!ofs->workdir) + sb->s_flags |= SB_RDONLY; + } + + err = ovl_check_overlapping_layers(sb, ofs); + if (err) + goto out_free_oe; + + /* Show index=off in /proc/mounts for forced r/o mount */ + if (!ofs->workdir) { + ofs->config.index = false; + if (ovl_upper_mnt(ofs) && ofs->config.nfs_export) { + pr_warn("NFS export requires an index dir, falling back to nfs_export=off.\n"); + ofs->config.nfs_export = false; + } + } + + if (ofs->config.metacopy && ofs->config.nfs_export) { + pr_warn("NFS export is not supported with metadata only copy up, falling back to nfs_export=off.\n"); + ofs->config.nfs_export = false; + } + + /* + * Support encoding decodable file handles with nfs_export=on + * and encoding non-decodable file handles with nfs_export=off + * if all layers support file handles. + */ + if (ofs->config.nfs_export) + sb->s_export_op = &ovl_export_operations; + else if (!ofs->nofh) + sb->s_export_op = &ovl_export_fid_operations; + + /* Never override disk quota limits or use reserved space */ + cap_lower(creator_cred->cap_effective, CAP_SYS_RESOURCE); + + sb->s_magic = OVERLAYFS_SUPER_MAGIC; + sb->s_xattr = ovl_xattr_handlers(ofs); + sb->s_fs_info = ofs; +#ifdef CONFIG_FS_POSIX_ACL + sb->s_flags |= SB_POSIXACL; +#endif + sb->s_iflags |= SB_I_SKIP_SYNC; + /* + * Ensure that umask handling is done by the filesystems used + * for the the upper layer instead of overlayfs as that would + * lead to unexpected results. + */ + sb->s_iflags |= SB_I_NOUMASK; + sb->s_iflags |= SB_I_EVM_HMAC_UNSUPPORTED; + + err = -ENOMEM; + sb->s_root = ovl_get_root(sb, ctx->upper.dentry, oe); + if (!sb->s_root) + goto out_free_oe; + + return 0; + +out_free_oe: + ovl_free_entry(oe); + return err; +} + +int ovl_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct ovl_fs *ofs = sb->s_fs_info; + int err; + + err = -EIO; + /* The fscontext fd may have been passed to another user namespace. */ + if (fc->user_ns != current_user_ns()) + goto out_err; + + ovl_set_d_op(sb); + + if (!ofs->creator_cred) { + err = -ENOMEM; + ofs->creator_cred = prepare_creds(); + if (!ofs->creator_cred) + goto out_err; + } + + with_ovl_creds(sb) + err = ovl_fill_super_creds(fc, sb); + +out_err: + if (err) { + ovl_free_fs(ofs); + sb->s_fs_info = NULL; + } + + return err; +} + +struct file_system_type ovl_fs_type = { + .owner = THIS_MODULE, + .name = "overlay", + .init_fs_context = ovl_init_fs_context, + .parameters = ovl_parameter_spec, + .fs_flags = FS_USERNS_MOUNT | FS_ALLOW_IDMAP, + .kill_sb = kill_anon_super, +}; +MODULE_ALIAS_FS("overlay"); + +static void ovl_inode_init_once(void *foo) +{ + struct ovl_inode *oi = foo; + + inode_init_once(&oi->vfs_inode); +} + +static int __init ovl_init(void) +{ + int err; + + ovl_inode_cachep = kmem_cache_create("ovl_inode", + sizeof(struct ovl_inode), 0, + (SLAB_RECLAIM_ACCOUNT| + SLAB_ACCOUNT), + ovl_inode_init_once); + if (ovl_inode_cachep == NULL) + return -ENOMEM; + + err = register_filesystem(&ovl_fs_type); + if (!err) + return 0; + + kmem_cache_destroy(ovl_inode_cachep); + + return err; +} + +static void __exit ovl_exit(void) +{ + unregister_filesystem(&ovl_fs_type); + + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(ovl_inode_cachep); +} + +module_init(ovl_init); +module_exit(ovl_exit); diff --git a/fs/overlayfs/util.c b/fs/overlayfs/util.c new file mode 100644 index 000000000..b41f4788e --- /dev/null +++ b/fs/overlayfs/util.c @@ -0,0 +1,1518 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2011 Novell Inc. + * Copyright (C) 2016 Red Hat, Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "overlayfs.h" + +/* Get write access to upper mnt - may fail if upper sb was remounted ro */ +int ovl_get_write_access(struct dentry *dentry) +{ + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + return mnt_get_write_access(ovl_upper_mnt(ofs)); +} + +/* Get write access to upper sb - may block if upper sb is frozen */ +void ovl_start_write(struct dentry *dentry) +{ + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + sb_start_write(ovl_upper_mnt(ofs)->mnt_sb); +} + +int ovl_want_write(struct dentry *dentry) +{ + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + return mnt_want_write(ovl_upper_mnt(ofs)); +} + +void ovl_put_write_access(struct dentry *dentry) +{ + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + mnt_put_write_access(ovl_upper_mnt(ofs)); +} + +void ovl_end_write(struct dentry *dentry) +{ + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + sb_end_write(ovl_upper_mnt(ofs)->mnt_sb); +} + +void ovl_drop_write(struct dentry *dentry) +{ + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + mnt_drop_write(ovl_upper_mnt(ofs)); +} + +struct dentry *ovl_workdir(struct dentry *dentry) +{ + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + return ofs->workdir; +} + +const struct cred *ovl_override_creds(struct super_block *sb) +{ + struct ovl_fs *ofs = OVL_FS(sb); + + return override_creds(ofs->creator_cred); +} + +/* + * Check if underlying fs supports file handles and try to determine encoding + * type, in order to deduce maximum inode number used by fs. + * + * Return 0 if file handles are not supported. + * Return 1 (FILEID_INO32_GEN) if fs uses the default 32bit inode encoding. + * Return -1 if fs uses a non default encoding with unknown inode size. + */ +int ovl_can_decode_fh(struct super_block *sb) +{ + if (!capable(CAP_DAC_READ_SEARCH)) + return 0; + + if (!exportfs_can_decode_fh(sb->s_export_op)) + return 0; + + if (sb->s_export_op->encode_fh == generic_encode_ino32_fh) + return FILEID_INO32_GEN; + + return -1; +} + +struct dentry *ovl_indexdir(struct super_block *sb) +{ + struct ovl_fs *ofs = OVL_FS(sb); + + return ofs->config.index ? ofs->workdir : NULL; +} + +/* Index all files on copy up. For now only enabled for NFS export */ +bool ovl_index_all(struct super_block *sb) +{ + struct ovl_fs *ofs = OVL_FS(sb); + + return ofs->config.nfs_export && ofs->config.index; +} + +/* Verify lower origin on lookup. For now only enabled for NFS export */ +bool ovl_verify_lower(struct super_block *sb) +{ + struct ovl_fs *ofs = OVL_FS(sb); + + return ofs->config.nfs_export && ofs->config.index; +} + +struct ovl_path *ovl_stack_alloc(unsigned int n) +{ + return kzalloc_objs(struct ovl_path, n); +} + +void ovl_stack_cpy(struct ovl_path *dst, struct ovl_path *src, unsigned int n) +{ + unsigned int i; + + memcpy(dst, src, sizeof(struct ovl_path) * n); + for (i = 0; i < n; i++) + dget(src[i].dentry); +} + +void ovl_stack_put(struct ovl_path *stack, unsigned int n) +{ + unsigned int i; + + for (i = 0; stack && i < n; i++) + dput(stack[i].dentry); +} + +void ovl_stack_free(struct ovl_path *stack, unsigned int n) +{ + ovl_stack_put(stack, n); + kfree(stack); +} + +struct ovl_entry *ovl_alloc_entry(unsigned int numlower) +{ + struct ovl_entry *oe; + + oe = kzalloc_flex(*oe, __lowerstack, numlower); + if (oe) + oe->__numlower = numlower; + + return oe; +} + +void ovl_free_entry(struct ovl_entry *oe) +{ + ovl_stack_put(ovl_lowerstack(oe), ovl_numlower(oe)); + kfree(oe); +} + +#define OVL_D_REVALIDATE (DCACHE_OP_REVALIDATE | DCACHE_OP_WEAK_REVALIDATE) + +bool ovl_dentry_remote(struct dentry *dentry) +{ + return dentry->d_flags & OVL_D_REVALIDATE; +} + +void ovl_dentry_update_reval(struct dentry *dentry, struct dentry *realdentry) +{ + if (!ovl_dentry_remote(realdentry)) + return; + + spin_lock(&dentry->d_lock); + dentry->d_flags |= realdentry->d_flags & OVL_D_REVALIDATE; + spin_unlock(&dentry->d_lock); +} + +void ovl_dentry_init_reval(struct dentry *dentry, struct dentry *upperdentry, + struct ovl_entry *oe) +{ + return ovl_dentry_init_flags(dentry, upperdentry, oe, OVL_D_REVALIDATE); +} + +void ovl_dentry_init_flags(struct dentry *dentry, struct dentry *upperdentry, + struct ovl_entry *oe, unsigned int mask) +{ + struct ovl_path *lowerstack = ovl_lowerstack(oe); + unsigned int i, flags = 0; + + if (upperdentry) + flags |= upperdentry->d_flags; + for (i = 0; i < ovl_numlower(oe) && lowerstack[i].dentry; i++) + flags |= lowerstack[i].dentry->d_flags; + + spin_lock(&dentry->d_lock); + dentry->d_flags &= ~mask; + dentry->d_flags |= flags & mask; + spin_unlock(&dentry->d_lock); +} + +bool ovl_dentry_weird(struct dentry *dentry) +{ + if (!d_can_lookup(dentry) && !d_is_file(dentry) && !d_is_symlink(dentry)) + return true; + + if (dentry->d_flags & (DCACHE_NEED_AUTOMOUNT | DCACHE_MANAGE_TRANSIT)) + return true; + + /* + * Exceptionally for layers with casefold, we accept that they have + * their own hash and compare operations + */ + if (sb_has_encoding(dentry->d_sb)) + return false; + + return dentry->d_flags & (DCACHE_OP_HASH | DCACHE_OP_COMPARE); +} + +enum ovl_path_type ovl_path_type(struct dentry *dentry) +{ + struct ovl_entry *oe = OVL_E(dentry); + enum ovl_path_type type = 0; + + if (ovl_dentry_upper(dentry)) { + type = __OVL_PATH_UPPER; + + /* + * Non-dir dentry can hold lower dentry of its copy up origin. + */ + if (ovl_numlower(oe)) { + if (ovl_test_flag(OVL_CONST_INO, d_inode(dentry))) + type |= __OVL_PATH_ORIGIN; + if (d_is_dir(dentry) || + !ovl_has_upperdata(d_inode(dentry))) + type |= __OVL_PATH_MERGE; + } + } else { + if (ovl_numlower(oe) > 1) + type |= __OVL_PATH_MERGE; + } + return type; +} + +void ovl_path_upper(struct dentry *dentry, struct path *path) +{ + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + + path->mnt = ovl_upper_mnt(ofs); + path->dentry = ovl_dentry_upper(dentry); +} + +void ovl_path_lower(struct dentry *dentry, struct path *path) +{ + struct ovl_entry *oe = OVL_E(dentry); + struct ovl_path *lowerpath = ovl_lowerstack(oe); + + if (ovl_numlower(oe)) { + path->mnt = lowerpath->layer->mnt; + path->dentry = lowerpath->dentry; + } else { + *path = (struct path) { }; + } +} + +void ovl_path_lowerdata(struct dentry *dentry, struct path *path) +{ + struct ovl_entry *oe = OVL_E(dentry); + struct ovl_path *lowerdata = ovl_lowerdata(oe); + struct dentry *lowerdata_dentry = ovl_lowerdata_dentry(oe); + + if (lowerdata_dentry) { + path->dentry = lowerdata_dentry; + /* + * Pairs with smp_wmb() in ovl_dentry_set_lowerdata(). + * Make sure that if lowerdata->dentry is visible, then + * datapath->layer is visible as well. + */ + smp_rmb(); + path->mnt = READ_ONCE(lowerdata->layer)->mnt; + } else { + *path = (struct path) { }; + } +} + +enum ovl_path_type ovl_path_real(struct dentry *dentry, struct path *path) +{ + enum ovl_path_type type = ovl_path_type(dentry); + + if (!OVL_TYPE_UPPER(type)) + ovl_path_lower(dentry, path); + else + ovl_path_upper(dentry, path); + + return type; +} + +enum ovl_path_type ovl_path_realdata(struct dentry *dentry, struct path *path) +{ + enum ovl_path_type type = ovl_path_type(dentry); + + WARN_ON_ONCE(d_is_dir(dentry)); + + if (!OVL_TYPE_UPPER(type) || OVL_TYPE_MERGE(type)) + ovl_path_lowerdata(dentry, path); + else + ovl_path_upper(dentry, path); + + return type; +} + +struct dentry *ovl_dentry_upper(struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + + return inode ? ovl_upperdentry_dereference(OVL_I(inode)) : NULL; +} + +struct dentry *ovl_dentry_lower(struct dentry *dentry) +{ + struct ovl_entry *oe = OVL_E(dentry); + + return ovl_numlower(oe) ? ovl_lowerstack(oe)->dentry : NULL; +} + +const struct ovl_layer *ovl_layer_lower(struct dentry *dentry) +{ + struct ovl_entry *oe = OVL_E(dentry); + + return ovl_numlower(oe) ? ovl_lowerstack(oe)->layer : NULL; +} + +/* + * ovl_dentry_lower() could return either a data dentry or metacopy dentry + * depending on what is stored in lowerstack[0]. At times we need to find + * lower dentry which has data (and not metacopy dentry). This helper + * returns the lower data dentry. + */ +struct dentry *ovl_dentry_lowerdata(struct dentry *dentry) +{ + return ovl_lowerdata_dentry(OVL_E(dentry)); +} + +int ovl_dentry_set_lowerdata(struct dentry *dentry, struct ovl_path *datapath) +{ + struct ovl_entry *oe = OVL_E(dentry); + struct ovl_path *lowerdata = ovl_lowerdata(oe); + struct dentry *datadentry = datapath->dentry; + + if (WARN_ON_ONCE(ovl_numlower(oe) <= 1)) + return -EIO; + + WRITE_ONCE(lowerdata->layer, datapath->layer); + /* + * Pairs with smp_rmb() in ovl_path_lowerdata(). + * Make sure that if lowerdata->dentry is visible, then + * lowerdata->layer is visible as well. + */ + smp_wmb(); + WRITE_ONCE(lowerdata->dentry, dget(datadentry)); + + ovl_dentry_update_reval(dentry, datadentry); + + return 0; +} + +struct dentry *ovl_dentry_real(struct dentry *dentry) +{ + return ovl_dentry_upper(dentry) ?: ovl_dentry_lower(dentry); +} + +struct dentry *ovl_i_dentry_upper(struct inode *inode) +{ + return ovl_upperdentry_dereference(OVL_I(inode)); +} + +struct inode *ovl_i_path_real(struct inode *inode, struct path *path) +{ + struct ovl_path *lowerpath = ovl_lowerpath(OVL_I_E(inode)); + + path->dentry = ovl_i_dentry_upper(inode); + if (!path->dentry) { + path->dentry = lowerpath->dentry; + path->mnt = lowerpath->layer->mnt; + } else { + path->mnt = ovl_upper_mnt(OVL_FS(inode->i_sb)); + } + + return path->dentry ? d_inode_rcu(path->dentry) : NULL; +} + +struct inode *ovl_inode_upper(struct inode *inode) +{ + struct dentry *upperdentry = ovl_i_dentry_upper(inode); + + return upperdentry ? d_inode(upperdentry) : NULL; +} + +struct inode *ovl_inode_lower(struct inode *inode) +{ + struct ovl_path *lowerpath = ovl_lowerpath(OVL_I_E(inode)); + + return lowerpath ? d_inode(lowerpath->dentry) : NULL; +} + +struct inode *ovl_inode_real(struct inode *inode) +{ + return ovl_inode_upper(inode) ?: ovl_inode_lower(inode); +} + +/* Return inode which contains lower data. Do not return metacopy */ +struct inode *ovl_inode_lowerdata(struct inode *inode) +{ + struct dentry *lowerdata = ovl_lowerdata_dentry(OVL_I_E(inode)); + + if (WARN_ON(!S_ISREG(inode->i_mode))) + return NULL; + + return lowerdata ? d_inode(lowerdata) : NULL; +} + +/* Return real inode which contains data. Does not return metacopy inode */ +struct inode *ovl_inode_realdata(struct inode *inode) +{ + struct inode *upperinode; + + upperinode = ovl_inode_upper(inode); + if (upperinode && ovl_has_upperdata(inode)) + return upperinode; + + return ovl_inode_lowerdata(inode); +} + +const char *ovl_lowerdata_redirect(struct inode *inode) +{ + return inode && S_ISREG(inode->i_mode) ? + OVL_I(inode)->lowerdata_redirect : NULL; +} + +struct ovl_dir_cache *ovl_dir_cache(struct inode *inode) +{ + return inode && S_ISDIR(inode->i_mode) ? OVL_I(inode)->cache : NULL; +} + +void ovl_set_dir_cache(struct inode *inode, struct ovl_dir_cache *cache) +{ + OVL_I(inode)->cache = cache; +} + +void ovl_dentry_set_flag(unsigned long flag, struct dentry *dentry) +{ + set_bit(flag, OVL_E_FLAGS(dentry)); +} + +void ovl_dentry_clear_flag(unsigned long flag, struct dentry *dentry) +{ + clear_bit(flag, OVL_E_FLAGS(dentry)); +} + +bool ovl_dentry_test_flag(unsigned long flag, struct dentry *dentry) +{ + return test_bit(flag, OVL_E_FLAGS(dentry)); +} + +bool ovl_dentry_is_opaque(struct dentry *dentry) +{ + return ovl_dentry_test_flag(OVL_E_OPAQUE, dentry); +} + +bool ovl_dentry_is_whiteout(struct dentry *dentry) +{ + return !dentry->d_inode && ovl_dentry_is_opaque(dentry); +} + +void ovl_dentry_set_opaque(struct dentry *dentry) +{ + ovl_dentry_set_flag(OVL_E_OPAQUE, dentry); +} + +bool ovl_dentry_has_xwhiteouts(struct dentry *dentry) +{ + return ovl_dentry_test_flag(OVL_E_XWHITEOUTS, dentry); +} + +void ovl_dentry_set_xwhiteouts(struct dentry *dentry) +{ + ovl_dentry_set_flag(OVL_E_XWHITEOUTS, dentry); +} + +/* + * ovl_layer_set_xwhiteouts() is called before adding the overlay dir + * dentry to dcache, while readdir of that same directory happens after + * the overlay dir dentry is in dcache, so if some cpu observes that + * ovl_dentry_is_xwhiteouts(), it will also observe layer->has_xwhiteouts + * for the layers where xwhiteouts marker was found in that merge dir. + */ +void ovl_layer_set_xwhiteouts(struct ovl_fs *ofs, + const struct ovl_layer *layer) +{ + if (layer->has_xwhiteouts) + return; + + /* Write once to read-mostly layer properties */ + ofs->layers[layer->idx].has_xwhiteouts = true; +} + +/* + * For hard links and decoded file handles, it's possible for ovl_dentry_upper() + * to return positive, while there's no actual upper alias for the inode. + * Copy up code needs to know about the existence of the upper alias, so it + * can't use ovl_dentry_upper(). + */ +bool ovl_dentry_has_upper_alias(struct dentry *dentry) +{ + return ovl_dentry_test_flag(OVL_E_UPPER_ALIAS, dentry); +} + +void ovl_dentry_set_upper_alias(struct dentry *dentry) +{ + ovl_dentry_set_flag(OVL_E_UPPER_ALIAS, dentry); +} + +static bool ovl_should_check_upperdata(struct inode *inode) +{ + if (!S_ISREG(inode->i_mode)) + return false; + + if (!ovl_inode_lower(inode)) + return false; + + return true; +} + +bool ovl_has_upperdata(struct inode *inode) +{ + if (!ovl_should_check_upperdata(inode)) + return true; + + if (!ovl_test_flag(OVL_UPPERDATA, inode)) + return false; + /* + * Pairs with smp_wmb() in ovl_set_upperdata(). Main user of + * ovl_has_upperdata() is ovl_copy_up_meta_inode_data(). Make sure + * if setting of OVL_UPPERDATA is visible, then effects of writes + * before that are visible too. + */ + smp_rmb(); + return true; +} + +void ovl_set_upperdata(struct inode *inode) +{ + /* + * Pairs with smp_rmb() in ovl_has_upperdata(). Make sure + * if OVL_UPPERDATA flag is visible, then effects of write operations + * before it are visible as well. + */ + smp_wmb(); + ovl_set_flag(OVL_UPPERDATA, inode); +} + +/* Caller should hold ovl_inode->lock */ +bool ovl_dentry_needs_data_copy_up_locked(struct dentry *dentry, int flags) +{ + if (!ovl_open_flags_need_copy_up(flags)) + return false; + + return !ovl_test_flag(OVL_UPPERDATA, d_inode(dentry)); +} + +bool ovl_dentry_needs_data_copy_up(struct dentry *dentry, int flags) +{ + if (!ovl_open_flags_need_copy_up(flags)) + return false; + + return !ovl_has_upperdata(d_inode(dentry)); +} + +const char *ovl_dentry_get_redirect(struct dentry *dentry) +{ + return OVL_I(d_inode(dentry))->redirect; +} + +void ovl_dentry_set_redirect(struct dentry *dentry, const char *redirect) +{ + struct ovl_inode *oi = OVL_I(d_inode(dentry)); + + kfree(oi->redirect); + oi->redirect = redirect; +} + +void ovl_inode_update(struct inode *inode, struct dentry *upperdentry) +{ + struct inode *upperinode = d_inode(upperdentry); + + WARN_ON(OVL_I(inode)->__upperdentry); + + /* + * Make sure upperdentry is consistent before making it visible + */ + smp_wmb(); + OVL_I(inode)->__upperdentry = upperdentry; + if (inode_unhashed(inode)) { + inode->i_private = upperinode; + __insert_inode_hash(inode, (unsigned long) upperinode); + } +} + +static void ovl_dir_version_inc(struct dentry *dentry, bool impurity) +{ + struct inode *inode = d_inode(dentry); + + WARN_ON(!inode_is_locked(inode)); + WARN_ON(!d_is_dir(dentry)); + /* + * Version is used by readdir code to keep cache consistent. + * For merge dirs (or dirs with origin) all changes need to be noted. + * For non-merge dirs, cache contains only impure entries (i.e. ones + * which have been copied up and have origins), so only need to note + * changes to impure entries. + */ + if (!ovl_dir_is_real(inode) || impurity) + OVL_I(inode)->version++; +} + +void ovl_dir_modified(struct dentry *dentry, bool impurity) +{ + /* Copy mtime/ctime */ + ovl_copyattr(d_inode(dentry)); + + ovl_dir_version_inc(dentry, impurity); +} + +u64 ovl_inode_version_get(struct inode *inode) +{ + WARN_ON(!inode_is_locked(inode)); + return OVL_I(inode)->version; +} + +bool ovl_is_whiteout(struct dentry *dentry) +{ + struct inode *inode = dentry->d_inode; + + return inode && IS_WHITEOUT(inode); +} + +/* + * Use this over ovl_is_whiteout for upper and lower files, as it also + * handles overlay.whiteout xattr whiteout files. + */ +bool ovl_path_is_whiteout(struct ovl_fs *ofs, const struct path *path) +{ + return ovl_is_whiteout(path->dentry) || + ovl_path_check_xwhiteout_xattr(ofs, path); +} + +struct file *ovl_path_open(const struct path *path, int flags) +{ + struct inode *inode = d_inode(path->dentry); + struct mnt_idmap *real_idmap = mnt_idmap(path->mnt); + int err, acc_mode; + + if (flags & ~(O_ACCMODE | O_LARGEFILE)) + BUG(); + + switch (flags & O_ACCMODE) { + case O_RDONLY: + acc_mode = MAY_READ; + break; + case O_WRONLY: + acc_mode = MAY_WRITE; + break; + default: + BUG(); + } + + err = inode_permission(real_idmap, inode, acc_mode | MAY_OPEN); + if (err) + return ERR_PTR(err); + + /* O_NOATIME is an optimization, don't fail if not permitted */ + if (inode_owner_or_capable(real_idmap, inode)) + flags |= O_NOATIME; + + return dentry_open(path, flags, current_cred()); +} + +/* Caller should hold ovl_inode->lock */ +static bool ovl_already_copied_up_locked(struct dentry *dentry, int flags) +{ + bool disconnected = dentry->d_flags & DCACHE_DISCONNECTED; + + if (ovl_dentry_upper(dentry) && + (ovl_dentry_has_upper_alias(dentry) || disconnected) && + !ovl_dentry_needs_data_copy_up_locked(dentry, flags)) + return true; + + return false; +} + +bool ovl_already_copied_up(struct dentry *dentry, int flags) +{ + bool disconnected = dentry->d_flags & DCACHE_DISCONNECTED; + + /* + * Check if copy-up has happened as well as for upper alias (in + * case of hard links) is there. + * + * Both checks are lockless: + * - false negatives: will recheck under oi->lock + * - false positives: + * + ovl_dentry_upper() uses memory barriers to ensure the + * upper dentry is up-to-date + * + ovl_dentry_has_upper_alias() relies on locking of + * upper parent i_rwsem to prevent reordering copy-up + * with rename. + */ + if (ovl_dentry_upper(dentry) && + (ovl_dentry_has_upper_alias(dentry) || disconnected) && + !ovl_dentry_needs_data_copy_up(dentry, flags)) + return true; + + return false; +} + +/* + * The copy up "transaction" keeps an elevated mnt write count on upper mnt, + * but leaves taking freeze protection on upper sb to lower level helpers. + */ +int ovl_copy_up_start(struct dentry *dentry, int flags) +{ + struct inode *inode = d_inode(dentry); + int err; + + err = ovl_inode_lock_interruptible(inode); + if (err) + return err; + + if (ovl_already_copied_up_locked(dentry, flags)) + err = 1; /* Already copied up */ + else + err = ovl_get_write_access(dentry); + if (err) + goto out_unlock; + + return 0; + +out_unlock: + ovl_inode_unlock(inode); + return err; +} + +void ovl_copy_up_end(struct dentry *dentry) +{ + ovl_put_write_access(dentry); + ovl_inode_unlock(d_inode(dentry)); +} + +bool ovl_path_check_origin_xattr(struct ovl_fs *ofs, const struct path *path) +{ + int res; + + res = ovl_path_getxattr(ofs, path, OVL_XATTR_ORIGIN, NULL, 0); + + /* Zero size value means "copied up but origin unknown" */ + if (res >= 0) + return true; + + return false; +} + +bool ovl_path_check_xwhiteout_xattr(struct ovl_fs *ofs, const struct path *path) +{ + struct dentry *dentry = path->dentry; + int res; + + /* xattr.whiteout must be a zero size regular file */ + if (!d_is_reg(dentry) || i_size_read(d_inode(dentry)) != 0) + return false; + + res = ovl_path_getxattr(ofs, path, OVL_XATTR_XWHITEOUT, NULL, 0); + return res >= 0; +} + +/* + * Load persistent uuid from xattr into s_uuid if found, or store a new + * random generated value in s_uuid and in xattr. + */ +bool ovl_init_uuid_xattr(struct super_block *sb, struct ovl_fs *ofs, + const struct path *upperpath) +{ + bool set = false; + uuid_t uuid; + int res; + + /* Try to load existing persistent uuid */ + res = ovl_path_getxattr(ofs, upperpath, OVL_XATTR_UUID, uuid.b, + UUID_SIZE); + if (res == UUID_SIZE) + goto set_uuid; + + if (res != -ENODATA) + goto fail; + + /* + * With uuid=auto, if uuid xattr is found, it will be used. + * If uuid xattrs is not found, generate a persistent uuid only on mount + * of new overlays where upper root dir is not yet marked as impure. + * An upper dir is marked as impure on copy up or lookup of its subdirs. + */ + if (ofs->config.uuid == OVL_UUID_AUTO) { + res = ovl_path_getxattr(ofs, upperpath, OVL_XATTR_IMPURE, NULL, + 0); + if (res > 0) { + /* Any mount of old overlay - downgrade to uuid=null */ + ofs->config.uuid = OVL_UUID_NULL; + return true; + } else if (res == -ENODATA) { + /* First mount of new overlay - upgrade to uuid=on */ + ofs->config.uuid = OVL_UUID_ON; + } else if (res < 0) { + goto fail; + } + + } + + /* Generate overlay instance uuid */ + uuid_gen(&uuid); + + /* Try to store persistent uuid */ + set = true; + res = ovl_setxattr(ofs, upperpath->dentry, OVL_XATTR_UUID, uuid.b, + UUID_SIZE); + if (res) + goto fail; + +set_uuid: + super_set_uuid(sb, uuid.b, sizeof(uuid)); + return true; + +fail: + ofs->config.uuid = OVL_UUID_NULL; + pr_warn("failed to %s uuid (%pd2, err=%i); falling back to uuid=null.\n", + set ? "set" : "get", upperpath->dentry, res); + return false; +} + +char ovl_get_dir_xattr_val(struct ovl_fs *ofs, const struct path *path, + enum ovl_xattr ox) +{ + int res; + char val; + + if (!d_is_dir(path->dentry)) + return 0; + + res = ovl_path_getxattr(ofs, path, ox, &val, 1); + return res == 1 ? val : 0; +} + +#define OVL_XATTR_OPAQUE_POSTFIX "opaque" +#define OVL_XATTR_REDIRECT_POSTFIX "redirect" +#define OVL_XATTR_ORIGIN_POSTFIX "origin" +#define OVL_XATTR_IMPURE_POSTFIX "impure" +#define OVL_XATTR_NLINK_POSTFIX "nlink" +#define OVL_XATTR_UPPER_POSTFIX "upper" +#define OVL_XATTR_UUID_POSTFIX "uuid" +#define OVL_XATTR_METACOPY_POSTFIX "metacopy" +#define OVL_XATTR_PROTATTR_POSTFIX "protattr" +#define OVL_XATTR_XWHITEOUT_POSTFIX "whiteout" + +#define OVL_XATTR_TAB_ENTRY(x) \ + [x] = { [false] = OVL_XATTR_TRUSTED_PREFIX x ## _POSTFIX, \ + [true] = OVL_XATTR_USER_PREFIX x ## _POSTFIX } + +const char *const ovl_xattr_table[][2] = { + OVL_XATTR_TAB_ENTRY(OVL_XATTR_OPAQUE), + OVL_XATTR_TAB_ENTRY(OVL_XATTR_REDIRECT), + OVL_XATTR_TAB_ENTRY(OVL_XATTR_ORIGIN), + OVL_XATTR_TAB_ENTRY(OVL_XATTR_IMPURE), + OVL_XATTR_TAB_ENTRY(OVL_XATTR_NLINK), + OVL_XATTR_TAB_ENTRY(OVL_XATTR_UPPER), + OVL_XATTR_TAB_ENTRY(OVL_XATTR_UUID), + OVL_XATTR_TAB_ENTRY(OVL_XATTR_METACOPY), + OVL_XATTR_TAB_ENTRY(OVL_XATTR_PROTATTR), + OVL_XATTR_TAB_ENTRY(OVL_XATTR_XWHITEOUT), +}; + +int ovl_check_setxattr(struct ovl_fs *ofs, struct dentry *upperdentry, + enum ovl_xattr ox, const void *value, size_t size, + int xerr) +{ + int err; + + if (ofs->noxattr) + return xerr; + + err = ovl_setxattr(ofs, upperdentry, ox, value, size); + + if (err == -EOPNOTSUPP) { + pr_warn("cannot set %s xattr on upper\n", ovl_xattr(ofs, ox)); + ofs->noxattr = true; + return xerr; + } + + return err; +} + +int ovl_set_impure(struct dentry *dentry, struct dentry *upperdentry) +{ + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + int err; + + if (ovl_test_flag(OVL_IMPURE, d_inode(dentry))) + return 0; + + /* + * Do not fail when upper doesn't support xattrs. + * Upper inodes won't have origin nor redirect xattr anyway. + */ + err = ovl_check_setxattr(ofs, upperdentry, OVL_XATTR_IMPURE, "y", 1, 0); + if (!err) + ovl_set_flag(OVL_IMPURE, d_inode(dentry)); + + return err; +} + + +#define OVL_PROTATTR_MAX 32 /* Reserved for future flags */ + +void ovl_check_protattr(struct inode *inode, struct dentry *upper) +{ + struct ovl_fs *ofs = OVL_FS(inode->i_sb); + u32 iflags = inode->i_flags & OVL_PROT_I_FLAGS_MASK; + char buf[OVL_PROTATTR_MAX+1]; + int res, n; + + res = ovl_getxattr_upper(ofs, upper, OVL_XATTR_PROTATTR, buf, + OVL_PROTATTR_MAX); + if (res < 0) + return; + + /* + * Initialize inode flags from overlay.protattr xattr and upper inode + * flags. If upper inode has those fileattr flags set (i.e. from old + * kernel), we do not clear them on ovl_get_inode(), but we will clear + * them on next fileattr_set(). + */ + for (n = 0; n < res; n++) { + if (buf[n] == 'a') + iflags |= S_APPEND; + else if (buf[n] == 'i') + iflags |= S_IMMUTABLE; + else + break; + } + + if (!res || n < res) { + pr_warn_ratelimited("incompatible overlay.protattr format (%pd2, len=%d)\n", + upper, res); + } else { + inode_set_flags(inode, iflags, OVL_PROT_I_FLAGS_MASK); + } +} + +int ovl_set_protattr(struct inode *inode, struct dentry *upper, + struct file_kattr *fa) +{ + struct ovl_fs *ofs = OVL_FS(inode->i_sb); + char buf[OVL_PROTATTR_MAX]; + int len = 0, err = 0; + u32 iflags = 0; + + BUILD_BUG_ON(HWEIGHT32(OVL_PROT_FS_FLAGS_MASK) > OVL_PROTATTR_MAX); + + if (fa->flags & FS_APPEND_FL) { + buf[len++] = 'a'; + iflags |= S_APPEND; + } + if (fa->flags & FS_IMMUTABLE_FL) { + buf[len++] = 'i'; + iflags |= S_IMMUTABLE; + } + + /* + * Do not allow to set protection flags when upper doesn't support + * xattrs, because we do not set those fileattr flags on upper inode. + * Remove xattr if it exist and all protection flags are cleared. + */ + if (len) { + err = ovl_check_setxattr(ofs, upper, OVL_XATTR_PROTATTR, + buf, len, -EPERM); + } else if (inode->i_flags & OVL_PROT_I_FLAGS_MASK) { + err = ovl_removexattr(ofs, upper, OVL_XATTR_PROTATTR); + if (err == -EOPNOTSUPP || err == -ENODATA) + err = 0; + } + if (err) + return err; + + inode_set_flags(inode, iflags, OVL_PROT_I_FLAGS_MASK); + + /* Mask out the fileattr flags that should not be set in upper inode */ + fa->flags &= ~OVL_PROT_FS_FLAGS_MASK; + fa->fsx_xflags &= ~OVL_PROT_FSX_FLAGS_MASK; + + return 0; +} + +/* + * Caller must hold a reference to inode to prevent it from being freed while + * it is marked inuse. + */ +bool ovl_inuse_trylock(struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + bool locked = false; + + spin_lock(&inode->i_lock); + if (!(inode_state_read(inode) & I_OVL_INUSE)) { + inode_state_set(inode, I_OVL_INUSE); + locked = true; + } + spin_unlock(&inode->i_lock); + + return locked; +} + +void ovl_inuse_unlock(struct dentry *dentry) +{ + if (dentry) { + struct inode *inode = d_inode(dentry); + + spin_lock(&inode->i_lock); + WARN_ON(!(inode_state_read(inode) & I_OVL_INUSE)); + inode_state_clear(inode, I_OVL_INUSE); + spin_unlock(&inode->i_lock); + } +} + +bool ovl_is_inuse(struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + bool inuse; + + spin_lock(&inode->i_lock); + inuse = (inode_state_read(inode) & I_OVL_INUSE); + spin_unlock(&inode->i_lock); + + return inuse; +} + +/* + * Does this overlay dentry need to be indexed on copy up? + */ +bool ovl_need_index(struct dentry *dentry) +{ + struct dentry *lower = ovl_dentry_lower(dentry); + + if (!lower || !ovl_indexdir(dentry->d_sb)) + return false; + + /* Index all files for NFS export and consistency verification */ + if (ovl_index_all(dentry->d_sb)) + return true; + + /* Index only lower hardlinks on copy up */ + if (!d_is_dir(lower) && d_inode(lower)->i_nlink > 1) + return true; + + return false; +} + +/* Caller must hold OVL_I(inode)->lock */ +static void ovl_cleanup_index(struct dentry *dentry) +{ + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + struct dentry *indexdir = ovl_indexdir(dentry->d_sb); + struct dentry *lowerdentry = ovl_dentry_lower(dentry); + struct dentry *upperdentry = ovl_dentry_upper(dentry); + struct dentry *index = NULL; + struct inode *inode; + struct qstr name = { }; + bool got_write = false; + int err; + + err = ovl_get_index_name(ofs, lowerdentry, &name); + if (err) + goto fail; + + err = ovl_want_write(dentry); + if (err) + goto fail; + + got_write = true; + inode = d_inode(upperdentry); + if (!S_ISDIR(inode->i_mode) && inode->i_nlink != 1) { + pr_warn_ratelimited("cleanup linked index (%pd2, ino=%llu, nlink=%u)\n", + upperdentry, inode->i_ino, inode->i_nlink); + /* + * We either have a bug with persistent union nlink or a lower + * hardlink was added while overlay is mounted. Adding a lower + * hardlink and then unlinking all overlay hardlinks would drop + * overlay nlink to zero before all upper inodes are unlinked. + * As a safety measure, when that situation is detected, set + * the overlay nlink to the index inode nlink minus one for the + * index entry itself. + */ + set_nlink(d_inode(dentry), inode->i_nlink - 1); + ovl_set_nlink_upper(dentry); + goto out; + } + + index = ovl_lookup_upper_unlocked(ofs, name.name, indexdir, name.len); + err = PTR_ERR(index); + if (IS_ERR(index)) { + index = NULL; + } else if (ovl_index_all(dentry->d_sb)) { + /* Whiteout orphan index to block future open by handle */ + err = ovl_cleanup_and_whiteout(OVL_FS(dentry->d_sb), + indexdir, index); + } else { + /* Cleanup orphan index entries */ + err = ovl_cleanup(ofs, indexdir, index); + } + if (err) + goto fail; + +out: + if (got_write) + ovl_drop_write(dentry); + kfree(name.name); + dput(index); + return; + +fail: + pr_err("cleanup index of '%pd2' failed (%i)\n", dentry, err); + goto out; +} + +/* + * Operations that change overlay inode and upper inode nlink need to be + * synchronized with copy up for persistent nlink accounting. + */ +int ovl_nlink_start(struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + int err; + + if (WARN_ON(!inode)) + return -ENOENT; + + /* + * With inodes index is enabled, we store the union overlay nlink + * in an xattr on the index inode. When whiting out an indexed lower, + * we need to decrement the overlay persistent nlink, but before the + * first copy up, we have no upper index inode to store the xattr. + * + * As a workaround, before whiteout/rename over an indexed lower, + * copy up to create the upper index. Creating the upper index will + * initialize the overlay nlink, so it could be dropped if unlink + * or rename succeeds. + * + * TODO: implement metadata only index copy up when called with + * ovl_copy_up_flags(dentry, O_PATH). + */ + if (ovl_need_index(dentry) && !ovl_dentry_has_upper_alias(dentry)) { + err = ovl_copy_up(dentry); + if (err) + return err; + } + + err = ovl_inode_lock_interruptible(inode); + if (err) + return err; + + err = ovl_want_write(dentry); + if (err) + goto out_unlock; + + if (d_is_dir(dentry) || !ovl_test_flag(OVL_INDEX, inode)) + return 0; + + /* + * The overlay inode nlink should be incremented/decremented IFF the + * upper operation succeeds, along with nlink change of upper inode. + * Therefore, before link/unlink/rename, we store the union nlink + * value relative to the upper inode nlink in an upper inode xattr. + */ + with_ovl_creds(dentry->d_sb) + err = ovl_set_nlink_upper(dentry); + if (err) + goto out_drop_write; + + return 0; + +out_drop_write: + ovl_drop_write(dentry); +out_unlock: + ovl_inode_unlock(inode); + + return err; +} + +void ovl_nlink_end(struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + + ovl_drop_write(dentry); + + if (ovl_test_flag(OVL_INDEX, inode) && inode->i_nlink == 0) { + with_ovl_creds(dentry->d_sb) + ovl_cleanup_index(dentry); + } + + ovl_inode_unlock(inode); +} + +/* + * err < 0, 0 if no metacopy xattr, metacopy data size if xattr found. + * an empty xattr returns OVL_METACOPY_MIN_SIZE to distinguish from no xattr value. + */ +int ovl_check_metacopy_xattr(struct ovl_fs *ofs, const struct path *path, + struct ovl_metacopy *data) +{ + int res; + + /* Only regular files can have metacopy xattr */ + if (!S_ISREG(d_inode(path->dentry)->i_mode)) + return 0; + + res = ovl_path_getxattr(ofs, path, OVL_XATTR_METACOPY, + data, data ? OVL_METACOPY_MAX_SIZE : 0); + if (res < 0) { + if (res == -ENODATA || res == -EOPNOTSUPP) + return 0; + /* + * getxattr on user.* may fail with EACCES in case there's no + * read permission on the inode. Not much we can do, other than + * tell the caller that this is not a metacopy inode. + */ + if (ofs->config.userxattr && res == -EACCES) + return 0; + goto out; + } + + if (res == 0) { + /* Emulate empty data for zero size metacopy xattr */ + res = OVL_METACOPY_MIN_SIZE; + if (data) { + memset(data, 0, res); + data->len = res; + } + } else if (res < OVL_METACOPY_MIN_SIZE) { + pr_warn_ratelimited("metacopy file '%pd' has too small xattr\n", + path->dentry); + return -EIO; + } else if (data) { + if (data->version != 0) { + pr_warn_ratelimited("metacopy file '%pd' has unsupported version\n", + path->dentry); + return -EIO; + } + if (res != data->len) { + pr_warn_ratelimited("metacopy file '%pd' has invalid xattr size\n", + path->dentry); + return -EIO; + } + } + + return res; +out: + pr_warn_ratelimited("failed to get metacopy (%i)\n", res); + return res; +} + +int ovl_set_metacopy_xattr(struct ovl_fs *ofs, struct dentry *d, struct ovl_metacopy *metacopy) +{ + size_t len = metacopy->len; + + /* If no flags or digest fall back to empty metacopy file */ + if (metacopy->version == 0 && metacopy->flags == 0 && metacopy->digest_algo == 0) + len = 0; + + return ovl_check_setxattr(ofs, d, OVL_XATTR_METACOPY, + metacopy, len, -EOPNOTSUPP); +} + +bool ovl_is_metacopy_dentry(struct dentry *dentry) +{ + struct ovl_entry *oe = OVL_E(dentry); + + if (!d_is_reg(dentry)) + return false; + + if (ovl_dentry_upper(dentry)) { + if (!ovl_has_upperdata(d_inode(dentry))) + return true; + return false; + } + + return (ovl_numlower(oe) > 1); +} + +char *ovl_get_redirect_xattr(struct ovl_fs *ofs, const struct path *path, int padding) +{ + int res; + char *s, *next, *buf = NULL; + + res = ovl_path_getxattr(ofs, path, OVL_XATTR_REDIRECT, NULL, 0); + if (res == -ENODATA || res == -EOPNOTSUPP) + return NULL; + if (res < 0) + goto fail; + if (res == 0) + goto invalid; + + buf = kzalloc(res + padding + 1, GFP_KERNEL); + if (!buf) + return ERR_PTR(-ENOMEM); + + res = ovl_path_getxattr(ofs, path, OVL_XATTR_REDIRECT, buf, res); + if (res < 0) + goto fail; + if (res == 0) + goto invalid; + + if (buf[0] == '/') { + for (s = buf; *s++ == '/'; s = next) { + next = strchrnul(s, '/'); + if (s == next) + goto invalid; + } + } else { + if (strchr(buf, '/') != NULL) + goto invalid; + } + + return buf; +invalid: + pr_warn_ratelimited("invalid redirect (%s)\n", buf); + res = -EINVAL; + goto err_free; +fail: + pr_warn_ratelimited("failed to get redirect (%i)\n", res); +err_free: + kfree(buf); + return ERR_PTR(res); +} + +/* Call with mounter creds as it may open the file */ +int ovl_ensure_verity_loaded(const struct path *datapath) +{ + struct inode *inode = d_inode(datapath->dentry); + struct file *filp; + + if (IS_VERITY(inode) && fsverity_get_info(inode) == NULL) { + /* + * If this inode was not yet opened, the verity info hasn't been + * loaded yet, so we need to do that here to force it into memory. + */ + filp = kernel_file_open(datapath, O_RDONLY, current_cred()); + if (IS_ERR(filp)) + return PTR_ERR(filp); + fput(filp); + } + + return 0; +} + +int ovl_validate_verity(struct ovl_fs *ofs, + const struct path *metapath, + const struct path *datapath) +{ + struct ovl_metacopy metacopy_data; + u8 actual_digest[FS_VERITY_MAX_DIGEST_SIZE]; + int xattr_digest_size, digest_size; + int xattr_size, err; + u8 verity_algo; + + if (!ofs->config.verity_mode || + /* Verity only works on regular files */ + !S_ISREG(d_inode(metapath->dentry)->i_mode)) + return 0; + + xattr_size = ovl_check_metacopy_xattr(ofs, metapath, &metacopy_data); + if (xattr_size < 0) + return xattr_size; + + if (!xattr_size || !metacopy_data.digest_algo) { + if (ofs->config.verity_mode == OVL_VERITY_REQUIRE) { + pr_warn_ratelimited("metacopy file '%pd' has no digest specified\n", + metapath->dentry); + return -EIO; + } + return 0; + } + + xattr_digest_size = ovl_metadata_digest_size(&metacopy_data); + + err = ovl_ensure_verity_loaded(datapath); + if (err < 0) { + pr_warn_ratelimited("lower file '%pd' failed to load fs-verity info\n", + datapath->dentry); + return -EIO; + } + + digest_size = fsverity_get_digest(d_inode(datapath->dentry), actual_digest, + &verity_algo, NULL); + if (digest_size == 0) { + pr_warn_ratelimited("lower file '%pd' has no fs-verity digest\n", datapath->dentry); + return -EIO; + } + + if (xattr_digest_size != digest_size || + metacopy_data.digest_algo != verity_algo || + memcmp(metacopy_data.digest, actual_digest, xattr_digest_size) != 0) { + pr_warn_ratelimited("lower file '%pd' has the wrong fs-verity digest\n", + datapath->dentry); + return -EIO; + } + + return 0; +} + +int ovl_get_verity_digest(struct ovl_fs *ofs, const struct path *src, + struct ovl_metacopy *metacopy) +{ + int err, digest_size; + + if (!ofs->config.verity_mode || !S_ISREG(d_inode(src->dentry)->i_mode)) + return 0; + + err = ovl_ensure_verity_loaded(src); + if (err < 0) { + pr_warn_ratelimited("lower file '%pd' failed to load fs-verity info\n", + src->dentry); + return -EIO; + } + + digest_size = fsverity_get_digest(d_inode(src->dentry), + metacopy->digest, &metacopy->digest_algo, NULL); + if (digest_size == 0 || + WARN_ON_ONCE(digest_size > FS_VERITY_MAX_DIGEST_SIZE)) { + if (ofs->config.verity_mode == OVL_VERITY_REQUIRE) { + pr_warn_ratelimited("lower file '%pd' has no fs-verity digest\n", + src->dentry); + return -EIO; + } + return 0; + } + + metacopy->len += digest_size; + return 0; +} + +/* + * ovl_sync_status() - Check fs sync status for volatile mounts + * + * Returns 1 if this is not a volatile mount and a real sync is required. + * + * Returns 0 if syncing can be skipped because mount is volatile, and no errors + * have occurred on the upperdir since the mount. + * + * Returns -errno if it is a volatile mount, and the error that occurred since + * the last mount. If the error code changes, it'll return the latest error + * code. + */ + +int ovl_sync_status(struct ovl_fs *ofs) +{ + struct vfsmount *mnt; + + if (ovl_should_sync(ofs)) + return 1; + + mnt = ovl_upper_mnt(ofs); + if (!mnt) + return 0; + + return errseq_check(&mnt->mnt_sb->s_wb_err, ofs->errseq); +} + +/* + * ovl_copyattr() - copy inode attributes from layer to ovl inode + * + * When overlay copies inode information from an upper or lower layer to the + * relevant overlay inode it will apply the idmapping of the upper or lower + * layer when doing so ensuring that the ovl inode ownership will correctly + * reflect the ownership of the idmapped upper or lower layer. For example, an + * idmapped upper or lower layer mapping id 1001 to id 1000 will take care to + * map any lower or upper inode owned by id 1001 to id 1000. These mapping + * helpers are nops when the relevant layer isn't idmapped. + */ +void ovl_copyattr(struct inode *inode) +{ + struct path realpath; + struct inode *realinode; + struct mnt_idmap *real_idmap; + vfsuid_t vfsuid; + vfsgid_t vfsgid; + + realinode = ovl_i_path_real(inode, &realpath); + real_idmap = mnt_idmap(realpath.mnt); + + spin_lock(&inode->i_lock); + vfsuid = i_uid_into_vfsuid(real_idmap, realinode); + vfsgid = i_gid_into_vfsgid(real_idmap, realinode); + + inode->i_uid = vfsuid_into_kuid(vfsuid); + inode->i_gid = vfsgid_into_kgid(vfsgid); + inode->i_mode = realinode->i_mode; + inode_set_atime_to_ts(inode, inode_get_atime(realinode)); + inode_set_mtime_to_ts(inode, inode_get_mtime(realinode)); + inode_set_ctime_to_ts(inode, inode_get_ctime(realinode)); + i_size_write(inode, i_size_read(realinode)); + spin_unlock(&inode->i_lock); +} diff --git a/fs/overlayfs/xattrs.c b/fs/overlayfs/xattrs.c new file mode 100644 index 000000000..5ae44b9c8 --- /dev/null +++ b/fs/overlayfs/xattrs.c @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include "overlayfs.h" + +static bool ovl_is_escaped_xattr(struct super_block *sb, const char *name) +{ + struct ovl_fs *ofs = sb->s_fs_info; + + if (ofs->config.userxattr) + return strncmp(name, OVL_XATTR_ESCAPE_USER_PREFIX, + OVL_XATTR_ESCAPE_USER_PREFIX_LEN) == 0; + else + return strncmp(name, OVL_XATTR_ESCAPE_TRUSTED_PREFIX, + OVL_XATTR_ESCAPE_TRUSTED_PREFIX_LEN) == 0; +} + +static bool ovl_is_own_xattr(struct super_block *sb, const char *name) +{ + struct ovl_fs *ofs = OVL_FS(sb); + + if (ofs->config.userxattr) + return strncmp(name, OVL_XATTR_USER_PREFIX, + OVL_XATTR_USER_PREFIX_LEN) == 0; + else + return strncmp(name, OVL_XATTR_TRUSTED_PREFIX, + OVL_XATTR_TRUSTED_PREFIX_LEN) == 0; +} + +bool ovl_is_private_xattr(struct super_block *sb, const char *name) +{ + return ovl_is_own_xattr(sb, name) && !ovl_is_escaped_xattr(sb, name); +} + +static int ovl_xattr_set(struct dentry *dentry, struct inode *inode, const char *name, + const void *value, size_t size, int flags) +{ + int err; + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + struct dentry *upperdentry = ovl_i_dentry_upper(inode); + struct dentry *realdentry = upperdentry ?: ovl_dentry_lower(dentry); + struct path realpath; + + if (!value && !upperdentry) { + ovl_path_lower(dentry, &realpath); + with_ovl_creds(dentry->d_sb) + err = vfs_getxattr(mnt_idmap(realpath.mnt), realdentry, name, NULL, 0); + if (err < 0) + goto out; + } + + if (!upperdentry) { + err = ovl_copy_up(dentry); + if (err) + goto out; + + realdentry = ovl_dentry_upper(dentry); + } + + err = ovl_want_write(dentry); + if (err) + goto out; + + with_ovl_creds(dentry->d_sb) { + if (value) { + err = ovl_do_setxattr(ofs, realdentry, name, value, size, flags); + } else { + WARN_ON(flags != XATTR_REPLACE); + err = ovl_do_removexattr(ofs, realdentry, name); + } + } + ovl_drop_write(dentry); + + /* copy c/mtime */ + ovl_copyattr(inode); +out: + return err; +} + +static int ovl_xattr_get(struct dentry *dentry, struct inode *inode, const char *name, + void *value, size_t size) +{ + struct path realpath; + + ovl_i_path_real(inode, &realpath); + /* Use vfs_getxattr(), not __vfs_getxattr(): it idmaps the security.capability rootid. */ + with_ovl_creds(dentry->d_sb) + return vfs_getxattr(mnt_idmap(realpath.mnt), realpath.dentry, name, value, size); +} + +static bool ovl_can_list(struct super_block *sb, const char *s) +{ + /* Never list private (.overlay) */ + if (ovl_is_private_xattr(sb, s)) + return false; + + /* List all non-trusted xattrs */ + if (strncmp(s, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN) != 0) + return true; + + /* list other trusted for superuser only */ + return ns_capable_noaudit(&init_user_ns, CAP_SYS_ADMIN); +} + +ssize_t ovl_listxattr(struct dentry *dentry, char *list, size_t size) +{ + struct dentry *realdentry = ovl_dentry_real(dentry); + struct ovl_fs *ofs = OVL_FS(dentry->d_sb); + ssize_t res; + size_t len; + char *s; + size_t prefix_len, name_len; + + with_ovl_creds(dentry->d_sb) + res = vfs_listxattr(realdentry, list, size); + if (res <= 0 || size == 0) + return res; + + prefix_len = ofs->config.userxattr ? + OVL_XATTR_USER_PREFIX_LEN : OVL_XATTR_TRUSTED_PREFIX_LEN; + + /* filter out private xattrs */ + for (s = list, len = res; len;) { + size_t slen = strnlen(s, len) + 1; + + /* underlying fs providing us with an broken xattr list? */ + if (WARN_ON(slen > len)) + return -EIO; + + len -= slen; + if (!ovl_can_list(dentry->d_sb, s)) { + res -= slen; + memmove(s, s + slen, len); + } else if (ovl_is_escaped_xattr(dentry->d_sb, s)) { + res -= OVL_XATTR_ESCAPE_PREFIX_LEN; + name_len = slen - prefix_len - OVL_XATTR_ESCAPE_PREFIX_LEN; + s += prefix_len; + memmove(s, s + OVL_XATTR_ESCAPE_PREFIX_LEN, name_len + len); + s += name_len; + } else { + s += slen; + } + } + + return res; +} + +static char *ovl_xattr_escape_name(const char *prefix, const char *name) +{ + size_t prefix_len = strlen(prefix); + size_t name_len = strlen(name); + size_t escaped_len; + char *escaped, *s; + + escaped_len = prefix_len + OVL_XATTR_ESCAPE_PREFIX_LEN + name_len; + if (escaped_len > XATTR_NAME_MAX) + return ERR_PTR(-EOPNOTSUPP); + + escaped = kmalloc(escaped_len + 1, GFP_KERNEL); + if (escaped == NULL) + return ERR_PTR(-ENOMEM); + + s = escaped; + memcpy(s, prefix, prefix_len); + s += prefix_len; + memcpy(s, OVL_XATTR_ESCAPE_PREFIX, OVL_XATTR_ESCAPE_PREFIX_LEN); + s += OVL_XATTR_ESCAPE_PREFIX_LEN; + memcpy(s, name, name_len + 1); + + return escaped; +} + +static int ovl_own_xattr_get(const struct xattr_handler *handler, + struct dentry *dentry, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + char *escaped; + int r; + + escaped = ovl_xattr_escape_name(handler->prefix, name); + if (IS_ERR(escaped)) + return PTR_ERR(escaped); + + r = ovl_xattr_get(dentry, inode, escaped, buffer, size); + + kfree(escaped); + + return r; +} + +static int ovl_own_xattr_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *dentry, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +{ + char *escaped; + int r; + + escaped = ovl_xattr_escape_name(handler->prefix, name); + if (IS_ERR(escaped)) + return PTR_ERR(escaped); + + r = ovl_xattr_set(dentry, inode, escaped, value, size, flags); + + kfree(escaped); + + return r; +} + +static int ovl_other_xattr_get(const struct xattr_handler *handler, + struct dentry *dentry, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + return ovl_xattr_get(dentry, inode, name, buffer, size); +} + +static int ovl_other_xattr_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *dentry, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +{ + return ovl_xattr_set(dentry, inode, name, value, size, flags); +} + +static const struct xattr_handler ovl_own_trusted_xattr_handler = { + .prefix = OVL_XATTR_TRUSTED_PREFIX, + .get = ovl_own_xattr_get, + .set = ovl_own_xattr_set, +}; + +static const struct xattr_handler ovl_own_user_xattr_handler = { + .prefix = OVL_XATTR_USER_PREFIX, + .get = ovl_own_xattr_get, + .set = ovl_own_xattr_set, +}; + +static const struct xattr_handler ovl_other_xattr_handler = { + .prefix = "", /* catch all */ + .get = ovl_other_xattr_get, + .set = ovl_other_xattr_set, +}; + +static const struct xattr_handler * const ovl_trusted_xattr_handlers[] = { + &ovl_own_trusted_xattr_handler, + &ovl_other_xattr_handler, + NULL +}; + +static const struct xattr_handler * const ovl_user_xattr_handlers[] = { + &ovl_own_user_xattr_handler, + &ovl_other_xattr_handler, + NULL +}; + +const struct xattr_handler * const *ovl_xattr_handlers(struct ovl_fs *ofs) +{ + return ofs->config.userxattr ? ovl_user_xattr_handlers : + ovl_trusted_xattr_handlers; +} diff --git a/fs/pidfs.c b/fs/pidfs.c new file mode 100644 index 000000000..a6a643f15 --- /dev/null +++ b/fs/pidfs.c @@ -0,0 +1,1195 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" +#include "mount.h" + +#define PIDFS_PID_DEAD ERR_PTR(-ESRCH) + +static struct kmem_cache *pidfs_attr_cachep __ro_after_init; + +static struct path pidfs_root_path = {}; + +static struct simple_xattr_cache pidfs_xa_cache; + +void pidfs_get_root(struct path *path) +{ + *path = pidfs_root_path; + path_get(path); +} + +enum pidfs_attr_mask_bits { + PIDFS_ATTR_BIT_EXIT = 0, + PIDFS_ATTR_BIT_COREDUMP = 1, +}; + +struct pidfs_anon_attr { + unsigned long attr_mask; + struct /* exit info */ { + __u64 cgroupid; + __s32 exit_code; + }; + __u32 coredump_mask; + __u32 coredump_signal; + __u32 coredump_code; +}; + +static struct rhashtable pidfs_ino_ht; + +static const struct rhashtable_params pidfs_ino_ht_params = { + .key_offset = offsetof(struct pid, ino), + .key_len = sizeof(u64), + .head_offset = offsetof(struct pid, pidfs_hash), + .automatic_shrinking = true, +}; + +/* + * inode number handling + * + * On 64 bit nothing special happens. The 64bit number assigned + * to struct pid is the inode number. + * + * On 32 bit the 64 bit number assigned to struct pid is split + * into two 32 bit numbers. The lower 32 bits are used as the + * inode number and the upper 32 bits are used as the inode + * generation number. + * + * On 32 bit pidfs_ino() will return the lower 32 bit. When + * pidfs_ino() returns zero a wrap around happened. When a + * wraparound happens the 64 bit number will be incremented by 1 + * so inode numbering starts at 1 again. + * + * On 64 bit comparing two pidfds is as simple as comparing + * inode numbers. + * + * When a wraparound happens on 32 bit multiple pidfds with the + * same inode number are likely to exist (This isn't a problem + * since before pidfs pidfds used the anonymous inode meaning + * all pidfds had the same inode number.). Userspace can + * reconstruct the 64 bit identifier by retrieving both the + * inode number and the inode generation number to compare or + * use file handles. + */ +struct pidfs_attr { + struct list_head xattrs; + union { + struct pidfs_anon_attr; + struct llist_node pidfs_llist; + }; +}; + +#if BITS_PER_LONG == 32 + +static DEFINE_SPINLOCK(pidfs_ino_lock); +static u64 pidfs_ino_nr = 1; + +static inline unsigned long pidfs_ino(u64 ino) +{ + return lower_32_bits(ino); +} + +/* On 32 bit the generation number are the upper 32 bits. */ +static inline u32 pidfs_gen(u64 ino) +{ + return upper_32_bits(ino); +} + +static inline u64 pidfs_alloc_ino(void) +{ + u64 ino; + + spin_lock(&pidfs_ino_lock); + if (pidfs_ino(pidfs_ino_nr) == 0) + pidfs_ino_nr++; + ino = pidfs_ino_nr++; + spin_unlock(&pidfs_ino_lock); + return ino; +} + +#else + +/* On 64 bit simply return ino. */ +static inline unsigned long pidfs_ino(u64 ino) +{ + return ino; +} + +/* On 64 bit the generation number is 0. */ +static inline u32 pidfs_gen(u64 ino) +{ + return 0; +} + +DEFINE_COOKIE(pidfs_ino_cookie); + +static u64 pidfs_alloc_ino(void) +{ + u64 ino; + + preempt_disable(); + ino = gen_cookie_next(&pidfs_ino_cookie); + preempt_enable(); + + VFS_WARN_ON_ONCE(ino < 1); + return ino; +} + +#endif + +void pidfs_prepare_pid(struct pid *pid) +{ + pid->stashed = NULL; + pid->attr = NULL; + pid->ino = 0; +} + +int pidfs_add_pid(struct pid *pid) +{ + int ret; + + pid->ino = pidfs_alloc_ino(); + ret = rhashtable_insert_fast(&pidfs_ino_ht, &pid->pidfs_hash, + pidfs_ino_ht_params); + if (unlikely(ret)) + pid->ino = 0; + return ret; +} + +void pidfs_remove_pid(struct pid *pid) +{ + if (likely(pid->ino)) + rhashtable_remove_fast(&pidfs_ino_ht, &pid->pidfs_hash, + pidfs_ino_ht_params); +} + +static LLIST_HEAD(pidfs_free_list); + +static void pidfs_free_attr_work(struct work_struct *work) +{ + struct pidfs_attr *attr, *next; + struct llist_node *head; + + head = llist_del_all(&pidfs_free_list); + llist_for_each_entry_safe(attr, next, head, pidfs_llist) { + simple_xattrs_free(&pidfs_xa_cache, &attr->xattrs, NULL); + kfree(attr); + } +} + +static DECLARE_WORK(pidfs_free_work, pidfs_free_attr_work); + +void pidfs_free_pid(struct pid *pid) +{ + struct pidfs_attr *attr = pid->attr; + + /* + * Any dentry must've been wiped from the pid by now. + * Otherwise there's a reference count bug. + */ + VFS_WARN_ON_ONCE(pid->stashed); + + /* + * This if an error occurred during e.g., task creation that + * causes us to never go through the exit path. + */ + if (unlikely(!attr)) + return; + + /* This never had a pidfd created. */ + if (IS_ERR(attr)) + return; + + if (likely(list_empty(&attr->xattrs))) + kfree(attr); + else if (llist_add(&attr->pidfs_llist, &pidfs_free_list)) + schedule_work(&pidfs_free_work); +} + +#ifdef CONFIG_PROC_FS +/** + * pidfd_show_fdinfo - print information about a pidfd + * @m: proc fdinfo file + * @f: file referencing a pidfd + * + * Pid: + * This function will print the pid that a given pidfd refers to in the + * pid namespace of the procfs instance. + * If the pid namespace of the process is not a descendant of the pid + * namespace of the procfs instance 0 will be shown as its pid. This is + * similar to calling getppid() on a process whose parent is outside of + * its pid namespace. + * + * NSpid: + * If pid namespaces are supported then this function will also print + * the pid of a given pidfd refers to for all descendant pid namespaces + * starting from the current pid namespace of the instance, i.e. the + * Pid field and the first entry in the NSpid field will be identical. + * If the pid namespace of the process is not a descendant of the pid + * namespace of the procfs instance 0 will be shown as its first NSpid + * entry and no others will be shown. + * Note that this differs from the Pid and NSpid fields in + * /proc//status where Pid and NSpid are always shown relative to + * the pid namespace of the procfs instance. The difference becomes + * obvious when sending around a pidfd between pid namespaces from a + * different branch of the tree, i.e. where no ancestral relation is + * present between the pid namespaces: + * - create two new pid namespaces ns1 and ns2 in the initial pid + * namespace (also take care to create new mount namespaces in the + * new pid namespace and mount procfs) + * - create a process with a pidfd in ns1 + * - send pidfd from ns1 to ns2 + * - read /proc/self/fdinfo/ and observe that both Pid and NSpid + * have exactly one entry, which is 0 + */ +static void pidfd_show_fdinfo(struct seq_file *m, struct file *f) +{ + struct pid *pid = pidfd_pid(f); + struct pid_namespace *ns; + pid_t nr = -1; + + if (likely(pid_has_task(pid, PIDTYPE_PID))) { + ns = proc_pid_ns(file_inode(m->file)->i_sb); + nr = pid_nr_ns(pid, ns); + } + + seq_put_decimal_ll(m, "Pid:\t", nr); + +#ifdef CONFIG_PID_NS + seq_put_decimal_ll(m, "\nNSpid:\t", nr); + if (nr > 0) { + int i; + + /* If nr is non-zero it means that 'pid' is valid and that + * ns, i.e. the pid namespace associated with the procfs + * instance, is in the pid namespace hierarchy of pid. + * Start at one below the already printed level. + */ + for (i = ns->level + 1; i <= pid->level; i++) + seq_put_decimal_ll(m, "\t", pid->numbers[i].nr); + } +#endif + seq_putc(m, '\n'); +} +#endif + +/* + * Poll support for process exit notification. + */ +static __poll_t pidfd_poll(struct file *file, struct poll_table_struct *pts) +{ + struct pid *pid = pidfd_pid(file); + struct task_struct *task; + __poll_t poll_flags = 0; + + poll_wait(file, &pid->wait_pidfd, pts); + /* + * Don't wake waiters if the thread-group leader exited + * prematurely. They either get notified when the last subthread + * exits or not at all if one of the remaining subthreads execs + * and assumes the struct pid of the old thread-group leader. + */ + guard(rcu)(); + task = pid_task(pid, PIDTYPE_PID); + if (!task) + poll_flags = EPOLLIN | EPOLLRDNORM | EPOLLHUP; + else if (task->exit_state && !delay_group_leader(task)) + poll_flags = EPOLLIN | EPOLLRDNORM; + + return poll_flags; +} + +static inline bool pid_in_current_pidns(const struct pid *pid) +{ + const struct pid_namespace *ns = task_active_pid_ns(current); + + if (ns->level <= pid->level) + return pid->numbers[ns->level].ns == ns; + + return false; +} + +static __u32 pidfs_coredump_mask(enum task_dumpable dumpable) +{ + switch (dumpable) { + case TASK_DUMPABLE_OWNER: + return PIDFD_COREDUMP_USER; + case TASK_DUMPABLE_ROOT: + return PIDFD_COREDUMP_ROOT; + case TASK_DUMPABLE_OFF: + return PIDFD_COREDUMP_SKIP; + default: + WARN_ON_ONCE(true); + } + + return 0; +} + +/* This must be updated whenever a new flag is added */ +#define PIDFD_INFO_SUPPORTED (PIDFD_INFO_PID | \ + PIDFD_INFO_CREDS | \ + PIDFD_INFO_CGROUPID | \ + PIDFD_INFO_EXIT | \ + PIDFD_INFO_COREDUMP | \ + PIDFD_INFO_SUPPORTED_MASK | \ + PIDFD_INFO_COREDUMP_SIGNAL | \ + PIDFD_INFO_COREDUMP_CODE) + +static long pidfd_info(struct file *file, unsigned int cmd, unsigned long arg) +{ + struct pidfd_info __user *uinfo = (struct pidfd_info __user *)arg; + struct task_struct *task __free(put_task) = NULL; + struct pid *pid = pidfd_pid(file); + size_t usize = _IOC_SIZE(cmd); + struct pidfd_info kinfo = {}; + struct user_namespace *user_ns; + struct pidfs_attr *attr; + const struct cred *c; + __u64 mask; + + BUILD_BUG_ON(sizeof(struct pidfd_info) != PIDFD_INFO_SIZE_VER3); + + if (!uinfo) + return -EINVAL; + if (usize < PIDFD_INFO_SIZE_VER0) + return -EINVAL; /* First version, no smaller struct possible */ + + if (copy_from_user(&mask, &uinfo->mask, sizeof(mask))) + return -EFAULT; + + /* + * Restrict information retrieval to tasks within the caller's pid + * namespace hierarchy. + */ + if (!pid_in_current_pidns(pid)) + return -EREMOTE; + + attr = READ_ONCE(pid->attr); + if (mask & PIDFD_INFO_EXIT) { + if (test_bit(PIDFS_ATTR_BIT_EXIT, &attr->attr_mask)) { + smp_rmb(); + kinfo.mask |= PIDFD_INFO_EXIT; +#ifdef CONFIG_CGROUPS + kinfo.cgroupid = attr->cgroupid; + kinfo.mask |= PIDFD_INFO_CGROUPID; +#endif + kinfo.exit_code = attr->exit_code; + } + } + + if (mask & PIDFD_INFO_COREDUMP) { + if (test_bit(PIDFS_ATTR_BIT_COREDUMP, &attr->attr_mask)) { + smp_rmb(); + kinfo.mask |= PIDFD_INFO_COREDUMP | PIDFD_INFO_COREDUMP_SIGNAL | PIDFD_INFO_COREDUMP_CODE; + kinfo.coredump_mask = attr->coredump_mask; + kinfo.coredump_signal = attr->coredump_signal; + kinfo.coredump_code = attr->coredump_code; + } + } + + task = get_pid_task(pid, PIDTYPE_PID); + if (!task) { + /* + * If the task has already been reaped, only exit + * information is available + */ + if (!(mask & PIDFD_INFO_EXIT)) + return -ESRCH; + + goto copy_out; + } + + c = get_task_cred(task); + if (!c) + return -ESRCH; + + if ((mask & PIDFD_INFO_COREDUMP) && !kinfo.coredump_mask) { + kinfo.coredump_mask = pidfs_coredump_mask(task_exec_state_get_dumpable(task)); + kinfo.mask |= PIDFD_INFO_COREDUMP; + /* No coredump actually took place, so no coredump signal. */ + } + + /* Unconditionally return identifiers and credentials, the rest only on request */ + + user_ns = current_user_ns(); + kinfo.ruid = from_kuid_munged(user_ns, c->uid); + kinfo.rgid = from_kgid_munged(user_ns, c->gid); + kinfo.euid = from_kuid_munged(user_ns, c->euid); + kinfo.egid = from_kgid_munged(user_ns, c->egid); + kinfo.suid = from_kuid_munged(user_ns, c->suid); + kinfo.sgid = from_kgid_munged(user_ns, c->sgid); + kinfo.fsuid = from_kuid_munged(user_ns, c->fsuid); + kinfo.fsgid = from_kgid_munged(user_ns, c->fsgid); + kinfo.mask |= PIDFD_INFO_CREDS; + put_cred(c); + +#ifdef CONFIG_CGROUPS + if (!kinfo.cgroupid) { + struct cgroup *cgrp; + + rcu_read_lock(); + cgrp = task_dfl_cgroup(task); + kinfo.cgroupid = cgroup_id(cgrp); + kinfo.mask |= PIDFD_INFO_CGROUPID; + rcu_read_unlock(); + } +#endif + + /* + * Copy pid/tgid last, to reduce the chances the information might be + * stale. Note that it is not possible to ensure it will be valid as the + * task might return as soon as the copy_to_user finishes, but that's ok + * and userspace expects that might happen and can act accordingly, so + * this is just best-effort. What we can do however is checking that all + * the fields are set correctly, or return ESRCH to avoid providing + * incomplete information. */ + + kinfo.ppid = task_ppid_vnr(task); + kinfo.tgid = task_tgid_vnr(task); + kinfo.pid = task_pid_vnr(task); + kinfo.mask |= PIDFD_INFO_PID; + + if (kinfo.pid == 0 || kinfo.tgid == 0) + return -ESRCH; + +copy_out: + if (mask & PIDFD_INFO_SUPPORTED_MASK) { + kinfo.mask |= PIDFD_INFO_SUPPORTED_MASK; + kinfo.supported_mask = PIDFD_INFO_SUPPORTED; + } + + /* Are there bits in the return mask not present in PIDFD_INFO_SUPPORTED? */ + WARN_ON_ONCE(~PIDFD_INFO_SUPPORTED & kinfo.mask); + /* + * If userspace and the kernel have the same struct size it can just + * be copied. If userspace provides an older struct, only the bits that + * userspace knows about will be copied. If userspace provides a new + * struct, only the bits that the kernel knows about will be copied. + */ + return copy_struct_to_user(uinfo, usize, &kinfo, sizeof(kinfo), NULL); +} + +static bool pidfs_ioctl_valid(unsigned int cmd) +{ + switch (cmd) { + case FS_IOC_GETVERSION: + case PIDFD_GET_CGROUP_NAMESPACE: + case PIDFD_GET_IPC_NAMESPACE: + case PIDFD_GET_MNT_NAMESPACE: + case PIDFD_GET_NET_NAMESPACE: + case PIDFD_GET_PID_FOR_CHILDREN_NAMESPACE: + case PIDFD_GET_TIME_NAMESPACE: + case PIDFD_GET_TIME_FOR_CHILDREN_NAMESPACE: + case PIDFD_GET_UTS_NAMESPACE: + case PIDFD_GET_USER_NAMESPACE: + case PIDFD_GET_PID_NAMESPACE: + return true; + } + + /* Extensible ioctls require some more careful checks. */ + switch (_IOC_NR(cmd)) { + case _IOC_NR(PIDFD_GET_INFO): + /* + * Try to prevent performing a pidfd ioctl when someone + * erronously mistook the file descriptor for a pidfd. + * This is not perfect but will catch most cases. + */ + return extensible_ioctl_valid(cmd, PIDFD_GET_INFO, PIDFD_INFO_SIZE_VER0); + } + + return false; +} + +static long pidfd_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + struct task_struct *task __free(put_task) = NULL; + struct nsproxy *nsp __free(put_nsproxy) = NULL; + struct ns_common *ns_common = NULL; + int error; + + if (!pidfs_ioctl_valid(cmd)) + return -ENOIOCTLCMD; + + if (cmd == FS_IOC_GETVERSION) { + if (!arg) + return -EINVAL; + + __u32 __user *argp = (__u32 __user *)arg; + return put_user(file_inode(file)->i_generation, argp); + } + + /* Extensible IOCTL that does not open namespace FDs, take a shortcut */ + if (_IOC_NR(cmd) == _IOC_NR(PIDFD_GET_INFO)) + return pidfd_info(file, cmd, arg); + + task = get_pid_task(pidfd_pid(file), PIDTYPE_PID); + if (!task) + return -ESRCH; + + if (arg) + return -EINVAL; + + /* + * We're trying to open a file descriptor to the namespace so perform a + * filesystem cred ptrace check. Hold @task's exec_update_lock for the + * duration of the ptrace check and the namespace lookup so that the + * credentials used for the access decision match those of @task at the + * time its namespace is read, preventing a concurrent execve() from + * swapping the task's credentials in between the check and the use. We + * mirror nsfs behavior. + */ + error = down_read_killable(&task->signal->exec_update_lock); + if (error) + return error; + + if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) { + error = -EACCES; + goto out_unlock; + } + + scoped_guard(task_lock, task) { + nsp = task->nsproxy; + if (nsp) + get_nsproxy(nsp); + } + if (!nsp) { + error = -ESRCH; /* just pretend it didn't exist */ + goto out_unlock; + } + + switch (cmd) { + /* Namespaces that hang of nsproxy. */ + case PIDFD_GET_CGROUP_NAMESPACE: +#ifdef CONFIG_CGROUPS + if (!ns_ref_get(nsp->cgroup_ns)) + break; + ns_common = to_ns_common(nsp->cgroup_ns); +#endif + break; + case PIDFD_GET_IPC_NAMESPACE: +#ifdef CONFIG_IPC_NS + if (!ns_ref_get(nsp->ipc_ns)) + break; + ns_common = to_ns_common(nsp->ipc_ns); +#endif + break; + case PIDFD_GET_MNT_NAMESPACE: + if (!ns_ref_get(nsp->mnt_ns)) + break; + ns_common = to_ns_common(nsp->mnt_ns); + break; + case PIDFD_GET_NET_NAMESPACE: +#ifdef CONFIG_NET_NS + if (!ns_ref_get(nsp->net_ns)) + break; + ns_common = to_ns_common(nsp->net_ns); +#endif + break; + case PIDFD_GET_PID_FOR_CHILDREN_NAMESPACE: +#ifdef CONFIG_PID_NS + if (!ns_ref_get(nsp->pid_ns_for_children)) + break; + ns_common = to_ns_common(nsp->pid_ns_for_children); +#endif + break; + case PIDFD_GET_TIME_NAMESPACE: +#ifdef CONFIG_TIME_NS + if (!ns_ref_get(nsp->time_ns)) + break; + ns_common = to_ns_common(nsp->time_ns); +#endif + break; + case PIDFD_GET_TIME_FOR_CHILDREN_NAMESPACE: +#ifdef CONFIG_TIME_NS + if (!ns_ref_get(nsp->time_ns_for_children)) + break; + ns_common = to_ns_common(nsp->time_ns_for_children); +#endif + break; + case PIDFD_GET_UTS_NAMESPACE: +#ifdef CONFIG_UTS_NS + if (!ns_ref_get(nsp->uts_ns)) + break; + ns_common = to_ns_common(nsp->uts_ns); +#endif + break; + /* Namespaces that don't hang of nsproxy. */ + case PIDFD_GET_USER_NAMESPACE: +#ifdef CONFIG_USER_NS + scoped_guard(rcu) { + struct user_namespace *user_ns; + + user_ns = task_cred_xxx(task, user_ns); + if (ns_ref_get(user_ns)) + ns_common = to_ns_common(user_ns); + } +#endif + break; + case PIDFD_GET_PID_NAMESPACE: +#ifdef CONFIG_PID_NS + scoped_guard(rcu) { + struct pid_namespace *pid_ns; + + pid_ns = task_active_pid_ns(task); + if (ns_ref_get(pid_ns)) + ns_common = to_ns_common(pid_ns); + } +#endif + break; + default: + error = -ENOIOCTLCMD; + } + + if (!error && !ns_common) + error = -EOPNOTSUPP; + +out_unlock: + up_read(&task->signal->exec_update_lock); + if (error) + return error; + + /* open_namespace() unconditionally consumes the reference */ + return open_namespace(ns_common); +} + +#ifdef CONFIG_COMPAT +static long pidfd_compat_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + if (cmd == FS_IOC32_GETVERSION) + cmd = FS_IOC_GETVERSION; + + return pidfd_ioctl(file, cmd, (unsigned long)compat_ptr(arg)); +} +#endif + +static int pidfs_file_release(struct inode *inode, struct file *file) +{ + struct pid *pid = inode->i_private; + struct task_struct *task; + + if (!(file->f_flags & PIDFD_AUTOKILL)) + return 0; + + guard(rcu)(); + task = pid_task(pid, PIDTYPE_TGID); + if (!task) + return 0; + + /* Not available for kthreads or user workers for now. */ + if (WARN_ON_ONCE(task->flags & (PF_KTHREAD | PF_USER_WORKER))) + return 0; + do_send_sig_info(SIGKILL, SEND_SIG_PRIV, task, PIDTYPE_TGID); + return 0; +} + +static const struct file_operations pidfs_file_operations = { + .release = pidfs_file_release, + .poll = pidfd_poll, +#ifdef CONFIG_PROC_FS + .show_fdinfo = pidfd_show_fdinfo, +#endif + .unlocked_ioctl = pidfd_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = pidfd_compat_ioctl, +#endif +}; + +struct pid *pidfd_pid(const struct file *file) +{ + if (file->f_op != &pidfs_file_operations) + return ERR_PTR(-EBADF); + return file_inode(file)->i_private; +} + +/* + * We're called from release_task(). We know there's at least one + * reference to struct pid being held that won't be released until the + * task has been reaped which cannot happen until we're out of + * release_task(). + * + * If this struct pid has at least once been referred to by a pidfd then + * pid->attr will be allocated. If not we mark the struct pid as dead so + * anyone who is trying to register it with pidfs will fail to do so. + * Otherwise we would hand out pidfs for reaped tasks without having + * exit information available. + * + * Worst case is that we've filled in the info and the pid gets freed + * right away in free_pid() when no one holds a pidfd anymore. Since + * pidfs_exit() currently is placed after exit_task_work() we know that + * it cannot be us aka the exiting task holding a pidfd to itself. + */ +void pidfs_exit(struct task_struct *tsk) +{ + struct pid *pid = task_pid(tsk); + struct pidfs_attr *attr; +#ifdef CONFIG_CGROUPS + struct cgroup *cgrp; +#endif + + might_sleep(); + + /* Synchronize with pidfs_register_pid(). */ + scoped_guard(spinlock_irq, &pid->wait_pidfd.lock) { + attr = pid->attr; + if (!attr) { + /* + * No one ever held a pidfd for this struct pid. + * Mark it as dead so no one can add a pidfs + * entry anymore. We're about to be reaped and + * so no exit information would be available. + */ + pid->attr = PIDFS_PID_DEAD; + return; + } + } + + /* + * If @pid->attr is set someone might still legitimately hold a + * pidfd to @pid or someone might concurrently still be getting + * a reference to an already stashed dentry from @pid->stashed. + * So defer cleaning @pid->attr until the last reference to @pid + * is put + */ + +#ifdef CONFIG_CGROUPS + rcu_read_lock(); + cgrp = task_dfl_cgroup(tsk); + attr->cgroupid = cgroup_id(cgrp); + rcu_read_unlock(); +#endif + attr->exit_code = tsk->exit_code; + + /* Ensure that PIDFD_GET_INFO sees either all or nothing. */ + smp_wmb(); + set_bit(PIDFS_ATTR_BIT_EXIT, &attr->attr_mask); +} + +#ifdef CONFIG_COREDUMP +void pidfs_coredump(const struct coredump_params *cprm) +{ + struct pid *pid = cprm->pid; + struct pidfs_attr *attr; + + attr = READ_ONCE(pid->attr); + + VFS_WARN_ON_ONCE(!attr); + VFS_WARN_ON_ONCE(attr == PIDFS_PID_DEAD); + + /* Note how we were coredumped and that we coredumped. */ + attr->coredump_mask = pidfs_coredump_mask(cprm->dumpable) | + PIDFD_COREDUMPED; + /* If coredumping is set to skip we should never end up here. */ + VFS_WARN_ON_ONCE(attr->coredump_mask & PIDFD_COREDUMP_SKIP); + /* Expose the signal number and code that caused the coredump. */ + attr->coredump_signal = cprm->siginfo->si_signo; + attr->coredump_code = cprm->siginfo->si_code; + smp_wmb(); + set_bit(PIDFS_ATTR_BIT_COREDUMP, &attr->attr_mask); +} +#endif + +static struct vfsmount *pidfs_mnt __ro_after_init; + +/* + * The vfs falls back to simple_setattr() if i_op->setattr() isn't + * implemented. Let's reject it completely until we have a clean + * permission concept for pidfds. + */ +static int pidfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +{ + return anon_inode_setattr(idmap, dentry, attr); +} + +static int pidfs_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, + unsigned int query_flags) +{ + return anon_inode_getattr(idmap, path, stat, request_mask, query_flags); +} + +static ssize_t pidfs_listxattr(struct dentry *dentry, char *buf, size_t size) +{ + struct inode *inode = d_inode(dentry); + struct pid *pid = inode->i_private; + + return simple_xattr_list(inode, &pid->attr->xattrs, buf, size); +} + +static const struct inode_operations pidfs_inode_operations = { + .getattr = pidfs_getattr, + .setattr = pidfs_setattr, + .listxattr = pidfs_listxattr, +}; + +static void pidfs_evict_inode(struct inode *inode) +{ + struct pid *pid = inode->i_private; + + clear_inode(inode); + put_pid(pid); +} + +static const struct super_operations pidfs_sops = { + .drop_inode = inode_just_drop, + .evict_inode = pidfs_evict_inode, + .statfs = simple_statfs, +}; + +/* + * 'lsof' has knowledge of out historical anon_inode use, and expects + * the pidfs dentry name to start with 'anon_inode'. + */ +static char *pidfs_dname(struct dentry *dentry, char *buffer, int buflen) +{ + return dynamic_dname(buffer, buflen, "anon_inode:[pidfd]"); +} + +const struct dentry_operations pidfs_dentry_operations = { + .d_dname = pidfs_dname, + .d_prune = stashed_dentry_prune, +}; + +static int pidfs_encode_fh(struct inode *inode, u32 *fh, int *max_len, + struct inode *parent) +{ + const struct pid *pid = inode->i_private; + + if (*max_len < 2) { + *max_len = 2; + return FILEID_INVALID; + } + + *max_len = 2; + *(u64 *)fh = pid->ino; + return FILEID_KERNFS; +} + +/* Find a struct pid based on the inode number. */ +static struct pid *pidfs_ino_get_pid(u64 ino) +{ + struct pid *pid; + struct pidfs_attr *attr; + + guard(rcu)(); + pid = rhashtable_lookup(&pidfs_ino_ht, &ino, pidfs_ino_ht_params); + if (!pid) + return NULL; + attr = READ_ONCE(pid->attr); + if (IS_ERR_OR_NULL(attr)) + return NULL; + if (test_bit(PIDFS_ATTR_BIT_EXIT, &attr->attr_mask)) + return NULL; + /* Within our pid namespace hierarchy? */ + if (pid_vnr(pid) == 0) + return NULL; + return get_pid(pid); +} + +static struct dentry *pidfs_fh_to_dentry(struct super_block *sb, + struct fid *fid, int fh_len, + int fh_type) +{ + int ret; + u64 pid_ino; + struct path path; + struct pid *pid; + + if (fh_len < 2) + return NULL; + + switch (fh_type) { + case FILEID_KERNFS: + pid_ino = *(u64 *)fid; + break; + default: + return NULL; + } + + pid = pidfs_ino_get_pid(pid_ino); + if (!pid) + return NULL; + + ret = path_from_stashed(&pid->stashed, pidfs_mnt, pid, &path); + if (ret < 0) + return ERR_PTR(ret); + + VFS_WARN_ON_ONCE(!pid->attr); + + mntput(path.mnt); + return path.dentry; +} + +static struct file *pidfs_dentry_open(const struct path *path, + unsigned int flags, + const struct cred *cred) +{ + struct file *file; + + /* pidfds are always O_RDWR. */ + file = dentry_open(path, flags | O_RDWR, cred); + /* do_dentry_open() strips O_EXCL and O_TRUNC. */ + if (!IS_ERR(file)) + file->f_flags |= flags & (PIDFD_THREAD | PIDFD_AUTOKILL); + return file; +} + +/* + * Make sure that we reject any nonsensical flags that users pass via + * open_by_handle_at(). Note that PIDFD_THREAD is defined as O_EXCL, and + * PIDFD_NONBLOCK as O_NONBLOCK. + */ +#define VALID_FILE_HANDLE_OPEN_FLAGS \ + (O_RDONLY | O_WRONLY | O_RDWR | O_NONBLOCK | O_CLOEXEC | O_EXCL) + +static int pidfs_export_permission(struct handle_to_path_ctx *ctx, + unsigned int oflags) +{ + if (oflags & ~(VALID_FILE_HANDLE_OPEN_FLAGS | O_LARGEFILE)) + return -EINVAL; + + /* + * pidfd_ino_get_pid() will verify that the struct pid is part + * of the caller's pid namespace hierarchy. No further + * permission checks are needed. + */ + return 0; +} + +static struct file *pidfs_export_open(const struct path *path, unsigned int oflags) +{ + /* + * Opening via file handle may never raise PIDFD_AUTOKILL. That can + * only be done at task creation! + */ + if (WARN_ON_ONCE(oflags & PIDFD_AUTOKILL)) + return ERR_PTR(-EINVAL); + /* Clear O_LARGEFILE as open_by_handle_at() forces it. */ + return pidfs_dentry_open(path, oflags & ~O_LARGEFILE, current_cred()); +} + +static const struct export_operations pidfs_export_operations = { + .encode_fh = pidfs_encode_fh, + .fh_to_dentry = pidfs_fh_to_dentry, + .open = pidfs_export_open, + .permission = pidfs_export_permission, +}; + +static int pidfs_init_inode(struct inode *inode, void *data) +{ + const struct pid *pid = data; + + inode->i_private = data; + inode->i_flags |= S_PRIVATE | S_ANON_INODE; + /* We allow to set xattrs. */ + inode->i_flags &= ~S_IMMUTABLE; + inode->i_mode |= S_IRWXU; + inode->i_op = &pidfs_inode_operations; + inode->i_fop = &pidfs_file_operations; + inode->i_ino = pidfs_ino(pid->ino); + inode->i_generation = pidfs_gen(pid->ino); + return 0; +} + +static void pidfs_put_data(void *data) +{ + struct pid *pid = data; + put_pid(pid); +} + +/** + * pidfs_register_pid_gfp - register a struct pid in pidfs with custom GFP + * flags + * @pid: pid to pin + * @gfp: GFP flags for memory allocation + * + * Register a struct pid in pidfs with custom GFP flags. + * + * Return: On success zero, on error a negative error code is returned. + */ +int pidfs_register_pid_gfp(struct pid *pid, gfp_t gfp) +{ + struct pidfs_attr *new_attr __free(kfree) = NULL; + struct pidfs_attr *attr; + + might_sleep(); + + if (!pid) + return 0; + + attr = READ_ONCE(pid->attr); + if (unlikely(attr == PIDFS_PID_DEAD)) + return PTR_ERR(PIDFS_PID_DEAD); + if (attr) + return 0; + + new_attr = kmem_cache_zalloc(pidfs_attr_cachep, gfp); + if (!new_attr) + return -ENOMEM; + + INIT_LIST_HEAD_RCU(&new_attr->xattrs); + + /* Synchronize with pidfs_exit(). */ + guard(spinlock_irq)(&pid->wait_pidfd.lock); + + attr = pid->attr; + if (unlikely(attr == PIDFS_PID_DEAD)) + return PTR_ERR(PIDFS_PID_DEAD); + if (unlikely(attr)) + return 0; + + pid->attr = no_free_ptr(new_attr); + return 0; +} + +static struct dentry *pidfs_stash_dentry(struct dentry **stashed, + struct dentry *dentry) +{ + int ret; + struct pid *pid = d_inode(dentry)->i_private; + + VFS_WARN_ON_ONCE(stashed != &pid->stashed); + + ret = pidfs_register_pid(pid); + if (ret) + return ERR_PTR(ret); + + return stash_dentry(stashed, dentry); +} + +static const struct stashed_operations pidfs_stashed_ops = { + .stash_dentry = pidfs_stash_dentry, + .init_inode = pidfs_init_inode, + .put_data = pidfs_put_data, +}; + +static int pidfs_xattr_get(const struct xattr_handler *handler, + struct dentry *unused, struct inode *inode, + const char *suffix, void *value, size_t size) +{ + struct pid *pid = inode->i_private; + const char *name = xattr_full_name(handler, suffix); + + return simple_xattr_get(&pidfs_xa_cache, &pid->attr->xattrs, name, value, size); +} + +static int pidfs_xattr_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, struct dentry *unused, + struct inode *inode, const char *suffix, + const void *value, size_t size, int flags) +{ + struct pid *pid = inode->i_private; + const char *name = xattr_full_name(handler, suffix); + struct simple_xattr *old_xattr; + + /* Ensure we're the only one to set @attr->xattrs. */ + WARN_ON_ONCE(!inode_is_locked(inode)); + + old_xattr = simple_xattr_set(&pidfs_xa_cache, &pid->attr->xattrs, name, value, size, flags); + if (IS_ERR(old_xattr)) + return PTR_ERR(old_xattr); + + simple_xattr_free_rcu(old_xattr); + return 0; +} + +static const struct xattr_handler pidfs_trusted_xattr_handler = { + .prefix = XATTR_TRUSTED_PREFIX, + .get = pidfs_xattr_get, + .set = pidfs_xattr_set, +}; + +static const struct xattr_handler *const pidfs_xattr_handlers[] = { + &pidfs_trusted_xattr_handler, + NULL +}; + +static int pidfs_init_fs_context(struct fs_context *fc) +{ + struct pseudo_fs_context *ctx; + + ctx = init_pseudo(fc, PID_FS_MAGIC); + if (!ctx) + return -ENOMEM; + + ctx->s_d_flags |= DCACHE_DONTCACHE; + ctx->ops = &pidfs_sops; + ctx->eops = &pidfs_export_operations; + ctx->dops = &pidfs_dentry_operations; + ctx->xattr = pidfs_xattr_handlers; + fc->s_fs_info = (void *)&pidfs_stashed_ops; + return 0; +} + +static struct file_system_type pidfs_type = { + .name = "pidfs", + .init_fs_context = pidfs_init_fs_context, + .kill_sb = kill_anon_super, +}; + +struct file *pidfs_alloc_file(struct pid *pid, unsigned int flags) +{ + struct path path __free(path_put) = {}; + int ret; + + /* + * Ensure that internal pidfd flags don't overlap with each + * other or with uapi pidfd flags. + */ + BUILD_BUG_ON(hweight32(PIDFD_THREAD | PIDFD_NONBLOCK | + PIDFD_STALE | PIDFD_AUTOKILL) != 4); + + ret = path_from_stashed(&pid->stashed, pidfs_mnt, get_pid(pid), &path); + if (ret < 0) + return ERR_PTR(ret); + + VFS_WARN_ON_ONCE(!pid->attr); + + flags &= ~PIDFD_STALE; + return pidfs_dentry_open(&path, flags, current_cred()); +} + +void __init pidfs_init(void) +{ + if (rhashtable_init(&pidfs_ino_ht, &pidfs_ino_ht_params)) + panic("Failed to initialize pidfs hashtable"); + + pidfs_attr_cachep = kmem_cache_create("pidfs_attr_cache", sizeof(struct pidfs_attr), 0, + (SLAB_HWCACHE_ALIGN | SLAB_RECLAIM_ACCOUNT | + SLAB_ACCOUNT | SLAB_PANIC), NULL); + + pidfs_mnt = kern_mount(&pidfs_type); + if (IS_ERR(pidfs_mnt)) + panic("Failed to mount pidfs pseudo filesystem"); + + pidfs_root_path.mnt = pidfs_mnt; + pidfs_root_path.dentry = pidfs_mnt->mnt_root; +} diff --git a/fs/pipe.c b/fs/pipe.c new file mode 100644 index 000000000..292425a83 --- /dev/null +++ b/fs/pipe.c @@ -0,0 +1,1633 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/pipe.c + * + * Copyright (C) 1991, 1992, 1999 Linus Torvalds + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "internal.h" + +/* + * New pipe buffers will be restricted to this size while the user is exceeding + * their pipe buffer quota. The general pipe use case needs at least two + * buffers: one for data yet to be read, and one for new data. If this is less + * than two, then a write to a non-empty pipe may block even if the pipe is not + * full. This can occur with GNU make jobserver or similar uses of pipes as + * semaphores: multiple processes may be waiting to write tokens back to the + * pipe before reading tokens: https://lore.kernel.org/lkml/1628086770.5rn8p04n6j.none@localhost/. + * + * Users can reduce their pipe buffers with F_SETPIPE_SZ below this at their + * own risk, namely: pipe writes to non-full pipes may block until the pipe is + * emptied. + */ +#define PIPE_MIN_DEF_BUFFERS 2 + +/* + * The max size that a non-root user is allowed to grow the pipe. Can + * be set by root in /proc/sys/fs/pipe-max-size + */ +static unsigned int pipe_max_size = 1048576; + +/* Maximum allocatable pages per user. Hard limit is unset by default, soft + * matches default values. + */ +static unsigned long pipe_user_pages_hard; +static unsigned long pipe_user_pages_soft = PIPE_DEF_BUFFERS * INR_OPEN_CUR; + +/* + * We use head and tail indices that aren't masked off, except at the point of + * dereference, but rather they're allowed to wrap naturally. This means there + * isn't a dead spot in the buffer, but the ring has to be a power of two and + * <= 2^31. + * -- David Howells 2019-09-23. + * + * Reads with count = 0 should always return 0. + * -- Julian Bradfield 1999-06-07. + * + * FIFOs and Pipes now generate SIGIO for both readers and writers. + * -- Jeremy Elson 2001-08-16 + * + * pipe_read & write cleanup + * -- Manfred Spraul 2002-05-09 + */ + +#ifdef CONFIG_PROVE_LOCKING +static int pipe_lock_cmp_fn(const struct lockdep_map *a, + const struct lockdep_map *b) +{ + return cmp_int((unsigned long) a, (unsigned long) b); +} +#endif + +void pipe_lock(struct pipe_inode_info *pipe) +{ + if (pipe->files) + mutex_lock(&pipe->mutex); +} +EXPORT_SYMBOL(pipe_lock); + +void pipe_unlock(struct pipe_inode_info *pipe) +{ + if (pipe->files) + mutex_unlock(&pipe->mutex); +} +EXPORT_SYMBOL(pipe_unlock); + +void pipe_double_lock(struct pipe_inode_info *pipe1, + struct pipe_inode_info *pipe2) +{ + BUG_ON(pipe1 == pipe2); + + if (pipe1 > pipe2) + swap(pipe1, pipe2); + + pipe_lock(pipe1); + pipe_lock(pipe2); +} + +static struct page *anon_pipe_prealloc_pop(struct anon_pipe_prealloc *prealloc) +{ + if (!prealloc->count) + return NULL; + + prealloc->count--; + + return prealloc->pages[prealloc->count]; +} + +/* Push a page to the prealloc pool. Returns true if added, false if full. */ +static bool anon_pipe_prealloc_push(struct anon_pipe_prealloc *prealloc, + struct page *page) +{ + if (prealloc->count >= PIPE_PREALLOC_MAX) + return false; + prealloc->pages[prealloc->count++] = page; + return true; +} + +/* + * Top up the pipe's own pool, then take pipe->mutex and return with it held. + * The shortfall is allocated outside the lock; the push and the caller's write + * then run under a single lock acquisition, avoiding a separate prefill + * lock/unlock cycle. anon_pipe_get_page() drains the pool instead of allocating + * under the lock. + */ +static void anon_pipe_prefill_and_lock(struct pipe_inode_info *pipe, size_t total_len) +{ + struct page *pages[PIPE_PREALLOC_MAX]; + unsigned int want, have, need, n = 0; + + want = min_t(unsigned int, DIV_ROUND_UP(total_len, PAGE_SIZE), + PIPE_PREALLOC_MAX); + /* Unlocked read; the pool is refilled under the lock below. */ + have = min_t(unsigned int, READ_ONCE(pipe->prealloc.count), want); + need = want - have; + + if (!need) { + mutex_lock(&pipe->mutex); + return; + } + + while (n < need) { + struct page *page = alloc_page(GFP_HIGHUSER | __GFP_ACCOUNT); + + if (!page) + break; + pages[n++] = page; + } + + mutex_lock(&pipe->mutex); + while (n && anon_pipe_prealloc_push(&pipe->prealloc, pages[n - 1])) + n--; + + /* + * Just flush any extra page that got affected by the TOCTOU + * effect + */ + while (n) + put_page(pages[--n]); +} + +/* + * Called with pipe->mutex held. Trim the pool down to PIPE_PREALLOC_KEEP under + * the lock, drop it, then free the excess outside the critical section. + */ +static void anon_pipe_trim_and_unlock(struct pipe_inode_info *pipe) +{ + struct page *excess[PIPE_PREALLOC_MAX]; + unsigned int nexcess = 0; + + while (pipe->prealloc.count > PIPE_PREALLOC_KEEP) + excess[nexcess++] = anon_pipe_prealloc_pop(&pipe->prealloc); + mutex_unlock(&pipe->mutex); + + while (nexcess) + put_page(excess[--nexcess]); +} + +static struct page *anon_pipe_get_page(struct pipe_inode_info *pipe) +{ + struct page *page; + + /* Drain the prealloc pool before allocating. Called with mutex held. */ + page = anon_pipe_prealloc_pop(&pipe->prealloc); + if (page) + return page; + + /* FWIW: This is called with pipe->mutex held */ + return alloc_page(GFP_HIGHUSER | __GFP_ACCOUNT); +} + +static void anon_pipe_put_page(struct pipe_inode_info *pipe, + struct page *page) +{ + if (page_count(page) == 1 && + anon_pipe_prealloc_push(&pipe->prealloc, page)) + return; + + put_page(page); +} + +static void anon_pipe_buf_release(struct pipe_inode_info *pipe, + struct pipe_buffer *buf) +{ + struct page *page = buf->page; + + anon_pipe_put_page(pipe, page); +} + +static bool anon_pipe_buf_try_steal(struct pipe_inode_info *pipe, + struct pipe_buffer *buf) +{ + struct page *page = buf->page; + + if (page_count(page) != 1) + return false; + memcg_kmem_uncharge_page(page, 0); + __SetPageLocked(page); + return true; +} + +/** + * generic_pipe_buf_try_steal - attempt to take ownership of a &pipe_buffer + * @pipe: the pipe that the buffer belongs to + * @buf: the buffer to attempt to steal + * + * Description: + * This function attempts to steal the &struct page attached to + * @buf. If successful, this function returns 0 and returns with + * the page locked. The caller may then reuse the page for whatever + * he wishes; the typical use is insertion into a different file + * page cache. + */ +bool generic_pipe_buf_try_steal(struct pipe_inode_info *pipe, + struct pipe_buffer *buf) +{ + struct page *page = buf->page; + + /* + * A reference of one is golden, that means that the owner of this + * page is the only one holding a reference to it. lock the page + * and return OK. + */ + if (page_count(page) == 1) { + lock_page(page); + return true; + } + return false; +} +EXPORT_SYMBOL(generic_pipe_buf_try_steal); + +/** + * generic_pipe_buf_get - get a reference to a &struct pipe_buffer + * @pipe: the pipe that the buffer belongs to + * @buf: the buffer to get a reference to + * + * Description: + * This function grabs an extra reference to @buf. It's used in + * the tee() system call, when we duplicate the buffers in one + * pipe into another. + */ +bool generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf) +{ + return try_get_page(buf->page); +} +EXPORT_SYMBOL(generic_pipe_buf_get); + +/** + * generic_pipe_buf_release - put a reference to a &struct pipe_buffer + * @pipe: the pipe that the buffer belongs to + * @buf: the buffer to put a reference to + * + * Description: + * This function releases a reference to @buf. + */ +void generic_pipe_buf_release(struct pipe_inode_info *pipe, + struct pipe_buffer *buf) +{ + put_page(buf->page); +} +EXPORT_SYMBOL(generic_pipe_buf_release); + +static const struct pipe_buf_operations anon_pipe_buf_ops = { + .release = anon_pipe_buf_release, + .try_steal = anon_pipe_buf_try_steal, + .get = generic_pipe_buf_get, +}; + +/* Done while waiting without holding the pipe lock - thus the READ_ONCE() */ +static inline bool pipe_readable(const struct pipe_inode_info *pipe) +{ + union pipe_index idx = { .head_tail = READ_ONCE(pipe->head_tail) }; + unsigned int writers = READ_ONCE(pipe->writers); + + return !pipe_empty(idx.head, idx.tail) || !writers; +} + +static inline unsigned int pipe_update_tail(struct pipe_inode_info *pipe, + struct pipe_buffer *buf, + unsigned int tail) +{ + pipe_buf_release(pipe, buf); + + /* + * If the pipe has a watch_queue, we need additional protection + * by the spinlock because notifications get posted with only + * this spinlock, no mutex + */ + if (pipe_has_watch_queue(pipe)) { + spin_lock_irq(&pipe->rd_wait.lock); +#ifdef CONFIG_WATCH_QUEUE + if (buf->flags & PIPE_BUF_FLAG_LOSS) + pipe->note_loss = true; +#endif + pipe->tail = ++tail; + spin_unlock_irq(&pipe->rd_wait.lock); + return tail; + } + + /* + * Without a watch_queue, we can simply increment the tail + * without the spinlock - the mutex is enough. + */ + pipe->tail = ++tail; + return tail; +} + +static ssize_t +anon_pipe_read(struct kiocb *iocb, struct iov_iter *to) +{ + size_t total_len = iov_iter_count(to); + struct file *filp = iocb->ki_filp; + struct pipe_inode_info *pipe = filp->private_data; + bool wake_writer = false, wake_next_reader = false; + ssize_t ret; + + /* Null read succeeds. */ + if (unlikely(total_len == 0)) + return 0; + + ret = 0; + mutex_lock(&pipe->mutex); + + /* + * We only wake up writers if the pipe was full when we started reading + * and it is no longer full after reading to avoid unnecessary wakeups. + * + * But when we do wake up writers, we do so using a sync wakeup + * (WF_SYNC), because we want them to get going and generate more + * data for us. + */ + for (;;) { + /* Read ->head with a barrier vs post_one_notification() */ + unsigned int head = smp_load_acquire(&pipe->head); + unsigned int tail = pipe->tail; + +#ifdef CONFIG_WATCH_QUEUE + if (pipe->note_loss) { + struct watch_notification n; + + if (total_len < 8) { + if (ret == 0) + ret = -ENOBUFS; + break; + } + + n.type = WATCH_TYPE_META; + n.subtype = WATCH_META_LOSS_NOTIFICATION; + n.info = watch_sizeof(n); + if (copy_to_iter(&n, sizeof(n), to) != sizeof(n)) { + if (ret == 0) + ret = -EFAULT; + break; + } + ret += sizeof(n); + total_len -= sizeof(n); + pipe->note_loss = false; + } +#endif + + if (!pipe_empty(head, tail)) { + struct pipe_buffer *buf = pipe_buf(pipe, tail); + size_t chars = buf->len; + size_t written; + int error; + + if (chars > total_len) { + if (buf->flags & PIPE_BUF_FLAG_WHOLE) { + if (ret == 0) + ret = -ENOBUFS; + break; + } + chars = total_len; + } + + error = pipe_buf_confirm(pipe, buf); + if (error) { + if (!ret) + ret = error; + break; + } + + written = copy_page_to_iter(buf->page, buf->offset, chars, to); + if (unlikely(written < chars)) { + if (!ret) + ret = -EFAULT; + break; + } + ret += chars; + buf->offset += chars; + buf->len -= chars; + + /* Was it a packet buffer? Clean up and exit */ + if (buf->flags & PIPE_BUF_FLAG_PACKET) { + total_len = chars; + buf->len = 0; + } + + if (!buf->len) { + wake_writer |= pipe_full(head, tail, pipe->max_usage); + tail = pipe_update_tail(pipe, buf, tail); + } + total_len -= chars; + if (!total_len) + break; /* common path: read succeeded */ + if (!pipe_empty(head, tail)) /* More to do? */ + continue; + } + + if (!pipe->writers) + break; + if (ret) + break; + if ((filp->f_flags & O_NONBLOCK) || + (iocb->ki_flags & IOCB_NOWAIT)) { + ret = -EAGAIN; + break; + } + mutex_unlock(&pipe->mutex); + /* + * We only get here if we didn't actually read anything. + * + * But because we didn't read anything, at this point we can + * just return directly with -ERESTARTSYS if we're interrupted, + * since we've done any required wakeups and there's no need + * to mark anything accessed. And we've dropped the lock. + */ + if (wait_event_interruptible_exclusive(pipe->rd_wait, pipe_readable(pipe)) < 0) + return -ERESTARTSYS; + + wake_next_reader = true; + mutex_lock(&pipe->mutex); + } + if (pipe_is_empty(pipe)) + wake_next_reader = false; + /* Consumed buffers may have refilled the pool; trim it and unlock. */ + anon_pipe_trim_and_unlock(pipe); + + if (wake_writer) + wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM); + if (wake_next_reader) + wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM); + kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); + return ret; +} + +static ssize_t +fifo_pipe_read(struct kiocb *iocb, struct iov_iter *to) +{ + int ret = anon_pipe_read(iocb, to); + if (ret > 0) + file_accessed(iocb->ki_filp); + return ret; +} + +static inline int is_packetized(struct file *file) +{ + return (file->f_flags & O_DIRECT) != 0; +} + +/* Done while waiting without holding the pipe lock - thus the READ_ONCE() */ +static inline bool pipe_writable(const struct pipe_inode_info *pipe) +{ + union pipe_index idx = { .head_tail = READ_ONCE(pipe->head_tail) }; + unsigned int max_usage = READ_ONCE(pipe->max_usage); + + return !pipe_full(idx.head, idx.tail, max_usage) || + !READ_ONCE(pipe->readers); +} + +static ssize_t +anon_pipe_write(struct kiocb *iocb, struct iov_iter *from) +{ + struct file *filp = iocb->ki_filp; + struct pipe_inode_info *pipe = filp->private_data; + unsigned int head; + ssize_t ret = 0; + size_t total_len = iov_iter_count(from); + ssize_t chars; + bool was_empty = false; + bool wake_next_writer = false; + + /* + * Reject writing to watch queue pipes before the point where we lock + * the pipe. + * Otherwise, lockdep would be unhappy if the caller already has another + * pipe locked. + * If we had to support locking a normal pipe and a notification pipe at + * the same time, we could set up lockdep annotations for that, but + * since we don't actually need that, it's simpler to just bail here. + */ + if (pipe_has_watch_queue(pipe)) + return -EXDEV; + + /* Null write succeeds. */ + if (unlikely(total_len == 0)) + return 0; + + anon_pipe_prefill_and_lock(pipe, total_len); + + if (!pipe->readers) { + if ((iocb->ki_flags & IOCB_NOSIGNAL) == 0) + send_sig(SIGPIPE, current, 0); + ret = -EPIPE; + goto out; + } + + /* + * If it wasn't empty we try to merge new data into + * the last buffer. + * + * That naturally merges small writes, but it also + * page-aligns the rest of the writes for large writes + * spanning multiple pages. + */ + head = pipe->head; + was_empty = pipe_empty(head, pipe->tail); + chars = total_len & (PAGE_SIZE-1); + if (chars && !was_empty) { + struct pipe_buffer *buf = pipe_buf(pipe, head - 1); + int offset = buf->offset + buf->len; + + if ((buf->flags & PIPE_BUF_FLAG_CAN_MERGE) && + offset + chars <= PAGE_SIZE) { + ret = pipe_buf_confirm(pipe, buf); + if (ret) + goto out; + + ret = copy_page_from_iter(buf->page, offset, chars, from); + if (unlikely(ret < chars)) { + ret = -EFAULT; + goto out; + } + + buf->len += ret; + if (!iov_iter_count(from)) + goto out; + } + } + + for (;;) { + if (!pipe->readers) { + if ((iocb->ki_flags & IOCB_NOSIGNAL) == 0) + send_sig(SIGPIPE, current, 0); + if (!ret) + ret = -EPIPE; + break; + } + + head = pipe->head; + if (!pipe_full(head, pipe->tail, pipe->max_usage)) { + struct pipe_buffer *buf; + struct page *page; + int copied; + + page = anon_pipe_get_page(pipe); + if (unlikely(!page)) { + if (!ret) + ret = -ENOMEM; + break; + } + + copied = copy_page_from_iter(page, 0, PAGE_SIZE, from); + if (unlikely(copied < PAGE_SIZE && iov_iter_count(from))) { + anon_pipe_put_page(pipe, page); + if (!ret) + ret = -EFAULT; + break; + } + + pipe->head = head + 1; + /* Insert it into the buffer array */ + buf = pipe_buf(pipe, head); + buf->page = page; + buf->ops = &anon_pipe_buf_ops; + buf->offset = 0; + if (is_packetized(filp)) + buf->flags = PIPE_BUF_FLAG_PACKET; + else + buf->flags = PIPE_BUF_FLAG_CAN_MERGE; + + buf->len = copied; + ret += copied; + + if (!iov_iter_count(from)) + break; + + continue; + } + + /* Wait for buffer space to become available. */ + if ((filp->f_flags & O_NONBLOCK) || + (iocb->ki_flags & IOCB_NOWAIT)) { + if (!ret) + ret = -EAGAIN; + break; + } + if (signal_pending(current)) { + if (!ret) + ret = -ERESTARTSYS; + break; + } + + /* + * We're going to release the pipe lock and wait for more + * space. We wake up any readers if necessary, and then + * after waiting we need to re-check whether the pipe + * become empty while we dropped the lock. + */ + mutex_unlock(&pipe->mutex); + if (was_empty) + wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM); + kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); + wait_event_interruptible_exclusive(pipe->wr_wait, pipe_writable(pipe)); + mutex_lock(&pipe->mutex); + was_empty = pipe_is_empty(pipe); + wake_next_writer = true; + } +out: + if (pipe_is_full(pipe)) + wake_next_writer = false; + anon_pipe_trim_and_unlock(pipe); + + /* + * If we do do a wakeup event, we do a 'sync' wakeup, because we + * want the reader to start processing things asap, rather than + * leave the data pending. + * + * This is particularly important for small writes, because of + * how (for example) the GNU make jobserver uses small writes to + * wake up pending jobs + * + * ->pseudo_edgetrigger enables per-write wakeups, see pipe_poll() + */ + if (was_empty || READ_ONCE(pipe->pseudo_edgetrigger)) + wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM); + kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); + if (wake_next_writer) + wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM); + return ret; +} + +static ssize_t +fifo_pipe_write(struct kiocb *iocb, struct iov_iter *from) +{ + int ret = anon_pipe_write(iocb, from); + if (ret > 0) { + struct file *filp = iocb->ki_filp; + if (sb_start_write_trylock(file_inode(filp)->i_sb)) { + int err = file_update_time(filp); + if (err) + ret = err; + sb_end_write(file_inode(filp)->i_sb); + } + } + return ret; +} + +static long pipe_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct pipe_inode_info *pipe = filp->private_data; + unsigned int count, head, tail; + + switch (cmd) { + case FIONREAD: + mutex_lock(&pipe->mutex); + count = 0; + head = pipe->head; + tail = pipe->tail; + + while (!pipe_empty(head, tail)) { + count += pipe_buf(pipe, tail)->len; + tail++; + } + mutex_unlock(&pipe->mutex); + + return put_user(count, (int __user *)arg); + +#ifdef CONFIG_WATCH_QUEUE + case IOC_WATCH_QUEUE_SET_SIZE: { + int ret; + mutex_lock(&pipe->mutex); + ret = watch_queue_set_size(pipe, arg); + mutex_unlock(&pipe->mutex); + return ret; + } + + case IOC_WATCH_QUEUE_SET_FILTER: + return watch_queue_set_filter( + pipe, (struct watch_notification_filter __user *)arg); +#endif + + default: + return -ENOIOCTLCMD; + } +} + +static __poll_t +pipe_poll(struct file *filp, poll_table *wait) +{ + __poll_t mask; + struct pipe_inode_info *pipe = filp->private_data; + union pipe_index idx; + + /* + * Legacy epoll(EPOLLET) users depend on historical per-write wakeups, + * see 3a34b13a88ca ("pipe: make pipe writes always wake up readers") + * and the ->pseudo_edgetrigger check in anon_pipe_write(). + * Currently io_uring sets EPOLLET for multishot polls, so it gets the + * same behaviour. + */ + if ((filp->f_mode & FMODE_READ) && + wait && (wait->_key & EPOLLET) && + unlikely(!READ_ONCE(pipe->pseudo_edgetrigger))) + WRITE_ONCE(pipe->pseudo_edgetrigger, true); + + /* + * Reading pipe state only -- no need for acquiring the semaphore. + * + * But because this is racy, the code has to add the + * entry to the poll table _first_ .. + */ + if (filp->f_mode & FMODE_READ) + poll_wait(filp, &pipe->rd_wait, wait); + if (filp->f_mode & FMODE_WRITE) + poll_wait(filp, &pipe->wr_wait, wait); + + /* + * .. and only then can you do the racy tests. That way, + * if something changes and you got it wrong, the poll + * table entry will wake you up and fix it. + */ + idx.head_tail = READ_ONCE(pipe->head_tail); + + mask = 0; + if (filp->f_mode & FMODE_READ) { + if (!pipe_empty(idx.head, idx.tail)) + mask |= EPOLLIN | EPOLLRDNORM; + if (!pipe->writers && filp->f_pipe != pipe->w_counter) + mask |= EPOLLHUP; + } + + if (filp->f_mode & FMODE_WRITE) { + if (!pipe_full(idx.head, idx.tail, pipe->max_usage)) + mask |= EPOLLOUT | EPOLLWRNORM; + /* + * Most Unices do not set EPOLLERR for FIFOs but on Linux they + * behave exactly like pipes for poll(). + */ + if (!pipe->readers) + mask |= EPOLLERR; + } + + return mask; +} + +static void put_pipe_info(struct inode *inode, struct pipe_inode_info *pipe) +{ + int kill = 0; + + spin_lock(&inode->i_lock); + if (!--pipe->files) { + inode->i_pipe = NULL; + kill = 1; + } + spin_unlock(&inode->i_lock); + + if (kill) + free_pipe_info(pipe); +} + +static int +pipe_release(struct inode *inode, struct file *file) +{ + struct pipe_inode_info *pipe = file->private_data; + + mutex_lock(&pipe->mutex); + if (file->f_mode & FMODE_READ) + pipe->readers--; + if (file->f_mode & FMODE_WRITE) + pipe->writers--; + + /* Was that the last reader or writer, but not the other side? */ + if (!pipe->readers != !pipe->writers) { + wake_up_interruptible_all(&pipe->rd_wait); + wake_up_interruptible_all(&pipe->wr_wait); + kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); + kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); + } + mutex_unlock(&pipe->mutex); + + put_pipe_info(inode, pipe); + return 0; +} + +static int +pipe_fasync(int fd, struct file *filp, int on) +{ + struct pipe_inode_info *pipe = filp->private_data; + int retval = 0; + + mutex_lock(&pipe->mutex); + if (filp->f_mode & FMODE_READ) + retval = fasync_helper(fd, filp, on, &pipe->fasync_readers); + if ((filp->f_mode & FMODE_WRITE) && retval >= 0) { + retval = fasync_helper(fd, filp, on, &pipe->fasync_writers); + if (retval < 0 && (filp->f_mode & FMODE_READ)) + /* this can happen only if on == T */ + fasync_helper(-1, filp, 0, &pipe->fasync_readers); + } + mutex_unlock(&pipe->mutex); + return retval; +} + +unsigned long account_pipe_buffers(struct user_struct *user, + unsigned long old, unsigned long new) +{ + return atomic_long_add_return(new - old, &user->pipe_bufs); +} + +bool too_many_pipe_buffers_soft(unsigned long user_bufs) +{ + unsigned long soft_limit = READ_ONCE(pipe_user_pages_soft); + + return soft_limit && user_bufs > soft_limit; +} + +bool too_many_pipe_buffers_hard(unsigned long user_bufs) +{ + unsigned long hard_limit = READ_ONCE(pipe_user_pages_hard); + + return hard_limit && user_bufs > hard_limit; +} + +bool pipe_is_unprivileged_user(void) +{ + return !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN); +} + +struct pipe_inode_info *alloc_pipe_info(void) +{ + struct pipe_inode_info *pipe; + unsigned long pipe_bufs = PIPE_DEF_BUFFERS; + struct user_struct *user = get_current_user(); + unsigned long user_bufs; + unsigned int max_size = READ_ONCE(pipe_max_size); + + pipe = kzalloc_obj(struct pipe_inode_info, GFP_KERNEL_ACCOUNT); + if (pipe == NULL) + goto out_free_uid; + + if (pipe_bufs * PAGE_SIZE > max_size && !capable(CAP_SYS_RESOURCE)) + pipe_bufs = max_size >> PAGE_SHIFT; + + user_bufs = account_pipe_buffers(user, 0, pipe_bufs); + + if (too_many_pipe_buffers_soft(user_bufs) && pipe_is_unprivileged_user()) { + user_bufs = account_pipe_buffers(user, pipe_bufs, PIPE_MIN_DEF_BUFFERS); + pipe_bufs = PIPE_MIN_DEF_BUFFERS; + } + + if (too_many_pipe_buffers_hard(user_bufs) && pipe_is_unprivileged_user()) + goto out_revert_acct; + + pipe->bufs = kzalloc_objs(struct pipe_buffer, pipe_bufs, + GFP_KERNEL_ACCOUNT); + + if (pipe->bufs) { + init_waitqueue_head(&pipe->rd_wait); + init_waitqueue_head(&pipe->wr_wait); + pipe->r_counter = pipe->w_counter = 1; + pipe->max_usage = pipe_bufs; + pipe->ring_size = pipe_bufs; + pipe->nr_accounted = pipe_bufs; + pipe->user = user; + mutex_init(&pipe->mutex); + lock_set_cmp_fn(&pipe->mutex, pipe_lock_cmp_fn, NULL); + return pipe; + } + +out_revert_acct: + (void) account_pipe_buffers(user, pipe_bufs, 0); + kfree(pipe); +out_free_uid: + free_uid(user); + return NULL; +} + +void free_pipe_info(struct pipe_inode_info *pipe) +{ + unsigned int i; + +#ifdef CONFIG_WATCH_QUEUE + if (pipe->watch_queue) + watch_queue_clear(pipe->watch_queue); +#endif + + (void) account_pipe_buffers(pipe->user, pipe->nr_accounted, 0); + free_uid(pipe->user); + for (i = 0; i < pipe->ring_size; i++) { + struct pipe_buffer *buf = pipe->bufs + i; + if (buf->ops) + pipe_buf_release(pipe, buf); + } +#ifdef CONFIG_WATCH_QUEUE + if (pipe->watch_queue) + put_watch_queue(pipe->watch_queue); +#endif + for (i = 0; i < pipe->prealloc.count; i++) + __free_page(pipe->prealloc.pages[i]); + kfree(pipe->bufs); + kfree(pipe); +} + +static struct vfsmount *pipe_mnt __ro_after_init; + +/* + * pipefs_dname() is called from d_path(). + */ +static char *pipefs_dname(struct dentry *dentry, char *buffer, int buflen) +{ + return dynamic_dname(buffer, buflen, "pipe:[%llu]", + d_inode(dentry)->i_ino); +} + +static const struct dentry_operations pipefs_dentry_operations = { + .d_dname = pipefs_dname, +}; + +static const struct file_operations pipeanon_fops; + +static struct inode * get_pipe_inode(void) +{ + struct inode *inode = new_inode_pseudo(pipe_mnt->mnt_sb); + struct pipe_inode_info *pipe; + + if (!inode) + goto fail_inode; + + inode->i_ino = get_next_ino(); + + pipe = alloc_pipe_info(); + if (!pipe) + goto fail_iput; + + inode->i_pipe = pipe; + pipe->files = 2; + pipe->readers = pipe->writers = 1; + inode->i_fop = &pipeanon_fops; + + /* + * Mark the inode dirty from the very beginning, + * that way it will never be moved to the dirty + * list because "mark_inode_dirty()" will think + * that it already _is_ on the dirty list. + */ + inode_state_assign_raw(inode, I_DIRTY); + inode->i_mode = S_IFIFO | S_IRUSR | S_IWUSR; + inode->i_uid = current_fsuid(); + inode->i_gid = current_fsgid(); + simple_inode_init_ts(inode); + + return inode; + +fail_iput: + iput(inode); + +fail_inode: + return NULL; +} + +int create_pipe_files(struct file **res, int flags) +{ + struct inode *inode = get_pipe_inode(); + struct file *f; + int error; + + if (!inode) + return -ENFILE; + + if (flags & O_NOTIFICATION_PIPE) { + error = watch_queue_init(inode->i_pipe); + if (error) { + free_pipe_info(inode->i_pipe); + iput(inode); + return error; + } + } + + f = alloc_file_pseudo(inode, pipe_mnt, "", + O_WRONLY | (flags & (O_NONBLOCK | O_DIRECT)), + &pipeanon_fops); + if (IS_ERR(f)) { + free_pipe_info(inode->i_pipe); + iput(inode); + return PTR_ERR(f); + } + + f->private_data = inode->i_pipe; + f->f_pipe = 0; + + res[0] = alloc_file_clone(f, O_RDONLY | (flags & O_NONBLOCK), + &pipeanon_fops); + if (IS_ERR(res[0])) { + put_pipe_info(inode, inode->i_pipe); + fput(f); + return PTR_ERR(res[0]); + } + res[0]->private_data = inode->i_pipe; + res[0]->f_pipe = 0; + res[1] = f; + stream_open(inode, res[0]); + stream_open(inode, res[1]); + + /* pipe groks IOCB_NOWAIT */ + res[0]->f_mode |= FMODE_NOWAIT; + res[1]->f_mode |= FMODE_NOWAIT; + + /* + * Disable permission and pre-content events, but enable legacy + * inotify events for legacy users. + */ + file_set_fsnotify_mode(res[0], FMODE_NONOTIFY_PERM); + file_set_fsnotify_mode(res[1], FMODE_NONOTIFY_PERM); + return 0; +} + +static int __do_pipe_flags(int *fd, struct file **files, int flags) +{ + int error; + int fdw, fdr; + + if (flags & ~(O_CLOEXEC | O_NONBLOCK | O_DIRECT | O_NOTIFICATION_PIPE)) + return -EINVAL; + + error = create_pipe_files(files, flags); + if (error) + return error; + + error = get_unused_fd_flags(flags); + if (error < 0) + goto err_read_pipe; + fdr = error; + + error = get_unused_fd_flags(flags); + if (error < 0) + goto err_fdr; + fdw = error; + + audit_fd_pair(fdr, fdw); + fd[0] = fdr; + fd[1] = fdw; + return 0; + + err_fdr: + put_unused_fd(fdr); + err_read_pipe: + fput(files[0]); + fput(files[1]); + return error; +} + +int do_pipe_flags(int *fd, int flags) +{ + struct file *files[2]; + int error = __do_pipe_flags(fd, files, flags); + if (!error) { + fd_install(fd[0], files[0]); + fd_install(fd[1], files[1]); + } + return error; +} + +/* + * sys_pipe() is the normal C calling standard for creating + * a pipe. It's not the way Unix traditionally does this, though. + */ +static int do_pipe2(int __user *fildes, int flags) +{ + struct file *files[2]; + int fd[2]; + int error; + + error = __do_pipe_flags(fd, files, flags); + if (!error) { + if (unlikely(copy_to_user(fildes, fd, sizeof(fd)))) { + fput(files[0]); + fput(files[1]); + put_unused_fd(fd[0]); + put_unused_fd(fd[1]); + error = -EFAULT; + } else { + fd_install(fd[0], files[0]); + fd_install(fd[1], files[1]); + } + } + return error; +} + +SYSCALL_DEFINE2(pipe2, int __user *, fildes, int, flags) +{ + return do_pipe2(fildes, flags); +} + +SYSCALL_DEFINE1(pipe, int __user *, fildes) +{ + return do_pipe2(fildes, 0); +} + +/* + * This is the stupid "wait for pipe to be readable or writable" + * model. + * + * See pipe_read/write() for the proper kind of exclusive wait, + * but that requires that we wake up any other readers/writers + * if we then do not end up reading everything (ie the whole + * "wake_next_reader/writer" logic in pipe_read/write()). + */ +void pipe_wait_readable(struct pipe_inode_info *pipe) +{ + pipe_unlock(pipe); + wait_event_interruptible(pipe->rd_wait, pipe_readable(pipe)); + pipe_lock(pipe); +} + +void pipe_wait_writable(struct pipe_inode_info *pipe) +{ + pipe_unlock(pipe); + wait_event_interruptible(pipe->wr_wait, pipe_writable(pipe)); + pipe_lock(pipe); +} + +/* + * This depends on both the wait (here) and the wakeup (wake_up_partner) + * holding the pipe lock, so "*cnt" is stable and we know a wakeup cannot + * race with the count check and waitqueue prep. + * + * Normally in order to avoid races, you'd do the prepare_to_wait() first, + * then check the condition you're waiting for, and only then sleep. But + * because of the pipe lock, we can check the condition before being on + * the wait queue. + * + * We use the 'rd_wait' waitqueue for pipe partner waiting. + */ +static int wait_for_partner(struct pipe_inode_info *pipe, unsigned int *cnt) +{ + DEFINE_WAIT(rdwait); + int cur = *cnt; + + while (cur == *cnt) { + prepare_to_wait(&pipe->rd_wait, &rdwait, TASK_INTERRUPTIBLE); + pipe_unlock(pipe); + schedule(); + finish_wait(&pipe->rd_wait, &rdwait); + pipe_lock(pipe); + if (signal_pending(current)) + break; + } + return cur == *cnt ? -ERESTARTSYS : 0; +} + +static void wake_up_partner(struct pipe_inode_info *pipe) +{ + wake_up_interruptible_all(&pipe->rd_wait); +} + +static int fifo_open(struct inode *inode, struct file *filp) +{ + bool is_pipe = inode->i_fop == &pipeanon_fops; + struct pipe_inode_info *pipe; + int ret; + + filp->f_pipe = 0; + + spin_lock(&inode->i_lock); + if (inode->i_pipe) { + pipe = inode->i_pipe; + pipe->files++; + spin_unlock(&inode->i_lock); + } else { + spin_unlock(&inode->i_lock); + pipe = alloc_pipe_info(); + if (!pipe) + return -ENOMEM; + pipe->files = 1; + spin_lock(&inode->i_lock); + if (unlikely(inode->i_pipe)) { + inode->i_pipe->files++; + spin_unlock(&inode->i_lock); + free_pipe_info(pipe); + pipe = inode->i_pipe; + } else { + inode->i_pipe = pipe; + spin_unlock(&inode->i_lock); + } + } + filp->private_data = pipe; + /* OK, we have a pipe and it's pinned down */ + + mutex_lock(&pipe->mutex); + + /* We can only do regular read/write on fifos */ + stream_open(inode, filp); + + switch (filp->f_mode & (FMODE_READ | FMODE_WRITE)) { + case FMODE_READ: + /* + * O_RDONLY + * POSIX.1 says that O_NONBLOCK means return with the FIFO + * opened, even when there is no process writing the FIFO. + */ + pipe->r_counter++; + if (pipe->readers++ == 0) + wake_up_partner(pipe); + + if (!is_pipe && !pipe->writers) { + if ((filp->f_flags & O_NONBLOCK)) { + /* suppress EPOLLHUP until we have + * seen a writer */ + filp->f_pipe = pipe->w_counter; + } else { + if (wait_for_partner(pipe, &pipe->w_counter)) + goto err_rd; + } + } + break; + + case FMODE_WRITE: + /* + * O_WRONLY + * POSIX.1 says that O_NONBLOCK means return -1 with + * errno=ENXIO when there is no process reading the FIFO. + */ + ret = -ENXIO; + if (!is_pipe && (filp->f_flags & O_NONBLOCK) && !pipe->readers) + goto err; + + pipe->w_counter++; + if (!pipe->writers++) + wake_up_partner(pipe); + + if (!is_pipe && !pipe->readers) { + if (wait_for_partner(pipe, &pipe->r_counter)) + goto err_wr; + } + break; + + case FMODE_READ | FMODE_WRITE: + /* + * O_RDWR + * POSIX.1 leaves this case "undefined" when O_NONBLOCK is set. + * This implementation will NEVER block on a O_RDWR open, since + * the process can at least talk to itself. + */ + + pipe->readers++; + pipe->writers++; + pipe->r_counter++; + pipe->w_counter++; + if (pipe->readers == 1 || pipe->writers == 1) + wake_up_partner(pipe); + break; + + default: + ret = -EINVAL; + goto err; + } + + /* Ok! */ + mutex_unlock(&pipe->mutex); + return 0; + +err_rd: + if (!--pipe->readers) + wake_up_interruptible(&pipe->wr_wait); + ret = -ERESTARTSYS; + goto err; + +err_wr: + if (!--pipe->writers) + wake_up_interruptible_all(&pipe->rd_wait); + ret = -ERESTARTSYS; + goto err; + +err: + mutex_unlock(&pipe->mutex); + + put_pipe_info(inode, pipe); + return ret; +} + +const struct file_operations pipefifo_fops = { + .open = fifo_open, + .read_iter = fifo_pipe_read, + .write_iter = fifo_pipe_write, + .poll = pipe_poll, + .unlocked_ioctl = pipe_ioctl, + .release = pipe_release, + .fasync = pipe_fasync, + .splice_write = iter_file_splice_write, +}; + +static const struct file_operations pipeanon_fops = { + .open = fifo_open, + .read_iter = anon_pipe_read, + .write_iter = anon_pipe_write, + .poll = pipe_poll, + .unlocked_ioctl = pipe_ioctl, + .release = pipe_release, + .fasync = pipe_fasync, + .splice_write = iter_file_splice_write, +}; + +/* + * Currently we rely on the pipe array holding a power-of-2 number + * of pages. Returns 0 on error. + */ +unsigned int round_pipe_size(unsigned int size) +{ + if (size > (1U << 31)) + return 0; + + /* Minimum pipe size, as required by POSIX */ + if (size < PAGE_SIZE) + return PAGE_SIZE; + + return roundup_pow_of_two(size); +} + +/* + * Resize the pipe ring to a number of slots. + * + * Note the pipe can be reduced in capacity, but only if the current + * occupancy doesn't exceed nr_slots; if it does, EBUSY will be + * returned instead. + */ +int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots) +{ + struct pipe_buffer *bufs; + unsigned int head, tail, mask, n; + + /* nr_slots larger than limits of pipe->{head,tail} */ + if (unlikely(nr_slots > (pipe_index_t)-1u)) + return -EINVAL; + + bufs = kzalloc_objs(*bufs, nr_slots, GFP_KERNEL_ACCOUNT | __GFP_NOWARN); + if (unlikely(!bufs)) + return -ENOMEM; + + spin_lock_irq(&pipe->rd_wait.lock); + mask = pipe->ring_size - 1; + head = pipe->head; + tail = pipe->tail; + + n = pipe_occupancy(head, tail); + if (nr_slots < n) { + spin_unlock_irq(&pipe->rd_wait.lock); + kfree(bufs); + return -EBUSY; + } + + /* + * The pipe array wraps around, so just start the new one at zero + * and adjust the indices. + */ + if (n > 0) { + unsigned int h = head & mask; + unsigned int t = tail & mask; + if (h > t) { + memcpy(bufs, pipe->bufs + t, + n * sizeof(struct pipe_buffer)); + } else { + unsigned int tsize = pipe->ring_size - t; + if (h > 0) + memcpy(bufs + tsize, pipe->bufs, + h * sizeof(struct pipe_buffer)); + memcpy(bufs, pipe->bufs + t, + tsize * sizeof(struct pipe_buffer)); + } + } + + head = n; + tail = 0; + + kfree(pipe->bufs); + pipe->bufs = bufs; + pipe->ring_size = nr_slots; + if (pipe->max_usage > nr_slots) + pipe->max_usage = nr_slots; + pipe->tail = tail; + pipe->head = head; + + if (!pipe_has_watch_queue(pipe)) { + pipe->max_usage = nr_slots; + pipe->nr_accounted = nr_slots; + } + + spin_unlock_irq(&pipe->rd_wait.lock); + + /* This might have made more room for writers */ + wake_up_interruptible(&pipe->wr_wait); + return 0; +} + +/* + * Allocate a new array of pipe buffers and copy the info over. Returns the + * pipe size if successful, or return -ERROR on error. + */ +static long pipe_set_size(struct pipe_inode_info *pipe, unsigned int arg) +{ + unsigned long user_bufs; + unsigned int nr_slots, size; + long ret = 0; + + if (pipe_has_watch_queue(pipe)) + return -EBUSY; + + size = round_pipe_size(arg); + nr_slots = size >> PAGE_SHIFT; + + if (!nr_slots) + return -EINVAL; + + /* + * If trying to increase the pipe capacity, check that an + * unprivileged user is not trying to exceed various limits + * (soft limit check here, hard limit check just below). + * Decreasing the pipe capacity is always permitted, even + * if the user is currently over a limit. + */ + if (nr_slots > pipe->max_usage && + size > pipe_max_size && !capable(CAP_SYS_RESOURCE)) + return -EPERM; + + user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_slots); + + if (nr_slots > pipe->max_usage && + (too_many_pipe_buffers_hard(user_bufs) || + too_many_pipe_buffers_soft(user_bufs)) && + pipe_is_unprivileged_user()) { + ret = -EPERM; + goto out_revert_acct; + } + + ret = pipe_resize_ring(pipe, nr_slots); + if (ret < 0) + goto out_revert_acct; + + return pipe->max_usage * PAGE_SIZE; + +out_revert_acct: + (void) account_pipe_buffers(pipe->user, nr_slots, pipe->nr_accounted); + return ret; +} + +/* + * Note that i_pipe and i_cdev share the same location, so checking ->i_pipe is + * not enough to verify that this is a pipe. + */ +struct pipe_inode_info *get_pipe_info(struct file *file, bool for_splice) +{ + struct pipe_inode_info *pipe = file->private_data; + + if (!pipe) + return NULL; + if (file->f_op != &pipefifo_fops && file->f_op != &pipeanon_fops) + return NULL; + if (for_splice && pipe_has_watch_queue(pipe)) + return NULL; + return pipe; +} + +long pipe_fcntl(struct file *file, unsigned int cmd, unsigned int arg) +{ + struct pipe_inode_info *pipe; + long ret; + + pipe = get_pipe_info(file, false); + if (!pipe) + return -EBADF; + + mutex_lock(&pipe->mutex); + + switch (cmd) { + case F_SETPIPE_SZ: + ret = pipe_set_size(pipe, arg); + break; + case F_GETPIPE_SZ: + ret = pipe->max_usage * PAGE_SIZE; + break; + default: + ret = -EINVAL; + break; + } + + mutex_unlock(&pipe->mutex); + return ret; +} + +static const struct super_operations pipefs_ops = { + .destroy_inode = free_inode_nonrcu, + .statfs = simple_statfs, +}; + +/* + * pipefs should _never_ be mounted by userland - too much of security hassle, + * no real gain from having the whole file system mounted. So we don't need + * any operations on the root directory. However, we need a non-trivial + * d_name - pipe: will go nicely and kill the special-casing in procfs. + */ + +static int pipefs_init_fs_context(struct fs_context *fc) +{ + struct pseudo_fs_context *ctx = init_pseudo(fc, PIPEFS_MAGIC); + if (!ctx) + return -ENOMEM; + ctx->ops = &pipefs_ops; + ctx->dops = &pipefs_dentry_operations; + return 0; +} + +static struct file_system_type pipe_fs_type = { + .name = "pipefs", + .init_fs_context = pipefs_init_fs_context, + .kill_sb = kill_anon_super, +}; + +#ifdef CONFIG_SYSCTL + +static ulong round_pipe_size_ul(ulong size) +{ + return round_pipe_size(size); +} + +static int u2k_pipe_maxsz(const ulong *u_ptr, uint *k_ptr) +{ + return proc_uint_u2k_conv_uop(u_ptr, k_ptr, round_pipe_size_ul); +} + +static int do_proc_uint_conv_pipe_maxsz(bool *negp, ulong *u_ptr, uint *k_ptr, + int dir, const struct ctl_table *table) +{ + return proc_uint_conv(u_ptr, k_ptr, dir, table, true, + u2k_pipe_maxsz, + proc_uint_k2u_conv); +} + +static int proc_dopipe_max_size(const struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + return proc_douintvec_conv(table, write, buffer, lenp, ppos, + do_proc_uint_conv_pipe_maxsz); +} + +static const struct ctl_table fs_pipe_sysctls[] = { + { + .procname = "pipe-max-size", + .data = &pipe_max_size, + .maxlen = sizeof(pipe_max_size), + .mode = 0644, + .proc_handler = proc_dopipe_max_size, + .extra1 = SYSCTL_ONE, + }, + { + .procname = "pipe-user-pages-hard", + .data = &pipe_user_pages_hard, + .maxlen = sizeof(pipe_user_pages_hard), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + }, + { + .procname = "pipe-user-pages-soft", + .data = &pipe_user_pages_soft, + .maxlen = sizeof(pipe_user_pages_soft), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + }, +}; +#endif + +static int __init init_pipe_fs(void) +{ + int err = register_filesystem(&pipe_fs_type); + + if (!err) { + pipe_mnt = kern_mount(&pipe_fs_type); + if (IS_ERR(pipe_mnt)) { + err = PTR_ERR(pipe_mnt); + unregister_filesystem(&pipe_fs_type); + } + } +#ifdef CONFIG_SYSCTL + register_sysctl_init("fs", fs_pipe_sysctls); +#endif + return err; +} + +fs_initcall(init_pipe_fs); diff --git a/fs/pnode.c b/fs/pnode.c new file mode 100644 index 000000000..5d91c3e58 --- /dev/null +++ b/fs/pnode.c @@ -0,0 +1,687 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/pnode.c + * + * (C) Copyright IBM Corporation 2005. + * Author : Ram Pai (linuxram@us.ibm.com) + */ +#include +#include +#include +#include +#include +#include "internal.h" +#include "pnode.h" + +/* return the next shared peer mount of @p */ +static inline struct mount *next_peer(struct mount *p) +{ + return list_entry(p->mnt_share.next, struct mount, mnt_share); +} + +static inline struct mount *first_slave(struct mount *p) +{ + return hlist_entry(p->mnt_slave_list.first, struct mount, mnt_slave); +} + +static inline struct mount *next_slave(struct mount *p) +{ + return hlist_entry(p->mnt_slave.next, struct mount, mnt_slave); +} + +/* locks: namespace_shared && is_mounted(mnt) */ +static struct mount *get_peer_under_root(struct mount *mnt, + struct mnt_namespace *ns, + const struct path *root) +{ + struct mount *m = mnt; + + do { + /* Check the namespace first for optimization */ + if (m->mnt_ns == ns && is_path_reachable(m, m->mnt.mnt_root, root)) + return m; + + m = next_peer(m); + } while (m != mnt); + + return NULL; +} + +/* + * Get ID of closest dominating peer group having a representative + * under the given root. + * + * locks: namespace_shared + */ +int get_dominating_id(struct mount *mnt, const struct path *root) +{ + struct mount *m; + + for (m = mnt->mnt_master; m != NULL; m = m->mnt_master) { + struct mount *d = get_peer_under_root(m, mnt->mnt_ns, root); + if (d) + return d->mnt_group_id; + } + + return 0; +} + +static inline bool will_be_unmounted(struct mount *m) +{ + return m->mnt.mnt_flags & MNT_UMOUNT; +} + +static void transfer_propagation(struct mount *mnt, struct mount *to) +{ + struct hlist_node *p = NULL, *n; + struct mount *m; + + hlist_for_each_entry_safe(m, n, &mnt->mnt_slave_list, mnt_slave) { + m->mnt_master = to; + if (!to) + hlist_del_init(&m->mnt_slave); + else + p = &m->mnt_slave; + } + if (p) + hlist_splice_init(&mnt->mnt_slave_list, p, &to->mnt_slave_list); +} + +/* + * EXCL[namespace_sem] + */ +void change_mnt_propagation(struct mount *mnt, int type) +{ + struct mount *m = mnt->mnt_master; + + if (type == MS_SHARED) { + set_mnt_shared(mnt); + return; + } + if (IS_MNT_SHARED(mnt)) { + if (list_empty(&mnt->mnt_share)) { + mnt_release_group_id(mnt); + } else { + m = next_peer(mnt); + list_del_init(&mnt->mnt_share); + mnt->mnt_group_id = 0; + } + CLEAR_MNT_SHARED(mnt); + transfer_propagation(mnt, m); + } + hlist_del_init(&mnt->mnt_slave); + if (type == MS_SLAVE) { + mnt->mnt_master = m; + if (m) + hlist_add_head(&mnt->mnt_slave, &m->mnt_slave_list); + } else { + mnt->mnt_master = NULL; + if (type == MS_UNBINDABLE) + mnt->mnt_t_flags |= T_UNBINDABLE; + else + mnt->mnt_t_flags &= ~T_UNBINDABLE; + } +} + +static struct mount *trace_transfers(struct mount *m) +{ + while (1) { + struct mount *next = next_peer(m); + + if (next != m) { + list_del_init(&m->mnt_share); + m->mnt_group_id = 0; + m->mnt_master = next; + } else { + if (IS_MNT_SHARED(m)) + mnt_release_group_id(m); + next = m->mnt_master; + } + hlist_del_init(&m->mnt_slave); + CLEAR_MNT_SHARED(m); + SET_MNT_MARK(m); + + if (!next || !will_be_unmounted(next)) + return next; + if (IS_MNT_MARKED(next)) + return next->mnt_master; + m = next; + } +} + +static void set_destinations(struct mount *m, struct mount *master) +{ + struct mount *next; + + while ((next = m->mnt_master) != master) { + m->mnt_master = master; + m = next; + } +} + +void bulk_make_private(struct list_head *set) +{ + struct mount *m; + + list_for_each_entry(m, set, mnt_list) + if (!IS_MNT_MARKED(m)) + set_destinations(m, trace_transfers(m)); + + list_for_each_entry(m, set, mnt_list) { + transfer_propagation(m, m->mnt_master); + m->mnt_master = NULL; + CLEAR_MNT_MARK(m); + } +} + +static struct mount *__propagation_next(struct mount *m, + struct mount *origin) +{ + while (1) { + struct mount *master = m->mnt_master; + + if (master == origin->mnt_master) { + struct mount *next = next_peer(m); + return (next == origin) ? NULL : next; + } else if (m->mnt_slave.next) + return next_slave(m); + + /* back at master */ + m = master; + } +} + +/* + * get the next mount in the propagation tree. + * @m: the mount seen last + * @origin: the original mount from where the tree walk initiated + * + * Note that peer groups form contiguous segments of slave lists. + * We rely on that in get_source() to be able to find out if + * vfsmount found while iterating with propagation_next() is + * a peer of one we'd found earlier. + */ +static struct mount *propagation_next(struct mount *m, + struct mount *origin) +{ + /* are there any slaves of this mount? */ + if (!IS_MNT_NEW(m) && !hlist_empty(&m->mnt_slave_list)) + return first_slave(m); + + return __propagation_next(m, origin); +} + +static struct mount *skip_propagation_subtree(struct mount *m, + struct mount *origin) +{ + /* + * Advance m past everything that gets propagation from it. + */ + struct mount *p = __propagation_next(m, origin); + + while (p && peers(m, p)) + p = __propagation_next(p, origin); + + return p; +} + +static struct mount *next_group(struct mount *m, struct mount *origin) +{ + while (1) { + while (1) { + struct mount *next; + if (!IS_MNT_NEW(m) && !hlist_empty(&m->mnt_slave_list)) + return first_slave(m); + next = next_peer(m); + if (m->mnt_group_id == origin->mnt_group_id) { + if (next == origin) + return NULL; + } else if (m->mnt_slave.next != &next->mnt_slave) + break; + m = next; + } + /* m is the last peer */ + while (1) { + struct mount *master = m->mnt_master; + if (m->mnt_slave.next) + return next_slave(m); + m = next_peer(master); + if (master->mnt_group_id == origin->mnt_group_id) + break; + if (master->mnt_slave.next == &m->mnt_slave) + break; + m = master; + } + if (m == origin) + return NULL; + } +} + +static bool need_secondary(struct mount *m, struct mountpoint *dest_mp) +{ + /* skip ones added by this propagate_mnt() */ + if (IS_MNT_NEW(m)) + return false; + /* skip if mountpoint isn't visible in m */ + if (!is_subdir(dest_mp->m_dentry, m->mnt.mnt_root)) + return false; + /* skip if m is in the anon_ns */ + if (is_anon_ns(m->mnt_ns)) + return false; + return true; +} + +static struct mount *find_master(struct mount *m, + struct mount *last_copy, + struct mount *original) +{ + struct mount *p; + + // ascend until there's a copy for something with the same master + for (;;) { + p = m->mnt_master; + if (!p || IS_MNT_MARKED(p)) + break; + m = p; + } + while (!peers(last_copy, original)) { + struct mount *parent = last_copy->mnt_parent; + if (parent->mnt_master == p) { + if (!peers(parent, m)) + last_copy = last_copy->mnt_master; + break; + } + last_copy = last_copy->mnt_master; + } + return last_copy; +} + +/** + * propagate_mnt() - create secondary copies for tree attachment + * @dest_mnt: destination mount. + * @dest_mp: destination mountpoint. + * @source_mnt: source mount. + * @tree_list: list of secondaries to be attached. + * + * Create secondary copies for attaching a tree with root @source_mnt + * at mount @dest_mnt with mountpoint @dest_mp. Link all new mounts + * into a propagation graph. Set mountpoints for all secondaries, + * link their roots into @tree_list via ->mnt_hash. + */ +int propagate_mnt(struct mount *dest_mnt, struct mountpoint *dest_mp, + struct mount *source_mnt, struct hlist_head *tree_list) +{ + struct mount *m, *n, *copy, *this; + int err = 0, type; + + if (dest_mnt->mnt_master) + SET_MNT_MARK(dest_mnt->mnt_master); + + /* iterate over peer groups, depth first */ + for (m = dest_mnt; m && !err; m = next_group(m, dest_mnt)) { + if (m == dest_mnt) { // have one for dest_mnt itself + copy = source_mnt; + type = CL_MAKE_SHARED; + n = next_peer(m); + if (n == m) + continue; + } else { + type = CL_SLAVE; + /* beginning of peer group among the slaves? */ + if (IS_MNT_SHARED(m)) + type |= CL_MAKE_SHARED; + n = m; + } + do { + if (!need_secondary(n, dest_mp)) + continue; + if (type & CL_SLAVE) // first in this peer group + copy = find_master(n, copy, source_mnt); + this = copy_tree(copy, copy->mnt.mnt_root, type); + if (IS_ERR(this)) { + err = PTR_ERR(this); + break; + } + scoped_guard(mount_locked_reader) + mnt_set_mountpoint(n, dest_mp, this); + if (n->mnt_master) + SET_MNT_MARK(n->mnt_master); + copy = this; + hlist_add_head(&this->mnt_hash, tree_list); + err = count_mounts(n->mnt_ns, this); + if (err) + break; + type = CL_MAKE_SHARED; + } while ((n = next_peer(n)) != m); + } + + hlist_for_each_entry(n, tree_list, mnt_hash) { + m = n->mnt_parent; + if (m->mnt_master) + CLEAR_MNT_MARK(m->mnt_master); + } + if (dest_mnt->mnt_master) + CLEAR_MNT_MARK(dest_mnt->mnt_master); + return err; +} + +/* + * return true if the refcount is greater than count + */ +static inline int do_refcount_check(struct mount *mnt, int count) +{ + return mnt_get_count(mnt) > count; +} + +/** + * propagation_would_overmount - check whether propagation from @from + * would overmount @to + * @from: shared mount + * @to: mount to check + * @mp: future mountpoint of @to on @from + * + * If @from propagates mounts to @to, @from and @to must either be peers + * or one of the masters in the hierarchy of masters of @to must be a + * peer of @from. + * + * If the root of the @to mount is equal to the future mountpoint @mp of + * the @to mount on @from then @to will be overmounted by whatever is + * propagated to it. + * + * Context: This function expects namespace_lock() to be held and that + * @mp is stable. + * Return: If @from overmounts @to, true is returned, false if not. + */ +bool propagation_would_overmount(const struct mount *from, + const struct mount *to, + const struct mountpoint *mp) +{ + if (!IS_MNT_SHARED(from)) + return false; + + if (to->mnt.mnt_root != mp->m_dentry) + return false; + + for (const struct mount *m = to; m; m = m->mnt_master) { + if (peers(from, m)) + return true; + } + + return false; +} + +/* + * check if the mount 'mnt' can be unmounted successfully. + * @mnt: the mount to be checked for unmount + * NOTE: unmounting 'mnt' would naturally propagate to all + * other mounts its parent propagates to. + * Check if any of these mounts that **do not have submounts** + * have more references than 'refcnt'. If so return busy. + * + * vfsmount lock must be held for write + */ +int propagate_mount_busy(struct mount *mnt, int refcnt) +{ + struct mount *parent = mnt->mnt_parent; + + /* + * quickly check if the current mount can be unmounted. + * If not, we don't have to go checking for all other + * mounts + */ + if (!list_empty(&mnt->mnt_mounts) || do_refcount_check(mnt, refcnt)) + return 1; + + if (mnt == parent) + return 0; + + for (struct mount *m = propagation_next(parent, parent); m; + m = propagation_next(m, parent)) { + struct list_head *head; + struct mount *child = __lookup_mnt(&m->mnt, mnt->mnt_mountpoint); + + if (!child) + continue; + + head = &child->mnt_mounts; + if (!list_empty(head)) { + /* + * a mount that covers child completely wouldn't prevent + * it being pulled out; any other would. + */ + if (!list_is_singular(head) || !child->overmount) + continue; + } + if (do_refcount_check(child, 1)) + return 1; + } + return 0; +} + +/* + * Clear MNT_LOCKED when it can be shown to be safe. + * + * mount_lock lock must be held for write + */ +void propagate_mount_unlock(struct mount *mnt) +{ + struct mount *parent = mnt->mnt_parent; + struct mount *m, *child; + + BUG_ON(parent == mnt); + + for (m = propagation_next(parent, parent); m; + m = propagation_next(m, parent)) { + child = __lookup_mnt(&m->mnt, mnt->mnt_mountpoint); + if (child) + child->mnt.mnt_flags &= ~MNT_LOCKED; + } +} + +static inline bool is_candidate(struct mount *m) +{ + return m->mnt_t_flags & T_UMOUNT_CANDIDATE; +} + +static void umount_one(struct mount *m, struct list_head *to_umount) +{ + m->mnt.mnt_flags |= MNT_UMOUNT; + list_del_init(&m->mnt_child); + move_from_ns(m); + list_add_tail(&m->mnt_list, to_umount); +} + +static void remove_from_candidate_list(struct mount *m) +{ + m->mnt_t_flags &= ~(T_MARKED | T_UMOUNT_CANDIDATE); + list_del_init(&m->mnt_list); +} + +static void gather_candidates(struct list_head *set, + struct list_head *candidates) +{ + struct mount *m, *p, *q; + + list_for_each_entry(m, set, mnt_list) { + if (is_candidate(m)) + continue; + m->mnt_t_flags |= T_UMOUNT_CANDIDATE; + p = m->mnt_parent; + q = propagation_next(p, p); + while (q) { + struct mount *child = __lookup_mnt(&q->mnt, + m->mnt_mountpoint); + if (child) { + /* + * We might've already run into this one. That + * must've happened on earlier iteration of the + * outer loop; in that case we can skip those + * parents that get propagation from q - there + * will be nothing new on those as well. + */ + if (is_candidate(child)) { + q = skip_propagation_subtree(q, p); + continue; + } + child->mnt_t_flags |= T_UMOUNT_CANDIDATE; + if (!will_be_unmounted(child)) + list_add(&child->mnt_list, candidates); + } + q = propagation_next(q, p); + } + } + list_for_each_entry(m, set, mnt_list) + m->mnt_t_flags &= ~T_UMOUNT_CANDIDATE; +} + +/* + * We know that some child of @m can't be unmounted. In all places where the + * chain of descent of @m has child not overmounting the root of parent, + * the parent can't be unmounted either. + */ +static void trim_ancestors(struct mount *m) +{ + struct mount *p; + + for (p = m->mnt_parent; is_candidate(p); m = p, p = p->mnt_parent) { + if (IS_MNT_MARKED(m)) // all candidates beneath are overmounts + return; + SET_MNT_MARK(m); + if (m != p->overmount) + p->mnt_t_flags &= ~T_UMOUNT_CANDIDATE; + } +} + +/* + * Find and exclude all umount candidates forbidden by @m + * (see Documentation/filesystems/propagate_umount.txt) + * If we can immediately tell that @m is OK to unmount (unlocked + * and all children are already committed to unmounting) commit + * to unmounting it. + * Only @m itself might be taken from the candidates list; + * anything found by trim_ancestors() is marked non-candidate + * and left on the list. + */ +static void trim_one(struct mount *m, struct list_head *to_umount) +{ + bool remove_this = false, found = false, umount_this = false; + struct mount *n; + + if (!is_candidate(m)) { // trim_ancestors() left it on list + remove_from_candidate_list(m); + return; + } + + list_for_each_entry(n, &m->mnt_mounts, mnt_child) { + if (!is_candidate(n)) { + found = true; + if (n != m->overmount) { + remove_this = true; + break; + } + } + } + if (found) { + trim_ancestors(m); + } else if (!IS_MNT_LOCKED(m) && list_empty(&m->mnt_mounts)) { + remove_this = true; + umount_this = true; + } + if (remove_this) { + remove_from_candidate_list(m); + if (umount_this) + umount_one(m, to_umount); + } +} + +static void handle_locked(struct mount *m, struct list_head *to_umount) +{ + struct mount *cutoff = m, *p; + + if (!is_candidate(m)) { // trim_ancestors() left it on list + remove_from_candidate_list(m); + return; + } + for (p = m; is_candidate(p); p = p->mnt_parent) { + remove_from_candidate_list(p); + if (!IS_MNT_LOCKED(p)) + cutoff = p->mnt_parent; + } + if (will_be_unmounted(p)) + cutoff = p; + while (m != cutoff) { + umount_one(m, to_umount); + m = m->mnt_parent; + } +} + +/* + * @m is not to going away, and it overmounts the top of a stack of mounts + * that are going away. We know that all of those are fully overmounted + * by the one above (@m being the topmost of the chain), so @m can be slid + * in place where the bottom of the stack is attached. + * + * NOTE: here we temporarily violate a constraint - two mounts end up with + * the same parent and mountpoint; that will be remedied as soon as we + * return from propagate_umount() - its caller (umount_tree()) will detach + * the stack from the parent it (and now @m) is attached to. umount_tree() + * might choose to keep unmounted pieces stuck to each other, but it always + * detaches them from the mounts that remain in the tree. + */ +static void reparent(struct mount *m) +{ + struct mount *p = m; + struct mountpoint *mp; + + do { + mp = p->mnt_mp; + p = p->mnt_parent; + } while (will_be_unmounted(p)); + + mnt_change_mountpoint(p, mp, m); + mnt_notify_add(m); +} + +/** + * propagate_umount - apply propagation rules to the set of mounts for umount() + * @set: the list of mounts to be unmounted. + * + * Collect all mounts that receive propagation from the mount in @set and have + * no obstacles to being unmounted. Add these additional mounts to the set. + * + * See Documentation/filesystems/propagate_umount.txt if you do anything in + * this area. + * + * Locks held: + * mount_lock (write_seqlock), namespace_sem (exclusive). + */ +void propagate_umount(struct list_head *set) +{ + struct mount *m, *p; + LIST_HEAD(to_umount); // committed to unmounting + LIST_HEAD(candidates); // undecided umount candidates + + // collect all candidates + gather_candidates(set, &candidates); + + // reduce the set until it's non-shifting + list_for_each_entry_safe(m, p, &candidates, mnt_list) + trim_one(m, &to_umount); + + // ... and non-revealing + while (!list_empty(&candidates)) { + m = list_first_entry(&candidates,struct mount, mnt_list); + handle_locked(m, &to_umount); + } + + // now to_umount consists of all acceptable candidates + // deal with reparenting of surviving overmounts on those + list_for_each_entry(m, &to_umount, mnt_list) { + struct mount *over = m->overmount; + if (over && !will_be_unmounted(over)) + reparent(over); + } + + // and fold them into the set + list_splice_tail_init(&to_umount, set); +} diff --git a/fs/pnode.h b/fs/pnode.h new file mode 100644 index 000000000..b029db225 --- /dev/null +++ b/fs/pnode.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * linux/fs/pnode.h + * + * (C) Copyright IBM Corporation 2005. + */ +#ifndef _LINUX_PNODE_H +#define _LINUX_PNODE_H + +#include +#include "mount.h" + +#define IS_MNT_SHARED(m) ((m)->mnt_t_flags & T_SHARED) +#define IS_MNT_SLAVE(m) ((m)->mnt_master) +#define IS_MNT_NEW(m) (!(m)->mnt_ns) +#define CLEAR_MNT_SHARED(m) ((m)->mnt_t_flags &= ~T_SHARED) +#define IS_MNT_UNBINDABLE(m) ((m)->mnt_t_flags & T_UNBINDABLE) +#define IS_MNT_MARKED(m) ((m)->mnt_t_flags & T_MARKED) +#define SET_MNT_MARK(m) ((m)->mnt_t_flags |= T_MARKED) +#define CLEAR_MNT_MARK(m) ((m)->mnt_t_flags &= ~T_MARKED) +#define IS_MNT_LOCKED(m) ((m)->mnt.mnt_flags & MNT_LOCKED) + +#define CL_EXPIRE 0x01 +#define CL_SLAVE 0x02 +#define CL_COPY_UNBINDABLE 0x04 +#define CL_MAKE_SHARED 0x08 +#define CL_PRIVATE 0x10 +#define CL_COPY_MNT_NS_FILE 0x40 + +/* + * EXCL[namespace_sem] + */ +static inline void set_mnt_shared(struct mount *mnt) +{ + mnt->mnt_t_flags &= ~T_SHARED_MASK; + mnt->mnt_t_flags |= T_SHARED; +} + +static inline bool peers(const struct mount *m1, const struct mount *m2) +{ + return m1->mnt_group_id == m2->mnt_group_id && m1->mnt_group_id; +} + +void change_mnt_propagation(struct mount *, int); +void bulk_make_private(struct list_head *); +int propagate_mnt(struct mount *, struct mountpoint *, struct mount *, + struct hlist_head *); +void propagate_umount(struct list_head *); +int propagate_mount_busy(struct mount *, int); +void propagate_mount_unlock(struct mount *); +void mnt_release_group_id(struct mount *); +int get_dominating_id(struct mount *mnt, const struct path *root); +int mnt_get_count(struct mount *mnt); +void mnt_set_mountpoint(struct mount *, struct mountpoint *, + struct mount *); +void mnt_change_mountpoint(struct mount *parent, struct mountpoint *mp, + struct mount *mnt); +struct mount *copy_tree(struct mount *, struct dentry *, int); +bool is_path_reachable(struct mount *, struct dentry *, + const struct path *root); +int count_mounts(struct mnt_namespace *ns, struct mount *mnt); +bool propagation_would_overmount(const struct mount *from, + const struct mount *to, + const struct mountpoint *mp); +#endif /* _LINUX_PNODE_H */ diff --git a/fs/posix_acl.c b/fs/posix_acl.c new file mode 100644 index 000000000..18b302f94 --- /dev/null +++ b/fs/posix_acl.c @@ -0,0 +1,1303 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2002,2003 by Andreas Gruenbacher + * + * Fixes from William Schumacher incorporated on 15 March 2001. + * (Reported by Charles Bertsch, ). + */ + +/* + * This file contains generic functions for manipulating + * POSIX 1003.1e draft standard 17 ACLs. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +static struct posix_acl **acl_by_type(struct inode *inode, int type) +{ + switch (type) { + case ACL_TYPE_ACCESS: + return &inode->i_acl; + case ACL_TYPE_DEFAULT: + return &inode->i_default_acl; + default: + BUG(); + } +} + +struct posix_acl *get_cached_acl(struct inode *inode, int type) +{ + struct posix_acl **p = acl_by_type(inode, type); + struct posix_acl *acl; + + for (;;) { + rcu_read_lock(); + acl = rcu_dereference(*p); + if (!acl || is_uncached_acl(acl) || + refcount_inc_not_zero(&acl->a_refcount)) + break; + rcu_read_unlock(); + cpu_relax(); + } + rcu_read_unlock(); + return acl; +} +EXPORT_SYMBOL(get_cached_acl); + +struct posix_acl *get_cached_acl_rcu(struct inode *inode, int type) +{ + struct posix_acl *acl = rcu_dereference(*acl_by_type(inode, type)); + + if (acl == ACL_DONT_CACHE) { + struct posix_acl *ret; + + ret = inode->i_op->get_inode_acl(inode, type, LOOKUP_RCU); + if (!IS_ERR(ret)) + acl = ret; + } + + return acl; +} +EXPORT_SYMBOL(get_cached_acl_rcu); + +void set_cached_acl(struct inode *inode, int type, struct posix_acl *acl) +{ + struct posix_acl **p = acl_by_type(inode, type); + struct posix_acl *old; + + old = xchg(p, posix_acl_dup(acl)); + if (!is_uncached_acl(old)) + posix_acl_release(old); +} +EXPORT_SYMBOL(set_cached_acl); + +static void __forget_cached_acl(struct posix_acl **p) +{ + struct posix_acl *old; + + /* + * ACL_DONT_CACHE is expected to be a "const" value and xchg it with + * ACL_NOT_CACHED would enable acl caching for the inode - + * clearly not what the caller has intended. + */ + if (READ_ONCE(*p) == ACL_DONT_CACHE) + return; + old = xchg(p, ACL_NOT_CACHED); + if (!is_uncached_acl(old)) + posix_acl_release(old); +} + +void forget_cached_acl(struct inode *inode, int type) +{ + __forget_cached_acl(acl_by_type(inode, type)); +} +EXPORT_SYMBOL(forget_cached_acl); + +void forget_all_cached_acls(struct inode *inode) +{ + __forget_cached_acl(&inode->i_acl); + __forget_cached_acl(&inode->i_default_acl); +} +EXPORT_SYMBOL(forget_all_cached_acls); + +static struct posix_acl *__get_acl(struct mnt_idmap *idmap, + struct dentry *dentry, struct inode *inode, + int type) +{ + struct posix_acl *sentinel; + struct posix_acl **p; + struct posix_acl *acl; + + /* + * The sentinel is used to detect when another operation like + * set_cached_acl() or forget_cached_acl() races with get_inode_acl(). + * It is guaranteed that is_uncached_acl(sentinel) is true. + */ + + acl = get_cached_acl(inode, type); + if (!is_uncached_acl(acl)) + return acl; + + if (!IS_POSIXACL(inode)) + return NULL; + + sentinel = uncached_acl_sentinel(current); + p = acl_by_type(inode, type); + + /* + * If the ACL isn't being read yet, set our sentinel. Otherwise, the + * current value of the ACL will not be ACL_NOT_CACHED and so our own + * sentinel will not be set; another task will update the cache. We + * could wait for that other task to complete its job, but it's easier + * to just call ->get_inode_acl to fetch the ACL ourself. (This is + * going to be an unlikely race.) + */ + cmpxchg(p, ACL_NOT_CACHED, sentinel); + + /* + * Normally, the ACL returned by ->get{_inode}_acl will be cached. + * A filesystem can prevent that by calling + * forget_cached_acl(inode, type) in ->get{_inode}_acl. + * + * If the filesystem doesn't have a get{_inode}_ acl() function at all, + * we'll just create the negative cache entry. + */ + if (dentry && inode->i_op->get_acl) { + acl = inode->i_op->get_acl(idmap, dentry, type); + } else if (inode->i_op->get_inode_acl) { + acl = inode->i_op->get_inode_acl(inode, type, false); + } else { + set_cached_acl(inode, type, NULL); + return NULL; + } + if (IS_ERR(acl)) { + /* + * Remove our sentinel so that we don't block future attempts + * to cache the ACL. + */ + cmpxchg(p, sentinel, ACL_NOT_CACHED); + return acl; + } + + /* + * Cache the result, but only if our sentinel is still in place. + */ + posix_acl_dup(acl); + if (unlikely(!try_cmpxchg(p, &sentinel, acl))) + posix_acl_release(acl); + return acl; +} + +struct posix_acl *get_inode_acl(struct inode *inode, int type) +{ + return __get_acl(&nop_mnt_idmap, NULL, inode, type); +} +EXPORT_SYMBOL(get_inode_acl); + +/* + * Init a fresh posix_acl + */ +void +posix_acl_init(struct posix_acl *acl, int count) +{ + refcount_set(&acl->a_refcount, 1); + acl->a_count = count; +} +EXPORT_SYMBOL(posix_acl_init); + +/* + * Allocate a new ACL with the specified number of entries. + */ +struct posix_acl * +posix_acl_alloc(unsigned int count, gfp_t flags) +{ + struct posix_acl *acl; + + acl = kmalloc_flex(*acl, a_entries, count, flags); + if (acl) + posix_acl_init(acl, count); + return acl; +} +EXPORT_SYMBOL(posix_acl_alloc); + +/* + * Clone an ACL. + */ +struct posix_acl * +posix_acl_clone(const struct posix_acl *acl, gfp_t flags) +{ + struct posix_acl *clone = NULL; + + if (acl) { + clone = kmemdup(acl, struct_size(acl, a_entries, acl->a_count), + flags); + if (clone) + refcount_set(&clone->a_refcount, 1); + } + return clone; +} +EXPORT_SYMBOL_GPL(posix_acl_clone); + +/* + * Check if an acl is valid. Returns 0 if it is, or -E... otherwise. + */ +int +posix_acl_valid(struct user_namespace *user_ns, const struct posix_acl *acl) +{ + const struct posix_acl_entry *pa, *pe; + int state = ACL_USER_OBJ; + int needs_mask = 0; + + FOREACH_ACL_ENTRY(pa, acl, pe) { + if (pa->e_perm & ~(ACL_READ|ACL_WRITE|ACL_EXECUTE)) + return -EINVAL; + switch (pa->e_tag) { + case ACL_USER_OBJ: + if (state == ACL_USER_OBJ) { + state = ACL_USER; + break; + } + return -EINVAL; + + case ACL_USER: + if (state != ACL_USER) + return -EINVAL; + if (!kuid_has_mapping(user_ns, pa->e_uid)) + return -EINVAL; + needs_mask = 1; + break; + + case ACL_GROUP_OBJ: + if (state == ACL_USER) { + state = ACL_GROUP; + break; + } + return -EINVAL; + + case ACL_GROUP: + if (state != ACL_GROUP) + return -EINVAL; + if (!kgid_has_mapping(user_ns, pa->e_gid)) + return -EINVAL; + needs_mask = 1; + break; + + case ACL_MASK: + if (state != ACL_GROUP) + return -EINVAL; + state = ACL_OTHER; + break; + + case ACL_OTHER: + if (state == ACL_OTHER || + (state == ACL_GROUP && !needs_mask)) { + state = 0; + break; + } + return -EINVAL; + + default: + return -EINVAL; + } + } + if (state == 0) + return 0; + return -EINVAL; +} +EXPORT_SYMBOL(posix_acl_valid); + +/* + * Returns 0 if the acl can be exactly represented in the traditional + * file mode permission bits, or else 1. Returns -E... on error. + */ +int +posix_acl_equiv_mode(const struct posix_acl *acl, umode_t *mode_p) +{ + const struct posix_acl_entry *pa, *pe; + umode_t mode = 0; + int not_equiv = 0; + + /* + * A null ACL can always be presented as mode bits. + */ + if (!acl) + return 0; + + FOREACH_ACL_ENTRY(pa, acl, pe) { + switch (pa->e_tag) { + case ACL_USER_OBJ: + mode |= (pa->e_perm & S_IRWXO) << 6; + break; + case ACL_GROUP_OBJ: + mode |= (pa->e_perm & S_IRWXO) << 3; + break; + case ACL_OTHER: + mode |= pa->e_perm & S_IRWXO; + break; + case ACL_MASK: + mode = (mode & ~S_IRWXG) | + ((pa->e_perm & S_IRWXO) << 3); + not_equiv = 1; + break; + case ACL_USER: + case ACL_GROUP: + not_equiv = 1; + break; + default: + return -EINVAL; + } + } + if (mode_p) + *mode_p = (*mode_p & ~S_IRWXUGO) | mode; + return not_equiv; +} +EXPORT_SYMBOL(posix_acl_equiv_mode); + +/* + * Create an ACL representing the file mode permission bits of an inode. + */ +struct posix_acl * +posix_acl_from_mode(umode_t mode, gfp_t flags) +{ + struct posix_acl *acl = posix_acl_alloc(3, flags); + if (!acl) + return ERR_PTR(-ENOMEM); + + acl->a_entries[0].e_tag = ACL_USER_OBJ; + acl->a_entries[0].e_perm = (mode & S_IRWXU) >> 6; + + acl->a_entries[1].e_tag = ACL_GROUP_OBJ; + acl->a_entries[1].e_perm = (mode & S_IRWXG) >> 3; + + acl->a_entries[2].e_tag = ACL_OTHER; + acl->a_entries[2].e_perm = (mode & S_IRWXO); + return acl; +} +EXPORT_SYMBOL(posix_acl_from_mode); + +/* + * Return 0 if current is granted want access to the inode + * by the acl. Returns -E... otherwise. + */ +int +posix_acl_permission(struct mnt_idmap *idmap, struct inode *inode, + const struct posix_acl *acl, int want) +{ + const struct posix_acl_entry *pa, *pe, *mask_obj; + struct user_namespace *fs_userns = i_user_ns(inode); + int found = 0; + vfsuid_t vfsuid; + vfsgid_t vfsgid; + + want &= MAY_READ | MAY_WRITE | MAY_EXEC; + + FOREACH_ACL_ENTRY(pa, acl, pe) { + switch(pa->e_tag) { + case ACL_USER_OBJ: + /* (May have been checked already) */ + vfsuid = i_uid_into_vfsuid(idmap, inode); + if (vfsuid_eq_kuid(vfsuid, current_fsuid())) + goto check_perm; + break; + case ACL_USER: + vfsuid = make_vfsuid(idmap, fs_userns, + pa->e_uid); + if (vfsuid_eq_kuid(vfsuid, current_fsuid())) + goto mask; + break; + case ACL_GROUP_OBJ: + vfsgid = i_gid_into_vfsgid(idmap, inode); + if (vfsgid_in_group_p(vfsgid)) { + found = 1; + if ((pa->e_perm & want) == want) + goto mask; + } + break; + case ACL_GROUP: + vfsgid = make_vfsgid(idmap, fs_userns, + pa->e_gid); + if (vfsgid_in_group_p(vfsgid)) { + found = 1; + if ((pa->e_perm & want) == want) + goto mask; + } + break; + case ACL_MASK: + break; + case ACL_OTHER: + if (found) + return -EACCES; + else + goto check_perm; + default: + return -EIO; + } + } + return -EIO; + +mask: + for (mask_obj = pa+1; mask_obj != pe; mask_obj++) { + if (mask_obj->e_tag == ACL_MASK) { + if ((pa->e_perm & mask_obj->e_perm & want) == want) + return 0; + return -EACCES; + } + } + +check_perm: + if ((pa->e_perm & want) == want) + return 0; + return -EACCES; +} + +/* + * Modify acl when creating a new inode. The caller must ensure the acl is + * only referenced once. + * + * mode_p initially must contain the mode parameter to the open() / creat() + * system calls. All permissions that are not granted by the acl are removed. + * The permissions in the acl are changed to reflect the mode_p parameter. + */ +static int posix_acl_create_masq(struct posix_acl *acl, umode_t *mode_p) +{ + struct posix_acl_entry *pa, *pe; + struct posix_acl_entry *group_obj = NULL, *mask_obj = NULL; + umode_t mode = *mode_p; + int not_equiv = 0; + + /* assert(atomic_read(acl->a_refcount) == 1); */ + + FOREACH_ACL_ENTRY(pa, acl, pe) { + switch(pa->e_tag) { + case ACL_USER_OBJ: + pa->e_perm &= (mode >> 6) | ~S_IRWXO; + mode &= (pa->e_perm << 6) | ~S_IRWXU; + break; + + case ACL_USER: + case ACL_GROUP: + not_equiv = 1; + break; + + case ACL_GROUP_OBJ: + group_obj = pa; + break; + + case ACL_OTHER: + pa->e_perm &= mode | ~S_IRWXO; + mode &= pa->e_perm | ~S_IRWXO; + break; + + case ACL_MASK: + mask_obj = pa; + not_equiv = 1; + break; + + default: + return -EIO; + } + } + + if (mask_obj) { + mask_obj->e_perm &= (mode >> 3) | ~S_IRWXO; + mode &= (mask_obj->e_perm << 3) | ~S_IRWXG; + } else { + if (!group_obj) + return -EIO; + group_obj->e_perm &= (mode >> 3) | ~S_IRWXO; + mode &= (group_obj->e_perm << 3) | ~S_IRWXG; + } + + *mode_p = (*mode_p & ~S_IRWXUGO) | mode; + return not_equiv; +} + +/* + * Modify the ACL for the chmod syscall. + */ +static int __posix_acl_chmod_masq(struct posix_acl *acl, umode_t mode) +{ + struct posix_acl_entry *group_obj = NULL, *mask_obj = NULL; + struct posix_acl_entry *pa, *pe; + + /* assert(atomic_read(acl->a_refcount) == 1); */ + + FOREACH_ACL_ENTRY(pa, acl, pe) { + switch(pa->e_tag) { + case ACL_USER_OBJ: + pa->e_perm = (mode & S_IRWXU) >> 6; + break; + + case ACL_USER: + case ACL_GROUP: + break; + + case ACL_GROUP_OBJ: + group_obj = pa; + break; + + case ACL_MASK: + mask_obj = pa; + break; + + case ACL_OTHER: + pa->e_perm = (mode & S_IRWXO); + break; + + default: + return -EIO; + } + } + + if (mask_obj) { + mask_obj->e_perm = (mode & S_IRWXG) >> 3; + } else { + if (!group_obj) + return -EIO; + group_obj->e_perm = (mode & S_IRWXG) >> 3; + } + + return 0; +} + +int +__posix_acl_create(struct posix_acl **acl, gfp_t gfp, umode_t *mode_p) +{ + struct posix_acl *clone = posix_acl_clone(*acl, gfp); + int err = -ENOMEM; + if (clone) { + err = posix_acl_create_masq(clone, mode_p); + if (err < 0) { + posix_acl_release(clone); + clone = NULL; + } + } + posix_acl_release(*acl); + *acl = clone; + return err; +} +EXPORT_SYMBOL(__posix_acl_create); + +int +__posix_acl_chmod(struct posix_acl **acl, gfp_t gfp, umode_t mode) +{ + struct posix_acl *clone = posix_acl_clone(*acl, gfp); + int err = -ENOMEM; + if (clone) { + err = __posix_acl_chmod_masq(clone, mode); + if (err) { + posix_acl_release(clone); + clone = NULL; + } + } + posix_acl_release(*acl); + *acl = clone; + return err; +} +EXPORT_SYMBOL(__posix_acl_chmod); + +/** + * posix_acl_chmod - chmod a posix acl + * + * @idmap: idmap of the mount @inode was found from + * @dentry: dentry to check permissions on + * @mode: the new mode of @inode + * + * If the dentry has been found through an idmapped mount the idmap of + * the vfsmount must be passed through @idmap. This function will then + * take care to map the inode according to @idmap before checking + * permissions. On non-idmapped mounts or if permission checking is to be + * performed on the raw inode simply pass @nop_mnt_idmap. + */ +int + posix_acl_chmod(struct mnt_idmap *idmap, struct dentry *dentry, + umode_t mode) +{ + struct inode *inode = d_inode(dentry); + struct posix_acl *acl; + int ret = 0; + + if (!IS_POSIXACL(inode)) + return 0; + if (!inode->i_op->set_acl) + return -EOPNOTSUPP; + + acl = get_inode_acl(inode, ACL_TYPE_ACCESS); + if (IS_ERR_OR_NULL(acl)) { + if (acl == ERR_PTR(-EOPNOTSUPP)) + return 0; + return PTR_ERR(acl); + } + + ret = __posix_acl_chmod(&acl, GFP_KERNEL, mode); + if (ret) + return ret; + ret = inode->i_op->set_acl(idmap, dentry, acl, ACL_TYPE_ACCESS); + posix_acl_release(acl); + return ret; +} +EXPORT_SYMBOL(posix_acl_chmod); + +int +posix_acl_create(struct inode *dir, umode_t *mode, + struct posix_acl **default_acl, struct posix_acl **acl) +{ + struct posix_acl *p; + struct posix_acl *clone; + int ret; + + *acl = NULL; + *default_acl = NULL; + + if (S_ISLNK(*mode) || !IS_POSIXACL(dir)) + return 0; + + p = get_inode_acl(dir, ACL_TYPE_DEFAULT); + if (!p || p == ERR_PTR(-EOPNOTSUPP)) { + *mode &= ~current_umask(); + return 0; + } + if (IS_ERR(p)) + return PTR_ERR(p); + + ret = -ENOMEM; + clone = posix_acl_clone(p, GFP_NOFS); + if (!clone) + goto err_release; + + ret = posix_acl_create_masq(clone, mode); + if (ret < 0) + goto err_release_clone; + + if (ret == 0) + posix_acl_release(clone); + else + *acl = clone; + + if (!S_ISDIR(*mode)) + posix_acl_release(p); + else + *default_acl = p; + + return 0; + +err_release_clone: + posix_acl_release(clone); +err_release: + posix_acl_release(p); + return ret; +} +EXPORT_SYMBOL_GPL(posix_acl_create); + +/** + * posix_acl_update_mode - update mode in set_acl + * @idmap: idmap of the mount @inode was found from + * @inode: target inode + * @mode_p: mode (pointer) for update + * @acl: acl pointer + * + * Update the file mode when setting an ACL: compute the new file permission + * bits based on the ACL. In addition, if the ACL is equivalent to the new + * file mode, set *@acl to NULL to indicate that no ACL should be set. + * + * As with chmod, clear the setgid bit if the caller is not in the owning group + * or capable of CAP_FSETID (see inode_change_ok). + * + * If the inode has been found through an idmapped mount the idmap of + * the vfsmount must be passed through @idmap. This function will then + * take care to map the inode according to @idmap before checking + * permissions. On non-idmapped mounts or if permission checking is to be + * performed on the raw inode simply pass @nop_mnt_idmap. + * + * Called from set_acl inode operations. + */ +int posix_acl_update_mode(struct mnt_idmap *idmap, + struct inode *inode, umode_t *mode_p, + struct posix_acl **acl) +{ + umode_t mode = inode->i_mode; + int error; + + error = posix_acl_equiv_mode(*acl, &mode); + if (error < 0) + return error; + if (error == 0) + *acl = NULL; + if (!in_group_or_capable(idmap, inode, + i_gid_into_vfsgid(idmap, inode))) + mode &= ~S_ISGID; + *mode_p = mode; + return 0; +} +EXPORT_SYMBOL(posix_acl_update_mode); + +/* + * Fix up the uids and gids in posix acl extended attributes in place. + */ +static int posix_acl_fix_xattr_common(const void *value, size_t size) +{ + const struct posix_acl_xattr_header *header = value; + int count; + + if (!header) + return -EINVAL; + if (size < sizeof(struct posix_acl_xattr_header)) + return -EINVAL; + if (header->a_version != cpu_to_le32(POSIX_ACL_XATTR_VERSION)) + return -EOPNOTSUPP; + + count = posix_acl_xattr_count(size); + if (count < 0) + return -EINVAL; + + return count; +} + +/** + * posix_acl_from_xattr - convert POSIX ACLs from backing store to VFS format + * @userns: the filesystem's idmapping + * @value: the uapi representation of POSIX ACLs + * @size: the size of @void + * + * Filesystems that store POSIX ACLs in the unaltered uapi format should use + * posix_acl_from_xattr() when reading them from the backing store and + * converting them into the struct posix_acl VFS format. The helper is + * specifically intended to be called from the acl inode operation. + * + * The posix_acl_from_xattr() function will map the raw {g,u}id values stored + * in ACL_{GROUP,USER} entries into idmapping in @userns. + * + * Note that posix_acl_from_xattr() does not take idmapped mounts into account. + * If it did it calling it from the get acl inode operation would return POSIX + * ACLs mapped according to an idmapped mount which would mean that the value + * couldn't be cached for the filesystem. Idmapped mounts are taken into + * account on the fly during permission checking or right at the VFS - + * userspace boundary before reporting them to the user. + * + * Return: Allocated struct posix_acl on success, NULL for a valid header but + * without actual POSIX ACL entries, or ERR_PTR() encoded error code. + */ +struct posix_acl *posix_acl_from_xattr(struct user_namespace *userns, + const void *value, size_t size) +{ + const struct posix_acl_xattr_header *header = value; + const struct posix_acl_xattr_entry *entry = (const void *)(header + 1), *end; + int count; + struct posix_acl *acl; + struct posix_acl_entry *acl_e; + + count = posix_acl_fix_xattr_common(value, size); + if (count < 0) + return ERR_PTR(count); + if (count == 0) + return NULL; + + acl = posix_acl_alloc(count, GFP_NOFS); + if (!acl) + return ERR_PTR(-ENOMEM); + acl_e = acl->a_entries; + + for (end = entry + count; entry != end; acl_e++, entry++) { + acl_e->e_tag = le16_to_cpu(entry->e_tag); + acl_e->e_perm = le16_to_cpu(entry->e_perm); + + switch(acl_e->e_tag) { + case ACL_USER_OBJ: + case ACL_GROUP_OBJ: + case ACL_MASK: + case ACL_OTHER: + break; + + case ACL_USER: + acl_e->e_uid = make_kuid(userns, + le32_to_cpu(entry->e_id)); + if (!uid_valid(acl_e->e_uid)) + goto fail; + break; + case ACL_GROUP: + acl_e->e_gid = make_kgid(userns, + le32_to_cpu(entry->e_id)); + if (!gid_valid(acl_e->e_gid)) + goto fail; + break; + + default: + goto fail; + } + } + return acl; + +fail: + posix_acl_release(acl); + return ERR_PTR(-EINVAL); +} +EXPORT_SYMBOL (posix_acl_from_xattr); + +/* + * Convert from in-memory to extended attribute representation. + */ +void * +posix_acl_to_xattr(struct user_namespace *user_ns, const struct posix_acl *acl, + size_t *sizep, gfp_t gfp) +{ + struct posix_acl_xattr_header *ext_acl; + struct posix_acl_xattr_entry *ext_entry; + size_t size; + int n; + + size = posix_acl_xattr_size(acl->a_count); + ext_acl = kmalloc(size, gfp); + if (!ext_acl) + return NULL; + + ext_entry = (void *)(ext_acl + 1); + ext_acl->a_version = cpu_to_le32(POSIX_ACL_XATTR_VERSION); + + for (n=0; n < acl->a_count; n++, ext_entry++) { + const struct posix_acl_entry *acl_e = &acl->a_entries[n]; + ext_entry->e_tag = cpu_to_le16(acl_e->e_tag); + ext_entry->e_perm = cpu_to_le16(acl_e->e_perm); + switch(acl_e->e_tag) { + case ACL_USER: + ext_entry->e_id = + cpu_to_le32(from_kuid(user_ns, acl_e->e_uid)); + break; + case ACL_GROUP: + ext_entry->e_id = + cpu_to_le32(from_kgid(user_ns, acl_e->e_gid)); + break; + default: + ext_entry->e_id = cpu_to_le32(ACL_UNDEFINED_ID); + break; + } + } + *sizep = size; + return ext_acl; +} +EXPORT_SYMBOL (posix_acl_to_xattr); + +/** + * vfs_posix_acl_to_xattr - convert from kernel to userspace representation + * @idmap: idmap of the mount + * @inode: inode the posix acls are set on + * @acl: the posix acls as represented by the vfs + * @buffer: the buffer into which to convert @acl + * @size: size of @buffer + * + * This converts @acl from the VFS representation in the filesystem idmapping + * to the uapi form reportable to userspace. And mount and caller idmappings + * are handled appropriately. + * + * Return: On success, the size of the stored uapi posix acls, on error a + * negative errno. + */ +static ssize_t vfs_posix_acl_to_xattr(struct mnt_idmap *idmap, + struct inode *inode, + const struct posix_acl *acl, void *buffer, + size_t size) + +{ + struct posix_acl_xattr_header *ext_acl = buffer; + struct posix_acl_xattr_entry *ext_entry; + struct user_namespace *fs_userns, *caller_userns; + ssize_t real_size, n; + vfsuid_t vfsuid; + vfsgid_t vfsgid; + + real_size = posix_acl_xattr_size(acl->a_count); + if (!buffer) + return real_size; + if (real_size > size) + return -ERANGE; + + ext_entry = (void *)(ext_acl + 1); + ext_acl->a_version = cpu_to_le32(POSIX_ACL_XATTR_VERSION); + + fs_userns = i_user_ns(inode); + caller_userns = current_user_ns(); + for (n=0; n < acl->a_count; n++, ext_entry++) { + const struct posix_acl_entry *acl_e = &acl->a_entries[n]; + ext_entry->e_tag = cpu_to_le16(acl_e->e_tag); + ext_entry->e_perm = cpu_to_le16(acl_e->e_perm); + switch(acl_e->e_tag) { + case ACL_USER: + vfsuid = make_vfsuid(idmap, fs_userns, acl_e->e_uid); + ext_entry->e_id = cpu_to_le32(from_kuid( + caller_userns, vfsuid_into_kuid(vfsuid))); + break; + case ACL_GROUP: + vfsgid = make_vfsgid(idmap, fs_userns, acl_e->e_gid); + ext_entry->e_id = cpu_to_le32(from_kgid( + caller_userns, vfsgid_into_kgid(vfsgid))); + break; + default: + ext_entry->e_id = cpu_to_le32(ACL_UNDEFINED_ID); + break; + } + } + return real_size; +} + +int +set_posix_acl(struct mnt_idmap *idmap, struct dentry *dentry, + int type, struct posix_acl *acl) +{ + struct inode *inode = d_inode(dentry); + + if (!IS_POSIXACL(inode)) + return -EOPNOTSUPP; + if (!inode->i_op->set_acl) + return -EOPNOTSUPP; + + if (type == ACL_TYPE_DEFAULT && !S_ISDIR(inode->i_mode)) + return acl ? -EACCES : 0; + if (!inode_owner_or_capable(idmap, inode)) + return -EPERM; + + if (acl) { + int ret = posix_acl_valid(inode->i_sb->s_user_ns, acl); + if (ret) + return ret; + } + return inode->i_op->set_acl(idmap, dentry, acl, type); +} +EXPORT_SYMBOL(set_posix_acl); + +int posix_acl_listxattr(struct inode *inode, char **buffer, + ssize_t *remaining_size) +{ + int err; + + if (!IS_POSIXACL(inode)) + return 0; + + if (inode->i_acl) { + err = xattr_list_one(buffer, remaining_size, + XATTR_NAME_POSIX_ACL_ACCESS); + if (err) + return err; + } + + if (inode->i_default_acl) { + err = xattr_list_one(buffer, remaining_size, + XATTR_NAME_POSIX_ACL_DEFAULT); + if (err) + return err; + } + + return 0; +} + +static bool +posix_acl_xattr_list(struct dentry *dentry) +{ + return IS_POSIXACL(d_backing_inode(dentry)); +} + +/* + * nop_posix_acl_access - legacy xattr handler for access POSIX ACLs + * + * This is the legacy POSIX ACL access xattr handler. It is used by some + * filesystems to implement their ->listxattr() inode operation. New code + * should never use them. + */ +const struct xattr_handler nop_posix_acl_access = { + .name = XATTR_NAME_POSIX_ACL_ACCESS, + .list = posix_acl_xattr_list, +}; +EXPORT_SYMBOL_GPL(nop_posix_acl_access); + +/* + * nop_posix_acl_default - legacy xattr handler for default POSIX ACLs + * + * This is the legacy POSIX ACL default xattr handler. It is used by some + * filesystems to implement their ->listxattr() inode operation. New code + * should never use them. + */ +const struct xattr_handler nop_posix_acl_default = { + .name = XATTR_NAME_POSIX_ACL_DEFAULT, + .list = posix_acl_xattr_list, +}; +EXPORT_SYMBOL_GPL(nop_posix_acl_default); + +int simple_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type) +{ + int error; + struct inode *inode = d_inode(dentry); + + if (type == ACL_TYPE_ACCESS) { + error = posix_acl_update_mode(idmap, inode, + &inode->i_mode, &acl); + if (error) + return error; + } + + inode_set_ctime_current(inode); + if (IS_I_VERSION(inode)) + inode_inc_iversion(inode); + set_cached_acl(inode, type, acl); + return 0; +} + +int simple_acl_create(struct inode *dir, struct inode *inode) +{ + struct posix_acl *default_acl, *acl; + int error; + + error = posix_acl_create(dir, &inode->i_mode, &default_acl, &acl); + if (error) + return error; + + set_cached_acl(inode, ACL_TYPE_DEFAULT, default_acl); + set_cached_acl(inode, ACL_TYPE_ACCESS, acl); + + if (default_acl) + posix_acl_release(default_acl); + if (acl) + posix_acl_release(acl); + return 0; +} + +static int vfs_set_acl_idmapped_mnt(struct mnt_idmap *idmap, + struct user_namespace *fs_userns, + struct posix_acl *acl) +{ + for (int n = 0; n < acl->a_count; n++) { + struct posix_acl_entry *acl_e = &acl->a_entries[n]; + + switch (acl_e->e_tag) { + case ACL_USER: + acl_e->e_uid = from_vfsuid(idmap, fs_userns, + VFSUIDT_INIT(acl_e->e_uid)); + break; + case ACL_GROUP: + acl_e->e_gid = from_vfsgid(idmap, fs_userns, + VFSGIDT_INIT(acl_e->e_gid)); + break; + } + } + + return 0; +} + +/** + * vfs_set_acl - set posix acls + * @idmap: idmap of the mount + * @dentry: the dentry based on which to set the posix acls + * @acl_name: the name of the posix acl + * @kacl: the posix acls in the appropriate VFS format + * + * This function sets @kacl. The caller must all posix_acl_release() on @kacl + * afterwards. + * + * Return: On success 0, on error negative errno. + */ +int vfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + const char *acl_name, struct posix_acl *kacl) +{ + int acl_type; + int error; + struct inode *inode = d_inode(dentry); + struct delegated_inode delegated_inode = { }; + + acl_type = posix_acl_type(acl_name); + if (acl_type < 0) + return -EINVAL; + + if (kacl) { + /* + * If we're on an idmapped mount translate from mount specific + * vfs{g,u}id_t into global filesystem k{g,u}id_t. + * Afterwards we can cache the POSIX ACLs filesystem wide and - + * if this is a filesystem with a backing store - ultimately + * translate them to backing store values. + */ + error = vfs_set_acl_idmapped_mnt(idmap, i_user_ns(inode), kacl); + if (error) + return error; + } + +retry_deleg: + inode_lock(inode); + + /* + * We only care about restrictions the inode struct itself places upon + * us otherwise POSIX ACLs aren't subject to any VFS restrictions. + */ + error = may_write_xattr(idmap, inode); + if (error) + goto out_inode_unlock; + + error = security_inode_set_acl(idmap, dentry, acl_name, kacl); + if (error) + goto out_inode_unlock; + + error = try_break_deleg(inode, 0, &delegated_inode); + if (error) + goto out_inode_unlock; + + if (likely(!is_bad_inode(inode))) + error = set_posix_acl(idmap, dentry, acl_type, kacl); + else + error = -EIO; + if (!error) { + fsnotify_xattr(dentry); + security_inode_post_set_acl(dentry, acl_name, kacl); + } + +out_inode_unlock: + inode_unlock(inode); + + if (is_delegated(&delegated_inode)) { + error = break_deleg_wait(&delegated_inode); + if (!error) + goto retry_deleg; + } + + return error; +} +EXPORT_SYMBOL_GPL(vfs_set_acl); + +/** + * vfs_get_acl - get posix acls + * @idmap: idmap of the mount + * @dentry: the dentry based on which to retrieve the posix acls + * @acl_name: the name of the posix acl + * + * This function retrieves @kacl from the filesystem. The caller must all + * posix_acl_release() on @kacl. + * + * Return: On success POSIX ACLs in VFS format, on error negative errno. + */ +struct posix_acl *vfs_get_acl(struct mnt_idmap *idmap, + struct dentry *dentry, const char *acl_name) +{ + struct inode *inode = d_inode(dentry); + struct posix_acl *acl; + int acl_type, error; + + acl_type = posix_acl_type(acl_name); + if (acl_type < 0) + return ERR_PTR(-EINVAL); + + /* + * The VFS has no restrictions on reading POSIX ACLs so calling + * something like xattr_permission() isn't needed. Only LSMs get a say. + */ + error = security_inode_get_acl(idmap, dentry, acl_name); + if (error) + return ERR_PTR(error); + + if (!IS_POSIXACL(inode)) + return ERR_PTR(-EOPNOTSUPP); + if (S_ISLNK(inode->i_mode)) + return ERR_PTR(-EOPNOTSUPP); + + acl = __get_acl(idmap, dentry, inode, acl_type); + if (IS_ERR(acl)) + return acl; + if (!acl) + return ERR_PTR(-ENODATA); + + return acl; +} +EXPORT_SYMBOL_GPL(vfs_get_acl); + +/** + * vfs_remove_acl - remove posix acls + * @idmap: idmap of the mount + * @dentry: the dentry based on which to retrieve the posix acls + * @acl_name: the name of the posix acl + * + * This function removes posix acls. + * + * Return: On success 0, on error negative errno. + */ +int vfs_remove_acl(struct mnt_idmap *idmap, struct dentry *dentry, + const char *acl_name) +{ + int acl_type; + int error; + struct inode *inode = d_inode(dentry); + struct delegated_inode delegated_inode = { }; + + acl_type = posix_acl_type(acl_name); + if (acl_type < 0) + return -EINVAL; + +retry_deleg: + inode_lock(inode); + + /* + * We only care about restrictions the inode struct itself places upon + * us otherwise POSIX ACLs aren't subject to any VFS restrictions. + */ + error = may_write_xattr(idmap, inode); + if (error) + goto out_inode_unlock; + + error = security_inode_remove_acl(idmap, dentry, acl_name); + if (error) + goto out_inode_unlock; + + error = try_break_deleg(inode, 0, &delegated_inode); + if (error) + goto out_inode_unlock; + + if (likely(!is_bad_inode(inode))) + error = set_posix_acl(idmap, dentry, acl_type, NULL); + else + error = -EIO; + if (!error) { + fsnotify_xattr(dentry); + security_inode_post_remove_acl(idmap, dentry, acl_name); + } + +out_inode_unlock: + inode_unlock(inode); + + if (is_delegated(&delegated_inode)) { + error = break_deleg_wait(&delegated_inode); + if (!error) + goto retry_deleg; + } + + return error; +} +EXPORT_SYMBOL_GPL(vfs_remove_acl); + +int do_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + const char *acl_name, const void *kvalue, size_t size) +{ + int error; + struct posix_acl *acl = NULL; + + if (size) { + /* + * Note that posix_acl_from_xattr() uses GFP_NOFS when it + * probably doesn't need to here. + */ + acl = posix_acl_from_xattr(current_user_ns(), kvalue, size); + if (IS_ERR(acl)) + return PTR_ERR(acl); + } + + error = vfs_set_acl(idmap, dentry, acl_name, acl); + posix_acl_release(acl); + return error; +} + +ssize_t do_get_acl(struct mnt_idmap *idmap, struct dentry *dentry, + const char *acl_name, void *kvalue, size_t size) +{ + ssize_t error; + struct posix_acl *acl; + + acl = vfs_get_acl(idmap, dentry, acl_name); + if (IS_ERR(acl)) + return PTR_ERR(acl); + + error = vfs_posix_acl_to_xattr(idmap, d_inode(dentry), + acl, kvalue, size); + posix_acl_release(acl); + return error; +} diff --git a/fs/proc/Kconfig b/fs/proc/Kconfig new file mode 100644 index 000000000..24f5a36f4 --- /dev/null +++ b/fs/proc/Kconfig @@ -0,0 +1,128 @@ +# SPDX-License-Identifier: GPL-2.0-only +config PROC_FS + bool "/proc file system support" if EXPERT + default y + help + This is a virtual file system providing information about the status + of the system. "Virtual" means that it doesn't take up any space on + your hard disk: the files are created on the fly by the kernel when + you try to access them. Also, you cannot read the files with older + version of the program less: you need to use more or cat. + + It's totally cool; for example, "cat /proc/interrupts" gives + information about what the different IRQs are used for at the moment + (there is a small number of Interrupt ReQuest lines in your computer + that are used by the attached devices to gain the CPU's attention -- + often a source of trouble if two devices are mistakenly configured + to use the same IRQ). The program procinfo to display some + information about your system gathered from the /proc file system. + + Before you can use the /proc file system, it has to be mounted, + meaning it has to be given a location in the directory hierarchy. + That location should be /proc. A command such as "mount -t proc proc + /proc" or the equivalent line in /etc/fstab does the job. + + The /proc file system is explained in the file + and on the proc(5) manpage + ("man 5 proc"). + + This option will enlarge your kernel by about 67 KB. Several + programs depend on this, so everyone should say Y here. + +config PROC_KCORE + bool "/proc/kcore support" if !ARM + depends on PROC_FS && MMU + select VMCORE_INFO + help + Provides a virtual ELF core file of the live kernel. This can + be read with gdb and other ELF tools. No modifications can be + made using this mechanism. + +config PROC_VMCORE + bool "/proc/vmcore support" + depends on PROC_FS && CRASH_DUMP + default y + help + Exports the dump image of crashed kernel in ELF format. + +config PROC_VMCORE_DEVICE_DUMP + bool "Device Hardware/Firmware Log Collection" + depends on PROC_VMCORE + default n + help + After kernel panic, device drivers can collect the device + specific snapshot of their hardware or firmware before the + underlying devices are initialized in crash recovery kernel. + Note that the device driver must be present in the crash + recovery kernel's initramfs to collect its underlying device + snapshot. + + If you say Y here, the collected device dumps will be added + as ELF notes to /proc/vmcore. You can still disable device + dump using the kernel command line option 'novmcoredd'. + +config NEED_PROC_VMCORE_DEVICE_RAM + bool + +config PROC_VMCORE_DEVICE_RAM + def_bool y + depends on PROC_VMCORE && NEED_PROC_VMCORE_DEVICE_RAM + depends on VIRTIO_MEM + help + If the elfcore hdr is allocated and prepared by the dump kernel + ("2nd kernel") instead of the crashed kernel, RAM provided by memory + devices such as virtio-mem will not be included in the dump + image, because only the device driver can properly detect them. + + With this config enabled, these RAM ranges will be queried from the + device drivers once the device gets probed, so they can be included + in the crash dump. + + Relevant architectures should select NEED_PROC_VMCORE_DEVICE_RAM. + +config SYSCTL + bool "Sysctl support (/proc/sys)" if EXPERT + depends on PROC_FS + default y + help + The sysctl interface provides a means of dynamically changing + certain kernel parameters and variables on the fly without requiring + a recompile of the kernel or reboot of the system. The primary + interface is through /proc/sys. If you say Y here a tree of + modifiable sysctl entries will be generated beneath the + /proc/sys directory. They are explained in the files + in . Note that enabling this + option will enlarge the kernel by at least 8 KB. + + As it is generally a good thing, you should say Y here unless + building a kernel for install/rescue disks or your system is very + limited in memory. + +config PROC_PAGE_MONITOR + default y + depends on PROC_FS && MMU + bool "Enable /proc page monitoring" if EXPERT + help + Various /proc files exist to monitor process memory utilization: + /proc/pid/smaps, /proc/pid/clear_refs, /proc/pid/pagemap, + /proc/kpagecount, and /proc/kpageflags. Disabling these + interfaces will reduce the size of the kernel by approximately 4kb. + +config PROC_CHILDREN + bool "Include /proc//task//children file" + depends on PROC_FS + default n + help + Provides a fast way to retrieve first level children pids of a task. See + for more information. + + Say Y if you are running any user-space software which takes benefit from + this interface. For example, rkt is such a piece of software. + +config PROC_PID_ARCH_STATUS + def_bool n + depends on PROC_FS + +config PROC_CPU_RESCTRL + def_bool n + depends on PROC_FS diff --git a/fs/proc/Makefile b/fs/proc/Makefile new file mode 100644 index 000000000..27e2b26b2 --- /dev/null +++ b/fs/proc/Makefile @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the Linux proc filesystem routines. +# + +obj-y += proc.o + +CFLAGS_task_mmu.o += -Wno-override-init +proc-y := nommu.o task_nommu.o +proc-$(CONFIG_MMU) := task_mmu.o + +proc-y += inode.o root.o base.o generic.o array.o \ + fd.o +proc-$(CONFIG_TTY) += proc_tty.o +proc-y += cmdline.o +proc-y += consoles.o +proc-y += cpuinfo.o +proc-y += devices.o +ifneq ($(CONFIG_GENERIC_IRQ_SHOW),y) +proc-y += interrupts.o +endif +proc-y += loadavg.o +proc-y += meminfo.o +proc-y += stat.o +proc-y += uptime.o +proc-y += util.o +proc-y += version.o +proc-y += softirqs.o +proc-y += namespaces.o +proc-y += self.o +proc-y += thread_self.o +proc-$(CONFIG_SYSCTL) += proc_sysctl.o +proc-$(CONFIG_NET) += proc_net.o +proc-$(CONFIG_PROC_KCORE) += kcore.o +proc-$(CONFIG_PROC_VMCORE) += vmcore.o +proc-$(CONFIG_PRINTK) += kmsg.o +proc-$(CONFIG_PROC_PAGE_MONITOR) += page.o +proc-$(CONFIG_BOOT_CONFIG) += bootconfig.o diff --git a/fs/proc/array.c b/fs/proc/array.c new file mode 100644 index 000000000..f6f75d206 --- /dev/null +++ b/fs/proc/array.c @@ -0,0 +1,822 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/proc/array.c + * + * Copyright (C) 1992 by Linus Torvalds + * based on ideas by Darren Senn + * + * Fixes: + * Michael. K. Johnson: stat,statm extensions. + * + * + * Pauline Middelink : Made cmdline,envline only break at '\0's, to + * make sure SET_PROCTITLE works. Also removed + * bad '!' which forced address recalculation for + * EVERY character on the current page. + * + * + * Danny ter Haar : added cpuinfo + * + * + * Alessandro Rubini : profile extension. + * + * + * Jeff Tranter : added BogoMips field to cpuinfo + * + * + * Bruno Haible : remove 4K limit for the maps file + * + * + * Yves Arrouye : remove removal of trailing spaces in get_array. + * + * + * Jerome Forissier : added per-CPU time information to /proc/stat + * and /proc//cpu extension + * + * - Incorporation and non-SMP safe operation + * of forissier patch in 2.1.78 by + * Hans Marcus + * + * aeb@cwi.nl : /proc/partitions + * + * + * Alan Cox : security fixes. + * + * + * Al Viro : safe handling of mm_struct + * + * Gerhard Wichert : added BIGMEM support + * Siemens AG + * + * Al Viro & Jeff Garzik : moved most of the thing into base.c and + * : proc_misc.c. The rest may eventually go into + * : base.c too. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "internal.h" + +void proc_task_name(struct seq_file *m, struct task_struct *p, bool escape) +{ + char tcomm[64]; + + /* + * Test before PF_KTHREAD because all workqueue worker threads are + * kernel threads. + */ + if (p->flags & PF_WQ_WORKER) + wq_worker_comm(tcomm, sizeof(tcomm), p); + else if (p->flags & PF_KTHREAD) + get_kthread_comm(tcomm, sizeof(tcomm), p); + else + get_task_comm(tcomm, p); + + if (escape) + seq_escape_str(m, tcomm, ESCAPE_SPACE | ESCAPE_SPECIAL, "\n\\"); + else + seq_printf(m, "%.64s", tcomm); +} + +/* + * The task state array is a strange "bitmap" of + * reasons to sleep. Thus "running" is zero, and + * you can test for combinations of others with + * simple bit tests. + */ +static const char * const task_state_array[] = { + + /* states in TASK_REPORT: */ + "R (running)", /* 0x00 */ + "S (sleeping)", /* 0x01 */ + "D (disk sleep)", /* 0x02 */ + "T (stopped)", /* 0x04 */ + "t (tracing stop)", /* 0x08 */ + "X (dead)", /* 0x10 */ + "Z (zombie)", /* 0x20 */ + "P (parked)", /* 0x40 */ + + /* states beyond TASK_REPORT: */ + "I (idle)", /* 0x80 */ +}; + +static inline const char *get_task_state(struct task_struct *tsk) +{ + BUILD_BUG_ON(1 + ilog2(TASK_REPORT_MAX) != ARRAY_SIZE(task_state_array)); + return task_state_array[task_state_index(tsk)]; +} + +static inline void task_state(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *p) +{ + struct user_namespace *user_ns = seq_user_ns(m); + struct group_info *group_info; + int g, umask = -1; + struct task_struct *tracer; + const struct cred *cred; + pid_t ppid, tpid = 0, tgid, ngid; + unsigned int max_fds = 0; + + rcu_read_lock(); + tracer = ptrace_parent(p); + if (tracer) + tpid = task_pid_nr_ns(tracer, ns); + + ppid = task_ppid_nr_ns(p, ns); + tgid = task_tgid_nr_ns(p, ns); + ngid = task_numa_group_id(p); + cred = get_task_cred(p); + + task_lock(p); + if (p->real_fs) + umask = p->real_fs->umask; + if (p->files) + max_fds = files_fdtable(p->files)->max_fds; + task_unlock(p); + rcu_read_unlock(); + + if (umask >= 0) + seq_printf(m, "Umask:\t%#04o\n", umask); + seq_puts(m, "State:\t"); + seq_puts(m, get_task_state(p)); + + seq_put_decimal_ull(m, "\nTgid:\t", tgid); + seq_put_decimal_ull(m, "\nNgid:\t", ngid); + seq_put_decimal_ull(m, "\nPid:\t", pid_nr_ns(pid, ns)); + seq_put_decimal_ull(m, "\nPPid:\t", ppid); + seq_put_decimal_ull(m, "\nTracerPid:\t", tpid); + seq_put_decimal_ull(m, "\nUid:\t", from_kuid_munged(user_ns, cred->uid)); + seq_put_decimal_ull(m, "\t", from_kuid_munged(user_ns, cred->euid)); + seq_put_decimal_ull(m, "\t", from_kuid_munged(user_ns, cred->suid)); + seq_put_decimal_ull(m, "\t", from_kuid_munged(user_ns, cred->fsuid)); + seq_put_decimal_ull(m, "\nGid:\t", from_kgid_munged(user_ns, cred->gid)); + seq_put_decimal_ull(m, "\t", from_kgid_munged(user_ns, cred->egid)); + seq_put_decimal_ull(m, "\t", from_kgid_munged(user_ns, cred->sgid)); + seq_put_decimal_ull(m, "\t", from_kgid_munged(user_ns, cred->fsgid)); + seq_put_decimal_ull(m, "\nFDSize:\t", max_fds); + + seq_puts(m, "\nGroups:\t"); + group_info = cred->group_info; + for (g = 0; g < group_info->ngroups; g++) + seq_put_decimal_ull(m, g ? " " : "", + from_kgid_munged(user_ns, group_info->gid[g])); + put_cred(cred); + /* Trailing space shouldn't have been added in the first place. */ + seq_putc(m, ' '); + +#ifdef CONFIG_PID_NS + seq_puts(m, "\nNStgid:"); + for (g = ns->level; g <= pid->level; g++) + seq_put_decimal_ull(m, "\t", task_tgid_nr_ns(p, pid->numbers[g].ns)); + seq_puts(m, "\nNSpid:"); + for (g = ns->level; g <= pid->level; g++) + seq_put_decimal_ull(m, "\t", task_pid_nr_ns(p, pid->numbers[g].ns)); + seq_puts(m, "\nNSpgid:"); + for (g = ns->level; g <= pid->level; g++) + seq_put_decimal_ull(m, "\t", task_pgrp_nr_ns(p, pid->numbers[g].ns)); + seq_puts(m, "\nNSsid:"); + for (g = ns->level; g <= pid->level; g++) + seq_put_decimal_ull(m, "\t", task_session_nr_ns(p, pid->numbers[g].ns)); +#endif + seq_putc(m, '\n'); + + seq_printf(m, "Kthread:\t%c\n", p->flags & PF_KTHREAD ? '1' : '0'); +} + +void render_sigset_t(struct seq_file *m, const char *header, + sigset_t *set) +{ + int i; + + seq_puts(m, header); + + i = _NSIG; + do { + int x = 0; + + i -= 4; + if (sigismember(set, i+1)) x |= 1; + if (sigismember(set, i+2)) x |= 2; + if (sigismember(set, i+3)) x |= 4; + if (sigismember(set, i+4)) x |= 8; + seq_putc(m, hex_asc[x]); + } while (i >= 4); + + seq_putc(m, '\n'); +} + +static void collect_sigign_sigcatch(struct task_struct *p, sigset_t *sigign, + sigset_t *sigcatch) +{ + struct k_sigaction *k; + int i; + + k = p->sighand->action; + for (i = 1; i <= _NSIG; ++i, ++k) { + if (k->sa.sa_handler == SIG_IGN) + sigaddset(sigign, i); + else if (k->sa.sa_handler != SIG_DFL) + sigaddset(sigcatch, i); + } +} + +static inline void task_sig(struct seq_file *m, struct task_struct *p) +{ + unsigned long flags; + sigset_t pending, shpending, blocked, ignored, caught; + int num_threads = 0; + unsigned int qsize = 0; + unsigned long qlim = 0; + + sigemptyset(&pending); + sigemptyset(&shpending); + sigemptyset(&blocked); + sigemptyset(&ignored); + sigemptyset(&caught); + + if (lock_task_sighand(p, &flags)) { + pending = p->pending.signal; + shpending = p->signal->shared_pending.signal; + blocked = p->blocked; + collect_sigign_sigcatch(p, &ignored, &caught); + num_threads = get_nr_threads(p); + rcu_read_lock(); + qsize = get_rlimit_value(task_ucounts(p), UCOUNT_RLIMIT_SIGPENDING); + rcu_read_unlock(); + qlim = task_rlimit(p, RLIMIT_SIGPENDING); + unlock_task_sighand(p, &flags); + } + + seq_put_decimal_ull(m, "Threads:\t", num_threads); + seq_put_decimal_ull(m, "\nSigQ:\t", qsize); + seq_put_decimal_ull(m, "/", qlim); + + /* render them all */ + render_sigset_t(m, "\nSigPnd:\t", &pending); + render_sigset_t(m, "ShdPnd:\t", &shpending); + render_sigset_t(m, "SigBlk:\t", &blocked); + render_sigset_t(m, "SigIgn:\t", &ignored); + render_sigset_t(m, "SigCgt:\t", &caught); +} + +static void render_cap_t(struct seq_file *m, const char *header, + kernel_cap_t *a) +{ + seq_puts(m, header); + seq_put_hex_ll(m, NULL, a->val, 16); + seq_putc(m, '\n'); +} + +static inline void task_cap(struct seq_file *m, struct task_struct *p) +{ + const struct cred *cred; + kernel_cap_t cap_inheritable, cap_permitted, cap_effective, + cap_bset, cap_ambient; + + rcu_read_lock(); + cred = __task_cred(p); + cap_inheritable = cred->cap_inheritable; + cap_permitted = cred->cap_permitted; + cap_effective = cred->cap_effective; + cap_bset = cred->cap_bset; + cap_ambient = cred->cap_ambient; + rcu_read_unlock(); + + render_cap_t(m, "CapInh:\t", &cap_inheritable); + render_cap_t(m, "CapPrm:\t", &cap_permitted); + render_cap_t(m, "CapEff:\t", &cap_effective); + render_cap_t(m, "CapBnd:\t", &cap_bset); + render_cap_t(m, "CapAmb:\t", &cap_ambient); +} + +static inline void task_seccomp(struct seq_file *m, struct task_struct *p) +{ + seq_put_decimal_ull(m, "NoNewPrivs:\t", task_no_new_privs(p)); +#ifdef CONFIG_SECCOMP + seq_put_decimal_ull(m, "\nSeccomp:\t", p->seccomp.mode); +#ifdef CONFIG_SECCOMP_FILTER + seq_put_decimal_ull(m, "\nSeccomp_filters:\t", + atomic_read(&p->seccomp.filter_count)); +#endif +#endif + seq_puts(m, "\nSpeculation_Store_Bypass:\t"); + switch (arch_prctl_spec_ctrl_get(p, PR_SPEC_STORE_BYPASS)) { + case -EINVAL: + seq_puts(m, "unknown"); + break; + case PR_SPEC_NOT_AFFECTED: + seq_puts(m, "not vulnerable"); + break; + case PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE: + seq_puts(m, "thread force mitigated"); + break; + case PR_SPEC_PRCTL | PR_SPEC_DISABLE: + seq_puts(m, "thread mitigated"); + break; + case PR_SPEC_PRCTL | PR_SPEC_ENABLE: + seq_puts(m, "thread vulnerable"); + break; + case PR_SPEC_DISABLE: + seq_puts(m, "globally mitigated"); + break; + default: + seq_puts(m, "vulnerable"); + break; + } + + seq_puts(m, "\nSpeculationIndirectBranch:\t"); + switch (arch_prctl_spec_ctrl_get(p, PR_SPEC_INDIRECT_BRANCH)) { + case -EINVAL: + seq_puts(m, "unsupported"); + break; + case PR_SPEC_NOT_AFFECTED: + seq_puts(m, "not affected"); + break; + case PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE: + seq_puts(m, "conditional force disabled"); + break; + case PR_SPEC_PRCTL | PR_SPEC_DISABLE: + seq_puts(m, "conditional disabled"); + break; + case PR_SPEC_PRCTL | PR_SPEC_ENABLE: + seq_puts(m, "conditional enabled"); + break; + case PR_SPEC_ENABLE: + seq_puts(m, "always enabled"); + break; + case PR_SPEC_DISABLE: + seq_puts(m, "always disabled"); + break; + default: + seq_puts(m, "unknown"); + break; + } + seq_putc(m, '\n'); +} + +static inline void task_context_switch_counts(struct seq_file *m, + struct task_struct *p) +{ + seq_put_decimal_ull(m, "voluntary_ctxt_switches:\t", p->nvcsw); + seq_put_decimal_ull(m, "\nnonvoluntary_ctxt_switches:\t", p->nivcsw); + seq_putc(m, '\n'); +} + +static void task_cpus_allowed(struct seq_file *m, struct task_struct *task) +{ + seq_printf(m, "Cpus_allowed:\t%*pb\n", + cpumask_pr_args(&task->cpus_mask)); + seq_printf(m, "Cpus_allowed_list:\t%*pbl\n", + cpumask_pr_args(&task->cpus_mask)); +} + +static inline void task_core_dumping(struct seq_file *m, struct task_struct *task) +{ + seq_put_decimal_ull(m, "CoreDumping:\t", !!task->signal->core_state); + seq_putc(m, '\n'); +} + +static inline void task_thp_status(struct seq_file *m, struct mm_struct *mm) +{ + bool thp_enabled = IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE); + + if (thp_enabled) + thp_enabled = !mm_flags_test(MMF_DISABLE_THP_COMPLETELY, mm); + seq_printf(m, "THP_enabled:\t%d\n", thp_enabled); +} + +static inline void task_untag_mask(struct seq_file *m, struct mm_struct *mm) +{ + seq_printf(m, "untag_mask:\t%#lx\n", mm_untag_mask(mm)); +} + +__weak void arch_proc_pid_thread_features(struct seq_file *m, + struct task_struct *task) +{ +} + +int proc_pid_status(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *task) +{ + struct mm_struct *mm = get_task_mm(task); + + seq_puts(m, "Name:\t"); + proc_task_name(m, task, true); + seq_putc(m, '\n'); + + task_state(m, ns, pid, task); + + if (mm) { + task_mem(m, mm); + task_core_dumping(m, task); + task_thp_status(m, mm); + task_untag_mask(m, mm); + mmput(mm); + } + task_sig(m, task); + task_cap(m, task); + task_seccomp(m, task); + task_cpus_allowed(m, task); + cpuset_task_status_allowed(m, task); + task_context_switch_counts(m, task); + arch_proc_pid_thread_features(m, task); + return 0; +} + +static int do_task_stat(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *task, int whole) +{ + unsigned long vsize, eip, esp, wchan = 0; + int priority, nice; + int tty_pgrp = -1, tty_nr = 0; + sigset_t sigign, sigcatch; + char state; + pid_t ppid = 0, pgid = -1, sid = -1; + int num_threads = 0; + int permitted; + struct mm_struct *mm; + unsigned long long start_time; + unsigned long cmin_flt, cmaj_flt, min_flt, maj_flt; + u64 cutime, cstime, cgtime, utime, stime, gtime; + unsigned long rsslim = 0; + unsigned long flags; + int exit_code = task->exit_code; + struct signal_struct *sig = task->signal; + int ret; + + ret = down_read_killable(&task->signal->exec_update_lock); + if (ret) + return ret; + + state = *get_task_state(task); + vsize = eip = esp = 0; + permitted = ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS | PTRACE_MODE_NOAUDIT); + mm = get_task_mm(task); + if (mm) { + vsize = task_vsize(mm); + /* + * esp and eip are intentionally zeroed out. There is no + * non-racy way to read them without freezing the task. + * Programs that need reliable values can use ptrace(2). + * + * The only exception is if the task is core dumping because + * a program is not able to use ptrace(2) in that case. It is + * safe because the task has stopped executing permanently. + */ + if (permitted && (task->flags & (PF_EXITING|PF_DUMPCORE|PF_POSTCOREDUMP))) { + if (try_get_task_stack(task)) { + eip = KSTK_EIP(task); + esp = KSTK_ESP(task); + put_task_stack(task); + } + } + } + + sigemptyset(&sigign); + sigemptyset(&sigcatch); + + if (lock_task_sighand(task, &flags)) { + if (sig->tty) { + struct pid *pgrp = tty_get_pgrp(sig->tty); + tty_pgrp = pid_nr_ns(pgrp, ns); + put_pid(pgrp); + tty_nr = new_encode_dev(tty_devnum(sig->tty)); + } + + num_threads = get_nr_threads(task); + collect_sigign_sigcatch(task, &sigign, &sigcatch); + + rsslim = READ_ONCE(sig->rlim[RLIMIT_RSS].rlim_cur); + + if (whole) { + if (sig->flags & (SIGNAL_GROUP_EXIT | SIGNAL_STOP_STOPPED)) + exit_code = sig->group_exit_code; + } + + sid = task_session_nr_ns(task, ns); + ppid = task_ppid_nr_ns(task, ns); + pgid = task_pgrp_nr_ns(task, ns); + + unlock_task_sighand(task, &flags); + } + + if (permitted && (!whole || num_threads < 2)) + wchan = !task_is_running(task); + + scoped_guard(rcu) { + scoped_seqlock_read (&sig->stats_lock, ss_lock_irqsave) { + cmin_flt = sig->cmin_flt; + cmaj_flt = sig->cmaj_flt; + cutime = sig->cutime; + cstime = sig->cstime; + cgtime = sig->cgtime; + + if (whole) { + struct task_struct *t; + + min_flt = sig->min_flt; + maj_flt = sig->maj_flt; + gtime = sig->gtime; + + __for_each_thread(sig, t) { + min_flt += t->min_flt; + maj_flt += t->maj_flt; + gtime += task_gtime(t); + } + } + } + } + + if (whole) { + thread_group_cputime_adjusted(task, &utime, &stime); + } else { + task_cputime_adjusted(task, &utime, &stime); + min_flt = task->min_flt; + maj_flt = task->maj_flt; + gtime = task_gtime(task); + } + + /* scale priority and nice values from timeslices to -20..20 */ + /* to make it look like a "normal" Unix priority/nice value */ + priority = task_prio(task); + nice = task_nice(task); + + /* apply timens offset for boottime and convert nsec -> ticks */ + start_time = + nsec_to_clock_t(timens_add_boottime_ns(task->start_boottime)); + + seq_put_decimal_ull(m, "", pid_nr_ns(pid, ns)); + seq_puts(m, " ("); + proc_task_name(m, task, false); + seq_puts(m, ") "); + seq_putc(m, state); + seq_put_decimal_ll(m, " ", ppid); + seq_put_decimal_ll(m, " ", pgid); + seq_put_decimal_ll(m, " ", sid); + seq_put_decimal_ll(m, " ", tty_nr); + seq_put_decimal_ll(m, " ", tty_pgrp); + seq_put_decimal_ull(m, " ", task->flags); + seq_put_decimal_ull(m, " ", min_flt); + seq_put_decimal_ull(m, " ", cmin_flt); + seq_put_decimal_ull(m, " ", maj_flt); + seq_put_decimal_ull(m, " ", cmaj_flt); + seq_put_decimal_ull(m, " ", nsec_to_clock_t(utime)); + seq_put_decimal_ull(m, " ", nsec_to_clock_t(stime)); + seq_put_decimal_ll(m, " ", nsec_to_clock_t(cutime)); + seq_put_decimal_ll(m, " ", nsec_to_clock_t(cstime)); + seq_put_decimal_ll(m, " ", priority); + seq_put_decimal_ll(m, " ", nice); + seq_put_decimal_ll(m, " ", num_threads); + seq_put_decimal_ull(m, " ", 0); + seq_put_decimal_ull(m, " ", start_time); + seq_put_decimal_ull(m, " ", vsize); + seq_put_decimal_ull(m, " ", mm ? get_mm_rss(mm) : 0); + seq_put_decimal_ull(m, " ", rsslim); + seq_put_decimal_ull(m, " ", mm ? (permitted ? mm->start_code : 1) : 0); + seq_put_decimal_ull(m, " ", mm ? (permitted ? mm->end_code : 1) : 0); + seq_put_decimal_ull(m, " ", (permitted && mm) ? mm->start_stack : 0); + seq_put_decimal_ull(m, " ", esp); + seq_put_decimal_ull(m, " ", eip); + /* The signal information here is obsolete. + * It must be decimal for Linux 2.0 compatibility. + * Use /proc/#/status for real-time signals. + */ + seq_put_decimal_ull(m, " ", task->pending.signal.sig[0] & 0x7fffffffUL); + seq_put_decimal_ull(m, " ", task->blocked.sig[0] & 0x7fffffffUL); + seq_put_decimal_ull(m, " ", sigign.sig[0] & 0x7fffffffUL); + seq_put_decimal_ull(m, " ", sigcatch.sig[0] & 0x7fffffffUL); + + /* + * We used to output the absolute kernel address, but that's an + * information leak - so instead we show a 0/1 flag here, to signal + * to user-space whether there's a wchan field in /proc/PID/wchan. + * + * This works with older implementations of procps as well. + */ + seq_put_decimal_ull(m, " ", wchan); + + seq_put_decimal_ull(m, " ", 0); + seq_put_decimal_ull(m, " ", 0); + seq_put_decimal_ll(m, " ", task->exit_signal); + seq_put_decimal_ll(m, " ", task_cpu(task)); + seq_put_decimal_ull(m, " ", task->rt_priority); + seq_put_decimal_ull(m, " ", task->policy); + seq_put_decimal_ull(m, " ", delayacct_blkio_ticks(task)); + seq_put_decimal_ull(m, " ", nsec_to_clock_t(gtime)); + seq_put_decimal_ll(m, " ", nsec_to_clock_t(cgtime)); + + if (mm && permitted) { + seq_put_decimal_ull(m, " ", mm->start_data); + seq_put_decimal_ull(m, " ", mm->end_data); + seq_put_decimal_ull(m, " ", mm->start_brk); + seq_put_decimal_ull(m, " ", mm->arg_start); + seq_put_decimal_ull(m, " ", mm->arg_end); + seq_put_decimal_ull(m, " ", mm->env_start); + seq_put_decimal_ull(m, " ", mm->env_end); + } else + seq_puts(m, " 0 0 0 0 0 0 0"); + + if (permitted) + seq_put_decimal_ll(m, " ", exit_code); + else + seq_puts(m, " 0"); + + seq_putc(m, '\n'); + up_read(&task->signal->exec_update_lock); + if (mm) + mmput(mm); + return 0; +} + +int proc_tid_stat(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *task) +{ + return do_task_stat(m, ns, pid, task, 0); +} + +int proc_tgid_stat(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *task) +{ + return do_task_stat(m, ns, pid, task, 1); +} + +int proc_pid_statm(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *task) +{ + struct mm_struct *mm = get_task_mm(task); + + if (mm) { + unsigned long size; + unsigned long resident = 0; + unsigned long shared = 0; + unsigned long text = 0; + unsigned long data = 0; + + size = task_statm(mm, &shared, &text, &data, &resident); + mmput(mm); + + /* + * For quick read, open code by putting numbers directly + * expected format is + * seq_printf(m, "%lu %lu %lu %lu 0 %lu 0\n", + * size, resident, shared, text, data); + */ + seq_put_decimal_ull(m, "", size); + seq_put_decimal_ull(m, " ", resident); + seq_put_decimal_ull(m, " ", shared); + seq_put_decimal_ull(m, " ", text); + seq_put_decimal_ull(m, " ", 0); + seq_put_decimal_ull(m, " ", data); + seq_put_decimal_ull(m, " ", 0); + seq_putc(m, '\n'); + } else { + seq_write(m, "0 0 0 0 0 0 0\n", 14); + } + return 0; +} + +#ifdef CONFIG_PROC_CHILDREN +static struct pid * +get_children_pid(struct inode *inode, struct pid *pid_prev, loff_t pos) +{ + struct task_struct *start, *task; + struct pid *pid = NULL; + + read_lock(&tasklist_lock); + + start = pid_task(proc_pid(inode), PIDTYPE_PID); + if (!start) + goto out; + + /* + * Lets try to continue searching first, this gives + * us significant speedup on children-rich processes. + */ + if (pid_prev) { + task = pid_task(pid_prev, PIDTYPE_PID); + if (task && task->real_parent == start && + !(list_empty(&task->sibling))) { + if (list_is_last(&task->sibling, &start->children)) + goto out; + task = list_first_entry(&task->sibling, + struct task_struct, sibling); + pid = get_pid(task_pid(task)); + goto out; + } + } + + /* + * Slow search case. + * + * We might miss some children here if children + * are exited while we were not holding the lock, + * but it was never promised to be accurate that + * much. + * + * "Just suppose that the parent sleeps, but N children + * exit after we printed their tids. Now the slow paths + * skips N extra children, we miss N tasks." (c) + * + * So one need to stop or freeze the leader and all + * its children to get a precise result. + */ + list_for_each_entry(task, &start->children, sibling) { + if (pos-- == 0) { + pid = get_pid(task_pid(task)); + break; + } + } + +out: + read_unlock(&tasklist_lock); + return pid; +} + +static int children_seq_show(struct seq_file *seq, void *v) +{ + struct inode *inode = file_inode(seq->file); + + seq_printf(seq, "%d ", pid_nr_ns(v, proc_pid_ns(inode->i_sb))); + return 0; +} + +static void *children_seq_start(struct seq_file *seq, loff_t *pos) +{ + return get_children_pid(file_inode(seq->file), NULL, *pos); +} + +static void *children_seq_next(struct seq_file *seq, void *v, loff_t *pos) +{ + struct pid *pid; + + pid = get_children_pid(file_inode(seq->file), v, *pos + 1); + put_pid(v); + + ++*pos; + return pid; +} + +static void children_seq_stop(struct seq_file *seq, void *v) +{ + put_pid(v); +} + +static const struct seq_operations children_seq_ops = { + .start = children_seq_start, + .next = children_seq_next, + .stop = children_seq_stop, + .show = children_seq_show, +}; + +static int children_seq_open(struct inode *inode, struct file *file) +{ + return seq_open(file, &children_seq_ops); +} + +const struct file_operations proc_tid_children_operations = { + .open = children_seq_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; +#endif /* CONFIG_PROC_CHILDREN */ diff --git a/fs/proc/base.c b/fs/proc/base.c new file mode 100644 index 000000000..6a39de424 --- /dev/null +++ b/fs/proc/base.c @@ -0,0 +1,3990 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/proc/base.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * proc base directory handling functions + * + * 1999, Al Viro. Rewritten. Now it covers the whole per-process part. + * Instead of using magical inumbers to determine the kind of object + * we allocate and fill in-core inodes upon lookup. They don't even + * go into icache. We cache the reference to task_struct upon lookup too. + * Eventually it should become a filesystem in its own. We don't use the + * rest of procfs anymore. + * + * + * Changelog: + * 17-Jan-2005 + * Allan Bezerra + * Bruna Moreira + * Edjard Mota + * Ilias Biris + * Mauricio Lin + * + * Embedded Linux Lab - 10LE Instituto Nokia de Tecnologia - INdT + * + * A new process specific entry (smaps) included in /proc. It shows the + * size of rss for each memory area. The maps entry lacks information + * about physical memory size (rss) for each mapped file, i.e., + * rss information for executables and library files. + * This additional information is useful for any tools that need to know + * about physical memory consumption for a process specific library. + * + * Changelog: + * 21-Feb-2005 + * Embedded Linux Lab - 10LE Instituto Nokia de Tecnologia - INdT + * Pud inclusion in the page table walking. + * + * ChangeLog: + * 10-Mar-2005 + * 10LE Instituto Nokia de Tecnologia - INdT: + * A better way to walks through the page table as suggested by Hugh Dickins. + * + * Simo Piiroinen : + * Smaps information related to shared, private, clean and dirty pages. + * + * Paul Mundt : + * Overall revision about smaps. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" +#include "fd.h" + +#include "../../lib/kstrtox.h" + +/* NOTE: + * Implementing inode permission operations in /proc is almost + * certainly an error. Permission checks need to happen during + * each system call not at open time. The reason is that most of + * what we wish to check for permissions in /proc varies at runtime. + * + * The classic example of a problem is opening file descriptors + * in /proc for a task before it execs a suid executable. + */ + +static u8 nlink_tid __ro_after_init; +static u8 nlink_tgid __ro_after_init; + +enum proc_mem_force { + PROC_MEM_FORCE_ALWAYS, + PROC_MEM_FORCE_PTRACE, + PROC_MEM_FORCE_NEVER +}; + +static enum proc_mem_force proc_mem_force_override __ro_after_init = + IS_ENABLED(CONFIG_PROC_MEM_NO_FORCE) ? PROC_MEM_FORCE_NEVER : + IS_ENABLED(CONFIG_PROC_MEM_FORCE_PTRACE) ? PROC_MEM_FORCE_PTRACE : + PROC_MEM_FORCE_ALWAYS; + +static const struct constant_table proc_mem_force_table[] __initconst = { + { "always", PROC_MEM_FORCE_ALWAYS }, + { "ptrace", PROC_MEM_FORCE_PTRACE }, + { "never", PROC_MEM_FORCE_NEVER }, + { } +}; + +static int __init early_proc_mem_force_override(char *buf) +{ + if (!buf) + return -EINVAL; + + /* + * lookup_constant() defaults to proc_mem_force_override to preseve + * the initial Kconfig choice in case an invalid param gets passed. + */ + proc_mem_force_override = lookup_constant(proc_mem_force_table, + buf, proc_mem_force_override); + + return 0; +} +early_param("proc_mem.force_override", early_proc_mem_force_override); + +struct pid_entry { + const char *name; + unsigned int len; + umode_t mode; + const struct inode_operations *iop; + const struct file_operations *fop; + union proc_op op; +}; + +#define NOD(NAME, MODE, IOP, FOP, OP) { \ + .name = (NAME), \ + .len = sizeof(NAME) - 1, \ + .mode = MODE, \ + .iop = IOP, \ + .fop = FOP, \ + .op = OP, \ +} + +#define DIR(NAME, MODE, iops, fops) \ + NOD(NAME, (S_IFDIR|(MODE)), &iops, &fops, {} ) +#define LNK(NAME, get_link) \ + NOD(NAME, (S_IFLNK|S_IRWXUGO), \ + &proc_pid_link_inode_operations, NULL, \ + { .proc_get_link = get_link } ) +#define REG(NAME, MODE, fops) \ + NOD(NAME, (S_IFREG|(MODE)), NULL, &fops, {}) +#define ONE(NAME, MODE, show) \ + NOD(NAME, (S_IFREG|(MODE)), \ + NULL, &proc_single_file_operations, \ + { .proc_show = show } ) +#define ATTR(LSMID, NAME, MODE) \ + NOD(NAME, (S_IFREG|(MODE)), \ + NULL, &proc_pid_attr_operations, \ + { .lsmid = LSMID }) + +/* + * Count the number of hardlinks for the pid_entry table, excluding the . + * and .. links. + */ +static unsigned int __init pid_entry_nlink(const struct pid_entry *entries, + unsigned int n) +{ + unsigned int i; + unsigned int count; + + count = 2; + for (i = 0; i < n; ++i) { + if (S_ISDIR(entries[i].mode)) + ++count; + } + + return count; +} + +static int get_task_root(struct task_struct *task, struct path *root) +{ + int result = -ENOENT; + + task_lock(task); + if (task->real_fs) { + get_fs_root(task->real_fs, root); + result = 0; + } + task_unlock(task); + return result; +} + +static int proc_cwd_link(struct dentry *dentry, struct path *path, + struct task_struct *task) +{ + int result = -ENOENT; + + task_lock(task); + if (task->real_fs) { + get_fs_pwd(task->real_fs, path); + result = 0; + } + task_unlock(task); + return result; +} + +static int proc_root_link(struct dentry *dentry, struct path *path, + struct task_struct *task) +{ + return get_task_root(task, path); +} + +/* + * If the user used setproctitle(), we just get the string from + * user space at arg_start, and limit it to a maximum of one page. + */ +static ssize_t get_mm_proctitle(struct mm_struct *mm, char __user *buf, + size_t count, unsigned long pos, + unsigned long arg_start) +{ + char *page; + int ret, got; + + if (pos >= PAGE_SIZE) + return 0; + + page = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!page) + return -ENOMEM; + + ret = 0; + got = access_remote_vm(mm, arg_start, page, PAGE_SIZE, FOLL_ANON); + if (got > 0) { + int len = strnlen(page, got); + + /* Include the NUL character if it was found */ + if (len < got) + len++; + + if (len > pos) { + len -= pos; + if (len > count) + len = count; + len -= copy_to_user(buf, page+pos, len); + if (!len) + len = -EFAULT; + ret = len; + } + } + kfree(page); + return ret; +} + +static ssize_t get_mm_cmdline(struct mm_struct *mm, char __user *buf, + size_t count, loff_t *ppos) +{ + unsigned long arg_start, arg_end, env_start, env_end; + unsigned long pos, len; + char *page, c; + + /* Check if process spawned far enough to have cmdline. */ + if (!mm->env_end) + return 0; + + spin_lock(&mm->arg_lock); + arg_start = mm->arg_start; + arg_end = mm->arg_end; + env_start = mm->env_start; + env_end = mm->env_end; + spin_unlock(&mm->arg_lock); + + if (arg_start >= arg_end) + return 0; + + /* + * We allow setproctitle() to overwrite the argument + * strings, and overflow past the original end. But + * only when it overflows into the environment area. + */ + if (env_start != arg_end || env_end < env_start) + env_start = env_end = arg_end; + len = env_end - arg_start; + + /* We're not going to care if "*ppos" has high bits set */ + pos = *ppos; + if (pos >= len) + return 0; + if (count > len - pos) + count = len - pos; + if (!count) + return 0; + + /* + * Magical special case: if the argv[] end byte is not + * zero, the user has overwritten it with setproctitle(3). + * + * Possible future enhancement: do this only once when + * pos is 0, and set a flag in the 'struct file'. + */ + if (access_remote_vm(mm, arg_end-1, &c, 1, FOLL_ANON) == 1 && c) + return get_mm_proctitle(mm, buf, count, pos, arg_start); + + /* + * For the non-setproctitle() case we limit things strictly + * to the [arg_start, arg_end[ range. + */ + pos += arg_start; + if (pos < arg_start || pos >= arg_end) + return 0; + if (count > arg_end - pos) + count = arg_end - pos; + + page = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!page) + return -ENOMEM; + + len = 0; + while (count) { + int got; + size_t size = min_t(size_t, PAGE_SIZE, count); + + got = access_remote_vm(mm, pos, page, size, FOLL_ANON); + if (got <= 0) + break; + got -= copy_to_user(buf, page, got); + if (unlikely(!got)) { + if (!len) + len = -EFAULT; + break; + } + pos += got; + buf += got; + len += got; + count -= got; + } + + kfree(page); + return len; +} + +static ssize_t get_task_cmdline(struct task_struct *tsk, char __user *buf, + size_t count, loff_t *pos) +{ + struct mm_struct *mm; + ssize_t ret; + + mm = get_task_mm(tsk); + if (!mm) + return 0; + + ret = get_mm_cmdline(mm, buf, count, pos); + mmput(mm); + return ret; +} + +static ssize_t proc_pid_cmdline_read(struct file *file, char __user *buf, + size_t count, loff_t *pos) +{ + struct task_struct *tsk; + ssize_t ret; + + BUG_ON(*pos < 0); + + tsk = get_proc_task(file_inode(file)); + if (!tsk) + return -ESRCH; + ret = get_task_cmdline(tsk, buf, count, pos); + put_task_struct(tsk); + if (ret > 0) + *pos += ret; + return ret; +} + +static const struct file_operations proc_pid_cmdline_ops = { + .read = proc_pid_cmdline_read, + .llseek = generic_file_llseek, +}; + +#ifdef CONFIG_KALLSYMS +/* + * Provides a wchan file via kallsyms in a proper one-value-per-file format. + * Returns the resolved symbol to user space. + */ +static int proc_pid_wchan(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *task) +{ + unsigned long wchan; + char symname[KSYM_NAME_LEN]; + int err; + + err = down_read_killable(&task->signal->exec_update_lock); + if (err) + return err; + if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) + goto print0; + + wchan = get_wchan(task); + if (wchan && !lookup_symbol_name(wchan, symname)) { + seq_puts(m, symname); + up_read(&task->signal->exec_update_lock); + return 0; + } + +print0: + seq_putc(m, '0'); + up_read(&task->signal->exec_update_lock); + return 0; +} +#endif /* CONFIG_KALLSYMS */ + +static int lock_trace(struct task_struct *task) +{ + int err = down_read_killable(&task->signal->exec_update_lock); + if (err) + return err; + if (!ptrace_may_access(task, PTRACE_MODE_ATTACH_FSCREDS)) { + up_read(&task->signal->exec_update_lock); + return -EPERM; + } + return 0; +} + +static void unlock_trace(struct task_struct *task) +{ + up_read(&task->signal->exec_update_lock); +} + +#ifdef CONFIG_STACKTRACE + +#define MAX_STACK_TRACE_DEPTH 64 + +static int proc_pid_stack(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *task) +{ + unsigned long *entries; + int err; + + /* + * The ability to racily run the kernel stack unwinder on a running task + * and then observe the unwinder output is scary; while it is useful for + * debugging kernel issues, it can also allow an attacker to leak kernel + * stack contents. + * Doing this in a manner that is at least safe from races would require + * some work to ensure that the remote task can not be scheduled; and + * even then, this would still expose the unwinder as local attack + * surface. + * Therefore, this interface is restricted to root. + */ + if (!file_ns_capable(m->file, &init_user_ns, CAP_SYS_ADMIN)) + return -EACCES; + + entries = kmalloc_array(MAX_STACK_TRACE_DEPTH, sizeof(*entries), + GFP_KERNEL); + if (!entries) + return -ENOMEM; + + err = lock_trace(task); + if (!err) { + unsigned int i, nr_entries; + + nr_entries = stack_trace_save_tsk(task, entries, + MAX_STACK_TRACE_DEPTH, 0); + + for (i = 0; i < nr_entries; i++) { + seq_printf(m, "[<0>] %pB\n", (void *)entries[i]); + } + + unlock_trace(task); + } + kfree(entries); + + return err; +} +#endif + +#ifdef CONFIG_SCHED_INFO +/* + * Provides /proc/PID/schedstat + */ +static int proc_pid_schedstat(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *task) +{ + if (unlikely(!sched_info_on())) + seq_puts(m, "0 0 0\n"); + else + seq_printf(m, "%llu %llu %lu\n", + (unsigned long long)task->se.sum_exec_runtime, + (unsigned long long)task->sched_info.run_delay, + task->sched_info.pcount); + + return 0; +} +#endif + +#ifdef CONFIG_LATENCYTOP +static int lstats_show_proc(struct seq_file *m, void *v) +{ + int i; + struct inode *inode = m->private; + struct task_struct *task = get_proc_task(inode); + + if (!task) + return -ESRCH; + seq_puts(m, "Latency Top version : v0.1\n"); + for (i = 0; i < LT_SAVECOUNT; i++) { + struct latency_record *lr = &task->latency_record[i]; + if (lr->backtrace[0]) { + int q; + seq_printf(m, "%i %li %li", + lr->count, lr->time, lr->max); + for (q = 0; q < LT_BACKTRACEDEPTH; q++) { + unsigned long bt = lr->backtrace[q]; + + if (!bt) + break; + seq_printf(m, " %ps", (void *)bt); + } + seq_putc(m, '\n'); + } + + } + put_task_struct(task); + return 0; +} + +static int lstats_open(struct inode *inode, struct file *file) +{ + return single_open(file, lstats_show_proc, inode); +} + +static ssize_t lstats_write(struct file *file, const char __user *buf, + size_t count, loff_t *offs) +{ + struct task_struct *task = get_proc_task(file_inode(file)); + + if (!task) + return -ESRCH; + clear_tsk_latency_tracing(task); + put_task_struct(task); + + return count; +} + +static const struct file_operations proc_lstats_operations = { + .open = lstats_open, + .read = seq_read, + .write = lstats_write, + .llseek = seq_lseek, + .release = single_release, +}; + +#endif + +static int proc_oom_score(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *task) +{ + unsigned long totalpages = totalram_pages() + total_swap_pages; + unsigned long points = 0; + long badness; + + badness = oom_badness(task, totalpages); + /* + * Special case OOM_SCORE_ADJ_MIN for all others scale the + * badness value into [0, 2000] range which we have been + * exporting for a long time so userspace might depend on it. + */ + if (badness != LONG_MIN) + points = (1000 + badness * 1000 / (long)totalpages) * 2 / 3; + + seq_printf(m, "%lu\n", points); + + return 0; +} + +struct limit_names { + const char *name; + const char *unit; +}; + +static const struct limit_names lnames[RLIM_NLIMITS] = { + [RLIMIT_CPU] = {"Max cpu time", "seconds"}, + [RLIMIT_FSIZE] = {"Max file size", "bytes"}, + [RLIMIT_DATA] = {"Max data size", "bytes"}, + [RLIMIT_STACK] = {"Max stack size", "bytes"}, + [RLIMIT_CORE] = {"Max core file size", "bytes"}, + [RLIMIT_RSS] = {"Max resident set", "bytes"}, + [RLIMIT_NPROC] = {"Max processes", "processes"}, + [RLIMIT_NOFILE] = {"Max open files", "files"}, + [RLIMIT_MEMLOCK] = {"Max locked memory", "bytes"}, + [RLIMIT_AS] = {"Max address space", "bytes"}, + [RLIMIT_LOCKS] = {"Max file locks", "locks"}, + [RLIMIT_SIGPENDING] = {"Max pending signals", "signals"}, + [RLIMIT_MSGQUEUE] = {"Max msgqueue size", "bytes"}, + [RLIMIT_NICE] = {"Max nice priority", NULL}, + [RLIMIT_RTPRIO] = {"Max realtime priority", NULL}, + [RLIMIT_RTTIME] = {"Max realtime timeout", "us"}, +}; + +/* Display limits for a process */ +static int proc_pid_limits(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *task) +{ + unsigned int i; + unsigned long flags; + + struct rlimit rlim[RLIM_NLIMITS]; + + if (!lock_task_sighand(task, &flags)) + return 0; + memcpy(rlim, task->signal->rlim, sizeof(struct rlimit) * RLIM_NLIMITS); + unlock_task_sighand(task, &flags); + + /* + * print the file header + */ + seq_puts(m, "Limit " + "Soft Limit " + "Hard Limit " + "Units \n"); + + for (i = 0; i < RLIM_NLIMITS; i++) { + if (rlim[i].rlim_cur == RLIM_INFINITY) + seq_printf(m, "%-25s %-20s ", + lnames[i].name, "unlimited"); + else + seq_printf(m, "%-25s %-20lu ", + lnames[i].name, rlim[i].rlim_cur); + + if (rlim[i].rlim_max == RLIM_INFINITY) + seq_printf(m, "%-20s ", "unlimited"); + else + seq_printf(m, "%-20lu ", rlim[i].rlim_max); + + if (lnames[i].unit) + seq_printf(m, "%-10s\n", lnames[i].unit); + else + seq_putc(m, '\n'); + } + + return 0; +} + +#ifdef CONFIG_HAVE_ARCH_TRACEHOOK +static int proc_pid_syscall(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *task) +{ + struct syscall_info info; + u64 *args = &info.data.args[0]; + int res; + + res = lock_trace(task); + if (res) + return res; + + if (task_current_syscall(task, &info)) + seq_puts(m, "running\n"); + else if (info.data.nr < 0) + seq_printf(m, "%d 0x%llx 0x%llx\n", + info.data.nr, info.sp, info.data.instruction_pointer); + else + seq_printf(m, + "%d 0x%llx 0x%llx 0x%llx 0x%llx 0x%llx 0x%llx 0x%llx 0x%llx\n", + info.data.nr, + args[0], args[1], args[2], args[3], args[4], args[5], + info.sp, info.data.instruction_pointer); + unlock_trace(task); + + return 0; +} +#endif /* CONFIG_HAVE_ARCH_TRACEHOOK */ + +/************************************************************************/ +/* Here the fs part begins */ +/************************************************************************/ + +int proc_nochmod_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +{ + int error; + struct inode *inode = d_inode(dentry); + + if (attr->ia_valid & ATTR_MODE) + return -EPERM; + + error = setattr_prepare(&nop_mnt_idmap, dentry, attr); + if (error) + return error; + + setattr_copy(&nop_mnt_idmap, inode, attr); + return 0; +} + +/* + * May current process learn task's sched/cmdline info (for hide_pid_min=1) + * or euid/egid (for hide_pid_min=2)? + */ +static bool has_pid_permissions(struct proc_fs_info *fs_info, + struct task_struct *task, + enum proc_hidepid hide_pid_min) +{ + /* + * If 'hidpid' mount option is set force a ptrace check, + * we indicate that we are using a filesystem syscall + * by passing PTRACE_MODE_READ_FSCREDS + */ + if (fs_info->hide_pid == HIDEPID_NOT_PTRACEABLE) + return ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS); + + if (fs_info->hide_pid < hide_pid_min) + return true; + if (in_group_p(fs_info->pid_gid)) + return true; + return ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS); +} + + +static int proc_pid_permission(struct mnt_idmap *idmap, + struct inode *inode, int mask) +{ + struct proc_fs_info *fs_info = proc_sb_info(inode->i_sb); + struct task_struct *task; + bool has_perms; + + task = get_proc_task(inode); + if (!task) + return -ESRCH; + has_perms = has_pid_permissions(fs_info, task, HIDEPID_NO_ACCESS); + put_task_struct(task); + + if (!has_perms) { + if (fs_info->hide_pid == HIDEPID_INVISIBLE) { + /* + * Let's make getdents(), stat(), and open() + * consistent with each other. If a process + * may not stat() a file, it shouldn't be seen + * in procfs at all. + */ + return -ENOENT; + } + + return -EPERM; + } + return generic_permission(&nop_mnt_idmap, inode, mask); +} + + + +static const struct inode_operations proc_def_inode_operations = { + .setattr = proc_nochmod_setattr, +}; + +static int proc_single_show(struct seq_file *m, void *v) +{ + struct inode *inode = m->private; + struct pid_namespace *ns = proc_pid_ns(inode->i_sb); + struct pid *pid = proc_pid(inode); + struct task_struct *task; + int ret; + + task = get_pid_task(pid, PIDTYPE_PID); + if (!task) + return -ESRCH; + + ret = PROC_I(inode)->op.proc_show(m, ns, pid, task); + + put_task_struct(task); + return ret; +} + +static int proc_single_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, proc_single_show, inode); +} + +static const struct file_operations proc_single_file_operations = { + .open = proc_single_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +/* + * proc_mem_open() can return errno, NULL or mm_struct*. + * + * - Returns NULL if the task has no mm (PF_KTHREAD or PF_EXITING) + * - Returns mm_struct* on success + * - Returns error code on failure + */ +struct mm_struct *proc_mem_open(struct inode *inode, unsigned int mode) +{ + struct task_struct *task = get_proc_task(inode); + struct mm_struct *mm; + + if (!task) + return ERR_PTR(-ESRCH); + + mm = mm_access(task, mode | PTRACE_MODE_FSCREDS); + put_task_struct(task); + + if (IS_ERR(mm)) + return mm == ERR_PTR(-ESRCH) ? NULL : mm; + + /* ensure this mm_struct can't be freed */ + mmgrab(mm); + /* but do not pin its memory */ + mmput(mm); + + return mm; +} + +static int __mem_open(struct inode *inode, struct file *file, unsigned int mode) +{ + struct mm_struct *mm = proc_mem_open(inode, mode); + + if (IS_ERR_OR_NULL(mm)) + return mm ? PTR_ERR(mm) : -ESRCH; + + file->private_data = mm; + return 0; +} + +static int mem_open(struct inode *inode, struct file *file) +{ + if (WARN_ON_ONCE(!(file->f_op->fop_flags & FOP_UNSIGNED_OFFSET))) + return -EINVAL; + return __mem_open(inode, file, PTRACE_MODE_ATTACH); +} + +static bool proc_mem_foll_force(struct file *file, struct mm_struct *mm) +{ + struct task_struct *task; + bool ptrace_active = false; + + switch (proc_mem_force_override) { + case PROC_MEM_FORCE_NEVER: + return false; + case PROC_MEM_FORCE_PTRACE: + task = get_proc_task(file_inode(file)); + if (task) { + ptrace_active = READ_ONCE(task->ptrace) && + READ_ONCE(task->mm) == mm && + READ_ONCE(task->parent) == current; + put_task_struct(task); + } + return ptrace_active; + default: + return true; + } +} + +static ssize_t mem_rw(struct file *file, char __user *buf, + size_t count, loff_t *ppos, int write) +{ + struct mm_struct *mm = file->private_data; + unsigned long addr = *ppos; + ssize_t copied; + char *page; + unsigned int flags; + + if (!mm) + return 0; + + page = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!page) + return -ENOMEM; + + copied = 0; + if (!mmget_not_zero(mm)) + goto free; + + flags = write ? FOLL_WRITE : 0; + if (proc_mem_foll_force(file, mm)) + flags |= FOLL_FORCE; + + while (count > 0) { + size_t this_len = min_t(size_t, count, PAGE_SIZE); + + if (write && copy_from_user(page, buf, this_len)) { + copied = -EFAULT; + break; + } + + this_len = access_remote_vm(mm, addr, page, this_len, flags); + if (!this_len) { + if (!copied) + copied = -EIO; + break; + } + + if (!write && copy_to_user(buf, page, this_len)) { + copied = -EFAULT; + break; + } + + buf += this_len; + addr += this_len; + copied += this_len; + count -= this_len; + } + *ppos = addr; + + mmput(mm); +free: + kfree(page); + return copied; +} + +static ssize_t mem_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + return mem_rw(file, buf, count, ppos, 0); +} + +static ssize_t mem_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + return mem_rw(file, (char __user*)buf, count, ppos, 1); +} + +loff_t mem_lseek(struct file *file, loff_t offset, int orig) +{ + switch (orig) { + case 0: + file->f_pos = offset; + break; + case 1: + file->f_pos += offset; + break; + default: + return -EINVAL; + } + force_successful_syscall_return(); + return file->f_pos; +} + +static int mem_release(struct inode *inode, struct file *file) +{ + struct mm_struct *mm = file->private_data; + if (mm) + mmdrop(mm); + return 0; +} + +static const struct file_operations proc_mem_operations = { + .llseek = mem_lseek, + .read = mem_read, + .write = mem_write, + .open = mem_open, + .release = mem_release, + .fop_flags = FOP_UNSIGNED_OFFSET, +}; + +static int environ_open(struct inode *inode, struct file *file) +{ + return __mem_open(inode, file, PTRACE_MODE_READ); +} + +static ssize_t environ_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + char *page; + unsigned long src = *ppos; + int ret = 0; + struct mm_struct *mm = file->private_data; + unsigned long env_start, env_end; + + /* Ensure the process spawned far enough to have an environment. */ + if (!mm || !mm->env_end) + return 0; + + page = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!page) + return -ENOMEM; + + ret = 0; + if (!mmget_not_zero(mm)) + goto free; + + spin_lock(&mm->arg_lock); + env_start = mm->env_start; + env_end = mm->env_end; + spin_unlock(&mm->arg_lock); + + while (count > 0) { + size_t this_len, max_len; + int retval; + + if (src >= (env_end - env_start)) + break; + + this_len = env_end - (env_start + src); + + max_len = min_t(size_t, PAGE_SIZE, count); + this_len = min(max_len, this_len); + + retval = access_remote_vm(mm, (env_start + src), page, this_len, FOLL_ANON); + + if (retval <= 0) { + ret = retval; + break; + } + + if (copy_to_user(buf, page, retval)) { + ret = -EFAULT; + break; + } + + ret += retval; + src += retval; + buf += retval; + count -= retval; + } + *ppos = src; + mmput(mm); + +free: + kfree(page); + return ret; +} + +static const struct file_operations proc_environ_operations = { + .open = environ_open, + .read = environ_read, + .llseek = generic_file_llseek, + .release = mem_release, +}; + +static int auxv_open(struct inode *inode, struct file *file) +{ + return __mem_open(inode, file, PTRACE_MODE_READ_FSCREDS); +} + +static ssize_t auxv_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct mm_struct *mm = file->private_data; + unsigned int nwords = 0; + + if (!mm) + return 0; + do { + nwords += 2; + } while (mm->saved_auxv[nwords - 2] != 0); /* AT_NULL */ + return simple_read_from_buffer(buf, count, ppos, mm->saved_auxv, + nwords * sizeof(mm->saved_auxv[0])); +} + +static const struct file_operations proc_auxv_operations = { + .open = auxv_open, + .read = auxv_read, + .llseek = generic_file_llseek, + .release = mem_release, +}; + +static ssize_t oom_adj_read(struct file *file, char __user *buf, size_t count, + loff_t *ppos) +{ + struct task_struct *task = get_proc_task(file_inode(file)); + char buffer[PROC_NUMBUF]; + int oom_adj = OOM_ADJUST_MIN; + size_t len; + + if (!task) + return -ESRCH; + if (task->signal->oom_score_adj == OOM_SCORE_ADJ_MAX) + oom_adj = OOM_ADJUST_MAX; + else + oom_adj = (task->signal->oom_score_adj * -OOM_DISABLE) / + OOM_SCORE_ADJ_MAX; + put_task_struct(task); + if (oom_adj > OOM_ADJUST_MAX) + oom_adj = OOM_ADJUST_MAX; + len = snprintf(buffer, sizeof(buffer), "%d\n", oom_adj); + return simple_read_from_buffer(buf, count, ppos, buffer, len); +} + +static int __set_oom_adj(struct file *file, int oom_adj, bool legacy) +{ + struct mm_struct *mm = NULL; + struct task_struct *task; + int err = 0; + + task = get_proc_task(file_inode(file)); + if (!task) + return -ESRCH; + + mutex_lock(&oom_adj_mutex); + if (legacy) { + if (oom_adj < task->signal->oom_score_adj && + !capable(CAP_SYS_RESOURCE)) { + err = -EACCES; + goto err_unlock; + } + /* + * /proc/pid/oom_adj is provided for legacy purposes, ask users to use + * /proc/pid/oom_score_adj instead. + */ + pr_warn_once("%s (%d): /proc/%d/oom_adj is deprecated, please use /proc/%d/oom_score_adj instead.\n", + current->comm, task_pid_nr(current), task_pid_nr(task), + task_pid_nr(task)); + } else { + if ((short)oom_adj < task->signal->oom_score_adj_min && + !capable(CAP_SYS_RESOURCE)) { + err = -EACCES; + goto err_unlock; + } + } + + /* + * Make sure we will check other processes sharing the mm if this is + * not vfrok which wants its own oom_score_adj. + * pin the mm so it doesn't go away and get reused after task_unlock + */ + if (!task->vfork_done) { + struct task_struct *p = find_lock_task_mm(task); + + if (p) { + if (mm_flags_test(MMF_MULTIPROCESS, p->mm)) { + mm = p->mm; + mmgrab(mm); + } + task_unlock(p); + } + } + + task->signal->oom_score_adj = oom_adj; + if (!legacy && has_capability_noaudit(current, CAP_SYS_RESOURCE)) + task->signal->oom_score_adj_min = (short)oom_adj; + trace_oom_score_adj_update(task); + + if (mm) { + struct task_struct *p; + + rcu_read_lock(); + for_each_process(p) { + if (same_thread_group(task, p)) + continue; + + /* do not touch kernel threads or the global init */ + if (p->flags & PF_KTHREAD || is_global_init(p)) + continue; + + task_lock(p); + if (!p->vfork_done && process_shares_mm(p, mm)) { + p->signal->oom_score_adj = oom_adj; + if (!legacy && has_capability_noaudit(current, CAP_SYS_RESOURCE)) + p->signal->oom_score_adj_min = (short)oom_adj; + } + task_unlock(p); + } + rcu_read_unlock(); + mmdrop(mm); + } +err_unlock: + mutex_unlock(&oom_adj_mutex); + put_task_struct(task); + return err; +} + +/* + * /proc/pid/oom_adj exists solely for backwards compatibility with previous + * kernels. The effective policy is defined by oom_score_adj, which has a + * different scale: oom_adj grew exponentially and oom_score_adj grows linearly. + * Values written to oom_adj are simply mapped linearly to oom_score_adj. + * Processes that become oom disabled via oom_adj will still be oom disabled + * with this implementation. + * + * oom_adj cannot be removed since existing userspace binaries use it. + */ +static ssize_t oom_adj_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + char buffer[PROC_NUMBUF] = {}; + int oom_adj; + int err; + + if (count > sizeof(buffer) - 1) + count = sizeof(buffer) - 1; + if (copy_from_user(buffer, buf, count)) { + err = -EFAULT; + goto out; + } + + err = kstrtoint(strstrip(buffer), 0, &oom_adj); + if (err) + goto out; + if ((oom_adj < OOM_ADJUST_MIN || oom_adj > OOM_ADJUST_MAX) && + oom_adj != OOM_DISABLE) { + err = -EINVAL; + goto out; + } + + /* + * Scale /proc/pid/oom_score_adj appropriately ensuring that a maximum + * value is always attainable. + */ + if (oom_adj == OOM_ADJUST_MAX) + oom_adj = OOM_SCORE_ADJ_MAX; + else + oom_adj = (oom_adj * OOM_SCORE_ADJ_MAX) / -OOM_DISABLE; + + err = __set_oom_adj(file, oom_adj, true); +out: + return err < 0 ? err : count; +} + +static const struct file_operations proc_oom_adj_operations = { + .read = oom_adj_read, + .write = oom_adj_write, + .llseek = generic_file_llseek, +}; + +static ssize_t oom_score_adj_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct task_struct *task = get_proc_task(file_inode(file)); + char buffer[PROC_NUMBUF]; + short oom_score_adj = OOM_SCORE_ADJ_MIN; + size_t len; + + if (!task) + return -ESRCH; + oom_score_adj = task->signal->oom_score_adj; + put_task_struct(task); + len = snprintf(buffer, sizeof(buffer), "%hd\n", oom_score_adj); + return simple_read_from_buffer(buf, count, ppos, buffer, len); +} + +static ssize_t oom_score_adj_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + char buffer[PROC_NUMBUF] = {}; + int oom_score_adj; + int err; + + if (count > sizeof(buffer) - 1) + count = sizeof(buffer) - 1; + if (copy_from_user(buffer, buf, count)) { + err = -EFAULT; + goto out; + } + + err = kstrtoint(strstrip(buffer), 0, &oom_score_adj); + if (err) + goto out; + if (oom_score_adj < OOM_SCORE_ADJ_MIN || + oom_score_adj > OOM_SCORE_ADJ_MAX) { + err = -EINVAL; + goto out; + } + + err = __set_oom_adj(file, oom_score_adj, false); +out: + return err < 0 ? err : count; +} + +static const struct file_operations proc_oom_score_adj_operations = { + .read = oom_score_adj_read, + .write = oom_score_adj_write, + .llseek = default_llseek, +}; + +#ifdef CONFIG_AUDIT +#define TMPBUFLEN 11 +static ssize_t proc_loginuid_read(struct file * file, char __user * buf, + size_t count, loff_t *ppos) +{ + struct inode * inode = file_inode(file); + struct task_struct *task = get_proc_task(inode); + ssize_t length; + char tmpbuf[TMPBUFLEN]; + + if (!task) + return -ESRCH; + length = scnprintf(tmpbuf, TMPBUFLEN, "%u", + from_kuid(file->f_cred->user_ns, + audit_get_loginuid(task))); + put_task_struct(task); + return simple_read_from_buffer(buf, count, ppos, tmpbuf, length); +} + +static ssize_t proc_loginuid_write(struct file * file, const char __user * buf, + size_t count, loff_t *ppos) +{ + struct inode * inode = file_inode(file); + uid_t loginuid; + kuid_t kloginuid; + int rv; + + /* Don't let kthreads write their own loginuid */ + if (current->flags & PF_KTHREAD) + return -EPERM; + + rcu_read_lock(); + if (current != pid_task(proc_pid(inode), PIDTYPE_PID)) { + rcu_read_unlock(); + return -EPERM; + } + rcu_read_unlock(); + + if (*ppos != 0) { + /* No partial writes. */ + return -EINVAL; + } + + rv = kstrtou32_from_user(buf, count, 10, &loginuid); + if (rv < 0) + return rv; + + /* is userspace tring to explicitly UNSET the loginuid? */ + if (loginuid == AUDIT_UID_UNSET) { + kloginuid = INVALID_UID; + } else { + kloginuid = make_kuid(file->f_cred->user_ns, loginuid); + if (!uid_valid(kloginuid)) + return -EINVAL; + } + + rv = audit_set_loginuid(kloginuid); + if (rv < 0) + return rv; + return count; +} + +static const struct file_operations proc_loginuid_operations = { + .read = proc_loginuid_read, + .write = proc_loginuid_write, + .llseek = generic_file_llseek, +}; + +static ssize_t proc_sessionid_read(struct file * file, char __user * buf, + size_t count, loff_t *ppos) +{ + struct inode * inode = file_inode(file); + struct task_struct *task = get_proc_task(inode); + ssize_t length; + char tmpbuf[TMPBUFLEN]; + + if (!task) + return -ESRCH; + length = scnprintf(tmpbuf, TMPBUFLEN, "%u", + audit_get_sessionid(task)); + put_task_struct(task); + return simple_read_from_buffer(buf, count, ppos, tmpbuf, length); +} + +static const struct file_operations proc_sessionid_operations = { + .read = proc_sessionid_read, + .llseek = generic_file_llseek, +}; +#endif + +#ifdef CONFIG_FAULT_INJECTION +static ssize_t proc_fault_inject_read(struct file * file, char __user * buf, + size_t count, loff_t *ppos) +{ + struct task_struct *task = get_proc_task(file_inode(file)); + char buffer[PROC_NUMBUF]; + size_t len; + int make_it_fail; + + if (!task) + return -ESRCH; + make_it_fail = task->make_it_fail; + put_task_struct(task); + + len = snprintf(buffer, sizeof(buffer), "%i\n", make_it_fail); + + return simple_read_from_buffer(buf, count, ppos, buffer, len); +} + +static ssize_t proc_fault_inject_write(struct file * file, + const char __user * buf, size_t count, loff_t *ppos) +{ + struct task_struct *task; + char buffer[PROC_NUMBUF] = {}; + int make_it_fail; + int rv; + + if (!capable(CAP_SYS_RESOURCE)) + return -EPERM; + + if (count > sizeof(buffer) - 1) + count = sizeof(buffer) - 1; + if (copy_from_user(buffer, buf, count)) + return -EFAULT; + rv = kstrtoint(strstrip(buffer), 0, &make_it_fail); + if (rv < 0) + return rv; + if (make_it_fail < 0 || make_it_fail > 1) + return -EINVAL; + + task = get_proc_task(file_inode(file)); + if (!task) + return -ESRCH; + task->make_it_fail = make_it_fail; + put_task_struct(task); + + return count; +} + +static const struct file_operations proc_fault_inject_operations = { + .read = proc_fault_inject_read, + .write = proc_fault_inject_write, + .llseek = generic_file_llseek, +}; + +static ssize_t proc_fail_nth_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct task_struct *task; + int err; + unsigned int n; + + err = kstrtouint_from_user(buf, count, 0, &n); + if (err) + return err; + + task = get_proc_task(file_inode(file)); + if (!task) + return -ESRCH; + task->fail_nth = n; + put_task_struct(task); + + return count; +} + +static ssize_t proc_fail_nth_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct task_struct *task; + char numbuf[PROC_NUMBUF]; + ssize_t len; + + task = get_proc_task(file_inode(file)); + if (!task) + return -ESRCH; + len = snprintf(numbuf, sizeof(numbuf), "%u\n", task->fail_nth); + put_task_struct(task); + return simple_read_from_buffer(buf, count, ppos, numbuf, len); +} + +static const struct file_operations proc_fail_nth_operations = { + .read = proc_fail_nth_read, + .write = proc_fail_nth_write, +}; +#endif + + +/* + * Print out various scheduling related per-task fields: + */ +static int sched_show(struct seq_file *m, void *v) +{ + struct inode *inode = m->private; + struct pid_namespace *ns = proc_pid_ns(inode->i_sb); + struct task_struct *p; + + p = get_proc_task(inode); + if (!p) + return -ESRCH; + proc_sched_show_task(p, ns, m); + + put_task_struct(p); + + return 0; +} + +static ssize_t +sched_write(struct file *file, const char __user *buf, + size_t count, loff_t *offset) +{ + struct inode *inode = file_inode(file); + struct task_struct *p; + + p = get_proc_task(inode); + if (!p) + return -ESRCH; + proc_sched_set_task(p); + + put_task_struct(p); + + return count; +} + +static int sched_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, sched_show, inode); +} + +static const struct file_operations proc_pid_sched_operations = { + .open = sched_open, + .read = seq_read, + .write = sched_write, + .llseek = seq_lseek, + .release = single_release, +}; + +#ifdef CONFIG_SCHED_AUTOGROUP +/* + * Print out autogroup related information: + */ +static int sched_autogroup_show(struct seq_file *m, void *v) +{ + struct inode *inode = m->private; + struct task_struct *p; + + p = get_proc_task(inode); + if (!p) + return -ESRCH; + proc_sched_autogroup_show_task(p, m); + + put_task_struct(p); + + return 0; +} + +static ssize_t +sched_autogroup_write(struct file *file, const char __user *buf, + size_t count, loff_t *offset) +{ + struct inode *inode = file_inode(file); + struct task_struct *p; + char buffer[PROC_NUMBUF] = {}; + int nice; + int err; + + if (count > sizeof(buffer) - 1) + count = sizeof(buffer) - 1; + if (copy_from_user(buffer, buf, count)) + return -EFAULT; + + err = kstrtoint(strstrip(buffer), 0, &nice); + if (err < 0) + return err; + + p = get_proc_task(inode); + if (!p) + return -ESRCH; + + err = proc_sched_autogroup_set_nice(p, nice); + if (err) + count = err; + + put_task_struct(p); + + return count; +} + +static int sched_autogroup_open(struct inode *inode, struct file *filp) +{ + int ret; + + ret = single_open(filp, sched_autogroup_show, NULL); + if (!ret) { + struct seq_file *m = filp->private_data; + + m->private = inode; + } + return ret; +} + +static const struct file_operations proc_pid_sched_autogroup_operations = { + .open = sched_autogroup_open, + .read = seq_read, + .write = sched_autogroup_write, + .llseek = seq_lseek, + .release = single_release, +}; + +#endif /* CONFIG_SCHED_AUTOGROUP */ + +#ifdef CONFIG_TIME_NS +static int timens_offsets_show(struct seq_file *m, void *v) +{ + struct task_struct *p; + + p = get_proc_task(file_inode(m->file)); + if (!p) + return -ESRCH; + proc_timens_show_offsets(p, m); + + put_task_struct(p); + + return 0; +} + +static ssize_t timens_offsets_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct inode *inode = file_inode(file); + struct proc_timens_offset offsets[2]; + char *kbuf = NULL, *pos, *next_line; + struct task_struct *p; + int ret, noffsets; + + /* Only allow < page size writes at the beginning of the file */ + if ((*ppos != 0) || (count >= PAGE_SIZE)) + return -EINVAL; + + /* Slurp in the user data */ + kbuf = memdup_user_nul(buf, count); + if (IS_ERR(kbuf)) + return PTR_ERR(kbuf); + + /* Parse the user data */ + ret = -EINVAL; + noffsets = 0; + for (pos = kbuf; pos; pos = next_line) { + struct proc_timens_offset *off = &offsets[noffsets]; + char clock[10]; + int err; + + /* Find the end of line and ensure we don't look past it */ + next_line = strchr(pos, '\n'); + if (next_line) { + *next_line = '\0'; + next_line++; + if (*next_line == '\0') + next_line = NULL; + } + + err = sscanf(pos, "%9s %lld %lu", clock, + &off->val.tv_sec, &off->val.tv_nsec); + if (err != 3 || off->val.tv_nsec >= NSEC_PER_SEC) + goto out; + + clock[sizeof(clock) - 1] = 0; + if (strcmp(clock, "monotonic") == 0 || + strcmp(clock, __stringify(CLOCK_MONOTONIC)) == 0) + off->clockid = CLOCK_MONOTONIC; + else if (strcmp(clock, "boottime") == 0 || + strcmp(clock, __stringify(CLOCK_BOOTTIME)) == 0) + off->clockid = CLOCK_BOOTTIME; + else + goto out; + + noffsets++; + if (noffsets == ARRAY_SIZE(offsets)) { + if (next_line) + count = next_line - kbuf; + break; + } + } + + ret = -ESRCH; + p = get_proc_task(inode); + if (!p) + goto out; + ret = proc_timens_set_offset(file, p, offsets, noffsets); + put_task_struct(p); + if (ret) + goto out; + + ret = count; +out: + kfree(kbuf); + return ret; +} + +static int timens_offsets_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, timens_offsets_show, inode); +} + +static const struct file_operations proc_timens_offsets_operations = { + .open = timens_offsets_open, + .read = seq_read, + .write = timens_offsets_write, + .llseek = seq_lseek, + .release = single_release, +}; +#endif /* CONFIG_TIME_NS */ + +static ssize_t comm_write(struct file *file, const char __user *buf, + size_t count, loff_t *offset) +{ + struct inode *inode = file_inode(file); + struct task_struct *p; + char buffer[TASK_COMM_LEN] = {}; + const size_t maxlen = sizeof(buffer) - 1; + + if (copy_from_user(buffer, buf, count > maxlen ? maxlen : count)) + return -EFAULT; + + p = get_proc_task(inode); + if (!p) + return -ESRCH; + + if (same_thread_group(current, p)) { + set_task_comm(p, buffer); + proc_comm_connector(p); + } + else + count = -EINVAL; + + put_task_struct(p); + + return count; +} + +static int comm_show(struct seq_file *m, void *v) +{ + struct inode *inode = m->private; + struct task_struct *p; + + p = get_proc_task(inode); + if (!p) + return -ESRCH; + + proc_task_name(m, p, false); + seq_putc(m, '\n'); + + put_task_struct(p); + + return 0; +} + +static int comm_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, comm_show, inode); +} + +static const struct file_operations proc_pid_set_comm_operations = { + .open = comm_open, + .read = seq_read, + .write = comm_write, + .llseek = seq_lseek, + .release = single_release, +}; + +static int proc_exe_link(struct dentry *dentry, struct path *exe_path, + struct task_struct *task) +{ + struct file *exe_file; + + exe_file = get_task_exe_file(task); + if (exe_file) { + *exe_path = exe_file->f_path; + path_get(&exe_file->f_path); + fput(exe_file); + return 0; + } else + return -ENOENT; +} + +static int call_proc_get_link(struct dentry *dentry, struct inode *inode, struct path *path_out) +{ + struct task_struct *task; + int ret; + + task = get_proc_task(inode); + if (!task) + return -ENOENT; + ret = down_read_killable(&task->signal->exec_update_lock); + if (ret) + goto out_put_task; + if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) { + ret = -EACCES; + goto out; + } + ret = PROC_I(inode)->op.proc_get_link(dentry, path_out, task); + +out: + up_read(&task->signal->exec_update_lock); +out_put_task: + put_task_struct(task); + return ret; +} + +static const char *proc_pid_get_link(struct dentry *dentry, + struct inode *inode, + struct delayed_call *done) +{ + struct path path; + int error; + + if (!dentry) + return ERR_PTR(-ECHILD); + error = call_proc_get_link(dentry, inode, &path); + if (!error) + error = nd_jump_link(&path); + return ERR_PTR(error); +} + +static int do_proc_readlink(const struct path *path, char __user *buffer, int buflen) +{ + char *tmp = kmalloc(PATH_MAX, GFP_KERNEL); + char *pathname; + int len; + + if (!tmp) + return -ENOMEM; + + pathname = d_path(path, tmp, PATH_MAX); + len = PTR_ERR(pathname); + if (IS_ERR(pathname)) + goto out; + len = tmp + PATH_MAX - 1 - pathname; + + if (len > buflen) + len = buflen; + if (copy_to_user(buffer, pathname, len)) + len = -EFAULT; + out: + kfree(tmp); + return len; +} + +static int proc_pid_readlink(struct dentry * dentry, char __user * buffer, int buflen) +{ + int error = -EACCES; + struct inode *inode = d_inode(dentry); + struct path path; + + error = call_proc_get_link(dentry, inode, &path); + if (!error) { + error = do_proc_readlink(&path, buffer, buflen); + path_put(&path); + } + return error; +} + +const struct inode_operations proc_pid_link_inode_operations = { + .readlink = proc_pid_readlink, + .get_link = proc_pid_get_link, + .setattr = proc_nochmod_setattr, +}; + + +/* building an inode */ + +void task_dump_owner(struct task_struct *task, umode_t mode, + kuid_t *ruid, kgid_t *rgid) +{ + /* Depending on the state of dumpable compute who should own a + * proc file for a task. + */ + const struct cred *cred; + kuid_t uid; + kgid_t gid; + + if (unlikely(task->flags & PF_KTHREAD)) { + *ruid = GLOBAL_ROOT_UID; + *rgid = GLOBAL_ROOT_GID; + return; + } + + /* Default to the tasks effective ownership */ + rcu_read_lock(); + cred = __task_cred(task); + uid = cred->euid; + gid = cred->egid; + + /* + * Before the /proc/pid/status file was created the only way to read + * the effective uid of a /process was to stat /proc/pid. Reading + * /proc/pid/status is slow enough that procps and other packages + * kept stating /proc/pid. To keep the rules in /proc simple I have + * made this apply to all per process world readable and executable + * directories. + */ + if (mode != (S_IFDIR | S_IRUGO | S_IXUGO)) { + struct task_exec_state *exec_state; + + exec_state = task_exec_state_rcu(task); + if (READ_ONCE(exec_state->dumpable) != TASK_DUMPABLE_OWNER) { + uid = make_kuid(exec_state->user_ns, 0); + if (!uid_valid(uid)) + uid = GLOBAL_ROOT_UID; + + gid = make_kgid(exec_state->user_ns, 0); + if (!gid_valid(gid)) + gid = GLOBAL_ROOT_GID; + } + } + rcu_read_unlock(); + + *ruid = uid; + *rgid = gid; +} + +void proc_pid_evict_inode(struct proc_inode *ei) +{ + struct pid *pid = ei->pid; + + if (S_ISDIR(ei->vfs_inode.i_mode)) { + spin_lock(&pid->lock); + hlist_del_init_rcu(&ei->sibling_inodes); + spin_unlock(&pid->lock); + } +} + +struct inode *proc_pid_make_inode(struct super_block *sb, + struct task_struct *task, umode_t mode) +{ + struct inode * inode; + struct proc_inode *ei; + struct pid *pid; + + /* We need a new inode */ + + inode = new_inode(sb); + if (!inode) + goto out; + + /* Common stuff */ + ei = PROC_I(inode); + inode->i_mode = mode; + inode->i_ino = get_next_ino(); + simple_inode_init_ts(inode); + inode->i_op = &proc_def_inode_operations; + + /* + * grab the reference to task. + */ + pid = get_task_pid(task, PIDTYPE_PID); + if (!pid) + goto out_unlock; + + /* Let the pid remember us for quick removal */ + ei->pid = pid; + + task_dump_owner(task, 0, &inode->i_uid, &inode->i_gid); + security_task_to_inode(task, inode); + +out: + return inode; + +out_unlock: + iput(inode); + return NULL; +} + +/* + * Generating an inode and adding it into @pid->inodes, so that task will + * invalidate inode's dentry before being released. + * + * This helper is used for creating dir-type entries under '/proc' and + * '/proc//task'. Other entries(eg. fd, stat) under '/proc/' + * can be released by invalidating '/proc/' dentry. + * In theory, dentries under '/proc//task' can also be released by + * invalidating '/proc/' dentry, we reserve it to handle single + * thread exiting situation: Any one of threads should invalidate its + * '/proc//task/' dentry before released. + */ +static struct inode *proc_pid_make_base_inode(struct super_block *sb, + struct task_struct *task, umode_t mode) +{ + struct inode *inode; + struct proc_inode *ei; + struct pid *pid; + + inode = proc_pid_make_inode(sb, task, mode); + if (!inode) + return NULL; + + /* Let proc_flush_pid find this directory inode */ + ei = PROC_I(inode); + pid = ei->pid; + spin_lock(&pid->lock); + hlist_add_head_rcu(&ei->sibling_inodes, &pid->inodes); + spin_unlock(&pid->lock); + + return inode; +} + +int pid_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int query_flags) +{ + struct inode *inode = d_inode(path->dentry); + struct proc_fs_info *fs_info = proc_sb_info(inode->i_sb); + struct task_struct *task; + + generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat); + + stat->uid = GLOBAL_ROOT_UID; + stat->gid = GLOBAL_ROOT_GID; + rcu_read_lock(); + task = pid_task(proc_pid(inode), PIDTYPE_PID); + if (task) { + if (!has_pid_permissions(fs_info, task, HIDEPID_INVISIBLE)) { + rcu_read_unlock(); + /* + * This doesn't prevent learning whether PID exists, + * it only makes getattr() consistent with readdir(). + */ + return -ENOENT; + } + task_dump_owner(task, inode->i_mode, &stat->uid, &stat->gid); + } + rcu_read_unlock(); + return 0; +} + +/* dentry stuff */ + +/* + * Set /... inode ownership (can change due to setuid(), etc.) + */ +void pid_update_inode(struct task_struct *task, struct inode *inode) +{ + task_dump_owner(task, inode->i_mode, &inode->i_uid, &inode->i_gid); + + inode->i_mode &= ~(S_ISUID | S_ISGID); + security_task_to_inode(task, inode); +} + +/* + * Rewrite the inode's ownerships here because the owning task may have + * performed a setuid(), etc. + * + */ +static int pid_revalidate(struct inode *dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + struct inode *inode; + struct task_struct *task; + int ret = 0; + + rcu_read_lock(); + inode = d_inode_rcu(dentry); + if (!inode) + goto out; + task = pid_task(proc_pid(inode), PIDTYPE_PID); + + if (task) { + pid_update_inode(task, inode); + ret = 1; + } +out: + rcu_read_unlock(); + return ret; +} + +static inline bool proc_inode_is_dead(struct inode *inode) +{ + return !proc_pid(inode)->tasks[PIDTYPE_PID].first; +} + +int pid_delete_dentry(const struct dentry *dentry) +{ + /* Is the task we represent dead? + * If so, then don't put the dentry on the lru list, + * kill it immediately. + */ + return proc_inode_is_dead(d_inode(dentry)); +} + +const struct dentry_operations pid_dentry_operations = +{ + .d_revalidate = pid_revalidate, + .d_delete = pid_delete_dentry, +}; + +/* Lookups */ + +/* + * Fill a directory entry. + * + * If possible create the dcache entry and derive our inode number and + * file type from dcache entry. + * + * Since all of the proc inode numbers are dynamically generated, the inode + * numbers do not exist until the inode is cache. This means creating + * the dcache entry in readdir is necessary to keep the inode numbers + * reported by readdir in sync with the inode numbers reported + * by stat. + */ +bool proc_fill_cache(struct file *file, struct dir_context *ctx, + const char *name, unsigned int len, + instantiate_t instantiate, struct task_struct *task, const void *ptr) +{ + struct dentry *child, *dir = file->f_path.dentry; + struct qstr qname = QSTR_INIT(name, len); + struct inode *inode; + unsigned type = DT_UNKNOWN; + ino_t ino = 1; + + child = try_lookup_noperm(&qname, dir); + if (IS_ERR(child)) + goto end_instantiate; + + if (!child) { + child = d_alloc_parallel(dir, &qname); + if (IS_ERR(child)) + goto end_instantiate; + if (d_in_lookup(child)) { + struct dentry *res; + res = instantiate(child, task, ptr); + d_lookup_done(child); + if (unlikely(res)) { + dput(child); + child = res; + if (IS_ERR(child)) + goto end_instantiate; + } + } + } + inode = d_inode(child); + ino = inode->i_ino; + type = inode->i_mode >> 12; + dput(child); +end_instantiate: + return dir_emit(ctx, name, len, ino, type); +} + +/* + * dname_to_vma_addr - maps a dentry name into two unsigned longs + * which represent vma start and end addresses. + */ +static int dname_to_vma_addr(struct dentry *dentry, + unsigned long *start, unsigned long *end) +{ + const char *str = dentry->d_name.name; + unsigned long long sval, eval; + unsigned int len; + + if (str[0] == '0' && str[1] != '-') + return -EINVAL; + len = _parse_integer(str, 16, &sval); + if (len & KSTRTOX_OVERFLOW) + return -EINVAL; + if (sval != (unsigned long)sval) + return -EINVAL; + str += len; + + if (*str != '-') + return -EINVAL; + str++; + + if (str[0] == '0' && str[1]) + return -EINVAL; + len = _parse_integer(str, 16, &eval); + if (len & KSTRTOX_OVERFLOW) + return -EINVAL; + if (eval != (unsigned long)eval) + return -EINVAL; + str += len; + + if (*str != '\0') + return -EINVAL; + + *start = sval; + *end = eval; + + return 0; +} + +static int map_files_d_revalidate(struct inode *dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + unsigned long vm_start, vm_end; + bool exact_vma_exists = false; + struct mm_struct *mm = NULL; + struct task_struct *task; + struct inode *inode; + int status = 0; + + if (flags & LOOKUP_RCU) + return -ECHILD; + + inode = d_inode(dentry); + task = get_proc_task(inode); + if (!task) + goto out_notask; + + mm = mm_access(task, PTRACE_MODE_READ_FSCREDS); + if (IS_ERR(mm)) + goto out; + + if (!dname_to_vma_addr(dentry, &vm_start, &vm_end)) { + status = mmap_read_lock_killable(mm); + if (!status) { + exact_vma_exists = !!find_exact_vma(mm, vm_start, + vm_end); + mmap_read_unlock(mm); + } + } + + mmput(mm); + + if (exact_vma_exists) { + task_dump_owner(task, 0, &inode->i_uid, &inode->i_gid); + + security_task_to_inode(task, inode); + status = 1; + } + +out: + put_task_struct(task); + +out_notask: + return status; +} + +static const struct dentry_operations tid_map_files_dentry_operations = { + .d_revalidate = map_files_d_revalidate, + .d_delete = pid_delete_dentry, +}; + +static int map_files_get_link(struct dentry *dentry, struct path *path, + struct task_struct *task) +{ + unsigned long vm_start, vm_end; + struct vm_area_struct *vma; + struct mm_struct *mm; + int rc; + + rc = -ENOENT; + mm = get_task_mm(task); + if (!mm) + goto out; + + rc = dname_to_vma_addr(dentry, &vm_start, &vm_end); + if (rc) + goto out_mmput; + + rc = mmap_read_lock_killable(mm); + if (rc) + goto out_mmput; + + rc = -ENOENT; + vma = find_exact_vma(mm, vm_start, vm_end); + if (vma && vma->vm_file) { + *path = *file_user_path(vma->vm_file); + path_get(path); + rc = 0; + } + mmap_read_unlock(mm); + +out_mmput: + mmput(mm); +out: + return rc; +} + +struct map_files_info { + unsigned long start; + unsigned long end; + fmode_t mode; +}; + +/* + * Only allow CAP_SYS_ADMIN and CAP_CHECKPOINT_RESTORE to follow the links, due + * to concerns about how the symlinks may be used to bypass permissions on + * ancestor directories in the path to the file in question. + */ +static const char * +proc_map_files_get_link(struct dentry *dentry, + struct inode *inode, + struct delayed_call *done) +{ + if (!checkpoint_restore_ns_capable(&init_user_ns)) + return ERR_PTR(-EPERM); + + return proc_pid_get_link(dentry, inode, done); +} + +/* + * Identical to proc_pid_link_inode_operations except for get_link() + */ +static const struct inode_operations proc_map_files_link_inode_operations = { + .readlink = proc_pid_readlink, + .get_link = proc_map_files_get_link, + .setattr = proc_nochmod_setattr, +}; + +static struct dentry * +proc_map_files_instantiate(struct dentry *dentry, + struct task_struct *task, const void *ptr) +{ + fmode_t mode = (fmode_t)(unsigned long)ptr; + struct proc_inode *ei; + struct inode *inode; + + inode = proc_pid_make_inode(dentry->d_sb, task, S_IFLNK | + ((mode & FMODE_READ ) ? S_IRUSR : 0) | + ((mode & FMODE_WRITE) ? S_IWUSR : 0)); + if (!inode) + return ERR_PTR(-ENOENT); + + ei = PROC_I(inode); + ei->op.proc_get_link = map_files_get_link; + + inode->i_op = &proc_map_files_link_inode_operations; + inode->i_size = 64; + + return proc_splice_unmountable(inode, dentry, + &tid_map_files_dentry_operations); +} + +static struct dentry *proc_map_files_lookup(struct inode *dir, + struct dentry *dentry, unsigned int flags) +{ + unsigned long vm_start, vm_end; + struct vm_area_struct *vma; + struct task_struct *task; + struct dentry *result; + struct mm_struct *mm; + + result = ERR_PTR(-ENOENT); + task = get_proc_task(dir); + if (!task) + goto out; + + result = ERR_PTR(-ENOENT); + if (dname_to_vma_addr(dentry, &vm_start, &vm_end)) + goto out_put_task; + + mm = mm_access(task, PTRACE_MODE_READ_FSCREDS); + if (IS_ERR(mm)) { + result = ERR_CAST(mm); + goto out_put_task; + } + + result = ERR_PTR(-EINTR); + if (mmap_read_lock_killable(mm)) + goto out_put_mm; + + result = ERR_PTR(-ENOENT); + vma = find_exact_vma(mm, vm_start, vm_end); + if (!vma) + goto out_no_vma; + + if (vma->vm_file) + result = proc_map_files_instantiate(dentry, task, + (void *)(unsigned long)vma->vm_file->f_mode); + +out_no_vma: + mmap_read_unlock(mm); +out_put_mm: + mmput(mm); +out_put_task: + put_task_struct(task); +out: + return result; +} + +static const struct inode_operations proc_map_files_inode_operations = { + .lookup = proc_map_files_lookup, + .permission = proc_fd_permission, + .setattr = proc_nochmod_setattr, +}; + +static int +proc_map_files_readdir(struct file *file, struct dir_context *ctx) +{ + struct vm_area_struct *vma; + struct task_struct *task; + struct mm_struct *mm; + unsigned long nr_files, pos, i; + GENRADIX(struct map_files_info) fa; + struct map_files_info *p; + int ret; + struct vma_iterator vmi; + + genradix_init(&fa); + + ret = -ENOENT; + task = get_proc_task(file_inode(file)); + if (!task) + goto out; + + ret = 0; + if (!dir_emit_dots(file, ctx)) + goto out_put_task; + + mm = mm_access(task, PTRACE_MODE_READ_FSCREDS); + if (IS_ERR(mm)) { + ret = PTR_ERR(mm); + /* if the task has no mm, the directory should just be empty */ + if (ret == -ESRCH) + ret = 0; + goto out_put_task; + } + + ret = mmap_read_lock_killable(mm); + if (ret) + goto out_put_mm; + + nr_files = 0; + + /* + * We need two passes here: + * + * 1) Collect vmas of mapped files with mmap_lock taken + * 2) Release mmap_lock and instantiate entries + * + * otherwise we get lockdep complained, since filldir() + * routine might require mmap_lock taken in might_fault(). + */ + + pos = 2; + vma_iter_init(&vmi, mm, 0); + for_each_vma(vmi, vma) { + if (!vma->vm_file) + continue; + if (++pos <= ctx->pos) + continue; + + p = genradix_ptr_alloc(&fa, nr_files++, GFP_KERNEL); + if (!p) { + ret = -ENOMEM; + mmap_read_unlock(mm); + goto out_put_mm; + } + + p->start = vma->vm_start; + p->end = vma->vm_end; + p->mode = vma->vm_file->f_mode; + } + mmap_read_unlock(mm); + + for (i = 0; i < nr_files; i++) { + char buf[4 * sizeof(long) + 2]; /* max: %lx-%lx\0 */ + unsigned int len; + + p = genradix_ptr(&fa, i); + len = snprintf(buf, sizeof(buf), "%lx-%lx", p->start, p->end); + if (!proc_fill_cache(file, ctx, + buf, len, + proc_map_files_instantiate, + task, + (void *)(unsigned long)p->mode)) + break; + ctx->pos++; + } + +out_put_mm: + mmput(mm); +out_put_task: + put_task_struct(task); +out: + genradix_free(&fa); + return ret; +} + +static const struct file_operations proc_map_files_operations = { + .read = generic_read_dir, + .iterate_shared = proc_map_files_readdir, + .llseek = generic_file_llseek, +}; + +#if defined(CONFIG_CHECKPOINT_RESTORE) && defined(CONFIG_POSIX_TIMERS) +struct timers_private { + struct pid *pid; + struct task_struct *task; + struct pid_namespace *ns; +}; + +static void *timers_start(struct seq_file *m, loff_t *pos) +{ + struct timers_private *tp = m->private; + + tp->task = get_pid_task(tp->pid, PIDTYPE_PID); + if (!tp->task) + return ERR_PTR(-ESRCH); + + rcu_read_lock(); + return seq_hlist_start_rcu(&tp->task->signal->posix_timers, *pos); +} + +static void *timers_next(struct seq_file *m, void *v, loff_t *pos) +{ + struct timers_private *tp = m->private; + + return seq_hlist_next_rcu(v, &tp->task->signal->posix_timers, pos); +} + +static void timers_stop(struct seq_file *m, void *v) +{ + struct timers_private *tp = m->private; + + if (tp->task) { + put_task_struct(tp->task); + tp->task = NULL; + rcu_read_unlock(); + } +} + +static int show_timer(struct seq_file *m, void *v) +{ + static const char * const nstr[] = { + [SIGEV_SIGNAL] = "signal", + [SIGEV_NONE] = "none", + [SIGEV_THREAD] = "thread", + }; + + struct k_itimer *timer = hlist_entry((struct hlist_node *)v, struct k_itimer, list); + struct timers_private *tp = m->private; + int notify = timer->it_sigev_notify; + + guard(spinlock_irq)(&timer->it_lock); + if (!posixtimer_valid(timer)) + return 0; + + seq_printf(m, "ID: %d\n", timer->it_id); + seq_printf(m, "signal: %d/%px\n", timer->sigq.info.si_signo, + timer->sigq.info.si_value.sival_ptr); + seq_printf(m, "notify: %s/%s.%d\n", nstr[notify & ~SIGEV_THREAD_ID], + (notify & SIGEV_THREAD_ID) ? "tid" : "pid", + pid_nr_ns(timer->it_pid, tp->ns)); + seq_printf(m, "ClockID: %d\n", timer->it_clock); + + return 0; +} + +static const struct seq_operations proc_timers_seq_ops = { + .start = timers_start, + .next = timers_next, + .stop = timers_stop, + .show = show_timer, +}; + +static int proc_timers_open(struct inode *inode, struct file *file) +{ + struct timers_private *tp; + + tp = __seq_open_private(file, &proc_timers_seq_ops, + sizeof(struct timers_private)); + if (!tp) + return -ENOMEM; + + tp->pid = proc_pid(inode); + tp->ns = proc_pid_ns(inode->i_sb); + return 0; +} + +static const struct file_operations proc_timers_operations = { + .open = proc_timers_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release_private, +}; +#endif + +static ssize_t timerslack_ns_write(struct file *file, const char __user *buf, + size_t count, loff_t *offset) +{ + struct inode *inode = file_inode(file); + struct task_struct *p; + u64 slack_ns; + int err; + + err = kstrtoull_from_user(buf, count, 10, &slack_ns); + if (err < 0) + return err; + + p = get_proc_task(inode); + if (!p) + return -ESRCH; + + if (p != current) { + rcu_read_lock(); + if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) { + rcu_read_unlock(); + count = -EPERM; + goto out; + } + rcu_read_unlock(); + + err = security_task_setscheduler(p); + if (err) { + count = err; + goto out; + } + } + + task_lock(p); + if (rt_or_dl_task_policy(p)) + slack_ns = 0; + else if (slack_ns == 0) + slack_ns = p->default_timer_slack_ns; + p->timer_slack_ns = slack_ns; + task_unlock(p); + +out: + put_task_struct(p); + + return count; +} + +static int timerslack_ns_show(struct seq_file *m, void *v) +{ + struct inode *inode = m->private; + struct task_struct *p; + int err = 0; + + p = get_proc_task(inode); + if (!p) + return -ESRCH; + + if (p != current) { + rcu_read_lock(); + if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) { + rcu_read_unlock(); + err = -EPERM; + goto out; + } + rcu_read_unlock(); + + err = security_task_getscheduler(p); + if (err) + goto out; + } + + task_lock(p); + seq_printf(m, "%llu\n", p->timer_slack_ns); + task_unlock(p); + +out: + put_task_struct(p); + + return err; +} + +static int timerslack_ns_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, timerslack_ns_show, inode); +} + +static const struct file_operations proc_pid_set_timerslack_ns_operations = { + .open = timerslack_ns_open, + .read = seq_read, + .write = timerslack_ns_write, + .llseek = seq_lseek, + .release = single_release, +}; + +static struct dentry *proc_pident_instantiate(struct dentry *dentry, + struct task_struct *task, const void *ptr) +{ + const struct pid_entry *p = ptr; + struct inode *inode; + struct proc_inode *ei; + + inode = proc_pid_make_inode(dentry->d_sb, task, p->mode); + if (!inode) + return ERR_PTR(-ENOENT); + + ei = PROC_I(inode); + if (S_ISDIR(inode->i_mode)) + set_nlink(inode, 2); /* Use getattr to fix if necessary */ + if (p->iop) + inode->i_op = p->iop; + if (p->fop) + inode->i_fop = p->fop; + ei->op = p->op; + pid_update_inode(task, inode); + return d_splice_alias_ops(inode, dentry, &pid_dentry_operations); +} + +static struct dentry *proc_pident_lookup(struct inode *dir, + struct dentry *dentry, + const struct pid_entry *p, + const struct pid_entry *end) +{ + struct task_struct *task = get_proc_task(dir); + struct dentry *res = ERR_PTR(-ENOENT); + + if (!task) + goto out_no_task; + + /* + * Yes, it does not scale. And it should not. Don't add + * new entries into /proc// without very good reasons. + */ + for (; p < end; p++) { + if (p->len != dentry->d_name.len) + continue; + if (!memcmp(dentry->d_name.name, p->name, p->len)) { + res = proc_pident_instantiate(dentry, task, p); + break; + } + } + put_task_struct(task); +out_no_task: + return res; +} + +static int proc_pident_readdir(struct file *file, struct dir_context *ctx, + const struct pid_entry *ents, unsigned int nents) +{ + struct task_struct *task = get_proc_task(file_inode(file)); + const struct pid_entry *p; + + if (!task) + return -ENOENT; + + if (!dir_emit_dots(file, ctx)) + goto out; + + if (ctx->pos >= nents + 2) + goto out; + + for (p = ents + (ctx->pos - 2); p < ents + nents; p++) { + if (!proc_fill_cache(file, ctx, p->name, p->len, + proc_pident_instantiate, task, p)) + break; + ctx->pos++; + } +out: + put_task_struct(task); + return 0; +} + +#ifdef CONFIG_SECURITY +static int proc_pid_attr_open(struct inode *inode, struct file *file) +{ + file->private_data = NULL; + __mem_open(inode, file, PTRACE_MODE_READ_FSCREDS); + return 0; +} + +static ssize_t proc_pid_attr_read(struct file * file, char __user * buf, + size_t count, loff_t *ppos) +{ + struct inode * inode = file_inode(file); + char *p = NULL; + ssize_t length; + struct task_struct *task = get_proc_task(inode); + + if (!task) + return -ESRCH; + + length = security_getprocattr(task, PROC_I(inode)->op.lsmid, + file->f_path.dentry->d_name.name, + &p); + put_task_struct(task); + if (length > 0) + length = simple_read_from_buffer(buf, count, ppos, p, length); + kfree(p); + return length; +} + +static ssize_t proc_pid_attr_write(struct file * file, const char __user * buf, + size_t count, loff_t *ppos) +{ + struct inode * inode = file_inode(file); + struct task_struct *task; + void *page; + int rv; + + /* A task may only write when it was the opener. */ + if (file->private_data != current->mm) + return -EPERM; + + rcu_read_lock(); + task = pid_task(proc_pid(inode), PIDTYPE_PID); + if (!task) { + rcu_read_unlock(); + return -ESRCH; + } + /* A task may only write its own attributes. */ + if (current != task) { + rcu_read_unlock(); + return -EACCES; + } + /* Prevent changes to overridden credentials. */ + if (current_cred() != current_real_cred()) { + rcu_read_unlock(); + return -EBUSY; + } + rcu_read_unlock(); + + if (count > PAGE_SIZE) + count = PAGE_SIZE; + + /* No partial writes. */ + if (*ppos != 0) + return -EINVAL; + + page = memdup_user(buf, count); + if (IS_ERR(page)) { + rv = PTR_ERR(page); + goto out; + } + + /* Guard against adverse ptrace interaction */ + rv = mutex_lock_interruptible(¤t->signal->cred_guard_mutex); + if (rv < 0) + goto out_free; + + rv = security_setprocattr(PROC_I(inode)->op.lsmid, + file->f_path.dentry->d_name.name, page, + count); + mutex_unlock(¤t->signal->cred_guard_mutex); +out_free: + kfree(page); +out: + return rv; +} + +static const struct file_operations proc_pid_attr_operations = { + .open = proc_pid_attr_open, + .read = proc_pid_attr_read, + .write = proc_pid_attr_write, + .llseek = generic_file_llseek, + .release = mem_release, +}; + +#define LSM_DIR_OPS(LSM) \ +static int proc_##LSM##_attr_dir_iterate(struct file *filp, \ + struct dir_context *ctx) \ +{ \ + return proc_pident_readdir(filp, ctx, \ + LSM##_attr_dir_stuff, \ + ARRAY_SIZE(LSM##_attr_dir_stuff)); \ +} \ +\ +static const struct file_operations proc_##LSM##_attr_dir_ops = { \ + .read = generic_read_dir, \ + .iterate_shared = proc_##LSM##_attr_dir_iterate, \ + .llseek = default_llseek, \ +}; \ +\ +static struct dentry *proc_##LSM##_attr_dir_lookup(struct inode *dir, \ + struct dentry *dentry, unsigned int flags) \ +{ \ + return proc_pident_lookup(dir, dentry, \ + LSM##_attr_dir_stuff, \ + LSM##_attr_dir_stuff + ARRAY_SIZE(LSM##_attr_dir_stuff)); \ +} \ +\ +static const struct inode_operations proc_##LSM##_attr_dir_inode_ops = { \ + .lookup = proc_##LSM##_attr_dir_lookup, \ + .getattr = pid_getattr, \ + .setattr = proc_nochmod_setattr, \ +} + +#ifdef CONFIG_SECURITY_SMACK +static const struct pid_entry smack_attr_dir_stuff[] = { + ATTR(LSM_ID_SMACK, "current", 0666), +}; +LSM_DIR_OPS(smack); +#endif + +#ifdef CONFIG_SECURITY_APPARMOR +static const struct pid_entry apparmor_attr_dir_stuff[] = { + ATTR(LSM_ID_APPARMOR, "current", 0666), + ATTR(LSM_ID_APPARMOR, "prev", 0444), + ATTR(LSM_ID_APPARMOR, "exec", 0666), +}; +LSM_DIR_OPS(apparmor); +#endif + +static const struct pid_entry attr_dir_stuff[] = { + ATTR(LSM_ID_UNDEF, "current", 0666), + ATTR(LSM_ID_UNDEF, "prev", 0444), + ATTR(LSM_ID_UNDEF, "exec", 0666), + ATTR(LSM_ID_UNDEF, "fscreate", 0666), + ATTR(LSM_ID_UNDEF, "keycreate", 0666), + ATTR(LSM_ID_UNDEF, "sockcreate", 0666), +#ifdef CONFIG_SECURITY_SMACK + DIR("smack", 0555, + proc_smack_attr_dir_inode_ops, proc_smack_attr_dir_ops), +#endif +#ifdef CONFIG_SECURITY_APPARMOR + DIR("apparmor", 0555, + proc_apparmor_attr_dir_inode_ops, proc_apparmor_attr_dir_ops), +#endif +}; + +static int proc_attr_dir_readdir(struct file *file, struct dir_context *ctx) +{ + return proc_pident_readdir(file, ctx, + attr_dir_stuff, ARRAY_SIZE(attr_dir_stuff)); +} + +static const struct file_operations proc_attr_dir_operations = { + .read = generic_read_dir, + .iterate_shared = proc_attr_dir_readdir, + .llseek = generic_file_llseek, +}; + +static struct dentry *proc_attr_dir_lookup(struct inode *dir, + struct dentry *dentry, unsigned int flags) +{ + return proc_pident_lookup(dir, dentry, + attr_dir_stuff, + attr_dir_stuff + ARRAY_SIZE(attr_dir_stuff)); +} + +static const struct inode_operations proc_attr_dir_inode_operations = { + .lookup = proc_attr_dir_lookup, + .getattr = pid_getattr, + .setattr = proc_nochmod_setattr, +}; + +#endif + +#ifdef CONFIG_ELF_CORE +static ssize_t proc_coredump_filter_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct task_struct *task = get_proc_task(file_inode(file)); + struct mm_struct *mm; + char buffer[PROC_NUMBUF]; + size_t len; + int ret; + + if (!task) + return -ESRCH; + + ret = 0; + mm = get_task_mm(task); + if (mm) { + unsigned long flags = __mm_flags_get_word(mm); + + len = snprintf(buffer, sizeof(buffer), "%08lx\n", + ((flags & MMF_DUMP_FILTER_MASK) >> + MMF_DUMP_FILTER_SHIFT)); + mmput(mm); + ret = simple_read_from_buffer(buf, count, ppos, buffer, len); + } + + put_task_struct(task); + + return ret; +} + +static ssize_t proc_coredump_filter_write(struct file *file, + const char __user *buf, + size_t count, + loff_t *ppos) +{ + struct task_struct *task; + struct mm_struct *mm; + unsigned int val; + int ret; + int i; + unsigned long mask; + + ret = kstrtouint_from_user(buf, count, 0, &val); + if (ret < 0) + return ret; + + ret = -ESRCH; + task = get_proc_task(file_inode(file)); + if (!task) + goto out_no_task; + + mm = get_task_mm(task); + if (!mm) + goto out_no_mm; + ret = 0; + + for (i = 0, mask = 1; i < MMF_DUMP_FILTER_BITS; i++, mask <<= 1) { + if (val & mask) + mm_flags_set(i + MMF_DUMP_FILTER_SHIFT, mm); + else + mm_flags_clear(i + MMF_DUMP_FILTER_SHIFT, mm); + } + + mmput(mm); + out_no_mm: + put_task_struct(task); + out_no_task: + if (ret < 0) + return ret; + return count; +} + +static const struct file_operations proc_coredump_filter_operations = { + .read = proc_coredump_filter_read, + .write = proc_coredump_filter_write, + .llseek = generic_file_llseek, +}; +#endif + +#ifdef CONFIG_TASK_IO_ACCOUNTING +static int do_io_accounting(struct task_struct *task, struct seq_file *m, int whole) +{ + struct task_io_accounting acct; + int result; + + result = down_read_killable(&task->signal->exec_update_lock); + if (result) + return result; + + if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) { + result = -EACCES; + goto out_unlock; + } + + if (whole) { + struct signal_struct *sig = task->signal; + struct task_struct *t; + + guard(rcu)(); + scoped_seqlock_read (&sig->stats_lock, ss_lock_irqsave) { + acct = sig->ioac; + __for_each_thread(sig, t) + task_io_accounting_add(&acct, &t->ioac); + + } + } else { + acct = task->ioac; + } + + seq_printf(m, + "rchar: %llu\n" + "wchar: %llu\n" + "syscr: %llu\n" + "syscw: %llu\n" + "read_bytes: %llu\n" + "write_bytes: %llu\n" + "cancelled_write_bytes: %llu\n", + (unsigned long long)acct.rchar, + (unsigned long long)acct.wchar, + (unsigned long long)acct.syscr, + (unsigned long long)acct.syscw, + (unsigned long long)acct.read_bytes, + (unsigned long long)acct.write_bytes, + (unsigned long long)acct.cancelled_write_bytes); + result = 0; + +out_unlock: + up_read(&task->signal->exec_update_lock); + return result; +} + +static int proc_tid_io_accounting(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *task) +{ + return do_io_accounting(task, m, 0); +} + +static int proc_tgid_io_accounting(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *task) +{ + return do_io_accounting(task, m, 1); +} +#endif /* CONFIG_TASK_IO_ACCOUNTING */ + +#ifdef CONFIG_USER_NS +static int proc_id_map_open(struct inode *inode, struct file *file, + const struct seq_operations *seq_ops) +{ + struct user_namespace *ns = NULL; + struct task_struct *task; + struct seq_file *seq; + int ret = -EINVAL; + + task = get_proc_task(inode); + if (task) { + rcu_read_lock(); + ns = get_user_ns(task_cred_xxx(task, user_ns)); + rcu_read_unlock(); + put_task_struct(task); + } + if (!ns) + goto err; + + ret = seq_open(file, seq_ops); + if (ret) + goto err_put_ns; + + seq = file->private_data; + seq->private = ns; + + return 0; +err_put_ns: + put_user_ns(ns); +err: + return ret; +} + +static int proc_id_map_release(struct inode *inode, struct file *file) +{ + struct seq_file *seq = file->private_data; + struct user_namespace *ns = seq->private; + put_user_ns(ns); + return seq_release(inode, file); +} + +static int proc_uid_map_open(struct inode *inode, struct file *file) +{ + return proc_id_map_open(inode, file, &proc_uid_seq_operations); +} + +static int proc_gid_map_open(struct inode *inode, struct file *file) +{ + return proc_id_map_open(inode, file, &proc_gid_seq_operations); +} + +static int proc_projid_map_open(struct inode *inode, struct file *file) +{ + return proc_id_map_open(inode, file, &proc_projid_seq_operations); +} + +static const struct file_operations proc_uid_map_operations = { + .open = proc_uid_map_open, + .write = proc_uid_map_write, + .read = seq_read, + .llseek = seq_lseek, + .release = proc_id_map_release, +}; + +static const struct file_operations proc_gid_map_operations = { + .open = proc_gid_map_open, + .write = proc_gid_map_write, + .read = seq_read, + .llseek = seq_lseek, + .release = proc_id_map_release, +}; + +static const struct file_operations proc_projid_map_operations = { + .open = proc_projid_map_open, + .write = proc_projid_map_write, + .read = seq_read, + .llseek = seq_lseek, + .release = proc_id_map_release, +}; + +static int proc_setgroups_open(struct inode *inode, struct file *file) +{ + struct user_namespace *ns = NULL; + struct task_struct *task; + int ret; + + ret = -ESRCH; + task = get_proc_task(inode); + if (task) { + rcu_read_lock(); + ns = get_user_ns(task_cred_xxx(task, user_ns)); + rcu_read_unlock(); + put_task_struct(task); + } + if (!ns) + goto err; + + if (file->f_mode & FMODE_WRITE) { + ret = -EACCES; + if (!ns_capable(ns, CAP_SYS_ADMIN)) + goto err_put_ns; + } + + ret = single_open(file, &proc_setgroups_show, ns); + if (ret) + goto err_put_ns; + + return 0; +err_put_ns: + put_user_ns(ns); +err: + return ret; +} + +static int proc_setgroups_release(struct inode *inode, struct file *file) +{ + struct seq_file *seq = file->private_data; + struct user_namespace *ns = seq->private; + int ret = single_release(inode, file); + put_user_ns(ns); + return ret; +} + +static const struct file_operations proc_setgroups_operations = { + .open = proc_setgroups_open, + .write = proc_setgroups_write, + .read = seq_read, + .llseek = seq_lseek, + .release = proc_setgroups_release, +}; +#endif /* CONFIG_USER_NS */ + +static int proc_pid_personality(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *task) +{ + int err = lock_trace(task); + if (!err) { + seq_printf(m, "%08x\n", task->personality); + unlock_trace(task); + } + return err; +} + +#ifdef CONFIG_LIVEPATCH +static int proc_pid_patch_state(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *task) +{ + seq_printf(m, "%d\n", task->patch_state); + return 0; +} +#endif /* CONFIG_LIVEPATCH */ + +#ifdef CONFIG_KSM +static int proc_pid_ksm_merging_pages(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *task) +{ + struct mm_struct *mm; + + mm = get_task_mm(task); + if (mm) { + seq_printf(m, "%lu\n", mm->ksm_merging_pages); + mmput(mm); + } + + return 0; +} +static int proc_pid_ksm_stat(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *task) +{ + struct mm_struct *mm; + int ret = 0; + + mm = get_task_mm(task); + if (mm) { + seq_printf(m, "ksm_rmap_items %lu\n", mm->ksm_rmap_items); + seq_printf(m, "ksm_zero_pages %ld\n", mm_ksm_zero_pages(mm)); + seq_printf(m, "ksm_merging_pages %lu\n", mm->ksm_merging_pages); + seq_printf(m, "ksm_process_profit %ld\n", ksm_process_profit(mm)); + seq_printf(m, "ksm_merge_any: %s\n", + mm_flags_test(MMF_VM_MERGE_ANY, mm) ? "yes" : "no"); + ret = mmap_read_lock_killable(mm); + if (ret) { + mmput(mm); + return ret; + } + seq_printf(m, "ksm_mergeable: %s\n", + ksm_process_mergeable(mm) ? "yes" : "no"); + mmap_read_unlock(mm); + mmput(mm); + } + + return 0; +} +#endif /* CONFIG_KSM */ + +#ifdef CONFIG_KSTACK_ERASE_METRICS +static int proc_stack_depth(struct seq_file *m, struct pid_namespace *ns, + struct pid *pid, struct task_struct *task) +{ + unsigned long prev_depth = THREAD_SIZE - + (task->prev_lowest_stack & (THREAD_SIZE - 1)); + unsigned long depth = THREAD_SIZE - + (task->lowest_stack & (THREAD_SIZE - 1)); + + seq_printf(m, "previous stack depth: %lu\nstack depth: %lu\n", + prev_depth, depth); + return 0; +} +#endif /* CONFIG_KSTACK_ERASE_METRICS */ + +/* + * Thread groups + */ +static const struct file_operations proc_task_operations; +static const struct inode_operations proc_task_inode_operations; + +static const struct pid_entry tgid_base_stuff[] = { + DIR("task", S_IRUGO|S_IXUGO, proc_task_inode_operations, proc_task_operations), + DIR("fd", S_IRUSR|S_IXUSR, proc_fd_inode_operations, proc_fd_operations), + DIR("map_files", S_IRUSR|S_IXUSR, proc_map_files_inode_operations, proc_map_files_operations), + DIR("fdinfo", S_IRUGO|S_IXUGO, proc_fdinfo_inode_operations, proc_fdinfo_operations), + DIR("ns", S_IRUSR|S_IXUGO, proc_ns_dir_inode_operations, proc_ns_dir_operations), +#ifdef CONFIG_NET + DIR("net", S_IRUGO|S_IXUGO, proc_net_inode_operations, proc_net_operations), +#endif + REG("environ", S_IRUSR, proc_environ_operations), + REG("auxv", S_IRUSR, proc_auxv_operations), + ONE("status", S_IRUGO, proc_pid_status), + ONE("personality", S_IRUSR, proc_pid_personality), + ONE("limits", S_IRUGO, proc_pid_limits), + REG("sched", S_IRUGO|S_IWUSR, proc_pid_sched_operations), +#ifdef CONFIG_SCHED_AUTOGROUP + REG("autogroup", S_IRUGO|S_IWUSR, proc_pid_sched_autogroup_operations), +#endif +#ifdef CONFIG_TIME_NS + REG("timens_offsets", S_IRUGO|S_IWUSR, proc_timens_offsets_operations), +#endif + REG("comm", S_IRUGO|S_IWUSR, proc_pid_set_comm_operations), +#ifdef CONFIG_HAVE_ARCH_TRACEHOOK + ONE("syscall", S_IRUSR, proc_pid_syscall), +#endif + REG("cmdline", S_IRUGO, proc_pid_cmdline_ops), + ONE("stat", S_IRUGO, proc_tgid_stat), + ONE("statm", S_IRUGO, proc_pid_statm), + REG("maps", S_IRUGO, proc_pid_maps_operations), +#ifdef CONFIG_NUMA + REG("numa_maps", S_IRUGO, proc_pid_numa_maps_operations), +#endif + REG("mem", S_IRUSR|S_IWUSR, proc_mem_operations), + LNK("cwd", proc_cwd_link), + LNK("root", proc_root_link), + LNK("exe", proc_exe_link), + REG("mounts", S_IRUGO, proc_mounts_operations), + REG("mountinfo", S_IRUGO, proc_mountinfo_operations), + REG("mountstats", S_IRUSR, proc_mountstats_operations), +#ifdef CONFIG_PROC_PAGE_MONITOR + REG("clear_refs", S_IWUSR, proc_clear_refs_operations), + REG("smaps", S_IRUGO, proc_pid_smaps_operations), + REG("smaps_rollup", S_IRUGO, proc_pid_smaps_rollup_operations), + REG("pagemap", S_IRUSR, proc_pagemap_operations), +#endif +#ifdef CONFIG_SECURITY + DIR("attr", S_IRUGO|S_IXUGO, proc_attr_dir_inode_operations, proc_attr_dir_operations), +#endif +#ifdef CONFIG_KALLSYMS + ONE("wchan", S_IRUGO, proc_pid_wchan), +#endif +#ifdef CONFIG_STACKTRACE + ONE("stack", S_IRUSR, proc_pid_stack), +#endif +#ifdef CONFIG_SCHED_INFO + ONE("schedstat", S_IRUGO, proc_pid_schedstat), +#endif +#ifdef CONFIG_LATENCYTOP + REG("latency", S_IRUGO, proc_lstats_operations), +#endif +#ifdef CONFIG_PROC_PID_CPUSET + ONE("cpuset", S_IRUGO, proc_cpuset_show), +#endif +#ifdef CONFIG_CGROUPS + ONE("cgroup", S_IRUGO, proc_cgroup_show), +#endif +#ifdef CONFIG_PROC_CPU_RESCTRL + ONE("cpu_resctrl_groups", S_IRUGO, proc_resctrl_show), +#endif + ONE("oom_score", S_IRUGO, proc_oom_score), + REG("oom_adj", S_IRUGO|S_IWUSR, proc_oom_adj_operations), + REG("oom_score_adj", S_IRUGO|S_IWUSR, proc_oom_score_adj_operations), +#ifdef CONFIG_AUDIT + REG("loginuid", S_IWUSR|S_IRUGO, proc_loginuid_operations), + REG("sessionid", S_IRUGO, proc_sessionid_operations), +#endif +#ifdef CONFIG_FAULT_INJECTION + REG("make-it-fail", S_IRUGO|S_IWUSR, proc_fault_inject_operations), + REG("fail-nth", 0644, proc_fail_nth_operations), +#endif +#ifdef CONFIG_ELF_CORE + REG("coredump_filter", S_IRUGO|S_IWUSR, proc_coredump_filter_operations), +#endif +#ifdef CONFIG_TASK_IO_ACCOUNTING + ONE("io", S_IRUSR, proc_tgid_io_accounting), +#endif +#ifdef CONFIG_USER_NS + REG("uid_map", S_IRUGO|S_IWUSR, proc_uid_map_operations), + REG("gid_map", S_IRUGO|S_IWUSR, proc_gid_map_operations), + REG("projid_map", S_IRUGO|S_IWUSR, proc_projid_map_operations), + REG("setgroups", S_IRUGO|S_IWUSR, proc_setgroups_operations), +#endif +#if defined(CONFIG_CHECKPOINT_RESTORE) && defined(CONFIG_POSIX_TIMERS) + REG("timers", S_IRUGO, proc_timers_operations), +#endif + REG("timerslack_ns", S_IRUGO|S_IWUGO, proc_pid_set_timerslack_ns_operations), +#ifdef CONFIG_LIVEPATCH + ONE("patch_state", S_IRUSR, proc_pid_patch_state), +#endif +#ifdef CONFIG_KSTACK_ERASE_METRICS + ONE("stack_depth", S_IRUGO, proc_stack_depth), +#endif +#ifdef CONFIG_PROC_PID_ARCH_STATUS + ONE("arch_status", S_IRUGO, proc_pid_arch_status), +#endif +#ifdef CONFIG_SECCOMP_CACHE_DEBUG + ONE("seccomp_cache", S_IRUSR, proc_pid_seccomp_cache), +#endif +#ifdef CONFIG_KSM + ONE("ksm_merging_pages", S_IRUSR, proc_pid_ksm_merging_pages), + ONE("ksm_stat", S_IRUSR, proc_pid_ksm_stat), +#endif +}; + +static int proc_tgid_base_readdir(struct file *file, struct dir_context *ctx) +{ + return proc_pident_readdir(file, ctx, + tgid_base_stuff, ARRAY_SIZE(tgid_base_stuff)); +} + +static const struct file_operations proc_tgid_base_operations = { + .read = generic_read_dir, + .iterate_shared = proc_tgid_base_readdir, + .llseek = generic_file_llseek, +}; + +struct pid *tgid_pidfd_to_pid(const struct file *file) +{ + if (file->f_op != &proc_tgid_base_operations) + return ERR_PTR(-EBADF); + + return proc_pid(file_inode(file)); +} + +static struct dentry *proc_tgid_base_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) +{ + return proc_pident_lookup(dir, dentry, + tgid_base_stuff, + tgid_base_stuff + ARRAY_SIZE(tgid_base_stuff)); +} + +static const struct inode_operations proc_tgid_base_inode_operations = { + .lookup = proc_tgid_base_lookup, + .getattr = pid_getattr, + .setattr = proc_nochmod_setattr, + .permission = proc_pid_permission, +}; + +/** + * proc_flush_pid - Remove dcache entries for @pid from the /proc dcache. + * @pid: pid that should be flushed. + * + * This function walks a list of inodes (that belong to any proc + * filesystem) that are attached to the pid and flushes them from + * the dentry cache. + * + * It is safe and reasonable to cache /proc entries for a task until + * that task exits. After that they just clog up the dcache with + * useless entries, possibly causing useful dcache entries to be + * flushed instead. This routine is provided to flush those useless + * dcache entries when a process is reaped. + * + * NOTE: This routine is just an optimization so it does not guarantee + * that no dcache entries will exist after a process is reaped + * it just makes it very unlikely that any will persist. + */ + +void proc_flush_pid(struct pid *pid) +{ + proc_invalidate_siblings_dcache(&pid->inodes, &pid->lock); +} + +static struct dentry *proc_pid_instantiate(struct dentry * dentry, + struct task_struct *task, const void *ptr) +{ + struct inode *inode; + + inode = proc_pid_make_base_inode(dentry->d_sb, task, + S_IFDIR | S_IRUGO | S_IXUGO); + if (!inode) + return ERR_PTR(-ENOENT); + + inode->i_op = &proc_tgid_base_inode_operations; + inode->i_fop = &proc_tgid_base_operations; + inode->i_flags|=S_IMMUTABLE; + + set_nlink(inode, nlink_tgid); + pid_update_inode(task, inode); + + return d_splice_alias_ops(inode, dentry, &pid_dentry_operations); +} + +struct dentry *proc_pid_lookup(struct dentry *dentry, unsigned int flags) +{ + struct task_struct *task; + unsigned tgid; + struct proc_fs_info *fs_info; + struct pid_namespace *ns; + struct dentry *result = ERR_PTR(-ENOENT); + + tgid = name_to_int(&dentry->d_name); + if (tgid == ~0U) + goto out; + + fs_info = proc_sb_info(dentry->d_sb); + ns = fs_info->pid_ns; + rcu_read_lock(); + task = find_task_by_pid_ns(tgid, ns); + if (task) + get_task_struct(task); + rcu_read_unlock(); + if (!task) + goto out; + + /* Limit procfs to only ptraceable tasks */ + if (fs_info->hide_pid == HIDEPID_NOT_PTRACEABLE) { + if (!has_pid_permissions(fs_info, task, HIDEPID_NO_ACCESS)) + goto out_put_task; + } + + result = proc_pid_instantiate(dentry, task, NULL); +out_put_task: + put_task_struct(task); +out: + return result; +} + +/* + * Find the first task with tgid >= tgid + * + */ +struct tgid_iter { + unsigned int tgid; + struct task_struct *task; + struct pid_namespace *const pid_ns; +}; + +static struct tgid_iter +make_tgid_iter(unsigned int init_tgid, struct pid_namespace *pid_ns) +{ + return (struct tgid_iter){ + .tgid = init_tgid - 1, + .pid_ns = pid_ns, + }; +} + +static bool next_tgid(struct tgid_iter *it) +{ + if (it->task) { + put_task_struct(it->task); + it->task = NULL; + } + + rcu_read_lock(); + while (1) { + it->tgid += 1; + const auto pid = find_ge_pid(it->tgid, it->pid_ns); + if (pid) { + it->tgid = pid_nr_ns(pid, it->pid_ns); + it->task = pid_task(pid, PIDTYPE_TGID); + if (it->task) { + get_task_struct(it->task); + rcu_read_unlock(); + return true; + } + } else { + rcu_read_unlock(); + return false; + } + } +} + +#define TGID_OFFSET (FIRST_PROCESS_ENTRY + 2) + +/* for the /proc/ directory itself, after non-process stuff has been done */ +int proc_pid_readdir(struct file *file, struct dir_context *ctx) +{ + struct proc_fs_info *fs_info = proc_sb_info(file_inode(file)->i_sb); + struct pid_namespace *pid_ns = proc_pid_ns(file_inode(file)->i_sb); + loff_t pos = ctx->pos; + + if (pos >= PID_MAX_LIMIT + TGID_OFFSET) + return 0; + + if (pos == TGID_OFFSET - 2) { + if (!dir_emit(ctx, "self", 4, self_inum, DT_LNK)) + return 0; + ctx->pos = pos = pos + 1; + } + if (pos == TGID_OFFSET - 1) { + if (!dir_emit(ctx, "thread-self", 11, thread_self_inum, DT_LNK)) + return 0; + ctx->pos = pos = pos + 1; + } + + auto iter = make_tgid_iter(pos - TGID_OFFSET, pid_ns); + while (next_tgid(&iter)) { + char name[10 + 1]; + unsigned int len; + + cond_resched(); + if (!has_pid_permissions(fs_info, iter.task, HIDEPID_INVISIBLE)) + continue; + + len = snprintf(name, sizeof(name), "%u", iter.tgid); + ctx->pos = iter.tgid + TGID_OFFSET; + if (!proc_fill_cache(file, ctx, name, len, + proc_pid_instantiate, iter.task, NULL)) { + put_task_struct(iter.task); + return 0; + } + } + ctx->pos = PID_MAX_LIMIT + TGID_OFFSET; + return 0; +} + +/* + * proc_tid_comm_permission is a special permission function exclusively + * used for the node /proc//task//comm. + * It bypasses generic permission checks in the case where a task of the same + * task group attempts to access the node. + * The rationale behind this is that glibc and bionic access this node for + * cross thread naming (pthread_set/getname_np(!self)). However, if + * PR_SET_DUMPABLE gets set to 0 this node among others becomes uid=0 gid=0, + * which locks out the cross thread naming implementation. + * This function makes sure that the node is always accessible for members of + * same thread group. + */ +static int proc_tid_comm_permission(struct mnt_idmap *idmap, + struct inode *inode, int mask) +{ + bool is_same_tgroup; + struct task_struct *task; + + task = get_proc_task(inode); + if (!task) + return -ESRCH; + is_same_tgroup = same_thread_group(current, task); + put_task_struct(task); + + if (likely(is_same_tgroup && !(mask & MAY_EXEC))) { + /* This file (/proc//task//comm) can always be + * read or written by the members of the corresponding + * thread group. + */ + return 0; + } + + return generic_permission(&nop_mnt_idmap, inode, mask); +} + +static const struct inode_operations proc_tid_comm_inode_operations = { + .setattr = proc_nochmod_setattr, + .permission = proc_tid_comm_permission, +}; + +/* + * Tasks + */ +static const struct pid_entry tid_base_stuff[] = { + DIR("fd", S_IRUSR|S_IXUSR, proc_fd_inode_operations, proc_fd_operations), + DIR("fdinfo", S_IRUGO|S_IXUGO, proc_fdinfo_inode_operations, proc_fdinfo_operations), + DIR("ns", S_IRUSR|S_IXUGO, proc_ns_dir_inode_operations, proc_ns_dir_operations), +#ifdef CONFIG_NET + DIR("net", S_IRUGO|S_IXUGO, proc_net_inode_operations, proc_net_operations), +#endif + REG("environ", S_IRUSR, proc_environ_operations), + REG("auxv", S_IRUSR, proc_auxv_operations), + ONE("status", S_IRUGO, proc_pid_status), + ONE("personality", S_IRUSR, proc_pid_personality), + ONE("limits", S_IRUGO, proc_pid_limits), + REG("sched", S_IRUGO|S_IWUSR, proc_pid_sched_operations), + NOD("comm", S_IFREG|S_IRUGO|S_IWUSR, + &proc_tid_comm_inode_operations, + &proc_pid_set_comm_operations, {}), +#ifdef CONFIG_HAVE_ARCH_TRACEHOOK + ONE("syscall", S_IRUSR, proc_pid_syscall), +#endif + REG("cmdline", S_IRUGO, proc_pid_cmdline_ops), + ONE("stat", S_IRUGO, proc_tid_stat), + ONE("statm", S_IRUGO, proc_pid_statm), + REG("maps", S_IRUGO, proc_pid_maps_operations), +#ifdef CONFIG_PROC_CHILDREN + REG("children", S_IRUGO, proc_tid_children_operations), +#endif +#ifdef CONFIG_NUMA + REG("numa_maps", S_IRUGO, proc_pid_numa_maps_operations), +#endif + REG("mem", S_IRUSR|S_IWUSR, proc_mem_operations), + LNK("cwd", proc_cwd_link), + LNK("root", proc_root_link), + LNK("exe", proc_exe_link), + REG("mounts", S_IRUGO, proc_mounts_operations), + REG("mountinfo", S_IRUGO, proc_mountinfo_operations), +#ifdef CONFIG_PROC_PAGE_MONITOR + REG("clear_refs", S_IWUSR, proc_clear_refs_operations), + REG("smaps", S_IRUGO, proc_pid_smaps_operations), + REG("smaps_rollup", S_IRUGO, proc_pid_smaps_rollup_operations), + REG("pagemap", S_IRUSR, proc_pagemap_operations), +#endif +#ifdef CONFIG_SECURITY + DIR("attr", S_IRUGO|S_IXUGO, proc_attr_dir_inode_operations, proc_attr_dir_operations), +#endif +#ifdef CONFIG_KALLSYMS + ONE("wchan", S_IRUGO, proc_pid_wchan), +#endif +#ifdef CONFIG_STACKTRACE + ONE("stack", S_IRUSR, proc_pid_stack), +#endif +#ifdef CONFIG_SCHED_INFO + ONE("schedstat", S_IRUGO, proc_pid_schedstat), +#endif +#ifdef CONFIG_LATENCYTOP + REG("latency", S_IRUGO, proc_lstats_operations), +#endif +#ifdef CONFIG_PROC_PID_CPUSET + ONE("cpuset", S_IRUGO, proc_cpuset_show), +#endif +#ifdef CONFIG_CGROUPS + ONE("cgroup", S_IRUGO, proc_cgroup_show), +#endif +#ifdef CONFIG_PROC_CPU_RESCTRL + ONE("cpu_resctrl_groups", S_IRUGO, proc_resctrl_show), +#endif + ONE("oom_score", S_IRUGO, proc_oom_score), + REG("oom_adj", S_IRUGO|S_IWUSR, proc_oom_adj_operations), + REG("oom_score_adj", S_IRUGO|S_IWUSR, proc_oom_score_adj_operations), +#ifdef CONFIG_AUDIT + REG("loginuid", S_IWUSR|S_IRUGO, proc_loginuid_operations), + REG("sessionid", S_IRUGO, proc_sessionid_operations), +#endif +#ifdef CONFIG_FAULT_INJECTION + REG("make-it-fail", S_IRUGO|S_IWUSR, proc_fault_inject_operations), + REG("fail-nth", 0644, proc_fail_nth_operations), +#endif +#ifdef CONFIG_TASK_IO_ACCOUNTING + ONE("io", S_IRUSR, proc_tid_io_accounting), +#endif +#ifdef CONFIG_USER_NS + REG("uid_map", S_IRUGO|S_IWUSR, proc_uid_map_operations), + REG("gid_map", S_IRUGO|S_IWUSR, proc_gid_map_operations), + REG("projid_map", S_IRUGO|S_IWUSR, proc_projid_map_operations), + REG("setgroups", S_IRUGO|S_IWUSR, proc_setgroups_operations), +#endif +#ifdef CONFIG_LIVEPATCH + ONE("patch_state", S_IRUSR, proc_pid_patch_state), +#endif +#ifdef CONFIG_PROC_PID_ARCH_STATUS + ONE("arch_status", S_IRUGO, proc_pid_arch_status), +#endif +#ifdef CONFIG_SECCOMP_CACHE_DEBUG + ONE("seccomp_cache", S_IRUSR, proc_pid_seccomp_cache), +#endif +#ifdef CONFIG_KSM + ONE("ksm_merging_pages", S_IRUSR, proc_pid_ksm_merging_pages), + ONE("ksm_stat", S_IRUSR, proc_pid_ksm_stat), +#endif +}; + +static int proc_tid_base_readdir(struct file *file, struct dir_context *ctx) +{ + return proc_pident_readdir(file, ctx, + tid_base_stuff, ARRAY_SIZE(tid_base_stuff)); +} + +static struct dentry *proc_tid_base_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) +{ + return proc_pident_lookup(dir, dentry, + tid_base_stuff, + tid_base_stuff + ARRAY_SIZE(tid_base_stuff)); +} + +static const struct file_operations proc_tid_base_operations = { + .read = generic_read_dir, + .iterate_shared = proc_tid_base_readdir, + .llseek = generic_file_llseek, +}; + +static const struct inode_operations proc_tid_base_inode_operations = { + .lookup = proc_tid_base_lookup, + .getattr = pid_getattr, + .setattr = proc_nochmod_setattr, +}; + +static struct dentry *proc_task_instantiate(struct dentry *dentry, + struct task_struct *task, const void *ptr) +{ + struct inode *inode; + inode = proc_pid_make_base_inode(dentry->d_sb, task, + S_IFDIR | S_IRUGO | S_IXUGO); + if (!inode) + return ERR_PTR(-ENOENT); + + inode->i_op = &proc_tid_base_inode_operations; + inode->i_fop = &proc_tid_base_operations; + inode->i_flags |= S_IMMUTABLE; + + set_nlink(inode, nlink_tid); + pid_update_inode(task, inode); + + return d_splice_alias_ops(inode, dentry, &pid_dentry_operations); +} + +static struct dentry *proc_task_lookup(struct inode *dir, struct dentry * dentry, unsigned int flags) +{ + struct task_struct *task; + struct task_struct *leader = get_proc_task(dir); + unsigned tid; + struct proc_fs_info *fs_info; + struct pid_namespace *ns; + struct dentry *result = ERR_PTR(-ENOENT); + + if (!leader) + goto out_no_task; + + tid = name_to_int(&dentry->d_name); + if (tid == ~0U) + goto out; + + fs_info = proc_sb_info(dentry->d_sb); + ns = fs_info->pid_ns; + rcu_read_lock(); + task = find_task_by_pid_ns(tid, ns); + if (task) + get_task_struct(task); + rcu_read_unlock(); + if (!task) + goto out; + if (!same_thread_group(leader, task)) + goto out_drop_task; + + result = proc_task_instantiate(dentry, task, NULL); +out_drop_task: + put_task_struct(task); +out: + put_task_struct(leader); +out_no_task: + return result; +} + +/* + * Find the first tid of a thread group to return to user space. + * + * Usually this is just the thread group leader, but if the users + * buffer was too small or there was a seek into the middle of the + * directory we have more work todo. + * + * In the case of a short read we start with find_task_by_pid. + * + * In the case of a seek we start with the leader and walk nr + * threads past it. + */ +static struct task_struct *first_tid(struct pid *pid, int tid, loff_t f_pos, + struct pid_namespace *ns) +{ + struct task_struct *pos, *task; + unsigned long nr = f_pos; + + if (nr != f_pos) /* 32bit overflow? */ + return NULL; + + rcu_read_lock(); + task = pid_task(pid, PIDTYPE_PID); + if (!task) + goto fail; + + /* Attempt to start with the tid of a thread */ + if (tid && nr) { + pos = find_task_by_pid_ns(tid, ns); + if (pos && same_thread_group(pos, task)) + goto found; + } + + /* If nr exceeds the number of threads there is nothing todo */ + if (nr >= get_nr_threads(task)) + goto fail; + + /* If we haven't found our starting place yet start + * with the leader and walk nr threads forward. + */ + for_each_thread(task, pos) { + if (!nr--) + goto found; + } +fail: + pos = NULL; + goto out; +found: + get_task_struct(pos); +out: + rcu_read_unlock(); + return pos; +} + +/* + * Find the next thread in the thread list. + * Return NULL if there is an error or no next thread. + * + * The reference to the input task_struct is released. + */ +static struct task_struct *next_tid(struct task_struct *start) +{ + struct task_struct *pos = NULL; + rcu_read_lock(); + if (pid_alive(start)) { + pos = __next_thread(start); + if (pos) + get_task_struct(pos); + } + rcu_read_unlock(); + put_task_struct(start); + return pos; +} + +/* for the /proc/TGID/task/ directories */ +static int proc_task_readdir(struct file *file, struct dir_context *ctx) +{ + struct inode *inode = file_inode(file); + struct task_struct *task; + struct pid_namespace *ns; + int tid; + + if (proc_inode_is_dead(inode)) + return -ENOENT; + + if (!dir_emit_dots(file, ctx)) + return 0; + + /* We cache the tgid value that the last readdir call couldn't + * return and lseek resets it to 0. + */ + ns = proc_pid_ns(inode->i_sb); + tid = (int)(intptr_t)file->private_data; + file->private_data = NULL; + for (task = first_tid(proc_pid(inode), tid, ctx->pos - 2, ns); + task; + task = next_tid(task), ctx->pos++) { + char name[10 + 1]; + unsigned int len; + + tid = task_pid_nr_ns(task, ns); + if (!tid) + continue; /* The task has just exited. */ + len = snprintf(name, sizeof(name), "%d", tid); + if (!proc_fill_cache(file, ctx, name, len, + proc_task_instantiate, task, NULL)) { + /* returning this tgid failed, save it as the first + * pid for the next readir call */ + file->private_data = (void *)(intptr_t)tid; + put_task_struct(task); + break; + } + } + + return 0; +} + +static int proc_task_getattr(struct mnt_idmap *idmap, + const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int query_flags) +{ + struct inode *inode = d_inode(path->dentry); + struct task_struct *p = get_proc_task(inode); + generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat); + + if (p) { + stat->nlink += get_nr_threads(p); + put_task_struct(p); + } + + return 0; +} + +/* + * proc_task_readdir() set @file->private_data to a positive integer + * value, so casting that to u64 is safe. generic_llseek_cookie() will + * set @cookie to 0, so casting to an int is safe. The WARN_ON_ONCE() is + * here to catch any unexpected change in behavior either in + * proc_task_readdir() or generic_llseek_cookie(). + */ +static loff_t proc_dir_llseek(struct file *file, loff_t offset, int whence) +{ + u64 cookie = (u64)(intptr_t)file->private_data; + loff_t off; + + off = generic_llseek_cookie(file, offset, whence, &cookie); + WARN_ON_ONCE(cookie > INT_MAX); + file->private_data = (void *)(intptr_t)cookie; /* serialized by f_pos_lock */ + return off; +} + +static const struct inode_operations proc_task_inode_operations = { + .lookup = proc_task_lookup, + .getattr = proc_task_getattr, + .setattr = proc_nochmod_setattr, + .permission = proc_pid_permission, +}; + +static const struct file_operations proc_task_operations = { + .read = generic_read_dir, + .iterate_shared = proc_task_readdir, + .llseek = proc_dir_llseek, +}; + +void __init set_proc_pid_nlink(void) +{ + nlink_tid = pid_entry_nlink(tid_base_stuff, ARRAY_SIZE(tid_base_stuff)); + nlink_tgid = pid_entry_nlink(tgid_base_stuff, ARRAY_SIZE(tgid_base_stuff)); +} diff --git a/fs/proc/bootconfig.c b/fs/proc/bootconfig.c new file mode 100644 index 000000000..87dcaae32 --- /dev/null +++ b/fs/proc/bootconfig.c @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * /proc/bootconfig - Extra boot configuration + */ +#include +#include +#include +#include +#include +#include +#include + +static char *saved_boot_config; + +static int boot_config_proc_show(struct seq_file *m, void *v) +{ + if (saved_boot_config) + seq_puts(m, saved_boot_config); + return 0; +} + +/* Rest size of buffer */ +#define rest(dst, end) ((end) > (dst) ? (end) - (dst) : 0) + +/* Return the needed total length if @size is 0 */ +static int __init copy_xbc_key_value_list(char *dst, size_t size) +{ + struct xbc_node *leaf, *vnode; + char *key, *end = dst + size; + const char *val; + char q; + int ret = 0; + + key = kzalloc(XBC_KEYLEN_MAX, GFP_KERNEL); + if (!key) + return -ENOMEM; + + xbc_for_each_key_value(leaf, val) { + ret = xbc_node_compose_key(leaf, key, XBC_KEYLEN_MAX); + if (ret < 0) + break; + ret = snprintf(dst, rest(dst, end), "%s = ", key); + if (ret < 0) + break; + dst += ret; + vnode = xbc_node_get_child(leaf); + if (vnode) { + xbc_array_for_each_value(vnode, val) { + if (strchr(val, '"')) + q = '\''; + else + q = '"'; + ret = snprintf(dst, rest(dst, end), "%c%s%c%s", + q, val, q, xbc_node_is_array(vnode) ? ", " : "\n"); + if (ret < 0) + goto out; + dst += ret; + } + } else { + ret = snprintf(dst, rest(dst, end), "\"\"\n"); + if (ret < 0) + break; + dst += ret; + } + } + if (cmdline_has_extra_options() && ret >= 0 && boot_command_line[0]) { + ret = snprintf(dst, rest(dst, end), "# Parameters from bootloader:\n# %s\n", + boot_command_line); + if (ret > 0) + dst += ret; + } +out: + kfree(key); + + return ret < 0 ? ret : dst - (end - size); +} + +static int __init proc_boot_config_init(void) +{ + int len; + + len = copy_xbc_key_value_list(NULL, 0); + if (len < 0) + return len; + + if (len > 0) { + saved_boot_config = kzalloc(len + 1, GFP_KERNEL); + if (!saved_boot_config) + return -ENOMEM; + + len = copy_xbc_key_value_list(saved_boot_config, len + 1); + if (len < 0) { + kfree(saved_boot_config); + return len; + } + } + + proc_create_single("bootconfig", 0, NULL, boot_config_proc_show); + + return 0; +} +fs_initcall(proc_boot_config_init); diff --git a/fs/proc/cmdline.c b/fs/proc/cmdline.c new file mode 100644 index 000000000..a6f761219 --- /dev/null +++ b/fs/proc/cmdline.c @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include "internal.h" + +static int cmdline_proc_show(struct seq_file *m, void *v) +{ + seq_puts(m, saved_command_line); + seq_putc(m, '\n'); + return 0; +} + +static int __init proc_cmdline_init(void) +{ + struct proc_dir_entry *pde; + + pde = proc_create_single("cmdline", 0, NULL, cmdline_proc_show); + pde_make_permanent(pde); + pde->size = saved_command_line_len + 1; + return 0; +} +fs_initcall(proc_cmdline_init); diff --git a/fs/proc/consoles.c b/fs/proc/consoles.c new file mode 100644 index 000000000..b7cab1ad9 --- /dev/null +++ b/fs/proc/consoles.c @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2010 Werner Fink, Jiri Slaby + */ + +#include +#include +#include +#include +#include + +/* + * This is handler for /proc/consoles + */ +static int show_console_dev(struct seq_file *m, void *v) +{ + static const struct { + short flag; + char name; + } con_flags[] = { + { CON_ENABLED, 'E' }, + { CON_CONSDEV, 'C' }, + { CON_BOOT, 'B' }, + { CON_NBCON, 'N' }, + { CON_PRINTBUFFER, 'p' }, + { CON_BRL, 'b' }, + { CON_ANYTIME, 'a' }, + }; + char flags[ARRAY_SIZE(con_flags) + 1]; + struct console *con = v; + unsigned int a; + dev_t dev = 0; + + if (con->device) { + const struct tty_driver *driver; + int index; + + /* + * Take console_lock to serialize device() callback with + * other console operations. For example, fg_console is + * modified under console_lock when switching vt. + */ + console_lock(); + driver = con->device(con, &index); + console_unlock(); + + if (driver) { + dev = MKDEV(driver->major, driver->minor_start); + dev += index; + } + } + + for (a = 0; a < ARRAY_SIZE(con_flags); a++) + flags[a] = (con->flags & con_flags[a].flag) ? + con_flags[a].name : ' '; + flags[a] = 0; + + seq_setwidth(m, 21 - 1); + seq_printf(m, "%s%d", con->name, con->index); + seq_pad(m, ' '); + seq_printf(m, "%c%c%c (%s)", con->read ? 'R' : '-', + ((con->flags & CON_NBCON) || con->write) ? 'W' : '-', + con->unblank ? 'U' : '-', flags); + if (dev) + seq_printf(m, " %4d:%d", MAJOR(dev), MINOR(dev)); + + seq_putc(m, '\n'); + return 0; +} + +static void *c_start(struct seq_file *m, loff_t *pos) + __acquires(&console_mutex) +{ + struct console *con; + loff_t off = 0; + + /* + * Hold the console_list_lock to guarantee safe traversal of the + * console list. SRCU cannot be used because there is no + * place to store the SRCU cookie. + */ + console_list_lock(); + for_each_console(con) + if (off++ == *pos) + break; + + return con; +} + +static void *c_next(struct seq_file *m, void *v, loff_t *pos) +{ + struct console *con = v; + + ++*pos; + return hlist_entry_safe(con->node.next, struct console, node); +} + +static void c_stop(struct seq_file *m, void *v) + __releases(&console_mutex) +{ + console_list_unlock(); +} + +static const struct seq_operations consoles_op = { + .start = c_start, + .next = c_next, + .stop = c_stop, + .show = show_console_dev +}; + +static int __init proc_consoles_init(void) +{ + proc_create_seq("consoles", 0, NULL, &consoles_op); + return 0; +} +fs_initcall(proc_consoles_init); diff --git a/fs/proc/cpuinfo.c b/fs/proc/cpuinfo.c new file mode 100644 index 000000000..f38bda5b8 --- /dev/null +++ b/fs/proc/cpuinfo.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include + +extern const struct seq_operations cpuinfo_op; + +static int cpuinfo_open(struct inode *inode, struct file *file) +{ + return seq_open(file, &cpuinfo_op); +} + +static const struct proc_ops cpuinfo_proc_ops = { + .proc_flags = PROC_ENTRY_PERMANENT, + .proc_open = cpuinfo_open, + .proc_read_iter = seq_read_iter, + .proc_lseek = seq_lseek, + .proc_release = seq_release, +}; + +static int __init proc_cpuinfo_init(void) +{ + proc_create("cpuinfo", 0, NULL, &cpuinfo_proc_ops); + return 0; +} +fs_initcall(proc_cpuinfo_init); diff --git a/fs/proc/devices.c b/fs/proc/devices.c new file mode 100644 index 000000000..fe7bfcb7d --- /dev/null +++ b/fs/proc/devices.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include "internal.h" + +static int devinfo_show(struct seq_file *f, void *v) +{ + int i = *(loff_t *) v; + + if (i < CHRDEV_MAJOR_MAX) { + if (i == 0) + seq_puts(f, "Character devices:\n"); + chrdev_show(f, i); + } +#ifdef CONFIG_BLOCK + else { + i -= CHRDEV_MAJOR_MAX; + if (i == 0) + seq_puts(f, "\nBlock devices:\n"); + blkdev_show(f, i); + } +#endif + return 0; +} + +static void *devinfo_start(struct seq_file *f, loff_t *pos) +{ + if (*pos < (BLKDEV_MAJOR_MAX + CHRDEV_MAJOR_MAX)) + return pos; + return NULL; +} + +static void *devinfo_next(struct seq_file *f, void *v, loff_t *pos) +{ + (*pos)++; + if (*pos >= (BLKDEV_MAJOR_MAX + CHRDEV_MAJOR_MAX)) + return NULL; + return pos; +} + +static void devinfo_stop(struct seq_file *f, void *v) +{ + /* Nothing to do */ +} + +static const struct seq_operations devinfo_ops = { + .start = devinfo_start, + .next = devinfo_next, + .stop = devinfo_stop, + .show = devinfo_show +}; + +static int __init proc_devices_init(void) +{ + struct proc_dir_entry *pde; + + pde = proc_create_seq("devices", 0, NULL, &devinfo_ops); + pde_make_permanent(pde); + return 0; +} +fs_initcall(proc_devices_init); diff --git a/fs/proc/fd.c b/fs/proc/fd.c new file mode 100644 index 000000000..0f9a1556f --- /dev/null +++ b/fs/proc/fd.c @@ -0,0 +1,407 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "../mount.h" +#include "internal.h" +#include "fd.h" + +static int seq_show(struct seq_file *m, void *v) +{ + struct files_struct *files = NULL; + int f_flags = 0, ret = -ENOENT; + struct file *file = NULL; + struct task_struct *task; + + task = get_proc_task(m->private); + if (!task) + return -ENOENT; + + task_lock(task); + files = task->files; + if (files) { + unsigned int fd = proc_fd(m->private); + + spin_lock(&files->file_lock); + file = files_lookup_fd_locked(files, fd); + if (file) { + f_flags = file->f_flags; + if (close_on_exec(fd, files)) + f_flags |= O_CLOEXEC; + + get_file(file); + ret = 0; + } + spin_unlock(&files->file_lock); + } + task_unlock(task); + put_task_struct(task); + + if (ret) + return ret; + + seq_printf(m, "pos:\t%lli\nflags:\t0%o\nmnt_id:\t%i\nino:\t%llu\n", + (long long)file->f_pos, f_flags, + real_mount(file->f_path.mnt)->mnt_id, + file_inode(file)->i_ino); + + /* show_fd_locks() never dereferences files, so a stale value is safe */ + show_fd_locks(m, file, files); + if (seq_has_overflowed(m)) + goto out; + + if (file->f_op->show_fdinfo) + file->f_op->show_fdinfo(m, file); + +out: + fput(file); + return 0; +} + +static int seq_fdinfo_open(struct inode *inode, struct file *file) +{ + return single_open(file, seq_show, inode); +} + +/* + * Shared /proc/pid/fdinfo and /proc/pid/fdinfo/fd permission helper to ensure + * that the current task has PTRACE_MODE_READ in addition to the normal + * POSIX-like checks. + */ +static int proc_fdinfo_permission(struct mnt_idmap *idmap, struct inode *inode, + int mask) +{ + bool allowed = false; + struct task_struct *task = get_proc_task(inode); + + if (!task) + return -ESRCH; + + allowed = ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS); + put_task_struct(task); + + if (!allowed) + return -EACCES; + + return generic_permission(idmap, inode, mask); +} + +static const struct inode_operations proc_fdinfo_file_inode_operations = { + .permission = proc_fdinfo_permission, + .setattr = proc_nochmod_setattr, +}; + +static const struct file_operations proc_fdinfo_file_operations = { + .open = seq_fdinfo_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static bool tid_fd_mode(struct task_struct *task, unsigned fd, fmode_t *mode) +{ + struct file *file; + + file = fget_task(task, fd); + if (file) { + *mode = file->f_mode; + fput(file); + } + return !!file; +} + +static void tid_fd_update_inode(struct task_struct *task, struct inode *inode, + fmode_t f_mode) +{ + task_dump_owner(task, 0, &inode->i_uid, &inode->i_gid); + + if (S_ISLNK(inode->i_mode)) { + unsigned i_mode = S_IFLNK; + if (f_mode & FMODE_READ) + i_mode |= S_IRUSR | S_IXUSR; + if (f_mode & FMODE_WRITE) + i_mode |= S_IWUSR | S_IXUSR; + inode->i_mode = i_mode; + } + security_task_to_inode(task, inode); +} + +static int tid_fd_revalidate(struct inode *dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + struct task_struct *task; + struct inode *inode; + unsigned int fd; + + if (flags & LOOKUP_RCU) + return -ECHILD; + + inode = d_inode(dentry); + task = get_proc_task(inode); + fd = proc_fd(inode); + + if (task) { + fmode_t f_mode; + if (tid_fd_mode(task, fd, &f_mode)) { + tid_fd_update_inode(task, inode, f_mode); + put_task_struct(task); + return 1; + } + put_task_struct(task); + } + return 0; +} + +static const struct dentry_operations tid_fd_dentry_operations = { + .d_revalidate = tid_fd_revalidate, + .d_delete = pid_delete_dentry, +}; + +static int proc_fd_link(struct dentry *dentry, struct path *path, + struct task_struct *task) +{ + int ret = -ENOENT; + unsigned int fd = proc_fd(d_inode(dentry)); + struct file *fd_file; + + fd_file = fget_task(task, fd); + if (fd_file) { + *path = fd_file->f_path; + path_get(&fd_file->f_path); + ret = 0; + fput(fd_file); + } + + return ret; +} + +struct fd_data { + fmode_t mode; + unsigned fd; +}; + +static struct dentry *proc_fd_instantiate(struct dentry *dentry, + struct task_struct *task, const void *ptr) +{ + const struct fd_data *data = ptr; + struct proc_inode *ei; + struct inode *inode; + + inode = proc_pid_make_inode(dentry->d_sb, task, S_IFLNK); + if (!inode) + return ERR_PTR(-ENOENT); + + ei = PROC_I(inode); + ei->fd = data->fd; + + inode->i_op = &proc_pid_link_inode_operations; + inode->i_size = 64; + + ei->op.proc_get_link = proc_fd_link; + tid_fd_update_inode(task, inode, data->mode); + + return proc_splice_unmountable(inode, dentry, + &tid_fd_dentry_operations); +} + +static struct dentry *proc_lookupfd_common(struct inode *dir, + struct dentry *dentry, + instantiate_t instantiate) +{ + struct task_struct *task = get_proc_task(dir); + struct fd_data data = {.fd = name_to_int(&dentry->d_name)}; + struct dentry *result = ERR_PTR(-ENOENT); + + if (!task) + goto out_no_task; + if (data.fd == ~0U) + goto out; + if (!tid_fd_mode(task, data.fd, &data.mode)) + goto out; + + result = instantiate(dentry, task, &data); +out: + put_task_struct(task); +out_no_task: + return result; +} + +static int proc_readfd_common(struct file *file, struct dir_context *ctx, + instantiate_t instantiate) +{ + struct task_struct *p = get_proc_task(file_inode(file)); + unsigned int fd; + + if (!p) + return -ENOENT; + + if (!dir_emit_dots(file, ctx)) + goto out; + + for (fd = ctx->pos - 2;; fd++) { + struct file *f; + struct fd_data data; + char name[10 + 1]; + unsigned int len; + + f = fget_task_next(p, &fd); + ctx->pos = fd + 2LL; + if (!f) + break; + data.mode = f->f_mode; + fput(f); + data.fd = fd; + + len = snprintf(name, sizeof(name), "%u", fd); + if (!proc_fill_cache(file, ctx, + name, len, instantiate, p, + &data)) + break; + cond_resched(); + } +out: + put_task_struct(p); + return 0; +} + +static int proc_readfd_count(struct inode *inode, loff_t *count) +{ + struct task_struct *p = get_proc_task(inode); + struct fdtable *fdt; + + if (!p) + return -ENOENT; + + task_lock(p); + if (p->files) { + rcu_read_lock(); + + fdt = files_fdtable(p->files); + *count = bitmap_weight(fdt->open_fds, fdt->max_fds); + + rcu_read_unlock(); + } + task_unlock(p); + + put_task_struct(p); + + return 0; +} + +static int proc_fd_iterate(struct file *file, struct dir_context *ctx) +{ + return proc_readfd_common(file, ctx, proc_fd_instantiate); +} + +const struct file_operations proc_fd_operations = { + .read = generic_read_dir, + .iterate_shared = proc_fd_iterate, + .llseek = generic_file_llseek, +}; + +static struct dentry *proc_lookupfd(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + return proc_lookupfd_common(dir, dentry, proc_fd_instantiate); +} + +/* + * /proc/pid/fd needs a special permission handler so that a process can still + * access /proc/self/fd after it has executed a setuid(). + */ +int proc_fd_permission(struct mnt_idmap *idmap, + struct inode *inode, int mask) +{ + struct task_struct *p; + int rv; + + rv = generic_permission(&nop_mnt_idmap, inode, mask); + if (rv == 0) + return rv; + + rcu_read_lock(); + p = pid_task(proc_pid(inode), PIDTYPE_PID); + if (p && same_thread_group(p, current)) + rv = 0; + rcu_read_unlock(); + + return rv; +} + +static int proc_fd_getattr(struct mnt_idmap *idmap, + const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int query_flags) +{ + struct inode *inode = d_inode(path->dentry); + + generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat); + return proc_readfd_count(inode, &stat->size); +} + +const struct inode_operations proc_fd_inode_operations = { + .lookup = proc_lookupfd, + .permission = proc_fd_permission, + .getattr = proc_fd_getattr, + .setattr = proc_nochmod_setattr, +}; + +static struct dentry *proc_fdinfo_instantiate(struct dentry *dentry, + struct task_struct *task, const void *ptr) +{ + const struct fd_data *data = ptr; + struct proc_inode *ei; + struct inode *inode; + + inode = proc_pid_make_inode(dentry->d_sb, task, S_IFREG | S_IRUGO); + if (!inode) + return ERR_PTR(-ENOENT); + + ei = PROC_I(inode); + ei->fd = data->fd; + + inode->i_op = &proc_fdinfo_file_inode_operations; + + inode->i_fop = &proc_fdinfo_file_operations; + tid_fd_update_inode(task, inode, 0); + + return proc_splice_unmountable(inode, dentry, + &tid_fd_dentry_operations); +} + +static struct dentry * +proc_lookupfdinfo(struct inode *dir, struct dentry *dentry, unsigned int flags) +{ + return proc_lookupfd_common(dir, dentry, proc_fdinfo_instantiate); +} + +static int proc_fdinfo_iterate(struct file *file, struct dir_context *ctx) +{ + return proc_readfd_common(file, ctx, + proc_fdinfo_instantiate); +} + +const struct inode_operations proc_fdinfo_inode_operations = { + .lookup = proc_lookupfdinfo, + .permission = proc_fdinfo_permission, + .setattr = proc_nochmod_setattr, +}; + +const struct file_operations proc_fdinfo_operations = { + .read = generic_read_dir, + .iterate_shared = proc_fdinfo_iterate, + .llseek = generic_file_llseek, +}; diff --git a/fs/proc/fd.h b/fs/proc/fd.h new file mode 100644 index 000000000..7e7265f7e --- /dev/null +++ b/fs/proc/fd.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PROCFS_FD_H__ +#define __PROCFS_FD_H__ + +#include + +extern const struct file_operations proc_fd_operations; +extern const struct inode_operations proc_fd_inode_operations; + +extern const struct file_operations proc_fdinfo_operations; +extern const struct inode_operations proc_fdinfo_inode_operations; + +extern int proc_fd_permission(struct mnt_idmap *idmap, + struct inode *inode, int mask); + +static inline unsigned int proc_fd(struct inode *inode) +{ + return PROC_I(inode)->fd; +} + +#endif /* __PROCFS_FD_H__ */ diff --git a/fs/proc/generic.c b/fs/proc/generic.c new file mode 100644 index 000000000..26086a283 --- /dev/null +++ b/fs/proc/generic.c @@ -0,0 +1,856 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * proc/fs/generic.c --- generic routines for the proc-fs + * + * This file contains generic proc-fs routines for handling + * directories and files. + * + * Copyright (C) 1991, 1992 Linus Torvalds. + * Copyright (C) 1997 Theodore Ts'o + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +static DEFINE_RWLOCK(proc_subdir_lock); + +struct kmem_cache *proc_dir_entry_cache __ro_after_init; + +void pde_free(struct proc_dir_entry *pde) +{ + if (S_ISLNK(pde->mode)) + kfree(pde->data); + if (pde->name != pde->inline_name) + kfree(pde->name); + kmem_cache_free(proc_dir_entry_cache, pde); +} + +static int proc_match(const char *name, struct proc_dir_entry *de, unsigned int len) +{ + if (len < de->namelen) + return -1; + if (len > de->namelen) + return 1; + + return memcmp(name, de->name, len); +} + +static struct proc_dir_entry *pde_subdir_first(struct proc_dir_entry *dir) +{ + return rb_entry_safe(rb_first(&dir->subdir), struct proc_dir_entry, + subdir_node); +} + +static struct proc_dir_entry *pde_subdir_next(struct proc_dir_entry *dir) +{ + return rb_entry_safe(rb_next(&dir->subdir_node), struct proc_dir_entry, + subdir_node); +} + +static struct proc_dir_entry *pde_subdir_find(struct proc_dir_entry *dir, + const char *name, + unsigned int len) +{ + struct rb_node *node = dir->subdir.rb_node; + + while (node) { + struct proc_dir_entry *de = rb_entry(node, + struct proc_dir_entry, + subdir_node); + int result = proc_match(name, de, len); + + if (result < 0) + node = node->rb_left; + else if (result > 0) + node = node->rb_right; + else + return de; + } + return NULL; +} + +static bool pde_subdir_insert(struct proc_dir_entry *dir, + struct proc_dir_entry *de) +{ + struct rb_root *root = &dir->subdir; + struct rb_node **new = &root->rb_node, *parent = NULL; + + /* Figure out where to put new node */ + while (*new) { + struct proc_dir_entry *this = rb_entry(*new, + struct proc_dir_entry, + subdir_node); + int result = proc_match(de->name, this, de->namelen); + + parent = *new; + if (result < 0) + new = &(*new)->rb_left; + else if (result > 0) + new = &(*new)->rb_right; + else + return false; + } + + /* Add new node and rebalance tree. */ + rb_link_node(&de->subdir_node, parent, new); + rb_insert_color(&de->subdir_node, root); + if (S_ISDIR(de->mode)) + dir->nlink++; + return true; +} + +static int proc_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *iattr) +{ + struct inode *inode = d_inode(dentry); + struct proc_dir_entry *de = PDE(inode); + int error; + + error = setattr_prepare(&nop_mnt_idmap, dentry, iattr); + if (error) + return error; + + setattr_copy(&nop_mnt_idmap, inode, iattr); + + proc_set_user(de, inode->i_uid, inode->i_gid); + de->mode = inode->i_mode; + return 0; +} + +static int proc_getattr(struct mnt_idmap *idmap, + const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int query_flags) +{ + struct inode *inode = d_inode(path->dentry); + struct proc_dir_entry *de = PDE(inode); + if (de) { + nlink_t nlink = READ_ONCE(de->nlink); + if (nlink > 0) { + set_nlink(inode, nlink); + } + } + + generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat); + return 0; +} + +static const struct inode_operations proc_file_inode_operations = { + .setattr = proc_setattr, +}; + +/* + * This function parses a name such as "tty/driver/serial", and + * returns the struct proc_dir_entry for "/proc/tty/driver", and + * returns "serial" in residual. + */ +static int __xlate_proc_name(const char *name, struct proc_dir_entry **ret, + const char **residual) +{ + const char *cp = name, *next; + struct proc_dir_entry *de; + + de = *ret ?: &proc_root; + while ((next = strchr(cp, '/')) != NULL) { + de = pde_subdir_find(de, cp, next - cp); + if (!de) { + WARN(1, "name '%s'\n", name); + return -ENOENT; + } + cp = next + 1; + } + *residual = cp; + *ret = de; + return 0; +} + +static int xlate_proc_name(const char *name, struct proc_dir_entry **ret, + const char **residual) +{ + int rv; + + read_lock(&proc_subdir_lock); + rv = __xlate_proc_name(name, ret, residual); + read_unlock(&proc_subdir_lock); + return rv; +} + +static DEFINE_IDA(proc_inum_ida); + +#define PROC_DYNAMIC_FIRST 0xF0000000U + +/* + * Return an inode number between PROC_DYNAMIC_FIRST and + * 0xffffffff, or zero on failure. + */ +int proc_alloc_inum(unsigned int *inum) +{ + int i; + + i = ida_alloc_max(&proc_inum_ida, UINT_MAX - PROC_DYNAMIC_FIRST, + GFP_KERNEL); + if (i < 0) + return i; + + *inum = PROC_DYNAMIC_FIRST + (unsigned int)i; + return 0; +} + +void proc_free_inum(unsigned int inum) +{ + ida_free(&proc_inum_ida, inum - PROC_DYNAMIC_FIRST); +} + +static int proc_misc_d_revalidate(struct inode *dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + if (flags & LOOKUP_RCU) + return -ECHILD; + + if (atomic_read(&PDE(d_inode(dentry))->in_use) < 0) + return 0; /* revalidate */ + return 1; +} + +static int proc_misc_d_delete(const struct dentry *dentry) +{ + return atomic_read(&PDE(d_inode(dentry))->in_use) < 0; +} + +static const struct dentry_operations proc_misc_dentry_ops = { + .d_revalidate = proc_misc_d_revalidate, + .d_delete = proc_misc_d_delete, +}; + +/* + * Don't create negative dentries here, return -ENOENT by hand + * instead. + */ +struct dentry *proc_lookup_de(struct inode *dir, struct dentry *dentry, + struct proc_dir_entry *de) +{ + struct inode *inode; + + read_lock(&proc_subdir_lock); + de = pde_subdir_find(de, dentry->d_name.name, dentry->d_name.len); + if (de) { + pde_get(de); + read_unlock(&proc_subdir_lock); + inode = proc_get_inode(dir->i_sb, de); + if (!inode) + return ERR_PTR(-ENOMEM); + if (de->flags & PROC_ENTRY_FORCE_LOOKUP) + return d_splice_alias_ops(inode, dentry, + &proc_net_dentry_ops); + return d_splice_alias_ops(inode, dentry, + &proc_misc_dentry_ops); + } + read_unlock(&proc_subdir_lock); + return ERR_PTR(-ENOENT); +} + +struct dentry *proc_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + struct proc_fs_info *fs_info = proc_sb_info(dir->i_sb); + + if (fs_info->pidonly == PROC_PIDONLY_ON) + return ERR_PTR(-ENOENT); + + return proc_lookup_de(dir, dentry, PDE(dir)); +} + +/* + * This returns non-zero if at EOF, so that the /proc + * root directory can use this and check if it should + * continue with the entries.. + * + * Note that the VFS-layer doesn't care about the return + * value of the readdir() call, as long as it's non-negative + * for success.. + */ +int proc_readdir_de(struct file *file, struct dir_context *ctx, + struct proc_dir_entry *de) +{ + int i; + + if (!dir_emit_dots(file, ctx)) + return 0; + + i = ctx->pos - 2; + read_lock(&proc_subdir_lock); + de = pde_subdir_first(de); + for (;;) { + if (!de) { + read_unlock(&proc_subdir_lock); + return 0; + } + if (!i) + break; + de = pde_subdir_next(de); + i--; + } + + do { + struct proc_dir_entry *next; + pde_get(de); + read_unlock(&proc_subdir_lock); + if (!dir_emit(ctx, de->name, de->namelen, + de->low_ino, de->mode >> 12)) { + pde_put(de); + return 0; + } + ctx->pos++; + read_lock(&proc_subdir_lock); + next = pde_subdir_next(de); + pde_put(de); + de = next; + } while (de); + read_unlock(&proc_subdir_lock); + return 1; +} + +int proc_readdir(struct file *file, struct dir_context *ctx) +{ + struct inode *inode = file_inode(file); + struct proc_fs_info *fs_info = proc_sb_info(inode->i_sb); + + if (fs_info->pidonly == PROC_PIDONLY_ON) + return 1; + + return proc_readdir_de(file, ctx, PDE(inode)); +} + +/* + * These are the generic /proc directory operations. They + * use the in-memory "struct proc_dir_entry" tree to parse + * the /proc directory. + */ +static const struct file_operations proc_dir_operations = { + .llseek = generic_file_llseek, + .read = generic_read_dir, + .iterate_shared = proc_readdir, +}; + +static int proc_net_d_revalidate(struct inode *dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + return 0; +} + +const struct dentry_operations proc_net_dentry_ops = { + .d_revalidate = proc_net_d_revalidate, + .d_delete = always_delete_dentry, +}; + +/* + * proc directories can do almost nothing.. + */ +static const struct inode_operations proc_dir_inode_operations = { + .lookup = proc_lookup, + .getattr = proc_getattr, + .setattr = proc_setattr, +}; + +static void pde_set_flags(struct proc_dir_entry *pde) +{ + const struct proc_ops *proc_ops = pde->proc_ops; + + if (!proc_ops) + return; + + if (proc_ops->proc_flags & PROC_ENTRY_PERMANENT) + pde->flags |= PROC_ENTRY_PERMANENT; + if (proc_ops->proc_read_iter) + pde->flags |= PROC_ENTRY_proc_read_iter; +#ifdef CONFIG_COMPAT + if (proc_ops->proc_compat_ioctl) + pde->flags |= PROC_ENTRY_proc_compat_ioctl; +#endif + if (proc_ops->proc_lseek) + pde->flags |= PROC_ENTRY_proc_lseek; +} + +/* returns the registered entry, or frees dp and returns NULL on failure */ +struct proc_dir_entry *proc_register(struct proc_dir_entry *dir, + struct proc_dir_entry *dp) +{ + if (proc_alloc_inum(&dp->low_ino)) + goto out_free_entry; + + if (!S_ISDIR(dp->mode)) + pde_set_flags(dp); + + write_lock(&proc_subdir_lock); + dp->parent = dir; + if (pde_subdir_insert(dir, dp) == false) { + WARN(1, "proc_dir_entry '%s/%s' already registered\n", + dir->name, dp->name); + write_unlock(&proc_subdir_lock); + goto out_free_inum; + } + write_unlock(&proc_subdir_lock); + + return dp; +out_free_inum: + proc_free_inum(dp->low_ino); +out_free_entry: + pde_free(dp); + return NULL; +} + +static struct proc_dir_entry *__proc_create(struct proc_dir_entry **parent, + const char *name, + umode_t mode, + nlink_t nlink) +{ + struct proc_dir_entry *ent = NULL; + const char *fn; + struct qstr qstr; + + if (xlate_proc_name(name, parent, &fn) != 0) + goto out; + qstr.name = fn; + qstr.len = strnlen(fn, NAME_MAX + 1); + if (qstr.len == 0) { + WARN(1, "empty name\n"); + return NULL; + } + if (qstr.len > NAME_MAX) { + WARN(1, "name too long\n"); + return NULL; + } + if (qstr.len == 1 && fn[0] == '.') { + WARN(1, "name '.'\n"); + return NULL; + } + if (qstr.len == 2 && fn[0] == '.' && fn[1] == '.') { + WARN(1, "name '..'\n"); + return NULL; + } + if (*parent == &proc_root && name_to_int(&qstr) != ~0U) { + WARN(1, "create '/proc/%s' by hand\n", qstr.name); + return NULL; + } + if (is_empty_pde(*parent)) { + WARN(1, "attempt to add to permanently empty directory"); + return NULL; + } + + ent = kmem_cache_zalloc(proc_dir_entry_cache, GFP_KERNEL); + if (!ent) + goto out; + + if (qstr.len + 1 <= SIZEOF_PDE_INLINE_NAME) { + ent->name = ent->inline_name; + } else { + ent->name = kmalloc(qstr.len + 1, GFP_KERNEL); + if (!ent->name) { + pde_free(ent); + return NULL; + } + } + + memcpy(ent->name, fn, qstr.len + 1); + ent->namelen = qstr.len; + ent->mode = mode; + ent->nlink = nlink; + ent->subdir = RB_ROOT; + refcount_set(&ent->refcnt, 1); + spin_lock_init(&ent->pde_unload_lock); + INIT_LIST_HEAD(&ent->pde_openers); + proc_set_user(ent, (*parent)->uid, (*parent)->gid); + + /* Revalidate everything under /proc/${pid}/net */ + if ((*parent)->flags & PROC_ENTRY_FORCE_LOOKUP) + pde_force_lookup(ent); + +out: + return ent; +} + +struct proc_dir_entry *proc_symlink(const char *name, + struct proc_dir_entry *parent, const char *dest) +{ + struct proc_dir_entry *ent; + + ent = __proc_create(&parent, name, + (S_IFLNK | S_IRUGO | S_IWUGO | S_IXUGO),1); + + if (ent) { + ent->size = strlen(dest); + ent->data = kmemdup(dest, ent->size + 1, GFP_KERNEL); + if (ent->data) { + ent->proc_iops = &proc_link_inode_operations; + ent = proc_register(parent, ent); + } else { + pde_free(ent); + ent = NULL; + } + } + return ent; +} +EXPORT_SYMBOL(proc_symlink); + +struct proc_dir_entry *_proc_mkdir(const char *name, umode_t mode, + struct proc_dir_entry *parent, void *data, bool force_lookup) +{ + struct proc_dir_entry *ent; + + if (mode == 0) + mode = S_IRUGO | S_IXUGO; + + ent = __proc_create(&parent, name, S_IFDIR | mode, 2); + if (ent) { + ent->data = data; + ent->proc_dir_ops = &proc_dir_operations; + ent->proc_iops = &proc_dir_inode_operations; + if (force_lookup) { + pde_force_lookup(ent); + } + ent = proc_register(parent, ent); + } + return ent; +} +EXPORT_SYMBOL_GPL(_proc_mkdir); + +struct proc_dir_entry *proc_mkdir_data(const char *name, umode_t mode, + struct proc_dir_entry *parent, void *data) +{ + return _proc_mkdir(name, mode, parent, data, false); +} +EXPORT_SYMBOL_GPL(proc_mkdir_data); + +struct proc_dir_entry *proc_mkdir_mode(const char *name, umode_t mode, + struct proc_dir_entry *parent) +{ + return proc_mkdir_data(name, mode, parent, NULL); +} +EXPORT_SYMBOL(proc_mkdir_mode); + +struct proc_dir_entry *proc_mkdir(const char *name, + struct proc_dir_entry *parent) +{ + return proc_mkdir_data(name, 0, parent, NULL); +} +EXPORT_SYMBOL(proc_mkdir); + +struct proc_dir_entry *proc_create_mount_point(const char *name) +{ + umode_t mode = S_IFDIR | S_IRUGO | S_IXUGO; + struct proc_dir_entry *ent, *parent = NULL; + + ent = __proc_create(&parent, name, mode, 2); + if (ent) { + ent->data = NULL; + ent->proc_dir_ops = NULL; + ent->proc_iops = NULL; + ent = proc_register(parent, ent); + } + return ent; +} +EXPORT_SYMBOL(proc_create_mount_point); + +struct proc_dir_entry *proc_create_reg(const char *name, umode_t mode, + struct proc_dir_entry **parent, void *data) +{ + struct proc_dir_entry *p; + + if ((mode & S_IFMT) == 0) + mode |= S_IFREG; + if ((mode & S_IALLUGO) == 0) + mode |= S_IRUGO; + if (WARN_ON_ONCE(!S_ISREG(mode))) + return NULL; + + p = __proc_create(parent, name, mode, 1); + if (p) { + p->proc_iops = &proc_file_inode_operations; + p->data = data; + } + return p; +} + +struct proc_dir_entry *proc_create_data(const char *name, umode_t mode, + struct proc_dir_entry *parent, + const struct proc_ops *proc_ops, void *data) +{ + struct proc_dir_entry *p; + + p = proc_create_reg(name, mode, &parent, data); + if (!p) + return NULL; + p->proc_ops = proc_ops; + return proc_register(parent, p); +} +EXPORT_SYMBOL(proc_create_data); + +struct proc_dir_entry *proc_create(const char *name, umode_t mode, + struct proc_dir_entry *parent, + const struct proc_ops *proc_ops) +{ + return proc_create_data(name, mode, parent, proc_ops, NULL); +} +EXPORT_SYMBOL(proc_create); + +static int proc_seq_open(struct inode *inode, struct file *file) +{ + struct proc_dir_entry *de = PDE(inode); + + if (de->state_size) + return seq_open_private(file, de->seq_ops, de->state_size); + return seq_open(file, de->seq_ops); +} + +static int proc_seq_release(struct inode *inode, struct file *file) +{ + struct proc_dir_entry *de = PDE(inode); + + if (de->state_size) + return seq_release_private(inode, file); + return seq_release(inode, file); +} + +static const struct proc_ops proc_seq_ops = { + /* not permanent -- can call into arbitrary seq_operations */ + .proc_open = proc_seq_open, + .proc_read_iter = seq_read_iter, + .proc_lseek = seq_lseek, + .proc_release = proc_seq_release, +}; + +struct proc_dir_entry *proc_create_seq_private(const char *name, umode_t mode, + struct proc_dir_entry *parent, const struct seq_operations *ops, + unsigned int state_size, void *data) +{ + struct proc_dir_entry *p; + + p = proc_create_reg(name, mode, &parent, data); + if (!p) + return NULL; + p->proc_ops = &proc_seq_ops; + p->seq_ops = ops; + p->state_size = state_size; + return proc_register(parent, p); +} +EXPORT_SYMBOL(proc_create_seq_private); + +static int proc_single_open(struct inode *inode, struct file *file) +{ + struct proc_dir_entry *de = PDE(inode); + + return single_open(file, de->single_show, de->data); +} + +static const struct proc_ops proc_single_ops = { + /* not permanent -- can call into arbitrary ->single_show */ + .proc_open = proc_single_open, + .proc_read_iter = seq_read_iter, + .proc_lseek = seq_lseek, + .proc_release = single_release, +}; + +struct proc_dir_entry *proc_create_single_data(const char *name, umode_t mode, + struct proc_dir_entry *parent, + int (*show)(struct seq_file *, void *), void *data) +{ + struct proc_dir_entry *p; + + p = proc_create_reg(name, mode, &parent, data); + if (!p) + return NULL; + p->proc_ops = &proc_single_ops; + p->single_show = show; + return proc_register(parent, p); +} +EXPORT_SYMBOL(proc_create_single_data); + +void proc_set_size(struct proc_dir_entry *de, loff_t size) +{ + de->size = size; +} +EXPORT_SYMBOL(proc_set_size); + +void proc_set_user(struct proc_dir_entry *de, kuid_t uid, kgid_t gid) +{ + de->uid = uid; + de->gid = gid; +} +EXPORT_SYMBOL(proc_set_user); + +void pde_put(struct proc_dir_entry *pde) +{ + if (refcount_dec_and_test(&pde->refcnt)) { + proc_free_inum(pde->low_ino); + pde_free(pde); + } +} + +static void pde_erase(struct proc_dir_entry *pde, struct proc_dir_entry *parent) +{ + rb_erase(&pde->subdir_node, &parent->subdir); + RB_CLEAR_NODE(&pde->subdir_node); + if (S_ISDIR(pde->mode)) + parent->nlink--; +} + +/* + * Remove a /proc entry and free it if it's not currently in use. + */ +void remove_proc_entry(const char *name, struct proc_dir_entry *parent) +{ + struct proc_dir_entry *de = NULL; + const char *fn = name; + unsigned int len; + + write_lock(&proc_subdir_lock); + if (__xlate_proc_name(name, &parent, &fn) != 0) { + write_unlock(&proc_subdir_lock); + return; + } + len = strlen(fn); + + de = pde_subdir_find(parent, fn, len); + if (de) { + if (unlikely(pde_is_permanent(de))) { + WARN(1, "removing permanent /proc entry '%s'", de->name); + de = NULL; + } else { + pde_erase(de, parent); + } + } + write_unlock(&proc_subdir_lock); + if (!de) { + WARN(1, "name '%s'\n", name); + return; + } + + proc_entry_rundown(de); + + WARN(pde_subdir_first(de), + "%s: removing non-empty directory '%s/%s', leaking at least '%s'\n", + __func__, de->parent->name, de->name, pde_subdir_first(de)->name); + pde_put(de); +} +EXPORT_SYMBOL(remove_proc_entry); + +int remove_proc_subtree(const char *name, struct proc_dir_entry *parent) +{ + struct proc_dir_entry *root = NULL, *de, *next; + const char *fn = name; + unsigned int len; + + write_lock(&proc_subdir_lock); + if (__xlate_proc_name(name, &parent, &fn) != 0) { + write_unlock(&proc_subdir_lock); + return -ENOENT; + } + len = strlen(fn); + + root = pde_subdir_find(parent, fn, len); + if (!root) { + write_unlock(&proc_subdir_lock); + return -ENOENT; + } + if (unlikely(pde_is_permanent(root))) { + write_unlock(&proc_subdir_lock); + WARN(1, "removing permanent /proc entry '%s/%s'", + root->parent->name, root->name); + return -EINVAL; + } + pde_erase(root, parent); + + de = root; + while (1) { + next = pde_subdir_first(de); + if (next) { + if (unlikely(pde_is_permanent(next))) { + write_unlock(&proc_subdir_lock); + WARN(1, "removing permanent /proc entry '%s/%s'", + next->parent->name, next->name); + return -EINVAL; + } + pde_erase(next, de); + de = next; + continue; + } + next = de->parent; + write_unlock(&proc_subdir_lock); + + proc_entry_rundown(de); + if (de == root) + break; + pde_put(de); + + write_lock(&proc_subdir_lock); + de = next; + } + pde_put(root); + return 0; +} +EXPORT_SYMBOL(remove_proc_subtree); + +void *proc_get_parent_data(const struct inode *inode) +{ + struct proc_dir_entry *de = PDE(inode); + return de->parent->data; +} +EXPORT_SYMBOL_GPL(proc_get_parent_data); + +void proc_remove(struct proc_dir_entry *de) +{ + if (de) + remove_proc_subtree(de->name, de->parent); +} +EXPORT_SYMBOL(proc_remove); + +/* + * Pull a user buffer into memory and pass it to the file's write handler if + * one is supplied. The ->write() method is permitted to modify the + * kernel-side buffer. + */ +ssize_t proc_simple_write(struct file *f, const char __user *ubuf, size_t size, + loff_t *_pos) +{ + struct proc_dir_entry *pde = PDE(file_inode(f)); + char *buf; + int ret; + + if (!pde->write) + return -EACCES; + if (size == 0 || size > PAGE_SIZE - 1) + return -EINVAL; + buf = memdup_user_nul(ubuf, size); + if (IS_ERR(buf)) + return PTR_ERR(buf); + ret = pde->write(f, buf, size); + kfree(buf); + return ret == 0 ? size : ret; +} + +/* + * Not exported to modules: + * modules' /proc files aren't permanent because modules aren't permanent. + */ +void impl_proc_make_permanent(struct proc_dir_entry *pde) +{ + if (pde) + pde_make_permanent(pde); +} diff --git a/fs/proc/inode.c b/fs/proc/inode.c new file mode 100644 index 000000000..b7634f975 --- /dev/null +++ b/fs/proc/inode.c @@ -0,0 +1,680 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/proc/inode.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +static void proc_evict_inode(struct inode *inode) +{ + struct ctl_table_header *head; + struct proc_inode *ei = PROC_I(inode); + + truncate_inode_pages_final(&inode->i_data); + clear_inode(inode); + + /* Stop tracking associated processes */ + if (ei->pid) + proc_pid_evict_inode(ei); + + head = ei->sysctl; + if (head) { + WRITE_ONCE(ei->sysctl, NULL); + proc_sys_evict_inode(inode, head); + } +} + +static struct kmem_cache *proc_inode_cachep __ro_after_init; +static struct kmem_cache *pde_opener_cache __ro_after_init; + +static struct inode *proc_alloc_inode(struct super_block *sb) +{ + struct proc_inode *ei; + + ei = alloc_inode_sb(sb, proc_inode_cachep, GFP_KERNEL); + if (!ei) + return NULL; + ei->pid = NULL; + ei->fd = 0; + ei->op.proc_get_link = NULL; + ei->pde = NULL; + ei->sysctl = NULL; + ei->sysctl_entry = NULL; + INIT_HLIST_NODE(&ei->sibling_inodes); + ei->ns_ops = NULL; + return &ei->vfs_inode; +} + +static void proc_free_inode(struct inode *inode) +{ + struct proc_inode *ei = PROC_I(inode); + + if (ei->pid) + put_pid(ei->pid); + /* Let go of any associated proc directory entry */ + if (ei->pde) + pde_put(ei->pde); + kmem_cache_free(proc_inode_cachep, PROC_I(inode)); +} + +static void init_once(void *foo) +{ + struct proc_inode *ei = (struct proc_inode *) foo; + + inode_init_once(&ei->vfs_inode); +} + +void __init proc_init_kmemcache(void) +{ + proc_inode_cachep = kmem_cache_create("proc_inode_cache", + sizeof(struct proc_inode), + 0, (SLAB_RECLAIM_ACCOUNT| + SLAB_ACCOUNT| + SLAB_PANIC), + init_once); + pde_opener_cache = + kmem_cache_create("pde_opener", sizeof(struct pde_opener), 0, + SLAB_ACCOUNT|SLAB_PANIC, NULL); + proc_dir_entry_cache = kmem_cache_create_usercopy( + "proc_dir_entry", SIZEOF_PDE, 0, SLAB_PANIC, + offsetof(struct proc_dir_entry, inline_name), + SIZEOF_PDE_INLINE_NAME, NULL); + BUILD_BUG_ON(sizeof(struct proc_dir_entry) >= SIZEOF_PDE); +} + +void proc_invalidate_siblings_dcache(struct hlist_head *inodes, spinlock_t *lock) +{ + struct hlist_node *node; + struct super_block *old_sb = NULL; + + rcu_read_lock(); + while ((node = hlist_first_rcu(inodes))) { + struct proc_inode *ei = hlist_entry(node, struct proc_inode, sibling_inodes); + struct super_block *sb; + struct inode *inode; + + spin_lock(lock); + hlist_del_init_rcu(&ei->sibling_inodes); + spin_unlock(lock); + + inode = &ei->vfs_inode; + sb = inode->i_sb; + if ((sb != old_sb) && !atomic_inc_not_zero(&sb->s_active)) + continue; + inode = igrab(inode); + rcu_read_unlock(); + if (sb != old_sb) { + if (old_sb) + deactivate_super(old_sb); + old_sb = sb; + } + if (unlikely(!inode)) { + rcu_read_lock(); + continue; + } + + if (S_ISDIR(inode->i_mode)) { + struct dentry *dir = d_find_any_alias(inode); + if (dir) { + d_invalidate(dir); + dput(dir); + } + } else { + struct dentry *dentry; + while ((dentry = d_find_alias(inode))) { + d_invalidate(dentry); + dput(dentry); + } + } + iput(inode); + + rcu_read_lock(); + } + rcu_read_unlock(); + if (old_sb) + deactivate_super(old_sb); +} + +static inline const char *hidepid2str(enum proc_hidepid v) +{ + switch (v) { + case HIDEPID_OFF: return "off"; + case HIDEPID_NO_ACCESS: return "noaccess"; + case HIDEPID_INVISIBLE: return "invisible"; + case HIDEPID_NOT_PTRACEABLE: return "ptraceable"; + } + WARN_ONCE(1, "bad hide_pid value: %d\n", v); + return "unknown"; +} + +static int proc_show_options(struct seq_file *seq, struct dentry *root) +{ + struct proc_fs_info *fs_info = proc_sb_info(root->d_sb); + + if (!gid_eq(fs_info->pid_gid, GLOBAL_ROOT_GID)) + seq_printf(seq, ",gid=%u", from_kgid_munged(&init_user_ns, fs_info->pid_gid)); + if (fs_info->hide_pid != HIDEPID_OFF) + seq_printf(seq, ",hidepid=%s", hidepid2str(fs_info->hide_pid)); + if (fs_info->pidonly != PROC_PIDONLY_OFF) + seq_printf(seq, ",subset=pid"); + + return 0; +} + +const struct super_operations proc_sops = { + .alloc_inode = proc_alloc_inode, + .free_inode = proc_free_inode, + .drop_inode = inode_just_drop, + .evict_inode = proc_evict_inode, + .statfs = simple_statfs, + .show_options = proc_show_options, +}; + +enum {BIAS = -1U<<31}; + +static inline int use_pde(struct proc_dir_entry *pde) +{ + return likely(atomic_inc_unless_negative(&pde->in_use)); +} + +static void unuse_pde(struct proc_dir_entry *pde) +{ + if (unlikely(atomic_dec_return(&pde->in_use) == BIAS)) + complete(pde->pde_unload_completion); +} + +/* + * At most 2 contexts can enter this function: the one doing the last + * close on the descriptor and whoever is deleting PDE itself. + * + * First to enter calls ->proc_release hook and signals its completion + * to the second one which waits and then does nothing. + * + * PDE is locked on entry, unlocked on exit. + */ +static void close_pdeo(struct proc_dir_entry *pde, struct pde_opener *pdeo) + __releases(&pde->pde_unload_lock) +{ + /* + * close() (proc_reg_release()) can't delete an entry and proceed: + * ->release hook needs to be available at the right moment. + * + * rmmod (remove_proc_entry() et al) can't delete an entry and proceed: + * "struct file" needs to be available at the right moment. + */ + if (pdeo->closing) { + /* somebody else is doing that, just wait */ + DECLARE_COMPLETION_ONSTACK(c); + pdeo->c = &c; + spin_unlock(&pde->pde_unload_lock); + wait_for_completion(&c); + } else { + struct file *file; + struct completion *c; + + pdeo->closing = true; + spin_unlock(&pde->pde_unload_lock); + + file = pdeo->file; + pde->proc_ops->proc_release(file_inode(file), file); + + spin_lock(&pde->pde_unload_lock); + /* Strictly after ->proc_release, see above. */ + list_del(&pdeo->lh); + c = pdeo->c; + spin_unlock(&pde->pde_unload_lock); + if (unlikely(c)) + complete(c); + kmem_cache_free(pde_opener_cache, pdeo); + } +} + +void proc_entry_rundown(struct proc_dir_entry *de) +{ + DECLARE_COMPLETION_ONSTACK(c); + /* Wait until all existing callers into module are done. */ + de->pde_unload_completion = &c; + if (atomic_add_return(BIAS, &de->in_use) != BIAS) + wait_for_completion(&c); + + /* ->pde_openers list can't grow from now on. */ + + spin_lock(&de->pde_unload_lock); + while (!list_empty(&de->pde_openers)) { + struct pde_opener *pdeo; + pdeo = list_first_entry(&de->pde_openers, struct pde_opener, lh); + close_pdeo(de, pdeo); + spin_lock(&de->pde_unload_lock); + } + spin_unlock(&de->pde_unload_lock); +} + +static loff_t proc_reg_llseek(struct file *file, loff_t offset, int whence) +{ + struct proc_dir_entry *pde = PDE(file_inode(file)); + loff_t rv = -EINVAL; + + if (pde_is_permanent(pde)) { + return pde->proc_ops->proc_lseek(file, offset, whence); + } else if (use_pde(pde)) { + rv = pde->proc_ops->proc_lseek(file, offset, whence); + unuse_pde(pde); + } + return rv; +} + +static ssize_t proc_reg_read_iter(struct kiocb *iocb, struct iov_iter *iter) +{ + struct proc_dir_entry *pde = PDE(file_inode(iocb->ki_filp)); + ssize_t ret; + + if (pde_is_permanent(pde)) + return pde->proc_ops->proc_read_iter(iocb, iter); + + if (!use_pde(pde)) + return -EIO; + ret = pde->proc_ops->proc_read_iter(iocb, iter); + unuse_pde(pde); + return ret; +} + +static ssize_t pde_read(struct proc_dir_entry *pde, struct file *file, char __user *buf, size_t count, loff_t *ppos) +{ + const auto read = pde->proc_ops->proc_read; + if (read) + return read(file, buf, count, ppos); + return -EIO; +} + +static ssize_t proc_reg_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) +{ + struct proc_dir_entry *pde = PDE(file_inode(file)); + ssize_t rv = -EIO; + + if (pde_is_permanent(pde)) { + return pde_read(pde, file, buf, count, ppos); + } else if (use_pde(pde)) { + rv = pde_read(pde, file, buf, count, ppos); + unuse_pde(pde); + } + return rv; +} + +static ssize_t pde_write(struct proc_dir_entry *pde, struct file *file, const char __user *buf, size_t count, loff_t *ppos) +{ + const auto write = pde->proc_ops->proc_write; + if (write) + return write(file, buf, count, ppos); + return -EIO; +} + +static ssize_t proc_reg_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) +{ + struct proc_dir_entry *pde = PDE(file_inode(file)); + ssize_t rv = -EIO; + + if (pde_is_permanent(pde)) { + return pde_write(pde, file, buf, count, ppos); + } else if (use_pde(pde)) { + rv = pde_write(pde, file, buf, count, ppos); + unuse_pde(pde); + } + return rv; +} + +static __poll_t pde_poll(struct proc_dir_entry *pde, struct file *file, struct poll_table_struct *pts) +{ + const auto poll = pde->proc_ops->proc_poll; + if (poll) + return poll(file, pts); + return DEFAULT_POLLMASK; +} + +static __poll_t proc_reg_poll(struct file *file, struct poll_table_struct *pts) +{ + struct proc_dir_entry *pde = PDE(file_inode(file)); + __poll_t rv = DEFAULT_POLLMASK; + + if (pde_is_permanent(pde)) { + return pde_poll(pde, file, pts); + } else if (use_pde(pde)) { + rv = pde_poll(pde, file, pts); + unuse_pde(pde); + } + return rv; +} + +static long pde_ioctl(struct proc_dir_entry *pde, struct file *file, unsigned int cmd, unsigned long arg) +{ + const auto ioctl = pde->proc_ops->proc_ioctl; + if (ioctl) + return ioctl(file, cmd, arg); + return -ENOTTY; +} + +static long proc_reg_unlocked_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + struct proc_dir_entry *pde = PDE(file_inode(file)); + long rv = -ENOTTY; + + if (pde_is_permanent(pde)) { + return pde_ioctl(pde, file, cmd, arg); + } else if (use_pde(pde)) { + rv = pde_ioctl(pde, file, cmd, arg); + unuse_pde(pde); + } + return rv; +} + +#ifdef CONFIG_COMPAT +static long pde_compat_ioctl(struct proc_dir_entry *pde, struct file *file, unsigned int cmd, unsigned long arg) +{ + const auto compat_ioctl = pde->proc_ops->proc_compat_ioctl; + if (compat_ioctl) + return compat_ioctl(file, cmd, arg); + return -ENOTTY; +} + +static long proc_reg_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + struct proc_dir_entry *pde = PDE(file_inode(file)); + long rv = -ENOTTY; + if (pde_is_permanent(pde)) { + return pde_compat_ioctl(pde, file, cmd, arg); + } else if (use_pde(pde)) { + rv = pde_compat_ioctl(pde, file, cmd, arg); + unuse_pde(pde); + } + return rv; +} +#endif + +static int pde_mmap(struct proc_dir_entry *pde, struct file *file, struct vm_area_struct *vma) +{ + const auto mmap = pde->proc_ops->proc_mmap; + if (mmap) + return mmap(file, vma); + return -EIO; +} + +static int proc_reg_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct proc_dir_entry *pde = PDE(file_inode(file)); + int rv = -EIO; + + if (pde_is_permanent(pde)) { + return pde_mmap(pde, file, vma); + } else if (use_pde(pde)) { + rv = pde_mmap(pde, file, vma); + unuse_pde(pde); + } + return rv; +} + +static unsigned long +pde_get_unmapped_area(struct proc_dir_entry *pde, struct file *file, unsigned long orig_addr, + unsigned long len, unsigned long pgoff, + unsigned long flags) +{ + if (pde->proc_ops->proc_get_unmapped_area) + return pde->proc_ops->proc_get_unmapped_area(file, orig_addr, len, pgoff, flags); + +#ifdef CONFIG_MMU + return mm_get_unmapped_area(file, orig_addr, len, pgoff, flags); +#endif + + return orig_addr; +} + +static unsigned long +proc_reg_get_unmapped_area(struct file *file, unsigned long orig_addr, + unsigned long len, unsigned long pgoff, + unsigned long flags) +{ + struct proc_dir_entry *pde = PDE(file_inode(file)); + unsigned long rv = -EIO; + + if (pde_is_permanent(pde)) { + return pde_get_unmapped_area(pde, file, orig_addr, len, pgoff, flags); + } else if (use_pde(pde)) { + rv = pde_get_unmapped_area(pde, file, orig_addr, len, pgoff, flags); + unuse_pde(pde); + } + return rv; +} + +static int proc_reg_open(struct inode *inode, struct file *file) +{ + struct proc_dir_entry *pde = PDE(inode); + int rv = 0; + typeof_member(struct proc_ops, proc_open) open; + struct pde_opener *pdeo; + + if (!pde_has_proc_lseek(pde)) + file->f_mode &= ~FMODE_LSEEK; + + if (pde_is_permanent(pde)) { + open = pde->proc_ops->proc_open; + if (open) + rv = open(inode, file); + return rv; + } + + /* + * Ensure that + * 1) PDE's ->release hook will be called no matter what + * either normally by close()/->release, or forcefully by + * rmmod/remove_proc_entry. + * + * 2) rmmod isn't blocked by opening file in /proc and sitting on + * the descriptor (including "rmmod foo release hook. + */ + if (!use_pde(pde)) + return -ENOENT; + + const auto release = pde->proc_ops->proc_release; + if (release) { + pdeo = kmem_cache_alloc(pde_opener_cache, GFP_KERNEL); + if (!pdeo) { + rv = -ENOMEM; + goto out_unuse; + } + } + + open = pde->proc_ops->proc_open; + if (open) + rv = open(inode, file); + + if (release) { + if (rv == 0) { + /* To know what to release. */ + pdeo->file = file; + pdeo->closing = false; + pdeo->c = NULL; + spin_lock(&pde->pde_unload_lock); + list_add(&pdeo->lh, &pde->pde_openers); + spin_unlock(&pde->pde_unload_lock); + } else + kmem_cache_free(pde_opener_cache, pdeo); + } + +out_unuse: + unuse_pde(pde); + return rv; +} + +static int proc_reg_release(struct inode *inode, struct file *file) +{ + struct proc_dir_entry *pde = PDE(inode); + struct pde_opener *pdeo; + + if (pde_is_permanent(pde)) { + const auto release = pde->proc_ops->proc_release; + if (release) + return release(inode, file); + return 0; + } + + spin_lock(&pde->pde_unload_lock); + list_for_each_entry(pdeo, &pde->pde_openers, lh) { + if (pdeo->file == file) { + close_pdeo(pde, pdeo); + return 0; + } + } + spin_unlock(&pde->pde_unload_lock); + return 0; +} + +static const struct file_operations proc_reg_file_ops = { + .llseek = proc_reg_llseek, + .read = proc_reg_read, + .write = proc_reg_write, + .poll = proc_reg_poll, + .unlocked_ioctl = proc_reg_unlocked_ioctl, + .mmap = proc_reg_mmap, + .get_unmapped_area = proc_reg_get_unmapped_area, + .open = proc_reg_open, + .release = proc_reg_release, +}; + +static const struct file_operations proc_iter_file_ops = { + .llseek = proc_reg_llseek, + .read_iter = proc_reg_read_iter, + .write = proc_reg_write, + .splice_read = copy_splice_read, + .poll = proc_reg_poll, + .unlocked_ioctl = proc_reg_unlocked_ioctl, + .mmap = proc_reg_mmap, + .get_unmapped_area = proc_reg_get_unmapped_area, + .open = proc_reg_open, + .release = proc_reg_release, +}; + +#ifdef CONFIG_COMPAT +static const struct file_operations proc_reg_file_ops_compat = { + .llseek = proc_reg_llseek, + .read = proc_reg_read, + .write = proc_reg_write, + .poll = proc_reg_poll, + .unlocked_ioctl = proc_reg_unlocked_ioctl, + .compat_ioctl = proc_reg_compat_ioctl, + .mmap = proc_reg_mmap, + .get_unmapped_area = proc_reg_get_unmapped_area, + .open = proc_reg_open, + .release = proc_reg_release, +}; + +static const struct file_operations proc_iter_file_ops_compat = { + .llseek = proc_reg_llseek, + .read_iter = proc_reg_read_iter, + .splice_read = copy_splice_read, + .write = proc_reg_write, + .poll = proc_reg_poll, + .unlocked_ioctl = proc_reg_unlocked_ioctl, + .compat_ioctl = proc_reg_compat_ioctl, + .mmap = proc_reg_mmap, + .get_unmapped_area = proc_reg_get_unmapped_area, + .open = proc_reg_open, + .release = proc_reg_release, +}; +#endif + +static void proc_put_link(void *p) +{ + unuse_pde(p); +} + +static const char *proc_get_link(struct dentry *dentry, + struct inode *inode, + struct delayed_call *done) +{ + struct proc_dir_entry *pde = PDE(inode); + if (!use_pde(pde)) + return ERR_PTR(-EINVAL); + set_delayed_call(done, proc_put_link, pde); + return pde->data; +} + +const struct inode_operations proc_link_inode_operations = { + .get_link = proc_get_link, +}; + +struct inode *proc_get_inode(struct super_block *sb, struct proc_dir_entry *de) +{ + struct inode *inode = new_inode(sb); + + if (!inode) { + pde_put(de); + return NULL; + } + + inode->i_private = de->data; + inode->i_ino = de->low_ino; + simple_inode_init_ts(inode); + PROC_I(inode)->pde = de; + if (is_empty_pde(de)) { + make_empty_dir_inode(inode); + return inode; + } + + if (de->mode) { + inode->i_mode = de->mode; + inode->i_uid = de->uid; + inode->i_gid = de->gid; + } + if (de->size) + inode->i_size = de->size; + if (de->nlink) + set_nlink(inode, de->nlink); + + if (S_ISREG(inode->i_mode)) { + inode->i_op = de->proc_iops; + if (pde_has_proc_read_iter(de)) + inode->i_fop = &proc_iter_file_ops; + else + inode->i_fop = &proc_reg_file_ops; +#ifdef CONFIG_COMPAT + if (pde_has_proc_compat_ioctl(de)) { + if (pde_has_proc_read_iter(de)) + inode->i_fop = &proc_iter_file_ops_compat; + else + inode->i_fop = &proc_reg_file_ops_compat; + } +#endif + } else if (S_ISDIR(inode->i_mode)) { + inode->i_op = de->proc_iops; + inode->i_fop = de->proc_dir_ops; + } else if (S_ISLNK(inode->i_mode)) { + inode->i_op = de->proc_iops; + inode->i_fop = NULL; + } else { + BUG(); + } + return inode; +} diff --git a/fs/proc/internal.h b/fs/proc/internal.h new file mode 100644 index 000000000..04bd6c9e6 --- /dev/null +++ b/fs/proc/internal.h @@ -0,0 +1,437 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Internal procfs definitions + * + * Copyright (C) 2004 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct ctl_table_header; +struct mempolicy; + +/* + * This is not completely implemented yet. The idea is to + * create an in-memory tree (like the actual /proc filesystem + * tree) of these proc_dir_entries, so that we can dynamically + * add new files to /proc. + * + * parent/subdir are used for the directory structure (every /proc file has a + * parent, but "subdir" is empty for all non-directory entries). + * subdir_node is used to build the rb tree "subdir" of the parent. + */ +struct proc_dir_entry { + /* + * number of callers into module in progress; + * negative -> it's going away RSN + */ + atomic_t in_use; + refcount_t refcnt; + struct list_head pde_openers; /* who did ->open, but not ->release */ + /* protects ->pde_openers and all struct pde_opener instances */ + spinlock_t pde_unload_lock; + struct completion *pde_unload_completion; + const struct inode_operations *proc_iops; + union { + const struct proc_ops *proc_ops; + const struct file_operations *proc_dir_ops; + }; + union { + const struct seq_operations *seq_ops; + int (*single_show)(struct seq_file *, void *); + }; + proc_write_t write; + void *data; + unsigned int state_size; + unsigned int low_ino; + nlink_t nlink; + kuid_t uid; + kgid_t gid; + loff_t size; + struct proc_dir_entry *parent; + struct rb_root subdir; + struct rb_node subdir_node; + char *name; + umode_t mode; + u8 flags; + u8 namelen; + char inline_name[]; +} __randomize_layout; + +#define SIZEOF_PDE ( \ + sizeof(struct proc_dir_entry) < 128 ? 128 : \ + sizeof(struct proc_dir_entry) < 192 ? 192 : \ + sizeof(struct proc_dir_entry) < 256 ? 256 : \ + sizeof(struct proc_dir_entry) < 512 ? 512 : \ + 0) +#define SIZEOF_PDE_INLINE_NAME (SIZEOF_PDE - sizeof(struct proc_dir_entry)) + +static inline bool pde_is_permanent(const struct proc_dir_entry *pde) +{ + return pde->flags & PROC_ENTRY_PERMANENT; +} + +/* This is for builtin code, not even for modules which are compiled in. */ +static inline void pde_make_permanent(struct proc_dir_entry *pde) +{ + /* Ensure magic flag does something. */ + static_assert(PROC_ENTRY_PERMANENT != 0); + pde->flags |= PROC_ENTRY_PERMANENT; +} + +static inline bool pde_has_proc_read_iter(const struct proc_dir_entry *pde) +{ + return pde->flags & PROC_ENTRY_proc_read_iter; +} + +static inline bool pde_has_proc_compat_ioctl(const struct proc_dir_entry *pde) +{ +#ifdef CONFIG_COMPAT + return pde->flags & PROC_ENTRY_proc_compat_ioctl; +#else + return false; +#endif +} + +static inline bool pde_has_proc_lseek(const struct proc_dir_entry *pde) +{ + return pde->flags & PROC_ENTRY_proc_lseek; +} + +extern struct kmem_cache *proc_dir_entry_cache; +void pde_free(struct proc_dir_entry *pde); + +union proc_op { + int (*proc_get_link)(struct dentry *, struct path *, struct task_struct *); + int (*proc_show)(struct seq_file *m, + struct pid_namespace *ns, struct pid *pid, + struct task_struct *task); + int lsmid; +}; + +struct proc_inode { + struct pid *pid; + unsigned int fd; + union proc_op op; + struct proc_dir_entry *pde; + struct ctl_table_header *sysctl; + const struct ctl_table *sysctl_entry; + struct hlist_node sibling_inodes; + const struct proc_ns_operations *ns_ops; + struct inode vfs_inode; +} __randomize_layout; + +/* + * General functions + */ +static inline struct proc_inode *PROC_I(const struct inode *inode) +{ + return container_of(inode, struct proc_inode, vfs_inode); +} + +static inline struct proc_dir_entry *PDE(const struct inode *inode) +{ + return PROC_I(inode)->pde; +} + +static inline struct pid *proc_pid(const struct inode *inode) +{ + return PROC_I(inode)->pid; +} + +static inline struct task_struct *get_proc_task(const struct inode *inode) +{ + return get_pid_task(proc_pid(inode), PIDTYPE_PID); +} + +void task_dump_owner(struct task_struct *task, umode_t mode, + kuid_t *ruid, kgid_t *rgid); + +unsigned name_to_int(const struct qstr *qstr); +/* + * Offset of the first process in the /proc root directory.. + */ +#define FIRST_PROCESS_ENTRY 256 + +/* Worst case buffer size needed for holding an integer. */ +#define PROC_NUMBUF 13 + +#ifdef CONFIG_PAGE_MAPCOUNT +/** + * folio_precise_page_mapcount() - Number of mappings of this folio page. + * @folio: The folio. + * @page: The page. + * + * The number of present user page table entries that reference this page + * as tracked via the RMAP: either referenced directly (PTE) or as part of + * a larger area that covers this page (e.g., PMD). + * + * Use this function only for the calculation of existing statistics + * (USS, PSS, mapcount_max) and for debugging purposes (/proc/kpagecount). + * + * Do not add new users. + * + * Returns: The number of mappings of this folio page. 0 for + * folios that are not mapped to user space or are not tracked via the RMAP + * (e.g., shared zeropage). + */ +static inline int folio_precise_page_mapcount(struct folio *folio, + struct page *page) +{ + int mapcount = atomic_read(&page->_mapcount) + 1; + + if (page_mapcount_is_type(mapcount)) + mapcount = 0; + if (folio_test_large(folio)) + mapcount += folio_entire_mapcount(folio); + + return mapcount; +} +#else /* !CONFIG_PAGE_MAPCOUNT */ +static inline int folio_precise_page_mapcount(struct folio *folio, + struct page *page) +{ + BUILD_BUG(); +} +#endif /* CONFIG_PAGE_MAPCOUNT */ + +/** + * folio_average_page_mapcount() - Average number of mappings per page in this + * folio + * @folio: The folio. + * + * The average number of user page table entries that reference each page in + * this folio as tracked via the RMAP: either referenced directly (PTE) or + * as part of a larger area that covers this page (e.g., PMD). + * + * The average is calculated by rounding to the nearest integer; however, + * to avoid duplicated code in current callers, the average is at least + * 1 if any page of the folio is mapped. + * + * Returns: The average number of mappings per page in this folio. + */ +static inline int folio_average_page_mapcount(struct folio *folio) +{ + int mapcount, entire_mapcount, avg; + + if (!folio_test_large(folio)) + return atomic_read(&folio->_mapcount) + 1; + + mapcount = folio_large_mapcount(folio); + if (unlikely(mapcount <= 0)) + return 0; + entire_mapcount = folio_entire_mapcount(folio); + if (mapcount <= entire_mapcount) + return entire_mapcount; + mapcount -= entire_mapcount; + + /* Round to closest integer ... */ + avg = ((unsigned int)mapcount + folio_large_nr_pages(folio) / 2) >> folio_large_order(folio); + /* ... but return at least 1. */ + return max_t(int, avg + entire_mapcount, 1); +} +/* + * array.c + */ +extern const struct file_operations proc_tid_children_operations; + +extern void proc_task_name(struct seq_file *m, struct task_struct *p, + bool escape); +extern int proc_tid_stat(struct seq_file *, struct pid_namespace *, + struct pid *, struct task_struct *); +extern int proc_tgid_stat(struct seq_file *, struct pid_namespace *, + struct pid *, struct task_struct *); +extern int proc_pid_status(struct seq_file *, struct pid_namespace *, + struct pid *, struct task_struct *); +extern int proc_pid_statm(struct seq_file *, struct pid_namespace *, + struct pid *, struct task_struct *); + +/* + * base.c + */ +extern const struct dentry_operations pid_dentry_operations; +extern int pid_getattr(struct mnt_idmap *, const struct path *, + struct kstat *, u32, unsigned int); +int proc_nochmod_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr); +extern void proc_pid_evict_inode(struct proc_inode *); +extern struct inode *proc_pid_make_inode(struct super_block *, struct task_struct *, umode_t); +extern void pid_update_inode(struct task_struct *, struct inode *); +extern int pid_delete_dentry(const struct dentry *); +extern int proc_pid_readdir(struct file *, struct dir_context *); +struct dentry *proc_pid_lookup(struct dentry *, unsigned int); +extern loff_t mem_lseek(struct file *, loff_t, int); + +/* Lookups */ +typedef struct dentry *instantiate_t(struct dentry *, + struct task_struct *, const void *); +bool proc_fill_cache(struct file *, struct dir_context *, const char *, unsigned int, + instantiate_t, struct task_struct *, const void *); + +/* + * generic.c + */ +struct proc_dir_entry *proc_create_reg(const char *name, umode_t mode, + struct proc_dir_entry **parent, void *data); +struct proc_dir_entry *proc_register(struct proc_dir_entry *dir, + struct proc_dir_entry *dp); +extern struct dentry *proc_lookup(struct inode *, struct dentry *, unsigned int); +struct dentry *proc_lookup_de(struct inode *, struct dentry *, struct proc_dir_entry *); +extern int proc_readdir(struct file *, struct dir_context *); +int proc_readdir_de(struct file *, struct dir_context *, struct proc_dir_entry *); + +static inline void pde_get(struct proc_dir_entry *pde) +{ + refcount_inc(&pde->refcnt); +} +extern void pde_put(struct proc_dir_entry *); + +static inline bool is_empty_pde(const struct proc_dir_entry *pde) +{ + return S_ISDIR(pde->mode) && !pde->proc_iops; +} +extern ssize_t proc_simple_write(struct file *, const char __user *, size_t, loff_t *); + +/* + * inode.c + */ +struct pde_opener { + struct list_head lh; + struct file *file; + bool closing; + struct completion *c; +} __randomize_layout; +extern const struct inode_operations proc_link_inode_operations; +extern const struct inode_operations proc_pid_link_inode_operations; +extern const struct super_operations proc_sops; + +void proc_init_kmemcache(void); +void proc_invalidate_siblings_dcache(struct hlist_head *inodes, spinlock_t *lock); +void set_proc_pid_nlink(void); +extern struct inode *proc_get_inode(struct super_block *, struct proc_dir_entry *); +extern void proc_entry_rundown(struct proc_dir_entry *); + +/* + * proc_namespaces.c + */ +extern const struct inode_operations proc_ns_dir_inode_operations; +extern const struct file_operations proc_ns_dir_operations; + +/* + * proc_net.c + */ +extern const struct file_operations proc_net_operations; +extern const struct inode_operations proc_net_inode_operations; + +#ifdef CONFIG_NET +extern int proc_net_init(void); +#else +static inline int proc_net_init(void) { return 0; } +#endif + +/* + * proc_self.c + */ +extern int proc_setup_self(struct super_block *); + +/* + * proc_thread_self.c + */ +extern int proc_setup_thread_self(struct super_block *); +extern void proc_thread_self_init(void); + +/* + * proc_sysctl.c + */ +#ifdef CONFIG_SYSCTL +extern int proc_sys_init(void); +extern void proc_sys_evict_inode(struct inode *inode, + struct ctl_table_header *head); +#else +static inline void proc_sys_init(void) { } +static inline void proc_sys_evict_inode(struct inode *inode, + struct ctl_table_header *head) { } +#endif + +/* + * proc_tty.c + */ +#ifdef CONFIG_TTY +extern void proc_tty_init(void); +#else +static inline void proc_tty_init(void) {} +#endif + +/* + * root.c + */ +extern struct proc_dir_entry proc_root; + +extern void proc_self_init(void); +extern unsigned self_inum, thread_self_inum; + +/* + * task_[no]mmu.c + */ +struct mem_size_stats; + +struct proc_maps_locking_ctx { + struct mm_struct *mm; +#ifdef CONFIG_PER_VMA_LOCK + bool mmap_locked; + struct vm_area_struct *locked_vma; +#endif +}; + +struct proc_maps_private { + struct inode *inode; + struct task_struct *task; + struct vma_iterator iter; + loff_t last_pos; + struct proc_maps_locking_ctx lock_ctx; +#ifdef CONFIG_NUMA + struct mempolicy *task_mempolicy; +#endif +} __randomize_layout; + +struct mm_struct *proc_mem_open(struct inode *inode, unsigned int mode); + +extern const struct file_operations proc_pid_maps_operations; +extern const struct file_operations proc_pid_numa_maps_operations; +extern const struct file_operations proc_pid_smaps_operations; +extern const struct file_operations proc_pid_smaps_rollup_operations; +extern const struct file_operations proc_clear_refs_operations; +extern const struct file_operations proc_pagemap_operations; + +extern unsigned long task_vsize(struct mm_struct *); +extern unsigned long task_statm(struct mm_struct *, + unsigned long *, unsigned long *, + unsigned long *, unsigned long *); +extern void task_mem(struct seq_file *, struct mm_struct *); + +extern const struct dentry_operations proc_net_dentry_ops; +static inline void pde_force_lookup(struct proc_dir_entry *pde) +{ + /* /proc/net/ entries can be changed under us by setns(CLONE_NEWNET) */ + pde->flags |= PROC_ENTRY_FORCE_LOOKUP; +} + +/* + * Add a new procfs dentry that can't serve as a mountpoint. That should + * encompass anything that is ephemeral and can just disappear while the + * process is still around. + */ +static inline struct dentry *proc_splice_unmountable(struct inode *inode, + struct dentry *dentry, const struct dentry_operations *d_ops) +{ + dont_mount(dentry); + return d_splice_alias_ops(inode, dentry, d_ops); +} diff --git a/fs/proc/interrupts.c b/fs/proc/interrupts.c new file mode 100644 index 000000000..714a22ded --- /dev/null +++ b/fs/proc/interrupts.c @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include + +/* + * /proc/interrupts + */ +static void *int_seq_start(struct seq_file *f, loff_t *pos) +{ + return *pos <= irq_get_nr_irqs() ? pos : NULL; +} + +static void *int_seq_next(struct seq_file *f, void *v, loff_t *pos) +{ + (*pos)++; + if (*pos > irq_get_nr_irqs()) + return NULL; + return pos; +} + +static void int_seq_stop(struct seq_file *f, void *v) +{ + /* Nothing to do */ +} + +static const struct seq_operations int_seq_ops = { + .start = int_seq_start, + .next = int_seq_next, + .stop = int_seq_stop, + .show = show_interrupts +}; + +static int __init proc_interrupts_init(void) +{ + proc_create_seq("interrupts", 0, NULL, &int_seq_ops); + return 0; +} +fs_initcall(proc_interrupts_init); diff --git a/fs/proc/kcore.c b/fs/proc/kcore.c new file mode 100644 index 000000000..390547b99 --- /dev/null +++ b/fs/proc/kcore.c @@ -0,0 +1,742 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/proc/kcore.c kernel ELF core dumper + * + * Modelled on fs/exec.c:aout_core_dump() + * Jeremy Fitzhardinge + * ELF version written by David Howells + * Modified and incorporated into 2.3.x by Tigran Aivazian + * Support to dump vmalloc'd areas (ELF only), Tigran Aivazian + * Safe accesses to vmalloc/direct-mapped discontiguous areas, Kanoj Sarcar + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +#ifndef ELF_CORE_EFLAGS +#define ELF_CORE_EFLAGS 0 +#endif + +static struct proc_dir_entry *proc_root_kcore; + + +#ifndef kc_vaddr_to_offset +#define kc_vaddr_to_offset(v) ((v) - PAGE_OFFSET) +#endif +#ifndef kc_offset_to_vaddr +#define kc_offset_to_vaddr(o) ((o) + PAGE_OFFSET) +#endif + +#ifndef kc_xlate_dev_mem_ptr +#define kc_xlate_dev_mem_ptr kc_xlate_dev_mem_ptr +static inline void *kc_xlate_dev_mem_ptr(phys_addr_t phys) +{ + return __va(phys); +} +#endif +#ifndef kc_unxlate_dev_mem_ptr +#define kc_unxlate_dev_mem_ptr kc_unxlate_dev_mem_ptr +static inline void kc_unxlate_dev_mem_ptr(phys_addr_t phys, void *virt) +{ +} +#endif + +static LIST_HEAD(kclist_head); +static int kcore_nphdr; +static size_t kcore_phdrs_len; +static size_t kcore_notes_len; +static size_t kcore_data_offset; +DEFINE_STATIC_PERCPU_RWSEM(kclist_lock); +static int kcore_need_update = 1; + +/* + * Returns > 0 for RAM pages, 0 for non-RAM pages, < 0 on error + * Same as oldmem_pfn_is_ram in vmcore + */ +static int (*mem_pfn_is_ram)(unsigned long pfn); + +int __init register_mem_pfn_is_ram(int (*fn)(unsigned long pfn)) +{ + if (mem_pfn_is_ram) + return -EBUSY; + mem_pfn_is_ram = fn; + return 0; +} + +static int pfn_is_ram(unsigned long pfn) +{ + if (mem_pfn_is_ram) + return mem_pfn_is_ram(pfn); + else + return 1; +} + +/* This doesn't grab kclist_lock, so it should only be used at init time. */ +void __init kclist_add(struct kcore_list *new, void *addr, size_t size, + int type) +{ + new->addr = (unsigned long)addr; + new->size = size; + new->type = type; + + list_add_tail(&new->list, &kclist_head); +} + +static void update_kcore_size(void) +{ + size_t try, size; + struct kcore_list *m; + + kcore_nphdr = 1; /* PT_NOTE */ + size = 0; + + list_for_each_entry(m, &kclist_head, list) { + try = kc_vaddr_to_offset((size_t)m->addr + m->size); + if (try > size) + size = try; + kcore_nphdr++; + } + + kcore_phdrs_len = kcore_nphdr * sizeof(struct elf_phdr); + kcore_notes_len = (4 * sizeof(struct elf_note) + + ALIGN(sizeof(NN_PRSTATUS), 4) + + ALIGN(sizeof(NN_PRPSINFO), 4) + + ALIGN(sizeof(NN_TASKSTRUCT), 4) + + VMCOREINFO_NOTE_NAME_BYTES + + ALIGN(sizeof(struct elf_prstatus), 4) + + ALIGN(sizeof(struct elf_prpsinfo), 4) + + ALIGN(arch_task_struct_size, 4) + + ALIGN(vmcoreinfo_size, 4)); + kcore_data_offset = PAGE_ALIGN(sizeof(struct elfhdr) + kcore_phdrs_len + + kcore_notes_len); + proc_root_kcore->size = kcore_data_offset + size; +} + +#ifdef CONFIG_HIGHMEM +/* + * If no highmem, we can assume [0...max_low_pfn) continuous range of memory + * because memory hole is not as big as !HIGHMEM case. + * (HIGHMEM is special because part of memory is _invisible_ from the kernel.) + */ +static int kcore_ram_list(struct list_head *head) +{ + struct kcore_list *ent; + + ent = kmalloc_obj(*ent); + if (!ent) + return -ENOMEM; + ent->addr = (unsigned long)__va(0); + ent->size = max_low_pfn << PAGE_SHIFT; + ent->type = KCORE_RAM; + list_add(&ent->list, head); + return 0; +} + +#else /* !CONFIG_HIGHMEM */ + +#ifdef CONFIG_SPARSEMEM_VMEMMAP +/* calculate vmemmap's address from given system ram pfn and register it */ +static int +get_sparsemem_vmemmap_info(struct kcore_list *ent, struct list_head *head) +{ + unsigned long pfn = __pa(ent->addr) >> PAGE_SHIFT; + unsigned long nr_pages = ent->size >> PAGE_SHIFT; + unsigned long start, end; + struct kcore_list *vmm, *tmp; + + + start = ((unsigned long)pfn_to_page(pfn)) & PAGE_MASK; + end = ((unsigned long)pfn_to_page(pfn + nr_pages)) - 1; + end = PAGE_ALIGN(end); + /* overlap check (because we have to align page */ + list_for_each_entry(tmp, head, list) { + if (tmp->type != KCORE_VMEMMAP) + continue; + if (start < tmp->addr + tmp->size) + if (end > tmp->addr) + end = tmp->addr; + } + if (start < end) { + vmm = kmalloc_obj(*vmm); + if (!vmm) + return 0; + vmm->addr = start; + vmm->size = end - start; + vmm->type = KCORE_VMEMMAP; + list_add_tail(&vmm->list, head); + } + return 1; + +} +#else +static int +get_sparsemem_vmemmap_info(struct kcore_list *ent, struct list_head *head) +{ + return 1; +} + +#endif + +static int +kclist_add_private(unsigned long pfn, unsigned long nr_pages, void *arg) +{ + struct list_head *head = (struct list_head *)arg; + struct kcore_list *ent; + struct page *p; + + if (!pfn_valid(pfn)) + return 1; + + p = pfn_to_page(pfn); + + ent = kmalloc_obj(*ent); + if (!ent) + return -ENOMEM; + ent->addr = (unsigned long)page_to_virt(p); + ent->size = nr_pages << PAGE_SHIFT; + + if (!virt_addr_valid((void *)ent->addr)) + goto free_out; + + /* cut not-mapped area. ....from ppc-32 code. */ + if (ULONG_MAX - ent->addr < ent->size) + ent->size = ULONG_MAX - ent->addr; + + /* + * We've already checked virt_addr_valid so we know this address + * is a valid pointer, therefore we can check against it to determine + * if we need to trim + */ + if (VMALLOC_START > ent->addr) { + if (VMALLOC_START - ent->addr < ent->size) + ent->size = VMALLOC_START - ent->addr; + } + + ent->type = KCORE_RAM; + list_add_tail(&ent->list, head); + + if (!get_sparsemem_vmemmap_info(ent, head)) { + list_del(&ent->list); + goto free_out; + } + + return 0; +free_out: + kfree(ent); + return 1; +} + +static int kcore_ram_list(struct list_head *list) +{ + int nid, ret; + unsigned long end_pfn; + + /* Not initialized....update now */ + /* find out "max pfn" */ + end_pfn = 0; + for_each_node_state(nid, N_MEMORY) { + unsigned long node_end; + node_end = node_end_pfn(nid); + if (end_pfn < node_end) + end_pfn = node_end; + } + /* scan 0 to max_pfn */ + ret = walk_system_ram_range(0, end_pfn, list, kclist_add_private); + if (ret) + return -ENOMEM; + return 0; +} +#endif /* CONFIG_HIGHMEM */ + +static int kcore_update_ram(void) +{ + LIST_HEAD(list); + LIST_HEAD(garbage); + struct kcore_list *tmp, *pos; + int ret = 0; + + percpu_down_write(&kclist_lock); + if (!xchg(&kcore_need_update, 0)) + goto out; + + ret = kcore_ram_list(&list); + if (ret) { + /* Couldn't get the RAM list, try again next time. */ + WRITE_ONCE(kcore_need_update, 1); + list_splice_tail(&list, &garbage); + goto out; + } + + list_for_each_entry_safe(pos, tmp, &kclist_head, list) { + if (pos->type == KCORE_RAM || pos->type == KCORE_VMEMMAP) + list_move(&pos->list, &garbage); + } + list_splice_tail(&list, &kclist_head); + + update_kcore_size(); + +out: + percpu_up_write(&kclist_lock); + list_for_each_entry_safe(pos, tmp, &garbage, list) { + list_del(&pos->list); + kfree(pos); + } + return ret; +} + +static void append_kcore_note(char *notes, size_t *i, const char *name, + unsigned int type, const void *desc, + size_t descsz) +{ + struct elf_note *note = (struct elf_note *)¬es[*i]; + + note->n_namesz = strlen(name) + 1; + note->n_descsz = descsz; + note->n_type = type; + *i += sizeof(*note); + memcpy(¬es[*i], name, note->n_namesz); + *i = ALIGN(*i + note->n_namesz, 4); + memcpy(¬es[*i], desc, descsz); + *i = ALIGN(*i + descsz, 4); +} + +static ssize_t read_kcore_iter(struct kiocb *iocb, struct iov_iter *iter) +{ + struct file *file = iocb->ki_filp; + char *buf = file->private_data; + loff_t *fpos = &iocb->ki_pos; + size_t phdrs_offset, notes_offset; + size_t page_offline_frozen = 1; + struct kcore_list *m; + size_t tsz; + unsigned long start; + size_t buflen = iov_iter_count(iter); + size_t orig_buflen = buflen; + int ret = 0; + + percpu_down_read(&kclist_lock); + /* + * Don't race against drivers that set PageOffline() and expect no + * further page access. + */ + page_offline_freeze(); + + phdrs_offset = sizeof(struct elfhdr); + notes_offset = phdrs_offset + kcore_phdrs_len; + + /* ELF file header. */ + if (buflen && *fpos < sizeof(struct elfhdr)) { + struct elfhdr ehdr = { + .e_ident = { + [EI_MAG0] = ELFMAG0, + [EI_MAG1] = ELFMAG1, + [EI_MAG2] = ELFMAG2, + [EI_MAG3] = ELFMAG3, + [EI_CLASS] = ELF_CLASS, + [EI_DATA] = ELF_DATA, + [EI_VERSION] = EV_CURRENT, + [EI_OSABI] = ELF_OSABI, + }, + .e_type = ET_CORE, + .e_machine = ELF_ARCH, + .e_version = EV_CURRENT, + .e_phoff = sizeof(struct elfhdr), + .e_flags = ELF_CORE_EFLAGS, + .e_ehsize = sizeof(struct elfhdr), + .e_phentsize = sizeof(struct elf_phdr), + .e_phnum = kcore_nphdr, + }; + + tsz = min_t(size_t, buflen, sizeof(struct elfhdr) - *fpos); + if (copy_to_iter((char *)&ehdr + *fpos, tsz, iter) != tsz) { + ret = -EFAULT; + goto out; + } + + buflen -= tsz; + *fpos += tsz; + } + + /* ELF program headers. */ + if (buflen && *fpos < phdrs_offset + kcore_phdrs_len) { + struct elf_phdr *phdrs, *phdr; + + phdrs = kzalloc(kcore_phdrs_len, GFP_KERNEL); + if (!phdrs) { + ret = -ENOMEM; + goto out; + } + + phdrs[0].p_type = PT_NOTE; + phdrs[0].p_offset = notes_offset; + phdrs[0].p_filesz = kcore_notes_len; + + phdr = &phdrs[1]; + list_for_each_entry(m, &kclist_head, list) { + phdr->p_type = PT_LOAD; + phdr->p_flags = PF_R | PF_W | PF_X; + phdr->p_offset = kc_vaddr_to_offset(m->addr) + + kcore_data_offset; + phdr->p_vaddr = (size_t)m->addr; + if (m->type == KCORE_RAM) + phdr->p_paddr = __pa(m->addr); + else if (m->type == KCORE_TEXT) + phdr->p_paddr = __pa_symbol(m->addr); + else + phdr->p_paddr = (elf_addr_t)-1; + phdr->p_filesz = phdr->p_memsz = m->size; + phdr->p_align = PAGE_SIZE; + phdr++; + } + + tsz = min_t(size_t, buflen, + phdrs_offset + kcore_phdrs_len - *fpos); + if (copy_to_iter((char *)phdrs + *fpos - phdrs_offset, tsz, + iter) != tsz) { + kfree(phdrs); + ret = -EFAULT; + goto out; + } + kfree(phdrs); + + buflen -= tsz; + *fpos += tsz; + } + + /* ELF note segment. */ + if (buflen && *fpos < notes_offset + kcore_notes_len) { + struct elf_prstatus prstatus = {}; + struct elf_prpsinfo prpsinfo = { + .pr_sname = 'R', + .pr_fname = "vmlinux", + }; + char *notes; + size_t i = 0; + + strscpy(prpsinfo.pr_psargs, saved_command_line, + sizeof(prpsinfo.pr_psargs)); + + notes = kzalloc(kcore_notes_len, GFP_KERNEL); + if (!notes) { + ret = -ENOMEM; + goto out; + } + + append_kcore_note(notes, &i, NN_PRSTATUS, NT_PRSTATUS, &prstatus, + sizeof(prstatus)); + append_kcore_note(notes, &i, NN_PRPSINFO, NT_PRPSINFO, &prpsinfo, + sizeof(prpsinfo)); + append_kcore_note(notes, &i, NN_TASKSTRUCT, NT_TASKSTRUCT, current, + arch_task_struct_size); + /* + * vmcoreinfo_size is mostly constant after init time, but it + * can be changed by crash_save_vmcoreinfo(). Racing here with a + * panic on another CPU before the machine goes down is insanely + * unlikely, but it's better to not leave potential buffer + * overflows lying around, regardless. + */ + append_kcore_note(notes, &i, VMCOREINFO_NOTE_NAME, 0, + vmcoreinfo_data, + min(vmcoreinfo_size, kcore_notes_len - i)); + + tsz = min_t(size_t, buflen, + notes_offset + kcore_notes_len - *fpos); + if (copy_to_iter(notes + *fpos - notes_offset, tsz, iter) != tsz) { + kfree(notes); + ret = -EFAULT; + goto out; + } + kfree(notes); + + buflen -= tsz; + *fpos += tsz; + } + + /* + * Check to see if our file offset matches with any of + * the addresses in the elf_phdr on our list. + */ + start = kc_offset_to_vaddr(*fpos - kcore_data_offset); + if ((tsz = (PAGE_SIZE - (start & ~PAGE_MASK))) > buflen) + tsz = buflen; + + m = NULL; + while (buflen) { + struct page *page; + unsigned long pfn; + phys_addr_t phys; + void *__start; + + /* + * If this is the first iteration or the address is not within + * the previous entry, search for a matching entry. + */ + if (!m || start < m->addr || start >= m->addr + m->size) { + struct kcore_list *pos; + + m = NULL; + list_for_each_entry(pos, &kclist_head, list) { + if (start >= pos->addr && + start < pos->addr + pos->size) { + m = pos; + break; + } + } + } + + if (page_offline_frozen++ % MAX_ORDER_NR_PAGES == 0) { + page_offline_thaw(); + cond_resched(); + page_offline_freeze(); + } + + if (!m) { + if (iov_iter_zero(tsz, iter) != tsz) { + ret = -EFAULT; + goto out; + } + goto skip; + } + + switch (m->type) { + case KCORE_VMALLOC: + { + const char *src = (char *)start; + size_t read = 0, left = tsz; + + /* + * vmalloc uses spinlocks, so we optimistically try to + * read memory. If this fails, fault pages in and try + * again until we are done. + */ + while (true) { + read += vread_iter(iter, src, left); + if (read == tsz) + break; + + src += read; + left -= read; + + if (fault_in_iov_iter_writeable(iter, left)) { + ret = -EFAULT; + goto out; + } + } + break; + } + case KCORE_USER: + /* User page is handled prior to normal kernel page: */ + if (copy_to_iter((char *)start, tsz, iter) != tsz) { + ret = -EFAULT; + goto out; + } + break; + case KCORE_RAM: + phys = __pa(start); + pfn = phys >> PAGE_SHIFT; + page = pfn_to_online_page(pfn); + + /* + * Don't read offline sections, logically offline pages + * (e.g., inflated in a balloon), hwpoisoned pages, + * and explicitly excluded physical ranges. + */ + if (!page || PageOffline(page) || + is_page_hwpoison(page) || !pfn_is_ram(pfn) || + pfn_is_unaccepted_memory(pfn)) { + if (iov_iter_zero(tsz, iter) != tsz) { + ret = -EFAULT; + goto out; + } + break; + } + fallthrough; + case KCORE_VMEMMAP: + case KCORE_TEXT: + if (m->type == KCORE_RAM) { + __start = kc_xlate_dev_mem_ptr(phys); + if (!__start) { + ret = -ENOMEM; + if (iov_iter_zero(tsz, iter) != tsz) + ret = -EFAULT; + goto out; + } + } else { + __start = (void *)start; + } + + /* + * Sadly we must use a bounce buffer here to be able to + * make use of copy_from_kernel_nofault(), as these + * memory regions might not always be mapped on all + * architectures. + */ + ret = copy_from_kernel_nofault(buf, __start, tsz); + if (m->type == KCORE_RAM) + kc_unxlate_dev_mem_ptr(phys, __start); + if (ret) { + if (iov_iter_zero(tsz, iter) != tsz) { + ret = -EFAULT; + goto out; + } + ret = 0; + /* + * We know the bounce buffer is safe to copy from, so + * use _copy_to_iter() directly. + */ + } else if (_copy_to_iter(buf, tsz, iter) != tsz) { + ret = -EFAULT; + goto out; + } + break; + default: + pr_warn_once("Unhandled KCORE type: %d\n", m->type); + if (iov_iter_zero(tsz, iter) != tsz) { + ret = -EFAULT; + goto out; + } + } +skip: + buflen -= tsz; + *fpos += tsz; + start += tsz; + tsz = (buflen > PAGE_SIZE ? PAGE_SIZE : buflen); + } + +out: + page_offline_thaw(); + percpu_up_read(&kclist_lock); + if (ret) + return ret; + return orig_buflen - buflen; +} + +static int open_kcore(struct inode *inode, struct file *filp) +{ + int ret = security_locked_down(LOCKDOWN_KCORE); + + if (!capable(CAP_SYS_RAWIO)) + return -EPERM; + + if (ret) + return ret; + + filp->private_data = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!filp->private_data) + return -ENOMEM; + + if (kcore_need_update) + kcore_update_ram(); + if (i_size_read(inode) != proc_root_kcore->size) { + inode_lock(inode); + i_size_write(inode, proc_root_kcore->size); + inode_unlock(inode); + } + return 0; +} + +static int release_kcore(struct inode *inode, struct file *file) +{ + kfree(file->private_data); + return 0; +} + +static const struct proc_ops kcore_proc_ops = { + .proc_flags = PROC_ENTRY_PERMANENT, + .proc_read_iter = read_kcore_iter, + .proc_open = open_kcore, + .proc_release = release_kcore, + .proc_lseek = default_llseek, +}; + +/* just remember that we have to update kcore */ +static int __meminit kcore_callback(struct notifier_block *self, + unsigned long action, void *arg) +{ + switch (action) { + case MEM_ONLINE: + case MEM_OFFLINE: + kcore_need_update = 1; + break; + } + return NOTIFY_OK; +} + + +static struct kcore_list kcore_vmalloc; + +#ifdef CONFIG_ARCH_PROC_KCORE_TEXT +static struct kcore_list kcore_text; +/* + * If defined, special segment is used for mapping kernel text instead of + * direct-map area. We need to create special TEXT section. + */ +static void __init proc_kcore_text_init(void) +{ + kclist_add(&kcore_text, _text, _end - _text, KCORE_TEXT); +} +#else +static void __init proc_kcore_text_init(void) +{ +} +#endif + +#if defined(CONFIG_MODULES) && defined(MODULES_VADDR) +/* + * MODULES_VADDR has no intersection with VMALLOC_ADDR. + */ +static struct kcore_list kcore_modules; +static void __init add_modules_range(void) +{ + if (MODULES_VADDR != VMALLOC_START && MODULES_END != VMALLOC_END) { + kclist_add(&kcore_modules, (void *)MODULES_VADDR, + MODULES_END - MODULES_VADDR, KCORE_VMALLOC); + } +} +#else +static void __init add_modules_range(void) +{ +} +#endif + +static int __init proc_kcore_init(void) +{ + proc_root_kcore = proc_create("kcore", S_IRUSR, NULL, &kcore_proc_ops); + if (!proc_root_kcore) { + pr_err("couldn't create /proc/kcore\n"); + return 0; /* Always returns 0. */ + } + /* Store text area if it's special */ + proc_kcore_text_init(); + /* Store vmalloc area */ + kclist_add(&kcore_vmalloc, (void *)VMALLOC_START, + VMALLOC_END - VMALLOC_START, KCORE_VMALLOC); + add_modules_range(); + /* Store direct-map area from physical memory map */ + kcore_update_ram(); + hotplug_memory_notifier(kcore_callback, DEFAULT_CALLBACK_PRI); + + return 0; +} +fs_initcall(proc_kcore_init); diff --git a/fs/proc/kmsg.c b/fs/proc/kmsg.c new file mode 100644 index 000000000..2fc92a13f --- /dev/null +++ b/fs/proc/kmsg.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/proc/kmsg.c + * + * Copyright (C) 1992 by Linus Torvalds + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +static int kmsg_open(struct inode * inode, struct file * file) +{ + return do_syslog(SYSLOG_ACTION_OPEN, NULL, 0, SYSLOG_FROM_PROC); +} + +static int kmsg_release(struct inode * inode, struct file * file) +{ + (void) do_syslog(SYSLOG_ACTION_CLOSE, NULL, 0, SYSLOG_FROM_PROC); + return 0; +} + +static ssize_t kmsg_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + if ((file->f_flags & O_NONBLOCK) && + !do_syslog(SYSLOG_ACTION_SIZE_UNREAD, NULL, 0, SYSLOG_FROM_PROC)) + return -EAGAIN; + return do_syslog(SYSLOG_ACTION_READ, buf, count, SYSLOG_FROM_PROC); +} + +static __poll_t kmsg_poll(struct file *file, poll_table *wait) +{ + poll_wait(file, &log_wait, wait); + if (do_syslog(SYSLOG_ACTION_SIZE_UNREAD, NULL, 0, SYSLOG_FROM_PROC)) + return EPOLLIN | EPOLLRDNORM; + return 0; +} + + +static const struct proc_ops kmsg_proc_ops = { + .proc_flags = PROC_ENTRY_PERMANENT, + .proc_read = kmsg_read, + .proc_poll = kmsg_poll, + .proc_open = kmsg_open, + .proc_release = kmsg_release, + .proc_lseek = generic_file_llseek, +}; + +static int __init proc_kmsg_init(void) +{ + proc_create("kmsg", S_IRUSR, NULL, &kmsg_proc_ops); + return 0; +} +fs_initcall(proc_kmsg_init); diff --git a/fs/proc/loadavg.c b/fs/proc/loadavg.c new file mode 100644 index 000000000..817981e57 --- /dev/null +++ b/fs/proc/loadavg.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +static int loadavg_proc_show(struct seq_file *m, void *v) +{ + unsigned long avnrun[3]; + + get_avenrun(avnrun, FIXED_1/200, 0); + + seq_printf(m, "%lu.%02lu %lu.%02lu %lu.%02lu %u/%d %d\n", + LOAD_INT(avnrun[0]), LOAD_FRAC(avnrun[0]), + LOAD_INT(avnrun[1]), LOAD_FRAC(avnrun[1]), + LOAD_INT(avnrun[2]), LOAD_FRAC(avnrun[2]), + nr_running(), nr_threads, + idr_get_cursor(&task_active_pid_ns(current)->idr) - 1); + return 0; +} + +static int __init proc_loadavg_init(void) +{ + struct proc_dir_entry *pde; + + pde = proc_create_single("loadavg", 0, NULL, loadavg_proc_show); + pde_make_permanent(pde); + return 0; +} +fs_initcall(proc_loadavg_init); diff --git a/fs/proc/meminfo.c b/fs/proc/meminfo.c new file mode 100644 index 000000000..b2813ff13 --- /dev/null +++ b/fs/proc/meminfo.c @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_CMA +#include +#endif +#include +#include +#include "internal.h" + +void __attribute__((weak)) arch_report_meminfo(struct seq_file *m) +{ +} + +static void show_val_kb(struct seq_file *m, const char *s, unsigned long num) +{ + seq_put_decimal_ull_width(m, s, num << (PAGE_SHIFT - 10), 8); + seq_write(m, " kB\n", 4); +} + +static int meminfo_proc_show(struct seq_file *m, void *v) +{ + struct sysinfo i; + unsigned long committed; + long cached; + long available; + unsigned long pages[NR_LRU_LISTS]; + unsigned long sreclaimable, sunreclaim; + int lru; + + si_meminfo(&i); + si_swapinfo(&i); + committed = vm_memory_committed(); + + cached = global_node_page_state(NR_FILE_PAGES) - + total_swapcache_pages() - i.bufferram; + if (cached < 0) + cached = 0; + + for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++) + pages[lru] = global_node_page_state(NR_LRU_BASE + lru); + + available = si_mem_available(); + sreclaimable = global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B); + sunreclaim = global_node_page_state_pages(NR_SLAB_UNRECLAIMABLE_B); + + show_val_kb(m, "MemTotal: ", i.totalram); + show_val_kb(m, "MemFree: ", i.freeram); + show_val_kb(m, "MemAvailable: ", available); + show_val_kb(m, "Buffers: ", i.bufferram); + show_val_kb(m, "Cached: ", cached); + show_val_kb(m, "SwapCached: ", total_swapcache_pages()); + show_val_kb(m, "Active: ", pages[LRU_ACTIVE_ANON] + + pages[LRU_ACTIVE_FILE]); + show_val_kb(m, "Inactive: ", pages[LRU_INACTIVE_ANON] + + pages[LRU_INACTIVE_FILE]); + show_val_kb(m, "Active(anon): ", pages[LRU_ACTIVE_ANON]); + show_val_kb(m, "Inactive(anon): ", pages[LRU_INACTIVE_ANON]); + show_val_kb(m, "Active(file): ", pages[LRU_ACTIVE_FILE]); + show_val_kb(m, "Inactive(file): ", pages[LRU_INACTIVE_FILE]); + show_val_kb(m, "Unevictable: ", pages[LRU_UNEVICTABLE]); + show_val_kb(m, "Mlocked: ", global_zone_page_state(NR_MLOCK)); + +#ifdef CONFIG_HIGHMEM + show_val_kb(m, "HighTotal: ", i.totalhigh); + show_val_kb(m, "HighFree: ", i.freehigh); + show_val_kb(m, "LowTotal: ", i.totalram - i.totalhigh); + show_val_kb(m, "LowFree: ", i.freeram - i.freehigh); +#endif + +#ifndef CONFIG_MMU + show_val_kb(m, "MmapCopy: ", + (unsigned long)atomic_long_read(&mmap_pages_allocated)); +#endif + + show_val_kb(m, "SwapTotal: ", i.totalswap); + show_val_kb(m, "SwapFree: ", i.freeswap); +#ifdef CONFIG_ZSWAP + show_val_kb(m, "Zswap: ", zswap_total_pages()); + seq_printf(m, "Zswapped: %8lu kB\n", + (unsigned long)atomic_long_read(&zswap_stored_pages) << + (PAGE_SHIFT - 10)); +#endif + show_val_kb(m, "Dirty: ", + global_node_page_state(NR_FILE_DIRTY)); + show_val_kb(m, "Writeback: ", + global_node_page_state(NR_WRITEBACK)); + show_val_kb(m, "AnonPages: ", + global_node_page_state(NR_ANON_MAPPED)); + show_val_kb(m, "Mapped: ", + global_node_page_state(NR_FILE_MAPPED)); + show_val_kb(m, "Shmem: ", i.sharedram); + show_val_kb(m, "KReclaimable: ", sreclaimable + + global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE)); + show_val_kb(m, "Slab: ", sreclaimable + sunreclaim); + show_val_kb(m, "SReclaimable: ", sreclaimable); + show_val_kb(m, "SUnreclaim: ", sunreclaim); + seq_printf(m, "KernelStack: %8lu kB\n", + global_node_page_state(NR_KERNEL_STACK_KB)); +#ifdef CONFIG_SHADOW_CALL_STACK + seq_printf(m, "ShadowCallStack:%8lu kB\n", + global_node_page_state(NR_KERNEL_SCS_KB)); +#endif + show_val_kb(m, "PageTables: ", + global_node_page_state(NR_PAGETABLE)); + show_val_kb(m, "SecPageTables: ", + global_node_page_state(NR_SECONDARY_PAGETABLE)); + + show_val_kb(m, "NFS_Unstable: ", 0); + show_val_kb(m, "Bounce: ", 0); + show_val_kb(m, "WritebackTmp: ", 0); + show_val_kb(m, "CommitLimit: ", vm_commit_limit()); + show_val_kb(m, "Committed_AS: ", committed); + seq_printf(m, "VmallocTotal: %8lu kB\n", + (unsigned long)VMALLOC_TOTAL >> 10); + show_val_kb(m, "VmallocUsed: ", + global_node_page_state(NR_VMALLOC)); + show_val_kb(m, "VmallocChunk: ", 0ul); + show_val_kb(m, "Percpu: ", pcpu_nr_pages()); + + memtest_report_meminfo(m); + +#ifdef CONFIG_MEMORY_FAILURE + seq_printf(m, "HardwareCorrupted: %5lu kB\n", + atomic_long_read(&num_poisoned_pages) << (PAGE_SHIFT - 10)); +#endif + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + show_val_kb(m, "AnonHugePages: ", + global_node_page_state(NR_ANON_THPS)); + show_val_kb(m, "ShmemHugePages: ", + global_node_page_state(NR_SHMEM_THPS)); + show_val_kb(m, "ShmemPmdMapped: ", + global_node_page_state(NR_SHMEM_PMDMAPPED)); + show_val_kb(m, "FileHugePages: ", + global_node_page_state(NR_FILE_THPS)); + show_val_kb(m, "FilePmdMapped: ", + global_node_page_state(NR_FILE_PMDMAPPED)); +#endif + +#ifdef CONFIG_CMA + show_val_kb(m, "CmaTotal: ", totalcma_pages); + show_val_kb(m, "CmaFree: ", + global_zone_page_state(NR_FREE_CMA_PAGES)); +#endif + +#ifdef CONFIG_UNACCEPTED_MEMORY + show_val_kb(m, "Unaccepted: ", + global_zone_page_state(NR_UNACCEPTED)); +#endif + show_val_kb(m, "Balloon: ", + global_node_page_state(NR_BALLOON_PAGES)); + + show_val_kb(m, "GPUActive: ", + global_node_page_state(NR_GPU_ACTIVE)); + + show_val_kb(m, "GPUReclaim: ", + global_node_page_state(NR_GPU_RECLAIM)); + + hugetlb_report_meminfo(m); + + arch_report_meminfo(m); + + return 0; +} + +static int __init proc_meminfo_init(void) +{ + struct proc_dir_entry *pde; + + pde = proc_create_single("meminfo", 0, NULL, meminfo_proc_show); + pde_make_permanent(pde); + return 0; +} +fs_initcall(proc_meminfo_init); diff --git a/fs/proc/namespaces.c b/fs/proc/namespaces.c new file mode 100644 index 000000000..ea6ec61a0 --- /dev/null +++ b/fs/proc/namespaces.c @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + + +static const struct proc_ns_operations *const ns_entries[] = { +#ifdef CONFIG_NET_NS + &netns_operations, +#endif +#ifdef CONFIG_UTS_NS + &utsns_operations, +#endif +#ifdef CONFIG_IPC_NS + &ipcns_operations, +#endif +#ifdef CONFIG_PID_NS + &pidns_operations, + &pidns_for_children_operations, +#endif +#ifdef CONFIG_USER_NS + &userns_operations, +#endif + &mntns_operations, +#ifdef CONFIG_CGROUPS + &cgroupns_operations, +#endif +#ifdef CONFIG_TIME_NS + &timens_operations, + &timens_for_children_operations, +#endif +}; + +static const char *proc_ns_get_link(struct dentry *dentry, + struct inode *inode, + struct delayed_call *done) +{ + const struct proc_ns_operations *ns_ops = PROC_I(inode)->ns_ops; + struct task_struct *task; + struct path ns_path; + int error; + + if (!dentry) + return ERR_PTR(-ECHILD); + + task = get_proc_task(inode); + if (!task) + return ERR_PTR(-EACCES); + + error = down_read_killable(&task->signal->exec_update_lock); + if (error) + goto out_put_task; + + error = -EACCES; + if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) + goto out; + + error = ns_get_path(&ns_path, task, ns_ops); + if (error) + goto out; + + error = nd_jump_link(&ns_path); +out: + up_read(&task->signal->exec_update_lock); +out_put_task: + put_task_struct(task); + return ERR_PTR(error); +} + +static int proc_ns_readlink(struct dentry *dentry, char __user *buffer, int buflen) +{ + struct inode *inode = d_inode(dentry); + const struct proc_ns_operations *ns_ops = PROC_I(inode)->ns_ops; + struct task_struct *task; + char name[50]; + int res = -EACCES; + + task = get_proc_task(inode); + if (!task) + return res; + + res = down_read_killable(&task->signal->exec_update_lock); + if (res) + goto out_put_task; + + res = -EACCES; + if (ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) { + res = ns_get_name(name, sizeof(name), task, ns_ops); + if (res >= 0) + res = readlink_copy(buffer, buflen, name, strlen(name)); + } + up_read(&task->signal->exec_update_lock); +out_put_task: + put_task_struct(task); + return res; +} + +static const struct inode_operations proc_ns_link_inode_operations = { + .readlink = proc_ns_readlink, + .get_link = proc_ns_get_link, + .setattr = proc_nochmod_setattr, +}; + +static struct dentry *proc_ns_instantiate(struct dentry *dentry, + struct task_struct *task, const void *ptr) +{ + const struct proc_ns_operations *ns_ops = ptr; + struct inode *inode; + struct proc_inode *ei; + + inode = proc_pid_make_inode(dentry->d_sb, task, S_IFLNK | S_IRWXUGO); + if (!inode) + return ERR_PTR(-ENOENT); + + ei = PROC_I(inode); + inode->i_op = &proc_ns_link_inode_operations; + ei->ns_ops = ns_ops; + pid_update_inode(task, inode); + + return d_splice_alias_ops(inode, dentry, &pid_dentry_operations); +} + +static int proc_ns_dir_readdir(struct file *file, struct dir_context *ctx) +{ + struct task_struct *task = get_proc_task(file_inode(file)); + const struct proc_ns_operations *const *entry, *const *last; + + if (!task) + return -ENOENT; + + if (!dir_emit_dots(file, ctx)) + goto out; + if (ctx->pos >= 2 + ARRAY_SIZE(ns_entries)) + goto out; + entry = ns_entries + (ctx->pos - 2); + last = &ns_entries[ARRAY_SIZE(ns_entries) - 1]; + while (entry <= last) { + const struct proc_ns_operations *ops = *entry; + if (!proc_fill_cache(file, ctx, ops->name, strlen(ops->name), + proc_ns_instantiate, task, ops)) + break; + ctx->pos++; + entry++; + } +out: + put_task_struct(task); + return 0; +} + +const struct file_operations proc_ns_dir_operations = { + .read = generic_read_dir, + .iterate_shared = proc_ns_dir_readdir, + .llseek = generic_file_llseek, +}; + +static struct dentry *proc_ns_dir_lookup(struct inode *dir, + struct dentry *dentry, unsigned int flags) +{ + struct task_struct *task = get_proc_task(dir); + const struct proc_ns_operations *const *entry, *const *last; + unsigned int len = dentry->d_name.len; + struct dentry *res = ERR_PTR(-ENOENT); + + if (!task) + goto out_no_task; + + last = &ns_entries[ARRAY_SIZE(ns_entries)]; + for (entry = ns_entries; entry < last; entry++) { + if (strlen((*entry)->name) != len) + continue; + if (!memcmp(dentry->d_name.name, (*entry)->name, len)) + break; + } + if (entry == last) + goto out; + + res = proc_ns_instantiate(dentry, task, *entry); +out: + put_task_struct(task); +out_no_task: + return res; +} + +const struct inode_operations proc_ns_dir_inode_operations = { + .lookup = proc_ns_dir_lookup, + .getattr = pid_getattr, + .setattr = proc_nochmod_setattr, +}; diff --git a/fs/proc/nommu.c b/fs/proc/nommu.c new file mode 100644 index 000000000..c6e7ebc63 --- /dev/null +++ b/fs/proc/nommu.c @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* nommu.c: mmu-less memory info files + * + * Copyright (C) 2004 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +/* + * display a single region to a sequenced file + */ +static int nommu_region_show(struct seq_file *m, struct vm_region *region) +{ + unsigned long ino = 0; + struct file *file; + dev_t dev = 0; + int flags; + + flags = region->vm_flags; + file = region->vm_file; + + if (file) { + struct inode *inode = file_inode(region->vm_file); + dev = inode->i_sb->s_dev; + ino = inode->i_ino; + } + + seq_setwidth(m, 25 + sizeof(void *) * 6 - 1); + seq_printf(m, + "%08lx-%08lx %c%c%c%c %08llx %02x:%02x %lu ", + region->vm_start, + region->vm_end, + flags & VM_READ ? 'r' : '-', + flags & VM_WRITE ? 'w' : '-', + flags & VM_EXEC ? 'x' : '-', + flags & VM_MAYSHARE ? flags & VM_SHARED ? 'S' : 's' : 'p', + ((loff_t)region->vm_pgoff) << PAGE_SHIFT, + MAJOR(dev), MINOR(dev), ino); + + if (file) { + seq_pad(m, ' '); + seq_path(m, file_user_path(file), ""); + } + + seq_putc(m, '\n'); + return 0; +} + +/* + * display a list of all the REGIONs the kernel knows about + * - nommu kernels have a single flat list + */ +static int nommu_region_list_show(struct seq_file *m, void *_p) +{ + struct rb_node *p = _p; + + return nommu_region_show(m, rb_entry(p, struct vm_region, vm_rb)); +} + +static void *nommu_region_list_start(struct seq_file *m, loff_t *_pos) +{ + struct rb_node *p; + loff_t pos = *_pos; + + down_read(&nommu_region_sem); + + for (p = rb_first(&nommu_region_tree); p; p = rb_next(p)) + if (pos-- == 0) + return p; + return NULL; +} + +static void nommu_region_list_stop(struct seq_file *m, void *v) +{ + up_read(&nommu_region_sem); +} + +static void *nommu_region_list_next(struct seq_file *m, void *v, loff_t *pos) +{ + (*pos)++; + return rb_next((struct rb_node *) v); +} + +static const struct seq_operations proc_nommu_region_list_seqop = { + .start = nommu_region_list_start, + .next = nommu_region_list_next, + .stop = nommu_region_list_stop, + .show = nommu_region_list_show +}; + +static int __init proc_nommu_init(void) +{ + proc_create_seq("maps", S_IRUGO, NULL, &proc_nommu_region_list_seqop); + return 0; +} + +fs_initcall(proc_nommu_init); diff --git a/fs/proc/page.c b/fs/proc/page.c new file mode 100644 index 000000000..260772b20 --- /dev/null +++ b/fs/proc/page.c @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +#define KPMSIZE sizeof(u64) +#define KPMMASK (KPMSIZE - 1) + +enum kpage_operation { + KPAGE_FLAGS, + KPAGE_COUNT, + KPAGE_CGROUP, +}; + +static inline unsigned long get_max_dump_pfn(void) +{ +#ifdef CONFIG_SPARSEMEM + /* + * The memmap of early sections is completely populated and marked + * online even if max_pfn does not fall on a section boundary - + * pfn_to_online_page() will succeed on all pages. Allow inspecting + * these memmaps. + */ + return round_up(max_pfn, PAGES_PER_SECTION); +#else + return max_pfn; +#endif +} + +static u64 get_kpage_count(const struct page *page) +{ + struct page_snapshot ps; + u64 ret; + + snapshot_page(&ps, page); + + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + ret = folio_precise_page_mapcount(&ps.folio_snapshot, + &ps.page_snapshot); + else + ret = folio_average_page_mapcount(&ps.folio_snapshot); + + return ret; +} + +static ssize_t kpage_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos, + enum kpage_operation op) +{ + const unsigned long max_dump_pfn = get_max_dump_pfn(); + u64 __user *out = (u64 __user *)buf; + struct page *page; + unsigned long src = *ppos; + unsigned long pfn; + ssize_t ret = 0; + u64 info; + + pfn = src / KPMSIZE; + if (src & KPMMASK || count & KPMMASK) + return -EINVAL; + if (src >= max_dump_pfn * KPMSIZE) + return 0; + count = min_t(unsigned long, count, (max_dump_pfn * KPMSIZE) - src); + + while (count > 0) { + /* + * TODO: ZONE_DEVICE support requires to identify + * memmaps that were actually initialized. + */ + page = pfn_to_online_page(pfn); + + if (page) { + switch (op) { + case KPAGE_FLAGS: + info = stable_page_flags(page); + break; + case KPAGE_COUNT: + info = get_kpage_count(page); + break; + case KPAGE_CGROUP: + info = page_cgroup_ino(page); + break; + default: + info = 0; + break; + } + } else + info = 0; + + if (put_user(info, out)) { + ret = -EFAULT; + break; + } + + pfn++; + out++; + count -= KPMSIZE; + + cond_resched(); + } + + *ppos += (char __user *)out - buf; + if (!ret) + ret = (char __user *)out - buf; + return ret; +} + +/* /proc/kpagecount - an array exposing page mapcounts + * + * Each entry is a u64 representing the corresponding + * physical page mapcount. + */ +static ssize_t kpagecount_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + return kpage_read(file, buf, count, ppos, KPAGE_COUNT); +} + +static const struct proc_ops kpagecount_proc_ops = { + .proc_flags = PROC_ENTRY_PERMANENT, + .proc_lseek = mem_lseek, + .proc_read = kpagecount_read, +}; + + +static inline u64 kpf_copy_bit(u64 kflags, int ubit, int kbit) +{ + return ((kflags >> kbit) & 1) << ubit; +} + +u64 stable_page_flags(const struct page *page) +{ + const struct folio *folio; + struct page_snapshot ps; + unsigned long k; + u64 u = 0; + + /* + * pseudo flag: KPF_NOPAGE + * it differentiates a memory hole from a page with no flags + */ + if (!page) + return BIT_ULL(KPF_NOPAGE); + + snapshot_page(&ps, page); + folio = &ps.folio_snapshot; + k = folio->flags.f; + + /* + * pseudo flags for the well known (anonymous) memory mapped pages + */ + if (folio_mapped(folio)) + u |= BIT_ULL(KPF_MMAP); + if (folio_test_anon(folio)) { + u |= BIT_ULL(KPF_ANON); + if (folio_test_ksm(folio)) + u |= BIT_ULL(KPF_KSM); + } + + /* + * compound pages: export both head/tail info + * they together define a compound page's start/end pos and order + */ + if (ps.idx == 0) + u |= kpf_copy_bit(k, KPF_COMPOUND_HEAD, PG_head); + else + u |= BIT_ULL(KPF_COMPOUND_TAIL); + if (folio_test_hugetlb(folio)) + u |= BIT_ULL(KPF_HUGE); + else if (folio_test_large(folio) && + folio_test_large_rmappable(folio)) { + /* Note: we indicate any THPs here, not just PMD-sized ones */ + u |= BIT_ULL(KPF_THP); + } else if (is_huge_zero_pfn(ps.pfn)) { + u |= BIT_ULL(KPF_ZERO_PAGE); + u |= BIT_ULL(KPF_THP); + } else if (is_zero_pfn(ps.pfn)) { + u |= BIT_ULL(KPF_ZERO_PAGE); + } + + if (ps.flags & PAGE_SNAPSHOT_PG_BUDDY) + u |= BIT_ULL(KPF_BUDDY); + + if (ps.flags & PAGE_SNAPSHOT_PG_IDLE) + u |= BIT_ULL(KPF_IDLE); + + if (folio_test_offline(folio)) + u |= BIT_ULL(KPF_OFFLINE); + if (folio_test_pgtable(folio)) + u |= BIT_ULL(KPF_PGTABLE); + if (folio_test_slab(folio)) + u |= BIT_ULL(KPF_SLAB); + + u |= kpf_copy_bit(k, KPF_LOCKED, PG_locked); + u |= kpf_copy_bit(k, KPF_DIRTY, PG_dirty); + u |= kpf_copy_bit(k, KPF_UPTODATE, PG_uptodate); + u |= kpf_copy_bit(k, KPF_WRITEBACK, PG_writeback); + + u |= kpf_copy_bit(k, KPF_LRU, PG_lru); + u |= kpf_copy_bit(k, KPF_REFERENCED, PG_referenced); + u |= kpf_copy_bit(k, KPF_ACTIVE, PG_active); + u |= kpf_copy_bit(k, KPF_RECLAIM, PG_reclaim); + + if (folio_test_swapcache(folio)) + u |= BIT_ULL(KPF_SWAPCACHE); + + u |= kpf_copy_bit(k, KPF_SWAPBACKED, PG_swapbacked); + u |= kpf_copy_bit(k, KPF_UNEVICTABLE, PG_unevictable); + u |= kpf_copy_bit(k, KPF_MLOCKED, PG_mlocked); + +#ifdef CONFIG_MEMORY_FAILURE + if (u & BIT_ULL(KPF_HUGE)) + u |= kpf_copy_bit(k, KPF_HWPOISON, PG_hwpoison); + else + u |= kpf_copy_bit(ps.page_snapshot.flags.f, KPF_HWPOISON, PG_hwpoison); +#endif + + u |= kpf_copy_bit(k, KPF_RESERVED, PG_reserved); + u |= kpf_copy_bit(k, KPF_OWNER_2, PG_owner_2); + u |= kpf_copy_bit(k, KPF_PRIVATE, PG_private); + u |= kpf_copy_bit(k, KPF_PRIVATE_2, PG_private_2); + u |= kpf_copy_bit(k, KPF_OWNER_PRIVATE, PG_owner_priv_1); + u |= kpf_copy_bit(k, KPF_ARCH, PG_arch_1); +#ifdef CONFIG_ARCH_USES_PG_ARCH_2 + u |= kpf_copy_bit(k, KPF_ARCH_2, PG_arch_2); +#endif +#ifdef CONFIG_ARCH_USES_PG_ARCH_3 + u |= kpf_copy_bit(k, KPF_ARCH_3, PG_arch_3); +#endif + + return u; +} +EXPORT_SYMBOL_GPL(stable_page_flags); + +/* /proc/kpageflags - an array exposing page flags + * + * Each entry is a u64 representing the corresponding + * physical page flags. + */ +static ssize_t kpageflags_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + return kpage_read(file, buf, count, ppos, KPAGE_FLAGS); +} + +static const struct proc_ops kpageflags_proc_ops = { + .proc_flags = PROC_ENTRY_PERMANENT, + .proc_lseek = mem_lseek, + .proc_read = kpageflags_read, +}; + +#ifdef CONFIG_MEMCG +static ssize_t kpagecgroup_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + return kpage_read(file, buf, count, ppos, KPAGE_CGROUP); +} +static const struct proc_ops kpagecgroup_proc_ops = { + .proc_flags = PROC_ENTRY_PERMANENT, + .proc_lseek = mem_lseek, + .proc_read = kpagecgroup_read, +}; +#endif /* CONFIG_MEMCG */ + +static int __init proc_page_init(void) +{ + proc_create("kpagecount", S_IRUSR, NULL, &kpagecount_proc_ops); + proc_create("kpageflags", S_IRUSR, NULL, &kpageflags_proc_ops); +#ifdef CONFIG_MEMCG + proc_create("kpagecgroup", S_IRUSR, NULL, &kpagecgroup_proc_ops); +#endif + return 0; +} +fs_initcall(proc_page_init); diff --git a/fs/proc/proc_net.c b/fs/proc/proc_net.c new file mode 100644 index 000000000..00cc385bc --- /dev/null +++ b/fs/proc/proc_net.c @@ -0,0 +1,426 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/proc/net.c + * + * Copyright (C) 2007 + * + * Author: Eric Biederman + * + * proc net directory handling functions + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +static inline struct net *PDE_NET(struct proc_dir_entry *pde) +{ + return pde->parent->data; +} + +static struct net *get_proc_net(const struct inode *inode) +{ + return maybe_get_net(PDE_NET(PDE(inode))); +} + +static int seq_open_net(struct inode *inode, struct file *file) +{ + unsigned int state_size = PDE(inode)->state_size; + struct seq_net_private *p; + struct net *net; + + WARN_ON_ONCE(state_size < sizeof(*p)); + + if (file->f_mode & FMODE_WRITE && !PDE(inode)->write) + return -EACCES; + + net = get_proc_net(inode); + if (!net) + return -ENXIO; + + p = __seq_open_private(file, PDE(inode)->seq_ops, state_size); + if (!p) { + put_net(net); + return -ENOMEM; + } +#ifdef CONFIG_NET_NS + p->net = net; + netns_tracker_alloc(net, &p->ns_tracker, GFP_KERNEL); +#endif + return 0; +} + +static void seq_file_net_put_net(struct seq_file *seq) +{ +#ifdef CONFIG_NET_NS + struct seq_net_private *priv = seq->private; + + put_net_track(priv->net, &priv->ns_tracker); +#else + put_net(&init_net); +#endif +} + +static int seq_release_net(struct inode *ino, struct file *f) +{ + struct seq_file *seq = f->private_data; + + seq_file_net_put_net(seq); + seq_release_private(ino, f); + return 0; +} + +static const struct proc_ops proc_net_seq_ops = { + .proc_open = seq_open_net, + .proc_read = seq_read, + .proc_write = proc_simple_write, + .proc_lseek = seq_lseek, + .proc_release = seq_release_net, +}; + +int bpf_iter_init_seq_net(void *priv_data, struct bpf_iter_aux_info *aux) +{ +#ifdef CONFIG_NET_NS + struct seq_net_private *p = priv_data; + + p->net = get_net_track(current->nsproxy->net_ns, &p->ns_tracker, + GFP_KERNEL); +#endif + return 0; +} + +void bpf_iter_fini_seq_net(void *priv_data) +{ +#ifdef CONFIG_NET_NS + struct seq_net_private *p = priv_data; + + put_net_track(p->net, &p->ns_tracker); +#endif +} + +struct proc_dir_entry *proc_create_net_data(const char *name, umode_t mode, + struct proc_dir_entry *parent, const struct seq_operations *ops, + unsigned int state_size, void *data) +{ + struct proc_dir_entry *p; + + p = proc_create_reg(name, mode, &parent, data); + if (!p) + return NULL; + pde_force_lookup(p); + p->proc_ops = &proc_net_seq_ops; + p->seq_ops = ops; + p->state_size = state_size; + return proc_register(parent, p); +} +EXPORT_SYMBOL_GPL(proc_create_net_data); + +/** + * proc_create_net_data_write - Create a writable net_ns-specific proc file + * @name: The name of the file. + * @mode: The file's access mode. + * @parent: The parent directory in which to create. + * @ops: The seq_file ops with which to read the file. + * @write: The write method with which to 'modify' the file. + * @state_size: The size of the per-file private state to allocate. + * @data: Data for retrieval by pde_data(). + * + * Create a network namespaced proc file in the @parent directory with the + * specified @name and @mode that allows reading of a file that displays a + * series of elements and also provides for the file accepting writes that have + * some arbitrary effect. + * + * The functions in the @ops table are used to iterate over items to be + * presented and extract the readable content using the seq_file interface. + * + * The @write function is called with the data copied into a kernel space + * scratch buffer and has a NUL appended for convenience. The buffer may be + * modified by the @write function. @write should return 0 on success. + * + * The @data value is accessible from the @show and @write functions by calling + * pde_data() on the file inode. The network namespace must be accessed by + * calling seq_file_net() on the seq_file struct. + */ +struct proc_dir_entry *proc_create_net_data_write(const char *name, umode_t mode, + struct proc_dir_entry *parent, + const struct seq_operations *ops, + proc_write_t write, + unsigned int state_size, void *data) +{ + struct proc_dir_entry *p; + + p = proc_create_reg(name, mode, &parent, data); + if (!p) + return NULL; + pde_force_lookup(p); + p->proc_ops = &proc_net_seq_ops; + p->seq_ops = ops; + p->state_size = state_size; + p->write = write; + return proc_register(parent, p); +} +EXPORT_SYMBOL_GPL(proc_create_net_data_write); + +static int single_open_net(struct inode *inode, struct file *file) +{ + struct proc_dir_entry *de = PDE(inode); + struct net *net; + int err; + + net = get_proc_net(inode); + if (!net) + return -ENXIO; + + err = single_open(file, de->single_show, net); + if (err) + put_net(net); + return err; +} + +static int single_release_net(struct inode *ino, struct file *f) +{ + struct seq_file *seq = f->private_data; + put_net(seq->private); + return single_release(ino, f); +} + +static const struct proc_ops proc_net_single_ops = { + .proc_open = single_open_net, + .proc_read = seq_read, + .proc_write = proc_simple_write, + .proc_lseek = seq_lseek, + .proc_release = single_release_net, +}; + +struct proc_dir_entry *proc_create_net_single(const char *name, umode_t mode, + struct proc_dir_entry *parent, + int (*show)(struct seq_file *, void *), void *data) +{ + struct proc_dir_entry *p; + + p = proc_create_reg(name, mode, &parent, data); + if (!p) + return NULL; + pde_force_lookup(p); + p->proc_ops = &proc_net_single_ops; + p->single_show = show; + return proc_register(parent, p); +} +EXPORT_SYMBOL_GPL(proc_create_net_single); + +/** + * proc_create_net_single_write - Create a writable net_ns-specific proc file + * @name: The name of the file. + * @mode: The file's access mode. + * @parent: The parent directory in which to create. + * @show: The seqfile show method with which to read the file. + * @write: The write method with which to 'modify' the file. + * @data: Data for retrieval by pde_data(). + * + * Create a network-namespaced proc file in the @parent directory with the + * specified @name and @mode that allows reading of a file that displays a + * single element rather than a series and also provides for the file accepting + * writes that have some arbitrary effect. + * + * The @show function is called to extract the readable content via the + * seq_file interface. + * + * The @write function is called with the data copied into a kernel space + * scratch buffer and has a NUL appended for convenience. The buffer may be + * modified by the @write function. @write should return 0 on success. + * + * The @data value is accessible from the @show and @write functions by calling + * pde_data() on the file inode. The network namespace must be accessed by + * calling seq_file_single_net() on the seq_file struct. + */ +struct proc_dir_entry *proc_create_net_single_write(const char *name, umode_t mode, + struct proc_dir_entry *parent, + int (*show)(struct seq_file *, void *), + proc_write_t write, + void *data) +{ + struct proc_dir_entry *p; + + p = proc_create_reg(name, mode, &parent, data); + if (!p) + return NULL; + pde_force_lookup(p); + p->proc_ops = &proc_net_single_ops; + p->single_show = show; + p->write = write; + return proc_register(parent, p); +} +EXPORT_SYMBOL_GPL(proc_create_net_single_write); + +static struct net *get_proc_task_net(struct inode *dir) +{ + struct task_struct *task; + struct nsproxy *ns; + struct net *net = NULL; + struct proc_fs_info *fs_info = proc_sb_info(dir->i_sb); + + rcu_read_lock(); + task = pid_task(proc_pid(dir), PIDTYPE_PID); + if (task != NULL) { + task_lock(task); + ns = task->nsproxy; + if (ns != NULL) + net = get_net(ns->net_ns); + task_unlock(task); + } + rcu_read_unlock(); + + if (net && (fs_info->pidonly == PROC_PIDONLY_ON) && + security_capable(fs_info->mounter_cred, net->user_ns, CAP_NET_ADMIN, CAP_OPT_NONE) < 0) { + put_net(net); + net = NULL; + } + + return net; +} + +static struct dentry *proc_tgid_net_lookup(struct inode *dir, + struct dentry *dentry, unsigned int flags) +{ + struct dentry *de; + struct net *net; + + de = ERR_PTR(-ENOENT); + net = get_proc_task_net(dir); + if (net != NULL) { + de = proc_lookup_de(dir, dentry, net->proc_net); + put_net(net); + } + return de; +} + +static int proc_tgid_net_getattr(struct mnt_idmap *idmap, + const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int query_flags) +{ + struct inode *inode = d_inode(path->dentry); + struct net *net; + + net = get_proc_task_net(inode); + + generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat); + + if (net != NULL) { + stat->nlink = net->proc_net->nlink; + put_net(net); + } + + return 0; +} + +const struct inode_operations proc_net_inode_operations = { + .lookup = proc_tgid_net_lookup, + .getattr = proc_tgid_net_getattr, + .setattr = proc_nochmod_setattr, +}; + +static int proc_tgid_net_readdir(struct file *file, struct dir_context *ctx) +{ + int ret; + struct net *net; + + ret = -EINVAL; + net = get_proc_task_net(file_inode(file)); + if (net != NULL) { + ret = proc_readdir_de(file, ctx, net->proc_net); + put_net(net); + } + return ret; +} + +const struct file_operations proc_net_operations = { + .llseek = generic_file_llseek, + .read = generic_read_dir, + .iterate_shared = proc_tgid_net_readdir, +}; + +static __net_init int proc_net_ns_init(struct net *net) +{ + struct proc_dir_entry *netd, *net_statd; + kuid_t uid; + kgid_t gid; + int err; + + /* + * This PDE acts only as an anchor for /proc/${pid}/net hierarchy. + * Corresponding inode (PDE(inode) == net->proc_net) is never + * instantiated therefore blanket zeroing is fine. + * net->proc_net_stat inode is instantiated normally. + */ + err = -ENOMEM; + netd = kmem_cache_zalloc(proc_dir_entry_cache, GFP_KERNEL); + if (!netd) + goto out; + + netd->subdir = RB_ROOT; + netd->data = net; + netd->nlink = 2; + netd->namelen = 3; + netd->parent = &proc_root; + netd->name = netd->inline_name; + memcpy(netd->name, "net", 4); + + uid = make_kuid(net->user_ns, 0); + if (!uid_valid(uid)) + uid = netd->uid; + + gid = make_kgid(net->user_ns, 0); + if (!gid_valid(gid)) + gid = netd->gid; + + proc_set_user(netd, uid, gid); + + /* Seed dentry revalidation for /proc/${pid}/net */ + pde_force_lookup(netd); + + err = -EEXIST; + net_statd = proc_net_mkdir(net, "stat", netd); + if (!net_statd) + goto free_net; + + net->proc_net = netd; + net->proc_net_stat = net_statd; + return 0; + +free_net: + pde_free(netd); +out: + return err; +} + +static __net_exit void proc_net_ns_exit(struct net *net) +{ + remove_proc_entry("stat", net->proc_net); + pde_free(net->proc_net); +} + +static struct pernet_operations __net_initdata proc_net_ns_ops = { + .init = proc_net_ns_init, + .exit = proc_net_ns_exit, +}; + +int __init proc_net_init(void) +{ + proc_symlink("net", NULL, "self/net"); + + return register_pernet_subsys(&proc_net_ns_ops); +} diff --git a/fs/proc/proc_sysctl.c b/fs/proc/proc_sysctl.c new file mode 100644 index 000000000..04a382178 --- /dev/null +++ b/fs/proc/proc_sysctl.c @@ -0,0 +1,1725 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * /proc/sys support + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +#define list_for_each_table_entry(entry, header) \ + entry = header->ctl_table; \ + for (size_t i = 0 ; i < header->ctl_table_size; ++i, entry++) + +static const struct dentry_operations proc_sys_dentry_operations; +static const struct file_operations proc_sys_file_operations; +static const struct inode_operations proc_sys_inode_operations; +static const struct file_operations proc_sys_dir_file_operations; +static const struct inode_operations proc_sys_dir_operations; + +/* + * Support for permanently empty directories. + * Must be non-empty to avoid sharing an address with other tables. + */ +static const struct ctl_table sysctl_mount_point[] = { + { } +}; + +/** + * register_sysctl_mount_point() - registers a sysctl mount point + * @path: path for the mount point + * + * Used to create a permanently empty directory to serve as mount point. + * There are some subtle but important permission checks this allows in the + * case of unprivileged mounts. + */ +struct ctl_table_header *register_sysctl_mount_point(const char *path) +{ + return register_sysctl_sz(path, sysctl_mount_point, 0); +} +EXPORT_SYMBOL(register_sysctl_mount_point); + +#define sysctl_is_perm_empty_ctl_header(hptr) \ + (hptr->type == SYSCTL_TABLE_TYPE_PERMANENTLY_EMPTY) +#define sysctl_set_perm_empty_ctl_header(hptr) \ + (hptr->type = SYSCTL_TABLE_TYPE_PERMANENTLY_EMPTY) +#define sysctl_clear_perm_empty_ctl_header(hptr) \ + (hptr->type = SYSCTL_TABLE_TYPE_DEFAULT) + +void proc_sys_poll_notify(struct ctl_table_poll *poll) +{ + if (!poll) + return; + + atomic_inc(&poll->event); + wake_up_interruptible(&poll->wait); +} + +static const struct ctl_table root_table[] = { + { + .procname = "", + .mode = S_IFDIR|S_IRUGO|S_IXUGO, + }, +}; +static struct ctl_table_root sysctl_table_root = { + .default_set.dir.header = { + {{.count = 1, + .nreg = 1, + .ctl_table = root_table }}, + .ctl_table_arg = root_table, + .root = &sysctl_table_root, + .set = &sysctl_table_root.default_set, + }, +}; + +static DEFINE_SPINLOCK(sysctl_lock); + +static void drop_sysctl_table(struct ctl_table_header *header); +static int sysctl_follow_link(struct ctl_table_header **phead, + const struct ctl_table **pentry); +static int insert_links(struct ctl_table_header *head); +static void put_links(struct ctl_table_header *header); + +static void sysctl_print_dir(struct ctl_dir *dir) +{ + if (dir->header.parent) + sysctl_print_dir(dir->header.parent); + pr_cont("%s/", dir->header.ctl_table[0].procname); +} + +static int namecmp(const char *name1, int len1, const char *name2, int len2) +{ + int cmp; + + cmp = memcmp(name1, name2, min(len1, len2)); + if (cmp == 0) + cmp = len1 - len2; + return cmp; +} + +static const struct ctl_table *find_entry(struct ctl_table_header **phead, + struct ctl_dir *dir, const char *name, int namelen) +{ + struct ctl_table_header *head; + const struct ctl_table *entry; + struct rb_node *node = dir->root.rb_node; + + lockdep_assert_held(&sysctl_lock); + + while (node) + { + struct ctl_node *ctl_node; + const char *procname; + int cmp; + + ctl_node = rb_entry(node, struct ctl_node, node); + head = ctl_node->header; + entry = &head->ctl_table[ctl_node - head->node]; + procname = entry->procname; + + cmp = namecmp(name, namelen, procname, strlen(procname)); + if (cmp < 0) + node = node->rb_left; + else if (cmp > 0) + node = node->rb_right; + else { + *phead = head; + return entry; + } + } + return NULL; +} + +static int insert_entry(struct ctl_table_header *head, const struct ctl_table *entry) +{ + struct rb_node *node = &head->node[entry - head->ctl_table].node; + struct rb_node **p = &head->parent->root.rb_node; + struct rb_node *parent = NULL; + const char *name = entry->procname; + int namelen = strlen(name); + + while (*p) { + struct ctl_table_header *parent_head; + const struct ctl_table *parent_entry; + struct ctl_node *parent_node; + const char *parent_name; + int cmp; + + parent = *p; + parent_node = rb_entry(parent, struct ctl_node, node); + parent_head = parent_node->header; + parent_entry = &parent_head->ctl_table[parent_node - parent_head->node]; + parent_name = parent_entry->procname; + + cmp = namecmp(name, namelen, parent_name, strlen(parent_name)); + if (cmp < 0) + p = &(*p)->rb_left; + else if (cmp > 0) + p = &(*p)->rb_right; + else { + pr_err("sysctl duplicate entry: "); + sysctl_print_dir(head->parent); + pr_cont("%s\n", entry->procname); + return -EEXIST; + } + } + + rb_link_node(node, parent, p); + rb_insert_color(node, &head->parent->root); + return 0; +} + +static void erase_entry(struct ctl_table_header *head, const struct ctl_table *entry) +{ + struct rb_node *node = &head->node[entry - head->ctl_table].node; + + rb_erase(node, &head->parent->root); +} + +static void init_header(struct ctl_table_header *head, + struct ctl_table_root *root, struct ctl_table_set *set, + struct ctl_node *node, const struct ctl_table *table, size_t table_size) +{ + head->ctl_table = table; + head->ctl_table_size = table_size; + head->ctl_table_arg = table; + head->used = 0; + head->count = 1; + head->nreg = 1; + head->unregistering = NULL; + head->root = root; + head->set = set; + head->parent = NULL; + head->node = node; + INIT_HLIST_HEAD(&head->inodes); + if (node) { + const struct ctl_table *entry; + + list_for_each_table_entry(entry, head) { + node->header = head; + node++; + } + } + if (table == sysctl_mount_point) + sysctl_set_perm_empty_ctl_header(head); +} + +static void erase_header(struct ctl_table_header *head) +{ + const struct ctl_table *entry; + + list_for_each_table_entry(entry, head) + erase_entry(head, entry); +} + +static int insert_header(struct ctl_dir *dir, struct ctl_table_header *header) +{ + const struct ctl_table *entry; + struct ctl_table_header *dir_h = &dir->header; + int err; + + + /* Is this a permanently empty directory? */ + if (sysctl_is_perm_empty_ctl_header(dir_h)) + return -EROFS; + + /* Am I creating a permanently empty directory? */ + if (sysctl_is_perm_empty_ctl_header(header)) { + if (!RB_EMPTY_ROOT(&dir->root)) + return -EINVAL; + sysctl_set_perm_empty_ctl_header(dir_h); + } + + dir_h->nreg++; + header->parent = dir; + err = insert_links(header); + if (err) + goto fail_links; + list_for_each_table_entry(entry, header) { + err = insert_entry(header, entry); + if (err) + goto fail; + } + return 0; +fail: + erase_header(header); + put_links(header); +fail_links: + if (header->ctl_table == sysctl_mount_point) + sysctl_clear_perm_empty_ctl_header(dir_h); + header->parent = NULL; + drop_sysctl_table(dir_h); + return err; +} + +static int use_table(struct ctl_table_header *p) +{ + lockdep_assert_held(&sysctl_lock); + + if (unlikely(p->unregistering)) + return 0; + p->used++; + return 1; +} + +static void unuse_table(struct ctl_table_header *p) +{ + lockdep_assert_held(&sysctl_lock); + + if (!--p->used) + if (unlikely(p->unregistering)) + complete(p->unregistering); +} + +static void proc_sys_invalidate_dcache(struct ctl_table_header *head) +{ + proc_invalidate_siblings_dcache(&head->inodes, &sysctl_lock); +} + +static void start_unregistering(struct ctl_table_header *p) +{ + /* will reacquire if has to wait */ + lockdep_assert_held(&sysctl_lock); + + /* + * if p->used is 0, nobody will ever touch that entry again; + * we'll eliminate all paths to it before dropping sysctl_lock + */ + if (unlikely(p->used)) { + struct completion wait; + init_completion(&wait); + p->unregistering = &wait; + spin_unlock(&sysctl_lock); + wait_for_completion(&wait); + } else { + /* anything non-NULL; we'll never dereference it */ + p->unregistering = ERR_PTR(-EINVAL); + spin_unlock(&sysctl_lock); + } + /* + * Invalidate dentries for unregistered sysctls: namespaced sysctls + * can have duplicate names and contaminate dcache very badly. + */ + proc_sys_invalidate_dcache(p); + /* + * do not remove from the list until nobody holds it; walking the + * list in do_sysctl() relies on that. + */ + spin_lock(&sysctl_lock); + erase_header(p); +} + +static struct ctl_table_header *sysctl_head_grab(struct ctl_table_header *head) +{ + BUG_ON(!head); + spin_lock(&sysctl_lock); + if (!use_table(head)) + head = ERR_PTR(-ENOENT); + spin_unlock(&sysctl_lock); + return head; +} + +static void sysctl_head_finish(struct ctl_table_header *head) +{ + if (!head) + return; + spin_lock(&sysctl_lock); + unuse_table(head); + spin_unlock(&sysctl_lock); +} + +static struct ctl_table_set * +lookup_header_set(struct ctl_table_root *root) +{ + struct ctl_table_set *set = &root->default_set; + if (root->lookup) + set = root->lookup(root); + return set; +} + +static const struct ctl_table *lookup_entry(struct ctl_table_header **phead, + struct ctl_dir *dir, + const char *name, int namelen) +{ + struct ctl_table_header *head; + const struct ctl_table *entry; + + spin_lock(&sysctl_lock); + entry = find_entry(&head, dir, name, namelen); + if (entry && use_table(head)) + *phead = head; + else + entry = NULL; + spin_unlock(&sysctl_lock); + return entry; +} + +static struct ctl_node *first_usable_entry(struct rb_node *node) +{ + struct ctl_node *ctl_node; + + for (;node; node = rb_next(node)) { + ctl_node = rb_entry(node, struct ctl_node, node); + if (use_table(ctl_node->header)) + return ctl_node; + } + return NULL; +} + +static void first_entry(struct ctl_dir *dir, + struct ctl_table_header **phead, const struct ctl_table **pentry) +{ + struct ctl_table_header *head = NULL; + const struct ctl_table *entry = NULL; + struct ctl_node *ctl_node; + + spin_lock(&sysctl_lock); + ctl_node = first_usable_entry(rb_first(&dir->root)); + spin_unlock(&sysctl_lock); + if (ctl_node) { + head = ctl_node->header; + entry = &head->ctl_table[ctl_node - head->node]; + } + *phead = head; + *pentry = entry; +} + +static void next_entry(struct ctl_table_header **phead, const struct ctl_table **pentry) +{ + struct ctl_table_header *head = *phead; + const struct ctl_table *entry = *pentry; + struct ctl_node *ctl_node = &head->node[entry - head->ctl_table]; + + spin_lock(&sysctl_lock); + unuse_table(head); + + ctl_node = first_usable_entry(rb_next(&ctl_node->node)); + spin_unlock(&sysctl_lock); + head = NULL; + if (ctl_node) { + head = ctl_node->header; + entry = &head->ctl_table[ctl_node - head->node]; + } + *phead = head; + *pentry = entry; +} + +/* + * sysctl_perm does NOT grant the superuser all rights automatically, because + * some sysctl variables are readonly even to root. + */ + +static int test_perm(int mode, int op) +{ + if (uid_eq(current_euid(), GLOBAL_ROOT_UID)) + mode >>= 6; + else if (in_egroup_p(GLOBAL_ROOT_GID)) + mode >>= 3; + if ((op & ~mode & (MAY_READ|MAY_WRITE|MAY_EXEC)) == 0) + return 0; + return -EACCES; +} + +static int sysctl_perm(struct ctl_table_header *head, const struct ctl_table *table, int op) +{ + struct ctl_table_root *root = head->root; + int mode; + + if (root->permissions) + mode = root->permissions(head, table); + else + mode = table->mode; + + return test_perm(mode, op); +} + +static struct inode *proc_sys_make_inode(struct super_block *sb, + struct ctl_table_header *head, const struct ctl_table *table) +{ + struct ctl_table_root *root = head->root; + struct inode *inode; + struct proc_inode *ei; + + inode = new_inode(sb); + if (!inode) + return ERR_PTR(-ENOMEM); + + inode->i_ino = get_next_ino(); + + ei = PROC_I(inode); + + spin_lock(&sysctl_lock); + if (unlikely(head->unregistering)) { + spin_unlock(&sysctl_lock); + iput(inode); + return ERR_PTR(-ENOENT); + } + ei->sysctl = head; + ei->sysctl_entry = table; + hlist_add_head_rcu(&ei->sibling_inodes, &head->inodes); + head->count++; + spin_unlock(&sysctl_lock); + + simple_inode_init_ts(inode); + inode->i_mode = table->mode; + if (!S_ISDIR(table->mode)) { + inode->i_mode |= S_IFREG; + inode->i_op = &proc_sys_inode_operations; + inode->i_fop = &proc_sys_file_operations; + } else { + inode->i_mode |= S_IFDIR; + inode->i_op = &proc_sys_dir_operations; + inode->i_fop = &proc_sys_dir_file_operations; + if (sysctl_is_perm_empty_ctl_header(head)) + make_empty_dir_inode(inode); + } + + inode->i_uid = GLOBAL_ROOT_UID; + inode->i_gid = GLOBAL_ROOT_GID; + if (root->set_ownership) + root->set_ownership(head, &inode->i_uid, &inode->i_gid); + + return inode; +} + +void proc_sys_evict_inode(struct inode *inode, struct ctl_table_header *head) +{ + spin_lock(&sysctl_lock); + hlist_del_init_rcu(&PROC_I(inode)->sibling_inodes); + if (!--head->count) + kfree_rcu(head, rcu); + spin_unlock(&sysctl_lock); +} + +static struct ctl_table_header *grab_header(struct inode *inode) +{ + struct ctl_table_header *head = PROC_I(inode)->sysctl; + if (!head) + head = &sysctl_table_root.default_set.dir.header; + return sysctl_head_grab(head); +} + +static struct dentry *proc_sys_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + struct ctl_table_header *head = grab_header(dir); + struct ctl_table_header *h = NULL; + const struct qstr *name = &dentry->d_name; + const struct ctl_table *p; + struct inode *inode; + struct dentry *err = ERR_PTR(-ENOENT); + struct ctl_dir *ctl_dir; + int ret; + + if (IS_ERR(head)) + return ERR_CAST(head); + + ctl_dir = container_of(head, struct ctl_dir, header); + + p = lookup_entry(&h, ctl_dir, name->name, name->len); + if (!p) + goto out; + + if (S_ISLNK(p->mode)) { + ret = sysctl_follow_link(&h, &p); + err = ERR_PTR(ret); + if (ret) + goto out; + } + + inode = proc_sys_make_inode(dir->i_sb, h ? h : head, p); + err = d_splice_alias_ops(inode, dentry, &proc_sys_dentry_operations); + +out: + if (h) + sysctl_head_finish(h); + sysctl_head_finish(head); + return err; +} + +static ssize_t proc_sys_call_handler(struct kiocb *iocb, struct iov_iter *iter, + int write) +{ + struct inode *inode = file_inode(iocb->ki_filp); + struct ctl_table_header *head = grab_header(inode); + const struct ctl_table *table = PROC_I(inode)->sysctl_entry; + size_t count = iov_iter_count(iter); + char *kbuf; + ssize_t error; + + if (IS_ERR(head)) + return PTR_ERR(head); + + /* + * At this point we know that the sysctl was not unregistered + * and won't be until we finish. + */ + error = -EPERM; + if (sysctl_perm(head, table, write ? MAY_WRITE : MAY_READ)) + goto out; + + /* if that can happen at all, it should be -EINVAL, not -EISDIR */ + error = -EINVAL; + if (!table->proc_handler) + goto out; + + /* don't even try if the size is too large */ + error = -ENOMEM; + if (count >= KMALLOC_MAX_SIZE) + goto out; + kbuf = kvzalloc(count + 1, GFP_KERNEL); + if (!kbuf) + goto out; + + if (write) { + error = -EFAULT; + if (!copy_from_iter_full(kbuf, count, iter)) + goto out_free_buf; + kbuf[count] = '\0'; + } + + error = BPF_CGROUP_RUN_PROG_SYSCTL(head, table, write, &kbuf, &count, + &iocb->ki_pos); + if (error) + goto out_free_buf; + + /* careful: calling conventions are nasty here */ + error = table->proc_handler(table, write, kbuf, &count, &iocb->ki_pos); + if (error) + goto out_free_buf; + + if (!write) { + error = -EFAULT; + if (copy_to_iter(kbuf, count, iter) < count) + goto out_free_buf; + } + + error = count; +out_free_buf: + kvfree(kbuf); +out: + sysctl_head_finish(head); + + return error; +} + +static ssize_t proc_sys_read(struct kiocb *iocb, struct iov_iter *iter) +{ + return proc_sys_call_handler(iocb, iter, 0); +} + +static ssize_t proc_sys_write(struct kiocb *iocb, struct iov_iter *iter) +{ + return proc_sys_call_handler(iocb, iter, 1); +} + +static int proc_sys_open(struct inode *inode, struct file *filp) +{ + struct ctl_table_header *head = grab_header(inode); + const struct ctl_table *table = PROC_I(inode)->sysctl_entry; + + /* sysctl was unregistered */ + if (IS_ERR(head)) + return PTR_ERR(head); + + if (table->poll) + filp->private_data = proc_sys_poll_event(table->poll); + + sysctl_head_finish(head); + + return 0; +} + +static __poll_t proc_sys_poll(struct file *filp, poll_table *wait) +{ + struct inode *inode = file_inode(filp); + struct ctl_table_header *head = grab_header(inode); + const struct ctl_table *table = PROC_I(inode)->sysctl_entry; + __poll_t ret = DEFAULT_POLLMASK; + unsigned long event; + + /* sysctl was unregistered */ + if (IS_ERR(head)) + return EPOLLERR | EPOLLHUP; + + if (!table->proc_handler) + goto out; + + if (!table->poll) + goto out; + + event = (unsigned long)filp->private_data; + poll_wait(filp, &table->poll->wait, wait); + + if (event != atomic_read(&table->poll->event)) { + filp->private_data = proc_sys_poll_event(table->poll); + ret = EPOLLIN | EPOLLRDNORM | EPOLLERR | EPOLLPRI; + } + +out: + sysctl_head_finish(head); + + return ret; +} + +static bool proc_sys_fill_cache(struct file *file, + struct dir_context *ctx, + struct ctl_table_header *head, + const struct ctl_table *table) +{ + struct dentry *child, *dir = file->f_path.dentry; + struct inode *inode; + struct qstr qname; + ino_t ino = 0; + unsigned type = DT_UNKNOWN; + + qname.name = table->procname; + qname.len = strlen(table->procname); + qname.hash = full_name_hash(dir, qname.name, qname.len); + + child = d_lookup(dir, &qname); + if (!child) { + child = d_alloc_parallel(dir, &qname); + if (IS_ERR(child)) + return false; + if (d_in_lookup(child)) { + struct dentry *res; + inode = proc_sys_make_inode(dir->d_sb, head, table); + res = d_splice_alias_ops(inode, child, + &proc_sys_dentry_operations); + d_lookup_done(child); + if (unlikely(res)) { + dput(child); + + if (IS_ERR(res)) + return false; + + child = res; + } + } + } + inode = d_inode(child); + ino = inode->i_ino; + type = inode->i_mode >> 12; + dput(child); + return dir_emit(ctx, qname.name, qname.len, ino, type); +} + +static bool proc_sys_link_fill_cache(struct file *file, + struct dir_context *ctx, + struct ctl_table_header *head, + const struct ctl_table *table) +{ + bool ret = true; + + head = sysctl_head_grab(head); + if (IS_ERR(head)) + return false; + + /* It is not an error if we can not follow the link ignore it */ + if (sysctl_follow_link(&head, &table)) + goto out; + + ret = proc_sys_fill_cache(file, ctx, head, table); +out: + sysctl_head_finish(head); + return ret; +} + +static int scan(struct ctl_table_header *head, const struct ctl_table *table, + unsigned long *pos, struct file *file, + struct dir_context *ctx) +{ + bool res; + + if ((*pos)++ < ctx->pos) + return true; + + if (unlikely(S_ISLNK(table->mode))) + res = proc_sys_link_fill_cache(file, ctx, head, table); + else + res = proc_sys_fill_cache(file, ctx, head, table); + + if (res) + ctx->pos = *pos; + + return res; +} + +static int proc_sys_readdir(struct file *file, struct dir_context *ctx) +{ + struct ctl_table_header *head = grab_header(file_inode(file)); + struct ctl_table_header *h = NULL; + const struct ctl_table *entry; + struct ctl_dir *ctl_dir; + unsigned long pos; + + if (IS_ERR(head)) + return PTR_ERR(head); + + ctl_dir = container_of(head, struct ctl_dir, header); + + if (!dir_emit_dots(file, ctx)) + goto out; + + pos = 2; + + for (first_entry(ctl_dir, &h, &entry); h; next_entry(&h, &entry)) { + if (!scan(h, entry, &pos, file, ctx)) { + sysctl_head_finish(h); + break; + } + } +out: + sysctl_head_finish(head); + return 0; +} + +static int proc_sys_permission(struct mnt_idmap *idmap, + struct inode *inode, int mask) +{ + /* + * sysctl entries that are not writeable, + * are _NOT_ writeable, capabilities or not. + */ + struct ctl_table_header *head; + const struct ctl_table *table; + int error; + + /* Executable files are not allowed under /proc/sys/ */ + if ((mask & MAY_EXEC) && S_ISREG(inode->i_mode)) + return -EACCES; + + head = grab_header(inode); + if (IS_ERR(head)) + return PTR_ERR(head); + + table = PROC_I(inode)->sysctl_entry; + if (!table) /* global root - r-xr-xr-x */ + error = mask & MAY_WRITE ? -EACCES : 0; + else /* Use the permissions on the sysctl table entry */ + error = sysctl_perm(head, table, mask & ~MAY_NOT_BLOCK); + + sysctl_head_finish(head); + return error; +} + +static int proc_sys_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, struct iattr *attr) +{ + struct inode *inode = d_inode(dentry); + int error; + + if (attr->ia_valid & (ATTR_MODE | ATTR_UID | ATTR_GID)) + return -EPERM; + + error = setattr_prepare(&nop_mnt_idmap, dentry, attr); + if (error) + return error; + + setattr_copy(&nop_mnt_idmap, inode, attr); + return 0; +} + +static int proc_sys_getattr(struct mnt_idmap *idmap, + const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int query_flags) +{ + struct inode *inode = d_inode(path->dentry); + struct ctl_table_header *head = grab_header(inode); + const struct ctl_table *table = PROC_I(inode)->sysctl_entry; + + if (IS_ERR(head)) + return PTR_ERR(head); + + generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat); + if (table) + stat->mode = (stat->mode & S_IFMT) | table->mode; + + sysctl_head_finish(head); + return 0; +} + +static const struct file_operations proc_sys_file_operations = { + .open = proc_sys_open, + .poll = proc_sys_poll, + .read_iter = proc_sys_read, + .write_iter = proc_sys_write, + .splice_read = copy_splice_read, + .splice_write = iter_file_splice_write, + .llseek = default_llseek, +}; + +static const struct file_operations proc_sys_dir_file_operations = { + .read = generic_read_dir, + .iterate_shared = proc_sys_readdir, + .llseek = generic_file_llseek, +}; + +static const struct inode_operations proc_sys_inode_operations = { + .permission = proc_sys_permission, + .setattr = proc_sys_setattr, + .getattr = proc_sys_getattr, +}; + +static const struct inode_operations proc_sys_dir_operations = { + .lookup = proc_sys_lookup, + .permission = proc_sys_permission, + .setattr = proc_sys_setattr, + .getattr = proc_sys_getattr, +}; + +static int proc_sys_revalidate(struct inode *dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + if (flags & LOOKUP_RCU) + return -ECHILD; + return !PROC_I(d_inode(dentry))->sysctl->unregistering; +} + +static int proc_sys_delete(const struct dentry *dentry) +{ + return !!PROC_I(d_inode(dentry))->sysctl->unregistering; +} + +static int sysctl_is_seen(struct ctl_table_header *p) +{ + struct ctl_table_set *set = p->set; + int res; + spin_lock(&sysctl_lock); + if (p->unregistering) + res = 0; + else if (!set->is_seen) + res = 1; + else + res = set->is_seen(set); + spin_unlock(&sysctl_lock); + return res; +} + +static int proc_sys_compare(const struct dentry *dentry, + unsigned int len, const char *str, const struct qstr *name) +{ + struct ctl_table_header *head; + struct inode *inode; + + if (name->len != len) + return 1; + if (memcmp(name->name, str, len)) + return 1; + + // false positive is fine here - we'll recheck anyway + if (d_in_lookup(dentry)) + return 0; + + inode = d_inode_rcu(dentry); + // we just might have run into dentry in the middle of __dentry_kill() + if (!inode) + return 1; + + head = READ_ONCE(PROC_I(inode)->sysctl); + return !head || !sysctl_is_seen(head); +} + +static const struct dentry_operations proc_sys_dentry_operations = { + .d_revalidate = proc_sys_revalidate, + .d_delete = proc_sys_delete, + .d_compare = proc_sys_compare, +}; + +static struct ctl_dir *find_subdir(struct ctl_dir *dir, + const char *name, int namelen) +{ + struct ctl_table_header *head; + const struct ctl_table *entry; + + entry = find_entry(&head, dir, name, namelen); + if (!entry) + return ERR_PTR(-ENOENT); + if (!S_ISDIR(entry->mode)) + return ERR_PTR(-ENOTDIR); + return container_of(head, struct ctl_dir, header); +} + +static struct ctl_dir *new_dir(struct ctl_table_set *set, + const char *name, int namelen) +{ + struct ctl_table *table; + struct ctl_dir *new; + struct ctl_node *node; + char *new_name; + + new = kzalloc(sizeof(*new) + sizeof(struct ctl_node) + + sizeof(struct ctl_table) + namelen + 1, + GFP_KERNEL); + if (!new) + return NULL; + + node = (struct ctl_node *)(new + 1); + table = (struct ctl_table *)(node + 1); + new_name = (char *)(table + 1); + memcpy(new_name, name, namelen); + table[0].procname = new_name; + table[0].mode = S_IFDIR|S_IRUGO|S_IXUGO; + init_header(&new->header, set->dir.header.root, set, node, table, 1); + + return new; +} + +/** + * get_subdir - find or create a subdir with the specified name. + * @dir: Directory to create the subdirectory in + * @name: The name of the subdirectory to find or create + * @namelen: The length of name + * + * Takes a directory with an elevated reference count so we know that + * if we drop the lock the directory will not go away. Upon success + * the reference is moved from @dir to the returned subdirectory. + * Upon error an error code is returned and the reference on @dir is + * simply dropped. + */ +static struct ctl_dir *get_subdir(struct ctl_dir *dir, + const char *name, int namelen) +{ + struct ctl_table_set *set = dir->header.set; + struct ctl_dir *subdir, *new = NULL; + int err; + + spin_lock(&sysctl_lock); + subdir = find_subdir(dir, name, namelen); + if (!IS_ERR(subdir)) + goto found; + if (PTR_ERR(subdir) != -ENOENT) + goto failed; + + spin_unlock(&sysctl_lock); + new = new_dir(set, name, namelen); + spin_lock(&sysctl_lock); + subdir = ERR_PTR(-ENOMEM); + if (!new) + goto failed; + + /* Was the subdir added while we dropped the lock? */ + subdir = find_subdir(dir, name, namelen); + if (!IS_ERR(subdir)) + goto found; + if (PTR_ERR(subdir) != -ENOENT) + goto failed; + + /* Nope. Use the our freshly made directory entry. */ + err = insert_header(dir, &new->header); + subdir = ERR_PTR(err); + if (err) + goto failed; + subdir = new; +found: + subdir->header.nreg++; +failed: + if (IS_ERR(subdir)) { + pr_err("sysctl could not get directory: "); + sysctl_print_dir(dir); + pr_cont("%*.*s %ld\n", namelen, namelen, name, + PTR_ERR(subdir)); + } + drop_sysctl_table(&dir->header); + if (new) + drop_sysctl_table(&new->header); + spin_unlock(&sysctl_lock); + return subdir; +} + +static struct ctl_dir *xlate_dir(struct ctl_table_set *set, struct ctl_dir *dir) +{ + struct ctl_dir *parent; + const char *procname; + if (!dir->header.parent) + return &set->dir; + parent = xlate_dir(set, dir->header.parent); + if (IS_ERR(parent)) + return parent; + procname = dir->header.ctl_table[0].procname; + return find_subdir(parent, procname, strlen(procname)); +} + +static int sysctl_follow_link(struct ctl_table_header **phead, + const struct ctl_table **pentry) +{ + struct ctl_table_header *head; + const struct ctl_table *entry; + struct ctl_table_root *root; + struct ctl_table_set *set; + struct ctl_dir *dir; + int ret; + + spin_lock(&sysctl_lock); + root = (*pentry)->data; + set = lookup_header_set(root); + dir = xlate_dir(set, (*phead)->parent); + if (IS_ERR(dir)) + ret = PTR_ERR(dir); + else { + const char *procname = (*pentry)->procname; + head = NULL; + entry = find_entry(&head, dir, procname, strlen(procname)); + ret = -ENOENT; + if (entry && use_table(head)) { + unuse_table(*phead); + *phead = head; + *pentry = entry; + ret = 0; + } + } + + spin_unlock(&sysctl_lock); + return ret; +} + +static int sysctl_err(const char *path, const struct ctl_table *table, char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + + pr_err("sysctl table check failed: %s/%s %pV\n", + path, table->procname, &vaf); + + va_end(args); + return -EINVAL; +} + +static int sysctl_check_table_array(const char *path, const struct ctl_table *table) +{ + unsigned int extra; + int err = 0; + + if ((table->proc_handler == proc_douintvec) || + (table->proc_handler == proc_douintvec_minmax)) { + if (table->maxlen != sizeof(unsigned int)) + err |= sysctl_err(path, table, "array not allowed"); + } + + if (table->proc_handler == proc_dou8vec_minmax) { + if (table->maxlen != sizeof(u8)) + err |= sysctl_err(path, table, "array not allowed"); + + if (table->extra1) { + extra = *(unsigned int *) table->extra1; + if (extra > 255U) + err |= sysctl_err(path, table, + "range value too large for proc_dou8vec_minmax"); + } + if (table->extra2) { + extra = *(unsigned int *) table->extra2; + if (extra > 255U) + err |= sysctl_err(path, table, + "range value too large for proc_dou8vec_minmax"); + } + } + + if (table->proc_handler == proc_dobool) { + if (table->maxlen != sizeof(bool)) + err |= sysctl_err(path, table, "array not allowed"); + } + + return err; +} + +static int sysctl_check_table(const char *path, struct ctl_table_header *header) +{ + const struct ctl_table *entry; + int err = 0; + list_for_each_table_entry(entry, header) { + if (!entry->procname) + err |= sysctl_err(path, entry, "procname is null"); + if ((entry->proc_handler == proc_dostring) || + (entry->proc_handler == proc_dobool) || + (entry->proc_handler == proc_dointvec) || + (entry->proc_handler == proc_douintvec) || + (entry->proc_handler == proc_douintvec_minmax) || + (entry->proc_handler == proc_dointvec_minmax) || + (entry->proc_handler == proc_dou8vec_minmax) || + (entry->proc_handler == proc_dointvec_jiffies) || + (entry->proc_handler == proc_dointvec_userhz_jiffies) || + (entry->proc_handler == proc_dointvec_ms_jiffies) || + (entry->proc_handler == proc_doulongvec_minmax) || + (entry->proc_handler == proc_doulongvec_ms_jiffies_minmax)) { + if (!entry->data) + err |= sysctl_err(path, entry, "No data"); + if (!entry->maxlen) + err |= sysctl_err(path, entry, "No maxlen"); + else + err |= sysctl_check_table_array(path, entry); + } + if (!entry->proc_handler) + err |= sysctl_err(path, entry, "No proc_handler"); + + if ((entry->mode & (S_IRUGO|S_IWUGO)) != entry->mode) + err |= sysctl_err(path, entry, "bogus .mode 0%o", + entry->mode); + } + return err; +} + +static struct ctl_table_header *new_links(struct ctl_dir *dir, struct ctl_table_header *head) +{ + struct ctl_table *link_table, *link; + struct ctl_table_header *links; + const struct ctl_table *entry; + struct ctl_node *node; + char *link_name; + int name_bytes; + + name_bytes = 0; + list_for_each_table_entry(entry, head) { + name_bytes += strlen(entry->procname) + 1; + } + + links = kzalloc(sizeof(struct ctl_table_header) + + sizeof(struct ctl_node)*head->ctl_table_size + + sizeof(struct ctl_table)*head->ctl_table_size + + name_bytes, + GFP_KERNEL); + + if (!links) + return NULL; + + node = (struct ctl_node *)(links + 1); + link_table = (struct ctl_table *)(node + head->ctl_table_size); + link_name = (char *)(link_table + head->ctl_table_size); + link = link_table; + + list_for_each_table_entry(entry, head) { + int len = strlen(entry->procname) + 1; + memcpy(link_name, entry->procname, len); + link->procname = link_name; + link->mode = S_IFLNK|S_IRWXUGO; + link->data = head->root; + link_name += len; + link++; + } + init_header(links, dir->header.root, dir->header.set, node, link_table, + head->ctl_table_size); + links->nreg = head->ctl_table_size; + + return links; +} + +static bool get_links(struct ctl_dir *dir, + struct ctl_table_header *header, + struct ctl_table_root *link_root) +{ + struct ctl_table_header *tmp_head; + const struct ctl_table *entry, *link; + + if (header->ctl_table_size == 0 || + sysctl_is_perm_empty_ctl_header(header)) + return true; + + /* Are there links available for every entry in table? */ + list_for_each_table_entry(entry, header) { + const char *procname = entry->procname; + link = find_entry(&tmp_head, dir, procname, strlen(procname)); + if (!link) + return false; + if (S_ISDIR(link->mode) && S_ISDIR(entry->mode)) + continue; + if (S_ISLNK(link->mode) && (link->data == link_root)) + continue; + return false; + } + + /* The checks passed. Increase the registration count on the links */ + list_for_each_table_entry(entry, header) { + const char *procname = entry->procname; + link = find_entry(&tmp_head, dir, procname, strlen(procname)); + tmp_head->nreg++; + } + return true; +} + +static int insert_links(struct ctl_table_header *head) +{ + struct ctl_table_set *root_set = &sysctl_table_root.default_set; + struct ctl_dir *core_parent; + struct ctl_table_header *links; + int err; + + if (head->set == root_set) + return 0; + + core_parent = xlate_dir(root_set, head->parent); + if (IS_ERR(core_parent)) + return 0; + + if (get_links(core_parent, head, head->root)) + return 0; + + core_parent->header.nreg++; + spin_unlock(&sysctl_lock); + + links = new_links(core_parent, head); + + spin_lock(&sysctl_lock); + err = -ENOMEM; + if (!links) + goto out; + + err = 0; + if (get_links(core_parent, head, head->root)) { + kfree(links); + goto out; + } + + err = insert_header(core_parent, links); + if (err) + kfree(links); +out: + drop_sysctl_table(&core_parent->header); + return err; +} + +/* Find the directory for the ctl_table. If one is not found create it. */ +static struct ctl_dir *sysctl_mkdir_p(struct ctl_dir *dir, const char *path) +{ + const char *name, *nextname; + + for (name = path; name; name = nextname) { + int namelen; + nextname = strchr(name, '/'); + if (nextname) { + namelen = nextname - name; + nextname++; + } else { + namelen = strlen(name); + } + if (namelen == 0) + continue; + + /* + * namelen ensures if name is "foo/bar/yay" only foo is + * registered first. We traverse as if using mkdir -p and + * return a ctl_dir for the last directory entry. + */ + dir = get_subdir(dir, name, namelen); + if (IS_ERR(dir)) + break; + } + return dir; +} + +/** + * __register_sysctl_table - register a leaf sysctl table + * @set: Sysctl tree to register on + * @path: The path to the directory the sysctl table is in. + * + * @table: the top-level table structure. This table should not be free'd + * after registration. So it should not be used on stack. It can either + * be a global or dynamically allocated by the caller and free'd later + * after sysctl unregistration. + * @table_size : The number of elements in table + * + * Register a sysctl table hierarchy. @table should be a filled in ctl_table + * array. + * + * The members of the &struct ctl_table structure are used as follows: + * procname - the name of the sysctl file under /proc/sys. Set to %NULL to not + * enter a sysctl file + * data - a pointer to data for use by proc_handler + * maxlen - the maximum size in bytes of the data + * mode - the file permissions for the /proc/sys file + * type - Defines the target type (described in struct definition) + * proc_handler - the text handler routine (described below) + * + * extra1, extra2 - extra pointers usable by the proc handler routines + * XXX: we should eventually modify these to use long min / max [0] + * [0] https://lkml.kernel.org/87zgpte9o4.fsf@email.froward.int.ebiederm.org + * + * Leaf nodes in the sysctl tree will be represented by a single file + * under /proc; non-leaf nodes are not allowed. + * + * There must be a proc_handler routine for any terminal nodes. + * Several default handlers are available to cover common cases - + * + * proc_dostring(), proc_dointvec(), proc_dointvec_jiffies(), + * proc_dointvec_userhz_jiffies(), proc_dointvec_minmax(), + * proc_doulongvec_ms_jiffies_minmax(), proc_doulongvec_minmax() + * + * It is the handler's job to read the input buffer from user memory + * and process it. The handler should return 0 on success. + * + * This routine returns %NULL on a failure to register, and a pointer + * to the table header on success. + */ +struct ctl_table_header *__register_sysctl_table( + struct ctl_table_set *set, + const char *path, const struct ctl_table *table, size_t table_size) +{ + struct ctl_table_root *root = set->dir.header.root; + struct ctl_table_header *header; + struct ctl_dir *dir; + struct ctl_node *node; + + header = kzalloc(sizeof(struct ctl_table_header) + + sizeof(struct ctl_node)*table_size, GFP_KERNEL_ACCOUNT); + if (!header) + return NULL; + + node = (struct ctl_node *)(header + 1); + init_header(header, root, set, node, table, table_size); + if (sysctl_check_table(path, header)) + goto fail; + + spin_lock(&sysctl_lock); + dir = &set->dir; + /* Reference moved down the directory tree get_subdir */ + dir->header.nreg++; + spin_unlock(&sysctl_lock); + + dir = sysctl_mkdir_p(dir, path); + if (IS_ERR(dir)) + goto fail; + spin_lock(&sysctl_lock); + if (insert_header(dir, header)) + goto fail_put_dir_locked; + + drop_sysctl_table(&dir->header); + spin_unlock(&sysctl_lock); + + return header; + +fail_put_dir_locked: + drop_sysctl_table(&dir->header); + spin_unlock(&sysctl_lock); +fail: + kfree(header); + return NULL; +} + +/** + * register_sysctl_sz - register a sysctl table + * @path: The path to the directory the sysctl table is in. If the path + * doesn't exist we will create it for you. + * @table: the table structure. The calller must ensure the life of the @table + * will be kept during the lifetime use of the syctl. It must not be freed + * until unregister_sysctl_table() is called with the given returned table + * with this registration. If your code is non modular then you don't need + * to call unregister_sysctl_table() and can instead use something like + * register_sysctl_init() which does not care for the result of the syctl + * registration. + * @table_size: The number of elements in table. + * + * Register a sysctl table. @table should be a filled in ctl_table + * array. A completely 0 filled entry terminates the table. + * + * See __register_sysctl_table for more details. + */ +struct ctl_table_header *register_sysctl_sz(const char *path, const struct ctl_table *table, + size_t table_size) +{ + return __register_sysctl_table(&sysctl_table_root.default_set, + path, table, table_size); +} +EXPORT_SYMBOL(register_sysctl_sz); + +/** + * __register_sysctl_init() - register sysctl table to path + * @path: path name for sysctl base. If that path doesn't exist we will create + * it for you. + * @table: This is the sysctl table that needs to be registered to the path. + * The caller must ensure the life of the @table will be kept during the + * lifetime use of the sysctl. + * @table_name: The name of sysctl table, only used for log printing when + * registration fails + * @table_size: The number of elements in table + * + * The sysctl interface is used by userspace to query or modify at runtime + * a predefined value set on a variable. These variables however have default + * values pre-set. Code which depends on these variables will always work even + * if register_sysctl() fails. If register_sysctl() fails you'd just loose the + * ability to query or modify the sysctls dynamically at run time. Chances of + * register_sysctl() failing on init are extremely low, and so for both reasons + * this function does not return any error as it is used by initialization code. + * + * Context: if your base directory does not exist it will be created for you. + */ +void __init __register_sysctl_init(const char *path, const struct ctl_table *table, + const char *table_name, size_t table_size) +{ + struct ctl_table_header *hdr = register_sysctl_sz(path, table, table_size); + + if (unlikely(!hdr)) { + pr_err("failed when register_sysctl_sz %s to %s\n", table_name, path); + return; + } + kmemleak_not_leak(hdr); +} + +static void put_links(struct ctl_table_header *header) +{ + struct ctl_table_set *root_set = &sysctl_table_root.default_set; + struct ctl_table_root *root = header->root; + struct ctl_dir *parent = header->parent; + struct ctl_dir *core_parent; + const struct ctl_table *entry; + + if (header->set == root_set) + return; + + core_parent = xlate_dir(root_set, parent); + if (IS_ERR(core_parent)) + return; + + list_for_each_table_entry(entry, header) { + struct ctl_table_header *link_head; + const struct ctl_table *link; + const char *name = entry->procname; + + link = find_entry(&link_head, core_parent, name, strlen(name)); + if (link && + ((S_ISDIR(link->mode) && S_ISDIR(entry->mode)) || + (S_ISLNK(link->mode) && (link->data == root)))) { + drop_sysctl_table(link_head); + } + else { + pr_err("sysctl link missing during unregister: "); + sysctl_print_dir(parent); + pr_cont("%s\n", name); + } + } +} + +static void drop_sysctl_table(struct ctl_table_header *header) +{ + struct ctl_dir *parent = header->parent; + + if (--header->nreg) + return; + + if (parent) { + put_links(header); + start_unregistering(header); + } + + if (!--header->count) + kfree_rcu(header, rcu); + + if (parent) + drop_sysctl_table(&parent->header); +} + +/** + * unregister_sysctl_table - unregister a sysctl table hierarchy + * @header: the header returned from register_sysctl or __register_sysctl_table + * + * Unregisters the sysctl table and all children. proc entries may not + * actually be removed until they are no longer used by anyone. + */ +void unregister_sysctl_table(struct ctl_table_header * header) +{ + might_sleep(); + + if (header == NULL) + return; + + spin_lock(&sysctl_lock); + drop_sysctl_table(header); + spin_unlock(&sysctl_lock); +} +EXPORT_SYMBOL(unregister_sysctl_table); + +void setup_sysctl_set(struct ctl_table_set *set, + struct ctl_table_root *root, + int (*is_seen)(struct ctl_table_set *)) +{ + memset(set, 0, sizeof(*set)); + set->is_seen = is_seen; + init_header(&set->dir.header, root, set, NULL, root_table, 1); +} + +void retire_sysctl_set(struct ctl_table_set *set) +{ + WARN_ON(!RB_EMPTY_ROOT(&set->dir.root)); +} + +int __init proc_sys_init(void) +{ + struct proc_dir_entry *proc_sys_root; + + proc_sys_root = proc_mkdir("sys", NULL); + proc_sys_root->proc_iops = &proc_sys_dir_operations; + proc_sys_root->proc_dir_ops = &proc_sys_dir_file_operations; + proc_sys_root->nlink = 0; + + return sysctl_init_bases(); +} + +struct sysctl_alias { + const char *kernel_param; + const char *sysctl_param; +}; + +/* + * Historically some settings had both sysctl and a command line parameter. + * With the generic sysctl. parameter support, we can handle them at a single + * place and only keep the historical name for compatibility. This is not meant + * to add brand new aliases. When adding existing aliases, consider whether + * the possibly different moment of changing the value (e.g. from early_param + * to the moment do_sysctl_args() is called) is an issue for the specific + * parameter. + */ +static const struct sysctl_alias sysctl_aliases[] = { + {"hardlockup_all_cpu_backtrace", "kernel.hardlockup_all_cpu_backtrace" }, + {"hung_task_panic", "kernel.hung_task_panic" }, + {"numa_zonelist_order", "vm.numa_zonelist_order" }, + {"softlockup_all_cpu_backtrace", "kernel.softlockup_all_cpu_backtrace" }, + { } +}; + +static const char *sysctl_find_alias(char *param) +{ + const struct sysctl_alias *alias; + + for (alias = &sysctl_aliases[0]; alias->kernel_param != NULL; alias++) { + if (strcmp(alias->kernel_param, param) == 0) + return alias->sysctl_param; + } + + return NULL; +} + +bool sysctl_is_alias(char *param) +{ + const char *alias = sysctl_find_alias(param); + + return alias != NULL; +} + +/* Set sysctl value passed on kernel command line. */ +static int process_sysctl_arg(char *param, char *val, + const char *unused, void *arg) +{ + char *path; + struct vfsmount **proc_mnt = arg; + struct file_system_type *proc_fs_type; + struct file *file; + int len; + int err; + loff_t pos = 0; + ssize_t wret; + + if (strncmp(param, "sysctl", sizeof("sysctl") - 1) == 0) { + param += sizeof("sysctl") - 1; + + if (param[0] != '/' && param[0] != '.') + return 0; + + param++; + } else { + param = (char *) sysctl_find_alias(param); + if (!param) + return 0; + } + + if (!val) + return -EINVAL; + len = strlen(val); + if (len == 0) + return -EINVAL; + + /* + * To set sysctl options, we use a temporary mount of proc, look up the + * respective sys/ file and write to it. To avoid mounting it when no + * options were given, we mount it only when the first sysctl option is + * found. Why not a persistent mount? There are problems with a + * persistent mount of proc in that it forces userspace not to use any + * proc mount options. + */ + if (!*proc_mnt) { + proc_fs_type = get_fs_type("proc"); + if (!proc_fs_type) { + pr_err("Failed to find procfs to set sysctl from command line\n"); + return 0; + } + *proc_mnt = kern_mount(proc_fs_type); + put_filesystem(proc_fs_type); + if (IS_ERR(*proc_mnt)) { + pr_err("Failed to mount procfs to set sysctl from command line\n"); + return 0; + } + } + + path = kasprintf(GFP_KERNEL, "sys/%s", param); + if (!path) + panic("%s: Failed to allocate path for %s\n", __func__, param); + strreplace(path, '.', '/'); + + file = file_open_root_mnt(*proc_mnt, path, O_WRONLY, 0); + if (IS_ERR(file)) { + err = PTR_ERR(file); + if (err == -ENOENT) + pr_err("Failed to set sysctl parameter '%s=%s': parameter not found\n", + param, val); + else if (err == -EACCES) + pr_err("Failed to set sysctl parameter '%s=%s': permission denied (read-only?)\n", + param, val); + else + pr_err("Error %pe opening proc file to set sysctl parameter '%s=%s'\n", + file, param, val); + goto out; + } + wret = kernel_write(file, val, len, &pos); + if (wret < 0) { + err = wret; + if (err == -EINVAL) + pr_err("Failed to set sysctl parameter '%s=%s': invalid value\n", + param, val); + else + pr_err("Error %pe writing to proc file to set sysctl parameter '%s=%s'\n", + ERR_PTR(err), param, val); + } else if (wret != len) { + pr_err("Wrote only %zd bytes of %d writing to proc file %s to set sysctl parameter '%s=%s\n", + wret, len, path, param, val); + } + + err = filp_close(file, NULL); + if (err) + pr_err("Error %pe closing proc file to set sysctl parameter '%s=%s\n", + ERR_PTR(err), param, val); +out: + kfree(path); + return 0; +} + +void do_sysctl_args(void) +{ + char *command_line; + struct vfsmount *proc_mnt = NULL; + + command_line = kstrdup(saved_command_line, GFP_KERNEL); + if (!command_line) + panic("%s: Failed to allocate copy of command line\n", __func__); + + parse_args("Setting sysctl args", command_line, + NULL, 0, -1, -1, &proc_mnt, process_sysctl_arg); + + if (proc_mnt) + kern_unmount(proc_mnt); + + kfree(command_line); +} diff --git a/fs/proc/proc_tty.c b/fs/proc/proc_tty.c new file mode 100644 index 000000000..5c6a5ceab --- /dev/null +++ b/fs/proc/proc_tty.c @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * proc_tty.c -- handles /proc/tty + * + * Copyright 1997, Theodore Ts'o + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +/* + * The /proc/tty directory inodes... + */ +static struct proc_dir_entry *proc_tty_driver; + +/* + * This is the handler for /proc/tty/drivers + */ +static void show_tty_range(struct seq_file *m, struct tty_driver *p, + dev_t from, int num) +{ + seq_printf(m, "%-20s ", p->driver_name ? p->driver_name : "unknown"); + seq_printf(m, "/dev/%-8s ", p->name); + if (p->num > 1) { + seq_printf(m, "%3d %d-%d ", MAJOR(from), MINOR(from), + MINOR(from) + num - 1); + } else { + seq_printf(m, "%3d %7d ", MAJOR(from), MINOR(from)); + } + switch (p->type) { + case TTY_DRIVER_TYPE_SYSTEM: + seq_puts(m, "system"); + if (p->subtype == SYSTEM_TYPE_TTY) + seq_puts(m, ":/dev/tty"); + else if (p->subtype == SYSTEM_TYPE_SYSCONS) + seq_puts(m, ":console"); + else if (p->subtype == SYSTEM_TYPE_CONSOLE) + seq_puts(m, ":vtmaster"); + break; + case TTY_DRIVER_TYPE_CONSOLE: + seq_puts(m, "console"); + break; + case TTY_DRIVER_TYPE_SERIAL: + seq_puts(m, "serial"); + break; + case TTY_DRIVER_TYPE_PTY: + if (p->subtype == PTY_TYPE_MASTER) + seq_puts(m, "pty:master"); + else if (p->subtype == PTY_TYPE_SLAVE) + seq_puts(m, "pty:slave"); + else + seq_puts(m, "pty"); + break; + default: + seq_printf(m, "type:%d.%d", p->type, p->subtype); + } + seq_putc(m, '\n'); +} + +static int show_tty_driver(struct seq_file *m, void *v) +{ + struct tty_driver *p = list_entry(v, struct tty_driver, tty_drivers); + dev_t from = MKDEV(p->major, p->minor_start); + dev_t to = from + p->num; + + if (&p->tty_drivers == tty_drivers.next) { + /* pseudo-drivers first */ + seq_printf(m, "%-20s /dev/%-8s ", "/dev/tty", "tty"); + seq_printf(m, "%3d %7d ", TTYAUX_MAJOR, 0); + seq_puts(m, "system:/dev/tty\n"); + seq_printf(m, "%-20s /dev/%-8s ", "/dev/console", "console"); + seq_printf(m, "%3d %7d ", TTYAUX_MAJOR, 1); + seq_puts(m, "system:console\n"); +#ifdef CONFIG_UNIX98_PTYS + seq_printf(m, "%-20s /dev/%-8s ", "/dev/ptmx", "ptmx"); + seq_printf(m, "%3d %7d ", TTYAUX_MAJOR, 2); + seq_puts(m, "system\n"); +#endif +#ifdef CONFIG_VT + seq_printf(m, "%-20s /dev/%-8s ", "/dev/vc/0", "vc/0"); + seq_printf(m, "%3d %7d ", TTY_MAJOR, 0); + seq_puts(m, "system:vtmaster\n"); +#endif + } + + while (MAJOR(from) < MAJOR(to)) { + dev_t next = MKDEV(MAJOR(from)+1, 0); + show_tty_range(m, p, from, next - from); + from = next; + } + if (from != to) + show_tty_range(m, p, from, to - from); + return 0; +} + +/* iterator */ +static void *t_start(struct seq_file *m, loff_t *pos) +{ + mutex_lock(&tty_mutex); + return seq_list_start(&tty_drivers, *pos); +} + +static void *t_next(struct seq_file *m, void *v, loff_t *pos) +{ + return seq_list_next(v, &tty_drivers, pos); +} + +static void t_stop(struct seq_file *m, void *v) +{ + mutex_unlock(&tty_mutex); +} + +static const struct seq_operations tty_drivers_op = { + .start = t_start, + .next = t_next, + .stop = t_stop, + .show = show_tty_driver +}; + +/* + * This function is called by tty_register_driver() to handle + * registering the driver's /proc handler into /proc/tty/driver/ + */ +void proc_tty_register_driver(struct tty_driver *driver) +{ + struct proc_dir_entry *ent; + + if (!driver->driver_name || driver->proc_entry || + !driver->ops->proc_show) + return; + + ent = proc_create_single_data(driver->driver_name, 0, proc_tty_driver, + driver->ops->proc_show, driver); + driver->proc_entry = ent; +} + +/* + * This function is called by tty_unregister_driver() + */ +void proc_tty_unregister_driver(struct tty_driver *driver) +{ + struct proc_dir_entry *ent; + + ent = driver->proc_entry; + if (!ent) + return; + + remove_proc_entry(ent->name, proc_tty_driver); + + driver->proc_entry = NULL; +} + +/* + * Called by proc_root_init() to initialize the /proc/tty subtree + */ +void __init proc_tty_init(void) +{ + if (!proc_mkdir("tty", NULL)) + return; + proc_mkdir("tty/ldisc", NULL); /* Preserved: it's userspace visible */ + /* + * /proc/tty/driver/serial reveals the exact character counts for + * serial links which is just too easy to abuse for inferring + * password lengths and inter-keystroke timings during password + * entry. + */ + proc_tty_driver = proc_mkdir_mode("tty/driver", S_IRUSR|S_IXUSR, NULL); + proc_create_seq("tty/ldiscs", 0, NULL, &tty_ldiscs_seq_ops); + proc_create_seq("tty/drivers", 0, NULL, &tty_drivers_op); +} diff --git a/fs/proc/root.c b/fs/proc/root.c new file mode 100644 index 000000000..99adddfeb --- /dev/null +++ b/fs/proc/root.c @@ -0,0 +1,468 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/proc/root.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * proc root directory handling functions + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +struct proc_fs_context { + struct pid_namespace *pid_ns; + unsigned int mask; + enum proc_hidepid hidepid; + int gid; + enum proc_pidonly pidonly; +}; + +enum proc_param { + Opt_gid, + Opt_hidepid, + Opt_subset, + Opt_pidns, +}; + +static const struct fs_parameter_spec proc_fs_parameters[] = { + fsparam_u32("gid", Opt_gid), + fsparam_string("hidepid", Opt_hidepid), + fsparam_string("subset", Opt_subset), + fsparam_file_or_string("pidns", Opt_pidns), + {} +}; + +static inline int valid_hidepid(unsigned int value) +{ + return (value == HIDEPID_OFF || + value == HIDEPID_NO_ACCESS || + value == HIDEPID_INVISIBLE || + value == HIDEPID_NOT_PTRACEABLE); +} + +static int proc_parse_hidepid_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct proc_fs_context *ctx = fc->fs_private; + struct fs_parameter_spec hidepid_u32_spec = fsparam_u32("hidepid", Opt_hidepid); + struct fs_parse_result result; + int base = (unsigned long)hidepid_u32_spec.data; + + if (param->type != fs_value_is_string) + return invalf(fc, "proc: unexpected type of hidepid value\n"); + + if (!kstrtouint(param->string, base, &result.uint_32)) { + if (!valid_hidepid(result.uint_32)) + return invalf(fc, "proc: unknown value of hidepid - %s\n", param->string); + ctx->hidepid = result.uint_32; + return 0; + } + + if (!strcmp(param->string, "off")) + ctx->hidepid = HIDEPID_OFF; + else if (!strcmp(param->string, "noaccess")) + ctx->hidepid = HIDEPID_NO_ACCESS; + else if (!strcmp(param->string, "invisible")) + ctx->hidepid = HIDEPID_INVISIBLE; + else if (!strcmp(param->string, "ptraceable")) + ctx->hidepid = HIDEPID_NOT_PTRACEABLE; + else + return invalf(fc, "proc: unknown value of hidepid - %s\n", param->string); + + return 0; +} + +static int proc_parse_subset_param(struct fs_context *fc, char *value) +{ + struct proc_fs_context *ctx = fc->fs_private; + + while (value) { + char *ptr = strchr(value, ','); + + if (ptr != NULL) + *ptr++ = '\0'; + + if (*value != '\0') { + if (!strcmp(value, "pid")) { + ctx->pidonly = PROC_PIDONLY_ON; + } else { + return invalf(fc, "proc: unsupported subset option - %s\n", value); + } + } + value = ptr; + } + + return 0; +} + +#ifdef CONFIG_PID_NS +static int proc_parse_pidns_param(struct fs_context *fc, + struct fs_parameter *param, + struct fs_parse_result *result) +{ + struct proc_fs_context *ctx = fc->fs_private; + struct pid_namespace *target, *active = task_active_pid_ns(current); + struct ns_common *ns; + struct file *ns_filp __free(fput) = NULL; + + switch (param->type) { + case fs_value_is_file: + /* came through fsconfig, steal the file reference */ + ns_filp = no_free_ptr(param->file); + break; + case fs_value_is_string: + ns_filp = filp_open(param->string, O_RDONLY, 0); + break; + default: + WARN_ON_ONCE(true); + break; + } + if (!ns_filp) + ns_filp = ERR_PTR(-EBADF); + if (IS_ERR(ns_filp)) { + errorfc(fc, "could not get file from pidns argument"); + return PTR_ERR(ns_filp); + } + + if (!proc_ns_file(ns_filp)) + return invalfc(fc, "pidns argument is not an nsfs file"); + ns = get_proc_ns(file_inode(ns_filp)); + if (ns->ns_type != CLONE_NEWPID) + return invalfc(fc, "pidns argument is not a pidns file"); + target = container_of(ns, struct pid_namespace, ns); + + /* + * pidns= is shorthand for joining the pidns to get a fsopen fd, so the + * permission model should be the same as pidns_install(). + */ + if (!ns_capable(target->user_ns, CAP_SYS_ADMIN)) { + errorfc(fc, "insufficient permissions to set pidns"); + return -EPERM; + } + if (!pidns_is_ancestor(target, active)) + return invalfc(fc, "cannot set pidns to non-descendant pidns"); + + put_pid_ns(ctx->pid_ns); + ctx->pid_ns = get_pid_ns(target); + put_user_ns(fc->user_ns); + fc->user_ns = get_user_ns(ctx->pid_ns->user_ns); + return 0; +} +#endif /* CONFIG_PID_NS */ + +static int proc_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct proc_fs_context *ctx = fc->fs_private; + struct fs_parse_result result; + int opt, err; + + opt = fs_parse(fc, proc_fs_parameters, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_gid: + ctx->gid = result.uint_32; + break; + + case Opt_hidepid: + err = proc_parse_hidepid_param(fc, param); + if (err) + return err; + break; + + case Opt_subset: + err = proc_parse_subset_param(fc, param->string); + if (err) + return err; + break; + + case Opt_pidns: +#ifdef CONFIG_PID_NS + /* + * We would have to RCU-protect every proc_pid_ns() or + * proc_sb_info() access if we allowed this to be reconfigured + * for an existing procfs instance. Luckily, procfs instances + * are cheap to create, and mount-beneath would let you + * atomically replace an instance even with overmounts. + */ + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE) { + errorfc(fc, "cannot reconfigure pidns for existing procfs"); + return -EBUSY; + } + err = proc_parse_pidns_param(fc, param, &result); + if (err) + return err; + break; +#else + errorfc(fc, "pidns mount flag not supported on this system"); + return -EOPNOTSUPP; +#endif + + default: + return -EINVAL; + } + + ctx->mask |= 1 << opt; + return 0; +} + +static int proc_apply_options(struct proc_fs_info *fs_info, + struct fs_context *fc, + struct user_namespace *user_ns) +{ + struct proc_fs_context *ctx = fc->fs_private; + + if ((ctx->mask & (1 << Opt_subset)) && + fc->purpose == FS_CONTEXT_FOR_RECONFIGURE && + ctx->pidonly != fs_info->pidonly) + return invalf(fc, "proc: subset=pid cannot be changed\n"); + + if (ctx->mask & (1 << Opt_gid)) + fs_info->pid_gid = make_kgid(user_ns, ctx->gid); + if (ctx->mask & (1 << Opt_hidepid)) + fs_info->hide_pid = ctx->hidepid; + if (ctx->mask & (1 << Opt_subset)) + fs_info->pidonly = ctx->pidonly; + if (ctx->mask & (1 << Opt_pidns) && + !WARN_ON_ONCE(fc->purpose == FS_CONTEXT_FOR_RECONFIGURE)) { + put_pid_ns(fs_info->pid_ns); + fs_info->pid_ns = get_pid_ns(ctx->pid_ns); + } + return 0; +} + +static int proc_fill_super(struct super_block *s, struct fs_context *fc) +{ + struct proc_fs_context *ctx = fc->fs_private; + struct inode *root_inode; + struct proc_fs_info *fs_info; + int ret; + + fs_info = kzalloc_obj(*fs_info); + if (!fs_info) + return -ENOMEM; + + fs_info->pid_ns = get_pid_ns(ctx->pid_ns); + fs_info->mounter_cred = get_cred(fc->cred); + ret = proc_apply_options(fs_info, fc, current_user_ns()); + if (ret) + return ret; + + /* User space would break if executables or devices appear on proc */ + s->s_iflags |= SB_I_NOEXEC | SB_I_NODEV; + s->s_flags |= SB_NODIRATIME | SB_NOSUID | SB_NOEXEC; + s->s_blocksize = 1024; + s->s_blocksize_bits = 10; + s->s_magic = PROC_SUPER_MAGIC; + s->s_op = &proc_sops; + s->s_time_gran = 1; + s->s_fs_info = fs_info; + + if (fs_info->pidonly == PROC_PIDONLY_ON) + s->s_iflags |= SB_I_RESTRICTED_VARIANT; + + /* + * procfs isn't actually a stacking filesystem; however, there is + * too much magic going on inside it to permit stacking things on + * top of it + */ + s->s_stack_depth = FILESYSTEM_MAX_STACK_DEPTH; + + /* procfs dentries and inodes don't require IO to create */ + s->s_shrink->seeks = 0; + + pde_get(&proc_root); + root_inode = proc_get_inode(s, &proc_root); + if (!root_inode) { + pr_err("proc_fill_super: get root inode failed\n"); + return -ENOMEM; + } + + s->s_root = d_make_root(root_inode); + if (!s->s_root) { + pr_err("proc_fill_super: allocate dentry failed\n"); + return -ENOMEM; + } + + ret = proc_setup_self(s); + if (ret) { + return ret; + } + return proc_setup_thread_self(s); +} + +static int proc_reconfigure(struct fs_context *fc) +{ + struct super_block *sb = fc->root->d_sb; + struct proc_fs_info *fs_info = proc_sb_info(sb); + + sync_filesystem(sb); + + return proc_apply_options(fs_info, fc, current_user_ns()); +} + +static int proc_get_tree(struct fs_context *fc) +{ + return get_tree_nodev(fc, proc_fill_super); +} + +static void proc_fs_context_free(struct fs_context *fc) +{ + struct proc_fs_context *ctx = fc->fs_private; + + put_pid_ns(ctx->pid_ns); + kfree(ctx); +} + +static const struct fs_context_operations proc_fs_context_ops = { + .free = proc_fs_context_free, + .parse_param = proc_parse_param, + .get_tree = proc_get_tree, + .reconfigure = proc_reconfigure, +}; + +static int proc_init_fs_context(struct fs_context *fc) +{ + struct proc_fs_context *ctx; + + ctx = kzalloc_obj(struct proc_fs_context); + if (!ctx) + return -ENOMEM; + + ctx->pid_ns = get_pid_ns(task_active_pid_ns(current)); + put_user_ns(fc->user_ns); + fc->user_ns = get_user_ns(ctx->pid_ns->user_ns); + fc->fs_private = ctx; + fc->ops = &proc_fs_context_ops; + return 0; +} + +static void proc_kill_sb(struct super_block *sb) +{ + struct proc_fs_info *fs_info = proc_sb_info(sb); + + kill_anon_super(sb); + if (fs_info) { + put_pid_ns(fs_info->pid_ns); + put_cred(fs_info->mounter_cred); + kfree_rcu(fs_info, rcu); + } +} + +static struct file_system_type proc_fs_type = { + .name = "proc", + .init_fs_context = proc_init_fs_context, + .parameters = proc_fs_parameters, + .kill_sb = proc_kill_sb, + .fs_flags = FS_USERNS_MOUNT | FS_USERNS_MOUNT_RESTRICTED | FS_DISALLOW_NOTIFY_PERM, +}; + +void __init proc_root_init(void) +{ + proc_init_kmemcache(); + set_proc_pid_nlink(); + proc_self_init(); + proc_thread_self_init(); + proc_symlink("mounts", NULL, "self/mounts"); + + proc_net_init(); + proc_mkdir("fs", NULL); + proc_mkdir("driver", NULL); + proc_create_mount_point("fs/nfsd"); /* somewhere for the nfsd filesystem to be mounted */ +#if defined(CONFIG_SUN_OPENPROMFS) || defined(CONFIG_SUN_OPENPROMFS_MODULE) + /* just give it a mountpoint */ + proc_create_mount_point("openprom"); +#endif + proc_tty_init(); + proc_mkdir("bus", NULL); + proc_sys_init(); + + /* + * Last things last. It is not like userspace processes eager + * to open /proc files exist at this point but register last + * anyway. + */ + register_filesystem(&proc_fs_type); +} + +static int proc_root_getattr(struct mnt_idmap *idmap, + const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int query_flags) +{ + generic_fillattr(&nop_mnt_idmap, request_mask, d_inode(path->dentry), + stat); + stat->nlink = proc_root.nlink + nr_processes(); + return 0; +} + +static struct dentry *proc_root_lookup(struct inode * dir, struct dentry * dentry, unsigned int flags) +{ + if (!proc_pid_lookup(dentry, flags)) + return NULL; + + return proc_lookup(dir, dentry, flags); +} + +static int proc_root_readdir(struct file *file, struct dir_context *ctx) +{ + if (ctx->pos < FIRST_PROCESS_ENTRY) { + int error = proc_readdir(file, ctx); + if (unlikely(error <= 0)) + return error; + ctx->pos = FIRST_PROCESS_ENTRY; + } + + return proc_pid_readdir(file, ctx); +} + +/* + * The root /proc directory is special, as it has the + * directories. Thus we don't use the generic + * directory handling functions for that.. + */ +static const struct file_operations proc_root_operations = { + .read = generic_read_dir, + .iterate_shared = proc_root_readdir, + .llseek = generic_file_llseek, +}; + +/* + * proc root can do almost nothing.. + */ +static const struct inode_operations proc_root_inode_operations = { + .lookup = proc_root_lookup, + .getattr = proc_root_getattr, +}; + +/* + * This is the root "inode" in the /proc tree.. + */ +struct proc_dir_entry proc_root = { + .low_ino = PROCFS_ROOT_INO, + .namelen = 5, + .mode = S_IFDIR | S_IRUGO | S_IXUGO, + .nlink = 2, + .refcnt = REFCOUNT_INIT(1), + .proc_iops = &proc_root_inode_operations, + .proc_dir_ops = &proc_root_operations, + .parent = &proc_root, + .subdir = RB_ROOT, + .name = "/proc", +}; diff --git a/fs/proc/self.c b/fs/proc/self.c new file mode 100644 index 000000000..56adf1c68 --- /dev/null +++ b/fs/proc/self.c @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include "internal.h" + +/* + * /proc/self: + */ +static const char *proc_self_get_link(struct dentry *dentry, + struct inode *inode, + struct delayed_call *done) +{ + struct pid_namespace *ns = proc_pid_ns(inode->i_sb); + pid_t tgid = task_tgid_nr_ns(current, ns); + char *name; + + if (!tgid) + return ERR_PTR(-ENOENT); + /* max length of unsigned int in decimal + NULL term */ + name = kmalloc(10 + 1, dentry ? GFP_KERNEL : GFP_ATOMIC); + if (unlikely(!name)) + return dentry ? ERR_PTR(-ENOMEM) : ERR_PTR(-ECHILD); + sprintf(name, "%u", tgid); + set_delayed_call(done, kfree_link, name); + return name; +} + +static const struct inode_operations proc_self_inode_operations = { + .get_link = proc_self_get_link, +}; + +unsigned self_inum __ro_after_init; + +int proc_setup_self(struct super_block *s) +{ + struct dentry *self; + int ret = -ENOMEM; + + self = d_alloc_name(s->s_root, "self"); + if (self) { + struct inode *inode = new_inode(s); + if (inode) { + inode->i_ino = self_inum; + simple_inode_init_ts(inode); + inode->i_mode = S_IFLNK | S_IRWXUGO; + inode->i_uid = GLOBAL_ROOT_UID; + inode->i_gid = GLOBAL_ROOT_GID; + inode->i_op = &proc_self_inode_operations; + d_make_persistent(self, inode); + ret = 0; + } + dput(self); + } + + if (ret) + pr_err("proc_fill_super: can't allocate /proc/self\n"); + + return ret; +} + +void __init proc_self_init(void) +{ + proc_alloc_inum(&self_inum); +} diff --git a/fs/proc/softirqs.c b/fs/proc/softirqs.c new file mode 100644 index 000000000..04bb29721 --- /dev/null +++ b/fs/proc/softirqs.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include "internal.h" + +/* + * /proc/softirqs ... display the number of softirqs + */ +static int show_softirqs(struct seq_file *p, void *v) +{ + int i, j; + + seq_puts(p, " "); + for_each_possible_cpu(i) + seq_printf(p, "CPU%-8d", i); + seq_putc(p, '\n'); + + for (i = 0; i < NR_SOFTIRQS; i++) { + seq_printf(p, "%12s:", softirq_to_name[i]); + for_each_possible_cpu(j) + seq_put_decimal_ull_width(p, " ", kstat_softirqs_cpu(i, j), 10); + seq_putc(p, '\n'); + } + return 0; +} + +static int __init proc_softirqs_init(void) +{ + struct proc_dir_entry *pde; + + pde = proc_create_single("softirqs", 0, NULL, show_softirqs); + pde_make_permanent(pde); + return 0; +} +fs_initcall(proc_softirqs_init); diff --git a/fs/proc/stat.c b/fs/proc/stat.c new file mode 100644 index 000000000..79b42f2bf --- /dev/null +++ b/fs/proc/stat.c @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef arch_irq_stat_cpu +#define arch_irq_stat_cpu(cpu) 0 +#endif + +static void show_irq_gap(struct seq_file *p, unsigned int gap) +{ + static const char zeros[] = " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0"; + + while (gap > 0) { + unsigned int inc; + + inc = min_t(unsigned int, gap, ARRAY_SIZE(zeros) / 2); + seq_write(p, zeros, 2 * inc); + gap -= inc; + } +} + +static void show_all_irqs(struct seq_file *p) +{ + unsigned int i, next = 0; + + for_each_active_irq(i) { + show_irq_gap(p, i - next); + seq_put_decimal_ull(p, " ", kstat_irqs_usr(i)); + next = i + 1; + } + show_irq_gap(p, irq_get_nr_irqs() - next); +} + +static int show_stat(struct seq_file *p, void *v) +{ + int i, j; + u64 user, nice, system, idle, iowait, irq, softirq, steal; + u64 guest, guest_nice; + u64 sum = 0; + u64 sum_softirq = 0; + unsigned int per_softirq_sums[NR_SOFTIRQS] = {0}; + struct timespec64 boottime; + + user = nice = system = idle = iowait = + irq = softirq = steal = 0; + guest = guest_nice = 0; + getboottime64(&boottime); + /* shift boot timestamp according to the timens offset */ + timens_sub_boottime(&boottime); + + for_each_possible_cpu(i) { + struct kernel_cpustat kcpustat; + u64 *cpustat = kcpustat.cpustat; + + kcpustat_cpu_fetch(&kcpustat, i); + + user += cpustat[CPUTIME_USER]; + nice += cpustat[CPUTIME_NICE]; + system += cpustat[CPUTIME_SYSTEM]; + idle += cpustat[CPUTIME_IDLE]; + iowait += cpustat[CPUTIME_IOWAIT]; + irq += cpustat[CPUTIME_IRQ]; + softirq += cpustat[CPUTIME_SOFTIRQ]; + steal += cpustat[CPUTIME_STEAL]; + guest += cpustat[CPUTIME_GUEST]; + guest_nice += cpustat[CPUTIME_GUEST_NICE]; + sum += kstat_cpu_irqs_sum(i); + sum += arch_irq_stat_cpu(i); + + for (j = 0; j < NR_SOFTIRQS; j++) { + unsigned int softirq_stat = kstat_softirqs_cpu(j, i); + + per_softirq_sums[j] += softirq_stat; + sum_softirq += softirq_stat; + } + } + + seq_put_decimal_ull(p, "cpu ", nsec_to_clock_t(user)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(nice)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(system)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(idle)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(iowait)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(irq)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(softirq)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(steal)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(guest)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(guest_nice)); + seq_putc(p, '\n'); + + for_each_online_cpu(i) { + struct kernel_cpustat kcpustat; + u64 *cpustat = kcpustat.cpustat; + + kcpustat_cpu_fetch(&kcpustat, i); + + /* Copy values here to work around gcc-2.95.3, gcc-2.96 */ + user = cpustat[CPUTIME_USER]; + nice = cpustat[CPUTIME_NICE]; + system = cpustat[CPUTIME_SYSTEM]; + idle = cpustat[CPUTIME_IDLE]; + iowait = cpustat[CPUTIME_IOWAIT]; + irq = cpustat[CPUTIME_IRQ]; + softirq = cpustat[CPUTIME_SOFTIRQ]; + steal = cpustat[CPUTIME_STEAL]; + guest = cpustat[CPUTIME_GUEST]; + guest_nice = cpustat[CPUTIME_GUEST_NICE]; + seq_printf(p, "cpu%d", i); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(user)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(nice)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(system)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(idle)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(iowait)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(irq)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(softirq)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(steal)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(guest)); + seq_put_decimal_ull(p, " ", nsec_to_clock_t(guest_nice)); + seq_putc(p, '\n'); + } + seq_put_decimal_ull(p, "intr ", (unsigned long long)sum); + + show_all_irqs(p); + + seq_printf(p, + "\nctxt %llu\n" + "btime %llu\n" + "processes %lu\n" + "procs_running %u\n" + "procs_blocked %u\n", + nr_context_switches(), + (unsigned long long)boottime.tv_sec, + total_forks, + nr_running(), + nr_iowait()); + + seq_put_decimal_ull(p, "softirq ", (unsigned long long)sum_softirq); + + for (i = 0; i < NR_SOFTIRQS; i++) + seq_put_decimal_ull(p, " ", per_softirq_sums[i]); + seq_putc(p, '\n'); + + return 0; +} + +static int stat_open(struct inode *inode, struct file *file) +{ + unsigned int size = 1024 + 128 * num_online_cpus(); + + /* minimum size to display an interrupt count : 2 bytes */ + size += 2 * irq_get_nr_irqs(); + return single_open_size(file, show_stat, NULL, size); +} + +static const struct proc_ops stat_proc_ops = { + .proc_flags = PROC_ENTRY_PERMANENT, + .proc_open = stat_open, + .proc_read_iter = seq_read_iter, + .proc_lseek = seq_lseek, + .proc_release = single_release, +}; + +static int __init proc_stat_init(void) +{ + proc_create("stat", 0, NULL, &stat_proc_ops); + return 0; +} +fs_initcall(proc_stat_init); diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c new file mode 100644 index 000000000..817e3e0f9 --- /dev/null +++ b/fs/proc/task_mmu.c @@ -0,0 +1,3631 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include "internal.h" + +#define SENTINEL_VMA_END -1 +#define SENTINEL_VMA_GATE -2 + +#define SEQ_PUT_DEC(str, val) \ + seq_put_decimal_ull_width(m, str, (val) << (PAGE_SHIFT-10), 8) +void task_mem(struct seq_file *m, struct mm_struct *mm) +{ + unsigned long text, lib, swap, anon, file, shmem; + unsigned long hiwater_vm, total_vm, hiwater_rss, total_rss; + + anon = get_mm_counter_sum(mm, MM_ANONPAGES); + file = get_mm_counter_sum(mm, MM_FILEPAGES); + shmem = get_mm_counter_sum(mm, MM_SHMEMPAGES); + + /* + * Note: to minimize their overhead, mm maintains hiwater_vm and + * hiwater_rss only when about to *lower* total_vm or rss. Any + * collector of these hiwater stats must therefore get total_vm + * and rss too, which will usually be the higher. Barriers? not + * worth the effort, such snapshots can always be inconsistent. + */ + hiwater_vm = total_vm = mm->total_vm; + if (hiwater_vm < mm->hiwater_vm) + hiwater_vm = mm->hiwater_vm; + hiwater_rss = total_rss = anon + file + shmem; + if (hiwater_rss < mm->hiwater_rss) + hiwater_rss = mm->hiwater_rss; + + /* split executable areas between text and lib */ + text = PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK); + text = min(text, mm->exec_vm << PAGE_SHIFT); + lib = (mm->exec_vm << PAGE_SHIFT) - text; + + swap = get_mm_counter_sum(mm, MM_SWAPENTS); + SEQ_PUT_DEC("VmPeak:\t", hiwater_vm); + SEQ_PUT_DEC(" kB\nVmSize:\t", total_vm); + SEQ_PUT_DEC(" kB\nVmLck:\t", mm->locked_vm); + SEQ_PUT_DEC(" kB\nVmPin:\t", atomic64_read(&mm->pinned_vm)); + SEQ_PUT_DEC(" kB\nVmHWM:\t", hiwater_rss); + SEQ_PUT_DEC(" kB\nVmRSS:\t", total_rss); + SEQ_PUT_DEC(" kB\nRssAnon:\t", anon); + SEQ_PUT_DEC(" kB\nRssFile:\t", file); + SEQ_PUT_DEC(" kB\nRssShmem:\t", shmem); + SEQ_PUT_DEC(" kB\nVmData:\t", mm->data_vm); + SEQ_PUT_DEC(" kB\nVmStk:\t", mm->stack_vm); + seq_put_decimal_ull_width(m, + " kB\nVmExe:\t", text >> 10, 8); + seq_put_decimal_ull_width(m, + " kB\nVmLib:\t", lib >> 10, 8); + seq_put_decimal_ull_width(m, + " kB\nVmPTE:\t", mm_pgtables_bytes(mm) >> 10, 8); + SEQ_PUT_DEC(" kB\nVmSwap:\t", swap); + seq_puts(m, " kB\n"); + hugetlb_report_usage(m, mm); +} +#undef SEQ_PUT_DEC + +unsigned long task_vsize(struct mm_struct *mm) +{ + return PAGE_SIZE * mm->total_vm; +} + +unsigned long task_statm(struct mm_struct *mm, + unsigned long *shared, unsigned long *text, + unsigned long *data, unsigned long *resident) +{ + *shared = get_mm_counter_sum(mm, MM_FILEPAGES) + + get_mm_counter_sum(mm, MM_SHMEMPAGES); + *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) + >> PAGE_SHIFT; + *data = mm->data_vm + mm->stack_vm; + *resident = *shared + get_mm_counter_sum(mm, MM_ANONPAGES); + return mm->total_vm; +} + +#ifdef CONFIG_NUMA +/* + * Save get_task_policy() for show_numa_map(). + */ +static void hold_task_mempolicy(struct proc_maps_private *priv) +{ + struct task_struct *task = priv->task; + + task_lock(task); + priv->task_mempolicy = get_task_policy(task); + mpol_get(priv->task_mempolicy); + task_unlock(task); +} +static void release_task_mempolicy(struct proc_maps_private *priv) +{ + mpol_put(priv->task_mempolicy); +} +#else +static void hold_task_mempolicy(struct proc_maps_private *priv) +{ +} +static void release_task_mempolicy(struct proc_maps_private *priv) +{ +} +#endif + +#ifdef CONFIG_PER_VMA_LOCK + +static inline int lock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) +{ + int ret = mmap_read_lock_killable(lock_ctx->mm); + + if (!ret) + lock_ctx->mmap_locked = true; + + return ret; +} + +static inline void unlock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) +{ + mmap_read_unlock(lock_ctx->mm); + lock_ctx->mmap_locked = false; +} + +static void reset_lock_ctx(struct proc_maps_locking_ctx *lock_ctx) +{ + lock_ctx->locked_vma = NULL; + lock_ctx->mmap_locked = false; +} + +static void unlock_ctx_vma(struct proc_maps_locking_ctx *lock_ctx) +{ + if (lock_ctx->locked_vma) { + vma_end_read(lock_ctx->locked_vma); + lock_ctx->locked_vma = NULL; + } +} + +static inline bool lock_vma_range(struct seq_file *m, + struct proc_maps_locking_ctx *lock_ctx) +{ + rcu_read_lock(); + reset_lock_ctx(lock_ctx); + + return true; +} + +static inline void unlock_vma_range(struct proc_maps_locking_ctx *lock_ctx) +{ + if (lock_ctx->mmap_locked) { + unlock_ctx_mm(lock_ctx); + } else { + unlock_ctx_vma(lock_ctx); + rcu_read_unlock(); + } +} + +static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv, + loff_t last_pos) +{ + struct proc_maps_locking_ctx *lock_ctx = &priv->lock_ctx; + struct vm_area_struct *vma; + + if (lock_ctx->mmap_locked) + return vma_next(&priv->iter); + + unlock_ctx_vma(lock_ctx); + vma = lock_next_vma(lock_ctx->mm, &priv->iter, last_pos); + if (!IS_ERR_OR_NULL(vma)) + lock_ctx->locked_vma = vma; + + return vma; +} + +static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, + loff_t pos) +{ + struct proc_maps_locking_ctx *lock_ctx = &priv->lock_ctx; + + if (lock_ctx->mmap_locked) + return false; + + rcu_read_unlock(); + mmap_read_lock(lock_ctx->mm); + /* Reinitialize the iterator after taking mmap_lock */ + vma_iter_set(&priv->iter, pos); + lock_ctx->mmap_locked = true; + + return true; +} + +static inline void drop_rcu(struct proc_maps_private *priv) +{ + if (priv->lock_ctx.mmap_locked) + return; + + rcu_read_unlock(); +} + +static inline void reacquire_rcu(struct proc_maps_private *priv) +{ + if (priv->lock_ctx.mmap_locked) + return; + + rcu_read_lock(); + /* Reinitialize the iterator. */ + vma_iter_set(&priv->iter, priv->lock_ctx.locked_vma->vm_end); +} + +#else /* CONFIG_PER_VMA_LOCK */ + +static inline int lock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) +{ + return mmap_read_lock_killable(lock_ctx->mm); +} + +static inline void unlock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) +{ + mmap_read_unlock(lock_ctx->mm); +} + +static inline bool lock_vma_range(struct seq_file *m, + struct proc_maps_locking_ctx *lock_ctx) +{ + return lock_ctx_mm(lock_ctx) == 0; +} + +static inline void unlock_vma_range(struct proc_maps_locking_ctx *lock_ctx) +{ + unlock_ctx_mm(lock_ctx); +} + +static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv, + loff_t last_pos) +{ + return vma_next(&priv->iter); +} + +static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, + loff_t pos) +{ + return false; +} + +static inline void drop_rcu(struct proc_maps_private *priv) {} +static inline void reacquire_rcu(struct proc_maps_private *priv) {} + +#endif /* CONFIG_PER_VMA_LOCK */ + +static struct vm_area_struct *proc_get_vma(struct seq_file *m, loff_t *ppos) +{ + struct proc_maps_private *priv = m->private; + struct vm_area_struct *vma; + +retry: + vma = get_next_vma(priv, *ppos); + /* EINTR of EAGAIN is possible */ + if (IS_ERR(vma)) { + if (PTR_ERR(vma) == -EAGAIN && fallback_to_mmap_lock(priv, *ppos)) + goto retry; + + return vma; + } + + /* Store previous position to be able to restart if needed */ + priv->last_pos = *ppos; + if (vma) { + /* + * Track the end of the reported vma to ensure position changes + * even if previous vma was merged with the next vma and we + * found the extended vma with the same vm_start. + */ + *ppos = vma->vm_end; + } else { + *ppos = SENTINEL_VMA_GATE; + vma = get_gate_vma(priv->lock_ctx.mm); + } + + return vma; +} + +static void *m_start(struct seq_file *m, loff_t *ppos) +{ + struct proc_maps_private *priv = m->private; + struct proc_maps_locking_ctx *lock_ctx; + loff_t last_addr = *ppos; + struct mm_struct *mm; + + /* See m_next(). Zero at the start or after lseek. */ + if (last_addr == SENTINEL_VMA_END) + return NULL; + + priv->task = get_proc_task(priv->inode); + if (!priv->task) + return ERR_PTR(-ESRCH); + + lock_ctx = &priv->lock_ctx; + mm = lock_ctx->mm; + if (!mm || !mmget_not_zero(mm)) { + put_task_struct(priv->task); + priv->task = NULL; + return NULL; + } + + if (!lock_vma_range(m, lock_ctx)) { + mmput(mm); + put_task_struct(priv->task); + priv->task = NULL; + return ERR_PTR(-EINTR); + } + + /* + * Reset current position if last_addr was set before + * and it's not a sentinel. + */ + if (last_addr > 0) + *ppos = last_addr = priv->last_pos; + vma_iter_init(&priv->iter, mm, (unsigned long)last_addr); + hold_task_mempolicy(priv); + if (last_addr == SENTINEL_VMA_GATE) + return get_gate_vma(mm); + + return proc_get_vma(m, ppos); +} + +static void *m_next(struct seq_file *m, void *v, loff_t *ppos) +{ + if (*ppos == SENTINEL_VMA_GATE) { + *ppos = SENTINEL_VMA_END; + return NULL; + } + return proc_get_vma(m, ppos); +} + +static void m_stop(struct seq_file *m, void *v) +{ + struct proc_maps_private *priv = m->private; + struct mm_struct *mm = priv->lock_ctx.mm; + + if (!priv->task) + return; + + release_task_mempolicy(priv); + unlock_vma_range(&priv->lock_ctx); + mmput(mm); + put_task_struct(priv->task); + priv->task = NULL; +} + +static int proc_maps_open(struct inode *inode, struct file *file, + const struct seq_operations *ops, int psize) +{ + struct proc_maps_private *priv = __seq_open_private(file, ops, psize); + + if (!priv) + return -ENOMEM; + + priv->inode = inode; + priv->lock_ctx.mm = proc_mem_open(inode, PTRACE_MODE_READ); + if (IS_ERR(priv->lock_ctx.mm)) { + int err = PTR_ERR(priv->lock_ctx.mm); + + seq_release_private(inode, file); + return err; + } + + return 0; +} + +static int proc_map_release(struct inode *inode, struct file *file) +{ + struct seq_file *seq = file->private_data; + struct proc_maps_private *priv = seq->private; + + if (priv->lock_ctx.mm) + mmdrop(priv->lock_ctx.mm); + + return seq_release_private(inode, file); +} + +static int do_maps_open(struct inode *inode, struct file *file, + const struct seq_operations *ops) +{ + return proc_maps_open(inode, file, ops, + sizeof(struct proc_maps_private)); +} + +static void get_vma_name(struct vm_area_struct *vma, + const struct path **path, + const char **name, + const char **name_fmt) +{ + struct anon_vma_name *anon_name = vma->vm_mm ? anon_vma_name(vma) : NULL; + + *name = NULL; + *path = NULL; + *name_fmt = NULL; + + /* + * Print the dentry name for named mappings, and a + * special [heap] marker for the heap: + */ + if (vma->vm_file) { + /* + * If user named this anon shared memory via + * prctl(PR_SET_VMA ..., use the provided name. + */ + if (anon_name) { + *name_fmt = "[anon_shmem:%s]"; + *name = anon_name->name; + } else { + *path = file_user_path(vma->vm_file); + } + return; + } + + if (vma->vm_ops && vma->vm_ops->name) { + *name = vma->vm_ops->name(vma); + if (*name) + return; + } + + *name = arch_vma_name(vma); + if (*name) + return; + + if (!vma->vm_mm) { + *name = "[vdso]"; + return; + } + + if (vma_is_initial_heap(vma)) { + *name = "[heap]"; + return; + } + + if (vma_is_initial_stack(vma)) { + *name = "[stack]"; + return; + } + + if (anon_name) { + *name_fmt = "[anon:%s]"; + *name = anon_name->name; + return; + } +} + +static void show_vma_header_prefix(struct seq_file *m, + unsigned long start, unsigned long end, + vm_flags_t flags, unsigned long long pgoff, + dev_t dev, u64 ino) +{ + seq_setwidth(m, 25 + sizeof(void *) * 6 - 1); + seq_put_hex_ll(m, NULL, start, 8); + seq_put_hex_ll(m, "-", end, 8); + seq_putc(m, ' '); + seq_putc(m, flags & VM_READ ? 'r' : '-'); + seq_putc(m, flags & VM_WRITE ? 'w' : '-'); + seq_putc(m, flags & VM_EXEC ? 'x' : '-'); + seq_putc(m, flags & VM_MAYSHARE ? 's' : 'p'); + seq_put_hex_ll(m, " ", pgoff, 8); + seq_put_hex_ll(m, " ", MAJOR(dev), 2); + seq_put_hex_ll(m, ":", MINOR(dev), 2); + seq_put_decimal_ull(m, " ", ino); + seq_putc(m, ' '); +} + +static void +show_map_vma(struct seq_file *m, struct vm_area_struct *vma) +{ + const struct path *path; + const char *name_fmt, *name; + vm_flags_t flags = vma->vm_flags; + u64 ino = 0; + unsigned long long pgoff = 0; + unsigned long start, end; + dev_t dev = 0; + + if (vma->vm_file) { + const struct inode *inode = file_user_inode(vma->vm_file); + + dev = inode->i_sb->s_dev; + ino = inode->i_ino; + pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT; + } + + start = vma->vm_start; + end = vma->vm_end; + show_vma_header_prefix(m, start, end, flags, pgoff, dev, ino); + + get_vma_name(vma, &path, &name, &name_fmt); + if (path) { + seq_pad(m, ' '); + seq_path(m, path, "\n"); + } else if (name_fmt) { + seq_pad(m, ' '); + seq_printf(m, name_fmt, name); + } else if (name) { + seq_pad(m, ' '); + seq_puts(m, name); + } + seq_putc(m, '\n'); +} + +static int show_map(struct seq_file *m, void *v) +{ + show_map_vma(m, v); + return 0; +} + +static const struct seq_operations proc_pid_maps_op = { + .start = m_start, + .next = m_next, + .stop = m_stop, + .show = show_map +}; + +static int pid_maps_open(struct inode *inode, struct file *file) +{ + return do_maps_open(inode, file, &proc_pid_maps_op); +} + +#define PROCMAP_QUERY_VMA_FLAGS ( \ + PROCMAP_QUERY_VMA_READABLE | \ + PROCMAP_QUERY_VMA_WRITABLE | \ + PROCMAP_QUERY_VMA_EXECUTABLE | \ + PROCMAP_QUERY_VMA_SHARED \ +) + +#define PROCMAP_QUERY_VALID_FLAGS_MASK ( \ + PROCMAP_QUERY_COVERING_OR_NEXT_VMA | \ + PROCMAP_QUERY_FILE_BACKED_VMA | \ + PROCMAP_QUERY_VMA_FLAGS \ +) + +#ifdef CONFIG_PER_VMA_LOCK + +static int query_vma_setup(struct proc_maps_locking_ctx *lock_ctx) +{ + reset_lock_ctx(lock_ctx); + + return 0; +} + +static void query_vma_teardown(struct proc_maps_locking_ctx *lock_ctx) +{ + if (lock_ctx->mmap_locked) + unlock_ctx_mm(lock_ctx); + else + unlock_ctx_vma(lock_ctx); +} + +static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ctx *lock_ctx, + unsigned long addr) +{ + struct mm_struct *mm = lock_ctx->mm; + struct vm_area_struct *vma; + struct vma_iterator vmi; + + if (lock_ctx->mmap_locked) + return find_vma(mm, addr); + + /* Unlock previously locked VMA and find the next one under RCU */ + unlock_ctx_vma(lock_ctx); + rcu_read_lock(); + vma_iter_init(&vmi, mm, addr); + vma = lock_next_vma(mm, &vmi, addr); + rcu_read_unlock(); + + if (!vma) + return NULL; + + if (!IS_ERR(vma)) { + lock_ctx->locked_vma = vma; + return vma; + } + + if (PTR_ERR(vma) == -EAGAIN) { + /* Fallback to mmap_lock on vma->vm_refcnt overflow */ + mmap_read_lock(mm); + vma = find_vma(mm, addr); + lock_ctx->mmap_locked = true; + } + + return vma; +} + +#else /* CONFIG_PER_VMA_LOCK */ + +static int query_vma_setup(struct proc_maps_locking_ctx *lock_ctx) +{ + return mmap_read_lock_killable(lock_ctx->mm); +} + +static void query_vma_teardown(struct proc_maps_locking_ctx *lock_ctx) +{ + mmap_read_unlock(lock_ctx->mm); +} + +static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ctx *lock_ctx, + unsigned long addr) +{ + return find_vma(lock_ctx->mm, addr); +} + +#endif /* CONFIG_PER_VMA_LOCK */ + +static struct vm_area_struct *query_matching_vma(struct proc_maps_locking_ctx *lock_ctx, + unsigned long addr, u32 flags) +{ + struct vm_area_struct *vma; + +next_vma: + vma = query_vma_find_by_addr(lock_ctx, addr); + if (IS_ERR(vma)) + return vma; + + if (!vma) + goto no_vma; + + /* user requested only file-backed VMA, keep iterating */ + if ((flags & PROCMAP_QUERY_FILE_BACKED_VMA) && !vma->vm_file) + goto skip_vma; + + /* VMA permissions should satisfy query flags */ + if (flags & PROCMAP_QUERY_VMA_FLAGS) { + u32 perm = 0; + + if (flags & PROCMAP_QUERY_VMA_READABLE) + perm |= VM_READ; + if (flags & PROCMAP_QUERY_VMA_WRITABLE) + perm |= VM_WRITE; + if (flags & PROCMAP_QUERY_VMA_EXECUTABLE) + perm |= VM_EXEC; + if (flags & PROCMAP_QUERY_VMA_SHARED) + perm |= VM_MAYSHARE; + + if ((vma->vm_flags & perm) != perm) + goto skip_vma; + } + + /* found covering VMA or user is OK with the matching next VMA */ + if ((flags & PROCMAP_QUERY_COVERING_OR_NEXT_VMA) || vma->vm_start <= addr) + return vma; + +skip_vma: + /* + * If the user needs closest matching VMA, keep iterating. + */ + addr = vma->vm_end; + if (flags & PROCMAP_QUERY_COVERING_OR_NEXT_VMA) + goto next_vma; + +no_vma: + return ERR_PTR(-ENOENT); +} + +static int do_procmap_query(struct mm_struct *mm, void __user *uarg) +{ + struct proc_maps_locking_ctx lock_ctx = { .mm = mm }; + struct procmap_query karg; + struct vm_area_struct *vma; + struct file *vm_file = NULL; + const char *name = NULL; + char build_id_buf[BUILD_ID_SIZE_MAX], *name_buf = NULL; + __u64 usize; + int err; + + if (copy_from_user(&usize, (void __user *)uarg, sizeof(usize))) + return -EFAULT; + /* argument struct can never be that large, reject abuse */ + if (usize > PAGE_SIZE) + return -E2BIG; + /* argument struct should have at least query_flags and query_addr fields */ + if (usize < offsetofend(struct procmap_query, query_addr)) + return -EINVAL; + err = copy_struct_from_user(&karg, sizeof(karg), uarg, usize); + if (err) + return err; + + /* reject unknown flags */ + if (karg.query_flags & ~PROCMAP_QUERY_VALID_FLAGS_MASK) + return -EINVAL; + /* either both buffer address and size are set, or both should be zero */ + if (!!karg.vma_name_size != !!karg.vma_name_addr) + return -EINVAL; + if (!!karg.build_id_size != !!karg.build_id_addr) + return -EINVAL; + + if (!mm || !mmget_not_zero(mm)) + return -ESRCH; + + err = query_vma_setup(&lock_ctx); + if (err) { + mmput(mm); + return err; + } + + vma = query_matching_vma(&lock_ctx, karg.query_addr, karg.query_flags); + if (IS_ERR(vma)) { + err = PTR_ERR(vma); + vma = NULL; + goto out; + } + + karg.vma_start = vma->vm_start; + karg.vma_end = vma->vm_end; + + karg.vma_flags = 0; + if (vma->vm_flags & VM_READ) + karg.vma_flags |= PROCMAP_QUERY_VMA_READABLE; + if (vma->vm_flags & VM_WRITE) + karg.vma_flags |= PROCMAP_QUERY_VMA_WRITABLE; + if (vma->vm_flags & VM_EXEC) + karg.vma_flags |= PROCMAP_QUERY_VMA_EXECUTABLE; + if (vma->vm_flags & VM_MAYSHARE) + karg.vma_flags |= PROCMAP_QUERY_VMA_SHARED; + + karg.vma_page_size = vma_kernel_pagesize(vma); + + if (vma->vm_file) { + const struct inode *inode = file_user_inode(vma->vm_file); + + karg.vma_offset = ((__u64)vma->vm_pgoff) << PAGE_SHIFT; + karg.dev_major = MAJOR(inode->i_sb->s_dev); + karg.dev_minor = MINOR(inode->i_sb->s_dev); + karg.inode = inode->i_ino; + } else { + karg.vma_offset = 0; + karg.dev_major = 0; + karg.dev_minor = 0; + karg.inode = 0; + } + + if (karg.vma_name_size) { + size_t name_buf_sz = min_t(size_t, PATH_MAX, karg.vma_name_size); + const struct path *path; + const char *name_fmt; + size_t name_sz = 0; + + get_vma_name(vma, &path, &name, &name_fmt); + + if (path || name_fmt || name) { + name_buf = kmalloc(name_buf_sz, GFP_KERNEL); + if (!name_buf) { + err = -ENOMEM; + goto out; + } + } + if (path) { + name = d_path(path, name_buf, name_buf_sz); + if (IS_ERR(name)) { + err = PTR_ERR(name); + goto out; + } + name_sz = name_buf + name_buf_sz - name; + } else if (name || name_fmt) { + name_sz = 1 + snprintf(name_buf, name_buf_sz, name_fmt ?: "%s", name); + name = name_buf; + } + if (name_sz > name_buf_sz) { + err = -ENAMETOOLONG; + goto out; + } + karg.vma_name_size = name_sz; + } + + if (karg.build_id_size && vma->vm_file) + vm_file = get_file(vma->vm_file); + + /* unlock vma or mmap_lock, and put mm_struct before copying data to user */ + query_vma_teardown(&lock_ctx); + mmput(mm); + + if (karg.build_id_size) { + __u32 build_id_sz; + + if (vm_file) + err = build_id_parse_file(vm_file, build_id_buf, &build_id_sz); + else + err = -ENOENT; + if (err) { + karg.build_id_size = 0; + } else { + if (karg.build_id_size < build_id_sz) { + err = -ENAMETOOLONG; + goto out_file; + } + karg.build_id_size = build_id_sz; + } + } + + if (vm_file) + fput(vm_file); + + if (karg.vma_name_size && copy_to_user(u64_to_user_ptr(karg.vma_name_addr), + name, karg.vma_name_size)) { + kfree(name_buf); + return -EFAULT; + } + kfree(name_buf); + + if (karg.build_id_size && copy_to_user(u64_to_user_ptr(karg.build_id_addr), + build_id_buf, karg.build_id_size)) + return -EFAULT; + + if (copy_to_user(uarg, &karg, min_t(size_t, sizeof(karg), usize))) + return -EFAULT; + + return 0; + +out: + query_vma_teardown(&lock_ctx); + mmput(mm); +out_file: + if (vm_file) + fput(vm_file); + kfree(name_buf); + return err; +} + +static long procfs_procmap_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + struct seq_file *seq = file->private_data; + struct proc_maps_private *priv = seq->private; + + switch (cmd) { + case PROCMAP_QUERY: + /* priv->lock_ctx.mm is set during file open operation */ + return do_procmap_query(priv->lock_ctx.mm, (void __user *)arg); + default: + return -ENOIOCTLCMD; + } +} + +const struct file_operations proc_pid_maps_operations = { + .open = pid_maps_open, + .read = seq_read, + .llseek = seq_lseek, + .release = proc_map_release, + .unlocked_ioctl = procfs_procmap_ioctl, + .compat_ioctl = compat_ptr_ioctl, +}; + +/* + * Proportional Set Size(PSS): my share of RSS. + * + * PSS of a process is the count of pages it has in memory, where each + * page is divided by the number of processes sharing it. So if a + * process has 1000 pages all to itself, and 1000 shared with one other + * process, its PSS will be 1500. + * + * To keep (accumulated) division errors low, we adopt a 64bit + * fixed-point pss counter to minimize division errors. So (pss >> + * PSS_SHIFT) would be the real byte count. + * + * A shift of 12 before division means (assuming 4K page size): + * - 1M 3-user-pages add up to 8KB errors; + * - supports mapcount up to 2^24, or 16M; + * - supports PSS up to 2^52 bytes, or 4PB. + */ +#define PSS_SHIFT 12 + +#ifdef CONFIG_PROC_PAGE_MONITOR +struct mem_size_stats { + unsigned long resident; + unsigned long shared_clean; + unsigned long shared_dirty; + unsigned long private_clean; + unsigned long private_dirty; + unsigned long referenced; + unsigned long anonymous; + unsigned long lazyfree; + unsigned long anonymous_thp; + unsigned long shmem_thp; + unsigned long file_thp; + unsigned long swap; + unsigned long shared_hugetlb; + unsigned long private_hugetlb; + unsigned long ksm; + u64 pss; + u64 pss_anon; + u64 pss_file; + u64 pss_shmem; + u64 pss_dirty; + u64 pss_locked; + u64 swap_pss; +}; + +static void smaps_page_accumulate(struct mem_size_stats *mss, + struct folio *folio, unsigned long size, unsigned long pss, + bool dirty, bool locked, bool private) +{ + mss->pss += pss; + + if (folio_test_anon(folio)) + mss->pss_anon += pss; + else if (folio_test_swapbacked(folio)) + mss->pss_shmem += pss; + else + mss->pss_file += pss; + + if (locked) + mss->pss_locked += pss; + + if (dirty || folio_test_dirty(folio)) { + mss->pss_dirty += pss; + if (private) + mss->private_dirty += size; + else + mss->shared_dirty += size; + } else { + if (private) + mss->private_clean += size; + else + mss->shared_clean += size; + } +} + +static void smaps_account(struct mem_size_stats *mss, struct page *page, + bool compound, bool young, bool dirty, bool locked, + bool present) +{ + struct folio *folio = page_folio(page); + int i, nr = compound ? compound_nr(page) : 1; + unsigned long size = nr * PAGE_SIZE; + bool exclusive; + int mapcount; + + /* + * First accumulate quantities that depend only on |size| and the type + * of the compound page. + */ + if (folio_test_anon(folio)) { + mss->anonymous += size; + if (!folio_test_swapbacked(folio) && !dirty && + !folio_test_dirty(folio)) + mss->lazyfree += size; + } + + if (folio_test_ksm(folio)) + mss->ksm += size; + + mss->resident += size; + /* Accumulate the size in pages that have been accessed. */ + if (young || folio_test_young(folio) || folio_test_referenced(folio)) + mss->referenced += size; + + /* + * Then accumulate quantities that may depend on sharing, or that may + * differ page-by-page. + * + * refcount == 1 for present entries guarantees that the folio is mapped + * exactly once. For large folios this implies that exactly one + * PTE/PMD/... maps (a part of) this folio. + * + * Treat all non-present entries (where relying on the mapcount and + * refcount doesn't make sense) as "maybe shared, but not sure how + * often". We treat device private entries as being fake-present. + * + * Note that it would not be safe to read the mapcount especially for + * pages referenced by migration entries, even with the PTL held. + */ + if (folio_ref_count(folio) == 1 || !present) { + smaps_page_accumulate(mss, folio, size, size << PSS_SHIFT, + dirty, locked, present); + return; + } + + if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT)) { + mapcount = folio_average_page_mapcount(folio); + exclusive = !folio_maybe_mapped_shared(folio); + } + + /* + * We obtain a snapshot of the mapcount. Without holding the folio lock + * this snapshot can be slightly wrong as we cannot always read the + * mapcount atomically. + */ + for (i = 0; i < nr; i++, page++) { + unsigned long pss = PAGE_SIZE << PSS_SHIFT; + + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) { + mapcount = folio_precise_page_mapcount(folio, page); + exclusive = mapcount < 2; + } + + if (mapcount >= 2) + pss /= mapcount; + smaps_page_accumulate(mss, folio, PAGE_SIZE, pss, + dirty, locked, exclusive); + } +} + +#ifdef CONFIG_SHMEM +static int smaps_pte_hole(unsigned long addr, unsigned long end, + __always_unused int depth, struct mm_walk *walk) +{ + struct mem_size_stats *mss = walk->private; + struct vm_area_struct *vma = walk->vma; + + mss->swap += shmem_partial_swap_usage(walk->vma->vm_file->f_mapping, + linear_page_index(vma, addr), + linear_page_index(vma, end)); + + return 0; +} +#else +#define smaps_pte_hole NULL +#endif /* CONFIG_SHMEM */ + +static void smaps_pte_hole_lookup(unsigned long addr, struct mm_walk *walk) +{ +#ifdef CONFIG_SHMEM + if (walk->ops->pte_hole) { + /* depth is not used */ + smaps_pte_hole(addr, addr + PAGE_SIZE, 0, walk); + } +#endif +} + +static void smaps_pte_entry(pte_t *pte, unsigned long addr, + struct mm_walk *walk) +{ + struct mem_size_stats *mss = walk->private; + struct vm_area_struct *vma = walk->vma; + bool locked = !!(vma->vm_flags & VM_LOCKED); + struct page *page = NULL; + bool present = false, young = false, dirty = false; + pte_t ptent = ptep_get(pte); + + if (pte_present(ptent)) { + page = vm_normal_page(vma, addr, ptent); + young = pte_young(ptent); + dirty = pte_dirty(ptent); + present = true; + } else if (pte_none(ptent)) { + smaps_pte_hole_lookup(addr, walk); + } else { + const softleaf_t entry = softleaf_from_pte(ptent); + + if (softleaf_is_swap(entry)) { + int mapcount; + + mss->swap += PAGE_SIZE; + mapcount = swp_swapcount(entry); + if (mapcount >= 2) { + u64 pss_delta = (u64)PAGE_SIZE << PSS_SHIFT; + + do_div(pss_delta, mapcount); + mss->swap_pss += pss_delta; + } else { + mss->swap_pss += (u64)PAGE_SIZE << PSS_SHIFT; + } + } else if (softleaf_has_pfn(entry)) { + if (softleaf_is_device_private(entry)) + present = true; + page = softleaf_to_page(entry); + } + } + + if (!page) + return; + + smaps_account(mss, page, false, young, dirty, locked, present); +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr, + struct mm_walk *walk) +{ + struct mem_size_stats *mss = walk->private; + struct vm_area_struct *vma = walk->vma; + bool locked = !!(vma->vm_flags & VM_LOCKED); + struct page *page = NULL; + bool present = false; + struct folio *folio; + + if (pmd_none(*pmd)) + return; + if (pmd_present(*pmd)) { + page = vm_normal_page_pmd(vma, addr, *pmd); + present = true; + } else if (unlikely(thp_migration_supported())) { + const softleaf_t entry = softleaf_from_pmd(*pmd); + + if (softleaf_has_pfn(entry)) + page = softleaf_to_page(entry); + } + if (IS_ERR_OR_NULL(page)) + return; + folio = page_folio(page); + if (folio_test_anon(folio)) + mss->anonymous_thp += HPAGE_PMD_SIZE; + else if (folio_test_swapbacked(folio)) + mss->shmem_thp += HPAGE_PMD_SIZE; + else if (folio_is_zone_device(folio)) + /* pass */; + else + mss->file_thp += HPAGE_PMD_SIZE; + + smaps_account(mss, page, true, pmd_young(*pmd), pmd_dirty(*pmd), + locked, present); +} +#else +static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr, + struct mm_walk *walk) +{ +} +#endif + +static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, + struct mm_walk *walk) +{ + struct vm_area_struct *vma = walk->vma; + pte_t *pte; + spinlock_t *ptl; + + ptl = pmd_trans_huge_lock(pmd, vma); + if (ptl) { + smaps_pmd_entry(pmd, addr, walk); + spin_unlock(ptl); + goto out; + } + + pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); + if (!pte) { + walk->action = ACTION_AGAIN; + return 0; + } + for (; addr != end; pte++, addr += PAGE_SIZE) + smaps_pte_entry(pte, addr, walk); + pte_unmap_unlock(pte - 1, ptl); +out: + cond_resched(); + return 0; +} + +static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma) +{ + /* + * Don't forget to update Documentation/ on changes. + * + * The length of the second argument of mnemonics[] + * needs to be 3 instead of previously set 2 + * (i.e. from [BITS_PER_LONG][2] to [BITS_PER_LONG][3]) + * to avoid spurious + * -Werror=unterminated-string-initialization warning + * with GCC 15 + */ + static const char mnemonics[BITS_PER_LONG][3] = { + /* + * In case if we meet a flag we don't know about. + */ + [0 ... (BITS_PER_LONG-1)] = "??", + + [ilog2(VM_READ)] = "rd", + [ilog2(VM_WRITE)] = "wr", + [ilog2(VM_EXEC)] = "ex", + [ilog2(VM_SHARED)] = "sh", + [ilog2(VM_MAYREAD)] = "mr", + [ilog2(VM_MAYWRITE)] = "mw", + [ilog2(VM_MAYEXEC)] = "me", + [ilog2(VM_MAYSHARE)] = "ms", + [ilog2(VM_GROWSDOWN)] = "gd", + [ilog2(VM_PFNMAP)] = "pf", + [ilog2(VM_MAYBE_GUARD)] = "gu", + [ilog2(VM_LOCKED)] = "lo", + [ilog2(VM_IO)] = "io", + [ilog2(VM_SEQ_READ)] = "sr", + [ilog2(VM_RAND_READ)] = "rr", + [ilog2(VM_DONTCOPY)] = "dc", + [ilog2(VM_DONTEXPAND)] = "de", + [ilog2(VM_LOCKONFAULT)] = "lf", + [ilog2(VM_ACCOUNT)] = "ac", + [ilog2(VM_NORESERVE)] = "nr", + [ilog2(VM_HUGETLB)] = "ht", + [ilog2(VM_SYNC)] = "sf", + [ilog2(VM_ARCH_1)] = "ar", + [ilog2(VM_WIPEONFORK)] = "wf", + [ilog2(VM_DONTDUMP)] = "dd", +#ifdef CONFIG_ARM64_BTI + [ilog2(VM_ARM64_BTI)] = "bt", +#endif +#ifdef CONFIG_MEM_SOFT_DIRTY + [ilog2(VM_SOFTDIRTY)] = "sd", +#endif + [ilog2(VM_MIXEDMAP)] = "mm", + [ilog2(VM_HUGEPAGE)] = "hg", + [ilog2(VM_NOHUGEPAGE)] = "nh", + [ilog2(VM_MERGEABLE)] = "mg", + [ilog2(VM_UFFD_MISSING)]= "um", + [ilog2(VM_UFFD_WP)] = "uw", +#ifdef CONFIG_ARM64_MTE + [ilog2(VM_MTE)] = "mt", + [ilog2(VM_MTE_ALLOWED)] = "", +#endif +#ifdef CONFIG_ARCH_HAS_PKEYS + /* These come out via ProtectionKey: */ + [ilog2(VM_PKEY_BIT0)] = "", + [ilog2(VM_PKEY_BIT1)] = "", + [ilog2(VM_PKEY_BIT2)] = "", +#if CONFIG_ARCH_PKEY_BITS > 3 + [ilog2(VM_PKEY_BIT3)] = "", +#endif +#if CONFIG_ARCH_PKEY_BITS > 4 + [ilog2(VM_PKEY_BIT4)] = "", +#endif +#endif /* CONFIG_ARCH_HAS_PKEYS */ +#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR + [ilog2(VM_UFFD_MINOR)] = "ui", +#endif /* CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */ +#ifdef CONFIG_USERFAULTFD_RWP + [ilog2(VM_UFFD_RWP)] = "ur", +#endif +#ifdef CONFIG_ARCH_HAS_USER_SHADOW_STACK + [ilog2(VM_SHADOW_STACK)] = "ss", +#endif +#if defined(CONFIG_64BIT) || defined(CONFIG_PPC32) + [ilog2(VM_DROPPABLE)] = "dp", +#endif +#ifdef CONFIG_64BIT + [ilog2(VM_SEALED)] = "sl", +#endif + }; + size_t i; + + seq_puts(m, "VmFlags: "); + for (i = 0; i < BITS_PER_LONG; i++) { + if (!mnemonics[i][0]) + continue; + if (vma->vm_flags & (1UL << i)) + seq_printf(m, "%s ", mnemonics[i]); + } + seq_putc(m, '\n'); +} + +#ifdef CONFIG_HUGETLB_PAGE +static int smaps_hugetlb_range(pte_t *pte, unsigned long hmask, + unsigned long addr, unsigned long end, + struct mm_walk *walk) +{ + struct mem_size_stats *mss = walk->private; + struct vm_area_struct *vma = walk->vma; + struct folio *folio = NULL; + bool present = false; + spinlock_t *ptl; + pte_t ptent; + + ptl = huge_pte_lock(hstate_vma(vma), walk->mm, pte); + ptent = huge_ptep_get(walk->mm, addr, pte); + if (pte_present(ptent)) { + folio = page_folio(pte_page(ptent)); + present = true; + } else { + const softleaf_t entry = softleaf_from_pte(ptent); + + if (softleaf_has_pfn(entry)) + folio = softleaf_to_folio(entry); + } + + if (folio) { + /* We treat non-present entries as "maybe shared". */ + if (!present || folio_maybe_mapped_shared(folio) || + hugetlb_pmd_shared(pte)) + mss->shared_hugetlb += huge_page_size(hstate_vma(vma)); + else + mss->private_hugetlb += huge_page_size(hstate_vma(vma)); + } + spin_unlock(ptl); + return 0; +} +#else +#define smaps_hugetlb_range NULL +#endif /* HUGETLB_PAGE */ + +static const struct mm_walk_ops smaps_walk_ops = { + .pmd_entry = smaps_pte_range, + .hugetlb_entry = smaps_hugetlb_range, + .walk_lock = PGWALK_RDLOCK, +}; + +static const struct mm_walk_ops smaps_shmem_walk_ops = { + .pmd_entry = smaps_pte_range, + .hugetlb_entry = smaps_hugetlb_range, + .pte_hole = smaps_pte_hole, + .walk_lock = PGWALK_RDLOCK, +}; + +#ifdef CONFIG_PER_VMA_LOCK + +static const struct mm_walk_ops smaps_walk_vma_lock_ops = { + .pmd_entry = smaps_pte_range, + .hugetlb_entry = smaps_hugetlb_range, + .walk_lock = PGWALK_VMA_RDLOCK_VERIFY, +}; + +static const struct mm_walk_ops smaps_shmem_walk_vma_lock_ops = { + .pmd_entry = smaps_pte_range, + .hugetlb_entry = smaps_hugetlb_range, + .pte_hole = smaps_pte_hole, + .walk_lock = PGWALK_VMA_RDLOCK_VERIFY, +}; + +static inline const struct mm_walk_ops * +get_smaps_walk_ops(struct proc_maps_private *priv) +{ + if (priv->lock_ctx.mmap_locked) + return &smaps_walk_ops; + return &smaps_walk_vma_lock_ops; +} + +static inline const struct mm_walk_ops * +get_smaps_shmem_walk_ops(struct proc_maps_private *priv) +{ + if (priv->lock_ctx.mmap_locked) + return &smaps_shmem_walk_ops; + return &smaps_shmem_walk_vma_lock_ops; +} + +#else /* CONFIG_PER_VMA_LOCK */ + +static inline const struct mm_walk_ops * +get_smaps_walk_ops(struct proc_maps_private *priv) +{ + return &smaps_walk_ops; +} + +static inline const struct mm_walk_ops * +get_smaps_shmem_walk_ops(struct proc_maps_private *priv) +{ + return &smaps_shmem_walk_ops; +} + +#endif /* CONFIG_PER_VMA_LOCK */ + +/* + * Gather mem stats from @vma with the indicated beginning + * address @start, and keep them in @mss. + * + * Use vm_start of @vma as the beginning address if @start is 0. + */ +static void smap_gather_stats(struct proc_maps_private *priv, + struct vm_area_struct *vma, + struct mem_size_stats *mss, unsigned long start) +{ + const struct mm_walk_ops *ops = get_smaps_walk_ops(priv); + + /* Invalid start */ + if (start >= vma->vm_end) + return; + + if (vma == get_gate_vma(priv->lock_ctx.mm)) + return; + + /* Might sleep. Drop RCU read lock but keep the VMA locked. */ + drop_rcu(priv); + + if (vma->vm_file && shmem_mapping(vma->vm_file->f_mapping)) { + /* + * For shared or readonly shmem mappings we know that all + * swapped out pages belong to the shmem object, and we can + * obtain the swap value much more efficiently. For private + * writable mappings, we might have COW pages that are + * not affected by the parent swapped out pages of the shmem + * object, so we have to distinguish them during the page walk. + * Unless we know that the shmem object (or the part mapped by + * our VMA) has no swapped out pages at all. + */ + unsigned long shmem_swapped = shmem_swap_usage(vma); + + if (!start && (!shmem_swapped || (vma->vm_flags & VM_SHARED) || + !(vma->vm_flags & VM_WRITE))) { + mss->swap += shmem_swapped; + } else { + ops = get_smaps_shmem_walk_ops(priv); + } + } + + if (!start) + walk_page_vma(vma, ops, mss); + else + walk_page_range(vma->vm_mm, start, vma->vm_end, ops, mss); + + reacquire_rcu(priv); +} + +#define SEQ_PUT_DEC(str, val) \ + seq_put_decimal_ull_width(m, str, (val) >> 10, 8) + +/* Show the contents common for smaps and smaps_rollup */ +static void __show_smap(struct seq_file *m, const struct mem_size_stats *mss, + bool rollup_mode) +{ + SEQ_PUT_DEC("Rss: ", mss->resident); + SEQ_PUT_DEC(" kB\nPss: ", mss->pss >> PSS_SHIFT); + SEQ_PUT_DEC(" kB\nPss_Dirty: ", mss->pss_dirty >> PSS_SHIFT); + if (rollup_mode) { + /* + * These are meaningful only for smaps_rollup, otherwise two of + * them are zero, and the other one is the same as Pss. + */ + SEQ_PUT_DEC(" kB\nPss_Anon: ", + mss->pss_anon >> PSS_SHIFT); + SEQ_PUT_DEC(" kB\nPss_File: ", + mss->pss_file >> PSS_SHIFT); + SEQ_PUT_DEC(" kB\nPss_Shmem: ", + mss->pss_shmem >> PSS_SHIFT); + } + SEQ_PUT_DEC(" kB\nShared_Clean: ", mss->shared_clean); + SEQ_PUT_DEC(" kB\nShared_Dirty: ", mss->shared_dirty); + SEQ_PUT_DEC(" kB\nPrivate_Clean: ", mss->private_clean); + SEQ_PUT_DEC(" kB\nPrivate_Dirty: ", mss->private_dirty); + SEQ_PUT_DEC(" kB\nReferenced: ", mss->referenced); + SEQ_PUT_DEC(" kB\nAnonymous: ", mss->anonymous); + SEQ_PUT_DEC(" kB\nKSM: ", mss->ksm); + SEQ_PUT_DEC(" kB\nLazyFree: ", mss->lazyfree); + SEQ_PUT_DEC(" kB\nAnonHugePages: ", mss->anonymous_thp); + SEQ_PUT_DEC(" kB\nShmemPmdMapped: ", mss->shmem_thp); + SEQ_PUT_DEC(" kB\nFilePmdMapped: ", mss->file_thp); + SEQ_PUT_DEC(" kB\nShared_Hugetlb: ", mss->shared_hugetlb); + seq_put_decimal_ull_width(m, " kB\nPrivate_Hugetlb: ", + mss->private_hugetlb >> 10, 7); + SEQ_PUT_DEC(" kB\nSwap: ", mss->swap); + SEQ_PUT_DEC(" kB\nSwapPss: ", + mss->swap_pss >> PSS_SHIFT); + SEQ_PUT_DEC(" kB\nLocked: ", + mss->pss_locked >> PSS_SHIFT); + seq_puts(m, " kB\n"); +} + +static int show_smap(struct seq_file *m, void *v) +{ + struct proc_maps_private *priv = m->private; + struct vm_area_struct *vma = v; + struct mem_size_stats mss = {}; + + smap_gather_stats(priv, vma, &mss, 0); + + show_map_vma(m, vma); + + SEQ_PUT_DEC("Size: ", vma->vm_end - vma->vm_start); + SEQ_PUT_DEC(" kB\nKernelPageSize: ", vma_kernel_pagesize(vma)); + SEQ_PUT_DEC(" kB\nMMUPageSize: ", vma_mmu_pagesize(vma)); + seq_puts(m, " kB\n"); + + __show_smap(m, &mss, false); + + seq_printf(m, "THPeligible: %8u\n", + !!thp_vma_allowable_orders(vma, vma->vm_flags, TVA_SMAPS, + THP_ORDERS_ALL)); + + if (arch_pkeys_enabled()) + seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma)); + show_smap_vma_flags(m, vma); + + return 0; +} + +static int show_smaps_rollup(struct seq_file *m, void *v) +{ + struct proc_maps_private *priv = m->private; + struct mem_size_stats mss = {}; + struct mm_struct *mm = priv->lock_ctx.mm; + struct vm_area_struct *vma; + unsigned long vma_start = 0, last_vma_end = 0; + int ret = 0; + VMA_ITERATOR(vmi, mm, 0); + + priv->task = get_proc_task(priv->inode); + if (!priv->task) + return -ESRCH; + + if (!mm || !mmget_not_zero(mm)) { + ret = -ESRCH; + goto out_put_task; + } + + ret = lock_ctx_mm(&priv->lock_ctx); + if (ret) + goto out_put_mm; + + hold_task_mempolicy(priv); + vma = vma_next(&vmi); + + if (unlikely(!vma)) + goto empty_set; + + vma_start = vma->vm_start; + do { + smap_gather_stats(priv, vma, &mss, 0); + last_vma_end = vma->vm_end; + + /* + * Release mmap_lock temporarily if someone wants to + * access it for write request. + */ + if (mmap_lock_is_contended(mm)) { + vma_iter_invalidate(&vmi); + unlock_ctx_mm(&priv->lock_ctx); + ret = lock_ctx_mm(&priv->lock_ctx); + if (ret) { + release_task_mempolicy(priv); + goto out_put_mm; + } + + /* + * After dropping the lock, there are four cases to + * consider. See the following example for explanation. + * + * +------+------+-----------+ + * | VMA1 | VMA2 | VMA3 | + * +------+------+-----------+ + * | | | | + * 4k 8k 16k 400k + * + * Suppose we drop the lock after reading VMA2 due to + * contention, then we get: + * + * last_vma_end = 16k + * + * 1) VMA2 is freed, but VMA3 exists: + * + * vma_next(vmi) will return VMA3. + * In this case, just continue from VMA3. + * + * 2) VMA2 still exists: + * + * vma_next(vmi) will return VMA3. + * In this case, just continue from VMA3. + * + * 3) No more VMAs can be found: + * + * vma_next(vmi) will return NULL. + * No more things to do, just break. + * + * 4) (last_vma_end - 1) is the middle of a vma (VMA'): + * + * vma_next(vmi) will return VMA' whose range + * contains last_vma_end. + * Iterate VMA' from last_vma_end. + */ + vma = vma_next(&vmi); + /* Case 3 above */ + if (!vma) + break; + + /* Case 1 and 2 above */ + if (vma->vm_start >= last_vma_end) { + smap_gather_stats(priv, vma, &mss, 0); + last_vma_end = vma->vm_end; + continue; + } + + /* Case 4 above */ + if (vma->vm_end > last_vma_end) { + smap_gather_stats(priv, vma, &mss, last_vma_end); + last_vma_end = vma->vm_end; + } + } + } for_each_vma(vmi, vma); + +empty_set: + show_vma_header_prefix(m, vma_start, last_vma_end, 0, 0, 0, 0); + seq_pad(m, ' '); + seq_puts(m, "[rollup]\n"); + + __show_smap(m, &mss, true); + + release_task_mempolicy(priv); + unlock_ctx_mm(&priv->lock_ctx); + +out_put_mm: + mmput(mm); +out_put_task: + put_task_struct(priv->task); + priv->task = NULL; + + return ret; +} +#undef SEQ_PUT_DEC + +static const struct seq_operations proc_pid_smaps_op = { + .start = m_start, + .next = m_next, + .stop = m_stop, + .show = show_smap +}; + +static int pid_smaps_open(struct inode *inode, struct file *file) +{ + return do_maps_open(inode, file, &proc_pid_smaps_op); +} + +static int smaps_rollup_open(struct inode *inode, struct file *file) +{ + int ret; + struct proc_maps_private *priv; + + priv = kzalloc_obj(*priv, GFP_KERNEL_ACCOUNT); + if (!priv) + return -ENOMEM; + + ret = single_open(file, show_smaps_rollup, priv); + if (ret) + goto out_free; + + priv->inode = inode; + priv->lock_ctx.mm = proc_mem_open(inode, PTRACE_MODE_READ); + if (IS_ERR_OR_NULL(priv->lock_ctx.mm)) { + ret = priv->lock_ctx.mm ? PTR_ERR(priv->lock_ctx.mm) : -ESRCH; + + single_release(inode, file); + goto out_free; + } + + return 0; + +out_free: + kfree(priv); + return ret; +} + +static int smaps_rollup_release(struct inode *inode, struct file *file) +{ + struct seq_file *seq = file->private_data; + struct proc_maps_private *priv = seq->private; + + if (priv->lock_ctx.mm) + mmdrop(priv->lock_ctx.mm); + + kfree(priv); + return single_release(inode, file); +} + +const struct file_operations proc_pid_smaps_operations = { + .open = pid_smaps_open, + .read = seq_read, + .llseek = seq_lseek, + .release = proc_map_release, +}; + +const struct file_operations proc_pid_smaps_rollup_operations = { + .open = smaps_rollup_open, + .read = seq_read, + .llseek = seq_lseek, + .release = smaps_rollup_release, +}; + +enum clear_refs_types { + CLEAR_REFS_ALL = 1, + CLEAR_REFS_ANON, + CLEAR_REFS_MAPPED, + CLEAR_REFS_SOFT_DIRTY, + CLEAR_REFS_MM_HIWATER_RSS, + CLEAR_REFS_LAST, +}; + +struct clear_refs_private { + enum clear_refs_types type; +}; + +static inline bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr, pte_t pte) +{ + struct folio *folio; + + if (!pte_write(pte)) + return false; + if (!is_cow_mapping(vma->vm_flags)) + return false; + if (likely(!mm_flags_test(MMF_HAS_PINNED, vma->vm_mm))) + return false; + folio = vm_normal_folio(vma, addr, pte); + if (!folio) + return false; + return folio_maybe_dma_pinned(folio); +} + +static inline void clear_soft_dirty(struct vm_area_struct *vma, + unsigned long addr, pte_t *pte) +{ + if (!pgtable_supports_soft_dirty()) + return; + /* + * The soft-dirty tracker uses #PF-s to catch writes + * to pages, so write-protect the pte as well. See the + * Documentation/admin-guide/mm/soft-dirty.rst for full description + * of how soft-dirty works. + */ + pte_t ptent = ptep_get(pte); + + if (pte_none(ptent)) + return; + + if (pte_present(ptent)) { + pte_t old_pte; + + if (pte_is_pinned(vma, addr, ptent)) + return; + old_pte = ptep_modify_prot_start(vma, addr, pte); + ptent = pte_wrprotect(old_pte); + ptent = pte_clear_soft_dirty(ptent); + ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent); + } else { + ptent = pte_swp_clear_soft_dirty(ptent); + set_pte_at(vma->vm_mm, addr, pte, ptent); + } +} + +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) +static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma, + unsigned long addr, pmd_t *pmdp) +{ + pmd_t old, pmd = *pmdp; + + if (!pgtable_supports_soft_dirty()) + return; + + if (pmd_present(pmd)) { + /* See comment in change_huge_pmd() */ + old = pmdp_invalidate(vma, addr, pmdp); + if (pmd_dirty(old)) + pmd = pmd_mkdirty(pmd); + if (pmd_young(old)) + pmd = pmd_mkyoung(pmd); + + pmd = pmd_wrprotect(pmd); + pmd = pmd_clear_soft_dirty(pmd); + + set_pmd_at(vma->vm_mm, addr, pmdp, pmd); + } else if (pmd_is_migration_entry(pmd)) { + pmd = pmd_swp_clear_soft_dirty(pmd); + set_pmd_at(vma->vm_mm, addr, pmdp, pmd); + } +} +#else +static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma, + unsigned long addr, pmd_t *pmdp) +{ +} +#endif + +static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, + unsigned long end, struct mm_walk *walk) +{ + struct clear_refs_private *cp = walk->private; + struct vm_area_struct *vma = walk->vma; + pte_t *pte, ptent; + spinlock_t *ptl; + struct folio *folio; + + ptl = pmd_trans_huge_lock(pmd, vma); + if (ptl) { + if (cp->type == CLEAR_REFS_SOFT_DIRTY) { + clear_soft_dirty_pmd(vma, addr, pmd); + goto out; + } + + if (!pmd_present(*pmd)) + goto out; + + folio = pmd_folio(*pmd); + + /* Clear accessed and referenced bits. */ + pmdp_test_and_clear_young(vma, addr, pmd); + folio_test_clear_young(folio); + folio_clear_referenced(folio); +out: + spin_unlock(ptl); + return 0; + } + + pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); + if (!pte) { + walk->action = ACTION_AGAIN; + return 0; + } + for (; addr != end; pte++, addr += PAGE_SIZE) { + ptent = ptep_get(pte); + + if (cp->type == CLEAR_REFS_SOFT_DIRTY) { + clear_soft_dirty(vma, addr, pte); + continue; + } + + if (!pte_present(ptent)) + continue; + + folio = vm_normal_folio(vma, addr, ptent); + if (!folio) + continue; + + /* Clear accessed and referenced bits. */ + ptep_test_and_clear_young(vma, addr, pte); + folio_test_clear_young(folio); + folio_clear_referenced(folio); + } + pte_unmap_unlock(pte - 1, ptl); + cond_resched(); + return 0; +} + +static int clear_refs_test_walk(unsigned long start, unsigned long end, + struct mm_walk *walk) +{ + struct clear_refs_private *cp = walk->private; + struct vm_area_struct *vma = walk->vma; + + if (vma->vm_flags & VM_PFNMAP) + return 1; + + /* + * Writing 1 to /proc/pid/clear_refs affects all pages. + * Writing 2 to /proc/pid/clear_refs only affects anonymous pages. + * Writing 3 to /proc/pid/clear_refs only affects file mapped pages. + * Writing 4 to /proc/pid/clear_refs affects all pages. + */ + if (cp->type == CLEAR_REFS_ANON && vma->vm_file) + return 1; + if (cp->type == CLEAR_REFS_MAPPED && !vma->vm_file) + return 1; + return 0; +} + +static const struct mm_walk_ops clear_refs_walk_ops = { + .pmd_entry = clear_refs_pte_range, + .test_walk = clear_refs_test_walk, + .walk_lock = PGWALK_WRLOCK, +}; + +static ssize_t clear_refs_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct task_struct *task; + char buffer[PROC_NUMBUF] = {}; + struct mm_struct *mm; + struct vm_area_struct *vma; + enum clear_refs_types type; + int itype; + int rv; + + if (count > sizeof(buffer) - 1) + count = sizeof(buffer) - 1; + if (copy_from_user(buffer, buf, count)) + return -EFAULT; + rv = kstrtoint(strstrip(buffer), 10, &itype); + if (rv < 0) + return rv; + type = (enum clear_refs_types)itype; + if (type < CLEAR_REFS_ALL || type >= CLEAR_REFS_LAST) + return -EINVAL; + + task = get_proc_task(file_inode(file)); + if (!task) + return -ESRCH; + mm = get_task_mm(task); + if (mm) { + VMA_ITERATOR(vmi, mm, 0); + struct mmu_notifier_range range; + struct clear_refs_private cp = { + .type = type, + }; + + if (mmap_write_lock_killable(mm)) { + count = -EINTR; + goto out_mm; + } + if (type == CLEAR_REFS_MM_HIWATER_RSS) { + /* + * Writing 5 to /proc/pid/clear_refs resets the peak + * resident set size to this mm's current rss value. + */ + reset_mm_hiwater_rss(mm); + goto out_unlock; + } + + if (type == CLEAR_REFS_SOFT_DIRTY) { + for_each_vma(vmi, vma) { + if (!(vma->vm_flags & VM_SOFTDIRTY)) + continue; + vm_flags_clear(vma, VM_SOFTDIRTY); + vma_set_page_prot(vma); + } + + inc_tlb_flush_pending(mm); + mmu_notifier_range_init(&range, MMU_NOTIFY_SOFT_DIRTY, + 0, mm, 0, -1UL); + mmu_notifier_invalidate_range_start(&range); + } + walk_page_range(mm, 0, -1, &clear_refs_walk_ops, &cp); + if (type == CLEAR_REFS_SOFT_DIRTY) { + mmu_notifier_invalidate_range_end(&range); + flush_tlb_mm(mm); + dec_tlb_flush_pending(mm); + } +out_unlock: + mmap_write_unlock(mm); +out_mm: + mmput(mm); + } + put_task_struct(task); + + return count; +} + +const struct file_operations proc_clear_refs_operations = { + .write = clear_refs_write, + .llseek = noop_llseek, +}; + +typedef struct { + u64 pme; +} pagemap_entry_t; + +struct pagemapread { + int pos, len; /* units: PM_ENTRY_BYTES, not bytes */ + pagemap_entry_t *buffer; + bool show_pfn; +}; + +#define PAGEMAP_WALK_SIZE (PMD_SIZE) +#define PAGEMAP_WALK_MASK (PMD_MASK) + +#define PM_ENTRY_BYTES sizeof(pagemap_entry_t) +#define PM_PFRAME_BITS 55 +#define PM_PFRAME_MASK GENMASK_ULL(PM_PFRAME_BITS - 1, 0) +#define PM_SOFT_DIRTY BIT_ULL(55) +#define PM_MMAP_EXCLUSIVE BIT_ULL(56) +#define PM_UFFD_WP BIT_ULL(57) +#define PM_GUARD_REGION BIT_ULL(58) +#define PM_FILE BIT_ULL(61) +#define PM_SWAP BIT_ULL(62) +#define PM_PRESENT BIT_ULL(63) + +#define PM_END_OF_BUFFER 1 + +static inline pagemap_entry_t make_pme(u64 frame, u64 flags) +{ + return (pagemap_entry_t) { .pme = (frame & PM_PFRAME_MASK) | flags }; +} + +static int add_to_pagemap(pagemap_entry_t *pme, struct pagemapread *pm) +{ + pm->buffer[pm->pos++] = *pme; + if (pm->pos >= pm->len) + return PM_END_OF_BUFFER; + return 0; +} + +static bool __folio_page_mapped_exclusively(struct folio *folio, struct page *page) +{ + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + return folio_precise_page_mapcount(folio, page) == 1; + return !folio_maybe_mapped_shared(folio); +} + +static int pagemap_pte_hole(unsigned long start, unsigned long end, + __always_unused int depth, struct mm_walk *walk) +{ + struct pagemapread *pm = walk->private; + unsigned long addr = start; + int err = 0; + + while (addr < end) { + struct vm_area_struct *vma = find_vma(walk->mm, addr); + pagemap_entry_t pme = make_pme(0, 0); + /* End of address space hole, which we mark as non-present. */ + unsigned long hole_end; + + if (vma) + hole_end = min(end, vma->vm_start); + else + hole_end = end; + + for (; addr < hole_end; addr += PAGE_SIZE) { + err = add_to_pagemap(&pme, pm); + if (err) + goto out; + } + + if (!vma) + break; + + /* Addresses in the VMA. */ + if (vma->vm_flags & VM_SOFTDIRTY) + pme = make_pme(0, PM_SOFT_DIRTY); + for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) { + err = add_to_pagemap(&pme, pm); + if (err) + goto out; + } + } +out: + return err; +} + +static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, + struct vm_area_struct *vma, unsigned long addr, pte_t pte) +{ + u64 frame = 0, flags = 0; + struct page *page = NULL; + struct folio *folio; + + if (pte_none(pte)) + goto out; + + if (pte_present(pte)) { + if (pm->show_pfn) + frame = pte_pfn(pte); + flags |= PM_PRESENT; + page = vm_normal_page(vma, addr, pte); + if (pte_soft_dirty(pte)) + flags |= PM_SOFT_DIRTY; + if (pte_uffd(pte)) + flags |= PM_UFFD_WP; + } else { + softleaf_t entry; + + if (pte_swp_soft_dirty(pte)) + flags |= PM_SOFT_DIRTY; + if (pte_swp_uffd(pte)) + flags |= PM_UFFD_WP; + entry = softleaf_from_pte(pte); + if (pm->show_pfn) { + pgoff_t offset; + + /* + * For PFN swap offsets, keeping the offset field + * to be PFN only to be compatible with old smaps. + */ + if (softleaf_has_pfn(entry)) + offset = softleaf_to_pfn(entry); + else + offset = swp_offset(entry); + frame = swp_type(entry) | + (offset << MAX_SWAPFILES_SHIFT); + } + flags |= PM_SWAP; + if (softleaf_has_pfn(entry)) + page = softleaf_to_page(entry); + if (softleaf_is_uffd_wp_marker(entry)) + flags |= PM_UFFD_WP; + if (softleaf_is_guard_marker(entry)) + flags |= PM_GUARD_REGION; + } + + if (page) { + folio = page_folio(page); + if (!folio_test_anon(folio)) + flags |= PM_FILE; + if ((flags & PM_PRESENT) && + __folio_page_mapped_exclusively(folio, page)) + flags |= PM_MMAP_EXCLUSIVE; + } + +out: + if (vma->vm_flags & VM_SOFTDIRTY) + flags |= PM_SOFT_DIRTY; + + return make_pme(frame, flags); +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static int pagemap_pmd_range_thp(pmd_t *pmdp, unsigned long addr, + unsigned long end, struct vm_area_struct *vma, + struct pagemapread *pm) +{ + unsigned int idx = (addr & ~PMD_MASK) >> PAGE_SHIFT; + u64 flags = 0, frame = 0; + pmd_t pmd = *pmdp; + struct page *page = NULL; + struct folio *folio = NULL; + int err = 0; + + if (vma->vm_flags & VM_SOFTDIRTY) + flags |= PM_SOFT_DIRTY; + + if (pmd_none(pmd)) + goto populate_pagemap; + + if (pmd_present(pmd)) { + page = pmd_page(pmd); + + flags |= PM_PRESENT; + if (pmd_soft_dirty(pmd)) + flags |= PM_SOFT_DIRTY; + if (pmd_uffd(pmd)) + flags |= PM_UFFD_WP; + if (pm->show_pfn) + frame = pmd_pfn(pmd) + idx; + } else if (thp_migration_supported()) { + const softleaf_t entry = softleaf_from_pmd(pmd); + unsigned long offset; + + if (pm->show_pfn) { + if (softleaf_has_pfn(entry)) + offset = softleaf_to_pfn(entry) + idx; + else + offset = swp_offset(entry) + idx; + frame = swp_type(entry) | + (offset << MAX_SWAPFILES_SHIFT); + } + flags |= PM_SWAP; + if (pmd_swp_soft_dirty(pmd)) + flags |= PM_SOFT_DIRTY; + if (pmd_swp_uffd(pmd)) + flags |= PM_UFFD_WP; + if (softleaf_has_pfn(entry)) + page = softleaf_to_page(entry); + } + + if (page) { + folio = page_folio(page); + if (!folio_test_anon(folio)) + flags |= PM_FILE; + } + +populate_pagemap: + for (; addr != end; addr += PAGE_SIZE, idx++) { + u64 cur_flags = flags; + pagemap_entry_t pme; + + if (folio && (flags & PM_PRESENT) && + __folio_page_mapped_exclusively(folio, page)) + cur_flags |= PM_MMAP_EXCLUSIVE; + + pme = make_pme(frame, cur_flags); + err = add_to_pagemap(&pme, pm); + if (err) + break; + if (pm->show_pfn) { + if (flags & PM_PRESENT) + frame++; + else if (flags & PM_SWAP) + frame += (1 << MAX_SWAPFILES_SHIFT); + } + } + return err; +} +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end, + struct mm_walk *walk) +{ + struct vm_area_struct *vma = walk->vma; + struct pagemapread *pm = walk->private; + spinlock_t *ptl; + pte_t *pte, *orig_pte; + int err = 0; + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + ptl = pmd_trans_huge_lock(pmdp, vma); + if (ptl) { + err = pagemap_pmd_range_thp(pmdp, addr, end, vma, pm); + spin_unlock(ptl); + return err; + } +#endif + + /* + * We can assume that @vma always points to a valid one and @end never + * goes beyond vma->vm_end. + */ + orig_pte = pte = pte_offset_map_lock(walk->mm, pmdp, addr, &ptl); + if (!pte) { + walk->action = ACTION_AGAIN; + return err; + } + for (; addr < end; pte++, addr += PAGE_SIZE) { + pagemap_entry_t pme; + + pme = pte_to_pagemap_entry(pm, vma, addr, ptep_get(pte)); + err = add_to_pagemap(&pme, pm); + if (err) + break; + } + pte_unmap_unlock(orig_pte, ptl); + + cond_resched(); + + return err; +} + +#ifdef CONFIG_HUGETLB_PAGE +/* This function walks within one hugetlb entry in the single call */ +static int pagemap_hugetlb_range(pte_t *ptep, unsigned long hmask, + unsigned long addr, unsigned long end, + struct mm_walk *walk) +{ + struct pagemapread *pm = walk->private; + struct vm_area_struct *vma = walk->vma; + u64 flags = 0, frame = 0; + spinlock_t *ptl; + int err = 0; + pte_t pte; + + if (vma->vm_flags & VM_SOFTDIRTY) + flags |= PM_SOFT_DIRTY; + + ptl = huge_pte_lock(hstate_vma(vma), walk->mm, ptep); + pte = huge_ptep_get(walk->mm, addr, ptep); + if (pte_present(pte)) { + struct folio *folio = page_folio(pte_page(pte)); + + if (!folio_test_anon(folio)) + flags |= PM_FILE; + + if (!folio_maybe_mapped_shared(folio) && + !hugetlb_pmd_shared(ptep)) + flags |= PM_MMAP_EXCLUSIVE; + + if (huge_pte_uffd(pte)) + flags |= PM_UFFD_WP; + + flags |= PM_PRESENT; + if (pm->show_pfn) + frame = pte_pfn(pte) + + ((addr & ~hmask) >> PAGE_SHIFT); + } else if (pte_swp_uffd_any(pte)) { + flags |= PM_UFFD_WP; + } + + for (; addr != end; addr += PAGE_SIZE) { + pagemap_entry_t pme = make_pme(frame, flags); + + err = add_to_pagemap(&pme, pm); + if (err) + break; + if (pm->show_pfn && (flags & PM_PRESENT)) + frame++; + } + + spin_unlock(ptl); + cond_resched(); + + return err; +} +#else +#define pagemap_hugetlb_range NULL +#endif /* HUGETLB_PAGE */ + +static const struct mm_walk_ops pagemap_ops = { + .pmd_entry = pagemap_pmd_range, + .pte_hole = pagemap_pte_hole, + .hugetlb_entry = pagemap_hugetlb_range, + .walk_lock = PGWALK_RDLOCK, +}; + +/* + * /proc/pid/pagemap - an array mapping virtual pages to pfns + * + * For each page in the address space, this file contains one 64-bit entry + * consisting of the following: + * + * Bits 0-54 page frame number (PFN) if present + * Bits 0-4 swap type if swapped + * Bits 5-54 swap offset if swapped + * Bit 55 pte is soft-dirty (see Documentation/admin-guide/mm/soft-dirty.rst) + * Bit 56 page exclusively mapped + * Bit 57 pte is tracked by userfaultfd (uffd-wp or RWP) + * Bit 58 pte is a guard region + * Bits 59-60 zero + * Bit 61 page is file-page or shared-anon + * Bit 62 page swapped + * Bit 63 page present + * + * If the page is not present but in swap, then the PFN contains an + * encoding of the swap file number and the page's offset into the + * swap. Unmapped pages return a null PFN. This allows determining + * precisely which pages are mapped (or in swap) and comparing mapped + * pages between processes. + * + * Efficient users of this interface will use /proc/pid/maps to + * determine which areas of memory are actually mapped and llseek to + * skip over unmapped regions. + */ +static ssize_t pagemap_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct mm_struct *mm = file->private_data; + struct pagemapread pm; + unsigned long src; + unsigned long svpfn; + unsigned long start_vaddr; + unsigned long end_vaddr; + int ret = 0, copied = 0; + + if (!mm || !mmget_not_zero(mm)) + goto out; + + ret = -EINVAL; + /* file position must be aligned */ + if ((*ppos % PM_ENTRY_BYTES) || (count % PM_ENTRY_BYTES)) + goto out_mm; + + ret = 0; + if (!count) + goto out_mm; + + /* do not disclose physical addresses: attack vector */ + pm.show_pfn = file_ns_capable(file, &init_user_ns, CAP_SYS_ADMIN); + + pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT); + pm.buffer = kmalloc_array(pm.len, PM_ENTRY_BYTES, GFP_KERNEL); + ret = -ENOMEM; + if (!pm.buffer) + goto out_mm; + + src = *ppos; + svpfn = src / PM_ENTRY_BYTES; + end_vaddr = mm->task_size; + + /* watch out for wraparound */ + start_vaddr = end_vaddr; + if (svpfn <= (ULONG_MAX >> PAGE_SHIFT)) { + unsigned long end; + + ret = mmap_read_lock_killable(mm); + if (ret) + goto out_free; + start_vaddr = untagged_addr_remote(mm, svpfn << PAGE_SHIFT); + mmap_read_unlock(mm); + + end = start_vaddr + ((count / PM_ENTRY_BYTES) << PAGE_SHIFT); + if (end >= start_vaddr && end < mm->task_size) + end_vaddr = end; + } + + /* Ensure the address is inside the task */ + if (start_vaddr > mm->task_size) + start_vaddr = end_vaddr; + + ret = 0; + while (count && (start_vaddr < end_vaddr)) { + int len; + unsigned long end; + + pm.pos = 0; + end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK; + /* overflow ? */ + if (end < start_vaddr || end > end_vaddr) + end = end_vaddr; + ret = mmap_read_lock_killable(mm); + if (ret) + goto out_free; + ret = walk_page_range(mm, start_vaddr, end, &pagemap_ops, &pm); + mmap_read_unlock(mm); + start_vaddr = end; + + len = min(count, PM_ENTRY_BYTES * pm.pos); + if (copy_to_user(buf, pm.buffer, len)) { + ret = -EFAULT; + goto out_free; + } + copied += len; + buf += len; + count -= len; + } + *ppos += copied; + if (!ret || ret == PM_END_OF_BUFFER) + ret = copied; + +out_free: + kfree(pm.buffer); +out_mm: + mmput(mm); +out: + return ret; +} + +static int pagemap_open(struct inode *inode, struct file *file) +{ + struct mm_struct *mm; + + mm = proc_mem_open(inode, PTRACE_MODE_READ); + if (IS_ERR_OR_NULL(mm)) + return mm ? PTR_ERR(mm) : -ESRCH; + file->private_data = mm; + return 0; +} + +static int pagemap_release(struct inode *inode, struct file *file) +{ + struct mm_struct *mm = file->private_data; + + if (mm) + mmdrop(mm); + return 0; +} + +#define PM_SCAN_CATEGORIES (PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | \ + PAGE_IS_FILE | PAGE_IS_PRESENT | \ + PAGE_IS_SWAPPED | PAGE_IS_PFNZERO | \ + PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY | \ + PAGE_IS_GUARD | PAGE_IS_ACCESSED) +#define PM_SCAN_FLAGS (PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC) + +struct pagemap_scan_private { + struct pm_scan_arg arg; + unsigned long masks_of_interest, cur_vma_category; + struct page_region *vec_buf; + unsigned long vec_buf_len, vec_buf_index, found_pages; + struct page_region __user *vec_out; +}; + +static unsigned long pagemap_page_category(struct pagemap_scan_private *p, + struct vm_area_struct *vma, + unsigned long addr, pte_t pte) +{ + unsigned long categories; + + if (pte_none(pte)) { + /* + * An unpopulated pte carries no uffd bit, i.e. it is not + * write-protected. The PAGE_IS_WRITTEN fast path in + * pagemap_scan_pmd_entry() is now gated on a VM_UFFD_WP VMA; + * gate the pte_none report here the same way so the two paths + * still agree. RWP has no such fast path and an unpopulated + * page is not part of the RWP working set, so it is reported as + * neither. + */ + if (userfaultfd_wp(vma)) + return PAGE_IS_WRITTEN; + return 0; + } + + if (pte_present(pte)) { + struct page *page; + + categories = PAGE_IS_PRESENT; + + if (!pte_uffd(pte)) { + if (userfaultfd_wp(vma)) + categories |= PAGE_IS_WRITTEN; + if (userfaultfd_rwp(vma)) + categories |= PAGE_IS_ACCESSED; + } + + if (p->masks_of_interest & PAGE_IS_FILE) { + page = vm_normal_page(vma, addr, pte); + if (page && !PageAnon(page)) + categories |= PAGE_IS_FILE; + } + + if (is_zero_pfn(pte_pfn(pte))) + categories |= PAGE_IS_PFNZERO; + if (pte_soft_dirty(pte)) + categories |= PAGE_IS_SOFT_DIRTY; + } else { + softleaf_t entry; + + categories = PAGE_IS_SWAPPED; + + if (!pte_swp_uffd_any(pte)) { + if (userfaultfd_wp(vma)) + categories |= PAGE_IS_WRITTEN; + if (userfaultfd_rwp(vma)) + categories |= PAGE_IS_ACCESSED; + } + + entry = softleaf_from_pte(pte); + if (softleaf_is_guard_marker(entry)) + categories |= PAGE_IS_GUARD; + else if ((p->masks_of_interest & PAGE_IS_FILE) && + softleaf_has_pfn(entry) && + !folio_test_anon(softleaf_to_folio(entry))) + categories |= PAGE_IS_FILE; + + if (pte_swp_soft_dirty(pte)) + categories |= PAGE_IS_SOFT_DIRTY; + } + + return categories; +} + +static void make_uffd_wp_pte(struct vm_area_struct *vma, + unsigned long addr, pte_t *pte, pte_t ptent) +{ + if (pte_present(ptent)) { + pte_t old_pte; + + old_pte = ptep_modify_prot_start(vma, addr, pte); + ptent = pte_mkuffd(old_pte); + ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent); + } else if (pte_none(ptent)) { + set_pte_at(vma->vm_mm, addr, pte, + make_pte_marker(PTE_MARKER_UFFD_WP)); + } else { + ptent = pte_swp_mkuffd(ptent); + set_pte_at(vma->vm_mm, addr, pte, ptent); + } +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static unsigned long pagemap_thp_category(struct pagemap_scan_private *p, + struct vm_area_struct *vma, + unsigned long addr, pmd_t pmd) +{ + unsigned long categories = PAGE_IS_HUGE; + + if (pmd_none(pmd)) + return categories; + + if (pmd_present(pmd)) { + struct page *page; + + categories |= PAGE_IS_PRESENT; + if (!pmd_uffd(pmd)) { + if (userfaultfd_wp(vma)) + categories |= PAGE_IS_WRITTEN; + if (userfaultfd_rwp(vma)) + categories |= PAGE_IS_ACCESSED; + } + + if (p->masks_of_interest & PAGE_IS_FILE) { + page = vm_normal_page_pmd(vma, addr, pmd); + if (page && !PageAnon(page)) + categories |= PAGE_IS_FILE; + } + + if (is_huge_zero_pmd(pmd)) + categories |= PAGE_IS_PFNZERO; + if (pmd_soft_dirty(pmd)) + categories |= PAGE_IS_SOFT_DIRTY; + } else { + categories |= PAGE_IS_SWAPPED; + if (!pmd_swp_uffd(pmd)) { + if (userfaultfd_wp(vma)) + categories |= PAGE_IS_WRITTEN; + if (userfaultfd_rwp(vma)) + categories |= PAGE_IS_ACCESSED; + } + if (pmd_swp_soft_dirty(pmd)) + categories |= PAGE_IS_SOFT_DIRTY; + + if (p->masks_of_interest & PAGE_IS_FILE) { + const softleaf_t entry = softleaf_from_pmd(pmd); + + if (softleaf_has_pfn(entry) && + !folio_test_anon(softleaf_to_folio(entry))) + categories |= PAGE_IS_FILE; + } + } + + return categories; +} + +static void make_uffd_wp_pmd(struct vm_area_struct *vma, + unsigned long addr, pmd_t *pmdp) +{ + pmd_t old, pmd = *pmdp; + + if (pmd_present(pmd)) { + old = pmdp_invalidate_ad(vma, addr, pmdp); + pmd = pmd_mkuffd(old); + set_pmd_at(vma->vm_mm, addr, pmdp, pmd); + } else if (pmd_is_migration_entry(pmd)) { + pmd = pmd_swp_mkuffd(pmd); + set_pmd_at(vma->vm_mm, addr, pmdp, pmd); + } +} +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +#ifdef CONFIG_HUGETLB_PAGE +static unsigned long pagemap_hugetlb_category(struct vm_area_struct *vma, + pte_t pte) +{ + unsigned long categories = PAGE_IS_HUGE; + + if (pte_none(pte)) + return categories; + + /* + * According to pagemap_hugetlb_range(), file-backed HugeTLB + * page cannot be swapped. So PAGE_IS_FILE is not checked for + * swapped pages. + */ + if (pte_present(pte)) { + categories |= PAGE_IS_PRESENT; + + if (!huge_pte_uffd(pte)) { + if (userfaultfd_wp(vma)) + categories |= PAGE_IS_WRITTEN; + if (userfaultfd_rwp(vma)) + categories |= PAGE_IS_ACCESSED; + } + if (!PageAnon(pte_page(pte))) + categories |= PAGE_IS_FILE; + if (is_zero_pfn(pte_pfn(pte))) + categories |= PAGE_IS_PFNZERO; + if (pte_soft_dirty(pte)) + categories |= PAGE_IS_SOFT_DIRTY; + } else { + categories |= PAGE_IS_SWAPPED; + + if (!pte_swp_uffd_any(pte)) { + if (userfaultfd_wp(vma)) + categories |= PAGE_IS_WRITTEN; + if (userfaultfd_rwp(vma)) + categories |= PAGE_IS_ACCESSED; + } + if (pte_swp_soft_dirty(pte)) + categories |= PAGE_IS_SOFT_DIRTY; + } + + return categories; +} + +static void make_uffd_wp_huge_pte(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + pte_t ptent) +{ + const unsigned long psize = huge_page_size(hstate_vma(vma)); + softleaf_t entry; + + if (huge_pte_none(ptent)) { + set_huge_pte_at(vma->vm_mm, addr, ptep, + make_pte_marker(PTE_MARKER_UFFD_WP), psize); + return; + } + + entry = softleaf_from_pte(ptent); + if (softleaf_is_hwpoison(entry) || softleaf_is_marker(entry)) + return; + + if (softleaf_is_migration(entry)) { + set_huge_pte_at(vma->vm_mm, addr, ptep, + pte_swp_mkuffd(ptent), psize); + } else { + pte_t old_pte, new_pte; + + old_pte = huge_ptep_modify_prot_start(vma, addr, ptep); + new_pte = huge_pte_mkuffd(old_pte); + huge_ptep_modify_prot_commit(vma, addr, ptep, old_pte, new_pte); + } +} +#endif /* CONFIG_HUGETLB_PAGE */ + +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE) +static void pagemap_scan_backout_range(struct pagemap_scan_private *p, + unsigned long addr, unsigned long end) +{ + struct page_region *cur_buf = &p->vec_buf[p->vec_buf_index]; + + if (!p->vec_buf) + return; + + if (cur_buf->start != addr) + cur_buf->end = addr; + else + cur_buf->start = cur_buf->end = 0; + + p->found_pages -= (end - addr) / PAGE_SIZE; +} +#endif + +static bool pagemap_scan_is_interesting_page(unsigned long categories, + const struct pagemap_scan_private *p) +{ + categories ^= p->arg.category_inverted; + if ((categories & p->arg.category_mask) != p->arg.category_mask) + return false; + if (p->arg.category_anyof_mask && !(categories & p->arg.category_anyof_mask)) + return false; + + return true; +} + +static bool pagemap_scan_is_interesting_vma(unsigned long categories, + const struct pagemap_scan_private *p) +{ + unsigned long required = p->arg.category_mask & PAGE_IS_WPALLOWED; + + categories ^= p->arg.category_inverted; + if ((categories & required) != required) + return false; + + return true; +} + +static int pagemap_scan_test_walk(unsigned long start, unsigned long end, + struct mm_walk *walk) +{ + struct pagemap_scan_private *p = walk->private; + struct vm_area_struct *vma = walk->vma; + unsigned long vma_category = 0; + bool wp_allowed = userfaultfd_wp_async(vma) && + userfaultfd_wp_use_markers(vma); + + if (!wp_allowed) { + /* User requested explicit failure over wp-async capability */ + if (p->arg.flags & PM_SCAN_CHECK_WPASYNC) + return -EPERM; + /* + * User requires wr-protect, and allows silently skipping + * unsupported vmas. + */ + if (p->arg.flags & PM_SCAN_WP_MATCHING) + return 1; + /* + * Then the request doesn't involve wr-protects at all, + * fall through to the rest checks, and allow vma walk. + */ + } + + if (vma->vm_flags & VM_PFNMAP) + return 1; + + if (wp_allowed) + vma_category |= PAGE_IS_WPALLOWED; + + if (vma->vm_flags & VM_SOFTDIRTY) + vma_category |= PAGE_IS_SOFT_DIRTY; + + if (!pagemap_scan_is_interesting_vma(vma_category, p)) + return 1; + + p->cur_vma_category = vma_category; + + return 0; +} + +static bool pagemap_scan_push_range(unsigned long categories, + struct pagemap_scan_private *p, + unsigned long addr, unsigned long end) +{ + struct page_region *cur_buf = &p->vec_buf[p->vec_buf_index]; + + /* + * When there is no output buffer provided at all, the sentinel values + * won't match here. There is no other way for `cur_buf->end` to be + * non-zero other than it being non-empty. + */ + if (addr == cur_buf->end && categories == cur_buf->categories) { + cur_buf->end = end; + return true; + } + + if (cur_buf->end) { + if (p->vec_buf_index >= p->vec_buf_len - 1) + return false; + + cur_buf = &p->vec_buf[++p->vec_buf_index]; + } + + cur_buf->start = addr; + cur_buf->end = end; + cur_buf->categories = categories; + + return true; +} + +static int pagemap_scan_output(unsigned long categories, + struct pagemap_scan_private *p, + unsigned long addr, unsigned long *end) +{ + unsigned long n_pages, total_pages; + int ret = 0; + + if (!p->vec_buf) + return 0; + + categories &= p->arg.return_mask; + + n_pages = (*end - addr) / PAGE_SIZE; + if (check_add_overflow(p->found_pages, n_pages, &total_pages) || + total_pages > p->arg.max_pages) { + size_t n_too_much = total_pages - p->arg.max_pages; + *end -= n_too_much * PAGE_SIZE; + n_pages -= n_too_much; + ret = -ENOSPC; + } + + if (!pagemap_scan_push_range(categories, p, addr, *end)) { + *end = addr; + n_pages = 0; + ret = -ENOSPC; + } + + p->found_pages += n_pages; + if (ret) + p->arg.walk_end = *end; + + return ret; +} + +static int pagemap_scan_thp_entry(pmd_t *pmd, unsigned long start, + unsigned long end, struct mm_walk *walk) +{ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + struct pagemap_scan_private *p = walk->private; + struct vm_area_struct *vma = walk->vma; + unsigned long categories; + spinlock_t *ptl; + int ret = 0; + + ptl = pmd_trans_huge_lock(pmd, vma); + if (!ptl) + return -ENOENT; + + categories = p->cur_vma_category | + pagemap_thp_category(p, vma, start, *pmd); + + if (!pagemap_scan_is_interesting_page(categories, p)) + goto out_unlock; + + ret = pagemap_scan_output(categories, p, start, &end); + if (start == end) + goto out_unlock; + + if (~p->arg.flags & PM_SCAN_WP_MATCHING) + goto out_unlock; + if (~categories & PAGE_IS_WRITTEN) + goto out_unlock; + + /* + * Break huge page into small pages if the WP operation + * needs to be performed on a portion of the huge page. + */ + if (end != start + HPAGE_SIZE) { + spin_unlock(ptl); + split_huge_pmd(vma, pmd, start); + pagemap_scan_backout_range(p, start, end); + /* Report as if there was no THP */ + return -ENOENT; + } + + make_uffd_wp_pmd(vma, start, pmd); + flush_tlb_range(vma, start, end); +out_unlock: + spin_unlock(ptl); + return ret; +#else /* !CONFIG_TRANSPARENT_HUGEPAGE */ + return -ENOENT; +#endif +} + +static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, + unsigned long end, struct mm_walk *walk) +{ + struct pagemap_scan_private *p = walk->private; + struct vm_area_struct *vma = walk->vma; + unsigned long addr, flush_end = 0; + pte_t *pte, *start_pte; + spinlock_t *ptl; + int ret; + + ret = pagemap_scan_thp_entry(pmd, start, end, walk); + if (ret != -ENOENT) + return ret; + + ret = 0; + start_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, start, &ptl); + if (!pte) { + walk->action = ACTION_AGAIN; + return 0; + } + + lazy_mmu_mode_enable(); + + if ((p->arg.flags & PM_SCAN_WP_MATCHING) && !p->vec_out) { + /* Fast path for performing exclusive WP */ + for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { + pte_t ptent = ptep_get(pte); + + if ((pte_present(ptent) && pte_uffd(ptent)) || + pte_swp_uffd_any(ptent)) + continue; + make_uffd_wp_pte(vma, addr, pte, ptent); + if (!flush_end) + start = addr; + flush_end = addr + PAGE_SIZE; + } + goto flush_and_return; + } + + if (userfaultfd_wp(vma) && !p->arg.category_anyof_mask && + !p->arg.category_inverted && + p->arg.category_mask == PAGE_IS_WRITTEN && + p->arg.return_mask == PAGE_IS_WRITTEN) { + for (addr = start; addr < end; pte++, addr += PAGE_SIZE) { + unsigned long next = addr + PAGE_SIZE; + pte_t ptent = ptep_get(pte); + + if ((pte_present(ptent) && pte_uffd(ptent)) || + pte_swp_uffd_any(ptent)) + continue; + ret = pagemap_scan_output(p->cur_vma_category | PAGE_IS_WRITTEN, + p, addr, &next); + if (next == addr) + break; + if (~p->arg.flags & PM_SCAN_WP_MATCHING) + continue; + make_uffd_wp_pte(vma, addr, pte, ptent); + if (!flush_end) + start = addr; + flush_end = next; + } + goto flush_and_return; + } + + for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { + pte_t ptent = ptep_get(pte); + unsigned long categories = p->cur_vma_category | + pagemap_page_category(p, vma, addr, ptent); + unsigned long next = addr + PAGE_SIZE; + + if (!pagemap_scan_is_interesting_page(categories, p)) + continue; + + ret = pagemap_scan_output(categories, p, addr, &next); + if (next == addr) + break; + + if (~p->arg.flags & PM_SCAN_WP_MATCHING) + continue; + if (~categories & PAGE_IS_WRITTEN) + continue; + + make_uffd_wp_pte(vma, addr, pte, ptent); + if (!flush_end) + start = addr; + flush_end = next; + } + +flush_and_return: + if (flush_end) + flush_tlb_range(vma, start, addr); + + lazy_mmu_mode_disable(); + pte_unmap_unlock(start_pte, ptl); + + cond_resched(); + return ret; +} + +#ifdef CONFIG_HUGETLB_PAGE +static int pagemap_scan_hugetlb_entry(pte_t *ptep, unsigned long hmask, + unsigned long start, unsigned long end, + struct mm_walk *walk) +{ + struct pagemap_scan_private *p = walk->private; + struct vm_area_struct *vma = walk->vma; + unsigned long categories; + spinlock_t *ptl; + int ret = 0; + pte_t pte; + + if (~p->arg.flags & PM_SCAN_WP_MATCHING) { + /* Go the short route when not write-protecting pages. */ + + pte = huge_ptep_get(walk->mm, start, ptep); + categories = p->cur_vma_category | + pagemap_hugetlb_category(vma, pte); + + if (!pagemap_scan_is_interesting_page(categories, p)) + return 0; + + return pagemap_scan_output(categories, p, start, &end); + } + + i_mmap_lock_write(vma->vm_file->f_mapping); + ptl = huge_pte_lock(hstate_vma(vma), vma->vm_mm, ptep); + + pte = huge_ptep_get(walk->mm, start, ptep); + categories = p->cur_vma_category | pagemap_hugetlb_category(vma, pte); + + if (!pagemap_scan_is_interesting_page(categories, p)) + goto out_unlock; + + ret = pagemap_scan_output(categories, p, start, &end); + if (start == end) + goto out_unlock; + + if (~categories & PAGE_IS_WRITTEN) + goto out_unlock; + + if (end != start + huge_page_size(hstate_vma(vma))) { + /* Partial HugeTLB page WP isn't possible. */ + pagemap_scan_backout_range(p, start, end); + p->arg.walk_end = start; + ret = 0; + goto out_unlock; + } + + make_uffd_wp_huge_pte(vma, start, ptep, pte); + flush_hugetlb_tlb_range(vma, start, end); + +out_unlock: + spin_unlock(ptl); + i_mmap_unlock_write(vma->vm_file->f_mapping); + + return ret; +} + +/* + * Write-protect the unpopulated hugetlb entries covering [addr, end) by + * installing uffd-wp markers inline, exactly as pagemap_scan_hugetlb_entry() + * does for populated entries. + * + * walk_hugetlb_range() currently calls ->pte_hole() once per huge page, so the + * loop normally runs a single iteration; it is written to cover the full range + * in case the walker ever coalesces adjacent holes. + * + * The obvious route -- uffd_wp_range() -> hugetlb_change_protection() -- + * cannot be used here: it takes hugetlb_vma_lock_write(), but the page-table + * walker (walk_hugetlb_range()) already holds hugetlb_vma_lock_read() on the + * same VMA, so the scanning thread would deadlock against itself. PMD sharing + * is disabled on uffd-wp VMAs (hugetlb_unshare_all_pmds() at registration), so + * the vma lock guards nothing that matters for these entries anyway. + */ +static int pagemap_scan_hugetlb_hole_wp(struct vm_area_struct *vma, + unsigned long addr, unsigned long end) +{ + struct hstate *h = hstate_vma(vma); + unsigned long psize = huge_page_size(h); + struct mm_struct *mm = vma->vm_mm; + spinlock_t *ptl; + pte_t *ptep; + pte_t pte; + + for (addr = ALIGN_DOWN(addr, psize); addr < end; addr += psize) { + ptep = huge_pte_alloc(mm, vma, addr, psize); + if (!ptep) + return -ENOMEM; + + i_mmap_lock_write(vma->vm_file->f_mapping); + ptl = huge_pte_lock(h, mm, ptep); + pte = huge_ptep_get(mm, addr, ptep); + make_uffd_wp_huge_pte(vma, addr, ptep, pte); + /* + * A none entry has no cached translation, so installing the + * marker needs no TLB flush. Flush only if a fault populated + * the entry between huge_pte_alloc() and the page table lock. + */ + if (!huge_pte_none(pte)) + flush_hugetlb_tlb_range(vma, addr, addr + psize); + spin_unlock(ptl); + i_mmap_unlock_write(vma->vm_file->f_mapping); + } + + return 0; +} +#else +#define pagemap_scan_hugetlb_entry NULL +static int pagemap_scan_hugetlb_hole_wp(struct vm_area_struct *vma, + unsigned long addr, unsigned long end) +{ + return 0; +} +#endif + +static int pagemap_scan_pte_hole(unsigned long addr, unsigned long end, + int depth, struct mm_walk *walk) +{ + struct pagemap_scan_private *p = walk->private; + struct vm_area_struct *vma = walk->vma; + unsigned long categories; + int ret, err; + + if (!vma) + return 0; + + /* + * In a uffd-wp VMA an unpopulated range is treated as written: + * uffd-wp registration populates page tables and installs markers + * with WP_UNPOPULATED, so a missing marker means the range was + * zapped. See the pte_none() handling in pagemap_page_category(). + * + * hugetlb differs, see pagemap_hugetlb_category(). + */ + categories = p->cur_vma_category; + if (userfaultfd_wp(vma) && !is_vm_hugetlb_page(vma)) + categories |= PAGE_IS_WRITTEN; + + if (!pagemap_scan_is_interesting_page(categories, p)) + return 0; + + ret = pagemap_scan_output(categories, p, addr, &end); + if (addr == end) + return ret; + + if (~p->arg.flags & PM_SCAN_WP_MATCHING) + return ret; + + if (is_vm_hugetlb_page(vma)) + err = pagemap_scan_hugetlb_hole_wp(vma, addr, end); + else + err = uffd_wp_range(vma, addr, end - addr, true); + if (err < 0) + ret = err; + + return ret; +} + +static const struct mm_walk_ops pagemap_scan_ops = { + .test_walk = pagemap_scan_test_walk, + .pmd_entry = pagemap_scan_pmd_entry, + .pte_hole = pagemap_scan_pte_hole, + .hugetlb_entry = pagemap_scan_hugetlb_entry, +}; + +static int pagemap_scan_get_args(struct pm_scan_arg *arg, + unsigned long uarg) +{ + if (copy_from_user(arg, (void __user *)uarg, sizeof(*arg))) + return -EFAULT; + + if (arg->size != sizeof(struct pm_scan_arg)) + return -EINVAL; + + /* Validate requested features */ + if (arg->flags & ~PM_SCAN_FLAGS) + return -EINVAL; + if ((arg->category_inverted | arg->category_mask | + arg->category_anyof_mask | arg->return_mask) & ~PM_SCAN_CATEGORIES) + return -EINVAL; + + arg->start = untagged_addr((unsigned long)arg->start); + arg->end = untagged_addr((unsigned long)arg->end); + arg->vec = untagged_addr((unsigned long)arg->vec); + + /* Validate memory pointers */ + if (!IS_ALIGNED(arg->start, PAGE_SIZE)) + return -EINVAL; + if (!access_ok((void __user *)(long)arg->start, arg->end - arg->start)) + return -EFAULT; + if (!arg->vec && arg->vec_len) + return -EINVAL; + if (UINT_MAX == SIZE_MAX && arg->vec_len > SIZE_MAX) + return -EINVAL; + if (arg->vec && !access_ok((void __user *)(long)arg->vec, + size_mul(arg->vec_len, sizeof(struct page_region)))) + return -EFAULT; + + /* Fixup default values */ + arg->end = ALIGN(arg->end, PAGE_SIZE); + arg->walk_end = 0; + if (!arg->max_pages) + arg->max_pages = ULONG_MAX; + + return 0; +} + +static int pagemap_scan_writeback_args(struct pm_scan_arg *arg, + unsigned long uargl) +{ + struct pm_scan_arg __user *uarg = (void __user *)uargl; + + if (copy_to_user(&uarg->walk_end, &arg->walk_end, sizeof(arg->walk_end))) + return -EFAULT; + + return 0; +} + +static int pagemap_scan_init_bounce_buffer(struct pagemap_scan_private *p) +{ + if (!p->arg.vec_len) + return 0; + + p->vec_buf_len = min_t(size_t, PAGEMAP_WALK_SIZE >> PAGE_SHIFT, + p->arg.vec_len); + p->vec_buf = kmalloc_objs(*p->vec_buf, p->vec_buf_len); + if (!p->vec_buf) + return -ENOMEM; + + p->vec_buf->start = p->vec_buf->end = 0; + p->vec_out = (struct page_region __user *)(long)p->arg.vec; + + return 0; +} + +static long pagemap_scan_flush_buffer(struct pagemap_scan_private *p) +{ + const struct page_region *buf = p->vec_buf; + long n = p->vec_buf_index; + + if (!p->vec_buf) + return 0; + + if (buf[n].end != buf[n].start) + n++; + + if (!n) + return 0; + + if (copy_to_user(p->vec_out, buf, n * sizeof(*buf))) + return -EFAULT; + + p->arg.vec_len -= n; + p->vec_out += n; + + p->vec_buf_index = 0; + p->vec_buf_len = min_t(size_t, p->vec_buf_len, p->arg.vec_len); + p->vec_buf->start = p->vec_buf->end = 0; + + return n; +} + +static long do_pagemap_scan(struct mm_struct *mm, unsigned long uarg) +{ + struct pagemap_scan_private p = {0}; + unsigned long walk_start; + size_t n_ranges_out = 0; + int ret; + + ret = pagemap_scan_get_args(&p.arg, uarg); + if (ret) + return ret; + + p.masks_of_interest = p.arg.category_mask | p.arg.category_anyof_mask | + p.arg.return_mask; + ret = pagemap_scan_init_bounce_buffer(&p); + if (ret) + return ret; + + for (walk_start = p.arg.start; walk_start < p.arg.end; + walk_start = p.arg.walk_end) { + struct mmu_notifier_range range; + long n_out; + + if (fatal_signal_pending(current)) { + ret = -EINTR; + break; + } + + ret = mmap_read_lock_killable(mm); + if (ret) + break; + + /* Protection change for the range is going to happen. */ + if (p.arg.flags & PM_SCAN_WP_MATCHING) { + mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA, 0, + mm, walk_start, p.arg.end); + mmu_notifier_invalidate_range_start(&range); + } + + ret = walk_page_range(mm, walk_start, p.arg.end, + &pagemap_scan_ops, &p); + + if (p.arg.flags & PM_SCAN_WP_MATCHING) + mmu_notifier_invalidate_range_end(&range); + + mmap_read_unlock(mm); + + n_out = pagemap_scan_flush_buffer(&p); + if (n_out < 0) + ret = n_out; + else + n_ranges_out += n_out; + + if (ret != -ENOSPC) + break; + + if (p.arg.vec_len == 0 || p.found_pages == p.arg.max_pages) + break; + } + + /* ENOSPC signifies early stop (buffer full) from the walk. */ + if (!ret || ret == -ENOSPC) + ret = n_ranges_out; + + /* The walk_end isn't set when ret is zero */ + if (!p.arg.walk_end) + p.arg.walk_end = p.arg.end; + if (pagemap_scan_writeback_args(&p.arg, uarg)) + ret = -EFAULT; + + kfree(p.vec_buf); + return ret; +} + +static long do_pagemap_cmd(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct mm_struct *mm = file->private_data; + + switch (cmd) { + case PAGEMAP_SCAN: + return do_pagemap_scan(mm, arg); + + default: + return -EINVAL; + } +} + +const struct file_operations proc_pagemap_operations = { + .llseek = mem_lseek, /* borrow this */ + .read = pagemap_read, + .open = pagemap_open, + .release = pagemap_release, + .unlocked_ioctl = do_pagemap_cmd, + .compat_ioctl = do_pagemap_cmd, +}; +#endif /* CONFIG_PROC_PAGE_MONITOR */ + +#ifdef CONFIG_NUMA + +struct numa_maps { + unsigned long pages; + unsigned long anon; + unsigned long active; + unsigned long writeback; + unsigned long mapcount_max; + unsigned long dirty; + unsigned long swapcache; + unsigned long node[MAX_NUMNODES]; +}; + +struct numa_maps_private { + struct proc_maps_private proc_maps; + struct numa_maps md; +}; + +static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty, + unsigned long nr_pages) +{ + struct folio *folio = page_folio(page); + int count; + + if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) + count = folio_precise_page_mapcount(folio, page); + else + count = folio_average_page_mapcount(folio); + + md->pages += nr_pages; + if (pte_dirty || folio_test_dirty(folio)) + md->dirty += nr_pages; + + if (folio_test_swapcache(folio)) + md->swapcache += nr_pages; + + if (folio_test_active(folio) || folio_test_unevictable(folio)) + md->active += nr_pages; + + if (folio_test_writeback(folio)) + md->writeback += nr_pages; + + if (folio_test_anon(folio)) + md->anon += nr_pages; + + if (count > md->mapcount_max) + md->mapcount_max = count; + + md->node[folio_nid(folio)] += nr_pages; +} + +static struct page *can_gather_numa_stats(pte_t pte, struct vm_area_struct *vma, + unsigned long addr) +{ + struct page *page; + int nid; + + if (!pte_present(pte)) + return NULL; + + page = vm_normal_page(vma, addr, pte); + if (!page || is_zone_device_page(page)) + return NULL; + + if (PageReserved(page)) + return NULL; + + nid = page_to_nid(page); + if (!node_isset(nid, node_states[N_MEMORY])) + return NULL; + + return page; +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static struct page *can_gather_numa_stats_pmd(pmd_t pmd, + struct vm_area_struct *vma, + unsigned long addr) +{ + struct page *page; + int nid; + + if (!pmd_present(pmd)) + return NULL; + + page = vm_normal_page_pmd(vma, addr, pmd); + if (!page) + return NULL; + + if (PageReserved(page)) + return NULL; + + nid = page_to_nid(page); + if (!node_isset(nid, node_states[N_MEMORY])) + return NULL; + + return page; +} +#endif + +static int gather_pte_stats(pmd_t *pmd, unsigned long addr, + unsigned long end, struct mm_walk *walk) +{ + struct numa_maps *md = walk->private; + struct vm_area_struct *vma = walk->vma; + spinlock_t *ptl; + pte_t *orig_pte; + pte_t *pte; + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + ptl = pmd_trans_huge_lock(pmd, vma); + if (ptl) { + struct page *page; + + page = can_gather_numa_stats_pmd(*pmd, vma, addr); + if (page) + gather_stats(page, md, pmd_dirty(*pmd), + HPAGE_PMD_SIZE/PAGE_SIZE); + spin_unlock(ptl); + return 0; + } +#endif + orig_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl); + if (!pte) { + walk->action = ACTION_AGAIN; + return 0; + } + do { + pte_t ptent = ptep_get(pte); + struct page *page = can_gather_numa_stats(ptent, vma, addr); + if (!page) + continue; + gather_stats(page, md, pte_dirty(ptent), 1); + + } while (pte++, addr += PAGE_SIZE, addr != end); + pte_unmap_unlock(orig_pte, ptl); + cond_resched(); + return 0; +} +#ifdef CONFIG_HUGETLB_PAGE +static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask, + unsigned long addr, unsigned long end, struct mm_walk *walk) +{ + pte_t huge_pte; + struct numa_maps *md; + struct page *page; + spinlock_t *ptl; + + ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte); + huge_pte = huge_ptep_get(walk->mm, addr, pte); + if (!pte_present(huge_pte)) + goto out; + + page = pte_page(huge_pte); + + md = walk->private; + gather_stats(page, md, pte_dirty(huge_pte), 1); +out: + spin_unlock(ptl); + return 0; +} + +#else +static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask, + unsigned long addr, unsigned long end, struct mm_walk *walk) +{ + return 0; +} +#endif + +static const struct mm_walk_ops show_numa_ops = { + .hugetlb_entry = gather_hugetlb_stats, + .pmd_entry = gather_pte_stats, + .walk_lock = PGWALK_RDLOCK, +}; + +#ifdef CONFIG_PER_VMA_LOCK +static const struct mm_walk_ops show_numa_vma_lock_ops = { + .hugetlb_entry = gather_hugetlb_stats, + .pmd_entry = gather_pte_stats, + .walk_lock = PGWALK_VMA_RDLOCK_VERIFY, +}; + +static inline const struct mm_walk_ops * +get_show_numa_ops(struct proc_maps_private *priv) +{ + if (priv->lock_ctx.mmap_locked) + return &show_numa_ops; + return &show_numa_vma_lock_ops; +} + +#else /* CONFIG_PER_VMA_LOCK */ + +static inline const struct mm_walk_ops * +get_show_numa_ops(struct proc_maps_private *priv) +{ + return &show_numa_ops; +} + +#endif /* CONFIG_PER_VMA_LOCK */ + +/* + * Display pages allocated per node and memory policy via /proc. + */ +static int show_numa_map(struct seq_file *m, void *v) +{ + struct numa_maps_private *numa_priv = m->private; + struct proc_maps_private *proc_priv = &numa_priv->proc_maps; + struct vm_area_struct *vma = v; + struct numa_maps *md = &numa_priv->md; + struct file *file = vma->vm_file; + struct mm_struct *mm = vma->vm_mm; + char buffer[64]; + struct mempolicy *pol; + pgoff_t ilx; + int nid; + + if (!mm) + return 0; + + /* Ensure we start with an empty set of numa_maps statistics. */ + memset(md, 0, sizeof(*md)); + + pol = __get_vma_policy(vma, vma->vm_start, &ilx); + if (pol) { + mpol_to_str(buffer, sizeof(buffer), pol); + mpol_cond_put(pol); + } else { + mpol_to_str(buffer, sizeof(buffer), proc_priv->task_mempolicy); + } + + seq_printf(m, "%08lx %s", vma->vm_start, buffer); + + if (file) { + seq_puts(m, " file="); + seq_path(m, file_user_path(file), "\n\t= "); + } else if (vma_is_initial_heap(vma)) { + seq_puts(m, " heap"); + } else if (vma_is_initial_stack(vma)) { + seq_puts(m, " stack"); + } + + if (is_vm_hugetlb_page(vma)) + seq_puts(m, " huge"); + + /* Skip walking pages if gate VMA */ + if (vma != get_gate_vma(proc_priv->lock_ctx.mm)) { + /* Might sleep. Drop RCU read lock but keep the VMA locked. */ + drop_rcu(proc_priv); + walk_page_vma(vma, get_show_numa_ops(proc_priv), md); + reacquire_rcu(proc_priv); + } + + if (!md->pages) + goto out; + + if (md->anon) + seq_printf(m, " anon=%lu", md->anon); + + if (md->dirty) + seq_printf(m, " dirty=%lu", md->dirty); + + if (md->pages != md->anon && md->pages != md->dirty) + seq_printf(m, " mapped=%lu", md->pages); + + if (md->mapcount_max > 1) + seq_printf(m, " mapmax=%lu", md->mapcount_max); + + if (md->swapcache) + seq_printf(m, " swapcache=%lu", md->swapcache); + + if (md->active < md->pages && !is_vm_hugetlb_page(vma)) + seq_printf(m, " active=%lu", md->active); + + if (md->writeback) + seq_printf(m, " writeback=%lu", md->writeback); + + for_each_node_state(nid, N_MEMORY) + if (md->node[nid]) + seq_printf(m, " N%d=%lu", nid, md->node[nid]); + + seq_printf(m, " kernelpagesize_kB=%lu", vma_kernel_pagesize(vma) >> 10); +out: + seq_putc(m, '\n'); + return 0; +} + +static const struct seq_operations proc_pid_numa_maps_op = { + .start = m_start, + .next = m_next, + .stop = m_stop, + .show = show_numa_map, +}; + +static int pid_numa_maps_open(struct inode *inode, struct file *file) +{ + return proc_maps_open(inode, file, &proc_pid_numa_maps_op, + sizeof(struct numa_maps_private)); +} + +const struct file_operations proc_pid_numa_maps_operations = { + .open = pid_numa_maps_open, + .read = seq_read, + .llseek = seq_lseek, + .release = proc_map_release, +}; + +#endif /* CONFIG_NUMA */ diff --git a/fs/proc/task_nommu.c b/fs/proc/task_nommu.c new file mode 100644 index 000000000..d362919f4 --- /dev/null +++ b/fs/proc/task_nommu.c @@ -0,0 +1,296 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +/* + * Logic: we've got two memory sums for each process, "shared", and + * "non-shared". Shared memory may get counted more than once, for + * each process that owns it. Non-shared memory is counted + * accurately. + */ +void task_mem(struct seq_file *m, struct mm_struct *mm) +{ + VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *vma; + struct vm_region *region; + unsigned long bytes = 0, sbytes = 0, slack = 0, size; + + mmap_read_lock(mm); + for_each_vma(vmi, vma) { + bytes += kobjsize(vma); + + region = vma->vm_region; + if (region) { + size = kobjsize(region); + size += region->vm_end - region->vm_start; + } else { + size = vma->vm_end - vma->vm_start; + } + + if (atomic_read(&mm->mm_count) > 1 || + is_nommu_shared_mapping(vma->vm_flags)) { + sbytes += size; + } else { + bytes += size; + if (region) + slack = region->vm_end - vma->vm_end; + } + } + + if (atomic_read(&mm->mm_count) > 1) + sbytes += kobjsize(mm); + else + bytes += kobjsize(mm); + + if (current->fs && current->fs->users > 1) + sbytes += kobjsize(current->fs); + else + bytes += kobjsize(current->fs); + + if (current->files && atomic_read(¤t->files->count) > 1) + sbytes += kobjsize(current->files); + else + bytes += kobjsize(current->files); + + if (current->sighand && refcount_read(¤t->sighand->count) > 1) + sbytes += kobjsize(current->sighand); + else + bytes += kobjsize(current->sighand); + + bytes += kobjsize(current); /* includes kernel stack */ + + mmap_read_unlock(mm); + + seq_printf(m, + "Mem:\t%8lu bytes\n" + "Slack:\t%8lu bytes\n" + "Shared:\t%8lu bytes\n", + bytes, slack, sbytes); +} + +unsigned long task_vsize(struct mm_struct *mm) +{ + VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *vma; + unsigned long vsize = 0; + + mmap_read_lock(mm); + for_each_vma(vmi, vma) + vsize += vma->vm_end - vma->vm_start; + mmap_read_unlock(mm); + return vsize; +} + +unsigned long task_statm(struct mm_struct *mm, + unsigned long *shared, unsigned long *text, + unsigned long *data, unsigned long *resident) +{ + VMA_ITERATOR(vmi, mm, 0); + struct vm_area_struct *vma; + struct vm_region *region; + unsigned long size = kobjsize(mm); + + mmap_read_lock(mm); + for_each_vma(vmi, vma) { + size += kobjsize(vma); + region = vma->vm_region; + if (region) { + size += kobjsize(region); + size += region->vm_end - region->vm_start; + } + } + + *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) + >> PAGE_SHIFT; + *data = (PAGE_ALIGN(mm->start_stack) - (mm->start_data & PAGE_MASK)) + >> PAGE_SHIFT; + mmap_read_unlock(mm); + size >>= PAGE_SHIFT; + size += *text + *data; + *resident = size; + return size; +} + +/* + * display a single VMA to a sequenced file + */ +static int nommu_vma_show(struct seq_file *m, struct vm_area_struct *vma) +{ + struct mm_struct *mm = vma->vm_mm; + unsigned long ino = 0; + struct file *file; + dev_t dev = 0; + int flags; + unsigned long long pgoff = 0; + + flags = vma->vm_flags; + file = vma->vm_file; + + if (file) { + struct inode *inode = file_inode(vma->vm_file); + dev = inode->i_sb->s_dev; + ino = inode->i_ino; + pgoff = (loff_t)vma->vm_pgoff << PAGE_SHIFT; + } + + seq_setwidth(m, 25 + sizeof(void *) * 6 - 1); + seq_printf(m, + "%08lx-%08lx %c%c%c%c %08llx %02x:%02x %lu ", + vma->vm_start, + vma->vm_end, + flags & VM_READ ? 'r' : '-', + flags & VM_WRITE ? 'w' : '-', + flags & VM_EXEC ? 'x' : '-', + flags & VM_MAYSHARE ? flags & VM_SHARED ? 'S' : 's' : 'p', + pgoff, + MAJOR(dev), MINOR(dev), ino); + + if (file) { + seq_pad(m, ' '); + seq_path(m, file_user_path(file), ""); + } else if (mm && vma_is_initial_stack(vma)) { + seq_pad(m, ' '); + seq_puts(m, "[stack]"); + } + + seq_putc(m, '\n'); + return 0; +} + +/* + * display mapping lines for a particular process's /proc/pid/maps + */ +static int show_map(struct seq_file *m, void *_p) +{ + return nommu_vma_show(m, _p); +} + +static struct vm_area_struct *proc_get_vma(struct proc_maps_private *priv, + loff_t *ppos) +{ + struct vm_area_struct *vma = vma_next(&priv->iter); + + if (vma) { + *ppos = vma->vm_start; + } else { + *ppos = -1UL; + } + + return vma; +} + +static void *m_start(struct seq_file *m, loff_t *ppos) +{ + struct proc_maps_private *priv = m->private; + unsigned long last_addr = *ppos; + struct mm_struct *mm; + + /* See proc_get_vma(). Zero at the start or after lseek. */ + if (last_addr == -1UL) + return NULL; + + /* pin the task and mm whilst we play with them */ + priv->task = get_proc_task(priv->inode); + if (!priv->task) + return ERR_PTR(-ESRCH); + + mm = priv->lock_ctx.mm; + if (!mm || !mmget_not_zero(mm)) { + put_task_struct(priv->task); + priv->task = NULL; + return NULL; + } + + if (mmap_read_lock_killable(mm)) { + mmput(mm); + put_task_struct(priv->task); + priv->task = NULL; + return ERR_PTR(-EINTR); + } + + vma_iter_init(&priv->iter, mm, last_addr); + + return proc_get_vma(priv, ppos); +} + +static void m_stop(struct seq_file *m, void *v) +{ + struct proc_maps_private *priv = m->private; + struct mm_struct *mm = priv->lock_ctx.mm; + + if (!priv->task) + return; + + mmap_read_unlock(mm); + mmput(mm); + put_task_struct(priv->task); + priv->task = NULL; +} + +static void *m_next(struct seq_file *m, void *_p, loff_t *ppos) +{ + return proc_get_vma(m->private, ppos); +} + +static const struct seq_operations proc_pid_maps_ops = { + .start = m_start, + .next = m_next, + .stop = m_stop, + .show = show_map +}; + +static int maps_open(struct inode *inode, struct file *file, + const struct seq_operations *ops) +{ + struct proc_maps_private *priv; + + priv = __seq_open_private(file, ops, sizeof(*priv)); + if (!priv) + return -ENOMEM; + + priv->inode = inode; + priv->lock_ctx.mm = proc_mem_open(inode, PTRACE_MODE_READ); + if (IS_ERR_OR_NULL(priv->lock_ctx.mm)) { + int err = priv->lock_ctx.mm ? PTR_ERR(priv->lock_ctx.mm) : -ESRCH; + + seq_release_private(inode, file); + return err; + } + + return 0; +} + + +static int map_release(struct inode *inode, struct file *file) +{ + struct seq_file *seq = file->private_data; + struct proc_maps_private *priv = seq->private; + + if (priv->lock_ctx.mm) + mmdrop(priv->lock_ctx.mm); + + return seq_release_private(inode, file); +} + +static int pid_maps_open(struct inode *inode, struct file *file) +{ + return maps_open(inode, file, &proc_pid_maps_ops); +} + +const struct file_operations proc_pid_maps_operations = { + .open = pid_maps_open, + .read = seq_read, + .llseek = seq_lseek, + .release = map_release, +}; + diff --git a/fs/proc/thread_self.c b/fs/proc/thread_self.c new file mode 100644 index 000000000..61ac62c3f --- /dev/null +++ b/fs/proc/thread_self.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include "internal.h" + +/* + * /proc/thread_self: + */ +static const char *proc_thread_self_get_link(struct dentry *dentry, + struct inode *inode, + struct delayed_call *done) +{ + struct pid_namespace *ns = proc_pid_ns(inode->i_sb); + pid_t tgid = task_tgid_nr_ns(current, ns); + pid_t pid = task_pid_nr_ns(current, ns); + char *name; + + if (!pid) + return ERR_PTR(-ENOENT); + name = kmalloc(10 + 6 + 10 + 1, dentry ? GFP_KERNEL : GFP_ATOMIC); + if (unlikely(!name)) + return dentry ? ERR_PTR(-ENOMEM) : ERR_PTR(-ECHILD); + sprintf(name, "%u/task/%u", tgid, pid); + set_delayed_call(done, kfree_link, name); + return name; +} + +static const struct inode_operations proc_thread_self_inode_operations = { + .get_link = proc_thread_self_get_link, +}; + +unsigned thread_self_inum __ro_after_init; + +int proc_setup_thread_self(struct super_block *s) +{ + struct dentry *thread_self; + int ret = -ENOMEM; + + thread_self = d_alloc_name(s->s_root, "thread-self"); + if (thread_self) { + struct inode *inode = new_inode(s); + if (inode) { + inode->i_ino = thread_self_inum; + simple_inode_init_ts(inode); + inode->i_mode = S_IFLNK | S_IRWXUGO; + inode->i_uid = GLOBAL_ROOT_UID; + inode->i_gid = GLOBAL_ROOT_GID; + inode->i_op = &proc_thread_self_inode_operations; + d_make_persistent(thread_self, inode); + ret = 0; + } + dput(thread_self); + } + + if (ret) + pr_err("proc_fill_super: can't allocate /proc/thread-self\n"); + return ret; +} + +void __init proc_thread_self_init(void) +{ + proc_alloc_inum(&thread_self_inum); +} diff --git a/fs/proc/uptime.c b/fs/proc/uptime.c new file mode 100644 index 000000000..433aa947c --- /dev/null +++ b/fs/proc/uptime.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +static int uptime_proc_show(struct seq_file *m, void *v) +{ + struct timespec64 uptime; + struct timespec64 idle; + u64 idle_nsec; + u32 rem; + int i; + + idle_nsec = 0; + for_each_possible_cpu(i) + idle_nsec += kcpustat_field(CPUTIME_IDLE, i); + + ktime_get_boottime_ts64(&uptime); + timens_add_boottime(&uptime); + + idle.tv_sec = div_u64_rem(idle_nsec, NSEC_PER_SEC, &rem); + idle.tv_nsec = rem; + seq_printf(m, "%lu.%02lu %lu.%02lu\n", + (unsigned long) uptime.tv_sec, + (uptime.tv_nsec / (NSEC_PER_SEC / 100)), + (unsigned long) idle.tv_sec, + (idle.tv_nsec / (NSEC_PER_SEC / 100))); + return 0; +} + +static int __init proc_uptime_init(void) +{ + struct proc_dir_entry *pde; + + pde = proc_create_single("uptime", 0, NULL, uptime_proc_show); + pde_make_permanent(pde); + return 0; +} +fs_initcall(proc_uptime_init); diff --git a/fs/proc/util.c b/fs/proc/util.c new file mode 100644 index 000000000..98f8adc17 --- /dev/null +++ b/fs/proc/util.c @@ -0,0 +1,24 @@ +#include +#include "internal.h" + +unsigned name_to_int(const struct qstr *qstr) +{ + const char *name = qstr->name; + int len = qstr->len; + unsigned n = 0; + + if (len > 1 && *name == '0') + goto out; + do { + unsigned c = *name++ - '0'; + if (c > 9) + goto out; + if (n >= (~0U-9)/10) + goto out; + n *= 10; + n += c; + } while (--len > 0); + return n; +out: + return ~0U; +} diff --git a/fs/proc/version.c b/fs/proc/version.c new file mode 100644 index 000000000..02e3c3cd4 --- /dev/null +++ b/fs/proc/version.c @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include "internal.h" + +static int version_proc_show(struct seq_file *m, void *v) +{ + seq_printf(m, linux_proc_banner, + utsname()->sysname, + utsname()->release, + utsname()->version); + return 0; +} + +static int __init proc_version_init(void) +{ + struct proc_dir_entry *pde; + + pde = proc_create_single("version", 0, NULL, version_proc_show); + pde_make_permanent(pde); + return 0; +} +fs_initcall(proc_version_init); diff --git a/fs/proc/vmcore.c b/fs/proc/vmcore.c new file mode 100644 index 000000000..44d154364 --- /dev/null +++ b/fs/proc/vmcore.c @@ -0,0 +1,1756 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * fs/proc/vmcore.c Interface for accessing the crash + * dump from the system's previous life. + * Heavily borrowed from fs/proc/kcore.c + * Created by: Hariprasad Nellitheertha (hari@in.ibm.com) + * Copyright (C) IBM Corporation, 2004. All rights reserved + * + */ + +#define pr_fmt(fmt) "vmcore: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +/* List representing chunks of contiguous memory areas and their offsets in + * vmcore file. + */ +static LIST_HEAD(vmcore_list); + +/* Stores the pointer to the buffer containing kernel elf core headers. */ +static char *elfcorebuf; +static size_t elfcorebuf_sz; +static size_t elfcorebuf_sz_orig; + +static char *elfnotes_buf; +static size_t elfnotes_sz; +/* Size of all notes minus the device dump notes */ +static size_t elfnotes_orig_sz; + +/* Total size of vmcore file. */ +static u64 vmcore_size; + +static struct proc_dir_entry *proc_vmcore; + +#ifdef CONFIG_PROC_VMCORE_DEVICE_DUMP +struct vmcoredd_node { + struct list_head list; /* List of dumps */ + void *buf; /* Buffer containing device's dump */ + unsigned int size; /* Size of the buffer */ +}; + +/* Device Dump list and mutex to synchronize access to list */ +static LIST_HEAD(vmcoredd_list); + +static bool vmcoredd_disabled; +core_param(novmcoredd, vmcoredd_disabled, bool, 0); +#endif /* CONFIG_PROC_VMCORE_DEVICE_DUMP */ + +/* Device Dump Size */ +static size_t vmcoredd_orig_sz; + +static DEFINE_MUTEX(vmcore_mutex); + +DEFINE_STATIC_SRCU(vmcore_cb_srcu); +/* List of registered vmcore callbacks. */ +static LIST_HEAD(vmcore_cb_list); +/* Whether the vmcore has been opened once. */ +static bool vmcore_opened; +/* Whether the vmcore is currently open. */ +static unsigned int vmcore_open; + +static void vmcore_process_device_ram(struct vmcore_cb *cb); + +void register_vmcore_cb(struct vmcore_cb *cb) +{ + INIT_LIST_HEAD(&cb->next); + mutex_lock(&vmcore_mutex); + list_add_tail(&cb->next, &vmcore_cb_list); + /* + * Registering a vmcore callback after the vmcore was opened is + * very unusual (e.g., manual driver loading). + */ + if (vmcore_opened) + pr_warn_once("Unexpected vmcore callback registration\n"); + if (!vmcore_open && cb->get_device_ram) + vmcore_process_device_ram(cb); + mutex_unlock(&vmcore_mutex); +} +EXPORT_SYMBOL_GPL(register_vmcore_cb); + +void unregister_vmcore_cb(struct vmcore_cb *cb) +{ + mutex_lock(&vmcore_mutex); + list_del_rcu(&cb->next); + /* + * Unregistering a vmcore callback after the vmcore was opened is + * very unusual (e.g., forced driver removal), but we cannot stop + * unregistering. + */ + if (vmcore_opened) + pr_warn_once("Unexpected vmcore callback unregistration\n"); + mutex_unlock(&vmcore_mutex); + + synchronize_srcu(&vmcore_cb_srcu); +} +EXPORT_SYMBOL_GPL(unregister_vmcore_cb); + +static bool pfn_is_ram(unsigned long pfn) +{ + struct vmcore_cb *cb; + bool ret = true; + + list_for_each_entry_srcu(cb, &vmcore_cb_list, next, + srcu_read_lock_held(&vmcore_cb_srcu)) { + if (unlikely(!cb->pfn_is_ram)) + continue; + ret = cb->pfn_is_ram(cb, pfn); + if (!ret) + break; + } + + return ret; +} + +static int open_vmcore(struct inode *inode, struct file *file) +{ + mutex_lock(&vmcore_mutex); + vmcore_opened = true; + if (vmcore_open + 1 == 0) { + mutex_unlock(&vmcore_mutex); + return -EBUSY; + } + vmcore_open++; + mutex_unlock(&vmcore_mutex); + + return 0; +} + +static int release_vmcore(struct inode *inode, struct file *file) +{ + mutex_lock(&vmcore_mutex); + vmcore_open--; + mutex_unlock(&vmcore_mutex); + + return 0; +} + +/* Reads a page from the oldmem device from given offset. */ +ssize_t read_from_oldmem(struct iov_iter *iter, size_t count, + u64 *ppos, bool encrypted) +{ + unsigned long pfn, offset; + ssize_t nr_bytes; + ssize_t read = 0, tmp; + int idx; + + if (!count) + return 0; + + offset = (unsigned long)(*ppos % PAGE_SIZE); + pfn = (unsigned long)(*ppos / PAGE_SIZE); + + idx = srcu_read_lock(&vmcore_cb_srcu); + do { + if (count > (PAGE_SIZE - offset)) + nr_bytes = PAGE_SIZE - offset; + else + nr_bytes = count; + + /* If pfn is not ram, return zeros for sparse dump files */ + if (!pfn_is_ram(pfn)) { + tmp = iov_iter_zero(nr_bytes, iter); + } else { + if (encrypted) + tmp = copy_oldmem_page_encrypted(iter, pfn, + nr_bytes, + offset); + else + tmp = copy_oldmem_page(iter, pfn, nr_bytes, + offset); + } + if (tmp < nr_bytes) { + srcu_read_unlock(&vmcore_cb_srcu, idx); + return -EFAULT; + } + + *ppos += nr_bytes; + count -= nr_bytes; + read += nr_bytes; + ++pfn; + offset = 0; + } while (count); + srcu_read_unlock(&vmcore_cb_srcu, idx); + + return read; +} + +/* + * Architectures may override this function to allocate ELF header in 2nd kernel + */ +int __weak elfcorehdr_alloc(unsigned long long *addr, unsigned long long *size) +{ + return 0; +} + +/* + * Architectures may override this function to free header + */ +void __weak elfcorehdr_free(unsigned long long addr) +{} + +/* + * Architectures may override this function to read from ELF header + */ +ssize_t __weak elfcorehdr_read(char *buf, size_t count, u64 *ppos) +{ + struct kvec kvec = { .iov_base = buf, .iov_len = count }; + struct iov_iter iter; + + iov_iter_kvec(&iter, ITER_DEST, &kvec, 1, count); + + return read_from_oldmem(&iter, count, ppos, false); +} + +/* + * Architectures may override this function to read from notes sections + */ +ssize_t __weak elfcorehdr_read_notes(char *buf, size_t count, u64 *ppos) +{ + struct kvec kvec = { .iov_base = buf, .iov_len = count }; + struct iov_iter iter; + + iov_iter_kvec(&iter, ITER_DEST, &kvec, 1, count); + + return read_from_oldmem(&iter, count, ppos, + cc_platform_has(CC_ATTR_MEM_ENCRYPT)); +} + +/* + * Architectures may override this function to map oldmem + */ +int __weak remap_oldmem_pfn_range(struct vm_area_struct *vma, + unsigned long from, unsigned long pfn, + unsigned long size, pgprot_t prot) +{ + prot = pgprot_encrypted(prot); + return remap_pfn_range(vma, from, pfn, size, prot); +} + +/* + * Architectures which support memory encryption override this. + */ +ssize_t __weak copy_oldmem_page_encrypted(struct iov_iter *iter, + unsigned long pfn, size_t csize, unsigned long offset) +{ + return copy_oldmem_page(iter, pfn, csize, offset); +} + +#ifdef CONFIG_PROC_VMCORE_DEVICE_DUMP +static int vmcoredd_copy_dumps(struct iov_iter *iter, u64 start, size_t size) +{ + struct vmcoredd_node *dump; + u64 offset = 0; + size_t tsz; + char *buf; + + list_for_each_entry(dump, &vmcoredd_list, list) { + if (start < offset + dump->size) { + tsz = min(offset + (u64)dump->size - start, (u64)size); + buf = dump->buf + start - offset; + if (copy_to_iter(buf, tsz, iter) < tsz) + return -EFAULT; + + size -= tsz; + start += tsz; + + /* Leave now if buffer filled already */ + if (!size) + return 0; + } + offset += dump->size; + } + + return 0; +} + +#ifdef CONFIG_MMU +static int vmcoredd_mmap_dumps(struct vm_area_struct *vma, unsigned long dst, + u64 start, size_t size) +{ + struct vmcoredd_node *dump; + u64 offset = 0; + size_t tsz; + char *buf; + + list_for_each_entry(dump, &vmcoredd_list, list) { + if (start < offset + dump->size) { + tsz = min(offset + (u64)dump->size - start, (u64)size); + buf = dump->buf + start - offset; + if (remap_vmalloc_range_partial(vma, dst, buf, 0, + tsz)) + return -EFAULT; + + size -= tsz; + start += tsz; + dst += tsz; + + /* Leave now if buffer filled already */ + if (!size) + return 0; + } + offset += dump->size; + } + + return 0; +} +#endif /* CONFIG_MMU */ +#endif /* CONFIG_PROC_VMCORE_DEVICE_DUMP */ + +/* Read from the ELF header and then the crash dump. On error, negative value is + * returned otherwise number of bytes read are returned. + */ +static ssize_t __read_vmcore(struct iov_iter *iter, loff_t *fpos) +{ + struct vmcore_range *m = NULL; + ssize_t acc = 0, tmp; + size_t tsz; + u64 start; + + if (!iov_iter_count(iter) || *fpos >= vmcore_size) + return 0; + + iov_iter_truncate(iter, vmcore_size - *fpos); + + /* Read ELF core header */ + if (*fpos < elfcorebuf_sz) { + tsz = min(elfcorebuf_sz - (size_t)*fpos, iov_iter_count(iter)); + if (copy_to_iter(elfcorebuf + *fpos, tsz, iter) < tsz) + return -EFAULT; + *fpos += tsz; + acc += tsz; + + /* leave now if filled buffer already */ + if (!iov_iter_count(iter)) + return acc; + } + + /* Read ELF note segment */ + if (*fpos < elfcorebuf_sz + elfnotes_sz) { + void *kaddr; + + /* We add device dumps before other elf notes because the + * other elf notes may not fill the elf notes buffer + * completely and we will end up with zero-filled data + * between the elf notes and the device dumps. Tools will + * then try to decode this zero-filled data as valid notes + * and we don't want that. Hence, adding device dumps before + * the other elf notes ensure that zero-filled data can be + * avoided. + */ +#ifdef CONFIG_PROC_VMCORE_DEVICE_DUMP + /* Read device dumps */ + if (*fpos < elfcorebuf_sz + vmcoredd_orig_sz) { + tsz = min(elfcorebuf_sz + vmcoredd_orig_sz - + (size_t)*fpos, iov_iter_count(iter)); + start = *fpos - elfcorebuf_sz; + if (vmcoredd_copy_dumps(iter, start, tsz)) + return -EFAULT; + + *fpos += tsz; + acc += tsz; + + /* leave now if filled buffer already */ + if (!iov_iter_count(iter)) + return acc; + } +#endif /* CONFIG_PROC_VMCORE_DEVICE_DUMP */ + + /* Read remaining elf notes */ + tsz = min(elfcorebuf_sz + elfnotes_sz - (size_t)*fpos, + iov_iter_count(iter)); + kaddr = elfnotes_buf + *fpos - elfcorebuf_sz - vmcoredd_orig_sz; + if (copy_to_iter(kaddr, tsz, iter) < tsz) + return -EFAULT; + + *fpos += tsz; + acc += tsz; + + /* leave now if filled buffer already */ + if (!iov_iter_count(iter)) + return acc; + + cond_resched(); + } + + list_for_each_entry(m, &vmcore_list, list) { + if (*fpos < m->offset + m->size) { + tsz = (size_t)min_t(unsigned long long, + m->offset + m->size - *fpos, + iov_iter_count(iter)); + start = m->paddr + *fpos - m->offset; + tmp = read_from_oldmem(iter, tsz, &start, + cc_platform_has(CC_ATTR_MEM_ENCRYPT)); + if (tmp < 0) + return tmp; + *fpos += tsz; + acc += tsz; + + /* leave now if filled buffer already */ + if (!iov_iter_count(iter)) + return acc; + } + + cond_resched(); + } + + return acc; +} + +static ssize_t read_vmcore(struct kiocb *iocb, struct iov_iter *iter) +{ + return __read_vmcore(iter, &iocb->ki_pos); +} + +/** + * vmcore_alloc_buf - allocate buffer in vmalloc memory + * @size: size of buffer + * + * If CONFIG_MMU is defined, use vmalloc_user() to allow users to mmap + * the buffer to user-space by means of remap_vmalloc_range(). + * + * If CONFIG_MMU is not defined, use vzalloc() since mmap_vmcore() is + * disabled and there's no need to allow users to mmap the buffer. + */ +static inline char *vmcore_alloc_buf(size_t size) +{ +#ifdef CONFIG_MMU + return vmalloc_user(size); +#else + return vzalloc(size); +#endif +} + +/* + * Disable mmap_vmcore() if CONFIG_MMU is not defined. MMU is + * essential for mmap_vmcore() in order to map physically + * non-contiguous objects (ELF header, ELF note segment and memory + * regions in the 1st kernel pointed to by PT_LOAD entries) into + * virtually contiguous user-space in ELF layout. + */ +#ifdef CONFIG_MMU + +/* + * The vmcore fault handler uses the page cache and fills data using the + * standard __read_vmcore() function. + * + * On s390 the fault handler is used for memory regions that can't be mapped + * directly with remap_pfn_range(). + */ +static vm_fault_t mmap_vmcore_fault(struct vm_fault *vmf) +{ +#ifdef CONFIG_S390 + struct address_space *mapping = vmf->vma->vm_file->f_mapping; + pgoff_t index = vmf->pgoff; + struct iov_iter iter; + struct kvec kvec; + struct page *page; + loff_t offset; + int rc; + + page = find_or_create_page(mapping, index, GFP_KERNEL); + if (!page) + return VM_FAULT_OOM; + if (!PageUptodate(page)) { + offset = (loff_t) index << PAGE_SHIFT; + kvec.iov_base = page_address(page); + kvec.iov_len = PAGE_SIZE; + iov_iter_kvec(&iter, ITER_DEST, &kvec, 1, PAGE_SIZE); + + rc = __read_vmcore(&iter, &offset); + if (rc < 0) { + unlock_page(page); + put_page(page); + return vmf_error(rc); + } + SetPageUptodate(page); + } + unlock_page(page); + vmf->page = page; + return 0; +#else + return VM_FAULT_SIGBUS; +#endif +} + +static const struct vm_operations_struct vmcore_mmap_ops = { + .fault = mmap_vmcore_fault, +}; + +/* + * remap_oldmem_pfn_checked - do remap_oldmem_pfn_range replacing all pages + * reported as not being ram with the zero page. + * + * @vma: vm_area_struct describing requested mapping + * @from: start remapping from + * @pfn: page frame number to start remapping to + * @size: remapping size + * @prot: protection bits + * + * Returns zero on success, -EAGAIN on failure. + */ +static int remap_oldmem_pfn_checked(struct vm_area_struct *vma, + unsigned long from, unsigned long pfn, + unsigned long size, pgprot_t prot) +{ + unsigned long map_size; + unsigned long pos_start, pos_end, pos; + unsigned long zeropage_pfn = zero_pfn(0); + size_t len = 0; + + pos_start = pfn; + pos_end = pfn + (size >> PAGE_SHIFT); + + for (pos = pos_start; pos < pos_end; ++pos) { + if (!pfn_is_ram(pos)) { + /* + * We hit a page which is not ram. Remap the continuous + * region between pos_start and pos-1 and replace + * the non-ram page at pos with the zero page. + */ + if (pos > pos_start) { + /* Remap continuous region */ + map_size = (pos - pos_start) << PAGE_SHIFT; + if (remap_oldmem_pfn_range(vma, from + len, + pos_start, map_size, + prot)) + goto fail; + len += map_size; + } + /* Remap the zero page */ + if (remap_oldmem_pfn_range(vma, from + len, + zeropage_pfn, + PAGE_SIZE, prot)) + goto fail; + len += PAGE_SIZE; + pos_start = pos + 1; + } + } + if (pos > pos_start) { + /* Remap the rest */ + map_size = (pos - pos_start) << PAGE_SHIFT; + if (remap_oldmem_pfn_range(vma, from + len, pos_start, + map_size, prot)) + goto fail; + } + return 0; +fail: + do_munmap(vma->vm_mm, from, len, NULL); + return -EAGAIN; +} + +static int vmcore_remap_oldmem_pfn(struct vm_area_struct *vma, + unsigned long from, unsigned long pfn, + unsigned long size, pgprot_t prot) +{ + int ret, idx; + + /* + * Check if a callback was registered to avoid looping over all + * pages without a reason. + */ + idx = srcu_read_lock(&vmcore_cb_srcu); + if (!list_empty(&vmcore_cb_list)) + ret = remap_oldmem_pfn_checked(vma, from, pfn, size, prot); + else + ret = remap_oldmem_pfn_range(vma, from, pfn, size, prot); + srcu_read_unlock(&vmcore_cb_srcu, idx); + return ret; +} + +static int mmap_vmcore(struct file *file, struct vm_area_struct *vma) +{ + size_t size = vma->vm_end - vma->vm_start; + u64 start, end, len, tsz; + struct vmcore_range *m; + + start = (u64)vma->vm_pgoff << PAGE_SHIFT; + end = start + size; + + if (size > vmcore_size || end > vmcore_size) + return -EINVAL; + + if (vma->vm_flags & (VM_WRITE | VM_EXEC)) + return -EPERM; + + vm_flags_mod(vma, VM_MIXEDMAP, VM_MAYWRITE | VM_MAYEXEC); + vma->vm_ops = &vmcore_mmap_ops; + + len = 0; + + if (start < elfcorebuf_sz) { + u64 pfn; + + tsz = min(elfcorebuf_sz - (size_t)start, size); + pfn = __pa(elfcorebuf + start) >> PAGE_SHIFT; + if (remap_pfn_range(vma, vma->vm_start, pfn, tsz, + vma->vm_page_prot)) + return -EAGAIN; + size -= tsz; + start += tsz; + len += tsz; + + if (size == 0) + return 0; + } + + if (start < elfcorebuf_sz + elfnotes_sz) { + void *kaddr; + + /* We add device dumps before other elf notes because the + * other elf notes may not fill the elf notes buffer + * completely and we will end up with zero-filled data + * between the elf notes and the device dumps. Tools will + * then try to decode this zero-filled data as valid notes + * and we don't want that. Hence, adding device dumps before + * the other elf notes ensure that zero-filled data can be + * avoided. This also ensures that the device dumps and + * other elf notes can be properly mmaped at page aligned + * address. + */ +#ifdef CONFIG_PROC_VMCORE_DEVICE_DUMP + /* Read device dumps */ + if (start < elfcorebuf_sz + vmcoredd_orig_sz) { + u64 start_off; + + tsz = min(elfcorebuf_sz + vmcoredd_orig_sz - + (size_t)start, size); + start_off = start - elfcorebuf_sz; + if (vmcoredd_mmap_dumps(vma, vma->vm_start + len, + start_off, tsz)) + goto fail; + + size -= tsz; + start += tsz; + len += tsz; + + /* leave now if filled buffer already */ + if (!size) + return 0; + } +#endif /* CONFIG_PROC_VMCORE_DEVICE_DUMP */ + + /* Read remaining elf notes */ + tsz = min(elfcorebuf_sz + elfnotes_sz - (size_t)start, size); + kaddr = elfnotes_buf + start - elfcorebuf_sz - vmcoredd_orig_sz; + if (remap_vmalloc_range_partial(vma, vma->vm_start + len, + kaddr, 0, tsz)) + goto fail; + + size -= tsz; + start += tsz; + len += tsz; + + if (size == 0) + return 0; + } + + list_for_each_entry(m, &vmcore_list, list) { + if (start < m->offset + m->size) { + u64 paddr = 0; + + tsz = (size_t)min_t(unsigned long long, + m->offset + m->size - start, size); + paddr = m->paddr + start - m->offset; + if (vmcore_remap_oldmem_pfn(vma, vma->vm_start + len, + paddr >> PAGE_SHIFT, tsz, + vma->vm_page_prot)) + goto fail; + size -= tsz; + start += tsz; + len += tsz; + + if (size == 0) + return 0; + } + } + + return 0; +fail: + do_munmap(vma->vm_mm, vma->vm_start, len, NULL); + return -EAGAIN; +} +#else +static int mmap_vmcore(struct file *file, struct vm_area_struct *vma) +{ + return -ENOSYS; +} +#endif + +static const struct proc_ops vmcore_proc_ops = { + .proc_open = open_vmcore, + .proc_release = release_vmcore, + .proc_read_iter = read_vmcore, + .proc_lseek = default_llseek, + .proc_mmap = mmap_vmcore, +}; + +static u64 get_vmcore_size(size_t elfsz, size_t elfnotesegsz, + struct list_head *vc_list) +{ + struct vmcore_range *m; + u64 size; + + size = elfsz + elfnotesegsz; + list_for_each_entry(m, vc_list, list) { + size += m->size; + } + return size; +} + +/** + * update_note_header_size_elf64 - update p_memsz member of each PT_NOTE entry + * + * @ehdr_ptr: ELF header + * + * This function updates p_memsz member of each PT_NOTE entry in the + * program header table pointed to by @ehdr_ptr to real size of ELF + * note segment. + */ +static int __init update_note_header_size_elf64(const Elf64_Ehdr *ehdr_ptr) +{ + int i, rc=0; + Elf64_Phdr *phdr_ptr; + Elf64_Nhdr *nhdr_ptr; + + phdr_ptr = (Elf64_Phdr *)(ehdr_ptr + 1); + for (i = 0; i < ehdr_ptr->e_phnum; i++, phdr_ptr++) { + void *notes_section; + u64 offset, max_sz, sz, real_sz = 0; + if (phdr_ptr->p_type != PT_NOTE) + continue; + max_sz = phdr_ptr->p_memsz; + offset = phdr_ptr->p_offset; + notes_section = kmalloc(max_sz, GFP_KERNEL); + if (!notes_section) + return -ENOMEM; + rc = elfcorehdr_read_notes(notes_section, max_sz, &offset); + if (rc < 0) { + kfree(notes_section); + return rc; + } + nhdr_ptr = notes_section; + while (nhdr_ptr->n_namesz != 0) { + sz = sizeof(Elf64_Nhdr) + + (((u64)nhdr_ptr->n_namesz + 3) & ~3) + + (((u64)nhdr_ptr->n_descsz + 3) & ~3); + if ((real_sz + sz) > max_sz) { + pr_warn("Warning: Exceeded p_memsz, dropping PT_NOTE entry n_namesz=0x%x, n_descsz=0x%x\n", + nhdr_ptr->n_namesz, nhdr_ptr->n_descsz); + break; + } + real_sz += sz; + nhdr_ptr = (Elf64_Nhdr*)((char*)nhdr_ptr + sz); + } + kfree(notes_section); + phdr_ptr->p_memsz = real_sz; + if (real_sz == 0) { + pr_warn("Warning: Zero PT_NOTE entries found\n"); + } + } + + return 0; +} + +/** + * get_note_number_and_size_elf64 - get the number of PT_NOTE program + * headers and sum of real size of their ELF note segment headers and + * data. + * + * @ehdr_ptr: ELF header + * @nr_ptnote: buffer for the number of PT_NOTE program headers + * @sz_ptnote: buffer for size of unique PT_NOTE program header + * + * This function is used to merge multiple PT_NOTE program headers + * into a unique single one. The resulting unique entry will have + * @sz_ptnote in its phdr->p_mem. + * + * It is assumed that program headers with PT_NOTE type pointed to by + * @ehdr_ptr has already been updated by update_note_header_size_elf64 + * and each of PT_NOTE program headers has actual ELF note segment + * size in its p_memsz member. + */ +static int __init get_note_number_and_size_elf64(const Elf64_Ehdr *ehdr_ptr, + int *nr_ptnote, u64 *sz_ptnote) +{ + int i; + Elf64_Phdr *phdr_ptr; + + *nr_ptnote = *sz_ptnote = 0; + + phdr_ptr = (Elf64_Phdr *)(ehdr_ptr + 1); + for (i = 0; i < ehdr_ptr->e_phnum; i++, phdr_ptr++) { + if (phdr_ptr->p_type != PT_NOTE) + continue; + *nr_ptnote += 1; + *sz_ptnote += phdr_ptr->p_memsz; + } + + return 0; +} + +/** + * copy_notes_elf64 - copy ELF note segments in a given buffer + * + * @ehdr_ptr: ELF header + * @notes_buf: buffer into which ELF note segments are copied + * + * This function is used to copy ELF note segment in the 1st kernel + * into the buffer @notes_buf in the 2nd kernel. It is assumed that + * size of the buffer @notes_buf is equal to or larger than sum of the + * real ELF note segment headers and data. + * + * It is assumed that program headers with PT_NOTE type pointed to by + * @ehdr_ptr has already been updated by update_note_header_size_elf64 + * and each of PT_NOTE program headers has actual ELF note segment + * size in its p_memsz member. + */ +static int __init copy_notes_elf64(const Elf64_Ehdr *ehdr_ptr, char *notes_buf) +{ + int i, rc=0; + Elf64_Phdr *phdr_ptr; + + phdr_ptr = (Elf64_Phdr*)(ehdr_ptr + 1); + + for (i = 0; i < ehdr_ptr->e_phnum; i++, phdr_ptr++) { + u64 offset; + if (phdr_ptr->p_type != PT_NOTE) + continue; + offset = phdr_ptr->p_offset; + rc = elfcorehdr_read_notes(notes_buf, phdr_ptr->p_memsz, + &offset); + if (rc < 0) + return rc; + notes_buf += phdr_ptr->p_memsz; + } + + return 0; +} + +/* Merges all the PT_NOTE headers into one. */ +static int __init merge_note_headers_elf64(char *elfptr, size_t *elfsz, + char **notes_buf, size_t *notes_sz) +{ + int i, nr_ptnote=0, rc=0; + char *tmp; + Elf64_Ehdr *ehdr_ptr; + Elf64_Phdr phdr; + u64 phdr_sz = 0, note_off; + + ehdr_ptr = (Elf64_Ehdr *)elfptr; + + rc = update_note_header_size_elf64(ehdr_ptr); + if (rc < 0) + return rc; + + rc = get_note_number_and_size_elf64(ehdr_ptr, &nr_ptnote, &phdr_sz); + if (rc < 0) + return rc; + + *notes_sz = roundup(phdr_sz, PAGE_SIZE); + *notes_buf = vmcore_alloc_buf(*notes_sz); + if (!*notes_buf) + return -ENOMEM; + + rc = copy_notes_elf64(ehdr_ptr, *notes_buf); + if (rc < 0) + return rc; + + /* Prepare merged PT_NOTE program header. */ + phdr.p_type = PT_NOTE; + phdr.p_flags = 0; + note_off = sizeof(Elf64_Ehdr) + + (ehdr_ptr->e_phnum - nr_ptnote +1) * sizeof(Elf64_Phdr); + phdr.p_offset = roundup(note_off, PAGE_SIZE); + phdr.p_vaddr = phdr.p_paddr = 0; + phdr.p_filesz = phdr.p_memsz = phdr_sz; + phdr.p_align = 4; + + /* Add merged PT_NOTE program header*/ + tmp = elfptr + sizeof(Elf64_Ehdr); + memcpy(tmp, &phdr, sizeof(phdr)); + tmp += sizeof(phdr); + + /* Remove unwanted PT_NOTE program headers. */ + i = (nr_ptnote - 1) * sizeof(Elf64_Phdr); + *elfsz = *elfsz - i; + memmove(tmp, tmp+i, ((*elfsz)-sizeof(Elf64_Ehdr)-sizeof(Elf64_Phdr))); + memset(elfptr + *elfsz, 0, i); + *elfsz = roundup(*elfsz, PAGE_SIZE); + + /* Modify e_phnum to reflect merged headers. */ + ehdr_ptr->e_phnum = ehdr_ptr->e_phnum - nr_ptnote + 1; + + /* Store the size of all notes. We need this to update the note + * header when the device dumps will be added. + */ + elfnotes_orig_sz = phdr.p_memsz; + + return 0; +} + +/** + * update_note_header_size_elf32 - update p_memsz member of each PT_NOTE entry + * + * @ehdr_ptr: ELF header + * + * This function updates p_memsz member of each PT_NOTE entry in the + * program header table pointed to by @ehdr_ptr to real size of ELF + * note segment. + */ +static int __init update_note_header_size_elf32(const Elf32_Ehdr *ehdr_ptr) +{ + int i, rc=0; + Elf32_Phdr *phdr_ptr; + Elf32_Nhdr *nhdr_ptr; + + phdr_ptr = (Elf32_Phdr *)(ehdr_ptr + 1); + for (i = 0; i < ehdr_ptr->e_phnum; i++, phdr_ptr++) { + void *notes_section; + u64 offset, max_sz, sz, real_sz = 0; + if (phdr_ptr->p_type != PT_NOTE) + continue; + max_sz = phdr_ptr->p_memsz; + offset = phdr_ptr->p_offset; + notes_section = kmalloc(max_sz, GFP_KERNEL); + if (!notes_section) + return -ENOMEM; + rc = elfcorehdr_read_notes(notes_section, max_sz, &offset); + if (rc < 0) { + kfree(notes_section); + return rc; + } + nhdr_ptr = notes_section; + while (nhdr_ptr->n_namesz != 0) { + sz = sizeof(Elf32_Nhdr) + + (((u64)nhdr_ptr->n_namesz + 3) & ~3) + + (((u64)nhdr_ptr->n_descsz + 3) & ~3); + if ((real_sz + sz) > max_sz) { + pr_warn("Warning: Exceeded p_memsz, dropping PT_NOTE entry n_namesz=0x%x, n_descsz=0x%x\n", + nhdr_ptr->n_namesz, nhdr_ptr->n_descsz); + break; + } + real_sz += sz; + nhdr_ptr = (Elf32_Nhdr*)((char*)nhdr_ptr + sz); + } + kfree(notes_section); + phdr_ptr->p_memsz = real_sz; + if (real_sz == 0) { + pr_warn("Warning: Zero PT_NOTE entries found\n"); + } + } + + return 0; +} + +/** + * get_note_number_and_size_elf32 - get the number of PT_NOTE program + * headers and sum of real size of their ELF note segment headers and + * data. + * + * @ehdr_ptr: ELF header + * @nr_ptnote: buffer for the number of PT_NOTE program headers + * @sz_ptnote: buffer for size of unique PT_NOTE program header + * + * This function is used to merge multiple PT_NOTE program headers + * into a unique single one. The resulting unique entry will have + * @sz_ptnote in its phdr->p_mem. + * + * It is assumed that program headers with PT_NOTE type pointed to by + * @ehdr_ptr has already been updated by update_note_header_size_elf32 + * and each of PT_NOTE program headers has actual ELF note segment + * size in its p_memsz member. + */ +static int __init get_note_number_and_size_elf32(const Elf32_Ehdr *ehdr_ptr, + int *nr_ptnote, u64 *sz_ptnote) +{ + int i; + Elf32_Phdr *phdr_ptr; + + *nr_ptnote = *sz_ptnote = 0; + + phdr_ptr = (Elf32_Phdr *)(ehdr_ptr + 1); + for (i = 0; i < ehdr_ptr->e_phnum; i++, phdr_ptr++) { + if (phdr_ptr->p_type != PT_NOTE) + continue; + *nr_ptnote += 1; + *sz_ptnote += phdr_ptr->p_memsz; + } + + return 0; +} + +/** + * copy_notes_elf32 - copy ELF note segments in a given buffer + * + * @ehdr_ptr: ELF header + * @notes_buf: buffer into which ELF note segments are copied + * + * This function is used to copy ELF note segment in the 1st kernel + * into the buffer @notes_buf in the 2nd kernel. It is assumed that + * size of the buffer @notes_buf is equal to or larger than sum of the + * real ELF note segment headers and data. + * + * It is assumed that program headers with PT_NOTE type pointed to by + * @ehdr_ptr has already been updated by update_note_header_size_elf32 + * and each of PT_NOTE program headers has actual ELF note segment + * size in its p_memsz member. + */ +static int __init copy_notes_elf32(const Elf32_Ehdr *ehdr_ptr, char *notes_buf) +{ + int i, rc=0; + Elf32_Phdr *phdr_ptr; + + phdr_ptr = (Elf32_Phdr*)(ehdr_ptr + 1); + + for (i = 0; i < ehdr_ptr->e_phnum; i++, phdr_ptr++) { + u64 offset; + if (phdr_ptr->p_type != PT_NOTE) + continue; + offset = phdr_ptr->p_offset; + rc = elfcorehdr_read_notes(notes_buf, phdr_ptr->p_memsz, + &offset); + if (rc < 0) + return rc; + notes_buf += phdr_ptr->p_memsz; + } + + return 0; +} + +/* Merges all the PT_NOTE headers into one. */ +static int __init merge_note_headers_elf32(char *elfptr, size_t *elfsz, + char **notes_buf, size_t *notes_sz) +{ + int i, nr_ptnote=0, rc=0; + char *tmp; + Elf32_Ehdr *ehdr_ptr; + Elf32_Phdr phdr; + u64 phdr_sz = 0, note_off; + + ehdr_ptr = (Elf32_Ehdr *)elfptr; + + rc = update_note_header_size_elf32(ehdr_ptr); + if (rc < 0) + return rc; + + rc = get_note_number_and_size_elf32(ehdr_ptr, &nr_ptnote, &phdr_sz); + if (rc < 0) + return rc; + + *notes_sz = roundup(phdr_sz, PAGE_SIZE); + *notes_buf = vmcore_alloc_buf(*notes_sz); + if (!*notes_buf) + return -ENOMEM; + + rc = copy_notes_elf32(ehdr_ptr, *notes_buf); + if (rc < 0) + return rc; + + /* Prepare merged PT_NOTE program header. */ + phdr.p_type = PT_NOTE; + phdr.p_flags = 0; + note_off = sizeof(Elf32_Ehdr) + + (ehdr_ptr->e_phnum - nr_ptnote +1) * sizeof(Elf32_Phdr); + phdr.p_offset = roundup(note_off, PAGE_SIZE); + phdr.p_vaddr = phdr.p_paddr = 0; + phdr.p_filesz = phdr.p_memsz = phdr_sz; + phdr.p_align = 4; + + /* Add merged PT_NOTE program header*/ + tmp = elfptr + sizeof(Elf32_Ehdr); + memcpy(tmp, &phdr, sizeof(phdr)); + tmp += sizeof(phdr); + + /* Remove unwanted PT_NOTE program headers. */ + i = (nr_ptnote - 1) * sizeof(Elf32_Phdr); + *elfsz = *elfsz - i; + memmove(tmp, tmp+i, ((*elfsz)-sizeof(Elf32_Ehdr)-sizeof(Elf32_Phdr))); + memset(elfptr + *elfsz, 0, i); + *elfsz = roundup(*elfsz, PAGE_SIZE); + + /* Modify e_phnum to reflect merged headers. */ + ehdr_ptr->e_phnum = ehdr_ptr->e_phnum - nr_ptnote + 1; + + /* Store the size of all notes. We need this to update the note + * header when the device dumps will be added. + */ + elfnotes_orig_sz = phdr.p_memsz; + + return 0; +} + +/* Add memory chunks represented by program headers to vmcore list. Also update + * the new offset fields of exported program headers. */ +static int __init process_ptload_program_headers_elf64(char *elfptr, + size_t elfsz, + size_t elfnotes_sz, + struct list_head *vc_list) +{ + int i; + Elf64_Ehdr *ehdr_ptr; + Elf64_Phdr *phdr_ptr; + loff_t vmcore_off; + + ehdr_ptr = (Elf64_Ehdr *)elfptr; + phdr_ptr = (Elf64_Phdr*)(elfptr + sizeof(Elf64_Ehdr)); /* PT_NOTE hdr */ + + /* Skip ELF header, program headers and ELF note segment. */ + vmcore_off = elfsz + elfnotes_sz; + + for (i = 0; i < ehdr_ptr->e_phnum; i++, phdr_ptr++) { + u64 paddr, start, end, size; + + if (phdr_ptr->p_type != PT_LOAD) + continue; + + paddr = phdr_ptr->p_offset; + start = rounddown(paddr, PAGE_SIZE); + end = roundup(paddr + phdr_ptr->p_memsz, PAGE_SIZE); + size = end - start; + + if (vmcore_alloc_add_range(vc_list, start, size)) + return -ENOMEM; + + /* Update the program header offset. */ + phdr_ptr->p_offset = vmcore_off + (paddr - start); + vmcore_off = vmcore_off + size; + } + return 0; +} + +static int __init process_ptload_program_headers_elf32(char *elfptr, + size_t elfsz, + size_t elfnotes_sz, + struct list_head *vc_list) +{ + int i; + Elf32_Ehdr *ehdr_ptr; + Elf32_Phdr *phdr_ptr; + loff_t vmcore_off; + + ehdr_ptr = (Elf32_Ehdr *)elfptr; + phdr_ptr = (Elf32_Phdr*)(elfptr + sizeof(Elf32_Ehdr)); /* PT_NOTE hdr */ + + /* Skip ELF header, program headers and ELF note segment. */ + vmcore_off = elfsz + elfnotes_sz; + + for (i = 0; i < ehdr_ptr->e_phnum; i++, phdr_ptr++) { + u64 paddr, start, end, size; + + if (phdr_ptr->p_type != PT_LOAD) + continue; + + paddr = phdr_ptr->p_offset; + start = rounddown(paddr, PAGE_SIZE); + end = roundup(paddr + phdr_ptr->p_memsz, PAGE_SIZE); + size = end - start; + + if (vmcore_alloc_add_range(vc_list, start, size)) + return -ENOMEM; + + /* Update the program header offset */ + phdr_ptr->p_offset = vmcore_off + (paddr - start); + vmcore_off = vmcore_off + size; + } + return 0; +} + +/* Sets offset fields of vmcore elements. */ +static void set_vmcore_list_offsets(size_t elfsz, size_t elfnotes_sz, + struct list_head *vc_list) +{ + struct vmcore_range *m; + loff_t vmcore_off; + + /* Skip ELF header, program headers and ELF note segment. */ + vmcore_off = elfsz + elfnotes_sz; + + list_for_each_entry(m, vc_list, list) { + m->offset = vmcore_off; + vmcore_off += m->size; + } +} + +static void free_elfcorebuf(void) +{ + free_pages((unsigned long)elfcorebuf, get_order(elfcorebuf_sz_orig)); + elfcorebuf = NULL; + vfree(elfnotes_buf); + elfnotes_buf = NULL; +} + +static int __init parse_crash_elf64_headers(void) +{ + int rc=0; + Elf64_Ehdr ehdr; + u64 addr; + + addr = elfcorehdr_addr; + + /* Read ELF header */ + rc = elfcorehdr_read((char *)&ehdr, sizeof(Elf64_Ehdr), &addr); + if (rc < 0) + return rc; + + /* Do some basic Verification. */ + if (memcmp(ehdr.e_ident, ELFMAG, SELFMAG) != 0 || + (ehdr.e_type != ET_CORE) || + !vmcore_elf64_check_arch(&ehdr) || + ehdr.e_ident[EI_CLASS] != ELFCLASS64 || + ehdr.e_ident[EI_VERSION] != EV_CURRENT || + ehdr.e_version != EV_CURRENT || + ehdr.e_ehsize != sizeof(Elf64_Ehdr) || + ehdr.e_phentsize != sizeof(Elf64_Phdr) || + ehdr.e_phnum == 0) { + pr_warn("Warning: Core image elf header is not sane\n"); + return -EINVAL; + } + + /* Read in all elf headers. */ + elfcorebuf_sz_orig = sizeof(Elf64_Ehdr) + + ehdr.e_phnum * sizeof(Elf64_Phdr); + elfcorebuf_sz = elfcorebuf_sz_orig; + elfcorebuf = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, + get_order(elfcorebuf_sz_orig)); + if (!elfcorebuf) + return -ENOMEM; + addr = elfcorehdr_addr; + rc = elfcorehdr_read(elfcorebuf, elfcorebuf_sz_orig, &addr); + if (rc < 0) + goto fail; + + /* Merge all PT_NOTE headers into one. */ + rc = merge_note_headers_elf64(elfcorebuf, &elfcorebuf_sz, + &elfnotes_buf, &elfnotes_sz); + if (rc) + goto fail; + rc = process_ptload_program_headers_elf64(elfcorebuf, elfcorebuf_sz, + elfnotes_sz, &vmcore_list); + if (rc) + goto fail; + set_vmcore_list_offsets(elfcorebuf_sz, elfnotes_sz, &vmcore_list); + return 0; +fail: + free_elfcorebuf(); + return rc; +} + +static int __init parse_crash_elf32_headers(void) +{ + int rc=0; + Elf32_Ehdr ehdr; + u64 addr; + + addr = elfcorehdr_addr; + + /* Read ELF header */ + rc = elfcorehdr_read((char *)&ehdr, sizeof(Elf32_Ehdr), &addr); + if (rc < 0) + return rc; + + /* Do some basic Verification. */ + if (memcmp(ehdr.e_ident, ELFMAG, SELFMAG) != 0 || + (ehdr.e_type != ET_CORE) || + !vmcore_elf32_check_arch(&ehdr) || + ehdr.e_ident[EI_CLASS] != ELFCLASS32|| + ehdr.e_ident[EI_VERSION] != EV_CURRENT || + ehdr.e_version != EV_CURRENT || + ehdr.e_ehsize != sizeof(Elf32_Ehdr) || + ehdr.e_phentsize != sizeof(Elf32_Phdr) || + ehdr.e_phnum == 0) { + pr_warn("Warning: Core image elf header is not sane\n"); + return -EINVAL; + } + + /* Read in all elf headers. */ + elfcorebuf_sz_orig = sizeof(Elf32_Ehdr) + ehdr.e_phnum * sizeof(Elf32_Phdr); + elfcorebuf_sz = elfcorebuf_sz_orig; + elfcorebuf = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, + get_order(elfcorebuf_sz_orig)); + if (!elfcorebuf) + return -ENOMEM; + addr = elfcorehdr_addr; + rc = elfcorehdr_read(elfcorebuf, elfcorebuf_sz_orig, &addr); + if (rc < 0) + goto fail; + + /* Merge all PT_NOTE headers into one. */ + rc = merge_note_headers_elf32(elfcorebuf, &elfcorebuf_sz, + &elfnotes_buf, &elfnotes_sz); + if (rc) + goto fail; + rc = process_ptload_program_headers_elf32(elfcorebuf, elfcorebuf_sz, + elfnotes_sz, &vmcore_list); + if (rc) + goto fail; + set_vmcore_list_offsets(elfcorebuf_sz, elfnotes_sz, &vmcore_list); + return 0; +fail: + free_elfcorebuf(); + return rc; +} + +static int __init parse_crash_elf_headers(void) +{ + unsigned char e_ident[EI_NIDENT]; + u64 addr; + int rc=0; + + addr = elfcorehdr_addr; + rc = elfcorehdr_read(e_ident, EI_NIDENT, &addr); + if (rc < 0) + return rc; + if (memcmp(e_ident, ELFMAG, SELFMAG) != 0) { + pr_warn("Warning: Core image elf header not found\n"); + return -EINVAL; + } + + if (e_ident[EI_CLASS] == ELFCLASS64) { + rc = parse_crash_elf64_headers(); + if (rc) + return rc; + } else if (e_ident[EI_CLASS] == ELFCLASS32) { + rc = parse_crash_elf32_headers(); + if (rc) + return rc; + } else { + pr_warn("Warning: Core image elf header is not sane\n"); + return -EINVAL; + } + + /* Determine vmcore size. */ + vmcore_size = get_vmcore_size(elfcorebuf_sz, elfnotes_sz, + &vmcore_list); + + return 0; +} + +#ifdef CONFIG_PROC_VMCORE_DEVICE_DUMP +/** + * vmcoredd_write_header - Write vmcore device dump header at the + * beginning of the dump's buffer. + * @buf: Output buffer where the note is written + * @data: Dump info + * @size: Size of the dump + * + * Fills beginning of the dump's buffer with vmcore device dump header. + */ +static void vmcoredd_write_header(void *buf, struct vmcoredd_data *data, + u32 size) +{ + struct vmcoredd_header *vdd_hdr = (struct vmcoredd_header *)buf; + + vdd_hdr->n_namesz = sizeof(vdd_hdr->name); + vdd_hdr->n_descsz = size + sizeof(vdd_hdr->dump_name); + vdd_hdr->n_type = NT_VMCOREDD; + + strscpy_pad(vdd_hdr->name, VMCOREDD_NOTE_NAME); + strscpy_pad(vdd_hdr->dump_name, data->dump_name); +} + +/** + * vmcoredd_update_program_headers - Update all ELF program headers + * @elfptr: Pointer to elf header + * @elfnotesz: Size of elf notes aligned to page size + * @vmcoreddsz: Size of device dumps to be added to elf note header + * + * Determine type of ELF header (Elf64 or Elf32) and update the elf note size. + * Also update the offsets of all the program headers after the elf note header. + */ +static void vmcoredd_update_program_headers(char *elfptr, size_t elfnotesz, + size_t vmcoreddsz) +{ + unsigned char *e_ident = (unsigned char *)elfptr; + u64 start, end, size; + loff_t vmcore_off; + u32 i; + + vmcore_off = elfcorebuf_sz + elfnotesz; + + if (e_ident[EI_CLASS] == ELFCLASS64) { + Elf64_Ehdr *ehdr = (Elf64_Ehdr *)elfptr; + Elf64_Phdr *phdr = (Elf64_Phdr *)(elfptr + sizeof(Elf64_Ehdr)); + + /* Update all program headers */ + for (i = 0; i < ehdr->e_phnum; i++, phdr++) { + if (phdr->p_type == PT_NOTE) { + /* Update note size */ + phdr->p_memsz = elfnotes_orig_sz + vmcoreddsz; + phdr->p_filesz = phdr->p_memsz; + continue; + } + + start = rounddown(phdr->p_offset, PAGE_SIZE); + end = roundup(phdr->p_offset + phdr->p_memsz, + PAGE_SIZE); + size = end - start; + phdr->p_offset = vmcore_off + (phdr->p_offset - start); + vmcore_off += size; + } + } else { + Elf32_Ehdr *ehdr = (Elf32_Ehdr *)elfptr; + Elf32_Phdr *phdr = (Elf32_Phdr *)(elfptr + sizeof(Elf32_Ehdr)); + + /* Update all program headers */ + for (i = 0; i < ehdr->e_phnum; i++, phdr++) { + if (phdr->p_type == PT_NOTE) { + /* Update note size */ + phdr->p_memsz = elfnotes_orig_sz + vmcoreddsz; + phdr->p_filesz = phdr->p_memsz; + continue; + } + + start = rounddown(phdr->p_offset, PAGE_SIZE); + end = roundup(phdr->p_offset + phdr->p_memsz, + PAGE_SIZE); + size = end - start; + phdr->p_offset = vmcore_off + (phdr->p_offset - start); + vmcore_off += size; + } + } +} + +/** + * vmcoredd_update_size - Update the total size of the device dumps and update + * ELF header + * @dump_size: Size of the current device dump to be added to total size + * + * Update the total size of all the device dumps and update the ELF program + * headers. Calculate the new offsets for the vmcore list and update the + * total vmcore size. + */ +static void vmcoredd_update_size(size_t dump_size) +{ + vmcoredd_orig_sz += dump_size; + elfnotes_sz = roundup(elfnotes_orig_sz, PAGE_SIZE) + vmcoredd_orig_sz; + vmcoredd_update_program_headers(elfcorebuf, elfnotes_sz, + vmcoredd_orig_sz); + + /* Update vmcore list offsets */ + set_vmcore_list_offsets(elfcorebuf_sz, elfnotes_sz, &vmcore_list); + + vmcore_size = get_vmcore_size(elfcorebuf_sz, elfnotes_sz, + &vmcore_list); + proc_vmcore->size = vmcore_size; +} + +/** + * vmcore_add_device_dump - Add a buffer containing device dump to vmcore + * @data: dump info. + * + * Allocate a buffer and invoke the calling driver's dump collect routine. + * Write ELF note at the beginning of the buffer to indicate vmcore device + * dump and add the dump to global list. + */ +int vmcore_add_device_dump(struct vmcoredd_data *data) +{ + struct vmcoredd_node *dump; + void *buf = NULL; + size_t data_size; + int ret; + + if (vmcoredd_disabled) { + pr_err_once("Device dump is disabled\n"); + return -EINVAL; + } + + if (!data || !strlen(data->dump_name) || + !data->vmcoredd_callback || !data->size) + return -EINVAL; + + dump = vzalloc(sizeof(*dump)); + if (!dump) + return -ENOMEM; + + /* Keep size of the buffer page aligned so that it can be mmaped */ + data_size = roundup(sizeof(struct vmcoredd_header) + data->size, + PAGE_SIZE); + + /* Allocate buffer for driver's to write their dumps */ + buf = vmcore_alloc_buf(data_size); + if (!buf) { + ret = -ENOMEM; + goto out_err; + } + + vmcoredd_write_header(buf, data, data_size - + sizeof(struct vmcoredd_header)); + + /* Invoke the driver's dump collection routing */ + ret = data->vmcoredd_callback(data, buf + + sizeof(struct vmcoredd_header)); + if (ret) + goto out_err; + + dump->buf = buf; + dump->size = data_size; + + /* Add the dump to driver sysfs list and update the elfcore hdr */ + scoped_guard(mutex, &vmcore_mutex) { + if (vmcore_opened) + pr_warn_once("Unexpected adding of device dump\n"); + if (vmcore_open) { + ret = -EBUSY; + goto out_err; + } + + list_add_tail(&dump->list, &vmcoredd_list); + vmcoredd_update_size(data_size); + } + return 0; + +out_err: + vfree(buf); + vfree(dump); + + return ret; +} +EXPORT_SYMBOL(vmcore_add_device_dump); +#endif /* CONFIG_PROC_VMCORE_DEVICE_DUMP */ + +#ifdef CONFIG_PROC_VMCORE_DEVICE_RAM +static int vmcore_realloc_elfcore_buffer_elf64(size_t new_size) +{ + char *elfcorebuf_new; + + if (WARN_ON_ONCE(new_size < elfcorebuf_sz)) + return -EINVAL; + if (get_order(elfcorebuf_sz_orig) == get_order(new_size)) { + elfcorebuf_sz_orig = new_size; + return 0; + } + + elfcorebuf_new = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, + get_order(new_size)); + if (!elfcorebuf_new) + return -ENOMEM; + memcpy(elfcorebuf_new, elfcorebuf, elfcorebuf_sz); + free_pages((unsigned long)elfcorebuf, get_order(elfcorebuf_sz_orig)); + elfcorebuf = elfcorebuf_new; + elfcorebuf_sz_orig = new_size; + return 0; +} + +static void vmcore_reset_offsets_elf64(void) +{ + Elf64_Phdr *phdr_start = (Elf64_Phdr *)(elfcorebuf + sizeof(Elf64_Ehdr)); + loff_t vmcore_off = elfcorebuf_sz + elfnotes_sz; + Elf64_Ehdr *ehdr = (Elf64_Ehdr *)elfcorebuf; + Elf64_Phdr *phdr; + int i; + + for (i = 0, phdr = phdr_start; i < ehdr->e_phnum; i++, phdr++) { + u64 start, end; + + /* + * After merge_note_headers_elf64() we should only have a single + * PT_NOTE entry that starts immediately after elfcorebuf_sz. + */ + if (phdr->p_type == PT_NOTE) { + phdr->p_offset = elfcorebuf_sz; + continue; + } + + start = rounddown(phdr->p_offset, PAGE_SIZE); + end = roundup(phdr->p_offset + phdr->p_memsz, PAGE_SIZE); + phdr->p_offset = vmcore_off + (phdr->p_offset - start); + vmcore_off = vmcore_off + end - start; + } + set_vmcore_list_offsets(elfcorebuf_sz, elfnotes_sz, &vmcore_list); +} + +static int vmcore_add_device_ram_elf64(struct list_head *list, size_t count) +{ + Elf64_Phdr *phdr_start = (Elf64_Phdr *)(elfcorebuf + sizeof(Elf64_Ehdr)); + Elf64_Ehdr *ehdr = (Elf64_Ehdr *)elfcorebuf; + struct vmcore_range *cur; + Elf64_Phdr *phdr; + size_t new_size; + int rc; + + if ((Elf32_Half)(ehdr->e_phnum + count) != ehdr->e_phnum + count) { + pr_err("too many device ram ranges\n"); + return -ENOSPC; + } + + /* elfcorebuf_sz must always cover full pages. */ + new_size = sizeof(Elf64_Ehdr) + + (ehdr->e_phnum + count) * sizeof(Elf64_Phdr); + new_size = roundup(new_size, PAGE_SIZE); + + /* + * Make sure we have sufficient space to include the new PT_LOAD + * entries. + */ + rc = vmcore_realloc_elfcore_buffer_elf64(new_size); + if (rc) { + pr_err("resizing elfcore failed\n"); + return rc; + } + + /* Modify our used elfcore buffer size to cover the new entries. */ + elfcorebuf_sz = new_size; + + /* Fill the added PT_LOAD entries. */ + phdr = phdr_start + ehdr->e_phnum; + list_for_each_entry(cur, list, list) { + WARN_ON_ONCE(!IS_ALIGNED(cur->paddr | cur->size, PAGE_SIZE)); + elfcorehdr_fill_device_ram_ptload_elf64(phdr, cur->paddr, cur->size); + + /* p_offset will be adjusted later. */ + phdr++; + ehdr->e_phnum++; + } + list_splice_tail(list, &vmcore_list); + + /* We changed elfcorebuf_sz and added new entries; reset all offsets. */ + vmcore_reset_offsets_elf64(); + + /* Finally, recalculate the total vmcore size. */ + vmcore_size = get_vmcore_size(elfcorebuf_sz, elfnotes_sz, + &vmcore_list); + proc_vmcore->size = vmcore_size; + return 0; +} + +static void vmcore_process_device_ram(struct vmcore_cb *cb) +{ + unsigned char *e_ident = (unsigned char *)elfcorebuf; + struct vmcore_range *first, *m; + LIST_HEAD(list); + int count; + + /* We only support Elf64 dumps for now. */ + if (WARN_ON_ONCE(e_ident[EI_CLASS] != ELFCLASS64)) { + pr_err("device ram ranges only support Elf64\n"); + return; + } + + if (cb->get_device_ram(cb, &list)) { + pr_err("obtaining device ram ranges failed\n"); + return; + } + count = list_count_nodes(&list); + if (!count) + return; + + /* + * For some reason these ranges are already know? Might happen + * with unusual register->unregister->register sequences; we'll simply + * sanity check using the first range. + */ + first = list_first_entry(&list, struct vmcore_range, list); + list_for_each_entry(m, &vmcore_list, list) { + unsigned long long m_end = m->paddr + m->size; + unsigned long long first_end = first->paddr + first->size; + + if (first->paddr < m_end && m->paddr < first_end) + goto out_free; + } + + /* If adding the mem nodes succeeds, they must not be freed. */ + if (!vmcore_add_device_ram_elf64(&list, count)) + return; +out_free: + vmcore_free_ranges(&list); +} +#else /* !CONFIG_PROC_VMCORE_DEVICE_RAM */ +static void vmcore_process_device_ram(struct vmcore_cb *cb) +{ +} +#endif /* CONFIG_PROC_VMCORE_DEVICE_RAM */ + +/* Free all dumps in vmcore device dump list */ +static void vmcore_free_device_dumps(void) +{ +#ifdef CONFIG_PROC_VMCORE_DEVICE_DUMP + mutex_lock(&vmcore_mutex); + while (!list_empty(&vmcoredd_list)) { + struct vmcoredd_node *dump; + + dump = list_first_entry(&vmcoredd_list, struct vmcoredd_node, + list); + list_del(&dump->list); + vfree(dump->buf); + vfree(dump); + } + mutex_unlock(&vmcore_mutex); +#endif /* CONFIG_PROC_VMCORE_DEVICE_DUMP */ +} + +/* Init function for vmcore module. */ +static int __init vmcore_init(void) +{ + int rc = 0; + + /* Allow architectures to allocate ELF header in 2nd kernel */ + rc = elfcorehdr_alloc(&elfcorehdr_addr, &elfcorehdr_size); + if (rc) + return rc; + /* + * If elfcorehdr= has been passed in cmdline or created in 2nd kernel, + * then capture the dump. + */ + if (!(is_vmcore_usable())) + return rc; + rc = parse_crash_elf_headers(); + if (rc) { + elfcorehdr_free(elfcorehdr_addr); + pr_warn("not initialized\n"); + return rc; + } + elfcorehdr_free(elfcorehdr_addr); + elfcorehdr_addr = ELFCORE_ADDR_ERR; + + proc_vmcore = proc_create("vmcore", S_IRUSR, NULL, &vmcore_proc_ops); + if (proc_vmcore) + proc_vmcore->size = vmcore_size; + return 0; +} +fs_initcall(vmcore_init); + +/* Cleanup function for vmcore module. */ +void vmcore_cleanup(void) +{ + if (proc_vmcore) { + proc_remove(proc_vmcore); + proc_vmcore = NULL; + } + + vmcore_free_ranges(&vmcore_list); + free_elfcorebuf(); + + /* clear vmcore device dump list */ + vmcore_free_device_dumps(); +} diff --git a/fs/proc_namespace.c b/fs/proc_namespace.c new file mode 100644 index 000000000..036356c0a --- /dev/null +++ b/fs/proc_namespace.c @@ -0,0 +1,337 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/proc_namespace.c - handling of /proc//{mounts,mountinfo,mountstats} + * + * In fact, that's a piece of procfs; it's *almost* isolated from + * the rest of fs/proc, but has rather close relationships with + * fs/namespace.c, thus here instead of fs/proc + * + */ +#include +#include +#include +#include +#include + +#include "proc/internal.h" /* only for get_proc_task() in ->open() */ + +#include "pnode.h" +#include "internal.h" + +static __poll_t mounts_poll(struct file *file, poll_table *wait) +{ + struct seq_file *m = file->private_data; + struct proc_mounts *p = m->private; + struct mnt_namespace *ns = p->ns; + __poll_t res = EPOLLIN | EPOLLRDNORM; + int event; + + poll_wait(file, &p->ns->poll, wait); + + event = READ_ONCE(ns->event); + if (m->poll_event != event) { + m->poll_event = event; + res |= EPOLLERR | EPOLLPRI; + } + + return res; +} + +struct proc_fs_opts { + int flag; + const char *str; +}; + +static int show_sb_opts(struct seq_file *m, struct super_block *sb) +{ + static const struct proc_fs_opts fs_opts[] = { + { SB_SYNCHRONOUS, ",sync" }, + { SB_DIRSYNC, ",dirsync" }, + { SB_MANDLOCK, ",mand" }, + { SB_LAZYTIME, ",lazytime" }, + { 0, NULL } + }; + const struct proc_fs_opts *fs_infop; + + for (fs_infop = fs_opts; fs_infop->flag; fs_infop++) { + if (sb->s_flags & fs_infop->flag) + seq_puts(m, fs_infop->str); + } + + return security_sb_show_options(m, sb); +} + +static void show_vfsmnt_opts(struct seq_file *m, struct vfsmount *mnt) +{ + static const struct proc_fs_opts mnt_opts[] = { + { MNT_NOSUID, ",nosuid" }, + { MNT_NODEV, ",nodev" }, + { MNT_NOEXEC, ",noexec" }, + { MNT_NOATIME, ",noatime" }, + { MNT_NODIRATIME, ",nodiratime" }, + { MNT_RELATIME, ",relatime" }, + { MNT_NOSYMFOLLOW, ",nosymfollow" }, + { 0, NULL } + }; + const struct proc_fs_opts *fs_infop; + + for (fs_infop = mnt_opts; fs_infop->flag; fs_infop++) { + if (mnt->mnt_flags & fs_infop->flag) + seq_puts(m, fs_infop->str); + } + + if (is_idmapped_mnt(mnt)) + seq_puts(m, ",idmapped"); +} + +static inline void mangle(struct seq_file *m, const char *s) +{ + seq_escape(m, s, " \t\n\\#"); +} + +static void show_type(struct seq_file *m, struct super_block *sb) +{ + mangle(m, sb->s_type->name); + if (sb->s_subtype) { + seq_putc(m, '.'); + mangle(m, sb->s_subtype); + } +} + +static int show_vfsmnt(struct seq_file *m, struct vfsmount *mnt) +{ + struct proc_mounts *p = m->private; + struct mount *r = real_mount(mnt); + struct path mnt_path = { .dentry = mnt->mnt_root, .mnt = mnt }; + struct super_block *sb = mnt_path.dentry->d_sb; + int err; + + if (sb->s_op->show_devname) { + err = sb->s_op->show_devname(m, mnt_path.dentry); + if (err) + goto out; + } else { + mangle(m, r->mnt_devname); + } + seq_putc(m, ' '); + /* mountpoints outside of chroot jail will give SEQ_SKIP on this */ + err = seq_path_root(m, &mnt_path, &p->root, " \t\n\\"); + if (err) + goto out; + seq_putc(m, ' '); + show_type(m, sb); + seq_puts(m, __mnt_is_readonly(mnt) ? " ro" : " rw"); + err = show_sb_opts(m, sb); + if (err) + goto out; + show_vfsmnt_opts(m, mnt); + if (sb->s_op->show_options) + err = sb->s_op->show_options(m, mnt_path.dentry); + seq_puts(m, " 0 0\n"); +out: + return err; +} + +static int show_mountinfo(struct seq_file *m, struct vfsmount *mnt) +{ + struct proc_mounts *p = m->private; + struct mount *r = real_mount(mnt); + struct super_block *sb = mnt->mnt_sb; + struct path mnt_path = { .dentry = mnt->mnt_root, .mnt = mnt }; + int err; + + seq_printf(m, "%i %i %u:%u ", r->mnt_id, r->mnt_parent->mnt_id, + MAJOR(sb->s_dev), MINOR(sb->s_dev)); + err = show_path(m, mnt->mnt_root); + if (err) + goto out; + seq_putc(m, ' '); + + /* mountpoints outside of chroot jail will give SEQ_SKIP on this */ + err = seq_path_root(m, &mnt_path, &p->root, " \t\n\\"); + if (err) + goto out; + + seq_puts(m, mnt->mnt_flags & MNT_READONLY ? " ro" : " rw"); + show_vfsmnt_opts(m, mnt); + + /* Tagged fields ("foo:X" or "bar") */ + if (IS_MNT_SHARED(r)) + seq_printf(m, " shared:%i", r->mnt_group_id); + if (IS_MNT_SLAVE(r)) { + int master = r->mnt_master->mnt_group_id; + int dom = get_dominating_id(r, &p->root); + seq_printf(m, " master:%i", master); + if (dom && dom != master) + seq_printf(m, " propagate_from:%i", dom); + } + if (IS_MNT_UNBINDABLE(r)) + seq_puts(m, " unbindable"); + + /* Filesystem specific data */ + seq_puts(m, " - "); + show_type(m, sb); + seq_putc(m, ' '); + if (sb->s_op->show_devname) { + err = sb->s_op->show_devname(m, mnt->mnt_root); + if (err) + goto out; + } else { + mangle(m, r->mnt_devname); + } + seq_puts(m, sb_rdonly(sb) ? " ro" : " rw"); + err = show_sb_opts(m, sb); + if (err) + goto out; + if (sb->s_op->show_options) + err = sb->s_op->show_options(m, mnt->mnt_root); + seq_putc(m, '\n'); +out: + return err; +} + +static int show_vfsstat(struct seq_file *m, struct vfsmount *mnt) +{ + struct proc_mounts *p = m->private; + struct mount *r = real_mount(mnt); + struct path mnt_path = { .dentry = mnt->mnt_root, .mnt = mnt }; + struct super_block *sb = mnt_path.dentry->d_sb; + int err; + + /* device */ + seq_puts(m, "device "); + if (sb->s_op->show_devname) { + err = sb->s_op->show_devname(m, mnt_path.dentry); + if (err) + goto out; + } else { + mangle(m, r->mnt_devname); + } + + /* mount point */ + seq_puts(m, " mounted on "); + /* mountpoints outside of chroot jail will give SEQ_SKIP on this */ + err = seq_path_root(m, &mnt_path, &p->root, " \t\n\\"); + if (err) + goto out; + seq_putc(m, ' '); + + /* file system type */ + seq_puts(m, "with fstype "); + show_type(m, sb); + + /* optional statistics */ + if (sb->s_op->show_stats) { + seq_putc(m, ' '); + err = sb->s_op->show_stats(m, mnt_path.dentry); + } + + seq_putc(m, '\n'); +out: + return err; +} + +static int mounts_open_common(struct inode *inode, struct file *file, + int (*show)(struct seq_file *, struct vfsmount *)) +{ + struct task_struct *task = get_proc_task(inode); + struct nsproxy *nsp; + struct mnt_namespace *ns = NULL; + struct path root; + struct proc_mounts *p; + struct seq_file *m; + int ret = -EINVAL; + + if (!task) + goto err; + + task_lock(task); + nsp = task->nsproxy; + if (!nsp || !nsp->mnt_ns) { + task_unlock(task); + put_task_struct(task); + goto err; + } + ns = nsp->mnt_ns; + get_mnt_ns(ns); + if (!task->real_fs) { + task_unlock(task); + put_task_struct(task); + ret = -ENOENT; + goto err_put_ns; + } + get_fs_root(task->real_fs, &root); + task_unlock(task); + put_task_struct(task); + + ret = seq_open_private(file, &mounts_op, sizeof(struct proc_mounts)); + if (ret) + goto err_put_path; + + m = file->private_data; + m->poll_event = ns->event; + + p = m->private; + p->ns = ns; + p->root = root; + p->show = show; + + return 0; + + err_put_path: + path_put(&root); + err_put_ns: + put_mnt_ns(ns); + err: + return ret; +} + +static int mounts_release(struct inode *inode, struct file *file) +{ + struct seq_file *m = file->private_data; + struct proc_mounts *p = m->private; + path_put(&p->root); + put_mnt_ns(p->ns); + return seq_release_private(inode, file); +} + +static int mounts_open(struct inode *inode, struct file *file) +{ + return mounts_open_common(inode, file, show_vfsmnt); +} + +static int mountinfo_open(struct inode *inode, struct file *file) +{ + return mounts_open_common(inode, file, show_mountinfo); +} + +static int mountstats_open(struct inode *inode, struct file *file) +{ + return mounts_open_common(inode, file, show_vfsstat); +} + +const struct file_operations proc_mounts_operations = { + .open = mounts_open, + .read_iter = seq_read_iter, + .splice_read = copy_splice_read, + .llseek = seq_lseek, + .release = mounts_release, + .poll = mounts_poll, +}; + +const struct file_operations proc_mountinfo_operations = { + .open = mountinfo_open, + .read_iter = seq_read_iter, + .splice_read = copy_splice_read, + .llseek = seq_lseek, + .release = mounts_release, + .poll = mounts_poll, +}; + +const struct file_operations proc_mountstats_operations = { + .open = mountstats_open, + .read_iter = seq_read_iter, + .splice_read = copy_splice_read, + .llseek = seq_lseek, + .release = mounts_release, +}; diff --git a/fs/pstore/Kconfig b/fs/pstore/Kconfig new file mode 100644 index 000000000..3acc38600 --- /dev/null +++ b/fs/pstore/Kconfig @@ -0,0 +1,191 @@ +# SPDX-License-Identifier: GPL-2.0-only +config PSTORE + tristate "Persistent store support" + default n + help + This option enables generic access to platform level + persistent storage via "pstore" filesystem that can + be mounted as /dev/pstore. Only useful if you have + a platform level driver that registers with pstore to + provide the data, so you probably should just go say "Y" + (or "M") to a platform specific persistent store driver + (e.g. ACPI_APEI on X86) which will select this for you. + If you don't have a platform persistent store driver, + say N. + +config PSTORE_DEFAULT_KMSG_BYTES + int "Default kernel log storage space" if EXPERT + depends on PSTORE + default "10240" + help + Defines default size of pstore kernel log storage. + Can be enlarged if needed, not recommended to shrink it. + +config PSTORE_COMPRESS + bool "Pstore compression (deflate)" + depends on PSTORE + select ZLIB_INFLATE + select ZLIB_DEFLATE + default y + help + Whether pstore records should be compressed before being written to + the backing store. This is implemented using the zlib 'deflate' + algorithm, using the library implementation instead of using the full + blown crypto API. This reduces the risk of secondary oopses or other + problems while pstore is recording panic metadata. + +config PSTORE_CONSOLE + bool "Log kernel console messages" + depends on PSTORE + help + When the option is enabled, pstore will log all kernel + messages, even if no oops or panic happened. + +config PSTORE_PMSG + bool "Log user space messages" + depends on PSTORE + select RT_MUTEXES + help + When the option is enabled, pstore will export a character + interface /dev/pmsg0 to log user space messages. On reboot + data can be retrieved from /sys/fs/pstore/pmsg-ramoops-[ID]. + + If unsure, say N. + +config PSTORE_FTRACE + bool "Persistent function tracer" + depends on PSTORE + depends on FUNCTION_TRACER + depends on DEBUG_FS + help + With this option kernel traces function calls into a persistent + ram buffer that can be decoded and dumped after reboot through + pstore filesystem. It can be used to determine what function + was last called before a reset or panic. + + If unsure, say N. + +config PSTORE_RAM + tristate "Log panic/oops to a RAM buffer" + depends on PSTORE + depends on HAS_IOMEM + select REED_SOLOMON + select REED_SOLOMON_ENC8 + select REED_SOLOMON_DEC8 + help + This enables panic and oops messages to be logged to a circular + buffer in RAM where it can be read back at some later point. + + Note that for historical reasons, the module will be named + "ramoops.ko". + + For more information, see Documentation/admin-guide/ramoops.rst. + +config PSTORE_ZONE + tristate + depends on PSTORE + help + The common layer for pstore/blk (and pstore/ram in the future) + to manage storage in zones. + +config PSTORE_BLK + tristate "Log panic/oops to a block device" + depends on PSTORE + depends on BLOCK + select PSTORE_ZONE + default n + help + This enables panic and oops message to be logged to a block dev + where it can be read back at some later point. + + For more information, see Documentation/admin-guide/pstore-blk.rst + + If unsure, say N. + +config PSTORE_BLK_BLKDEV + string "block device identifier" + depends on PSTORE_BLK + default "" + help + Which block device should be used for pstore/blk. + + It accepts the following variants: + 1) device number in hexadecimal representation, + with no leading 0x, for example b302. + 2) /dev/ represents the device name of disk + 3) /dev/ represents the device name and number + of partition - device number of disk plus the partition number + 4) /dev/p - same as the above, this form is + used when disk name of partitioned disk ends with a digit. + 5) PARTUUID=00112233-4455-6677-8899-AABBCCDDEEFF representing the + unique id of a partition if the partition table provides it. + The UUID may be either an EFI/GPT UUID, or refer to an MSDOS + partition using the format SSSSSSSS-PP, where SSSSSSSS is a zero- + filled hex representation of the 32-bit "NT disk signature", and PP + is a zero-filled hex representation of the 1-based partition number. + 6) PARTUUID=/PARTNROFF= to select a partition in relation + to a partition with a known unique id. + 7) : major and minor number of the device separated by + a colon. + + NOTE that, both Kconfig and module parameters can configure + pstore/blk, but module parameters have priority over Kconfig. + +config PSTORE_BLK_KMSG_SIZE + int "Size in Kbytes of kmsg dump log to store" + depends on PSTORE_BLK + default 64 + help + This just sets size of kmsg dump (oops, panic, etc) log for + pstore/blk. The size is in KB and must be a multiple of 4. + + NOTE that, both Kconfig and module parameters can configure + pstore/blk, but module parameters have priority over Kconfig. + +config PSTORE_BLK_MAX_REASON + int "Maximum kmsg dump reason to store" + depends on PSTORE_BLK + default 2 + help + The maximum reason for kmsg dumps to store. The default is + 2 (KMSG_DUMP_OOPS), see include/linux/kmsg_dump.h's + enum kmsg_dump_reason for more details. + + NOTE that, both Kconfig and module parameters can configure + pstore/blk, but module parameters have priority over Kconfig. + +config PSTORE_BLK_PMSG_SIZE + int "Size in Kbytes of pmsg to store" + depends on PSTORE_BLK + depends on PSTORE_PMSG + default 64 + help + This just sets size of pmsg (pmsg_size) for pstore/blk. The size is + in KB and must be a multiple of 4. + + NOTE that, both Kconfig and module parameters can configure + pstore/blk, but module parameters have priority over Kconfig. + +config PSTORE_BLK_CONSOLE_SIZE + int "Size in Kbytes of console log to store" + depends on PSTORE_BLK + depends on PSTORE_CONSOLE + default 64 + help + This just sets size of console log (console_size) to store via + pstore/blk. The size is in KB and must be a multiple of 4. + + NOTE that, both Kconfig and module parameters can configure + pstore/blk, but module parameters have priority over Kconfig. + +config PSTORE_BLK_FTRACE_SIZE + int "Size in Kbytes of ftrace log to store" + depends on PSTORE_BLK + depends on PSTORE_FTRACE + default 64 + help + This just sets size of ftrace log (ftrace_size) for pstore/blk. The + size is in KB and must be a multiple of 4. + + NOTE that, both Kconfig and module parameters can configure + pstore/blk, but module parameters have priority over Kconfig. diff --git a/fs/pstore/Makefile b/fs/pstore/Makefile new file mode 100644 index 000000000..c270467ae --- /dev/null +++ b/fs/pstore/Makefile @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the linux pstorefs routines. +# + +obj-$(CONFIG_PSTORE) += pstore.o + +pstore-objs += inode.o platform.o +pstore-$(CONFIG_PSTORE_FTRACE) += ftrace.o + +pstore-$(CONFIG_PSTORE_PMSG) += pmsg.o + +ramoops-objs += ram.o ram_core.o +obj-$(CONFIG_PSTORE_RAM) += ramoops.o + +pstore_zone-objs += zone.o +obj-$(CONFIG_PSTORE_ZONE) += pstore_zone.o + +pstore_blk-objs += blk.o +obj-$(CONFIG_PSTORE_BLK) += pstore_blk.o diff --git a/fs/pstore/blk.c b/fs/pstore/blk.c new file mode 100644 index 000000000..12070b6a7 --- /dev/null +++ b/fs/pstore/blk.c @@ -0,0 +1,361 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Implements pstore backend driver that write to block (or non-block) storage + * devices, using the pstore/zone API. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static long kmsg_size = CONFIG_PSTORE_BLK_KMSG_SIZE; +module_param(kmsg_size, long, 0400); +MODULE_PARM_DESC(kmsg_size, "kmsg dump record size in kbytes"); + +static int max_reason = CONFIG_PSTORE_BLK_MAX_REASON; +module_param(max_reason, int, 0400); +MODULE_PARM_DESC(max_reason, + "maximum reason for kmsg dump (default 2: Oops and Panic)"); + +#if IS_ENABLED(CONFIG_PSTORE_PMSG) +static long pmsg_size = CONFIG_PSTORE_BLK_PMSG_SIZE; +#else +static long pmsg_size = -1; +#endif +module_param(pmsg_size, long, 0400); +MODULE_PARM_DESC(pmsg_size, "pmsg size in kbytes"); + +#if IS_ENABLED(CONFIG_PSTORE_CONSOLE) +static long console_size = CONFIG_PSTORE_BLK_CONSOLE_SIZE; +#else +static long console_size = -1; +#endif +module_param(console_size, long, 0400); +MODULE_PARM_DESC(console_size, "console size in kbytes"); + +#if IS_ENABLED(CONFIG_PSTORE_FTRACE) +static long ftrace_size = CONFIG_PSTORE_BLK_FTRACE_SIZE; +#else +static long ftrace_size = -1; +#endif +module_param(ftrace_size, long, 0400); +MODULE_PARM_DESC(ftrace_size, "ftrace size in kbytes"); + +static bool best_effort; +module_param(best_effort, bool, 0400); +MODULE_PARM_DESC(best_effort, "use best effort to write (i.e. do not require storage driver pstore support, default: off)"); + +/* + * blkdev - the block device to use for pstore storage + * See Documentation/admin-guide/pstore-blk.rst for details. + */ +static char blkdev[80] = CONFIG_PSTORE_BLK_BLKDEV; +module_param_string(blkdev, blkdev, 80, 0400); +MODULE_PARM_DESC(blkdev, "block device for pstore storage"); + +/* + * All globals must only be accessed under the pstore_blk_lock + * during the register/unregister functions. + */ +static DEFINE_MUTEX(pstore_blk_lock); +static struct file *psblk_file; +static struct pstore_device_info *pstore_device_info; + +#define check_size(name, alignsize) ({ \ + long _##name_ = (name); \ + _##name_ = _##name_ <= 0 ? 0 : (_##name_ * 1024); \ + if (_##name_ & ((alignsize) - 1)) { \ + pr_info(#name " must align to %d\n", \ + (alignsize)); \ + _##name_ = ALIGN(name, (alignsize)); \ + } \ + _##name_; \ +}) + +#define verify_size(name, alignsize, enabled) { \ + long _##name_; \ + if (enabled) \ + _##name_ = check_size(name, alignsize); \ + else \ + _##name_ = 0; \ + /* Synchronize module parameters with results. */ \ + name = _##name_ / 1024; \ + dev->zone.name = _##name_; \ +} + +static int __register_pstore_device(struct pstore_device_info *dev) +{ + int ret; + + lockdep_assert_held(&pstore_blk_lock); + + if (!dev) { + pr_err("NULL device info\n"); + return -EINVAL; + } + if (!dev->zone.total_size) { + pr_err("zero sized device\n"); + return -EINVAL; + } + if (!dev->zone.read) { + pr_err("no read handler for device\n"); + return -EINVAL; + } + if (!dev->zone.write) { + pr_err("no write handler for device\n"); + return -EINVAL; + } + + /* someone already registered before */ + if (pstore_device_info) + return -EBUSY; + + /* zero means no limit on which backends attempt to store. */ + if (!dev->flags) + dev->flags = UINT_MAX; + + /* Copy in module parameters. */ + verify_size(kmsg_size, 4096, dev->flags & PSTORE_FLAGS_DMESG); + verify_size(pmsg_size, 4096, dev->flags & PSTORE_FLAGS_PMSG); + verify_size(console_size, 4096, dev->flags & PSTORE_FLAGS_CONSOLE); + verify_size(ftrace_size, 4096, dev->flags & PSTORE_FLAGS_FTRACE); + dev->zone.max_reason = max_reason; + + /* Initialize required zone ownership details. */ + dev->zone.name = KBUILD_MODNAME; + dev->zone.owner = THIS_MODULE; + + ret = register_pstore_zone(&dev->zone); + if (ret == 0) + pstore_device_info = dev; + + return ret; +} +/** + * register_pstore_device() - register non-block device to pstore/blk + * + * @dev: non-block device information + * + * Return: + * * 0 - OK + * * Others - something error. + */ +int register_pstore_device(struct pstore_device_info *dev) +{ + int ret; + + mutex_lock(&pstore_blk_lock); + ret = __register_pstore_device(dev); + mutex_unlock(&pstore_blk_lock); + + return ret; +} +EXPORT_SYMBOL_GPL(register_pstore_device); + +static void __unregister_pstore_device(struct pstore_device_info *dev) +{ + lockdep_assert_held(&pstore_blk_lock); + if (pstore_device_info && pstore_device_info == dev) { + unregister_pstore_zone(&dev->zone); + pstore_device_info = NULL; + } +} + +/** + * unregister_pstore_device() - unregister non-block device from pstore/blk + * + * @dev: non-block device information + */ +void unregister_pstore_device(struct pstore_device_info *dev) +{ + mutex_lock(&pstore_blk_lock); + __unregister_pstore_device(dev); + mutex_unlock(&pstore_blk_lock); +} +EXPORT_SYMBOL_GPL(unregister_pstore_device); + +static ssize_t psblk_generic_blk_read(char *buf, size_t bytes, loff_t pos) +{ + return kernel_read(psblk_file, buf, bytes, &pos); +} + +static ssize_t psblk_generic_blk_write(const char *buf, size_t bytes, + loff_t pos) +{ + /* Console/Ftrace backend may handle buffer until flush dirty zones */ + if (in_interrupt() || irqs_disabled()) + return -EBUSY; + return kernel_write(psblk_file, buf, bytes, &pos); +} + +/* + * This takes its configuration only from the module parameters now. + */ +static int __register_pstore_blk(struct pstore_device_info *dev, + const char *devpath) +{ + int ret = -ENODEV; + + lockdep_assert_held(&pstore_blk_lock); + + psblk_file = filp_open(devpath, O_RDWR | O_DSYNC | O_NOATIME | O_EXCL, 0); + if (IS_ERR(psblk_file)) { + ret = PTR_ERR(psblk_file); + pr_err("failed to open '%s': %d!\n", devpath, ret); + goto err; + } + + if (!S_ISBLK(file_inode(psblk_file)->i_mode)) { + pr_err("'%s' is not block device!\n", devpath); + goto err_fput; + } + + dev->zone.total_size = + bdev_nr_bytes(I_BDEV(psblk_file->f_mapping->host)); + + ret = __register_pstore_device(dev); + if (ret) + goto err_fput; + + return 0; + +err_fput: + fput(psblk_file); +err: + psblk_file = NULL; + + return ret; +} + +/* get information of pstore/blk */ +int pstore_blk_get_config(struct pstore_blk_config *info) +{ + strscpy(info->device, blkdev); + info->max_reason = max_reason; + info->kmsg_size = check_size(kmsg_size, 4096); + info->pmsg_size = check_size(pmsg_size, 4096); + info->ftrace_size = check_size(ftrace_size, 4096); + info->console_size = check_size(console_size, 4096); + + return 0; +} +EXPORT_SYMBOL_GPL(pstore_blk_get_config); + + +#ifndef MODULE +static const char devname[] = "/dev/pstore-blk"; +static __init const char *early_boot_devpath(const char *initial_devname) +{ + /* + * During early boot the real root file system hasn't been + * mounted yet, and no device nodes are present yet. Use the + * same scheme to find the device that we use for mounting + * the root file system. + */ + dev_t dev; + + if (early_lookup_bdev(initial_devname, &dev)) { + pr_err("failed to resolve '%s'!\n", initial_devname); + return initial_devname; + } + + init_unlink(devname); + init_mknod(devname, S_IFBLK | 0600, new_encode_dev(dev)); + + return devname; +} +#else +static inline const char *early_boot_devpath(const char *initial_devname) +{ + return initial_devname; +} +#endif + +static int __init __best_effort_init(void) +{ + struct pstore_device_info *best_effort_dev; + int ret; + + /* No best-effort mode requested. */ + if (!best_effort) + return 0; + + /* Reject an empty blkdev. */ + if (!blkdev[0]) { + pr_err("blkdev empty with best_effort=Y\n"); + return -EINVAL; + } + + best_effort_dev = kzalloc_obj(*best_effort_dev); + if (!best_effort_dev) + return -ENOMEM; + + best_effort_dev->zone.read = psblk_generic_blk_read; + best_effort_dev->zone.write = psblk_generic_blk_write; + + ret = __register_pstore_blk(best_effort_dev, + early_boot_devpath(blkdev)); + if (ret) + kfree(best_effort_dev); + else + pr_info("attached %s (%lu) (no dedicated panic_write!)\n", + blkdev, best_effort_dev->zone.total_size); + + return ret; +} + +static void __exit __best_effort_exit(void) +{ + /* + * Currently, the only user of psblk_file is best_effort, so + * we can assume that pstore_device_info is associated with it. + * Once there are "real" blk devices, there will need to be a + * dedicated pstore_blk_info, etc. + */ + if (psblk_file) { + struct pstore_device_info *dev = pstore_device_info; + + __unregister_pstore_device(dev); + kfree(dev); + fput(psblk_file); + psblk_file = NULL; + } +} + +static int __init pstore_blk_init(void) +{ + int ret; + + mutex_lock(&pstore_blk_lock); + ret = __best_effort_init(); + mutex_unlock(&pstore_blk_lock); + + return ret; +} +late_initcall(pstore_blk_init); + +static void __exit pstore_blk_exit(void) +{ + mutex_lock(&pstore_blk_lock); + __best_effort_exit(); + /* If we've been asked to unload, unregister any remaining device. */ + __unregister_pstore_device(pstore_device_info); + mutex_unlock(&pstore_blk_lock); +} +module_exit(pstore_blk_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("WeiXiong Liao "); +MODULE_AUTHOR("Kees Cook "); +MODULE_DESCRIPTION("pstore backend for block devices"); diff --git a/fs/pstore/ftrace.c b/fs/pstore/ftrace.c new file mode 100644 index 000000000..f937c6a33 --- /dev/null +++ b/fs/pstore/ftrace.c @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2012 Google, Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +/* This doesn't need to be atomic: speed is chosen over correctness here. */ +static u64 pstore_ftrace_stamp; + +static inline unsigned long adjust_ip(unsigned long ip) +{ +#if defined(CONFIG_RANDOMIZE_BASE) && !defined(PSTORE_CPU_IN_IP) && IS_BUILTIN(CONFIG_PSTORE) + if (core_kernel_text(ip)) + return ip - kaslr_offset(); + + __clear_bit(BITS_PER_LONG - 1, &ip); +#endif + return ip; +} + +inline unsigned long decode_ip(unsigned long ip) +{ +#if defined(CONFIG_RANDOMIZE_BASE) && !defined(PSTORE_CPU_IN_IP) && IS_BUILTIN(CONFIG_PSTORE) + if (test_bit(BITS_PER_LONG - 1, &ip)) + return ip + kaslr_offset(); + + __set_bit(BITS_PER_LONG - 1, &ip); + +#endif + return ip; +} + +static void notrace pstore_ftrace_call(unsigned long ip, + unsigned long parent_ip, + struct ftrace_ops *op, + struct ftrace_regs *fregs) +{ + int bit; + unsigned long flags; + struct pstore_ftrace_record rec = {}; + struct pstore_record record = { + .type = PSTORE_TYPE_FTRACE, + .buf = (char *)&rec, + .size = sizeof(rec), + .psi = psinfo, + }; + + if (unlikely(oops_in_progress)) + return; + + bit = ftrace_test_recursion_trylock(ip, parent_ip); + if (bit < 0) + return; + + local_irq_save(flags); + + rec.ip = adjust_ip(ip); + rec.parent_ip = adjust_ip(parent_ip); + pstore_ftrace_write_timestamp(&rec, pstore_ftrace_stamp++); + pstore_ftrace_encode_cpu(&rec, raw_smp_processor_id()); + psinfo->write(&record); + + local_irq_restore(flags); + ftrace_test_recursion_unlock(bit); +} + +static struct ftrace_ops pstore_ftrace_ops __read_mostly = { + .func = pstore_ftrace_call, +}; + +static DEFINE_MUTEX(pstore_ftrace_lock); +static bool record_ftrace; +module_param(record_ftrace, bool, 0400); +MODULE_PARM_DESC(record_ftrace, + "enable ftrace recording immediately (default: off)"); + +static int pstore_set_ftrace_enabled(bool on) +{ + ssize_t ret; + + if (on == record_ftrace) + return 0; + + if (on) { + ftrace_ops_set_global_filter(&pstore_ftrace_ops); + ret = register_ftrace_function(&pstore_ftrace_ops); + } else { + ret = unregister_ftrace_function(&pstore_ftrace_ops); + } + + if (ret) { + pr_err("%s: unable to %sregister ftrace ops: %zd\n", + __func__, on ? "" : "un", ret); + } else { + record_ftrace = on; + } + + return ret; +} + +static ssize_t pstore_ftrace_knob_write(struct file *f, const char __user *buf, + size_t count, loff_t *ppos) +{ + u8 on; + ssize_t ret; + + ret = kstrtou8_from_user(buf, count, 2, &on); + if (ret) + return ret; + + mutex_lock(&pstore_ftrace_lock); + ret = pstore_set_ftrace_enabled(on); + mutex_unlock(&pstore_ftrace_lock); + + if (ret == 0) + ret = count; + + return ret; +} + +static ssize_t pstore_ftrace_knob_read(struct file *f, char __user *buf, + size_t count, loff_t *ppos) +{ + char val[] = { '0' + record_ftrace, '\n' }; + + return simple_read_from_buffer(buf, count, ppos, val, sizeof(val)); +} + +static const struct file_operations pstore_knob_fops = { + .open = simple_open, + .read = pstore_ftrace_knob_read, + .write = pstore_ftrace_knob_write, +}; + +static struct dentry *pstore_ftrace_dir; + +void pstore_register_ftrace(void) +{ + if (!psinfo->write) + return; + + pstore_ftrace_dir = debugfs_create_dir("pstore", NULL); + + pstore_set_ftrace_enabled(record_ftrace); + + debugfs_create_file("record_ftrace", 0600, pstore_ftrace_dir, NULL, + &pstore_knob_fops); +} + +void pstore_unregister_ftrace(void) +{ + mutex_lock(&pstore_ftrace_lock); + if (record_ftrace) { + unregister_ftrace_function(&pstore_ftrace_ops); + record_ftrace = false; + } + mutex_unlock(&pstore_ftrace_lock); + + debugfs_remove_recursive(pstore_ftrace_dir); +} + +ssize_t pstore_ftrace_combine_log(char **dest_log, size_t *dest_log_size, + const char *src_log, size_t src_log_size) +{ + size_t dest_size, src_size, total, dest_off, src_off; + size_t dest_idx = 0, src_idx = 0, merged_idx = 0; + void *merged_buf; + struct pstore_ftrace_record *drec, *srec, *mrec; + size_t record_size = sizeof(struct pstore_ftrace_record); + + dest_off = *dest_log_size % record_size; + dest_size = *dest_log_size - dest_off; + + src_off = src_log_size % record_size; + src_size = src_log_size - src_off; + + total = dest_size + src_size; + merged_buf = kmalloc(total, GFP_KERNEL); + if (!merged_buf) + return -ENOMEM; + + drec = (struct pstore_ftrace_record *)(*dest_log + dest_off); + srec = (struct pstore_ftrace_record *)(src_log + src_off); + mrec = (struct pstore_ftrace_record *)(merged_buf); + + while (dest_size > 0 && src_size > 0) { + if (pstore_ftrace_read_timestamp(&drec[dest_idx]) < + pstore_ftrace_read_timestamp(&srec[src_idx])) { + mrec[merged_idx++] = drec[dest_idx++]; + dest_size -= record_size; + } else { + mrec[merged_idx++] = srec[src_idx++]; + src_size -= record_size; + } + } + + while (dest_size > 0) { + mrec[merged_idx++] = drec[dest_idx++]; + dest_size -= record_size; + } + + while (src_size > 0) { + mrec[merged_idx++] = srec[src_idx++]; + src_size -= record_size; + } + + kfree(*dest_log); + *dest_log = merged_buf; + *dest_log_size = total; + + return 0; +} +EXPORT_SYMBOL_GPL(pstore_ftrace_combine_log); diff --git a/fs/pstore/inode.c b/fs/pstore/inode.c new file mode 100644 index 000000000..cf499ba72 --- /dev/null +++ b/fs/pstore/inode.c @@ -0,0 +1,528 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Persistent Storage - ramfs parts. + * + * Copyright (C) 2010 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +#define PSTORE_NAMELEN 64 + +static DEFINE_MUTEX(records_list_lock); +static LIST_HEAD(records_list); + +static DEFINE_MUTEX(pstore_sb_lock); +static struct super_block *pstore_sb; + +DEFINE_FREE(pstore_iput, struct inode *, if (_T) iput(_T)) + +struct pstore_private { + struct list_head list; + struct dentry *dentry; + struct pstore_record *record; + size_t total_size; +}; + +struct pstore_ftrace_seq_data { + const void *ptr; + size_t off; + size_t size; +}; + +#define REC_SIZE sizeof(struct pstore_ftrace_record) + +static void free_pstore_private(struct pstore_private *private) +{ + if (!private) + return; + if (private->record) { + kvfree(private->record->buf); + kfree(private->record->priv); + kfree(private->record); + } + kfree(private); +} +DEFINE_FREE(pstore_private, struct pstore_private *, free_pstore_private(_T)); + +static void *pstore_ftrace_seq_start(struct seq_file *s, loff_t *pos) +{ + struct pstore_private *ps = s->private; + struct pstore_ftrace_seq_data *data __free(kfree) = NULL; + + data = kzalloc_obj(*data); + if (!data) + return NULL; + + data->off = ps->record->size % REC_SIZE; + data->off += *pos * REC_SIZE; + if (data->off + REC_SIZE > ps->record->size) + return NULL; + + return_ptr(data); +} + +static void pstore_ftrace_seq_stop(struct seq_file *s, void *v) +{ + kfree(v); +} + +static void *pstore_ftrace_seq_next(struct seq_file *s, void *v, loff_t *pos) +{ + struct pstore_private *ps = s->private; + struct pstore_ftrace_seq_data *data = v; + + (*pos)++; + data->off += REC_SIZE; + if (data->off + REC_SIZE > ps->record->size) + return NULL; + + return data; +} + +static int pstore_ftrace_seq_show(struct seq_file *s, void *v) +{ + struct pstore_private *ps = s->private; + struct pstore_ftrace_seq_data *data = v; + struct pstore_ftrace_record *rec; + unsigned long ip, parent_ip; + + if (!data) + return 0; + + rec = (struct pstore_ftrace_record *)(ps->record->buf + data->off); + + ip = decode_ip(rec->ip); + parent_ip = decode_ip(rec->parent_ip); + seq_printf(s, "CPU:%d ts:%llu %08lx %08lx %ps <- %pS\n", + pstore_ftrace_decode_cpu(rec), + pstore_ftrace_read_timestamp(rec), + ip, parent_ip, (void *)ip, (void *)parent_ip); + + return 0; +} + +static const struct seq_operations pstore_ftrace_seq_ops = { + .start = pstore_ftrace_seq_start, + .next = pstore_ftrace_seq_next, + .stop = pstore_ftrace_seq_stop, + .show = pstore_ftrace_seq_show, +}; + +static ssize_t pstore_file_read(struct file *file, char __user *userbuf, + size_t count, loff_t *ppos) +{ + struct seq_file *sf = file->private_data; + struct pstore_private *ps = sf->private; + + if (ps->record->type == PSTORE_TYPE_FTRACE) + return seq_read(file, userbuf, count, ppos); + return simple_read_from_buffer(userbuf, count, ppos, + ps->record->buf, ps->total_size); +} + +static int pstore_file_open(struct inode *inode, struct file *file) +{ + struct pstore_private *ps = inode->i_private; + struct seq_file *sf; + int err; + const struct seq_operations *sops = NULL; + + if (ps->record->type == PSTORE_TYPE_FTRACE) + sops = &pstore_ftrace_seq_ops; + + err = seq_open(file, sops); + if (err < 0) + return err; + + sf = file->private_data; + sf->private = ps; + + return 0; +} + +static loff_t pstore_file_llseek(struct file *file, loff_t off, int whence) +{ + struct seq_file *sf = file->private_data; + + if (sf->op) + return seq_lseek(file, off, whence); + return default_llseek(file, off, whence); +} + +static const struct file_operations pstore_file_operations = { + .open = pstore_file_open, + .read = pstore_file_read, + .llseek = pstore_file_llseek, + .release = seq_release, +}; + +/* + * When a file is unlinked from our file system we call the + * platform driver to erase the record from persistent store. + */ +static int pstore_unlink(struct inode *dir, struct dentry *dentry) +{ + struct pstore_private *p = d_inode(dentry)->i_private; + struct pstore_record *record = p->record; + + if (!record->psi->erase) + return -EPERM; + + /* Make sure we can't race while removing this file. */ + scoped_guard(mutex, &records_list_lock) { + if (!list_empty(&p->list)) + list_del_init(&p->list); + else + return -ENOENT; + p->dentry = NULL; + } + + scoped_guard(mutex, &record->psi->read_mutex) + record->psi->erase(record); + + return simple_unlink(dir, dentry); +} + +static void pstore_evict_inode(struct inode *inode) +{ + struct pstore_private *p = inode->i_private; + + clear_inode(inode); + free_pstore_private(p); +} + +static const struct inode_operations pstore_dir_inode_operations = { + .lookup = simple_lookup, + .unlink = pstore_unlink, +}; + +static struct inode *pstore_get_inode(struct super_block *sb) +{ + struct inode *inode = new_inode(sb); + if (inode) { + inode->i_ino = get_next_ino(); + simple_inode_init_ts(inode); + } + return inode; +} + +enum { + Opt_kmsg_bytes +}; + +static const struct fs_parameter_spec pstore_param_spec[] = { + fsparam_u32 ("kmsg_bytes", Opt_kmsg_bytes), + {} +}; + +struct pstore_context { + unsigned int kmsg_bytes; +}; + +static int pstore_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct pstore_context *ctx = fc->fs_private; + struct fs_parse_result result; + int opt; + + opt = fs_parse(fc, pstore_param_spec, param, &result); + /* pstore has historically ignored invalid kmsg_bytes param */ + if (opt < 0) + return 0; + + switch (opt) { + case Opt_kmsg_bytes: + ctx->kmsg_bytes = result.uint_32; + break; + default: + return -EINVAL; + } + + return 0; +} + +/* + * Display the mount options in /proc/mounts. + */ +static int pstore_show_options(struct seq_file *m, struct dentry *root) +{ + if (kmsg_bytes != CONFIG_PSTORE_DEFAULT_KMSG_BYTES) + seq_printf(m, ",kmsg_bytes=%u", kmsg_bytes); + return 0; +} + +static int pstore_reconfigure(struct fs_context *fc) +{ + struct pstore_context *ctx = fc->fs_private; + + sync_filesystem(fc->root->d_sb); + pstore_set_kmsg_bytes(ctx->kmsg_bytes); + + return 0; +} + +static const struct super_operations pstore_ops = { + .statfs = simple_statfs, + .drop_inode = inode_just_drop, + .evict_inode = pstore_evict_inode, + .show_options = pstore_show_options, +}; + +static struct dentry *psinfo_lock_root(void) +{ + struct dentry *root; + + guard(mutex)(&pstore_sb_lock); + /* + * Having no backend is fine -- no records appear. + * Not being mounted is fine -- nothing to do. + */ + if (!psinfo || !pstore_sb) + return NULL; + + root = pstore_sb->s_root; + inode_lock_nested(d_inode(root), I_MUTEX_PARENT); + + return root; +} + +int pstore_put_backend_records(struct pstore_info *psi) +{ + struct pstore_private *pos, *tmp; + struct dentry *root; + + root = psinfo_lock_root(); + if (!root) + return 0; + + scoped_guard(mutex, &records_list_lock) { + list_for_each_entry_safe(pos, tmp, &records_list, list) { + if (pos->record->psi == psi) { + list_del_init(&pos->list); + locked_recursive_removal(pos->dentry, NULL); + pos->dentry = NULL; + } + } + } + + inode_unlock(d_inode(root)); + + return 0; +} + +/* + * Make a regular file in the root directory of our file system. + * Load it up with "size" bytes of data from "buf". + * Set the mtime & ctime to the date that this record was originally stored. + */ +int pstore_mkfile(struct dentry *root, struct pstore_record *record) +{ + struct dentry *dentry; + struct inode *inode __free(pstore_iput) = NULL; + char name[PSTORE_NAMELEN]; + struct pstore_private *private __free(pstore_private) = NULL, *pos; + size_t size = record->size + record->ecc_notice_size; + + if (WARN_ON(!inode_is_locked(d_inode(root)))) + return -EINVAL; + + guard(mutex)(&records_list_lock); + + /* Skip records that are already present in the filesystem. */ + list_for_each_entry(pos, &records_list, list) { + if (pos->record->type == record->type && + pos->record->id == record->id && + pos->record->psi == record->psi) + return -EEXIST; + } + + inode = pstore_get_inode(root->d_sb); + if (!inode) + return -ENOMEM; + inode->i_mode = S_IFREG | 0444; + inode->i_fop = &pstore_file_operations; + scnprintf(name, sizeof(name), "%s-%s-%llu%s", + pstore_type_to_name(record->type), + record->psi->name, record->id, + record->compressed ? ".enc.z" : ""); + + private = kzalloc_obj(*private); + if (!private) + return -ENOMEM; + + dentry = d_alloc_name(root, name); + if (!dentry) + return -ENOMEM; + + private->dentry = dentry; // borrowed + private->record = record; + inode->i_size = private->total_size = size; + inode->i_private = private; + + if (record->time.tv_sec) + inode_set_mtime_to_ts(inode, + inode_set_ctime_to_ts(inode, record->time)); + + d_make_persistent(dentry, no_free_ptr(inode)); + dput(dentry); + + list_add(&(no_free_ptr(private))->list, &records_list); + + return 0; +} + +/* + * Read all the records from the persistent store. Create + * files in our filesystem. Don't warn about -EEXIST errors + * when we are re-scanning the backing store looking to add new + * error records. + */ +void pstore_get_records(int quiet) +{ + struct dentry *root; + + root = psinfo_lock_root(); + if (!root) + return; + + pstore_get_backend_records(psinfo, root, quiet); + inode_unlock(d_inode(root)); +} + +static int pstore_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct pstore_context *ctx = fc->fs_private; + struct inode *inode; + + sb->s_maxbytes = MAX_LFS_FILESIZE; + sb->s_blocksize = PAGE_SIZE; + sb->s_blocksize_bits = PAGE_SHIFT; + sb->s_magic = PSTOREFS_MAGIC; + sb->s_op = &pstore_ops; + sb->s_time_gran = 1; + + pstore_set_kmsg_bytes(ctx->kmsg_bytes); + + inode = pstore_get_inode(sb); + if (inode) { + inode->i_mode = S_IFDIR | 0750; + inode->i_op = &pstore_dir_inode_operations; + inode->i_fop = &simple_dir_operations; + inc_nlink(inode); + } + sb->s_root = d_make_root(inode); + if (!sb->s_root) + return -ENOMEM; + + scoped_guard(mutex, &pstore_sb_lock) + pstore_sb = sb; + + pstore_get_records(0); + + return 0; +} + +static int pstore_get_tree(struct fs_context *fc) +{ + if (fc->root) + return pstore_reconfigure(fc); + + return get_tree_single(fc, pstore_fill_super); +} + +static void pstore_free_fc(struct fs_context *fc) +{ + kfree(fc->fs_private); +} + +static const struct fs_context_operations pstore_context_ops = { + .parse_param = pstore_parse_param, + .get_tree = pstore_get_tree, + .reconfigure = pstore_reconfigure, + .free = pstore_free_fc, +}; + +static void pstore_kill_sb(struct super_block *sb) +{ + guard(mutex)(&pstore_sb_lock); + WARN_ON(pstore_sb && pstore_sb != sb); + + kill_anon_super(sb); + pstore_sb = NULL; + + guard(mutex)(&records_list_lock); + INIT_LIST_HEAD(&records_list); +} + +static int pstore_init_fs_context(struct fs_context *fc) +{ + struct pstore_context *ctx; + + ctx = kzalloc_obj(struct pstore_context); + if (!ctx) + return -ENOMEM; + + /* + * Global kmsg_bytes is initialized to default, and updated + * every time we (re)mount the single-sb filesystem with the + * option specified. + */ + ctx->kmsg_bytes = kmsg_bytes; + + fc->fs_private = ctx; + fc->ops = &pstore_context_ops; + + return 0; +} + +static struct file_system_type pstore_fs_type = { + .owner = THIS_MODULE, + .name = "pstore", + .kill_sb = pstore_kill_sb, + .init_fs_context = pstore_init_fs_context, + .parameters = pstore_param_spec, +}; + +int __init pstore_init_fs(void) +{ + int err; + + /* Create a convenient mount point for people to access pstore */ + err = sysfs_create_mount_point(fs_kobj, "pstore"); + if (err) + goto out; + + err = register_filesystem(&pstore_fs_type); + if (err < 0) + sysfs_remove_mount_point(fs_kobj, "pstore"); + +out: + return err; +} + +void __exit pstore_exit_fs(void) +{ + unregister_filesystem(&pstore_fs_type); + sysfs_remove_mount_point(fs_kobj, "pstore"); +} diff --git a/fs/pstore/internal.h b/fs/pstore/internal.h new file mode 100644 index 000000000..079284120 --- /dev/null +++ b/fs/pstore/internal.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PSTORE_INTERNAL_H__ +#define __PSTORE_INTERNAL_H__ + +#include +#include +#include + +extern unsigned int kmsg_bytes; + +#ifdef CONFIG_PSTORE_FTRACE +extern unsigned long decode_ip(unsigned long ip); +extern void pstore_register_ftrace(void); +extern void pstore_unregister_ftrace(void); +ssize_t pstore_ftrace_combine_log(char **dest_log, size_t *dest_log_size, + const char *src_log, size_t src_log_size); +#else +static inline void pstore_register_ftrace(void) {} +static inline void pstore_unregister_ftrace(void) {} +static inline unsigned long decode_ip(unsigned long ip) { return ip; } +static inline ssize_t +pstore_ftrace_combine_log(char **dest_log, size_t *dest_log_size, + const char *src_log, size_t src_log_size) +{ + *dest_log_size = 0; + return 0; +} +#endif + +#ifdef CONFIG_PSTORE_PMSG +extern void pstore_register_pmsg(void); +extern void pstore_unregister_pmsg(void); +#else +static inline void pstore_register_pmsg(void) {} +static inline void pstore_unregister_pmsg(void) {} +#endif + +extern struct pstore_info *psinfo; + +extern void pstore_set_kmsg_bytes(unsigned int bytes); +extern void pstore_get_records(int); +extern void pstore_get_backend_records(struct pstore_info *psi, + struct dentry *root, int quiet); +extern int pstore_put_backend_records(struct pstore_info *psi); +extern int pstore_mkfile(struct dentry *root, + struct pstore_record *record); +extern void pstore_record_init(struct pstore_record *record, + struct pstore_info *psi); + +/* Called during pstore init/exit. */ +int __init pstore_init_fs(void); +void __exit pstore_exit_fs(void); + +#endif diff --git a/fs/pstore/platform.c b/fs/pstore/platform.c new file mode 100644 index 000000000..1d76c9d92 --- /dev/null +++ b/fs/pstore/platform.c @@ -0,0 +1,766 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Persistent Storage - platform driver interface parts. + * + * Copyright (C) 2007-2008 Google, Inc. + * Copyright (C) 2010 Intel Corporation + */ + +#define pr_fmt(fmt) "pstore: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +/* + * We defer making "oops" entries appear in pstore - see + * whether the system is actually still running well enough + * to let someone see the entry + */ +static int pstore_update_ms = -1; +module_param_named(update_ms, pstore_update_ms, int, 0600); +MODULE_PARM_DESC(update_ms, "milliseconds before pstore updates its content " + "(default is -1, which means runtime updates are disabled; " + "enabling this option may not be safe; it may lead to further " + "corruption on Oopses)"); + +/* Names should be in the same order as the enum pstore_type_id */ +static const char * const pstore_type_names[] = { + "dmesg", + "mce", + "console", + "ftrace", + "rtas", + "powerpc-ofw", + "powerpc-common", + "pmsg", + "powerpc-opal", +}; + +static int pstore_new_entry; + +static void pstore_timefunc(struct timer_list *); +static DEFINE_TIMER(pstore_timer, pstore_timefunc); + +static void pstore_dowork(struct work_struct *); +static DECLARE_WORK(pstore_work, pstore_dowork); + +/* + * psinfo_lock protects "psinfo" during calls to + * pstore_register(), pstore_unregister(), and + * the filesystem mount/unmount routines. + */ +static DEFINE_MUTEX(psinfo_lock); +struct pstore_info *psinfo; + +static char *backend; +module_param(backend, charp, 0444); +MODULE_PARM_DESC(backend, "specific backend to use"); + +/* + * pstore no longer implements compression via the crypto API, and only + * supports zlib deflate compression implemented using the zlib library + * interface. This removes additional complexity which is hard to justify for a + * diagnostic facility that has to operate in conditions where the system may + * have become unstable. Zlib deflate is comparatively small in terms of code + * size, and compresses ASCII text comparatively well. In terms of compression + * speed, deflate is not the best performer but for recording the log output on + * a kernel panic, this is not considered critical. + * + * The only remaining arguments supported by the compress= module parameter are + * 'deflate' and 'none'. To retain compatibility with existing installations, + * all other values are logged and replaced with 'deflate'. + */ +static char *compress = "deflate"; +module_param(compress, charp, 0444); +MODULE_PARM_DESC(compress, "compression to use"); + +/* How much of the kernel log to snapshot */ +unsigned int kmsg_bytes = CONFIG_PSTORE_DEFAULT_KMSG_BYTES; +module_param(kmsg_bytes, uint, 0444); +MODULE_PARM_DESC(kmsg_bytes, "amount of kernel log to snapshot (in bytes)"); + +static void *compress_workspace; + +/* + * Compression is only used for dmesg output, which consists of low-entropy + * ASCII text, and so we can assume worst-case 60%. + */ +#define DMESG_COMP_PERCENT 60 + +static char *big_oops_buf; +static size_t max_compressed_size; + +void pstore_set_kmsg_bytes(unsigned int bytes) +{ + WRITE_ONCE(kmsg_bytes, bytes); +} + +/* Tag each group of saved records with a sequence number */ +static int oopscount; + +const char *pstore_type_to_name(enum pstore_type_id type) +{ + BUILD_BUG_ON(ARRAY_SIZE(pstore_type_names) != PSTORE_TYPE_MAX); + + if (WARN_ON_ONCE(type >= PSTORE_TYPE_MAX)) + return "unknown"; + + return pstore_type_names[type]; +} +EXPORT_SYMBOL_GPL(pstore_type_to_name); + +enum pstore_type_id pstore_name_to_type(const char *name) +{ + int i; + + for (i = 0; i < PSTORE_TYPE_MAX; i++) { + if (!strcmp(pstore_type_names[i], name)) + return i; + } + + return PSTORE_TYPE_MAX; +} +EXPORT_SYMBOL_GPL(pstore_name_to_type); + +static void pstore_timer_kick(void) +{ + if (pstore_update_ms < 0) + return; + + mod_timer(&pstore_timer, jiffies + msecs_to_jiffies(pstore_update_ms)); +} + +static bool pstore_cannot_block_path(enum kmsg_dump_reason reason) +{ + /* + * In case of NMI path, pstore shouldn't be blocked + * regardless of reason. + */ + if (in_nmi()) + return true; + + switch (reason) { + /* In panic case, other cpus are stopped by smp_send_stop(). */ + case KMSG_DUMP_PANIC: + /* + * Emergency restart shouldn't be blocked by spinning on + * pstore_info::buf_lock. + */ + case KMSG_DUMP_EMERG: + return true; + default: + return false; + } +} + +static int pstore_compress(const void *in, void *out, + unsigned int inlen, unsigned int outlen) +{ + struct z_stream_s zstream = { + .next_in = in, + .avail_in = inlen, + .next_out = out, + .avail_out = outlen, + .workspace = compress_workspace, + }; + int ret; + + if (!IS_ENABLED(CONFIG_PSTORE_COMPRESS)) + return -EINVAL; + + ret = zlib_deflateInit2(&zstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY); + if (ret != Z_OK) + return -EINVAL; + + ret = zlib_deflate(&zstream, Z_FINISH); + if (ret != Z_STREAM_END) + return -EINVAL; + + ret = zlib_deflateEnd(&zstream); + if (ret != Z_OK) + pr_warn_once("zlib_deflateEnd() failed: %d\n", ret); + + return zstream.total_out; +} + +static void allocate_buf_for_compression(void) +{ + size_t compressed_size; + char *buf; + + /* Skip if not built-in or compression disabled. */ + if (!IS_ENABLED(CONFIG_PSTORE_COMPRESS) || !compress || + !strcmp(compress, "none")) { + compress = NULL; + return; + } + + if (strcmp(compress, "deflate")) { + pr_err("Unsupported compression '%s', falling back to deflate\n", + compress); + compress = "deflate"; + } + + /* + * The compression buffer only needs to be as large as the maximum + * uncompressed record size, since any record that would be expanded by + * compression is just stored uncompressed. + */ + compressed_size = (psinfo->bufsize * 100) / DMESG_COMP_PERCENT; + buf = kvzalloc(compressed_size, GFP_KERNEL); + if (!buf) { + pr_err("Failed %zu byte compression buffer allocation for: %s\n", + psinfo->bufsize, compress); + return; + } + + compress_workspace = + vmalloc(zlib_deflate_workspacesize(MAX_WBITS, DEF_MEM_LEVEL)); + if (!compress_workspace) { + pr_err("Failed to allocate zlib deflate workspace\n"); + kvfree(buf); + return; + } + + /* A non-NULL big_oops_buf indicates compression is available. */ + big_oops_buf = buf; + max_compressed_size = compressed_size; + + pr_info("Using crash dump compression: %s\n", compress); +} + +static void free_buf_for_compression(void) +{ + if (IS_ENABLED(CONFIG_PSTORE_COMPRESS) && compress_workspace) { + vfree(compress_workspace); + compress_workspace = NULL; + } + + kvfree(big_oops_buf); + big_oops_buf = NULL; + max_compressed_size = 0; +} + +void pstore_record_init(struct pstore_record *record, + struct pstore_info *psinfo) +{ + memset(record, 0, sizeof(*record)); + + record->psi = psinfo; + + /* Report zeroed timestamp if called before timekeeping has resumed. */ + record->time = ns_to_timespec64(ktime_get_real_fast_ns()); +} + +/* + * callback from kmsg_dump. Save as much as we can (up to kmsg_bytes) from the + * end of the buffer. + */ +static void pstore_dump(struct kmsg_dumper *dumper, + struct kmsg_dump_detail *detail) +{ + struct kmsg_dump_iter iter; + unsigned int remaining = READ_ONCE(kmsg_bytes); + unsigned long total = 0; + const char *why; + unsigned int part = 1; + unsigned long flags = 0; + int saved_ret = 0; + int ret; + + why = kmsg_dump_reason_str(detail->reason); + + if (pstore_cannot_block_path(detail->reason)) { + if (!raw_spin_trylock_irqsave(&psinfo->buf_lock, flags)) { + pr_err("dump skipped in %s path because of concurrent dump\n", + in_nmi() ? "NMI" : why); + return; + } + } else { + raw_spin_lock_irqsave(&psinfo->buf_lock, flags); + } + + kmsg_dump_rewind(&iter); + + oopscount++; + while (total < remaining) { + char *dst; + size_t dst_size; + int header_size; + int zipped_len = -1; + size_t dump_size; + struct pstore_record record; + + pstore_record_init(&record, psinfo); + record.type = PSTORE_TYPE_DMESG; + record.count = oopscount; + record.reason = detail->reason; + record.part = part; + record.buf = psinfo->buf; + + dst = big_oops_buf ?: psinfo->buf; + dst_size = max_compressed_size ?: psinfo->bufsize; + + /* Write dump header. */ + header_size = snprintf(dst, dst_size, "%s#%d Part%u\n", why, + oopscount, part); + dst_size -= header_size; + + /* Write dump contents. */ + if (!kmsg_dump_get_buffer(&iter, true, dst + header_size, + dst_size, &dump_size)) + break; + + if (big_oops_buf) { + zipped_len = pstore_compress(dst, psinfo->buf, + header_size + dump_size, + psinfo->bufsize); + + if (zipped_len > 0) { + record.compressed = true; + record.size = zipped_len; + } else { + /* + * Compression failed, so the buffer is most + * likely filled with binary data that does not + * compress as well as ASCII text. Copy as much + * of the uncompressed data as possible into + * the pstore record, and discard the rest. + */ + record.size = psinfo->bufsize; + memcpy(psinfo->buf, dst, psinfo->bufsize); + } + } else { + record.size = header_size + dump_size; + } + + ret = psinfo->write(&record); + if (ret == 0 && detail->reason == KMSG_DUMP_OOPS) { + pstore_new_entry = 1; + pstore_timer_kick(); + } else { + /* Preserve only the first non-zero returned value. */ + if (!saved_ret) + saved_ret = ret; + } + + total += record.size; + part++; + } + raw_spin_unlock_irqrestore(&psinfo->buf_lock, flags); + + if (saved_ret) { + pr_err_once("backend (%s) writing error (%d)\n", psinfo->name, + saved_ret); + } +} + +static struct kmsg_dumper pstore_dumper = { + .dump = pstore_dump, +}; + +/* + * Register with kmsg_dump to save last part of console log on panic. + */ +static void pstore_register_kmsg(void) +{ + kmsg_dump_register(&pstore_dumper); +} + +static void pstore_unregister_kmsg(void) +{ + kmsg_dump_unregister(&pstore_dumper); +} + +#ifdef CONFIG_PSTORE_CONSOLE +static void pstore_console_write(struct console *con, const char *s, unsigned c) +{ + struct pstore_record record; + + if (!c) + return; + + pstore_record_init(&record, psinfo); + record.type = PSTORE_TYPE_CONSOLE; + + record.buf = (char *)s; + record.size = c; + psinfo->write(&record); +} + +static struct console pstore_console = { + .write = pstore_console_write, + .index = -1, +}; + +static void pstore_register_console(void) +{ + /* Show which backend is going to get console writes. */ + strscpy(pstore_console.name, psinfo->name, + sizeof(pstore_console.name)); + /* + * Always initialize flags here since prior unregister_console() + * calls may have changed settings (specifically CON_ENABLED). + */ + pstore_console.flags = CON_PRINTBUFFER | CON_ENABLED | CON_ANYTIME; + register_console(&pstore_console); +} + +static void pstore_unregister_console(void) +{ + unregister_console(&pstore_console); +} +#else +static void pstore_register_console(void) {} +static void pstore_unregister_console(void) {} +#endif + +static int pstore_write_user_compat(struct pstore_record *record, + const char __user *buf) +{ + int ret = 0; + + if (record->buf) + return -EINVAL; + + record->buf = vmemdup_user(buf, record->size); + if (IS_ERR(record->buf)) { + ret = PTR_ERR(record->buf); + goto out; + } + + ret = record->psi->write(record); + + kvfree(record->buf); +out: + record->buf = NULL; + + return unlikely(ret < 0) ? ret : record->size; +} + +/* + * platform specific persistent storage driver registers with + * us here. If pstore is already mounted, call the platform + * read function right away to populate the file system. If not + * then the pstore mount code will call us later to fill out + * the file system. + */ +int pstore_register(struct pstore_info *psi) +{ + char *new_backend; + + if (backend && strcmp(backend, psi->name)) { + pr_warn("backend '%s' already in use: ignoring '%s'\n", + backend, psi->name); + return -EBUSY; + } + + /* Sanity check flags. */ + if (!psi->flags) { + pr_warn("backend '%s' must support at least one frontend\n", + psi->name); + return -EINVAL; + } + + /* Check for required functions. */ + if (!psi->read || !psi->write) { + pr_warn("backend '%s' must implement read() and write()\n", + psi->name); + return -EINVAL; + } + + new_backend = kstrdup(psi->name, GFP_KERNEL); + if (!new_backend) + return -ENOMEM; + + mutex_lock(&psinfo_lock); + if (psinfo) { + pr_warn("backend '%s' already loaded: ignoring '%s'\n", + psinfo->name, psi->name); + mutex_unlock(&psinfo_lock); + kfree(new_backend); + return -EBUSY; + } + + if (!psi->write_user) + psi->write_user = pstore_write_user_compat; + psinfo = psi; + mutex_init(&psinfo->read_mutex); + raw_spin_lock_init(&psinfo->buf_lock); + + if (psi->flags & PSTORE_FLAGS_DMESG) + allocate_buf_for_compression(); + + pstore_get_records(0); + + if (psi->flags & PSTORE_FLAGS_DMESG) { + pstore_dumper.max_reason = psinfo->max_reason; + pstore_register_kmsg(); + } + if (psi->flags & PSTORE_FLAGS_CONSOLE) + pstore_register_console(); + if (psi->flags & PSTORE_FLAGS_FTRACE) + pstore_register_ftrace(); + if (psi->flags & PSTORE_FLAGS_PMSG) + pstore_register_pmsg(); + + /* Start watching for new records, if desired. */ + pstore_timer_kick(); + + /* + * Update the module parameter backend, so it is visible + * through /sys/module/pstore/parameters/backend + */ + backend = new_backend; + + pr_info("Registered %s as persistent store backend\n", psi->name); + + mutex_unlock(&psinfo_lock); + return 0; +} +EXPORT_SYMBOL_GPL(pstore_register); + +void pstore_unregister(struct pstore_info *psi) +{ + /* It's okay to unregister nothing. */ + if (!psi) + return; + + mutex_lock(&psinfo_lock); + + /* Only one backend can be registered at a time. */ + if (WARN_ON(psi != psinfo)) { + mutex_unlock(&psinfo_lock); + return; + } + + /* Unregister all callbacks. */ + if (psi->flags & PSTORE_FLAGS_PMSG) + pstore_unregister_pmsg(); + if (psi->flags & PSTORE_FLAGS_FTRACE) + pstore_unregister_ftrace(); + if (psi->flags & PSTORE_FLAGS_CONSOLE) + pstore_unregister_console(); + if (psi->flags & PSTORE_FLAGS_DMESG) + pstore_unregister_kmsg(); + + /* Stop timer and make sure all work has finished. */ + timer_delete_sync(&pstore_timer); + flush_work(&pstore_work); + + /* Remove all backend records from filesystem tree. */ + pstore_put_backend_records(psi); + + free_buf_for_compression(); + + psinfo = NULL; + kfree(backend); + backend = NULL; + + pr_info("Unregistered %s as persistent store backend\n", psi->name); + mutex_unlock(&psinfo_lock); +} +EXPORT_SYMBOL_GPL(pstore_unregister); + +static void decompress_record(struct pstore_record *record, + struct z_stream_s *zstream) +{ + int ret; + int unzipped_len; + char *unzipped, *workspace; + size_t max_uncompressed_size; + + if (!IS_ENABLED(CONFIG_PSTORE_COMPRESS) || !record->compressed) + return; + + /* Only PSTORE_TYPE_DMESG support compression. */ + if (record->type != PSTORE_TYPE_DMESG) { + pr_warn("ignored compressed record type %d\n", record->type); + return; + } + + /* Missing compression buffer means compression was not initialized. */ + if (!zstream->workspace) { + pr_warn("no decompression method initialized!\n"); + return; + } + + ret = zlib_inflateReset(zstream); + if (ret != Z_OK) { + pr_err("zlib_inflateReset() failed, ret = %d!\n", ret); + return; + } + + /* Allocate enough space to hold max decompression and ECC. */ + max_uncompressed_size = 3 * psinfo->bufsize; + workspace = kvzalloc(max_uncompressed_size + record->ecc_notice_size, + GFP_KERNEL); + if (!workspace) + return; + + zstream->next_in = record->buf; + zstream->avail_in = record->size; + zstream->next_out = workspace; + zstream->avail_out = max_uncompressed_size; + + ret = zlib_inflate(zstream, Z_FINISH); + if (ret != Z_STREAM_END) { + pr_err_ratelimited("zlib_inflate() failed, ret = %d!\n", ret); + kvfree(workspace); + return; + } + + unzipped_len = zstream->total_out; + + /* Append ECC notice to decompressed buffer. */ + memcpy(workspace + unzipped_len, record->buf + record->size, + record->ecc_notice_size); + + /* Copy decompressed contents into an minimum-sized allocation. */ + unzipped = kvmemdup(workspace, unzipped_len + record->ecc_notice_size, + GFP_KERNEL); + kvfree(workspace); + if (!unzipped) + return; + + /* Swap out compressed contents with decompressed contents. */ + kvfree(record->buf); + record->buf = unzipped; + record->size = unzipped_len; + record->compressed = false; +} + +/* + * Read all the records from one persistent store backend. Create + * files in our filesystem. Don't warn about -EEXIST errors + * when we are re-scanning the backing store looking to add new + * error records. + */ +void pstore_get_backend_records(struct pstore_info *psi, + struct dentry *root, int quiet) +{ + int failed = 0; + unsigned int stop_loop = 65536; + struct z_stream_s zstream = {}; + + if (!psi || !root) + return; + + if (IS_ENABLED(CONFIG_PSTORE_COMPRESS) && compress) { + zstream.workspace = kvmalloc(zlib_inflate_workspacesize(), + GFP_KERNEL); + zlib_inflateInit2(&zstream, -DEF_WBITS); + } + + mutex_lock(&psi->read_mutex); + if (psi->open && psi->open(psi)) + goto out; + + /* + * Backend callback read() allocates record.buf. decompress_record() + * may reallocate record.buf. On success, pstore_mkfile() will keep + * the record.buf, so free it only on failure. + */ + for (; stop_loop; stop_loop--) { + struct pstore_record *record; + int rc; + + record = kzalloc_obj(*record); + if (!record) { + pr_err("out of memory creating record\n"); + break; + } + pstore_record_init(record, psi); + + record->size = psi->read(record); + + /* No more records left in backend? */ + if (record->size <= 0) { + kfree(record); + break; + } + + decompress_record(record, &zstream); + rc = pstore_mkfile(root, record); + if (rc) { + /* pstore_mkfile() did not take record, so free it. */ + kvfree(record->buf); + kfree(record->priv); + kfree(record); + if (rc != -EEXIST || !quiet) + failed++; + } + } + if (psi->close) + psi->close(psi); +out: + mutex_unlock(&psi->read_mutex); + + if (IS_ENABLED(CONFIG_PSTORE_COMPRESS) && compress) { + if (zlib_inflateEnd(&zstream) != Z_OK) + pr_warn("zlib_inflateEnd() failed\n"); + kvfree(zstream.workspace); + } + + if (failed) + pr_warn("failed to create %d record(s) from '%s'\n", + failed, psi->name); + if (!stop_loop) + pr_err("looping? Too many records seen from '%s'\n", + psi->name); +} + +static void pstore_dowork(struct work_struct *work) +{ + pstore_get_records(1); +} + +static void pstore_timefunc(struct timer_list *unused) +{ + if (pstore_new_entry) { + pstore_new_entry = 0; + schedule_work(&pstore_work); + } + + pstore_timer_kick(); +} + +static int __init pstore_init(void) +{ + int ret; + + ret = pstore_init_fs(); + if (ret) + free_buf_for_compression(); + + return ret; +} +late_initcall(pstore_init); + +static void __exit pstore_exit(void) +{ + pstore_exit_fs(); +} +module_exit(pstore_exit) + +MODULE_AUTHOR("Tony Luck "); +MODULE_DESCRIPTION("Persistent Storage - platform driver interface"); +MODULE_LICENSE("GPL"); diff --git a/fs/pstore/pmsg.c b/fs/pstore/pmsg.c new file mode 100644 index 000000000..55f139afa --- /dev/null +++ b/fs/pstore/pmsg.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2014 Google, Inc. + */ + +#include +#include +#include +#include +#include "internal.h" + +static DEFINE_MUTEX(pmsg_lock); + +static ssize_t write_pmsg(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct pstore_record record; + int ret; + + if (!count) + return 0; + + pstore_record_init(&record, psinfo); + record.type = PSTORE_TYPE_PMSG; + record.size = count; + + /* check outside lock, page in any data. write_user also checks */ + if (!access_ok(buf, count)) + return -EFAULT; + + mutex_lock(&pmsg_lock); + ret = psinfo->write_user(&record, buf); + mutex_unlock(&pmsg_lock); + return ret ? ret : count; +} + +static const struct file_operations pmsg_fops = { + .owner = THIS_MODULE, + .llseek = noop_llseek, + .write = write_pmsg, +}; + +static struct class *pmsg_class; +static int pmsg_major; +#define PMSG_NAME "pmsg" +#undef pr_fmt +#define pr_fmt(fmt) PMSG_NAME ": " fmt + +static char *pmsg_devnode(const struct device *dev, umode_t *mode) +{ + if (mode) + *mode = 0220; + return NULL; +} + +void pstore_register_pmsg(void) +{ + struct device *pmsg_device; + + pmsg_major = register_chrdev(0, PMSG_NAME, &pmsg_fops); + if (pmsg_major < 0) { + pr_err("register_chrdev failed\n"); + goto err; + } + + pmsg_class = class_create(PMSG_NAME); + if (IS_ERR(pmsg_class)) { + pr_err("device class file already in use\n"); + goto err_class; + } + pmsg_class->devnode = pmsg_devnode; + + pmsg_device = device_create(pmsg_class, NULL, MKDEV(pmsg_major, 0), + NULL, "%s%d", PMSG_NAME, 0); + if (IS_ERR(pmsg_device)) { + pr_err("failed to create device\n"); + goto err_device; + } + return; + +err_device: + class_destroy(pmsg_class); +err_class: + unregister_chrdev(pmsg_major, PMSG_NAME); +err: + return; +} + +void pstore_unregister_pmsg(void) +{ + device_destroy(pmsg_class, MKDEV(pmsg_major, 0)); + class_destroy(pmsg_class); + unregister_chrdev(pmsg_major, PMSG_NAME); +} diff --git a/fs/pstore/ram.c b/fs/pstore/ram.c new file mode 100644 index 000000000..2eb0d4fa2 --- /dev/null +++ b/fs/pstore/ram.c @@ -0,0 +1,999 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * RAM Oops/Panic logger + * + * Copyright (C) 2010 Marco Stornelli + * Copyright (C) 2011 Kees Cook + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" +#include "ram_internal.h" + +#define RAMOOPS_KERNMSG_HDR "====" +#define MIN_MEM_SIZE 4096UL + +static ulong record_size = MIN_MEM_SIZE; +module_param(record_size, ulong, 0400); +MODULE_PARM_DESC(record_size, + "size of each dump done on oops/panic"); + +static ulong ramoops_console_size = MIN_MEM_SIZE; +module_param_named(console_size, ramoops_console_size, ulong, 0400); +MODULE_PARM_DESC(console_size, "size of kernel console log"); + +static ulong ramoops_ftrace_size = MIN_MEM_SIZE; +module_param_named(ftrace_size, ramoops_ftrace_size, ulong, 0400); +MODULE_PARM_DESC(ftrace_size, "size of ftrace log"); + +static ulong ramoops_pmsg_size = MIN_MEM_SIZE; +module_param_named(pmsg_size, ramoops_pmsg_size, ulong, 0400); +MODULE_PARM_DESC(pmsg_size, "size of user space message log"); + +static unsigned long long mem_address; +module_param_hw(mem_address, ullong, other, 0400); +MODULE_PARM_DESC(mem_address, + "start of reserved RAM used to store oops/panic logs"); + +static char *mem_name; +module_param_named(mem_name, mem_name, charp, 0400); +MODULE_PARM_DESC(mem_name, "name of kernel param that holds addr"); + +static ulong mem_size; +module_param(mem_size, ulong, 0400); +MODULE_PARM_DESC(mem_size, + "size of reserved RAM used to store oops/panic logs"); + +static unsigned int mem_type; +module_param(mem_type, uint, 0400); +MODULE_PARM_DESC(mem_type, + "memory type: 0=write-combined (default), 1=unbuffered, 2=cached"); + +static int ramoops_max_reason = -1; +module_param_named(max_reason, ramoops_max_reason, int, 0400); +MODULE_PARM_DESC(max_reason, + "maximum reason for kmsg dump (default 2: Oops and Panic) "); + +static int ramoops_ecc; +module_param_named(ecc, ramoops_ecc, int, 0400); +MODULE_PARM_DESC(ecc, + "if non-zero, the option enables ECC support and specifies " + "ECC buffer size in bytes (1 is a special value, means 16 " + "bytes ECC)"); + +static int ramoops_dump_oops = -1; +module_param_named(dump_oops, ramoops_dump_oops, int, 0400); +MODULE_PARM_DESC(dump_oops, + "(deprecated: use max_reason instead) set to 1 to dump oopses & panics, 0 to only dump panics"); + +struct ramoops_context { + struct persistent_ram_zone **dprzs; /* Oops dump zones */ + struct persistent_ram_zone *cprz; /* Console zone */ + struct persistent_ram_zone **fprzs; /* Ftrace zones */ + struct persistent_ram_zone *mprz; /* PMSG zone */ + phys_addr_t phys_addr; + unsigned long size; + unsigned int memtype; + size_t record_size; + size_t console_size; + size_t ftrace_size; + size_t pmsg_size; + u32 flags; + struct persistent_ram_ecc_info ecc_info; + unsigned int max_dump_cnt; + unsigned int dump_write_cnt; + /* _read_cnt need clear on ramoops_pstore_open */ + unsigned int dump_read_cnt; + unsigned int console_read_cnt; + unsigned int max_ftrace_cnt; + unsigned int ftrace_read_cnt; + unsigned int pmsg_read_cnt; + struct pstore_info pstore; +}; + +static struct platform_device *dummy; + +static int ramoops_pstore_open(struct pstore_info *psi) +{ + struct ramoops_context *cxt = psi->data; + + cxt->dump_read_cnt = 0; + cxt->console_read_cnt = 0; + cxt->ftrace_read_cnt = 0; + cxt->pmsg_read_cnt = 0; + return 0; +} + +static struct persistent_ram_zone * +ramoops_get_next_prz(struct persistent_ram_zone *przs[], int id, + struct pstore_record *record) +{ + struct persistent_ram_zone *prz; + + /* Give up if we never existed or have hit the end. */ + if (!przs) + return NULL; + + prz = przs[id]; + if (!prz) + return NULL; + + /* Update old/shadowed buffer. */ + if (prz->type == PSTORE_TYPE_DMESG) + persistent_ram_save_old(prz); + + if (!persistent_ram_old_size(prz)) + return NULL; + + record->type = prz->type; + record->id = id; + + return prz; +} + +static int ramoops_read_kmsg_hdr(char *buffer, struct timespec64 *time, + bool *compressed) +{ + char data_type; + int header_length = 0; + + if (sscanf(buffer, RAMOOPS_KERNMSG_HDR "%lld.%lu-%c\n%n", + (time64_t *)&time->tv_sec, &time->tv_nsec, &data_type, + &header_length) == 3) { + time->tv_nsec *= 1000; + if (data_type == 'C') + *compressed = true; + else + *compressed = false; + } else if (sscanf(buffer, RAMOOPS_KERNMSG_HDR "%lld.%lu\n%n", + (time64_t *)&time->tv_sec, &time->tv_nsec, + &header_length) == 2) { + time->tv_nsec *= 1000; + *compressed = false; + } else { + time->tv_sec = 0; + time->tv_nsec = 0; + *compressed = false; + } + return header_length; +} + +static bool prz_ok(struct persistent_ram_zone *prz) +{ + return !!prz && !!(persistent_ram_old_size(prz) + + persistent_ram_ecc_string(prz, NULL, 0)); +} + +static ssize_t ramoops_pstore_read(struct pstore_record *record) +{ + ssize_t size = 0; + struct ramoops_context *cxt = record->psi->data; + struct persistent_ram_zone *prz = NULL; + int header_length = 0; + bool free_prz = false; + + /* + * Ramoops headers provide time stamps for PSTORE_TYPE_DMESG, but + * PSTORE_TYPE_CONSOLE and PSTORE_TYPE_FTRACE don't currently have + * valid time stamps, so it is initialized to zero. + */ + record->time.tv_sec = 0; + record->time.tv_nsec = 0; + record->compressed = false; + + /* Find the next valid persistent_ram_zone for DMESG */ + while (cxt->dump_read_cnt < cxt->max_dump_cnt && !prz) { + prz = ramoops_get_next_prz(cxt->dprzs, cxt->dump_read_cnt++, + record); + if (!prz_ok(prz)) + continue; + header_length = ramoops_read_kmsg_hdr(persistent_ram_old(prz), + &record->time, + &record->compressed); + /* Clear and skip this DMESG record if it has no valid header */ + if (!header_length) { + persistent_ram_free_old(prz); + persistent_ram_zap(prz); + prz = NULL; + } + } + + if (!prz_ok(prz) && !cxt->console_read_cnt++) + prz = ramoops_get_next_prz(&cxt->cprz, 0 /* single */, record); + + if (!prz_ok(prz) && !cxt->pmsg_read_cnt++) + prz = ramoops_get_next_prz(&cxt->mprz, 0 /* single */, record); + + /* ftrace is last since it may want to dynamically allocate memory. */ + if (!prz_ok(prz)) { + if (!(cxt->flags & RAMOOPS_FLAG_FTRACE_PER_CPU) && + !cxt->ftrace_read_cnt++) { + prz = ramoops_get_next_prz(cxt->fprzs, 0 /* single */, + record); + } else { + /* + * Build a new dummy record which combines all the + * per-cpu records including metadata and ecc info. + */ + struct persistent_ram_zone *tmp_prz, *prz_next; + + tmp_prz = kzalloc_obj(struct persistent_ram_zone); + if (!tmp_prz) + return -ENOMEM; + prz = tmp_prz; + free_prz = true; + + while (cxt->ftrace_read_cnt < cxt->max_ftrace_cnt) { + prz_next = ramoops_get_next_prz(cxt->fprzs, + cxt->ftrace_read_cnt++, record); + + if (!prz_ok(prz_next)) + continue; + + tmp_prz->ecc_info = prz_next->ecc_info; + tmp_prz->corrected_bytes += + prz_next->corrected_bytes; + tmp_prz->bad_blocks += prz_next->bad_blocks; + + size = pstore_ftrace_combine_log( + &tmp_prz->old_log, + &tmp_prz->old_log_size, + prz_next->old_log, + prz_next->old_log_size); + if (size) + goto out; + } + record->id = 0; + } + } + + if (!prz_ok(prz)) { + size = 0; + goto out; + } + + size = persistent_ram_old_size(prz) - header_length; + + /* ECC correction notice */ + record->ecc_notice_size = persistent_ram_ecc_string(prz, NULL, 0); + + record->buf = kvzalloc(size + record->ecc_notice_size + 1, GFP_KERNEL); + if (record->buf == NULL) { + size = -ENOMEM; + goto out; + } + + memcpy(record->buf, (char *)persistent_ram_old(prz) + header_length, + size); + + persistent_ram_ecc_string(prz, record->buf + size, + record->ecc_notice_size + 1); + +out: + if (free_prz) { + kvfree(prz->old_log); + kfree(prz); + } + + return size; +} + +static size_t ramoops_write_kmsg_hdr(struct persistent_ram_zone *prz, + struct pstore_record *record) +{ + char hdr[36]; /* "===="(4), %lld(20), "."(1), %06lu(6), "-%c\n"(3) */ + size_t len; + + len = scnprintf(hdr, sizeof(hdr), + RAMOOPS_KERNMSG_HDR "%lld.%06lu-%c\n", + (time64_t)record->time.tv_sec, + record->time.tv_nsec / 1000, + record->compressed ? 'C' : 'D'); + persistent_ram_write(prz, hdr, len); + + return len; +} + +static int notrace ramoops_pstore_write(struct pstore_record *record) +{ + struct ramoops_context *cxt = record->psi->data; + struct persistent_ram_zone *prz; + size_t size, hlen; + + if (record->type == PSTORE_TYPE_CONSOLE) { + if (!cxt->cprz) + return -ENOMEM; + persistent_ram_write(cxt->cprz, record->buf, record->size); + return 0; + } else if (record->type == PSTORE_TYPE_FTRACE) { + int zonenum; + + if (!cxt->fprzs) + return -ENOMEM; + /* + * Choose zone by if we're using per-cpu buffers. + */ + if (cxt->flags & RAMOOPS_FLAG_FTRACE_PER_CPU) + zonenum = smp_processor_id(); + else + zonenum = 0; + + persistent_ram_write(cxt->fprzs[zonenum], record->buf, + record->size); + return 0; + } else if (record->type == PSTORE_TYPE_PMSG) { + pr_warn_ratelimited("PMSG shouldn't call %s\n", __func__); + return -EINVAL; + } + + if (record->type != PSTORE_TYPE_DMESG) + return -EINVAL; + + /* + * We could filter on record->reason here if we wanted to (which + * would duplicate what happened before the "max_reason" setting + * was added), but that would defeat the purpose of a system + * changing printk.always_kmsg_dump, so instead log everything that + * the kmsg dumper sends us, since it should be doing the filtering + * based on the combination of printk.always_kmsg_dump and our + * requested "max_reason". + */ + + /* + * Explicitly only take the first part of any new crash. + * If our buffer is larger than kmsg_bytes, this can never happen, + * and if our buffer is smaller than kmsg_bytes, we don't want the + * report split across multiple records. + */ + if (record->part != 1) + return -ENOSPC; + + if (!cxt->dprzs) + return -ENOSPC; + + prz = cxt->dprzs[cxt->dump_write_cnt]; + + /* + * Since this is a new crash dump, we need to reset the buffer in + * case it still has an old dump present. Without this, the new dump + * will get appended, which would seriously confuse anything trying + * to check dump file contents. Specifically, ramoops_read_kmsg_hdr() + * expects to find a dump header in the beginning of buffer data, so + * we must to reset the buffer values, in order to ensure that the + * header will be written to the beginning of the buffer. + */ + persistent_ram_zap(prz); + + /* Build header and append record contents. */ + hlen = ramoops_write_kmsg_hdr(prz, record); + if (!hlen) + return -ENOMEM; + + size = record->size; + if (size + hlen > prz->buffer_size) + size = prz->buffer_size - hlen; + persistent_ram_write(prz, record->buf, size); + + cxt->dump_write_cnt = (cxt->dump_write_cnt + 1) % cxt->max_dump_cnt; + + return 0; +} + +static int notrace ramoops_pstore_write_user(struct pstore_record *record, + const char __user *buf) +{ + if (record->type == PSTORE_TYPE_PMSG) { + struct ramoops_context *cxt = record->psi->data; + + if (!cxt->mprz) + return -ENOMEM; + return persistent_ram_write_user(cxt->mprz, buf, record->size); + } + + return -EINVAL; +} + +static int ramoops_pstore_erase(struct pstore_record *record) +{ + struct ramoops_context *cxt = record->psi->data; + struct persistent_ram_zone *prz; + + switch (record->type) { + case PSTORE_TYPE_DMESG: + if (record->id >= cxt->max_dump_cnt) + return -EINVAL; + prz = cxt->dprzs[record->id]; + break; + case PSTORE_TYPE_CONSOLE: + prz = cxt->cprz; + break; + case PSTORE_TYPE_FTRACE: + if (record->id >= cxt->max_ftrace_cnt) + return -EINVAL; + prz = cxt->fprzs[record->id]; + break; + case PSTORE_TYPE_PMSG: + prz = cxt->mprz; + break; + default: + return -EINVAL; + } + + persistent_ram_free_old(prz); + persistent_ram_zap(prz); + + return 0; +} + +static struct ramoops_context oops_cxt = { + .pstore = { + .owner = THIS_MODULE, + .name = "ramoops", + .open = ramoops_pstore_open, + .read = ramoops_pstore_read, + .write = ramoops_pstore_write, + .write_user = ramoops_pstore_write_user, + .erase = ramoops_pstore_erase, + }, +}; + +static void ramoops_free_przs(struct ramoops_context *cxt) +{ + int i; + + /* Free pmsg PRZ */ + persistent_ram_free(&cxt->mprz); + + /* Free console PRZ */ + persistent_ram_free(&cxt->cprz); + + /* Free dump PRZs */ + if (cxt->dprzs) { + for (i = 0; i < cxt->max_dump_cnt; i++) + persistent_ram_free(&cxt->dprzs[i]); + + kfree(cxt->dprzs); + cxt->dprzs = NULL; + cxt->max_dump_cnt = 0; + } + + /* Free ftrace PRZs */ + if (cxt->fprzs) { + for (i = 0; i < cxt->max_ftrace_cnt; i++) + persistent_ram_free(&cxt->fprzs[i]); + kfree(cxt->fprzs); + cxt->fprzs = NULL; + cxt->max_ftrace_cnt = 0; + } +} + +static int ramoops_init_przs(const char *name, + struct device *dev, struct ramoops_context *cxt, + struct persistent_ram_zone ***przs, + phys_addr_t *paddr, size_t mem_sz, + ssize_t record_size, + unsigned int *cnt, u32 sig, u32 flags) +{ + int err = -ENOMEM; + int i; + size_t zone_sz; + struct persistent_ram_zone **prz_ar; + + /* Allocate nothing for 0 mem_sz or 0 record_size. */ + if (mem_sz == 0 || record_size == 0) { + *cnt = 0; + return 0; + } + + /* + * If we have a negative record size, calculate it based on + * mem_sz / *cnt. If we have a positive record size, calculate + * cnt from mem_sz / record_size. + */ + if (record_size < 0) { + if (*cnt == 0) + return 0; + record_size = mem_sz / *cnt; + if (record_size == 0) { + dev_err(dev, "%s record size == 0 (%zu / %u)\n", + name, mem_sz, *cnt); + goto fail; + } + } else { + *cnt = mem_sz / record_size; + if (*cnt == 0) { + dev_err(dev, "%s record count == 0 (%zu / %zu)\n", + name, mem_sz, record_size); + goto fail; + } + } + + if (*paddr + mem_sz - cxt->phys_addr > cxt->size) { + dev_err(dev, "no room for %s mem region (0x%zx@0x%llx) in (0x%lx@0x%llx)\n", + name, + mem_sz, (unsigned long long)*paddr, + cxt->size, (unsigned long long)cxt->phys_addr); + goto fail; + } + + zone_sz = mem_sz / *cnt; + zone_sz = ALIGN_DOWN(zone_sz, 2); + if (!zone_sz) { + dev_err(dev, "%s zone size == 0\n", name); + goto fail; + } + + prz_ar = kzalloc_objs(**przs, *cnt); + if (!prz_ar) + goto fail; + + for (i = 0; i < *cnt; i++) { + char *label; + + if (*cnt == 1) + label = kasprintf(GFP_KERNEL, "ramoops:%s", name); + else + label = kasprintf(GFP_KERNEL, "ramoops:%s(%d/%d)", + name, i, *cnt - 1); + prz_ar[i] = persistent_ram_new(*paddr, zone_sz, sig, + &cxt->ecc_info, + cxt->memtype, flags, label); + kfree(label); + if (IS_ERR(prz_ar[i])) { + err = PTR_ERR(prz_ar[i]); + dev_err(dev, "failed to request %s mem region (0x%zx@0x%llx): %d\n", + name, record_size, + (unsigned long long)*paddr, err); + + while (i > 0) { + i--; + persistent_ram_free(&prz_ar[i]); + } + kfree(prz_ar); + prz_ar = NULL; + goto fail; + } + *paddr += zone_sz; + prz_ar[i]->type = pstore_name_to_type(name); + } + + *przs = prz_ar; + return 0; + +fail: + *cnt = 0; + return err; +} + +static int ramoops_init_prz(const char *name, + struct device *dev, struct ramoops_context *cxt, + struct persistent_ram_zone **prz, + phys_addr_t *paddr, size_t sz, u32 sig) +{ + char *label; + + if (!sz) + return 0; + + if (*paddr + sz - cxt->phys_addr > cxt->size) { + dev_err(dev, "no room for %s mem region (0x%zx@0x%llx) in (0x%lx@0x%llx)\n", + name, sz, (unsigned long long)*paddr, + cxt->size, (unsigned long long)cxt->phys_addr); + return -ENOMEM; + } + + label = kasprintf(GFP_KERNEL, "ramoops:%s", name); + *prz = persistent_ram_new(*paddr, sz, sig, &cxt->ecc_info, + cxt->memtype, PRZ_FLAG_ZAP_OLD, label); + kfree(label); + if (IS_ERR(*prz)) { + int err = PTR_ERR(*prz); + + dev_err(dev, "failed to request %s mem region (0x%zx@0x%llx): %d\n", + name, sz, (unsigned long long)*paddr, err); + return err; + } + + *paddr += sz; + (*prz)->type = pstore_name_to_type(name); + + return 0; +} + +/* Read a u32 from a dt property and make sure it's safe for an int. */ +static int ramoops_parse_dt_u32(struct platform_device *pdev, + const char *propname, + u32 default_value, u32 *value) +{ + u32 val32 = 0; + int ret; + + ret = of_property_read_u32(pdev->dev.of_node, propname, &val32); + if (ret == -EINVAL) { + /* field is missing, use default value. */ + val32 = default_value; + } else if (ret < 0) { + dev_err(&pdev->dev, "failed to parse property %s: %d\n", + propname, ret); + return ret; + } + + /* Sanity check our results. */ + if (val32 > INT_MAX) { + dev_err(&pdev->dev, "%s %u > INT_MAX\n", propname, val32); + return -EOVERFLOW; + } + + *value = val32; + return 0; +} + +static int ramoops_parse_dt(struct platform_device *pdev, + struct ramoops_platform_data *pdata) +{ + struct device_node *of_node = pdev->dev.of_node; + struct device_node *parent_node; + struct resource *res; + u32 value; + int ret; + + dev_dbg(&pdev->dev, "using Device Tree\n"); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) { + dev_err(&pdev->dev, + "failed to locate DT /reserved-memory resource\n"); + return -EINVAL; + } + + pdata->mem_size = resource_size(res); + pdata->mem_address = res->start; + /* + * Setting "unbuffered" is deprecated and will be ignored if + * "mem_type" is also specified. + */ + pdata->mem_type = of_property_read_bool(of_node, "unbuffered"); + /* + * Setting "no-dump-oops" is deprecated and will be ignored if + * "max_reason" is also specified. + */ + if (of_property_read_bool(of_node, "no-dump-oops")) + pdata->max_reason = KMSG_DUMP_PANIC; + else + pdata->max_reason = KMSG_DUMP_OOPS; + +#define parse_u32(name, field, default_value) { \ + ret = ramoops_parse_dt_u32(pdev, name, default_value, \ + &value); \ + if (ret < 0) \ + return ret; \ + field = value; \ + } + + parse_u32("mem-type", pdata->mem_type, pdata->mem_type); + parse_u32("record-size", pdata->record_size, 0); + parse_u32("console-size", pdata->console_size, 0); + parse_u32("ftrace-size", pdata->ftrace_size, 0); + parse_u32("pmsg-size", pdata->pmsg_size, 0); + parse_u32("ecc-size", pdata->ecc_info.ecc_size, 0); + parse_u32("flags", pdata->flags, 0); + parse_u32("max-reason", pdata->max_reason, pdata->max_reason); + +#undef parse_u32 + + /* + * Some old Chromebooks relied on the kernel setting the + * console_size and pmsg_size to the record size since that's + * what the downstream kernel did. These same Chromebooks had + * "ramoops" straight under the root node which isn't + * according to the current upstream bindings (though it was + * arguably acceptable under a prior version of the bindings). + * Let's make those old Chromebooks work by detecting that + * we're not a child of "reserved-memory" and mimicking the + * expected behavior. + */ + parent_node = of_get_parent(of_node); + if (!of_node_name_eq(parent_node, "reserved-memory") && + !pdata->console_size && !pdata->ftrace_size && + !pdata->pmsg_size && !pdata->ecc_info.ecc_size) { + pdata->console_size = pdata->record_size; + pdata->pmsg_size = pdata->record_size; + } + of_node_put(parent_node); + + return 0; +} + +static int ramoops_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct ramoops_platform_data *pdata = dev->platform_data; + struct ramoops_platform_data pdata_local; + struct ramoops_context *cxt = &oops_cxt; + size_t dump_mem_sz; + phys_addr_t paddr; + int err = -EINVAL; + + /* + * Only a single ramoops area allowed at a time, so fail extra + * probes. + */ + if (cxt->max_dump_cnt) { + pr_err("already initialized\n"); + goto fail_out; + } + + if (dev_of_node(dev) && !pdata) { + pdata = &pdata_local; + memset(pdata, 0, sizeof(*pdata)); + + err = ramoops_parse_dt(pdev, pdata); + if (err < 0) + goto fail_out; + } + + /* Make sure we didn't get bogus platform data pointer. */ + if (!pdata) { + pr_err("NULL platform data\n"); + err = -EINVAL; + goto fail_out; + } + + if (!pdata->mem_size || (!pdata->record_size && !pdata->console_size && + !pdata->ftrace_size && !pdata->pmsg_size)) { + pr_err("The memory size and the record/console size must be " + "non-zero\n"); + err = -EINVAL; + goto fail_out; + } + + if (pdata->record_size && !is_power_of_2(pdata->record_size)) + pdata->record_size = rounddown_pow_of_two(pdata->record_size); + if (pdata->console_size && !is_power_of_2(pdata->console_size)) + pdata->console_size = rounddown_pow_of_two(pdata->console_size); + if (pdata->ftrace_size && !is_power_of_2(pdata->ftrace_size)) + pdata->ftrace_size = rounddown_pow_of_two(pdata->ftrace_size); + if (pdata->pmsg_size && !is_power_of_2(pdata->pmsg_size)) + pdata->pmsg_size = rounddown_pow_of_two(pdata->pmsg_size); + + cxt->size = pdata->mem_size; + cxt->phys_addr = pdata->mem_address; + cxt->memtype = pdata->mem_type; + cxt->record_size = pdata->record_size; + cxt->console_size = pdata->console_size; + cxt->ftrace_size = pdata->ftrace_size; + cxt->pmsg_size = pdata->pmsg_size; + cxt->flags = pdata->flags; + cxt->ecc_info = pdata->ecc_info; + + paddr = cxt->phys_addr; + + dump_mem_sz = cxt->size - cxt->console_size - cxt->ftrace_size + - cxt->pmsg_size; + err = ramoops_init_przs("dmesg", dev, cxt, &cxt->dprzs, &paddr, + dump_mem_sz, cxt->record_size, + &cxt->max_dump_cnt, 0, 0); + if (err) + goto fail_init; + + err = ramoops_init_prz("console", dev, cxt, &cxt->cprz, &paddr, + cxt->console_size, 0); + if (err) + goto fail_init; + + err = ramoops_init_prz("pmsg", dev, cxt, &cxt->mprz, &paddr, + cxt->pmsg_size, 0); + if (err) + goto fail_init; + + cxt->max_ftrace_cnt = (cxt->flags & RAMOOPS_FLAG_FTRACE_PER_CPU) + ? nr_cpu_ids + : 1; + err = ramoops_init_przs("ftrace", dev, cxt, &cxt->fprzs, &paddr, + cxt->ftrace_size, -1, + &cxt->max_ftrace_cnt, LINUX_VERSION_CODE, + (cxt->flags & RAMOOPS_FLAG_FTRACE_PER_CPU) + ? PRZ_FLAG_NO_LOCK : 0); + if (err) + goto fail_init; + + cxt->pstore.data = cxt; + /* + * Prepare frontend flags based on which areas are initialized. + * For ramoops_init_przs() cases, the "max count" variable tells + * if there are regions present. For ramoops_init_prz() cases, + * the single region size is how to check. + */ + cxt->pstore.flags = 0; + if (cxt->max_dump_cnt) { + cxt->pstore.flags |= PSTORE_FLAGS_DMESG; + cxt->pstore.max_reason = pdata->max_reason; + } + if (cxt->console_size) + cxt->pstore.flags |= PSTORE_FLAGS_CONSOLE; + if (cxt->max_ftrace_cnt) + cxt->pstore.flags |= PSTORE_FLAGS_FTRACE; + if (cxt->pmsg_size) + cxt->pstore.flags |= PSTORE_FLAGS_PMSG; + + /* + * Since bufsize is only used for dmesg crash dumps, it + * must match the size of the dprz record (after PRZ header + * and ECC bytes have been accounted for). + */ + if (cxt->pstore.flags & PSTORE_FLAGS_DMESG) { + cxt->pstore.bufsize = cxt->dprzs[0]->buffer_size; + cxt->pstore.buf = kvzalloc(cxt->pstore.bufsize, GFP_KERNEL); + if (!cxt->pstore.buf) { + pr_err("cannot allocate pstore crash dump buffer\n"); + err = -ENOMEM; + goto fail_clear; + } + } + + err = pstore_register(&cxt->pstore); + if (err) { + pr_err("registering with pstore failed\n"); + goto fail_buf; + } + + /* + * Update the module parameter variables as well so they are visible + * through /sys/module/ramoops/parameters/ + */ + mem_size = pdata->mem_size; + mem_address = pdata->mem_address; + record_size = pdata->record_size; + ramoops_max_reason = pdata->max_reason; + ramoops_console_size = pdata->console_size; + ramoops_pmsg_size = pdata->pmsg_size; + ramoops_ftrace_size = pdata->ftrace_size; + mem_type = pdata->mem_type; + ramoops_ecc = pdata->ecc_info.ecc_size; + + pr_info("using 0x%lx@0x%llx, ecc: %d\n", + cxt->size, (unsigned long long)cxt->phys_addr, + cxt->ecc_info.ecc_size); + + return 0; + +fail_buf: + kvfree(cxt->pstore.buf); +fail_clear: + cxt->pstore.bufsize = 0; +fail_init: + ramoops_free_przs(cxt); +fail_out: + return err; +} + +static void ramoops_remove(struct platform_device *pdev) +{ + struct ramoops_context *cxt = &oops_cxt; + + pstore_unregister(&cxt->pstore); + + kvfree(cxt->pstore.buf); + cxt->pstore.bufsize = 0; + + ramoops_free_przs(cxt); +} + +static const struct of_device_id dt_match[] = { + { .compatible = "ramoops" }, + {} +}; +MODULE_DEVICE_TABLE(of, dt_match); + +static struct platform_driver ramoops_driver = { + .probe = ramoops_probe, + .remove = ramoops_remove, + .driver = { + .name = "ramoops", + .of_match_table = dt_match, + }, +}; + +static inline void ramoops_unregister_dummy(void) +{ + platform_device_unregister(dummy); + dummy = NULL; +} + +static void __init ramoops_register_dummy(void) +{ + struct ramoops_platform_data pdata; + + if (mem_name) { + phys_addr_t start; + phys_addr_t size; + + if (reserve_mem_find_by_name(mem_name, &start, &size)) { + mem_address = start; + mem_size = size; + } + } + + /* + * Prepare a dummy platform data structure to carry the module + * parameters. If mem_size isn't set, then there are no module + * parameters, and we can skip this. + */ + if (!mem_size) + return; + + pr_info("using module parameters\n"); + + memset(&pdata, 0, sizeof(pdata)); + pdata.mem_size = mem_size; + pdata.mem_address = mem_address; + pdata.mem_type = mem_type; + pdata.record_size = record_size; + pdata.console_size = ramoops_console_size; + pdata.ftrace_size = ramoops_ftrace_size; + pdata.pmsg_size = ramoops_pmsg_size; + /* If "max_reason" is set, its value has priority over "dump_oops". */ + if (ramoops_max_reason >= 0) + pdata.max_reason = ramoops_max_reason; + /* Otherwise, if "dump_oops" is set, parse it into "max_reason". */ + else if (ramoops_dump_oops != -1) + pdata.max_reason = ramoops_dump_oops ? KMSG_DUMP_OOPS + : KMSG_DUMP_PANIC; + /* And if neither are explicitly set, use the default. */ + else + pdata.max_reason = KMSG_DUMP_OOPS; + pdata.flags = RAMOOPS_FLAG_FTRACE_PER_CPU; + + /* + * For backwards compatibility ramoops.ecc=1 means 16 bytes ECC + * (using 1 byte for ECC isn't much of use anyway). + */ + pdata.ecc_info.ecc_size = ramoops_ecc == 1 ? 16 : ramoops_ecc; + + dummy = platform_device_register_data(NULL, "ramoops", -1, + &pdata, sizeof(pdata)); + if (IS_ERR(dummy)) { + pr_info("could not create platform device: %ld\n", + PTR_ERR(dummy)); + dummy = NULL; + } +} + +static int __init ramoops_init(void) +{ + int ret; + + ramoops_register_dummy(); + ret = platform_driver_register(&ramoops_driver); + if (ret != 0) + ramoops_unregister_dummy(); + + return ret; +} +postcore_initcall(ramoops_init); + +static void __exit ramoops_exit(void) +{ + platform_driver_unregister(&ramoops_driver); + ramoops_unregister_dummy(); +} +module_exit(ramoops_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Marco Stornelli "); +MODULE_DESCRIPTION("RAM Oops/Panic logger/driver"); diff --git a/fs/pstore/ram_core.c b/fs/pstore/ram_core.c new file mode 100644 index 000000000..b4ec8009b --- /dev/null +++ b/fs/pstore/ram_core.c @@ -0,0 +1,642 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2012 Google, Inc. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ram_internal.h" + +/** + * struct persistent_ram_buffer - persistent circular RAM buffer + * + * @sig: Signature to indicate header (PERSISTENT_RAM_SIG xor PRZ-type value) + * @start: First valid byte in the buffer. + * @size: Number of valid bytes in the buffer. + * @data: The contents of the buffer. + */ +struct persistent_ram_buffer { + uint32_t sig; + atomic_t start; + atomic_t size; + uint8_t data[]; +}; + +#define PERSISTENT_RAM_SIG (0x43474244) /* DBGC */ + +static inline size_t buffer_size(struct persistent_ram_zone *prz) +{ + return atomic_read(&prz->buffer->size); +} + +static inline size_t buffer_start(struct persistent_ram_zone *prz) +{ + return atomic_read(&prz->buffer->start); +} + +/* increase and wrap the start pointer, returning the old value */ +static size_t buffer_start_add(struct persistent_ram_zone *prz, size_t a) +{ + int old; + int new; + unsigned long flags = 0; + + if (!(prz->flags & PRZ_FLAG_NO_LOCK)) + raw_spin_lock_irqsave(&prz->buffer_lock, flags); + + old = atomic_read(&prz->buffer->start); + new = old + a; + while (unlikely(new >= prz->buffer_size)) + new -= prz->buffer_size; + atomic_set(&prz->buffer->start, new); + + if (!(prz->flags & PRZ_FLAG_NO_LOCK)) + raw_spin_unlock_irqrestore(&prz->buffer_lock, flags); + + return old; +} + +/* increase the size counter until it hits the max size */ +static void buffer_size_add(struct persistent_ram_zone *prz, size_t a) +{ + size_t old; + size_t new; + unsigned long flags = 0; + + if (!(prz->flags & PRZ_FLAG_NO_LOCK)) + raw_spin_lock_irqsave(&prz->buffer_lock, flags); + + old = atomic_read(&prz->buffer->size); + if (old == prz->buffer_size) + goto exit; + + new = old + a; + if (new > prz->buffer_size) + new = prz->buffer_size; + atomic_set(&prz->buffer->size, new); + +exit: + if (!(prz->flags & PRZ_FLAG_NO_LOCK)) + raw_spin_unlock_irqrestore(&prz->buffer_lock, flags); +} + +static void notrace persistent_ram_encode_rs8(struct persistent_ram_zone *prz, + uint8_t *data, size_t len, uint8_t *ecc) +{ + int i; + + /* Initialize the parity buffer */ + memset(prz->ecc_info.par, 0, + prz->ecc_info.ecc_size * sizeof(prz->ecc_info.par[0])); + encode_rs8(prz->rs_decoder, data, len, prz->ecc_info.par, 0); + for (i = 0; i < prz->ecc_info.ecc_size; i++) + ecc[i] = prz->ecc_info.par[i]; +} + +static int persistent_ram_decode_rs8(struct persistent_ram_zone *prz, + void *data, size_t len, uint8_t *ecc) +{ + int i; + + for (i = 0; i < prz->ecc_info.ecc_size; i++) + prz->ecc_info.par[i] = ecc[i]; + return decode_rs8(prz->rs_decoder, data, prz->ecc_info.par, len, + NULL, 0, NULL, 0, NULL); +} + +static void notrace persistent_ram_update_ecc(struct persistent_ram_zone *prz, + unsigned int start, unsigned int count) +{ + struct persistent_ram_buffer *buffer = prz->buffer; + uint8_t *buffer_end = buffer->data + prz->buffer_size; + uint8_t *block; + uint8_t *par; + int ecc_block_size = prz->ecc_info.block_size; + int ecc_size = prz->ecc_info.ecc_size; + int size = ecc_block_size; + + if (!ecc_size) + return; + + block = buffer->data + (start & ~(ecc_block_size - 1)); + par = prz->par_buffer + (start / ecc_block_size) * ecc_size; + + do { + if (block + ecc_block_size > buffer_end) + size = buffer_end - block; + persistent_ram_encode_rs8(prz, block, size, par); + block += ecc_block_size; + par += ecc_size; + } while (block < buffer->data + start + count); +} + +static void persistent_ram_update_header_ecc(struct persistent_ram_zone *prz) +{ + struct persistent_ram_buffer *buffer = prz->buffer; + + if (!prz->ecc_info.ecc_size) + return; + + persistent_ram_encode_rs8(prz, (uint8_t *)buffer, sizeof(*buffer), + prz->par_header); +} + +static void persistent_ram_ecc_old(struct persistent_ram_zone *prz) +{ + struct persistent_ram_buffer *buffer = prz->buffer; + uint8_t *block; + uint8_t *par; + + if (!prz->ecc_info.ecc_size) + return; + + block = buffer->data; + par = prz->par_buffer; + while (block < buffer->data + buffer_size(prz)) { + int numerr; + int size = prz->ecc_info.block_size; + if (block + size > buffer->data + prz->buffer_size) + size = buffer->data + prz->buffer_size - block; + numerr = persistent_ram_decode_rs8(prz, block, size, par); + if (numerr > 0) { + pr_devel("error in block %p, %d\n", block, numerr); + prz->corrected_bytes += numerr; + } else if (numerr < 0) { + pr_devel("uncorrectable error in block %p\n", block); + prz->bad_blocks++; + } + block += prz->ecc_info.block_size; + par += prz->ecc_info.ecc_size; + } +} + +static int persistent_ram_init_ecc(struct persistent_ram_zone *prz, + struct persistent_ram_ecc_info *ecc_info) +{ + int numerr; + struct persistent_ram_buffer *buffer = prz->buffer; + size_t ecc_blocks; + size_t ecc_total; + + if (!ecc_info || !ecc_info->ecc_size) + return 0; + + prz->ecc_info.block_size = ecc_info->block_size ?: 128; + prz->ecc_info.ecc_size = ecc_info->ecc_size ?: 16; + prz->ecc_info.symsize = ecc_info->symsize ?: 8; + prz->ecc_info.poly = ecc_info->poly ?: 0x11d; + + ecc_blocks = DIV_ROUND_UP(prz->buffer_size - prz->ecc_info.ecc_size, + prz->ecc_info.block_size + + prz->ecc_info.ecc_size); + ecc_total = (ecc_blocks + 1) * prz->ecc_info.ecc_size; + if (ecc_total >= prz->buffer_size) { + pr_err("%s: invalid ecc_size %u (total %zu, buffer size %zu)\n", + __func__, prz->ecc_info.ecc_size, + ecc_total, prz->buffer_size); + return -EINVAL; + } + + prz->buffer_size -= ecc_total; + prz->par_buffer = buffer->data + prz->buffer_size; + prz->par_header = prz->par_buffer + + ecc_blocks * prz->ecc_info.ecc_size; + + /* + * first consecutive root is 0 + * primitive element to generate roots = 1 + */ + prz->rs_decoder = init_rs(prz->ecc_info.symsize, prz->ecc_info.poly, + 0, 1, prz->ecc_info.ecc_size); + if (prz->rs_decoder == NULL) { + pr_info("init_rs failed\n"); + return -EINVAL; + } + + /* allocate workspace instead of using stack VLA */ + prz->ecc_info.par = kmalloc_objs(*prz->ecc_info.par, + prz->ecc_info.ecc_size); + if (!prz->ecc_info.par) { + pr_err("cannot allocate ECC parity workspace\n"); + return -ENOMEM; + } + + prz->corrected_bytes = 0; + prz->bad_blocks = 0; + + numerr = persistent_ram_decode_rs8(prz, buffer, sizeof(*buffer), + prz->par_header); + if (numerr > 0) { + pr_info("error in header, %d\n", numerr); + prz->corrected_bytes += numerr; + } else if (numerr < 0) { + pr_info_ratelimited("uncorrectable error in header\n"); + prz->bad_blocks++; + } + + return 0; +} + +ssize_t persistent_ram_ecc_string(struct persistent_ram_zone *prz, + char *str, size_t len) +{ + ssize_t ret; + + if (!prz->ecc_info.ecc_size) + return 0; + + if (prz->corrected_bytes || prz->bad_blocks) + ret = snprintf(str, len, "" + "\nECC: %d Corrected bytes, %d unrecoverable blocks\n", + prz->corrected_bytes, prz->bad_blocks); + else + ret = snprintf(str, len, "\nECC: No errors detected\n"); + + return ret; +} + +static void notrace persistent_ram_update(struct persistent_ram_zone *prz, + const void *s, unsigned int start, unsigned int count) +{ + struct persistent_ram_buffer *buffer = prz->buffer; + memcpy_toio(buffer->data + start, s, count); + persistent_ram_update_ecc(prz, start, count); +} + +static int notrace persistent_ram_update_user(struct persistent_ram_zone *prz, + const void __user *s, unsigned int start, unsigned int count) +{ + struct persistent_ram_buffer *buffer = prz->buffer; + int ret = unlikely(copy_from_user(buffer->data + start, s, count)) ? + -EFAULT : 0; + persistent_ram_update_ecc(prz, start, count); + return ret; +} + +void persistent_ram_save_old(struct persistent_ram_zone *prz) +{ + struct persistent_ram_buffer *buffer = prz->buffer; + size_t size = buffer_size(prz); + size_t start = buffer_start(prz); + + if (!size) + return; + + /* + * If the existing buffer is differently sized, free it so a new + * one is allocated. This can happen when persistent_ram_save_old() + * is called early in boot and later for a timer-triggered + * survivable crash when the crash dumps don't match in size + * (which would be extremely unlikely given kmsg buffers usually + * exceed prz buffer sizes). + */ + if (prz->old_log && prz->old_log_size != size) + persistent_ram_free_old(prz); + + if (!prz->old_log) { + persistent_ram_ecc_old(prz); + prz->old_log = kvzalloc(size, GFP_KERNEL); + } + if (!prz->old_log) { + pr_err("failed to allocate buffer\n"); + return; + } + + prz->old_log_size = size; + memcpy_fromio(prz->old_log, &buffer->data[start], size - start); + memcpy_fromio(prz->old_log + size - start, &buffer->data[0], start); +} + +int notrace persistent_ram_write(struct persistent_ram_zone *prz, + const void *s, unsigned int count) +{ + int rem; + int c = count; + size_t start; + + if (unlikely(c > prz->buffer_size)) { + s += c - prz->buffer_size; + c = prz->buffer_size; + } + + buffer_size_add(prz, c); + + start = buffer_start_add(prz, c); + + rem = prz->buffer_size - start; + if (unlikely(rem < c)) { + persistent_ram_update(prz, s, start, rem); + s += rem; + c -= rem; + start = 0; + } + persistent_ram_update(prz, s, start, c); + + persistent_ram_update_header_ecc(prz); + + return count; +} + +int notrace persistent_ram_write_user(struct persistent_ram_zone *prz, + const void __user *s, unsigned int count) +{ + int rem, ret = 0, c = count; + size_t start; + + if (unlikely(c > prz->buffer_size)) { + s += c - prz->buffer_size; + c = prz->buffer_size; + } + + buffer_size_add(prz, c); + + start = buffer_start_add(prz, c); + + rem = prz->buffer_size - start; + if (unlikely(rem < c)) { + ret = persistent_ram_update_user(prz, s, start, rem); + s += rem; + c -= rem; + start = 0; + } + if (likely(!ret)) + ret = persistent_ram_update_user(prz, s, start, c); + + persistent_ram_update_header_ecc(prz); + + return unlikely(ret) ? ret : count; +} + +size_t persistent_ram_old_size(struct persistent_ram_zone *prz) +{ + return prz->old_log_size; +} + +void *persistent_ram_old(struct persistent_ram_zone *prz) +{ + return prz->old_log; +} + +void persistent_ram_free_old(struct persistent_ram_zone *prz) +{ + kvfree(prz->old_log); + prz->old_log = NULL; + prz->old_log_size = 0; +} + +void persistent_ram_zap(struct persistent_ram_zone *prz) +{ + atomic_set(&prz->buffer->start, 0); + atomic_set(&prz->buffer->size, 0); + persistent_ram_update_header_ecc(prz); +} + +#define MEM_TYPE_WCOMBINE 0 +#define MEM_TYPE_NONCACHED 1 +#define MEM_TYPE_NORMAL 2 + +static void *persistent_ram_vmap(phys_addr_t start, size_t size, + unsigned int memtype) +{ + struct page **pages; + phys_addr_t page_start; + unsigned int page_count; + pgprot_t prot; + unsigned int i; + void *vaddr; + + page_start = start - offset_in_page(start); + page_count = DIV_ROUND_UP(size + offset_in_page(start), PAGE_SIZE); + + switch (memtype) { + case MEM_TYPE_NORMAL: + prot = PAGE_KERNEL; + break; + case MEM_TYPE_NONCACHED: + prot = pgprot_noncached(PAGE_KERNEL); + break; + case MEM_TYPE_WCOMBINE: + prot = pgprot_writecombine(PAGE_KERNEL); + break; + default: + pr_err("invalid mem_type=%d\n", memtype); + return NULL; + } + + pages = kmalloc_objs(struct page *, page_count); + if (!pages) { + pr_err("%s: Failed to allocate array for %u pages\n", + __func__, page_count); + return NULL; + } + + for (i = 0; i < page_count; i++) { + phys_addr_t addr = page_start + i * PAGE_SIZE; + pages[i] = pfn_to_page(addr >> PAGE_SHIFT); + } + /* + * VM_IOREMAP used here to bypass this region during vread_iter() + * and kmap_atomic() (i.e. kcore) to avoid __va() failures. + */ + vaddr = vmap(pages, page_count, VM_MAP | VM_IOREMAP, prot); + kfree(pages); + + /* + * vmap() may fail and return NULL. Do not add the offset in this + * case, otherwise a NULL mapping would appear successful. + */ + if (!vaddr) + return NULL; + + /* + * Since vmap() uses page granularity, we must add the offset + * into the page here, to get the byte granularity address + * into the mapping to represent the actual "start" location. + */ + return vaddr + offset_in_page(start); +} + +static void *persistent_ram_iomap(phys_addr_t start, size_t size, + unsigned int memtype, char *label) +{ + void *va; + + if (!request_mem_region(start, size, label ?: "ramoops")) { + pr_err("request mem region (%s 0x%llx@0x%llx) failed\n", + label ?: "ramoops", + (unsigned long long)size, (unsigned long long)start); + return NULL; + } + + if (memtype) + va = ioremap(start, size); + else + va = ioremap_wc(start, size); + + /* We must release the mem region if ioremap fails. */ + if (!va) + release_mem_region(start, size); + + /* + * Since request_mem_region() and ioremap() are byte-granularity + * there is no need handle anything special like we do when the + * vmap() case in persistent_ram_vmap() above. + */ + return va; +} + +static int persistent_ram_buffer_map(phys_addr_t start, phys_addr_t size, + struct persistent_ram_zone *prz, int memtype) +{ + prz->paddr = start; + prz->size = size; + + if (pfn_valid(start >> PAGE_SHIFT)) + prz->vaddr = persistent_ram_vmap(start, size, memtype); + else + prz->vaddr = persistent_ram_iomap(start, size, memtype, + prz->label); + + if (!prz->vaddr) { + pr_err("%s: Failed to map 0x%llx pages at 0x%llx\n", __func__, + (unsigned long long)size, (unsigned long long)start); + return -ENOMEM; + } + + prz->buffer = prz->vaddr; + prz->buffer_size = size - sizeof(struct persistent_ram_buffer); + + return 0; +} + +static int persistent_ram_post_init(struct persistent_ram_zone *prz, u32 sig, + struct persistent_ram_ecc_info *ecc_info) +{ + int ret; + bool zap = !!(prz->flags & PRZ_FLAG_ZAP_OLD); + + ret = persistent_ram_init_ecc(prz, ecc_info); + if (ret) { + pr_warn("ECC failed %s\n", prz->label); + return ret; + } + + sig ^= PERSISTENT_RAM_SIG; + + if (prz->buffer->sig == sig) { + if (buffer_size(prz) == 0 && buffer_start(prz) == 0) { + pr_debug("found existing empty buffer\n"); + return 0; + } + + if (buffer_size(prz) > prz->buffer_size || + buffer_start(prz) > buffer_size(prz)) { + pr_info("found existing invalid buffer, size %zu, start %zu\n", + buffer_size(prz), buffer_start(prz)); + zap = true; + } else { + pr_debug("found existing buffer, size %zu, start %zu\n", + buffer_size(prz), buffer_start(prz)); + persistent_ram_save_old(prz); + } + } else { + pr_debug("no valid data in buffer (sig = 0x%08x)\n", + prz->buffer->sig); + prz->buffer->sig = sig; + zap = true; + } + + /* Reset missing, invalid, or single-use memory area. */ + if (zap) + persistent_ram_zap(prz); + + return 0; +} + +void persistent_ram_free(struct persistent_ram_zone **_prz) +{ + struct persistent_ram_zone *prz; + + if (!_prz) + return; + + prz = *_prz; + if (!prz) + return; + + if (prz->vaddr) { + if (pfn_valid(prz->paddr >> PAGE_SHIFT)) { + /* We must vunmap() at page-granularity. */ + vunmap(prz->vaddr - offset_in_page(prz->paddr)); + } else { + iounmap(prz->vaddr); + release_mem_region(prz->paddr, prz->size); + } + prz->vaddr = NULL; + } + if (prz->rs_decoder) { + free_rs(prz->rs_decoder); + prz->rs_decoder = NULL; + } + kfree(prz->ecc_info.par); + prz->ecc_info.par = NULL; + + persistent_ram_free_old(prz); + kfree(prz->label); + kfree(prz); + *_prz = NULL; +} + +struct persistent_ram_zone *persistent_ram_new(phys_addr_t start, size_t size, + u32 sig, struct persistent_ram_ecc_info *ecc_info, + unsigned int memtype, u32 flags, char *label) +{ + struct persistent_ram_zone *prz; + int ret = -ENOMEM; + + prz = kzalloc_obj(struct persistent_ram_zone); + if (!prz) { + pr_err("failed to allocate persistent ram zone\n"); + goto err; + } + + /* Initialize general buffer state. */ + raw_spin_lock_init(&prz->buffer_lock); + prz->flags = flags; + prz->label = kstrdup(label, GFP_KERNEL); + if (!prz->label) + goto err; + + ret = persistent_ram_buffer_map(start, size, prz, memtype); + if (ret) + goto err; + + ret = persistent_ram_post_init(prz, sig, ecc_info); + if (ret) + goto err; + + pr_debug("attached %s 0x%zx@0x%llx: %zu header, %zu data, %zu ecc (%d/%d)\n", + prz->label, prz->size, (unsigned long long)prz->paddr, + sizeof(*prz->buffer), prz->buffer_size, + prz->size - sizeof(*prz->buffer) - prz->buffer_size, + prz->ecc_info.ecc_size, prz->ecc_info.block_size); + + return prz; +err: + persistent_ram_free(&prz); + return ERR_PTR(ret); +} diff --git a/fs/pstore/ram_internal.h b/fs/pstore/ram_internal.h new file mode 100644 index 000000000..5f6946983 --- /dev/null +++ b/fs/pstore/ram_internal.h @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2010 Marco Stornelli + * Copyright (C) 2011 Kees Cook + * Copyright (C) 2011 Google, Inc. + */ + +#include + +/* + * Choose whether access to the RAM zone requires locking or not. If a zone + * can be written to from different CPUs like with ftrace for example, then + * PRZ_FLAG_NO_LOCK is used. For all other cases, locking is required. + */ +#define PRZ_FLAG_NO_LOCK BIT(0) +/* + * If a PRZ should only have a single-boot lifetime, this marks it as + * getting wiped after its contents get copied out after boot. + */ +#define PRZ_FLAG_ZAP_OLD BIT(1) + +/** + * struct persistent_ram_zone - Details of a persistent RAM zone (PRZ) + * used as a pstore backend + * + * @paddr: physical address of the mapped RAM area + * @size: size of mapping + * @label: unique name of this PRZ + * @type: frontend type for this PRZ + * @flags: holds PRZ_FLAGS_* bits + * + * @buffer_lock: + * locks access to @buffer "size" bytes and "start" offset + * @buffer: + * pointer to actual RAM area managed by this PRZ + * @buffer_size: + * bytes in @buffer->data (not including any trailing ECC bytes) + * + * @par_buffer: + * pointer into @buffer->data containing ECC bytes for @buffer->data + * @par_header: + * pointer into @buffer->data containing ECC bytes for @buffer header + * (i.e. all fields up to @data) + * @rs_decoder: + * RSLIB instance for doing ECC calculations + * @corrected_bytes: + * ECC corrected bytes accounting since boot + * @bad_blocks: + * ECC uncorrectable bytes accounting since boot + * @ecc_info: + * ECC configuration details + * + * @old_log: + * saved copy of @buffer->data prior to most recent wipe + * @old_log_size: + * bytes contained in @old_log + * + */ +struct persistent_ram_zone { + phys_addr_t paddr; + size_t size; + void *vaddr; + char *label; + enum pstore_type_id type; + u32 flags; + + raw_spinlock_t buffer_lock; + struct persistent_ram_buffer *buffer; + size_t buffer_size; + + char *par_buffer; + char *par_header; + struct rs_control *rs_decoder; + int corrected_bytes; + int bad_blocks; + struct persistent_ram_ecc_info ecc_info; + + char *old_log; + size_t old_log_size; +}; + +struct persistent_ram_zone *persistent_ram_new(phys_addr_t start, size_t size, + u32 sig, struct persistent_ram_ecc_info *ecc_info, + unsigned int memtype, u32 flags, char *label); +void persistent_ram_free(struct persistent_ram_zone **_prz); +void persistent_ram_zap(struct persistent_ram_zone *prz); + +int persistent_ram_write(struct persistent_ram_zone *prz, const void *s, + unsigned int count); +int persistent_ram_write_user(struct persistent_ram_zone *prz, + const void __user *s, unsigned int count); + +void persistent_ram_save_old(struct persistent_ram_zone *prz); +size_t persistent_ram_old_size(struct persistent_ram_zone *prz); +void *persistent_ram_old(struct persistent_ram_zone *prz); +void persistent_ram_free_old(struct persistent_ram_zone *prz); +ssize_t persistent_ram_ecc_string(struct persistent_ram_zone *prz, + char *str, size_t len); diff --git a/fs/pstore/zone.c b/fs/pstore/zone.c new file mode 100644 index 000000000..a3b003f9a --- /dev/null +++ b/fs/pstore/zone.c @@ -0,0 +1,1475 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Provide a pstore intermediate backend, organized into kernel memory + * allocated zones that are then mapped and flushed into a single + * contiguous region on a storage backend of some kind (block, mtd, etc). + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +/** + * struct psz_buffer - header of zone to flush to storage + * + * @sig: signature to indicate header (PSZ_SIG xor PSZONE-type value) + * @datalen: length of data in @data + * @start: offset into @data where the beginning of the stored bytes begin + * @data: zone data. + */ +struct psz_buffer { +#define PSZ_SIG (0x43474244) /* DBGC */ + uint32_t sig; + atomic_t datalen; + atomic_t start; + uint8_t data[]; +}; + +/** + * struct psz_kmsg_header - kmsg dump-specific header to flush to storage + * + * @magic: magic num for kmsg dump header + * @time: kmsg dump trigger time + * @compressed: whether compressed + * @counter: kmsg dump counter + * @reason: the kmsg dump reason (e.g. oops, panic, etc) + * @data: pointer to log data + * + * This is a sub-header for a kmsg dump, trailing after &psz_buffer. + */ +struct psz_kmsg_header { +#define PSTORE_KMSG_HEADER_MAGIC 0x4dfc3ae5 /* Just a random number */ + uint32_t magic; + struct timespec64 time; + bool compressed; + uint32_t counter; + enum kmsg_dump_reason reason; + uint8_t data[]; +}; + +/** + * struct pstore_zone - single stored buffer + * + * @off: zone offset of storage + * @type: front-end type for this zone + * @name: front-end name for this zone + * @buffer: pointer to data buffer managed by this zone + * @oldbuf: pointer to old data buffer + * @buffer_size: bytes in @buffer->data + * @should_recover: whether this zone should recover from storage + * @dirty: whether the data in @buffer dirty + * + * zone structure in memory. + */ +struct pstore_zone { + loff_t off; + const char *name; + enum pstore_type_id type; + + struct psz_buffer *buffer; + struct psz_buffer *oldbuf; + size_t buffer_size; + bool should_recover; + atomic_t dirty; +}; + +/** + * struct psz_context - all about running state of pstore/zone + * + * @kpszs: kmsg dump storage zones + * @ppsz: pmsg storage zone + * @cpsz: console storage zone + * @fpszs: ftrace storage zones + * @kmsg_max_cnt: max count of @kpszs + * @kmsg_read_cnt: counter of total read kmsg dumps + * @kmsg_write_cnt: counter of total kmsg dump writes + * @pmsg_read_cnt: counter of total read pmsg zone + * @console_read_cnt: counter of total read console zone + * @ftrace_max_cnt: max count of @fpszs + * @ftrace_read_cnt: counter of max read ftrace zone + * @oops_counter: counter of oops dumps + * @panic_counter: counter of panic dumps + * @recovered: whether finished recovering data from storage + * @on_panic: whether panic is happening + * @pstore_zone_info_lock: lock to @pstore_zone_info + * @pstore_zone_info: information from backend + * @pstore: structure for pstore + */ +struct psz_context { + struct pstore_zone **kpszs; + struct pstore_zone *ppsz; + struct pstore_zone *cpsz; + struct pstore_zone **fpszs; + unsigned int kmsg_max_cnt; + unsigned int kmsg_read_cnt; + unsigned int kmsg_write_cnt; + unsigned int pmsg_read_cnt; + unsigned int console_read_cnt; + unsigned int ftrace_max_cnt; + unsigned int ftrace_read_cnt; + /* + * These counters should be calculated during recovery. + * It records the oops/panic times after crashes rather than boots. + */ + unsigned int oops_counter; + unsigned int panic_counter; + atomic_t recovered; + atomic_t on_panic; + + /* + * pstore_zone_info_lock protects this entire structure during calls + * to register_pstore_zone()/unregister_pstore_zone(). + */ + struct mutex pstore_zone_info_lock; + struct pstore_zone_info *pstore_zone_info; + struct pstore_info pstore; +}; +static struct psz_context pstore_zone_cxt; + +static void psz_flush_all_dirty_zones(struct work_struct *); +static DECLARE_DELAYED_WORK(psz_cleaner, psz_flush_all_dirty_zones); + +/** + * enum psz_flush_mode - flush mode for psz_zone_write() + * + * @FLUSH_NONE: do not flush to storage but update data on memory + * @FLUSH_PART: just flush part of data including meta data to storage + * @FLUSH_META: just flush meta data of zone to storage + * @FLUSH_ALL: flush all of zone + */ +enum psz_flush_mode { + FLUSH_NONE = 0, + FLUSH_PART, + FLUSH_META, + FLUSH_ALL, +}; + +static inline int buffer_datalen(struct pstore_zone *zone) +{ + return atomic_read(&zone->buffer->datalen); +} + +static inline int buffer_start(struct pstore_zone *zone) +{ + return atomic_read(&zone->buffer->start); +} + +static inline bool is_on_panic(void) +{ + return atomic_read(&pstore_zone_cxt.on_panic); +} + +static ssize_t psz_zone_read_buffer(struct pstore_zone *zone, char *buf, + size_t len, unsigned long off) +{ + if (!buf || !zone || !zone->buffer) + return -EINVAL; + if (off > zone->buffer_size) + return -EINVAL; + len = min_t(size_t, len, zone->buffer_size - off); + memcpy(buf, zone->buffer->data + off, len); + return len; +} + +static int psz_zone_read_oldbuf(struct pstore_zone *zone, char *buf, + size_t len, unsigned long off) +{ + if (!buf || !zone || !zone->oldbuf) + return -EINVAL; + if (off > zone->buffer_size) + return -EINVAL; + len = min_t(size_t, len, zone->buffer_size - off); + memcpy(buf, zone->oldbuf->data + off, len); + return 0; +} + +static int psz_zone_write(struct pstore_zone *zone, + enum psz_flush_mode flush_mode, const char *buf, + size_t len, unsigned long off) +{ + struct pstore_zone_info *info = pstore_zone_cxt.pstore_zone_info; + ssize_t wcnt = 0; + ssize_t (*writeop)(const char *buf, size_t bytes, loff_t pos); + size_t wlen; + + if (off > zone->buffer_size) + return -EINVAL; + + wlen = min_t(size_t, len, zone->buffer_size - off); + if (buf && wlen) { + memcpy(zone->buffer->data + off, buf, wlen); + atomic_set(&zone->buffer->datalen, wlen + off); + } + + /* avoid damaging old records */ + if (!is_on_panic() && !atomic_read(&pstore_zone_cxt.recovered)) + goto dirty; + + writeop = is_on_panic() ? info->panic_write : info->write; + if (!writeop) + goto dirty; + + switch (flush_mode) { + case FLUSH_NONE: + if (unlikely(buf && wlen)) + goto dirty; + return 0; + case FLUSH_PART: + wcnt = writeop((const char *)zone->buffer->data + off, wlen, + zone->off + sizeof(*zone->buffer) + off); + if (wcnt != wlen) + goto dirty; + fallthrough; + case FLUSH_META: + wlen = sizeof(struct psz_buffer); + wcnt = writeop((const char *)zone->buffer, wlen, zone->off); + if (wcnt != wlen) + goto dirty; + break; + case FLUSH_ALL: + wlen = zone->buffer_size + sizeof(*zone->buffer); + wcnt = writeop((const char *)zone->buffer, wlen, zone->off); + if (wcnt != wlen) + goto dirty; + break; + } + + return 0; +dirty: + /* no need to mark it dirty if going to try next zone */ + if (wcnt == -ENOMSG) + return -ENOMSG; + atomic_set(&zone->dirty, true); + /* flush dirty zones nicely */ + if (wcnt == -EBUSY && !is_on_panic()) + schedule_delayed_work(&psz_cleaner, msecs_to_jiffies(500)); + return -EBUSY; +} + +static int psz_flush_dirty_zone(struct pstore_zone *zone) +{ + int ret; + + if (unlikely(!zone)) + return -EINVAL; + + if (unlikely(!atomic_read(&pstore_zone_cxt.recovered))) + return -EBUSY; + + if (!atomic_xchg(&zone->dirty, false)) + return 0; + + ret = psz_zone_write(zone, FLUSH_ALL, NULL, 0, 0); + if (ret) + atomic_set(&zone->dirty, true); + return ret; +} + +static int psz_flush_dirty_zones(struct pstore_zone **zones, unsigned int cnt) +{ + int i, ret; + struct pstore_zone *zone; + + if (!zones) + return -EINVAL; + + for (i = 0; i < cnt; i++) { + zone = zones[i]; + if (!zone) + return -EINVAL; + ret = psz_flush_dirty_zone(zone); + if (ret) + return ret; + } + return 0; +} + +static int psz_move_zone(struct pstore_zone *old, struct pstore_zone *new) +{ + const char *data = (const char *)old->buffer->data; + int ret; + + ret = psz_zone_write(new, FLUSH_ALL, data, buffer_datalen(old), 0); + if (ret) { + atomic_set(&new->buffer->datalen, 0); + atomic_set(&new->dirty, false); + return ret; + } + atomic_set(&old->buffer->datalen, 0); + return 0; +} + +static void psz_flush_all_dirty_zones(struct work_struct *work) +{ + struct psz_context *cxt = &pstore_zone_cxt; + int ret = 0; + + if (cxt->ppsz) + ret |= psz_flush_dirty_zone(cxt->ppsz); + if (cxt->cpsz) + ret |= psz_flush_dirty_zone(cxt->cpsz); + if (cxt->kpszs) + ret |= psz_flush_dirty_zones(cxt->kpszs, cxt->kmsg_max_cnt); + if (cxt->fpszs) + ret |= psz_flush_dirty_zones(cxt->fpszs, cxt->ftrace_max_cnt); + if (ret && cxt->pstore_zone_info) + schedule_delayed_work(&psz_cleaner, msecs_to_jiffies(1000)); +} + +static int psz_kmsg_recover_data(struct psz_context *cxt) +{ + struct pstore_zone_info *info = cxt->pstore_zone_info; + struct pstore_zone *zone = NULL; + struct psz_buffer *buf; + unsigned long i; + ssize_t rcnt; + + if (!info->read) + return -EINVAL; + + for (i = 0; i < cxt->kmsg_max_cnt; i++) { + zone = cxt->kpszs[i]; + if (unlikely(!zone)) + return -EINVAL; + if (atomic_read(&zone->dirty)) { + unsigned int wcnt = cxt->kmsg_write_cnt; + struct pstore_zone *new = cxt->kpszs[wcnt]; + int ret; + + ret = psz_move_zone(zone, new); + if (ret) { + pr_err("move zone from %lu to %d failed\n", + i, wcnt); + return ret; + } + cxt->kmsg_write_cnt = (wcnt + 1) % cxt->kmsg_max_cnt; + } + if (!zone->should_recover) + continue; + buf = zone->buffer; + rcnt = info->read((char *)buf, zone->buffer_size + sizeof(*buf), + zone->off); + if (rcnt != zone->buffer_size + sizeof(*buf)) + return rcnt < 0 ? rcnt : -EIO; + } + return 0; +} + +static int psz_kmsg_recover_meta(struct psz_context *cxt) +{ + struct pstore_zone_info *info = cxt->pstore_zone_info; + struct pstore_zone *zone; + ssize_t rcnt, len; + struct psz_buffer *buf; + struct psz_kmsg_header *hdr; + struct timespec64 time = { }; + unsigned long i; + /* + * Recover may happen on panic, we can't allocate any memory by kmalloc. + * So, we use local array instead. + */ + char buffer_header[sizeof(*buf) + sizeof(*hdr)] = {0}; + + if (!info->read) + return -EINVAL; + + len = sizeof(*buf) + sizeof(*hdr); + buf = (struct psz_buffer *)buffer_header; + for (i = 0; i < cxt->kmsg_max_cnt; i++) { + zone = cxt->kpszs[i]; + if (unlikely(!zone)) + return -EINVAL; + + rcnt = info->read((char *)buf, len, zone->off); + if (rcnt == -ENOMSG) { + pr_debug("%s with id %lu may be broken, skip\n", + zone->name, i); + continue; + } else if (rcnt != len) { + pr_err("read %s with id %lu failed\n", zone->name, i); + return rcnt < 0 ? rcnt : -EIO; + } + + if (buf->sig != zone->buffer->sig) { + pr_debug("no valid data in kmsg dump zone %lu\n", i); + continue; + } + + if (zone->buffer_size < atomic_read(&buf->datalen)) { + pr_info("found overtop zone: %s: id %lu, off %lld, size %zu\n", + zone->name, i, zone->off, + zone->buffer_size); + continue; + } + + hdr = (struct psz_kmsg_header *)buf->data; + if (hdr->magic != PSTORE_KMSG_HEADER_MAGIC) { + pr_info("found invalid zone: %s: id %lu, off %lld, size %zu\n", + zone->name, i, zone->off, + zone->buffer_size); + continue; + } + + /* + * we get the newest zone, and the next one must be the oldest + * or unused zone, because we do write one by one like a circle. + */ + if (hdr->time.tv_sec >= time.tv_sec) { + time.tv_sec = hdr->time.tv_sec; + cxt->kmsg_write_cnt = (i + 1) % cxt->kmsg_max_cnt; + } + + if (hdr->reason == KMSG_DUMP_OOPS) + cxt->oops_counter = + max(cxt->oops_counter, hdr->counter); + else if (hdr->reason == KMSG_DUMP_PANIC) + cxt->panic_counter = + max(cxt->panic_counter, hdr->counter); + + if (!atomic_read(&buf->datalen)) { + pr_debug("found erased zone: %s: id %lu, off %lld, size %zu, datalen %d\n", + zone->name, i, zone->off, + zone->buffer_size, + atomic_read(&buf->datalen)); + continue; + } + + if (!is_on_panic()) + zone->should_recover = true; + pr_debug("found nice zone: %s: id %lu, off %lld, size %zu, datalen %d\n", + zone->name, i, zone->off, + zone->buffer_size, atomic_read(&buf->datalen)); + } + + return 0; +} + +static int psz_kmsg_recover(struct psz_context *cxt) +{ + int ret; + + if (!cxt->kpszs) + return 0; + + ret = psz_kmsg_recover_meta(cxt); + if (ret) + goto recover_fail; + + ret = psz_kmsg_recover_data(cxt); + if (ret) + goto recover_fail; + + return 0; +recover_fail: + pr_debug("psz_recover_kmsg failed\n"); + return ret; +} + +static int psz_recover_zone(struct psz_context *cxt, struct pstore_zone *zone) +{ + struct pstore_zone_info *info = cxt->pstore_zone_info; + struct psz_buffer *oldbuf, tmpbuf; + int ret = 0; + char *buf; + ssize_t rcnt, len, start, off; + + if (!zone || zone->oldbuf) + return 0; + + if (is_on_panic()) { + /* save data as much as possible */ + psz_flush_dirty_zone(zone); + return 0; + } + + if (unlikely(!info->read)) + return -EINVAL; + + len = sizeof(struct psz_buffer); + rcnt = info->read((char *)&tmpbuf, len, zone->off); + if (rcnt != len) { + pr_debug("read zone %s failed\n", zone->name); + return rcnt < 0 ? rcnt : -EIO; + } + + if (tmpbuf.sig != zone->buffer->sig) { + pr_debug("no valid data in zone %s\n", zone->name); + return 0; + } + + if (zone->buffer_size < atomic_read(&tmpbuf.datalen) || + zone->buffer_size < atomic_read(&tmpbuf.start)) { + pr_info("found overtop zone: %s: off %lld, size %zu\n", + zone->name, zone->off, zone->buffer_size); + /* just keep going */ + return 0; + } + + if (!atomic_read(&tmpbuf.datalen)) { + pr_debug("found erased zone: %s: off %lld, size %zu, datalen %d\n", + zone->name, zone->off, zone->buffer_size, + atomic_read(&tmpbuf.datalen)); + return 0; + } + + pr_debug("found nice zone: %s: off %lld, size %zu, datalen %d\n", + zone->name, zone->off, zone->buffer_size, + atomic_read(&tmpbuf.datalen)); + + len = atomic_read(&tmpbuf.datalen) + sizeof(*oldbuf); + oldbuf = kzalloc(len, GFP_KERNEL); + if (!oldbuf) + return -ENOMEM; + + memcpy(oldbuf, &tmpbuf, sizeof(*oldbuf)); + buf = (char *)oldbuf + sizeof(*oldbuf); + len = atomic_read(&oldbuf->datalen); + start = atomic_read(&oldbuf->start); + off = zone->off + sizeof(*oldbuf); + + /* get part of data */ + rcnt = info->read(buf, len - start, off + start); + if (rcnt != len - start) { + pr_err("read zone %s failed\n", zone->name); + ret = rcnt < 0 ? rcnt : -EIO; + goto free_oldbuf; + } + + /* get the rest of data */ + rcnt = info->read(buf + len - start, start, off); + if (rcnt != start) { + pr_err("read zone %s failed\n", zone->name); + ret = rcnt < 0 ? rcnt : -EIO; + goto free_oldbuf; + } + + zone->oldbuf = oldbuf; + psz_flush_dirty_zone(zone); + return 0; + +free_oldbuf: + kfree(oldbuf); + return ret; +} + +static int psz_recover_zones(struct psz_context *cxt, + struct pstore_zone **zones, unsigned int cnt) +{ + int ret; + unsigned int i; + struct pstore_zone *zone; + + if (!zones) + return 0; + + for (i = 0; i < cnt; i++) { + zone = zones[i]; + if (unlikely(!zone)) + continue; + ret = psz_recover_zone(cxt, zone); + if (ret) + goto recover_fail; + } + + return 0; +recover_fail: + pr_debug("recover %s[%u] failed\n", zone->name, i); + return ret; +} + +/** + * psz_recovery() - recover data from storage + * @cxt: the context of pstore/zone + * + * recovery means reading data back from storage after rebooting + * + * Return: 0 on success, others on failure. + */ +static inline int psz_recovery(struct psz_context *cxt) +{ + int ret; + + if (atomic_read(&cxt->recovered)) + return 0; + + ret = psz_kmsg_recover(cxt); + if (ret) + goto out; + + ret = psz_recover_zone(cxt, cxt->ppsz); + if (ret) + goto out; + + ret = psz_recover_zone(cxt, cxt->cpsz); + if (ret) + goto out; + + ret = psz_recover_zones(cxt, cxt->fpszs, cxt->ftrace_max_cnt); + +out: + if (unlikely(ret)) + pr_err("recover failed\n"); + else { + pr_debug("recover end!\n"); + atomic_set(&cxt->recovered, 1); + } + return ret; +} + +static int psz_pstore_open(struct pstore_info *psi) +{ + struct psz_context *cxt = psi->data; + + cxt->kmsg_read_cnt = 0; + cxt->pmsg_read_cnt = 0; + cxt->console_read_cnt = 0; + cxt->ftrace_read_cnt = 0; + return 0; +} + +static inline bool psz_old_ok(struct pstore_zone *zone) +{ + if (zone && zone->oldbuf && atomic_read(&zone->oldbuf->datalen)) + return true; + return false; +} + +static inline bool psz_ok(struct pstore_zone *zone) +{ + if (zone && zone->buffer && buffer_datalen(zone)) + return true; + return false; +} + +static inline int psz_kmsg_erase(struct psz_context *cxt, + struct pstore_zone *zone, struct pstore_record *record) +{ + struct psz_buffer *buffer = zone->buffer; + struct psz_kmsg_header *hdr = + (struct psz_kmsg_header *)buffer->data; + size_t size; + + if (unlikely(!psz_ok(zone))) + return 0; + + /* this zone is already updated, no need to erase */ + if (record->count != hdr->counter) + return 0; + + size = buffer_datalen(zone) + sizeof(*zone->buffer); + atomic_set(&zone->buffer->datalen, 0); + if (cxt->pstore_zone_info->erase) + return cxt->pstore_zone_info->erase(size, zone->off); + else + return psz_zone_write(zone, FLUSH_META, NULL, 0, 0); +} + +static inline int psz_record_erase(struct psz_context *cxt, + struct pstore_zone *zone) +{ + if (unlikely(!psz_old_ok(zone))) + return 0; + + kfree(zone->oldbuf); + zone->oldbuf = NULL; + /* + * if there are new data in zone buffer, that means the old data + * are already invalid. It is no need to flush 0 (erase) to + * block device. + */ + if (!buffer_datalen(zone)) + return psz_zone_write(zone, FLUSH_META, NULL, 0, 0); + psz_flush_dirty_zone(zone); + return 0; +} + +static int psz_pstore_erase(struct pstore_record *record) +{ + struct psz_context *cxt = record->psi->data; + + switch (record->type) { + case PSTORE_TYPE_DMESG: + if (record->id >= cxt->kmsg_max_cnt) + return -EINVAL; + return psz_kmsg_erase(cxt, cxt->kpszs[record->id], record); + case PSTORE_TYPE_PMSG: + return psz_record_erase(cxt, cxt->ppsz); + case PSTORE_TYPE_CONSOLE: + return psz_record_erase(cxt, cxt->cpsz); + case PSTORE_TYPE_FTRACE: + if (record->id >= cxt->ftrace_max_cnt) + return -EINVAL; + return psz_record_erase(cxt, cxt->fpszs[record->id]); + default: return -EINVAL; + } +} + +static void psz_write_kmsg_hdr(struct pstore_zone *zone, + struct pstore_record *record) +{ + struct psz_context *cxt = record->psi->data; + struct psz_buffer *buffer = zone->buffer; + struct psz_kmsg_header *hdr = + (struct psz_kmsg_header *)buffer->data; + + hdr->magic = PSTORE_KMSG_HEADER_MAGIC; + hdr->compressed = record->compressed; + hdr->time.tv_sec = record->time.tv_sec; + hdr->time.tv_nsec = record->time.tv_nsec; + hdr->reason = record->reason; + if (hdr->reason == KMSG_DUMP_OOPS) + hdr->counter = ++cxt->oops_counter; + else if (hdr->reason == KMSG_DUMP_PANIC) + hdr->counter = ++cxt->panic_counter; + else + hdr->counter = 0; +} + +/* + * In case zone is broken, which may occur to MTD device, we try each zones, + * start at cxt->kmsg_write_cnt. + */ +static inline int notrace psz_kmsg_write_record(struct psz_context *cxt, + struct pstore_record *record) +{ + size_t size, hlen; + struct pstore_zone *zone; + unsigned int i; + + for (i = 0; i < cxt->kmsg_max_cnt; i++) { + unsigned int zonenum, len; + int ret; + + zonenum = (cxt->kmsg_write_cnt + i) % cxt->kmsg_max_cnt; + zone = cxt->kpszs[zonenum]; + if (unlikely(!zone)) + return -ENOSPC; + + /* avoid destroying old data, allocate a new one */ + len = zone->buffer_size + sizeof(*zone->buffer); + zone->oldbuf = zone->buffer; + zone->buffer = kzalloc(len, GFP_ATOMIC); + if (!zone->buffer) { + zone->buffer = zone->oldbuf; + return -ENOMEM; + } + zone->buffer->sig = zone->oldbuf->sig; + + pr_debug("write %s to zone id %d\n", zone->name, zonenum); + psz_write_kmsg_hdr(zone, record); + hlen = sizeof(struct psz_kmsg_header); + size = min_t(size_t, record->size, zone->buffer_size - hlen); + ret = psz_zone_write(zone, FLUSH_ALL, record->buf, size, hlen); + if (likely(!ret || ret != -ENOMSG)) { + cxt->kmsg_write_cnt = zonenum + 1; + cxt->kmsg_write_cnt %= cxt->kmsg_max_cnt; + /* no need to try next zone, free last zone buffer */ + kfree(zone->oldbuf); + zone->oldbuf = NULL; + return ret; + } + + pr_debug("zone %u may be broken, try next dmesg zone\n", + zonenum); + kfree(zone->buffer); + zone->buffer = zone->oldbuf; + zone->oldbuf = NULL; + } + + return -EBUSY; +} + +static int notrace psz_kmsg_write(struct psz_context *cxt, + struct pstore_record *record) +{ + int ret; + + /* + * Explicitly only take the first part of any new crash. + * If our buffer is larger than kmsg_bytes, this can never happen, + * and if our buffer is smaller than kmsg_bytes, we don't want the + * report split across multiple records. + */ + if (record->part != 1) + return -ENOSPC; + + if (!cxt->kpszs) + return -ENOSPC; + + ret = psz_kmsg_write_record(cxt, record); + if (!ret && is_on_panic()) { + /* ensure all data are flushed to storage when panic */ + pr_debug("try to flush other dirty zones\n"); + psz_flush_all_dirty_zones(NULL); + } + + /* always return 0 as we had handled it on buffer */ + return 0; +} + +static int notrace psz_record_write(struct pstore_zone *zone, + struct pstore_record *record) +{ + size_t start, rem; + bool is_full_data = false; + char *buf; + int cnt; + + if (!zone || !record) + return -ENOSPC; + + if (atomic_read(&zone->buffer->datalen) >= zone->buffer_size) + is_full_data = true; + + cnt = record->size; + buf = record->buf; + if (unlikely(cnt > zone->buffer_size)) { + buf += cnt - zone->buffer_size; + cnt = zone->buffer_size; + } + + start = buffer_start(zone); + rem = zone->buffer_size - start; + if (unlikely(rem < cnt)) { + psz_zone_write(zone, FLUSH_PART, buf, rem, start); + buf += rem; + cnt -= rem; + start = 0; + is_full_data = true; + } + + atomic_set(&zone->buffer->start, cnt + start); + psz_zone_write(zone, FLUSH_PART, buf, cnt, start); + + /** + * psz_zone_write will set datalen as start + cnt. + * It works if actual data length is lesser than buffer size. + * If data length is greater than buffer size, pmsg will rewrite to + * the beginning of the zone, which makes buffer->datalen wrong. + * So we should reset datalen as buffer size once actual data length + * is greater than buffer size. + */ + if (is_full_data) { + atomic_set(&zone->buffer->datalen, zone->buffer_size); + psz_zone_write(zone, FLUSH_META, NULL, 0, 0); + } + return 0; +} + +static int notrace psz_pstore_write(struct pstore_record *record) +{ + struct psz_context *cxt = record->psi->data; + + if (record->type == PSTORE_TYPE_DMESG && + record->reason == KMSG_DUMP_PANIC) + atomic_set(&cxt->on_panic, 1); + + /* + * If on panic, do not write anything except panic records. + * Fix the case when panic_write prints log that wakes up + * console backend. + */ + if (is_on_panic() && record->type != PSTORE_TYPE_DMESG) + return -EBUSY; + + switch (record->type) { + case PSTORE_TYPE_DMESG: + return psz_kmsg_write(cxt, record); + case PSTORE_TYPE_CONSOLE: + return psz_record_write(cxt->cpsz, record); + case PSTORE_TYPE_PMSG: + return psz_record_write(cxt->ppsz, record); + case PSTORE_TYPE_FTRACE: { + int zonenum = smp_processor_id(); + + if (!cxt->fpszs) + return -ENOSPC; + return psz_record_write(cxt->fpszs[zonenum], record); + } + default: + return -EINVAL; + } +} + +static struct pstore_zone *psz_read_next_zone(struct psz_context *cxt) +{ + struct pstore_zone *zone = NULL; + + while (cxt->kmsg_read_cnt < cxt->kmsg_max_cnt) { + zone = cxt->kpszs[cxt->kmsg_read_cnt++]; + if (psz_ok(zone)) + return zone; + } + + if (cxt->ftrace_read_cnt < cxt->ftrace_max_cnt) + /* + * No need psz_old_ok(). Let psz_ftrace_read() do so for + * combination. psz_ftrace_read() should traverse over + * all zones in case of some zone without data. + */ + return cxt->fpszs[cxt->ftrace_read_cnt++]; + + if (cxt->pmsg_read_cnt == 0) { + cxt->pmsg_read_cnt++; + zone = cxt->ppsz; + if (psz_old_ok(zone)) + return zone; + } + + if (cxt->console_read_cnt == 0) { + cxt->console_read_cnt++; + zone = cxt->cpsz; + if (psz_old_ok(zone)) + return zone; + } + + return NULL; +} + +static int psz_kmsg_read_hdr(struct pstore_zone *zone, + struct pstore_record *record) +{ + struct psz_buffer *buffer = zone->buffer; + struct psz_kmsg_header *hdr = + (struct psz_kmsg_header *)buffer->data; + + if (hdr->magic != PSTORE_KMSG_HEADER_MAGIC) + return -EINVAL; + record->compressed = hdr->compressed; + record->time.tv_sec = hdr->time.tv_sec; + record->time.tv_nsec = hdr->time.tv_nsec; + record->reason = hdr->reason; + record->count = hdr->counter; + return 0; +} + +static ssize_t psz_kmsg_read(struct pstore_zone *zone, + struct pstore_record *record) +{ + ssize_t size, hlen = 0; + + size = buffer_datalen(zone); + /* Clear and skip this kmsg dump record if it has no valid header */ + if (psz_kmsg_read_hdr(zone, record)) { + atomic_set(&zone->buffer->datalen, 0); + atomic_set(&zone->dirty, 0); + return -ENOMSG; + } + size -= sizeof(struct psz_kmsg_header); + + if (!record->compressed) { + char *buf = kasprintf(GFP_KERNEL, "%s: Total %d times\n", + kmsg_dump_reason_str(record->reason), + record->count); + if (!buf) + return -ENOMEM; + hlen = strlen(buf); + record->buf = krealloc(buf, hlen + size, GFP_KERNEL); + if (!record->buf) { + kfree(buf); + return -ENOMEM; + } + } else { + record->buf = kmalloc(size, GFP_KERNEL); + if (!record->buf) + return -ENOMEM; + } + + size = psz_zone_read_buffer(zone, record->buf + hlen, size, + sizeof(struct psz_kmsg_header)); + if (unlikely(size < 0)) { + kfree(record->buf); + return -ENOMSG; + } + + return size + hlen; +} + +/* try to combine all ftrace zones */ +static ssize_t psz_ftrace_read(struct pstore_zone *zone, + struct pstore_record *record) +{ + struct psz_context *cxt; + struct psz_buffer *buf; + int ret; + + if (!zone || !record) + return -ENOSPC; + + if (!psz_old_ok(zone)) + goto out; + + buf = (struct psz_buffer *)zone->oldbuf; + if (!buf) + return -ENOMSG; + + ret = pstore_ftrace_combine_log(&record->buf, &record->size, + (char *)buf->data, atomic_read(&buf->datalen)); + if (unlikely(ret)) + return ret; + +out: + cxt = record->psi->data; + if (cxt->ftrace_read_cnt < cxt->ftrace_max_cnt) + /* then, read next ftrace zone */ + return -ENOMSG; + record->id = 0; + return record->size ? record->size : -ENOMSG; +} + +static ssize_t psz_record_read(struct pstore_zone *zone, + struct pstore_record *record) +{ + size_t len; + struct psz_buffer *buf; + + if (!zone || !record) + return -ENOSPC; + + buf = (struct psz_buffer *)zone->oldbuf; + if (!buf) + return -ENOMSG; + + len = atomic_read(&buf->datalen); + record->buf = kmalloc(len, GFP_KERNEL); + if (!record->buf) + return -ENOMEM; + + if (unlikely(psz_zone_read_oldbuf(zone, record->buf, len, 0))) { + kfree(record->buf); + return -ENOMSG; + } + + return len; +} + +static ssize_t psz_pstore_read(struct pstore_record *record) +{ + struct psz_context *cxt = record->psi->data; + ssize_t (*readop)(struct pstore_zone *zone, + struct pstore_record *record); + struct pstore_zone *zone; + ssize_t ret; + + /* before read, we must recover from storage */ + ret = psz_recovery(cxt); + if (ret) + return ret; + +next_zone: + zone = psz_read_next_zone(cxt); + if (!zone) + return 0; + + record->type = zone->type; + switch (record->type) { + case PSTORE_TYPE_DMESG: + readop = psz_kmsg_read; + record->id = cxt->kmsg_read_cnt - 1; + break; + case PSTORE_TYPE_FTRACE: + readop = psz_ftrace_read; + break; + case PSTORE_TYPE_CONSOLE: + case PSTORE_TYPE_PMSG: + readop = psz_record_read; + break; + default: + goto next_zone; + } + + ret = readop(zone, record); + if (ret == -ENOMSG) + goto next_zone; + return ret; +} + +static struct psz_context pstore_zone_cxt = { + .pstore_zone_info_lock = + __MUTEX_INITIALIZER(pstore_zone_cxt.pstore_zone_info_lock), + .recovered = ATOMIC_INIT(0), + .on_panic = ATOMIC_INIT(0), + .pstore = { + .owner = THIS_MODULE, + .open = psz_pstore_open, + .read = psz_pstore_read, + .write = psz_pstore_write, + .erase = psz_pstore_erase, + }, +}; + +static void psz_free_zone(struct pstore_zone **pszone) +{ + struct pstore_zone *zone = *pszone; + + if (!zone) + return; + + kfree(zone->buffer); + kfree(zone); + *pszone = NULL; +} + +static void psz_free_zones(struct pstore_zone ***pszones, unsigned int *cnt) +{ + struct pstore_zone **zones = *pszones; + + if (!zones) + return; + + while (*cnt > 0) { + (*cnt)--; + psz_free_zone(&(zones[*cnt])); + } + kfree(zones); + *pszones = NULL; +} + +static void psz_free_all_zones(struct psz_context *cxt) +{ + if (cxt->kpszs) + psz_free_zones(&cxt->kpszs, &cxt->kmsg_max_cnt); + if (cxt->ppsz) + psz_free_zone(&cxt->ppsz); + if (cxt->cpsz) + psz_free_zone(&cxt->cpsz); + if (cxt->fpszs) + psz_free_zones(&cxt->fpszs, &cxt->ftrace_max_cnt); +} + +static struct pstore_zone *psz_init_zone(enum pstore_type_id type, + loff_t *off, size_t size) +{ + struct pstore_zone_info *info = pstore_zone_cxt.pstore_zone_info; + struct pstore_zone *zone; + const char *name = pstore_type_to_name(type); + + if (!size) + return NULL; + + if (*off + size > info->total_size) { + pr_err("no room for %s (0x%zx@0x%llx over 0x%lx)\n", + name, size, *off, info->total_size); + return ERR_PTR(-ENOMEM); + } + + zone = kzalloc_obj(struct pstore_zone); + if (!zone) + return ERR_PTR(-ENOMEM); + + zone->buffer = kmalloc(size, GFP_KERNEL); + if (!zone->buffer) { + kfree(zone); + return ERR_PTR(-ENOMEM); + } + memset(zone->buffer, 0xFF, size); + zone->off = *off; + zone->name = name; + zone->type = type; + zone->buffer_size = size - sizeof(struct psz_buffer); + zone->buffer->sig = type ^ PSZ_SIG; + zone->oldbuf = NULL; + atomic_set(&zone->dirty, 0); + atomic_set(&zone->buffer->datalen, 0); + atomic_set(&zone->buffer->start, 0); + + *off += size; + + pr_debug("pszone %s: off 0x%llx, %zu header, %zu data\n", zone->name, + zone->off, sizeof(*zone->buffer), zone->buffer_size); + return zone; +} + +static struct pstore_zone **psz_init_zones(enum pstore_type_id type, + loff_t *off, size_t total_size, ssize_t record_size, + unsigned int *cnt) +{ + struct pstore_zone_info *info = pstore_zone_cxt.pstore_zone_info; + struct pstore_zone **zones, *zone; + const char *name = pstore_type_to_name(type); + int c, i; + + *cnt = 0; + if (!total_size || !record_size) + return NULL; + + if (*off + total_size > info->total_size) { + pr_err("no room for zones %s (0x%zx@0x%llx over 0x%lx)\n", + name, total_size, *off, info->total_size); + return ERR_PTR(-ENOMEM); + } + + c = total_size / record_size; + if (unlikely(!c)) { + pr_err("zone %s total_size too small\n", name); + return ERR_PTR(-EINVAL); + } + + zones = kzalloc_objs(*zones, c); + if (!zones) { + pr_err("allocate for zones %s failed\n", name); + return ERR_PTR(-ENOMEM); + } + + for (i = 0; i < c; i++) { + zone = psz_init_zone(type, off, record_size); + if (!zone || IS_ERR(zone)) { + pr_err("initialize zones %s failed\n", name); + psz_free_zones(&zones, &i); + return (void *)zone; + } + zones[i] = zone; + } + + *cnt = c; + return zones; +} + +static int psz_alloc_zones(struct psz_context *cxt) +{ + struct pstore_zone_info *info = cxt->pstore_zone_info; + loff_t off = 0; + int err; + size_t off_size = 0; + + off_size += info->pmsg_size; + cxt->ppsz = psz_init_zone(PSTORE_TYPE_PMSG, &off, info->pmsg_size); + if (IS_ERR(cxt->ppsz)) { + err = PTR_ERR(cxt->ppsz); + cxt->ppsz = NULL; + goto free_out; + } + + off_size += info->console_size; + cxt->cpsz = psz_init_zone(PSTORE_TYPE_CONSOLE, &off, + info->console_size); + if (IS_ERR(cxt->cpsz)) { + err = PTR_ERR(cxt->cpsz); + cxt->cpsz = NULL; + goto free_out; + } + + off_size += info->ftrace_size; + cxt->fpszs = psz_init_zones(PSTORE_TYPE_FTRACE, &off, + info->ftrace_size, + info->ftrace_size / nr_cpu_ids, + &cxt->ftrace_max_cnt); + if (IS_ERR(cxt->fpszs)) { + err = PTR_ERR(cxt->fpszs); + cxt->fpszs = NULL; + goto free_out; + } + + cxt->kpszs = psz_init_zones(PSTORE_TYPE_DMESG, &off, + info->total_size - off_size, + info->kmsg_size, &cxt->kmsg_max_cnt); + if (IS_ERR(cxt->kpszs)) { + err = PTR_ERR(cxt->kpszs); + cxt->kpszs = NULL; + goto free_out; + } + + return 0; +free_out: + psz_free_all_zones(cxt); + return err; +} + +/** + * register_pstore_zone() - register to pstore/zone + * + * @info: back-end driver information. See &struct pstore_zone_info. + * + * Only one back-end at one time. + * + * Return: 0 on success, others on failure. + */ +int register_pstore_zone(struct pstore_zone_info *info) +{ + int err = -EINVAL; + struct psz_context *cxt = &pstore_zone_cxt; + + if (info->total_size < 4096) { + pr_warn("total_size must be >= 4096\n"); + return -EINVAL; + } + if (info->total_size > SZ_128M) { + pr_warn("capping size to 128MiB\n"); + info->total_size = SZ_128M; + } + + if (!info->kmsg_size && !info->pmsg_size && !info->console_size && + !info->ftrace_size) { + pr_warn("at least one record size must be non-zero\n"); + return -EINVAL; + } + + if (!info->name || !info->name[0]) + return -EINVAL; + +#define check_size(name, size) { \ + if (info->name > 0 && info->name < (size)) { \ + pr_err(#name " must be over %d\n", (size)); \ + return -EINVAL; \ + } \ + if (info->name & (size - 1)) { \ + pr_err(#name " must be a multiple of %d\n", \ + (size)); \ + return -EINVAL; \ + } \ + } + + check_size(total_size, 4096); + check_size(kmsg_size, SECTOR_SIZE); + check_size(pmsg_size, SECTOR_SIZE); + check_size(console_size, SECTOR_SIZE); + check_size(ftrace_size, SECTOR_SIZE); + +#undef check_size + + /* + * the @read and @write must be applied. + * if no @read, pstore may mount failed. + * if no @write, pstore do not support to remove record file. + */ + if (!info->read || !info->write) { + pr_err("no valid general read/write interface\n"); + return -EINVAL; + } + + mutex_lock(&cxt->pstore_zone_info_lock); + if (cxt->pstore_zone_info) { + pr_warn("'%s' already loaded: ignoring '%s'\n", + cxt->pstore_zone_info->name, info->name); + mutex_unlock(&cxt->pstore_zone_info_lock); + return -EBUSY; + } + cxt->pstore_zone_info = info; + + pr_debug("register %s with properties:\n", info->name); + pr_debug("\ttotal size : %ld Bytes\n", info->total_size); + pr_debug("\tkmsg size : %ld Bytes\n", info->kmsg_size); + pr_debug("\tpmsg size : %ld Bytes\n", info->pmsg_size); + pr_debug("\tconsole size : %ld Bytes\n", info->console_size); + pr_debug("\tftrace size : %ld Bytes\n", info->ftrace_size); + + err = psz_alloc_zones(cxt); + if (err) { + pr_err("alloc zones failed\n"); + goto fail_out; + } + + if (info->kmsg_size) { + cxt->pstore.bufsize = cxt->kpszs[0]->buffer_size - + sizeof(struct psz_kmsg_header); + cxt->pstore.buf = kzalloc(cxt->pstore.bufsize, GFP_KERNEL); + if (!cxt->pstore.buf) { + err = -ENOMEM; + goto fail_free; + } + } + cxt->pstore.data = cxt; + + pr_info("registered %s as backend for", info->name); + cxt->pstore.max_reason = info->max_reason; + cxt->pstore.name = info->name; + if (info->kmsg_size) { + cxt->pstore.flags |= PSTORE_FLAGS_DMESG; + pr_cont(" kmsg(%s", + kmsg_dump_reason_str(cxt->pstore.max_reason)); + if (cxt->pstore_zone_info->panic_write) + pr_cont(",panic_write"); + pr_cont(")"); + } + if (info->pmsg_size) { + cxt->pstore.flags |= PSTORE_FLAGS_PMSG; + pr_cont(" pmsg"); + } + if (info->console_size) { + cxt->pstore.flags |= PSTORE_FLAGS_CONSOLE; + pr_cont(" console"); + } + if (info->ftrace_size) { + cxt->pstore.flags |= PSTORE_FLAGS_FTRACE; + pr_cont(" ftrace"); + } + pr_cont("\n"); + + err = pstore_register(&cxt->pstore); + if (err) { + pr_err("registering with pstore failed\n"); + goto fail_free; + } + mutex_unlock(&pstore_zone_cxt.pstore_zone_info_lock); + + return 0; + +fail_free: + kfree(cxt->pstore.buf); + cxt->pstore.buf = NULL; + cxt->pstore.bufsize = 0; + psz_free_all_zones(cxt); +fail_out: + pstore_zone_cxt.pstore_zone_info = NULL; + mutex_unlock(&pstore_zone_cxt.pstore_zone_info_lock); + return err; +} +EXPORT_SYMBOL_GPL(register_pstore_zone); + +/** + * unregister_pstore_zone() - unregister to pstore/zone + * + * @info: back-end driver information. See struct pstore_zone_info. + */ +void unregister_pstore_zone(struct pstore_zone_info *info) +{ + struct psz_context *cxt = &pstore_zone_cxt; + + mutex_lock(&cxt->pstore_zone_info_lock); + if (!cxt->pstore_zone_info) { + mutex_unlock(&cxt->pstore_zone_info_lock); + return; + } + + /* Stop incoming writes from pstore. */ + pstore_unregister(&cxt->pstore); + + /* Flush any pending writes. */ + psz_flush_all_dirty_zones(NULL); + flush_delayed_work(&psz_cleaner); + + /* Clean up allocations. */ + kfree(cxt->pstore.buf); + cxt->pstore.buf = NULL; + cxt->pstore.bufsize = 0; + cxt->pstore_zone_info = NULL; + + psz_free_all_zones(cxt); + + /* Clear counters and zone state. */ + cxt->oops_counter = 0; + cxt->panic_counter = 0; + atomic_set(&cxt->recovered, 0); + atomic_set(&cxt->on_panic, 0); + + mutex_unlock(&cxt->pstore_zone_info_lock); +} +EXPORT_SYMBOL_GPL(unregister_pstore_zone); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("WeiXiong Liao "); +MODULE_AUTHOR("Kees Cook "); +MODULE_DESCRIPTION("Storage Manager for pstore/blk"); diff --git a/fs/qnx4/Kconfig b/fs/qnx4/Kconfig new file mode 100644 index 000000000..a2eb826e7 --- /dev/null +++ b/fs/qnx4/Kconfig @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0-only +config QNX4FS_FS + tristate "QNX4 file system support (read only)" + depends on BLOCK + select BUFFER_HEAD + help + This is the file system used by the real-time operating systems + QNX 4 and QNX 6 (the latter is also called QNX RTP). + Further information is available at . + Say Y if you intend to mount QNX hard disks or floppies. + + To compile this file system support as a module, choose M here: the + module will be called qnx4. + + If you don't know whether you need it, then you don't need it: + answer N. diff --git a/fs/qnx4/Makefile b/fs/qnx4/Makefile new file mode 100644 index 000000000..42d4fc263 --- /dev/null +++ b/fs/qnx4/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the linux qnx4-filesystem routines. +# + +obj-$(CONFIG_QNX4FS_FS) += qnx4.o + +qnx4-objs := inode.o dir.o namei.o bitmap.o diff --git a/fs/qnx4/bitmap.c b/fs/qnx4/bitmap.c new file mode 100644 index 000000000..163afc4ba --- /dev/null +++ b/fs/qnx4/bitmap.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * QNX4 file system, Linux implementation. + * + * Version : 0.2.1 + * + * Using parts of the xiafs filesystem. + * + * History : + * + * 28-05-1998 by Richard Frowijn : first release. + * 20-06-1998 by Frank Denis : basic optimisations. + * 25-06-1998 by Frank Denis : qnx4_is_free, qnx4_set_bitmap, qnx4_bmap . + * 28-06-1998 by Frank Denis : qnx4_free_inode (to be fixed) . + */ + +#include +#include +#include "qnx4.h" + +unsigned long qnx4_count_free_blocks(struct super_block *sb) +{ + int start = le32_to_cpu(qnx4_sb(sb)->BitMap->di_first_xtnt.xtnt_blk) - 1; + int total = 0; + int total_free = 0; + int offset = 0; + int size = le32_to_cpu(qnx4_sb(sb)->BitMap->di_size); + struct buffer_head *bh; + + while (total < size) { + int bytes = min(size - total, QNX4_BLOCK_SIZE); + + if ((bh = sb_bread(sb, start + offset)) == NULL) { + printk(KERN_ERR "qnx4: I/O error in counting free blocks\n"); + break; + } + total_free += bytes * BITS_PER_BYTE - + memweight(bh->b_data, bytes); + brelse(bh); + total += bytes; + offset++; + } + + return total_free; +} diff --git a/fs/qnx4/dir.c b/fs/qnx4/dir.c new file mode 100644 index 000000000..a9038d231 --- /dev/null +++ b/fs/qnx4/dir.c @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * QNX4 file system, Linux implementation. + * + * Version : 0.2.1 + * + * Using parts of the xiafs filesystem. + * + * History : + * + * 28-05-1998 by Richard Frowijn : first release. + * 20-06-1998 by Frank Denis : Linux 2.1.99+ & dcache support. + */ + +#include +#include +#include "qnx4.h" + +static int qnx4_readdir(struct file *file, struct dir_context *ctx) +{ + struct inode *inode = file_inode(file); + unsigned int offset; + struct buffer_head *bh; + unsigned long blknum; + int ix, ino; + int size; + + QNX4DEBUG((KERN_INFO "qnx4_readdir:i_size = %ld\n", (long) inode->i_size)); + QNX4DEBUG((KERN_INFO "pos = %ld\n", (long) ctx->pos)); + + while (ctx->pos < inode->i_size) { + blknum = qnx4_block_map(inode, ctx->pos >> QNX4_BLOCK_SIZE_BITS); + bh = sb_bread(inode->i_sb, blknum); + if (bh == NULL) { + printk(KERN_ERR "qnx4_readdir: bread failed (%ld)\n", blknum); + return 0; + } + ix = (ctx->pos >> QNX4_DIR_ENTRY_SIZE_BITS) % QNX4_INODES_PER_BLOCK; + for (; ix < QNX4_INODES_PER_BLOCK; ix++, ctx->pos += QNX4_DIR_ENTRY_SIZE) { + union qnx4_directory_entry *de; + const char *fname; + + offset = ix * QNX4_DIR_ENTRY_SIZE; + de = (union qnx4_directory_entry *) (bh->b_data + offset); + + fname = get_entry_fname(de, &size); + if (!fname) + continue; + + if (!(de->de_status & QNX4_FILE_LINK)) { + ino = blknum * QNX4_INODES_PER_BLOCK + ix - 1; + } else { + ino = ( le32_to_cpu(de->link.dl_inode_blk) - 1 ) * + QNX4_INODES_PER_BLOCK + + de->link.dl_inode_ndx; + } + + QNX4DEBUG((KERN_INFO "qnx4_readdir:%.*s\n", size, fname)); + if (!dir_emit(ctx, fname, size, ino, DT_UNKNOWN)) { + brelse(bh); + return 0; + } + } + brelse(bh); + } + return 0; +} + +const struct file_operations qnx4_dir_operations = +{ + .llseek = generic_file_llseek, + .read = generic_read_dir, + .iterate_shared = qnx4_readdir, + .fsync = simple_fsync, + .setlease = generic_setlease, +}; + +const struct inode_operations qnx4_dir_inode_operations = +{ + .lookup = qnx4_lookup, +}; diff --git a/fs/qnx4/inode.c b/fs/qnx4/inode.c new file mode 100644 index 000000000..42fcd500f --- /dev/null +++ b/fs/qnx4/inode.c @@ -0,0 +1,436 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * QNX4 file system, Linux implementation. + * + * Version : 0.2.1 + * + * Using parts of the xiafs filesystem. + * + * History : + * + * 01-06-1998 by Richard Frowijn : first release. + * 20-06-1998 by Frank Denis : Linux 2.1.99+ support, boot signature, misc. + * 30-06-1998 by Frank Denis : first step to write inodes. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "qnx4.h" + +#define QNX4_VERSION 4 +#define QNX4_BMNAME ".bitmap" + +static const struct super_operations qnx4_sops; + +static struct inode *qnx4_alloc_inode(struct super_block *sb); +static void qnx4_free_inode(struct inode *inode); +static int qnx4_statfs(struct dentry *, struct kstatfs *); +static int qnx4_get_tree(struct fs_context *fc); + +static const struct super_operations qnx4_sops = +{ + .alloc_inode = qnx4_alloc_inode, + .free_inode = qnx4_free_inode, + .statfs = qnx4_statfs, +}; + +static int qnx4_reconfigure(struct fs_context *fc) +{ + struct super_block *sb = fc->root->d_sb; + struct qnx4_sb_info *qs; + + sync_filesystem(sb); + qs = qnx4_sb(sb); + qs->Version = QNX4_VERSION; + fc->sb_flags |= SB_RDONLY; + return 0; +} + +static const struct fs_context_operations qnx4_context_opts = { + .get_tree = qnx4_get_tree, + .reconfigure = qnx4_reconfigure, +}; + +static int qnx4_get_block( struct inode *inode, sector_t iblock, struct buffer_head *bh, int create ) +{ + unsigned long phys; + + QNX4DEBUG((KERN_INFO "qnx4: qnx4_get_block inode=[%llu] iblock=[%ld]\n", inode->i_ino, iblock)); + + phys = qnx4_block_map( inode, iblock ); + if ( phys ) { + // logical block is before EOF + map_bh(bh, inode->i_sb, phys); + } + return 0; +} + +static inline u32 try_extent(qnx4_xtnt_t *extent, u32 *offset) +{ + u32 size = le32_to_cpu(extent->xtnt_size); + if (*offset < size) + return le32_to_cpu(extent->xtnt_blk) + *offset - 1; + *offset -= size; + return 0; +} + +unsigned long qnx4_block_map( struct inode *inode, long iblock ) +{ + int ix; + long i_xblk; + struct buffer_head *bh = NULL; + struct qnx4_xblk *xblk = NULL; + struct qnx4_inode_entry *qnx4_inode = qnx4_raw_inode(inode); + u16 nxtnt = le16_to_cpu(qnx4_inode->di_num_xtnts); + u32 offset = iblock; + u32 block = try_extent(&qnx4_inode->di_first_xtnt, &offset); + + if (block) { + // iblock is in the first extent. This is easy. + } else { + // iblock is beyond first extent. We have to follow the extent chain. + i_xblk = le32_to_cpu(qnx4_inode->di_xblk); + ix = 0; + while ( --nxtnt > 0 ) { + if ( ix == 0 ) { + // read next xtnt block. + bh = sb_bread(inode->i_sb, i_xblk - 1); + if ( !bh ) { + QNX4DEBUG((KERN_ERR "qnx4: I/O error reading xtnt block [%ld])\n", i_xblk - 1)); + return -EIO; + } + xblk = (struct qnx4_xblk*)bh->b_data; + if ( memcmp( xblk->xblk_signature, "IamXblk", 7 ) ) { + QNX4DEBUG((KERN_ERR "qnx4: block at %ld is not a valid xtnt\n", qnx4_inode->i_xblk)); + return -EIO; + } + } + block = try_extent(&xblk->xblk_xtnts[ix], &offset); + if (block) { + // got it! + break; + } + if ( ++ix >= xblk->xblk_num_xtnts ) { + i_xblk = le32_to_cpu(xblk->xblk_next_xblk); + ix = 0; + brelse( bh ); + bh = NULL; + } + } + if ( bh ) + brelse( bh ); + } + + QNX4DEBUG((KERN_INFO "qnx4: mapping block %ld of inode %llu = %ld\n", iblock, inode->i_ino, block)); + return block; +} + +static int qnx4_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + u64 id = huge_encode_dev(sb->s_bdev->bd_dev); + + buf->f_type = sb->s_magic; + buf->f_bsize = sb->s_blocksize; + buf->f_blocks = le32_to_cpu(qnx4_sb(sb)->BitMap->di_size) * 8; + buf->f_bfree = qnx4_count_free_blocks(sb); + buf->f_bavail = buf->f_bfree; + buf->f_namelen = QNX4_NAME_MAX; + buf->f_fsid = u64_to_fsid(id); + + return 0; +} + +/* + * Check the root directory of the filesystem to make sure + * it really _is_ a qnx4 filesystem, and to check the size + * of the directory entry. + */ +static const char *qnx4_checkroot(struct super_block *sb, + struct qnx4_super_block *s) +{ + struct buffer_head *bh; + struct qnx4_inode_entry *rootdir; + int rd, rl; + int i, j; + + if (s->RootDir.di_fname[0] != '/' || s->RootDir.di_fname[1] != '\0') + return "no qnx4 filesystem (no root dir)."; + QNX4DEBUG((KERN_NOTICE "QNX4 filesystem found on dev %s.\n", sb->s_id)); + rd = le32_to_cpu(s->RootDir.di_first_xtnt.xtnt_blk) - 1; + rl = le32_to_cpu(s->RootDir.di_first_xtnt.xtnt_size); + for (j = 0; j < rl; j++) { + bh = sb_bread(sb, rd + j); /* root dir, first block */ + if (bh == NULL) + return "unable to read root entry."; + rootdir = (struct qnx4_inode_entry *) bh->b_data; + for (i = 0; i < QNX4_INODES_PER_BLOCK; i++, rootdir++) { + QNX4DEBUG((KERN_INFO "rootdir entry found : [%s]\n", rootdir->di_fname)); + if (strcmp(rootdir->di_fname, QNX4_BMNAME) != 0) + continue; + qnx4_sb(sb)->BitMap = kmemdup(rootdir, + sizeof(struct qnx4_inode_entry), + GFP_KERNEL); + brelse(bh); + if (!qnx4_sb(sb)->BitMap) + return "not enough memory for bitmap inode"; + /* keep bitmap inode known */ + return NULL; + } + brelse(bh); + } + return "bitmap file not found."; +} + +static int qnx4_fill_super(struct super_block *s, struct fs_context *fc) +{ + struct buffer_head *bh; + struct inode *root; + const char *errmsg; + struct qnx4_sb_info *qs; + int silent = fc->sb_flags & SB_SILENT; + + qs = kzalloc_obj(struct qnx4_sb_info); + if (!qs) + return -ENOMEM; + s->s_fs_info = qs; + + if (!sb_set_blocksize(s, QNX4_BLOCK_SIZE)) + return -EINVAL; + + s->s_op = &qnx4_sops; + s->s_magic = QNX4_SUPER_MAGIC; + s->s_flags |= SB_RDONLY; /* Yup, read-only yet */ + s->s_time_min = 0; + s->s_time_max = U32_MAX; + + /* Check the superblock signature. Since the qnx4 code is + dangerous, we should leave as quickly as possible + if we don't belong here... */ + bh = sb_bread(s, 1); + if (!bh) { + printk(KERN_ERR "qnx4: unable to read the superblock\n"); + return -EINVAL; + } + + /* check before allocating dentries, inodes, .. */ + errmsg = qnx4_checkroot(s, (struct qnx4_super_block *) bh->b_data); + brelse(bh); + if (errmsg != NULL) { + if (!silent) + printk(KERN_ERR "qnx4: %s\n", errmsg); + return -EINVAL; + } + + /* does root not have inode number QNX4_ROOT_INO ?? */ + root = qnx4_iget(s, QNX4_ROOT_INO * QNX4_INODES_PER_BLOCK); + if (IS_ERR(root)) { + printk(KERN_ERR "qnx4: get inode failed\n"); + return PTR_ERR(root); + } + + s->s_root = d_make_root(root); + if (s->s_root == NULL) + return -ENOMEM; + + return 0; +} + +static int qnx4_get_tree(struct fs_context *fc) +{ + return get_tree_bdev(fc, qnx4_fill_super); +} + +static int qnx4_init_fs_context(struct fs_context *fc) +{ + fc->ops = &qnx4_context_opts; + + return 0; +} + +static void qnx4_kill_sb(struct super_block *sb) +{ + struct qnx4_sb_info *qs = qnx4_sb(sb); + kill_block_super(sb); + if (qs) { + kfree(qs->BitMap); + kfree(qs); + } +} + +static int qnx4_read_folio(struct file *file, struct folio *folio) +{ + return block_read_full_folio(folio, qnx4_get_block); +} + +static sector_t qnx4_bmap(struct address_space *mapping, sector_t block) +{ + return generic_block_bmap(mapping,block,qnx4_get_block); +} + +static const struct address_space_operations qnx4_aops = { + .read_folio = qnx4_read_folio, + .bmap = qnx4_bmap +}; + +struct inode *qnx4_iget(struct super_block *sb, unsigned long ino) +{ + struct buffer_head *bh; + struct qnx4_inode_entry *raw_inode; + int block; + struct qnx4_inode_entry *qnx4_inode; + struct inode *inode; + + inode = iget_locked(sb, ino); + if (!inode) + return ERR_PTR(-ENOMEM); + if (!(inode_state_read_once(inode) & I_NEW)) + return inode; + + qnx4_inode = qnx4_raw_inode(inode); + inode->i_mode = 0; + + QNX4DEBUG((KERN_INFO "reading inode : [%d]\n", ino)); + if (!ino) { + printk(KERN_ERR "qnx4: bad inode number on dev %s: %lu is " + "out of range\n", + sb->s_id, ino); + iget_failed(inode); + return ERR_PTR(-EIO); + } + block = ino / QNX4_INODES_PER_BLOCK; + + if (!(bh = sb_bread(sb, block))) { + printk(KERN_ERR "qnx4: major problem: unable to read inode from dev " + "%s\n", sb->s_id); + iget_failed(inode); + return ERR_PTR(-EIO); + } + raw_inode = ((struct qnx4_inode_entry *) bh->b_data) + + (ino % QNX4_INODES_PER_BLOCK); + + inode->i_mode = le16_to_cpu(raw_inode->di_mode); + i_uid_write(inode, (uid_t)le16_to_cpu(raw_inode->di_uid)); + i_gid_write(inode, (gid_t)le16_to_cpu(raw_inode->di_gid)); + set_nlink(inode, le16_to_cpu(raw_inode->di_nlink)); + inode->i_size = le32_to_cpu(raw_inode->di_size); + inode_set_mtime(inode, le32_to_cpu(raw_inode->di_mtime), 0); + inode_set_atime(inode, le32_to_cpu(raw_inode->di_atime), 0); + inode_set_ctime(inode, le32_to_cpu(raw_inode->di_ctime), 0); + inode->i_blocks = le32_to_cpu(raw_inode->di_first_xtnt.xtnt_size); + + memcpy(qnx4_inode, raw_inode, QNX4_DIR_ENTRY_SIZE); + if (S_ISREG(inode->i_mode)) { + inode->i_fop = &generic_ro_fops; + inode->i_mapping->a_ops = &qnx4_aops; + qnx4_i(inode)->mmu_private = inode->i_size; + } else if (S_ISDIR(inode->i_mode)) { + inode->i_op = &qnx4_dir_inode_operations; + inode->i_fop = &qnx4_dir_operations; + } else if (S_ISLNK(inode->i_mode)) { + inode->i_op = &page_symlink_inode_operations; + inode_nohighmem(inode); + inode->i_mapping->a_ops = &qnx4_aops; + qnx4_i(inode)->mmu_private = inode->i_size; + } else { + printk(KERN_ERR "qnx4: bad inode %lu on dev %s\n", + ino, sb->s_id); + iget_failed(inode); + brelse(bh); + return ERR_PTR(-EIO); + } + brelse(bh); + unlock_new_inode(inode); + return inode; +} + +static struct kmem_cache *qnx4_inode_cachep; + +static struct inode *qnx4_alloc_inode(struct super_block *sb) +{ + struct qnx4_inode_info *ei; + ei = alloc_inode_sb(sb, qnx4_inode_cachep, GFP_KERNEL); + if (!ei) + return NULL; + return &ei->vfs_inode; +} + +static void qnx4_free_inode(struct inode *inode) +{ + kmem_cache_free(qnx4_inode_cachep, qnx4_i(inode)); +} + +static void init_once(void *foo) +{ + struct qnx4_inode_info *ei = (struct qnx4_inode_info *) foo; + + inode_init_once(&ei->vfs_inode); +} + +static int init_inodecache(void) +{ + qnx4_inode_cachep = kmem_cache_create("qnx4_inode_cache", + sizeof(struct qnx4_inode_info), + 0, (SLAB_RECLAIM_ACCOUNT| + SLAB_ACCOUNT), + init_once); + if (qnx4_inode_cachep == NULL) + return -ENOMEM; + return 0; +} + +static void destroy_inodecache(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(qnx4_inode_cachep); +} + +static struct file_system_type qnx4_fs_type = { + .owner = THIS_MODULE, + .name = "qnx4", + .kill_sb = qnx4_kill_sb, + .fs_flags = FS_REQUIRES_DEV, + .init_fs_context = qnx4_init_fs_context, +}; +MODULE_ALIAS_FS("qnx4"); + +static int __init init_qnx4_fs(void) +{ + int err; + + err = init_inodecache(); + if (err) + return err; + + err = register_filesystem(&qnx4_fs_type); + if (err) { + destroy_inodecache(); + return err; + } + + printk(KERN_INFO "QNX4 filesystem 0.2.3 registered.\n"); + return 0; +} + +static void __exit exit_qnx4_fs(void) +{ + unregister_filesystem(&qnx4_fs_type); + destroy_inodecache(); +} + +module_init(init_qnx4_fs) +module_exit(exit_qnx4_fs) +MODULE_DESCRIPTION("QNX4 file system"); +MODULE_LICENSE("GPL"); + diff --git a/fs/qnx4/namei.c b/fs/qnx4/namei.c new file mode 100644 index 000000000..bb8db6550 --- /dev/null +++ b/fs/qnx4/namei.c @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * QNX4 file system, Linux implementation. + * + * Version : 0.2.1 + * + * Using parts of the xiafs filesystem. + * + * History : + * + * 01-06-1998 by Richard Frowijn : first release. + * 21-06-1998 by Frank Denis : dcache support, fixed error codes. + * 04-07-1998 by Frank Denis : first step for rmdir/unlink. + */ + +#include +#include "qnx4.h" + + +/* + * check if the filename is correct. For some obscure reason, qnx writes a + * new file twice in the directory entry, first with all possible options at 0 + * and for a second time the way it is, they want us not to access the qnx + * filesystem when whe are using linux. + */ +static int qnx4_match(int len, const char *name, + struct buffer_head *bh, unsigned long *offset) +{ + union qnx4_directory_entry *de; + const char *fname; + int fnamelen; + + if (bh == NULL) { + printk(KERN_WARNING "qnx4: matching unassigned buffer !\n"); + return 0; + } + de = (union qnx4_directory_entry *) (bh->b_data + *offset); + *offset += QNX4_DIR_ENTRY_SIZE; + + fname = get_entry_fname(de, &fnamelen); + if (!fname || len != fnamelen) + return 0; + + if (strncmp(name, fname, len) == 0) + return 1; + + return 0; +} + +static struct buffer_head *qnx4_find_entry(int len, struct inode *dir, + const char *name, struct qnx4_inode_entry **res_dir, int *ino) +{ + unsigned long block, offset, blkofs; + struct buffer_head *bh; + + *res_dir = NULL; + bh = NULL; + block = offset = blkofs = 0; + while (blkofs * QNX4_BLOCK_SIZE + offset < dir->i_size) { + if (!bh) { + block = qnx4_block_map(dir, blkofs); + if (block) + bh = sb_bread(dir->i_sb, block); + if (!bh) { + blkofs++; + continue; + } + } + *res_dir = (struct qnx4_inode_entry *) (bh->b_data + offset); + if (qnx4_match(len, name, bh, &offset)) { + *ino = block * QNX4_INODES_PER_BLOCK + + (offset / QNX4_DIR_ENTRY_SIZE) - 1; + return bh; + } + if (offset < bh->b_size) { + continue; + } + brelse(bh); + bh = NULL; + offset = 0; + blkofs++; + } + brelse(bh); + *res_dir = NULL; + return NULL; +} + +struct dentry * qnx4_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) +{ + int ino; + struct qnx4_inode_entry *de; + struct qnx4_link_info *lnk; + struct buffer_head *bh; + const char *name = dentry->d_name.name; + int len = dentry->d_name.len; + struct inode *foundinode = NULL; + + if (!(bh = qnx4_find_entry(len, dir, name, &de, &ino))) + goto out; + /* The entry is linked, let's get the real info */ + if ((de->di_status & QNX4_FILE_LINK) == QNX4_FILE_LINK) { + lnk = (struct qnx4_link_info *) de; + ino = (le32_to_cpu(lnk->dl_inode_blk) - 1) * + QNX4_INODES_PER_BLOCK + + lnk->dl_inode_ndx; + } + brelse(bh); + + foundinode = qnx4_iget(dir->i_sb, ino); + if (IS_ERR(foundinode)) + QNX4DEBUG((KERN_ERR "qnx4: lookup->iget -> error %ld\n", + PTR_ERR(foundinode))); +out: + return d_splice_alias(foundinode, dentry); +} diff --git a/fs/qnx4/qnx4.h b/fs/qnx4/qnx4.h new file mode 100644 index 000000000..5c2b1fb6b --- /dev/null +++ b/fs/qnx4/qnx4.h @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include + +#define QNX4_DEBUG 0 + +#if QNX4_DEBUG +#define QNX4DEBUG(X) printk X +#else +#define QNX4DEBUG(X) (void) 0 +#endif + +struct qnx4_sb_info { + unsigned int Version; /* may be useful */ + struct qnx4_inode_entry *BitMap; /* useful */ +}; + +struct qnx4_inode_info { + struct qnx4_inode_entry raw; + loff_t mmu_private; + struct inode vfs_inode; +}; + +extern struct inode *qnx4_iget(struct super_block *, unsigned long); +extern struct dentry *qnx4_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags); +extern unsigned long qnx4_count_free_blocks(struct super_block *sb); +extern unsigned long qnx4_block_map(struct inode *inode, long iblock); + +extern const struct inode_operations qnx4_dir_inode_operations; +extern const struct file_operations qnx4_dir_operations; +extern int qnx4_is_free(struct super_block *sb, long block); + +static inline struct qnx4_sb_info *qnx4_sb(struct super_block *sb) +{ + return sb->s_fs_info; +} + +static inline struct qnx4_inode_info *qnx4_i(struct inode *inode) +{ + return container_of(inode, struct qnx4_inode_info, vfs_inode); +} + +static inline struct qnx4_inode_entry *qnx4_raw_inode(struct inode *inode) +{ + return &qnx4_i(inode)->raw; +} + +/* + * A qnx4 directory entry is an inode entry or link info + * depending on the status field in the last byte. The + * first byte is where the name start either way, and a + * zero means it's empty. + * + * Also, due to a bug in gcc, we don't want to use the + * real (differently sized) name arrays in the inode and + * link entries, but always the 'de_name[]' one in the + * fake struct entry. + * + * See + * + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=99578#c6 + * + * for details, but basically gcc will take the size of the + * 'name' array from one of the used union entries randomly. + * + * This use of 'de_name[]' (48 bytes) avoids the false positive + * warnings that would happen if gcc decides to use 'inode.di_name' + * (16 bytes) even when the pointer and size were to come from + * 'link.dl_name' (48 bytes). + * + * In all cases the actual name pointer itself is the same, it's + * only the gcc internal 'what is the size of this field' logic + * that can get confused. + */ +union qnx4_directory_entry { + struct { + const char de_name[48]; + u8 de_pad[15]; + u8 de_status; + }; + struct qnx4_inode_entry inode; + struct qnx4_link_info link; +}; + +static inline const char *get_entry_fname(union qnx4_directory_entry *de, + int *size) +{ + /* Make sure the status byte is in the same place for all structs. */ + BUILD_BUG_ON(offsetof(struct qnx4_inode_entry, di_status) != + offsetof(struct qnx4_link_info, dl_status)); + BUILD_BUG_ON(offsetof(struct qnx4_inode_entry, di_status) != + offsetof(union qnx4_directory_entry, de_status)); + + if (!de->de_name[0]) + return NULL; + if (!(de->de_status & (QNX4_FILE_USED|QNX4_FILE_LINK))) + return NULL; + if (!(de->de_status & QNX4_FILE_LINK)) + *size = sizeof(de->inode.di_fname); + else + *size = sizeof(de->link.dl_fname); + + *size = strnlen(de->de_name, *size); + + return de->de_name; +} diff --git a/fs/qnx6/Kconfig b/fs/qnx6/Kconfig new file mode 100644 index 000000000..8e865d722 --- /dev/null +++ b/fs/qnx6/Kconfig @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: GPL-2.0-only +config QNX6FS_FS + tristate "QNX6 file system support (read only)" + depends on BLOCK && CRC32 + select BUFFER_HEAD + help + This is the file system used by the real-time operating systems + QNX 6 (also called QNX RTP). + Further information is available at . + Say Y if you intend to mount QNX hard disks or floppies formatted + with a mkqnx6fs. + However, keep in mind that this currently is a readonly driver! + + To compile this file system support as a module, choose M here: the + module will be called qnx6. + + If you don't know whether you need it, then you don't need it: + answer N. + +config QNX6FS_DEBUG + bool "QNX6 debugging information" + depends on QNX6FS_FS + help + Turns on extended debugging output. + + If you are not a developer working on the QNX6FS, you probably don't + want this: + answer N. diff --git a/fs/qnx6/Makefile b/fs/qnx6/Makefile new file mode 100644 index 000000000..bd4e6c8ef --- /dev/null +++ b/fs/qnx6/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the linux qnx4-filesystem routines. +# + +obj-$(CONFIG_QNX6FS_FS) += qnx6.o + +qnx6-objs := inode.o dir.o namei.o super_mmi.o +ccflags-$(CONFIG_QNX6FS_DEBUG) += -DDEBUG diff --git a/fs/qnx6/dir.c b/fs/qnx6/dir.c new file mode 100644 index 000000000..56bbaffe4 --- /dev/null +++ b/fs/qnx6/dir.c @@ -0,0 +1,284 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * QNX6 file system, Linux implementation. + * + * Version : 1.0.0 + * + * History : + * + * 01-02-2012 by Kai Bankett (chaosman@ontika.net) : first release. + * 16-02-2012 pagemap extension by Al Viro + * + */ + +#include +#include "qnx6.h" + +static unsigned qnx6_lfile_checksum(char *name, unsigned size) +{ + unsigned crc = 0; + char *end = name + size; + while (name < end) { + crc = ((crc >> 1) + *(name++)) ^ + ((crc & 0x00000001) ? 0x80000000 : 0); + } + return crc; +} + +static void *qnx6_get_folio(struct inode *dir, unsigned long n, + struct folio **foliop) +{ + struct folio *folio = read_mapping_folio(dir->i_mapping, n, NULL); + + if (IS_ERR(folio)) + return folio; + *foliop = folio; + return kmap_local_folio(folio, 0); +} + +static unsigned last_entry(struct inode *inode, unsigned long page_nr) +{ + unsigned long last_byte = inode->i_size; + last_byte -= page_nr << PAGE_SHIFT; + if (last_byte > PAGE_SIZE) + last_byte = PAGE_SIZE; + return last_byte / QNX6_DIR_ENTRY_SIZE; +} + +static struct qnx6_long_filename *qnx6_longname(struct super_block *sb, + struct qnx6_long_dir_entry *de, + struct folio **foliop) +{ + struct qnx6_sb_info *sbi = QNX6_SB(sb); + u32 s = fs32_to_cpu(sbi, de->de_long_inode); /* in block units */ + u32 n = s >> (PAGE_SHIFT - sb->s_blocksize_bits); /* in pages */ + u32 offs; + struct address_space *mapping = sbi->longfile->i_mapping; + struct folio *folio = read_mapping_folio(mapping, n, NULL); + + if (IS_ERR(folio)) + return ERR_CAST(folio); + offs = offset_in_folio(folio, s << sb->s_blocksize_bits); + *foliop = folio; + return kmap_local_folio(folio, offs); +} + +static int qnx6_dir_longfilename(struct inode *inode, + struct qnx6_long_dir_entry *de, + struct dir_context *ctx, + unsigned de_inode) +{ + struct qnx6_long_filename *lf; + struct super_block *s = inode->i_sb; + struct qnx6_sb_info *sbi = QNX6_SB(s); + struct folio *folio; + int lf_size; + + if (de->de_size != 0xff) { + /* error - long filename entries always have size 0xff + in direntry */ + pr_err("invalid direntry size (%i).\n", de->de_size); + return 0; + } + lf = qnx6_longname(s, de, &folio); + if (IS_ERR(lf)) { + pr_err("Error reading longname\n"); + return 0; + } + + lf_size = fs16_to_cpu(sbi, lf->lf_size); + + if (lf_size > QNX6_LONG_NAME_MAX) { + pr_debug("file %s\n", lf->lf_fname); + pr_err("Filename too long (%i)\n", lf_size); + folio_release_kmap(folio, lf); + return 0; + } + + /* calc & validate longfilename checksum + mmi 3g filesystem does not have that checksum */ + if (!test_opt(s, MMI_FS) && fs32_to_cpu(sbi, de->de_checksum) != + qnx6_lfile_checksum(lf->lf_fname, lf_size)) + pr_info("long filename checksum error.\n"); + + pr_debug("qnx6_readdir:%.*s inode:%u\n", + lf_size, lf->lf_fname, de_inode); + if (!dir_emit(ctx, lf->lf_fname, lf_size, de_inode, DT_UNKNOWN)) { + folio_release_kmap(folio, lf); + return 0; + } + + folio_release_kmap(folio, lf); + /* success */ + return 1; +} + +static int qnx6_readdir(struct file *file, struct dir_context *ctx) +{ + struct inode *inode = file_inode(file); + struct super_block *s = inode->i_sb; + struct qnx6_sb_info *sbi = QNX6_SB(s); + loff_t pos = ctx->pos & ~(QNX6_DIR_ENTRY_SIZE - 1); + unsigned long npages = dir_pages(inode); + unsigned long n = pos >> PAGE_SHIFT; + unsigned offset = (pos & ~PAGE_MASK) / QNX6_DIR_ENTRY_SIZE; + bool done = false; + + ctx->pos = pos; + if (ctx->pos >= inode->i_size) + return 0; + + for ( ; !done && n < npages; n++, offset = 0) { + struct qnx6_dir_entry *de; + struct folio *folio; + char *kaddr = qnx6_get_folio(inode, n, &folio); + struct qnx6_dir_entry *limit; + + if (IS_ERR(kaddr)) { + pr_err("%s(): read failed\n", __func__); + ctx->pos = (n + 1) << PAGE_SHIFT; + return PTR_ERR(kaddr); + } + de = (struct qnx6_dir_entry *)kaddr + offset; + limit = (struct qnx6_dir_entry *)kaddr + last_entry(inode, n); + for (; de < limit; de++, ctx->pos += QNX6_DIR_ENTRY_SIZE) { + int size = de->de_size; + u32 no_inode = fs32_to_cpu(sbi, de->de_inode); + + if (!no_inode || !size) + continue; + + if (size > QNX6_SHORT_NAME_MAX) { + /* long filename detected + get the filename from long filename + structure / block */ + if (!qnx6_dir_longfilename(inode, + (struct qnx6_long_dir_entry *)de, + ctx, no_inode)) { + done = true; + break; + } + } else { + pr_debug("%s():%.*s inode:%u\n", + __func__, size, de->de_fname, + no_inode); + if (!dir_emit(ctx, de->de_fname, size, + no_inode, DT_UNKNOWN)) { + done = true; + break; + } + } + } + folio_release_kmap(folio, kaddr); + } + return 0; +} + +/* + * check if the long filename is correct. + */ +static unsigned qnx6_long_match(int len, const char *name, + struct qnx6_long_dir_entry *de, struct inode *dir) +{ + struct super_block *s = dir->i_sb; + struct qnx6_sb_info *sbi = QNX6_SB(s); + struct folio *folio; + int thislen; + struct qnx6_long_filename *lf = qnx6_longname(s, de, &folio); + + if (IS_ERR(lf)) + return 0; + + thislen = fs16_to_cpu(sbi, lf->lf_size); + if (len != thislen) { + folio_release_kmap(folio, lf); + return 0; + } + if (memcmp(name, lf->lf_fname, len) == 0) { + folio_release_kmap(folio, lf); + return fs32_to_cpu(sbi, de->de_inode); + } + folio_release_kmap(folio, lf); + return 0; +} + +/* + * check if the filename is correct. + */ +static unsigned qnx6_match(struct super_block *s, int len, const char *name, + struct qnx6_dir_entry *de) +{ + struct qnx6_sb_info *sbi = QNX6_SB(s); + if (memcmp(name, de->de_fname, len) == 0) + return fs32_to_cpu(sbi, de->de_inode); + return 0; +} + + +unsigned qnx6_find_ino(int len, struct inode *dir, const char *name) +{ + struct super_block *s = dir->i_sb; + struct qnx6_inode_info *ei = QNX6_I(dir); + struct folio *folio; + unsigned long start, n; + unsigned long npages = dir_pages(dir); + unsigned ino; + struct qnx6_dir_entry *de; + struct qnx6_long_dir_entry *lde; + + if (npages == 0) + return 0; + start = ei->i_dir_start_lookup; + if (start >= npages) + start = 0; + n = start; + + do { + de = qnx6_get_folio(dir, n, &folio); + if (!IS_ERR(de)) { + int limit = last_entry(dir, n); + int i; + + for (i = 0; i < limit; i++, de++) { + if (len <= QNX6_SHORT_NAME_MAX) { + /* short filename */ + if (len != de->de_size) + continue; + ino = qnx6_match(s, len, name, de); + if (ino) + goto found; + } else if (de->de_size == 0xff) { + /* deal with long filename */ + lde = (struct qnx6_long_dir_entry *)de; + ino = qnx6_long_match(len, + name, lde, dir); + if (ino) + goto found; + } else + pr_err("undefined filename size in inode.\n"); + } + folio_release_kmap(folio, de - i); + } + + if (++n >= npages) + n = 0; + } while (n != start); + return 0; + +found: + ei->i_dir_start_lookup = n; + folio_release_kmap(folio, de); + return ino; +} + +const struct file_operations qnx6_dir_operations = { + .llseek = generic_file_llseek, + .read = generic_read_dir, + .iterate_shared = qnx6_readdir, + .fsync = simple_fsync, + .setlease = generic_setlease, +}; + +const struct inode_operations qnx6_dir_inode_operations = { + .lookup = qnx6_lookup, +}; diff --git a/fs/qnx6/inode.c b/fs/qnx6/inode.c new file mode 100644 index 000000000..6de49333a --- /dev/null +++ b/fs/qnx6/inode.c @@ -0,0 +1,694 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * QNX6 file system, Linux implementation. + * + * Version : 1.0.0 + * + * History : + * + * 01-02-2012 by Kai Bankett (chaosman@ontika.net) : first release. + * 16-02-2012 pagemap extension by Al Viro + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "qnx6.h" + +static const struct super_operations qnx6_sops; + +static void qnx6_put_super(struct super_block *sb); +static struct inode *qnx6_alloc_inode(struct super_block *sb); +static void qnx6_free_inode(struct inode *inode); +static int qnx6_reconfigure(struct fs_context *fc); +static int qnx6_statfs(struct dentry *dentry, struct kstatfs *buf); +static int qnx6_show_options(struct seq_file *seq, struct dentry *root); + +static const struct super_operations qnx6_sops = { + .alloc_inode = qnx6_alloc_inode, + .free_inode = qnx6_free_inode, + .put_super = qnx6_put_super, + .statfs = qnx6_statfs, + .show_options = qnx6_show_options, +}; + +static int qnx6_show_options(struct seq_file *seq, struct dentry *root) +{ + struct super_block *sb = root->d_sb; + struct qnx6_sb_info *sbi = QNX6_SB(sb); + + if (sbi->s_mount_opt & QNX6_MOUNT_MMI_FS) + seq_puts(seq, ",mmi_fs"); + return 0; +} + +static int qnx6_reconfigure(struct fs_context *fc) +{ + struct super_block *sb = fc->root->d_sb; + + sync_filesystem(sb); + fc->sb_flags |= SB_RDONLY; + return 0; +} + +static unsigned qnx6_get_devblock(struct super_block *sb, __fs32 block) +{ + struct qnx6_sb_info *sbi = QNX6_SB(sb); + return fs32_to_cpu(sbi, block) + sbi->s_blks_off; +} + +static unsigned qnx6_block_map(struct inode *inode, unsigned iblock); + +static int qnx6_get_block(struct inode *inode, sector_t iblock, + struct buffer_head *bh, int create) +{ + unsigned phys; + + pr_debug("qnx6_get_block inode=[%llu] iblock=[%ld]\n", + inode->i_ino, (unsigned long)iblock); + + phys = qnx6_block_map(inode, iblock); + if (phys) { + /* logical block is before EOF */ + map_bh(bh, inode->i_sb, phys); + } + return 0; +} + +static int qnx6_check_blockptr(__fs32 ptr) +{ + if (ptr == ~(__fs32)0) { + pr_err("hit unused blockpointer.\n"); + return 0; + } + return 1; +} + +static int qnx6_read_folio(struct file *file, struct folio *folio) +{ + return mpage_read_folio(folio, qnx6_get_block); +} + +static void qnx6_readahead(struct readahead_control *rac) +{ + mpage_readahead(rac, qnx6_get_block); +} + +/* + * returns the block number for the no-th element in the tree + * inodebits requred as there are multiple inodes in one inode block + */ +static unsigned qnx6_block_map(struct inode *inode, unsigned no) +{ + struct super_block *s = inode->i_sb; + struct qnx6_sb_info *sbi = QNX6_SB(s); + struct qnx6_inode_info *ei = QNX6_I(inode); + unsigned block = 0; + struct buffer_head *bh; + __fs32 ptr; + int levelptr; + int ptrbits = sbi->s_ptrbits; + int bitdelta; + u32 mask = (1 << ptrbits) - 1; + int depth = ei->di_filelevels; + int i; + + bitdelta = ptrbits * depth; + levelptr = no >> bitdelta; + + if (levelptr > QNX6_NO_DIRECT_POINTERS - 1) { + pr_err("Requested file block number (%u) too big.", no); + return 0; + } + + block = qnx6_get_devblock(s, ei->di_block_ptr[levelptr]); + + for (i = 0; i < depth; i++) { + bh = sb_bread(s, block); + if (!bh) { + pr_err("Error reading block (%u)\n", block); + return 0; + } + bitdelta -= ptrbits; + levelptr = (no >> bitdelta) & mask; + ptr = ((__fs32 *)bh->b_data)[levelptr]; + + if (!qnx6_check_blockptr(ptr)) + return 0; + + block = qnx6_get_devblock(s, ptr); + brelse(bh); + } + return block; +} + +static int qnx6_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + struct qnx6_sb_info *sbi = QNX6_SB(sb); + u64 id = huge_encode_dev(sb->s_bdev->bd_dev); + + buf->f_type = sb->s_magic; + buf->f_bsize = sb->s_blocksize; + buf->f_blocks = fs32_to_cpu(sbi, sbi->sb->sb_num_blocks); + buf->f_bfree = fs32_to_cpu(sbi, sbi->sb->sb_free_blocks); + buf->f_files = fs32_to_cpu(sbi, sbi->sb->sb_num_inodes); + buf->f_ffree = fs32_to_cpu(sbi, sbi->sb->sb_free_inodes); + buf->f_bavail = buf->f_bfree; + buf->f_namelen = QNX6_LONG_NAME_MAX; + buf->f_fsid = u64_to_fsid(id); + + return 0; +} + +/* + * Check the root directory of the filesystem to make sure + * it really _is_ a qnx6 filesystem, and to check the size + * of the directory entry. + */ +static const char *qnx6_checkroot(struct super_block *s) +{ + int error = 0; + struct qnx6_dir_entry *dir_entry; + struct inode *root = d_inode(s->s_root); + struct address_space *mapping = root->i_mapping; + struct folio *folio = read_mapping_folio(mapping, 0, NULL); + + if (IS_ERR(folio)) + return "error reading root directory"; + dir_entry = kmap_local_folio(folio, 0); + if (memcmp(dir_entry[0].de_fname, ".", 2) || + memcmp(dir_entry[1].de_fname, "..", 3)) + error = 1; + folio_release_kmap(folio, dir_entry); + if (error) + return "error reading root directory."; + return NULL; +} + +#ifdef CONFIG_QNX6FS_DEBUG +void qnx6_superblock_debug(struct qnx6_super_block *sb, struct super_block *s) +{ + struct qnx6_sb_info *sbi = QNX6_SB(s); + + pr_debug("magic: %08x\n", fs32_to_cpu(sbi, sb->sb_magic)); + pr_debug("checksum: %08x\n", fs32_to_cpu(sbi, sb->sb_checksum)); + pr_debug("serial: %llx\n", fs64_to_cpu(sbi, sb->sb_serial)); + pr_debug("flags: %08x\n", fs32_to_cpu(sbi, sb->sb_flags)); + pr_debug("blocksize: %08x\n", fs32_to_cpu(sbi, sb->sb_blocksize)); + pr_debug("num_inodes: %08x\n", fs32_to_cpu(sbi, sb->sb_num_inodes)); + pr_debug("free_inodes: %08x\n", fs32_to_cpu(sbi, sb->sb_free_inodes)); + pr_debug("num_blocks: %08x\n", fs32_to_cpu(sbi, sb->sb_num_blocks)); + pr_debug("free_blocks: %08x\n", fs32_to_cpu(sbi, sb->sb_free_blocks)); + pr_debug("inode_levels: %02x\n", sb->Inode.levels); +} +#endif + +enum { + Opt_mmifs +}; + +struct qnx6_context { + unsigned long s_mount_opts; +}; + +static const struct fs_parameter_spec qnx6_param_spec[] = { + fsparam_flag ("mmi_fs", Opt_mmifs), + {} +}; + +static int qnx6_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct qnx6_context *ctx = fc->fs_private; + struct fs_parse_result result; + int opt; + + opt = fs_parse(fc, qnx6_param_spec, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_mmifs: + ctx->s_mount_opts |= QNX6_MOUNT_MMI_FS; + break; + default: + return -EINVAL; + } + return 0; +} + +static struct buffer_head *qnx6_check_first_superblock(struct super_block *s, + int offset, int silent) +{ + struct qnx6_sb_info *sbi = QNX6_SB(s); + struct buffer_head *bh; + struct qnx6_super_block *sb; + + /* Check the superblock signatures + start with the first superblock */ + bh = sb_bread(s, offset); + if (!bh) { + pr_err("unable to read the first superblock\n"); + return NULL; + } + sb = (struct qnx6_super_block *)bh->b_data; + if (fs32_to_cpu(sbi, sb->sb_magic) != QNX6_SUPER_MAGIC) { + sbi->s_bytesex = BYTESEX_BE; + if (fs32_to_cpu(sbi, sb->sb_magic) == QNX6_SUPER_MAGIC) { + /* we got a big endian fs */ + pr_debug("fs got different endianness.\n"); + return bh; + } else + sbi->s_bytesex = BYTESEX_LE; + if (!silent) { + if (offset == 0) { + pr_err("wrong signature (magic) in superblock #1.\n"); + } else { + pr_info("wrong signature (magic) at position (0x%lx) - will try alternative position (0x0000).\n", + offset * s->s_blocksize); + } + } + brelse(bh); + return NULL; + } + return bh; +} + +static struct inode *qnx6_private_inode(struct super_block *s, + struct qnx6_root_node *p); + +static int qnx6_fill_super(struct super_block *s, struct fs_context *fc) +{ + struct buffer_head *bh1 = NULL, *bh2 = NULL; + struct qnx6_super_block *sb1 = NULL, *sb2 = NULL; + struct qnx6_sb_info *sbi; + struct qnx6_context *ctx = fc->fs_private; + struct inode *root; + const char *errmsg; + struct qnx6_sb_info *qs; + int ret = -EINVAL; + u64 offset; + int bootblock_offset = QNX6_BOOTBLOCK_SIZE; + int silent = fc->sb_flags & SB_SILENT; + + qs = kzalloc_obj(struct qnx6_sb_info); + if (!qs) + return -ENOMEM; + s->s_fs_info = qs; + qs->s_mount_opt = ctx->s_mount_opts; + + /* Superblock always is 512 Byte long */ + if (!sb_set_blocksize(s, QNX6_SUPERBLOCK_SIZE)) { + pr_err("unable to set blocksize\n"); + goto outnobh; + } + + if (qs->s_mount_opt == QNX6_MOUNT_MMI_FS) { + sb1 = qnx6_mmi_fill_super(s, silent); + if (sb1) + goto mmi_success; + else + goto outnobh; + } + sbi = QNX6_SB(s); + sbi->s_bytesex = BYTESEX_LE; + /* Check the superblock signatures + start with the first superblock */ + bh1 = qnx6_check_first_superblock(s, + bootblock_offset / QNX6_SUPERBLOCK_SIZE, silent); + if (!bh1) { + /* try again without bootblock offset */ + bh1 = qnx6_check_first_superblock(s, 0, silent); + if (!bh1) { + pr_err("unable to read the first superblock\n"); + goto outnobh; + } + /* seems that no bootblock at partition start */ + bootblock_offset = 0; + } + sb1 = (struct qnx6_super_block *)bh1->b_data; + +#ifdef CONFIG_QNX6FS_DEBUG + qnx6_superblock_debug(sb1, s); +#endif + + /* checksum check - start at byte 8 and end at byte 512 */ + if (fs32_to_cpu(sbi, sb1->sb_checksum) != + crc32_be(0, (char *)(bh1->b_data + 8), 504)) { + pr_err("superblock #1 checksum error\n"); + goto out; + } + + /* set new blocksize */ + if (!sb_set_blocksize(s, fs32_to_cpu(sbi, sb1->sb_blocksize))) { + pr_err("unable to set blocksize\n"); + goto out; + } + /* blocksize invalidates bh - pull it back in */ + brelse(bh1); + bh1 = sb_bread(s, bootblock_offset >> s->s_blocksize_bits); + if (!bh1) + goto outnobh; + sb1 = (struct qnx6_super_block *)bh1->b_data; + + /* calculate second superblock blocknumber */ + offset = fs32_to_cpu(sbi, sb1->sb_num_blocks) + + (bootblock_offset >> s->s_blocksize_bits) + + (QNX6_SUPERBLOCK_AREA >> s->s_blocksize_bits); + + /* set bootblock offset */ + sbi->s_blks_off = (bootblock_offset >> s->s_blocksize_bits) + + (QNX6_SUPERBLOCK_AREA >> s->s_blocksize_bits); + + /* next the second superblock */ + bh2 = sb_bread(s, offset); + if (!bh2) { + pr_err("unable to read the second superblock\n"); + goto out; + } + sb2 = (struct qnx6_super_block *)bh2->b_data; + if (fs32_to_cpu(sbi, sb2->sb_magic) != QNX6_SUPER_MAGIC) { + if (!silent) + pr_err("wrong signature (magic) in superblock #2.\n"); + goto out; + } + + /* checksum check - start at byte 8 and end at byte 512 */ + if (fs32_to_cpu(sbi, sb2->sb_checksum) != + crc32_be(0, (char *)(bh2->b_data + 8), 504)) { + pr_err("superblock #2 checksum error\n"); + goto out; + } + + if (fs64_to_cpu(sbi, sb1->sb_serial) >= + fs64_to_cpu(sbi, sb2->sb_serial)) { + /* superblock #1 active */ + sbi->sb_buf = bh1; + sbi->sb = (struct qnx6_super_block *)bh1->b_data; + brelse(bh2); + pr_info("superblock #1 active\n"); + } else { + /* superblock #2 active */ + sbi->sb_buf = bh2; + sbi->sb = (struct qnx6_super_block *)bh2->b_data; + brelse(bh1); + pr_info("superblock #2 active\n"); + } +mmi_success: + /* sanity check - limit maximum indirect pointer levels */ + if (sb1->Inode.levels > QNX6_PTR_MAX_LEVELS) { + pr_err("too many inode levels (max %i, sb %i)\n", + QNX6_PTR_MAX_LEVELS, sb1->Inode.levels); + goto out; + } + if (sb1->Longfile.levels > QNX6_PTR_MAX_LEVELS) { + pr_err("too many longfilename levels (max %i, sb %i)\n", + QNX6_PTR_MAX_LEVELS, sb1->Longfile.levels); + goto out; + } + s->s_op = &qnx6_sops; + s->s_magic = QNX6_SUPER_MAGIC; + s->s_flags |= SB_RDONLY; /* Yup, read-only yet */ + s->s_time_min = 0; + s->s_time_max = U32_MAX; + + /* ease the later tree level calculations */ + sbi = QNX6_SB(s); + sbi->s_ptrbits = ilog2(s->s_blocksize / 4); + sbi->inodes = qnx6_private_inode(s, &sb1->Inode); + if (!sbi->inodes) + goto out; + sbi->longfile = qnx6_private_inode(s, &sb1->Longfile); + if (!sbi->longfile) + goto out1; + + /* prefetch root inode */ + root = qnx6_iget(s, QNX6_ROOT_INO); + if (IS_ERR(root)) { + pr_err("get inode failed\n"); + ret = PTR_ERR(root); + goto out2; + } + + ret = -ENOMEM; + s->s_root = d_make_root(root); + if (!s->s_root) + goto out2; + + ret = -EINVAL; + errmsg = qnx6_checkroot(s); + if (errmsg != NULL) { + if (!silent) + pr_err("%s\n", errmsg); + goto out3; + } + return 0; + +out3: + dput(s->s_root); + s->s_root = NULL; +out2: + iput(sbi->longfile); +out1: + iput(sbi->inodes); +out: + brelse(bh1); + brelse(bh2); +outnobh: + kfree(qs); + s->s_fs_info = NULL; + return ret; +} + +static void qnx6_put_super(struct super_block *sb) +{ + struct qnx6_sb_info *qs = QNX6_SB(sb); + brelse(qs->sb_buf); + iput(qs->longfile); + iput(qs->inodes); + kfree(qs); + sb->s_fs_info = NULL; + return; +} + +static sector_t qnx6_bmap(struct address_space *mapping, sector_t block) +{ + return generic_block_bmap(mapping, block, qnx6_get_block); +} +static const struct address_space_operations qnx6_aops = { + .read_folio = qnx6_read_folio, + .readahead = qnx6_readahead, + .bmap = qnx6_bmap +}; + +static struct inode *qnx6_private_inode(struct super_block *s, + struct qnx6_root_node *p) +{ + struct inode *inode = new_inode(s); + if (inode) { + struct qnx6_inode_info *ei = QNX6_I(inode); + struct qnx6_sb_info *sbi = QNX6_SB(s); + inode->i_size = fs64_to_cpu(sbi, p->size); + memcpy(ei->di_block_ptr, p->ptr, sizeof(p->ptr)); + ei->di_filelevels = p->levels; + inode->i_mode = S_IFREG | S_IRUSR; /* probably wrong */ + inode->i_mapping->a_ops = &qnx6_aops; + } + return inode; +} + +struct inode *qnx6_iget(struct super_block *sb, unsigned ino) +{ + struct qnx6_sb_info *sbi = QNX6_SB(sb); + struct qnx6_inode_entry *raw_inode; + struct inode *inode; + struct qnx6_inode_info *ei; + struct address_space *mapping; + struct folio *folio; + u32 n, offs; + + inode = iget_locked(sb, ino); + if (!inode) + return ERR_PTR(-ENOMEM); + if (!(inode_state_read_once(inode) & I_NEW)) + return inode; + + ei = QNX6_I(inode); + + inode->i_mode = 0; + + if (ino == 0) { + pr_err("bad inode number on dev %s: %u is out of range\n", + sb->s_id, ino); + iget_failed(inode); + return ERR_PTR(-EIO); + } + n = (ino - 1) >> (PAGE_SHIFT - QNX6_INODE_SIZE_BITS); + mapping = sbi->inodes->i_mapping; + folio = read_mapping_folio(mapping, n, NULL); + if (IS_ERR(folio)) { + pr_err("major problem: unable to read inode from dev %s\n", + sb->s_id); + iget_failed(inode); + return ERR_CAST(folio); + } + offs = offset_in_folio(folio, (ino - 1) << QNX6_INODE_SIZE_BITS); + raw_inode = kmap_local_folio(folio, offs); + + inode->i_mode = fs16_to_cpu(sbi, raw_inode->di_mode); + i_uid_write(inode, (uid_t)fs32_to_cpu(sbi, raw_inode->di_uid)); + i_gid_write(inode, (gid_t)fs32_to_cpu(sbi, raw_inode->di_gid)); + inode->i_size = fs64_to_cpu(sbi, raw_inode->di_size); + inode_set_mtime(inode, fs32_to_cpu(sbi, raw_inode->di_mtime), 0); + inode_set_atime(inode, fs32_to_cpu(sbi, raw_inode->di_atime), 0); + inode_set_ctime(inode, fs32_to_cpu(sbi, raw_inode->di_ctime), 0); + + /* calc blocks based on 512 byte blocksize */ + inode->i_blocks = (inode->i_size + 511) >> 9; + + memcpy(&ei->di_block_ptr, &raw_inode->di_block_ptr, + sizeof(raw_inode->di_block_ptr)); + ei->di_filelevels = raw_inode->di_filelevels; + + if (S_ISREG(inode->i_mode)) { + inode->i_fop = &generic_ro_fops; + inode->i_mapping->a_ops = &qnx6_aops; + } else if (S_ISDIR(inode->i_mode)) { + inode->i_op = &qnx6_dir_inode_operations; + inode->i_fop = &qnx6_dir_operations; + inode->i_mapping->a_ops = &qnx6_aops; + } else if (S_ISLNK(inode->i_mode)) { + inode->i_op = &page_symlink_inode_operations; + inode_nohighmem(inode); + inode->i_mapping->a_ops = &qnx6_aops; + } else + init_special_inode(inode, inode->i_mode, 0); + folio_release_kmap(folio, raw_inode); + unlock_new_inode(inode); + return inode; +} + +static struct kmem_cache *qnx6_inode_cachep; + +static struct inode *qnx6_alloc_inode(struct super_block *sb) +{ + struct qnx6_inode_info *ei; + ei = alloc_inode_sb(sb, qnx6_inode_cachep, GFP_KERNEL); + if (!ei) + return NULL; + return &ei->vfs_inode; +} + +static void qnx6_free_inode(struct inode *inode) +{ + kmem_cache_free(qnx6_inode_cachep, QNX6_I(inode)); +} + +static void init_once(void *foo) +{ + struct qnx6_inode_info *ei = (struct qnx6_inode_info *) foo; + + inode_init_once(&ei->vfs_inode); +} + +static int init_inodecache(void) +{ + qnx6_inode_cachep = kmem_cache_create("qnx6_inode_cache", + sizeof(struct qnx6_inode_info), + 0, (SLAB_RECLAIM_ACCOUNT| + SLAB_ACCOUNT), + init_once); + if (!qnx6_inode_cachep) + return -ENOMEM; + return 0; +} + +static void destroy_inodecache(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(qnx6_inode_cachep); +} + +static int qnx6_get_tree(struct fs_context *fc) +{ + return get_tree_bdev(fc, qnx6_fill_super); +} + +static void qnx6_free_fc(struct fs_context *fc) +{ + kfree(fc->fs_private); +} + +static const struct fs_context_operations qnx6_context_ops = { + .parse_param = qnx6_parse_param, + .get_tree = qnx6_get_tree, + .reconfigure = qnx6_reconfigure, + .free = qnx6_free_fc, +}; + +static int qnx6_init_fs_context(struct fs_context *fc) +{ + struct qnx6_context *ctx; + + ctx = kzalloc_obj(struct qnx6_context); + if (!ctx) + return -ENOMEM; + fc->ops = &qnx6_context_ops; + fc->fs_private = ctx; + + return 0; +} + +static struct file_system_type qnx6_fs_type = { + .owner = THIS_MODULE, + .name = "qnx6", + .kill_sb = kill_block_super, + .fs_flags = FS_REQUIRES_DEV, + .init_fs_context = qnx6_init_fs_context, + .parameters = qnx6_param_spec, +}; +MODULE_ALIAS_FS("qnx6"); + +static int __init init_qnx6_fs(void) +{ + int err; + + err = init_inodecache(); + if (err) + return err; + + err = register_filesystem(&qnx6_fs_type); + if (err) { + destroy_inodecache(); + return err; + } + + pr_info("QNX6 filesystem 1.0.0 registered.\n"); + return 0; +} + +static void __exit exit_qnx6_fs(void) +{ + unregister_filesystem(&qnx6_fs_type); + destroy_inodecache(); +} + +module_init(init_qnx6_fs) +module_exit(exit_qnx6_fs) +MODULE_DESCRIPTION("QNX6 file system"); +MODULE_LICENSE("GPL"); diff --git a/fs/qnx6/namei.c b/fs/qnx6/namei.c new file mode 100644 index 000000000..0f0755a9e --- /dev/null +++ b/fs/qnx6/namei.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * QNX6 file system, Linux implementation. + * + * Version : 1.0.0 + * + * History : + * + * 01-02-2012 by Kai Bankett (chaosman@ontika.net) : first release. + * 16-02-2012 pagemap extension by Al Viro + * + */ + +#include "qnx6.h" + +struct dentry *qnx6_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + unsigned ino; + struct inode *foundinode = NULL; + const char *name = dentry->d_name.name; + int len = dentry->d_name.len; + + if (len > QNX6_LONG_NAME_MAX) + return ERR_PTR(-ENAMETOOLONG); + + ino = qnx6_find_ino(len, dir, name); + if (ino) { + foundinode = qnx6_iget(dir->i_sb, ino); + if (IS_ERR(foundinode)) + pr_debug("lookup->iget -> error %ld\n", + PTR_ERR(foundinode)); + } else { + pr_debug("%s(): not found %s\n", __func__, name); + } + return d_splice_alias(foundinode, dentry); +} diff --git a/fs/qnx6/qnx6.h b/fs/qnx6/qnx6.h new file mode 100644 index 000000000..56ed13674 --- /dev/null +++ b/fs/qnx6/qnx6.h @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * QNX6 file system, Linux implementation. + * + * Version : 1.0.0 + * + * History : + * + * 01-02-2012 by Kai Bankett (chaosman@ontika.net) : first release. + * 16-02-2012 page map extension by Al Viro + * + */ + +#ifdef pr_fmt +#undef pr_fmt +#endif + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include + +typedef __u16 __bitwise __fs16; +typedef __u32 __bitwise __fs32; +typedef __u64 __bitwise __fs64; + +#include + +struct qnx6_sb_info { + struct buffer_head *sb_buf; /* superblock buffer */ + struct qnx6_super_block *sb; /* our superblock */ + int s_blks_off; /* blkoffset fs-startpoint */ + int s_ptrbits; /* indirect pointer bitfield */ + unsigned long s_mount_opt; /* all mount options */ + int s_bytesex; /* holds endianess info */ + struct inode * inodes; + struct inode * longfile; +}; + +struct qnx6_inode_info { + __fs32 di_block_ptr[QNX6_NO_DIRECT_POINTERS]; + __u8 di_filelevels; + __u32 i_dir_start_lookup; + struct inode vfs_inode; +}; + +extern struct inode *qnx6_iget(struct super_block *sb, unsigned ino); +extern struct dentry *qnx6_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags); + +#ifdef CONFIG_QNX6FS_DEBUG +extern void qnx6_superblock_debug(struct qnx6_super_block *, + struct super_block *); +#endif + +extern const struct inode_operations qnx6_dir_inode_operations; +extern const struct file_operations qnx6_dir_operations; + +static inline struct qnx6_sb_info *QNX6_SB(struct super_block *sb) +{ + return sb->s_fs_info; +} + +static inline struct qnx6_inode_info *QNX6_I(struct inode *inode) +{ + return container_of(inode, struct qnx6_inode_info, vfs_inode); +} + +#define clear_opt(o, opt) (o &= ~(QNX6_MOUNT_##opt)) +#define set_opt(o, opt) (o |= (QNX6_MOUNT_##opt)) +#define test_opt(sb, opt) (QNX6_SB(sb)->s_mount_opt & \ + QNX6_MOUNT_##opt) +enum { + BYTESEX_LE, + BYTESEX_BE, +}; + +static inline __u64 fs64_to_cpu(struct qnx6_sb_info *sbi, __fs64 n) +{ + if (sbi->s_bytesex == BYTESEX_LE) + return le64_to_cpu((__force __le64)n); + else + return be64_to_cpu((__force __be64)n); +} + +static inline __fs64 cpu_to_fs64(struct qnx6_sb_info *sbi, __u64 n) +{ + if (sbi->s_bytesex == BYTESEX_LE) + return (__force __fs64)cpu_to_le64(n); + else + return (__force __fs64)cpu_to_be64(n); +} + +static inline __u32 fs32_to_cpu(struct qnx6_sb_info *sbi, __fs32 n) +{ + if (sbi->s_bytesex == BYTESEX_LE) + return le32_to_cpu((__force __le32)n); + else + return be32_to_cpu((__force __be32)n); +} + +static inline __fs32 cpu_to_fs32(struct qnx6_sb_info *sbi, __u32 n) +{ + if (sbi->s_bytesex == BYTESEX_LE) + return (__force __fs32)cpu_to_le32(n); + else + return (__force __fs32)cpu_to_be32(n); +} + +static inline __u16 fs16_to_cpu(struct qnx6_sb_info *sbi, __fs16 n) +{ + if (sbi->s_bytesex == BYTESEX_LE) + return le16_to_cpu((__force __le16)n); + else + return be16_to_cpu((__force __be16)n); +} + +static inline __fs16 cpu_to_fs16(struct qnx6_sb_info *sbi, __u16 n) +{ + if (sbi->s_bytesex == BYTESEX_LE) + return (__force __fs16)cpu_to_le16(n); + else + return (__force __fs16)cpu_to_be16(n); +} + +extern struct qnx6_super_block *qnx6_mmi_fill_super(struct super_block *s, + int silent); + +unsigned qnx6_find_ino(int len, struct inode *dir, const char *name); diff --git a/fs/qnx6/super_mmi.c b/fs/qnx6/super_mmi.c new file mode 100644 index 000000000..b8afb6f38 --- /dev/null +++ b/fs/qnx6/super_mmi.c @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * QNX6 file system, Linux implementation. + * + * Version : 1.0.0 + * + * History : + * + * 01-02-2012 by Kai Bankett (chaosman@ontika.net) : first release. + * + */ + +#include +#include +#include +#include "qnx6.h" + +static void qnx6_mmi_copy_sb(struct qnx6_super_block *qsb, + struct qnx6_mmi_super_block *sb) +{ + qsb->sb_magic = sb->sb_magic; + qsb->sb_checksum = sb->sb_checksum; + qsb->sb_serial = sb->sb_serial; + qsb->sb_blocksize = sb->sb_blocksize; + qsb->sb_num_inodes = sb->sb_num_inodes; + qsb->sb_free_inodes = sb->sb_free_inodes; + qsb->sb_num_blocks = sb->sb_num_blocks; + qsb->sb_free_blocks = sb->sb_free_blocks; + + /* the rest of the superblock is the same */ + memcpy(&qsb->Inode, &sb->Inode, sizeof(sb->Inode)); + memcpy(&qsb->Bitmap, &sb->Bitmap, sizeof(sb->Bitmap)); + memcpy(&qsb->Longfile, &sb->Longfile, sizeof(sb->Longfile)); +} + +struct qnx6_super_block *qnx6_mmi_fill_super(struct super_block *s, int silent) +{ + struct buffer_head *bh1, *bh2 = NULL; + struct qnx6_mmi_super_block *sb1, *sb2; + struct qnx6_super_block *qsb = NULL; + struct qnx6_sb_info *sbi; + __u64 offset; + + /* Check the superblock signatures + start with the first superblock */ + bh1 = sb_bread(s, 0); + if (!bh1) { + pr_err("Unable to read first mmi superblock\n"); + return NULL; + } + sb1 = (struct qnx6_mmi_super_block *)bh1->b_data; + sbi = QNX6_SB(s); + if (fs32_to_cpu(sbi, sb1->sb_magic) != QNX6_SUPER_MAGIC) { + if (!silent) { + pr_err("wrong signature (magic) in superblock #1.\n"); + goto out; + } + } + + /* checksum check - start at byte 8 and end at byte 512 */ + if (fs32_to_cpu(sbi, sb1->sb_checksum) != + crc32_be(0, (char *)(bh1->b_data + 8), 504)) { + pr_err("superblock #1 checksum error\n"); + goto out; + } + + /* calculate second superblock blocknumber */ + offset = fs32_to_cpu(sbi, sb1->sb_num_blocks) + QNX6_SUPERBLOCK_AREA / + fs32_to_cpu(sbi, sb1->sb_blocksize); + + /* set new blocksize */ + if (!sb_set_blocksize(s, fs32_to_cpu(sbi, sb1->sb_blocksize))) { + pr_err("unable to set blocksize\n"); + goto out; + } + /* blocksize invalidates bh - pull it back in */ + brelse(bh1); + bh1 = sb_bread(s, 0); + if (!bh1) + goto out; + sb1 = (struct qnx6_mmi_super_block *)bh1->b_data; + + /* read second superblock */ + bh2 = sb_bread(s, offset); + if (!bh2) { + pr_err("unable to read the second superblock\n"); + goto out; + } + sb2 = (struct qnx6_mmi_super_block *)bh2->b_data; + if (fs32_to_cpu(sbi, sb2->sb_magic) != QNX6_SUPER_MAGIC) { + if (!silent) + pr_err("wrong signature (magic) in superblock #2.\n"); + goto out; + } + + /* checksum check - start at byte 8 and end at byte 512 */ + if (fs32_to_cpu(sbi, sb2->sb_checksum) + != crc32_be(0, (char *)(bh2->b_data + 8), 504)) { + pr_err("superblock #1 checksum error\n"); + goto out; + } + + qsb = kmalloc_obj(*qsb); + if (!qsb) { + pr_err("unable to allocate memory.\n"); + goto out; + } + + if (fs64_to_cpu(sbi, sb1->sb_serial) > + fs64_to_cpu(sbi, sb2->sb_serial)) { + /* superblock #1 active */ + qnx6_mmi_copy_sb(qsb, sb1); +#ifdef CONFIG_QNX6FS_DEBUG + qnx6_superblock_debug(qsb, s); +#endif + memcpy(bh1->b_data, qsb, sizeof(struct qnx6_super_block)); + + sbi->sb_buf = bh1; + sbi->sb = (struct qnx6_super_block *)bh1->b_data; + brelse(bh2); + pr_info("superblock #1 active\n"); + } else { + /* superblock #2 active */ + qnx6_mmi_copy_sb(qsb, sb2); +#ifdef CONFIG_QNX6FS_DEBUG + qnx6_superblock_debug(qsb, s); +#endif + memcpy(bh2->b_data, qsb, sizeof(struct qnx6_super_block)); + + sbi->sb_buf = bh2; + sbi->sb = (struct qnx6_super_block *)bh2->b_data; + brelse(bh1); + pr_info("superblock #2 active\n"); + } + kfree(qsb); + + /* offset for mmi_fs is just SUPERBLOCK_AREA bytes */ + sbi->s_blks_off = QNX6_SUPERBLOCK_AREA / s->s_blocksize; + + /* success */ + return sbi->sb; + +out: + if (bh1 != NULL) + brelse(bh1); + if (bh2 != NULL) + brelse(bh2); + return NULL; +} diff --git a/fs/quota/Kconfig b/fs/quota/Kconfig new file mode 100644 index 000000000..68eb44132 --- /dev/null +++ b/fs/quota/Kconfig @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Quota configuration +# + +config QUOTA + bool "Quota support" + select QUOTACTL + help + If you say Y here, you will be able to set per user limits for disk + usage (also called disk quotas). Currently, it works for the + ext2, ext3, ext4, f2fs, jfs and ocfs2 file systems. Note that gfs2 + and xfs use their own quota system. Ext3 and ext4 also support + journaled quotas for which you don't need to run quotacheck(8) after + an unclean shutdown. For further details, read the Quota mini-HOWTO, + available from , or the + documentation provided with the quota tools. Probably the quota + support is only useful for multi user systems. If unsure, say N. + +config QUOTA_NETLINK_INTERFACE + bool "Report quota messages through netlink interface" + depends on QUOTACTL && NET + help + If you say Y here, quota warnings (about exceeding softlimit, reaching + hardlimit, etc.) will be reported through netlink interface. If unsure, + say Y. + +config QUOTA_DEBUG + bool "Additional quota sanity checks" + depends on QUOTA + default n + help + If you say Y here, quota subsystem will perform some additional + sanity checks of quota internal structures. If unsure, say N. + +# Generic support for tree structured quota files. Selected when needed. +config QUOTA_TREE + tristate + +config QFMT_V1 + tristate "Old quota format support" + depends on QUOTA + help + This quota format was (is) used by kernels earlier than 2.4.22. If + you have quota working and you don't want to convert to new quota + format say Y here. + +config QFMT_V2 + tristate "Quota format vfsv0 and vfsv1 support" + depends on QUOTA + select QUOTA_TREE + help + This config option enables kernel support for vfsv0 and vfsv1 quota + formats. Both these formats support 32-bit UIDs/GIDs and vfsv1 format + also supports 64-bit inode and block quota limits. If you need this + functionality say Y here. + +config QUOTACTL + bool + default n diff --git a/fs/quota/Makefile b/fs/quota/Makefile new file mode 100644 index 000000000..9160639da --- /dev/null +++ b/fs/quota/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_QUOTA) += dquot.o +obj-$(CONFIG_QFMT_V1) += quota_v1.o +obj-$(CONFIG_QFMT_V2) += quota_v2.o +obj-$(CONFIG_QUOTA_TREE) += quota_tree.o +obj-$(CONFIG_QUOTACTL) += quota.o kqid.o +obj-$(CONFIG_QUOTA_NETLINK_INTERFACE) += netlink.o diff --git a/fs/quota/compat.h b/fs/quota/compat.h new file mode 100644 index 000000000..ef7d1e12d --- /dev/null +++ b/fs/quota/compat.h @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +struct compat_if_dqblk { + compat_u64 dqb_bhardlimit; + compat_u64 dqb_bsoftlimit; + compat_u64 dqb_curspace; + compat_u64 dqb_ihardlimit; + compat_u64 dqb_isoftlimit; + compat_u64 dqb_curinodes; + compat_u64 dqb_btime; + compat_u64 dqb_itime; + compat_uint_t dqb_valid; +}; + +struct compat_fs_qfilestat { + compat_u64 dqb_bhardlimit; + compat_u64 qfs_nblks; + compat_uint_t qfs_nextents; +}; + +struct compat_fs_quota_stat { + __s8 qs_version; + __u16 qs_flags; + __s8 qs_pad; + struct compat_fs_qfilestat qs_uquota; + struct compat_fs_qfilestat qs_gquota; + compat_uint_t qs_incoredqs; + compat_int_t qs_btimelimit; + compat_int_t qs_itimelimit; + compat_int_t qs_rtbtimelimit; + __u16 qs_bwarnlimit; + __u16 qs_iwarnlimit; +}; diff --git a/fs/quota/dquot.c b/fs/quota/dquot.c new file mode 100644 index 000000000..204afc5e9 --- /dev/null +++ b/fs/quota/dquot.c @@ -0,0 +1,2998 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Implementation of the diskquota system for the LINUX operating system. QUOTA + * is implemented using the BSD system call interface as the means of + * communication with the user level. This file contains the generic routines + * called by the different filesystems on allocation of an inode or block. + * These routines take care of the administration needed to have a consistent + * diskquota tracking system. The ideas of both user and group quotas are based + * on the Melbourne quota system as used on BSD derived systems. The internal + * implementation is based on one of the several variants of the LINUX + * inode-subsystem with added complexity of the diskquota system. + * + * Author: Marco van Wieringen + * + * Fixes: Dmitry Gorodchanin , 11 Feb 96 + * + * Revised list management to avoid races + * -- Bill Hawes, , 9/98 + * + * Fixed races in dquot_transfer(), dqget() and dquot_alloc_...(). + * As the consequence the locking was moved from dquot_decr_...(), + * dquot_incr_...() to calling functions. + * invalidate_dquots() now writes modified dquots. + * Serialized quota_off() and quota_on() for mount point. + * Fixed a few bugs in grow_dquots(). + * Fixed deadlock in write_dquot() - we no longer account quotas on + * quota files + * remove_dquot_ref() moved to inode.c - it now traverses through inodes + * add_dquot_ref() restarts after blocking + * Added check for bogus uid and fixed check for group in quotactl. + * Jan Kara, , sponsored by SuSE CR, 10-11/99 + * + * Used struct list_head instead of own list struct + * Invalidation of referenced dquots is no longer possible + * Improved free_dquots list management + * Quota and i_blocks are now updated in one place to avoid races + * Warnings are now delayed so we won't block in critical section + * Write updated not to require dquot lock + * Jan Kara, , 9/2000 + * + * Added dynamic quota structure allocation + * Jan Kara 12/2000 + * + * Rewritten quota interface. Implemented new quota format and + * formats registering. + * Jan Kara, , 2001,2002 + * + * New SMP locking. + * Jan Kara, , 10/2002 + * + * Added journalled quota support, fix lock inversion problems + * Jan Kara, , 2003,2004 + * + * (C) Copyright 1994 - 1997 Marco van Wieringen + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +/* + * There are five quota SMP locks: + * * dq_list_lock protects all lists with quotas and quota formats. + * * dquot->dq_dqb_lock protects data from dq_dqb + * * inode->i_lock protects inode->i_blocks, i_bytes and also guards + * consistency of dquot->dq_dqb with inode->i_blocks, i_bytes so that + * dquot_transfer() can stabilize amount it transfers + * * dq_data_lock protects mem_dqinfo structures and modifications of dquot + * pointers in the inode + * * dq_state_lock protects modifications of quota state (on quotaon and + * quotaoff) and readers who care about latest values take it as well. + * + * The spinlock ordering is hence: + * dq_data_lock > dq_list_lock > i_lock > dquot->dq_dqb_lock, + * dq_list_lock > dq_state_lock + * + * Note that some things (eg. sb pointer, type, id) doesn't change during + * the life of the dquot structure and so needn't to be protected by a lock + * + * Operation accessing dquots via inode pointers are protected by dquot_srcu. + * Operation of reading pointer needs srcu_read_lock(&dquot_srcu), and + * synchronize_srcu(&dquot_srcu) is called after clearing pointers from + * inode and before dropping dquot references to avoid use of dquots after + * they are freed. dq_data_lock is used to serialize the pointer setting and + * clearing operations. + * Special care needs to be taken about S_NOQUOTA inode flag (marking that + * inode is a quota file). Functions adding pointers from inode to dquots have + * to check this flag under dq_data_lock and then (if S_NOQUOTA is not set) they + * have to do all pointer modifications before dropping dq_data_lock. This makes + * sure they cannot race with quotaon which first sets S_NOQUOTA flag and + * then drops all pointers to dquots from an inode. + * + * Each dquot has its dq_lock mutex. Dquot is locked when it is being read to + * memory (or space for it is being allocated) on the first dqget(), when it is + * being written out, and when it is being released on the last dqput(). The + * allocation and release operations are serialized by the dq_lock and by + * checking the use count in dquot_release(). + * + * Lock ordering (including related VFS locks) is the following: + * s_umount > i_mutex > journal_lock > dquot->dq_lock > dqio_sem + */ + +static __cacheline_aligned_in_smp DEFINE_SPINLOCK(dq_list_lock); +static __cacheline_aligned_in_smp DEFINE_SPINLOCK(dq_state_lock); +__cacheline_aligned_in_smp DEFINE_SPINLOCK(dq_data_lock); +EXPORT_SYMBOL(dq_data_lock); +DEFINE_STATIC_SRCU(dquot_srcu); + +static DECLARE_WAIT_QUEUE_HEAD(dquot_ref_wq); + +void __quota_error(struct super_block *sb, const char *func, + const char *fmt, ...) +{ + if (printk_ratelimit()) { + va_list args; + struct va_format vaf; + + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + printk(KERN_ERR "Quota error (device %s): %s: %pV\n", + sb->s_id, func, &vaf); + + va_end(args); + } +} +EXPORT_SYMBOL(__quota_error); + +#ifdef CONFIG_QUOTA_DEBUG +static char *quotatypes[] = INITQFNAMES; +#endif +static struct quota_format_type *quota_formats; /* List of registered formats */ +static struct quota_module_name module_names[] = INIT_QUOTA_MODULE_NAMES; + +/* SLAB cache for dquot structures */ +static struct kmem_cache *dquot_cachep; + +/* workqueue for work quota_release_work*/ +static struct workqueue_struct *quota_unbound_wq; + +void register_quota_format(struct quota_format_type *fmt) +{ + spin_lock(&dq_list_lock); + fmt->qf_next = quota_formats; + quota_formats = fmt; + spin_unlock(&dq_list_lock); +} +EXPORT_SYMBOL(register_quota_format); + +void unregister_quota_format(struct quota_format_type *fmt) +{ + struct quota_format_type **actqf; + + spin_lock(&dq_list_lock); + for (actqf = "a_formats; *actqf && *actqf != fmt; + actqf = &(*actqf)->qf_next) + ; + if (*actqf) + *actqf = (*actqf)->qf_next; + spin_unlock(&dq_list_lock); +} +EXPORT_SYMBOL(unregister_quota_format); + +static struct quota_format_type *find_quota_format(int id) +{ + struct quota_format_type *actqf; + + spin_lock(&dq_list_lock); + for (actqf = quota_formats; actqf && actqf->qf_fmt_id != id; + actqf = actqf->qf_next) + ; + if (!actqf || !try_module_get(actqf->qf_owner)) { + int qm; + + spin_unlock(&dq_list_lock); + + for (qm = 0; module_names[qm].qm_fmt_id && + module_names[qm].qm_fmt_id != id; qm++) + ; + if (!module_names[qm].qm_fmt_id || + request_module(module_names[qm].qm_mod_name)) + return NULL; + + spin_lock(&dq_list_lock); + for (actqf = quota_formats; actqf && actqf->qf_fmt_id != id; + actqf = actqf->qf_next) + ; + if (actqf && !try_module_get(actqf->qf_owner)) + actqf = NULL; + } + spin_unlock(&dq_list_lock); + return actqf; +} + +static void put_quota_format(struct quota_format_type *fmt) +{ + module_put(fmt->qf_owner); +} + +/* + * Dquot List Management: + * The quota code uses five lists for dquot management: the inuse_list, + * releasing_dquots, free_dquots, dqi_dirty_list, and dquot_hash[] array. + * A single dquot structure may be on some of those lists, depending on + * its current state. + * + * All dquots are placed to the end of inuse_list when first created, and this + * list is used for invalidate operation, which must look at every dquot. + * + * When the last reference of a dquot is dropped, the dquot is added to + * releasing_dquots. We'll then queue work item which will call + * synchronize_srcu() and after that perform the final cleanup of all the + * dquots on the list. Each cleaned up dquot is moved to free_dquots list. + * Both releasing_dquots and free_dquots use the dq_free list_head in the dquot + * struct. + * + * Unused and cleaned up dquots are in the free_dquots list and this list is + * searched whenever we need an available dquot. Dquots are removed from the + * list as soon as they are used again and dqstats.free_dquots gives the number + * of dquots on the list. When dquot is invalidated it's completely released + * from memory. + * + * Dirty dquots are added to the dqi_dirty_list of quota_info when mark + * dirtied, and this list is searched when writing dirty dquots back to + * quota file. Note that some filesystems do dirty dquot tracking on their + * own (e.g. in a journal) and thus don't use dqi_dirty_list. + * + * Dquots with a specific identity (device, type and id) are placed on + * one of the dquot_hash[] hash chains. The provides an efficient search + * mechanism to locate a specific dquot. + */ + +static LIST_HEAD(inuse_list); +static LIST_HEAD(free_dquots); +static LIST_HEAD(releasing_dquots); +static unsigned int dq_hash_bits, dq_hash_mask; +static struct hlist_head *dquot_hash; + +struct dqstats dqstats; +EXPORT_SYMBOL(dqstats); + +static qsize_t inode_get_rsv_space(struct inode *inode); +static qsize_t __inode_get_rsv_space(struct inode *inode); +static int __dquot_initialize(struct inode *inode, int type); + +static void quota_release_workfn(struct work_struct *work); +static DECLARE_DELAYED_WORK(quota_release_work, quota_release_workfn); + +static inline unsigned int +hashfn(const struct super_block *sb, struct kqid qid) +{ + unsigned int id = from_kqid(&init_user_ns, qid); + int type = qid.type; + unsigned long tmp; + + tmp = (((unsigned long)sb>>L1_CACHE_SHIFT) ^ id) * (MAXQUOTAS - type); + return (tmp + (tmp >> dq_hash_bits)) & dq_hash_mask; +} + +/* + * Following list functions expect dq_list_lock to be held + */ +static inline void insert_dquot_hash(struct dquot *dquot) +{ + struct hlist_head *head; + head = dquot_hash + hashfn(dquot->dq_sb, dquot->dq_id); + hlist_add_head(&dquot->dq_hash, head); +} + +static inline void remove_dquot_hash(struct dquot *dquot) +{ + hlist_del_init(&dquot->dq_hash); +} + +static struct dquot *find_dquot(unsigned int hashent, struct super_block *sb, + struct kqid qid) +{ + struct dquot *dquot; + + hlist_for_each_entry(dquot, dquot_hash+hashent, dq_hash) + if (dquot->dq_sb == sb && qid_eq(dquot->dq_id, qid)) + return dquot; + + return NULL; +} + +/* Add a dquot to the tail of the free list */ +static inline void put_dquot_last(struct dquot *dquot) +{ + list_add_tail(&dquot->dq_free, &free_dquots); + dqstats_inc(DQST_FREE_DQUOTS); +} + +static inline void put_releasing_dquots(struct dquot *dquot) +{ + list_add_tail(&dquot->dq_free, &releasing_dquots); + set_bit(DQ_RELEASING_B, &dquot->dq_flags); +} + +static inline void remove_free_dquot(struct dquot *dquot) +{ + if (list_empty(&dquot->dq_free)) + return; + list_del_init(&dquot->dq_free); + if (!test_bit(DQ_RELEASING_B, &dquot->dq_flags)) + dqstats_dec(DQST_FREE_DQUOTS); + else + clear_bit(DQ_RELEASING_B, &dquot->dq_flags); +} + +static inline void put_inuse(struct dquot *dquot) +{ + /* We add to the back of inuse list so we don't have to restart + * when traversing this list and we block */ + list_add_tail(&dquot->dq_inuse, &inuse_list); + dqstats_inc(DQST_ALLOC_DQUOTS); +} + +static inline void remove_inuse(struct dquot *dquot) +{ + dqstats_dec(DQST_ALLOC_DQUOTS); + list_del(&dquot->dq_inuse); +} +/* + * End of list functions needing dq_list_lock + */ + +static void wait_on_dquot(struct dquot *dquot) +{ + mutex_lock(&dquot->dq_lock); + mutex_unlock(&dquot->dq_lock); +} + +static inline int dquot_active(struct dquot *dquot) +{ + return test_bit(DQ_ACTIVE_B, &dquot->dq_flags); +} + +static struct dquot *__dqgrab(struct dquot *dquot) +{ + lockdep_assert_held(&dq_list_lock); + if (!atomic_read(&dquot->dq_count)) + remove_free_dquot(dquot); + atomic_inc(&dquot->dq_count); + return dquot; +} + +/* + * Get reference to dquot when we got pointer to it by some other means. The + * dquot has to be active and the caller has to make sure it cannot get + * deactivated under our hands. + */ +struct dquot *dqgrab(struct dquot *dquot) +{ + spin_lock(&dq_list_lock); + WARN_ON_ONCE(!dquot_active(dquot)); + dquot = __dqgrab(dquot); + spin_unlock(&dq_list_lock); + + return dquot; +} +EXPORT_SYMBOL_GPL(dqgrab); + +static inline int dquot_dirty(struct dquot *dquot) +{ + return test_bit(DQ_MOD_B, &dquot->dq_flags); +} + +static inline int mark_dquot_dirty(struct dquot *dquot) +{ + return dquot->dq_sb->dq_op->mark_dirty(dquot); +} + +/* Mark dquot dirty in atomic manner, and return it's old dirty flag state */ +int dquot_mark_dquot_dirty(struct dquot *dquot) +{ + int ret = 1; + + if (!dquot_active(dquot)) + return 0; + + if (sb_dqopt(dquot->dq_sb)->flags & DQUOT_NOLIST_DIRTY) + return test_and_set_bit(DQ_MOD_B, &dquot->dq_flags); + + /* If quota is dirty already, we don't have to acquire dq_list_lock */ + if (dquot_dirty(dquot)) + return 1; + + spin_lock(&dq_list_lock); + if (!test_and_set_bit(DQ_MOD_B, &dquot->dq_flags)) { + list_add(&dquot->dq_dirty, &sb_dqopt(dquot->dq_sb)-> + info[dquot->dq_id.type].dqi_dirty_list); + ret = 0; + } + spin_unlock(&dq_list_lock); + return ret; +} +EXPORT_SYMBOL(dquot_mark_dquot_dirty); + +/* Dirtify all the dquots - this can block when journalling */ +static inline int mark_all_dquot_dirty(struct dquot __rcu * const *dquots) +{ + int ret, err, cnt; + struct dquot *dquot; + + ret = err = 0; + for (cnt = 0; cnt < MAXQUOTAS; cnt++) { + dquot = srcu_dereference(dquots[cnt], &dquot_srcu); + if (dquot) + /* Even in case of error we have to continue */ + ret = mark_dquot_dirty(dquot); + if (!err && ret < 0) + err = ret; + } + return err; +} + +static inline void dqput_all(struct dquot **dquot) +{ + unsigned int cnt; + + for (cnt = 0; cnt < MAXQUOTAS; cnt++) + dqput(dquot[cnt]); +} + +static inline int clear_dquot_dirty(struct dquot *dquot) +{ + if (sb_dqopt(dquot->dq_sb)->flags & DQUOT_NOLIST_DIRTY) + return test_and_clear_bit(DQ_MOD_B, &dquot->dq_flags); + + spin_lock(&dq_list_lock); + if (!test_and_clear_bit(DQ_MOD_B, &dquot->dq_flags)) { + spin_unlock(&dq_list_lock); + return 0; + } + list_del_init(&dquot->dq_dirty); + spin_unlock(&dq_list_lock); + return 1; +} + +void mark_info_dirty(struct super_block *sb, int type) +{ + spin_lock(&dq_data_lock); + sb_dqopt(sb)->info[type].dqi_flags |= DQF_INFO_DIRTY; + spin_unlock(&dq_data_lock); +} +EXPORT_SYMBOL(mark_info_dirty); + +/* + * Read dquot from disk and alloc space for it + */ + +int dquot_acquire(struct dquot *dquot) +{ + int ret = 0, ret2 = 0; + unsigned int memalloc; + struct quota_info *dqopt = sb_dqopt(dquot->dq_sb); + + mutex_lock(&dquot->dq_lock); + memalloc = memalloc_nofs_save(); + if (!test_bit(DQ_READ_B, &dquot->dq_flags)) { + ret = dqopt->ops[dquot->dq_id.type]->read_dqblk(dquot); + if (ret < 0) + goto out_iolock; + } + /* Make sure flags update is visible after dquot has been filled */ + smp_mb__before_atomic(); + set_bit(DQ_READ_B, &dquot->dq_flags); + /* Instantiate dquot if needed */ + if (!dquot_active(dquot) && !dquot->dq_off) { + ret = dqopt->ops[dquot->dq_id.type]->commit_dqblk(dquot); + /* Write the info if needed */ + if (info_dirty(&dqopt->info[dquot->dq_id.type])) { + ret2 = dqopt->ops[dquot->dq_id.type]->write_file_info( + dquot->dq_sb, dquot->dq_id.type); + } + if (ret < 0) + goto out_iolock; + if (ret2 < 0) { + ret = ret2; + goto out_iolock; + } + } + /* + * Make sure flags update is visible after on-disk struct has been + * allocated. Paired with smp_rmb() in dqget(). + */ + smp_mb__before_atomic(); + set_bit(DQ_ACTIVE_B, &dquot->dq_flags); +out_iolock: + memalloc_nofs_restore(memalloc); + mutex_unlock(&dquot->dq_lock); + return ret; +} +EXPORT_SYMBOL(dquot_acquire); + +/* + * Write dquot to disk + */ +int dquot_commit(struct dquot *dquot) +{ + int ret = 0; + unsigned int memalloc; + struct quota_info *dqopt = sb_dqopt(dquot->dq_sb); + + mutex_lock(&dquot->dq_lock); + memalloc = memalloc_nofs_save(); + if (!clear_dquot_dirty(dquot)) + goto out_lock; + /* Inactive dquot can be only if there was error during read/init + * => we have better not writing it */ + if (dquot_active(dquot)) + ret = dqopt->ops[dquot->dq_id.type]->commit_dqblk(dquot); + else + ret = -EIO; +out_lock: + memalloc_nofs_restore(memalloc); + mutex_unlock(&dquot->dq_lock); + return ret; +} +EXPORT_SYMBOL(dquot_commit); + +/* + * Release dquot + */ +int dquot_release(struct dquot *dquot) +{ + int ret = 0, ret2 = 0; + unsigned int memalloc; + struct quota_info *dqopt = sb_dqopt(dquot->dq_sb); + + mutex_lock(&dquot->dq_lock); + memalloc = memalloc_nofs_save(); + /* Check whether we are not racing with some other dqget() */ + if (dquot_is_busy(dquot)) + goto out_dqlock; + if (dqopt->ops[dquot->dq_id.type]->release_dqblk) { + ret = dqopt->ops[dquot->dq_id.type]->release_dqblk(dquot); + /* Write the info */ + if (info_dirty(&dqopt->info[dquot->dq_id.type])) { + ret2 = dqopt->ops[dquot->dq_id.type]->write_file_info( + dquot->dq_sb, dquot->dq_id.type); + } + if (ret >= 0) + ret = ret2; + } + clear_bit(DQ_ACTIVE_B, &dquot->dq_flags); +out_dqlock: + memalloc_nofs_restore(memalloc); + mutex_unlock(&dquot->dq_lock); + return ret; +} +EXPORT_SYMBOL(dquot_release); + +void dquot_destroy(struct dquot *dquot) +{ + kmem_cache_free(dquot_cachep, dquot); +} +EXPORT_SYMBOL(dquot_destroy); + +static inline void do_destroy_dquot(struct dquot *dquot) +{ + dquot->dq_sb->dq_op->destroy_dquot(dquot); +} + +/* Invalidate all dquots on the list. Note that this function is called after + * quota is disabled and pointers from inodes removed so there cannot be new + * quota users. There can still be some users of quotas due to inodes being + * just deleted or pruned by prune_icache() (those are not attached to any + * list) or parallel quotactl call. We have to wait for such users. + */ +static void invalidate_dquots(struct super_block *sb, int type) +{ + struct dquot *dquot, *tmp; + +restart: + flush_delayed_work("a_release_work); + + spin_lock(&dq_list_lock); + list_for_each_entry_safe(dquot, tmp, &inuse_list, dq_inuse) { + if (dquot->dq_sb != sb) + continue; + if (dquot->dq_id.type != type) + continue; + /* Wait for dquot users */ + if (atomic_read(&dquot->dq_count)) { + atomic_inc(&dquot->dq_count); + spin_unlock(&dq_list_lock); + /* + * Once dqput() wakes us up, we know it's time to free + * the dquot. + * IMPORTANT: we rely on the fact that there is always + * at most one process waiting for dquot to free. + * Otherwise dq_count would be > 1 and we would never + * wake up. + */ + wait_event(dquot_ref_wq, + atomic_read(&dquot->dq_count) == 1); + dqput(dquot); + /* At this moment dquot() need not exist (it could be + * reclaimed by prune_dqcache(). Hence we must + * restart. */ + goto restart; + } + /* + * The last user already dropped its reference but dquot didn't + * get fully cleaned up yet. Restart the scan which flushes the + * work cleaning up released dquots. + */ + if (test_bit(DQ_RELEASING_B, &dquot->dq_flags)) { + spin_unlock(&dq_list_lock); + goto restart; + } + /* + * Quota now has no users and it has been written on last + * dqput() + */ + remove_dquot_hash(dquot); + remove_free_dquot(dquot); + remove_inuse(dquot); + do_destroy_dquot(dquot); + } + spin_unlock(&dq_list_lock); +} + +/* Call callback for every active dquot on given filesystem */ +int dquot_scan_active(struct super_block *sb, + int (*fn)(struct dquot *dquot, unsigned long priv), + unsigned long priv) +{ + struct dquot *dquot, *old_dquot = NULL; + int ret = 0; + + WARN_ON_ONCE(!rwsem_is_locked(&sb->s_umount)); + + spin_lock(&dq_list_lock); + list_for_each_entry(dquot, &inuse_list, dq_inuse) { + if (!dquot_active(dquot)) + continue; + if (dquot->dq_sb != sb) + continue; + __dqgrab(dquot); + spin_unlock(&dq_list_lock); + dqput(old_dquot); + old_dquot = dquot; + /* + * ->release_dquot() can be racing with us. Our reference + * protects us from dquot_release() proceeding so just wait for + * any outstanding call and recheck the DQ_ACTIVE_B after that. + */ + wait_on_dquot(dquot); + if (dquot_active(dquot)) { + ret = fn(dquot, priv); + if (ret < 0) + goto out; + } + spin_lock(&dq_list_lock); + /* We are safe to continue now because our dquot could not + * be moved out of the inuse list while we hold the reference */ + } + spin_unlock(&dq_list_lock); +out: + dqput(old_dquot); + return ret; +} +EXPORT_SYMBOL(dquot_scan_active); + +static inline int dquot_write_dquot(struct dquot *dquot) +{ + int ret = dquot->dq_sb->dq_op->write_dquot(dquot); + if (ret < 0) { + quota_error(dquot->dq_sb, "Can't write quota structure " + "(error %d). Quota may get out of sync!", ret); + /* Clear dirty bit anyway to avoid infinite loop. */ + clear_dquot_dirty(dquot); + } + return ret; +} + +/* Write all dquot structures to quota files */ +int dquot_writeback_dquots(struct super_block *sb, int type) +{ + struct list_head dirty; + struct dquot *dquot; + struct quota_info *dqopt = sb_dqopt(sb); + int cnt; + int err, ret = 0; + + WARN_ON_ONCE(!rwsem_is_locked(&sb->s_umount)); + + flush_delayed_work("a_release_work); + + for (cnt = 0; cnt < MAXQUOTAS; cnt++) { + if (type != -1 && cnt != type) + continue; + if (!sb_has_quota_active(sb, cnt)) + continue; + spin_lock(&dq_list_lock); + /* Move list away to avoid livelock. */ + list_replace_init(&dqopt->info[cnt].dqi_dirty_list, &dirty); + while (!list_empty(&dirty)) { + dquot = list_first_entry(&dirty, struct dquot, + dq_dirty); + + WARN_ON(!dquot_active(dquot)); + /* If the dquot is releasing we should not touch it */ + if (test_bit(DQ_RELEASING_B, &dquot->dq_flags)) { + spin_unlock(&dq_list_lock); + flush_delayed_work("a_release_work); + spin_lock(&dq_list_lock); + continue; + } + + /* Now we have active dquot from which someone is + * holding reference so we can safely just increase + * use count */ + __dqgrab(dquot); + spin_unlock(&dq_list_lock); + err = dquot_write_dquot(dquot); + if (err && !ret) + ret = err; + dqput(dquot); + spin_lock(&dq_list_lock); + } + spin_unlock(&dq_list_lock); + } + + for (cnt = 0; cnt < MAXQUOTAS; cnt++) + if ((cnt == type || type == -1) && sb_has_quota_active(sb, cnt) + && info_dirty(&dqopt->info[cnt])) + sb->dq_op->write_info(sb, cnt); + dqstats_inc(DQST_SYNCS); + + return ret; +} +EXPORT_SYMBOL(dquot_writeback_dquots); + +/* Write all dquot structures to disk and make them visible from userspace */ +int dquot_quota_sync(struct super_block *sb, int type) +{ + struct quota_info *dqopt = sb_dqopt(sb); + int cnt; + int ret; + + ret = dquot_writeback_dquots(sb, type); + if (ret) + return ret; + if (dqopt->flags & DQUOT_QUOTA_SYS_FILE) + return 0; + + /* This is not very clever (and fast) but currently I don't know about + * any other simple way of getting quota data to disk and we must get + * them there for userspace to be visible... */ + if (sb->s_op->sync_fs) { + ret = sb->s_op->sync_fs(sb, 1); + if (ret) + return ret; + } + ret = sync_blockdev(sb->s_bdev); + if (ret) + return ret; + + /* + * Now when everything is written we can discard the pagecache so + * that userspace sees the changes. + */ + for (cnt = 0; cnt < MAXQUOTAS; cnt++) { + if (type != -1 && cnt != type) + continue; + if (!sb_has_quota_active(sb, cnt)) + continue; + inode_lock(dqopt->files[cnt]); + truncate_inode_pages(&dqopt->files[cnt]->i_data, 0); + inode_unlock(dqopt->files[cnt]); + } + + return 0; +} +EXPORT_SYMBOL(dquot_quota_sync); + +static unsigned long +dqcache_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) +{ + struct dquot *dquot; + unsigned long freed = 0; + + spin_lock(&dq_list_lock); + while (!list_empty(&free_dquots) && sc->nr_to_scan) { + dquot = list_first_entry(&free_dquots, struct dquot, dq_free); + remove_dquot_hash(dquot); + remove_free_dquot(dquot); + remove_inuse(dquot); + do_destroy_dquot(dquot); + sc->nr_to_scan--; + freed++; + } + spin_unlock(&dq_list_lock); + return freed; +} + +static unsigned long +dqcache_shrink_count(struct shrinker *shrink, struct shrink_control *sc) +{ + return vfs_pressure_ratio( + percpu_counter_read_positive(&dqstats.counter[DQST_FREE_DQUOTS])); +} + +/* + * Safely release dquot and put reference to dquot. + */ +static void quota_release_workfn(struct work_struct *work) +{ + struct dquot *dquot; + struct list_head rls_head; + + spin_lock(&dq_list_lock); + /* Exchange the list head to avoid livelock. */ + list_replace_init(&releasing_dquots, &rls_head); + spin_unlock(&dq_list_lock); + synchronize_srcu(&dquot_srcu); + +restart: + spin_lock(&dq_list_lock); + while (!list_empty(&rls_head)) { + dquot = list_first_entry(&rls_head, struct dquot, dq_free); + WARN_ON_ONCE(atomic_read(&dquot->dq_count)); + /* + * Note that DQ_RELEASING_B protects us from racing with + * invalidate_dquots() calls so we are safe to work with the + * dquot even after we drop dq_list_lock. + */ + if (dquot_dirty(dquot)) { + spin_unlock(&dq_list_lock); + /* Commit dquot before releasing */ + dquot_write_dquot(dquot); + goto restart; + } + if (dquot_active(dquot)) { + spin_unlock(&dq_list_lock); + dquot->dq_sb->dq_op->release_dquot(dquot); + goto restart; + } + /* Dquot is inactive and clean, now move it to free list */ + remove_free_dquot(dquot); + put_dquot_last(dquot); + } + spin_unlock(&dq_list_lock); +} + +/* + * Put reference to dquot + */ +void dqput(struct dquot *dquot) +{ + if (!dquot) + return; +#ifdef CONFIG_QUOTA_DEBUG + if (!atomic_read(&dquot->dq_count)) { + quota_error(dquot->dq_sb, "trying to free free dquot of %s %d", + quotatypes[dquot->dq_id.type], + from_kqid(&init_user_ns, dquot->dq_id)); + BUG(); + } +#endif + dqstats_inc(DQST_DROPS); + + spin_lock(&dq_list_lock); + if (atomic_read(&dquot->dq_count) > 1) { + /* We have more than one user... nothing to do */ + atomic_dec(&dquot->dq_count); + /* Releasing dquot during quotaoff phase? */ + if (!sb_has_quota_active(dquot->dq_sb, dquot->dq_id.type) && + atomic_read(&dquot->dq_count) == 1) + wake_up(&dquot_ref_wq); + spin_unlock(&dq_list_lock); + return; + } + + /* Need to release dquot? */ + WARN_ON_ONCE(!list_empty(&dquot->dq_free)); + put_releasing_dquots(dquot); + atomic_dec(&dquot->dq_count); + spin_unlock(&dq_list_lock); + queue_delayed_work(quota_unbound_wq, "a_release_work, 1); +} +EXPORT_SYMBOL(dqput); + +struct dquot *dquot_alloc(struct super_block *sb, int type) +{ + return kmem_cache_zalloc(dquot_cachep, GFP_NOFS); +} +EXPORT_SYMBOL(dquot_alloc); + +static struct dquot *get_empty_dquot(struct super_block *sb, int type) +{ + struct dquot *dquot; + + dquot = sb->dq_op->alloc_dquot(sb, type); + if(!dquot) + return NULL; + + mutex_init(&dquot->dq_lock); + INIT_LIST_HEAD(&dquot->dq_free); + INIT_LIST_HEAD(&dquot->dq_inuse); + INIT_HLIST_NODE(&dquot->dq_hash); + INIT_LIST_HEAD(&dquot->dq_dirty); + dquot->dq_sb = sb; + dquot->dq_id = make_kqid_invalid(type); + atomic_set(&dquot->dq_count, 1); + spin_lock_init(&dquot->dq_dqb_lock); + + return dquot; +} + +/* + * Get reference to dquot + * + * Locking is slightly tricky here. We are guarded from parallel quotaoff() + * destroying our dquot by: + * a) checking for quota flags under dq_list_lock and + * b) getting a reference to dquot before we release dq_list_lock + */ +struct dquot *dqget(struct super_block *sb, struct kqid qid) +{ + unsigned int hashent = hashfn(sb, qid); + struct dquot *dquot, *empty = NULL; + + if (!qid_has_mapping(sb->s_user_ns, qid)) + return ERR_PTR(-EINVAL); + + if (!sb_has_quota_active(sb, qid.type)) + return ERR_PTR(-ESRCH); +we_slept: + spin_lock(&dq_list_lock); + spin_lock(&dq_state_lock); + if (!sb_has_quota_active(sb, qid.type)) { + spin_unlock(&dq_state_lock); + spin_unlock(&dq_list_lock); + dquot = ERR_PTR(-ESRCH); + goto out; + } + spin_unlock(&dq_state_lock); + + dquot = find_dquot(hashent, sb, qid); + if (!dquot) { + if (!empty) { + spin_unlock(&dq_list_lock); + empty = get_empty_dquot(sb, qid.type); + if (!empty) + schedule(); /* Try to wait for a moment... */ + goto we_slept; + } + dquot = empty; + empty = NULL; + dquot->dq_id = qid; + /* all dquots go on the inuse_list */ + put_inuse(dquot); + /* hash it first so it can be found */ + insert_dquot_hash(dquot); + spin_unlock(&dq_list_lock); + dqstats_inc(DQST_LOOKUPS); + } else { + __dqgrab(dquot); + spin_unlock(&dq_list_lock); + dqstats_inc(DQST_CACHE_HITS); + dqstats_inc(DQST_LOOKUPS); + } + /* Wait for dq_lock - after this we know that either dquot_release() is + * already finished or it will be canceled due to dq_count > 0 test */ + wait_on_dquot(dquot); + /* Read the dquot / allocate space in quota file */ + if (!dquot_active(dquot)) { + int err; + + err = sb->dq_op->acquire_dquot(dquot); + if (err < 0) { + dqput(dquot); + dquot = ERR_PTR(err); + goto out; + } + } + /* + * Make sure following reads see filled structure - paired with + * smp_mb__before_atomic() in dquot_acquire(). + */ + smp_rmb(); + /* Has somebody invalidated entry under us? */ + WARN_ON_ONCE(hlist_unhashed(&dquot->dq_hash)); +out: + if (empty) + do_destroy_dquot(empty); + + return dquot; +} +EXPORT_SYMBOL(dqget); + +static inline struct dquot __rcu **i_dquot(struct inode *inode) +{ + return inode->i_sb->s_op->get_dquots(inode); +} + +static int dqinit_needed(struct inode *inode, int type) +{ + struct dquot __rcu * const *dquots; + int cnt; + + if (IS_NOQUOTA(inode)) + return 0; + + dquots = i_dquot(inode); + if (type != -1) + return !dquots[type]; + for (cnt = 0; cnt < MAXQUOTAS; cnt++) + if (!dquots[cnt]) + return 1; + return 0; +} + +/* This routine is guarded by s_umount semaphore */ +static int add_dquot_ref(struct super_block *sb, int type) +{ + struct inode *inode, *old_inode = NULL; +#ifdef CONFIG_QUOTA_DEBUG + int reserved = 0; +#endif + int err = 0; + + spin_lock(&sb->s_inode_list_lock); + list_for_each_entry(inode, &sb->s_inodes, i_sb_list) { + spin_lock(&inode->i_lock); + if ((inode_state_read(inode) & (I_FREEING | I_WILL_FREE | I_NEW)) || + !atomic_read(&inode->i_writecount) || + !dqinit_needed(inode, type)) { + spin_unlock(&inode->i_lock); + continue; + } + __iget(inode); + spin_unlock(&inode->i_lock); + spin_unlock(&sb->s_inode_list_lock); + +#ifdef CONFIG_QUOTA_DEBUG + if (unlikely(inode_get_rsv_space(inode) > 0)) + reserved = 1; +#endif + iput(old_inode); + err = __dquot_initialize(inode, type); + if (err) { + iput(inode); + goto out; + } + + /* + * We hold a reference to 'inode' so it couldn't have been + * removed from s_inodes list while we dropped the + * s_inode_list_lock. We cannot iput the inode now as we can be + * holding the last reference and we cannot iput it under + * s_inode_list_lock. So we keep the reference and iput it + * later. + */ + old_inode = inode; + cond_resched(); + spin_lock(&sb->s_inode_list_lock); + } + spin_unlock(&sb->s_inode_list_lock); + iput(old_inode); +out: +#ifdef CONFIG_QUOTA_DEBUG + if (reserved) { + quota_error(sb, "Writes happened before quota was turned on " + "thus quota information is probably inconsistent. " + "Please run quotacheck(8)"); + } +#endif + return err; +} + +static void remove_dquot_ref(struct super_block *sb, int type) +{ + struct inode *inode; +#ifdef CONFIG_QUOTA_DEBUG + int reserved = 0; +#endif + + spin_lock(&sb->s_inode_list_lock); + list_for_each_entry(inode, &sb->s_inodes, i_sb_list) { + /* + * We have to scan also I_NEW inodes because they can already + * have quota pointer initialized. Luckily, we need to touch + * only quota pointers and these have separate locking + * (dq_data_lock). + */ + spin_lock(&dq_data_lock); + if (!IS_NOQUOTA(inode)) { + struct dquot __rcu **dquots = i_dquot(inode); + struct dquot *dquot = srcu_dereference_check( + dquots[type], &dquot_srcu, + lockdep_is_held(&dq_data_lock)); + +#ifdef CONFIG_QUOTA_DEBUG + if (unlikely(inode_get_rsv_space(inode) > 0)) + reserved = 1; +#endif + rcu_assign_pointer(dquots[type], NULL); + if (dquot) + dqput(dquot); + } + spin_unlock(&dq_data_lock); + } + spin_unlock(&sb->s_inode_list_lock); +#ifdef CONFIG_QUOTA_DEBUG + if (reserved) { + printk(KERN_WARNING "VFS (%s): Writes happened after quota" + " was disabled thus quota information is probably " + "inconsistent. Please run quotacheck(8).\n", sb->s_id); + } +#endif +} + +/* Gather all references from inodes and drop them */ +static void drop_dquot_ref(struct super_block *sb, int type) +{ + if (sb->dq_op) + remove_dquot_ref(sb, type); +} + +static inline +void dquot_free_reserved_space(struct dquot *dquot, qsize_t number) +{ + if (dquot->dq_dqb.dqb_rsvspace >= number) + dquot->dq_dqb.dqb_rsvspace -= number; + else { + WARN_ON_ONCE(1); + dquot->dq_dqb.dqb_rsvspace = 0; + } + if (dquot->dq_dqb.dqb_curspace + dquot->dq_dqb.dqb_rsvspace <= + dquot->dq_dqb.dqb_bsoftlimit) + dquot->dq_dqb.dqb_btime = (time64_t) 0; + clear_bit(DQ_BLKS_B, &dquot->dq_flags); +} + +static void dquot_decr_inodes(struct dquot *dquot, qsize_t number) +{ + if (sb_dqopt(dquot->dq_sb)->flags & DQUOT_NEGATIVE_USAGE || + dquot->dq_dqb.dqb_curinodes >= number) + dquot->dq_dqb.dqb_curinodes -= number; + else + dquot->dq_dqb.dqb_curinodes = 0; + if (dquot->dq_dqb.dqb_curinodes <= dquot->dq_dqb.dqb_isoftlimit) + dquot->dq_dqb.dqb_itime = (time64_t) 0; + clear_bit(DQ_INODES_B, &dquot->dq_flags); +} + +static void dquot_decr_space(struct dquot *dquot, qsize_t number) +{ + if (sb_dqopt(dquot->dq_sb)->flags & DQUOT_NEGATIVE_USAGE || + dquot->dq_dqb.dqb_curspace >= number) + dquot->dq_dqb.dqb_curspace -= number; + else + dquot->dq_dqb.dqb_curspace = 0; + if (dquot->dq_dqb.dqb_curspace + dquot->dq_dqb.dqb_rsvspace <= + dquot->dq_dqb.dqb_bsoftlimit) + dquot->dq_dqb.dqb_btime = (time64_t) 0; + clear_bit(DQ_BLKS_B, &dquot->dq_flags); +} + +struct dquot_warn { + struct super_block *w_sb; + struct kqid w_dq_id; + short w_type; +}; + +static int warning_issued(struct dquot *dquot, const int warntype) +{ + int flag = (warntype == QUOTA_NL_BHARDWARN || + warntype == QUOTA_NL_BSOFTLONGWARN) ? DQ_BLKS_B : + ((warntype == QUOTA_NL_IHARDWARN || + warntype == QUOTA_NL_ISOFTLONGWARN) ? DQ_INODES_B : 0); + + if (!flag) + return 0; + return test_and_set_bit(flag, &dquot->dq_flags); +} + +static void prepare_warning(struct dquot_warn *warn, struct dquot *dquot, + int warntype) +{ + if (warning_issued(dquot, warntype)) + return; + warn->w_type = warntype; + warn->w_sb = dquot->dq_sb; + warn->w_dq_id = dquot->dq_id; +} + +/* + * Write warnings to the console and send warning messages over netlink. + * + * Note that this function can call into tty and networking code. + */ +static void flush_warnings(struct dquot_warn *warn) +{ + int i; + + for (i = 0; i < MAXQUOTAS; i++) { + if (warn[i].w_type == QUOTA_NL_NOWARN) + continue; + + quota_send_warning(warn[i].w_dq_id, + warn[i].w_sb->s_dev, warn[i].w_type); + } +} + +static int ignore_hardlimit(struct dquot *dquot) +{ + struct mem_dqinfo *info = &sb_dqopt(dquot->dq_sb)->info[dquot->dq_id.type]; + + return capable(CAP_SYS_RESOURCE) && + (info->dqi_format->qf_fmt_id != QFMT_VFS_OLD || + !(info->dqi_flags & DQF_ROOT_SQUASH)); +} + +static int dquot_add_inodes(struct dquot *dquot, qsize_t inodes, + struct dquot_warn *warn) +{ + qsize_t newinodes; + int ret = 0; + + spin_lock(&dquot->dq_dqb_lock); + newinodes = dquot->dq_dqb.dqb_curinodes + inodes; + if (!sb_has_quota_limits_enabled(dquot->dq_sb, dquot->dq_id.type) || + test_bit(DQ_FAKE_B, &dquot->dq_flags)) + goto add; + + if (dquot->dq_dqb.dqb_ihardlimit && + newinodes > dquot->dq_dqb.dqb_ihardlimit && + !ignore_hardlimit(dquot)) { + prepare_warning(warn, dquot, QUOTA_NL_IHARDWARN); + ret = -EDQUOT; + goto out; + } + + if (dquot->dq_dqb.dqb_isoftlimit && + newinodes > dquot->dq_dqb.dqb_isoftlimit && + dquot->dq_dqb.dqb_itime && + ktime_get_real_seconds() >= dquot->dq_dqb.dqb_itime && + !ignore_hardlimit(dquot)) { + prepare_warning(warn, dquot, QUOTA_NL_ISOFTLONGWARN); + ret = -EDQUOT; + goto out; + } + + if (dquot->dq_dqb.dqb_isoftlimit && + newinodes > dquot->dq_dqb.dqb_isoftlimit && + dquot->dq_dqb.dqb_itime == 0) { + prepare_warning(warn, dquot, QUOTA_NL_ISOFTWARN); + dquot->dq_dqb.dqb_itime = ktime_get_real_seconds() + + sb_dqopt(dquot->dq_sb)->info[dquot->dq_id.type].dqi_igrace; + } +add: + dquot->dq_dqb.dqb_curinodes = newinodes; + +out: + spin_unlock(&dquot->dq_dqb_lock); + return ret; +} + +static int dquot_add_space(struct dquot *dquot, qsize_t space, + qsize_t rsv_space, unsigned int flags, + struct dquot_warn *warn) +{ + qsize_t tspace; + struct super_block *sb = dquot->dq_sb; + int ret = 0; + + spin_lock(&dquot->dq_dqb_lock); + if (!sb_has_quota_limits_enabled(sb, dquot->dq_id.type) || + test_bit(DQ_FAKE_B, &dquot->dq_flags)) + goto finish; + + tspace = dquot->dq_dqb.dqb_curspace + dquot->dq_dqb.dqb_rsvspace + + space + rsv_space; + + if (dquot->dq_dqb.dqb_bhardlimit && + tspace > dquot->dq_dqb.dqb_bhardlimit && + !ignore_hardlimit(dquot)) { + if (flags & DQUOT_SPACE_WARN) + prepare_warning(warn, dquot, QUOTA_NL_BHARDWARN); + ret = -EDQUOT; + goto finish; + } + + if (dquot->dq_dqb.dqb_bsoftlimit && + tspace > dquot->dq_dqb.dqb_bsoftlimit && + dquot->dq_dqb.dqb_btime && + ktime_get_real_seconds() >= dquot->dq_dqb.dqb_btime && + !ignore_hardlimit(dquot)) { + if (flags & DQUOT_SPACE_WARN) + prepare_warning(warn, dquot, QUOTA_NL_BSOFTLONGWARN); + ret = -EDQUOT; + goto finish; + } + + if (dquot->dq_dqb.dqb_bsoftlimit && + tspace > dquot->dq_dqb.dqb_bsoftlimit && + dquot->dq_dqb.dqb_btime == 0) { + if (flags & DQUOT_SPACE_WARN) { + prepare_warning(warn, dquot, QUOTA_NL_BSOFTWARN); + dquot->dq_dqb.dqb_btime = ktime_get_real_seconds() + + sb_dqopt(sb)->info[dquot->dq_id.type].dqi_bgrace; + } else { + /* + * We don't allow preallocation to exceed softlimit so exceeding will + * be always printed + */ + ret = -EDQUOT; + goto finish; + } + } +finish: + /* + * We have to be careful and go through warning generation & grace time + * setting even if DQUOT_SPACE_NOFAIL is set. That's why we check it + * only here... + */ + if (flags & DQUOT_SPACE_NOFAIL) + ret = 0; + if (!ret) { + dquot->dq_dqb.dqb_rsvspace += rsv_space; + dquot->dq_dqb.dqb_curspace += space; + } + spin_unlock(&dquot->dq_dqb_lock); + return ret; +} + +static int info_idq_free(struct dquot *dquot, qsize_t inodes) +{ + qsize_t newinodes; + + if (test_bit(DQ_FAKE_B, &dquot->dq_flags) || + dquot->dq_dqb.dqb_curinodes <= dquot->dq_dqb.dqb_isoftlimit || + !sb_has_quota_limits_enabled(dquot->dq_sb, dquot->dq_id.type)) + return QUOTA_NL_NOWARN; + + newinodes = dquot->dq_dqb.dqb_curinodes - inodes; + if (newinodes <= dquot->dq_dqb.dqb_isoftlimit) + return QUOTA_NL_ISOFTBELOW; + if (dquot->dq_dqb.dqb_curinodes >= dquot->dq_dqb.dqb_ihardlimit && + newinodes < dquot->dq_dqb.dqb_ihardlimit) + return QUOTA_NL_IHARDBELOW; + return QUOTA_NL_NOWARN; +} + +static int info_bdq_free(struct dquot *dquot, qsize_t space) +{ + qsize_t tspace; + + tspace = dquot->dq_dqb.dqb_curspace + dquot->dq_dqb.dqb_rsvspace; + + if (test_bit(DQ_FAKE_B, &dquot->dq_flags) || + tspace <= dquot->dq_dqb.dqb_bsoftlimit) + return QUOTA_NL_NOWARN; + + if (tspace - space <= dquot->dq_dqb.dqb_bsoftlimit) + return QUOTA_NL_BSOFTBELOW; + if (tspace >= dquot->dq_dqb.dqb_bhardlimit && + tspace - space < dquot->dq_dqb.dqb_bhardlimit) + return QUOTA_NL_BHARDBELOW; + return QUOTA_NL_NOWARN; +} + +static int inode_quota_active(const struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + + if (IS_NOQUOTA(inode)) + return 0; + return sb_any_quota_loaded(sb) & ~sb_any_quota_suspended(sb); +} + +/* + * Initialize quota pointers in inode + * + * It is better to call this function outside of any transaction as it + * might need a lot of space in journal for dquot structure allocation. + */ +static int __dquot_initialize(struct inode *inode, int type) +{ + int cnt, init_needed = 0; + struct dquot __rcu **dquots; + struct dquot *got[MAXQUOTAS] = {}; + struct super_block *sb = inode->i_sb; + qsize_t rsv; + int ret = 0; + + if (!inode_quota_active(inode)) + return 0; + + dquots = i_dquot(inode); + + /* First get references to structures we might need. */ + for (cnt = 0; cnt < MAXQUOTAS; cnt++) { + struct kqid qid; + kprojid_t projid; + int rc; + struct dquot *dquot; + + if (type != -1 && cnt != type) + continue; + /* + * The i_dquot should have been initialized in most cases, + * we check it without locking here to avoid unnecessary + * dqget()/dqput() calls. + */ + if (dquots[cnt]) + continue; + + if (!sb_has_quota_active(sb, cnt)) + continue; + + init_needed = 1; + + switch (cnt) { + case USRQUOTA: + qid = make_kqid_uid(inode->i_uid); + break; + case GRPQUOTA: + qid = make_kqid_gid(inode->i_gid); + break; + case PRJQUOTA: + rc = inode->i_sb->dq_op->get_projid(inode, &projid); + if (rc) + continue; + qid = make_kqid_projid(projid); + break; + } + dquot = dqget(sb, qid); + if (IS_ERR(dquot)) { + /* We raced with somebody turning quotas off... */ + if (PTR_ERR(dquot) != -ESRCH) { + ret = PTR_ERR(dquot); + goto out_put; + } + dquot = NULL; + } + got[cnt] = dquot; + } + + /* All required i_dquot has been initialized */ + if (!init_needed) + return 0; + + spin_lock(&dq_data_lock); + if (IS_NOQUOTA(inode)) + goto out_lock; + for (cnt = 0; cnt < MAXQUOTAS; cnt++) { + if (type != -1 && cnt != type) + continue; + /* Avoid races with quotaoff() */ + if (!sb_has_quota_active(sb, cnt)) + continue; + /* We could race with quotaon or dqget() could have failed */ + if (!got[cnt]) + continue; + if (!dquots[cnt]) { + rcu_assign_pointer(dquots[cnt], got[cnt]); + got[cnt] = NULL; + /* + * Make quota reservation system happy if someone + * did a write before quota was turned on + */ + rsv = inode_get_rsv_space(inode); + if (unlikely(rsv)) { + struct dquot *dquot = srcu_dereference_check( + dquots[cnt], &dquot_srcu, + lockdep_is_held(&dq_data_lock)); + + spin_lock(&inode->i_lock); + /* Get reservation again under proper lock */ + rsv = __inode_get_rsv_space(inode); + spin_lock(&dquot->dq_dqb_lock); + dquot->dq_dqb.dqb_rsvspace += rsv; + spin_unlock(&dquot->dq_dqb_lock); + spin_unlock(&inode->i_lock); + } + } + } +out_lock: + spin_unlock(&dq_data_lock); +out_put: + /* Drop unused references */ + dqput_all(got); + + return ret; +} + +int dquot_initialize(struct inode *inode) +{ + return __dquot_initialize(inode, -1); +} +EXPORT_SYMBOL(dquot_initialize); + +bool dquot_initialize_needed(struct inode *inode) +{ + struct dquot __rcu **dquots; + int i; + + if (!inode_quota_active(inode)) + return false; + + dquots = i_dquot(inode); + for (i = 0; i < MAXQUOTAS; i++) + if (!dquots[i] && sb_has_quota_active(inode->i_sb, i)) + return true; + return false; +} +EXPORT_SYMBOL(dquot_initialize_needed); + +/* + * Release all quotas referenced by inode. + * + * This function only be called on inode free or converting + * a file to quota file, no other users for the i_dquot in + * both cases, so we needn't call synchronize_srcu() after + * clearing i_dquot. + */ +static void __dquot_drop(struct inode *inode) +{ + int cnt; + struct dquot __rcu **dquots = i_dquot(inode); + struct dquot *put[MAXQUOTAS]; + + spin_lock(&dq_data_lock); + for (cnt = 0; cnt < MAXQUOTAS; cnt++) { + put[cnt] = srcu_dereference_check(dquots[cnt], &dquot_srcu, + lockdep_is_held(&dq_data_lock)); + rcu_assign_pointer(dquots[cnt], NULL); + } + spin_unlock(&dq_data_lock); + dqput_all(put); +} + +void dquot_drop(struct inode *inode) +{ + struct dquot __rcu * const *dquots; + int cnt; + + if (IS_NOQUOTA(inode)) + return; + + /* + * Test before calling to rule out calls from proc and such + * where we are not allowed to block. Note that this is + * actually reliable test even without the lock - the caller + * must assure that nobody can come after the DQUOT_DROP and + * add quota pointers back anyway. + */ + dquots = i_dquot(inode); + for (cnt = 0; cnt < MAXQUOTAS; cnt++) { + if (dquots[cnt]) + break; + } + + if (cnt < MAXQUOTAS) + __dquot_drop(inode); +} +EXPORT_SYMBOL(dquot_drop); + +/* + * inode_reserved_space is managed internally by quota, and protected by + * i_lock similar to i_blocks+i_bytes. + */ +static qsize_t *inode_reserved_space(struct inode * inode) +{ + /* Filesystem must explicitly define it's own method in order to use + * quota reservation interface */ + BUG_ON(!inode->i_sb->dq_op->get_reserved_space); + return inode->i_sb->dq_op->get_reserved_space(inode); +} + +static qsize_t __inode_get_rsv_space(struct inode *inode) +{ + if (!inode->i_sb->dq_op->get_reserved_space) + return 0; + return *inode_reserved_space(inode); +} + +static qsize_t inode_get_rsv_space(struct inode *inode) +{ + qsize_t ret; + + if (!inode->i_sb->dq_op->get_reserved_space) + return 0; + spin_lock(&inode->i_lock); + ret = __inode_get_rsv_space(inode); + spin_unlock(&inode->i_lock); + return ret; +} + +/* + * This functions updates i_blocks+i_bytes fields and quota information + * (together with appropriate checks). + * + * NOTE: We absolutely rely on the fact that caller dirties the inode + * (usually helpers in quotaops.h care about this) and holds a handle for + * the current transaction so that dquot write and inode write go into the + * same transaction. + */ + +/* + * This operation can block, but only after everything is updated + */ +int __dquot_alloc_space(struct inode *inode, qsize_t number, int flags) +{ + int cnt, ret = 0, index; + struct dquot_warn warn[MAXQUOTAS]; + int reserve = flags & DQUOT_SPACE_RESERVE; + struct dquot __rcu **dquots; + struct dquot *dquot; + + if (!inode_quota_active(inode)) { + if (reserve) { + spin_lock(&inode->i_lock); + *inode_reserved_space(inode) += number; + spin_unlock(&inode->i_lock); + } else { + inode_add_bytes(inode, number); + } + goto out; + } + + for (cnt = 0; cnt < MAXQUOTAS; cnt++) + warn[cnt].w_type = QUOTA_NL_NOWARN; + + dquots = i_dquot(inode); + index = srcu_read_lock(&dquot_srcu); + spin_lock(&inode->i_lock); + for (cnt = 0; cnt < MAXQUOTAS; cnt++) { + dquot = srcu_dereference(dquots[cnt], &dquot_srcu); + if (!dquot) + continue; + if (reserve) { + ret = dquot_add_space(dquot, 0, number, flags, &warn[cnt]); + } else { + ret = dquot_add_space(dquot, number, 0, flags, &warn[cnt]); + } + if (ret) { + /* Back out changes we already did */ + for (cnt--; cnt >= 0; cnt--) { + dquot = srcu_dereference(dquots[cnt], &dquot_srcu); + if (!dquot) + continue; + spin_lock(&dquot->dq_dqb_lock); + if (reserve) + dquot_free_reserved_space(dquot, number); + else + dquot_decr_space(dquot, number); + spin_unlock(&dquot->dq_dqb_lock); + } + spin_unlock(&inode->i_lock); + goto out_flush_warn; + } + } + if (reserve) + *inode_reserved_space(inode) += number; + else + __inode_add_bytes(inode, number); + spin_unlock(&inode->i_lock); + + if (reserve) + goto out_flush_warn; + ret = mark_all_dquot_dirty(dquots); +out_flush_warn: + srcu_read_unlock(&dquot_srcu, index); + flush_warnings(warn); +out: + return ret; +} +EXPORT_SYMBOL(__dquot_alloc_space); + +/* + * This operation can block, but only after everything is updated + */ +int dquot_alloc_inode(struct inode *inode) +{ + int cnt, ret = 0, index; + struct dquot_warn warn[MAXQUOTAS]; + struct dquot __rcu * const *dquots; + struct dquot *dquot; + + if (!inode_quota_active(inode)) + return 0; + for (cnt = 0; cnt < MAXQUOTAS; cnt++) + warn[cnt].w_type = QUOTA_NL_NOWARN; + + dquots = i_dquot(inode); + index = srcu_read_lock(&dquot_srcu); + spin_lock(&inode->i_lock); + for (cnt = 0; cnt < MAXQUOTAS; cnt++) { + dquot = srcu_dereference(dquots[cnt], &dquot_srcu); + if (!dquot) + continue; + ret = dquot_add_inodes(dquot, 1, &warn[cnt]); + if (ret) { + for (cnt--; cnt >= 0; cnt--) { + dquot = srcu_dereference(dquots[cnt], &dquot_srcu); + if (!dquot) + continue; + /* Back out changes we already did */ + spin_lock(&dquot->dq_dqb_lock); + dquot_decr_inodes(dquot, 1); + spin_unlock(&dquot->dq_dqb_lock); + } + goto warn_put_all; + } + } + +warn_put_all: + spin_unlock(&inode->i_lock); + if (ret == 0) + ret = mark_all_dquot_dirty(dquots); + srcu_read_unlock(&dquot_srcu, index); + flush_warnings(warn); + return ret; +} +EXPORT_SYMBOL(dquot_alloc_inode); + +/* + * Convert in-memory reserved quotas to real consumed quotas + */ +void dquot_claim_space_nodirty(struct inode *inode, qsize_t number) +{ + struct dquot __rcu **dquots; + struct dquot *dquot; + int cnt, index; + + if (!inode_quota_active(inode)) { + spin_lock(&inode->i_lock); + *inode_reserved_space(inode) -= number; + __inode_add_bytes(inode, number); + spin_unlock(&inode->i_lock); + return; + } + + dquots = i_dquot(inode); + index = srcu_read_lock(&dquot_srcu); + spin_lock(&inode->i_lock); + /* Claim reserved quotas to allocated quotas */ + for (cnt = 0; cnt < MAXQUOTAS; cnt++) { + dquot = srcu_dereference(dquots[cnt], &dquot_srcu); + if (dquot) { + spin_lock(&dquot->dq_dqb_lock); + if (WARN_ON_ONCE(dquot->dq_dqb.dqb_rsvspace < number)) + number = dquot->dq_dqb.dqb_rsvspace; + dquot->dq_dqb.dqb_curspace += number; + dquot->dq_dqb.dqb_rsvspace -= number; + spin_unlock(&dquot->dq_dqb_lock); + } + } + /* Update inode bytes */ + *inode_reserved_space(inode) -= number; + __inode_add_bytes(inode, number); + spin_unlock(&inode->i_lock); + mark_all_dquot_dirty(dquots); + srcu_read_unlock(&dquot_srcu, index); +} +EXPORT_SYMBOL(dquot_claim_space_nodirty); + +/* + * Convert allocated space back to in-memory reserved quotas + */ +void dquot_reclaim_space_nodirty(struct inode *inode, qsize_t number) +{ + struct dquot __rcu **dquots; + struct dquot *dquot; + int cnt, index; + + if (!inode_quota_active(inode)) { + spin_lock(&inode->i_lock); + *inode_reserved_space(inode) += number; + __inode_sub_bytes(inode, number); + spin_unlock(&inode->i_lock); + return; + } + + dquots = i_dquot(inode); + index = srcu_read_lock(&dquot_srcu); + spin_lock(&inode->i_lock); + /* Claim reserved quotas to allocated quotas */ + for (cnt = 0; cnt < MAXQUOTAS; cnt++) { + dquot = srcu_dereference(dquots[cnt], &dquot_srcu); + if (dquot) { + spin_lock(&dquot->dq_dqb_lock); + if (WARN_ON_ONCE(dquot->dq_dqb.dqb_curspace < number)) + number = dquot->dq_dqb.dqb_curspace; + dquot->dq_dqb.dqb_rsvspace += number; + dquot->dq_dqb.dqb_curspace -= number; + spin_unlock(&dquot->dq_dqb_lock); + } + } + /* Update inode bytes */ + *inode_reserved_space(inode) += number; + __inode_sub_bytes(inode, number); + spin_unlock(&inode->i_lock); + mark_all_dquot_dirty(dquots); + srcu_read_unlock(&dquot_srcu, index); +} +EXPORT_SYMBOL(dquot_reclaim_space_nodirty); + +/* + * This operation can block, but only after everything is updated + */ +void __dquot_free_space(struct inode *inode, qsize_t number, int flags) +{ + unsigned int cnt; + struct dquot_warn warn[MAXQUOTAS]; + struct dquot __rcu **dquots; + struct dquot *dquot; + int reserve = flags & DQUOT_SPACE_RESERVE, index; + + if (!inode_quota_active(inode)) { + if (reserve) { + spin_lock(&inode->i_lock); + *inode_reserved_space(inode) -= number; + spin_unlock(&inode->i_lock); + } else { + inode_sub_bytes(inode, number); + } + return; + } + + dquots = i_dquot(inode); + index = srcu_read_lock(&dquot_srcu); + spin_lock(&inode->i_lock); + for (cnt = 0; cnt < MAXQUOTAS; cnt++) { + int wtype; + + warn[cnt].w_type = QUOTA_NL_NOWARN; + dquot = srcu_dereference(dquots[cnt], &dquot_srcu); + if (!dquot) + continue; + spin_lock(&dquot->dq_dqb_lock); + wtype = info_bdq_free(dquot, number); + if (wtype != QUOTA_NL_NOWARN) + prepare_warning(&warn[cnt], dquot, wtype); + if (reserve) + dquot_free_reserved_space(dquot, number); + else + dquot_decr_space(dquot, number); + spin_unlock(&dquot->dq_dqb_lock); + } + if (reserve) + *inode_reserved_space(inode) -= number; + else + __inode_sub_bytes(inode, number); + spin_unlock(&inode->i_lock); + + if (reserve) + goto out_unlock; + mark_all_dquot_dirty(dquots); +out_unlock: + srcu_read_unlock(&dquot_srcu, index); + flush_warnings(warn); +} +EXPORT_SYMBOL(__dquot_free_space); + +/* + * This operation can block, but only after everything is updated + */ +void dquot_free_inode(struct inode *inode) +{ + unsigned int cnt; + struct dquot_warn warn[MAXQUOTAS]; + struct dquot __rcu * const *dquots; + struct dquot *dquot; + int index; + + if (!inode_quota_active(inode)) + return; + + dquots = i_dquot(inode); + index = srcu_read_lock(&dquot_srcu); + spin_lock(&inode->i_lock); + for (cnt = 0; cnt < MAXQUOTAS; cnt++) { + int wtype; + warn[cnt].w_type = QUOTA_NL_NOWARN; + dquot = srcu_dereference(dquots[cnt], &dquot_srcu); + if (!dquot) + continue; + spin_lock(&dquot->dq_dqb_lock); + wtype = info_idq_free(dquot, 1); + if (wtype != QUOTA_NL_NOWARN) + prepare_warning(&warn[cnt], dquot, wtype); + dquot_decr_inodes(dquot, 1); + spin_unlock(&dquot->dq_dqb_lock); + } + spin_unlock(&inode->i_lock); + mark_all_dquot_dirty(dquots); + srcu_read_unlock(&dquot_srcu, index); + flush_warnings(warn); +} +EXPORT_SYMBOL(dquot_free_inode); + +/* + * Transfer the number of inode and blocks from one diskquota to an other. + * On success, dquot references in transfer_to are consumed and references + * to original dquots that need to be released are placed there. On failure, + * references are kept untouched. + * + * This operation can block, but only after everything is updated + * A transaction must be started when entering this function. + * + * We are holding reference on transfer_from & transfer_to, no need to + * protect them by srcu_read_lock(). + */ +int __dquot_transfer(struct inode *inode, struct dquot **transfer_to) +{ + qsize_t cur_space; + qsize_t rsv_space = 0; + qsize_t inode_usage = 1; + struct dquot __rcu **dquots; + struct dquot *transfer_from[MAXQUOTAS] = {}; + int cnt, index, ret = 0, err; + char is_valid[MAXQUOTAS] = {}; + struct dquot_warn warn_to[MAXQUOTAS]; + struct dquot_warn warn_from_inodes[MAXQUOTAS]; + struct dquot_warn warn_from_space[MAXQUOTAS]; + + if (IS_NOQUOTA(inode)) + return 0; + + if (inode->i_sb->dq_op->get_inode_usage) { + ret = inode->i_sb->dq_op->get_inode_usage(inode, &inode_usage); + if (ret) + return ret; + } + + /* Initialize the arrays */ + for (cnt = 0; cnt < MAXQUOTAS; cnt++) { + warn_to[cnt].w_type = QUOTA_NL_NOWARN; + warn_from_inodes[cnt].w_type = QUOTA_NL_NOWARN; + warn_from_space[cnt].w_type = QUOTA_NL_NOWARN; + } + + spin_lock(&dq_data_lock); + spin_lock(&inode->i_lock); + if (IS_NOQUOTA(inode)) { /* File without quota accounting? */ + spin_unlock(&inode->i_lock); + spin_unlock(&dq_data_lock); + return 0; + } + cur_space = __inode_get_bytes(inode); + rsv_space = __inode_get_rsv_space(inode); + dquots = i_dquot(inode); + /* + * Build the transfer_from list, check limits, and update usage in + * the target structures. + */ + for (cnt = 0; cnt < MAXQUOTAS; cnt++) { + /* + * Skip changes for same uid or gid or for turned off quota-type. + */ + if (!transfer_to[cnt]) + continue; + /* Avoid races with quotaoff() */ + if (!sb_has_quota_active(inode->i_sb, cnt)) + continue; + is_valid[cnt] = 1; + transfer_from[cnt] = srcu_dereference_check(dquots[cnt], + &dquot_srcu, lockdep_is_held(&dq_data_lock)); + ret = dquot_add_inodes(transfer_to[cnt], inode_usage, + &warn_to[cnt]); + if (ret) + goto over_quota; + ret = dquot_add_space(transfer_to[cnt], cur_space, rsv_space, + DQUOT_SPACE_WARN, &warn_to[cnt]); + if (ret) { + spin_lock(&transfer_to[cnt]->dq_dqb_lock); + dquot_decr_inodes(transfer_to[cnt], inode_usage); + spin_unlock(&transfer_to[cnt]->dq_dqb_lock); + goto over_quota; + } + } + + /* Decrease usage for source structures and update quota pointers */ + for (cnt = 0; cnt < MAXQUOTAS; cnt++) { + if (!is_valid[cnt]) + continue; + /* Due to IO error we might not have transfer_from[] structure */ + if (transfer_from[cnt]) { + int wtype; + + spin_lock(&transfer_from[cnt]->dq_dqb_lock); + wtype = info_idq_free(transfer_from[cnt], inode_usage); + if (wtype != QUOTA_NL_NOWARN) + prepare_warning(&warn_from_inodes[cnt], + transfer_from[cnt], wtype); + wtype = info_bdq_free(transfer_from[cnt], + cur_space + rsv_space); + if (wtype != QUOTA_NL_NOWARN) + prepare_warning(&warn_from_space[cnt], + transfer_from[cnt], wtype); + dquot_decr_inodes(transfer_from[cnt], inode_usage); + dquot_decr_space(transfer_from[cnt], cur_space); + dquot_free_reserved_space(transfer_from[cnt], + rsv_space); + spin_unlock(&transfer_from[cnt]->dq_dqb_lock); + } + rcu_assign_pointer(dquots[cnt], transfer_to[cnt]); + } + spin_unlock(&inode->i_lock); + spin_unlock(&dq_data_lock); + + /* + * These arrays are local and we hold dquot references so we don't need + * the srcu protection but still take dquot_srcu to avoid warning in + * mark_all_dquot_dirty(). + */ + index = srcu_read_lock(&dquot_srcu); + err = mark_all_dquot_dirty((struct dquot __rcu **)transfer_from); + if (err < 0) + ret = err; + err = mark_all_dquot_dirty((struct dquot __rcu **)transfer_to); + if (err < 0) + ret = err; + srcu_read_unlock(&dquot_srcu, index); + + flush_warnings(warn_to); + flush_warnings(warn_from_inodes); + flush_warnings(warn_from_space); + /* Pass back references to put */ + for (cnt = 0; cnt < MAXQUOTAS; cnt++) + if (is_valid[cnt]) + transfer_to[cnt] = transfer_from[cnt]; + return ret; +over_quota: + /* Back out changes we already did */ + for (cnt--; cnt >= 0; cnt--) { + if (!is_valid[cnt]) + continue; + spin_lock(&transfer_to[cnt]->dq_dqb_lock); + dquot_decr_inodes(transfer_to[cnt], inode_usage); + dquot_decr_space(transfer_to[cnt], cur_space); + dquot_free_reserved_space(transfer_to[cnt], rsv_space); + spin_unlock(&transfer_to[cnt]->dq_dqb_lock); + } + spin_unlock(&inode->i_lock); + spin_unlock(&dq_data_lock); + flush_warnings(warn_to); + return ret; +} +EXPORT_SYMBOL(__dquot_transfer); + +/* Wrapper for transferring ownership of an inode for uid/gid only + * Called from FSXXX_setattr() + */ +int dquot_transfer(struct mnt_idmap *idmap, struct inode *inode, + struct iattr *iattr) +{ + struct dquot *transfer_to[MAXQUOTAS] = {}; + struct dquot *dquot; + struct super_block *sb = inode->i_sb; + int ret; + + if (!inode_quota_active(inode)) + return 0; + + if (i_uid_needs_update(idmap, iattr, inode)) { + kuid_t kuid = from_vfsuid(idmap, i_user_ns(inode), + iattr->ia_vfsuid); + + dquot = dqget(sb, make_kqid_uid(kuid)); + if (IS_ERR(dquot)) { + if (PTR_ERR(dquot) != -ESRCH) { + ret = PTR_ERR(dquot); + goto out_put; + } + dquot = NULL; + } + transfer_to[USRQUOTA] = dquot; + } + if (i_gid_needs_update(idmap, iattr, inode)) { + kgid_t kgid = from_vfsgid(idmap, i_user_ns(inode), + iattr->ia_vfsgid); + + dquot = dqget(sb, make_kqid_gid(kgid)); + if (IS_ERR(dquot)) { + if (PTR_ERR(dquot) != -ESRCH) { + ret = PTR_ERR(dquot); + goto out_put; + } + dquot = NULL; + } + transfer_to[GRPQUOTA] = dquot; + } + ret = __dquot_transfer(inode, transfer_to); +out_put: + dqput_all(transfer_to); + return ret; +} +EXPORT_SYMBOL(dquot_transfer); + +/* + * Write info of quota file to disk + */ +int dquot_commit_info(struct super_block *sb, int type) +{ + struct quota_info *dqopt = sb_dqopt(sb); + + return dqopt->ops[type]->write_file_info(sb, type); +} +EXPORT_SYMBOL(dquot_commit_info); + +int dquot_get_next_id(struct super_block *sb, struct kqid *qid) +{ + struct quota_info *dqopt = sb_dqopt(sb); + + if (!sb_has_quota_active(sb, qid->type)) + return -ESRCH; + if (!dqopt->ops[qid->type]->get_next_id) + return -ENOSYS; + return dqopt->ops[qid->type]->get_next_id(sb, qid); +} +EXPORT_SYMBOL(dquot_get_next_id); + +/* + * Definitions of diskquota operations. + */ +const struct dquot_operations dquot_operations = { + .write_dquot = dquot_commit, + .acquire_dquot = dquot_acquire, + .release_dquot = dquot_release, + .mark_dirty = dquot_mark_dquot_dirty, + .write_info = dquot_commit_info, + .alloc_dquot = dquot_alloc, + .destroy_dquot = dquot_destroy, + .get_next_id = dquot_get_next_id, +}; +EXPORT_SYMBOL(dquot_operations); + +/* + * Generic helper for ->open on filesystems supporting disk quotas. + */ +int dquot_file_open(struct inode *inode, struct file *file) +{ + int error; + + error = generic_file_open(inode, file); + if (!error && (file->f_mode & FMODE_WRITE)) + error = dquot_initialize(inode); + return error; +} +EXPORT_SYMBOL(dquot_file_open); + +static void vfs_cleanup_quota_inode(struct super_block *sb, int type) +{ + struct quota_info *dqopt = sb_dqopt(sb); + struct inode *inode = dqopt->files[type]; + + if (!inode) + return; + if (!(dqopt->flags & DQUOT_QUOTA_SYS_FILE)) { + inode_lock(inode); + inode->i_flags &= ~S_NOQUOTA; + inode_unlock(inode); + } + dqopt->files[type] = NULL; + iput(inode); +} + +/* + * Turn quota off on a device. type == -1 ==> quotaoff for all types (umount) + */ +int dquot_disable(struct super_block *sb, int type, unsigned int flags) +{ + int cnt; + struct quota_info *dqopt = sb_dqopt(sb); + + rwsem_assert_held_write(&sb->s_umount); + + /* Cannot turn off usage accounting without turning off limits, or + * suspend quotas and simultaneously turn quotas off. */ + if ((flags & DQUOT_USAGE_ENABLED && !(flags & DQUOT_LIMITS_ENABLED)) + || (flags & DQUOT_SUSPENDED && flags & (DQUOT_LIMITS_ENABLED | + DQUOT_USAGE_ENABLED))) + return -EINVAL; + + /* + * Skip everything if there's nothing to do. We have to do this because + * sometimes we are called when fill_super() failed and calling + * sync_fs() in such cases does no good. + */ + if (!sb_any_quota_loaded(sb)) + return 0; + + for (cnt = 0; cnt < MAXQUOTAS; cnt++) { + if (type != -1 && cnt != type) + continue; + if (!sb_has_quota_loaded(sb, cnt)) + continue; + + if (flags & DQUOT_SUSPENDED) { + spin_lock(&dq_state_lock); + dqopt->flags |= + dquot_state_flag(DQUOT_SUSPENDED, cnt); + spin_unlock(&dq_state_lock); + } else { + spin_lock(&dq_state_lock); + dqopt->flags &= ~dquot_state_flag(flags, cnt); + /* Turning off suspended quotas? */ + if (!sb_has_quota_loaded(sb, cnt) && + sb_has_quota_suspended(sb, cnt)) { + dqopt->flags &= ~dquot_state_flag( + DQUOT_SUSPENDED, cnt); + spin_unlock(&dq_state_lock); + vfs_cleanup_quota_inode(sb, cnt); + continue; + } + spin_unlock(&dq_state_lock); + } + + /* We still have to keep quota loaded? */ + if (sb_has_quota_loaded(sb, cnt) && !(flags & DQUOT_SUSPENDED)) + continue; + + /* Note: these are blocking operations */ + drop_dquot_ref(sb, cnt); + invalidate_dquots(sb, cnt); + /* + * Now all dquots should be invalidated, all writes done so we + * should be only users of the info. No locks needed. + */ + if (info_dirty(&dqopt->info[cnt])) + sb->dq_op->write_info(sb, cnt); + if (dqopt->ops[cnt]->free_file_info) + dqopt->ops[cnt]->free_file_info(sb, cnt); + put_quota_format(dqopt->info[cnt].dqi_format); + dqopt->info[cnt].dqi_flags = 0; + dqopt->info[cnt].dqi_igrace = 0; + dqopt->info[cnt].dqi_bgrace = 0; + dqopt->ops[cnt] = NULL; + } + + /* Skip syncing and setting flags if quota files are hidden */ + if (dqopt->flags & DQUOT_QUOTA_SYS_FILE) + goto put_inodes; + + /* Sync the superblock so that buffers with quota data are written to + * disk (and so userspace sees correct data afterwards). */ + if (sb->s_op->sync_fs) + sb->s_op->sync_fs(sb, 1); + sync_blockdev(sb->s_bdev); + /* Now the quota files are just ordinary files and we can set the + * inode flags back. Moreover we discard the pagecache so that + * userspace sees the writes we did bypassing the pagecache. We + * must also discard the blockdev buffers so that we see the + * changes done by userspace on the next quotaon() */ + for (cnt = 0; cnt < MAXQUOTAS; cnt++) + if (!sb_has_quota_loaded(sb, cnt) && dqopt->files[cnt]) { + inode_lock(dqopt->files[cnt]); + truncate_inode_pages(&dqopt->files[cnt]->i_data, 0); + inode_unlock(dqopt->files[cnt]); + } + if (sb->s_bdev) + invalidate_bdev(sb->s_bdev); +put_inodes: + /* We are done when suspending quotas */ + if (flags & DQUOT_SUSPENDED) + return 0; + + for (cnt = 0; cnt < MAXQUOTAS; cnt++) + if (!sb_has_quota_loaded(sb, cnt)) + vfs_cleanup_quota_inode(sb, cnt); + return 0; +} +EXPORT_SYMBOL(dquot_disable); + +int dquot_quota_off(struct super_block *sb, int type) +{ + return dquot_disable(sb, type, + DQUOT_USAGE_ENABLED | DQUOT_LIMITS_ENABLED); +} +EXPORT_SYMBOL(dquot_quota_off); + +/* + * Turn quotas on on a device + */ + +static int vfs_setup_quota_inode(struct inode *inode, int type) +{ + struct super_block *sb = inode->i_sb; + struct quota_info *dqopt = sb_dqopt(sb); + + if (is_bad_inode(inode)) + return -EUCLEAN; + if (!S_ISREG(inode->i_mode)) + return -EACCES; + if (IS_RDONLY(inode)) + return -EROFS; + if (sb_has_quota_loaded(sb, type)) + return -EBUSY; + + /* + * Quota files should never be encrypted. They should be thought of as + * filesystem metadata, not user data. New-style internal quota files + * cannot be encrypted by users anyway, but old-style external quota + * files could potentially be incorrectly created in an encrypted + * directory, hence this explicit check. Some reasons why encrypted + * quota files don't work include: (1) some filesystems that support + * encryption don't handle it in their quota_read and quota_write, and + * (2) cleaning up encrypted quota files at unmount would need special + * consideration, as quota files are cleaned up later than user files. + */ + if (IS_ENCRYPTED(inode)) + return -EINVAL; + + dqopt->files[type] = igrab(inode); + if (!dqopt->files[type]) + return -EIO; + if (!(dqopt->flags & DQUOT_QUOTA_SYS_FILE)) { + /* We don't want quota and atime on quota files (deadlocks + * possible) Also nobody should write to the file - we use + * special IO operations which ignore the immutable bit. */ + inode_lock(inode); + inode->i_flags |= S_NOQUOTA; + inode_unlock(inode); + /* + * When S_NOQUOTA is set, remove dquot references as no more + * references can be added + */ + __dquot_drop(inode); + } + return 0; +} + +int dquot_load_quota_sb(struct super_block *sb, int type, int format_id, + unsigned int flags) +{ + struct quota_format_type *fmt; + struct quota_info *dqopt = sb_dqopt(sb); + int error; + + lockdep_assert_held_write(&sb->s_umount); + + /* Just unsuspend quotas? */ + if (WARN_ON_ONCE(flags & DQUOT_SUSPENDED)) + return -EINVAL; + + fmt = find_quota_format(format_id); + if (!fmt) + return -ESRCH; + if (!sb->dq_op || !sb->s_qcop || + (type == PRJQUOTA && sb->dq_op->get_projid == NULL)) { + error = -EINVAL; + goto out_fmt; + } + /* Filesystems outside of init_user_ns not yet supported */ + if (sb->s_user_ns != &init_user_ns) { + error = -EINVAL; + goto out_fmt; + } + /* Usage always has to be set... */ + if (!(flags & DQUOT_USAGE_ENABLED)) { + error = -EINVAL; + goto out_fmt; + } + if (sb_has_quota_loaded(sb, type)) { + error = -EBUSY; + goto out_fmt; + } + + if (!(dqopt->flags & DQUOT_QUOTA_SYS_FILE)) { + /* As we bypass the pagecache we must now flush all the + * dirty data and invalidate caches so that kernel sees + * changes from userspace. It is not enough to just flush + * the quota file since if blocksize < pagesize, invalidation + * of the cache could fail because of other unrelated dirty + * data */ + sync_filesystem(sb); + invalidate_bdev(sb->s_bdev); + } + + error = -EINVAL; + if (!fmt->qf_ops->check_quota_file(sb, type)) + goto out_fmt; + + dqopt->ops[type] = fmt->qf_ops; + dqopt->info[type].dqi_format = fmt; + dqopt->info[type].dqi_fmt_id = format_id; + INIT_LIST_HEAD(&dqopt->info[type].dqi_dirty_list); + error = dqopt->ops[type]->read_file_info(sb, type); + if (error < 0) + goto out_fmt; + if (dqopt->flags & DQUOT_QUOTA_SYS_FILE) { + spin_lock(&dq_data_lock); + dqopt->info[type].dqi_flags |= DQF_SYS_FILE; + spin_unlock(&dq_data_lock); + } + spin_lock(&dq_state_lock); + dqopt->flags |= dquot_state_flag(flags, type); + spin_unlock(&dq_state_lock); + + error = add_dquot_ref(sb, type); + if (error) + dquot_disable(sb, type, + DQUOT_USAGE_ENABLED | DQUOT_LIMITS_ENABLED); + + return error; +out_fmt: + put_quota_format(fmt); + + return error; +} +EXPORT_SYMBOL(dquot_load_quota_sb); + +/* + * More powerful function for turning on quotas on given quota inode allowing + * setting of individual quota flags + */ +int dquot_load_quota_inode(struct inode *inode, int type, int format_id, + unsigned int flags) +{ + int err; + + err = vfs_setup_quota_inode(inode, type); + if (err < 0) + return err; + err = dquot_load_quota_sb(inode->i_sb, type, format_id, flags); + if (err < 0) + vfs_cleanup_quota_inode(inode->i_sb, type); + return err; +} +EXPORT_SYMBOL(dquot_load_quota_inode); + +/* Reenable quotas on remount RW */ +int dquot_resume(struct super_block *sb, int type) +{ + struct quota_info *dqopt = sb_dqopt(sb); + int ret = 0, cnt; + unsigned int flags; + + rwsem_assert_held_write(&sb->s_umount); + + for (cnt = 0; cnt < MAXQUOTAS; cnt++) { + if (type != -1 && cnt != type) + continue; + if (!sb_has_quota_suspended(sb, cnt)) + continue; + + spin_lock(&dq_state_lock); + flags = dqopt->flags & dquot_state_flag(DQUOT_USAGE_ENABLED | + DQUOT_LIMITS_ENABLED, + cnt); + dqopt->flags &= ~dquot_state_flag(DQUOT_STATE_FLAGS, cnt); + spin_unlock(&dq_state_lock); + + flags = dquot_generic_flag(flags, cnt); + ret = dquot_load_quota_sb(sb, cnt, dqopt->info[cnt].dqi_fmt_id, + flags); + if (ret < 0) + vfs_cleanup_quota_inode(sb, cnt); + } + + return ret; +} +EXPORT_SYMBOL(dquot_resume); + +int dquot_quota_on(struct super_block *sb, int type, int format_id, + const struct path *path) +{ + int error = security_quota_on(path->dentry); + if (error) + return error; + /* Quota file not on the same filesystem? */ + if (path->dentry->d_sb != sb) + error = -EXDEV; + else + error = dquot_load_quota_inode(d_inode(path->dentry), type, + format_id, DQUOT_USAGE_ENABLED | + DQUOT_LIMITS_ENABLED); + return error; +} +EXPORT_SYMBOL(dquot_quota_on); + +/* + * This function is used when filesystem needs to initialize quotas + * during mount time. + */ +int dquot_quota_on_mount(struct super_block *sb, char *qf_name, + int format_id, int type) +{ + struct dentry *dentry; + int error; + + dentry = lookup_noperm_positive_unlocked(&QSTR(qf_name), sb->s_root); + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + + error = security_quota_on(dentry); + if (!error) + error = dquot_load_quota_inode(d_inode(dentry), type, format_id, + DQUOT_USAGE_ENABLED | DQUOT_LIMITS_ENABLED); + + dput(dentry); + return error; +} +EXPORT_SYMBOL(dquot_quota_on_mount); + +static int dquot_quota_enable(struct super_block *sb, unsigned int flags) +{ + int ret; + int type; + struct quota_info *dqopt = sb_dqopt(sb); + + if (!(dqopt->flags & DQUOT_QUOTA_SYS_FILE)) + return -ENOSYS; + /* Accounting cannot be turned on while fs is mounted */ + flags &= ~(FS_QUOTA_UDQ_ACCT | FS_QUOTA_GDQ_ACCT | FS_QUOTA_PDQ_ACCT); + if (!flags) + return -EINVAL; + for (type = 0; type < MAXQUOTAS; type++) { + if (!(flags & qtype_enforce_flag(type))) + continue; + /* Can't enforce without accounting */ + if (!sb_has_quota_usage_enabled(sb, type)) { + ret = -EINVAL; + goto out_err; + } + if (sb_has_quota_limits_enabled(sb, type)) { + /* compatible with XFS */ + ret = -EEXIST; + goto out_err; + } + spin_lock(&dq_state_lock); + dqopt->flags |= dquot_state_flag(DQUOT_LIMITS_ENABLED, type); + spin_unlock(&dq_state_lock); + } + return 0; +out_err: + /* Backout enforcement enablement we already did */ + for (type--; type >= 0; type--) { + if (flags & qtype_enforce_flag(type)) + dquot_disable(sb, type, DQUOT_LIMITS_ENABLED); + } + return ret; +} + +static int dquot_quota_disable(struct super_block *sb, unsigned int flags) +{ + int ret; + int type; + struct quota_info *dqopt = sb_dqopt(sb); + + if (!(dqopt->flags & DQUOT_QUOTA_SYS_FILE)) + return -ENOSYS; + /* + * We don't support turning off accounting via quotactl. In principle + * quota infrastructure can do this but filesystems don't expect + * userspace to be able to do it. + */ + if (flags & + (FS_QUOTA_UDQ_ACCT | FS_QUOTA_GDQ_ACCT | FS_QUOTA_PDQ_ACCT)) + return -EOPNOTSUPP; + + /* Filter out limits not enabled */ + for (type = 0; type < MAXQUOTAS; type++) + if (!sb_has_quota_limits_enabled(sb, type)) + flags &= ~qtype_enforce_flag(type); + /* Nothing left? */ + if (!flags) + return -EEXIST; + for (type = 0; type < MAXQUOTAS; type++) { + if (flags & qtype_enforce_flag(type)) { + ret = dquot_disable(sb, type, DQUOT_LIMITS_ENABLED); + if (ret < 0) + goto out_err; + } + } + return 0; +out_err: + /* Backout enforcement disabling we already did */ + for (type--; type >= 0; type--) { + if (flags & qtype_enforce_flag(type)) { + spin_lock(&dq_state_lock); + dqopt->flags |= + dquot_state_flag(DQUOT_LIMITS_ENABLED, type); + spin_unlock(&dq_state_lock); + } + } + return ret; +} + +/* Generic routine for getting common part of quota structure */ +static void do_get_dqblk(struct dquot *dquot, struct qc_dqblk *di) +{ + struct mem_dqblk *dm = &dquot->dq_dqb; + + memset(di, 0, sizeof(*di)); + spin_lock(&dquot->dq_dqb_lock); + di->d_spc_hardlimit = dm->dqb_bhardlimit; + di->d_spc_softlimit = dm->dqb_bsoftlimit; + di->d_ino_hardlimit = dm->dqb_ihardlimit; + di->d_ino_softlimit = dm->dqb_isoftlimit; + di->d_space = dm->dqb_curspace + dm->dqb_rsvspace; + di->d_ino_count = dm->dqb_curinodes; + di->d_spc_timer = dm->dqb_btime; + di->d_ino_timer = dm->dqb_itime; + spin_unlock(&dquot->dq_dqb_lock); +} + +int dquot_get_dqblk(struct super_block *sb, struct kqid qid, + struct qc_dqblk *di) +{ + struct dquot *dquot; + + dquot = dqget(sb, qid); + if (IS_ERR(dquot)) + return PTR_ERR(dquot); + do_get_dqblk(dquot, di); + dqput(dquot); + + return 0; +} +EXPORT_SYMBOL(dquot_get_dqblk); + +int dquot_get_next_dqblk(struct super_block *sb, struct kqid *qid, + struct qc_dqblk *di) +{ + struct dquot *dquot; + int err; + + if (!sb->dq_op->get_next_id) + return -ENOSYS; + err = sb->dq_op->get_next_id(sb, qid); + if (err < 0) + return err; + dquot = dqget(sb, *qid); + if (IS_ERR(dquot)) + return PTR_ERR(dquot); + do_get_dqblk(dquot, di); + dqput(dquot); + + return 0; +} +EXPORT_SYMBOL(dquot_get_next_dqblk); + +#define VFS_QC_MASK \ + (QC_SPACE | QC_SPC_SOFT | QC_SPC_HARD | \ + QC_INO_COUNT | QC_INO_SOFT | QC_INO_HARD | \ + QC_SPC_TIMER | QC_INO_TIMER) + +/* Generic routine for setting common part of quota structure */ +static int do_set_dqblk(struct dquot *dquot, struct qc_dqblk *di) +{ + struct mem_dqblk *dm = &dquot->dq_dqb; + int check_blim = 0, check_ilim = 0; + struct mem_dqinfo *dqi = &sb_dqopt(dquot->dq_sb)->info[dquot->dq_id.type]; + int ret; + + if (di->d_fieldmask & ~VFS_QC_MASK) + return -EINVAL; + + if (((di->d_fieldmask & QC_SPC_SOFT) && + di->d_spc_softlimit > dqi->dqi_max_spc_limit) || + ((di->d_fieldmask & QC_SPC_HARD) && + di->d_spc_hardlimit > dqi->dqi_max_spc_limit) || + ((di->d_fieldmask & QC_INO_SOFT) && + (di->d_ino_softlimit > dqi->dqi_max_ino_limit)) || + ((di->d_fieldmask & QC_INO_HARD) && + (di->d_ino_hardlimit > dqi->dqi_max_ino_limit))) + return -ERANGE; + + spin_lock(&dquot->dq_dqb_lock); + if (di->d_fieldmask & QC_SPACE) { + dm->dqb_curspace = di->d_space - dm->dqb_rsvspace; + check_blim = 1; + set_bit(DQ_LASTSET_B + QIF_SPACE_B, &dquot->dq_flags); + } + + if (di->d_fieldmask & QC_SPC_SOFT) + dm->dqb_bsoftlimit = di->d_spc_softlimit; + if (di->d_fieldmask & QC_SPC_HARD) + dm->dqb_bhardlimit = di->d_spc_hardlimit; + if (di->d_fieldmask & (QC_SPC_SOFT | QC_SPC_HARD)) { + check_blim = 1; + set_bit(DQ_LASTSET_B + QIF_BLIMITS_B, &dquot->dq_flags); + } + + if (di->d_fieldmask & QC_INO_COUNT) { + dm->dqb_curinodes = di->d_ino_count; + check_ilim = 1; + set_bit(DQ_LASTSET_B + QIF_INODES_B, &dquot->dq_flags); + } + + if (di->d_fieldmask & QC_INO_SOFT) + dm->dqb_isoftlimit = di->d_ino_softlimit; + if (di->d_fieldmask & QC_INO_HARD) + dm->dqb_ihardlimit = di->d_ino_hardlimit; + if (di->d_fieldmask & (QC_INO_SOFT | QC_INO_HARD)) { + check_ilim = 1; + set_bit(DQ_LASTSET_B + QIF_ILIMITS_B, &dquot->dq_flags); + } + + if (di->d_fieldmask & QC_SPC_TIMER) { + dm->dqb_btime = di->d_spc_timer; + check_blim = 1; + set_bit(DQ_LASTSET_B + QIF_BTIME_B, &dquot->dq_flags); + } + + if (di->d_fieldmask & QC_INO_TIMER) { + dm->dqb_itime = di->d_ino_timer; + check_ilim = 1; + set_bit(DQ_LASTSET_B + QIF_ITIME_B, &dquot->dq_flags); + } + + if (check_blim) { + if (!dm->dqb_bsoftlimit || + dm->dqb_curspace + dm->dqb_rsvspace <= dm->dqb_bsoftlimit) { + dm->dqb_btime = 0; + clear_bit(DQ_BLKS_B, &dquot->dq_flags); + } else if (!(di->d_fieldmask & QC_SPC_TIMER)) + /* Set grace only if user hasn't provided his own... */ + dm->dqb_btime = ktime_get_real_seconds() + dqi->dqi_bgrace; + } + if (check_ilim) { + if (!dm->dqb_isoftlimit || + dm->dqb_curinodes <= dm->dqb_isoftlimit) { + dm->dqb_itime = 0; + clear_bit(DQ_INODES_B, &dquot->dq_flags); + } else if (!(di->d_fieldmask & QC_INO_TIMER)) + /* Set grace only if user hasn't provided his own... */ + dm->dqb_itime = ktime_get_real_seconds() + dqi->dqi_igrace; + } + if (dm->dqb_bhardlimit || dm->dqb_bsoftlimit || dm->dqb_ihardlimit || + dm->dqb_isoftlimit) + clear_bit(DQ_FAKE_B, &dquot->dq_flags); + else + set_bit(DQ_FAKE_B, &dquot->dq_flags); + spin_unlock(&dquot->dq_dqb_lock); + ret = mark_dquot_dirty(dquot); + if (ret < 0) + return ret; + return 0; +} + +int dquot_set_dqblk(struct super_block *sb, struct kqid qid, + struct qc_dqblk *di) +{ + struct dquot *dquot; + int rc; + + dquot = dqget(sb, qid); + if (IS_ERR(dquot)) { + rc = PTR_ERR(dquot); + goto out; + } + rc = do_set_dqblk(dquot, di); + dqput(dquot); +out: + return rc; +} +EXPORT_SYMBOL(dquot_set_dqblk); + +/* Generic routine for getting common part of quota file information */ +int dquot_get_state(struct super_block *sb, struct qc_state *state) +{ + struct mem_dqinfo *mi; + struct qc_type_state *tstate; + struct quota_info *dqopt = sb_dqopt(sb); + int type; + + memset(state, 0, sizeof(*state)); + for (type = 0; type < MAXQUOTAS; type++) { + if (!sb_has_quota_active(sb, type)) + continue; + tstate = state->s_state + type; + mi = sb_dqopt(sb)->info + type; + tstate->flags = QCI_ACCT_ENABLED; + spin_lock(&dq_data_lock); + if (mi->dqi_flags & DQF_SYS_FILE) + tstate->flags |= QCI_SYSFILE; + if (mi->dqi_flags & DQF_ROOT_SQUASH) + tstate->flags |= QCI_ROOT_SQUASH; + if (sb_has_quota_limits_enabled(sb, type)) + tstate->flags |= QCI_LIMITS_ENFORCED; + tstate->spc_timelimit = mi->dqi_bgrace; + tstate->ino_timelimit = mi->dqi_igrace; + if (dqopt->files[type]) { + tstate->ino = dqopt->files[type]->i_ino; + tstate->blocks = dqopt->files[type]->i_blocks; + } + tstate->nextents = 1; /* We don't know... */ + spin_unlock(&dq_data_lock); + } + return 0; +} +EXPORT_SYMBOL(dquot_get_state); + +/* Generic routine for setting common part of quota file information */ +int dquot_set_dqinfo(struct super_block *sb, int type, struct qc_info *ii) +{ + struct mem_dqinfo *mi; + + if ((ii->i_fieldmask & QC_WARNS_MASK) || + (ii->i_fieldmask & QC_RT_SPC_TIMER)) + return -EINVAL; + if (!sb_has_quota_active(sb, type)) + return -ESRCH; + mi = sb_dqopt(sb)->info + type; + if (ii->i_fieldmask & QC_FLAGS) { + if ((ii->i_flags & QCI_ROOT_SQUASH && + mi->dqi_format->qf_fmt_id != QFMT_VFS_OLD)) + return -EINVAL; + } + spin_lock(&dq_data_lock); + if (ii->i_fieldmask & QC_SPC_TIMER) + mi->dqi_bgrace = ii->i_spc_timelimit; + if (ii->i_fieldmask & QC_INO_TIMER) + mi->dqi_igrace = ii->i_ino_timelimit; + if (ii->i_fieldmask & QC_FLAGS) { + if (ii->i_flags & QCI_ROOT_SQUASH) + mi->dqi_flags |= DQF_ROOT_SQUASH; + else + mi->dqi_flags &= ~DQF_ROOT_SQUASH; + } + spin_unlock(&dq_data_lock); + mark_info_dirty(sb, type); + /* Force write to disk */ + return sb->dq_op->write_info(sb, type); +} +EXPORT_SYMBOL(dquot_set_dqinfo); + +const struct quotactl_ops dquot_quotactl_sysfile_ops = { + .quota_enable = dquot_quota_enable, + .quota_disable = dquot_quota_disable, + .quota_sync = dquot_quota_sync, + .get_state = dquot_get_state, + .set_info = dquot_set_dqinfo, + .get_dqblk = dquot_get_dqblk, + .get_nextdqblk = dquot_get_next_dqblk, + .set_dqblk = dquot_set_dqblk +}; +EXPORT_SYMBOL(dquot_quotactl_sysfile_ops); + +static int do_proc_dqstats(const struct ctl_table *table, int write, + void *buffer, size_t *lenp, loff_t *ppos) +{ + unsigned int type = (unsigned long *)table->data - dqstats.stat; + s64 value = percpu_counter_sum(&dqstats.counter[type]); + + /* Filter negative values for non-monotonic counters */ + if (value < 0 && (type == DQST_ALLOC_DQUOTS || + type == DQST_FREE_DQUOTS)) + value = 0; + + /* Update global table */ + dqstats.stat[type] = value; + return proc_doulongvec_minmax(table, write, buffer, lenp, ppos); +} + +static const struct ctl_table fs_dqstats_table[] = { + { + .procname = "lookups", + .data = &dqstats.stat[DQST_LOOKUPS], + .maxlen = sizeof(unsigned long), + .mode = 0444, + .proc_handler = do_proc_dqstats, + }, + { + .procname = "drops", + .data = &dqstats.stat[DQST_DROPS], + .maxlen = sizeof(unsigned long), + .mode = 0444, + .proc_handler = do_proc_dqstats, + }, + { + .procname = "reads", + .data = &dqstats.stat[DQST_READS], + .maxlen = sizeof(unsigned long), + .mode = 0444, + .proc_handler = do_proc_dqstats, + }, + { + .procname = "writes", + .data = &dqstats.stat[DQST_WRITES], + .maxlen = sizeof(unsigned long), + .mode = 0444, + .proc_handler = do_proc_dqstats, + }, + { + .procname = "cache_hits", + .data = &dqstats.stat[DQST_CACHE_HITS], + .maxlen = sizeof(unsigned long), + .mode = 0444, + .proc_handler = do_proc_dqstats, + }, + { + .procname = "allocated_dquots", + .data = &dqstats.stat[DQST_ALLOC_DQUOTS], + .maxlen = sizeof(unsigned long), + .mode = 0444, + .proc_handler = do_proc_dqstats, + }, + { + .procname = "free_dquots", + .data = &dqstats.stat[DQST_FREE_DQUOTS], + .maxlen = sizeof(unsigned long), + .mode = 0444, + .proc_handler = do_proc_dqstats, + }, + { + .procname = "syncs", + .data = &dqstats.stat[DQST_SYNCS], + .maxlen = sizeof(unsigned long), + .mode = 0444, + .proc_handler = do_proc_dqstats, + }, +}; + +static int __init dquot_init(void) +{ + int i, ret; + unsigned long nr_hash; + struct shrinker *dqcache_shrinker; + + printk(KERN_NOTICE "VFS: Disk quotas %s\n", __DQUOT_VERSION__); + + register_sysctl_init("fs/quota", fs_dqstats_table); + + dquot_cachep = kmem_cache_create("dquot", + sizeof(struct dquot), sizeof(unsigned long) * 4, + (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| + SLAB_PANIC), + NULL); + + dquot_hash = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!dquot_hash) + panic("Cannot create dquot hash table"); + + ret = percpu_counter_init_many(dqstats.counter, 0, GFP_KERNEL, + _DQST_DQSTAT_LAST); + if (ret) + panic("Cannot create dquot stat counters"); + + /* Find power-of-two hlist_heads which can fit into allocation */ + nr_hash = PAGE_SIZE / sizeof(struct hlist_head); + dq_hash_bits = ilog2(nr_hash); + + nr_hash = 1UL << dq_hash_bits; + dq_hash_mask = nr_hash - 1; + for (i = 0; i < nr_hash; i++) + INIT_HLIST_HEAD(dquot_hash + i); + + pr_info("VFS: Dquot-cache hash table entries: %ld (%ld bytes)\n", + nr_hash, PAGE_SIZE); + + dqcache_shrinker = shrinker_alloc(0, "dquota-cache"); + if (!dqcache_shrinker) + panic("Cannot allocate dquot shrinker"); + + dqcache_shrinker->count_objects = dqcache_shrink_count; + dqcache_shrinker->scan_objects = dqcache_shrink_scan; + + shrinker_register(dqcache_shrinker); + + quota_unbound_wq = alloc_workqueue("quota_events_unbound", + WQ_UNBOUND | WQ_MEM_RECLAIM, WQ_MAX_ACTIVE); + if (!quota_unbound_wq) + panic("Cannot create quota_unbound_wq\n"); + + return 0; +} +fs_initcall(dquot_init); diff --git a/fs/quota/kqid.c b/fs/quota/kqid.c new file mode 100644 index 000000000..f814fa90a --- /dev/null +++ b/fs/quota/kqid.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include + +/** + * qid_eq - Test to see if to kquid values are the same + * @left: A qid value + * @right: Another quid value + * + * Return true if the two qid values are equal and false otherwise. + */ +bool qid_eq(struct kqid left, struct kqid right) +{ + if (left.type != right.type) + return false; + switch(left.type) { + case USRQUOTA: + return uid_eq(left.uid, right.uid); + case GRPQUOTA: + return gid_eq(left.gid, right.gid); + case PRJQUOTA: + return projid_eq(left.projid, right.projid); + default: + BUG(); + } +} +EXPORT_SYMBOL(qid_eq); + +/** + * qid_lt - Test to see if one qid value is less than another + * @left: The possibly lesser qid value + * @right: The possibly greater qid value + * + * Return true if left is less than right and false otherwise. + */ +bool qid_lt(struct kqid left, struct kqid right) +{ + if (left.type < right.type) + return true; + if (left.type > right.type) + return false; + switch (left.type) { + case USRQUOTA: + return uid_lt(left.uid, right.uid); + case GRPQUOTA: + return gid_lt(left.gid, right.gid); + case PRJQUOTA: + return projid_lt(left.projid, right.projid); + default: + BUG(); + } +} +EXPORT_SYMBOL(qid_lt); + +/** + * from_kqid - Create a qid from a kqid user-namespace pair. + * @targ: The user namespace we want a qid in. + * @kqid: The kernel internal quota identifier to start with. + * + * Map @kqid into the user-namespace specified by @targ and + * return the resulting qid. + * + * There is always a mapping into the initial user_namespace. + * + * If @kqid has no mapping in @targ (qid_t)-1 is returned. + */ +qid_t from_kqid(struct user_namespace *targ, struct kqid kqid) +{ + switch (kqid.type) { + case USRQUOTA: + return from_kuid(targ, kqid.uid); + case GRPQUOTA: + return from_kgid(targ, kqid.gid); + case PRJQUOTA: + return from_kprojid(targ, kqid.projid); + default: + BUG(); + } +} +EXPORT_SYMBOL(from_kqid); + +/** + * from_kqid_munged - Create a qid from a kqid user-namespace pair. + * @targ: The user namespace we want a qid in. + * @kqid: The kernel internal quota identifier to start with. + * + * Map @kqid into the user-namespace specified by @targ and + * return the resulting qid. + * + * There is always a mapping into the initial user_namespace. + * + * Unlike from_kqid from_kqid_munged never fails and always + * returns a valid projid. This makes from_kqid_munged + * appropriate for use in places where failing to provide + * a qid_t is not a good option. + * + * If @kqid has no mapping in @targ the kqid.type specific + * overflow identifier is returned. + */ +qid_t from_kqid_munged(struct user_namespace *targ, struct kqid kqid) +{ + switch (kqid.type) { + case USRQUOTA: + return from_kuid_munged(targ, kqid.uid); + case GRPQUOTA: + return from_kgid_munged(targ, kqid.gid); + case PRJQUOTA: + return from_kprojid_munged(targ, kqid.projid); + default: + BUG(); + } +} +EXPORT_SYMBOL(from_kqid_munged); + +/** + * qid_valid - Report if a valid value is stored in a kqid. + * @qid: The kernel internal quota identifier to test. + */ +bool qid_valid(struct kqid qid) +{ + switch (qid.type) { + case USRQUOTA: + return uid_valid(qid.uid); + case GRPQUOTA: + return gid_valid(qid.gid); + case PRJQUOTA: + return projid_valid(qid.projid); + default: + BUG(); + } +} +EXPORT_SYMBOL(qid_valid); diff --git a/fs/quota/netlink.c b/fs/quota/netlink.c new file mode 100644 index 000000000..95acdae39 --- /dev/null +++ b/fs/quota/netlink.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include + +static const struct genl_multicast_group quota_mcgrps[] = { + { .name = "events", }, +}; + +/* Netlink family structure for quota */ +static struct genl_family quota_genl_family __ro_after_init = { + .module = THIS_MODULE, + .hdrsize = 0, + .name = "VFS_DQUOT", + .version = 1, + .maxattr = QUOTA_NL_A_MAX, + .mcgrps = quota_mcgrps, + .n_mcgrps = ARRAY_SIZE(quota_mcgrps), +}; + +/** + * quota_send_warning - Send warning to userspace about exceeded quota + * @qid: The kernel internal quota identifier. + * @dev: The device on which the fs is mounted (sb->s_dev) + * @warntype: The type of the warning: QUOTA_NL_... + * + * This can be used by filesystems (including those which don't use + * dquot) to send a message to userspace relating to quota limits. + * + */ + +void quota_send_warning(struct kqid qid, dev_t dev, + const char warntype) +{ + static atomic_t seq; + struct sk_buff *skb; + void *msg_head; + int ret; + int msg_size = 4 * nla_total_size(sizeof(u32)) + + 2 * nla_total_size_64bit(sizeof(u64)); + + /* We have to allocate using GFP_NOFS as we are called from a + * filesystem performing write and thus further recursion into + * the fs to free some data could cause deadlocks. */ + skb = genlmsg_new(msg_size, GFP_NOFS); + if (!skb) { + printk(KERN_ERR + "VFS: Not enough memory to send quota warning.\n"); + return; + } + msg_head = genlmsg_put(skb, 0, atomic_add_return(1, &seq), + "a_genl_family, 0, QUOTA_NL_C_WARNING); + if (!msg_head) { + printk(KERN_ERR + "VFS: Cannot store netlink header in quota warning.\n"); + goto err_out; + } + ret = nla_put_u32(skb, QUOTA_NL_A_QTYPE, qid.type); + if (ret) + goto attr_err_out; + ret = nla_put_u64_64bit(skb, QUOTA_NL_A_EXCESS_ID, + from_kqid_munged(&init_user_ns, qid), + QUOTA_NL_A_PAD); + if (ret) + goto attr_err_out; + ret = nla_put_u32(skb, QUOTA_NL_A_WARNING, warntype); + if (ret) + goto attr_err_out; + ret = nla_put_u32(skb, QUOTA_NL_A_DEV_MAJOR, MAJOR(dev)); + if (ret) + goto attr_err_out; + ret = nla_put_u32(skb, QUOTA_NL_A_DEV_MINOR, MINOR(dev)); + if (ret) + goto attr_err_out; + ret = nla_put_u64_64bit(skb, QUOTA_NL_A_CAUSED_ID, + from_kuid_munged(&init_user_ns, current_uid()), + QUOTA_NL_A_PAD); + if (ret) + goto attr_err_out; + genlmsg_end(skb, msg_head); + + genlmsg_multicast("a_genl_family, skb, 0, 0, GFP_NOFS); + return; +attr_err_out: + printk(KERN_ERR "VFS: Not enough space to compose quota message!\n"); +err_out: + kfree_skb(skb); +} +EXPORT_SYMBOL(quota_send_warning); + +static int __init quota_init(void) +{ + if (genl_register_family("a_genl_family) != 0) + printk(KERN_ERR + "VFS: Failed to create quota netlink interface.\n"); + return 0; +}; +fs_initcall(quota_init); diff --git a/fs/quota/quota.c b/fs/quota/quota.c new file mode 100644 index 000000000..33bacd707 --- /dev/null +++ b/fs/quota/quota.c @@ -0,0 +1,1010 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Quota code necessary even when VFS quota support is not compiled + * into the kernel. The interesting stuff is over in dquot.c, here + * we have symbols for initial quotactl(2) handling, the sysctl(2) + * variables, etc - things needed even when quota support disabled. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "compat.h" +#include "../internal.h" + +static int check_quotactl_permission(struct super_block *sb, int type, int cmd, + qid_t id) +{ + switch (cmd) { + /* these commands do not require any special privilegues */ + case Q_GETFMT: + case Q_SYNC: + case Q_GETINFO: + case Q_XGETQSTAT: + case Q_XGETQSTATV: + case Q_XQUOTASYNC: + break; + /* allow to query information for dquots we "own" */ + case Q_GETQUOTA: + case Q_XGETQUOTA: + if ((type == USRQUOTA && uid_eq(current_euid(), make_kuid(current_user_ns(), id))) || + (type == GRPQUOTA && in_egroup_p(make_kgid(current_user_ns(), id)))) + break; + fallthrough; + default: + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + } + + return security_quotactl(cmd, type, id, sb); +} + +static void quota_sync_one(struct super_block *sb, void *arg) +{ + int type = *(int *)arg; + + if (sb->s_qcop && sb->s_qcop->quota_sync && + (sb->s_quota_types & (1 << type))) + sb->s_qcop->quota_sync(sb, type); +} + +static int quota_sync_all(int type) +{ + int ret; + + ret = security_quotactl(Q_SYNC, type, 0, NULL); + if (!ret) + iterate_supers(quota_sync_one, &type); + return ret; +} + +unsigned int qtype_enforce_flag(int type) +{ + switch (type) { + case USRQUOTA: + return FS_QUOTA_UDQ_ENFD; + case GRPQUOTA: + return FS_QUOTA_GDQ_ENFD; + case PRJQUOTA: + return FS_QUOTA_PDQ_ENFD; + } + return 0; +} + +static int quota_quotaon(struct super_block *sb, int type, qid_t id, + const struct path *path) +{ + if (!sb->s_qcop->quota_on && !sb->s_qcop->quota_enable) + return -ENOSYS; + if (sb->s_qcop->quota_enable) + return sb->s_qcop->quota_enable(sb, qtype_enforce_flag(type)); + if (IS_ERR(path)) + return PTR_ERR(path); + return sb->s_qcop->quota_on(sb, type, id, path); +} + +static int quota_quotaoff(struct super_block *sb, int type) +{ + if (!sb->s_qcop->quota_off && !sb->s_qcop->quota_disable) + return -ENOSYS; + if (sb->s_qcop->quota_disable) + return sb->s_qcop->quota_disable(sb, qtype_enforce_flag(type)); + return sb->s_qcop->quota_off(sb, type); +} + +static int quota_getfmt(struct super_block *sb, int type, void __user *addr) +{ + __u32 fmt; + + if (!sb_has_quota_active(sb, type)) + return -ESRCH; + fmt = sb_dqopt(sb)->info[type].dqi_format->qf_fmt_id; + if (copy_to_user(addr, &fmt, sizeof(fmt))) + return -EFAULT; + return 0; +} + +static int quota_getinfo(struct super_block *sb, int type, void __user *addr) +{ + struct qc_state state; + struct qc_type_state *tstate; + struct if_dqinfo uinfo; + int ret; + + if (!sb->s_qcop->get_state) + return -ENOSYS; + ret = sb->s_qcop->get_state(sb, &state); + if (ret) + return ret; + tstate = state.s_state + type; + if (!(tstate->flags & QCI_ACCT_ENABLED)) + return -ESRCH; + memset(&uinfo, 0, sizeof(uinfo)); + uinfo.dqi_bgrace = tstate->spc_timelimit; + uinfo.dqi_igrace = tstate->ino_timelimit; + if (tstate->flags & QCI_SYSFILE) + uinfo.dqi_flags |= DQF_SYS_FILE; + if (tstate->flags & QCI_ROOT_SQUASH) + uinfo.dqi_flags |= DQF_ROOT_SQUASH; + uinfo.dqi_valid = IIF_ALL; + if (copy_to_user(addr, &uinfo, sizeof(uinfo))) + return -EFAULT; + return 0; +} + +static int quota_setinfo(struct super_block *sb, int type, void __user *addr) +{ + struct if_dqinfo info; + struct qc_info qinfo; + + if (copy_from_user(&info, addr, sizeof(info))) + return -EFAULT; + if (!sb->s_qcop->set_info) + return -ENOSYS; + if (info.dqi_valid & ~(IIF_FLAGS | IIF_BGRACE | IIF_IGRACE)) + return -EINVAL; + memset(&qinfo, 0, sizeof(qinfo)); + if (info.dqi_valid & IIF_FLAGS) { + if (info.dqi_flags & ~DQF_SETINFO_MASK) + return -EINVAL; + if (info.dqi_flags & DQF_ROOT_SQUASH) + qinfo.i_flags |= QCI_ROOT_SQUASH; + qinfo.i_fieldmask |= QC_FLAGS; + } + if (info.dqi_valid & IIF_BGRACE) { + qinfo.i_spc_timelimit = info.dqi_bgrace; + qinfo.i_fieldmask |= QC_SPC_TIMER; + } + if (info.dqi_valid & IIF_IGRACE) { + qinfo.i_ino_timelimit = info.dqi_igrace; + qinfo.i_fieldmask |= QC_INO_TIMER; + } + return sb->s_qcop->set_info(sb, type, &qinfo); +} + +static inline qsize_t qbtos(qsize_t blocks) +{ + return blocks << QIF_DQBLKSIZE_BITS; +} + +static inline qsize_t stoqb(qsize_t space) +{ + return (space + QIF_DQBLKSIZE - 1) >> QIF_DQBLKSIZE_BITS; +} + +static void copy_to_if_dqblk(struct if_dqblk *dst, struct qc_dqblk *src) +{ + memset(dst, 0, sizeof(*dst)); + dst->dqb_bhardlimit = stoqb(src->d_spc_hardlimit); + dst->dqb_bsoftlimit = stoqb(src->d_spc_softlimit); + dst->dqb_curspace = src->d_space; + dst->dqb_ihardlimit = src->d_ino_hardlimit; + dst->dqb_isoftlimit = src->d_ino_softlimit; + dst->dqb_curinodes = src->d_ino_count; + dst->dqb_btime = src->d_spc_timer; + dst->dqb_itime = src->d_ino_timer; + dst->dqb_valid = QIF_ALL; +} + +static int quota_getquota(struct super_block *sb, int type, qid_t id, + void __user *addr) +{ + struct kqid qid; + struct qc_dqblk fdq; + struct if_dqblk idq; + int ret; + + if (!sb->s_qcop->get_dqblk) + return -ENOSYS; + qid = make_kqid(current_user_ns(), type, id); + if (!qid_has_mapping(sb->s_user_ns, qid)) + return -EINVAL; + ret = sb->s_qcop->get_dqblk(sb, qid, &fdq); + if (ret) + return ret; + copy_to_if_dqblk(&idq, &fdq); + + if (compat_need_64bit_alignment_fixup()) { + struct compat_if_dqblk __user *compat_dqblk = addr; + + if (copy_to_user(compat_dqblk, &idq, sizeof(*compat_dqblk))) + return -EFAULT; + if (put_user(idq.dqb_valid, &compat_dqblk->dqb_valid)) + return -EFAULT; + } else { + if (copy_to_user(addr, &idq, sizeof(idq))) + return -EFAULT; + } + return 0; +} + +/* + * Return quota for next active quota >= this id, if any exists, + * otherwise return -ENOENT via ->get_nextdqblk + */ +static int quota_getnextquota(struct super_block *sb, int type, qid_t id, + void __user *addr) +{ + struct kqid qid; + struct qc_dqblk fdq; + struct if_nextdqblk idq; + int ret; + + if (!sb->s_qcop->get_nextdqblk) + return -ENOSYS; + qid = make_kqid(current_user_ns(), type, id); + if (!qid_has_mapping(sb->s_user_ns, qid)) + return -EINVAL; + ret = sb->s_qcop->get_nextdqblk(sb, &qid, &fdq); + if (ret) + return ret; + /* struct if_nextdqblk is a superset of struct if_dqblk */ + copy_to_if_dqblk((struct if_dqblk *)&idq, &fdq); + idq.dqb_id = from_kqid(current_user_ns(), qid); + if (copy_to_user(addr, &idq, sizeof(idq))) + return -EFAULT; + return 0; +} + +static void copy_from_if_dqblk(struct qc_dqblk *dst, struct if_dqblk *src) +{ + dst->d_spc_hardlimit = qbtos(src->dqb_bhardlimit); + dst->d_spc_softlimit = qbtos(src->dqb_bsoftlimit); + dst->d_space = src->dqb_curspace; + dst->d_ino_hardlimit = src->dqb_ihardlimit; + dst->d_ino_softlimit = src->dqb_isoftlimit; + dst->d_ino_count = src->dqb_curinodes; + dst->d_spc_timer = src->dqb_btime; + dst->d_ino_timer = src->dqb_itime; + + dst->d_fieldmask = 0; + if (src->dqb_valid & QIF_BLIMITS) + dst->d_fieldmask |= QC_SPC_SOFT | QC_SPC_HARD; + if (src->dqb_valid & QIF_SPACE) + dst->d_fieldmask |= QC_SPACE; + if (src->dqb_valid & QIF_ILIMITS) + dst->d_fieldmask |= QC_INO_SOFT | QC_INO_HARD; + if (src->dqb_valid & QIF_INODES) + dst->d_fieldmask |= QC_INO_COUNT; + if (src->dqb_valid & QIF_BTIME) + dst->d_fieldmask |= QC_SPC_TIMER; + if (src->dqb_valid & QIF_ITIME) + dst->d_fieldmask |= QC_INO_TIMER; +} + +static int quota_setquota(struct super_block *sb, int type, qid_t id, + void __user *addr) +{ + struct qc_dqblk fdq; + struct if_dqblk idq; + struct kqid qid; + + if (compat_need_64bit_alignment_fixup()) { + struct compat_if_dqblk __user *compat_dqblk = addr; + + if (copy_from_user(&idq, compat_dqblk, sizeof(*compat_dqblk)) || + get_user(idq.dqb_valid, &compat_dqblk->dqb_valid)) + return -EFAULT; + } else { + if (copy_from_user(&idq, addr, sizeof(idq))) + return -EFAULT; + } + if (!sb->s_qcop->set_dqblk) + return -ENOSYS; + qid = make_kqid(current_user_ns(), type, id); + if (!qid_has_mapping(sb->s_user_ns, qid)) + return -EINVAL; + copy_from_if_dqblk(&fdq, &idq); + return sb->s_qcop->set_dqblk(sb, qid, &fdq); +} + +static int quota_enable(struct super_block *sb, void __user *addr) +{ + __u32 flags; + + if (copy_from_user(&flags, addr, sizeof(flags))) + return -EFAULT; + if (!sb->s_qcop->quota_enable) + return -ENOSYS; + return sb->s_qcop->quota_enable(sb, flags); +} + +static int quota_disable(struct super_block *sb, void __user *addr) +{ + __u32 flags; + + if (copy_from_user(&flags, addr, sizeof(flags))) + return -EFAULT; + if (!sb->s_qcop->quota_disable) + return -ENOSYS; + return sb->s_qcop->quota_disable(sb, flags); +} + +static int quota_state_to_flags(struct qc_state *state) +{ + int flags = 0; + + if (state->s_state[USRQUOTA].flags & QCI_ACCT_ENABLED) + flags |= FS_QUOTA_UDQ_ACCT; + if (state->s_state[USRQUOTA].flags & QCI_LIMITS_ENFORCED) + flags |= FS_QUOTA_UDQ_ENFD; + if (state->s_state[GRPQUOTA].flags & QCI_ACCT_ENABLED) + flags |= FS_QUOTA_GDQ_ACCT; + if (state->s_state[GRPQUOTA].flags & QCI_LIMITS_ENFORCED) + flags |= FS_QUOTA_GDQ_ENFD; + if (state->s_state[PRJQUOTA].flags & QCI_ACCT_ENABLED) + flags |= FS_QUOTA_PDQ_ACCT; + if (state->s_state[PRJQUOTA].flags & QCI_LIMITS_ENFORCED) + flags |= FS_QUOTA_PDQ_ENFD; + return flags; +} + +static int quota_getstate(struct super_block *sb, int type, + struct fs_quota_stat *fqs) +{ + struct qc_state state; + int ret; + + memset(&state, 0, sizeof (struct qc_state)); + ret = sb->s_qcop->get_state(sb, &state); + if (ret < 0) + return ret; + + memset(fqs, 0, sizeof(*fqs)); + fqs->qs_version = FS_QSTAT_VERSION; + fqs->qs_flags = quota_state_to_flags(&state); + /* No quota enabled? */ + if (!fqs->qs_flags) + return -ENOSYS; + fqs->qs_incoredqs = state.s_incoredqs; + + fqs->qs_btimelimit = state.s_state[type].spc_timelimit; + fqs->qs_itimelimit = state.s_state[type].ino_timelimit; + fqs->qs_rtbtimelimit = state.s_state[type].rt_spc_timelimit; + fqs->qs_bwarnlimit = state.s_state[type].spc_warnlimit; + fqs->qs_iwarnlimit = state.s_state[type].ino_warnlimit; + + /* Inodes may be allocated even if inactive; copy out if present */ + if (state.s_state[USRQUOTA].ino) { + fqs->qs_uquota.qfs_ino = state.s_state[USRQUOTA].ino; + fqs->qs_uquota.qfs_nblks = state.s_state[USRQUOTA].blocks; + fqs->qs_uquota.qfs_nextents = state.s_state[USRQUOTA].nextents; + } + if (state.s_state[GRPQUOTA].ino) { + fqs->qs_gquota.qfs_ino = state.s_state[GRPQUOTA].ino; + fqs->qs_gquota.qfs_nblks = state.s_state[GRPQUOTA].blocks; + fqs->qs_gquota.qfs_nextents = state.s_state[GRPQUOTA].nextents; + } + if (state.s_state[PRJQUOTA].ino) { + /* + * Q_XGETQSTAT doesn't have room for both group and project + * quotas. So, allow the project quota values to be copied out + * only if there is no group quota information available. + */ + if (!(state.s_state[GRPQUOTA].flags & QCI_ACCT_ENABLED)) { + fqs->qs_gquota.qfs_ino = state.s_state[PRJQUOTA].ino; + fqs->qs_gquota.qfs_nblks = + state.s_state[PRJQUOTA].blocks; + fqs->qs_gquota.qfs_nextents = + state.s_state[PRJQUOTA].nextents; + } + } + return 0; +} + +static int compat_copy_fs_qfilestat(struct compat_fs_qfilestat __user *to, + struct fs_qfilestat *from) +{ + if (copy_to_user(to, from, sizeof(*to)) || + put_user(from->qfs_nextents, &to->qfs_nextents)) + return -EFAULT; + return 0; +} + +static int compat_copy_fs_quota_stat(struct compat_fs_quota_stat __user *to, + struct fs_quota_stat *from) +{ + if (put_user(from->qs_version, &to->qs_version) || + put_user(from->qs_flags, &to->qs_flags) || + put_user(from->qs_pad, &to->qs_pad) || + compat_copy_fs_qfilestat(&to->qs_uquota, &from->qs_uquota) || + compat_copy_fs_qfilestat(&to->qs_gquota, &from->qs_gquota) || + put_user(from->qs_incoredqs, &to->qs_incoredqs) || + put_user(from->qs_btimelimit, &to->qs_btimelimit) || + put_user(from->qs_itimelimit, &to->qs_itimelimit) || + put_user(from->qs_rtbtimelimit, &to->qs_rtbtimelimit) || + put_user(from->qs_bwarnlimit, &to->qs_bwarnlimit) || + put_user(from->qs_iwarnlimit, &to->qs_iwarnlimit)) + return -EFAULT; + return 0; +} + +static int quota_getxstate(struct super_block *sb, int type, void __user *addr) +{ + struct fs_quota_stat fqs; + int ret; + + if (!sb->s_qcop->get_state) + return -ENOSYS; + ret = quota_getstate(sb, type, &fqs); + if (ret) + return ret; + + if (compat_need_64bit_alignment_fixup()) + return compat_copy_fs_quota_stat(addr, &fqs); + if (copy_to_user(addr, &fqs, sizeof(fqs))) + return -EFAULT; + return 0; +} + +static int quota_getstatev(struct super_block *sb, int type, + struct fs_quota_statv *fqs) +{ + struct qc_state state; + int ret; + + memset(&state, 0, sizeof (struct qc_state)); + ret = sb->s_qcop->get_state(sb, &state); + if (ret < 0) + return ret; + + memset(fqs, 0, sizeof(*fqs)); + fqs->qs_version = FS_QSTAT_VERSION; + fqs->qs_flags = quota_state_to_flags(&state); + /* No quota enabled? */ + if (!fqs->qs_flags) + return -ENOSYS; + fqs->qs_incoredqs = state.s_incoredqs; + + fqs->qs_btimelimit = state.s_state[type].spc_timelimit; + fqs->qs_itimelimit = state.s_state[type].ino_timelimit; + fqs->qs_rtbtimelimit = state.s_state[type].rt_spc_timelimit; + fqs->qs_bwarnlimit = state.s_state[type].spc_warnlimit; + fqs->qs_iwarnlimit = state.s_state[type].ino_warnlimit; + fqs->qs_rtbwarnlimit = state.s_state[type].rt_spc_warnlimit; + + /* Inodes may be allocated even if inactive; copy out if present */ + if (state.s_state[USRQUOTA].ino) { + fqs->qs_uquota.qfs_ino = state.s_state[USRQUOTA].ino; + fqs->qs_uquota.qfs_nblks = state.s_state[USRQUOTA].blocks; + fqs->qs_uquota.qfs_nextents = state.s_state[USRQUOTA].nextents; + } + if (state.s_state[GRPQUOTA].ino) { + fqs->qs_gquota.qfs_ino = state.s_state[GRPQUOTA].ino; + fqs->qs_gquota.qfs_nblks = state.s_state[GRPQUOTA].blocks; + fqs->qs_gquota.qfs_nextents = state.s_state[GRPQUOTA].nextents; + } + if (state.s_state[PRJQUOTA].ino) { + fqs->qs_pquota.qfs_ino = state.s_state[PRJQUOTA].ino; + fqs->qs_pquota.qfs_nblks = state.s_state[PRJQUOTA].blocks; + fqs->qs_pquota.qfs_nextents = state.s_state[PRJQUOTA].nextents; + } + return 0; +} + +static int quota_getxstatev(struct super_block *sb, int type, void __user *addr) +{ + struct fs_quota_statv fqs; + int ret; + + if (!sb->s_qcop->get_state) + return -ENOSYS; + + memset(&fqs, 0, sizeof(fqs)); + if (copy_from_user(&fqs, addr, 1)) /* Just read qs_version */ + return -EFAULT; + + /* If this kernel doesn't support user specified version, fail */ + switch (fqs.qs_version) { + case FS_QSTATV_VERSION1: + break; + default: + return -EINVAL; + } + ret = quota_getstatev(sb, type, &fqs); + if (!ret && copy_to_user(addr, &fqs, sizeof(fqs))) + return -EFAULT; + return ret; +} + +/* + * XFS defines BBTOB and BTOBB macros inside fs/xfs/ and we cannot move them + * out of there as xfsprogs rely on definitions being in that header file. So + * just define same functions here for quota purposes. + */ +#define XFS_BB_SHIFT 9 + +static inline u64 quota_bbtob(u64 blocks) +{ + return blocks << XFS_BB_SHIFT; +} + +static inline u64 quota_btobb(u64 bytes) +{ + return (bytes + (1 << XFS_BB_SHIFT) - 1) >> XFS_BB_SHIFT; +} + +static inline s64 copy_from_xfs_dqblk_ts(const struct fs_disk_quota *d, + __s32 timer, __s8 timer_hi) +{ + if (d->d_fieldmask & FS_DQ_BIGTIME) + return (u32)timer | (s64)timer_hi << 32; + return timer; +} + +static void copy_from_xfs_dqblk(struct qc_dqblk *dst, struct fs_disk_quota *src) +{ + dst->d_spc_hardlimit = quota_bbtob(src->d_blk_hardlimit); + dst->d_spc_softlimit = quota_bbtob(src->d_blk_softlimit); + dst->d_ino_hardlimit = src->d_ino_hardlimit; + dst->d_ino_softlimit = src->d_ino_softlimit; + dst->d_space = quota_bbtob(src->d_bcount); + dst->d_ino_count = src->d_icount; + dst->d_ino_timer = copy_from_xfs_dqblk_ts(src, src->d_itimer, + src->d_itimer_hi); + dst->d_spc_timer = copy_from_xfs_dqblk_ts(src, src->d_btimer, + src->d_btimer_hi); + dst->d_ino_warns = src->d_iwarns; + dst->d_spc_warns = src->d_bwarns; + dst->d_rt_spc_hardlimit = quota_bbtob(src->d_rtb_hardlimit); + dst->d_rt_spc_softlimit = quota_bbtob(src->d_rtb_softlimit); + dst->d_rt_space = quota_bbtob(src->d_rtbcount); + dst->d_rt_spc_timer = copy_from_xfs_dqblk_ts(src, src->d_rtbtimer, + src->d_rtbtimer_hi); + dst->d_rt_spc_warns = src->d_rtbwarns; + dst->d_fieldmask = 0; + if (src->d_fieldmask & FS_DQ_ISOFT) + dst->d_fieldmask |= QC_INO_SOFT; + if (src->d_fieldmask & FS_DQ_IHARD) + dst->d_fieldmask |= QC_INO_HARD; + if (src->d_fieldmask & FS_DQ_BSOFT) + dst->d_fieldmask |= QC_SPC_SOFT; + if (src->d_fieldmask & FS_DQ_BHARD) + dst->d_fieldmask |= QC_SPC_HARD; + if (src->d_fieldmask & FS_DQ_RTBSOFT) + dst->d_fieldmask |= QC_RT_SPC_SOFT; + if (src->d_fieldmask & FS_DQ_RTBHARD) + dst->d_fieldmask |= QC_RT_SPC_HARD; + if (src->d_fieldmask & FS_DQ_BTIMER) + dst->d_fieldmask |= QC_SPC_TIMER; + if (src->d_fieldmask & FS_DQ_ITIMER) + dst->d_fieldmask |= QC_INO_TIMER; + if (src->d_fieldmask & FS_DQ_RTBTIMER) + dst->d_fieldmask |= QC_RT_SPC_TIMER; + if (src->d_fieldmask & FS_DQ_BWARNS) + dst->d_fieldmask |= QC_SPC_WARNS; + if (src->d_fieldmask & FS_DQ_IWARNS) + dst->d_fieldmask |= QC_INO_WARNS; + if (src->d_fieldmask & FS_DQ_RTBWARNS) + dst->d_fieldmask |= QC_RT_SPC_WARNS; + if (src->d_fieldmask & FS_DQ_BCOUNT) + dst->d_fieldmask |= QC_SPACE; + if (src->d_fieldmask & FS_DQ_ICOUNT) + dst->d_fieldmask |= QC_INO_COUNT; + if (src->d_fieldmask & FS_DQ_RTBCOUNT) + dst->d_fieldmask |= QC_RT_SPACE; +} + +static void copy_qcinfo_from_xfs_dqblk(struct qc_info *dst, + struct fs_disk_quota *src) +{ + memset(dst, 0, sizeof(*dst)); + dst->i_spc_timelimit = src->d_btimer; + dst->i_ino_timelimit = src->d_itimer; + dst->i_rt_spc_timelimit = src->d_rtbtimer; + dst->i_ino_warnlimit = src->d_iwarns; + dst->i_spc_warnlimit = src->d_bwarns; + dst->i_rt_spc_warnlimit = src->d_rtbwarns; + if (src->d_fieldmask & FS_DQ_BWARNS) + dst->i_fieldmask |= QC_SPC_WARNS; + if (src->d_fieldmask & FS_DQ_IWARNS) + dst->i_fieldmask |= QC_INO_WARNS; + if (src->d_fieldmask & FS_DQ_RTBWARNS) + dst->i_fieldmask |= QC_RT_SPC_WARNS; + if (src->d_fieldmask & FS_DQ_BTIMER) + dst->i_fieldmask |= QC_SPC_TIMER; + if (src->d_fieldmask & FS_DQ_ITIMER) + dst->i_fieldmask |= QC_INO_TIMER; + if (src->d_fieldmask & FS_DQ_RTBTIMER) + dst->i_fieldmask |= QC_RT_SPC_TIMER; +} + +static int quota_setxquota(struct super_block *sb, int type, qid_t id, + void __user *addr) +{ + struct fs_disk_quota fdq; + struct qc_dqblk qdq; + struct kqid qid; + + if (copy_from_user(&fdq, addr, sizeof(fdq))) + return -EFAULT; + if (!sb->s_qcop->set_dqblk) + return -ENOSYS; + qid = make_kqid(current_user_ns(), type, id); + if (!qid_has_mapping(sb->s_user_ns, qid)) + return -EINVAL; + /* Are we actually setting timer / warning limits for all users? */ + if (from_kqid(sb->s_user_ns, qid) == 0 && + fdq.d_fieldmask & (FS_DQ_WARNS_MASK | FS_DQ_TIMER_MASK)) { + struct qc_info qinfo; + int ret; + + if (!sb->s_qcop->set_info) + return -EINVAL; + copy_qcinfo_from_xfs_dqblk(&qinfo, &fdq); + ret = sb->s_qcop->set_info(sb, type, &qinfo); + if (ret) + return ret; + /* These are already done */ + fdq.d_fieldmask &= ~(FS_DQ_WARNS_MASK | FS_DQ_TIMER_MASK); + } + copy_from_xfs_dqblk(&qdq, &fdq); + return sb->s_qcop->set_dqblk(sb, qid, &qdq); +} + +static inline void copy_to_xfs_dqblk_ts(const struct fs_disk_quota *d, + __s32 *timer_lo, __s8 *timer_hi, s64 timer) +{ + *timer_lo = timer; + if (d->d_fieldmask & FS_DQ_BIGTIME) + *timer_hi = timer >> 32; +} + +static inline bool want_bigtime(s64 timer) +{ + return timer > S32_MAX || timer < S32_MIN; +} + +static void copy_to_xfs_dqblk(struct fs_disk_quota *dst, struct qc_dqblk *src, + int type, qid_t id) +{ + memset(dst, 0, sizeof(*dst)); + if (want_bigtime(src->d_ino_timer) || want_bigtime(src->d_spc_timer) || + want_bigtime(src->d_rt_spc_timer)) + dst->d_fieldmask |= FS_DQ_BIGTIME; + dst->d_version = FS_DQUOT_VERSION; + dst->d_id = id; + if (type == USRQUOTA) + dst->d_flags = FS_USER_QUOTA; + else if (type == PRJQUOTA) + dst->d_flags = FS_PROJ_QUOTA; + else + dst->d_flags = FS_GROUP_QUOTA; + dst->d_blk_hardlimit = quota_btobb(src->d_spc_hardlimit); + dst->d_blk_softlimit = quota_btobb(src->d_spc_softlimit); + dst->d_ino_hardlimit = src->d_ino_hardlimit; + dst->d_ino_softlimit = src->d_ino_softlimit; + dst->d_bcount = quota_btobb(src->d_space); + dst->d_icount = src->d_ino_count; + copy_to_xfs_dqblk_ts(dst, &dst->d_itimer, &dst->d_itimer_hi, + src->d_ino_timer); + copy_to_xfs_dqblk_ts(dst, &dst->d_btimer, &dst->d_btimer_hi, + src->d_spc_timer); + dst->d_iwarns = src->d_ino_warns; + dst->d_bwarns = src->d_spc_warns; + dst->d_rtb_hardlimit = quota_btobb(src->d_rt_spc_hardlimit); + dst->d_rtb_softlimit = quota_btobb(src->d_rt_spc_softlimit); + dst->d_rtbcount = quota_btobb(src->d_rt_space); + copy_to_xfs_dqblk_ts(dst, &dst->d_rtbtimer, &dst->d_rtbtimer_hi, + src->d_rt_spc_timer); + dst->d_rtbwarns = src->d_rt_spc_warns; +} + +static int quota_getxquota(struct super_block *sb, int type, qid_t id, + void __user *addr) +{ + struct fs_disk_quota fdq; + struct qc_dqblk qdq; + struct kqid qid; + int ret; + + if (!sb->s_qcop->get_dqblk) + return -ENOSYS; + qid = make_kqid(current_user_ns(), type, id); + if (!qid_has_mapping(sb->s_user_ns, qid)) + return -EINVAL; + ret = sb->s_qcop->get_dqblk(sb, qid, &qdq); + if (ret) + return ret; + copy_to_xfs_dqblk(&fdq, &qdq, type, id); + if (copy_to_user(addr, &fdq, sizeof(fdq))) + return -EFAULT; + return ret; +} + +/* + * Return quota for next active quota >= this id, if any exists, + * otherwise return -ENOENT via ->get_nextdqblk. + */ +static int quota_getnextxquota(struct super_block *sb, int type, qid_t id, + void __user *addr) +{ + struct fs_disk_quota fdq; + struct qc_dqblk qdq; + struct kqid qid; + qid_t id_out; + int ret; + + if (!sb->s_qcop->get_nextdqblk) + return -ENOSYS; + qid = make_kqid(current_user_ns(), type, id); + if (!qid_has_mapping(sb->s_user_ns, qid)) + return -EINVAL; + ret = sb->s_qcop->get_nextdqblk(sb, &qid, &qdq); + if (ret) + return ret; + id_out = from_kqid(current_user_ns(), qid); + copy_to_xfs_dqblk(&fdq, &qdq, type, id_out); + if (copy_to_user(addr, &fdq, sizeof(fdq))) + return -EFAULT; + return ret; +} + +static int quota_rmxquota(struct super_block *sb, void __user *addr) +{ + __u32 flags; + + if (copy_from_user(&flags, addr, sizeof(flags))) + return -EFAULT; + if (!sb->s_qcop->rm_xquota) + return -ENOSYS; + return sb->s_qcop->rm_xquota(sb, flags); +} + +/* Copy parameters and call proper function */ +static int do_quotactl(struct super_block *sb, int type, int cmd, qid_t id, + void __user *addr, const struct path *path) +{ + int ret; + + type = array_index_nospec(type, MAXQUOTAS); + /* + * Quota not supported on this fs? Check this before s_quota_types + * since they needn't be set if quota is not supported at all. + */ + if (!sb->s_qcop) + return -ENOSYS; + if (!(sb->s_quota_types & (1 << type))) + return -EINVAL; + + ret = check_quotactl_permission(sb, type, cmd, id); + if (ret < 0) + return ret; + + switch (cmd) { + case Q_QUOTAON: + return quota_quotaon(sb, type, id, path); + case Q_QUOTAOFF: + return quota_quotaoff(sb, type); + case Q_GETFMT: + return quota_getfmt(sb, type, addr); + case Q_GETINFO: + return quota_getinfo(sb, type, addr); + case Q_SETINFO: + return quota_setinfo(sb, type, addr); + case Q_GETQUOTA: + return quota_getquota(sb, type, id, addr); + case Q_GETNEXTQUOTA: + return quota_getnextquota(sb, type, id, addr); + case Q_SETQUOTA: + return quota_setquota(sb, type, id, addr); + case Q_SYNC: + if (!sb->s_qcop->quota_sync) + return -ENOSYS; + return sb->s_qcop->quota_sync(sb, type); + case Q_XQUOTAON: + return quota_enable(sb, addr); + case Q_XQUOTAOFF: + return quota_disable(sb, addr); + case Q_XQUOTARM: + return quota_rmxquota(sb, addr); + case Q_XGETQSTAT: + return quota_getxstate(sb, type, addr); + case Q_XGETQSTATV: + return quota_getxstatev(sb, type, addr); + case Q_XSETQLIM: + return quota_setxquota(sb, type, id, addr); + case Q_XGETQUOTA: + return quota_getxquota(sb, type, id, addr); + case Q_XGETNEXTQUOTA: + return quota_getnextxquota(sb, type, id, addr); + case Q_XQUOTASYNC: + if (sb_rdonly(sb)) + return -EROFS; + /* XFS quotas are fully coherent now, making this call a noop */ + return 0; + default: + return -EINVAL; + } +} + +/* Return 1 if 'cmd' will block on frozen filesystem */ +static int quotactl_cmd_write(int cmd) +{ + /* + * We cannot allow Q_GETQUOTA and Q_GETNEXTQUOTA without write access + * as dquot_acquire() may allocate space for new structure and OCFS2 + * needs to increment on-disk use count. + */ + switch (cmd) { + case Q_GETFMT: + case Q_GETINFO: + case Q_SYNC: + case Q_XGETQSTAT: + case Q_XGETQSTATV: + case Q_XGETQUOTA: + case Q_XGETNEXTQUOTA: + case Q_XQUOTASYNC: + return 0; + } + return 1; +} + +/* Return true if quotactl command is manipulating quota on/off state */ +static bool quotactl_cmd_onoff(int cmd) +{ + return (cmd == Q_QUOTAON) || (cmd == Q_QUOTAOFF) || + (cmd == Q_XQUOTAON) || (cmd == Q_XQUOTAOFF); +} + +/* + * look up a superblock on which quota ops will be performed + * - use the name of a block device to find the superblock thereon + */ +static struct super_block *quotactl_block(const char __user *special, int cmd) +{ +#ifdef CONFIG_BLOCK + struct super_block *sb; + CLASS(filename, tmp)(special); + bool excl = false, thawed = false; + int error; + dev_t dev; + + if (IS_ERR(tmp)) + return ERR_CAST(tmp); + error = lookup_bdev(tmp->name, &dev); + if (error) + return ERR_PTR(error); + + if (quotactl_cmd_onoff(cmd)) { + excl = true; + thawed = true; + } else if (quotactl_cmd_write(cmd)) { + thawed = true; + } + +retry: + sb = user_get_super(dev, excl); + if (!sb) + return ERR_PTR(-ENODEV); + if (thawed && sb->s_writers.frozen != SB_UNFROZEN) { + if (excl) + up_write(&sb->s_umount); + else + up_read(&sb->s_umount); + /* Wait for sb to unfreeze */ + sb_start_write(sb); + sb_end_write(sb); + put_super(sb); + cond_resched(); + goto retry; + } + return sb; + +#else + return ERR_PTR(-ENODEV); +#endif +} + +/* + * This is the system call interface. This communicates with + * the user-level programs. Currently this only supports diskquota + * calls. Maybe we need to add the process quotas etc. in the future, + * but we probably should use rlimits for that. + */ +SYSCALL_DEFINE4(quotactl, unsigned int, cmd, const char __user *, special, + qid_t, id, void __user *, addr) +{ + uint cmds, type; + struct super_block *sb = NULL; + struct path path, *pathp = NULL; + int ret; + + cmds = cmd >> SUBCMDSHIFT; + type = cmd & SUBCMDMASK; + + if (type >= MAXQUOTAS) + return -EINVAL; + + /* + * As a special case Q_SYNC can be called without a specific device. + * It will iterate all superblocks that have quota enabled and call + * the sync action on each of them. + */ + if (!special) { + if (cmds == Q_SYNC) + return quota_sync_all(type); + return -ENODEV; + } + + /* + * Path for quotaon has to be resolved before grabbing superblock + * because that gets s_umount sem which is also possibly needed by path + * resolution (think about autofs) and thus deadlocks could arise. + */ + if (cmds == Q_QUOTAON) { + ret = user_path_at(AT_FDCWD, addr, LOOKUP_FOLLOW|LOOKUP_AUTOMOUNT, &path); + if (ret) + pathp = ERR_PTR(ret); + else + pathp = &path; + } + + sb = quotactl_block(special, cmds); + if (IS_ERR(sb)) { + ret = PTR_ERR(sb); + goto out; + } + + ret = do_quotactl(sb, type, cmds, id, addr, pathp); + + if (!quotactl_cmd_onoff(cmds)) + drop_super(sb); + else + drop_super_exclusive(sb); +out: + if (pathp && !IS_ERR(pathp)) + path_put(pathp); + return ret; +} + +SYSCALL_DEFINE4(quotactl_fd, unsigned int, fd, unsigned int, cmd, + qid_t, id, void __user *, addr) +{ + struct super_block *sb; + unsigned int cmds = cmd >> SUBCMDSHIFT; + unsigned int type = cmd & SUBCMDMASK; + CLASS(fd_raw, f)(fd); + int ret; + + if (fd_empty(f)) + return -EBADF; + + if (type >= MAXQUOTAS) + return -EINVAL; + + if (quotactl_cmd_write(cmds)) { + ret = mnt_want_write(fd_file(f)->f_path.mnt); + if (ret) + return ret; + } + + sb = fd_file(f)->f_path.mnt->mnt_sb; + if (quotactl_cmd_onoff(cmds)) + down_write(&sb->s_umount); + else + down_read(&sb->s_umount); + + ret = do_quotactl(sb, type, cmds, id, addr, ERR_PTR(-EINVAL)); + + if (quotactl_cmd_onoff(cmds)) + up_write(&sb->s_umount); + else + up_read(&sb->s_umount); + + if (quotactl_cmd_write(cmds)) + mnt_drop_write(fd_file(f)->f_path.mnt); + return ret; +} diff --git a/fs/quota/quota_tree.c b/fs/quota/quota_tree.c new file mode 100644 index 000000000..afceef3dd --- /dev/null +++ b/fs/quota/quota_tree.c @@ -0,0 +1,853 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * vfsv0 quota IO operations on file + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "quota_tree.h" + +MODULE_AUTHOR("Jan Kara"); +MODULE_DESCRIPTION("Quota trie support"); +MODULE_LICENSE("GPL"); + +/* + * Maximum quota tree depth we support. Only to limit recursion when working + * with the tree. + */ +#define MAX_QTREE_DEPTH 6 + +#define __QUOTA_QT_PARANOIA + +static int __get_index(struct qtree_mem_dqinfo *info, qid_t id, int depth) +{ + unsigned int epb = info->dqi_usable_bs >> 2; + + depth = info->dqi_qtree_depth - depth - 1; + while (depth--) + id /= epb; + return id % epb; +} + +static int get_index(struct qtree_mem_dqinfo *info, struct kqid qid, int depth) +{ + qid_t id = from_kqid(&init_user_ns, qid); + + return __get_index(info, id, depth); +} + +/* Number of entries in one blocks */ +static int qtree_dqstr_in_blk(struct qtree_mem_dqinfo *info) +{ + return (info->dqi_usable_bs - sizeof(struct qt_disk_dqdbheader)) + / info->dqi_entry_size; +} + +static ssize_t read_blk(struct qtree_mem_dqinfo *info, uint blk, char *buf) +{ + struct super_block *sb = info->dqi_sb; + + memset(buf, 0, info->dqi_usable_bs); + return sb->s_op->quota_read(sb, info->dqi_type, buf, + info->dqi_usable_bs, (loff_t)blk << info->dqi_blocksize_bits); +} + +static ssize_t write_blk(struct qtree_mem_dqinfo *info, uint blk, char *buf) +{ + struct super_block *sb = info->dqi_sb; + ssize_t ret; + + ret = sb->s_op->quota_write(sb, info->dqi_type, buf, + info->dqi_usable_bs, (loff_t)blk << info->dqi_blocksize_bits); + if (ret != info->dqi_usable_bs) { + quota_error(sb, "dquota write failed"); + if (ret >= 0) + ret = -EIO; + } + return ret; +} + +static inline int do_check_range(struct super_block *sb, const char *val_name, + uint val, uint min_val, uint max_val) +{ + if (val < min_val || val > max_val) { + quota_error(sb, "Getting %s %u out of range %u-%u", + val_name, val, min_val, max_val); + return -EUCLEAN; + } + + return 0; +} + +static int check_dquot_block_header(struct qtree_mem_dqinfo *info, + struct qt_disk_dqdbheader *dh) +{ + int err = 0; + + err = do_check_range(info->dqi_sb, "dqdh_next_free", + le32_to_cpu(dh->dqdh_next_free), 0, + info->dqi_blocks - 1); + if (err) + return err; + err = do_check_range(info->dqi_sb, "dqdh_prev_free", + le32_to_cpu(dh->dqdh_prev_free), 0, + info->dqi_blocks - 1); + if (err) + return err; + err = do_check_range(info->dqi_sb, "dqdh_entries", + le16_to_cpu(dh->dqdh_entries), 0, + qtree_dqstr_in_blk(info)); + + return err; +} + +/* Remove empty block from list and return it */ +static int get_free_dqblk(struct qtree_mem_dqinfo *info) +{ + char *buf = kmalloc(info->dqi_usable_bs, GFP_KERNEL); + struct qt_disk_dqdbheader *dh = (struct qt_disk_dqdbheader *)buf; + int ret, blk; + + if (!buf) + return -ENOMEM; + if (info->dqi_free_blk) { + blk = info->dqi_free_blk; + ret = read_blk(info, blk, buf); + if (ret < 0) + goto out_buf; + ret = check_dquot_block_header(info, dh); + if (ret) + goto out_buf; + info->dqi_free_blk = le32_to_cpu(dh->dqdh_next_free); + } + else { + memset(buf, 0, info->dqi_usable_bs); + /* Assure block allocation... */ + ret = write_blk(info, info->dqi_blocks, buf); + if (ret < 0) + goto out_buf; + blk = info->dqi_blocks++; + } + mark_info_dirty(info->dqi_sb, info->dqi_type); + ret = blk; +out_buf: + kfree(buf); + return ret; +} + +/* Insert empty block to the list */ +static int put_free_dqblk(struct qtree_mem_dqinfo *info, char *buf, uint blk) +{ + struct qt_disk_dqdbheader *dh = (struct qt_disk_dqdbheader *)buf; + int err; + + dh->dqdh_next_free = cpu_to_le32(info->dqi_free_blk); + dh->dqdh_prev_free = cpu_to_le32(0); + dh->dqdh_entries = cpu_to_le16(0); + err = write_blk(info, blk, buf); + if (err < 0) + return err; + info->dqi_free_blk = blk; + mark_info_dirty(info->dqi_sb, info->dqi_type); + return 0; +} + +/* Remove given block from the list of blocks with free entries */ +static int remove_free_dqentry(struct qtree_mem_dqinfo *info, char *buf, + uint blk) +{ + char *tmpbuf = kmalloc(info->dqi_usable_bs, GFP_KERNEL); + struct qt_disk_dqdbheader *dh = (struct qt_disk_dqdbheader *)buf; + uint nextblk = le32_to_cpu(dh->dqdh_next_free); + uint prevblk = le32_to_cpu(dh->dqdh_prev_free); + int err; + + if (!tmpbuf) + return -ENOMEM; + if (nextblk) { + err = read_blk(info, nextblk, tmpbuf); + if (err < 0) + goto out_buf; + ((struct qt_disk_dqdbheader *)tmpbuf)->dqdh_prev_free = + dh->dqdh_prev_free; + err = write_blk(info, nextblk, tmpbuf); + if (err < 0) + goto out_buf; + } + if (prevblk) { + err = read_blk(info, prevblk, tmpbuf); + if (err < 0) + goto out_buf; + ((struct qt_disk_dqdbheader *)tmpbuf)->dqdh_next_free = + dh->dqdh_next_free; + err = write_blk(info, prevblk, tmpbuf); + if (err < 0) + goto out_buf; + } else { + info->dqi_free_entry = nextblk; + mark_info_dirty(info->dqi_sb, info->dqi_type); + } + kfree(tmpbuf); + dh->dqdh_next_free = dh->dqdh_prev_free = cpu_to_le32(0); + /* No matter whether write succeeds block is out of list */ + if (write_blk(info, blk, buf) < 0) + quota_error(info->dqi_sb, "Can't write block (%u) " + "with free entries", blk); + return 0; +out_buf: + kfree(tmpbuf); + return err; +} + +/* Insert given block to the beginning of list with free entries */ +static int insert_free_dqentry(struct qtree_mem_dqinfo *info, char *buf, + uint blk) +{ + char *tmpbuf = kmalloc(info->dqi_usable_bs, GFP_KERNEL); + struct qt_disk_dqdbheader *dh = (struct qt_disk_dqdbheader *)buf; + int err; + + if (!tmpbuf) + return -ENOMEM; + dh->dqdh_next_free = cpu_to_le32(info->dqi_free_entry); + dh->dqdh_prev_free = cpu_to_le32(0); + err = write_blk(info, blk, buf); + if (err < 0) + goto out_buf; + if (info->dqi_free_entry) { + err = read_blk(info, info->dqi_free_entry, tmpbuf); + if (err < 0) + goto out_buf; + ((struct qt_disk_dqdbheader *)tmpbuf)->dqdh_prev_free = + cpu_to_le32(blk); + err = write_blk(info, info->dqi_free_entry, tmpbuf); + if (err < 0) + goto out_buf; + } + kfree(tmpbuf); + info->dqi_free_entry = blk; + mark_info_dirty(info->dqi_sb, info->dqi_type); + return 0; +out_buf: + kfree(tmpbuf); + return err; +} + +/* Is the entry in the block free? */ +int qtree_entry_unused(struct qtree_mem_dqinfo *info, char *disk) +{ + int i; + + for (i = 0; i < info->dqi_entry_size; i++) + if (disk[i]) + return 0; + return 1; +} +EXPORT_SYMBOL(qtree_entry_unused); + +/* Find space for dquot */ +static uint find_free_dqentry(struct qtree_mem_dqinfo *info, + struct dquot *dquot, int *err) +{ + uint blk, i; + struct qt_disk_dqdbheader *dh; + char *buf = kmalloc(info->dqi_usable_bs, GFP_KERNEL); + char *ddquot; + + *err = 0; + if (!buf) { + *err = -ENOMEM; + return 0; + } + dh = (struct qt_disk_dqdbheader *)buf; + if (info->dqi_free_entry) { + blk = info->dqi_free_entry; + *err = read_blk(info, blk, buf); + if (*err < 0) + goto out_buf; + *err = check_dquot_block_header(info, dh); + if (*err) + goto out_buf; + } else { + blk = get_free_dqblk(info); + if ((int)blk < 0) { + *err = blk; + kfree(buf); + return 0; + } + memset(buf, 0, info->dqi_usable_bs); + /* This is enough as the block is already zeroed and the entry + * list is empty... */ + info->dqi_free_entry = blk; + mark_info_dirty(dquot->dq_sb, dquot->dq_id.type); + } + /* Block will be full? */ + if (le16_to_cpu(dh->dqdh_entries) + 1 >= qtree_dqstr_in_blk(info)) { + *err = remove_free_dqentry(info, buf, blk); + if (*err < 0) { + quota_error(dquot->dq_sb, "Can't remove block (%u) " + "from entry free list", blk); + goto out_buf; + } + } + le16_add_cpu(&dh->dqdh_entries, 1); + /* Find free structure in block */ + ddquot = buf + sizeof(struct qt_disk_dqdbheader); + for (i = 0; i < qtree_dqstr_in_blk(info); i++) { + if (qtree_entry_unused(info, ddquot)) + break; + ddquot += info->dqi_entry_size; + } +#ifdef __QUOTA_QT_PARANOIA + if (i == qtree_dqstr_in_blk(info)) { + quota_error(dquot->dq_sb, "Data block full but it shouldn't"); + *err = -EIO; + goto out_buf; + } +#endif + *err = write_blk(info, blk, buf); + if (*err < 0) { + quota_error(dquot->dq_sb, "Can't write quota data block %u", + blk); + goto out_buf; + } + dquot->dq_off = ((loff_t)blk << info->dqi_blocksize_bits) + + sizeof(struct qt_disk_dqdbheader) + + i * info->dqi_entry_size; + kfree(buf); + return blk; +out_buf: + kfree(buf); + return 0; +} + +/* Insert reference to structure into the trie */ +static int do_insert_tree(struct qtree_mem_dqinfo *info, struct dquot *dquot, + uint *blks, int depth) +{ + char *buf = kmalloc(info->dqi_usable_bs, GFP_KERNEL); + int ret = 0, newson = 0, newact = 0; + __le32 *ref; + uint newblk; + int i; + + if (!buf) + return -ENOMEM; + if (!blks[depth]) { + ret = get_free_dqblk(info); + if (ret < 0) + goto out_buf; + for (i = 0; i < depth; i++) + if (ret == blks[i]) { + quota_error(dquot->dq_sb, + "Free block already used in tree: block %u", + ret); + ret = -EIO; + goto out_buf; + } + blks[depth] = ret; + memset(buf, 0, info->dqi_usable_bs); + newact = 1; + } else { + ret = read_blk(info, blks[depth], buf); + if (ret < 0) { + quota_error(dquot->dq_sb, "Can't read tree quota " + "block %u", blks[depth]); + goto out_buf; + } + } + ref = (__le32 *)buf; + newblk = le32_to_cpu(ref[get_index(info, dquot->dq_id, depth)]); + ret = do_check_range(dquot->dq_sb, "block", newblk, 0, + info->dqi_blocks - 1); + if (ret) + goto out_buf; + if (!newblk) { + newson = 1; + } else { + for (i = 0; i <= depth; i++) + if (newblk == blks[i]) { + quota_error(dquot->dq_sb, + "Cycle in quota tree detected: block %u index %u", + blks[depth], + get_index(info, dquot->dq_id, depth)); + ret = -EIO; + goto out_buf; + } + } + blks[depth + 1] = newblk; + if (depth == info->dqi_qtree_depth - 1) { +#ifdef __QUOTA_QT_PARANOIA + if (newblk) { + quota_error(dquot->dq_sb, "Inserting already present " + "quota entry (block %u)", + le32_to_cpu(ref[get_index(info, + dquot->dq_id, depth)])); + ret = -EIO; + goto out_buf; + } +#endif + blks[depth + 1] = find_free_dqentry(info, dquot, &ret); + } else { + ret = do_insert_tree(info, dquot, blks, depth + 1); + } + if (newson && ret >= 0) { + ref[get_index(info, dquot->dq_id, depth)] = + cpu_to_le32(blks[depth + 1]); + ret = write_blk(info, blks[depth], buf); + } else if (newact && ret < 0) { + put_free_dqblk(info, buf, blks[depth]); + } +out_buf: + kfree(buf); + return ret; +} + +/* Wrapper for inserting quota structure into tree */ +static inline int dq_insert_tree(struct qtree_mem_dqinfo *info, + struct dquot *dquot) +{ + uint blks[MAX_QTREE_DEPTH] = { QT_TREEOFF }; + +#ifdef __QUOTA_QT_PARANOIA + if (info->dqi_blocks <= QT_TREEOFF) { + quota_error(dquot->dq_sb, "Quota tree root isn't allocated!"); + return -EIO; + } +#endif + if (info->dqi_qtree_depth >= MAX_QTREE_DEPTH) { + quota_error(dquot->dq_sb, "Quota tree depth too big!"); + return -EIO; + } + return do_insert_tree(info, dquot, blks, 0); +} + +/* + * We don't have to be afraid of deadlocks as we never have quotas on quota + * files... + */ +int qtree_write_dquot(struct qtree_mem_dqinfo *info, struct dquot *dquot) +{ + int type = dquot->dq_id.type; + struct super_block *sb = dquot->dq_sb; + ssize_t ret; + char *ddquot = kmalloc(info->dqi_entry_size, GFP_KERNEL); + + if (!ddquot) + return -ENOMEM; + + /* dq_off is guarded by dqio_sem */ + if (!dquot->dq_off) { + ret = dq_insert_tree(info, dquot); + if (ret < 0) { + quota_error(sb, "Error %zd occurred while creating " + "quota", ret); + kfree(ddquot); + return ret; + } + } + spin_lock(&dquot->dq_dqb_lock); + info->dqi_ops->mem2disk_dqblk(ddquot, dquot); + spin_unlock(&dquot->dq_dqb_lock); + ret = sb->s_op->quota_write(sb, type, ddquot, info->dqi_entry_size, + dquot->dq_off); + if (ret != info->dqi_entry_size) { + quota_error(sb, "dquota write failed"); + if (ret >= 0) + ret = -ENOSPC; + } else { + ret = 0; + } + dqstats_inc(DQST_WRITES); + kfree(ddquot); + + return ret; +} +EXPORT_SYMBOL(qtree_write_dquot); + +/* Free dquot entry in data block */ +static int free_dqentry(struct qtree_mem_dqinfo *info, struct dquot *dquot, + uint blk) +{ + struct qt_disk_dqdbheader *dh; + char *buf = kmalloc(info->dqi_usable_bs, GFP_KERNEL); + int ret = 0; + + if (!buf) + return -ENOMEM; + if (dquot->dq_off >> info->dqi_blocksize_bits != blk) { + quota_error(dquot->dq_sb, "Quota structure has offset to " + "other block (%u) than it should (%u)", blk, + (uint)(dquot->dq_off >> info->dqi_blocksize_bits)); + ret = -EIO; + goto out_buf; + } + ret = read_blk(info, blk, buf); + if (ret < 0) { + quota_error(dquot->dq_sb, "Can't read quota data block %u", + blk); + goto out_buf; + } + dh = (struct qt_disk_dqdbheader *)buf; + ret = check_dquot_block_header(info, dh); + if (ret) + goto out_buf; + le16_add_cpu(&dh->dqdh_entries, -1); + if (!le16_to_cpu(dh->dqdh_entries)) { /* Block got free? */ + ret = remove_free_dqentry(info, buf, blk); + if (ret >= 0) + ret = put_free_dqblk(info, buf, blk); + if (ret < 0) { + quota_error(dquot->dq_sb, "Can't move quota data block " + "(%u) to free list", blk); + goto out_buf; + } + } else { + memset(buf + + (dquot->dq_off & ((1 << info->dqi_blocksize_bits) - 1)), + 0, info->dqi_entry_size); + if (le16_to_cpu(dh->dqdh_entries) == + qtree_dqstr_in_blk(info) - 1) { + /* Insert will write block itself */ + ret = insert_free_dqentry(info, buf, blk); + if (ret < 0) { + quota_error(dquot->dq_sb, "Can't insert quota " + "data block (%u) to free entry list", blk); + goto out_buf; + } + } else { + ret = write_blk(info, blk, buf); + if (ret < 0) { + quota_error(dquot->dq_sb, "Can't write quota " + "data block %u", blk); + goto out_buf; + } + } + } + dquot->dq_off = 0; /* Quota is now unattached */ +out_buf: + kfree(buf); + return ret; +} + +/* Remove reference to dquot from tree */ +static int remove_tree(struct qtree_mem_dqinfo *info, struct dquot *dquot, + uint *blks, int depth) +{ + char *buf = kmalloc(info->dqi_usable_bs, GFP_KERNEL); + int ret = 0; + uint newblk; + __le32 *ref = (__le32 *)buf; + int i; + + if (!buf) + return -ENOMEM; + ret = read_blk(info, blks[depth], buf); + if (ret < 0) { + quota_error(dquot->dq_sb, "Can't read quota data block %u", + blks[depth]); + goto out_buf; + } + newblk = le32_to_cpu(ref[get_index(info, dquot->dq_id, depth)]); + ret = do_check_range(dquot->dq_sb, "block", newblk, QT_TREEOFF, + info->dqi_blocks - 1); + if (ret) + goto out_buf; + + for (i = 0; i <= depth; i++) + if (newblk == blks[i]) { + quota_error(dquot->dq_sb, + "Cycle in quota tree detected: block %u index %u", + blks[depth], + get_index(info, dquot->dq_id, depth)); + ret = -EIO; + goto out_buf; + } + if (depth == info->dqi_qtree_depth - 1) { + ret = free_dqentry(info, dquot, newblk); + blks[depth + 1] = 0; + } else { + blks[depth + 1] = newblk; + ret = remove_tree(info, dquot, blks, depth + 1); + } + if (ret >= 0 && !blks[depth + 1]) { + ref[get_index(info, dquot->dq_id, depth)] = cpu_to_le32(0); + /* Block got empty? */ + for (i = 0; i < (info->dqi_usable_bs >> 2) && !ref[i]; i++) + ; + /* Don't put the root block into the free block list */ + if (i == (info->dqi_usable_bs >> 2) + && blks[depth] != QT_TREEOFF) { + put_free_dqblk(info, buf, blks[depth]); + blks[depth] = 0; + } else { + ret = write_blk(info, blks[depth], buf); + if (ret < 0) + quota_error(dquot->dq_sb, + "Can't write quota tree block %u", + blks[depth]); + } + } +out_buf: + kfree(buf); + return ret; +} + +/* Delete dquot from tree */ +int qtree_delete_dquot(struct qtree_mem_dqinfo *info, struct dquot *dquot) +{ + uint blks[MAX_QTREE_DEPTH] = { QT_TREEOFF }; + + if (!dquot->dq_off) /* Even not allocated? */ + return 0; + if (info->dqi_qtree_depth >= MAX_QTREE_DEPTH) { + quota_error(dquot->dq_sb, "Quota tree depth too big!"); + return -EIO; + } + return remove_tree(info, dquot, blks, 0); +} +EXPORT_SYMBOL(qtree_delete_dquot); + +/* Find entry in block */ +static loff_t find_block_dqentry(struct qtree_mem_dqinfo *info, + struct dquot *dquot, uint blk) +{ + char *buf = kmalloc(info->dqi_usable_bs, GFP_KERNEL); + loff_t ret = 0; + int i; + char *ddquot; + + if (!buf) + return -ENOMEM; + ret = read_blk(info, blk, buf); + if (ret < 0) { + quota_error(dquot->dq_sb, "Can't read quota tree " + "block %u", blk); + goto out_buf; + } + ddquot = buf + sizeof(struct qt_disk_dqdbheader); + for (i = 0; i < qtree_dqstr_in_blk(info); i++) { + if (info->dqi_ops->is_id(ddquot, dquot)) + break; + ddquot += info->dqi_entry_size; + } + if (i == qtree_dqstr_in_blk(info)) { + quota_error(dquot->dq_sb, + "Quota for id %u referenced but not present", + from_kqid(&init_user_ns, dquot->dq_id)); + ret = -EIO; + goto out_buf; + } else { + ret = ((loff_t)blk << info->dqi_blocksize_bits) + sizeof(struct + qt_disk_dqdbheader) + i * info->dqi_entry_size; + } +out_buf: + kfree(buf); + return ret; +} + +/* Find entry for given id in the tree */ +static loff_t find_tree_dqentry(struct qtree_mem_dqinfo *info, + struct dquot *dquot, uint *blks, int depth) +{ + char *buf = kmalloc(info->dqi_usable_bs, GFP_KERNEL); + loff_t ret = 0; + __le32 *ref = (__le32 *)buf; + uint blk; + int i; + + if (!buf) + return -ENOMEM; + ret = read_blk(info, blks[depth], buf); + if (ret < 0) { + quota_error(dquot->dq_sb, "Can't read quota tree block %u", + blks[depth]); + goto out_buf; + } + ret = 0; + blk = le32_to_cpu(ref[get_index(info, dquot->dq_id, depth)]); + if (!blk) /* No reference? */ + goto out_buf; + ret = do_check_range(dquot->dq_sb, "block", blk, QT_TREEOFF, + info->dqi_blocks - 1); + if (ret) + goto out_buf; + + /* Check for cycles in the tree */ + for (i = 0; i <= depth; i++) + if (blk == blks[i]) { + quota_error(dquot->dq_sb, + "Cycle in quota tree detected: block %u index %u", + blks[depth], + get_index(info, dquot->dq_id, depth)); + ret = -EIO; + goto out_buf; + } + blks[depth + 1] = blk; + if (depth < info->dqi_qtree_depth - 1) + ret = find_tree_dqentry(info, dquot, blks, depth + 1); + else + ret = find_block_dqentry(info, dquot, blk); +out_buf: + kfree(buf); + return ret; +} + +/* Find entry for given id in the tree - wrapper function */ +static inline loff_t find_dqentry(struct qtree_mem_dqinfo *info, + struct dquot *dquot) +{ + uint blks[MAX_QTREE_DEPTH] = { QT_TREEOFF }; + + if (info->dqi_qtree_depth >= MAX_QTREE_DEPTH) { + quota_error(dquot->dq_sb, "Quota tree depth too big!"); + return -EIO; + } + return find_tree_dqentry(info, dquot, blks, 0); +} + +int qtree_read_dquot(struct qtree_mem_dqinfo *info, struct dquot *dquot) +{ + int type = dquot->dq_id.type; + struct super_block *sb = dquot->dq_sb; + loff_t offset; + char *ddquot; + int ret = 0; + +#ifdef __QUOTA_QT_PARANOIA + /* Invalidated quota? */ + if (!sb_dqopt(dquot->dq_sb)->files[type]) { + quota_error(sb, "Quota invalidated while reading!"); + return -EIO; + } +#endif + /* Do we know offset of the dquot entry in the quota file? */ + if (!dquot->dq_off) { + offset = find_dqentry(info, dquot); + if (offset <= 0) { /* Entry not present? */ + if (offset < 0) + quota_error(sb,"Can't read quota structure " + "for id %u", + from_kqid(&init_user_ns, + dquot->dq_id)); + dquot->dq_off = 0; + set_bit(DQ_FAKE_B, &dquot->dq_flags); + memset(&dquot->dq_dqb, 0, sizeof(struct mem_dqblk)); + ret = offset; + goto out; + } + dquot->dq_off = offset; + } + ddquot = kmalloc(info->dqi_entry_size, GFP_KERNEL); + if (!ddquot) + return -ENOMEM; + ret = sb->s_op->quota_read(sb, type, ddquot, info->dqi_entry_size, + dquot->dq_off); + if (ret != info->dqi_entry_size) { + if (ret >= 0) + ret = -EIO; + quota_error(sb, "Error while reading quota structure for id %u", + from_kqid(&init_user_ns, dquot->dq_id)); + set_bit(DQ_FAKE_B, &dquot->dq_flags); + memset(&dquot->dq_dqb, 0, sizeof(struct mem_dqblk)); + kfree(ddquot); + goto out; + } + spin_lock(&dquot->dq_dqb_lock); + info->dqi_ops->disk2mem_dqblk(dquot, ddquot); + if (!dquot->dq_dqb.dqb_bhardlimit && + !dquot->dq_dqb.dqb_bsoftlimit && + !dquot->dq_dqb.dqb_ihardlimit && + !dquot->dq_dqb.dqb_isoftlimit) + set_bit(DQ_FAKE_B, &dquot->dq_flags); + spin_unlock(&dquot->dq_dqb_lock); + kfree(ddquot); +out: + dqstats_inc(DQST_READS); + return ret; +} +EXPORT_SYMBOL(qtree_read_dquot); + +/* Check whether dquot should not be deleted. We know we are + * the only one operating on dquot (thanks to dq_lock) */ +int qtree_release_dquot(struct qtree_mem_dqinfo *info, struct dquot *dquot) +{ + if (test_bit(DQ_FAKE_B, &dquot->dq_flags) && + !(dquot->dq_dqb.dqb_curinodes | dquot->dq_dqb.dqb_curspace)) + return qtree_delete_dquot(info, dquot); + return 0; +} +EXPORT_SYMBOL(qtree_release_dquot); + +static int find_next_id(struct qtree_mem_dqinfo *info, qid_t *id, + unsigned int blk, int depth) +{ + char *buf = kmalloc(info->dqi_usable_bs, GFP_KERNEL); + __le32 *ref = (__le32 *)buf; + ssize_t ret; + unsigned int epb = info->dqi_usable_bs >> 2; + unsigned int level_inc = 1; + int i; + + if (!buf) + return -ENOMEM; + + for (i = depth; i < info->dqi_qtree_depth - 1; i++) + level_inc *= epb; + + ret = read_blk(info, blk, buf); + if (ret < 0) { + quota_error(info->dqi_sb, + "Can't read quota tree block %u", blk); + goto out_buf; + } + for (i = __get_index(info, *id, depth); i < epb; i++) { + uint blk_no = le32_to_cpu(ref[i]); + + if (blk_no == 0) { + *id += level_inc; + continue; + } + ret = do_check_range(info->dqi_sb, "block", blk_no, 0, + info->dqi_blocks - 1); + if (ret) + goto out_buf; + if (depth == info->dqi_qtree_depth - 1) { + ret = 0; + goto out_buf; + } + ret = find_next_id(info, id, blk_no, depth + 1); + if (ret != -ENOENT) + break; + } + if (i == epb) { + ret = -ENOENT; + goto out_buf; + } +out_buf: + kfree(buf); + return ret; +} + +int qtree_get_next_id(struct qtree_mem_dqinfo *info, struct kqid *qid) +{ + qid_t id = from_kqid(&init_user_ns, *qid); + int ret; + + ret = find_next_id(info, &id, QT_TREEOFF, 0); + if (ret < 0) + return ret; + *qid = make_kqid(&init_user_ns, qid->type, id); + return 0; +} +EXPORT_SYMBOL(qtree_get_next_id); diff --git a/fs/quota/quota_tree.h b/fs/quota/quota_tree.h new file mode 100644 index 000000000..31cf27e0e --- /dev/null +++ b/fs/quota/quota_tree.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Definitions of structures for vfsv0 quota format + */ + +#ifndef _LINUX_QUOTA_TREE_H +#define _LINUX_QUOTA_TREE_H + +#include +#include + +/* + * Structure of header of block with quota structures. It is padded to 16 bytes so + * there will be space for exactly 21 quota-entries in a block + */ +struct qt_disk_dqdbheader { + __le32 dqdh_next_free; /* Number of next block with free entry */ + __le32 dqdh_prev_free; /* Number of previous block with free entry */ + __le16 dqdh_entries; /* Number of valid entries in block */ + __le16 dqdh_pad1; + __le32 dqdh_pad2; +}; + +#define QT_TREEOFF 1 /* Offset of tree in file in blocks */ + +#endif /* _LINUX_QUOTAIO_TREE_H */ diff --git a/fs/quota/quota_v1.c b/fs/quota/quota_v1.c new file mode 100644 index 000000000..6f7f0b4af --- /dev/null +++ b/fs/quota/quota_v1.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "quotaio_v1.h" + +MODULE_AUTHOR("Jan Kara"); +MODULE_DESCRIPTION("Old quota format support"); +MODULE_LICENSE("GPL"); + +#define QUOTABLOCK_BITS 10 +#define QUOTABLOCK_SIZE (1 << QUOTABLOCK_BITS) + +static inline qsize_t v1_stoqb(qsize_t space) +{ + return (space + QUOTABLOCK_SIZE - 1) >> QUOTABLOCK_BITS; +} + +static inline qsize_t v1_qbtos(qsize_t blocks) +{ + return blocks << QUOTABLOCK_BITS; +} + +static void v1_disk2mem_dqblk(struct mem_dqblk *m, struct v1_disk_dqblk *d) +{ + m->dqb_ihardlimit = d->dqb_ihardlimit; + m->dqb_isoftlimit = d->dqb_isoftlimit; + m->dqb_curinodes = d->dqb_curinodes; + m->dqb_bhardlimit = v1_qbtos(d->dqb_bhardlimit); + m->dqb_bsoftlimit = v1_qbtos(d->dqb_bsoftlimit); + m->dqb_curspace = v1_qbtos(d->dqb_curblocks); + m->dqb_itime = d->dqb_itime; + m->dqb_btime = d->dqb_btime; +} + +static void v1_mem2disk_dqblk(struct v1_disk_dqblk *d, struct mem_dqblk *m) +{ + d->dqb_ihardlimit = m->dqb_ihardlimit; + d->dqb_isoftlimit = m->dqb_isoftlimit; + d->dqb_curinodes = m->dqb_curinodes; + d->dqb_bhardlimit = v1_stoqb(m->dqb_bhardlimit); + d->dqb_bsoftlimit = v1_stoqb(m->dqb_bsoftlimit); + d->dqb_curblocks = v1_stoqb(m->dqb_curspace); + d->dqb_itime = m->dqb_itime; + d->dqb_btime = m->dqb_btime; +} + +static int v1_read_dqblk(struct dquot *dquot) +{ + int type = dquot->dq_id.type; + struct v1_disk_dqblk dqblk; + struct quota_info *dqopt = sb_dqopt(dquot->dq_sb); + + if (!dqopt->files[type]) + return -EINVAL; + + /* Set structure to 0s in case read fails/is after end of file */ + memset(&dqblk, 0, sizeof(struct v1_disk_dqblk)); + dquot->dq_sb->s_op->quota_read(dquot->dq_sb, type, (char *)&dqblk, + sizeof(struct v1_disk_dqblk), + v1_dqoff(from_kqid(&init_user_ns, dquot->dq_id))); + + v1_disk2mem_dqblk(&dquot->dq_dqb, &dqblk); + if (dquot->dq_dqb.dqb_bhardlimit == 0 && + dquot->dq_dqb.dqb_bsoftlimit == 0 && + dquot->dq_dqb.dqb_ihardlimit == 0 && + dquot->dq_dqb.dqb_isoftlimit == 0) + set_bit(DQ_FAKE_B, &dquot->dq_flags); + dqstats_inc(DQST_READS); + + return 0; +} + +static int v1_commit_dqblk(struct dquot *dquot) +{ + short type = dquot->dq_id.type; + ssize_t ret; + struct v1_disk_dqblk dqblk; + + v1_mem2disk_dqblk(&dqblk, &dquot->dq_dqb); + if (((type == USRQUOTA) && uid_eq(dquot->dq_id.uid, GLOBAL_ROOT_UID)) || + ((type == GRPQUOTA) && gid_eq(dquot->dq_id.gid, GLOBAL_ROOT_GID))) { + dqblk.dqb_btime = + sb_dqopt(dquot->dq_sb)->info[type].dqi_bgrace; + dqblk.dqb_itime = + sb_dqopt(dquot->dq_sb)->info[type].dqi_igrace; + } + ret = 0; + if (sb_dqopt(dquot->dq_sb)->files[type]) + ret = dquot->dq_sb->s_op->quota_write(dquot->dq_sb, type, + (char *)&dqblk, sizeof(struct v1_disk_dqblk), + v1_dqoff(from_kqid(&init_user_ns, dquot->dq_id))); + if (ret != sizeof(struct v1_disk_dqblk)) { + quota_error(dquot->dq_sb, "dquota write failed"); + if (ret >= 0) + ret = -EIO; + goto out; + } + ret = 0; + +out: + dqstats_inc(DQST_WRITES); + + return ret; +} + +/* Magics of new quota format */ +#define V2_INITQMAGICS {\ + 0xd9c01f11, /* USRQUOTA */\ + 0xd9c01927 /* GRPQUOTA */\ +} + +/* Header of new quota format */ +struct v2_disk_dqheader { + __le32 dqh_magic; /* Magic number identifying file */ + __le32 dqh_version; /* File version */ +}; + +static int v1_check_quota_file(struct super_block *sb, int type) +{ + struct inode *inode = sb_dqopt(sb)->files[type]; + ulong blocks; + size_t off; + struct v2_disk_dqheader dqhead; + ssize_t size; + loff_t isize; + static const uint quota_magics[] = V2_INITQMAGICS; + + isize = i_size_read(inode); + if (!isize) + return 0; + blocks = isize >> BLOCK_SIZE_BITS; + off = isize & (BLOCK_SIZE - 1); + if ((blocks % sizeof(struct v1_disk_dqblk) * BLOCK_SIZE + off) % + sizeof(struct v1_disk_dqblk)) + return 0; + /* Doublecheck whether we didn't get file with new format - with old + * quotactl() this could happen */ + size = sb->s_op->quota_read(sb, type, (char *)&dqhead, + sizeof(struct v2_disk_dqheader), 0); + if (size != sizeof(struct v2_disk_dqheader)) + return 1; /* Probably not new format */ + if (le32_to_cpu(dqhead.dqh_magic) != quota_magics[type]) + return 1; /* Definitely not new format */ + printk(KERN_INFO + "VFS: %s: Refusing to turn on old quota format on given file." + " It probably contains newer quota format.\n", sb->s_id); + return 0; /* Seems like a new format file -> refuse it */ +} + +static int v1_read_file_info(struct super_block *sb, int type) +{ + struct quota_info *dqopt = sb_dqopt(sb); + struct v1_disk_dqblk dqblk; + unsigned int memalloc; + int ret; + + down_read(&dqopt->dqio_sem); + memalloc = memalloc_nofs_save(); + ret = sb->s_op->quota_read(sb, type, (char *)&dqblk, + sizeof(struct v1_disk_dqblk), v1_dqoff(0)); + if (ret != sizeof(struct v1_disk_dqblk)) { + if (ret >= 0) + ret = -EIO; + goto out; + } + ret = 0; + /* limits are stored as unsigned 32-bit data */ + dqopt->info[type].dqi_max_spc_limit = 0xffffffffULL << QUOTABLOCK_BITS; + dqopt->info[type].dqi_max_ino_limit = 0xffffffff; + dqopt->info[type].dqi_igrace = + dqblk.dqb_itime ? dqblk.dqb_itime : MAX_IQ_TIME; + dqopt->info[type].dqi_bgrace = + dqblk.dqb_btime ? dqblk.dqb_btime : MAX_DQ_TIME; +out: + memalloc_nofs_restore(memalloc); + up_read(&dqopt->dqio_sem); + return ret; +} + +static int v1_write_file_info(struct super_block *sb, int type) +{ + struct quota_info *dqopt = sb_dqopt(sb); + struct v1_disk_dqblk dqblk; + unsigned int memalloc; + int ret; + + down_write(&dqopt->dqio_sem); + memalloc = memalloc_nofs_save(); + ret = sb->s_op->quota_read(sb, type, (char *)&dqblk, + sizeof(struct v1_disk_dqblk), v1_dqoff(0)); + if (ret != sizeof(struct v1_disk_dqblk)) { + if (ret >= 0) + ret = -EIO; + goto out; + } + spin_lock(&dq_data_lock); + dqopt->info[type].dqi_flags &= ~DQF_INFO_DIRTY; + dqblk.dqb_itime = dqopt->info[type].dqi_igrace; + dqblk.dqb_btime = dqopt->info[type].dqi_bgrace; + spin_unlock(&dq_data_lock); + ret = sb->s_op->quota_write(sb, type, (char *)&dqblk, + sizeof(struct v1_disk_dqblk), v1_dqoff(0)); + if (ret == sizeof(struct v1_disk_dqblk)) + ret = 0; + else if (ret >= 0) + ret = -EIO; +out: + memalloc_nofs_restore(memalloc); + up_write(&dqopt->dqio_sem); + return ret; +} + +static const struct quota_format_ops v1_format_ops = { + .check_quota_file = v1_check_quota_file, + .read_file_info = v1_read_file_info, + .write_file_info = v1_write_file_info, + .read_dqblk = v1_read_dqblk, + .commit_dqblk = v1_commit_dqblk, +}; + +static struct quota_format_type v1_quota_format = { + .qf_fmt_id = QFMT_VFS_OLD, + .qf_ops = &v1_format_ops, + .qf_owner = THIS_MODULE +}; + +static int __init init_v1_quota_format(void) +{ + register_quota_format(&v1_quota_format); + return 0; +} + +static void __exit exit_v1_quota_format(void) +{ + unregister_quota_format(&v1_quota_format); +} + +module_init(init_v1_quota_format); +module_exit(exit_v1_quota_format); + diff --git a/fs/quota/quota_v2.c b/fs/quota/quota_v2.c new file mode 100644 index 000000000..a24fab5f9 --- /dev/null +++ b/fs/quota/quota_v2.c @@ -0,0 +1,455 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * vfsv0 quota IO operations on file + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "quota_tree.h" +#include "quotaio_v2.h" + +MODULE_AUTHOR("Jan Kara"); +MODULE_DESCRIPTION("Quota format v2 support"); +MODULE_LICENSE("GPL"); + +static void v2r0_mem2diskdqb(void *dp, struct dquot *dquot); +static void v2r0_disk2memdqb(struct dquot *dquot, void *dp); +static int v2r0_is_id(void *dp, struct dquot *dquot); +static void v2r1_mem2diskdqb(void *dp, struct dquot *dquot); +static void v2r1_disk2memdqb(struct dquot *dquot, void *dp); +static int v2r1_is_id(void *dp, struct dquot *dquot); + +static const struct qtree_fmt_operations v2r0_qtree_ops = { + .mem2disk_dqblk = v2r0_mem2diskdqb, + .disk2mem_dqblk = v2r0_disk2memdqb, + .is_id = v2r0_is_id, +}; + +static const struct qtree_fmt_operations v2r1_qtree_ops = { + .mem2disk_dqblk = v2r1_mem2diskdqb, + .disk2mem_dqblk = v2r1_disk2memdqb, + .is_id = v2r1_is_id, +}; + +#define QUOTABLOCK_BITS 10 +#define QUOTABLOCK_SIZE (1 << QUOTABLOCK_BITS) + +static inline qsize_t v2_stoqb(qsize_t space) +{ + return (space + QUOTABLOCK_SIZE - 1) >> QUOTABLOCK_BITS; +} + +static inline qsize_t v2_qbtos(qsize_t blocks) +{ + return blocks << QUOTABLOCK_BITS; +} + +static int v2_read_header(struct super_block *sb, int type, + struct v2_disk_dqheader *dqhead) +{ + ssize_t size; + + size = sb->s_op->quota_read(sb, type, (char *)dqhead, + sizeof(struct v2_disk_dqheader), 0); + if (size != sizeof(struct v2_disk_dqheader)) { + quota_error(sb, "Failed header read: expected=%zd got=%zd", + sizeof(struct v2_disk_dqheader), size); + if (size < 0) + return size; + return -EIO; + } + return 0; +} + +/* Check whether given file is really vfsv0 quotafile */ +static int v2_check_quota_file(struct super_block *sb, int type) +{ + struct v2_disk_dqheader dqhead; + static const uint quota_magics[] = V2_INITQMAGICS; + static const uint quota_versions[] = V2_INITQVERSIONS; + + if (v2_read_header(sb, type, &dqhead)) + return 0; + if (le32_to_cpu(dqhead.dqh_magic) != quota_magics[type] || + le32_to_cpu(dqhead.dqh_version) > quota_versions[type]) + return 0; + return 1; +} + +/* Read information header from quota file */ +static int v2_read_file_info(struct super_block *sb, int type) +{ + struct v2_disk_dqinfo dinfo; + struct v2_disk_dqheader dqhead; + struct quota_info *dqopt = sb_dqopt(sb); + struct mem_dqinfo *info = &dqopt->info[type]; + struct qtree_mem_dqinfo *qinfo; + ssize_t size; + unsigned int version; + unsigned int memalloc; + int ret; + + down_read(&dqopt->dqio_sem); + memalloc = memalloc_nofs_save(); + ret = v2_read_header(sb, type, &dqhead); + if (ret < 0) + goto out; + version = le32_to_cpu(dqhead.dqh_version); + if ((info->dqi_fmt_id == QFMT_VFS_V0 && version != 0) || + (info->dqi_fmt_id == QFMT_VFS_V1 && version != 1)) { + ret = -EINVAL; + goto out; + } + + size = sb->s_op->quota_read(sb, type, (char *)&dinfo, + sizeof(struct v2_disk_dqinfo), V2_DQINFOOFF); + if (size != sizeof(struct v2_disk_dqinfo)) { + quota_error(sb, "Can't read info structure"); + if (size < 0) + ret = size; + else + ret = -EIO; + goto out; + } + info->dqi_priv = kmalloc_obj(struct qtree_mem_dqinfo); + if (!info->dqi_priv) { + ret = -ENOMEM; + goto out; + } + qinfo = info->dqi_priv; + if (version == 0) { + /* limits are stored as unsigned 32-bit data */ + info->dqi_max_spc_limit = 0xffffffffLL << QUOTABLOCK_BITS; + info->dqi_max_ino_limit = 0xffffffff; + } else { + /* + * Used space is stored as unsigned 64-bit value in bytes but + * quota core supports only signed 64-bit values so use that + * as a limit + */ + info->dqi_max_spc_limit = 0x7fffffffffffffffLL; /* 2^63-1 */ + info->dqi_max_ino_limit = 0x7fffffffffffffffLL; + } + info->dqi_bgrace = le32_to_cpu(dinfo.dqi_bgrace); + info->dqi_igrace = le32_to_cpu(dinfo.dqi_igrace); + /* No flags currently supported */ + info->dqi_flags = 0; + qinfo->dqi_sb = sb; + qinfo->dqi_type = type; + qinfo->dqi_blocks = le32_to_cpu(dinfo.dqi_blocks); + qinfo->dqi_free_blk = le32_to_cpu(dinfo.dqi_free_blk); + qinfo->dqi_free_entry = le32_to_cpu(dinfo.dqi_free_entry); + qinfo->dqi_blocksize_bits = V2_DQBLKSIZE_BITS; + qinfo->dqi_usable_bs = 1 << V2_DQBLKSIZE_BITS; + qinfo->dqi_qtree_depth = qtree_depth(qinfo); + if (version == 0) { + qinfo->dqi_entry_size = sizeof(struct v2r0_disk_dqblk); + qinfo->dqi_ops = &v2r0_qtree_ops; + } else { + qinfo->dqi_entry_size = sizeof(struct v2r1_disk_dqblk); + qinfo->dqi_ops = &v2r1_qtree_ops; + } + ret = -EUCLEAN; + /* Some sanity checks of the read headers... */ + if ((loff_t)qinfo->dqi_blocks << qinfo->dqi_blocksize_bits > + i_size_read(sb_dqopt(sb)->files[type])) { + quota_error(sb, "Number of blocks too big for quota file size (%llu > %llu).", + (loff_t)qinfo->dqi_blocks << qinfo->dqi_blocksize_bits, + i_size_read(sb_dqopt(sb)->files[type])); + goto out_free; + } + if (qinfo->dqi_free_blk && (qinfo->dqi_free_blk <= QT_TREEOFF || + qinfo->dqi_free_blk >= qinfo->dqi_blocks)) { + quota_error(sb, "Free block number %u out of range (%u, %u).", + qinfo->dqi_free_blk, QT_TREEOFF, qinfo->dqi_blocks); + goto out_free; + } + if (qinfo->dqi_free_entry && (qinfo->dqi_free_entry <= QT_TREEOFF || + qinfo->dqi_free_entry >= qinfo->dqi_blocks)) { + quota_error(sb, "Block with free entry %u out of range (%u, %u).", + qinfo->dqi_free_entry, QT_TREEOFF, + qinfo->dqi_blocks); + goto out_free; + } + ret = 0; +out_free: + if (ret) { + kfree(info->dqi_priv); + info->dqi_priv = NULL; + } +out: + memalloc_nofs_restore(memalloc); + up_read(&dqopt->dqio_sem); + return ret; +} + +/* Write information header to quota file */ +static int v2_write_file_info(struct super_block *sb, int type) +{ + struct v2_disk_dqinfo dinfo; + struct quota_info *dqopt = sb_dqopt(sb); + struct mem_dqinfo *info = &dqopt->info[type]; + struct qtree_mem_dqinfo *qinfo = info->dqi_priv; + ssize_t size; + unsigned int memalloc; + + down_write(&dqopt->dqio_sem); + memalloc = memalloc_nofs_save(); + spin_lock(&dq_data_lock); + info->dqi_flags &= ~DQF_INFO_DIRTY; + dinfo.dqi_bgrace = cpu_to_le32(info->dqi_bgrace); + dinfo.dqi_igrace = cpu_to_le32(info->dqi_igrace); + /* No flags currently supported */ + dinfo.dqi_flags = cpu_to_le32(0); + spin_unlock(&dq_data_lock); + dinfo.dqi_blocks = cpu_to_le32(qinfo->dqi_blocks); + dinfo.dqi_free_blk = cpu_to_le32(qinfo->dqi_free_blk); + dinfo.dqi_free_entry = cpu_to_le32(qinfo->dqi_free_entry); + size = sb->s_op->quota_write(sb, type, (char *)&dinfo, + sizeof(struct v2_disk_dqinfo), V2_DQINFOOFF); + memalloc_nofs_restore(memalloc); + up_write(&dqopt->dqio_sem); + if (size != sizeof(struct v2_disk_dqinfo)) { + quota_error(sb, "Can't write info structure"); + return size < 0 ? size : -EIO; + } + return 0; +} + +static void v2r0_disk2memdqb(struct dquot *dquot, void *dp) +{ + struct v2r0_disk_dqblk *d = dp, empty; + struct mem_dqblk *m = &dquot->dq_dqb; + + m->dqb_ihardlimit = le32_to_cpu(d->dqb_ihardlimit); + m->dqb_isoftlimit = le32_to_cpu(d->dqb_isoftlimit); + m->dqb_curinodes = le32_to_cpu(d->dqb_curinodes); + m->dqb_itime = le64_to_cpu(d->dqb_itime); + m->dqb_bhardlimit = v2_qbtos(le32_to_cpu(d->dqb_bhardlimit)); + m->dqb_bsoftlimit = v2_qbtos(le32_to_cpu(d->dqb_bsoftlimit)); + m->dqb_curspace = le64_to_cpu(d->dqb_curspace); + m->dqb_btime = le64_to_cpu(d->dqb_btime); + /* We need to escape back all-zero structure */ + memset(&empty, 0, sizeof(struct v2r0_disk_dqblk)); + empty.dqb_itime = cpu_to_le64(1); + if (!memcmp(&empty, dp, sizeof(struct v2r0_disk_dqblk))) + m->dqb_itime = 0; +} + +static void v2r0_mem2diskdqb(void *dp, struct dquot *dquot) +{ + struct v2r0_disk_dqblk *d = dp; + struct mem_dqblk *m = &dquot->dq_dqb; + struct qtree_mem_dqinfo *info = + sb_dqinfo(dquot->dq_sb, dquot->dq_id.type)->dqi_priv; + + d->dqb_ihardlimit = cpu_to_le32(m->dqb_ihardlimit); + d->dqb_isoftlimit = cpu_to_le32(m->dqb_isoftlimit); + d->dqb_curinodes = cpu_to_le32(m->dqb_curinodes); + d->dqb_itime = cpu_to_le64(m->dqb_itime); + d->dqb_bhardlimit = cpu_to_le32(v2_stoqb(m->dqb_bhardlimit)); + d->dqb_bsoftlimit = cpu_to_le32(v2_stoqb(m->dqb_bsoftlimit)); + d->dqb_curspace = cpu_to_le64(m->dqb_curspace); + d->dqb_btime = cpu_to_le64(m->dqb_btime); + d->dqb_id = cpu_to_le32(from_kqid(&init_user_ns, dquot->dq_id)); + if (qtree_entry_unused(info, dp)) + d->dqb_itime = cpu_to_le64(1); +} + +static int v2r0_is_id(void *dp, struct dquot *dquot) +{ + struct v2r0_disk_dqblk *d = dp; + struct qtree_mem_dqinfo *info = + sb_dqinfo(dquot->dq_sb, dquot->dq_id.type)->dqi_priv; + + if (qtree_entry_unused(info, dp)) + return 0; + return qid_eq(make_kqid(&init_user_ns, dquot->dq_id.type, + le32_to_cpu(d->dqb_id)), + dquot->dq_id); +} + +static void v2r1_disk2memdqb(struct dquot *dquot, void *dp) +{ + struct v2r1_disk_dqblk *d = dp, empty; + struct mem_dqblk *m = &dquot->dq_dqb; + + m->dqb_ihardlimit = le64_to_cpu(d->dqb_ihardlimit); + m->dqb_isoftlimit = le64_to_cpu(d->dqb_isoftlimit); + m->dqb_curinodes = le64_to_cpu(d->dqb_curinodes); + m->dqb_itime = le64_to_cpu(d->dqb_itime); + m->dqb_bhardlimit = v2_qbtos(le64_to_cpu(d->dqb_bhardlimit)); + m->dqb_bsoftlimit = v2_qbtos(le64_to_cpu(d->dqb_bsoftlimit)); + m->dqb_curspace = le64_to_cpu(d->dqb_curspace); + m->dqb_btime = le64_to_cpu(d->dqb_btime); + /* We need to escape back all-zero structure */ + memset(&empty, 0, sizeof(struct v2r1_disk_dqblk)); + empty.dqb_itime = cpu_to_le64(1); + if (!memcmp(&empty, dp, sizeof(struct v2r1_disk_dqblk))) + m->dqb_itime = 0; +} + +static void v2r1_mem2diskdqb(void *dp, struct dquot *dquot) +{ + struct v2r1_disk_dqblk *d = dp; + struct mem_dqblk *m = &dquot->dq_dqb; + struct qtree_mem_dqinfo *info = + sb_dqinfo(dquot->dq_sb, dquot->dq_id.type)->dqi_priv; + + d->dqb_ihardlimit = cpu_to_le64(m->dqb_ihardlimit); + d->dqb_isoftlimit = cpu_to_le64(m->dqb_isoftlimit); + d->dqb_curinodes = cpu_to_le64(m->dqb_curinodes); + d->dqb_itime = cpu_to_le64(m->dqb_itime); + d->dqb_bhardlimit = cpu_to_le64(v2_stoqb(m->dqb_bhardlimit)); + d->dqb_bsoftlimit = cpu_to_le64(v2_stoqb(m->dqb_bsoftlimit)); + d->dqb_curspace = cpu_to_le64(m->dqb_curspace); + d->dqb_btime = cpu_to_le64(m->dqb_btime); + d->dqb_id = cpu_to_le32(from_kqid(&init_user_ns, dquot->dq_id)); + d->dqb_pad = 0; + if (qtree_entry_unused(info, dp)) + d->dqb_itime = cpu_to_le64(1); +} + +static int v2r1_is_id(void *dp, struct dquot *dquot) +{ + struct v2r1_disk_dqblk *d = dp; + struct qtree_mem_dqinfo *info = + sb_dqinfo(dquot->dq_sb, dquot->dq_id.type)->dqi_priv; + + if (qtree_entry_unused(info, dp)) + return 0; + return qid_eq(make_kqid(&init_user_ns, dquot->dq_id.type, + le32_to_cpu(d->dqb_id)), + dquot->dq_id); +} + +static int v2_read_dquot(struct dquot *dquot) +{ + struct quota_info *dqopt = sb_dqopt(dquot->dq_sb); + int ret; + unsigned int memalloc; + + down_read(&dqopt->dqio_sem); + memalloc = memalloc_nofs_save(); + ret = qtree_read_dquot( + sb_dqinfo(dquot->dq_sb, dquot->dq_id.type)->dqi_priv, + dquot); + memalloc_nofs_restore(memalloc); + up_read(&dqopt->dqio_sem); + return ret; +} + +static int v2_write_dquot(struct dquot *dquot) +{ + struct quota_info *dqopt = sb_dqopt(dquot->dq_sb); + int ret; + bool alloc = false; + unsigned int memalloc; + + /* + * If space for dquot is already allocated, we don't need any + * protection as we'll only overwrite the place of dquot. We are + * still protected by concurrent writes of the same dquot by + * dquot->dq_lock. + */ + if (!dquot->dq_off) { + alloc = true; + down_write(&dqopt->dqio_sem); + } else { + down_read(&dqopt->dqio_sem); + } + memalloc = memalloc_nofs_save(); + ret = qtree_write_dquot( + sb_dqinfo(dquot->dq_sb, dquot->dq_id.type)->dqi_priv, + dquot); + memalloc_nofs_restore(memalloc); + if (alloc) + up_write(&dqopt->dqio_sem); + else + up_read(&dqopt->dqio_sem); + return ret; +} + +static int v2_release_dquot(struct dquot *dquot) +{ + struct quota_info *dqopt = sb_dqopt(dquot->dq_sb); + unsigned int memalloc; + int ret; + + down_write(&dqopt->dqio_sem); + memalloc = memalloc_nofs_save(); + ret = qtree_release_dquot(sb_dqinfo(dquot->dq_sb, dquot->dq_id.type)->dqi_priv, dquot); + memalloc_nofs_restore(memalloc); + up_write(&dqopt->dqio_sem); + + return ret; +} + +static int v2_free_file_info(struct super_block *sb, int type) +{ + kfree(sb_dqinfo(sb, type)->dqi_priv); + return 0; +} + +static int v2_get_next_id(struct super_block *sb, struct kqid *qid) +{ + struct quota_info *dqopt = sb_dqopt(sb); + unsigned int memalloc; + int ret; + + down_read(&dqopt->dqio_sem); + memalloc = memalloc_nofs_save(); + ret = qtree_get_next_id(sb_dqinfo(sb, qid->type)->dqi_priv, qid); + memalloc_nofs_restore(memalloc); + up_read(&dqopt->dqio_sem); + return ret; +} + +static const struct quota_format_ops v2_format_ops = { + .check_quota_file = v2_check_quota_file, + .read_file_info = v2_read_file_info, + .write_file_info = v2_write_file_info, + .free_file_info = v2_free_file_info, + .read_dqblk = v2_read_dquot, + .commit_dqblk = v2_write_dquot, + .release_dqblk = v2_release_dquot, + .get_next_id = v2_get_next_id, +}; + +static struct quota_format_type v2r0_quota_format = { + .qf_fmt_id = QFMT_VFS_V0, + .qf_ops = &v2_format_ops, + .qf_owner = THIS_MODULE +}; + +static struct quota_format_type v2r1_quota_format = { + .qf_fmt_id = QFMT_VFS_V1, + .qf_ops = &v2_format_ops, + .qf_owner = THIS_MODULE +}; + +static int __init init_v2_quota_format(void) +{ + register_quota_format(&v2r0_quota_format); + register_quota_format(&v2r1_quota_format); + return 0; +} + +static void __exit exit_v2_quota_format(void) +{ + unregister_quota_format(&v2r0_quota_format); + unregister_quota_format(&v2r1_quota_format); +} + +module_init(init_v2_quota_format); +module_exit(exit_v2_quota_format); diff --git a/fs/quota/quotaio_v1.h b/fs/quota/quotaio_v1.h new file mode 100644 index 000000000..31dca9a89 --- /dev/null +++ b/fs/quota/quotaio_v1.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_QUOTAIO_V1_H +#define _LINUX_QUOTAIO_V1_H + +#include + +/* + * The following constants define the amount of time given a user + * before the soft limits are treated as hard limits (usually resulting + * in an allocation failure). The timer is started when the user crosses + * their soft limit, it is reset when they go below their soft limit. + */ +#define MAX_IQ_TIME 604800 /* (7*24*60*60) 1 week */ +#define MAX_DQ_TIME 604800 /* (7*24*60*60) 1 week */ + +/* + * The following structure defines the format of the disk quota file + * (as it appears on disk) - the file is an array of these structures + * indexed by user or group number. + */ +struct v1_disk_dqblk { + __u32 dqb_bhardlimit; /* absolute limit on disk blks alloc */ + __u32 dqb_bsoftlimit; /* preferred limit on disk blks */ + __u32 dqb_curblocks; /* current block count */ + __u32 dqb_ihardlimit; /* absolute limit on allocated inodes */ + __u32 dqb_isoftlimit; /* preferred inode limit */ + __u32 dqb_curinodes; /* current # allocated inodes */ + + /* below fields differ in length on 32-bit vs 64-bit architectures */ + unsigned long dqb_btime; /* time limit for excessive disk use */ + unsigned long dqb_itime; /* time limit for excessive inode use */ +}; + +#define v1_dqoff(UID) ((loff_t)((UID) * sizeof (struct v1_disk_dqblk))) + +#endif /* _LINUX_QUOTAIO_V1_H */ diff --git a/fs/quota/quotaio_v2.h b/fs/quota/quotaio_v2.h new file mode 100644 index 000000000..43cf0f0e2 --- /dev/null +++ b/fs/quota/quotaio_v2.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Definitions of structures for vfsv0 quota format + */ + +#ifndef _LINUX_QUOTAIO_V2_H +#define _LINUX_QUOTAIO_V2_H + +#include +#include + +/* + * Definitions of magics and versions of current quota files + */ +#define V2_INITQMAGICS {\ + 0xd9c01f11, /* USRQUOTA */\ + 0xd9c01927, /* GRPQUOTA */\ + 0xd9c03f14, /* PRJQUOTA */\ +} + +#define V2_INITQVERSIONS {\ + 1, /* USRQUOTA */\ + 1, /* GRPQUOTA */\ + 1, /* PRJQUOTA */\ +} + +/* First generic header */ +struct v2_disk_dqheader { + __le32 dqh_magic; /* Magic number identifying file */ + __le32 dqh_version; /* File version */ +}; + +/* + * The following structure defines the format of the disk quota file + * (as it appears on disk) - the file is a radix tree whose leaves point + * to blocks of these structures. + */ +struct v2r0_disk_dqblk { + __le32 dqb_id; /* id this quota applies to */ + __le32 dqb_ihardlimit; /* absolute limit on allocated inodes */ + __le32 dqb_isoftlimit; /* preferred inode limit */ + __le32 dqb_curinodes; /* current # allocated inodes */ + __le32 dqb_bhardlimit; /* absolute limit on disk space (in QUOTABLOCK_SIZE) */ + __le32 dqb_bsoftlimit; /* preferred limit on disk space (in QUOTABLOCK_SIZE) */ + __le64 dqb_curspace; /* current space occupied (in bytes) */ + __le64 dqb_btime; /* time limit for excessive disk use */ + __le64 dqb_itime; /* time limit for excessive inode use */ +}; + +struct v2r1_disk_dqblk { + __le32 dqb_id; /* id this quota applies to */ + __le32 dqb_pad; + __le64 dqb_ihardlimit; /* absolute limit on allocated inodes */ + __le64 dqb_isoftlimit; /* preferred inode limit */ + __le64 dqb_curinodes; /* current # allocated inodes */ + __le64 dqb_bhardlimit; /* absolute limit on disk space (in QUOTABLOCK_SIZE) */ + __le64 dqb_bsoftlimit; /* preferred limit on disk space (in QUOTABLOCK_SIZE) */ + __le64 dqb_curspace; /* current space occupied (in bytes) */ + __le64 dqb_btime; /* time limit for excessive disk use */ + __le64 dqb_itime; /* time limit for excessive inode use */ +}; + +/* Header with type and version specific information */ +struct v2_disk_dqinfo { + __le32 dqi_bgrace; /* Time before block soft limit becomes hard limit */ + __le32 dqi_igrace; /* Time before inode soft limit becomes hard limit */ + __le32 dqi_flags; /* Flags for quotafile (DQF_*) */ + __le32 dqi_blocks; /* Number of blocks in file */ + __le32 dqi_free_blk; /* Number of first free block in the list */ + __le32 dqi_free_entry; /* Number of block with at least one free entry */ +}; + +#define V2_DQINFOOFF sizeof(struct v2_disk_dqheader) /* Offset of info header in file */ +#define V2_DQBLKSIZE_BITS 10 /* Size of leaf block in tree */ + +#endif /* _LINUX_QUOTAIO_V2_H */ diff --git a/fs/ramfs/Makefile b/fs/ramfs/Makefile new file mode 100644 index 000000000..d1198adb5 --- /dev/null +++ b/fs/ramfs/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the linux ramfs routines. +# + +obj-y += ramfs.o + +file-mmu-y := file-nommu.o +file-mmu-$(CONFIG_MMU) := file-mmu.o +ramfs-objs += inode.o $(file-mmu-y) diff --git a/fs/ramfs/file-mmu.c b/fs/ramfs/file-mmu.c new file mode 100644 index 000000000..c3ed1c511 --- /dev/null +++ b/fs/ramfs/file-mmu.c @@ -0,0 +1,55 @@ +/* file-mmu.c: ramfs MMU-based file operations + * + * Resizable simple ram filesystem for Linux. + * + * Copyright (C) 2000 Linus Torvalds. + * 2000 Transmeta Corp. + * + * Usage limits added by David Gibson, Linuxcare Australia. + * This file is released under the GPL. + */ + +/* + * NOTE! This filesystem is probably most useful + * not as a real filesystem, but as an example of + * how virtual filesystems can be written. + * + * It doesn't get much simpler than this. Consider + * that this file implements the full semantics of + * a POSIX-compliant read-write filesystem. + * + * Note in particular how the filesystem does not + * need to implement any data structures of its own + * to keep track of the virtual data: using the VFS + * caches is sufficient. + */ + +#include +#include +#include +#include + +#include "internal.h" + +static unsigned long ramfs_mmu_get_unmapped_area(struct file *file, + unsigned long addr, unsigned long len, unsigned long pgoff, + unsigned long flags) +{ + return mm_get_unmapped_area(file, addr, len, pgoff, flags); +} + +const struct file_operations ramfs_file_operations = { + .read_iter = generic_file_read_iter, + .write_iter = generic_file_write_iter, + .mmap_prepare = generic_file_mmap_prepare, + .fsync = noop_fsync, + .splice_read = filemap_splice_read, + .splice_write = iter_file_splice_write, + .llseek = generic_file_llseek, + .get_unmapped_area = ramfs_mmu_get_unmapped_area, +}; + +const struct inode_operations ramfs_file_inode_operations = { + .setattr = simple_setattr, + .getattr = simple_getattr, +}; diff --git a/fs/ramfs/file-nommu.c b/fs/ramfs/file-nommu.c new file mode 100644 index 000000000..2f79bcb89 --- /dev/null +++ b/fs/ramfs/file-nommu.c @@ -0,0 +1,273 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* file-nommu.c: no-MMU version of ramfs + * + * Copyright (C) 2005 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "internal.h" + +static int ramfs_nommu_setattr(struct mnt_idmap *, struct dentry *, struct iattr *); +static unsigned long ramfs_nommu_get_unmapped_area(struct file *file, + unsigned long addr, + unsigned long len, + unsigned long pgoff, + unsigned long flags); +static int ramfs_nommu_mmap_prepare(struct vm_area_desc *desc); + +static unsigned ramfs_mmap_capabilities(struct file *file) +{ + return NOMMU_MAP_DIRECT | NOMMU_MAP_COPY | NOMMU_MAP_READ | + NOMMU_MAP_WRITE | NOMMU_MAP_EXEC; +} + +const struct file_operations ramfs_file_operations = { + .mmap_capabilities = ramfs_mmap_capabilities, + .mmap_prepare = ramfs_nommu_mmap_prepare, + .get_unmapped_area = ramfs_nommu_get_unmapped_area, + .read_iter = generic_file_read_iter, + .write_iter = generic_file_write_iter, + .fsync = noop_fsync, + .splice_read = filemap_splice_read, + .splice_write = iter_file_splice_write, + .llseek = generic_file_llseek, +}; + +const struct inode_operations ramfs_file_inode_operations = { + .setattr = ramfs_nommu_setattr, + .getattr = simple_getattr, +}; + +/*****************************************************************************/ +/* + * add a contiguous set of pages into a ramfs inode when it's truncated from + * size 0 on the assumption that it's going to be used for an mmap of shared + * memory + */ +int ramfs_nommu_expand_for_mapping(struct inode *inode, size_t newsize) +{ + unsigned long npages, xpages, loop; + struct page *pages; + unsigned order; + void *data; + int ret; + gfp_t gfp = mapping_gfp_mask(inode->i_mapping); + + /* make various checks */ + order = get_order(newsize); + if (unlikely(order > MAX_PAGE_ORDER)) + return -EFBIG; + + ret = inode_newsize_ok(inode, newsize); + if (ret) + return ret; + + i_size_write(inode, newsize); + + /* allocate enough contiguous pages to be able to satisfy the + * request */ + pages = alloc_pages(gfp, order); + if (!pages) + return -ENOMEM; + + /* split the high-order page into an array of single pages */ + xpages = 1UL << order; + npages = (newsize + PAGE_SIZE - 1) >> PAGE_SHIFT; + + split_page(pages, order); + + /* trim off any pages we don't actually require */ + for (loop = npages; loop < xpages; loop++) + __free_page(pages + loop); + + /* clear the memory we allocated */ + newsize = PAGE_SIZE * npages; + data = page_address(pages); + memset(data, 0, newsize); + + /* attach all the pages to the inode's address space */ + for (loop = 0; loop < npages; loop++) { + struct page *page = pages + loop; + + ret = add_to_page_cache_lru(page, inode->i_mapping, loop, + gfp); + if (ret < 0) + goto add_error; + + /* prevent the page from being discarded on memory pressure */ + SetPageDirty(page); + SetPageUptodate(page); + + unlock_page(page); + put_page(page); + } + + return 0; + +add_error: + while (loop < npages) + __free_page(pages + loop++); + return ret; +} + +/*****************************************************************************/ +/* + * + */ +static int ramfs_nommu_resize(struct inode *inode, loff_t newsize, loff_t size) +{ + int ret; + + /* assume a truncate from zero size is going to be for the purposes of + * shared mmap */ + if (size == 0) { + if (unlikely(newsize >> 32)) + return -EFBIG; + + return ramfs_nommu_expand_for_mapping(inode, newsize); + } + + /* check that a decrease in size doesn't cut off any shared mappings */ + if (newsize < size) { + ret = nommu_shrink_inode_mappings(inode, size, newsize); + if (ret < 0) + return ret; + } + + truncate_setsize(inode, newsize); + return 0; +} + +/*****************************************************************************/ +/* + * handle a change of attributes + * - we're specifically interested in a change of size + */ +static int ramfs_nommu_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, struct iattr *ia) +{ + struct inode *inode = d_inode(dentry); + unsigned int old_ia_valid = ia->ia_valid; + int ret = 0; + + /* POSIX UID/GID verification for setting inode attributes */ + ret = setattr_prepare(&nop_mnt_idmap, dentry, ia); + if (ret) + return ret; + + /* pick out size-changing events */ + if (ia->ia_valid & ATTR_SIZE) { + loff_t size = inode->i_size; + + if (ia->ia_size != size) { + ret = ramfs_nommu_resize(inode, ia->ia_size, size); + if (ret < 0 || ia->ia_valid == ATTR_SIZE) + goto out; + } else { + /* we skipped the truncate but must still update + * timestamps + */ + ia->ia_valid |= ATTR_MTIME|ATTR_CTIME; + } + } + + setattr_copy(&nop_mnt_idmap, inode, ia); + out: + ia->ia_valid = old_ia_valid; + return ret; +} + +/*****************************************************************************/ +/* + * try to determine where a shared mapping can be made + * - we require that: + * - the pages to be mapped must exist + * - the pages be physically contiguous in sequence + */ +static unsigned long ramfs_nommu_get_unmapped_area(struct file *file, + unsigned long addr, unsigned long len, + unsigned long pgoff, unsigned long flags) +{ + unsigned long maxpages, lpages, nr_folios, loop, ret, nr_pages, pfn; + struct inode *inode = file_inode(file); + struct folio_batch fbatch; + loff_t isize; + + /* the mapping mustn't extend beyond the EOF */ + lpages = (len + PAGE_SIZE - 1) >> PAGE_SHIFT; + isize = i_size_read(inode); + + ret = -ENOSYS; + maxpages = (isize + PAGE_SIZE - 1) >> PAGE_SHIFT; + if (pgoff >= maxpages) + goto out; + + if (maxpages - pgoff < lpages) + goto out; + + /* gang-find the pages */ + folio_batch_init(&fbatch); + nr_pages = 0; +repeat: + nr_folios = filemap_get_folios_contig(inode->i_mapping, &pgoff, + ULONG_MAX, &fbatch); + if (!nr_folios) { + ret = -ENOSYS; + return ret; + } + + if (ret == -ENOSYS) { + ret = (unsigned long) folio_address(fbatch.folios[0]); + pfn = folio_pfn(fbatch.folios[0]); + } + /* check the pages for physical adjacency */ + for (loop = 0; loop < nr_folios; loop++) { + if (pfn + nr_pages != folio_pfn(fbatch.folios[loop])) { + ret = -ENOSYS; + goto out_free; /* leave if not physical adjacent */ + } + nr_pages += folio_nr_pages(fbatch.folios[loop]); + if (nr_pages >= lpages) + goto out_free; /* successfully found desired pages*/ + } + + if (nr_pages < lpages) { + folio_batch_release(&fbatch); + goto repeat; /* loop if pages are missing */ + } + /* okay - all conditions fulfilled */ + +out_free: + folio_batch_release(&fbatch); +out: + return ret; +} + +/*****************************************************************************/ +/* + * set up a mapping for shared memory segments + */ +static int ramfs_nommu_mmap_prepare(struct vm_area_desc *desc) +{ + if (!is_nommu_shared_vma_flags(&desc->vma_flags)) + return -ENOSYS; + + file_accessed(desc->file); + desc->vm_ops = &generic_file_vm_ops; + return 0; +} diff --git a/fs/ramfs/inode.c b/fs/ramfs/inode.c new file mode 100644 index 000000000..0a88ede48 --- /dev/null +++ b/fs/ramfs/inode.c @@ -0,0 +1,329 @@ +/* + * Resizable simple ram filesystem for Linux. + * + * Copyright (C) 2000 Linus Torvalds. + * 2000 Transmeta Corp. + * + * Usage limits added by David Gibson, Linuxcare Australia. + * This file is released under the GPL. + */ + +/* + * NOTE! This filesystem is probably most useful + * not as a real filesystem, but as an example of + * how virtual filesystems can be written. + * + * It doesn't get much simpler than this. Consider + * that this file implements the full semantics of + * a POSIX-compliant read-write filesystem. + * + * Note in particular how the filesystem does not + * need to implement any data structures of its own + * to keep track of the virtual data: using the VFS + * caches is sufficient. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +struct ramfs_mount_opts { + umode_t mode; +}; + +struct ramfs_fs_info { + struct ramfs_mount_opts mount_opts; +}; + +#define RAMFS_DEFAULT_MODE 0755 + +static const struct super_operations ramfs_ops; +static const struct inode_operations ramfs_dir_inode_operations; + +struct inode *ramfs_get_inode(struct super_block *sb, + const struct inode *dir, umode_t mode, dev_t dev) +{ + struct inode * inode = new_inode(sb); + + if (inode) { + inode->i_ino = get_next_ino(); + inode_init_owner(&nop_mnt_idmap, inode, dir, mode); + inode->i_mapping->a_ops = &ram_aops; + mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER); + mapping_set_unevictable(inode->i_mapping); + simple_inode_init_ts(inode); + switch (mode & S_IFMT) { + default: + init_special_inode(inode, mode, dev); + break; + case S_IFREG: + inode->i_op = &ramfs_file_inode_operations; + inode->i_fop = &ramfs_file_operations; + break; + case S_IFDIR: + inode->i_op = &ramfs_dir_inode_operations; + inode->i_fop = &simple_dir_operations; + + /* directory inodes start off with i_nlink == 2 (for "." entry) */ + inc_nlink(inode); + break; + case S_IFLNK: + inode->i_op = &page_symlink_inode_operations; + inode_nohighmem(inode); + break; + } + } + return inode; +} + +/* + * File creation. Allocate an inode, and we're done.. + */ +/* SMP-safe */ +static int +ramfs_mknod(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t dev) +{ + struct inode * inode = ramfs_get_inode(dir->i_sb, dir, mode, dev); + int error = -ENOSPC; + + if (inode) { + error = security_inode_init_security(inode, dir, + &dentry->d_name, NULL, + NULL); + if (error) { + iput(inode); + goto out; + } + + d_make_persistent(dentry, inode); + error = 0; + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + } +out: + return error; +} + +static struct dentry *ramfs_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + int retval = ramfs_mknod(&nop_mnt_idmap, dir, dentry, mode, 0); + if (!retval) + inc_nlink(dir); + return ERR_PTR(retval); +} + +static int ramfs_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + return ramfs_mknod(&nop_mnt_idmap, dir, dentry, mode | S_IFREG, 0); +} + +static int ramfs_symlink(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const char *symname) +{ + struct inode *inode; + int error = -ENOSPC; + + inode = ramfs_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0); + if (inode) { + int l = strlen(symname)+1; + + error = security_inode_init_security(inode, dir, + &dentry->d_name, NULL, + NULL); + if (error) { + iput(inode); + goto out; + } + + error = page_symlink(inode, symname, l); + if (!error) { + d_make_persistent(dentry, inode); + inode_set_mtime_to_ts(dir, + inode_set_ctime_current(dir)); + } else + iput(inode); + } +out: + return error; +} + +static int ramfs_tmpfile(struct mnt_idmap *idmap, + struct inode *dir, struct file *file, umode_t mode) +{ + struct inode *inode; + int error; + + inode = ramfs_get_inode(dir->i_sb, dir, mode, 0); + if (!inode) + return -ENOSPC; + + error = security_inode_init_security(inode, dir, + &file_dentry(file)->d_name, NULL, + NULL); + if (error) { + iput(inode); + goto out; + } + + d_tmpfile(file, inode); +out: + return finish_open_simple(file, error); +} + +static const struct inode_operations ramfs_dir_inode_operations = { + .create = ramfs_create, + .lookup = simple_lookup, + .link = simple_link, + .unlink = simple_unlink, + .symlink = ramfs_symlink, + .mkdir = ramfs_mkdir, + .rmdir = simple_rmdir, + .mknod = ramfs_mknod, + .rename = simple_rename, + .tmpfile = ramfs_tmpfile, +}; + +/* + * Display the mount options in /proc/mounts. + */ +static int ramfs_show_options(struct seq_file *m, struct dentry *root) +{ + struct ramfs_fs_info *fsi = root->d_sb->s_fs_info; + + if (fsi->mount_opts.mode != RAMFS_DEFAULT_MODE) + seq_printf(m, ",mode=%o", fsi->mount_opts.mode); + return 0; +} + +static const struct super_operations ramfs_ops = { + .statfs = simple_statfs, + .drop_inode = inode_just_drop, + .show_options = ramfs_show_options, +}; + +enum ramfs_param { + Opt_mode, +}; + +const struct fs_parameter_spec ramfs_fs_parameters[] = { + fsparam_u32oct("mode", Opt_mode), + {} +}; + +static int ramfs_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct fs_parse_result result; + struct ramfs_fs_info *fsi = fc->s_fs_info; + int opt; + + opt = fs_parse(fc, ramfs_fs_parameters, param, &result); + if (opt == -ENOPARAM) { + opt = vfs_parse_fs_param_source(fc, param); + if (opt != -ENOPARAM) + return opt; + /* + * We might like to report bad mount options here; + * but traditionally ramfs has ignored all mount options, + * and as it is used as a !CONFIG_SHMEM simple substitute + * for tmpfs, better continue to ignore other mount options. + */ + return 0; + } + if (opt < 0) + return opt; + + switch (opt) { + case Opt_mode: + fsi->mount_opts.mode = result.uint_32 & S_IALLUGO; + break; + } + + return 0; +} + +static int ramfs_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct ramfs_fs_info *fsi = sb->s_fs_info; + struct inode *inode; + + sb->s_maxbytes = MAX_LFS_FILESIZE; + sb->s_blocksize = PAGE_SIZE; + sb->s_blocksize_bits = PAGE_SHIFT; + sb->s_magic = RAMFS_MAGIC; + sb->s_op = &ramfs_ops; + sb->s_d_flags = DCACHE_DONTCACHE; + sb->s_time_gran = 1; + + inode = ramfs_get_inode(sb, NULL, S_IFDIR | fsi->mount_opts.mode, 0); + sb->s_root = d_make_root(inode); + if (!sb->s_root) + return -ENOMEM; + + return 0; +} + +static int ramfs_get_tree(struct fs_context *fc) +{ + return get_tree_nodev(fc, ramfs_fill_super); +} + +static void ramfs_free_fc(struct fs_context *fc) +{ + kfree(fc->s_fs_info); +} + +static const struct fs_context_operations ramfs_context_ops = { + .free = ramfs_free_fc, + .parse_param = ramfs_parse_param, + .get_tree = ramfs_get_tree, +}; + +int ramfs_init_fs_context(struct fs_context *fc) +{ + struct ramfs_fs_info *fsi; + + fsi = kzalloc_obj(*fsi); + if (!fsi) + return -ENOMEM; + + fsi->mount_opts.mode = RAMFS_DEFAULT_MODE; + fc->s_fs_info = fsi; + fc->ops = &ramfs_context_ops; + return 0; +} + +void ramfs_kill_sb(struct super_block *sb) +{ + kfree(sb->s_fs_info); + kill_anon_super(sb); +} + +static struct file_system_type ramfs_fs_type = { + .name = "ramfs", + .init_fs_context = ramfs_init_fs_context, + .parameters = ramfs_fs_parameters, + .kill_sb = ramfs_kill_sb, + .fs_flags = FS_USERNS_MOUNT, +}; + +static int __init init_ramfs_fs(void) +{ + return register_filesystem(&ramfs_fs_type); +} +fs_initcall(init_ramfs_fs); diff --git a/fs/ramfs/internal.h b/fs/ramfs/internal.h new file mode 100644 index 000000000..3257fc180 --- /dev/null +++ b/fs/ramfs/internal.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* internal.h: ramfs internal definitions + * + * Copyright (C) 2005 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + + +extern const struct inode_operations ramfs_file_inode_operations; diff --git a/fs/read_write.c b/fs/read_write.c new file mode 100644 index 000000000..e8c14e276 --- /dev/null +++ b/fs/read_write.c @@ -0,0 +1,1822 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/read_write.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +#include +#include + +const struct file_operations generic_ro_fops = { + .llseek = generic_file_llseek, + .read_iter = generic_file_read_iter, + .mmap_prepare = generic_file_readonly_mmap_prepare, + .splice_read = filemap_splice_read, + .setlease = generic_setlease, +}; + +EXPORT_SYMBOL(generic_ro_fops); + +static inline bool unsigned_offsets(struct file *file) +{ + return file->f_op->fop_flags & FOP_UNSIGNED_OFFSET; +} + +/** + * vfs_setpos_cookie - update the file offset for lseek and reset cookie + * @file: file structure in question + * @offset: file offset to seek to + * @maxsize: maximum file size + * @cookie: cookie to reset + * + * Update the file offset to the value specified by @offset if the given + * offset is valid and it is not equal to the current file offset and + * reset the specified cookie to indicate that a seek happened. + * + * Return the specified offset on success and -EINVAL on invalid offset. + */ +static loff_t vfs_setpos_cookie(struct file *file, loff_t offset, + loff_t maxsize, u64 *cookie) +{ + if (offset < 0 && !unsigned_offsets(file)) + return -EINVAL; + if (offset > maxsize) + return -EINVAL; + + if (offset != file->f_pos) { + file->f_pos = offset; + if (cookie) + *cookie = 0; + } + return offset; +} + +/** + * vfs_setpos - update the file offset for lseek + * @file: file structure in question + * @offset: file offset to seek to + * @maxsize: maximum file size + * + * This is a low-level filesystem helper for updating the file offset to + * the value specified by @offset if the given offset is valid and it is + * not equal to the current file offset. + * + * Return the specified offset on success and -EINVAL on invalid offset. + */ +loff_t vfs_setpos(struct file *file, loff_t offset, loff_t maxsize) +{ + return vfs_setpos_cookie(file, offset, maxsize, NULL); +} +EXPORT_SYMBOL(vfs_setpos); + +/** + * must_set_pos - check whether f_pos has to be updated + * @file: file to seek on + * @offset: offset to use + * @whence: type of seek operation + * @eof: end of file + * + * Check whether f_pos needs to be updated and update @offset according + * to @whence. + * + * Return: 0 if f_pos doesn't need to be updated, 1 if f_pos has to be + * updated, and negative error code on failure. + */ +static int must_set_pos(struct file *file, loff_t *offset, int whence, loff_t eof) +{ + switch (whence) { + case SEEK_END: + *offset += eof; + break; + case SEEK_CUR: + /* + * Here we special-case the lseek(fd, 0, SEEK_CUR) + * position-querying operation. Avoid rewriting the "same" + * f_pos value back to the file because a concurrent read(), + * write() or lseek() might have altered it + */ + if (*offset == 0) { + *offset = file->f_pos; + return 0; + } + break; + case SEEK_DATA: + /* + * In the generic case the entire file is data, so as long as + * offset isn't at the end of the file then the offset is data. + */ + if ((unsigned long long)*offset >= eof) + return -ENXIO; + break; + case SEEK_HOLE: + /* + * There is a virtual hole at the end of the file, so as long as + * offset isn't i_size or larger, return i_size. + */ + if ((unsigned long long)*offset >= eof) + return -ENXIO; + *offset = eof; + break; + } + + return 1; +} + +/** + * generic_file_llseek_size - generic llseek implementation for regular files + * @file: file structure to seek on + * @offset: file offset to seek to + * @whence: type of seek + * @maxsize: max size of this file in file system + * @eof: offset used for SEEK_END position + * + * This is a variant of generic_file_llseek that allows passing in a custom + * maximum file size and a custom EOF position, for e.g. hashed directories + * + * Synchronization: + * SEEK_SET and SEEK_END are unsynchronized (but atomic on 64bit platforms) + * SEEK_CUR is synchronized against other SEEK_CURs, but not read/writes. + * read/writes behave like SEEK_SET against seeks. + */ +loff_t +generic_file_llseek_size(struct file *file, loff_t offset, int whence, + loff_t maxsize, loff_t eof) +{ + int ret; + + ret = must_set_pos(file, &offset, whence, eof); + if (ret < 0) + return ret; + if (ret == 0) + return offset; + + if (whence == SEEK_CUR) { + /* + * If the file requires locking via f_pos_lock we know + * that mutual exclusion for SEEK_CUR on the same file + * is guaranteed. If the file isn't locked, we take + * f_lock to protect against f_pos races with other + * SEEK_CURs. + */ + if (file_seek_cur_needs_f_lock(file)) { + guard(spinlock)(&file->f_lock); + return vfs_setpos(file, file->f_pos + offset, maxsize); + } + return vfs_setpos(file, file->f_pos + offset, maxsize); + } + + return vfs_setpos(file, offset, maxsize); +} +EXPORT_SYMBOL(generic_file_llseek_size); + +/** + * generic_llseek_cookie - versioned llseek implementation + * @file: file structure to seek on + * @offset: file offset to seek to + * @whence: type of seek + * @cookie: cookie to update + * + * See generic_file_llseek for a general description and locking assumptions. + * + * In contrast to generic_file_llseek, this function also resets a + * specified cookie to indicate a seek took place. + */ +loff_t generic_llseek_cookie(struct file *file, loff_t offset, int whence, + u64 *cookie) +{ + struct inode *inode = file->f_mapping->host; + loff_t maxsize = inode->i_sb->s_maxbytes; + loff_t eof = i_size_read(inode); + int ret; + + if (WARN_ON_ONCE(!cookie)) + return -EINVAL; + + /* + * Require that this is only used for directories that guarantee + * synchronization between readdir and seek so that an update to + * @cookie is correctly synchronized with concurrent readdir. + */ + if (WARN_ON_ONCE(!(file->f_mode & FMODE_ATOMIC_POS))) + return -EINVAL; + + ret = must_set_pos(file, &offset, whence, eof); + if (ret < 0) + return ret; + if (ret == 0) + return offset; + + /* No need to hold f_lock because we know that f_pos_lock is held. */ + if (whence == SEEK_CUR) + return vfs_setpos_cookie(file, file->f_pos + offset, maxsize, cookie); + + return vfs_setpos_cookie(file, offset, maxsize, cookie); +} +EXPORT_SYMBOL(generic_llseek_cookie); + +/** + * generic_file_llseek - generic llseek implementation for regular files + * @file: file structure to seek on + * @offset: file offset to seek to + * @whence: type of seek + * + * This is a generic implementation of ->llseek useable for all normal local + * filesystems. It just updates the file offset to the value specified by + * @offset and @whence. + */ +loff_t generic_file_llseek(struct file *file, loff_t offset, int whence) +{ + struct inode *inode = file->f_mapping->host; + + return generic_file_llseek_size(file, offset, whence, + inode->i_sb->s_maxbytes, + i_size_read(inode)); +} +EXPORT_SYMBOL(generic_file_llseek); + +/** + * fixed_size_llseek - llseek implementation for fixed-sized devices + * @file: file structure to seek on + * @offset: file offset to seek to + * @whence: type of seek + * @size: size of the file + * + */ +loff_t fixed_size_llseek(struct file *file, loff_t offset, int whence, loff_t size) +{ + switch (whence) { + case SEEK_SET: case SEEK_CUR: case SEEK_END: + return generic_file_llseek_size(file, offset, whence, + size, size); + default: + return -EINVAL; + } +} +EXPORT_SYMBOL(fixed_size_llseek); + +/** + * no_seek_end_llseek - llseek implementation for fixed-sized devices + * @file: file structure to seek on + * @offset: file offset to seek to + * @whence: type of seek + * + */ +loff_t no_seek_end_llseek(struct file *file, loff_t offset, int whence) +{ + switch (whence) { + case SEEK_SET: case SEEK_CUR: + return generic_file_llseek_size(file, offset, whence, + OFFSET_MAX, 0); + default: + return -EINVAL; + } +} +EXPORT_SYMBOL(no_seek_end_llseek); + +/** + * no_seek_end_llseek_size - llseek implementation for fixed-sized devices + * @file: file structure to seek on + * @offset: file offset to seek to + * @whence: type of seek + * @size: maximal offset allowed + * + */ +loff_t no_seek_end_llseek_size(struct file *file, loff_t offset, int whence, loff_t size) +{ + switch (whence) { + case SEEK_SET: case SEEK_CUR: + return generic_file_llseek_size(file, offset, whence, + size, 0); + default: + return -EINVAL; + } +} +EXPORT_SYMBOL(no_seek_end_llseek_size); + +/** + * noop_llseek - No Operation Performed llseek implementation + * @file: file structure to seek on + * @offset: file offset to seek to + * @whence: type of seek + * + * This is an implementation of ->llseek useable for the rare special case when + * userspace expects the seek to succeed but the (device) file is actually not + * able to perform the seek. In this case you use noop_llseek() instead of + * falling back to the default implementation of ->llseek. + */ +loff_t noop_llseek(struct file *file, loff_t offset, int whence) +{ + return file->f_pos; +} +EXPORT_SYMBOL(noop_llseek); + +loff_t default_llseek(struct file *file, loff_t offset, int whence) +{ + struct inode *inode = file_inode(file); + loff_t retval; + + retval = inode_lock_killable(inode); + if (retval) + return retval; + switch (whence) { + case SEEK_END: + offset += i_size_read(inode); + break; + case SEEK_CUR: + if (offset == 0) { + retval = file->f_pos; + goto out; + } + offset += file->f_pos; + break; + case SEEK_DATA: + /* + * In the generic case the entire file is data, so as + * long as offset isn't at the end of the file then the + * offset is data. + */ + if (offset >= inode->i_size) { + retval = -ENXIO; + goto out; + } + break; + case SEEK_HOLE: + /* + * There is a virtual hole at the end of the file, so + * as long as offset isn't i_size or larger, return + * i_size. + */ + if (offset >= inode->i_size) { + retval = -ENXIO; + goto out; + } + offset = inode->i_size; + break; + } + retval = -EINVAL; + if (offset >= 0 || unsigned_offsets(file)) { + if (offset != file->f_pos) + file->f_pos = offset; + retval = offset; + } +out: + inode_unlock(inode); + return retval; +} +EXPORT_SYMBOL(default_llseek); + +loff_t vfs_llseek(struct file *file, loff_t offset, int whence) +{ + if (!(file->f_mode & FMODE_LSEEK)) + return -ESPIPE; + return file->f_op->llseek(file, offset, whence); +} +EXPORT_SYMBOL(vfs_llseek); + +static off_t ksys_lseek(unsigned int fd, off_t offset, unsigned int whence) +{ + off_t retval; + CLASS(fd_pos, f)(fd); + if (fd_empty(f)) + return -EBADF; + + retval = -EINVAL; + if (whence <= SEEK_MAX) { + loff_t res = vfs_llseek(fd_file(f), offset, whence); + retval = res; + if (res != (loff_t)retval) + retval = -EOVERFLOW; /* LFS: should only happen on 32 bit platforms */ + } + return retval; +} + +SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, whence) +{ + return ksys_lseek(fd, offset, whence); +} + +#ifdef CONFIG_COMPAT +COMPAT_SYSCALL_DEFINE3(lseek, unsigned int, fd, compat_off_t, offset, unsigned int, whence) +{ + return ksys_lseek(fd, offset, whence); +} +#endif + +#if !defined(CONFIG_64BIT) || defined(CONFIG_COMPAT) || \ + defined(__ARCH_WANT_SYS_LLSEEK) +SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high, + unsigned long, offset_low, loff_t __user *, result, + unsigned int, whence) +{ + int retval; + CLASS(fd_pos, f)(fd); + loff_t offset; + + if (fd_empty(f)) + return -EBADF; + + if (whence > SEEK_MAX) + return -EINVAL; + + offset = vfs_llseek(fd_file(f), ((loff_t) offset_high << 32) | offset_low, + whence); + + retval = (int)offset; + if (offset >= 0) { + retval = -EFAULT; + if (!copy_to_user(result, &offset, sizeof(offset))) + retval = 0; + } + return retval; +} +#endif + +int rw_verify_area(int read_write, struct file *file, const loff_t *ppos, size_t count) +{ + int mask = read_write == READ ? MAY_READ : MAY_WRITE; + int ret; + + if (unlikely((ssize_t) count < 0)) + return -EINVAL; + + if (ppos) { + loff_t pos = *ppos; + + if (unlikely(pos < 0)) { + if (!unsigned_offsets(file)) + return -EINVAL; + if (count >= -pos) /* both values are in 0..LLONG_MAX */ + return -EOVERFLOW; + } else if (unlikely((loff_t) (pos + count) < 0)) { + if (!unsigned_offsets(file)) + return -EINVAL; + } + } + + ret = security_file_permission(file, mask); + if (ret) + return ret; + + return fsnotify_file_area_perm(file, mask, ppos, count); +} +EXPORT_SYMBOL(rw_verify_area); + +static ssize_t new_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos) +{ + struct kiocb kiocb; + struct iov_iter iter; + ssize_t ret; + + init_sync_kiocb(&kiocb, filp); + kiocb.ki_pos = (ppos ? *ppos : 0); + iov_iter_ubuf(&iter, ITER_DEST, buf, len); + + ret = filp->f_op->read_iter(&kiocb, &iter); + BUG_ON(ret == -EIOCBQUEUED); + if (ppos) + *ppos = kiocb.ki_pos; + return ret; +} + +static int warn_unsupported(struct file *file, const char *op) +{ + pr_warn_ratelimited( + "kernel %s not supported for file %pD4 (pid: %d comm: %.20s)\n", + op, file, current->pid, current->comm); + return -EINVAL; +} + +ssize_t __kernel_read(struct file *file, void *buf, size_t count, loff_t *pos) +{ + struct kvec iov = { + .iov_base = buf, + .iov_len = min_t(size_t, count, MAX_RW_COUNT), + }; + struct kiocb kiocb; + struct iov_iter iter; + ssize_t ret; + + if (WARN_ON_ONCE(!(file->f_mode & FMODE_READ))) + return -EINVAL; + if (!(file->f_mode & FMODE_CAN_READ)) + return -EINVAL; + /* + * Also fail if ->read_iter and ->read are both wired up as that + * implies very convoluted semantics. + */ + if (unlikely(!file->f_op->read_iter || file->f_op->read)) + return warn_unsupported(file, "read"); + + init_sync_kiocb(&kiocb, file); + kiocb.ki_pos = pos ? *pos : 0; + iov_iter_kvec(&iter, ITER_DEST, &iov, 1, iov.iov_len); + ret = file->f_op->read_iter(&kiocb, &iter); + if (ret > 0) { + if (pos) + *pos = kiocb.ki_pos; + fsnotify_access(file); + add_rchar(current, ret); + } + inc_syscr(current); + return ret; +} + +ssize_t kernel_read(struct file *file, void *buf, size_t count, loff_t *pos) +{ + ssize_t ret; + + ret = rw_verify_area(READ, file, pos, count); + if (ret) + return ret; + return __kernel_read(file, buf, count, pos); +} +EXPORT_SYMBOL(kernel_read); + +ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos) +{ + ssize_t ret; + + if (!(file->f_mode & FMODE_READ)) + return -EBADF; + if (!(file->f_mode & FMODE_CAN_READ)) + return -EINVAL; + if (unlikely(!access_ok(buf, count))) + return -EFAULT; + + ret = rw_verify_area(READ, file, pos, count); + if (ret) + return ret; + if (count > MAX_RW_COUNT) + count = MAX_RW_COUNT; + + if (file->f_op->read) + ret = file->f_op->read(file, buf, count, pos); + else if (file->f_op->read_iter) + ret = new_sync_read(file, buf, count, pos); + else + ret = -EINVAL; + if (ret > 0) { + fsnotify_access(file); + add_rchar(current, ret); + } + inc_syscr(current); + return ret; +} + +static ssize_t new_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos) +{ + struct kiocb kiocb; + struct iov_iter iter; + ssize_t ret; + + init_sync_kiocb(&kiocb, filp); + kiocb.ki_pos = (ppos ? *ppos : 0); + iov_iter_ubuf(&iter, ITER_SOURCE, (void __user *)buf, len); + + ret = filp->f_op->write_iter(&kiocb, &iter); + BUG_ON(ret == -EIOCBQUEUED); + if (ret > 0 && ppos) + *ppos = kiocb.ki_pos; + return ret; +} + +/* caller is responsible for file_start_write/file_end_write */ +ssize_t __kernel_write_iter(struct file *file, struct iov_iter *from, loff_t *pos) +{ + struct kiocb kiocb; + ssize_t ret; + + if (WARN_ON_ONCE(!(file->f_mode & FMODE_WRITE))) + return -EBADF; + if (!(file->f_mode & FMODE_CAN_WRITE)) + return -EINVAL; + /* + * Also fail if ->write_iter and ->write are both wired up as that + * implies very convoluted semantics. + */ + if (unlikely(!file->f_op->write_iter || file->f_op->write)) + return warn_unsupported(file, "write"); + + init_sync_kiocb(&kiocb, file); + kiocb.ki_pos = pos ? *pos : 0; + ret = file->f_op->write_iter(&kiocb, from); + if (ret > 0) { + if (pos) + *pos = kiocb.ki_pos; + fsnotify_modify(file); + add_wchar(current, ret); + } + inc_syscw(current); + return ret; +} + +/* caller is responsible for file_start_write/file_end_write */ +ssize_t __kernel_write(struct file *file, const void *buf, size_t count, loff_t *pos) +{ + struct kvec iov = { + .iov_base = (void *)buf, + .iov_len = min_t(size_t, count, MAX_RW_COUNT), + }; + struct iov_iter iter; + iov_iter_kvec(&iter, ITER_SOURCE, &iov, 1, iov.iov_len); + return __kernel_write_iter(file, &iter, pos); +} +/* + * autofs is one of the few internal kernel users that actually + * wants this _and_ can be built as a module. So we need to export + * this symbol for autofs, even though it really isn't appropriate + * for any other kernel modules. + */ +EXPORT_SYMBOL_FOR_MODULES(__kernel_write, "autofs4"); + +ssize_t kernel_write(struct file *file, const void *buf, size_t count, + loff_t *pos) +{ + ssize_t ret; + + ret = rw_verify_area(WRITE, file, pos, count); + if (ret) + return ret; + + file_start_write(file); + ret = __kernel_write(file, buf, count, pos); + file_end_write(file); + return ret; +} +EXPORT_SYMBOL(kernel_write); + +ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos) +{ + ssize_t ret; + + if (!(file->f_mode & FMODE_WRITE)) + return -EBADF; + if (!(file->f_mode & FMODE_CAN_WRITE)) + return -EINVAL; + if (unlikely(!access_ok(buf, count))) + return -EFAULT; + + ret = rw_verify_area(WRITE, file, pos, count); + if (ret) + return ret; + if (count > MAX_RW_COUNT) + count = MAX_RW_COUNT; + file_start_write(file); + if (file->f_op->write) + ret = file->f_op->write(file, buf, count, pos); + else if (file->f_op->write_iter) + ret = new_sync_write(file, buf, count, pos); + else + ret = -EINVAL; + if (ret > 0) { + fsnotify_modify(file); + add_wchar(current, ret); + } + inc_syscw(current); + file_end_write(file); + return ret; +} + +/* file_ppos returns &file->f_pos or NULL if file is stream */ +static inline loff_t *file_ppos(struct file *file) +{ + return file->f_mode & FMODE_STREAM ? NULL : &file->f_pos; +} + +ssize_t ksys_read(unsigned int fd, char __user *buf, size_t count) +{ + CLASS(fd_pos, f)(fd); + ssize_t ret = -EBADF; + + if (!fd_empty(f)) { + loff_t pos, *ppos = file_ppos(fd_file(f)); + if (ppos) { + pos = *ppos; + ppos = &pos; + } + ret = vfs_read(fd_file(f), buf, count, ppos); + if (ret >= 0 && ppos) + fd_file(f)->f_pos = pos; + } + return ret; +} + +SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count) +{ + return ksys_read(fd, buf, count); +} + +ssize_t ksys_write(unsigned int fd, const char __user *buf, size_t count) +{ + CLASS(fd_pos, f)(fd); + ssize_t ret = -EBADF; + + if (!fd_empty(f)) { + loff_t pos, *ppos = file_ppos(fd_file(f)); + if (ppos) { + pos = *ppos; + ppos = &pos; + } + ret = vfs_write(fd_file(f), buf, count, ppos); + if (ret >= 0 && ppos) + fd_file(f)->f_pos = pos; + } + + return ret; +} + +SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf, + size_t, count) +{ + return ksys_write(fd, buf, count); +} + +ssize_t ksys_pread64(unsigned int fd, char __user *buf, size_t count, + loff_t pos) +{ + if (pos < 0) + return -EINVAL; + + CLASS(fd, f)(fd); + if (fd_empty(f)) + return -EBADF; + + if (fd_file(f)->f_mode & FMODE_PREAD) + return vfs_read(fd_file(f), buf, count, &pos); + + return -ESPIPE; +} + +SYSCALL_DEFINE4(pread64, unsigned int, fd, char __user *, buf, + size_t, count, loff_t, pos) +{ + return ksys_pread64(fd, buf, count, pos); +} + +#if defined(CONFIG_COMPAT) && defined(__ARCH_WANT_COMPAT_PREAD64) +COMPAT_SYSCALL_DEFINE5(pread64, unsigned int, fd, char __user *, buf, + size_t, count, compat_arg_u64_dual(pos)) +{ + return ksys_pread64(fd, buf, count, compat_arg_u64_glue(pos)); +} +#endif + +ssize_t ksys_pwrite64(unsigned int fd, const char __user *buf, + size_t count, loff_t pos) +{ + if (pos < 0) + return -EINVAL; + + CLASS(fd, f)(fd); + if (fd_empty(f)) + return -EBADF; + + if (fd_file(f)->f_mode & FMODE_PWRITE) + return vfs_write(fd_file(f), buf, count, &pos); + + return -ESPIPE; +} + +SYSCALL_DEFINE4(pwrite64, unsigned int, fd, const char __user *, buf, + size_t, count, loff_t, pos) +{ + return ksys_pwrite64(fd, buf, count, pos); +} + +#if defined(CONFIG_COMPAT) && defined(__ARCH_WANT_COMPAT_PWRITE64) +COMPAT_SYSCALL_DEFINE5(pwrite64, unsigned int, fd, const char __user *, buf, + size_t, count, compat_arg_u64_dual(pos)) +{ + return ksys_pwrite64(fd, buf, count, compat_arg_u64_glue(pos)); +} +#endif + +static ssize_t do_iter_readv_writev(struct file *filp, struct iov_iter *iter, + loff_t *ppos, int type, rwf_t flags) +{ + struct kiocb kiocb; + ssize_t ret; + + init_sync_kiocb(&kiocb, filp); + ret = kiocb_set_rw_flags(&kiocb, flags, type); + if (ret) + return ret; + kiocb.ki_pos = (ppos ? *ppos : 0); + + if (type == READ) + ret = filp->f_op->read_iter(&kiocb, iter); + else + ret = filp->f_op->write_iter(&kiocb, iter); + BUG_ON(ret == -EIOCBQUEUED); + if (ppos) + *ppos = kiocb.ki_pos; + return ret; +} + +/* Do it by hand, with file-ops */ +static ssize_t do_loop_readv_writev(struct file *filp, struct iov_iter *iter, + loff_t *ppos, int type, rwf_t flags) +{ + ssize_t ret = 0; + + if (flags & ~RWF_HIPRI) + return -EOPNOTSUPP; + + while (iov_iter_count(iter)) { + ssize_t nr; + + if (type == READ) { + nr = filp->f_op->read(filp, iter_iov_addr(iter), + iter_iov_len(iter), ppos); + } else { + nr = filp->f_op->write(filp, iter_iov_addr(iter), + iter_iov_len(iter), ppos); + } + + if (nr < 0) { + if (!ret) + ret = nr; + break; + } + ret += nr; + if (nr != iter_iov_len(iter)) + break; + iov_iter_advance(iter, nr); + } + + return ret; +} + +ssize_t vfs_iocb_iter_read(struct file *file, struct kiocb *iocb, + struct iov_iter *iter) +{ + size_t tot_len; + ssize_t ret = 0; + + if (!file->f_op->read_iter) + return -EINVAL; + if (!(file->f_mode & FMODE_READ)) + return -EBADF; + if (!(file->f_mode & FMODE_CAN_READ)) + return -EINVAL; + + tot_len = iov_iter_count(iter); + if (!tot_len) + goto out; + ret = rw_verify_area(READ, file, &iocb->ki_pos, tot_len); + if (ret < 0) + return ret; + + ret = file->f_op->read_iter(iocb, iter); +out: + if (ret >= 0) + fsnotify_access(file); + return ret; +} +EXPORT_SYMBOL(vfs_iocb_iter_read); + +ssize_t vfs_iter_read(struct file *file, struct iov_iter *iter, loff_t *ppos, + rwf_t flags) +{ + size_t tot_len; + ssize_t ret = 0; + + if (!file->f_op->read_iter) + return -EINVAL; + if (!(file->f_mode & FMODE_READ)) + return -EBADF; + if (!(file->f_mode & FMODE_CAN_READ)) + return -EINVAL; + + tot_len = iov_iter_count(iter); + if (!tot_len) + goto out; + ret = rw_verify_area(READ, file, ppos, tot_len); + if (ret < 0) + return ret; + + ret = do_iter_readv_writev(file, iter, ppos, READ, flags); +out: + if (ret >= 0) + fsnotify_access(file); + return ret; +} +EXPORT_SYMBOL(vfs_iter_read); + +/* + * Caller is responsible for calling kiocb_end_write() on completion + * if async iocb was queued. + */ +ssize_t vfs_iocb_iter_write(struct file *file, struct kiocb *iocb, + struct iov_iter *iter) +{ + size_t tot_len; + ssize_t ret = 0; + + if (!file->f_op->write_iter) + return -EINVAL; + if (!(file->f_mode & FMODE_WRITE)) + return -EBADF; + if (!(file->f_mode & FMODE_CAN_WRITE)) + return -EINVAL; + + tot_len = iov_iter_count(iter); + if (!tot_len) + return 0; + ret = rw_verify_area(WRITE, file, &iocb->ki_pos, tot_len); + if (ret < 0) + return ret; + + kiocb_start_write(iocb); + ret = file->f_op->write_iter(iocb, iter); + if (ret != -EIOCBQUEUED) + kiocb_end_write(iocb); + if (ret > 0) + fsnotify_modify(file); + + return ret; +} +EXPORT_SYMBOL(vfs_iocb_iter_write); + +ssize_t vfs_iter_write(struct file *file, struct iov_iter *iter, loff_t *ppos, + rwf_t flags) +{ + size_t tot_len; + ssize_t ret; + + if (!(file->f_mode & FMODE_WRITE)) + return -EBADF; + if (!(file->f_mode & FMODE_CAN_WRITE)) + return -EINVAL; + if (!file->f_op->write_iter) + return -EINVAL; + + tot_len = iov_iter_count(iter); + if (!tot_len) + return 0; + + ret = rw_verify_area(WRITE, file, ppos, tot_len); + if (ret < 0) + return ret; + + file_start_write(file); + ret = do_iter_readv_writev(file, iter, ppos, WRITE, flags); + if (ret > 0) + fsnotify_modify(file); + file_end_write(file); + + return ret; +} +EXPORT_SYMBOL(vfs_iter_write); + +static ssize_t vfs_readv(struct file *file, const struct iovec __user *vec, + unsigned long vlen, loff_t *pos, rwf_t flags) +{ + struct iovec iovstack[UIO_FASTIOV]; + struct iovec *iov = iovstack; + struct iov_iter iter; + size_t tot_len; + ssize_t ret = 0; + + if (!(file->f_mode & FMODE_READ)) + return -EBADF; + if (!(file->f_mode & FMODE_CAN_READ)) + return -EINVAL; + + ret = import_iovec(ITER_DEST, vec, vlen, ARRAY_SIZE(iovstack), &iov, + &iter); + if (ret < 0) + return ret; + + tot_len = iov_iter_count(&iter); + if (!tot_len) + goto out; + + ret = rw_verify_area(READ, file, pos, tot_len); + if (ret < 0) + goto out; + + if (file->f_op->read_iter) + ret = do_iter_readv_writev(file, &iter, pos, READ, flags); + else + ret = do_loop_readv_writev(file, &iter, pos, READ, flags); +out: + if (ret >= 0) + fsnotify_access(file); + kfree(iov); + return ret; +} + +static ssize_t vfs_writev(struct file *file, const struct iovec __user *vec, + unsigned long vlen, loff_t *pos, rwf_t flags) +{ + struct iovec iovstack[UIO_FASTIOV]; + struct iovec *iov = iovstack; + struct iov_iter iter; + size_t tot_len; + ssize_t ret = 0; + + if (!(file->f_mode & FMODE_WRITE)) + return -EBADF; + if (!(file->f_mode & FMODE_CAN_WRITE)) + return -EINVAL; + + ret = import_iovec(ITER_SOURCE, vec, vlen, ARRAY_SIZE(iovstack), &iov, + &iter); + if (ret < 0) + return ret; + + tot_len = iov_iter_count(&iter); + if (!tot_len) + goto out; + + ret = rw_verify_area(WRITE, file, pos, tot_len); + if (ret < 0) + goto out; + + file_start_write(file); + if (file->f_op->write_iter) + ret = do_iter_readv_writev(file, &iter, pos, WRITE, flags); + else + ret = do_loop_readv_writev(file, &iter, pos, WRITE, flags); + if (ret > 0) + fsnotify_modify(file); + file_end_write(file); +out: + kfree(iov); + return ret; +} + +static ssize_t do_readv(unsigned long fd, const struct iovec __user *vec, + unsigned long vlen, rwf_t flags) +{ + CLASS(fd_pos, f)(fd); + ssize_t ret = -EBADF; + + if (!fd_empty(f)) { + loff_t pos, *ppos = file_ppos(fd_file(f)); + if (ppos) { + pos = *ppos; + ppos = &pos; + } + ret = vfs_readv(fd_file(f), vec, vlen, ppos, flags); + if (ret >= 0 && ppos) + fd_file(f)->f_pos = pos; + } + + if (ret > 0) + add_rchar(current, ret); + inc_syscr(current); + return ret; +} + +static ssize_t do_writev(unsigned long fd, const struct iovec __user *vec, + unsigned long vlen, rwf_t flags) +{ + CLASS(fd_pos, f)(fd); + ssize_t ret = -EBADF; + + if (!fd_empty(f)) { + loff_t pos, *ppos = file_ppos(fd_file(f)); + if (ppos) { + pos = *ppos; + ppos = &pos; + } + ret = vfs_writev(fd_file(f), vec, vlen, ppos, flags); + if (ret >= 0 && ppos) + fd_file(f)->f_pos = pos; + } + + if (ret > 0) + add_wchar(current, ret); + inc_syscw(current); + return ret; +} + +static inline loff_t pos_from_hilo(unsigned long high, unsigned long low) +{ +#define HALF_LONG_BITS (BITS_PER_LONG / 2) + return (((loff_t)high << HALF_LONG_BITS) << HALF_LONG_BITS) | low; +} + +static ssize_t do_preadv(unsigned long fd, const struct iovec __user *vec, + unsigned long vlen, loff_t pos, rwf_t flags) +{ + ssize_t ret = -EBADF; + + if (pos < 0) + return -EINVAL; + + CLASS(fd, f)(fd); + if (!fd_empty(f)) { + ret = -ESPIPE; + if (fd_file(f)->f_mode & FMODE_PREAD) + ret = vfs_readv(fd_file(f), vec, vlen, &pos, flags); + } + + if (ret > 0) + add_rchar(current, ret); + inc_syscr(current); + return ret; +} + +static ssize_t do_pwritev(unsigned long fd, const struct iovec __user *vec, + unsigned long vlen, loff_t pos, rwf_t flags) +{ + ssize_t ret = -EBADF; + + if (pos < 0) + return -EINVAL; + + CLASS(fd, f)(fd); + if (!fd_empty(f)) { + ret = -ESPIPE; + if (fd_file(f)->f_mode & FMODE_PWRITE) + ret = vfs_writev(fd_file(f), vec, vlen, &pos, flags); + } + + if (ret > 0) + add_wchar(current, ret); + inc_syscw(current); + return ret; +} + +SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec, + unsigned long, vlen) +{ + return do_readv(fd, vec, vlen, 0); +} + +SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec, + unsigned long, vlen) +{ + return do_writev(fd, vec, vlen, 0); +} + +SYSCALL_DEFINE5(preadv, unsigned long, fd, const struct iovec __user *, vec, + unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h) +{ + loff_t pos = pos_from_hilo(pos_h, pos_l); + + return do_preadv(fd, vec, vlen, pos, 0); +} + +SYSCALL_DEFINE6(preadv2, unsigned long, fd, const struct iovec __user *, vec, + unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h, + rwf_t, flags) +{ + loff_t pos = pos_from_hilo(pos_h, pos_l); + + if (pos == -1) + return do_readv(fd, vec, vlen, flags); + + return do_preadv(fd, vec, vlen, pos, flags); +} + +SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec, + unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h) +{ + loff_t pos = pos_from_hilo(pos_h, pos_l); + + return do_pwritev(fd, vec, vlen, pos, 0); +} + +SYSCALL_DEFINE6(pwritev2, unsigned long, fd, const struct iovec __user *, vec, + unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h, + rwf_t, flags) +{ + loff_t pos = pos_from_hilo(pos_h, pos_l); + + if (pos == -1) + return do_writev(fd, vec, vlen, flags); + + return do_pwritev(fd, vec, vlen, pos, flags); +} + +/* + * Various compat syscalls. Note that they all pretend to take a native + * iovec - import_iovec will properly treat those as compat_iovecs based on + * in_compat_syscall(). + */ +#ifdef CONFIG_COMPAT +#ifdef __ARCH_WANT_COMPAT_SYS_PREADV64 +COMPAT_SYSCALL_DEFINE4(preadv64, unsigned long, fd, + const struct iovec __user *, vec, + unsigned long, vlen, loff_t, pos) +{ + return do_preadv(fd, vec, vlen, pos, 0); +} +#endif + +COMPAT_SYSCALL_DEFINE5(preadv, compat_ulong_t, fd, + const struct iovec __user *, vec, + compat_ulong_t, vlen, u32, pos_low, u32, pos_high) +{ + loff_t pos = ((loff_t)pos_high << 32) | pos_low; + + return do_preadv(fd, vec, vlen, pos, 0); +} + +#ifdef __ARCH_WANT_COMPAT_SYS_PREADV64V2 +COMPAT_SYSCALL_DEFINE5(preadv64v2, unsigned long, fd, + const struct iovec __user *, vec, + unsigned long, vlen, loff_t, pos, rwf_t, flags) +{ + if (pos == -1) + return do_readv(fd, vec, vlen, flags); + return do_preadv(fd, vec, vlen, pos, flags); +} +#endif + +COMPAT_SYSCALL_DEFINE6(preadv2, compat_ulong_t, fd, + const struct iovec __user *, vec, + compat_ulong_t, vlen, u32, pos_low, u32, pos_high, + rwf_t, flags) +{ + loff_t pos = ((loff_t)pos_high << 32) | pos_low; + + if (pos == -1) + return do_readv(fd, vec, vlen, flags); + return do_preadv(fd, vec, vlen, pos, flags); +} + +#ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64 +COMPAT_SYSCALL_DEFINE4(pwritev64, unsigned long, fd, + const struct iovec __user *, vec, + unsigned long, vlen, loff_t, pos) +{ + return do_pwritev(fd, vec, vlen, pos, 0); +} +#endif + +COMPAT_SYSCALL_DEFINE5(pwritev, compat_ulong_t, fd, + const struct iovec __user *,vec, + compat_ulong_t, vlen, u32, pos_low, u32, pos_high) +{ + loff_t pos = ((loff_t)pos_high << 32) | pos_low; + + return do_pwritev(fd, vec, vlen, pos, 0); +} + +#ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64V2 +COMPAT_SYSCALL_DEFINE5(pwritev64v2, unsigned long, fd, + const struct iovec __user *, vec, + unsigned long, vlen, loff_t, pos, rwf_t, flags) +{ + if (pos == -1) + return do_writev(fd, vec, vlen, flags); + return do_pwritev(fd, vec, vlen, pos, flags); +} +#endif + +COMPAT_SYSCALL_DEFINE6(pwritev2, compat_ulong_t, fd, + const struct iovec __user *,vec, + compat_ulong_t, vlen, u32, pos_low, u32, pos_high, rwf_t, flags) +{ + loff_t pos = ((loff_t)pos_high << 32) | pos_low; + + if (pos == -1) + return do_writev(fd, vec, vlen, flags); + return do_pwritev(fd, vec, vlen, pos, flags); +} +#endif /* CONFIG_COMPAT */ + +static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos, + size_t count, loff_t max) +{ + struct inode *in_inode, *out_inode; + struct pipe_inode_info *opipe; + loff_t pos; + loff_t out_pos; + ssize_t retval; + int fl; + + /* + * Get input file, and verify that it is ok.. + */ + CLASS(fd, in)(in_fd); + if (fd_empty(in)) + return -EBADF; + if (!(fd_file(in)->f_mode & FMODE_READ)) + return -EBADF; + if (!ppos) { + pos = fd_file(in)->f_pos; + } else { + pos = *ppos; + if (!(fd_file(in)->f_mode & FMODE_PREAD)) + return -ESPIPE; + } + retval = rw_verify_area(READ, fd_file(in), &pos, count); + if (retval < 0) + return retval; + if (count > MAX_RW_COUNT) + count = MAX_RW_COUNT; + + /* + * Get output file, and verify that it is ok.. + */ + CLASS(fd, out)(out_fd); + if (fd_empty(out)) + return -EBADF; + if (!(fd_file(out)->f_mode & FMODE_WRITE)) + return -EBADF; + in_inode = file_inode(fd_file(in)); + out_inode = file_inode(fd_file(out)); + out_pos = fd_file(out)->f_pos; + + if (!max) + max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes); + + if (unlikely(pos + count > max)) { + if (pos >= max) + return -EOVERFLOW; + count = max - pos; + } + + fl = 0; +#if 0 + /* + * We need to debate whether we can enable this or not. The + * man page documents EAGAIN return for the output at least, + * and the application is arguably buggy if it doesn't expect + * EAGAIN on a non-blocking file descriptor. + */ + if (fd_file(in)->f_flags & O_NONBLOCK) + fl = SPLICE_F_NONBLOCK; +#endif + opipe = get_pipe_info(fd_file(out), true); + if (!opipe) { + retval = rw_verify_area(WRITE, fd_file(out), &out_pos, count); + if (retval < 0) + return retval; + retval = do_splice_direct(fd_file(in), &pos, fd_file(out), &out_pos, + count, fl); + } else { + if (fd_file(out)->f_flags & O_NONBLOCK) + fl |= SPLICE_F_NONBLOCK; + + retval = splice_file_to_pipe(fd_file(in), opipe, &pos, count, fl); + } + + if (retval > 0) { + add_rchar(current, retval); + add_wchar(current, retval); + fsnotify_access(fd_file(in)); + fsnotify_modify(fd_file(out)); + fd_file(out)->f_pos = out_pos; + if (ppos) + *ppos = pos; + else + fd_file(in)->f_pos = pos; + } + + inc_syscr(current); + inc_syscw(current); + if (pos > max) + retval = -EOVERFLOW; + return retval; +} + +SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count) +{ + loff_t pos; + off_t off; + ssize_t ret; + + if (offset) { + if (unlikely(get_user(off, offset))) + return -EFAULT; + pos = off; + ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS); + if (unlikely(put_user(pos, offset))) + return -EFAULT; + return ret; + } + + return do_sendfile(out_fd, in_fd, NULL, count, 0); +} + +SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count) +{ + loff_t pos; + ssize_t ret; + + if (offset) { + if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t)))) + return -EFAULT; + ret = do_sendfile(out_fd, in_fd, &pos, count, 0); + if (unlikely(put_user(pos, offset))) + return -EFAULT; + return ret; + } + + return do_sendfile(out_fd, in_fd, NULL, count, 0); +} + +#ifdef CONFIG_COMPAT +COMPAT_SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, + compat_off_t __user *, offset, compat_size_t, count) +{ + loff_t pos; + off_t off; + ssize_t ret; + + if (offset) { + if (unlikely(get_user(off, offset))) + return -EFAULT; + pos = off; + ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS); + if (unlikely(put_user(pos, offset))) + return -EFAULT; + return ret; + } + + return do_sendfile(out_fd, in_fd, NULL, count, 0); +} + +COMPAT_SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, + compat_loff_t __user *, offset, compat_size_t, count) +{ + loff_t pos; + ssize_t ret; + + if (offset) { + if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t)))) + return -EFAULT; + ret = do_sendfile(out_fd, in_fd, &pos, count, 0); + if (unlikely(put_user(pos, offset))) + return -EFAULT; + return ret; + } + + return do_sendfile(out_fd, in_fd, NULL, count, 0); +} +#endif + +/* + * Performs necessary checks before doing a file copy + * + * Can adjust amount of bytes to copy via @req_count argument. + * Returns appropriate error code that caller should return or + * zero in case the copy should be allowed. + */ +static int generic_copy_file_checks(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + size_t *req_count, unsigned int flags) +{ + struct inode *inode_in = file_inode(file_in); + struct inode *inode_out = file_inode(file_out); + uint64_t count = *req_count; + loff_t size_in; + int ret; + + ret = generic_file_rw_checks(file_in, file_out); + if (ret) + return ret; + + /* + * We allow some filesystems to handle cross sb copy, but passing + * a file of the wrong filesystem type to filesystem driver can result + * in an attempt to dereference the wrong type of ->private_data, so + * avoid doing that until we really have a good reason. + * + * nfs and cifs define several different file_system_type structures + * and several different sets of file_operations, but they all end up + * using the same ->copy_file_range() function pointer. + */ + if (flags & COPY_FILE_SPLICE) { + /* cross sb splice is allowed */ + } else if (file_out->f_op->copy_file_range) { + if (file_in->f_op->copy_file_range != + file_out->f_op->copy_file_range) + return -EXDEV; + } else if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb) { + return -EXDEV; + } + + /* Don't touch certain kinds of inodes */ + if (IS_IMMUTABLE(inode_out)) + return -EPERM; + + if (IS_SWAPFILE(inode_in) || IS_SWAPFILE(inode_out)) + return -ETXTBSY; + + /* Ensure offsets don't wrap. */ + if (pos_in + count < pos_in || pos_out + count < pos_out) + return -EOVERFLOW; + + /* Shorten the copy to EOF */ + size_in = i_size_read(inode_in); + if (pos_in >= size_in) + count = 0; + else + count = min(count, size_in - (uint64_t)pos_in); + + ret = generic_write_check_limits(file_out, pos_out, &count); + if (ret) + return ret; + + /* Don't allow overlapped copying within the same file. */ + if (inode_in == inode_out && + pos_out + count > pos_in && + pos_out < pos_in + count) + return -EINVAL; + + *req_count = count; + return 0; +} + +/* + * copy_file_range() differs from regular file read and write in that it + * specifically allows return partial success. When it does so is up to + * the copy_file_range method. + */ +ssize_t vfs_copy_file_range(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + size_t len, unsigned int flags) +{ + ssize_t ret; + bool splice = flags & COPY_FILE_SPLICE; + bool samesb = file_inode(file_in)->i_sb == file_inode(file_out)->i_sb; + + if (flags & ~COPY_FILE_SPLICE) + return -EINVAL; + + ret = generic_copy_file_checks(file_in, pos_in, file_out, pos_out, &len, + flags); + if (unlikely(ret)) + return ret; + + ret = rw_verify_area(READ, file_in, &pos_in, len); + if (unlikely(ret)) + return ret; + + ret = rw_verify_area(WRITE, file_out, &pos_out, len); + if (unlikely(ret)) + return ret; + + if (len == 0) + return 0; + + /* + * Make sure return value doesn't overflow in 32bit compat mode. Also + * limit the size for all cases except when calling ->copy_file_range(). + */ + if (splice || !file_out->f_op->copy_file_range || in_compat_syscall()) + len = min_t(size_t, MAX_RW_COUNT, len); + + file_start_write(file_out); + + /* + * Cloning is supported by more file systems, so we implement copy on + * same sb using clone, but for filesystems where both clone and copy + * are supported (e.g. nfs,cifs), we only call the copy method. + */ + if (!splice && file_out->f_op->copy_file_range) { + ret = file_out->f_op->copy_file_range(file_in, pos_in, + file_out, pos_out, + len, flags); + } else if (!splice && file_in->f_op->remap_file_range && samesb) { + ret = file_in->f_op->remap_file_range(file_in, pos_in, + file_out, pos_out, len, REMAP_FILE_CAN_SHORTEN); + /* fallback to splice */ + if (ret <= 0) + splice = true; + } else if (samesb) { + /* Fallback to splice for same sb copy for backward compat */ + splice = true; + } + + file_end_write(file_out); + + if (!splice) + goto done; + + /* + * We can get here for same sb copy of filesystems that do not implement + * ->copy_file_range() in case filesystem does not support clone or in + * case filesystem supports clone but rejected the clone request (e.g. + * because it was not block aligned). + * + * In both cases, fall back to kernel copy so we are able to maintain a + * consistent story about which filesystems support copy_file_range() + * and which filesystems do not, that will allow userspace tools to + * make consistent desicions w.r.t using copy_file_range(). + * + * We also get here if caller (e.g. nfsd) requested COPY_FILE_SPLICE + * for server-side-copy between any two sb. + * + * In any case, we call do_splice_direct() and not splice_file_range(), + * without file_start_write() held, to avoid possible deadlocks related + * to splicing from input file, while file_start_write() is held on + * the output file on a different sb. + */ + ret = do_splice_direct(file_in, &pos_in, file_out, &pos_out, len, 0); +done: + if (ret > 0) { + fsnotify_access(file_in); + add_rchar(current, ret); + fsnotify_modify(file_out); + add_wchar(current, ret); + } + + inc_syscr(current); + inc_syscw(current); + + return ret; +} +EXPORT_SYMBOL(vfs_copy_file_range); + +SYSCALL_DEFINE6(copy_file_range, int, fd_in, loff_t __user *, off_in, + int, fd_out, loff_t __user *, off_out, + size_t, len, unsigned int, flags) +{ + loff_t pos_in; + loff_t pos_out; + ssize_t ret = -EBADF; + + CLASS(fd, f_in)(fd_in); + if (fd_empty(f_in)) + return -EBADF; + + CLASS(fd, f_out)(fd_out); + if (fd_empty(f_out)) + return -EBADF; + + if (off_in) { + if (copy_from_user(&pos_in, off_in, sizeof(loff_t))) + return -EFAULT; + } else { + pos_in = fd_file(f_in)->f_pos; + } + + if (off_out) { + if (copy_from_user(&pos_out, off_out, sizeof(loff_t))) + return -EFAULT; + } else { + pos_out = fd_file(f_out)->f_pos; + } + + if (flags != 0) + return -EINVAL; + + ret = vfs_copy_file_range(fd_file(f_in), pos_in, fd_file(f_out), pos_out, len, + flags); + if (ret > 0) { + pos_in += ret; + pos_out += ret; + + if (off_in) { + if (copy_to_user(off_in, &pos_in, sizeof(loff_t))) + ret = -EFAULT; + } else { + fd_file(f_in)->f_pos = pos_in; + } + + if (off_out) { + if (copy_to_user(off_out, &pos_out, sizeof(loff_t))) + ret = -EFAULT; + } else { + fd_file(f_out)->f_pos = pos_out; + } + } + return ret; +} + +/* + * Don't operate on ranges the page cache doesn't support, and don't exceed the + * LFS limits. If pos is under the limit it becomes a short access. If it + * exceeds the limit we return -EFBIG. + */ +int generic_write_check_limits(struct file *file, loff_t pos, loff_t *count) +{ + struct inode *inode = file->f_mapping->host; + loff_t max_size = inode->i_sb->s_maxbytes; + loff_t limit = rlimit(RLIMIT_FSIZE); + + if (limit != RLIM_INFINITY) { + if (pos >= limit) { + send_sig(SIGXFSZ, current, 0); + return -EFBIG; + } + *count = min(*count, limit - pos); + } + + if (!(file->f_flags & O_LARGEFILE)) + max_size = MAX_NON_LFS; + + if (unlikely(pos >= max_size)) + return -EFBIG; + + *count = min(*count, max_size - pos); + + return 0; +} +EXPORT_SYMBOL_GPL(generic_write_check_limits); + +/* Like generic_write_checks(), but takes size of write instead of iter. */ +int generic_write_checks_count(struct kiocb *iocb, loff_t *count) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file->f_mapping->host; + + if (IS_SWAPFILE(inode)) + return -ETXTBSY; + + if (!*count) + return 0; + + if (iocb->ki_flags & IOCB_APPEND) + iocb->ki_pos = i_size_read(inode); + + if ((iocb->ki_flags & IOCB_NOWAIT) && + !((iocb->ki_flags & IOCB_DIRECT) || + (file->f_op->fop_flags & FOP_BUFFER_WASYNC))) + return -EINVAL; + + return generic_write_check_limits(iocb->ki_filp, iocb->ki_pos, count); +} +EXPORT_SYMBOL(generic_write_checks_count); + +/* + * Performs necessary checks before doing a write + * + * Can adjust writing position or amount of bytes to write. + * Returns appropriate error code that caller should return or + * zero in case that write should be allowed. + */ +ssize_t generic_write_checks(struct kiocb *iocb, struct iov_iter *from) +{ + loff_t count = iov_iter_count(from); + int ret; + + ret = generic_write_checks_count(iocb, &count); + if (ret) + return ret; + + iov_iter_truncate(from, count); + return iov_iter_count(from); +} +EXPORT_SYMBOL(generic_write_checks); + +/* + * Performs common checks before doing a file copy/clone + * from @file_in to @file_out. + */ +int generic_file_rw_checks(struct file *file_in, struct file *file_out) +{ + struct inode *inode_in = file_inode(file_in); + struct inode *inode_out = file_inode(file_out); + + /* Don't copy dirs, pipes, sockets... */ + if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode)) + return -EISDIR; + if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode)) + return -EINVAL; + + if (!(file_in->f_mode & FMODE_READ) || + !(file_out->f_mode & FMODE_WRITE) || + (file_out->f_flags & O_APPEND)) + return -EBADF; + + return 0; +} + +int generic_atomic_write_valid(struct kiocb *iocb, struct iov_iter *iter) +{ + size_t len = iov_iter_count(iter); + + if (!iter_is_ubuf(iter)) + return -EINVAL; + + if (!is_power_of_2(len)) + return -EINVAL; + + if (!IS_ALIGNED(iocb->ki_pos, len)) + return -EINVAL; + + if (!(iocb->ki_flags & IOCB_DIRECT)) + return -EOPNOTSUPP; + + return 0; +} +EXPORT_SYMBOL_GPL(generic_atomic_write_valid); diff --git a/fs/readdir.c b/fs/readdir.c new file mode 100644 index 000000000..76bb1ae3a --- /dev/null +++ b/fs/readdir.c @@ -0,0 +1,573 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/readdir.c + * + * Copyright (C) 1995 Linus Torvalds + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define dirent_size(dirent, len) offsetof(typeof(*(dirent)), d_name[len]) + +/* + * Some filesystems were never converted to '->iterate_shared()' + * and their directory iterators want the inode lock held for + * writing. This wrapper allows for converting from the shared + * semantics to the exclusive inode use. + */ +int wrap_directory_iterator(struct file *file, + struct dir_context *ctx, + int (*iter)(struct file *, struct dir_context *)) +{ + struct inode *inode = file_inode(file); + int ret; + + /* + * We'd love to have an 'inode_upgrade_trylock()' operation, + * see the comment in mmap_upgrade_trylock() in mm/memory.c. + * + * But considering this is for "filesystems that never got + * converted", it really doesn't matter. + * + * Also note that since we have to return with the lock held + * for reading, we can't use the "killable()" locking here, + * since we do need to get the lock even if we're dying. + * + * We could do the write part killably and then get the read + * lock unconditionally if it mattered, but see above on why + * this does the very simplistic conversion. + */ + up_read(&inode->i_rwsem); + down_write(&inode->i_rwsem); + + /* + * Since we dropped the inode lock, we should do the + * DEADDIR test again. See 'iterate_dir()' below. + * + * Note that we don't need to re-do the f_pos games, + * since the file must be locked wrt f_pos anyway. + */ + ret = -ENOENT; + if (!IS_DEADDIR(inode)) + ret = iter(file, ctx); + + downgrade_write(&inode->i_rwsem); + return ret; +} +EXPORT_SYMBOL(wrap_directory_iterator); + +/* + * Note the "unsafe_put_user()" semantics: we goto a + * label for errors. + */ +#define unsafe_copy_dirent_name(_dst, _src, _len, label) do { \ + char __user *dst = (_dst); \ + const char *src = (_src); \ + size_t len = (_len); \ + unsafe_put_user(0, dst+len, label); \ + unsafe_copy_to_user(dst, src, len, label); \ +} while (0) + + +int iterate_dir(struct file *file, struct dir_context *ctx) +{ + struct inode *inode = file_inode(file); + int res = -ENOTDIR; + + if (!file->f_op->iterate_shared) + goto out; + + res = security_file_permission(file, MAY_READ); + if (res) + goto out; + + res = fsnotify_file_perm(file, MAY_READ); + if (res) + goto out; + + res = down_read_killable(&inode->i_rwsem); + if (res) + goto out; + + res = -ENOENT; + if (!IS_DEADDIR(inode)) { + ctx->pos = file->f_pos; + res = file->f_op->iterate_shared(file, ctx); + file->f_pos = ctx->pos; + fsnotify_access(file); + file_accessed(file); + } + inode_unlock_shared(inode); +out: + return res; +} +EXPORT_SYMBOL(iterate_dir); + +/* + * POSIX says that a dirent name cannot contain NULL or a '/'. + * + * It's not 100% clear what we should really do in this case. + * The filesystem is clearly corrupted, but returning a hard + * error means that you now don't see any of the other names + * either, so that isn't a perfect alternative. + * + * And if you return an error, what error do you use? Several + * filesystems seem to have decided on EUCLEAN being the error + * code for EFSCORRUPTED, and that may be the error to use. Or + * just EIO, which is perhaps more obvious to users. + * + * In order to see the other file names in the directory, the + * caller might want to make this a "soft" error: skip the + * entry, and return the error at the end instead. + * + * Note that this should likely do a "memchr(name, 0, len)" + * check too, since that would be filesystem corruption as + * well. However, that case can't actually confuse user space, + * which has to do a strlen() on the name anyway to find the + * filename length, and the above "soft error" worry means + * that it's probably better left alone until we have that + * issue clarified. + * + * Note the PATH_MAX check - it's arbitrary but the real + * kernel limit on a possible path component, not NAME_MAX, + * which is the technical standard limit. + */ +static int verify_dirent_name(const char *name, int len) +{ + if (len <= 0 || len >= PATH_MAX) + return -EIO; + if (memchr(name, '/', len)) + return -EIO; + return 0; +} + +/* + * Traditional linux readdir() handling.. + * + * "count=1" is a special case, meaning that the buffer is one + * dirent-structure in size and that the code can't handle more + * anyway. Thus the special "fillonedir()" function for that + * case (the low-level handlers don't need to care about this). + */ + +#ifdef __ARCH_WANT_OLD_READDIR + +struct old_linux_dirent { + unsigned long d_ino; + unsigned long d_offset; + unsigned short d_namlen; + char d_name[]; +}; + +struct readdir_callback { + struct dir_context ctx; + struct old_linux_dirent __user * dirent; + int result; +}; + +static bool fillonedir(struct dir_context *ctx, const char *name, int namlen, + loff_t offset, u64 ino, unsigned int d_type) +{ + struct readdir_callback *buf = + container_of(ctx, struct readdir_callback, ctx); + struct old_linux_dirent __user * dirent; + unsigned long d_ino; + + if (buf->result) + return false; + buf->result = verify_dirent_name(name, namlen); + if (buf->result) + return false; + d_ino = ino; + if (sizeof(d_ino) < sizeof(ino) && d_ino != ino) { + buf->result = -EOVERFLOW; + return false; + } + buf->result++; + dirent = buf->dirent; + scoped_user_write_access_size(dirent, dirent_size(dirent, namlen + 1), efault) { + unsafe_put_user(d_ino, &dirent->d_ino, efault); + unsafe_put_user(offset, &dirent->d_offset, efault); + unsafe_put_user(namlen, &dirent->d_namlen, efault); + unsafe_copy_dirent_name(dirent->d_name, name, namlen, efault); + } + return true; +efault: + buf->result = -EFAULT; + return false; +} + +SYSCALL_DEFINE3(old_readdir, unsigned int, fd, + struct old_linux_dirent __user *, dirent, unsigned int, count) +{ + int error; + CLASS(fd_pos, f)(fd); + struct readdir_callback buf = { + .ctx.actor = fillonedir, + .ctx.count = 1, /* Hint to fs: just one entry. */ + .dirent = dirent + }; + + if (fd_empty(f)) + return -EBADF; + + error = iterate_dir(fd_file(f), &buf.ctx); + if (buf.result) + error = buf.result; + + return error; +} + +#endif /* __ARCH_WANT_OLD_READDIR */ + +/* + * New, all-improved, singing, dancing, iBCS2-compliant getdents() + * interface. + */ +struct linux_dirent { + unsigned long d_ino; + unsigned long d_off; + unsigned short d_reclen; + char d_name[]; +}; + +struct getdents_callback { + struct dir_context ctx; + struct linux_dirent __user * current_dir; + int prev_reclen; + int error; +}; + +static bool filldir(struct dir_context *ctx, const char *name, int namlen, + loff_t offset, u64 ino, unsigned int d_type) +{ + struct linux_dirent __user *dirent, *prev; + struct getdents_callback *buf = + container_of(ctx, struct getdents_callback, ctx); + unsigned long d_ino; + int reclen = ALIGN(dirent_size(dirent, namlen + 2), sizeof(long)); + int prev_reclen; + unsigned int flags = d_type; + + BUILD_BUG_ON(FILLDIR_FLAG_NOINTR & S_DT_MASK); + d_type &= S_DT_MASK; + + buf->error = verify_dirent_name(name, namlen); + if (unlikely(buf->error)) + return false; + buf->error = -EINVAL; /* only used if we fail.. */ + if (reclen > ctx->count) + return false; + d_ino = ino; + if (sizeof(d_ino) < sizeof(ino) && d_ino != ino) { + buf->error = -EOVERFLOW; + return false; + } + prev_reclen = buf->prev_reclen; + if (!(flags & FILLDIR_FLAG_NOINTR) && prev_reclen && signal_pending(current)) + return false; + dirent = buf->current_dir; + prev = (void __user *) dirent - prev_reclen; + scoped_user_write_access_size(prev, reclen + prev_reclen, efault) { + /* This might be 'dirent->d_off', but if so it will get overwritten */ + unsafe_put_user(offset, &prev->d_off, efault); + unsafe_put_user(d_ino, &dirent->d_ino, efault); + unsafe_put_user(reclen, &dirent->d_reclen, efault); + unsafe_put_user(d_type, (char __user *)dirent + reclen - 1, efault); + unsafe_copy_dirent_name(dirent->d_name, name, namlen, efault); + } + + buf->current_dir = (void __user *)dirent + reclen; + buf->prev_reclen = reclen; + ctx->count -= reclen; + return true; +efault: + buf->error = -EFAULT; + return false; +} + +SYSCALL_DEFINE3(getdents, unsigned int, fd, + struct linux_dirent __user *, dirent, unsigned int, count) +{ + CLASS(fd_pos, f)(fd); + struct getdents_callback buf = { + .ctx.actor = filldir, + .ctx.count = count, + .ctx.dt_flags_mask = FILLDIR_FLAG_NOINTR, + .current_dir = dirent + }; + int error; + + if (fd_empty(f)) + return -EBADF; + + error = iterate_dir(fd_file(f), &buf.ctx); + if (error >= 0) + error = buf.error; + if (buf.prev_reclen) { + struct linux_dirent __user * lastdirent; + lastdirent = (void __user *)buf.current_dir - buf.prev_reclen; + + if (put_user(buf.ctx.pos, &lastdirent->d_off)) + error = -EFAULT; + else + error = count - buf.ctx.count; + } + return error; +} + +struct getdents_callback64 { + struct dir_context ctx; + struct linux_dirent64 __user * current_dir; + int prev_reclen; + int error; +}; + +static bool filldir64(struct dir_context *ctx, const char *name, int namlen, + loff_t offset, u64 ino, unsigned int d_type) +{ + struct linux_dirent64 __user *dirent, *prev; + struct getdents_callback64 *buf = + container_of(ctx, struct getdents_callback64, ctx); + int reclen = ALIGN(dirent_size(dirent, namlen + 1), sizeof(u64)); + int prev_reclen; + unsigned int flags = d_type; + + BUILD_BUG_ON(FILLDIR_FLAG_NOINTR & S_DT_MASK); + d_type &= S_DT_MASK; + + buf->error = verify_dirent_name(name, namlen); + if (unlikely(buf->error)) + return false; + buf->error = -EINVAL; /* only used if we fail.. */ + if (reclen > ctx->count) + return false; + prev_reclen = buf->prev_reclen; + if (!(flags & FILLDIR_FLAG_NOINTR) && prev_reclen && signal_pending(current)) + return false; + dirent = buf->current_dir; + prev = (void __user *)dirent - prev_reclen; + scoped_user_write_access_size(prev, reclen + prev_reclen, efault) { + /* This might be 'dirent->d_off', but if so it will get overwritten */ + unsafe_put_user(offset, &prev->d_off, efault); + unsafe_put_user(ino, &dirent->d_ino, efault); + unsafe_put_user(reclen, &dirent->d_reclen, efault); + unsafe_put_user(d_type, &dirent->d_type, efault); + unsafe_copy_dirent_name(dirent->d_name, name, namlen, efault); + } + + buf->prev_reclen = reclen; + buf->current_dir = (void __user *)dirent + reclen; + ctx->count -= reclen; + return true; + +efault: + buf->error = -EFAULT; + return false; +} + +SYSCALL_DEFINE3(getdents64, unsigned int, fd, + struct linux_dirent64 __user *, dirent, unsigned int, count) +{ + CLASS(fd_pos, f)(fd); + struct getdents_callback64 buf = { + .ctx.actor = filldir64, + .ctx.count = count, + .ctx.dt_flags_mask = FILLDIR_FLAG_NOINTR, + .current_dir = dirent + }; + int error; + + if (fd_empty(f)) + return -EBADF; + + error = iterate_dir(fd_file(f), &buf.ctx); + if (error >= 0) + error = buf.error; + if (buf.prev_reclen) { + struct linux_dirent64 __user * lastdirent; + typeof(lastdirent->d_off) d_off = buf.ctx.pos; + + lastdirent = (void __user *) buf.current_dir - buf.prev_reclen; + if (put_user(d_off, &lastdirent->d_off)) + error = -EFAULT; + else + error = count - buf.ctx.count; + } + return error; +} + +#ifdef CONFIG_COMPAT +struct compat_old_linux_dirent { + compat_ulong_t d_ino; + compat_ulong_t d_offset; + unsigned short d_namlen; + char d_name[]; +}; + +struct compat_readdir_callback { + struct dir_context ctx; + struct compat_old_linux_dirent __user *dirent; + int result; +}; + +static bool compat_fillonedir(struct dir_context *ctx, const char *name, + int namlen, loff_t offset, u64 ino, + unsigned int d_type) +{ + struct compat_readdir_callback *buf = + container_of(ctx, struct compat_readdir_callback, ctx); + struct compat_old_linux_dirent __user *dirent; + compat_ulong_t d_ino; + + if (buf->result) + return false; + buf->result = verify_dirent_name(name, namlen); + if (buf->result) + return false; + d_ino = ino; + if (sizeof(d_ino) < sizeof(ino) && d_ino != ino) { + buf->result = -EOVERFLOW; + return false; + } + buf->result++; + dirent = buf->dirent; + scoped_user_write_access_size(dirent, dirent_size(dirent, namlen + 1), efault) { + unsafe_put_user(d_ino, &dirent->d_ino, efault); + unsafe_put_user(offset, &dirent->d_offset, efault); + unsafe_put_user(namlen, &dirent->d_namlen, efault); + unsafe_copy_dirent_name(dirent->d_name, name, namlen, efault); + } + return true; +efault: + buf->result = -EFAULT; + return false; +} + +COMPAT_SYSCALL_DEFINE3(old_readdir, unsigned int, fd, + struct compat_old_linux_dirent __user *, dirent, unsigned int, count) +{ + int error; + CLASS(fd_pos, f)(fd); + struct compat_readdir_callback buf = { + .ctx.actor = compat_fillonedir, + .ctx.count = 1, /* Hint to fs: just one entry. */ + .dirent = dirent + }; + + if (fd_empty(f)) + return -EBADF; + + error = iterate_dir(fd_file(f), &buf.ctx); + if (buf.result) + error = buf.result; + + return error; +} + +struct compat_linux_dirent { + compat_ulong_t d_ino; + compat_ulong_t d_off; + unsigned short d_reclen; + char d_name[]; +}; + +struct compat_getdents_callback { + struct dir_context ctx; + struct compat_linux_dirent __user *current_dir; + int prev_reclen; + int error; +}; + +static bool compat_filldir(struct dir_context *ctx, const char *name, int namlen, + loff_t offset, u64 ino, unsigned int d_type) +{ + struct compat_linux_dirent __user *dirent, *prev; + struct compat_getdents_callback *buf = + container_of(ctx, struct compat_getdents_callback, ctx); + compat_ulong_t d_ino; + int reclen = ALIGN(dirent_size(dirent, namlen + 2), sizeof(compat_long_t)); + int prev_reclen; + unsigned int flags = d_type; + + BUILD_BUG_ON(FILLDIR_FLAG_NOINTR & S_DT_MASK); + d_type &= S_DT_MASK; + + buf->error = verify_dirent_name(name, namlen); + if (unlikely(buf->error)) + return false; + buf->error = -EINVAL; /* only used if we fail.. */ + if (reclen > ctx->count) + return false; + d_ino = ino; + if (sizeof(d_ino) < sizeof(ino) && d_ino != ino) { + buf->error = -EOVERFLOW; + return false; + } + prev_reclen = buf->prev_reclen; + if (!(flags & FILLDIR_FLAG_NOINTR) && prev_reclen && signal_pending(current)) + return false; + dirent = buf->current_dir; + prev = (void __user *) dirent - prev_reclen; + scoped_user_write_access_size(prev, reclen + prev_reclen, efault) { + unsafe_put_user(offset, &prev->d_off, efault); + unsafe_put_user(d_ino, &dirent->d_ino, efault); + unsafe_put_user(reclen, &dirent->d_reclen, efault); + unsafe_put_user(d_type, (char __user *)dirent + reclen - 1, efault); + unsafe_copy_dirent_name(dirent->d_name, name, namlen, efault); + } + + buf->prev_reclen = reclen; + buf->current_dir = (void __user *)dirent + reclen; + ctx->count -= reclen; + return true; +efault: + buf->error = -EFAULT; + return false; +} + +COMPAT_SYSCALL_DEFINE3(getdents, unsigned int, fd, + struct compat_linux_dirent __user *, dirent, unsigned int, count) +{ + CLASS(fd_pos, f)(fd); + struct compat_getdents_callback buf = { + .ctx.actor = compat_filldir, + .ctx.count = count, + .ctx.dt_flags_mask = FILLDIR_FLAG_NOINTR, + .current_dir = dirent, + }; + int error; + + if (fd_empty(f)) + return -EBADF; + + error = iterate_dir(fd_file(f), &buf.ctx); + if (error >= 0) + error = buf.error; + if (buf.prev_reclen) { + struct compat_linux_dirent __user * lastdirent; + lastdirent = (void __user *)buf.current_dir - buf.prev_reclen; + + if (put_user(buf.ctx.pos, &lastdirent->d_off)) + error = -EFAULT; + else + error = count - buf.ctx.count; + } + return error; +} +#endif diff --git a/fs/remap_range.c b/fs/remap_range.c new file mode 100644 index 000000000..26afbbbfb --- /dev/null +++ b/fs/remap_range.c @@ -0,0 +1,567 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +#include +#include + +/* + * Performs necessary checks before doing a clone. + * + * Can adjust amount of bytes to clone via @req_count argument. + * Returns appropriate error code that caller should return or + * zero in case the clone should be allowed. + */ +static int generic_remap_checks(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + loff_t *req_count, unsigned int remap_flags) +{ + struct inode *inode_in = file_in->f_mapping->host; + struct inode *inode_out = file_out->f_mapping->host; + uint64_t count = *req_count; + uint64_t bcount; + loff_t size_in, size_out; + loff_t bs = inode_out->i_sb->s_blocksize; + int ret; + + /* The start of both ranges must be aligned to an fs block. */ + if (!IS_ALIGNED(pos_in, bs) || !IS_ALIGNED(pos_out, bs)) + return -EINVAL; + + /* Ensure offsets don't wrap. */ + if (pos_in + count < pos_in || pos_out + count < pos_out) + return -EINVAL; + + size_in = i_size_read(inode_in); + size_out = i_size_read(inode_out); + + /* Dedupe requires both ranges to be within EOF. */ + if ((remap_flags & REMAP_FILE_DEDUP) && + (pos_in >= size_in || pos_in + count > size_in || + pos_out >= size_out || pos_out + count > size_out)) + return -EINVAL; + + /* Ensure the infile range is within the infile. */ + if (pos_in >= size_in) + return -EINVAL; + count = min(count, size_in - (uint64_t)pos_in); + + ret = generic_write_check_limits(file_out, pos_out, &count); + if (ret) + return ret; + + /* + * If the user wanted us to link to the infile's EOF, round up to the + * next block boundary for this check. + * + * Otherwise, make sure the count is also block-aligned, having + * already confirmed the starting offsets' block alignment. + */ + if (pos_in + count == size_in && + (!(remap_flags & REMAP_FILE_DEDUP) || pos_out + count == size_out)) { + bcount = ALIGN(size_in, bs) - pos_in; + } else { + if (!IS_ALIGNED(count, bs)) + count = ALIGN_DOWN(count, bs); + bcount = count; + } + + /* Don't allow overlapped cloning within the same file. */ + if (inode_in == inode_out && + pos_out + bcount > pos_in && + pos_out < pos_in + bcount) + return -EINVAL; + + /* + * We shortened the request but the caller can't deal with that, so + * bounce the request back to userspace. + */ + if (*req_count != count && !(remap_flags & REMAP_FILE_CAN_SHORTEN)) + return -EINVAL; + + *req_count = count; + return 0; +} + +int remap_verify_area(struct file *file, loff_t pos, loff_t len, bool write) +{ + int mask = write ? MAY_WRITE : MAY_READ; + loff_t tmp; + int ret; + + if (unlikely(pos < 0 || len < 0)) + return -EINVAL; + + if (unlikely(check_add_overflow(pos, len, &tmp))) + return -EINVAL; + + ret = security_file_permission(file, mask); + if (ret) + return ret; + + return fsnotify_file_area_perm(file, mask, &pos, len); +} +EXPORT_SYMBOL_GPL(remap_verify_area); + +/* + * Ensure that we don't remap a partial EOF block in the middle of something + * else. Assume that the offsets have already been checked for block + * alignment. + * + * For clone we only link a partial EOF block above or at the destination file's + * EOF. For deduplication we accept a partial EOF block only if it ends at the + * destination file's EOF (can not link it into the middle of a file). + * + * Shorten the request if possible. + */ +static int generic_remap_check_len(struct inode *inode_in, + struct inode *inode_out, + loff_t pos_out, + loff_t *len, + unsigned int remap_flags) +{ + u64 blkmask = i_blocksize(inode_in) - 1; + loff_t new_len = *len; + + if ((*len & blkmask) == 0) + return 0; + + if (pos_out + *len < i_size_read(inode_out)) + new_len &= ~blkmask; + + if (new_len == *len) + return 0; + + if (remap_flags & REMAP_FILE_CAN_SHORTEN) { + *len = new_len; + return 0; + } + + return (remap_flags & REMAP_FILE_DEDUP) ? -EBADE : -EINVAL; +} + +/* Read a page's worth of file data into the page cache. */ +static struct folio *vfs_dedupe_get_folio(struct file *file, loff_t pos) +{ + return read_mapping_folio(file->f_mapping, pos >> PAGE_SHIFT, file); +} + +/* + * Lock two folios, ensuring that we lock in offset order if the folios + * are from the same file. + */ +static void vfs_lock_two_folios(struct folio *folio1, struct folio *folio2) +{ + /* Always lock in order of increasing index. */ + if (folio1->index > folio2->index) + swap(folio1, folio2); + + folio_lock(folio1); + if (folio1 != folio2) + folio_lock(folio2); +} + +/* Unlock two folios, being careful not to unlock the same folio twice. */ +static void vfs_unlock_two_folios(struct folio *folio1, struct folio *folio2) +{ + folio_unlock(folio1); + if (folio1 != folio2) + folio_unlock(folio2); +} + +/* + * Compare extents of two files to see if they are the same. + * Caller must have locked both inodes to prevent write races. + */ +static int vfs_dedupe_file_range_compare(struct file *src, loff_t srcoff, + struct file *dest, loff_t dstoff, + loff_t len, bool *is_same) +{ + bool same = true; + int error = -EINVAL; + + while (len) { + struct folio *src_folio, *dst_folio; + void *src_addr, *dst_addr; + loff_t cmp_len = min(PAGE_SIZE - offset_in_page(srcoff), + PAGE_SIZE - offset_in_page(dstoff)); + + cmp_len = min(cmp_len, len); + if (cmp_len <= 0) + goto out_error; + + src_folio = vfs_dedupe_get_folio(src, srcoff); + if (IS_ERR(src_folio)) { + error = PTR_ERR(src_folio); + goto out_error; + } + dst_folio = vfs_dedupe_get_folio(dest, dstoff); + if (IS_ERR(dst_folio)) { + error = PTR_ERR(dst_folio); + folio_put(src_folio); + goto out_error; + } + + vfs_lock_two_folios(src_folio, dst_folio); + + /* + * Now that we've locked both folios, make sure they're still + * mapped to the file data we're interested in. If not, + * someone is invalidating pages on us and we lose. + */ + if (!folio_test_uptodate(src_folio) || !folio_test_uptodate(dst_folio) || + src_folio->mapping != src->f_mapping || + dst_folio->mapping != dest->f_mapping) { + same = false; + goto unlock; + } + + src_addr = kmap_local_folio(src_folio, + offset_in_folio(src_folio, srcoff)); + dst_addr = kmap_local_folio(dst_folio, + offset_in_folio(dst_folio, dstoff)); + + flush_dcache_folio(src_folio); + flush_dcache_folio(dst_folio); + + if (memcmp(src_addr, dst_addr, cmp_len)) + same = false; + + kunmap_local(dst_addr); + kunmap_local(src_addr); +unlock: + vfs_unlock_two_folios(src_folio, dst_folio); + folio_put(dst_folio); + folio_put(src_folio); + + if (!same) + break; + + srcoff += cmp_len; + dstoff += cmp_len; + len -= cmp_len; + } + + *is_same = same; + return 0; + +out_error: + return error; +} + +/* + * Check that the two inodes are eligible for cloning, the ranges make + * sense, and then flush all dirty data. Caller must ensure that the + * inodes have been locked against any other modifications. + * + * If there's an error, then the usual negative error code is returned. + * Otherwise returns 0 with *len set to the request length. + */ +int +__generic_remap_file_range_prep(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + loff_t *len, unsigned int remap_flags, + const struct iomap_ops *dax_read_ops) +{ + struct inode *inode_in = file_inode(file_in); + struct inode *inode_out = file_inode(file_out); + bool same_inode = (inode_in == inode_out); + int ret; + + /* Don't touch certain kinds of inodes */ + if (IS_IMMUTABLE(inode_out)) + return -EPERM; + + if (IS_SWAPFILE(inode_in) || IS_SWAPFILE(inode_out)) + return -ETXTBSY; + + /* Don't reflink dirs, pipes, sockets... */ + if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode)) + return -EISDIR; + if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode)) + return -EINVAL; + + /* Zero length dedupe exits immediately; reflink goes to EOF. */ + if (*len == 0) { + loff_t isize = i_size_read(inode_in); + + if ((remap_flags & REMAP_FILE_DEDUP) || pos_in == isize) + return 0; + if (pos_in > isize) + return -EINVAL; + *len = isize - pos_in; + if (*len == 0) + return 0; + } + + /* Check that we don't violate system file offset limits. */ + ret = generic_remap_checks(file_in, pos_in, file_out, pos_out, len, + remap_flags); + if (ret || *len == 0) + return ret; + + /* Wait for the completion of any pending IOs on both files */ + inode_dio_wait(inode_in); + if (!same_inode) + inode_dio_wait(inode_out); + + ret = filemap_write_and_wait_range(inode_in->i_mapping, + pos_in, pos_in + *len - 1); + if (ret) + return ret; + + ret = filemap_write_and_wait_range(inode_out->i_mapping, + pos_out, pos_out + *len - 1); + if (ret) + return ret; + + /* + * Check that the extents are the same. + */ + if (remap_flags & REMAP_FILE_DEDUP) { + bool is_same = false; + + if (!IS_DAX(inode_in)) + ret = vfs_dedupe_file_range_compare(file_in, pos_in, + file_out, pos_out, *len, &is_same); + else if (dax_read_ops) + ret = dax_dedupe_file_range_compare(inode_in, pos_in, + inode_out, pos_out, *len, &is_same, + dax_read_ops); + else + return -EINVAL; + if (ret) + return ret; + if (!is_same) + return -EBADE; + } + + ret = generic_remap_check_len(inode_in, inode_out, pos_out, len, + remap_flags); + if (ret || *len == 0) + return ret; + + /* If can't alter the file contents, we're done. */ + if (!(remap_flags & REMAP_FILE_DEDUP)) + ret = file_modified(file_out); + + return ret; +} + +int generic_remap_file_range_prep(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + loff_t *len, unsigned int remap_flags) +{ + return __generic_remap_file_range_prep(file_in, pos_in, file_out, + pos_out, len, remap_flags, NULL); +} +EXPORT_SYMBOL(generic_remap_file_range_prep); + +loff_t vfs_clone_file_range(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + loff_t len, unsigned int remap_flags) +{ + loff_t ret; + + WARN_ON_ONCE(remap_flags & REMAP_FILE_DEDUP); + + if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb) + return -EXDEV; + + ret = generic_file_rw_checks(file_in, file_out); + if (ret < 0) + return ret; + + if (!file_in->f_op->remap_file_range) + return -EOPNOTSUPP; + + ret = remap_verify_area(file_in, pos_in, len, false); + if (ret) + return ret; + + ret = remap_verify_area(file_out, pos_out, len, true); + if (ret) + return ret; + + file_start_write(file_out); + ret = file_in->f_op->remap_file_range(file_in, pos_in, + file_out, pos_out, len, remap_flags); + file_end_write(file_out); + if (ret < 0) + return ret; + + fsnotify_access(file_in); + fsnotify_modify(file_out); + return ret; +} +EXPORT_SYMBOL(vfs_clone_file_range); + +/* Check whether we are allowed to dedupe the destination file */ +static bool may_dedupe_file(struct file *file) +{ + struct mnt_idmap *idmap = file_mnt_idmap(file); + struct inode *inode = file_inode(file); + + if (capable(CAP_SYS_ADMIN)) + return true; + if (file->f_mode & FMODE_WRITE) + return true; + if (vfsuid_eq_kuid(i_uid_into_vfsuid(idmap, inode), current_fsuid())) + return true; + if (!inode_permission(idmap, inode, MAY_WRITE)) + return true; + return false; +} + +loff_t vfs_dedupe_file_range_one(struct file *src_file, loff_t src_pos, + struct file *dst_file, loff_t dst_pos, + loff_t len, unsigned int remap_flags) +{ + loff_t ret; + + WARN_ON_ONCE(remap_flags & ~(REMAP_FILE_DEDUP | + REMAP_FILE_CAN_SHORTEN)); + + /* + * This is redundant if called from vfs_dedupe_file_range(), but other + * callers need it and it's not performance sesitive... + */ + ret = remap_verify_area(src_file, src_pos, len, false); + if (ret) + return ret; + + ret = remap_verify_area(dst_file, dst_pos, len, true); + if (ret) + return ret; + + /* + * This needs to be called after remap_verify_area() because of + * sb_start_write() and before may_dedupe_file() because the mount's + * MAY_WRITE need to be checked with mnt_get_write_access_file() held. + */ + ret = mnt_want_write_file(dst_file); + if (ret) + return ret; + + ret = -EPERM; + if (!may_dedupe_file(dst_file)) + goto out_drop_write; + + ret = -EXDEV; + if (file_inode(src_file)->i_sb != file_inode(dst_file)->i_sb) + goto out_drop_write; + + ret = -EISDIR; + if (S_ISDIR(file_inode(dst_file)->i_mode)) + goto out_drop_write; + + ret = -EINVAL; + if (!dst_file->f_op->remap_file_range) + goto out_drop_write; + + if (len == 0) { + ret = 0; + goto out_drop_write; + } + + ret = dst_file->f_op->remap_file_range(src_file, src_pos, dst_file, + dst_pos, len, remap_flags | REMAP_FILE_DEDUP); +out_drop_write: + mnt_drop_write_file(dst_file); + + return ret; +} +EXPORT_SYMBOL(vfs_dedupe_file_range_one); + +int vfs_dedupe_file_range(struct file *file, struct file_dedupe_range *same) +{ + struct file_dedupe_range_info *info; + struct inode *src = file_inode(file); + u64 off; + u64 len; + int i; + int ret; + u16 count = same->dest_count; + loff_t deduped; + + if (!(file->f_mode & FMODE_READ)) + return -EINVAL; + + if (same->reserved1 || same->reserved2) + return -EINVAL; + + off = same->src_offset; + len = same->src_length; + + if (S_ISDIR(src->i_mode)) + return -EISDIR; + + if (!S_ISREG(src->i_mode)) + return -EINVAL; + + if (!file->f_op->remap_file_range) + return -EOPNOTSUPP; + + ret = remap_verify_area(file, off, len, false); + if (ret < 0) + return ret; + ret = 0; + + if (off + len > i_size_read(src)) + return -EINVAL; + + /* Arbitrary 1G limit on a single dedupe request, can be raised. */ + len = min_t(u64, len, 1 << 30); + + /* pre-format output fields to sane values */ + for (i = 0; i < count; i++) { + same->info[i].bytes_deduped = 0ULL; + same->info[i].status = FILE_DEDUPE_RANGE_SAME; + } + + for (i = 0, info = same->info; i < count; i++, info++) { + CLASS(fd, dst_fd)(info->dest_fd); + + if (fd_empty(dst_fd)) { + info->status = -EBADF; + goto next_loop; + } + + if (info->reserved) { + info->status = -EINVAL; + goto next_loop; + } + + deduped = vfs_dedupe_file_range_one(file, off, fd_file(dst_fd), + info->dest_offset, len, + REMAP_FILE_CAN_SHORTEN); + if (deduped == -EBADE) + info->status = FILE_DEDUPE_RANGE_DIFFERS; + else if (deduped < 0) + info->status = deduped; + else + info->bytes_deduped = len; + +next_loop: + if (fatal_signal_pending(current)) + break; + } + return ret; +} +EXPORT_SYMBOL(vfs_dedupe_file_range); diff --git a/fs/resctrl/Kconfig b/fs/resctrl/Kconfig new file mode 100644 index 000000000..21671301b --- /dev/null +++ b/fs/resctrl/Kconfig @@ -0,0 +1,39 @@ +config RESCTRL_FS + bool "CPU Resource Control Filesystem (resctrl)" + depends on ARCH_HAS_CPU_RESCTRL + select KERNFS + select PROC_CPU_RESCTRL if PROC_FS + help + Some architectures provide hardware facilities to group tasks and + monitor and control their usage of memory system resources such as + caches and memory bandwidth. Examples of such facilities include + Intel's Resource Director Technology (Intel(R) RDT) and AMD's + Platform Quality of Service (AMD QoS). + + If your system has the necessary support and you want to be able to + assign tasks to groups and manipulate the associated resource + monitors and controls from userspace, say Y here to get a mountable + 'resctrl' filesystem that lets you do just that. + + If nothing mounts or prods the 'resctrl' filesystem, resource + controls and monitors are left in a quiescent, permissive state. + + On architectures where this can be disabled independently, it is + safe to say N. + + See for more information. + +config RESCTRL_FS_PSEUDO_LOCK + bool + depends on RESCTRL_FS + help + Software mechanism to pin data in a cache portion using + micro-architecture specific knowledge. + +config RESCTRL_RMID_DEPENDS_ON_CLOSID + bool + depends on RESCTRL_FS + help + Enabled by the architecture when the RMID values depend on the CLOSID. + This causes the CLOSID allocator to search for CLOSID with clean + RMID. diff --git a/fs/resctrl/Makefile b/fs/resctrl/Makefile new file mode 100644 index 000000000..e67f34d22 --- /dev/null +++ b/fs/resctrl/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_RESCTRL_FS) += rdtgroup.o ctrlmondata.o monitor.o +obj-$(CONFIG_RESCTRL_FS_PSEUDO_LOCK) += pseudo_lock.o + +# To allow define_trace.h's recursive include: +CFLAGS_monitor.o = -I$(src) diff --git a/fs/resctrl/ctrlmondata.c b/fs/resctrl/ctrlmondata.c new file mode 100644 index 000000000..18ec9f564 --- /dev/null +++ b/fs/resctrl/ctrlmondata.c @@ -0,0 +1,1075 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Resource Director Technology(RDT) + * - Cache Allocation code. + * + * Copyright (C) 2016 Intel Corporation + * + * Authors: + * Fenghua Yu + * Tony Luck + * + * More information about RDT be found in the Intel (R) x86 Architecture + * Software Developer Manual June 2016, volume 3, section 17.17. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +struct rdt_parse_data { + u32 closid; + enum rdtgrp_mode mode; + char *buf; +}; + +typedef int (ctrlval_parser_t)(struct rdt_parse_data *data, + struct resctrl_schema *s, + struct rdt_ctrl_domain *d); + +/* + * Check whether MBA bandwidth percentage value is correct. The value is + * checked against the minimum and max bandwidth values specified by the + * hardware. The allocated bandwidth percentage is rounded to the next + * control step available on the hardware. + */ +static bool bw_validate(char *buf, u32 *data, struct rdt_resource *r) +{ + int ret; + u32 bw; + + /* + * Only linear delay values is supported for current Intel SKUs. + */ + if (!r->membw.delay_linear && r->membw.arch_needs_linear) { + rdt_last_cmd_puts("No support for non-linear MB domains\n"); + return false; + } + + ret = kstrtou32(buf, 10, &bw); + if (ret) { + rdt_last_cmd_printf("Invalid MB value %s\n", buf); + return false; + } + + /* Nothing else to do if software controller is enabled. */ + if (is_mba_sc(r)) { + *data = bw; + return true; + } + + if (bw < r->membw.min_bw || bw > r->membw.max_bw) { + rdt_last_cmd_printf("MB value %u out of range [%d,%d]\n", + bw, r->membw.min_bw, r->membw.max_bw); + return false; + } + + *data = roundup(bw, (unsigned long)r->membw.bw_gran); + return true; +} + +static int parse_bw(struct rdt_parse_data *data, struct resctrl_schema *s, + struct rdt_ctrl_domain *d) +{ + struct resctrl_staged_config *cfg; + struct rdt_resource *r = s->res; + u32 closid = data->closid; + u32 bw_val; + + cfg = &d->staged_config[s->conf_type]; + if (cfg->have_new_ctrl) { + rdt_last_cmd_printf("Duplicate domain %d\n", d->hdr.id); + return -EINVAL; + } + + if (!bw_validate(data->buf, &bw_val, r)) + return -EINVAL; + + if (is_mba_sc(r)) { + d->mbps_val[closid] = bw_val; + return 0; + } + + cfg->new_ctrl = bw_val; + cfg->have_new_ctrl = true; + + return 0; +} + +/* + * Check whether a cache bit mask is valid. + * On Intel CPUs, non-contiguous 1s value support is indicated by CPUID: + * - CPUID.0x10.1:ECX[3]: L3 non-contiguous 1s value supported if 1 + * - CPUID.0x10.2:ECX[3]: L2 non-contiguous 1s value supported if 1 + * + * Haswell does not support a non-contiguous 1s value and additionally + * requires at least two bits set. + * AMD allows non-contiguous bitmasks. + */ +static bool cbm_validate(char *buf, u32 *data, struct rdt_resource *r) +{ + u32 supported_bits = BIT_MASK(r->cache.cbm_len) - 1; + unsigned int cbm_len = r->cache.cbm_len; + unsigned long first_bit, zero_bit, val; + int ret; + + ret = kstrtoul(buf, 16, &val); + if (ret) { + rdt_last_cmd_printf("Non-hex character in the mask %s\n", buf); + return false; + } + + if ((r->cache.min_cbm_bits > 0 && val == 0) || val > supported_bits) { + rdt_last_cmd_puts("Mask out of range\n"); + return false; + } + + first_bit = find_first_bit(&val, cbm_len); + zero_bit = find_next_zero_bit(&val, cbm_len, first_bit); + + /* Are non-contiguous bitmasks allowed? */ + if (!r->cache.arch_has_sparse_bitmasks && + (find_next_bit(&val, cbm_len, zero_bit) < cbm_len)) { + rdt_last_cmd_printf("The mask %lx has non-consecutive 1-bits\n", val); + return false; + } + + if ((zero_bit - first_bit) < r->cache.min_cbm_bits) { + rdt_last_cmd_printf("Need at least %d bits in the mask\n", + r->cache.min_cbm_bits); + return false; + } + + *data = val; + return true; +} + +/* + * Read one cache bit mask (hex). Check that it is valid for the current + * resource type. + */ +static int parse_cbm(struct rdt_parse_data *data, struct resctrl_schema *s, + struct rdt_ctrl_domain *d) +{ + enum rdtgrp_mode mode = data->mode; + struct resctrl_staged_config *cfg; + struct rdt_resource *r = s->res; + u32 closid = data->closid; + u32 cbm_val; + + cfg = &d->staged_config[s->conf_type]; + if (cfg->have_new_ctrl) { + rdt_last_cmd_printf("Duplicate domain %d\n", d->hdr.id); + return -EINVAL; + } + + /* + * Cannot set up more than one pseudo-locked region in a cache + * hierarchy. + */ + if (mode == RDT_MODE_PSEUDO_LOCKSETUP && + rdtgroup_pseudo_locked_in_hierarchy(d)) { + rdt_last_cmd_puts("Pseudo-locked region in hierarchy\n"); + return -EINVAL; + } + + if (!cbm_validate(data->buf, &cbm_val, r)) + return -EINVAL; + + if ((mode == RDT_MODE_EXCLUSIVE || mode == RDT_MODE_SHAREABLE) && + rdtgroup_cbm_overlaps_pseudo_locked(d, cbm_val)) { + rdt_last_cmd_puts("CBM overlaps with pseudo-locked region\n"); + return -EINVAL; + } + + /* + * The CBM may not overlap with the CBM of another closid if + * either is exclusive. + */ + if (rdtgroup_cbm_overlaps(s, d, cbm_val, closid, true)) { + rdt_last_cmd_puts("Overlaps with exclusive group\n"); + return -EINVAL; + } + + if (rdtgroup_cbm_overlaps(s, d, cbm_val, closid, false)) { + if (mode == RDT_MODE_EXCLUSIVE || + mode == RDT_MODE_PSEUDO_LOCKSETUP) { + rdt_last_cmd_puts("Overlaps with other group\n"); + return -EINVAL; + } + } + + cfg->new_ctrl = cbm_val; + cfg->have_new_ctrl = true; + + return 0; +} + +/* + * For each domain in this resource we expect to find a series of: + * id=mask + * separated by ";". The "id" is in decimal, and must match one of + * the "id"s for this resource. + */ +static int parse_line(char *line, struct resctrl_schema *s, + struct rdtgroup *rdtgrp) +{ + enum resctrl_conf_type t = s->conf_type; + ctrlval_parser_t *parse_ctrlval = NULL; + struct resctrl_staged_config *cfg; + struct rdt_resource *r = s->res; + struct rdt_parse_data data; + struct rdt_ctrl_domain *d; + char *dom = NULL, *id; + unsigned long dom_id; + + /* Walking r->domains, ensure it can't race with cpuhp */ + lockdep_assert_cpus_held(); + + switch (r->schema_fmt) { + case RESCTRL_SCHEMA_BITMAP: + parse_ctrlval = &parse_cbm; + break; + case RESCTRL_SCHEMA_RANGE: + parse_ctrlval = &parse_bw; + break; + } + + if (WARN_ON_ONCE(!parse_ctrlval)) + return -EINVAL; + + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP && + (r->rid == RDT_RESOURCE_MBA || r->rid == RDT_RESOURCE_SMBA)) { + rdt_last_cmd_puts("Cannot pseudo-lock MBA resource\n"); + return -EINVAL; + } + +next: + if (!line || line[0] == '\0') + return 0; + dom = strsep(&line, ";"); + id = strsep(&dom, "="); + if (!dom || kstrtoul(id, 10, &dom_id)) { + rdt_last_cmd_puts("Missing '=' or non-numeric domain\n"); + return -EINVAL; + } + dom = strim(dom); + list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) { + if (d->hdr.id == dom_id) { + data.buf = dom; + data.closid = rdtgrp->closid; + data.mode = rdtgrp->mode; + if (parse_ctrlval(&data, s, d)) + return -EINVAL; + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { + cfg = &d->staged_config[t]; + /* + * In pseudo-locking setup mode and just + * parsed a valid CBM that should be + * pseudo-locked. Only one locked region per + * resource group and domain so just do + * the required initialization for single + * region and return. + */ + rdtgrp->plr->s = s; + rdtgrp->plr->d = d; + rdtgrp->plr->cbm = cfg->new_ctrl; + d->plr = rdtgrp->plr; + return 0; + } + goto next; + } + } + return -EINVAL; +} + +static int rdtgroup_parse_resource(char *resname, char *tok, + struct rdtgroup *rdtgrp) +{ + struct resctrl_schema *s; + + list_for_each_entry(s, &resctrl_schema_all, list) { + if (!strcmp(resname, s->name) && rdtgrp->closid < s->num_closid) + return parse_line(tok, s, rdtgrp); + } + rdt_last_cmd_printf("Unknown or unsupported resource name '%s'\n", resname); + return -EINVAL; +} + +ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off) +{ + struct resctrl_schema *s; + struct rdtgroup *rdtgrp; + struct rdt_resource *r; + char *tok, *resname; + int ret = 0; + + rdtgrp = rdtgroup_kn_lock_live(of->kn); + if (!rdtgrp) { + rdtgroup_kn_unlock(of->kn); + return -ENOENT; + } + + /* Valid input requires a trailing newline */ + if (nbytes == 0 || buf[nbytes - 1] != '\n') { + rdt_last_cmd_puts("schemata: Invalid input\n"); + ret = -EINVAL; + goto out_unlock; + } + + buf[nbytes - 1] = '\0'; + + /* + * No changes to pseudo-locked region allowed. It has to be removed + * and re-created instead. + */ + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) { + ret = -EINVAL; + rdt_last_cmd_puts("Resource group is pseudo-locked\n"); + goto out_unlock; + } + + rdt_staged_configs_clear(); + + while ((tok = strsep(&buf, "\n")) != NULL) { + resname = strim(strsep(&tok, ":")); + if (!tok) { + rdt_last_cmd_puts("Missing ':'\n"); + ret = -EINVAL; + goto out_clear_staged; + } + if (tok[0] == '\0') { + rdt_last_cmd_printf("Missing '%s' value\n", resname); + ret = -EINVAL; + goto out_clear_staged; + } + ret = rdtgroup_parse_resource(resname, tok, rdtgrp); + if (ret) + goto out_clear_staged; + } + + list_for_each_entry(s, &resctrl_schema_all, list) { + r = s->res; + + /* + * Writes to mba_sc resources update the software controller, + * not the control MSR. + */ + if (is_mba_sc(r)) + continue; + + ret = resctrl_arch_update_domains(r, rdtgrp->closid); + if (ret) + goto out_clear_staged; + } + + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { + /* + * If pseudo-locking fails we keep the resource group in + * mode RDT_MODE_PSEUDO_LOCKSETUP with its class of service + * active and updated for just the domain the pseudo-locked + * region was requested for. + */ + ret = rdtgroup_pseudo_lock_create(rdtgrp); + } + +out_clear_staged: + rdt_staged_configs_clear(); +out_unlock: + rdtgroup_kn_unlock(of->kn); + return ret ?: nbytes; +} + +static void show_doms(struct seq_file *s, struct resctrl_schema *schema, + char *resource_name, int closid) +{ + struct rdt_resource *r = schema->res; + struct rdt_ctrl_domain *dom; + bool sep = false; + u32 ctrl_val; + + /* Walking r->domains, ensure it can't race with cpuhp */ + lockdep_assert_cpus_held(); + + if (resource_name) + seq_printf(s, "%*s:", max_name_width, resource_name); + list_for_each_entry_rcu(dom, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) { + if (sep) + seq_puts(s, ";"); + + if (is_mba_sc(r)) + ctrl_val = dom->mbps_val[closid]; + else + ctrl_val = resctrl_arch_get_config(r, dom, closid, + schema->conf_type); + + seq_printf(s, schema->fmt_str, dom->hdr.id, ctrl_val); + sep = true; + } + seq_puts(s, "\n"); +} + +int rdtgroup_schemata_show(struct kernfs_open_file *of, + struct seq_file *s, void *v) +{ + struct resctrl_schema *schema; + struct rdtgroup *rdtgrp; + int ret = 0; + u32 closid; + + rdtgrp = rdtgroup_kn_lock_live(of->kn); + if (rdtgrp) { + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { + list_for_each_entry(schema, &resctrl_schema_all, list) { + seq_printf(s, "%s:uninitialized\n", schema->name); + } + } else if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) { + if (!rdtgrp->plr->d) { + rdt_last_cmd_puts("Cache domain offline\n"); + ret = -ENODEV; + } else { + seq_printf(s, "%s:%d=%x\n", + rdtgrp->plr->s->res->name, + rdtgrp->plr->d->hdr.id, + rdtgrp->plr->cbm); + } + } else { + closid = rdtgrp->closid; + list_for_each_entry(schema, &resctrl_schema_all, list) { + if (closid < schema->num_closid) + show_doms(s, schema, schema->name, closid); + } + } + } else { + ret = -ENOENT; + } + rdtgroup_kn_unlock(of->kn); + return ret; +} + +static int smp_mon_event_count(void *arg) +{ + mon_event_count(arg); + + return 0; +} + +ssize_t rdtgroup_mba_mbps_event_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off) +{ + struct rdtgroup *rdtgrp; + int ret = 0; + + rdtgrp = rdtgroup_kn_lock_live(of->kn); + if (!rdtgrp) { + rdtgroup_kn_unlock(of->kn); + return -ENOENT; + } + + /* Valid input requires a trailing newline */ + if (nbytes == 0 || buf[nbytes - 1] != '\n') { + rdt_last_cmd_puts("mba_MBps_event: Invalid input\n"); + ret = -EINVAL; + goto out_unlock; + } + + buf[nbytes - 1] = '\0'; + + if (!strcmp(buf, "mbm_local_bytes")) { + if (resctrl_is_mon_event_enabled(QOS_L3_MBM_LOCAL_EVENT_ID)) + rdtgrp->mba_mbps_event = QOS_L3_MBM_LOCAL_EVENT_ID; + else + ret = -EINVAL; + } else if (!strcmp(buf, "mbm_total_bytes")) { + if (resctrl_is_mon_event_enabled(QOS_L3_MBM_TOTAL_EVENT_ID)) + rdtgrp->mba_mbps_event = QOS_L3_MBM_TOTAL_EVENT_ID; + else + ret = -EINVAL; + } else { + ret = -EINVAL; + } + + if (ret) + rdt_last_cmd_printf("Unsupported event id '%s'\n", buf); + +out_unlock: + rdtgroup_kn_unlock(of->kn); + + return ret ?: nbytes; +} + +int rdtgroup_mba_mbps_event_show(struct kernfs_open_file *of, + struct seq_file *s, void *v) +{ + struct rdtgroup *rdtgrp; + int ret = 0; + + rdtgrp = rdtgroup_kn_lock_live(of->kn); + + if (rdtgrp) { + switch (rdtgrp->mba_mbps_event) { + case QOS_L3_MBM_LOCAL_EVENT_ID: + seq_puts(s, "mbm_local_bytes\n"); + break; + case QOS_L3_MBM_TOTAL_EVENT_ID: + seq_puts(s, "mbm_total_bytes\n"); + break; + default: + pr_warn_once("Bad event %d\n", rdtgrp->mba_mbps_event); + ret = -EINVAL; + break; + } + } else { + ret = -ENOENT; + } + + rdtgroup_kn_unlock(of->kn); + + return ret; +} + +struct rdt_domain_hdr *resctrl_find_domain(struct list_head *h, int id, + struct list_head **pos) +{ + struct rdt_domain_hdr *d; + struct list_head *l; + + lockdep_assert_cpus_held(); + + list_for_each(l, h) { + d = list_entry(l, struct rdt_domain_hdr, list); + /* When id is found, return its domain. */ + if (id == d->id) + return d; + /* Stop searching when finding id's position in sorted list. */ + if (id < d->id) + break; + } + + if (pos) + *pos = l; + + return NULL; +} + +void mon_event_read(struct rmid_read *rr, struct rdt_resource *r, + struct rdt_domain_hdr *hdr, struct rdtgroup *rdtgrp, + cpumask_t *cpumask, struct mon_evt *evt, int first) +{ + int cpu; + + /* When picking a CPU from cpu_mask, ensure it can't race with cpuhp */ + lockdep_assert_cpus_held(); + + /* + * Setup the parameters to pass to mon_event_count() to read the data. + */ + rr->rgrp = rdtgrp; + rr->evt = evt; + rr->r = r; + rr->hdr = hdr; + rr->first = first; + if (resctrl_arch_mbm_cntr_assign_enabled(r) && + resctrl_is_mbm_event(evt->evtid)) { + rr->is_mbm_cntr = true; + } else { + rr->arch_mon_ctx = resctrl_arch_mon_ctx_alloc(r, evt->evtid); + if (IS_ERR(rr->arch_mon_ctx)) { + rr->err = -EINVAL; + return; + } + } + + if (evt->any_cpu) { + mon_event_count(rr); + goto out_ctx_free; + } + + cpu = cpumask_any_housekeeping(cpumask, RESCTRL_PICK_ANY_CPU); + + /* + * cpumask_any_housekeeping() prefers housekeeping CPUs, but + * are all the CPUs nohz_full? If yes, pick a CPU to IPI. + * MPAM's resctrl_arch_rmid_read() is unable to read the + * counters on some platforms if its called in IRQ context. + */ + if (tick_nohz_full_cpu(cpu)) + smp_call_function_any(cpumask, mon_event_count, rr, 1); + else + smp_call_on_cpu(cpu, smp_mon_event_count, rr, false); + +out_ctx_free: + if (rr->arch_mon_ctx) + resctrl_arch_mon_ctx_free(r, evt->evtid, rr->arch_mon_ctx); +} + +/* + * Decimal place precision to use for each number of fixed-point + * binary bits computed from ceil(binary_bits * log10(2)) except + * binary_bits == 0 which will print "value.0" + */ +static const unsigned int decplaces[MAX_BINARY_BITS + 1] = { + [0] = 1, + [1] = 1, + [2] = 1, + [3] = 1, + [4] = 2, + [5] = 2, + [6] = 2, + [7] = 3, + [8] = 3, + [9] = 3, + [10] = 4, + [11] = 4, + [12] = 4, + [13] = 4, + [14] = 5, + [15] = 5, + [16] = 5, + [17] = 6, + [18] = 6, + [19] = 6, + [20] = 7, + [21] = 7, + [22] = 7, + [23] = 7, + [24] = 8, + [25] = 8, + [26] = 8, + [27] = 9 +}; + +static void print_event_value(struct seq_file *m, unsigned int binary_bits, u64 val) +{ + unsigned long long frac = 0; + + if (binary_bits) { + /* Mask off the integer part of the fixed-point value. */ + frac = val & GENMASK_ULL(binary_bits - 1, 0); + + /* + * Multiply by 10^{desired decimal places}. The integer part of + * the fixed point value is now almost what is needed. + */ + frac *= int_pow(10ull, decplaces[binary_bits]); + + /* + * Round to nearest by adding a value that would be a "1" in the + * binary_bits + 1 place. Integer part of fixed point value is + * now the needed value. + */ + frac += 1ull << (binary_bits - 1); + + /* + * Extract the integer part of the value. This is the decimal + * representation of the original fixed-point fractional value. + */ + frac >>= binary_bits; + } + + /* + * "frac" is now in the range [0 .. 10^decplaces). I.e. string + * representation will fit into chosen number of decimal places. + */ + seq_printf(m, "%llu.%0*llu\n", val >> binary_bits, decplaces[binary_bits], frac); +} + +int rdtgroup_mondata_show(struct seq_file *m, void *arg) +{ + struct kernfs_open_file *of = m->private; + enum resctrl_res_level resid; + struct rdt_domain_hdr *hdr; + struct rmid_read rr = {0}; + struct rdtgroup *rdtgrp; + int domid, cpu, ret = 0; + struct rdt_resource *r; + struct cacheinfo *ci; + struct mon_evt *evt; + struct mon_data *md; + + rdtgrp = rdtgroup_kn_lock_live(of->kn); + if (!rdtgrp) { + ret = -ENOENT; + goto out; + } + + md = of->kn->priv; + if (WARN_ON_ONCE(!md)) { + ret = -EIO; + goto out; + } + + resid = md->rid; + domid = md->domid; + evt = md->evt; + r = resctrl_arch_get_resource(resid); + + if (md->sum) { + struct rdt_l3_mon_domain *d; + + if (WARN_ON_ONCE(resid != RDT_RESOURCE_L3)) { + ret = -EINVAL; + goto out; + } + + /* + * This file requires summing across all domains that share + * the L3 cache id that was provided in the "domid" field of the + * struct mon_data. Search all domains in the resource for + * one that matches this cache id. + */ + list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) { + if (d->ci_id == domid) { + cpu = cpumask_any(&d->hdr.cpu_mask); + ci = get_cpu_cacheinfo_level(cpu, RESCTRL_L3_CACHE); + if (!ci) + continue; + rr.ci = ci; + mon_event_read(&rr, r, NULL, rdtgrp, + &ci->shared_cpu_map, evt, false); + goto checkresult; + } + } + ret = -ENOENT; + goto out; + } else { + /* + * This file provides data from a single domain. Search + * the resource to find the domain with "domid". + */ + hdr = resctrl_find_domain(&r->mon_domains, domid, NULL); + if (!hdr) { + ret = -ENOENT; + goto out; + } + mon_event_read(&rr, r, hdr, rdtgrp, &hdr->cpu_mask, evt, false); + } + +checkresult: + + /* + * -ENOENT is a special case, set only when "mbm_event" counter assignment + * mode is enabled and no counter has been assigned. + */ + if (rr.err == -EIO) + seq_puts(m, "Error\n"); + else if (rr.err == -EINVAL) + seq_puts(m, "Unavailable\n"); + else if (rr.err == -ENOENT) + seq_puts(m, "Unassigned\n"); + else if (evt->is_floating_point) + print_event_value(m, evt->binary_bits, rr.val); + else + seq_printf(m, "%llu\n", rr.val); + +out: + rdtgroup_kn_unlock(of->kn); + return ret; +} + +int resctrl_io_alloc_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) +{ + struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); + struct rdt_resource *r; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + + r = s->res; + if (r->cache.io_alloc_capable) { + if (resctrl_arch_get_io_alloc_enabled(r)) + seq_puts(seq, "enabled\n"); + else + seq_puts(seq, "disabled\n"); + } else { + seq_puts(seq, "not supported\n"); + } + + info_kn_unlock(of->kn); + + return 0; +} + +/* + * resctrl_io_alloc_closid_supported() - io_alloc feature utilizes the + * highest CLOSID value to direct I/O traffic. Ensure that io_alloc_closid + * is in the supported range. + */ +static bool resctrl_io_alloc_closid_supported(u32 io_alloc_closid) +{ + return io_alloc_closid < closids_supported(); +} + +/* + * Initialize io_alloc CLOSID cache resource CBM with all usable (shared + * and unused) cache portions. + */ +static int resctrl_io_alloc_init_cbm(struct resctrl_schema *s, u32 closid) +{ + enum resctrl_conf_type peer_type; + struct rdt_resource *r = s->res; + struct rdt_ctrl_domain *d; + int ret; + + rdt_staged_configs_clear(); + + ret = rdtgroup_init_cat(s, closid); + if (ret < 0) + goto out; + + /* Keep CDP_CODE and CDP_DATA of io_alloc CLOSID's CBM in sync. */ + if (resctrl_arch_get_cdp_enabled(r->rid)) { + peer_type = resctrl_peer_type(s->conf_type); + list_for_each_entry_rcu(d, &s->res->ctrl_domains, hdr.list, lockdep_is_cpus_held()) + memcpy(&d->staged_config[peer_type], + &d->staged_config[s->conf_type], + sizeof(d->staged_config[0])); + } + + ret = resctrl_arch_update_domains(r, closid); +out: + rdt_staged_configs_clear(); + return ret; +} + +/* + * resctrl_io_alloc_closid() - io_alloc feature routes I/O traffic using + * the highest available CLOSID. Retrieve the maximum CLOSID supported by the + * resource. Note that if Code Data Prioritization (CDP) is enabled, the number + * of available CLOSIDs is reduced by half. + */ +u32 resctrl_io_alloc_closid(struct rdt_resource *r) +{ + if (resctrl_arch_get_cdp_enabled(r->rid)) + return resctrl_arch_get_num_closid(r) / 2 - 1; + else + return resctrl_arch_get_num_closid(r) - 1; +} + +ssize_t resctrl_io_alloc_write(struct kernfs_open_file *of, char *buf, + size_t nbytes, loff_t off) +{ + struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); + struct rdt_resource *r; + char const *grp_name; + u32 io_alloc_closid; + bool enable; + int ret; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + + r = s->res; + rdt_last_cmd_clear(); + + ret = kstrtobool(buf, &enable); + if (ret) { + rdt_last_cmd_puts("io_alloc: Invalid input\n"); + goto out_unlock; + } + + if (!r->cache.io_alloc_capable) { + rdt_last_cmd_printf("io_alloc is not supported on %s\n", s->name); + ret = -ENODEV; + goto out_unlock; + } + + /* If the feature is already up to date, no action is needed. */ + if (resctrl_arch_get_io_alloc_enabled(r) == enable) + goto out_unlock; + + io_alloc_closid = resctrl_io_alloc_closid(r); + if (!resctrl_io_alloc_closid_supported(io_alloc_closid)) { + rdt_last_cmd_printf("io_alloc CLOSID (ctrl_hw_id) %u is not available\n", + io_alloc_closid); + ret = -EINVAL; + goto out_unlock; + } + + if (enable) { + if (!closid_alloc_fixed(io_alloc_closid)) { + grp_name = rdtgroup_name_by_closid(io_alloc_closid); + WARN_ON_ONCE(!grp_name); + rdt_last_cmd_printf("CLOSID (ctrl_hw_id) %u for io_alloc is used by %s group\n", + io_alloc_closid, grp_name ? grp_name : "another"); + ret = -ENOSPC; + goto out_unlock; + } + + ret = resctrl_io_alloc_init_cbm(s, io_alloc_closid); + if (ret) { + rdt_last_cmd_puts("Failed to initialize io_alloc allocations\n"); + closid_free(io_alloc_closid); + goto out_unlock; + } + } else { + closid_free(io_alloc_closid); + } + + ret = resctrl_arch_io_alloc_enable(r, enable); + if (enable && ret) { + rdt_last_cmd_puts("Failed to enable io_alloc feature\n"); + closid_free(io_alloc_closid); + } + +out_unlock: + info_kn_unlock(of->kn); + + return ret ?: nbytes; +} + +int resctrl_io_alloc_cbm_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) +{ + struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); + struct rdt_resource *r; + int ret = 0; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + + rdt_last_cmd_clear(); + + r = s->res; + if (!r->cache.io_alloc_capable) { + rdt_last_cmd_printf("io_alloc is not supported on %s\n", s->name); + ret = -ENODEV; + goto out_unlock; + } + + if (!resctrl_arch_get_io_alloc_enabled(r)) { + rdt_last_cmd_printf("io_alloc is not enabled on %s\n", s->name); + ret = -EINVAL; + goto out_unlock; + } + + /* + * When CDP is enabled, the CBMs of the highest CLOSID of CDP_CODE and + * CDP_DATA are kept in sync. As a result, the io_alloc CBMs shown for + * either CDP resource are identical and accurately represent the CBMs + * used for I/O. + */ + show_doms(seq, s, NULL, resctrl_io_alloc_closid(r)); + +out_unlock: + info_kn_unlock(of->kn); + return ret; +} + +static int resctrl_io_alloc_parse_line(char *line, struct rdt_resource *r, + struct resctrl_schema *s, u32 closid) +{ + enum resctrl_conf_type peer_type; + unsigned long dom_id = ULONG_MAX; + struct rdt_parse_data data; + struct rdt_ctrl_domain *d; + bool update_all = false; + char *dom = NULL, *id; + +next: + if (!line || line[0] == '\0') + return 0; + + if (update_all) { + rdt_last_cmd_puts("Configurations after global '*'\n"); + return -EINVAL; + } + + dom = strsep(&line, ";"); + id = strsep(&dom, "="); + + if (dom && !strcmp(id, "*")) { + update_all = true; + } else if (!dom || kstrtoul(id, 10, &dom_id)) { + rdt_last_cmd_puts("Missing '=' or non-numeric domain\n"); + return -EINVAL; + } + + dom = strim(dom); + list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) { + if (update_all || d->hdr.id == dom_id) { + data.buf = dom; + data.mode = RDT_MODE_SHAREABLE; + data.closid = closid; + if (parse_cbm(&data, s, d)) + return -EINVAL; + /* + * Keep io_alloc CLOSID's CBM of CDP_CODE and CDP_DATA + * in sync. + */ + if (resctrl_arch_get_cdp_enabled(r->rid)) { + peer_type = resctrl_peer_type(s->conf_type); + memcpy(&d->staged_config[peer_type], + &d->staged_config[s->conf_type], + sizeof(d->staged_config[0])); + } + if (!update_all) + goto next; + } + } + + if (update_all) + goto next; + + rdt_last_cmd_printf("Invalid domain %lu\n", dom_id); + return -EINVAL; +} + +ssize_t resctrl_io_alloc_cbm_write(struct kernfs_open_file *of, char *buf, + size_t nbytes, loff_t off) +{ + struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); + struct rdt_resource *r; + u32 io_alloc_closid; + int ret = 0; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + rdt_last_cmd_clear(); + + r = s->res; + + /* Valid input requires a trailing newline */ + if (nbytes == 0 || buf[nbytes - 1] != '\n') { + rdt_last_cmd_puts("io_alloc_cbm: Invalid input\n"); + ret = -EINVAL; + goto out_unlock; + } + + buf[nbytes - 1] = '\0'; + + if (!r->cache.io_alloc_capable) { + rdt_last_cmd_printf("io_alloc is not supported on %s\n", s->name); + ret = -ENODEV; + goto out_unlock; + } + + if (!resctrl_arch_get_io_alloc_enabled(r)) { + rdt_last_cmd_printf("io_alloc is not enabled on %s\n", s->name); + ret = -EINVAL; + goto out_unlock; + } + + io_alloc_closid = resctrl_io_alloc_closid(r); + + rdt_staged_configs_clear(); + ret = resctrl_io_alloc_parse_line(buf, r, s, io_alloc_closid); + if (ret) + goto out_clear_configs; + + ret = resctrl_arch_update_domains(r, io_alloc_closid); + +out_clear_configs: + rdt_staged_configs_clear(); +out_unlock: + info_kn_unlock(of->kn); + + return ret ?: nbytes; +} diff --git a/fs/resctrl/internal.h b/fs/resctrl/internal.h new file mode 100644 index 000000000..e62a277de --- /dev/null +++ b/fs/resctrl/internal.h @@ -0,0 +1,520 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _FS_RESCTRL_INTERNAL_H +#define _FS_RESCTRL_INTERNAL_H + +#include +#include +#include +#include + +#define CQM_LIMBOCHECK_INTERVAL 1000 + +/** + * cpumask_any_housekeeping() - Choose any CPU in @mask, preferring those that + * aren't marked nohz_full + * @mask: The mask to pick a CPU from. + * @exclude_cpu:The CPU to avoid picking. + * + * Returns a CPU from @mask, but not @exclude_cpu. If there are housekeeping + * CPUs that don't use nohz_full, these are preferred. Pass + * RESCTRL_PICK_ANY_CPU to avoid excluding any CPUs. + * + * When a CPU is excluded, returns >= nr_cpu_ids if no CPUs are available. + */ +static inline unsigned int +cpumask_any_housekeeping(const struct cpumask *mask, int exclude_cpu) +{ + unsigned int cpu; + + /* Try to find a CPU that isn't nohz_full to use in preference */ + if (tick_nohz_full_enabled()) { + cpu = cpumask_any_andnot_but(mask, tick_nohz_full_mask, exclude_cpu); + if (cpu < nr_cpu_ids) + return cpu; + } + + return cpumask_any_but(mask, exclude_cpu); +} + +struct rdt_fs_context { + struct kernfs_fs_context kfc; + bool enable_cdpl2; + bool enable_cdpl3; + bool enable_mba_mbps; + bool enable_debug; +}; + +static inline struct rdt_fs_context *rdt_fc2context(struct fs_context *fc) +{ + struct kernfs_fs_context *kfc = fc->fs_private; + + return container_of(kfc, struct rdt_fs_context, kfc); +} + +/** + * struct mon_evt - Properties of a monitor event + * @evtid: event id + * @rid: resource id for this event + * @name: name of the event + * @evt_cfg: Event configuration value that represents the + * memory transactions (e.g., READS_TO_LOCAL_MEM, + * READS_TO_REMOTE_MEM) being tracked by @evtid. + * Only valid if @evtid is an MBM event. + * @configurable: true if the event is configurable + * @any_cpu: true if the event can be read from any CPU + * @is_floating_point: event values are displayed in floating point format + * @binary_bits: number of fixed-point binary bits from architecture, + * only valid if @is_floating_point is true + * @enabled: true if the event is enabled + * @arch_priv: Architecture private data for this event. + * The @arch_priv provided by the architecture via + * resctrl_enable_mon_event(). + */ +struct mon_evt { + enum resctrl_event_id evtid; + enum resctrl_res_level rid; + char *name; + u32 evt_cfg; + bool configurable; + bool any_cpu; + bool is_floating_point; + unsigned int binary_bits; + bool enabled; + void *arch_priv; +}; + +extern struct mon_evt mon_event_all[QOS_NUM_EVENTS]; + +#define for_each_mon_event(mevt) for (mevt = &mon_event_all[QOS_FIRST_EVENT]; \ + mevt < &mon_event_all[QOS_NUM_EVENTS]; mevt++) + +/* Limit for mon_evt::binary_bits */ +#define MAX_BINARY_BITS 27 + +/** + * struct mon_data - Monitoring details for each event file. + * @list: Member of the global @mon_data_kn_priv_list list. + * @rid: Resource id associated with the event file. + * @evt: Event structure associated with the event file. + * @sum: Set for RDT_RESOURCE_L3 when event must be summed + * across multiple domains. + * @domid: When @sum is zero this is the domain to which + * the event file belongs. When @sum is one this + * is the id of the L3 cache that all domains to be + * summed share. + * + * Pointed to by the kernfs kn->priv field of monitoring event files. + * Readers and writers must hold rdtgroup_mutex. + */ +struct mon_data { + struct list_head list; + enum resctrl_res_level rid; + struct mon_evt *evt; + int domid; + bool sum; +}; + +/** + * struct rmid_read - Data passed across smp_call*() to read event count. + * @rgrp: Resource group for which the counter is being read. If it is a parent + * resource group then its event count is summed with the count from all + * its child resource groups. + * @r: Resource describing the properties of the event being read. + * @hdr: Header of domain that the counter should be read from. If NULL then + * sum all domains in @r sharing L3 @ci.id + * @evt: Which monitor event to read. + * @first: Initialize MBM counter when true. + * @ci: Cacheinfo for L3. Only set when @hdr is NULL. Used when summing + * domains. + * @is_mbm_cntr: true if "mbm_event" counter assignment mode is enabled and it + * is an MBM event. + * @err: Error encountered when reading counter. + * @val: Returned value of event counter. If @rgrp is a parent resource + * group, @val includes the sum of event counts from its child + * resource groups. If @hdr is NULL, @val includes the sum of all + * domains in @r sharing @ci.id, (summed across child resource groups + * if @rgrp is a parent resource group). + * @arch_mon_ctx: Hardware monitor allocated for this read request (MPAM only). + */ +struct rmid_read { + struct rdtgroup *rgrp; + struct rdt_resource *r; + struct rdt_domain_hdr *hdr; + struct mon_evt *evt; + bool first; + struct cacheinfo *ci; + bool is_mbm_cntr; + int err; + u64 val; + void *arch_mon_ctx; +}; + +extern struct list_head resctrl_schema_all; + +extern bool resctrl_mounted; + +enum rdt_group_type { + RDTCTRL_GROUP = 0, + RDTMON_GROUP, + RDT_NUM_GROUP, +}; + +/** + * enum rdtgrp_mode - Mode of a RDT resource group + * @RDT_MODE_SHAREABLE: This resource group allows sharing of its allocations + * @RDT_MODE_EXCLUSIVE: No sharing of this resource group's allocations allowed + * @RDT_MODE_PSEUDO_LOCKSETUP: Resource group will be used for Pseudo-Locking + * @RDT_MODE_PSEUDO_LOCKED: No sharing of this resource group's allocations + * allowed AND the allocations are Cache Pseudo-Locked + * @RDT_NUM_MODES: Total number of modes + * + * The mode of a resource group enables control over the allowed overlap + * between allocations associated with different resource groups (classes + * of service). User is able to modify the mode of a resource group by + * writing to the "mode" resctrl file associated with the resource group. + * + * The "shareable", "exclusive", and "pseudo-locksetup" modes are set by + * writing the appropriate text to the "mode" file. A resource group enters + * "pseudo-locked" mode after the schemata is written while the resource + * group is in "pseudo-locksetup" mode. + */ +enum rdtgrp_mode { + RDT_MODE_SHAREABLE = 0, + RDT_MODE_EXCLUSIVE, + RDT_MODE_PSEUDO_LOCKSETUP, + RDT_MODE_PSEUDO_LOCKED, + + /* Must be last */ + RDT_NUM_MODES, +}; + +/** + * struct mongroup - store mon group's data in resctrl fs. + * @mon_data_kn: kernfs node for the mon_data directory + * @parent: parent rdtgrp + * @crdtgrp_list: child rdtgroup node list + * @rmid: rmid for this rdtgroup + */ +struct mongroup { + struct kernfs_node *mon_data_kn; + struct rdtgroup *parent; + struct list_head crdtgrp_list; + u32 rmid; +}; + +/** + * struct rdtgroup - store rdtgroup's data in resctrl file system. + * @kn: kernfs node + * @rdtgroup_list: linked list for all rdtgroups + * @closid: closid for this rdtgroup + * @cpu_mask: CPUs assigned to this rdtgroup + * @flags: status bits + * @waitcount: how many cpus expect to find this + * group when they acquire rdtgroup_mutex + * @type: indicates type of this rdtgroup - either + * monitor only or ctrl_mon group + * @mon: mongroup related data + * @mode: mode of resource group + * @mba_mbps_event: input monitoring event id when mba_sc is enabled + * @plr: pseudo-locked region + */ +struct rdtgroup { + struct kernfs_node *kn; + struct list_head rdtgroup_list; + u32 closid; + struct cpumask cpu_mask; + int flags; + atomic_t waitcount; + enum rdt_group_type type; + struct mongroup mon; + enum rdtgrp_mode mode; + enum resctrl_event_id mba_mbps_event; + struct pseudo_lock_region *plr; +}; + +/* rdtgroup.flags */ +#define RDT_DELETED 1 + +/* rftype.flags */ +#define RFTYPE_FLAGS_CPUS_LIST 1 + +/* + * Define the file type flags for base and info directories. + */ +#define RFTYPE_INFO BIT(0) + +#define RFTYPE_BASE BIT(1) + +#define RFTYPE_CTRL BIT(4) + +#define RFTYPE_MON BIT(5) + +#define RFTYPE_TOP BIT(6) + +#define RFTYPE_RES_CACHE BIT(8) + +#define RFTYPE_RES_MB BIT(9) + +#define RFTYPE_DEBUG BIT(10) + +#define RFTYPE_ASSIGN_CONFIG BIT(11) + +#define RFTYPE_RES_PERF_PKG BIT(12) + +#define RFTYPE_CTRL_INFO (RFTYPE_INFO | RFTYPE_CTRL) + +#define RFTYPE_MON_INFO (RFTYPE_INFO | RFTYPE_MON) + +#define RFTYPE_TOP_INFO (RFTYPE_INFO | RFTYPE_TOP) + +#define RFTYPE_CTRL_BASE (RFTYPE_BASE | RFTYPE_CTRL) + +#define RFTYPE_MON_BASE (RFTYPE_BASE | RFTYPE_MON) + +/* List of all resource groups */ +extern struct list_head rdt_all_groups; + +extern int max_name_width; + +/** + * struct rftype - describe each file in the resctrl file system + * @name: File name + * @mode: Access mode + * @kf_ops: File operations + * @flags: File specific RFTYPE_FLAGS_* flags + * @fflags: File specific RFTYPE_* flags + * @seq_show: Show content of the file + * @write: Write to the file + */ +struct rftype { + char *name; + umode_t mode; + const struct kernfs_ops *kf_ops; + unsigned long flags; + unsigned long fflags; + + int (*seq_show)(struct kernfs_open_file *of, + struct seq_file *sf, void *v); + /* + * write() is the generic write callback which maps directly to + * kernfs write operation and overrides all other operations. + * Maximum write size is determined by ->max_write_len. + */ + ssize_t (*write)(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off); +}; + +/** + * struct mbm_state - status for each MBM counter in each domain + * @prev_bw_bytes: Previous bytes value read for bandwidth calculation + * @prev_bw: The most recent bandwidth in MBps + */ +struct mbm_state { + u64 prev_bw_bytes; + u32 prev_bw; +}; + +extern struct mutex rdtgroup_mutex; + +static inline const char *rdt_kn_name(const struct kernfs_node *kn) +{ + return rcu_dereference_check(kn->name, lockdep_is_held(&rdtgroup_mutex)); +} + +extern struct rdtgroup rdtgroup_default; + +extern struct dentry *debugfs_resctrl; + +extern enum resctrl_event_id mba_mbps_default_event; + +void rdt_last_cmd_clear(void); + +void rdt_last_cmd_puts(const char *s); + +__printf(1, 2) +void rdt_last_cmd_printf(const char *fmt, ...); + +struct rdtgroup *rdtgroup_kn_lock_live(struct kernfs_node *kn); +void rdtgroup_kn_unlock(struct kernfs_node *kn); +bool info_kn_lock(struct kernfs_node *kn); +void info_kn_unlock(struct kernfs_node *kn); + +int rdtgroup_kn_mode_restrict(struct rdtgroup *r, const char *name); + +int rdtgroup_kn_mode_restore(struct rdtgroup *r, const char *name, + umode_t mask); + +ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off); + +int rdtgroup_schemata_show(struct kernfs_open_file *of, + struct seq_file *s, void *v); + +ssize_t rdtgroup_mba_mbps_event_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off); + +int rdtgroup_mba_mbps_event_show(struct kernfs_open_file *of, + struct seq_file *s, void *v); + +bool rdtgroup_cbm_overlaps(struct resctrl_schema *s, struct rdt_ctrl_domain *d, + unsigned long cbm, int closid, bool exclusive); + +unsigned int rdtgroup_cbm_to_size(struct rdt_resource *r, struct rdt_ctrl_domain *d, + unsigned long cbm); + +enum rdtgrp_mode rdtgroup_mode_by_closid(int closid); + +int rdtgroup_tasks_assigned(struct rdtgroup *r); + +int closids_supported(void); + +void closid_free(int closid); + +int setup_rmid_lru_list(void); + +void free_rmid_lru_list(void); + +int alloc_rmid(u32 closid); + +void free_rmid(u32 closid, u32 rmid); + +int resctrl_l3_mon_resource_init(void); + +void resctrl_l3_mon_resource_exit(void); + +void mon_event_count(void *info); + +int rdtgroup_mondata_show(struct seq_file *m, void *arg); + +void mon_event_read(struct rmid_read *rr, struct rdt_resource *r, + struct rdt_domain_hdr *hdr, struct rdtgroup *rdtgrp, + cpumask_t *cpumask, struct mon_evt *evt, int first); + +void mbm_setup_overflow_handler(struct rdt_l3_mon_domain *dom, + unsigned long delay_ms, + int exclude_cpu); + +void mbm_handle_overflow(struct work_struct *work); + +bool is_mba_sc(struct rdt_resource *r); + +void cqm_setup_limbo_handler(struct rdt_l3_mon_domain *dom, unsigned long delay_ms, + int exclude_cpu); + +void cqm_handle_limbo(struct work_struct *work); + +bool has_busy_rmid(struct rdt_l3_mon_domain *d); + +void __check_limbo(struct rdt_l3_mon_domain *d, bool force_free); + +void resctrl_file_fflags_init(const char *config, unsigned long fflags); + +void resctrl_file_mode_init(const char *config, umode_t mode); + +void rdt_staged_configs_clear(void); + +bool closid_allocated(unsigned int closid); + +bool closid_alloc_fixed(u32 closid); + +int resctrl_find_cleanest_closid(void); + +void *rdt_kn_parent_priv(struct kernfs_node *kn); + +int resctrl_mbm_assign_mode_show(struct kernfs_open_file *of, struct seq_file *s, void *v); + +ssize_t resctrl_mbm_assign_mode_write(struct kernfs_open_file *of, char *buf, + size_t nbytes, loff_t off); + +void resctrl_bmec_files_show(struct rdt_resource *r, struct kernfs_node *l3_mon_kn, + bool show); + +int resctrl_num_mbm_cntrs_show(struct kernfs_open_file *of, struct seq_file *s, void *v); + +int resctrl_available_mbm_cntrs_show(struct kernfs_open_file *of, struct seq_file *s, + void *v); + +void rdtgroup_assign_cntrs(struct rdtgroup *rdtgrp); + +void rdtgroup_unassign_cntrs(struct rdtgroup *rdtgrp); + +int event_filter_show(struct kernfs_open_file *of, struct seq_file *seq, void *v); + +ssize_t event_filter_write(struct kernfs_open_file *of, char *buf, size_t nbytes, + loff_t off); + +int resctrl_mbm_assign_on_mkdir_show(struct kernfs_open_file *of, + struct seq_file *s, void *v); + +ssize_t resctrl_mbm_assign_on_mkdir_write(struct kernfs_open_file *of, char *buf, + size_t nbytes, loff_t off); + +int mbm_L3_assignments_show(struct kernfs_open_file *of, struct seq_file *s, void *v); + +ssize_t mbm_L3_assignments_write(struct kernfs_open_file *of, char *buf, size_t nbytes, + loff_t off); +int resctrl_io_alloc_show(struct kernfs_open_file *of, struct seq_file *seq, void *v); + +int rdtgroup_init_cat(struct resctrl_schema *s, u32 closid); + +enum resctrl_conf_type resctrl_peer_type(enum resctrl_conf_type my_type); + +ssize_t resctrl_io_alloc_write(struct kernfs_open_file *of, char *buf, + size_t nbytes, loff_t off); + +const char *rdtgroup_name_by_closid(u32 closid); +int resctrl_io_alloc_cbm_show(struct kernfs_open_file *of, struct seq_file *seq, + void *v); +ssize_t resctrl_io_alloc_cbm_write(struct kernfs_open_file *of, char *buf, + size_t nbytes, loff_t off); +u32 resctrl_io_alloc_closid(struct rdt_resource *r); + +#ifdef CONFIG_RESCTRL_FS_PSEUDO_LOCK +int rdtgroup_locksetup_enter(struct rdtgroup *rdtgrp); + +int rdtgroup_locksetup_exit(struct rdtgroup *rdtgrp); + +bool rdtgroup_cbm_overlaps_pseudo_locked(struct rdt_ctrl_domain *d, unsigned long cbm); + +bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_ctrl_domain *d); + +int rdt_pseudo_lock_init(void); + +void rdt_pseudo_lock_release(void); + +int rdtgroup_pseudo_lock_create(struct rdtgroup *rdtgrp); + +void rdtgroup_pseudo_lock_remove(struct rdtgroup *rdtgrp); + +#else +static inline int rdtgroup_locksetup_enter(struct rdtgroup *rdtgrp) +{ + return -EOPNOTSUPP; +} + +static inline int rdtgroup_locksetup_exit(struct rdtgroup *rdtgrp) +{ + return -EOPNOTSUPP; +} + +static inline bool rdtgroup_cbm_overlaps_pseudo_locked(struct rdt_ctrl_domain *d, unsigned long cbm) +{ + return false; +} + +static inline bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_ctrl_domain *d) +{ + return false; +} + +static inline int rdt_pseudo_lock_init(void) { return 0; } +static inline void rdt_pseudo_lock_release(void) { } +static inline int rdtgroup_pseudo_lock_create(struct rdtgroup *rdtgrp) +{ + return -EOPNOTSUPP; +} + +static inline void rdtgroup_pseudo_lock_remove(struct rdtgroup *rdtgrp) { } +#endif /* CONFIG_RESCTRL_FS_PSEUDO_LOCK */ + +#endif /* _FS_RESCTRL_INTERNAL_H */ diff --git a/fs/resctrl/monitor.c b/fs/resctrl/monitor.c new file mode 100644 index 000000000..73413cb12 --- /dev/null +++ b/fs/resctrl/monitor.c @@ -0,0 +1,1985 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Resource Director Technology(RDT) + * - Monitoring code + * + * Copyright (C) 2017 Intel Corporation + * + * Author: + * Vikas Shivappa + * + * This replaces the cqm.c based on perf but we reuse a lot of + * code and datastructures originally from Peter Zijlstra and Matt Fleming. + * + * More information about RDT be found in the Intel (R) x86 Architecture + * Software Developer Manual June 2016, volume 3, section 17.17. + */ + +#define pr_fmt(fmt) "resctrl: " fmt + +#include +#include +#include +#include + +#include "internal.h" + +#define CREATE_TRACE_POINTS + +#include "monitor_trace.h" + +/** + * struct rmid_entry - dirty tracking for all RMID. + * @closid: The CLOSID for this entry. + * @rmid: The RMID for this entry. + * @busy: The number of domains with cached data using this RMID. + * @list: Member of the rmid_free_lru list when busy == 0. + * + * Depending on the architecture the correct monitor is accessed using + * both @closid and @rmid, or @rmid only. + * + * Take the rdtgroup_mutex when accessing. + */ +struct rmid_entry { + u32 closid; + u32 rmid; + int busy; + struct list_head list; +}; + +/* + * @rmid_free_lru - A least recently used list of free RMIDs + * These RMIDs are guaranteed to have an occupancy less than the + * threshold occupancy + */ +static LIST_HEAD(rmid_free_lru); + +/* + * @closid_num_dirty_rmid The number of dirty RMID each CLOSID has. + * Only allocated when CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID is defined. + * Indexed by CLOSID. Protected by rdtgroup_mutex. + */ +static u32 *closid_num_dirty_rmid; + +/* + * @rmid_limbo_count - count of currently unused but (potentially) + * dirty RMIDs. + * This counts RMIDs that no one is currently using but that + * may have a occupancy value > resctrl_rmid_realloc_threshold. User can + * change the threshold occupancy value. + */ +static unsigned int rmid_limbo_count; + +/* + * @rmid_entry - The entry in the limbo and free lists. + */ +static struct rmid_entry *rmid_ptrs; + +/* + * This is the threshold cache occupancy in bytes at which we will consider an + * RMID available for re-allocation. + */ +unsigned int resctrl_rmid_realloc_threshold; + +/* + * This is the maximum value for the reallocation threshold, in bytes. + */ +unsigned int resctrl_rmid_realloc_limit; + +/* + * x86 and arm64 differ in their handling of monitoring. + * x86's RMID are independent numbers, there is only one source of traffic + * with an RMID value of '1'. + * arm64's PMG extends the PARTID/CLOSID space, there are multiple sources of + * traffic with a PMG value of '1', one for each CLOSID, meaning the RMID + * value is no longer unique. + * To account for this, resctrl uses an index. On x86 this is just the RMID, + * on arm64 it encodes the CLOSID and RMID. This gives a unique number. + * + * The domain's rmid_busy_llc and rmid_ptrs[] are sized by index. The arch code + * must accept an attempt to read every index. + */ +static inline struct rmid_entry *__rmid_entry(u32 idx) +{ + struct rmid_entry *entry; + u32 closid, rmid; + + entry = &rmid_ptrs[idx]; + resctrl_arch_rmid_idx_decode(idx, &closid, &rmid); + + WARN_ON_ONCE(entry->closid != closid); + WARN_ON_ONCE(entry->rmid != rmid); + + return entry; +} + +static void limbo_release_entry(struct rmid_entry *entry) +{ + lockdep_assert_held(&rdtgroup_mutex); + + rmid_limbo_count--; + list_add_tail(&entry->list, &rmid_free_lru); + + if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) + closid_num_dirty_rmid[entry->closid]--; +} + +/* + * Check the RMIDs that are marked as busy for this domain. If the + * reported LLC occupancy is below the threshold clear the busy bit and + * decrement the count. If the busy count gets to zero on an RMID, we + * free the RMID + */ +void __check_limbo(struct rdt_l3_mon_domain *d, bool force_free) +{ + struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); + u32 idx_limit = resctrl_arch_system_num_rmid_idx(); + struct rmid_entry *entry; + bool rmid_dirty = true; + u32 idx, cur_idx = 1; + void *arch_mon_ctx; + void *arch_priv; + u64 val = 0; + + arch_priv = mon_event_all[QOS_L3_OCCUP_EVENT_ID].arch_priv; + arch_mon_ctx = resctrl_arch_mon_ctx_alloc(r, QOS_L3_OCCUP_EVENT_ID); + if (IS_ERR(arch_mon_ctx)) { + pr_warn_ratelimited("Failed to allocate monitor context: %pe", + arch_mon_ctx); + return; + } + + /* + * Skip RMID 0 and start from RMID 1 and check all the RMIDs that + * are marked as busy for occupancy < threshold. If the occupancy + * is less than the threshold decrement the busy counter of the + * RMID and move it to the free list when the counter reaches 0. + */ + for (;;) { + idx = find_next_bit(d->rmid_busy_llc, idx_limit, cur_idx); + if (idx >= idx_limit) + break; + + entry = __rmid_entry(idx); + if (!force_free) { + if (resctrl_arch_rmid_read(r, &d->hdr, entry->closid, + entry->rmid, QOS_L3_OCCUP_EVENT_ID, + arch_priv, &val, arch_mon_ctx)) { + rmid_dirty = true; + } else { + rmid_dirty = (val >= resctrl_rmid_realloc_threshold); + + /* + * x86's CLOSID and RMID are independent numbers, + * so the entry's CLOSID is an empty CLOSID + * (X86_RESCTRL_EMPTY_CLOSID). On Arm the RMID + * (PMG) extends the CLOSID (PARTID) space with + * bits that aren't used to select the configuration. + * It is thus necessary to track both CLOSID and + * RMID because there may be dependencies between + * them on some architectures. + */ + trace_mon_llc_occupancy_limbo(entry->closid, entry->rmid, + d->hdr.id, val); + } + } + + if (force_free || !rmid_dirty) { + clear_bit(idx, d->rmid_busy_llc); + if (!--entry->busy) + limbo_release_entry(entry); + } + cur_idx = idx + 1; + } + + resctrl_arch_mon_ctx_free(r, QOS_L3_OCCUP_EVENT_ID, arch_mon_ctx); +} + +bool has_busy_rmid(struct rdt_l3_mon_domain *d) +{ + u32 idx_limit = resctrl_arch_system_num_rmid_idx(); + + return find_first_bit(d->rmid_busy_llc, idx_limit) != idx_limit; +} + +static struct rmid_entry *resctrl_find_free_rmid(u32 closid) +{ + struct rmid_entry *itr; + u32 itr_idx, cmp_idx; + + if (list_empty(&rmid_free_lru)) + return rmid_limbo_count ? ERR_PTR(-EBUSY) : ERR_PTR(-ENOSPC); + + list_for_each_entry(itr, &rmid_free_lru, list) { + /* + * Get the index of this free RMID, and the index it would need + * to be if it were used with this CLOSID. + * If the CLOSID is irrelevant on this architecture, the two + * index values are always the same on every entry and thus the + * very first entry will be returned. + */ + itr_idx = resctrl_arch_rmid_idx_encode(itr->closid, itr->rmid); + cmp_idx = resctrl_arch_rmid_idx_encode(closid, itr->rmid); + + if (itr_idx == cmp_idx) + return itr; + } + + return ERR_PTR(-ENOSPC); +} + +/** + * resctrl_find_cleanest_closid() - Find a CLOSID where all the associated + * RMID are clean, or the CLOSID that has + * the most clean RMID. + * + * MPAM's equivalent of RMID are per-CLOSID, meaning a freshly allocated CLOSID + * may not be able to allocate clean RMID. To avoid this the allocator will + * choose the CLOSID with the most clean RMID. + * + * When the CLOSID and RMID are independent numbers, the first free CLOSID will + * be returned. + * + * Return: Free CLOSID on success, < 0 on failure. + */ +int resctrl_find_cleanest_closid(void) +{ + u32 cleanest_closid = ~0; + int i = 0; + + lockdep_assert_held(&rdtgroup_mutex); + + if (!IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) + return -EIO; + + for (i = 0; i < closids_supported(); i++) { + int num_dirty; + + if (closid_allocated(i)) + continue; + + num_dirty = closid_num_dirty_rmid[i]; + if (num_dirty == 0) + return i; + + if (cleanest_closid == ~0) + cleanest_closid = i; + + if (num_dirty < closid_num_dirty_rmid[cleanest_closid]) + cleanest_closid = i; + } + + if (cleanest_closid == ~0) + return -ENOSPC; + + return cleanest_closid; +} + +/* + * For MPAM the RMID value is not unique, and has to be considered with + * the CLOSID. The (CLOSID, RMID) pair is allocated on all domains, which + * allows all domains to be managed by a single free list. + * Each domain also has a rmid_busy_llc to reduce the work of the limbo handler. + */ +int alloc_rmid(u32 closid) +{ + struct rmid_entry *entry; + + lockdep_assert_held(&rdtgroup_mutex); + + entry = resctrl_find_free_rmid(closid); + if (IS_ERR(entry)) + return PTR_ERR(entry); + + list_del(&entry->list); + return entry->rmid; +} + +static void add_rmid_to_limbo(struct rmid_entry *entry) +{ + struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); + struct rdt_l3_mon_domain *d; + u32 idx; + + lockdep_assert_held(&rdtgroup_mutex); + + /* Walking r->domains, ensure it can't race with cpuhp */ + lockdep_assert_cpus_held(); + + idx = resctrl_arch_rmid_idx_encode(entry->closid, entry->rmid); + + entry->busy = 0; + list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) { + /* + * For the first limbo RMID in the domain, + * setup up the limbo worker. + */ + if (!has_busy_rmid(d)) + cqm_setup_limbo_handler(d, CQM_LIMBOCHECK_INTERVAL, + RESCTRL_PICK_ANY_CPU); + set_bit(idx, d->rmid_busy_llc); + entry->busy++; + } + + rmid_limbo_count++; + if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) + closid_num_dirty_rmid[entry->closid]++; +} + +void free_rmid(u32 closid, u32 rmid) +{ + u32 idx = resctrl_arch_rmid_idx_encode(closid, rmid); + struct rmid_entry *entry; + + lockdep_assert_held(&rdtgroup_mutex); + + /* + * Do not allow the default rmid to be free'd. Comparing by index + * allows architectures that ignore the closid parameter to avoid an + * unnecessary check. + */ + if (!resctrl_arch_mon_capable() || + idx == resctrl_arch_rmid_idx_encode(RESCTRL_RESERVED_CLOSID, + RESCTRL_RESERVED_RMID)) + return; + + entry = __rmid_entry(idx); + + if (resctrl_is_mon_event_enabled(QOS_L3_OCCUP_EVENT_ID)) + add_rmid_to_limbo(entry); + else + list_add_tail(&entry->list, &rmid_free_lru); +} + +static struct mbm_state *get_mbm_state(struct rdt_l3_mon_domain *d, u32 closid, + u32 rmid, enum resctrl_event_id evtid) +{ + u32 idx = resctrl_arch_rmid_idx_encode(closid, rmid); + struct mbm_state *state; + + if (!resctrl_is_mbm_event(evtid)) + return NULL; + + state = d->mbm_states[MBM_STATE_IDX(evtid)]; + + return state ? &state[idx] : NULL; +} + +/* + * mbm_cntr_get() - Return the counter ID for the matching @evtid and @rdtgrp. + * + * Return: + * Valid counter ID on success, or -ENOENT on failure. + */ +static int mbm_cntr_get(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + struct rdtgroup *rdtgrp, enum resctrl_event_id evtid) +{ + int cntr_id; + + if (!r->mon.mbm_cntr_assignable) + return -ENOENT; + + if (!resctrl_is_mbm_event(evtid)) + return -ENOENT; + + for (cntr_id = 0; cntr_id < r->mon.num_mbm_cntrs; cntr_id++) { + if (d->cntr_cfg[cntr_id].rdtgrp == rdtgrp && + d->cntr_cfg[cntr_id].evtid == evtid) + return cntr_id; + } + + return -ENOENT; +} + +/* + * mbm_cntr_alloc() - Initialize and return a new counter ID in the domain @d. + * Caller must ensure that the specified event is not assigned already. + * + * Return: + * Valid counter ID on success, or -ENOSPC on failure. + */ +static int mbm_cntr_alloc(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + struct rdtgroup *rdtgrp, enum resctrl_event_id evtid) +{ + int cntr_id; + + for (cntr_id = 0; cntr_id < r->mon.num_mbm_cntrs; cntr_id++) { + if (!d->cntr_cfg[cntr_id].rdtgrp) { + d->cntr_cfg[cntr_id].rdtgrp = rdtgrp; + d->cntr_cfg[cntr_id].evtid = evtid; + return cntr_id; + } + } + + return -ENOSPC; +} + +/* + * mbm_cntr_free() - Clear the counter ID configuration details in the domain @d. + */ +static void mbm_cntr_free(struct rdt_l3_mon_domain *d, int cntr_id) +{ + memset(&d->cntr_cfg[cntr_id], 0, sizeof(*d->cntr_cfg)); +} + +static int __l3_mon_event_count(struct rdtgroup *rdtgrp, struct rmid_read *rr) +{ + int cpu = smp_processor_id(); + u32 closid = rdtgrp->closid; + u32 rmid = rdtgrp->mon.rmid; + struct rdt_l3_mon_domain *d; + int cntr_id = -ENOENT; + struct mbm_state *m; + u64 tval = 0; + + if (!domain_header_is_valid(rr->hdr, RESCTRL_MON_DOMAIN, RDT_RESOURCE_L3)) { + rr->err = -EIO; + return -EINVAL; + } + d = container_of(rr->hdr, struct rdt_l3_mon_domain, hdr); + + if (rr->is_mbm_cntr) { + cntr_id = mbm_cntr_get(rr->r, d, rdtgrp, rr->evt->evtid); + if (cntr_id < 0) { + rr->err = -ENOENT; + return -EINVAL; + } + } + + if (rr->first) { + if (rr->is_mbm_cntr) + resctrl_arch_reset_cntr(rr->r, d, closid, rmid, cntr_id, rr->evt->evtid); + else + resctrl_arch_reset_rmid(rr->r, d, closid, rmid, rr->evt->evtid); + m = get_mbm_state(d, closid, rmid, rr->evt->evtid); + if (m) + memset(m, 0, sizeof(struct mbm_state)); + return 0; + } + + /* Reading a single domain, must be on a CPU in that domain. */ + if (!cpumask_test_cpu(cpu, &d->hdr.cpu_mask)) { + rr->err = -EIO; + return -EINVAL; + } + if (rr->is_mbm_cntr) + rr->err = resctrl_arch_cntr_read(rr->r, d, closid, rmid, cntr_id, + rr->evt->evtid, &tval); + else + rr->err = resctrl_arch_rmid_read(rr->r, rr->hdr, closid, rmid, + rr->evt->evtid, rr->evt->arch_priv, + &tval, rr->arch_mon_ctx); + if (rr->err) + return rr->err; + + rr->val += tval; + + return 0; +} + +static int __l3_mon_event_count_sum(struct rdtgroup *rdtgrp, struct rmid_read *rr) +{ + int cpu = smp_processor_id(); + u32 closid = rdtgrp->closid; + u32 rmid = rdtgrp->mon.rmid; + struct rdt_l3_mon_domain *d; + u64 tval = 0; + int err, ret; + + /* + * Summing across domains is only done for systems that implement + * Sub-NUMA Cluster. There is no overlap with systems that support + * assignable counters. + */ + if (rr->is_mbm_cntr) { + pr_warn_once("Summing domains using assignable counters is not supported\n"); + rr->err = -EINVAL; + return -EINVAL; + } + + /* Summing domains that share a cache, must be on a CPU for that cache. */ + if (!cpumask_test_cpu(cpu, &rr->ci->shared_cpu_map)) { + rr->err = -EIO; + return -EINVAL; + } + + /* + * Legacy files must report the sum of an event across all + * domains that share the same L3 cache instance. + * Report success if a read from any domain succeeds, -EINVAL + * (translated to "Unavailable" for user space) if reading from + * all domains fail for any reason. + */ + ret = -EINVAL; + /* + * RCU list being traversed with CPU hotplug lock held. lockdep + * unable to help prove this here since this work is scheduled via + * smp_call*(). Not called from MBM overflow handler. + */ + list_for_each_entry(d, &rr->r->mon_domains, hdr.list) { + if (d->ci_id != rr->ci->id) + continue; + err = resctrl_arch_rmid_read(rr->r, &d->hdr, closid, rmid, + rr->evt->evtid, rr->evt->arch_priv, + &tval, rr->arch_mon_ctx); + if (!err) { + rr->val += tval; + ret = 0; + } + } + + if (ret) + rr->err = ret; + + return ret; +} + +static int __mon_event_count(struct rdtgroup *rdtgrp, struct rmid_read *rr) +{ + switch (rr->r->rid) { + case RDT_RESOURCE_L3: + WARN_ON_ONCE(rr->evt->any_cpu); + if (rr->hdr) + return __l3_mon_event_count(rdtgrp, rr); + else + return __l3_mon_event_count_sum(rdtgrp, rr); + case RDT_RESOURCE_PERF_PKG: { + u64 tval = 0; + + rr->err = resctrl_arch_rmid_read(rr->r, rr->hdr, rdtgrp->closid, + rdtgrp->mon.rmid, rr->evt->evtid, + rr->evt->arch_priv, + &tval, rr->arch_mon_ctx); + if (rr->err) + return rr->err; + + rr->val += tval; + + return 0; + } + default: + rr->err = -EINVAL; + return -EINVAL; + } +} + +/* + * mbm_bw_count() - Update bw count from values previously read by + * __mon_event_count(). + * @rdtgrp: resctrl group associated with the CLOSID and RMID to identify + * the cached mbm_state. + * @rr: The struct rmid_read populated by __mon_event_count(). + * + * Supporting function to calculate the memory bandwidth + * and delta bandwidth in MBps. The chunks value previously read by + * __mon_event_count() is compared with the chunks value from the previous + * invocation. This must be called once per second to maintain values in MBps. + */ +static void mbm_bw_count(struct rdtgroup *rdtgrp, struct rmid_read *rr) +{ + u64 cur_bw, bytes, cur_bytes; + u32 closid = rdtgrp->closid; + u32 rmid = rdtgrp->mon.rmid; + struct rdt_l3_mon_domain *d; + struct mbm_state *m; + + if (!domain_header_is_valid(rr->hdr, RESCTRL_MON_DOMAIN, RDT_RESOURCE_L3)) + return; + d = container_of(rr->hdr, struct rdt_l3_mon_domain, hdr); + m = get_mbm_state(d, closid, rmid, rr->evt->evtid); + if (WARN_ON_ONCE(!m)) + return; + + cur_bytes = rr->val; + bytes = cur_bytes - m->prev_bw_bytes; + m->prev_bw_bytes = cur_bytes; + + cur_bw = bytes / SZ_1M; + + m->prev_bw = cur_bw; +} + +/* + * This is scheduled by mon_event_read() to read the CQM/MBM counters + * on a domain. + */ +void mon_event_count(void *info) +{ + struct rdtgroup *rdtgrp, *entry; + struct rmid_read *rr = info; + struct list_head *head; + int ret; + + rdtgrp = rr->rgrp; + + ret = __mon_event_count(rdtgrp, rr); + + /* + * For Ctrl groups read data from child monitor groups and + * add them together. Count events which are read successfully. + * Discard the rmid_read's reporting errors. + */ + head = &rdtgrp->mon.crdtgrp_list; + + if (rdtgrp->type == RDTCTRL_GROUP) { + list_for_each_entry(entry, head, mon.crdtgrp_list) { + if (__mon_event_count(entry, rr) == 0) + ret = 0; + } + } + + /* + * __mon_event_count() calls for newly created monitor groups may + * report -EINVAL/Unavailable if the monitor hasn't seen any traffic. + * Discard error if any of the monitor event reads succeeded. + */ + if (ret == 0) + rr->err = 0; +} + +/* + * Find the software controller's ctrl domain that contains @cpu on resource @r. + * + * Only called from the mbm_over worker via update_mba_bw() where the returned + * domain is kept alive by cancel_delayed_work_sync() in + * resctrl_offline_ctrl_domain(). This drains this worker and then waits on + * rdtgroup_mutex held here before the architecture can free the ctrl domain. + * + * Context: Call from RCU read-side critical section. + */ +static struct rdt_ctrl_domain *get_sc_ctrl_domain_from_cpu(int cpu, + struct rdt_resource *r) +{ + struct rdt_ctrl_domain *d; + + list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list) { + /* Find the domain that contains this CPU */ + if (cpumask_test_cpu(cpu, &d->hdr.cpu_mask)) + return d; + } + + return NULL; +} + +/* + * Feedback loop for MBA software controller (mba_sc) + * + * mba_sc is a feedback loop where we periodically read MBM counters and + * adjust the bandwidth percentage values via the IA32_MBA_THRTL_MSRs so + * that: + * + * current bandwidth(cur_bw) < user specified bandwidth(user_bw) + * + * This uses the MBM counters to measure the bandwidth and MBA throttle + * MSRs to control the bandwidth for a particular rdtgrp. It builds on the + * fact that resctrl rdtgroups have both monitoring and control. + * + * The frequency of the checks is 1s and we just tag along the MBM overflow + * timer. Having 1s interval makes the calculation of bandwidth simpler. + * + * Although MBA's goal is to restrict the bandwidth to a maximum, there may + * be a need to increase the bandwidth to avoid unnecessarily restricting + * the L2 <-> L3 traffic. + * + * Since MBA controls the L2 external bandwidth where as MBM measures the + * L3 external bandwidth the following sequence could lead to such a + * situation. + * + * Consider an rdtgroup which had high L3 <-> memory traffic in initial + * phases -> mba_sc kicks in and reduced bandwidth percentage values -> but + * after some time rdtgroup has mostly L2 <-> L3 traffic. + * + * In this case we may restrict the rdtgroup's L2 <-> L3 traffic as its + * throttle MSRs already have low percentage values. To avoid + * unnecessarily restricting such rdtgroups, we also increase the bandwidth. + */ +static void update_mba_bw(struct rdtgroup *rgrp, struct rdt_l3_mon_domain *dom_mbm) +{ + u32 closid, rmid, cur_msr_val, new_msr_val; + struct mbm_state *pmbm_data, *cmbm_data; + struct rdt_ctrl_domain *dom_mba; + enum resctrl_event_id evt_id; + struct rdt_resource *r_mba; + struct list_head *head; + struct rdtgroup *entry; + u32 cur_bw, user_bw; + + r_mba = resctrl_arch_get_resource(RDT_RESOURCE_MBA); + evt_id = rgrp->mba_mbps_event; + + closid = rgrp->closid; + rmid = rgrp->mon.rmid; + pmbm_data = get_mbm_state(dom_mbm, closid, rmid, evt_id); + if (WARN_ON_ONCE(!pmbm_data)) + return; + + guard(rcu)(); + dom_mba = get_sc_ctrl_domain_from_cpu(smp_processor_id(), r_mba); + if (!dom_mba) { + pr_warn_once("Failure to get domain for MBA update\n"); + return; + } + + cur_bw = pmbm_data->prev_bw; + user_bw = dom_mba->mbps_val[closid]; + + /* MBA resource doesn't support CDP */ + cur_msr_val = resctrl_arch_get_config(r_mba, dom_mba, closid, CDP_NONE); + + /* + * For Ctrl groups read data from child monitor groups. + */ + head = &rgrp->mon.crdtgrp_list; + list_for_each_entry(entry, head, mon.crdtgrp_list) { + cmbm_data = get_mbm_state(dom_mbm, entry->closid, entry->mon.rmid, evt_id); + if (WARN_ON_ONCE(!cmbm_data)) + return; + cur_bw += cmbm_data->prev_bw; + } + + /* + * Scale up/down the bandwidth linearly for the ctrl group. The + * bandwidth step is the bandwidth granularity specified by the + * hardware. + * Always increase throttling if current bandwidth is above the + * target set by user. + * But avoid thrashing up and down on every poll by checking + * whether a decrease in throttling is likely to push the group + * back over target. E.g. if currently throttling to 30% of bandwidth + * on a system with 10% granularity steps, check whether moving to + * 40% would go past the limit by multiplying current bandwidth by + * "(30 + 10) / 30". + */ + if (cur_msr_val > r_mba->membw.min_bw && user_bw < cur_bw) { + new_msr_val = cur_msr_val - r_mba->membw.bw_gran; + } else if (cur_msr_val < MAX_MBA_BW && + (user_bw > (cur_bw * (cur_msr_val + r_mba->membw.min_bw) / cur_msr_val))) { + new_msr_val = cur_msr_val + r_mba->membw.bw_gran; + } else { + return; + } + + resctrl_arch_update_one(r_mba, dom_mba, closid, CDP_NONE, new_msr_val); +} + +static void mbm_update_one_event(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + struct rdtgroup *rdtgrp, enum resctrl_event_id evtid) +{ + struct rmid_read rr = {0}; + + rr.r = r; + rr.hdr = &d->hdr; + rr.evt = &mon_event_all[evtid]; + if (resctrl_arch_mbm_cntr_assign_enabled(r)) { + rr.is_mbm_cntr = true; + } else { + rr.arch_mon_ctx = resctrl_arch_mon_ctx_alloc(rr.r, evtid); + if (IS_ERR(rr.arch_mon_ctx)) { + pr_warn_ratelimited("Failed to allocate monitor context: %pe", + rr.arch_mon_ctx); + return; + } + } + + __mon_event_count(rdtgrp, &rr); + + /* + * If the software controller is enabled, compute the + * bandwidth for this event id. + */ + if (is_mba_sc(NULL)) + mbm_bw_count(rdtgrp, &rr); + + if (rr.arch_mon_ctx) + resctrl_arch_mon_ctx_free(rr.r, evtid, rr.arch_mon_ctx); +} + +static void mbm_update(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + struct rdtgroup *rdtgrp) +{ + /* + * This is protected from concurrent reads from user as both + * the user and overflow handler hold the global mutex. + */ + if (resctrl_is_mon_event_enabled(QOS_L3_MBM_TOTAL_EVENT_ID)) + mbm_update_one_event(r, d, rdtgrp, QOS_L3_MBM_TOTAL_EVENT_ID); + + if (resctrl_is_mon_event_enabled(QOS_L3_MBM_LOCAL_EVENT_ID)) + mbm_update_one_event(r, d, rdtgrp, QOS_L3_MBM_LOCAL_EVENT_ID); +} + +/* + * Handler to scan the limbo list and move the RMIDs + * to free list whose occupancy < threshold_occupancy. + */ +void cqm_handle_limbo(struct work_struct *work) +{ + unsigned long delay = msecs_to_jiffies(CQM_LIMBOCHECK_INTERVAL); + struct rdt_l3_mon_domain *d; + + /* + * Safe to run without CPU hotplug lock. Work is guaranteed to be + * canceled before the domain structure is removed. + */ + mutex_lock(&rdtgroup_mutex); + + /* + * Ensure the worker is dedicated to a CPU as intended and not + * relocated by workqueue subsystem as part of CPU going offline. + */ + if (!is_percpu_thread()) + goto out_unlock; + + d = container_of(work, struct rdt_l3_mon_domain, cqm_limbo.work); + + /* Domain is going offline */ + if (cpumask_empty(&d->hdr.cpu_mask)) + goto out_unlock; + + __check_limbo(d, false); + + if (has_busy_rmid(d)) { + d->cqm_work_cpu = cpumask_any_housekeeping(&d->hdr.cpu_mask, + RESCTRL_PICK_ANY_CPU); + schedule_delayed_work_on(d->cqm_work_cpu, &d->cqm_limbo, + delay); + } + +out_unlock: + mutex_unlock(&rdtgroup_mutex); +} + +/** + * cqm_setup_limbo_handler() - Schedule the limbo handler to run for this + * domain. + * @dom: The domain the limbo handler should run for. + * @delay_ms: How far in the future the handler should run. + * @exclude_cpu: Which CPU the handler should not run on, + * RESCTRL_PICK_ANY_CPU to pick any CPU. + */ +void cqm_setup_limbo_handler(struct rdt_l3_mon_domain *dom, unsigned long delay_ms, + int exclude_cpu) +{ + unsigned long delay = msecs_to_jiffies(delay_ms); + int cpu; + + cpu = cpumask_any_housekeeping(&dom->hdr.cpu_mask, exclude_cpu); + dom->cqm_work_cpu = cpu; + + if (cpu < nr_cpu_ids) + schedule_delayed_work_on(cpu, &dom->cqm_limbo, delay); +} + +void mbm_handle_overflow(struct work_struct *work) +{ + unsigned long delay = msecs_to_jiffies(MBM_OVERFLOW_INTERVAL); + struct rdtgroup *prgrp, *crgrp; + struct rdt_l3_mon_domain *d; + struct list_head *head; + struct rdt_resource *r; + + /* + * Safe to run without CPU hotplug lock. Work is guaranteed to be + * canceled before the domain structure is removed. + */ + mutex_lock(&rdtgroup_mutex); + + /* + * If the filesystem has been unmounted this work no longer needs to + * run. + */ + if (!resctrl_mounted || !resctrl_arch_mon_capable()) + goto out_unlock; + + /* + * Ensure the worker is dedicated to a CPU and not relocated by + * workqueue subsystem as part of CPU going offline since reading + * events depend on smp_processor_id(). After passing this check + * smp_processor_id() is valid for entire duration of this worker + * since it runs with rdtgroup_mutex held and the offline handler needs + * rdtgroup_mutex to offline the CPU being run on here. + */ + if (!is_percpu_thread()) + goto out_unlock; + + r = resctrl_arch_get_resource(RDT_RESOURCE_L3); + d = container_of(work, struct rdt_l3_mon_domain, mbm_over.work); + + /* Domain is going offline */ + if (cpumask_empty(&d->hdr.cpu_mask)) + goto out_unlock; + + list_for_each_entry(prgrp, &rdt_all_groups, rdtgroup_list) { + mbm_update(r, d, prgrp); + + head = &prgrp->mon.crdtgrp_list; + list_for_each_entry(crgrp, head, mon.crdtgrp_list) + mbm_update(r, d, crgrp); + + if (is_mba_sc(NULL)) + update_mba_bw(prgrp, d); + } + + /* + * Re-check for housekeeping CPUs. This allows the overflow handler to + * move off a nohz_full CPU quickly. + */ + d->mbm_work_cpu = cpumask_any_housekeeping(&d->hdr.cpu_mask, + RESCTRL_PICK_ANY_CPU); + schedule_delayed_work_on(d->mbm_work_cpu, &d->mbm_over, delay); + +out_unlock: + mutex_unlock(&rdtgroup_mutex); +} + +/** + * mbm_setup_overflow_handler() - Schedule the overflow handler to run for this + * domain. + * @dom: The domain the overflow handler should run for. + * @delay_ms: How far in the future the handler should run. + * @exclude_cpu: Which CPU the handler should not run on, + * RESCTRL_PICK_ANY_CPU to pick any CPU. + */ +void mbm_setup_overflow_handler(struct rdt_l3_mon_domain *dom, unsigned long delay_ms, + int exclude_cpu) +{ + unsigned long delay = msecs_to_jiffies(delay_ms); + int cpu; + + /* + * When a domain comes online there is no guarantee the filesystem is + * mounted. If not, there is no need to catch counter overflow. + */ + if (!resctrl_mounted || !resctrl_arch_mon_capable()) + return; + cpu = cpumask_any_housekeeping(&dom->hdr.cpu_mask, exclude_cpu); + dom->mbm_work_cpu = cpu; + + if (cpu < nr_cpu_ids) + schedule_delayed_work_on(cpu, &dom->mbm_over, delay); +} + +int setup_rmid_lru_list(void) +{ + struct rmid_entry *entry = NULL; + u32 idx_limit; + u32 idx; + int i; + + if (!resctrl_arch_mon_capable()) + return 0; + + /* + * Called on every mount, but the number of RMIDs cannot change + * after the first mount, so keep using the same set of rmid_ptrs[] + * until resctrl_exit(). Note that the limbo handler continues to + * access rmid_ptrs[] after resctrl is unmounted. + */ + if (rmid_ptrs) + return 0; + + idx_limit = resctrl_arch_system_num_rmid_idx(); + rmid_ptrs = kzalloc_objs(struct rmid_entry, idx_limit); + if (!rmid_ptrs) + return -ENOMEM; + + for (i = 0; i < idx_limit; i++) { + entry = &rmid_ptrs[i]; + INIT_LIST_HEAD(&entry->list); + + resctrl_arch_rmid_idx_decode(i, &entry->closid, &entry->rmid); + list_add_tail(&entry->list, &rmid_free_lru); + } + + /* + * RESCTRL_RESERVED_CLOSID and RESCTRL_RESERVED_RMID are special and + * are always allocated. These are used for the rdtgroup_default + * control group, which was setup earlier in rdtgroup_setup_default(). + */ + idx = resctrl_arch_rmid_idx_encode(RESCTRL_RESERVED_CLOSID, + RESCTRL_RESERVED_RMID); + entry = __rmid_entry(idx); + list_del(&entry->list); + + return 0; +} + +void free_rmid_lru_list(void) +{ + if (!resctrl_arch_mon_capable()) + return; + + mutex_lock(&rdtgroup_mutex); + kfree(rmid_ptrs); + rmid_ptrs = NULL; + mutex_unlock(&rdtgroup_mutex); +} + +#define MON_EVENT(_eventid, _name, _res, _fp) \ + [_eventid] = { \ + .name = _name, \ + .evtid = _eventid, \ + .rid = _res, \ + .is_floating_point = _fp, \ +} + +/* + * All available events. Architecture code marks the ones that + * are supported by a system using resctrl_enable_mon_event() + * to set .enabled. + */ +struct mon_evt mon_event_all[QOS_NUM_EVENTS] = { + MON_EVENT(QOS_L3_OCCUP_EVENT_ID, "llc_occupancy", RDT_RESOURCE_L3, false), + MON_EVENT(QOS_L3_MBM_TOTAL_EVENT_ID, "mbm_total_bytes", RDT_RESOURCE_L3, false), + MON_EVENT(QOS_L3_MBM_LOCAL_EVENT_ID, "mbm_local_bytes", RDT_RESOURCE_L3, false), + MON_EVENT(PMT_EVENT_ENERGY, "core_energy", RDT_RESOURCE_PERF_PKG, true), + MON_EVENT(PMT_EVENT_ACTIVITY, "activity", RDT_RESOURCE_PERF_PKG, true), + MON_EVENT(PMT_EVENT_STALLS_LLC_HIT, "stalls_llc_hit", RDT_RESOURCE_PERF_PKG, false), + MON_EVENT(PMT_EVENT_C1_RES, "c1_res", RDT_RESOURCE_PERF_PKG, false), + MON_EVENT(PMT_EVENT_UNHALTED_CORE_CYCLES, "unhalted_core_cycles", RDT_RESOURCE_PERF_PKG, false), + MON_EVENT(PMT_EVENT_STALLS_LLC_MISS, "stalls_llc_miss", RDT_RESOURCE_PERF_PKG, false), + MON_EVENT(PMT_EVENT_AUTO_C6_RES, "c6_res", RDT_RESOURCE_PERF_PKG, false), + MON_EVENT(PMT_EVENT_UNHALTED_REF_CYCLES, "unhalted_ref_cycles", RDT_RESOURCE_PERF_PKG, false), + MON_EVENT(PMT_EVENT_UOPS_RETIRED, "uops_retired", RDT_RESOURCE_PERF_PKG, false), +}; + +bool resctrl_enable_mon_event(enum resctrl_event_id eventid, bool any_cpu, + unsigned int binary_bits, void *arch_priv) +{ + if (WARN_ON_ONCE(eventid < QOS_FIRST_EVENT || eventid >= QOS_NUM_EVENTS || + binary_bits > MAX_BINARY_BITS)) + return false; + if (mon_event_all[eventid].enabled) { + pr_warn("Duplicate enable for event %d\n", eventid); + return false; + } + if (binary_bits && !mon_event_all[eventid].is_floating_point) { + pr_warn("Event %d may not be floating point\n", eventid); + return false; + } + + mon_event_all[eventid].any_cpu = any_cpu; + mon_event_all[eventid].binary_bits = binary_bits; + mon_event_all[eventid].arch_priv = arch_priv; + mon_event_all[eventid].enabled = true; + + return true; +} + +bool resctrl_is_mon_event_enabled(enum resctrl_event_id eventid) +{ + return eventid >= QOS_FIRST_EVENT && eventid < QOS_NUM_EVENTS && + mon_event_all[eventid].enabled; +} + +u32 resctrl_get_mon_evt_cfg(enum resctrl_event_id evtid) +{ + return mon_event_all[evtid].evt_cfg; +} + +/** + * struct mbm_transaction - Memory transaction an MBM event can be configured with. + * @name: Name of memory transaction (read, write ...). + * @val: The bit (eg. READS_TO_LOCAL_MEM or READS_TO_REMOTE_MEM) used to + * represent the memory transaction within an event's configuration. + */ +struct mbm_transaction { + char name[32]; + u32 val; +}; + +/* Decoded values for each type of memory transaction. */ +static struct mbm_transaction mbm_transactions[NUM_MBM_TRANSACTIONS] = { + {"local_reads", READS_TO_LOCAL_MEM}, + {"remote_reads", READS_TO_REMOTE_MEM}, + {"local_non_temporal_writes", NON_TEMP_WRITE_TO_LOCAL_MEM}, + {"remote_non_temporal_writes", NON_TEMP_WRITE_TO_REMOTE_MEM}, + {"local_reads_slow_memory", READS_TO_LOCAL_S_MEM}, + {"remote_reads_slow_memory", READS_TO_REMOTE_S_MEM}, + {"dirty_victim_writes_all", DIRTY_VICTIMS_TO_ALL_MEM}, +}; + +int event_filter_show(struct kernfs_open_file *of, struct seq_file *seq, void *v) +{ + struct mon_evt *mevt = rdt_kn_parent_priv(of->kn); + struct rdt_resource *r; + bool sep = false; + int ret = 0, i; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + rdt_last_cmd_clear(); + + r = resctrl_arch_get_resource(mevt->rid); + if (!resctrl_arch_mbm_cntr_assign_enabled(r)) { + rdt_last_cmd_puts("mbm_event counter assignment mode is not enabled\n"); + ret = -EINVAL; + goto out_unlock; + } + + for (i = 0; i < NUM_MBM_TRANSACTIONS; i++) { + if (mevt->evt_cfg & mbm_transactions[i].val) { + if (sep) + seq_putc(seq, ','); + seq_printf(seq, "%s", mbm_transactions[i].name); + sep = true; + } + } + seq_putc(seq, '\n'); + +out_unlock: + info_kn_unlock(of->kn); + + return ret; +} + +int resctrl_mbm_assign_on_mkdir_show(struct kernfs_open_file *of, struct seq_file *s, + void *v) +{ + struct rdt_resource *r = rdt_kn_parent_priv(of->kn); + int ret = 0; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + rdt_last_cmd_clear(); + + if (!resctrl_arch_mbm_cntr_assign_enabled(r)) { + rdt_last_cmd_puts("mbm_event counter assignment mode is not enabled\n"); + ret = -EINVAL; + goto out_unlock; + } + + seq_printf(s, "%u\n", r->mon.mbm_assign_on_mkdir); + +out_unlock: + info_kn_unlock(of->kn); + + return ret; +} + +ssize_t resctrl_mbm_assign_on_mkdir_write(struct kernfs_open_file *of, char *buf, + size_t nbytes, loff_t off) +{ + struct rdt_resource *r = rdt_kn_parent_priv(of->kn); + bool value; + int ret; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + rdt_last_cmd_clear(); + + ret = kstrtobool(buf, &value); + if (ret) { + rdt_last_cmd_puts("mbm_assign_on_mkdir: Invalid input\n"); + goto out_unlock; + } + + if (!resctrl_arch_mbm_cntr_assign_enabled(r)) { + rdt_last_cmd_puts("mbm_event counter assignment mode is not enabled\n"); + ret = -EINVAL; + goto out_unlock; + } + + r->mon.mbm_assign_on_mkdir = value; + +out_unlock: + info_kn_unlock(of->kn); + + return ret ?: nbytes; +} + +/* + * mbm_cntr_free_all() - Clear all the counter ID configuration details in the + * domain @d. Called when mbm_assign_mode is changed. + */ +static void mbm_cntr_free_all(struct rdt_resource *r, struct rdt_l3_mon_domain *d) +{ + memset(d->cntr_cfg, 0, sizeof(*d->cntr_cfg) * r->mon.num_mbm_cntrs); +} + +/* + * resctrl_reset_rmid_all() - Reset all non-architecture states for all the + * supported RMIDs. + */ +static void resctrl_reset_rmid_all(struct rdt_resource *r, struct rdt_l3_mon_domain *d) +{ + u32 idx_limit = resctrl_arch_system_num_rmid_idx(); + enum resctrl_event_id evt; + int idx; + + for_each_mbm_event_id(evt) { + if (!resctrl_is_mon_event_enabled(evt)) + continue; + idx = MBM_STATE_IDX(evt); + memset(d->mbm_states[idx], 0, sizeof(*d->mbm_states[0]) * idx_limit); + } +} + +/* + * rdtgroup_assign_cntr() - Assign/unassign the counter ID for the event, RMID + * pair in the domain. + * + * Assign the counter if @assign is true else unassign the counter. Reset the + * associated non-architectural state. + */ +static void rdtgroup_assign_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + enum resctrl_event_id evtid, u32 rmid, u32 closid, + u32 cntr_id, bool assign) +{ + struct mbm_state *m; + + resctrl_arch_config_cntr(r, d, evtid, rmid, closid, cntr_id, assign); + + m = get_mbm_state(d, closid, rmid, evtid); + if (m) + memset(m, 0, sizeof(*m)); +} + +/* + * rdtgroup_alloc_assign_cntr() - Allocate a counter ID and assign it to the event + * pointed to by @mevt and the resctrl group @rdtgrp within the domain @d. + * + * Return: + * 0 on success, < 0 on failure. + */ +static int rdtgroup_alloc_assign_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + struct rdtgroup *rdtgrp, struct mon_evt *mevt) +{ + int cntr_id; + + /* No action required if the counter is assigned already. */ + cntr_id = mbm_cntr_get(r, d, rdtgrp, mevt->evtid); + if (cntr_id >= 0) + return 0; + + cntr_id = mbm_cntr_alloc(r, d, rdtgrp, mevt->evtid); + if (cntr_id < 0) { + rdt_last_cmd_printf("Failed to allocate counter for %s in domain %d\n", + mevt->name, d->hdr.id); + return cntr_id; + } + + rdtgroup_assign_cntr(r, d, mevt->evtid, rdtgrp->mon.rmid, rdtgrp->closid, cntr_id, true); + + return 0; +} + +/* + * rdtgroup_assign_cntr_event() - Assign a hardware counter for the event in + * @mevt to the resctrl group @rdtgrp. Assign counters to all domains if @d is + * NULL; otherwise, assign the counter to the specified domain @d. + * + * If all counters in a domain are already in use, rdtgroup_alloc_assign_cntr() + * will fail. When attempting to assign counters to all domains, carry on trying + * to assign counters after a failure since only some domains may have counters + * and the goal is to assign counters where possible. If any counter assignment + * fails, return the error from the last failing assignment. + * + * Return: + * 0 on success, < 0 on failure. + */ +static int rdtgroup_assign_cntr_event(struct rdt_l3_mon_domain *d, struct rdtgroup *rdtgrp, + struct mon_evt *mevt) +{ + struct rdt_resource *r = resctrl_arch_get_resource(mevt->rid); + int ret = 0; + + if (!d) { + list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) { + int err; + + err = rdtgroup_alloc_assign_cntr(r, d, rdtgrp, mevt); + if (err) + ret = err; + } + } else { + ret = rdtgroup_alloc_assign_cntr(r, d, rdtgrp, mevt); + } + + return ret; +} + +/* + * rdtgroup_assign_cntrs() - Assign counters to MBM events. Called when + * a new group is created. + * + * Each group can accommodate two counters per domain: one for the total + * event and one for the local event. Assignments may fail due to the limited + * number of counters. However, it is not necessary to fail the group creation + * and thus no failure is returned. Users have the option to modify the + * counter assignments after the group has been created. + */ +void rdtgroup_assign_cntrs(struct rdtgroup *rdtgrp) +{ + struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); + + if (!r->mon_capable || !resctrl_arch_mbm_cntr_assign_enabled(r) || + !r->mon.mbm_assign_on_mkdir) + return; + + if (resctrl_is_mon_event_enabled(QOS_L3_MBM_TOTAL_EVENT_ID)) + rdtgroup_assign_cntr_event(NULL, rdtgrp, + &mon_event_all[QOS_L3_MBM_TOTAL_EVENT_ID]); + + if (resctrl_is_mon_event_enabled(QOS_L3_MBM_LOCAL_EVENT_ID)) + rdtgroup_assign_cntr_event(NULL, rdtgrp, + &mon_event_all[QOS_L3_MBM_LOCAL_EVENT_ID]); +} + +/* + * rdtgroup_free_unassign_cntr() - Unassign and reset the counter ID configuration + * for the event pointed to by @mevt within the domain @d and resctrl group @rdtgrp. + */ +static void rdtgroup_free_unassign_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + struct rdtgroup *rdtgrp, struct mon_evt *mevt) +{ + int cntr_id; + + cntr_id = mbm_cntr_get(r, d, rdtgrp, mevt->evtid); + + /* If there is no cntr_id assigned, nothing to do */ + if (cntr_id < 0) + return; + + rdtgroup_assign_cntr(r, d, mevt->evtid, rdtgrp->mon.rmid, rdtgrp->closid, cntr_id, false); + + mbm_cntr_free(d, cntr_id); +} + +/* + * rdtgroup_unassign_cntr_event() - Unassign a hardware counter associated with + * the event structure @mevt from the domain @d and the group @rdtgrp. Unassign + * the counters from all the domains if @d is NULL else unassign from @d. + */ +static void rdtgroup_unassign_cntr_event(struct rdt_l3_mon_domain *d, struct rdtgroup *rdtgrp, + struct mon_evt *mevt) +{ + struct rdt_resource *r = resctrl_arch_get_resource(mevt->rid); + + if (!d) { + list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) + rdtgroup_free_unassign_cntr(r, d, rdtgrp, mevt); + } else { + rdtgroup_free_unassign_cntr(r, d, rdtgrp, mevt); + } +} + +/* + * rdtgroup_unassign_cntrs() - Unassign the counters associated with MBM events. + * Called when a group is deleted. + */ +void rdtgroup_unassign_cntrs(struct rdtgroup *rdtgrp) +{ + struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); + + if (!r->mon_capable || !resctrl_arch_mbm_cntr_assign_enabled(r)) + return; + + if (resctrl_is_mon_event_enabled(QOS_L3_MBM_TOTAL_EVENT_ID)) + rdtgroup_unassign_cntr_event(NULL, rdtgrp, + &mon_event_all[QOS_L3_MBM_TOTAL_EVENT_ID]); + + if (resctrl_is_mon_event_enabled(QOS_L3_MBM_LOCAL_EVENT_ID)) + rdtgroup_unassign_cntr_event(NULL, rdtgrp, + &mon_event_all[QOS_L3_MBM_LOCAL_EVENT_ID]); +} + +static int resctrl_parse_mem_transactions(char *tok, u32 *val) +{ + u32 temp_val = 0; + char *evt_str; + bool found; + int i; + +next_config: + if (!tok || tok[0] == '\0') { + *val = temp_val; + return 0; + } + + /* Start processing the strings for each memory transaction type */ + evt_str = strim(strsep(&tok, ",")); + found = false; + for (i = 0; i < NUM_MBM_TRANSACTIONS; i++) { + if (!strcmp(mbm_transactions[i].name, evt_str)) { + temp_val |= mbm_transactions[i].val; + found = true; + break; + } + } + + if (!found) { + rdt_last_cmd_printf("Invalid memory transaction type %s\n", evt_str); + return -EINVAL; + } + + goto next_config; +} + +/* + * rdtgroup_update_cntr_event - Update the counter assignments for the event + * in a group. + * @r: Resource to which update needs to be done. + * @rdtgrp: Resctrl group. + * @evtid: MBM monitor event. + */ +static void rdtgroup_update_cntr_event(struct rdt_resource *r, struct rdtgroup *rdtgrp, + enum resctrl_event_id evtid) +{ + struct rdt_l3_mon_domain *d; + int cntr_id; + + list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) { + cntr_id = mbm_cntr_get(r, d, rdtgrp, evtid); + if (cntr_id >= 0) + rdtgroup_assign_cntr(r, d, evtid, rdtgrp->mon.rmid, + rdtgrp->closid, cntr_id, true); + } +} + +/* + * resctrl_update_cntr_allrdtgrp - Update the counter assignments for the event + * for all the groups. + * @mevt MBM Monitor event. + */ +static void resctrl_update_cntr_allrdtgrp(struct mon_evt *mevt) +{ + struct rdt_resource *r = resctrl_arch_get_resource(mevt->rid); + struct rdtgroup *prgrp, *crgrp; + + /* + * Find all the groups where the event is assigned and update the + * configuration of existing assignments. + */ + list_for_each_entry(prgrp, &rdt_all_groups, rdtgroup_list) { + rdtgroup_update_cntr_event(r, prgrp, mevt->evtid); + + list_for_each_entry(crgrp, &prgrp->mon.crdtgrp_list, mon.crdtgrp_list) + rdtgroup_update_cntr_event(r, crgrp, mevt->evtid); + } +} + +ssize_t event_filter_write(struct kernfs_open_file *of, char *buf, size_t nbytes, + loff_t off) +{ + struct mon_evt *mevt = rdt_kn_parent_priv(of->kn); + struct rdt_resource *r; + u32 evt_cfg = 0; + int ret = 0; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + + rdt_last_cmd_clear(); + + /* Valid input requires a trailing newline */ + if (nbytes == 0 || buf[nbytes - 1] != '\n') { + rdt_last_cmd_puts("event_filter: Invalid input\n"); + ret = -EINVAL; + goto out_unlock; + } + + buf[nbytes - 1] = '\0'; + + r = resctrl_arch_get_resource(mevt->rid); + if (!resctrl_arch_mbm_cntr_assign_enabled(r)) { + rdt_last_cmd_puts("mbm_event counter assignment mode is not enabled\n"); + ret = -EINVAL; + goto out_unlock; + } + if (!r->mon.mbm_cntr_configurable) { + rdt_last_cmd_puts("event_filter is not configurable\n"); + ret = -EPERM; + goto out_unlock; + } + + ret = resctrl_parse_mem_transactions(buf, &evt_cfg); + if (!ret && mevt->evt_cfg != evt_cfg) { + mevt->evt_cfg = evt_cfg; + resctrl_update_cntr_allrdtgrp(mevt); + } + +out_unlock: + info_kn_unlock(of->kn); + + return ret ?: nbytes; +} + +int resctrl_mbm_assign_mode_show(struct kernfs_open_file *of, + struct seq_file *s, void *v) +{ + struct rdt_resource *r = rdt_kn_parent_priv(of->kn); + bool enabled; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + enabled = resctrl_arch_mbm_cntr_assign_enabled(r); + + if (r->mon.mbm_cntr_assignable) { + if (enabled) + seq_puts(s, "[mbm_event]\n"); + else + seq_puts(s, "[default]\n"); + + if (!r->mon.mbm_cntr_assign_fixed) { + if (enabled) + seq_puts(s, "default\n"); + else + seq_puts(s, "mbm_event\n"); + } + } else { + seq_puts(s, "[default]\n"); + } + + info_kn_unlock(of->kn); + + return 0; +} + +ssize_t resctrl_mbm_assign_mode_write(struct kernfs_open_file *of, char *buf, + size_t nbytes, loff_t off) +{ + struct rdt_resource *r = rdt_kn_parent_priv(of->kn); + struct rdt_l3_mon_domain *d; + int ret = 0; + bool enable; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + + rdt_last_cmd_clear(); + + /* Valid input requires a trailing newline */ + if (nbytes == 0 || buf[nbytes - 1] != '\n') { + rdt_last_cmd_puts("mbm_assign_mode: Invalid input\n"); + ret = -EINVAL; + goto out_unlock; + } + + buf[nbytes - 1] = '\0'; + + if (!strcmp(buf, "default")) { + enable = 0; + } else if (!strcmp(buf, "mbm_event")) { + if (r->mon.mbm_cntr_assignable) { + enable = 1; + } else { + ret = -EINVAL; + rdt_last_cmd_puts("mbm_event mode is not supported\n"); + goto out_unlock; + } + } else { + ret = -EINVAL; + rdt_last_cmd_puts("Unsupported assign mode\n"); + goto out_unlock; + } + + if (enable != resctrl_arch_mbm_cntr_assign_enabled(r)) { + if (r->mon.mbm_cntr_assign_fixed) { + ret = -EINVAL; + rdt_last_cmd_puts("Counter assignment mode is not configurable\n"); + goto out_unlock; + } + + ret = resctrl_arch_mbm_cntr_assign_set(r, enable); + if (ret) + goto out_unlock; + + /* Update the visibility of BMEC related files */ + resctrl_bmec_files_show(r, NULL, !enable); + + /* + * Initialize the default memory transaction values for + * total and local events. + */ + if (resctrl_is_mon_event_enabled(QOS_L3_MBM_TOTAL_EVENT_ID)) + mon_event_all[QOS_L3_MBM_TOTAL_EVENT_ID].evt_cfg = r->mon.mbm_cfg_mask; + if (resctrl_is_mon_event_enabled(QOS_L3_MBM_LOCAL_EVENT_ID)) + mon_event_all[QOS_L3_MBM_LOCAL_EVENT_ID].evt_cfg = r->mon.mbm_cfg_mask & + (READS_TO_LOCAL_MEM | + READS_TO_LOCAL_S_MEM | + NON_TEMP_WRITE_TO_LOCAL_MEM); + /* Enable auto assignment when switching to "mbm_event" mode */ + if (enable) + r->mon.mbm_assign_on_mkdir = true; + /* + * Reset all the non-achitectural RMID state and assignable counters. + */ + list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) { + mbm_cntr_free_all(r, d); + resctrl_reset_rmid_all(r, d); + } + } + +out_unlock: + info_kn_unlock(of->kn); + + return ret ?: nbytes; +} + +int resctrl_num_mbm_cntrs_show(struct kernfs_open_file *of, + struct seq_file *s, void *v) +{ + struct rdt_resource *r = rdt_kn_parent_priv(of->kn); + struct rdt_l3_mon_domain *dom; + bool sep = false; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + + list_for_each_entry_rcu(dom, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) { + if (sep) + seq_putc(s, ';'); + + seq_printf(s, "%d=%d", dom->hdr.id, r->mon.num_mbm_cntrs); + sep = true; + } + seq_putc(s, '\n'); + + info_kn_unlock(of->kn); + return 0; +} + +int resctrl_available_mbm_cntrs_show(struct kernfs_open_file *of, + struct seq_file *s, void *v) +{ + struct rdt_resource *r = rdt_kn_parent_priv(of->kn); + struct rdt_l3_mon_domain *dom; + bool sep = false; + u32 cntrs, i; + int ret = 0; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + + rdt_last_cmd_clear(); + + if (!resctrl_arch_mbm_cntr_assign_enabled(r)) { + rdt_last_cmd_puts("mbm_event counter assignment mode is not enabled\n"); + ret = -EINVAL; + goto out_unlock; + } + + list_for_each_entry_rcu(dom, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) { + if (sep) + seq_putc(s, ';'); + + cntrs = 0; + for (i = 0; i < r->mon.num_mbm_cntrs; i++) { + if (!dom->cntr_cfg[i].rdtgrp) + cntrs++; + } + + seq_printf(s, "%d=%u", dom->hdr.id, cntrs); + sep = true; + } + seq_putc(s, '\n'); + +out_unlock: + info_kn_unlock(of->kn); + + return ret; +} + +int mbm_L3_assignments_show(struct kernfs_open_file *of, struct seq_file *s, void *v) +{ + struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); + struct rdt_l3_mon_domain *d; + struct rdtgroup *rdtgrp; + struct mon_evt *mevt; + int ret = 0; + bool sep; + + rdtgrp = rdtgroup_kn_lock_live(of->kn); + if (!rdtgrp) { + ret = -ENOENT; + goto out_unlock; + } + + if (!resctrl_arch_mbm_cntr_assign_enabled(r)) { + rdt_last_cmd_puts("mbm_event counter assignment mode is not enabled\n"); + ret = -EINVAL; + goto out_unlock; + } + + for_each_mon_event(mevt) { + if (mevt->rid != r->rid || !mevt->enabled || !resctrl_is_mbm_event(mevt->evtid)) + continue; + + sep = false; + seq_printf(s, "%s:", mevt->name); + list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) { + if (sep) + seq_putc(s, ';'); + + if (mbm_cntr_get(r, d, rdtgrp, mevt->evtid) < 0) + seq_printf(s, "%d=_", d->hdr.id); + else + seq_printf(s, "%d=e", d->hdr.id); + + sep = true; + } + seq_putc(s, '\n'); + } + +out_unlock: + rdtgroup_kn_unlock(of->kn); + + return ret; +} + +/* + * mbm_get_mon_event_by_name() - Return the mon_evt entry for the matching + * event name. + */ +static struct mon_evt *mbm_get_mon_event_by_name(struct rdt_resource *r, char *name) +{ + struct mon_evt *mevt; + + for_each_mon_event(mevt) { + if (mevt->rid == r->rid && mevt->enabled && + resctrl_is_mbm_event(mevt->evtid) && + !strcmp(mevt->name, name)) + return mevt; + } + + return NULL; +} + +static int rdtgroup_modify_assign_state(char *assign, struct rdt_l3_mon_domain *d, + struct rdtgroup *rdtgrp, struct mon_evt *mevt) +{ + int ret = 0; + + if (!assign || strlen(assign) != 1) + return -EINVAL; + + switch (*assign) { + case 'e': + ret = rdtgroup_assign_cntr_event(d, rdtgrp, mevt); + break; + case '_': + rdtgroup_unassign_cntr_event(d, rdtgrp, mevt); + break; + default: + ret = -EINVAL; + break; + } + + return ret; +} + +static int resctrl_parse_mbm_assignment(struct rdt_resource *r, struct rdtgroup *rdtgrp, + char *event, char *tok) +{ + struct rdt_l3_mon_domain *d; + unsigned long dom_id = 0; + char *dom_str, *id_str; + struct mon_evt *mevt; + int ret; + + mevt = mbm_get_mon_event_by_name(r, event); + if (!mevt) { + rdt_last_cmd_printf("Invalid event %s\n", event); + return -ENOENT; + } + +next: + if (!tok || tok[0] == '\0') + return 0; + + /* Start processing the strings for each domain */ + dom_str = strim(strsep(&tok, ";")); + + id_str = strsep(&dom_str, "="); + + /* Check for domain id '*' which means all domains */ + if (id_str && *id_str == '*') { + ret = rdtgroup_modify_assign_state(dom_str, NULL, rdtgrp, mevt); + if (ret) + rdt_last_cmd_printf("Assign operation '%s:*=%s' failed\n", + event, dom_str); + return ret; + } else if (!id_str || kstrtoul(id_str, 10, &dom_id)) { + rdt_last_cmd_puts("Missing domain id\n"); + return -EINVAL; + } + + /* Verify if the dom_id is valid */ + list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) { + if (d->hdr.id == dom_id) { + ret = rdtgroup_modify_assign_state(dom_str, d, rdtgrp, mevt); + if (ret) { + rdt_last_cmd_printf("Assign operation '%s:%ld=%s' failed\n", + event, dom_id, dom_str); + return ret; + } + goto next; + } + } + + rdt_last_cmd_printf("Invalid domain id %ld\n", dom_id); + return -EINVAL; +} + +ssize_t mbm_L3_assignments_write(struct kernfs_open_file *of, char *buf, + size_t nbytes, loff_t off) +{ + struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); + struct rdtgroup *rdtgrp; + char *token, *event; + int ret = 0; + + rdtgrp = rdtgroup_kn_lock_live(of->kn); + if (!rdtgrp) { + rdtgroup_kn_unlock(of->kn); + return -ENOENT; + } + + /* Valid input requires a trailing newline */ + if (nbytes == 0 || buf[nbytes - 1] != '\n') { + rdt_last_cmd_puts("mbm_L3_assignments: Invalid input\n"); + ret = -EINVAL; + goto out_unlock; + } + + buf[nbytes - 1] = '\0'; + + if (!resctrl_arch_mbm_cntr_assign_enabled(r)) { + rdt_last_cmd_puts("mbm_event mode is not enabled\n"); + ret = -EINVAL; + goto out_unlock; + } + + while ((token = strsep(&buf, "\n")) != NULL) { + /* + * The write command follows the following format: + * ":=" + * Extract the event name first. + */ + event = strsep(&token, ":"); + + ret = resctrl_parse_mbm_assignment(r, rdtgrp, event, token); + if (ret) + break; + } + +out_unlock: + rdtgroup_kn_unlock(of->kn); + + return ret ?: nbytes; +} + +static int closid_num_dirty_rmid_alloc(struct rdt_resource *r) +{ + if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) { + u32 num_closid = resctrl_arch_get_num_closid(r); + u32 *tmp; + + /* For ARM memory ordering access to closid_num_dirty_rmid */ + mutex_lock(&rdtgroup_mutex); + + /* + * If the architecture hasn't provided a sanitised value here, + * this may result in larger arrays than necessary. Resctrl will + * use a smaller system wide value based on the resources in + * use. + */ + tmp = kcalloc(num_closid, sizeof(*tmp), GFP_KERNEL); + if (!tmp) { + mutex_unlock(&rdtgroup_mutex); + return -ENOMEM; + } + + closid_num_dirty_rmid = tmp; + + mutex_unlock(&rdtgroup_mutex); + } + + return 0; +} + +static void closid_num_dirty_rmid_free(void) +{ + if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) { + mutex_lock(&rdtgroup_mutex); + kfree(closid_num_dirty_rmid); + closid_num_dirty_rmid = NULL; + mutex_unlock(&rdtgroup_mutex); + } +} + +/** + * resctrl_l3_mon_resource_init() - Initialise global monitoring structures. + * + * Allocate and initialise global monitor resources that do not belong to a + * specific domain. i.e. the closid_num_dirty_rmid[] used to find the CLOSID + * with the cleanest set of RMIDs. + * Called once during boot after the struct rdt_resource's have been configured + * but before the filesystem is mounted. + * Resctrl's cpuhp callbacks may be called before this point to bring a domain + * online. + * + * Return: 0 for success, or -ENOMEM. + */ +int resctrl_l3_mon_resource_init(void) +{ + struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); + int ret; + + if (!r->mon_capable) + return 0; + + ret = closid_num_dirty_rmid_alloc(r); + if (ret) + return ret; + + if (resctrl_arch_is_evt_configurable(QOS_L3_MBM_TOTAL_EVENT_ID)) { + mon_event_all[QOS_L3_MBM_TOTAL_EVENT_ID].configurable = true; + resctrl_file_fflags_init("mbm_total_bytes_config", + RFTYPE_MON_INFO | RFTYPE_RES_CACHE); + } + if (resctrl_arch_is_evt_configurable(QOS_L3_MBM_LOCAL_EVENT_ID)) { + mon_event_all[QOS_L3_MBM_LOCAL_EVENT_ID].configurable = true; + resctrl_file_fflags_init("mbm_local_bytes_config", + RFTYPE_MON_INFO | RFTYPE_RES_CACHE); + } + + if (resctrl_is_mon_event_enabled(QOS_L3_MBM_LOCAL_EVENT_ID)) + mba_mbps_default_event = QOS_L3_MBM_LOCAL_EVENT_ID; + else if (resctrl_is_mon_event_enabled(QOS_L3_MBM_TOTAL_EVENT_ID)) + mba_mbps_default_event = QOS_L3_MBM_TOTAL_EVENT_ID; + + if (r->mon.mbm_cntr_assignable) { + if (resctrl_is_mon_event_enabled(QOS_L3_MBM_TOTAL_EVENT_ID)) + mon_event_all[QOS_L3_MBM_TOTAL_EVENT_ID].evt_cfg = r->mon.mbm_cfg_mask; + if (resctrl_is_mon_event_enabled(QOS_L3_MBM_LOCAL_EVENT_ID)) + mon_event_all[QOS_L3_MBM_LOCAL_EVENT_ID].evt_cfg = r->mon.mbm_cfg_mask & + (READS_TO_LOCAL_MEM | + READS_TO_LOCAL_S_MEM | + NON_TEMP_WRITE_TO_LOCAL_MEM); + r->mon.mbm_assign_on_mkdir = true; + resctrl_file_fflags_init("num_mbm_cntrs", + RFTYPE_MON_INFO | RFTYPE_RES_CACHE); + resctrl_file_fflags_init("available_mbm_cntrs", + RFTYPE_MON_INFO | RFTYPE_RES_CACHE); + resctrl_file_fflags_init("event_filter", RFTYPE_ASSIGN_CONFIG); + if (r->mon.mbm_cntr_configurable) + resctrl_file_mode_init("event_filter", 0644); + resctrl_file_fflags_init("mbm_assign_on_mkdir", RFTYPE_MON_INFO | + RFTYPE_RES_CACHE); + resctrl_file_fflags_init("mbm_L3_assignments", RFTYPE_MON_BASE); + } + + return 0; +} + +void resctrl_l3_mon_resource_exit(void) +{ + struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); + + if (!r->mon_capable) + return; + + closid_num_dirty_rmid_free(); +} diff --git a/fs/resctrl/monitor_trace.h b/fs/resctrl/monitor_trace.h new file mode 100644 index 000000000..fdf49f225 --- /dev/null +++ b/fs/resctrl/monitor_trace.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM resctrl + +#if !defined(_FS_RESCTRL_MONITOR_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _FS_RESCTRL_MONITOR_TRACE_H + +#include + +TRACE_EVENT(mon_llc_occupancy_limbo, + TP_PROTO(u32 ctrl_hw_id, u32 mon_hw_id, int domain_id, u64 llc_occupancy_bytes), + TP_ARGS(ctrl_hw_id, mon_hw_id, domain_id, llc_occupancy_bytes), + TP_STRUCT__entry(__field(u32, ctrl_hw_id) + __field(u32, mon_hw_id) + __field(int, domain_id) + __field(u64, llc_occupancy_bytes)), + TP_fast_assign(__entry->ctrl_hw_id = ctrl_hw_id; + __entry->mon_hw_id = mon_hw_id; + __entry->domain_id = domain_id; + __entry->llc_occupancy_bytes = llc_occupancy_bytes;), + TP_printk("ctrl_hw_id=%u mon_hw_id=%u domain_id=%d llc_occupancy_bytes=%llu", + __entry->ctrl_hw_id, __entry->mon_hw_id, __entry->domain_id, + __entry->llc_occupancy_bytes) + ); + +#endif /* _FS_RESCTRL_MONITOR_TRACE_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . + +#define TRACE_INCLUDE_FILE monitor_trace + +#include diff --git a/fs/resctrl/pseudo_lock.c b/fs/resctrl/pseudo_lock.c new file mode 100644 index 000000000..dea2b4bf9 --- /dev/null +++ b/fs/resctrl/pseudo_lock.c @@ -0,0 +1,1099 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Resource Director Technology (RDT) + * + * Pseudo-locking support built on top of Cache Allocation Technology (CAT) + * + * Copyright (C) 2018 Intel Corporation + * + * Author: Reinette Chatre + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +/* + * Major number assigned to and shared by all devices exposing + * pseudo-locked regions. + */ +static unsigned int pseudo_lock_major; + +static unsigned long pseudo_lock_minor_avail = GENMASK(MINORBITS, 0); + +static char *pseudo_lock_devnode(const struct device *dev, umode_t *mode) +{ + const struct rdtgroup *rdtgrp; + + rdtgrp = dev_get_drvdata(dev); + if (mode) + *mode = 0600; + guard(mutex)(&rdtgroup_mutex); + return kasprintf(GFP_KERNEL, "pseudo_lock/%s", rdt_kn_name(rdtgrp->kn)); +} + +static const struct class pseudo_lock_class = { + .name = "pseudo_lock", + .devnode = pseudo_lock_devnode, +}; + +/** + * pseudo_lock_minor_get - Obtain available minor number + * @minor: Pointer to where new minor number will be stored + * + * A bitmask is used to track available minor numbers. Here the next free + * minor number is marked as unavailable and returned. + * + * Return: 0 on success, <0 on failure. + */ +static int pseudo_lock_minor_get(unsigned int *minor) +{ + unsigned long first_bit; + + first_bit = find_first_bit(&pseudo_lock_minor_avail, MINORBITS); + + if (first_bit == MINORBITS) + return -ENOSPC; + + __clear_bit(first_bit, &pseudo_lock_minor_avail); + *minor = first_bit; + + return 0; +} + +/** + * pseudo_lock_minor_release - Return minor number to available + * @minor: The minor number made available + */ +static void pseudo_lock_minor_release(unsigned int minor) +{ + __set_bit(minor, &pseudo_lock_minor_avail); +} + +/** + * region_find_by_minor - Locate a pseudo-lock region by inode minor number + * @minor: The minor number of the device representing pseudo-locked region + * + * When the character device is accessed we need to determine which + * pseudo-locked region it belongs to. This is done by matching the minor + * number of the device to the pseudo-locked region it belongs. + * + * Minor numbers are assigned at the time a pseudo-locked region is associated + * with a cache instance. + * + * Return: On success return pointer to resource group owning the pseudo-locked + * region, NULL on failure. + */ +static struct rdtgroup *region_find_by_minor(unsigned int minor) +{ + struct rdtgroup *rdtgrp, *rdtgrp_match = NULL; + + list_for_each_entry(rdtgrp, &rdt_all_groups, rdtgroup_list) { + if (rdtgrp->plr && rdtgrp->plr->minor == minor) { + rdtgrp_match = rdtgrp; + break; + } + } + return rdtgrp_match; +} + +/** + * struct pseudo_lock_pm_req - A power management QoS request list entry + * @list: Entry within the @pm_reqs list for a pseudo-locked region + * @req: PM QoS request + */ +struct pseudo_lock_pm_req { + struct list_head list; + struct dev_pm_qos_request req; +}; + +static void pseudo_lock_cstates_relax(struct pseudo_lock_region *plr) +{ + struct pseudo_lock_pm_req *pm_req, *next; + + list_for_each_entry_safe(pm_req, next, &plr->pm_reqs, list) { + dev_pm_qos_remove_request(&pm_req->req); + list_del(&pm_req->list); + kfree(pm_req); + } +} + +/** + * pseudo_lock_cstates_constrain - Restrict cores from entering C6 + * @plr: Pseudo-locked region + * + * To prevent the cache from being affected by power management entering + * C6 has to be avoided. This is accomplished by requesting a latency + * requirement lower than lowest C6 exit latency of all supported + * platforms as found in the cpuidle state tables in the intel_idle driver. + * At this time it is possible to do so with a single latency requirement + * for all supported platforms. + * + * Since Goldmont is supported, which is affected by X86_BUG_MONITOR, + * the ACPI latencies need to be considered while keeping in mind that C2 + * may be set to map to deeper sleep states. In this case the latency + * requirement needs to prevent entering C2 also. + * + * Return: 0 on success, <0 on failure + */ +static int pseudo_lock_cstates_constrain(struct pseudo_lock_region *plr) +{ + struct pseudo_lock_pm_req *pm_req; + int cpu; + int ret; + + for_each_cpu(cpu, &plr->d->hdr.cpu_mask) { + pm_req = kzalloc_obj(*pm_req); + if (!pm_req) { + rdt_last_cmd_puts("Failure to allocate memory for PM QoS\n"); + ret = -ENOMEM; + goto out_err; + } + ret = dev_pm_qos_add_request(get_cpu_device(cpu), + &pm_req->req, + DEV_PM_QOS_RESUME_LATENCY, + 30); + if (ret < 0) { + rdt_last_cmd_printf("Failed to add latency req CPU%d\n", + cpu); + kfree(pm_req); + ret = -1; + goto out_err; + } + list_add(&pm_req->list, &plr->pm_reqs); + } + + return 0; + +out_err: + pseudo_lock_cstates_relax(plr); + return ret; +} + +/** + * pseudo_lock_region_clear - Reset pseudo-lock region data + * @plr: pseudo-lock region + * + * All content of the pseudo-locked region is reset - any memory allocated + * freed. + * + * Return: void + */ +static void pseudo_lock_region_clear(struct pseudo_lock_region *plr) +{ + plr->size = 0; + plr->line_size = 0; + kfree(plr->kmem); + plr->kmem = NULL; + plr->s = NULL; + if (plr->d) + plr->d->plr = NULL; + plr->d = NULL; + plr->cbm = 0; + plr->debugfs_dir = NULL; +} + +/** + * pseudo_lock_region_init - Initialize pseudo-lock region information + * @plr: pseudo-lock region + * + * Called after user provided a schemata to be pseudo-locked. From the + * schemata the &struct pseudo_lock_region is on entry already initialized + * with the resource, domain, and capacity bitmask. Here the information + * required for pseudo-locking is deduced from this data and &struct + * pseudo_lock_region initialized further. This information includes: + * - size in bytes of the region to be pseudo-locked + * - cache line size to know the stride with which data needs to be accessed + * to be pseudo-locked + * - a cpu associated with the cache instance on which the pseudo-locking + * flow can be executed + * + * Return: 0 on success, <0 on failure. Descriptive error will be written + * to last_cmd_status buffer. + */ +static int pseudo_lock_region_init(struct pseudo_lock_region *plr) +{ + enum resctrl_scope scope = plr->s->res->ctrl_scope; + struct cacheinfo *ci; + int ret; + + if (WARN_ON_ONCE(scope != RESCTRL_L2_CACHE && scope != RESCTRL_L3_CACHE)) + return -ENODEV; + + /* Pick the first cpu we find that is associated with the cache. */ + plr->cpu = cpumask_first(&plr->d->hdr.cpu_mask); + + if (!cpu_online(plr->cpu)) { + rdt_last_cmd_printf("CPU %u associated with cache not online\n", + plr->cpu); + ret = -ENODEV; + goto out_region; + } + + ci = get_cpu_cacheinfo_level(plr->cpu, scope); + if (ci) { + plr->line_size = ci->coherency_line_size; + plr->size = rdtgroup_cbm_to_size(plr->s->res, plr->d, plr->cbm); + return 0; + } + + ret = -1; + rdt_last_cmd_puts("Unable to determine cache line size\n"); +out_region: + pseudo_lock_region_clear(plr); + return ret; +} + +/** + * pseudo_lock_init - Initialize a pseudo-lock region + * @rdtgrp: resource group to which new pseudo-locked region will belong + * + * A pseudo-locked region is associated with a resource group. When this + * association is created the pseudo-locked region is initialized. The + * details of the pseudo-locked region are not known at this time so only + * allocation is done and association established. + * + * Return: 0 on success, <0 on failure + */ +static int pseudo_lock_init(struct rdtgroup *rdtgrp) +{ + struct pseudo_lock_region *plr; + + plr = kzalloc_obj(*plr); + if (!plr) + return -ENOMEM; + + init_waitqueue_head(&plr->lock_thread_wq); + INIT_LIST_HEAD(&plr->pm_reqs); + rdtgrp->plr = plr; + return 0; +} + +/** + * pseudo_lock_region_alloc - Allocate kernel memory that will be pseudo-locked + * @plr: pseudo-lock region + * + * Initialize the details required to set up the pseudo-locked region and + * allocate the contiguous memory that will be pseudo-locked to the cache. + * + * Return: 0 on success, <0 on failure. Descriptive error will be written + * to last_cmd_status buffer. + */ +static int pseudo_lock_region_alloc(struct pseudo_lock_region *plr) +{ + int ret; + + ret = pseudo_lock_region_init(plr); + if (ret < 0) + return ret; + + /* + * We do not yet support contiguous regions larger than + * KMALLOC_MAX_SIZE. + */ + if (plr->size > KMALLOC_MAX_SIZE) { + rdt_last_cmd_puts("Requested region exceeds maximum size\n"); + ret = -E2BIG; + goto out_region; + } + + plr->kmem = kzalloc(plr->size, GFP_KERNEL); + if (!plr->kmem) { + rdt_last_cmd_puts("Unable to allocate memory\n"); + ret = -ENOMEM; + goto out_region; + } + + ret = 0; + goto out; +out_region: + pseudo_lock_region_clear(plr); +out: + return ret; +} + +/** + * pseudo_lock_free - Free a pseudo-locked region + * @rdtgrp: resource group to which pseudo-locked region belonged + * + * The pseudo-locked region's resources have already been released, or not + * yet created at this point. Now it can be freed and disassociated from the + * resource group. + * + * Return: void + */ +static void pseudo_lock_free(struct rdtgroup *rdtgrp) +{ + pseudo_lock_region_clear(rdtgrp->plr); + kfree(rdtgrp->plr); + rdtgrp->plr = NULL; +} + +/** + * rdtgroup_monitor_in_progress - Test if monitoring in progress + * @rdtgrp: resource group being queried + * + * Return: 1 if monitor groups have been created for this resource + * group, 0 otherwise. + */ +static int rdtgroup_monitor_in_progress(struct rdtgroup *rdtgrp) +{ + return !list_empty(&rdtgrp->mon.crdtgrp_list); +} + +/** + * rdtgroup_locksetup_user_restrict - Restrict user access to group + * @rdtgrp: resource group needing access restricted + * + * A resource group used for cache pseudo-locking cannot have cpus or tasks + * assigned to it. This is communicated to the user by restricting access + * to all the files that can be used to make such changes. + * + * Permissions restored with rdtgroup_locksetup_user_restore() + * + * Return: 0 on success, <0 on failure. If a failure occurs during the + * restriction of access an attempt will be made to restore permissions but + * the state of the mode of these files will be uncertain when a failure + * occurs. + */ +static int rdtgroup_locksetup_user_restrict(struct rdtgroup *rdtgrp) +{ + int ret; + + ret = rdtgroup_kn_mode_restrict(rdtgrp, "tasks"); + if (ret) + return ret; + + ret = rdtgroup_kn_mode_restrict(rdtgrp, "cpus"); + if (ret) + goto err_tasks; + + ret = rdtgroup_kn_mode_restrict(rdtgrp, "cpus_list"); + if (ret) + goto err_cpus; + + if (resctrl_arch_mon_capable()) { + ret = rdtgroup_kn_mode_restrict(rdtgrp, "mon_groups"); + if (ret) + goto err_cpus_list; + } + + ret = 0; + goto out; + +err_cpus_list: + rdtgroup_kn_mode_restore(rdtgrp, "cpus_list", 0777); +err_cpus: + rdtgroup_kn_mode_restore(rdtgrp, "cpus", 0777); +err_tasks: + rdtgroup_kn_mode_restore(rdtgrp, "tasks", 0777); +out: + return ret; +} + +/** + * rdtgroup_locksetup_user_restore - Restore user access to group + * @rdtgrp: resource group needing access restored + * + * Restore all file access previously removed using + * rdtgroup_locksetup_user_restrict() + * + * Return: 0 on success, <0 on failure. If a failure occurs during the + * restoration of access an attempt will be made to restrict permissions + * again but the state of the mode of these files will be uncertain when + * a failure occurs. + */ +static int rdtgroup_locksetup_user_restore(struct rdtgroup *rdtgrp) +{ + int ret; + + ret = rdtgroup_kn_mode_restore(rdtgrp, "tasks", 0777); + if (ret) + return ret; + + ret = rdtgroup_kn_mode_restore(rdtgrp, "cpus", 0777); + if (ret) + goto err_tasks; + + ret = rdtgroup_kn_mode_restore(rdtgrp, "cpus_list", 0777); + if (ret) + goto err_cpus; + + if (resctrl_arch_mon_capable()) { + ret = rdtgroup_kn_mode_restore(rdtgrp, "mon_groups", 0777); + if (ret) + goto err_cpus_list; + } + + ret = 0; + goto out; + +err_cpus_list: + rdtgroup_kn_mode_restrict(rdtgrp, "cpus_list"); +err_cpus: + rdtgroup_kn_mode_restrict(rdtgrp, "cpus"); +err_tasks: + rdtgroup_kn_mode_restrict(rdtgrp, "tasks"); +out: + return ret; +} + +/** + * rdtgroup_locksetup_enter - Resource group enters locksetup mode + * @rdtgrp: resource group requested to enter locksetup mode + * + * A resource group enters locksetup mode to reflect that it would be used + * to represent a pseudo-locked region and is in the process of being set + * up to do so. A resource group used for a pseudo-locked region would + * lose the closid associated with it so we cannot allow it to have any + * tasks or cpus assigned nor permit tasks or cpus to be assigned in the + * future. Monitoring of a pseudo-locked region is not allowed either. + * + * The above and more restrictions on a pseudo-locked region are checked + * for and enforced before the resource group enters the locksetup mode. + * + * Returns: 0 if the resource group successfully entered locksetup mode, <0 + * on failure. On failure the last_cmd_status buffer is updated with text to + * communicate details of failure to the user. + */ +int rdtgroup_locksetup_enter(struct rdtgroup *rdtgrp) +{ + int ret; + + /* + * The default resource group can neither be removed nor lose the + * default closid associated with it. + */ + if (rdtgrp == &rdtgroup_default) { + rdt_last_cmd_puts("Cannot pseudo-lock default group\n"); + return -EINVAL; + } + + /* + * Cache Pseudo-locking not supported when CDP is enabled. + * + * Some things to consider if you would like to enable this + * support (using L3 CDP as example): + * - When CDP is enabled two separate resources are exposed, + * L3DATA and L3CODE, but they are actually on the same cache. + * The implication for pseudo-locking is that if a + * pseudo-locked region is created on a domain of one + * resource (eg. L3CODE), then a pseudo-locked region cannot + * be created on that same domain of the other resource + * (eg. L3DATA). This is because the creation of a + * pseudo-locked region involves a call to wbinvd that will + * affect all cache allocations on particular domain. + * - Considering the previous, it may be possible to only + * expose one of the CDP resources to pseudo-locking and + * hide the other. For example, we could consider to only + * expose L3DATA and since the L3 cache is unified it is + * still possible to place instructions there are execute it. + * - If only one region is exposed to pseudo-locking we should + * still keep in mind that availability of a portion of cache + * for pseudo-locking should take into account both resources. + * Similarly, if a pseudo-locked region is created in one + * resource, the portion of cache used by it should be made + * unavailable to all future allocations from both resources. + */ + if (resctrl_arch_get_cdp_enabled(RDT_RESOURCE_L3) || + resctrl_arch_get_cdp_enabled(RDT_RESOURCE_L2)) { + rdt_last_cmd_puts("CDP enabled\n"); + return -EINVAL; + } + + /* + * Not knowing the bits to disable prefetching implies that this + * platform does not support Cache Pseudo-Locking. + */ + if (resctrl_arch_get_prefetch_disable_bits() == 0) { + rdt_last_cmd_puts("Pseudo-locking not supported\n"); + return -EINVAL; + } + + if (rdtgroup_monitor_in_progress(rdtgrp)) { + rdt_last_cmd_puts("Monitoring in progress\n"); + return -EINVAL; + } + + if (rdtgroup_tasks_assigned(rdtgrp)) { + rdt_last_cmd_puts("Tasks assigned to resource group\n"); + return -EINVAL; + } + + if (!cpumask_empty(&rdtgrp->cpu_mask)) { + rdt_last_cmd_puts("CPUs assigned to resource group\n"); + return -EINVAL; + } + + if (rdtgroup_locksetup_user_restrict(rdtgrp)) { + rdt_last_cmd_puts("Unable to modify resctrl permissions\n"); + return -EIO; + } + + ret = pseudo_lock_init(rdtgrp); + if (ret) { + rdt_last_cmd_puts("Unable to init pseudo-lock region\n"); + goto out_release; + } + + /* + * If this system is capable of monitoring a rmid would have been + * allocated when the control group was created. This is not needed + * anymore when this group would be used for pseudo-locking. This + * is safe to call on platforms not capable of monitoring. + */ + free_rmid(rdtgrp->closid, rdtgrp->mon.rmid); + + ret = 0; + goto out; + +out_release: + rdtgroup_locksetup_user_restore(rdtgrp); +out: + return ret; +} + +/** + * rdtgroup_locksetup_exit - resource group exist locksetup mode + * @rdtgrp: resource group + * + * When a resource group exits locksetup mode the earlier restrictions are + * lifted. + * + * Return: 0 on success, <0 on failure + */ +int rdtgroup_locksetup_exit(struct rdtgroup *rdtgrp) +{ + int ret; + + if (resctrl_arch_mon_capable()) { + ret = alloc_rmid(rdtgrp->closid); + if (ret < 0) { + rdt_last_cmd_puts("Out of RMIDs\n"); + return ret; + } + rdtgrp->mon.rmid = ret; + } + + ret = rdtgroup_locksetup_user_restore(rdtgrp); + if (ret) { + free_rmid(rdtgrp->closid, rdtgrp->mon.rmid); + return ret; + } + + pseudo_lock_free(rdtgrp); + return 0; +} + +/** + * rdtgroup_cbm_overlaps_pseudo_locked - Test if CBM or portion is pseudo-locked + * @d: RDT domain + * @cbm: CBM to test + * + * @d represents a cache instance and @cbm a capacity bitmask that is + * considered for it. Determine if @cbm overlaps with any existing + * pseudo-locked region on @d. + * + * @cbm is unsigned long, even if only 32 bits are used, to make the + * bitmap functions work correctly. + * + * Return: true if @cbm overlaps with pseudo-locked region on @d, false + * otherwise. + */ +bool rdtgroup_cbm_overlaps_pseudo_locked(struct rdt_ctrl_domain *d, unsigned long cbm) +{ + unsigned int cbm_len; + unsigned long cbm_b; + + if (d->plr) { + cbm_len = d->plr->s->res->cache.cbm_len; + cbm_b = d->plr->cbm; + if (bitmap_intersects(&cbm, &cbm_b, cbm_len)) + return true; + } + return false; +} + +/** + * rdtgroup_pseudo_locked_in_hierarchy - Pseudo-locked region in cache hierarchy + * @d: RDT domain under test + * + * The setup of a pseudo-locked region affects all cache instances within + * the hierarchy of the region. It is thus essential to know if any + * pseudo-locked regions exist within a cache hierarchy to prevent any + * attempts to create new pseudo-locked regions in the same hierarchy. + * + * Return: true if a pseudo-locked region exists in the hierarchy of @d or + * if it is not possible to test due to memory allocation issue, + * false otherwise. + */ +bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_ctrl_domain *d) +{ + struct rdt_ctrl_domain *d_i; + cpumask_var_t cpu_with_psl; + struct rdt_resource *r; + bool ret = false; + + /* Walking r->domains, ensure it can't race with cpuhp */ + lockdep_assert_cpus_held(); + + if (!zalloc_cpumask_var(&cpu_with_psl, GFP_KERNEL)) + return true; + + /* + * First determine which cpus have pseudo-locked regions + * associated with them. + */ + for_each_alloc_capable_rdt_resource(r) { + list_for_each_entry_rcu(d_i, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) { + if (d_i->plr) + cpumask_or(cpu_with_psl, cpu_with_psl, + &d_i->hdr.cpu_mask); + } + } + + /* + * Next test if new pseudo-locked region would intersect with + * existing region. + */ + if (cpumask_intersects(&d->hdr.cpu_mask, cpu_with_psl)) + ret = true; + + free_cpumask_var(cpu_with_psl); + return ret; +} + +/** + * pseudo_lock_measure_cycles - Trigger latency measure to pseudo-locked region + * @rdtgrp: Resource group to which the pseudo-locked region belongs. + * @sel: Selector of which measurement to perform on a pseudo-locked region. + * + * The measurement of latency to access a pseudo-locked region should be + * done from a cpu that is associated with that pseudo-locked region. + * Determine which cpu is associated with this region and start a thread on + * that cpu to perform the measurement, wait for that thread to complete. + * + * Return: 0 on success, <0 on failure + */ +static int pseudo_lock_measure_cycles(struct rdtgroup *rdtgrp, int sel) +{ + struct pseudo_lock_region *plr = rdtgrp->plr; + struct task_struct *thread; + unsigned int cpu; + int ret = -1; + + cpus_read_lock(); + mutex_lock(&rdtgroup_mutex); + + if (rdtgrp->flags & RDT_DELETED) { + ret = -ENODEV; + goto out; + } + + if (!plr->d) { + ret = -ENODEV; + goto out; + } + + plr->thread_done = 0; + cpu = cpumask_first(&plr->d->hdr.cpu_mask); + if (!cpu_online(cpu)) { + ret = -ENODEV; + goto out; + } + + plr->cpu = cpu; + + if (sel == 1) + thread = kthread_run_on_cpu(resctrl_arch_measure_cycles_lat_fn, + plr, cpu, "pseudo_lock_measure/%u"); + else if (sel == 2) + thread = kthread_run_on_cpu(resctrl_arch_measure_l2_residency, + plr, cpu, "pseudo_lock_measure/%u"); + else if (sel == 3) + thread = kthread_run_on_cpu(resctrl_arch_measure_l3_residency, + plr, cpu, "pseudo_lock_measure/%u"); + else + goto out; + + if (IS_ERR(thread)) { + ret = PTR_ERR(thread); + goto out; + } + + ret = wait_event_interruptible(plr->lock_thread_wq, + plr->thread_done == 1); + if (ret < 0) + goto out; + + ret = 0; + +out: + mutex_unlock(&rdtgroup_mutex); + cpus_read_unlock(); + return ret; +} + +static ssize_t pseudo_lock_measure_trigger(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct rdtgroup *rdtgrp = file->private_data; + size_t buf_size; + char buf[32]; + int ret; + int sel; + + buf_size = min(count, (sizeof(buf) - 1)); + if (copy_from_user(buf, user_buf, buf_size)) + return -EFAULT; + + buf[buf_size] = '\0'; + ret = kstrtoint(buf, 10, &sel); + if (ret == 0) { + if (sel != 1 && sel != 2 && sel != 3) + return -EINVAL; + ret = pseudo_lock_measure_cycles(rdtgrp, sel); + if (ret == 0) + ret = count; + } + + return ret; +} + +static const struct file_operations pseudo_measure_fops = { + .write = pseudo_lock_measure_trigger, + .open = simple_open, + .llseek = default_llseek, +}; + +/** + * rdtgroup_pseudo_lock_create - Create a pseudo-locked region + * @rdtgrp: resource group to which pseudo-lock region belongs + * + * Called when a resource group in the pseudo-locksetup mode receives a + * valid schemata that should be pseudo-locked. Since the resource group is + * in pseudo-locksetup mode the &struct pseudo_lock_region has already been + * allocated and initialized with the essential information. If a failure + * occurs the resource group remains in the pseudo-locksetup mode with the + * &struct pseudo_lock_region associated with it, but cleared from all + * information and ready for the user to re-attempt pseudo-locking by + * writing the schemata again. + * + * Return: 0 if the pseudo-locked region was successfully pseudo-locked, <0 + * on failure. Descriptive error will be written to last_cmd_status buffer. + */ +int rdtgroup_pseudo_lock_create(struct rdtgroup *rdtgrp) +{ + struct pseudo_lock_region *plr = rdtgrp->plr; + struct task_struct *thread; + unsigned int new_minor; + struct device *dev; + char *kn_name __free(kfree) = NULL; + int ret; + + ret = pseudo_lock_region_alloc(plr); + if (ret < 0) + return ret; + + ret = pseudo_lock_cstates_constrain(plr); + if (ret < 0) { + ret = -EINVAL; + goto out_region; + } + kn_name = kstrdup(rdt_kn_name(rdtgrp->kn), GFP_KERNEL); + if (!kn_name) { + ret = -ENOMEM; + goto out_cstates; + } + + plr->thread_done = 0; + + thread = kthread_run_on_cpu(resctrl_arch_pseudo_lock_fn, plr, + plr->cpu, "pseudo_lock/%u"); + if (IS_ERR(thread)) { + ret = PTR_ERR(thread); + rdt_last_cmd_printf("Locking thread returned error %d\n", ret); + goto out_cstates; + } + + ret = wait_event_interruptible(plr->lock_thread_wq, + plr->thread_done == 1); + if (ret < 0) { + /* + * If the thread does not get on the CPU for whatever + * reason and the process which sets up the region is + * interrupted then this will leave the thread in runnable + * state and once it gets on the CPU it will dereference + * the cleared, but not freed, plr struct resulting in an + * empty pseudo-locking loop. + */ + rdt_last_cmd_puts("Locking thread interrupted\n"); + goto out_cstates; + } + + ret = pseudo_lock_minor_get(&new_minor); + if (ret < 0) { + rdt_last_cmd_puts("Unable to obtain a new minor number\n"); + goto out_cstates; + } + + /* + * Unlock access but do not release the reference. The + * pseudo-locked region will still be here on return. + * + * The mutex has to be released temporarily to avoid a potential + * deadlock with the mm->mmap_lock which is obtained in the + * device_create() and debugfs_create_dir() callpath below as well as + * before the mmap() callback is called. + */ + mutex_unlock(&rdtgroup_mutex); + + if (!IS_ERR_OR_NULL(debugfs_resctrl)) { + plr->debugfs_dir = debugfs_create_dir(kn_name, debugfs_resctrl); + if (!IS_ERR_OR_NULL(plr->debugfs_dir)) + debugfs_create_file("pseudo_lock_measure", 0200, + plr->debugfs_dir, rdtgrp, + &pseudo_measure_fops); + } + + dev = device_create(&pseudo_lock_class, NULL, + MKDEV(pseudo_lock_major, new_minor), + rdtgrp, "%s", kn_name); + + mutex_lock(&rdtgroup_mutex); + + if (IS_ERR(dev)) { + ret = PTR_ERR(dev); + rdt_last_cmd_printf("Failed to create character device: %d\n", + ret); + goto out_debugfs; + } + + /* We released the mutex - check if group was removed while we did so */ + if (rdtgrp->flags & RDT_DELETED) { + ret = -ENODEV; + goto out_device; + } + + plr->minor = new_minor; + + rdtgrp->mode = RDT_MODE_PSEUDO_LOCKED; + closid_free(rdtgrp->closid); + rdtgroup_kn_mode_restore(rdtgrp, "cpus", 0444); + rdtgroup_kn_mode_restore(rdtgrp, "cpus_list", 0444); + + ret = 0; + goto out; + +out_device: + device_destroy(&pseudo_lock_class, MKDEV(pseudo_lock_major, new_minor)); +out_debugfs: + debugfs_remove_recursive(plr->debugfs_dir); + pseudo_lock_minor_release(new_minor); +out_cstates: + pseudo_lock_cstates_relax(plr); +out_region: + pseudo_lock_region_clear(plr); +out: + return ret; +} + +/** + * rdtgroup_pseudo_lock_remove - Remove a pseudo-locked region + * @rdtgrp: resource group to which the pseudo-locked region belongs + * + * The removal of a pseudo-locked region can be initiated when the resource + * group is removed from user space via a "rmdir" from userspace or the + * unmount of the resctrl filesystem. On removal the resource group does + * not go back to pseudo-locksetup mode before it is removed, instead it is + * removed directly. There is thus asymmetry with the creation where the + * &struct pseudo_lock_region is removed here while it was not created in + * rdtgroup_pseudo_lock_create(). + * + * Return: void + */ +void rdtgroup_pseudo_lock_remove(struct rdtgroup *rdtgrp) +{ + struct pseudo_lock_region *plr = rdtgrp->plr; + + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { + /* + * Default group cannot be a pseudo-locked region so we can + * free closid here. + */ + closid_free(rdtgrp->closid); + goto free; + } + + pseudo_lock_cstates_relax(plr); + debugfs_remove_recursive(rdtgrp->plr->debugfs_dir); + device_destroy(&pseudo_lock_class, MKDEV(pseudo_lock_major, plr->minor)); + pseudo_lock_minor_release(plr->minor); + +free: + pseudo_lock_free(rdtgrp); +} + +static int pseudo_lock_dev_open(struct inode *inode, struct file *filp) +{ + struct rdtgroup *rdtgrp; + + mutex_lock(&rdtgroup_mutex); + + rdtgrp = region_find_by_minor(iminor(inode)); + if (!rdtgrp) { + mutex_unlock(&rdtgroup_mutex); + return -ENODEV; + } + + filp->private_data = rdtgrp; + atomic_inc(&rdtgrp->waitcount); + /* Perform a non-seekable open - llseek is not supported */ + filp->f_mode &= ~(FMODE_LSEEK | FMODE_PREAD | FMODE_PWRITE); + + mutex_unlock(&rdtgroup_mutex); + + return 0; +} + +static int pseudo_lock_dev_release(struct inode *inode, struct file *filp) +{ + struct rdtgroup *rdtgrp; + + mutex_lock(&rdtgroup_mutex); + rdtgrp = filp->private_data; + WARN_ON(!rdtgrp); + if (!rdtgrp) { + mutex_unlock(&rdtgroup_mutex); + return -ENODEV; + } + filp->private_data = NULL; + atomic_dec(&rdtgrp->waitcount); + mutex_unlock(&rdtgroup_mutex); + return 0; +} + +static int pseudo_lock_dev_mremap(struct vm_area_struct *area) +{ + /* Not supported */ + return -EINVAL; +} + +static const struct vm_operations_struct pseudo_mmap_ops = { + .mremap = pseudo_lock_dev_mremap, +}; + +static int pseudo_lock_dev_mmap_prepare(struct vm_area_desc *desc) +{ + unsigned long off = desc->pgoff << PAGE_SHIFT; + unsigned long vsize = vma_desc_size(desc); + struct file *filp = desc->file; + struct pseudo_lock_region *plr; + struct rdtgroup *rdtgrp; + unsigned long physical; + unsigned long psize; + + mutex_lock(&rdtgroup_mutex); + + rdtgrp = filp->private_data; + WARN_ON(!rdtgrp); + if (!rdtgrp) { + mutex_unlock(&rdtgroup_mutex); + return -ENODEV; + } + + plr = rdtgrp->plr; + + if (!plr->d) { + mutex_unlock(&rdtgroup_mutex); + return -ENODEV; + } + + /* + * Task is required to run with affinity to the cpus associated + * with the pseudo-locked region. If this is not the case the task + * may be scheduled elsewhere and invalidate entries in the + * pseudo-locked region. + */ + if (!cpumask_subset(current->cpus_ptr, &plr->d->hdr.cpu_mask)) { + mutex_unlock(&rdtgroup_mutex); + return -EINVAL; + } + + physical = __pa(plr->kmem) >> PAGE_SHIFT; + psize = plr->size - off; + + if (off > plr->size) { + mutex_unlock(&rdtgroup_mutex); + return -ENOSPC; + } + + /* + * Ensure changes are carried directly to the memory being mapped, + * do not allow copy-on-write mapping. + */ + if (!vma_desc_test(desc, VMA_SHARED_BIT)) { + mutex_unlock(&rdtgroup_mutex); + return -EINVAL; + } + + if (vsize > psize) { + mutex_unlock(&rdtgroup_mutex); + return -ENOSPC; + } + + memset(plr->kmem + off, 0, vsize); + + desc->vm_ops = &pseudo_mmap_ops; + mmap_action_remap_full(desc, physical + desc->pgoff); + + mutex_unlock(&rdtgroup_mutex); + return 0; +} + +static const struct file_operations pseudo_lock_dev_fops = { + .owner = THIS_MODULE, + .read = NULL, + .write = NULL, + .open = pseudo_lock_dev_open, + .release = pseudo_lock_dev_release, + .mmap_prepare = pseudo_lock_dev_mmap_prepare, +}; + +int rdt_pseudo_lock_init(void) +{ + int ret; + + ret = register_chrdev(0, "pseudo_lock", &pseudo_lock_dev_fops); + if (ret < 0) + return ret; + + pseudo_lock_major = ret; + + ret = class_register(&pseudo_lock_class); + if (ret) { + unregister_chrdev(pseudo_lock_major, "pseudo_lock"); + return ret; + } + + return 0; +} + +void rdt_pseudo_lock_release(void) +{ + class_unregister(&pseudo_lock_class); + unregister_chrdev(pseudo_lock_major, "pseudo_lock"); + pseudo_lock_major = 0; +} diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c new file mode 100644 index 000000000..5dcbb0a96 --- /dev/null +++ b/fs/resctrl/rdtgroup.c @@ -0,0 +1,4921 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * User interface for Resource Allocation in Resource Director Technology(RDT) + * + * Copyright (C) 2016 Intel Corporation + * + * Author: Fenghua Yu + * + * More information about RDT be found in the Intel (R) x86 Architecture + * Software Developer Manual. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "internal.h" + +/* Mutex to protect rdtgroup access. */ +DEFINE_MUTEX(rdtgroup_mutex); + +static struct kernfs_root *rdt_root; + +struct rdtgroup rdtgroup_default; + +LIST_HEAD(rdt_all_groups); + +/* list of entries for the schemata file */ +LIST_HEAD(resctrl_schema_all); + +/* + * List of struct mon_data containing private data of event files for use by + * rdtgroup_mondata_show(). Protected by rdtgroup_mutex. + */ +static LIST_HEAD(mon_data_kn_priv_list); + +/* The filesystem can only be mounted once. */ +bool resctrl_mounted; + +/* Kernel fs node for "info" directory under root */ +static struct kernfs_node *kn_info; + +/* Kernel fs node for "mon_groups" directory under root */ +static struct kernfs_node *kn_mongrp; + +/* Kernel fs node for "mon_data" directory under root */ +static struct kernfs_node *kn_mondata; + +/* + * Used to store the max resource name width to display the schemata names in + * a tabular format. + */ +int max_name_width; + +static struct seq_buf last_cmd_status; + +static char last_cmd_status_buf[512]; + +static int rdtgroup_setup_root(struct rdt_fs_context *ctx); + +static void rdtgroup_destroy_root(void); + +struct dentry *debugfs_resctrl; + +/* + * Memory bandwidth monitoring event to use for the default CTRL_MON group + * and each new CTRL_MON group created by the user. Only relevant when + * the filesystem is mounted with the "mba_MBps" option so it does not + * matter that it remains uninitialized on systems that do not support + * the "mba_MBps" option. + */ +enum resctrl_event_id mba_mbps_default_event; + +static bool resctrl_debug; + +void rdt_last_cmd_clear(void) +{ + lockdep_assert_held(&rdtgroup_mutex); + seq_buf_clear(&last_cmd_status); +} + +void rdt_last_cmd_puts(const char *s) +{ + lockdep_assert_held(&rdtgroup_mutex); + seq_buf_puts(&last_cmd_status, s); +} + +void rdt_last_cmd_printf(const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + lockdep_assert_held(&rdtgroup_mutex); + seq_buf_vprintf(&last_cmd_status, fmt, ap); + va_end(ap); +} + +void rdt_staged_configs_clear(void) +{ + struct rdt_ctrl_domain *dom; + struct rdt_resource *r; + + lockdep_assert_held(&rdtgroup_mutex); + + for_each_alloc_capable_rdt_resource(r) { + list_for_each_entry_rcu(dom, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) + memset(dom->staged_config, 0, sizeof(dom->staged_config)); + } +} + +static bool resctrl_is_mbm_enabled(void) +{ + return (resctrl_is_mon_event_enabled(QOS_L3_MBM_TOTAL_EVENT_ID) || + resctrl_is_mon_event_enabled(QOS_L3_MBM_LOCAL_EVENT_ID)); +} + +/* + * Trivial allocator for CLOSIDs. Use BITMAP APIs to manipulate a bitmap + * of free CLOSIDs. + * + * Using a global CLOSID across all resources has some advantages and + * some drawbacks: + * + We can simply set current's closid to assign a task to a resource + * group. + * + Context switch code can avoid extra memory references deciding which + * CLOSID to load into the PQR_ASSOC MSR + * - We give up some options in configuring resource groups across multi-socket + * systems. + * - Our choices on how to configure each resource become progressively more + * limited as the number of resources grows. + */ +static unsigned long *closid_free_map; + +static int closid_free_map_len; + +int closids_supported(void) +{ + return closid_free_map_len; +} + +static int closid_init(void) +{ + struct resctrl_schema *s; + u32 rdt_min_closid = ~0; + + /* Monitor only platforms still call closid_init() */ + if (list_empty(&resctrl_schema_all)) + return 0; + + /* Compute rdt_min_closid across all resources */ + list_for_each_entry(s, &resctrl_schema_all, list) + rdt_min_closid = min(rdt_min_closid, s->num_closid); + + closid_free_map = bitmap_alloc(rdt_min_closid, GFP_KERNEL); + if (!closid_free_map) + return -ENOMEM; + bitmap_fill(closid_free_map, rdt_min_closid); + + /* RESCTRL_RESERVED_CLOSID is always reserved for the default group */ + __clear_bit(RESCTRL_RESERVED_CLOSID, closid_free_map); + closid_free_map_len = rdt_min_closid; + + return 0; +} + +static void closid_exit(void) +{ + bitmap_free(closid_free_map); + closid_free_map = NULL; +} + +static int closid_alloc(void) +{ + int cleanest_closid; + u32 closid; + + lockdep_assert_held(&rdtgroup_mutex); + + if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID) && + resctrl_is_mon_event_enabled(QOS_L3_OCCUP_EVENT_ID)) { + cleanest_closid = resctrl_find_cleanest_closid(); + if (cleanest_closid < 0) + return cleanest_closid; + closid = cleanest_closid; + } else { + closid = find_first_bit(closid_free_map, closid_free_map_len); + if (closid == closid_free_map_len) + return -ENOSPC; + } + __clear_bit(closid, closid_free_map); + + return closid; +} + +void closid_free(int closid) +{ + lockdep_assert_held(&rdtgroup_mutex); + + __set_bit(closid, closid_free_map); +} + +/** + * closid_allocated - test if provided closid is in use + * @closid: closid to be tested + * + * Return: true if @closid is currently associated with a resource group, + * false if @closid is free + */ +bool closid_allocated(unsigned int closid) +{ + lockdep_assert_held(&rdtgroup_mutex); + + return !test_bit(closid, closid_free_map); +} + +bool closid_alloc_fixed(u32 closid) +{ + return __test_and_clear_bit(closid, closid_free_map); +} + +/** + * rdtgroup_mode_by_closid - Return mode of resource group with closid + * @closid: closid if the resource group + * + * Each resource group is associated with a @closid. Here the mode + * of a resource group can be queried by searching for it using its closid. + * + * Return: mode as &enum rdtgrp_mode of resource group with closid @closid + */ +enum rdtgrp_mode rdtgroup_mode_by_closid(int closid) +{ + struct rdtgroup *rdtgrp; + + list_for_each_entry(rdtgrp, &rdt_all_groups, rdtgroup_list) { + if (rdtgrp->closid == closid) + return rdtgrp->mode; + } + + return RDT_NUM_MODES; +} + +static const char * const rdt_mode_str[] = { + [RDT_MODE_SHAREABLE] = "shareable", + [RDT_MODE_EXCLUSIVE] = "exclusive", + [RDT_MODE_PSEUDO_LOCKSETUP] = "pseudo-locksetup", + [RDT_MODE_PSEUDO_LOCKED] = "pseudo-locked", +}; + +/** + * rdtgroup_mode_str - Return the string representation of mode + * @mode: the resource group mode as &enum rdtgroup_mode + * + * Return: string representation of valid mode, "unknown" otherwise + */ +static const char *rdtgroup_mode_str(enum rdtgrp_mode mode) +{ + if (mode < RDT_MODE_SHAREABLE || mode >= RDT_NUM_MODES) + return "unknown"; + + return rdt_mode_str[mode]; +} + +/* set uid and gid of rdtgroup dirs and files to that of the creator */ +static int rdtgroup_kn_set_ugid(struct kernfs_node *kn) +{ + struct iattr iattr = { .ia_valid = ATTR_UID | ATTR_GID, + .ia_uid = current_fsuid(), + .ia_gid = current_fsgid(), }; + + if (uid_eq(iattr.ia_uid, GLOBAL_ROOT_UID) && + gid_eq(iattr.ia_gid, GLOBAL_ROOT_GID)) + return 0; + + return kernfs_setattr(kn, &iattr); +} + +static int rdtgroup_add_file(struct kernfs_node *parent_kn, struct rftype *rft) +{ + struct kernfs_node *kn; + int ret; + + kn = __kernfs_create_file(parent_kn, rft->name, rft->mode, + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, + 0, rft->kf_ops, rft, NULL, NULL); + if (IS_ERR(kn)) + return PTR_ERR(kn); + + ret = rdtgroup_kn_set_ugid(kn); + if (ret) { + kernfs_remove(kn); + return ret; + } + + return 0; +} + +static int rdtgroup_seqfile_show(struct seq_file *m, void *arg) +{ + struct kernfs_open_file *of = m->private; + struct rftype *rft = of->kn->priv; + + if (rft->seq_show) + return rft->seq_show(of, m, arg); + return 0; +} + +static ssize_t rdtgroup_file_write(struct kernfs_open_file *of, char *buf, + size_t nbytes, loff_t off) +{ + struct rftype *rft = of->kn->priv; + + if (rft->write) + return rft->write(of, buf, nbytes, off); + + return -EINVAL; +} + +static const struct kernfs_ops rdtgroup_kf_single_ops = { + .atomic_write_len = PAGE_SIZE, + .write = rdtgroup_file_write, + .seq_show = rdtgroup_seqfile_show, +}; + +static const struct kernfs_ops kf_mondata_ops = { + .atomic_write_len = PAGE_SIZE, + .seq_show = rdtgroup_mondata_show, +}; + +static bool is_cpu_list(struct kernfs_open_file *of) +{ + struct rftype *rft = of->kn->priv; + + return rft->flags & RFTYPE_FLAGS_CPUS_LIST; +} + +static int rdtgroup_cpus_show(struct kernfs_open_file *of, + struct seq_file *s, void *v) +{ + struct rdtgroup *rdtgrp; + struct cpumask *mask; + int ret = 0; + + rdtgrp = rdtgroup_kn_lock_live(of->kn); + + if (rdtgrp) { + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) { + if (!rdtgrp->plr->d) { + rdt_last_cmd_puts("Cache domain offline\n"); + ret = -ENODEV; + } else { + mask = &rdtgrp->plr->d->hdr.cpu_mask; + seq_printf(s, is_cpu_list(of) ? + "%*pbl\n" : "%*pb\n", + cpumask_pr_args(mask)); + } + } else { + seq_printf(s, is_cpu_list(of) ? "%*pbl\n" : "%*pb\n", + cpumask_pr_args(&rdtgrp->cpu_mask)); + } + } else { + ret = -ENOENT; + } + rdtgroup_kn_unlock(of->kn); + + return ret; +} + +/* + * Update the PGR_ASSOC MSR on all cpus in @cpu_mask, + * + * Per task closids/rmids must have been set up before calling this function. + * @r may be NULL. + */ +static void +update_closid_rmid(const struct cpumask *cpu_mask, struct rdtgroup *r) +{ + struct resctrl_cpu_defaults defaults, *p = NULL; + + if (r) { + defaults.closid = r->closid; + defaults.rmid = r->mon.rmid; + p = &defaults; + } + + on_each_cpu_mask(cpu_mask, resctrl_arch_sync_cpu_closid_rmid, p, 1); +} + +static int cpus_mon_write(struct rdtgroup *rdtgrp, cpumask_var_t newmask, + cpumask_var_t tmpmask) +{ + struct rdtgroup *prgrp = rdtgrp->mon.parent, *crgrp; + struct list_head *head; + + /* Check whether cpus belong to parent ctrl group */ + cpumask_andnot(tmpmask, newmask, &prgrp->cpu_mask); + if (!cpumask_empty(tmpmask)) { + rdt_last_cmd_puts("Can only add CPUs to mongroup that belong to parent\n"); + return -EINVAL; + } + + /* Check whether cpus are dropped from this group */ + cpumask_andnot(tmpmask, &rdtgrp->cpu_mask, newmask); + if (!cpumask_empty(tmpmask)) { + /* Give any dropped cpus to parent rdtgroup */ + cpumask_or(&prgrp->cpu_mask, &prgrp->cpu_mask, tmpmask); + update_closid_rmid(tmpmask, prgrp); + } + + /* + * If we added cpus, remove them from previous group that owned them + * and update per-cpu rmid + */ + cpumask_andnot(tmpmask, newmask, &rdtgrp->cpu_mask); + if (!cpumask_empty(tmpmask)) { + head = &prgrp->mon.crdtgrp_list; + list_for_each_entry(crgrp, head, mon.crdtgrp_list) { + if (crgrp == rdtgrp) + continue; + cpumask_andnot(&crgrp->cpu_mask, &crgrp->cpu_mask, + tmpmask); + } + update_closid_rmid(tmpmask, rdtgrp); + } + + /* Done pushing/pulling - update this group with new mask */ + cpumask_copy(&rdtgrp->cpu_mask, newmask); + + return 0; +} + +static void cpumask_rdtgrp_clear(struct rdtgroup *r, struct cpumask *m) +{ + struct rdtgroup *crgrp; + + cpumask_andnot(&r->cpu_mask, &r->cpu_mask, m); + /* update the child mon group masks as well*/ + list_for_each_entry(crgrp, &r->mon.crdtgrp_list, mon.crdtgrp_list) + cpumask_and(&crgrp->cpu_mask, &r->cpu_mask, &crgrp->cpu_mask); +} + +static int cpus_ctrl_write(struct rdtgroup *rdtgrp, cpumask_var_t newmask, + cpumask_var_t tmpmask, cpumask_var_t tmpmask1) +{ + struct rdtgroup *r, *crgrp; + struct list_head *head; + + /* Check whether cpus are dropped from this group */ + cpumask_andnot(tmpmask, &rdtgrp->cpu_mask, newmask); + if (!cpumask_empty(tmpmask)) { + /* Can't drop from default group */ + if (rdtgrp == &rdtgroup_default) { + rdt_last_cmd_puts("Can't drop CPUs from default group\n"); + return -EINVAL; + } + + /* Give any dropped cpus to rdtgroup_default */ + cpumask_or(&rdtgroup_default.cpu_mask, + &rdtgroup_default.cpu_mask, tmpmask); + update_closid_rmid(tmpmask, &rdtgroup_default); + } + + /* + * If we added cpus, remove them from previous group and + * the prev group's child groups that owned them + * and update per-cpu closid/rmid. + */ + cpumask_andnot(tmpmask, newmask, &rdtgrp->cpu_mask); + if (!cpumask_empty(tmpmask)) { + list_for_each_entry(r, &rdt_all_groups, rdtgroup_list) { + if (r == rdtgrp) + continue; + cpumask_and(tmpmask1, &r->cpu_mask, tmpmask); + if (!cpumask_empty(tmpmask1)) + cpumask_rdtgrp_clear(r, tmpmask1); + } + update_closid_rmid(tmpmask, rdtgrp); + } + + /* Done pushing/pulling - update this group with new mask */ + cpumask_copy(&rdtgrp->cpu_mask, newmask); + + /* + * Clear child mon group masks since there is a new parent mask + * now and update the rmid for the cpus the child lost. + */ + head = &rdtgrp->mon.crdtgrp_list; + list_for_each_entry(crgrp, head, mon.crdtgrp_list) { + cpumask_and(tmpmask, &rdtgrp->cpu_mask, &crgrp->cpu_mask); + update_closid_rmid(tmpmask, rdtgrp); + cpumask_clear(&crgrp->cpu_mask); + } + + return 0; +} + +static ssize_t rdtgroup_cpus_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off) +{ + cpumask_var_t tmpmask = CPUMASK_VAR_NULL, newmask = CPUMASK_VAR_NULL; + cpumask_var_t tmpmask1 = CPUMASK_VAR_NULL; + struct rdtgroup *rdtgrp; + int ret; + + rdtgrp = rdtgroup_kn_lock_live(of->kn); + if (!rdtgrp) { + ret = -ENOENT; + goto out_unlock; + } + + if (!buf) { + rdt_last_cmd_printf("%s: Invalid input\n", + is_cpu_list(of) ? "cpus_list" : "cpus"); + ret = -EINVAL; + goto out_unlock; + } + + if (!zalloc_cpumask_var(&tmpmask, GFP_KERNEL) || + !zalloc_cpumask_var(&newmask, GFP_KERNEL) || + !zalloc_cpumask_var(&tmpmask1, GFP_KERNEL)) { + rdt_last_cmd_printf("%s: Kernel allocation failure\n", + is_cpu_list(of) ? "cpus_list" : "cpus"); + ret = -ENOMEM; + goto out_free; + } + + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED || + rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { + ret = -EINVAL; + rdt_last_cmd_puts("Pseudo-locking in progress\n"); + goto out_free; + } + + if (is_cpu_list(of)) + ret = cpulist_parse(buf, newmask); + else + ret = cpumask_parse(buf, newmask); + + if (ret) { + rdt_last_cmd_puts("Bad CPU list/mask\n"); + goto out_free; + } + + /* check that user didn't specify any offline cpus */ + cpumask_andnot(tmpmask, newmask, cpu_online_mask); + if (!cpumask_empty(tmpmask)) { + ret = -EINVAL; + rdt_last_cmd_puts("Can only assign online CPUs\n"); + goto out_free; + } + + if (rdtgrp->type == RDTCTRL_GROUP) + ret = cpus_ctrl_write(rdtgrp, newmask, tmpmask, tmpmask1); + else if (rdtgrp->type == RDTMON_GROUP) + ret = cpus_mon_write(rdtgrp, newmask, tmpmask); + else + ret = -EINVAL; + +out_free: + free_cpumask_var(tmpmask); + free_cpumask_var(newmask); + free_cpumask_var(tmpmask1); +out_unlock: + rdtgroup_kn_unlock(of->kn); + + return ret ?: nbytes; +} + +/** + * rdtgroup_remove - the helper to remove resource group safely + * @rdtgrp: resource group to remove + * + * On resource group creation via a mkdir, an extra kernfs_node reference is + * taken to ensure that the rdtgroup structure remains accessible for the + * rdtgroup_kn_unlock() calls where it is removed. The default group is + * statically allocated: it does not have an extra reference but will have + * RDT_DELETED set on unmount to support safe access to its associated files + * via rdtgroup_kn_lock_live/rdtgroup_kn_unlock(). + * + * For all but the default group: drop the extra reference, then free the + * rdtgroup structure. + * + * Return: void + */ +static void rdtgroup_remove(struct rdtgroup *rdtgrp) +{ + if (rdtgrp == &rdtgroup_default) + return; + kernfs_put(rdtgrp->kn); + kfree(rdtgrp); +} + +static void _update_task_closid_rmid(void *task) +{ + /* + * If the task is still current on this CPU, update PQR_ASSOC MSR. + * Otherwise, the MSR is updated when the task is scheduled in. + */ + if (task == current) + resctrl_arch_sched_in(task); +} + +static void update_task_closid_rmid(struct task_struct *t) +{ + if (IS_ENABLED(CONFIG_SMP) && task_curr(t)) + smp_call_function_single(task_cpu(t), _update_task_closid_rmid, t, 1); + else + _update_task_closid_rmid(t); +} + +static bool task_in_rdtgroup(struct task_struct *tsk, struct rdtgroup *rdtgrp) +{ + u32 closid, rmid = rdtgrp->mon.rmid; + + if (rdtgrp->type == RDTCTRL_GROUP) + closid = rdtgrp->closid; + else if (rdtgrp->type == RDTMON_GROUP) + closid = rdtgrp->mon.parent->closid; + else + return false; + + return resctrl_arch_match_closid(tsk, closid) && + resctrl_arch_match_rmid(tsk, closid, rmid); +} + +static int __rdtgroup_move_task(struct task_struct *tsk, + struct rdtgroup *rdtgrp) +{ + /* If the task is already in rdtgrp, no need to move the task. */ + if (task_in_rdtgroup(tsk, rdtgrp)) + return 0; + + /* + * Set the task's closid/rmid before the PQR_ASSOC MSR can be + * updated by them. + * + * For ctrl_mon groups, move both closid and rmid. + * For monitor groups, can move the tasks only from + * their parent CTRL group. + */ + if (rdtgrp->type == RDTMON_GROUP && + !resctrl_arch_match_closid(tsk, rdtgrp->mon.parent->closid)) { + rdt_last_cmd_puts("Can't move task to different control group\n"); + return -EINVAL; + } + + if (rdtgrp->type == RDTMON_GROUP) + resctrl_arch_set_closid_rmid(tsk, rdtgrp->mon.parent->closid, + rdtgrp->mon.rmid); + else + resctrl_arch_set_closid_rmid(tsk, rdtgrp->closid, + rdtgrp->mon.rmid); + + /* + * Ensure the task's closid and rmid are written before determining if + * the task is current that will decide if it will be interrupted. + * This pairs with the full barrier between the rq->curr update and + * resctrl_arch_sched_in() during context switch. + */ + smp_mb(); + + /* + * By now, the task's closid and rmid are set. If the task is current + * on a CPU, the PQR_ASSOC MSR needs to be updated to make the resource + * group go into effect. If the task is not current, the MSR will be + * updated when the task is scheduled in. + */ + update_task_closid_rmid(tsk); + + return 0; +} + +static bool is_closid_match(struct task_struct *t, struct rdtgroup *r) +{ + return (resctrl_arch_alloc_capable() && (r->type == RDTCTRL_GROUP) && + resctrl_arch_match_closid(t, r->closid)); +} + +static bool is_rmid_match(struct task_struct *t, struct rdtgroup *r) +{ + return (resctrl_arch_mon_capable() && (r->type == RDTMON_GROUP) && + resctrl_arch_match_rmid(t, r->mon.parent->closid, + r->mon.rmid)); +} + +/** + * rdtgroup_tasks_assigned - Test if tasks have been assigned to resource group + * @r: Resource group + * + * Return: 1 if tasks have been assigned to @r, 0 otherwise + */ +int rdtgroup_tasks_assigned(struct rdtgroup *r) +{ + struct task_struct *p, *t; + int ret = 0; + + lockdep_assert_held(&rdtgroup_mutex); + + rcu_read_lock(); + for_each_process_thread(p, t) { + if (is_closid_match(t, r) || is_rmid_match(t, r)) { + ret = 1; + break; + } + } + rcu_read_unlock(); + + return ret; +} + +static int rdtgroup_task_write_permission(struct task_struct *task, + struct kernfs_open_file *of) +{ + const struct cred *tcred = get_task_cred(task); + const struct cred *cred = current_cred(); + int ret = 0; + + /* + * Even if we're attaching all tasks in the thread group, we only + * need to check permissions on one of them. + */ + if (!uid_eq(cred->euid, GLOBAL_ROOT_UID) && + !uid_eq(cred->euid, tcred->uid) && + !uid_eq(cred->euid, tcred->suid)) { + rdt_last_cmd_printf("No permission to move task %d\n", task->pid); + ret = -EPERM; + } + + put_cred(tcred); + return ret; +} + +static int rdtgroup_move_task(pid_t pid, struct rdtgroup *rdtgrp, + struct kernfs_open_file *of) +{ + struct task_struct *tsk; + int ret; + + rcu_read_lock(); + if (pid) { + tsk = find_task_by_vpid(pid); + if (!tsk) { + rcu_read_unlock(); + rdt_last_cmd_printf("No task %d\n", pid); + return -ESRCH; + } + } else { + tsk = current; + } + + get_task_struct(tsk); + rcu_read_unlock(); + + ret = rdtgroup_task_write_permission(tsk, of); + if (!ret) + ret = __rdtgroup_move_task(tsk, rdtgrp); + + put_task_struct(tsk); + return ret; +} + +static ssize_t rdtgroup_tasks_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off) +{ + struct rdtgroup *rdtgrp; + char *pid_str; + int ret = 0; + pid_t pid; + + rdtgrp = rdtgroup_kn_lock_live(of->kn); + if (!rdtgrp) { + rdtgroup_kn_unlock(of->kn); + return -ENOENT; + } + + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED || + rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { + ret = -EINVAL; + rdt_last_cmd_puts("Pseudo-locking in progress\n"); + goto unlock; + } + + while (buf && buf[0] != '\0' && buf[0] != '\n') { + pid_str = strim(strsep(&buf, ",")); + + if (kstrtoint(pid_str, 0, &pid)) { + rdt_last_cmd_printf("Task list parsing error pid %s\n", pid_str); + ret = -EINVAL; + break; + } + + if (pid < 0) { + rdt_last_cmd_printf("Invalid pid %d\n", pid); + ret = -EINVAL; + break; + } + + ret = rdtgroup_move_task(pid, rdtgrp, of); + if (ret) { + rdt_last_cmd_printf("Error while processing task %d\n", pid); + break; + } + } + +unlock: + rdtgroup_kn_unlock(of->kn); + + return ret ?: nbytes; +} + +static void show_rdt_tasks(struct rdtgroup *r, struct seq_file *s) +{ + struct task_struct *p, *t; + pid_t pid; + + rcu_read_lock(); + for_each_process_thread(p, t) { + if (is_closid_match(t, r) || is_rmid_match(t, r)) { + pid = task_pid_vnr(t); + if (pid) + seq_printf(s, "%d\n", pid); + } + } + rcu_read_unlock(); +} + +static int rdtgroup_tasks_show(struct kernfs_open_file *of, + struct seq_file *s, void *v) +{ + struct rdtgroup *rdtgrp; + int ret = 0; + + rdtgrp = rdtgroup_kn_lock_live(of->kn); + if (rdtgrp) + show_rdt_tasks(rdtgrp, s); + else + ret = -ENOENT; + rdtgroup_kn_unlock(of->kn); + + return ret; +} + +static int rdtgroup_closid_show(struct kernfs_open_file *of, + struct seq_file *s, void *v) +{ + struct rdtgroup *rdtgrp; + int ret = 0; + + rdtgrp = rdtgroup_kn_lock_live(of->kn); + if (rdtgrp) + seq_printf(s, "%u\n", rdtgrp->closid); + else + ret = -ENOENT; + rdtgroup_kn_unlock(of->kn); + + return ret; +} + +static int rdtgroup_rmid_show(struct kernfs_open_file *of, + struct seq_file *s, void *v) +{ + struct rdtgroup *rdtgrp; + int ret = 0; + + rdtgrp = rdtgroup_kn_lock_live(of->kn); + if (rdtgrp) + seq_printf(s, "%u\n", rdtgrp->mon.rmid); + else + ret = -ENOENT; + rdtgroup_kn_unlock(of->kn); + + return ret; +} + +#ifdef CONFIG_PROC_CPU_RESCTRL +/* + * A task can only be part of one resctrl control group and of one monitor + * group which is associated to that control group. + * + * 1) res: + * mon: + * + * resctrl is not available. + * + * 2) res:/ + * mon: + * + * Task is part of the root resctrl control group, and it is not associated + * to any monitor group. + * + * 3) res:/ + * mon:mon0 + * + * Task is part of the root resctrl control group and monitor group mon0. + * + * 4) res:group0 + * mon: + * + * Task is part of resctrl control group group0, and it is not associated + * to any monitor group. + * + * 5) res:group0 + * mon:mon1 + * + * Task is part of resctrl control group group0 and monitor group mon1. + */ +int proc_resctrl_show(struct seq_file *s, struct pid_namespace *ns, + struct pid *pid, struct task_struct *tsk) +{ + struct rdtgroup *rdtg; + int ret = 0; + + mutex_lock(&rdtgroup_mutex); + + /* Return empty if resctrl has not been mounted. */ + if (!resctrl_mounted) { + seq_puts(s, "res:\nmon:\n"); + goto unlock; + } + + list_for_each_entry(rdtg, &rdt_all_groups, rdtgroup_list) { + struct rdtgroup *crg; + + /* + * Task information is only relevant for shareable + * and exclusive groups. + */ + if (rdtg->mode != RDT_MODE_SHAREABLE && + rdtg->mode != RDT_MODE_EXCLUSIVE) + continue; + + if (!resctrl_arch_match_closid(tsk, rdtg->closid)) + continue; + + seq_printf(s, "res:%s%s\n", (rdtg == &rdtgroup_default) ? "/" : "", + rdt_kn_name(rdtg->kn)); + seq_puts(s, "mon:"); + list_for_each_entry(crg, &rdtg->mon.crdtgrp_list, + mon.crdtgrp_list) { + if (!resctrl_arch_match_rmid(tsk, crg->mon.parent->closid, + crg->mon.rmid)) + continue; + seq_printf(s, "%s", rdt_kn_name(crg->kn)); + break; + } + seq_putc(s, '\n'); + goto unlock; + } + /* + * The above search should succeed. Otherwise return + * with an error. + */ + ret = -ENOENT; +unlock: + mutex_unlock(&rdtgroup_mutex); + + return ret; +} +#endif + +static int rdt_last_cmd_status_show(struct kernfs_open_file *of, + struct seq_file *seq, void *v) +{ + int len; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + len = seq_buf_used(&last_cmd_status); + if (len) { + seq_printf(seq, "%.*s", len, last_cmd_status_buf); + if (seq_buf_has_overflowed(&last_cmd_status)) + seq_puts(seq, "[truncated]\n"); + } else { + seq_puts(seq, "ok\n"); + } + info_kn_unlock(of->kn); + return 0; +} + +void *rdt_kn_parent_priv(struct kernfs_node *kn) +{ + /* + * The parent pointer is only valid within RCU section since it can be + * replaced. + */ + guard(rcu)(); + return rcu_dereference(kn->__parent)->priv; +} + +static int rdt_num_closids_show(struct kernfs_open_file *of, + struct seq_file *seq, void *v) +{ + struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); + + if (!info_kn_lock(of->kn)) + return -ENOENT; + seq_printf(seq, "%u\n", s->num_closid); + info_kn_unlock(of->kn); + + return 0; +} + +static int rdt_default_ctrl_show(struct kernfs_open_file *of, + struct seq_file *seq, void *v) +{ + struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); + struct rdt_resource *r; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + r = s->res; + seq_printf(seq, "%x\n", resctrl_get_default_ctrl(r)); + info_kn_unlock(of->kn); + + return 0; +} + +static int rdt_min_cbm_bits_show(struct kernfs_open_file *of, + struct seq_file *seq, void *v) +{ + struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); + struct rdt_resource *r; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + r = s->res; + seq_printf(seq, "%u\n", r->cache.min_cbm_bits); + info_kn_unlock(of->kn); + + return 0; +} + +static int rdt_shareable_bits_show(struct kernfs_open_file *of, + struct seq_file *seq, void *v) +{ + struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); + struct rdt_resource *r; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + r = s->res; + seq_printf(seq, "%x\n", r->cache.shareable_bits); + info_kn_unlock(of->kn); + + return 0; +} + +/* + * rdt_bit_usage_show - Display current usage of resources + * + * A domain is a shared resource that can now be allocated differently. Here + * we display the current regions of the domain as an annotated bitmask. + * For each domain of this resource its allocation bitmask + * is annotated as below to indicate the current usage of the corresponding bit: + * 0 - currently unused + * X - currently available for sharing and used by software and hardware + * H - currently used by hardware only but available for software use + * S - currently used and shareable by software only + * E - currently used exclusively by one resource group + * P - currently pseudo-locked by one resource group + */ +static int rdt_bit_usage_show(struct kernfs_open_file *of, + struct seq_file *seq, void *v) +{ + struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); + /* + * Use unsigned long even though only 32 bits are used to ensure + * test_bit() is used safely. + */ + unsigned long sw_shareable = 0, hw_shareable = 0; + unsigned long exclusive = 0, pseudo_locked = 0; + struct rdt_ctrl_domain *dom; + int i, hwb, swb, excl, psl; + struct rdt_resource *r; + enum rdtgrp_mode mode; + bool sep = false; + u32 ctrl_val; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + r = s->res; + list_for_each_entry_rcu(dom, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) { + if (sep) + seq_putc(seq, ';'); + hw_shareable = r->cache.shareable_bits; + sw_shareable = 0; + exclusive = 0; + seq_printf(seq, "%d=", dom->hdr.id); + for (i = 0; i < closids_supported(); i++) { + if (!closid_allocated(i) || + (resctrl_arch_get_io_alloc_enabled(r) && + i == resctrl_io_alloc_closid(r))) + continue; + ctrl_val = resctrl_arch_get_config(r, dom, i, + s->conf_type); + mode = rdtgroup_mode_by_closid(i); + switch (mode) { + case RDT_MODE_SHAREABLE: + sw_shareable |= ctrl_val; + break; + case RDT_MODE_EXCLUSIVE: + exclusive |= ctrl_val; + break; + case RDT_MODE_PSEUDO_LOCKSETUP: + /* + * RDT_MODE_PSEUDO_LOCKSETUP is possible + * here but not included since the CBM + * associated with this CLOSID in this mode + * is not initialized and no task or cpu can be + * assigned this CLOSID. + */ + break; + case RDT_MODE_PSEUDO_LOCKED: + case RDT_NUM_MODES: + WARN(1, + "invalid mode for closid %d\n", i); + break; + } + } + + /* + * When the "io_alloc" feature is enabled, a portion of the cache + * is configured for shared use between hardware and software. + * Also, when CDP is enabled the CBMs of CDP_CODE and CDP_DATA + * resources are kept in sync. So, the CBMs for "io_alloc" can + * be accessed through either resource. + */ + if (resctrl_arch_get_io_alloc_enabled(r)) { + ctrl_val = resctrl_arch_get_config(r, dom, + resctrl_io_alloc_closid(r), + s->conf_type); + hw_shareable |= ctrl_val; + } + + for (i = r->cache.cbm_len - 1; i >= 0; i--) { + pseudo_locked = dom->plr ? dom->plr->cbm : 0; + hwb = test_bit(i, &hw_shareable); + swb = test_bit(i, &sw_shareable); + excl = test_bit(i, &exclusive); + psl = test_bit(i, &pseudo_locked); + if (hwb && swb) + seq_putc(seq, 'X'); + else if (hwb && !swb) + seq_putc(seq, 'H'); + else if (!hwb && swb) + seq_putc(seq, 'S'); + else if (excl) + seq_putc(seq, 'E'); + else if (psl) + seq_putc(seq, 'P'); + else /* Unused bits remain */ + seq_putc(seq, '0'); + } + sep = true; + } + seq_putc(seq, '\n'); + info_kn_unlock(of->kn); + return 0; +} + +static int rdt_min_bw_show(struct kernfs_open_file *of, + struct seq_file *seq, void *v) +{ + struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); + struct rdt_resource *r; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + r = s->res; + seq_printf(seq, "%u\n", r->membw.min_bw); + info_kn_unlock(of->kn); + + return 0; +} + +static int rdt_num_rmids_show(struct kernfs_open_file *of, + struct seq_file *seq, void *v) +{ + struct rdt_resource *r = rdt_kn_parent_priv(of->kn); + + if (!info_kn_lock(of->kn)) + return -ENOENT; + seq_printf(seq, "%u\n", r->mon.num_rmid); + + info_kn_unlock(of->kn); + + return 0; +} + +static int rdt_mon_features_show(struct kernfs_open_file *of, + struct seq_file *seq, void *v) +{ + struct rdt_resource *r = rdt_kn_parent_priv(of->kn); + struct mon_evt *mevt; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + for_each_mon_event(mevt) { + if (mevt->rid != r->rid || !mevt->enabled) + continue; + seq_printf(seq, "%s\n", mevt->name); + if (mevt->configurable && + !resctrl_arch_mbm_cntr_assign_enabled(r)) + seq_printf(seq, "%s_config\n", mevt->name); + } + + info_kn_unlock(of->kn); + + return 0; +} + +static int rdt_bw_gran_show(struct kernfs_open_file *of, + struct seq_file *seq, void *v) +{ + struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); + struct rdt_resource *r; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + r = s->res; + seq_printf(seq, "%u\n", r->membw.bw_gran); + info_kn_unlock(of->kn); + + return 0; +} + +static int rdt_delay_linear_show(struct kernfs_open_file *of, + struct seq_file *seq, void *v) +{ + struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); + struct rdt_resource *r; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + r = s->res; + seq_printf(seq, "%u\n", r->membw.delay_linear); + info_kn_unlock(of->kn); + + return 0; +} + +static int max_threshold_occ_show(struct kernfs_open_file *of, + struct seq_file *seq, void *v) +{ + if (!info_kn_lock(of->kn)) + return -ENOENT; + seq_printf(seq, "%u\n", resctrl_rmid_realloc_threshold); + info_kn_unlock(of->kn); + + return 0; +} + +static int rdt_thread_throttle_mode_show(struct kernfs_open_file *of, + struct seq_file *seq, void *v) +{ + struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); + struct rdt_resource *r; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + + r = s->res; + switch (r->membw.throttle_mode) { + case THREAD_THROTTLE_PER_THREAD: + seq_puts(seq, "per-thread\n"); + break; + case THREAD_THROTTLE_MAX: + seq_puts(seq, "max\n"); + break; + case THREAD_THROTTLE_UNDEFINED: + seq_puts(seq, "undefined\n"); + break; + default: + WARN_ON_ONCE(1); + break; + } + + info_kn_unlock(of->kn); + return 0; +} + +static ssize_t max_threshold_occ_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off) +{ + unsigned int bytes; + int ret; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + + rdt_last_cmd_clear(); + + ret = kstrtouint(buf, 0, &bytes); + if (ret) { + rdt_last_cmd_puts("max_threshold_occupancy: Invalid input\n"); + goto out_unlock; + } + + if (bytes > resctrl_rmid_realloc_limit) { + rdt_last_cmd_printf("max_threshold_occupancy: Exceeds limit (before adjustment) of %u bytes\n", + resctrl_rmid_realloc_limit); + ret = -EINVAL; + goto out_unlock; + } + + resctrl_rmid_realloc_threshold = resctrl_arch_round_mon_val(bytes); + +out_unlock: + info_kn_unlock(of->kn); + + return ret ?: nbytes; +} + +/* + * rdtgroup_mode_show - Display mode of this resource group + */ +static int rdtgroup_mode_show(struct kernfs_open_file *of, + struct seq_file *s, void *v) +{ + struct rdtgroup *rdtgrp; + + rdtgrp = rdtgroup_kn_lock_live(of->kn); + if (!rdtgrp) { + rdtgroup_kn_unlock(of->kn); + return -ENOENT; + } + + seq_printf(s, "%s\n", rdtgroup_mode_str(rdtgrp->mode)); + + rdtgroup_kn_unlock(of->kn); + return 0; +} + +enum resctrl_conf_type resctrl_peer_type(enum resctrl_conf_type my_type) +{ + switch (my_type) { + case CDP_CODE: + return CDP_DATA; + case CDP_DATA: + return CDP_CODE; + default: + case CDP_NONE: + return CDP_NONE; + } +} + +static int rdt_has_sparse_bitmasks_show(struct kernfs_open_file *of, + struct seq_file *seq, void *v) +{ + struct resctrl_schema *s = rdt_kn_parent_priv(of->kn); + struct rdt_resource *r; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + r = s->res; + seq_printf(seq, "%u\n", r->cache.arch_has_sparse_bitmasks); + + info_kn_unlock(of->kn); + + return 0; +} + +/** + * __rdtgroup_cbm_overlaps - Does CBM for intended closid overlap with other + * @r: Resource to which domain instance @d belongs. + * @d: The domain instance for which @closid is being tested. + * @cbm: Capacity bitmask being tested. + * @closid: Intended closid for @cbm. + * @type: CDP type of @r. + * @exclusive: Only check if overlaps with exclusive resource groups + * + * Checks if provided @cbm intended to be used for @closid on domain + * @d overlaps with any other closids or other hardware usage associated + * with this domain. If @exclusive is true then only overlaps with + * resource groups in exclusive mode will be considered. If @exclusive + * is false then overlaps with any resource group or hardware entities + * will be considered. + * + * @cbm is unsigned long, even if only 32 bits are used, to make the + * bitmap functions work correctly. + * + * Return: false if CBM does not overlap, true if it does. + */ +static bool __rdtgroup_cbm_overlaps(struct rdt_resource *r, struct rdt_ctrl_domain *d, + unsigned long cbm, int closid, + enum resctrl_conf_type type, bool exclusive) +{ + enum rdtgrp_mode mode; + unsigned long ctrl_b; + int i; + + /* Check for any overlap with regions used by hardware directly */ + if (!exclusive) { + ctrl_b = r->cache.shareable_bits; + if (bitmap_intersects(&cbm, &ctrl_b, r->cache.cbm_len)) + return true; + } + + /* Check for overlap with other resource groups */ + for (i = 0; i < closids_supported(); i++) { + ctrl_b = resctrl_arch_get_config(r, d, i, type); + mode = rdtgroup_mode_by_closid(i); + if (closid_allocated(i) && i != closid && + mode != RDT_MODE_PSEUDO_LOCKSETUP) { + if (bitmap_intersects(&cbm, &ctrl_b, r->cache.cbm_len)) { + if (exclusive) { + if (mode == RDT_MODE_EXCLUSIVE) + return true; + continue; + } + return true; + } + } + } + + return false; +} + +/** + * rdtgroup_cbm_overlaps - Does CBM overlap with other use of hardware + * @s: Schema for the resource to which domain instance @d belongs. + * @d: The domain instance for which @closid is being tested. + * @cbm: Capacity bitmask being tested. + * @closid: Intended closid for @cbm. + * @exclusive: Only check if overlaps with exclusive resource groups + * + * Resources that can be allocated using a CBM can use the CBM to control + * the overlap of these allocations. rdtgroup_cmb_overlaps() is the test + * for overlap. Overlap test is not limited to the specific resource for + * which the CBM is intended though - when dealing with CDP resources that + * share the underlying hardware the overlap check should be performed on + * the CDP resource sharing the hardware also. + * + * Refer to description of __rdtgroup_cbm_overlaps() for the details of the + * overlap test. + * + * Return: true if CBM overlap detected, false if there is no overlap + */ +bool rdtgroup_cbm_overlaps(struct resctrl_schema *s, struct rdt_ctrl_domain *d, + unsigned long cbm, int closid, bool exclusive) +{ + enum resctrl_conf_type peer_type = resctrl_peer_type(s->conf_type); + struct rdt_resource *r = s->res; + + if (__rdtgroup_cbm_overlaps(r, d, cbm, closid, s->conf_type, + exclusive)) + return true; + + if (!resctrl_arch_get_cdp_enabled(r->rid)) + return false; + return __rdtgroup_cbm_overlaps(r, d, cbm, closid, peer_type, exclusive); +} + +/** + * rdtgroup_mode_test_exclusive - Test if this resource group can be exclusive + * @rdtgrp: Resource group identified through its closid. + * + * An exclusive resource group implies that there should be no sharing of + * its allocated resources. At the time this group is considered to be + * exclusive this test can determine if its current schemata supports this + * setting by testing for overlap with all other resource groups. + * + * Return: true if resource group can be exclusive, false if there is overlap + * with allocations of other resource groups and thus this resource group + * cannot be exclusive. + */ +static bool rdtgroup_mode_test_exclusive(struct rdtgroup *rdtgrp) +{ + int closid = rdtgrp->closid; + struct rdt_ctrl_domain *d; + struct resctrl_schema *s; + struct rdt_resource *r; + bool has_cache = false; + u32 ctrl; + + /* Walking r->domains, ensure it can't race with cpuhp */ + lockdep_assert_cpus_held(); + + list_for_each_entry(s, &resctrl_schema_all, list) { + r = s->res; + if (r->rid == RDT_RESOURCE_MBA || r->rid == RDT_RESOURCE_SMBA) + continue; + has_cache = true; + list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) { + ctrl = resctrl_arch_get_config(r, d, closid, + s->conf_type); + if (rdtgroup_cbm_overlaps(s, d, ctrl, closid, false)) { + rdt_last_cmd_puts("Schemata overlaps\n"); + return false; + } + } + } + + if (!has_cache) { + rdt_last_cmd_puts("Cannot be exclusive without CAT/CDP\n"); + return false; + } + + return true; +} + +/* + * rdtgroup_mode_write - Modify the resource group's mode + */ +static ssize_t rdtgroup_mode_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off) +{ + struct rdtgroup *rdtgrp; + enum rdtgrp_mode mode; + int ret = 0; + + rdtgrp = rdtgroup_kn_lock_live(of->kn); + if (!rdtgrp) { + rdtgroup_kn_unlock(of->kn); + return -ENOENT; + } + + /* Valid input requires a trailing newline */ + if (nbytes == 0 || buf[nbytes - 1] != '\n') { + rdt_last_cmd_puts("mode: Invalid input\n"); + ret = -EINVAL; + goto out; + } + + buf[nbytes - 1] = '\0'; + + mode = rdtgrp->mode; + + if ((!strcmp(buf, "shareable") && mode == RDT_MODE_SHAREABLE) || + (!strcmp(buf, "exclusive") && mode == RDT_MODE_EXCLUSIVE) || + (!strcmp(buf, "pseudo-locksetup") && + mode == RDT_MODE_PSEUDO_LOCKSETUP) || + (!strcmp(buf, "pseudo-locked") && mode == RDT_MODE_PSEUDO_LOCKED)) + goto out; + + if (mode == RDT_MODE_PSEUDO_LOCKED) { + rdt_last_cmd_puts("Cannot change pseudo-locked group\n"); + ret = -EINVAL; + goto out; + } + + if (!strcmp(buf, "shareable")) { + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { + ret = rdtgroup_locksetup_exit(rdtgrp); + if (ret) + goto out; + } + rdtgrp->mode = RDT_MODE_SHAREABLE; + } else if (!strcmp(buf, "exclusive")) { + if (!rdtgroup_mode_test_exclusive(rdtgrp)) { + ret = -EINVAL; + goto out; + } + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { + ret = rdtgroup_locksetup_exit(rdtgrp); + if (ret) + goto out; + } + rdtgrp->mode = RDT_MODE_EXCLUSIVE; + } else if (IS_ENABLED(CONFIG_RESCTRL_FS_PSEUDO_LOCK) && + !strcmp(buf, "pseudo-locksetup")) { + ret = rdtgroup_locksetup_enter(rdtgrp); + if (ret) + goto out; + rdtgrp->mode = RDT_MODE_PSEUDO_LOCKSETUP; + } else { + rdt_last_cmd_puts("Unknown or unsupported mode\n"); + ret = -EINVAL; + } + +out: + rdtgroup_kn_unlock(of->kn); + return ret ?: nbytes; +} + +/** + * rdtgroup_cbm_to_size - Translate CBM to size in bytes + * @r: RDT resource to which @d belongs. + * @d: RDT domain instance. + * @cbm: bitmask for which the size should be computed. + * + * The bitmask provided associated with the RDT domain instance @d will be + * translated into how many bytes it represents. The size in bytes is + * computed by first dividing the total cache size by the CBM length to + * determine how many bytes each bit in the bitmask represents. The result + * is multiplied with the number of bits set in the bitmask. + * + * @cbm is unsigned long, even if only 32 bits are used to make the + * bitmap functions work correctly. + * + * Return: Size (in bytes) of cache portion represented by CBM, 0 on failure. + */ +unsigned int rdtgroup_cbm_to_size(struct rdt_resource *r, + struct rdt_ctrl_domain *d, unsigned long cbm) +{ + unsigned int size = 0; + struct cacheinfo *ci; + int num_b; + + if (WARN_ON_ONCE(r->ctrl_scope != RESCTRL_L2_CACHE && r->ctrl_scope != RESCTRL_L3_CACHE)) + return size; + + num_b = bitmap_weight(&cbm, r->cache.cbm_len); + ci = get_cpu_cacheinfo_level(cpumask_any(&d->hdr.cpu_mask), r->ctrl_scope); + if (ci) + size = ci->size / r->cache.cbm_len * num_b; + + return size; +} + +bool is_mba_sc(struct rdt_resource *r) +{ + if (!r) + r = resctrl_arch_get_resource(RDT_RESOURCE_MBA); + + /* + * The software controller support is only applicable to MBA resource. + * Make sure to check for resource type. + */ + if (r->rid != RDT_RESOURCE_MBA) + return false; + + return r->membw.mba_sc; +} + +/* + * rdtgroup_size_show - Display size in bytes of allocated regions + * + * The "size" file mirrors the layout of the "schemata" file, printing the + * size in bytes of each region instead of the capacity bitmask. + */ +static int rdtgroup_size_show(struct kernfs_open_file *of, + struct seq_file *s, void *v) +{ + struct resctrl_schema *schema; + enum resctrl_conf_type type; + struct rdt_ctrl_domain *d; + struct rdtgroup *rdtgrp; + struct rdt_resource *r; + unsigned int size; + int ret = 0; + u32 closid; + bool sep; + u32 ctrl; + + rdtgrp = rdtgroup_kn_lock_live(of->kn); + if (!rdtgrp) { + rdtgroup_kn_unlock(of->kn); + return -ENOENT; + } + + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) { + if (!rdtgrp->plr->d) { + rdt_last_cmd_puts("Cache domain offline\n"); + ret = -ENODEV; + } else { + seq_printf(s, "%*s:", max_name_width, + rdtgrp->plr->s->name); + size = rdtgroup_cbm_to_size(rdtgrp->plr->s->res, + rdtgrp->plr->d, + rdtgrp->plr->cbm); + seq_printf(s, "%d=%u\n", rdtgrp->plr->d->hdr.id, size); + } + goto out; + } + + closid = rdtgrp->closid; + + list_for_each_entry(schema, &resctrl_schema_all, list) { + r = schema->res; + type = schema->conf_type; + sep = false; + seq_printf(s, "%*s:", max_name_width, schema->name); + list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) { + if (sep) + seq_putc(s, ';'); + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) { + size = 0; + } else { + if (is_mba_sc(r)) + ctrl = d->mbps_val[closid]; + else + ctrl = resctrl_arch_get_config(r, d, + closid, + type); + if (r->rid == RDT_RESOURCE_MBA || + r->rid == RDT_RESOURCE_SMBA) + size = ctrl; + else + size = rdtgroup_cbm_to_size(r, d, ctrl); + } + seq_printf(s, "%d=%u", d->hdr.id, size); + sep = true; + } + seq_putc(s, '\n'); + } + +out: + rdtgroup_kn_unlock(of->kn); + + return ret; +} + +static void mondata_config_read(struct resctrl_mon_config_info *mon_info) +{ + smp_call_function_any(&mon_info->d->hdr.cpu_mask, + resctrl_arch_mon_event_config_read, mon_info, 1); +} + +static int mbm_config_show(struct seq_file *s, struct rdt_resource *r, u32 evtid) +{ + struct resctrl_mon_config_info mon_info; + struct rdt_l3_mon_domain *dom; + bool sep = false; + + lockdep_assert_cpus_held(); + lockdep_assert_held(&rdtgroup_mutex); + + list_for_each_entry_rcu(dom, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) { + if (sep) + seq_puts(s, ";"); + + memset(&mon_info, 0, sizeof(struct resctrl_mon_config_info)); + mon_info.r = r; + mon_info.d = dom; + mon_info.evtid = evtid; + mondata_config_read(&mon_info); + + seq_printf(s, "%d=0x%02x", dom->hdr.id, mon_info.mon_config); + sep = true; + } + seq_puts(s, "\n"); + + return 0; +} + +static int mbm_total_bytes_config_show(struct kernfs_open_file *of, + struct seq_file *seq, void *v) +{ + struct rdt_resource *r = rdt_kn_parent_priv(of->kn); + + if (!info_kn_lock(of->kn)) + return -ENOENT; + + mbm_config_show(seq, r, QOS_L3_MBM_TOTAL_EVENT_ID); + + info_kn_unlock(of->kn); + return 0; +} + +static int mbm_local_bytes_config_show(struct kernfs_open_file *of, + struct seq_file *seq, void *v) +{ + struct rdt_resource *r = rdt_kn_parent_priv(of->kn); + + if (!info_kn_lock(of->kn)) + return -ENOENT; + + mbm_config_show(seq, r, QOS_L3_MBM_LOCAL_EVENT_ID); + + info_kn_unlock(of->kn); + return 0; +} + +static void mbm_config_write_domain(struct rdt_resource *r, + struct rdt_l3_mon_domain *d, u32 evtid, u32 val) +{ + struct resctrl_mon_config_info mon_info = {0}; + + /* + * Read the current config value first. If both are the same then + * no need to write it again. + */ + mon_info.r = r; + mon_info.d = d; + mon_info.evtid = evtid; + mondata_config_read(&mon_info); + if (mon_info.mon_config == val) + return; + + mon_info.mon_config = val; + + /* + * Update MSR_IA32_EVT_CFG_BASE MSR on one of the CPUs in the + * domain. The MSRs offset from MSR MSR_IA32_EVT_CFG_BASE + * are scoped at the domain level. Writing any of these MSRs + * on one CPU is observed by all the CPUs in the domain. + */ + smp_call_function_any(&d->hdr.cpu_mask, resctrl_arch_mon_event_config_write, + &mon_info, 1); + + /* + * When an Event Configuration is changed, the bandwidth counters + * for all RMIDs and Events will be cleared by the hardware. The + * hardware also sets MSR_IA32_QM_CTR.Unavailable (bit 62) for + * every RMID on the next read to any event for every RMID. + * Subsequent reads will have MSR_IA32_QM_CTR.Unavailable (bit 62) + * cleared while it is tracked by the hardware. Clear the + * mbm_local and mbm_total counts for all the RMIDs. + */ + resctrl_arch_reset_rmid_all(r, d); +} + +static int mon_config_write(struct rdt_resource *r, char *tok, u32 evtid) +{ + char *dom_str = NULL, *id_str; + struct rdt_l3_mon_domain *d; + unsigned long dom_id, val; + + /* Walking r->domains, ensure it can't race with cpuhp */ + lockdep_assert_cpus_held(); + +next: + if (!tok || tok[0] == '\0') + return 0; + + /* Start processing the strings for each domain */ + dom_str = strim(strsep(&tok, ";")); + id_str = strsep(&dom_str, "="); + + if (!id_str || kstrtoul(id_str, 10, &dom_id)) { + rdt_last_cmd_puts("Missing '=' or non-numeric domain id\n"); + return -EINVAL; + } + + if (!dom_str || kstrtoul(dom_str, 16, &val)) { + rdt_last_cmd_puts("Non-numeric event configuration value\n"); + return -EINVAL; + } + + /* Value from user cannot be more than the supported set of events */ + if ((val & r->mon.mbm_cfg_mask) != val) { + rdt_last_cmd_printf("Invalid event configuration: max valid mask is 0x%02x\n", + r->mon.mbm_cfg_mask); + return -EINVAL; + } + + list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) { + if (d->hdr.id == dom_id) { + mbm_config_write_domain(r, d, evtid, val); + goto next; + } + } + + return -EINVAL; +} + +static ssize_t mbm_total_bytes_config_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, + loff_t off) +{ + struct rdt_resource *r = rdt_kn_parent_priv(of->kn); + int ret; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + + rdt_last_cmd_clear(); + + /* Valid input requires a trailing newline */ + if (nbytes == 0 || buf[nbytes - 1] != '\n') { + rdt_last_cmd_puts("mbm_total_bytes_config: Invalid input\n"); + ret = -EINVAL; + goto out_unlock; + } + + buf[nbytes - 1] = '\0'; + + ret = mon_config_write(r, buf, QOS_L3_MBM_TOTAL_EVENT_ID); + +out_unlock: + info_kn_unlock(of->kn); + + return ret ?: nbytes; +} + +static ssize_t mbm_local_bytes_config_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, + loff_t off) +{ + struct rdt_resource *r = rdt_kn_parent_priv(of->kn); + int ret; + + if (!info_kn_lock(of->kn)) + return -ENOENT; + + rdt_last_cmd_clear(); + + /* Valid input requires a trailing newline */ + if (nbytes == 0 || buf[nbytes - 1] != '\n') { + rdt_last_cmd_puts("mbm_local_bytes_config: Invalid input\n"); + ret = -EINVAL; + goto out_unlock; + } + + buf[nbytes - 1] = '\0'; + + ret = mon_config_write(r, buf, QOS_L3_MBM_LOCAL_EVENT_ID); + +out_unlock: + info_kn_unlock(of->kn); + + return ret ?: nbytes; +} + +/* + * resctrl_bmec_files_show() — Controls the visibility of BMEC-related resctrl + * files. When @show is true, the files are displayed; when false, the files + * are hidden. + * Don't treat kernfs_find_and_get failure as an error, since this function may + * be called regardless of whether BMEC is supported or the event is enabled. + */ +void resctrl_bmec_files_show(struct rdt_resource *r, struct kernfs_node *l3_mon_kn, + bool show) +{ + struct kernfs_node *kn_config, *mon_kn = NULL; + char name[32]; + + if (!l3_mon_kn) { + sprintf(name, "%s_MON", r->name); + mon_kn = kernfs_find_and_get(kn_info, name); + if (!mon_kn) + return; + l3_mon_kn = mon_kn; + } + + kn_config = kernfs_find_and_get(l3_mon_kn, "mbm_total_bytes_config"); + if (kn_config) { + kernfs_show(kn_config, show); + kernfs_put(kn_config); + } + + kn_config = kernfs_find_and_get(l3_mon_kn, "mbm_local_bytes_config"); + if (kn_config) { + kernfs_show(kn_config, show); + kernfs_put(kn_config); + } + + /* Release the reference only if it was acquired */ + if (mon_kn) + kernfs_put(mon_kn); +} + +const char *rdtgroup_name_by_closid(u32 closid) +{ + struct rdtgroup *rdtgrp; + + list_for_each_entry(rdtgrp, &rdt_all_groups, rdtgroup_list) { + if (rdtgrp->closid == closid) + return rdt_kn_name(rdtgrp->kn); + } + + return NULL; +} + +/* rdtgroup information files for one cache resource. */ +static struct rftype res_common_files[] = { + { + .name = "last_cmd_status", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = rdt_last_cmd_status_show, + .fflags = RFTYPE_TOP_INFO, + }, + { + .name = "mbm_assign_on_mkdir", + .mode = 0644, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = resctrl_mbm_assign_on_mkdir_show, + .write = resctrl_mbm_assign_on_mkdir_write, + }, + { + .name = "num_closids", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = rdt_num_closids_show, + .fflags = RFTYPE_CTRL_INFO, + }, + { + .name = "mon_features", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = rdt_mon_features_show, + .fflags = RFTYPE_MON_INFO, + }, + { + .name = "available_mbm_cntrs", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = resctrl_available_mbm_cntrs_show, + }, + { + .name = "num_rmids", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = rdt_num_rmids_show, + .fflags = RFTYPE_MON_INFO, + }, + { + .name = "cbm_mask", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = rdt_default_ctrl_show, + .fflags = RFTYPE_CTRL_INFO | RFTYPE_RES_CACHE, + }, + { + .name = "num_mbm_cntrs", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = resctrl_num_mbm_cntrs_show, + }, + { + .name = "min_cbm_bits", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = rdt_min_cbm_bits_show, + .fflags = RFTYPE_CTRL_INFO | RFTYPE_RES_CACHE, + }, + { + .name = "shareable_bits", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = rdt_shareable_bits_show, + .fflags = RFTYPE_CTRL_INFO | RFTYPE_RES_CACHE, + }, + { + .name = "bit_usage", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = rdt_bit_usage_show, + .fflags = RFTYPE_CTRL_INFO | RFTYPE_RES_CACHE, + }, + { + .name = "min_bandwidth", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = rdt_min_bw_show, + .fflags = RFTYPE_CTRL_INFO | RFTYPE_RES_MB, + }, + { + .name = "bandwidth_gran", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = rdt_bw_gran_show, + .fflags = RFTYPE_CTRL_INFO | RFTYPE_RES_MB, + }, + { + .name = "delay_linear", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = rdt_delay_linear_show, + .fflags = RFTYPE_CTRL_INFO | RFTYPE_RES_MB, + }, + /* + * Platform specific which (if any) capabilities are provided by + * thread_throttle_mode. Defer "fflags" initialization to platform + * discovery. + */ + { + .name = "thread_throttle_mode", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = rdt_thread_throttle_mode_show, + }, + { + .name = "io_alloc", + .mode = 0644, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = resctrl_io_alloc_show, + .write = resctrl_io_alloc_write, + }, + { + .name = "io_alloc_cbm", + .mode = 0644, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = resctrl_io_alloc_cbm_show, + .write = resctrl_io_alloc_cbm_write, + }, + { + .name = "max_threshold_occupancy", + .mode = 0644, + .kf_ops = &rdtgroup_kf_single_ops, + .write = max_threshold_occ_write, + .seq_show = max_threshold_occ_show, + .fflags = RFTYPE_MON_INFO | RFTYPE_RES_CACHE, + }, + { + .name = "mbm_total_bytes_config", + .mode = 0644, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = mbm_total_bytes_config_show, + .write = mbm_total_bytes_config_write, + }, + { + .name = "mbm_local_bytes_config", + .mode = 0644, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = mbm_local_bytes_config_show, + .write = mbm_local_bytes_config_write, + }, + { + .name = "event_filter", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = event_filter_show, + .write = event_filter_write, + }, + { + .name = "mbm_L3_assignments", + .mode = 0644, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = mbm_L3_assignments_show, + .write = mbm_L3_assignments_write, + }, + { + .name = "mbm_assign_mode", + .mode = 0644, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = resctrl_mbm_assign_mode_show, + .write = resctrl_mbm_assign_mode_write, + .fflags = RFTYPE_MON_INFO | RFTYPE_RES_CACHE, + }, + { + .name = "cpus", + .mode = 0644, + .kf_ops = &rdtgroup_kf_single_ops, + .write = rdtgroup_cpus_write, + .seq_show = rdtgroup_cpus_show, + .fflags = RFTYPE_BASE, + }, + { + .name = "cpus_list", + .mode = 0644, + .kf_ops = &rdtgroup_kf_single_ops, + .write = rdtgroup_cpus_write, + .seq_show = rdtgroup_cpus_show, + .flags = RFTYPE_FLAGS_CPUS_LIST, + .fflags = RFTYPE_BASE, + }, + { + .name = "tasks", + .mode = 0644, + .kf_ops = &rdtgroup_kf_single_ops, + .write = rdtgroup_tasks_write, + .seq_show = rdtgroup_tasks_show, + .fflags = RFTYPE_BASE, + }, + { + .name = "mon_hw_id", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = rdtgroup_rmid_show, + .fflags = RFTYPE_MON_BASE | RFTYPE_DEBUG, + }, + { + .name = "schemata", + .mode = 0644, + .kf_ops = &rdtgroup_kf_single_ops, + .write = rdtgroup_schemata_write, + .seq_show = rdtgroup_schemata_show, + .fflags = RFTYPE_CTRL_BASE, + }, + { + .name = "mba_MBps_event", + .mode = 0644, + .kf_ops = &rdtgroup_kf_single_ops, + .write = rdtgroup_mba_mbps_event_write, + .seq_show = rdtgroup_mba_mbps_event_show, + }, + { + .name = "mode", + .mode = 0644, + .kf_ops = &rdtgroup_kf_single_ops, + .write = rdtgroup_mode_write, + .seq_show = rdtgroup_mode_show, + .fflags = RFTYPE_CTRL_BASE, + }, + { + .name = "size", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = rdtgroup_size_show, + .fflags = RFTYPE_CTRL_BASE, + }, + { + .name = "sparse_masks", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = rdt_has_sparse_bitmasks_show, + .fflags = RFTYPE_CTRL_INFO | RFTYPE_RES_CACHE, + }, + { + .name = "ctrl_hw_id", + .mode = 0444, + .kf_ops = &rdtgroup_kf_single_ops, + .seq_show = rdtgroup_closid_show, + .fflags = RFTYPE_CTRL_BASE | RFTYPE_DEBUG, + }, +}; + +static int rdtgroup_add_files(struct kernfs_node *kn, unsigned long fflags) +{ + struct rftype *rfts, *rft; + int ret, len; + + rfts = res_common_files; + len = ARRAY_SIZE(res_common_files); + + lockdep_assert_held(&rdtgroup_mutex); + + if (resctrl_debug) + fflags |= RFTYPE_DEBUG; + + for (rft = rfts; rft < rfts + len; rft++) { + if (rft->fflags && ((fflags & rft->fflags) == rft->fflags)) { + ret = rdtgroup_add_file(kn, rft); + if (ret) + goto error; + } + } + + return 0; +error: + pr_warn("Failed to add %s, err=%d\n", rft->name, ret); + while (--rft >= rfts) { + if ((fflags & rft->fflags) == rft->fflags) + kernfs_remove_by_name(kn, rft->name); + } + return ret; +} + +static struct rftype *rdtgroup_get_rftype_by_name(const char *name) +{ + struct rftype *rfts, *rft; + int len; + + rfts = res_common_files; + len = ARRAY_SIZE(res_common_files); + + for (rft = rfts; rft < rfts + len; rft++) { + if (!strcmp(rft->name, name)) + return rft; + } + + return NULL; +} + +static void thread_throttle_mode_init(void) +{ + enum membw_throttle_mode throttle_mode = THREAD_THROTTLE_UNDEFINED; + struct rdt_resource *r_mba, *r_smba; + + r_mba = resctrl_arch_get_resource(RDT_RESOURCE_MBA); + if (r_mba->alloc_capable && + r_mba->membw.throttle_mode != THREAD_THROTTLE_UNDEFINED) + throttle_mode = r_mba->membw.throttle_mode; + + r_smba = resctrl_arch_get_resource(RDT_RESOURCE_SMBA); + if (r_smba->alloc_capable && + r_smba->membw.throttle_mode != THREAD_THROTTLE_UNDEFINED) + throttle_mode = r_smba->membw.throttle_mode; + + if (throttle_mode == THREAD_THROTTLE_UNDEFINED) + return; + + resctrl_file_fflags_init("thread_throttle_mode", + RFTYPE_CTRL_INFO | RFTYPE_RES_MB); +} + +/* + * The resctrl file "io_alloc" is added using L3 resource. However, it results + * in this file being visible for *all* cache resources (eg. L2 cache), + * whether it supports "io_alloc" or not. + */ +static void io_alloc_init(void) +{ + struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); + + if (r->cache.io_alloc_capable) { + resctrl_file_fflags_init("io_alloc", RFTYPE_CTRL_INFO | + RFTYPE_RES_CACHE); + resctrl_file_fflags_init("io_alloc_cbm", + RFTYPE_CTRL_INFO | RFTYPE_RES_CACHE); + } +} + +void resctrl_file_fflags_init(const char *config, unsigned long fflags) +{ + struct rftype *rft; + + rft = rdtgroup_get_rftype_by_name(config); + if (rft) + rft->fflags = fflags; +} + +void resctrl_file_mode_init(const char *config, umode_t mode) +{ + struct rftype *rft; + + rft = rdtgroup_get_rftype_by_name(config); + if (rft) + rft->mode = mode; +} + +/** + * rdtgroup_kn_mode_restrict - Restrict user access to named resctrl file + * @r: The resource group with which the file is associated. + * @name: Name of the file + * + * The permissions of named resctrl file, directory, or link are modified + * to not allow read, write, or execute by any user. + * + * WARNING: This function is intended to communicate to the user that the + * resctrl file has been locked down - that it is not relevant to the + * particular state the system finds itself in. It should not be relied + * on to protect from user access because after the file's permissions + * are restricted the user can still change the permissions using chmod + * from the command line. + * + * Return: 0 on success, <0 on failure. + */ +int rdtgroup_kn_mode_restrict(struct rdtgroup *r, const char *name) +{ + struct iattr iattr = {.ia_valid = ATTR_MODE,}; + struct kernfs_node *kn; + int ret = 0; + + kn = kernfs_find_and_get_ns(r->kn, name, NULL); + if (!kn) + return -ENOENT; + + switch (kernfs_type(kn)) { + case KERNFS_DIR: + iattr.ia_mode = S_IFDIR; + break; + case KERNFS_FILE: + iattr.ia_mode = S_IFREG; + break; + case KERNFS_LINK: + iattr.ia_mode = S_IFLNK; + break; + } + + ret = kernfs_setattr(kn, &iattr); + kernfs_put(kn); + return ret; +} + +/** + * rdtgroup_kn_mode_restore - Restore user access to named resctrl file + * @r: The resource group with which the file is associated. + * @name: Name of the file + * @mask: Mask of permissions that should be restored + * + * Restore the permissions of the named file. If @name is a directory the + * permissions of its parent will be used. + * + * Return: 0 on success, <0 on failure. + */ +int rdtgroup_kn_mode_restore(struct rdtgroup *r, const char *name, + umode_t mask) +{ + struct iattr iattr = {.ia_valid = ATTR_MODE,}; + struct kernfs_node *kn, *parent; + struct rftype *rfts, *rft; + int ret, len; + + rfts = res_common_files; + len = ARRAY_SIZE(res_common_files); + + for (rft = rfts; rft < rfts + len; rft++) { + if (!strcmp(rft->name, name)) + iattr.ia_mode = rft->mode & mask; + } + + kn = kernfs_find_and_get_ns(r->kn, name, NULL); + if (!kn) + return -ENOENT; + + switch (kernfs_type(kn)) { + case KERNFS_DIR: + parent = kernfs_get_parent(kn); + if (parent) { + iattr.ia_mode |= parent->mode; + kernfs_put(parent); + } + iattr.ia_mode |= S_IFDIR; + break; + case KERNFS_FILE: + iattr.ia_mode |= S_IFREG; + break; + case KERNFS_LINK: + iattr.ia_mode |= S_IFLNK; + break; + } + + ret = kernfs_setattr(kn, &iattr); + kernfs_put(kn); + return ret; +} + +static int resctrl_mkdir_event_configs(struct rdt_resource *r, struct kernfs_node *l3_mon_kn) +{ + struct kernfs_node *kn_subdir, *kn_subdir2; + struct mon_evt *mevt; + int ret; + + kn_subdir = kernfs_create_dir(l3_mon_kn, "event_configs", l3_mon_kn->mode, NULL); + if (IS_ERR(kn_subdir)) + return PTR_ERR(kn_subdir); + + ret = rdtgroup_kn_set_ugid(kn_subdir); + if (ret) + return ret; + + for_each_mon_event(mevt) { + if (mevt->rid != r->rid || !mevt->enabled || !resctrl_is_mbm_event(mevt->evtid)) + continue; + + kn_subdir2 = kernfs_create_dir(kn_subdir, mevt->name, kn_subdir->mode, mevt); + if (IS_ERR(kn_subdir2)) + return PTR_ERR(kn_subdir2); + + ret = rdtgroup_kn_set_ugid(kn_subdir2); + if (ret) + return ret; + + ret = rdtgroup_add_files(kn_subdir2, RFTYPE_ASSIGN_CONFIG); + if (ret) + return ret; + } + + return 0; +} + +static int rdtgroup_mkdir_info_resdir(void *priv, char *name, + unsigned long fflags) +{ + struct kernfs_node *kn_subdir; + struct rdt_resource *r; + int ret; + + kn_subdir = kernfs_create_dir(kn_info, name, + kn_info->mode, priv); + if (IS_ERR(kn_subdir)) + return PTR_ERR(kn_subdir); + + ret = rdtgroup_kn_set_ugid(kn_subdir); + if (ret) + return ret; + + ret = rdtgroup_add_files(kn_subdir, fflags); + if (ret) + return ret; + + if ((fflags & RFTYPE_MON_INFO) == RFTYPE_MON_INFO) { + r = priv; + if (r->mon.mbm_cntr_assignable) { + ret = resctrl_mkdir_event_configs(r, kn_subdir); + if (ret) + return ret; + /* + * Hide BMEC related files if mbm_event mode + * is enabled. + */ + if (resctrl_arch_mbm_cntr_assign_enabled(r)) + resctrl_bmec_files_show(r, kn_subdir, false); + } + } + + kernfs_activate(kn_subdir); + + return ret; +} + +static unsigned long fflags_from_resource(struct rdt_resource *r) +{ + switch (r->rid) { + case RDT_RESOURCE_L3: + case RDT_RESOURCE_L2: + return RFTYPE_RES_CACHE; + case RDT_RESOURCE_MBA: + case RDT_RESOURCE_SMBA: + return RFTYPE_RES_MB; + case RDT_RESOURCE_PERF_PKG: + return RFTYPE_RES_PERF_PKG; + } + + return WARN_ON_ONCE(1); +} + +static int rdtgroup_create_info_dir(struct kernfs_node *parent_kn) +{ + struct resctrl_schema *s; + struct rdt_resource *r; + unsigned long fflags; + char name[32]; + int ret; + + /* create the directory */ + kn_info = kernfs_create_dir(parent_kn, "info", parent_kn->mode, NULL); + if (IS_ERR(kn_info)) + return PTR_ERR(kn_info); + + ret = rdtgroup_add_files(kn_info, RFTYPE_TOP_INFO); + if (ret) + goto out_destroy; + + /* loop over enabled controls, these are all alloc_capable */ + list_for_each_entry(s, &resctrl_schema_all, list) { + r = s->res; + fflags = fflags_from_resource(r) | RFTYPE_CTRL_INFO; + ret = rdtgroup_mkdir_info_resdir(s, s->name, fflags); + if (ret) + goto out_destroy; + } + + for_each_mon_capable_rdt_resource(r) { + fflags = fflags_from_resource(r) | RFTYPE_MON_INFO; + sprintf(name, "%s_MON", r->name); + ret = rdtgroup_mkdir_info_resdir(r, name, fflags); + if (ret) + goto out_destroy; + } + + ret = rdtgroup_kn_set_ugid(kn_info); + if (ret) + goto out_destroy; + + kernfs_activate(kn_info); + + return 0; + +out_destroy: + kernfs_remove(kn_info); + return ret; +} + +static int +mongroup_create_dir(struct kernfs_node *parent_kn, struct rdtgroup *prgrp, + char *name, struct kernfs_node **dest_kn) +{ + struct kernfs_node *kn; + int ret; + + /* create the directory */ + kn = kernfs_create_dir(parent_kn, name, parent_kn->mode, prgrp); + if (IS_ERR(kn)) + return PTR_ERR(kn); + + if (dest_kn) + *dest_kn = kn; + + ret = rdtgroup_kn_set_ugid(kn); + if (ret) + goto out_destroy; + + kernfs_activate(kn); + + return 0; + +out_destroy: + kernfs_remove(kn); + return ret; +} + +static inline bool is_mba_linear(void) +{ + return resctrl_arch_get_resource(RDT_RESOURCE_MBA)->membw.delay_linear; +} + +static int mba_sc_domain_allocate(struct rdt_resource *r, struct rdt_ctrl_domain *d) +{ + u32 num_closid = resctrl_arch_get_num_closid(r); + int cpu = cpumask_any(&d->hdr.cpu_mask); + int i; + + d->mbps_val = kcalloc_node(num_closid, sizeof(*d->mbps_val), + GFP_KERNEL, cpu_to_node(cpu)); + if (!d->mbps_val) + return -ENOMEM; + + for (i = 0; i < num_closid; i++) + d->mbps_val[i] = MBA_MAX_MBPS; + + return 0; +} + +static void mba_sc_domain_destroy(struct rdt_resource *r, + struct rdt_ctrl_domain *d) +{ + kfree(d->mbps_val); + d->mbps_val = NULL; +} + +/* + * The MBA software controller is supported only if MBM is supported and MBA is + * in linear scale, and the MBM monitor scope is the same as MBA control scope. + * + * The software controller cannot be supported when the MBM counters are + * assignable. There is no guarantee that MBM counters are assigned to the + * event backing the software controller in all monitoring domains of all + * monitoring groups. + */ +static bool supports_mba_mbps(void) +{ + struct rdt_resource *rmbm = resctrl_arch_get_resource(RDT_RESOURCE_L3); + struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_MBA); + + return (resctrl_is_mbm_enabled() && + r->alloc_capable && is_mba_linear() && + r->ctrl_scope == rmbm->mon_scope && + !rmbm->mon.mbm_cntr_assignable); +} + +/* + * Enable or disable the MBA software controller + * which helps user specify bandwidth in MBps. + */ +static int set_mba_sc(bool mba_sc) +{ + struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_MBA); + u32 num_closid = resctrl_arch_get_num_closid(r); + struct rdt_ctrl_domain *d; + unsigned long fflags; + int i; + + if (!supports_mba_mbps() || mba_sc == is_mba_sc(r)) + return -EINVAL; + + r->membw.mba_sc = mba_sc; + + rdtgroup_default.mba_mbps_event = mba_mbps_default_event; + + list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) { + for (i = 0; i < num_closid; i++) + d->mbps_val[i] = MBA_MAX_MBPS; + } + + fflags = mba_sc ? RFTYPE_CTRL_BASE | RFTYPE_MON_BASE : 0; + resctrl_file_fflags_init("mba_MBps_event", fflags); + + return 0; +} + +/* + * We don't allow rdtgroup directories to be created anywhere + * except the root directory. Thus when looking for the rdtgroup + * structure for a kernfs node we are either looking at a directory, + * in which case the rdtgroup structure is pointed at by the "priv" + * field, otherwise we have a file, and need only look to the parent + * to find the rdtgroup. + */ +static struct rdtgroup *kernfs_to_rdtgroup(struct kernfs_node *kn) +{ + if (kernfs_type(kn) == KERNFS_DIR) { + /* + * All the resource directories use "kn->priv" + * to point to the "struct rdtgroup" for the + * resource. "info" and its subdirectories don't + * have rdtgroup structures, so return NULL here. + */ + if (kn == kn_info || + rcu_access_pointer(kn->__parent) == kn_info) + return NULL; + else + return kn->priv; + } else { + return rdt_kn_parent_priv(kn); + } +} + +static void rdtgroup_kn_get(struct rdtgroup *rdtgrp, struct kernfs_node *kn) +{ + atomic_inc(&rdtgrp->waitcount); + kernfs_break_active_protection(kn); +} + +static void rdtgroup_kn_put(struct rdtgroup *rdtgrp, struct kernfs_node *kn) +{ + bool needs_free; + + if (!atomic_dec_and_mutex_lock(&rdtgrp->waitcount, &rdtgroup_mutex)) { + kernfs_unbreak_active_protection(kn); + return; + } + + needs_free = rdtgrp->flags & RDT_DELETED; + + mutex_unlock(&rdtgroup_mutex); + + kernfs_unbreak_active_protection(kn); + + if (needs_free) { + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP || + rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) + rdtgroup_pseudo_lock_remove(rdtgrp); + rdtgroup_remove(rdtgrp); + } +} + +struct rdtgroup *rdtgroup_kn_lock_live(struct kernfs_node *kn) +{ + struct rdtgroup *rdtgrp = kernfs_to_rdtgroup(kn); + + if (!rdtgrp) + return NULL; + + rdtgroup_kn_get(rdtgrp, kn); + + cpus_read_lock(); + mutex_lock(&rdtgroup_mutex); + rdt_last_cmd_clear(); + + /* Was this group deleted while we waited? */ + if (rdtgrp->flags & RDT_DELETED) { + /* + * It is safe to dereference kn to obtain the resource group's + * name because one extra reference to kn is obtained + * during resource group creation that will be released by + * rdtgroup_remove() called by rdtgroup_kn_put(). + */ + rdt_last_cmd_printf("Resource group %s deleted. No commands possible.\n", + rdt_kn_name(rdtgrp->kn)); + return NULL; + } + + return rdtgrp; +} + +void rdtgroup_kn_unlock(struct kernfs_node *kn) +{ + struct rdtgroup *rdtgrp = kernfs_to_rdtgroup(kn); + + if (!rdtgrp) + return; + + mutex_unlock(&rdtgroup_mutex); + cpus_read_unlock(); + + rdtgroup_kn_put(rdtgrp, kn); +} + +/* + * Accessing the kn after breaking active protection is safe since the open + * of resctrl file holds a kernfs base reference (different from active + * protection) on the kn ensuring that it remains accessible even if it was + * unlinked. Each kn in turn holds base reference to parent so the kn's + * genealogy remains in memory until all base references dropped. + */ +static bool is_active_resctrl_node(struct kernfs_node *kn) +{ + struct kernfs_node *p; + bool match = false; + + guard(rcu)(); + p = kn; + while (p) { + if (p == rdtgroup_default.kn) { + match = true; + break; + } + p = rcu_dereference(p->__parent); + } + + return match; +} + +bool info_kn_lock(struct kernfs_node *kn) +{ + kernfs_break_active_protection(kn); + cpus_read_lock(); + mutex_lock(&rdtgroup_mutex); + + /* + * Check both if resctrl is torn down (!rdtgroup_default.kn) and + * if the reader's kernfs_node originates from a dead mount. + */ + if (!rdtgroup_default.kn || !is_active_resctrl_node(kn)) { + mutex_unlock(&rdtgroup_mutex); + cpus_read_unlock(); + kernfs_unbreak_active_protection(kn); + return false; + } + + return true; +} + +void info_kn_unlock(struct kernfs_node *kn) +{ + mutex_unlock(&rdtgroup_mutex); + cpus_read_unlock(); + kernfs_unbreak_active_protection(kn); +} + +static int mkdir_mondata_all(struct kernfs_node *parent_kn, + struct rdtgroup *prgrp, + struct kernfs_node **mon_data_kn); + +static void rdt_disable_ctx(void) +{ + resctrl_arch_set_cdp_enabled(RDT_RESOURCE_L3, false); + resctrl_arch_set_cdp_enabled(RDT_RESOURCE_L2, false); + set_mba_sc(false); + + resctrl_debug = false; +} + +static int rdt_enable_ctx(struct rdt_fs_context *ctx) +{ + int ret = 0; + + if (ctx->enable_cdpl2) { + ret = resctrl_arch_set_cdp_enabled(RDT_RESOURCE_L2, true); + if (ret) + goto out_done; + } + + if (ctx->enable_cdpl3) { + ret = resctrl_arch_set_cdp_enabled(RDT_RESOURCE_L3, true); + if (ret) + goto out_cdpl2; + } + + if (ctx->enable_mba_mbps) { + ret = set_mba_sc(true); + if (ret) + goto out_cdpl3; + } + + if (ctx->enable_debug) + resctrl_debug = true; + + return 0; + +out_cdpl3: + resctrl_arch_set_cdp_enabled(RDT_RESOURCE_L3, false); +out_cdpl2: + resctrl_arch_set_cdp_enabled(RDT_RESOURCE_L2, false); +out_done: + return ret; +} + +static int schemata_list_add(struct rdt_resource *r, enum resctrl_conf_type type) +{ + struct resctrl_schema *s; + const char *suffix = ""; + int ret, cl; + + s = kzalloc_obj(*s); + if (!s) + return -ENOMEM; + + s->res = r; + s->num_closid = resctrl_arch_get_num_closid(r); + if (resctrl_arch_get_cdp_enabled(r->rid)) + s->num_closid /= 2; + + s->conf_type = type; + switch (type) { + case CDP_CODE: + suffix = "CODE"; + break; + case CDP_DATA: + suffix = "DATA"; + break; + case CDP_NONE: + suffix = ""; + break; + } + + ret = snprintf(s->name, sizeof(s->name), "%s%s", r->name, suffix); + if (ret >= sizeof(s->name)) { + kfree(s); + return -EINVAL; + } + + cl = strlen(s->name); + + /* + * If CDP is supported by this resource, but not enabled, + * include the suffix. This ensures the tabular format of the + * schemata file does not change between mounts of the filesystem. + */ + if (r->cdp_capable && !resctrl_arch_get_cdp_enabled(r->rid)) + cl += 4; + + if (cl > max_name_width) + max_name_width = cl; + + switch (r->schema_fmt) { + case RESCTRL_SCHEMA_BITMAP: + s->fmt_str = "%d=%x"; + break; + case RESCTRL_SCHEMA_RANGE: + s->fmt_str = "%d=%u"; + break; + } + + if (WARN_ON_ONCE(!s->fmt_str)) { + kfree(s); + return -EINVAL; + } + + INIT_LIST_HEAD(&s->list); + list_add(&s->list, &resctrl_schema_all); + + return 0; +} + +static int schemata_list_create(void) +{ + struct rdt_resource *r; + int ret = 0; + + for_each_alloc_capable_rdt_resource(r) { + if (resctrl_arch_get_cdp_enabled(r->rid)) { + ret = schemata_list_add(r, CDP_CODE); + if (ret) + break; + + ret = schemata_list_add(r, CDP_DATA); + } else { + ret = schemata_list_add(r, CDP_NONE); + } + + if (ret) + break; + } + + return ret; +} + +static void schemata_list_destroy(void) +{ + struct resctrl_schema *s, *tmp; + + list_for_each_entry_safe(s, tmp, &resctrl_schema_all, list) { + list_del(&s->list); + kfree(s); + } +} + +/* + * Move tasks from one to the other group. If @from is NULL, then all tasks + * in the systems are moved unconditionally (used for teardown). + * + * If @mask is not NULL the cpus on which moved tasks are running are set + * in that mask so the update smp function call is restricted to affected + * cpus. + */ +static void rdt_move_group_tasks(struct rdtgroup *from, struct rdtgroup *to, + struct cpumask *mask) +{ + struct task_struct *p, *t; + + read_lock(&tasklist_lock); + for_each_process_thread(p, t) { + if (!from || is_closid_match(t, from) || + is_rmid_match(t, from)) { + resctrl_arch_set_closid_rmid(t, to->closid, + to->mon.rmid); + + /* + * Order the closid/rmid stores above before the loads + * in task_curr(). This pairs with the full barrier + * between the rq->curr update and + * resctrl_arch_sched_in() during context switch. + */ + smp_mb(); + + /* + * If the task is on a CPU, set the CPU in the mask. + * The detection is inaccurate as tasks might move or + * schedule before the smp function call takes place. + * In such a case the function call is pointless, but + * there is no other side effect. + */ + if (IS_ENABLED(CONFIG_SMP) && mask && task_curr(t)) + cpumask_set_cpu(task_cpu(t), mask); + } + } + read_unlock(&tasklist_lock); +} + +static void free_all_child_rdtgrp(struct rdtgroup *rdtgrp) +{ + struct rdtgroup *sentry, *stmp; + struct list_head *head; + + head = &rdtgrp->mon.crdtgrp_list; + list_for_each_entry_safe(sentry, stmp, head, mon.crdtgrp_list) { + rdtgroup_unassign_cntrs(sentry); + free_rmid(sentry->closid, sentry->mon.rmid); + list_del(&sentry->mon.crdtgrp_list); + + if (atomic_read(&sentry->waitcount) != 0) + sentry->flags = RDT_DELETED; + else + rdtgroup_remove(sentry); + } +} + +/* + * Forcibly remove all of subdirectories under root. + */ +static void rmdir_all_sub(void) +{ + struct rdtgroup *rdtgrp, *tmp; + + /* Move all tasks to the default resource group */ + rdt_move_group_tasks(NULL, &rdtgroup_default, NULL); + + list_for_each_entry_safe(rdtgrp, tmp, &rdt_all_groups, rdtgroup_list) { + /* Free any child rmids */ + free_all_child_rdtgrp(rdtgrp); + + /* Remove each rdtgroup other than root */ + if (rdtgrp == &rdtgroup_default) + continue; + + /* + * Give any CPUs back to the default group. We cannot copy + * cpu_online_mask because a CPU might have executed the + * offline callback already, but is still marked online. + */ + cpumask_or(&rdtgroup_default.cpu_mask, + &rdtgroup_default.cpu_mask, &rdtgrp->cpu_mask); + + rdtgroup_unassign_cntrs(rdtgrp); + + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP || + rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) { + rdtgroup_pseudo_lock_remove(rdtgrp); + } else { + /* Pseudo-locked group's RMID is freed during setup. */ + free_rmid(rdtgrp->closid, rdtgrp->mon.rmid); + } + + kernfs_remove(rdtgrp->kn); + list_del(&rdtgrp->rdtgroup_list); + + if (atomic_read(&rdtgrp->waitcount) != 0) + rdtgrp->flags = RDT_DELETED; + else + rdtgroup_remove(rdtgrp); + } + /* Notify online CPUs to update per cpu storage and PQR_ASSOC MSR */ + update_closid_rmid(cpu_online_mask, &rdtgroup_default); + + kernfs_remove(kn_info); + kernfs_remove(kn_mongrp); + kernfs_remove(kn_mondata); +} + +/** + * mon_get_kn_priv() - Get the mon_data priv data for this event. + * + * The same values are used across the mon_data directories of all control and + * monitor groups for the same event in the same domain. Keep a list of + * allocated structures and re-use an existing one with the same values for + * @rid, @domid, etc. + * + * @rid: The resource id for the event file being created. + * @domid: The domain id for the event file being created. + * @mevt: The type of event file being created. + * @do_sum: Whether SNC summing monitors are being created. Only set + * when @rid == RDT_RESOURCE_L3. + * + * Return: Pointer to mon_data private data of the event, NULL on failure. + */ +static struct mon_data *mon_get_kn_priv(enum resctrl_res_level rid, int domid, + struct mon_evt *mevt, + bool do_sum) +{ + struct mon_data *priv; + + lockdep_assert_held(&rdtgroup_mutex); + + list_for_each_entry(priv, &mon_data_kn_priv_list, list) { + if (priv->rid == rid && priv->domid == domid && + priv->sum == do_sum && priv->evt == mevt) + return priv; + } + + priv = kzalloc_obj(*priv); + if (!priv) + return NULL; + + priv->rid = rid; + priv->domid = domid; + priv->sum = do_sum; + priv->evt = mevt; + list_add_tail(&priv->list, &mon_data_kn_priv_list); + + return priv; +} + +/** + * mon_put_kn_priv() - Free all allocated mon_data structures. + * + * Called when resctrl file system is unmounted. + */ +static void mon_put_kn_priv(void) +{ + struct mon_data *priv, *tmp; + + lockdep_assert_held(&rdtgroup_mutex); + + list_for_each_entry_safe(priv, tmp, &mon_data_kn_priv_list, list) { + list_del(&priv->list); + kfree(priv); + } +} + +static void resctrl_fs_teardown(void) +{ + lockdep_assert_held(&rdtgroup_mutex); + + /* Cleared by rdtgroup_destroy_root() */ + if (!rdtgroup_default.kn) + return; + + rmdir_all_sub(); + rdtgroup_unassign_cntrs(&rdtgroup_default); + mon_put_kn_priv(); + rdt_pseudo_lock_release(); + rdtgroup_default.mode = RDT_MODE_SHAREABLE; + rdtgroup_default.flags = RDT_DELETED; + closid_exit(); + schemata_list_destroy(); + rdtgroup_destroy_root(); +} + +static void resctrl_unmount(void) +{ + struct rdt_resource *r; + + cpus_read_lock(); + mutex_lock(&rdtgroup_mutex); + + rdt_disable_ctx(); + + /* Put everything back to default values. */ + for_each_alloc_capable_rdt_resource(r) + resctrl_arch_reset_all_ctrls(r); + + resctrl_fs_teardown(); + if (resctrl_arch_alloc_capable()) + resctrl_arch_disable_alloc(); + if (resctrl_arch_mon_capable()) + resctrl_arch_disable_mon(); + resctrl_mounted = false; + mutex_unlock(&rdtgroup_mutex); + cpus_read_unlock(); +} + +static int rdt_get_tree(struct fs_context *fc) +{ + struct rdt_fs_context *ctx = rdt_fc2context(fc); + unsigned long flags = RFTYPE_CTRL_BASE; + struct kernfs_node *rdt_root_kn; + struct rdt_l3_mon_domain *dom; + struct rdt_resource *r; + int ret; + + DO_ONCE_SLEEPABLE(resctrl_arch_pre_mount); + + cpus_read_lock(); + mutex_lock(&rdtgroup_mutex); + /* + * resctrl file system can only be mounted once. + */ + if (resctrl_mounted) { + ret = -EBUSY; + goto out; + } + + /* Avoid races from pending operations from a previous mount */ + if (atomic_read(&rdtgroup_default.waitcount) != 0) { + ret = -EBUSY; + goto out; + } + + ret = setup_rmid_lru_list(); + if (ret) + goto out; + + ret = rdtgroup_setup_root(ctx); + if (ret) + goto out; + + ret = rdt_enable_ctx(ctx); + if (ret) + goto out_root; + + ret = schemata_list_create(); + if (ret) + goto out_schemata_free; + + ret = closid_init(); + if (ret) + goto out_schemata_free; + + if (resctrl_arch_mon_capable()) + flags |= RFTYPE_MON; + + ret = rdtgroup_add_files(rdtgroup_default.kn, flags); + if (ret) + goto out_closid_exit; + + kernfs_activate(rdtgroup_default.kn); + + ret = rdtgroup_create_info_dir(rdtgroup_default.kn); + if (ret < 0) + goto out_closid_exit; + + if (resctrl_arch_mon_capable()) { + ret = mongroup_create_dir(rdtgroup_default.kn, + &rdtgroup_default, "mon_groups", + &kn_mongrp); + if (ret < 0) + goto out_info; + + rdtgroup_assign_cntrs(&rdtgroup_default); + + ret = mkdir_mondata_all(rdtgroup_default.kn, + &rdtgroup_default, &kn_mondata); + if (ret < 0) + goto out_mongrp; + rdtgroup_default.mon.mon_data_kn = kn_mondata; + } + + ret = rdt_pseudo_lock_init(); + if (ret) + goto out_mondata; + + if (resctrl_arch_alloc_capable()) + resctrl_arch_enable_alloc(); + if (resctrl_arch_mon_capable()) + resctrl_arch_enable_mon(); + + if (resctrl_arch_alloc_capable() || resctrl_arch_mon_capable()) + resctrl_mounted = true; + + if (resctrl_is_mbm_enabled()) { + r = resctrl_arch_get_resource(RDT_RESOURCE_L3); + list_for_each_entry_rcu(dom, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) + mbm_setup_overflow_handler(dom, MBM_OVERFLOW_INTERVAL, + RESCTRL_PICK_ANY_CPU); + } + + /* + * Ensure root remains accessible after mutex is unlocked so that + * kernfs_kill_sb() can run safely if called by kernfs_get_tree()'s + * failure path after creating a superblock but before taking reference + * on root kn (for example, if unable to get inode for root kn). + */ + kernfs_get(rdtgroup_default.kn); + + /* + * Make backup of the current root kn being created to be used in + * kernfs_put(). The additional reference taken above will prevent the + * kn from being freed before kernfs_kill_sb() can run but + * rdtgroup_default.kn may be set to NULL via rdtgroup_destroy_root() + * and its backing root (rdt_root) could be overwritten before + * kernfs_put() can run. + */ + rdt_root_kn = rdtgroup_default.kn; + + rdt_last_cmd_clear(); + mutex_unlock(&rdtgroup_mutex); + cpus_read_unlock(); + + ret = kernfs_get_tree(fc); + /* + * resctrl can only be mounted once, new superblock only expected + * to be created once. + */ + if (!ctx->kfc.new_sb_created) + resctrl_unmount(); + kernfs_put(rdt_root_kn); + return ret; + +out_mondata: + if (resctrl_arch_mon_capable()) + kernfs_remove(kn_mondata); +out_mongrp: + if (resctrl_arch_mon_capable()) { + mon_put_kn_priv(); + rdtgroup_unassign_cntrs(&rdtgroup_default); + kernfs_remove(kn_mongrp); + } +out_info: + kernfs_remove(kn_info); +out_closid_exit: + closid_exit(); +out_schemata_free: + schemata_list_destroy(); + rdt_disable_ctx(); +out_root: + rdtgroup_destroy_root(); +out: + mutex_unlock(&rdtgroup_mutex); + cpus_read_unlock(); + return ret; +} + +enum rdt_param { + Opt_cdp, + Opt_cdpl2, + Opt_mba_mbps, + Opt_debug, + nr__rdt_params +}; + +static const struct fs_parameter_spec rdt_fs_parameters[] = { + fsparam_flag("cdp", Opt_cdp), + fsparam_flag("cdpl2", Opt_cdpl2), + fsparam_flag("mba_MBps", Opt_mba_mbps), + fsparam_flag("debug", Opt_debug), + {} +}; + +static int rdt_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct rdt_fs_context *ctx = rdt_fc2context(fc); + struct fs_parse_result result; + const char *msg; + int opt; + + opt = fs_parse(fc, rdt_fs_parameters, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_cdp: + ctx->enable_cdpl3 = true; + return 0; + case Opt_cdpl2: + ctx->enable_cdpl2 = true; + return 0; + case Opt_mba_mbps: + msg = "mba_MBps requires MBM (mbm_event mode not supported) and linear scale MBA at L3 scope"; + if (!supports_mba_mbps()) + return invalfc(fc, msg); + ctx->enable_mba_mbps = true; + return 0; + case Opt_debug: + ctx->enable_debug = true; + return 0; + } + + return -EINVAL; +} + +static void rdt_fs_context_free(struct fs_context *fc) +{ + struct rdt_fs_context *ctx = rdt_fc2context(fc); + + kernfs_free_fs_context(fc); + kfree(ctx); +} + +static const struct fs_context_operations rdt_fs_context_ops = { + .free = rdt_fs_context_free, + .parse_param = rdt_parse_param, + .get_tree = rdt_get_tree, +}; + +static int rdt_init_fs_context(struct fs_context *fc) +{ + struct rdt_fs_context *ctx; + + ctx = kzalloc_obj(*ctx); + if (!ctx) + return -ENOMEM; + + ctx->kfc.magic = RDTGROUP_SUPER_MAGIC; + fc->fs_private = &ctx->kfc; + fc->ops = &rdt_fs_context_ops; + put_user_ns(fc->user_ns); + fc->user_ns = get_user_ns(&init_user_ns); + fc->global = true; + return 0; +} + +static void rdt_kill_sb(struct super_block *sb) +{ + resctrl_unmount(); + kernfs_kill_sb(sb); +} + +static struct file_system_type rdt_fs_type = { + .name = "resctrl", + .init_fs_context = rdt_init_fs_context, + .parameters = rdt_fs_parameters, + .kill_sb = rdt_kill_sb, +}; + +static int mon_addfile(struct kernfs_node *parent_kn, const char *name, + void *priv) +{ + struct kernfs_node *kn; + int ret = 0; + + kn = __kernfs_create_file(parent_kn, name, 0444, + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, 0, + &kf_mondata_ops, priv, NULL, NULL); + if (IS_ERR(kn)) + return PTR_ERR(kn); + + ret = rdtgroup_kn_set_ugid(kn); + if (ret) { + kernfs_remove(kn); + return ret; + } + + return ret; +} + +static void mon_rmdir_one_subdir(struct kernfs_node *pkn, char *name, char *subname) +{ + struct kernfs_node *kn; + + kn = kernfs_find_and_get(pkn, name); + if (!kn) + return; + kernfs_put(kn); + + if (kn->dir.subdirs <= 1) + kernfs_remove(kn); + else + kernfs_remove_by_name(kn, subname); +} + +/* + * Remove files and directories for one SNC node. If it is the last node + * sharing an L3 cache, then remove the upper level directory containing + * the "sum" files too. + */ +static void rmdir_mondata_subdir_allrdtgrp_snc(struct rdt_resource *r, + struct rdt_domain_hdr *hdr) +{ + struct rdtgroup *prgrp, *crgrp; + struct rdt_l3_mon_domain *d; + char subname[32]; + char name[32]; + + if (!domain_header_is_valid(hdr, RESCTRL_MON_DOMAIN, RDT_RESOURCE_L3)) + return; + + d = container_of(hdr, struct rdt_l3_mon_domain, hdr); + sprintf(name, "mon_%s_%02d", r->name, d->ci_id); + sprintf(subname, "mon_sub_%s_%02d", r->name, hdr->id); + + list_for_each_entry(prgrp, &rdt_all_groups, rdtgroup_list) { + mon_rmdir_one_subdir(prgrp->mon.mon_data_kn, name, subname); + + list_for_each_entry(crgrp, &prgrp->mon.crdtgrp_list, mon.crdtgrp_list) + mon_rmdir_one_subdir(crgrp->mon.mon_data_kn, name, subname); + } +} + +/* + * Remove all subdirectories of mon_data of ctrl_mon groups + * and monitor groups for the given domain. + */ +static void rmdir_mondata_subdir_allrdtgrp(struct rdt_resource *r, + struct rdt_domain_hdr *hdr) +{ + struct rdtgroup *prgrp, *crgrp; + char name[32]; + + if (r->rid == RDT_RESOURCE_L3 && r->mon_scope == RESCTRL_L3_NODE) { + rmdir_mondata_subdir_allrdtgrp_snc(r, hdr); + return; + } + + sprintf(name, "mon_%s_%02d", r->name, hdr->id); + list_for_each_entry(prgrp, &rdt_all_groups, rdtgroup_list) { + kernfs_remove_by_name(prgrp->mon.mon_data_kn, name); + + list_for_each_entry(crgrp, &prgrp->mon.crdtgrp_list, mon.crdtgrp_list) + kernfs_remove_by_name(crgrp->mon.mon_data_kn, name); + } +} + +/* + * Create a directory for a domain and populate it with monitor files. Create + * summing monitors when @hdr is NULL. No need to initialize summing monitors. + */ +static struct kernfs_node *_mkdir_mondata_subdir(struct kernfs_node *parent_kn, char *name, + struct rdt_domain_hdr *hdr, + struct rdt_resource *r, + struct rdtgroup *prgrp, int domid) +{ + struct rmid_read rr = {0}; + struct kernfs_node *kn; + struct mon_data *priv; + struct mon_evt *mevt; + int ret; + + kn = kernfs_create_dir(parent_kn, name, parent_kn->mode, prgrp); + if (IS_ERR(kn)) + return kn; + + ret = rdtgroup_kn_set_ugid(kn); + if (ret) + goto out_destroy; + + for_each_mon_event(mevt) { + if (mevt->rid != r->rid || !mevt->enabled) + continue; + priv = mon_get_kn_priv(r->rid, domid, mevt, !hdr); + if (WARN_ON_ONCE(!priv)) { + ret = -EINVAL; + goto out_destroy; + } + + ret = mon_addfile(kn, mevt->name, priv); + if (ret) + goto out_destroy; + + if (hdr && resctrl_is_mbm_event(mevt->evtid)) + mon_event_read(&rr, r, hdr, prgrp, &hdr->cpu_mask, mevt, true); + } + + return kn; +out_destroy: + kernfs_remove(kn); + return ERR_PTR(ret); +} + +static int mkdir_mondata_subdir_snc(struct kernfs_node *parent_kn, + struct rdt_domain_hdr *hdr, + struct rdt_resource *r, struct rdtgroup *prgrp) +{ + struct kernfs_node *ckn, *kn; + struct rdt_l3_mon_domain *d; + char name[32]; + + if (!domain_header_is_valid(hdr, RESCTRL_MON_DOMAIN, RDT_RESOURCE_L3)) + return -EINVAL; + + d = container_of(hdr, struct rdt_l3_mon_domain, hdr); + sprintf(name, "mon_%s_%02d", r->name, d->ci_id); + kn = kernfs_find_and_get(parent_kn, name); + if (kn) { + /* + * rdtgroup_mutex will prevent this directory from being + * removed. No need to keep this hold. + */ + kernfs_put(kn); + } else { + kn = _mkdir_mondata_subdir(parent_kn, name, NULL, r, prgrp, d->ci_id); + if (IS_ERR(kn)) + return PTR_ERR(kn); + } + + sprintf(name, "mon_sub_%s_%02d", r->name, hdr->id); + ckn = _mkdir_mondata_subdir(kn, name, hdr, r, prgrp, hdr->id); + if (IS_ERR(ckn)) { + kernfs_remove(kn); + return PTR_ERR(ckn); + } + + kernfs_activate(kn); + return 0; +} + +static int mkdir_mondata_subdir(struct kernfs_node *parent_kn, + struct rdt_domain_hdr *hdr, + struct rdt_resource *r, struct rdtgroup *prgrp) +{ + struct kernfs_node *kn; + char name[32]; + + lockdep_assert_held(&rdtgroup_mutex); + + if (r->rid == RDT_RESOURCE_L3 && r->mon_scope == RESCTRL_L3_NODE) + return mkdir_mondata_subdir_snc(parent_kn, hdr, r, prgrp); + + sprintf(name, "mon_%s_%02d", r->name, hdr->id); + kn = _mkdir_mondata_subdir(parent_kn, name, hdr, r, prgrp, hdr->id); + if (IS_ERR(kn)) + return PTR_ERR(kn); + + kernfs_activate(kn); + return 0; +} + +/* + * Add all subdirectories of mon_data for "ctrl_mon" groups + * and "monitor" groups with given domain id. + */ +static void mkdir_mondata_subdir_allrdtgrp(struct rdt_resource *r, + struct rdt_domain_hdr *hdr) +{ + struct kernfs_node *parent_kn; + struct rdtgroup *prgrp, *crgrp; + struct list_head *head; + + list_for_each_entry(prgrp, &rdt_all_groups, rdtgroup_list) { + parent_kn = prgrp->mon.mon_data_kn; + mkdir_mondata_subdir(parent_kn, hdr, r, prgrp); + + head = &prgrp->mon.crdtgrp_list; + list_for_each_entry(crgrp, head, mon.crdtgrp_list) { + parent_kn = crgrp->mon.mon_data_kn; + mkdir_mondata_subdir(parent_kn, hdr, r, crgrp); + } + } +} + +static int mkdir_mondata_subdir_alldom(struct kernfs_node *parent_kn, + struct rdt_resource *r, + struct rdtgroup *prgrp) +{ + struct rdt_domain_hdr *hdr; + int ret; + + /* Walking r->domains, ensure it can't race with cpuhp */ + lockdep_assert_cpus_held(); + + list_for_each_entry_rcu(hdr, &r->mon_domains, list, lockdep_is_cpus_held()) { + ret = mkdir_mondata_subdir(parent_kn, hdr, r, prgrp); + if (ret) + return ret; + } + + return 0; +} + +/* + * This creates a directory mon_data which contains the monitored data. + * + * mon_data has one directory for each domain which are named + * in the format mon__. For ex: A mon_data + * with L3 domain looks as below: + * ./mon_data: + * mon_L3_00 + * mon_L3_01 + * mon_L3_02 + * ... + * + * Each domain directory has one file per event: + * ./mon_L3_00/: + * llc_occupancy + * + */ +static int mkdir_mondata_all(struct kernfs_node *parent_kn, + struct rdtgroup *prgrp, + struct kernfs_node **dest_kn) +{ + struct rdt_resource *r; + struct kernfs_node *kn; + int ret; + + /* + * Create the mon_data directory first. + */ + ret = mongroup_create_dir(parent_kn, prgrp, "mon_data", &kn); + if (ret) + return ret; + + if (dest_kn) + *dest_kn = kn; + + /* + * Create the subdirectories for each domain. Note that all events + * in a domain like L3 are grouped into a resource whose domain is L3 + */ + for_each_mon_capable_rdt_resource(r) { + ret = mkdir_mondata_subdir_alldom(kn, r, prgrp); + if (ret) + goto out_destroy; + } + + return 0; + +out_destroy: + kernfs_remove(kn); + return ret; +} + +/** + * cbm_ensure_valid - Enforce validity on provided CBM + * @_val: Candidate CBM + * @r: RDT resource to which the CBM belongs + * + * The provided CBM represents all cache portions available for use. This + * may be represented by a bitmap that does not consist of contiguous ones + * and thus be an invalid CBM. + * Here the provided CBM is forced to be a valid CBM by only considering + * the first set of contiguous bits as valid and clearing all bits. + * The intention here is to provide a valid default CBM with which a new + * resource group is initialized. The user can follow this with a + * modification to the CBM if the default does not satisfy the + * requirements. + * + * Return: A CBM that is valid for resource @r. + */ +static u32 cbm_ensure_valid(u32 _val, struct rdt_resource *r) +{ + unsigned int cbm_len = r->cache.cbm_len; + unsigned long first_bit, zero_bit; + unsigned long val; + + if (!_val || r->cache.arch_has_sparse_bitmasks) + return _val; + + val = _val; + first_bit = find_first_bit(&val, cbm_len); + zero_bit = find_next_zero_bit(&val, cbm_len, first_bit); + + /* Clear any remaining bits to ensure contiguous region */ + bitmap_clear(&val, zero_bit, cbm_len - zero_bit); + return (u32)val; +} + +/* + * Initialize cache resources per RDT domain + * + * Set the RDT domain up to start off with all usable allocations. That is, + * all shareable and unused bits. All-zero CBM is invalid. + */ +static int __init_one_rdt_domain(struct rdt_ctrl_domain *d, struct resctrl_schema *s, + u32 closid) +{ + enum resctrl_conf_type peer_type = resctrl_peer_type(s->conf_type); + enum resctrl_conf_type t = s->conf_type; + struct resctrl_staged_config *cfg; + struct rdt_resource *r = s->res; + u32 used_b = 0, unused_b = 0; + unsigned long tmp_cbm; + enum rdtgrp_mode mode; + u32 peer_ctl, ctrl_val; + int i; + + cfg = &d->staged_config[t]; + cfg->have_new_ctrl = false; + cfg->new_ctrl = r->cache.shareable_bits; + used_b = r->cache.shareable_bits; + for (i = 0; i < closids_supported(); i++) { + if (closid_allocated(i) && i != closid) { + mode = rdtgroup_mode_by_closid(i); + if (mode == RDT_MODE_PSEUDO_LOCKSETUP) + /* + * ctrl values for locksetup aren't relevant + * until the schemata is written, and the mode + * becomes RDT_MODE_PSEUDO_LOCKED. + */ + continue; + /* + * If CDP is active include peer domain's + * usage to ensure there is no overlap + * with an exclusive group. + */ + if (resctrl_arch_get_cdp_enabled(r->rid)) + peer_ctl = resctrl_arch_get_config(r, d, i, + peer_type); + else + peer_ctl = 0; + ctrl_val = resctrl_arch_get_config(r, d, i, + s->conf_type); + used_b |= ctrl_val | peer_ctl; + if (mode == RDT_MODE_SHAREABLE) + cfg->new_ctrl |= ctrl_val | peer_ctl; + } + } + if (d->plr && d->plr->cbm > 0) + used_b |= d->plr->cbm; + unused_b = used_b ^ (BIT_MASK(r->cache.cbm_len) - 1); + unused_b &= BIT_MASK(r->cache.cbm_len) - 1; + cfg->new_ctrl |= unused_b; + /* + * Force the initial CBM to be valid, user can + * modify the CBM based on system availability. + */ + cfg->new_ctrl = cbm_ensure_valid(cfg->new_ctrl, r); + /* + * Assign the u32 CBM to an unsigned long to ensure that + * bitmap_weight() does not access out-of-bound memory. + */ + tmp_cbm = cfg->new_ctrl; + if (bitmap_weight(&tmp_cbm, r->cache.cbm_len) < r->cache.min_cbm_bits) { + rdt_last_cmd_printf("No space on %s:%d\n", s->name, d->hdr.id); + return -ENOSPC; + } + cfg->have_new_ctrl = true; + + return 0; +} + +/* + * Initialize cache resources with default values. + * + * A new RDT group is being created on an allocation capable (CAT) + * supporting system. Set this group up to start off with all usable + * allocations. + * + * If there are no more shareable bits available on any domain then + * the entire allocation will fail. + */ +int rdtgroup_init_cat(struct resctrl_schema *s, u32 closid) +{ + struct rdt_ctrl_domain *d; + int ret; + + list_for_each_entry_rcu(d, &s->res->ctrl_domains, hdr.list, lockdep_is_cpus_held()) { + ret = __init_one_rdt_domain(d, s, closid); + if (ret < 0) + return ret; + } + + return 0; +} + +/* Initialize MBA resource with default values. */ +static void rdtgroup_init_mba(struct rdt_resource *r, u32 closid) +{ + struct resctrl_staged_config *cfg; + struct rdt_ctrl_domain *d; + + list_for_each_entry_rcu(d, &r->ctrl_domains, hdr.list, lockdep_is_cpus_held()) { + if (is_mba_sc(r)) { + d->mbps_val[closid] = MBA_MAX_MBPS; + continue; + } + + cfg = &d->staged_config[CDP_NONE]; + cfg->new_ctrl = resctrl_get_default_ctrl(r); + cfg->have_new_ctrl = true; + } +} + +/* Initialize the RDT group's allocations. */ +static int rdtgroup_init_alloc(struct rdtgroup *rdtgrp) +{ + struct resctrl_schema *s; + struct rdt_resource *r; + int ret = 0; + + rdt_staged_configs_clear(); + + list_for_each_entry(s, &resctrl_schema_all, list) { + r = s->res; + if (r->rid == RDT_RESOURCE_MBA || + r->rid == RDT_RESOURCE_SMBA) { + rdtgroup_init_mba(r, rdtgrp->closid); + if (is_mba_sc(r)) + continue; + } else { + ret = rdtgroup_init_cat(s, rdtgrp->closid); + if (ret < 0) + goto out; + } + + ret = resctrl_arch_update_domains(r, rdtgrp->closid); + if (ret < 0) { + rdt_last_cmd_puts("Failed to initialize allocations\n"); + goto out; + } + } + + rdtgrp->mode = RDT_MODE_SHAREABLE; + +out: + rdt_staged_configs_clear(); + return ret; +} + +static int mkdir_rdt_prepare_rmid_alloc(struct rdtgroup *rdtgrp) +{ + int ret; + + if (!resctrl_arch_mon_capable()) + return 0; + + ret = alloc_rmid(rdtgrp->closid); + if (ret < 0) { + rdt_last_cmd_puts("Out of RMIDs\n"); + return ret; + } + rdtgrp->mon.rmid = ret; + + rdtgroup_assign_cntrs(rdtgrp); + + ret = mkdir_mondata_all(rdtgrp->kn, rdtgrp, &rdtgrp->mon.mon_data_kn); + if (ret) { + rdt_last_cmd_puts("kernfs subdir error\n"); + rdtgroup_unassign_cntrs(rdtgrp); + free_rmid(rdtgrp->closid, rdtgrp->mon.rmid); + return ret; + } + + return 0; +} + +static void mkdir_rdt_prepare_rmid_free(struct rdtgroup *rgrp) +{ + if (resctrl_arch_mon_capable()) { + rdtgroup_unassign_cntrs(rgrp); + free_rmid(rgrp->closid, rgrp->mon.rmid); + } +} + +/* + * We allow creating mon groups only with in a directory called "mon_groups" + * which is present in every ctrl_mon group. Check if this is a valid + * "mon_groups" directory. + * + * 1. The directory should be named "mon_groups". + * 2. The mon group itself should "not" be named "mon_groups". + * This makes sure "mon_groups" directory always has a ctrl_mon group + * as parent. + */ +static bool is_mon_groups(struct kernfs_node *kn, const char *name) +{ + return (!strcmp(rdt_kn_name(kn), "mon_groups") && + strcmp(name, "mon_groups")); +} + +static int mkdir_rdt_prepare(struct kernfs_node *parent_kn, + const char *name, umode_t mode, + enum rdt_group_type rtype, struct rdtgroup **r) +{ + struct rdtgroup *prdtgrp, *rdtgrp; + unsigned long files = 0; + struct kernfs_node *kn; + int ret; + + prdtgrp = rdtgroup_kn_lock_live(parent_kn); + if (!prdtgrp) { + ret = -ENODEV; + goto out_unlock; + } + + /* + * Check that the parent directory for a monitor group is a "mon_groups" + * directory. + */ + if (rtype == RDTMON_GROUP && !is_mon_groups(parent_kn, name)) { + ret = -EPERM; + goto out_unlock; + } + + if (rtype == RDTMON_GROUP && + (prdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP || + prdtgrp->mode == RDT_MODE_PSEUDO_LOCKED)) { + ret = -EINVAL; + rdt_last_cmd_puts("Pseudo-locking in progress\n"); + goto out_unlock; + } + + /* allocate the rdtgroup. */ + rdtgrp = kzalloc_obj(*rdtgrp); + if (!rdtgrp) { + ret = -ENOSPC; + rdt_last_cmd_puts("Kernel out of memory\n"); + goto out_unlock; + } + *r = rdtgrp; + rdtgrp->mon.parent = prdtgrp; + rdtgrp->type = rtype; + INIT_LIST_HEAD(&rdtgrp->mon.crdtgrp_list); + + /* kernfs creates the directory for rdtgrp */ + kn = kernfs_create_dir(parent_kn, name, mode, rdtgrp); + if (IS_ERR(kn)) { + ret = PTR_ERR(kn); + rdt_last_cmd_puts("kernfs create error\n"); + goto out_free_rgrp; + } + rdtgrp->kn = kn; + + /* + * kernfs_remove() will drop the reference count on "kn" which + * will free it. But we still need it to stick around for the + * rdtgroup_kn_unlock(kn) call. Take one extra reference here, + * which will be dropped by kernfs_put() in rdtgroup_remove(). + */ + kernfs_get(kn); + + ret = rdtgroup_kn_set_ugid(kn); + if (ret) { + rdt_last_cmd_puts("kernfs perm error\n"); + goto out_destroy; + } + + if (rtype == RDTCTRL_GROUP) { + files = RFTYPE_BASE | RFTYPE_CTRL; + if (resctrl_arch_mon_capable()) + files |= RFTYPE_MON; + } else { + files = RFTYPE_BASE | RFTYPE_MON; + } + + ret = rdtgroup_add_files(kn, files); + if (ret) { + rdt_last_cmd_puts("kernfs fill error\n"); + goto out_destroy; + } + + /* + * The caller unlocks the parent_kn upon success. + */ + return 0; + +out_destroy: + kernfs_put(rdtgrp->kn); + kernfs_remove(rdtgrp->kn); +out_free_rgrp: + kfree(rdtgrp); +out_unlock: + rdtgroup_kn_unlock(parent_kn); + return ret; +} + +static void mkdir_rdt_prepare_clean(struct rdtgroup *rgrp) +{ + kernfs_remove(rgrp->kn); + rdtgroup_remove(rgrp); +} + +/* + * Create a monitor group under "mon_groups" directory of a control + * and monitor group(ctrl_mon). This is a resource group + * to monitor a subset of tasks and cpus in its parent ctrl_mon group. + */ +static int rdtgroup_mkdir_mon(struct kernfs_node *parent_kn, + const char *name, umode_t mode) +{ + struct rdtgroup *rdtgrp, *prgrp; + int ret; + + ret = mkdir_rdt_prepare(parent_kn, name, mode, RDTMON_GROUP, &rdtgrp); + if (ret) + return ret; + + prgrp = rdtgrp->mon.parent; + rdtgrp->closid = prgrp->closid; + + ret = mkdir_rdt_prepare_rmid_alloc(rdtgrp); + if (ret) { + mkdir_rdt_prepare_clean(rdtgrp); + goto out_unlock; + } + + kernfs_activate(rdtgrp->kn); + + /* + * Add the rdtgrp to the list of rdtgrps the parent + * ctrl_mon group has to track. + */ + list_add_tail(&rdtgrp->mon.crdtgrp_list, &prgrp->mon.crdtgrp_list); + +out_unlock: + rdtgroup_kn_unlock(parent_kn); + return ret; +} + +/* + * These are rdtgroups created under the root directory. Can be used + * to allocate and monitor resources. + */ +static int rdtgroup_mkdir_ctrl_mon(struct kernfs_node *parent_kn, + const char *name, umode_t mode) +{ + struct rdtgroup *rdtgrp; + struct kernfs_node *kn; + u32 closid; + int ret; + + ret = mkdir_rdt_prepare(parent_kn, name, mode, RDTCTRL_GROUP, &rdtgrp); + if (ret) + return ret; + + kn = rdtgrp->kn; + ret = closid_alloc(); + if (ret < 0) { + rdt_last_cmd_puts("Out of CLOSIDs\n"); + goto out_common_fail; + } + closid = ret; + ret = 0; + + rdtgrp->closid = closid; + + ret = mkdir_rdt_prepare_rmid_alloc(rdtgrp); + if (ret) + goto out_closid_free; + + kernfs_activate(rdtgrp->kn); + + ret = rdtgroup_init_alloc(rdtgrp); + if (ret < 0) + goto out_rmid_free; + + list_add(&rdtgrp->rdtgroup_list, &rdt_all_groups); + + if (resctrl_arch_mon_capable()) { + /* + * Create an empty mon_groups directory to hold the subset + * of tasks and cpus to monitor. + */ + ret = mongroup_create_dir(kn, rdtgrp, "mon_groups", NULL); + if (ret) { + rdt_last_cmd_puts("kernfs subdir error\n"); + goto out_del_list; + } + if (is_mba_sc(NULL)) + rdtgrp->mba_mbps_event = mba_mbps_default_event; + } + + goto out_unlock; + +out_del_list: + list_del(&rdtgrp->rdtgroup_list); +out_rmid_free: + mkdir_rdt_prepare_rmid_free(rdtgrp); +out_closid_free: + closid_free(closid); +out_common_fail: + mkdir_rdt_prepare_clean(rdtgrp); +out_unlock: + rdtgroup_kn_unlock(parent_kn); + return ret; +} + +static int rdtgroup_mkdir(struct kernfs_node *parent_kn, const char *name, + umode_t mode) +{ + /* Do not accept '\n' to avoid unparsable situation. */ + if (strchr(name, '\n')) + return -EINVAL; + + /* + * If the parent directory is the root directory and RDT + * allocation is supported, add a control and monitoring + * subdirectory + */ + if (resctrl_arch_alloc_capable() && parent_kn == rdtgroup_default.kn) + return rdtgroup_mkdir_ctrl_mon(parent_kn, name, mode); + + /* Else, attempt to add a monitoring subdirectory. */ + if (resctrl_arch_mon_capable()) + return rdtgroup_mkdir_mon(parent_kn, name, mode); + + return -EPERM; +} + +static int rdtgroup_rmdir_mon(struct rdtgroup *rdtgrp, cpumask_var_t tmpmask) +{ + struct rdtgroup *prdtgrp = rdtgrp->mon.parent; + u32 closid, rmid; + int cpu; + + /* Give any tasks back to the parent group */ + rdt_move_group_tasks(rdtgrp, prdtgrp, tmpmask); + + /* + * Update per cpu closid/rmid of the moved CPUs first. + * Note: the closid will not change, but the arch code still needs it. + */ + closid = prdtgrp->closid; + rmid = prdtgrp->mon.rmid; + for_each_cpu(cpu, &rdtgrp->cpu_mask) + resctrl_arch_set_cpu_default_closid_rmid(cpu, closid, rmid); + + /* + * Update the MSR on moved CPUs and CPUs which have moved + * task running on them. + */ + cpumask_or(tmpmask, tmpmask, &rdtgrp->cpu_mask); + update_closid_rmid(tmpmask, NULL); + + rdtgrp->flags = RDT_DELETED; + + rdtgroup_unassign_cntrs(rdtgrp); + + free_rmid(rdtgrp->closid, rdtgrp->mon.rmid); + + /* + * Remove the rdtgrp from the parent ctrl_mon group's list + */ + WARN_ON(list_empty(&prdtgrp->mon.crdtgrp_list)); + list_del(&rdtgrp->mon.crdtgrp_list); + + kernfs_remove(rdtgrp->kn); + + return 0; +} + +static int rdtgroup_ctrl_remove(struct rdtgroup *rdtgrp) +{ + rdtgrp->flags = RDT_DELETED; + list_del(&rdtgrp->rdtgroup_list); + + kernfs_remove(rdtgrp->kn); + return 0; +} + +static int rdtgroup_rmdir_ctrl(struct rdtgroup *rdtgrp, cpumask_var_t tmpmask) +{ + u32 closid, rmid; + int cpu; + + /* Give any tasks back to the default group */ + rdt_move_group_tasks(rdtgrp, &rdtgroup_default, tmpmask); + + /* Give any CPUs back to the default group */ + cpumask_or(&rdtgroup_default.cpu_mask, + &rdtgroup_default.cpu_mask, &rdtgrp->cpu_mask); + + /* Update per cpu closid and rmid of the moved CPUs first */ + closid = rdtgroup_default.closid; + rmid = rdtgroup_default.mon.rmid; + for_each_cpu(cpu, &rdtgrp->cpu_mask) + resctrl_arch_set_cpu_default_closid_rmid(cpu, closid, rmid); + + /* + * Update the MSR on moved CPUs and CPUs which have moved + * task running on them. + */ + cpumask_or(tmpmask, tmpmask, &rdtgrp->cpu_mask); + update_closid_rmid(tmpmask, NULL); + + rdtgroup_unassign_cntrs(rdtgrp); + + free_rmid(rdtgrp->closid, rdtgrp->mon.rmid); + closid_free(rdtgrp->closid); + + rdtgroup_ctrl_remove(rdtgrp); + + /* + * Free all the child monitor group rmids. + */ + free_all_child_rdtgrp(rdtgrp); + + return 0; +} + +static struct kernfs_node *rdt_kn_parent(struct kernfs_node *kn) +{ + /* + * Valid within the RCU section it was obtained or while rdtgroup_mutex + * is held. + */ + return rcu_dereference_check(kn->__parent, lockdep_is_held(&rdtgroup_mutex)); +} + +static int rdtgroup_rmdir(struct kernfs_node *kn) +{ + struct kernfs_node *parent_kn; + struct rdtgroup *rdtgrp; + cpumask_var_t tmpmask; + int ret = 0; + + if (!zalloc_cpumask_var(&tmpmask, GFP_KERNEL)) + return -ENOMEM; + + rdtgrp = rdtgroup_kn_lock_live(kn); + if (!rdtgrp) { + ret = -EPERM; + goto out; + } + parent_kn = rdt_kn_parent(kn); + + /* + * If the rdtgroup is a ctrl_mon group and parent directory + * is the root directory, remove the ctrl_mon group. + * + * If the rdtgroup is a mon group and parent directory + * is a valid "mon_groups" directory, remove the mon group. + */ + if (rdtgrp->type == RDTCTRL_GROUP && parent_kn == rdtgroup_default.kn && + rdtgrp != &rdtgroup_default) { + if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP || + rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) { + ret = rdtgroup_ctrl_remove(rdtgrp); + } else { + ret = rdtgroup_rmdir_ctrl(rdtgrp, tmpmask); + } + } else if (rdtgrp->type == RDTMON_GROUP && + is_mon_groups(parent_kn, rdt_kn_name(kn))) { + ret = rdtgroup_rmdir_mon(rdtgrp, tmpmask); + } else { + ret = -EPERM; + } + +out: + rdtgroup_kn_unlock(kn); + free_cpumask_var(tmpmask); + return ret; +} + +/** + * mongrp_reparent() - replace parent CTRL_MON group of a MON group + * @rdtgrp: the MON group whose parent should be replaced + * @new_prdtgrp: replacement parent CTRL_MON group for @rdtgrp + * @cpus: cpumask provided by the caller for use during this call + * + * Replaces the parent CTRL_MON group for a MON group, resulting in all member + * tasks' CLOSID immediately changing to that of the new parent group. + * Monitoring data for the group is unaffected by this operation. + */ +static void mongrp_reparent(struct rdtgroup *rdtgrp, + struct rdtgroup *new_prdtgrp, + cpumask_var_t cpus) +{ + struct rdtgroup *prdtgrp = rdtgrp->mon.parent; + + WARN_ON(rdtgrp->type != RDTMON_GROUP); + WARN_ON(new_prdtgrp->type != RDTCTRL_GROUP); + + /* Nothing to do when simply renaming a MON group. */ + if (prdtgrp == new_prdtgrp) + return; + + WARN_ON(list_empty(&prdtgrp->mon.crdtgrp_list)); + list_move_tail(&rdtgrp->mon.crdtgrp_list, + &new_prdtgrp->mon.crdtgrp_list); + + rdtgrp->mon.parent = new_prdtgrp; + rdtgrp->closid = new_prdtgrp->closid; + + /* Propagate updated closid to all tasks in this group. */ + rdt_move_group_tasks(rdtgrp, rdtgrp, cpus); + + update_closid_rmid(cpus, NULL); +} + +static int rdtgroup_rename(struct kernfs_node *kn, + struct kernfs_node *new_parent, const char *new_name) +{ + struct kernfs_node *kn_parent; + struct rdtgroup *new_prdtgrp; + struct rdtgroup *rdtgrp; + cpumask_var_t tmpmask; + int ret; + + rdtgrp = kernfs_to_rdtgroup(kn); + new_prdtgrp = kernfs_to_rdtgroup(new_parent); + if (!rdtgrp || !new_prdtgrp) + return -ENOENT; + + /* Release both kernfs active_refs before obtaining rdtgroup mutex. */ + rdtgroup_kn_get(rdtgrp, kn); + rdtgroup_kn_get(new_prdtgrp, new_parent); + + mutex_lock(&rdtgroup_mutex); + + rdt_last_cmd_clear(); + + /* + * Don't allow kernfs_to_rdtgroup() to return a parent rdtgroup if + * either kernfs_node is a file. + */ + if (kernfs_type(kn) != KERNFS_DIR || + kernfs_type(new_parent) != KERNFS_DIR) { + rdt_last_cmd_puts("Source and destination must be directories"); + ret = -EPERM; + goto out; + } + + if ((rdtgrp->flags & RDT_DELETED) || (new_prdtgrp->flags & RDT_DELETED)) { + ret = -ENOENT; + goto out; + } + + kn_parent = rdt_kn_parent(kn); + if (rdtgrp->type != RDTMON_GROUP || !kn_parent || + !is_mon_groups(kn_parent, rdt_kn_name(kn))) { + rdt_last_cmd_puts("Source must be a MON group\n"); + ret = -EPERM; + goto out; + } + + if (!is_mon_groups(new_parent, new_name)) { + rdt_last_cmd_puts("Destination must be a mon_groups subdirectory\n"); + ret = -EPERM; + goto out; + } + + /* + * If the MON group is monitoring CPUs, the CPUs must be assigned to the + * current parent CTRL_MON group and therefore cannot be assigned to + * the new parent, making the move illegal. + */ + if (!cpumask_empty(&rdtgrp->cpu_mask) && + rdtgrp->mon.parent != new_prdtgrp) { + rdt_last_cmd_puts("Cannot move a MON group that monitors CPUs\n"); + ret = -EPERM; + goto out; + } + + /* + * Allocate the cpumask for use in mongrp_reparent() to avoid the + * possibility of failing to allocate it after kernfs_rename() has + * succeeded. + */ + if (!zalloc_cpumask_var(&tmpmask, GFP_KERNEL)) { + ret = -ENOMEM; + goto out; + } + + /* + * Perform all input validation and allocations needed to ensure + * mongrp_reparent() will succeed before calling kernfs_rename(), + * otherwise it would be necessary to revert this call if + * mongrp_reparent() failed. + */ + ret = kernfs_rename(kn, new_parent, new_name); + if (!ret) + mongrp_reparent(rdtgrp, new_prdtgrp, tmpmask); + + free_cpumask_var(tmpmask); + +out: + mutex_unlock(&rdtgroup_mutex); + rdtgroup_kn_put(rdtgrp, kn); + rdtgroup_kn_put(new_prdtgrp, new_parent); + return ret; +} + +static int rdtgroup_show_options(struct seq_file *seq, struct kernfs_root *kf) +{ + if (resctrl_arch_get_cdp_enabled(RDT_RESOURCE_L3)) + seq_puts(seq, ",cdp"); + + if (resctrl_arch_get_cdp_enabled(RDT_RESOURCE_L2)) + seq_puts(seq, ",cdpl2"); + + if (is_mba_sc(resctrl_arch_get_resource(RDT_RESOURCE_MBA))) + seq_puts(seq, ",mba_MBps"); + + if (resctrl_debug) + seq_puts(seq, ",debug"); + + return 0; +} + +static struct kernfs_syscall_ops rdtgroup_kf_syscall_ops = { + .mkdir = rdtgroup_mkdir, + .rmdir = rdtgroup_rmdir, + .rename = rdtgroup_rename, + .show_options = rdtgroup_show_options, +}; + +static int rdtgroup_setup_root(struct rdt_fs_context *ctx) +{ + rdt_root = kernfs_create_root(&rdtgroup_kf_syscall_ops, + KERNFS_ROOT_CREATE_DEACTIVATED | + KERNFS_ROOT_EXTRA_OPEN_PERM_CHECK, + &rdtgroup_default); + if (IS_ERR(rdt_root)) + return PTR_ERR(rdt_root); + + ctx->kfc.root = rdt_root; + rdtgroup_default.kn = kernfs_root_to_node(rdt_root); + rdtgroup_default.flags = 0; + + return 0; +} + +static void rdtgroup_destroy_root(void) +{ + lockdep_assert_held(&rdtgroup_mutex); + + kernfs_destroy_root(rdt_root); + rdtgroup_default.kn = NULL; +} + +static void rdtgroup_setup_default(void) +{ + mutex_lock(&rdtgroup_mutex); + + rdtgroup_default.closid = RESCTRL_RESERVED_CLOSID; + rdtgroup_default.mon.rmid = RESCTRL_RESERVED_RMID; + rdtgroup_default.type = RDTCTRL_GROUP; + INIT_LIST_HEAD(&rdtgroup_default.mon.crdtgrp_list); + + list_add(&rdtgroup_default.rdtgroup_list, &rdt_all_groups); + + mutex_unlock(&rdtgroup_mutex); +} + +static void domain_destroy_l3_mon_state(struct rdt_l3_mon_domain *d) +{ + int idx; + + kfree(d->cntr_cfg); + bitmap_free(d->rmid_busy_llc); + for_each_mbm_idx(idx) { + kfree(d->mbm_states[idx]); + d->mbm_states[idx] = NULL; + } +} + +void resctrl_offline_ctrl_domain(struct rdt_resource *r, struct rdt_ctrl_domain *d) +{ + /* + * mbm_handle_overflow() may dereference this ctrl domain via + * update_mba_bw()->get_sc_ctrl_domain_from_cpu(). The architecture has + * unlinked the domain from the RCU list and waited a grace period, so + * no new worker iteration can find it; drain any worker that already + * holds a pointer to it before the architecture frees the domain. + * + * Software controller is enabled/disabled on mount/unmount with + * cpus_read_lock() held. Running here with cpus_write_lock() so + * there are no concurrent changes to software controller status. + */ + if (r->rid == RDT_RESOURCE_MBA && is_mba_sc(r)) { + struct rdt_resource *l3 = resctrl_arch_get_resource(RDT_RESOURCE_L3); + struct rdt_l3_mon_domain *mon_d; + + list_for_each_entry_rcu(mon_d, &l3->mon_domains, hdr.list, lockdep_is_cpus_held()) { + if (mon_d->hdr.id == d->hdr.id) { + cancel_delayed_work_sync(&mon_d->mbm_over); + break; + } + } + } + + mutex_lock(&rdtgroup_mutex); + + if (supports_mba_mbps() && r->rid == RDT_RESOURCE_MBA) + mba_sc_domain_destroy(r, d); + + mutex_unlock(&rdtgroup_mutex); +} + +void resctrl_offline_mon_domain(struct rdt_resource *r, struct rdt_domain_hdr *hdr) +{ + struct rdt_l3_mon_domain *d; + + /* + * Called by architecture under CPU hotplug lock as it prepares to remove + * the domain which is guaranteed to be accessible here. + * The domain has been unlinked from the RCU list and a grace period + * has elapsed, so no new worker can be scheduled. Drain any worker that + * is in flight or pending before letting architecture proceed to free + * the domain that has the workers' struct delayed_work embedded. + * Do so before taking rdtgroup_mutex since the workers also acquire it. + */ + if (r->rid == RDT_RESOURCE_L3 && + domain_header_is_valid(hdr, RESCTRL_MON_DOMAIN, RDT_RESOURCE_L3)) { + d = container_of(hdr, struct rdt_l3_mon_domain, hdr); + if (resctrl_is_mbm_enabled()) + cancel_delayed_work_sync(&d->mbm_over); + if (resctrl_is_mon_event_enabled(QOS_L3_OCCUP_EVENT_ID)) + cancel_delayed_work_sync(&d->cqm_limbo); + } + + mutex_lock(&rdtgroup_mutex); + + /* + * If resctrl is mounted, remove all the + * per domain monitor data directories. + */ + if (resctrl_mounted && resctrl_arch_mon_capable()) + rmdir_mondata_subdir_allrdtgrp(r, hdr); + + if (r->rid != RDT_RESOURCE_L3) + goto out_unlock; + + if (!domain_header_is_valid(hdr, RESCTRL_MON_DOMAIN, RDT_RESOURCE_L3)) + goto out_unlock; + + d = container_of(hdr, struct rdt_l3_mon_domain, hdr); + if (resctrl_is_mon_event_enabled(QOS_L3_OCCUP_EVENT_ID) && has_busy_rmid(d)) { + /* + * When a package is going down, forcefully + * decrement rmid->ebusy. There is no way to know + * that the L3 was flushed and hence may lead to + * incorrect counts in rare scenarios, but leaving + * the RMID as busy creates RMID leaks if the + * package never comes back. + */ + __check_limbo(d, true); + } + + domain_destroy_l3_mon_state(d); +out_unlock: + mutex_unlock(&rdtgroup_mutex); +} + +/** + * domain_setup_l3_mon_state() - Initialise domain monitoring structures. + * @r: The resource for the newly online domain. + * @d: The newly online domain. + * + * Allocate monitor resources that belong to this domain. + * Called when the first CPU of a domain comes online, regardless of whether + * the filesystem is mounted. + * During boot this may be called before global allocations have been made by + * resctrl_l3_mon_resource_init(). + * + * Called during CPU online that may run as soon as CPU online callbacks + * are set up during resctrl initialization. The number of supported RMIDs + * may be reduced if additional mon_capable resources are enumerated + * at mount time. This means the rdt_l3_mon_domain::mbm_states[] and + * rdt_l3_mon_domain::rmid_busy_llc allocations may be larger than needed. + * + * Return: 0 for success, or -ENOMEM. + */ +static int domain_setup_l3_mon_state(struct rdt_resource *r, struct rdt_l3_mon_domain *d) +{ + u32 idx_limit = resctrl_arch_system_num_rmid_idx(); + size_t tsize = sizeof(*d->mbm_states[0]); + enum resctrl_event_id eventid; + int idx; + + if (resctrl_is_mon_event_enabled(QOS_L3_OCCUP_EVENT_ID)) { + d->rmid_busy_llc = bitmap_zalloc(idx_limit, GFP_KERNEL); + if (!d->rmid_busy_llc) + return -ENOMEM; + } + + for_each_mbm_event_id(eventid) { + if (!resctrl_is_mon_event_enabled(eventid)) + continue; + idx = MBM_STATE_IDX(eventid); + d->mbm_states[idx] = kcalloc(idx_limit, tsize, GFP_KERNEL); + if (!d->mbm_states[idx]) + goto cleanup; + } + + if (resctrl_is_mbm_enabled() && r->mon.mbm_cntr_assignable) { + tsize = sizeof(*d->cntr_cfg); + d->cntr_cfg = kcalloc(r->mon.num_mbm_cntrs, tsize, GFP_KERNEL); + if (!d->cntr_cfg) + goto cleanup; + } + + return 0; +cleanup: + bitmap_free(d->rmid_busy_llc); + for_each_mbm_idx(idx) { + kfree(d->mbm_states[idx]); + d->mbm_states[idx] = NULL; + } + + return -ENOMEM; +} + +int resctrl_online_ctrl_domain(struct rdt_resource *r, struct rdt_ctrl_domain *d) +{ + int err = 0; + + mutex_lock(&rdtgroup_mutex); + + if (supports_mba_mbps() && r->rid == RDT_RESOURCE_MBA) { + /* RDT_RESOURCE_MBA is never mon_capable */ + err = mba_sc_domain_allocate(r, d); + } + + mutex_unlock(&rdtgroup_mutex); + + return err; +} + +int resctrl_online_mon_domain(struct rdt_resource *r, struct rdt_domain_hdr *hdr) +{ + struct rdt_l3_mon_domain *d; + int err = -EINVAL; + + mutex_lock(&rdtgroup_mutex); + + if (r->rid != RDT_RESOURCE_L3) + goto mkdir; + + if (!domain_header_is_valid(hdr, RESCTRL_MON_DOMAIN, RDT_RESOURCE_L3)) + goto out_unlock; + + d = container_of(hdr, struct rdt_l3_mon_domain, hdr); + err = domain_setup_l3_mon_state(r, d); + if (err) + goto out_unlock; + + if (resctrl_is_mbm_enabled()) { + INIT_DELAYED_WORK(&d->mbm_over, mbm_handle_overflow); + mbm_setup_overflow_handler(d, MBM_OVERFLOW_INTERVAL, + RESCTRL_PICK_ANY_CPU); + } + + if (resctrl_is_mon_event_enabled(QOS_L3_OCCUP_EVENT_ID)) + INIT_DELAYED_WORK(&d->cqm_limbo, cqm_handle_limbo); + +mkdir: + err = 0; + /* + * If the filesystem is not mounted then only the default resource group + * exists. Creation of its directories is deferred until mount time + * by rdt_get_tree() calling mkdir_mondata_all(). + * If resctrl is mounted, add per domain monitor data directories. + */ + if (resctrl_mounted && resctrl_arch_mon_capable()) + mkdir_mondata_subdir_allrdtgrp(r, hdr); + +out_unlock: + mutex_unlock(&rdtgroup_mutex); + + return err; +} + +void resctrl_online_cpu(unsigned int cpu) +{ + mutex_lock(&rdtgroup_mutex); + /* The CPU is set in default rdtgroup after online. */ + cpumask_set_cpu(cpu, &rdtgroup_default.cpu_mask); + mutex_unlock(&rdtgroup_mutex); +} + +static void clear_childcpus(struct rdtgroup *r, unsigned int cpu) +{ + struct rdtgroup *cr; + + list_for_each_entry(cr, &r->mon.crdtgrp_list, mon.crdtgrp_list) { + if (cpumask_test_and_clear_cpu(cpu, &cr->cpu_mask)) + break; + } +} + +static struct rdt_l3_mon_domain *get_mon_domain_from_cpu(int cpu, + struct rdt_resource *r) +{ + struct rdt_l3_mon_domain *d; + + lockdep_assert_cpus_held(); + + list_for_each_entry_rcu(d, &r->mon_domains, hdr.list, lockdep_is_cpus_held()) { + /* Find the domain that contains this CPU */ + if (cpumask_test_cpu(cpu, &d->hdr.cpu_mask)) + return d; + } + + return NULL; +} + +void resctrl_offline_cpu(unsigned int cpu) +{ + struct rdt_resource *l3 = resctrl_arch_get_resource(RDT_RESOURCE_L3); + struct rdt_l3_mon_domain *d; + struct rdtgroup *rdtgrp; + + mutex_lock(&rdtgroup_mutex); + list_for_each_entry(rdtgrp, &rdt_all_groups, rdtgroup_list) { + if (cpumask_test_and_clear_cpu(cpu, &rdtgrp->cpu_mask)) { + clear_childcpus(rdtgrp, cpu); + break; + } + } + + if (!l3->mon_capable) + goto out_unlock; + + d = get_mon_domain_from_cpu(cpu, l3); + if (d) { + if (resctrl_is_mbm_enabled() && cpu == d->mbm_work_cpu) { + mutex_unlock(&rdtgroup_mutex); + cancel_delayed_work_sync(&d->mbm_over); + mutex_lock(&rdtgroup_mutex); + mbm_setup_overflow_handler(d, 0, cpu); + } + if (resctrl_is_mon_event_enabled(QOS_L3_OCCUP_EVENT_ID) && + cpu == d->cqm_work_cpu && has_busy_rmid(d)) { + mutex_unlock(&rdtgroup_mutex); + cancel_delayed_work_sync(&d->cqm_limbo); + mutex_lock(&rdtgroup_mutex); + cqm_setup_limbo_handler(d, 0, cpu); + } + } + +out_unlock: + mutex_unlock(&rdtgroup_mutex); +} + +/* + * resctrl_init - resctrl filesystem initialization + * + * Setup resctrl file system including set up root, create mount point, + * register resctrl filesystem, and initialize files under root directory. + * + * Return: 0 on success or -errno + */ +int resctrl_init(void) +{ + int ret = 0; + + seq_buf_init(&last_cmd_status, last_cmd_status_buf, + sizeof(last_cmd_status_buf)); + + rdtgroup_setup_default(); + + thread_throttle_mode_init(); + + io_alloc_init(); + + ret = resctrl_l3_mon_resource_init(); + if (ret) + return ret; + + ret = sysfs_create_mount_point(fs_kobj, "resctrl"); + if (ret) { + resctrl_l3_mon_resource_exit(); + return ret; + } + + ret = register_filesystem(&rdt_fs_type); + if (ret) + goto cleanup_mountpoint; + + /* + * Adding the resctrl debugfs directory here may not be ideal since + * it would let the resctrl debugfs directory appear on the debugfs + * filesystem before the resctrl filesystem is mounted. + * It may also be ok since that would enable debugging of RDT before + * resctrl is mounted. + * The reason why the debugfs directory is created here and not in + * rdt_get_tree() is because rdt_get_tree() takes rdtgroup_mutex and + * during the debugfs directory creation also &sb->s_type->i_mutex_key + * (the lockdep class of inode->i_rwsem). Other filesystem + * interactions (eg. SyS_getdents) have the lock ordering: + * &sb->s_type->i_mutex_key --> &mm->mmap_lock + * During mmap(), called with &mm->mmap_lock, the rdtgroup_mutex + * is taken, thus creating dependency: + * &mm->mmap_lock --> rdtgroup_mutex for the latter that can cause + * issues considering the other two lock dependencies. + * By creating the debugfs directory here we avoid a dependency + * that may cause deadlock (even though file operations cannot + * occur until the filesystem is mounted, but I do not know how to + * tell lockdep that). + */ + debugfs_resctrl = debugfs_create_dir("resctrl", NULL); + + return 0; + +cleanup_mountpoint: + sysfs_remove_mount_point(fs_kobj, "resctrl"); + resctrl_l3_mon_resource_exit(); + + return ret; +} + +static bool resctrl_online_domains_exist(void) +{ + struct rdt_resource *r; + + /* + * Only walk capable resources to allow resctrl_arch_get_resource() + * to return dummy 'not capable' resources. + */ + for_each_alloc_capable_rdt_resource(r) { + if (!list_empty(&r->ctrl_domains)) + return true; + } + + for_each_mon_capable_rdt_resource(r) { + if (!list_empty(&r->mon_domains)) + return true; + } + + return false; +} + +/** + * resctrl_exit() - Remove the resctrl filesystem and free resources. + * + * Called by the architecture code in response to a fatal error. + * Removes resctrl files and structures from kernfs to prevent further + * configuration. + * + * When called by the architecture code, all CPUs and resctrl domains must be + * offline. This ensures the limbo and overflow handlers are not scheduled to + * run, meaning the data structures they access can be freed by + * resctrl_l3_mon_resource_exit(). + * + * After resctrl_exit() returns, the architecture code should return an + * error from all resctrl_arch_ functions that can do this. + * resctrl_arch_get_resource() must continue to return struct rdt_resources + * with the correct rid field to ensure the filesystem can be unmounted. + */ +void resctrl_exit(void) +{ + cpus_read_lock(); + WARN_ON_ONCE(resctrl_online_domains_exist()); + + mutex_lock(&rdtgroup_mutex); + resctrl_fs_teardown(); + mutex_unlock(&rdtgroup_mutex); + + cpus_read_unlock(); + + debugfs_remove_recursive(debugfs_resctrl); + debugfs_resctrl = NULL; + unregister_filesystem(&rdt_fs_type); + + /* + * Do not remove the sysfs mount point added by resctrl_init() so that + * it can be used to umount resctrl. + */ + + resctrl_l3_mon_resource_exit(); + free_rmid_lru_list(); +} diff --git a/fs/romfs/Kconfig b/fs/romfs/Kconfig new file mode 100644 index 000000000..f24a96a33 --- /dev/null +++ b/fs/romfs/Kconfig @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: GPL-2.0-only +config ROMFS_FS + tristate "ROM file system support" + depends on BLOCK || MTD + help + This is a very small read-only file system mainly intended for + initial ram disks of installation disks, but it could be used for + other read-only media as well. Read + for details. + + To compile this file system support as a module, choose M here: the + module will be called romfs. Note that the file system of your + root partition (the one containing the directory /) cannot be a + module. + + If you don't know whether you need it, then you don't need it: + answer N. + +# +# Select the backing stores to be supported +# +choice + prompt "RomFS backing stores" + depends on ROMFS_FS + default ROMFS_BACKED_BY_BLOCK + help + Select the backing stores to be supported. + +config ROMFS_BACKED_BY_BLOCK + bool "Block device-backed ROM file system support" + depends on BLOCK + help + This permits ROMFS to use block devices buffered through the page + cache as the medium from which to retrieve data. It does not allow + direct mapping of the medium. + + If unsure, answer Y. + +config ROMFS_BACKED_BY_MTD + bool "MTD-backed ROM file system support" + depends on MTD=y || (ROMFS_FS=m && MTD) + help + This permits ROMFS to use MTD based devices directly, without the + intercession of the block layer (which may have been disabled). It + also allows direct mapping of MTD devices through romfs files under + NOMMU conditions if the underlying device is directly addressable by + the CPU. + + If unsure, answer Y. + +config ROMFS_BACKED_BY_BOTH + bool "Both the above" + depends on BLOCK && (MTD=y || (ROMFS_FS=m && MTD)) +endchoice + + +config ROMFS_ON_BLOCK + bool + default y if ROMFS_BACKED_BY_BLOCK || ROMFS_BACKED_BY_BOTH + select BUFFER_HEAD + +config ROMFS_ON_MTD + bool + default y if ROMFS_BACKED_BY_MTD || ROMFS_BACKED_BY_BOTH diff --git a/fs/romfs/Makefile b/fs/romfs/Makefile new file mode 100644 index 000000000..844928f15 --- /dev/null +++ b/fs/romfs/Makefile @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the linux RomFS filesystem routines. +# + +obj-$(CONFIG_ROMFS_FS) += romfs.o + +romfs-y := storage.o super.o + +ifneq ($(CONFIG_MMU),y) +romfs-$(CONFIG_ROMFS_ON_MTD) += mmap-nommu.o +endif + diff --git a/fs/romfs/internal.h b/fs/romfs/internal.h new file mode 100644 index 000000000..e90df22bd --- /dev/null +++ b/fs/romfs/internal.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* RomFS internal definitions + * + * Copyright © 2007 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include + +struct romfs_inode_info { + struct inode vfs_inode; + unsigned long i_metasize; /* size of non-data area */ + unsigned long i_dataoffset; /* from the start of fs */ +}; + +static inline size_t romfs_maxsize(struct super_block *sb) +{ + return (size_t) (unsigned long) sb->s_fs_info; +} + +static inline struct romfs_inode_info *ROMFS_I(struct inode *inode) +{ + return container_of(inode, struct romfs_inode_info, vfs_inode); +} + +/* + * mmap-nommu.c + */ +#if !defined(CONFIG_MMU) && defined(CONFIG_ROMFS_ON_MTD) +extern const struct file_operations romfs_ro_fops; +#else +#define romfs_ro_fops generic_ro_fops +#endif + +/* + * storage.c + */ +extern int romfs_dev_read(struct super_block *sb, unsigned long pos, + void *buf, size_t buflen); +extern ssize_t romfs_dev_strnlen(struct super_block *sb, + unsigned long pos, size_t maxlen); +extern int romfs_dev_strcmp(struct super_block *sb, unsigned long pos, + const char *str, size_t size); diff --git a/fs/romfs/mmap-nommu.c b/fs/romfs/mmap-nommu.c new file mode 100644 index 000000000..7c3a1a7fe --- /dev/null +++ b/fs/romfs/mmap-nommu.c @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* NOMMU mmap support for RomFS on MTD devices + * + * Copyright © 2007 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include "internal.h" + +/* + * try to determine where a shared mapping can be made + * - only supported for NOMMU at the moment (MMU can't doesn't copy private + * mappings) + * - attempts to map through to the underlying MTD device + */ +static unsigned long romfs_get_unmapped_area(struct file *file, + unsigned long addr, + unsigned long len, + unsigned long pgoff, + unsigned long flags) +{ + struct inode *inode = file->f_mapping->host; + struct mtd_info *mtd = inode->i_sb->s_mtd; + unsigned long isize, offset, maxpages, lpages; + int ret; + + if (!mtd) + return (unsigned long) -ENOSYS; + + /* the mapping mustn't extend beyond the EOF */ + lpages = (len + PAGE_SIZE - 1) >> PAGE_SHIFT; + isize = i_size_read(inode); + offset = pgoff << PAGE_SHIFT; + + maxpages = (isize + PAGE_SIZE - 1) >> PAGE_SHIFT; + if ((pgoff >= maxpages) || (maxpages - pgoff < lpages)) + return (unsigned long) -EINVAL; + + if (addr != 0) + return (unsigned long) -EINVAL; + + if (len > mtd->size || pgoff >= (mtd->size >> PAGE_SHIFT)) + return (unsigned long) -EINVAL; + + offset += ROMFS_I(inode)->i_dataoffset; + if (offset >= mtd->size) + return (unsigned long) -EINVAL; + /* the mapping mustn't extend beyond the EOF */ + if ((offset + len) > mtd->size) + len = mtd->size - offset; + + ret = mtd_get_unmapped_area(mtd, len, offset, flags); + if (ret == -EOPNOTSUPP) + ret = -ENOSYS; + return (unsigned long) ret; +} + +/* + * permit a R/O mapping to be made directly through onto an MTD device if + * possible + */ +static int romfs_mmap_prepare(struct vm_area_desc *desc) +{ + return is_nommu_shared_vma_flags(&desc->vma_flags) ? 0 : -ENOSYS; +} + +static unsigned romfs_mmap_capabilities(struct file *file) +{ + struct mtd_info *mtd = file_inode(file)->i_sb->s_mtd; + + if (!mtd) + return NOMMU_MAP_COPY; + return mtd_mmap_capabilities(mtd); +} + +const struct file_operations romfs_ro_fops = { + .llseek = generic_file_llseek, + .read_iter = generic_file_read_iter, + .splice_read = filemap_splice_read, + .mmap_prepare = romfs_mmap_prepare, + .get_unmapped_area = romfs_get_unmapped_area, + .mmap_capabilities = romfs_mmap_capabilities, +}; diff --git a/fs/romfs/storage.c b/fs/romfs/storage.c new file mode 100644 index 000000000..b57b3ffcb --- /dev/null +++ b/fs/romfs/storage.c @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* RomFS storage access routines + * + * Copyright © 2007 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#include +#include +#include +#include "internal.h" + +#if !defined(CONFIG_ROMFS_ON_MTD) && !defined(CONFIG_ROMFS_ON_BLOCK) +#error no ROMFS backing store interface configured +#endif + +#ifdef CONFIG_ROMFS_ON_MTD +#define ROMFS_MTD_READ(sb, ...) mtd_read((sb)->s_mtd, ##__VA_ARGS__) + +/* + * read data from an romfs image on an MTD device + */ +static int romfs_mtd_read(struct super_block *sb, unsigned long pos, + void *buf, size_t buflen) +{ + size_t rlen; + int ret; + + ret = ROMFS_MTD_READ(sb, pos, buflen, &rlen, buf); + return (ret < 0 || rlen != buflen) ? -EIO : 0; +} + +/* + * determine the length of a string in a romfs image on an MTD device + */ +static ssize_t romfs_mtd_strnlen(struct super_block *sb, + unsigned long pos, size_t maxlen) +{ + ssize_t n = 0; + size_t segment; + u_char buf[16], *p; + size_t len; + int ret; + + /* scan the string up to 16 bytes at a time */ + while (maxlen > 0) { + segment = min_t(size_t, maxlen, 16); + ret = ROMFS_MTD_READ(sb, pos, segment, &len, buf); + if (ret < 0) + return ret; + p = memchr(buf, 0, len); + if (p) + return n + (p - buf); + maxlen -= len; + pos += len; + n += len; + } + + return n; +} + +/* + * compare a string to one in a romfs image on MTD + * - return 1 if matched, 0 if differ, -ve if error + */ +static int romfs_mtd_strcmp(struct super_block *sb, unsigned long pos, + const char *str, size_t size) +{ + u_char buf[17]; + size_t len, segment; + int ret; + + /* scan the string up to 16 bytes at a time, and attempt to grab the + * trailing NUL whilst we're at it */ + buf[0] = 0xff; + + while (size > 0) { + segment = min_t(size_t, size + 1, 17); + ret = ROMFS_MTD_READ(sb, pos, segment, &len, buf); + if (ret < 0) + return ret; + len--; + if (memcmp(buf, str, len) != 0) + return 0; + buf[0] = buf[len]; + size -= len; + pos += len; + str += len; + } + + /* check the trailing NUL was */ + if (buf[0]) + return 0; + + return 1; +} +#endif /* CONFIG_ROMFS_ON_MTD */ + +#ifdef CONFIG_ROMFS_ON_BLOCK +/* + * read data from an romfs image on a block device + */ +static int romfs_blk_read(struct super_block *sb, unsigned long pos, + void *buf, size_t buflen) +{ + struct buffer_head *bh; + unsigned long offset; + size_t segment; + + /* copy the string up to blocksize bytes at a time */ + while (buflen > 0) { + offset = pos & (ROMBSIZE - 1); + segment = min_t(size_t, buflen, ROMBSIZE - offset); + bh = sb_bread(sb, pos >> ROMBSBITS); + if (!bh) + return -EIO; + memcpy(buf, bh->b_data + offset, segment); + brelse(bh); + buf += segment; + buflen -= segment; + pos += segment; + } + + return 0; +} + +/* + * determine the length of a string in romfs on a block device + */ +static ssize_t romfs_blk_strnlen(struct super_block *sb, + unsigned long pos, size_t limit) +{ + struct buffer_head *bh; + unsigned long offset; + ssize_t n = 0; + size_t segment; + u_char *buf, *p; + + /* scan the string up to blocksize bytes at a time */ + while (limit > 0) { + offset = pos & (ROMBSIZE - 1); + segment = min_t(size_t, limit, ROMBSIZE - offset); + bh = sb_bread(sb, pos >> ROMBSBITS); + if (!bh) + return -EIO; + buf = bh->b_data + offset; + p = memchr(buf, 0, segment); + brelse(bh); + if (p) + return n + (p - buf); + limit -= segment; + pos += segment; + n += segment; + } + + return n; +} + +/* + * compare a string to one in a romfs image on a block device + * - return 1 if matched, 0 if differ, -ve if error + */ +static int romfs_blk_strcmp(struct super_block *sb, unsigned long pos, + const char *str, size_t size) +{ + struct buffer_head *bh; + unsigned long offset; + size_t segment; + bool matched, terminated = false; + + /* compare string up to a block at a time */ + while (size > 0) { + offset = pos & (ROMBSIZE - 1); + segment = min_t(size_t, size, ROMBSIZE - offset); + bh = sb_bread(sb, pos >> ROMBSBITS); + if (!bh) + return -EIO; + matched = (memcmp(bh->b_data + offset, str, segment) == 0); + + size -= segment; + pos += segment; + str += segment; + if (matched && size == 0 && offset + segment < ROMBSIZE) { + if (!bh->b_data[offset + segment]) + terminated = true; + else + matched = false; + } + brelse(bh); + if (!matched) + return 0; + } + + if (!terminated) { + /* the terminating NUL must be on the first byte of the next + * block */ + BUG_ON((pos & (ROMBSIZE - 1)) != 0); + bh = sb_bread(sb, pos >> ROMBSBITS); + if (!bh) + return -EIO; + matched = !bh->b_data[0]; + brelse(bh); + if (!matched) + return 0; + } + + return 1; +} +#endif /* CONFIG_ROMFS_ON_BLOCK */ + +/* + * read data from the romfs image + */ +int romfs_dev_read(struct super_block *sb, unsigned long pos, + void *buf, size_t buflen) +{ + size_t limit; + + limit = romfs_maxsize(sb); + if (pos >= limit || buflen > limit - pos) + return -EIO; + +#ifdef CONFIG_ROMFS_ON_MTD + if (sb->s_mtd) + return romfs_mtd_read(sb, pos, buf, buflen); +#endif +#ifdef CONFIG_ROMFS_ON_BLOCK + if (sb->s_bdev) + return romfs_blk_read(sb, pos, buf, buflen); +#endif + return -EIO; +} + +/* + * determine the length of a string in romfs + */ +ssize_t romfs_dev_strnlen(struct super_block *sb, + unsigned long pos, size_t maxlen) +{ + size_t limit; + + limit = romfs_maxsize(sb); + if (pos >= limit) + return -EIO; + if (maxlen > limit - pos) + maxlen = limit - pos; + +#ifdef CONFIG_ROMFS_ON_MTD + if (sb->s_mtd) + return romfs_mtd_strnlen(sb, pos, maxlen); +#endif +#ifdef CONFIG_ROMFS_ON_BLOCK + if (sb->s_bdev) + return romfs_blk_strnlen(sb, pos, maxlen); +#endif + return -EIO; +} + +/* + * compare a string to one in romfs + * - the string to be compared to, str, may not be NUL-terminated; instead the + * string is of the specified size + * - return 1 if matched, 0 if differ, -ve if error + */ +int romfs_dev_strcmp(struct super_block *sb, unsigned long pos, + const char *str, size_t size) +{ + size_t limit; + + limit = romfs_maxsize(sb); + if (pos >= limit) + return -EIO; + if (size > ROMFS_MAXFN) + return -ENAMETOOLONG; + if (size + 1 > limit - pos) + return -EIO; + +#ifdef CONFIG_ROMFS_ON_MTD + if (sb->s_mtd) + return romfs_mtd_strcmp(sb, pos, str, size); +#endif +#ifdef CONFIG_ROMFS_ON_BLOCK + if (sb->s_bdev) + return romfs_blk_strcmp(sb, pos, str, size); +#endif + return -EIO; +} diff --git a/fs/romfs/super.c b/fs/romfs/super.c new file mode 100644 index 000000000..7f783d617 --- /dev/null +++ b/fs/romfs/super.c @@ -0,0 +1,672 @@ +/* Block- or MTD-based romfs + * + * Copyright © 2007 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + * + * Derived from: ROMFS file system, Linux implementation + * + * Copyright © 1997-1999 Janos Farkas + * + * Using parts of the minix filesystem + * Copyright © 1991, 1992 Linus Torvalds + * + * and parts of the affs filesystem additionally + * Copyright © 1993 Ray Burr + * Copyright © 1996 Hans-Joachim Widmaier + * + * Changes + * Changed for 2.1.19 modules + * Jan 1997 Initial release + * Jun 1997 2.1.43+ changes + * Proper page locking in read_folio + * Changed to work with 2.1.45+ fs + * Jul 1997 Fixed follow_link + * 2.1.47 + * lookup shouldn't return -ENOENT + * from Horst von Brand: + * fail on wrong checksum + * double unlock_super was possible + * correct namelen for statfs + * spotted by Bill Hawes: + * readlink shouldn't iput() + * Jun 1998 2.1.106 from Avery Pennarun: glibc scandir() + * exposed a problem in readdir + * 2.1.107 code-freeze spellchecker run + * Aug 1998 2.1.118+ VFS changes + * Sep 1998 2.1.122 another VFS change (follow_link) + * Apr 1999 2.2.7 no more EBADF checking in + * lookup/readdir, use ERR_PTR + * Jun 1999 2.3.6 d_alloc_root use changed + * 2.3.9 clean up usage of ENOENT/negative + * dentries in lookup + * clean up page flags setting + * (error, uptodate, locking) in + * in read_folio + * use init_special_inode for + * fifos/sockets (and streamline) in + * read_inode, fix _ops table order + * Aug 1999 2.3.16 __initfunc() => __init change + * Oct 1999 2.3.24 page->owner hack obsoleted + * Nov 1999 2.3.27 2.3.25+ page->offset => index change + * + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public Licence + * as published by the Free Software Foundation; either version + * 2 of the Licence, or (at your option) any later version. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +static struct kmem_cache *romfs_inode_cachep; + +static const umode_t romfs_modemap[8] = { + 0, /* hard link */ + S_IFDIR | 0644, /* directory */ + S_IFREG | 0644, /* regular file */ + S_IFLNK | 0777, /* symlink */ + S_IFBLK | 0600, /* blockdev */ + S_IFCHR | 0600, /* chardev */ + S_IFSOCK | 0644, /* socket */ + S_IFIFO | 0644 /* FIFO */ +}; + +static const unsigned char romfs_dtype_table[] = { + DT_UNKNOWN, DT_DIR, DT_REG, DT_LNK, DT_BLK, DT_CHR, DT_SOCK, DT_FIFO +}; + +static struct inode *romfs_iget(struct super_block *sb, unsigned long pos); + +/* + * read a page worth of data from the image + */ +static int romfs_read_folio(struct file *file, struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + loff_t offset, size; + unsigned long fillsize, pos; + void *buf; + int ret; + + buf = kmap_local_folio(folio, 0); + + offset = folio_pos(folio); + size = i_size_read(inode); + fillsize = 0; + ret = 0; + if (offset < size) { + size -= offset; + fillsize = size > PAGE_SIZE ? PAGE_SIZE : size; + + pos = ROMFS_I(inode)->i_dataoffset + offset; + + ret = romfs_dev_read(inode->i_sb, pos, buf, fillsize); + if (ret < 0) { + fillsize = 0; + ret = -EIO; + } + } + + buf = folio_zero_tail(folio, fillsize, buf + fillsize); + kunmap_local(buf); + folio_end_read(folio, ret == 0); + return ret; +} + +static const struct address_space_operations romfs_aops = { + .read_folio = romfs_read_folio +}; + +/* + * read the entries from a directory + */ +static int romfs_readdir(struct file *file, struct dir_context *ctx) +{ + struct inode *i = file_inode(file); + struct romfs_inode ri; + unsigned long offset, maxoff; + int j, ino, nextfh; + char fsname[ROMFS_MAXFN]; /* XXX dynamic? */ + int ret; + + maxoff = romfs_maxsize(i->i_sb); + + offset = ctx->pos; + if (!offset) { + offset = i->i_ino & ROMFH_MASK; + ret = romfs_dev_read(i->i_sb, offset, &ri, ROMFH_SIZE); + if (ret < 0) + goto out; + offset = be32_to_cpu(ri.spec) & ROMFH_MASK; + } + + /* Not really failsafe, but we are read-only... */ + for (;;) { + if (!offset || offset >= maxoff) { + offset = maxoff; + ctx->pos = offset; + goto out; + } + ctx->pos = offset; + + /* Fetch inode info */ + ret = romfs_dev_read(i->i_sb, offset, &ri, ROMFH_SIZE); + if (ret < 0) + goto out; + + j = romfs_dev_strnlen(i->i_sb, offset + ROMFH_SIZE, + sizeof(fsname) - 1); + if (j < 0) + goto out; + + ret = romfs_dev_read(i->i_sb, offset + ROMFH_SIZE, fsname, j); + if (ret < 0) + goto out; + fsname[j] = '\0'; + + ino = offset; + nextfh = be32_to_cpu(ri.next); + if ((nextfh & ROMFH_TYPE) == ROMFH_HRD) + ino = be32_to_cpu(ri.spec); + if (!dir_emit(ctx, fsname, j, ino, + romfs_dtype_table[nextfh & ROMFH_TYPE])) + goto out; + + offset = nextfh & ROMFH_MASK; + } +out: + return 0; +} + +/* + * look up an entry in a directory + */ +static struct dentry *romfs_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + unsigned long offset, maxoff; + struct inode *inode = NULL; + struct romfs_inode ri; + const char *name; /* got from dentry */ + int len, ret; + + offset = dir->i_ino & ROMFH_MASK; + ret = romfs_dev_read(dir->i_sb, offset, &ri, ROMFH_SIZE); + if (ret < 0) + goto error; + + /* search all the file entries in the list starting from the one + * pointed to by the directory's special data */ + maxoff = romfs_maxsize(dir->i_sb); + offset = be32_to_cpu(ri.spec) & ROMFH_MASK; + + name = dentry->d_name.name; + len = dentry->d_name.len; + + for (;;) { + if (!offset || offset >= maxoff) + break; + + ret = romfs_dev_read(dir->i_sb, offset, &ri, sizeof(ri)); + if (ret < 0) + goto error; + + /* try to match the first 16 bytes of name */ + ret = romfs_dev_strcmp(dir->i_sb, offset + ROMFH_SIZE, name, + len); + if (ret < 0) + goto error; + if (ret == 1) { + /* Hard link handling */ + if ((be32_to_cpu(ri.next) & ROMFH_TYPE) == ROMFH_HRD) + offset = be32_to_cpu(ri.spec) & ROMFH_MASK; + inode = romfs_iget(dir->i_sb, offset); + if (IS_ERR(inode)) + return ERR_CAST(inode); + break; + } + + /* next entry */ + offset = be32_to_cpu(ri.next) & ROMFH_MASK; + } + + return d_splice_alias(inode, dentry); +error: + return ERR_PTR(ret); +} + +static const struct file_operations romfs_dir_operations = { + .read = generic_read_dir, + .iterate_shared = romfs_readdir, + .llseek = generic_file_llseek, +}; + +static const struct inode_operations romfs_dir_inode_operations = { + .lookup = romfs_lookup, +}; + +#define ROMFS_MAX_HARDLINK_DEPTH 64 + +/* + * get a romfs inode based on its position in the image (which doubles as the + * inode number) + */ +static struct inode *romfs_iget(struct super_block *sb, unsigned long pos) +{ + struct romfs_inode_info *inode; + struct romfs_inode ri; + struct inode *i; + unsigned long nlen; + unsigned nextfh; + unsigned int depth = 0; + int ret; + umode_t mode; + + /* we might have to traverse a chain of "hard link" file entries to get + * to the actual file */ + for (;;) { + ret = romfs_dev_read(sb, pos, &ri, sizeof(ri)); + if (ret < 0) + goto error; + + /* XXX: do romfs_checksum here too (with name) */ + + nextfh = be32_to_cpu(ri.next); + if ((nextfh & ROMFH_TYPE) != ROMFH_HRD) + break; + + if (++depth > ROMFS_MAX_HARDLINK_DEPTH) + return ERR_PTR(-ELOOP); + + pos = be32_to_cpu(ri.spec) & ROMFH_MASK; + } + + /* determine the length of the filename */ + nlen = romfs_dev_strnlen(sb, pos + ROMFH_SIZE, ROMFS_MAXFN); + if (IS_ERR_VALUE(nlen)) + goto eio; + + /* get an inode for this image position */ + i = iget_locked(sb, pos); + if (!i) + return ERR_PTR(-ENOMEM); + + if (!(inode_state_read_once(i) & I_NEW)) + return i; + + /* precalculate the data offset */ + inode = ROMFS_I(i); + inode->i_metasize = (ROMFH_SIZE + nlen + 1 + ROMFH_PAD) & ROMFH_MASK; + inode->i_dataoffset = pos + inode->i_metasize; + + set_nlink(i, 1); /* Hard to decide.. */ + i->i_size = be32_to_cpu(ri.size); + inode_set_mtime_to_ts(i, + inode_set_atime_to_ts(i, inode_set_ctime(i, 0, 0))); + + /* set up mode and ops */ + mode = romfs_modemap[nextfh & ROMFH_TYPE]; + + switch (nextfh & ROMFH_TYPE) { + case ROMFH_DIR: + i->i_size = ROMFS_I(i)->i_metasize; + i->i_op = &romfs_dir_inode_operations; + i->i_fop = &romfs_dir_operations; + if (nextfh & ROMFH_EXEC) + mode |= S_IXUGO; + break; + case ROMFH_REG: + i->i_fop = &romfs_ro_fops; + i->i_data.a_ops = &romfs_aops; + if (nextfh & ROMFH_EXEC) + mode |= S_IXUGO; + break; + case ROMFH_SYM: + i->i_op = &page_symlink_inode_operations; + inode_nohighmem(i); + i->i_data.a_ops = &romfs_aops; + mode |= S_IRWXUGO; + break; + default: + /* depending on MBZ for sock/fifos */ + nextfh = be32_to_cpu(ri.spec); + init_special_inode(i, mode, MKDEV(nextfh >> 16, + nextfh & 0xffff)); + break; + } + + i->i_mode = mode; + i->i_blocks = (i->i_size + 511) >> 9; + + unlock_new_inode(i); + return i; + +eio: + ret = -EIO; +error: + pr_err("read error for inode 0x%lx\n", pos); + return ERR_PTR(ret); +} + +/* + * allocate a new inode + */ +static struct inode *romfs_alloc_inode(struct super_block *sb) +{ + struct romfs_inode_info *inode; + + inode = alloc_inode_sb(sb, romfs_inode_cachep, GFP_KERNEL); + return inode ? &inode->vfs_inode : NULL; +} + +/* + * return a spent inode to the slab cache + */ +static void romfs_free_inode(struct inode *inode) +{ + kmem_cache_free(romfs_inode_cachep, ROMFS_I(inode)); +} + +/* + * get filesystem statistics + */ +static int romfs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + u64 id = 0; + + /* When calling huge_encode_dev(), + * use sb->s_bdev->bd_dev when, + * - CONFIG_ROMFS_ON_BLOCK defined + * use sb->s_dev when, + * - CONFIG_ROMFS_ON_BLOCK undefined and + * - CONFIG_ROMFS_ON_MTD defined + * leave id as 0 when, + * - CONFIG_ROMFS_ON_BLOCK undefined and + * - CONFIG_ROMFS_ON_MTD undefined + */ + if (sb->s_bdev) + id = huge_encode_dev(sb->s_bdev->bd_dev); + else if (sb->s_dev) + id = huge_encode_dev(sb->s_dev); + + buf->f_type = ROMFS_MAGIC; + buf->f_namelen = ROMFS_MAXFN; + buf->f_bsize = ROMBSIZE; + buf->f_bfree = buf->f_bavail = buf->f_ffree; + buf->f_blocks = + (romfs_maxsize(dentry->d_sb) + ROMBSIZE - 1) >> ROMBSBITS; + buf->f_fsid = u64_to_fsid(id); + return 0; +} + +/* + * remounting must involve read-only + */ +static int romfs_reconfigure(struct fs_context *fc) +{ + sync_filesystem(fc->root->d_sb); + fc->sb_flags |= SB_RDONLY; + return 0; +} + +static const struct super_operations romfs_super_ops = { + .alloc_inode = romfs_alloc_inode, + .free_inode = romfs_free_inode, + .statfs = romfs_statfs, +}; + +/* + * checksum check on part of a romfs filesystem + */ +static __u32 romfs_checksum(const void *data, int size) +{ + const __be32 *ptr = data; + __u32 sum; + + sum = 0; + size >>= 2; + while (size > 0) { + sum += be32_to_cpu(*ptr++); + size--; + } + return sum; +} + +/* + * fill in the superblock + */ +static int romfs_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct romfs_super_block *rsb; + struct inode *root; + unsigned long pos, img_size; + const char *storage; + size_t len; + int ret; + +#ifdef CONFIG_BLOCK + if (!sb->s_mtd) { + if (!sb_set_blocksize(sb, ROMBSIZE)) { + errorf(fc, "romfs: unable to set blocksize\n"); + return -EINVAL; + } + } else { + sb->s_blocksize = ROMBSIZE; + sb->s_blocksize_bits = blksize_bits(ROMBSIZE); + } +#endif + + sb->s_maxbytes = 0xFFFFFFFF; + sb->s_magic = ROMFS_MAGIC; + sb->s_flags |= SB_RDONLY | SB_NOATIME; + sb->s_time_min = 0; + sb->s_time_max = 0; + sb->s_op = &romfs_super_ops; + +#ifdef CONFIG_ROMFS_ON_MTD + /* Use same dev ID from the underlying mtdblock device */ + if (sb->s_mtd) + sb->s_dev = MKDEV(MTD_BLOCK_MAJOR, sb->s_mtd->index); +#endif + /* read the image superblock and check it */ + rsb = kmalloc(512, GFP_KERNEL); + if (!rsb) + return -ENOMEM; + + sb->s_fs_info = (void *) 512; + ret = romfs_dev_read(sb, 0, rsb, 512); + if (ret < 0) + goto error_rsb; + + img_size = be32_to_cpu(rsb->size); + + if (sb->s_mtd && img_size > sb->s_mtd->size) + goto error_rsb_inval; + + sb->s_fs_info = (void *) img_size; + + if (rsb->word0 != ROMSB_WORD0 || rsb->word1 != ROMSB_WORD1 || + img_size < ROMFH_SIZE) { + if (!(fc->sb_flags & SB_SILENT)) + errorf(fc, "VFS: Can't find a romfs filesystem on dev %s.\n", + sb->s_id); + goto error_rsb_inval; + } + + if (romfs_checksum(rsb, min_t(size_t, img_size, 512))) { + pr_err("bad initial checksum on dev %s.\n", sb->s_id); + goto error_rsb_inval; + } + + storage = sb->s_mtd ? "MTD" : "the block layer"; + + len = strnlen(rsb->name, ROMFS_MAXFN); + if (!(fc->sb_flags & SB_SILENT)) + pr_notice("Mounting image '%*.*s' through %s\n", + (unsigned) len, (unsigned) len, rsb->name, storage); + + kfree(rsb); + rsb = NULL; + + /* find the root directory */ + pos = (ROMFH_SIZE + len + 1 + ROMFH_PAD) & ROMFH_MASK; + + root = romfs_iget(sb, pos); + if (IS_ERR(root)) + return PTR_ERR(root); + + sb->s_root = d_make_root(root); + if (!sb->s_root) + return -ENOMEM; + + return 0; + +error_rsb_inval: + ret = -EINVAL; +error_rsb: + kfree(rsb); + return ret; +} + +/* + * get a superblock for mounting + */ +static int romfs_get_tree(struct fs_context *fc) +{ + int ret = -EINVAL; + +#ifdef CONFIG_ROMFS_ON_MTD + ret = get_tree_mtd(fc, romfs_fill_super); +#endif +#ifdef CONFIG_ROMFS_ON_BLOCK + if (ret == -EINVAL) + ret = get_tree_bdev(fc, romfs_fill_super); +#endif + return ret; +} + +static const struct fs_context_operations romfs_context_ops = { + .get_tree = romfs_get_tree, + .reconfigure = romfs_reconfigure, +}; + +/* + * Set up the filesystem mount context. + */ +static int romfs_init_fs_context(struct fs_context *fc) +{ + fc->ops = &romfs_context_ops; + return 0; +} + +/* + * destroy a romfs superblock in the appropriate manner + */ +static void romfs_kill_sb(struct super_block *sb) +{ + generic_shutdown_super(sb); + +#ifdef CONFIG_ROMFS_ON_MTD + if (sb->s_mtd) { + put_mtd_device(sb->s_mtd); + sb->s_mtd = NULL; + } +#endif +#ifdef CONFIG_ROMFS_ON_BLOCK + if (sb->s_bdev) { + sync_blockdev(sb->s_bdev); + fs_bdev_file_release(sb->s_bdev_file, sb); + } +#endif +} + +static struct file_system_type romfs_fs_type = { + .owner = THIS_MODULE, + .name = "romfs", + .init_fs_context = romfs_init_fs_context, + .kill_sb = romfs_kill_sb, + .fs_flags = FS_REQUIRES_DEV, +}; +MODULE_ALIAS_FS("romfs"); + +/* + * inode storage initialiser + */ +static void romfs_i_init_once(void *_inode) +{ + struct romfs_inode_info *inode = _inode; + + inode_init_once(&inode->vfs_inode); +} + +/* + * romfs module initialisation + */ +static int __init init_romfs_fs(void) +{ + int ret; + + pr_info("ROMFS MTD (C) 2007 Red Hat, Inc.\n"); + + romfs_inode_cachep = + kmem_cache_create("romfs_i", + sizeof(struct romfs_inode_info), 0, + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, + romfs_i_init_once); + + if (!romfs_inode_cachep) { + pr_err("Failed to initialise inode cache\n"); + return -ENOMEM; + } + ret = register_filesystem(&romfs_fs_type); + if (ret) { + pr_err("Failed to register filesystem\n"); + goto error_register; + } + return 0; + +error_register: + kmem_cache_destroy(romfs_inode_cachep); + return ret; +} + +/* + * romfs module removal + */ +static void __exit exit_romfs_fs(void) +{ + unregister_filesystem(&romfs_fs_type); + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(romfs_inode_cachep); +} + +module_init(init_romfs_fs); +module_exit(exit_romfs_fs); + +MODULE_DESCRIPTION("Direct-MTD Capable RomFS"); +MODULE_AUTHOR("Red Hat, Inc."); +MODULE_LICENSE("GPL"); /* Actually dual-licensed, but it doesn't matter for */ diff --git a/fs/select.c b/fs/select.c new file mode 100644 index 000000000..95d765310 --- /dev/null +++ b/fs/select.c @@ -0,0 +1,1438 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * This file contains the procedures for the handling of select and poll + * + * Created for Linux based loosely upon Mathius Lattner's minix + * patches by Peter MacDonald. Heavily edited by Linus. + * + * 4 February 1994 + * COFF/ELF binary emulation. If the process has the STICKY_TIMEOUTS + * flag set in its personality we do *not* modify the given timeout + * parameter to reflect time remaining. + * + * 24 January 2000 + * Changed sys_poll()/do_poll() to use PAGE_SIZE chunk-based allocation + * of fds to overcome nfds < 16390 descriptors limit (Tigran Aivazian). + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include /* for STICKY_TIMEOUTS */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include + + +/* + * Estimate expected accuracy in ns from a timeval. + * + * After quite a bit of churning around, we've settled on + * a simple thing of taking 0.1% of the timeout as the + * slack, with a cap of 100 msec. + * "nice" tasks get a 0.5% slack instead. + * + * Consider this comment an open invitation to come up with even + * better solutions.. + */ + +#define MAX_SLACK (100 * NSEC_PER_MSEC) + +static long __estimate_accuracy(struct timespec64 *tv) +{ + long slack; + int divfactor = 1000; + + if (tv->tv_sec < 0) + return 0; + + if (task_nice(current) > 0) + divfactor = divfactor / 5; + + if (tv->tv_sec > MAX_SLACK / (NSEC_PER_SEC/divfactor)) + return MAX_SLACK; + + slack = tv->tv_nsec / divfactor; + slack += tv->tv_sec * (NSEC_PER_SEC/divfactor); + + if (slack > MAX_SLACK) + return MAX_SLACK; + + return slack; +} + +u64 select_estimate_accuracy(struct timespec64 *tv) +{ + u64 ret; + struct timespec64 now; + u64 slack = current->timer_slack_ns; + + if (slack == 0) + return 0; + + ktime_get_ts64(&now); + now = timespec64_sub(*tv, now); + ret = __estimate_accuracy(&now); + if (ret < slack) + return slack; + return ret; +} + + + +struct poll_table_page { + struct poll_table_page * next; + struct poll_table_entry * entry; + struct poll_table_entry entries[]; +}; + +#define POLL_TABLE_FULL(table) \ + ((unsigned long)((table)->entry+1) > PAGE_SIZE + (unsigned long)(table)) + +/* + * Ok, Peter made a complicated, but straightforward multiple_wait() function. + * I have rewritten this, taking some shortcuts: This code may not be easy to + * follow, but it should be free of race-conditions, and it's practical. If you + * understand what I'm doing here, then you understand how the linux + * sleep/wakeup mechanism works. + * + * Two very simple procedures, poll_wait() and poll_freewait() make all the + * work. poll_wait() is an inline-function defined in , + * as all select/poll functions have to call it to add an entry to the + * poll table. + */ +static void __pollwait(struct file *filp, wait_queue_head_t *wait_address, + poll_table *p); + +void poll_initwait(struct poll_wqueues *pwq) +{ + init_poll_funcptr(&pwq->pt, __pollwait); + pwq->polling_task = current; + pwq->triggered = 0; + pwq->error = 0; + pwq->table = NULL; + pwq->inline_index = 0; +} +EXPORT_SYMBOL(poll_initwait); + +static void free_poll_entry(struct poll_table_entry *entry) +{ + remove_wait_queue(entry->wait_address, &entry->wait); + fput(entry->filp); +} + +void poll_freewait(struct poll_wqueues *pwq) +{ + struct poll_table_page * p = pwq->table; + int i; + for (i = 0; i < pwq->inline_index; i++) + free_poll_entry(pwq->inline_entries + i); + while (p) { + struct poll_table_entry * entry; + struct poll_table_page *old; + + entry = p->entry; + do { + entry--; + free_poll_entry(entry); + } while (entry > p->entries); + old = p; + p = p->next; + kfree(old); + } +} +EXPORT_SYMBOL(poll_freewait); + +static struct poll_table_entry *poll_get_entry(struct poll_wqueues *p) +{ + struct poll_table_page *table = p->table; + + if (p->inline_index < N_INLINE_POLL_ENTRIES) + return p->inline_entries + p->inline_index++; + + if (!table || POLL_TABLE_FULL(table)) { + struct poll_table_page *new_table; + + new_table = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!new_table) { + p->error = -ENOMEM; + return NULL; + } + new_table->entry = new_table->entries; + new_table->next = table; + p->table = new_table; + table = new_table; + } + + return table->entry++; +} + +static int __pollwake(wait_queue_entry_t *wait, unsigned mode, int sync, void *key) +{ + struct poll_wqueues *pwq = wait->private; + DECLARE_WAITQUEUE(dummy_wait, pwq->polling_task); + + /* + * Although this function is called under waitqueue lock, LOCK + * doesn't imply write barrier and the users expect write + * barrier semantics on wakeup functions. The following + * smp_wmb() is equivalent to smp_wmb() in try_to_wake_up() + * and is paired with smp_store_mb() in poll_schedule_timeout. + */ + smp_wmb(); + WRITE_ONCE(pwq->triggered, 1); + + /* + * Perform the default wake up operation using a dummy + * waitqueue. + * + * TODO: This is hacky but there currently is no interface to + * pass in @sync. @sync is scheduled to be removed and once + * that happens, wake_up_process() can be used directly. + */ + return default_wake_function(&dummy_wait, mode, sync, key); +} + +static int pollwake(wait_queue_entry_t *wait, unsigned mode, int sync, void *key) +{ + struct poll_table_entry *entry; + + entry = container_of(wait, struct poll_table_entry, wait); + if (key && !(key_to_poll(key) & entry->key)) + return 0; + return __pollwake(wait, mode, sync, key); +} + +/* Add a new entry */ +static void __pollwait(struct file *filp, wait_queue_head_t *wait_address, + poll_table *p) +{ + struct poll_wqueues *pwq = container_of(p, struct poll_wqueues, pt); + struct poll_table_entry *entry = poll_get_entry(pwq); + if (!entry) + return; + entry->filp = get_file(filp); + entry->wait_address = wait_address; + entry->key = p->_key; + init_waitqueue_func_entry(&entry->wait, pollwake); + entry->wait.private = pwq; + add_wait_queue(wait_address, &entry->wait); +} + +static int poll_schedule_timeout(struct poll_wqueues *pwq, int state, + ktime_t *expires, unsigned long slack) +{ + int rc = -EINTR; + + set_current_state(state); + if (!READ_ONCE(pwq->triggered)) + rc = schedule_hrtimeout_range(expires, slack, HRTIMER_MODE_ABS); + __set_current_state(TASK_RUNNING); + + /* + * Prepare for the next iteration. + * + * The following smp_store_mb() serves two purposes. First, it's + * the counterpart rmb of the wmb in pollwake() such that data + * written before wake up is always visible after wake up. + * Second, the full barrier guarantees that triggered clearing + * doesn't pass event check of the next iteration. Note that + * this problem doesn't exist for the first iteration as + * add_wait_queue() has full barrier semantics. + */ + smp_store_mb(pwq->triggered, 0); + + return rc; +} + +/** + * poll_select_set_timeout - helper function to setup the timeout value + * @to: pointer to timespec64 variable for the final timeout + * @sec: seconds (from user space) + * @nsec: nanoseconds (from user space) + * + * Note, we do not use a timespec for the user space value here, That + * way we can use the function for timeval and compat interfaces as well. + * + * Returns -EINVAL if sec/nsec are not normalized. Otherwise 0. + */ +int poll_select_set_timeout(struct timespec64 *to, time64_t sec, long nsec) +{ + struct timespec64 ts = {.tv_sec = sec, .tv_nsec = nsec}; + + if (!timespec64_valid(&ts)) + return -EINVAL; + + /* Optimize for the zero timeout value here */ + if (!sec && !nsec) { + to->tv_sec = to->tv_nsec = 0; + } else { + ktime_get_ts64(to); + *to = timespec64_add_safe(*to, ts); + } + return 0; +} + +enum poll_time_type { + PT_TIMEVAL = 0, + PT_OLD_TIMEVAL = 1, + PT_TIMESPEC = 2, + PT_OLD_TIMESPEC = 3, +}; + +static int poll_select_finish(struct timespec64 *end_time, + void __user *p, + enum poll_time_type pt_type, int ret) +{ + struct timespec64 rts; + + restore_saved_sigmask_unless(ret == -ERESTARTNOHAND); + + if (!p) + return ret; + + if (current->personality & STICKY_TIMEOUTS) + goto sticky; + + /* No update for zero timeout */ + if (!end_time->tv_sec && !end_time->tv_nsec) + return ret; + + ktime_get_ts64(&rts); + rts = timespec64_sub(*end_time, rts); + if (rts.tv_sec < 0) + rts.tv_sec = rts.tv_nsec = 0; + + + switch (pt_type) { + case PT_TIMEVAL: + { + struct __kernel_old_timeval rtv; + + if (sizeof(rtv) > sizeof(rtv.tv_sec) + sizeof(rtv.tv_usec)) + memset(&rtv, 0, sizeof(rtv)); + rtv.tv_sec = rts.tv_sec; + rtv.tv_usec = rts.tv_nsec / NSEC_PER_USEC; + if (!copy_to_user(p, &rtv, sizeof(rtv))) + return ret; + } + break; + case PT_OLD_TIMEVAL: + { + struct old_timeval32 rtv; + + rtv.tv_sec = rts.tv_sec; + rtv.tv_usec = rts.tv_nsec / NSEC_PER_USEC; + if (!copy_to_user(p, &rtv, sizeof(rtv))) + return ret; + } + break; + case PT_TIMESPEC: + if (!put_timespec64(&rts, p)) + return ret; + break; + case PT_OLD_TIMESPEC: + if (!put_old_timespec32(&rts, p)) + return ret; + break; + default: + BUG(); + } + /* + * If an application puts its timeval in read-only memory, we + * don't want the Linux-specific update to the timeval to + * cause a fault after the select has completed + * successfully. However, because we're not updating the + * timeval, we can't restart the system call. + */ + +sticky: + if (ret == -ERESTARTNOHAND) + ret = -EINTR; + return ret; +} + +/* + * Scalable version of the fd_set. + */ + +typedef struct { + unsigned long *in, *out, *ex; + unsigned long *res_in, *res_out, *res_ex; +} fd_set_bits; + +/* + * How many longwords for "nr" bits? + */ +#define FDS_BITPERLONG (8*sizeof(long)) +#define FDS_LONGS(nr) (((nr)+FDS_BITPERLONG-1)/FDS_BITPERLONG) +#define FDS_BYTES(nr) (FDS_LONGS(nr)*sizeof(long)) + +/* + * Use "unsigned long" accesses to let user-mode fd_set's be long-aligned. + */ +static inline +int get_fd_set(unsigned long nr, void __user *ufdset, unsigned long *fdset) +{ + nr = FDS_BYTES(nr); + if (ufdset) + return copy_from_user(fdset, ufdset, nr) ? -EFAULT : 0; + + memset(fdset, 0, nr); + return 0; +} + +static inline unsigned long __must_check +set_fd_set(unsigned long nr, void __user *ufdset, unsigned long *fdset) +{ + if (ufdset) + return __copy_to_user(ufdset, fdset, FDS_BYTES(nr)); + return 0; +} + +static inline +void zero_fd_set(unsigned long nr, unsigned long *fdset) +{ + memset(fdset, 0, FDS_BYTES(nr)); +} + +#define FDS_IN(fds, n) (fds->in + n) +#define FDS_OUT(fds, n) (fds->out + n) +#define FDS_EX(fds, n) (fds->ex + n) + +#define BITS(fds, n) (*FDS_IN(fds, n)|*FDS_OUT(fds, n)|*FDS_EX(fds, n)) + +static int max_select_fd(unsigned long n, fd_set_bits *fds) +{ + unsigned long *open_fds; + unsigned long set; + int max; + struct fdtable *fdt; + + /* handle last in-complete long-word first */ + set = ~(~0UL << (n & (BITS_PER_LONG-1))); + n /= BITS_PER_LONG; + fdt = files_fdtable(current->files); + open_fds = fdt->open_fds + n; + max = 0; + if (set) { + set &= BITS(fds, n); + if (set) { + if (!(set & ~*open_fds)) + goto get_max; + return -EBADF; + } + } + while (n) { + open_fds--; + n--; + set = BITS(fds, n); + if (!set) + continue; + if (set & ~*open_fds) + return -EBADF; + if (max) + continue; +get_max: + do { + max++; + set >>= 1; + } while (set); + max += n * BITS_PER_LONG; + } + + return max; +} + +#define POLLIN_SET (EPOLLRDNORM | EPOLLRDBAND | EPOLLIN | EPOLLHUP | EPOLLERR |\ + EPOLLNVAL) +#define POLLOUT_SET (EPOLLWRBAND | EPOLLWRNORM | EPOLLOUT | EPOLLERR |\ + EPOLLNVAL) +#define POLLEX_SET (EPOLLPRI | EPOLLNVAL) + +static inline __poll_t select_poll_one(int fd, poll_table *wait, unsigned long in, + unsigned long out, unsigned long bit, + __poll_t ll_flag) +{ + CLASS(fd, f)(fd); + + if (fd_empty(f)) + return EPOLLNVAL; + + wait->_key = POLLEX_SET | ll_flag; + if (in & bit) + wait->_key |= POLLIN_SET; + if (out & bit) + wait->_key |= POLLOUT_SET; + + return vfs_poll(fd_file(f), wait); +} + +static noinline_for_stack int do_select(int n, fd_set_bits *fds, struct timespec64 *end_time) +{ + ktime_t expire, *to = NULL; + struct poll_wqueues table; + poll_table *wait; + int retval, i, timed_out = 0; + u64 slack = 0; + __poll_t busy_flag = net_busy_loop_on() ? POLL_BUSY_LOOP : 0; + unsigned long busy_start = 0; + + rcu_read_lock(); + retval = max_select_fd(n, fds); + rcu_read_unlock(); + + if (retval < 0) + return retval; + n = retval; + + poll_initwait(&table); + wait = &table.pt; + if (end_time && !end_time->tv_sec && !end_time->tv_nsec) { + wait->_qproc = NULL; + timed_out = 1; + } + + if (end_time && !timed_out) + slack = select_estimate_accuracy(end_time); + + retval = 0; + for (;;) { + unsigned long *rinp, *routp, *rexp, *inp, *outp, *exp; + bool can_busy_loop = false; + + inp = fds->in; outp = fds->out; exp = fds->ex; + rinp = fds->res_in; routp = fds->res_out; rexp = fds->res_ex; + + for (i = 0; i < n; ++rinp, ++routp, ++rexp) { + unsigned long in, out, ex, all_bits, bit = 1, j; + unsigned long res_in = 0, res_out = 0, res_ex = 0; + __poll_t mask; + + in = *inp++; out = *outp++; ex = *exp++; + all_bits = in | out | ex; + if (all_bits == 0) { + i += BITS_PER_LONG; + continue; + } + + for (j = 0; j < BITS_PER_LONG; ++j, ++i, bit <<= 1) { + if (i >= n) + break; + if (!(bit & all_bits)) + continue; + mask = select_poll_one(i, wait, in, out, bit, + busy_flag); + if ((mask & POLLIN_SET) && (in & bit)) { + res_in |= bit; + retval++; + wait->_qproc = NULL; + } + if ((mask & POLLOUT_SET) && (out & bit)) { + res_out |= bit; + retval++; + wait->_qproc = NULL; + } + if ((mask & POLLEX_SET) && (ex & bit)) { + res_ex |= bit; + retval++; + wait->_qproc = NULL; + } + /* got something, stop busy polling */ + if (retval) { + can_busy_loop = false; + busy_flag = 0; + + /* + * only remember a returned + * POLL_BUSY_LOOP if we asked for it + */ + } else if (busy_flag & mask) + can_busy_loop = true; + + } + if (res_in) + *rinp = res_in; + if (res_out) + *routp = res_out; + if (res_ex) + *rexp = res_ex; + cond_resched(); + } + wait->_qproc = NULL; + if (retval || timed_out || signal_pending(current)) + break; + if (table.error) { + retval = table.error; + break; + } + + /* only if found POLL_BUSY_LOOP sockets && not out of time */ + if (can_busy_loop && !need_resched()) { + if (!busy_start) { + busy_start = busy_loop_current_time(); + continue; + } + if (!busy_loop_timeout(busy_start)) + continue; + } + busy_flag = 0; + + /* + * If this is the first loop and we have a timeout + * given, then we convert to ktime_t and set the to + * pointer to the expiry value. + */ + if (end_time && !to) { + expire = timespec64_to_ktime(*end_time); + to = &expire; + } + + if (!poll_schedule_timeout(&table, TASK_INTERRUPTIBLE, + to, slack)) + timed_out = 1; + } + + poll_freewait(&table); + + return retval; +} + +/* + * We can actually return ERESTARTSYS instead of EINTR, but I'd + * like to be certain this leads to no problems. So I return + * EINTR just for safety. + * + * Update: ERESTARTSYS breaks at least the xview clock binary, so + * I'm trying ERESTARTNOHAND which restart only when you want to. + */ +int core_sys_select(int n, fd_set __user *inp, fd_set __user *outp, + fd_set __user *exp, struct timespec64 *end_time) +{ + fd_set_bits fds; + void *bits; + int ret, max_fds; + size_t size, alloc_size; + struct fdtable *fdt; + /* Allocate small arguments on the stack to save memory and be faster */ + long stack_fds[SELECT_STACK_ALLOC/sizeof(long)]; + + ret = -EINVAL; + if (unlikely(n < 0)) + goto out_nofds; + + /* max_fds can increase, so grab it once to avoid race */ + rcu_read_lock(); + fdt = files_fdtable(current->files); + max_fds = fdt->max_fds; + rcu_read_unlock(); + if (n > max_fds) + n = max_fds; + + /* + * We need 6 bitmaps (in/out/ex for both incoming and outgoing), + * since we used fdset we need to allocate memory in units of + * long-words. + */ + size = FDS_BYTES(n); + bits = stack_fds; + if (size > sizeof(stack_fds) / 6) { + /* Not enough space in on-stack array; must use kmalloc */ + ret = -ENOMEM; + if (size > (SIZE_MAX / 6)) + goto out_nofds; + + alloc_size = 6 * size; + bits = kvmalloc(alloc_size, GFP_KERNEL); + if (!bits) + goto out_nofds; + } + fds.in = bits; + fds.out = bits + size; + fds.ex = bits + 2*size; + fds.res_in = bits + 3*size; + fds.res_out = bits + 4*size; + fds.res_ex = bits + 5*size; + + if ((ret = get_fd_set(n, inp, fds.in)) || + (ret = get_fd_set(n, outp, fds.out)) || + (ret = get_fd_set(n, exp, fds.ex))) + goto out; + zero_fd_set(n, fds.res_in); + zero_fd_set(n, fds.res_out); + zero_fd_set(n, fds.res_ex); + + ret = do_select(n, &fds, end_time); + + if (ret < 0) + goto out; + if (!ret) { + ret = -ERESTARTNOHAND; + if (signal_pending(current)) + goto out; + ret = 0; + } + + if (set_fd_set(n, inp, fds.res_in) || + set_fd_set(n, outp, fds.res_out) || + set_fd_set(n, exp, fds.res_ex)) + ret = -EFAULT; + +out: + if (bits != stack_fds) + kvfree(bits); +out_nofds: + return ret; +} + +static int kern_select(int n, fd_set __user *inp, fd_set __user *outp, + fd_set __user *exp, struct __kernel_old_timeval __user *tvp) +{ + struct timespec64 end_time, *to = NULL; + struct __kernel_old_timeval tv; + int ret; + + if (tvp) { + if (copy_from_user(&tv, tvp, sizeof(tv))) + return -EFAULT; + + /* + * Reject negative components before normalisation. The seconds + * sum below is performed in signed long and a crafted negative + * timeval can wrap to a positive value that passes + * timespec64_valid() and turns into an effectively-infinite + * deadline via timespec64_add_safe()'s saturation, instead of + * the -EINVAL POSIX requires for negative timeouts. + */ + if (tv.tv_sec < 0 || tv.tv_usec < 0) + return -EINVAL; + + to = &end_time; + if (poll_select_set_timeout(to, + tv.tv_sec + (tv.tv_usec / USEC_PER_SEC), + (tv.tv_usec % USEC_PER_SEC) * NSEC_PER_USEC)) + return -EINVAL; + } + + ret = core_sys_select(n, inp, outp, exp, to); + return poll_select_finish(&end_time, tvp, PT_TIMEVAL, ret); +} + +SYSCALL_DEFINE5(select, int, n, fd_set __user *, inp, fd_set __user *, outp, + fd_set __user *, exp, struct __kernel_old_timeval __user *, tvp) +{ + return kern_select(n, inp, outp, exp, tvp); +} + +static long do_pselect(int n, fd_set __user *inp, fd_set __user *outp, + fd_set __user *exp, void __user *tsp, + const sigset_t __user *sigmask, size_t sigsetsize, + enum poll_time_type type) +{ + struct timespec64 ts, end_time, *to = NULL; + int ret; + + if (tsp) { + switch (type) { + case PT_TIMESPEC: + if (get_timespec64(&ts, tsp)) + return -EFAULT; + break; + case PT_OLD_TIMESPEC: + if (get_old_timespec32(&ts, tsp)) + return -EFAULT; + break; + default: + BUG(); + } + + to = &end_time; + if (poll_select_set_timeout(to, ts.tv_sec, ts.tv_nsec)) + return -EINVAL; + } + + ret = set_user_sigmask(sigmask, sigsetsize); + if (ret) + return ret; + + ret = core_sys_select(n, inp, outp, exp, to); + return poll_select_finish(&end_time, tsp, type, ret); +} + +/* + * Most architectures can't handle 7-argument syscalls. So we provide a + * 6-argument version where the sixth argument is a pointer to a structure + * which has a pointer to the sigset_t itself followed by a size_t containing + * the sigset size. + */ +struct sigset_argpack { + sigset_t __user *p; + size_t size; +}; + +static inline int get_sigset_argpack(struct sigset_argpack *to, + struct sigset_argpack __user *from) +{ + // the path is hot enough for overhead of copy_from_user() to matter + if (from) { + scoped_user_read_access(from, Efault) { + unsafe_get_user(to->p, &from->p, Efault); + unsafe_get_user(to->size, &from->size, Efault); + } + } + return 0; +Efault: + return -EFAULT; +} + +SYSCALL_DEFINE6(pselect6, int, n, fd_set __user *, inp, fd_set __user *, outp, + fd_set __user *, exp, struct __kernel_timespec __user *, tsp, + void __user *, sig) +{ + struct sigset_argpack x = {NULL, 0}; + + if (get_sigset_argpack(&x, sig)) + return -EFAULT; + + return do_pselect(n, inp, outp, exp, tsp, x.p, x.size, PT_TIMESPEC); +} + +#if defined(CONFIG_COMPAT_32BIT_TIME) && !defined(CONFIG_64BIT) + +SYSCALL_DEFINE6(pselect6_time32, int, n, fd_set __user *, inp, fd_set __user *, outp, + fd_set __user *, exp, struct old_timespec32 __user *, tsp, + void __user *, sig) +{ + struct sigset_argpack x = {NULL, 0}; + + if (get_sigset_argpack(&x, sig)) + return -EFAULT; + + return do_pselect(n, inp, outp, exp, tsp, x.p, x.size, PT_OLD_TIMESPEC); +} + +#endif + +#ifdef __ARCH_WANT_SYS_OLD_SELECT +struct sel_arg_struct { + unsigned long n; + fd_set __user *inp, *outp, *exp; + struct __kernel_old_timeval __user *tvp; +}; + +SYSCALL_DEFINE1(old_select, struct sel_arg_struct __user *, arg) +{ + struct sel_arg_struct a; + + if (copy_from_user(&a, arg, sizeof(a))) + return -EFAULT; + return kern_select(a.n, a.inp, a.outp, a.exp, a.tvp); +} +#endif + +struct poll_list { + struct poll_list *next; + unsigned int len; + struct pollfd entries[] __counted_by(len); +}; + +#define POLLFD_PER_PAGE ((PAGE_SIZE-sizeof(struct poll_list)) / sizeof(struct pollfd)) + +/* + * Fish for pollable events on the pollfd->fd file descriptor. We're only + * interested in events matching the pollfd->events mask, and the result + * matching that mask is both recorded in pollfd->revents and returned. The + * pwait poll_table will be used by the fd-provided poll handler for waiting, + * if pwait->_qproc is non-NULL. + */ +static inline __poll_t do_pollfd(struct pollfd *pollfd, poll_table *pwait, + bool *can_busy_poll, + __poll_t busy_flag) +{ + int fd = pollfd->fd; + __poll_t mask, filter; + + if (unlikely(fd < 0)) + return 0; + + CLASS(fd, f)(fd); + if (fd_empty(f)) + return EPOLLNVAL; + + /* userland u16 ->events contains POLL... bitmap */ + filter = demangle_poll(pollfd->events) | EPOLLERR | EPOLLHUP; + pwait->_key = filter | busy_flag; + mask = vfs_poll(fd_file(f), pwait); + if (mask & busy_flag) + *can_busy_poll = true; + return mask & filter; /* Mask out unneeded events. */ +} + +static int do_poll(struct poll_list *list, struct poll_wqueues *wait, + struct timespec64 *end_time) +{ + poll_table* pt = &wait->pt; + ktime_t expire, *to = NULL; + int timed_out = 0, count = 0; + u64 slack = 0; + __poll_t busy_flag = net_busy_loop_on() ? POLL_BUSY_LOOP : 0; + unsigned long busy_start = 0; + + /* Optimise the no-wait case */ + if (end_time && !end_time->tv_sec && !end_time->tv_nsec) { + pt->_qproc = NULL; + timed_out = 1; + } + + if (end_time && !timed_out) + slack = select_estimate_accuracy(end_time); + + for (;;) { + struct poll_list *walk; + bool can_busy_loop = false; + + for (walk = list; walk != NULL; walk = walk->next) { + struct pollfd * pfd, * pfd_end; + + pfd = walk->entries; + pfd_end = pfd + walk->len; + for (; pfd != pfd_end; pfd++) { + __poll_t mask; + /* + * Fish for events. If we found one, record it + * and kill poll_table->_qproc, so we don't + * needlessly register any other waiters after + * this. They'll get immediately deregistered + * when we break out and return. + */ + mask = do_pollfd(pfd, pt, &can_busy_loop, busy_flag); + pfd->revents = mangle_poll(mask); + if (mask) { + count++; + pt->_qproc = NULL; + /* found something, stop busy polling */ + busy_flag = 0; + can_busy_loop = false; + } + } + } + /* + * All waiters have already been registered, so don't provide + * a poll_table->_qproc to them on the next loop iteration. + */ + pt->_qproc = NULL; + if (!count) { + count = wait->error; + if (signal_pending(current)) + count = -ERESTARTNOHAND; + } + if (count || timed_out) + break; + + /* only if found POLL_BUSY_LOOP sockets && not out of time */ + if (can_busy_loop && !need_resched()) { + if (!busy_start) { + busy_start = busy_loop_current_time(); + continue; + } + if (!busy_loop_timeout(busy_start)) + continue; + } + busy_flag = 0; + + /* + * If this is the first loop and we have a timeout + * given, then we convert to ktime_t and set the to + * pointer to the expiry value. + */ + if (end_time && !to) { + expire = timespec64_to_ktime(*end_time); + to = &expire; + } + + if (!poll_schedule_timeout(wait, TASK_INTERRUPTIBLE, to, slack)) + timed_out = 1; + } + return count; +} + +#define N_STACK_PPS ((sizeof(stack_pps) - sizeof(struct poll_list)) / \ + sizeof(struct pollfd)) + +static int do_sys_poll(struct pollfd __user *ufds, unsigned int nfds, + struct timespec64 *end_time) +{ + struct poll_wqueues table; + int err = -EFAULT, fdcount; + /* Allocate small arguments on the stack to save memory and be + faster - use long to make sure the buffer is aligned properly + on 64 bit archs to avoid unaligned access */ + long stack_pps[POLL_STACK_ALLOC/sizeof(long)]; + struct poll_list *const head = (struct poll_list *)stack_pps; + struct poll_list *walk = head; + unsigned int todo = nfds; + unsigned int len; + + if (nfds > rlimit(RLIMIT_NOFILE)) + return -EINVAL; + + len = min_t(unsigned int, nfds, N_STACK_PPS); + for (;;) { + walk->next = NULL; + walk->len = len; + if (!len) + break; + + if (copy_from_user(walk->entries, ufds + nfds-todo, + sizeof(struct pollfd) * walk->len)) + goto out_fds; + + if (walk->len >= todo) + break; + todo -= walk->len; + + len = min(todo, POLLFD_PER_PAGE); + walk = walk->next = kmalloc_flex(*walk, entries, len); + if (!walk) { + err = -ENOMEM; + goto out_fds; + } + } + + poll_initwait(&table); + fdcount = do_poll(head, &table, end_time); + poll_freewait(&table); + + scoped_user_write_access_size(ufds, nfds * sizeof(*ufds), out_fds) { + struct pollfd __user *_ufds = ufds; + + for (walk = head; walk; walk = walk->next) { + struct pollfd *fds = walk->entries; + unsigned int j; + + for (j = walk->len; j; fds++, _ufds++, j--) + unsafe_put_user(fds->revents, &_ufds->revents, out_fds); + } + } + + err = fdcount; +out_fds: + walk = head->next; + while (walk) { + struct poll_list *pos = walk; + walk = walk->next; + kfree(pos); + } + + return err; +} + +static long do_restart_poll(struct restart_block *restart_block) +{ + struct pollfd __user *ufds = restart_block->poll.ufds; + int nfds = restart_block->poll.nfds; + struct timespec64 *to = NULL; + int ret; + + if (restart_block->poll.has_timeout) + to = &restart_block->poll.end_time; + + ret = do_sys_poll(ufds, nfds, to); + + if (ret == -ERESTARTNOHAND) + ret = set_restart_fn(restart_block, do_restart_poll); + + return ret; +} + +SYSCALL_DEFINE3(poll, struct pollfd __user *, ufds, unsigned int, nfds, + int, timeout_msecs) +{ + struct timespec64 end_time, *to = NULL; + int ret; + + if (timeout_msecs >= 0) { + to = &end_time; + poll_select_set_timeout(to, timeout_msecs / MSEC_PER_SEC, + NSEC_PER_MSEC * (timeout_msecs % MSEC_PER_SEC)); + } + + ret = do_sys_poll(ufds, nfds, to); + + if (ret == -ERESTARTNOHAND) { + struct restart_block *restart_block; + + restart_block = ¤t->restart_block; + restart_block->poll.ufds = ufds; + restart_block->poll.nfds = nfds; + + if (timeout_msecs >= 0) { + restart_block->poll.end_time = end_time; + restart_block->poll.has_timeout = 1; + } else + restart_block->poll.has_timeout = 0; + + ret = set_restart_fn(restart_block, do_restart_poll); + } + return ret; +} + +SYSCALL_DEFINE5(ppoll, struct pollfd __user *, ufds, unsigned int, nfds, + struct __kernel_timespec __user *, tsp, const sigset_t __user *, sigmask, + size_t, sigsetsize) +{ + struct timespec64 ts, end_time, *to = NULL; + int ret; + + if (tsp) { + if (get_timespec64(&ts, tsp)) + return -EFAULT; + + to = &end_time; + if (poll_select_set_timeout(to, ts.tv_sec, ts.tv_nsec)) + return -EINVAL; + } + + ret = set_user_sigmask(sigmask, sigsetsize); + if (ret) + return ret; + + ret = do_sys_poll(ufds, nfds, to); + return poll_select_finish(&end_time, tsp, PT_TIMESPEC, ret); +} + +#if defined(CONFIG_COMPAT_32BIT_TIME) && !defined(CONFIG_64BIT) + +SYSCALL_DEFINE5(ppoll_time32, struct pollfd __user *, ufds, unsigned int, nfds, + struct old_timespec32 __user *, tsp, const sigset_t __user *, sigmask, + size_t, sigsetsize) +{ + struct timespec64 ts, end_time, *to = NULL; + int ret; + + if (tsp) { + if (get_old_timespec32(&ts, tsp)) + return -EFAULT; + + to = &end_time; + if (poll_select_set_timeout(to, ts.tv_sec, ts.tv_nsec)) + return -EINVAL; + } + + ret = set_user_sigmask(sigmask, sigsetsize); + if (ret) + return ret; + + ret = do_sys_poll(ufds, nfds, to); + return poll_select_finish(&end_time, tsp, PT_OLD_TIMESPEC, ret); +} +#endif + +#ifdef CONFIG_COMPAT +#define __COMPAT_NFDBITS (8 * sizeof(compat_ulong_t)) + +/* + * Ooo, nasty. We need here to frob 32-bit unsigned longs to + * 64-bit unsigned longs. + */ +static +int compat_get_fd_set(unsigned long nr, compat_ulong_t __user *ufdset, + unsigned long *fdset) +{ + if (ufdset) { + return compat_get_bitmap(fdset, ufdset, nr); + } else { + zero_fd_set(nr, fdset); + return 0; + } +} + +static +int compat_set_fd_set(unsigned long nr, compat_ulong_t __user *ufdset, + unsigned long *fdset) +{ + if (!ufdset) + return 0; + return compat_put_bitmap(ufdset, fdset, nr); +} + + +/* + * This is a virtual copy of sys_select from fs/select.c and probably + * should be compared to it from time to time + */ + +/* + * We can actually return ERESTARTSYS instead of EINTR, but I'd + * like to be certain this leads to no problems. So I return + * EINTR just for safety. + * + * Update: ERESTARTSYS breaks at least the xview clock binary, so + * I'm trying ERESTARTNOHAND which restart only when you want to. + */ +static int compat_core_sys_select(int n, compat_ulong_t __user *inp, + compat_ulong_t __user *outp, compat_ulong_t __user *exp, + struct timespec64 *end_time) +{ + fd_set_bits fds; + void *bits; + int size, max_fds, ret = -EINVAL; + struct fdtable *fdt; + long stack_fds[SELECT_STACK_ALLOC/sizeof(long)]; + + if (n < 0) + goto out_nofds; + + /* max_fds can increase, so grab it once to avoid race */ + rcu_read_lock(); + fdt = files_fdtable(current->files); + max_fds = fdt->max_fds; + rcu_read_unlock(); + if (n > max_fds) + n = max_fds; + + /* + * We need 6 bitmaps (in/out/ex for both incoming and outgoing), + * since we used fdset we need to allocate memory in units of + * long-words. + */ + size = FDS_BYTES(n); + bits = stack_fds; + if (size > sizeof(stack_fds) / 6) { + bits = kmalloc_array(6, size, GFP_KERNEL); + ret = -ENOMEM; + if (!bits) + goto out_nofds; + } + fds.in = (unsigned long *) bits; + fds.out = (unsigned long *) (bits + size); + fds.ex = (unsigned long *) (bits + 2*size); + fds.res_in = (unsigned long *) (bits + 3*size); + fds.res_out = (unsigned long *) (bits + 4*size); + fds.res_ex = (unsigned long *) (bits + 5*size); + + if ((ret = compat_get_fd_set(n, inp, fds.in)) || + (ret = compat_get_fd_set(n, outp, fds.out)) || + (ret = compat_get_fd_set(n, exp, fds.ex))) + goto out; + zero_fd_set(n, fds.res_in); + zero_fd_set(n, fds.res_out); + zero_fd_set(n, fds.res_ex); + + ret = do_select(n, &fds, end_time); + + if (ret < 0) + goto out; + if (!ret) { + ret = -ERESTARTNOHAND; + if (signal_pending(current)) + goto out; + ret = 0; + } + + if (compat_set_fd_set(n, inp, fds.res_in) || + compat_set_fd_set(n, outp, fds.res_out) || + compat_set_fd_set(n, exp, fds.res_ex)) + ret = -EFAULT; +out: + if (bits != stack_fds) + kfree(bits); +out_nofds: + return ret; +} + +static int do_compat_select(int n, compat_ulong_t __user *inp, + compat_ulong_t __user *outp, compat_ulong_t __user *exp, + struct old_timeval32 __user *tvp) +{ + struct timespec64 end_time, *to = NULL; + struct old_timeval32 tv; + int ret; + + if (tvp) { + if (copy_from_user(&tv, tvp, sizeof(tv))) + return -EFAULT; + + to = &end_time; + if (poll_select_set_timeout(to, + tv.tv_sec + (tv.tv_usec / USEC_PER_SEC), + (tv.tv_usec % USEC_PER_SEC) * NSEC_PER_USEC)) + return -EINVAL; + } + + ret = compat_core_sys_select(n, inp, outp, exp, to); + return poll_select_finish(&end_time, tvp, PT_OLD_TIMEVAL, ret); +} + +COMPAT_SYSCALL_DEFINE5(select, int, n, compat_ulong_t __user *, inp, + compat_ulong_t __user *, outp, compat_ulong_t __user *, exp, + struct old_timeval32 __user *, tvp) +{ + return do_compat_select(n, inp, outp, exp, tvp); +} + +struct compat_sel_arg_struct { + compat_ulong_t n; + compat_uptr_t inp; + compat_uptr_t outp; + compat_uptr_t exp; + compat_uptr_t tvp; +}; + +COMPAT_SYSCALL_DEFINE1(old_select, struct compat_sel_arg_struct __user *, arg) +{ + struct compat_sel_arg_struct a; + + if (copy_from_user(&a, arg, sizeof(a))) + return -EFAULT; + return do_compat_select(a.n, compat_ptr(a.inp), compat_ptr(a.outp), + compat_ptr(a.exp), compat_ptr(a.tvp)); +} + +static long do_compat_pselect(int n, compat_ulong_t __user *inp, + compat_ulong_t __user *outp, compat_ulong_t __user *exp, + void __user *tsp, compat_sigset_t __user *sigmask, + compat_size_t sigsetsize, enum poll_time_type type) +{ + struct timespec64 ts, end_time, *to = NULL; + int ret; + + if (tsp) { + switch (type) { + case PT_OLD_TIMESPEC: + if (get_old_timespec32(&ts, tsp)) + return -EFAULT; + break; + case PT_TIMESPEC: + if (get_timespec64(&ts, tsp)) + return -EFAULT; + break; + default: + BUG(); + } + + to = &end_time; + if (poll_select_set_timeout(to, ts.tv_sec, ts.tv_nsec)) + return -EINVAL; + } + + ret = set_compat_user_sigmask(sigmask, sigsetsize); + if (ret) + return ret; + + ret = compat_core_sys_select(n, inp, outp, exp, to); + return poll_select_finish(&end_time, tsp, type, ret); +} + +struct compat_sigset_argpack { + compat_uptr_t p; + compat_size_t size; +}; +static inline int get_compat_sigset_argpack(struct compat_sigset_argpack *to, + struct compat_sigset_argpack __user *from) +{ + if (from) { + scoped_user_read_access(from, efault) { + unsafe_get_user(to->p, &from->p, efault); + unsafe_get_user(to->size, &from->size, efault); + } + } + return 0; +efault: + return -EFAULT; +} + +COMPAT_SYSCALL_DEFINE6(pselect6_time64, int, n, compat_ulong_t __user *, inp, + compat_ulong_t __user *, outp, compat_ulong_t __user *, exp, + struct __kernel_timespec __user *, tsp, void __user *, sig) +{ + struct compat_sigset_argpack x = {0, 0}; + + if (get_compat_sigset_argpack(&x, sig)) + return -EFAULT; + + return do_compat_pselect(n, inp, outp, exp, tsp, compat_ptr(x.p), + x.size, PT_TIMESPEC); +} + +#if defined(CONFIG_COMPAT_32BIT_TIME) + +COMPAT_SYSCALL_DEFINE6(pselect6_time32, int, n, compat_ulong_t __user *, inp, + compat_ulong_t __user *, outp, compat_ulong_t __user *, exp, + struct old_timespec32 __user *, tsp, void __user *, sig) +{ + struct compat_sigset_argpack x = {0, 0}; + + if (get_compat_sigset_argpack(&x, sig)) + return -EFAULT; + + return do_compat_pselect(n, inp, outp, exp, tsp, compat_ptr(x.p), + x.size, PT_OLD_TIMESPEC); +} + +#endif + +#if defined(CONFIG_COMPAT_32BIT_TIME) +COMPAT_SYSCALL_DEFINE5(ppoll_time32, struct pollfd __user *, ufds, + unsigned int, nfds, struct old_timespec32 __user *, tsp, + const compat_sigset_t __user *, sigmask, compat_size_t, sigsetsize) +{ + struct timespec64 ts, end_time, *to = NULL; + int ret; + + if (tsp) { + if (get_old_timespec32(&ts, tsp)) + return -EFAULT; + + to = &end_time; + if (poll_select_set_timeout(to, ts.tv_sec, ts.tv_nsec)) + return -EINVAL; + } + + ret = set_compat_user_sigmask(sigmask, sigsetsize); + if (ret) + return ret; + + ret = do_sys_poll(ufds, nfds, to); + return poll_select_finish(&end_time, tsp, PT_OLD_TIMESPEC, ret); +} +#endif + +/* New compat syscall for 64 bit time_t*/ +COMPAT_SYSCALL_DEFINE5(ppoll_time64, struct pollfd __user *, ufds, + unsigned int, nfds, struct __kernel_timespec __user *, tsp, + const compat_sigset_t __user *, sigmask, compat_size_t, sigsetsize) +{ + struct timespec64 ts, end_time, *to = NULL; + int ret; + + if (tsp) { + if (get_timespec64(&ts, tsp)) + return -EFAULT; + + to = &end_time; + if (poll_select_set_timeout(to, ts.tv_sec, ts.tv_nsec)) + return -EINVAL; + } + + ret = set_compat_user_sigmask(sigmask, sigsetsize); + if (ret) + return ret; + + ret = do_sys_poll(ufds, nfds, to); + return poll_select_finish(&end_time, tsp, PT_TIMESPEC, ret); +} + +#endif diff --git a/fs/seq_file.c b/fs/seq_file.c new file mode 100644 index 000000000..456c78719 --- /dev/null +++ b/fs/seq_file.c @@ -0,0 +1,1143 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/seq_file.c + * + * helper functions for making synthetic files from sequences of records. + * initial implementation -- AV, Oct 2001. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +static struct kmem_cache *seq_file_cache __ro_after_init; + +static void seq_set_overflow(struct seq_file *m) +{ + m->count = m->size; +} + +static void *seq_buf_alloc(unsigned long size) +{ + if (unlikely(size > MAX_RW_COUNT)) + return NULL; + + return kvmalloc(size, GFP_KERNEL_ACCOUNT); +} + +/** + * seq_open - initialize sequential file + * @file: file we initialize + * @op: method table describing the sequence + * + * seq_open() sets @file, associating it with a sequence described + * by @op. @op->start() sets the iterator up and returns the first + * element of sequence. @op->stop() shuts it down. @op->next() + * returns the next element of sequence. @op->show() prints element + * into the buffer. In case of error ->start() and ->next() return + * ERR_PTR(error). In the end of sequence they return %NULL. ->show() + * returns 0 in case of success and negative number in case of error. + * Returning SEQ_SKIP means "discard this element and move on". + * Note: seq_open() will allocate a struct seq_file and store its + * pointer in @file->private_data. This pointer should not be modified. + */ +int seq_open(struct file *file, const struct seq_operations *op) +{ + struct seq_file *p; + + WARN_ON(file->private_data); + + p = kmem_cache_zalloc(seq_file_cache, GFP_KERNEL); + if (!p) + return -ENOMEM; + + file->private_data = p; + + mutex_init(&p->lock); + p->op = op; + + // No refcounting: the lifetime of 'p' is constrained + // to the lifetime of the file. + p->file = file; + + /* + * seq_files support lseek() and pread(). They do not implement + * write() at all, but we clear FMODE_PWRITE here for historical + * reasons. + * + * If a client of seq_files a) implements file.write() and b) wishes to + * support pwrite() then that client will need to implement its own + * file.open() which calls seq_open() and then sets FMODE_PWRITE. + */ + file->f_mode &= ~FMODE_PWRITE; + return 0; +} +EXPORT_SYMBOL(seq_open); + +static int traverse(struct seq_file *m, loff_t offset) +{ + loff_t pos = 0; + int error = 0; + void *p; + + m->index = 0; + m->count = m->from = 0; + if (!offset) + return 0; + + if (!m->buf) { + m->buf = seq_buf_alloc(m->size = PAGE_SIZE); + if (!m->buf) + return -ENOMEM; + } + p = m->op->start(m, &m->index); + while (p) { + error = PTR_ERR(p); + if (IS_ERR(p)) + break; + error = m->op->show(m, p); + if (error < 0) + break; + if (unlikely(error)) { + error = 0; + m->count = 0; + } + if (seq_has_overflowed(m)) + goto Eoverflow; + p = m->op->next(m, p, &m->index); + if (pos + m->count > offset) { + m->from = offset - pos; + m->count -= m->from; + break; + } + pos += m->count; + m->count = 0; + if (pos == offset) + break; + } + m->op->stop(m, p); + return error; + +Eoverflow: + m->op->stop(m, p); + kvfree(m->buf); + m->count = 0; + m->buf = seq_buf_alloc(m->size <<= 1); + return !m->buf ? -ENOMEM : -EAGAIN; +} + +/** + * seq_read - ->read() method for sequential files. + * @file: the file to read from + * @buf: the buffer to read to + * @size: the maximum number of bytes to read + * @ppos: the current position in the file + * + * Ready-made ->f_op->read() + */ +ssize_t seq_read(struct file *file, char __user *buf, size_t size, loff_t *ppos) +{ + struct iovec iov = { .iov_base = buf, .iov_len = size}; + struct kiocb kiocb; + struct iov_iter iter; + ssize_t ret; + + init_sync_kiocb(&kiocb, file); + iov_iter_init(&iter, ITER_DEST, &iov, 1, size); + + kiocb.ki_pos = *ppos; + ret = seq_read_iter(&kiocb, &iter); + *ppos = kiocb.ki_pos; + return ret; +} +EXPORT_SYMBOL(seq_read); + +/* + * Ready-made ->f_op->read_iter() + */ +ssize_t seq_read_iter(struct kiocb *iocb, struct iov_iter *iter) +{ + struct seq_file *m = iocb->ki_filp->private_data; + size_t copied = 0; + size_t n; + void *p; + int err = 0; + + if (!iov_iter_count(iter)) + return 0; + + mutex_lock(&m->lock); + + /* + * if request is to read from zero offset, reset iterator to first + * record as it might have been already advanced by previous requests + */ + if (iocb->ki_pos == 0) { + m->index = 0; + m->count = 0; + } + + /* Don't assume ki_pos is where we left it */ + if (unlikely(iocb->ki_pos != m->read_pos)) { + while ((err = traverse(m, iocb->ki_pos)) == -EAGAIN) + ; + if (err) { + /* With prejudice... */ + m->read_pos = 0; + m->index = 0; + m->count = 0; + goto Done; + } else { + m->read_pos = iocb->ki_pos; + } + } + + /* grab buffer if we didn't have one */ + if (!m->buf) { + m->buf = seq_buf_alloc(m->size = PAGE_SIZE); + if (!m->buf) + goto Enomem; + } + // something left in the buffer - copy it out first + if (m->count) { + n = copy_to_iter(m->buf + m->from, m->count, iter); + m->count -= n; + m->from += n; + copied += n; + if (m->count) // hadn't managed to copy everything + goto Done; + } + // get a non-empty record in the buffer + m->from = 0; + p = m->op->start(m, &m->index); + while (1) { + err = PTR_ERR(p); + if (!p || IS_ERR(p)) // EOF or an error + break; + err = m->op->show(m, p); + if (err < 0) // hard error + break; + if (unlikely(err)) // ->show() says "skip it" + m->count = 0; + if (unlikely(!m->count)) { // empty record + p = m->op->next(m, p, &m->index); + continue; + } + if (!seq_has_overflowed(m)) // got it + goto Fill; + // need a bigger buffer + m->op->stop(m, p); + kvfree(m->buf); + m->count = 0; + m->buf = seq_buf_alloc(m->size <<= 1); + if (!m->buf) + goto Enomem; + p = m->op->start(m, &m->index); + } + // EOF or an error + m->op->stop(m, p); + m->count = 0; + goto Done; +Fill: + // one non-empty record is in the buffer; if they want more, + // try to fit more in, but in any case we need to advance + // the iterator once for every record shown. + while (1) { + size_t offs = m->count; + loff_t pos = m->index; + + p = m->op->next(m, p, &m->index); + if (pos == m->index) { + pr_info_ratelimited("buggy .next function %ps did not update position index\n", + m->op->next); + m->index++; + } + if (!p || IS_ERR(p)) // no next record for us + break; + if (m->count >= iov_iter_count(iter)) + break; + err = m->op->show(m, p); + if (err > 0) { // ->show() says "skip it" + m->count = offs; + } else if (err || seq_has_overflowed(m)) { + m->count = offs; + break; + } + } + m->op->stop(m, p); + n = copy_to_iter(m->buf, m->count, iter); + copied += n; + m->count -= n; + m->from = n; +Done: + if (unlikely(!copied)) { + copied = m->count ? -EFAULT : err; + } else { + iocb->ki_pos += copied; + m->read_pos += copied; + } + mutex_unlock(&m->lock); + return copied; +Enomem: + err = -ENOMEM; + goto Done; +} +EXPORT_SYMBOL(seq_read_iter); + +/** + * seq_lseek - ->llseek() method for sequential files. + * @file: the file in question + * @offset: new position + * @whence: 0 for absolute, 1 for relative position + * + * Ready-made ->f_op->llseek() + */ +loff_t seq_lseek(struct file *file, loff_t offset, int whence) +{ + struct seq_file *m = file->private_data; + loff_t retval = -EINVAL; + + mutex_lock(&m->lock); + switch (whence) { + case SEEK_CUR: + offset += file->f_pos; + fallthrough; + case SEEK_SET: + if (offset < 0) + break; + retval = offset; + if (offset != m->read_pos) { + while ((retval = traverse(m, offset)) == -EAGAIN) + ; + if (retval) { + /* with extreme prejudice... */ + file->f_pos = 0; + m->read_pos = 0; + m->index = 0; + m->count = 0; + } else { + m->read_pos = offset; + retval = file->f_pos = offset; + } + } else { + file->f_pos = offset; + } + } + mutex_unlock(&m->lock); + return retval; +} +EXPORT_SYMBOL(seq_lseek); + +/** + * seq_release - free the structures associated with sequential file. + * @inode: its inode + * @file: file in question + * + * Frees the structures associated with sequential file; can be used + * as ->f_op->release() if you don't have private data to destroy. + */ +int seq_release(struct inode *inode, struct file *file) +{ + struct seq_file *m = file->private_data; + kvfree(m->buf); + kmem_cache_free(seq_file_cache, m); + return 0; +} +EXPORT_SYMBOL(seq_release); + +/** + * seq_escape_mem - print data into buffer, escaping some characters + * @m: target buffer + * @src: source buffer + * @len: size of source buffer + * @flags: flags to pass to string_escape_mem() + * @esc: set of characters that need escaping + * + * Puts data into buffer, replacing each occurrence of character from + * given class (defined by @flags and @esc) with printable escaped sequence. + * + * Use seq_has_overflowed() to check for errors. + */ +void seq_escape_mem(struct seq_file *m, const char *src, size_t len, + unsigned int flags, const char *esc) +{ + char *buf; + size_t size = seq_get_buf(m, &buf); + int ret; + + ret = string_escape_mem(src, len, buf, size, flags, esc); + seq_commit(m, ret < size ? ret : -1); +} +EXPORT_SYMBOL(seq_escape_mem); + +void seq_vprintf(struct seq_file *m, const char *f, va_list args) +{ + int len; + + if (m->count < m->size) { + len = vsnprintf(m->buf + m->count, m->size - m->count, f, args); + if (m->count + len < m->size) { + m->count += len; + return; + } + } + seq_set_overflow(m); +} +EXPORT_SYMBOL(seq_vprintf); + +void seq_printf(struct seq_file *m, const char *f, ...) +{ + va_list args; + + va_start(args, f); + seq_vprintf(m, f, args); + va_end(args); +} +EXPORT_SYMBOL(seq_printf); + +#ifdef CONFIG_BINARY_PRINTF +void seq_bprintf(struct seq_file *m, const char *f, const u32 *binary) +{ + int len; + + if (m->count < m->size) { + len = bstr_printf(m->buf + m->count, m->size - m->count, f, + binary); + if (m->count + len < m->size) { + m->count += len; + return; + } + } + seq_set_overflow(m); +} +EXPORT_SYMBOL(seq_bprintf); +#endif /* CONFIG_BINARY_PRINTF */ + +/** + * seq_mangle_path - mangle and copy path to buffer beginning + * @s: buffer start + * @p: beginning of path in above buffer + * @esc: set of characters that need escaping + * + * Copy the path from @p to @s, replacing each occurrence of character from + * @esc with usual octal escape. + * Returns pointer past last written character in @s, or NULL in case of + * failure. + */ +char *seq_mangle_path(char *s, const char *p, const char *esc) +{ + while (s <= p) { + char c = *p++; + if (!c) { + return s; + } else if (!strchr(esc, c)) { + *s++ = c; + } else if (s + 4 > p) { + break; + } else { + *s++ = '\\'; + *s++ = '0' + ((c & 0300) >> 6); + *s++ = '0' + ((c & 070) >> 3); + *s++ = '0' + (c & 07); + } + } + return NULL; +} + +/** + * seq_path - seq_file interface to print a pathname + * @m: the seq_file handle + * @path: the struct path to print + * @esc: set of characters to escape in the output + * + * return the absolute path of 'path', as represented by the + * dentry / mnt pair in the path parameter. + */ +int seq_path(struct seq_file *m, const struct path *path, const char *esc) +{ + char *buf; + size_t size = seq_get_buf(m, &buf); + int res = -1; + + if (size) { + char *p = d_path(path, buf, size); + if (!IS_ERR(p)) { + char *end = seq_mangle_path(buf, p, esc); + if (end) + res = end - buf; + } + } + seq_commit(m, res); + + return res; +} +EXPORT_SYMBOL(seq_path); + +/** + * seq_file_path - seq_file interface to print a pathname of a file + * @m: the seq_file handle + * @file: the struct file to print + * @esc: set of characters to escape in the output + * + * return the absolute path to the file. + */ +int seq_file_path(struct seq_file *m, struct file *file, const char *esc) +{ + return seq_path(m, &file->f_path, esc); +} +EXPORT_SYMBOL(seq_file_path); + +/* + * Same as seq_path, but relative to supplied root. + */ +int seq_path_root(struct seq_file *m, const struct path *path, + const struct path *root, const char *esc) +{ + char *buf; + size_t size = seq_get_buf(m, &buf); + int res = -ENAMETOOLONG; + + if (size) { + char *p; + + p = __d_path(path, root, buf, size); + if (!p) + return SEQ_SKIP; + res = PTR_ERR(p); + if (!IS_ERR(p)) { + char *end = seq_mangle_path(buf, p, esc); + if (end) + res = end - buf; + else + res = -ENAMETOOLONG; + } + } + seq_commit(m, res); + + return res < 0 && res != -ENAMETOOLONG ? res : 0; +} + +/* + * returns the path of the 'dentry' from the root of its filesystem. + */ +int seq_dentry(struct seq_file *m, struct dentry *dentry, const char *esc) +{ + char *buf; + size_t size = seq_get_buf(m, &buf); + int res = -1; + + if (size) { + char *p = dentry_path(dentry, buf, size); + if (!IS_ERR(p)) { + char *end = seq_mangle_path(buf, p, esc); + if (end) + res = end - buf; + } + } + seq_commit(m, res); + + return res; +} +EXPORT_SYMBOL(seq_dentry); + +void *single_start(struct seq_file *p, loff_t *pos) +{ + return *pos ? NULL : SEQ_START_TOKEN; +} + +static void *single_next(struct seq_file *p, void *v, loff_t *pos) +{ + ++*pos; + return NULL; +} + +static void single_stop(struct seq_file *p, void *v) +{ +} + +int single_open(struct file *file, int (*show)(struct seq_file *, void *), + void *data) +{ + struct seq_operations *op = kmalloc_obj(*op, GFP_KERNEL_ACCOUNT); + int res = -ENOMEM; + + if (op) { + op->start = single_start; + op->next = single_next; + op->stop = single_stop; + op->show = show; + res = seq_open(file, op); + if (!res) + ((struct seq_file *)file->private_data)->private = data; + else + kfree(op); + } + return res; +} +EXPORT_SYMBOL(single_open); + +int single_open_size(struct file *file, int (*show)(struct seq_file *, void *), + void *data, size_t size) +{ + char *buf = seq_buf_alloc(size); + int ret; + if (!buf) + return -ENOMEM; + ret = single_open(file, show, data); + if (ret) { + kvfree(buf); + return ret; + } + ((struct seq_file *)file->private_data)->buf = buf; + ((struct seq_file *)file->private_data)->size = size; + return 0; +} +EXPORT_SYMBOL(single_open_size); + +int single_release(struct inode *inode, struct file *file) +{ + const struct seq_operations *op = ((struct seq_file *)file->private_data)->op; + int res = seq_release(inode, file); + kfree(op); + return res; +} +EXPORT_SYMBOL(single_release); + +int seq_release_private(struct inode *inode, struct file *file) +{ + struct seq_file *seq = file->private_data; + + kfree(seq->private); + seq->private = NULL; + return seq_release(inode, file); +} +EXPORT_SYMBOL(seq_release_private); + +void *__seq_open_private(struct file *f, const struct seq_operations *ops, + int psize) +{ + int rc; + void *private; + struct seq_file *seq; + + private = kzalloc(psize, GFP_KERNEL_ACCOUNT); + if (private == NULL) + goto out; + + rc = seq_open(f, ops); + if (rc < 0) + goto out_free; + + seq = f->private_data; + seq->private = private; + return private; + +out_free: + kfree(private); +out: + return NULL; +} +EXPORT_SYMBOL(__seq_open_private); + +int seq_open_private(struct file *filp, const struct seq_operations *ops, + int psize) +{ + return __seq_open_private(filp, ops, psize) ? 0 : -ENOMEM; +} +EXPORT_SYMBOL(seq_open_private); + +void seq_putc(struct seq_file *m, char c) +{ + if (m->count >= m->size) + return; + + m->buf[m->count++] = c; +} +EXPORT_SYMBOL(seq_putc); + +void __seq_puts(struct seq_file *m, const char *s) +{ + seq_write(m, s, strlen(s)); +} +EXPORT_SYMBOL(__seq_puts); + +/** + * seq_put_decimal_ull_width - A helper routine for putting decimal numbers + * without rich format of printf(). + * only 'unsigned long long' is supported. + * @m: seq_file identifying the buffer to which data should be written + * @delimiter: a string which is printed before the number + * @num: the number + * @width: a minimum field width + * + * This routine will put strlen(delimiter) + number into seq_filed. + * This routine is very quick when you show lots of numbers. + * In usual cases, it will be better to use seq_printf(). It's easier to read. + */ +void seq_put_decimal_ull_width(struct seq_file *m, const char *delimiter, + unsigned long long num, unsigned int width) +{ + int len; + + if (m->count + 2 >= m->size) /* we'll write 2 bytes at least */ + goto overflow; + + if (delimiter && delimiter[0]) { + if (delimiter[1] == 0) + seq_putc(m, delimiter[0]); + else + seq_puts(m, delimiter); + } + + if (!width) + width = 1; + + if (m->count + width >= m->size) + goto overflow; + + len = num_to_str(m->buf + m->count, m->size - m->count, num, width); + if (!len) + goto overflow; + + m->count += len; + return; + +overflow: + seq_set_overflow(m); +} + +void seq_put_decimal_ull(struct seq_file *m, const char *delimiter, + unsigned long long num) +{ + return seq_put_decimal_ull_width(m, delimiter, num, 0); +} +EXPORT_SYMBOL(seq_put_decimal_ull); + +/** + * seq_put_hex_ll - put a number in hexadecimal notation + * @m: seq_file identifying the buffer to which data should be written + * @delimiter: a string which is printed before the number + * @v: the number + * @width: a minimum field width + * + * seq_put_hex_ll(m, "", v, 8) is equal to seq_printf(m, "%08llx", v) + * + * This routine is very quick when you show lots of numbers. + * In usual cases, it will be better to use seq_printf(). It's easier to read. + */ +void seq_put_hex_ll(struct seq_file *m, const char *delimiter, + unsigned long long v, unsigned int width) +{ + unsigned int len; + int i; + + if (delimiter && delimiter[0]) { + if (delimiter[1] == 0) + seq_putc(m, delimiter[0]); + else + seq_puts(m, delimiter); + } + + /* If x is 0, the result of __builtin_clzll is undefined */ + if (v == 0) + len = 1; + else + len = (sizeof(v) * 8 - __builtin_clzll(v) + 3) / 4; + + if (len < width) + len = width; + + if (m->count + len > m->size) { + seq_set_overflow(m); + return; + } + + for (i = len - 1; i >= 0; i--) { + m->buf[m->count + i] = hex_asc[0xf & v]; + v = v >> 4; + } + m->count += len; +} + +void seq_put_decimal_ll(struct seq_file *m, const char *delimiter, long long num) +{ + int len; + + if (m->count + 3 >= m->size) /* we'll write 2 bytes at least */ + goto overflow; + + if (delimiter && delimiter[0]) { + if (delimiter[1] == 0) + seq_putc(m, delimiter[0]); + else + seq_puts(m, delimiter); + } + + if (m->count + 2 >= m->size) + goto overflow; + + if (num < 0) { + m->buf[m->count++] = '-'; + num = -num; + } + + if (num < 10) { + m->buf[m->count++] = num + '0'; + return; + } + + len = num_to_str(m->buf + m->count, m->size - m->count, num, 0); + if (!len) + goto overflow; + + m->count += len; + return; + +overflow: + seq_set_overflow(m); +} +EXPORT_SYMBOL(seq_put_decimal_ll); + +/** + * seq_write - write arbitrary data to buffer + * @seq: seq_file identifying the buffer to which data should be written + * @data: data address + * @len: number of bytes + * + * Return 0 on success, non-zero otherwise. + */ +int seq_write(struct seq_file *seq, const void *data, size_t len) +{ + if (seq->count + len < seq->size) { + memcpy(seq->buf + seq->count, data, len); + seq->count += len; + return 0; + } + seq_set_overflow(seq); + return -1; +} +EXPORT_SYMBOL(seq_write); + +/** + * seq_pad - write padding spaces to buffer + * @m: seq_file identifying the buffer to which data should be written + * @c: the byte to append after padding if non-zero + */ +void seq_pad(struct seq_file *m, char c) +{ + int size = m->pad_until - m->count; + if (size > 0) { + if (size + m->count > m->size) { + seq_set_overflow(m); + return; + } + memset(m->buf + m->count, ' ', size); + m->count += size; + } + if (c) + seq_putc(m, c); +} +EXPORT_SYMBOL(seq_pad); + +/* A complete analogue of print_hex_dump() */ +void seq_hex_dump(struct seq_file *m, const char *prefix_str, int prefix_type, + int rowsize, int groupsize, const void *buf, size_t len, + bool ascii) +{ + const u8 *ptr = buf; + int i, linelen, remaining = len; + char *buffer; + size_t size; + int ret; + + if (rowsize != 16 && rowsize != 32) + rowsize = 16; + + for (i = 0; i < len && !seq_has_overflowed(m); i += rowsize) { + linelen = min(remaining, rowsize); + remaining -= rowsize; + + switch (prefix_type) { + case DUMP_PREFIX_ADDRESS: + seq_printf(m, "%s%p: ", prefix_str, ptr + i); + break; + case DUMP_PREFIX_OFFSET: + seq_printf(m, "%s%.8x: ", prefix_str, i); + break; + default: + seq_printf(m, "%s", prefix_str); + break; + } + + size = seq_get_buf(m, &buffer); + ret = hex_dump_to_buffer(ptr + i, linelen, rowsize, groupsize, + buffer, size, ascii); + seq_commit(m, ret < size ? ret : -1); + + seq_putc(m, '\n'); + } +} +EXPORT_SYMBOL(seq_hex_dump); + +struct list_head *seq_list_start(struct list_head *head, loff_t pos) +{ + struct list_head *lh; + + list_for_each(lh, head) + if (pos-- == 0) + return lh; + + return NULL; +} +EXPORT_SYMBOL(seq_list_start); + +struct list_head *seq_list_start_head(struct list_head *head, loff_t pos) +{ + if (!pos) + return head; + + return seq_list_start(head, pos - 1); +} +EXPORT_SYMBOL(seq_list_start_head); + +struct list_head *seq_list_next(void *v, struct list_head *head, loff_t *ppos) +{ + struct list_head *lh; + + lh = ((struct list_head *)v)->next; + ++*ppos; + return lh == head ? NULL : lh; +} +EXPORT_SYMBOL(seq_list_next); + +struct list_head *seq_list_start_rcu(struct list_head *head, loff_t pos) +{ + struct list_head *lh; + + list_for_each_rcu(lh, head) + if (pos-- == 0) + return lh; + + return NULL; +} +EXPORT_SYMBOL(seq_list_start_rcu); + +struct list_head *seq_list_start_head_rcu(struct list_head *head, loff_t pos) +{ + if (!pos) + return head; + + return seq_list_start_rcu(head, pos - 1); +} +EXPORT_SYMBOL(seq_list_start_head_rcu); + +struct list_head *seq_list_next_rcu(void *v, struct list_head *head, + loff_t *ppos) +{ + struct list_head *lh; + + lh = list_next_rcu((struct list_head *)v); + ++*ppos; + return lh == head ? NULL : lh; +} +EXPORT_SYMBOL(seq_list_next_rcu); + +/** + * seq_hlist_start - start an iteration of a hlist + * @head: the head of the hlist + * @pos: the start position of the sequence + * + * Called at seq_file->op->start(). + */ +struct hlist_node *seq_hlist_start(struct hlist_head *head, loff_t pos) +{ + struct hlist_node *node; + + hlist_for_each(node, head) + if (pos-- == 0) + return node; + return NULL; +} +EXPORT_SYMBOL(seq_hlist_start); + +/** + * seq_hlist_start_head - start an iteration of a hlist + * @head: the head of the hlist + * @pos: the start position of the sequence + * + * Called at seq_file->op->start(). Call this function if you want to + * print a header at the top of the output. + */ +struct hlist_node *seq_hlist_start_head(struct hlist_head *head, loff_t pos) +{ + if (!pos) + return SEQ_START_TOKEN; + + return seq_hlist_start(head, pos - 1); +} +EXPORT_SYMBOL(seq_hlist_start_head); + +/** + * seq_hlist_next - move to the next position of the hlist + * @v: the current iterator + * @head: the head of the hlist + * @ppos: the current position + * + * Called at seq_file->op->next(). + */ +struct hlist_node *seq_hlist_next(void *v, struct hlist_head *head, + loff_t *ppos) +{ + struct hlist_node *node = v; + + ++*ppos; + if (v == SEQ_START_TOKEN) + return head->first; + else + return node->next; +} +EXPORT_SYMBOL(seq_hlist_next); + +/** + * seq_hlist_start_rcu - start an iteration of a hlist protected by RCU + * @head: the head of the hlist + * @pos: the start position of the sequence + * + * Called at seq_file->op->start(). + * + * This list-traversal primitive may safely run concurrently with + * the _rcu list-mutation primitives such as hlist_add_head_rcu() + * as long as the traversal is guarded by rcu_read_lock(). + */ +struct hlist_node *seq_hlist_start_rcu(struct hlist_head *head, + loff_t pos) +{ + struct hlist_node *node; + + __hlist_for_each_rcu(node, head) + if (pos-- == 0) + return node; + return NULL; +} +EXPORT_SYMBOL(seq_hlist_start_rcu); + +/** + * seq_hlist_start_head_rcu - start an iteration of a hlist protected by RCU + * @head: the head of the hlist + * @pos: the start position of the sequence + * + * Called at seq_file->op->start(). Call this function if you want to + * print a header at the top of the output. + * + * This list-traversal primitive may safely run concurrently with + * the _rcu list-mutation primitives such as hlist_add_head_rcu() + * as long as the traversal is guarded by rcu_read_lock(). + */ +struct hlist_node *seq_hlist_start_head_rcu(struct hlist_head *head, + loff_t pos) +{ + if (!pos) + return SEQ_START_TOKEN; + + return seq_hlist_start_rcu(head, pos - 1); +} +EXPORT_SYMBOL(seq_hlist_start_head_rcu); + +/** + * seq_hlist_next_rcu - move to the next position of the hlist protected by RCU + * @v: the current iterator + * @head: the head of the hlist + * @ppos: the current position + * + * Called at seq_file->op->next(). + * + * This list-traversal primitive may safely run concurrently with + * the _rcu list-mutation primitives such as hlist_add_head_rcu() + * as long as the traversal is guarded by rcu_read_lock(). + */ +struct hlist_node *seq_hlist_next_rcu(void *v, + struct hlist_head *head, + loff_t *ppos) +{ + struct hlist_node *node = v; + + ++*ppos; + if (v == SEQ_START_TOKEN) + return rcu_dereference(head->first); + else + return rcu_dereference(node->next); +} +EXPORT_SYMBOL(seq_hlist_next_rcu); + +/** + * seq_hlist_start_percpu - start an iteration of a percpu hlist array + * @head: pointer to percpu array of struct hlist_heads + * @cpu: pointer to cpu "cursor" + * @pos: start position of sequence + * + * Called at seq_file->op->start(). + */ +struct hlist_node * +seq_hlist_start_percpu(struct hlist_head __percpu *head, int *cpu, loff_t pos) +{ + struct hlist_node *node; + + for_each_possible_cpu(*cpu) { + hlist_for_each(node, per_cpu_ptr(head, *cpu)) { + if (pos-- == 0) + return node; + } + } + return NULL; +} +EXPORT_SYMBOL(seq_hlist_start_percpu); + +/** + * seq_hlist_next_percpu - move to the next position of the percpu hlist array + * @v: pointer to current hlist_node + * @head: pointer to percpu array of struct hlist_heads + * @cpu: pointer to cpu "cursor" + * @pos: start position of sequence + * + * Called at seq_file->op->next(). + */ +struct hlist_node * +seq_hlist_next_percpu(void *v, struct hlist_head __percpu *head, + int *cpu, loff_t *pos) +{ + struct hlist_node *node = v; + + ++*pos; + + if (node->next) + return node->next; + + for (*cpu = cpumask_next(*cpu, cpu_possible_mask); *cpu < nr_cpu_ids; + *cpu = cpumask_next(*cpu, cpu_possible_mask)) { + struct hlist_head *bucket = per_cpu_ptr(head, *cpu); + + if (!hlist_empty(bucket)) + return bucket->first; + } + return NULL; +} +EXPORT_SYMBOL(seq_hlist_next_percpu); + +void __init seq_file_init(void) +{ + seq_file_cache = KMEM_CACHE(seq_file, SLAB_ACCOUNT|SLAB_PANIC); +} diff --git a/fs/signalfd.c b/fs/signalfd.c new file mode 100644 index 000000000..dff53745e --- /dev/null +++ b/fs/signalfd.c @@ -0,0 +1,351 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/signalfd.c + * + * Copyright (C) 2003 Linus Torvalds + * + * Mon Mar 5, 2007: Davide Libenzi + * Changed ->read() to return a siginfo strcture instead of signal number. + * Fixed locking in ->poll(). + * Added sighand-detach notification. + * Added fd re-use in sys_signalfd() syscall. + * Now using anonymous inode source. + * Thanks to Oleg Nesterov for useful code review and suggestions. + * More comments and suggestions from Arnd Bergmann. + * Sat May 19, 2007: Davi E. M. Arnaut + * Retrieve multiple signals with one read() call + * Sun Jul 15, 2007: Davide Libenzi + * Attach to the sighand only during read() and poll(). + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +void signalfd_cleanup(struct sighand_struct *sighand) +{ + wake_up_pollfree(&sighand->signalfd_wqh); +} + +struct signalfd_ctx { + sigset_t sigmask; +}; + +static int signalfd_release(struct inode *inode, struct file *file) +{ + kfree(file->private_data); + return 0; +} + +static __poll_t signalfd_poll(struct file *file, poll_table *wait) +{ + struct signalfd_ctx *ctx = file->private_data; + __poll_t events = 0; + + poll_wait(file, ¤t->sighand->signalfd_wqh, wait); + + spin_lock_irq(¤t->sighand->siglock); + if (next_signal(¤t->pending, &ctx->sigmask) || + next_signal(¤t->signal->shared_pending, + &ctx->sigmask)) + events |= EPOLLIN; + spin_unlock_irq(¤t->sighand->siglock); + + return events; +} + +/* + * Copied from copy_siginfo_to_user() in kernel/signal.c + */ +static int signalfd_copyinfo(struct iov_iter *to, kernel_siginfo_t const *kinfo) +{ + struct signalfd_siginfo new; + + BUILD_BUG_ON(sizeof(struct signalfd_siginfo) != 128); + + /* + * Unused members should be zero ... + */ + memset(&new, 0, sizeof(new)); + + /* + * If you change siginfo_t structure, please be sure + * this code is fixed accordingly. + */ + new.ssi_signo = kinfo->si_signo; + new.ssi_errno = kinfo->si_errno; + new.ssi_code = kinfo->si_code; + switch (siginfo_layout(kinfo->si_signo, kinfo->si_code)) { + case SIL_KILL: + new.ssi_pid = kinfo->si_pid; + new.ssi_uid = kinfo->si_uid; + break; + case SIL_TIMER: + new.ssi_tid = kinfo->si_tid; + new.ssi_overrun = kinfo->si_overrun; + new.ssi_ptr = (long) kinfo->si_ptr; + new.ssi_int = kinfo->si_int; + break; + case SIL_POLL: + new.ssi_band = kinfo->si_band; + new.ssi_fd = kinfo->si_fd; + break; + case SIL_FAULT_BNDERR: + case SIL_FAULT_PKUERR: + case SIL_FAULT_PERF_EVENT: + /* + * Fall through to the SIL_FAULT case. SIL_FAULT_BNDERR, + * SIL_FAULT_PKUERR, and SIL_FAULT_PERF_EVENT are only + * generated by faults that deliver them synchronously to + * userspace. In case someone injects one of these signals + * and signalfd catches it treat it as SIL_FAULT. + */ + case SIL_FAULT: + new.ssi_addr = (long) kinfo->si_addr; + break; + case SIL_FAULT_TRAPNO: + new.ssi_addr = (long) kinfo->si_addr; + new.ssi_trapno = kinfo->si_trapno; + break; + case SIL_FAULT_MCEERR: + new.ssi_addr = (long) kinfo->si_addr; + new.ssi_addr_lsb = (short) kinfo->si_addr_lsb; + break; + case SIL_CHLD: + new.ssi_pid = kinfo->si_pid; + new.ssi_uid = kinfo->si_uid; + new.ssi_status = kinfo->si_status; + new.ssi_utime = kinfo->si_utime; + new.ssi_stime = kinfo->si_stime; + break; + case SIL_RT: + /* + * This case catches also the signals queued by sigqueue(). + */ + new.ssi_pid = kinfo->si_pid; + new.ssi_uid = kinfo->si_uid; + new.ssi_ptr = (long) kinfo->si_ptr; + new.ssi_int = kinfo->si_int; + break; + case SIL_SYS: + new.ssi_call_addr = (long) kinfo->si_call_addr; + new.ssi_syscall = kinfo->si_syscall; + new.ssi_arch = kinfo->si_arch; + break; + } + + if (!copy_to_iter_full(&new, sizeof(struct signalfd_siginfo), to)) + return -EFAULT; + + return sizeof(struct signalfd_siginfo); +} + +static ssize_t signalfd_dequeue(struct signalfd_ctx *ctx, kernel_siginfo_t *info, + int nonblock) +{ + enum pid_type type; + ssize_t ret; + DECLARE_WAITQUEUE(wait, current); + + spin_lock_irq(¤t->sighand->siglock); + ret = dequeue_signal(&ctx->sigmask, info, &type); + switch (ret) { + case 0: + if (!nonblock) + break; + ret = -EAGAIN; + fallthrough; + default: + spin_unlock_irq(¤t->sighand->siglock); + return ret; + } + + add_wait_queue(¤t->sighand->signalfd_wqh, &wait); + for (;;) { + set_current_state(TASK_INTERRUPTIBLE); + ret = dequeue_signal(&ctx->sigmask, info, &type); + if (ret != 0) + break; + if (signal_pending(current)) { + ret = -ERESTARTSYS; + break; + } + spin_unlock_irq(¤t->sighand->siglock); + schedule(); + spin_lock_irq(¤t->sighand->siglock); + } + spin_unlock_irq(¤t->sighand->siglock); + + remove_wait_queue(¤t->sighand->signalfd_wqh, &wait); + __set_current_state(TASK_RUNNING); + + return ret; +} + +/* + * Returns a multiple of the size of a "struct signalfd_siginfo", or a negative + * error code. The "count" parameter must be at least the size of a + * "struct signalfd_siginfo". + */ +static ssize_t signalfd_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct file *file = iocb->ki_filp; + struct signalfd_ctx *ctx = file->private_data; + size_t count = iov_iter_count(to); + ssize_t ret, total = 0; + kernel_siginfo_t info; + bool nonblock; + + count /= sizeof(struct signalfd_siginfo); + if (!count) + return -EINVAL; + + nonblock = file->f_flags & O_NONBLOCK || iocb->ki_flags & IOCB_NOWAIT; + do { + ret = signalfd_dequeue(ctx, &info, nonblock); + if (unlikely(ret <= 0)) + break; + ret = signalfd_copyinfo(to, &info); + if (ret < 0) + break; + total += ret; + nonblock = 1; + } while (--count); + + return total ? total: ret; +} + +#ifdef CONFIG_PROC_FS +static void signalfd_show_fdinfo(struct seq_file *m, struct file *f) +{ + struct signalfd_ctx *ctx = f->private_data; + sigset_t sigmask; + + sigmask = ctx->sigmask; + signotset(&sigmask); + render_sigset_t(m, "sigmask:\t", &sigmask); +} +#endif + +static const struct file_operations signalfd_fops = { +#ifdef CONFIG_PROC_FS + .show_fdinfo = signalfd_show_fdinfo, +#endif + .release = signalfd_release, + .poll = signalfd_poll, + .read_iter = signalfd_read_iter, + .llseek = noop_llseek, +}; + +static int do_signalfd4(int ufd, sigset_t *mask, int flags) +{ + /* Check the SFD_* constants for consistency. */ + BUILD_BUG_ON(SFD_CLOEXEC != O_CLOEXEC); + BUILD_BUG_ON(SFD_NONBLOCK != O_NONBLOCK); + + if (flags & ~(SFD_CLOEXEC | SFD_NONBLOCK)) + return -EINVAL; + + sigdelsetmask(mask, sigmask(SIGKILL) | sigmask(SIGSTOP)); + signotset(mask); + + if (ufd == -1) { + int fd; + struct signalfd_ctx *ctx __free(kfree) = NULL; + + ctx = kmalloc_obj(*ctx); + if (!ctx) + return -ENOMEM; + + ctx->sigmask = *mask; + + fd = FD_ADD(flags & O_CLOEXEC, + anon_inode_getfile_fmode( + "[signalfd]", &signalfd_fops, ctx, + O_RDWR | (flags & O_NONBLOCK), FMODE_NOWAIT)); + if (fd >= 0) + retain_and_null_ptr(ctx); + return fd; + } else { + struct signalfd_ctx *ctx; + + CLASS(fd, f)(ufd); + if (fd_empty(f)) + return -EBADF; + ctx = fd_file(f)->private_data; + if (fd_file(f)->f_op != &signalfd_fops) + return -EINVAL; + spin_lock_irq(¤t->sighand->siglock); + ctx->sigmask = *mask; + spin_unlock_irq(¤t->sighand->siglock); + + wake_up(¤t->sighand->signalfd_wqh); + } + + return ufd; +} + +SYSCALL_DEFINE4(signalfd4, int, ufd, sigset_t __user *, user_mask, + size_t, sizemask, int, flags) +{ + sigset_t mask; + + if (sizemask != sizeof(sigset_t)) + return -EINVAL; + if (copy_from_user(&mask, user_mask, sizeof(mask))) + return -EFAULT; + return do_signalfd4(ufd, &mask, flags); +} + +SYSCALL_DEFINE3(signalfd, int, ufd, sigset_t __user *, user_mask, + size_t, sizemask) +{ + sigset_t mask; + + if (sizemask != sizeof(sigset_t)) + return -EINVAL; + if (copy_from_user(&mask, user_mask, sizeof(mask))) + return -EFAULT; + return do_signalfd4(ufd, &mask, 0); +} + +#ifdef CONFIG_COMPAT +static long do_compat_signalfd4(int ufd, + const compat_sigset_t __user *user_mask, + compat_size_t sigsetsize, int flags) +{ + sigset_t mask; + + if (sigsetsize != sizeof(compat_sigset_t)) + return -EINVAL; + if (get_compat_sigset(&mask, user_mask)) + return -EFAULT; + return do_signalfd4(ufd, &mask, flags); +} + +COMPAT_SYSCALL_DEFINE4(signalfd4, int, ufd, + const compat_sigset_t __user *, user_mask, + compat_size_t, sigsetsize, + int, flags) +{ + return do_compat_signalfd4(ufd, user_mask, sigsetsize, flags); +} + +COMPAT_SYSCALL_DEFINE3(signalfd, int, ufd, + const compat_sigset_t __user *, user_mask, + compat_size_t, sigsetsize) +{ + return do_compat_signalfd4(ufd, user_mask, sigsetsize, 0); +} +#endif diff --git a/fs/smb/Kconfig b/fs/smb/Kconfig new file mode 100644 index 000000000..e549e189e --- /dev/null +++ b/fs/smb/Kconfig @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# smbfs configuration + +source "fs/smb/client/Kconfig" +source "fs/smb/server/Kconfig" +source "fs/smb/smbdirect/Kconfig" + +config SMBFS + tristate + default y if CIFS=y || SMB_SERVER=y + default m if CIFS=m || SMB_SERVER=m + +config SMB_KUNIT_TESTS + tristate "KUnit tests for SMB" if !KUNIT_ALL_TESTS + depends on SMBFS && KUNIT + default KUNIT_ALL_TESTS + help + This builds the SMB KUnit tests. + + KUnit tests run during boot and output the results to the debug log + in TAP format (https://testanything.org/). Only useful for kernel devs + running KUnit test harness and are not for inclusion into a production + build. + + For more information on KUnit and unit tests in general please refer + to the KUnit documentation in Documentation/dev-tools/kunit/. + + If unsure, say N. diff --git a/fs/smb/Makefile b/fs/smb/Makefile new file mode 100644 index 000000000..353b1c2ee --- /dev/null +++ b/fs/smb/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_SMBFS) += common/ +obj-$(CONFIG_SMBDIRECT) += smbdirect/ +obj-$(CONFIG_CIFS) += client/ +obj-$(CONFIG_SMB_SERVER) += server/ diff --git a/fs/smb/client/.gitignore b/fs/smb/client/.gitignore new file mode 100644 index 000000000..66e6e2ade --- /dev/null +++ b/fs/smb/client/.gitignore @@ -0,0 +1,4 @@ +smb1_mapping_table.c +smb1_err_dos_map.c +smb1_err_srv_map.c +smb2_mapping_table.c diff --git a/fs/smb/client/Kconfig b/fs/smb/client/Kconfig new file mode 100644 index 000000000..2b7db5fb0 --- /dev/null +++ b/fs/smb/client/Kconfig @@ -0,0 +1,232 @@ +# SPDX-License-Identifier: GPL-2.0-only +config CIFS + tristate "SMB3 and CIFS support (advanced network filesystem)" + depends on INET + select NLS + select NLS_UCS2_UTILS + select CRYPTO + select CRYPTO_AEAD2 + select CRYPTO_CCM + select CRYPTO_GCM + select CRYPTO_AES + select CRYPTO_LIB_AES_CBC_MACS + select CRYPTO_LIB_ARC4 + select CRYPTO_LIB_MD5 + select CRYPTO_LIB_SHA256 + select CRYPTO_LIB_SHA512 + select KEYS + select DNS_RESOLVER + select ASN1 + select OID_REGISTRY + select NETFS_SUPPORT + help + This is the client VFS module for the SMB3 family of network file + protocols (including the most recent, most secure dialect SMB3.1.1). + This module also includes support for earlier dialects such as + SMB2.1, SMB2 and even the old Common Internet File System (CIFS) + protocol. CIFS was the successor to the original network filesystem + protocol, Server Message Block (SMB ie SMB1), the native file sharing + mechanism for most early PC operating systems. + + The SMB3.1.1 protocol is supported by most modern operating systems + and NAS appliances (e.g. Samba, Windows 11, Windows Server 2022, + MacOS) and even in the cloud (e.g. Microsoft Azure) and also by the + Linux kernel server, ksmbd. Support for the older CIFS protocol was + included in Windows NT4, 2000 and XP (and later). Use of dialects + older than SMB2.1 is often discouraged on public networks. + This module also provides limited support for OS/2 and Windows ME + and similar very old servers. + + This module provides an advanced network file system client for + mounting to SMB3 (and CIFS) compliant servers. It includes support + for DFS (hierarchical name space), secure per-user session + establishment via Kerberos or NTLMv2, RDMA (smbdirect), advanced + security features, per-share encryption, packet-signing, snapshots, + directory leases, safe distributed caching (leases), multichannel, + Unicode and other internationalization improvements. + + In general, the default dialects, SMB3 and later, enable better + performance, security and features, than would be possible with CIFS. + + If you need to mount to Samba, Azure, ksmbd, Macs or Windows from this + machine, say Y. + +config CIFS_STATS2 + bool "Extended statistics" + depends on CIFS + default y + help + Enabling this option will allow more detailed statistics on SMB + request timing to be displayed in /proc/fs/cifs/DebugData and also + allow optional logging of slow responses to dmesg (depending on the + value of /proc/fs/cifs/cifsFYI). See Documentation/admin-guide/cifs/usage.rst + for more details. These additional statistics may have a minor effect + on performance and memory utilization. + + If unsure, say Y. + +config CIFS_ALLOW_INSECURE_LEGACY + bool "Support legacy servers which use less secure dialects" + depends on CIFS + default y + help + Modern dialects, SMB2.1 and later (including SMB3 and 3.1.1), have + additional security features, including protection against + man-in-the-middle attacks and stronger crypto hashes, so the use + of legacy dialects (SMB1/CIFS and SMB2.0) is discouraged. + + Disabling this option prevents users from using vers=1.0 or vers=2.0 + on mounts with cifs.ko + + If unsure, say Y. + +config CIFS_UPCALL + bool "Kerberos/SPNEGO advanced session setup" + depends on CIFS + help + Enables an upcall mechanism for CIFS which accesses userspace helper + utilities to provide SPNEGO packaged (RFC 4178) Kerberos tickets + which are needed to mount to certain secure servers (for which more + secure Kerberos authentication is required). If unsure, say Y. + +config CIFS_XATTR + bool "CIFS extended attributes" + depends on CIFS + help + Extended attributes are name:value pairs associated with inodes by + the kernel or by users (see the attr(5) manual page for details). + CIFS maps the name of extended attributes beginning with the user + namespace prefix to SMB/CIFS EAs. EAs are stored on Windows + servers without the user namespace prefix, but their names are + seen by Linux cifs clients prefaced by the user namespace prefix. + The system namespace (used by some filesystems to store ACLs) is + not supported at this time. + + If unsure, say Y. + +config CIFS_POSIX + bool "CIFS POSIX Extensions" + depends on CIFS && CIFS_ALLOW_INSECURE_LEGACY && CIFS_XATTR + help + Enabling this option will cause the cifs client to attempt to + negotiate a feature of the older cifs dialect with servers, such as + Samba 3.0.5 or later, that optionally can handle more POSIX like + (rather than Windows like) file behavior. It also enables support + for POSIX ACLs (getfacl and setfacl) to servers (such as Samba 3.10 + and later) which can negotiate CIFS POSIX ACL support. This config + option is not needed when mounting with SMB3.1.1. If unsure, say N. + +config CIFS_DEBUG + bool "Enable CIFS debugging routines" + default y + depends on CIFS + help + Enabling this option adds helpful debugging messages to + the cifs code which increases the size of the cifs module. + If unsure, say Y. + +config CIFS_DEBUG2 + bool "Enable additional CIFS debugging routines" + depends on CIFS_DEBUG + help + Enabling this option adds a few more debugging routines + to the cifs code which slightly increases the size of + the cifs module and can cause additional logging of debug + messages in some error paths, slowing performance. This + option can be turned off unless you are debugging + cifs problems. If unsure, say N. + +config CIFS_DEBUG_DUMP_KEYS + bool "Dump encryption keys for offline decryption (Unsafe)" + depends on CIFS_DEBUG + help + Enabling this will dump the encryption and decryption keys + used to communicate on an encrypted share connection on the + console. This allows Wireshark to decrypt and dissect + encrypted network captures. Enable this carefully. + If unsure, say N. + +config CIFS_DFS_UPCALL + bool "DFS feature support" + depends on CIFS + help + Distributed File System (DFS) support is used to access shares + transparently in an enterprise name space, even if the share + moves to a different server. This feature also enables + an upcall mechanism for CIFS which contacts userspace helper + utilities to provide server name resolution (host names to + IP addresses) which is needed in order to reconnect to + servers if their addresses change or for implicit mounts of + DFS junction points. If unsure, say Y. + +config CIFS_SWN_UPCALL + bool "SWN feature support" + depends on CIFS + help + The Service Witness Protocol (SWN) is used to get notifications + from a highly available server of resource state changes. This + feature enables an upcall mechanism for CIFS which contacts a + userspace daemon to establish the DCE/RPC connection to retrieve + the cluster available interfaces and resource change notifications. + If unsure, say Y. + +config CIFS_NFSD_EXPORT + bool "Allow nfsd to export CIFS file system" + depends on CIFS && BROKEN + help + Allows NFS server to export a CIFS mounted share (nfsd over cifs) + +if CIFS + +config CIFS_SMB_DIRECT + bool "SMB Direct support" + depends on CIFS && INFINIBAND && INFINIBAND_ADDR_TRANS + depends on CIFS=m || INFINIBAND=y + select SMBDIRECT + help + Enables SMB Direct support for SMB 3.0, 3.02 and 3.1.1. + SMB Direct allows transferring SMB packets over RDMA. If unsure, + say Y. + +config CIFS_FSCACHE + bool "Provide CIFS client caching support" + depends on CIFS=m && FSCACHE || CIFS=y && FSCACHE=y + help + Makes CIFS FS-Cache capable. Say Y here if you want your CIFS data + to be cached locally on disk through the general filesystem cache + manager. If unsure, say N. + +config CIFS_ROOT + bool "SMB root file system (Experimental)" + depends on CIFS=y && IP_PNP + help + Enables root file system support over SMB protocol. + + Most people say N here. + +config CIFS_COMPRESSION + bool "SMB message compression (Experimental)" + depends on CIFS + default n + help + Enables over-the-wire message compression for SMB 3.1.1 + mounts when negotiated with the server. + + Only write requests with data size >= PAGE_SIZE will be + compressed to avoid wasting resources. + + Say Y here if you want SMB traffic to be compressed. + If unsure, say N. + +config SMB1_KUNIT_TESTS + tristate "KUnit tests for SMB1" + depends on SMB_KUNIT_TESTS && CIFS_ALLOW_INSECURE_LEGACY + default SMB_KUNIT_TESTS + help + This builds the SMB1-specific KUnit tests. + + These tests are only enabled when legacy insecure SMB1 support + (CIFS_ALLOW_INSECURE_LEGACY) is enabled. + + If unsure, say N. +endif diff --git a/fs/smb/client/Makefile b/fs/smb/client/Makefile new file mode 100644 index 000000000..fc6b9d35c --- /dev/null +++ b/fs/smb/client/Makefile @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for Linux CIFS/SMB2/SMB3 VFS client +# +ccflags-y += -I$(src) # needed for trace events +obj-$(CONFIG_CIFS) += cifs.o + +cifs-y := trace.o cifsfs.o cifs_debug.o connect.o dir.o file.o \ + inode.o link.o misc.o netmisc.o smbencrypt.o transport.o \ + cached_dir.o cifs_unicode.o cifsencrypt.o \ + readdir.o ioctl.o sess.o export.o unc.o winucase.o \ + smb2ops.o smb2maperror.o smb2transport.o \ + smb2misc.o smb2pdu.o smb2inode.o smb2file.o cifsacl.o fs_context.o \ + dns_resolve.o cifs_spnego_negtokeninit.asn1.o asn1.o \ + namespace.o reparse.o + +$(obj)/asn1.o: $(obj)/cifs_spnego_negtokeninit.asn1.h + +$(obj)/cifs_spnego_negtokeninit.asn1.o: $(obj)/cifs_spnego_negtokeninit.asn1.c $(obj)/cifs_spnego_negtokeninit.asn1.h + +cifs-$(CONFIG_CIFS_XATTR) += xattr.o + +cifs-$(CONFIG_CIFS_UPCALL) += cifs_spnego.o + +cifs-$(CONFIG_CIFS_DFS_UPCALL) += dfs_cache.o dfs.o + +cifs-$(CONFIG_CIFS_SWN_UPCALL) += netlink.o cifs_swn.o + +cifs-$(CONFIG_CIFS_FSCACHE) += fscache.o + +cifs-$(CONFIG_CIFS_SMB_DIRECT) += smbdirect.o + +cifs-$(CONFIG_CIFS_ROOT) += cifsroot.o + +cifs-$(CONFIG_CIFS_ALLOW_INSECURE_LEGACY) += \ + cifssmb.o \ + smb1debug.o \ + smb1encrypt.o \ + smb1maperror.o \ + smb1misc.o \ + smb1ops.o \ + smb1session.o \ + smb1transport.o + +cifs-$(CONFIG_CIFS_COMPRESSION) += compress.o + +ifneq ($(CONFIG_CIFS_ALLOW_INSECURE_LEGACY),) +# +# Build the SMB1 error mapping tables from nterr.h and smberr.h +# +smb1-gen-y := smb1_mapping_table.c \ + smb1_err_dos_map.c \ + smb1_err_srv_map.c + +$(obj)/smb1_mapping_table.c: $(src)/nterr.h $(src)/gen_smb1_mapping FORCE + $(call if_changed,gen_smb1_mapping) + +$(obj)/smb1_err_%.c: $(src)/smberr.h $(src)/gen_smb1_mapping FORCE + $(call if_changed,gen_smb1_mapping) + +$(obj)/smb1maperror.o: $(addprefix $(obj)/, $(smb1-gen-y)) + +quiet_cmd_gen_smb1_mapping = GEN $@ + cmd_gen_smb1_mapping = perl $(src)/gen_smb1_mapping $< $@ +endif + +# +# Build the SMB2 error mapping table from smb2status.h +# +$(obj)/smb2_mapping_table.c: $(src)/../common/smb2status.h \ + $(src)/gen_smb2_mapping FORCE + $(call if_changed,gen_smb2_mapping) + +$(obj)/smb2maperror.o: $(obj)/smb2_mapping_table.c + +quiet_cmd_gen_smb2_mapping = GEN $@ + cmd_gen_smb2_mapping = perl $(src)/gen_smb2_mapping $< $@ + +obj-$(CONFIG_SMB1_KUNIT_TESTS) += smb1maperror_test.o +obj-$(CONFIG_SMB_KUNIT_TESTS) += smb2maperror_test.o + +# Let Kbuild handle tracking and cleaning +targets += smb2_mapping_table.c $(smb1-gen-y) diff --git a/fs/smb/client/asn1.c b/fs/smb/client/asn1.c new file mode 100644 index 000000000..214a44509 --- /dev/null +++ b/fs/smb/client/asn1.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#include +#include +#include +#include "cifsglob.h" +#include "cifs_debug.h" +#include "cifsproto.h" +#include "cifs_spnego_negtokeninit.asn1.h" + +int +decode_negTokenInit(unsigned char *security_blob, int length, + struct TCP_Server_Info *server) +{ + if (asn1_ber_decoder(&cifs_spnego_negtokeninit_decoder, server, + security_blob, length) == 0) + return 1; + else + return 0; +} + +int cifs_gssapi_this_mech(void *context, size_t hdrlen, + unsigned char tag, const void *value, size_t vlen) +{ + enum OID oid; + + oid = look_up_OID(value, vlen); + if (oid != OID_spnego) { + char buf[50]; + + sprint_oid(value, vlen, buf, sizeof(buf)); + cifs_dbg(FYI, "Error decoding negTokenInit header: unexpected OID %s\n", + buf); + return -EBADMSG; + } + return 0; +} + +int cifs_neg_token_init_mech_type(void *context, size_t hdrlen, + unsigned char tag, + const void *value, size_t vlen) +{ + struct TCP_Server_Info *server = context; + enum OID oid; + + oid = look_up_OID(value, vlen); + if (oid == OID_mskrb5) + server->sec_mskerberos = true; + else if (oid == OID_krb5u2u) + server->sec_kerberosu2u = true; + else if (oid == OID_krb5) + server->sec_kerberos = true; + else if (oid == OID_ntlmssp) + server->sec_ntlmssp = true; + else if (oid == OID_IAKerb) + server->sec_iakerb = true; + else { + char buf[50]; + + sprint_oid(value, vlen, buf, sizeof(buf)); + cifs_dbg(FYI, "Decoding negTokenInit: unsupported OID %s\n", + buf); + } + return 0; +} diff --git a/fs/smb/client/cached_dir.c b/fs/smb/client/cached_dir.c new file mode 100644 index 000000000..88d5e9a32 --- /dev/null +++ b/fs/smb/client/cached_dir.c @@ -0,0 +1,875 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Functions to handle the cached directory entries + * + * Copyright (c) 2022, Ronnie Sahlberg + */ + +#include +#include "cifsglob.h" +#include "cifsproto.h" +#include "cifs_debug.h" +#include "smb2proto.h" +#include "cached_dir.h" + +static struct cached_fid *init_cached_dir(const char *path); +static void free_cached_dir(struct cached_fid *cfid); +static void smb2_close_cached_fid(struct kref *ref); +static void cfids_laundromat_worker(struct work_struct *work); +static void close_cached_dir_locked(struct cached_fid *cfid); + +struct cached_dir_dentry { + struct list_head entry; + struct dentry *dentry; +}; + +static struct cached_fid *find_or_create_cached_dir(struct cached_fids *cfids, + const char *path, + bool lookup_only, + __u32 max_cached_dirs) +{ + struct cached_fid *cfid; + + list_for_each_entry(cfid, &cfids->entries, entry) { + if (!strcmp(cfid->path, path)) { + /* + * If it doesn't have a lease it is either not yet + * fully cached or it may be in the process of + * being deleted due to a lease break. + */ + if (!is_valid_cached_dir(cfid)) + return NULL; + kref_get(&cfid->refcount); + return cfid; + } + } + if (lookup_only) { + return NULL; + } + if (cfids->num_entries >= max_cached_dirs) { + return NULL; + } + cfid = init_cached_dir(path); + if (cfid == NULL) { + return NULL; + } + cfid->cfids = cfids; + cfids->num_entries++; + list_add(&cfid->entry, &cfids->entries); + cfid->on_list = true; + kref_get(&cfid->refcount); + /* + * Set @cfid->has_lease to true during construction so that the lease + * reference can be put in cached_dir_lease_break() due to a potential + * lease break right after the request is sent or while @cfid is still + * being cached, or if a reconnection is triggered during construction. + * Concurrent processes won't be to use it yet due to @cfid->time being + * zero. + */ + cfid->has_lease = true; + + return cfid; +} + +static struct dentry * +path_to_dentry(struct cifs_sb_info *cifs_sb, const char *path) +{ + struct dentry *dentry; + const char *s, *p; + char sep; + + sep = CIFS_DIR_SEP(cifs_sb); + dentry = dget(cifs_sb->root); + s = path; + + do { + struct inode *dir = d_inode(dentry); + struct dentry *child; + + if (!S_ISDIR(dir->i_mode)) { + dput(dentry); + dentry = ERR_PTR(-ENOTDIR); + break; + } + + /* skip separators */ + while (*s == sep) + s++; + if (!*s) + break; + p = s++; + /* next separator */ + while (*s && *s != sep) + s++; + + child = lookup_noperm_positive_unlocked(&QSTR_LEN(p, s - p), + dentry); + dput(dentry); + dentry = child; + } while (!IS_ERR(dentry)); + return dentry; +} + +static const char *path_no_prefix(struct cifs_sb_info *cifs_sb, + const char *path) +{ + size_t len = 0; + + if (!*path) + return path; + + if ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_USE_PREFIX_PATH) && + cifs_sb->prepath) { + len = strlen(cifs_sb->prepath) + 1; + if (unlikely(len > strlen(path))) + return ERR_PTR(-EINVAL); + } + return path + len; +} + +/* + * Open the and cache a directory handle. + * If error then *cfid is not initialized. + */ +int open_cached_dir(unsigned int xid, struct cifs_tcon *tcon, + const char *path, + struct cifs_sb_info *cifs_sb, + bool lookup_only, struct cached_fid **ret_cfid) +{ + struct cifs_ses *ses; + struct TCP_Server_Info *server; + struct cifs_open_parms oparms; + struct smb2_create_rsp *o_rsp = NULL; + struct smb2_query_info_rsp *qi_rsp = NULL; + int resp_buftype[2]; + struct smb_rqst rqst[2]; + struct kvec rsp_iov[2]; + struct kvec open_iov[SMB2_CREATE_IOV_SIZE]; + struct kvec qi_iov[1]; + int rc, flags = 0; + __le16 *utf16_path = NULL; + u8 oplock = SMB2_OPLOCK_LEVEL_II; + struct cifs_fid *pfid; + struct dentry *dentry = NULL; + struct cached_fid *cfid; + struct cached_fids *cfids; + const char *npath; + int retries = 0, cur_sleep = 0; + __le32 lease_flags = 0; + + if (cifs_sb->root == NULL) + return -ENOENT; + + if (tcon == NULL) + return -EOPNOTSUPP; + + ses = tcon->ses; + cfids = tcon->cfids; + + if (cfids == NULL) + return -EOPNOTSUPP; + +replay_again: + /* reinitialize for possible replay */ + flags = 0; + oplock = SMB2_OPLOCK_LEVEL_II; + server = cifs_pick_channel(ses); + + if (!server->ops->new_lease_key) + return smb_EIO(smb_eio_trace_no_lease_key); + + utf16_path = cifs_convert_path_to_utf16(path, cifs_sb); + if (!utf16_path) + return -ENOMEM; + + spin_lock(&cfids->cfid_list_lock); + cfid = find_or_create_cached_dir(cfids, path, lookup_only, tcon->max_cached_dirs); + if (cfid == NULL) { + spin_unlock(&cfids->cfid_list_lock); + kfree(utf16_path); + return -ENOENT; + } + /* + * Return cached fid if it is valid (has a lease and has a time). + * Otherwise, it is either a new entry or laundromat worker removed it + * from @cfids->entries. Caller will put last reference if the latter. + */ + if (is_valid_cached_dir(cfid)) { + cfid->last_access_time = jiffies; + spin_unlock(&cfids->cfid_list_lock); + *ret_cfid = cfid; + kfree(utf16_path); + return 0; + } + spin_unlock(&cfids->cfid_list_lock); + + pfid = &cfid->fid; + + /* + * Skip any prefix paths in @path as lookup_noperm_positive_unlocked() ends up + * calling ->lookup() which already adds those through + * build_path_from_dentry(). Also, do it earlier as we might reconnect + * below when trying to send compounded request and then potentially + * having a different prefix path (e.g. after DFS failover). + */ + npath = path_no_prefix(cifs_sb, path); + if (IS_ERR(npath)) { + rc = PTR_ERR(npath); + goto out; + } + + if (!npath[0]) { + dentry = dget(cifs_sb->root); + } else { + dentry = path_to_dentry(cifs_sb, npath); + if (IS_ERR(dentry)) { + rc = -ENOENT; + goto out; + } + if (dentry->d_parent && server->dialect >= SMB30_PROT_ID) { + struct cached_fid *parent_cfid; + + spin_lock(&cfids->cfid_list_lock); + list_for_each_entry(parent_cfid, &cfids->entries, entry) { + if (parent_cfid->dentry == dentry->d_parent) { + cifs_dbg(FYI, "found a parent cached file handle\n"); + if (is_valid_cached_dir(parent_cfid)) { + lease_flags + |= SMB2_LEASE_FLAG_PARENT_LEASE_KEY_SET_LE; + memcpy(pfid->parent_lease_key, + parent_cfid->fid.lease_key, + SMB2_LEASE_KEY_SIZE); + } + break; + } + } + spin_unlock(&cfids->cfid_list_lock); + } + } + cfid->dentry = dentry; + cfid->tcon = tcon; + + /* + * We do not hold the lock for the open because in case + * SMB2_open needs to reconnect. + * This is safe because no other thread will be able to get a ref + * to the cfid until we have finished opening the file and (possibly) + * acquired a lease. + */ + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; + + server->ops->new_lease_key(pfid); + + memset(rqst, 0, sizeof(rqst)); + resp_buftype[0] = resp_buftype[1] = CIFS_NO_BUFFER; + memset(rsp_iov, 0, sizeof(rsp_iov)); + + /* Open */ + memset(&open_iov, 0, sizeof(open_iov)); + rqst[0].rq_iov = open_iov; + rqst[0].rq_nvec = SMB2_CREATE_IOV_SIZE; + + oparms = (struct cifs_open_parms) { + .tcon = tcon, + .path = path, + .create_options = cifs_create_options(cifs_sb, CREATE_NOT_FILE), + .desired_access = FILE_READ_DATA | FILE_READ_ATTRIBUTES | + FILE_READ_EA, + .disposition = FILE_OPEN, + .fid = pfid, + .lease_flags = lease_flags, + .replay = !!(retries), + }; + + rc = SMB2_open_init(tcon, server, + &rqst[0], &oplock, &oparms, utf16_path); + if (rc) + goto oshr_free; + + if (oplock != SMB2_OPLOCK_LEVEL_II) { + rc = -EINVAL; + cifs_dbg(FYI, "%s: Oplock level %d not suitable for cached directory\n", + __func__, oplock); + goto oshr_free; + } + + smb2_set_next_command(tcon, &rqst[0]); + + memset(&qi_iov, 0, sizeof(qi_iov)); + rqst[1].rq_iov = qi_iov; + rqst[1].rq_nvec = 1; + + rc = SMB2_query_info_init(tcon, server, + &rqst[1], COMPOUND_FID, + COMPOUND_FID, FILE_ALL_INFORMATION, + SMB2_O_INFO_FILE, 0, + sizeof(struct smb2_file_all_info) + + PATH_MAX * 2, 0, NULL); + if (rc) + goto oshr_free; + + smb2_set_related(&rqst[1]); + + if (retries) { + /* Back-off before retry */ + if (cur_sleep) + msleep(cur_sleep); + + smb2_set_replay(server, &rqst[0]); + smb2_set_replay(server, &rqst[1]); + } + + rc = compound_send_recv(xid, ses, server, + flags, 2, rqst, + resp_buftype, rsp_iov); + if (rc) { + if (rc == -EREMCHG) { + tcon->need_reconnect = true; + pr_warn_once("server share %s deleted\n", + tcon->tree_name); + } + goto oshr_free; + } + cfid->is_open = true; + + spin_lock(&cfids->cfid_list_lock); + + o_rsp = (struct smb2_create_rsp *)rsp_iov[0].iov_base; + oparms.fid->persistent_fid = o_rsp->PersistentFileId; + oparms.fid->volatile_fid = o_rsp->VolatileFileId; +#ifdef CONFIG_CIFS_DEBUG2 + oparms.fid->mid = le64_to_cpu(o_rsp->hdr.MessageId); +#endif /* CIFS_DEBUG2 */ + + + if (o_rsp->OplockLevel != SMB2_OPLOCK_LEVEL_LEASE) { + spin_unlock(&cfids->cfid_list_lock); + rc = -EINVAL; + goto oshr_free; + } + + rc = smb2_parse_contexts(server, rsp_iov, + &oparms.fid->epoch, + oparms.fid->lease_key, + &oplock, NULL, NULL); + if (rc) { + spin_unlock(&cfids->cfid_list_lock); + goto oshr_free; + } + + rc = -EINVAL; + if (!(oplock & SMB2_LEASE_READ_CACHING_HE)) { + spin_unlock(&cfids->cfid_list_lock); + goto oshr_free; + } + qi_rsp = (struct smb2_query_info_rsp *)rsp_iov[1].iov_base; + if (le32_to_cpu(qi_rsp->OutputBufferLength) < sizeof(struct smb2_file_all_info)) { + spin_unlock(&cfids->cfid_list_lock); + goto oshr_free; + } + if (!smb2_validate_and_copy_iov( + le16_to_cpu(qi_rsp->OutputBufferOffset), + sizeof(struct smb2_file_all_info), + &rsp_iov[1], sizeof(struct smb2_file_all_info), + (char *)&cfid->file_all_info)) + cfid->file_all_info_is_valid = true; + + cfid->time = jiffies; + cfid->last_access_time = jiffies; + spin_unlock(&cfids->cfid_list_lock); + /* At this point the directory handle is fully cached */ + rc = 0; + +oshr_free: + SMB2_open_free(&rqst[0]); + SMB2_query_info_free(&rqst[1]); + free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base); + free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base); +out: + if (rc) { + spin_lock(&cfids->cfid_list_lock); + if (cfid->on_list) { + list_del(&cfid->entry); + cfid->on_list = false; + cfids->num_entries--; + } + if (cfid->has_lease) { + /* + * We are guaranteed to have two references at this + * point. One for the caller and one for a potential + * lease. Release one here, and the second below. + */ + cfid->has_lease = false; + close_cached_dir_locked(cfid); + } + spin_unlock(&cfids->cfid_list_lock); + + close_cached_dir(cfid); + } else { + *ret_cfid = cfid; + atomic_inc(&tcon->num_remote_opens); + } + kfree(utf16_path); + + if (is_replayable_error(rc) && + smb2_should_replay(tcon, &retries, &cur_sleep)) + goto replay_again; + + return rc; +} + +int open_cached_dir_by_dentry(struct cifs_tcon *tcon, + struct dentry *dentry, + struct cached_fid **ret_cfid) +{ + struct cached_fid *cfid; + struct cached_fids *cfids = tcon->cfids; + + if (cfids == NULL) + return -EOPNOTSUPP; + + if (!dentry) + return -ENOENT; + + spin_lock(&cfids->cfid_list_lock); + list_for_each_entry(cfid, &cfids->entries, entry) { + if (cfid->dentry == dentry) { + if (!is_valid_cached_dir(cfid)) + break; + cifs_dbg(FYI, "found a cached file handle by dentry\n"); + kref_get(&cfid->refcount); + *ret_cfid = cfid; + cfid->last_access_time = jiffies; + spin_unlock(&cfids->cfid_list_lock); + return 0; + } + } + spin_unlock(&cfids->cfid_list_lock); + return -ENOENT; +} + +static void +smb2_close_cached_fid(struct kref *ref) +__releases(&cfid->cfids->cfid_list_lock) +{ + struct cached_fid *cfid = container_of(ref, struct cached_fid, + refcount); + int rc; + + lockdep_assert_held(&cfid->cfids->cfid_list_lock); + + if (cfid->on_list) { + list_del(&cfid->entry); + cfid->on_list = false; + cfid->cfids->num_entries--; + } + spin_unlock(&cfid->cfids->cfid_list_lock); + + dput(cfid->dentry); + cfid->dentry = NULL; + + if (cfid->is_open) { + rc = SMB2_close(0, cfid->tcon, cfid->fid.persistent_fid, + cfid->fid.volatile_fid); + if (rc) /* should we retry on -EBUSY or -EAGAIN? */ + cifs_dbg(VFS, "close cached dir rc %d\n", rc); + } + + free_cached_dir(cfid); +} + +void drop_cached_dir_by_name(const unsigned int xid, struct cifs_tcon *tcon, + const char *name, struct cifs_sb_info *cifs_sb) +{ + struct cached_fid *cfid = NULL; + int rc; + + rc = open_cached_dir(xid, tcon, name, cifs_sb, true, &cfid); + if (rc) { + cifs_dbg(FYI, "no cached dir found for rmdir(%s)\n", name); + return; + } + spin_lock(&cfid->cfids->cfid_list_lock); + if (cfid->has_lease) { + cfid->has_lease = false; + close_cached_dir_locked(cfid); + } + spin_unlock(&cfid->cfids->cfid_list_lock); + close_cached_dir(cfid); +} + +/** + * close_cached_dir - drop a reference of a cached dir + * + * The release function will be called with cfid_list_lock held to remove the + * cached dirs from the list before any other thread can take another @cfid + * ref. Must not be called with cfid_list_lock held; use + * close_cached_dir_locked() called instead. + * + * @cfid: cached dir + */ +void close_cached_dir(struct cached_fid *cfid) +{ + lockdep_assert_not_held(&cfid->cfids->cfid_list_lock); + kref_put_lock(&cfid->refcount, smb2_close_cached_fid, &cfid->cfids->cfid_list_lock); +} + +/** + * close_cached_dir_locked - put a reference of a cached dir with + * cfid_list_lock held + * + * Calling close_cached_dir() with cfid_list_lock held has the potential effect + * of causing a deadlock if the invariant of refcount >= 2 is false. + * + * This function is used in paths that hold cfid_list_lock and expect at least + * two references. If that invariant is violated, WARNs and returns without + * dropping a reference; the final put must still go through + * close_cached_dir(). + * + * @cfid: cached dir + */ +static void close_cached_dir_locked(struct cached_fid *cfid) +{ + lockdep_assert_held(&cfid->cfids->cfid_list_lock); + + if (WARN_ON(kref_read(&cfid->refcount) < 2)) + return; + + kref_put(&cfid->refcount, smb2_close_cached_fid); +} + +/* + * Called from cifs_kill_sb when we unmount a share + */ +void close_all_cached_dirs(struct cifs_sb_info *cifs_sb) +{ + struct rb_root *root = &cifs_sb->tlink_tree; + struct rb_node *node; + struct cached_fid *cfid; + struct cifs_tcon *tcon; + struct tcon_link *tlink; + struct cached_fids *cfids; + struct cached_dir_dentry *tmp_list, *q; + LIST_HEAD(entry); + + spin_lock(&cifs_sb->tlink_tree_lock); + for (node = rb_first(root); node; node = rb_next(node)) { + tlink = rb_entry(node, struct tcon_link, tl_rbnode); + tcon = tlink_tcon(tlink); + if (IS_ERR(tcon)) + continue; + cfids = tcon->cfids; + if (cfids == NULL) + continue; + spin_lock(&cfids->cfid_list_lock); + list_for_each_entry(cfid, &cfids->entries, entry) { + tmp_list = kmalloc_obj(*tmp_list, GFP_ATOMIC); + if (tmp_list == NULL) { + /* + * If the malloc() fails, we won't drop all + * dentries, and unmounting is likely to trigger + * a 'Dentry still in use' error. + */ + cifs_tcon_dbg(VFS, "Out of memory while dropping dentries\n"); + spin_unlock(&cfids->cfid_list_lock); + spin_unlock(&cifs_sb->tlink_tree_lock); + goto done; + } + + tmp_list->dentry = cfid->dentry; + cfid->dentry = NULL; + + list_add_tail(&tmp_list->entry, &entry); + } + spin_unlock(&cfids->cfid_list_lock); + } + spin_unlock(&cifs_sb->tlink_tree_lock); + +done: + list_for_each_entry_safe(tmp_list, q, &entry, entry) { + list_del(&tmp_list->entry); + dput(tmp_list->dentry); + kfree(tmp_list); + } + + /* Flush any pending work that will drop dentries */ + flush_workqueue(cfid_put_wq); +} + +/* + * Invalidate all cached dirs when a TCON has been reset + * due to a session loss. + */ +void invalidate_all_cached_dirs(struct cifs_tcon *tcon, bool sync) +{ + struct cached_fids *cfids = tcon->cfids; + struct cached_fid *cfid, *q; + + if (cfids == NULL) + return; + + /* + * Mark all the cfids as closed, and move them to the cfids->dying list. + * They'll be cleaned up by laundromat. Take a reference to each cfid + * during this process. + */ + spin_lock(&cfids->cfid_list_lock); + list_for_each_entry_safe(cfid, q, &cfids->entries, entry) { + list_move(&cfid->entry, &cfids->dying); + cfids->num_entries--; + cfid->is_open = false; + cfid->on_list = false; + if (cfid->has_lease) { + /* + * The lease was never cancelled from the server, + * so steal that reference. + */ + cfid->has_lease = false; + } else + kref_get(&cfid->refcount); + } + spin_unlock(&cfids->cfid_list_lock); + + /* run laundromat unconditionally now as there might have been previously queued work */ + mod_delayed_work(cfid_put_wq, &cfids->laundromat_work, 0); + if (sync) + flush_delayed_work(&cfids->laundromat_work); +} + +static void +cached_dir_offload_close(struct work_struct *work) +{ + struct cached_fid *cfid = container_of(work, + struct cached_fid, close_work); + struct cifs_tcon *tcon = cfid->tcon; + + WARN_ON(cfid->on_list); + + close_cached_dir(cfid); + cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_cached_close); +} + +/* + * Release the cached directory's dentry, and then queue work to drop cached + * directory itself (closing on server if needed). + * + * Must be called with a reference to the cached_fid and a reference to the + * tcon. + */ +static void cached_dir_put_work(struct work_struct *work) +{ + struct cached_fid *cfid = container_of(work, struct cached_fid, + put_work); + dput(cfid->dentry); + cfid->dentry = NULL; + + queue_work(serverclose_wq, &cfid->close_work); +} + +bool cached_dir_lease_break(struct cifs_tcon *tcon, __u8 lease_key[16]) +{ + struct cached_fids *cfids = tcon->cfids; + struct cached_fid *cfid; + + if (cfids == NULL) + return false; + + spin_lock(&cfids->cfid_list_lock); + list_for_each_entry(cfid, &cfids->entries, entry) { + if (cfid->has_lease && + !memcmp(lease_key, + cfid->fid.lease_key, + SMB2_LEASE_KEY_SIZE)) { + cfid->has_lease = false; + cfid->time = 0; + /* + * We found a lease remove it from the list + * so no threads can access it. + */ + list_del(&cfid->entry); + cfid->on_list = false; + cfids->num_entries--; + + ++tcon->tc_count; + trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, + netfs_trace_tcon_ref_get_cached_lease_break); + queue_work(cfid_put_wq, &cfid->put_work); + spin_unlock(&cfids->cfid_list_lock); + return true; + } + } + spin_unlock(&cfids->cfid_list_lock); + return false; +} + +static struct cached_fid *init_cached_dir(const char *path) +{ + struct cached_fid *cfid; + + cfid = kzalloc_obj(*cfid, GFP_ATOMIC); + if (!cfid) + return NULL; + cfid->path = kstrdup(path, GFP_ATOMIC); + if (!cfid->path) { + kfree(cfid); + return NULL; + } + + INIT_WORK(&cfid->close_work, cached_dir_offload_close); + INIT_WORK(&cfid->put_work, cached_dir_put_work); + INIT_LIST_HEAD(&cfid->entry); + INIT_LIST_HEAD(&cfid->dirents.entries); + mutex_init(&cfid->dirents.de_mutex); + kref_init(&cfid->refcount); + return cfid; +} + +static void free_cached_dir(struct cached_fid *cfid) +{ + struct cached_dirent *dirent, *q; + + WARN_ON(work_pending(&cfid->close_work)); + WARN_ON(work_pending(&cfid->put_work)); + + dput(cfid->dentry); + cfid->dentry = NULL; + + /* + * Delete all cached dirent names + */ + list_for_each_entry_safe(dirent, q, &cfid->dirents.entries, entry) { + list_del(&dirent->entry); + kfree(dirent->name); + kfree(dirent); + } + + /* adjust tcon-level counters and reset per-dir accounting */ + if (cfid->cfids) { + if (cfid->dirents.entries_count) + atomic_long_sub((long)cfid->dirents.entries_count, + &cfid->cfids->total_dirents_entries); + if (cfid->dirents.bytes_used) { + atomic64_sub((long long)cfid->dirents.bytes_used, + &cfid->cfids->total_dirents_bytes); + atomic64_sub((long long)cfid->dirents.bytes_used, + &cifs_dircache_bytes_used); + } + } + cfid->dirents.entries_count = 0; + cfid->dirents.bytes_used = 0; + + kfree(cfid->path); + cfid->path = NULL; + kfree(cfid); +} + +static void cfids_laundromat_worker(struct work_struct *work) +{ + struct cached_fids *cfids; + struct cached_fid *cfid, *q; + LIST_HEAD(entry); + + cfids = container_of(work, struct cached_fids, laundromat_work.work); + + spin_lock(&cfids->cfid_list_lock); + /* move cfids->dying to the local list */ + list_cut_before(&entry, &cfids->dying, &cfids->dying); + + list_for_each_entry_safe(cfid, q, &cfids->entries, entry) { + if (cfid->last_access_time && + time_after(jiffies, cfid->last_access_time + HZ * dir_cache_timeout)) { + cfid->on_list = false; + list_move(&cfid->entry, &entry); + cfids->num_entries--; + if (cfid->has_lease) { + /* + * Our lease has not yet been cancelled from the + * server. Steal that reference. + */ + cfid->has_lease = false; + } else + kref_get(&cfid->refcount); + } + } + spin_unlock(&cfids->cfid_list_lock); + + list_for_each_entry_safe(cfid, q, &entry, entry) { + list_del(&cfid->entry); + + dput(cfid->dentry); + cfid->dentry = NULL; + + if (cfid->is_open) { + spin_lock(&cfid->tcon->tc_lock); + ++cfid->tcon->tc_count; + trace_smb3_tcon_ref(cfid->tcon->debug_id, cfid->tcon->tc_count, + netfs_trace_tcon_ref_get_cached_laundromat); + spin_unlock(&cfid->tcon->tc_lock); + queue_work(serverclose_wq, &cfid->close_work); + } else + /* + * Drop the ref-count from above, either the lease-ref (if there + * was one) or the extra one acquired. + */ + close_cached_dir(cfid); + } + queue_delayed_work(cfid_put_wq, &cfids->laundromat_work, + dir_cache_timeout * HZ); +} + +struct cached_fids *init_cached_dirs(void) +{ + struct cached_fids *cfids; + + cfids = kzalloc_obj(*cfids); + if (!cfids) + return NULL; + spin_lock_init(&cfids->cfid_list_lock); + INIT_LIST_HEAD(&cfids->entries); + INIT_LIST_HEAD(&cfids->dying); + + INIT_DELAYED_WORK(&cfids->laundromat_work, cfids_laundromat_worker); + queue_delayed_work(cfid_put_wq, &cfids->laundromat_work, + dir_cache_timeout * HZ); + + atomic_long_set(&cfids->total_dirents_entries, 0); + atomic64_set(&cfids->total_dirents_bytes, 0); + + return cfids; +} + +/* + * Called from tconInfoFree when we are tearing down the tcon. + * There are no active users or open files/directories at this point. + */ +void free_cached_dirs(struct cached_fids *cfids) +{ + struct cached_fid *cfid, *q; + LIST_HEAD(entry); + + if (cfids == NULL) + return; + + cancel_delayed_work_sync(&cfids->laundromat_work); + + spin_lock(&cfids->cfid_list_lock); + list_for_each_entry_safe(cfid, q, &cfids->entries, entry) { + cfid->on_list = false; + cfid->is_open = false; + list_move(&cfid->entry, &entry); + } + list_for_each_entry_safe(cfid, q, &cfids->dying, entry) { + cfid->on_list = false; + cfid->is_open = false; + list_move(&cfid->entry, &entry); + } + spin_unlock(&cfids->cfid_list_lock); + + list_for_each_entry_safe(cfid, q, &entry, entry) { + list_del(&cfid->entry); + free_cached_dir(cfid); + } + + kfree(cfids); +} diff --git a/fs/smb/client/cached_dir.h b/fs/smb/client/cached_dir.h new file mode 100644 index 000000000..fc756836d --- /dev/null +++ b/fs/smb/client/cached_dir.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Functions to handle the cached directory entries + * + * Copyright (c) 2022, Ronnie Sahlberg + */ + +#ifndef _CACHED_DIR_H +#define _CACHED_DIR_H + + +struct cached_dirent { + struct list_head entry; + char *name; + int namelen; + loff_t pos; + struct cifs_fattr fattr; +}; + +struct cached_dirents { + bool is_valid:1; + bool is_failed:1; + struct file *file; /* + * Used to associate the cache with a single + * open file instance. + */ + struct mutex de_mutex; + loff_t pos; /* Expected ctx->pos */ + struct list_head entries; + /* accounting for cached entries in this directory */ + unsigned long entries_count; + unsigned long bytes_used; +}; + +struct cached_fid { + struct list_head entry; + struct cached_fids *cfids; + const char *path; + bool has_lease; + bool is_open; + bool on_list; + bool file_all_info_is_valid; + unsigned long time; /* jiffies of when lease was taken */ + unsigned long last_access_time; /* jiffies of when last accessed */ + struct kref refcount; + struct cifs_fid fid; + struct cifs_tcon *tcon; + struct dentry *dentry; + struct work_struct put_work; + struct work_struct close_work; + struct cached_dirents dirents; + + /* Must be last as it ends in a flexible-array member. */ + struct smb2_file_all_info file_all_info; +}; + +/* default MAX_CACHED_FIDS is 16 */ +struct cached_fids { + /* Must be held when: + * - accessing the cfids->entries list + * - accessing the cfids->dying list + */ + spinlock_t cfid_list_lock; + int num_entries; + struct list_head entries; + struct list_head dying; + struct delayed_work laundromat_work; + /* aggregate accounting for all cached dirents under this tcon */ + atomic_long_t total_dirents_entries; + atomic64_t total_dirents_bytes; +}; + +/* Module-wide directory cache accounting (defined in cifsfs.c) */ +extern atomic64_t cifs_dircache_bytes_used; /* bytes across all mounts */ + +static inline bool +is_valid_cached_dir(struct cached_fid *cfid) +{ + return cfid->time && cfid->has_lease; +} + +struct cached_fids *init_cached_dirs(void); +void free_cached_dirs(struct cached_fids *cfids); +int open_cached_dir(unsigned int xid, struct cifs_tcon *tcon, const char *path, + struct cifs_sb_info *cifs_sb, bool lookup_only, + struct cached_fid **ret_cfid); +int open_cached_dir_by_dentry(struct cifs_tcon *tcon, struct dentry *dentry, + struct cached_fid **ret_cfid); +void close_cached_dir(struct cached_fid *cfid); +void drop_cached_dir_by_name(const unsigned int xid, struct cifs_tcon *tcon, + const char *name, struct cifs_sb_info *cifs_sb); +void close_all_cached_dirs(struct cifs_sb_info *cifs_sb); +void invalidate_all_cached_dirs(struct cifs_tcon *tcon, bool sync); +bool cached_dir_lease_break(struct cifs_tcon *tcon, __u8 lease_key[16]); + +#endif /* _CACHED_DIR_H */ diff --git a/fs/smb/client/cifs_debug.c b/fs/smb/client/cifs_debug.c new file mode 100644 index 000000000..4ed4f55a0 --- /dev/null +++ b/fs/smb/client/cifs_debug.c @@ -0,0 +1,1297 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * + * Copyright (C) International Business Machines Corp., 2000,2005 + * + * Modified by Steve French (sfrench@us.ibm.com) + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "cifsglob.h" +#include "cifsproto.h" +#include "cifs_debug.h" +#include "cifsfs.h" +#include "fs_context.h" +#ifdef CONFIG_CIFS_DFS_UPCALL +#include "dfs_cache.h" +#endif +#ifdef CONFIG_CIFS_SMB_DIRECT +#include "smbdirect.h" +#endif +#include "cifs_swn.h" +#include "cached_dir.h" + +void +cifs_dump_mem(char *label, void *data, int length) +{ + pr_debug("%s: dump of %d bytes of data at 0x%p\n", label, length, data); + print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_OFFSET, 16, 4, + data, length, true); +} + +void cifs_dump_mids(struct TCP_Server_Info *server) +{ +#ifdef CONFIG_CIFS_DEBUG2 + struct mid_q_entry *mid_entry; + + if (server == NULL) + return; + + cifs_dbg(VFS, "Dump pending requests:\n"); + spin_lock(&server->mid_queue_lock); + list_for_each_entry(mid_entry, &server->pending_mid_q, qhead) { + cifs_dbg(VFS, "State: %d Cmd: %d Pid: %d Cbdata: %p Mid %llu\n", + mid_entry->mid_state, + le16_to_cpu(mid_entry->command), + mid_entry->pid, + mid_entry->callback_data, + mid_entry->mid); +#ifdef CONFIG_CIFS_STATS2 + cifs_dbg(VFS, "IsLarge: %d buf: %p time rcv: %ld now: %ld\n", + mid_entry->large_buf, + mid_entry->resp_buf, + mid_entry->when_received, + jiffies); +#endif /* STATS2 */ + cifs_dbg(VFS, "IsMult: %d IsEnd: %d\n", + mid_entry->multiRsp, mid_entry->multiEnd); + if (mid_entry->resp_buf) { + server->ops->dump_detail(mid_entry->resp_buf, + mid_entry->response_pdu_len, server); + cifs_dump_mem("existing buf: ", mid_entry->resp_buf, 62); + } + } + spin_unlock(&server->mid_queue_lock); +#endif /* CONFIG_CIFS_DEBUG2 */ +} + +#ifdef CONFIG_PROC_FS +static void cifs_debug_tcon(struct seq_file *m, struct cifs_tcon *tcon) +{ + __u32 dev_type = le32_to_cpu(tcon->fsDevInfo.DeviceType); + + seq_printf(m, "%s Mounts: %d ", tcon->tree_name, tcon->tc_count); + if (tcon->nativeFileSystem) + seq_printf(m, "Type: %s ", tcon->nativeFileSystem); + seq_printf(m, "DevInfo: 0x%x Attributes: 0x%x\n\tPathComponentMax: %d Status: %d", + le32_to_cpu(tcon->fsDevInfo.DeviceCharacteristics), + le32_to_cpu(tcon->fsAttrInfo.Attributes), + le32_to_cpu(tcon->fsAttrInfo.MaxPathNameComponentLength), + tcon->status); + if (dev_type == FILE_DEVICE_DISK) + seq_puts(m, " type: DISK "); + else if (dev_type == FILE_DEVICE_CD_ROM) + seq_puts(m, " type: CDROM "); + else + seq_printf(m, " type: %d ", dev_type); + + seq_printf(m, "Serial Number: 0x%x", tcon->vol_serial_number); + + if ((tcon->seal) || + (tcon->ses->session_flags & SMB2_SESSION_FLAG_ENCRYPT_DATA) || + (tcon->share_flags & SHI1005_FLAGS_ENCRYPT_DATA)) + seq_puts(m, " encrypted"); + if (tcon->nocase) + seq_printf(m, " nocase"); + if (tcon->unix_ext) + seq_printf(m, " POSIX Extensions"); + if (tcon->ses->server->ops->dump_share_caps) + tcon->ses->server->ops->dump_share_caps(m, tcon); + if (tcon->use_witness) + seq_puts(m, " Witness"); + if (tcon->broken_sparse_sup) + seq_puts(m, " nosparse"); + if (tcon->need_reconnect) + seq_puts(m, "\tDISCONNECTED "); + spin_lock(&tcon->tc_lock); + if (tcon->origin_fullpath) { + seq_printf(m, "\n\tDFS origin fullpath: %s", + tcon->origin_fullpath); + } + spin_unlock(&tcon->tc_lock); + seq_putc(m, '\n'); +} + +static void +cifs_dump_channel(struct seq_file *m, int i, struct cifs_chan *chan) +{ + struct TCP_Server_Info *server = chan->server; + + if (!server) { + seq_printf(m, "\n\n\t\tChannel: %d DISABLED", i+1); + return; + } + + seq_printf(m, "\n\n\t\tChannel: %d ConnectionId: 0x%llx" + "\n\t\tNumber of credits: %d,%d,%d Dialect 0x%x" + "\n\t\tTCP status: %d Instance: %d" + "\n\t\tLocal Users To Server: %d SecMode: 0x%x Req On Wire: %d" + "\n\t\tIn Send: %d In MaxReq Wait: %d", + i+1, server->conn_id, + server->credits, + server->echo_credits, + server->oplock_credits, + server->dialect, + server->tcpStatus, + server->reconnect_instance, + server->srv_count, + server->sec_mode, + in_flight(server), + atomic_read(&server->in_send), + atomic_read(&server->num_waiters)); +#ifdef CONFIG_NET_NS + if (server->net) + seq_printf(m, " Net namespace: %u ", server->net->ns.inum); +#endif /* NET_NS */ + +} + +static inline const char *smb_speed_to_str(size_t bps) +{ + size_t mbps = bps / 1000 / 1000; + + switch (mbps) { + case SPEED_10: + return "10Mbps"; + case SPEED_100: + return "100Mbps"; + case SPEED_1000: + return "1Gbps"; + case SPEED_2500: + return "2.5Gbps"; + case SPEED_5000: + return "5Gbps"; + case SPEED_10000: + return "10Gbps"; + case SPEED_14000: + return "14Gbps"; + case SPEED_20000: + return "20Gbps"; + case SPEED_25000: + return "25Gbps"; + case SPEED_40000: + return "40Gbps"; + case SPEED_50000: + return "50Gbps"; + case SPEED_56000: + return "56Gbps"; + case SPEED_100000: + return "100Gbps"; + case SPEED_200000: + return "200Gbps"; + case SPEED_400000: + return "400Gbps"; + case SPEED_800000: + return "800Gbps"; + default: + return "Unknown"; + } +} + +static void +cifs_dump_iface(struct seq_file *m, struct cifs_server_iface *iface) +{ + struct sockaddr_in *ipv4 = (struct sockaddr_in *)&iface->sockaddr; + struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)&iface->sockaddr; + + seq_printf(m, "\tSpeed: %s\n", smb_speed_to_str(iface->speed)); + seq_puts(m, "\t\tCapabilities: "); + if (iface->rdma_capable) + seq_puts(m, "rdma "); + if (iface->rss_capable) + seq_puts(m, "rss "); + if (!iface->rdma_capable && !iface->rss_capable) + seq_puts(m, "None"); + seq_putc(m, '\n'); + if (iface->sockaddr.ss_family == AF_INET) + seq_printf(m, "\t\tIPv4: %pI4\n", &ipv4->sin_addr); + else if (iface->sockaddr.ss_family == AF_INET6) + seq_printf(m, "\t\tIPv6: %pI6\n", &ipv6->sin6_addr); + if (!iface->is_active) + seq_puts(m, "\t\t[for-cleanup]\n"); +} + +static int cifs_debug_files_proc_show(struct seq_file *m, void *v) +{ + struct TCP_Server_Info *server; + struct cifs_ses *ses; + struct cifs_tcon *tcon; + struct cifsFileInfo *cfile; + struct inode *inode; + struct cifsInodeInfo *cinode; + char lease[4]; + int n; + + seq_puts(m, "# Version:1\n"); + seq_puts(m, "# Format:\n"); + seq_puts(m, "# "); +#ifdef CONFIG_CIFS_DEBUG2 + seq_puts(m, " \n"); +#else + seq_puts(m, " \n"); +#endif /* CIFS_DEBUG2 */ + spin_lock(&cifs_tcp_ses_lock); + list_for_each_entry(server, &cifs_tcp_ses_list, tcp_ses_list) { + list_for_each_entry(ses, &server->smb_ses_list, smb_ses_list) { + if (cifs_ses_exiting(ses)) + continue; + list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { + spin_lock(&tcon->open_file_lock); + list_for_each_entry(cfile, &tcon->openFileList, tlist) { + seq_printf(m, + "0x%x 0x%llx 0x%llx 0x%x %d %d %d %pd", + tcon->tid, + ses->Suid, + cfile->fid.persistent_fid, + cfile->f_flags, + cfile->count, + cfile->pid, + from_kuid(&init_user_ns, cfile->uid), + cfile->dentry); + + /* Append lease/oplock caching state as RHW letters */ + inode = d_inode(cfile->dentry); + cinode = NULL; + n = 0; + if (inode) { + cinode = CIFS_I(inode); + if (CIFS_CACHE_READ(cinode)) + lease[n++] = 'R'; + if (CIFS_CACHE_HANDLE(cinode)) + lease[n++] = 'H'; + if (CIFS_CACHE_WRITE(cinode)) + lease[n++] = 'W'; + } + lease[n] = '\0'; + seq_puts(m, " "); + if (n) + seq_printf(m, "%s", lease); + else + seq_puts(m, "NONE"); + + seq_puts(m, " "); + if (cinode && cinode->lease_granted) + seq_printf(m, "%pUl", cinode->lease_key); + else + seq_puts(m, "-"); + +#ifdef CONFIG_CIFS_DEBUG2 + seq_printf(m, " %llu", cfile->fid.mid); +#endif /* CONFIG_CIFS_DEBUG2 */ + seq_printf(m, "\n"); + } + spin_unlock(&tcon->open_file_lock); + } + } + } + spin_unlock(&cifs_tcp_ses_lock); + seq_putc(m, '\n'); + return 0; +} + +static int cifs_debug_dirs_proc_show(struct seq_file *m, void *v) +{ + struct list_head *stmp, *tmp, *tmp1; + struct TCP_Server_Info *server; + struct cifs_ses *ses; + struct cifs_tcon *tcon; + struct cached_fids *cfids; + struct cached_fid *cfid; + LIST_HEAD(entry); + + seq_puts(m, "# Version:1\n"); +#ifdef CONFIG_CIFS_DEBUG + seq_puts(m, "# Write 0 to this file to drop all cached directory entries\n"); +#endif /* CONFIG_CIFS_DEBUG */ + seq_puts(m, "# Format:\n"); + seq_puts(m, "# \n"); + + spin_lock(&cifs_tcp_ses_lock); + list_for_each(stmp, &cifs_tcp_ses_list) { + server = list_entry(stmp, struct TCP_Server_Info, + tcp_ses_list); + list_for_each(tmp, &server->smb_ses_list) { + ses = list_entry(tmp, struct cifs_ses, smb_ses_list); + list_for_each(tmp1, &ses->tcon_list) { + tcon = list_entry(tmp1, struct cifs_tcon, tcon_list); + cfids = tcon->cfids; + if (!cfids) + continue; + spin_lock(&cfids->cfid_list_lock); /* check lock ordering */ + seq_printf(m, "Num entries: %d, cached_dirents: %lu entries, %llu bytes\n", + cfids->num_entries, + (unsigned long)atomic_long_read(&cfids->total_dirents_entries), + (unsigned long long)atomic64_read(&cfids->total_dirents_bytes)); + list_for_each_entry(cfid, &cfids->entries, entry) { + seq_printf(m, "0x%x 0x%llx 0x%llx ", + tcon->tid, + ses->Suid, + cfid->fid.persistent_fid); + if (cfid->has_lease) + seq_printf(m, "%pUl ", cfid->fid.lease_key); + else + seq_puts(m, "- "); + seq_printf(m, "%s", cfid->path); + if (cfid->file_all_info_is_valid) + seq_printf(m, "\tvalid file info"); + if (cfid->dirents.is_valid) + seq_printf(m, ", valid dirents"); + if (!list_empty(&cfid->dirents.entries)) + seq_printf(m, ", dirents: %lu entries, %lu bytes", + cfid->dirents.entries_count, cfid->dirents.bytes_used); + seq_printf(m, "\n"); + } + spin_unlock(&cfids->cfid_list_lock); + } + } + } + spin_unlock(&cifs_tcp_ses_lock); + seq_putc(m, '\n'); + return 0; +} + +#ifdef CONFIG_CIFS_DEBUG +static int cifs_debug_dirs_proc_open(struct inode *inode, struct file *file) +{ + return single_open(file, cifs_debug_dirs_proc_show, NULL); +} + +/* Drop all cached directory entries across all CIFS mounts. */ +static ssize_t cifs_debug_dirs_proc_write(struct file *file, const char __user *buffer, + size_t count, loff_t *ppos) +{ + int rc, v; + + rc = kstrtoint_from_user(buffer, count, 10, &v); + if (rc) + return rc; + + if (v == 0) { + struct TCP_Server_Info *server; + struct cifs_ses *ses; + struct cifs_tcon *tcon; + + spin_lock(&cifs_tcp_ses_lock); + list_for_each_entry(server, &cifs_tcp_ses_list, tcp_ses_list) { + list_for_each_entry(ses, &server->smb_ses_list, smb_ses_list) { + if (cifs_ses_exiting(ses)) + continue; + list_for_each_entry(tcon, &ses->tcon_list, tcon_list) + invalidate_all_cached_dirs(tcon, false); + } + } + spin_unlock(&cifs_tcp_ses_lock); + } + + return count; +} + +static const struct proc_ops cifs_debug_dirs_proc_ops = { + .proc_open = cifs_debug_dirs_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = cifs_debug_dirs_proc_write, +}; +#endif /* CONFIG_CIFS_DEBUG */ + +static __always_inline const char *compression_alg_str(__le16 alg) +{ + switch (alg) { + case SMB3_COMPRESS_NONE: + return "NONE"; + case SMB3_COMPRESS_LZNT1: + return "LZNT1"; + case SMB3_COMPRESS_LZ77: + return "LZ77"; + case SMB3_COMPRESS_LZ77_HUFF: + return "LZ77-Huffman"; + case SMB3_COMPRESS_PATTERN: + return "Pattern_V1"; + default: + return "invalid"; + } +} + +static __always_inline const char *cipher_alg_str(__le16 cipher) +{ + switch (cipher) { + case SMB2_ENCRYPTION_AES128_CCM: + return "AES128-CCM"; + case SMB2_ENCRYPTION_AES128_GCM: + return "AES128-GCM"; + case SMB2_ENCRYPTION_AES256_CCM: + return "AES256-CCM"; + case SMB2_ENCRYPTION_AES256_GCM: + return "AES256-GCM"; + default: + return "UNKNOWN"; + } +} + +static int cifs_debug_data_proc_show(struct seq_file *m, void *v) +{ + struct mid_q_entry *mid_entry; + struct TCP_Server_Info *server; + struct TCP_Server_Info *chan_server; + struct cifs_ses *ses; + struct cifs_tcon *tcon; + struct cifs_server_iface *iface; + size_t iface_weight = 0, iface_min_speed = 0; + struct cifs_server_iface *last_iface = NULL; + int c, i, j; + + seq_puts(m, + "Display Internal CIFS Data Structures for Debugging\n" + "---------------------------------------------------\n"); + seq_printf(m, "CIFS Version %s\n", CIFS_VERSION); + seq_printf(m, "Features:"); +#ifdef CONFIG_CIFS_DFS_UPCALL + seq_printf(m, " DFS"); +#endif +#ifdef CONFIG_CIFS_FSCACHE + seq_printf(m, ",FSCACHE"); +#endif +#ifdef CONFIG_CIFS_SMB_DIRECT + seq_printf(m, ",SMB_DIRECT"); +#endif +#ifdef CONFIG_CIFS_STATS2 + seq_printf(m, ",STATS2"); +#else + seq_printf(m, ",STATS"); +#endif +#ifdef CONFIG_CIFS_DEBUG2 + seq_printf(m, ",DEBUG2"); +#elif defined(CONFIG_CIFS_DEBUG) + seq_printf(m, ",DEBUG"); +#endif +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + seq_printf(m, ",ALLOW_INSECURE_LEGACY"); +#endif +#ifdef CONFIG_CIFS_POSIX + seq_printf(m, ",CIFS_POSIX"); +#endif +#ifdef CONFIG_CIFS_UPCALL + seq_printf(m, ",UPCALL(SPNEGO)"); +#endif +#ifdef CONFIG_CIFS_XATTR + seq_printf(m, ",XATTR"); +#endif + seq_printf(m, ",ACL"); +#ifdef CONFIG_CIFS_SWN_UPCALL + seq_puts(m, ",WITNESS"); +#endif +#ifdef CONFIG_CIFS_COMPRESSION + seq_puts(m, ",COMPRESSION"); +#endif + seq_putc(m, '\n'); + seq_printf(m, "CIFSMaxBufSize: %d\n", CIFSMaxBufSize); + seq_printf(m, "Active VFS Requests: %d\n", GlobalTotalActiveXid); + + seq_printf(m, "\nServers: "); + + c = 0; + spin_lock(&cifs_tcp_ses_lock); + list_for_each_entry(server, &cifs_tcp_ses_list, tcp_ses_list) { + /* channel info will be printed as a part of sessions below */ + if (SERVER_IS_CHAN(server)) + continue; + + c++; + seq_printf(m, "\n%d) ConnectionId: 0x%llx ", + c, server->conn_id); + + spin_lock(&server->srv_lock); + if (server->hostname) + seq_printf(m, "Hostname: %s ", server->hostname); + seq_printf(m, "\nClientGUID: %pUL", server->client_guid); + spin_unlock(&server->srv_lock); +#ifdef CONFIG_CIFS_SMB_DIRECT + smbd_debug_proc_show(server, m); +#endif + seq_printf(m, "\nNumber of credits: %d,%d,%d Dialect 0x%x", + server->credits, + server->echo_credits, + server->oplock_credits, + server->dialect); + if (server->sign) + seq_printf(m, " signed"); + if (server->posix_ext_supported) + seq_printf(m, " posix"); + if (server->nosharesock) + seq_printf(m, " nosharesock"); + + seq_printf(m, "\nServer capabilities: 0x%x", server->capabilities); + + if (server->rdma) + seq_printf(m, "\nRDMA "); + seq_printf(m, "\nTCP status: %d Instance: %d" + "\nLocal Users To Server: %d SecMode: 0x%x Req On Wire: %d", + server->tcpStatus, + server->reconnect_instance, + server->srv_count, + server->sec_mode, in_flight(server)); +#ifdef CONFIG_NET_NS + if (server->net) + seq_printf(m, " Net namespace: %u ", server->net->ns.inum); +#endif /* NET_NS */ + + seq_printf(m, "\nIn Send: %d In MaxReq Wait: %d", + atomic_read(&server->in_send), + atomic_read(&server->num_waiters)); + + if (server->leaf_fullpath) { + seq_printf(m, "\nDFS leaf full path: %s", + server->leaf_fullpath); + } + + seq_puts(m, "\nCompression: "); + if (!IS_ENABLED(CONFIG_CIFS_COMPRESSION)) + seq_puts(m, "no built-in support"); + else if (!server->compression.requested) + seq_puts(m, "disabled on mount"); + else if (server->compression.enabled) + seq_printf(m, "enabled (%s)", compression_alg_str(server->compression.alg)); + else + seq_puts(m, "disabled (not supported by this server)"); + + /* Show negotiated encryption cipher, even if not required */ + seq_puts(m, "\nEncryption: "); + if (server->cipher_type) + seq_printf(m, "Negotiated cipher (%s)", cipher_alg_str(server->cipher_type)); + + seq_printf(m, "\n\n\tSessions: "); + i = 0; + list_for_each_entry(ses, &server->smb_ses_list, smb_ses_list) { + spin_lock(&ses->ses_lock); + if (ses->ses_status == SES_EXITING) { + spin_unlock(&ses->ses_lock); + continue; + } + i++; + if ((ses->serverDomain == NULL) || + (ses->serverOS == NULL) || + (ses->serverNOS == NULL)) { + seq_printf(m, "\n\t%d) Address: %s Uses: %d Capability: 0x%x\tSession Status: %d ", + i, ses->ip_addr, ses->ses_count, + ses->capabilities, ses->ses_status); + if (ses->session_flags & SMB2_SESSION_FLAG_IS_GUEST) + seq_printf(m, "Guest "); + else if (ses->session_flags & SMB2_SESSION_FLAG_IS_NULL) + seq_printf(m, "Anonymous "); + } else { + seq_printf(m, + "\n\t%d) Name: %s Domain: %s Uses: %d OS: %s " + "\n\tNOS: %s\tCapability: 0x%x" + "\n\tSMB session status: %d ", + i, ses->ip_addr, ses->serverDomain, + ses->ses_count, ses->serverOS, ses->serverNOS, + ses->capabilities, ses->ses_status); + } + if (ses->expired_pwd) + seq_puts(m, "password no longer valid "); + spin_unlock(&ses->ses_lock); + + seq_printf(m, "\n\tSecurity type: %s ", + get_security_type_str(server->ops->select_sectype(server, ses->sectype))); + + /* dump session id helpful for use with network trace */ + seq_printf(m, " SessionId: 0x%llx", ses->Suid); + if (ses->session_flags & SMB2_SESSION_FLAG_ENCRYPT_DATA) + seq_puts(m, " encrypted"); + if (ses->sign) + seq_puts(m, " signed"); + + seq_printf(m, "\n\tUser: %d Cred User: %d", + from_kuid(&init_user_ns, ses->linux_uid), + from_kuid(&init_user_ns, ses->cred_uid)); + + if (ses->dfs_root_ses) { + seq_printf(m, "\n\tDFS root session id: 0x%llx", + ses->dfs_root_ses->Suid); + } + + spin_lock(&ses->chan_lock); + if (CIFS_CHAN_NEEDS_RECONNECT(ses, 0)) + seq_puts(m, "\tPrimary channel: DISCONNECTED "); + if (CIFS_CHAN_IN_RECONNECT(ses, 0)) + seq_puts(m, "\t[RECONNECTING] "); + + if (ses->chan_count > 1) { + seq_printf(m, "\n\n\tExtra Channels: %zu ", + ses->chan_count-1); + for (j = 1; j < ses->chan_count; j++) { + cifs_dump_channel(m, j, &ses->chans[j]); + if (CIFS_CHAN_NEEDS_RECONNECT(ses, j)) + seq_puts(m, "\tDISCONNECTED "); + if (CIFS_CHAN_IN_RECONNECT(ses, j)) + seq_puts(m, "\t[RECONNECTING] "); + } + } + spin_unlock(&ses->chan_lock); + + seq_puts(m, "\n\n\tShares: "); + j = 0; + + seq_printf(m, "\n\t%d) IPC: ", j); + if (ses->tcon_ipc) + cifs_debug_tcon(m, ses->tcon_ipc); + else + seq_puts(m, "none\n"); + + list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { + ++j; + seq_printf(m, "\n\t%d) ", j); + cifs_debug_tcon(m, tcon); + } + + spin_lock(&ses->iface_lock); + if (ses->iface_count) + seq_printf(m, "\n\n\tServer interfaces: %zu" + "\tLast updated: %lu seconds ago", + ses->iface_count, + (jiffies - ses->iface_last_update) / HZ); + + last_iface = list_last_entry(&ses->iface_list, + struct cifs_server_iface, + iface_head); + iface_min_speed = last_iface->speed; + + j = 0; + list_for_each_entry(iface, &ses->iface_list, + iface_head) { + seq_printf(m, "\n\t%d)", ++j); + cifs_dump_iface(m, iface); + + iface_weight = iface->speed / iface_min_speed; + seq_printf(m, "\t\tWeight (cur,total): (%zu,%zu)" + "\n\t\tAllocated channels: %u\n", + iface->weight_fulfilled, + iface_weight, + iface->num_channels); + + if (is_ses_using_iface(ses, iface)) + seq_puts(m, "\t\t[CONNECTED]\n"); + } + spin_unlock(&ses->iface_lock); + + seq_puts(m, "\n\n\tMIDs: "); + spin_lock(&ses->chan_lock); + for (j = 0; j < ses->chan_count; j++) { + chan_server = ses->chans[j].server; + if (!chan_server) + continue; + + if (list_empty(&chan_server->pending_mid_q)) + continue; + + seq_printf(m, "\n\tServer ConnectionId: 0x%llx", + chan_server->conn_id); + spin_lock(&chan_server->mid_queue_lock); + list_for_each_entry(mid_entry, &chan_server->pending_mid_q, qhead) { + seq_printf(m, "\n\t\tState: %d com: %d pid: %d cbdata: %p mid %llu", + mid_entry->mid_state, + le16_to_cpu(mid_entry->command), + mid_entry->pid, + mid_entry->callback_data, + mid_entry->mid); + } + spin_unlock(&chan_server->mid_queue_lock); + } + spin_unlock(&ses->chan_lock); + seq_puts(m, "\n--\n"); + } + if (i == 0) + seq_printf(m, "\n\t\t[NONE]"); + } + if (c == 0) + seq_printf(m, "\n\t[NONE]"); + + spin_unlock(&cifs_tcp_ses_lock); + seq_putc(m, '\n'); + cifs_swn_dump(m); + + /* BB add code to dump additional info such as TCP session info now */ + return 0; +} + +static ssize_t cifs_stats_proc_write(struct file *file, + const char __user *buffer, size_t count, loff_t *ppos) +{ + bool bv; + int rc; + struct TCP_Server_Info *server; + struct cifs_ses *ses; + struct cifs_tcon *tcon; + + rc = kstrtobool_from_user(buffer, count, &bv); + if (rc == 0) { +#ifdef CONFIG_CIFS_STATS2 + int i; + + atomic_set(&total_buf_alloc_count, 0); + atomic_set(&total_small_buf_alloc_count, 0); +#endif /* CONFIG_CIFS_STATS2 */ + atomic_set(&tcpSesReconnectCount, 0); + atomic_set(&tconInfoReconnectCount, 0); + + spin_lock(&GlobalMid_Lock); + GlobalMaxActiveXid = 0; + GlobalCurrentXid = 0; + spin_unlock(&GlobalMid_Lock); + spin_lock(&cifs_tcp_ses_lock); + list_for_each_entry(server, &cifs_tcp_ses_list, tcp_ses_list) { + server->max_in_flight = 0; +#ifdef CONFIG_CIFS_STATS2 + for (i = 0; i < NUMBER_OF_SMB2_COMMANDS; i++) { + atomic_set(&server->num_cmds[i], 0); + atomic_set(&server->smb2slowcmd[i], 0); + server->time_per_cmd[i] = 0; + server->slowest_cmd[i] = 0; + server->fastest_cmd[0] = 0; + } +#endif /* CONFIG_CIFS_STATS2 */ + list_for_each_entry(ses, &server->smb_ses_list, smb_ses_list) { + if (cifs_ses_exiting(ses)) + continue; + list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { + atomic_set(&tcon->num_smbs_sent, 0); + spin_lock(&tcon->stat_lock); + tcon->bytes_read = 0; + tcon->bytes_written = 0; + tcon->stats_from_time = ktime_get_real_seconds(); + spin_unlock(&tcon->stat_lock); + if (server->ops->clear_stats) + server->ops->clear_stats(tcon); + } + } + } + spin_unlock(&cifs_tcp_ses_lock); + } else { + return rc; + } + + return count; +} + +static int cifs_stats_proc_show(struct seq_file *m, void *v) +{ + int i; +#ifdef CONFIG_CIFS_STATS2 + int j; +#endif /* STATS2 */ + struct TCP_Server_Info *server; + struct cifs_ses *ses; + struct cifs_tcon *tcon; + + seq_printf(m, "Resources in use\nCIFS Session: %d\n", + sesInfoAllocCount.counter); + seq_printf(m, "Share (unique mount targets): %d\n", + tconInfoAllocCount.counter); + seq_printf(m, "SMB Request/Response Buffer: %d Pool size: %d\n", + buf_alloc_count.counter, + cifs_min_rcv + tcpSesAllocCount.counter); + seq_printf(m, "SMB Small Req/Resp Buffer: %d Pool size: %d\n", + small_buf_alloc_count.counter, cifs_min_small); +#ifdef CONFIG_CIFS_STATS2 + seq_printf(m, "Total Large %d Small %d Allocations\n", + atomic_read(&total_buf_alloc_count), + atomic_read(&total_small_buf_alloc_count)); +#endif /* CONFIG_CIFS_STATS2 */ + + seq_printf(m, "Operations (MIDs): %d\n", atomic_read(&mid_count)); + seq_printf(m, + "\n%d session %d share reconnects\n", + tcpSesReconnectCount.counter, tconInfoReconnectCount.counter); + + seq_printf(m, + "Total vfs operations: %d maximum at one time: %d\n", + GlobalCurrentXid, GlobalMaxActiveXid); + + i = 0; + spin_lock(&cifs_tcp_ses_lock); + list_for_each_entry(server, &cifs_tcp_ses_list, tcp_ses_list) { + seq_printf(m, "\nMax requests in flight: %d", server->max_in_flight); +#ifdef CONFIG_CIFS_STATS2 + seq_puts(m, "\nTotal time spent processing by command. Time "); + seq_printf(m, "units are jiffies (%d per second)\n", HZ); + seq_puts(m, " SMB3 CMD\tNumber\tTotal Time\tFastest\tSlowest\n"); + seq_puts(m, " --------\t------\t----------\t-------\t-------\n"); + for (j = 0; j < NUMBER_OF_SMB2_COMMANDS; j++) + seq_printf(m, " %d\t\t%d\t%llu\t\t%u\t%u\n", j, + atomic_read(&server->num_cmds[j]), + server->time_per_cmd[j], + server->fastest_cmd[j], + server->slowest_cmd[j]); + for (j = 0; j < NUMBER_OF_SMB2_COMMANDS; j++) + if (atomic_read(&server->smb2slowcmd[j])) { + spin_lock(&server->srv_lock); + seq_printf(m, " %d slow responses from %s for command %d\n", + atomic_read(&server->smb2slowcmd[j]), + server->hostname, j); + spin_unlock(&server->srv_lock); + } +#endif /* STATS2 */ + list_for_each_entry(ses, &server->smb_ses_list, smb_ses_list) { + if (cifs_ses_exiting(ses)) + continue; + list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { + i++; + seq_printf(m, "\n%d) %s", i, tcon->tree_name); + if (tcon->need_reconnect) + seq_puts(m, "\tDISCONNECTED "); + seq_printf(m, "\nSMBs: %d since %ptTs UTC", + atomic_read(&tcon->num_smbs_sent), + &tcon->stats_from_time); + if (server->ops->print_stats) + server->ops->print_stats(m, tcon); + } + } + } + spin_unlock(&cifs_tcp_ses_lock); + + seq_putc(m, '\n'); + return 0; +} + +static int cifs_stats_proc_open(struct inode *inode, struct file *file) +{ + return single_open(file, cifs_stats_proc_show, NULL); +} + +static const struct proc_ops cifs_stats_proc_ops = { + .proc_open = cifs_stats_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = cifs_stats_proc_write, +}; + +#ifdef CONFIG_CIFS_SMB_DIRECT +#define PROC_FILE_DEFINE(name) \ +static ssize_t name##_write(struct file *file, const char __user *buffer, \ + size_t count, loff_t *ppos) \ +{ \ + int rc; \ + rc = kstrtoint_from_user(buffer, count, 10, &name); \ + if (rc) \ + return rc; \ + return count; \ +} \ +static int name##_proc_show(struct seq_file *m, void *v) \ +{ \ + seq_printf(m, "%d\n", name); \ + return 0; \ +} \ +static int name##_open(struct inode *inode, struct file *file) \ +{ \ + return single_open(file, name##_proc_show, NULL); \ +} \ +\ +static const struct proc_ops cifs_##name##_proc_fops = { \ + .proc_open = name##_open, \ + .proc_read = seq_read, \ + .proc_lseek = seq_lseek, \ + .proc_release = single_release, \ + .proc_write = name##_write, \ +} + +PROC_FILE_DEFINE(rdma_readwrite_threshold); +PROC_FILE_DEFINE(smbd_max_frmr_depth); +PROC_FILE_DEFINE(smbd_keep_alive_interval); +PROC_FILE_DEFINE(smbd_max_receive_size); +PROC_FILE_DEFINE(smbd_max_fragmented_recv_size); +PROC_FILE_DEFINE(smbd_max_send_size); +PROC_FILE_DEFINE(smbd_send_credit_target); +PROC_FILE_DEFINE(smbd_receive_credit_max); +#endif + +static struct proc_dir_entry *proc_fs_cifs; +static const struct proc_ops cifsFYI_proc_ops; +static const struct proc_ops cifs_lookup_cache_proc_ops; +static const struct proc_ops traceSMB_proc_ops; +static const struct proc_ops cifs_security_flags_proc_ops; +static const struct proc_ops cifs_linux_ext_proc_ops; +static const struct proc_ops cifs_mount_params_proc_ops; + +void +cifs_proc_init(void) +{ + proc_fs_cifs = proc_mkdir("fs/cifs", NULL); + if (proc_fs_cifs == NULL) + return; + + proc_create_single("DebugData", 0, proc_fs_cifs, + cifs_debug_data_proc_show); + + proc_create_single("open_files", 0400, proc_fs_cifs, + cifs_debug_files_proc_show); + +#ifdef CONFIG_CIFS_DEBUG + proc_create("open_dirs", 0600, proc_fs_cifs, &cifs_debug_dirs_proc_ops); +#else /* CONFIG_CIFS_DEBUG */ + proc_create_single("open_dirs", 0400, proc_fs_cifs, cifs_debug_dirs_proc_show); +#endif /* !CONFIG_CIFS_DEBUG */ + proc_create("Stats", 0644, proc_fs_cifs, &cifs_stats_proc_ops); + proc_create("cifsFYI", 0644, proc_fs_cifs, &cifsFYI_proc_ops); + proc_create("traceSMB", 0644, proc_fs_cifs, &traceSMB_proc_ops); + proc_create("LinuxExtensionsEnabled", 0644, proc_fs_cifs, + &cifs_linux_ext_proc_ops); + proc_create("SecurityFlags", 0644, proc_fs_cifs, + &cifs_security_flags_proc_ops); + proc_create("LookupCacheEnabled", 0644, proc_fs_cifs, + &cifs_lookup_cache_proc_ops); + + proc_create("mount_params", 0444, proc_fs_cifs, &cifs_mount_params_proc_ops); + +#ifdef CONFIG_CIFS_DFS_UPCALL + proc_create("dfscache", 0644, proc_fs_cifs, &dfscache_proc_ops); +#endif + +#ifdef CONFIG_CIFS_SMB_DIRECT + proc_create("rdma_readwrite_threshold", 0644, proc_fs_cifs, + &cifs_rdma_readwrite_threshold_proc_fops); + proc_create("smbd_max_frmr_depth", 0644, proc_fs_cifs, + &cifs_smbd_max_frmr_depth_proc_fops); + proc_create("smbd_keep_alive_interval", 0644, proc_fs_cifs, + &cifs_smbd_keep_alive_interval_proc_fops); + proc_create("smbd_max_receive_size", 0644, proc_fs_cifs, + &cifs_smbd_max_receive_size_proc_fops); + proc_create("smbd_max_fragmented_recv_size", 0644, proc_fs_cifs, + &cifs_smbd_max_fragmented_recv_size_proc_fops); + proc_create("smbd_max_send_size", 0644, proc_fs_cifs, + &cifs_smbd_max_send_size_proc_fops); + proc_create("smbd_send_credit_target", 0644, proc_fs_cifs, + &cifs_smbd_send_credit_target_proc_fops); + proc_create("smbd_receive_credit_max", 0644, proc_fs_cifs, + &cifs_smbd_receive_credit_max_proc_fops); +#endif +} + +void +cifs_proc_clean(void) +{ + if (proc_fs_cifs == NULL) + return; + + remove_proc_entry("DebugData", proc_fs_cifs); + remove_proc_entry("open_files", proc_fs_cifs); + remove_proc_entry("open_dirs", proc_fs_cifs); + remove_proc_entry("cifsFYI", proc_fs_cifs); + remove_proc_entry("traceSMB", proc_fs_cifs); + remove_proc_entry("Stats", proc_fs_cifs); + remove_proc_entry("SecurityFlags", proc_fs_cifs); + remove_proc_entry("LinuxExtensionsEnabled", proc_fs_cifs); + remove_proc_entry("LookupCacheEnabled", proc_fs_cifs); + remove_proc_entry("mount_params", proc_fs_cifs); + +#ifdef CONFIG_CIFS_DFS_UPCALL + remove_proc_entry("dfscache", proc_fs_cifs); +#endif +#ifdef CONFIG_CIFS_SMB_DIRECT + remove_proc_entry("rdma_readwrite_threshold", proc_fs_cifs); + remove_proc_entry("smbd_max_frmr_depth", proc_fs_cifs); + remove_proc_entry("smbd_keep_alive_interval", proc_fs_cifs); + remove_proc_entry("smbd_max_receive_size", proc_fs_cifs); + remove_proc_entry("smbd_max_fragmented_recv_size", proc_fs_cifs); + remove_proc_entry("smbd_max_send_size", proc_fs_cifs); + remove_proc_entry("smbd_send_credit_target", proc_fs_cifs); + remove_proc_entry("smbd_receive_credit_max", proc_fs_cifs); +#endif + remove_proc_entry("fs/cifs", NULL); +} + +static int cifsFYI_proc_show(struct seq_file *m, void *v) +{ + seq_printf(m, "%d\n", cifsFYI); + return 0; +} + +static int cifsFYI_proc_open(struct inode *inode, struct file *file) +{ + return single_open(file, cifsFYI_proc_show, NULL); +} + +static ssize_t cifsFYI_proc_write(struct file *file, const char __user *buffer, + size_t count, loff_t *ppos) +{ + char c[2] = { '\0' }; + bool bv; + int rc; + + rc = get_user(c[0], buffer); + if (rc) + return rc; + if (kstrtobool(c, &bv) == 0) + cifsFYI = bv; + else if ((c[0] > '1') && (c[0] <= '9')) + cifsFYI = (int) (c[0] - '0'); /* see cifs_debug.h for meanings */ + else + return -EINVAL; + + return count; +} + +static const struct proc_ops cifsFYI_proc_ops = { + .proc_open = cifsFYI_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = cifsFYI_proc_write, +}; + +static int cifs_linux_ext_proc_show(struct seq_file *m, void *v) +{ + seq_printf(m, "%d\n", linuxExtEnabled); + return 0; +} + +static int cifs_linux_ext_proc_open(struct inode *inode, struct file *file) +{ + return single_open(file, cifs_linux_ext_proc_show, NULL); +} + +static ssize_t cifs_linux_ext_proc_write(struct file *file, + const char __user *buffer, size_t count, loff_t *ppos) +{ + int rc; + + rc = kstrtobool_from_user(buffer, count, &linuxExtEnabled); + if (rc) + return rc; + + return count; +} + +static const struct proc_ops cifs_linux_ext_proc_ops = { + .proc_open = cifs_linux_ext_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = cifs_linux_ext_proc_write, +}; + +static int cifs_lookup_cache_proc_show(struct seq_file *m, void *v) +{ + seq_printf(m, "%d\n", lookupCacheEnabled); + return 0; +} + +static int cifs_lookup_cache_proc_open(struct inode *inode, struct file *file) +{ + return single_open(file, cifs_lookup_cache_proc_show, NULL); +} + +static ssize_t cifs_lookup_cache_proc_write(struct file *file, + const char __user *buffer, size_t count, loff_t *ppos) +{ + int rc; + + rc = kstrtobool_from_user(buffer, count, &lookupCacheEnabled); + if (rc) + return rc; + + return count; +} + +static const struct proc_ops cifs_lookup_cache_proc_ops = { + .proc_open = cifs_lookup_cache_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = cifs_lookup_cache_proc_write, +}; + +static int traceSMB_proc_show(struct seq_file *m, void *v) +{ + seq_printf(m, "%d\n", traceSMB); + return 0; +} + +static int traceSMB_proc_open(struct inode *inode, struct file *file) +{ + return single_open(file, traceSMB_proc_show, NULL); +} + +static ssize_t traceSMB_proc_write(struct file *file, const char __user *buffer, + size_t count, loff_t *ppos) +{ + int rc; + + rc = kstrtobool_from_user(buffer, count, &traceSMB); + if (rc) + return rc; + + return count; +} + +static const struct proc_ops traceSMB_proc_ops = { + .proc_open = traceSMB_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = traceSMB_proc_write, +}; + +static int cifs_security_flags_proc_show(struct seq_file *m, void *v) +{ + seq_printf(m, "0x%x\n", global_secflags); + return 0; +} + +static int cifs_security_flags_proc_open(struct inode *inode, struct file *file) +{ + return single_open(file, cifs_security_flags_proc_show, NULL); +} + +/* + * Ensure that if someone sets a MUST flag, that we disable all other MAY + * flags except for the ones corresponding to the given MUST flag. If there are + * multiple MUST flags, then try to prefer more secure ones. + */ +static void +cifs_security_flags_handle_must_flags(unsigned int *flags) +{ + unsigned int signflags = *flags & (CIFSSEC_MUST_SIGN | CIFSSEC_MUST_SEAL); + + if ((*flags & CIFSSEC_MUST_KRB5) == CIFSSEC_MUST_KRB5) + *flags = CIFSSEC_MUST_KRB5; + else if ((*flags & CIFSSEC_MUST_NTLMSSP) == CIFSSEC_MUST_NTLMSSP) + *flags = CIFSSEC_MUST_NTLMSSP; + else if ((*flags & CIFSSEC_MUST_NTLMV2) == CIFSSEC_MUST_NTLMV2) + *flags = CIFSSEC_MUST_NTLMV2; + + *flags |= signflags; +} + +static ssize_t cifs_security_flags_proc_write(struct file *file, + const char __user *buffer, size_t count, loff_t *ppos) +{ + int rc; + unsigned int flags; + char flags_string[12]; + bool bv; + + if ((count < 1) || (count > 11)) + return -EINVAL; + + memset(flags_string, 0, sizeof(flags_string)); + + if (copy_from_user(flags_string, buffer, count)) + return -EFAULT; + + if (count < 3) { + /* single char or single char followed by null */ + if (kstrtobool(flags_string, &bv) == 0) { + global_secflags = bv ? CIFSSEC_MAX : CIFSSEC_DEF; + return count; + } else if (!isdigit(flags_string[0])) { + cifs_dbg(VFS, "Invalid SecurityFlags: %s\n", + flags_string); + return -EINVAL; + } + } + + /* else we have a number */ + rc = kstrtouint(flags_string, 0, &flags); + if (rc) { + cifs_dbg(VFS, "Invalid SecurityFlags: %s\n", + flags_string); + return rc; + } + + cifs_dbg(FYI, "sec flags 0x%x\n", flags); + + if (flags == 0) { + cifs_dbg(VFS, "Invalid SecurityFlags: %s\n", flags_string); + return -EINVAL; + } + + if (flags & ~CIFSSEC_MASK) { + cifs_dbg(VFS, "Unsupported security flags: 0x%x\n", + flags & ~CIFSSEC_MASK); + return -EINVAL; + } + + cifs_security_flags_handle_must_flags(&flags); + + /* flags look ok - update the global security flags for cifs module */ + global_secflags = flags; + if (global_secflags & CIFSSEC_MUST_SIGN) { + /* requiring signing implies signing is allowed */ + global_secflags |= CIFSSEC_MAY_SIGN; + cifs_dbg(FYI, "packet signing now required\n"); + } else if ((global_secflags & CIFSSEC_MAY_SIGN) == 0) { + cifs_dbg(FYI, "packet signing disabled\n"); + } + /* BB should we turn on MAY flags for other MUST options? */ + return count; +} + +static const struct proc_ops cifs_security_flags_proc_ops = { + .proc_open = cifs_security_flags_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = cifs_security_flags_proc_write, +}; + +/* To make it easier to debug, can help to show mount params */ +static int cifs_mount_params_proc_show(struct seq_file *m, void *v) +{ + const struct fs_parameter_spec *p; + const char *type; + + for (p = smb3_fs_parameters; p->name; p++) { + /* cannot use switch with pointers... */ + if (!p->type) { + if (p->flags == fs_param_neg_with_no) + type = "noflag"; + else + type = "flag"; + } else if (p->type == fs_param_is_bool) + type = "bool"; + else if (p->type == fs_param_is_u32) + type = "u32"; + else if (p->type == fs_param_is_u64) + type = "u64"; + else if (p->type == fs_param_is_string) + type = "string"; + else + type = "unknown"; + + seq_printf(m, "%s:%s\n", p->name, type); + } + + return 0; +} + +static int cifs_mount_params_proc_open(struct inode *inode, struct file *file) +{ + return single_open(file, cifs_mount_params_proc_show, NULL); +} + +static const struct proc_ops cifs_mount_params_proc_ops = { + .proc_open = cifs_mount_params_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + /* No need for write for now */ + /* .proc_write = cifs_mount_params_proc_write, */ +}; + +#else +void cifs_proc_init(void) +{ +} + +void cifs_proc_clean(void) +{ +} +#endif /* PROC_FS */ diff --git a/fs/smb/client/cifs_debug.h b/fs/smb/client/cifs_debug.h new file mode 100644 index 000000000..00650929a --- /dev/null +++ b/fs/smb/client/cifs_debug.h @@ -0,0 +1,159 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * + * Copyright (c) International Business Machines Corp., 2000,2002 + * Modified by Steve French (sfrench@us.ibm.com) + */ + +#ifndef _H_CIFS_DEBUG +#define _H_CIFS_DEBUG + +#ifdef pr_fmt +#undef pr_fmt +#endif + +#define pr_fmt(fmt) "CIFS: " fmt + +void cifs_dump_mem(char *label, void *data, int length); +void cifs_dump_mids(struct TCP_Server_Info *server); +extern bool traceSMB; /* flag which enables the function below */ +void dump_smb(void *buf, int smb_buf_length); +#define CIFS_INFO 0x01 +#define CIFS_RC 0x02 +#define CIFS_TIMER 0x04 + +#define VFS 1 +#define FYI 2 +extern int cifsFYI; +#ifdef CONFIG_CIFS_DEBUG2 +#define NOISY 4 +#else +#define NOISY 0 +#endif +#define ONCE 8 + +/* + * debug ON + * -------- + */ +#ifdef CONFIG_CIFS_DEBUG + + +/* + * When adding tracepoints and debug messages we have various choices. + * Some considerations: + * + * Use cifs_dbg(VFS, ...) for things we always want logged, and the user to see + * cifs_info(...) slightly less important, admin can filter via loglevel > 6 + * cifs_dbg(FYI, ...) minor debugging messages, off by default + * trace_smb3_* ftrace functions are preferred for complex debug messages + * intended for developers or experienced admins, off by default + */ + +/* Information level messages, minor events */ +#define cifs_info_func(ratefunc, fmt, ...) \ + pr_info_ ## ratefunc(fmt, ##__VA_ARGS__) + +#define cifs_info(fmt, ...) \ + cifs_info_func(ratelimited, fmt, ##__VA_ARGS__) + +/* information message: e.g., configuration, major event */ +#define cifs_dbg_func(ratefunc, type, fmt, ...) \ +do { \ + if ((type) & FYI && cifsFYI & CIFS_INFO) { \ + pr_debug_ ## ratefunc("%s: " fmt, \ + __FILE__, ##__VA_ARGS__); \ + } else if ((type) & VFS) { \ + pr_err_ ## ratefunc("VFS: " fmt, ##__VA_ARGS__); \ + } else if ((type) & NOISY && (NOISY != 0)) { \ + pr_debug_ ## ratefunc(fmt, ##__VA_ARGS__); \ + } \ +} while (0) + +#define cifs_dbg(type, fmt, ...) \ +do { \ + if ((type) & ONCE) \ + cifs_dbg_func(once, type, fmt, ##__VA_ARGS__); \ + else \ + cifs_dbg_func(ratelimited, type, fmt, ##__VA_ARGS__); \ +} while (0) + +#define cifs_server_dbg_func(ratefunc, type, fmt, ...) \ +do { \ + spin_lock(&server->srv_lock); \ + if ((type) & FYI && cifsFYI & CIFS_INFO) { \ + pr_debug_ ## ratefunc("%s: \\\\%s " fmt, \ + __FILE__, server->hostname, \ + ##__VA_ARGS__); \ + } else if ((type) & VFS) { \ + pr_err_ ## ratefunc("VFS: \\\\%s " fmt, \ + server->hostname, ##__VA_ARGS__); \ + } else if ((type) & NOISY && (NOISY != 0)) { \ + pr_debug_ ## ratefunc("\\\\%s " fmt, \ + server->hostname, ##__VA_ARGS__); \ + } \ + spin_unlock(&server->srv_lock); \ +} while (0) + +#define cifs_server_dbg(type, fmt, ...) \ +do { \ + if ((type) & ONCE) \ + cifs_server_dbg_func(once, type, fmt, ##__VA_ARGS__); \ + else \ + cifs_server_dbg_func(ratelimited, type, fmt, \ + ##__VA_ARGS__); \ +} while (0) + +#define cifs_tcon_dbg_func(ratefunc, type, fmt, ...) \ +do { \ + const char *tn = ""; \ + if (tcon && tcon->tree_name) \ + tn = tcon->tree_name; \ + if ((type) & FYI && cifsFYI & CIFS_INFO) { \ + pr_debug_ ## ratefunc("%s: %s " fmt, \ + __FILE__, tn, ##__VA_ARGS__); \ + } else if ((type) & VFS) { \ + pr_err_ ## ratefunc("VFS: %s " fmt, tn, ##__VA_ARGS__); \ + } else if ((type) & NOISY && (NOISY != 0)) { \ + pr_debug_ ## ratefunc("%s " fmt, tn, ##__VA_ARGS__); \ + } \ +} while (0) + +#define cifs_tcon_dbg(type, fmt, ...) \ +do { \ + if ((type) & ONCE) \ + cifs_tcon_dbg_func(once, type, fmt, ##__VA_ARGS__); \ + else \ + cifs_tcon_dbg_func(ratelimited, type, fmt, \ + ##__VA_ARGS__); \ +} while (0) + +/* + * debug OFF + * --------- + */ +#else /* _CIFS_DEBUG */ +#define cifs_dbg(type, fmt, ...) \ +do { \ + if (0) \ + pr_debug(fmt, ##__VA_ARGS__); \ +} while (0) + +#define cifs_server_dbg(type, fmt, ...) \ +do { \ + if (0) \ + pr_debug("\\\\%s " fmt, \ + server->hostname, ##__VA_ARGS__); \ +} while (0) + +#define cifs_tcon_dbg(type, fmt, ...) \ +do { \ + if (0) \ + pr_debug("%s " fmt, tcon->tree_name, ##__VA_ARGS__); \ +} while (0) + +#define cifs_info(fmt, ...) \ + pr_info(fmt, ##__VA_ARGS__) +#endif + +#endif /* _H_CIFS_DEBUG */ diff --git a/fs/smb/client/cifs_fs_sb.h b/fs/smb/client/cifs_fs_sb.h new file mode 100644 index 000000000..d6494e1d9 --- /dev/null +++ b/fs/smb/client/cifs_fs_sb.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +/* + * + * Copyright (c) International Business Machines Corp., 2002,2004 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ +#include + +#ifndef _CIFS_FS_SB_H +#define _CIFS_FS_SB_H + +#include + +#define CIFS_MOUNT_NO_PERM 1 /* do not do client vfs_perm check */ +#define CIFS_MOUNT_SET_UID 2 /* set current's euid in create etc. */ +#define CIFS_MOUNT_SERVER_INUM 4 /* inode numbers from uniqueid from server */ +#define CIFS_MOUNT_DIRECT_IO 8 /* do not write nor read through page cache */ +#define CIFS_MOUNT_NO_XATTR 0x10 /* if set - disable xattr support */ +#define CIFS_MOUNT_MAP_SPECIAL_CHR 0x20 /* remap illegal chars in filenames */ +#define CIFS_MOUNT_POSIX_PATHS 0x40 /* Negotiate posix pathnames if possible*/ +#define CIFS_MOUNT_UNX_EMUL 0x80 /* Network compat with SFUnix emulation */ +#define CIFS_MOUNT_NO_BRL 0x100 /* No sending byte range locks to srv */ +#define CIFS_MOUNT_CIFS_ACL 0x200 /* send ACL requests to non-POSIX srv */ +#define CIFS_MOUNT_OVERR_UID 0x400 /* override uid returned from server */ +#define CIFS_MOUNT_OVERR_GID 0x800 /* override gid returned from server */ +#define CIFS_MOUNT_DYNPERM 0x1000 /* allow in-memory only mode setting */ +#define CIFS_MOUNT_NOPOSIXBRL 0x2000 /* mandatory not posix byte range lock */ +#define CIFS_MOUNT_NOSSYNC 0x4000 /* don't do slow SMBflush on every sync*/ +#define CIFS_MOUNT_FSCACHE 0x8000 /* local caching enabled */ +#define CIFS_MOUNT_MF_SYMLINKS 0x10000 /* Minshall+French Symlinks enabled */ +#define CIFS_MOUNT_MULTIUSER 0x20000 /* multiuser mount */ +#define CIFS_MOUNT_STRICT_IO 0x40000 /* strict cache mode */ +#define CIFS_MOUNT_RWPIDFORWARD 0x80000 /* use pid forwarding for rw */ +#define CIFS_MOUNT_POSIXACL 0x100000 /* mirror of SB_POSIXACL in mnt_cifs_flags */ +#define CIFS_MOUNT_CIFS_BACKUPUID 0x200000 /* backup intent bit for a user */ +#define CIFS_MOUNT_CIFS_BACKUPGID 0x400000 /* backup intent bit for a group */ +#define CIFS_MOUNT_MAP_SFM_CHR 0x800000 /* SFM/MAC mapping for illegal chars */ +#define CIFS_MOUNT_USE_PREFIX_PATH 0x1000000 /* make subpath with unaccessible + * root mountable + */ +#define CIFS_MOUNT_UID_FROM_ACL 0x2000000 /* try to get UID via special SID */ +#define CIFS_MOUNT_NO_HANDLE_CACHE 0x4000000 /* disable caching dir handles */ +#define CIFS_MOUNT_NO_DFS 0x8000000 /* disable DFS resolving */ +#define CIFS_MOUNT_MODE_FROM_SID 0x10000000 /* retrieve mode from special ACE */ +#define CIFS_MOUNT_RO_CACHE 0x20000000 /* assumes share will not change */ +#define CIFS_MOUNT_RW_CACHE 0x40000000 /* assumes only client accessing */ +#define CIFS_MOUNT_SHUTDOWN 0x80000000 + +struct cifs_sb_info { + struct rb_root tlink_tree; + struct list_head tcon_sb_link; + spinlock_t tlink_tree_lock; + struct tcon_link *master_tlink; + struct nls_table *local_nls; + struct smb3_fs_context *ctx; + atomic_t active; + atomic_t mnt_cifs_flags; + atomic_t outstanding_rreq; /* nr of rreqs not yet fully deinitialized */ + struct delayed_work prune_tlinks; + struct rcu_head rcu; + + /* only used when CIFS_MOUNT_USE_PREFIX_PATH is set */ + char *prepath; + + /* + * Indicate whether serverino option was turned off later + * (cifs_autodisable_serverino) in order to match new mounts. + */ + bool mnt_cifs_serverino_autodisabled; + /* + * Available once the mount has completed. + */ + struct dentry *root; +}; +#endif /* _CIFS_FS_SB_H */ diff --git a/fs/smb/client/cifs_ioctl.h b/fs/smb/client/cifs_ioctl.h new file mode 100644 index 000000000..147496ac9 --- /dev/null +++ b/fs/smb/client/cifs_ioctl.h @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +/* + * + * Structure definitions for io control for cifs/smb3 + * + * Copyright (c) 2015 Steve French + * + */ + +struct smb_mnt_fs_info { + __u32 version; /* 0001 */ + __u16 protocol_id; + __u16 tcon_flags; + __u32 vol_serial_number; + __u32 vol_create_time; + __u32 share_caps; + __u32 share_flags; + __u32 sector_flags; + __u32 optimal_sector_size; + __u32 max_bytes_chunk; + __u32 fs_attributes; + __u32 max_path_component; + __u32 device_type; + __u32 device_characteristics; + __u32 maximal_access; + __u64 cifs_posix_caps; +} __packed; + +struct smb_mnt_tcon_info { + __u32 tid; + __u64 session_id; +} __packed; + +struct smb_snapshot_array { + __u32 number_of_snapshots; + __u32 number_of_snapshots_returned; + __u32 snapshot_array_size; + /* snapshots[]; */ +} __packed; + +/* query_info flags */ +#define PASSTHRU_QUERY_INFO 0x00000000 +#define PASSTHRU_FSCTL 0x00000001 +#define PASSTHRU_SET_INFO 0x00000002 +struct smb_query_info { + __u32 info_type; + __u32 file_info_class; + __u32 additional_information; + __u32 flags; + __u32 input_buffer_length; + __u32 output_buffer_length; + /* char buffer[]; */ +} __packed; + +/* + * Dumping the commonly used 16 byte (e.g. CCM and GCM128) keys still supported + * for backlevel compatibility, but is not sufficient for dumping the less + * frequently used GCM256 (32 byte) keys (see the newer "CIFS_DUMP_FULL_KEY" + * ioctl for dumping decryption info for GCM256 mounts) + */ +struct smb3_key_debug_info { + __u64 Suid; + __u16 cipher_type; + __u8 auth_key[SMB2_NTLMV2_SESSKEY_SIZE]; + __u8 smb3encryptionkey[SMB3_SIGN_KEY_SIZE]; + __u8 smb3decryptionkey[SMB3_SIGN_KEY_SIZE]; +} __packed; + +/* + * Dump variable-sized keys + */ +struct smb3_full_key_debug_info { + /* INPUT: size of userspace buffer */ + __u32 in_size; + + /* + * INPUT: 0 for current user, otherwise session to dump + * OUTPUT: session id that was dumped + */ + __u64 session_id; + __u16 cipher_type; + __u8 session_key_length; + __u8 server_in_key_length; + __u8 server_out_key_length; + __u8 data[]; + /* + * return this struct with the keys appended at the end: + * __u8 session_key[session_key_length]; + * __u8 server_in_key[server_in_key_length]; + * __u8 server_out_key[server_out_key_length]; + */ +} __packed; + +struct smb3_notify { + __u32 completion_filter; + bool watch_tree; +} __packed; + +struct smb3_notify_info { + __u32 completion_filter; + bool watch_tree; + __u32 data_len; /* size of notify data below */ + __u8 notify_data[]; +} __packed; + +#define CIFS_IOCTL_MAGIC 0xCF +#define CIFS_IOC_COPYCHUNK_FILE _IOW(CIFS_IOCTL_MAGIC, 3, int) +#define CIFS_IOC_SET_INTEGRITY _IO(CIFS_IOCTL_MAGIC, 4) +#define CIFS_IOC_GET_MNT_INFO _IOR(CIFS_IOCTL_MAGIC, 5, struct smb_mnt_fs_info) +#define CIFS_ENUMERATE_SNAPSHOTS _IOR(CIFS_IOCTL_MAGIC, 6, struct smb_snapshot_array) +#define CIFS_QUERY_INFO _IOWR(CIFS_IOCTL_MAGIC, 7, struct smb_query_info) +#define CIFS_DUMP_KEY _IOWR(CIFS_IOCTL_MAGIC, 8, struct smb3_key_debug_info) +#define CIFS_IOC_NOTIFY _IOW(CIFS_IOCTL_MAGIC, 9, struct smb3_notify) +#define CIFS_DUMP_FULL_KEY _IOWR(CIFS_IOCTL_MAGIC, 10, struct smb3_full_key_debug_info) +#define CIFS_IOC_NOTIFY_INFO _IOWR(CIFS_IOCTL_MAGIC, 11, struct smb3_notify_info) +#define CIFS_IOC_GET_TCON_INFO _IOR(CIFS_IOCTL_MAGIC, 12, struct smb_mnt_tcon_info) +#define CIFS_IOC_SHUTDOWN _IOR('X', 125, __u32) + +/* + * Flags for going down operation + */ +#define CIFS_GOING_FLAGS_DEFAULT 0x0 /* going down */ +#define CIFS_GOING_FLAGS_LOGFLUSH 0x1 /* flush log but not data */ +#define CIFS_GOING_FLAGS_NOLOGFLUSH 0x2 /* don't flush log nor data */ diff --git a/fs/smb/client/cifs_spnego.c b/fs/smb/client/cifs_spnego.c new file mode 100644 index 000000000..44c407275 --- /dev/null +++ b/fs/smb/client/cifs_spnego.c @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * SPNEGO upcall management for CIFS + * + * Copyright (c) 2007 Red Hat, Inc. + * Author(s): Jeff Layton (jlayton@redhat.com) + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "cifsglob.h" +#include "cifs_spnego.h" +#include "cifs_debug.h" +#include "cifsproto.h" +static const struct cred *spnego_cred; + +/* create a new cifs key */ +static int +cifs_spnego_key_instantiate(struct key *key, struct key_preparsed_payload *prep) +{ + char *payload = kmemdup(prep->data, prep->datalen, GFP_KERNEL); + + if (!payload) + return -ENOMEM; + + /* attach the data */ + key->payload.data[0] = payload; + return 0; +} + +static void +cifs_spnego_key_destroy(struct key *key) +{ + kfree(key->payload.data[0]); +} + +static int +cifs_spnego_key_vet_description(const char *description) +{ + /* + * cifs.spnego descriptions are authority-bearing inputs to cifs.upcall. + * They are only valid when produced by CIFS while using the private + * spnego_cred installed below. Do not let userspace create this type + * of key through request_key(2)/add_key(2), since the helper treats + * pid/uid/creduid/upcall_target as kernel-originating fields. + */ + if (current_cred() != spnego_cred) + return -EPERM; + return 0; +} + +/* + * keytype for CIFS spnego keys + */ +struct key_type cifs_spnego_key_type = { + .name = "cifs.spnego", + .vet_description = cifs_spnego_key_vet_description, + .instantiate = cifs_spnego_key_instantiate, + .destroy = cifs_spnego_key_destroy, + .describe = user_describe, +}; + +/* length of longest version string e.g. strlen("ver=0xFF") */ +#define MAX_VER_STR_LEN 8 + +/* length of longest security mechanism name, eg in future could have + * strlen(";sec=ntlmsspi") */ +#define MAX_MECH_STR_LEN 13 + +/* strlen of ";host=" */ +#define HOST_KEY_LEN 6 + +/* strlen of ";ip4=" or ";ip6=" */ +#define IP_KEY_LEN 5 + +/* strlen of ";uid=0x" */ +#define UID_KEY_LEN 7 + +/* strlen of ";creduid=0x" */ +#define CREDUID_KEY_LEN 11 + +/* strlen of ";user=" */ +#define USER_KEY_LEN 6 + +/* strlen of ";pid=0x" */ +#define PID_KEY_LEN 7 + +/* strlen of ";upcall_target=" */ +#define UPCALL_TARGET_KEY_LEN 15 + +/* get a key struct with a SPNEGO security blob, suitable for session setup */ +struct key * +cifs_get_spnego_key(struct cifs_ses *sesInfo, + struct TCP_Server_Info *server) +{ + struct sockaddr_in *sa = (struct sockaddr_in *) &server->dstaddr; + struct sockaddr_in6 *sa6 = (struct sockaddr_in6 *) &server->dstaddr; + char *description, *dp; + size_t desc_len; + struct key *spnego_key; + const char *hostname = server->hostname; + + /* length of fields (with semicolons): ver=0xyz ip4=ipaddress + host=hostname sec=mechanism uid=0xFF user=username */ + desc_len = MAX_VER_STR_LEN + + HOST_KEY_LEN + strlen(hostname) + + IP_KEY_LEN + INET6_ADDRSTRLEN + + MAX_MECH_STR_LEN + + UID_KEY_LEN + (sizeof(uid_t) * 2) + + CREDUID_KEY_LEN + (sizeof(uid_t) * 2) + + PID_KEY_LEN + (sizeof(pid_t) * 2) + 1; + + if (sesInfo->user_name) + desc_len += USER_KEY_LEN + strlen(sesInfo->user_name); + + if (sesInfo->upcall_target == UPTARGET_MOUNT) + desc_len += UPCALL_TARGET_KEY_LEN + 5; // strlen("mount") + else + desc_len += UPCALL_TARGET_KEY_LEN + 3; // strlen("app") + + spnego_key = ERR_PTR(-ENOMEM); + description = kzalloc(desc_len, GFP_KERNEL); + if (description == NULL) + goto out; + + dp = description; + /* start with version and hostname portion of UNC string */ + spnego_key = ERR_PTR(-EINVAL); + dp += sprintf(dp, "ver=0x%x;host=%s;", CIFS_SPNEGO_UPCALL_VERSION, + hostname); + + /* add the server address */ + if (server->dstaddr.ss_family == AF_INET) + dp += sprintf(dp, "ip4=%pI4", &sa->sin_addr); + else if (server->dstaddr.ss_family == AF_INET6) + dp += sprintf(dp, "ip6=%pI6", &sa6->sin6_addr); + else + goto out; + + /* for now, only sec=krb5 and sec=mskrb5 and iakerb are valid */ + if (server->sec_kerberos) + dp += sprintf(dp, ";sec=krb5"); + else if (server->sec_mskerberos) + dp += sprintf(dp, ";sec=mskrb5"); + else if (server->sec_iakerb) + dp += sprintf(dp, ";sec=iakerb"); + else { + cifs_dbg(VFS, "unknown or missing server auth type, use krb5\n"); + dp += sprintf(dp, ";sec=krb5"); + } + + dp += sprintf(dp, ";uid=0x%x", + from_kuid_munged(&init_user_ns, sesInfo->linux_uid)); + + dp += sprintf(dp, ";creduid=0x%x", + from_kuid_munged(&init_user_ns, sesInfo->cred_uid)); + + if (sesInfo->user_name) + dp += sprintf(dp, ";user=%s", sesInfo->user_name); + + dp += sprintf(dp, ";pid=0x%x", current->pid); + + if (sesInfo->upcall_target == UPTARGET_MOUNT) + dp += sprintf(dp, ";upcall_target=mount"); + else + dp += sprintf(dp, ";upcall_target=app"); + + cifs_dbg(FYI, "key description = %s\n", description); + scoped_with_creds(spnego_cred) + spnego_key = request_key(&cifs_spnego_key_type, description, ""); + trace_smb3_kerberos_auth(server, sesInfo, PTR_ERR_OR_ZERO(spnego_key)); + +#ifdef CONFIG_CIFS_DEBUG2 + if (cifsFYI && !IS_ERR(spnego_key)) { + struct cifs_spnego_msg *msg = spnego_key->payload.data[0]; + cifs_dump_mem("SPNEGO reply blob:", msg->data, min(1024U, + msg->secblob_len + msg->sesskey_len)); + } +#endif /* CONFIG_CIFS_DEBUG2 */ + +out: + kfree(description); + return spnego_key; +} + +int +init_cifs_spnego(void) +{ + struct cred *cred; + struct key *keyring; + int ret; + + cifs_dbg(FYI, "Registering the %s key type\n", + cifs_spnego_key_type.name); + + /* + * Create an override credential set with special thread keyring for + * spnego upcalls. + */ + + cred = prepare_kernel_cred(&init_task); + if (!cred) + return -ENOMEM; + + keyring = keyring_alloc(".cifs_spnego", + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, cred, + (KEY_POS_ALL & ~KEY_POS_SETATTR) | + KEY_USR_VIEW | KEY_USR_READ, + KEY_ALLOC_NOT_IN_QUOTA, NULL, NULL); + if (IS_ERR(keyring)) { + ret = PTR_ERR(keyring); + goto failed_put_cred; + } + + ret = register_key_type(&cifs_spnego_key_type); + if (ret < 0) + goto failed_put_key; + + /* + * instruct request_key() to use this special keyring as a cache for + * the results it looks up + */ + set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags); + cred->thread_keyring = keyring; + cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING; + spnego_cred = cred; + + cifs_dbg(FYI, "cifs spnego keyring: %d\n", key_serial(keyring)); + return 0; + +failed_put_key: + key_put(keyring); +failed_put_cred: + put_cred(cred); + return ret; +} + +void +exit_cifs_spnego(void) +{ + key_revoke(spnego_cred->thread_keyring); + unregister_key_type(&cifs_spnego_key_type); + put_cred(spnego_cred); + cifs_dbg(FYI, "Unregistered %s key type\n", cifs_spnego_key_type.name); +} diff --git a/fs/smb/client/cifs_spnego.h b/fs/smb/client/cifs_spnego.h new file mode 100644 index 000000000..987768348 --- /dev/null +++ b/fs/smb/client/cifs_spnego.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +/* + * SPNEGO upcall management for CIFS + * + * Copyright (c) 2007 Red Hat, Inc. + * Author(s): Jeff Layton (jlayton@redhat.com) + * Steve French (sfrench@us.ibm.com) + * + */ + +#ifndef _CIFS_SPNEGO_H +#define _CIFS_SPNEGO_H + +#define CIFS_SPNEGO_UPCALL_VERSION 2 + +/* + * The version field should always be set to CIFS_SPNEGO_UPCALL_VERSION. + * The flags field is for future use. The request-key callout should set + * sesskey_len and secblob_len, and then concatenate the SessKey+SecBlob + * and stuff it in the data field. + */ +struct cifs_spnego_msg { + uint32_t version; + uint32_t flags; + uint32_t sesskey_len; + uint32_t secblob_len; + uint8_t data[]; +}; + +extern struct key_type cifs_spnego_key_type; +struct key *cifs_get_spnego_key(struct cifs_ses *sesInfo, + struct TCP_Server_Info *server); + +#endif /* _CIFS_SPNEGO_H */ diff --git a/fs/smb/client/cifs_spnego_negtokeninit.asn1 b/fs/smb/client/cifs_spnego_negtokeninit.asn1 new file mode 100644 index 000000000..181c08388 --- /dev/null +++ b/fs/smb/client/cifs_spnego_negtokeninit.asn1 @@ -0,0 +1,40 @@ +GSSAPI ::= + [APPLICATION 0] IMPLICIT SEQUENCE { + thisMech + OBJECT IDENTIFIER ({cifs_gssapi_this_mech}), + negotiationToken + NegotiationToken + } + +MechType ::= OBJECT IDENTIFIER ({cifs_neg_token_init_mech_type}) + +MechTypeList ::= SEQUENCE OF MechType + +NegHints ::= SEQUENCE { + hintName + [0] GeneralString OPTIONAL, + hintAddress + [1] OCTET STRING OPTIONAL + } + +NegTokenInit2 ::= + SEQUENCE { + mechTypes + [0] MechTypeList OPTIONAL, + reqFlags + [1] BIT STRING OPTIONAL, + mechToken + [2] OCTET STRING OPTIONAL, + negHints + [3] NegHints OPTIONAL, + mechListMIC + [3] OCTET STRING OPTIONAL + } + +NegotiationToken ::= + CHOICE { + negTokenInit + [0] NegTokenInit2, + negTokenTarg + [1] ANY + } diff --git a/fs/smb/client/cifs_swn.c b/fs/smb/client/cifs_swn.c new file mode 100644 index 000000000..fe10719e6 --- /dev/null +++ b/fs/smb/client/cifs_swn.c @@ -0,0 +1,876 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Witness Service client for CIFS + * + * Copyright (c) 2020 Samuel Cabrero + */ + +#include +#include +#include + +#include "cifs_swn.h" +#include "cifsglob.h" +#include "cifsproto.h" +#include "fscache.h" +#include "cifs_debug.h" +#include "netlink.h" + +static DEFINE_IDR(cifs_swnreg_idr); +static DEFINE_MUTEX(cifs_swnreg_idr_mutex); + +struct cifs_swn_reg { + int id; + struct kref ref_count; + + const char *net_name; + const char *share_name; + bool net_name_notify; + bool share_name_notify; + bool ip_notify; +}; + +struct cifs_swn_reg_info { + int id; + unsigned int ref_count; + const char *net_name; + const char *share_name; + bool net_name_notify; + bool share_name_notify; + bool ip_notify; +}; + +static void cifs_swn_snapshot_reg(struct cifs_swn_reg *swnreg, + struct cifs_swn_reg_info *info) +{ + info->id = swnreg->id; + info->ref_count = kref_read(&swnreg->ref_count); + info->net_name = swnreg->net_name; + info->share_name = swnreg->share_name; + info->net_name_notify = swnreg->net_name_notify; + info->share_name_notify = swnreg->share_name_notify; + info->ip_notify = swnreg->ip_notify; +} + +static int cifs_swn_dup_reg(struct cifs_swn_reg *swnreg, + struct cifs_swn_reg_info *info) +{ + cifs_swn_snapshot_reg(swnreg, info); + + info->net_name = kstrdup(swnreg->net_name, GFP_KERNEL); + if (!info->net_name) + return -ENOMEM; + + info->share_name = kstrdup(swnreg->share_name, GFP_KERNEL); + if (!info->share_name) { + kfree(info->net_name); + return -ENOMEM; + } + + return 0; +} + +static void cifs_swn_free_reg_info(struct cifs_swn_reg_info *info) +{ + kfree(info->net_name); + kfree(info->share_name); +} + +static int cifs_swn_auth_info_krb(struct cifs_tcon *tcon, struct sk_buff *skb) +{ + int ret; + + ret = nla_put_flag(skb, CIFS_GENL_ATTR_SWN_KRB_AUTH); + if (ret < 0) + return ret; + + return 0; +} + +static int cifs_swn_auth_info_ntlm(struct cifs_tcon *tcon, struct sk_buff *skb) +{ + int ret; + + if (tcon->ses->user_name != NULL) { + ret = nla_put_string(skb, CIFS_GENL_ATTR_SWN_USER_NAME, tcon->ses->user_name); + if (ret < 0) + return ret; + } + + if (tcon->ses->password != NULL) { + ret = nla_put_string(skb, CIFS_GENL_ATTR_SWN_PASSWORD, tcon->ses->password); + if (ret < 0) + return ret; + } + + if (tcon->ses->domainName != NULL) { + ret = nla_put_string(skb, CIFS_GENL_ATTR_SWN_DOMAIN_NAME, tcon->ses->domainName); + if (ret < 0) + return ret; + } + + return 0; +} + +/* + * Sends a register message to the userspace daemon based on the registration. + * The authentication information to connect to the witness service is bundled + * into the message. + */ +static int cifs_swn_send_register_message(struct cifs_swn_reg_info *swnreg, + struct cifs_tcon *tcon) +{ + struct sk_buff *skb; + struct genlmsghdr *hdr; + enum securityEnum authtype; + struct sockaddr_storage *addr; + int ret; + + skb = genlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); + if (!skb) + return -ENOMEM; + + hdr = genlmsg_put(skb, 0, 0, &cifs_genl_family, 0, CIFS_GENL_CMD_SWN_REGISTER); + if (hdr == NULL) { + ret = -ENOMEM; + goto nlmsg_fail; + } + + ret = nla_put_u32(skb, CIFS_GENL_ATTR_SWN_REGISTRATION_ID, swnreg->id); + if (ret < 0) + goto nlmsg_fail; + + ret = nla_put_string(skb, CIFS_GENL_ATTR_SWN_NET_NAME, swnreg->net_name); + if (ret < 0) + goto nlmsg_fail; + + ret = nla_put_string(skb, CIFS_GENL_ATTR_SWN_SHARE_NAME, swnreg->share_name); + if (ret < 0) + goto nlmsg_fail; + + /* + * If there is an address stored use it instead of the server address, because we are + * in the process of reconnecting to it after a share has been moved or we have been + * told to switch to it (client move message). In these cases we unregister from the + * server address and register to the new address when we receive the notification. + */ + if (tcon->ses->server->use_swn_dstaddr) + addr = &tcon->ses->server->swn_dstaddr; + else + addr = &tcon->ses->server->dstaddr; + + ret = nla_put(skb, CIFS_GENL_ATTR_SWN_IP, sizeof(struct sockaddr_storage), addr); + if (ret < 0) + goto nlmsg_fail; + + if (swnreg->net_name_notify) { + ret = nla_put_flag(skb, CIFS_GENL_ATTR_SWN_NET_NAME_NOTIFY); + if (ret < 0) + goto nlmsg_fail; + } + + if (swnreg->share_name_notify) { + ret = nla_put_flag(skb, CIFS_GENL_ATTR_SWN_SHARE_NAME_NOTIFY); + if (ret < 0) + goto nlmsg_fail; + } + + if (swnreg->ip_notify) { + ret = nla_put_flag(skb, CIFS_GENL_ATTR_SWN_IP_NOTIFY); + if (ret < 0) + goto nlmsg_fail; + } + + authtype = cifs_select_sectype(tcon->ses->server, tcon->ses->sectype); + switch (authtype) { + case Kerberos: + ret = cifs_swn_auth_info_krb(tcon, skb); + if (ret < 0) { + cifs_dbg(VFS, "%s: Failed to get kerberos auth info: %d\n", __func__, ret); + goto nlmsg_fail; + } + break; + case NTLMv2: + case RawNTLMSSP: + ret = cifs_swn_auth_info_ntlm(tcon, skb); + if (ret < 0) { + cifs_dbg(VFS, "%s: Failed to get NTLM auth info: %d\n", __func__, ret); + goto nlmsg_fail; + } + break; + default: + cifs_dbg(VFS, "%s: secType %d not supported!\n", __func__, authtype); + ret = -EINVAL; + goto nlmsg_fail; + } + + genlmsg_end(skb, hdr); + genlmsg_multicast(&cifs_genl_family, skb, 0, CIFS_GENL_MCGRP_SWN, GFP_ATOMIC); + + cifs_dbg(FYI, "%s: Message to register for network name %s with id %d sent\n", __func__, + swnreg->net_name, swnreg->id); + + return 0; + +nlmsg_fail: + genlmsg_cancel(skb, hdr); + nlmsg_free(skb); + return ret; +} + +/* + * Sends an uregister message to the userspace daemon based on the registration + */ +static int cifs_swn_send_unregister_message(struct cifs_swn_reg_info *swnreg, + struct cifs_tcon *tcon) +{ + struct sk_buff *skb; + struct genlmsghdr *hdr; + int ret; + + skb = genlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL); + if (skb == NULL) + return -ENOMEM; + + hdr = genlmsg_put(skb, 0, 0, &cifs_genl_family, 0, CIFS_GENL_CMD_SWN_UNREGISTER); + if (hdr == NULL) { + ret = -ENOMEM; + goto nlmsg_fail; + } + + ret = nla_put_u32(skb, CIFS_GENL_ATTR_SWN_REGISTRATION_ID, swnreg->id); + if (ret < 0) + goto nlmsg_fail; + + ret = nla_put_string(skb, CIFS_GENL_ATTR_SWN_NET_NAME, swnreg->net_name); + if (ret < 0) + goto nlmsg_fail; + + ret = nla_put_string(skb, CIFS_GENL_ATTR_SWN_SHARE_NAME, swnreg->share_name); + if (ret < 0) + goto nlmsg_fail; + + ret = nla_put(skb, CIFS_GENL_ATTR_SWN_IP, sizeof(struct sockaddr_storage), + &tcon->ses->server->dstaddr); + if (ret < 0) + goto nlmsg_fail; + + if (swnreg->net_name_notify) { + ret = nla_put_flag(skb, CIFS_GENL_ATTR_SWN_NET_NAME_NOTIFY); + if (ret < 0) + goto nlmsg_fail; + } + + if (swnreg->share_name_notify) { + ret = nla_put_flag(skb, CIFS_GENL_ATTR_SWN_SHARE_NAME_NOTIFY); + if (ret < 0) + goto nlmsg_fail; + } + + if (swnreg->ip_notify) { + ret = nla_put_flag(skb, CIFS_GENL_ATTR_SWN_IP_NOTIFY); + if (ret < 0) + goto nlmsg_fail; + } + + genlmsg_end(skb, hdr); + genlmsg_multicast(&cifs_genl_family, skb, 0, CIFS_GENL_MCGRP_SWN, GFP_ATOMIC); + + cifs_dbg(FYI, "%s: Message to unregister for network name %s with id %d sent\n", __func__, + swnreg->net_name, swnreg->id); + + return 0; + +nlmsg_fail: + genlmsg_cancel(skb, hdr); + nlmsg_free(skb); + return ret; +} + +/* + * Allocation-free mirror of extract_hostname() + extract_sharename() from + * fs/smb/client/unc.c. Those helpers kmalloc(GFP_KERNEL); this runs under + * cifs_tcp_ses_lock and tcon->tc_lock, both spinlocks, so we mirror their + * parsing in place against the caller's stable net_name/share_name strings. + * Keep in sync with unc.c. + */ +static bool cifs_swn_tcon_matches(struct cifs_tcon *tcon, + const char *net_name, + const char *share_name) +{ + const char *unc = tcon->tree_name; + const char *host, *share, *delim; + size_t host_len, share_len; + + if (!tcon->use_witness) + return false; + + /* extract_hostname: require strlen(unc) >= 3 */ + if (strnlen(unc, 3) < 3) + return false; + /* extract_hostname: skip all leading '\' characters */ + for (host = unc; *host == '\\'; host++) + ; + if (!*host) + return false; + delim = strchr(host, '\\'); + if (!delim) + return false; + host_len = delim - host; + if (strlen(net_name) != host_len || + strncasecmp(host, net_name, host_len)) + return false; + + /* extract_sharename: start at unc + 2, then first '\' onward */ + share = unc + 2; + delim = strchr(share, '\\'); + if (!delim) + return false; + share = delim + 1; + share_len = strlen(share); + + return strlen(share_name) == share_len && + !strncasecmp(share, share_name, share_len); +} + +/* + * One SWN registration id represents one net/share name pair. Multiple + * mounted tcons can therefore share the id. Pick a live representative at + * use time instead of caching the first tcon pointer in the registration. + */ +static struct cifs_tcon *cifs_swn_get_tcon(struct cifs_swn_reg_info *swnreg) +{ + struct TCP_Server_Info *server; + struct cifs_ses *ses; + struct cifs_tcon *tcon; + + spin_lock(&cifs_tcp_ses_lock); + list_for_each_entry(server, &cifs_tcp_ses_list, tcp_ses_list) { + list_for_each_entry(ses, &server->smb_ses_list, smb_ses_list) { + list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { + spin_lock(&tcon->tc_lock); + if (tcon->status == TID_EXITING || + !cifs_swn_tcon_matches(tcon, swnreg->net_name, + swnreg->share_name)) { + spin_unlock(&tcon->tc_lock); + continue; + } + ++tcon->tc_count; + trace_smb3_tcon_ref(tcon->debug_id, + tcon->tc_count, + netfs_trace_tcon_ref_get_swn_notify); + spin_unlock(&tcon->tc_lock); + spin_unlock(&cifs_tcp_ses_lock); + return tcon; + } + } + } + spin_unlock(&cifs_tcp_ses_lock); + return NULL; +} + +/* + * Try to find a matching registration for the tcon's server name and share name. + * Calls to this function must be protected by cifs_swnreg_idr_mutex. + * TODO Try to avoid memory allocations + */ +static struct cifs_swn_reg *cifs_find_swn_reg(struct cifs_tcon *tcon) +{ + struct cifs_swn_reg *swnreg; + int id; + const char *share_name; + const char *net_name; + + net_name = extract_hostname(tcon->tree_name); + if (IS_ERR(net_name)) { + int ret; + + ret = PTR_ERR(net_name); + cifs_dbg(VFS, "%s: failed to extract host name from target '%s': %d\n", + __func__, tcon->tree_name, ret); + return ERR_PTR(-EINVAL); + } + + share_name = extract_sharename(tcon->tree_name); + if (IS_ERR(share_name)) { + int ret; + + ret = PTR_ERR(share_name); + cifs_dbg(VFS, "%s: failed to extract share name from target '%s': %d\n", + __func__, tcon->tree_name, ret); + kfree(net_name); + return ERR_PTR(-EINVAL); + } + + idr_for_each_entry(&cifs_swnreg_idr, swnreg, id) { + if (strcasecmp(swnreg->net_name, net_name) != 0 + || strcasecmp(swnreg->share_name, share_name) != 0) { + continue; + } + + cifs_dbg(FYI, "Existing swn registration for %s:%s found\n", swnreg->net_name, + swnreg->share_name); + + kfree(net_name); + kfree(share_name); + + return swnreg; + } + + kfree(net_name); + kfree(share_name); + + return ERR_PTR(-EEXIST); +} + +/* + * Get a registration for the tcon's server and share name, allocating a new one if it does not + * exist. + */ +static struct cifs_swn_reg *cifs_get_swn_reg(struct cifs_tcon *tcon) +{ + struct cifs_swn_reg *reg = NULL; + int ret; + + mutex_lock(&cifs_swnreg_idr_mutex); + + /* Check if we are already registered for this network and share names */ + reg = cifs_find_swn_reg(tcon); + if (!IS_ERR(reg)) { + kref_get(®->ref_count); + goto unlock; + } else if (PTR_ERR(reg) != -EEXIST) { + goto unlock; + } + + reg = kmalloc_obj(struct cifs_swn_reg, GFP_KERNEL); + if (reg == NULL) { + ret = -ENOMEM; + goto fail_unlock; + } + + kref_init(®->ref_count); + + reg->id = idr_alloc(&cifs_swnreg_idr, reg, 1, 0, GFP_KERNEL); + if (reg->id < 0) { + cifs_dbg(FYI, "%s: failed to allocate registration id\n", __func__); + ret = reg->id; + goto fail; + } + + reg->net_name = extract_hostname(tcon->tree_name); + if (IS_ERR(reg->net_name)) { + ret = PTR_ERR(reg->net_name); + cifs_dbg(VFS, "%s: failed to extract host name from target: %d\n", __func__, ret); + goto fail_idr; + } + + reg->share_name = extract_sharename(tcon->tree_name); + if (IS_ERR(reg->share_name)) { + ret = PTR_ERR(reg->share_name); + cifs_dbg(VFS, "%s: failed to extract share name from target: %d\n", __func__, ret); + goto fail_net_name; + } + + reg->net_name_notify = true; + reg->share_name_notify = true; + reg->ip_notify = (tcon->capabilities & SMB2_SHARE_CAP_SCALEOUT); +unlock: + mutex_unlock(&cifs_swnreg_idr_mutex); + + return reg; + +fail_net_name: + kfree(reg->net_name); +fail_idr: + idr_remove(&cifs_swnreg_idr, reg->id); +fail: + kfree(reg); +fail_unlock: + mutex_unlock(&cifs_swnreg_idr_mutex); + return ERR_PTR(ret); +} + +static void cifs_swn_reg_release(struct kref *ref) +{ + struct cifs_swn_reg *swnreg = container_of(ref, struct cifs_swn_reg, ref_count); + + idr_remove(&cifs_swnreg_idr, swnreg->id); + kfree(swnreg->net_name); + kfree(swnreg->share_name); + kfree(swnreg); +} + +static void cifs_put_swn_reg_locked(struct cifs_swn_reg *swnreg, + struct cifs_tcon *tcon) +{ + if (kref_read(&swnreg->ref_count) == 1) { + struct cifs_swn_reg_info swnreg_info; + int ret; + + cifs_swn_snapshot_reg(swnreg, &swnreg_info); + ret = cifs_swn_send_unregister_message(&swnreg_info, tcon); + if (ret < 0) + cifs_dbg(VFS, "%s: Failed to send unregister message: %d\n", + __func__, ret); + } + + kref_put(&swnreg->ref_count, cifs_swn_reg_release); +} + +static int cifs_swn_resource_state_changed(struct cifs_tcon *tcon, const char *name, int state) +{ + switch (state) { + case CIFS_SWN_RESOURCE_STATE_UNAVAILABLE: + cifs_dbg(FYI, "%s: resource name '%s' become unavailable\n", __func__, name); + cifs_signal_cifsd_for_reconnect(tcon->ses->server, true); + break; + case CIFS_SWN_RESOURCE_STATE_AVAILABLE: + cifs_dbg(FYI, "%s: resource name '%s' become available\n", __func__, name); + cifs_signal_cifsd_for_reconnect(tcon->ses->server, true); + break; + case CIFS_SWN_RESOURCE_STATE_UNKNOWN: + cifs_dbg(FYI, "%s: resource name '%s' changed to unknown state\n", __func__, name); + break; + } + return 0; +} + +static bool cifs_sockaddr_equal(struct sockaddr_storage *addr1, struct sockaddr_storage *addr2) +{ + if (addr1->ss_family != addr2->ss_family) + return false; + + if (addr1->ss_family == AF_INET) { + return (memcmp(&((const struct sockaddr_in *)addr1)->sin_addr, + &((const struct sockaddr_in *)addr2)->sin_addr, + sizeof(struct in_addr)) == 0); + } + + if (addr1->ss_family == AF_INET6) { + return (memcmp(&((const struct sockaddr_in6 *)addr1)->sin6_addr, + &((const struct sockaddr_in6 *)addr2)->sin6_addr, + sizeof(struct in6_addr)) == 0); + } + + return false; +} + +static int cifs_swn_store_swn_addr(const struct sockaddr_storage *new, + const struct sockaddr_storage *old, + struct sockaddr_storage *dst) +{ + __be16 port = cpu_to_be16(CIFS_PORT); + + if (old->ss_family == AF_INET) { + struct sockaddr_in *ipv4 = (struct sockaddr_in *)old; + + port = ipv4->sin_port; + } else if (old->ss_family == AF_INET6) { + struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)old; + + port = ipv6->sin6_port; + } + + if (new->ss_family == AF_INET) { + struct sockaddr_in *ipv4 = (struct sockaddr_in *)new; + + ipv4->sin_port = port; + } else if (new->ss_family == AF_INET6) { + struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)new; + + ipv6->sin6_port = port; + } + + *dst = *new; + + return 0; +} + +static int cifs_swn_reconnect(struct cifs_tcon *tcon, struct sockaddr_storage *addr) +{ + int ret = 0; + + /* Store the reconnect address */ + cifs_server_lock(tcon->ses->server); + if (cifs_sockaddr_equal(&tcon->ses->server->dstaddr, addr)) + goto unlock; + + ret = cifs_swn_store_swn_addr(addr, &tcon->ses->server->dstaddr, + &tcon->ses->server->swn_dstaddr); + if (ret < 0) { + cifs_dbg(VFS, "%s: failed to store address: %d\n", __func__, ret); + goto unlock; + } + tcon->ses->server->use_swn_dstaddr = true; + + /* + * Unregister to stop receiving notifications for the old IP address. + */ + ret = cifs_swn_unregister(tcon); + if (ret < 0) { + cifs_dbg(VFS, "%s: Failed to unregister for witness notifications: %d\n", + __func__, ret); + goto unlock; + } + + /* + * And register to receive notifications for the new IP address now that we have + * stored the new address. + */ + ret = cifs_swn_register(tcon); + if (ret < 0) { + cifs_dbg(VFS, "%s: Failed to register for witness notifications: %d\n", + __func__, ret); + goto unlock; + } + + cifs_signal_cifsd_for_reconnect(tcon->ses->server, false); + +unlock: + cifs_server_unlock(tcon->ses->server); + + return ret; +} + +static int cifs_swn_client_move(struct cifs_tcon *tcon, struct sockaddr_storage *addr) +{ + struct sockaddr_in *ipv4 = (struct sockaddr_in *)addr; + struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)addr; + + if (addr->ss_family == AF_INET) + cifs_dbg(FYI, "%s: move to %pI4\n", __func__, &ipv4->sin_addr); + else if (addr->ss_family == AF_INET6) + cifs_dbg(FYI, "%s: move to %pI6\n", __func__, &ipv6->sin6_addr); + + return cifs_swn_reconnect(tcon, addr); +} + +int cifs_swn_notify(struct sk_buff *skb, struct genl_info *info) +{ + struct cifs_swn_reg *swnreg; + struct cifs_swn_reg_info swnreg_info; + struct cifs_tcon *tcon; + char name[256]; + int type; + int ret = 0; + + if (info->attrs[CIFS_GENL_ATTR_SWN_REGISTRATION_ID]) { + int swnreg_id; + + swnreg_id = nla_get_u32(info->attrs[CIFS_GENL_ATTR_SWN_REGISTRATION_ID]); + mutex_lock(&cifs_swnreg_idr_mutex); + swnreg = idr_find(&cifs_swnreg_idr, swnreg_id); + if (swnreg == NULL) { + mutex_unlock(&cifs_swnreg_idr_mutex); + cifs_dbg(FYI, "%s: registration id %d not found\n", __func__, swnreg_id); + return -EINVAL; + } + ret = cifs_swn_dup_reg(swnreg, &swnreg_info); + mutex_unlock(&cifs_swnreg_idr_mutex); + if (ret) + return ret; + } else { + cifs_dbg(FYI, "%s: missing registration id attribute\n", __func__); + return -EINVAL; + } + + tcon = cifs_swn_get_tcon(&swnreg_info); + if (!tcon) { + cifs_dbg(FYI, "%s: registration id %d has no live tcon\n", + __func__, swnreg_info.id); + ret = -ENODEV; + goto free_info; + } + + if (info->attrs[CIFS_GENL_ATTR_SWN_NOTIFICATION_TYPE]) { + type = nla_get_u32(info->attrs[CIFS_GENL_ATTR_SWN_NOTIFICATION_TYPE]); + } else { + cifs_dbg(FYI, "%s: missing notification type attribute\n", __func__); + ret = -EINVAL; + goto out; + } + + switch (type) { + case CIFS_SWN_NOTIFICATION_RESOURCE_CHANGE: { + int state; + + if (info->attrs[CIFS_GENL_ATTR_SWN_RESOURCE_NAME]) { + nla_strscpy(name, info->attrs[CIFS_GENL_ATTR_SWN_RESOURCE_NAME], + sizeof(name)); + } else { + cifs_dbg(FYI, "%s: missing resource name attribute\n", __func__); + ret = -EINVAL; + goto out; + } + if (info->attrs[CIFS_GENL_ATTR_SWN_RESOURCE_STATE]) { + state = nla_get_u32(info->attrs[CIFS_GENL_ATTR_SWN_RESOURCE_STATE]); + } else { + cifs_dbg(FYI, "%s: missing resource state attribute\n", __func__); + ret = -EINVAL; + goto out; + } + ret = cifs_swn_resource_state_changed(tcon, name, state); + break; + } + case CIFS_SWN_NOTIFICATION_CLIENT_MOVE: { + struct sockaddr_storage addr; + + if (info->attrs[CIFS_GENL_ATTR_SWN_IP]) { + nla_memcpy(&addr, info->attrs[CIFS_GENL_ATTR_SWN_IP], sizeof(addr)); + } else { + cifs_dbg(FYI, "%s: missing IP address attribute\n", __func__); + ret = -EINVAL; + goto out; + } + ret = cifs_swn_client_move(tcon, &addr); + break; + } + default: + cifs_dbg(FYI, "%s: unknown notification type %d\n", __func__, type); + break; + } + +out: + cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_swn_notify); +free_info: + cifs_swn_free_reg_info(&swnreg_info); + return ret; +} + +int cifs_swn_register(struct cifs_tcon *tcon) +{ + struct cifs_swn_reg *swnreg; + struct cifs_swn_reg_info swnreg_info; + int ret; + + swnreg = cifs_get_swn_reg(tcon); + if (IS_ERR(swnreg)) + return PTR_ERR(swnreg); + + cifs_swn_snapshot_reg(swnreg, &swnreg_info); + ret = cifs_swn_send_register_message(&swnreg_info, tcon); + if (ret < 0) { + cifs_dbg(VFS, "%s: Failed to send swn register message: %d\n", __func__, ret); + /* Do not put the swnreg or return error, the echo task will retry */ + } + + return 0; +} + +int cifs_swn_unregister(struct cifs_tcon *tcon) +{ + struct cifs_swn_reg *swnreg; + + mutex_lock(&cifs_swnreg_idr_mutex); + + swnreg = cifs_find_swn_reg(tcon); + if (IS_ERR(swnreg)) { + mutex_unlock(&cifs_swnreg_idr_mutex); + return PTR_ERR(swnreg); + } + + cifs_put_swn_reg_locked(swnreg, tcon); + mutex_unlock(&cifs_swnreg_idr_mutex); + + return 0; +} + +/* + * Snapshot one registration under cifs_swnreg_idr_mutex and return. Callers + * intentionally do the per-registration network/genlmsg work without the + * mutex held, both to keep the critical section short and to avoid nesting + * cifs_swnreg_idr_mutex inside the higher tc_lock when a live tcon is then + * pinned for the send. + */ +static int cifs_swn_get_next_reg_info(int *id, struct cifs_swn_reg_info *info) +{ + struct cifs_swn_reg *swnreg; + int ret = 0; + + mutex_lock(&cifs_swnreg_idr_mutex); + swnreg = idr_get_next(&cifs_swnreg_idr, id); + if (swnreg) { + ret = cifs_swn_dup_reg(swnreg, info); + if (!ret) { + *id = swnreg->id + 1; + ret = 1; + } + } + mutex_unlock(&cifs_swnreg_idr_mutex); + + return ret; +} + +void cifs_swn_dump(struct seq_file *m) +{ + struct cifs_swn_reg_info swnreg_info; + struct cifs_tcon *tcon; + struct sockaddr_in *sa; + struct sockaddr_in6 *sa6; + int id = 0; + int ret; + + seq_puts(m, "Witness registrations:"); + + while ((ret = cifs_swn_get_next_reg_info(&id, &swnreg_info)) > 0) { + seq_printf(m, "\nId: %u Refs: %u Network name: '%s'%s Share name: '%s'%s Ip address: ", + swnreg_info.id, swnreg_info.ref_count, + swnreg_info.net_name, swnreg_info.net_name_notify ? "(y)" : "(n)", + swnreg_info.share_name, swnreg_info.share_name_notify ? "(y)" : "(n)"); + + tcon = cifs_swn_get_tcon(&swnreg_info); + if (!tcon) { + seq_puts(m, "(no live tcon)"); + goto next; + } + + switch (tcon->ses->server->dstaddr.ss_family) { + case AF_INET: + sa = (struct sockaddr_in *)&tcon->ses->server->dstaddr; + seq_printf(m, "%pI4", &sa->sin_addr.s_addr); + break; + case AF_INET6: + sa6 = (struct sockaddr_in6 *)&tcon->ses->server->dstaddr; + seq_printf(m, "%pI6", &sa6->sin6_addr.s6_addr); + if (sa6->sin6_scope_id) + seq_printf(m, "%%%u", sa6->sin6_scope_id); + break; + default: + seq_puts(m, "(unknown)"); + } + cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_swn_notify); +next: + seq_printf(m, "%s", swnreg_info.ip_notify ? "(y)" : "(n)"); + cifs_swn_free_reg_info(&swnreg_info); + } + if (ret < 0) + seq_printf(m, "\nFailed to snapshot witness registration: %d", ret); + seq_puts(m, "\n"); +} + +void cifs_swn_check(void) +{ + struct cifs_swn_reg_info swnreg_info; + struct cifs_tcon *tcon; + int id = 0; + int ret; + + while ((ret = cifs_swn_get_next_reg_info(&id, &swnreg_info)) > 0) { + tcon = cifs_swn_get_tcon(&swnreg_info); + if (!tcon) { + cifs_dbg(FYI, "%s: registration id %d has no live tcon\n", + __func__, swnreg_info.id); + goto free_info; + } + + ret = cifs_swn_send_register_message(&swnreg_info, tcon); + if (ret < 0) + cifs_dbg(FYI, "%s: Failed to send register message: %d\n", __func__, ret); + cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_swn_notify); +free_info: + cifs_swn_free_reg_info(&swnreg_info); + } + if (ret < 0) + cifs_dbg(FYI, "%s: Failed to snapshot registration: %d\n", __func__, ret); +} diff --git a/fs/smb/client/cifs_swn.h b/fs/smb/client/cifs_swn.h new file mode 100644 index 000000000..955d07b69 --- /dev/null +++ b/fs/smb/client/cifs_swn.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Witness Service client for CIFS + * + * Copyright (c) 2020 Samuel Cabrero + */ + +#ifndef _CIFS_SWN_H +#define _CIFS_SWN_H +#include "cifsglob.h" + +struct cifs_tcon; +struct sk_buff; +struct genl_info; + +#ifdef CONFIG_CIFS_SWN_UPCALL +int cifs_swn_register(struct cifs_tcon *tcon); + +int cifs_swn_unregister(struct cifs_tcon *tcon); + +int cifs_swn_notify(struct sk_buff *skb, struct genl_info *info); + +void cifs_swn_dump(struct seq_file *m); + +void cifs_swn_check(void); + +static inline bool cifs_swn_set_server_dstaddr(struct TCP_Server_Info *server) +{ + if (server->use_swn_dstaddr) { + server->dstaddr = server->swn_dstaddr; + return true; + } + return false; +} + +static inline void cifs_swn_reset_server_dstaddr(struct TCP_Server_Info *server) +{ + server->use_swn_dstaddr = false; +} + +#else + +static inline int cifs_swn_register(struct cifs_tcon *tcon) { return 0; } +static inline int cifs_swn_unregister(struct cifs_tcon *tcon) { return 0; } +static inline int cifs_swn_notify(struct sk_buff *s, struct genl_info *i) { return 0; } +static inline void cifs_swn_dump(struct seq_file *m) {} +static inline void cifs_swn_check(void) {} +static inline bool cifs_swn_set_server_dstaddr(struct TCP_Server_Info *server) { return false; } +static inline void cifs_swn_reset_server_dstaddr(struct TCP_Server_Info *server) {} + +#endif /* CONFIG_CIFS_SWN_UPCALL */ +#endif /* _CIFS_SWN_H */ diff --git a/fs/smb/client/cifs_unicode.c b/fs/smb/client/cifs_unicode.c new file mode 100644 index 000000000..4a8a591f4 --- /dev/null +++ b/fs/smb/client/cifs_unicode.c @@ -0,0 +1,631 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * + * Copyright (c) International Business Machines Corp., 2000,2009 + * Modified by Steve French (sfrench@us.ibm.com) + */ +#include +#include +#include +#include "cifs_fs_sb.h" +#include "cifs_unicode.h" +#include "cifsglob.h" +#include "cifs_debug.h" + +/* Convert character using the SFU - "Services for Unix" remapping range */ +static bool +convert_sfu_char(const __u16 src_char, char *target) +{ + /* + * BB: Cannot handle remapping UNI_SLASH until all the calls to + * build_path_from_dentry are modified, as they use slash as + * separator. + */ + switch (src_char) { + case UNI_COLON: + *target = ':'; + break; + case UNI_ASTERISK: + *target = '*'; + break; + case UNI_QUESTION: + *target = '?'; + break; + case UNI_PIPE: + *target = '|'; + break; + case UNI_GRTRTHAN: + *target = '>'; + break; + case UNI_LESSTHAN: + *target = '<'; + break; + default: + return false; + } + return true; +} + +/* Convert character using the SFM - "Services for Mac" remapping range */ +static bool +convert_sfm_char(const __u16 src_char, char *target) +{ + if (src_char >= 0xF001 && src_char <= 0xF01F) { + *target = src_char - 0xF000; + return true; + } + switch (src_char) { + case SFM_COLON: + *target = ':'; + break; + case SFM_DOUBLEQUOTE: + *target = '"'; + break; + case SFM_ASTERISK: + *target = '*'; + break; + case SFM_QUESTION: + *target = '?'; + break; + case SFM_PIPE: + *target = '|'; + break; + case SFM_GRTRTHAN: + *target = '>'; + break; + case SFM_LESSTHAN: + *target = '<'; + break; + case SFM_SPACE: + *target = ' '; + break; + case SFM_PERIOD: + *target = '.'; + break; + default: + return false; + } + return true; +} + + +/* + * cifs_mapchar - convert a host-endian char to proper char in codepage + * @target - where converted character should be copied + * @src_char - 2 byte host-endian source character + * @cp - codepage to which character should be converted + * @map_type - How should the 7 NTFS/SMB reserved characters be mapped to UCS2? + * + * This function handles the conversion of a single character. It is the + * responsibility of the caller to ensure that the target buffer is large + * enough to hold the result of the conversion (at least NLS_MAX_CHARSET_SIZE). + */ +static int +cifs_mapchar(char *target, const __u16 *from, const struct nls_table *cp, + int maptype) +{ + int len = 1; + __u16 src_char; + + src_char = *from; + + if ((maptype == SFM_MAP_UNI_RSVD) && convert_sfm_char(src_char, target)) + return len; + else if ((maptype == SFU_MAP_UNI_RSVD) && + convert_sfu_char(src_char, target)) + return len; + + /* if character not one of seven in special remap set */ + len = cp->uni2char(src_char, target, NLS_MAX_CHARSET_SIZE); + if (len <= 0) + goto surrogate_pair; + + return len; + +surrogate_pair: + /* convert SURROGATE_PAIR and IVS */ + if (strcmp(cp->charset, "utf8")) + goto unknown; + len = utf16s_to_utf8s(from, 3, UTF16_LITTLE_ENDIAN, target, 6); + if (len <= 0) + goto unknown; + return len; + +unknown: + *target = '?'; + len = 1; + return len; +} + +/* + * cifs_from_utf16 - convert utf16le string to local charset + * @to - destination buffer + * @from - source buffer + * @tolen - destination buffer size (in bytes) + * @fromlen - source buffer size (in bytes) + * @codepage - codepage to which characters should be converted + * @mapchar - should characters be remapped according to the mapchars option? + * + * Convert a little-endian utf16le string (as sent by the server) to a string + * in the provided codepage. The tolen and fromlen parameters are to ensure + * that the code doesn't walk off of the end of the buffer (which is always + * a danger if the alignment of the source buffer is off). The destination + * string is always properly null terminated and fits in the destination + * buffer. Returns the length of the destination string in bytes (including + * null terminator). + * + * Note that some windows versions actually send multiword UTF-16 characters + * instead of straight UTF16-2. The linux nls routines however aren't able to + * deal with those characters properly. In the event that we get some of + * those characters, they won't be translated properly. + */ +int +cifs_from_utf16(char *to, const __le16 *from, int tolen, int fromlen, + const struct nls_table *codepage, int map_type) +{ + int i, charlen, safelen; + int outlen = 0; + int nullsize = nls_nullsize(codepage); + int fromwords = fromlen / 2; + char tmp[NLS_MAX_CHARSET_SIZE]; + __u16 ftmp[3]; /* ftmp[3] = 3array x 2bytes = 6bytes UTF-16 */ + + /* + * because the chars can be of varying widths, we need to take care + * not to overflow the destination buffer when we get close to the + * end of it. Until we get to this offset, we don't need to check + * for overflow however. + */ + safelen = tolen - (NLS_MAX_CHARSET_SIZE + nullsize); + + for (i = 0; i < fromwords; i++) { + ftmp[0] = get_unaligned_le16(&from[i]); + if (ftmp[0] == 0) + break; + if (i + 1 < fromwords) + ftmp[1] = get_unaligned_le16(&from[i + 1]); + else + ftmp[1] = 0; + if (i + 2 < fromwords) + ftmp[2] = get_unaligned_le16(&from[i + 2]); + else + ftmp[2] = 0; + + /* + * check to see if converting this character might make the + * conversion bleed into the null terminator + */ + if (outlen >= safelen) { + charlen = cifs_mapchar(tmp, ftmp, codepage, map_type); + if ((outlen + charlen) > (tolen - nullsize)) + break; + } + + /* put converted char into 'to' buffer */ + charlen = cifs_mapchar(&to[outlen], ftmp, codepage, map_type); + outlen += charlen; + + /* charlen (=bytes of UTF-8 for 1 character) + * 4bytes UTF-8(surrogate pair) is charlen=4 + * (4bytes UTF-16 code) + * 7-8bytes UTF-8(IVS) is charlen=3+4 or 4+4 + * (2 UTF-8 pairs divided to 2 UTF-16 pairs) */ + if (charlen == 4) + i++; + else if (charlen >= 5) + /* 5-6bytes UTF-8 */ + i += 2; + } + + /* properly null-terminate string */ + for (i = 0; i < nullsize; i++) + to[outlen++] = 0; + + return outlen; +} + +/* + * NAME: cifs_strtoUTF16() + * + * FUNCTION: Convert character string to unicode string + * + */ +int +cifs_strtoUTF16(__le16 *to, const char *from, int len, + const struct nls_table *codepage) +{ + int charlen; + int i; + wchar_t wchar_to; /* needed to quiet sparse */ + + /* special case for utf8 to handle no plane0 chars */ + if (!strcmp(codepage->charset, "utf8")) { + /* + * convert utf8 -> utf16, we assume we have enough space + * as caller should have assumed conversion does not overflow + * in destination len is length in wchar_t units (16bits) + */ + i = utf8s_to_utf16s(from, len, UTF16_LITTLE_ENDIAN, + (wchar_t *) to, len); + + /* if success terminate and exit */ + if (i >= 0) + goto success; + /* + * if fails fall back to UCS encoding as this + * function should not return negative values + * currently can fail only if source contains + * invalid encoded characters + */ + } + + for (i = 0; len && *from; i++, from += charlen, len -= charlen) { + charlen = codepage->char2uni(from, len, &wchar_to); + if (charlen < 1) { + cifs_dbg(VFS, "strtoUTF16: char2uni of 0x%x returned %d\n", + *from, charlen); + /* A question mark */ + wchar_to = 0x003f; + charlen = 1; + } + put_unaligned_le16(wchar_to, &to[i]); + } + +success: + put_unaligned_le16(0, &to[i]); + return i; +} + +/* + * cifs_utf16_bytes - how long will a string be after conversion? + * @utf16 - pointer to input string + * @maxbytes - don't go past this many bytes of input string + * @codepage - destination codepage + * + * Walk a utf16le string and return the number of bytes that the string will + * be after being converted to the given charset, not including any null + * termination required. Don't walk past maxbytes in the source buffer. + */ +int +cifs_utf16_bytes(const __le16 *from, int maxbytes, + const struct nls_table *codepage) +{ + int i; + int charlen, outlen = 0; + int maxwords = maxbytes / 2; + char tmp[NLS_MAX_CHARSET_SIZE]; + __u16 ftmp[3]; + + for (i = 0; i < maxwords; i++) { + ftmp[0] = get_unaligned_le16(&from[i]); + if (ftmp[0] == 0) + break; + if (i + 1 < maxwords) + ftmp[1] = get_unaligned_le16(&from[i + 1]); + else + ftmp[1] = 0; + if (i + 2 < maxwords) + ftmp[2] = get_unaligned_le16(&from[i + 2]); + else + ftmp[2] = 0; + + charlen = cifs_mapchar(tmp, ftmp, codepage, NO_MAP_UNI_RSVD); + outlen += charlen; + } + + return outlen; +} + +/* + * cifs_strndup_from_utf16 - copy a string from wire format to the local + * codepage + * @src - source string + * @maxlen - don't walk past this many bytes in the source string + * @is_unicode - is this a unicode string? + * @codepage - destination codepage + * + * Take a string given by the server, convert it to the local codepage and + * put it in a new buffer. Returns a pointer to the new string or NULL on + * error. + */ +char * +cifs_strndup_from_utf16(const char *src, const int maxlen, + const bool is_unicode, const struct nls_table *codepage) +{ + int len; + char *dst; + + if (is_unicode) { + len = cifs_utf16_bytes((__le16 *) src, maxlen, codepage); + len += nls_nullsize(codepage); + dst = kmalloc(len, GFP_KERNEL); + if (!dst) + return NULL; + cifs_from_utf16(dst, (__le16 *) src, len, maxlen, codepage, + NO_MAP_UNI_RSVD); + } else { + dst = kstrndup(src, maxlen, GFP_KERNEL); + } + + return dst; +} + +static __le16 convert_to_sfu_char(char src_char) +{ + __le16 dest_char; + + switch (src_char) { + case ':': + dest_char = cpu_to_le16(UNI_COLON); + break; + case '*': + dest_char = cpu_to_le16(UNI_ASTERISK); + break; + case '?': + dest_char = cpu_to_le16(UNI_QUESTION); + break; + case '<': + dest_char = cpu_to_le16(UNI_LESSTHAN); + break; + case '>': + dest_char = cpu_to_le16(UNI_GRTRTHAN); + break; + case '|': + dest_char = cpu_to_le16(UNI_PIPE); + break; + default: + dest_char = 0; + } + + return dest_char; +} + +static __le16 convert_to_sfm_char(char src_char, bool end_of_string) +{ + __le16 dest_char; + + if (src_char >= 0x01 && src_char <= 0x1F) { + dest_char = cpu_to_le16(src_char + 0xF000); + return dest_char; + } + switch (src_char) { + case ':': + dest_char = cpu_to_le16(SFM_COLON); + break; + case '"': + dest_char = cpu_to_le16(SFM_DOUBLEQUOTE); + break; + case '*': + dest_char = cpu_to_le16(SFM_ASTERISK); + break; + case '?': + dest_char = cpu_to_le16(SFM_QUESTION); + break; + case '<': + dest_char = cpu_to_le16(SFM_LESSTHAN); + break; + case '>': + dest_char = cpu_to_le16(SFM_GRTRTHAN); + break; + case '|': + dest_char = cpu_to_le16(SFM_PIPE); + break; + case '.': + if (end_of_string) + dest_char = cpu_to_le16(SFM_PERIOD); + else + dest_char = 0; + break; + case ' ': + if (end_of_string) + dest_char = cpu_to_le16(SFM_SPACE); + else + dest_char = 0; + break; + default: + dest_char = 0; + } + + return dest_char; +} + +/* + * Convert 16 bit Unicode pathname to wire format from string in current code + * page. Conversion may involve remapping up the six characters that are + * only legal in POSIX-like OS (if they are present in the string). Path + * names are little endian 16 bit Unicode on the wire + */ +int +cifsConvertToUTF16(__le16 *target, const char *source, int srclen, + const struct nls_table *cp, int map_chars) +{ + int i, charlen; + int j = 0; + char src_char; + __le16 dst_char; + wchar_t tmp; + wchar_t *wchar_to; /* UTF-16 */ + int ret; + unicode_t u; + + if (map_chars == NO_MAP_UNI_RSVD) + return cifs_strtoUTF16(target, source, PATH_MAX, cp); + + wchar_to = kzalloc(6, GFP_KERNEL); + + for (i = 0; i < srclen; j++) { + src_char = source[i]; + charlen = 1; + + /* check if end of string */ + if (src_char == 0) + goto ctoUTF16_out; + + /* see if we must remap this char */ + if (map_chars == SFU_MAP_UNI_RSVD) + dst_char = convert_to_sfu_char(src_char); + else if (map_chars == SFM_MAP_UNI_RSVD) { + bool end_of_string; + + /** + * Remap spaces and periods found at the end of every + * component of the path. The special cases of '.' and + * '..' are need to be handled because of symlinks. + * They are treated as non-end-of-string to avoid + * remapping and breaking symlinks pointing to . or .. + **/ + if ((i == 0 || source[i-1] == '\\') && + source[i] == '.' && + (i == srclen-1 || source[i+1] == '\\')) + end_of_string = false; /* "." case */ + else if (i >= 1 && + (i == 1 || source[i-2] == '\\') && + source[i-1] == '.' && + source[i] == '.' && + (i == srclen-1 || source[i+1] == '\\')) + end_of_string = false; /* ".." case */ + else if ((i == srclen - 1) || (source[i+1] == '\\')) + end_of_string = true; + else + end_of_string = false; + + dst_char = convert_to_sfm_char(src_char, end_of_string); + } else + dst_char = 0; + /* + * FIXME: We can not handle remapping backslash (UNI_SLASH) + * until all the calls to build_path_from_dentry are modified, + * as they use backslash as separator. + */ + if (dst_char == 0) { + charlen = cp->char2uni(source + i, srclen - i, &tmp); + dst_char = cpu_to_le16(tmp); + + /* + * if no match, use question mark, which at least in + * some cases serves as wild card + */ + if (charlen > 0) + goto ctoUTF16; + + /* convert SURROGATE_PAIR */ + if (strcmp(cp->charset, "utf8") || !wchar_to) + goto unknown; + if (*(source + i) & 0x80) { + charlen = utf8_to_utf32(source + i, 6, &u); + if (charlen < 0) + goto unknown; + } else + goto unknown; + ret = utf8s_to_utf16s(source + i, charlen, + UTF16_LITTLE_ENDIAN, + wchar_to, 6); + if (ret < 0) + goto unknown; + + i += charlen; + dst_char = cpu_to_le16(*wchar_to); + if (charlen <= 3) + /* 1-3bytes UTF-8 to 2bytes UTF-16 */ + put_unaligned(dst_char, &target[j]); + else if (charlen == 4) { + /* 4bytes UTF-8(surrogate pair) to 4bytes UTF-16 + * 7-8bytes UTF-8(IVS) divided to 2 UTF-16 + * (charlen=3+4 or 4+4) */ + put_unaligned(dst_char, &target[j]); + dst_char = cpu_to_le16(*(wchar_to + 1)); + j++; + put_unaligned(dst_char, &target[j]); + } else if (charlen >= 5) { + /* 5-6bytes UTF-8 to 6bytes UTF-16 */ + put_unaligned(dst_char, &target[j]); + dst_char = cpu_to_le16(*(wchar_to + 1)); + j++; + put_unaligned(dst_char, &target[j]); + dst_char = cpu_to_le16(*(wchar_to + 2)); + j++; + put_unaligned(dst_char, &target[j]); + } + continue; + +unknown: + dst_char = cpu_to_le16(0x003f); + charlen = 1; + } + +ctoUTF16: + /* + * character may take more than one byte in the source string, + * but will take exactly two bytes in the target string + */ + i += charlen; + put_unaligned(dst_char, &target[j]); + } + +ctoUTF16_out: + put_unaligned(0, &target[j]); /* Null terminate target unicode string */ + kfree(wchar_to); + return j; +} + +/* + * cifs_local_to_utf16_bytes - how long will a string be after conversion? + * @from - pointer to input string + * @maxbytes - don't go past this many bytes of input string + * @codepage - source codepage + * + * Walk a string and return the number of bytes that the string will + * be after being converted to the given charset, not including any null + * termination required. Don't walk past maxbytes in the source buffer. + */ + +static int +cifs_local_to_utf16_bytes(const char *from, int len, + const struct nls_table *codepage) +{ + int charlen; + int i; + wchar_t wchar_to; + + for (i = 0; len && *from; i++, from += charlen, len -= charlen) { + charlen = codepage->char2uni(from, len, &wchar_to); + /* Failed conversion defaults to a question mark */ + if (charlen < 1) + charlen = 1; + } + return 2 * i; /* UTF16 characters are two bytes */ +} + +/* + * cifs_strndup_to_utf16 - copy a string to wire format from the local codepage + * @src - source string + * @maxlen - don't walk past this many bytes in the source string + * @utf16_len - the length of the allocated string in bytes (including null) + * @cp - source codepage + * @remap - map special chars + * + * Take a string convert it from the local codepage to UTF16 and + * put it in a new buffer. Returns a pointer to the new string or NULL on + * error. + */ +__le16 * +cifs_strndup_to_utf16(const char *src, const int maxlen, int *utf16_len, + const struct nls_table *cp, int remap) +{ + int len; + __le16 *dst; + + if (!src) + return NULL; + + len = cifs_local_to_utf16_bytes(src, maxlen, cp); + len += 2; /* NULL */ + dst = kmalloc(len, GFP_KERNEL); + if (!dst) { + *utf16_len = 0; + return NULL; + } + cifsConvertToUTF16(dst, src, strlen(src), cp, remap); + *utf16_len = len; + return dst; +} diff --git a/fs/smb/client/cifs_unicode.h b/fs/smb/client/cifs_unicode.h new file mode 100644 index 000000000..3e9cd9acf --- /dev/null +++ b/fs/smb/client/cifs_unicode.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * cifs_unicode: Unicode kernel case support + * + * Function: + * Convert a unicode character to upper or lower case using + * compressed tables. + * + * Copyright (c) International Business Machines Corp., 2000,2009 + * + * Notes: + * These APIs are based on the C library functions. The semantics + * should match the C functions but with expanded size operands. + * + * The upper/lower functions are based on a table created by mkupr. + * This is a compressed table of upper and lower case conversion. + */ +#ifndef _CIFS_UNICODE_H +#define _CIFS_UNICODE_H + +#include +#include +#include +#include "../../nls/nls_ucs2_utils.h" +#include "cifsglob.h" + +/* + * Macs use an older "SFM" mapping of the symbols above. Fortunately it does + * not conflict (although almost does) with the mapping above. + */ + +#define SFM_DOUBLEQUOTE ((__u16) 0xF020) +#define SFM_ASTERISK ((__u16) 0xF021) +#define SFM_QUESTION ((__u16) 0xF025) +#define SFM_COLON ((__u16) 0xF022) +#define SFM_GRTRTHAN ((__u16) 0xF024) +#define SFM_LESSTHAN ((__u16) 0xF023) +#define SFM_PIPE ((__u16) 0xF027) +#define SFM_SLASH ((__u16) 0xF026) +#define SFM_SPACE ((__u16) 0xF028) +#define SFM_PERIOD ((__u16) 0xF029) + +/* + * Mapping mechanism to use when one of the seven reserved characters is + * encountered. We can only map using one of the mechanisms at a time + * since otherwise readdir could return directory entries which we would + * not be able to open + * + * NO_MAP_UNI_RSVD = do not perform any remapping of the character + * SFM_MAP_UNI_RSVD = map reserved characters using SFM scheme (MAC compatible) + * SFU_MAP_UNI_RSVD = map reserved characters ala SFU ("mapchars" option) + * + */ +#define NO_MAP_UNI_RSVD 0 +#define SFM_MAP_UNI_RSVD 1 +#define SFU_MAP_UNI_RSVD 2 + +int cifs_from_utf16(char *to, const __le16 *from, int tolen, int fromlen, + const struct nls_table *codepage, int map_type); +int cifs_utf16_bytes(const __le16 *from, int maxbytes, + const struct nls_table *codepage); +int cifs_strtoUTF16(__le16 *to, const char *from, int len, + const struct nls_table *codepage); +char *cifs_strndup_from_utf16(const char *src, const int maxlen, + const bool is_unicode, + const struct nls_table *codepage); +int cifsConvertToUTF16(__le16 *target, const char *source, int srclen, + const struct nls_table *cp, int map_chars); +__le16 *cifs_strndup_to_utf16(const char *src, const int maxlen, + int *utf16_len, const struct nls_table *cp, + int remap); +wchar_t cifs_toupper(wchar_t in); + +static inline int cifs_remap(const struct cifs_sb_info *cifs_sb) +{ + unsigned int sbflags = cifs_sb_flags(cifs_sb); + + if (sbflags & CIFS_MOUNT_MAP_SFM_CHR) + return SFM_MAP_UNI_RSVD; + if (sbflags & CIFS_MOUNT_MAP_SPECIAL_CHR) + return SFU_MAP_UNI_RSVD; + + return NO_MAP_UNI_RSVD; +} + +#endif /* _CIFS_UNICODE_H */ diff --git a/fs/smb/client/cifsacl.c b/fs/smb/client/cifsacl.c new file mode 100644 index 000000000..12005f463 --- /dev/null +++ b/fs/smb/client/cifsacl.c @@ -0,0 +1,2002 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * Copyright (C) International Business Machines Corp., 2007,2008 + * Author(s): Steve French (sfrench@us.ibm.com) + * + * Contains the routines for mapping CIFS/NTFS ACLs + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "cifsglob.h" +#include "cifsacl.h" +#include "cifsproto.h" +#include "cifs_debug.h" +#include "fs_context.h" +#include "cifs_fs_sb.h" +#include "cifs_unicode.h" + +/* security id for everyone/world system group */ +static const struct smb_sid sid_everyone = { + 1, 1, {0, 0, 0, 0, 0, 1}, {0} }; +/* security id for Authenticated Users system group */ +static const struct smb_sid sid_authusers = { + 1, 1, {0, 0, 0, 0, 0, 5}, {cpu_to_le32(11)} }; + +/* S-1-22-1 Unmapped Unix users */ +static const struct smb_sid sid_unix_users = {1, 1, {0, 0, 0, 0, 0, 22}, + {cpu_to_le32(1), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} }; + +/* S-1-22-2 Unmapped Unix groups */ +static const struct smb_sid sid_unix_groups = { 1, 1, {0, 0, 0, 0, 0, 22}, + {cpu_to_le32(2), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} }; + +/* + * See https://technet.microsoft.com/en-us/library/hh509017(v=ws.10).aspx + */ + +/* S-1-5-88 MS NFS and Apple style UID/GID/mode */ + +/* S-1-5-88-1 Unix uid */ +static const struct smb_sid sid_unix_NFS_users = { 1, 2, {0, 0, 0, 0, 0, 5}, + {cpu_to_le32(88), + cpu_to_le32(1), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} }; + +/* S-1-5-88-2 Unix gid */ +static const struct smb_sid sid_unix_NFS_groups = { 1, 2, {0, 0, 0, 0, 0, 5}, + {cpu_to_le32(88), + cpu_to_le32(2), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} }; + +/* S-1-5-88-3 Unix mode */ +static const struct smb_sid sid_unix_NFS_mode = { 1, 2, {0, 0, 0, 0, 0, 5}, + {cpu_to_le32(88), + cpu_to_le32(3), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} }; + +static const struct cred *root_cred; + +static int +cifs_idmap_key_instantiate(struct key *key, struct key_preparsed_payload *prep) +{ + char *payload; + + if (prep->datalen > U16_MAX) + return -EINVAL; + + /* + * If the payload is less than or equal to the size of a pointer, then + * an allocation here is wasteful. Just copy the data directly to the + * payload.value union member instead. + * + * With this however, you must check the datalen before trying to + * dereference payload.data! + */ + if (prep->datalen <= sizeof(key->payload)) { + key->payload.data[0] = NULL; + memcpy(&key->payload, prep->data, prep->datalen); + } else { + payload = kmemdup(prep->data, prep->datalen, GFP_KERNEL); + if (!payload) + return -ENOMEM; + key->payload.data[0] = payload; + } + + key->datalen = prep->datalen; + return 0; +} + +static inline void +cifs_idmap_key_destroy(struct key *key) +{ + if (key->datalen > sizeof(key->payload)) + kfree(key->payload.data[0]); +} + +static struct key_type cifs_idmap_key_type = { + .name = "cifs.idmap", + .instantiate = cifs_idmap_key_instantiate, + .destroy = cifs_idmap_key_destroy, + .describe = user_describe, +}; + +static char * +sid_to_key_str(struct smb_sid *sidptr, unsigned int type) +{ + int i, len; + unsigned int saval; + char *sidstr, *strptr; + unsigned long long id_auth_val; + + /* 3 bytes for prefix */ + sidstr = kmalloc(3 + SID_STRING_BASE_SIZE + + (SID_STRING_SUBAUTH_SIZE * sidptr->num_subauth), + GFP_KERNEL); + if (!sidstr) + return sidstr; + + strptr = sidstr; + len = sprintf(strptr, "%cs:S-%hhu", type == SIDOWNER ? 'o' : 'g', + sidptr->revision); + strptr += len; + + /* The authority field is a single 48-bit number */ + id_auth_val = (unsigned long long)sidptr->authority[5]; + id_auth_val |= (unsigned long long)sidptr->authority[4] << 8; + id_auth_val |= (unsigned long long)sidptr->authority[3] << 16; + id_auth_val |= (unsigned long long)sidptr->authority[2] << 24; + id_auth_val |= (unsigned long long)sidptr->authority[1] << 32; + id_auth_val |= (unsigned long long)sidptr->authority[0] << 48; + + /* + * MS-DTYP states that if the authority is >= 2^32, then it should be + * expressed as a hex value. + */ + if (id_auth_val <= UINT_MAX) + len = sprintf(strptr, "-%llu", id_auth_val); + else + len = sprintf(strptr, "-0x%llx", id_auth_val); + + strptr += len; + + for (i = 0; i < sidptr->num_subauth; ++i) { + saval = le32_to_cpu(sidptr->sub_auth[i]); + len = sprintf(strptr, "-%u", saval); + strptr += len; + } + + return sidstr; +} + +/* + * if the two SIDs (roughly equivalent to a UUID for a user or group) are + * the same returns zero, if they do not match returns non-zero. + */ +static int +compare_sids(const struct smb_sid *ctsid, const struct smb_sid *cwsid) +{ + int i; + int num_subauth, num_sat, num_saw; + + if ((!ctsid) || (!cwsid)) + return 1; + + /* compare the revision */ + if (ctsid->revision != cwsid->revision) { + if (ctsid->revision > cwsid->revision) + return 1; + else + return -1; + } + + /* compare all of the six auth values */ + for (i = 0; i < NUM_AUTHS; ++i) { + if (ctsid->authority[i] != cwsid->authority[i]) { + if (ctsid->authority[i] > cwsid->authority[i]) + return 1; + else + return -1; + } + } + + /* compare all of the subauth values if any */ + num_sat = ctsid->num_subauth; + num_saw = cwsid->num_subauth; + num_subauth = min(num_sat, num_saw); + if (num_subauth) { + for (i = 0; i < num_subauth; ++i) { + if (ctsid->sub_auth[i] != cwsid->sub_auth[i]) { + if (le32_to_cpu(ctsid->sub_auth[i]) > + le32_to_cpu(cwsid->sub_auth[i])) + return 1; + else + return -1; + } + } + } + + return 0; /* sids compare/match */ +} + +static bool +is_well_known_sid(const struct smb_sid *psid, uint32_t *puid, bool is_group) +{ + int i; + int num_subauth; + const struct smb_sid *pwell_known_sid; + + if (!psid || (puid == NULL)) + return false; + + num_subauth = psid->num_subauth; + + /* check if Mac (or Windows NFS) vs. Samba format for Unix owner SID */ + if (num_subauth == 2) { + if (is_group) + pwell_known_sid = &sid_unix_groups; + else + pwell_known_sid = &sid_unix_users; + } else if (num_subauth == 3) { + if (is_group) + pwell_known_sid = &sid_unix_NFS_groups; + else + pwell_known_sid = &sid_unix_NFS_users; + } else + return false; + + /* compare the revision */ + if (psid->revision != pwell_known_sid->revision) + return false; + + /* compare all of the six auth values */ + for (i = 0; i < NUM_AUTHS; ++i) { + if (psid->authority[i] != pwell_known_sid->authority[i]) { + cifs_dbg(FYI, "auth %d did not match\n", i); + return false; + } + } + + if (num_subauth == 2) { + if (psid->sub_auth[0] != pwell_known_sid->sub_auth[0]) + return false; + + *puid = le32_to_cpu(psid->sub_auth[1]); + } else /* 3 subauths, ie Windows/Mac style */ { + *puid = le32_to_cpu(psid->sub_auth[0]); + if ((psid->sub_auth[0] != pwell_known_sid->sub_auth[0]) || + (psid->sub_auth[1] != pwell_known_sid->sub_auth[1])) + return false; + + *puid = le32_to_cpu(psid->sub_auth[2]); + } + + cifs_dbg(FYI, "Unix UID %d returned from SID\n", *puid); + return true; /* well known sid found, uid returned */ +} + +static __u16 +cifs_copy_sid(struct smb_sid *dst, const struct smb_sid *src) +{ + int i; + __u16 size = 1 + 1 + 6; + + dst->revision = src->revision; + dst->num_subauth = min_t(u8, src->num_subauth, SID_MAX_SUB_AUTHORITIES); + for (i = 0; i < NUM_AUTHS; ++i) + dst->authority[i] = src->authority[i]; + for (i = 0; i < dst->num_subauth; ++i) + dst->sub_auth[i] = src->sub_auth[i]; + size += (dst->num_subauth * 4); + + return size; +} + +static int parse_sid(const struct smb_sid *psid, const char *end_of_acl) +{ + unsigned int sid_len; + + /* SID must contain the fixed header before num_subauth is trusted. */ + if (end_of_acl < (const char *)psid + CIFS_SID_BASE_SIZE) { + cifs_dbg(VFS, "ACL too small to parse SID %p\n", psid); + return -EINVAL; + } + if (psid->num_subauth > SID_MAX_SUB_AUTHORITIES) { + cifs_dbg(VFS, "SID contains too many subauthorities %u\n", + psid->num_subauth); + return -EINVAL; + } + + sid_len = CIFS_SID_BASE_SIZE + psid->num_subauth * sizeof(__le32); + if (end_of_acl < (const char *)psid + sid_len) { + cifs_dbg(VFS, "ACL too small to parse SID %p\n", psid); + return -EINVAL; + } + +#ifdef CONFIG_CIFS_DEBUG2 + if (psid->num_subauth) { + int i; + + cifs_dbg(FYI, "SID revision %d num_auth %d\n", + psid->revision, psid->num_subauth); + + for (i = 0; i < psid->num_subauth; i++) { + cifs_dbg(FYI, "SID sub_auth[%d]: 0x%x\n", + i, le32_to_cpu(psid->sub_auth[i])); + } + + cifs_dbg(FYI, "RID 0x%x\n", + le32_to_cpu(psid->sub_auth[psid->num_subauth - 1])); + } +#endif + + return 0; +} + +static int sid_from_sd(const struct smb_ntsd *pntsd, __u32 secdesclen, + __u32 sid_offset, struct smb_sid **sid) +{ + struct smb_sid *psid; + char *end_of_acl; + + if (secdesclen < sizeof(struct smb_ntsd)) { + cifs_dbg(VFS, "ACL too small to parse security descriptor\n"); + return -EINVAL; + } + end_of_acl = (char *)pntsd + secdesclen; + + if (sid_offset < sizeof(struct smb_ntsd) || + sid_offset > secdesclen - CIFS_SID_BASE_SIZE) { + cifs_dbg(VFS, "Server returned illegal SID offset\n"); + return -EINVAL; + } + + psid = (struct smb_sid *)((char *)pntsd + sid_offset); + if (parse_sid(psid, end_of_acl)) + return -EINVAL; + + *sid = psid; + return 0; +} + +static int +id_to_sid(unsigned int cid, uint sidtype, struct smb_sid *ssid) +{ + int rc; + struct key *sidkey; + struct smb_sid *ksid; + unsigned int ksid_size; + char desc[3 + 10 + 1]; /* 3 byte prefix + 10 bytes for value + NULL */ + const struct cred *saved_cred; + + rc = snprintf(desc, sizeof(desc), "%ci:%u", + sidtype == SIDOWNER ? 'o' : 'g', cid); + if (rc >= sizeof(desc)) + return -EINVAL; + + rc = 0; + saved_cred = override_creds(root_cred); + sidkey = request_key(&cifs_idmap_key_type, desc, ""); + if (IS_ERR(sidkey)) { + rc = -EINVAL; + cifs_dbg(FYI, "%s: Can't map %cid %u to a SID\n", + __func__, sidtype == SIDOWNER ? 'u' : 'g', cid); + goto out_revert_creds; + } else if (sidkey->datalen < CIFS_SID_BASE_SIZE) { + rc = smb_EIO1(smb_eio_trace_malformed_sid_key, sidkey->datalen); + cifs_dbg(FYI, "%s: Downcall contained malformed key (datalen=%hu)\n", + __func__, sidkey->datalen); + goto invalidate_key; + } + + /* + * A sid is usually too large to be embedded in payload.value, but if + * there are no subauthorities and the host has 8-byte pointers, then + * it could be. + */ + ksid = sidkey->datalen <= sizeof(sidkey->payload) ? + (struct smb_sid *)&sidkey->payload : + (struct smb_sid *)sidkey->payload.data[0]; + + ksid_size = CIFS_SID_BASE_SIZE + (ksid->num_subauth * sizeof(__le32)); + if (ksid_size > sidkey->datalen) { + rc = smb_EIO2(smb_eio_trace_malformed_ksid_key, + ksid_size, sidkey->datalen); + cifs_dbg(FYI, "%s: Downcall contained malformed key (datalen=%hu, ksid_size=%u)\n", + __func__, sidkey->datalen, ksid_size); + goto invalidate_key; + } + + cifs_copy_sid(ssid, ksid); +out_key_put: + key_put(sidkey); +out_revert_creds: + revert_creds(saved_cred); + return rc; + +invalidate_key: + key_invalidate(sidkey); + goto out_key_put; +} + +int +sid_to_id(struct cifs_sb_info *cifs_sb, struct smb_sid *psid, + struct cifs_fattr *fattr, uint sidtype) +{ + struct key *sidkey; + char *sidstr; + const struct cred *saved_cred; + kuid_t fuid = cifs_sb->ctx->linux_uid; + kgid_t fgid = cifs_sb->ctx->linux_gid; + + /* + * If we have too many subauthorities, then something is really wrong. + * Just return an error. + */ + if (unlikely(psid->num_subauth > SID_MAX_SUB_AUTHORITIES)) { + cifs_dbg(FYI, "%s: %u subauthorities is too many!\n", + __func__, psid->num_subauth); + return smb_EIO2(smb_eio_trace_sid_too_many_auth, + psid->num_subauth, SID_MAX_SUB_AUTHORITIES); + } + + if ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_UID_FROM_ACL) || + (cifs_sb_master_tcon(cifs_sb)->posix_extensions)) { + uint32_t unix_id; + bool is_group; + + if (sidtype != SIDOWNER) + is_group = true; + else + is_group = false; + + if (is_well_known_sid(psid, &unix_id, is_group) == false) + goto try_upcall_to_get_id; + + if (is_group) { + kgid_t gid; + gid_t id; + + id = (gid_t)unix_id; + gid = make_kgid(&init_user_ns, id); + if (gid_valid(gid)) { + fgid = gid; + goto got_valid_id; + } + } else { + kuid_t uid; + uid_t id; + + id = (uid_t)unix_id; + uid = make_kuid(&init_user_ns, id); + if (uid_valid(uid)) { + fuid = uid; + goto got_valid_id; + } + } + /* If unable to find uid/gid easily from SID try via upcall */ + } + +try_upcall_to_get_id: + sidstr = sid_to_key_str(psid, sidtype); + if (!sidstr) + return -ENOMEM; + + saved_cred = override_creds(root_cred); + sidkey = request_key(&cifs_idmap_key_type, sidstr, ""); + if (IS_ERR(sidkey)) { + cifs_dbg(FYI, "%s: Can't map SID %s to a %cid\n", + __func__, sidstr, sidtype == SIDOWNER ? 'u' : 'g'); + goto out_revert_creds; + } + + /* + * FIXME: Here we assume that uid_t and gid_t are same size. It's + * probably a safe assumption but might be better to check based on + * sidtype. + */ + BUILD_BUG_ON(sizeof(uid_t) != sizeof(gid_t)); + if (sidkey->datalen != sizeof(uid_t)) { + cifs_dbg(FYI, "%s: Downcall contained malformed key (datalen=%hu)\n", + __func__, sidkey->datalen); + key_invalidate(sidkey); + goto out_key_put; + } + + if (sidtype == SIDOWNER) { + kuid_t uid; + uid_t id; + memcpy(&id, &sidkey->payload.data[0], sizeof(uid_t)); + uid = make_kuid(&init_user_ns, id); + if (uid_valid(uid)) + fuid = uid; + } else { + kgid_t gid; + gid_t id; + memcpy(&id, &sidkey->payload.data[0], sizeof(gid_t)); + gid = make_kgid(&init_user_ns, id); + if (gid_valid(gid)) + fgid = gid; + } + +out_key_put: + key_put(sidkey); +out_revert_creds: + revert_creds(saved_cred); + kfree(sidstr); + + /* + * Note that we return 0 here unconditionally. If the mapping + * fails then we just fall back to using the ctx->linux_uid/linux_gid. + */ +got_valid_id: + if (sidtype == SIDOWNER) + fattr->cf_uid = fuid; + else + fattr->cf_gid = fgid; + + return 0; +} + +int +init_cifs_idmap(void) +{ + struct cred *cred; + struct key *keyring; + int ret; + + cifs_dbg(FYI, "Registering the %s key type\n", + cifs_idmap_key_type.name); + + /* create an override credential set with a special thread keyring in + * which requests are cached + * + * this is used to prevent malicious redirections from being installed + * with add_key(). + */ + cred = prepare_kernel_cred(&init_task); + if (!cred) + return -ENOMEM; + + keyring = keyring_alloc(".cifs_idmap", + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, cred, + (KEY_POS_ALL & ~KEY_POS_SETATTR) | + KEY_USR_VIEW | KEY_USR_READ, + KEY_ALLOC_NOT_IN_QUOTA, NULL, NULL); + if (IS_ERR(keyring)) { + ret = PTR_ERR(keyring); + goto failed_put_cred; + } + + ret = register_key_type(&cifs_idmap_key_type); + if (ret < 0) + goto failed_put_key; + + /* instruct request_key() to use this special keyring as a cache for + * the results it looks up */ + set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags); + cred->thread_keyring = keyring; + cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING; + root_cred = cred; + + cifs_dbg(FYI, "cifs idmap keyring: %d\n", key_serial(keyring)); + return 0; + +failed_put_key: + key_put(keyring); +failed_put_cred: + put_cred(cred); + return ret; +} + +void +exit_cifs_idmap(void) +{ + key_revoke(root_cred->thread_keyring); + unregister_key_type(&cifs_idmap_key_type); + put_cred(root_cred); + cifs_dbg(FYI, "Unregistered %s key type\n", cifs_idmap_key_type.name); +} + +/* copy ntsd, owner sid, and group sid from a security descriptor to another */ +static int copy_sec_desc(const struct smb_ntsd *pntsd, struct smb_ntsd *pnntsd, + __u32 sidsoffset, __u32 secdesclen, + __u32 *pnsecdesclen, struct smb_sid *pownersid, + struct smb_sid *pgrpsid) +{ + struct smb_sid *owner_sid_ptr, *group_sid_ptr; + struct smb_sid *nowner_sid_ptr, *ngroup_sid_ptr; + int rc; + + /* copy security descriptor control portion */ + pnntsd->revision = pntsd->revision; + pnntsd->type = pntsd->type; + pnntsd->dacloffset = cpu_to_le32(sizeof(struct smb_ntsd)); + pnntsd->sacloffset = 0; + pnntsd->osidoffset = cpu_to_le32(sidsoffset); + pnntsd->gsidoffset = cpu_to_le32(sidsoffset + sizeof(struct smb_sid)); + + /* copy owner sid */ + if (pownersid) { + owner_sid_ptr = pownersid; + } else { + rc = sid_from_sd(pntsd, secdesclen, + le32_to_cpu(pntsd->osidoffset), &owner_sid_ptr); + if (rc) + return rc; + } + nowner_sid_ptr = (struct smb_sid *)((char *)pnntsd + sidsoffset); + cifs_copy_sid(nowner_sid_ptr, owner_sid_ptr); + + /* copy group sid */ + if (pgrpsid) { + group_sid_ptr = pgrpsid; + } else { + rc = sid_from_sd(pntsd, secdesclen, + le32_to_cpu(pntsd->gsidoffset), &group_sid_ptr); + if (rc) + return rc; + } + ngroup_sid_ptr = (struct smb_sid *)((char *)pnntsd + sidsoffset + + sizeof(struct smb_sid)); + cifs_copy_sid(ngroup_sid_ptr, group_sid_ptr); + + *pnsecdesclen = sidsoffset + (2 * sizeof(struct smb_sid)); + return 0; +} + +/* + change posix mode to reflect permissions + pmode is the existing mode (we only want to overwrite part of this + bits to set can be: S_IRWXU, S_IRWXG or S_IRWXO ie 00700 or 00070 or 00007 +*/ +static void access_flags_to_mode(__le32 ace_flags, int type, umode_t *pmode, + umode_t *pdenied, umode_t mask) +{ + __u32 flags = le32_to_cpu(ace_flags); + /* + * Do not assume "preferred" or "canonical" order. + * The first DENY or ALLOW ACE which matches perfectly is + * the permission to be used. Once allowed or denied, same + * permission in later ACEs do not matter. + */ + + /* If not already allowed, deny these bits */ + if (type == ACCESS_DENIED) { + if (flags & GENERIC_ALL && + !(*pmode & mask & 0777)) + *pdenied |= mask & 0777; + + if (((flags & GENERIC_WRITE) || + ((flags & FILE_WRITE_RIGHTS) == FILE_WRITE_RIGHTS)) && + !(*pmode & mask & 0222)) + *pdenied |= mask & 0222; + + if (((flags & GENERIC_READ) || + ((flags & FILE_READ_RIGHTS) == FILE_READ_RIGHTS)) && + !(*pmode & mask & 0444)) + *pdenied |= mask & 0444; + + if (((flags & GENERIC_EXECUTE) || + ((flags & FILE_EXEC_RIGHTS) == FILE_EXEC_RIGHTS)) && + !(*pmode & mask & 0111)) + *pdenied |= mask & 0111; + + return; + } else if (type != ACCESS_ALLOWED) { + cifs_dbg(VFS, "unknown access control type %d\n", type); + return; + } + /* else ACCESS_ALLOWED type */ + + if ((flags & GENERIC_ALL) && + !(*pdenied & mask & 0777)) { + *pmode |= mask & 0777; + cifs_dbg(NOISY, "all perms\n"); + return; + } + + if (((flags & GENERIC_WRITE) || + ((flags & FILE_WRITE_RIGHTS) == FILE_WRITE_RIGHTS)) && + !(*pdenied & mask & 0222)) + *pmode |= mask & 0222; + + if (((flags & GENERIC_READ) || + ((flags & FILE_READ_RIGHTS) == FILE_READ_RIGHTS)) && + !(*pdenied & mask & 0444)) + *pmode |= mask & 0444; + + if (((flags & GENERIC_EXECUTE) || + ((flags & FILE_EXEC_RIGHTS) == FILE_EXEC_RIGHTS)) && + !(*pdenied & mask & 0111)) + *pmode |= mask & 0111; + + /* If DELETE_CHILD is set only on an owner ACE, set sticky bit */ + if (flags & FILE_DELETE_CHILD) { + if (mask == ACL_OWNER_MASK) { + if (!(*pdenied & 01000)) + *pmode |= 01000; + } else if (!(*pdenied & 01000)) { + *pmode &= ~01000; + *pdenied |= 01000; + } + } + + cifs_dbg(NOISY, "access flags 0x%x mode now %04o\n", flags, *pmode); + return; +} + +/* + Generate access flags to reflect permissions mode is the existing mode. + This function is called for every ACE in the DACL whose SID matches + with either owner or group or everyone. +*/ + +static void mode_to_access_flags(umode_t mode, umode_t bits_to_use, + __u32 *pace_flags) +{ + /* reset access mask */ + *pace_flags = 0x0; + + /* bits to use are either S_IRWXU or S_IRWXG or S_IRWXO */ + mode &= bits_to_use; + + /* check for R/W/X UGO since we do not know whose flags + is this but we have cleared all the bits sans RWX for + either user or group or other as per bits_to_use */ + if (mode & S_IRUGO) + *pace_flags |= SET_FILE_READ_RIGHTS; + if (mode & S_IWUGO) + *pace_flags |= SET_FILE_WRITE_RIGHTS; + if (mode & S_IXUGO) + *pace_flags |= SET_FILE_EXEC_RIGHTS; + + cifs_dbg(NOISY, "mode: %04o, access flags now 0x%x\n", + mode, *pace_flags); + return; +} + +static __u16 cifs_copy_ace(struct smb_ace *dst, struct smb_ace *src, struct smb_sid *psid) +{ + __u16 size = 1 + 1 + 2 + 4; + + dst->type = src->type; + dst->flags = src->flags; + dst->access_req = src->access_req; + + /* Check if there's a replacement sid specified */ + if (psid) + size += cifs_copy_sid(&dst->sid, psid); + else + size += cifs_copy_sid(&dst->sid, &src->sid); + + dst->size = cpu_to_le16(size); + + return size; +} + +static __u16 fill_ace_for_sid(struct smb_ace *pntace, + const struct smb_sid *psid, __u64 nmode, + umode_t bits, __u8 access_type, + bool allow_delete_child) +{ + int i; + __u16 size = 0; + __u32 access_req = 0; + + pntace->type = access_type; + pntace->flags = 0x0; + mode_to_access_flags(nmode, bits, &access_req); + + if (access_type == ACCESS_ALLOWED && allow_delete_child) + access_req |= FILE_DELETE_CHILD; + + if (access_type == ACCESS_ALLOWED && !access_req) + access_req = SET_MINIMUM_RIGHTS; + else if (access_type == ACCESS_DENIED) + access_req &= ~SET_MINIMUM_RIGHTS; + + pntace->access_req = cpu_to_le32(access_req); + + pntace->sid.revision = psid->revision; + pntace->sid.num_subauth = psid->num_subauth; + for (i = 0; i < NUM_AUTHS; i++) + pntace->sid.authority[i] = psid->authority[i]; + for (i = 0; i < psid->num_subauth; i++) + pntace->sid.sub_auth[i] = psid->sub_auth[i]; + + size = 1 + 1 + 2 + 4 + 1 + 1 + 6 + (psid->num_subauth * 4); + pntace->size = cpu_to_le16(size); + + return size; +} + + +#ifdef CONFIG_CIFS_DEBUG2 +static void dump_ace(struct smb_ace *pace, char *end_of_acl) +{ + int num_subauth; + + /* validate that we do not go past end of acl */ + + if (le16_to_cpu(pace->size) < 16) { + cifs_dbg(VFS, "ACE too small %d\n", le16_to_cpu(pace->size)); + return; + } + + if (end_of_acl < (char *)pace + le16_to_cpu(pace->size)) { + cifs_dbg(VFS, "ACL too small to parse ACE\n"); + return; + } + + num_subauth = pace->sid.num_subauth; + if (num_subauth) { + int i; + cifs_dbg(FYI, "ACE revision %d num_auth %d type %d flags %d size %d\n", + pace->sid.revision, pace->sid.num_subauth, pace->type, + pace->flags, le16_to_cpu(pace->size)); + for (i = 0; i < num_subauth; ++i) { + cifs_dbg(FYI, "ACE sub_auth[%d]: 0x%x\n", + i, le32_to_cpu(pace->sid.sub_auth[i])); + } + + /* BB add length check to make sure that we do not have huge + num auths and therefore go off the end */ + } + + return; +} +#endif + +static int validate_dacl(struct smb_acl *pdacl, char *end_of_acl) +{ + int i, ace_hdr_size, ace_size, min_ace_size; + u16 dacl_size, num_aces; + char *acl_base, *end_of_dacl; + struct smb_ace *pace; + + if (!pdacl) + return 0; + + if (end_of_acl < (char *)pdacl + sizeof(struct smb_acl)) { + cifs_dbg(VFS, "ACL too small to parse DACL\n"); + return -EINVAL; + } + + dacl_size = le16_to_cpu(pdacl->size); + if (dacl_size < sizeof(struct smb_acl) || + end_of_acl < (char *)pdacl + dacl_size) { + cifs_dbg(VFS, "ACL too small to parse DACL\n"); + return -EINVAL; + } + + num_aces = le16_to_cpu(pdacl->num_aces); + if (!num_aces) + return 0; + + ace_hdr_size = offsetof(struct smb_ace, sid) + + offsetof(struct smb_sid, sub_auth); + min_ace_size = ace_hdr_size + sizeof(__le32); + if (num_aces > (dacl_size - sizeof(struct smb_acl)) / min_ace_size) { + cifs_dbg(VFS, "ACL too small to parse DACL\n"); + return -EINVAL; + } + + end_of_dacl = (char *)pdacl + dacl_size; + acl_base = (char *)pdacl; + ace_size = sizeof(struct smb_acl); + + for (i = 0; i < num_aces; ++i) { + if (end_of_dacl - acl_base < ace_size) { + cifs_dbg(VFS, "ACL too small to parse ACE\n"); + return -EINVAL; + } + + pace = (struct smb_ace *)(acl_base + ace_size); + acl_base = (char *)pace; + + if (end_of_dacl - acl_base < ace_hdr_size || + pace->sid.num_subauth == 0 || + pace->sid.num_subauth > SID_MAX_SUB_AUTHORITIES) { + cifs_dbg(VFS, "ACL too small to parse ACE\n"); + return -EINVAL; + } + + ace_size = ace_hdr_size + sizeof(__le32) * pace->sid.num_subauth; + if (end_of_dacl - acl_base < ace_size || + le16_to_cpu(pace->size) < ace_size) { + cifs_dbg(VFS, "ACL too small to parse ACE\n"); + return -EINVAL; + } + + ace_size = le16_to_cpu(pace->size); + if (end_of_dacl - acl_base < ace_size) { + cifs_dbg(VFS, "ACL too small to parse ACE\n"); + return -EINVAL; + } + } + + return 0; +} + +static void parse_dacl(struct smb_acl *pdacl, char *end_of_acl, + struct smb_sid *pownersid, struct smb_sid *pgrpsid, + struct cifs_fattr *fattr, bool mode_from_special_sid) +{ + int i; + u16 num_aces = 0; + int acl_size; + char *acl_base; + struct smb_ace **ppace; + + /* BB need to add parm so we can store the SID BB */ + + if (!pdacl) { + /* no DACL in the security descriptor, set + all the permissions for user/group/other */ + fattr->cf_mode |= 0777; + return; + } + + if (validate_dacl(pdacl, end_of_acl)) + return; + + cifs_dbg(NOISY, "DACL revision %d size %d num aces %d\n", + le16_to_cpu(pdacl->revision), le16_to_cpu(pdacl->size), + le16_to_cpu(pdacl->num_aces)); + + /* reset rwx permissions for user/group/other. + Also, if num_aces is 0 i.e. DACL has no ACEs, + user/group/other have no permissions */ + fattr->cf_mode &= ~(0777); + + acl_base = (char *)pdacl; + acl_size = sizeof(struct smb_acl); + + num_aces = le16_to_cpu(pdacl->num_aces); + if (num_aces > 0) { + umode_t denied_mode = 0; + + ppace = kmalloc_objs(struct smb_ace *, num_aces); + if (!ppace) + return; + + for (i = 0; i < num_aces; ++i) { + ppace[i] = (struct smb_ace *) (acl_base + acl_size); + +#ifdef CONFIG_CIFS_DEBUG2 + dump_ace(ppace[i], + (char *)pdacl + le16_to_cpu(pdacl->size)); +#endif + if (mode_from_special_sid && + ppace[i]->sid.num_subauth >= 3 && + (compare_sids(&(ppace[i]->sid), + &sid_unix_NFS_mode) == 0)) { + /* + * Full permissions are: + * 07777 = S_ISUID | S_ISGID | S_ISVTX | + * S_IRWXU | S_IRWXG | S_IRWXO + */ + fattr->cf_mode &= ~07777; + fattr->cf_mode |= + le32_to_cpu(ppace[i]->sid.sub_auth[2]) & 07777; + break; + } else { + if (compare_sids(&(ppace[i]->sid), pownersid) == 0) { + access_flags_to_mode(ppace[i]->access_req, + ppace[i]->type, + &fattr->cf_mode, + &denied_mode, + ACL_OWNER_MASK); + } else if (compare_sids(&(ppace[i]->sid), pgrpsid) == 0) { + access_flags_to_mode(ppace[i]->access_req, + ppace[i]->type, + &fattr->cf_mode, + &denied_mode, + ACL_GROUP_MASK); + } else if ((compare_sids(&(ppace[i]->sid), &sid_everyone) == 0) || + (compare_sids(&(ppace[i]->sid), &sid_authusers) == 0)) { + access_flags_to_mode(ppace[i]->access_req, + ppace[i]->type, + &fattr->cf_mode, + &denied_mode, + ACL_EVERYONE_MASK); + } + } + + +/* memcpy((void *)(&(cifscred->aces[i])), + (void *)ppace[i], + sizeof(struct smb_ace)); */ + + acl_base = (char *)ppace[i]; + acl_size = le16_to_cpu(ppace[i]->size); + } + + kfree(ppace); + } + + return; +} + +unsigned int setup_authusers_ACE(struct smb_ace *pntace) +{ + int i; + unsigned int ace_size = 20; + + pntace->type = ACCESS_ALLOWED_ACE_TYPE; + pntace->flags = 0x0; + pntace->access_req = cpu_to_le32(GENERIC_ALL); + pntace->sid.num_subauth = 1; + pntace->sid.revision = 1; + for (i = 0; i < NUM_AUTHS; i++) + pntace->sid.authority[i] = sid_authusers.authority[i]; + + pntace->sid.sub_auth[0] = sid_authusers.sub_auth[0]; + + /* size = 1 + 1 + 2 + 4 + 1 + 1 + 6 + (psid->num_subauth*4) */ + pntace->size = cpu_to_le16(ace_size); + return ace_size; +} + +/* + * Fill in the special SID based on the mode. See + * https://technet.microsoft.com/en-us/library/hh509017(v=ws.10).aspx + */ +unsigned int setup_special_mode_ACE(struct smb_ace *pntace, + bool posix, + __u64 nmode) +{ + int i; + unsigned int ace_size = 28; + + if (posix) + pntace->type = ACCESS_ALLOWED_ACE_TYPE; + else + pntace->type = ACCESS_DENIED_ACE_TYPE; + pntace->flags = 0x0; + pntace->access_req = 0; + pntace->sid.num_subauth = 3; + pntace->sid.revision = 1; + for (i = 0; i < NUM_AUTHS; i++) + pntace->sid.authority[i] = sid_unix_NFS_mode.authority[i]; + + pntace->sid.sub_auth[0] = sid_unix_NFS_mode.sub_auth[0]; + pntace->sid.sub_auth[1] = sid_unix_NFS_mode.sub_auth[1]; + pntace->sid.sub_auth[2] = cpu_to_le32(nmode & 07777); + + /* size = 1 + 1 + 2 + 4 + 1 + 1 + 6 + (psid->num_subauth*4) */ + pntace->size = cpu_to_le16(ace_size); + return ace_size; +} + +unsigned int setup_special_user_owner_ACE(struct smb_ace *pntace) +{ + int i; + unsigned int ace_size = 28; + + pntace->type = ACCESS_ALLOWED_ACE_TYPE; + pntace->flags = 0x0; + pntace->access_req = cpu_to_le32(GENERIC_ALL); + pntace->sid.num_subauth = 3; + pntace->sid.revision = 1; + for (i = 0; i < NUM_AUTHS; i++) + pntace->sid.authority[i] = sid_unix_NFS_users.authority[i]; + + pntace->sid.sub_auth[0] = sid_unix_NFS_users.sub_auth[0]; + pntace->sid.sub_auth[1] = sid_unix_NFS_users.sub_auth[1]; + pntace->sid.sub_auth[2] = cpu_to_le32(current_fsgid().val); + + /* size = 1 + 1 + 2 + 4 + 1 + 1 + 6 + (psid->num_subauth*4) */ + pntace->size = cpu_to_le16(ace_size); + return ace_size; +} + +static void populate_new_aces(char *nacl_base, + struct smb_sid *pownersid, + struct smb_sid *pgrpsid, + __u64 *pnmode, u16 *pnum_aces, u16 *pnsize, + bool modefromsid, + bool posix) +{ + __u64 nmode; + u16 num_aces = 0; + u16 nsize = 0; + __u64 user_mode; + __u64 group_mode; + __u64 other_mode; + __u64 deny_user_mode = 0; + __u64 deny_group_mode = 0; + bool sticky_set = false; + struct smb_ace *pnntace = NULL; + + nmode = *pnmode; + num_aces = *pnum_aces; + nsize = *pnsize; + + if (modefromsid || posix) { + pnntace = (struct smb_ace *) (nacl_base + nsize); + nsize += setup_special_mode_ACE(pnntace, posix, nmode); + num_aces++; + if (modefromsid) { + pnntace = (struct smb_ace *) (nacl_base + nsize); + nsize += setup_authusers_ACE(pnntace); + num_aces++; + } + goto set_size; + } + + /* + * We'll try to keep the mode as requested by the user. + * But in cases where we cannot meaningfully convert that + * into ACL, return back the updated mode, so that it is + * updated in the inode. + */ + + if (!memcmp(pownersid, pgrpsid, sizeof(struct smb_sid))) { + /* + * Case when owner and group SIDs are the same. + * Set the more restrictive of the two modes. + */ + user_mode = nmode & (nmode << 3) & 0700; + group_mode = nmode & (nmode >> 3) & 0070; + } else { + user_mode = nmode & 0700; + group_mode = nmode & 0070; + } + + other_mode = nmode & 0007; + + /* We need DENY ACE when the perm is more restrictive than the next sets. */ + deny_user_mode = ~(user_mode) & ((group_mode << 3) | (other_mode << 6)) & 0700; + deny_group_mode = ~(group_mode) & (other_mode << 3) & 0070; + + *pnmode = user_mode | group_mode | other_mode | (nmode & ~0777); + + /* This tells if we should allow delete child for group and everyone. */ + if (nmode & 01000) + sticky_set = true; + + if (deny_user_mode) { + pnntace = (struct smb_ace *) (nacl_base + nsize); + nsize += fill_ace_for_sid(pnntace, pownersid, deny_user_mode, + 0700, ACCESS_DENIED, false); + num_aces++; + } + + /* Group DENY ACE does not conflict with owner ALLOW ACE. Keep in preferred order*/ + if (deny_group_mode && !(deny_group_mode & (user_mode >> 3))) { + pnntace = (struct smb_ace *) (nacl_base + nsize); + nsize += fill_ace_for_sid(pnntace, pgrpsid, deny_group_mode, + 0070, ACCESS_DENIED, false); + num_aces++; + } + + pnntace = (struct smb_ace *) (nacl_base + nsize); + nsize += fill_ace_for_sid(pnntace, pownersid, user_mode, + 0700, ACCESS_ALLOWED, true); + num_aces++; + + /* Group DENY ACE conflicts with owner ALLOW ACE. So keep it after. */ + if (deny_group_mode && (deny_group_mode & (user_mode >> 3))) { + pnntace = (struct smb_ace *) (nacl_base + nsize); + nsize += fill_ace_for_sid(pnntace, pgrpsid, deny_group_mode, + 0070, ACCESS_DENIED, false); + num_aces++; + } + + pnntace = (struct smb_ace *) (nacl_base + nsize); + nsize += fill_ace_for_sid(pnntace, pgrpsid, group_mode, + 0070, ACCESS_ALLOWED, !sticky_set); + num_aces++; + + pnntace = (struct smb_ace *) (nacl_base + nsize); + nsize += fill_ace_for_sid(pnntace, &sid_everyone, other_mode, + 0007, ACCESS_ALLOWED, !sticky_set); + num_aces++; + +set_size: + *pnum_aces = num_aces; + *pnsize = nsize; +} + +static __u16 replace_sids_and_copy_aces(struct smb_acl *pdacl, struct smb_acl *pndacl, + struct smb_sid *pownersid, struct smb_sid *pgrpsid, + struct smb_sid *pnownersid, struct smb_sid *pngrpsid, + int *aclflag) +{ + int i; + u16 size = 0; + struct smb_ace *pntace = NULL; + char *acl_base = NULL; + u16 src_num_aces = 0; + u16 nsize = 0; + struct smb_ace *pnntace = NULL; + char *nacl_base = NULL; + u16 ace_size = 0; + + acl_base = (char *)pdacl; + size = sizeof(struct smb_acl); + src_num_aces = le16_to_cpu(pdacl->num_aces); + + nacl_base = (char *)pndacl; + nsize = sizeof(struct smb_acl); + + /* Go through all the ACEs */ + for (i = 0; i < src_num_aces; ++i) { + pntace = (struct smb_ace *) (acl_base + size); + pnntace = (struct smb_ace *) (nacl_base + nsize); + + if (pnownersid && compare_sids(&pntace->sid, pownersid) == 0) { + ace_size = cifs_copy_ace(pnntace, pntace, pnownersid); + *aclflag |= CIFS_ACL_DACL; + } else if (pngrpsid && compare_sids(&pntace->sid, pgrpsid) == 0) { + ace_size = cifs_copy_ace(pnntace, pntace, pngrpsid); + *aclflag |= CIFS_ACL_DACL; + } else { + ace_size = cifs_copy_ace(pnntace, pntace, NULL); + } + + size += le16_to_cpu(pntace->size); + nsize += ace_size; + } + + return nsize; +} + +static int set_chmod_dacl(struct smb_acl *pdacl, struct smb_acl *pndacl, + struct smb_sid *pownersid, struct smb_sid *pgrpsid, + __u64 *pnmode, bool mode_from_sid, bool posix) +{ + int i; + u16 size = 0; + struct smb_ace *pntace = NULL; + char *acl_base = NULL; + u16 src_num_aces = 0; + u16 nsize = 0; + struct smb_ace *pnntace = NULL; + char *nacl_base = NULL; + u16 num_aces = 0; + bool new_aces_set = false; + + /* Assuming that pndacl and pnmode are never NULL */ + nacl_base = (char *)pndacl; + nsize = sizeof(struct smb_acl); + + /* If pdacl is NULL, we don't have a src. Simply populate new ACL. */ + if (!pdacl || posix) { + populate_new_aces(nacl_base, + pownersid, pgrpsid, + pnmode, &num_aces, &nsize, + mode_from_sid, posix); + goto finalize_dacl; + } + + acl_base = (char *)pdacl; + size = sizeof(struct smb_acl); + src_num_aces = le16_to_cpu(pdacl->num_aces); + + /* Retain old ACEs which we can retain */ + for (i = 0; i < src_num_aces; ++i) { + pntace = (struct smb_ace *) (acl_base + size); + + if (!new_aces_set && (pntace->flags & INHERITED_ACE)) { + /* Place the new ACEs in between existing explicit and inherited */ + populate_new_aces(nacl_base, + pownersid, pgrpsid, + pnmode, &num_aces, &nsize, + mode_from_sid, posix); + + new_aces_set = true; + } + + /* If it's any one of the ACE we're replacing, skip! */ + if (((compare_sids(&pntace->sid, &sid_unix_NFS_mode) == 0) || + (compare_sids(&pntace->sid, pownersid) == 0) || + (compare_sids(&pntace->sid, pgrpsid) == 0) || + (compare_sids(&pntace->sid, &sid_everyone) == 0) || + (compare_sids(&pntace->sid, &sid_authusers) == 0))) { + goto next_ace; + } + + /* update the pointer to the next ACE to populate*/ + pnntace = (struct smb_ace *) (nacl_base + nsize); + + nsize += cifs_copy_ace(pnntace, pntace, NULL); + num_aces++; + +next_ace: + size += le16_to_cpu(pntace->size); + } + + /* If inherited ACEs are not present, place the new ones at the tail */ + if (!new_aces_set) { + populate_new_aces(nacl_base, + pownersid, pgrpsid, + pnmode, &num_aces, &nsize, + mode_from_sid, posix); + + new_aces_set = true; + } + +finalize_dacl: + pndacl->num_aces = cpu_to_le16(num_aces); + pndacl->size = cpu_to_le16(nsize); + + return 0; +} + +static bool dacl_offset_valid(unsigned int acl_len, __u32 dacloffset) +{ + if (acl_len < sizeof(struct smb_acl)) + return false; + + if (dacloffset < sizeof(struct smb_ntsd)) + return false; + + return dacloffset <= acl_len - sizeof(struct smb_acl); +} + + +/* Convert CIFS ACL to POSIX form */ +static int parse_sec_desc(struct cifs_sb_info *cifs_sb, + struct smb_ntsd *pntsd, int acl_len, struct cifs_fattr *fattr, + bool get_mode_from_special_sid) +{ + int rc = 0; + struct smb_sid *owner_sid_ptr, *group_sid_ptr; + struct smb_acl *dacl_ptr; /* no need for SACL ptr */ + char *end_of_acl; + __u32 dacloffset, osidoffset, gsidoffset; + + if (pntsd == NULL) + return smb_EIO(smb_eio_trace_null_pointers); + if (acl_len < (int)sizeof(struct smb_ntsd)) { + cifs_dbg(VFS, "ACL too small to parse security descriptor\n"); + return -EINVAL; + } + end_of_acl = ((char *)pntsd) + acl_len; + + osidoffset = le32_to_cpu(pntsd->osidoffset); + gsidoffset = le32_to_cpu(pntsd->gsidoffset); + dacloffset = le32_to_cpu(pntsd->dacloffset); + cifs_dbg(NOISY, "revision %d type 0x%x ooffset 0x%x goffset 0x%x sacloffset 0x%x dacloffset 0x%x\n", + pntsd->revision, pntsd->type, osidoffset, gsidoffset, + le32_to_cpu(pntsd->sacloffset), dacloffset); +/* cifs_dump_mem("owner_sid: ", owner_sid_ptr, 64); */ + rc = sid_from_sd(pntsd, acl_len, osidoffset, &owner_sid_ptr); + if (rc) { + cifs_dbg(FYI, "%s: Error %d parsing Owner SID\n", __func__, rc); + return rc; + } + rc = sid_to_id(cifs_sb, owner_sid_ptr, fattr, SIDOWNER); + if (rc) { + cifs_dbg(FYI, "%s: Error %d mapping Owner SID to uid\n", + __func__, rc); + return rc; + } + + rc = sid_from_sd(pntsd, acl_len, gsidoffset, &group_sid_ptr); + if (rc) { + cifs_dbg(FYI, "%s: Error %d parsing Group SID\n", + __func__, rc); + return rc; + } + rc = sid_to_id(cifs_sb, group_sid_ptr, fattr, SIDGROUP); + if (rc) { + cifs_dbg(FYI, "%s: Error %d mapping Group SID to gid\n", + __func__, rc); + return rc; + } + + if (dacloffset) { + if (!dacl_offset_valid(acl_len, dacloffset)) { + cifs_dbg(VFS, "Server returned illegal DACL offset\n"); + return -EINVAL; + } + + dacl_ptr = (struct smb_acl *)((char *)pntsd + dacloffset); + parse_dacl(dacl_ptr, end_of_acl, owner_sid_ptr, + group_sid_ptr, fattr, get_mode_from_special_sid); + } else { + cifs_dbg(FYI, "no ACL\n"); /* BB grant all or default perms? */ + } + + return rc; +} + +/* Convert permission bits from mode to equivalent CIFS ACL */ +static int build_sec_desc(struct smb_ntsd *pntsd, struct smb_ntsd *pnntsd, + __u32 secdesclen, __u32 *pnsecdesclen, __u64 *pnmode, kuid_t uid, kgid_t gid, + bool mode_from_sid, bool id_from_sid, bool posix, int *aclflag) +{ + int rc = 0; + __u32 dacloffset; + __u32 ndacloffset; + __u32 sidsoffset; + struct smb_sid *owner_sid_ptr, *group_sid_ptr; + struct smb_sid *nowner_sid_ptr = NULL, *ngroup_sid_ptr = NULL; + struct smb_acl *dacl_ptr = NULL; /* no need for SACL ptr */ + struct smb_acl *ndacl_ptr = NULL; /* no need for SACL ptr */ + char *end_of_acl; + u16 size = 0; + __u32 osidoffset, gsidoffset; + + if (secdesclen < sizeof(struct smb_ntsd)) { + cifs_dbg(VFS, "ACL too small to parse security descriptor\n"); + return -EINVAL; + } + end_of_acl = ((char *)pntsd) + secdesclen; + + dacloffset = le32_to_cpu(pntsd->dacloffset); + if (dacloffset) { + if (!dacl_offset_valid(secdesclen, dacloffset)) { + cifs_dbg(VFS, "Server returned illegal DACL offset\n"); + return -EINVAL; + } + + dacl_ptr = (struct smb_acl *)((char *)pntsd + dacloffset); + rc = validate_dacl(dacl_ptr, end_of_acl); + if (rc) + return rc; + } + + osidoffset = le32_to_cpu(pntsd->osidoffset); + gsidoffset = le32_to_cpu(pntsd->gsidoffset); + rc = sid_from_sd(pntsd, secdesclen, osidoffset, &owner_sid_ptr); + if (rc) { + cifs_dbg(FYI, "%s: Error %d parsing Owner SID\n", __func__, rc); + return rc; + } + rc = sid_from_sd(pntsd, secdesclen, gsidoffset, &group_sid_ptr); + if (rc) { + cifs_dbg(FYI, "%s: Error %d parsing Group SID\n", __func__, rc); + return rc; + } + + if (pnmode && *pnmode != NO_CHANGE_64) { /* chmod */ + ndacloffset = sizeof(struct smb_ntsd); + ndacl_ptr = (struct smb_acl *)((char *)pnntsd + ndacloffset); + ndacl_ptr->revision = + dacloffset ? dacl_ptr->revision : cpu_to_le16(ACL_REVISION); + + ndacl_ptr->size = cpu_to_le16(0); + ndacl_ptr->num_aces = cpu_to_le16(0); + + rc = set_chmod_dacl(dacl_ptr, ndacl_ptr, owner_sid_ptr, group_sid_ptr, + pnmode, mode_from_sid, posix); + + sidsoffset = ndacloffset + le16_to_cpu(ndacl_ptr->size); + /* copy the non-dacl portion of secdesc */ + rc = copy_sec_desc(pntsd, pnntsd, sidsoffset, secdesclen, + pnsecdesclen, NULL, NULL); + if (rc) + return rc; + + *aclflag |= CIFS_ACL_DACL; + } else { + ndacloffset = sizeof(struct smb_ntsd); + ndacl_ptr = (struct smb_acl *)((char *)pnntsd + ndacloffset); + ndacl_ptr->revision = + dacloffset ? dacl_ptr->revision : cpu_to_le16(ACL_REVISION); + ndacl_ptr->num_aces = dacl_ptr ? dacl_ptr->num_aces : 0; + + if (uid_valid(uid)) { /* chown */ + uid_t id; + nowner_sid_ptr = kzalloc_obj(struct smb_sid); + if (!nowner_sid_ptr) { + rc = -ENOMEM; + goto chown_chgrp_exit; + } + id = from_kuid(&init_user_ns, uid); + if (id_from_sid) { + struct owner_sid *osid = (struct owner_sid *)nowner_sid_ptr; + /* Populate the user ownership fields S-1-5-88-1 */ + osid->Revision = 1; + osid->NumAuth = 3; + osid->Authority[5] = 5; + osid->SubAuthorities[0] = cpu_to_le32(88); + osid->SubAuthorities[1] = cpu_to_le32(1); + osid->SubAuthorities[2] = cpu_to_le32(id); + + } else { /* lookup sid with upcall */ + rc = id_to_sid(id, SIDOWNER, nowner_sid_ptr); + if (rc) { + cifs_dbg(FYI, "%s: Mapping error %d for owner id %d\n", + __func__, rc, id); + goto chown_chgrp_exit; + } + } + *aclflag |= CIFS_ACL_OWNER; + } + if (gid_valid(gid)) { /* chgrp */ + gid_t id; + ngroup_sid_ptr = kzalloc_obj(struct smb_sid); + if (!ngroup_sid_ptr) { + rc = -ENOMEM; + goto chown_chgrp_exit; + } + id = from_kgid(&init_user_ns, gid); + if (id_from_sid) { + struct owner_sid *gsid = (struct owner_sid *)ngroup_sid_ptr; + /* Populate the group ownership fields S-1-5-88-2 */ + gsid->Revision = 1; + gsid->NumAuth = 3; + gsid->Authority[5] = 5; + gsid->SubAuthorities[0] = cpu_to_le32(88); + gsid->SubAuthorities[1] = cpu_to_le32(2); + gsid->SubAuthorities[2] = cpu_to_le32(id); + + } else { /* lookup sid with upcall */ + rc = id_to_sid(id, SIDGROUP, ngroup_sid_ptr); + if (rc) { + cifs_dbg(FYI, "%s: Mapping error %d for group id %d\n", + __func__, rc, id); + goto chown_chgrp_exit; + } + } + *aclflag |= CIFS_ACL_GROUP; + } + + if (dacloffset) { + /* Replace ACEs for old owner with new one */ + size = replace_sids_and_copy_aces(dacl_ptr, ndacl_ptr, + owner_sid_ptr, group_sid_ptr, + nowner_sid_ptr, ngroup_sid_ptr, + aclflag); + ndacl_ptr->size = cpu_to_le16(size); + } + + sidsoffset = ndacloffset + le16_to_cpu(ndacl_ptr->size); + /* copy the non-dacl portion of secdesc */ + rc = copy_sec_desc(pntsd, pnntsd, sidsoffset, secdesclen, + pnsecdesclen, nowner_sid_ptr, ngroup_sid_ptr); + if (rc) + goto chown_chgrp_exit; + +chown_chgrp_exit: + /* errors could jump here. So make sure we return soon after this */ + kfree(nowner_sid_ptr); + kfree(ngroup_sid_ptr); + } + + return rc; +} + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY +struct smb_ntsd *get_cifs_acl_by_fid(struct cifs_sb_info *cifs_sb, + const struct cifs_fid *cifsfid, u32 *pacllen, + u32 info) +{ + struct smb_ntsd *pntsd = NULL; + unsigned int xid; + int rc; + struct tcon_link *tlink = cifs_sb_tlink(cifs_sb); + + if (IS_ERR(tlink)) + return ERR_CAST(tlink); + + xid = get_xid(); + rc = CIFSSMBGetCIFSACL(xid, tlink_tcon(tlink), cifsfid->netfid, &pntsd, + pacllen, info); + free_xid(xid); + + cifs_put_tlink(tlink); + + cifs_dbg(FYI, "%s: rc = %d ACL len %d\n", __func__, rc, *pacllen); + if (rc) + return ERR_PTR(rc); + return pntsd; +} + +static struct smb_ntsd *get_cifs_acl_by_path(struct cifs_sb_info *cifs_sb, + const char *path, u32 *pacllen, u32 info) +{ + struct smb_ntsd *pntsd = NULL; + int oplock = 0; + unsigned int xid; + int rc; + struct cifs_tcon *tcon; + struct tcon_link *tlink = cifs_sb_tlink(cifs_sb); + struct cifs_fid fid; + struct cifs_open_parms oparms; + + if (IS_ERR(tlink)) + return ERR_CAST(tlink); + + tcon = tlink_tcon(tlink); + xid = get_xid(); + + oparms = (struct cifs_open_parms) { + .tcon = tcon, + .cifs_sb = cifs_sb, + .desired_access = READ_CONTROL, + .create_options = cifs_create_options(cifs_sb, 0), + .disposition = FILE_OPEN, + .path = path, + .fid = &fid, + }; + + if (info & SACL_SECINFO) + oparms.desired_access |= SYSTEM_SECURITY; + + rc = CIFS_open(xid, &oparms, &oplock, NULL); + if (!rc) { + rc = CIFSSMBGetCIFSACL(xid, tcon, fid.netfid, &pntsd, pacllen, info); + CIFSSMBClose(xid, tcon, fid.netfid); + } + + cifs_put_tlink(tlink); + free_xid(xid); + + cifs_dbg(FYI, "%s: rc = %d ACL len %d\n", __func__, rc, *pacllen); + if (rc) + return ERR_PTR(rc); + return pntsd; +} + +/* Retrieve an ACL from the server */ +struct smb_ntsd *get_cifs_acl(struct cifs_sb_info *cifs_sb, + struct inode *inode, const char *path, + u32 *pacllen, u32 info) +{ + struct smb_ntsd *pntsd = NULL; + struct cifsFileInfo *open_file = NULL; + + if (inode) + open_file = find_readable_file(CIFS_I(inode), FIND_FSUID_ONLY); + if (!open_file) + return get_cifs_acl_by_path(cifs_sb, path, pacllen, info); + + pntsd = get_cifs_acl_by_fid(cifs_sb, &open_file->fid, pacllen, info); + cifsFileInfo_put(open_file); + return pntsd; +} + + /* Set an ACL on the server */ +int set_cifs_acl(struct smb_ntsd *pnntsd, __u32 acllen, + struct inode *inode, const char *path, int aclflag) +{ + int oplock = 0; + unsigned int xid; + int rc, access_flags = 0; + struct cifs_tcon *tcon; + struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); + struct tcon_link *tlink = cifs_sb_tlink(cifs_sb); + struct cifs_fid fid; + struct cifs_open_parms oparms; + + if (IS_ERR(tlink)) + return PTR_ERR(tlink); + + tcon = tlink_tcon(tlink); + xid = get_xid(); + + if (aclflag & CIFS_ACL_OWNER || aclflag & CIFS_ACL_GROUP) + access_flags |= WRITE_OWNER; + if (aclflag & CIFS_ACL_SACL) + access_flags |= SYSTEM_SECURITY; + if (aclflag & CIFS_ACL_DACL) + access_flags |= WRITE_DAC; + + oparms = (struct cifs_open_parms) { + .tcon = tcon, + .cifs_sb = cifs_sb, + .desired_access = access_flags, + .create_options = cifs_create_options(cifs_sb, 0), + .disposition = FILE_OPEN, + .path = path, + .fid = &fid, + }; + + rc = CIFS_open(xid, &oparms, &oplock, NULL); + if (rc) { + cifs_dbg(VFS, "Unable to open file to set ACL\n"); + goto out; + } + + rc = CIFSSMBSetCIFSACL(xid, tcon, fid.netfid, pnntsd, acllen, aclflag); + cifs_dbg(NOISY, "SetCIFSACL rc = %d\n", rc); + + CIFSSMBClose(xid, tcon, fid.netfid); +out: + free_xid(xid); + cifs_put_tlink(tlink); + return rc; +} +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + +/* Translate the CIFS ACL (similar to NTFS ACL) for a file into mode bits */ +int +cifs_acl_to_fattr(struct cifs_sb_info *cifs_sb, struct cifs_fattr *fattr, + struct inode *inode, bool mode_from_special_sid, + const char *path, const struct cifs_fid *pfid) +{ + struct smb_ntsd *pntsd = NULL; + u32 acllen = 0; + int rc = 0; + struct tcon_link *tlink = cifs_sb_tlink(cifs_sb); + struct smb_version_operations *ops; + const u32 info = OWNER_SECINFO | GROUP_SECINFO | DACL_SECINFO; + + cifs_dbg(NOISY, "converting ACL to mode for %s\n", path); + + if (IS_ERR(tlink)) + return PTR_ERR(tlink); + + ops = tlink_tcon(tlink)->ses->server->ops; + + if (pfid && (ops->get_acl_by_fid)) + pntsd = ops->get_acl_by_fid(cifs_sb, pfid, &acllen, info); + else if (ops->get_acl) + pntsd = ops->get_acl(cifs_sb, inode, path, &acllen, info); + else { + cifs_put_tlink(tlink); + return -EOPNOTSUPP; + } + /* if we can retrieve the ACL, now parse Access Control Entries, ACEs */ + if (IS_ERR(pntsd)) { + rc = PTR_ERR(pntsd); + cifs_dbg(VFS, "%s: error %d getting sec desc\n", __func__, rc); + } else if (mode_from_special_sid) { + rc = parse_sec_desc(cifs_sb, pntsd, acllen, fattr, true); + kfree(pntsd); + } else { + /* get approximated mode from ACL */ + rc = parse_sec_desc(cifs_sb, pntsd, acllen, fattr, false); + kfree(pntsd); + if (rc) + cifs_dbg(VFS, "parse sec desc failed rc = %d\n", rc); + } + + cifs_put_tlink(tlink); + + return rc; +} + +/* Convert mode bits to an ACL so we can update the ACL on the server */ +int +id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode, + kuid_t uid, kgid_t gid) +{ + int rc = 0; + int aclflag = 0; + __u32 secdesclen = 0; + __u32 nsecdesclen = 0; + __u32 dacloffset = 0; + struct smb_acl *dacl_ptr = NULL; + struct smb_ntsd *pntsd = NULL; /* acl obtained from server */ + struct smb_ntsd *pnntsd = NULL; /* modified acl to be sent to server */ + struct cifs_sb_info *cifs_sb = CIFS_SB(inode); + unsigned int sbflags; + struct tcon_link *tlink; + struct smb_version_operations *ops; + bool mode_from_sid, id_from_sid; + const u32 info = OWNER_SECINFO | GROUP_SECINFO | DACL_SECINFO; + bool posix; + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) + return PTR_ERR(tlink); + posix = tlink_tcon(tlink)->posix_extensions; + + ops = tlink_tcon(tlink)->ses->server->ops; + + cifs_dbg(NOISY, "set ACL from mode for %s\n", path); + + /* Get the security descriptor */ + + if (ops->get_acl == NULL) { + cifs_put_tlink(tlink); + return -EOPNOTSUPP; + } + + pntsd = ops->get_acl(cifs_sb, inode, path, &secdesclen, info); + if (IS_ERR(pntsd)) { + rc = PTR_ERR(pntsd); + cifs_dbg(VFS, "%s: error %d getting sec desc\n", __func__, rc); + cifs_put_tlink(tlink); + return rc; + } + + sbflags = cifs_sb_flags(cifs_sb); + mode_from_sid = sbflags & CIFS_MOUNT_MODE_FROM_SID; + id_from_sid = sbflags & CIFS_MOUNT_UID_FROM_ACL; + + /* Potentially, five new ACEs can be added to the ACL for U,G,O mapping */ + if (pnmode && *pnmode != NO_CHANGE_64) { /* chmod */ + if (posix) + nsecdesclen = 1 * sizeof(struct smb_ace); + else if (mode_from_sid) + nsecdesclen = secdesclen + (2 * sizeof(struct smb_ace)); + else /* cifsacl */ + nsecdesclen = secdesclen + (5 * sizeof(struct smb_ace)); + } else { /* chown */ + /* When ownership changes, changes new owner sid length could be different */ + nsecdesclen = sizeof(struct smb_ntsd) + (sizeof(struct smb_sid) * 2); + dacloffset = le32_to_cpu(pntsd->dacloffset); + if (dacloffset) { + if (!dacl_offset_valid(secdesclen, dacloffset)) { + cifs_dbg(VFS, "Server returned illegal DACL offset\n"); + rc = -EINVAL; + goto id_mode_to_cifs_acl_exit; + } + + dacl_ptr = (struct smb_acl *)((char *)pntsd + dacloffset); + rc = validate_dacl(dacl_ptr, (char *)pntsd + secdesclen); + if (rc) { + kfree(pntsd); + cifs_put_tlink(tlink); + return rc; + } + if (mode_from_sid) + nsecdesclen += + le16_to_cpu(dacl_ptr->num_aces) * sizeof(struct smb_ace); + else /* cifsacl */ + nsecdesclen += le16_to_cpu(dacl_ptr->size); + } + } + + /* + * Add three ACEs for owner, group, everyone getting rid of other ACEs + * as chmod disables ACEs and set the security descriptor. Allocate + * memory for the smb header, set security descriptor request security + * descriptor parameters, and security descriptor itself + */ + nsecdesclen = max_t(u32, nsecdesclen, DEFAULT_SEC_DESC_LEN); + pnntsd = kzalloc(nsecdesclen, GFP_KERNEL); + if (!pnntsd) { + kfree(pntsd); + cifs_put_tlink(tlink); + return -ENOMEM; + } + + rc = build_sec_desc(pntsd, pnntsd, secdesclen, &nsecdesclen, pnmode, uid, gid, + mode_from_sid, id_from_sid, posix, &aclflag); + + cifs_dbg(NOISY, "build_sec_desc rc: %d\n", rc); + + if (rc != 0) + goto id_mode_to_cifs_acl_exit; + + if (aclflag == 0) { + cifs_dbg(FYI, "set_cifs_acl aclflag=0, no change mapped\n"); + goto id_mode_to_cifs_acl_exit; + } + + if (ops->set_acl == NULL) { + rc = -EOPNOTSUPP; + goto id_mode_to_cifs_acl_exit; + } + + /* Set the security descriptor */ + rc = ops->set_acl(pnntsd, nsecdesclen, inode, path, aclflag); + cifs_dbg(NOISY, "set_cifs_acl rc: %d\n", rc); + +id_mode_to_cifs_acl_exit: + cifs_put_tlink(tlink); + + kfree(pnntsd); + kfree(pntsd); + return rc; +} + +struct posix_acl *cifs_get_acl(struct mnt_idmap *idmap, + struct dentry *dentry, int type) +{ +#if defined(CONFIG_CIFS_ALLOW_INSECURE_LEGACY) && defined(CONFIG_CIFS_POSIX) + struct posix_acl *acl = NULL; + ssize_t rc = -EOPNOTSUPP; + unsigned int xid; + struct super_block *sb = dentry->d_sb; + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + struct tcon_link *tlink; + struct cifs_tcon *pTcon; + const char *full_path; + void *page; + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) + return ERR_CAST(tlink); + pTcon = tlink_tcon(tlink); + + xid = get_xid(); + page = alloc_dentry_path(); + + full_path = build_path_from_dentry(dentry, page); + if (IS_ERR(full_path)) { + acl = ERR_CAST(full_path); + goto out; + } + + /* return alt name if available as pseudo attr */ + switch (type) { + case ACL_TYPE_ACCESS: + if (sb->s_flags & SB_POSIXACL) + rc = cifs_do_get_acl(xid, pTcon, full_path, &acl, + ACL_TYPE_ACCESS, + cifs_sb->local_nls, + cifs_remap(cifs_sb)); + break; + + case ACL_TYPE_DEFAULT: + if (sb->s_flags & SB_POSIXACL) + rc = cifs_do_get_acl(xid, pTcon, full_path, &acl, + ACL_TYPE_DEFAULT, + cifs_sb->local_nls, + cifs_remap(cifs_sb)); + break; + } + + if (rc < 0) { + if (rc == -EINVAL) + acl = ERR_PTR(-EOPNOTSUPP); + else + acl = ERR_PTR(rc); + } + +out: + free_dentry_path(page); + free_xid(xid); + cifs_put_tlink(tlink); + return acl; +#else + return ERR_PTR(-EOPNOTSUPP); +#endif +} + +int cifs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type) +{ +#if defined(CONFIG_CIFS_ALLOW_INSECURE_LEGACY) && defined(CONFIG_CIFS_POSIX) + int rc = -EOPNOTSUPP; + unsigned int xid; + struct super_block *sb = dentry->d_sb; + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + struct tcon_link *tlink; + struct cifs_tcon *pTcon; + const char *full_path; + void *page; + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) + return PTR_ERR(tlink); + pTcon = tlink_tcon(tlink); + + xid = get_xid(); + page = alloc_dentry_path(); + + full_path = build_path_from_dentry(dentry, page); + if (IS_ERR(full_path)) { + rc = PTR_ERR(full_path); + goto out; + } + + if (!acl) + goto out; + + /* return dos attributes as pseudo xattr */ + /* return alt name if available as pseudo attr */ + + /* if proc/fs/cifs/streamstoxattr is set then + search server for EAs or streams to + returns as xattrs */ + if (posix_acl_xattr_size(acl->a_count) > CIFSMaxBufSize) { + cifs_dbg(FYI, "size of EA value too large\n"); + rc = -EOPNOTSUPP; + goto out; + } + + switch (type) { + case ACL_TYPE_ACCESS: + if (sb->s_flags & SB_POSIXACL) + rc = cifs_do_set_acl(xid, pTcon, full_path, acl, + ACL_TYPE_ACCESS, + cifs_sb->local_nls, + cifs_remap(cifs_sb)); + break; + + case ACL_TYPE_DEFAULT: + if (sb->s_flags & SB_POSIXACL) + rc = cifs_do_set_acl(xid, pTcon, full_path, acl, + ACL_TYPE_DEFAULT, + cifs_sb->local_nls, + cifs_remap(cifs_sb)); + break; + } + +out: + free_dentry_path(page); + free_xid(xid); + cifs_put_tlink(tlink); + return rc; +#else + return -EOPNOTSUPP; +#endif +} diff --git a/fs/smb/client/cifsacl.h b/fs/smb/client/cifsacl.h new file mode 100644 index 000000000..31b51a8fc --- /dev/null +++ b/fs/smb/client/cifsacl.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +/* + * + * Copyright (c) International Business Machines Corp., 2007 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ + +#ifndef _CIFSACL_H +#define _CIFSACL_H + +#include "../common/smbacl.h" + +#define READ_BIT 0x4 +#define WRITE_BIT 0x2 +#define EXEC_BIT 0x1 + +#define ACL_OWNER_MASK 0700 +#define ACL_GROUP_MASK 0070 +#define ACL_EVERYONE_MASK 0007 + +#define UBITSHIFT 6 +#define GBITSHIFT 3 + +/* + * Security Descriptor length containing DACL with 3 ACEs (one each for + * owner, group and world). + */ +#define DEFAULT_SEC_DESC_LEN (sizeof(struct smb_ntsd) + \ + sizeof(struct smb_acl) + \ + (sizeof(struct smb_ace) * 4)) + +/* + * The current SMB3 form of security descriptor is similar to what was used for + * cifs (see above) but some fields are split, and fields in the struct below + * matches names of fields to the spec, MS-DTYP (see sections 2.4.5 and + * 2.4.6). Note that "CamelCase" fields are used in this struct in order to + * match the MS-DTYP and MS-SMB2 specs which define the wire format. + */ +struct smb3_sd { + __u8 Revision; /* revision level, MUST be one */ + __u8 Sbz1; /* only meaningful if 'RM' flag set below */ + __le16 Control; + __le32 OffsetOwner; + __le32 OffsetGroup; + __le32 OffsetSacl; + __le32 OffsetDacl; +} __packed; + +/* Meaning of 'Control' field flags */ +#define ACL_CONTROL_SR 0x8000 /* Self relative */ +#define ACL_CONTROL_RM 0x4000 /* Resource manager control bits */ +#define ACL_CONTROL_PS 0x2000 /* SACL protected from inherits */ +#define ACL_CONTROL_PD 0x1000 /* DACL protected from inherits */ +#define ACL_CONTROL_SI 0x0800 /* SACL Auto-Inherited */ +#define ACL_CONTROL_DI 0x0400 /* DACL Auto-Inherited */ +#define ACL_CONTROL_SC 0x0200 /* SACL computed through inheritance */ +#define ACL_CONTROL_DC 0x0100 /* DACL computed through inheritance */ +#define ACL_CONTROL_SS 0x0080 /* Create server ACL */ +#define ACL_CONTROL_DT 0x0040 /* DACL provided by trusted source */ +#define ACL_CONTROL_SD 0x0020 /* SACL defaulted */ +#define ACL_CONTROL_SP 0x0010 /* SACL is present on object */ +#define ACL_CONTROL_DD 0x0008 /* DACL defaulted */ +#define ACL_CONTROL_DP 0x0004 /* DACL is present on object */ +#define ACL_CONTROL_GD 0x0002 /* Group was defaulted */ +#define ACL_CONTROL_OD 0x0001 /* User was defaulted */ + +/* Meaning of AclRevision flags */ +#define ACL_REVISION 0x02 /* See section 2.4.4.1 of MS-DTYP */ +#define ACL_REVISION_DS 0x04 /* Additional AceTypes allowed */ + +struct smb3_acl { + u8 AclRevision; /* revision level */ + u8 Sbz1; /* MBZ */ + __le16 AclSize; + __le16 AceCount; + __le16 Sbz2; /* MBZ */ +} __packed; + +/* + * Used to store the special 'NFS SIDs' used to persist the POSIX uid and gid + * See http://technet.microsoft.com/en-us/library/hh509017(v=ws.10).aspx + */ +struct owner_sid { + u8 Revision; + u8 NumAuth; + u8 Authority[6]; + __le32 SubAuthorities[3]; +} __packed; + +struct owner_group_sids { + struct owner_sid owner; + struct owner_sid group; +} __packed; + +/* + * Minimum security identifier can be one for system defined Users + * and Groups such as NULL SID and World or Built-in accounts such + * as Administrator and Guest and consists of + * Revision + Num (Sub)Auths + Authority + Domain (one Subauthority) + */ +#define MIN_SID_LEN (1 + 1 + 6 + 4) /* in bytes */ + +/* + * Minimum security descriptor can be one without any SACL and DACL and can + * consist of revision, type, and two sids of minimum size for owner and group + */ +#define MIN_SEC_DESC_LEN (sizeof(struct smb_ntsd) + (2 * MIN_SID_LEN)) + +#endif /* _CIFSACL_H */ diff --git a/fs/smb/client/cifsencrypt.c b/fs/smb/client/cifsencrypt.c new file mode 100644 index 000000000..34804e984 --- /dev/null +++ b/fs/smb/client/cifsencrypt.c @@ -0,0 +1,514 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * Encryption and hashing operations relating to NTLM, NTLMv2. See MS-NLMP + * for more detailed information + * + * Copyright (C) International Business Machines Corp., 2005,2013 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ + +#include +#include +#include "cifsglob.h" +#include "cifs_debug.h" +#include "cifs_unicode.h" +#include "cifsproto.h" +#include "ntlmssp.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static size_t cifs_sig_step(void *iter_base, size_t progress, size_t len, + void *priv, void *priv2) +{ + struct cifs_calc_sig_ctx *ctx = priv; + + if (ctx->md5) + md5_update(ctx->md5, iter_base, len); + else if (ctx->hmac) + hmac_sha256_update(ctx->hmac, iter_base, len); + else + aes_cmac_update(ctx->cmac, iter_base, len); + return 0; /* Return value is length *not* processed, i.e. 0. */ +} + +static void cifs_sig_final(struct cifs_calc_sig_ctx *ctx, u8 *out) +{ + if (ctx->md5) + md5_final(ctx->md5, out); + else if (ctx->hmac) + hmac_sha256_final(ctx->hmac, out); + else + aes_cmac_final(ctx->cmac, out); +} + +/* + * Pass the data from an iterator into a hash. + */ +static int cifs_sig_iter(const struct iov_iter *iter, size_t maxsize, + struct cifs_calc_sig_ctx *ctx) +{ + struct iov_iter tmp_iter = *iter; + size_t did; + + did = iterate_and_advance_kernel(&tmp_iter, maxsize, ctx, NULL, + cifs_sig_step); + if (did != maxsize) + return smb_EIO2(smb_eio_trace_sig_iter, did, maxsize); + return 0; +} + +int __cifs_calc_signature(struct smb_rqst *rqst, struct TCP_Server_Info *server, + char *signature, struct cifs_calc_sig_ctx *ctx) +{ + struct iov_iter iter; + ssize_t rc; + size_t size = 0; + + for (int i = 0; i < rqst->rq_nvec; i++) + size += rqst->rq_iov[i].iov_len; + + iov_iter_kvec(&iter, ITER_SOURCE, rqst->rq_iov, rqst->rq_nvec, size); + + if (iov_iter_count(&iter) <= 4) + return smb_EIO2(smb_eio_trace_sig_data_too_small, + iov_iter_count(&iter), 4); + + rc = cifs_sig_iter(&iter, iov_iter_count(&iter), ctx); + if (rc < 0) + return rc; + + rc = cifs_sig_iter(&rqst->rq_iter, iov_iter_count(&rqst->rq_iter), ctx); + if (rc < 0) + return rc; + + cifs_sig_final(ctx, signature); + return 0; +} + +/* Build a proper attribute value/target info pairs blob. + * Fill in netbios and dns domain name and workstation name + * and client time (total five av pairs and + one end of fields indicator. + * Allocate domain name which gets freed when session struct is deallocated. + */ +static int +build_avpair_blob(struct cifs_ses *ses, const struct nls_table *nls_cp) +{ + unsigned int dlen; + unsigned int size = 2 * sizeof(struct ntlmssp2_name); + char *defdmname = "WORKGROUP"; + unsigned char *blobptr; + struct ntlmssp2_name *attrptr; + + if (!ses->domainName) { + ses->domainName = kstrdup(defdmname, GFP_KERNEL); + if (!ses->domainName) + return -ENOMEM; + } + + dlen = strlen(ses->domainName); + + /* + * The length of this blob is two times the size of a + * structure (av pair) which holds name/size + * ( for NTLMSSP_AV_NB_DOMAIN_NAME followed by NTLMSSP_AV_EOL ) + + * unicode length of a netbios domain name + */ + kfree_sensitive(ses->auth_key.response); + ses->auth_key.len = size + 2 * dlen; + ses->auth_key.response = kzalloc(ses->auth_key.len, GFP_KERNEL); + if (!ses->auth_key.response) { + ses->auth_key.len = 0; + return -ENOMEM; + } + + blobptr = ses->auth_key.response; + attrptr = (struct ntlmssp2_name *) blobptr; + + /* + * As defined in MS-NTLM 3.3.2, just this av pair field + * is sufficient as part of the temp + */ + attrptr->type = cpu_to_le16(NTLMSSP_AV_NB_DOMAIN_NAME); + attrptr->length = cpu_to_le16(2 * dlen); + blobptr = (unsigned char *)attrptr + sizeof(struct ntlmssp2_name); + cifs_strtoUTF16((__le16 *)blobptr, ses->domainName, dlen, nls_cp); + + return 0; +} + +#define AV_TYPE(av) (le16_to_cpu(av->type)) +#define AV_LEN(av) (le16_to_cpu(av->length)) +#define AV_DATA_PTR(av) ((void *)av->data) + +#define av_for_each_entry(ses, av) \ + for (av = NULL; (av = find_next_av(ses, av));) + +static struct ntlmssp2_name *find_next_av(struct cifs_ses *ses, + struct ntlmssp2_name *av) +{ + u16 len; + u8 *end; + + end = (u8 *)ses->auth_key.response + ses->auth_key.len; + if (!av) { + if (unlikely(!ses->auth_key.response || !ses->auth_key.len)) + return NULL; + av = (void *)ses->auth_key.response; + } else { + av = (void *)((u8 *)av + sizeof(*av) + AV_LEN(av)); + } + + if ((u8 *)av + sizeof(*av) > end) + return NULL; + + len = AV_LEN(av); + if (AV_TYPE(av) == NTLMSSP_AV_EOL) + return NULL; + if ((u8 *)av + sizeof(*av) + len > end) + return NULL; + return av; +} + +/* + * Check if server has provided av pair of @type in the NTLMSSP + * CHALLENGE_MESSAGE blob. + */ +static int find_av_name(struct cifs_ses *ses, u16 type, char **name, u16 maxlen) +{ + const struct nls_table *nlsc = ses->local_nls; + struct ntlmssp2_name *av; + u16 len, nlen; + + if (*name) + return 0; + + av_for_each_entry(ses, av) { + len = AV_LEN(av); + if (AV_TYPE(av) != type || !len) + continue; + if (!IS_ALIGNED(len, sizeof(__le16))) { + cifs_dbg(VFS | ONCE, "%s: bad length(%u) for type %u\n", + __func__, len, type); + continue; + } + nlen = len / sizeof(__le16); + if (nlen <= maxlen) { + ++nlen; + *name = kmalloc(nlen, GFP_KERNEL); + if (!*name) + return -ENOMEM; + cifs_from_utf16(*name, AV_DATA_PTR(av), nlen, + len, nlsc, NO_MAP_UNI_RSVD); + break; + } + } + return 0; +} + +/* Server has provided av pairs/target info in the type 2 challenge + * packet and we have plucked it and stored within smb session. + * We parse that blob here to find the server given timestamp + * as part of ntlmv2 authentication (or local current time as + * default in case of failure) + */ +static __le64 find_timestamp(struct cifs_ses *ses) +{ + struct ntlmssp2_name *av; + struct timespec64 ts; + + av_for_each_entry(ses, av) { + if (AV_TYPE(av) == NTLMSSP_AV_TIMESTAMP && + AV_LEN(av) == sizeof(u64)) + return *((__le64 *)AV_DATA_PTR(av)); + } + ktime_get_real_ts64(&ts); + return cpu_to_le64(cifs_UnixTimeToNT(ts)); +} + +static int calc_ntlmv2_hash(struct cifs_ses *ses, char *ntlmv2_hash, + const struct nls_table *nls_cp) +{ + int len; + char nt_hash[CIFS_NTHASH_SIZE]; + struct hmac_md5_ctx hmac_ctx; + __le16 *user; + wchar_t *domain; + wchar_t *server; + + /* calculate md4 hash of password */ + E_md4hash(ses->password, nt_hash, nls_cp); + + hmac_md5_init_usingrawkey(&hmac_ctx, nt_hash, CIFS_NTHASH_SIZE); + + /* convert ses->user_name to unicode */ + len = ses->user_name ? strlen(ses->user_name) : 0; + user = kmalloc(2 + (len * 2), GFP_KERNEL); + if (user == NULL) + return -ENOMEM; + + if (len) { + len = cifs_strtoUTF16(user, ses->user_name, len, nls_cp); + UniStrupr(user); + } else { + *(u16 *)user = 0; + } + + hmac_md5_update(&hmac_ctx, (const u8 *)user, 2 * len); + kfree(user); + + /* convert ses->domainName to unicode and uppercase */ + if (ses->domainName) { + len = strlen(ses->domainName); + + domain = kmalloc(2 + (len * 2), GFP_KERNEL); + if (domain == NULL) + return -ENOMEM; + + len = cifs_strtoUTF16((__le16 *)domain, ses->domainName, len, + nls_cp); + hmac_md5_update(&hmac_ctx, (const u8 *)domain, 2 * len); + kfree(domain); + } else { + /* We use ses->ip_addr if no domain name available */ + len = strlen(ses->ip_addr); + + server = kmalloc(2 + (len * 2), GFP_KERNEL); + if (server == NULL) + return -ENOMEM; + + len = cifs_strtoUTF16((__le16 *)server, ses->ip_addr, len, nls_cp); + hmac_md5_update(&hmac_ctx, (const u8 *)server, 2 * len); + kfree(server); + } + + hmac_md5_final(&hmac_ctx, ntlmv2_hash); + return 0; +} + +static void CalcNTLMv2_response(const struct cifs_ses *ses, char *ntlmv2_hash) +{ + struct ntlmv2_resp *ntlmv2 = (struct ntlmv2_resp *) + (ses->auth_key.response + CIFS_SESS_KEY_SIZE); + unsigned int hash_len; + + /* The MD5 hash starts at challenge_key.key */ + hash_len = ses->auth_key.len - (CIFS_SESS_KEY_SIZE + + offsetof(struct ntlmv2_resp, challenge.key[0])); + + if (ses->server->negflavor == CIFS_NEGFLAVOR_EXTENDED) + memcpy(ntlmv2->challenge.key, ses->ntlmssp->cryptkey, CIFS_SERVER_CHALLENGE_SIZE); + else + memcpy(ntlmv2->challenge.key, ses->server->cryptkey, CIFS_SERVER_CHALLENGE_SIZE); + + /* Note that the HMAC-MD5 value overwrites ntlmv2->challenge.key */ + hmac_md5_usingrawkey(ntlmv2_hash, CIFS_HMAC_MD5_HASH_SIZE, + ntlmv2->challenge.key, hash_len, + ntlmv2->ntlmv2_hash); +} + +/* + * Set up NTLMv2 response blob with SPN (cifs/) appended to the + * existing list of AV pairs. + */ +static int set_auth_key_response(struct cifs_ses *ses) +{ + size_t baselen = CIFS_SESS_KEY_SIZE + sizeof(struct ntlmv2_resp); + size_t len, spnlen, tilen = 0, num_avs = 2 /* SPN + EOL */; + struct TCP_Server_Info *server = ses->server; + char *spn __free(kfree) = NULL; + struct ntlmssp2_name *av; + char *rsp = NULL; + int rc; + + spnlen = strlen(server->hostname); + len = sizeof("cifs/") + spnlen; + spn = kmalloc(len, GFP_KERNEL); + if (!spn) { + rc = -ENOMEM; + goto out; + } + + spnlen = scnprintf(spn, len, "cifs/%.*s", + (int)spnlen, server->hostname); + + av_for_each_entry(ses, av) + tilen += sizeof(*av) + AV_LEN(av); + + len = baselen + tilen + spnlen * sizeof(__le16) + num_avs * sizeof(*av); + rsp = kmalloc(len, GFP_KERNEL); + if (!rsp) { + rc = -ENOMEM; + goto out; + } + + memcpy(rsp + baselen, ses->auth_key.response, tilen); + av = (void *)(rsp + baselen + tilen); + av->type = cpu_to_le16(NTLMSSP_AV_TARGET_NAME); + av->length = cpu_to_le16(spnlen * sizeof(__le16)); + cifs_strtoUTF16((__le16 *)av->data, spn, spnlen, ses->local_nls); + av = (void *)((__u8 *)av + sizeof(*av) + AV_LEN(av)); + av->type = cpu_to_le16(NTLMSSP_AV_EOL); + av->length = 0; + + rc = 0; + ses->auth_key.len = len; +out: + ses->auth_key.response = rsp; + return rc; +} + +int +setup_ntlmv2_rsp(struct cifs_ses *ses, const struct nls_table *nls_cp) +{ + unsigned char *tiblob = NULL; /* target info blob */ + struct ntlmv2_resp *ntlmv2; + char ntlmv2_hash[16]; + __le64 rsp_timestamp; + __u64 cc; + int rc; + + if (nls_cp == NULL) { + cifs_dbg(VFS, "%s called with nls_cp==NULL\n", __func__); + return -EINVAL; + } + + if (ses->server->negflavor == CIFS_NEGFLAVOR_EXTENDED) { + if (!ses->domainName) { + if (ses->domainAuto) { + /* + * Domain (workgroup) hasn't been specified in + * mount options, so try to find it in + * CHALLENGE_MESSAGE message and then use it as + * part of NTLMv2 authentication. + */ + rc = find_av_name(ses, NTLMSSP_AV_NB_DOMAIN_NAME, + &ses->domainName, + CIFS_MAX_DOMAINNAME_LEN); + if (rc) + goto setup_ntlmv2_rsp_ret; + } else { + ses->domainName = kstrdup("", GFP_KERNEL); + if (!ses->domainName) { + rc = -ENOMEM; + goto setup_ntlmv2_rsp_ret; + } + } + } + rc = find_av_name(ses, NTLMSSP_AV_DNS_DOMAIN_NAME, + &ses->dns_dom, CIFS_MAX_DOMAINNAME_LEN); + if (rc) + goto setup_ntlmv2_rsp_ret; + } else { + rc = build_avpair_blob(ses, nls_cp); + if (rc) { + cifs_dbg(VFS, "error %d building av pair blob\n", rc); + goto setup_ntlmv2_rsp_ret; + } + } + + /* Must be within 5 minutes of the server (or in range +/-2h + * in case of Mac OS X), so simply carry over server timestamp + * (as Windows 7 does) + */ + rsp_timestamp = find_timestamp(ses); + get_random_bytes(&cc, sizeof(cc)); + + cifs_server_lock(ses->server); + + tiblob = ses->auth_key.response; + rc = set_auth_key_response(ses); + if (rc) { + ses->auth_key.len = 0; + goto unlock; + } + + ntlmv2 = (struct ntlmv2_resp *) + (ses->auth_key.response + CIFS_SESS_KEY_SIZE); + ntlmv2->blob_signature = cpu_to_le32(0x00000101); + ntlmv2->reserved = 0; + ntlmv2->time = rsp_timestamp; + ntlmv2->client_chal = cc; + ntlmv2->reserved2 = 0; + + if (fips_enabled) { + cifs_dbg(VFS, "NTLMv2 support is disabled due to FIPS\n"); + rc = -EOPNOTSUPP; + goto unlock; + } + + /* calculate ntlmv2_hash */ + rc = calc_ntlmv2_hash(ses, ntlmv2_hash, nls_cp); + if (rc) { + cifs_dbg(VFS, "Could not get NTLMv2 hash, rc=%d\n", rc); + goto unlock; + } + + /* calculate first part of the client response (CR1) */ + CalcNTLMv2_response(ses, ntlmv2_hash); + + /* now calculate the session key for NTLMv2 */ + hmac_md5_usingrawkey(ntlmv2_hash, CIFS_HMAC_MD5_HASH_SIZE, + ntlmv2->ntlmv2_hash, CIFS_HMAC_MD5_HASH_SIZE, + ses->auth_key.response); + rc = 0; +unlock: + cifs_server_unlock(ses->server); +setup_ntlmv2_rsp_ret: + kfree_sensitive(tiblob); + + return rc; +} + +int +calc_seckey(struct cifs_ses *ses) +{ + unsigned char sec_key[CIFS_SESS_KEY_SIZE]; /* a nonce */ + struct arc4_ctx *ctx_arc4; + + if (fips_enabled) + return -ENODEV; + + get_random_bytes(sec_key, CIFS_SESS_KEY_SIZE); + + ctx_arc4 = kmalloc_obj(*ctx_arc4); + if (!ctx_arc4) { + cifs_dbg(VFS, "Could not allocate arc4 context\n"); + return -ENOMEM; + } + + arc4_setkey(ctx_arc4, ses->auth_key.response, CIFS_SESS_KEY_SIZE); + arc4_crypt(ctx_arc4, ses->ntlmssp->ciphertext, sec_key, + CIFS_CPHTXT_SIZE); + + /* make secondary_key/nonce as session key */ + memcpy(ses->auth_key.response, sec_key, CIFS_SESS_KEY_SIZE); + /* and make len as that of session key only */ + ses->auth_key.len = CIFS_SESS_KEY_SIZE; + + memzero_explicit(sec_key, CIFS_SESS_KEY_SIZE); + kfree_sensitive(ctx_arc4); + return 0; +} + +void +cifs_crypto_secmech_release(struct TCP_Server_Info *server) +{ + if (server->secmech.enc) { + crypto_free_aead(server->secmech.enc); + server->secmech.enc = NULL; + } + if (server->secmech.dec) { + crypto_free_aead(server->secmech.dec); + server->secmech.dec = NULL; + } +} diff --git a/fs/smb/client/cifsfs.c b/fs/smb/client/cifsfs.c new file mode 100644 index 000000000..a1dacc7d8 --- /dev/null +++ b/fs/smb/client/cifsfs.c @@ -0,0 +1,2225 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * Copyright (C) International Business Machines Corp., 2002,2008 + * Author(s): Steve French (sfrench@us.ibm.com) + * + * Common Internet FileSystem (CIFS) client + * + */ + +/* Note that BB means BUGBUG (ie something to fix eventually) */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "cifsfs.h" +#define DECLARE_GLOBALS_HERE +#include "cifsglob.h" +#include "cifsproto.h" +#include "smb2proto.h" +#include "cifs_debug.h" +#include "cifs_fs_sb.h" +#include "cifs_spnego.h" +#include "fscache.h" +#ifdef CONFIG_CIFS_DFS_UPCALL +#include "dfs_cache.h" +#endif +#ifdef CONFIG_CIFS_SWN_UPCALL +#include "netlink.h" +#endif +#include "fs_context.h" +#include "cached_dir.h" + +/* + * DOS dates from 1980/1/1 through 2107/12/31 + * Protocol specifications indicate the range should be to 119, which + * limits maximum year to 2099. But this range has not been checked. + */ +#define SMB_DATE_MAX (127<<9 | 12<<5 | 31) +#define SMB_DATE_MIN (0<<9 | 1<<5 | 1) +#define SMB_TIME_MAX (23<<11 | 59<<5 | 29) + +int cifsFYI = 0; +bool traceSMB; +bool enable_oplocks = true; +bool linuxExtEnabled = true; +bool lookupCacheEnabled = true; +bool disable_legacy_dialects; /* false by default */ +bool enable_gcm_256 = true; +bool require_gcm_256; /* false by default */ +bool enable_negotiate_signing; /* false by default */ +unsigned int global_secflags = CIFSSEC_DEF; +/* unsigned int ntlmv2_support = 0; */ + +/* + * Global transaction id (XID) information + */ +unsigned int GlobalCurrentXid; /* protected by GlobalMid_Lock */ +unsigned int GlobalTotalActiveXid; /* prot by GlobalMid_Lock */ +unsigned int GlobalMaxActiveXid; /* prot by GlobalMid_Lock */ +DEFINE_SPINLOCK(GlobalMid_Lock); /* protects above & list operations on midQ entries */ + +/* + * Global counters, updated atomically + */ +atomic_t sesInfoAllocCount; +atomic_t tconInfoAllocCount; +atomic_t tcpSesNextId; +atomic_t tcpSesAllocCount; +atomic_t tcpSesReconnectCount; +atomic_t tconInfoReconnectCount; + +atomic_t mid_count; +atomic_t buf_alloc_count; +atomic_t small_buf_alloc_count; +#ifdef CONFIG_CIFS_STATS2 +atomic_t total_buf_alloc_count; +atomic_t total_small_buf_alloc_count; +#endif/* STATS2 */ +struct list_head cifs_tcp_ses_list; +DEFINE_SPINLOCK(cifs_tcp_ses_lock); +static const struct super_operations cifs_super_ops; +unsigned int CIFSMaxBufSize = CIFS_MAX_MSGSIZE; +module_param(CIFSMaxBufSize, uint, 0444); +MODULE_PARM_DESC(CIFSMaxBufSize, "Network buffer size (not including header) " + "for CIFS requests. " + "Default: 16384 Range: 8192 to 130048"); +unsigned int cifs_min_rcv = CIFS_MIN_RCV_POOL; +module_param(cifs_min_rcv, uint, 0444); +MODULE_PARM_DESC(cifs_min_rcv, "Network buffers in pool. Default: 4 Range: " + "1 to 64"); +unsigned int cifs_min_small = 30; +module_param(cifs_min_small, uint, 0444); +MODULE_PARM_DESC(cifs_min_small, "Small network buffers in pool. Default: 30 " + "Range: 2 to 256"); +unsigned int cifs_max_pending = CIFS_MAX_REQ; +module_param(cifs_max_pending, uint, 0444); +MODULE_PARM_DESC(cifs_max_pending, "Simultaneous requests to server for " + "CIFS/SMB1 dialect (N/A for SMB3) " + "Default: 32767 Range: 2 to 32767."); +unsigned int dir_cache_timeout = 30; +module_param(dir_cache_timeout, uint, 0644); +MODULE_PARM_DESC(dir_cache_timeout, "Number of seconds to cache directory contents for which we have a lease. Default: 30 " + "Range: 1 to 65000 seconds, 0 to disable caching dir contents"); +/* Module-wide total cached dirents (in bytes) across all tcons */ +atomic64_t cifs_dircache_bytes_used = ATOMIC64_INIT(0); + +atomic_t cifs_sillycounter; +atomic_t cifs_tmpcounter; + +#ifdef CONFIG_CIFS_STATS2 +unsigned int slow_rsp_threshold = 1; +module_param(slow_rsp_threshold, uint, 0644); +MODULE_PARM_DESC(slow_rsp_threshold, "Amount of time (in seconds) to wait " + "before logging that a response is delayed. " + "Default: 1 (if set to 0 disables msg)."); +#endif /* STATS2 */ + +module_param(enable_oplocks, bool, 0644); +MODULE_PARM_DESC(enable_oplocks, "Enable or disable oplocks. Default: y/Y/1"); + +module_param(enable_gcm_256, bool, 0644); +MODULE_PARM_DESC(enable_gcm_256, "Enable requesting strongest (256 bit) GCM encryption. Default: y/Y/1"); + +module_param(require_gcm_256, bool, 0644); +MODULE_PARM_DESC(require_gcm_256, "Require strongest (256 bit) GCM encryption. Default: n/N/0"); + +module_param(enable_negotiate_signing, bool, 0644); +MODULE_PARM_DESC(enable_negotiate_signing, "Enable negotiating packet signing algorithm with server. Default: n/N/0"); + +module_param(disable_legacy_dialects, bool, 0644); +MODULE_PARM_DESC(disable_legacy_dialects, "To improve security it may be " + "helpful to restrict the ability to " + "override the default dialects (SMB2.1, " + "SMB3 and SMB3.02) on mount with old " + "dialects (CIFS/SMB1 and SMB2) since " + "vers=1.0 (CIFS/SMB1) and vers=2.0 are weaker" + " and less secure. Default: n/N/0"); + +struct workqueue_struct *cifsiod_wq; +struct workqueue_struct *decrypt_wq; +struct workqueue_struct *fileinfo_put_wq; +struct workqueue_struct *cifsoplockd_wq; +struct workqueue_struct *deferredclose_wq; +struct workqueue_struct *serverclose_wq; +struct workqueue_struct *cfid_put_wq; +__u32 cifs_lock_secret; + +/* + * Bumps refcount for cifs super block. + * Note that it should be only called if a reference to VFS super block is + * already held, e.g. in open-type syscalls context. Otherwise it can race with + * atomic_dec_and_test in deactivate_locked_super. + */ +void +cifs_sb_active(struct super_block *sb) +{ + struct cifs_sb_info *server = CIFS_SB(sb); + + if (atomic_inc_return(&server->active) == 1) + atomic_inc(&sb->s_active); +} + +void +cifs_sb_deactive(struct super_block *sb) +{ + struct cifs_sb_info *server = CIFS_SB(sb); + + if (atomic_dec_and_test(&server->active)) + deactivate_super(sb); +} + +static int +cifs_read_super(struct super_block *sb) +{ + struct cifs_sb_info *cifs_sb; + struct cifs_tcon *tcon; + unsigned int sbflags; + struct timespec64 ts; + struct inode *inode; + int rc = 0; + + cifs_sb = CIFS_SB(sb); + tcon = cifs_sb_master_tcon(cifs_sb); + sbflags = cifs_sb_flags(cifs_sb); + + if (sbflags & CIFS_MOUNT_POSIXACL) + sb->s_flags |= SB_POSIXACL; + + if (tcon->snapshot_time) + sb->s_flags |= SB_RDONLY; + + if (tcon->ses->capabilities & tcon->ses->server->vals->cap_large_files) + sb->s_maxbytes = MAX_LFS_FILESIZE; + else + sb->s_maxbytes = MAX_NON_LFS; + + /* + * Some very old servers like DOS and OS/2 used 2 second granularity + * (while all current servers use 100ns granularity - see MS-DTYP) + * but 1 second is the maximum allowed granularity for the VFS + * so for old servers set time granularity to 1 second while for + * everything else (current servers) set it to 100ns. + */ + if ((tcon->ses->server->vals->protocol_id == SMB10_PROT_ID) && + ((tcon->ses->capabilities & + tcon->ses->server->vals->cap_nt_find) == 0) && + !tcon->unix_ext) { + sb->s_time_gran = 1000000000; /* 1 second is max allowed gran */ + ts = cnvrtDosUnixTm(cpu_to_le16(SMB_DATE_MIN), 0, 0); + sb->s_time_min = ts.tv_sec; + ts = cnvrtDosUnixTm(cpu_to_le16(SMB_DATE_MAX), + cpu_to_le16(SMB_TIME_MAX), 0); + sb->s_time_max = ts.tv_sec; + } else { + /* + * Almost every server, including all SMB2+, uses DCE TIME + * ie 100 nanosecond units, since 1601. See MS-DTYP and MS-FSCC + */ + sb->s_time_gran = 100; + ts = cifs_NTtimeToUnix(0); + sb->s_time_min = ts.tv_sec; + ts = cifs_NTtimeToUnix(cpu_to_le64(S64_MAX)); + sb->s_time_max = ts.tv_sec; + } + + sb->s_magic = CIFS_SUPER_MAGIC; + sb->s_op = &cifs_super_ops; + sb->s_xattr = cifs_xattr_handlers; + rc = super_setup_bdi(sb); + if (rc) + goto out_no_root; + /* tune readahead according to rsize if readahead size not set on mount */ + if (cifs_sb->ctx->rsize == 0) + cifs_sb->ctx->rsize = + tcon->ses->server->ops->negotiate_rsize(tcon, cifs_sb->ctx); + if (cifs_sb->ctx->rasize) + sb->s_bdi->ra_pages = cifs_sb->ctx->rasize / PAGE_SIZE; + else + sb->s_bdi->ra_pages = 2 * (cifs_sb->ctx->rsize / PAGE_SIZE); + + sb->s_blocksize = CIFS_MAX_MSGSIZE; + sb->s_blocksize_bits = 14; /* default 2**14 = CIFS_MAX_MSGSIZE */ + inode = cifs_root_iget(sb); + + if (IS_ERR(inode)) { + rc = PTR_ERR(inode); + goto out_no_root; + } + + if (tcon->nocase) + set_default_d_op(sb, &cifs_ci_dentry_ops); + else + set_default_d_op(sb, &cifs_dentry_ops); + + sb->s_root = d_make_root(inode); + if (!sb->s_root) { + rc = -ENOMEM; + goto out_no_root; + } + +#ifdef CONFIG_CIFS_NFSD_EXPORT + if (sbflags & CIFS_MOUNT_SERVER_INUM) { + cifs_dbg(FYI, "export ops supported\n"); + sb->s_export_op = &cifs_export_ops; + } +#endif /* CONFIG_CIFS_NFSD_EXPORT */ + + return 0; + +out_no_root: + cifs_dbg(VFS, "%s: get root inode failed\n", __func__); + return rc; +} + +static void cifs_kill_sb(struct super_block *sb) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + + /* + * We need to release all dentries for the cached directories + * and close all deferred file handles before we kill the sb. + */ + if (cifs_sb->root) { + close_all_cached_dirs(cifs_sb); + cifs_close_all_deferred_files_sb(cifs_sb); + + /* Wait for all pending oplock breaks to complete */ + flush_workqueue(cifsoplockd_wq); + /* Wait for all opened files to release */ + flush_workqueue(deferredclose_wq); + + /* + * Wait for all in-flight netfs I/O requests to finish their + * cleanup_work so that any cifsFileInfo final puts they queue + * to fileinfo_put_wq/serverclose_wq have been queued, then + * drain the workqueue so the cfile dentry refs are dropped to + * avoid the busy dentry warning. + */ + wait_var_event(&cifs_sb->outstanding_rreq, + !atomic_read(&cifs_sb->outstanding_rreq)); + flush_workqueue(serverclose_wq); + flush_workqueue(fileinfo_put_wq); + + /* finally release root dentry */ + dput(cifs_sb->root); + cifs_sb->root = NULL; + } + + kill_anon_super(sb); + cifs_umount(cifs_sb); +} + +static int +cifs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb); + struct TCP_Server_Info *server = tcon->ses->server; + unsigned int xid; + int rc = 0; + const char *full_path; + void *page; + + xid = get_xid(); + page = alloc_dentry_path(); + + full_path = build_path_from_dentry(dentry, page); + if (IS_ERR(full_path)) { + rc = PTR_ERR(full_path); + goto statfs_out; + } + + if (le32_to_cpu(tcon->fsAttrInfo.MaxPathNameComponentLength) > 0) + buf->f_namelen = + le32_to_cpu(tcon->fsAttrInfo.MaxPathNameComponentLength); + else + buf->f_namelen = PATH_MAX; + + buf->f_fsid.val[0] = tcon->vol_serial_number; + /* are using part of create time for more randomness, see man statfs */ + buf->f_fsid.val[1] = (int)le64_to_cpu(tcon->vol_create_time); + + buf->f_files = 0; /* undefined */ + buf->f_ffree = 0; /* unlimited */ + + if (server->ops->queryfs) + rc = server->ops->queryfs(xid, tcon, full_path, cifs_sb, buf); + +statfs_out: + free_dentry_path(page); + free_xid(xid); + return rc; +} + +static long cifs_fallocate(struct file *file, int mode, loff_t off, loff_t len) +{ + struct cifs_tcon *tcon = cifs_sb_master_tcon(CIFS_SB(file)); + struct TCP_Server_Info *server = tcon->ses->server; + struct inode *inode = file_inode(file); + int rc; + + if (!server->ops->fallocate) + return -EOPNOTSUPP; + + rc = inode_lock_killable(inode); + if (rc) + return rc; + + netfs_wait_for_outstanding_io(inode); + + rc = file_modified(file); + if (rc) + goto out_unlock; + + rc = server->ops->fallocate(file, tcon, mode, off, len); + +out_unlock: + inode_unlock(inode); + return rc; +} + +static int cifs_permission(struct mnt_idmap *idmap, + struct inode *inode, int mask) +{ + unsigned int sbflags = cifs_sb_flags(CIFS_SB(inode)); + + if (sbflags & CIFS_MOUNT_NO_PERM) { + if ((mask & MAY_EXEC) && !execute_ok(inode)) + return -EACCES; + else + return 0; + } else /* file mode might have been restricted at mount time + on the client (above and beyond ACL on servers) for + servers which do not support setting and viewing mode bits, + so allowing client to check permissions is useful */ + return generic_permission(&nop_mnt_idmap, inode, mask); +} + +static struct kmem_cache *cifs_inode_cachep; +static struct kmem_cache *cifs_req_cachep; +static struct kmem_cache *cifs_mid_cachep; +static struct kmem_cache *cifs_sm_req_cachep; +static struct kmem_cache *cifs_io_request_cachep; +static struct kmem_cache *cifs_io_subrequest_cachep; +mempool_t *cifs_sm_req_poolp; +mempool_t *cifs_req_poolp; +mempool_t cifs_mid_pool; +mempool_t cifs_io_request_pool; +mempool_t cifs_io_subrequest_pool; + +static struct inode * +cifs_alloc_inode(struct super_block *sb) +{ + struct cifsInodeInfo *cifs_inode; + cifs_inode = alloc_inode_sb(sb, cifs_inode_cachep, GFP_KERNEL); + if (!cifs_inode) + return NULL; + cifs_inode->cifsAttrs = ATTR_ARCHIVE; /* default */ + cifs_inode->time = 0; + cifs_inode->time_last_write = 0; + /* + * Until the file is open and we have gotten oplock info back from the + * server, can not assume caching of file data or metadata. + */ + cifs_set_oplock_level(cifs_inode, 0); + cifs_inode->lease_granted = false; + cifs_inode->flags = 0; + spin_lock_init(&cifs_inode->writers_lock); + cifs_inode->writers = 0; + cifs_inode->netfs.inode.i_blkbits = 14; /* 2**14 = CIFS_MAX_MSGSIZE */ + cifs_inode->netfs._remote_i_size = 0; + cifs_inode->netfs._zero_point = 0; + cifs_inode->uniqueid = 0; + cifs_inode->createtime = 0; + cifs_inode->epoch = 0; + spin_lock_init(&cifs_inode->open_file_lock); + generate_random_uuid(cifs_inode->lease_key); + cifs_inode->symlink_target = NULL; + + /* + * Can not set i_flags here - they get immediately overwritten to zero + * by the VFS. + */ + /* cifs_inode->netfs.inode.i_flags = S_NOATIME | S_NOCMTIME; */ + INIT_LIST_HEAD(&cifs_inode->openFileList); + INIT_LIST_HEAD(&cifs_inode->llist); + INIT_LIST_HEAD(&cifs_inode->deferred_closes); + spin_lock_init(&cifs_inode->deferred_lock); + return &cifs_inode->netfs.inode; +} + +static void +cifs_free_inode(struct inode *inode) +{ + struct cifsInodeInfo *cinode = CIFS_I(inode); + + if (S_ISLNK(inode->i_mode)) + kfree(cinode->symlink_target); + kmem_cache_free(cifs_inode_cachep, cinode); +} + +static void +cifs_evict_inode(struct inode *inode) +{ + netfs_wait_for_outstanding_io(inode); + truncate_inode_pages_final(&inode->i_data); + if (inode_state_read_once(inode) & I_PINNING_NETFS_WB) + cifs_fscache_unuse_inode_cookie(inode, true); + cifs_fscache_release_inode_cookie(inode); + clear_inode(inode); +} + +static void +cifs_show_address(struct seq_file *s, struct TCP_Server_Info *server) +{ + struct sockaddr_in *sa = (struct sockaddr_in *) &server->dstaddr; + struct sockaddr_in6 *sa6 = (struct sockaddr_in6 *) &server->dstaddr; + + seq_puts(s, ",addr="); + + switch (server->dstaddr.ss_family) { + case AF_INET: + seq_printf(s, "%pI4", &sa->sin_addr.s_addr); + break; + case AF_INET6: + seq_printf(s, "%pI6", &sa6->sin6_addr.s6_addr); + if (sa6->sin6_scope_id) + seq_printf(s, "%%%u", sa6->sin6_scope_id); + break; + default: + seq_puts(s, "(unknown)"); + } + if (server->rdma) + seq_puts(s, ",rdma"); +} + +static void +cifs_show_security(struct seq_file *s, struct cifs_ses *ses) +{ + if (ses->sectype == Unspecified) { + if (ses->user_name == NULL) + seq_puts(s, ",sec=none"); + return; + } + + seq_puts(s, ",sec="); + + switch (ses->sectype) { + case NTLMv2: + seq_puts(s, "ntlmv2"); + break; + case Kerberos: + seq_puts(s, "krb5"); + break; + case RawNTLMSSP: + seq_puts(s, "ntlmssp"); + break; + default: + /* shouldn't ever happen */ + seq_puts(s, "unknown"); + break; + } + + if (ses->sign) + seq_puts(s, "i"); + + if (ses->sectype == Kerberos) + seq_printf(s, ",cruid=%u", + from_kuid_munged(&init_user_ns, ses->cred_uid)); +} + +static void +cifs_show_cache_flavor(struct seq_file *s, struct cifs_sb_info *cifs_sb) +{ + unsigned int sbflags = cifs_sb_flags(cifs_sb); + + seq_puts(s, ",cache="); + + if (sbflags & CIFS_MOUNT_STRICT_IO) + seq_puts(s, "strict"); + else if (sbflags & CIFS_MOUNT_DIRECT_IO) + seq_puts(s, "none"); + else if (sbflags & CIFS_MOUNT_RW_CACHE) + seq_puts(s, "singleclient"); /* assume only one client access */ + else if (sbflags & CIFS_MOUNT_RO_CACHE) + seq_puts(s, "ro"); /* read only caching assumed */ + else + seq_puts(s, "loose"); +} + +/* + * cifs_show_devname() is used so we show the mount device name with correct + * format (e.g. forward slashes vs. back slashes) in /proc/mounts + */ +static int cifs_show_devname(struct seq_file *m, struct dentry *root) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(root->d_sb); + char *devname = kstrdup(cifs_sb->ctx->source, GFP_KERNEL); + + if (devname == NULL) + seq_puts(m, "none"); + else { + convert_delimiter(devname, '/'); + /* escape all spaces in share names */ + seq_escape(m, devname, " \t"); + kfree(devname); + } + return 0; +} + +static void +cifs_show_upcall_target(struct seq_file *s, struct cifs_sb_info *cifs_sb) +{ + if (cifs_sb->ctx->upcall_target == UPTARGET_UNSPECIFIED) { + seq_puts(s, ",upcall_target=app"); + return; + } + + seq_puts(s, ",upcall_target="); + + switch (cifs_sb->ctx->upcall_target) { + case UPTARGET_APP: + seq_puts(s, "app"); + break; + case UPTARGET_MOUNT: + seq_puts(s, "mount"); + break; + default: + /* shouldn't ever happen */ + seq_puts(s, "unknown"); + break; + } +} + +/* + * cifs_show_options() is for displaying mount options in /proc/mounts. + * Not all settable options are displayed but most of the important + * ones are. + */ +static int +cifs_show_options(struct seq_file *s, struct dentry *root) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(root->d_sb); + struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb); + struct sockaddr *srcaddr; + unsigned int sbflags; + + srcaddr = (struct sockaddr *)&tcon->ses->server->srcaddr; + + seq_show_option(s, "vers", tcon->ses->server->vals->version_string); + cifs_show_security(s, tcon->ses); + cifs_show_cache_flavor(s, cifs_sb); + cifs_show_upcall_target(s, cifs_sb); + + if (tcon->no_lease) + seq_puts(s, ",nolease"); + if (cifs_sb->ctx->multiuser) + seq_puts(s, ",multiuser"); + else if (tcon->ses->user_name) + seq_show_option(s, "username", tcon->ses->user_name); + + if (tcon->ses->domainName && tcon->ses->domainName[0] != 0) + seq_show_option(s, "domain", tcon->ses->domainName); + + if (srcaddr->sa_family != AF_UNSPEC) { + struct sockaddr_in *saddr4; + struct sockaddr_in6 *saddr6; + saddr4 = (struct sockaddr_in *)srcaddr; + saddr6 = (struct sockaddr_in6 *)srcaddr; + if (srcaddr->sa_family == AF_INET6) + seq_printf(s, ",srcaddr=%pI6c", + &saddr6->sin6_addr); + else if (srcaddr->sa_family == AF_INET) + seq_printf(s, ",srcaddr=%pI4", + &saddr4->sin_addr.s_addr); + else + seq_printf(s, ",srcaddr=BAD-AF:%i", + (int)(srcaddr->sa_family)); + } + + sbflags = cifs_sb_flags(cifs_sb); + seq_printf(s, ",uid=%u", + from_kuid_munged(&init_user_ns, cifs_sb->ctx->linux_uid)); + if (sbflags & CIFS_MOUNT_OVERR_UID) + seq_puts(s, ",forceuid"); + else + seq_puts(s, ",noforceuid"); + + seq_printf(s, ",gid=%u", + from_kgid_munged(&init_user_ns, cifs_sb->ctx->linux_gid)); + if (sbflags & CIFS_MOUNT_OVERR_GID) + seq_puts(s, ",forcegid"); + else + seq_puts(s, ",noforcegid"); + + cifs_show_address(s, tcon->ses->server); + + if (!tcon->unix_ext) + seq_printf(s, ",file_mode=0%ho,dir_mode=0%ho", + cifs_sb->ctx->file_mode, + cifs_sb->ctx->dir_mode); + if (cifs_sb->ctx->iocharset) + seq_printf(s, ",iocharset=%s", cifs_sb->ctx->iocharset); + if (tcon->ses->unicode == 0) + seq_puts(s, ",nounicode"); + else if (tcon->ses->unicode == 1) + seq_puts(s, ",unicode"); + if (tcon->seal) + seq_puts(s, ",seal"); + else if (tcon->ses->server->ignore_signature) + seq_puts(s, ",signloosely"); + if (cifs_sb->ctx->compress) + seq_puts(s, ",compress"); + if (tcon->nocase) + seq_puts(s, ",nocase"); + if (tcon->nodelete) + seq_puts(s, ",nodelete"); + if (cifs_sb->ctx->no_sparse) + seq_puts(s, ",nosparse"); + if (tcon->local_lease) + seq_puts(s, ",locallease"); + if (tcon->retry) + seq_puts(s, ",hard"); + else + seq_puts(s, ",soft"); + if (tcon->use_persistent) + seq_puts(s, ",persistenthandles"); + else if (tcon->use_resilient) + seq_puts(s, ",resilienthandles"); + if (tcon->posix_extensions) + seq_puts(s, ",posix"); + else if (tcon->unix_ext) + seq_puts(s, ",unix"); + else + seq_puts(s, ",nounix"); + if (sbflags & CIFS_MOUNT_NO_DFS) + seq_puts(s, ",nodfs"); + if (sbflags & CIFS_MOUNT_POSIX_PATHS) + seq_puts(s, ",posixpaths"); + if (sbflags & CIFS_MOUNT_SET_UID) + seq_puts(s, ",setuids"); + if (sbflags & CIFS_MOUNT_UID_FROM_ACL) + seq_puts(s, ",idsfromsid"); + if (sbflags & CIFS_MOUNT_SERVER_INUM) + seq_puts(s, ",serverino"); + if (sbflags & CIFS_MOUNT_RWPIDFORWARD) + seq_puts(s, ",rwpidforward"); + if (sbflags & CIFS_MOUNT_NOPOSIXBRL) + seq_puts(s, ",forcemand"); + if (sbflags & CIFS_MOUNT_NO_XATTR) + seq_puts(s, ",nouser_xattr"); + if (sbflags & CIFS_MOUNT_MAP_SPECIAL_CHR) + seq_puts(s, ",mapchars"); + if (sbflags & CIFS_MOUNT_MAP_SFM_CHR) + seq_puts(s, ",mapposix"); + if (sbflags & CIFS_MOUNT_UNX_EMUL) + seq_puts(s, ",sfu"); + if (sbflags & CIFS_MOUNT_NO_BRL) + seq_puts(s, ",nobrl"); + if (sbflags & CIFS_MOUNT_NO_HANDLE_CACHE) + seq_puts(s, ",nohandlecache"); + if (sbflags & CIFS_MOUNT_MODE_FROM_SID) + seq_puts(s, ",modefromsid"); + if (sbflags & CIFS_MOUNT_CIFS_ACL) + seq_puts(s, ",cifsacl"); + if (sbflags & CIFS_MOUNT_DYNPERM) + seq_puts(s, ",dynperm"); + if (root->d_sb->s_flags & SB_POSIXACL) + seq_puts(s, ",acl"); + if (sbflags & CIFS_MOUNT_MF_SYMLINKS) + seq_puts(s, ",mfsymlinks"); + if (sbflags & CIFS_MOUNT_FSCACHE) + seq_puts(s, ",fsc"); + if (sbflags & CIFS_MOUNT_NOSSYNC) + seq_puts(s, ",nostrictsync"); + if (sbflags & CIFS_MOUNT_NO_PERM) + seq_puts(s, ",noperm"); + if (sbflags & CIFS_MOUNT_CIFS_BACKUPUID) + seq_printf(s, ",backupuid=%u", + from_kuid_munged(&init_user_ns, + cifs_sb->ctx->backupuid)); + if (sbflags & CIFS_MOUNT_CIFS_BACKUPGID) + seq_printf(s, ",backupgid=%u", + from_kgid_munged(&init_user_ns, + cifs_sb->ctx->backupgid)); + seq_show_option(s, "reparse", + cifs_reparse_type_str(cifs_sb->ctx->reparse_type)); + if (cifs_sb->ctx->nonativesocket) + seq_puts(s, ",nonativesocket"); + else + seq_puts(s, ",nativesocket"); + seq_show_option(s, "symlink", + cifs_symlink_type_str(cifs_symlink_type(cifs_sb))); + + seq_printf(s, ",rsize=%u", cifs_sb->ctx->rsize); + seq_printf(s, ",wsize=%u", cifs_sb->ctx->wsize); + seq_printf(s, ",bsize=%u", cifs_sb->ctx->bsize); + if (cifs_sb->ctx->rasize) + seq_printf(s, ",rasize=%u", cifs_sb->ctx->rasize); + if (tcon->ses->server->min_offload) + seq_printf(s, ",esize=%u", tcon->ses->server->min_offload); + if (tcon->ses->server->retrans) + seq_printf(s, ",retrans=%u", tcon->ses->server->retrans); + seq_printf(s, ",echo_interval=%lu", + tcon->ses->server->echo_interval / HZ); + + /* Only display the following if overridden on mount */ + if (tcon->ses->server->max_credits != SMB2_MAX_CREDITS_AVAILABLE) + seq_printf(s, ",max_credits=%u", tcon->ses->server->max_credits); + if (tcon->ses->server->tcp_nodelay) + seq_puts(s, ",tcpnodelay"); + if (tcon->ses->server->noautotune) + seq_puts(s, ",noautotune"); + if (tcon->ses->server->noblocksnd) + seq_puts(s, ",noblocksend"); + if (tcon->ses->server->nosharesock) + seq_puts(s, ",nosharesock"); + + if (tcon->snapshot_time) + seq_printf(s, ",snapshot=%llu", tcon->snapshot_time); + if (tcon->handle_timeout) + seq_printf(s, ",handletimeout=%u", tcon->handle_timeout); + if (tcon->max_cached_dirs != MAX_CACHED_FIDS) + seq_printf(s, ",max_cached_dirs=%u", tcon->max_cached_dirs); + + /* + * Display file and directory attribute timeout in seconds. + * If file and directory attribute timeout the same then actimeo + * was likely specified on mount + */ + if (cifs_sb->ctx->acdirmax == cifs_sb->ctx->acregmax) + seq_printf(s, ",actimeo=%lu", cifs_sb->ctx->acregmax / HZ); + else { + seq_printf(s, ",acdirmax=%lu", cifs_sb->ctx->acdirmax / HZ); + seq_printf(s, ",acregmax=%lu", cifs_sb->ctx->acregmax / HZ); + } + seq_printf(s, ",closetimeo=%lu", cifs_sb->ctx->closetimeo / HZ); + + if (tcon->ses->chan_max > 1) + seq_printf(s, ",multichannel,max_channels=%zu", + tcon->ses->chan_max); + + if (tcon->use_witness) + seq_puts(s, ",witness"); + + return 0; +} + +static void cifs_umount_begin(struct super_block *sb) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + struct cifs_tcon *tcon; + + if (cifs_sb == NULL) + return; + + tcon = cifs_sb_master_tcon(cifs_sb); + + spin_lock(&cifs_tcp_ses_lock); + spin_lock(&tcon->tc_lock); + trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, + netfs_trace_tcon_ref_see_umount); + if ((tcon->tc_count > 1) || (tcon->status == TID_EXITING)) { + /* we have other mounts to same share or we have + already tried to umount this and woken up + all waiting network requests, nothing to do */ + spin_unlock(&tcon->tc_lock); + spin_unlock(&cifs_tcp_ses_lock); + return; + } + /* + * can not set tcon->status to TID_EXITING yet since we don't know if umount -f will + * fail later (e.g. due to open files). TID_EXITING will be set just before tdis req sent + */ + spin_unlock(&tcon->tc_lock); + spin_unlock(&cifs_tcp_ses_lock); + + /* cancel_brl_requests(tcon); */ /* BB mark all brl mids as exiting */ + /* cancel_notify_requests(tcon); */ + if (tcon->ses && tcon->ses->server) { + cifs_dbg(FYI, "wake up tasks now - umount begin not complete\n"); + wake_up_all(&tcon->ses->server->request_q); + wake_up_all(&tcon->ses->server->response_q); + msleep(1); /* yield */ + /* we have to kick the requests once more */ + wake_up_all(&tcon->ses->server->response_q); + msleep(1); + } + + return; +} + +static int cifs_freeze(struct super_block *sb) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + struct cifs_tcon *tcon; + + if (cifs_sb == NULL) + return 0; + + tcon = cifs_sb_master_tcon(cifs_sb); + + cifs_close_all_deferred_files(tcon); + return 0; +} + +#ifdef CONFIG_CIFS_STATS2 +static int cifs_show_stats(struct seq_file *s, struct dentry *root) +{ + /* BB FIXME */ + return 0; +} +#endif + +static int cifs_write_inode(struct inode *inode, struct writeback_control *wbc) +{ + return netfs_unpin_writeback(inode, wbc); +} + +static int cifs_drop_inode(struct inode *inode) +{ + unsigned int sbflags = cifs_sb_flags(CIFS_SB(inode)); + + /* no serverino => unconditional eviction */ + return !(sbflags & CIFS_MOUNT_SERVER_INUM) || + inode_generic_drop(inode); +} + +static const struct super_operations cifs_super_ops = { + .statfs = cifs_statfs, + .alloc_inode = cifs_alloc_inode, + .write_inode = cifs_write_inode, + .free_inode = cifs_free_inode, + .drop_inode = cifs_drop_inode, + .evict_inode = cifs_evict_inode, +/* .show_path = cifs_show_path, */ /* Would we ever need show path? */ + .show_devname = cifs_show_devname, +/* .delete_inode = cifs_delete_inode, */ /* Do not need above + function unless later we add lazy close of inodes or unless the + kernel forgets to call us with the same number of releases (closes) + as opens */ + .show_options = cifs_show_options, + .umount_begin = cifs_umount_begin, + .freeze_fs = cifs_freeze, +#ifdef CONFIG_CIFS_STATS2 + .show_stats = cifs_show_stats, +#endif +}; + +/* + * Get root dentry from superblock according to prefix path mount option. + * Return dentry with refcount + 1 on success and NULL otherwise. + */ +static struct dentry * +cifs_get_root(struct smb3_fs_context *ctx, struct super_block *sb) +{ + struct dentry *dentry; + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + char *full_path = NULL; + char *s, *p; + char sep; + + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_USE_PREFIX_PATH) + return dget(sb->s_root); + + full_path = cifs_build_path_to_root(ctx, cifs_sb, + cifs_sb_master_tcon(cifs_sb), 0); + if (full_path == NULL) + return ERR_PTR(-ENOMEM); + + cifs_dbg(FYI, "Get root dentry for %s\n", full_path); + + sep = CIFS_DIR_SEP(cifs_sb); + dentry = dget(sb->s_root); + s = full_path; + + do { + struct inode *dir = d_inode(dentry); + struct dentry *child; + + if (!S_ISDIR(dir->i_mode)) { + dput(dentry); + dentry = ERR_PTR(-ENOTDIR); + break; + } + + /* skip separators */ + while (*s == sep) + s++; + if (!*s) + break; + p = s++; + /* next separator */ + while (*s && *s != sep) + s++; + + child = lookup_noperm_positive_unlocked(&QSTR_LEN(p, s - p), + dentry); + dput(dentry); + dentry = child; + } while (!IS_ERR(dentry)); + kfree(full_path); + return dentry; +} + +struct dentry * +cifs_smb3_do_mount(struct fs_context *fc, struct smb3_fs_context *old_ctx) +{ + struct cifs_mnt_data mnt_data; + struct cifs_sb_info *cifs_sb; + struct super_block *sb; + struct dentry *root; + unsigned int saved_sb_flags; + int rc; + + if (cifsFYI) { + cifs_dbg(FYI, "%s: devname=%s sb_flags=0x%x\n", __func__, + old_ctx->source, fc->sb_flags); + } else { + cifs_info("Attempting to mount %s\n", old_ctx->source); + } + cifs_sb = kzalloc_obj(*cifs_sb); + if (!cifs_sb) + return ERR_PTR(-ENOMEM); + + cifs_sb->ctx = kzalloc_obj(struct smb3_fs_context); + if (!cifs_sb->ctx) { + root = ERR_PTR(-ENOMEM); + goto out; + } + rc = smb3_fs_context_dup(cifs_sb->ctx, old_ctx); + if (rc) { + root = ERR_PTR(rc); + goto out; + } + + rc = cifs_setup_cifs_sb(cifs_sb); + if (rc) { + root = ERR_PTR(rc); + goto out; + } + + rc = cifs_mount(cifs_sb, cifs_sb->ctx); + if (rc) { + if (!(fc->sb_flags & SB_SILENT)) + cifs_dbg(VFS, "cifs_mount failed w/return code = %d\n", + rc); + root = ERR_PTR(rc); + goto out; + } + + mnt_data.ctx = cifs_sb->ctx; + mnt_data.cifs_sb = cifs_sb; + mnt_data.flags = 0; + + /* + * sb->s_flags is set from fc->sb_flags by alloc_super(). CIFS has + * historically forced SB_NODIRATIME | SB_NOATIME on every mount and + * ignored the caller-supplied SB_* flags. Preserve that behaviour by + * overriding fc->sb_flags around the sget_fc() call. + * + * Hand cifs_sb to sget_fc() via fc->s_fs_info; sget_fc() copies it + * onto sb->s_fs_info before running set() and clears fc->s_fs_info + * on successful publish. Pass the rest of the per-mount context to + * cifs_match_super() through fc->sget_key. + */ + saved_sb_flags = fc->sb_flags; + fc->sb_flags = SB_NODIRATIME | SB_NOATIME; + fc->s_fs_info = cifs_sb; + fc->sget_key = &mnt_data; + sb = sget_fc(fc, cifs_match_super, set_anon_super_fc); + fc->sget_key = NULL; + fc->s_fs_info = NULL; + fc->sb_flags = saved_sb_flags; + if (IS_ERR(sb)) { + cifs_umount(cifs_sb); + return ERR_CAST(sb); + } + + if (sb->s_root) { + cifs_dbg(FYI, "Use existing superblock\n"); + cifs_umount(cifs_sb); + cifs_sb = NULL; + } else { + rc = cifs_read_super(sb); + if (rc) { + root = ERR_PTR(rc); + goto out_super; + } + + sb->s_flags |= SB_ACTIVE; + } + + root = cifs_get_root(cifs_sb ? cifs_sb->ctx : old_ctx, sb); + if (IS_ERR(root)) + goto out_super; + + if (cifs_sb) + cifs_sb->root = dget(root); + + cifs_dbg(FYI, "dentry root is: %p\n", root); + return root; + +out_super: + deactivate_locked_super(sb); + return root; +out: + kfree(cifs_sb->prepath); + smb3_cleanup_fs_context(cifs_sb->ctx); + kfree(cifs_sb); + return root; +} + +static loff_t cifs_llseek(struct file *file, loff_t offset, int whence) +{ + struct cifsFileInfo *cfile = file->private_data; + struct cifs_tcon *tcon; + + /* + * whence == SEEK_END || SEEK_DATA || SEEK_HOLE => we must revalidate + * the cached file length + */ + if (whence != SEEK_SET && whence != SEEK_CUR) { + int rc; + struct inode *inode = file_inode(file); + + /* + * We need to be sure that all dirty pages are written and the + * server has the newest file length. + */ + if (!CIFS_CACHE_READ(CIFS_I(inode)) && inode->i_mapping && + inode->i_mapping->nrpages != 0) { + rc = filemap_fdatawait(inode->i_mapping); + if (rc) { + mapping_set_error(inode->i_mapping, rc); + return rc; + } + } + /* + * Some applications poll for the file length in this strange + * way so we must seek to end on non-oplocked files by + * setting the revalidate time to zero. + */ + CIFS_I(inode)->time = 0; + + rc = cifs_revalidate_file_attr(file); + if (rc < 0) + return (loff_t)rc; + } + if (cfile && cfile->tlink) { + tcon = tlink_tcon(cfile->tlink); + if (tcon->ses->server->ops->llseek) + return tcon->ses->server->ops->llseek(file, tcon, + offset, whence); + } + return generic_file_llseek(file, offset, whence); +} + +static int +cifs_setlease(struct file *file, int arg, struct file_lease **lease, void **priv) +{ + /* + * Note that this is called by vfs setlease with i_lock held to + * protect *lease from going away. + */ + struct inode *inode = file_inode(file); + struct cifsFileInfo *cfile = file->private_data; + + /* Check if file is oplocked if this is request for new lease */ + if (arg == F_UNLCK || + ((arg == F_RDLCK) && CIFS_CACHE_READ(CIFS_I(inode))) || + ((arg == F_WRLCK) && CIFS_CACHE_WRITE(CIFS_I(inode)))) + return generic_setlease(file, arg, lease, priv); + else if (tlink_tcon(cfile->tlink)->local_lease && + !CIFS_CACHE_READ(CIFS_I(inode))) + /* + * If the server claims to support oplock on this file, then we + * still need to check oplock even if the local_lease mount + * option is set, but there are servers which do not support + * oplock for which this mount option may be useful if the user + * knows that the file won't be changed on the server by anyone + * else. + */ + return generic_setlease(file, arg, lease, priv); + else + return -EAGAIN; +} + +struct file_system_type cifs_fs_type = { + .owner = THIS_MODULE, + .name = "cifs", + .init_fs_context = smb3_init_fs_context, + .parameters = smb3_fs_parameters, + .kill_sb = cifs_kill_sb, + .fs_flags = FS_RENAME_DOES_D_MOVE, +}; +MODULE_ALIAS_FS("cifs"); + +struct file_system_type smb3_fs_type = { + .owner = THIS_MODULE, + .name = "smb3", + .init_fs_context = smb3_init_fs_context, + .parameters = smb3_fs_parameters, + .kill_sb = cifs_kill_sb, + .fs_flags = FS_RENAME_DOES_D_MOVE, +}; +MODULE_ALIAS_FS("smb3"); +MODULE_ALIAS("smb3"); + +int cifs_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(dentry->d_sb); + struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb); + struct inode *inode = d_inode(dentry); + u32 attrs; + + /* Preserve FS_COMPR_FL previously reported by cifs_ioctl(). */ + if (CIFS_I(inode)->cifsAttrs & ATTR_COMPRESSED) + fa->flags |= FS_COMPR_FL; + + /* + * FS_CASEFOLD_FL is defined by UAPI as a folder attribute, + * and userspace tools (e.g., lsattr) display it only on + * directories. Confine the case-handling bits to directories + * to match that convention; for non-directories the share's + * case semantics are still discoverable through the parent. + */ + if (!S_ISDIR(inode->i_mode)) + return 0; + + /* + * The server's FS_ATTRIBUTE_INFORMATION response, cached on + * the tcon at mount, reflects the share's case-handling + * semantics after any POSIX extensions negotiation. Prefer + * it over the client-local nocase mount option, which only + * governs dentry comparison on this superblock. + * + * QueryFSInfo is best-effort at mount; when it did not + * populate fsAttrInfo, MaxPathNameComponentLength remains + * zero. In that case fall back to nocase so the reporting + * matches the comparison behavior installed on the sb. + */ + if (le32_to_cpu(tcon->fsAttrInfo.MaxPathNameComponentLength) == 0) { + if (tcon->nocase) { + fa->fsx_xflags |= FS_XFLAG_CASEFOLD; + fa->flags |= FS_CASEFOLD_FL; + } + return 0; + } + attrs = le32_to_cpu(tcon->fsAttrInfo.Attributes); + if (!(attrs & FILE_CASE_SENSITIVE_SEARCH)) { + fa->fsx_xflags |= FS_XFLAG_CASEFOLD; + fa->flags |= FS_CASEFOLD_FL; + } + if (!(attrs & FILE_CASE_PRESERVED_NAMES)) + fa->fsx_xflags |= FS_XFLAG_CASENONPRESERVING; + return 0; +} + +const struct inode_operations cifs_dir_inode_ops = { + .create = cifs_create, + .atomic_open = cifs_atomic_open, + .tmpfile = cifs_tmpfile, + .lookup = cifs_lookup, + .getattr = cifs_getattr, + .unlink = cifs_unlink, + .link = cifs_hardlink, + .mkdir = cifs_mkdir, + .rmdir = cifs_rmdir, + .rename = cifs_rename2, + .permission = cifs_permission, + .setattr = cifs_setattr, + .symlink = cifs_symlink, + .mknod = cifs_mknod, + .listxattr = cifs_listxattr, + .get_acl = cifs_get_acl, + .set_acl = cifs_set_acl, + .fileattr_get = cifs_fileattr_get, +}; + +const struct inode_operations cifs_file_inode_ops = { + .setattr = cifs_setattr, + .getattr = cifs_getattr, + .permission = cifs_permission, + .listxattr = cifs_listxattr, + .fiemap = cifs_fiemap, + .get_acl = cifs_get_acl, + .set_acl = cifs_set_acl, + .fileattr_get = cifs_fileattr_get, +}; + +const char *cifs_get_link(struct dentry *dentry, struct inode *inode, + struct delayed_call *done) +{ + char *target_path; + + if (!dentry) + return ERR_PTR(-ECHILD); + + target_path = kmalloc(PATH_MAX, GFP_KERNEL); + if (!target_path) + return ERR_PTR(-ENOMEM); + + spin_lock(&inode->i_lock); + if (likely(CIFS_I(inode)->symlink_target)) { + strscpy(target_path, CIFS_I(inode)->symlink_target, PATH_MAX); + } else { + kfree(target_path); + target_path = ERR_PTR(-EOPNOTSUPP); + } + spin_unlock(&inode->i_lock); + + if (!IS_ERR(target_path)) + set_delayed_call(done, kfree_link, target_path); + + return target_path; +} + +const struct inode_operations cifs_symlink_inode_ops = { + .get_link = cifs_get_link, + .setattr = cifs_setattr, + .permission = cifs_permission, + .listxattr = cifs_listxattr, +}; + +/* + * Advance the EOF marker to after the source range. + */ +static int cifs_precopy_set_eof(struct inode *src_inode, struct cifsInodeInfo *src_cifsi, + struct cifs_tcon *src_tcon, + unsigned int xid, loff_t src_end) +{ + struct cifsFileInfo *writeable_srcfile; + int rc = -EINVAL; + + writeable_srcfile = find_writable_file(src_cifsi, FIND_FSUID_ONLY); + if (writeable_srcfile) { + if (src_tcon->ses->server->ops->set_file_size) + rc = src_tcon->ses->server->ops->set_file_size( + xid, src_tcon, writeable_srcfile, + src_inode->i_size, true /* no need to set sparse */); + else + rc = -ENOSYS; + cifsFileInfo_put(writeable_srcfile); + cifs_dbg(FYI, "SetFSize for copychunk rc = %d\n", rc); + } + + if (rc < 0) + goto set_failed; + + netfs_resize_file(&src_cifsi->netfs, src_end, true); + fscache_resize_cookie(cifs_inode_cookie(src_inode), src_end); + return 0; + +set_failed: + return filemap_write_and_wait(src_inode->i_mapping); +} + +/* + * Flush out either the folio that overlaps the beginning of a range in which + * pos resides or the folio that overlaps the end of a range unless that folio + * is entirely within the range we're going to invalidate. We extend the flush + * bounds to encompass the folio. + */ +static int cifs_flush_folio(struct inode *inode, loff_t pos, loff_t *_fstart, loff_t *_fend, + bool first) +{ + struct folio *folio; + unsigned long long fpos, fend; + pgoff_t index = pos / PAGE_SIZE; + size_t size; + int rc = 0; + + folio = filemap_get_folio(inode->i_mapping, index); + if (IS_ERR(folio)) + return 0; + + size = folio_size(folio); + fpos = folio_pos(folio); + fend = fpos + size - 1; + *_fstart = min_t(unsigned long long, *_fstart, fpos); + *_fend = max_t(unsigned long long, *_fend, fend); + if ((first && pos == fpos) || (!first && pos == fend)) + goto out; + + rc = filemap_write_and_wait_range(inode->i_mapping, fpos, fend); +out: + folio_put(folio); + return rc; +} + +static loff_t cifs_remap_file_range(struct file *src_file, loff_t off, + struct file *dst_file, loff_t destoff, loff_t len, + unsigned int remap_flags) +{ + struct inode *src_inode = file_inode(src_file); + struct inode *target_inode = file_inode(dst_file); + struct cifsInodeInfo *src_cifsi = CIFS_I(src_inode); + struct cifsInodeInfo *target_cifsi = CIFS_I(target_inode); + struct cifsFileInfo *smb_file_src = src_file->private_data; + struct cifsFileInfo *smb_file_target = dst_file->private_data; + struct cifs_tcon *target_tcon, *src_tcon; + unsigned long long i_size, new_size; + unsigned long long destend, fstart, fend; + unsigned int xid; + int rc; + + if (remap_flags & REMAP_FILE_DEDUP) + return -EOPNOTSUPP; + if (remap_flags & ~REMAP_FILE_ADVISORY) + return -EINVAL; + + cifs_dbg(FYI, "clone range\n"); + + xid = get_xid(); + + if (!smb_file_src || !smb_file_target) { + rc = -EBADF; + cifs_dbg(VFS, "missing cifsFileInfo on copy range src file\n"); + goto out; + } + + src_tcon = tlink_tcon(smb_file_src->tlink); + target_tcon = tlink_tcon(smb_file_target->tlink); + + /* + * Note: cifs case is easier than btrfs since server responsible for + * checks for proper open modes and file type and if it wants + * server could even support copy of range where source = target + */ + lock_two_nondirectories(target_inode, src_inode); + + if (len == 0) + len = src_inode->i_size - off; + + cifs_dbg(FYI, "clone range\n"); + + /* Flush the source buffer */ + rc = filemap_write_and_wait_range(src_inode->i_mapping, off, + off + len - 1); + if (rc) + goto unlock; + + /* The server-side copy will fail if the source crosses the EOF marker. + * Advance the EOF marker after the flush above to the end of the range + * if it's short of that. + */ + if (netfs_read_remote_i_size(src_inode) < off + len) { + rc = cifs_precopy_set_eof(src_inode, src_cifsi, src_tcon, xid, off + len); + if (rc < 0) + goto unlock; + } + + new_size = destoff + len; + destend = destoff + len - 1; + + /* Flush the folios at either end of the destination range to prevent + * accidental loss of dirty data outside of the range. + */ + fstart = destoff; + fend = destend; + + rc = cifs_flush_folio(target_inode, destoff, &fstart, &fend, true); + if (rc) + goto unlock; + rc = cifs_flush_folio(target_inode, destend, &fstart, &fend, false); + if (rc) + goto unlock; + + spin_lock(&target_inode->i_lock); + if (fend > target_cifsi->netfs._zero_point) + netfs_write_zero_point(target_inode, fend + 1); + i_size = target_inode->i_size; + spin_unlock(&target_inode->i_lock); + + /* Discard all the folios that overlap the destination region. */ + cifs_dbg(FYI, "about to discard pages %llx-%llx\n", fstart, fend); + truncate_inode_pages_range(&target_inode->i_data, fstart, fend); + + fscache_invalidate(cifs_inode_cookie(target_inode), NULL, i_size, 0); + + rc = -EOPNOTSUPP; + if (target_tcon->ses->server->ops->duplicate_extents) { + rc = target_tcon->ses->server->ops->duplicate_extents(xid, + smb_file_src, smb_file_target, off, len, destoff); + if (rc == 0 && new_size > i_size) { + truncate_setsize(target_inode, new_size); + fscache_resize_cookie(cifs_inode_cookie(target_inode), + new_size); + } else if (rc == -EOPNOTSUPP) { + /* + * copy_file_range syscall man page indicates EINVAL + * is returned e.g when "fd_in and fd_out refer to the + * same file and the source and target ranges overlap." + * Test generic/157 was what showed these cases where + * we need to remap EOPNOTSUPP to EINVAL + */ + if (off >= src_inode->i_size) { + rc = -EINVAL; + } else if (src_inode == target_inode) { + if (off + len > destoff) + rc = -EINVAL; + } + } + if (rc == 0) { + spin_lock(&target_inode->i_lock); + if (new_size > target_cifsi->netfs._zero_point) + netfs_write_zero_point(target_inode, new_size); + spin_unlock(&target_inode->i_lock); + } + } + + /* + * On success, duplicate_extents already updated the target inode attrs + * or marked them stale if the refresh failed. On failure, mark attrs + * stale because EOF may have changed before the clone failed. + */ + if (rc) + CIFS_I(target_inode)->time = 0; +unlock: + /* although unlocking in the reverse order from locking is not + strictly necessary here it is a little cleaner to be consistent */ + unlock_two_nondirectories(src_inode, target_inode); +out: + free_xid(xid); + return rc < 0 ? rc : len; +} + +ssize_t cifs_file_copychunk_range(unsigned int xid, + struct file *src_file, loff_t off, + struct file *dst_file, loff_t destoff, + size_t len, unsigned int flags) +{ + struct inode *src_inode = file_inode(src_file); + struct inode *target_inode = file_inode(dst_file); + struct cifsInodeInfo *src_cifsi = CIFS_I(src_inode); + struct cifsInodeInfo *target_cifsi = CIFS_I(target_inode); + struct cifsFileInfo *smb_file_src; + struct cifsFileInfo *smb_file_target; + struct cifs_tcon *src_tcon; + struct cifs_tcon *target_tcon; + ssize_t rc; + + cifs_dbg(FYI, "copychunk range\n"); + + if (!src_file->private_data || !dst_file->private_data) { + rc = -EBADF; + cifs_dbg(VFS, "missing cifsFileInfo on copy range src file\n"); + goto out; + } + + rc = -EXDEV; + smb_file_target = dst_file->private_data; + smb_file_src = src_file->private_data; + src_tcon = tlink_tcon(smb_file_src->tlink); + target_tcon = tlink_tcon(smb_file_target->tlink); + + if (src_tcon->ses != target_tcon->ses) { + cifs_dbg(FYI, "source and target of copy not on same server\n"); + goto out; + } + + rc = -EOPNOTSUPP; + if (!target_tcon->ses->server->ops->copychunk_range) + goto out; + + /* + * Note: cifs case is easier than btrfs since server responsible for + * checks for proper open modes and file type and if it wants + * server could even support copy of range where source = target + */ + lock_two_nondirectories(target_inode, src_inode); + + cifs_dbg(FYI, "about to flush pages\n"); + + rc = filemap_write_and_wait_range(src_inode->i_mapping, off, + off + len - 1); + if (rc) + goto unlock; + + /* The server-side copy will fail if the source crosses the EOF marker. + * Advance the EOF marker after the flush above to the end of the range + * if it's short of that. + */ + if (netfs_read_remote_i_size(src_inode) < off + len) { + rc = cifs_precopy_set_eof(src_inode, src_cifsi, src_tcon, xid, off + len); + if (rc < 0) + goto unlock; + } + + /* Flush and invalidate all the folios in the destination region. If + * the copy was successful, then some of the flush is extra overhead, + * but we need to allow for the copy failing in some way (eg. ENOSPC). + */ + rc = filemap_invalidate_inode(target_inode, true, destoff, destoff + len - 1); + if (rc) + goto unlock; + + fscache_invalidate(cifs_inode_cookie(target_inode), NULL, + i_size_read(target_inode), 0); + + rc = file_modified(dst_file); + if (!rc) { + rc = target_tcon->ses->server->ops->copychunk_range(xid, + smb_file_src, smb_file_target, off, len, destoff); + if (rc > 0 && destoff + rc > i_size_read(target_inode)) { + truncate_setsize(target_inode, destoff + rc); + netfs_resize_file(&target_cifsi->netfs, + i_size_read(target_inode), true); + fscache_resize_cookie(cifs_inode_cookie(target_inode), + i_size_read(target_inode)); + } + if (rc > 0) { + spin_lock(&target_inode->i_lock); + if (destoff + rc > target_cifsi->netfs._zero_point) + netfs_write_zero_point(target_inode, destoff + rc); + spin_unlock(&target_inode->i_lock); + } + } + + file_accessed(src_file); + + /* force revalidate of size and timestamps of target file now + * that target is updated on the server + */ + CIFS_I(target_inode)->time = 0; + +unlock: + /* although unlocking in the reverse order from locking is not + * strictly necessary here it is a little cleaner to be consistent + */ + unlock_two_nondirectories(src_inode, target_inode); + +out: + return rc; +} + +/* + * Directory operations under CIFS/SMB2/SMB3 are synchronous, so fsync() + * is a dummy operation. + */ +static int cifs_dir_fsync(struct file *file, loff_t start, loff_t end, int datasync) +{ + cifs_dbg(FYI, "Sync directory - name: %pD datasync: 0x%x\n", + file, datasync); + + return 0; +} + +static ssize_t cifs_copy_file_range(struct file *src_file, loff_t off, + struct file *dst_file, loff_t destoff, + size_t len, unsigned int flags) +{ + unsigned int xid = get_xid(); + ssize_t rc; + struct cifsFileInfo *cfile = dst_file->private_data; + + if (cfile->swapfile) { + rc = -EOPNOTSUPP; + free_xid(xid); + return rc; + } + + rc = cifs_file_copychunk_range(xid, src_file, off, dst_file, destoff, + len, flags); + free_xid(xid); + + if (rc == -EOPNOTSUPP || rc == -EXDEV) + rc = splice_copy_file_range(src_file, off, dst_file, + destoff, len); + return rc; +} + +const struct file_operations cifs_file_ops = { + .read_iter = cifs_loose_read_iter, + .write_iter = cifs_file_write_iter, + .open = cifs_open, + .release = cifs_close, + .lock = cifs_lock, + .flock = cifs_flock, + .fsync = cifs_fsync, + .flush = cifs_flush, + .mmap_prepare = cifs_file_mmap_prepare, + .splice_read = filemap_splice_read, + .splice_write = iter_file_splice_write, + .llseek = cifs_llseek, + .unlocked_ioctl = cifs_ioctl, + .copy_file_range = cifs_copy_file_range, + .remap_file_range = cifs_remap_file_range, + .setlease = cifs_setlease, + .fallocate = cifs_fallocate, +}; + +const struct file_operations cifs_file_strict_ops = { + .read_iter = cifs_strict_readv, + .write_iter = cifs_strict_writev, + .open = cifs_open, + .release = cifs_close, + .lock = cifs_lock, + .flock = cifs_flock, + .fsync = cifs_strict_fsync, + .flush = cifs_flush, + .mmap_prepare = cifs_file_strict_mmap_prepare, + .splice_read = filemap_splice_read, + .splice_write = iter_file_splice_write, + .llseek = cifs_llseek, + .unlocked_ioctl = cifs_ioctl, + .copy_file_range = cifs_copy_file_range, + .remap_file_range = cifs_remap_file_range, + .setlease = cifs_setlease, + .fallocate = cifs_fallocate, +}; + +const struct file_operations cifs_file_direct_ops = { + .read_iter = netfs_unbuffered_read_iter, + .write_iter = cifs_direct_write_iter, + .open = cifs_open, + .release = cifs_close, + .lock = cifs_lock, + .flock = cifs_flock, + .fsync = cifs_fsync, + .flush = cifs_flush, + .mmap_prepare = cifs_file_mmap_prepare, + .splice_read = copy_splice_read, + .splice_write = iter_file_splice_write, + .unlocked_ioctl = cifs_ioctl, + .copy_file_range = cifs_copy_file_range, + .remap_file_range = cifs_remap_file_range, + .llseek = cifs_llseek, + .setlease = cifs_setlease, + .fallocate = cifs_fallocate, +}; + +const struct file_operations cifs_file_nobrl_ops = { + .read_iter = cifs_loose_read_iter, + .write_iter = cifs_file_write_iter, + .open = cifs_open, + .release = cifs_close, + .fsync = cifs_fsync, + .flush = cifs_flush, + .mmap_prepare = cifs_file_mmap_prepare, + .splice_read = filemap_splice_read, + .splice_write = iter_file_splice_write, + .llseek = cifs_llseek, + .unlocked_ioctl = cifs_ioctl, + .copy_file_range = cifs_copy_file_range, + .remap_file_range = cifs_remap_file_range, + .setlease = cifs_setlease, + .fallocate = cifs_fallocate, +}; + +const struct file_operations cifs_file_strict_nobrl_ops = { + .read_iter = cifs_strict_readv, + .write_iter = cifs_strict_writev, + .open = cifs_open, + .release = cifs_close, + .fsync = cifs_strict_fsync, + .flush = cifs_flush, + .mmap_prepare = cifs_file_strict_mmap_prepare, + .splice_read = filemap_splice_read, + .splice_write = iter_file_splice_write, + .llseek = cifs_llseek, + .unlocked_ioctl = cifs_ioctl, + .copy_file_range = cifs_copy_file_range, + .remap_file_range = cifs_remap_file_range, + .setlease = cifs_setlease, + .fallocate = cifs_fallocate, +}; + +const struct file_operations cifs_file_direct_nobrl_ops = { + .read_iter = netfs_unbuffered_read_iter, + .write_iter = cifs_direct_write_iter, + .open = cifs_open, + .release = cifs_close, + .fsync = cifs_fsync, + .flush = cifs_flush, + .mmap_prepare = cifs_file_mmap_prepare, + .splice_read = copy_splice_read, + .splice_write = iter_file_splice_write, + .unlocked_ioctl = cifs_ioctl, + .copy_file_range = cifs_copy_file_range, + .remap_file_range = cifs_remap_file_range, + .llseek = cifs_llseek, + .setlease = cifs_setlease, + .fallocate = cifs_fallocate, +}; + +const struct file_operations cifs_dir_ops = { + .iterate_shared = cifs_readdir, + .release = cifs_closedir, + .read = generic_read_dir, + .unlocked_ioctl = cifs_ioctl, + .copy_file_range = cifs_copy_file_range, + .remap_file_range = cifs_remap_file_range, + .llseek = generic_file_llseek, + .fsync = cifs_dir_fsync, +}; + +static void +cifs_init_once(void *inode) +{ + struct cifsInodeInfo *cifsi = inode; + + inode_init_once(&cifsi->netfs.inode); + init_rwsem(&cifsi->lock_sem); +} + +static int __init +cifs_init_inodecache(void) +{ + cifs_inode_cachep = kmem_cache_create("cifs_inode_cache", + sizeof(struct cifsInodeInfo), + 0, (SLAB_RECLAIM_ACCOUNT| + SLAB_ACCOUNT), + cifs_init_once); + if (cifs_inode_cachep == NULL) + return -ENOMEM; + + return 0; +} + +static void +cifs_destroy_inodecache(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(cifs_inode_cachep); +} + +static int +cifs_init_request_bufs(void) +{ + /* + * SMB2 maximum header size is bigger than CIFS one - no problems to + * allocate some more bytes for CIFS. + */ + size_t max_hdr_size = MAX_SMB2_HDR_SIZE; + + if (CIFSMaxBufSize < 8192) { + /* Buffer size can not be smaller than 2 * PATH_MAX since maximum + Unicode path name has to fit in any SMB/CIFS path based frames */ + CIFSMaxBufSize = 8192; + } else if (CIFSMaxBufSize > 1024*127) { + CIFSMaxBufSize = 1024 * 127; + } else { + CIFSMaxBufSize &= 0x1FE00; /* Round size to even 512 byte mult*/ + } +/* + cifs_dbg(VFS, "CIFSMaxBufSize %d 0x%x\n", + CIFSMaxBufSize, CIFSMaxBufSize); +*/ + cifs_req_cachep = kmem_cache_create_usercopy("cifs_request", + CIFSMaxBufSize + max_hdr_size, 0, + SLAB_HWCACHE_ALIGN, 0, + CIFSMaxBufSize + max_hdr_size, + NULL); + if (cifs_req_cachep == NULL) + return -ENOMEM; + + if (cifs_min_rcv < 1) + cifs_min_rcv = 1; + else if (cifs_min_rcv > 64) { + cifs_min_rcv = 64; + cifs_dbg(VFS, "cifs_min_rcv set to maximum (64)\n"); + } + + cifs_req_poolp = mempool_create_slab_pool(cifs_min_rcv, + cifs_req_cachep); + + if (cifs_req_poolp == NULL) { + kmem_cache_destroy(cifs_req_cachep); + return -ENOMEM; + } + /* MAX_CIFS_SMALL_BUFFER_SIZE bytes is enough for most SMB responses and + almost all handle based requests (but not write response, nor is it + sufficient for path based requests). A smaller size would have + been more efficient (compacting multiple slab items on one 4k page) + for the case in which debug was on, but this larger size allows + more SMBs to use small buffer alloc and is still much more + efficient to alloc 1 per page off the slab compared to 17K (5page) + alloc of large cifs buffers even when page debugging is on */ + cifs_sm_req_cachep = kmem_cache_create_usercopy("cifs_small_rq", + MAX_CIFS_SMALL_BUFFER_SIZE, 0, SLAB_HWCACHE_ALIGN, + 0, MAX_CIFS_SMALL_BUFFER_SIZE, NULL); + if (cifs_sm_req_cachep == NULL) { + mempool_destroy(cifs_req_poolp); + kmem_cache_destroy(cifs_req_cachep); + return -ENOMEM; + } + + if (cifs_min_small < 2) + cifs_min_small = 2; + else if (cifs_min_small > 256) { + cifs_min_small = 256; + cifs_dbg(FYI, "cifs_min_small set to maximum (256)\n"); + } + + cifs_sm_req_poolp = mempool_create_slab_pool(cifs_min_small, + cifs_sm_req_cachep); + + if (cifs_sm_req_poolp == NULL) { + mempool_destroy(cifs_req_poolp); + kmem_cache_destroy(cifs_req_cachep); + kmem_cache_destroy(cifs_sm_req_cachep); + return -ENOMEM; + } + + return 0; +} + +static void +cifs_destroy_request_bufs(void) +{ + mempool_destroy(cifs_req_poolp); + kmem_cache_destroy(cifs_req_cachep); + mempool_destroy(cifs_sm_req_poolp); + kmem_cache_destroy(cifs_sm_req_cachep); +} + +static int init_mids(void) +{ + cifs_mid_cachep = kmem_cache_create("cifs_mpx_ids", + sizeof(struct mid_q_entry), 0, + SLAB_HWCACHE_ALIGN, NULL); + if (cifs_mid_cachep == NULL) + return -ENOMEM; + + /* 3 is a reasonable minimum number of simultaneous operations */ + if (mempool_init_slab_pool(&cifs_mid_pool, 3, cifs_mid_cachep) < 0) { + kmem_cache_destroy(cifs_mid_cachep); + return -ENOMEM; + } + + return 0; +} + +static void destroy_mids(void) +{ + mempool_exit(&cifs_mid_pool); + kmem_cache_destroy(cifs_mid_cachep); +} + +static int cifs_init_netfs(void) +{ + cifs_io_request_cachep = + kmem_cache_create("cifs_io_request", + sizeof(struct cifs_io_request), 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!cifs_io_request_cachep) + goto nomem_req; + + if (mempool_init_slab_pool(&cifs_io_request_pool, 100, cifs_io_request_cachep) < 0) + goto nomem_reqpool; + + cifs_io_subrequest_cachep = + kmem_cache_create("cifs_io_subrequest", + sizeof(struct cifs_io_subrequest), 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!cifs_io_subrequest_cachep) + goto nomem_subreq; + + if (mempool_init_slab_pool(&cifs_io_subrequest_pool, 100, cifs_io_subrequest_cachep) < 0) + goto nomem_subreqpool; + + return 0; + +nomem_subreqpool: + kmem_cache_destroy(cifs_io_subrequest_cachep); +nomem_subreq: + mempool_exit(&cifs_io_request_pool); +nomem_reqpool: + kmem_cache_destroy(cifs_io_request_cachep); +nomem_req: + return -ENOMEM; +} + +static void cifs_destroy_netfs(void) +{ + mempool_exit(&cifs_io_subrequest_pool); + kmem_cache_destroy(cifs_io_subrequest_cachep); + mempool_exit(&cifs_io_request_pool); + kmem_cache_destroy(cifs_io_request_cachep); +} + +static int __init +init_cifs(void) +{ + int rc = 0; + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + rc = smb1_init_maperror(); + if (rc) + return rc; +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + + rc = smb2_init_maperror(); + if (rc) + return rc; + + cifs_proc_init(); + INIT_LIST_HEAD(&cifs_tcp_ses_list); +/* + * Initialize Global counters + */ + atomic_set(&sesInfoAllocCount, 0); + atomic_set(&tconInfoAllocCount, 0); + atomic_set(&tcpSesNextId, 0); + atomic_set(&tcpSesAllocCount, 0); + atomic_set(&tcpSesReconnectCount, 0); + atomic_set(&tconInfoReconnectCount, 0); + + atomic_set(&buf_alloc_count, 0); + atomic_set(&small_buf_alloc_count, 0); +#ifdef CONFIG_CIFS_STATS2 + atomic_set(&total_buf_alloc_count, 0); + atomic_set(&total_small_buf_alloc_count, 0); + if (slow_rsp_threshold < 1) + cifs_dbg(FYI, "slow_response_threshold msgs disabled\n"); + else if (slow_rsp_threshold > 32767) + cifs_dbg(VFS, + "slow response threshold set higher than recommended (0 to 32767)\n"); +#endif /* CONFIG_CIFS_STATS2 */ + + atomic_set(&mid_count, 0); + GlobalCurrentXid = 0; + GlobalTotalActiveXid = 0; + GlobalMaxActiveXid = 0; + + cifs_lock_secret = get_random_u32(); + + if (cifs_max_pending < 2) { + cifs_max_pending = 2; + cifs_dbg(FYI, "cifs_max_pending set to min of 2\n"); + } else if (cifs_max_pending > CIFS_MAX_REQ) { + cifs_max_pending = CIFS_MAX_REQ; + cifs_dbg(FYI, "cifs_max_pending set to max of %u\n", + CIFS_MAX_REQ); + } + + /* Limit max to about 18 hours, and setting to zero disables directory entry caching */ + if (dir_cache_timeout > 65000) { + dir_cache_timeout = 65000; + cifs_dbg(VFS, "dir_cache_timeout set to max of 65000 seconds\n"); + } + + cifsiod_wq = alloc_workqueue("cifsiod", + WQ_FREEZABLE | WQ_MEM_RECLAIM | WQ_PERCPU, + 0); + if (!cifsiod_wq) { + rc = -ENOMEM; + goto out_clean_proc; + } + + /* + * Consider in future setting limit!=0 maybe to min(num_of_cores - 1, 3) + * so that we don't launch too many worker threads but + * Documentation/core-api/workqueue.rst recommends setting it to 0 + */ + + /* WQ_UNBOUND allows decrypt tasks to run on any CPU */ + decrypt_wq = alloc_workqueue("smb3decryptd", + WQ_UNBOUND|WQ_FREEZABLE|WQ_MEM_RECLAIM, 0); + if (!decrypt_wq) { + rc = -ENOMEM; + goto out_destroy_cifsiod_wq; + } + + fileinfo_put_wq = alloc_workqueue("cifsfileinfoput", + WQ_UNBOUND|WQ_FREEZABLE|WQ_MEM_RECLAIM, 0); + if (!fileinfo_put_wq) { + rc = -ENOMEM; + goto out_destroy_decrypt_wq; + } + + cifsoplockd_wq = alloc_workqueue("cifsoplockd", + WQ_FREEZABLE | WQ_MEM_RECLAIM | WQ_PERCPU, + 0); + if (!cifsoplockd_wq) { + rc = -ENOMEM; + goto out_destroy_fileinfo_put_wq; + } + + deferredclose_wq = alloc_workqueue("deferredclose", + WQ_FREEZABLE | WQ_MEM_RECLAIM | WQ_PERCPU, + 0); + if (!deferredclose_wq) { + rc = -ENOMEM; + goto out_destroy_cifsoplockd_wq; + } + + serverclose_wq = alloc_workqueue("serverclose", + WQ_FREEZABLE | WQ_MEM_RECLAIM | WQ_PERCPU, + 0); + if (!serverclose_wq) { + rc = -ENOMEM; + goto out_destroy_deferredclose_wq; + } + + cfid_put_wq = alloc_workqueue("cfid_put_wq", + WQ_FREEZABLE | WQ_MEM_RECLAIM | WQ_PERCPU, + 0); + if (!cfid_put_wq) { + rc = -ENOMEM; + goto out_destroy_serverclose_wq; + } + + rc = cifs_init_inodecache(); + if (rc) + goto out_destroy_cfid_put_wq; + + rc = cifs_init_netfs(); + if (rc) + goto out_destroy_inodecache; + + rc = init_mids(); + if (rc) + goto out_destroy_netfs; + + rc = cifs_init_request_bufs(); + if (rc) + goto out_destroy_mids; + +#ifdef CONFIG_CIFS_DFS_UPCALL + rc = dfs_cache_init(); + if (rc) + goto out_destroy_request_bufs; +#endif /* CONFIG_CIFS_DFS_UPCALL */ +#ifdef CONFIG_CIFS_UPCALL + rc = init_cifs_spnego(); + if (rc) + goto out_destroy_dfs_cache; +#endif /* CONFIG_CIFS_UPCALL */ +#ifdef CONFIG_CIFS_SWN_UPCALL + rc = cifs_genl_init(); + if (rc) + goto out_register_key_type; +#endif /* CONFIG_CIFS_SWN_UPCALL */ + + rc = init_cifs_idmap(); + if (rc) + goto out_cifs_swn_init; + + rc = register_filesystem(&cifs_fs_type); + if (rc) + goto out_init_cifs_idmap; + + rc = register_filesystem(&smb3_fs_type); + if (rc) { + unregister_filesystem(&cifs_fs_type); + goto out_init_cifs_idmap; + } + + return 0; + +out_init_cifs_idmap: + exit_cifs_idmap(); +out_cifs_swn_init: +#ifdef CONFIG_CIFS_SWN_UPCALL + cifs_genl_exit(); +out_register_key_type: +#endif +#ifdef CONFIG_CIFS_UPCALL + exit_cifs_spnego(); +out_destroy_dfs_cache: +#endif +#ifdef CONFIG_CIFS_DFS_UPCALL + dfs_cache_destroy(); +out_destroy_request_bufs: +#endif + cifs_destroy_request_bufs(); +out_destroy_mids: + destroy_mids(); +out_destroy_netfs: + cifs_destroy_netfs(); +out_destroy_inodecache: + cifs_destroy_inodecache(); +out_destroy_cfid_put_wq: + destroy_workqueue(cfid_put_wq); +out_destroy_serverclose_wq: + destroy_workqueue(serverclose_wq); +out_destroy_deferredclose_wq: + destroy_workqueue(deferredclose_wq); +out_destroy_cifsoplockd_wq: + destroy_workqueue(cifsoplockd_wq); +out_destroy_fileinfo_put_wq: + destroy_workqueue(fileinfo_put_wq); +out_destroy_decrypt_wq: + destroy_workqueue(decrypt_wq); +out_destroy_cifsiod_wq: + destroy_workqueue(cifsiod_wq); +out_clean_proc: + cifs_proc_clean(); + return rc; +} + +static void __exit +exit_cifs(void) +{ + cifs_dbg(NOISY, "exit_smb3\n"); + unregister_filesystem(&cifs_fs_type); + unregister_filesystem(&smb3_fs_type); + cifs_release_automount_timer(); + exit_cifs_idmap(); +#ifdef CONFIG_CIFS_SWN_UPCALL + cifs_genl_exit(); +#endif +#ifdef CONFIG_CIFS_UPCALL + exit_cifs_spnego(); +#endif +#ifdef CONFIG_CIFS_DFS_UPCALL + dfs_cache_destroy(); +#endif + cifs_destroy_request_bufs(); + destroy_mids(); + cifs_destroy_netfs(); + cifs_destroy_inodecache(); + destroy_workqueue(deferredclose_wq); + destroy_workqueue(cifsoplockd_wq); + destroy_workqueue(decrypt_wq); + destroy_workqueue(fileinfo_put_wq); + destroy_workqueue(serverclose_wq); + destroy_workqueue(cfid_put_wq); + destroy_workqueue(cifsiod_wq); + cifs_proc_clean(); +} + +MODULE_AUTHOR("Steve French"); +MODULE_LICENSE("GPL"); /* combination of LGPL + GPL source behaves as GPL */ +MODULE_DESCRIPTION + ("VFS to access SMB3 servers e.g. Samba, Macs, Azure and Windows (and " + "also older servers complying with the SNIA CIFS Specification)"); +MODULE_VERSION(CIFS_VERSION); +MODULE_SOFTDEP("nls"); +MODULE_SOFTDEP("aes"); +MODULE_SOFTDEP("aead2"); +MODULE_SOFTDEP("ccm"); +MODULE_SOFTDEP("gcm"); +module_init(init_cifs) +module_exit(exit_cifs) diff --git a/fs/smb/client/cifsfs.h b/fs/smb/client/cifsfs.h new file mode 100644 index 000000000..287c63239 --- /dev/null +++ b/fs/smb/client/cifsfs.h @@ -0,0 +1,171 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +/* + * + * Copyright (c) International Business Machines Corp., 2002, 2007 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ + +#ifndef _CIFSFS_H +#define _CIFSFS_H + +#include +#include + +#define ROOT_I 2 + +extern atomic_t cifs_sillycounter; +extern atomic_t cifs_tmpcounter; + +/* + * ino_t is 32-bits on 32-bit arch. We have to squash the 64-bit value down + * so that it will fit. We use hash_64 to convert the value to 31 bits, and + * then add 1, to ensure that we don't end up with a 0 as the value. + */ +static inline ino_t +cifs_uniqueid_to_ino_t(u64 fileid) +{ + if ((sizeof(ino_t)) < (sizeof(u64))) + return (ino_t)hash_64(fileid, (sizeof(ino_t) * 8) - 1) + 1; + + return (ino_t)fileid; + +} + +static inline void cifs_set_time(struct dentry *dentry, unsigned long time) +{ + dentry->d_fsdata = (void *) time; +} + +static inline unsigned long cifs_get_time(struct dentry *dentry) +{ + return (unsigned long) dentry->d_fsdata; +} + +extern struct file_system_type cifs_fs_type, smb3_fs_type; +extern const struct address_space_operations cifs_addr_ops; +extern const struct address_space_operations cifs_addr_ops_smallbuf; + +/* Functions related to super block operations */ +void cifs_sb_active(struct super_block *sb); +void cifs_sb_deactive(struct super_block *sb); + +/* Functions related to inodes */ +extern const struct inode_operations cifs_dir_inode_ops; +struct inode *cifs_root_iget(struct super_block *sb); +int cifs_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *direntry, umode_t mode); +int cifs_atomic_open(struct inode *dir, struct dentry *direntry, + struct file *file, unsigned int oflags, umode_t mode); +int cifs_tmpfile(struct mnt_idmap *idmap, struct inode *dir, + struct file *file, umode_t mode); +struct dentry *cifs_lookup(struct inode *parent_dir_inode, + struct dentry *direntry, unsigned int flags); +int cifs_unlink(struct inode *dir, struct dentry *dentry); +int cifs_hardlink(struct dentry *old_file, struct inode *inode, + struct dentry *direntry); +int cifs_mknod(struct mnt_idmap *idmap, struct inode *inode, + struct dentry *direntry, umode_t mode, dev_t device_number); +struct dentry *cifs_mkdir(struct mnt_idmap *idmap, struct inode *inode, + struct dentry *direntry, umode_t mode); +int cifs_rmdir(struct inode *inode, struct dentry *direntry); +int cifs_rename2(struct mnt_idmap *idmap, struct inode *source_dir, + struct dentry *source_dentry, struct inode *target_dir, + struct dentry *target_dentry, unsigned int flags); +int cifs_revalidate_file_attr(struct file *filp); +int cifs_revalidate_dentry_attr(struct dentry *dentry); +int cifs_revalidate_file(struct file *filp); +int cifs_revalidate_dentry(struct dentry *dentry); +int cifs_revalidate_mapping(struct inode *inode); +int cifs_zap_mapping(struct inode *inode); +int cifs_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags); +int cifs_setattr(struct mnt_idmap *idmap, struct dentry *direntry, + struct iattr *attrs); +int cifs_fiemap(struct inode *inode, struct fiemap_extent_info *fei, u64 start, + u64 len); + +extern const struct inode_operations cifs_file_inode_ops; +extern const struct inode_operations cifs_symlink_inode_ops; +extern const struct inode_operations cifs_namespace_inode_operations; + +struct file_kattr; +int cifs_fileattr_get(struct dentry *dentry, struct file_kattr *fa); + + +/* Functions related to files and directories */ +extern const struct netfs_request_ops cifs_req_ops; +extern const struct file_operations cifs_file_ops; +extern const struct file_operations cifs_file_direct_ops; /* if directio mnt */ +extern const struct file_operations cifs_file_strict_ops; /* if strictio mnt */ +extern const struct file_operations cifs_file_nobrl_ops; /* no brlocks */ +extern const struct file_operations cifs_file_direct_nobrl_ops; +extern const struct file_operations cifs_file_strict_nobrl_ops; +int cifs_open(struct inode *inode, struct file *file); +int cifs_close(struct inode *inode, struct file *file); +int cifs_closedir(struct inode *inode, struct file *file); +ssize_t cifs_strict_readv(struct kiocb *iocb, struct iov_iter *to); +ssize_t cifs_strict_writev(struct kiocb *iocb, struct iov_iter *from); +ssize_t cifs_file_write_iter(struct kiocb *iocb, struct iov_iter *from); +ssize_t cifs_direct_write_iter(struct kiocb *iocb, struct iov_iter *from); +ssize_t cifs_loose_read_iter(struct kiocb *iocb, struct iov_iter *iter); +int cifs_flock(struct file *file, int cmd, struct file_lock *fl); +int cifs_lock(struct file *file, int cmd, struct file_lock *flock); +int cifs_fsync(struct file *file, loff_t start, loff_t end, int datasync); +int cifs_strict_fsync(struct file *file, loff_t start, loff_t end, + int datasync); +int cifs_flush(struct file *file, fl_owner_t id); +int cifs_file_mmap_prepare(struct vm_area_desc *desc); +int cifs_file_strict_mmap_prepare(struct vm_area_desc *desc); +extern const struct file_operations cifs_dir_ops; +int cifs_readdir(struct file *file, struct dir_context *ctx); + +/* Functions related to dir entries */ +extern const struct dentry_operations cifs_dentry_ops; +extern const struct dentry_operations cifs_ci_dentry_ops; + +struct vfsmount *cifs_d_automount(struct path *path); + +/* Functions related to symlinks */ +const char *cifs_get_link(struct dentry *dentry, struct inode *inode, + struct delayed_call *done); +int cifs_symlink(struct mnt_idmap *idmap, struct inode *inode, + struct dentry *direntry, const char *symname); + +#ifdef CONFIG_CIFS_XATTR +extern const struct xattr_handler * const cifs_xattr_handlers[]; +ssize_t cifs_listxattr(struct dentry *direntry, char *data, size_t buf_size); +#else +# define cifs_xattr_handlers NULL +# define cifs_listxattr NULL +#endif + +ssize_t cifs_file_copychunk_range(unsigned int xid, struct file *src_file, + loff_t off, struct file *dst_file, + loff_t destoff, size_t len, + unsigned int flags); + +long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg); +void cifs_setsize(struct inode *inode, loff_t offset); + +struct fs_context; +struct smb3_fs_context; +struct dentry *cifs_smb3_do_mount(struct fs_context *fc, + struct smb3_fs_context *old_ctx); + +char *cifs_silly_fullpath(struct dentry *dentry); + +#define CIFS_TMPNAME_PREFIX ".__smbfile_tmp" +#define CIFS_TMPNAME_LEN (DNAME_INLINE_LEN - 1) + +#define CIFS_SILLYNAME_PREFIX ".__smbfile_silly" +#define CIFS_SILLYNAME_LEN (DNAME_INLINE_LEN - 1) + +#ifdef CONFIG_CIFS_NFSD_EXPORT +extern const struct export_operations cifs_export_ops; +#endif /* CONFIG_CIFS_NFSD_EXPORT */ + +/* when changing internal version - update following two lines at same time */ +#define SMB3_PRODUCT_BUILD 61 +#define CIFS_VERSION "2.61" +#endif /* _CIFSFS_H */ diff --git a/fs/smb/client/cifsglob.h b/fs/smb/client/cifsglob.h new file mode 100644 index 000000000..79e4e84f8 --- /dev/null +++ b/fs/smb/client/cifsglob.h @@ -0,0 +1,2404 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +/* + * + * Copyright (C) International Business Machines Corp., 2002,2008 + * Author(s): Steve French (sfrench@us.ibm.com) + * Jeremy Allison (jra@samba.org) + * + */ +#ifndef _CIFS_GLOB_H +#define _CIFS_GLOB_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "cifs_fs_sb.h" +#include "cifsacl.h" +#include +#include "../common/smbglob.h" +#include "../common/smb2pdu.h" +#include "../common/fscc.h" +#include "smb2pdu.h" +#include "smb1pdu.h" +#include + +#define SMB_PATH_MAX 260 +#define CIFS_PORT 445 +#define RFC1001_PORT 139 + +/* + * The sizes of various internal tables and strings + */ +#define MAX_UID_INFO 16 +#define MAX_SES_INFO 2 +#define MAX_TCON_INFO 4 + +#define MAX_TREE_SIZE (2 + CIFS_NI_MAXHOST + 1 + CIFS_MAX_SHARE_LEN + 1) + +#define CIFS_MIN_RCV_POOL 4 + +#define MAX_REOPEN_ATT 5 /* these many maximum attempts to reopen a file */ +/* + * default attribute cache timeout (jiffies) + */ +#define CIFS_DEF_ACTIMEO (1 * HZ) + +/* + * max sleep time before retry to server + */ +#define CIFS_MAX_SLEEP 2000 + +/* + * max attribute cache timeout (jiffies) - 2^30 + */ +#define CIFS_MAX_ACTIMEO (1 << 30) + +/* + * Max persistent and resilient handle timeout (milliseconds). + * Windows durable max was 960000 (16 minutes) + */ +#define SMB3_MAX_HANDLE_TIMEOUT 960000 + +/* + * MAX_REQ is the maximum number of requests that WE will send + * on one socket concurrently. + */ +#define CIFS_MAX_REQ 32767 + +#define RFC1001_NAME_LEN 15 +#define RFC1001_NAME_LEN_WITH_NULL (RFC1001_NAME_LEN + 1) + +/* maximum length of ip addr as a string (including ipv6 and sctp) */ +#define SERVER_NAME_LENGTH 80 +#define SERVER_NAME_LEN_WITH_NULL (SERVER_NAME_LENGTH + 1) + +/* echo interval in seconds */ +#define SMB_ECHO_INTERVAL_MIN 1 +#define SMB_ECHO_INTERVAL_MAX 600 +#define SMB_ECHO_INTERVAL_DEFAULT 60 + +/* smb multichannel query server interfaces interval in seconds */ +#define SMB_INTERFACE_POLL_INTERVAL 600 + +/* maximum number of PDUs in one compound */ +#define MAX_COMPOUND 10 + +/* + * Default number of credits to keep available for SMB3. + * This value is chosen somewhat arbitrarily. The Windows client + * defaults to 128 credits, the Windows server allows clients up to + * 512 credits (or 8K for later versions), and the NetApp server + * does not limit clients at all. Choose a high enough default value + * such that the client shouldn't limit performance, but allow mount + * to override (until you approach 64K, where we limit credits to 65000 + * to reduce possibility of seeing more server credit overflow bugs. + */ +#define SMB2_MAX_CREDITS_AVAILABLE 32000 + +#ifndef XATTR_DOS_ATTRIB +#define XATTR_DOS_ATTRIB "user.DOSATTRIB" +#endif + +#define CIFS_MAX_WORKSTATION_LEN (__NEW_UTS_LEN + 1) /* reasonable max for client */ + +#define CIFS_DFS_ROOT_SES(ses) ((ses)->dfs_root_ses ?: (ses)) + +/* + * CIFS vfs client Status information (based on what we know.) + */ + +/* associated with each connection */ +enum statusEnum { + CifsNew = 0, + CifsGood, + CifsExiting, + CifsNeedReconnect, + CifsNeedNegotiate, + CifsInNegotiate, +}; + +/* associated with each smb session */ +enum ses_status_enum { + SES_NEW = 0, + SES_GOOD, + SES_EXITING, + SES_NEED_RECON, + SES_IN_SETUP +}; + +/* associated with each tree connection to the server */ +enum tid_status_enum { + TID_NEW = 0, + TID_GOOD, + TID_EXITING, + TID_NEED_RECON, + TID_NEED_TCON, + TID_IN_TCON, + TID_NEED_FILES_INVALIDATE, /* currently unused */ + TID_IN_FILES_INVALIDATE +}; + +enum securityEnum { + Unspecified = 0, /* not specified */ + NTLMv2, /* Legacy NTLM auth with NTLMv2 hash */ + RawNTLMSSP, /* NTLMSSP without SPNEGO, NTLMv2 hash */ + Kerberos, /* Kerberos via SPNEGO */ + IAKerb, /* Kerberos proxy */ +}; + +enum upcall_target_enum { + UPTARGET_UNSPECIFIED, /* not specified, defaults to app */ + UPTARGET_MOUNT, /* upcall to the mount namespace */ + UPTARGET_APP, /* upcall to the application namespace which did the mount */ +}; + +enum cifs_reparse_type { + CIFS_REPARSE_TYPE_NONE, + CIFS_REPARSE_TYPE_NFS, + CIFS_REPARSE_TYPE_WSL, + CIFS_REPARSE_TYPE_DEFAULT = CIFS_REPARSE_TYPE_NFS, +}; + +static inline const char *cifs_reparse_type_str(enum cifs_reparse_type type) +{ + switch (type) { + case CIFS_REPARSE_TYPE_NONE: + return "none"; + case CIFS_REPARSE_TYPE_NFS: + return "nfs"; + case CIFS_REPARSE_TYPE_WSL: + return "wsl"; + default: + return "unknown"; + } +} + +enum cifs_symlink_type { + CIFS_SYMLINK_TYPE_DEFAULT, + CIFS_SYMLINK_TYPE_NONE, + CIFS_SYMLINK_TYPE_NATIVE, + CIFS_SYMLINK_TYPE_UNIX, + CIFS_SYMLINK_TYPE_MFSYMLINKS, + CIFS_SYMLINK_TYPE_SFU, + CIFS_SYMLINK_TYPE_NFS, + CIFS_SYMLINK_TYPE_WSL, +}; + +static inline const char *cifs_symlink_type_str(enum cifs_symlink_type type) +{ + switch (type) { + case CIFS_SYMLINK_TYPE_NONE: + return "none"; + case CIFS_SYMLINK_TYPE_NATIVE: + return "native"; + case CIFS_SYMLINK_TYPE_UNIX: + return "unix"; + case CIFS_SYMLINK_TYPE_MFSYMLINKS: + return "mfsymlinks"; + case CIFS_SYMLINK_TYPE_SFU: + return "sfu"; + case CIFS_SYMLINK_TYPE_NFS: + return "nfs"; + case CIFS_SYMLINK_TYPE_WSL: + return "wsl"; + default: + return "unknown"; + } +} + +struct session_key { + unsigned int len; + char *response; +}; + +/* encryption related structure/fields, not specific to a sec mech */ +struct cifs_secmech { + struct crypto_aead *enc; /* smb3 encryption AEAD TFM (AES-CCM and AES-GCM) */ + struct crypto_aead *dec; /* smb3 decryption AEAD TFM (AES-CCM and AES-GCM) */ +}; + +/* per smb session structure/fields */ +struct ntlmssp_auth { + bool sesskey_per_smbsess; /* whether session key is per smb session */ + __u32 client_flags; /* sent by client in type 1 ntlmsssp exchange */ + __u32 server_flags; /* sent by server in type 2 ntlmssp exchange */ + unsigned char ciphertext[CIFS_CPHTXT_SIZE]; /* sent to server */ + char cryptkey[CIFS_CRYPTO_KEY_SIZE]; /* used by ntlmssp */ +}; + +struct cifs_cred { + int uid; + int gid; + int mode; + int cecount; + struct smb_sid osid; + struct smb_sid gsid; + struct cifs_ntace *ntaces; + struct smb_ace *aces; +}; + +struct cifs_open_info_data { + bool adjust_tz; + bool reparse_point; + bool contains_posix_file_info; + bool unknown_nlink; + struct { + /* ioctl response buffer */ + struct { + int buftype; + struct kvec iov; + } io; + __u32 tag; + struct reparse_data_buffer *buf; + } reparse; + struct { + __u8 eas[SMB2_WSL_MAX_QUERY_EA_RESP_SIZE]; + unsigned int eas_len; + } wsl; + char *symlink_target; + struct smb_sid posix_owner; + struct smb_sid posix_group; + union { + struct smb2_file_all_info fi; + struct smb311_posix_qinfo posix_fi; + }; +}; + +/* + ***************************************************************** + * Except the CIFS PDUs themselves all the + * globally interesting structs should go here + ***************************************************************** + */ + +/* + * A smb_rqst represents a complete request to be issued to a server. It's + * formed by a kvec array, followed by an array of pages. Page data is assumed + * to start at the beginning of the first page. + */ +struct smb_rqst { + struct kvec *rq_iov; /* array of kvecs */ + unsigned int rq_nvec; /* number of kvecs in array */ + struct iov_iter rq_iter; /* Data iterator */ + struct folio_queue *rq_buffer; /* Buffer for encryption */ +}; + +struct mid_q_entry; +struct TCP_Server_Info; +struct cifsFileInfo; +struct cifs_ses; +struct cifs_tcon; +struct dfs_info3_param; +struct cifs_fattr; +struct smb3_fs_context; +struct cifs_fid; +struct cifs_io_subrequest; +struct cifs_io_parms; +struct cifs_search_info; +struct cifsInodeInfo; +struct cifs_open_parms; +struct cifs_credits; + +struct smb_version_operations { + int (*send_cancel)(struct cifs_ses *ses, struct TCP_Server_Info *server, + struct smb_rqst *rqst, struct mid_q_entry *mid, + unsigned int xid); + bool (*compare_fids)(struct cifsFileInfo *, struct cifsFileInfo *); + /* setup request: allocate mid, sign message */ + struct mid_q_entry *(*setup_request)(struct cifs_ses *, + struct TCP_Server_Info *, + struct smb_rqst *); + /* setup async request: allocate mid, sign message */ + struct mid_q_entry *(*setup_async_request)(struct TCP_Server_Info *, + struct smb_rqst *); + /* check response: verify signature, map error */ + int (*check_receive)(struct mid_q_entry *, struct TCP_Server_Info *, + bool); + void (*add_credits)(struct TCP_Server_Info *server, + struct cifs_credits *credits, + const int optype); + void (*set_credits)(struct TCP_Server_Info *, const int); + int * (*get_credits_field)(struct TCP_Server_Info *, const int); + unsigned int (*get_credits)(struct mid_q_entry *); + __u64 (*get_next_mid)(struct TCP_Server_Info *); + void (*revert_current_mid)(struct TCP_Server_Info *server, + const unsigned int val); + /* data offset from read response message */ + unsigned int (*read_data_offset)(char *); + /* + * Data length from read response message + * When in_remaining is true, the returned data length is in + * message field DataRemaining for out-of-band data read (e.g through + * Memory Registration RDMA write in SMBD). + * Otherwise, the returned data length is in message field DataLength. + */ + unsigned int (*read_data_length)(char *, bool in_remaining); + /* map smb to linux error */ + int (*map_error)(char *, bool); + /* find mid corresponding to the response message */ + struct mid_q_entry *(*find_mid)(struct TCP_Server_Info *server, char *buf); + void (*dump_detail)(void *buf, size_t buf_len, struct TCP_Server_Info *ptcp_info); + void (*clear_stats)(struct cifs_tcon *); + void (*print_stats)(struct seq_file *m, struct cifs_tcon *); + void (*dump_share_caps)(struct seq_file *, struct cifs_tcon *); + /* verify the message */ + int (*check_message)(char *buf, unsigned int pdu_len, unsigned int len, + struct TCP_Server_Info *server); + bool (*is_oplock_break)(char *, struct TCP_Server_Info *); + int (*handle_cancelled_mid)(struct mid_q_entry *, struct TCP_Server_Info *); + void (*downgrade_oplock)(struct TCP_Server_Info *server, + struct cifsInodeInfo *cinode, __u32 oplock, + __u16 epoch, bool *purge_cache); + /* process transaction2 response */ + bool (*check_trans2)(struct mid_q_entry *, struct TCP_Server_Info *, + char *, int); + /* check if we need to negotiate */ + bool (*need_neg)(struct TCP_Server_Info *); + /* negotiate to the server */ + int (*negotiate)(const unsigned int xid, + struct cifs_ses *ses, + struct TCP_Server_Info *server); + /* set negotiated write size */ + unsigned int (*negotiate_wsize)(struct cifs_tcon *tcon, struct smb3_fs_context *ctx); + /* set negotiated read size */ + unsigned int (*negotiate_rsize)(struct cifs_tcon *tcon, struct smb3_fs_context *ctx); + /* setup smb sessionn */ + int (*sess_setup)(const unsigned int, struct cifs_ses *, + struct TCP_Server_Info *server, + const struct nls_table *); + /* close smb session */ + int (*logoff)(const unsigned int, struct cifs_ses *); + /* connect to a server share */ + int (*tree_connect)(const unsigned int, struct cifs_ses *, const char *, + struct cifs_tcon *, const struct nls_table *); + /* close tree connection */ + int (*tree_disconnect)(const unsigned int, struct cifs_tcon *); + /* get DFS referrals */ + int (*get_dfs_refer)(const unsigned int, struct cifs_ses *, + const char *, struct dfs_info3_param **, + unsigned int *, const struct nls_table *, int); + /* informational QFS call */ + void (*qfs_tcon)(const unsigned int, struct cifs_tcon *, + struct cifs_sb_info *); + /* query for server interfaces */ + int (*query_server_interfaces)(const unsigned int, struct cifs_tcon *, + bool); + /* check if a path is accessible or not */ + int (*is_path_accessible)(const unsigned int, struct cifs_tcon *, + struct cifs_sb_info *, const char *); + /* query path data from the server */ + int (*query_path_info)(const unsigned int xid, + struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, + const char *full_path, + struct cifs_open_info_data *data); + /* query file data from the server */ + int (*query_file_info)(const unsigned int xid, struct cifs_tcon *tcon, + struct cifsFileInfo *cfile, struct cifs_open_info_data *data); + /* query reparse point to determine which type of special file */ + int (*query_reparse_point)(const unsigned int xid, + struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, + const char *full_path, + u32 *tag, struct kvec *rsp, + int *rsp_buftype); + /* get server index number */ + int (*get_srv_inum)(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, const char *full_path, u64 *uniqueid, + struct cifs_open_info_data *data); + /* set size by path */ + int (*set_path_size)(const unsigned int, struct cifs_tcon *, + const char *, __u64, struct cifs_sb_info *, bool, + struct dentry *); + /* set size by file handle */ + int (*set_file_size)(const unsigned int, struct cifs_tcon *, + struct cifsFileInfo *, __u64, bool); + /* set attributes */ + int (*set_file_info)(struct inode *, const char *, FILE_BASIC_INFO *, + const unsigned int); + int (*set_compression)(const unsigned int, struct cifs_tcon *, + struct cifsFileInfo *, __u16); + /* check if we can send an echo or nor */ + bool (*can_echo)(struct TCP_Server_Info *); + /* send echo request */ + int (*echo)(struct TCP_Server_Info *); + /* create directory */ + int (*posix_mkdir)(const unsigned int xid, struct inode *inode, + umode_t mode, struct cifs_tcon *tcon, + const char *full_path, + struct cifs_sb_info *cifs_sb); + int (*mkdir)(const unsigned int xid, struct inode *inode, umode_t mode, + struct cifs_tcon *tcon, const char *name, + struct cifs_sb_info *sb); + /* set info on created directory */ + void (*mkdir_setinfo)(struct inode *, const char *, + struct cifs_sb_info *, struct cifs_tcon *, + const unsigned int); + /* remove directory */ + int (*rmdir)(const unsigned int, struct cifs_tcon *, const char *, + struct cifs_sb_info *); + /* unlink file */ + int (*unlink)(const unsigned int, struct cifs_tcon *, const char *, + struct cifs_sb_info *, struct dentry *); + /* open, rename and delete file */ + int (*rename_pending_delete)(const char *, struct dentry *, + const unsigned int); + /* send rename request */ + int (*rename)(const unsigned int xid, + struct cifs_tcon *tcon, + struct dentry *source_dentry, + const char *from_name, const char *to_name, + struct cifs_sb_info *cifs_sb); + /* send create hardlink request */ + int (*create_hardlink)(const unsigned int xid, + struct cifs_tcon *tcon, + struct dentry *source_dentry, + const char *from_name, const char *to_name, + struct cifs_sb_info *cifs_sb); + /* query symlink target */ + int (*query_symlink)(const unsigned int xid, + struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, + const char *full_path, + char **target_path); + /* open a file for non-posix mounts */ + int (*open)(const unsigned int xid, struct cifs_open_parms *oparms, __u32 *oplock, + void *buf); + /* set fid protocol-specific info */ + void (*set_fid)(struct cifsFileInfo *, struct cifs_fid *, __u32); + /* close a file */ + int (*close)(const unsigned int, struct cifs_tcon *, + struct cifs_fid *); + /* close a file, returning file attributes and timestamps */ + int (*close_getattr)(const unsigned int xid, struct cifs_tcon *tcon, + struct cifsFileInfo *pfile_info); + /* send a flush request to the server */ + int (*flush)(const unsigned int, struct cifs_tcon *, struct cifs_fid *); + /* async read from the server */ + int (*async_readv)(struct cifs_io_subrequest *); + /* async write to the server */ + void (*async_writev)(struct cifs_io_subrequest *); + /* sync read from the server */ + int (*sync_read)(const unsigned int, struct cifs_fid *, + struct cifs_io_parms *, unsigned int *, char **, + int *); + /* sync write to the server */ + int (*sync_write)(const unsigned int, struct cifs_fid *, + struct cifs_io_parms *, unsigned int *, struct kvec *, + unsigned long); + /* open dir, start readdir */ + int (*query_dir_first)(const unsigned int, struct cifs_tcon *, + const char *, struct cifs_sb_info *, + struct cifs_fid *, __u16, + struct cifs_search_info *); + /* continue readdir */ + int (*query_dir_next)(const unsigned int, struct cifs_tcon *, + struct cifs_fid *, + __u16, struct cifs_search_info *srch_inf); + /* close dir */ + int (*close_dir)(const unsigned int, struct cifs_tcon *, + struct cifs_fid *); + /* calculate a size of SMB message */ + unsigned int (*calc_smb_size)(void *buf); + /* check for STATUS_PENDING and process the response if yes */ + bool (*is_status_pending)(char *buf, struct TCP_Server_Info *server); + /* check for STATUS_NETWORK_SESSION_EXPIRED */ + bool (*is_session_expired)(char *); + /* send oplock break response */ + int (*oplock_response)(struct cifs_tcon *tcon, __u64 persistent_fid, + __u64 volatile_fid, __u16 net_fid, + struct cifsInodeInfo *cifs_inode, + unsigned int oplock); + /* query remote filesystem */ + int (*queryfs)(const unsigned int, struct cifs_tcon *, + const char *, struct cifs_sb_info *, struct kstatfs *); + /* send mandatory brlock to the server */ + int (*mand_lock)(const unsigned int, struct cifsFileInfo *, __u64, + __u64, __u32, int, int, bool); + /* unlock range of mandatory locks */ + int (*mand_unlock_range)(struct cifsFileInfo *, struct file_lock *, + const unsigned int); + /* push brlocks from the cache to the server */ + int (*push_mand_locks)(struct cifsFileInfo *); + /* get lease key of the inode */ + void (*get_lease_key)(struct inode *, struct cifs_fid *); + /* set lease key of the inode */ + void (*set_lease_key)(struct inode *, struct cifs_fid *); + /* generate new lease key */ + void (*new_lease_key)(struct cifs_fid *); + int (*generate_signingkey)(struct cifs_ses *ses, + struct TCP_Server_Info *server); + int (*set_integrity)(const unsigned int, struct cifs_tcon *tcon, + struct cifsFileInfo *src_file); + int (*enum_snapshots)(const unsigned int xid, struct cifs_tcon *tcon, + struct cifsFileInfo *src_file, void __user *); + int (*notify)(const unsigned int xid, struct file *pfile, + void __user *pbuf, bool return_changes); + int (*query_mf_symlink)(unsigned int, struct cifs_tcon *, + struct cifs_sb_info *, const unsigned char *, + char *, unsigned int *); + int (*create_mf_symlink)(unsigned int, struct cifs_tcon *, + struct cifs_sb_info *, const unsigned char *, + char *, unsigned int *); + /* if we can do cache read operations */ + bool (*is_read_op)(__u32); + /* set oplock level for the inode */ + void (*set_oplock_level)(struct cifsInodeInfo *cinode, __u32 oplock, __u16 epoch, + bool *purge_cache); + /* create lease context buffer for CREATE request */ + char * (*create_lease_buf)(u8 *lease_key, u8 oplock, u8 *parent_lease_key, __le32 le_flags); + /* parse lease context buffer and return oplock/epoch info */ + __u8 (*parse_lease_buf)(void *buf, __u16 *epoch, char *lkey); + ssize_t (*copychunk_range)(const unsigned int, + struct cifsFileInfo *src_file, + struct cifsFileInfo *target_file, + u64 src_off, u64 len, u64 dest_off); + int (*duplicate_extents)(const unsigned int, struct cifsFileInfo *src, + struct cifsFileInfo *target_file, u64 src_off, u64 len, + u64 dest_off); + int (*validate_negotiate)(const unsigned int, struct cifs_tcon *); + ssize_t (*query_all_EAs)(const unsigned int, struct cifs_tcon *, + const unsigned char *, const unsigned char *, char *, + size_t, struct cifs_sb_info *); + int (*set_EA)(const unsigned int, struct cifs_tcon *, const char *, + const char *, const void *, const __u16, + const struct nls_table *, struct cifs_sb_info *); + struct smb_ntsd * (*get_acl)(struct cifs_sb_info *cifssb, struct inode *ino, + const char *patch, u32 *plen, u32 info); + struct smb_ntsd * (*get_acl_by_fid)(struct cifs_sb_info *cifssmb, + const struct cifs_fid *pfid, u32 *plen, u32 info); + int (*set_acl)(struct smb_ntsd *pntsd, __u32 len, struct inode *ino, const char *path, + int flag); + /* writepages retry size */ + unsigned int (*wp_retry_size)(struct inode *); + /* get mtu credits */ + int (*wait_mtu_credits)(struct TCP_Server_Info *, size_t, + size_t *, struct cifs_credits *); + /* adjust previously taken mtu credits to request size */ + int (*adjust_credits)(struct TCP_Server_Info *server, + struct cifs_io_subrequest *subreq, + unsigned int /*enum smb3_rw_credits_trace*/ trace); + /* check if we need to issue closedir */ + bool (*dir_needs_close)(struct cifsFileInfo *); + long (*fallocate)(struct file *, struct cifs_tcon *, int, loff_t, + loff_t); + /* init transform (compress/encrypt) request */ + int (*init_transform_rq)(struct TCP_Server_Info *, int num_rqst, + struct smb_rqst *, struct smb_rqst *); + int (*is_transform_hdr)(void *buf); + int (*receive_transform)(struct TCP_Server_Info *, + struct mid_q_entry **, char **, int *); + enum securityEnum (*select_sectype)(struct TCP_Server_Info *, + enum securityEnum); + int (*next_header)(struct TCP_Server_Info *server, char *buf, + unsigned int *noff); + /* ioctl passthrough for query_info */ + int (*ioctl_query_info)(const unsigned int xid, + struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, + __le16 *path, int is_dir, + unsigned long p); + /* make unix special files (block, char, fifo, socket) */ + int (*make_node)(unsigned int xid, + struct inode *inode, + struct dentry *dentry, + struct cifs_tcon *tcon, + const char *full_path, + umode_t mode, + dev_t device_number); + /* version specific fiemap implementation */ + int (*fiemap)(struct cifs_tcon *tcon, struct cifsFileInfo *, + struct fiemap_extent_info *, u64, u64); + /* version specific llseek implementation */ + loff_t (*llseek)(struct file *, struct cifs_tcon *, loff_t, int); + /* Check for STATUS_IO_TIMEOUT */ + bool (*is_status_io_timeout)(char *buf); + /* Check for STATUS_NETWORK_NAME_DELETED */ + bool (*is_network_name_deleted)(char *buf, struct TCP_Server_Info *srv); + struct reparse_data_buffer * (*get_reparse_point_buffer)(const struct kvec *rsp_iov, + u32 *plen); + struct inode * (*create_reparse_inode)(struct cifs_open_info_data *data, + struct super_block *sb, + const unsigned int xid, + struct cifs_tcon *tcon, + const char *full_path, + bool directory, + struct kvec *reparse_iov, + struct kvec *xattr_iov); +}; + +#define HEADER_SIZE(server) (server->vals->header_size) +#define MAX_HEADER_SIZE(server) (server->vals->max_header_size) +#define MID_HEADER_SIZE(server) (HEADER_SIZE(server) - 1) + +/** + * CIFS superblock mount flags (mnt_cifs_flags) to consider when + * trying to reuse existing superblock for a new mount + */ +#define CIFS_MOUNT_MASK (CIFS_MOUNT_NO_PERM | CIFS_MOUNT_SET_UID | \ + CIFS_MOUNT_SERVER_INUM | CIFS_MOUNT_DIRECT_IO | \ + CIFS_MOUNT_NO_XATTR | CIFS_MOUNT_MAP_SPECIAL_CHR | \ + CIFS_MOUNT_MAP_SFM_CHR | \ + CIFS_MOUNT_UNX_EMUL | CIFS_MOUNT_NO_BRL | \ + CIFS_MOUNT_CIFS_ACL | CIFS_MOUNT_OVERR_UID | \ + CIFS_MOUNT_OVERR_GID | CIFS_MOUNT_DYNPERM | \ + CIFS_MOUNT_NOPOSIXBRL | CIFS_MOUNT_NOSSYNC | \ + CIFS_MOUNT_FSCACHE | CIFS_MOUNT_MF_SYMLINKS | \ + CIFS_MOUNT_MULTIUSER | CIFS_MOUNT_STRICT_IO | \ + CIFS_MOUNT_CIFS_BACKUPUID | CIFS_MOUNT_CIFS_BACKUPGID | \ + CIFS_MOUNT_UID_FROM_ACL | CIFS_MOUNT_NO_HANDLE_CACHE | \ + CIFS_MOUNT_NO_DFS | CIFS_MOUNT_MODE_FROM_SID | \ + CIFS_MOUNT_RO_CACHE | CIFS_MOUNT_RW_CACHE) + +/** + * Generic VFS superblock mount flags (s_flags) to consider when + * trying to reuse existing superblock for a new mount + */ +#define CIFS_MS_MASK (SB_RDONLY | SB_MANDLOCK | SB_NOEXEC | SB_NOSUID | \ + SB_NODEV | SB_SYNCHRONOUS) + +struct cifs_mnt_data { + struct cifs_sb_info *cifs_sb; + struct smb3_fs_context *ctx; + int flags; +}; + +struct TCP_Server_Info { + struct list_head tcp_ses_list; + struct list_head smb_ses_list; + struct list_head rlist; /* reconnect list */ + spinlock_t srv_lock; /* protect anything here that is not protected */ + __u64 conn_id; /* connection identifier (useful for debugging) */ + int srv_count; /* reference counter */ + int rfc1001_sessinit; /* whether to estasblish netbios session */ + bool with_rfc1001; /* if netbios session is used */ + /* 15 character server name + 0x20 16th byte indicating type = srv */ + char server_RFC1001_name[RFC1001_NAME_LEN_WITH_NULL]; + struct smb_version_operations *ops; + struct smb_version_values *vals; + /* updates to tcpStatus protected by cifs_tcp_ses_lock */ + enum statusEnum tcpStatus; /* what we think the status is */ + char *hostname; /* hostname portion of UNC string */ + struct socket *ssocket; + struct sockaddr_storage dstaddr; + struct sockaddr_storage srcaddr; /* locally bind to this IP */ +#ifdef CONFIG_NET_NS + struct net *net; +#endif + wait_queue_head_t response_q; + wait_queue_head_t request_q; /* if more than maxmpx to srvr must block*/ + spinlock_t mid_queue_lock; /* protect mid queue */ + spinlock_t mid_counter_lock; + struct list_head pending_mid_q; + bool noblocksnd; /* use blocking sendmsg */ + bool noautotune; /* do not autotune send buf sizes */ + bool nosharesock; + bool tcp_nodelay; + bool terminate; + int credits; /* send no more requests at once */ + unsigned int max_credits; /* can override large 32000 default at mnt */ + unsigned int in_flight; /* number of requests on the wire to server */ + unsigned int max_in_flight; /* max number of requests that were on wire */ + spinlock_t req_lock; /* protect the two values above */ + struct mutex _srv_mutex; + unsigned int nofs_flag; + struct task_struct *tsk; + char server_GUID[16]; + __u16 sec_mode; + bool sign; /* is signing enabled on this connection? */ + bool ignore_signature:1; /* skip validation of signatures in SMB2/3 rsp */ + bool session_estab; /* mark when very first sess is established */ + int echo_credits; /* echo reserved slots */ + int oplock_credits; /* oplock break reserved slots */ + bool echoes:1; /* enable echoes */ + __u8 client_guid[SMB2_CLIENT_GUID_SIZE]; /* Client GUID */ + u16 dialect; /* dialect index that server chose */ + bool oplocks:1; /* enable oplocks */ + unsigned int maxReq; /* Clients should submit no more */ + /* than maxReq distinct unanswered SMBs to the server when using */ + /* multiplexed reads or writes (for SMB1/CIFS only, not SMB2/SMB3) */ + unsigned int maxBuf; /* maxBuf specifies the maximum */ + /* message size the server can send or receive for non-raw SMBs */ + /* maxBuf is returned by SMB NegotiateProtocol so maxBuf is only 0 */ + /* when socket is setup (and during reconnect) before NegProt sent */ + unsigned int max_rw; /* maxRw specifies the maximum */ + /* message size the server can send or receive for */ + /* SMB_COM_WRITE_RAW or SMB_COM_READ_RAW. */ + unsigned int capabilities; /* selective disabling of caps by smb sess */ + int timeAdj; /* Adjust for difference in server time zone in sec */ + __u64 current_mid; /* multiplex id - rotating counter, protected by mid_counter_lock */ + char cryptkey[CIFS_CRYPTO_KEY_SIZE]; /* used by ntlm, ntlmv2 etc */ + /* 16th byte of RFC1001 workstation name is always null */ + char workstation_RFC1001_name[RFC1001_NAME_LEN_WITH_NULL]; + __u32 sequence_number; /* for signing, protected by srv_mutex */ + __u32 reconnect_instance; /* incremented on each reconnect */ + __le32 session_key_id; /* retrieved from negotiate response and send in session setup request */ + struct session_key session_key; + unsigned long lstrp; /* when we got last response from this server */ + unsigned long neg_start; /* when negotiate started (jiffies) */ + unsigned long reconn_delay; /* when resched session and tcon reconnect */ + struct cifs_secmech secmech; /* crypto sec mech functs, descriptors */ +#define CIFS_NEGFLAVOR_UNENCAP 1 /* wct == 17, but no ext_sec */ +#define CIFS_NEGFLAVOR_EXTENDED 2 /* wct == 17, ext_sec bit set */ + char negflavor; /* NEGOTIATE response flavor */ + /* extended security flavors that server supports */ + bool sec_ntlmssp; /* supports NTLMSSP */ + bool sec_kerberosu2u; /* supports U2U Kerberos */ + bool sec_kerberos; /* supports plain Kerberos */ + bool sec_mskerberos; /* supports legacy MS Kerberos */ + bool sec_iakerb; /* supports pass-through auth for Kerberos (krb5 proxy) */ + bool large_buf; /* is current buffer large? */ + /* use SMBD connection instead of socket */ + bool rdma; + /* point to the SMBD connection if RDMA is used instead of socket */ + struct smbd_connection *smbd_conn; + struct delayed_work echo; /* echo ping workqueue job */ + char *smallbuf; /* pointer to current "small" buffer */ + char *bigbuf; /* pointer to current "big" buffer */ + /* Total size of this PDU. Only valid from cifs_demultiplex_thread */ + unsigned int pdu_size; + unsigned int total_read; /* total amount of data read in this pass */ + atomic_t in_send; /* requests trying to send */ + atomic_t num_waiters; /* blocked waiting to get in sendrecv */ +#ifdef CONFIG_CIFS_STATS2 + atomic_t num_cmds[NUMBER_OF_SMB2_COMMANDS]; /* total requests by cmd */ + atomic_t smb2slowcmd[NUMBER_OF_SMB2_COMMANDS]; /* count resps > 1 sec */ + __u64 time_per_cmd[NUMBER_OF_SMB2_COMMANDS]; /* total time per cmd */ + __u32 slowest_cmd[NUMBER_OF_SMB2_COMMANDS]; + __u32 fastest_cmd[NUMBER_OF_SMB2_COMMANDS]; +#endif /* STATS2 */ + unsigned int max_read; + unsigned int max_write; + unsigned int min_offload; + /* + * If payload is less than or equal to the threshold, + * use RDMA send/recv to send upper layer I/O. + * If payload is more than the threshold, + * use RDMA read/write through memory registration for I/O. + */ + unsigned int rdma_readwrite_threshold; + unsigned int retrans; + struct { + bool requested; /* "compress" mount option set*/ + bool enabled; /* actually negotiated with server */ + bool chained; /* chained transforms were negotiated */ + bool pattern; /* Pattern_V1 chained payloads were negotiated */ + __le16 alg; /* preferred alg negotiated with server */ + } compression; + __u16 signing_algorithm; + __le16 cipher_type; + /* save initial negprot hash */ + __u8 preauth_sha_hash[SMB2_PREAUTH_HASH_SIZE]; + bool signing_negotiated; /* true if valid signing context rcvd from server */ + bool posix_ext_supported; + struct delayed_work reconnect; /* reconnect workqueue job */ + struct mutex reconnect_mutex; /* prevent simultaneous reconnects */ + unsigned long echo_interval; + + /* + * Number of targets available for reconnect. The more targets + * the more tasks have to wait to let the demultiplex thread + * reconnect. + */ + int nr_targets; + bool noblockcnt; /* use non-blocking connect() */ + + /* + * If this is a session channel, + * primary_server holds the ref-counted + * pointer to primary channel connection for the session. + */ +#define SERVER_IS_CHAN(server) (!!(server)->primary_server) + struct TCP_Server_Info *primary_server; + __u16 channel_sequence_num; /* incremented on primary channel on each chan reconnect */ + +#ifdef CONFIG_CIFS_SWN_UPCALL + bool use_swn_dstaddr; + struct sockaddr_storage swn_dstaddr; +#endif + /* + * Canonical DFS referral path used in cifs_reconnect() for failover as + * well as in DFS cache refresher. + * + * format: \\HOST\SHARE[\OPTIONAL PATH] + */ + char *leaf_fullpath; + bool dfs_conn:1; + char dns_dom[CIFS_MAX_DOMAINNAME_LEN + 1]; +}; + +static inline bool is_smb1(const struct TCP_Server_Info *server) +{ + return server->vals->protocol_id == SMB10_PROT_ID; +} + +static inline void cifs_server_lock(struct TCP_Server_Info *server) +{ + unsigned int nofs_flag = memalloc_nofs_save(); + + mutex_lock(&server->_srv_mutex); + server->nofs_flag = nofs_flag; +} + +static inline void cifs_server_unlock(struct TCP_Server_Info *server) +{ + unsigned int nofs_flag = server->nofs_flag; + + mutex_unlock(&server->_srv_mutex); + memalloc_nofs_restore(nofs_flag); +} + +struct cifs_credits { + unsigned int value; + unsigned int instance; + unsigned int in_flight_check; + unsigned int rreq_debug_id; + unsigned int rreq_debug_index; +}; + +static inline unsigned int +in_flight(struct TCP_Server_Info *server) +{ + unsigned int num; + + spin_lock(&server->req_lock); + num = server->in_flight; + spin_unlock(&server->req_lock); + return num; +} + +static inline bool +has_credits(struct TCP_Server_Info *server, int *credits, int num_credits) +{ + int num; + + spin_lock(&server->req_lock); + num = *credits; + spin_unlock(&server->req_lock); + return num >= num_credits; +} + +static inline void +add_credits(struct TCP_Server_Info *server, struct cifs_credits *credits, + const int optype) +{ + server->ops->add_credits(server, credits, optype); +} + +static inline void +add_credits_and_wake_if(struct TCP_Server_Info *server, + struct cifs_credits *credits, const int optype) +{ + if (credits->value) { + server->ops->add_credits(server, credits, optype); + wake_up(&server->request_q); + credits->value = 0; + } +} + +static inline void +set_credits(struct TCP_Server_Info *server, const int val) +{ + server->ops->set_credits(server, val); +} + +static inline int +adjust_credits(struct TCP_Server_Info *server, struct cifs_io_subrequest *subreq, + unsigned int /* enum smb3_rw_credits_trace */ trace) +{ + return server->ops->adjust_credits ? + server->ops->adjust_credits(server, subreq, trace) : 0; +} + +static inline __le64 +get_next_mid64(struct TCP_Server_Info *server) +{ + return cpu_to_le64(server->ops->get_next_mid(server)); +} + +static inline __le16 +get_next_mid(struct TCP_Server_Info *server) +{ + __u16 mid = server->ops->get_next_mid(server); + /* + * The value in the SMB header should be little endian for easy + * on-the-wire decoding. + */ + return cpu_to_le16(mid); +} + +static inline void +revert_current_mid(struct TCP_Server_Info *server, const unsigned int val) +{ + if (server->ops->revert_current_mid) + server->ops->revert_current_mid(server, val); +} + +static inline void +revert_current_mid_from_hdr(struct TCP_Server_Info *server, + const struct smb2_hdr *shdr) +{ + unsigned int num = le16_to_cpu(shdr->CreditCharge); + + return revert_current_mid(server, num > 0 ? num : 1); +} + +/* + * When the server supports very large reads and writes via POSIX extensions, + * we can allow up to 2^24-1, minus the size of a READ/WRITE_AND_X header, not + * including the RFC1001 length. + * + * Note that this might make for "interesting" allocation problems during + * writeback however as we have to allocate an array of pointers for the + * pages. A 16M write means ~32kb page array with PAGE_SIZE == 4096. + * + * For reads, there is a similar problem as we need to allocate an array + * of kvecs to handle the receive, though that should only need to be done + * once. + */ +#define CIFS_MAX_WSIZE ((1<<24) - 1 - sizeof(WRITE_REQ)) +#define CIFS_MAX_RSIZE ((1<<24) - sizeof(READ_RSP)) + +/* + * When the server doesn't allow large posix writes, only allow a rsize/wsize + * of 2^17-1 minus the size of the call header. That allows for a read or + * write up to the maximum size described by RFC1002. + */ +#define CIFS_MAX_RFC1002_WSIZE ((1<<17) - 1 - sizeof(WRITE_REQ)) +#define CIFS_MAX_RFC1002_RSIZE ((1<<17) - 1 - sizeof(READ_RSP)) + +/* + * Windows only supports a max of 60kb reads and 65535 byte writes. Default to + * those values when posix extensions aren't in force. In actuality here, we + * use 65536 to allow for a write that is a multiple of 4k. Most servers seem + * to be ok with the extra byte even though Windows doesn't send writes that + * are that large. + * + * Citation: + * + * https://blogs.msdn.com/b/openspecification/archive/2009/04/10/smb-maximum-transmit-buffer-size-and-performance-tuning.aspx + */ +#define CIFS_DEFAULT_NON_POSIX_RSIZE (60 * 1024) +#define CIFS_DEFAULT_NON_POSIX_WSIZE (65536) + +/* + * Macros to allow the TCP_Server_Info->net field and related code to drop out + * when CONFIG_NET_NS isn't set. + */ + +#ifdef CONFIG_NET_NS + +static inline struct net *cifs_net_ns(struct TCP_Server_Info *srv) +{ + return srv->net; +} + +static inline void cifs_set_net_ns(struct TCP_Server_Info *srv, struct net *net) +{ + srv->net = net; +} + +#else + +static inline struct net *cifs_net_ns(struct TCP_Server_Info *srv) +{ + return &init_net; +} + +static inline void cifs_set_net_ns(struct TCP_Server_Info *srv, struct net *net) +{ +} + +#endif + +struct cifs_server_iface { + struct list_head iface_head; + struct kref refcount; + size_t speed; + size_t weight_fulfilled; + unsigned int num_channels; + unsigned int rdma_capable : 1; + unsigned int rss_capable : 1; + unsigned int is_active : 1; /* unset if non existent */ + struct sockaddr_storage sockaddr; +}; + +/* release iface when last ref is dropped */ +static inline void +release_iface(struct kref *ref) +{ + struct cifs_server_iface *iface = container_of(ref, + struct cifs_server_iface, + refcount); + kfree(iface); +} + +struct cifs_chan { + unsigned int in_reconnect : 1; /* if session setup in progress for this channel */ + struct TCP_Server_Info *server; + struct cifs_server_iface *iface; /* interface in use */ + __u8 signkey[SMB3_SIGN_KEY_SIZE]; +}; + +#define CIFS_SES_FLAG_SCALE_CHANNELS (0x1) +#define CIFS_SES_FLAGS_PENDING_QUERY_INTERFACES (0x2) + +/* + * Session structure. One of these for each uid session with a particular host + */ +struct cifs_ses { + struct list_head smb_ses_list; + struct list_head rlist; /* reconnect list */ + struct list_head tcon_list; + struct list_head dlist; /* dfs list */ + struct cifs_tcon *tcon_ipc; + spinlock_t ses_lock; /* protect anything here that is not protected */ + struct mutex session_mutex; + struct TCP_Server_Info *server; /* pointer to server info */ + int ses_count; /* reference counter */ + enum ses_status_enum ses_status; /* updates protected by cifs_tcp_ses_lock */ + unsigned int overrideSecFlg; /* if non-zero override global sec flags */ + char *serverOS; /* name of operating system underlying server */ + char *serverNOS; /* name of network operating system of server */ + char *serverDomain; /* security realm of server */ + __u64 Suid; /* remote smb uid */ + kuid_t linux_uid; /* overriding owner of files on the mount */ + kuid_t cred_uid; /* owner of credentials */ + unsigned int capabilities; + char ip_addr[INET6_ADDRSTRLEN + 1]; /* Max ipv6 (or v4) addr string len */ + char *user_name; /* must not be null except during init of sess + and after mount option parsing we fill it */ + char *domainName; + char *password; + char *password2; /* When key rotation used, new password may be set before it expires */ + char workstation_name[CIFS_MAX_WORKSTATION_LEN]; + struct session_key auth_key; + struct ntlmssp_auth *ntlmssp; /* ciphertext, flags, server challenge */ + enum securityEnum sectype; /* what security flavor was specified? */ + enum upcall_target_enum upcall_target; /* what upcall target was specified? */ + bool sign; /* is signing required? */ + bool domainAuto:1; + bool expired_pwd; /* track if access denied or expired pwd so can know if need to update */ + int unicode; + unsigned int flags; + __u16 session_flags; + __u8 smb3signingkey[SMB3_SIGN_KEY_SIZE]; + __u8 smb3encryptionkey[SMB3_ENC_DEC_KEY_SIZE]; + __u8 smb3decryptionkey[SMB3_ENC_DEC_KEY_SIZE]; + __u8 preauth_sha_hash[SMB2_PREAUTH_HASH_SIZE]; + + /* + * Network interfaces available on the server this session is + * connected to. + * + * Other channels can be opened by connecting and binding this + * session to interfaces from this list. + * + * iface_lock should be taken when accessing any of these fields + */ + spinlock_t iface_lock; + /* ========= begin: protected by iface_lock ======== */ + struct list_head iface_list; + size_t iface_count; + unsigned long iface_last_update; /* jiffies */ + /* ========= end: protected by iface_lock ======== */ + + spinlock_t chan_lock; + /* ========= begin: protected by chan_lock ======== */ +#define CIFS_MAX_CHANNELS 16 +#define CIFS_INVAL_CHAN_INDEX (-1) +#define CIFS_ALL_CHANNELS_SET(ses) \ + ((1UL << (ses)->chan_count) - 1) +#define CIFS_ALL_CHANS_GOOD(ses) \ + (!(ses)->chans_need_reconnect) +#define CIFS_ALL_CHANS_NEED_RECONNECT(ses) \ + ((ses)->chans_need_reconnect == CIFS_ALL_CHANNELS_SET(ses)) +#define CIFS_SET_ALL_CHANS_NEED_RECONNECT(ses) \ + ((ses)->chans_need_reconnect = CIFS_ALL_CHANNELS_SET(ses)) +#define CIFS_CHAN_NEEDS_RECONNECT(ses, index) \ + test_bit((index), &(ses)->chans_need_reconnect) +#define CIFS_CHAN_IN_RECONNECT(ses, index) \ + ((ses)->chans[(index)].in_reconnect) + + struct cifs_chan chans[CIFS_MAX_CHANNELS]; + size_t chan_count; + size_t chan_max; + atomic_t chan_seq; /* round robin state */ + + /* + * chans_need_reconnect is a bitmap indicating which of the channels + * under this smb session needs to be reconnected. + * If not multichannel session, only one bit will be used. + * + * We will ask for sess and tcon reconnection only if all the + * channels are marked for needing reconnection. This will + * enable the sessions on top to continue to live till any + * of the channels below are active. + */ + unsigned long chans_need_reconnect; + /* ========= end: protected by chan_lock ======== */ + struct cifs_ses *dfs_root_ses; + struct nls_table *local_nls; + char *dns_dom; /* FQDN of the domain */ +}; + +static inline bool +cap_unix(struct cifs_ses *ses) +{ + return ses->server->vals->cap_unix & ses->capabilities; +} + +/* + * common struct for holding inode info when searching for or updating an + * inode with new info + */ + +#define CIFS_FATTR_JUNCTION 0x1 +#define CIFS_FATTR_DELETE_PENDING 0x2 +#define CIFS_FATTR_NEED_REVAL 0x4 +#define CIFS_FATTR_INO_COLLISION 0x8 +#define CIFS_FATTR_UNKNOWN_NLINK 0x10 +#define CIFS_FATTR_FAKE_ROOT_INO 0x20 + +struct cifs_fattr { + u32 cf_flags; + u32 cf_cifsattrs; + u64 cf_uniqueid; + u64 cf_eof; + u64 cf_bytes; + u64 cf_createtime; + kuid_t cf_uid; + kgid_t cf_gid; + umode_t cf_mode; + dev_t cf_rdev; + unsigned int cf_nlink; + unsigned int cf_dtype; + struct timespec64 cf_atime; + struct timespec64 cf_mtime; + struct timespec64 cf_ctime; + u32 cf_cifstag; + char *cf_symlink_target; +}; + +/* + * there is one of these for each connection to a resource on a particular + * session + */ +struct cifs_tcon { + struct list_head tcon_list; + int debug_id; /* Debugging for tracing */ + int tc_count; + struct list_head rlist; /* reconnect list */ + spinlock_t tc_lock; /* protect anything here that is not protected */ + atomic_t num_local_opens; /* num of all opens including disconnected */ + atomic_t num_remote_opens; /* num of all network opens on server */ + struct list_head openFileList; + spinlock_t open_file_lock; /* protects list above */ + struct cifs_ses *ses; /* pointer to session associated with */ + char tree_name[MAX_TREE_SIZE + 1]; /* UNC name of resource in ASCII */ + char *nativeFileSystem; + char *password; /* for share-level security */ + __u32 tid; /* The 4 byte tree id */ + __u16 Flags; /* optional support bits */ + enum tid_status_enum status; + atomic_t num_smbs_sent; + union { + struct { + atomic_t num_writes; + atomic_t num_reads; + atomic_t num_flushes; + atomic_t num_oplock_brks; + atomic_t num_opens; + atomic_t num_closes; + atomic_t num_deletes; + atomic_t num_mkdirs; + atomic_t num_posixopens; + atomic_t num_posixmkdirs; + atomic_t num_rmdirs; + atomic_t num_renames; + atomic_t num_t2renames; + atomic_t num_ffirst; + atomic_t num_fnext; + atomic_t num_fclose; + atomic_t num_hardlinks; + atomic_t num_symlinks; + atomic_t num_locks; + atomic_t num_acl_get; + atomic_t num_acl_set; + } cifs_stats; + struct { + atomic_t smb2_com_sent[NUMBER_OF_SMB2_COMMANDS]; + atomic_t smb2_com_failed[NUMBER_OF_SMB2_COMMANDS]; + } smb2_stats; + } stats; + __u64 bytes_read; + __u64 bytes_written; + spinlock_t stat_lock; /* protects the two fields above */ + time64_t stats_from_time; + FILE_SYSTEM_DEVICE_INFO fsDevInfo; + FILE_SYSTEM_ATTRIBUTE_INFO fsAttrInfo; /* ok if fs name truncated */ + FILE_SYSTEM_UNIX_INFO fsUnixInfo; + bool ipc:1; /* set if connection to IPC$ share (always also pipe) */ + bool pipe:1; /* set if connection to pipe share */ + bool print:1; /* set if connection to printer share */ + bool retry:1; + bool nocase:1; + bool nohandlecache:1; /* if strange server resource prob can turn off */ + bool nodelete:1; + bool seal:1; /* transport encryption for this mounted share */ + bool unix_ext:1; /* if false disable Linux extensions to CIFS protocol + for this mount even if server would support */ + bool posix_extensions; /* if true SMB3.11 posix extensions enabled */ + bool local_lease:1; /* check leases (only) on local system not remote */ + bool broken_posix_open; /* e.g. Samba server versions < 3.3.2, 3.2.9 */ + bool broken_sparse_sup; /* if server or share does not support sparse */ + bool need_reconnect:1; /* connection reset, tid now invalid */ + bool need_reopen_files:1; /* need to reopen tcon file handles */ + bool use_resilient:1; /* use resilient instead of durable handles */ + bool use_persistent:1; /* use persistent instead of durable handles */ + bool no_lease:1; /* Do not request leases on files or directories */ + bool use_witness:1; /* use witness protocol */ + bool dummy:1; /* dummy tcon used for reconnecting channels */ + __le32 capabilities; + __u32 share_flags; + __u32 maximal_access; + __u32 vol_serial_number; + __le64 vol_create_time; + __u64 snapshot_time; /* for timewarp tokens - timestamp of snapshot */ + __u32 handle_timeout; /* persistent and durable handle timeout in ms */ + __u32 ss_flags; /* sector size flags */ + __u32 perf_sector_size; /* best sector size for perf */ + __u32 max_chunks; + __u32 max_bytes_chunk; + __u32 max_bytes_copy; + __u32 max_cached_dirs; +#ifdef CONFIG_CIFS_FSCACHE + u64 resource_id; /* server resource id */ + bool fscache_acquired; /* T if we've tried acquiring a cookie */ + struct fscache_volume *fscache; /* cookie for share */ + struct mutex fscache_lock; /* Prevent regetting a cookie */ +#endif + struct list_head pending_opens; /* list of incomplete opens */ + struct cached_fids *cfids; + struct list_head cifs_sb_list; + spinlock_t sb_list_lock; +#ifdef CONFIG_CIFS_DFS_UPCALL + struct delayed_work dfs_cache_work; + struct list_head dfs_ses_list; +#endif + struct delayed_work query_interfaces; /* query interfaces workqueue job */ + char *origin_fullpath; /* canonical copy of smb3_fs_context::source */ +}; + +/* + * This is a refcounted and timestamped container for a tcon pointer. The + * container holds a tcon reference. It is considered safe to free one of + * these when the tl_count goes to 0. The tl_time is the time of the last + * "get" on the container. + */ +struct tcon_link { + struct rb_node tl_rbnode; + kuid_t tl_uid; + unsigned long tl_flags; +#define TCON_LINK_MASTER 0 +#define TCON_LINK_PENDING 1 +#define TCON_LINK_IN_TREE 2 + unsigned long tl_time; + atomic_t tl_count; + struct cifs_tcon *tl_tcon; +}; + +struct tcon_link *cifs_sb_tlink(struct cifs_sb_info *cifs_sb); +void smb3_free_compound_rqst(int num_rqst, struct smb_rqst *rqst); + +static inline struct cifs_tcon * +tlink_tcon(struct tcon_link *tlink) +{ + return tlink->tl_tcon; +} + +static inline struct tcon_link * +cifs_sb_master_tlink(struct cifs_sb_info *cifs_sb) +{ + return cifs_sb->master_tlink; +} + +void cifs_put_tlink(struct tcon_link *tlink); + +static inline struct tcon_link * +cifs_get_tlink(struct tcon_link *tlink) +{ + if (tlink && !IS_ERR(tlink)) + atomic_inc(&tlink->tl_count); + return tlink; +} + +/* This function is always expected to succeed */ +struct cifs_tcon *cifs_sb_master_tcon(struct cifs_sb_info *cifs_sb); + +#define CIFS_OPLOCK_NO_CHANGE 0xfe + +struct cifs_pending_open { + struct list_head olist; + struct tcon_link *tlink; + __u8 lease_key[16]; + __u32 oplock; +}; + +struct cifs_deferred_close { + struct list_head dlist; + struct tcon_link *tlink; + __u16 netfid; + __u64 persistent_fid; + __u64 volatile_fid; +}; + +/* + * This info hangs off the cifsFileInfo structure, pointed to by llist. + * This is used to track byte stream locks on the file + */ +struct cifsLockInfo { + struct list_head llist; /* pointer to next cifsLockInfo */ + struct list_head blist; /* pointer to locks blocked on this */ + wait_queue_head_t block_q; + __u64 offset; + __u64 length; + __u32 pid; + __u16 type; + __u16 flags; +}; + +/* + * One of these for each open instance of a file + */ +struct cifs_search_info { + loff_t index_of_last_entry; + __u16 entries_in_buffer; + __u16 info_level; + __u32 resume_key; + char *ntwrk_buf_start; + char *srch_entries_start; + char *last_entry; + const char *presume_name; + unsigned int resume_name_len; + bool endOfSearch:1; + bool emptyDir:1; + bool unicode:1; + bool smallBuf:1; /* so we know which buf_release function to call */ + bool is_dynamic_buf:1; /* dynamically allocated buffer - can be variable size */ +}; + +#define ACL_NO_MODE ((umode_t)(-1)) +struct cifs_open_parms { + struct cifs_tcon *tcon; + struct cifs_sb_info *cifs_sb; + int disposition; + int desired_access; + int create_options; + const char *path; + struct cifs_fid *fid; + umode_t mode; + bool reconnect:1; + bool replay:1; /* indicates that this open is for a replay */ + struct kvec *ea_cctx; + __le32 lease_flags; +}; + +struct cifs_fid { + __u16 netfid; + __u64 persistent_fid; /* persist file id for smb2 */ + __u64 volatile_fid; /* volatile file id for smb2 */ + __u8 lease_key[SMB2_LEASE_KEY_SIZE]; /* lease key for smb2 */ + __u8 parent_lease_key[SMB2_LEASE_KEY_SIZE]; + __u8 create_guid[16]; + __u32 access; + struct cifs_pending_open *pending_open; + __u16 epoch; +#ifdef CONFIG_CIFS_DEBUG2 + __u64 mid; +#endif /* CIFS_DEBUG2 */ + bool purge_cache; +}; + +struct cifs_fid_locks { + struct list_head llist; + struct cifsFileInfo *cfile; /* fid that owns locks */ + struct list_head locks; /* locks held by fid above */ +}; + +struct cifsFileInfo { + /* following two lists are protected by tcon->open_file_lock */ + struct list_head tlist; /* pointer to next fid owned by tcon */ + struct list_head flist; /* next fid (file instance) for this inode */ + /* lock list below protected by cifsi->lock_sem */ + struct cifs_fid_locks *llist; /* brlocks held by this fid */ + kuid_t uid; /* allows finding which FileInfo structure */ + __u32 pid; /* process id who opened file */ + struct cifs_fid fid; /* file id from remote */ + struct list_head rlist; /* reconnect list */ + /* BB add lock scope info here if needed */ + /* lock scope id (0 if none) */ + struct dentry *dentry; + struct tcon_link *tlink; + unsigned int f_flags; + bool invalidHandle:1; /* file closed via session abend */ + bool swapfile:1; + bool oplock_break_cancelled:1; + bool status_file_deleted:1; /* file has been deleted */ + bool offload:1; /* offload final part of _put to a wq */ + __u16 oplock_epoch; /* epoch from the lease break */ + __u32 oplock_level; /* oplock/lease level from the lease break */ + int count; + spinlock_t file_info_lock; /* protects four flag/count fields above */ + struct mutex fh_mutex; /* prevents reopen race after dead ses*/ + struct cifs_search_info srch_inf; + struct work_struct oplock_break; /* work for oplock breaks */ + struct work_struct put; /* work for the final part of _put */ + struct work_struct serverclose; /* work for serverclose */ + struct delayed_work deferred; + bool deferred_close_scheduled; /* Flag to indicate close is scheduled */ + char *symlink_target; +}; + +struct cifs_io_parms { + __u16 netfid; + __u64 persistent_fid; /* persist file id for smb2 */ + __u64 volatile_fid; /* volatile file id for smb2 */ + __u32 pid; + __u64 offset; + unsigned int length; + struct cifs_tcon *tcon; + struct TCP_Server_Info *server; +}; + +struct cifs_io_request { + struct netfs_io_request rreq; + struct cifsFileInfo *cfile; + pid_t pid; +}; + +/* asynchronous read support */ +struct cifs_io_subrequest { + union { + struct netfs_io_subrequest subreq; + struct netfs_io_request *rreq; + struct cifs_io_request *req; + }; + ssize_t got_bytes; + unsigned int xid; + int result; + bool have_xid; + bool replay; + unsigned int retries; /* number of retries so far */ + unsigned int cur_sleep; /* time to sleep before replay */ + struct kvec iov[2]; + struct TCP_Server_Info *server; +#ifdef CONFIG_CIFS_SMB_DIRECT + struct smbdirect_mr_io *mr; +#endif + struct cifs_credits credits; +}; + +/* + * Take a reference on the file private data. Must be called with + * cfile->file_info_lock held. + */ +static inline void +cifsFileInfo_get_locked(struct cifsFileInfo *cifs_file) +{ + ++cifs_file->count; +} + +struct cifsFileInfo *cifsFileInfo_get(struct cifsFileInfo *cifs_file); +void _cifsFileInfo_put(struct cifsFileInfo *cifs_file, + bool wait_oplock_handler, bool offload); +void cifsFileInfo_put(struct cifsFileInfo *cifs_file); +int cifs_file_flush(const unsigned int xid, struct inode *inode, + struct cifsFileInfo *cfile); +int cifs_file_set_size(const unsigned int xid, struct dentry *dentry, + const char *full_path, struct cifsFileInfo *open_file, + loff_t size); + +#define CIFS_CACHE_READ_FLG 1 +#define CIFS_CACHE_HANDLE_FLG 2 +#define CIFS_CACHE_RH_FLG (CIFS_CACHE_READ_FLG | CIFS_CACHE_HANDLE_FLG) +#define CIFS_CACHE_WRITE_FLG 4 +#define CIFS_CACHE_RW_FLG (CIFS_CACHE_READ_FLG | CIFS_CACHE_WRITE_FLG) +#define CIFS_CACHE_RHW_FLG (CIFS_CACHE_RW_FLG | CIFS_CACHE_HANDLE_FLG) + +enum cifs_inode_flags { + CIFS_INODE_PENDING_OPLOCK_BREAK, /* oplock break in progress */ + CIFS_INODE_PENDING_WRITERS, /* Writes in progress */ + CIFS_INODE_FLAG_UNUSED, /* Unused flag */ + CIFS_INO_DELETE_PENDING, /* delete pending on server */ + CIFS_INO_INVALID_MAPPING, /* pagecache is invalid */ + CIFS_INO_LOCK, /* lock bit for synchronization */ + CIFS_INO_TMPFILE, /* for O_TMPFILE inodes */ + CIFS_INO_CLOSE_ON_LOCK, /* Not to defer the close when lock is set */ +}; + +struct cifsInodeInfo { + struct netfs_inode netfs; /* Netfslib context and vfs inode */ + bool can_cache_brlcks; + struct list_head llist; /* locks helb by this inode */ + /* + * NOTE: Some code paths call down_read(lock_sem) twice, so + * we must always use cifs_down_write() instead of down_write() + * for this semaphore to avoid deadlocks. + */ + struct rw_semaphore lock_sem; /* protect the fields above */ + /* BB add in lists for dirty pages i.e. write caching info for oplock */ + struct list_head openFileList; + spinlock_t open_file_lock; /* protects openFileList */ + __u32 cifsAttrs; /* e.g. DOS archive bit, sparse, compressed, system */ + unsigned int oplock; /* oplock/lease level we have */ + __u16 epoch; /* used to track lease state changes */ + unsigned long flags; + spinlock_t writers_lock; + unsigned int writers; /* Number of writers on this inode */ + unsigned long time; /* jiffies of last update of inode */ + unsigned long time_last_write; /* jiffies of last writable close or truncate */ + u64 uniqueid; /* server inode number */ + u64 createtime; /* creation time on server */ + __u8 lease_key[SMB2_LEASE_KEY_SIZE]; /* lease key for this inode */ + struct list_head deferred_closes; /* list of deferred closes */ + spinlock_t deferred_lock; /* protection on deferred list */ + bool lease_granted; /* Flag to indicate whether lease or oplock is granted. */ + char *symlink_target; + __u32 reparse_tag; +}; + +static inline struct cifsInodeInfo * +CIFS_I(struct inode *inode) +{ + return container_of(inode, struct cifsInodeInfo, netfs.inode); +} + +static inline void *cinode_to_fsinfo(struct cifsInodeInfo *cinode) +{ + return cinode->netfs.inode.i_sb->s_fs_info; +} + +static inline void *super_to_fsinfo(struct super_block *sb) +{ + return sb->s_fs_info; +} + +static inline void *inode_to_fsinfo(struct inode *inode) +{ + return inode->i_sb->s_fs_info; +} + +static inline void *file_to_fsinfo(struct file *file) +{ + return file_inode(file)->i_sb->s_fs_info; +} + +static inline void *dentry_to_fsinfo(struct dentry *dentry) +{ + return dentry->d_sb->s_fs_info; +} + +static inline void *const_dentry_to_fsinfo(const struct dentry *dentry) +{ + return dentry->d_sb->s_fs_info; +} + +#define CIFS_SB(_ptr) \ + ((struct cifs_sb_info *) \ + _Generic((_ptr), \ + struct cifsInodeInfo * : cinode_to_fsinfo, \ + const struct dentry * : const_dentry_to_fsinfo, \ + struct super_block * : super_to_fsinfo, \ + struct dentry * : dentry_to_fsinfo, \ + struct inode * : inode_to_fsinfo, \ + struct file * : file_to_fsinfo)(_ptr)) + +/* + * Use atomic_t for @cifs_sb->mnt_cifs_flags as it is currently accessed + * locklessly and may be changed concurrently by mount/remount and reconnect + * paths. + */ +static inline unsigned int cifs_sb_flags(const struct cifs_sb_info *cifs_sb) +{ + return atomic_read(&cifs_sb->mnt_cifs_flags); +} + +static inline char CIFS_DIR_SEP(const struct cifs_sb_info *cifs_sb) +{ + return (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_POSIX_PATHS) ? '/' : '\\'; +} + +static inline void +convert_delimiter(char *path, char delim) +{ + char old_delim, *pos; + + if (delim == '/') + old_delim = '\\'; + else + old_delim = '/'; + + pos = path; + while ((pos = strchr(pos, old_delim))) + *pos = delim; +} + +#define cifs_stats_inc atomic_inc + +static inline void cifs_stats_bytes_written(struct cifs_tcon *tcon, + unsigned int bytes) +{ + if (bytes) { + spin_lock(&tcon->stat_lock); + tcon->bytes_written += bytes; + spin_unlock(&tcon->stat_lock); + } +} + +static inline void cifs_stats_bytes_read(struct cifs_tcon *tcon, + unsigned int bytes) +{ + spin_lock(&tcon->stat_lock); + tcon->bytes_read += bytes; + spin_unlock(&tcon->stat_lock); +} + + +/* + * This is the prototype for the mid receive function. This function is for + * receiving the rest of the SMB frame, starting with the WordCount (which is + * just after the MID in struct smb_hdr). Note: + * + * - This will be called by cifsd, with no locks held. + * - The mid will still be on the pending_mid_q. + * - mid->resp_buf will point to the current buffer. + * + * Returns zero on a successful receive, or an error. The receive state in + * the TCP_Server_Info will also be updated. + */ +typedef int (*mid_receive_t)(struct TCP_Server_Info *server, + struct mid_q_entry *mid); + +/* + * This is the prototype for the mid callback function. This is called once the + * mid has been received off of the socket. When creating one, take special + * care to avoid deadlocks. Things to bear in mind: + * + * - it will be called by cifsd, with no locks held + * - the mid will be removed from any lists + */ +typedef void (*mid_callback_t)(struct TCP_Server_Info *srv, struct mid_q_entry *mid); + +/* + * This is the protopyte for mid handle function. This is called once the mid + * has been recognized after decryption of the message. + */ +typedef int (*mid_handle_t)(struct TCP_Server_Info *server, + struct mid_q_entry *mid); + +/* one of these for every pending CIFS request to the server */ +struct mid_q_entry { + struct list_head qhead; /* mids waiting on reply from this server */ + refcount_t refcount; + __u64 mid; /* multiplex id */ + __u16 credits; /* number of credits consumed by this mid */ + __u16 credits_received; /* number of credits from the response */ + __u32 pid; /* process id */ + __u32 sequence_number; /* for CIFS signing */ + unsigned int sr_flags; /* Flags passed to send_recv() */ + unsigned long when_alloc; /* when mid was created */ +#ifdef CONFIG_CIFS_STATS2 + unsigned long when_sent; /* time when smb send finished */ + unsigned long when_received; /* when demux complete (taken off wire) */ +#endif + mid_receive_t receive; /* call receive callback */ + mid_callback_t callback; /* call completion callback */ + mid_handle_t handle; /* call handle mid callback */ + void *callback_data; /* general purpose pointer for callback */ + struct task_struct *creator; + void *resp_buf; /* pointer to received SMB header */ + unsigned int resp_buf_size; + u32 response_pdu_len; + int mid_state; /* wish this were enum but can not pass to wait_event */ + int mid_rc; /* rc for MID_RC */ + __le16 command; /* smb command code */ + unsigned int optype; /* operation type */ + spinlock_t mid_lock; + bool wait_cancelled:1; /* Cancelled while waiting for response */ + bool deleted_from_q:1; /* Whether Mid has been dequeued frem pending_mid_q */ + bool large_buf:1; /* if valid response, is pointer to large buf */ + bool multiRsp:1; /* multiple trans2 responses for one request */ + bool multiEnd:1; /* both received */ + bool decrypted:1; /* decrypted entry */ +}; + +struct close_cancelled_open { + struct cifs_fid fid; + struct cifs_tcon *tcon; + struct work_struct work; + __u64 mid; + __u16 cmd; +}; + +/* Make code in transport.c a little cleaner by moving + update of optional stats into function below */ +static inline void cifs_in_send_inc(struct TCP_Server_Info *server) +{ + atomic_inc(&server->in_send); +} + +static inline void cifs_in_send_dec(struct TCP_Server_Info *server) +{ + atomic_dec(&server->in_send); +} + +static inline void cifs_num_waiters_inc(struct TCP_Server_Info *server) +{ + atomic_inc(&server->num_waiters); +} + +static inline void cifs_num_waiters_dec(struct TCP_Server_Info *server) +{ + atomic_dec(&server->num_waiters); +} + +#ifdef CONFIG_CIFS_STATS2 +static inline void cifs_save_when_sent(struct mid_q_entry *mid) +{ + mid->when_sent = jiffies; +} +#else +static inline void cifs_save_when_sent(struct mid_q_entry *mid) +{ +} +#endif + +/* for pending dnotify requests */ +struct dir_notify_req { + struct list_head lhead; + __le16 Pid; + __le16 PidHigh; + __u16 Mid; + __u16 Tid; + __u16 Uid; + __u16 netfid; + __u32 filter; /* CompletionFilter (for multishot) */ + int multishot; + struct file *pfile; +}; + +struct dfs_info3_param { + int flags; /* DFSREF_REFERRAL_SERVER, DFSREF_STORAGE_SERVER*/ + int path_consumed; + int server_type; + int ref_flag; + char *path_name; + char *node_name; + int ttl; +}; + +struct file_list { + struct list_head list; + struct cifsFileInfo *cfile; +}; + +struct cifs_mount_ctx { + struct cifs_sb_info *cifs_sb; + struct smb3_fs_context *fs_ctx; + unsigned int xid; + struct TCP_Server_Info *server; + struct cifs_ses *ses; + struct cifs_tcon *tcon; +}; + +struct mchan_mount { + struct work_struct work; + struct cifs_ses *ses; +}; + +static inline void __free_dfs_info_param(struct dfs_info3_param *param) +{ + kfree(param->path_name); + kfree(param->node_name); +} + +static inline void free_dfs_info_param(struct dfs_info3_param *param) +{ + if (param) + __free_dfs_info_param(param); +} + +static inline void zfree_dfs_info_param(struct dfs_info3_param *param) +{ + if (param) { + __free_dfs_info_param(param); + memset(param, 0, sizeof(*param)); + } +} + +static inline void free_dfs_info_array(struct dfs_info3_param *param, + int number_of_items) +{ + int i; + + if ((number_of_items == 0) || (param == NULL)) + return; + for (i = 0; i < number_of_items; i++) { + kfree(param[i].path_name); + kfree(param[i].node_name); + } + kfree(param); +} + +static inline bool is_interrupt_error(int error) +{ + switch (error) { + case -EINTR: + case -ERESTARTSYS: + case -ERESTARTNOHAND: + case -ERESTARTNOINTR: + return true; + } + return false; +} + +static inline bool is_retryable_error(int error) +{ + if (is_interrupt_error(error) || error == -EAGAIN) + return true; + return false; +} + +static inline bool is_replayable_error(int error) +{ + if (error == -EAGAIN || error == -ECONNABORTED) + return true; + return false; +} + + +enum cifs_find_flags { + FIND_ANY = 0U, + FIND_FSUID_ONLY = (1U << 0), + FIND_WITH_DELETE = (1U << 1), + FIND_NO_PENDING_DELETE = (1U << 2), + FIND_OPEN_FLAGS = (1U << 3), +}; + +#define MID_FREE 0 +#define MID_REQUEST_ALLOCATED 1 +#define MID_REQUEST_SUBMITTED 2 +#define MID_RESPONSE_RECEIVED 4 +#define MID_RETRY_NEEDED 8 /* session closed while this request out */ +#define MID_RESPONSE_MALFORMED 0x10 +#define MID_SHUTDOWN 0x20 +#define MID_RESPONSE_READY 0x40 /* ready for other process handle the rsp */ +#define MID_RC 0x80 /* mid_rc contains custom rc */ + +/* Types of response buffer returned from SendReceive2 */ +#define CIFS_NO_BUFFER 0 /* Response buffer not returned */ +#define CIFS_SMALL_BUFFER 1 +#define CIFS_LARGE_BUFFER 2 +#define CIFS_DYNAMIC_BUFFER 3 /* Dynamically allocated buffer */ +#define CIFS_IOVEC 4 /* array of response buffers */ + +/* Type of Request to SendReceive2 */ +#define CIFS_BLOCKING_OP 1 /* operation can block */ +#define CIFS_NON_BLOCKING 2 /* do not block waiting for credits */ +#define CIFS_TIMEOUT_MASK 0x003 /* only one of above set in req */ +#define CIFS_LOG_ERROR 0x010 /* log NT STATUS if non-zero */ +#define CIFS_LARGE_BUF_OP 0x020 /* large request buffer */ +#define CIFS_NO_RSP_BUF 0x040 /* no response buffer required */ + +/* Type of request operation */ +#define CIFS_ECHO_OP 0x080 /* echo request */ +#define CIFS_OBREAK_OP 0x0100 /* oplock break request */ +#define CIFS_NEG_OP 0x0200 /* negotiate request */ +#define CIFS_CP_CREATE_CLOSE_OP 0x0400 /* compound create+close request */ +/* Lower bitmask values are reserved by others below. */ +#define CIFS_SESS_OP 0x2000 /* session setup request */ +#define CIFS_OP_MASK 0x2780 /* mask request type */ + +#define CIFS_HAS_CREDITS 0x0400 /* already has credits */ +#define CIFS_TRANSFORM_REQ 0x0800 /* transform request before sending */ +#define CIFS_NO_SRV_RSP 0x1000 /* there is no server response */ +#define CIFS_COMPRESS_REQ 0x4000 /* compress request before sending */ +#define CIFS_INTERRUPTIBLE_WAIT 0x8000 /* Interruptible wait (e.g. lock request) */ +#define CIFS_WINDOWS_LOCK 0x10000 /* We're trying to get a Windows lock */ + +/* Security Flags: indicate type of session setup needed */ +#define CIFSSEC_MAY_SIGN 0x00001 +#define CIFSSEC_MAY_NTLMV2 0x00004 +#define CIFSSEC_MAY_KRB5 0x00008 +#define CIFSSEC_MAY_SEAL 0x00040 +#define CIFSSEC_MAY_NTLMSSP 0x00080 /* raw ntlmssp with ntlmv2 */ + +#define CIFSSEC_MUST_SIGN 0x01001 +/* note that only one of the following can be set so the +result of setting MUST flags more than once will be to +require use of the stronger protocol */ +#define CIFSSEC_MUST_NTLMV2 0x04004 +#define CIFSSEC_MUST_KRB5 0x08008 +#ifdef CONFIG_CIFS_UPCALL +#define CIFSSEC_MASK 0xCF0CF /* flags supported if no weak allowed */ +#else +#define CIFSSEC_MASK 0xC70C7 /* flags supported if no weak allowed */ +#endif /* UPCALL */ +#define CIFSSEC_MUST_SEAL 0x40040 +#define CIFSSEC_MUST_NTLMSSP 0x80080 /* raw ntlmssp with ntlmv2 */ + +#define CIFSSEC_DEF (CIFSSEC_MAY_SIGN | CIFSSEC_MAY_NTLMV2 | CIFSSEC_MAY_NTLMSSP | CIFSSEC_MAY_SEAL) +#define CIFSSEC_MAX (CIFSSEC_MAY_SIGN | CIFSSEC_MUST_KRB5 | CIFSSEC_MAY_SEAL) +#define CIFSSEC_AUTH_MASK (CIFSSEC_MAY_NTLMV2 | CIFSSEC_MAY_KRB5 | CIFSSEC_MAY_NTLMSSP) +/* + ***************************************************************** + * All constants go here + ***************************************************************** + */ + +#define UID_HASH (16) + +/* + * Note that ONE module should define _DECLARE_GLOBALS_HERE to cause the + * following to be declared. + */ + +/**************************************************************************** + * LOCK ORDERING NOTES: + **************************************************************************** + * Here are all the locks (spinlock, mutex, semaphore) in cifs.ko, arranged according + * to the locking order. i.e. if two locks are to be held together, the lock that + * appears higher in this list needs to be taken before the other. + * + * If you hold a lock that is lower in this list, and you need to take a higher lock + * (or if you think that one of the functions that you're calling may need to), first + * drop the lock you hold, pick up the higher lock, then the lower one. This will + * ensure that locks are picked up only in one direction in the below table + * (top to bottom). + * + * Also, if you expect a function to be called with a lock held, explicitly document + * this in the comments on top of your function definition. + * + * And also, try to keep the critical sections (lock hold time) to be as minimal as + * possible. Blocking / calling other functions with a lock held always increase + * the risk of a possible deadlock. + * + * Following this rule will avoid unnecessary deadlocks, which can get really hard to + * debug. Also, any new lock that you introduce, please add to this list in the correct + * order. + * + * Please populate this list whenever you introduce new locks in your changes. Or in + * case I've missed some existing locks. Please ensure that it's added in the list + * based on the locking order expected. + * + * ===================================================================================== + * Lock Protects Initialization fn + * ===================================================================================== + * cifs_mount_mutex mount/unmount operations + * vol_list_lock + * vol_info->ctx_lock vol_info->ctx + * cifs_sb_info->tlink_tree_lock cifs_sb_info->tlink_tree cifs_setup_cifs_sb + * TCP_Server_Info-> TCP_Server_Info cifs_get_tcp_session + * reconnect_mutex + * cifs_ses->session_mutex cifs_ses sesInfoAlloc + * TCP_Server_Info->srv_mutex TCP_Server_Info cifs_get_tcp_session + * cifs_tcp_ses_lock cifs_tcp_ses_list sesInfoAlloc + * cifs_tcon->open_file_lock cifs_tcon->openFileList tconInfoAlloc + * cifs_tcon->pending_opens + * cifs_tcon->stat_lock cifs_tcon->bytes_read tconInfoAlloc + * cifs_tcon->bytes_written + * cifs_tcon->fscache_lock cifs_tcon->fscache tconInfoAlloc + * cifs_tcon->sb_list_lock cifs_tcon->cifs_sb_list tconInfoAlloc + * GlobalMid_Lock GlobalMaxActiveXid init_cifs + * GlobalCurrentXid + * GlobalTotalActiveXid + * TCP_Server_Info->srv_lock (anything in struct not protected by another lock and can change) + * TCP_Server_Info->mid_queue_lock TCP_Server_Info->pending_mid_q cifs_get_tcp_session + * mid_q_entry->deleted_from_q + * TCP_Server_Info->mid_counter_lock TCP_Server_Info->current_mid cifs_get_tcp_session + * TCP_Server_Info->req_lock TCP_Server_Info->in_flight cifs_get_tcp_session + * ->credits + * ->echo_credits + * ->oplock_credits + * ->reconnect_instance + * cifs_ses->ses_lock (anything that is not protected by another lock and can change) + * sesInfoAlloc + * cifs_ses->iface_lock cifs_ses->iface_list sesInfoAlloc + * ->iface_count + * ->iface_last_update + * cifs_ses->chan_lock cifs_ses->chans sesInfoAlloc + * ->chans_need_reconnect + * ->chans_in_reconnect + * cifs_tcon->tc_lock (anything that is not protected by another lock and can change) + * tcon_info_alloc + * cifs_swnreg_idr_mutex cifs_swnreg_idr cifs_swn.c + * (witness service registration, accesses tcon fields under tc_lock) + * inode->i_rwsem, taken by fs/netfs/locking.c e.g. should be taken before cifsInodeInfo locks + * cifsInodeInfo->open_file_lock cifsInodeInfo->openFileList cifs_alloc_inode + * cifsInodeInfo->writers_lock cifsInodeInfo->writers cifsInodeInfo_alloc + * cifsInodeInfo->lock_sem cifsInodeInfo->llist cifs_init_once + * ->can_cache_brlcks + * cifsInodeInfo->deferred_lock cifsInodeInfo->deferred_closes cifsInodeInfo_alloc + * cached_fids->cfid_list_lock cifs_tcon->cfids->entries init_cached_dirs + * cached_fid->dirents.de_mutex cached_fid->dirents alloc_cached_dir + * cifsFileInfo->fh_mutex cifsFileInfo cifs_new_fileinfo + * cifsFileInfo->file_info_lock cifsFileInfo->count cifs_new_fileinfo + * ->invalidHandle initiate_cifs_search + * ->oplock_break_cancelled + * smbdirect_mr->mutex RDMA memory region management (SMBDirect only) + * mid_q_entry->mid_lock mid_q_entry->callback alloc_mid + * smb2_mid_entry_alloc + * (Any fields of mid_q_entry that will need protection) + ****************************************************************************/ + +#ifdef DECLARE_GLOBALS_HERE +#define GLOBAL_EXTERN +#else +#define GLOBAL_EXTERN extern +#endif + +/* + * the list of TCP_Server_Info structures, ie each of the sockets + * connecting our client to a distinct server (ip address), is + * chained together by cifs_tcp_ses_list. The list of all our SMB + * sessions (and from that the tree connections) can be found + * by iterating over cifs_tcp_ses_list + */ +extern struct list_head cifs_tcp_ses_list; + +/* + * This lock protects the cifs_tcp_ses_list, the list of smb sessions per + * tcp session, and the list of tcon's per smb session. It also protects + * the reference counters for the server, smb session, and tcon. + * generally the locks should be taken in order tcp_ses_lock before + * tcon->open_file_lock and that before file->file_info_lock since the + * structure order is cifs_socket-->cifs_ses-->cifs_tcon-->cifs_file + */ +extern spinlock_t cifs_tcp_ses_lock; + +/* + * Global transaction id (XID) information + */ +extern unsigned int GlobalCurrentXid; /* protected by GlobalMid_Lock */ +extern unsigned int GlobalTotalActiveXid; /* prot by GlobalMid_Lock */ +extern unsigned int GlobalMaxActiveXid; /* prot by GlobalMid_Lock */ +extern spinlock_t GlobalMid_Lock; /* protects above & list operations on midQ entries */ + +/* + * Global counters, updated atomically + */ +extern atomic_t sesInfoAllocCount; +extern atomic_t tconInfoAllocCount; +extern atomic_t tcpSesNextId; +extern atomic_t tcpSesAllocCount; +extern atomic_t tcpSesReconnectCount; +extern atomic_t tconInfoReconnectCount; + +/* Various Debug counters */ +extern atomic_t buf_alloc_count; /* current number allocated */ +extern atomic_t small_buf_alloc_count; +#ifdef CONFIG_CIFS_STATS2 +extern atomic_t total_buf_alloc_count; /* total allocated over all time */ +extern atomic_t total_small_buf_alloc_count; +extern unsigned int slow_rsp_threshold; /* number of secs before logging */ +#endif + +/* Misc globals */ +extern bool enable_oplocks; /* enable or disable oplocks */ +extern bool lookupCacheEnabled; +extern unsigned int global_secflags; /* if on, session setup sent + with more secure ntlmssp2 challenge/resp */ +extern unsigned int sign_CIFS_PDUs; /* enable smb packet signing */ +extern bool enable_gcm_256; /* allow optional negotiate of strongest signing (aes-gcm-256) */ +extern bool require_gcm_256; /* require use of strongest signing (aes-gcm-256) */ +extern bool enable_negotiate_signing; /* request use of faster (GMAC) signing if available */ +extern bool linuxExtEnabled;/*enable Linux/Unix CIFS extensions*/ +extern unsigned int CIFSMaxBufSize; /* max size not including hdr */ +extern unsigned int cifs_min_rcv; /* min size of big ntwrk buf pool */ +extern unsigned int cifs_min_small; /* min size of small buf pool */ +extern unsigned int cifs_max_pending; /* MAX requests at once to server*/ +extern unsigned int dir_cache_timeout; /* max time for directory lease caching of dir */ +extern bool disable_legacy_dialects; /* forbid vers=1.0 and vers=2.0 mounts */ +extern atomic_t mid_count; + +void cifs_oplock_break(struct work_struct *work); +void cifs_queue_oplock_break(struct cifsFileInfo *cfile); +void smb2_deferred_work_close(struct work_struct *work); + +extern const struct slow_work_ops cifs_oplock_break_ops; +extern struct workqueue_struct *cifsiod_wq; +extern struct workqueue_struct *decrypt_wq; +extern struct workqueue_struct *fileinfo_put_wq; +extern struct workqueue_struct *cifsoplockd_wq; +extern struct workqueue_struct *deferredclose_wq; +extern struct workqueue_struct *serverclose_wq; +extern struct workqueue_struct *cfid_put_wq; +extern __u32 cifs_lock_secret; + +extern mempool_t *cifs_sm_req_poolp; +extern mempool_t *cifs_req_poolp; +extern mempool_t cifs_mid_pool; +extern mempool_t cifs_io_request_pool; +extern mempool_t cifs_io_subrequest_pool; + +/* Operations for different SMB versions */ +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY +extern struct smb_version_operations smb20_operations; +extern struct smb_version_values smb20_values; +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ +extern struct smb_version_operations smb21_operations; +extern struct smb_version_values smb21_values; +extern struct smb_version_values smbdefault_values; +extern struct smb_version_values smb3any_values; +extern struct smb_version_operations smb30_operations; +extern struct smb_version_values smb30_values; +/*extern struct smb_version_operations smb302_operations;*/ /* not needed yet */ +extern struct smb_version_values smb302_values; +extern struct smb_version_operations smb311_operations; +extern struct smb_version_values smb311_values; + +static inline char *get_security_type_str(enum securityEnum sectype) +{ + switch (sectype) { + case RawNTLMSSP: + return "RawNTLMSSP"; + case Kerberos: + return "Kerberos"; + case NTLMv2: + return "NTLMv2"; + case IAKerb: + return "IAKerb"; + default: + return "Unknown"; + } +} + +static inline bool is_smb1_server(struct TCP_Server_Info *server) +{ + return strcmp(server->vals->version_string, SMB1_VERSION_STRING) == 0; +} + +static inline bool is_tcon_dfs(struct cifs_tcon *tcon) +{ + /* + * For SMB1, see MS-CIFS 2.4.55 SMB_COM_TREE_CONNECT_ANDX (0x75) and MS-CIFS 3.3.4.4 DFS + * Subsystem Notifies That a Share Is a DFS Share. + * + * For SMB2+, see MS-SMB2 2.2.10 SMB2 TREE_CONNECT Response and MS-SMB2 3.3.4.14 Server + * Application Updates a Share. + */ + if (!tcon || !tcon->ses || !tcon->ses->server) + return false; + return is_smb1_server(tcon->ses->server) ? tcon->Flags & SMB_SHARE_IS_IN_DFS : + tcon->share_flags & (SHI1005_FLAGS_DFS | SHI1005_FLAGS_DFS_ROOT); +} + +static inline bool cifs_is_referral_server(struct cifs_tcon *tcon, + const struct dfs_info3_param *ref) +{ + /* + * Check if all targets are capable of handling DFS referrals as per + * MS-DFSC 2.2.4 RESP_GET_DFS_REFERRAL. + */ + return is_tcon_dfs(tcon) || (ref && (ref->flags & DFSREF_REFERRAL_SERVER)); +} + +static inline u64 cifs_flock_len(const struct file_lock *fl) +{ + return (u64)fl->fl_end - fl->fl_start + 1; +} + +static inline size_t ntlmssp_workstation_name_size(const struct cifs_ses *ses) +{ + if (WARN_ON_ONCE(!ses || !ses->server)) + return 0; + /* + * Make workstation name no more than 15 chars when using insecure dialects as some legacy + * servers do require it during NTLMSSP. + */ + if (ses->server->dialect <= SMB20_PROT_ID) + return min_t(size_t, sizeof(ses->workstation_name), RFC1001_NAME_LEN_WITH_NULL); + return sizeof(ses->workstation_name); +} + +static inline void move_cifs_info_to_smb2(struct smb2_file_all_info *dst, const FILE_ALL_INFO *src) +{ + memcpy(dst, src, (size_t)((u8 *)&src->EASize - (u8 *)src)); + dst->IndexNumber = 0; + dst->EASize = src->EASize; + dst->AccessFlags = 0; + dst->CurrentByteOffset = 0; + dst->Mode = 0; + dst->AlignmentRequirement = 0; + dst->FileNameLength = src->FileNameLength; +} + +#define CIFS_OPARMS(_cifs_sb, _tcon, _path, _da, _cd, _co, _mode) \ + ((struct cifs_open_parms) { \ + .tcon = _tcon, \ + .path = _path, \ + .desired_access = (_da), \ + .disposition = (_cd), \ + .create_options = cifs_create_options(_cifs_sb, (_co)), \ + .mode = (_mode), \ + .cifs_sb = _cifs_sb, \ + }) + +struct smb2_compound_vars { + struct cifs_open_parms oparms; + struct kvec rsp_iov[MAX_COMPOUND]; + struct smb_rqst rqst[MAX_COMPOUND]; + struct kvec open_iov[SMB2_CREATE_IOV_SIZE]; + struct kvec qi_iov; + struct kvec io_iov[SMB2_IOCTL_IOV_SIZE]; + struct kvec si_iov[SMB2_SET_INFO_IOV_SIZE]; + struct kvec hl_iov[SMB2_SET_INFO_IOV_SIZE]; + struct kvec unlink_iov[SMB2_SET_INFO_IOV_SIZE]; + struct kvec rename_iov[SMB2_SET_INFO_IOV_SIZE]; + struct kvec close_iov; + struct smb2_file_rename_info_hdr rename_info; + struct smb2_file_link_info_hdr link_info; + struct kvec ea_iov; +}; + +static inline bool cifs_ses_exiting(struct cifs_ses *ses) +{ + bool ret; + + spin_lock(&ses->ses_lock); + ret = ses->ses_status == SES_EXITING; + spin_unlock(&ses->ses_lock); + return ret; +} + +static inline bool cifs_netbios_name(const char *name, size_t namelen) +{ + bool ret = false; + size_t i; + + if (namelen >= 1 && namelen <= RFC1001_NAME_LEN) { + for (i = 0; i < namelen; i++) { + const unsigned char c = name[i]; + + if (c == '\\' || c == '/' || c == ':' || c == '*' || + c == '?' || c == '"' || c == '<' || c == '>' || + c == '|' || c == '.') + return false; + if (!ret && isalpha(c)) + ret = true; + } + } + return ret; +} + +/* + * Execute mid callback atomically - ensures callback runs exactly once + * and prevents sleeping in atomic context. + */ +static inline void mid_execute_callback(struct TCP_Server_Info *server, + struct mid_q_entry *mid) +{ + mid_callback_t callback; + + spin_lock(&mid->mid_lock); + callback = mid->callback; + mid->callback = NULL; /* Mark as executed, */ + spin_unlock(&mid->mid_lock); + + if (callback) + callback(server, mid); +} + +#define CIFS_REPARSE_SUPPORT(tcon) \ + ((tcon)->posix_extensions || \ + (le32_to_cpu((tcon)->fsAttrInfo.Attributes) & \ + FILE_SUPPORTS_REPARSE_POINTS)) + +struct cifs_calc_sig_ctx { + struct md5_ctx *md5; + struct hmac_sha256_ctx *hmac; + struct aes_cmac_ctx *cmac; +}; + +#define CIFS_RECONN_DELAY_SECS 30 +#define CIFS_MAX_RECONN_DELAY (4 * CIFS_RECONN_DELAY_SECS) + +static inline void cifs_queue_server_reconn(struct TCP_Server_Info *server) +{ + if (!delayed_work_pending(&server->reconnect)) { + WRITE_ONCE(server->reconn_delay, 0); + mod_delayed_work(cifsiod_wq, &server->reconnect, 0); + } +} + +static inline void cifs_requeue_server_reconn(struct TCP_Server_Info *server) +{ + unsigned long delay = READ_ONCE(server->reconn_delay); + + delay = umin(delay + CIFS_RECONN_DELAY_SECS, CIFS_MAX_RECONN_DELAY); + WRITE_ONCE(server->reconn_delay, delay); + queue_delayed_work(cifsiod_wq, &server->reconnect, delay * HZ); +} + +static inline bool __cifs_cache_state_check(struct cifsInodeInfo *cinode, + unsigned int oplock_flags, + unsigned int sb_flags) +{ + unsigned int sflags = cifs_sb_flags(CIFS_SB(cinode)); + unsigned int oplock = READ_ONCE(cinode->oplock); + + return (oplock & oplock_flags) || (sflags & sb_flags); +} + +#define CIFS_CACHE_READ(cinode) \ + __cifs_cache_state_check(cinode, CIFS_CACHE_READ_FLG, \ + CIFS_MOUNT_RO_CACHE) +#define CIFS_CACHE_HANDLE(cinode) \ + __cifs_cache_state_check(cinode, CIFS_CACHE_HANDLE_FLG, 0) +#define CIFS_CACHE_WRITE(cinode) \ + __cifs_cache_state_check(cinode, CIFS_CACHE_WRITE_FLG, \ + CIFS_MOUNT_RW_CACHE) + +static inline void cifs_reset_oplock(struct cifsInodeInfo *cinode) +{ + scoped_guard(spinlock, &cinode->open_file_lock) + WRITE_ONCE(cinode->oplock, 0); +} + +static inline bool cifs_forced_shutdown(const struct cifs_sb_info *sbi) +{ + return cifs_sb_flags(sbi) & CIFS_MOUNT_SHUTDOWN; +} + +static inline int cifs_open_create_options(unsigned int oflags, int opts) +{ + /* O_SYNC also has bit for O_DSYNC so following check picks up either */ + if (oflags & O_SYNC) + opts |= CREATE_WRITE_THROUGH; + if (oflags & O_DIRECT) + opts |= CREATE_NO_BUFFER; + if (oflags & O_TMPFILE) + opts |= CREATE_DELETE_ON_CLOSE; + return opts; +} + +/* + * inode->i_blocks is counted in 512-byte units, independent of + * inode->i_blksize. + */ +#define CIFS_INO_BLOCK_SIZE 512ULL +#define CIFS_INO_BLOCKS(size) \ + DIV_ROUND_UP_ULL((u64)(size), CIFS_INO_BLOCK_SIZE) +#define CIFS_INO_BYTES(blocks) ((u64)(blocks) * CIFS_INO_BLOCK_SIZE) + +#endif /* _CIFS_GLOB_H */ diff --git a/fs/smb/client/cifspdu.h b/fs/smb/client/cifspdu.h new file mode 100644 index 000000000..78512af18 --- /dev/null +++ b/fs/smb/client/cifspdu.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +/* + * + * Copyright (c) International Business Machines Corp., 2002,2009 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ + +#ifndef _CIFSPDU_H +#define _CIFSPDU_H + +#endif /* _CIFSPDU_H */ diff --git a/fs/smb/client/cifsproto.h b/fs/smb/client/cifsproto.h new file mode 100644 index 000000000..00168839c --- /dev/null +++ b/fs/smb/client/cifsproto.h @@ -0,0 +1,615 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +/* + * + * Copyright (c) International Business Machines Corp., 2002,2008 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ +#ifndef _CIFSPROTO_H +#define _CIFSPROTO_H +#include +#include +#include "cifsglob.h" +#include "trace.h" +#ifdef CONFIG_CIFS_DFS_UPCALL +#include "dfs_cache.h" +#endif +#include "smb1proto.h" + +struct statfs; +struct smb_rqst; +struct smb3_fs_context; +struct fs_context; + +/* + ***************************************************************** + * All Prototypes + ***************************************************************** + */ + +void *cifs_buf_get(void); +void cifs_buf_release(void *buf_to_free); +void *cifs_small_buf_get(void); +void cifs_small_buf_release(void *buf_to_free); +void free_rsp_buf(int resp_buftype, void *rsp); +int smb_send_kvec(struct TCP_Server_Info *server, struct msghdr *smb_msg, + size_t *sent); +unsigned int _get_xid(void); +void _free_xid(unsigned int xid); +#define get_xid() \ +({ \ + unsigned int __xid = _get_xid(); \ + cifs_dbg(FYI, "VFS: in %s as Xid: %u with uid: %d\n", \ + __func__, __xid, \ + from_kuid(&init_user_ns, current_fsuid())); \ + trace_smb3_enter(__xid, __func__); \ + __xid; \ +}) + +#define free_xid(curr_xid) \ +do { \ + _free_xid(curr_xid); \ + cifs_dbg(FYI, "VFS: leaving %s (xid = %u) rc = %d\n", \ + __func__, curr_xid, (int)rc); \ + if (rc) \ + trace_smb3_exit_err(curr_xid, __func__, (int)rc); \ + else \ + trace_smb3_exit_done(curr_xid, __func__); \ +} while (0) +int init_cifs_idmap(void); +void exit_cifs_idmap(void); +int init_cifs_spnego(void); +void exit_cifs_spnego(void); +const char *build_path_from_dentry(struct dentry *direntry, void *page); +char *__build_path_from_dentry_optional_prefix(struct dentry *direntry, + void *page, const char *tree, + int tree_len, bool prefix); +char *build_path_from_dentry_optional_prefix(struct dentry *direntry, + void *page, bool prefix); +static inline void *alloc_dentry_path(void) +{ + return __getname(); +} + +static inline void free_dentry_path(void *page) +{ + if (page) + __putname(page); +} + +char *cifs_build_path_to_root(struct smb3_fs_context *ctx, + struct cifs_sb_info *cifs_sb, + struct cifs_tcon *tcon, int add_treename); +char *cifs_build_devname(char *nodename, const char *prepath); +void delete_mid(struct TCP_Server_Info *server, struct mid_q_entry *mid); +void __release_mid(struct TCP_Server_Info *server, + struct mid_q_entry *midEntry); +void cifs_wake_up_task(struct TCP_Server_Info *server, + struct mid_q_entry *mid); +int cifs_handle_standard(struct TCP_Server_Info *server, + struct mid_q_entry *mid); +char *smb3_fs_context_fullpath(const struct smb3_fs_context *ctx, char dirsep); +int smb3_parse_devname(const char *devname, struct smb3_fs_context *ctx); +int cifs_ipaddr_cmp(struct sockaddr *srcaddr, struct sockaddr *rhs); +bool cifs_match_ipaddr(struct sockaddr *srcaddr, struct sockaddr *rhs); +int cifs_discard_remaining_data(struct TCP_Server_Info *server); +int cifs_call_async(struct TCP_Server_Info *server, struct smb_rqst *rqst, + mid_receive_t receive, mid_callback_t callback, + mid_handle_t handle, void *cbdata, const int flags, + const struct cifs_credits *exist_credits); +struct TCP_Server_Info *cifs_pick_channel(struct cifs_ses *ses); +int cifs_send_recv(const unsigned int xid, struct cifs_ses *ses, + struct TCP_Server_Info *server, struct smb_rqst *rqst, + int *resp_buf_type, const int flags, struct kvec *resp_iov); +int compound_send_recv(const unsigned int xid, struct cifs_ses *ses, + struct TCP_Server_Info *server, const int flags, + const int num_rqst, struct smb_rqst *rqst, + int *resp_buf_type, struct kvec *resp_iov); +int cifs_sync_mid_result(struct mid_q_entry *mid, + struct TCP_Server_Info *server); +int __smb_send_rqst(struct TCP_Server_Info *server, int num_rqst, + struct smb_rqst *rqst); +int wait_for_free_request(struct TCP_Server_Info *server, const int flags, + unsigned int *instance); +int cifs_wait_mtu_credits(struct TCP_Server_Info *server, size_t size, + size_t *num, struct cifs_credits *credits); + +static inline int +send_cancel(struct cifs_ses *ses, struct TCP_Server_Info *server, + struct smb_rqst *rqst, struct mid_q_entry *mid, + unsigned int xid) +{ + return server->ops->send_cancel ? + server->ops->send_cancel(ses, server, rqst, mid, xid) : 0; +} + +int wait_for_response(struct TCP_Server_Info *server, struct mid_q_entry *mid); + +void smb2_query_server_interfaces(struct work_struct *work); +void cifs_signal_cifsd_for_reconnect(struct TCP_Server_Info *server, + bool all_channels); +void cifs_mark_tcp_ses_conns_for_reconnect(struct TCP_Server_Info *server, + bool mark_smb_session); +int cifs_reconnect(struct TCP_Server_Info *server, bool mark_smb_session); +bool backup_cred(struct cifs_sb_info *cifs_sb); +bool is_size_safe_to_change(struct cifsInodeInfo *cifsInode, __u64 end_of_file, + bool from_readdir); +void cifs_write_subrequest_terminated(struct cifs_io_subrequest *wdata, + ssize_t result); +struct cifsFileInfo *find_writable_file(struct cifsInodeInfo *cifs_inode, + int flags); +int __cifs_get_writable_file(struct cifsInodeInfo *cifs_inode, + unsigned int find_flags, unsigned int open_flags, + struct cifsFileInfo **ret_file); +int cifs_get_writable_path(struct cifs_tcon *tcon, const char *name, + struct inode *inode, int flags, + struct cifsFileInfo **ret_file); +struct cifsFileInfo *__find_readable_file(struct cifsInodeInfo *cifs_inode, + unsigned int find_flags, + unsigned int open_flags); +int cifs_get_readable_path(struct cifs_tcon *tcon, const char *name, + struct cifsFileInfo **ret_file); +int cifs_get_hardlink_path(struct cifs_tcon *tcon, struct inode *inode, + struct file *file); +int decode_negTokenInit(unsigned char *security_blob, int length, + struct TCP_Server_Info *server); +int cifs_convert_address(struct sockaddr *dst, const char *src, int len); +void cifs_set_port(struct sockaddr *addr, const unsigned short int port); +struct timespec64 cifs_NTtimeToUnix(__le64 ntutc); +u64 cifs_UnixTimeToNT(struct timespec64 t); +struct timespec64 cnvrtDosUnixTm(__le16 le_date, __le16 le_time, int offset); +void cifs_set_oplock_level(struct cifsInodeInfo *cinode, __u32 oplock); +int cifs_get_writer(struct cifsInodeInfo *cinode); +void cifs_put_writer(struct cifsInodeInfo *cinode); +void cifs_done_oplock_break(struct cifsInodeInfo *cinode); +int cifs_unlock_range(struct cifsFileInfo *cfile, struct file_lock *flock, + unsigned int xid); +int cifs_push_mandatory_locks(struct cifsFileInfo *cfile); + +void cifs_down_write(struct rw_semaphore *sem); +struct cifsFileInfo *cifs_new_fileinfo(struct cifs_fid *fid, struct file *file, + struct tcon_link *tlink, __u32 oplock, + const char *symlink_target); +int cifs_posix_open(const char *full_path, struct inode **pinode, + struct super_block *sb, int mode, unsigned int f_flags, + __u32 *poplock, __u16 *pnetfid, unsigned int xid); +void cifs_fill_uniqueid(struct super_block *sb, struct cifs_fattr *fattr); +void cifs_unix_basic_to_fattr(struct cifs_fattr *fattr, + FILE_UNIX_BASIC_INFO *info, + struct cifs_sb_info *cifs_sb); +void cifs_dir_info_to_fattr(struct cifs_fattr *fattr, + FILE_DIRECTORY_INFO *info, + struct cifs_sb_info *cifs_sb); +int cifs_fattr_to_inode(struct inode *inode, struct cifs_fattr *fattr, + bool from_readdir); +struct inode *cifs_iget(struct super_block *sb, struct cifs_fattr *fattr); + +int cifs_get_inode_info(struct inode **inode, const char *full_path, + struct cifs_open_info_data *data, + struct super_block *sb, int xid, + const struct cifs_fid *fid); +int smb311_posix_get_inode_info(struct inode **inode, const char *full_path, + struct cifs_open_info_data *data, + struct super_block *sb, + const unsigned int xid); +int cifs_get_inode_info_unix(struct inode **pinode, + const unsigned char *full_path, + struct super_block *sb, unsigned int xid); +int cifs_set_file_info(struct inode *inode, struct iattr *attrs, + unsigned int xid, const char *full_path, __u32 dosattr); +int cifs_rename_pending_delete(const char *full_path, struct dentry *dentry, + const unsigned int xid); +int sid_to_id(struct cifs_sb_info *cifs_sb, struct smb_sid *psid, + struct cifs_fattr *fattr, uint sidtype); +int cifs_acl_to_fattr(struct cifs_sb_info *cifs_sb, struct cifs_fattr *fattr, + struct inode *inode, bool mode_from_special_sid, + const char *path, const struct cifs_fid *pfid); +int id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode, + kuid_t uid, kgid_t gid); +struct smb_ntsd *get_cifs_acl(struct cifs_sb_info *cifs_sb, + struct inode *inode, const char *path, + u32 *pacllen, u32 info); +struct smb_ntsd *get_cifs_acl_by_fid(struct cifs_sb_info *cifs_sb, + const struct cifs_fid *cifsfid, + u32 *pacllen, u32 info); +struct posix_acl *cifs_get_acl(struct mnt_idmap *idmap, struct dentry *dentry, + int type); +int cifs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type); +int set_cifs_acl(struct smb_ntsd *pnntsd, __u32 acllen, struct inode *inode, + const char *path, int aclflag); +unsigned int setup_authusers_ACE(struct smb_ace *pntace); +unsigned int setup_special_mode_ACE(struct smb_ace *pntace, bool posix, + __u64 nmode); +unsigned int setup_special_user_owner_ACE(struct smb_ace *pntace); + +void dequeue_mid(struct TCP_Server_Info *server, struct mid_q_entry *mid, + bool malformed); +int cifs_read_from_socket(struct TCP_Server_Info *server, char *buf, + unsigned int to_read); +ssize_t cifs_discard_from_socket(struct TCP_Server_Info *server, + size_t to_read); +int cifs_read_iter_from_socket(struct TCP_Server_Info *server, + struct iov_iter *iter, unsigned int to_read); +int cifs_setup_cifs_sb(struct cifs_sb_info *cifs_sb); +void cifs_mount_put_conns(struct cifs_mount_ctx *mnt_ctx); +int cifs_mount_get_session(struct cifs_mount_ctx *mnt_ctx); +int cifs_is_path_remote(struct cifs_mount_ctx *mnt_ctx); +int cifs_mount_get_tcon(struct cifs_mount_ctx *mnt_ctx); +int cifs_match_super(struct super_block *sb, struct fs_context *fc); +int cifs_mount(struct cifs_sb_info *cifs_sb, struct smb3_fs_context *ctx); +void cifs_umount(struct cifs_sb_info *cifs_sb); +void cifs_mark_open_files_invalid(struct cifs_tcon *tcon); +void cifs_reopen_persistent_handles(struct cifs_tcon *tcon); + +bool cifs_find_lock_conflict(struct cifsFileInfo *cfile, __u64 offset, + __u64 length, __u8 type, __u16 flags, + struct cifsLockInfo **conf_lock, int rw_check); +void cifs_add_pending_open(struct cifs_fid *fid, struct tcon_link *tlink, + struct cifs_pending_open *open); +void cifs_add_pending_open_locked(struct cifs_fid *fid, + struct tcon_link *tlink, + struct cifs_pending_open *open); +void cifs_del_pending_open(struct cifs_pending_open *open); + +bool cifs_is_deferred_close(struct cifsFileInfo *cfile, + struct cifs_deferred_close **pdclose); + +void cifs_add_deferred_close(struct cifsFileInfo *cfile, + struct cifs_deferred_close *dclose); + +void cifs_del_deferred_close(struct cifsFileInfo *cfile); + +void cifs_close_deferred_file(struct cifsInodeInfo *cifs_inode); + +void cifs_close_all_deferred_files(struct cifs_tcon *tcon); + +void cifs_close_all_deferred_files_sb(struct cifs_sb_info *cifs_sb); +void cifs_close_deferred_file_under_dentry(struct cifs_tcon *tcon, + struct dentry *dentry); + +void cifs_mark_open_handles_for_deleted_file(struct inode *inode, + const char *path); + +struct TCP_Server_Info *cifs_get_tcp_session(struct smb3_fs_context *ctx, + struct TCP_Server_Info *primary_server); +void cifs_put_tcp_session(struct TCP_Server_Info *server, int from_reconnect); +void cifs_put_tcon(struct cifs_tcon *tcon, enum smb3_tcon_ref_trace trace); + +void cifs_release_automount_timer(void); + +void cifs_proc_init(void); +void cifs_proc_clean(void); + +void cifs_move_llist(struct list_head *source, struct list_head *dest); +void cifs_free_llist(struct list_head *llist); +void cifs_del_lock_waiters(struct cifsLockInfo *lock); + +int cifs_tree_connect(const unsigned int xid, struct cifs_tcon *tcon); + +int cifs_negotiate_protocol(const unsigned int xid, struct cifs_ses *ses, + struct TCP_Server_Info *server); +int cifs_setup_session(const unsigned int xid, struct cifs_ses *ses, + struct TCP_Server_Info *server, + struct nls_table *nls_info); +int cifs_enable_signing(struct TCP_Server_Info *server, + bool mnt_sign_required); + +int parse_dfs_referrals(struct get_dfs_referral_rsp *rsp, u32 rsp_size, + unsigned int *num_of_nodes, + struct dfs_info3_param **target_nodes, + const struct nls_table *nls_codepage, int remap, + const char *searchName, bool is_unicode); + +struct cifs_ses *sesInfoAlloc(void); +void sesInfoFree(struct cifs_ses *buf_to_free); +struct cifs_tcon *tcon_info_alloc(bool dir_leases_enabled, + enum smb3_tcon_ref_trace trace); +void tconInfoFree(struct cifs_tcon *tcon, enum smb3_tcon_ref_trace trace); + +int setup_ntlmv2_rsp(struct cifs_ses *ses, const struct nls_table *nls_cp); +void cifs_crypto_secmech_release(struct TCP_Server_Info *server); +int calc_seckey(struct cifs_ses *ses); +int generate_smb30signingkey(struct cifs_ses *ses, + struct TCP_Server_Info *server); +int generate_smb311signingkey(struct cifs_ses *ses, + struct TCP_Server_Info *server); + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ +void cifs_autodisable_serverino(struct cifs_sb_info *cifs_sb, const char *reason, int rc); +bool couldbe_mf_symlink(const struct cifs_fattr *fattr); +int check_mf_symlink(unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, struct cifs_fattr *fattr, + const unsigned char *path); +int E_md4hash(const unsigned char *passwd, unsigned char *p16, + const struct nls_table *codepage); + +struct TCP_Server_Info *cifs_find_tcp_session(struct smb3_fs_context *ctx); + +struct cifs_tcon *cifs_setup_ipc(struct cifs_ses *ses, bool seal); + +void __cifs_put_smb_ses(struct cifs_ses *ses); + +struct cifs_ses *cifs_get_smb_ses(struct TCP_Server_Info *server, + struct smb3_fs_context *ctx); + +int cifs_readv_receive(struct TCP_Server_Info *server, + struct mid_q_entry *mid); + +int cifs_query_mf_symlink(unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, + const unsigned char *path, char *pbuf, + unsigned int *pbytes_read); +int cifs_create_mf_symlink(unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, + const unsigned char *path, char *pbuf, + unsigned int *pbytes_written); +int __cifs_calc_signature(struct smb_rqst *rqst, + struct TCP_Server_Info *server, char *signature, + struct cifs_calc_sig_ctx *ctx); +enum securityEnum cifs_select_sectype(struct TCP_Server_Info *server, + enum securityEnum requested); + +int cifs_try_adding_channels(struct cifs_ses *ses); +int smb3_update_ses_channels(struct cifs_ses *ses, + struct TCP_Server_Info *server, + bool from_reconnect, bool disable_mchan); +bool is_ses_using_iface(struct cifs_ses *ses, struct cifs_server_iface *iface); + +int cifs_ses_get_chan_index(struct cifs_ses *ses, + struct TCP_Server_Info *server); +void cifs_chan_set_in_reconnect(struct cifs_ses *ses, + struct TCP_Server_Info *server); +void cifs_chan_clear_in_reconnect(struct cifs_ses *ses, + struct TCP_Server_Info *server); +void cifs_chan_set_need_reconnect(struct cifs_ses *ses, + struct TCP_Server_Info *server); +void cifs_chan_clear_need_reconnect(struct cifs_ses *ses, + struct TCP_Server_Info *server); +bool cifs_chan_needs_reconnect(struct cifs_ses *ses, + struct TCP_Server_Info *server); +bool cifs_chan_is_iface_active(struct cifs_ses *ses, + struct TCP_Server_Info *server); +void cifs_decrease_secondary_channels(struct cifs_ses *ses, + bool disable_mchan); +void cifs_chan_update_iface(struct cifs_ses *ses, + struct TCP_Server_Info *server); +int SMB3_request_interfaces(const unsigned int xid, struct cifs_tcon *tcon, + bool in_mount); + +void extract_unc_hostname(const char *unc, const char **h, size_t *len); +int copy_path_name(char *dst, const char *src); +int smb2_parse_query_directory(struct cifs_tcon *tcon, struct kvec *rsp_iov, + int resp_buftype, + struct cifs_search_info *srch_inf); + +struct super_block *cifs_get_dfs_tcon_super(struct cifs_tcon *tcon); +void cifs_put_tcp_super(struct super_block *sb); +int cifs_update_super_prepath(struct cifs_sb_info *cifs_sb, char *prefix); +char *extract_hostname(const char *unc); +char *extract_sharename(const char *unc); +int parse_reparse_point(struct reparse_data_buffer *buf, u32 plen, + struct cifs_sb_info *cifs_sb, const char *full_path, + struct cifs_open_info_data *data); +int __cifs_sfu_make_node(unsigned int xid, struct inode *inode, + struct dentry *dentry, struct cifs_tcon *tcon, + const char *full_path, umode_t mode, dev_t dev, + const char *symname); +int cifs_sfu_make_node(unsigned int xid, struct inode *inode, + struct dentry *dentry, struct cifs_tcon *tcon, + const char *full_path, umode_t mode, dev_t dev); +umode_t wire_mode_to_posix(u32 wire, bool is_dir); + +#ifdef CONFIG_CIFS_DFS_UPCALL +static inline int get_dfs_path(const unsigned int xid, struct cifs_ses *ses, + const char *old_path, + const struct nls_table *nls_codepage, + struct dfs_info3_param *referral, int remap) +{ + return dfs_cache_find(xid, ses, nls_codepage, remap, old_path, + referral, NULL); +} + +int match_target_ip(struct TCP_Server_Info *server, const char *host, + size_t hostlen, bool *result); +int cifs_inval_name_dfs_link_error(const unsigned int xid, + struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, + const char *full_path, bool *islink); +#else +static inline int cifs_inval_name_dfs_link_error(const unsigned int xid, + struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, + const char *full_path, + bool *islink) +{ + *islink = false; + return 0; +} +#endif + +static inline int cifs_create_options(struct cifs_sb_info *cifs_sb, int options) +{ + if (cifs_sb && (backup_cred(cifs_sb))) + return options | CREATE_OPEN_BACKUP_INTENT; + else + return options; +} + +int cifs_wait_for_server_reconnect(struct TCP_Server_Info *server, bool retry); + +static inline void cifs_put_smb_ses(struct cifs_ses *ses) +{ + __cifs_put_smb_ses(ses); +} + +/* Get an active reference of @ses and its children. + * + * NOTE: make sure to call this function when incrementing reference count of + * @ses to ensure that any DFS root session attached to it (@ses->dfs_root_ses) + * will also get its reference count incremented. + * + * cifs_put_smb_ses() will put all references, so call it when you're done. + */ +static inline void cifs_smb_ses_inc_refcount(struct cifs_ses *ses) +{ + lockdep_assert_held(&cifs_tcp_ses_lock); + ses->ses_count++; +} + +static inline bool dfs_src_pathname_equal(const char *s1, const char *s2) +{ + if (strlen(s1) != strlen(s2)) + return false; + for (; *s1; s1++, s2++) { + if (*s1 == '/' || *s1 == '\\') { + if (*s2 != '/' && *s2 != '\\') + return false; + } else if (tolower(*s1) != tolower(*s2)) + return false; + } + return true; +} + +static inline void smb_get_mid(struct mid_q_entry *mid) +{ + refcount_inc(&mid->refcount); +} + +static inline void release_mid(struct TCP_Server_Info *server, struct mid_q_entry *mid) +{ + if (refcount_dec_and_test(&mid->refcount)) + __release_mid(server, mid); +} + +static inline void cifs_free_open_info(struct cifs_open_info_data *data) +{ + kfree(data->symlink_target); + free_rsp_buf(data->reparse.io.buftype, data->reparse.io.iov.iov_base); + memset(data, 0, sizeof(*data)); +} + +static inline int smb_EIO(enum smb_eio_trace trace) +{ + trace_smb3_eio(trace, 0, 0); + return -EIO; +} + +static inline int smb_EIO1(enum smb_eio_trace trace, unsigned long info) +{ + trace_smb3_eio(trace, info, 0); + return -EIO; +} + +static inline int smb_EIO2(enum smb_eio_trace trace, unsigned long info, unsigned long info2) +{ + trace_smb3_eio(trace, info, info2); + return -EIO; +} + +static inline int cifs_get_num_sgs(const struct smb_rqst *rqst, + int num_rqst, + const u8 *sig) +{ + unsigned int len, skip; + unsigned int nents = 0; + unsigned long addr; + size_t data_size; + int i, j; + + /* + * The first rqst has a transform header where the first 20 bytes are + * not part of the encrypted blob. + */ + skip = 20; + + /* Assumes the first rqst has a transform header as the first iov. + * I.e. + * rqst[0].rq_iov[0] is transform header + * rqst[0].rq_iov[1+] data to be encrypted/decrypted + * rqst[1+].rq_iov[0+] data to be encrypted/decrypted + */ + for (i = 0; i < num_rqst; i++) { + data_size = iov_iter_count(&rqst[i].rq_iter); + + /* We really don't want a mixture of pinned and unpinned pages + * in the sglist. It's hard to keep track of which is what. + * Instead, we convert to a BVEC-type iterator higher up. + */ + if (data_size && + WARN_ON_ONCE(user_backed_iter(&rqst[i].rq_iter))) + return smb_EIO(smb_eio_trace_user_iter); + + /* We also don't want to have any extra refs or pins to clean + * up in the sglist. + */ + if (data_size && + WARN_ON_ONCE(iov_iter_extract_will_pin(&rqst[i].rq_iter))) + return smb_EIO(smb_eio_trace_extract_will_pin); + + for (j = 0; j < rqst[i].rq_nvec; j++) { + struct kvec *iov = &rqst[i].rq_iov[j]; + + addr = (unsigned long)iov->iov_base + skip; + if (is_vmalloc_or_module_addr((void *)addr)) { + len = iov->iov_len - skip; + nents += DIV_ROUND_UP(offset_in_page(addr) + len, + PAGE_SIZE); + } else { + nents++; + } + skip = 0; + } + if (data_size) + nents += iov_iter_npages(&rqst[i].rq_iter, INT_MAX); + } + nents += DIV_ROUND_UP(offset_in_page(sig) + SMB2_SIGNATURE_SIZE, PAGE_SIZE); + return nents; +} + +/* We can not use the normal sg_set_buf() as we will sometimes pass a + * stack object as buf. + */ +static inline void cifs_sg_set_buf(struct sg_table *sgtable, + const void *buf, + unsigned int buflen) +{ + unsigned long addr = (unsigned long)buf; + unsigned int off = offset_in_page(addr); + + addr &= PAGE_MASK; + if (is_vmalloc_or_module_addr((void *)addr)) { + do { + unsigned int len = min_t(unsigned int, buflen, PAGE_SIZE - off); + + sg_set_page(&sgtable->sgl[sgtable->nents++], + vmalloc_to_page((void *)addr), len, off); + + off = 0; + addr += PAGE_SIZE; + buflen -= len; + } while (buflen); + } else { + sg_set_page(&sgtable->sgl[sgtable->nents++], + virt_to_page((void *)addr), buflen, off); + } +} + +static inline int cifs_get_writable_file(struct cifsInodeInfo *cifs_inode, + unsigned int find_flags, + struct cifsFileInfo **ret_file) +{ + find_flags &= ~FIND_OPEN_FLAGS; + return __cifs_get_writable_file(cifs_inode, find_flags, 0, ret_file); +} + +static inline struct cifsFileInfo * +find_readable_file(struct cifsInodeInfo *cinode, unsigned int find_flags) +{ + find_flags &= ~FIND_OPEN_FLAGS; + find_flags |= FIND_NO_PENDING_DELETE; + return __find_readable_file(cinode, find_flags, 0); +} + +#endif /* _CIFSPROTO_H */ diff --git a/fs/smb/client/cifsroot.c b/fs/smb/client/cifsroot.c new file mode 100644 index 000000000..56ec1b233 --- /dev/null +++ b/fs/smb/client/cifsroot.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * SMB root file system support + * + * Copyright (c) 2019 Paulo Alcantara + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DEFAULT_MNT_OPTS \ + "vers=1.0,cifsacl,mfsymlinks,rsize=1048576,wsize=65536,uid=0,gid=0," \ + "hard,rootfs" + +static char root_dev[2048] __initdata = ""; +static char root_opts[1024] __initdata = DEFAULT_MNT_OPTS; + +static __be32 __init parse_srvaddr(char *start, char *end) +{ + /* TODO: ipv6 support */ + char addr[sizeof("aaa.bbb.ccc.ddd")]; + int i = 0; + + while (start < end && i < sizeof(addr) - 1) { + if (isdigit(*start) || *start == '.') + addr[i++] = *start; + start++; + } + addr[i] = '\0'; + return in_aton(addr); +} + +/* cifsroot=///[,options] */ +static int __init cifs_root_setup(char *line) +{ + char *s; + int len; + __be32 srvaddr = htonl(INADDR_NONE); + + ROOT_DEV = Root_CIFS; + + if (strlen(line) > 3 && line[0] == '/' && line[1] == '/') { + s = strchr(&line[2], '/'); + if (!s || s[1] == '\0') + return 1; + + /* make s point to ',' or '\0' at end of line */ + s = strchrnul(s, ','); + /* len is strlen(unc) + '\0' */ + len = s - line + 1; + if (len > sizeof(root_dev)) { + pr_err("Root-CIFS: UNC path too long\n"); + return 1; + } + strscpy(root_dev, line, len); + srvaddr = parse_srvaddr(&line[2], s); + if (*s) { + int n = snprintf(root_opts, + sizeof(root_opts), "%s,%s", + DEFAULT_MNT_OPTS, s + 1); + if (n >= sizeof(root_opts)) { + pr_err("Root-CIFS: mount options string too long\n"); + root_opts[sizeof(root_opts)-1] = '\0'; + return 1; + } + } + } + + root_server_addr = srvaddr; + + return 1; +} + +__setup("cifsroot=", cifs_root_setup); + +int __init cifs_root_data(char **dev, char **opts) +{ + if (!root_dev[0] || root_server_addr == htonl(INADDR_NONE)) { + pr_err("Root-CIFS: no SMB server address\n"); + return -1; + } + + *dev = root_dev; + *opts = root_opts; + + return 0; +} diff --git a/fs/smb/client/cifssmb.c b/fs/smb/client/cifssmb.c new file mode 100644 index 000000000..1f7751225 --- /dev/null +++ b/fs/smb/client/cifssmb.c @@ -0,0 +1,6411 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * Copyright (C) International Business Machines Corp., 2002,2010 + * Author(s): Steve French (sfrench@us.ibm.com) + * + * Contains the routines for constructing the SMB PDUs themselves + * + */ + + /* SMB/CIFS PDU handling routines here - except for leftovers in connect.c */ + /* These are mostly routines that operate on a pathname, or on a tree id */ + /* (mounted volume), but there are eight handle based routines which must be */ + /* treated slightly differently for reconnection purposes since we never */ + /* want to reuse a stale file handle and only the caller knows the file info */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "cifsglob.h" +#include "cifsproto.h" +#include "smb1proto.h" +#include "../common/smbfsctl.h" +#include "cifsfs.h" +#include "cifsacl.h" +#include "cifs_unicode.h" +#include "cifs_debug.h" +#include "fscache.h" +#include "smbdirect.h" +#ifdef CONFIG_CIFS_DFS_UPCALL +#include "dfs_cache.h" +#endif + +#ifdef CONFIG_CIFS_POSIX +static struct { + int index; + char *name; +} protocols[] = { + {CIFS_PROT, "\2NT LM 0.12"}, + {POSIX_PROT, "\2POSIX 2"}, + {BAD_PROT, "\2"} +}; +#else +static struct { + int index; + char *name; +} protocols[] = { + {CIFS_PROT, "\2NT LM 0.12"}, + {BAD_PROT, "\2"} +}; +#endif + +/* define the number of elements in the cifs dialect array */ +#ifdef CONFIG_CIFS_POSIX +#define CIFS_NUM_PROT 2 +#else /* not posix */ +#define CIFS_NUM_PROT 1 +#endif /* CIFS_POSIX */ + + +/* reconnect the socket, tcon, and smb session if needed */ +static int +cifs_reconnect_tcon(struct cifs_tcon *tcon, int smb_command) +{ + struct TCP_Server_Info *server; + struct cifs_ses *ses; + int rc; + + /* + * SMBs NegProt, SessSetup, uLogoff do not have tcon yet so check for + * tcp and smb session status done differently for those three - in the + * calling routine + */ + if (!tcon) + return 0; + + ses = tcon->ses; + server = ses->server; + + /* + * only tree disconnect, open, and write, (and ulogoff which does not + * have tcon) are allowed as we start umount + */ + spin_lock(&tcon->tc_lock); + if (tcon->status == TID_EXITING) { + if (smb_command != SMB_COM_TREE_DISCONNECT) { + spin_unlock(&tcon->tc_lock); + cifs_dbg(FYI, "can not send cmd %d while umounting\n", + smb_command); + return -ENODEV; + } + } + spin_unlock(&tcon->tc_lock); + +again: + rc = cifs_wait_for_server_reconnect(server, tcon->retry); + if (rc) + return rc; + + spin_lock(&ses->chan_lock); + if (!cifs_chan_needs_reconnect(ses, server) && !tcon->need_reconnect) { + spin_unlock(&ses->chan_lock); + return 0; + } + spin_unlock(&ses->chan_lock); + + mutex_lock(&ses->session_mutex); + /* + * Handle the case where a concurrent thread failed to negotiate or + * killed a channel. + */ + spin_lock(&server->srv_lock); + switch (server->tcpStatus) { + case CifsExiting: + spin_unlock(&server->srv_lock); + mutex_unlock(&ses->session_mutex); + return -EHOSTDOWN; + case CifsNeedReconnect: + spin_unlock(&server->srv_lock); + mutex_unlock(&ses->session_mutex); + if (!tcon->retry) + return -EHOSTDOWN; + goto again; + default: + break; + } + spin_unlock(&server->srv_lock); + + /* + * need to prevent multiple threads trying to simultaneously + * reconnect the same SMB session + */ + spin_lock(&ses->ses_lock); + spin_lock(&ses->chan_lock); + if (!cifs_chan_needs_reconnect(ses, server) && + ses->ses_status == SES_GOOD) { + spin_unlock(&ses->chan_lock); + spin_unlock(&ses->ses_lock); + + /* this means that we only need to tree connect */ + if (tcon->need_reconnect) + goto skip_sess_setup; + + mutex_unlock(&ses->session_mutex); + goto out; + } + spin_unlock(&ses->chan_lock); + spin_unlock(&ses->ses_lock); + + rc = cifs_negotiate_protocol(0, ses, server); + if (rc) { + mutex_unlock(&ses->session_mutex); + if (!tcon->retry) + return -EHOSTDOWN; + goto again; + } + rc = cifs_setup_session(0, ses, server, ses->local_nls); + if ((rc == -EACCES) || (rc == -EHOSTDOWN) || (rc == -EKEYREVOKED)) { + /* + * Try alternate password for next reconnect if an alternate + * password is available. + */ + if (ses->password2) + swap(ses->password2, ses->password); + } + + /* do we need to reconnect tcon? */ + if (rc || !tcon->need_reconnect) { + mutex_unlock(&ses->session_mutex); + goto out; + } + +skip_sess_setup: + cifs_mark_open_files_invalid(tcon); + rc = cifs_tree_connect(0, tcon); + mutex_unlock(&ses->session_mutex); + cifs_dbg(FYI, "reconnect tcon rc = %d\n", rc); + + if (rc) { + pr_warn_once("reconnect tcon failed rc = %d\n", rc); + goto out; + } + + atomic_inc(&tconInfoReconnectCount); + + /* tell server Unix caps we support */ + if (cap_unix(ses)) + reset_cifs_unix_caps(0, tcon, NULL, NULL); + + /* + * Removed call to reopen open files here. It is safer (and faster) to + * reopen files one at a time as needed in read and write. + * + * FIXME: what about file locks? don't we need to reclaim them ASAP? + */ + +out: + /* + * Check if handle based operation so we know whether we can continue + * or not without returning to caller to reset file handle + */ + switch (smb_command) { + case SMB_COM_READ_ANDX: + case SMB_COM_WRITE_ANDX: + case SMB_COM_CLOSE: + case SMB_COM_FIND_CLOSE2: + case SMB_COM_LOCKING_ANDX: + rc = -EAGAIN; + } + + return rc; +} + +/* Allocate and return pointer to an SMB request buffer, and set basic + SMB information in the SMB header. If the return code is zero, this + function must have filled in request_buf pointer */ +static int +small_smb_init(int smb_command, int wct, struct cifs_tcon *tcon, + void **request_buf) +{ + unsigned int in_len; + int rc; + + rc = cifs_reconnect_tcon(tcon, smb_command); + if (rc) + return rc; + + *request_buf = cifs_small_buf_get(); + if (*request_buf == NULL) { + /* BB should we add a retry in here if not a writepage? */ + return -ENOMEM; + } + + in_len = header_assemble((struct smb_hdr *) *request_buf, smb_command, + tcon, wct); + + if (tcon != NULL) + cifs_stats_inc(&tcon->num_smbs_sent); + + return in_len; +} + +int +small_smb_init_no_tc(const int smb_command, const int wct, + struct cifs_ses *ses, void **request_buf) +{ + int rc; + struct smb_hdr *buffer; + + rc = small_smb_init(smb_command, wct, NULL, request_buf); + if (rc < 0) + return rc; + + buffer = (struct smb_hdr *)*request_buf; + buffer->Mid = get_next_mid(ses->server); + if (ses->capabilities & CAP_UNICODE) + buffer->Flags2 |= SMBFLG2_UNICODE; + if (ses->capabilities & CAP_STATUS32) + buffer->Flags2 |= SMBFLG2_ERR_STATUS; + + /* uid, tid can stay at zero as set in header assemble */ + + /* BB add support for turning on the signing when + this function is used after 1st of session setup requests */ + + return rc; +} + +/* If the return code is zero, this function must fill in request_buf pointer */ +static int +__smb_init(int smb_command, int wct, struct cifs_tcon *tcon, + void **request_buf, void **response_buf) +{ + unsigned int in_len; + + *request_buf = cifs_buf_get(); + if (*request_buf == NULL) { + /* BB should we add a retry in here if not a writepage? */ + return -ENOMEM; + } + /* Although the original thought was we needed the response buf for */ + /* potential retries of smb operations it turns out we can determine */ + /* from the mid flags when the request buffer can be resent without */ + /* having to use a second distinct buffer for the response */ + if (response_buf) + *response_buf = *request_buf; + + in_len = header_assemble((struct smb_hdr *)*request_buf, smb_command, tcon, + wct); + + if (tcon != NULL) + cifs_stats_inc(&tcon->num_smbs_sent); + + return in_len; +} + +/* If the return code is zero, this function must fill in request_buf pointer */ +static int +smb_init(int smb_command, int wct, struct cifs_tcon *tcon, + void **request_buf, void **response_buf) +{ + int rc; + + rc = cifs_reconnect_tcon(tcon, smb_command); + if (rc) + return rc; + + return __smb_init(smb_command, wct, tcon, request_buf, response_buf); +} + +static int +smb_init_no_reconnect(int smb_command, int wct, struct cifs_tcon *tcon, + void **request_buf, void **response_buf) +{ + spin_lock(&tcon->ses->chan_lock); + if (cifs_chan_needs_reconnect(tcon->ses, tcon->ses->server) || + tcon->need_reconnect) { + spin_unlock(&tcon->ses->chan_lock); + return -EHOSTDOWN; + } + spin_unlock(&tcon->ses->chan_lock); + + return __smb_init(smb_command, wct, tcon, request_buf, response_buf); +} + +static int validate_t2(struct smb_t2_rsp *pSMB) +{ + unsigned int total_size; + + /* check for plausible wct */ + if (pSMB->hdr.WordCount < 10) + goto vt2_err; + + /* check for parm and data offset going beyond end of smb */ + if (get_unaligned_le16(&pSMB->t2_rsp.ParameterOffset) > 1024 || + get_unaligned_le16(&pSMB->t2_rsp.DataOffset) > 1024) + goto vt2_err; + + total_size = get_unaligned_le16(&pSMB->t2_rsp.ParameterCount); + if (total_size >= 512) + goto vt2_err; + + /* check that bcc is at least as big as parms + data, and that it is + * less than negotiated smb buffer + */ + total_size += get_unaligned_le16(&pSMB->t2_rsp.DataCount); + if (total_size > get_bcc(&pSMB->hdr) || + total_size >= CIFSMaxBufSize + MAX_CIFS_HDR_SIZE) + goto vt2_err; + + return 0; +vt2_err: + cifs_dump_mem("Invalid transact2 SMB: ", (char *)pSMB, + sizeof(struct smb_t2_rsp) + 16); + return -EINVAL; +} + +static int +decode_ext_sec_blob(struct cifs_ses *ses, SMB_NEGOTIATE_RSP *pSMBr) +{ + int rc = 0; + u16 count; + char *guid = pSMBr->u.extended_response.GUID; + struct TCP_Server_Info *server = ses->server; + + count = get_bcc(&pSMBr->hdr); + if (count < SMB1_CLIENT_GUID_SIZE) + return smb_EIO2(smb_eio_trace_neg_sec_blob_too_small, + count, SMB1_CLIENT_GUID_SIZE); + + spin_lock(&cifs_tcp_ses_lock); + if (server->srv_count > 1) { + spin_unlock(&cifs_tcp_ses_lock); + if (memcmp(server->server_GUID, guid, SMB1_CLIENT_GUID_SIZE) != 0) { + cifs_dbg(FYI, "server UID changed\n"); + memcpy(server->server_GUID, guid, SMB1_CLIENT_GUID_SIZE); + } + } else { + spin_unlock(&cifs_tcp_ses_lock); + memcpy(server->server_GUID, guid, SMB1_CLIENT_GUID_SIZE); + } + + if (count == SMB1_CLIENT_GUID_SIZE) { + server->sec_ntlmssp = true; + } else { + count -= SMB1_CLIENT_GUID_SIZE; + rc = decode_negTokenInit( + pSMBr->u.extended_response.SecurityBlob, count, server); + if (rc != 1) + return -EINVAL; + } + + return 0; +} + +static bool +should_set_ext_sec_flag(enum securityEnum sectype) +{ + switch (sectype) { + case RawNTLMSSP: + case Kerberos: + return true; + case Unspecified: + if (global_secflags & + (CIFSSEC_MAY_KRB5 | CIFSSEC_MAY_NTLMSSP)) + return true; + fallthrough; + default: + return false; + } +} + +int +CIFSSMBNegotiate(const unsigned int xid, + struct cifs_ses *ses, + struct TCP_Server_Info *server) +{ + SMB_NEGOTIATE_REQ *pSMB; + SMB_NEGOTIATE_RSP *pSMBr; + unsigned int in_len; + int rc = 0; + int bytes_returned; + int i; + u16 count; + + if (!server) { + WARN(1, "%s: server is NULL!\n", __func__); + return smb_EIO(smb_eio_trace_null_pointers); + } + + rc = smb_init(SMB_COM_NEGOTIATE, 0, NULL /* no tcon yet */ , + (void **) &pSMB, (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + pSMB->hdr.Mid = get_next_mid(server); + pSMB->hdr.Flags2 |= SMBFLG2_ERR_STATUS; + + if (ses->unicode != 0) + pSMB->hdr.Flags2 |= SMBFLG2_UNICODE; + + if (should_set_ext_sec_flag(ses->sectype)) { + cifs_dbg(FYI, "Requesting extended security\n"); + pSMB->hdr.Flags2 |= SMBFLG2_EXT_SEC; + } + + count = 0; + /* + * We know that all the name entries in the protocols array + * are short (< 16 bytes anyway) and are NUL terminated. + */ + for (i = 0; i < CIFS_NUM_PROT; i++) { + size_t len = strlen(protocols[i].name) + 1; + + memcpy(&pSMB->DialectsArray[count], protocols[i].name, len); + count += len; + } + in_len += count; + pSMB->ByteCount = cpu_to_le16(count); + + rc = SendReceive(xid, ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc != 0) + goto neg_err_exit; + + server->dialect = le16_to_cpu(pSMBr->DialectIndex); + cifs_dbg(FYI, "Dialect: %d\n", server->dialect); + /* Check wct = 1 error case */ + if ((pSMBr->hdr.WordCount <= 13) || (server->dialect == BAD_PROT)) { + /* core returns wct = 1, but we do not ask for core - otherwise + small wct just comes when dialect index is -1 indicating we + could not negotiate a common dialect */ + rc = -EOPNOTSUPP; + goto neg_err_exit; + } else if (pSMBr->hdr.WordCount != 17) { + /* unknown wct */ + rc = -EOPNOTSUPP; + goto neg_err_exit; + } + /* else wct == 17, NTLM or better */ + + server->sec_mode = pSMBr->SecurityMode; + if ((server->sec_mode & SECMODE_USER) == 0) + cifs_dbg(FYI, "share mode security\n"); + + /* one byte, so no need to convert this or EncryptionKeyLen from + little endian */ + server->maxReq = min_t(unsigned int, le16_to_cpu(pSMBr->MaxMpxCount), + cifs_max_pending); + set_credits(server, server->maxReq); + /* probably no need to store and check maxvcs */ + server->maxBuf = le32_to_cpu(pSMBr->MaxBufferSize); + /* set up max_read for readahead check */ + server->max_read = server->maxBuf; + server->max_rw = le32_to_cpu(pSMBr->MaxRawSize); + cifs_dbg(NOISY, "Max buf = %d\n", ses->server->maxBuf); + server->capabilities = le32_to_cpu(pSMBr->Capabilities); + server->session_key_id = pSMBr->SessionKey; + server->timeAdj = (int)(__s16)le16_to_cpu(pSMBr->ServerTimeZone); + server->timeAdj *= 60; + + if (pSMBr->EncryptionKeyLength == CIFS_CRYPTO_KEY_SIZE) { + server->negflavor = CIFS_NEGFLAVOR_UNENCAP; + memcpy(ses->server->cryptkey, pSMBr->u.EncryptionKey, + CIFS_CRYPTO_KEY_SIZE); + } else if (pSMBr->hdr.Flags2 & SMBFLG2_EXT_SEC || + server->capabilities & CAP_EXTENDED_SECURITY) { + server->negflavor = CIFS_NEGFLAVOR_EXTENDED; + rc = decode_ext_sec_blob(ses, pSMBr); + } else if (server->sec_mode & SECMODE_PW_ENCRYPT) { + /* no crypt key only if plain text pwd */ + rc = smb_EIO(smb_eio_trace_neg_no_crypt_key); + } else { + server->negflavor = CIFS_NEGFLAVOR_UNENCAP; + server->capabilities &= ~CAP_EXTENDED_SECURITY; + } + + if (!rc) + rc = cifs_enable_signing(server, ses->sign); +neg_err_exit: + cifs_buf_release(pSMB); + + cifs_dbg(FYI, "negprot rc %d\n", rc); + return rc; +} + +/* + * Issue a TREE_CONNECT request. + */ +int +CIFSTCon(const unsigned int xid, struct cifs_ses *ses, + const char *tree, struct cifs_tcon *tcon, + const struct nls_table *nls_codepage) +{ + struct smb_hdr *smb_buffer; + struct smb_hdr *smb_buffer_response; + TCONX_REQ *pSMB; + TCONX_RSP *pSMBr; + unsigned char *bcc_ptr; + int rc = 0; + int length, in_len; + __u16 bytes_left, count; + + if (ses == NULL) + return smb_EIO(smb_eio_trace_null_pointers); + + smb_buffer = cifs_buf_get(); + if (smb_buffer == NULL) + return -ENOMEM; + + smb_buffer_response = smb_buffer; + + in_len = header_assemble(smb_buffer, SMB_COM_TREE_CONNECT_ANDX, + NULL /*no tid */, 4 /*wct */); + + smb_buffer->Mid = get_next_mid(ses->server); + smb_buffer->Uid = ses->Suid; + pSMB = (TCONX_REQ *) smb_buffer; + pSMBr = (TCONX_RSP *) smb_buffer_response; + + pSMB->AndXCommand = 0xFF; + pSMB->Flags = cpu_to_le16(TCON_EXTENDED_SECINFO); + bcc_ptr = &pSMB->Password[0]; + + pSMB->PasswordLength = cpu_to_le16(1); /* minimum */ + *bcc_ptr = 0; /* password is null byte */ + bcc_ptr++; /* skip password */ + /* already aligned so no need to do it below */ + + if (ses->server->sign) + smb_buffer->Flags2 |= SMBFLG2_SECURITY_SIGNATURE; + + if (ses->capabilities & CAP_STATUS32) + smb_buffer->Flags2 |= SMBFLG2_ERR_STATUS; + + if (ses->capabilities & CAP_DFS) + smb_buffer->Flags2 |= SMBFLG2_DFS; + + if (ses->capabilities & CAP_UNICODE) { + smb_buffer->Flags2 |= SMBFLG2_UNICODE; + length = + cifs_strtoUTF16((__le16 *) bcc_ptr, tree, + 6 /* max utf8 char length in bytes */ * + (/* server len*/ + 256 /* share len */), nls_codepage); + bcc_ptr += 2 * length; /* convert num 16 bit words to bytes */ + bcc_ptr += 2; /* skip trailing null */ + } else { /* ASCII */ + strcpy(bcc_ptr, tree); + bcc_ptr += strlen(tree) + 1; + } + strcpy(bcc_ptr, "?????"); + bcc_ptr += strlen("?????"); + bcc_ptr += 1; + count = bcc_ptr - &pSMB->Password[0]; + in_len += count; + pSMB->ByteCount = cpu_to_le16(count); + + rc = SendReceive(xid, ses, smb_buffer, in_len, smb_buffer_response, + &length, 0); + + /* above now done in SendReceive */ + if (rc == 0) { + bool is_unicode; + + tcon->tid = smb_buffer_response->Tid; + bcc_ptr = pByteArea(smb_buffer_response); + bytes_left = get_bcc(smb_buffer_response); + length = strnlen(bcc_ptr, bytes_left - 2); + if (smb_buffer->Flags2 & SMBFLG2_UNICODE) + is_unicode = true; + else + is_unicode = false; + + + /* skip service field (NB: this field is always ASCII) */ + if (length == 3) { + if ((bcc_ptr[0] == 'I') && (bcc_ptr[1] == 'P') && + (bcc_ptr[2] == 'C')) { + cifs_dbg(FYI, "IPC connection\n"); + tcon->ipc = true; + tcon->pipe = true; + } + } else if (length == 2) { + if ((bcc_ptr[0] == 'A') && (bcc_ptr[1] == ':')) { + /* the most common case */ + cifs_dbg(FYI, "disk share connection\n"); + } + } + bcc_ptr += length + 1; + bytes_left -= (length + 1); + strscpy(tcon->tree_name, tree, sizeof(tcon->tree_name)); + + /* mostly informational -- no need to fail on error here */ + kfree(tcon->nativeFileSystem); + tcon->nativeFileSystem = cifs_strndup_from_utf16(bcc_ptr, + bytes_left, is_unicode, + nls_codepage); + + cifs_dbg(FYI, "nativeFileSystem=%s\n", tcon->nativeFileSystem); + + if ((smb_buffer_response->WordCount == 3) || + (smb_buffer_response->WordCount == 7)) + /* field is in same location */ + tcon->Flags = le16_to_cpu(pSMBr->OptionalSupport); + else + tcon->Flags = 0; + cifs_dbg(FYI, "Tcon flags: 0x%x\n", tcon->Flags); + + /* + * reset_cifs_unix_caps calls QFSInfo which requires + * need_reconnect to be false, but we would not need to call + * reset_caps if this were not a reconnect case so must check + * need_reconnect flag here. The caller will also clear + * need_reconnect when tcon was successful but needed to be + * cleared earlier in the case of unix extensions reconnect + */ + if (tcon->need_reconnect && tcon->unix_ext) { + cifs_dbg(FYI, "resetting caps for %s\n", tcon->tree_name); + tcon->need_reconnect = false; + reset_cifs_unix_caps(xid, tcon, NULL, NULL); + } + } + cifs_buf_release(smb_buffer); + return rc; +} + +int +CIFSSMBTDis(const unsigned int xid, struct cifs_tcon *tcon) +{ + struct smb_hdr *smb_buffer; + unsigned int in_len; + int rc = 0; + + cifs_dbg(FYI, "In tree disconnect\n"); + + /* BB: do we need to check this? These should never be NULL. */ + if ((tcon->ses == NULL) || (tcon->ses->server == NULL)) + return smb_EIO(smb_eio_trace_null_pointers); + + /* + * No need to return error on this operation if tid invalidated and + * closed on server already e.g. due to tcp session crashing. Also, + * the tcon is no longer on the list, so no need to take lock before + * checking this. + */ + spin_lock(&tcon->ses->chan_lock); + if ((tcon->need_reconnect) || CIFS_ALL_CHANS_NEED_RECONNECT(tcon->ses)) { + spin_unlock(&tcon->ses->chan_lock); + return smb_EIO(smb_eio_trace_tdis_in_reconnect); + } + spin_unlock(&tcon->ses->chan_lock); + + rc = small_smb_init(SMB_COM_TREE_DISCONNECT, 0, tcon, + (void **)&smb_buffer); + if (rc < 0) + return rc; + in_len = rc; + + rc = SendReceiveNoRsp(xid, tcon->ses, (char *)smb_buffer, in_len, 0); + cifs_small_buf_release(smb_buffer); + if (rc) + cifs_dbg(FYI, "Tree disconnect failed %d\n", rc); + + /* No need to return error on this operation if tid invalidated and + closed on server already e.g. due to tcp session crashing */ + if (rc == -EAGAIN) + rc = 0; + + return rc; +} + +/* + * This is a no-op for now. We're not really interested in the reply, but + * rather in the fact that the server sent one and that server->lstrp + * gets updated. + * + * FIXME: maybe we should consider checking that the reply matches request? + */ +static void +cifs_echo_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid) +{ + struct cifs_credits credits = { .value = 1, .instance = 0 }; + + release_mid(server, mid); + add_credits(server, &credits, CIFS_ECHO_OP); +} + +int +CIFSSMBEcho(struct TCP_Server_Info *server) +{ + ECHO_REQ *smb; + int rc = 0; + struct kvec iov[1]; + struct smb_rqst rqst = { + .rq_iov = iov, + .rq_nvec = ARRAY_SIZE(iov), + }; + unsigned int in_len; + + cifs_dbg(FYI, "In echo request\n"); + + rc = small_smb_init(SMB_COM_ECHO, 0, NULL, (void **)&smb); + if (rc < 0) + return rc; + in_len = rc; + + if (server->capabilities & CAP_UNICODE) + smb->hdr.Flags2 |= SMBFLG2_UNICODE; + + /* set up echo request */ + smb->hdr.Tid = 0xffff; + smb->hdr.WordCount = 1; + put_unaligned_le16(1, &smb->EchoCount); + put_bcc(1, &smb->hdr); + smb->Data[0] = 'a'; + in_len += 3; + + iov[0].iov_len = in_len; + iov[0].iov_base = smb; + + rc = cifs_call_async(server, &rqst, NULL, cifs_echo_callback, NULL, + server, CIFS_NON_BLOCKING | CIFS_ECHO_OP, NULL); + if (rc) + cifs_dbg(FYI, "Echo request failed: %d\n", rc); + + cifs_small_buf_release(smb); + + return rc; +} + +int +CIFSSMBLogoff(const unsigned int xid, struct cifs_ses *ses) +{ + LOGOFF_ANDX_REQ *pSMB; + unsigned int in_len; + int rc = 0; + + cifs_dbg(FYI, "In SMBLogoff for session disconnect\n"); + + /* + * BB: do we need to check validity of ses and server? They should + * always be valid since we have an active reference. If not, that + * should probably be a BUG() + */ + if (!ses || !ses->server) + return smb_EIO(smb_eio_trace_null_pointers); + + mutex_lock(&ses->session_mutex); + spin_lock(&ses->chan_lock); + if (CIFS_ALL_CHANS_NEED_RECONNECT(ses)) { + spin_unlock(&ses->chan_lock); + goto session_already_dead; /* no need to send SMBlogoff if uid + already closed due to reconnect */ + } + spin_unlock(&ses->chan_lock); + + rc = small_smb_init(SMB_COM_LOGOFF_ANDX, 2, NULL, (void **)&pSMB); + if (rc < 0) { + mutex_unlock(&ses->session_mutex); + return rc; + } + in_len = rc; + + pSMB->hdr.Mid = get_next_mid(ses->server); + + if (ses->server->sign) + pSMB->hdr.Flags2 |= SMBFLG2_SECURITY_SIGNATURE; + + pSMB->hdr.Uid = ses->Suid; + + pSMB->AndXCommand = 0xFF; + rc = SendReceiveNoRsp(xid, ses, (char *) pSMB, in_len, 0); + cifs_small_buf_release(pSMB); +session_already_dead: + mutex_unlock(&ses->session_mutex); + + /* if session dead then we do not need to do ulogoff, + since server closed smb session, no sense reporting + error */ + if (rc == -EAGAIN) + rc = 0; + return rc; +} + +int +CIFSPOSIXDelFile(const unsigned int xid, struct cifs_tcon *tcon, + const char *fileName, __u16 type, + const struct nls_table *nls_codepage, int remap) +{ + TRANSACTION2_SPI_REQ *pSMB = NULL; + TRANSACTION2_SPI_RSP *pSMBr = NULL; + struct unlink_psx_rq *pRqD; + unsigned int in_len; + int name_len; + int rc = 0; + int bytes_returned = 0; + __u16 params, param_offset, offset, byte_count; + + cifs_dbg(FYI, "In POSIX delete\n"); +PsxDelete: + rc = smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + name_len = + cifsConvertToUTF16((__le16 *) pSMB->FileName, fileName, + PATH_MAX, nls_codepage, remap); + name_len++; /* trailing null */ + name_len *= 2; + } else { + name_len = copy_path_name(pSMB->FileName, fileName); + } + + params = 6 + name_len; + pSMB->MaxParameterCount = cpu_to_le16(2); + pSMB->MaxDataCount = 0; /* BB double check this with jra */ + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + param_offset = offsetof(struct smb_com_transaction2_spi_req, + InformationLevel); + offset = param_offset + params; + + /* Setup pointer to Request Data (inode type). */ + pRqD = (struct unlink_psx_rq *)((char *)(pSMB) + offset); + pRqD->type = cpu_to_le16(type); + pSMB->ParameterOffset = cpu_to_le16(param_offset); + pSMB->DataOffset = cpu_to_le16(offset); + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_SET_PATH_INFORMATION); + byte_count = 3 /* pad */ + params + sizeof(struct unlink_psx_rq); + + pSMB->DataCount = cpu_to_le16(sizeof(struct unlink_psx_rq)); + pSMB->TotalDataCount = cpu_to_le16(sizeof(struct unlink_psx_rq)); + pSMB->ParameterCount = cpu_to_le16(params); + pSMB->TotalParameterCount = pSMB->ParameterCount; + pSMB->InformationLevel = cpu_to_le16(SMB_POSIX_UNLINK); + pSMB->Reserved4 = 0; + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) + cifs_dbg(FYI, "Posix delete returned %d\n", rc); + cifs_buf_release(pSMB); + + cifs_stats_inc(&tcon->stats.cifs_stats.num_deletes); + + if (rc == -EAGAIN) + goto PsxDelete; + + return rc; +} + +int +CIFSSMBDelFile(const unsigned int xid, struct cifs_tcon *tcon, const char *name, + struct cifs_sb_info *cifs_sb, struct dentry *dentry) +{ + DELETE_FILE_REQ *pSMB = NULL; + DELETE_FILE_RSP *pSMBr = NULL; + unsigned int in_len; + int rc = 0; + int bytes_returned; + int name_len; + int remap = cifs_remap(cifs_sb); + +DelFileRetry: + rc = smb_init(SMB_COM_DELETE, 1, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + name_len = cifsConvertToUTF16((__le16 *) pSMB->fileName, name, + PATH_MAX, cifs_sb->local_nls, + remap); + name_len++; /* trailing null */ + name_len *= 2; + } else { + name_len = copy_path_name(pSMB->fileName, name); + } + pSMB->SearchAttributes = + cpu_to_le16(ATTR_READONLY | ATTR_HIDDEN | ATTR_SYSTEM); + pSMB->BufferFormat = 0x04; + in_len += name_len + 1; + pSMB->ByteCount = cpu_to_le16(name_len + 1); + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + cifs_stats_inc(&tcon->stats.cifs_stats.num_deletes); + if (rc) + cifs_dbg(FYI, "Error in RMFile = %d\n", rc); + + cifs_buf_release(pSMB); + if (rc == -EAGAIN) + goto DelFileRetry; + + return rc; +} + +int +CIFSSMBRmDir(const unsigned int xid, struct cifs_tcon *tcon, const char *name, + struct cifs_sb_info *cifs_sb) +{ + DELETE_DIRECTORY_REQ *pSMB = NULL; + DELETE_DIRECTORY_RSP *pSMBr = NULL; + unsigned int in_len; + int rc = 0; + int bytes_returned; + int name_len; + int remap = cifs_remap(cifs_sb); + + cifs_dbg(FYI, "In CIFSSMBRmDir\n"); +RmDirRetry: + rc = smb_init(SMB_COM_DELETE_DIRECTORY, 0, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + name_len = cifsConvertToUTF16((__le16 *) pSMB->DirName, name, + PATH_MAX, cifs_sb->local_nls, + remap); + name_len++; /* trailing null */ + name_len *= 2; + } else { + name_len = copy_path_name(pSMB->DirName, name); + } + + pSMB->BufferFormat = 0x04; + in_len += name_len + 1; + pSMB->ByteCount = cpu_to_le16(name_len + 1); + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + cifs_stats_inc(&tcon->stats.cifs_stats.num_rmdirs); + if (rc) + cifs_dbg(FYI, "Error in RMDir = %d\n", rc); + + cifs_buf_release(pSMB); + if (rc == -EAGAIN) + goto RmDirRetry; + return rc; +} + +int +CIFSSMBMkDir(const unsigned int xid, struct inode *inode, umode_t mode, + struct cifs_tcon *tcon, const char *name, + struct cifs_sb_info *cifs_sb) +{ + int rc = 0; + CREATE_DIRECTORY_REQ *pSMB = NULL; + CREATE_DIRECTORY_RSP *pSMBr = NULL; + unsigned int in_len; + int bytes_returned; + int name_len; + int remap = cifs_remap(cifs_sb); + + cifs_dbg(FYI, "In CIFSSMBMkDir\n"); +MkDirRetry: + rc = smb_init(SMB_COM_CREATE_DIRECTORY, 0, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + name_len = cifsConvertToUTF16((__le16 *) pSMB->DirName, name, + PATH_MAX, cifs_sb->local_nls, + remap); + name_len++; /* trailing null */ + name_len *= 2; + } else { + name_len = copy_path_name(pSMB->DirName, name); + } + + pSMB->BufferFormat = 0x04; + in_len += name_len + 1; + pSMB->ByteCount = cpu_to_le16(name_len + 1); + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + cifs_stats_inc(&tcon->stats.cifs_stats.num_mkdirs); + if (rc) + cifs_dbg(FYI, "Error in Mkdir = %d\n", rc); + + cifs_buf_release(pSMB); + if (rc == -EAGAIN) + goto MkDirRetry; + return rc; +} + +int +CIFSPOSIXCreate(const unsigned int xid, struct cifs_tcon *tcon, + __u32 posix_flags, __u64 mode, __u16 *netfid, + FILE_UNIX_BASIC_INFO *pRetData, __u32 *pOplock, + const char *name, const struct nls_table *nls_codepage, + int remap) +{ + TRANSACTION2_SPI_REQ *pSMB = NULL; + TRANSACTION2_SPI_RSP *pSMBr = NULL; + unsigned int in_len; + int name_len; + int rc = 0; + int bytes_returned = 0; + __u16 params, param_offset, offset, byte_count, count; + OPEN_PSX_REQ *pdata; + OPEN_PSX_RSP *psx_rsp; + + cifs_dbg(FYI, "In POSIX Create\n"); +PsxCreat: + rc = smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + name_len = + cifsConvertToUTF16((__le16 *) pSMB->FileName, name, + PATH_MAX, nls_codepage, remap); + name_len++; /* trailing null */ + name_len *= 2; + } else { + name_len = copy_path_name(pSMB->FileName, name); + } + + params = 6 + name_len; + count = sizeof(OPEN_PSX_REQ); + pSMB->MaxParameterCount = cpu_to_le16(2); + pSMB->MaxDataCount = cpu_to_le16(1000); /* large enough */ + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + param_offset = offsetof(struct smb_com_transaction2_spi_req, + InformationLevel); + offset = param_offset + params; + pdata = (OPEN_PSX_REQ *)((char *)(pSMB) + offset); + pdata->Level = cpu_to_le16(SMB_QUERY_FILE_UNIX_BASIC); + pdata->Permissions = cpu_to_le64(mode); + pdata->PosixOpenFlags = cpu_to_le32(posix_flags); + pdata->OpenFlags = cpu_to_le32(*pOplock); + pSMB->ParameterOffset = cpu_to_le16(param_offset); + pSMB->DataOffset = cpu_to_le16(offset); + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_SET_PATH_INFORMATION); + byte_count = 3 /* pad */ + params + count; + + pSMB->DataCount = cpu_to_le16(count); + pSMB->ParameterCount = cpu_to_le16(params); + pSMB->TotalDataCount = pSMB->DataCount; + pSMB->TotalParameterCount = pSMB->ParameterCount; + pSMB->InformationLevel = cpu_to_le16(SMB_POSIX_OPEN); + pSMB->Reserved4 = 0; + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) { + cifs_dbg(FYI, "Posix create returned %d\n", rc); + goto psx_create_err; + } + + cifs_dbg(FYI, "copying inode info\n"); + rc = validate_t2((struct smb_t2_rsp *)pSMBr); + + if (rc || get_bcc(&pSMBr->hdr) < sizeof(OPEN_PSX_RSP)) { + rc = smb_EIO2(smb_eio_trace_create_rsp_too_small, + get_bcc(&pSMBr->hdr), sizeof(OPEN_PSX_RSP)); + goto psx_create_err; + } + + /* copy return information to pRetData */ + psx_rsp = (OPEN_PSX_RSP *) + ((char *)pSMBr + le16_to_cpu(pSMBr->t2.DataOffset)); + + *pOplock = le16_to_cpu(psx_rsp->OplockFlags); + if (netfid) + *netfid = psx_rsp->Fid; /* cifs fid stays in le */ + /* Let caller know file was created so we can set the mode. */ + /* Do we care about the CreateAction in any other cases? */ + if (cpu_to_le32(FILE_CREATE) == psx_rsp->CreateAction) + *pOplock |= CIFS_CREATE_ACTION; + /* check to make sure response data is there */ + if (psx_rsp->ReturnedLevel != cpu_to_le16(SMB_QUERY_FILE_UNIX_BASIC)) { + pRetData->Type = cpu_to_le32(-1); /* unknown */ + cifs_dbg(NOISY, "unknown type\n"); + } else { + if (get_bcc(&pSMBr->hdr) < sizeof(OPEN_PSX_RSP) + + sizeof(FILE_UNIX_BASIC_INFO)) { + cifs_dbg(VFS, "Open response data too small\n"); + pRetData->Type = cpu_to_le32(-1); + goto psx_create_err; + } + memcpy(pRetData, + (char *)psx_rsp + sizeof(OPEN_PSX_RSP), + sizeof(*pRetData)); + } + +psx_create_err: + cifs_buf_release(pSMB); + + if (posix_flags & SMB_O_DIRECTORY) + cifs_stats_inc(&tcon->stats.cifs_stats.num_posixmkdirs); + else + cifs_stats_inc(&tcon->stats.cifs_stats.num_posixopens); + + if (rc == -EAGAIN) + goto PsxCreat; + + return rc; +} + +static __u16 convert_disposition(int disposition) +{ + __u16 ofun = 0; + + switch (disposition) { + case FILE_SUPERSEDE: + ofun = SMBOPEN_OCREATE | SMBOPEN_OTRUNC; + break; + case FILE_OPEN: + ofun = SMBOPEN_OAPPEND; + break; + case FILE_CREATE: + ofun = SMBOPEN_OCREATE; + break; + case FILE_OPEN_IF: + ofun = SMBOPEN_OCREATE | SMBOPEN_OAPPEND; + break; + case FILE_OVERWRITE: + ofun = SMBOPEN_OTRUNC; + break; + case FILE_OVERWRITE_IF: + ofun = SMBOPEN_OCREATE | SMBOPEN_OTRUNC; + break; + default: + cifs_dbg(FYI, "unknown disposition %d\n", disposition); + ofun = SMBOPEN_OAPPEND; /* regular open */ + } + return ofun; +} + +static int +access_flags_to_smbopen_mode(const int access_flags) +{ + /* + * SYSTEM_SECURITY grants both read and write access to SACL, treat is as read/write. + * MAXIMUM_ALLOWED grants as many access as possible, so treat it as read/write too. + * SYNCHRONIZE as is does not grant any specific access, so do not check its mask. + * If only SYNCHRONIZE bit is specified then fallback to read access. + */ + bool with_write_flags = access_flags & (FILE_WRITE_DATA | FILE_APPEND_DATA | FILE_WRITE_EA | + FILE_DELETE_CHILD | FILE_WRITE_ATTRIBUTES | DELETE | + WRITE_DAC | WRITE_OWNER | SYSTEM_SECURITY | + MAXIMUM_ALLOWED | GENERIC_WRITE | GENERIC_ALL); + bool with_read_flags = access_flags & (FILE_READ_DATA | FILE_READ_EA | FILE_EXECUTE | + FILE_READ_ATTRIBUTES | READ_CONTROL | + SYSTEM_SECURITY | MAXIMUM_ALLOWED | GENERIC_ALL | + GENERIC_EXECUTE | GENERIC_READ); + bool with_execute_flags = access_flags & (FILE_EXECUTE | MAXIMUM_ALLOWED | GENERIC_ALL | + GENERIC_EXECUTE); + + if (with_write_flags && with_read_flags) + return SMBOPEN_READWRITE; + else if (with_write_flags) + return SMBOPEN_WRITE; + else if (with_execute_flags) + return SMBOPEN_EXECUTE; + else + return SMBOPEN_READ; +} + +int +SMBLegacyOpen(const unsigned int xid, struct cifs_tcon *tcon, + const char *fileName, const int openDisposition, + const int access_flags, const int create_options, __u16 *netfid, + int *pOplock, FILE_ALL_INFO *pfile_info, + const struct nls_table *nls_codepage, int remap) +{ + int rc; + OPENX_REQ *pSMB = NULL; + OPENX_RSP *pSMBr = NULL; + unsigned int in_len; + int bytes_returned; + int name_len; + __u16 count; + +OldOpenRetry: + rc = smb_init(SMB_COM_OPEN_ANDX, 15, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + pSMB->AndXCommand = 0xFF; /* none */ + + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + count = 1; /* account for one byte pad to word boundary */ + name_len = + cifsConvertToUTF16((__le16 *) (pSMB->fileName + 1), + fileName, PATH_MAX, nls_codepage, remap); + name_len++; /* trailing null */ + name_len *= 2; + } else { + count = 0; /* no pad */ + name_len = copy_path_name(pSMB->fileName, fileName); + } + if (*pOplock & REQ_OPLOCK) + pSMB->OpenFlags = cpu_to_le16(REQ_OPLOCK); + else if (*pOplock & REQ_BATCHOPLOCK) + pSMB->OpenFlags = cpu_to_le16(REQ_BATCHOPLOCK); + + pSMB->OpenFlags |= cpu_to_le16(REQ_MORE_INFO); + pSMB->Mode = cpu_to_le16(access_flags_to_smbopen_mode(access_flags)); + pSMB->Mode |= cpu_to_le16(0x40); /* deny none */ + /* set file as system file if special file such as fifo, + * socket, char or block and server expecting SFU style and + no Unix extensions */ + + if (create_options & CREATE_OPTION_SPECIAL) + pSMB->FileAttributes = cpu_to_le16(ATTR_SYSTEM); + else /* BB FIXME BB */ + pSMB->FileAttributes = cpu_to_le16(0/*ATTR_NORMAL*/); + + if (create_options & CREATE_OPTION_READONLY) + pSMB->FileAttributes |= cpu_to_le16(ATTR_READONLY); + + /* BB FIXME BB */ +/* pSMB->CreateOptions = cpu_to_le32(create_options & + CREATE_OPTIONS_MASK); */ + /* BB FIXME END BB */ + + pSMB->Sattr = cpu_to_le16(ATTR_HIDDEN | ATTR_SYSTEM | ATTR_DIRECTORY); + pSMB->OpenFunction = cpu_to_le16(convert_disposition(openDisposition)); + count += name_len; + in_len += count; + + pSMB->ByteCount = cpu_to_le16(count); + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *)pSMBr, &bytes_returned, 0); + cifs_stats_inc(&tcon->stats.cifs_stats.num_opens); + if (rc) { + cifs_dbg(FYI, "Error in Open = %d\n", rc); + } else { + /* BB verify if wct == 15 */ + +/* *pOplock = pSMBr->OplockLevel; */ /* BB take from action field*/ + + *netfid = pSMBr->Fid; /* cifs fid stays in le */ + /* Let caller know file was created so we can set the mode. */ + /* Do we care about the CreateAction in any other cases? */ + /* BB FIXME BB */ +/* if (cpu_to_le32(FILE_CREATE) == pSMBr->CreateAction) + *pOplock |= CIFS_CREATE_ACTION; */ + /* BB FIXME END */ + + if (pfile_info) { + pfile_info->CreationTime = 0; /* BB convert CreateTime*/ + pfile_info->LastAccessTime = 0; /* BB fixme */ + pfile_info->LastWriteTime = 0; /* BB fixme */ + pfile_info->ChangeTime = 0; /* BB fixme */ + pfile_info->Attributes = + cpu_to_le32(le16_to_cpu(pSMBr->FileAttributes)); + /* the file_info buf is endian converted by caller */ + pfile_info->AllocationSize = + cpu_to_le64(le32_to_cpu(pSMBr->EndOfFile)); + pfile_info->EndOfFile = pfile_info->AllocationSize; + pfile_info->NumberOfLinks = cpu_to_le32(1); + pfile_info->DeletePending = 0; /* successful open = not delete pending */ + } + } + + cifs_buf_release(pSMB); + if (rc == -EAGAIN) + goto OldOpenRetry; + return rc; +} + +int +CIFS_open(const unsigned int xid, struct cifs_open_parms *oparms, int *oplock, + FILE_ALL_INFO *buf) +{ + int rc; + OPEN_REQ *req = NULL; + OPEN_RSP *rsp = NULL; + int bytes_returned; + int name_len; + __u16 count; + struct cifs_sb_info *cifs_sb = oparms->cifs_sb; + struct cifs_tcon *tcon = oparms->tcon; + int remap = cifs_remap(cifs_sb); + const struct nls_table *nls = cifs_sb->local_nls; + int create_options = oparms->create_options; + int desired_access = oparms->desired_access; + int disposition = oparms->disposition; + const char *path = oparms->path; + unsigned int in_len; + +openRetry: + rc = smb_init(SMB_COM_NT_CREATE_ANDX, 24, tcon, (void **)&req, + (void **)&rsp); + if (rc < 0) + return rc; + in_len = rc; + + /* no commands go after this */ + req->AndXCommand = 0xFF; + + if (req->hdr.Flags2 & SMBFLG2_UNICODE) { + /* account for one byte pad to word boundary */ + count = 1; + name_len = cifsConvertToUTF16((__le16 *)(req->fileName + 1), + path, PATH_MAX, nls, remap); + /* trailing null */ + name_len++; + name_len *= 2; + req->NameLength = cpu_to_le16(name_len); + } else { + /* BB improve check for buffer overruns BB */ + /* no pad */ + count = 0; + name_len = copy_path_name(req->fileName, path); + req->NameLength = cpu_to_le16(name_len); + } + + if (*oplock & REQ_OPLOCK) + req->OpenFlags = cpu_to_le32(REQ_OPLOCK); + else if (*oplock & REQ_BATCHOPLOCK) + req->OpenFlags = cpu_to_le32(REQ_BATCHOPLOCK); + + req->DesiredAccess = cpu_to_le32(desired_access); + req->AllocationSize = 0; + + /* + * Set file as system file if special file such as fifo, socket, char + * or block and server expecting SFU style and no Unix extensions. + */ + if (create_options & CREATE_OPTION_SPECIAL) + req->FileAttributes = cpu_to_le32(ATTR_SYSTEM); + else + req->FileAttributes = cpu_to_le32(ATTR_NORMAL); + + /* + * XP does not handle ATTR_POSIX_SEMANTICS but it helps speed up case + * sensitive checks for other servers such as Samba. + */ + if (tcon->ses->capabilities & CAP_UNIX) + req->FileAttributes |= cpu_to_le32(ATTR_POSIX_SEMANTICS); + + if (create_options & CREATE_OPTION_READONLY) + req->FileAttributes |= cpu_to_le32(ATTR_READONLY); + + req->ShareAccess = cpu_to_le32(FILE_SHARE_ALL); + req->CreateDisposition = cpu_to_le32(disposition); + req->CreateOptions = cpu_to_le32(create_options & CREATE_OPTIONS_MASK); + + /* BB Experiment with various impersonation levels and verify */ + req->ImpersonationLevel = cpu_to_le32(SECURITY_IMPERSONATION); + req->SecurityFlags = SECURITY_CONTEXT_TRACKING|SECURITY_EFFECTIVE_ONLY; + + count += name_len; + in_len += count; + + req->ByteCount = cpu_to_le16(count); + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *)req, in_len, + (struct smb_hdr *)rsp, &bytes_returned, 0); + cifs_stats_inc(&tcon->stats.cifs_stats.num_opens); + if (rc) { + cifs_dbg(FYI, "Error in Open = %d\n", rc); + cifs_buf_release(req); + if (rc == -EAGAIN) + goto openRetry; + return rc; + } + + /* 1 byte no need to le_to_cpu */ + *oplock = rsp->OplockLevel; + /* cifs fid stays in le */ + oparms->fid->netfid = rsp->Fid; + oparms->fid->access = desired_access; + + /* Let caller know file was created so we can set the mode. */ + /* Do we care about the CreateAction in any other cases? */ + if (cpu_to_le32(FILE_CREATE) == rsp->CreateAction) + *oplock |= CIFS_CREATE_ACTION; + + if (buf) { + /* copy commonly used attributes */ + memcpy(&buf->common_attributes, + &rsp->common_attributes, + sizeof(buf->common_attributes)); + /* the file_info buf is endian converted by caller */ + buf->AllocationSize = rsp->AllocationSize; + buf->EndOfFile = rsp->EndOfFile; + buf->NumberOfLinks = cpu_to_le32(1); + buf->DeletePending = 0; /* successful open = not delete pending */ + } + + cifs_buf_release(req); + return rc; +} + +static void +cifs_readv_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid) +{ + struct cifs_io_subrequest *rdata = mid->callback_data; + struct netfs_inode *ictx = netfs_inode(rdata->rreq->inode); + struct cifs_tcon *tcon = tlink_tcon(rdata->req->cfile->tlink); + struct inode *inode = &ictx->inode; + struct smb_rqst rqst = { .rq_iov = rdata->iov, + .rq_nvec = 1, + .rq_iter = rdata->subreq.io_iter }; + struct cifs_credits credits = { + .value = 1, + .instance = 0, + .rreq_debug_id = rdata->rreq->debug_id, + .rreq_debug_index = rdata->subreq.debug_index, + }; + unsigned int rreq_debug_id = rdata->rreq->debug_id; + unsigned int subreq_debug_index = rdata->subreq.debug_index; + + cifs_dbg(FYI, "%s: mid=%llu state=%d result=%d bytes=%zu\n", + __func__, mid->mid, mid->mid_state, rdata->result, + rdata->subreq.len); + + switch (mid->mid_state) { + case MID_RESPONSE_RECEIVED: + /* result already set, check signature */ + if (server->sign) { + int rc = 0; + + iov_iter_truncate(&rqst.rq_iter, rdata->got_bytes); + rc = cifs_verify_signature(&rqst, server, + mid->sequence_number); + if (rc) + cifs_dbg(VFS, "SMB signature verification returned error = %d\n", + rc); + } + /* FIXME: should this be counted toward the initiating task? */ + task_io_account_read(rdata->got_bytes); + cifs_stats_bytes_read(tcon, rdata->got_bytes); + break; + case MID_REQUEST_SUBMITTED: + trace_netfs_sreq(&rdata->subreq, netfs_sreq_trace_io_req_submitted); + goto do_retry; + case MID_RETRY_NEEDED: + trace_netfs_sreq(&rdata->subreq, netfs_sreq_trace_io_retry_needed); +do_retry: + __set_bit(NETFS_SREQ_NEED_RETRY, &rdata->subreq.flags); + rdata->result = -EAGAIN; + if (server->sign && rdata->got_bytes) + /* reset bytes number since we can not check a sign */ + rdata->got_bytes = 0; + /* FIXME: should this be counted toward the initiating task? */ + task_io_account_read(rdata->got_bytes); + cifs_stats_bytes_read(tcon, rdata->got_bytes); + break; + case MID_RESPONSE_MALFORMED: + trace_netfs_sreq(&rdata->subreq, netfs_sreq_trace_io_malformed); + rdata->result = smb_EIO(smb_eio_trace_read_rsp_malformed); + break; + default: + trace_netfs_sreq(&rdata->subreq, netfs_sreq_trace_io_unknown); + rdata->result = smb_EIO1(smb_eio_trace_read_mid_state_unknown, + mid->mid_state); + break; + } + + if (rdata->result == -ENODATA) { + rdata->result = 0; + __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags); + trace_smb3_read_err(rdata->rreq->debug_id, + rdata->subreq.debug_index, + rdata->xid, + rdata->req->cfile->fid.persistent_fid, + tcon->tid, tcon->ses->Suid, + rdata->subreq.start + rdata->subreq.transferred, + rdata->subreq.len - rdata->subreq.transferred, + rdata->result); + } else { + size_t trans = rdata->subreq.transferred + rdata->got_bytes; + if (trans < rdata->subreq.len && + rdata->subreq.start + trans >= netfs_read_remote_i_size(inode)) { + rdata->result = 0; + __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags); + } else if (rdata->got_bytes > 0) { + __set_bit(NETFS_SREQ_MADE_PROGRESS, &rdata->subreq.flags); + } + if (rdata->got_bytes) + __set_bit(NETFS_SREQ_MADE_PROGRESS, &rdata->subreq.flags); + trace_smb3_read_done(rdata->rreq->debug_id, + rdata->subreq.debug_index, + rdata->xid, + rdata->req->cfile->fid.persistent_fid, + tcon->tid, tcon->ses->Suid, + rdata->subreq.start + rdata->subreq.transferred, + rdata->got_bytes); + } + + trace_smb3_rw_credits(rreq_debug_id, subreq_debug_index, rdata->credits.value, + server->credits, server->in_flight, + 0, cifs_trace_rw_credits_read_response_clear); + rdata->credits.value = 0; + rdata->subreq.error = rdata->result; + rdata->subreq.transferred += rdata->got_bytes; + trace_netfs_sreq(&rdata->subreq, netfs_sreq_trace_io_progress); + netfs_read_subreq_terminated(&rdata->subreq); + release_mid(server, mid); + add_credits(server, &credits, 0); + trace_smb3_rw_credits(rreq_debug_id, subreq_debug_index, 0, + server->credits, server->in_flight, + credits.value, cifs_trace_rw_credits_read_response_add); +} + +/* cifs_async_readv - send an async write, and set up mid to handle result */ +int +cifs_async_readv(struct cifs_io_subrequest *rdata) +{ + int rc; + READ_REQ *smb = NULL; + int wct; + struct cifs_tcon *tcon = tlink_tcon(rdata->req->cfile->tlink); + struct smb_rqst rqst = { .rq_iov = rdata->iov, + .rq_nvec = 1 }; + unsigned int in_len; + + cifs_dbg(FYI, "%s: offset=%llu bytes=%zu\n", + __func__, rdata->subreq.start, rdata->subreq.len); + + if (tcon->ses->capabilities & CAP_LARGE_FILES) + wct = 12; + else { + wct = 10; /* old style read */ + if ((rdata->subreq.start >> 32) > 0) { + /* can not handle this big offset for old */ + return smb_EIO(smb_eio_trace_read_too_far); + } + } + + rc = small_smb_init(SMB_COM_READ_ANDX, wct, tcon, (void **)&smb); + if (rc < 0) + return rc; + in_len = rc; + + smb->hdr.Pid = cpu_to_le16((__u16)rdata->req->pid); + smb->hdr.PidHigh = cpu_to_le16((__u16)(rdata->req->pid >> 16)); + + smb->AndXCommand = 0xFF; /* none */ + smb->Fid = rdata->req->cfile->fid.netfid; + smb->OffsetLow = cpu_to_le32(rdata->subreq.start & 0xFFFFFFFF); + if (wct == 12) + smb->OffsetHigh = cpu_to_le32(rdata->subreq.start >> 32); + smb->Remaining = 0; + smb->MaxCount = cpu_to_le16(rdata->subreq.len & 0xFFFF); + smb->MaxCountHigh = cpu_to_le32(rdata->subreq.len >> 16); + if (wct == 12) + smb->ByteCount = 0; + else { + /* old style read */ + struct smb_com_readx_req *smbr = + (struct smb_com_readx_req *)smb; + smbr->ByteCount = 0; + } + + /* 4 for RFC1001 length + 1 for BCC */ + rdata->iov[0].iov_base = smb; + rdata->iov[0].iov_len = in_len; + + trace_smb3_read_enter(rdata->rreq->debug_id, + rdata->subreq.debug_index, + rdata->xid, + rdata->req->cfile->fid.netfid, + tcon->tid, tcon->ses->Suid, + rdata->subreq.start, rdata->subreq.len); + + rc = cifs_call_async(tcon->ses->server, &rqst, cifs_readv_receive, + cifs_readv_callback, NULL, rdata, 0, NULL); + + if (rc == 0) + cifs_stats_inc(&tcon->stats.cifs_stats.num_reads); + cifs_small_buf_release(smb); + return rc; +} + +int +CIFSSMBRead(const unsigned int xid, struct cifs_io_parms *io_parms, + unsigned int *nbytes, char **buf, int *pbuf_type) +{ + int rc = -EACCES; + READ_REQ *pSMB = NULL; + READ_RSP *pSMBr = NULL; + char *pReadData = NULL; + int wct; + int resp_buf_type = 0; + struct kvec iov[1]; + struct kvec rsp_iov; + __u32 pid = io_parms->pid; + __u16 netfid = io_parms->netfid; + __u64 offset = io_parms->offset; + struct cifs_tcon *tcon = io_parms->tcon; + unsigned int in_len; + unsigned int count = io_parms->length; + + cifs_dbg(FYI, "Reading %d bytes on fid %d\n", count, netfid); + if (tcon->ses->capabilities & CAP_LARGE_FILES) + wct = 12; + else { + wct = 10; /* old style read */ + if ((offset >> 32) > 0) { + /* can not handle this big offset for old */ + return smb_EIO(smb_eio_trace_read_too_far); + } + } + + *nbytes = 0; + rc = small_smb_init(SMB_COM_READ_ANDX, wct, tcon, (void **) &pSMB); + if (rc < 0) + return rc; + in_len = rc; + + pSMB->hdr.Pid = cpu_to_le16((__u16)pid); + pSMB->hdr.PidHigh = cpu_to_le16((__u16)(pid >> 16)); + + /* tcon and ses pointer are checked in smb_init */ + if (!tcon->ses->server) { + cifs_small_buf_release(pSMB); + return -ECONNABORTED; + } + + pSMB->AndXCommand = 0xFF; /* none */ + pSMB->Fid = netfid; + pSMB->OffsetLow = cpu_to_le32(offset & 0xFFFFFFFF); + if (wct == 12) + pSMB->OffsetHigh = cpu_to_le32(offset >> 32); + + pSMB->Remaining = 0; + pSMB->MaxCount = cpu_to_le16(count & 0xFFFF); + pSMB->MaxCountHigh = cpu_to_le32(count >> 16); + if (wct == 12) + pSMB->ByteCount = 0; /* no need to do le conversion since 0 */ + else { + /* old style read */ + struct smb_com_readx_req *pSMBW = + (struct smb_com_readx_req *)pSMB; + pSMBW->ByteCount = 0; + } + + iov[0].iov_base = (char *)pSMB; + iov[0].iov_len = in_len; + rc = SendReceive2(xid, tcon->ses, iov, 1, &resp_buf_type, + CIFS_LOG_ERROR, &rsp_iov); + cifs_small_buf_release(pSMB); + cifs_stats_inc(&tcon->stats.cifs_stats.num_reads); + pSMBr = (READ_RSP *)rsp_iov.iov_base; + if (rc) { + cifs_dbg(VFS, "Send error in read = %d\n", rc); + } else { + int data_length = le16_to_cpu(pSMBr->DataLengthHigh); + data_length = data_length << 16; + data_length += le16_to_cpu(pSMBr->DataLength); + *nbytes = data_length; + + /*check that DataLength would not go beyond end of SMB */ + if ((data_length > CIFSMaxBufSize) + || (data_length > count)) { + cifs_dbg(FYI, "bad length %d for count %d\n", + data_length, count); + rc = smb_EIO2(smb_eio_trace_read_overlarge, + data_length, count); + *nbytes = 0; + } else { + pReadData = (char *) (&pSMBr->hdr.Protocol) + + le16_to_cpu(pSMBr->DataOffset); +/* if (rc = copy_to_user(buf, pReadData, data_length)) { + cifs_dbg(VFS, "Faulting on read rc = %d\n",rc); + rc = -EFAULT; + }*/ /* can not use copy_to_user when using page cache*/ + if (*buf) + memcpy(*buf, pReadData, data_length); + } + } + + if (*buf) { + free_rsp_buf(resp_buf_type, rsp_iov.iov_base); + } else if (resp_buf_type != CIFS_NO_BUFFER) { + /* return buffer to caller to free */ + *buf = rsp_iov.iov_base; + if (resp_buf_type == CIFS_SMALL_BUFFER) + *pbuf_type = CIFS_SMALL_BUFFER; + else if (resp_buf_type == CIFS_LARGE_BUFFER) + *pbuf_type = CIFS_LARGE_BUFFER; + } /* else no valid buffer on return - leave as null */ + + /* Note: On -EAGAIN error only caller can retry on handle based calls + since file handle passed in no longer valid */ + return rc; +} + + +int +CIFSSMBWrite(const unsigned int xid, struct cifs_io_parms *io_parms, + unsigned int *nbytes, const char *buf) +{ + int rc = -EACCES; + WRITE_REQ *pSMB = NULL; + WRITE_RSP *pSMBr = NULL; + int bytes_returned, wct; + __u32 bytes_sent; + __u16 byte_count; + __u32 pid = io_parms->pid; + __u16 netfid = io_parms->netfid; + __u64 offset = io_parms->offset; + struct cifs_tcon *tcon = io_parms->tcon; + unsigned int count = io_parms->length, in_len; + + *nbytes = 0; + + /* cifs_dbg(FYI, "write at %lld %d bytes\n", offset, count);*/ + if (tcon->ses == NULL) + return -ECONNABORTED; + + if (tcon->ses->capabilities & CAP_LARGE_FILES) + wct = 14; + else { + wct = 12; + if ((offset >> 32) > 0) { + /* can not handle big offset for old srv */ + return smb_EIO(smb_eio_trace_write_too_far); + } + } + + rc = smb_init(SMB_COM_WRITE_ANDX, wct, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + pSMB->hdr.Pid = cpu_to_le16((__u16)pid); + pSMB->hdr.PidHigh = cpu_to_le16((__u16)(pid >> 16)); + + /* tcon and ses pointer are checked in smb_init */ + if (!tcon->ses->server) { + cifs_buf_release(pSMB); + return -ECONNABORTED; + } + + pSMB->AndXCommand = 0xFF; /* none */ + pSMB->Fid = netfid; + pSMB->OffsetLow = cpu_to_le32(offset & 0xFFFFFFFF); + if (wct == 14) + pSMB->OffsetHigh = cpu_to_le32(offset >> 32); + + pSMB->Reserved = 0xFFFFFFFF; + pSMB->WriteMode = 0; + pSMB->Remaining = 0; + + /* Can increase buffer size if buffer is big enough in some cases ie we + can send more if LARGE_WRITE_X capability returned by the server and if + our buffer is big enough or if we convert to iovecs on socket writes + and eliminate the copy to the CIFS buffer */ + if (tcon->ses->capabilities & CAP_LARGE_WRITE_X) { + bytes_sent = min_t(const unsigned int, CIFSMaxBufSize, count); + } else { + bytes_sent = (tcon->ses->server->maxBuf - MAX_CIFS_HDR_SIZE) + & ~0xFF; + } + + if (bytes_sent > count) + bytes_sent = count; + pSMB->DataOffset = + cpu_to_le16(offsetof(struct smb_com_write_req, Data)); + if (buf) + memcpy(pSMB->Data, buf, bytes_sent); + else if (count != 0) { + /* No buffer */ + cifs_buf_release(pSMB); + return -EINVAL; + } /* else setting file size with write of zero bytes */ + if (wct == 14) + byte_count = bytes_sent + 1; /* pad */ + else /* wct == 12 */ + byte_count = bytes_sent + 5; /* bigger pad, smaller smb hdr */ + + pSMB->DataLengthLow = cpu_to_le16(bytes_sent & 0xFFFF); + pSMB->DataLengthHigh = cpu_to_le16(bytes_sent >> 16); + in_len += byte_count; + + if (wct == 14) + pSMB->ByteCount = cpu_to_le16(byte_count); + else { /* old style write has byte count 4 bytes earlier + so 4 bytes pad */ + struct smb_com_writex_req *pSMBW = + (struct smb_com_writex_req *)pSMB; + pSMBW->ByteCount = cpu_to_le16(byte_count); + } + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + cifs_stats_inc(&tcon->stats.cifs_stats.num_writes); + if (rc) { + cifs_dbg(FYI, "Send error in write = %d\n", rc); + } else { + *nbytes = le16_to_cpu(pSMBr->CountHigh); + *nbytes = (*nbytes) << 16; + *nbytes += le16_to_cpu(pSMBr->Count); + + /* + * Mask off high 16 bits when bytes written as returned by the + * server is greater than bytes requested by the client. Some + * OS/2 servers are known to set incorrect CountHigh values. + */ + if (*nbytes > count) + *nbytes &= 0xFFFF; + } + + cifs_buf_release(pSMB); + + /* Note: On -EAGAIN error only caller can retry on handle based calls + since file handle passed in no longer valid */ + + return rc; +} + +/* + * Check the mid_state and signature on received buffer (if any), and queue the + * workqueue completion task. + */ +static void +cifs_writev_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid) +{ + struct cifs_io_subrequest *wdata = mid->callback_data; + struct cifs_tcon *tcon = tlink_tcon(wdata->req->cfile->tlink); + WRITE_RSP *smb = (WRITE_RSP *)mid->resp_buf; + struct cifs_credits credits = { + .value = 1, + .instance = 0, + .rreq_debug_id = wdata->rreq->debug_id, + .rreq_debug_index = wdata->subreq.debug_index, + }; + ssize_t result; + size_t written; + + switch (mid->mid_state) { + case MID_RESPONSE_RECEIVED: + result = cifs_check_receive(mid, tcon->ses->server, 0); + if (result != 0) + break; + + written = le16_to_cpu(smb->CountHigh); + written <<= 16; + written += le16_to_cpu(smb->Count); + /* + * Mask off high 16 bits when bytes written as returned + * by the server is greater than bytes requested by the + * client. OS/2 servers are known to set incorrect + * CountHigh values. + */ + if (written > wdata->subreq.len) + written &= 0xFFFF; + + if (written < wdata->subreq.len) { + result = -ENOSPC; + } else { + result = written; + if (written > 0) + __set_bit(NETFS_SREQ_MADE_PROGRESS, &wdata->subreq.flags); + } + break; + case MID_REQUEST_SUBMITTED: + trace_netfs_sreq(&wdata->subreq, netfs_sreq_trace_io_req_submitted); + __set_bit(NETFS_SREQ_NEED_RETRY, &wdata->subreq.flags); + result = -EAGAIN; + break; + case MID_RETRY_NEEDED: + trace_netfs_sreq(&wdata->subreq, netfs_sreq_trace_io_retry_needed); + __set_bit(NETFS_SREQ_NEED_RETRY, &wdata->subreq.flags); + result = -EAGAIN; + break; + case MID_RESPONSE_MALFORMED: + trace_netfs_sreq(&wdata->subreq, netfs_sreq_trace_io_malformed); + result = smb_EIO(smb_eio_trace_write_rsp_malformed); + break; + default: + trace_netfs_sreq(&wdata->subreq, netfs_sreq_trace_io_unknown); + result = smb_EIO1(smb_eio_trace_write_mid_state_unknown, + mid->mid_state); + break; + } + + trace_smb3_rw_credits(credits.rreq_debug_id, credits.rreq_debug_index, + wdata->credits.value, + server->credits, server->in_flight, + 0, cifs_trace_rw_credits_write_response_clear); + wdata->credits.value = 0; + cifs_write_subrequest_terminated(wdata, result); + release_mid(server, mid); + trace_smb3_rw_credits(credits.rreq_debug_id, credits.rreq_debug_index, 0, + server->credits, server->in_flight, + credits.value, cifs_trace_rw_credits_write_response_add); + add_credits(tcon->ses->server, &credits, 0); +} + +/* cifs_async_writev - send an async write, and set up mid to handle result */ +void +cifs_async_writev(struct cifs_io_subrequest *wdata) +{ + int rc = -EACCES; + WRITE_REQ *req = NULL; + int wct; + struct cifs_tcon *tcon = tlink_tcon(wdata->req->cfile->tlink); + struct kvec iov[1]; + struct smb_rqst rqst = { }; + unsigned int in_len; + + if (tcon->ses->capabilities & CAP_LARGE_FILES) { + wct = 14; + } else { + wct = 12; + if (wdata->subreq.start >> 32 > 0) { + /* can not handle big offset for old srv */ + rc = smb_EIO(smb_eio_trace_write_too_far); + goto out; + } + } + + rc = small_smb_init(SMB_COM_WRITE_ANDX, wct, tcon, (void **)&req); + if (rc < 0) + goto async_writev_out; + in_len = rc; + + req->hdr.Pid = cpu_to_le16((__u16)wdata->req->pid); + req->hdr.PidHigh = cpu_to_le16((__u16)(wdata->req->pid >> 16)); + + req->AndXCommand = 0xFF; /* none */ + req->Fid = wdata->req->cfile->fid.netfid; + req->OffsetLow = cpu_to_le32(wdata->subreq.start & 0xFFFFFFFF); + if (wct == 14) + req->OffsetHigh = cpu_to_le32(wdata->subreq.start >> 32); + req->Reserved = 0xFFFFFFFF; + req->WriteMode = 0; + req->Remaining = 0; + + req->DataOffset = + cpu_to_le16(offsetof(struct smb_com_write_req, Data)); + + iov[0].iov_base = req; + iov[0].iov_len = in_len + 1; /* +1 for BCC */ + + rqst.rq_iov = iov; + rqst.rq_nvec = 1; + rqst.rq_iter = wdata->subreq.io_iter; + + cifs_dbg(FYI, "async write at %llu %zu bytes\n", + wdata->subreq.start, wdata->subreq.len); + + req->DataLengthLow = cpu_to_le16(wdata->subreq.len & 0xFFFF); + req->DataLengthHigh = cpu_to_le16(wdata->subreq.len >> 16); + + if (wct == 14) { + in_len += wdata->subreq.len + 1; + put_bcc(wdata->subreq.len + 1, &req->hdr); + } else { + /* wct == 12 */ + struct smb_com_writex_req *reqw = + (struct smb_com_writex_req *)req; + in_len += wdata->subreq.len + 5; + put_bcc(wdata->subreq.len + 5, &reqw->hdr); + iov[0].iov_len += 4; /* pad bigger by four bytes */ + } + + rc = cifs_call_async(tcon->ses->server, &rqst, NULL, + cifs_writev_callback, NULL, wdata, 0, NULL); + /* Can't touch wdata if rc == 0 */ + if (rc == 0) + cifs_stats_inc(&tcon->stats.cifs_stats.num_writes); + +async_writev_out: + cifs_small_buf_release(req); +out: + if (rc) { + add_credits_and_wake_if(wdata->server, &wdata->credits, 0); + cifs_write_subrequest_terminated(wdata, rc); + } +} + +int +CIFSSMBWrite2(const unsigned int xid, struct cifs_io_parms *io_parms, + unsigned int *nbytes, struct kvec *iov, int n_vec) +{ + int rc; + WRITE_REQ *pSMB = NULL; + int wct; + int smb_hdr_len; + int resp_buf_type = 0; + __u32 pid = io_parms->pid; + __u16 netfid = io_parms->netfid; + __u64 offset = io_parms->offset; + struct cifs_tcon *tcon = io_parms->tcon; + unsigned int count = io_parms->length; + struct kvec rsp_iov; + unsigned int in_len; + + *nbytes = 0; + + cifs_dbg(FYI, "write2 at %lld %d bytes\n", (long long)offset, count); + + if (tcon->ses->capabilities & CAP_LARGE_FILES) { + wct = 14; + } else { + wct = 12; + if ((offset >> 32) > 0) { + /* can not handle big offset for old srv */ + return smb_EIO(smb_eio_trace_write_too_far); + } + } + rc = small_smb_init(SMB_COM_WRITE_ANDX, wct, tcon, (void **) &pSMB); + if (rc < 0) + return rc; + in_len = rc; + + pSMB->hdr.Pid = cpu_to_le16((__u16)pid); + pSMB->hdr.PidHigh = cpu_to_le16((__u16)(pid >> 16)); + + /* tcon and ses pointer are checked in smb_init */ + if (!tcon->ses->server) { + cifs_small_buf_release(pSMB); + return -ECONNABORTED; + } + + pSMB->AndXCommand = 0xFF; /* none */ + pSMB->Fid = netfid; + pSMB->OffsetLow = cpu_to_le32(offset & 0xFFFFFFFF); + if (wct == 14) + pSMB->OffsetHigh = cpu_to_le32(offset >> 32); + pSMB->Reserved = 0xFFFFFFFF; + pSMB->WriteMode = 0; + pSMB->Remaining = 0; + + pSMB->DataOffset = + cpu_to_le16(offsetof(struct smb_com_write_req, Data)); + + pSMB->DataLengthLow = cpu_to_le16(count & 0xFFFF); + pSMB->DataLengthHigh = cpu_to_le16(count >> 16); + /* header + 1 byte pad */ + smb_hdr_len = in_len + 1; + if (wct == 14) + in_len += count + 1; + else /* wct == 12 */ + in_len += count + 5; /* smb data starts later */ + if (wct == 14) + pSMB->ByteCount = cpu_to_le16(count + 1); + else /* wct == 12 */ /* bigger pad, smaller smb hdr, keep offset ok */ { + struct smb_com_writex_req *pSMBW = + (struct smb_com_writex_req *)pSMB; + pSMBW->ByteCount = cpu_to_le16(count + 5); + } + iov[0].iov_base = pSMB; + if (wct == 14) + iov[0].iov_len = smb_hdr_len + 4; + else /* wct == 12 pad bigger by four bytes */ + iov[0].iov_len = smb_hdr_len + 8; + + rc = SendReceive2(xid, tcon->ses, iov, n_vec + 1, &resp_buf_type, 0, + &rsp_iov); + cifs_small_buf_release(pSMB); + cifs_stats_inc(&tcon->stats.cifs_stats.num_writes); + if (rc) { + cifs_dbg(FYI, "Send error Write2 = %d\n", rc); + } else if (resp_buf_type == 0) { + /* presumably this can not happen, but best to be safe */ + rc = smb_EIO1(smb_eio_trace_write_bad_buf_type, resp_buf_type); + } else { + WRITE_RSP *pSMBr = (WRITE_RSP *)rsp_iov.iov_base; + *nbytes = le16_to_cpu(pSMBr->CountHigh); + *nbytes = (*nbytes) << 16; + *nbytes += le16_to_cpu(pSMBr->Count); + + /* + * Mask off high 16 bits when bytes written as returned by the + * server is greater than bytes requested by the client. OS/2 + * servers are known to set incorrect CountHigh values. + */ + if (*nbytes > count) + *nbytes &= 0xFFFF; + } + + free_rsp_buf(resp_buf_type, rsp_iov.iov_base); + + /* Note: On -EAGAIN error only caller can retry on handle based calls + since file handle passed in no longer valid */ + + return rc; +} + +int cifs_lockv(const unsigned int xid, struct cifs_tcon *tcon, + const __u16 netfid, const __u8 lock_type, const __u32 num_unlock, + const __u32 num_lock, LOCKING_ANDX_RANGE *buf) +{ + int rc = 0; + LOCK_REQ *pSMB = NULL; + struct kvec iov[2]; + struct kvec rsp_iov; + unsigned int in_len; + int resp_buf_type; + __u16 count; + + cifs_dbg(FYI, "cifs_lockv num lock %d num unlock %d\n", + num_lock, num_unlock); + + rc = small_smb_init(SMB_COM_LOCKING_ANDX, 8, tcon, (void **) &pSMB); + if (rc < 0) + return rc; + in_len = rc; + + pSMB->Timeout = 0; + pSMB->NumberOfLocks = cpu_to_le16(num_lock); + pSMB->NumberOfUnlocks = cpu_to_le16(num_unlock); + pSMB->LockType = lock_type; + pSMB->AndXCommand = 0xFF; /* none */ + pSMB->Fid = netfid; /* netfid stays le */ + + count = (num_unlock + num_lock) * sizeof(LOCKING_ANDX_RANGE); + in_len += count; + pSMB->ByteCount = cpu_to_le16(count); + + iov[0].iov_base = (char *)pSMB; + iov[0].iov_len = in_len - + (num_unlock + num_lock) * sizeof(LOCKING_ANDX_RANGE); + iov[1].iov_base = (char *)buf; + iov[1].iov_len = (num_unlock + num_lock) * sizeof(LOCKING_ANDX_RANGE); + + cifs_stats_inc(&tcon->stats.cifs_stats.num_locks); + rc = SendReceive2(xid, tcon->ses, iov, 2, &resp_buf_type, + CIFS_NO_RSP_BUF, &rsp_iov); + cifs_small_buf_release(pSMB); + if (rc) + cifs_dbg(FYI, "Send error in cifs_lockv = %d\n", rc); + + return rc; +} + +int +CIFSSMBLock(const unsigned int xid, struct cifs_tcon *tcon, + const __u16 smb_file_id, const __u32 netpid, const __u64 len, + const __u64 offset, const __u32 numUnlock, + const __u32 numLock, const __u8 lockType, + const bool waitFlag, const __u8 oplock_level) +{ + int rc = 0; + LOCK_REQ *pSMB = NULL; +/* LOCK_RSP *pSMBr = NULL; */ /* No response data other than rc to parse */ + unsigned int in_len; + int bytes_returned; + int flags = CIFS_WINDOWS_LOCK | CIFS_INTERRUPTIBLE_WAIT; + __u16 count; + + cifs_dbg(FYI, "CIFSSMBLock timeout %d numLock %d\n", + (int)waitFlag, numLock); + rc = small_smb_init(SMB_COM_LOCKING_ANDX, 8, tcon, (void **) &pSMB); + + if (rc < 0) + return rc; + in_len = rc; + + if (lockType == LOCKING_ANDX_OPLOCK_RELEASE) { + /* no response expected */ + flags = CIFS_NO_SRV_RSP | CIFS_NON_BLOCKING | CIFS_OBREAK_OP; + pSMB->Timeout = 0; + } else if (waitFlag) { + flags = CIFS_BLOCKING_OP; /* blocking operation, no timeout */ + pSMB->Timeout = cpu_to_le32(-1);/* blocking - do not time out */ + } else { + pSMB->Timeout = 0; + } + + pSMB->NumberOfLocks = cpu_to_le16(numLock); + pSMB->NumberOfUnlocks = cpu_to_le16(numUnlock); + pSMB->LockType = lockType; + pSMB->OplockLevel = oplock_level; + pSMB->AndXCommand = 0xFF; /* none */ + pSMB->Fid = smb_file_id; /* netfid stays le */ + + if ((numLock != 0) || (numUnlock != 0)) { + pSMB->Locks[0].Pid = cpu_to_le16(netpid); + /* BB where to store pid high? */ + pSMB->Locks[0].LengthLow = cpu_to_le32((u32)len); + pSMB->Locks[0].LengthHigh = cpu_to_le32((u32)(len>>32)); + pSMB->Locks[0].OffsetLow = cpu_to_le32((u32)offset); + pSMB->Locks[0].OffsetHigh = cpu_to_le32((u32)(offset>>32)); + count = sizeof(LOCKING_ANDX_RANGE); + } else { + /* oplock break */ + count = 0; + } + in_len += count; + pSMB->ByteCount = cpu_to_le16(count); + + if (waitFlag) + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMB, &bytes_returned, + flags); + else + rc = SendReceiveNoRsp(xid, tcon->ses, (char *)pSMB, in_len, flags); + cifs_small_buf_release(pSMB); + cifs_stats_inc(&tcon->stats.cifs_stats.num_locks); + if (rc) + cifs_dbg(FYI, "Send error in Lock = %d\n", rc); + + /* Note: On -EAGAIN error only caller can retry on handle based calls + since file handle passed in no longer valid */ + return rc; +} + +int +CIFSSMBPosixLock(const unsigned int xid, struct cifs_tcon *tcon, + const __u16 smb_file_id, const __u32 netpid, + const loff_t start_offset, const __u64 len, + struct file_lock *pLockData, const __u16 lock_type, + const bool waitFlag) +{ + struct smb_com_transaction2_sfi_req *pSMB = NULL; + struct smb_com_transaction2_sfi_rsp *pSMBr = NULL; + struct cifs_posix_lock *parm_data; + unsigned int in_len; + int rc = 0; + int sr_flags = CIFS_INTERRUPTIBLE_WAIT; + int bytes_returned = 0; + int resp_buf_type = 0; + __u16 params, param_offset, offset, byte_count, count; + struct kvec iov[1]; + struct kvec rsp_iov; + + cifs_dbg(FYI, "Posix Lock\n"); + + rc = small_smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB); + if (rc < 0) + return rc; + in_len = rc; + + pSMBr = (struct smb_com_transaction2_sfi_rsp *)pSMB; + + params = 6; + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Reserved2 = 0; + param_offset = offsetof(struct smb_com_transaction2_sfi_req, Fid); + offset = param_offset + params; + + count = sizeof(struct cifs_posix_lock); + pSMB->MaxParameterCount = cpu_to_le16(2); + pSMB->MaxDataCount = cpu_to_le16(1000); /* BB find max SMB from sess */ + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + if (pLockData) + pSMB->SubCommand = cpu_to_le16(TRANS2_QUERY_FILE_INFORMATION); + else + pSMB->SubCommand = cpu_to_le16(TRANS2_SET_FILE_INFORMATION); + byte_count = 3 /* pad */ + params + count; + pSMB->DataCount = cpu_to_le16(count); + pSMB->ParameterCount = cpu_to_le16(params); + pSMB->TotalDataCount = pSMB->DataCount; + pSMB->TotalParameterCount = pSMB->ParameterCount; + pSMB->ParameterOffset = cpu_to_le16(param_offset); + parm_data = (struct cifs_posix_lock *)(((char *)pSMB) + offset); + + parm_data->lock_type = cpu_to_le16(lock_type); + if (waitFlag) { + sr_flags |= CIFS_BLOCKING_OP; /* blocking operation, no timeout */ + parm_data->lock_flags = cpu_to_le16(1); + pSMB->Timeout = cpu_to_le32(-1); + } else + pSMB->Timeout = 0; + + parm_data->pid = cpu_to_le32(netpid); + parm_data->start = cpu_to_le64(start_offset); + parm_data->length = cpu_to_le64(len); /* normalize negative numbers */ + + pSMB->DataOffset = cpu_to_le16(offset); + pSMB->Fid = smb_file_id; + pSMB->InformationLevel = cpu_to_le16(SMB_SET_POSIX_LOCK); + pSMB->Reserved4 = 0; + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + if (waitFlag) { + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, + sr_flags); + } else { + iov[0].iov_base = (char *)pSMB; + iov[0].iov_len = in_len; + rc = SendReceive2(xid, tcon->ses, iov, 1 /* num iovecs */, + &resp_buf_type, sr_flags, &rsp_iov); + pSMBr = (struct smb_com_transaction2_sfi_rsp *)rsp_iov.iov_base; + } + cifs_small_buf_release(pSMB); + + if (rc) { + cifs_dbg(FYI, "Send error in Posix Lock = %d\n", rc); + } else if (pLockData) { + /* lock structure can be returned on get */ + __u16 data_offset; + __u16 data_count; + rc = validate_t2((struct smb_t2_rsp *)pSMBr); + + if (rc || get_bcc(&pSMBr->hdr) < sizeof(*parm_data)) { + rc = smb_EIO2(smb_eio_trace_lock_bcc_too_small, + get_bcc(&pSMBr->hdr), sizeof(*parm_data)); + goto plk_err_exit; + } + data_offset = le16_to_cpu(pSMBr->t2.DataOffset); + data_count = le16_to_cpu(pSMBr->t2.DataCount); + if (data_count < sizeof(struct cifs_posix_lock)) { + rc = smb_EIO2(smb_eio_trace_lock_data_too_small, + data_count, sizeof(struct cifs_posix_lock)); + goto plk_err_exit; + } + parm_data = (struct cifs_posix_lock *) + ((char *)&pSMBr->hdr.Protocol + data_offset); + if (parm_data->lock_type == cpu_to_le16(CIFS_UNLCK)) + pLockData->c.flc_type = F_UNLCK; + else { + if (parm_data->lock_type == + cpu_to_le16(CIFS_RDLCK)) + pLockData->c.flc_type = F_RDLCK; + else if (parm_data->lock_type == + cpu_to_le16(CIFS_WRLCK)) + pLockData->c.flc_type = F_WRLCK; + + pLockData->fl_start = le64_to_cpu(parm_data->start); + pLockData->fl_end = pLockData->fl_start + + (le64_to_cpu(parm_data->length) ? + le64_to_cpu(parm_data->length) - 1 : 0); + pLockData->c.flc_pid = -le32_to_cpu(parm_data->pid); + } + } + +plk_err_exit: + free_rsp_buf(resp_buf_type, rsp_iov.iov_base); + + /* Note: On -EAGAIN error only caller can retry on handle based calls + since file handle passed in no longer valid */ + + return rc; +} + + +int +CIFSSMBClose(const unsigned int xid, struct cifs_tcon *tcon, int smb_file_id) +{ + int rc = 0; + CLOSE_REQ *pSMB = NULL; + unsigned int in_len; + + cifs_dbg(FYI, "In CIFSSMBClose\n"); + +/* do not retry on dead session on close */ + rc = small_smb_init(SMB_COM_CLOSE, 3, tcon, (void **) &pSMB); + if (rc == -EAGAIN) + return 0; + if (rc < 0) + return rc; + in_len = rc; + + pSMB->FileID = (__u16) smb_file_id; + pSMB->LastWriteTime = 0xFFFFFFFF; + pSMB->ByteCount = 0; + rc = SendReceiveNoRsp(xid, tcon->ses, (char *) pSMB, in_len, 0); + cifs_small_buf_release(pSMB); + cifs_stats_inc(&tcon->stats.cifs_stats.num_closes); + if (rc) { + if (rc != -EINTR) { + /* EINTR is expected when user ctl-c to kill app */ + cifs_dbg(VFS, "Send error in Close = %d\n", rc); + } + } + + /* Since session is dead, file will be closed on server already */ + if (rc == -EAGAIN) + rc = 0; + + return rc; +} + +int +CIFSSMBFlush(const unsigned int xid, struct cifs_tcon *tcon, int smb_file_id) +{ + int rc = 0; + FLUSH_REQ *pSMB = NULL; + unsigned int in_len; + + cifs_dbg(FYI, "In CIFSSMBFlush\n"); + + rc = small_smb_init(SMB_COM_FLUSH, 1, tcon, (void **) &pSMB); + if (rc < 0) + return rc; + in_len = rc; + + pSMB->FileID = (__u16) smb_file_id; + pSMB->ByteCount = 0; + rc = SendReceiveNoRsp(xid, tcon->ses, (char *) pSMB, in_len, 0); + cifs_small_buf_release(pSMB); + cifs_stats_inc(&tcon->stats.cifs_stats.num_flushes); + if (rc) + cifs_dbg(VFS, "Send error in Flush = %d\n", rc); + + return rc; +} + +int CIFSSMBRename(const unsigned int xid, struct cifs_tcon *tcon, + struct dentry *source_dentry, + const char *from_name, const char *to_name, + struct cifs_sb_info *cifs_sb) +{ + int rc = 0; + RENAME_REQ *pSMB = NULL; + RENAME_RSP *pSMBr = NULL; + unsigned int in_len; + int bytes_returned; + int name_len, name_len2; + __u16 count; + int remap = cifs_remap(cifs_sb); + + cifs_dbg(FYI, "In CIFSSMBRename\n"); +renameRetry: + rc = smb_init(SMB_COM_RENAME, 1, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + pSMB->BufferFormat = 0x04; + pSMB->SearchAttributes = + cpu_to_le16(ATTR_READONLY | ATTR_HIDDEN | ATTR_SYSTEM | + ATTR_DIRECTORY); + + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + name_len = cifsConvertToUTF16((__le16 *) pSMB->OldFileName, + from_name, PATH_MAX, + cifs_sb->local_nls, remap); + name_len++; /* trailing null */ + name_len *= 2; + pSMB->OldFileName[name_len] = 0x04; /* pad */ + /* protocol requires ASCII signature byte on Unicode string */ + pSMB->OldFileName[name_len + 1] = 0x00; + name_len2 = + cifsConvertToUTF16((__le16 *)&pSMB->OldFileName[name_len+2], + to_name, PATH_MAX, cifs_sb->local_nls, + remap); + name_len2 += 1 /* trailing null */ + 1 /* Signature word */ ; + name_len2 *= 2; /* convert to bytes */ + } else { + name_len = copy_path_name(pSMB->OldFileName, from_name); + name_len2 = copy_path_name(pSMB->OldFileName+name_len+1, to_name); + pSMB->OldFileName[name_len] = 0x04; /* 2nd buffer format */ + name_len2++; /* signature byte */ + } + + count = 1 /* 1st signature byte */ + name_len + name_len2; + in_len += count; + pSMB->ByteCount = cpu_to_le16(count); + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + cifs_stats_inc(&tcon->stats.cifs_stats.num_renames); + if (rc) + cifs_dbg(FYI, "Send error in rename = %d\n", rc); + + cifs_buf_release(pSMB); + + if (rc == -EAGAIN) + goto renameRetry; + + return rc; +} + +int CIFSSMBRenameOpenFile(const unsigned int xid, struct cifs_tcon *pTcon, + int netfid, const char *target_name, + const struct nls_table *nls_codepage, int remap) +{ + struct smb_com_transaction2_sfi_req *pSMB = NULL; + struct smb_com_transaction2_sfi_rsp *pSMBr = NULL; + struct set_file_rename *rename_info; + unsigned int in_len; + char *data_offset; + char dummy_string[30]; + int rc = 0; + int bytes_returned = 0; + int len_of_str; + __u16 params, param_offset, offset, count, byte_count; + + cifs_dbg(FYI, "Rename to File by handle\n"); + rc = smb_init(SMB_COM_TRANSACTION2, 15, pTcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + params = 6; + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + param_offset = offsetof(struct smb_com_transaction2_sfi_req, Fid); + offset = param_offset + params; + + data_offset = (char *)(pSMB) + offset; + rename_info = (struct set_file_rename *) data_offset; + pSMB->MaxParameterCount = cpu_to_le16(2); + pSMB->MaxDataCount = cpu_to_le16(1000); /* BB find max SMB from sess */ + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_SET_FILE_INFORMATION); + byte_count = 3 /* pad */ + params; + pSMB->ParameterCount = cpu_to_le16(params); + pSMB->TotalParameterCount = pSMB->ParameterCount; + pSMB->ParameterOffset = cpu_to_le16(param_offset); + pSMB->DataOffset = cpu_to_le16(offset); + /* construct random name ".cifs_tmp" */ + rename_info->overwrite = cpu_to_le32(1); + rename_info->root_fid = 0; + /* unicode only call */ + if (target_name == NULL) { + sprintf(dummy_string, "cifs%x", pSMB->hdr.Mid); + len_of_str = + cifsConvertToUTF16((__le16 *)rename_info->target_name, + dummy_string, 24, nls_codepage, remap); + } else { + len_of_str = + cifsConvertToUTF16((__le16 *)rename_info->target_name, + target_name, PATH_MAX, nls_codepage, + remap); + } + rename_info->target_name_len = cpu_to_le32(2 * len_of_str); + count = sizeof(struct set_file_rename) + (2 * len_of_str); + byte_count += count; + pSMB->DataCount = cpu_to_le16(count); + pSMB->TotalDataCount = pSMB->DataCount; + pSMB->Fid = netfid; + pSMB->InformationLevel = + cpu_to_le16(SMB_SET_FILE_RENAME_INFORMATION); + pSMB->Reserved4 = 0; + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + rc = SendReceive(xid, pTcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + cifs_stats_inc(&pTcon->stats.cifs_stats.num_t2renames); + if (rc) + cifs_dbg(FYI, "Send error in Rename (by file handle) = %d\n", + rc); + + cifs_buf_release(pSMB); + + /* Note: On -EAGAIN error only caller can retry on handle based calls + since file handle passed in no longer valid */ + + return rc; +} + +int +CIFSUnixCreateSymLink(const unsigned int xid, struct cifs_tcon *tcon, + const char *fromName, const char *toName, + const struct nls_table *nls_codepage, int remap) +{ + TRANSACTION2_SPI_REQ *pSMB = NULL; + TRANSACTION2_SPI_RSP *pSMBr = NULL; + unsigned int in_len; + char *data_offset; + int name_len; + int name_len_target; + int rc = 0; + int bytes_returned = 0; + __u16 params, param_offset, offset, byte_count; + + cifs_dbg(FYI, "In Symlink Unix style\n"); +createSymLinkRetry: + rc = smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + name_len = + cifsConvertToUTF16((__le16 *) pSMB->FileName, fromName, + /* find define for this maxpathcomponent */ + PATH_MAX, nls_codepage, remap); + name_len++; /* trailing null */ + name_len *= 2; + + } else { + name_len = copy_path_name(pSMB->FileName, fromName); + } + params = 6 + name_len; + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + param_offset = offsetof(struct smb_com_transaction2_spi_req, + InformationLevel); + offset = param_offset + params; + + data_offset = (char *)pSMB + offset; + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + name_len_target = + cifsConvertToUTF16((__le16 *) data_offset, toName, + /* find define for this maxpathcomponent */ + PATH_MAX, nls_codepage, remap); + name_len_target++; /* trailing null */ + name_len_target *= 2; + } else { + name_len_target = copy_path_name(data_offset, toName); + } + + pSMB->MaxParameterCount = cpu_to_le16(2); + /* BB find exact max on data count below from sess */ + pSMB->MaxDataCount = cpu_to_le16(1000); + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_SET_PATH_INFORMATION); + byte_count = 3 /* pad */ + params + name_len_target; + pSMB->DataCount = cpu_to_le16(name_len_target); + pSMB->ParameterCount = cpu_to_le16(params); + pSMB->TotalDataCount = pSMB->DataCount; + pSMB->TotalParameterCount = pSMB->ParameterCount; + pSMB->ParameterOffset = cpu_to_le16(param_offset); + pSMB->DataOffset = cpu_to_le16(offset); + pSMB->InformationLevel = cpu_to_le16(SMB_SET_FILE_UNIX_LINK); + pSMB->Reserved4 = 0; + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + cifs_stats_inc(&tcon->stats.cifs_stats.num_symlinks); + if (rc) + cifs_dbg(FYI, "Send error in SetPathInfo create symlink = %d\n", + rc); + + cifs_buf_release(pSMB); + + if (rc == -EAGAIN) + goto createSymLinkRetry; + + return rc; +} + +int +CIFSUnixCreateHardLink(const unsigned int xid, struct cifs_tcon *tcon, + const char *fromName, const char *toName, + const struct nls_table *nls_codepage, int remap) +{ + TRANSACTION2_SPI_REQ *pSMB = NULL; + TRANSACTION2_SPI_RSP *pSMBr = NULL; + unsigned int in_len; + char *data_offset; + int name_len; + int name_len_target; + int rc = 0; + int bytes_returned = 0; + __u16 params, param_offset, offset, byte_count; + + cifs_dbg(FYI, "In Create Hard link Unix style\n"); +createHardLinkRetry: + rc = smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + name_len = cifsConvertToUTF16((__le16 *) pSMB->FileName, toName, + PATH_MAX, nls_codepage, remap); + name_len++; /* trailing null */ + name_len *= 2; + + } else { + name_len = copy_path_name(pSMB->FileName, toName); + } + params = 6 + name_len; + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + param_offset = offsetof(struct smb_com_transaction2_spi_req, + InformationLevel); + offset = param_offset + params; + + data_offset = (char *)pSMB + offset; + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + name_len_target = + cifsConvertToUTF16((__le16 *) data_offset, fromName, + PATH_MAX, nls_codepage, remap); + name_len_target++; /* trailing null */ + name_len_target *= 2; + } else { + name_len_target = copy_path_name(data_offset, fromName); + } + + pSMB->MaxParameterCount = cpu_to_le16(2); + /* BB find exact max on data count below from sess*/ + pSMB->MaxDataCount = cpu_to_le16(1000); + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_SET_PATH_INFORMATION); + byte_count = 3 /* pad */ + params + name_len_target; + pSMB->ParameterCount = cpu_to_le16(params); + pSMB->TotalParameterCount = pSMB->ParameterCount; + pSMB->DataCount = cpu_to_le16(name_len_target); + pSMB->TotalDataCount = pSMB->DataCount; + pSMB->ParameterOffset = cpu_to_le16(param_offset); + pSMB->DataOffset = cpu_to_le16(offset); + pSMB->InformationLevel = cpu_to_le16(SMB_SET_FILE_UNIX_HLINK); + pSMB->Reserved4 = 0; + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + cifs_stats_inc(&tcon->stats.cifs_stats.num_hardlinks); + if (rc) + cifs_dbg(FYI, "Send error in SetPathInfo (hard link) = %d\n", + rc); + + cifs_buf_release(pSMB); + if (rc == -EAGAIN) + goto createHardLinkRetry; + + return rc; +} + +int CIFSCreateHardLink(const unsigned int xid, + struct cifs_tcon *tcon, + struct dentry *source_dentry, + const char *from_name, const char *to_name, + struct cifs_sb_info *cifs_sb) +{ + int rc = 0; + NT_RENAME_REQ *pSMB = NULL; + RENAME_RSP *pSMBr = NULL; + unsigned int in_len; + int bytes_returned; + int name_len, name_len2; + __u16 count; + int remap = cifs_remap(cifs_sb); + + cifs_dbg(FYI, "In CIFSCreateHardLink\n"); +winCreateHardLinkRetry: + + rc = smb_init(SMB_COM_NT_RENAME, 4, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + pSMB->SearchAttributes = + cpu_to_le16(ATTR_READONLY | ATTR_HIDDEN | ATTR_SYSTEM | + ATTR_DIRECTORY); + pSMB->Flags = cpu_to_le16(CREATE_HARD_LINK); + pSMB->ClusterCount = 0; + + pSMB->BufferFormat = 0x04; + + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + name_len = + cifsConvertToUTF16((__le16 *) pSMB->OldFileName, from_name, + PATH_MAX, cifs_sb->local_nls, remap); + name_len++; /* trailing null */ + name_len *= 2; + + /* protocol specifies ASCII buffer format (0x04) for unicode */ + pSMB->OldFileName[name_len] = 0x04; + pSMB->OldFileName[name_len + 1] = 0x00; /* pad */ + name_len2 = + cifsConvertToUTF16((__le16 *)&pSMB->OldFileName[name_len+2], + to_name, PATH_MAX, cifs_sb->local_nls, + remap); + name_len2 += 1 /* trailing null */ + 1 /* Signature word */ ; + name_len2 *= 2; /* convert to bytes */ + } else { + name_len = copy_path_name(pSMB->OldFileName, from_name); + pSMB->OldFileName[name_len] = 0x04; /* 2nd buffer format */ + name_len2 = copy_path_name(pSMB->OldFileName+name_len+1, to_name); + name_len2++; /* signature byte */ + } + + count = 1 /* string type byte */ + name_len + name_len2; + in_len += count; + pSMB->ByteCount = cpu_to_le16(count); + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + cifs_stats_inc(&tcon->stats.cifs_stats.num_hardlinks); + if (rc) + cifs_dbg(FYI, "Send error in hard link (NT rename) = %d\n", rc); + + cifs_buf_release(pSMB); + if (rc == -EAGAIN) + goto winCreateHardLinkRetry; + + return rc; +} + +int +CIFSSMBUnixQuerySymLink(const unsigned int xid, struct cifs_tcon *tcon, + const unsigned char *searchName, char **symlinkinfo, + const struct nls_table *nls_codepage, int remap) +{ +/* SMB_QUERY_FILE_UNIX_LINK */ + TRANSACTION2_QPI_REQ *pSMB = NULL; + TRANSACTION2_QPI_RSP *pSMBr = NULL; + unsigned int in_len; + int rc = 0; + int bytes_returned; + int name_len; + __u16 params, byte_count; + char *data_start; + + cifs_dbg(FYI, "In QPathSymLinkInfo (Unix) for path %s\n", searchName); + +querySymLinkRetry: + rc = smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + name_len = + cifsConvertToUTF16((__le16 *) pSMB->FileName, + searchName, PATH_MAX, nls_codepage, + remap); + name_len++; /* trailing null */ + name_len *= 2; + } else { + name_len = copy_path_name(pSMB->FileName, searchName); + } + + params = 2 /* level */ + 4 /* rsrvd */ + name_len /* incl null */ ; + pSMB->TotalDataCount = 0; + pSMB->MaxParameterCount = cpu_to_le16(2); + pSMB->MaxDataCount = cpu_to_le16(CIFSMaxBufSize); + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + pSMB->ParameterOffset = cpu_to_le16(offsetof( + struct smb_com_transaction2_qpi_req, InformationLevel)); + pSMB->DataCount = 0; + pSMB->DataOffset = 0; + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_QUERY_PATH_INFORMATION); + byte_count = params + 1 /* pad */ ; + pSMB->TotalParameterCount = cpu_to_le16(params); + pSMB->ParameterCount = pSMB->TotalParameterCount; + pSMB->InformationLevel = cpu_to_le16(SMB_QUERY_FILE_UNIX_LINK); + pSMB->Reserved4 = 0; + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) { + cifs_dbg(FYI, "Send error in QuerySymLinkInfo = %d\n", rc); + } else { + /* decode response */ + + rc = validate_t2((struct smb_t2_rsp *)pSMBr); + /* BB also check enough total bytes returned */ + if (rc || get_bcc(&pSMBr->hdr) < 2) + rc = smb_EIO2(smb_eio_trace_qsym_bcc_too_small, + get_bcc(&pSMBr->hdr), 2); + else { + bool is_unicode; + u16 count = le16_to_cpu(pSMBr->t2.DataCount); + + data_start = ((char *) &pSMBr->hdr.Protocol) + + le16_to_cpu(pSMBr->t2.DataOffset); + + if (pSMBr->hdr.Flags2 & SMBFLG2_UNICODE) + is_unicode = true; + else + is_unicode = false; + + /* BB FIXME investigate remapping reserved chars here */ + *symlinkinfo = cifs_strndup_from_utf16(data_start, + count, is_unicode, nls_codepage); + if (!*symlinkinfo) + rc = -ENOMEM; + } + } + cifs_buf_release(pSMB); + if (rc == -EAGAIN) + goto querySymLinkRetry; + return rc; +} + +int cifs_query_reparse_point(const unsigned int xid, + struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, + const char *full_path, + u32 *tag, struct kvec *rsp, + int *rsp_buftype) +{ + struct reparse_data_buffer *buf; + struct cifs_open_parms oparms; + TRANSACT_IOCTL_REQ *io_req = NULL; + TRANSACT_IOCTL_RSP *io_rsp = NULL; + struct cifs_fid fid; + unsigned int in_len; + __u32 data_offset, data_count, len; + __u8 *start, *end; + int io_rsp_len; + int oplock = 0; + int rc; + + cifs_tcon_dbg(FYI, "%s: path=%s\n", __func__, full_path); + + if (cap_unix(tcon->ses)) + return -EOPNOTSUPP; + + if (!CIFS_REPARSE_SUPPORT(tcon)) + return -EOPNOTSUPP; + + oparms = (struct cifs_open_parms) { + .tcon = tcon, + .cifs_sb = cifs_sb, + .desired_access = FILE_READ_ATTRIBUTES, + .create_options = cifs_create_options(cifs_sb, + OPEN_REPARSE_POINT), + .disposition = FILE_OPEN, + .path = full_path, + .fid = &fid, + }; + + rc = CIFS_open(xid, &oparms, &oplock, NULL); + if (rc) + return rc; + + rc = smb_init(SMB_COM_NT_TRANSACT, 23, tcon, + (void **)&io_req, (void **)&io_rsp); + if (rc < 0) + goto error; + in_len = rc; + + io_req->TotalParameterCount = 0; + io_req->TotalDataCount = 0; + io_req->MaxParameterCount = cpu_to_le32(0); + /* BB find exact data count max from sess structure BB */ + io_req->MaxDataCount = cpu_to_le32(CIFSMaxBufSize & 0xFFFFFF00); + io_req->MaxSetupCount = 1; + io_req->Reserved = 0; + io_req->ParameterOffset = 0; + io_req->DataCount = 0; + io_req->DataOffset = 0; + io_req->SetupCount = 4; + io_req->SubCommand = cpu_to_le16(NT_TRANSACT_IOCTL); + io_req->ParameterCount = io_req->TotalParameterCount; + io_req->FunctionCode = cpu_to_le32(FSCTL_GET_REPARSE_POINT); + io_req->IsFsctl = 1; + io_req->IsRootFlag = 0; + io_req->Fid = fid.netfid; + io_req->ByteCount = 0; + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *)io_req, in_len, + (struct smb_hdr *)io_rsp, &io_rsp_len, 0); + if (rc) + goto error; + + data_offset = le32_to_cpu(io_rsp->DataOffset); + data_count = le32_to_cpu(io_rsp->DataCount); + if (get_bcc(&io_rsp->hdr) < 2 || data_offset > 512 || + !data_count || data_count > 2048) { + rc = smb_EIO2(smb_eio_trace_qreparse_sizes_wrong, + get_bcc(&io_rsp->hdr), data_count); + goto error; + } + + /* SetupCount must be 1, otherwise offset to ByteCount is incorrect. */ + if (io_rsp->SetupCount != 1) { + rc = smb_EIO2(smb_eio_trace_qreparse_setup_count, + io_rsp->SetupCount, 1); + goto error; + } + + /* + * ReturnedDataLen is output length of executed IOCTL. + * DataCount is output length transferred over network. + * Check that we have full FSCTL_GET_REPARSE_POINT buffer. + */ + if (data_count != le16_to_cpu(io_rsp->ReturnedDataLen)) { + rc = smb_EIO2(smb_eio_trace_qreparse_ret_datalen, + data_count, le16_to_cpu(io_rsp->ReturnedDataLen)); + goto error; + } + + end = 2 + get_bcc(&io_rsp->hdr) + (__u8 *)&io_rsp->ByteCount; + start = (__u8 *)&io_rsp->hdr.Protocol + data_offset; + if (start >= end) { + rc = smb_EIO2(smb_eio_trace_qreparse_data_area, + (unsigned long)start - (unsigned long)io_rsp, + (unsigned long)end - (unsigned long)io_rsp); + goto error; + } + + data_count = le16_to_cpu(io_rsp->ByteCount); + buf = (struct reparse_data_buffer *)start; + len = sizeof(*buf); + if (data_count < len || + data_count < le16_to_cpu(buf->ReparseDataLength) + len) { + rc = smb_EIO2(smb_eio_trace_qreparse_rep_datalen, + data_count, le16_to_cpu(buf->ReparseDataLength) + len); + goto error; + } + + *tag = le32_to_cpu(buf->ReparseTag); + rsp->iov_base = io_rsp; + rsp->iov_len = io_rsp_len; + *rsp_buftype = CIFS_LARGE_BUFFER; + CIFSSMBClose(xid, tcon, fid.netfid); + return 0; + +error: + cifs_buf_release(io_req); + CIFSSMBClose(xid, tcon, fid.netfid); + return rc; +} + +struct inode *cifs_create_reparse_inode(struct cifs_open_info_data *data, + struct super_block *sb, + const unsigned int xid, + struct cifs_tcon *tcon, + const char *full_path, + bool directory, + struct kvec *reparse_iov, + struct kvec *xattr_iov) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + struct cifs_open_parms oparms; + TRANSACT_IOCTL_REQ *io_req; + struct inode *new = NULL; + struct kvec in_iov[2]; + struct kvec out_iov; + struct cifs_fid fid; + unsigned int in_len; + int oplock = 0; + int buf_type = 0; + int rc; + + cifs_tcon_dbg(FYI, "%s: path=%s\n", __func__, full_path); + + /* + * If server filesystem does not support reparse points then do not + * attempt to create reparse point. This will prevent creating unusable + * empty object on the server. + */ + if (!CIFS_REPARSE_SUPPORT(tcon)) + return ERR_PTR(-EOPNOTSUPP); + +#ifndef CONFIG_CIFS_XATTR + if (xattr_iov) + return ERR_PTR(-EOPNOTSUPP); +#endif + + oparms = CIFS_OPARMS(cifs_sb, tcon, full_path, + FILE_READ_ATTRIBUTES | FILE_WRITE_DATA | FILE_WRITE_EA, + FILE_CREATE, + (directory ? CREATE_NOT_FILE : CREATE_NOT_DIR) | OPEN_REPARSE_POINT, + ACL_NO_MODE); + oparms.fid = &fid; + + rc = CIFS_open(xid, &oparms, &oplock, NULL); + if (rc) + return ERR_PTR(rc); + +#ifdef CONFIG_CIFS_XATTR + if (xattr_iov) { + struct smb2_file_full_ea_info *ea; + + ea = &((struct smb2_create_ea_ctx *)xattr_iov->iov_base)->ea; + while (1) { + rc = CIFSSMBSetEA(xid, + tcon, + full_path, + &ea->ea_data[0], + &ea->ea_data[ea->ea_name_length+1], + le16_to_cpu(ea->ea_value_length), + cifs_sb->local_nls, + cifs_sb); + if (rc) + goto out_close; + if (le32_to_cpu(ea->next_entry_offset) == 0) + break; + ea = (struct smb2_file_full_ea_info *)((u8 *)ea + + le32_to_cpu(ea->next_entry_offset)); + } + } +#endif + + rc = smb_init(SMB_COM_NT_TRANSACT, 23, tcon, (void **)&io_req, NULL); + if (rc < 0) + goto out_close; + in_len = rc; + in_len += sizeof(io_req->Pad); + + /* NT IOCTL response contains one-word long output setup buffer with size of output data. */ + io_req->MaxSetupCount = 1; + /* NT IOCTL response does not contain output parameters. */ + io_req->MaxParameterCount = cpu_to_le32(0); + /* FSCTL_SET_REPARSE_POINT response contains empty output data. */ + io_req->MaxDataCount = cpu_to_le32(0); + + io_req->TotalParameterCount = cpu_to_le32(0); + io_req->TotalDataCount = cpu_to_le32(reparse_iov->iov_len); + io_req->ParameterCount = io_req->TotalParameterCount; + io_req->ParameterOffset = cpu_to_le32(0); + io_req->DataCount = io_req->TotalDataCount; + io_req->DataOffset = cpu_to_le32(offsetof(typeof(*io_req), Data)); + io_req->SetupCount = 4; + io_req->SubCommand = cpu_to_le16(NT_TRANSACT_IOCTL); + io_req->FunctionCode = cpu_to_le32(FSCTL_SET_REPARSE_POINT); + io_req->Fid = fid.netfid; + io_req->IsFsctl = 1; + io_req->IsRootFlag = 0; + io_req->ByteCount = cpu_to_le16(le32_to_cpu(io_req->DataCount) + sizeof(io_req->Pad)); + + in_iov[0].iov_base = (char *)io_req; + in_iov[0].iov_len = in_len; + in_iov[1] = *reparse_iov; + rc = SendReceive2(xid, tcon->ses, in_iov, ARRAY_SIZE(in_iov), &buf_type, + CIFS_NO_RSP_BUF, &out_iov); + + cifs_buf_release(io_req); + + if (!rc) + rc = cifs_get_inode_info(&new, full_path, data, sb, xid, NULL); + +out_close: + CIFSSMBClose(xid, tcon, fid.netfid); + + /* + * If CREATE was successful but FSCTL_SET_REPARSE_POINT failed then + * remove the intermediate object created by CREATE. Otherwise + * empty object stay on the server when reparse call failed. + */ + if (rc) + CIFSSMBDelFile(xid, tcon, full_path, cifs_sb, NULL); + + return rc ? ERR_PTR(rc) : new; +} + +int +CIFSSMB_set_compression(const unsigned int xid, struct cifs_tcon *tcon, + __u16 fid, __u16 compression_state) +{ + int rc = 0; + int bytes_returned; + struct smb_com_transaction_compr_ioctl_req *pSMB; + struct smb_com_transaction_ioctl_rsp *pSMBr; + unsigned int in_len; + + cifs_dbg(FYI, "Set compression for %u\n", fid); + rc = smb_init(SMB_COM_NT_TRANSACT, 23, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + pSMB->compression_state = cpu_to_le16(compression_state); + + pSMB->TotalParameterCount = 0; + pSMB->TotalDataCount = cpu_to_le32(2); + pSMB->MaxParameterCount = 0; + pSMB->MaxDataCount = 0; + pSMB->MaxSetupCount = 4; + pSMB->Reserved = 0; + pSMB->ParameterOffset = 0; + pSMB->DataCount = cpu_to_le32(2); + pSMB->DataOffset = + cpu_to_le32(offsetof(struct smb_com_transaction_compr_ioctl_req, + compression_state)); /* 84 */ + pSMB->SetupCount = 4; + pSMB->SubCommand = cpu_to_le16(NT_TRANSACT_IOCTL); + pSMB->ParameterCount = 0; + pSMB->FunctionCode = cpu_to_le32(FSCTL_SET_COMPRESSION); + pSMB->IsFsctl = 1; /* FSCTL */ + pSMB->IsRootFlag = 0; + pSMB->Fid = fid; /* file handle always le */ + /* 3 byte pad, followed by 2 byte compress state */ + pSMB->ByteCount = cpu_to_le16(5); + in_len += 5; + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) + cifs_dbg(FYI, "Send error in SetCompression = %d\n", rc); + + cifs_buf_release(pSMB); + + /* + * Note: On -EAGAIN error only caller can retry on handle based calls + * since file handle passed in no longer valid. + */ + return rc; +} + + +#ifdef CONFIG_CIFS_POSIX + +#ifdef CONFIG_FS_POSIX_ACL +/** + * cifs_init_posix_acl - convert ACL from cifs to POSIX ACL format + * @ace: POSIX ACL entry to store converted ACL into + * @cifs_ace: ACL in cifs format + * + * Convert an Access Control Entry from wire format to local POSIX xattr + * format. + * + * Note that the @cifs_uid member is used to store both {g,u}id_t. + */ +static void cifs_init_posix_acl(struct posix_acl_entry *ace, + struct cifs_posix_ace *cifs_ace) +{ + /* u8 cifs fields do not need le conversion */ + ace->e_perm = cifs_ace->cifs_e_perm; + ace->e_tag = cifs_ace->cifs_e_tag; + + switch (ace->e_tag) { + case ACL_USER: + ace->e_uid = make_kuid(&init_user_ns, + le64_to_cpu(cifs_ace->cifs_uid)); + break; + case ACL_GROUP: + ace->e_gid = make_kgid(&init_user_ns, + le64_to_cpu(cifs_ace->cifs_uid)); + break; + } + return; +} + +/** + * cifs_to_posix_acl - copy cifs ACL format to POSIX ACL format + * @acl: ACLs returned in POSIX ACL format + * @src: ACLs in cifs format + * @acl_type: type of POSIX ACL requested + * @size_of_data_area: size of SMB we got + * + * This function converts ACLs from cifs format to POSIX ACL format. + * If @acl is NULL then the size of the buffer required to store POSIX ACLs in + * their uapi format is returned. + */ +static int cifs_to_posix_acl(struct posix_acl **acl, char *src, + const int acl_type, const int size_of_data_area) +{ + int size = 0; + __u16 count; + struct cifs_posix_ace *pACE; + struct cifs_posix_acl *cifs_acl = (struct cifs_posix_acl *)src; + struct posix_acl *kacl = NULL; + struct posix_acl_entry *pa, *pe; + + if (le16_to_cpu(cifs_acl->version) != CIFS_ACL_VERSION) + return -EOPNOTSUPP; + + if (acl_type == ACL_TYPE_ACCESS) { + count = le16_to_cpu(cifs_acl->access_entry_count); + pACE = &cifs_acl->ace_array[0]; + size = sizeof(struct cifs_posix_acl); + size += sizeof(struct cifs_posix_ace) * count; + /* check if we would go beyond end of SMB */ + if (size_of_data_area < size) { + cifs_dbg(FYI, "bad CIFS POSIX ACL size %d vs. %d\n", + size_of_data_area, size); + return -EINVAL; + } + } else if (acl_type == ACL_TYPE_DEFAULT) { + count = le16_to_cpu(cifs_acl->access_entry_count); + size = sizeof(struct cifs_posix_acl); + size += sizeof(struct cifs_posix_ace) * count; + /* skip past access ACEs to get to default ACEs */ + pACE = &cifs_acl->ace_array[count]; + count = le16_to_cpu(cifs_acl->default_entry_count); + size += sizeof(struct cifs_posix_ace) * count; + /* check if we would go beyond end of SMB */ + if (size_of_data_area < size) + return -EINVAL; + } else { + /* illegal type */ + return -EINVAL; + } + + /* Allocate number of POSIX ACLs to store in VFS format. */ + kacl = posix_acl_alloc(count, GFP_NOFS); + if (!kacl) + return -ENOMEM; + + FOREACH_ACL_ENTRY(pa, kacl, pe) { + cifs_init_posix_acl(pa, pACE); + pACE++; + } + + *acl = kacl; + return 0; +} + +/** + * cifs_init_ace - convert ACL entry from POSIX ACL to cifs format + * @cifs_ace: the cifs ACL entry to store into + * @local_ace: the POSIX ACL entry to convert + */ +static void cifs_init_ace(struct cifs_posix_ace *cifs_ace, + const struct posix_acl_entry *local_ace) +{ + cifs_ace->cifs_e_perm = local_ace->e_perm; + cifs_ace->cifs_e_tag = local_ace->e_tag; + + switch (local_ace->e_tag) { + case ACL_USER: + cifs_ace->cifs_uid = + cpu_to_le64(from_kuid(&init_user_ns, local_ace->e_uid)); + break; + case ACL_GROUP: + cifs_ace->cifs_uid = + cpu_to_le64(from_kgid(&init_user_ns, local_ace->e_gid)); + break; + default: + cifs_ace->cifs_uid = cpu_to_le64(-1); + } +} + +/** + * posix_acl_to_cifs - convert ACLs from POSIX ACL to cifs format + * @parm_data: ACLs in cifs format to convert to + * @acl: ACLs in POSIX ACL format to convert from + * @acl_type: the type of POSIX ACLs stored in @acl + * + * Return: the number cifs ACL entries after conversion + */ +static __u16 posix_acl_to_cifs(char *parm_data, const struct posix_acl *acl, + const int acl_type) +{ + __u16 rc = 0; + struct cifs_posix_acl *cifs_acl = (struct cifs_posix_acl *)parm_data; + const struct posix_acl_entry *pa, *pe; + int count; + int i = 0; + + if ((acl == NULL) || (cifs_acl == NULL)) + return 0; + + count = acl->a_count; + cifs_dbg(FYI, "setting acl with %d entries\n", count); + + /* + * Note that the uapi POSIX ACL version is verified by the VFS and is + * independent of the cifs ACL version. Changing the POSIX ACL version + * is a uapi change and if it's changed we will pass down the POSIX ACL + * version in struct posix_acl from the VFS. For now there's really + * only one that all filesystems know how to deal with. + */ + cifs_acl->version = cpu_to_le16(1); + if (acl_type == ACL_TYPE_ACCESS) { + cifs_acl->access_entry_count = cpu_to_le16(count); + cifs_acl->default_entry_count = cpu_to_le16(0xFFFF); + } else if (acl_type == ACL_TYPE_DEFAULT) { + cifs_acl->default_entry_count = cpu_to_le16(count); + cifs_acl->access_entry_count = cpu_to_le16(0xFFFF); + } else { + cifs_dbg(FYI, "unknown ACL type %d\n", acl_type); + return 0; + } + FOREACH_ACL_ENTRY(pa, acl, pe) { + cifs_init_ace(&cifs_acl->ace_array[i++], pa); + } + if (rc == 0) { + rc = (__u16)(count * sizeof(struct cifs_posix_ace)); + rc += sizeof(struct cifs_posix_acl); + /* BB add check to make sure ACL does not overflow SMB */ + } + return rc; +} + +int cifs_do_get_acl(const unsigned int xid, struct cifs_tcon *tcon, + const unsigned char *searchName, struct posix_acl **acl, + const int acl_type, const struct nls_table *nls_codepage, + int remap) +{ +/* SMB_QUERY_POSIX_ACL */ + TRANSACTION2_QPI_REQ *pSMB = NULL; + TRANSACTION2_QPI_RSP *pSMBr = NULL; + unsigned int in_len; + int rc = 0; + int bytes_returned; + int name_len; + __u16 params, byte_count; + + cifs_dbg(FYI, "In GetPosixACL (Unix) for path %s\n", searchName); + +queryAclRetry: + rc = smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + name_len = + cifsConvertToUTF16((__le16 *) pSMB->FileName, + searchName, PATH_MAX, nls_codepage, + remap); + name_len++; /* trailing null */ + name_len *= 2; + pSMB->FileName[name_len] = 0; + pSMB->FileName[name_len+1] = 0; + } else { + name_len = copy_path_name(pSMB->FileName, searchName); + } + + params = 2 /* level */ + 4 /* rsrvd */ + name_len /* incl null */ ; + pSMB->TotalDataCount = 0; + pSMB->MaxParameterCount = cpu_to_le16(2); + /* BB find exact max data count below from sess structure BB */ + pSMB->MaxDataCount = cpu_to_le16(4000); + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + pSMB->ParameterOffset = cpu_to_le16( + offsetof(struct smb_com_transaction2_qpi_req, + InformationLevel)); + pSMB->DataCount = 0; + pSMB->DataOffset = 0; + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_QUERY_PATH_INFORMATION); + byte_count = params + 1 /* pad */ ; + pSMB->TotalParameterCount = cpu_to_le16(params); + pSMB->ParameterCount = pSMB->TotalParameterCount; + pSMB->InformationLevel = cpu_to_le16(SMB_QUERY_POSIX_ACL); + pSMB->Reserved4 = 0; + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + cifs_stats_inc(&tcon->stats.cifs_stats.num_acl_get); + if (rc) { + cifs_dbg(FYI, "Send error in Query POSIX ACL = %d\n", rc); + } else { + /* decode response */ + + rc = validate_t2((struct smb_t2_rsp *)pSMBr); + /* BB also check enough total bytes returned */ + if (rc || get_bcc(&pSMBr->hdr) < 2) + rc = smb_EIO2(smb_eio_trace_getacl_bcc_too_small, + get_bcc(&pSMBr->hdr), 2); + else { + __u16 data_offset = le16_to_cpu(pSMBr->t2.DataOffset); + __u16 count = le16_to_cpu(pSMBr->t2.DataCount); + rc = cifs_to_posix_acl(acl, + (char *)&pSMBr->hdr.Protocol+data_offset, + acl_type, count); + } + } + cifs_buf_release(pSMB); + /* + * The else branch after SendReceive() doesn't return EAGAIN so if we + * allocated @acl in cifs_to_posix_acl() we are guaranteed to return + * here and don't leak POSIX ACLs. + */ + if (rc == -EAGAIN) + goto queryAclRetry; + return rc; +} + +int cifs_do_set_acl(const unsigned int xid, struct cifs_tcon *tcon, + const unsigned char *fileName, const struct posix_acl *acl, + const int acl_type, const struct nls_table *nls_codepage, + int remap) +{ + struct smb_com_transaction2_spi_req *pSMB = NULL; + struct smb_com_transaction2_spi_rsp *pSMBr = NULL; + unsigned int in_len; + char *parm_data; + int name_len; + int rc = 0; + int bytes_returned = 0; + __u16 params, byte_count, data_count, param_offset, offset; + + cifs_dbg(FYI, "In SetPosixACL (Unix) for path %s\n", fileName); +setAclRetry: + rc = smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + name_len = + cifsConvertToUTF16((__le16 *) pSMB->FileName, fileName, + PATH_MAX, nls_codepage, remap); + name_len++; /* trailing null */ + name_len *= 2; + } else { + name_len = copy_path_name(pSMB->FileName, fileName); + } + params = 6 + name_len; + pSMB->MaxParameterCount = cpu_to_le16(2); + /* BB find max SMB size from sess */ + pSMB->MaxDataCount = cpu_to_le16(1000); + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + param_offset = offsetof(struct smb_com_transaction2_spi_req, + InformationLevel); + offset = param_offset + params; + parm_data = ((char *)pSMB) + offset; + pSMB->ParameterOffset = cpu_to_le16(param_offset); + + /* convert to on the wire format for POSIX ACL */ + data_count = posix_acl_to_cifs(parm_data, acl, acl_type); + + if (data_count == 0) { + rc = -EOPNOTSUPP; + goto setACLerrorExit; + } + pSMB->DataOffset = cpu_to_le16(offset); + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_SET_PATH_INFORMATION); + pSMB->InformationLevel = cpu_to_le16(SMB_SET_POSIX_ACL); + byte_count = 3 /* pad */ + params + data_count; + pSMB->DataCount = cpu_to_le16(data_count); + pSMB->TotalDataCount = pSMB->DataCount; + pSMB->ParameterCount = cpu_to_le16(params); + pSMB->TotalParameterCount = pSMB->ParameterCount; + pSMB->Reserved4 = 0; + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) + cifs_dbg(FYI, "Set POSIX ACL returned %d\n", rc); + +setACLerrorExit: + cifs_buf_release(pSMB); + if (rc == -EAGAIN) + goto setAclRetry; + return rc; +} +#else +int cifs_do_get_acl(const unsigned int xid, struct cifs_tcon *tcon, + const unsigned char *searchName, struct posix_acl **acl, + const int acl_type, const struct nls_table *nls_codepage, + int remap) +{ + return -EOPNOTSUPP; +} + +int cifs_do_set_acl(const unsigned int xid, struct cifs_tcon *tcon, + const unsigned char *fileName, const struct posix_acl *acl, + const int acl_type, const struct nls_table *nls_codepage, + int remap) +{ + return -EOPNOTSUPP; +} +#endif /* CONFIG_FS_POSIX_ACL */ + +int +CIFSGetExtAttr(const unsigned int xid, struct cifs_tcon *tcon, + const int netfid, __u64 *pExtAttrBits, __u64 *pMask) +{ + int rc = 0; + struct smb_t2_qfi_req *pSMB = NULL; + struct smb_t2_qfi_rsp *pSMBr = NULL; + unsigned int in_len; + int bytes_returned; + __u16 params, byte_count; + + cifs_dbg(FYI, "In GetExtAttr\n"); + if (tcon == NULL) + return -ENODEV; + +GetExtAttrRetry: + rc = smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + params = 2 /* level */ + 2 /* fid */; + pSMB->t2.TotalDataCount = 0; + pSMB->t2.MaxParameterCount = cpu_to_le16(4); + /* BB find exact max data count below from sess structure BB */ + pSMB->t2.MaxDataCount = cpu_to_le16(4000); + pSMB->t2.MaxSetupCount = 0; + pSMB->t2.Reserved = 0; + pSMB->t2.Flags = 0; + pSMB->t2.Timeout = 0; + pSMB->t2.Reserved2 = 0; + pSMB->t2.ParameterOffset = cpu_to_le16(offsetof(struct smb_t2_qfi_req, + Fid)); + pSMB->t2.DataCount = 0; + pSMB->t2.DataOffset = 0; + pSMB->t2.SetupCount = 1; + pSMB->t2.Reserved3 = 0; + pSMB->t2.SubCommand = cpu_to_le16(TRANS2_QUERY_FILE_INFORMATION); + byte_count = params + 1 /* pad */ ; + pSMB->t2.TotalParameterCount = cpu_to_le16(params); + pSMB->t2.ParameterCount = pSMB->t2.TotalParameterCount; + pSMB->InformationLevel = cpu_to_le16(SMB_QUERY_ATTR_FLAGS); + pSMB->Pad = 0; + pSMB->Fid = netfid; + in_len += byte_count; + pSMB->t2.ByteCount = cpu_to_le16(byte_count); + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) { + cifs_dbg(FYI, "error %d in GetExtAttr\n", rc); + } else { + /* decode response */ + rc = validate_t2((struct smb_t2_rsp *)pSMBr); + /* BB also check enough total bytes returned */ + if (rc || get_bcc(&pSMBr->hdr) < 2) + /* If rc should we check for EOPNOSUPP and + disable the srvino flag? or in caller? */ + rc = smb_EIO2(smb_eio_trace_getextattr_bcc_too_small, + get_bcc(&pSMBr->hdr), 2); + else { + __u16 data_offset = le16_to_cpu(pSMBr->t2.DataOffset); + __u16 count = le16_to_cpu(pSMBr->t2.DataCount); + struct file_chattr_info *pfinfo; + + if (count != 16) { + cifs_dbg(FYI, "Invalid size ret in GetExtAttr\n"); + rc = smb_EIO2(smb_eio_trace_getextattr_inv_size, + count, 16); + goto GetExtAttrOut; + } + pfinfo = (struct file_chattr_info *) + (data_offset + (char *) &pSMBr->hdr.Protocol); + *pExtAttrBits = le64_to_cpu(pfinfo->mode); + *pMask = le64_to_cpu(pfinfo->mask); + } + } +GetExtAttrOut: + cifs_buf_release(pSMB); + if (rc == -EAGAIN) + goto GetExtAttrRetry; + return rc; +} + +#endif /* CONFIG_POSIX */ + +/* + * Initialize NT TRANSACT SMB into small smb request buffer. This assumes that + * all NT TRANSACTS that we init here have total parm and data under about 400 + * bytes (to fit in small cifs buffer size), which is the case so far, it + * easily fits. NB: Setup words themselves and ByteCount MaxSetupCount (size of + * returned setup area) and MaxParameterCount (returned parms size) must be set + * by caller + */ +static int +smb_init_nttransact(const __u16 sub_command, const int setup_count, + const int parm_len, struct cifs_tcon *tcon, + void **ret_buf) +{ + int rc; + __u32 temp_offset; + struct smb_com_ntransact_req *pSMB; + unsigned int in_len; + + rc = small_smb_init(SMB_COM_NT_TRANSACT, 19 + setup_count, tcon, + (void **)&pSMB); + if (rc < 0) + return rc; + in_len = rc; + *ret_buf = (void *)pSMB; + pSMB->Reserved = 0; + pSMB->TotalParameterCount = cpu_to_le32(parm_len); + pSMB->TotalDataCount = 0; + pSMB->MaxDataCount = cpu_to_le32(CIFSMaxBufSize & 0xFFFFFF00); + pSMB->ParameterCount = pSMB->TotalParameterCount; + pSMB->DataCount = pSMB->TotalDataCount; + temp_offset = offsetof(struct smb_com_ntransact_req, Parms) + + (setup_count * 2); + pSMB->ParameterOffset = cpu_to_le32(temp_offset); + pSMB->DataOffset = cpu_to_le32(temp_offset + parm_len); + pSMB->SetupCount = setup_count; /* no need to le convert byte fields */ + pSMB->SubCommand = cpu_to_le16(sub_command); + return in_len; +} + +static int +validate_ntransact(char *buf, char **ppparm, char **ppdata, + __u32 *pparmlen, __u32 *pdatalen) +{ + char *end_of_smb; + __u32 data_count, data_offset, parm_count, parm_offset; + struct smb_com_ntransact_rsp *pSMBr; + u16 bcc; + + *pdatalen = 0; + *pparmlen = 0; + + if (buf == NULL) + return -EINVAL; + + pSMBr = (struct smb_com_ntransact_rsp *)buf; + + bcc = get_bcc(&pSMBr->hdr); + end_of_smb = 2 /* sizeof byte count */ + bcc + + (char *)&pSMBr->ByteCount; + + data_offset = le32_to_cpu(pSMBr->DataOffset); + data_count = le32_to_cpu(pSMBr->DataCount); + parm_offset = le32_to_cpu(pSMBr->ParameterOffset); + parm_count = le32_to_cpu(pSMBr->ParameterCount); + + *ppparm = (char *)&pSMBr->hdr.Protocol + parm_offset; + *ppdata = (char *)&pSMBr->hdr.Protocol + data_offset; + + /* should we also check that parm and data areas do not overlap? */ + if (*ppparm > end_of_smb) { + cifs_dbg(FYI, "parms start after end of smb\n"); + return -EINVAL; + } else if (parm_count + *ppparm > end_of_smb) { + cifs_dbg(FYI, "parm end after end of smb\n"); + return -EINVAL; + } else if (*ppdata > end_of_smb) { + cifs_dbg(FYI, "data starts after end of smb\n"); + return -EINVAL; + } else if (data_count + *ppdata > end_of_smb) { + cifs_dbg(FYI, "data %p + count %d (%p) past smb end %p start %p\n", + *ppdata, data_count, (data_count + *ppdata), + end_of_smb, pSMBr); + return -EINVAL; + } else if (parm_count + data_count > bcc) { + cifs_dbg(FYI, "parm count and data count larger than SMB\n"); + return -EINVAL; + } + *pdatalen = data_count; + *pparmlen = parm_count; + return 0; +} + +/* Get Security Descriptor (by handle) from remote server for a file or dir */ +int +CIFSSMBGetCIFSACL(const unsigned int xid, struct cifs_tcon *tcon, __u16 fid, + struct smb_ntsd **acl_inf, __u32 *pbuflen, __u32 info) +{ + int rc = 0; + int buf_type = 0; + QUERY_SEC_DESC_REQ *pSMB; + struct kvec iov[1]; + struct kvec rsp_iov; + unsigned int in_len; + + cifs_dbg(FYI, "GetCifsACL\n"); + + *pbuflen = 0; + *acl_inf = NULL; + + rc = smb_init_nttransact(NT_TRANSACT_QUERY_SECURITY_DESC, 0, + 8 /* parm len */, tcon, (void **) &pSMB); + if (rc < 0) + return rc; + in_len = rc; + + pSMB->MaxParameterCount = cpu_to_le32(4); + /* BB TEST with big acls that might need to be e.g. larger than 16K */ + pSMB->MaxSetupCount = 0; + pSMB->Fid = fid; /* file handle always le */ + pSMB->AclFlags = cpu_to_le32(info); + pSMB->ByteCount = cpu_to_le16(11); /* 3 bytes pad + 8 bytes parm */ + in_len += 11; + iov[0].iov_base = (char *)pSMB; + iov[0].iov_len = in_len; + + rc = SendReceive2(xid, tcon->ses, iov, 1 /* num iovec */, &buf_type, + 0, &rsp_iov); + cifs_small_buf_release(pSMB); + cifs_stats_inc(&tcon->stats.cifs_stats.num_acl_get); + if (rc) { + cifs_dbg(FYI, "Send error in QuerySecDesc = %d\n", rc); + } else { /* decode response */ + __le32 *parm; + __u32 parm_len; + __u32 acl_len; + struct smb_com_ntransact_rsp *pSMBr; + char *pdata; + +/* validate_nttransact */ + rc = validate_ntransact(rsp_iov.iov_base, (char **)&parm, + &pdata, &parm_len, pbuflen); + if (rc) + goto qsec_out; + pSMBr = (struct smb_com_ntransact_rsp *)rsp_iov.iov_base; + + cifs_dbg(FYI, "smb %p parm %p data %p\n", + pSMBr, parm, *acl_inf); + + if (le32_to_cpu(pSMBr->ParameterCount) != 4) { + rc = smb_EIO2(smb_eio_trace_getcifsacl_param_count, + le32_to_cpu(pSMBr->ParameterCount), 4); + *pbuflen = 0; + goto qsec_out; + } + +/* BB check that data area is minimum length and as big as acl_len */ + + acl_len = le32_to_cpu(*parm); + if (acl_len != *pbuflen) { + cifs_dbg(VFS, "acl length %d does not match %d\n", + acl_len, *pbuflen); + if (*pbuflen > acl_len) + *pbuflen = acl_len; + } + + /* check if buffer is big enough for the acl + header followed by the smallest SID */ + if ((*pbuflen < sizeof(struct smb_ntsd) + 8) || + (*pbuflen >= 64 * 1024)) { + cifs_dbg(VFS, "bad acl length %d\n", *pbuflen); + rc = -EINVAL; + *pbuflen = 0; + } else { + *acl_inf = kmemdup(pdata, *pbuflen, GFP_KERNEL); + if (*acl_inf == NULL) { + *pbuflen = 0; + rc = -ENOMEM; + } + } + } +qsec_out: + free_rsp_buf(buf_type, rsp_iov.iov_base); + return rc; +} + +int +CIFSSMBSetCIFSACL(const unsigned int xid, struct cifs_tcon *tcon, __u16 fid, + struct smb_ntsd *pntsd, __u32 acllen, int aclflag) +{ + __u16 byte_count, param_count, data_count, param_offset, data_offset; + int rc = 0; + int bytes_returned = 0; + SET_SEC_DESC_REQ *pSMB = NULL; + unsigned int in_len; + void *pSMBr; + +setCifsAclRetry: + rc = smb_init(SMB_COM_NT_TRANSACT, 19, tcon, (void **) &pSMB, &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + + param_count = 8; + param_offset = offsetof(struct smb_com_transaction_ssec_req, Fid); + data_count = acllen; + data_offset = param_offset + param_count; + byte_count = 3 /* pad */ + param_count; + + pSMB->DataCount = cpu_to_le32(data_count); + pSMB->TotalDataCount = pSMB->DataCount; + pSMB->MaxParameterCount = cpu_to_le32(4); + pSMB->MaxDataCount = cpu_to_le32(16384); + pSMB->ParameterCount = cpu_to_le32(param_count); + pSMB->ParameterOffset = cpu_to_le32(param_offset); + pSMB->TotalParameterCount = pSMB->ParameterCount; + pSMB->DataOffset = cpu_to_le32(data_offset); + pSMB->SetupCount = 0; + pSMB->SubCommand = cpu_to_le16(NT_TRANSACT_SET_SECURITY_DESC); + pSMB->ByteCount = cpu_to_le16(byte_count+data_count); + + pSMB->Fid = fid; /* file handle always le */ + pSMB->Reserved2 = 0; + pSMB->AclFlags = cpu_to_le32(aclflag); + + if (pntsd && acllen) { + memcpy((char *)pSMBr + data_offset, pntsd, acllen); + in_len += byte_count + data_count; + } else + in_len += byte_count; + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + + cifs_dbg(FYI, "SetCIFSACL bytes_returned: %d, rc: %d\n", + bytes_returned, rc); + if (rc) + cifs_dbg(FYI, "Set CIFS ACL returned %d\n", rc); + cifs_buf_release(pSMB); + + if (rc == -EAGAIN) + goto setCifsAclRetry; + + return (rc); +} + + +/* Legacy Query Path Information call for lookup to old servers such + as Win9x/WinME */ +int +SMBQueryInformation(const unsigned int xid, struct cifs_tcon *tcon, + const char *search_name, FILE_ALL_INFO *data, + const struct nls_table *nls_codepage, int remap) +{ + QUERY_INFORMATION_REQ *pSMB; + QUERY_INFORMATION_RSP *pSMBr; + unsigned int in_len; + int rc = 0; + int bytes_returned; + int name_len; + + cifs_dbg(FYI, "In SMBQPath path %s\n", search_name); +QInfRetry: + rc = smb_init(SMB_COM_QUERY_INFORMATION, 0, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + name_len = + cifsConvertToUTF16((__le16 *) pSMB->FileName, + search_name, PATH_MAX, nls_codepage, + remap); + name_len++; /* trailing null */ + name_len *= 2; + } else { + name_len = copy_path_name(pSMB->FileName, search_name); + } + pSMB->BufferFormat = 0x04; + name_len++; /* account for buffer type byte */ + in_len += name_len; + pSMB->ByteCount = cpu_to_le16(name_len); + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) { + cifs_dbg(FYI, "Send error in QueryInfo = %d\n", rc); + } else if (data) { + struct timespec64 ts; + __u32 time = le32_to_cpu(pSMBr->last_write_time); + + /* decode response */ + /* BB FIXME - add time zone adjustment BB */ + memset(data, 0, sizeof(FILE_ALL_INFO)); + ts.tv_nsec = 0; + ts.tv_sec = time; + /* decode time fields */ + data->ChangeTime = cpu_to_le64(cifs_UnixTimeToNT(ts)); + data->LastWriteTime = data->ChangeTime; + data->LastAccessTime = 0; + data->AllocationSize = + cpu_to_le64(le32_to_cpu(pSMBr->size)); + data->EndOfFile = data->AllocationSize; + data->Attributes = + cpu_to_le32(le16_to_cpu(pSMBr->attr)); + } else { + /* bad buffer passed in */ + rc = smb_EIO(smb_eio_trace_null_pointers); + } + + cifs_buf_release(pSMB); + + if (rc == -EAGAIN) + goto QInfRetry; + + return rc; +} + +int +CIFSSMBQFileInfo(const unsigned int xid, struct cifs_tcon *tcon, + u16 netfid, FILE_ALL_INFO *pFindData) +{ + struct smb_t2_qfi_req *pSMB = NULL; + struct smb_t2_qfi_rsp *pSMBr = NULL; + unsigned int in_len; + int rc = 0; + int bytes_returned; + __u16 params, byte_count; + +QFileInfoRetry: + rc = smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + params = 2 /* level */ + 2 /* fid */; + pSMB->t2.TotalDataCount = 0; + pSMB->t2.MaxParameterCount = cpu_to_le16(4); + /* BB find exact max data count below from sess structure BB */ + pSMB->t2.MaxDataCount = cpu_to_le16(CIFSMaxBufSize); + pSMB->t2.MaxSetupCount = 0; + pSMB->t2.Reserved = 0; + pSMB->t2.Flags = 0; + pSMB->t2.Timeout = 0; + pSMB->t2.Reserved2 = 0; + pSMB->t2.ParameterOffset = cpu_to_le16(offsetof(struct smb_t2_qfi_req, + Fid)); + pSMB->t2.DataCount = 0; + pSMB->t2.DataOffset = 0; + pSMB->t2.SetupCount = 1; + pSMB->t2.Reserved3 = 0; + pSMB->t2.SubCommand = cpu_to_le16(TRANS2_QUERY_FILE_INFORMATION); + byte_count = params + 1 /* pad */ ; + pSMB->t2.TotalParameterCount = cpu_to_le16(params); + pSMB->t2.ParameterCount = pSMB->t2.TotalParameterCount; + pSMB->InformationLevel = cpu_to_le16(SMB_QUERY_FILE_ALL_INFO); + pSMB->Pad = 0; + pSMB->Fid = netfid; + in_len += byte_count; + pSMB->t2.ByteCount = cpu_to_le16(byte_count); + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) { + cifs_dbg(FYI, "Send error in QFileInfo = %d\n", rc); + } else { /* decode response */ + rc = validate_t2((struct smb_t2_rsp *)pSMBr); + + if (rc) /* BB add auto retry on EOPNOTSUPP? */ + rc = smb_EIO2(smb_eio_trace_qfileinfo_invalid, + get_bcc(&pSMBr->hdr), 40); + else if (get_bcc(&pSMBr->hdr) < 40) + rc = smb_EIO2(smb_eio_trace_qfileinfo_bcc_too_small, + get_bcc(&pSMBr->hdr), 40); + else if (pFindData) { + __u16 data_offset = le16_to_cpu(pSMBr->t2.DataOffset); + memcpy((char *) pFindData, + (char *) &pSMBr->hdr.Protocol + + data_offset, sizeof(FILE_ALL_INFO)); + } else + rc = -ENOMEM; + } + cifs_buf_release(pSMB); + if (rc == -EAGAIN) + goto QFileInfoRetry; + + return rc; +} + +int +CIFSSMBQPathInfo(const unsigned int xid, struct cifs_tcon *tcon, + const char *search_name, FILE_ALL_INFO *data, + int legacy /* old style infolevel */, + const struct nls_table *nls_codepage, int remap) +{ + /* level 263 SMB_QUERY_FILE_ALL_INFO */ + TRANSACTION2_QPI_REQ *pSMB = NULL; + TRANSACTION2_QPI_RSP *pSMBr = NULL; + unsigned int in_len; + int rc = 0; + int bytes_returned; + int name_len; + __u16 params, byte_count; + + /* cifs_dbg(FYI, "In QPathInfo path %s\n", search_name); */ +QPathInfoRetry: + rc = smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + name_len = + cifsConvertToUTF16((__le16 *) pSMB->FileName, search_name, + PATH_MAX, nls_codepage, remap); + name_len++; /* trailing null */ + name_len *= 2; + } else { + name_len = copy_path_name(pSMB->FileName, search_name); + } + + params = 2 /* level */ + 4 /* reserved */ + name_len /* includes NUL */; + pSMB->TotalDataCount = 0; + pSMB->MaxParameterCount = cpu_to_le16(2); + /* BB find exact max SMB PDU from sess structure BB */ + pSMB->MaxDataCount = cpu_to_le16(4000); + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + pSMB->ParameterOffset = cpu_to_le16(offsetof( + struct smb_com_transaction2_qpi_req, InformationLevel)); + pSMB->DataCount = 0; + pSMB->DataOffset = 0; + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_QUERY_PATH_INFORMATION); + byte_count = params + 1 /* pad */ ; + pSMB->TotalParameterCount = cpu_to_le16(params); + pSMB->ParameterCount = pSMB->TotalParameterCount; + if (legacy) + pSMB->InformationLevel = cpu_to_le16(SMB_INFO_STANDARD); + else + pSMB->InformationLevel = cpu_to_le16(SMB_QUERY_FILE_ALL_INFO); + pSMB->Reserved4 = 0; + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) { + cifs_dbg(FYI, "Send error in QPathInfo = %d\n", rc); + } else { /* decode response */ + rc = validate_t2((struct smb_t2_rsp *)pSMBr); + + if (rc) /* BB add auto retry on EOPNOTSUPP? */ + rc = smb_EIO2(smb_eio_trace_qpathinfo_invalid, + get_bcc(&pSMBr->hdr), 40); + else if (!legacy && get_bcc(&pSMBr->hdr) < 40) + rc = smb_EIO2(smb_eio_trace_qpathinfo_bcc_too_small, + get_bcc(&pSMBr->hdr), 40); + else if (legacy && get_bcc(&pSMBr->hdr) < 24) + /* 24 or 26 expected but we do not read last field */ + rc = smb_EIO2(smb_eio_trace_qpathinfo_bcc_too_small, + get_bcc(&pSMBr->hdr), 24); + else if (data) { + int size; + __u16 data_offset = le16_to_cpu(pSMBr->t2.DataOffset); + + /* + * On legacy responses we do not read the last field, + * EAsize, fortunately since it varies by subdialect and + * also note it differs on Set vs Get, ie two bytes or 4 + * bytes depending but we don't care here. + */ + if (legacy) + size = sizeof(FILE_INFO_STANDARD); + else + size = sizeof(FILE_ALL_INFO); + memcpy((char *) data, (char *) &pSMBr->hdr.Protocol + + data_offset, size); + } else + rc = -ENOMEM; + } + cifs_buf_release(pSMB); + if (rc == -EAGAIN) + goto QPathInfoRetry; + + return rc; +} + +int +CIFSSMBUnixQFileInfo(const unsigned int xid, struct cifs_tcon *tcon, + u16 netfid, FILE_UNIX_BASIC_INFO *pFindData) +{ + struct smb_t2_qfi_req *pSMB = NULL; + struct smb_t2_qfi_rsp *pSMBr = NULL; + unsigned int in_len; + int rc = 0; + int bytes_returned; + __u16 params, byte_count; + +UnixQFileInfoRetry: + rc = smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + params = 2 /* level */ + 2 /* fid */; + pSMB->t2.TotalDataCount = 0; + pSMB->t2.MaxParameterCount = cpu_to_le16(4); + /* BB find exact max data count below from sess structure BB */ + pSMB->t2.MaxDataCount = cpu_to_le16(CIFSMaxBufSize); + pSMB->t2.MaxSetupCount = 0; + pSMB->t2.Reserved = 0; + pSMB->t2.Flags = 0; + pSMB->t2.Timeout = 0; + pSMB->t2.Reserved2 = 0; + pSMB->t2.ParameterOffset = cpu_to_le16(offsetof(struct smb_t2_qfi_req, + Fid)); + pSMB->t2.DataCount = 0; + pSMB->t2.DataOffset = 0; + pSMB->t2.SetupCount = 1; + pSMB->t2.Reserved3 = 0; + pSMB->t2.SubCommand = cpu_to_le16(TRANS2_QUERY_FILE_INFORMATION); + byte_count = params + 1 /* pad */ ; + pSMB->t2.TotalParameterCount = cpu_to_le16(params); + pSMB->t2.ParameterCount = pSMB->t2.TotalParameterCount; + pSMB->InformationLevel = cpu_to_le16(SMB_QUERY_FILE_UNIX_BASIC); + pSMB->Pad = 0; + pSMB->Fid = netfid; + in_len += byte_count; + pSMB->t2.ByteCount = cpu_to_le16(byte_count); + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) { + cifs_dbg(FYI, "Send error in UnixQFileInfo = %d\n", rc); + } else { /* decode response */ + rc = validate_t2((struct smb_t2_rsp *)pSMBr); + + if (rc || get_bcc(&pSMBr->hdr) < sizeof(FILE_UNIX_BASIC_INFO)) { + cifs_dbg(VFS, "Malformed FILE_UNIX_BASIC_INFO response. Unix Extensions can be disabled on mount by specifying the nosfu mount option.\n"); + rc = smb_EIO2(smb_eio_trace_unixqfileinfo_bcc_too_small, + get_bcc(&pSMBr->hdr), sizeof(FILE_UNIX_BASIC_INFO)); + } else { + __u16 data_offset = le16_to_cpu(pSMBr->t2.DataOffset); + memcpy((char *) pFindData, + (char *) &pSMBr->hdr.Protocol + + data_offset, + sizeof(FILE_UNIX_BASIC_INFO)); + } + } + + cifs_buf_release(pSMB); + if (rc == -EAGAIN) + goto UnixQFileInfoRetry; + + return rc; +} + +int +CIFSSMBUnixQPathInfo(const unsigned int xid, struct cifs_tcon *tcon, + const unsigned char *searchName, + FILE_UNIX_BASIC_INFO *pFindData, + const struct nls_table *nls_codepage, int remap) +{ +/* SMB_QUERY_FILE_UNIX_BASIC */ + TRANSACTION2_QPI_REQ *pSMB = NULL; + TRANSACTION2_QPI_RSP *pSMBr = NULL; + unsigned int in_len; + int rc = 0; + int bytes_returned = 0; + int name_len; + __u16 params, byte_count; + + cifs_dbg(FYI, "In QPathInfo (Unix) the path %s\n", searchName); +UnixQPathInfoRetry: + rc = smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + name_len = + cifsConvertToUTF16((__le16 *) pSMB->FileName, searchName, + PATH_MAX, nls_codepage, remap); + name_len++; /* trailing null */ + name_len *= 2; + } else { + name_len = copy_path_name(pSMB->FileName, searchName); + } + + params = 2 /* level */ + 4 /* reserved */ + name_len /* includes NUL */; + pSMB->TotalDataCount = 0; + pSMB->MaxParameterCount = cpu_to_le16(2); + /* BB find exact max SMB PDU from sess structure BB */ + pSMB->MaxDataCount = cpu_to_le16(4000); + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + pSMB->ParameterOffset = cpu_to_le16(offsetof( + struct smb_com_transaction2_qpi_req, InformationLevel)); + pSMB->DataCount = 0; + pSMB->DataOffset = 0; + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_QUERY_PATH_INFORMATION); + byte_count = params + 1 /* pad */ ; + pSMB->TotalParameterCount = cpu_to_le16(params); + pSMB->ParameterCount = pSMB->TotalParameterCount; + pSMB->InformationLevel = cpu_to_le16(SMB_QUERY_FILE_UNIX_BASIC); + pSMB->Reserved4 = 0; + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) { + cifs_dbg(FYI, "Send error in UnixQPathInfo = %d\n", rc); + } else { /* decode response */ + rc = validate_t2((struct smb_t2_rsp *)pSMBr); + + if (rc || get_bcc(&pSMBr->hdr) < sizeof(FILE_UNIX_BASIC_INFO)) { + cifs_dbg(VFS, "Malformed FILE_UNIX_BASIC_INFO response. Unix Extensions can be disabled on mount by specifying the nosfu mount option.\n"); + rc = smb_EIO2(smb_eio_trace_unixqpathinfo_bcc_too_small, + get_bcc(&pSMBr->hdr), sizeof(FILE_UNIX_BASIC_INFO)); + } else { + __u16 data_offset = le16_to_cpu(pSMBr->t2.DataOffset); + memcpy((char *) pFindData, + (char *) &pSMBr->hdr.Protocol + + data_offset, + sizeof(FILE_UNIX_BASIC_INFO)); + } + } + cifs_buf_release(pSMB); + if (rc == -EAGAIN) + goto UnixQPathInfoRetry; + + return rc; +} + +/* xid, tcon, searchName and codepage are input parms, rest are returned */ +int +CIFSFindFirst(const unsigned int xid, struct cifs_tcon *tcon, + const char *searchName, struct cifs_sb_info *cifs_sb, + __u16 *pnetfid, __u16 search_flags, + struct cifs_search_info *psrch_inf, bool msearch) +{ +/* level 257 SMB_ */ + TRANSACTION2_FFIRST_REQ *pSMB = NULL; + TRANSACTION2_FFIRST_RSP *pSMBr = NULL; + T2_FFIRST_RSP_PARMS *parms; + struct nls_table *nls_codepage; + unsigned int in_len, lnoff; + __u16 params, byte_count; + int bytes_returned = 0; + int name_len, remap; + int rc = 0; + + cifs_dbg(FYI, "In FindFirst for %s\n", searchName); + +findFirstRetry: + rc = smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + nls_codepage = cifs_sb->local_nls; + remap = cifs_remap(cifs_sb); + + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + name_len = + cifsConvertToUTF16((__le16 *) pSMB->FileName, searchName, + PATH_MAX, nls_codepage, remap); + /* We can not add the asterisk earlier in case + it got remapped to 0xF03A as if it were part of the + directory name instead of a wildcard */ + name_len *= 2; + if (msearch) { + pSMB->FileName[name_len] = CIFS_DIR_SEP(cifs_sb); + pSMB->FileName[name_len+1] = 0; + pSMB->FileName[name_len+2] = '*'; + pSMB->FileName[name_len+3] = 0; + name_len += 4; /* now the trailing null */ + /* null terminate just in case */ + pSMB->FileName[name_len] = 0; + pSMB->FileName[name_len+1] = 0; + name_len += 2; + } else if (!searchName[0]) { + pSMB->FileName[0] = CIFS_DIR_SEP(cifs_sb); + pSMB->FileName[1] = 0; + pSMB->FileName[2] = 0; + pSMB->FileName[3] = 0; + name_len = 4; + } + } else { + name_len = copy_path_name(pSMB->FileName, searchName); + if (msearch) { + if (WARN_ON_ONCE(name_len > PATH_MAX-2)) + name_len = PATH_MAX-2; + /* overwrite nul byte */ + pSMB->FileName[name_len-1] = CIFS_DIR_SEP(cifs_sb); + pSMB->FileName[name_len] = '*'; + pSMB->FileName[name_len+1] = 0; + name_len += 2; + } else if (!searchName[0]) { + pSMB->FileName[0] = CIFS_DIR_SEP(cifs_sb); + pSMB->FileName[1] = 0; + name_len = 2; + } + } + + params = 12 + name_len /* includes null */ ; + pSMB->TotalDataCount = 0; /* no EAs */ + pSMB->MaxParameterCount = cpu_to_le16(10); + pSMB->MaxDataCount = cpu_to_le16(CIFSMaxBufSize & 0xFFFFFF00); + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + byte_count = params + 1 /* pad */ ; + pSMB->TotalParameterCount = cpu_to_le16(params); + pSMB->ParameterCount = pSMB->TotalParameterCount; + pSMB->ParameterOffset = cpu_to_le16( + offsetof(struct smb_com_transaction2_ffirst_req, SearchAttributes)); + pSMB->DataCount = 0; + pSMB->DataOffset = 0; + pSMB->SetupCount = 1; /* one byte, no need to make endian neutral */ + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_FIND_FIRST); + pSMB->SearchAttributes = + cpu_to_le16(ATTR_READONLY | ATTR_HIDDEN | ATTR_SYSTEM | + ATTR_DIRECTORY); + pSMB->SearchCount = cpu_to_le16(msearch ? CIFSMaxBufSize/sizeof(FILE_UNIX_INFO) : 1); + pSMB->SearchFlags = cpu_to_le16(search_flags); + pSMB->InformationLevel = cpu_to_le16(psrch_inf->info_level); + + /* BB what should we set StorageType to? Does it matter? BB */ + pSMB->SearchStorageType = 0; + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + cifs_stats_inc(&tcon->stats.cifs_stats.num_ffirst); + + if (rc) { + /* + * BB: add logic to retry regular search if Unix search rejected + * unexpectedly by server. + */ + /* BB: add code to handle unsupported level rc */ + cifs_dbg(FYI, "Error in FindFirst = %d\n", rc); + cifs_buf_release(pSMB); + /* + * BB: eventually could optimize out free and realloc of buf for + * this case. + */ + if (rc == -EAGAIN) + goto findFirstRetry; + return rc; + } + /* decode response */ + rc = validate_t2((struct smb_t2_rsp *)pSMBr); + if (rc) { + cifs_buf_release(pSMB); + return rc; + } + + psrch_inf->unicode = !!(pSMBr->hdr.Flags2 & SMBFLG2_UNICODE); + psrch_inf->ntwrk_buf_start = (char *)pSMBr; + psrch_inf->smallBuf = false; + psrch_inf->srch_entries_start = (char *)&pSMBr->hdr.Protocol + + le16_to_cpu(pSMBr->t2.DataOffset); + + parms = (T2_FFIRST_RSP_PARMS *)((char *)&pSMBr->hdr.Protocol + + le16_to_cpu(pSMBr->t2.ParameterOffset)); + psrch_inf->endOfSearch = !!parms->EndofSearch; + + psrch_inf->entries_in_buffer = le16_to_cpu(parms->SearchCount); + psrch_inf->index_of_last_entry = 2 /* skip . and .. */ + + psrch_inf->entries_in_buffer; + lnoff = le16_to_cpu(parms->LastNameOffset); + if (CIFSMaxBufSize < lnoff) { + cifs_dbg(VFS, "ignoring corrupt resume name\n"); + psrch_inf->last_entry = NULL; + } else { + psrch_inf->last_entry = psrch_inf->srch_entries_start + lnoff; + if (pnetfid) + *pnetfid = parms->SearchHandle; + } + return 0; +} + +int CIFSFindNext(const unsigned int xid, struct cifs_tcon *tcon, + __u16 searchHandle, __u16 search_flags, + struct cifs_search_info *psrch_inf) +{ + TRANSACTION2_FNEXT_REQ *pSMB = NULL; + TRANSACTION2_FNEXT_RSP *pSMBr = NULL; + T2_FNEXT_RSP_PARMS *parms; + unsigned int name_len, in_len; + unsigned int lnoff; + __u16 params, byte_count; + char *response_data; + int bytes_returned; + int rc = 0; + + cifs_dbg(FYI, "In FindNext\n"); + + if (psrch_inf->endOfSearch) + return -ENOENT; + + rc = smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + params = 14; /* includes 2 bytes of null string, converted to LE below*/ + byte_count = 0; + pSMB->TotalDataCount = 0; /* no EAs */ + pSMB->MaxParameterCount = cpu_to_le16(8); + pSMB->MaxDataCount = cpu_to_le16(CIFSMaxBufSize & 0xFFFFFF00); + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + pSMB->ParameterOffset = cpu_to_le16( + offsetof(struct smb_com_transaction2_fnext_req, SearchHandle)); + pSMB->DataCount = 0; + pSMB->DataOffset = 0; + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_FIND_NEXT); + pSMB->SearchHandle = searchHandle; /* always kept as le */ + pSMB->SearchCount = + cpu_to_le16(CIFSMaxBufSize / sizeof(FILE_UNIX_INFO)); + pSMB->InformationLevel = cpu_to_le16(psrch_inf->info_level); + pSMB->ResumeKey = psrch_inf->resume_key; + pSMB->SearchFlags = cpu_to_le16(search_flags); + + name_len = psrch_inf->resume_name_len; + params += name_len; + if (name_len < PATH_MAX) { + memcpy(pSMB->ResumeFileName, psrch_inf->presume_name, name_len); + byte_count += name_len; + /* 14 byte parm len above enough for 2 byte null terminator */ + pSMB->ResumeFileName[name_len] = 0; + pSMB->ResumeFileName[name_len+1] = 0; + } else { + cifs_buf_release(pSMB); + return -EINVAL; + } + byte_count = params + 1 /* pad */ ; + pSMB->TotalParameterCount = cpu_to_le16(params); + pSMB->ParameterCount = pSMB->TotalParameterCount; + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + cifs_stats_inc(&tcon->stats.cifs_stats.num_fnext); + + if (rc) { + cifs_buf_release(pSMB); + if (rc == -EBADF) { + psrch_inf->endOfSearch = true; + rc = 0; /* search probably was closed at end of search*/ + } else { + cifs_dbg(FYI, "FindNext returned = %d\n", rc); + } + return rc; + } + + /* decode response */ + rc = validate_t2((struct smb_t2_rsp *)pSMBr); + if (rc) { + cifs_buf_release(pSMB); + return rc; + } + /* BB fixme add lock for file (srch_info) struct here */ + psrch_inf->unicode = !!(pSMBr->hdr.Flags2 & SMBFLG2_UNICODE); + response_data = (char *)&pSMBr->hdr.Protocol + + le16_to_cpu(pSMBr->t2.ParameterOffset); + parms = (T2_FNEXT_RSP_PARMS *)response_data; + response_data = (char *)&pSMBr->hdr.Protocol + + le16_to_cpu(pSMBr->t2.DataOffset); + + if (psrch_inf->smallBuf) + cifs_small_buf_release(psrch_inf->ntwrk_buf_start); + else + cifs_buf_release(psrch_inf->ntwrk_buf_start); + + psrch_inf->srch_entries_start = response_data; + psrch_inf->ntwrk_buf_start = (char *)pSMB; + psrch_inf->smallBuf = false; + psrch_inf->endOfSearch = !!parms->EndofSearch; + psrch_inf->entries_in_buffer = le16_to_cpu(parms->SearchCount); + psrch_inf->index_of_last_entry += psrch_inf->entries_in_buffer; + lnoff = le16_to_cpu(parms->LastNameOffset); + if (CIFSMaxBufSize < lnoff) { + cifs_dbg(VFS, "ignoring corrupt resume name\n"); + psrch_inf->last_entry = NULL; + } else { + psrch_inf->last_entry = + psrch_inf->srch_entries_start + lnoff; + } + /* BB fixme add unlock here */ + + /* + * BB: On error, should we leave previous search buf + * (and count and last entry fields) intact or free the previous one? + * + * Note: On -EAGAIN error only caller can retry on handle based calls + * since file handle passed in no longer valid. + */ + return 0; +} + +int +CIFSFindClose(const unsigned int xid, struct cifs_tcon *tcon, + const __u16 searchHandle) +{ + int rc = 0; + FINDCLOSE_REQ *pSMB = NULL; + unsigned int in_len; + + cifs_dbg(FYI, "In CIFSSMBFindClose\n"); + rc = small_smb_init(SMB_COM_FIND_CLOSE2, 1, tcon, (void **)&pSMB); + + /* no sense returning error if session restarted + as file handle has been closed */ + if (rc == -EAGAIN) + return 0; + if (rc < 0) + return rc; + in_len = rc; + + pSMB->FileID = searchHandle; + pSMB->ByteCount = 0; + rc = SendReceiveNoRsp(xid, tcon->ses, (char *) pSMB, in_len, 0); + cifs_small_buf_release(pSMB); + if (rc) + cifs_dbg(VFS, "Send error in FindClose = %d\n", rc); + + cifs_stats_inc(&tcon->stats.cifs_stats.num_fclose); + + /* Since session is dead, search handle closed on server already */ + if (rc == -EAGAIN) + rc = 0; + + return rc; +} + +int +CIFSGetSrvInodeNumber(const unsigned int xid, struct cifs_tcon *tcon, + const char *search_name, __u64 *inode_number, + const struct nls_table *nls_codepage, int remap) +{ + int rc = 0; + TRANSACTION2_QPI_REQ *pSMB = NULL; + TRANSACTION2_QPI_RSP *pSMBr = NULL; + unsigned int in_len; + int name_len, bytes_returned; + __u16 params, byte_count; + + cifs_dbg(FYI, "In GetSrvInodeNum for %s\n", search_name); + if (tcon == NULL) + return -ENODEV; + +GetInodeNumberRetry: + rc = smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + name_len = + cifsConvertToUTF16((__le16 *) pSMB->FileName, + search_name, PATH_MAX, nls_codepage, + remap); + name_len++; /* trailing null */ + name_len *= 2; + } else { + name_len = copy_path_name(pSMB->FileName, search_name); + } + + params = 2 /* level */ + 4 /* rsrvd */ + name_len /* incl null */ ; + pSMB->TotalDataCount = 0; + pSMB->MaxParameterCount = cpu_to_le16(2); + /* BB find exact max data count below from sess structure BB */ + pSMB->MaxDataCount = cpu_to_le16(4000); + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + pSMB->ParameterOffset = cpu_to_le16(offsetof( + struct smb_com_transaction2_qpi_req, InformationLevel)); + pSMB->DataCount = 0; + pSMB->DataOffset = 0; + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_QUERY_PATH_INFORMATION); + byte_count = params + 1 /* pad */ ; + pSMB->TotalParameterCount = cpu_to_le16(params); + pSMB->ParameterCount = pSMB->TotalParameterCount; + pSMB->InformationLevel = cpu_to_le16(SMB_QUERY_FILE_INTERNAL_INFO); + pSMB->Reserved4 = 0; + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) { + cifs_dbg(FYI, "error %d in QueryInternalInfo\n", rc); + } else { + /* decode response */ + rc = validate_t2((struct smb_t2_rsp *)pSMBr); + /* BB also check enough total bytes returned */ + if (rc || get_bcc(&pSMBr->hdr) < 2) + /* If rc should we check for EOPNOSUPP and + disable the srvino flag? or in caller? */ + rc = smb_EIO2(smb_eio_trace_getsrvinonum_bcc_too_small, + get_bcc(&pSMBr->hdr), 2); + else { + __u16 data_offset = le16_to_cpu(pSMBr->t2.DataOffset); + __u16 count = le16_to_cpu(pSMBr->t2.DataCount); + struct file_internal_info *pfinfo; + /* BB Do we need a cast or hash here ? */ + if (count < 8) { + cifs_dbg(FYI, "Invalid size ret in QryIntrnlInf\n"); + rc = smb_EIO2(smb_eio_trace_getsrvinonum_size, + count, 8); + goto GetInodeNumOut; + } + pfinfo = (struct file_internal_info *) + (data_offset + (char *) &pSMBr->hdr.Protocol); + *inode_number = le64_to_cpu(pfinfo->UniqueId); + } + } +GetInodeNumOut: + cifs_buf_release(pSMB); + if (rc == -EAGAIN) + goto GetInodeNumberRetry; + return rc; +} + +int +CIFSGetDFSRefer(const unsigned int xid, struct cifs_ses *ses, + const char *search_name, struct dfs_info3_param **target_nodes, + unsigned int *num_of_nodes, + const struct nls_table *nls_codepage, int remap) +{ +/* TRANS2_GET_DFS_REFERRAL */ + TRANSACTION2_GET_DFS_REFER_REQ *pSMB = NULL; + TRANSACTION2_GET_DFS_REFER_RSP *pSMBr = NULL; + unsigned int in_len; + int rc = 0; + int bytes_returned; + int name_len; + __u16 params, byte_count; + *num_of_nodes = 0; + *target_nodes = NULL; + + cifs_dbg(FYI, "In GetDFSRefer the path %s\n", search_name); + if (ses == NULL || ses->tcon_ipc == NULL) + return -ENODEV; + +getDFSRetry: + /* + * Use smb_init_no_reconnect() instead of smb_init() as + * CIFSGetDFSRefer() may be called from cifs_reconnect_tcon() and thus + * causing an infinite recursion. + */ + rc = smb_init(SMB_COM_TRANSACTION2, 15, ses->tcon_ipc, + (void **)&pSMB, (void **)&pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + /* server pointer checked in called function, + but should never be null here anyway */ + pSMB->hdr.Mid = get_next_mid(ses->server); + pSMB->hdr.Tid = ses->tcon_ipc->tid; + pSMB->hdr.Uid = ses->Suid; + if (ses->capabilities & CAP_STATUS32) + pSMB->hdr.Flags2 |= SMBFLG2_ERR_STATUS; + if (ses->capabilities & CAP_DFS) + pSMB->hdr.Flags2 |= SMBFLG2_DFS; + + if (ses->capabilities & CAP_UNICODE) { + pSMB->hdr.Flags2 |= SMBFLG2_UNICODE; + name_len = + cifsConvertToUTF16((__le16 *) pSMB->RequestFileName, + search_name, PATH_MAX, nls_codepage, + remap); + name_len++; /* trailing null */ + name_len *= 2; + } else { /* BB improve the check for buffer overruns BB */ + name_len = copy_path_name(pSMB->RequestFileName, search_name); + } + + if (ses->server->sign) + pSMB->hdr.Flags2 |= SMBFLG2_SECURITY_SIGNATURE; + + pSMB->hdr.Uid = ses->Suid; + + params = 2 /* level */ + name_len /*includes null */ ; + pSMB->TotalDataCount = 0; + pSMB->DataCount = 0; + pSMB->DataOffset = 0; + pSMB->MaxParameterCount = 0; + /* BB find exact max SMB PDU from sess structure BB */ + pSMB->MaxDataCount = cpu_to_le16(4000); + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + pSMB->ParameterOffset = cpu_to_le16(offsetof( + struct smb_com_transaction2_get_dfs_refer_req, MaxReferralLevel)); + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_GET_DFS_REFERRAL); + byte_count = params + 3 /* pad */ ; + pSMB->ParameterCount = cpu_to_le16(params); + pSMB->TotalParameterCount = pSMB->ParameterCount; + pSMB->MaxReferralLevel = cpu_to_le16(3); + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + + rc = SendReceive(xid, ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) { + cifs_dbg(FYI, "Send error in GetDFSRefer = %d\n", rc); + goto GetDFSRefExit; + } + rc = validate_t2((struct smb_t2_rsp *)pSMBr); + + /* BB Also check if enough total bytes returned? */ + if (rc || get_bcc(&pSMBr->hdr) < 17) { + rc = smb_EIO2(smb_eio_trace_getdfsrefer_bcc_too_small, + get_bcc(&pSMBr->hdr), 17); + goto GetDFSRefExit; + } + + cifs_dbg(FYI, "Decoding GetDFSRefer response BCC: %d Offset %d\n", + get_bcc(&pSMBr->hdr), le16_to_cpu(pSMBr->t2.DataOffset)); + + /* parse returned result into more usable form */ + rc = parse_dfs_referrals(&pSMBr->dfs_data, + le16_to_cpu(pSMBr->t2.DataCount), + num_of_nodes, target_nodes, nls_codepage, + remap, search_name, + (pSMBr->hdr.Flags2 & SMBFLG2_UNICODE) != 0); + +GetDFSRefExit: + cifs_buf_release(pSMB); + + if (rc == -EAGAIN) + goto getDFSRetry; + + return rc; +} + +/* Query File System Info such as free space to old servers such as Win 9x */ +int +SMBOldQFSInfo(const unsigned int xid, struct cifs_tcon *tcon, + struct kstatfs *FSData) +{ +/* level 0x01 SMB_QUERY_FILE_SYSTEM_INFO */ + TRANSACTION2_QFSI_REQ *pSMB = NULL; + TRANSACTION2_QFSI_RSP *pSMBr = NULL; + FILE_SYSTEM_ALLOC_INFO *response_data; + unsigned int in_len; + int rc = 0; + int bytes_returned = 0; + __u16 params, byte_count; + + cifs_dbg(FYI, "OldQFSInfo\n"); +oldQFSInfoRetry: + rc = smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + params = 2; /* level */ + pSMB->TotalDataCount = 0; + pSMB->MaxParameterCount = cpu_to_le16(2); + pSMB->MaxDataCount = cpu_to_le16(1000); + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + byte_count = params + 1 /* pad */ ; + pSMB->TotalParameterCount = cpu_to_le16(params); + pSMB->ParameterCount = pSMB->TotalParameterCount; + pSMB->ParameterOffset = cpu_to_le16(offsetof( + struct smb_com_transaction2_qfsi_req, InformationLevel)); + pSMB->DataCount = 0; + pSMB->DataOffset = 0; + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_QUERY_FS_INFORMATION); + pSMB->InformationLevel = cpu_to_le16(SMB_INFO_ALLOCATION); + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) { + cifs_dbg(FYI, "Send error in QFSInfo = %d\n", rc); + } else { /* decode response */ + rc = validate_t2((struct smb_t2_rsp *)pSMBr); + + if (rc || get_bcc(&pSMBr->hdr) < 18) + rc = smb_EIO2(smb_eio_trace_oldqfsinfo_bcc_too_small, + get_bcc(&pSMBr->hdr), 18); + else { + __u16 data_offset = le16_to_cpu(pSMBr->t2.DataOffset); + cifs_dbg(FYI, "qfsinf resp BCC: %d Offset %d\n", + get_bcc(&pSMBr->hdr), data_offset); + + response_data = (FILE_SYSTEM_ALLOC_INFO *) + (((char *) &pSMBr->hdr.Protocol) + data_offset); + FSData->f_bsize = + le16_to_cpu(response_data->BytesPerSector) * + le32_to_cpu(response_data-> + SectorsPerAllocationUnit); + /* + * much prefer larger but if server doesn't report + * a valid size than 4K is a reasonable minimum + */ + if (FSData->f_bsize < 512) + FSData->f_bsize = 4096; + + FSData->f_blocks = + le32_to_cpu(response_data->TotalAllocationUnits); + FSData->f_bfree = FSData->f_bavail = + le32_to_cpu(response_data->FreeAllocationUnits); + cifs_dbg(FYI, "Blocks: %lld Free: %lld Block size %ld\n", + (unsigned long long)FSData->f_blocks, + (unsigned long long)FSData->f_bfree, + FSData->f_bsize); + } + } + cifs_buf_release(pSMB); + + if (rc == -EAGAIN) + goto oldQFSInfoRetry; + + return rc; +} + +int +CIFSSMBQFSInfo(const unsigned int xid, struct cifs_tcon *tcon, + struct kstatfs *FSData) +{ +/* level 0x103 SMB_QUERY_FILE_SYSTEM_INFO */ + TRANSACTION2_QFSI_REQ *pSMB = NULL; + TRANSACTION2_QFSI_RSP *pSMBr = NULL; + FILE_SYSTEM_SIZE_INFO *response_data; + unsigned int in_len; + int rc = 0; + int bytes_returned = 0; + __u16 params, byte_count; + + cifs_dbg(FYI, "In QFSInfo\n"); +QFSInfoRetry: + rc = smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + params = 2; /* level */ + pSMB->TotalDataCount = 0; + pSMB->MaxParameterCount = cpu_to_le16(2); + pSMB->MaxDataCount = cpu_to_le16(1000); + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + byte_count = params + 1 /* pad */ ; + pSMB->TotalParameterCount = cpu_to_le16(params); + pSMB->ParameterCount = pSMB->TotalParameterCount; + pSMB->ParameterOffset = cpu_to_le16(offsetof( + struct smb_com_transaction2_qfsi_req, InformationLevel)); + pSMB->DataCount = 0; + pSMB->DataOffset = 0; + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_QUERY_FS_INFORMATION); + pSMB->InformationLevel = cpu_to_le16(SMB_QUERY_FS_SIZE_INFO); + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) { + cifs_dbg(FYI, "Send error in QFSInfo = %d\n", rc); + } else { /* decode response */ + rc = validate_t2((struct smb_t2_rsp *)pSMBr); + + if (rc || get_bcc(&pSMBr->hdr) < 24) + rc = smb_EIO2(smb_eio_trace_qfsinfo_bcc_too_small, + get_bcc(&pSMBr->hdr), 24); + else { + __u16 data_offset = le16_to_cpu(pSMBr->t2.DataOffset); + + response_data = + (FILE_SYSTEM_SIZE_INFO + *) (((char *) &pSMBr->hdr.Protocol) + + data_offset); + FSData->f_bsize = + le32_to_cpu(response_data->BytesPerSector) * + le32_to_cpu(response_data-> + SectorsPerAllocationUnit); + /* + * much prefer larger but if server doesn't report + * a valid size than 4K is a reasonable minimum + */ + if (FSData->f_bsize < 512) + FSData->f_bsize = 4096; + + FSData->f_blocks = + le64_to_cpu(response_data->TotalAllocationUnits); + FSData->f_bfree = FSData->f_bavail = + le64_to_cpu(response_data->AvailableAllocationUnits); + cifs_dbg(FYI, "Blocks: %lld Free: %lld Block size %ld\n", + (unsigned long long)FSData->f_blocks, + (unsigned long long)FSData->f_bfree, + FSData->f_bsize); + } + } + cifs_buf_release(pSMB); + + if (rc == -EAGAIN) + goto QFSInfoRetry; + + return rc; +} + +int +CIFSSMBQFSAttributeInfo(const unsigned int xid, struct cifs_tcon *tcon) +{ +/* level 0x105 SMB_QUERY_FILE_SYSTEM_INFO */ + TRANSACTION2_QFSI_REQ *pSMB = NULL; + TRANSACTION2_QFSI_RSP *pSMBr = NULL; + FILE_SYSTEM_ATTRIBUTE_INFO *response_data; + unsigned int in_len; + int rc = 0; + int bytes_returned = 0; + __u16 params, byte_count; + + cifs_dbg(FYI, "In QFSAttributeInfo\n"); +QFSAttributeRetry: + rc = smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + params = 2; /* level */ + pSMB->TotalDataCount = 0; + pSMB->MaxParameterCount = cpu_to_le16(2); + /* BB find exact max SMB PDU from sess structure BB */ + pSMB->MaxDataCount = cpu_to_le16(1000); + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + byte_count = params + 1 /* pad */ ; + pSMB->TotalParameterCount = cpu_to_le16(params); + pSMB->ParameterCount = pSMB->TotalParameterCount; + pSMB->ParameterOffset = cpu_to_le16(offsetof( + struct smb_com_transaction2_qfsi_req, InformationLevel)); + pSMB->DataCount = 0; + pSMB->DataOffset = 0; + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_QUERY_FS_INFORMATION); + pSMB->InformationLevel = cpu_to_le16(SMB_QUERY_FS_ATTRIBUTE_INFO); + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) { + cifs_dbg(VFS, "Send error in QFSAttributeInfo = %d\n", rc); + } else { /* decode response */ + rc = validate_t2((struct smb_t2_rsp *)pSMBr); + + if (rc || get_bcc(&pSMBr->hdr) < 13) { + /* BB also check if enough bytes returned */ + rc = smb_EIO2(smb_eio_trace_qfsattrinfo_bcc_too_small, + get_bcc(&pSMBr->hdr), 13); + } else { + __u16 data_offset = le16_to_cpu(pSMBr->t2.DataOffset); + response_data = + (FILE_SYSTEM_ATTRIBUTE_INFO + *) (((char *) &pSMBr->hdr.Protocol) + + data_offset); + memcpy(&tcon->fsAttrInfo, response_data, + sizeof(FILE_SYSTEM_ATTRIBUTE_INFO)); + } + } + cifs_buf_release(pSMB); + + if (rc == -EAGAIN) + goto QFSAttributeRetry; + + return rc; +} + +int +CIFSSMBQFSDeviceInfo(const unsigned int xid, struct cifs_tcon *tcon) +{ +/* level 0x104 SMB_QUERY_FILE_SYSTEM_INFO */ + TRANSACTION2_QFSI_REQ *pSMB = NULL; + TRANSACTION2_QFSI_RSP *pSMBr = NULL; + FILE_SYSTEM_DEVICE_INFO *response_data; + unsigned int in_len; + int rc = 0; + int bytes_returned = 0; + __u16 params, byte_count; + + cifs_dbg(FYI, "In QFSDeviceInfo\n"); +QFSDeviceRetry: + rc = smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + params = 2; /* level */ + pSMB->TotalDataCount = 0; + pSMB->MaxParameterCount = cpu_to_le16(2); + /* BB find exact max SMB PDU from sess structure BB */ + pSMB->MaxDataCount = cpu_to_le16(1000); + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + byte_count = params + 1 /* pad */ ; + pSMB->TotalParameterCount = cpu_to_le16(params); + pSMB->ParameterCount = pSMB->TotalParameterCount; + pSMB->ParameterOffset = cpu_to_le16(offsetof( + struct smb_com_transaction2_qfsi_req, InformationLevel)); + + pSMB->DataCount = 0; + pSMB->DataOffset = 0; + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_QUERY_FS_INFORMATION); + pSMB->InformationLevel = cpu_to_le16(SMB_QUERY_FS_DEVICE_INFO); + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) { + cifs_dbg(FYI, "Send error in QFSDeviceInfo = %d\n", rc); + } else { /* decode response */ + rc = validate_t2((struct smb_t2_rsp *)pSMBr); + + if (rc || get_bcc(&pSMBr->hdr) < + sizeof(FILE_SYSTEM_DEVICE_INFO)) + rc = smb_EIO2(smb_eio_trace_qfsdevinfo_bcc_too_small, + get_bcc(&pSMBr->hdr), + sizeof(FILE_SYSTEM_DEVICE_INFO)); + else { + __u16 data_offset = le16_to_cpu(pSMBr->t2.DataOffset); + response_data = + (FILE_SYSTEM_DEVICE_INFO *) + (((char *) &pSMBr->hdr.Protocol) + + data_offset); + memcpy(&tcon->fsDevInfo, response_data, + sizeof(FILE_SYSTEM_DEVICE_INFO)); + } + } + cifs_buf_release(pSMB); + + if (rc == -EAGAIN) + goto QFSDeviceRetry; + + return rc; +} + +int +CIFSSMBQFSUnixInfo(const unsigned int xid, struct cifs_tcon *tcon) +{ +/* level 0x200 SMB_QUERY_CIFS_UNIX_INFO */ + TRANSACTION2_QFSI_REQ *pSMB = NULL; + TRANSACTION2_QFSI_RSP *pSMBr = NULL; + FILE_SYSTEM_UNIX_INFO *response_data; + unsigned int in_len; + int rc = 0; + int bytes_returned = 0; + __u16 params, byte_count; + + cifs_dbg(FYI, "In QFSUnixInfo\n"); +QFSUnixRetry: + rc = smb_init_no_reconnect(SMB_COM_TRANSACTION2, 15, tcon, + (void **) &pSMB, (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + params = 2; /* level */ + pSMB->TotalDataCount = 0; + pSMB->DataCount = 0; + pSMB->DataOffset = 0; + pSMB->MaxParameterCount = cpu_to_le16(2); + /* BB find exact max SMB PDU from sess structure BB */ + pSMB->MaxDataCount = cpu_to_le16(100); + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + byte_count = params + 1 /* pad */ ; + pSMB->ParameterCount = cpu_to_le16(params); + pSMB->TotalParameterCount = pSMB->ParameterCount; + pSMB->ParameterOffset = cpu_to_le16(offsetof(struct + smb_com_transaction2_qfsi_req, InformationLevel)); + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_QUERY_FS_INFORMATION); + pSMB->InformationLevel = cpu_to_le16(SMB_QUERY_CIFS_UNIX_INFO); + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) { + cifs_dbg(VFS, "Send error in QFSUnixInfo = %d\n", rc); + } else { /* decode response */ + rc = validate_t2((struct smb_t2_rsp *)pSMBr); + + if (rc || get_bcc(&pSMBr->hdr) < 13) { + rc = smb_EIO2(smb_eio_trace_qfsunixinfo_bcc_too_small, + get_bcc(&pSMBr->hdr), 13); + } else { + __u16 data_offset = le16_to_cpu(pSMBr->t2.DataOffset); + response_data = + (FILE_SYSTEM_UNIX_INFO + *) (((char *) &pSMBr->hdr.Protocol) + + data_offset); + memcpy(&tcon->fsUnixInfo, response_data, + sizeof(FILE_SYSTEM_UNIX_INFO)); + } + } + cifs_buf_release(pSMB); + + if (rc == -EAGAIN) + goto QFSUnixRetry; + + + return rc; +} + +int +CIFSSMBSetFSUnixInfo(const unsigned int xid, struct cifs_tcon *tcon, __u64 cap) +{ +/* level 0x200 SMB_SET_CIFS_UNIX_INFO */ + TRANSACTION2_SETFSI_REQ *pSMB = NULL; + TRANSACTION2_SETFSI_RSP *pSMBr = NULL; + unsigned int in_len; + int rc = 0; + int bytes_returned = 0; + __u16 params, param_offset, offset, byte_count; + + cifs_dbg(FYI, "In SETFSUnixInfo\n"); +SETFSUnixRetry: + /* BB switch to small buf init to save memory */ + rc = smb_init_no_reconnect(SMB_COM_TRANSACTION2, 15, tcon, + (void **) &pSMB, (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + params = 4; /* 2 bytes zero followed by info level. */ + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + param_offset = offsetof(struct smb_com_transaction2_setfsi_req, FileNum); + offset = param_offset + params; + + pSMB->MaxParameterCount = cpu_to_le16(4); + /* BB find exact max SMB PDU from sess structure BB */ + pSMB->MaxDataCount = cpu_to_le16(100); + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_SET_FS_INFORMATION); + byte_count = 1 /* pad */ + params + 12; + + pSMB->DataCount = cpu_to_le16(12); + pSMB->ParameterCount = cpu_to_le16(params); + pSMB->TotalDataCount = pSMB->DataCount; + pSMB->TotalParameterCount = pSMB->ParameterCount; + pSMB->ParameterOffset = cpu_to_le16(param_offset); + pSMB->DataOffset = cpu_to_le16(offset); + + /* Params. */ + pSMB->FileNum = 0; + pSMB->InformationLevel = cpu_to_le16(SMB_SET_CIFS_UNIX_INFO); + + /* Data. */ + pSMB->ClientUnixMajor = cpu_to_le16(CIFS_UNIX_MAJOR_VERSION); + pSMB->ClientUnixMinor = cpu_to_le16(CIFS_UNIX_MINOR_VERSION); + pSMB->ClientUnixCap = cpu_to_le64(cap); + + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) { + cifs_dbg(VFS, "Send error in SETFSUnixInfo = %d\n", rc); + } else { /* decode response */ + rc = validate_t2((struct smb_t2_rsp *)pSMBr); + if (rc) + rc = -EIO; /* bad smb */ + } + cifs_buf_release(pSMB); + + if (rc == -EAGAIN) + goto SETFSUnixRetry; + + return rc; +} + + + +int +CIFSSMBQFSPosixInfo(const unsigned int xid, struct cifs_tcon *tcon, + struct kstatfs *FSData) +{ +/* level 0x201 SMB_QUERY_CIFS_POSIX_INFO */ + TRANSACTION2_QFSI_REQ *pSMB = NULL; + TRANSACTION2_QFSI_RSP *pSMBr = NULL; + FILE_SYSTEM_POSIX_INFO *response_data; + unsigned int in_len; + int rc = 0; + int bytes_returned = 0; + __u16 params, byte_count; + + cifs_dbg(FYI, "In QFSPosixInfo\n"); +QFSPosixRetry: + rc = smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + params = 2; /* level */ + pSMB->TotalDataCount = 0; + pSMB->DataCount = 0; + pSMB->DataOffset = 0; + pSMB->MaxParameterCount = cpu_to_le16(2); + /* BB find exact max SMB PDU from sess structure BB */ + pSMB->MaxDataCount = cpu_to_le16(100); + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + byte_count = params + 1 /* pad */ ; + pSMB->ParameterCount = cpu_to_le16(params); + pSMB->TotalParameterCount = pSMB->ParameterCount; + pSMB->ParameterOffset = cpu_to_le16(offsetof(struct + smb_com_transaction2_qfsi_req, InformationLevel)); + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_QUERY_FS_INFORMATION); + pSMB->InformationLevel = cpu_to_le16(SMB_QUERY_POSIX_FS_INFO); + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) { + cifs_dbg(FYI, "Send error in QFSUnixInfo = %d\n", rc); + } else { /* decode response */ + rc = validate_t2((struct smb_t2_rsp *)pSMBr); + + if (rc || get_bcc(&pSMBr->hdr) < 13) { + rc = smb_EIO2(smb_eio_trace_qfsposixinfo_bcc_too_small, + get_bcc(&pSMBr->hdr), 13); + } else { + __u16 data_offset = le16_to_cpu(pSMBr->t2.DataOffset); + response_data = + (FILE_SYSTEM_POSIX_INFO + *) (((char *) &pSMBr->hdr.Protocol) + + data_offset); + FSData->f_bsize = + le32_to_cpu(response_data->BlockSize); + /* + * much prefer larger but if server doesn't report + * a valid size than 4K is a reasonable minimum + */ + if (FSData->f_bsize < 512) + FSData->f_bsize = 4096; + + FSData->f_blocks = + le64_to_cpu(response_data->TotalBlocks); + FSData->f_bfree = + le64_to_cpu(response_data->BlocksAvail); + if (response_data->UserBlocksAvail == cpu_to_le64(-1)) { + FSData->f_bavail = FSData->f_bfree; + } else { + FSData->f_bavail = + le64_to_cpu(response_data->UserBlocksAvail); + } + if (response_data->TotalFileNodes != cpu_to_le64(-1)) + FSData->f_files = + le64_to_cpu(response_data->TotalFileNodes); + if (response_data->FreeFileNodes != cpu_to_le64(-1)) + FSData->f_ffree = + le64_to_cpu(response_data->FreeFileNodes); + } + } + cifs_buf_release(pSMB); + + if (rc == -EAGAIN) + goto QFSPosixRetry; + + return rc; +} + + +/* + * We can not use write of zero bytes trick to set file size due to need for + * large file support. Also note that this SetPathInfo is preferred to + * SetFileInfo based method in next routine which is only needed to work around + * a sharing violation bugin Samba which this routine can run into. + */ +int +CIFSSMBSetEOF(const unsigned int xid, struct cifs_tcon *tcon, + const char *file_name, __u64 size, struct cifs_sb_info *cifs_sb, + bool set_allocation, struct dentry *dentry) +{ + struct smb_com_transaction2_spi_req *pSMB = NULL; + struct smb_com_transaction2_spi_rsp *pSMBr = NULL; + struct file_end_of_file_info *parm_data; + unsigned int in_len; + int name_len; + int rc = 0; + int bytes_returned = 0; + int remap = cifs_remap(cifs_sb); + + __u16 params, byte_count, data_count, param_offset, offset; + + cifs_dbg(FYI, "In SetEOF\n"); +SetEOFRetry: + rc = smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + name_len = + cifsConvertToUTF16((__le16 *) pSMB->FileName, file_name, + PATH_MAX, cifs_sb->local_nls, remap); + name_len++; /* trailing null */ + name_len *= 2; + } else { + name_len = copy_path_name(pSMB->FileName, file_name); + } + params = 6 + name_len; + data_count = sizeof(struct file_end_of_file_info); + pSMB->MaxParameterCount = cpu_to_le16(2); + pSMB->MaxDataCount = cpu_to_le16(4100); + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + param_offset = offsetof(struct smb_com_transaction2_spi_req, + InformationLevel); + offset = param_offset + params; + if (set_allocation) { + if (tcon->ses->capabilities & CAP_INFOLEVEL_PASSTHRU) + pSMB->InformationLevel = + cpu_to_le16(SMB_SET_FILE_ALLOCATION_INFO2); + else + pSMB->InformationLevel = + cpu_to_le16(SMB_SET_FILE_ALLOCATION_INFO); + } else /* Set File Size */ { + if (tcon->ses->capabilities & CAP_INFOLEVEL_PASSTHRU) + pSMB->InformationLevel = + cpu_to_le16(SMB_SET_FILE_END_OF_FILE_INFO2); + else + pSMB->InformationLevel = + cpu_to_le16(SMB_SET_FILE_END_OF_FILE_INFO); + } + + parm_data = + (struct file_end_of_file_info *) (((char *) &pSMB->hdr.Protocol) + + offset); + pSMB->ParameterOffset = cpu_to_le16(param_offset); + pSMB->DataOffset = cpu_to_le16(offset); + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_SET_PATH_INFORMATION); + byte_count = 3 /* pad */ + params + data_count; + pSMB->DataCount = cpu_to_le16(data_count); + pSMB->TotalDataCount = pSMB->DataCount; + pSMB->ParameterCount = cpu_to_le16(params); + pSMB->TotalParameterCount = pSMB->ParameterCount; + pSMB->Reserved4 = 0; + in_len += byte_count; + parm_data->FileSize = cpu_to_le64(size); + pSMB->ByteCount = cpu_to_le16(byte_count); + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) + cifs_dbg(FYI, "SetPathInfo (file size) returned %d\n", rc); + + cifs_buf_release(pSMB); + + if (rc == -EAGAIN) + goto SetEOFRetry; + + return rc; +} + +int +CIFSSMBSetFileSize(const unsigned int xid, struct cifs_tcon *tcon, + struct cifsFileInfo *cfile, __u64 size, bool set_allocation) +{ + struct smb_com_transaction2_sfi_req *pSMB = NULL; + struct file_end_of_file_info *parm_data; + unsigned int in_len; + int rc = 0; + __u16 params, param_offset, offset, byte_count, count; + + cifs_dbg(FYI, "SetFileSize (via SetFileInfo) %lld\n", + (long long)size); + rc = small_smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB); + if (rc < 0) + return rc; + in_len = rc; + + pSMB->hdr.Pid = cpu_to_le16((__u16)cfile->pid); + pSMB->hdr.PidHigh = cpu_to_le16((__u16)(cfile->pid >> 16)); + + params = 6; + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + param_offset = offsetof(struct smb_com_transaction2_sfi_req, Fid); + offset = param_offset + params; + + count = sizeof(struct file_end_of_file_info); + pSMB->MaxParameterCount = cpu_to_le16(2); + /* BB find exact max SMB PDU from sess structure BB */ + pSMB->MaxDataCount = cpu_to_le16(1000); + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_SET_FILE_INFORMATION); + byte_count = 3 /* pad */ + params + count; + pSMB->DataCount = cpu_to_le16(count); + pSMB->ParameterCount = cpu_to_le16(params); + pSMB->TotalDataCount = pSMB->DataCount; + pSMB->TotalParameterCount = pSMB->ParameterCount; + pSMB->ParameterOffset = cpu_to_le16(param_offset); + parm_data = + (struct file_end_of_file_info *)(((char *)pSMB) + offset); + pSMB->DataOffset = cpu_to_le16(offset); + parm_data->FileSize = cpu_to_le64(size); + pSMB->Fid = cfile->fid.netfid; + if (set_allocation) { + if (tcon->ses->capabilities & CAP_INFOLEVEL_PASSTHRU) + pSMB->InformationLevel = + cpu_to_le16(SMB_SET_FILE_ALLOCATION_INFO2); + else + pSMB->InformationLevel = + cpu_to_le16(SMB_SET_FILE_ALLOCATION_INFO); + } else /* Set File Size */ { + if (tcon->ses->capabilities & CAP_INFOLEVEL_PASSTHRU) + pSMB->InformationLevel = + cpu_to_le16(SMB_SET_FILE_END_OF_FILE_INFO2); + else + pSMB->InformationLevel = + cpu_to_le16(SMB_SET_FILE_END_OF_FILE_INFO); + } + pSMB->Reserved4 = 0; + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + rc = SendReceiveNoRsp(xid, tcon->ses, (char *) pSMB, in_len, 0); + cifs_small_buf_release(pSMB); + if (rc) { + cifs_dbg(FYI, "Send error in SetFileInfo (SetFileSize) = %d\n", + rc); + } + + /* Note: On -EAGAIN error only caller can retry on handle based calls + since file handle passed in no longer valid */ + + return rc; +} + +int +SMBSetInformation(const unsigned int xid, struct cifs_tcon *tcon, + const char *fileName, __le32 attributes, __le64 write_time, + const struct nls_table *nls_codepage, + struct cifs_sb_info *cifs_sb) +{ + SETATTR_REQ *pSMB; + SETATTR_RSP *pSMBr; + struct timespec64 ts; + unsigned int in_len; + int bytes_returned; + int name_len; + int rc; + + cifs_dbg(FYI, "In %s path %s\n", __func__, fileName); + +retry: + rc = smb_init(SMB_COM_SETATTR, 8, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + name_len = + cifsConvertToUTF16((__le16 *) pSMB->fileName, + fileName, PATH_MAX, nls_codepage, + cifs_remap(cifs_sb)); + name_len++; /* trailing null */ + name_len *= 2; + } else { + name_len = copy_path_name(pSMB->fileName, fileName); + } + /* Only few attributes can be set by this command, others are not accepted by Win9x. */ + pSMB->attr = cpu_to_le16(le32_to_cpu(attributes) & + (ATTR_READONLY | ATTR_HIDDEN | ATTR_SYSTEM | ATTR_ARCHIVE)); + /* Zero write time value (in both NT and SETATTR formats) means to not change it. */ + if (le64_to_cpu(write_time) != 0) { + ts = cifs_NTtimeToUnix(write_time); + pSMB->last_write_time = cpu_to_le32(ts.tv_sec); + } + pSMB->BufferFormat = 0x04; + name_len++; /* account for buffer type byte */ + in_len += name_len; + pSMB->ByteCount = cpu_to_le16(name_len); + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) + cifs_dbg(FYI, "Send error in %s = %d\n", __func__, rc); + + cifs_buf_release(pSMB); + + if (rc == -EAGAIN) + goto retry; + + return rc; +} + +/* Some legacy servers such as NT4 require that the file times be set on + an open handle, rather than by pathname - this is awkward due to + potential access conflicts on the open, but it is unavoidable for these + old servers since the only other choice is to go from 100 nanosecond DCE + time and resort to the original setpathinfo level which takes the ancient + DOS time format with 2 second granularity */ +int +CIFSSMBSetFileInfo(const unsigned int xid, struct cifs_tcon *tcon, + const FILE_BASIC_INFO *data, __u16 fid, __u32 pid_of_opener) +{ + struct smb_com_transaction2_sfi_req *pSMB = NULL; + unsigned int in_len; + char *data_offset; + int rc = 0; + __u16 params, param_offset, offset, byte_count, count; + + cifs_dbg(FYI, "Set Times (via SetFileInfo)\n"); + rc = small_smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB); + if (rc < 0) + return rc; + in_len = rc; + + pSMB->hdr.Pid = cpu_to_le16((__u16)pid_of_opener); + pSMB->hdr.PidHigh = cpu_to_le16((__u16)(pid_of_opener >> 16)); + + params = 6; + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + param_offset = offsetof(struct smb_com_transaction2_sfi_req, Fid); + offset = param_offset + params; + + data_offset = (char *)pSMB + offset; + + count = sizeof(FILE_BASIC_INFO); + pSMB->MaxParameterCount = cpu_to_le16(2); + /* BB find max SMB PDU from sess */ + pSMB->MaxDataCount = cpu_to_le16(1000); + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_SET_FILE_INFORMATION); + byte_count = 3 /* pad */ + params + count; + pSMB->DataCount = cpu_to_le16(count); + pSMB->ParameterCount = cpu_to_le16(params); + pSMB->TotalDataCount = pSMB->DataCount; + pSMB->TotalParameterCount = pSMB->ParameterCount; + pSMB->ParameterOffset = cpu_to_le16(param_offset); + pSMB->DataOffset = cpu_to_le16(offset); + pSMB->Fid = fid; + if (tcon->ses->capabilities & CAP_INFOLEVEL_PASSTHRU) + pSMB->InformationLevel = cpu_to_le16(SMB_SET_FILE_BASIC_INFO2); + else + pSMB->InformationLevel = cpu_to_le16(SMB_SET_FILE_BASIC_INFO); + pSMB->Reserved4 = 0; + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + memcpy(data_offset, data, sizeof(FILE_BASIC_INFO)); + rc = SendReceiveNoRsp(xid, tcon->ses, (char *) pSMB, in_len, 0); + cifs_small_buf_release(pSMB); + if (rc) + cifs_dbg(FYI, "Send error in Set Time (SetFileInfo) = %d\n", + rc); + + /* Note: On -EAGAIN error only caller can retry on handle based calls + since file handle passed in no longer valid */ + + return rc; +} + +int +CIFSSMBSetFileDisposition(const unsigned int xid, struct cifs_tcon *tcon, + bool delete_file, __u16 fid, __u32 pid_of_opener) +{ + struct smb_com_transaction2_sfi_req *pSMB = NULL; + unsigned int in_len; + char *data_offset; + int rc = 0; + __u16 params, param_offset, offset, byte_count, count; + + cifs_dbg(FYI, "Set File Disposition (via SetFileInfo)\n"); + rc = small_smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB); + if (rc < 0) + return rc; + in_len = rc; + + pSMB->hdr.Pid = cpu_to_le16((__u16)pid_of_opener); + pSMB->hdr.PidHigh = cpu_to_le16((__u16)(pid_of_opener >> 16)); + + params = 6; + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + param_offset = offsetof(struct smb_com_transaction2_sfi_req, Fid); + offset = param_offset + params; + data_offset = (char *)(pSMB) + offset; + + count = 1; + pSMB->MaxParameterCount = cpu_to_le16(2); + /* BB find max SMB PDU from sess */ + pSMB->MaxDataCount = cpu_to_le16(1000); + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_SET_FILE_INFORMATION); + byte_count = 3 /* pad */ + params + count; + pSMB->DataCount = cpu_to_le16(count); + pSMB->ParameterCount = cpu_to_le16(params); + pSMB->TotalDataCount = pSMB->DataCount; + pSMB->TotalParameterCount = pSMB->ParameterCount; + pSMB->ParameterOffset = cpu_to_le16(param_offset); + pSMB->DataOffset = cpu_to_le16(offset); + pSMB->Fid = fid; + pSMB->InformationLevel = cpu_to_le16(SMB_SET_FILE_DISPOSITION_INFO); + pSMB->Reserved4 = 0; + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + *data_offset = delete_file ? 1 : 0; + rc = SendReceiveNoRsp(xid, tcon->ses, (char *) pSMB, in_len, 0); + cifs_small_buf_release(pSMB); + if (rc) + cifs_dbg(FYI, "Send error in SetFileDisposition = %d\n", rc); + + return rc; +} + +int +CIFSSMBSetPathInfo(const unsigned int xid, struct cifs_tcon *tcon, + const char *fileName, const FILE_BASIC_INFO *data, + const struct nls_table *nls_codepage, + struct cifs_sb_info *cifs_sb) +{ + TRANSACTION2_SPI_REQ *pSMB = NULL; + TRANSACTION2_SPI_RSP *pSMBr = NULL; + unsigned int in_len; + int name_len; + int rc = 0; + int bytes_returned = 0; + char *data_offset; + __u16 params, param_offset, offset, byte_count, count; + int remap = cifs_remap(cifs_sb); + + cifs_dbg(FYI, "In SetTimes\n"); + +SetTimesRetry: + rc = smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + name_len = + cifsConvertToUTF16((__le16 *) pSMB->FileName, fileName, + PATH_MAX, nls_codepage, remap); + name_len++; /* trailing null */ + name_len *= 2; + } else { + name_len = copy_path_name(pSMB->FileName, fileName); + } + + params = 6 + name_len; + count = sizeof(FILE_BASIC_INFO); + pSMB->MaxParameterCount = cpu_to_le16(2); + /* BB find max SMB PDU from sess structure BB */ + pSMB->MaxDataCount = cpu_to_le16(1000); + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + param_offset = offsetof(struct smb_com_transaction2_spi_req, + InformationLevel); + offset = param_offset + params; + data_offset = (char *)pSMB + offsetof(typeof(*pSMB), hdr.Protocol) + offset; + pSMB->ParameterOffset = cpu_to_le16(param_offset); + pSMB->DataOffset = cpu_to_le16(offset); + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_SET_PATH_INFORMATION); + byte_count = 3 /* pad */ + params + count; + + pSMB->DataCount = cpu_to_le16(count); + pSMB->ParameterCount = cpu_to_le16(params); + pSMB->TotalDataCount = pSMB->DataCount; + pSMB->TotalParameterCount = pSMB->ParameterCount; + if (tcon->ses->capabilities & CAP_INFOLEVEL_PASSTHRU) + pSMB->InformationLevel = cpu_to_le16(SMB_SET_FILE_BASIC_INFO2); + else + pSMB->InformationLevel = cpu_to_le16(SMB_SET_FILE_BASIC_INFO); + pSMB->Reserved4 = 0; + in_len += byte_count; + memcpy(data_offset, data, sizeof(FILE_BASIC_INFO)); + pSMB->ByteCount = cpu_to_le16(byte_count); + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) + cifs_dbg(FYI, "SetPathInfo (times) returned %d\n", rc); + + cifs_buf_release(pSMB); + + if (rc == -EAGAIN) + goto SetTimesRetry; + + return rc; +} + +static void +cifs_fill_unix_set_info(FILE_UNIX_BASIC_INFO *data_offset, + const struct cifs_unix_set_info_args *args) +{ + u64 uid = NO_CHANGE_64, gid = NO_CHANGE_64; + u64 mode = args->mode; + + if (uid_valid(args->uid)) + uid = from_kuid(&init_user_ns, args->uid); + if (gid_valid(args->gid)) + gid = from_kgid(&init_user_ns, args->gid); + + /* + * Samba server ignores set of file size to zero due to bugs in some + * older clients, but we should be precise - we use SetFileSize to + * set file size and do not want to truncate file size to zero + * accidentally as happened on one Samba server beta by putting + * zero instead of -1 here + */ + data_offset->EndOfFile = cpu_to_le64(NO_CHANGE_64); + data_offset->NumOfBytes = cpu_to_le64(NO_CHANGE_64); + data_offset->LastStatusChange = cpu_to_le64(args->ctime); + data_offset->LastAccessTime = cpu_to_le64(args->atime); + data_offset->LastModificationTime = cpu_to_le64(args->mtime); + data_offset->Uid = cpu_to_le64(uid); + data_offset->Gid = cpu_to_le64(gid); + /* better to leave device as zero when it is */ + data_offset->DevMajor = cpu_to_le64(MAJOR(args->device)); + data_offset->DevMinor = cpu_to_le64(MINOR(args->device)); + data_offset->Permissions = cpu_to_le64(mode); + + if (S_ISREG(mode)) + data_offset->Type = cpu_to_le32(UNIX_FILE); + else if (S_ISDIR(mode)) + data_offset->Type = cpu_to_le32(UNIX_DIR); + else if (S_ISLNK(mode)) + data_offset->Type = cpu_to_le32(UNIX_SYMLINK); + else if (S_ISCHR(mode)) + data_offset->Type = cpu_to_le32(UNIX_CHARDEV); + else if (S_ISBLK(mode)) + data_offset->Type = cpu_to_le32(UNIX_BLOCKDEV); + else if (S_ISFIFO(mode)) + data_offset->Type = cpu_to_le32(UNIX_FIFO); + else if (S_ISSOCK(mode)) + data_offset->Type = cpu_to_le32(UNIX_SOCKET); +} + +int +CIFSSMBUnixSetFileInfo(const unsigned int xid, struct cifs_tcon *tcon, + const struct cifs_unix_set_info_args *args, + u16 fid, u32 pid_of_opener) +{ + struct smb_com_transaction2_sfi_req *pSMB = NULL; + unsigned int in_len; + char *data_offset; + int rc = 0; + u16 params, param_offset, offset, byte_count, count; + + cifs_dbg(FYI, "Set Unix Info (via SetFileInfo)\n"); + rc = small_smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB); + if (rc < 0) + return rc; + in_len = rc; + + pSMB->hdr.Pid = cpu_to_le16((__u16)pid_of_opener); + pSMB->hdr.PidHigh = cpu_to_le16((__u16)(pid_of_opener >> 16)); + + params = 6; + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + param_offset = offsetof(struct smb_com_transaction2_sfi_req, Fid); + offset = param_offset + params; + + data_offset = (char *)pSMB + offset; + + count = sizeof(FILE_UNIX_BASIC_INFO); + + pSMB->MaxParameterCount = cpu_to_le16(2); + /* BB find max SMB PDU from sess */ + pSMB->MaxDataCount = cpu_to_le16(1000); + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_SET_FILE_INFORMATION); + byte_count = 3 /* pad */ + params + count; + pSMB->DataCount = cpu_to_le16(count); + pSMB->ParameterCount = cpu_to_le16(params); + pSMB->TotalDataCount = pSMB->DataCount; + pSMB->TotalParameterCount = pSMB->ParameterCount; + pSMB->ParameterOffset = cpu_to_le16(param_offset); + pSMB->DataOffset = cpu_to_le16(offset); + pSMB->Fid = fid; + pSMB->InformationLevel = cpu_to_le16(SMB_SET_FILE_UNIX_BASIC); + pSMB->Reserved4 = 0; + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + + cifs_fill_unix_set_info((FILE_UNIX_BASIC_INFO *)data_offset, args); + + rc = SendReceiveNoRsp(xid, tcon->ses, (char *) pSMB, in_len, 0); + cifs_small_buf_release(pSMB); + if (rc) + cifs_dbg(FYI, "Send error in Set Time (SetFileInfo) = %d\n", + rc); + + /* Note: On -EAGAIN error only caller can retry on handle based calls + since file handle passed in no longer valid */ + + return rc; +} + +int +CIFSSMBUnixSetPathInfo(const unsigned int xid, struct cifs_tcon *tcon, + const char *file_name, + const struct cifs_unix_set_info_args *args, + const struct nls_table *nls_codepage, int remap) +{ + TRANSACTION2_SPI_REQ *pSMB = NULL; + TRANSACTION2_SPI_RSP *pSMBr = NULL; + unsigned int in_len; + int name_len; + int rc = 0; + int bytes_returned = 0; + FILE_UNIX_BASIC_INFO *data_offset; + __u16 params, param_offset, offset, count, byte_count; + + cifs_dbg(FYI, "In SetUID/GID/Mode\n"); +setPermsRetry: + rc = smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + name_len = + cifsConvertToUTF16((__le16 *) pSMB->FileName, file_name, + PATH_MAX, nls_codepage, remap); + name_len++; /* trailing null */ + name_len *= 2; + } else { + name_len = copy_path_name(pSMB->FileName, file_name); + } + + params = 6 + name_len; + count = sizeof(FILE_UNIX_BASIC_INFO); + pSMB->MaxParameterCount = cpu_to_le16(2); + /* BB find max SMB PDU from sess structure BB */ + pSMB->MaxDataCount = cpu_to_le16(1000); + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + param_offset = offsetof(struct smb_com_transaction2_spi_req, + InformationLevel); + offset = param_offset + params; + data_offset = (FILE_UNIX_BASIC_INFO *)((char *) pSMB + offset); + memset(data_offset, 0, count); + pSMB->DataOffset = cpu_to_le16(offset); + pSMB->ParameterOffset = cpu_to_le16(param_offset); + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_SET_PATH_INFORMATION); + byte_count = 3 /* pad */ + params + count; + pSMB->ParameterCount = cpu_to_le16(params); + pSMB->DataCount = cpu_to_le16(count); + pSMB->TotalParameterCount = pSMB->ParameterCount; + pSMB->TotalDataCount = pSMB->DataCount; + pSMB->InformationLevel = cpu_to_le16(SMB_SET_FILE_UNIX_BASIC); + pSMB->Reserved4 = 0; + in_len += byte_count; + + cifs_fill_unix_set_info(data_offset, args); + + pSMB->ByteCount = cpu_to_le16(byte_count); + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) + cifs_dbg(FYI, "SetPathInfo (perms) returned %d\n", rc); + + cifs_buf_release(pSMB); + if (rc == -EAGAIN) + goto setPermsRetry; + return rc; +} + +#ifdef CONFIG_CIFS_XATTR +/* + * Do a path-based QUERY_ALL_EAS call and parse the result. This is a common + * function used by listxattr and getxattr type calls. When ea_name is set, + * it looks for that attribute name and stuffs that value into the EAData + * buffer. When ea_name is NULL, it stuffs a list of attribute names into the + * buffer. In both cases, the return value is either the length of the + * resulting data or a negative error code. If EAData is a NULL pointer then + * the data isn't copied to it, but the length is returned. + */ +ssize_t +CIFSSMBQAllEAs(const unsigned int xid, struct cifs_tcon *tcon, + const unsigned char *searchName, const unsigned char *ea_name, + char *EAData, size_t buf_size, + struct cifs_sb_info *cifs_sb) +{ + /* BB assumes one setup word */ + TRANSACTION2_QPI_REQ *pSMB = NULL; + TRANSACTION2_QPI_RSP *pSMBr = NULL; + int remap = cifs_remap(cifs_sb); + struct nls_table *nls_codepage = cifs_sb->local_nls; + unsigned int in_len; + int rc = 0; + int bytes_returned; + int list_len; + struct fealist *ea_response_data; + struct fea *temp_fea; + char *temp_ptr; + char *end_of_smb; + __u16 params, byte_count, data_offset; + unsigned int ea_name_len = ea_name ? strlen(ea_name) : 0; + + cifs_dbg(FYI, "In Query All EAs path %s\n", searchName); +QAllEAsRetry: + rc = smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + list_len = + cifsConvertToUTF16((__le16 *) pSMB->FileName, searchName, + PATH_MAX, nls_codepage, remap); + list_len++; /* trailing null */ + list_len *= 2; + } else { + list_len = copy_path_name(pSMB->FileName, searchName); + } + + params = 2 /* level */ + 4 /* reserved */ + list_len /* includes NUL */; + pSMB->TotalDataCount = 0; + pSMB->MaxParameterCount = cpu_to_le16(2); + /* BB find exact max SMB PDU from sess structure BB */ + pSMB->MaxDataCount = cpu_to_le16(CIFSMaxBufSize); + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + pSMB->ParameterOffset = cpu_to_le16(offsetof( + struct smb_com_transaction2_qpi_req, InformationLevel)); + pSMB->DataCount = 0; + pSMB->DataOffset = 0; + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_QUERY_PATH_INFORMATION); + byte_count = params + 1 /* pad */ ; + pSMB->TotalParameterCount = cpu_to_le16(params); + pSMB->ParameterCount = pSMB->TotalParameterCount; + pSMB->InformationLevel = cpu_to_le16(SMB_INFO_QUERY_ALL_EAS); + pSMB->Reserved4 = 0; + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) { + cifs_dbg(FYI, "Send error in QueryAllEAs = %d\n", rc); + goto QAllEAsOut; + } + + + /* BB also check enough total bytes returned */ + /* BB we need to improve the validity checking + of these trans2 responses */ + + rc = validate_t2((struct smb_t2_rsp *)pSMBr); + if (rc || get_bcc(&pSMBr->hdr) < 4) { + rc = smb_EIO2(smb_eio_trace_qalleas_bcc_too_small, + get_bcc(&pSMBr->hdr), 4); + goto QAllEAsOut; + } + + /* check that length of list is not more than bcc */ + /* check that each entry does not go beyond length + of list */ + /* check that each element of each entry does not + go beyond end of list */ + /* validate_trans2_offsets() */ + /* BB check if start of smb + data_offset > &bcc+ bcc */ + + data_offset = le16_to_cpu(pSMBr->t2.DataOffset); + ea_response_data = (struct fealist *) + (((char *) &pSMBr->hdr.Protocol) + data_offset); + + list_len = le32_to_cpu(ea_response_data->list_len); + cifs_dbg(FYI, "ea length %d\n", list_len); + if (list_len <= 8) { + cifs_dbg(FYI, "empty EA list returned from server\n"); + /* didn't find the named attribute */ + if (ea_name) + rc = -ENODATA; + goto QAllEAsOut; + } + + /* make sure list_len doesn't go past end of SMB */ + end_of_smb = (char *)pByteArea(&pSMBr->hdr) + get_bcc(&pSMBr->hdr); + if ((char *)ea_response_data + list_len > end_of_smb) { + cifs_dbg(FYI, "EA list appears to go beyond SMB\n"); + rc = smb_EIO2(smb_eio_trace_qalleas_overlong, + (unsigned long)ea_response_data + list_len - (unsigned long)pSMBr, + (unsigned long)end_of_smb - (unsigned long)pSMBr); + goto QAllEAsOut; + } + + /* account for ea list len */ + list_len -= 4; + temp_fea = &ea_response_data->list; + temp_ptr = (char *)temp_fea; + while (list_len > 0) { + unsigned int name_len; + __u16 value_len; + + list_len -= 4; + temp_ptr += 4; + /* make sure we can read name_len and value_len */ + if (list_len < 0) { + cifs_dbg(FYI, "EA entry goes beyond length of list\n"); + rc = smb_EIO1(smb_eio_trace_qalleas_ea_overlong, list_len); + goto QAllEAsOut; + } + + name_len = temp_fea->name_len; + value_len = le16_to_cpu(temp_fea->value_len); + list_len -= name_len + 1 + value_len; + if (list_len < 0) { + cifs_dbg(FYI, "EA entry goes beyond length of list\n"); + rc = smb_EIO1(smb_eio_trace_qalleas_ea_overlong, list_len); + goto QAllEAsOut; + } + + if (ea_name) { + if (ea_name_len == name_len && + memcmp(ea_name, temp_ptr, name_len) == 0) { + temp_ptr += name_len + 1; + rc = value_len; + if (buf_size == 0) + goto QAllEAsOut; + if ((size_t)value_len > buf_size) { + rc = -ERANGE; + goto QAllEAsOut; + } + memcpy(EAData, temp_ptr, value_len); + goto QAllEAsOut; + } + } else { + /* account for prefix user. and trailing null */ + rc += (5 + 1 + name_len); + if (rc < (int) buf_size) { + memcpy(EAData, "user.", 5); + EAData += 5; + memcpy(EAData, temp_ptr, name_len); + EAData += name_len; + /* null terminate name */ + *EAData = 0; + ++EAData; + } else if (buf_size == 0) { + /* skip copy - calc size only */ + } else { + /* stop before overrun buffer */ + rc = -ERANGE; + break; + } + } + temp_ptr += name_len + 1 + value_len; + temp_fea = (struct fea *)temp_ptr; + } + + /* didn't find the named attribute */ + if (ea_name) + rc = -ENODATA; + +QAllEAsOut: + cifs_buf_release(pSMB); + if (rc == -EAGAIN) + goto QAllEAsRetry; + + return (ssize_t)rc; +} + +int +CIFSSMBSetEA(const unsigned int xid, struct cifs_tcon *tcon, + const char *fileName, const char *ea_name, const void *ea_value, + const __u16 ea_value_len, const struct nls_table *nls_codepage, + struct cifs_sb_info *cifs_sb) +{ + struct smb_com_transaction2_spi_req *pSMB = NULL; + struct smb_com_transaction2_spi_rsp *pSMBr = NULL; + struct fealist *parm_data; + unsigned int in_len; + int name_len; + int rc = 0; + int bytes_returned = 0; + __u16 params, param_offset, byte_count, offset, count; + int remap = cifs_remap(cifs_sb); + + cifs_dbg(FYI, "In SetEA\n"); +SetEARetry: + rc = smb_init(SMB_COM_TRANSACTION2, 15, tcon, (void **) &pSMB, + (void **) &pSMBr); + if (rc < 0) + return rc; + in_len = rc; + + if (pSMB->hdr.Flags2 & SMBFLG2_UNICODE) { + name_len = + cifsConvertToUTF16((__le16 *) pSMB->FileName, fileName, + PATH_MAX, nls_codepage, remap); + name_len++; /* trailing null */ + name_len *= 2; + } else { + name_len = copy_path_name(pSMB->FileName, fileName); + } + + params = 6 + name_len; + + /* done calculating parms using name_len of file name, + now use name_len to calculate length of ea name + we are going to create in the inode xattrs */ + if (ea_name == NULL) + name_len = 0; + else + name_len = strnlen(ea_name, 255); + + count = sizeof(*parm_data) + 1 + ea_value_len + name_len; + pSMB->MaxParameterCount = cpu_to_le16(2); + /* BB find max SMB PDU from sess */ + pSMB->MaxDataCount = cpu_to_le16(1000); + pSMB->MaxSetupCount = 0; + pSMB->Reserved = 0; + pSMB->Flags = 0; + pSMB->Timeout = 0; + pSMB->Reserved2 = 0; + param_offset = offsetof(struct smb_com_transaction2_spi_req, + InformationLevel); + offset = param_offset + params; + pSMB->InformationLevel = + cpu_to_le16(SMB_SET_FILE_EA); + + parm_data = (void *)pSMB + offset; + pSMB->ParameterOffset = cpu_to_le16(param_offset); + pSMB->DataOffset = cpu_to_le16(offset); + pSMB->SetupCount = 1; + pSMB->Reserved3 = 0; + pSMB->SubCommand = cpu_to_le16(TRANS2_SET_PATH_INFORMATION); + byte_count = 3 /* pad */ + params + count; + pSMB->DataCount = cpu_to_le16(count); + parm_data->list_len = cpu_to_le32(count); + parm_data->list.EA_flags = 0; + /* we checked above that name len is less than 255 */ + parm_data->list.name_len = (__u8)name_len; + /* EA names are always ASCII and NUL-terminated */ + strscpy(parm_data->list.name, ea_name ?: "", name_len + 1); + parm_data->list.value_len = cpu_to_le16(ea_value_len); + /* caller ensures that ea_value_len is less than 64K but + we need to ensure that it fits within the smb */ + + /*BB add length check to see if it would fit in + negotiated SMB buffer size BB */ + /* if (ea_value_len > buffer_size - 512 (enough for header)) */ + if (ea_value_len) + memcpy(parm_data->list.name + name_len + 1, + ea_value, ea_value_len); + + pSMB->TotalDataCount = pSMB->DataCount; + pSMB->ParameterCount = cpu_to_le16(params); + pSMB->TotalParameterCount = pSMB->ParameterCount; + pSMB->Reserved4 = 0; + in_len += byte_count; + pSMB->ByteCount = cpu_to_le16(byte_count); + rc = SendReceive(xid, tcon->ses, (struct smb_hdr *) pSMB, in_len, + (struct smb_hdr *) pSMBr, &bytes_returned, 0); + if (rc) + cifs_dbg(FYI, "SetPathInfo (EA) returned %d\n", rc); + + cifs_buf_release(pSMB); + + if (rc == -EAGAIN) + goto SetEARetry; + + return rc; +} +#endif diff --git a/fs/smb/client/compress.c b/fs/smb/client/compress.c new file mode 100644 index 000000000..8f0860970 --- /dev/null +++ b/fs/smb/client/compress.c @@ -0,0 +1,372 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2024, SUSE LLC + * + * Authors: Enzo Matsumiya + * + * This file implements I/O compression support for SMB2 messages (SMB 3.1.1 only). + * See compress/ for implementation details of each algorithm. + * + * References: + * MS-SMB2 "3.1.4.4 Compressing the Message" + * MS-SMB2 "3.1.5.3 Decompressing the Chained Message" + * MS-XCA - for details of the supported algorithms + */ +#include +#include +#include +#include + +#include "cifsglob.h" +#include "../common/smb2pdu.h" +#include "cifsproto.h" +#include "smb2proto.h" + +#include "../common/compress/lz77.h" +#include "compress.h" + +/* + * The heuristic_*() functions below try to determine data compressibility. + * + * Derived from fs/btrfs/compression.c, changing coding style, some parameters, and removing + * unused parts. + * + * Read that file for better and more detailed explanation of the calculations. + * + * The algorithms are ran in a collected sample of the input (uncompressed) data. + * The sample is formed of 2K reads in PAGE_SIZE intervals, with a maximum size of 4M. + * + * Parsing the sample goes from "low-hanging fruits" (fastest algorithms, likely compressible) + * to "need more analysis" (likely uncompressible). + */ + +struct bucket { + unsigned int count; +}; + +static inline size_t pow4(size_t n) +{ + return n * n * n * n; +} + +/* + * has_low_entropy() - Compute Shannon entropy of the sampled data. + * @bkt: Bytes counts of the sample. + * @slen: Size of the sample. + * + * Return: true if the level (percentage of number of bits that would be required to + * compress the data) is below the minimum threshold. + * + * Note: + * There _is_ an entropy level here that's > 65 (minimum threshold) that would indicate a + * possibility of compression, but compressing, or even further analysing, it would waste so much + * resources that it's simply not worth it. + * + * Also Shannon entropy is the last computed heuristic; if we got this far and ended up + * with uncertainty, just stay on the safe side and call it uncompressible. + */ +static bool has_low_entropy(struct bucket *bkt, size_t slen) +{ + const size_t threshold = 65, max_entropy = 8 * ilog2(16); + size_t i, p, p2, len, sum = 0; + + len = ilog2(pow4(slen)); + + for (i = 0; i < 256 && bkt[i].count > 0; i++) { + p = bkt[i].count; + p2 = ilog2(pow4(p)); + sum += p * (len - p2); + } + + sum /= slen; + + return ((sum * 100 / max_entropy) <= threshold); +} + +#define BYTE_DIST_BAD 0 +#define BYTE_DIST_GOOD 1 +#define BYTE_DIST_MAYBE 2 +/* + * calc_byte_distribution() - Compute byte distribution on the sampled data. + * @bkt: Byte counts of the sample. + * @slen: Size of the sample. + * + * Return: + * BYTE_DIST_BAD: A "hard no" for compression -- a computed uniform distribution of + * the bytes (e.g. random or encrypted data). + * BYTE_DIST_GOOD: High probability (normal (Gaussian) distribution) of the data being + * compressible. + * BYTE_DIST_MAYBE: When computed byte distribution resulted in "low > n < high" + * grounds. has_low_entropy() should be used for a final decision. + */ +static int calc_byte_distribution(struct bucket *bkt, size_t slen) +{ + const size_t low = 64, high = 200, threshold = slen * 90 / 100; + size_t sum = 0; + int i; + + for (i = 0; i < low; i++) + sum += bkt[i].count; + + if (sum > threshold) + return BYTE_DIST_BAD; + + for (; i < high && bkt[i].count > 0; i++) { + sum += bkt[i].count; + if (sum > threshold) + break; + } + + if (i <= low) + return BYTE_DIST_GOOD; + + if (i >= high) + return BYTE_DIST_BAD; + + return BYTE_DIST_MAYBE; +} + +static bool is_mostly_ascii(const struct bucket *bkt) +{ + size_t count = 0; + int i; + + for (i = 0; i < 256; i++) + if (bkt[i].count > 0) + /* Too many non-ASCII (0-63) bytes. */ + if (++count > 64) + return false; + + return true; +} + +static bool has_repeated_data(const u8 *sample, size_t len) +{ + size_t s = len / 2; + + return (!memcmp(&sample[0], &sample[s], s)); +} + +static int cmp_bkt(const void *_a, const void *_b) +{ + const struct bucket *a = _a, *b = _b; + + /* Reverse sort. */ + if (a->count > b->count) + return -1; + + return 1; +} + +/* + * Collect some 2K samples with 2K gaps between. + */ +static int collect_sample(const struct iov_iter *source, ssize_t max, u8 *sample) +{ + struct iov_iter iter = *source; + size_t s = 0; + + while (iov_iter_count(&iter) >= SZ_2K) { + size_t part = umin(umin(iov_iter_count(&iter), SZ_2K), max); + size_t n; + + n = copy_from_iter(sample + s, part, &iter); + if (n != part) + return -EFAULT; + + s += n; + max -= n; + + if (iov_iter_count(&iter) < PAGE_SIZE - SZ_2K) + break; + + iov_iter_advance(&iter, SZ_2K); + } + + return s; +} + +/* + * is_compressible() - Determines if a chunk of data is compressible. + * @data: Iterator containing uncompressed data. + * + * Return: true if @data is compressible, false otherwise. + * + * Tests shows that this function is quite reliable in predicting data compressibility, + * matching close to 1:1 with the behaviour of LZ77 compression success and failures. + */ +static bool is_compressible(const struct iov_iter *data) +{ + const size_t read_size = SZ_2K, bkt_size = 256, max = SZ_4M; + struct bucket *bkt = NULL; + size_t len; + u8 *sample; + bool ret = false; + int i; + + /* Preventive double check -- already checked in should_compress(). */ + len = iov_iter_count(data); + if (unlikely(len < read_size)) + return ret; + + if (len - read_size > max) + len = max; + + sample = kvzalloc(len, GFP_KERNEL); + if (!sample) { + WARN_ON_ONCE(1); + + return ret; + } + + /* Sample 2K bytes per page of the uncompressed data. */ + i = collect_sample(data, len, sample); + if (i <= 0) { + WARN_ON_ONCE(1); + + goto out; + } + + len = i; + ret = true; + + if (has_repeated_data(sample, len)) + goto out; + + bkt = kzalloc_objs(*bkt, bkt_size); + if (!bkt) { + WARN_ON_ONCE(1); + ret = false; + + goto out; + } + + for (i = 0; i < len; i++) + bkt[sample[i]].count++; + + if (is_mostly_ascii(bkt)) + goto out; + + /* Sort in descending order */ + sort(bkt, bkt_size, sizeof(*bkt), cmp_bkt, NULL); + + i = calc_byte_distribution(bkt, len); + if (i != BYTE_DIST_MAYBE) { + ret = !!i; + + goto out; + } + + ret = has_low_entropy(bkt, len); +out: + kvfree(sample); + kfree(bkt); + + return ret; +} + +/* + * should_compress() - Determines if a request (write) or the response to a + * request (read) should be compressed. + * @tcon: tcon of the request is being sent to + * @rqst: request to evaluate + * + * Return: true iff: + * - compression was successfully negotiated with server + * - server has enabled compression for the share + * - it's a read or write request + * - (write only) request length is >= SMB_COMPRESS_MIN_LEN + * - (write only) is_compressible() returns 1 + * + * Return false otherwise. + */ +bool should_compress(const struct cifs_tcon *tcon, const struct smb_rqst *rq) +{ + const struct smb2_hdr *shdr = rq->rq_iov->iov_base; + + if (unlikely(!tcon || !tcon->ses || !tcon->ses->server)) + return false; + + if (!tcon->ses->server->compression.enabled) + return false; + + if (!(tcon->share_flags & SMB2_SHAREFLAG_COMPRESS_DATA)) + return false; + + if (shdr->Command == SMB2_WRITE) { + const struct smb2_write_req *wreq = rq->rq_iov->iov_base; + + if (le32_to_cpu(wreq->Length) < SMB_COMPRESS_MIN_LEN) + return false; + + return is_compressible(&rq->rq_iter); + } + + return (shdr->Command == SMB2_READ); +} + +int smb_compress(struct TCP_Server_Info *server, struct smb_rqst *rq, compress_send_fn send_fn) +{ + struct iov_iter iter; + u32 slen, dlen; + void *src, *dst = NULL; + int ret; + + if (!server || !rq || !rq->rq_iov || !rq->rq_iov->iov_base) + return -EINVAL; + + if (rq->rq_iov->iov_len != sizeof(struct smb2_write_req)) + return -EINVAL; + + slen = iov_iter_count(&rq->rq_iter); + src = kvzalloc(slen, GFP_KERNEL); + if (!src) { + ret = -ENOMEM; + goto err_free; + } + + /* Keep the original iter intact. */ + iter = rq->rq_iter; + + if (!copy_from_iter_full(src, slen, &iter)) { + ret = smb_EIO(smb_eio_trace_compress_copy); + goto err_free; + } + + dlen = smb_lz77_compressed_alloc_size(slen); + dst = kvzalloc(dlen, GFP_KERNEL); + if (!dst) { + ret = -ENOMEM; + goto err_free; + } + + ret = smb_lz77_compress(src, slen, dst, &dlen); + if (!ret) { + struct smb2_compression_hdr hdr = { 0 }; + struct smb_rqst comp_rq = { .rq_nvec = 3, }; + struct kvec iov[3]; + + hdr.ProtocolId = SMB2_COMPRESSION_TRANSFORM_ID; + hdr.OriginalCompressedSegmentSize = cpu_to_le32(slen); + hdr.CompressionAlgorithm = SMB3_COMPRESS_LZ77; + hdr.Flags = SMB2_COMPRESSION_FLAG_NONE; + hdr.Offset = cpu_to_le32(rq->rq_iov[0].iov_len); + + iov[0].iov_base = &hdr; + iov[0].iov_len = sizeof(hdr); + iov[1] = rq->rq_iov[0]; + iov[2].iov_base = dst; + iov[2].iov_len = dlen; + + comp_rq.rq_iov = iov; + + ret = send_fn(server, 1, &comp_rq); + } else if (ret == -EMSGSIZE || dlen >= slen) { + ret = send_fn(server, 1, rq); + } +err_free: + kvfree(dst); + kvfree(src); + + return ret; +} diff --git a/fs/smb/client/compress.h b/fs/smb/client/compress.h new file mode 100644 index 000000000..e08e6d339 --- /dev/null +++ b/fs/smb/client/compress.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2024, SUSE LLC + * + * Authors: Enzo Matsumiya + * + * This file implements I/O compression support for SMB2 messages (SMB 3.1.1 only). + * See compress/ for implementation details of each algorithm. + * + * References: + * MS-SMB2 "3.1.4.4 Compressing the Message" - for compression details + * MS-SMB2 "3.1.5.3 Decompressing the Chained Message" - for decompression details + * MS-XCA - for details of the supported algorithms + */ +#ifndef _SMB_COMPRESS_H +#define _SMB_COMPRESS_H + +#include +#include +#include "../common/smb2pdu.h" +#include "../common/compress/compress.h" +#include "cifsglob.h" + +/* sizeof(smb2_compression_hdr) - sizeof(OriginalPayloadSize) */ +#define SMB_COMPRESS_HDR_LEN 16 +/* sizeof(smb2_compression_payload_hdr) - sizeof(OriginalPayloadSize) */ +#define SMB_COMPRESS_PAYLOAD_HDR_LEN 8 +#define SMB_COMPRESS_MIN_LEN PAGE_SIZE + +#ifdef CONFIG_CIFS_COMPRESSION +typedef int (*compress_send_fn)(struct TCP_Server_Info *, int, struct smb_rqst *); + + +int smb_compress(struct TCP_Server_Info *server, struct smb_rqst *rq, + compress_send_fn send_fn); +bool should_compress(const struct cifs_tcon *tcon, const struct smb_rqst *rq); + +#else /* !CONFIG_CIFS_COMPRESSION */ +static inline int smb_compress(void *unused1, void *unused2, void *unused3) +{ + return -EOPNOTSUPP; +} + +static inline bool should_compress(void *unused1, void *unused2) +{ + return false; +} + +#endif /* !CONFIG_CIFS_COMPRESSION */ +#endif /* _SMB_COMPRESS_H */ diff --git a/fs/smb/client/connect.c b/fs/smb/client/connect.c new file mode 100644 index 000000000..ba749ec25 --- /dev/null +++ b/fs/smb/client/connect.c @@ -0,0 +1,4542 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * Copyright (C) International Business Machines Corp., 2002,2011 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "cifsglob.h" +#include "cifsproto.h" +#include "cifs_unicode.h" +#include "cifs_debug.h" +#include "cifs_fs_sb.h" +#include "ntlmssp.h" +#include "nterr.h" +#include "rfc1002pdu.h" +#include "fscache.h" +#include "smb2proto.h" +#include "smbdirect.h" +#include "dns_resolve.h" +#ifdef CONFIG_CIFS_DFS_UPCALL +#include "dfs.h" +#include "dfs_cache.h" +#endif +#include "fs_context.h" +#include "cifs_swn.h" + +/* FIXME: should these be tunable? */ +#define TLINK_ERROR_EXPIRE (1 * HZ) +#define TLINK_IDLE_EXPIRE (600 * HZ) + +/* Drop the connection to not overload the server */ +#define MAX_STATUS_IO_TIMEOUT 5 + +static int ip_connect(struct TCP_Server_Info *server); +static int generic_ip_connect(struct TCP_Server_Info *server); +static void tlink_rb_insert(struct rb_root *root, struct tcon_link *new_tlink); +static void cifs_prune_tlinks(struct work_struct *work); + +static struct mchan_mount *mchan_mount_alloc(struct cifs_ses *ses); +static void mchan_mount_free(struct mchan_mount *mchan_mount); +static void mchan_mount_work_fn(struct work_struct *work); + +/* + * Resolve hostname and set ip addr in tcp ses. Useful for hostnames that may + * get their ip addresses changed at some point. + * + * This should be called with server->srv_mutex held. + */ +static int reconn_set_ipaddr_from_hostname(struct TCP_Server_Info *server) +{ + struct sockaddr_storage ss; + int rc; + + if (!server->hostname) + return -EINVAL; + + /* if server hostname isn't populated, there's nothing to do here */ + if (server->hostname[0] == '\0') + return 0; + + spin_lock(&server->srv_lock); + ss = server->dstaddr; + spin_unlock(&server->srv_lock); + + rc = dns_resolve_name(server->dns_dom, server->hostname, + strlen(server->hostname), + (struct sockaddr *)&ss); + if (!rc) { + spin_lock(&server->srv_lock); + memcpy(&server->dstaddr, &ss, sizeof(server->dstaddr)); + spin_unlock(&server->srv_lock); + } + return rc; +} + +void smb2_query_server_interfaces(struct work_struct *work) +{ + int rc; + int xid; + struct cifs_tcon *tcon = container_of(work, + struct cifs_tcon, + query_interfaces.work); + struct TCP_Server_Info *server = tcon->ses->server; + + /* + * query server network interfaces, in case they change + */ + if (!server->ops->query_server_interfaces) + return; + + xid = get_xid(); + rc = server->ops->query_server_interfaces(xid, tcon, false); + free_xid(xid); + + if (rc) + cifs_dbg(FYI, "%s: failed to query server interfaces: %d\n", + __func__, rc); + + queue_delayed_work(cifsiod_wq, &tcon->query_interfaces, + (SMB_INTERFACE_POLL_INTERVAL * HZ)); +} + +#define set_need_reco(server) \ +do { \ + spin_lock(&server->srv_lock); \ + if (server->tcpStatus != CifsExiting) \ + server->tcpStatus = CifsNeedReconnect; \ + spin_unlock(&server->srv_lock); \ +} while (0) + +/* + * Update the tcpStatus for the server. + * This is used to signal the cifsd thread to call cifs_reconnect + * ONLY cifsd thread should call cifs_reconnect. For any other + * thread, use this function + * + * @server: the tcp ses for which reconnect is needed + * @all_channels: if this needs to be done for all channels + */ +void +cifs_signal_cifsd_for_reconnect(struct TCP_Server_Info *server, + bool all_channels) +{ + struct TCP_Server_Info *nserver; + struct cifs_ses *ses; + LIST_HEAD(reco); + int i; + + /* if we need to signal just this channel */ + if (!all_channels) { + set_need_reco(server); + return; + } + + if (SERVER_IS_CHAN(server)) + server = server->primary_server; + scoped_guard(spinlock, &cifs_tcp_ses_lock) { + set_need_reco(server); + list_for_each_entry(ses, &server->smb_ses_list, smb_ses_list) { + spin_lock(&ses->ses_lock); + if (ses->ses_status == SES_EXITING) { + spin_unlock(&ses->ses_lock); + continue; + } + spin_lock(&ses->chan_lock); + for (i = 1; i < ses->chan_count; i++) { + nserver = ses->chans[i].server; + if (!nserver) + continue; + nserver->srv_count++; + list_add(&nserver->rlist, &reco); + } + spin_unlock(&ses->chan_lock); + spin_unlock(&ses->ses_lock); + } + } + + list_for_each_entry_safe(server, nserver, &reco, rlist) { + list_del_init(&server->rlist); + set_need_reco(server); + cifs_put_tcp_session(server, 0); + } +} + +/* + * Mark all sessions and tcons for reconnect. + * IMPORTANT: make sure that this gets called only from + * cifsd thread. For any other thread, use + * cifs_signal_cifsd_for_reconnect + * + * @server: the tcp ses for which reconnect is needed + * @server needs to be previously set to CifsNeedReconnect. + * @mark_smb_session: whether even sessions need to be marked + */ +void +cifs_mark_tcp_ses_conns_for_reconnect(struct TCP_Server_Info *server, + bool mark_smb_session) +{ + struct TCP_Server_Info *pserver; + struct cifs_ses *ses, *nses; + struct cifs_tcon *tcon; + + /* + * before reconnecting the tcp session, mark the smb session (uid) and the tid bad so they + * are not used until reconnected. + */ + cifs_dbg(FYI, "%s: marking necessary sessions and tcons for reconnect\n", __func__); + + /* If server is a channel, select the primary channel */ + pserver = SERVER_IS_CHAN(server) ? server->primary_server : server; + + /* + * if the server has been marked for termination, there is a + * chance that the remaining channels all need reconnect. To be + * on the safer side, mark the session and trees for reconnect + * for this scenario. This might cause a few redundant session + * setup and tree connect requests, but it is better than not doing + * a tree connect when needed, and all following requests failing + */ + if (server->terminate) { + mark_smb_session = true; + server = pserver; + } + + spin_lock(&cifs_tcp_ses_lock); + list_for_each_entry_safe(ses, nses, &pserver->smb_ses_list, smb_ses_list) { + spin_lock(&ses->ses_lock); + if (ses->ses_status == SES_EXITING) { + spin_unlock(&ses->ses_lock); + continue; + } + spin_unlock(&ses->ses_lock); + + spin_lock(&ses->chan_lock); + if (cifs_ses_get_chan_index(ses, server) == + CIFS_INVAL_CHAN_INDEX) { + spin_unlock(&ses->chan_lock); + continue; + } + + if (!cifs_chan_is_iface_active(ses, server)) { + spin_unlock(&ses->chan_lock); + cifs_chan_update_iface(ses, server); + spin_lock(&ses->chan_lock); + } + + if (!mark_smb_session && cifs_chan_needs_reconnect(ses, server)) { + spin_unlock(&ses->chan_lock); + continue; + } + + if (mark_smb_session) + CIFS_SET_ALL_CHANS_NEED_RECONNECT(ses); + else + cifs_chan_set_need_reconnect(ses, server); + + cifs_dbg(FYI, "%s: channel connect bitmap: 0x%lx\n", + __func__, ses->chans_need_reconnect); + + /* If all channels need reconnect, then tcon needs reconnect */ + if (!mark_smb_session && !CIFS_ALL_CHANS_NEED_RECONNECT(ses)) { + spin_unlock(&ses->chan_lock); + continue; + } + spin_unlock(&ses->chan_lock); + + spin_lock(&ses->ses_lock); + ses->ses_status = SES_NEED_RECON; + spin_unlock(&ses->ses_lock); + + list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { + tcon->need_reconnect = true; + spin_lock(&tcon->tc_lock); + tcon->status = TID_NEED_RECON; + spin_unlock(&tcon->tc_lock); + + cancel_delayed_work(&tcon->query_interfaces); + } + if (ses->tcon_ipc) { + ses->tcon_ipc->need_reconnect = true; + spin_lock(&ses->tcon_ipc->tc_lock); + ses->tcon_ipc->status = TID_NEED_RECON; + spin_unlock(&ses->tcon_ipc->tc_lock); + } + } + spin_unlock(&cifs_tcp_ses_lock); +} + +static void +cifs_abort_connection(struct TCP_Server_Info *server) +{ + struct mid_q_entry *mid, *nmid; + struct list_head retry_list; + + server->maxBuf = 0; + server->max_read = 0; + + /* do not want to be sending data on a socket we are freeing */ + cifs_dbg(FYI, "%s: tearing down socket\n", __func__); + cifs_server_lock(server); + if (server->ssocket) { + cifs_dbg(FYI, "State: 0x%x Flags: 0x%lx\n", server->ssocket->state, + server->ssocket->flags); + kernel_sock_shutdown(server->ssocket, SHUT_WR); + cifs_dbg(FYI, "Post shutdown state: 0x%x Flags: 0x%lx\n", server->ssocket->state, + server->ssocket->flags); + sock_release(server->ssocket); + server->ssocket = NULL; + } else if (cifs_rdma_enabled(server)) { + smbd_destroy(server); + } + server->sequence_number = 0; + server->session_estab = false; + kfree_sensitive(server->session_key.response); + server->session_key.response = NULL; + server->session_key.len = 0; + server->lstrp = jiffies; + + /* mark submitted MIDs for retry and issue callback */ + INIT_LIST_HEAD(&retry_list); + cifs_dbg(FYI, "%s: moving mids to private list\n", __func__); + spin_lock(&server->mid_queue_lock); + list_for_each_entry_safe(mid, nmid, &server->pending_mid_q, qhead) { + smb_get_mid(mid); + if (mid->mid_state == MID_REQUEST_SUBMITTED) + mid->mid_state = MID_RETRY_NEEDED; + list_move(&mid->qhead, &retry_list); + mid->deleted_from_q = true; + } + spin_unlock(&server->mid_queue_lock); + cifs_server_unlock(server); + + cifs_dbg(FYI, "%s: issuing mid callbacks\n", __func__); + list_for_each_entry_safe(mid, nmid, &retry_list, qhead) { + list_del_init(&mid->qhead); + mid_execute_callback(server, mid); + release_mid(server, mid); + } +} + +static bool cifs_tcp_ses_needs_reconnect(struct TCP_Server_Info *server, int num_targets) +{ + spin_lock(&server->srv_lock); + server->nr_targets = num_targets; + if (server->tcpStatus == CifsExiting) { + /* the demux thread will exit normally next time through the loop */ + spin_unlock(&server->srv_lock); + wake_up(&server->response_q); + return false; + } + + cifs_dbg(FYI, "Mark tcp session as need reconnect\n"); + trace_smb3_reconnect(server->current_mid, server->conn_id, + server->hostname); + server->tcpStatus = CifsNeedReconnect; + + spin_unlock(&server->srv_lock); + return true; +} + +/* + * cifs tcp session reconnection + * + * mark tcp session as reconnecting so temporarily locked + * mark all smb sessions as reconnecting for tcp session + * reconnect tcp session + * wake up waiters on reconnection? - (not needed currently) + * + * if mark_smb_session is passed as true, unconditionally mark + * the smb session (and tcon) for reconnect as well. This value + * doesn't really matter for non-multichannel scenario. + * + */ +static int __cifs_reconnect(struct TCP_Server_Info *server, + bool mark_smb_session, bool once) +{ + int rc = 0; + + if (!cifs_tcp_ses_needs_reconnect(server, 1)) + return 0; + + /* + * if smb session has been marked for reconnect, also reconnect all + * connections. This way, the other connections do not end up bad. + */ + if (mark_smb_session) + cifs_signal_cifsd_for_reconnect(server, mark_smb_session); + + cifs_mark_tcp_ses_conns_for_reconnect(server, mark_smb_session); + + cifs_abort_connection(server); + + do { + try_to_freeze(); + cifs_server_lock(server); + + if (!cifs_swn_set_server_dstaddr(server) && + !SERVER_IS_CHAN(server)) { + /* resolve the hostname again to make sure that IP address is up-to-date */ + rc = reconn_set_ipaddr_from_hostname(server); + cifs_dbg(FYI, "%s: reconn_set_ipaddr_from_hostname: rc=%d\n", __func__, rc); + } + + if (cifs_rdma_enabled(server)) + rc = smbd_reconnect(server); + else + rc = generic_ip_connect(server); + if (rc) { + cifs_server_unlock(server); + cifs_dbg(FYI, "%s: reconnect error %d\n", __func__, rc); + /* If was asked to reconnect only once, do not try it more times */ + if (once) + break; + msleep(3000); + } else { + atomic_inc(&tcpSesReconnectCount); + set_credits(server, 1); + spin_lock(&server->srv_lock); + if (server->tcpStatus != CifsExiting) + server->tcpStatus = CifsNeedNegotiate; + spin_unlock(&server->srv_lock); + cifs_swn_reset_server_dstaddr(server); + cifs_server_unlock(server); + cifs_queue_server_reconn(server); + } + } while (server->tcpStatus == CifsNeedReconnect); + + spin_lock(&server->srv_lock); + if (server->tcpStatus == CifsNeedNegotiate) + mod_delayed_work(cifsiod_wq, &server->echo, 0); + spin_unlock(&server->srv_lock); + + wake_up(&server->response_q); + return rc; +} + +#ifdef CONFIG_CIFS_DFS_UPCALL +static int __reconnect_target_locked(struct TCP_Server_Info *server, + const char *target) +{ + int rc; + char *hostname; + + if (!cifs_swn_set_server_dstaddr(server)) { + if (server->hostname != target) { + hostname = extract_hostname(target); + if (!IS_ERR(hostname)) { + spin_lock(&server->srv_lock); + kfree(server->hostname); + server->hostname = hostname; + spin_unlock(&server->srv_lock); + } else { + cifs_dbg(FYI, "%s: couldn't extract hostname or address from dfs target: %pe\n", + __func__, hostname); + cifs_dbg(FYI, "%s: default to last target server: %s\n", __func__, + server->hostname); + } + } + /* resolve the hostname again to make sure that IP address is up-to-date. */ + rc = reconn_set_ipaddr_from_hostname(server); + cifs_dbg(FYI, "%s: reconn_set_ipaddr_from_hostname: rc=%d\n", __func__, rc); + } + /* Reconnect the socket */ + if (cifs_rdma_enabled(server)) + rc = smbd_reconnect(server); + else + rc = generic_ip_connect(server); + + return rc; +} + +static int reconnect_target_locked(struct TCP_Server_Info *server, + struct dfs_cache_tgt_list *tl, + struct dfs_cache_tgt_iterator **target_hint) +{ + struct dfs_cache_tgt_iterator *tit; + int rc; + + *target_hint = NULL; + + /* If dfs target list is empty, then reconnect to last server */ + tit = dfs_cache_get_tgt_iterator(tl); + if (!tit) + return __reconnect_target_locked(server, server->hostname); + + /* Otherwise, try every dfs target in @tl */ + do { + const char *target = dfs_cache_get_tgt_name(tit); + + spin_lock(&server->srv_lock); + if (server->tcpStatus != CifsNeedReconnect) { + spin_unlock(&server->srv_lock); + return -ECONNRESET; + } + spin_unlock(&server->srv_lock); + rc = __reconnect_target_locked(server, target); + if (!rc) { + *target_hint = tit; + break; + } + } while ((tit = dfs_cache_get_next_tgt(tl, tit))); + return rc; +} + +static int reconnect_dfs_server(struct TCP_Server_Info *server) +{ + struct dfs_cache_tgt_iterator *target_hint = NULL; + const char *ref_path = server->leaf_fullpath + 1; + DFS_CACHE_TGT_LIST(tl); + int num_targets = 0; + int rc = 0; + + /* + * Determine the number of dfs targets the referral path in @cifs_sb resolves to. + * + * smb2_reconnect() needs to know how long it should wait based upon the number of dfs + * targets (server->nr_targets). It's also possible that the cached referral was cleared + * through /proc/fs/cifs/dfscache or the target list is empty due to server settings after + * refreshing the referral, so, in this case, default it to 1. + */ + if (!dfs_cache_noreq_find(ref_path, NULL, &tl)) + num_targets = dfs_cache_get_nr_tgts(&tl); + if (!num_targets) + num_targets = 1; + + if (!cifs_tcp_ses_needs_reconnect(server, num_targets)) + return 0; + + /* + * Unconditionally mark all sessions & tcons for reconnect as we might be connecting to a + * different server or share during failover. It could be improved by adding some logic to + * only do that in case it connects to a different server or share, though. + */ + cifs_mark_tcp_ses_conns_for_reconnect(server, true); + + cifs_abort_connection(server); + + do { + try_to_freeze(); + cifs_server_lock(server); + + rc = reconnect_target_locked(server, &tl, &target_hint); + if (rc) { + /* Failed to reconnect socket */ + cifs_server_unlock(server); + cifs_dbg(FYI, "%s: reconnect error %d\n", __func__, rc); + msleep(3000); + continue; + } + /* + * Socket was created. Update tcp session status to CifsNeedNegotiate so that a + * process waiting for reconnect will know it needs to re-establish session and tcon + * through the reconnected target server. + */ + atomic_inc(&tcpSesReconnectCount); + set_credits(server, 1); + spin_lock(&server->srv_lock); + if (server->tcpStatus != CifsExiting) + server->tcpStatus = CifsNeedNegotiate; + spin_unlock(&server->srv_lock); + cifs_swn_reset_server_dstaddr(server); + cifs_server_unlock(server); + cifs_queue_server_reconn(server); + } while (server->tcpStatus == CifsNeedReconnect); + + dfs_cache_noreq_update_tgthint(ref_path, target_hint); + dfs_cache_free_tgts(&tl); + + /* Need to set up echo worker again once connection has been established */ + spin_lock(&server->srv_lock); + if (server->tcpStatus == CifsNeedNegotiate) + mod_delayed_work(cifsiod_wq, &server->echo, 0); + spin_unlock(&server->srv_lock); + + wake_up(&server->response_q); + return rc; +} + +static int +_cifs_reconnect(struct TCP_Server_Info *server, bool mark_smb_session, bool once) +{ + if (!server->leaf_fullpath) + return __cifs_reconnect(server, mark_smb_session, once); + return reconnect_dfs_server(server); +} +#else +static int +_cifs_reconnect(struct TCP_Server_Info *server, bool mark_smb_session, bool once) +{ + return __cifs_reconnect(server, mark_smb_session, once); +} +#endif + +int +cifs_reconnect(struct TCP_Server_Info *server, bool mark_smb_session) +{ + return _cifs_reconnect(server, mark_smb_session, false); +} + +static int +cifs_reconnect_once(struct TCP_Server_Info *server) +{ + return _cifs_reconnect(server, true, true); +} + +static void +cifs_echo_request(struct work_struct *work) +{ + int rc; + struct TCP_Server_Info *server = container_of(work, + struct TCP_Server_Info, echo.work); + + /* + * We cannot send an echo if it is disabled. + * Also, no need to ping if we got a response recently. + */ + + if (server->tcpStatus == CifsNeedReconnect || + server->tcpStatus == CifsExiting || + server->tcpStatus == CifsNew || + (server->ops->can_echo && !server->ops->can_echo(server)) || + time_before(jiffies, server->lstrp + server->echo_interval - HZ)) + goto requeue_echo; + + rc = server->ops->echo ? server->ops->echo(server) : -ENOSYS; + cifs_server_dbg(FYI, "send echo request: rc = %d\n", rc); + + /* Check witness registrations */ + cifs_swn_check(); + +requeue_echo: + queue_delayed_work(cifsiod_wq, &server->echo, server->echo_interval); +} + +static bool +allocate_buffers(struct TCP_Server_Info *server) +{ + if (!server->bigbuf) { + server->bigbuf = (char *)cifs_buf_get(); + if (!server->bigbuf) { + cifs_server_dbg(VFS, "No memory for large SMB response\n"); + msleep(3000); + /* retry will check if exiting */ + return false; + } + } else if (server->large_buf) { + /* we are reusing a dirty large buf, clear its start */ + memset(server->bigbuf, 0, HEADER_SIZE(server)); + } + + if (!server->smallbuf) { + server->smallbuf = (char *)cifs_small_buf_get(); + if (!server->smallbuf) { + cifs_server_dbg(VFS, "No memory for SMB response\n"); + msleep(1000); + /* retry will check if exiting */ + return false; + } + /* beginning of smb buffer is cleared in our buf_get */ + } else { + /* if existing small buf clear beginning */ + memset(server->smallbuf, 0, HEADER_SIZE(server)); + } + + return true; +} + +static bool +server_unresponsive(struct TCP_Server_Info *server) +{ + /* + * If we're in the process of mounting a share or reconnecting a session + * and the server abruptly shut down (e.g. socket wasn't closed, packet + * had been ACK'ed but no SMB response), don't wait longer than 20s from + * when negotiate actually started. + */ + spin_lock(&server->srv_lock); + if (server->tcpStatus == CifsInNegotiate && + time_after(jiffies, server->neg_start + 20 * HZ)) { + spin_unlock(&server->srv_lock); + cifs_reconnect(server, false); + return true; + } + /* + * We need to wait 3 echo intervals to make sure we handle such + * situations right: + * 1s client sends a normal SMB request + * 2s client gets a response + * 30s echo workqueue job pops, and decides we got a response recently + * and don't need to send another + * ... + * 65s kernel_recvmsg times out, and we see that we haven't gotten + * a response in >60s. + */ + if ((server->tcpStatus == CifsGood || + server->tcpStatus == CifsNeedNegotiate) && + (!server->ops->can_echo || server->ops->can_echo(server)) && + time_after(jiffies, server->lstrp + 3 * server->echo_interval)) { + spin_unlock(&server->srv_lock); + cifs_server_dbg(VFS, "has not responded in %lu seconds. Reconnecting...\n", + (3 * server->echo_interval) / HZ); + cifs_reconnect(server, false); + return true; + } + spin_unlock(&server->srv_lock); + + return false; +} + +static inline bool +zero_credits(struct TCP_Server_Info *server) +{ + int val; + + spin_lock(&server->req_lock); + val = server->credits + server->echo_credits + server->oplock_credits; + if (server->in_flight == 0 && val == 0) { + spin_unlock(&server->req_lock); + return true; + } + spin_unlock(&server->req_lock); + return false; +} + +static int +cifs_readv_from_socket(struct TCP_Server_Info *server, struct msghdr *smb_msg) +{ + int length = 0; + int total_read; + + for (total_read = 0; msg_data_left(smb_msg); total_read += length) { + try_to_freeze(); + + /* reconnect if no credits and no requests in flight */ + if (zero_credits(server)) { + cifs_reconnect(server, false); + return -ECONNABORTED; + } + + if (server_unresponsive(server)) + return -ECONNABORTED; + if (cifs_rdma_enabled(server) && server->smbd_conn) + length = smbd_recv(server->smbd_conn, smb_msg); + else + length = sock_recvmsg(server->ssocket, smb_msg, 0); + + spin_lock(&server->srv_lock); + if (server->tcpStatus == CifsExiting) { + spin_unlock(&server->srv_lock); + return -ESHUTDOWN; + } + + if (server->tcpStatus == CifsNeedReconnect) { + spin_unlock(&server->srv_lock); + cifs_reconnect(server, false); + return -ECONNABORTED; + } + spin_unlock(&server->srv_lock); + + if (length == -ERESTARTSYS || + length == -EAGAIN || + length == -EINTR) { + /* + * Minimum sleep to prevent looping, allowing socket + * to clear and app threads to set tcpStatus + * CifsNeedReconnect if server hung. + */ + usleep_range(1000, 2000); + length = 0; + continue; + } + + if (length <= 0) { + cifs_dbg(FYI, "Received no data or error: %d\n", length); + cifs_reconnect(server, false); + return -ECONNABORTED; + } + } + return total_read; +} + +int +cifs_read_from_socket(struct TCP_Server_Info *server, char *buf, + unsigned int to_read) +{ + struct msghdr smb_msg = {}; + struct kvec iov = {.iov_base = buf, .iov_len = to_read}; + + iov_iter_kvec(&smb_msg.msg_iter, ITER_DEST, &iov, 1, to_read); + + return cifs_readv_from_socket(server, &smb_msg); +} + +ssize_t +cifs_discard_from_socket(struct TCP_Server_Info *server, size_t to_read) +{ + struct msghdr smb_msg = {}; + + /* + * iov_iter_discard already sets smb_msg.type and count and iov_offset + * and cifs_readv_from_socket sets msg_control and msg_controllen + * so little to initialize in struct msghdr + */ + iov_iter_discard(&smb_msg.msg_iter, ITER_DEST, to_read); + + return cifs_readv_from_socket(server, &smb_msg); +} + +int +cifs_read_iter_from_socket(struct TCP_Server_Info *server, struct iov_iter *iter, + unsigned int to_read) +{ + struct msghdr smb_msg = { .msg_iter = *iter }; + + iov_iter_truncate(&smb_msg.msg_iter, to_read); + return cifs_readv_from_socket(server, &smb_msg); +} + +static bool +is_smb_response(struct TCP_Server_Info *server, unsigned char type) +{ + /* + * The first byte big endian of the length field, + * is actually not part of the length but the type + * with the most common, zero, as regular data. + */ + switch (type) { + case RFC1002_SESSION_MESSAGE: + /* Regular SMB response */ + return true; + case RFC1002_SESSION_KEEP_ALIVE: + /* + * RFC 1002 session keep alive can sent by the server only when + * we established a RFC 1002 session. But Samba servers send + * RFC 1002 session keep alive also over port 445 on which + * RFC 1002 session is not established. + */ + cifs_dbg(FYI, "RFC 1002 session keep alive\n"); + break; + case RFC1002_POSITIVE_SESSION_RESPONSE: + /* + * RFC 1002 positive session response cannot be returned + * for SMB request. RFC 1002 session response is handled + * exclusively in ip_rfc1001_connect() function. + */ + cifs_server_dbg(VFS, "RFC 1002 positive session response (unexpected)\n"); + cifs_reconnect(server, true); + break; + case RFC1002_NEGATIVE_SESSION_RESPONSE: + /* + * We get this from Windows 98 instead of an error on + * SMB negprot response, when we have not established + * RFC 1002 session (which means ip_rfc1001_connect() + * was skipped). Note that same still happens with + * Windows Server 2022 when connecting via port 139. + * So for this case when mount option -o nonbsessinit + * was not specified, try to reconnect with establishing + * RFC 1002 session. If new socket establishment with + * RFC 1002 session was successful then return to the + * mid's caller -EAGAIN, so it can retry the request. + */ + if (!cifs_rdma_enabled(server) && + server->tcpStatus == CifsInNegotiate && + !server->with_rfc1001 && + server->rfc1001_sessinit != 0) { + int rc, mid_rc; + struct mid_q_entry *mid, *nmid; + LIST_HEAD(dispose_list); + + cifs_dbg(FYI, "RFC 1002 negative session response during SMB Negotiate, retrying with NetBIOS session\n"); + + /* + * Before reconnect, delete all pending mids for this + * server, so reconnect would not signal connection + * aborted error to mid's callbacks. Note that for this + * server there should be exactly one pending mid + * corresponding to SMB1/SMB2 Negotiate packet. + */ + spin_lock(&server->mid_queue_lock); + list_for_each_entry_safe(mid, nmid, &server->pending_mid_q, qhead) { + smb_get_mid(mid); + list_move(&mid->qhead, &dispose_list); + mid->deleted_from_q = true; + } + spin_unlock(&server->mid_queue_lock); + + /* Now try to reconnect once with NetBIOS session. */ + server->with_rfc1001 = true; + rc = cifs_reconnect_once(server); + + /* + * If reconnect was successful then indicate -EAGAIN + * to mid's caller. If reconnect failed with -EAGAIN + * then mask it as -EHOSTDOWN, so mid's caller would + * know that it failed. + */ + if (rc == 0) + mid_rc = -EAGAIN; + else if (rc == -EAGAIN) + mid_rc = -EHOSTDOWN; + else + mid_rc = rc; + + /* + * After reconnect (either successful or unsuccessful) + * deliver reconnect status to mid's caller via mid's + * callback. Use MID_RC state which indicates that the + * return code should be read from mid_rc member. + */ + list_for_each_entry_safe(mid, nmid, &dispose_list, qhead) { + list_del_init(&mid->qhead); + mid->mid_rc = mid_rc; + mid->mid_state = MID_RC; + mid_execute_callback(server, mid); + release_mid(server, mid); + } + + /* + * If reconnect failed then wait two seconds. In most + * cases we were been called from the mount context and + * delivered failure to mid's callback will stop this + * receiver task thread and fails the mount process. + * So wait two seconds to prevent another reconnect + * in this task thread, which would be useless as the + * mount context will fail at all. + */ + if (rc != 0) + msleep(2000); + } else { + cifs_server_dbg(VFS, "RFC 1002 negative session response (unexpected)\n"); + cifs_reconnect(server, true); + } + break; + case RFC1002_RETARGET_SESSION_RESPONSE: + cifs_server_dbg(VFS, "RFC 1002 retarget session response (unexpected)\n"); + cifs_reconnect(server, true); + break; + default: + cifs_server_dbg(VFS, "RFC 1002 unknown response type 0x%x\n", type); + cifs_reconnect(server, true); + } + + return false; +} + +void +dequeue_mid(struct TCP_Server_Info *server, struct mid_q_entry *mid, bool malformed) +{ +#ifdef CONFIG_CIFS_STATS2 + mid->when_received = jiffies; +#endif + spin_lock(&server->mid_queue_lock); + if (!malformed) + mid->mid_state = MID_RESPONSE_RECEIVED; + else + mid->mid_state = MID_RESPONSE_MALFORMED; + /* + * Trying to handle/dequeue a mid after the send_recv() + * function has finished processing it is a bug. + */ + if (mid->deleted_from_q == true) { + spin_unlock(&server->mid_queue_lock); + pr_warn_once("trying to dequeue a deleted mid\n"); + } else { + list_del_init(&mid->qhead); + mid->deleted_from_q = true; + spin_unlock(&server->mid_queue_lock); + } +} + +static unsigned int +smb2_get_credits_from_hdr(char *buffer, struct TCP_Server_Info *server) +{ + struct smb2_hdr *shdr = (struct smb2_hdr *)buffer; + + /* + * SMB1 does not use credits. + */ + if (is_smb1(server)) + return 0; + + return le16_to_cpu(shdr->CreditRequest); +} + +static void +handle_mid(struct mid_q_entry *mid, struct TCP_Server_Info *server, + char *buf, int malformed) +{ + if (server->ops->check_trans2 && + server->ops->check_trans2(mid, server, buf, malformed)) + return; + mid->credits_received = smb2_get_credits_from_hdr(buf, server); + mid->resp_buf = buf; + mid->large_buf = server->large_buf; + /* Was previous buf put in mpx struct for multi-rsp? */ + if (!mid->multiRsp) { + /* smb buffer will be freed by user thread */ + if (server->large_buf) + server->bigbuf = NULL; + else + server->smallbuf = NULL; + } + dequeue_mid(server, mid, malformed); +} + +int +cifs_enable_signing(struct TCP_Server_Info *server, bool mnt_sign_required) +{ + bool srv_sign_required = server->sec_mode & server->vals->signing_required; + bool srv_sign_enabled = server->sec_mode & server->vals->signing_enabled; + bool mnt_sign_enabled; + + /* + * Is signing required by mnt options? If not then check + * global_secflags to see if it is there. + */ + if (!mnt_sign_required) + mnt_sign_required = ((global_secflags & CIFSSEC_MUST_SIGN) == + CIFSSEC_MUST_SIGN); + + /* + * If signing is required then it's automatically enabled too, + * otherwise, check to see if the secflags allow it. + */ + mnt_sign_enabled = mnt_sign_required ? mnt_sign_required : + (global_secflags & CIFSSEC_MAY_SIGN); + + /* If server requires signing, does client allow it? */ + if (srv_sign_required) { + if (!mnt_sign_enabled) { + cifs_dbg(VFS, "Server requires signing, but it's disabled in SecurityFlags!\n"); + return -EOPNOTSUPP; + } + server->sign = true; + } + + /* If client requires signing, does server allow it? */ + if (mnt_sign_required) { + if (!srv_sign_enabled) { + cifs_dbg(VFS, "Server does not support signing!\n"); + return -EOPNOTSUPP; + } + server->sign = true; + } + + if (cifs_rdma_enabled(server) && server->sign) + cifs_dbg(VFS, "Signing is enabled, and RDMA read/write will be disabled\n"); + + return 0; +} + +static noinline_for_stack void +clean_demultiplex_info(struct TCP_Server_Info *server) +{ + int length; + + /* take it off the list, if it's not already */ + spin_lock(&server->srv_lock); + list_del_init(&server->tcp_ses_list); + spin_unlock(&server->srv_lock); + + cancel_delayed_work_sync(&server->echo); + + spin_lock(&server->srv_lock); + server->tcpStatus = CifsExiting; + spin_unlock(&server->srv_lock); + wake_up_all(&server->response_q); + + /* check if we have blocked requests that need to free */ + spin_lock(&server->req_lock); + if (server->credits <= 0) + server->credits = 1; + spin_unlock(&server->req_lock); + /* + * Although there should not be any requests blocked on this queue it + * can not hurt to be paranoid and try to wake up requests that may + * haven been blocked when more than 50 at time were on the wire to the + * same server - they now will see the session is in exit state and get + * out of SendReceive. + */ + wake_up_all(&server->request_q); + /* give those requests time to exit */ + msleep(125); + if (cifs_rdma_enabled(server)) + smbd_destroy(server); + if (server->ssocket) { + sock_release(server->ssocket); + server->ssocket = NULL; + } + + if (!list_empty(&server->pending_mid_q)) { + struct mid_q_entry *mid_entry; + struct list_head *tmp, *tmp2; + LIST_HEAD(dispose_list); + + spin_lock(&server->mid_queue_lock); + list_for_each_safe(tmp, tmp2, &server->pending_mid_q) { + mid_entry = list_entry(tmp, struct mid_q_entry, qhead); + cifs_dbg(FYI, "Clearing mid %llu\n", mid_entry->mid); + smb_get_mid(mid_entry); + mid_entry->mid_state = MID_SHUTDOWN; + list_move(&mid_entry->qhead, &dispose_list); + mid_entry->deleted_from_q = true; + } + spin_unlock(&server->mid_queue_lock); + + /* now walk dispose list and issue callbacks */ + list_for_each_safe(tmp, tmp2, &dispose_list) { + mid_entry = list_entry(tmp, struct mid_q_entry, qhead); + cifs_dbg(FYI, "Callback mid %llu\n", mid_entry->mid); + list_del_init(&mid_entry->qhead); + mid_execute_callback(server, mid_entry); + release_mid(server, mid_entry); + } + /* 1/8th of sec is more than enough time for them to exit */ + msleep(125); + } + + if (!list_empty(&server->pending_mid_q)) { + /* + * mpx threads have not exited yet give them at least the smb + * send timeout time for long ops. + * + * Due to delays on oplock break requests, we need to wait at + * least 45 seconds before giving up on a request getting a + * response and going ahead and killing cifsd. + */ + cifs_dbg(FYI, "Wait for exit from demultiplex thread\n"); + msleep(46000); + /* + * If threads still have not exited they are probably never + * coming home not much else we can do but free the memory. + */ + } + + put_net(cifs_net_ns(server)); + kfree(server->leaf_fullpath); + kfree(server->hostname); + kfree(server); + + length = atomic_dec_return(&tcpSesAllocCount); + if (length > 0) + mempool_resize(cifs_req_poolp, length + cifs_min_rcv); +} + +static int +standard_receive3(struct TCP_Server_Info *server, struct mid_q_entry *mid) +{ + int length; + char *buf = server->smallbuf; + unsigned int pdu_length = server->pdu_size; + + /* make sure this will fit in a large buffer */ + if (pdu_length > CIFSMaxBufSize + MAX_HEADER_SIZE(server)) { + cifs_server_dbg(VFS, "SMB response too long (%u bytes)\n", pdu_length); + cifs_reconnect(server, true); + return -ECONNABORTED; + } + + /* switch to large buffer if too big for a small one */ + if (pdu_length > MAX_CIFS_SMALL_BUFFER_SIZE) { + server->large_buf = true; + memcpy(server->bigbuf, buf, server->total_read); + buf = server->bigbuf; + } + + /* now read the rest */ + length = cifs_read_from_socket(server, buf + HEADER_SIZE(server) - 1, + pdu_length - MID_HEADER_SIZE(server)); + + if (length < 0) + return length; + server->total_read += length; + + dump_smb(buf, server->total_read); + + return cifs_handle_standard(server, mid); +} + +int +cifs_handle_standard(struct TCP_Server_Info *server, struct mid_q_entry *mid) +{ + char *buf = server->large_buf ? server->bigbuf : server->smallbuf; + int rc; + + /* + * We know that we received enough to get to the MID as we + * checked the pdu_length earlier. Now check to see + * if the rest of the header is OK. + * + * 48 bytes is enough to display the header and a little bit + * into the payload for debugging purposes. + */ + rc = server->ops->check_message(buf, server->pdu_size, + server->total_read, server); + if (rc) + cifs_dump_mem("Bad SMB: ", buf, + min_t(unsigned int, server->total_read, 48)); + + if (server->ops->is_session_expired && + server->ops->is_session_expired(buf)) { + cifs_reconnect(server, true); + return -1; + } + + if (server->ops->is_status_pending && + server->ops->is_status_pending(buf, server)) + return -1; + + if (!mid) + return rc; + + handle_mid(mid, server, buf, rc); + return 0; +} + +static void +smb2_add_credits_from_hdr(char *buffer, struct TCP_Server_Info *server) +{ + struct smb2_hdr *shdr = (struct smb2_hdr *)buffer; + int scredits, in_flight; + + /* + * SMB1 does not use credits. + */ + if (is_smb1(server)) + return; + + if (shdr->CreditRequest) { + spin_lock(&server->req_lock); + server->credits += le16_to_cpu(shdr->CreditRequest); + scredits = server->credits; + in_flight = server->in_flight; + spin_unlock(&server->req_lock); + wake_up(&server->request_q); + + trace_smb3_hdr_credits(server->current_mid, + server->conn_id, server->hostname, scredits, + le16_to_cpu(shdr->CreditRequest), in_flight); + cifs_server_dbg(FYI, "%s: added %u credits total=%d\n", + __func__, le16_to_cpu(shdr->CreditRequest), + scredits); + } +} + + +static int +cifs_demultiplex_thread(void *p) +{ + int i, num_mids, length; + struct TCP_Server_Info *server = p; + unsigned int pdu_length; + unsigned int next_offset; + char *buf = NULL; + struct task_struct *task_to_wake = NULL; + struct mid_q_entry *mids[MAX_COMPOUND]; + char *bufs[MAX_COMPOUND]; + unsigned int noreclaim_flag, num_io_timeout = 0; + bool pending_reconnect = false; + + noreclaim_flag = memalloc_noreclaim_save(); + cifs_dbg(FYI, "Demultiplex PID: %d\n", task_pid_nr(current)); + + length = atomic_inc_return(&tcpSesAllocCount); + if (length > 1) + mempool_resize(cifs_req_poolp, length + cifs_min_rcv); + + set_freezable(); + allow_kernel_signal(SIGKILL); + while (server->tcpStatus != CifsExiting) { + if (try_to_freeze()) + continue; + + if (!allocate_buffers(server)) + continue; + + server->large_buf = false; + buf = server->smallbuf; + pdu_length = 4; /* enough to get RFC1001 header */ + + length = cifs_read_from_socket(server, buf, pdu_length); + if (length < 0) + continue; + + server->total_read = 0; + + /* + * The right amount was read from socket - 4 bytes, + * so we can now interpret the length field. + */ + pdu_length = be32_to_cpup(((__be32 *)buf)) & 0xffffff; + + cifs_dbg(FYI, "RFC1002 header 0x%x\n", pdu_length); + if (!is_smb_response(server, buf[0])) + continue; + + pending_reconnect = false; +next_pdu: + server->pdu_size = pdu_length; + + /* make sure we have enough to get to the MID */ + if (server->pdu_size < MID_HEADER_SIZE(server)) { + cifs_server_dbg(VFS, "SMB response too short (%u bytes)\n", + server->pdu_size); + cifs_reconnect(server, true); + continue; + } + + /* read down to the MID */ + length = cifs_read_from_socket(server, buf, + MID_HEADER_SIZE(server)); + if (length < 0) + continue; + server->total_read += length; + + if (server->ops->next_header) { + if (server->ops->next_header(server, buf, &next_offset)) { + cifs_dbg(VFS, "%s: malformed response (next_offset=%u)\n", + __func__, next_offset); + cifs_reconnect(server, true); + continue; + } + if (next_offset) + server->pdu_size = next_offset; + } + + memset(mids, 0, sizeof(mids)); + memset(bufs, 0, sizeof(bufs)); + num_mids = 0; + + if (server->ops->is_transform_hdr && + server->ops->receive_transform && + server->ops->is_transform_hdr(buf)) { + length = server->ops->receive_transform(server, + mids, + bufs, + &num_mids); + } else { + mids[0] = server->ops->find_mid(server, buf); + bufs[0] = buf; + num_mids = 1; + + if (mids[0]) + mids[0]->response_pdu_len = pdu_length; + if (!mids[0] || !mids[0]->receive) + length = standard_receive3(server, mids[0]); + else + length = mids[0]->receive(server, mids[0]); + } + + if (length < 0) { + for (i = 0; i < num_mids; i++) + if (mids[i]) + release_mid(server, mids[i]); + continue; + } + + if (server->ops->is_status_io_timeout && + server->ops->is_status_io_timeout(buf)) { + num_io_timeout++; + if (num_io_timeout > MAX_STATUS_IO_TIMEOUT) { + cifs_server_dbg(VFS, + "Number of request timeouts exceeded %d. Reconnecting", + MAX_STATUS_IO_TIMEOUT); + + pending_reconnect = true; + num_io_timeout = 0; + } + } + + server->lstrp = jiffies; + + for (i = 0; i < num_mids; i++) { + if (mids[i] != NULL) { + mids[i]->resp_buf_size = server->pdu_size; + + if (bufs[i] != NULL) { + if (server->ops->is_network_name_deleted && + server->ops->is_network_name_deleted(bufs[i], + server)) { + cifs_server_dbg(FYI, + "Share deleted. Reconnect needed"); + } + } + + if (!mids[i]->multiRsp || mids[i]->multiEnd) + mid_execute_callback(server, mids[i]); + + release_mid(server, mids[i]); + } else if (server->ops->is_oplock_break && + server->ops->is_oplock_break(bufs[i], + server)) { + smb2_add_credits_from_hdr(bufs[i], server); + cifs_dbg(FYI, "Received oplock break\n"); + } else { + cifs_server_dbg(VFS, "No task to wake, unknown frame received! NumMids %d\n", + atomic_read(&mid_count)); + cifs_dump_mem("Received Data is: ", bufs[i], + HEADER_SIZE(server)); + smb2_add_credits_from_hdr(bufs[i], server); +#ifdef CONFIG_CIFS_DEBUG2 + if (server->ops->dump_detail) + server->ops->dump_detail(bufs[i], pdu_length, + server); + cifs_dump_mids(server); +#endif /* CIFS_DEBUG2 */ + } + } + + if (pdu_length > server->pdu_size) { + if (!allocate_buffers(server)) + continue; + pdu_length -= server->pdu_size; + server->total_read = 0; + server->large_buf = false; + buf = server->smallbuf; + goto next_pdu; + } + + /* do this reconnect at the very end after processing all MIDs */ + if (pending_reconnect) + cifs_reconnect(server, true); + + } /* end while !EXITING */ + + /* buffer usually freed in free_mid - need to free it here on exit */ + cifs_buf_release(server->bigbuf); + if (server->smallbuf) /* no sense logging a debug message if NULL */ + cifs_small_buf_release(server->smallbuf); + + task_to_wake = xchg(&server->tsk, NULL); + clean_demultiplex_info(server); + + /* if server->tsk was NULL then wait for a signal before exiting */ + if (!task_to_wake) { + set_current_state(TASK_INTERRUPTIBLE); + while (!signal_pending(current)) { + schedule(); + set_current_state(TASK_INTERRUPTIBLE); + } + set_current_state(TASK_RUNNING); + } + + memalloc_noreclaim_restore(noreclaim_flag); + module_put_and_kthread_exit(0); +} + +int +cifs_ipaddr_cmp(struct sockaddr *srcaddr, struct sockaddr *rhs) +{ + struct sockaddr_in *saddr4 = (struct sockaddr_in *)srcaddr; + struct sockaddr_in *vaddr4 = (struct sockaddr_in *)rhs; + struct sockaddr_in6 *saddr6 = (struct sockaddr_in6 *)srcaddr; + struct sockaddr_in6 *vaddr6 = (struct sockaddr_in6 *)rhs; + + switch (srcaddr->sa_family) { + case AF_UNSPEC: + switch (rhs->sa_family) { + case AF_UNSPEC: + return 0; + case AF_INET: + case AF_INET6: + return 1; + default: + return -1; + } + case AF_INET: { + switch (rhs->sa_family) { + case AF_UNSPEC: + return -1; + case AF_INET: + return memcmp(saddr4, vaddr4, + sizeof(struct sockaddr_in)); + case AF_INET6: + return 1; + default: + return -1; + } + } + case AF_INET6: { + switch (rhs->sa_family) { + case AF_UNSPEC: + case AF_INET: + return -1; + case AF_INET6: + return memcmp(saddr6, + vaddr6, + sizeof(struct sockaddr_in6)); + default: + return -1; + } + } + default: + return -1; /* don't expect to be here */ + } +} + +/* + * Returns true if srcaddr isn't specified and rhs isn't specified, or + * if srcaddr is specified and matches the IP address of the rhs argument + */ +bool +cifs_match_ipaddr(struct sockaddr *srcaddr, struct sockaddr *rhs) +{ + switch (srcaddr->sa_family) { + case AF_UNSPEC: + return (rhs->sa_family == AF_UNSPEC); + case AF_INET: { + struct sockaddr_in *saddr4 = (struct sockaddr_in *)srcaddr; + struct sockaddr_in *vaddr4 = (struct sockaddr_in *)rhs; + + return (saddr4->sin_addr.s_addr == vaddr4->sin_addr.s_addr); + } + case AF_INET6: { + struct sockaddr_in6 *saddr6 = (struct sockaddr_in6 *)srcaddr; + struct sockaddr_in6 *vaddr6 = (struct sockaddr_in6 *)rhs; + + return (ipv6_addr_equal(&saddr6->sin6_addr, &vaddr6->sin6_addr) + && saddr6->sin6_scope_id == vaddr6->sin6_scope_id); + } + default: + WARN_ON(1); + return false; /* don't expect to be here */ + } +} + +/* + * If no port is specified in addr structure, we try to match with 445 port + * and if it fails - with 139 ports. It should be called only if address + * families of server and addr are equal. + */ +static bool +match_port(struct TCP_Server_Info *server, struct sockaddr *addr) +{ + __be16 port, *sport; + + /* SMBDirect manages its own ports, don't match it here */ + if (server->rdma) + return true; + + switch (addr->sa_family) { + case AF_INET: + sport = &((struct sockaddr_in *) &server->dstaddr)->sin_port; + port = ((struct sockaddr_in *) addr)->sin_port; + break; + case AF_INET6: + sport = &((struct sockaddr_in6 *) &server->dstaddr)->sin6_port; + port = ((struct sockaddr_in6 *) addr)->sin6_port; + break; + default: + WARN_ON(1); + return false; + } + + if (!port) { + port = htons(CIFS_PORT); + if (port == *sport) + return true; + + port = htons(RFC1001_PORT); + } + + return port == *sport; +} + +static bool match_server_address(struct TCP_Server_Info *server, struct sockaddr *addr) +{ + if (!cifs_match_ipaddr(addr, (struct sockaddr *)&server->dstaddr)) + return false; + + return true; +} + +static bool +match_security(struct TCP_Server_Info *server, struct smb3_fs_context *ctx) +{ + /* + * The select_sectype function should either return the ctx->sectype + * that was specified, or "Unspecified" if that sectype was not + * compatible with the given NEGOTIATE request. + */ + if (server->ops->select_sectype(server, ctx->sectype) + == Unspecified) + return false; + + /* + * Now check if signing mode is acceptable. No need to check + * global_secflags at this point since if MUST_SIGN is set then + * the server->sign had better be too. + */ + if (ctx->sign && !server->sign) + return false; + + return true; +} + +/* this function must be called with srv_lock held */ +static int match_server(struct TCP_Server_Info *server, + struct smb3_fs_context *ctx, + bool match_super) +{ + struct sockaddr *addr = (struct sockaddr *)&ctx->dstaddr; + + lockdep_assert_held(&server->srv_lock); + + if (ctx->nosharesock) + return 0; + + /* this server does not share socket */ + if (server->nosharesock) + return 0; + + if (!match_super && (ctx->dfs_conn || server->dfs_conn)) + return 0; + + /* If multidialect negotiation see if existing sessions match one */ + if (strcmp(ctx->vals->version_string, SMB3ANY_VERSION_STRING) == 0) { + if (server->vals->protocol_id < SMB30_PROT_ID) + return 0; + } else if (strcmp(ctx->vals->version_string, + SMBDEFAULT_VERSION_STRING) == 0) { + if (server->vals->protocol_id < SMB21_PROT_ID) + return 0; + } else if ((server->vals != ctx->vals) || (server->ops != ctx->ops)) + return 0; + + if (!net_eq(cifs_net_ns(server), current->nsproxy->net_ns)) + return 0; + + if (!cifs_match_ipaddr((struct sockaddr *)&ctx->srcaddr, + (struct sockaddr *)&server->srcaddr)) + return 0; + + if (strcasecmp(server->hostname, ctx->server_hostname) || + !match_server_address(server, addr) || + !match_port(server, addr)) + return 0; + + if (!match_security(server, ctx)) + return 0; + + if (server->echo_interval != ctx->echo_interval * HZ) + return 0; + + if (server->rdma != ctx->rdma) + return 0; + + if (server->ignore_signature != ctx->ignore_signature) + return 0; + + if (server->min_offload != ctx->min_offload) + return 0; + + if (server->retrans != ctx->retrans) + return 0; + + return 1; +} + +struct TCP_Server_Info * +cifs_find_tcp_session(struct smb3_fs_context *ctx) +{ + struct TCP_Server_Info *server; + + spin_lock(&cifs_tcp_ses_lock); + list_for_each_entry(server, &cifs_tcp_ses_list, tcp_ses_list) { + spin_lock(&server->srv_lock); + /* + * Skip ses channels since they're only handled in lower layers + * (e.g. cifs_send_recv). + */ + if (SERVER_IS_CHAN(server) || + !match_server(server, ctx, false)) { + spin_unlock(&server->srv_lock); + continue; + } + spin_unlock(&server->srv_lock); + + ++server->srv_count; + spin_unlock(&cifs_tcp_ses_lock); + cifs_dbg(FYI, "Existing tcp session with server found\n"); + return server; + } + spin_unlock(&cifs_tcp_ses_lock); + return NULL; +} + +void +cifs_put_tcp_session(struct TCP_Server_Info *server, int from_reconnect) +{ + struct task_struct *task; + + spin_lock(&cifs_tcp_ses_lock); + if (--server->srv_count > 0) { + spin_unlock(&cifs_tcp_ses_lock); + return; + } + + /* srv_count can never go negative */ + WARN_ON(server->srv_count < 0); + + list_del_init(&server->tcp_ses_list); + spin_unlock(&cifs_tcp_ses_lock); + + cancel_delayed_work_sync(&server->echo); + + if (from_reconnect) + /* + * Avoid deadlock here: reconnect work calls + * cifs_put_tcp_session() at its end. Need to be sure + * that reconnect work does nothing with server pointer after + * that step. + */ + cancel_delayed_work(&server->reconnect); + else + cancel_delayed_work_sync(&server->reconnect); + + /* For secondary channels, we pick up ref-count on the primary server */ + if (SERVER_IS_CHAN(server)) + cifs_put_tcp_session(server->primary_server, from_reconnect); + + spin_lock(&server->srv_lock); + server->tcpStatus = CifsExiting; + spin_unlock(&server->srv_lock); + + cifs_crypto_secmech_release(server); + + kfree_sensitive(server->session_key.response); + server->session_key.response = NULL; + server->session_key.len = 0; + + task = xchg(&server->tsk, NULL); + if (task) + send_sig(SIGKILL, task, 1); +} + +struct TCP_Server_Info * +cifs_get_tcp_session(struct smb3_fs_context *ctx, + struct TCP_Server_Info *primary_server) +{ + struct TCP_Server_Info *tcp_ses = NULL; + int rc; + + cifs_dbg(FYI, "UNC: %s\n", ctx->UNC); + + /* see if we already have a matching tcp_ses */ + tcp_ses = cifs_find_tcp_session(ctx); + if (tcp_ses) + return tcp_ses; + + tcp_ses = kzalloc_obj(struct TCP_Server_Info); + if (!tcp_ses) { + rc = -ENOMEM; + goto out_err; + } + + tcp_ses->hostname = kstrdup(ctx->server_hostname, GFP_KERNEL); + if (!tcp_ses->hostname) { + rc = -ENOMEM; + goto out_err; + } + + if (ctx->leaf_fullpath) { + tcp_ses->leaf_fullpath = kstrdup(ctx->leaf_fullpath, GFP_KERNEL); + if (!tcp_ses->leaf_fullpath) { + rc = -ENOMEM; + goto out_err; + } + } + if (ctx->dns_dom) + strscpy(tcp_ses->dns_dom, ctx->dns_dom); + + if (ctx->nosharesock) + tcp_ses->nosharesock = true; + tcp_ses->dfs_conn = ctx->dfs_conn; + + tcp_ses->ops = ctx->ops; + tcp_ses->vals = ctx->vals; + cifs_set_net_ns(tcp_ses, get_net(current->nsproxy->net_ns)); + + tcp_ses->sign = ctx->sign; + tcp_ses->conn_id = atomic_inc_return(&tcpSesNextId); + tcp_ses->noblockcnt = ctx->rootfs; + tcp_ses->noblocksnd = ctx->noblocksnd || ctx->rootfs; + tcp_ses->noautotune = ctx->noautotune; + tcp_ses->tcp_nodelay = ctx->sockopt_tcp_nodelay; + tcp_ses->rdma = ctx->rdma; + tcp_ses->in_flight = 0; + tcp_ses->max_in_flight = 0; + tcp_ses->credits = 1; + if (primary_server) { + spin_lock(&cifs_tcp_ses_lock); + ++primary_server->srv_count; + spin_unlock(&cifs_tcp_ses_lock); + tcp_ses->primary_server = primary_server; + } + init_waitqueue_head(&tcp_ses->response_q); + init_waitqueue_head(&tcp_ses->request_q); + INIT_LIST_HEAD(&tcp_ses->pending_mid_q); + mutex_init(&tcp_ses->_srv_mutex); + memcpy(tcp_ses->workstation_RFC1001_name, + ctx->source_rfc1001_name, RFC1001_NAME_LEN_WITH_NULL); + memcpy(tcp_ses->server_RFC1001_name, + ctx->target_rfc1001_name, RFC1001_NAME_LEN_WITH_NULL); + tcp_ses->rfc1001_sessinit = ctx->rfc1001_sessinit; + tcp_ses->with_rfc1001 = false; + tcp_ses->session_estab = false; + tcp_ses->sequence_number = 0; + tcp_ses->channel_sequence_num = 0; /* only tracked for primary channel */ + tcp_ses->reconnect_instance = 1; + tcp_ses->lstrp = jiffies; + tcp_ses->compression.requested = ctx->compress; + spin_lock_init(&tcp_ses->req_lock); + spin_lock_init(&tcp_ses->srv_lock); + spin_lock_init(&tcp_ses->mid_queue_lock); + spin_lock_init(&tcp_ses->mid_counter_lock); + INIT_LIST_HEAD(&tcp_ses->tcp_ses_list); + INIT_LIST_HEAD(&tcp_ses->smb_ses_list); + INIT_DELAYED_WORK(&tcp_ses->echo, cifs_echo_request); + INIT_DELAYED_WORK(&tcp_ses->reconnect, smb2_reconnect_server); + mutex_init(&tcp_ses->reconnect_mutex); + memcpy(&tcp_ses->srcaddr, &ctx->srcaddr, + sizeof(tcp_ses->srcaddr)); + memcpy(&tcp_ses->dstaddr, &ctx->dstaddr, + sizeof(tcp_ses->dstaddr)); + if (ctx->use_client_guid) + memcpy(tcp_ses->client_guid, ctx->client_guid, + SMB2_CLIENT_GUID_SIZE); + else + generate_random_uuid(tcp_ses->client_guid); + /* + * at this point we are the only ones with the pointer + * to the struct since the kernel thread not created yet + * no need to spinlock this init of tcpStatus or srv_count + */ + tcp_ses->tcpStatus = CifsNew; + ++tcp_ses->srv_count; + tcp_ses->echo_interval = ctx->echo_interval * HZ; + + if (tcp_ses->rdma) { +#ifndef CONFIG_CIFS_SMB_DIRECT + cifs_dbg(VFS, "CONFIG_CIFS_SMB_DIRECT is not enabled\n"); + rc = -ENOENT; + goto out_err_crypto_release; +#endif + tcp_ses->smbd_conn = smbd_get_connection( + tcp_ses, (struct sockaddr *)&ctx->dstaddr); + if (tcp_ses->smbd_conn) { + cifs_dbg(VFS, "RDMA transport established\n"); + rc = 0; + goto smbd_connected; + } else { + rc = -ENOENT; + goto out_err_crypto_release; + } + } + rc = ip_connect(tcp_ses); + if (rc < 0) { + cifs_dbg(VFS, "Error connecting to socket. Aborting operation.\n"); + goto out_err_crypto_release; + } +smbd_connected: + /* + * since we're in a cifs function already, we know that + * this will succeed. No need for try_module_get(). + */ + __module_get(THIS_MODULE); + tcp_ses->min_offload = ctx->min_offload; + tcp_ses->retrans = ctx->retrans; + /* + * at this point we are the only ones with the pointer + * to the struct since the kernel thread not created yet + * no need to spinlock this update of tcpStatus + */ + tcp_ses->tcpStatus = CifsNeedNegotiate; + + if ((ctx->max_credits < 20) || (ctx->max_credits > 60000)) + tcp_ses->max_credits = SMB2_MAX_CREDITS_AVAILABLE; + else + tcp_ses->max_credits = ctx->max_credits; + + tcp_ses->nr_targets = 1; + tcp_ses->ignore_signature = ctx->ignore_signature; + + tcp_ses->tsk = kthread_create(cifs_demultiplex_thread, + tcp_ses, "cifsd"); + if (IS_ERR(tcp_ses->tsk)) { + rc = PTR_ERR(tcp_ses->tsk); + cifs_dbg(VFS, "error %d create cifsd thread\n", rc); + module_put(THIS_MODULE); + goto out_err_crypto_release; + } + /* thread created, put it on the list */ + spin_lock(&cifs_tcp_ses_lock); + list_add(&tcp_ses->tcp_ses_list, &cifs_tcp_ses_list); + spin_unlock(&cifs_tcp_ses_lock); + + /* queue echo request delayed work */ + queue_delayed_work(cifsiod_wq, &tcp_ses->echo, tcp_ses->echo_interval); + + /* + * Use split create/wake logic to ensure that tcp_ses is fully populated + * and tcp_ses->tsk is valid + */ + wake_up_process(tcp_ses->tsk); + + return tcp_ses; + +out_err_crypto_release: + cifs_crypto_secmech_release(tcp_ses); + + put_net(cifs_net_ns(tcp_ses)); + +out_err: + if (tcp_ses) { + if (SERVER_IS_CHAN(tcp_ses)) + cifs_put_tcp_session(tcp_ses->primary_server, false); + kfree(tcp_ses->hostname); + kfree(tcp_ses->leaf_fullpath); + if (tcp_ses->ssocket) + sock_release(tcp_ses->ssocket); + kfree(tcp_ses); + } + return ERR_PTR(rc); +} + +/* this function must be called with ses_lock and chan_lock held */ +static int match_session(struct cifs_ses *ses, + struct smb3_fs_context *ctx, + bool match_super) +{ + struct TCP_Server_Info *server = ses->server; + enum securityEnum ctx_sec, ses_sec; + + if (!match_super && ctx->dfs_root_ses != ses->dfs_root_ses) + return 0; + + /* + * If an existing session is limited to less channels than + * requested, it should not be reused + */ + if (ses->chan_max < ctx->max_channels) + return 0; + + ctx_sec = server->ops->select_sectype(server, ctx->sectype); + ses_sec = server->ops->select_sectype(server, ses->sectype); + + if (ctx_sec != ses_sec) + return 0; + + switch (ctx_sec) { + case IAKerb: + case Kerberos: + if (!uid_eq(ctx->cred_uid, ses->cred_uid)) + return 0; + if (strncmp(ses->user_name ?: "", + ctx->username ?: "", + CIFS_MAX_USERNAME_LEN)) + return 0; + break; + case NTLMv2: + case RawNTLMSSP: + default: + /* NULL username means anonymous session */ + if (ses->user_name == NULL) { + if (!ctx->nullauth) + return 0; + break; + } + + /* anything else takes username/password */ + if (strncmp(ses->user_name, + ctx->username ? ctx->username : "", + CIFS_MAX_USERNAME_LEN)) + return 0; + if ((ctx->username && strlen(ctx->username) != 0) && + ses->password != NULL) { + + /* New mount can only share sessions with an existing mount if: + * 1. Both password and password2 match, or + * 2. password2 of the old mount matches password of the new mount + * and password of the old mount matches password2 of the new + * mount + */ + if (ses->password2 != NULL && ctx->password2 != NULL) { + if (!((strncmp(ses->password, ctx->password ? + ctx->password : "", CIFS_MAX_PASSWORD_LEN) == 0 && + strncmp(ses->password2, ctx->password2, + CIFS_MAX_PASSWORD_LEN) == 0) || + (strncmp(ses->password, ctx->password2, + CIFS_MAX_PASSWORD_LEN) == 0 && + strncmp(ses->password2, ctx->password ? + ctx->password : "", CIFS_MAX_PASSWORD_LEN) == 0))) + return 0; + + } else if ((ses->password2 == NULL && ctx->password2 != NULL) || + (ses->password2 != NULL && ctx->password2 == NULL)) { + return 0; + + } else { + if (strncmp(ses->password, ctx->password ? + ctx->password : "", CIFS_MAX_PASSWORD_LEN)) + return 0; + } + } + } + + if (strcmp(ctx->local_nls->charset, ses->local_nls->charset)) + return 0; + + return 1; +} + +/** + * cifs_setup_ipc - helper to setup the IPC tcon for the session + * @ses: smb session to issue the request on + * @seal: if encryption is requested + * + * A new IPC connection is made and stored in the session + * tcon_ipc. The IPC tcon has the same lifetime as the session. + */ +struct cifs_tcon *cifs_setup_ipc(struct cifs_ses *ses, bool seal) +{ + int rc = 0, xid; + struct cifs_tcon *tcon; + char unc[SERVER_NAME_LENGTH + sizeof("//x/IPC$")] = {0}; + struct TCP_Server_Info *server = ses->server; + + /* + * If the mount request that resulted in the creation of the + * session requires encryption, force IPC to be encrypted too. + */ + if (seal && !(server->capabilities & SMB2_GLOBAL_CAP_ENCRYPTION)) { + cifs_server_dbg(VFS, "IPC: server doesn't support encryption\n"); + return ERR_PTR(-EOPNOTSUPP); + } + + /* no need to setup directory caching on IPC share, so pass in false */ + tcon = tcon_info_alloc(false, netfs_trace_tcon_ref_new_ipc); + if (tcon == NULL) + return ERR_PTR(-ENOMEM); + + spin_lock(&server->srv_lock); + scnprintf(unc, sizeof(unc), "\\\\%s\\IPC$", server->hostname); + spin_unlock(&server->srv_lock); + + xid = get_xid(); + tcon->ses = ses; + tcon->ipc = true; + tcon->seal = seal; + rc = server->ops->tree_connect(xid, ses, unc, tcon, ses->local_nls); + free_xid(xid); + + if (rc) { + cifs_server_dbg(VFS | ONCE, "failed to connect to IPC (rc=%d)\n", rc); + tconInfoFree(tcon, netfs_trace_tcon_ref_free_ipc_fail); + return ERR_PTR(rc); + } + + cifs_dbg(FYI, "IPC tcon rc=%d ipc tid=0x%x\n", rc, tcon->tid); + + spin_lock(&tcon->tc_lock); + tcon->status = TID_GOOD; + spin_unlock(&tcon->tc_lock); + return tcon; +} + +static struct cifs_ses * +cifs_find_smb_ses(struct TCP_Server_Info *server, struct smb3_fs_context *ctx) +{ + struct cifs_ses *ses, *ret = NULL; + + spin_lock(&cifs_tcp_ses_lock); + list_for_each_entry(ses, &server->smb_ses_list, smb_ses_list) { + spin_lock(&ses->ses_lock); + if (ses->ses_status == SES_EXITING) { + spin_unlock(&ses->ses_lock); + continue; + } + spin_lock(&ses->chan_lock); + if (match_session(ses, ctx, false)) { + spin_unlock(&ses->chan_lock); + spin_unlock(&ses->ses_lock); + ret = ses; + break; + } + spin_unlock(&ses->chan_lock); + spin_unlock(&ses->ses_lock); + } + if (ret) + cifs_smb_ses_inc_refcount(ret); + spin_unlock(&cifs_tcp_ses_lock); + return ret; +} + +void __cifs_put_smb_ses(struct cifs_ses *ses) +{ + struct TCP_Server_Info *server = ses->server; + struct cifs_tcon *tcon; + unsigned int xid; + size_t i; + bool do_logoff; + int rc; + + spin_lock(&cifs_tcp_ses_lock); + spin_lock(&ses->ses_lock); + cifs_dbg(FYI, "%s: id=0x%llx ses_count=%d ses_status=%u ipc=%s\n", + __func__, ses->Suid, ses->ses_count, ses->ses_status, + ses->tcon_ipc ? ses->tcon_ipc->tree_name : "none"); + if (ses->ses_status == SES_EXITING || --ses->ses_count > 0) { + spin_unlock(&ses->ses_lock); + spin_unlock(&cifs_tcp_ses_lock); + return; + } + /* ses_count can never go negative */ + WARN_ON(ses->ses_count < 0); + + spin_lock(&ses->chan_lock); + cifs_chan_clear_need_reconnect(ses, server); + spin_unlock(&ses->chan_lock); + + do_logoff = ses->ses_status == SES_GOOD && server->ops->logoff; + ses->ses_status = SES_EXITING; + tcon = ses->tcon_ipc; + ses->tcon_ipc = NULL; + spin_unlock(&ses->ses_lock); + spin_unlock(&cifs_tcp_ses_lock); + + /* + * On session close, the IPC is closed and the server must release all + * tcons of the session. No need to send a tree disconnect here. + * + * Besides, it will make the server to not close durable and resilient + * files on session close, as specified in MS-SMB2 3.3.5.6 Receiving an + * SMB2 LOGOFF Request. + */ + tconInfoFree(tcon, netfs_trace_tcon_ref_free_ipc); + if (do_logoff) { + xid = get_xid(); + rc = server->ops->logoff(xid, ses); + cifs_server_dbg(FYI, "%s: Session Logoff: rc=%d\n", + __func__, rc); + _free_xid(xid); + } + + spin_lock(&cifs_tcp_ses_lock); + list_del_init(&ses->smb_ses_list); + spin_unlock(&cifs_tcp_ses_lock); + + /* close any extra channels */ + for (i = 1; i < ses->chan_count; i++) { + if (ses->chans[i].iface) { + kref_put(&ses->chans[i].iface->refcount, release_iface); + ses->chans[i].iface = NULL; + } + cifs_put_tcp_session(ses->chans[i].server, 0); + ses->chans[i].server = NULL; + } + + /* we now account for primary channel in iface->refcount */ + if (ses->chans[0].iface) { + kref_put(&ses->chans[0].iface->refcount, release_iface); + ses->chans[0].server = NULL; + } + + sesInfoFree(ses); + cifs_put_tcp_session(server, 0); +} + +#ifdef CONFIG_KEYS + +/* Populate username and pw fields from keyring if possible */ +static int +cifs_set_cifscreds(struct smb3_fs_context *ctx, struct cifs_ses *ses) +{ + int rc = 0; + int is_domain = 0; + const char *delim, *payload; + size_t desc_sz; + char *desc; + ssize_t len; + struct key *key; + struct TCP_Server_Info *server = ses->server; + struct sockaddr_in *sa; + struct sockaddr_in6 *sa6; + const struct user_key_payload *upayload; + + /* "cifs:a:" and "cifs:d:" are the same length; +1 for NUL terminator */ + desc_sz = strlen("cifs:a:") + CIFS_MAX_DOMAINNAME_LEN + 1; + desc = kmalloc(desc_sz, GFP_KERNEL); + if (!desc) + return -ENOMEM; + + /* try to find an address key first */ + switch (server->dstaddr.ss_family) { + case AF_INET: + sa = (struct sockaddr_in *)&server->dstaddr; + snprintf(desc, desc_sz, "cifs:a:%pI4", &sa->sin_addr.s_addr); + break; + case AF_INET6: + sa6 = (struct sockaddr_in6 *)&server->dstaddr; + snprintf(desc, desc_sz, "cifs:a:%pI6c", &sa6->sin6_addr.s6_addr); + break; + default: + cifs_dbg(FYI, "Bad ss_family (%hu)\n", + server->dstaddr.ss_family); + rc = -EINVAL; + goto out_err; + } + + cifs_dbg(FYI, "%s: desc=%s\n", __func__, desc); + key = request_key(&key_type_logon, desc, ""); + if (IS_ERR(key)) { + if (!ses->domainName) { + cifs_dbg(FYI, "domainName is NULL\n"); + rc = PTR_ERR(key); + goto out_err; + } + + /* didn't work, try to find a domain key */ + snprintf(desc, desc_sz, "cifs:d:%s", ses->domainName); + cifs_dbg(FYI, "%s: desc=%s\n", __func__, desc); + key = request_key(&key_type_logon, desc, ""); + if (IS_ERR(key)) { + rc = PTR_ERR(key); + goto out_err; + } + is_domain = 1; + } + + down_read(&key->sem); + upayload = user_key_payload_locked(key); + if (IS_ERR_OR_NULL(upayload)) { + rc = upayload ? PTR_ERR(upayload) : -EINVAL; + goto out_key_put; + } + + /* find first : in payload */ + payload = upayload->data; + delim = strnchr(payload, upayload->datalen, ':'); + if (!delim) { + cifs_dbg(FYI, "Unable to find ':' in payload (datalen=%d)\n", + upayload->datalen); + rc = -EINVAL; + goto out_key_put; + } + + len = delim - payload; + if (len > CIFS_MAX_USERNAME_LEN || len <= 0) { + cifs_dbg(FYI, "Bad value from username search (len=%zd)\n", + len); + rc = -EINVAL; + goto out_key_put; + } + + ctx->username = kstrndup(payload, len, GFP_KERNEL); + if (!ctx->username) { + cifs_dbg(FYI, "Unable to allocate %zd bytes for username\n", + len); + rc = -ENOMEM; + goto out_key_put; + } + cifs_dbg(FYI, "%s: username=%s\n", __func__, ctx->username); + + len = key->datalen - (len + 1); + if (len > CIFS_MAX_PASSWORD_LEN || len <= 0) { + cifs_dbg(FYI, "Bad len for password search (len=%zd)\n", len); + rc = -EINVAL; + kfree(ctx->username); + ctx->username = NULL; + goto out_key_put; + } + + ++delim; + /* BB consider adding support for password2 (Key Rotation) for multiuser in future */ + ctx->password = kstrndup(delim, len, GFP_KERNEL); + if (!ctx->password) { + cifs_dbg(FYI, "Unable to allocate %zd bytes for password\n", + len); + rc = -ENOMEM; + kfree(ctx->username); + ctx->username = NULL; + goto out_key_put; + } + + /* + * If we have a domain key then we must set the domainName in the + * for the request. + */ + if (is_domain && ses->domainName) { + ctx->domainname = kstrdup(ses->domainName, GFP_KERNEL); + if (!ctx->domainname) { + cifs_dbg(FYI, "Unable to allocate %zd bytes for domain\n", + len); + rc = -ENOMEM; + kfree(ctx->username); + ctx->username = NULL; + kfree_sensitive(ctx->password); + /* no need to free ctx->password2 since not allocated in this path */ + ctx->password = NULL; + goto out_key_put; + } + } + + strscpy(ctx->workstation_name, ses->workstation_name, sizeof(ctx->workstation_name)); + +out_key_put: + up_read(&key->sem); + key_put(key); +out_err: + kfree(desc); + cifs_dbg(FYI, "%s: returning %d\n", __func__, rc); + return rc; +} +#else /* ! CONFIG_KEYS */ +static inline int +cifs_set_cifscreds(struct smb3_fs_context *ctx __maybe_unused, + struct cifs_ses *ses __maybe_unused) +{ + return -ENOSYS; +} +#endif /* CONFIG_KEYS */ + +/** + * cifs_get_smb_ses - get a session matching @ctx data from @server + * @server: server to setup the session to + * @ctx: superblock configuration context to use to setup the session + * + * This function assumes it is being called from cifs_mount() where we + * already got a server reference (server refcount +1). See + * cifs_get_tcon() for refcount explanations. + */ +struct cifs_ses * +cifs_get_smb_ses(struct TCP_Server_Info *server, struct smb3_fs_context *ctx) +{ + struct sockaddr_in6 *addr6 = (struct sockaddr_in6 *)&server->dstaddr; + struct sockaddr_in *addr = (struct sockaddr_in *)&server->dstaddr; + struct cifs_tcon *ipc; + struct cifs_ses *ses; + unsigned int xid; + int retries = 0; + size_t len; + int rc = 0; + + xid = get_xid(); + + ses = cifs_find_smb_ses(server, ctx); + if (ses) { + cifs_dbg(FYI, "Existing smb sess found (status=%d)\n", + ses->ses_status); + + spin_lock(&ses->chan_lock); + if (cifs_chan_needs_reconnect(ses, server)) { + spin_unlock(&ses->chan_lock); + cifs_dbg(FYI, "Session needs reconnect\n"); + + mutex_lock(&ses->session_mutex); + +retry_old_session: + rc = cifs_negotiate_protocol(xid, ses, server); + if (rc) { + mutex_unlock(&ses->session_mutex); + /* problem -- put our ses reference */ + cifs_put_smb_ses(ses); + free_xid(xid); + return ERR_PTR(rc); + } + + rc = cifs_setup_session(xid, ses, server, + ctx->local_nls); + if (rc) { + if (((rc == -EACCES) || (rc == -EKEYEXPIRED) || + (rc == -EKEYREVOKED)) && !retries && ses->password2) { + retries++; + cifs_dbg(FYI, "Session reconnect failed, retrying with alternate password\n"); + swap(ses->password, ses->password2); + goto retry_old_session; + } + mutex_unlock(&ses->session_mutex); + /* problem -- put our reference */ + cifs_put_smb_ses(ses); + free_xid(xid); + return ERR_PTR(rc); + } + mutex_unlock(&ses->session_mutex); + + spin_lock(&ses->chan_lock); + } + spin_unlock(&ses->chan_lock); + + /* existing SMB ses has a server reference already */ + cifs_put_tcp_session(server, 0); + free_xid(xid); + return ses; + } + + rc = -ENOMEM; + + cifs_dbg(FYI, "Existing smb sess not found\n"); + ses = sesInfoAlloc(); + if (ses == NULL) + goto get_ses_fail; + + /* new SMB session uses our server ref */ + ses->server = server; + if (server->dstaddr.ss_family == AF_INET6) + sprintf(ses->ip_addr, "%pI6", &addr6->sin6_addr); + else + sprintf(ses->ip_addr, "%pI4", &addr->sin_addr); + + if (ctx->username) { + ses->user_name = kstrdup(ctx->username, GFP_KERNEL); + if (!ses->user_name) + goto get_ses_fail; + } + + /* ctx->password freed at unmount */ + if (ctx->password) { + ses->password = kstrdup(ctx->password, GFP_KERNEL); + if (!ses->password) + goto get_ses_fail; + } + /* ctx->password freed at unmount */ + if (ctx->password2) { + ses->password2 = kstrdup(ctx->password2, GFP_KERNEL); + if (!ses->password2) + goto get_ses_fail; + } + if (ctx->domainname) { + ses->domainName = kstrdup(ctx->domainname, GFP_KERNEL); + if (!ses->domainName) + goto get_ses_fail; + + len = strnlen(ctx->domainname, CIFS_MAX_DOMAINNAME_LEN); + if (!cifs_netbios_name(ctx->domainname, len)) { + ses->dns_dom = kstrndup(ctx->domainname, + len, GFP_KERNEL); + if (!ses->dns_dom) + goto get_ses_fail; + } + } + + strscpy(ses->workstation_name, ctx->workstation_name, sizeof(ses->workstation_name)); + + if (ctx->domainauto) + ses->domainAuto = ctx->domainauto; + ses->cred_uid = ctx->cred_uid; + ses->linux_uid = ctx->linux_uid; + + ses->unicode = ctx->unicode; + ses->sectype = ctx->sectype; + ses->sign = ctx->sign; + + /* + *Explicitly marking upcall_target mount option for easier handling + * by cifs_spnego.c and eventually cifs.upcall.c + */ + + switch (ctx->upcall_target) { + case UPTARGET_UNSPECIFIED: /* default to app */ + case UPTARGET_APP: + ses->upcall_target = UPTARGET_APP; + break; + case UPTARGET_MOUNT: + ses->upcall_target = UPTARGET_MOUNT; + break; + default: + // should never happen + ses->upcall_target = UPTARGET_APP; + break; + } + + ses->local_nls = load_nls(ctx->local_nls->charset); + + /* add server as first channel */ + spin_lock(&ses->chan_lock); + ses->chans[0].server = server; + ses->chan_count = 1; + ses->chan_max = ctx->multichannel ? ctx->max_channels:1; + ses->chans_need_reconnect = 1; + spin_unlock(&ses->chan_lock); + +retry_new_session: + mutex_lock(&ses->session_mutex); + rc = cifs_negotiate_protocol(xid, ses, server); + if (!rc) + rc = cifs_setup_session(xid, ses, server, ctx->local_nls); + mutex_unlock(&ses->session_mutex); + + /* each channel uses a different signing key */ + spin_lock(&ses->chan_lock); + memcpy(ses->chans[0].signkey, ses->smb3signingkey, + sizeof(ses->smb3signingkey)); + spin_unlock(&ses->chan_lock); + + if (rc) { + if (((rc == -EACCES) || (rc == -EKEYEXPIRED) || + (rc == -EKEYREVOKED)) && !retries && ses->password2) { + retries++; + cifs_dbg(FYI, "Session setup failed, retrying with alternate password\n"); + swap(ses->password, ses->password2); + goto retry_new_session; + } else + goto get_ses_fail; + } + + /* + * success, put it on the list and add it as first channel + * note: the session becomes active soon after this. So you'll + * need to lock before changing something in the session. + */ + spin_lock(&cifs_tcp_ses_lock); + ses->dfs_root_ses = ctx->dfs_root_ses; + list_add(&ses->smb_ses_list, &server->smb_ses_list); + spin_unlock(&cifs_tcp_ses_lock); + + ipc = cifs_setup_ipc(ses, ctx->seal); + spin_lock(&cifs_tcp_ses_lock); + spin_lock(&ses->ses_lock); + ses->tcon_ipc = !IS_ERR(ipc) ? ipc : NULL; + spin_unlock(&ses->ses_lock); + spin_unlock(&cifs_tcp_ses_lock); + + free_xid(xid); + + return ses; + +get_ses_fail: + sesInfoFree(ses); + free_xid(xid); + return ERR_PTR(rc); +} + +/* this function must be called with tc_lock held */ +static int match_tcon(struct cifs_tcon *tcon, struct smb3_fs_context *ctx) +{ + struct TCP_Server_Info *server = tcon->ses->server; + + if (tcon->status == TID_EXITING) + return 0; + + if (tcon->origin_fullpath) { + if (!ctx->source || + !dfs_src_pathname_equal(ctx->source, + tcon->origin_fullpath)) + return 0; + } else if (!server->leaf_fullpath && + strncmp(tcon->tree_name, ctx->UNC, MAX_TREE_SIZE)) { + return 0; + } + if (tcon->seal != ctx->seal) + return 0; + if (tcon->snapshot_time != ctx->snapshot_time) + return 0; + if (tcon->handle_timeout != ctx->handle_timeout) + return 0; + if (tcon->no_lease != ctx->no_lease) + return 0; + if (tcon->nodelete != ctx->nodelete) + return 0; + if (tcon->posix_extensions != ctx->linux_ext) + return 0; + return 1; +} + +static struct cifs_tcon * +cifs_find_tcon(struct cifs_ses *ses, struct smb3_fs_context *ctx) +{ + struct cifs_tcon *tcon; + + spin_lock(&cifs_tcp_ses_lock); + list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { + spin_lock(&tcon->tc_lock); + if (!match_tcon(tcon, ctx)) { + spin_unlock(&tcon->tc_lock); + continue; + } + ++tcon->tc_count; + trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, + netfs_trace_tcon_ref_get_find); + spin_unlock(&tcon->tc_lock); + spin_unlock(&cifs_tcp_ses_lock); + return tcon; + } + spin_unlock(&cifs_tcp_ses_lock); + return NULL; +} + +void +cifs_put_tcon(struct cifs_tcon *tcon, enum smb3_tcon_ref_trace trace) +{ + unsigned int xid; + struct cifs_ses *ses; + LIST_HEAD(ses_list); + + /* + * IPC tcon share the lifetime of their session and are + * destroyed in the session put function + */ + if (tcon == NULL || tcon->ipc) + return; + + ses = tcon->ses; + cifs_dbg(FYI, "%s: tc_count=%d\n", __func__, tcon->tc_count); + spin_lock(&cifs_tcp_ses_lock); + spin_lock(&tcon->tc_lock); + trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count - 1, trace); + if (--tcon->tc_count > 0) { + spin_unlock(&tcon->tc_lock); + spin_unlock(&cifs_tcp_ses_lock); + return; + } + + /* tc_count can never go negative */ + WARN_ON(tcon->tc_count < 0); + + list_del_init(&tcon->tcon_list); + tcon->status = TID_EXITING; + spin_unlock(&tcon->tc_lock); + spin_unlock(&cifs_tcp_ses_lock); + + /* cancel polling of interfaces */ + cancel_delayed_work_sync(&tcon->query_interfaces); +#ifdef CONFIG_CIFS_DFS_UPCALL + cancel_delayed_work_sync(&tcon->dfs_cache_work); + list_replace_init(&tcon->dfs_ses_list, &ses_list); +#endif + + if (tcon->use_witness) { + int rc; + + rc = cifs_swn_unregister(tcon); + if (rc < 0) { + cifs_dbg(VFS, "%s: Failed to unregister for witness notifications: %d\n", + __func__, rc); + } + } + + xid = get_xid(); + if (ses->server->ops->tree_disconnect) + ses->server->ops->tree_disconnect(xid, tcon); + _free_xid(xid); + + cifs_fscache_release_super_cookie(tcon); + tconInfoFree(tcon, netfs_trace_tcon_ref_free); + cifs_put_smb_ses(ses); +#ifdef CONFIG_CIFS_DFS_UPCALL + dfs_put_root_smb_sessions(&ses_list); +#endif +} + +/** + * cifs_get_tcon - get a tcon matching @ctx data from @ses + * @ses: smb session to issue the request on + * @ctx: the superblock configuration context to use for building the + * + * - tcon refcount is the number of mount points using the tcon. + * - ses refcount is the number of tcon using the session. + * + * 1. This function assumes it is being called from cifs_mount() where + * we already got a session reference (ses refcount +1). + * + * 2. Since we're in the context of adding a mount point, the end + * result should be either: + * + * a) a new tcon already allocated with refcount=1 (1 mount point) and + * its session refcount incremented (1 new tcon). This +1 was + * already done in (1). + * + * b) an existing tcon with refcount+1 (add a mount point to it) and + * identical ses refcount (no new tcon). Because of (1) we need to + * decrement the ses refcount. + */ +static struct cifs_tcon * +cifs_get_tcon(struct cifs_ses *ses, struct smb3_fs_context *ctx) +{ + struct cifs_tcon *tcon; + bool nohandlecache; + int rc, xid; + + tcon = cifs_find_tcon(ses, ctx); + if (tcon) { + /* + * tcon has refcount already incremented but we need to + * decrement extra ses reference gotten by caller (case b) + */ + cifs_dbg(FYI, "Found match on UNC path\n"); + cifs_put_smb_ses(ses); + return tcon; + } + + if (!ses->server->ops->tree_connect) { + rc = -ENOSYS; + goto out_fail; + } + + if (ses->server->dialect >= SMB20_PROT_ID && + (ses->server->capabilities & SMB2_GLOBAL_CAP_DIRECTORY_LEASING)) + nohandlecache = ctx->nohandlecache || !dir_cache_timeout; + else + nohandlecache = true; + tcon = tcon_info_alloc(!nohandlecache, netfs_trace_tcon_ref_new); + if (tcon == NULL) { + rc = -ENOMEM; + goto out_fail; + } + tcon->nohandlecache = nohandlecache; + + if (ctx->snapshot_time) { + if (ses->server->vals->protocol_id == 0) { + cifs_dbg(VFS, + "Use SMB2 or later for snapshot mount option\n"); + rc = -EOPNOTSUPP; + goto out_fail; + } else + tcon->snapshot_time = ctx->snapshot_time; + } + + if (ctx->handle_timeout) { + if (ses->server->vals->protocol_id == 0) { + cifs_dbg(VFS, + "Use SMB2.1 or later for handle timeout option\n"); + rc = -EOPNOTSUPP; + goto out_fail; + } else + tcon->handle_timeout = ctx->handle_timeout; + } + + tcon->ses = ses; + if (ctx->password) { + tcon->password = kstrdup(ctx->password, GFP_KERNEL); + if (!tcon->password) { + rc = -ENOMEM; + goto out_fail; + } + } + + if (ctx->seal) { + if (ses->server->vals->protocol_id == 0) { + cifs_dbg(VFS, + "SMB3 or later required for encryption\n"); + rc = -EOPNOTSUPP; + goto out_fail; + } else if (tcon->ses->server->capabilities & + SMB2_GLOBAL_CAP_ENCRYPTION) + tcon->seal = true; + else { + cifs_dbg(VFS, "Encryption is not supported on share\n"); + rc = -EOPNOTSUPP; + goto out_fail; + } + } + + if (ctx->linux_ext) { + if (ses->server->posix_ext_supported) { + tcon->posix_extensions = true; + pr_warn_once("SMB3.11 POSIX Extensions are experimental\n"); + } else if ((ses->server->vals->protocol_id == SMB311_PROT_ID) || + (strcmp(ses->server->vals->version_string, + SMB3ANY_VERSION_STRING) == 0) || + (strcmp(ses->server->vals->version_string, + SMBDEFAULT_VERSION_STRING) == 0)) { + cifs_dbg(VFS, "Server does not support mounting with posix SMB3.11 extensions\n"); + rc = -EOPNOTSUPP; + goto out_fail; + } else if (ses->server->vals->protocol_id == SMB10_PROT_ID) + if (cap_unix(ses)) + cifs_dbg(FYI, "Unix Extensions requested on SMB1 mount\n"); + else { + cifs_dbg(VFS, "SMB1 Unix Extensions not supported by server\n"); + rc = -EOPNOTSUPP; + goto out_fail; + } else { + cifs_dbg(VFS, + "Check vers= mount option. SMB3.11 disabled but required for POSIX extensions\n"); + rc = -EOPNOTSUPP; + goto out_fail; + } + } + + xid = get_xid(); + rc = ses->server->ops->tree_connect(xid, ses, ctx->UNC, tcon, + ctx->local_nls); + free_xid(xid); + cifs_dbg(FYI, "Tcon rc = %d\n", rc); + if (rc) + goto out_fail; + + tcon->use_persistent = false; + /* check if SMB2 or later, CIFS does not support persistent handles */ + if (ctx->persistent) { + if (ses->server->vals->protocol_id == 0) { + cifs_dbg(VFS, + "SMB3 or later required for persistent handles\n"); + rc = -EOPNOTSUPP; + goto out_fail; + } else if (ses->server->capabilities & + SMB2_GLOBAL_CAP_PERSISTENT_HANDLES) + tcon->use_persistent = true; + else /* persistent handles requested but not supported */ { + cifs_dbg(VFS, + "Persistent handles not supported on share\n"); + rc = -EOPNOTSUPP; + goto out_fail; + } + } else if ((tcon->capabilities & SMB2_SHARE_CAP_CONTINUOUS_AVAILABILITY) + && (ses->server->capabilities & SMB2_GLOBAL_CAP_PERSISTENT_HANDLES) + && (ctx->nopersistent == false)) { + cifs_dbg(FYI, "enabling persistent handles\n"); + tcon->use_persistent = true; + } else if (ctx->resilient) { + if (ses->server->vals->protocol_id == 0) { + cifs_dbg(VFS, + "SMB2.1 or later required for resilient handles\n"); + rc = -EOPNOTSUPP; + goto out_fail; + } + tcon->use_resilient = true; + } + + tcon->use_witness = false; + if (IS_ENABLED(CONFIG_CIFS_SWN_UPCALL) && ctx->witness) { + if (ses->server->vals->protocol_id >= SMB30_PROT_ID) { + if (tcon->capabilities & SMB2_SHARE_CAP_CLUSTER) { + /* + * Set witness in use flag in first place + * to retry registration in the echo task + */ + tcon->use_witness = true; + /* And try to register immediately */ + rc = cifs_swn_register(tcon); + if (rc < 0) { + cifs_dbg(VFS, "Failed to register for witness notifications: %d\n", rc); + goto out_fail; + } + } else { + /* TODO: try to extend for non-cluster uses (eg multichannel) */ + cifs_dbg(VFS, "witness requested on mount but no CLUSTER capability on share\n"); + rc = -EOPNOTSUPP; + goto out_fail; + } + } else { + cifs_dbg(VFS, "SMB3 or later required for witness option\n"); + rc = -EOPNOTSUPP; + goto out_fail; + } + } + + /* If the user really knows what they are doing they can override */ + if (tcon->share_flags & SMB2_SHAREFLAG_NO_CACHING) { + if (ctx->cache_ro) + cifs_dbg(VFS, "cache=ro requested on mount but NO_CACHING flag set on share\n"); + else if (ctx->cache_rw) + cifs_dbg(VFS, "cache=singleclient requested on mount but NO_CACHING flag set on share\n"); + } + + if (ctx->no_lease) { + if (ses->server->vals->protocol_id == 0) { + cifs_dbg(VFS, + "SMB2 or later required for nolease option\n"); + rc = -EOPNOTSUPP; + goto out_fail; + } else + tcon->no_lease = ctx->no_lease; + } + + /* + * We can have only one retry value for a connection to a share so for + * resources mounted more than once to the same server share the last + * value passed in for the retry flag is used. + */ + tcon->retry = ctx->retry; + tcon->nocase = ctx->nocase; + tcon->broken_sparse_sup = ctx->no_sparse; + tcon->max_cached_dirs = ctx->max_cached_dirs; + tcon->nodelete = ctx->nodelete; + tcon->local_lease = ctx->local_lease; + tcon->status = TID_GOOD; + + if (ses->server->dialect >= SMB30_PROT_ID && + (ses->server->capabilities & SMB2_GLOBAL_CAP_MULTI_CHANNEL)) { + /* schedule query interfaces poll */ + queue_delayed_work(cifsiod_wq, &tcon->query_interfaces, + (SMB_INTERFACE_POLL_INTERVAL * HZ)); + } + spin_lock(&cifs_tcp_ses_lock); + list_add(&tcon->tcon_list, &ses->tcon_list); + spin_unlock(&cifs_tcp_ses_lock); + + return tcon; + +out_fail: + tconInfoFree(tcon, netfs_trace_tcon_ref_free_fail); + return ERR_PTR(rc); +} + +void +cifs_put_tlink(struct tcon_link *tlink) +{ + if (!tlink || IS_ERR(tlink)) + return; + + if (!atomic_dec_and_test(&tlink->tl_count) || + test_bit(TCON_LINK_IN_TREE, &tlink->tl_flags)) { + tlink->tl_time = jiffies; + return; + } + + if (!IS_ERR(tlink_tcon(tlink))) + cifs_put_tcon(tlink_tcon(tlink), netfs_trace_tcon_ref_put_tlink); + kfree(tlink); +} + +static int +compare_mount_options(struct super_block *sb, struct cifs_mnt_data *mnt_data) +{ + struct cifs_sb_info *old = CIFS_SB(sb); + struct cifs_sb_info *new = mnt_data->cifs_sb; + unsigned int oldflags = cifs_sb_flags(old) & CIFS_MOUNT_MASK; + unsigned int newflags = cifs_sb_flags(new) & CIFS_MOUNT_MASK; + + if ((sb->s_flags & CIFS_MS_MASK) != (mnt_data->flags & CIFS_MS_MASK)) + return 0; + + if (old->mnt_cifs_serverino_autodisabled) + newflags &= ~CIFS_MOUNT_SERVER_INUM; + + if (oldflags != newflags) + return 0; + + /* + * We want to share sb only if we don't specify an r/wsize or + * specified r/wsize is greater than or equal to existing one. + */ + if (new->ctx->wsize && new->ctx->wsize < old->ctx->wsize) + return 0; + + if (new->ctx->rsize && new->ctx->rsize < old->ctx->rsize) + return 0; + + if (!uid_eq(old->ctx->linux_uid, new->ctx->linux_uid) || + !gid_eq(old->ctx->linux_gid, new->ctx->linux_gid)) + return 0; + + if (old->ctx->file_mode != new->ctx->file_mode || + old->ctx->dir_mode != new->ctx->dir_mode) + return 0; + + if (strcmp(old->local_nls->charset, new->local_nls->charset)) + return 0; + + if (old->ctx->acregmax != new->ctx->acregmax) + return 0; + if (old->ctx->acdirmax != new->ctx->acdirmax) + return 0; + if (old->ctx->closetimeo != new->ctx->closetimeo) + return 0; + if (old->ctx->reparse_type != new->ctx->reparse_type) + return 0; + if (old->ctx->nonativesocket != new->ctx->nonativesocket) + return 0; + if (old->ctx->symlink_type != new->ctx->symlink_type) + return 0; + + return 1; +} + +static int match_prepath(struct super_block *sb, + struct cifs_tcon *tcon, + struct cifs_mnt_data *mnt_data) +{ + struct smb3_fs_context *ctx = mnt_data->ctx; + struct cifs_sb_info *old = CIFS_SB(sb); + struct cifs_sb_info *new = mnt_data->cifs_sb; + bool old_set = (cifs_sb_flags(old) & CIFS_MOUNT_USE_PREFIX_PATH) && + old->prepath; + bool new_set = (cifs_sb_flags(new) & CIFS_MOUNT_USE_PREFIX_PATH) && + new->prepath; + + if (tcon->origin_fullpath && + dfs_src_pathname_equal(tcon->origin_fullpath, ctx->source)) + return 1; + + if (old_set && new_set && !strcmp(new->prepath, old->prepath)) + return 1; + else if (!old_set && !new_set) + return 1; + + return 0; +} + +int +cifs_match_super(struct super_block *sb, struct fs_context *fc) +{ + struct cifs_mnt_data *mnt_data = fc->sget_key; + struct smb3_fs_context *ctx; + struct cifs_sb_info *cifs_sb; + struct TCP_Server_Info *tcp_srv; + struct cifs_ses *ses; + struct cifs_tcon *tcon; + struct tcon_link *tlink; + int rc = 0; + + spin_lock(&cifs_tcp_ses_lock); + cifs_sb = CIFS_SB(sb); + + /* We do not want to use a superblock that has been shutdown */ + if (cifs_forced_shutdown(cifs_sb)) { + spin_unlock(&cifs_tcp_ses_lock); + return 0; + } + + tlink = cifs_get_tlink(cifs_sb_master_tlink(cifs_sb)); + if (IS_ERR_OR_NULL(tlink)) { + pr_warn_once("%s: skip super matching due to bad tlink(%p)\n", + __func__, tlink); + spin_unlock(&cifs_tcp_ses_lock); + return 0; + } + tcon = tlink_tcon(tlink); + ses = tcon->ses; + tcp_srv = ses->server; + + ctx = mnt_data->ctx; + + spin_lock(&tcp_srv->srv_lock); + spin_lock(&ses->ses_lock); + spin_lock(&ses->chan_lock); + spin_lock(&tcon->tc_lock); + if (!match_server(tcp_srv, ctx, true) || + !match_session(ses, ctx, true) || + !match_tcon(tcon, ctx) || + !match_prepath(sb, tcon, mnt_data)) { + rc = 0; + goto out; + } + + rc = compare_mount_options(sb, mnt_data); +out: + spin_unlock(&tcon->tc_lock); + spin_unlock(&ses->chan_lock); + spin_unlock(&ses->ses_lock); + spin_unlock(&tcp_srv->srv_lock); + + spin_unlock(&cifs_tcp_ses_lock); + cifs_put_tlink(tlink); + return rc; +} + +#ifdef CONFIG_DEBUG_LOCK_ALLOC +static struct lock_class_key cifs_key[2]; +static struct lock_class_key cifs_slock_key[2]; + +static inline void +cifs_reclassify_socket4(struct socket *sock) +{ + struct sock *sk = sock->sk; + + BUG_ON(!sock_allow_reclassification(sk)); + sock_lock_init_class_and_name(sk, "slock-AF_INET-CIFS", + &cifs_slock_key[0], "sk_lock-AF_INET-CIFS", &cifs_key[0]); +} + +static inline void +cifs_reclassify_socket6(struct socket *sock) +{ + struct sock *sk = sock->sk; + + BUG_ON(!sock_allow_reclassification(sk)); + sock_lock_init_class_and_name(sk, "slock-AF_INET6-CIFS", + &cifs_slock_key[1], "sk_lock-AF_INET6-CIFS", &cifs_key[1]); +} +#else +static inline void +cifs_reclassify_socket4(struct socket *sock) +{ +} + +static inline void +cifs_reclassify_socket6(struct socket *sock) +{ +} +#endif + +/* See RFC1001 section 14 on representation of Netbios names */ +static void rfc1002mangle(char *target, char *source, unsigned int length) +{ + unsigned int i, j; + + for (i = 0, j = 0; i < (length); i++) { + /* mask a nibble at a time and encode */ + target[j] = 'A' + (0x0F & (source[i] >> 4)); + target[j+1] = 'A' + (0x0F & source[i]); + j += 2; + } + +} + +static int +bind_socket(struct TCP_Server_Info *server) +{ + int rc = 0; + + if (server->srcaddr.ss_family != AF_UNSPEC) { + /* Bind to the specified local IP address */ + struct socket *socket = server->ssocket; + + rc = kernel_bind(socket, + (struct sockaddr_unsized *) &server->srcaddr, + sizeof(server->srcaddr)); + if (rc < 0) { + struct sockaddr_in *saddr4; + struct sockaddr_in6 *saddr6; + + saddr4 = (struct sockaddr_in *)&server->srcaddr; + saddr6 = (struct sockaddr_in6 *)&server->srcaddr; + if (saddr6->sin6_family == AF_INET6) + cifs_server_dbg(VFS, "Failed to bind to: %pI6c, error: %d\n", + &saddr6->sin6_addr, rc); + else + cifs_server_dbg(VFS, "Failed to bind to: %pI4, error: %d\n", + &saddr4->sin_addr.s_addr, rc); + } + } + return rc; +} + +static int +smb_recv_kvec(struct TCP_Server_Info *server, struct msghdr *msg, size_t *recv) +{ + int rc = 0; + int retries = 0; + int msg_flags = server->noblocksnd ? MSG_DONTWAIT : 0; + + *recv = 0; + + while (msg_data_left(msg)) { + rc = sock_recvmsg(server->ssocket, msg, msg_flags); + if (rc == -EAGAIN) { + retries++; + if (retries >= 14 || + (!server->noblocksnd && (retries > 2))) { + cifs_server_dbg(VFS, "sends on sock %p stuck for 15 seconds\n", + server->ssocket); + return -EAGAIN; + } + msleep(1 << retries); + continue; + } + + if (rc < 0) + return rc; + + if (rc == 0) { + cifs_dbg(FYI, "Received no data (TCP RST)\n"); + return -ECONNABORTED; + } + + /* recv was at least partially successful */ + *recv += rc; + retries = 0; /* in case we get ENOSPC on the next send */ + } + return 0; +} + +static int +ip_rfc1001_connect(struct TCP_Server_Info *server) +{ + int rc = 0; + /* + * some servers require RFC1001 sessinit before sending + * negprot - BB check reconnection in case where second + * sessinit is sent but no second negprot + */ + struct rfc1002_session_packet req = {}; + struct rfc1002_session_packet resp = {}; + struct msghdr msg = {}; + struct kvec iov = {}; + unsigned int len; + size_t sent; + size_t recv; + + req.trailer.session_req.called_len = sizeof(req.trailer.session_req.called_name); + + if (server->server_RFC1001_name[0] != 0) + rfc1002mangle(req.trailer.session_req.called_name, + server->server_RFC1001_name, + RFC1001_NAME_LEN_WITH_NULL); + else + rfc1002mangle(req.trailer.session_req.called_name, + DEFAULT_CIFS_CALLED_NAME, + RFC1001_NAME_LEN_WITH_NULL); + + req.trailer.session_req.calling_len = sizeof(req.trailer.session_req.calling_name); + + /* calling name ends in null (byte 16) from old smb convention */ + if (server->workstation_RFC1001_name[0] != 0) + rfc1002mangle(req.trailer.session_req.calling_name, + server->workstation_RFC1001_name, + RFC1001_NAME_LEN_WITH_NULL); + else + rfc1002mangle(req.trailer.session_req.calling_name, + "LINUX_CIFS_CLNT", + RFC1001_NAME_LEN_WITH_NULL); + + /* + * As per rfc1002, @len must be the number of bytes that follows the + * length field of a rfc1002 session request payload. + */ + len = sizeof(req.trailer.session_req); + req.type = RFC1002_SESSION_REQUEST; + req.flags = 0; + req.length = cpu_to_be16(len); + len += offsetof(typeof(req), trailer.session_req); + iov.iov_base = &req; + iov.iov_len = len; + iov_iter_kvec(&msg.msg_iter, ITER_SOURCE, &iov, 1, len); + rc = smb_send_kvec(server, &msg, &sent); + if (rc < 0 || len != sent) + return (rc == -EINTR || rc == -EAGAIN) ? rc : -ECONNABORTED; + + /* + * RFC1001 layer in at least one server requires very short break before + * negprot presumably because not expecting negprot to follow so fast. + * For example DOS SMB servers cannot process negprot if it was received + * before the server sent response for SESSION_REQUEST packet. So, wait + * for the response, read it and parse it as it can contain useful error + * information (e.g. specified server name was incorrect). For example + * even the latest Windows Server 2022 SMB1 server over port 139 send + * error if its server name was in SESSION_REQUEST packet incorrect. + * Nowadays usage of port 139 is not common, so waiting for reply here + * does not slowing down mounting of common case (over port 445). + */ + len = offsetof(typeof(resp), trailer); + iov.iov_base = &resp; + iov.iov_len = len; + iov_iter_kvec(&msg.msg_iter, ITER_DEST, &iov, 1, len); + rc = smb_recv_kvec(server, &msg, &recv); + if (rc < 0 || recv != len) + return (rc == -EINTR || rc == -EAGAIN) ? rc : -ECONNABORTED; + + switch (resp.type) { + case RFC1002_POSITIVE_SESSION_RESPONSE: + if (be16_to_cpu(resp.length) != 0) { + cifs_dbg(VFS, "RFC 1002 positive session response but with invalid non-zero length %u\n", + be16_to_cpu(resp.length)); + return smb_EIO(smb_eio_trace_rx_pos_sess_resp); + } + cifs_dbg(FYI, "RFC 1002 positive session response"); + break; + case RFC1002_NEGATIVE_SESSION_RESPONSE: + /* Read RFC1002 response error code and convert it to errno in rc */ + len = sizeof(resp.trailer.neg_ses_resp_error_code); + iov.iov_base = &resp.trailer.neg_ses_resp_error_code; + iov.iov_len = len; + iov_iter_kvec(&msg.msg_iter, ITER_DEST, &iov, 1, len); + if (be16_to_cpu(resp.length) == len && + smb_recv_kvec(server, &msg, &recv) == 0 && + recv == len) { + cifs_dbg(VFS, "RFC 1002 negative session response with error 0x%x\n", + resp.trailer.neg_ses_resp_error_code); + switch (resp.trailer.neg_ses_resp_error_code) { + case RFC1002_NOT_LISTENING_CALLED: + /* server does not listen for specified server name */ + fallthrough; + case RFC1002_NOT_PRESENT: + /* server name is incorrect */ + rc = -ENOENT; + cifs_dbg(VFS, "Server rejected NetBIOS servername %.15s\n", + server->server_RFC1001_name[0] ? + server->server_RFC1001_name : + DEFAULT_CIFS_CALLED_NAME); + cifs_dbg(VFS, "Specify correct NetBIOS servername in source path or with -o servern= option\n"); + break; + case RFC1002_NOT_LISTENING_CALLING: + /* client name was not accepted by server */ + rc = -EACCES; + cifs_dbg(VFS, "Server rejected NetBIOS clientname %.15s\n", + server->workstation_RFC1001_name[0] ? + server->workstation_RFC1001_name : + "LINUX_CIFS_CLNT"); + cifs_dbg(VFS, "Specify correct NetBIOS clientname with -o netbiosname= option\n"); + break; + case RFC1002_INSUFFICIENT_RESOURCE: + /* remote server resource error */ + smb_EIO(smb_eio_trace_rx_insuff_res); + rc = -EREMOTEIO; + break; + case RFC1002_UNSPECIFIED_ERROR: + default: + /* other/unknown error */ + rc = smb_EIO(smb_eio_trace_rx_unspec_error); + break; + } + } else { + cifs_dbg(VFS, "RFC 1002 negative session response\n"); + rc = smb_EIO(smb_eio_trace_rx_neg_sess_resp); + } + return rc; + case RFC1002_RETARGET_SESSION_RESPONSE: + cifs_dbg(VFS, "RFC 1002 retarget session response\n"); + if (be16_to_cpu(resp.length) == sizeof(resp.trailer.retarget_resp)) { + len = sizeof(resp.trailer.retarget_resp); + iov.iov_base = &resp.trailer.retarget_resp; + iov.iov_len = len; + iov_iter_kvec(&msg.msg_iter, ITER_DEST, &iov, 1, len); + if (smb_recv_kvec(server, &msg, &recv) == 0 && recv == len) { + cifs_dbg(VFS, "Server wants to redirect connection\n"); + cifs_dbg(VFS, "Remount with options -o ip=%pI4,port=%u\n", + &resp.trailer.retarget_resp.retarget_ip_addr, + be16_to_cpu(resp.trailer.retarget_resp.port)); + } + } + cifs_dbg(VFS, "Closing connection\n"); + /* FIXME: Should we automatically redirect to new retarget_resp server? */ + return -EMULTIHOP; + default: + cifs_dbg(VFS, "RFC 1002 unknown response type 0x%x\n", resp.type); + return smb_EIO1(smb_eio_trace_rx_unknown_resp, resp.type); + } + + server->with_rfc1001 = true; + return 0; +} + +static int +generic_ip_connect(struct TCP_Server_Info *server) +{ + struct sockaddr *saddr; + struct socket *socket; + int slen, sfamily; + __be16 sport; + int rc = 0; + + saddr = (struct sockaddr *) &server->dstaddr; + + if (server->dstaddr.ss_family == AF_INET6) { + struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)&server->dstaddr; + + sport = ipv6->sin6_port; + slen = sizeof(struct sockaddr_in6); + sfamily = AF_INET6; + cifs_dbg(FYI, "%s: connecting to [%pI6]:%d\n", __func__, &ipv6->sin6_addr, + ntohs(sport)); + } else { + struct sockaddr_in *ipv4 = (struct sockaddr_in *)&server->dstaddr; + + sport = ipv4->sin_port; + slen = sizeof(struct sockaddr_in); + sfamily = AF_INET; + cifs_dbg(FYI, "%s: connecting to %pI4:%d\n", __func__, &ipv4->sin_addr, + ntohs(sport)); + } + + if (server->ssocket) { + socket = server->ssocket; + } else { + struct net *net = cifs_net_ns(server); + struct sock *sk; + + rc = sock_create_kern(net, sfamily, SOCK_STREAM, + IPPROTO_TCP, &server->ssocket); + if (rc < 0) { + cifs_server_dbg(VFS, "Error %d creating socket\n", rc); + return rc; + } + + sk = server->ssocket->sk; + sk_net_refcnt_upgrade(sk); + + /* BB other socket options to set KEEPALIVE, NODELAY? */ + cifs_dbg(FYI, "Socket created\n"); + socket = server->ssocket; + socket->sk->sk_allocation = GFP_NOFS; + socket->sk->sk_use_task_frag = false; + if (sfamily == AF_INET6) + cifs_reclassify_socket6(socket); + else + cifs_reclassify_socket4(socket); + } + + rc = bind_socket(server); + if (rc < 0) + return rc; + + /* + * Eventually check for other socket options to change from + * the default. sock_setsockopt not used because it expects + * user space buffer + */ + socket->sk->sk_rcvtimeo = 7 * HZ; + socket->sk->sk_sndtimeo = 5 * HZ; + + /* make the bufsizes depend on wsize/rsize and max requests */ + if (server->noautotune) { + if (socket->sk->sk_sndbuf < (200 * 1024)) + socket->sk->sk_sndbuf = 200 * 1024; + if (socket->sk->sk_rcvbuf < (140 * 1024)) + socket->sk->sk_rcvbuf = 140 * 1024; + } + + if (server->tcp_nodelay) + tcp_sock_set_nodelay(socket->sk); + + cifs_dbg(FYI, "sndbuf %d rcvbuf %d rcvtimeo 0x%lx\n", + socket->sk->sk_sndbuf, + socket->sk->sk_rcvbuf, socket->sk->sk_rcvtimeo); + + rc = kernel_connect(socket, (struct sockaddr_unsized *)saddr, slen, + server->noblockcnt ? O_NONBLOCK : 0); + /* + * When mounting SMB root file systems, we do not want to block in + * connect. Otherwise bail out and then let cifs_reconnect() perform + * reconnect failover - if possible. + */ + if (server->noblockcnt && rc == -EINPROGRESS) + rc = 0; + if (rc < 0) { + cifs_dbg(FYI, "Error %d connecting to server\n", rc); + trace_smb3_connect_err(server->hostname, server->conn_id, &server->dstaddr, rc); + sock_release(socket); + server->ssocket = NULL; + return rc; + } + trace_smb3_connect_done(server->hostname, server->conn_id, &server->dstaddr); + + /* + * Establish RFC1001 NetBIOS session when it was explicitly requested + * by mount option -o nbsessinit, or when connecting to default RFC1001 + * server port (139) and it was not explicitly disabled by mount option + * -o nonbsessinit. + */ + if (server->with_rfc1001 || + server->rfc1001_sessinit == 1 || + (server->rfc1001_sessinit == -1 && sport == htons(RFC1001_PORT))) + rc = ip_rfc1001_connect(server); + + return rc; +} + +static int +ip_connect(struct TCP_Server_Info *server) +{ + __be16 *sport; + struct sockaddr_in6 *addr6 = (struct sockaddr_in6 *)&server->dstaddr; + struct sockaddr_in *addr = (struct sockaddr_in *)&server->dstaddr; + + if (server->dstaddr.ss_family == AF_INET6) + sport = &addr6->sin6_port; + else + sport = &addr->sin_port; + + if (*sport == 0) { + int rc; + + /* try with 445 port at first */ + *sport = htons(CIFS_PORT); + + rc = generic_ip_connect(server); + if (rc >= 0) + return rc; + + /* if it failed, try with 139 port */ + *sport = htons(RFC1001_PORT); + } + + return generic_ip_connect(server); +} + +int cifs_setup_cifs_sb(struct cifs_sb_info *cifs_sb) +{ + struct smb3_fs_context *ctx = cifs_sb->ctx; + unsigned int sbflags; + int rc = 0; + + INIT_DELAYED_WORK(&cifs_sb->prune_tlinks, cifs_prune_tlinks); + INIT_LIST_HEAD(&cifs_sb->tcon_sb_link); + + spin_lock_init(&cifs_sb->tlink_tree_lock); + cifs_sb->tlink_tree = RB_ROOT; + atomic_set(&cifs_sb->outstanding_rreq, 0); + + cifs_dbg(FYI, "file mode: %04ho dir mode: %04ho\n", + ctx->file_mode, ctx->dir_mode); + + /* this is needed for ASCII cp to Unicode converts */ + if (ctx->iocharset == NULL) { + /* load_nls_default cannot return null */ + cifs_sb->local_nls = load_nls_default(); + } else { + cifs_sb->local_nls = load_nls(ctx->iocharset); + if (cifs_sb->local_nls == NULL) { + cifs_dbg(VFS, "CIFS mount error: iocharset %s not found\n", + ctx->iocharset); + return -ELIBACC; + } + } + ctx->local_nls = cifs_sb->local_nls; + + sbflags = smb3_update_mnt_flags(cifs_sb); + + if (ctx->direct_io) + cifs_dbg(FYI, "mounting share using direct i/o\n"); + if (ctx->cache_ro) { + cifs_dbg(VFS, "mounting share with read only caching. Ensure that the share will not be modified while in use.\n"); + sbflags |= CIFS_MOUNT_RO_CACHE; + } else if (ctx->cache_rw) { + cifs_dbg(VFS, "mounting share in single client RW caching mode. Ensure that no other systems will be accessing the share.\n"); + sbflags |= CIFS_MOUNT_RO_CACHE | CIFS_MOUNT_RW_CACHE; + } + + if ((ctx->cifs_acl) && (ctx->dynperm)) + cifs_dbg(VFS, "mount option dynperm ignored if cifsacl mount option supported\n"); + + if (ctx->prepath) { + cifs_sb->prepath = kstrdup(ctx->prepath, GFP_KERNEL); + if (cifs_sb->prepath == NULL) + rc = -ENOMEM; + else + sbflags |= CIFS_MOUNT_USE_PREFIX_PATH; + } + + atomic_set(&cifs_sb->mnt_cifs_flags, sbflags); + return rc; +} + +/* Release all succeed connections */ +void cifs_mount_put_conns(struct cifs_mount_ctx *mnt_ctx) +{ + struct cifs_sb_info *cifs_sb = mnt_ctx->cifs_sb; + int rc = 0; + + if (mnt_ctx->tcon) + cifs_put_tcon(mnt_ctx->tcon, netfs_trace_tcon_ref_put_mnt_ctx); + else if (mnt_ctx->ses) + cifs_put_smb_ses(mnt_ctx->ses); + else if (mnt_ctx->server) + cifs_put_tcp_session(mnt_ctx->server, 0); + mnt_ctx->ses = NULL; + mnt_ctx->tcon = NULL; + mnt_ctx->server = NULL; + atomic_andnot(CIFS_MOUNT_POSIX_PATHS, &cifs_sb->mnt_cifs_flags); + free_xid(mnt_ctx->xid); +} + +int cifs_mount_get_session(struct cifs_mount_ctx *mnt_ctx) +{ + struct TCP_Server_Info *server = NULL; + struct smb3_fs_context *ctx; + struct cifs_ses *ses = NULL; + unsigned int xid; + int rc = 0; + + xid = get_xid(); + + if (WARN_ON_ONCE(!mnt_ctx || !mnt_ctx->fs_ctx)) { + rc = -EINVAL; + goto out; + } + ctx = mnt_ctx->fs_ctx; + + /* get a reference to a tcp session */ + server = cifs_get_tcp_session(ctx, NULL); + if (IS_ERR(server)) { + rc = PTR_ERR(server); + server = NULL; + goto out; + } + + /* get a reference to a SMB session */ + ses = cifs_get_smb_ses(server, ctx); + if (IS_ERR(ses)) { + rc = PTR_ERR(ses); + ses = NULL; + goto out; + } + + if ((ctx->persistent == true) && (!(ses->server->capabilities & + SMB2_GLOBAL_CAP_PERSISTENT_HANDLES))) { + cifs_server_dbg(VFS, "persistent handles not supported by server\n"); + rc = -EOPNOTSUPP; + } + +out: + mnt_ctx->xid = xid; + mnt_ctx->server = server; + mnt_ctx->ses = ses; + mnt_ctx->tcon = NULL; + + return rc; +} + +int cifs_mount_get_tcon(struct cifs_mount_ctx *mnt_ctx) +{ + struct TCP_Server_Info *server; + struct cifs_tcon *tcon = NULL; + struct cifs_sb_info *cifs_sb; + struct smb3_fs_context *ctx; + unsigned int sbflags; + int rc = 0; + + if (WARN_ON_ONCE(!mnt_ctx)) + return -EINVAL; + if (WARN_ON_ONCE(!mnt_ctx->server || !mnt_ctx->ses || + !mnt_ctx->fs_ctx || !mnt_ctx->cifs_sb)) { + mnt_ctx->tcon = NULL; + return -EINVAL; + } + server = mnt_ctx->server; + ctx = mnt_ctx->fs_ctx; + cifs_sb = mnt_ctx->cifs_sb; + + /* search for existing tcon to this server share */ + tcon = cifs_get_tcon(mnt_ctx->ses, ctx); + if (IS_ERR(tcon)) { + rc = PTR_ERR(tcon); + tcon = NULL; + goto out; + } + + /* + * if new SMB3.11 POSIX extensions are supported, do not change anything in the + * path (i.e., do not remap / and \ and do not map any special characters) + */ + if (tcon->posix_extensions) { + atomic_or(CIFS_MOUNT_POSIX_PATHS, &cifs_sb->mnt_cifs_flags); + atomic_andnot(CIFS_MOUNT_MAP_SFM_CHR | + CIFS_MOUNT_MAP_SPECIAL_CHR, + &cifs_sb->mnt_cifs_flags); + } + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + /* tell server which Unix caps we support */ + if (cap_unix(tcon->ses)) { + /* + * reset of caps checks mount to see if unix extensions disabled + * for just this mount. + */ + reset_cifs_unix_caps(mnt_ctx->xid, tcon, cifs_sb, ctx); + spin_lock(&tcon->ses->server->srv_lock); + if ((tcon->ses->server->tcpStatus == CifsNeedReconnect) && + (le64_to_cpu(tcon->fsUnixInfo.Capability) & + CIFS_UNIX_TRANSPORT_ENCRYPTION_MANDATORY_CAP)) { + spin_unlock(&tcon->ses->server->srv_lock); + rc = -EACCES; + goto out; + } + spin_unlock(&tcon->ses->server->srv_lock); + } else +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + tcon->unix_ext = 0; /* server does not support them */ + + sbflags = cifs_sb_flags(cifs_sb); + /* do not care if a following call succeed - informational */ + if (!tcon->pipe && server->ops->qfs_tcon) { + server->ops->qfs_tcon(mnt_ctx->xid, tcon, cifs_sb); + if (sbflags & CIFS_MOUNT_RO_CACHE) { + if (tcon->fsDevInfo.DeviceCharacteristics & + cpu_to_le32(FILE_READ_ONLY_DEVICE)) + cifs_dbg(VFS, "mounted to read only share\n"); + else if (!(sbflags & CIFS_MOUNT_RW_CACHE)) + cifs_dbg(VFS, "read only mount of RW share\n"); + /* no need to log a RW mount of a typical RW share */ + } + } + + cifs_negotiate_iosize(server, cifs_sb->ctx, tcon); + /* + * The cookie is initialized from volume info returned above. + * Inside cifs_fscache_get_super_cookie it checks + * that we do not get super cookie twice. + */ + if (sbflags & CIFS_MOUNT_FSCACHE) + cifs_fscache_get_super_cookie(tcon); + +out: + mnt_ctx->tcon = tcon; + return rc; +} + +static int mount_setup_tlink(struct cifs_sb_info *cifs_sb, struct cifs_ses *ses, + struct cifs_tcon *tcon) +{ + struct tcon_link *tlink; + + /* hang the tcon off of the superblock */ + tlink = kzalloc_obj(*tlink); + if (tlink == NULL) + return -ENOMEM; + + tlink->tl_uid = ses->linux_uid; + tlink->tl_tcon = tcon; + tlink->tl_time = jiffies; + set_bit(TCON_LINK_MASTER, &tlink->tl_flags); + set_bit(TCON_LINK_IN_TREE, &tlink->tl_flags); + + cifs_sb->master_tlink = tlink; + spin_lock(&cifs_sb->tlink_tree_lock); + tlink_rb_insert(&cifs_sb->tlink_tree, tlink); + spin_unlock(&cifs_sb->tlink_tree_lock); + + spin_lock(&tcon->sb_list_lock); + list_add(&cifs_sb->tcon_sb_link, &tcon->cifs_sb_list); + spin_unlock(&tcon->sb_list_lock); + + queue_delayed_work(cifsiod_wq, &cifs_sb->prune_tlinks, + TLINK_IDLE_EXPIRE); + return 0; +} + +static int +cifs_are_all_path_components_accessible(struct TCP_Server_Info *server, + unsigned int xid, + struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, + char *full_path, + int added_treename) +{ + int rc; + char *s; + char sep, tmp; + int skip = added_treename ? 1 : 0; + + sep = CIFS_DIR_SEP(cifs_sb); + s = full_path; + + rc = server->ops->is_path_accessible(xid, tcon, cifs_sb, ""); + while (rc == 0) { + /* skip separators */ + while (*s == sep) + s++; + if (!*s) + break; + /* next separator */ + while (*s && *s != sep) + s++; + /* + * if the treename is added, we then have to skip the first + * part within the separators + */ + if (skip) { + skip = 0; + continue; + } + /* + * temporarily null-terminate the path at the end of + * the current component + */ + tmp = *s; + *s = 0; + rc = server->ops->is_path_accessible(xid, tcon, cifs_sb, + full_path); + *s = tmp; + } + return rc; +} + +/* + * Check if path is remote (i.e. a DFS share). + * + * Return -EREMOTE if it is, otherwise 0 or -errno. + */ +int cifs_is_path_remote(struct cifs_mount_ctx *mnt_ctx) +{ + int rc; + struct cifs_sb_info *cifs_sb = mnt_ctx->cifs_sb; + struct TCP_Server_Info *server = mnt_ctx->server; + unsigned int xid = mnt_ctx->xid; + struct cifs_tcon *tcon = mnt_ctx->tcon; + struct smb3_fs_context *ctx = mnt_ctx->fs_ctx; + char *full_path; + + if (!server->ops->is_path_accessible) + return -EOPNOTSUPP; + + /* + * cifs_build_path_to_root works only when we have a valid tcon + */ + full_path = cifs_build_path_to_root(ctx, cifs_sb, tcon, + tcon->Flags & SMB_SHARE_IS_IN_DFS); + if (full_path == NULL) + return -ENOMEM; + + cifs_dbg(FYI, "%s: full_path: %s\n", __func__, full_path); + + rc = server->ops->is_path_accessible(xid, tcon, cifs_sb, + full_path); + if (rc != 0 && rc != -EREMOTE) + goto out; + + if (rc != -EREMOTE) { + rc = cifs_are_all_path_components_accessible(server, xid, tcon, + cifs_sb, full_path, tcon->Flags & SMB_SHARE_IS_IN_DFS); + if (rc != 0) { + cifs_server_dbg(VFS, "cannot query dirs between root and final path, enabling CIFS_MOUNT_USE_PREFIX_PATH\n"); + atomic_or(CIFS_MOUNT_USE_PREFIX_PATH, + &cifs_sb->mnt_cifs_flags); + rc = 0; + } + } + +out: + kfree(full_path); + return rc; +} + +static struct mchan_mount * +mchan_mount_alloc(struct cifs_ses *ses) +{ + struct mchan_mount *mchan_mount; + + mchan_mount = kzalloc_obj(*mchan_mount); + if (!mchan_mount) + return ERR_PTR(-ENOMEM); + + INIT_WORK(&mchan_mount->work, mchan_mount_work_fn); + + spin_lock(&cifs_tcp_ses_lock); + cifs_smb_ses_inc_refcount(ses); + spin_unlock(&cifs_tcp_ses_lock); + mchan_mount->ses = ses; + + return mchan_mount; +} + +static void +mchan_mount_free(struct mchan_mount *mchan_mount) +{ + cifs_put_smb_ses(mchan_mount->ses); + kfree(mchan_mount); +} + +static void +mchan_mount_work_fn(struct work_struct *work) +{ + struct mchan_mount *mchan_mount = container_of(work, struct mchan_mount, work); + + smb3_update_ses_channels(mchan_mount->ses, + mchan_mount->ses->server, + false /* from_reconnect */, + false /* disable_mchan */); + + mchan_mount_free(mchan_mount); +} + +#ifdef CONFIG_CIFS_DFS_UPCALL +int cifs_mount(struct cifs_sb_info *cifs_sb, struct smb3_fs_context *ctx) +{ + struct cifs_mount_ctx mnt_ctx = { .cifs_sb = cifs_sb, .fs_ctx = ctx, }; + struct mchan_mount *mchan_mount = NULL; + int rc; + + rc = dfs_mount_share(&mnt_ctx); + if (rc) + goto error; + + if (ctx->multichannel) { + mchan_mount = mchan_mount_alloc(mnt_ctx.ses); + if (IS_ERR(mchan_mount)) { + rc = PTR_ERR(mchan_mount); + goto error; + } + } + + if (!ctx->dfs_conn) + goto out; + + /* + * After reconnecting to a different server, unique ids won't match anymore, so we disable + * serverino. This prevents dentry revalidation to think the dentry are stale (ESTALE). + */ + cifs_autodisable_serverino(cifs_sb, "DFS failover may potentially connect to a different server, inode numbers won't match anymore", 0); + /* + * Force the use of prefix path to support failover on DFS paths that resolve to targets + * that have different prefix paths. + */ + atomic_or(CIFS_MOUNT_USE_PREFIX_PATH, &cifs_sb->mnt_cifs_flags); + kfree(cifs_sb->prepath); + cifs_sb->prepath = ctx->prepath; + ctx->prepath = NULL; + +out: + rc = mount_setup_tlink(cifs_sb, mnt_ctx.ses, mnt_ctx.tcon); + if (rc) + goto error; + + if (ctx->multichannel) + queue_work(cifsiod_wq, &mchan_mount->work); + + free_xid(mnt_ctx.xid); + return rc; + +error: + if (ctx->multichannel && !IS_ERR_OR_NULL(mchan_mount)) + mchan_mount_free(mchan_mount); + cifs_mount_put_conns(&mnt_ctx); + return rc; +} +#else +int cifs_mount(struct cifs_sb_info *cifs_sb, struct smb3_fs_context *ctx) +{ + int rc = 0; + struct cifs_mount_ctx mnt_ctx = { .cifs_sb = cifs_sb, .fs_ctx = ctx, }; + struct mchan_mount *mchan_mount = NULL; + + rc = cifs_mount_get_session(&mnt_ctx); + if (rc) + goto error; + + rc = cifs_mount_get_tcon(&mnt_ctx); + if (!rc) { + /* + * Prevent superblock from being created with any missing + * connections. + */ + if (WARN_ON(!mnt_ctx.server)) + rc = -EHOSTDOWN; + else if (WARN_ON(!mnt_ctx.ses)) + rc = -EACCES; + else if (WARN_ON(!mnt_ctx.tcon)) + rc = -ENOENT; + } + if (rc) + goto error; + + rc = cifs_is_path_remote(&mnt_ctx); + if (rc == -EREMOTE) + rc = -EOPNOTSUPP; + if (rc) + goto error; + + if (ctx->multichannel) { + mchan_mount = mchan_mount_alloc(mnt_ctx.ses); + if (IS_ERR(mchan_mount)) { + rc = PTR_ERR(mchan_mount); + goto error; + } + } + + rc = mount_setup_tlink(cifs_sb, mnt_ctx.ses, mnt_ctx.tcon); + if (rc) + goto error; + + if (ctx->multichannel) + queue_work(cifsiod_wq, &mchan_mount->work); + + free_xid(mnt_ctx.xid); + return rc; + +error: + if (ctx->multichannel && !IS_ERR_OR_NULL(mchan_mount)) + mchan_mount_free(mchan_mount); + cifs_mount_put_conns(&mnt_ctx); + return rc; +} +#endif + +static void delayed_free(struct rcu_head *p) +{ + struct cifs_sb_info *cifs_sb = container_of(p, struct cifs_sb_info, rcu); + + unload_nls(cifs_sb->local_nls); + smb3_cleanup_fs_context(cifs_sb->ctx); + kfree(cifs_sb); +} + +void +cifs_umount(struct cifs_sb_info *cifs_sb) +{ + struct rb_root *root = &cifs_sb->tlink_tree; + struct rb_node *node; + struct tcon_link *tlink; + struct cifs_tcon *tcon = NULL; + + cancel_delayed_work_sync(&cifs_sb->prune_tlinks); + + if (cifs_sb->master_tlink) { + tcon = cifs_sb->master_tlink->tl_tcon; + if (tcon) { + spin_lock(&tcon->sb_list_lock); + list_del_init(&cifs_sb->tcon_sb_link); + spin_unlock(&tcon->sb_list_lock); + } + } + + spin_lock(&cifs_sb->tlink_tree_lock); + while ((node = rb_first(root))) { + tlink = rb_entry(node, struct tcon_link, tl_rbnode); + cifs_get_tlink(tlink); + clear_bit(TCON_LINK_IN_TREE, &tlink->tl_flags); + rb_erase(node, root); + + spin_unlock(&cifs_sb->tlink_tree_lock); + cifs_put_tlink(tlink); + spin_lock(&cifs_sb->tlink_tree_lock); + } + spin_unlock(&cifs_sb->tlink_tree_lock); + + flush_workqueue(serverclose_wq); + flush_workqueue(fileinfo_put_wq); + + kfree(cifs_sb->prepath); + call_rcu(&cifs_sb->rcu, delayed_free); +} + +int +cifs_negotiate_protocol(const unsigned int xid, struct cifs_ses *ses, + struct TCP_Server_Info *server) +{ + bool in_retry = false; + int rc = 0; + + if (!server->ops->need_neg || !server->ops->negotiate) + return -ENOSYS; + +retry: + /* only send once per connect */ + spin_lock(&server->srv_lock); + if (server->tcpStatus != CifsGood && + server->tcpStatus != CifsNew && + server->tcpStatus != CifsNeedNegotiate) { + spin_unlock(&server->srv_lock); + return -EHOSTDOWN; + } + + if (!server->ops->need_neg(server) && + server->tcpStatus == CifsGood) { + spin_unlock(&server->srv_lock); + return 0; + } + + server->tcpStatus = CifsInNegotiate; + server->neg_start = jiffies; + spin_unlock(&server->srv_lock); + + rc = server->ops->negotiate(xid, ses, server); + if (rc == -EAGAIN) { + /* Allow one retry attempt */ + if (!in_retry) { + in_retry = true; + goto retry; + } + rc = -EHOSTDOWN; + } + if (rc == 0) { + spin_lock(&server->srv_lock); + if (server->tcpStatus == CifsInNegotiate) + server->tcpStatus = CifsGood; + else + rc = -EHOSTDOWN; + spin_unlock(&server->srv_lock); + } else { + spin_lock(&server->srv_lock); + if (server->tcpStatus == CifsInNegotiate) + server->tcpStatus = CifsNeedNegotiate; + spin_unlock(&server->srv_lock); + } + + return rc; +} + +int +cifs_setup_session(const unsigned int xid, struct cifs_ses *ses, + struct TCP_Server_Info *server, + struct nls_table *nls_info) +{ + int rc = 0; + struct TCP_Server_Info *pserver = SERVER_IS_CHAN(server) ? server->primary_server : server; + struct sockaddr_in6 *addr6 = (struct sockaddr_in6 *)&pserver->dstaddr; + struct sockaddr_in *addr = (struct sockaddr_in *)&pserver->dstaddr; + bool is_binding = false; + bool new_ses; + + spin_lock(&ses->ses_lock); + new_ses = ses->ses_status == SES_NEW; + cifs_dbg(FYI, "%s: channel connect bitmap: 0x%lx\n", + __func__, ses->chans_need_reconnect); + + if (ses->ses_status != SES_GOOD && + ses->ses_status != SES_NEW && + ses->ses_status != SES_NEED_RECON) { + spin_unlock(&ses->ses_lock); + return -EHOSTDOWN; + } + + /* only send once per connect */ + spin_lock(&ses->chan_lock); + if (CIFS_ALL_CHANS_GOOD(ses)) { + if (ses->ses_status == SES_NEED_RECON) + ses->ses_status = SES_GOOD; + spin_unlock(&ses->chan_lock); + spin_unlock(&ses->ses_lock); + return 0; + } + + cifs_chan_set_in_reconnect(ses, server); + is_binding = !CIFS_ALL_CHANS_NEED_RECONNECT(ses); + spin_unlock(&ses->chan_lock); + + if (!is_binding) { + ses->ses_status = SES_IN_SETUP; + + /* force iface_list refresh */ + spin_lock(&ses->iface_lock); + ses->iface_last_update = 0; + spin_unlock(&ses->iface_lock); + } + spin_unlock(&ses->ses_lock); + + /* update ses ip_addr only for primary chan */ + if (server == pserver) { + if (server->dstaddr.ss_family == AF_INET6) + scnprintf(ses->ip_addr, sizeof(ses->ip_addr), "%pI6", &addr6->sin6_addr); + else + scnprintf(ses->ip_addr, sizeof(ses->ip_addr), "%pI4", &addr->sin_addr); + } + + if (!is_binding) { + ses->capabilities = server->capabilities; + if (!linuxExtEnabled) + ses->capabilities &= (~server->vals->cap_unix); + + /* + * Check if the server supports specified encoding mode. + * Zero value in vals->cap_unicode indidcates that chosen + * protocol dialect does not support non-UNICODE mode. + */ + if (ses->unicode == 1 && server->vals->cap_unicode != 0 && + !(server->capabilities & server->vals->cap_unicode)) { + cifs_dbg(VFS, "Server does not support mounting in UNICODE mode\n"); + rc = -EOPNOTSUPP; + } else if (ses->unicode == 0 && server->vals->cap_unicode == 0) { + cifs_dbg(VFS, "Server does not support mounting in non-UNICODE mode\n"); + rc = -EOPNOTSUPP; + } else if (ses->unicode == 0) { + /* + * When UNICODE mode was explicitly disabled then + * do not announce client UNICODE capability. + */ + ses->capabilities &= (~server->vals->cap_unicode); + } + + if (ses->auth_key.response) { + cifs_dbg(FYI, "Free previous auth_key.response = %p\n", + ses->auth_key.response); + kfree_sensitive(ses->auth_key.response); + ses->auth_key.response = NULL; + ses->auth_key.len = 0; + } + } + + cifs_dbg(FYI, "Security Mode: 0x%x Capabilities: 0x%x TimeAdjust: %d\n", + server->sec_mode, server->capabilities, server->timeAdj); + + if (!rc) { + if (server->ops->sess_setup) + rc = server->ops->sess_setup(xid, ses, server, nls_info); + else + rc = -ENOSYS; + } + + if (rc) { + if (new_ses) { + cifs_server_dbg(VFS, "failed to create a new SMB session with %s: %d\n", + get_security_type_str(ses->sectype), rc); + } + spin_lock(&ses->ses_lock); + if (ses->ses_status == SES_IN_SETUP) + ses->ses_status = SES_NEED_RECON; + spin_lock(&ses->chan_lock); + cifs_chan_clear_in_reconnect(ses, server); + spin_unlock(&ses->chan_lock); + spin_unlock(&ses->ses_lock); + } else { + spin_lock(&ses->ses_lock); + if (ses->ses_status == SES_IN_SETUP) + ses->ses_status = SES_GOOD; + spin_lock(&ses->chan_lock); + cifs_chan_clear_in_reconnect(ses, server); + cifs_chan_clear_need_reconnect(ses, server); + spin_unlock(&ses->chan_lock); + spin_unlock(&ses->ses_lock); + } + + return rc; +} + +static int +cifs_set_vol_auth(struct smb3_fs_context *ctx, struct cifs_ses *ses) +{ + ctx->sectype = ses->sectype; + + /* krb5 is special, since we don't need username or pw */ + if (ctx->sectype == Kerberos) + return 0; + + return cifs_set_cifscreds(ctx, ses); +} + +static struct cifs_tcon * +cifs_construct_tcon(struct cifs_sb_info *cifs_sb, kuid_t fsuid) +{ + int rc; + struct cifs_tcon *master_tcon = cifs_sb_master_tcon(cifs_sb); + struct cifs_ses *ses; + struct cifs_tcon *tcon = NULL; + struct smb3_fs_context *ctx; + char *origin_fullpath = NULL; + + ctx = kzalloc_obj(*ctx); + if (ctx == NULL) + return ERR_PTR(-ENOMEM); + + ctx->local_nls = cifs_sb->local_nls; + ctx->linux_uid = fsuid; + ctx->cred_uid = fsuid; + ctx->UNC = master_tcon->tree_name; + ctx->retry = master_tcon->retry; + ctx->nocase = master_tcon->nocase; + ctx->nohandlecache = master_tcon->nohandlecache; + ctx->local_lease = master_tcon->local_lease; + ctx->no_lease = master_tcon->no_lease; + ctx->resilient = master_tcon->use_resilient; + ctx->persistent = master_tcon->use_persistent; + ctx->handle_timeout = master_tcon->handle_timeout; + ctx->no_linux_ext = !master_tcon->unix_ext; + ctx->linux_ext = master_tcon->posix_extensions; + ctx->sectype = master_tcon->ses->sectype; + ctx->sign = master_tcon->ses->sign; + ctx->seal = master_tcon->seal; + ctx->witness = master_tcon->use_witness; + ctx->dfs_root_ses = master_tcon->ses->dfs_root_ses; + ctx->unicode = master_tcon->ses->unicode; + + rc = cifs_set_vol_auth(ctx, master_tcon->ses); + if (rc) { + tcon = ERR_PTR(rc); + goto out; + } + + /* get a reference for the same TCP session */ + spin_lock(&cifs_tcp_ses_lock); + ++master_tcon->ses->server->srv_count; + spin_unlock(&cifs_tcp_ses_lock); + + ses = cifs_get_smb_ses(master_tcon->ses->server, ctx); + if (IS_ERR(ses)) { + tcon = ERR_CAST(ses); + cifs_put_tcp_session(master_tcon->ses->server, 0); + goto out; + } + +#ifdef CONFIG_CIFS_DFS_UPCALL + spin_lock(&master_tcon->tc_lock); + if (master_tcon->origin_fullpath) { + spin_unlock(&master_tcon->tc_lock); + origin_fullpath = dfs_get_path(cifs_sb, cifs_sb->ctx->source); + if (IS_ERR(origin_fullpath)) { + tcon = ERR_CAST(origin_fullpath); + origin_fullpath = NULL; + cifs_put_smb_ses(ses); + goto out; + } + } else { + spin_unlock(&master_tcon->tc_lock); + } +#endif + + tcon = cifs_get_tcon(ses, ctx); + if (IS_ERR(tcon)) { + cifs_put_smb_ses(ses); + goto out; + } + +#ifdef CONFIG_CIFS_DFS_UPCALL + if (origin_fullpath) { + spin_lock(&tcon->tc_lock); + tcon->origin_fullpath = origin_fullpath; + spin_unlock(&tcon->tc_lock); + origin_fullpath = NULL; + queue_delayed_work(dfscache_wq, &tcon->dfs_cache_work, + dfs_cache_get_ttl() * HZ); + } +#endif + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + if (cap_unix(ses)) + reset_cifs_unix_caps(0, tcon, NULL, ctx); +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + +out: + kfree(ctx->username); + kfree(ctx->domainname); + kfree_sensitive(ctx->password); + kfree(origin_fullpath); + kfree(ctx); + + return tcon; +} + +struct cifs_tcon * +cifs_sb_master_tcon(struct cifs_sb_info *cifs_sb) +{ + return tlink_tcon(cifs_sb_master_tlink(cifs_sb)); +} + +/* find and return a tlink with given uid */ +static struct tcon_link * +tlink_rb_search(struct rb_root *root, kuid_t uid) +{ + struct rb_node *node = root->rb_node; + struct tcon_link *tlink; + + while (node) { + tlink = rb_entry(node, struct tcon_link, tl_rbnode); + + if (uid_gt(tlink->tl_uid, uid)) + node = node->rb_left; + else if (uid_lt(tlink->tl_uid, uid)) + node = node->rb_right; + else + return tlink; + } + return NULL; +} + +/* insert a tcon_link into the tree */ +static void +tlink_rb_insert(struct rb_root *root, struct tcon_link *new_tlink) +{ + struct rb_node **new = &(root->rb_node), *parent = NULL; + struct tcon_link *tlink; + + while (*new) { + tlink = rb_entry(*new, struct tcon_link, tl_rbnode); + parent = *new; + + if (uid_gt(tlink->tl_uid, new_tlink->tl_uid)) + new = &((*new)->rb_left); + else + new = &((*new)->rb_right); + } + + rb_link_node(&new_tlink->tl_rbnode, parent, new); + rb_insert_color(&new_tlink->tl_rbnode, root); +} + +/* + * Find or construct an appropriate tcon given a cifs_sb and the fsuid of the + * current task. + * + * If the superblock doesn't refer to a multiuser mount, then just return + * the master tcon for the mount. + * + * First, search the rbtree for an existing tcon for this fsuid. If one + * exists, then check to see if it's pending construction. If it is then wait + * for construction to complete. Once it's no longer pending, check to see if + * it failed and either return an error or retry construction, depending on + * the timeout. + * + * If one doesn't exist then insert a new tcon_link struct into the tree and + * try to construct a new one. + * + * REMEMBER to call cifs_put_tlink() after successful calls to cifs_sb_tlink, + * to avoid refcount issues + */ +struct tcon_link * +cifs_sb_tlink(struct cifs_sb_info *cifs_sb) +{ + struct tcon_link *tlink, *newtlink; + kuid_t fsuid = current_fsuid(); + int err; + + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MULTIUSER)) + return cifs_get_tlink(cifs_sb_master_tlink(cifs_sb)); + + spin_lock(&cifs_sb->tlink_tree_lock); + tlink = tlink_rb_search(&cifs_sb->tlink_tree, fsuid); + if (tlink) + cifs_get_tlink(tlink); + spin_unlock(&cifs_sb->tlink_tree_lock); + + if (tlink == NULL) { + newtlink = kzalloc_obj(*tlink); + if (newtlink == NULL) + return ERR_PTR(-ENOMEM); + newtlink->tl_uid = fsuid; + newtlink->tl_tcon = ERR_PTR(-EACCES); + set_bit(TCON_LINK_PENDING, &newtlink->tl_flags); + set_bit(TCON_LINK_IN_TREE, &newtlink->tl_flags); + cifs_get_tlink(newtlink); + + spin_lock(&cifs_sb->tlink_tree_lock); + /* was one inserted after previous search? */ + tlink = tlink_rb_search(&cifs_sb->tlink_tree, fsuid); + if (tlink) { + cifs_get_tlink(tlink); + spin_unlock(&cifs_sb->tlink_tree_lock); + kfree(newtlink); + goto wait_for_construction; + } + tlink = newtlink; + tlink_rb_insert(&cifs_sb->tlink_tree, tlink); + spin_unlock(&cifs_sb->tlink_tree_lock); + } else { +wait_for_construction: + err = wait_on_bit(&tlink->tl_flags, TCON_LINK_PENDING, + TASK_INTERRUPTIBLE); + if (err) { + cifs_put_tlink(tlink); + return ERR_PTR(-ERESTARTSYS); + } + + /* if it's good, return it */ + if (!IS_ERR(tlink->tl_tcon)) + return tlink; + + /* return error if we tried this already recently */ + if (time_before(jiffies, tlink->tl_time + TLINK_ERROR_EXPIRE)) { + err = PTR_ERR(tlink->tl_tcon); + cifs_put_tlink(tlink); + return ERR_PTR(err); + } + + if (test_and_set_bit(TCON_LINK_PENDING, &tlink->tl_flags)) + goto wait_for_construction; + } + + tlink->tl_tcon = cifs_construct_tcon(cifs_sb, fsuid); + clear_bit(TCON_LINK_PENDING, &tlink->tl_flags); + wake_up_bit(&tlink->tl_flags, TCON_LINK_PENDING); + + if (IS_ERR(tlink->tl_tcon)) { + err = PTR_ERR(tlink->tl_tcon); + if (err == -ENOKEY) + err = -EACCES; + cifs_put_tlink(tlink); + return ERR_PTR(err); + } + + return tlink; +} + +/* + * periodic workqueue job that scans tcon_tree for a superblock and closes + * out tcons. + */ +static void +cifs_prune_tlinks(struct work_struct *work) +{ + struct cifs_sb_info *cifs_sb = container_of(work, struct cifs_sb_info, + prune_tlinks.work); + struct rb_root *root = &cifs_sb->tlink_tree; + struct rb_node *node; + struct rb_node *tmp; + struct tcon_link *tlink; + + /* + * Because we drop the spinlock in the loop in order to put the tlink + * it's not guarded against removal of links from the tree. The only + * places that remove entries from the tree are this function and + * umounts. Because this function is non-reentrant and is canceled + * before umount can proceed, this is safe. + */ + spin_lock(&cifs_sb->tlink_tree_lock); + node = rb_first(root); + while (node != NULL) { + tmp = node; + node = rb_next(tmp); + tlink = rb_entry(tmp, struct tcon_link, tl_rbnode); + + if (test_bit(TCON_LINK_MASTER, &tlink->tl_flags) || + atomic_read(&tlink->tl_count) != 0 || + time_after(tlink->tl_time + TLINK_IDLE_EXPIRE, jiffies)) + continue; + + cifs_get_tlink(tlink); + clear_bit(TCON_LINK_IN_TREE, &tlink->tl_flags); + rb_erase(tmp, root); + + spin_unlock(&cifs_sb->tlink_tree_lock); + cifs_put_tlink(tlink); + spin_lock(&cifs_sb->tlink_tree_lock); + } + spin_unlock(&cifs_sb->tlink_tree_lock); + + queue_delayed_work(cifsiod_wq, &cifs_sb->prune_tlinks, + TLINK_IDLE_EXPIRE); +} + +#ifndef CONFIG_CIFS_DFS_UPCALL +int cifs_tree_connect(const unsigned int xid, struct cifs_tcon *tcon) +{ + const struct smb_version_operations *ops = tcon->ses->server->ops; + int rc; + + /* only send once per connect */ + spin_lock(&tcon->tc_lock); + + /* if tcon is marked for needing reconnect, update state */ + if (tcon->need_reconnect) + tcon->status = TID_NEED_TCON; + + if (tcon->status == TID_GOOD) { + spin_unlock(&tcon->tc_lock); + return 0; + } + + if (tcon->status != TID_NEW && + tcon->status != TID_NEED_TCON) { + spin_unlock(&tcon->tc_lock); + return -EHOSTDOWN; + } + + tcon->status = TID_IN_TCON; + spin_unlock(&tcon->tc_lock); + + rc = ops->tree_connect(xid, tcon->ses, tcon->tree_name, + tcon, tcon->ses->local_nls); + if (rc) { + spin_lock(&tcon->tc_lock); + if (tcon->status == TID_IN_TCON) + tcon->status = TID_NEED_TCON; + spin_unlock(&tcon->tc_lock); + } else { + spin_lock(&tcon->tc_lock); + if (tcon->status == TID_IN_TCON) + tcon->status = TID_GOOD; + tcon->need_reconnect = false; + spin_unlock(&tcon->tc_lock); + } + + return rc; +} +#endif diff --git a/fs/smb/client/dfs.c b/fs/smb/client/dfs.c new file mode 100644 index 000000000..f65a8a90b --- /dev/null +++ b/fs/smb/client/dfs.c @@ -0,0 +1,467 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2022 Paulo Alcantara + */ + +#include "cifsproto.h" +#include "cifs_debug.h" +#include "dns_resolve.h" +#include "fs_context.h" +#include "dfs.h" + +#define DFS_DOM(ctx) (ctx->dfs_root_ses ? ctx->dfs_root_ses->dns_dom : NULL) + +/** + * dfs_parse_target_referral - set fs context for dfs target referral + * + * @full_path: full path in UNC format. + * @ref: dfs referral pointer. + * @ctx: smb3 fs context pointer. + * + * Return zero if dfs referral was parsed correctly, otherwise non-zero. + */ +int dfs_parse_target_referral(const char *full_path, const struct dfs_info3_param *ref, + struct smb3_fs_context *ctx) +{ + int rc; + const char *prepath = NULL; + char *path; + + if (!full_path || !*full_path || !ref || !ctx) + return -EINVAL; + + if (WARN_ON_ONCE(!ref->node_name || ref->path_consumed < 0)) + return -EINVAL; + + if (strlen(full_path) - ref->path_consumed) { + prepath = full_path + ref->path_consumed; + /* skip initial delimiter */ + if (*prepath == '/' || *prepath == '\\') + prepath++; + } + + path = cifs_build_devname(ref->node_name, prepath); + if (IS_ERR(path)) + return PTR_ERR(path); + + rc = smb3_parse_devname(path, ctx); + if (rc) + goto out; + + rc = dns_resolve_unc(DFS_DOM(ctx), path, + (struct sockaddr *)&ctx->dstaddr); +out: + kfree(path); + return rc; +} + +static int get_session(struct cifs_mount_ctx *mnt_ctx, const char *full_path) +{ + struct smb3_fs_context *ctx = mnt_ctx->fs_ctx; + int rc; + + ctx->leaf_fullpath = (char *)full_path; + ctx->dns_dom = DFS_DOM(ctx); + rc = cifs_mount_get_session(mnt_ctx); + ctx->leaf_fullpath = ctx->dns_dom = NULL; + + return rc; +} + +/* + * Get an active reference of @ses so that next call to cifs_put_tcon() won't + * release it as any new DFS referrals must go through its IPC tcon. + */ +static void set_root_smb_session(struct cifs_mount_ctx *mnt_ctx) +{ + struct smb3_fs_context *ctx = mnt_ctx->fs_ctx; + struct cifs_ses *ses = mnt_ctx->ses; + + if (ses) { + spin_lock(&cifs_tcp_ses_lock); + cifs_smb_ses_inc_refcount(ses); + spin_unlock(&cifs_tcp_ses_lock); + } + ctx->dfs_root_ses = ses; +} + +static inline int parse_dfs_target(struct smb3_fs_context *ctx, + struct dfs_ref_walk *rw, + struct dfs_info3_param *tgt) +{ + int rc; + const char *fpath = ref_walk_fpath(rw) + 1; + + rc = ref_walk_get_tgt(rw, tgt); + if (!rc) + rc = dfs_parse_target_referral(fpath, tgt, ctx); + return rc; +} + +static int setup_dfs_ref(struct dfs_info3_param *tgt, struct dfs_ref_walk *rw) +{ + struct cifs_sb_info *cifs_sb = rw->mnt_ctx->cifs_sb; + struct smb3_fs_context *ctx = rw->mnt_ctx->fs_ctx; + char *ref_path, *full_path; + int rc; + + set_root_smb_session(rw->mnt_ctx); + ref_walk_ses(rw) = ctx->dfs_root_ses; + + full_path = smb3_fs_context_fullpath(ctx, CIFS_DIR_SEP(cifs_sb)); + if (IS_ERR(full_path)) + return PTR_ERR(full_path); + + if (!tgt || (tgt->server_type == DFS_TYPE_LINK && + DFS_INTERLINK(tgt->flags))) + ref_path = dfs_get_path(cifs_sb, ctx->UNC); + else + ref_path = dfs_get_path(cifs_sb, full_path); + if (IS_ERR(ref_path)) { + rc = PTR_ERR(ref_path); + kfree(full_path); + return rc; + } + ref_walk_path(rw) = ref_path; + ref_walk_fpath(rw) = full_path; + + return dfs_get_referral(rw->mnt_ctx, + ref_walk_path(rw) + 1, + ref_walk_tl(rw)); +} + +static int __dfs_referral_walk(struct dfs_ref_walk *rw) +{ + struct smb3_fs_context *ctx = rw->mnt_ctx->fs_ctx; + struct cifs_mount_ctx *mnt_ctx = rw->mnt_ctx; + struct dfs_info3_param tgt = {}; + int rc = -ENOENT; + +again: + do { + ctx->dfs_root_ses = ref_walk_ses(rw); + while (ref_walk_next_tgt(rw)) { + rc = parse_dfs_target(ctx, rw, &tgt); + if (rc) + continue; + + cifs_mount_put_conns(mnt_ctx); + rc = get_session(mnt_ctx, ref_walk_path(rw)); + if (rc) + continue; + + rc = cifs_mount_get_tcon(mnt_ctx); + if (rc) { + if (tgt.server_type == DFS_TYPE_LINK && + DFS_INTERLINK(tgt.flags)) + rc = -EREMOTE; + } else { + rc = cifs_is_path_remote(mnt_ctx); + if (!rc) { + ref_walk_set_tgt_hint(rw); + break; + } + } + if (rc == -EREMOTE) { + rc = ref_walk_advance(rw); + if (!rc) { + rc = setup_dfs_ref(&tgt, rw); + if (rc) + break; + ref_walk_mark_end(rw); + goto again; + } + } + } + } while (rc && ref_walk_descend(rw)); + + free_dfs_info_param(&tgt); + return rc; +} + +static int dfs_referral_walk(struct cifs_mount_ctx *mnt_ctx, + struct dfs_ref_walk **rw) +{ + int rc; + + *rw = ref_walk_alloc(); + if (IS_ERR(*rw)) { + rc = PTR_ERR(*rw); + *rw = NULL; + return rc; + } + + ref_walk_init(*rw, mnt_ctx); + rc = setup_dfs_ref(NULL, *rw); + if (!rc) + rc = __dfs_referral_walk(*rw); + return rc; +} + +static int __dfs_mount_share(struct cifs_mount_ctx *mnt_ctx) +{ + struct cifs_sb_info *cifs_sb = mnt_ctx->cifs_sb; + struct smb3_fs_context *ctx = mnt_ctx->fs_ctx; + struct dfs_ref_walk *rw = NULL; + struct cifs_tcon *tcon; + char *origin_fullpath; + int rc; + + origin_fullpath = dfs_get_path(cifs_sb, ctx->source); + if (IS_ERR(origin_fullpath)) + return PTR_ERR(origin_fullpath); + + rc = dfs_referral_walk(mnt_ctx, &rw); + if (!rc) { + /* + * Prevent superblock from being created with any missing + * connections. + */ + if (WARN_ON(!mnt_ctx->server)) + rc = -EHOSTDOWN; + else if (WARN_ON(!mnt_ctx->ses)) + rc = -EACCES; + else if (WARN_ON(!mnt_ctx->tcon)) + rc = -ENOENT; + } + if (rc) + goto out; + + tcon = mnt_ctx->tcon; + spin_lock(&tcon->tc_lock); + tcon->origin_fullpath = origin_fullpath; + origin_fullpath = NULL; + ref_walk_set_tcon(rw, tcon); + spin_unlock(&tcon->tc_lock); + queue_delayed_work(dfscache_wq, &tcon->dfs_cache_work, + dfs_cache_get_ttl() * HZ); + +out: + kfree(origin_fullpath); + ref_walk_free(rw); + return rc; +} + +/* + * If @ctx->dfs_automount, then update @ctx->dstaddr earlier with the DFS root + * server from where we'll start following any referrals. Otherwise rely on the + * value provided by mount(2) as the user might not have dns_resolver key set up + * and therefore failing to upcall to resolve UNC hostname under @ctx->source. + */ +static int update_fs_context_dstaddr(struct smb3_fs_context *ctx) +{ + struct sockaddr *addr = (struct sockaddr *)&ctx->dstaddr; + int rc = 0; + + if (!ctx->nodfs && ctx->dfs_automount) { + rc = dns_resolve_unc(NULL, ctx->source, addr); + if (!rc) + cifs_set_port(addr, ctx->port); + ctx->dfs_automount = false; + } + return rc; +} + +int dfs_mount_share(struct cifs_mount_ctx *mnt_ctx) +{ + struct smb3_fs_context *ctx = mnt_ctx->fs_ctx; + bool nodfs = ctx->nodfs; + int rc; + + rc = update_fs_context_dstaddr(ctx); + if (rc) + return rc; + + rc = get_session(mnt_ctx, NULL); + if (rc) + return rc; + + /* + * If called with 'nodfs' mount option, then skip DFS resolving. Otherwise unconditionally + * try to get an DFS referral (even cached) to determine whether it is an DFS mount. + * + * Skip prefix path to provide support for DFS referrals from w2k8 servers which don't seem + * to respond with PATH_NOT_COVERED to requests that include the prefix. + */ + if (!nodfs) { + rc = dfs_get_referral(mnt_ctx, ctx->UNC + 1, NULL); + if (rc) { + cifs_dbg(FYI, "%s: no dfs referral for %s: %d\n", + __func__, ctx->UNC + 1, rc); + cifs_dbg(FYI, "%s: assuming non-dfs mount...\n", __func__); + nodfs = true; + } + } + if (nodfs) { + rc = cifs_mount_get_tcon(mnt_ctx); + if (!rc) + rc = cifs_is_path_remote(mnt_ctx); + return rc; + } + + if (!ctx->dfs_conn) { + ctx->dfs_conn = true; + cifs_mount_put_conns(mnt_ctx); + rc = get_session(mnt_ctx, NULL); + } + if (!rc) + rc = __dfs_mount_share(mnt_ctx); + return rc; +} + +static int target_share_matches_server(struct TCP_Server_Info *server, char *share, + bool *target_match) +{ + int rc = 0; + const char *dfs_host; + size_t dfs_host_len; + + *target_match = true; + extract_unc_hostname(share, &dfs_host, &dfs_host_len); + + /* Check if hostnames or addresses match */ + cifs_server_lock(server); + if (dfs_host_len != strlen(server->hostname) || + strncasecmp(dfs_host, server->hostname, dfs_host_len)) { + cifs_dbg(FYI, "%s: %.*s doesn't match %s\n", __func__, + (int)dfs_host_len, dfs_host, server->hostname); + rc = match_target_ip(server, dfs_host, dfs_host_len, target_match); + if (rc) + cifs_dbg(VFS, "%s: failed to match target ip: %d\n", __func__, rc); + } + cifs_server_unlock(server); + return rc; +} + +static int tree_connect_dfs_target(const unsigned int xid, + struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, + char *tree, bool islink, + struct dfs_cache_tgt_list *tl) +{ + const struct smb_version_operations *ops = tcon->ses->server->ops; + struct TCP_Server_Info *server = tcon->ses->server; + struct dfs_cache_tgt_iterator *tit; + char *share = NULL, *prefix = NULL; + bool target_match; + int rc = -ENOENT; + + /* Try to tree connect to all dfs targets */ + for (tit = dfs_cache_get_tgt_iterator(tl); + tit; tit = dfs_cache_get_next_tgt(tl, tit)) { + kfree(share); + kfree(prefix); + share = prefix = NULL; + + /* Check if share matches with tcp ses */ + rc = dfs_cache_get_tgt_share(server->leaf_fullpath + 1, tit, &share, &prefix); + if (rc) { + cifs_dbg(VFS, "%s: failed to parse target share: %d\n", __func__, rc); + break; + } + + rc = target_share_matches_server(server, share, &target_match); + if (rc) + break; + if (!target_match) { + rc = -EHOSTUNREACH; + continue; + } + + dfs_cache_noreq_update_tgthint(server->leaf_fullpath + 1, tit); + scnprintf(tree, MAX_TREE_SIZE, "\\%s", share); + rc = ops->tree_connect(xid, tcon->ses, tree, + tcon, tcon->ses->local_nls); + if (islink && !rc && cifs_sb) + rc = cifs_update_super_prepath(cifs_sb, prefix); + break; + } + + kfree(share); + kfree(prefix); + dfs_cache_free_tgts(tl); + return rc; +} + +int cifs_tree_connect(const unsigned int xid, struct cifs_tcon *tcon) +{ + int rc; + struct TCP_Server_Info *server = tcon->ses->server; + const struct smb_version_operations *ops = server->ops; + DFS_CACHE_TGT_LIST(tl); + struct cifs_sb_info *cifs_sb = NULL; + struct super_block *sb = NULL; + struct dfs_info3_param ref = {0}; + char *tree; + + /* only send once per connect */ + spin_lock(&tcon->tc_lock); + + /* if tcon is marked for needing reconnect, update state */ + if (tcon->need_reconnect) + tcon->status = TID_NEED_TCON; + + if (tcon->status == TID_GOOD) { + spin_unlock(&tcon->tc_lock); + return 0; + } + + if (tcon->status != TID_NEW && + tcon->status != TID_NEED_TCON) { + spin_unlock(&tcon->tc_lock); + return -EHOSTDOWN; + } + + tcon->status = TID_IN_TCON; + spin_unlock(&tcon->tc_lock); + + tree = kzalloc(MAX_TREE_SIZE, GFP_KERNEL); + if (!tree) { + rc = -ENOMEM; + goto out; + } + + if (tcon->ipc) { + cifs_server_lock(server); + scnprintf(tree, MAX_TREE_SIZE, "\\\\%s\\IPC$", server->hostname); + cifs_server_unlock(server); + rc = ops->tree_connect(xid, tcon->ses, tree, + tcon, tcon->ses->local_nls); + goto out; + } + + sb = cifs_get_dfs_tcon_super(tcon); + if (!IS_ERR(sb)) + cifs_sb = CIFS_SB(sb); + + /* Tree connect to last share in @tcon->tree_name if no DFS referral */ + if (!server->leaf_fullpath || + dfs_cache_noreq_find(server->leaf_fullpath + 1, &ref, &tl)) { + rc = ops->tree_connect(xid, tcon->ses, tcon->tree_name, + tcon, tcon->ses->local_nls); + goto out; + } + + rc = tree_connect_dfs_target(xid, tcon, cifs_sb, tree, ref.server_type == DFS_TYPE_LINK, + &tl); + free_dfs_info_param(&ref); + +out: + kfree(tree); + cifs_put_tcp_super(sb); + + if (rc) { + spin_lock(&tcon->tc_lock); + if (tcon->status == TID_IN_TCON) + tcon->status = TID_NEED_TCON; + spin_unlock(&tcon->tc_lock); + } else { + spin_lock(&tcon->tc_lock); + if (tcon->status == TID_IN_TCON) + tcon->status = TID_GOOD; + tcon->need_reconnect = false; + spin_unlock(&tcon->tc_lock); + } + + return rc; +} diff --git a/fs/smb/client/dfs.h b/fs/smb/client/dfs.h new file mode 100644 index 000000000..16ac2cdd5 --- /dev/null +++ b/fs/smb/client/dfs.h @@ -0,0 +1,199 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2022 Paulo Alcantara + */ + +#ifndef _CIFS_DFS_H +#define _CIFS_DFS_H + +#include "cifsglob.h" +#include "cifsproto.h" +#include "fs_context.h" +#include "dfs_cache.h" +#include "cifs_unicode.h" +#include +#include + +#define DFS_INTERLINK(v) \ + (((v) & DFSREF_REFERRAL_SERVER) && !((v) & DFSREF_STORAGE_SERVER)) + +struct dfs_ref { + char *path; + char *full_path; + struct cifs_ses *ses; + struct dfs_cache_tgt_list tl; + struct dfs_cache_tgt_iterator *tit; +}; + +struct dfs_ref_walk { + struct cifs_mount_ctx *mnt_ctx; + struct dfs_ref *ref; + struct dfs_ref refs[MAX_NESTED_LINKS]; +}; + +#define ref_walk_start(w) ((w)->refs) +#define ref_walk_end(w) (&(w)->refs[ARRAY_SIZE((w)->refs) - 1]) +#define ref_walk_cur(w) ((w)->ref) +#define ref_walk_descend(w) (--ref_walk_cur(w) >= ref_walk_start(w)) + +#define ref_walk_tit(w) (ref_walk_cur(w)->tit) +#define ref_walk_path(w) (ref_walk_cur(w)->path) +#define ref_walk_fpath(w) (ref_walk_cur(w)->full_path) +#define ref_walk_tl(w) (&ref_walk_cur(w)->tl) +#define ref_walk_ses(w) (ref_walk_cur(w)->ses) + +static inline struct dfs_ref_walk *ref_walk_alloc(void) +{ + struct dfs_ref_walk *rw; + + rw = kmalloc_obj(*rw); + if (!rw) + return ERR_PTR(-ENOMEM); + return rw; +} + +static inline void ref_walk_init(struct dfs_ref_walk *rw, + struct cifs_mount_ctx *mnt_ctx) +{ + memset(rw, 0, sizeof(*rw)); + rw->mnt_ctx = mnt_ctx; + ref_walk_cur(rw) = ref_walk_start(rw); +} + +static inline void __ref_walk_free(struct dfs_ref *ref) +{ + kfree(ref->path); + kfree(ref->full_path); + dfs_cache_free_tgts(&ref->tl); + if (ref->ses) + cifs_put_smb_ses(ref->ses); + memset(ref, 0, sizeof(*ref)); +} + +static inline void ref_walk_free(struct dfs_ref_walk *rw) +{ + struct dfs_ref *ref; + + if (!rw) + return; + + for (ref = ref_walk_start(rw); ref <= ref_walk_end(rw); ref++) + __ref_walk_free(ref); + kfree(rw); +} + +static inline int ref_walk_advance(struct dfs_ref_walk *rw) +{ + struct dfs_ref *ref = ref_walk_cur(rw) + 1; + + if (ref > ref_walk_end(rw)) + return -ELOOP; + __ref_walk_free(ref); + ref_walk_cur(rw) = ref; + return 0; +} + +static inline struct dfs_cache_tgt_iterator * +ref_walk_next_tgt(struct dfs_ref_walk *rw) +{ + struct dfs_ref *ref = ref_walk_cur(rw); + struct dfs_cache_tgt_iterator *tit; + + if (IS_ERR(ref->tit)) + return NULL; + + if (!ref->tit) + tit = dfs_cache_get_tgt_iterator(&ref->tl); + else + tit = dfs_cache_get_next_tgt(&ref->tl, ref->tit); + + if (!tit) { + ref->tit = ERR_PTR(-ENOENT); + return NULL; + } + ref->tit = tit; + return ref->tit; +} + +static inline int ref_walk_get_tgt(struct dfs_ref_walk *rw, + struct dfs_info3_param *tgt) +{ + zfree_dfs_info_param(tgt); + return dfs_cache_get_tgt_referral(ref_walk_path(rw) + 1, + ref_walk_tit(rw), tgt); +} + +static inline void ref_walk_set_tgt_hint(struct dfs_ref_walk *rw) +{ + dfs_cache_noreq_update_tgthint(ref_walk_path(rw) + 1, + ref_walk_tit(rw)); +} + +static inline void ref_walk_set_tcon(struct dfs_ref_walk *rw, + struct cifs_tcon *tcon) +{ + struct dfs_ref *ref = ref_walk_start(rw); + + for (; ref <= ref_walk_cur(rw); ref++) { + if (WARN_ON_ONCE(!ref->ses)) + continue; + list_add(&ref->ses->dlist, &tcon->dfs_ses_list); + ref->ses = NULL; + } +} + +static inline void ref_walk_mark_end(struct dfs_ref_walk *rw) +{ + struct dfs_ref *ref = ref_walk_cur(rw) - 1; + + WARN_ON_ONCE(ref < ref_walk_start(rw)); + dfs_cache_noreq_update_tgthint(ref->path + 1, ref->tit); + ref->tit = ERR_PTR(-ENOENT); /* end marker */ +} + +int dfs_parse_target_referral(const char *full_path, + const struct dfs_info3_param *ref, + struct smb3_fs_context *ctx); +int dfs_mount_share(struct cifs_mount_ctx *mnt_ctx); + +static inline char *dfs_get_path(struct cifs_sb_info *cifs_sb, const char *path) +{ + return dfs_cache_canonical_path(path, cifs_sb->local_nls, cifs_remap(cifs_sb)); +} + +static inline int dfs_get_referral(struct cifs_mount_ctx *mnt_ctx, + const char *path, + struct dfs_cache_tgt_list *tl) +{ + struct smb3_fs_context *ctx = mnt_ctx->fs_ctx; + struct cifs_sb_info *cifs_sb = mnt_ctx->cifs_sb; + struct cifs_ses *rses = ctx->dfs_root_ses ?: mnt_ctx->ses; + + return dfs_cache_find(mnt_ctx->xid, rses, cifs_sb->local_nls, + cifs_remap(cifs_sb), path, NULL, tl); +} + +/* + * cifs_get_smb_ses() already guarantees an active reference of + * @ses->dfs_root_ses when a new session is created, so we need to put extra + * references of all DFS root sessions that were used across the mount process + * in dfs_mount_share(). + */ +static inline void dfs_put_root_smb_sessions(struct list_head *head) +{ + struct cifs_ses *ses, *n; + + list_for_each_entry_safe(ses, n, head, dlist) { + list_del_init(&ses->dlist); + cifs_put_smb_ses(ses); + } +} + +static inline const char *dfs_ses_refpath(struct cifs_ses *ses) +{ + const char *path = ses->server->leaf_fullpath; + + return path ? path + 1 : ERR_PTR(-ENOENT); +} + +#endif /* _CIFS_DFS_H */ diff --git a/fs/smb/client/dfs_cache.c b/fs/smb/client/dfs_cache.c new file mode 100644 index 000000000..8cd93cd2f --- /dev/null +++ b/fs/smb/client/dfs_cache.c @@ -0,0 +1,1356 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DFS referral cache routines + * + * Copyright (c) 2018-2019 Paulo Alcantara + */ + +#include +#include +#include +#include +#include +#include +#include +#include "cifsglob.h" +#include "smb2pdu.h" +#include "smb2proto.h" +#include "cifsproto.h" +#include "cifs_debug.h" +#include "cifs_unicode.h" +#include "smb2glob.h" +#include "dns_resolve.h" +#include "dfs.h" + +#include "dfs_cache.h" + +#define CACHE_HTABLE_SIZE 512 +#define CACHE_MAX_ENTRIES 1024 +#define CACHE_MIN_TTL 120 /* 2 minutes */ +#define CACHE_DEFAULT_TTL 300 /* 5 minutes */ + +struct cache_dfs_tgt { + char *name; + int path_consumed; + struct list_head list; +}; + +struct cache_entry { + struct hlist_node hlist; + const char *path; + int hdr_flags; /* RESP_GET_DFS_REFERRAL.ReferralHeaderFlags */ + int ttl; /* DFS_REREFERRAL_V3.TimeToLive */ + int srvtype; /* DFS_REREFERRAL_V3.ServerType */ + int ref_flags; /* DFS_REREFERRAL_V3.ReferralEntryFlags */ + struct timespec64 etime; + int path_consumed; /* RESP_GET_DFS_REFERRAL.PathConsumed */ + int numtgts; + struct list_head tlist; + struct cache_dfs_tgt *tgthint; +}; + +static struct kmem_cache *cache_slab __read_mostly; +struct workqueue_struct *dfscache_wq; + +atomic_t dfs_cache_ttl; + +static struct nls_table *cache_cp; + +/* + * Number of entries in the cache + */ +static atomic_t cache_count; + +static struct hlist_head cache_htable[CACHE_HTABLE_SIZE]; +static DECLARE_RWSEM(htable_rw_lock); + +/** + * dfs_cache_canonical_path - get a canonical DFS path + * + * @path: DFS path + * @cp: codepage + * @remap: mapping type + * + * Return canonical path if success, otherwise error. + */ +char *dfs_cache_canonical_path(const char *path, const struct nls_table *cp, int remap) +{ + char *tmp; + int plen = 0; + char *npath; + + if (!path || strlen(path) < 3 || (*path != '\\' && *path != '/')) + return ERR_PTR(-EINVAL); + + if (unlikely(strcmp(cp->charset, cache_cp->charset))) { + tmp = (char *)cifs_strndup_to_utf16(path, strlen(path), &plen, cp, remap); + if (!tmp) { + cifs_dbg(VFS, "%s: failed to convert path to utf16\n", __func__); + return ERR_PTR(-EINVAL); + } + + npath = cifs_strndup_from_utf16(tmp, plen, true, cache_cp); + kfree(tmp); + + if (!npath) { + cifs_dbg(VFS, "%s: failed to convert path from utf16\n", __func__); + return ERR_PTR(-EINVAL); + } + } else { + npath = kstrdup(path, GFP_KERNEL); + if (!npath) + return ERR_PTR(-ENOMEM); + } + convert_delimiter(npath, '\\'); + return npath; +} + +static inline bool cache_entry_expired(const struct cache_entry *ce) +{ + struct timespec64 ts; + + ktime_get_coarse_real_ts64(&ts); + return timespec64_compare(&ts, &ce->etime) >= 0; +} + +static inline void free_tgts(struct cache_entry *ce) +{ + struct cache_dfs_tgt *t, *n; + + list_for_each_entry_safe(t, n, &ce->tlist, list) { + list_del(&t->list); + kfree(t->name); + kfree(t); + } +} + +static inline void flush_cache_ent(struct cache_entry *ce) +{ + cifs_dbg(FYI, "%s: %s\n", __func__, ce->path); + hlist_del_init(&ce->hlist); + kfree(ce->path); + free_tgts(ce); + atomic_dec(&cache_count); + kmem_cache_free(cache_slab, ce); +} + +static void flush_cache_ents(void) +{ + int i; + + for (i = 0; i < CACHE_HTABLE_SIZE; i++) { + struct hlist_head *l = &cache_htable[i]; + struct hlist_node *n; + struct cache_entry *ce; + + hlist_for_each_entry_safe(ce, n, l, hlist) { + if (!hlist_unhashed(&ce->hlist)) + flush_cache_ent(ce); + } + } +} + +/* + * dfs cache /proc file + */ +static int dfscache_proc_show(struct seq_file *m, void *v) +{ + int i; + struct cache_entry *ce; + struct cache_dfs_tgt *t; + + seq_puts(m, "DFS cache\n---------\n"); + + down_read(&htable_rw_lock); + for (i = 0; i < CACHE_HTABLE_SIZE; i++) { + struct hlist_head *l = &cache_htable[i]; + + hlist_for_each_entry(ce, l, hlist) { + if (hlist_unhashed(&ce->hlist)) + continue; + + seq_printf(m, + "cache entry: path=%s,type=%s,ttl=%d,etime=%ld,hdr_flags=0x%x,ref_flags=0x%x,interlink=%s,path_consumed=%d,expired=%s\n", + ce->path, ce->srvtype == DFS_TYPE_ROOT ? "root" : "link", + ce->ttl, ce->etime.tv_nsec, ce->hdr_flags, ce->ref_flags, + str_yes_no(DFS_INTERLINK(ce->hdr_flags)), + ce->path_consumed, str_yes_no(cache_entry_expired(ce))); + + list_for_each_entry(t, &ce->tlist, list) { + seq_printf(m, " %s%s\n", + t->name, + READ_ONCE(ce->tgthint) == t ? " (target hint)" : ""); + } + } + } + up_read(&htable_rw_lock); + + return 0; +} + +static ssize_t dfscache_proc_write(struct file *file, const char __user *buffer, + size_t count, loff_t *ppos) +{ + char c; + int rc; + + rc = get_user(c, buffer); + if (rc) + return rc; + + if (c != '0') + return -EINVAL; + + cifs_dbg(FYI, "clearing dfs cache\n"); + + down_write(&htable_rw_lock); + flush_cache_ents(); + up_write(&htable_rw_lock); + + return count; +} + +static int dfscache_proc_open(struct inode *inode, struct file *file) +{ + return single_open(file, dfscache_proc_show, NULL); +} + +const struct proc_ops dfscache_proc_ops = { + .proc_open = dfscache_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = dfscache_proc_write, +}; + +#ifdef CONFIG_CIFS_DEBUG2 +static inline void dump_tgts(const struct cache_entry *ce) +{ + struct cache_dfs_tgt *t; + + cifs_dbg(FYI, "target list:\n"); + list_for_each_entry(t, &ce->tlist, list) { + cifs_dbg(FYI, " %s%s\n", t->name, + READ_ONCE(ce->tgthint) == t ? " (target hint)" : ""); + } +} + +static inline void dump_ce(const struct cache_entry *ce) +{ + cifs_dbg(FYI, "cache entry: path=%s,type=%s,ttl=%d,etime=%ld,hdr_flags=0x%x,ref_flags=0x%x,interlink=%s,path_consumed=%d,expired=%s\n", + ce->path, + ce->srvtype == DFS_TYPE_ROOT ? "root" : "link", ce->ttl, + ce->etime.tv_nsec, + ce->hdr_flags, ce->ref_flags, + str_yes_no(DFS_INTERLINK(ce->hdr_flags)), + ce->path_consumed, + str_yes_no(cache_entry_expired(ce))); + dump_tgts(ce); +} + +static inline void dump_refs(const struct dfs_info3_param *refs, int numrefs) +{ + int i; + + cifs_dbg(FYI, "DFS referrals returned by the server:\n"); + for (i = 0; i < numrefs; i++) { + const struct dfs_info3_param *ref = &refs[i]; + + cifs_dbg(FYI, + "\n" + "flags: 0x%x\n" + "path_consumed: %d\n" + "server_type: 0x%x\n" + "ref_flag: 0x%x\n" + "path_name: %s\n" + "node_name: %s\n" + "ttl: %d (%dm)\n", + ref->flags, ref->path_consumed, ref->server_type, + ref->ref_flag, ref->path_name, ref->node_name, + ref->ttl, ref->ttl / 60); + } +} +#else +#define dump_tgts(e) +#define dump_ce(e) +#define dump_refs(r, n) +#endif + +/** + * dfs_cache_init - Initialize DFS referral cache. + * + * Return zero if initialized successfully, otherwise non-zero. + */ +int dfs_cache_init(void) +{ + int rc; + int i; + + dfscache_wq = alloc_workqueue("cifs-dfscache", + WQ_UNBOUND|WQ_FREEZABLE|WQ_MEM_RECLAIM, + 0); + if (!dfscache_wq) + return -ENOMEM; + + cache_slab = kmem_cache_create("cifs_dfs_cache", + sizeof(struct cache_entry), 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!cache_slab) { + rc = -ENOMEM; + goto out_destroy_wq; + } + + for (i = 0; i < CACHE_HTABLE_SIZE; i++) + INIT_HLIST_HEAD(&cache_htable[i]); + + atomic_set(&cache_count, 0); + atomic_set(&dfs_cache_ttl, CACHE_DEFAULT_TTL); + cache_cp = load_nls("utf8"); + if (!cache_cp) + cache_cp = load_nls_default(); + + cifs_dbg(FYI, "%s: initialized DFS referral cache\n", __func__); + return 0; + +out_destroy_wq: + destroy_workqueue(dfscache_wq); + return rc; +} + +static int cache_entry_hash(const void *data, int size, unsigned int *hash) +{ + int i, clen; + const unsigned char *s = data; + wchar_t c; + unsigned int h = 0; + + for (i = 0; i < size; i += clen) { + clen = cache_cp->char2uni(&s[i], size - i, &c); + if (unlikely(clen < 0)) { + cifs_dbg(VFS, "%s: can't convert char\n", __func__); + return clen; + } + c = cifs_toupper(c); + h = jhash(&c, sizeof(c), h); + } + *hash = h % CACHE_HTABLE_SIZE; + return 0; +} + +/* Return target hint of a DFS cache entry */ +static inline char *get_tgt_name(const struct cache_entry *ce) +{ + struct cache_dfs_tgt *t = READ_ONCE(ce->tgthint); + + return t ? t->name : ERR_PTR(-ENOENT); +} + +/* Return expire time out of a new entry's TTL */ +static inline struct timespec64 get_expire_time(int ttl) +{ + struct timespec64 ts = { + .tv_sec = ttl, + .tv_nsec = 0, + }; + struct timespec64 now; + + ktime_get_coarse_real_ts64(&now); + return timespec64_add(now, ts); +} + +/* Allocate a new DFS target */ +static struct cache_dfs_tgt *alloc_target(const char *name, int path_consumed) +{ + struct cache_dfs_tgt *t; + + t = kmalloc_obj(*t, GFP_KERNEL); + if (!t) + return ERR_PTR(-ENOMEM); + t->name = kstrdup(name, GFP_KERNEL); + if (!t->name) { + kfree(t); + return ERR_PTR(-ENOMEM); + } + t->path_consumed = path_consumed; + INIT_LIST_HEAD(&t->list); + return t; +} + +/* + * Copy DFS referral information to a cache entry and conditionally update + * target hint. + */ +static int copy_ref_data(const struct dfs_info3_param *refs, int numrefs, + struct cache_entry *ce, const char *tgthint) +{ + struct cache_dfs_tgt *target; + int i; + + ce->ttl = max_t(int, refs[0].ttl, CACHE_MIN_TTL); + ce->etime = get_expire_time(ce->ttl); + ce->srvtype = refs[0].server_type; + ce->hdr_flags = refs[0].flags; + ce->ref_flags = refs[0].ref_flag; + ce->path_consumed = refs[0].path_consumed; + + for (i = 0; i < numrefs; i++) { + struct cache_dfs_tgt *t; + + t = alloc_target(refs[i].node_name, refs[i].path_consumed); + if (IS_ERR(t)) { + free_tgts(ce); + return PTR_ERR(t); + } + if (tgthint && !strcasecmp(t->name, tgthint)) { + list_add(&t->list, &ce->tlist); + tgthint = NULL; + } else { + list_add_tail(&t->list, &ce->tlist); + } + ce->numtgts++; + } + + target = list_first_entry_or_null(&ce->tlist, struct cache_dfs_tgt, + list); + WRITE_ONCE(ce->tgthint, target); + + return 0; +} + +/* Allocate a new cache entry */ +static struct cache_entry *alloc_cache_entry(struct dfs_info3_param *refs, int numrefs) +{ + struct cache_entry *ce; + int rc; + + ce = kmem_cache_zalloc(cache_slab, GFP_KERNEL); + if (!ce) + return ERR_PTR(-ENOMEM); + + ce->path = refs[0].path_name; + refs[0].path_name = NULL; + + INIT_HLIST_NODE(&ce->hlist); + INIT_LIST_HEAD(&ce->tlist); + + rc = copy_ref_data(refs, numrefs, ce, NULL); + if (rc) { + kfree(ce->path); + kmem_cache_free(cache_slab, ce); + ce = ERR_PTR(rc); + } + return ce; +} + +/* Remove all referrals that have a single target or oldest entry */ +static void purge_cache(void) +{ + int i; + struct cache_entry *ce; + struct cache_entry *oldest = NULL; + + for (i = 0; i < CACHE_HTABLE_SIZE; i++) { + struct hlist_head *l = &cache_htable[i]; + struct hlist_node *n; + + hlist_for_each_entry_safe(ce, n, l, hlist) { + if (hlist_unhashed(&ce->hlist)) + continue; + if (ce->numtgts == 1) + flush_cache_ent(ce); + else if (!oldest || + timespec64_compare(&ce->etime, + &oldest->etime) < 0) + oldest = ce; + } + } + + if (atomic_read(&cache_count) >= CACHE_MAX_ENTRIES && oldest) + flush_cache_ent(oldest); +} + +/* Add a new DFS cache entry */ +static struct cache_entry *add_cache_entry_locked(struct dfs_info3_param *refs, + int numrefs) +{ + int rc; + struct cache_entry *ce; + unsigned int hash; + int ttl; + + WARN_ON(!rwsem_is_locked(&htable_rw_lock)); + + if (atomic_read(&cache_count) >= CACHE_MAX_ENTRIES) { + cifs_dbg(FYI, "%s: reached max cache size (%d)\n", __func__, CACHE_MAX_ENTRIES); + purge_cache(); + } + + rc = cache_entry_hash(refs[0].path_name, strlen(refs[0].path_name), &hash); + if (rc) + return ERR_PTR(rc); + + ce = alloc_cache_entry(refs, numrefs); + if (IS_ERR(ce)) + return ce; + + ttl = min_t(int, atomic_read(&dfs_cache_ttl), ce->ttl); + atomic_set(&dfs_cache_ttl, ttl); + + hlist_add_head(&ce->hlist, &cache_htable[hash]); + dump_ce(ce); + + atomic_inc(&cache_count); + + return ce; +} + +/* Check if two DFS paths are equal. @s1 and @s2 are expected to be in @cache_cp's charset */ +static bool dfs_path_equal(const char *s1, int len1, const char *s2, int len2) +{ + int i, l1, l2; + wchar_t c1, c2; + + if (len1 != len2) + return false; + + for (i = 0; i < len1; i += l1) { + l1 = cache_cp->char2uni(&s1[i], len1 - i, &c1); + l2 = cache_cp->char2uni(&s2[i], len2 - i, &c2); + if (unlikely(l1 < 0 && l2 < 0)) { + if (s1[i] != s2[i]) + return false; + l1 = 1; + continue; + } + if (l1 != l2) + return false; + if (cifs_toupper(c1) != cifs_toupper(c2)) + return false; + } + return true; +} + +static struct cache_entry *__lookup_cache_entry(const char *path, unsigned int hash, int len) +{ + struct cache_entry *ce; + + hlist_for_each_entry(ce, &cache_htable[hash], hlist) { + if (dfs_path_equal(ce->path, strlen(ce->path), path, len)) { + dump_ce(ce); + return ce; + } + } + return ERR_PTR(-ENOENT); +} + +/* + * Find a DFS cache entry in hash table and optionally check prefix path against normalized @path. + * + * Use whole path components in the match. Must be called with htable_rw_lock held. + * + * Return cached entry if successful. + * Return ERR_PTR(-ENOENT) if the entry is not found. + * Return error ptr otherwise. + */ +static struct cache_entry *lookup_cache_entry(const char *path) +{ + struct cache_entry *ce; + int cnt = 0; + const char *s = path, *e; + char sep = *s; + unsigned int hash; + int rc; + + while ((s = strchr(s, sep)) && ++cnt < 3) + s++; + + if (cnt < 3) { + rc = cache_entry_hash(path, strlen(path), &hash); + if (rc) + return ERR_PTR(rc); + return __lookup_cache_entry(path, hash, strlen(path)); + } + /* + * Handle paths that have more than two path components and are a complete prefix of the DFS + * referral request path (@path). + * + * See MS-DFSC 3.2.5.5 "Receiving a Root Referral Request or Link Referral Request". + */ + e = path + strlen(path) - 1; + while (e > s) { + int len; + + /* skip separators */ + while (e > s && *e == sep) + e--; + if (e == s) + break; + + len = e + 1 - path; + rc = cache_entry_hash(path, len, &hash); + if (rc) + return ERR_PTR(rc); + ce = __lookup_cache_entry(path, hash, len); + if (!IS_ERR(ce)) + return ce; + + /* backward until separator */ + while (e > s && *e != sep) + e--; + } + return ERR_PTR(-ENOENT); +} + +/** + * dfs_cache_destroy - destroy DFS referral cache + */ +void dfs_cache_destroy(void) +{ + unload_nls(cache_cp); + flush_cache_ents(); + kmem_cache_destroy(cache_slab); + destroy_workqueue(dfscache_wq); + + cifs_dbg(FYI, "%s: destroyed DFS referral cache\n", __func__); +} + +/* Update a cache entry with the new referral in @refs */ +static int update_cache_entry_locked(struct cache_entry *ce, const struct dfs_info3_param *refs, + int numrefs) +{ + struct cache_dfs_tgt *target; + char *th = NULL; + int rc; + + WARN_ON(!rwsem_is_locked(&htable_rw_lock)); + + target = READ_ONCE(ce->tgthint); + if (target) { + th = kstrdup(target->name, GFP_KERNEL); + if (!th) + return -ENOMEM; + } + + free_tgts(ce); + ce->numtgts = 0; + + rc = copy_ref_data(refs, numrefs, ce, th); + + kfree(th); + + return rc; +} + +static int get_dfs_referral(const unsigned int xid, struct cifs_ses *ses, const char *path, + struct dfs_info3_param **refs, int *numrefs) +{ + int rc; + int i; + + *refs = NULL; + *numrefs = 0; + + if (!ses || !ses->server || !ses->server->ops->get_dfs_refer) + return -EOPNOTSUPP; + if (unlikely(!cache_cp)) + return -EINVAL; + + cifs_dbg(FYI, "%s: ipc=%s referral=%s\n", __func__, ses->tcon_ipc->tree_name, path); + rc = ses->server->ops->get_dfs_refer(xid, ses, path, refs, numrefs, cache_cp, + NO_MAP_UNI_RSVD); + if (!rc) { + struct dfs_info3_param *ref = *refs; + + for (i = 0; i < *numrefs; i++) + convert_delimiter(ref[i].path_name, '\\'); + } + return rc; +} + +/* + * Find, create or update a DFS cache entry. + * + * If the entry wasn't found, it will create a new one. Or if it was found but + * expired, then it will update the entry accordingly. + * + * For interlinks, cifs_mount() and expand_dfs_referral() are supposed to + * handle them properly. + * + * On success, return entry with acquired lock for reading, otherwise error ptr. + */ +static struct cache_entry *cache_refresh_path(const unsigned int xid, + struct cifs_ses *ses, + const char *path, + bool force_refresh) +{ + struct dfs_info3_param *refs = NULL; + struct cache_entry *ce; + int numrefs = 0; + int rc; + + cifs_dbg(FYI, "%s: search path: %s\n", __func__, path); + + down_read(&htable_rw_lock); + + ce = lookup_cache_entry(path); + if (!IS_ERR(ce)) { + if (!force_refresh && !cache_entry_expired(ce)) + return ce; + } else if (PTR_ERR(ce) != -ENOENT) { + up_read(&htable_rw_lock); + return ce; + } + + /* + * Unlock shared access as we don't want to hold any locks while getting + * a new referral. The @ses used for performing the I/O could be + * reconnecting and it acquires @htable_rw_lock to look up the dfs cache + * in order to failover -- if necessary. + */ + up_read(&htable_rw_lock); + + /* + * Either the entry was not found, or it is expired, or it is a forced + * refresh. + * Request a new DFS referral in order to create or update a cache entry. + */ + rc = get_dfs_referral(xid, ses, path, &refs, &numrefs); + if (rc) { + ce = ERR_PTR(rc); + goto out; + } + + dump_refs(refs, numrefs); + + down_write(&htable_rw_lock); + /* Re-check as another task might have it added or refreshed already */ + ce = lookup_cache_entry(path); + if (!IS_ERR(ce)) { + if (force_refresh || cache_entry_expired(ce)) { + rc = update_cache_entry_locked(ce, refs, numrefs); + if (rc) + ce = ERR_PTR(rc); + } + } else if (PTR_ERR(ce) == -ENOENT) { + ce = add_cache_entry_locked(refs, numrefs); + } + + if (IS_ERR(ce)) { + up_write(&htable_rw_lock); + goto out; + } + + downgrade_write(&htable_rw_lock); +out: + free_dfs_info_array(refs, numrefs); + return ce; +} + +/* + * Set up a DFS referral from a given cache entry. + * + * Must be called with htable_rw_lock held. + */ +static int setup_referral(const char *path, struct cache_entry *ce, + struct dfs_info3_param *ref, const char *target) +{ + int rc; + + cifs_dbg(FYI, "%s: set up new ref\n", __func__); + + memset(ref, 0, sizeof(*ref)); + + ref->path_name = kstrdup(path, GFP_KERNEL); + if (!ref->path_name) + return -ENOMEM; + + ref->node_name = kstrdup(target, GFP_KERNEL); + if (!ref->node_name) { + rc = -ENOMEM; + goto err_free_path; + } + + ref->path_consumed = ce->path_consumed; + ref->ttl = ce->ttl; + ref->server_type = ce->srvtype; + ref->ref_flag = ce->ref_flags; + ref->flags = ce->hdr_flags; + + return 0; + +err_free_path: + kfree(ref->path_name); + ref->path_name = NULL; + return rc; +} + +/* Return target list of a DFS cache entry */ +static int get_targets(struct cache_entry *ce, struct dfs_cache_tgt_list *tl) +{ + int rc; + struct list_head *head = &tl->tl_list; + struct cache_dfs_tgt *t; + struct dfs_cache_tgt_iterator *it, *nit; + + memset(tl, 0, sizeof(*tl)); + INIT_LIST_HEAD(head); + + list_for_each_entry(t, &ce->tlist, list) { + it = kzalloc_obj(*it, GFP_ATOMIC); + if (!it) { + rc = -ENOMEM; + goto err_free_it; + } + + it->it_name = kstrdup(t->name, GFP_ATOMIC); + if (!it->it_name) { + kfree(it); + rc = -ENOMEM; + goto err_free_it; + } + it->it_path_consumed = t->path_consumed; + + if (READ_ONCE(ce->tgthint) == t) + list_add(&it->it_list, head); + else + list_add_tail(&it->it_list, head); + } + + tl->tl_numtgts = ce->numtgts; + + return 0; + +err_free_it: + list_for_each_entry_safe(it, nit, head, it_list) { + list_del(&it->it_list); + kfree(it->it_name); + kfree(it); + } + return rc; +} + +/** + * dfs_cache_find - find a DFS cache entry + * + * If it doesn't find the cache entry, then it will get a DFS referral + * for @path and create a new entry. + * + * In case the cache entry exists but expired, it will get a DFS referral + * for @path and then update the respective cache entry. + * + * These parameters are passed down to the get_dfs_refer() call if it + * needs to be issued: + * @xid: syscall xid + * @ses: smb session to issue the request on + * @cp: codepage + * @remap: path character remapping type + * @path: path to lookup in DFS referral cache. + * + * @ref: when non-NULL, store single DFS referral result in it. + * @tgt_list: when non-NULL, store complete DFS target list in it. + * + * Return zero if the target was found, otherwise non-zero. + */ +int dfs_cache_find(const unsigned int xid, struct cifs_ses *ses, const struct nls_table *cp, + int remap, const char *path, struct dfs_info3_param *ref, + struct dfs_cache_tgt_list *tgt_list) +{ + int rc; + const char *npath; + struct cache_entry *ce; + + npath = dfs_cache_canonical_path(path, cp, remap); + if (IS_ERR(npath)) + return PTR_ERR(npath); + + ce = cache_refresh_path(xid, ses, npath, false); + if (IS_ERR(ce)) { + rc = PTR_ERR(ce); + goto out_free_path; + } + + if (ref) + rc = setup_referral(path, ce, ref, get_tgt_name(ce)); + else + rc = 0; + if (!rc && tgt_list) + rc = get_targets(ce, tgt_list); + + up_read(&htable_rw_lock); + +out_free_path: + kfree(npath); + return rc; +} + +/** + * dfs_cache_noreq_find - find a DFS cache entry without sending any requests to + * the currently connected server. + * + * NOTE: This function will neither update a cache entry in case it was + * expired, nor create a new cache entry if @path hasn't been found. It heavily + * relies on an existing cache entry. + * + * @path: canonical DFS path to lookup in the DFS referral cache. + * @ref: when non-NULL, store single DFS referral result in it. + * @tgt_list: when non-NULL, store complete DFS target list in it. + * + * Return 0 if successful. + * Return -ENOENT if the entry was not found. + * Return non-zero for other errors. + */ +int dfs_cache_noreq_find(const char *path, struct dfs_info3_param *ref, + struct dfs_cache_tgt_list *tgt_list) +{ + int rc; + struct cache_entry *ce; + + cifs_dbg(FYI, "%s: path: %s\n", __func__, path); + + down_read(&htable_rw_lock); + + ce = lookup_cache_entry(path); + if (IS_ERR(ce)) { + rc = PTR_ERR(ce); + goto out_unlock; + } + + if (ref) + rc = setup_referral(path, ce, ref, get_tgt_name(ce)); + else + rc = 0; + if (!rc && tgt_list) + rc = get_targets(ce, tgt_list); + +out_unlock: + up_read(&htable_rw_lock); + return rc; +} + +/** + * dfs_cache_noreq_update_tgthint - update target hint of a DFS cache entry + * without sending any requests to the currently connected server. + * + * NOTE: This function will neither update a cache entry in case it was + * expired, nor create a new cache entry if @path hasn't been found. It heavily + * relies on an existing cache entry. + * + * @path: canonical DFS path to lookup in DFS referral cache. + * @it: target iterator which contains the target hint to update the cache + * entry with. + * + * Return zero if the target hint was updated successfully, otherwise non-zero. + */ +void dfs_cache_noreq_update_tgthint(const char *path, const struct dfs_cache_tgt_iterator *it) +{ + struct cache_dfs_tgt *t; + struct cache_entry *ce; + + if (!path || !it) + return; + + cifs_dbg(FYI, "%s: path: %s\n", __func__, path); + + down_read(&htable_rw_lock); + + ce = lookup_cache_entry(path); + if (IS_ERR(ce)) + goto out_unlock; + + t = READ_ONCE(ce->tgthint); + + if (unlikely(!strcasecmp(it->it_name, t->name))) + goto out_unlock; + + list_for_each_entry(t, &ce->tlist, list) { + if (!strcasecmp(t->name, it->it_name)) { + WRITE_ONCE(ce->tgthint, t); + cifs_dbg(FYI, "%s: new target hint: %s\n", __func__, + it->it_name); + break; + } + } + +out_unlock: + up_read(&htable_rw_lock); +} + +/** + * dfs_cache_get_tgt_referral - returns a DFS referral (@ref) from a given + * target iterator (@it). + * + * @path: canonical DFS path to lookup in DFS referral cache. + * @it: DFS target iterator. + * @ref: DFS referral pointer to set up the gathered information. + * + * Return zero if the DFS referral was set up correctly, otherwise non-zero. + */ +int dfs_cache_get_tgt_referral(const char *path, const struct dfs_cache_tgt_iterator *it, + struct dfs_info3_param *ref) +{ + int rc; + struct cache_entry *ce; + + if (!it || !ref) + return -EINVAL; + + cifs_dbg(FYI, "%s: path: %s\n", __func__, path); + + down_read(&htable_rw_lock); + + ce = lookup_cache_entry(path); + if (IS_ERR(ce)) { + rc = PTR_ERR(ce); + goto out_unlock; + } + + cifs_dbg(FYI, "%s: target name: %s\n", __func__, it->it_name); + + rc = setup_referral(path, ce, ref, it->it_name); + +out_unlock: + up_read(&htable_rw_lock); + return rc; +} + +/* Extract share from DFS target and return a pointer to prefix path or NULL */ +static const char *parse_target_share(const char *target, char **share) +{ + const char *s, *seps = "/\\"; + size_t len; + + s = strpbrk(target + 1, seps); + if (!s) + return ERR_PTR(-EINVAL); + + len = strcspn(s + 1, seps); + if (!len) + return ERR_PTR(-EINVAL); + s += len; + + len = s - target + 1; + *share = kstrndup(target, len, GFP_KERNEL); + if (!*share) + return ERR_PTR(-ENOMEM); + + s = target + len; + return s + strspn(s, seps); +} + +/** + * dfs_cache_get_tgt_share - parse a DFS target + * + * @path: DFS full path + * @it: DFS target iterator. + * @share: tree name. + * @prefix: prefix path. + * + * Return zero if target was parsed correctly, otherwise non-zero. + */ +int dfs_cache_get_tgt_share(char *path, const struct dfs_cache_tgt_iterator *it, char **share, + char **prefix) +{ + char sep; + char *target_share; + char *ppath = NULL; + const char *target_ppath, *dfsref_ppath; + size_t target_pplen, dfsref_pplen; + size_t len, c; + + if (!it || !path || !share || !prefix || strlen(path) < it->it_path_consumed) + return -EINVAL; + + sep = it->it_name[0]; + if (sep != '\\' && sep != '/') + return -EINVAL; + + target_ppath = parse_target_share(it->it_name, &target_share); + if (IS_ERR(target_ppath)) + return PTR_ERR(target_ppath); + + /* point to prefix in DFS referral path */ + dfsref_ppath = path + it->it_path_consumed; + dfsref_ppath += strspn(dfsref_ppath, "/\\"); + + target_pplen = strlen(target_ppath); + dfsref_pplen = strlen(dfsref_ppath); + + /* merge prefix paths from DFS referral path and target node */ + if (target_pplen || dfsref_pplen) { + len = target_pplen + dfsref_pplen + 2; + ppath = kzalloc(len, GFP_KERNEL); + if (!ppath) { + kfree(target_share); + return -ENOMEM; + } + c = strscpy(ppath, target_ppath, len); + if (c && dfsref_pplen) + ppath[c] = sep; + strlcat(ppath, dfsref_ppath, len); + } + *share = target_share; + *prefix = ppath; + return 0; +} + +static bool target_share_equal(struct cifs_tcon *tcon, const char *s1) +{ + struct TCP_Server_Info *server = tcon->ses->server; + const char *s2 = &tcon->tree_name[1]; + struct sockaddr_storage ss; + bool match; + int rc; + + if (strcasecmp(s2, s1)) + return false; + + /* + * Resolve share's hostname and check if server address matches. Otherwise just ignore it + * as we could not have upcall to resolve hostname or failed to convert ip address. + */ + rc = dns_resolve_unc(server->dns_dom, s1, (struct sockaddr *)&ss); + if (rc < 0) + return true; + + cifs_server_lock(server); + match = cifs_match_ipaddr((struct sockaddr *)&server->dstaddr, (struct sockaddr *)&ss); + cifs_dbg(FYI, "%s: [share=%s] ipaddr matched: %s\n", __func__, s1, str_yes_no(match)); + cifs_server_unlock(server); + + return match; +} + +static bool is_ses_good(struct cifs_tcon *tcon, struct cifs_ses *ses) +{ + struct TCP_Server_Info *server = ses->server; + struct cifs_tcon *ipc = NULL; + bool ret; + + spin_lock(&cifs_tcp_ses_lock); + spin_lock(&ses->ses_lock); + spin_lock(&ses->chan_lock); + + ret = !cifs_chan_needs_reconnect(ses, server) && + ses->ses_status == SES_GOOD; + + spin_unlock(&ses->chan_lock); + + if (!ret) + goto out; + + if (likely(ses->tcon_ipc)) { + if (ses->tcon_ipc->need_reconnect) { + ret = false; + goto out; + } + } else { + spin_unlock(&ses->ses_lock); + spin_unlock(&cifs_tcp_ses_lock); + + ipc = cifs_setup_ipc(ses, tcon->seal); + + spin_lock(&cifs_tcp_ses_lock); + spin_lock(&ses->ses_lock); + if (!IS_ERR(ipc)) { + if (!ses->tcon_ipc) { + ses->tcon_ipc = ipc; + ipc = NULL; + } + } else { + ret = false; + ipc = NULL; + } + } + +out: + spin_unlock(&ses->ses_lock); + spin_unlock(&cifs_tcp_ses_lock); + if (ipc && server->ops->tree_disconnect) { + unsigned int xid = get_xid(); + + (void)server->ops->tree_disconnect(xid, ipc); + _free_xid(xid); + } + tconInfoFree(ipc, netfs_trace_tcon_ref_free_ipc); + return ret; +} + +/* Refresh dfs referral of @ses */ +static void refresh_ses_referral(struct cifs_tcon *tcon, struct cifs_ses *ses) +{ + struct cache_entry *ce; + unsigned int xid; + const char *path; + int rc = 0; + + xid = get_xid(); + + path = dfs_ses_refpath(ses); + if (IS_ERR(path)) { + rc = PTR_ERR(path); + goto out; + } + + ses = CIFS_DFS_ROOT_SES(ses); + if (!is_ses_good(tcon, ses)) { + cifs_dbg(FYI, "%s: skip cache refresh due to disconnected ipc\n", + __func__); + goto out; + } + + ce = cache_refresh_path(xid, ses, path, false); + if (!IS_ERR(ce)) + up_read(&htable_rw_lock); + else + rc = PTR_ERR(ce); + +out: + free_xid(xid); +} + +static int __refresh_tcon_referral(struct cifs_tcon *tcon, + const char *path, + struct dfs_info3_param *refs, + int numrefs, bool force_refresh) +{ + struct cache_entry *ce; + bool reconnect = force_refresh; + int rc = 0; + int i; + + if (unlikely(!numrefs)) + return 0; + + if (force_refresh) { + for (i = 0; i < numrefs; i++) { + /* TODO: include prefix paths in the matching */ + if (target_share_equal(tcon, refs[i].node_name)) { + reconnect = false; + break; + } + } + } + + down_write(&htable_rw_lock); + ce = lookup_cache_entry(path); + if (!IS_ERR(ce)) { + if (force_refresh || cache_entry_expired(ce)) + rc = update_cache_entry_locked(ce, refs, numrefs); + } else if (PTR_ERR(ce) == -ENOENT) { + ce = add_cache_entry_locked(refs, numrefs); + } + up_write(&htable_rw_lock); + + if (IS_ERR(ce)) + rc = PTR_ERR(ce); + if (reconnect) { + cifs_tcon_dbg(FYI, "%s: mark for reconnect\n", __func__); + cifs_signal_cifsd_for_reconnect(tcon->ses->server, true); + } + return rc; +} + +static void refresh_tcon_referral(struct cifs_tcon *tcon, bool force_refresh) +{ + struct dfs_info3_param *refs = NULL; + struct cache_entry *ce; + struct cifs_ses *ses; + bool needs_refresh; + const char *path; + unsigned int xid; + int numrefs = 0; + int rc = 0; + + xid = get_xid(); + ses = tcon->ses; + + path = dfs_ses_refpath(ses); + if (IS_ERR(path)) { + rc = PTR_ERR(path); + goto out; + } + + down_read(&htable_rw_lock); + ce = lookup_cache_entry(path); + needs_refresh = force_refresh || IS_ERR(ce) || cache_entry_expired(ce); + if (!needs_refresh) { + up_read(&htable_rw_lock); + goto out; + } + up_read(&htable_rw_lock); + + ses = CIFS_DFS_ROOT_SES(ses); + if (!is_ses_good(tcon, ses)) { + cifs_dbg(FYI, "%s: skip cache refresh due to disconnected ipc\n", + __func__); + goto out; + } + + rc = get_dfs_referral(xid, ses, path, &refs, &numrefs); + if (!rc) { + rc = __refresh_tcon_referral(tcon, path, refs, + numrefs, force_refresh); + } + +out: + free_xid(xid); + free_dfs_info_array(refs, numrefs); +} + +/** + * dfs_cache_remount_fs - remount a DFS share + * + * Reconfigure dfs mount by forcing a new DFS referral and if the currently cached targets do not + * match any of the new targets, mark it for reconnect. + * + * @cifs_sb: cifs superblock. + * + * Return zero if remounted, otherwise non-zero. + */ +int dfs_cache_remount_fs(struct cifs_sb_info *cifs_sb) +{ + struct cifs_tcon *tcon; + + if (!cifs_sb || !cifs_sb->master_tlink) + return -EINVAL; + + tcon = cifs_sb_master_tcon(cifs_sb); + + spin_lock(&tcon->tc_lock); + if (!tcon->origin_fullpath) { + spin_unlock(&tcon->tc_lock); + cifs_dbg(FYI, "%s: not a dfs mount\n", __func__); + return 0; + } + spin_unlock(&tcon->tc_lock); + + /* + * After reconnecting to a different server, unique ids won't match anymore, so we disable + * serverino. This prevents dentry revalidation to think the dentry are stale (ESTALE). + */ + cifs_autodisable_serverino(cifs_sb, "DFS failover may potentially connect to a different server, inode numbers won't match anymore", 0); + /* + * Force the use of prefix path to support failover on DFS paths that resolve to targets + * that have different prefix paths. + */ + atomic_or(CIFS_MOUNT_USE_PREFIX_PATH, &cifs_sb->mnt_cifs_flags); + + refresh_tcon_referral(tcon, true); + return 0; +} + +/* Refresh all DFS referrals related to DFS tcon */ +void dfs_cache_refresh(struct work_struct *work) +{ + struct cifs_tcon *tcon; + struct cifs_ses *ses; + + tcon = container_of(work, struct cifs_tcon, dfs_cache_work.work); + + list_for_each_entry(ses, &tcon->dfs_ses_list, dlist) + refresh_ses_referral(tcon, ses); + refresh_tcon_referral(tcon, false); + + queue_delayed_work(dfscache_wq, &tcon->dfs_cache_work, + atomic_read(&dfs_cache_ttl) * HZ); +} diff --git a/fs/smb/client/dfs_cache.h b/fs/smb/client/dfs_cache.h new file mode 100644 index 000000000..c99dc3546 --- /dev/null +++ b/fs/smb/client/dfs_cache.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * DFS referral cache routines + * + * Copyright (c) 2018-2019 Paulo Alcantara + */ + +#ifndef _CIFS_DFS_CACHE_H +#define _CIFS_DFS_CACHE_H + +#include +#include +#include +#include "cifsglob.h" + +extern struct workqueue_struct *dfscache_wq; +extern atomic_t dfs_cache_ttl; + +#define DFS_CACHE_TGT_LIST_INIT(var) \ + { .tl_numtgts = 0, .tl_list = LIST_HEAD_INIT((var).tl_list), } + +#define DFS_CACHE_TGT_LIST(var) \ + struct dfs_cache_tgt_list var = DFS_CACHE_TGT_LIST_INIT(var) + +struct dfs_cache_tgt_list { + int tl_numtgts; + struct list_head tl_list; +}; + +struct dfs_cache_tgt_iterator { + char *it_name; + int it_path_consumed; + struct list_head it_list; +}; + +int dfs_cache_init(void); +void dfs_cache_destroy(void); +extern const struct proc_ops dfscache_proc_ops; + +int dfs_cache_find(const unsigned int xid, struct cifs_ses *ses, + const struct nls_table *cp, int remap, const char *path, + struct dfs_info3_param *ref, + struct dfs_cache_tgt_list *tgt_list); +int dfs_cache_noreq_find(const char *path, struct dfs_info3_param *ref, + struct dfs_cache_tgt_list *tgt_list); +void dfs_cache_noreq_update_tgthint(const char *path, + const struct dfs_cache_tgt_iterator *it); +int dfs_cache_get_tgt_referral(const char *path, + const struct dfs_cache_tgt_iterator *it, + struct dfs_info3_param *ref); +int dfs_cache_get_tgt_share(char *path, + const struct dfs_cache_tgt_iterator *it, + char **share, char **prefix); +char *dfs_cache_canonical_path(const char *path, const struct nls_table *cp, + int remap); +int dfs_cache_remount_fs(struct cifs_sb_info *cifs_sb); +void dfs_cache_refresh(struct work_struct *work); + +static inline struct dfs_cache_tgt_iterator * +dfs_cache_get_next_tgt(struct dfs_cache_tgt_list *tl, + struct dfs_cache_tgt_iterator *it) +{ + if (!tl || !tl->tl_numtgts || list_empty(&tl->tl_list) || + !it || list_is_last(&it->it_list, &tl->tl_list)) + return NULL; + return list_next_entry(it, it_list); +} + +static inline struct dfs_cache_tgt_iterator * +dfs_cache_get_tgt_iterator(struct dfs_cache_tgt_list *tl) +{ + if (!tl) + return NULL; + return list_first_entry_or_null(&tl->tl_list, + struct dfs_cache_tgt_iterator, + it_list); +} + +static inline void dfs_cache_free_tgts(struct dfs_cache_tgt_list *tl) +{ + struct dfs_cache_tgt_iterator *it, *nit; + + if (!tl || !tl->tl_numtgts || list_empty(&tl->tl_list)) + return; + list_for_each_entry_safe(it, nit, &tl->tl_list, it_list) { + list_del(&it->it_list); + kfree(it->it_name); + kfree(it); + } + tl->tl_numtgts = 0; +} + +static inline const char * +dfs_cache_get_tgt_name(const struct dfs_cache_tgt_iterator *it) +{ + return it ? it->it_name : NULL; +} + +static inline int +dfs_cache_get_nr_tgts(const struct dfs_cache_tgt_list *tl) +{ + return tl ? tl->tl_numtgts : 0; +} + +static inline int dfs_cache_get_ttl(void) +{ + return atomic_read(&dfs_cache_ttl); +} + +#endif /* _CIFS_DFS_CACHE_H */ diff --git a/fs/smb/client/dir.c b/fs/smb/client/dir.c new file mode 100644 index 000000000..7803bd5bd --- /dev/null +++ b/fs/smb/client/dir.c @@ -0,0 +1,1212 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * vfs operations that deal with dentries + * + * Copyright (C) International Business Machines Corp., 2002,2009 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ +#include +#include +#include +#include +#include +#include +#include "cifsfs.h" +#include "cifsglob.h" +#include "cifsproto.h" +#include "cifs_debug.h" +#include "cifs_fs_sb.h" +#include "cifs_unicode.h" +#include "fs_context.h" +#include "cifs_ioctl.h" +#include "fscache.h" +#include "cached_dir.h" + +static void +renew_parental_timestamps(struct dentry *direntry) +{ + /* BB check if there is a way to get the kernel to do this or if we + really need this */ + do { + cifs_set_time(direntry, jiffies); + direntry = direntry->d_parent; + } while (!IS_ROOT(direntry)); +} + +char * +cifs_build_path_to_root(struct smb3_fs_context *ctx, struct cifs_sb_info *cifs_sb, + struct cifs_tcon *tcon, int add_treename) +{ + int pplen = ctx->prepath ? strlen(ctx->prepath) + 1 : 0; + int dfsplen; + char *full_path = NULL; + + /* if no prefix path, simply set path to the root of share to "" */ + if (pplen == 0) { + full_path = kzalloc(1, GFP_KERNEL); + return full_path; + } + + if (add_treename) + dfsplen = strnlen(tcon->tree_name, MAX_TREE_SIZE + 1); + else + dfsplen = 0; + + full_path = kmalloc(dfsplen + pplen + 1, GFP_KERNEL); + if (full_path == NULL) + return full_path; + + if (dfsplen) + memcpy(full_path, tcon->tree_name, dfsplen); + full_path[dfsplen] = CIFS_DIR_SEP(cifs_sb); + memcpy(full_path + dfsplen + 1, ctx->prepath, pplen); + convert_delimiter(full_path, CIFS_DIR_SEP(cifs_sb)); + return full_path; +} + +/* Note: caller must free return buffer */ +const char * +build_path_from_dentry(struct dentry *direntry, void *page) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(direntry->d_sb); + struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb); + bool prefix = tcon->Flags & SMB_SHARE_IS_IN_DFS; + + return build_path_from_dentry_optional_prefix(direntry, page, + prefix); +} + +char *__build_path_from_dentry_optional_prefix(struct dentry *direntry, void *page, + const char *tree, int tree_len, + bool prefix) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(direntry); + unsigned int sbflags = cifs_sb_flags(cifs_sb); + char dirsep = CIFS_DIR_SEP(cifs_sb); + int pplen = 0; + int dfsplen; + char *s; + + if (unlikely(!page)) + return ERR_PTR(-ENOMEM); + + if (prefix) + dfsplen = strnlen(tree, tree_len + 1); + else + dfsplen = 0; + + if (sbflags & CIFS_MOUNT_USE_PREFIX_PATH) + pplen = cifs_sb->prepath ? strlen(cifs_sb->prepath) + 1 : 0; + + s = dentry_path_raw(direntry, page, PATH_MAX); + if (IS_ERR(s)) + return s; + if (!s[1]) // for root we want "", not "/" + s++; + if (s < (char *)page + pplen + dfsplen) + return ERR_PTR(-ENAMETOOLONG); + if (pplen) { + cifs_dbg(FYI, "using cifs_sb prepath <%s>\n", cifs_sb->prepath); + s -= pplen; + memcpy(s + 1, cifs_sb->prepath, pplen - 1); + *s = '/'; + } + if (dirsep != '/') { + /* BB test paths to Windows with '/' in the midst of prepath */ + char *p; + + for (p = s; *p; p++) + if (*p == '/') + *p = dirsep; + } + if (dfsplen) { + s -= dfsplen; + memcpy(s, tree, dfsplen); + if (sbflags & CIFS_MOUNT_POSIX_PATHS) { + int i; + for (i = 0; i < dfsplen; i++) { + if (s[i] == '\\') + s[i] = '/'; + } + } + } + return s; +} + +char *build_path_from_dentry_optional_prefix(struct dentry *direntry, void *page, + bool prefix) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(direntry->d_sb); + struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb); + + return __build_path_from_dentry_optional_prefix(direntry, page, tcon->tree_name, + MAX_TREE_SIZE, prefix); +} + +/* + * Don't allow path components longer than the server max. + * Don't allow the separator character in a path component. + * The VFS will not allow "/", but "\" is allowed by posix. + */ +static int +check_name(struct dentry *direntry, struct cifs_tcon *tcon) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(direntry); + int i; + + if (unlikely(tcon->fsAttrInfo.MaxPathNameComponentLength && + direntry->d_name.len > + le32_to_cpu(tcon->fsAttrInfo.MaxPathNameComponentLength))) + return -ENAMETOOLONG; + + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_POSIX_PATHS)) { + for (i = 0; i < direntry->d_name.len; i++) { + if (direntry->d_name.name[i] == '\\') { + cifs_dbg(FYI, "Invalid file name\n"); + return -EINVAL; + } + } + } + return 0; +} + +static char *alloc_parent_path(struct dentry *dentry, size_t namelen) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(dentry); + void *page = alloc_dentry_path(); + const char *path; + size_t size; + char *npath; + + path = build_path_from_dentry(dentry->d_parent, page); + if (IS_ERR(path)) { + npath = ERR_CAST(path); + goto out; + } + + size = strlen(path) + namelen + 2; + npath = kmalloc(size, GFP_KERNEL); + if (!npath) + npath = ERR_PTR(-ENOMEM); + else + scnprintf(npath, size, "%s%c", path, CIFS_DIR_SEP(cifs_sb)); +out: + free_dentry_path(page); + return npath; +} + +/* Inode operations in similar order to how they appear in Linux file fs.h */ +static int __cifs_do_create(struct inode *dir, struct dentry *direntry, + const char *full_path, unsigned int xid, + struct tcon_link *tlink, unsigned int oflags, + umode_t mode, __u32 *oplock, struct cifs_fid *fid, + struct cifs_open_info_data *buf, + struct inode **inode) +{ + int rc = -ENOENT; + int create_options = CREATE_NOT_DIR; + int desired_access; + struct cifs_sb_info *cifs_sb = CIFS_SB(dir); + struct cifs_tcon *tcon = tlink_tcon(tlink); + struct inode *newinode = NULL; + unsigned int sbflags = cifs_sb_flags(cifs_sb); + int disposition; + struct TCP_Server_Info *server = tcon->ses->server; + struct cifs_open_parms oparms; + struct cached_fid *parent_cfid = NULL; + int rdwr_for_fscache = 0; + __le32 lease_flags = 0; + + *inode = NULL; + *oplock = 0; + if (tcon->ses->server->oplocks) + *oplock = REQ_OPLOCK; + + /* If we're caching, we need to be able to fill in around partial writes. */ + if (cifs_fscache_enabled(dir) && (oflags & O_ACCMODE) == O_WRONLY) + rdwr_for_fscache = 1; + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + if (tcon->unix_ext && cap_unix(tcon->ses) && !tcon->broken_posix_open && + (CIFS_UNIX_POSIX_PATH_OPS_CAP & + le64_to_cpu(tcon->fsUnixInfo.Capability))) { + rc = cifs_posix_open(full_path, &newinode, dir->i_sb, mode, + oflags, oplock, &fid->netfid, xid); + switch (rc) { + case 0: + if (newinode == NULL) { + /* query inode info */ + goto cifs_create_get_file_info; + } + + if ((oflags & __O_REGULAR) && !S_ISREG(newinode->i_mode)) { + CIFSSMBClose(xid, tcon, fid->netfid); + iput(newinode); + return -EFTYPE; + } + + if (S_ISDIR(newinode->i_mode)) { + CIFSSMBClose(xid, tcon, fid->netfid); + iput(newinode); + return -EISDIR; + } + + if (!S_ISREG(newinode->i_mode)) { + /* + * The server may allow us to open things like + * FIFOs, but the client isn't set up to deal + * with that. If it's not a regular file, just + * close it and proceed as if it were a normal + * lookup. + */ + CIFSSMBClose(xid, tcon, fid->netfid); + goto cifs_create_get_file_info; + } + /* success, no need to query */ + goto cifs_create_set_dentry; + + case -ENOENT: + goto cifs_create_get_file_info; + + case -EIO: + case -EINVAL: + /* + * EIO could indicate that (posix open) operation is not + * supported, despite what server claimed in capability + * negotiation. + * + * POSIX open in samba versions 3.3.1 and earlier could + * incorrectly fail with invalid parameter. + */ + tcon->broken_posix_open = true; + break; + + case -EREMOTE: + case -EOPNOTSUPP: + /* + * EREMOTE indicates DFS junction, which is not handled + * in posix open. If either that or op not supported + * returned, follow the normal lookup. + */ + break; + + default: + return rc; + } + /* + * fallthrough to retry, using older open call, this is case + * where server does not support this SMB level, and falsely + * claims capability (also get here for DFS case which should be + * rare for path not covered on files) + */ + } +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + + desired_access = 0; + if (OPEN_FMODE(oflags) & FMODE_READ) + desired_access |= GENERIC_READ; /* is this too little? */ + if (OPEN_FMODE(oflags) & FMODE_WRITE) + desired_access |= GENERIC_WRITE; + if (rdwr_for_fscache == 1) + desired_access |= GENERIC_READ; + if (oflags & O_TMPFILE) + desired_access |= DELETE; + + disposition = FILE_OVERWRITE_IF; + if (oflags & O_CREAT) { + if (oflags & O_EXCL) + disposition = FILE_CREATE; + else if (oflags & O_TRUNC) + disposition = FILE_OVERWRITE_IF; + else + disposition = FILE_OPEN_IF; + } else if (oflags & O_TMPFILE) { + disposition = FILE_CREATE; + } else { + cifs_dbg(FYI, "Create flag not set in create function\n"); + } + + /* + * BB add processing to set equivalent of mode - e.g. via CreateX with + * ACLs + */ + + if (!server->ops->open) + return -EOPNOTSUPP; + + create_options |= cifs_open_create_options(oflags, create_options); + /* + * if we're not using unix extensions, see if we need to set + * ATTR_READONLY on the create call + */ + if (!tcon->unix_ext && (mode & S_IWUGO) == 0) + create_options |= CREATE_OPTION_READONLY; + + +retry_open: + if (tcon->cfids && direntry->d_parent && server->dialect >= SMB30_PROT_ID) { + parent_cfid = NULL; + spin_lock(&tcon->cfids->cfid_list_lock); + list_for_each_entry(parent_cfid, &tcon->cfids->entries, entry) { + if (parent_cfid->dentry == direntry->d_parent) { + cifs_dbg(FYI, "found a parent cached file handle\n"); + if (is_valid_cached_dir(parent_cfid)) { + lease_flags + |= SMB2_LEASE_FLAG_PARENT_LEASE_KEY_SET_LE; + memcpy(fid->parent_lease_key, + parent_cfid->fid.lease_key, + SMB2_LEASE_KEY_SIZE); + parent_cfid->dirents.is_valid = false; + parent_cfid->dirents.is_failed = true; + } + break; + } + } + spin_unlock(&tcon->cfids->cfid_list_lock); + } + + oparms = (struct cifs_open_parms) { + .tcon = tcon, + .cifs_sb = cifs_sb, + .desired_access = desired_access, + .create_options = cifs_create_options(cifs_sb, create_options), + .disposition = disposition, + .path = full_path, + .fid = fid, + .lease_flags = lease_flags, + .mode = mode, + }; + rc = server->ops->open(xid, &oparms, oplock, buf); + if (rc) { + cifs_dbg(FYI, "cifs_create returned 0x%x\n", rc); + if (rc == -EACCES && rdwr_for_fscache == 1) { + desired_access &= ~GENERIC_READ; + rdwr_for_fscache = 2; + goto retry_open; + } + return rc; + } + if (rdwr_for_fscache == 2) + cifs_invalidate_cache(dir, FSCACHE_INVAL_DIO_WRITE); + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + /* + * If Open reported that we actually created a file then we now have to + * set the mode if possible. + */ + if ((tcon->unix_ext) && (*oplock & CIFS_CREATE_ACTION)) { + struct cifs_unix_set_info_args args = { + .mode = mode, + .ctime = NO_CHANGE_64, + .atime = NO_CHANGE_64, + .mtime = NO_CHANGE_64, + .device = 0, + }; + + if (sbflags & CIFS_MOUNT_SET_UID) { + args.uid = current_fsuid(); + if (dir->i_mode & S_ISGID) + args.gid = dir->i_gid; + else + args.gid = current_fsgid(); + } else { + args.uid = INVALID_UID; /* no change */ + args.gid = INVALID_GID; /* no change */ + } + CIFSSMBUnixSetFileInfo(xid, tcon, &args, fid->netfid, + current->tgid); + } else { + /* + * BB implement mode setting via Windows security + * descriptors e.g. + */ + /* CIFSSMBWinSetPerms(xid,tcon,path,mode,-1,-1,nls);*/ + + /* Could set r/o dos attribute if mode & 0222 == 0 */ + } + +cifs_create_get_file_info: + /* server might mask mode so we have to query for it */ + if (tcon->unix_ext) + rc = cifs_get_inode_info_unix(&newinode, full_path, dir->i_sb, + xid); + else { +#else + { +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + /* TODO: Add support for calling POSIX query info here, but passing in fid */ + rc = cifs_get_inode_info(&newinode, full_path, buf, dir->i_sb, xid, fid); + if (newinode) { + if (server->ops->set_lease_key) + server->ops->set_lease_key(newinode, fid); + if ((*oplock & CIFS_CREATE_ACTION) && S_ISREG(newinode->i_mode)) { + if (sbflags & CIFS_MOUNT_DYNPERM) + newinode->i_mode = mode; + if (sbflags & CIFS_MOUNT_SET_UID) { + newinode->i_uid = current_fsuid(); + if (dir->i_mode & S_ISGID) + newinode->i_gid = dir->i_gid; + else + newinode->i_gid = current_fsgid(); + } + } + } + } + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY +cifs_create_set_dentry: +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + if (rc != 0) { + cifs_dbg(FYI, "Create worked, get_inode_info failed rc = %d\n", + rc); + goto out_err; + } + + if (newinode) { + if ((oflags & __O_REGULAR) && !S_ISREG(newinode->i_mode)) { + rc = -EFTYPE; + goto out_err; + } + if (S_ISDIR(newinode->i_mode)) { + rc = -EISDIR; + goto out_err; + } + } + + *inode = newinode; + return rc; + +out_err: + if (server->ops->close) + server->ops->close(xid, tcon, fid); + if (newinode) + iput(newinode); + return rc; +} + +static int cifs_do_create(struct inode *dir, struct dentry *direntry, + unsigned int xid, struct tcon_link *tlink, + unsigned int oflags, umode_t mode, + __u32 *oplock, struct cifs_fid *fid, + struct cifs_open_info_data *buf, + struct inode **inode) +{ + void *page = alloc_dentry_path(); + const char *full_path; + int rc; + + full_path = build_path_from_dentry(direntry, page); + if (IS_ERR(full_path)) { + rc = PTR_ERR(full_path); + } else { + rc = __cifs_do_create(dir, direntry, full_path, xid, + tlink, oflags, mode, oplock, + fid, buf, inode); + } + free_dentry_path(page); + return rc; +} + + +/* + * Look up, create and open a CIFS file. + * + * The initial dentry state is in-lookup or hashed-negative. On success, dentry + * will become hashed-positive by calling d_splice_alias() if in-lookup, + * otherwise d_instantiate(). + */ +int cifs_atomic_open(struct inode *dir, struct dentry *direntry, + struct file *file, unsigned int oflags, umode_t mode) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(dir); + struct cifs_open_info_data buf = {}; + struct TCP_Server_Info *server; + struct cifsFileInfo *file_info; + struct cifs_pending_open open; + struct cifs_fid fid = {}; + struct tcon_link *tlink; + struct cifs_tcon *tcon; + unsigned int sbflags; + struct dentry *alias; + struct inode *inode; + unsigned int xid; + __u32 oplock; + int rc; + + if (unlikely(cifs_forced_shutdown(cifs_sb))) + return smb_EIO(smb_eio_trace_forced_shutdown); + + /* + * Posix open is only called (at lookup time) for file create now. For + * opens (rather than creates), because we do not know if it is a file + * or directory yet, and current Samba no longer allows us to do posix + * open on dirs, we could end up wasting an open call on what turns out + * to be a dir. For file opens, we wait to call posix open till + * cifs_open. It could be added to atomic_open in the future but the + * performance tradeoff of the extra network request when EISDIR or + * EACCES is returned would have to be weighed against the 50% reduction + * in network traffic in the other paths. + */ + if (!(oflags & O_CREAT)) { + /* + * Check for hashed negative dentry. We have already revalidated + * the dentry and it is fine. No need to perform another lookup. + */ + if (!d_in_lookup(direntry)) + return -ENOENT; + + return finish_no_open(file, cifs_lookup(dir, direntry, 0)); + } + + xid = get_xid(); + + cifs_dbg(FYI, "parent inode = 0x%p name is: %pd and dentry = 0x%p\n", + dir, direntry, direntry); + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) { + rc = PTR_ERR(tlink); + goto out_free_xid; + } + + tcon = tlink_tcon(tlink); + + rc = check_name(direntry, tcon); + if (rc) + goto out; + + server = tcon->ses->server; + + if (server->ops->new_lease_key) + server->ops->new_lease_key(&fid); + + cifs_add_pending_open(&fid, tlink, &open); + + rc = cifs_do_create(dir, direntry, xid, tlink, oflags, mode, + &oplock, &fid, &buf, &inode); + if (rc) { + cifs_del_pending_open(&open); + goto out; + } + + if (d_in_lookup(direntry)) { + alias = d_splice_alias(inode, direntry); + if (!IS_ERR_OR_NULL(alias)) + direntry = alias; + } else { + d_instantiate(direntry, inode); + } + + if ((oflags & (O_CREAT | O_EXCL)) == (O_CREAT | O_EXCL)) + file->f_mode |= FMODE_CREATED; + + rc = finish_open(file, direntry, generic_file_open); + if (rc) { + if (server->ops->close) + server->ops->close(xid, tcon, &fid); + cifs_del_pending_open(&open); + goto out; + } + + sbflags = cifs_sb_flags(cifs_sb); + if ((file->f_flags & O_DIRECT) && (sbflags & CIFS_MOUNT_STRICT_IO)) { + if (sbflags & CIFS_MOUNT_NO_BRL) + file->f_op = &cifs_file_direct_nobrl_ops; + else + file->f_op = &cifs_file_direct_ops; + } + + file_info = cifs_new_fileinfo(&fid, file, tlink, oplock, buf.symlink_target); + if (file_info == NULL) { + if (server->ops->close) + server->ops->close(xid, tcon, &fid); + cifs_del_pending_open(&open); + rc = -ENOMEM; + goto out; + } + + fscache_use_cookie(cifs_inode_cookie(file_inode(file)), + file->f_mode & FMODE_WRITE); + +out: + cifs_put_tlink(tlink); +out_free_xid: + free_xid(xid); + cifs_free_open_info(&buf); + return rc; +} + +/* + * Create a CIFS file. + * + * The initial dentry state is hashed-negative. On success, dentry will become + * hashed-positive by calling d_instantiate(). + */ +int cifs_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *direntry, umode_t mode) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(dir); + int rc; + unsigned int xid = get_xid(); + /* + * BB below access is probably too much for mknod to request + * but we have to do query and setpathinfo so requesting + * less could fail (unless we want to request getatr and setatr + * permissions (only). At least for POSIX we do not have to + * request so much. + */ + unsigned oflags = O_EXCL | O_CREAT | O_RDWR; + struct tcon_link *tlink; + struct cifs_tcon *tcon; + struct TCP_Server_Info *server; + struct inode *inode; + struct cifs_fid fid; + __u32 oplock; + struct cifs_open_info_data buf = {}; + + cifs_dbg(FYI, "cifs_create parent inode = 0x%p name is: %pd and dentry = 0x%p\n", + dir, direntry, direntry); + + if (unlikely(cifs_forced_shutdown(cifs_sb))) { + rc = smb_EIO(smb_eio_trace_forced_shutdown); + goto out_free_xid; + } + + tlink = cifs_sb_tlink(cifs_sb); + rc = PTR_ERR(tlink); + if (IS_ERR(tlink)) + goto out_free_xid; + + tcon = tlink_tcon(tlink); + server = tcon->ses->server; + + if (server->ops->new_lease_key) + server->ops->new_lease_key(&fid); + + rc = cifs_do_create(dir, direntry, xid, tlink, oflags, + mode, &oplock, &fid, &buf, &inode); + if (!rc) { + d_instantiate(direntry, inode); + if (server->ops->close) + server->ops->close(xid, tcon, &fid); + } + + cifs_free_open_info(&buf); + cifs_put_tlink(tlink); +out_free_xid: + free_xid(xid); + return rc; +} + +int cifs_mknod(struct mnt_idmap *idmap, struct inode *inode, + struct dentry *direntry, umode_t mode, dev_t device_number) +{ + int rc = -EPERM; + unsigned int xid; + struct cifs_sb_info *cifs_sb; + struct tcon_link *tlink; + struct cifs_tcon *tcon; + const char *full_path; + void *page; + + if (!old_valid_dev(device_number)) + return -EINVAL; + + cifs_sb = CIFS_SB(inode->i_sb); + if (unlikely(cifs_forced_shutdown(cifs_sb))) + return smb_EIO(smb_eio_trace_forced_shutdown); + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) + return PTR_ERR(tlink); + + page = alloc_dentry_path(); + tcon = tlink_tcon(tlink); + xid = get_xid(); + + full_path = build_path_from_dentry(direntry, page); + if (IS_ERR(full_path)) { + rc = PTR_ERR(full_path); + goto mknod_out; + } + + trace_smb3_mknod_enter(xid, tcon->tid, tcon->ses->Suid, full_path); + + rc = tcon->ses->server->ops->make_node(xid, inode, direntry, tcon, + full_path, mode, + device_number); + +mknod_out: + if (rc) + trace_smb3_mknod_err(xid, tcon->tid, tcon->ses->Suid, rc); + else + trace_smb3_mknod_done(xid, tcon->tid, tcon->ses->Suid); + + free_dentry_path(page); + free_xid(xid); + cifs_put_tlink(tlink); + return rc; +} + +struct dentry * +cifs_lookup(struct inode *parent_dir_inode, struct dentry *direntry, + unsigned int flags) +{ + unsigned int xid; + int rc = 0; /* to get around spurious gcc warning, set to zero here */ + struct cifs_sb_info *cifs_sb; + struct tcon_link *tlink; + struct cifs_tcon *pTcon; + struct inode *newInode = NULL; + const char *full_path; + void *page; + int retry_count = 0; + struct dentry *de; + + xid = get_xid(); + + cifs_dbg(FYI, "parent inode = 0x%p name is: %pd and dentry = 0x%p\n", + parent_dir_inode, direntry, direntry); + + /* check whether path exists */ + + cifs_sb = CIFS_SB(parent_dir_inode->i_sb); + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) { + de = ERR_CAST(tlink); + goto free_xid; + } + pTcon = tlink_tcon(tlink); + + rc = check_name(direntry, pTcon); + if (unlikely(rc)) { + de = ERR_PTR(rc); + goto put_tlink; + } + + /* can not grab the rename sem here since it would + deadlock in the cases (beginning of sys_rename itself) + in which we already have the sb rename sem */ + page = alloc_dentry_path(); + full_path = build_path_from_dentry(direntry, page); + if (IS_ERR(full_path)) { + de = ERR_CAST(full_path); + goto free_dentry_path; + } + + if (d_really_is_positive(direntry)) { + cifs_dbg(FYI, "non-NULL inode in lookup\n"); + } else { + struct cached_fid *cfid = NULL; + + cifs_dbg(FYI, "NULL inode in lookup\n"); + + /* + * We can only rely on negative dentries having the same + * spelling as the cached dirent if case insensitivity is + * forced on mount. + * + * XXX: if servers correctly announce Case Sensitivity Search + * on GetInfo of FileFSAttributeInformation, then we can take + * correct action even if case insensitive is not forced on + * mount. + */ + if (pTcon->nocase && !open_cached_dir_by_dentry(pTcon, direntry->d_parent, &cfid)) { + /* + * dentry is negative and parent is fully cached: + * we can assume file does not exist + */ + if (cfid->dirents.is_valid) { + close_cached_dir(cfid); + goto out; + } + close_cached_dir(cfid); + } + } + cifs_dbg(FYI, "Full path: %s inode = 0x%p\n", + full_path, d_inode(direntry)); + +again: + if (pTcon->posix_extensions) { + rc = smb311_posix_get_inode_info(&newInode, full_path, NULL, + parent_dir_inode->i_sb, xid); + } else if (pTcon->unix_ext) { + rc = cifs_get_inode_info_unix(&newInode, full_path, + parent_dir_inode->i_sb, xid); + } else { + rc = cifs_get_inode_info(&newInode, full_path, NULL, + parent_dir_inode->i_sb, xid, NULL); + } + + if (rc == 0) { + /* since paths are not looked up by component - the parent + directories are presumed to be good here */ + renew_parental_timestamps(direntry); + } else if (rc == -EAGAIN && retry_count++ < 10) { + goto again; + } else if (rc == -ENOENT) { + cifs_set_time(direntry, jiffies); + newInode = NULL; + } else { + if (rc != -EACCES) { + cifs_dbg(FYI, "Unexpected lookup error %d\n", rc); + /* We special case check for Access Denied - since that + is a common return code */ + } + newInode = ERR_PTR(rc); + } + +out: + de = d_splice_alias(newInode, direntry); +free_dentry_path: + free_dentry_path(page); +put_tlink: + cifs_put_tlink(tlink); +free_xid: + free_xid(xid); + return de; +} + +static int +cifs_d_revalidate(struct inode *dir, const struct qstr *name, + struct dentry *direntry, unsigned int flags) +{ + if (flags & LOOKUP_RCU) + return -ECHILD; + + if (d_really_is_positive(direntry)) { + int rc; + struct inode *inode = d_inode(direntry); + + if ((flags & LOOKUP_REVAL) && !CIFS_CACHE_READ(CIFS_I(inode))) + CIFS_I(inode)->time = 0; /* force reval */ + + rc = cifs_revalidate_dentry(direntry); + if (rc) { + cifs_dbg(FYI, "cifs_revalidate_dentry failed with rc=%d", rc); + switch (rc) { + case -ENOENT: + case -ESTALE: + /* + * Those errors mean the dentry is invalid + * (file was deleted or recreated) + */ + return 0; + default: + /* + * Otherwise some unexpected error happened + * report it as-is to VFS layer + */ + return rc; + } + } + else { + /* + * If the inode wasn't known to be a dfs entry when + * the dentry was instantiated, such as when created + * via ->readdir(), it needs to be set now since the + * attributes will have been updated by + * cifs_revalidate_dentry(). + */ + if (IS_AUTOMOUNT(inode) && + !(direntry->d_flags & DCACHE_NEED_AUTOMOUNT)) { + spin_lock(&direntry->d_lock); + direntry->d_flags |= DCACHE_NEED_AUTOMOUNT; + spin_unlock(&direntry->d_lock); + } + + return 1; + } + } else { + struct cifs_sb_info *cifs_sb = CIFS_SB(dir->i_sb); + struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb); + struct cached_fid *cfid; + + if (!open_cached_dir_by_dentry(tcon, direntry->d_parent, &cfid)) { + /* + * dentry is negative and parent is fully cached: + * we can assume file does not exist + */ + if (cfid->dirents.is_valid) { + close_cached_dir(cfid); + return 1; + } + close_cached_dir(cfid); + } + } + + /* + * This may be nfsd (or something), anyway, we can't see the + * intent of this. So, since this can be for creation, drop it. + */ + if (!flags) + return 0; + + /* + * Drop the negative dentry, in order to make sure to use the + * case sensitive name which is specified by user if this is + * for creation. + */ + if (flags & (LOOKUP_CREATE | LOOKUP_RENAME_TARGET)) + return 0; + + if (time_after(jiffies, cifs_get_time(direntry) + HZ) || !lookupCacheEnabled) + return 0; + + return 1; +} + +/* static int cifs_d_delete(struct dentry *direntry) +{ + int rc = 0; + + cifs_dbg(FYI, "In cifs d_delete, name = %pd\n", direntry); + + return rc; +} */ + +const struct dentry_operations cifs_dentry_ops = { + .d_revalidate = cifs_d_revalidate, + .d_automount = cifs_d_automount, +/* d_delete: cifs_d_delete, */ /* not needed except for debugging */ +}; + +static int cifs_ci_hash(const struct dentry *dentry, struct qstr *q) +{ + struct nls_table *codepage = CIFS_SB(dentry->d_sb)->local_nls; + unsigned long hash; + wchar_t c; + int i, charlen; + + hash = init_name_hash(dentry); + for (i = 0; i < q->len; i += charlen) { + charlen = codepage->char2uni(&q->name[i], q->len - i, &c); + /* error out if we can't convert the character */ + if (unlikely(charlen < 0)) + return charlen; + hash = partial_name_hash(cifs_toupper(c), hash); + } + q->hash = end_name_hash(hash); + + return 0; +} + +static int cifs_ci_compare(const struct dentry *dentry, + unsigned int len, const char *str, const struct qstr *name) +{ + struct nls_table *codepage = CIFS_SB(dentry->d_sb)->local_nls; + wchar_t c1, c2; + int i, l1, l2; + + /* + * We make the assumption here that uppercase characters in the local + * codepage are always the same length as their lowercase counterparts. + * + * If that's ever not the case, then this will fail to match it. + */ + if (name->len != len) + return 1; + + for (i = 0; i < len; i += l1) { + /* Convert characters in both strings to UTF-16. */ + l1 = codepage->char2uni(&str[i], len - i, &c1); + l2 = codepage->char2uni(&name->name[i], name->len - i, &c2); + + /* + * If we can't convert either character, just declare it to + * be 1 byte long and compare the original byte. + */ + if (unlikely(l1 < 0 && l2 < 0)) { + if (str[i] != name->name[i]) + return 1; + l1 = 1; + continue; + } + + /* + * Here, we again ass|u|me that upper/lowercase versions of + * a character are the same length in the local NLS. + */ + if (l1 != l2) + return 1; + + /* Now compare uppercase versions of these characters */ + if (cifs_toupper(c1) != cifs_toupper(c2)) + return 1; + } + + return 0; +} + +static int set_tmpfile_attr(const unsigned int xid, unsigned int oflags, + struct inode *inode, const char *full_path, + struct TCP_Server_Info *server) +{ + struct cifsInodeInfo *cinode = CIFS_I(inode); + FILE_BASIC_INFO fi; + + cinode->cifsAttrs |= ATTR_HIDDEN; + if (oflags & O_EXCL) + cinode->cifsAttrs |= ATTR_TEMPORARY; + + fi = (FILE_BASIC_INFO) { + .Attributes = cpu_to_le32(cinode->cifsAttrs), + }; + return server->ops->set_file_info(inode, full_path, &fi, xid); +} + +/* + * Create a hidden temporary CIFS file with delete-on-close bit set. + * + * The initial dentry state is unhashed-negative. On success, dentry will + * become unhashed-positive by calling d_instantiate(). + */ +int cifs_tmpfile(struct mnt_idmap *idmap, struct inode *dir, + struct file *file, umode_t mode) +{ + struct dentry *dentry = file->f_path.dentry; + struct cifs_sb_info *cifs_sb = CIFS_SB(dir); + size_t namesize = CIFS_TMPNAME_LEN + 1; + char *path __free(kfree) = NULL, *name; + unsigned int oflags = file->f_flags; + int retries = 0, max_retries = 16; + struct TCP_Server_Info *server; + struct cifs_pending_open open; + struct cifsFileInfo *cfile; + struct cifs_fid fid = {}; + struct tcon_link *tlink; + struct cifs_tcon *tcon; + unsigned int sbflags; + struct inode *inode; + unsigned int xid; + __u32 oplock; + int namelen; + int rc; + + if (unlikely(cifs_forced_shutdown(cifs_sb))) + return smb_EIO(smb_eio_trace_forced_shutdown); + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) + return PTR_ERR(tlink); + tcon = tlink_tcon(tlink); + server = tcon->ses->server; + + xid = get_xid(); + + if (server->vals->protocol_id < SMB20_PROT_ID) { + cifs_dbg(VFS | ONCE, "O_TMPFILE is supported only in SMB2+\n"); + rc = -EOPNOTSUPP; + goto out; + } + + if (server->ops->new_lease_key) + server->ops->new_lease_key(&fid); + cifs_add_pending_open(&fid, tlink, &open); + + path = alloc_parent_path(dentry, namesize - 1); + if (IS_ERR(path)) { + cifs_del_pending_open(&open); + rc = PTR_ERR(path); + path = NULL; + goto out; + } + + name = path + strlen(path); + do { + /* Append tmpfile name to @path */ + namelen = scnprintf(name, namesize, CIFS_TMPNAME_PREFIX "%x", + atomic_inc_return(&cifs_tmpcounter)); + rc = __cifs_do_create(dir, dentry, path, xid, tlink, oflags, + mode, &oplock, &fid, NULL, &inode); + if (!rc) { + rc = d_mark_tmpfile_name(file, &QSTR_LEN(name, namelen)); + if (rc) { + cifs_dbg(VFS | ONCE, "%s: failed to set filename in dentry: %d\n", + __func__, rc); + rc = -EISDIR; + iput(inode); + goto err_open; + } + set_nlink(inode, 0); + mark_inode_dirty(inode); + d_instantiate(dentry, inode); + break; + } + } while (unlikely(rc == -EEXIST) && ++retries < max_retries); + + if (rc) { + cifs_del_pending_open(&open); + goto out; + } + + rc = finish_open(file, dentry, generic_file_open); + if (rc) + goto err_open; + + sbflags = cifs_sb_flags(cifs_sb); + if ((file->f_flags & O_DIRECT) && (sbflags & CIFS_MOUNT_STRICT_IO)) { + if (sbflags & CIFS_MOUNT_NO_BRL) + file->f_op = &cifs_file_direct_nobrl_ops; + else + file->f_op = &cifs_file_direct_ops; + } + + cfile = cifs_new_fileinfo(&fid, file, tlink, oplock, NULL); + if (!cfile) { + rc = -ENOMEM; + goto err_open; + } + + rc = set_tmpfile_attr(xid, oflags, inode, path, server); + if (rc) + goto out; + + fscache_use_cookie(cifs_inode_cookie(file_inode(file)), + file->f_mode & FMODE_WRITE); +out: + cifs_put_tlink(tlink); + free_xid(xid); + return rc; +err_open: + cifs_del_pending_open(&open); + if (server->ops->close) + server->ops->close(xid, tcon, &fid); + goto out; +} + +char *cifs_silly_fullpath(struct dentry *dentry) +{ + unsigned char name[CIFS_SILLYNAME_LEN + 1]; + int retries = 0, max_retries = 16; + size_t namesize = sizeof(name); + struct dentry *sdentry = NULL; + char *path; + + do { + dput(sdentry); + scnprintf(name, namesize, CIFS_SILLYNAME_PREFIX "%x", + atomic_inc_return(&cifs_sillycounter)); + sdentry = lookup_noperm(&QSTR(name), dentry->d_parent); + if (IS_ERR(sdentry)) + return ERR_CAST(sdentry); + if (d_is_negative(sdentry)) { + dput(sdentry); + path = alloc_parent_path(dentry, CIFS_SILLYNAME_LEN); + if (!IS_ERR(path)) + strcat(path, name); + return path; + } + } while (++retries < max_retries); + dput(sdentry); + return ERR_PTR(-EBUSY); +} + +const struct dentry_operations cifs_ci_dentry_ops = { + .d_revalidate = cifs_d_revalidate, + .d_hash = cifs_ci_hash, + .d_compare = cifs_ci_compare, + .d_automount = cifs_d_automount, +}; diff --git a/fs/smb/client/dns_resolve.c b/fs/smb/client/dns_resolve.c new file mode 100644 index 000000000..de7f4b384 --- /dev/null +++ b/fs/smb/client/dns_resolve.c @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * Copyright (c) 2007 Igor Mammedov + * Author(s): Igor Mammedov (niallain@gmail.com) + * Steve French (sfrench@us.ibm.com) + * Wang Lei (wang840925@gmail.com) + * David Howells (dhowells@redhat.com) + * + * Contains the CIFS DFS upcall routines used for hostname to + * IP address translation. + * + */ + +#include +#include +#include +#include "dns_resolve.h" +#include "cifsglob.h" +#include "cifsproto.h" +#include "cifs_debug.h" + +static int resolve_name(const char *name, size_t namelen, struct sockaddr *addr) +{ + char *ip; + int rc; + + rc = dns_query(current->nsproxy->net_ns, NULL, name, + namelen, NULL, &ip, NULL, false); + if (rc < 0) { + cifs_dbg(FYI, "%s: unable to resolve: %*.*s\n", + __func__, (int)namelen, (int)namelen, name); + } else { + cifs_dbg(FYI, "%s: resolved: %*.*s to %s\n", + __func__, (int)namelen, (int)namelen, name, ip); + + rc = cifs_convert_address(addr, ip, strlen(ip)); + kfree(ip); + if (!rc) { + cifs_dbg(FYI, "%s: unable to determine ip address\n", + __func__); + rc = -EHOSTUNREACH; + } else { + rc = 0; + } + } + return rc; +} + +/** + * dns_resolve_name - Perform an upcall to resolve hostname to an ip address. + * @dom: DNS domain name (or NULL) + * @name: Name to look up + * @namelen: Length of name + * @ip_addr: Where to return the IP address + * + * Returns zero on success, -ve code otherwise. + */ +int dns_resolve_name(const char *dom, const char *name, + size_t namelen, struct sockaddr *ip_addr) +{ + size_t len; + char *s; + int rc; + + cifs_dbg(FYI, "%s: dom=%s name=%.*s\n", __func__, dom, (int)namelen, name); + if (!ip_addr || !name || !*name || !namelen) + return -EINVAL; + + cifs_dbg(FYI, "%s: hostname=%.*s\n", __func__, (int)namelen, name); + /* Try to interpret hostname as an IPv4 or IPv6 address */ + rc = cifs_convert_address(ip_addr, name, namelen); + if (rc > 0) { + cifs_dbg(FYI, "%s: unc is IP, skipping dns upcall: %*.*s\n", + __func__, (int)namelen, (int)namelen, name); + return 0; + } + + /* + * If @name contains a NetBIOS name and @dom has been specified, then + * convert @name to an FQDN and try resolving it first. + */ + if (dom && *dom && cifs_netbios_name(name, namelen)) { + len = strnlen(dom, CIFS_MAX_DOMAINNAME_LEN) + namelen + 2; + s = kmalloc(len, GFP_KERNEL); + if (!s) + return -ENOMEM; + + scnprintf(s, len, "%.*s.%s", (int)namelen, name, dom); + rc = resolve_name(s, len - 1, ip_addr); + kfree(s); + if (!rc) + return 0; + } + return resolve_name(name, namelen, ip_addr); +} diff --git a/fs/smb/client/dns_resolve.h b/fs/smb/client/dns_resolve.h new file mode 100644 index 000000000..951fbab5e --- /dev/null +++ b/fs/smb/client/dns_resolve.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +/* + * DNS Resolver upcall management for CIFS DFS + * Handles host name to IP address resolution + * + * Copyright (c) International Business Machines Corp., 2008 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ + +#ifndef _DNS_RESOLVE_H +#define _DNS_RESOLVE_H + +#include +#include "cifsglob.h" +#include "cifsproto.h" + +int dns_resolve_name(const char *dom, const char *name, size_t namelen, + struct sockaddr *ip_addr); + +static inline int dns_resolve_unc(const char *dom, const char *unc, + struct sockaddr *ip_addr) +{ + const char *name; + size_t namelen; + + if (!unc || strlen(unc) < 3) + return -EINVAL; + + extract_unc_hostname(unc, &name, &namelen); + if (!namelen) + return -EINVAL; + + return dns_resolve_name(dom, name, namelen, ip_addr); +} + +#endif /* _DNS_RESOLVE_H */ diff --git a/fs/smb/client/export.c b/fs/smb/client/export.c new file mode 100644 index 000000000..d606e8cbc --- /dev/null +++ b/fs/smb/client/export.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * Copyright (C) International Business Machines Corp., 2007 + * Author(s): Steve French (sfrench@us.ibm.com) + * + * Common Internet FileSystem (CIFS) client + * + * Operations related to support for exporting files via NFSD + * + */ + + /* + * See Documentation/filesystems/nfs/exporting.rst + * and examples in fs/exportfs + * + * Since cifs is a network file system, an "fsid" must be included for + * any nfs exports file entries which refer to cifs paths. In addition + * the cifs mount must be mounted with the "serverino" option (ie use stable + * server inode numbers instead of locally generated temporary ones). + * Although cifs inodes do not use generation numbers (have generation number + * of zero) - the inode number alone should be good enough for simple cases + * in which users want to export cifs shares with NFS. The decode and encode + * could be improved by using a new routine which expects 64 bit inode numbers + * instead of the default 32 bit routines in fs/exportfs + * + */ + +#include +#include +#include "cifsglob.h" +#include "cifs_debug.h" +#include "cifsfs.h" + +#ifdef CONFIG_CIFS_NFSD_EXPORT +static struct dentry *cifs_get_parent(struct dentry *dentry) +{ + /* BB need to add code here eventually to enable export via NFSD */ + cifs_dbg(FYI, "get parent for %p\n", dentry); + return ERR_PTR(-EACCES); +} + +const struct export_operations cifs_export_ops = { + .encode_fh = generic_encode_ino32_fh, + .get_parent = cifs_get_parent, +/* + * Following export operations are mandatory for NFS export support: + * .fh_to_dentry = + */ +}; + +#endif /* CONFIG_CIFS_NFSD_EXPORT */ + diff --git a/fs/smb/client/file.c b/fs/smb/client/file.c new file mode 100644 index 000000000..ac89c1ba5 --- /dev/null +++ b/fs/smb/client/file.c @@ -0,0 +1,3518 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * vfs operations that deal with files + * + * Copyright (C) International Business Machines Corp., 2002,2010 + * Author(s): Steve French (sfrench@us.ibm.com) + * Jeremy Allison (jra@samba.org) + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "cifsfs.h" +#include "cifsglob.h" +#include "cifsproto.h" +#include "smb2proto.h" +#include "cifs_unicode.h" +#include "cifs_debug.h" +#include "cifs_fs_sb.h" +#include "fscache.h" +#include "smbdirect.h" +#include "fs_context.h" +#include "cifs_ioctl.h" +#include "cached_dir.h" +#include + +static int cifs_reopen_file(struct cifsFileInfo *cfile, bool can_flush); + +/* + * Prepare a subrequest to upload to the server. We need to allocate credits + * so that we know the maximum amount of data that we can include in it. + */ +static void cifs_prepare_write(struct netfs_io_subrequest *subreq) +{ + struct cifs_io_subrequest *wdata = + container_of(subreq, struct cifs_io_subrequest, subreq); + struct cifs_io_request *req = wdata->req; + struct netfs_io_stream *stream = &req->rreq.io_streams[subreq->stream_nr]; + struct TCP_Server_Info *server; + struct cifsFileInfo *open_file = req->cfile; + struct cifs_sb_info *cifs_sb = CIFS_SB(wdata->rreq->inode->i_sb); + size_t wsize = req->rreq.wsize; + int rc; + + if (!wdata->have_xid) { + wdata->xid = get_xid(); + wdata->have_xid = true; + } + + server = cifs_pick_channel(tlink_tcon(open_file->tlink)->ses); + wdata->server = server; + + if (cifs_sb->ctx->wsize == 0) + cifs_negotiate_wsize(server, cifs_sb->ctx, + tlink_tcon(req->cfile->tlink)); + +retry: + if (open_file->invalidHandle) { + rc = cifs_reopen_file(open_file, false); + if (rc < 0) { + if (rc == -EAGAIN) + goto retry; + subreq->error = rc; + return netfs_prepare_write_failed(subreq); + } + } + + rc = server->ops->wait_mtu_credits(server, wsize, &stream->sreq_max_len, + &wdata->credits); + if (rc < 0) { + subreq->error = rc; + return netfs_prepare_write_failed(subreq); + } + + wdata->credits.rreq_debug_id = subreq->rreq->debug_id; + wdata->credits.rreq_debug_index = subreq->debug_index; + wdata->credits.in_flight_check = 1; + trace_smb3_rw_credits(wdata->rreq->debug_id, + wdata->subreq.debug_index, + wdata->credits.value, + server->credits, server->in_flight, + wdata->credits.value, + cifs_trace_rw_credits_write_prepare); + +#ifdef CONFIG_CIFS_SMB_DIRECT + if (server->smbd_conn) { + const struct smbdirect_socket_parameters *sp = + smbd_get_parameters(server->smbd_conn); + + stream->sreq_max_segs = sp->max_frmr_depth; + } +#endif +} + +/* + * Issue a subrequest to upload to the server. + */ +static void cifs_issue_write(struct netfs_io_subrequest *subreq) +{ + struct cifs_io_subrequest *wdata = + container_of(subreq, struct cifs_io_subrequest, subreq); + struct cifs_sb_info *sbi = CIFS_SB(subreq->rreq->inode->i_sb); + int rc; + + if (cifs_forced_shutdown(sbi)) { + rc = smb_EIO(smb_eio_trace_forced_shutdown); + goto fail; + } + + rc = adjust_credits(wdata->server, wdata, cifs_trace_rw_credits_issue_write_adjust); + if (rc) + goto fail; + + rc = -EAGAIN; + if (wdata->req->cfile->invalidHandle) + goto fail; + + wdata->server->ops->async_writev(wdata); +out: + return; + +fail: + if (rc == -EAGAIN) + trace_netfs_sreq(subreq, netfs_sreq_trace_retry); + else + trace_netfs_sreq(subreq, netfs_sreq_trace_fail); + add_credits_and_wake_if(wdata->server, &wdata->credits, 0); + cifs_write_subrequest_terminated(wdata, rc); + goto out; +} + +static void cifs_netfs_invalidate_cache(struct netfs_io_request *wreq) +{ + cifs_invalidate_cache(wreq->inode, 0); +} + +/* + * Negotiate the size of a read operation on behalf of the netfs library. + */ +static int cifs_prepare_read(struct netfs_io_subrequest *subreq) +{ + struct netfs_io_request *rreq = subreq->rreq; + struct cifs_io_subrequest *rdata = container_of(subreq, struct cifs_io_subrequest, subreq); + struct cifs_io_request *req = container_of(subreq->rreq, struct cifs_io_request, rreq); + struct TCP_Server_Info *server; + struct cifs_sb_info *cifs_sb = CIFS_SB(rreq->inode->i_sb); + size_t size; + int rc = 0; + + if (!rdata->have_xid) { + rdata->xid = get_xid(); + rdata->have_xid = true; + } + + server = cifs_pick_channel(tlink_tcon(req->cfile->tlink)->ses); + rdata->server = server; + + if (cifs_sb->ctx->rsize == 0) + cifs_negotiate_rsize(server, cifs_sb->ctx, + tlink_tcon(req->cfile->tlink)); + + rc = server->ops->wait_mtu_credits(server, cifs_sb->ctx->rsize, + &size, &rdata->credits); + if (rc) + return rc; + + rreq->io_streams[0].sreq_max_len = size; + + rdata->credits.in_flight_check = 1; + rdata->credits.rreq_debug_id = rreq->debug_id; + rdata->credits.rreq_debug_index = subreq->debug_index; + + trace_smb3_rw_credits(rdata->rreq->debug_id, + rdata->subreq.debug_index, + rdata->credits.value, + server->credits, server->in_flight, 0, + cifs_trace_rw_credits_read_submit); + +#ifdef CONFIG_CIFS_SMB_DIRECT + if (server->smbd_conn) { + const struct smbdirect_socket_parameters *sp = + smbd_get_parameters(server->smbd_conn); + + rreq->io_streams[0].sreq_max_segs = sp->max_frmr_depth; + } +#endif + return 0; +} + +/* + * Issue a read operation on behalf of the netfs helper functions. We're asked + * to make a read of a certain size at a point in the file. We are permitted + * to only read a portion of that, but as long as we read something, the netfs + * helper will call us again so that we can issue another read. + */ +static void cifs_issue_read(struct netfs_io_subrequest *subreq) +{ + struct netfs_io_request *rreq = subreq->rreq; + struct cifs_io_subrequest *rdata = container_of(subreq, struct cifs_io_subrequest, subreq); + struct cifs_io_request *req = container_of(subreq->rreq, struct cifs_io_request, rreq); + struct TCP_Server_Info *server = rdata->server; + int rc = 0; + + cifs_dbg(FYI, "%s: op=%08x[%x] mapping=%p len=%zu/%zu\n", + __func__, rreq->debug_id, subreq->debug_index, rreq->mapping, + subreq->transferred, subreq->len); + + rc = adjust_credits(server, rdata, cifs_trace_rw_credits_issue_read_adjust); + if (rc) + goto failed; + + if (req->cfile->invalidHandle) { + do { + rc = cifs_reopen_file(req->cfile, true); + } while (rc == -EAGAIN); + if (rc) + goto failed; + } + + if (subreq->rreq->origin != NETFS_UNBUFFERED_READ && + subreq->rreq->origin != NETFS_DIO_READ) + __set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags); + + trace_netfs_sreq(subreq, netfs_sreq_trace_submit); + rc = rdata->server->ops->async_readv(rdata); + if (rc) + goto failed; + return; + +failed: + add_credits_and_wake_if(rdata->server, &rdata->credits, 0); + subreq->error = rc; + netfs_read_subreq_terminated(subreq); +} + +/* + * Writeback calls this when it finds a folio that needs uploading. This isn't + * called if writeback only has copy-to-cache to deal with. + */ +static void cifs_begin_writeback(struct netfs_io_request *wreq) +{ + struct cifs_io_request *req = container_of(wreq, struct cifs_io_request, rreq); + int ret; + + ret = cifs_get_writable_file(CIFS_I(wreq->inode), FIND_ANY, &req->cfile); + if (ret) { + cifs_dbg(VFS, "No writable handle in writepages ret=%d\n", ret); + return; + } + + wreq->io_streams[0].avail = true; +} + +/* + * Initialise a request. + */ +static int cifs_init_request(struct netfs_io_request *rreq, struct file *file) +{ + struct cifs_io_request *req = container_of(rreq, struct cifs_io_request, rreq); + struct cifs_sb_info *cifs_sb = CIFS_SB(rreq->inode); + struct cifsFileInfo *open_file = NULL; + + rreq->rsize = cifs_sb->ctx->rsize; + rreq->wsize = cifs_sb->ctx->wsize; + req->pid = current->tgid; // Ummm... This may be a workqueue + + if (file) { + open_file = file->private_data; + rreq->netfs_priv = file->private_data; + req->cfile = cifsFileInfo_get(open_file); + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_RWPIDFORWARD) + req->pid = req->cfile->pid; + } else if (rreq->origin != NETFS_WRITEBACK) { + WARN_ON_ONCE(1); + return smb_EIO1(smb_eio_trace_not_netfs_writeback, rreq->origin); + } + + atomic_inc(&cifs_sb->outstanding_rreq); + return 0; +} + +/* + * Completion of a request operation. + */ +static void cifs_rreq_done(struct netfs_io_request *rreq) +{ + struct timespec64 atime, mtime; + struct inode *inode = rreq->inode; + + /* we do not want atime to be less than mtime, it broke some apps */ + atime = inode_set_atime_to_ts(inode, current_time(inode)); + mtime = inode_get_mtime(inode); + if (timespec64_compare(&atime, &mtime) < 0) + inode_set_atime_to_ts(inode, inode_get_mtime(inode)); +} + +static void cifs_free_request(struct netfs_io_request *rreq) +{ + struct cifs_io_request *req = container_of(rreq, struct cifs_io_request, rreq); + struct cifs_sb_info *cifs_sb = CIFS_SB(rreq->inode->i_sb); + + if (req->cfile) + cifsFileInfo_put(req->cfile); + + if (atomic_dec_and_test(&cifs_sb->outstanding_rreq)) + wake_up_var(&cifs_sb->outstanding_rreq); +} + +static void cifs_free_subrequest(struct netfs_io_subrequest *subreq) +{ + struct cifs_io_subrequest *rdata = + container_of(subreq, struct cifs_io_subrequest, subreq); + int rc = subreq->error; + + if (rdata->subreq.source == NETFS_DOWNLOAD_FROM_SERVER) { +#ifdef CONFIG_CIFS_SMB_DIRECT + if (rdata->mr) { + smbd_deregister_mr(rdata->mr); + rdata->mr = NULL; + } +#endif + } + + if (rdata->credits.value != 0) { + trace_smb3_rw_credits(rdata->rreq->debug_id, + rdata->subreq.debug_index, + rdata->credits.value, + rdata->server ? rdata->server->credits : 0, + rdata->server ? rdata->server->in_flight : 0, + -rdata->credits.value, + cifs_trace_rw_credits_free_subreq); + if (rdata->server) + add_credits_and_wake_if(rdata->server, &rdata->credits, 0); + else + rdata->credits.value = 0; + } + + if (rdata->have_xid) + free_xid(rdata->xid); +} + +const struct netfs_request_ops cifs_req_ops = { + .request_pool = &cifs_io_request_pool, + .subrequest_pool = &cifs_io_subrequest_pool, + .init_request = cifs_init_request, + .free_request = cifs_free_request, + .free_subrequest = cifs_free_subrequest, + .prepare_read = cifs_prepare_read, + .issue_read = cifs_issue_read, + .done = cifs_rreq_done, + .begin_writeback = cifs_begin_writeback, + .prepare_write = cifs_prepare_write, + .issue_write = cifs_issue_write, + .invalidate_cache = cifs_netfs_invalidate_cache, +}; + +/* + * Mark as invalid, all open files on tree connections since they + * were closed when session to server was lost. + */ +void +cifs_mark_open_files_invalid(struct cifs_tcon *tcon) +{ + struct cifsFileInfo *open_file = NULL; + struct list_head *tmp; + struct list_head *tmp1; + + /* only send once per connect */ + spin_lock(&tcon->tc_lock); + if (tcon->need_reconnect) + tcon->status = TID_NEED_RECON; + + if (tcon->status != TID_NEED_RECON) { + spin_unlock(&tcon->tc_lock); + return; + } + tcon->status = TID_IN_FILES_INVALIDATE; + spin_unlock(&tcon->tc_lock); + + /* list all files open on tree connection and mark them invalid */ + spin_lock(&tcon->open_file_lock); + list_for_each_safe(tmp, tmp1, &tcon->openFileList) { + open_file = list_entry(tmp, struct cifsFileInfo, tlist); + open_file->invalidHandle = true; + open_file->oplock_break_cancelled = true; + } + spin_unlock(&tcon->open_file_lock); + + invalidate_all_cached_dirs(tcon, true); + spin_lock(&tcon->tc_lock); + if (tcon->status == TID_IN_FILES_INVALIDATE) + tcon->status = TID_NEED_TCON; + spin_unlock(&tcon->tc_lock); + + /* + * BB Add call to evict_inodes(sb) for all superblocks mounted + * to this tcon. + */ +} + +static inline int cifs_convert_flags(unsigned int oflags, int rdwr_for_fscache) +{ + int flags = 0; + + if (oflags & O_TMPFILE) + flags |= DELETE; + + if ((oflags & O_ACCMODE) == O_RDONLY) + return flags | GENERIC_READ; + if ((oflags & O_ACCMODE) == O_WRONLY) { + return flags | (rdwr_for_fscache == 1 ? + (GENERIC_READ | GENERIC_WRITE) : GENERIC_WRITE); + } + if ((oflags & O_ACCMODE) == O_RDWR) { + /* GENERIC_ALL is too much permission to request + can cause unnecessary access denied on create */ + /* return GENERIC_ALL; */ + return flags | GENERIC_READ | GENERIC_WRITE; + } + + return flags | READ_CONTROL | FILE_WRITE_ATTRIBUTES | + FILE_READ_ATTRIBUTES | FILE_WRITE_EA | FILE_APPEND_DATA | + FILE_WRITE_DATA | FILE_READ_DATA; +} + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY +static u32 cifs_posix_convert_flags(unsigned int flags) +{ + u32 posix_flags = 0; + + if ((flags & O_ACCMODE) == O_RDONLY) + posix_flags = SMB_O_RDONLY; + else if ((flags & O_ACCMODE) == O_WRONLY) + posix_flags = SMB_O_WRONLY; + else if ((flags & O_ACCMODE) == O_RDWR) + posix_flags = SMB_O_RDWR; + + if (flags & O_CREAT) { + posix_flags |= SMB_O_CREAT; + if (flags & O_EXCL) + posix_flags |= SMB_O_EXCL; + } else if (flags & O_EXCL) + cifs_dbg(FYI, "Application %s pid %d has incorrectly set O_EXCL flag but not O_CREAT on file open. Ignoring O_EXCL\n", + current->comm, current->tgid); + + if (flags & O_TRUNC) + posix_flags |= SMB_O_TRUNC; + /* be safe and imply O_SYNC for O_DSYNC */ + if (flags & O_DSYNC) + posix_flags |= SMB_O_SYNC; + if (flags & O_DIRECTORY) + posix_flags |= SMB_O_DIRECTORY; + if (flags & O_NOFOLLOW) + posix_flags |= SMB_O_NOFOLLOW; + if (flags & O_DIRECT) + posix_flags |= SMB_O_DIRECT; + + return posix_flags; +} +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + +static inline int cifs_get_disposition(unsigned int flags) +{ + if ((flags & (O_CREAT | O_EXCL)) == (O_CREAT | O_EXCL)) + return FILE_CREATE; + else if ((flags & (O_CREAT | O_TRUNC)) == (O_CREAT | O_TRUNC)) + return FILE_OVERWRITE_IF; + else if ((flags & O_CREAT) == O_CREAT) + return FILE_OPEN_IF; + else if ((flags & O_TRUNC) == O_TRUNC) + return FILE_OVERWRITE; + else + return FILE_OPEN; +} + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY +int cifs_posix_open(const char *full_path, struct inode **pinode, + struct super_block *sb, int mode, unsigned int f_flags, + __u32 *poplock, __u16 *pnetfid, unsigned int xid) +{ + int rc; + FILE_UNIX_BASIC_INFO *presp_data; + __u32 posix_flags = 0; + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + struct cifs_fattr fattr; + struct tcon_link *tlink; + struct cifs_tcon *tcon; + + cifs_dbg(FYI, "posix open %s\n", full_path); + + presp_data = kzalloc_obj(FILE_UNIX_BASIC_INFO); + if (presp_data == NULL) + return -ENOMEM; + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) { + rc = PTR_ERR(tlink); + goto posix_open_ret; + } + + tcon = tlink_tcon(tlink); + mode &= ~current_umask(); + + posix_flags = cifs_posix_convert_flags(f_flags); + rc = CIFSPOSIXCreate(xid, tcon, posix_flags, mode, pnetfid, presp_data, + poplock, full_path, cifs_sb->local_nls, + cifs_remap(cifs_sb)); + cifs_put_tlink(tlink); + + if (rc) + goto posix_open_ret; + + if (presp_data->Type == cpu_to_le32(-1)) + goto posix_open_ret; /* open ok, caller does qpathinfo */ + + if (!pinode) + goto posix_open_ret; /* caller does not need info */ + + cifs_unix_basic_to_fattr(&fattr, presp_data, cifs_sb); + + /* get new inode and set it up */ + if (*pinode == NULL) { + cifs_fill_uniqueid(sb, &fattr); + *pinode = cifs_iget(sb, &fattr); + if (!*pinode) { + rc = -ENOMEM; + goto posix_open_ret; + } + } else { + cifs_revalidate_mapping(*pinode); + rc = cifs_fattr_to_inode(*pinode, &fattr, false); + } + +posix_open_ret: + kfree(presp_data); + return rc; +} +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + +static int cifs_nt_open(const char *full_path, struct inode *inode, struct cifs_sb_info *cifs_sb, + struct cifs_tcon *tcon, unsigned int f_flags, __u32 *oplock, + struct cifs_fid *fid, unsigned int xid, struct cifs_open_info_data *buf) +{ + int rc; + int desired_access; + int disposition; + int create_options = CREATE_NOT_DIR; + struct TCP_Server_Info *server = tcon->ses->server; + struct cifs_open_parms oparms; + int rdwr_for_fscache = 0; + + if (!server->ops->open) + return -ENOSYS; + + /* If we're caching, we need to be able to fill in around partial writes. */ + if (cifs_fscache_enabled(inode) && (f_flags & O_ACCMODE) == O_WRONLY) + rdwr_for_fscache = 1; + + desired_access = cifs_convert_flags(f_flags, rdwr_for_fscache); + +/********************************************************************* + * open flag mapping table: + * + * POSIX Flag CIFS Disposition + * ---------- ---------------- + * O_CREAT FILE_OPEN_IF + * O_CREAT | O_EXCL FILE_CREATE + * O_CREAT | O_TRUNC FILE_OVERWRITE_IF + * O_TRUNC FILE_OVERWRITE + * none of the above FILE_OPEN + * + * Note that there is not a direct match between disposition + * FILE_SUPERSEDE (ie create whether or not file exists although + * O_CREAT | O_TRUNC is similar but truncates the existing + * file rather than creating a new file as FILE_SUPERSEDE does + * (which uses the attributes / metadata passed in on open call) + *? + *? O_SYNC is a reasonable match to CIFS writethrough flag + *? and the read write flags match reasonably. O_LARGEFILE + *? is irrelevant because largefile support is always used + *? by this client. Flags O_APPEND, O_DIRECT, O_DIRECTORY, + * O_FASYNC, O_NOFOLLOW, O_NONBLOCK need further investigation + *********************************************************************/ + + disposition = cifs_get_disposition(f_flags); + /* BB pass O_SYNC flag through on file attributes .. BB */ + create_options |= cifs_open_create_options(f_flags, create_options); + +retry_open: + oparms = (struct cifs_open_parms) { + .tcon = tcon, + .cifs_sb = cifs_sb, + .desired_access = desired_access, + .create_options = cifs_create_options(cifs_sb, create_options), + .disposition = disposition, + .path = full_path, + .fid = fid, + }; + + rc = server->ops->open(xid, &oparms, oplock, buf); + if (rc) { + if (rc == -EACCES && rdwr_for_fscache == 1) { + desired_access = cifs_convert_flags(f_flags, 0); + rdwr_for_fscache = 2; + goto retry_open; + } + return rc; + } + if (rdwr_for_fscache == 2) + cifs_invalidate_cache(inode, FSCACHE_INVAL_DIO_WRITE); + + /* TODO: Add support for calling posix query info but with passing in fid */ + if (tcon->unix_ext) + rc = cifs_get_inode_info_unix(&inode, full_path, inode->i_sb, + xid); + else + rc = cifs_get_inode_info(&inode, full_path, buf, inode->i_sb, + xid, fid); + + if (rc) { + server->ops->close(xid, tcon, fid); + if (rc == -ESTALE) + rc = -EOPENSTALE; + } + + return rc; +} + +static bool +cifs_has_mand_locks(struct cifsInodeInfo *cinode) +{ + struct cifs_fid_locks *cur; + bool has_locks = false; + + down_read(&cinode->lock_sem); + list_for_each_entry(cur, &cinode->llist, llist) { + if (!list_empty(&cur->locks)) { + has_locks = true; + break; + } + } + up_read(&cinode->lock_sem); + return has_locks; +} + +void +cifs_down_write(struct rw_semaphore *sem) +{ + while (!down_write_trylock(sem)) + msleep(10); +} + +static void cifsFileInfo_put_work(struct work_struct *work); +void serverclose_work(struct work_struct *work); + +struct cifsFileInfo *cifs_new_fileinfo(struct cifs_fid *fid, struct file *file, + struct tcon_link *tlink, __u32 oplock, + const char *symlink_target) +{ + struct dentry *dentry = file_dentry(file); + struct inode *inode = d_inode(dentry); + struct cifsInodeInfo *cinode = CIFS_I(inode); + struct cifsFileInfo *cfile; + struct cifs_fid_locks *fdlocks; + struct cifs_tcon *tcon = tlink_tcon(tlink); + struct TCP_Server_Info *server = tcon->ses->server; + + cfile = kzalloc_obj(struct cifsFileInfo); + if (cfile == NULL) + return cfile; + + fdlocks = kzalloc_obj(struct cifs_fid_locks); + if (!fdlocks) { + kfree(cfile); + return NULL; + } + + if (symlink_target) { + cfile->symlink_target = kstrdup(symlink_target, GFP_KERNEL); + if (!cfile->symlink_target) { + kfree(fdlocks); + kfree(cfile); + return NULL; + } + } + + INIT_LIST_HEAD(&fdlocks->locks); + fdlocks->cfile = cfile; + cfile->llist = fdlocks; + + cfile->count = 1; + cfile->pid = current->tgid; + cfile->uid = current_fsuid(); + cfile->dentry = dget(dentry); + cfile->f_flags = file->f_flags; + cfile->invalidHandle = false; + cfile->deferred_close_scheduled = false; + cfile->status_file_deleted = file->f_flags & O_TMPFILE; + cfile->tlink = cifs_get_tlink(tlink); + INIT_WORK(&cfile->oplock_break, cifs_oplock_break); + INIT_WORK(&cfile->put, cifsFileInfo_put_work); + INIT_WORK(&cfile->serverclose, serverclose_work); + INIT_DELAYED_WORK(&cfile->deferred, smb2_deferred_work_close); + mutex_init(&cfile->fh_mutex); + spin_lock_init(&cfile->file_info_lock); + + /* + * If the server returned a read oplock and we have mandatory brlocks, + * set oplock level to None. + */ + if (server->ops->is_read_op(oplock) && cifs_has_mand_locks(cinode)) { + cifs_dbg(FYI, "Reset oplock val from read to None due to mand locks\n"); + oplock = 0; + } + + cifs_down_write(&cinode->lock_sem); + list_add(&fdlocks->llist, &cinode->llist); + up_write(&cinode->lock_sem); + + spin_lock(&tcon->open_file_lock); + if (fid->pending_open->oplock != CIFS_OPLOCK_NO_CHANGE && oplock) + oplock = fid->pending_open->oplock; + list_del(&fid->pending_open->olist); + + list_add(&cfile->tlist, &tcon->openFileList); + atomic_inc(&tcon->num_local_opens); + + /* if readable file instance put first in list*/ + spin_lock(&cinode->open_file_lock); + if (file->f_flags & O_TMPFILE) + set_bit(CIFS_INO_TMPFILE, &cinode->flags); + fid->purge_cache = false; + server->ops->set_fid(cfile, fid, oplock); + + if (file->f_mode & FMODE_READ) + list_add(&cfile->flist, &cinode->openFileList); + else + list_add_tail(&cfile->flist, &cinode->openFileList); + spin_unlock(&cinode->open_file_lock); + spin_unlock(&tcon->open_file_lock); + + if (fid->purge_cache) + cifs_zap_mapping(inode); + + file->private_data = cfile; + return cfile; +} + +struct cifsFileInfo * +cifsFileInfo_get(struct cifsFileInfo *cifs_file) +{ + spin_lock(&cifs_file->file_info_lock); + cifsFileInfo_get_locked(cifs_file); + spin_unlock(&cifs_file->file_info_lock); + return cifs_file; +} + +static void cifsFileInfo_put_final(struct cifsFileInfo *cifs_file) +{ + struct inode *inode = d_inode(cifs_file->dentry); + struct cifsInodeInfo *cifsi = CIFS_I(inode); + struct cifsLockInfo *li, *tmp; + + /* + * Delete any outstanding lock records. We'll lose them when the file + * is closed anyway. + */ + cifs_down_write(&cifsi->lock_sem); + list_for_each_entry_safe(li, tmp, &cifs_file->llist->locks, llist) { + list_del(&li->llist); + cifs_del_lock_waiters(li); + kfree(li); + } + list_del(&cifs_file->llist->llist); + kfree(cifs_file->llist); + up_write(&cifsi->lock_sem); + + cifs_put_tlink(cifs_file->tlink); + dput(cifs_file->dentry); + kfree(cifs_file->symlink_target); + kfree(cifs_file); +} + +static void cifsFileInfo_put_work(struct work_struct *work) +{ + struct cifsFileInfo *cifs_file = container_of(work, + struct cifsFileInfo, put); + + cifsFileInfo_put_final(cifs_file); +} + +void serverclose_work(struct work_struct *work) +{ + struct cifsFileInfo *cifs_file = container_of(work, + struct cifsFileInfo, serverclose); + + struct cifs_tcon *tcon = tlink_tcon(cifs_file->tlink); + + struct TCP_Server_Info *server = tcon->ses->server; + int rc = 0; + int retries = 0; + int MAX_RETRIES = 4; + + do { + if (server->ops->close_getattr) + rc = server->ops->close_getattr(0, tcon, cifs_file); + else if (server->ops->close) + rc = server->ops->close(0, tcon, &cifs_file->fid); + + if (rc == -EBUSY || rc == -EAGAIN) { + retries++; + msleep(250); + } + } while ((rc == -EBUSY || rc == -EAGAIN) && (retries < MAX_RETRIES) + ); + + if (retries == MAX_RETRIES) + pr_warn("Serverclose failed %d times, giving up\n", MAX_RETRIES); + + if (cifs_file->offload) + queue_work(fileinfo_put_wq, &cifs_file->put); + else + cifsFileInfo_put_final(cifs_file); +} + +/** + * cifsFileInfo_put - release a reference of file priv data + * + * Always potentially wait for oplock handler. See _cifsFileInfo_put(). + * + * @cifs_file: cifs/smb3 specific info (eg refcounts) for an open file + */ +void cifsFileInfo_put(struct cifsFileInfo *cifs_file) +{ + _cifsFileInfo_put(cifs_file, true, true); +} + +/** + * _cifsFileInfo_put - release a reference of file priv data + * + * This may involve closing the filehandle @cifs_file out on the + * server. Must be called without holding tcon->open_file_lock, + * cinode->open_file_lock and cifs_file->file_info_lock. + * + * If @wait_for_oplock_handler is true and we are releasing the last + * reference, wait for any running oplock break handler of the file + * and cancel any pending one. + * + * @cifs_file: cifs/smb3 specific info (eg refcounts) for an open file + * @wait_oplock_handler: must be false if called from oplock_break_handler + * @offload: not offloaded on close and oplock breaks + * + */ +void _cifsFileInfo_put(struct cifsFileInfo *cifs_file, + bool wait_oplock_handler, bool offload) +{ + struct inode *inode = d_inode(cifs_file->dentry); + struct cifs_tcon *tcon = tlink_tcon(cifs_file->tlink); + struct TCP_Server_Info *server = tcon->ses->server; + struct cifsInodeInfo *cifsi = CIFS_I(inode); + struct super_block *sb = inode->i_sb; + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + struct cifs_fid fid = {}; + struct cifs_pending_open open; + bool oplock_break_cancelled; + bool serverclose_offloaded = false; + + spin_lock(&tcon->open_file_lock); + spin_lock(&cifsi->open_file_lock); + spin_lock(&cifs_file->file_info_lock); + + cifs_file->offload = offload; + if (--cifs_file->count > 0) { + spin_unlock(&cifs_file->file_info_lock); + spin_unlock(&cifsi->open_file_lock); + spin_unlock(&tcon->open_file_lock); + return; + } + spin_unlock(&cifs_file->file_info_lock); + + if (server->ops->get_lease_key) + server->ops->get_lease_key(inode, &fid); + + /* store open in pending opens to make sure we don't miss lease break */ + cifs_add_pending_open_locked(&fid, cifs_file->tlink, &open); + + /* remove it from the lists */ + list_del(&cifs_file->flist); + list_del(&cifs_file->tlist); + atomic_dec(&tcon->num_local_opens); + + if (list_empty(&cifsi->openFileList)) { + cifs_dbg(FYI, "closing last open instance for inode %p\n", + d_inode(cifs_file->dentry)); + /* + * In strict cache mode we need invalidate mapping on the last + * close because it may cause a error when we open this file + * again and get at least level II oplock. + */ + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_STRICT_IO) + set_bit(CIFS_INO_INVALID_MAPPING, &cifsi->flags); + cifs_set_oplock_level(cifsi, 0); + } + + if (OPEN_FMODE(cifs_file->f_flags) & FMODE_WRITE) { + /* Stamp while open_file_lock is held; covers all close paths + * including background I/O. Pairs with smp_load_acquire() in + * is_size_safe_to_change(). + */ + smp_store_release(&cifsi->time_last_write, jiffies); + } + + spin_unlock(&cifsi->open_file_lock); + spin_unlock(&tcon->open_file_lock); + + oplock_break_cancelled = wait_oplock_handler ? + cancel_work_sync(&cifs_file->oplock_break) : false; + + if (!tcon->need_reconnect && !cifs_file->invalidHandle) { + struct TCP_Server_Info *server = tcon->ses->server; + unsigned int xid; + int rc = 0; + + xid = get_xid(); + if (server->ops->close_getattr) + rc = server->ops->close_getattr(xid, tcon, cifs_file); + else if (server->ops->close) + rc = server->ops->close(xid, tcon, &cifs_file->fid); + _free_xid(xid); + + if (rc == -EBUSY || rc == -EAGAIN) { + // Server close failed, hence offloading it as an async op + queue_work(serverclose_wq, &cifs_file->serverclose); + serverclose_offloaded = true; + } + } + + if (oplock_break_cancelled) + cifs_done_oplock_break(cifsi); + + cifs_del_pending_open(&open); + + // if serverclose has been offloaded to wq (on failure), it will + // handle offloading put as well. If serverclose not offloaded, + // we need to handle offloading put here. + if (!serverclose_offloaded) { + if (offload) + queue_work(fileinfo_put_wq, &cifs_file->put); + else + cifsFileInfo_put_final(cifs_file); + } +} + +int cifs_file_flush(const unsigned int xid, struct inode *inode, + struct cifsFileInfo *cfile) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(inode); + struct cifs_tcon *tcon; + int rc; + + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NOSSYNC) + return 0; + + if (cfile && (OPEN_FMODE(cfile->f_flags) & FMODE_WRITE)) { + tcon = tlink_tcon(cfile->tlink); + return tcon->ses->server->ops->flush(xid, tcon, + &cfile->fid); + } + rc = cifs_get_writable_file(CIFS_I(inode), FIND_ANY, &cfile); + if (!rc) { + tcon = tlink_tcon(cfile->tlink); + rc = tcon->ses->server->ops->flush(xid, tcon, &cfile->fid); + cifsFileInfo_put(cfile); + } else if (rc == -EBADF) { + rc = 0; + } + return rc; +} + +static int cifs_do_truncate(const unsigned int xid, struct dentry *dentry) +{ + struct cifsInodeInfo *cinode = CIFS_I(d_inode(dentry)); + struct inode *inode = d_inode(dentry); + struct cifsFileInfo *cfile = NULL; + struct TCP_Server_Info *server; + struct cifs_tcon *tcon; + int rc; + + rc = filemap_write_and_wait(inode->i_mapping); + if (is_interrupt_error(rc)) + return -ERESTARTSYS; + mapping_set_error(inode->i_mapping, rc); + + cfile = find_writable_file(cinode, FIND_FSUID_ONLY); + rc = cifs_file_flush(xid, inode, cfile); + if (!rc) { + if (cfile) { + tcon = tlink_tcon(cfile->tlink); + server = tcon->ses->server; + rc = server->ops->set_file_size(xid, tcon, + cfile, 0, false); + } + if (!rc) { + netfs_resize_file(&cinode->netfs, 0, true); + cifs_setsize(inode, 0); + } + } + if (cfile) + cifsFileInfo_put(cfile); + return rc; +} + +int cifs_open(struct inode *inode, struct file *file) + +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(inode); + struct cifs_open_info_data data = {}; + struct cifsFileInfo *cfile = NULL; + struct TCP_Server_Info *server; + struct cifs_pending_open open; + bool posix_open_ok = false; + struct cifs_fid fid = {}; + struct tcon_link *tlink; + struct cifs_tcon *tcon; + const char *full_path; + unsigned int sbflags; + int rc = -EACCES; + unsigned int xid; + __u32 oplock; + void *page; + + xid = get_xid(); + + if (unlikely(cifs_forced_shutdown(cifs_sb))) { + free_xid(xid); + return smb_EIO(smb_eio_trace_forced_shutdown); + } + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) { + free_xid(xid); + return PTR_ERR(tlink); + } + tcon = tlink_tcon(tlink); + server = tcon->ses->server; + + page = alloc_dentry_path(); + full_path = build_path_from_dentry(file_dentry(file), page); + if (IS_ERR(full_path)) { + rc = PTR_ERR(full_path); + goto out; + } + + cifs_dbg(FYI, "inode = 0x%p file flags are 0x%x for %s\n", + inode, file->f_flags, full_path); + + sbflags = cifs_sb_flags(cifs_sb); + if ((file->f_flags & O_DIRECT) && (sbflags & CIFS_MOUNT_STRICT_IO)) { + if (sbflags & CIFS_MOUNT_NO_BRL) + file->f_op = &cifs_file_direct_nobrl_ops; + else + file->f_op = &cifs_file_direct_ops; + } + + if (file->f_flags & O_TRUNC) { + rc = cifs_do_truncate(xid, file_dentry(file)); + if (rc) + goto out; + } + + /* Get the cached handle as SMB2 close is deferred */ + if (OPEN_FMODE(file->f_flags) & FMODE_WRITE) { + rc = __cifs_get_writable_file(CIFS_I(inode), + FIND_FSUID_ONLY | + FIND_NO_PENDING_DELETE | + FIND_OPEN_FLAGS, + file->f_flags, &cfile); + } else { + cfile = __find_readable_file(CIFS_I(inode), + FIND_NO_PENDING_DELETE | + FIND_OPEN_FLAGS, + file->f_flags); + rc = cfile ? 0 : -ENOENT; + } + if (rc == 0) { + trace_smb3_open_cached(xid, tcon->tid, tcon->ses->Suid, + cfile->fid.persistent_fid, + file->f_flags, cfile->f_flags); + file->private_data = cfile; + spin_lock(&CIFS_I(inode)->deferred_lock); + cifs_del_deferred_close(cfile); + spin_unlock(&CIFS_I(inode)->deferred_lock); + goto use_cache; + } + /* hard link on the deferred close file */ + rc = cifs_get_hardlink_path(tcon, inode, file); + if (rc) + cifs_close_deferred_file(CIFS_I(inode)); + + if (server->oplocks) + oplock = REQ_OPLOCK; + else + oplock = 0; + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + if (!tcon->broken_posix_open && tcon->unix_ext && + cap_unix(tcon->ses) && (CIFS_UNIX_POSIX_PATH_OPS_CAP & + le64_to_cpu(tcon->fsUnixInfo.Capability))) { + /* can not refresh inode info since size could be stale */ + rc = cifs_posix_open(full_path, &inode, inode->i_sb, + cifs_sb->ctx->file_mode /* ignored */, + file->f_flags, &oplock, &fid.netfid, xid); + if (rc == 0) { + cifs_dbg(FYI, "posix open succeeded\n"); + posix_open_ok = true; + } else if ((rc == -EINVAL) || (rc == -EOPNOTSUPP)) { + if (tcon->ses->serverNOS) + cifs_dbg(VFS, "server %s of type %s returned unexpected error on SMB posix open, disabling posix open support. Check if server update available.\n", + tcon->ses->ip_addr, + tcon->ses->serverNOS); + tcon->broken_posix_open = true; + } else if ((rc != -EIO) && (rc != -EREMOTE) && + (rc != -EOPNOTSUPP)) /* path not found or net err */ + goto out; + /* + * Else fallthrough to retry open the old way on network i/o + * or DFS errors. + */ + } +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + + if (server->ops->get_lease_key) + server->ops->get_lease_key(inode, &fid); + + cifs_add_pending_open(&fid, tlink, &open); + + if (!posix_open_ok) { + if (server->ops->get_lease_key) + server->ops->get_lease_key(inode, &fid); + + rc = cifs_nt_open(full_path, inode, cifs_sb, tcon, file->f_flags, &oplock, &fid, + xid, &data); + if (rc) { + cifs_del_pending_open(&open); + goto out; + } + } + + cfile = cifs_new_fileinfo(&fid, file, tlink, oplock, data.symlink_target); + if (cfile == NULL) { + if (server->ops->close) + server->ops->close(xid, tcon, &fid); + cifs_del_pending_open(&open); + rc = -ENOMEM; + goto out; + } + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + if ((oplock & CIFS_CREATE_ACTION) && !posix_open_ok && tcon->unix_ext) { + /* + * Time to set mode which we can not set earlier due to + * problems creating new read-only files. + */ + struct cifs_unix_set_info_args args = { + .mode = inode->i_mode, + .uid = INVALID_UID, /* no change */ + .gid = INVALID_GID, /* no change */ + .ctime = NO_CHANGE_64, + .atime = NO_CHANGE_64, + .mtime = NO_CHANGE_64, + .device = 0, + }; + CIFSSMBUnixSetFileInfo(xid, tcon, &args, fid.netfid, + cfile->pid); + } +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + +use_cache: + fscache_use_cookie(cifs_inode_cookie(file_inode(file)), + file->f_mode & FMODE_WRITE); + if (!(file->f_flags & O_DIRECT)) + goto out; + if ((file->f_flags & (O_ACCMODE | O_APPEND)) == O_RDONLY) + goto out; + cifs_invalidate_cache(file_inode(file), FSCACHE_INVAL_DIO_WRITE); + +out: + free_dentry_path(page); + free_xid(xid); + cifs_put_tlink(tlink); + cifs_free_open_info(&data); + return rc; +} + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY +static int cifs_push_posix_locks(struct cifsFileInfo *cfile); +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + +/* + * Try to reacquire byte range locks that were released when session + * to server was lost. + */ +static int +cifs_relock_file(struct cifsFileInfo *cfile) +{ + struct cifsInodeInfo *cinode = CIFS_I(d_inode(cfile->dentry)); + struct cifs_tcon *tcon = tlink_tcon(cfile->tlink); + int rc = 0; +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + struct cifs_sb_info *cifs_sb = CIFS_SB(cinode); +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + + down_read_nested(&cinode->lock_sem, SINGLE_DEPTH_NESTING); + if (cinode->can_cache_brlcks) { + /* can cache locks - no need to relock */ + up_read(&cinode->lock_sem); + return rc; + } + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + if (cap_unix(tcon->ses) && + (CIFS_UNIX_FCNTL_CAP & le64_to_cpu(tcon->fsUnixInfo.Capability)) && + ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NOPOSIXBRL) == 0)) + rc = cifs_push_posix_locks(cfile); + else +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + rc = tcon->ses->server->ops->push_mand_locks(cfile); + + up_read(&cinode->lock_sem); + return rc; +} + +static int +cifs_reopen_file(struct cifsFileInfo *cfile, bool can_flush) +{ + int rc = -EACCES; + unsigned int xid; + __u32 oplock; + struct cifs_sb_info *cifs_sb; + struct cifs_tcon *tcon; + struct TCP_Server_Info *server; + struct cifsInodeInfo *cinode; + struct inode *inode; + void *page; + const char *full_path; + int desired_access; + int disposition = FILE_OPEN; + int create_options = CREATE_NOT_DIR; + struct cifs_open_parms oparms; + int rdwr_for_fscache = 0; + + xid = get_xid(); + mutex_lock(&cfile->fh_mutex); + if (!cfile->invalidHandle) { + mutex_unlock(&cfile->fh_mutex); + free_xid(xid); + return 0; + } + + inode = d_inode(cfile->dentry); + cifs_sb = CIFS_SB(inode->i_sb); + tcon = tlink_tcon(cfile->tlink); + server = tcon->ses->server; + + /* + * Can not grab rename sem here because various ops, including those + * that already have the rename sem can end up causing writepage to get + * called and if the server was down that means we end up here, and we + * can never tell if the caller already has the rename_sem. + */ + page = alloc_dentry_path(); + full_path = build_path_from_dentry(cfile->dentry, page); + if (IS_ERR(full_path)) { + mutex_unlock(&cfile->fh_mutex); + free_dentry_path(page); + free_xid(xid); + return PTR_ERR(full_path); + } + + cifs_dbg(FYI, "inode = 0x%p file flags 0x%x for %s\n", + inode, cfile->f_flags, full_path); + + if (tcon->ses->server->oplocks) + oplock = REQ_OPLOCK; + else + oplock = 0; + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + if (tcon->unix_ext && cap_unix(tcon->ses) && + (CIFS_UNIX_POSIX_PATH_OPS_CAP & + le64_to_cpu(tcon->fsUnixInfo.Capability))) { + /* + * O_CREAT, O_EXCL and O_TRUNC already had their effect on the + * original open. Must mask them off for a reopen. + */ + unsigned int oflags = cfile->f_flags & + ~(O_CREAT | O_EXCL | O_TRUNC); + + rc = cifs_posix_open(full_path, NULL, inode->i_sb, + cifs_sb->ctx->file_mode /* ignored */, + oflags, &oplock, &cfile->fid.netfid, xid); + if (rc == 0) { + cifs_dbg(FYI, "posix reopen succeeded\n"); + oparms.reconnect = true; + goto reopen_success; + } + /* + * fallthrough to retry open the old way on errors, especially + * in the reconnect path it is important to retry hard + */ + } +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + + /* If we're caching, we need to be able to fill in around partial writes. */ + if (cifs_fscache_enabled(inode) && (cfile->f_flags & O_ACCMODE) == O_WRONLY) + rdwr_for_fscache = 1; + + desired_access = cifs_convert_flags(cfile->f_flags, rdwr_for_fscache); + create_options |= cifs_open_create_options(cfile->f_flags, + create_options); + + if (server->ops->get_lease_key) + server->ops->get_lease_key(inode, &cfile->fid); + +retry_open: + oparms = (struct cifs_open_parms) { + .tcon = tcon, + .cifs_sb = cifs_sb, + .desired_access = desired_access, + .create_options = cifs_create_options(cifs_sb, create_options), + .disposition = disposition, + .path = full_path, + .fid = &cfile->fid, + .reconnect = true, + }; + + /* + * Can not refresh inode by passing in file_info buf to be returned by + * ops->open and then calling get_inode_info with returned buf since + * file might have write behind data that needs to be flushed and server + * version of file size can be stale. If we knew for sure that inode was + * not dirty locally we could do this. + */ + rc = server->ops->open(xid, &oparms, &oplock, NULL); + if (rc == -ENOENT && oparms.reconnect == false) { + /* durable handle timeout is expired - open the file again */ + rc = server->ops->open(xid, &oparms, &oplock, NULL); + /* indicate that we need to relock the file */ + oparms.reconnect = true; + } + if (rc == -EACCES && rdwr_for_fscache == 1) { + desired_access = cifs_convert_flags(cfile->f_flags, 0); + rdwr_for_fscache = 2; + goto retry_open; + } + + if (rc) { + mutex_unlock(&cfile->fh_mutex); + cifs_dbg(FYI, "cifs_reopen returned 0x%x\n", rc); + cifs_dbg(FYI, "oplock: %d\n", oplock); + goto reopen_error_exit; + } + + if (rdwr_for_fscache == 2) + cifs_invalidate_cache(inode, FSCACHE_INVAL_DIO_WRITE); + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY +reopen_success: +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + cfile->invalidHandle = false; + mutex_unlock(&cfile->fh_mutex); + cinode = CIFS_I(inode); + + if (can_flush) { + rc = filemap_write_and_wait(inode->i_mapping); + if (!is_interrupt_error(rc)) + mapping_set_error(inode->i_mapping, rc); + + if (tcon->posix_extensions) { + rc = smb311_posix_get_inode_info(&inode, full_path, + NULL, inode->i_sb, xid); + } else if (tcon->unix_ext) { + rc = cifs_get_inode_info_unix(&inode, full_path, + inode->i_sb, xid); + } else { + rc = cifs_get_inode_info(&inode, full_path, NULL, + inode->i_sb, xid, NULL); + } + } + /* + * Else we are writing out data to server already and could deadlock if + * we tried to flush data, and since we do not know if we have data that + * would invalidate the current end of file on the server we can not go + * to the server to get the new inode info. + */ + + /* + * If the server returned a read oplock and we have mandatory brlocks, + * set oplock level to None. + */ + if (server->ops->is_read_op(oplock) && cifs_has_mand_locks(cinode)) { + cifs_dbg(FYI, "Reset oplock val from read to None due to mand locks\n"); + oplock = 0; + } + + scoped_guard(spinlock, &cinode->open_file_lock) + server->ops->set_fid(cfile, &cfile->fid, oplock); + if (oparms.reconnect) + cifs_relock_file(cfile); + +reopen_error_exit: + free_dentry_path(page); + free_xid(xid); + return rc; +} + +void smb2_deferred_work_close(struct work_struct *work) +{ + struct cifsFileInfo *cfile = container_of(work, + struct cifsFileInfo, deferred.work); + struct cifsInodeInfo *cinode = CIFS_I(d_inode(cfile->dentry)); + + spin_lock(&cinode->deferred_lock); + cifs_del_deferred_close(cfile); + cfile->deferred_close_scheduled = false; + spin_unlock(&cinode->deferred_lock); + _cifsFileInfo_put(cfile, true, false); +} + +static bool +smb2_can_defer_close(struct inode *inode, struct cifs_deferred_close *dclose) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); + struct cifsInodeInfo *cinode = CIFS_I(inode); + unsigned int oplock = READ_ONCE(cinode->oplock); + + return cifs_sb->ctx->closetimeo && cinode->lease_granted && dclose && + (oplock == CIFS_CACHE_RHW_FLG || oplock == CIFS_CACHE_RH_FLG) && + !test_bit(CIFS_INO_CLOSE_ON_LOCK, &cinode->flags); + +} + +int cifs_close(struct inode *inode, struct file *file) +{ + struct cifsFileInfo *cfile; + struct cifsInodeInfo *cinode = CIFS_I(inode); + struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); + struct cifs_deferred_close *dclose; + struct cifs_tcon *tcon; + + cifs_fscache_unuse_inode_cookie(inode, file->f_mode & FMODE_WRITE); + + if (file->private_data != NULL) { + cfile = file->private_data; + file->private_data = NULL; + dclose = kmalloc_obj(struct cifs_deferred_close); + if ((cfile->status_file_deleted == false) && + (smb2_can_defer_close(inode, dclose))) { + if (test_and_clear_bit(NETFS_ICTX_MODIFIED_ATTR, &cinode->netfs.flags)) { + inode_set_mtime_to_ts(inode, + inode_set_ctime_current(inode)); + } + spin_lock(&cinode->deferred_lock); + cifs_add_deferred_close(cfile, dclose); + if (cfile->deferred_close_scheduled && + delayed_work_pending(&cfile->deferred)) { + /* + * If there is no pending work, mod_delayed_work queues new work. + * So, Increase the ref count to avoid use-after-free. + */ + if (!mod_delayed_work(deferredclose_wq, + &cfile->deferred, cifs_sb->ctx->closetimeo)) + cifsFileInfo_get(cfile); + } else { + /* Deferred close for files */ + tcon = tlink_tcon(cfile->tlink); + trace_smb3_close_cached(tcon->tid, tcon->ses->Suid, + cfile->fid.persistent_fid, + cifs_sb->ctx->closetimeo); + queue_delayed_work(deferredclose_wq, + &cfile->deferred, cifs_sb->ctx->closetimeo); + cfile->deferred_close_scheduled = true; + spin_unlock(&cinode->deferred_lock); + return 0; + } + spin_unlock(&cinode->deferred_lock); + _cifsFileInfo_put(cfile, true, false); + } else { + _cifsFileInfo_put(cfile, true, false); + kfree(dclose); + } + } + + /* return code from the ->release op is always ignored */ + return 0; +} + +void +cifs_reopen_persistent_handles(struct cifs_tcon *tcon) +{ + struct cifsFileInfo *open_file, *tmp; + LIST_HEAD(tmp_list); + + if (!tcon->use_persistent || !tcon->need_reopen_files) + return; + + tcon->need_reopen_files = false; + + cifs_dbg(FYI, "Reopen persistent handles\n"); + + /* list all files open on tree connection, reopen resilient handles */ + spin_lock(&tcon->open_file_lock); + list_for_each_entry(open_file, &tcon->openFileList, tlist) { + if (!open_file->invalidHandle) + continue; + cifsFileInfo_get(open_file); + list_add_tail(&open_file->rlist, &tmp_list); + } + spin_unlock(&tcon->open_file_lock); + + list_for_each_entry_safe(open_file, tmp, &tmp_list, rlist) { + if (cifs_reopen_file(open_file, false /* do not flush */)) + tcon->need_reopen_files = true; + list_del_init(&open_file->rlist); + cifsFileInfo_put(open_file); + } +} + +int cifs_closedir(struct inode *inode, struct file *file) +{ + int rc = 0; + unsigned int xid; + struct cifsFileInfo *cfile = file->private_data; + struct cifs_tcon *tcon; + struct TCP_Server_Info *server; + char *buf; + + cifs_dbg(FYI, "Closedir inode = 0x%p\n", inode); + + if (cfile == NULL) + return rc; + + xid = get_xid(); + tcon = tlink_tcon(cfile->tlink); + server = tcon->ses->server; + + cifs_dbg(FYI, "Freeing private data in close dir\n"); + spin_lock(&cfile->file_info_lock); + if (server->ops->dir_needs_close(cfile)) { + cfile->invalidHandle = true; + spin_unlock(&cfile->file_info_lock); + if (server->ops->close_dir) + rc = server->ops->close_dir(xid, tcon, &cfile->fid); + else + rc = -ENOSYS; + cifs_dbg(FYI, "Closing uncompleted readdir with rc %d\n", rc); + /* not much we can do if it fails anyway, ignore rc */ + rc = 0; + } else + spin_unlock(&cfile->file_info_lock); + + buf = cfile->srch_inf.ntwrk_buf_start; + if (buf) { + cifs_dbg(FYI, "closedir free smb buf in srch struct\n"); + cfile->srch_inf.ntwrk_buf_start = NULL; + if (cfile->srch_inf.smallBuf) + cifs_small_buf_release(buf); + else if (cfile->srch_inf.is_dynamic_buf) + kfree(buf); + else + cifs_buf_release(buf); + } + + cifs_put_tlink(cfile->tlink); + kfree(file->private_data); + file->private_data = NULL; + /* BB can we lock the filestruct while this is going on? */ + free_xid(xid); + return rc; +} + +static struct cifsLockInfo * +cifs_lock_init(__u64 offset, __u64 length, __u8 type, __u16 flags) +{ + struct cifsLockInfo *lock = + kmalloc_obj(struct cifsLockInfo); + if (!lock) + return lock; + lock->offset = offset; + lock->length = length; + lock->type = type; + lock->pid = current->tgid; + lock->flags = flags; + INIT_LIST_HEAD(&lock->blist); + init_waitqueue_head(&lock->block_q); + return lock; +} + +void +cifs_del_lock_waiters(struct cifsLockInfo *lock) +{ + struct cifsLockInfo *li, *tmp; + list_for_each_entry_safe(li, tmp, &lock->blist, blist) { + list_del_init(&li->blist); + wake_up(&li->block_q); + } +} + +#define CIFS_LOCK_OP 0 +#define CIFS_READ_OP 1 +#define CIFS_WRITE_OP 2 + +/* @rw_check : 0 - no op, 1 - read, 2 - write */ +static bool +cifs_find_fid_lock_conflict(struct cifs_fid_locks *fdlocks, __u64 offset, + __u64 length, __u8 type, __u16 flags, + struct cifsFileInfo *cfile, + struct cifsLockInfo **conf_lock, int rw_check) +{ + struct cifsLockInfo *li; + struct cifsFileInfo *cur_cfile = fdlocks->cfile; + struct TCP_Server_Info *server = tlink_tcon(cfile->tlink)->ses->server; + + list_for_each_entry(li, &fdlocks->locks, llist) { + if (offset + length <= li->offset || + offset >= li->offset + li->length) + continue; + if (rw_check != CIFS_LOCK_OP && current->tgid == li->pid && + server->ops->compare_fids(cfile, cur_cfile)) { + /* shared lock prevents write op through the same fid */ + if (!(li->type & server->vals->shared_lock_type) || + rw_check != CIFS_WRITE_OP) + continue; + } + if ((type & server->vals->shared_lock_type) && + ((server->ops->compare_fids(cfile, cur_cfile) && + current->tgid == li->pid) || type == li->type)) + continue; + if (rw_check == CIFS_LOCK_OP && + (flags & FL_OFDLCK) && (li->flags & FL_OFDLCK) && + server->ops->compare_fids(cfile, cur_cfile)) + continue; + if (conf_lock) + *conf_lock = li; + trace_smb3_lock_conflict(cfile->fid.persistent_fid, + offset, length, type, + li->offset, li->length, li->type, li->pid); + return true; + } + return false; +} + +bool +cifs_find_lock_conflict(struct cifsFileInfo *cfile, __u64 offset, __u64 length, + __u8 type, __u16 flags, + struct cifsLockInfo **conf_lock, int rw_check) +{ + bool rc = false; + struct cifs_fid_locks *cur; + struct cifsInodeInfo *cinode = CIFS_I(d_inode(cfile->dentry)); + + list_for_each_entry(cur, &cinode->llist, llist) { + rc = cifs_find_fid_lock_conflict(cur, offset, length, type, + flags, cfile, conf_lock, + rw_check); + if (rc) + break; + } + + return rc; +} + +/* + * Check if there is another lock that prevents us to set the lock (mandatory + * style). If such a lock exists, update the flock structure with its + * properties. Otherwise, set the flock type to F_UNLCK if we can cache brlocks + * or leave it the same if we can't. Returns 0 if we don't need to request to + * the server or 1 otherwise. + */ +static int +cifs_lock_test(struct cifsFileInfo *cfile, __u64 offset, __u64 length, + __u8 type, struct file_lock *flock) +{ + int rc = 0; + struct cifsLockInfo *conf_lock; + struct cifsInodeInfo *cinode = CIFS_I(d_inode(cfile->dentry)); + struct TCP_Server_Info *server = tlink_tcon(cfile->tlink)->ses->server; + bool exist; + + down_read(&cinode->lock_sem); + + exist = cifs_find_lock_conflict(cfile, offset, length, type, + flock->c.flc_flags, &conf_lock, + CIFS_LOCK_OP); + if (exist) { + flock->fl_start = conf_lock->offset; + flock->fl_end = conf_lock->offset + conf_lock->length - 1; + flock->c.flc_pid = conf_lock->pid; + if (conf_lock->type & server->vals->shared_lock_type) + flock->c.flc_type = F_RDLCK; + else + flock->c.flc_type = F_WRLCK; + } else if (!cinode->can_cache_brlcks) + rc = 1; + else + flock->c.flc_type = F_UNLCK; + + up_read(&cinode->lock_sem); + return rc; +} + +static void +cifs_lock_add(struct cifsFileInfo *cfile, struct cifsLockInfo *lock) +{ + struct cifsInodeInfo *cinode = CIFS_I(d_inode(cfile->dentry)); + cifs_down_write(&cinode->lock_sem); + list_add_tail(&lock->llist, &cfile->llist->locks); + up_write(&cinode->lock_sem); +} + +/* + * Set the byte-range lock (mandatory style). Returns: + * 1) 0, if we set the lock and don't need to request to the server; + * 2) 1, if no locks prevent us but we need to request to the server; + * 3) -EACCES, if there is a lock that prevents us and wait is false. + */ +static int +cifs_lock_add_if(struct cifsFileInfo *cfile, struct cifsLockInfo *lock, + bool wait, unsigned int xid) +{ + struct cifsLockInfo *conf_lock; + struct cifsInodeInfo *cinode = CIFS_I(d_inode(cfile->dentry)); + bool exist; + int rc = 0; + +try_again: + exist = false; + cifs_down_write(&cinode->lock_sem); + + exist = cifs_find_lock_conflict(cfile, lock->offset, lock->length, + lock->type, lock->flags, &conf_lock, + CIFS_LOCK_OP); + if (!exist && cinode->can_cache_brlcks) { + struct cifs_tcon *tcon = tlink_tcon(cfile->tlink); + + list_add_tail(&lock->llist, &cfile->llist->locks); + trace_smb3_lock_cached(xid, cfile->fid.persistent_fid, + tcon->tid, tcon->ses->Suid, + lock->offset, lock->length, + lock->type, 1, 0); + up_write(&cinode->lock_sem); + return rc; + } + + if (!exist) + rc = 1; + else if (!wait) + rc = -EACCES; + else { + list_add_tail(&lock->blist, &conf_lock->blist); + up_write(&cinode->lock_sem); + rc = wait_event_interruptible(lock->block_q, + (lock->blist.prev == &lock->blist) && + (lock->blist.next == &lock->blist)); + if (!rc) + goto try_again; + cifs_down_write(&cinode->lock_sem); + list_del_init(&lock->blist); + } + + up_write(&cinode->lock_sem); + return rc; +} + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY +/* + * Check if there is another lock that prevents us to set the lock (posix + * style). If such a lock exists, update the flock structure with its + * properties. Otherwise, set the flock type to F_UNLCK if we can cache brlocks + * or leave it the same if we can't. Returns 0 if we don't need to request to + * the server or 1 otherwise. + */ +static int +cifs_posix_lock_test(struct file *file, struct file_lock *flock) +{ + int rc = 0; + struct cifsInodeInfo *cinode = CIFS_I(file_inode(file)); + unsigned char saved_type = flock->c.flc_type; + + if ((flock->c.flc_flags & FL_POSIX) == 0) + return 1; + + down_read(&cinode->lock_sem); + posix_test_lock(file, flock); + + if (lock_is_unlock(flock) && !cinode->can_cache_brlcks) { + flock->c.flc_type = saved_type; + rc = 1; + } + + up_read(&cinode->lock_sem); + return rc; +} + +/* + * Set the byte-range lock (posix style). Returns: + * 1) <0, if the error occurs while setting the lock; + * 2) 0, if we set the lock and don't need to request to the server; + * 3) FILE_LOCK_DEFERRED, if we will wait for some other file_lock; + * 4) FILE_LOCK_DEFERRED + 1, if we need to request to the server. + */ +static int +cifs_posix_lock_set(struct file *file, struct file_lock *flock) +{ + struct cifsInodeInfo *cinode = CIFS_I(file_inode(file)); + int rc = FILE_LOCK_DEFERRED + 1; + + if ((flock->c.flc_flags & FL_POSIX) == 0) + return rc; + + cifs_down_write(&cinode->lock_sem); + if (!cinode->can_cache_brlcks) { + up_write(&cinode->lock_sem); + return rc; + } + + rc = posix_lock_file(file, flock, NULL); + up_write(&cinode->lock_sem); + return rc; +} + +int +cifs_push_mandatory_locks(struct cifsFileInfo *cfile) +{ + unsigned int xid; + int rc = 0, stored_rc; + struct cifsLockInfo *li, *tmp; + struct cifs_tcon *tcon; + unsigned int num, max_num, max_buf; + LOCKING_ANDX_RANGE *buf, *cur; + static const int types[] = { + LOCKING_ANDX_LARGE_FILES, + LOCKING_ANDX_SHARED_LOCK | LOCKING_ANDX_LARGE_FILES + }; + int i; + + xid = get_xid(); + tcon = tlink_tcon(cfile->tlink); + + /* + * Accessing maxBuf is racy with cifs_reconnect - need to store value + * and check it before using. + */ + max_buf = tcon->ses->server->maxBuf; + if (max_buf < (sizeof(struct smb_hdr) + sizeof(LOCKING_ANDX_RANGE))) { + free_xid(xid); + return -EINVAL; + } + + BUILD_BUG_ON(sizeof(struct smb_hdr) + sizeof(LOCKING_ANDX_RANGE) > + PAGE_SIZE); + max_buf = min_t(unsigned int, max_buf - sizeof(struct smb_hdr), + PAGE_SIZE); + max_num = (max_buf - sizeof(struct smb_hdr)) / + sizeof(LOCKING_ANDX_RANGE); + buf = kzalloc_objs(LOCKING_ANDX_RANGE, max_num); + if (!buf) { + free_xid(xid); + return -ENOMEM; + } + + for (i = 0; i < 2; i++) { + cur = buf; + num = 0; + list_for_each_entry_safe(li, tmp, &cfile->llist->locks, llist) { + if (li->type != types[i]) + continue; + cur->Pid = cpu_to_le16(li->pid); + cur->LengthLow = cpu_to_le32((u32)li->length); + cur->LengthHigh = cpu_to_le32((u32)(li->length>>32)); + cur->OffsetLow = cpu_to_le32((u32)li->offset); + cur->OffsetHigh = cpu_to_le32((u32)(li->offset>>32)); + if (++num == max_num) { + stored_rc = cifs_lockv(xid, tcon, + cfile->fid.netfid, + (__u8)li->type, 0, num, + buf); + if (stored_rc) + rc = stored_rc; + cur = buf; + num = 0; + } else + cur++; + } + + if (num) { + stored_rc = cifs_lockv(xid, tcon, cfile->fid.netfid, + (__u8)types[i], 0, num, buf); + if (stored_rc) + rc = stored_rc; + } + } + + kfree(buf); + free_xid(xid); + return rc; +} + +static __u32 +hash_lockowner(fl_owner_t owner) +{ + return cifs_lock_secret ^ hash32_ptr((const void *)owner); +} +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + +struct lock_to_push { + struct list_head llist; + __u64 offset; + __u64 length; + __u32 pid; + __u16 netfid; + __u8 type; +}; + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY +static int +cifs_push_posix_locks(struct cifsFileInfo *cfile) +{ + struct inode *inode = d_inode(cfile->dentry); + struct cifs_tcon *tcon = tlink_tcon(cfile->tlink); + struct file_lock *flock; + struct file_lock_context *flctx = locks_inode_context(inode); + unsigned int count = 0, i; + int rc = 0, xid, type; + struct list_head locks_to_send, *el; + struct lock_to_push *lck, *tmp; + __u64 length; + + xid = get_xid(); + + if (!flctx) + goto out; + + spin_lock(&flctx->flc_lock); + list_for_each(el, &flctx->flc_posix) { + count++; + } + spin_unlock(&flctx->flc_lock); + + INIT_LIST_HEAD(&locks_to_send); + + /* + * Allocating count locks is enough because no FL_POSIX locks can be + * added to the list while we are holding cinode->lock_sem that + * protects locking operations of this inode. + */ + for (i = 0; i < count; i++) { + lck = kmalloc_obj(struct lock_to_push); + if (!lck) { + rc = -ENOMEM; + goto err_out; + } + list_add_tail(&lck->llist, &locks_to_send); + } + + el = locks_to_send.next; + spin_lock(&flctx->flc_lock); + for_each_file_lock(flock, &flctx->flc_posix) { + unsigned char ftype = flock->c.flc_type; + + if (el == &locks_to_send) { + /* + * The list ended. We don't have enough allocated + * structures - something is really wrong. + */ + cifs_dbg(VFS, "Can't push all brlocks!\n"); + break; + } + length = cifs_flock_len(flock); + if (ftype == F_RDLCK || ftype == F_SHLCK) + type = CIFS_RDLCK; + else + type = CIFS_WRLCK; + lck = list_entry(el, struct lock_to_push, llist); + lck->pid = hash_lockowner(flock->c.flc_owner); + lck->netfid = cfile->fid.netfid; + lck->length = length; + lck->type = type; + lck->offset = flock->fl_start; + } + spin_unlock(&flctx->flc_lock); + + list_for_each_entry_safe(lck, tmp, &locks_to_send, llist) { + int stored_rc; + + stored_rc = CIFSSMBPosixLock(xid, tcon, lck->netfid, lck->pid, + lck->offset, lck->length, NULL, + lck->type, 0); + if (stored_rc) + rc = stored_rc; + list_del(&lck->llist); + kfree(lck); + } + +out: + free_xid(xid); + return rc; +err_out: + list_for_each_entry_safe(lck, tmp, &locks_to_send, llist) { + list_del(&lck->llist); + kfree(lck); + } + goto out; +} +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + +static int +cifs_push_locks(struct cifsFileInfo *cfile) +{ + struct cifsInodeInfo *cinode = CIFS_I(d_inode(cfile->dentry)); + struct cifs_tcon *tcon = tlink_tcon(cfile->tlink); + int rc = 0; +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + struct cifs_sb_info *cifs_sb = CIFS_SB(cinode); +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + + /* we are going to update can_cache_brlcks here - need a write access */ + cifs_down_write(&cinode->lock_sem); + if (!cinode->can_cache_brlcks) { + up_write(&cinode->lock_sem); + return rc; + } + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + if (cap_unix(tcon->ses) && + (CIFS_UNIX_FCNTL_CAP & le64_to_cpu(tcon->fsUnixInfo.Capability)) && + ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NOPOSIXBRL) == 0)) + rc = cifs_push_posix_locks(cfile); + else +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + rc = tcon->ses->server->ops->push_mand_locks(cfile); + + cinode->can_cache_brlcks = false; + up_write(&cinode->lock_sem); + return rc; +} + +static void +cifs_read_flock(struct file_lock *flock, __u32 *type, int *lock, int *unlock, + bool *wait_flag, struct TCP_Server_Info *server) +{ + if (flock->c.flc_flags & FL_POSIX) + cifs_dbg(FYI, "Posix\n"); + if (flock->c.flc_flags & FL_FLOCK) + cifs_dbg(FYI, "Flock\n"); + if (flock->c.flc_flags & FL_SLEEP) { + cifs_dbg(FYI, "Blocking lock\n"); + *wait_flag = true; + } + if (flock->c.flc_flags & FL_ACCESS) + cifs_dbg(FYI, "Process suspended by mandatory locking - not implemented yet\n"); + if (flock->c.flc_flags & FL_LEASE) + cifs_dbg(FYI, "Lease on file - not implemented yet\n"); + if (flock->c.flc_flags & + (~(FL_POSIX | FL_FLOCK | FL_SLEEP | + FL_ACCESS | FL_LEASE | FL_CLOSE | FL_OFDLCK))) + cifs_dbg(FYI, "Unknown lock flags 0x%x\n", + flock->c.flc_flags); + + *type = server->vals->large_lock_type; + if (lock_is_write(flock)) { + cifs_dbg(FYI, "F_WRLCK\n"); + *type |= server->vals->exclusive_lock_type; + *lock = 1; + } else if (lock_is_unlock(flock)) { + cifs_dbg(FYI, "F_UNLCK\n"); + *type |= server->vals->unlock_lock_type; + *unlock = 1; + /* Check if unlock includes more than one lock range */ + } else if (lock_is_read(flock)) { + cifs_dbg(FYI, "F_RDLCK\n"); + *type |= server->vals->shared_lock_type; + *lock = 1; + } else if (flock->c.flc_type == F_EXLCK) { + cifs_dbg(FYI, "F_EXLCK\n"); + *type |= server->vals->exclusive_lock_type; + *lock = 1; + } else if (flock->c.flc_type == F_SHLCK) { + cifs_dbg(FYI, "F_SHLCK\n"); + *type |= server->vals->shared_lock_type; + *lock = 1; + } else + cifs_dbg(FYI, "Unknown type of lock\n"); +} + +static int +cifs_getlk(struct file *file, struct file_lock *flock, __u32 type, + bool wait_flag, bool posix_lck, unsigned int xid) +{ + int rc = 0; + __u64 length = cifs_flock_len(flock); + struct cifsFileInfo *cfile = (struct cifsFileInfo *)file->private_data; + struct cifs_tcon *tcon = tlink_tcon(cfile->tlink); + struct TCP_Server_Info *server = tcon->ses->server; +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + __u16 netfid = cfile->fid.netfid; + + if (posix_lck) { + int posix_lock_type; + + rc = cifs_posix_lock_test(file, flock); + if (!rc) + return rc; + + if (type & server->vals->shared_lock_type) + posix_lock_type = CIFS_RDLCK; + else + posix_lock_type = CIFS_WRLCK; + rc = CIFSSMBPosixLock(xid, tcon, netfid, + hash_lockowner(flock->c.flc_owner), + flock->fl_start, length, flock, + posix_lock_type, wait_flag); + return rc; + } +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + + rc = cifs_lock_test(cfile, flock->fl_start, length, type, flock); + if (!rc) + return rc; + + /* BB we could chain these into one lock request BB */ + rc = server->ops->mand_lock(xid, cfile, flock->fl_start, length, type, + 1, 0, false); + if (rc == 0) { + rc = server->ops->mand_lock(xid, cfile, flock->fl_start, length, + type, 0, 1, false); + flock->c.flc_type = F_UNLCK; + if (rc != 0) + cifs_dbg(VFS, "Error unlocking previously locked range %d during test of lock\n", + rc); + return 0; + } + + if (type & server->vals->shared_lock_type) { + flock->c.flc_type = F_WRLCK; + return 0; + } + + type &= ~server->vals->exclusive_lock_type; + + rc = server->ops->mand_lock(xid, cfile, flock->fl_start, length, + type | server->vals->shared_lock_type, + 1, 0, false); + if (rc == 0) { + rc = server->ops->mand_lock(xid, cfile, flock->fl_start, length, + type | server->vals->shared_lock_type, 0, 1, false); + flock->c.flc_type = F_RDLCK; + if (rc != 0) + cifs_dbg(VFS, "Error unlocking previously locked range %d during test of lock\n", + rc); + } else + flock->c.flc_type = F_WRLCK; + + return 0; +} + +void +cifs_move_llist(struct list_head *source, struct list_head *dest) +{ + struct list_head *li, *tmp; + list_for_each_safe(li, tmp, source) + list_move(li, dest); +} + +int +cifs_get_hardlink_path(struct cifs_tcon *tcon, struct inode *inode, + struct file *file) +{ + struct cifsFileInfo *open_file = NULL; + struct cifsInodeInfo *cinode = CIFS_I(inode); + int rc = 0; + + spin_lock(&tcon->open_file_lock); + spin_lock(&cinode->open_file_lock); + + list_for_each_entry(open_file, &cinode->openFileList, flist) { + if (file->f_flags == open_file->f_flags) { + rc = -EINVAL; + break; + } + } + + spin_unlock(&cinode->open_file_lock); + spin_unlock(&tcon->open_file_lock); + return rc; +} + +void +cifs_free_llist(struct list_head *llist) +{ + struct cifsLockInfo *li, *tmp; + list_for_each_entry_safe(li, tmp, llist, llist) { + cifs_del_lock_waiters(li); + list_del(&li->llist); + kfree(li); + } +} + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY +int +cifs_unlock_range(struct cifsFileInfo *cfile, struct file_lock *flock, + unsigned int xid) +{ + int rc = 0, stored_rc; + static const int types[] = { + LOCKING_ANDX_LARGE_FILES, + LOCKING_ANDX_SHARED_LOCK | LOCKING_ANDX_LARGE_FILES + }; + unsigned int i; + unsigned int max_num, num, max_buf; + LOCKING_ANDX_RANGE *buf, *cur; + struct cifs_tcon *tcon = tlink_tcon(cfile->tlink); + struct cifsInodeInfo *cinode = CIFS_I(d_inode(cfile->dentry)); + struct cifsLockInfo *li, *tmp; + __u64 length = cifs_flock_len(flock); + LIST_HEAD(tmp_llist); + + /* + * Accessing maxBuf is racy with cifs_reconnect - need to store value + * and check it before using. + */ + max_buf = tcon->ses->server->maxBuf; + if (max_buf < (sizeof(struct smb_hdr) + sizeof(LOCKING_ANDX_RANGE))) + return -EINVAL; + + BUILD_BUG_ON(sizeof(struct smb_hdr) + sizeof(LOCKING_ANDX_RANGE) > + PAGE_SIZE); + max_buf = min_t(unsigned int, max_buf - sizeof(struct smb_hdr), + PAGE_SIZE); + max_num = (max_buf - sizeof(struct smb_hdr)) / + sizeof(LOCKING_ANDX_RANGE); + buf = kzalloc_objs(LOCKING_ANDX_RANGE, max_num); + if (!buf) + return -ENOMEM; + + cifs_down_write(&cinode->lock_sem); + for (i = 0; i < 2; i++) { + cur = buf; + num = 0; + list_for_each_entry_safe(li, tmp, &cfile->llist->locks, llist) { + if (flock->fl_start > li->offset || + (flock->fl_start + length) < + (li->offset + li->length)) + continue; + if (current->tgid != li->pid) + continue; + if (types[i] != li->type) + continue; + if (cinode->can_cache_brlcks) { + /* + * We can cache brlock requests - simply remove + * a lock from the file's list. + */ + list_del(&li->llist); + cifs_del_lock_waiters(li); + kfree(li); + continue; + } + cur->Pid = cpu_to_le16(li->pid); + cur->LengthLow = cpu_to_le32((u32)li->length); + cur->LengthHigh = cpu_to_le32((u32)(li->length>>32)); + cur->OffsetLow = cpu_to_le32((u32)li->offset); + cur->OffsetHigh = cpu_to_le32((u32)(li->offset>>32)); + /* + * We need to save a lock here to let us add it again to + * the file's list if the unlock range request fails on + * the server. + */ + list_move(&li->llist, &tmp_llist); + if (++num == max_num) { + stored_rc = cifs_lockv(xid, tcon, + cfile->fid.netfid, + li->type, num, 0, buf); + if (stored_rc) { + /* + * We failed on the unlock range + * request - add all locks from the tmp + * list to the head of the file's list. + */ + cifs_move_llist(&tmp_llist, + &cfile->llist->locks); + rc = stored_rc; + } else + /* + * The unlock range request succeed - + * free the tmp list. + */ + cifs_free_llist(&tmp_llist); + cur = buf; + num = 0; + } else + cur++; + } + if (num) { + stored_rc = cifs_lockv(xid, tcon, cfile->fid.netfid, + types[i], num, 0, buf); + if (stored_rc) { + cifs_move_llist(&tmp_llist, + &cfile->llist->locks); + rc = stored_rc; + } else + cifs_free_llist(&tmp_llist); + } + } + + up_write(&cinode->lock_sem); + kfree(buf); + return rc; +} +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + +static int +cifs_setlk(struct file *file, struct file_lock *flock, __u32 type, + bool wait_flag, bool posix_lck, int lock, int unlock, + unsigned int xid) +{ + int rc = 0; + __u64 length = cifs_flock_len(flock); + struct cifsFileInfo *cfile = (struct cifsFileInfo *)file->private_data; + struct cifs_tcon *tcon = tlink_tcon(cfile->tlink); + struct TCP_Server_Info *server = tcon->ses->server; + struct inode *inode = d_inode(cfile->dentry); + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + if (posix_lck) { + int posix_lock_type; + + rc = cifs_posix_lock_set(file, flock); + if (rc <= FILE_LOCK_DEFERRED) + return rc; + + if (type & server->vals->shared_lock_type) + posix_lock_type = CIFS_RDLCK; + else + posix_lock_type = CIFS_WRLCK; + + if (unlock == 1) + posix_lock_type = CIFS_UNLCK; + + rc = CIFSSMBPosixLock(xid, tcon, cfile->fid.netfid, + hash_lockowner(flock->c.flc_owner), + flock->fl_start, length, + NULL, posix_lock_type, wait_flag); + goto out; + } +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + if (lock) { + struct cifsLockInfo *lock; + + lock = cifs_lock_init(flock->fl_start, length, type, + flock->c.flc_flags); + if (!lock) + return -ENOMEM; + + rc = cifs_lock_add_if(cfile, lock, wait_flag, xid); + if (rc < 0) { + kfree(lock); + return rc; + } + if (!rc) + goto out; + + /* + * Windows 7 server can delay breaking lease from read to None + * if we set a byte-range lock on a file - break it explicitly + * before sending the lock to the server to be sure the next + * read won't conflict with non-overlapted locks due to + * pagereading. + */ + if (!CIFS_CACHE_WRITE(CIFS_I(inode)) && + CIFS_CACHE_READ(CIFS_I(inode))) { + cifs_zap_mapping(inode); + cifs_dbg(FYI, "Set no oplock for inode=%p due to mand locks\n", + inode); + cifs_reset_oplock(CIFS_I(inode)); + } + + rc = server->ops->mand_lock(xid, cfile, flock->fl_start, length, + type, 1, 0, wait_flag); + if (rc) { + kfree(lock); + return rc; + } + + cifs_lock_add(cfile, lock); + } else if (unlock) + rc = server->ops->mand_unlock_range(cfile, flock, xid); + +out: + if ((flock->c.flc_flags & FL_POSIX) || (flock->c.flc_flags & FL_FLOCK)) { + /* + * If this is a request to remove all locks because we + * are closing the file, it doesn't matter if the + * unlocking failed as both cifs.ko and the SMB server + * remove the lock on file close + */ + if (rc) { + cifs_dbg(VFS, "%s failed rc=%d\n", __func__, rc); + if (!(flock->c.flc_flags & FL_CLOSE)) + return rc; + } + rc = locks_lock_file_wait(file, flock); + } + return rc; +} + +int cifs_flock(struct file *file, int cmd, struct file_lock *fl) +{ + int rc, xid; + int lock = 0, unlock = 0; + bool wait_flag = false; + bool posix_lck = false; + struct cifs_sb_info *cifs_sb; + struct cifs_tcon *tcon; + struct cifsFileInfo *cfile; + __u32 type; + + xid = get_xid(); + + if (!(fl->c.flc_flags & FL_FLOCK)) { + rc = -ENOLCK; + free_xid(xid); + return rc; + } + + cfile = (struct cifsFileInfo *)file->private_data; + tcon = tlink_tcon(cfile->tlink); + + cifs_read_flock(fl, &type, &lock, &unlock, &wait_flag, + tcon->ses->server); + cifs_sb = CIFS_SB(file); + + if (cap_unix(tcon->ses) && + (CIFS_UNIX_FCNTL_CAP & le64_to_cpu(tcon->fsUnixInfo.Capability)) && + ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NOPOSIXBRL) == 0)) + posix_lck = true; + + if (!lock && !unlock) { + /* + * if no lock or unlock then nothing to do since we do not + * know what it is + */ + rc = -EOPNOTSUPP; + free_xid(xid); + return rc; + } + + rc = cifs_setlk(file, fl, type, wait_flag, posix_lck, lock, unlock, + xid); + free_xid(xid); + return rc; + + +} + +int cifs_lock(struct file *file, int cmd, struct file_lock *flock) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(file); + struct cifsFileInfo *cfile; + int lock = 0, unlock = 0; + bool wait_flag = false; + bool posix_lck = false; + struct cifs_tcon *tcon; + __u32 type; + int rc, xid; + + rc = -EACCES; + xid = get_xid(); + + cifs_dbg(FYI, "%s: %pD2 cmd=0x%x type=0x%x flags=0x%x r=%lld:%lld\n", __func__, file, cmd, + flock->c.flc_flags, flock->c.flc_type, + (long long)flock->fl_start, + (long long)flock->fl_end); + + cfile = (struct cifsFileInfo *)file->private_data; + tcon = tlink_tcon(cfile->tlink); + + cifs_read_flock(flock, &type, &lock, &unlock, &wait_flag, + tcon->ses->server); + set_bit(CIFS_INO_CLOSE_ON_LOCK, &CIFS_I(d_inode(cfile->dentry))->flags); + + if (cap_unix(tcon->ses) && + (CIFS_UNIX_FCNTL_CAP & le64_to_cpu(tcon->fsUnixInfo.Capability)) && + ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NOPOSIXBRL) == 0)) + posix_lck = true; + /* + * BB add code here to normalize offset and length to account for + * negative length which we can not accept over the wire. + */ + if (IS_GETLK(cmd)) { + rc = cifs_getlk(file, flock, type, wait_flag, posix_lck, xid); + free_xid(xid); + return rc; + } + + if (!lock && !unlock) { + /* + * if no lock or unlock then nothing to do since we do not + * know what it is + */ + free_xid(xid); + return -EOPNOTSUPP; + } + + rc = cifs_setlk(file, flock, type, wait_flag, posix_lck, lock, unlock, + xid); + free_xid(xid); + return rc; +} + +static void cifs_update_i_blocks_for_write(struct inode *inode, loff_t start, + loff_t end) +{ + struct cifsInodeInfo *cinode = CIFS_I(inode); + u64 allocated_end = CIFS_INO_BYTES(inode->i_blocks); + u64 blocks; + + if (cinode->cifsAttrs & FILE_ATTRIBUTE_SPARSE_FILE) + return; + + /* + * Grow the local estimate only across the currently known allocated + * prefix. A write beyond that may leave a hole. + */ + if ((u64)start > allocated_end) + return; + + blocks = CIFS_INO_BLOCKS(end); + if ((u64)inode->i_blocks < blocks) + inode->i_blocks = blocks; +} + +static void cifs_update_i_blocks_after_write(struct kiocb *iocb, + ssize_t written) +{ + struct inode *inode = file_inode(iocb->ki_filp); + loff_t end = iocb->ki_pos; + + if (written <= 0) + return; + + spin_lock(&inode->i_lock); + cifs_update_i_blocks_for_write(inode, end - written, end); + spin_unlock(&inode->i_lock); +} + +void cifs_write_subrequest_terminated(struct cifs_io_subrequest *wdata, ssize_t result) +{ + struct netfs_io_request *wreq = wdata->rreq; + struct inode *inode = wreq->inode; + struct netfs_inode *ictx = netfs_inode(inode); + loff_t wrend; + + if (result > 0) { + spin_lock(&inode->i_lock); + + wrend = wdata->subreq.start + wdata->subreq.transferred + result; + + if (wrend > ictx->_zero_point && + (wdata->rreq->origin == NETFS_UNBUFFERED_WRITE || + wdata->rreq->origin == NETFS_DIO_WRITE)) + netfs_write_zero_point(inode, wrend); + if (wrend > ictx->_remote_i_size) + netfs_resize_file(ictx, wrend, true); + cifs_update_i_blocks_for_write(inode, wdata->subreq.start, + wrend); + + spin_unlock(&inode->i_lock); + } + + netfs_write_subrequest_terminated(&wdata->subreq, result); +} + +static bool open_flags_match(struct cifsInodeInfo *cinode, + unsigned int oflags, unsigned int cflags) +{ + struct inode *inode = &cinode->netfs.inode; + int crw = 0, orw = 0; + + oflags &= ~(O_CREAT | O_EXCL | O_TRUNC); + cflags &= ~(O_CREAT | O_EXCL | O_TRUNC); + + if (cifs_fscache_enabled(inode)) { + if (OPEN_FMODE(cflags) & FMODE_WRITE) + crw = 1; + if (OPEN_FMODE(oflags) & FMODE_WRITE) + orw = 1; + } + if (cifs_convert_flags(oflags, orw) != cifs_convert_flags(cflags, crw)) + return false; + + return (oflags & (O_SYNC | O_DIRECT)) == (cflags & (O_SYNC | O_DIRECT)); +} + +struct cifsFileInfo *__find_readable_file(struct cifsInodeInfo *cifs_inode, + unsigned int find_flags, + unsigned int open_flags) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(cifs_inode); + bool fsuid_only = find_flags & FIND_FSUID_ONLY; + struct cifsFileInfo *open_file = NULL; + + /* only filter by fsuid on multiuser mounts */ + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MULTIUSER)) + fsuid_only = false; + + spin_lock(&cifs_inode->open_file_lock); + /* we could simply get the first_list_entry since write-only entries + are always at the end of the list but since the first entry might + have a close pending, we go through the whole list */ + list_for_each_entry(open_file, &cifs_inode->openFileList, flist) { + if (fsuid_only && !uid_eq(open_file->uid, current_fsuid())) + continue; + if ((find_flags & FIND_NO_PENDING_DELETE) && + open_file->status_file_deleted) + continue; + if ((find_flags & FIND_OPEN_FLAGS) && + !open_flags_match(cifs_inode, open_flags, + open_file->f_flags)) + continue; + if (OPEN_FMODE(open_file->f_flags) & FMODE_READ) { + if ((!open_file->invalidHandle)) { + /* found a good file */ + /* lock it so it will not be closed on us */ + cifsFileInfo_get(open_file); + spin_unlock(&cifs_inode->open_file_lock); + return open_file; + } /* else might as well continue, and look for + another, or simply have the caller reopen it + again rather than trying to fix this handle */ + } else /* write only file */ + break; /* write only files are last so must be done */ + } + spin_unlock(&cifs_inode->open_file_lock); + return NULL; +} + +/* Return -EBADF if no handle is found and general rc otherwise */ +int __cifs_get_writable_file(struct cifsInodeInfo *cifs_inode, + unsigned int find_flags, unsigned int open_flags, + struct cifsFileInfo **ret_file) +{ + struct cifsFileInfo *open_file, *inv_file = NULL; + bool fsuid_only, with_delete; + struct cifs_sb_info *cifs_sb; + bool any_available = false; + unsigned int refind = 0; + *ret_file = NULL; + int rc = -EBADF; + + /* + * Having a null inode here (because mapping->host was set to zero by + * the VFS or MM) should not happen but we had reports of on oops (due + * to it being zero) during stress testcases so we need to check for it + */ + + if (cifs_inode == NULL) { + cifs_dbg(VFS, "Null inode passed to cifs_writeable_file\n"); + dump_stack(); + return rc; + } + + if (test_bit(CIFS_INO_TMPFILE, &cifs_inode->flags)) + find_flags = FIND_ANY; + + cifs_sb = CIFS_SB(cifs_inode); + + with_delete = find_flags & FIND_WITH_DELETE; + fsuid_only = find_flags & FIND_FSUID_ONLY; + /* only filter by fsuid on multiuser mounts */ + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MULTIUSER)) + fsuid_only = false; + + spin_lock(&cifs_inode->open_file_lock); +refind_writable: + if (refind > MAX_REOPEN_ATT) { + spin_unlock(&cifs_inode->open_file_lock); + return rc; + } + list_for_each_entry(open_file, &cifs_inode->openFileList, flist) { + if (!any_available && open_file->pid != current->tgid) + continue; + if (fsuid_only && !uid_eq(open_file->uid, current_fsuid())) + continue; + if (with_delete && !(open_file->fid.access & DELETE)) + continue; + if ((find_flags & FIND_NO_PENDING_DELETE) && + open_file->status_file_deleted) + continue; + if ((find_flags & FIND_OPEN_FLAGS) && + !open_flags_match(cifs_inode, open_flags, + open_file->f_flags)) + continue; + if (OPEN_FMODE(open_file->f_flags) & FMODE_WRITE) { + if (!open_file->invalidHandle) { + /* found a good writable file */ + cifsFileInfo_get(open_file); + spin_unlock(&cifs_inode->open_file_lock); + *ret_file = open_file; + return 0; + } else { + if (!inv_file) + inv_file = open_file; + } + } + } + /* couldn't find usable FH with same pid, try any available */ + if (!any_available) { + any_available = true; + goto refind_writable; + } + + if (inv_file) { + any_available = false; + cifsFileInfo_get(inv_file); + } + + spin_unlock(&cifs_inode->open_file_lock); + + if (inv_file) { + rc = cifs_reopen_file(inv_file, false); + if (!rc) { + *ret_file = inv_file; + return 0; + } + + spin_lock(&cifs_inode->open_file_lock); + list_move_tail(&inv_file->flist, &cifs_inode->openFileList); + spin_unlock(&cifs_inode->open_file_lock); + cifsFileInfo_put(inv_file); + ++refind; + inv_file = NULL; + spin_lock(&cifs_inode->open_file_lock); + goto refind_writable; + } + + return rc; +} + +struct cifsFileInfo * +find_writable_file(struct cifsInodeInfo *cifs_inode, int flags) +{ + struct cifsFileInfo *cfile; + int rc; + + rc = cifs_get_writable_file(cifs_inode, flags, &cfile); + if (rc) + cifs_dbg(FYI, "Couldn't find writable handle rc=%d\n", rc); + + return cfile; +} + +int cifs_get_writable_path(struct cifs_tcon *tcon, const char *name, + struct inode *inode, int flags, + struct cifsFileInfo **ret_file) +{ + struct cifsFileInfo *cfile; + void *page; + + *ret_file = NULL; + + if (inode) + return cifs_get_writable_file(CIFS_I(inode), flags, ret_file); + + page = alloc_dentry_path(); + spin_lock(&tcon->open_file_lock); + list_for_each_entry(cfile, &tcon->openFileList, tlist) { + struct cifsInodeInfo *cinode; + const char *full_path = build_path_from_dentry(cfile->dentry, page); + if (IS_ERR(full_path)) { + spin_unlock(&tcon->open_file_lock); + free_dentry_path(page); + return PTR_ERR(full_path); + } + if (strcmp(full_path, name)) + continue; + + cinode = CIFS_I(d_inode(cfile->dentry)); + spin_unlock(&tcon->open_file_lock); + free_dentry_path(page); + return cifs_get_writable_file(cinode, flags, ret_file); + } + + spin_unlock(&tcon->open_file_lock); + free_dentry_path(page); + return -ENOENT; +} + +int +cifs_get_readable_path(struct cifs_tcon *tcon, const char *name, + struct cifsFileInfo **ret_file) +{ + struct cifsFileInfo *cfile; + void *page = alloc_dentry_path(); + + *ret_file = NULL; + + spin_lock(&tcon->open_file_lock); + list_for_each_entry(cfile, &tcon->openFileList, tlist) { + struct cifsInodeInfo *cinode; + const char *full_path = build_path_from_dentry(cfile->dentry, page); + if (IS_ERR(full_path)) { + spin_unlock(&tcon->open_file_lock); + free_dentry_path(page); + return PTR_ERR(full_path); + } + if (strcmp(full_path, name)) + continue; + + cinode = CIFS_I(d_inode(cfile->dentry)); + spin_unlock(&tcon->open_file_lock); + free_dentry_path(page); + *ret_file = find_readable_file(cinode, FIND_ANY); + return *ret_file ? 0 : -ENOENT; + } + + spin_unlock(&tcon->open_file_lock); + free_dentry_path(page); + return -ENOENT; +} + +/* + * Flush data on a strict file. + */ +int cifs_strict_fsync(struct file *file, loff_t start, loff_t end, + int datasync) +{ + struct cifsFileInfo *smbfile = file->private_data; + struct inode *inode = file_inode(file); + unsigned int xid; + int rc; + + rc = file_write_and_wait_range(file, start, end); + if (rc) { + trace_cifs_fsync_err(inode->i_ino, rc); + return rc; + } + + cifs_dbg(FYI, "%s: name=%pD datasync=0x%x\n", __func__, file, datasync); + + if (!CIFS_CACHE_READ(CIFS_I(inode))) { + rc = cifs_zap_mapping(inode); + cifs_dbg(FYI, "%s: invalidate mapping: rc = %d\n", __func__, rc); + } + + xid = get_xid(); + rc = cifs_file_flush(xid, inode, smbfile); + free_xid(xid); + return rc; +} + +/* + * Flush data on a non-strict data. + */ +int cifs_fsync(struct file *file, loff_t start, loff_t end, int datasync) +{ + unsigned int xid; + int rc = 0; + struct cifs_tcon *tcon; + struct TCP_Server_Info *server; + struct cifsFileInfo *smbfile = file->private_data; + struct inode *inode = file_inode(file); + struct cifs_sb_info *cifs_sb = CIFS_SB(file); + + rc = file_write_and_wait_range(file, start, end); + if (rc) { + trace_cifs_fsync_err(file_inode(file)->i_ino, rc); + return rc; + } + + xid = get_xid(); + + cifs_dbg(FYI, "Sync file - name: %pD datasync: 0x%x\n", + file, datasync); + + tcon = tlink_tcon(smbfile->tlink); + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NOSSYNC)) { + server = tcon->ses->server; + if (server->ops->flush == NULL) { + rc = -ENOSYS; + goto fsync_exit; + } + + if ((OPEN_FMODE(smbfile->f_flags) & FMODE_WRITE) == 0) { + smbfile = find_writable_file(CIFS_I(inode), FIND_ANY); + if (smbfile) { + rc = server->ops->flush(xid, tcon, &smbfile->fid); + cifsFileInfo_put(smbfile); + } else + cifs_dbg(FYI, "ignore fsync for file not open for write\n"); + } else + rc = server->ops->flush(xid, tcon, &smbfile->fid); + } + +fsync_exit: + free_xid(xid); + return rc; +} + +/* + * As file closes, flush all cached write data for this inode checking + * for write behind errors. + */ +int cifs_flush(struct file *file, fl_owner_t id) +{ + struct inode *inode = file_inode(file); + int rc = 0; + + if (file->f_mode & FMODE_WRITE) + rc = filemap_write_and_wait(inode->i_mapping); + + cifs_dbg(FYI, "Flush inode %p file %p rc %d\n", inode, file, rc); + if (rc) { + /* get more nuanced writeback errors */ + rc = filemap_check_wb_err(file->f_mapping, 0); + trace_cifs_flush_err(inode->i_ino, rc); + } + return rc; +} + +static ssize_t +cifs_writev(struct kiocb *iocb, struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct cifsFileInfo *cfile = (struct cifsFileInfo *)file->private_data; + struct inode *inode = file->f_mapping->host; + struct cifsInodeInfo *cinode = CIFS_I(inode); + struct TCP_Server_Info *server = tlink_tcon(cfile->tlink)->ses->server; + struct cifs_sb_info *cifs_sb = CIFS_SB(inode); + ssize_t rc; + + rc = netfs_start_io_write(inode); + if (rc < 0) + return rc; + + /* + * We need to hold the sem to be sure nobody modifies lock list + * with a brlock that prevents writing. + */ + down_read(&cinode->lock_sem); + + rc = generic_write_checks(iocb, from); + if (rc <= 0) + goto out; + + if ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NOPOSIXBRL) && + (cifs_find_lock_conflict(cfile, iocb->ki_pos, iov_iter_count(from), + server->vals->exclusive_lock_type, 0, + NULL, CIFS_WRITE_OP))) { + rc = -EACCES; + goto out; + } + + rc = netfs_buffered_write_iter_locked(iocb, from, NULL); + cifs_update_i_blocks_after_write(iocb, rc); + +out: + up_read(&cinode->lock_sem); + netfs_end_io_write(inode); + if (rc > 0) + rc = generic_write_sync(iocb, rc); + return rc; +} + +ssize_t +cifs_strict_writev(struct kiocb *iocb, struct iov_iter *from) +{ + struct inode *inode = file_inode(iocb->ki_filp); + struct cifsInodeInfo *cinode = CIFS_I(inode); + struct cifs_sb_info *cifs_sb = CIFS_SB(inode); + struct cifsFileInfo *cfile = (struct cifsFileInfo *) + iocb->ki_filp->private_data; + struct cifs_tcon *tcon = tlink_tcon(cfile->tlink); + ssize_t written; + + written = cifs_get_writer(cinode); + if (written) + return written; + + if (CIFS_CACHE_WRITE(cinode)) { + if (cap_unix(tcon->ses) && + (CIFS_UNIX_FCNTL_CAP & le64_to_cpu(tcon->fsUnixInfo.Capability)) && + ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NOPOSIXBRL) == 0)) { + written = netfs_file_write_iter(iocb, from); + cifs_update_i_blocks_after_write(iocb, written); + goto out; + } + written = cifs_writev(iocb, from); + goto out; + } + /* + * For non-oplocked files in strict cache mode we need to write the data + * to the server exactly from the pos to pos+len-1 rather than flush all + * affected pages because it may cause a error with mandatory locks on + * these pages but not on the region from pos to ppos+len-1. + */ + written = netfs_file_write_iter(iocb, from); + cifs_update_i_blocks_after_write(iocb, written); + if (CIFS_CACHE_READ(cinode)) { + /* + * We have read level caching and we have just sent a write + * request to the server thus making data in the cache stale. + * Zap the cache and set oplock/lease level to NONE to avoid + * reading stale data from the cache. All subsequent read + * operations will read new data from the server. + */ + cifs_zap_mapping(inode); + cifs_dbg(FYI, "Set Oplock/Lease to NONE for inode=%p after write\n", + inode); + cifs_reset_oplock(cinode); + } +out: + cifs_put_writer(cinode); + return written; +} + +ssize_t cifs_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + ssize_t written; + + written = netfs_file_write_iter(iocb, from); + cifs_update_i_blocks_after_write(iocb, written); + return written; +} + +ssize_t cifs_loose_read_iter(struct kiocb *iocb, struct iov_iter *iter) +{ + ssize_t rc; + struct inode *inode = file_inode(iocb->ki_filp); + + if (iocb->ki_flags & IOCB_DIRECT) + return netfs_unbuffered_read_iter(iocb, iter); + + rc = cifs_revalidate_mapping(inode); + if (rc) + return rc; + + return netfs_file_read_iter(iocb, iter); +} + +ssize_t cifs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + struct inode *inode = file_inode(iocb->ki_filp); + struct cifsInodeInfo *cinode = CIFS_I(inode); + ssize_t written; + int rc; + + if (iocb->ki_filp->f_flags & O_DIRECT) { + written = netfs_unbuffered_write_iter(iocb, from); + cifs_update_i_blocks_after_write(iocb, written); + if (written > 0 && CIFS_CACHE_READ(cinode)) { + cifs_zap_mapping(inode); + cifs_dbg(FYI, + "Set no oplock for inode=%p after a write operation\n", + inode); + cifs_reset_oplock(cinode); + } + return written; + } + + written = cifs_get_writer(cinode); + if (written) + return written; + + written = netfs_file_write_iter(iocb, from); + cifs_update_i_blocks_after_write(iocb, written); + + if (!CIFS_CACHE_WRITE(CIFS_I(inode))) { + rc = filemap_fdatawrite(inode->i_mapping); + if (rc) + cifs_dbg(FYI, "cifs_file_write_iter: %d rc on %p inode\n", + rc, inode); + } + + cifs_put_writer(cinode); + return written; +} + +ssize_t +cifs_strict_readv(struct kiocb *iocb, struct iov_iter *to) +{ + struct inode *inode = file_inode(iocb->ki_filp); + struct cifsInodeInfo *cinode = CIFS_I(inode); + struct cifs_sb_info *cifs_sb = CIFS_SB(inode); + struct cifsFileInfo *cfile = (struct cifsFileInfo *) + iocb->ki_filp->private_data; + struct cifs_tcon *tcon = tlink_tcon(cfile->tlink); + int rc = -EACCES; + + /* + * In strict cache mode we need to read from the server all the time + * if we don't have level II oplock because the server can delay mtime + * change - so we can't make a decision about inode invalidating. + * And we can also fail with pagereading if there are mandatory locks + * on pages affected by this read but not on the region from pos to + * pos+len-1. + */ + if (!CIFS_CACHE_READ(cinode)) + return netfs_unbuffered_read_iter(iocb, to); + + if ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NOPOSIXBRL) == 0) { + if (iocb->ki_flags & IOCB_DIRECT) + return netfs_unbuffered_read_iter(iocb, to); + return netfs_buffered_read_iter(iocb, to); + } + + /* + * We need to hold the sem to be sure nobody modifies lock list + * with a brlock that prevents reading. + */ + if (iocb->ki_flags & IOCB_DIRECT) { + rc = netfs_start_io_direct(inode); + if (rc < 0) + goto out; + rc = -EACCES; + down_read(&cinode->lock_sem); + if (!cifs_find_lock_conflict( + cfile, iocb->ki_pos, iov_iter_count(to), + tcon->ses->server->vals->shared_lock_type, + 0, NULL, CIFS_READ_OP)) + rc = netfs_unbuffered_read_iter_locked(iocb, to); + up_read(&cinode->lock_sem); + netfs_end_io_direct(inode); + } else { + rc = netfs_start_io_read(inode); + if (rc < 0) + goto out; + rc = -EACCES; + down_read(&cinode->lock_sem); + if (!cifs_find_lock_conflict( + cfile, iocb->ki_pos, iov_iter_count(to), + tcon->ses->server->vals->shared_lock_type, + 0, NULL, CIFS_READ_OP)) + rc = filemap_read(iocb, to, 0); + up_read(&cinode->lock_sem); + netfs_end_io_read(inode); + } +out: + return rc; +} + +static vm_fault_t cifs_page_mkwrite(struct vm_fault *vmf) +{ + return netfs_page_mkwrite(vmf, NULL); +} + +static const struct vm_operations_struct cifs_file_vm_ops = { + .fault = filemap_fault, + .map_pages = filemap_map_pages, + .page_mkwrite = cifs_page_mkwrite, +}; + +int cifs_file_strict_mmap_prepare(struct vm_area_desc *desc) +{ + int xid, rc = 0; + struct inode *inode = file_inode(desc->file); + + xid = get_xid(); + + if (!CIFS_CACHE_READ(CIFS_I(inode))) + rc = cifs_zap_mapping(inode); + if (!rc) + rc = generic_file_mmap_prepare(desc); + if (!rc) + desc->vm_ops = &cifs_file_vm_ops; + + free_xid(xid); + return rc; +} + +int cifs_file_mmap_prepare(struct vm_area_desc *desc) +{ + int rc, xid; + + xid = get_xid(); + + rc = cifs_revalidate_file(desc->file); + if (rc) + cifs_dbg(FYI, "Validation prior to mmap failed, error=%d\n", + rc); + if (!rc) + rc = generic_file_mmap_prepare(desc); + if (!rc) + desc->vm_ops = &cifs_file_vm_ops; + + free_xid(xid); + return rc; +} + +static int is_inode_writable(struct cifsInodeInfo *cifs_inode) +{ + struct cifsFileInfo *open_file; + + spin_lock(&cifs_inode->open_file_lock); + list_for_each_entry(open_file, &cifs_inode->openFileList, flist) { + if (OPEN_FMODE(open_file->f_flags) & FMODE_WRITE) { + spin_unlock(&cifs_inode->open_file_lock); + return 1; + } + } + spin_unlock(&cifs_inode->open_file_lock); + return 0; +} + +/* We do not want to update the file size from server for inodes + open for write - to avoid races with writepage extending + the file - in the future we could consider allowing + refreshing the inode only on increases in the file size + but this is tricky to do without racing with writebehind + page caching in the current Linux kernel design */ +bool is_size_safe_to_change(struct cifsInodeInfo *cifsInode, __u64 end_of_file, + bool from_readdir) +{ + struct cifs_sb_info *cifs_sb; + unsigned long tlw; + + if (!cifsInode) + return true; + + cifs_sb = CIFS_SB(cifsInode); + + if (is_inode_writable(cifsInode) || + ((cifsInode->oplock & CIFS_CACHE_RW_FLG) != 0 && from_readdir)) { + /* This inode is open for write at least once */ + + /* + * Readdir data is unreliable when we have writable handles or + * an exclusive lease -- never allow it to change i_size, even + * on direct-IO mounts where the server's directory metadata + * can still lag behind the actual file state. + */ + if (from_readdir) + return false; + + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_DIRECT_IO) { + /* since no page cache to corrupt on directio + we can change size safely */ + return true; + } + + if (i_size_read(&cifsInode->netfs.inode) < end_of_file) + return true; + + return false; + } + + /* + * No writable handles open. Check whether we are within the attribute + * cache validity window of a recent local modification. + * + * For the close() path: _cifsFileInfo_put() stamps time_last_write + * (via smp_store_release()) before releasing open_file_lock. That + * spin_unlock() is a store-release that pairs with the spin_lock() + * (load-acquire) in is_inode_writable() above, so if + * is_inode_writable() returned false the smp_load_acquire() below is + * guaranteed to observe any time_last_write update from a concurrent + * close(), covering all close paths including background I/O. + * + * For the setattr/truncate paths: those callers use smp_store_release() + * directly; the smp_load_acquire() below pairs with that store. There + * is no shared lock between setattr and readdir, so this relies on + * acquire-release semantics alone. The store propagation latency on + * weakly-ordered architectures (nanoseconds) is negligible relative to + * the acregmax window (seconds) and the readdir RPC round-trip + * (milliseconds), making this a sound design choice in practice. + * + * time_last_write == 0 means the inode has never been written locally; + * skip the window check to avoid false positives near boot time when + * jiffies is still close to INITIAL_JIFFIES on 32-bit systems. + */ + if (from_readdir) { + /* Pairs with smp_store_release() in _cifsFileInfo_put() and setattr. */ + tlw = smp_load_acquire(&cifsInode->time_last_write); + if (tlw && time_before(jiffies, tlw + cifs_sb->ctx->acregmax)) + return false; + } + + return true; +} + +void cifs_oplock_break(struct work_struct *work) +{ + struct cifsFileInfo *cfile = container_of(work, struct cifsFileInfo, + oplock_break); + struct inode *inode = d_inode(cfile->dentry); + struct super_block *sb = inode->i_sb; + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + struct cifsInodeInfo *cinode = CIFS_I(inode); + bool cache_read, cache_write, cache_handle; + struct cifs_tcon *tcon; + struct TCP_Server_Info *server; + struct tcon_link *tlink; + unsigned int oplock; + int rc = 0; + bool purge_cache = false, oplock_break_cancelled; + __u64 persistent_fid, volatile_fid; + __u16 net_fid; + + wait_on_bit(&cinode->flags, CIFS_INODE_PENDING_WRITERS, + TASK_UNINTERRUPTIBLE); + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) + goto out; + tcon = tlink_tcon(tlink); + server = tcon->ses->server; + + scoped_guard(spinlock, &cinode->open_file_lock) { + unsigned int sbflags = cifs_sb_flags(cifs_sb); + + server->ops->downgrade_oplock(server, cinode, cfile->oplock_level, + cfile->oplock_epoch, &purge_cache); + oplock = READ_ONCE(cinode->oplock); + cache_read = (oplock & CIFS_CACHE_READ_FLG) || + (sbflags & CIFS_MOUNT_RO_CACHE); + cache_write = (oplock & CIFS_CACHE_WRITE_FLG) || + (sbflags & CIFS_MOUNT_RW_CACHE); + cache_handle = oplock & CIFS_CACHE_HANDLE_FLG; + } + + if (!cache_write && cache_read && cifs_has_mand_locks(cinode)) { + cifs_dbg(FYI, "Reset oplock to None for inode=%p due to mand locks\n", + inode); + cifs_reset_oplock(cinode); + oplock = 0; + cache_read = cache_write = cache_handle = false; + } + + if (S_ISREG(inode->i_mode)) { + if (cache_read) + break_lease(inode, O_RDONLY); + else + break_lease(inode, O_WRONLY); + rc = filemap_fdatawrite(inode->i_mapping); + if (!cache_read || purge_cache) { + rc = filemap_fdatawait(inode->i_mapping); + mapping_set_error(inode->i_mapping, rc); + cifs_zap_mapping(inode); + } + cifs_dbg(FYI, "Oplock flush inode %p rc %d\n", inode, rc); + if (cache_write) + goto oplock_break_ack; + } + + rc = cifs_push_locks(cfile); + if (rc) + cifs_dbg(VFS, "Push locks rc = %d\n", rc); + +oplock_break_ack: + /* + * When oplock break is received and there are no active + * file handles but cached, then schedule deferred close immediately. + * So, new open will not use cached handle. + */ + + if (!cache_handle && !list_empty(&cinode->deferred_closes)) + cifs_close_deferred_file(cinode); + + persistent_fid = cfile->fid.persistent_fid; + volatile_fid = cfile->fid.volatile_fid; + net_fid = cfile->fid.netfid; + oplock_break_cancelled = cfile->oplock_break_cancelled; + + _cifsFileInfo_put(cfile, false /* do not wait for ourself */, false); + /* + * MS-SMB2 3.2.5.19.1 and 3.2.5.19.2 (and MS-CIFS 3.2.5.42) do not require + * an acknowledgment to be sent when the file has already been closed. + */ + spin_lock(&cinode->open_file_lock); + /* check list empty since can race with kill_sb calling tree disconnect */ + if (!oplock_break_cancelled && !list_empty(&cinode->openFileList)) { + spin_unlock(&cinode->open_file_lock); + rc = server->ops->oplock_response(tcon, persistent_fid, + volatile_fid, net_fid, + cinode, oplock); + cifs_dbg(FYI, "Oplock release rc = %d\n", rc); + } else + spin_unlock(&cinode->open_file_lock); + + cifs_put_tlink(tlink); +out: + cifs_done_oplock_break(cinode); +} + +static int cifs_swap_activate(struct swap_info_struct *sis, + struct file *swap_file, sector_t *span) +{ + struct cifsFileInfo *cfile = swap_file->private_data; + struct inode *inode = swap_file->f_mapping->host; + unsigned long blocks; + long long isize; + + cifs_dbg(FYI, "swap activate\n"); + + if (!swap_file->f_mapping->a_ops->swap_rw) + /* Cannot support swap */ + return -EINVAL; + + spin_lock(&inode->i_lock); + blocks = inode->i_blocks; + isize = inode->i_size; + spin_unlock(&inode->i_lock); + if (blocks*512 < isize) { + pr_warn("swap activate: swapfile has holes\n"); + return -EINVAL; + } + *span = sis->pages; + + pr_warn_once("Swap support over SMB3 is experimental\n"); + + /* + * TODO: consider adding ACL (or documenting how) to prevent other + * users (on this or other systems) from reading it + */ + + + /* TODO: add sk_set_memalloc(inet) or similar */ + + if (cfile) + cfile->swapfile = true; + /* + * TODO: Since file already open, we can't open with DENY_ALL here + * but we could add call to grab a byte range lock to prevent others + * from reading or writing the file + */ + + sis->flags |= SWP_FS_OPS; + return add_swap_extent(sis, 0, sis->max, 0); +} + +static void cifs_swap_deactivate(struct file *file) +{ + struct cifsFileInfo *cfile = file->private_data; + + cifs_dbg(FYI, "swap deactivate\n"); + + /* TODO: undo sk_set_memalloc(inet) will eventually be needed */ + + if (cfile) + cfile->swapfile = false; + + /* do we need to unpin (or unlock) the file */ +} + +/** + * cifs_swap_rw - SMB3 address space operation for swap I/O + * @iocb: target I/O control block + * @iter: I/O buffer + * + * Perform IO to the swap-file. This is much like direct IO. + */ +static int cifs_swap_rw(struct kiocb *iocb, struct iov_iter *iter) +{ + ssize_t ret; + + if (iov_iter_rw(iter) == READ) + ret = netfs_unbuffered_read_iter_locked(iocb, iter); + else + ret = netfs_unbuffered_write_iter_locked(iocb, iter, NULL); + if (ret < 0) + return ret; + return 0; +} + +const struct address_space_operations cifs_addr_ops = { + .read_folio = netfs_read_folio, + .readahead = netfs_readahead, + .writepages = netfs_writepages, + .dirty_folio = netfs_dirty_folio, + .release_folio = netfs_release_folio, + .direct_IO = noop_direct_IO, + .invalidate_folio = netfs_invalidate_folio, + .migrate_folio = filemap_migrate_folio, + /* + * TODO: investigate and if useful we could add an is_dirty_writeback + * helper if needed + */ + .swap_activate = cifs_swap_activate, + .swap_deactivate = cifs_swap_deactivate, + .swap_rw = cifs_swap_rw, +}; + +/* + * cifs_readahead requires the server to support a buffer large enough to + * contain the header plus one complete page of data. Otherwise, we need + * to leave cifs_readahead out of the address space operations. + */ +const struct address_space_operations cifs_addr_ops_smallbuf = { + .read_folio = netfs_read_folio, + .writepages = netfs_writepages, + .dirty_folio = netfs_dirty_folio, + .release_folio = netfs_release_folio, + .invalidate_folio = netfs_invalidate_folio, + .migrate_folio = filemap_migrate_folio, +}; diff --git a/fs/smb/client/fs_context.c b/fs/smb/client/fs_context.c new file mode 100644 index 000000000..abc29500a --- /dev/null +++ b/fs/smb/client/fs_context.c @@ -0,0 +1,2258 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2020, Microsoft Corporation. + * + * Author(s): Steve French + * David Howells + */ + +/* +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_CIFS_DFS_UPCALL +#include "dfs_cache.h" +#endif +*/ + +#include +#include +#include +#include +#include +#include +#include +#include "cifsfs.h" +#include "cifsglob.h" +#include "cifsproto.h" +#include "cifs_unicode.h" +#include "cifs_debug.h" +#include "cifs_fs_sb.h" +#include "ntlmssp.h" +#include "nterr.h" +#include "rfc1002pdu.h" +#include "fs_context.h" + +DEFINE_MUTEX(cifs_mount_mutex); + +static const match_table_t cifs_smb_version_tokens = { + { Smb_1, SMB1_VERSION_STRING }, + { Smb_20, SMB20_VERSION_STRING}, + { Smb_21, SMB21_VERSION_STRING }, + { Smb_30, SMB30_VERSION_STRING }, + { Smb_302, SMB302_VERSION_STRING }, + { Smb_302, ALT_SMB302_VERSION_STRING }, + { Smb_311, SMB311_VERSION_STRING }, + { Smb_311, ALT_SMB311_VERSION_STRING }, + { Smb_3any, SMB3ANY_VERSION_STRING }, + { Smb_default, SMBDEFAULT_VERSION_STRING }, + { Smb_version_err, NULL } +}; + +static const match_table_t cifs_secflavor_tokens = { + { Opt_sec_krb5, "krb5" }, + { Opt_sec_krb5i, "krb5i" }, + { Opt_sec_krb5p, "krb5p" }, + { Opt_sec_ntlmsspi, "ntlmsspi" }, + { Opt_sec_ntlmssp, "ntlmssp" }, + { Opt_sec_ntlmv2, "nontlm" }, + { Opt_sec_ntlmv2, "ntlmv2" }, + { Opt_sec_ntlmv2i, "ntlmv2i" }, + { Opt_sec_none, "none" }, + + { Opt_sec_err, NULL } +}; + +static const match_table_t cifs_upcall_target = { + { Opt_upcall_target_mount, "mount" }, + { Opt_upcall_target_application, "app" }, + { Opt_upcall_target_err, NULL } +}; + +const struct fs_parameter_spec smb3_fs_parameters[] = { + /* Mount options that take no arguments */ + fsparam_flag_no("user_xattr", Opt_user_xattr), + fsparam_flag_no("forceuid", Opt_forceuid), + fsparam_flag_no("multichannel", Opt_multichannel), + fsparam_flag_no("forcegid", Opt_forcegid), + fsparam_flag("noblocksend", Opt_noblocksend), + fsparam_flag("noautotune", Opt_noautotune), + fsparam_flag_no("lease", Opt_lease), + fsparam_flag_no("hard", Opt_hard), + fsparam_flag_no("soft", Opt_soft), + fsparam_flag_no("perm", Opt_perm), + fsparam_flag("nodelete", Opt_nodelete), + fsparam_flag_no("mapposix", Opt_mapposix), + fsparam_flag("mapchars", Opt_mapchars), + fsparam_flag("nomapchars", Opt_nomapchars), + fsparam_flag_no("sfu", Opt_sfu), + fsparam_flag("nodfs", Opt_nodfs), + fsparam_flag_no("posixpaths", Opt_posixpaths), + fsparam_flag_no("unix", Opt_unix), + fsparam_flag_no("linux", Opt_unix), + fsparam_flag_no("posix", Opt_unix), + fsparam_flag("nocase", Opt_nocase), + fsparam_flag("ignorecase", Opt_nocase), + fsparam_flag_no("brl", Opt_brl), + fsparam_flag_no("handlecache", Opt_handlecache), + fsparam_flag("forcemandatorylock", Opt_forcemandatorylock), + fsparam_flag("forcemand", Opt_forcemandatorylock), + fsparam_flag("setuidfromacl", Opt_setuidfromacl), + fsparam_flag("idsfromsid", Opt_setuidfromacl), + fsparam_flag_no("setuids", Opt_setuids), + fsparam_flag_no("dynperm", Opt_dynperm), + fsparam_flag_no("intr", Opt_intr), + fsparam_flag_no("strictsync", Opt_strictsync), + fsparam_flag_no("serverino", Opt_serverino), + fsparam_flag("rwpidforward", Opt_rwpidforward), + fsparam_flag("cifsacl", Opt_cifsacl), + fsparam_flag_no("acl", Opt_acl), + fsparam_flag("locallease", Opt_locallease), + fsparam_flag("sign", Opt_sign), + fsparam_flag("ignore_signature", Opt_ignore_signature), + fsparam_flag("signloosely", Opt_ignore_signature), + fsparam_flag("seal", Opt_seal), + fsparam_flag("noac", Opt_noac), + fsparam_flag("fsc", Opt_fsc), + fsparam_flag("mfsymlinks", Opt_mfsymlinks), + fsparam_flag("multiuser", Opt_multiuser), + fsparam_flag("sloppy", Opt_sloppy), + fsparam_flag("nosharesock", Opt_nosharesock), + fsparam_flag_no("persistenthandles", Opt_persistent), + fsparam_flag_no("resilienthandles", Opt_resilient), + fsparam_flag_no("tcpnodelay", Opt_tcp_nodelay), + fsparam_flag("nosparse", Opt_nosparse), + fsparam_flag("domainauto", Opt_domainauto), + fsparam_flag("rdma", Opt_rdma), + fsparam_flag("modesid", Opt_modesid), + fsparam_flag("modefromsid", Opt_modesid), + fsparam_flag("rootfs", Opt_rootfs), + fsparam_flag("compress", Opt_compress), + fsparam_flag("witness", Opt_witness), + fsparam_flag_no("nativesocket", Opt_nativesocket), + fsparam_flag_no("unicode", Opt_unicode), + fsparam_flag_no("nbsessinit", Opt_nbsessinit), + + /* Mount options which take uid or gid */ + fsparam_uid("backupuid", Opt_backupuid), + fsparam_gid("backupgid", Opt_backupgid), + fsparam_uid("uid", Opt_uid), + fsparam_uid("cruid", Opt_cruid), + fsparam_gid("gid", Opt_gid), + + /* Mount options which take numeric value */ + fsparam_u32("file_mode", Opt_file_mode), + fsparam_u32("dirmode", Opt_dirmode), + fsparam_u32("dir_mode", Opt_dirmode), + fsparam_u32("port", Opt_port), + fsparam_u32("min_enc_offload", Opt_min_enc_offload), + fsparam_u32("retrans", Opt_retrans), + fsparam_u32("esize", Opt_min_enc_offload), + fsparam_u32("bsize", Opt_blocksize), + fsparam_u32("rasize", Opt_rasize), + fsparam_u32("rsize", Opt_rsize), + fsparam_u32("wsize", Opt_wsize), + fsparam_u32("actimeo", Opt_actimeo), + fsparam_u32("acdirmax", Opt_acdirmax), + fsparam_u32("acregmax", Opt_acregmax), + fsparam_u32("closetimeo", Opt_closetimeo), + fsparam_u32("echo_interval", Opt_echo_interval), + fsparam_u32("max_credits", Opt_max_credits), + fsparam_u32("max_cached_dirs", Opt_max_cached_dirs), + fsparam_u32("handletimeout", Opt_handletimeout), + fsparam_u64("snapshot", Opt_snapshot), + fsparam_u32("max_channels", Opt_max_channels), + + /* Mount options which take string value */ + fsparam_string("source", Opt_source), + fsparam_string("user", Opt_user), + fsparam_string("username", Opt_user), + fsparam_string("pass", Opt_pass), + fsparam_string("password", Opt_pass), + fsparam_string("pass2", Opt_pass2), + fsparam_string("password2", Opt_pass2), + fsparam_string("ip", Opt_ip), + fsparam_string("addr", Opt_ip), + fsparam_string("domain", Opt_domain), + fsparam_string("dom", Opt_domain), + fsparam_string("srcaddr", Opt_srcaddr), + fsparam_string("iocharset", Opt_iocharset), + fsparam_string("netbiosname", Opt_netbiosname), + fsparam_string("servern", Opt_servern), + fsparam_string("ver", Opt_ver), + fsparam_string("vers", Opt_vers), + fsparam_string("sec", Opt_sec), + fsparam_string("cache", Opt_cache), + fsparam_string("reparse", Opt_reparse), + fsparam_string("upcall_target", Opt_upcalltarget), + fsparam_string("symlink", Opt_symlink), + fsparam_string("symlinkroot", Opt_symlinkroot), + + /* Arguments that should be ignored */ + fsparam_flag("guest", Opt_ignore), + fsparam_flag("noatime", Opt_ignore), + fsparam_flag("relatime", Opt_ignore), + fsparam_flag("_netdev", Opt_ignore), + fsparam_flag_no("suid", Opt_ignore), + fsparam_flag_no("exec", Opt_ignore), + fsparam_flag_no("dev", Opt_ignore), + fsparam_flag_no("mand", Opt_ignore), + fsparam_flag_no("auto", Opt_ignore), + fsparam_string("cred", Opt_ignore), + fsparam_string("credentials", Opt_ignore), + /* + * UNC and prefixpath is now extracted from Opt_source + * in the new mount API so we can just ignore them going forward. + */ + fsparam_string("unc", Opt_ignore), + fsparam_string("prefixpath", Opt_ignore), + {} +}; + +static int +cifs_parse_security_flavors(struct fs_context *fc, char *value, struct smb3_fs_context *ctx) +{ + + substring_t args[MAX_OPT_ARGS]; + + /* + * With mount options, the last one should win. Reset any existing + * settings back to default. + */ + ctx->sectype = Unspecified; + ctx->sign = false; + + switch (match_token(value, cifs_secflavor_tokens, args)) { + case Opt_sec_krb5p: + cifs_errorf(fc, "sec=krb5p is not supported. Use sec=krb5,seal instead\n"); + return 1; + case Opt_sec_krb5i: + ctx->sign = true; + fallthrough; + case Opt_sec_krb5: + ctx->sectype = Kerberos; + break; + case Opt_sec_ntlmsspi: + ctx->sign = true; + fallthrough; + case Opt_sec_ntlmssp: + ctx->sectype = RawNTLMSSP; + break; + case Opt_sec_ntlmv2i: + ctx->sign = true; + fallthrough; + case Opt_sec_ntlmv2: + ctx->sectype = NTLMv2; + break; + case Opt_sec_none: + ctx->nullauth = 1; + kfree(ctx->username); + ctx->username = NULL; + break; + default: + cifs_errorf(fc, "bad security option: %s\n", value); + return 1; + } + + return 0; +} + +static int +cifs_parse_upcall_target(struct fs_context *fc, char *value, struct smb3_fs_context *ctx) +{ + substring_t args[MAX_OPT_ARGS]; + + ctx->upcall_target = UPTARGET_UNSPECIFIED; + + switch (match_token(value, cifs_upcall_target, args)) { + case Opt_upcall_target_mount: + ctx->upcall_target = UPTARGET_MOUNT; + break; + case Opt_upcall_target_application: + ctx->upcall_target = UPTARGET_APP; + break; + + default: + cifs_errorf(fc, "bad upcall target: %s\n", value); + return 1; + } + + return 0; +} + +static const match_table_t cifs_cacheflavor_tokens = { + { Opt_cache_loose, "loose" }, + { Opt_cache_strict, "strict" }, + { Opt_cache_none, "none" }, + { Opt_cache_ro, "ro" }, + { Opt_cache_rw, "singleclient" }, + { Opt_cache_err, NULL } +}; + +static int +cifs_parse_cache_flavor(struct fs_context *fc, char *value, struct smb3_fs_context *ctx) +{ + substring_t args[MAX_OPT_ARGS]; + + switch (match_token(value, cifs_cacheflavor_tokens, args)) { + case Opt_cache_loose: + ctx->direct_io = false; + ctx->strict_io = false; + ctx->cache_ro = false; + ctx->cache_rw = false; + break; + case Opt_cache_strict: + ctx->direct_io = false; + ctx->strict_io = true; + ctx->cache_ro = false; + ctx->cache_rw = false; + break; + case Opt_cache_none: + ctx->direct_io = true; + ctx->strict_io = false; + ctx->cache_ro = false; + ctx->cache_rw = false; + break; + case Opt_cache_ro: + ctx->direct_io = false; + ctx->strict_io = false; + ctx->cache_ro = true; + ctx->cache_rw = false; + break; + case Opt_cache_rw: + ctx->direct_io = false; + ctx->strict_io = false; + ctx->cache_ro = false; + ctx->cache_rw = true; + break; + default: + cifs_errorf(fc, "bad cache= option: %s\n", value); + return 1; + } + return 0; +} + +static const match_table_t reparse_flavor_tokens = { + { Opt_reparse_default, "default" }, + { Opt_reparse_none, "none" }, + { Opt_reparse_nfs, "nfs" }, + { Opt_reparse_wsl, "wsl" }, + { Opt_reparse_err, NULL }, +}; + +static int parse_reparse_flavor(struct fs_context *fc, char *value, + struct smb3_fs_context *ctx) +{ + substring_t args[MAX_OPT_ARGS]; + + switch (match_token(value, reparse_flavor_tokens, args)) { + case Opt_reparse_default: + ctx->reparse_type = CIFS_REPARSE_TYPE_DEFAULT; + break; + case Opt_reparse_none: + ctx->reparse_type = CIFS_REPARSE_TYPE_NONE; + break; + case Opt_reparse_nfs: + ctx->reparse_type = CIFS_REPARSE_TYPE_NFS; + break; + case Opt_reparse_wsl: + ctx->reparse_type = CIFS_REPARSE_TYPE_WSL; + break; + default: + cifs_errorf(fc, "bad reparse= option: %s\n", value); + return 1; + } + return 0; +} + +static const match_table_t symlink_flavor_tokens = { + { Opt_symlink_default, "default" }, + { Opt_symlink_none, "none" }, + { Opt_symlink_native, "native" }, + { Opt_symlink_unix, "unix" }, + { Opt_symlink_mfsymlinks, "mfsymlinks" }, + { Opt_symlink_sfu, "sfu" }, + { Opt_symlink_nfs, "nfs" }, + { Opt_symlink_wsl, "wsl" }, + { Opt_symlink_err, NULL }, +}; + +static int parse_symlink_flavor(struct fs_context *fc, char *value, + struct smb3_fs_context *ctx) +{ + substring_t args[MAX_OPT_ARGS]; + + switch (match_token(value, symlink_flavor_tokens, args)) { + case Opt_symlink_default: + ctx->symlink_type = CIFS_SYMLINK_TYPE_DEFAULT; + break; + case Opt_symlink_none: + ctx->symlink_type = CIFS_SYMLINK_TYPE_NONE; + break; + case Opt_symlink_native: + ctx->symlink_type = CIFS_SYMLINK_TYPE_NATIVE; + break; + case Opt_symlink_unix: + ctx->symlink_type = CIFS_SYMLINK_TYPE_UNIX; + break; + case Opt_symlink_mfsymlinks: + ctx->symlink_type = CIFS_SYMLINK_TYPE_MFSYMLINKS; + break; + case Opt_symlink_sfu: + ctx->symlink_type = CIFS_SYMLINK_TYPE_SFU; + break; + case Opt_symlink_nfs: + ctx->symlink_type = CIFS_SYMLINK_TYPE_NFS; + break; + case Opt_symlink_wsl: + ctx->symlink_type = CIFS_SYMLINK_TYPE_WSL; + break; + default: + cifs_errorf(fc, "bad symlink= option: %s\n", value); + return 1; + } + return 0; +} + +#define DUP_CTX_STR(field) \ +do { \ + if (ctx->field) { \ + new_ctx->field = kstrdup(ctx->field, GFP_KERNEL); \ + if (new_ctx->field == NULL) { \ + smb3_cleanup_fs_context_contents(new_ctx); \ + return -ENOMEM; \ + } \ + } \ +} while (0) + +int +smb3_fs_context_dup(struct smb3_fs_context *new_ctx, struct smb3_fs_context *ctx) +{ + memcpy(new_ctx, ctx, sizeof(*ctx)); + new_ctx->prepath = NULL; + new_ctx->nodename = NULL; + new_ctx->username = NULL; + new_ctx->password = NULL; + new_ctx->password2 = NULL; + new_ctx->server_hostname = NULL; + new_ctx->domainname = NULL; + new_ctx->UNC = NULL; + new_ctx->source = NULL; + new_ctx->iocharset = NULL; + new_ctx->leaf_fullpath = NULL; + new_ctx->dns_dom = NULL; + new_ctx->symlinkroot = NULL; + /* + * Make sure to stay in sync with smb3_cleanup_fs_context_contents() + */ + DUP_CTX_STR(prepath); + DUP_CTX_STR(username); + DUP_CTX_STR(password); + DUP_CTX_STR(password2); + DUP_CTX_STR(server_hostname); + DUP_CTX_STR(UNC); + DUP_CTX_STR(source); + DUP_CTX_STR(domainname); + DUP_CTX_STR(nodename); + DUP_CTX_STR(iocharset); + DUP_CTX_STR(leaf_fullpath); + DUP_CTX_STR(dns_dom); + DUP_CTX_STR(symlinkroot); + + return 0; +} + +static int +cifs_parse_smb_version(struct fs_context *fc, char *value, struct smb3_fs_context *ctx, bool is_smb3) +{ + substring_t args[MAX_OPT_ARGS]; + + switch (match_token(value, cifs_smb_version_tokens, args)) { +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + case Smb_1: + if (disable_legacy_dialects) { + cifs_errorf(fc, "mount with legacy dialect disabled\n"); + return 1; + } + if (is_smb3) { + cifs_errorf(fc, "vers=1.0 (cifs) not permitted when mounting with smb3\n"); + return 1; + } + cifs_errorf(fc, "Use of the less secure dialect vers=1.0 is not recommended unless required for access to very old servers\n"); + ctx->ops = &smb1_operations; + ctx->vals = &smb1_values; + break; + case Smb_20: + if (disable_legacy_dialects) { + cifs_errorf(fc, "mount with legacy dialect disabled\n"); + return 1; + } + if (is_smb3) { + cifs_errorf(fc, "vers=2.0 not permitted when mounting with smb3\n"); + return 1; + } + ctx->ops = &smb20_operations; + ctx->vals = &smb20_values; + break; +#else + case Smb_1: + cifs_errorf(fc, "vers=1.0 (cifs) mount not permitted when legacy dialects disabled\n"); + return 1; + case Smb_20: + cifs_errorf(fc, "vers=2.0 mount not permitted when legacy dialects disabled\n"); + return 1; +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + case Smb_21: + ctx->ops = &smb21_operations; + ctx->vals = &smb21_values; + break; + case Smb_30: + ctx->ops = &smb30_operations; + ctx->vals = &smb30_values; + break; + case Smb_302: + ctx->ops = &smb30_operations; /* currently identical with 3.0 */ + ctx->vals = &smb302_values; + break; + case Smb_311: + ctx->ops = &smb311_operations; + ctx->vals = &smb311_values; + break; + case Smb_3any: + ctx->ops = &smb30_operations; /* currently identical with 3.0 */ + ctx->vals = &smb3any_values; + break; + case Smb_default: + ctx->ops = &smb30_operations; + ctx->vals = &smbdefault_values; + break; + default: + cifs_errorf(fc, "Unknown vers= option specified: %s\n", value); + return 1; + } + return 0; +} + +/* + * Remove duplicate path delimiters. Windows is supposed to do that + * but there are some bugs that prevent rename from working if there are + * multiple delimiters. + * + * Return a sanitized duplicate of @path or NULL for empty prefix paths. + * Otherwise, return ERR_PTR. + * + * @gfp indicates the GFP_* flags for kstrdup. + * The caller is responsible for freeing the original. + */ +#define IS_DELIM(c) ((c) == '/' || (c) == '\\') +char *cifs_sanitize_prepath(char *prepath, gfp_t gfp) +{ + char *cursor1 = prepath, *cursor2 = prepath; + char *s; + + /* skip all prepended delimiters */ + while (IS_DELIM(*cursor1)) + cursor1++; + + /* exit in case of only delimiters */ + if (!*cursor1) + return NULL; + + /* copy the first letter */ + *cursor2 = *cursor1; + + /* copy the remainder... */ + while (*(cursor1++)) { + /* ... skipping all duplicated delimiters */ + if (IS_DELIM(*cursor1) && IS_DELIM(*cursor2)) + continue; + *(++cursor2) = *cursor1; + } + + /* if the last character is a delimiter, skip it */ + if (IS_DELIM(*(cursor2 - 1))) + cursor2--; + + *cursor2 = '\0'; + if (!*prepath) + return NULL; + s = kstrdup(prepath, gfp); + if (!s) + return ERR_PTR(-ENOMEM); + return s; +} + +/* + * Return full path based on the values of @ctx->{UNC,prepath}. + * + * It is assumed that both values were already parsed by smb3_parse_devname(). + */ +char *smb3_fs_context_fullpath(const struct smb3_fs_context *ctx, char dirsep) +{ + size_t ulen, plen; + char *s; + + ulen = strlen(ctx->UNC); + plen = ctx->prepath ? strlen(ctx->prepath) + 1 : 0; + + s = kmalloc(ulen + plen + 1, GFP_KERNEL); + if (!s) + return ERR_PTR(-ENOMEM); + memcpy(s, ctx->UNC, ulen); + if (plen) { + s[ulen] = dirsep; + memcpy(s + ulen + 1, ctx->prepath, plen); + } + s[ulen + plen] = '\0'; + convert_delimiter(s, dirsep); + return s; +} + +/* + * Parse a devname into substrings and populate the ctx->UNC and ctx->prepath + * fields with the result. Returns 0 on success and an error otherwise + * (e.g. ENOMEM or EINVAL) + */ +int +smb3_parse_devname(const char *devname, struct smb3_fs_context *ctx) +{ + char *pos; + const char *delims = "/\\"; + size_t len; + int rc; + + if (unlikely(!devname || !*devname)) { + cifs_dbg(VFS, "Device name not specified\n"); + return -EINVAL; + } + + /* make sure we have a valid UNC double delimiter prefix */ + len = strspn(devname, delims); + if (len != 2) { + cifs_dbg(VFS, "UNC: path must begin with // or \\\\\n"); + return -EINVAL; + } + + /* find delimiter between host and sharename */ + pos = strpbrk(devname + 2, delims); + if (!pos) { + cifs_dbg(VFS, "UNC: missing delimiter between hostname and share name\n"); + return -EINVAL; + } + + /* record the server hostname */ + kfree(ctx->server_hostname); + ctx->server_hostname = kstrndup(devname + 2, pos - devname - 2, GFP_KERNEL); + if (!ctx->server_hostname) + return -ENOMEM; + + /* skip past delimiter */ + ++pos; + + /* now go until next delimiter or end of string */ + len = strcspn(pos, delims); + if (!len) { + cifs_dbg(VFS, "UNC: missing share name\n"); + return -EINVAL; + } + + /* move "pos" up to delimiter or NULL */ + pos += len; + kfree(ctx->UNC); + ctx->UNC = kstrndup(devname, pos - devname, GFP_KERNEL); + if (!ctx->UNC) + return -ENOMEM; + + convert_delimiter(ctx->UNC, '\\'); + + /* skip any delimiter */ + if (*pos == '/' || *pos == '\\') + pos++; + + kfree(ctx->prepath); + ctx->prepath = NULL; + + /* If pos is NULL then no prepath */ + if (!*pos) + return 0; + + ctx->prepath = cifs_sanitize_prepath(pos, GFP_KERNEL); + if (IS_ERR(ctx->prepath)) { + rc = PTR_ERR(ctx->prepath); + ctx->prepath = NULL; + return rc; + } + + return 0; +} + +static int smb3_handle_conflicting_options(struct fs_context *fc) +{ + struct smb3_fs_context *ctx = smb3_fc2context(fc); + + if (ctx->rdma && ctx->vals->protocol_id < SMB30_PROT_ID) { + cifs_errorf(fc, "SMB Direct requires Version >=3.0\n"); + return -EOPNOTSUPP; + } + + if (ctx->multiuser && !IS_ENABLED(CONFIG_KEYS)) { + cifs_errorf(fc, "Multiuser mounts require kernels with CONFIG_KEYS enabled\n"); + return -EOPNOTSUPP; + } + + if (ctx->multiuser && ctx->upcall_target == UPTARGET_MOUNT) { + cifs_errorf(fc, "multiuser mount option not supported with upcalltarget set as 'mount'\n"); + return -EINVAL; + } + + if (ctx->uid_specified && !ctx->forceuid_specified) { + ctx->override_uid = 1; + pr_notice("enabling forceuid mount option implicitly because uid= option is specified\n"); + } + + if (ctx->gid_specified && !ctx->forcegid_specified) { + ctx->override_gid = 1; + pr_notice("enabling forcegid mount option implicitly because gid= option is specified\n"); + } + + if (ctx->override_uid && !ctx->uid_specified) { + ctx->override_uid = 0; + pr_notice("ignoring forceuid mount option specified with no uid= option\n"); + } + + if (ctx->override_gid && !ctx->gid_specified) { + ctx->override_gid = 0; + pr_notice("ignoring forcegid mount option specified with no gid= option\n"); + } + + if (ctx->multichannel_specified) { + if (ctx->multichannel) { + if (!ctx->max_channels_specified) { + ctx->max_channels = 2; + } else if (ctx->max_channels == 1) { + cifs_errorf(fc, + "max_channels must be greater than 1 when multichannel is enabled\n"); + return -EINVAL; + } + } else { + if (!ctx->max_channels_specified) { + ctx->max_channels = 1; + } else if (ctx->max_channels > 1) { + cifs_errorf(fc, + "max_channels must be equal to 1 when multichannel is disabled\n"); + return -EINVAL; + } + } + } else if (ctx->max_channels_specified) { + if (ctx->max_channels > 1) + ctx->multichannel = true; + else + ctx->multichannel = false; + } + + /* clear parse-time latches so they don't persist across remounts */ + ctx->multichannel_specified = false; + ctx->max_channels_specified = false; + + return 0; +} + +static void smb3_fs_context_free(struct fs_context *fc); +static int smb3_fs_context_parse_param(struct fs_context *fc, + struct fs_parameter *param); +static int smb3_fs_context_parse_monolithic(struct fs_context *fc, + void *data); +static int smb3_get_tree(struct fs_context *fc); +static void smb3_sync_ses_chan_max(struct cifs_ses *ses, size_t max_channels); +static int smb3_reconfigure(struct fs_context *fc); + +static const struct fs_context_operations smb3_fs_context_ops = { + .free = smb3_fs_context_free, + .parse_param = smb3_fs_context_parse_param, + .parse_monolithic = smb3_fs_context_parse_monolithic, + .get_tree = smb3_get_tree, + .reconfigure = smb3_reconfigure, +}; + +/* + * Parse a monolithic block of data from sys_mount(). + * smb3_fs_context_parse_monolithic - Parse key[=val][,key[=val]]* mount data + * @ctx: The superblock configuration to fill in. + * @data: The data to parse + * + * Parse a blob of data that's in key[=val][,key[=val]]* form. This can be + * called from the ->monolithic_mount_data() fs_context operation. + * + * Returns 0 on success or the error returned by the ->parse_option() fs_context + * operation on failure. + */ +static int smb3_fs_context_parse_monolithic(struct fs_context *fc, + void *data) +{ + char *options = data, *key; + int ret = 0; + + if (!options) + return 0; + + ret = security_sb_eat_lsm_opts(options, &fc->security); + if (ret) + return ret; + + /* BB Need to add support for sep= here TBD */ + while ((key = strsep(&options, ",")) != NULL) { + size_t len; + char *value; + + if (*key == 0) + break; + + /* Check if following character is the deliminator If yes, + * we have encountered a double deliminator reset the NULL + * character to the deliminator + */ + while (options && options[0] == ',') { + len = strlen(key); + strcpy(key + len, options); + options = strchr(options, ','); + if (options) + *options++ = 0; + } + + + value = strchr(key, '='); + if (value) { + if (value == key) + continue; + *value++ = 0; + } + + ret = vfs_parse_fs_string(fc, key, value); + if (ret < 0) + break; + } + return ret; +} + +/* + * smb3_fs_context_validate - check initial-mount-only constraints: + * UNC presence, address resolution, dialect warnings + * + * @fc: generic mount context + */ +static int smb3_fs_context_validate(struct fs_context *fc) +{ + struct smb3_fs_context *ctx = smb3_fc2context(fc); + int rc; + + rc = smb3_handle_conflicting_options(fc); + if (rc) + return rc; + + if (ctx->got_version == false) + pr_warn_once("No dialect specified on mount. Default has changed to a more secure dialect, SMB2.1 or later (e.g. SMB3.1.1), from CIFS (SMB1). To use the less secure SMB1 dialect to access old servers which do not support SMB3.1.1 (or even SMB3 or SMB2.1) specify vers=1.0 on mount.\n"); + + + if (!ctx->UNC) { + cifs_errorf(fc, "CIFS mount error: No usable UNC path provided in device string!\n"); + return -1; + } + + /* make sure UNC has a share name */ + if (strlen(ctx->UNC) < 3 || !strchr(ctx->UNC + 3, '\\')) { + cifs_errorf(fc, "Malformed UNC. Unable to find share name.\n"); + return -ENOENT; + } + + if (!ctx->got_ip) { + int len; + const char *slash; + + /* No ip= option specified? Try to get it from UNC */ + /* Use the address part of the UNC. */ + slash = strchr(&ctx->UNC[2], '\\'); + len = slash - &ctx->UNC[2]; + if (!cifs_convert_address((struct sockaddr *)&ctx->dstaddr, + &ctx->UNC[2], len)) { + pr_err("Unable to determine destination address\n"); + return -EHOSTUNREACH; + } + } + + /* set the port that we got earlier */ + cifs_set_port((struct sockaddr *)&ctx->dstaddr, ctx->port); + + return 0; +} + +static int smb3_get_tree_common(struct fs_context *fc) +{ + struct smb3_fs_context *ctx = smb3_fc2context(fc); + struct dentry *root; + int rc = 0; + + root = cifs_smb3_do_mount(fc, ctx); + if (IS_ERR(root)) + return PTR_ERR(root); + + fc->root = root; + + return rc; +} + +/* + * Create an SMB3 superblock from the parameters passed. + */ +static int smb3_get_tree(struct fs_context *fc) +{ + int err = smb3_fs_context_validate(fc); + int ret; + + if (err) + return err; + cifs_mount_lock(); + ret = smb3_get_tree_common(fc); + cifs_mount_unlock(); + return ret; +} + +static void smb3_fs_context_free(struct fs_context *fc) +{ + struct smb3_fs_context *ctx = smb3_fc2context(fc); + + smb3_cleanup_fs_context(ctx); +} + +/* + * Compare the old and new proposed context during reconfigure + * and check if the changes are compatible. + */ +static int smb3_verify_reconfigure_ctx(struct fs_context *fc, + struct smb3_fs_context *new_ctx, + struct smb3_fs_context *old_ctx, bool need_recon) +{ + if (new_ctx->posix_paths != old_ctx->posix_paths) { + cifs_errorf(fc, "can not change posixpaths during remount\n"); + return -EINVAL; + } + if (new_ctx->sectype != old_ctx->sectype) { + cifs_errorf(fc, "can not change sec during remount\n"); + return -EINVAL; + } + if (new_ctx->multiuser != old_ctx->multiuser) { + cifs_errorf(fc, "can not change multiuser during remount\n"); + return -EINVAL; + } + if (new_ctx->UNC && + (!old_ctx->UNC || strcmp(new_ctx->UNC, old_ctx->UNC))) { + cifs_errorf(fc, "can not change UNC during remount\n"); + return -EINVAL; + } + if (new_ctx->username && + (!old_ctx->username || strcmp(new_ctx->username, old_ctx->username))) { + cifs_errorf(fc, "can not change username during remount\n"); + return -EINVAL; + } + if (new_ctx->password && + (!old_ctx->password || strcmp(new_ctx->password, old_ctx->password))) { + if (need_recon == false) { + cifs_errorf(fc, + "can not change password of active session during remount\n"); + return -EINVAL; + } else if (old_ctx->sectype == Kerberos) { + cifs_errorf(fc, + "can not change password for Kerberos via remount\n"); + return -EINVAL; + } + } + if (new_ctx->domainname && + (!old_ctx->domainname || strcmp(new_ctx->domainname, old_ctx->domainname))) { + cifs_errorf(fc, "can not change domainname during remount\n"); + return -EINVAL; + } + if (strcmp(new_ctx->workstation_name, old_ctx->workstation_name)) { + cifs_errorf(fc, "can not change workstation_name during remount\n"); + return -EINVAL; + } + if (new_ctx->nodename && + (!old_ctx->nodename || strcmp(new_ctx->nodename, old_ctx->nodename))) { + cifs_errorf(fc, "can not change nodename during remount\n"); + return -EINVAL; + } + if (new_ctx->iocharset && + (!old_ctx->iocharset || strcmp(new_ctx->iocharset, old_ctx->iocharset))) { + cifs_errorf(fc, "can not change iocharset during remount\n"); + return -EINVAL; + } + if (new_ctx->unicode != old_ctx->unicode) { + cifs_errorf(fc, "can not change unicode during remount\n"); + return -EINVAL; + } + if (new_ctx->rfc1001_sessinit != old_ctx->rfc1001_sessinit) { + cifs_errorf(fc, "can not change nbsessinit during remount\n"); + return -EINVAL; + } + + return 0; +} + +#define STEAL_STRING(cifs_sb, ctx, field) \ +do { \ + kfree(ctx->field); \ + ctx->field = cifs_sb->ctx->field; \ + cifs_sb->ctx->field = NULL; \ +} while (0) + +#define STEAL_STRING_SENSITIVE(cifs_sb, ctx, field) \ +do { \ + kfree_sensitive(ctx->field); \ + ctx->field = cifs_sb->ctx->field; \ + cifs_sb->ctx->field = NULL; \ +} while (0) + +int smb3_sync_session_ctx_passwords(struct cifs_sb_info *cifs_sb, struct cifs_ses *ses) +{ + char *password = NULL, *password2 = NULL; + + if (ses->password && + cifs_sb->ctx->password && + strcmp(ses->password, cifs_sb->ctx->password)) { + password = kstrdup(ses->password, GFP_KERNEL); + if (!password) + return -ENOMEM; + } + if (ses->password2 && + cifs_sb->ctx->password2 && + strcmp(ses->password2, cifs_sb->ctx->password2)) { + password2 = kstrdup(ses->password2, GFP_KERNEL); + if (!password2) { + kfree_sensitive(password); + return -ENOMEM; + } + } + + if (password) { + kfree_sensitive(cifs_sb->ctx->password); + cifs_sb->ctx->password = password; + } + if (password2) { + kfree_sensitive(cifs_sb->ctx->password2); + cifs_sb->ctx->password2 = password2; + } + + return 0; +} + +/* + * smb3_sync_ses_chan_max - Synchronize the session's maximum channel count + * @ses: pointer to the old CIFS session structure + * @max_channels: new maximum number of channels to allow + * + * Updates the session's chan_max field to the new value, protecting the update + * with the session's channel lock. This should be called whenever the maximum + * allowed channels for a session changes (e.g., after a remount or reconfigure). + */ +static void smb3_sync_ses_chan_max(struct cifs_ses *ses, size_t max_channels) +{ + spin_lock(&ses->chan_lock); + ses->chan_max = max_channels; + spin_unlock(&ses->chan_lock); +} + +static int smb3_reconfigure(struct fs_context *fc) +{ + struct smb3_fs_context *ctx = smb3_fc2context(fc); + struct smb3_fs_context *new_ctx = NULL; + struct smb3_fs_context *old_ctx = NULL; + struct dentry *root = fc->root; + struct cifs_sb_info *cifs_sb = CIFS_SB(root->d_sb); + struct cifs_ses *ses = cifs_sb_master_tcon(cifs_sb)->ses; + unsigned int rsize = ctx->rsize, wsize = ctx->wsize; + char *new_password = NULL, *new_password2 = NULL; + bool need_recon = false; + bool need_mchan_update; + int rc; + + if (ses->expired_pwd) + need_recon = true; + + rc = smb3_verify_reconfigure_ctx(fc, ctx, cifs_sb->ctx, need_recon); + if (rc) + return rc; + + rc = smb3_handle_conflicting_options(fc); + if (rc) + return rc; + + old_ctx = kzalloc_obj(*old_ctx); + if (!old_ctx) + return -ENOMEM; + + rc = smb3_fs_context_dup(old_ctx, cifs_sb->ctx); + if (rc) + goto free_old_ctx; + + /* + * We can not change UNC/username/password/domainname/ + * workstation_name/nodename/iocharset + * during reconnect so ignore what we have in the new context and + * just use what we already have in cifs_sb->ctx. + */ + STEAL_STRING(cifs_sb, ctx, UNC); + STEAL_STRING(cifs_sb, ctx, source); + STEAL_STRING(cifs_sb, ctx, username); + STEAL_STRING(cifs_sb, ctx, domainname); + STEAL_STRING(cifs_sb, ctx, nodename); + STEAL_STRING(cifs_sb, ctx, iocharset); + + if (!need_recon) { + STEAL_STRING_SENSITIVE(cifs_sb, ctx, password); + } else { + if (ctx->password) { + new_password = kstrdup(ctx->password, GFP_KERNEL); + if (!new_password) { + rc = -ENOMEM; + goto restore_ctx; + } + } else { + STEAL_STRING_SENSITIVE(cifs_sb, ctx, password); + } + } + + /* + * if a new password2 has been specified, then reset it's value + * inside the ses struct + */ + if (ctx->password2) { + new_password2 = kstrdup(ctx->password2, GFP_KERNEL); + if (!new_password2) { + rc = -ENOMEM; + goto restore_ctx; + } + } else { + STEAL_STRING_SENSITIVE(cifs_sb, ctx, password2); + } + + /* if rsize or wsize not passed in on remount, use previous values */ + ctx->rsize = rsize ? CIFS_ALIGN_RSIZE(fc, rsize) : cifs_sb->ctx->rsize; + ctx->wsize = wsize ? CIFS_ALIGN_WSIZE(fc, wsize) : cifs_sb->ctx->wsize; + + new_ctx = kzalloc_obj(*new_ctx); + if (!new_ctx) { + rc = -ENOMEM; + goto restore_ctx; + } + + rc = smb3_fs_context_dup(new_ctx, ctx); + if (rc) + goto restore_ctx; + + need_mchan_update = ctx->multichannel != cifs_sb->ctx->multichannel || + ctx->max_channels != cifs_sb->ctx->max_channels; + + /* + * we may update the passwords in the ses struct below. Make sure we do + * not race with smb2_reconnect + */ + mutex_lock(&ses->session_mutex); + + /* + * smb2_reconnect may swap password and password2 in case session setup + * failed. First get ctx passwords in sync with ses passwords. It should + * be done before committing new passwords. + */ + rc = smb3_sync_session_ctx_passwords(cifs_sb, ses); + if (rc) { + mutex_unlock(&ses->session_mutex); + goto cleanup_new_ctx; + } + + /* + * If multichannel or max_channels has changed, update the session's channels accordingly. + * This may add or remove channels to match the new configuration. + */ + if (need_mchan_update) { + /* Prevent concurrent scaling operations */ + spin_lock(&ses->ses_lock); + if (ses->flags & CIFS_SES_FLAG_SCALE_CHANNELS) { + spin_unlock(&ses->ses_lock); + mutex_unlock(&ses->session_mutex); + rc = -EINVAL; + goto cleanup_new_ctx; + } + ses->flags |= CIFS_SES_FLAG_SCALE_CHANNELS; + spin_unlock(&ses->ses_lock); + } + + /* + * Commit session passwords before any channel work so newly added + * channels authenticate with the new credentials. + */ + if (new_password) { + kfree_sensitive(ses->password); + ses->password = new_password; + new_password = NULL; + } + if (new_password2) { + kfree_sensitive(ses->password2); + ses->password2 = new_password2; + new_password2 = NULL; + } + + if (need_mchan_update) { + /* Synchronize ses->chan_max with the new mount context */ + smb3_sync_ses_chan_max(ses, ctx->max_channels); + + mutex_unlock(&ses->session_mutex); + + smb3_update_ses_channels(ses, ses->server, + false /* from_reconnect */, + false /* disable_mchan */); + + /* Clear scaling flag after operation */ + spin_lock(&ses->ses_lock); + ses->flags &= ~CIFS_SES_FLAG_SCALE_CHANNELS; + spin_unlock(&ses->ses_lock); + } else { + mutex_unlock(&ses->session_mutex); + } + + smb3_cleanup_fs_context_contents(cifs_sb->ctx); + memcpy(cifs_sb->ctx, new_ctx, sizeof(*new_ctx)); + kfree(new_ctx); + new_ctx = NULL; + smb3_cleanup_fs_context(old_ctx); + old_ctx = NULL; + smb3_update_mnt_flags(cifs_sb); +#ifdef CONFIG_CIFS_DFS_UPCALL + if (!rc) + rc = dfs_cache_remount_fs(cifs_sb); +#endif + + return rc; + +cleanup_new_ctx: + smb3_cleanup_fs_context_contents(new_ctx); +restore_ctx: + kfree(new_ctx); + kfree_sensitive(new_password); + kfree_sensitive(new_password2); + smb3_cleanup_fs_context_contents(cifs_sb->ctx); + memcpy(cifs_sb->ctx, old_ctx, sizeof(*old_ctx)); +free_old_ctx: + kfree(old_ctx); + + return rc; +} + +static int smb3_fs_context_parse_param(struct fs_context *fc, + struct fs_parameter *param) +{ + struct fs_parse_result result; + struct smb3_fs_context *ctx = smb3_fc2context(fc); + int i, opt; + bool is_smb3 = !strcmp(fc->fs_type->name, "smb3"); + bool skip_parsing = false; + char *hostname; + + cifs_dbg(FYI, "CIFS: parsing cifs mount option '%s'\n", param->key); + + /* + * fs_parse can not handle string options with an empty value so + * we will need special handling of them. + */ + if (param->type == fs_value_is_string && param->string[0] == 0) { + if (!strcmp("pass", param->key) || !strcmp("password", param->key)) { + skip_parsing = true; + opt = Opt_pass; + } else if (!strcmp("user", param->key) || !strcmp("username", param->key)) { + skip_parsing = true; + opt = Opt_user; + } else if (!strcmp("pass2", param->key) || !strcmp("password2", param->key)) { + skip_parsing = true; + opt = Opt_pass2; + } + } + + if (!skip_parsing) { + opt = fs_parse(fc, smb3_fs_parameters, param, &result); + if (opt < 0) + return ctx->sloppy ? 1 : opt; + } + + switch (opt) { + case Opt_compress: + if (!IS_ENABLED(CONFIG_CIFS_COMPRESSION)) { + cifs_errorf(fc, "CONFIG_CIFS_COMPRESSION kernel config option is unset\n"); + goto cifs_parse_mount_err; + } + ctx->compress = true; + cifs_dbg(VFS, "SMB3 compression support is experimental\n"); + break; + case Opt_nodfs: + ctx->nodfs = 1; + break; + case Opt_hard: + if (result.negated) { + if (ctx->retry == 1) + cifs_dbg(VFS, "conflicting hard vs. soft mount options\n"); + ctx->retry = 0; + } else + ctx->retry = 1; + break; + case Opt_soft: + if (result.negated) + ctx->retry = 1; + else { + if (ctx->retry == 1) + cifs_dbg(VFS, "conflicting hard vs soft mount options\n"); + ctx->retry = 0; + } + break; + case Opt_mapposix: + if (result.negated) + ctx->remap = false; + else { + ctx->remap = true; + ctx->sfu_remap = false; /* disable SFU mapping */ + } + break; + case Opt_mapchars: + if (result.negated) + ctx->sfu_remap = false; + else { + ctx->sfu_remap = true; + ctx->remap = false; /* disable SFM (mapposix) mapping */ + } + break; + case Opt_user_xattr: + if (result.negated) + ctx->no_xattr = 1; + else + ctx->no_xattr = 0; + break; + case Opt_forceuid: + if (result.negated) + ctx->override_uid = 0; + else + ctx->override_uid = 1; + ctx->forceuid_specified = true; + break; + case Opt_forcegid: + if (result.negated) + ctx->override_gid = 0; + else + ctx->override_gid = 1; + ctx->forcegid_specified = true; + break; + case Opt_perm: + if (result.negated) + ctx->noperm = 1; + else + ctx->noperm = 0; + break; + case Opt_dynperm: + if (result.negated) + ctx->dynperm = 0; + else + ctx->dynperm = 1; + break; + case Opt_sfu: + if (result.negated) + ctx->sfu_emul = 0; + else + ctx->sfu_emul = 1; + break; + case Opt_noblocksend: + ctx->noblocksnd = 1; + break; + case Opt_noautotune: + ctx->noautotune = 1; + break; + case Opt_lease: + ctx->no_lease = result.negated; + break; + case Opt_nosparse: + ctx->no_sparse = 1; + break; + case Opt_nodelete: + ctx->nodelete = 1; + break; + case Opt_multichannel: + ctx->multichannel_specified = true; + if (result.negated) + ctx->multichannel = false; + else + ctx->multichannel = true; + break; + case Opt_uid: + ctx->linux_uid = result.uid; + ctx->uid_specified = true; + break; + case Opt_cruid: + ctx->cred_uid = result.uid; + ctx->cruid_specified = true; + break; + case Opt_backupuid: + ctx->backupuid = result.uid; + ctx->backupuid_specified = true; + break; + case Opt_backupgid: + ctx->backupgid = result.gid; + ctx->backupgid_specified = true; + break; + case Opt_gid: + ctx->linux_gid = result.gid; + ctx->gid_specified = true; + break; + case Opt_port: + ctx->port = result.uint_32; + break; + case Opt_file_mode: + ctx->file_mode = result.uint_32; + break; + case Opt_dirmode: + ctx->dir_mode = result.uint_32; + break; + case Opt_min_enc_offload: + ctx->min_offload = result.uint_32; + break; + case Opt_retrans: + ctx->retrans = result.uint_32; + break; + case Opt_blocksize: + /* + * inode blocksize realistically should never need to be + * less than 16K or greater than 16M and default is 1MB. + * Note that small inode block sizes (e.g. 64K) can lead + * to very poor performance of common tools like cp and scp + */ + if ((result.uint_32 < CIFS_MAX_MSGSIZE) || + (result.uint_32 > (4 * SMB3_DEFAULT_IOSIZE))) { + cifs_errorf(fc, "%s: Invalid blocksize\n", + __func__); + goto cifs_parse_mount_err; + } + ctx->bsize = CIFS_ALIGN_BSIZE(fc, result.uint_32); + ctx->got_bsize = true; + break; + case Opt_rasize: + /* + * readahead size realistically should never need to be + * less than 1M (CIFS_DEFAULT_IOSIZE) or greater than 32M + * (perhaps an exception should be considered in the + * for the case of a large number of channels + * when multichannel is negotiated) since that would lead + * to plenty of parallel I/O in flight to the server. + * Note that smaller read ahead sizes would + * hurt performance of common tools like cp and scp + * which often trigger sequential i/o with read ahead + */ + if ((result.uint_32 > (8 * SMB3_DEFAULT_IOSIZE)) || + (result.uint_32 < CIFS_DEFAULT_IOSIZE)) { + cifs_errorf(fc, "%s: Invalid rasize %d vs. %d\n", + __func__, result.uint_32, SMB3_DEFAULT_IOSIZE); + goto cifs_parse_mount_err; + } + ctx->rasize = result.uint_32; + break; + case Opt_rsize: + ctx->rsize = CIFS_ALIGN_RSIZE(fc, result.uint_32); + ctx->got_rsize = true; + ctx->vol_rsize = ctx->rsize; + break; + case Opt_wsize: + ctx->wsize = CIFS_ALIGN_WSIZE(fc, result.uint_32); + ctx->got_wsize = true; + ctx->vol_wsize = ctx->wsize; + break; + case Opt_acregmax: + if (result.uint_32 > CIFS_MAX_ACTIMEO / HZ) { + cifs_errorf(fc, "acregmax too large\n"); + goto cifs_parse_mount_err; + } + ctx->acregmax = HZ * result.uint_32; + break; + case Opt_acdirmax: + if (result.uint_32 > CIFS_MAX_ACTIMEO / HZ) { + cifs_errorf(fc, "acdirmax too large\n"); + goto cifs_parse_mount_err; + } + ctx->acdirmax = HZ * result.uint_32; + break; + case Opt_actimeo: + if (result.uint_32 > CIFS_MAX_ACTIMEO / HZ) { + cifs_errorf(fc, "timeout too large\n"); + goto cifs_parse_mount_err; + } + if ((ctx->acdirmax != CIFS_DEF_ACTIMEO) || + (ctx->acregmax != CIFS_DEF_ACTIMEO)) { + cifs_errorf(fc, "actimeo ignored since acregmax or acdirmax specified\n"); + break; + } + ctx->acdirmax = ctx->acregmax = HZ * result.uint_32; + break; + case Opt_closetimeo: + if (result.uint_32 > SMB3_MAX_DCLOSETIMEO / HZ) { + cifs_errorf(fc, "closetimeo too large\n"); + goto cifs_parse_mount_err; + } + ctx->closetimeo = HZ * result.uint_32; + break; + case Opt_echo_interval: + if (result.uint_32 < SMB_ECHO_INTERVAL_MIN || + result.uint_32 > SMB_ECHO_INTERVAL_MAX) { + cifs_errorf(fc, "echo interval is out of bounds\n"); + goto cifs_parse_mount_err; + } + ctx->echo_interval = result.uint_32; + break; + case Opt_snapshot: + ctx->snapshot_time = result.uint_64; + break; + case Opt_max_credits: + if (result.uint_32 < 20 || result.uint_32 > 60000) { + cifs_errorf(fc, "%s: Invalid max_credits value\n", + __func__); + goto cifs_parse_mount_err; + } + ctx->max_credits = result.uint_32; + break; + case Opt_max_channels: + ctx->max_channels_specified = true; + if (result.uint_32 < 1 || result.uint_32 > CIFS_MAX_CHANNELS) { + cifs_errorf(fc, "%s: Invalid max_channels value, needs to be 1-%d\n", + __func__, CIFS_MAX_CHANNELS); + goto cifs_parse_mount_err; + } + ctx->max_channels = result.uint_32; + break; + case Opt_max_cached_dirs: + if (result.uint_32 < 1) { + cifs_errorf(fc, "%s: Invalid max_cached_dirs, needs to be 1 or more\n", + __func__); + goto cifs_parse_mount_err; + } + ctx->max_cached_dirs = result.uint_32; + break; + case Opt_handletimeout: + ctx->handle_timeout = result.uint_32; + if (ctx->handle_timeout > SMB3_MAX_HANDLE_TIMEOUT) { + cifs_errorf(fc, "Invalid handle cache timeout, longer than 16 minutes\n"); + goto cifs_parse_mount_err; + } + break; + case Opt_source: + kfree(ctx->UNC); + ctx->UNC = NULL; + switch (smb3_parse_devname(param->string, ctx)) { + case 0: + break; + case -ENOMEM: + cifs_errorf(fc, "Unable to allocate memory for devname\n"); + goto cifs_parse_mount_err; + case -EINVAL: + cifs_errorf(fc, "Malformed UNC in devname\n"); + goto cifs_parse_mount_err; + default: + cifs_errorf(fc, "Unknown error parsing devname\n"); + goto cifs_parse_mount_err; + } + kfree(ctx->source); + ctx->source = smb3_fs_context_fullpath(ctx, '/'); + if (IS_ERR(ctx->source)) { + ctx->source = NULL; + cifs_errorf(fc, "OOM when copying UNC string\n"); + goto cifs_parse_mount_err; + } + kfree(fc->source); + fc->source = kstrdup(ctx->source, GFP_KERNEL); + if (fc->source == NULL) { + cifs_errorf(fc, "OOM when copying UNC string\n"); + goto cifs_parse_mount_err; + } + hostname = extract_hostname(ctx->UNC); + if (IS_ERR(hostname)) { + cifs_errorf(fc, "Cannot extract hostname from UNC string\n"); + goto cifs_parse_mount_err; + } + /* last byte, type, is 0x20 for servr type */ + memset(ctx->target_rfc1001_name, 0x20, RFC1001_NAME_LEN_WITH_NULL); + for (i = 0; i < RFC1001_NAME_LEN && hostname[i] != 0; i++) + ctx->target_rfc1001_name[i] = toupper(hostname[i]); + kfree(hostname); + break; + case Opt_user: + kfree(ctx->username); + ctx->username = NULL; + if (ctx->nullauth) + break; + if (strlen(param->string) == 0) { + /* null user, ie. anonymous authentication */ + ctx->nullauth = 1; + break; + } + + if (strnlen(param->string, CIFS_MAX_USERNAME_LEN) == + CIFS_MAX_USERNAME_LEN) { + pr_warn("username too long\n"); + goto cifs_parse_mount_err; + } + ctx->username = no_free_ptr(param->string); + break; + case Opt_pass: + kfree_sensitive(ctx->password); + ctx->password = NULL; + if (strlen(param->string) == 0) + break; + ctx->password = no_free_ptr(param->string); + break; + case Opt_pass2: + kfree_sensitive(ctx->password2); + ctx->password2 = NULL; + if (strlen(param->string) == 0) + break; + ctx->password2 = no_free_ptr(param->string); + break; + case Opt_ip: + if (strlen(param->string) == 0) { + ctx->got_ip = false; + break; + } + if (!cifs_convert_address((struct sockaddr *)&ctx->dstaddr, + param->string, + strlen(param->string))) { + pr_err("bad ip= option (%s)\n", param->string); + goto cifs_parse_mount_err; + } + ctx->got_ip = true; + break; + case Opt_domain: + if (strnlen(param->string, CIFS_MAX_DOMAINNAME_LEN) + == CIFS_MAX_DOMAINNAME_LEN) { + pr_warn("domain name too long\n"); + goto cifs_parse_mount_err; + } + + kfree(ctx->domainname); + ctx->domainname = no_free_ptr(param->string); + cifs_dbg(FYI, "Domain name set\n"); + break; + case Opt_srcaddr: + if (!cifs_convert_address( + (struct sockaddr *)&ctx->srcaddr, + param->string, strlen(param->string))) { + pr_warn("Could not parse srcaddr: %s\n", + param->string); + goto cifs_parse_mount_err; + } + break; + case Opt_iocharset: + if (strnlen(param->string, 1024) >= 65) { + pr_warn("iocharset name too long\n"); + goto cifs_parse_mount_err; + } + + if (strncasecmp(param->string, "default", 7) != 0) { + kfree(ctx->iocharset); + ctx->iocharset = no_free_ptr(param->string); + } + /* if iocharset not set then load_nls_default + * is used by caller + */ + cifs_dbg(FYI, "iocharset set to %s\n", ctx->iocharset); + break; + case Opt_netbiosname: + memset(ctx->source_rfc1001_name, 0x20, + RFC1001_NAME_LEN); + /* + * FIXME: are there cases in which a comma can + * be valid in workstation netbios name (and + * need special handling)? + */ + for (i = 0; i < RFC1001_NAME_LEN; i++) { + /* don't ucase netbiosname for user */ + if (param->string[i] == 0) + break; + ctx->source_rfc1001_name[i] = param->string[i]; + } + /* The string has 16th byte zero still from + * set at top of the function + */ + if (i == RFC1001_NAME_LEN && param->string[i] != 0) + pr_warn("netbiosname longer than 15 truncated\n"); + break; + case Opt_servern: + /* last byte, type, is 0x20 for servr type */ + memset(ctx->target_rfc1001_name, 0x20, + RFC1001_NAME_LEN_WITH_NULL); + /* + * BB are there cases in which a comma can be valid in this + * workstation netbios name (and need special handling)? + */ + + /* user or mount helper must uppercase the netbios name */ + for (i = 0; i < 15; i++) { + if (param->string[i] == 0) + break; + ctx->target_rfc1001_name[i] = param->string[i]; + } + + /* The string has 16th byte zero still from set at top of function */ + if (i == RFC1001_NAME_LEN && param->string[i] != 0) + pr_warn("server netbiosname longer than 15 truncated\n"); + break; + case Opt_nbsessinit: + ctx->rfc1001_sessinit = !result.negated; + cifs_dbg(FYI, "rfc1001_sessinit set to %d\n", ctx->rfc1001_sessinit); + break; + case Opt_ver: + /* version of mount userspace tools, not dialect */ + /* If interface changes in mount.cifs bump to new ver */ + if (strncasecmp(param->string, "1", 1) == 0) { + if (strlen(param->string) > 1) { + pr_warn("Bad mount helper ver=%s. Did you want SMB1 (CIFS) dialect and mean to type vers=1.0 instead?\n", + param->string); + goto cifs_parse_mount_err; + } + /* This is the default */ + break; + } + /* For all other value, error */ + pr_warn("Invalid mount helper version specified\n"); + goto cifs_parse_mount_err; + case Opt_vers: + /* protocol version (dialect) */ + if (cifs_parse_smb_version(fc, param->string, ctx, is_smb3) != 0) + goto cifs_parse_mount_err; + ctx->got_version = true; + break; + case Opt_sec: + if (cifs_parse_security_flavors(fc, param->string, ctx) != 0) + goto cifs_parse_mount_err; + break; + case Opt_upcalltarget: + if (cifs_parse_upcall_target(fc, param->string, ctx) != 0) + goto cifs_parse_mount_err; + break; + case Opt_cache: + if (cifs_parse_cache_flavor(fc, param->string, ctx) != 0) + goto cifs_parse_mount_err; + break; + case Opt_witness: +#ifndef CONFIG_CIFS_SWN_UPCALL + cifs_errorf(fc, "Witness support needs CONFIG_CIFS_SWN_UPCALL config option\n"); + goto cifs_parse_mount_err; +#endif + ctx->witness = true; + pr_warn_once("Witness protocol support is experimental\n"); + break; + case Opt_unicode: + ctx->unicode = !result.negated; + cifs_dbg(FYI, "unicode set to %d\n", ctx->unicode); + break; + case Opt_rootfs: +#ifndef CONFIG_CIFS_ROOT + cifs_dbg(VFS, "rootfs support requires CONFIG_CIFS_ROOT config option\n"); + goto cifs_parse_mount_err; +#endif + ctx->rootfs = true; + break; + case Opt_posixpaths: + if (result.negated) + ctx->posix_paths = 0; + else + ctx->posix_paths = 1; + break; + case Opt_unix: + if (result.negated) { + if (ctx->linux_ext == 1) + pr_warn_once("conflicting posix mount options specified\n"); + ctx->linux_ext = 0; + ctx->no_linux_ext = 1; + } else { + if (ctx->no_linux_ext == 1) + pr_warn_once("conflicting posix mount options specified\n"); + ctx->linux_ext = 1; + ctx->no_linux_ext = 0; + ctx->nonativesocket = 1; /* POSIX mounts use NFS style reparse points */ + } + break; + case Opt_nocase: + ctx->nocase = 1; + break; + case Opt_brl: + if (result.negated) { + /* + * turn off mandatory locking in mode + * if remote locking is turned off since the + * local vfs will do advisory + */ + if (ctx->file_mode == + (S_IALLUGO & ~(S_ISUID | S_IXGRP))) + ctx->file_mode = S_IALLUGO; + ctx->nobrl = 1; + } else + ctx->nobrl = 0; + break; + case Opt_handlecache: + if (result.negated) + ctx->nohandlecache = 1; + else + ctx->nohandlecache = 0; + break; + case Opt_forcemandatorylock: + ctx->mand_lock = 1; + break; + case Opt_setuids: + ctx->setuids = result.negated; + break; + case Opt_intr: + ctx->intr = !result.negated; + break; + case Opt_setuidfromacl: + ctx->setuidfromacl = 1; + break; + case Opt_strictsync: + ctx->nostrictsync = result.negated; + break; + case Opt_serverino: + ctx->server_ino = !result.negated; + break; + case Opt_rwpidforward: + ctx->rwpidforward = 1; + break; + case Opt_modesid: + ctx->mode_ace = 1; + break; + case Opt_cifsacl: + ctx->cifs_acl = !result.negated; + break; + case Opt_acl: + ctx->no_psx_acl = result.negated; + break; + case Opt_locallease: + ctx->local_lease = 1; + break; + case Opt_sign: + ctx->sign = true; + break; + case Opt_ignore_signature: + ctx->sign = true; + ctx->ignore_signature = true; + break; + case Opt_seal: + /* we do not do the following in secFlags because seal + * is a per tree connection (mount) not a per socket + * or per-smb connection option in the protocol + * vol->secFlg |= CIFSSEC_MUST_SEAL; + */ + ctx->seal = 1; + break; + case Opt_noac: + pr_warn("Mount option noac not supported. Instead set /proc/fs/cifs/LookupCacheEnabled to 0\n"); + break; + case Opt_fsc: +#ifndef CONFIG_CIFS_FSCACHE + cifs_errorf(fc, "FS-Cache support needs CONFIG_CIFS_FSCACHE kernel config option set\n"); + goto cifs_parse_mount_err; +#endif + ctx->fsc = true; + break; + case Opt_mfsymlinks: + ctx->mfsymlinks = true; + break; + case Opt_multiuser: + ctx->multiuser = true; + break; + case Opt_sloppy: + ctx->sloppy = true; + break; + case Opt_nosharesock: + ctx->nosharesock = true; + break; + case Opt_persistent: + if (result.negated) { + ctx->nopersistent = true; + if (ctx->persistent) { + cifs_errorf(fc, "persistenthandles mount options conflict\n"); + goto cifs_parse_mount_err; + } + } else { + ctx->persistent = true; + if ((ctx->nopersistent) || (ctx->resilient)) { + cifs_errorf(fc, "persistenthandles mount options conflict\n"); + goto cifs_parse_mount_err; + } + } + break; + case Opt_resilient: + if (result.negated) { + ctx->resilient = false; /* already the default */ + } else { + ctx->resilient = true; + if (ctx->persistent) { + cifs_errorf(fc, "persistenthandles mount options conflict\n"); + goto cifs_parse_mount_err; + } + } + break; + case Opt_tcp_nodelay: + /* tcp nodelay should not usually be needed since we CORK/UNCORK the socket */ + if (result.negated) + ctx->sockopt_tcp_nodelay = false; + else + ctx->sockopt_tcp_nodelay = true; + break; + case Opt_domainauto: + ctx->domainauto = true; + break; + case Opt_rdma: + ctx->rdma = true; + break; + case Opt_reparse: + if (parse_reparse_flavor(fc, param->string, ctx)) + goto cifs_parse_mount_err; + break; + case Opt_nativesocket: + ctx->nonativesocket = result.negated; + break; + case Opt_symlink: + if (parse_symlink_flavor(fc, param->string, ctx)) + goto cifs_parse_mount_err; + break; + case Opt_symlinkroot: + if (param->string[0] != '/') { + cifs_errorf(fc, "symlinkroot mount options must be absolute path\n"); + goto cifs_parse_mount_err; + } + if (strnlen(param->string, PATH_MAX) == PATH_MAX) { + cifs_errorf(fc, "symlinkroot path too long (max path length: %u)\n", + PATH_MAX - 1); + goto cifs_parse_mount_err; + } + kfree(ctx->symlinkroot); + ctx->symlinkroot = param->string; + param->string = NULL; + break; + } + /* case Opt_ignore: - is ignored as expected ... */ + + return 0; + + cifs_parse_mount_err: + kfree_sensitive(ctx->password); + ctx->password = NULL; + kfree_sensitive(ctx->password2); + ctx->password2 = NULL; + kfree(ctx->source); + ctx->source = NULL; + kfree(fc->source); + fc->source = NULL; + return -EINVAL; +} + +int smb3_init_fs_context(struct fs_context *fc) +{ + struct smb3_fs_context *ctx; + char *nodename = utsname()->nodename; + int i; + + ctx = kzalloc_obj(struct smb3_fs_context); + if (unlikely(!ctx)) + return -ENOMEM; + + strscpy(ctx->workstation_name, nodename, sizeof(ctx->workstation_name)); + + /* + * does not have to be perfect mapping since field is + * informational, only used for servers that do not support + * port 445 and it can be overridden at mount time + */ + memset(ctx->source_rfc1001_name, 0x20, RFC1001_NAME_LEN); + for (i = 0; i < strnlen(nodename, RFC1001_NAME_LEN); i++) + ctx->source_rfc1001_name[i] = toupper(nodename[i]); + ctx->source_rfc1001_name[RFC1001_NAME_LEN] = 0; + + /* + * null target name indicates to use *SMBSERVR default called name + * if we end up sending RFC1001 session initialize + */ + ctx->target_rfc1001_name[0] = 0; + + ctx->rfc1001_sessinit = -1; /* autodetect based on port number */ + + ctx->cred_uid = current_uid(); + ctx->linux_uid = current_uid(); + ctx->linux_gid = current_gid(); + /* By default 4MB read ahead size, 1MB block size */ + ctx->bsize = CIFS_DEFAULT_IOSIZE; /* can improve cp performance significantly */ + ctx->rasize = 0; /* 0 = use default (ie negotiated rsize) for read ahead pages */ + + /* + * default to SFM style remapping of seven reserved characters + * unless user overrides it or we negotiate CIFS POSIX where + * it is unnecessary. Can not simultaneously use more than one mapping + * since then readdir could list files that open could not open + */ + ctx->remap = true; + + /* default to only allowing write access to owner of the mount */ + ctx->dir_mode = ctx->file_mode = S_IRUGO | S_IXUGO | S_IWUSR; + + /* ctx->retry default is 0 (i.e. "soft" limited retry not hard retry) */ + /* default is always to request posix paths. */ + ctx->posix_paths = 1; + /* default to using server inode numbers where available */ + ctx->server_ino = 1; + + /* default is to use strict cifs caching semantics */ + ctx->strict_io = true; + + ctx->acregmax = CIFS_DEF_ACTIMEO; + ctx->acdirmax = CIFS_DEF_ACTIMEO; + ctx->closetimeo = SMB3_DEF_DCLOSETIMEO; + ctx->max_cached_dirs = MAX_CACHED_FIDS; + /* Most clients set timeout to 0, allows server to use its default */ + ctx->handle_timeout = 0; /* See MS-SMB2 spec section 2.2.14.2.12 */ + + /* offer SMB2.1 and later (SMB3 etc). Secure and widely accepted */ + ctx->ops = &smb30_operations; + ctx->vals = &smbdefault_values; + + ctx->echo_interval = SMB_ECHO_INTERVAL_DEFAULT; + + /* default to no multichannel (single server connection) */ + ctx->multichannel = false; + ctx->multichannel_specified = false; + ctx->max_channels_specified = false; + ctx->max_channels = 1; + + ctx->backupuid_specified = false; /* no backup intent for a user */ + ctx->backupgid_specified = false; /* no backup intent for a group */ + + ctx->retrans = 0; + ctx->reparse_type = CIFS_REPARSE_TYPE_DEFAULT; + ctx->symlink_type = CIFS_SYMLINK_TYPE_DEFAULT; + ctx->nonativesocket = 0; + + ctx->unicode = -1; /* autodetect, but prefer UNICODE mode */ + +/* + * short int override_uid = -1; + * short int override_gid = -1; + * char *nodename = strdup(utsname()->nodename); + * struct sockaddr *dstaddr = (struct sockaddr *)&vol->dstaddr; + */ + + fc->fs_private = ctx; + fc->ops = &smb3_fs_context_ops; + return 0; +} + +void +smb3_cleanup_fs_context_contents(struct smb3_fs_context *ctx) +{ + if (ctx == NULL) + return; + + /* + * Make sure this stays in sync with smb3_fs_context_dup() + */ + kfree(ctx->username); + ctx->username = NULL; + kfree_sensitive(ctx->password); + ctx->password = NULL; + kfree_sensitive(ctx->password2); + ctx->password2 = NULL; + kfree(ctx->server_hostname); + ctx->server_hostname = NULL; + kfree(ctx->UNC); + ctx->UNC = NULL; + kfree(ctx->source); + ctx->source = NULL; + kfree(ctx->domainname); + ctx->domainname = NULL; + kfree(ctx->nodename); + ctx->nodename = NULL; + kfree(ctx->iocharset); + ctx->iocharset = NULL; + kfree(ctx->prepath); + ctx->prepath = NULL; + kfree(ctx->leaf_fullpath); + ctx->leaf_fullpath = NULL; + kfree(ctx->dns_dom); + ctx->dns_dom = NULL; + kfree(ctx->symlinkroot); + ctx->symlinkroot = NULL; +} + +void +smb3_cleanup_fs_context(struct smb3_fs_context *ctx) +{ + if (!ctx) + return; + smb3_cleanup_fs_context_contents(ctx); + kfree(ctx); +} + +unsigned int smb3_update_mnt_flags(struct cifs_sb_info *cifs_sb) +{ + unsigned int sbflags = cifs_sb_flags(cifs_sb); + struct smb3_fs_context *ctx = cifs_sb->ctx; + + if (ctx->nodfs) + sbflags |= CIFS_MOUNT_NO_DFS; + else + sbflags &= ~CIFS_MOUNT_NO_DFS; + + if (ctx->noperm) + sbflags |= CIFS_MOUNT_NO_PERM; + else + sbflags &= ~CIFS_MOUNT_NO_PERM; + + if (ctx->setuids) + sbflags |= CIFS_MOUNT_SET_UID; + else + sbflags &= ~CIFS_MOUNT_SET_UID; + + if (ctx->setuidfromacl) + sbflags |= CIFS_MOUNT_UID_FROM_ACL; + else + sbflags &= ~CIFS_MOUNT_UID_FROM_ACL; + + if (ctx->server_ino) + sbflags |= CIFS_MOUNT_SERVER_INUM; + else + sbflags &= ~CIFS_MOUNT_SERVER_INUM; + + if (ctx->remap) + sbflags |= CIFS_MOUNT_MAP_SFM_CHR; + else + sbflags &= ~CIFS_MOUNT_MAP_SFM_CHR; + + if (ctx->sfu_remap) + sbflags |= CIFS_MOUNT_MAP_SPECIAL_CHR; + else + sbflags &= ~CIFS_MOUNT_MAP_SPECIAL_CHR; + + if (ctx->no_xattr) + sbflags |= CIFS_MOUNT_NO_XATTR; + else + sbflags &= ~CIFS_MOUNT_NO_XATTR; + + if (ctx->sfu_emul) + sbflags |= CIFS_MOUNT_UNX_EMUL; + else + sbflags &= ~CIFS_MOUNT_UNX_EMUL; + + if (ctx->nobrl) + sbflags |= CIFS_MOUNT_NO_BRL; + else + sbflags &= ~CIFS_MOUNT_NO_BRL; + + if (ctx->nohandlecache) + sbflags |= CIFS_MOUNT_NO_HANDLE_CACHE; + else + sbflags &= ~CIFS_MOUNT_NO_HANDLE_CACHE; + + if (ctx->nostrictsync) + sbflags |= CIFS_MOUNT_NOSSYNC; + else + sbflags &= ~CIFS_MOUNT_NOSSYNC; + + if (ctx->mand_lock) + sbflags |= CIFS_MOUNT_NOPOSIXBRL; + else + sbflags &= ~CIFS_MOUNT_NOPOSIXBRL; + + if (ctx->rwpidforward) + sbflags |= CIFS_MOUNT_RWPIDFORWARD; + else + sbflags &= ~CIFS_MOUNT_RWPIDFORWARD; + + if (ctx->mode_ace) + sbflags |= CIFS_MOUNT_MODE_FROM_SID; + else + sbflags &= ~CIFS_MOUNT_MODE_FROM_SID; + + if (ctx->cifs_acl) + sbflags |= CIFS_MOUNT_CIFS_ACL; + else + sbflags &= ~CIFS_MOUNT_CIFS_ACL; + + if (ctx->backupuid_specified) + sbflags |= CIFS_MOUNT_CIFS_BACKUPUID; + else + sbflags &= ~CIFS_MOUNT_CIFS_BACKUPUID; + + if (ctx->backupgid_specified) + sbflags |= CIFS_MOUNT_CIFS_BACKUPGID; + else + sbflags &= ~CIFS_MOUNT_CIFS_BACKUPGID; + + if (ctx->override_uid) + sbflags |= CIFS_MOUNT_OVERR_UID; + else + sbflags &= ~CIFS_MOUNT_OVERR_UID; + + if (ctx->override_gid) + sbflags |= CIFS_MOUNT_OVERR_GID; + else + sbflags &= ~CIFS_MOUNT_OVERR_GID; + + if (ctx->dynperm) + sbflags |= CIFS_MOUNT_DYNPERM; + else + sbflags &= ~CIFS_MOUNT_DYNPERM; + + if (ctx->fsc) + sbflags |= CIFS_MOUNT_FSCACHE; + else + sbflags &= ~CIFS_MOUNT_FSCACHE; + + if (ctx->multiuser) + sbflags |= CIFS_MOUNT_MULTIUSER | CIFS_MOUNT_NO_PERM; + else + sbflags &= ~CIFS_MOUNT_MULTIUSER; + + + if (ctx->strict_io) + sbflags |= CIFS_MOUNT_STRICT_IO; + else + sbflags &= ~CIFS_MOUNT_STRICT_IO; + + if (ctx->direct_io) + sbflags |= CIFS_MOUNT_DIRECT_IO; + else + sbflags &= ~CIFS_MOUNT_DIRECT_IO; + + if (ctx->mfsymlinks) + sbflags |= CIFS_MOUNT_MF_SYMLINKS; + else + sbflags &= ~CIFS_MOUNT_MF_SYMLINKS; + + if (ctx->mfsymlinks && ctx->sfu_emul) { + /* + * Our SFU ("Services for Unix") emulation allows now + * creating new and reading existing SFU symlinks. + * Older Linux kernel versions were not able to neither + * read existing nor create new SFU symlinks. But + * creating and reading SFU style mknod and FIFOs was + * supported for long time. When "mfsymlinks" and + * "sfu" are both enabled at the same time, it allows + * reading both types of symlinks, but will only create + * them with mfsymlinks format. This allows better + * Apple compatibility, compatibility with older Linux + * kernel clients (probably better for Samba too) + * while still recognizing old Windows style symlinks. + */ + cifs_dbg(VFS, "mount options mfsymlinks and sfu both enabled\n"); + } + sbflags &= ~CIFS_MOUNT_SHUTDOWN; + atomic_set(&cifs_sb->mnt_cifs_flags, sbflags); + return sbflags; +} diff --git a/fs/smb/client/fs_context.h b/fs/smb/client/fs_context.h new file mode 100644 index 000000000..a80a5caff --- /dev/null +++ b/fs/smb/client/fs_context.h @@ -0,0 +1,431 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020, Microsoft Corporation. + * + * Author(s): Steve French + * David Howells + */ + +#ifndef _FS_CONTEXT_H +#define _FS_CONTEXT_H + +#include "cifsglob.h" +#include +#include + +/* Log errors in fs_context (new mount api) but also in dmesg (old style) */ +#define cifs_errorf(fc, fmt, ...) \ + do { \ + errorf(fc, fmt, ## __VA_ARGS__); \ + cifs_dbg(VFS, fmt, ## __VA_ARGS__); \ + } while (0) + +static inline size_t cifs_io_align(struct fs_context *fc, + const char *name, size_t size) +{ + if (!size || !IS_ALIGNED(size, PAGE_SIZE)) { + cifs_errorf(fc, "unaligned %s, making it a multiple of %lu bytes\n", + name, PAGE_SIZE); + size = umax(round_down(size, PAGE_SIZE), PAGE_SIZE); + } + return size; +} + +#define CIFS_ALIGN_WSIZE(_fc, _size) cifs_io_align(_fc, "wsize", _size) +#define CIFS_ALIGN_RSIZE(_fc, _size) cifs_io_align(_fc, "rsize", _size) +#define CIFS_ALIGN_BSIZE(_fc, _size) cifs_io_align(_fc, "bsize", _size) + +enum smb_version { + Smb_1 = 1, + Smb_20, + Smb_21, + Smb_30, + Smb_302, + Smb_311, + Smb_3any, + Smb_default, + Smb_version_err +}; + +enum { + Opt_cache_loose, + Opt_cache_strict, + Opt_cache_none, + Opt_cache_ro, + Opt_cache_rw, + Opt_cache_err +}; + +enum cifs_reparse_parm { + Opt_reparse_default, + Opt_reparse_none, + Opt_reparse_nfs, + Opt_reparse_wsl, + Opt_reparse_err +}; + +enum cifs_symlink_parm { + Opt_symlink_default, + Opt_symlink_none, + Opt_symlink_native, + Opt_symlink_unix, + Opt_symlink_mfsymlinks, + Opt_symlink_sfu, + Opt_symlink_nfs, + Opt_symlink_wsl, + Opt_symlink_err +}; + +enum cifs_sec_param { + Opt_sec_krb5, + Opt_sec_krb5i, + Opt_sec_krb5p, + Opt_sec_ntlmsspi, + Opt_sec_ntlmssp, + Opt_sec_ntlmv2, + Opt_sec_ntlmv2i, + Opt_sec_none, + + Opt_sec_err +}; + +enum cifs_upcall_target_param { + Opt_upcall_target_mount, + Opt_upcall_target_application, + Opt_upcall_target_err +}; + +enum cifs_param { + /* Mount options that take no arguments */ + Opt_user_xattr, + Opt_forceuid, + Opt_forcegid, + Opt_noblocksend, + Opt_noautotune, + Opt_lease, + Opt_nosparse, + Opt_hard, + Opt_soft, + Opt_perm, + Opt_nodelete, + Opt_mapposix, + Opt_mapchars, + Opt_nomapchars, + Opt_sfu, + Opt_nodfs, + Opt_posixpaths, + Opt_unix, + Opt_nocase, + Opt_brl, + Opt_handlecache, + Opt_forcemandatorylock, + Opt_setuidfromacl, + Opt_setuids, + Opt_dynperm, + Opt_intr, + Opt_strictsync, + Opt_serverino, + Opt_rwpidforward, + Opt_cifsacl, + Opt_acl, + Opt_locallease, + Opt_sign, + Opt_ignore_signature, + Opt_seal, + Opt_noac, + Opt_fsc, + Opt_mfsymlinks, + Opt_multiuser, + Opt_sloppy, + Opt_nosharesock, + Opt_persistent, + Opt_resilient, + Opt_tcp_nodelay, + Opt_domainauto, + Opt_rdma, + Opt_modesid, + Opt_rootfs, + Opt_multichannel, + Opt_compress, + Opt_witness, + Opt_is_upcall_target_mount, + Opt_is_upcall_target_application, + Opt_unicode, + + /* Mount options which take numeric value */ + Opt_backupuid, + Opt_backupgid, + Opt_uid, + Opt_cruid, + Opt_gid, + Opt_port, + Opt_file_mode, + Opt_dirmode, + Opt_min_enc_offload, + Opt_retrans, + Opt_blocksize, + Opt_rasize, + Opt_rsize, + Opt_wsize, + Opt_actimeo, + Opt_acdirmax, + Opt_acregmax, + Opt_closetimeo, + Opt_echo_interval, + Opt_max_credits, + Opt_max_cached_dirs, + Opt_snapshot, + Opt_max_channels, + Opt_handletimeout, + + /* Mount options which take string value */ + Opt_source, + Opt_user, + Opt_pass, + Opt_pass2, + Opt_ip, + Opt_domain, + Opt_srcaddr, + Opt_iocharset, + Opt_netbiosname, + Opt_servern, + Opt_nbsessinit, + Opt_ver, + Opt_vers, + Opt_sec, + Opt_cache, + Opt_reparse, + Opt_upcalltarget, + Opt_nativesocket, + Opt_symlink, + Opt_symlinkroot, + + /* Mount options to be ignored */ + Opt_ignore, + + Opt_err +}; + +struct smb3_fs_context { + bool forceuid_specified; + bool forcegid_specified; + bool uid_specified; + bool cruid_specified; + bool gid_specified; + bool sloppy; + bool got_ip; + bool got_version; + bool got_rsize; + bool got_wsize; + bool got_bsize; + unsigned short port; + + char *username; + char *password; + char *password2; + char *domainname; + char *source; + char *server_hostname; + char *UNC; + char *nodename; + char workstation_name[CIFS_MAX_WORKSTATION_LEN]; + char *iocharset; /* local code page for mapping to and from Unicode */ + char source_rfc1001_name[RFC1001_NAME_LEN_WITH_NULL]; /* clnt nb name */ + char target_rfc1001_name[RFC1001_NAME_LEN_WITH_NULL]; /* srvr nb name */ + int rfc1001_sessinit; + kuid_t cred_uid; + kuid_t linux_uid; + kgid_t linux_gid; + kuid_t backupuid; + kgid_t backupgid; + umode_t file_mode; + umode_t dir_mode; + enum securityEnum sectype; /* sectype requested via mnt opts */ + enum upcall_target_enum upcall_target; /* where to upcall for mount */ + bool sign; /* was signing requested via mnt opts? */ + bool ignore_signature:1; + bool retry:1; + bool intr:1; + bool setuids:1; + bool setuidfromacl:1; + bool override_uid:1; + bool override_gid:1; + bool dynperm:1; + bool noperm:1; + bool nodelete:1; + bool mode_ace:1; + bool no_psx_acl:1; /* set if posix acl support should be disabled */ + bool cifs_acl:1; + bool backupuid_specified; /* mount option backupuid is specified */ + bool backupgid_specified; /* mount option backupgid is specified */ + bool no_xattr:1; /* set if xattr (EA) support should be disabled*/ + bool server_ino:1; /* use inode numbers from server ie UniqueId */ + bool direct_io:1; + bool strict_io:1; /* strict cache behavior */ + bool cache_ro:1; + bool cache_rw:1; + bool remap:1; /* set to remap seven reserved chars in filenames */ + bool sfu_remap:1; /* remap seven reserved chars ala SFU */ + bool posix_paths:1; /* unset to not ask for posix pathnames. */ + bool no_linux_ext:1; + bool linux_ext:1; + bool sfu_emul:1; + bool nullauth:1; /* attempt to authenticate with null user */ + bool nocase:1; /* request case insensitive filenames */ + bool nobrl:1; /* disable sending byte range locks to srv */ + bool nohandlecache:1; /* disable caching dir handles if srvr probs */ + bool mand_lock:1; /* send mandatory not posix byte range lock reqs */ + bool seal:1; /* request transport encryption on share */ + bool nodfs:1; /* Do not request DFS, even if available */ + bool local_lease:1; /* check leases only on local system, not remote */ + bool noblocksnd:1; + bool noautotune:1; + bool nostrictsync:1; /* do not force expensive SMBflush on every sync */ + bool no_lease:1; /* disable requesting leases */ + bool no_sparse:1; /* do not attempt to set files sparse */ + bool fsc:1; /* enable fscache */ + bool mfsymlinks:1; /* use Minshall+French Symlinks */ + bool multiuser:1; + bool rwpidforward:1; /* pid forward for read/write operations */ + bool nosharesock:1; + bool persistent:1; + bool nopersistent:1; + bool resilient:1; /* noresilient not required since not fored for CA */ + bool domainauto:1; + bool rdma:1; + bool multichannel:1; + bool multichannel_specified:1; /* true if user specified multichannel or nomultichannel */ + bool max_channels_specified:1; /* true if user specified max_channels */ + bool use_client_guid:1; + /* reuse existing guid for multichannel */ + u8 client_guid[SMB2_CLIENT_GUID_SIZE]; + /* User-specified original r/wsize value */ + unsigned int vol_rsize; + unsigned int vol_wsize; + unsigned int bsize; + unsigned int rasize; + unsigned int rsize; + unsigned int wsize; + unsigned int min_offload; + unsigned int retrans; + bool sockopt_tcp_nodelay:1; + /* attribute cache timeout for files and directories in jiffies */ + unsigned long acregmax; + unsigned long acdirmax; + /* timeout for deferred close of files in jiffies */ + unsigned long closetimeo; + struct smb_version_operations *ops; + struct smb_version_values *vals; + char *prepath; + struct sockaddr_storage dstaddr; /* destination address */ + struct sockaddr_storage srcaddr; /* allow binding to a local IP */ + struct nls_table *local_nls; /* This is a copy of the pointer in cifs_sb */ + unsigned int echo_interval; /* echo interval in secs */ + __u64 snapshot_time; /* needed for timewarp tokens */ + __u32 handle_timeout; /* persistent and durable handle timeout in ms */ + unsigned int max_credits; /* smb3 max_credits 10 < credits < 60000 */ + unsigned int max_channels; + unsigned int max_cached_dirs; + bool compress; /* enable SMB2 messages (READ/WRITE) de/compression */ + bool rootfs:1; /* if it's a SMB root file system */ + bool witness:1; /* use witness protocol */ + int unicode; + char *leaf_fullpath; + struct cifs_ses *dfs_root_ses; + bool dfs_automount:1; /* set for dfs automount only */ + enum cifs_reparse_type reparse_type; + enum cifs_symlink_type symlink_type; + bool nonativesocket:1; + bool dfs_conn:1; /* set for dfs mounts */ + char *dns_dom; + char *symlinkroot; /* top level directory for native SMB symlinks in absolute format */ +}; + +extern const struct fs_parameter_spec smb3_fs_parameters[]; + +static inline enum cifs_symlink_type cifs_symlink_type(struct cifs_sb_info *cifs_sb) +{ + bool posix = cifs_sb_master_tcon(cifs_sb)->posix_extensions; + + if (cifs_sb->ctx->symlink_type != CIFS_SYMLINK_TYPE_DEFAULT) + return cifs_sb->ctx->symlink_type; + + if (cifs_sb->ctx->mfsymlinks) + return CIFS_SYMLINK_TYPE_MFSYMLINKS; + else if (cifs_sb->ctx->sfu_emul) + return CIFS_SYMLINK_TYPE_SFU; + else if (cifs_sb->ctx->linux_ext && !cifs_sb->ctx->no_linux_ext) + return posix ? CIFS_SYMLINK_TYPE_NATIVE : CIFS_SYMLINK_TYPE_UNIX; + else if (cifs_sb->ctx->reparse_type != CIFS_REPARSE_TYPE_NONE) + return CIFS_SYMLINK_TYPE_NATIVE; + return CIFS_SYMLINK_TYPE_NONE; +} + +int smb3_init_fs_context(struct fs_context *fc); +void smb3_cleanup_fs_context_contents(struct smb3_fs_context *ctx); +void smb3_cleanup_fs_context(struct smb3_fs_context *ctx); + +static inline struct smb3_fs_context *smb3_fc2context(const struct fs_context *fc) +{ + return fc->fs_private; +} + +int smb3_fs_context_dup(struct smb3_fs_context *new_ctx, + struct smb3_fs_context *ctx); +int smb3_sync_session_ctx_passwords(struct cifs_sb_info *cifs_sb, + struct cifs_ses *ses); +unsigned int smb3_update_mnt_flags(struct cifs_sb_info *cifs_sb); + +/* + * max deferred close timeout (jiffies) - 2^30 + */ +#define SMB3_MAX_DCLOSETIMEO (1 << 30) +#define SMB3_DEF_DCLOSETIMEO (1 * HZ) /* even 1 sec enough to help eg open/write/close/open/read */ +#define MAX_CACHED_FIDS 16 +char *cifs_sanitize_prepath(char *prepath, gfp_t gfp); + +extern struct mutex cifs_mount_mutex; + +static inline void cifs_mount_lock(void) +{ + mutex_lock(&cifs_mount_mutex); +} + +static inline void cifs_mount_unlock(void) +{ + mutex_unlock(&cifs_mount_mutex); +} + +static inline void cifs_negotiate_rsize(struct TCP_Server_Info *server, + struct smb3_fs_context *ctx, + struct cifs_tcon *tcon) +{ + unsigned int size; + + size = umax(server->ops->negotiate_rsize(tcon, ctx), PAGE_SIZE); + if (ctx->rsize) + size = umax(umin(ctx->rsize, size), PAGE_SIZE); + ctx->rsize = round_down(size, PAGE_SIZE); +} + +static inline void cifs_negotiate_wsize(struct TCP_Server_Info *server, + struct smb3_fs_context *ctx, + struct cifs_tcon *tcon) +{ + unsigned int size; + + size = umax(server->ops->negotiate_wsize(tcon, ctx), PAGE_SIZE); + if (ctx->wsize) + size = umax(umin(ctx->wsize, size), PAGE_SIZE); + ctx->wsize = round_down(size, PAGE_SIZE); +} + +static inline void cifs_negotiate_iosize(struct TCP_Server_Info *server, + struct smb3_fs_context *ctx, + struct cifs_tcon *tcon) +{ + cifs_negotiate_rsize(server, ctx, tcon); + cifs_negotiate_wsize(server, ctx, tcon); +} + +#endif diff --git a/fs/smb/client/fscache.c b/fs/smb/client/fscache.c new file mode 100644 index 000000000..9f66bbcd3 --- /dev/null +++ b/fs/smb/client/fscache.c @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * CIFS filesystem cache interface + * + * Copyright (c) 2010 Novell, Inc. + * Author(s): Suresh Jayaraman + * + */ +#include "fscache.h" +#include "cifsglob.h" +#include "cifs_debug.h" +#include "cifs_fs_sb.h" +#include "cifsproto.h" + +/* + * Key for fscache inode. [!] Contents must match comparisons in cifs_find_inode(). + */ +struct cifs_fscache_inode_key { + + __le64 uniqueid; /* server inode number */ + __le64 createtime; /* creation time on server */ + u8 type; /* S_IFMT file type */ +} __packed; + +static void cifs_fscache_fill_volume_coherency( + struct cifs_tcon *tcon, + struct cifs_fscache_volume_coherency_data *cd) +{ + memset(cd, 0, sizeof(*cd)); + cd->resource_id = cpu_to_le64(tcon->resource_id); + cd->vol_create_time = tcon->vol_create_time; + cd->vol_serial_number = cpu_to_le32(tcon->vol_serial_number); +} + +int cifs_fscache_get_super_cookie(struct cifs_tcon *tcon) +{ + struct cifs_fscache_volume_coherency_data cd; + struct TCP_Server_Info *server = tcon->ses->server; + struct fscache_volume *vcookie; + const struct sockaddr *sa = (struct sockaddr *)&server->dstaddr; + char *sharename; + char *key; + int ret = -ENOMEM; + + if (tcon->fscache_acquired) + return 0; + + mutex_lock(&tcon->fscache_lock); + if (tcon->fscache_acquired) { + mutex_unlock(&tcon->fscache_lock); + return 0; + } + tcon->fscache_acquired = true; + + tcon->fscache = NULL; + switch (sa->sa_family) { + case AF_INET: + case AF_INET6: + break; + default: + mutex_unlock(&tcon->fscache_lock); + cifs_dbg(VFS, "Unknown network family '%d'\n", sa->sa_family); + return -EINVAL; + } + + memset(&key, 0, sizeof(key)); + + sharename = extract_sharename(tcon->tree_name); + if (IS_ERR(sharename)) { + mutex_unlock(&tcon->fscache_lock); + cifs_dbg(FYI, "%s: couldn't extract sharename\n", __func__); + return PTR_ERR(sharename); + } + + strreplace(sharename, '/', ';'); + + key = kasprintf(GFP_KERNEL, "cifs,%pISpc,%s", sa, sharename); + if (!key) + goto out; + + cifs_fscache_fill_volume_coherency(tcon, &cd); + vcookie = fscache_acquire_volume(key, + NULL, /* preferred_cache */ + &cd, sizeof(cd)); + cifs_dbg(FYI, "%s: (%s/0x%p)\n", __func__, key, vcookie); + if (IS_ERR(vcookie)) { + if (vcookie != ERR_PTR(-EBUSY)) { + ret = PTR_ERR(vcookie); + goto out_2; + } + pr_err("Cache volume key already in use (%s)\n", key); + vcookie = NULL; + trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, + netfs_trace_tcon_ref_see_fscache_collision); + } else { + trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, + netfs_trace_tcon_ref_see_fscache_okay); + } + + tcon->fscache = vcookie; + ret = 0; +out_2: + kfree(key); +out: + kfree(sharename); + mutex_unlock(&tcon->fscache_lock); + return ret; +} + +void cifs_fscache_release_super_cookie(struct cifs_tcon *tcon) +{ + struct cifs_fscache_volume_coherency_data cd; + + cifs_dbg(FYI, "%s: (0x%p)\n", __func__, tcon->fscache); + + cifs_fscache_fill_volume_coherency(tcon, &cd); + fscache_relinquish_volume(tcon->fscache, &cd, false); + tcon->fscache = NULL; + trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, + netfs_trace_tcon_ref_see_fscache_relinq); +} + +void cifs_fscache_get_inode_cookie(struct inode *inode) +{ + struct cifs_fscache_inode_coherency_data cd; + struct cifs_fscache_inode_key key; + struct cifsInodeInfo *cifsi = CIFS_I(inode); + struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); + struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb); + + key.uniqueid = cpu_to_le64(cifsi->uniqueid); + key.createtime = cpu_to_le64(cifsi->createtime); + key.type = (inode->i_mode & S_IFMT) >> 12; + cifs_fscache_fill_coherency(&cifsi->netfs.inode, &cd); + + cifsi->netfs.cache = + fscache_acquire_cookie(tcon->fscache, 0, + &key, sizeof(key), + &cd, sizeof(cd), + i_size_read(&cifsi->netfs.inode)); + if (cifsi->netfs.cache) + mapping_set_release_always(inode->i_mapping); +} + +void cifs_fscache_unuse_inode_cookie(struct inode *inode, bool update) +{ + if (update) { + struct cifs_fscache_inode_coherency_data cd; + loff_t i_size = i_size_read(inode); + + cifs_fscache_fill_coherency(inode, &cd); + fscache_unuse_cookie(cifs_inode_cookie(inode), &cd, &i_size); + } else { + fscache_unuse_cookie(cifs_inode_cookie(inode), NULL, NULL); + } +} + +void cifs_fscache_release_inode_cookie(struct inode *inode) +{ + struct cifsInodeInfo *cifsi = CIFS_I(inode); + struct fscache_cookie *cookie = cifs_inode_cookie(inode); + + if (cookie) { + cifs_dbg(FYI, "%s: (0x%p)\n", __func__, cookie); + fscache_relinquish_cookie(cookie, false); + cifsi->netfs.cache = NULL; + } +} diff --git a/fs/smb/client/fscache.h b/fs/smb/client/fscache.h new file mode 100644 index 000000000..352122288 --- /dev/null +++ b/fs/smb/client/fscache.h @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +/* + * CIFS filesystem cache interface definitions + * + * Copyright (c) 2010 Novell, Inc. + * Authors(s): Suresh Jayaraman (sjayaraman@suse.de> + * + */ +#ifndef _CIFS_FSCACHE_H +#define _CIFS_FSCACHE_H + +#include +#include + +#include "cifsglob.h" + +/* + * Coherency data attached to CIFS volume within the cache + */ +struct cifs_fscache_volume_coherency_data { + __le64 resource_id; /* unique server resource id */ + __le64 vol_create_time; + __le32 vol_serial_number; +} __packed; + +/* + * Coherency data attached to CIFS inode within the cache. + */ +struct cifs_fscache_inode_coherency_data { + __le64 last_write_time_sec; + __le64 last_change_time_sec; + __le32 last_write_time_nsec; + __le32 last_change_time_nsec; +}; + +#ifdef CONFIG_CIFS_FSCACHE + +/* + * fscache.c + */ +int cifs_fscache_get_super_cookie(struct cifs_tcon *tcon); +void cifs_fscache_release_super_cookie(struct cifs_tcon *tcon); +void cifs_fscache_get_inode_cookie(struct inode *inode); +void cifs_fscache_unuse_inode_cookie(struct inode *inode, bool update); +void cifs_fscache_release_inode_cookie(struct inode *inode); +int cifs_fscache_get_super_cookie(struct cifs_tcon *tcon); +void cifs_fscache_release_super_cookie(struct cifs_tcon *tcon); + +void cifs_fscache_get_inode_cookie(struct inode *inode); +void cifs_fscache_release_inode_cookie(struct inode *inode); +void cifs_fscache_unuse_inode_cookie(struct inode *inode, bool update); + +static inline +void cifs_fscache_fill_coherency(struct inode *inode, + struct cifs_fscache_inode_coherency_data *cd) +{ + struct timespec64 ctime = inode_get_ctime(inode); + struct timespec64 mtime = inode_get_mtime(inode); + + memset(cd, 0, sizeof(*cd)); + cd->last_write_time_sec = cpu_to_le64(mtime.tv_sec); + cd->last_write_time_nsec = cpu_to_le32(mtime.tv_nsec); + cd->last_change_time_sec = cpu_to_le64(ctime.tv_sec); + cd->last_change_time_nsec = cpu_to_le32(ctime.tv_nsec); +} + + +static inline struct fscache_cookie *cifs_inode_cookie(struct inode *inode) +{ + return netfs_i_cookie(&CIFS_I(inode)->netfs); +} + +static inline void cifs_invalidate_cache(struct inode *inode, unsigned int flags) +{ + struct cifs_fscache_inode_coherency_data cd; + + cifs_fscache_fill_coherency(inode, &cd); + fscache_invalidate(cifs_inode_cookie(inode), &cd, + i_size_read(inode), flags); +} + +static inline bool cifs_fscache_enabled(struct inode *inode) +{ + return fscache_cookie_enabled(cifs_inode_cookie(inode)); +} + +#else /* CONFIG_CIFS_FSCACHE */ +static inline +void cifs_fscache_fill_coherency(struct inode *inode, + struct cifs_fscache_inode_coherency_data *cd) +{ +} + +static inline int cifs_fscache_get_super_cookie(struct cifs_tcon *tcon) { return 0; } +static inline void cifs_fscache_release_super_cookie(struct cifs_tcon *tcon) {} + +static inline void cifs_fscache_get_inode_cookie(struct inode *inode) {} +static inline void cifs_fscache_release_inode_cookie(struct inode *inode) {} +static inline void cifs_fscache_unuse_inode_cookie(struct inode *inode, bool update) {} +static inline struct fscache_cookie *cifs_inode_cookie(struct inode *inode) { return NULL; } +static inline void cifs_invalidate_cache(struct inode *inode, unsigned int flags) {} +static inline bool cifs_fscache_enabled(struct inode *inode) { return false; } + +#endif /* CONFIG_CIFS_FSCACHE */ + +#endif /* _CIFS_FSCACHE_H */ diff --git a/fs/smb/client/gen_smb1_mapping b/fs/smb/client/gen_smb1_mapping new file mode 100644 index 000000000..c2b2939a8 --- /dev/null +++ b/fs/smb/client/gen_smb1_mapping @@ -0,0 +1,124 @@ +#!/usr/bin/perl -w +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Script to generate SMB1 error mapping tables. +# +# Copyright (C) 2026 KylinSoft Co., Ltd. All rights reserved. +# Author(s): Huiwen He +# ChenXiaoSong +# +use strict; + +if ($#ARGV != 1) { + print STDERR "Usage: $0 \n"; + exit(2); +} + +# Parse input parameters and extract filenames +my $in_file = $ARGV[0]; +my $out_file = $ARGV[1]; +my $input_name = (split m|/|, $in_file)[-1]; +my $output_name = (split m|/|, $out_file)[-1]; +my $script_name = (split m|/|, $0)[-1]; +my @list = (); +my %seen = (); +my $current_class = ""; + +# Parse annotated entries from the input file +open(my $in, "<", $in_file) or die "Cannot open $in_file: $!"; +if ($in_file =~ /nterr\.h$/) { + while (<$in>) { + # Handle backslash line continuation + $_ .= <$in> while s/\\\s*\n//; + + # Match #define NT_STATUS_... followed by // CLASS, CODE or /* CLASS, CODE */ + my $re = qr{^\s*#define\s+(NT_STATUS_[A-Za-z0-9_]+)\s+(.+?)\s*} . + qr{(?://\s*|/\*\s*)([A-Z0-9_]+)\s*,\s*([A-Za-z0-9_]+)}; + + if (/$re/) { + my ($name, $val_str, $class, $code) = ($1, $2, $3, $4); + + # Skip duplicate macro names + next if $seen{$name}++; + + # Clean up value string (remove parens, spaces) + $val_str =~ s/[\s\(\)]//g; + my $val = 0; + foreach my $part (split(/\|/, $val_str)) { + $val |= hex($part); + } + push @list, { val => $val, name => $name, class => $class, code => $code }; + } elsif (/^\s*#define\s+NT_STATUS_.*(?:\/\/|\/\*)/) { + # Error if macro has a comment (// or /*) but fails mapping format + die "Error: Invalid mapping comment format in $in_file: $_"; + } + } +} elsif ($in_file =~ /smberr\.h$/) { + while (<$in>) { + # Handle backslash line continuation + $_ .= <$in> while s/\\\s*\n//; + + # Detect current error class from header comments (ERRDOS or ERRSRV) + if (/generated with the (\w+) error class/) { + $current_class = $1; + } + + # Match #define ERR/Err_... followed by // -POSIX_ERR or /* -POSIX_ERR */ + if (/^\s*#define\s+((?:ERR|Err)[A-Za-z0-9_]+)\s+([0-9a-fA-FxX]+)\s*(?:\/\/|\/\*)\s*(-[A-Z0-9_]+)/) { + my ($name, $val_str, $error) = ($1, $2, $3); + my $val = ($val_str =~ /^0x/i) ? hex($val_str) : $val_str; + push @list, { val => $val, name => $name, error => $error, class => $current_class }; + } elsif ($current_class && /^\s*#define\s+(?:ERR|Err).*?(?:\/\/|\/\*)/) { + # Error if macro has a comment (// or /*) but fails mapping format + die "Error: Invalid mapping comment format in $in_file: $_"; + } + } +} +close($in); + +# Fail if no entries were found to avoid broken builds +die "Error: No mapping entries found in $in_file\n" unless @list; + +# Sort entries numerically by value +@list = sort { $a->{val} <=> $b->{val} } @list; + +# Generate the C mapping table output file +open(my $out, ">", $out_file) or die "Cannot open $out_file: $!"; +print $out "/* Autogenerated from $input_name by $script_name */\n\n"; + +if ($output_name eq "smb1_mapping_table.c") { + # Generate NT status -> DOS error mapping file + + my $count = scalar @list; + my $full_names = ""; + + for (my $i = 0; $i < $count; $i++) { + my $e = $list[$i]; + my $val = $e->{val}; + + $full_names .= $e->{name}; + + # Merge synonyms + if ($i < $count - 1 && $list[$i + 1]->{val} == $val) { + $full_names .= " or "; + next; + } + + printf $out "\t{ %s, %s, 0x%08x, \"%s\" },\n", $e->{class}, $e->{code}, $val, $full_names; + + $full_names = ""; + } +} elsif ($output_name eq "smb1_err_dos_map.c" || $output_name eq "smb1_err_srv_map.c") { + # Generate SMB1 error -> POSIX error mapping file + + # Filtered by exact output filename + my $filter = ($output_name eq "smb1_err_dos_map.c") ? "ERRDOS" : "ERRSRV"; + foreach my $e (@list) { + if (!$filter || $e->{class} eq $filter) { + printf $out "\t{%s, %s},\n", $e->{name}, $e->{error}; + } + } +} else { + die "Error: Unsupported output target: $output_name\n"; +} +close($out); diff --git a/fs/smb/client/gen_smb2_mapping b/fs/smb/client/gen_smb2_mapping new file mode 100644 index 000000000..eb9fa727d --- /dev/null +++ b/fs/smb/client/gen_smb2_mapping @@ -0,0 +1,86 @@ +#!/usr/bin/perl -w +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Generate an SMB2 status -> error mapping table, +# sorted by NT status code (cpu-endian, ascending). +# +# Copyright (C) 2025 Red Hat, Inc. All Rights Reserved. +# Written by David Howells (dhowells@redhat.com) +# +use strict; + +if ($#ARGV != 1) { + print STDERR "Format: ", $0, " \n"; + exit(2); +} + +my %statuses = (); +my @list = (); + +# +# Read the file +# +open IN_FILE, "<$ARGV[0]" || die; +while () { + chomp; + + if (m!^#define\s*([A-Za-z0-9_]+)\s+cpu_to_le32[(]([0-9a-fA-Fx]+)[)]\s+//\s+([-A-Z0-9_]+)!) { + my $status = $1; + my $code = $2; + my $ncode = hex($2); + my $error = $3; + my $s; + + next if ($status =~ /^STATUS_SEVERITY/); + + die "Duplicate status $status" + if exists($statuses{$status}); + + my %s = ( + status => $status, + code => $code, + ncode => $ncode, + error => $error + ); + $statuses{$status} = \%s; + push @list, \%s; + } +} +close IN_FILE || die; + + +@list = sort( { $a->{ncode} <=> $b->{ncode} } @list); + +open OUT_FILE, ">$ARGV[1]" || die; +my $list_size = scalar @list; +my $full_status = ""; +for (my $i = 0; $i < $list_size; $i++) { + my $entry = $list[$i]; + my $status = $entry->{status}; + my $code = $entry->{code}; + my $ncode = $entry->{ncode}; + my $error = $entry->{error}; + + next if ($ncode == 0); + + $full_status .= $status; + # There may be synonyms + if ($i < $list_size - 1) { + my $next_entry = $list[$i + 1]; + my $next_ncode = $next_entry->{ncode}; + if ($next_ncode == $ncode) { + $full_status .= " or "; + next; + } + } + + my $pad = " "; + if (length($full_status) < 40) { + my $n = 40 - length($full_status); + $pad = "\t" x ((($n-1) / 8) + 1); + } + print(OUT_FILE "{ $code, $error, \"$full_status\" },\n"); + + $full_status = ""; +} +close OUT_FILE || die; diff --git a/fs/smb/client/inode.c b/fs/smb/client/inode.c new file mode 100644 index 000000000..18f562ac1 --- /dev/null +++ b/fs/smb/client/inode.c @@ -0,0 +1,3559 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * Copyright (C) International Business Machines Corp., 2002,2010 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "cifsfs.h" +#include "cifsglob.h" +#include "cifsproto.h" +#include "smb2proto.h" +#include "cifs_debug.h" +#include "cifs_fs_sb.h" +#include "cifs_unicode.h" +#include "fscache.h" +#include "fs_context.h" +#include "cifs_ioctl.h" +#include "cached_dir.h" +#include "reparse.h" + +static void cifs_invalidate_cached_dir(struct cifs_tcon *tcon, + struct dentry *parent) +{ + struct cached_fid *parent_cfid = NULL; + + if (!tcon || !parent) + return; + + if (!open_cached_dir_by_dentry(tcon, parent, &parent_cfid)) { + mutex_lock(&parent_cfid->dirents.de_mutex); + parent_cfid->dirents.is_valid = false; + parent_cfid->dirents.is_failed = true; + mutex_unlock(&parent_cfid->dirents.de_mutex); + close_cached_dir(parent_cfid); + } +} + +/* + * Set parameters for the netfs library + */ +static void cifs_set_netfs_context(struct inode *inode) +{ + struct cifsInodeInfo *cifs_i = CIFS_I(inode); + + netfs_inode_init(&cifs_i->netfs, &cifs_req_ops, true); +} + +static void cifs_set_ops(struct inode *inode) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(inode); + struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb); + struct netfs_inode *ictx = netfs_inode(inode); + unsigned int sbflags = cifs_sb_flags(cifs_sb); + + switch (inode->i_mode & S_IFMT) { + case S_IFREG: + inode->i_op = &cifs_file_inode_ops; + if (sbflags & CIFS_MOUNT_DIRECT_IO) { + set_bit(NETFS_ICTX_UNBUFFERED, &ictx->flags); + if (sbflags & CIFS_MOUNT_NO_BRL) + inode->i_fop = &cifs_file_direct_nobrl_ops; + else + inode->i_fop = &cifs_file_direct_ops; + } else if (sbflags & CIFS_MOUNT_STRICT_IO) { + if (sbflags & CIFS_MOUNT_NO_BRL) + inode->i_fop = &cifs_file_strict_nobrl_ops; + else + inode->i_fop = &cifs_file_strict_ops; + } else if (sbflags & CIFS_MOUNT_NO_BRL) + inode->i_fop = &cifs_file_nobrl_ops; + else { /* not direct, send byte range locks */ + inode->i_fop = &cifs_file_ops; + } + + /* check if server can support readahead */ + if (tcon->ses->server->max_read < PAGE_SIZE + MAX_CIFS_HDR_SIZE) + inode->i_data.a_ops = &cifs_addr_ops_smallbuf; + else + inode->i_data.a_ops = &cifs_addr_ops; + mapping_set_large_folios(inode->i_mapping); + break; + case S_IFDIR: + if (IS_AUTOMOUNT(inode)) { + inode->i_op = &cifs_namespace_inode_operations; + } else { + inode->i_op = &cifs_dir_inode_ops; + inode->i_fop = &cifs_dir_ops; + } + break; + case S_IFLNK: + inode->i_op = &cifs_symlink_inode_ops; + break; + default: + init_special_inode(inode, inode->i_mode, inode->i_rdev); + break; + } +} + +/* check inode attributes against fattr. If they don't match, tag the + * inode for cache invalidation + */ +static void +cifs_revalidate_cache(struct inode *inode, struct cifs_fattr *fattr) +{ + struct cifs_fscache_inode_coherency_data cd; + struct cifsInodeInfo *cifs_i = CIFS_I(inode); + struct timespec64 mtime; + + cifs_dbg(FYI, "%s: revalidating inode %llu\n", + __func__, cifs_i->uniqueid); + + if (inode_state_read_once(inode) & I_NEW) { + cifs_dbg(FYI, "%s: inode %llu is new\n", + __func__, cifs_i->uniqueid); + return; + } + + /* don't bother with revalidation if we have an oplock */ + if (CIFS_CACHE_READ(cifs_i)) { + cifs_dbg(FYI, "%s: inode %llu is oplocked\n", + __func__, cifs_i->uniqueid); + return; + } + + /* revalidate if mtime or size have changed */ + fattr->cf_mtime = timestamp_truncate(fattr->cf_mtime, inode); + mtime = inode_get_mtime(inode); + if (timespec64_equal(&mtime, &fattr->cf_mtime) && + netfs_read_remote_i_size(inode) == fattr->cf_eof) { + cifs_dbg(FYI, "%s: inode %llu is unchanged\n", + __func__, cifs_i->uniqueid); + return; + } + + cifs_dbg(FYI, "%s: invalidating inode %llu mapping\n", + __func__, cifs_i->uniqueid); + set_bit(CIFS_INO_INVALID_MAPPING, &cifs_i->flags); + /* Invalidate fscache cookie */ + cifs_fscache_fill_coherency(&cifs_i->netfs.inode, &cd); + fscache_invalidate(cifs_inode_cookie(inode), &cd, i_size_read(inode), 0); +} + +/* + * copy nlink to the inode, unless it wasn't provided. Provide + * sane values if we don't have an existing one and none was provided + */ +static void +cifs_nlink_fattr_to_inode(struct inode *inode, struct cifs_fattr *fattr) +{ + /* + * if we're in a situation where we can't trust what we + * got from the server (readdir, some non-unix cases) + * fake reasonable values + */ + if (fattr->cf_flags & CIFS_FATTR_UNKNOWN_NLINK) { + /* only provide fake values on a new inode */ + if (inode_state_read_once(inode) & I_NEW) { + if (fattr->cf_cifsattrs & ATTR_DIRECTORY) + set_nlink(inode, 2); + else + set_nlink(inode, 1); + } + return; + } + + /* we trust the server, so update it */ + set_nlink(inode, fattr->cf_nlink); +} + +/* populate an inode with info from a cifs_fattr struct */ +int +cifs_fattr_to_inode(struct inode *inode, struct cifs_fattr *fattr, + bool from_readdir) +{ + struct cifsInodeInfo *cifs_i = CIFS_I(inode); + struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); + + if (!(inode_state_read_once(inode) & I_NEW) && + unlikely(inode_wrong_type(inode, fattr->cf_mode))) { + CIFS_I(inode)->time = 0; /* force reval */ + return -ESTALE; + } + cifs_revalidate_cache(inode, fattr); + + spin_lock(&inode->i_lock); + if (inode_state_read_once(inode) & I_NEW) + netfs_write_zero_point(inode, fattr->cf_eof); + + fattr->cf_mtime = timestamp_truncate(fattr->cf_mtime, inode); + fattr->cf_atime = timestamp_truncate(fattr->cf_atime, inode); + fattr->cf_ctime = timestamp_truncate(fattr->cf_ctime, inode); + /* we do not want atime to be less than mtime, it broke some apps */ + if (timespec64_compare(&fattr->cf_atime, &fattr->cf_mtime) < 0) + inode_set_atime_to_ts(inode, fattr->cf_mtime); + else + inode_set_atime_to_ts(inode, fattr->cf_atime); + inode_set_mtime_to_ts(inode, fattr->cf_mtime); + inode_set_ctime_to_ts(inode, fattr->cf_ctime); + inode->i_rdev = fattr->cf_rdev; + cifs_nlink_fattr_to_inode(inode, fattr); + inode->i_uid = fattr->cf_uid; + inode->i_gid = fattr->cf_gid; + + /* if dynperm is set, don't clobber existing mode */ + if ((inode_state_read(inode) & I_NEW) || + !(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_DYNPERM)) + inode->i_mode = fattr->cf_mode; + + cifs_i->cifsAttrs = fattr->cf_cifsattrs; + cifs_i->reparse_tag = fattr->cf_cifstag; + + if (fattr->cf_flags & CIFS_FATTR_NEED_REVAL) + cifs_i->time = 0; + else + cifs_i->time = jiffies; + + if (fattr->cf_flags & CIFS_FATTR_DELETE_PENDING) + set_bit(CIFS_INO_DELETE_PENDING, &cifs_i->flags); + else + clear_bit(CIFS_INO_DELETE_PENDING, &cifs_i->flags); + + netfs_write_remote_i_size(inode, fattr->cf_eof); + /* + * Can't safely change the file size here if the client is writing to + * it due to potential races. + */ + if (is_size_safe_to_change(cifs_i, fattr->cf_eof, from_readdir)) { + i_size_write(inode, fattr->cf_eof); + inode->i_blocks = CIFS_INO_BLOCKS(fattr->cf_bytes); + } else if (from_readdir && i_size_read(inode) != fattr->cf_eof) { + cifs_i->time = 0; + } + + if (S_ISLNK(fattr->cf_mode) && fattr->cf_symlink_target) { + kfree(cifs_i->symlink_target); + cifs_i->symlink_target = fattr->cf_symlink_target; + fattr->cf_symlink_target = NULL; + } + spin_unlock(&inode->i_lock); + + if (fattr->cf_flags & CIFS_FATTR_JUNCTION) + inode->i_flags |= S_AUTOMOUNT; + if (inode_state_read_once(inode) & I_NEW) { + cifs_set_netfs_context(inode); + cifs_set_ops(inode); + } + return 0; +} + +void +cifs_fill_uniqueid(struct super_block *sb, struct cifs_fattr *fattr) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SERVER_INUM)) + fattr->cf_uniqueid = iunique(sb, ROOT_I); +} + +/* Fill a cifs_fattr struct with info from FILE_UNIX_BASIC_INFO. */ +void +cifs_unix_basic_to_fattr(struct cifs_fattr *fattr, FILE_UNIX_BASIC_INFO *info, + struct cifs_sb_info *cifs_sb) +{ + unsigned int sbflags; + + memset(fattr, 0, sizeof(*fattr)); + fattr->cf_uniqueid = le64_to_cpu(info->UniqueId); + fattr->cf_bytes = le64_to_cpu(info->NumOfBytes); + fattr->cf_eof = le64_to_cpu(info->EndOfFile); + + fattr->cf_atime = cifs_NTtimeToUnix(info->LastAccessTime); + fattr->cf_mtime = cifs_NTtimeToUnix(info->LastModificationTime); + fattr->cf_ctime = cifs_NTtimeToUnix(info->LastStatusChange); + /* old POSIX extensions don't get create time */ + + fattr->cf_mode = le64_to_cpu(info->Permissions); + + /* + * Since we set the inode type below we need to mask off + * to avoid strange results if bits set above. + */ + fattr->cf_mode &= ~S_IFMT; + switch (le32_to_cpu(info->Type)) { + case UNIX_FILE: + fattr->cf_mode |= S_IFREG; + fattr->cf_dtype = DT_REG; + break; + case UNIX_SYMLINK: + fattr->cf_mode |= S_IFLNK; + fattr->cf_dtype = DT_LNK; + break; + case UNIX_DIR: + fattr->cf_mode |= S_IFDIR; + fattr->cf_dtype = DT_DIR; + break; + case UNIX_CHARDEV: + fattr->cf_mode |= S_IFCHR; + fattr->cf_dtype = DT_CHR; + fattr->cf_rdev = MKDEV(le64_to_cpu(info->DevMajor), + le64_to_cpu(info->DevMinor) & MINORMASK); + break; + case UNIX_BLOCKDEV: + fattr->cf_mode |= S_IFBLK; + fattr->cf_dtype = DT_BLK; + fattr->cf_rdev = MKDEV(le64_to_cpu(info->DevMajor), + le64_to_cpu(info->DevMinor) & MINORMASK); + break; + case UNIX_FIFO: + fattr->cf_mode |= S_IFIFO; + fattr->cf_dtype = DT_FIFO; + break; + case UNIX_SOCKET: + fattr->cf_mode |= S_IFSOCK; + fattr->cf_dtype = DT_SOCK; + break; + default: + /* safest to call it a file if we do not know */ + fattr->cf_mode |= S_IFREG; + fattr->cf_dtype = DT_REG; + cifs_dbg(FYI, "unknown type %d\n", le32_to_cpu(info->Type)); + break; + } + + sbflags = cifs_sb_flags(cifs_sb); + fattr->cf_uid = cifs_sb->ctx->linux_uid; + if (!(sbflags & CIFS_MOUNT_OVERR_UID)) { + u64 id = le64_to_cpu(info->Uid); + if (id < ((uid_t)-1)) { + kuid_t uid = make_kuid(&init_user_ns, id); + if (uid_valid(uid)) + fattr->cf_uid = uid; + } + } + + fattr->cf_gid = cifs_sb->ctx->linux_gid; + if (!(sbflags & CIFS_MOUNT_OVERR_GID)) { + u64 id = le64_to_cpu(info->Gid); + if (id < ((gid_t)-1)) { + kgid_t gid = make_kgid(&init_user_ns, id); + if (gid_valid(gid)) + fattr->cf_gid = gid; + } + } + + fattr->cf_nlink = le64_to_cpu(info->Nlinks); +} + +/* + * Fill a cifs_fattr struct with fake inode info. + * + * Needed to setup cifs_fattr data for the directory which is the + * junction to the new submount (ie to setup the fake directory + * which represents a DFS referral or reparse mount point). + */ +static void cifs_create_junction_fattr(struct cifs_fattr *fattr, + struct super_block *sb) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + + cifs_dbg(FYI, "%s: creating fake fattr\n", __func__); + + memset(fattr, 0, sizeof(*fattr)); + fattr->cf_mode = S_IFDIR | S_IXUGO | S_IRWXU; + fattr->cf_uid = cifs_sb->ctx->linux_uid; + fattr->cf_gid = cifs_sb->ctx->linux_gid; + ktime_get_coarse_real_ts64(&fattr->cf_mtime); + fattr->cf_atime = fattr->cf_ctime = fattr->cf_mtime; + fattr->cf_nlink = 2; + fattr->cf_flags = CIFS_FATTR_JUNCTION; +} + +/* Update inode with final fattr data */ +static int update_inode_info(struct super_block *sb, + struct cifs_fattr *fattr, + struct inode **inode) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + int rc = 0; + + if (!*inode) { + *inode = cifs_iget(sb, fattr); + if (!*inode) + rc = -ENOMEM; + return rc; + } + /* We already have inode, update it. + * + * If file type or uniqueid is different, return error. + */ + if (unlikely((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SERVER_INUM) && + CIFS_I(*inode)->uniqueid != fattr->cf_uniqueid)) { + CIFS_I(*inode)->time = 0; /* force reval */ + return -ESTALE; + } + return cifs_fattr_to_inode(*inode, fattr, false); +} + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY +static int +cifs_get_file_info_unix(struct file *filp) +{ + int rc; + unsigned int xid; + FILE_UNIX_BASIC_INFO find_data; + struct cifs_fattr fattr = {}; + struct inode *inode = file_inode(filp); + struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); + struct cifsFileInfo *cfile = filp->private_data; + struct cifs_tcon *tcon = tlink_tcon(cfile->tlink); + + xid = get_xid(); + + if (cfile->symlink_target) { + fattr.cf_symlink_target = kstrdup(cfile->symlink_target, GFP_KERNEL); + if (!fattr.cf_symlink_target) { + rc = -ENOMEM; + goto cifs_gfiunix_out; + } + } + + rc = CIFSSMBUnixQFileInfo(xid, tcon, cfile->fid.netfid, &find_data); + if (!rc) { + cifs_unix_basic_to_fattr(&fattr, &find_data, cifs_sb); + } else if (rc == -EREMOTE) { + cifs_create_junction_fattr(&fattr, inode->i_sb); + } else + goto cifs_gfiunix_out; + + rc = cifs_fattr_to_inode(inode, &fattr, false); + +cifs_gfiunix_out: + free_xid(xid); + return rc; +} + +static int cifs_get_unix_fattr(const unsigned char *full_path, + struct super_block *sb, + struct cifs_fattr *fattr, + struct inode **pinode, + const unsigned int xid) +{ + struct TCP_Server_Info *server; + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + FILE_UNIX_BASIC_INFO find_data; + struct cifs_tcon *tcon; + struct tcon_link *tlink; + int rc, tmprc; + + cifs_dbg(FYI, "Getting info on %s\n", full_path); + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) + return PTR_ERR(tlink); + tcon = tlink_tcon(tlink); + server = tcon->ses->server; + + /* could have done a find first instead but this returns more info */ + rc = CIFSSMBUnixQPathInfo(xid, tcon, full_path, &find_data, + cifs_sb->local_nls, cifs_remap(cifs_sb)); + cifs_dbg(FYI, "%s: query path info: rc = %d\n", __func__, rc); + cifs_put_tlink(tlink); + + if (!rc) { + cifs_unix_basic_to_fattr(fattr, &find_data, cifs_sb); + } else if (rc == -EREMOTE) { + cifs_create_junction_fattr(fattr, sb); + rc = 0; + } else { + return rc; + } + + if (!*pinode) + cifs_fill_uniqueid(sb, fattr); + + /* check for Minshall+French symlinks */ + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MF_SYMLINKS) { + tmprc = check_mf_symlink(xid, tcon, cifs_sb, fattr, full_path); + cifs_dbg(FYI, "check_mf_symlink: %d\n", tmprc); + } + + if (S_ISLNK(fattr->cf_mode) && !fattr->cf_symlink_target) { + if (!server->ops->query_symlink) + return -EOPNOTSUPP; + rc = server->ops->query_symlink(xid, tcon, + cifs_sb, full_path, + &fattr->cf_symlink_target); + cifs_dbg(FYI, "%s: query_symlink: %d\n", __func__, rc); + } + return rc; +} + +int cifs_get_inode_info_unix(struct inode **pinode, + const unsigned char *full_path, + struct super_block *sb, unsigned int xid) +{ + struct cifs_fattr fattr = {}; + int rc; + + rc = cifs_get_unix_fattr(full_path, sb, &fattr, pinode, xid); + if (rc) + goto out; + + rc = update_inode_info(sb, &fattr, pinode); +out: + kfree(fattr.cf_symlink_target); + return rc; +} +#else +static inline int cifs_get_unix_fattr(const unsigned char *full_path, + struct super_block *sb, + struct cifs_fattr *fattr, + struct inode **pinode, + const unsigned int xid) +{ + return -EOPNOTSUPP; +} + +int cifs_get_inode_info_unix(struct inode **pinode, + const unsigned char *full_path, + struct super_block *sb, unsigned int xid) +{ + return -EOPNOTSUPP; +} +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + +static int +cifs_sfu_type(struct cifs_fattr *fattr, const char *path, + struct cifs_sb_info *cifs_sb, unsigned int xid) +{ + int rc; + __u32 oplock; + struct tcon_link *tlink; + struct cifs_tcon *tcon; + struct cifs_fid fid; + struct cifs_open_parms oparms; + struct cifs_io_parms io_parms = {0}; + char *symlink_buf_utf16; + unsigned int symlink_len_utf16; + char buf[24]; + unsigned int bytes_read; + char *pbuf; + int buf_type = CIFS_NO_BUFFER; + + pbuf = buf; + + fattr->cf_mode &= ~S_IFMT; + + if (fattr->cf_eof == 0) { + cifs_dbg(FYI, "Fifo\n"); + fattr->cf_mode |= S_IFIFO; + fattr->cf_dtype = DT_FIFO; + return 0; + } else if (fattr->cf_eof > 1 && fattr->cf_eof < 8) { + fattr->cf_mode |= S_IFREG; + fattr->cf_dtype = DT_REG; + return -EINVAL; /* EOPNOTSUPP? */ + } + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) + return PTR_ERR(tlink); + tcon = tlink_tcon(tlink); + + oparms = (struct cifs_open_parms) { + .tcon = tcon, + .cifs_sb = cifs_sb, + .desired_access = GENERIC_READ, + .create_options = cifs_create_options(cifs_sb, CREATE_NOT_DIR), + .disposition = FILE_OPEN, + .path = path, + .fid = &fid, + }; + + if (tcon->ses->server->oplocks) + oplock = REQ_OPLOCK; + else + oplock = 0; + rc = tcon->ses->server->ops->open(xid, &oparms, &oplock, NULL); + if (rc) { + cifs_dbg(FYI, "check sfu type of %s, open rc = %d\n", path, rc); + cifs_put_tlink(tlink); + return rc; + } + + /* Read header */ + io_parms.netfid = fid.netfid; + io_parms.pid = current->tgid; + io_parms.tcon = tcon; + io_parms.offset = 0; + io_parms.length = 24; + + rc = tcon->ses->server->ops->sync_read(xid, &fid, &io_parms, + &bytes_read, &pbuf, &buf_type); + if ((rc == 0) && (bytes_read >= 8)) { + if (memcmp("IntxBLK\0", pbuf, 8) == 0) { + cifs_dbg(FYI, "Block device\n"); + fattr->cf_mode |= S_IFBLK; + fattr->cf_dtype = DT_BLK; + if (bytes_read == 24) { + /* we have enough to decode dev num */ + __u64 mjr; /* major */ + __u64 mnr; /* minor */ + mjr = le64_to_cpu(*(__le64 *)(pbuf+8)); + mnr = le64_to_cpu(*(__le64 *)(pbuf+16)); + fattr->cf_rdev = MKDEV(mjr, mnr); + } else if (bytes_read == 16) { + /* + * Windows NFS server before Windows Server 2012 + * stores major and minor number in SFU-modified + * style, just as 32-bit numbers. Recognize it. + */ + __u32 mjr; /* major */ + __u32 mnr; /* minor */ + mjr = le32_to_cpu(*(__le32 *)(pbuf+8)); + mnr = le32_to_cpu(*(__le32 *)(pbuf+12)); + fattr->cf_rdev = MKDEV(mjr, mnr); + } + } else if (memcmp("IntxCHR\0", pbuf, 8) == 0) { + cifs_dbg(FYI, "Char device\n"); + fattr->cf_mode |= S_IFCHR; + fattr->cf_dtype = DT_CHR; + if (bytes_read == 24) { + /* we have enough to decode dev num */ + __u64 mjr; /* major */ + __u64 mnr; /* minor */ + mjr = le64_to_cpu(*(__le64 *)(pbuf+8)); + mnr = le64_to_cpu(*(__le64 *)(pbuf+16)); + fattr->cf_rdev = MKDEV(mjr, mnr); + } else if (bytes_read == 16) { + /* + * Windows NFS server before Windows Server 2012 + * stores major and minor number in SFU-modified + * style, just as 32-bit numbers. Recognize it. + */ + __u32 mjr; /* major */ + __u32 mnr; /* minor */ + mjr = le32_to_cpu(*(__le32 *)(pbuf+8)); + mnr = le32_to_cpu(*(__le32 *)(pbuf+12)); + fattr->cf_rdev = MKDEV(mjr, mnr); + } + } else if (memcmp("LnxSOCK", pbuf, 8) == 0) { + cifs_dbg(FYI, "Socket\n"); + fattr->cf_mode |= S_IFSOCK; + fattr->cf_dtype = DT_SOCK; + } else if (memcmp("IntxLNK\1", pbuf, 8) == 0) { + cifs_dbg(FYI, "Symlink\n"); + fattr->cf_mode |= S_IFLNK; + fattr->cf_dtype = DT_LNK; + if ((fattr->cf_eof > 8) && (fattr->cf_eof % 2 == 0)) { + symlink_buf_utf16 = kmalloc(fattr->cf_eof-8 + 1, GFP_KERNEL); + if (symlink_buf_utf16) { + io_parms.offset = 8; + io_parms.length = fattr->cf_eof-8 + 1; + buf_type = CIFS_NO_BUFFER; + rc = tcon->ses->server->ops->sync_read(xid, &fid, &io_parms, + &symlink_len_utf16, + &symlink_buf_utf16, + &buf_type); + /* + * Check that read buffer has valid length and does not + * contain UTF-16 null codepoint (via UniStrnlen() call) + * because Linux cannot process symlink with null byte. + */ + if ((rc == 0) && + (symlink_len_utf16 > 0) && + (symlink_len_utf16 < fattr->cf_eof-8 + 1) && + (symlink_len_utf16 % 2 == 0) && + (UniStrnlen((wchar_t *)symlink_buf_utf16, symlink_len_utf16/2) == symlink_len_utf16/2)) { + fattr->cf_symlink_target = + cifs_strndup_from_utf16(symlink_buf_utf16, + symlink_len_utf16, + true, + cifs_sb->local_nls); + if (!fattr->cf_symlink_target) + rc = -ENOMEM; + } + kfree(symlink_buf_utf16); + } else { + rc = -ENOMEM; + } + } + } else if (memcmp("LnxFIFO", pbuf, 8) == 0) { + cifs_dbg(FYI, "FIFO\n"); + fattr->cf_mode |= S_IFIFO; + fattr->cf_dtype = DT_FIFO; + } else { + fattr->cf_mode |= S_IFREG; /* file? */ + fattr->cf_dtype = DT_REG; + rc = -EOPNOTSUPP; + } + } else if ((rc == 0) && (bytes_read == 1) && (pbuf[0] == '\0')) { + cifs_dbg(FYI, "Socket\n"); + fattr->cf_mode |= S_IFSOCK; + fattr->cf_dtype = DT_SOCK; + } else { + fattr->cf_mode |= S_IFREG; /* then it is a file */ + fattr->cf_dtype = DT_REG; + rc = -EOPNOTSUPP; /* or some unknown SFU type */ + } + + tcon->ses->server->ops->close(xid, tcon, &fid); + cifs_put_tlink(tlink); + return rc; +} + +#define SFBITS_MASK (S_ISVTX | S_ISGID | S_ISUID) /* SETFILEBITS valid bits */ + +/* + * Fetch mode bits as provided by SFU. + * + * FIXME: Doesn't this clobber the type bit we got from cifs_sfu_type ? + */ +static int cifs_sfu_mode(struct cifs_fattr *fattr, const unsigned char *path, + struct cifs_sb_info *cifs_sb, unsigned int xid) +{ +#ifdef CONFIG_CIFS_XATTR + ssize_t rc; + char ea_value[4]; + __u32 mode; + struct tcon_link *tlink; + struct cifs_tcon *tcon; + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) + return PTR_ERR(tlink); + tcon = tlink_tcon(tlink); + + if (tcon->ses->server->ops->query_all_EAs == NULL) { + cifs_put_tlink(tlink); + return -EOPNOTSUPP; + } + + rc = tcon->ses->server->ops->query_all_EAs(xid, tcon, path, + "SETFILEBITS", ea_value, 4 /* size of buf */, + cifs_sb); + cifs_put_tlink(tlink); + if (rc < 0) + return (int)rc; + else if (rc > 3) { + mode = le32_to_cpu(*((__le32 *)ea_value)); + fattr->cf_mode &= ~SFBITS_MASK; + cifs_dbg(FYI, "special bits 0%o org mode 0%o\n", + mode, fattr->cf_mode); + fattr->cf_mode = (mode & SFBITS_MASK) | fattr->cf_mode; + cifs_dbg(FYI, "special mode bits 0%o\n", mode); + } + + return 0; +#else + return -EOPNOTSUPP; +#endif +} + +#define POSIX_TYPE_FILE 0 +#define POSIX_TYPE_DIR 1 +#define POSIX_TYPE_SYMLINK 2 +#define POSIX_TYPE_CHARDEV 3 +#define POSIX_TYPE_BLKDEV 4 +#define POSIX_TYPE_FIFO 5 +#define POSIX_TYPE_SOCKET 6 + +#define POSIX_X_OTH 0000001 +#define POSIX_W_OTH 0000002 +#define POSIX_R_OTH 0000004 +#define POSIX_X_GRP 0000010 +#define POSIX_W_GRP 0000020 +#define POSIX_R_GRP 0000040 +#define POSIX_X_USR 0000100 +#define POSIX_W_USR 0000200 +#define POSIX_R_USR 0000400 +#define POSIX_STICKY 0001000 +#define POSIX_SET_GID 0002000 +#define POSIX_SET_UID 0004000 + +#define POSIX_OTH_MASK 0000007 +#define POSIX_GRP_MASK 0000070 +#define POSIX_USR_MASK 0000700 +#define POSIX_PERM_MASK 0000777 +#define POSIX_FILETYPE_MASK 0070000 + +#define POSIX_FILETYPE_SHIFT 12 + +static u32 wire_perms_to_posix(u32 wire) +{ + u32 mode = 0; + + mode |= (wire & POSIX_X_OTH) ? S_IXOTH : 0; + mode |= (wire & POSIX_W_OTH) ? S_IWOTH : 0; + mode |= (wire & POSIX_R_OTH) ? S_IROTH : 0; + mode |= (wire & POSIX_X_GRP) ? S_IXGRP : 0; + mode |= (wire & POSIX_W_GRP) ? S_IWGRP : 0; + mode |= (wire & POSIX_R_GRP) ? S_IRGRP : 0; + mode |= (wire & POSIX_X_USR) ? S_IXUSR : 0; + mode |= (wire & POSIX_W_USR) ? S_IWUSR : 0; + mode |= (wire & POSIX_R_USR) ? S_IRUSR : 0; + mode |= (wire & POSIX_STICKY) ? S_ISVTX : 0; + mode |= (wire & POSIX_SET_GID) ? S_ISGID : 0; + mode |= (wire & POSIX_SET_UID) ? S_ISUID : 0; + + return mode; +} + +static u32 posix_filetypes[] = { + S_IFREG, + S_IFDIR, + S_IFLNK, + S_IFCHR, + S_IFBLK, + S_IFIFO, + S_IFSOCK +}; + +static u32 wire_filetype_to_posix(u32 wire_type) +{ + if (wire_type >= ARRAY_SIZE(posix_filetypes)) { + pr_warn("Unexpected type %u", wire_type); + return 0; + } + return posix_filetypes[wire_type]; +} + +umode_t wire_mode_to_posix(u32 wire, bool is_dir) +{ + u32 wire_type; + u32 mode; + + wire_type = (wire & POSIX_FILETYPE_MASK) >> POSIX_FILETYPE_SHIFT; + /* older servers do not set POSIX file type in the mode field in the response */ + if ((wire_type == 0) && is_dir) + mode = wire_perms_to_posix(wire) | S_IFDIR; + else + mode = (wire_perms_to_posix(wire) | wire_filetype_to_posix(wire_type)); + return (umode_t)mode; +} + +/* Fill a cifs_fattr struct with info from POSIX info struct */ +static void smb311_posix_info_to_fattr(struct cifs_fattr *fattr, + struct cifs_open_info_data *data, + struct super_block *sb) +{ + struct smb311_posix_qinfo *info = &data->posix_fi; + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb); + + memset(fattr, 0, sizeof(*fattr)); + + /* no fattr->flags to set */ + fattr->cf_cifsattrs = le32_to_cpu(info->DosAttributes); + fattr->cf_uniqueid = le64_to_cpu(info->Inode); + + if (info->LastAccessTime) + fattr->cf_atime = cifs_NTtimeToUnix(info->LastAccessTime); + else + ktime_get_coarse_real_ts64(&fattr->cf_atime); + + fattr->cf_ctime = cifs_NTtimeToUnix(info->ChangeTime); + fattr->cf_mtime = cifs_NTtimeToUnix(info->LastWriteTime); + + if (data->adjust_tz) { + fattr->cf_ctime.tv_sec += tcon->ses->server->timeAdj; + fattr->cf_mtime.tv_sec += tcon->ses->server->timeAdj; + } + + /* + * The srv fs device id is overridden on network mount so setting + * @fattr->cf_rdev isn't needed here. + */ + fattr->cf_eof = le64_to_cpu(info->EndOfFile); + fattr->cf_bytes = le64_to_cpu(info->AllocationSize); + fattr->cf_createtime = le64_to_cpu(info->CreationTime); + fattr->cf_nlink = le32_to_cpu(info->HardLinks); + fattr->cf_mode = wire_mode_to_posix(le32_to_cpu(info->Mode), + fattr->cf_cifsattrs & ATTR_DIRECTORY); + + if (cifs_open_data_reparse(data) && + cifs_reparse_point_to_fattr(cifs_sb, fattr, data)) + goto out_reparse; + + fattr->cf_dtype = S_DT(fattr->cf_mode); + +out_reparse: + if (S_ISLNK(fattr->cf_mode)) { + if (likely(data->symlink_target)) + fattr->cf_eof = strnlen(data->symlink_target, PATH_MAX); + fattr->cf_symlink_target = data->symlink_target; + data->symlink_target = NULL; + } + sid_to_id(cifs_sb, &data->posix_owner, fattr, SIDOWNER); + sid_to_id(cifs_sb, &data->posix_group, fattr, SIDGROUP); + + cifs_dbg(FYI, "POSIX query info: mode 0x%x uniqueid 0x%llx nlink %d\n", + fattr->cf_mode, fattr->cf_uniqueid, fattr->cf_nlink); +} + +static void cifs_open_info_to_fattr(struct cifs_fattr *fattr, + struct cifs_open_info_data *data, + struct super_block *sb) +{ + struct smb2_file_all_info *info = &data->fi; + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb); + + memset(fattr, 0, sizeof(*fattr)); + if (data->unknown_nlink) + fattr->cf_flags |= CIFS_FATTR_UNKNOWN_NLINK; + fattr->cf_cifsattrs = le32_to_cpu(info->Attributes); + if (info->DeletePending) + fattr->cf_flags |= CIFS_FATTR_DELETE_PENDING; + + if (info->LastAccessTime) + fattr->cf_atime = cifs_NTtimeToUnix(info->LastAccessTime); + else + ktime_get_coarse_real_ts64(&fattr->cf_atime); + + fattr->cf_ctime = cifs_NTtimeToUnix(info->ChangeTime); + fattr->cf_mtime = cifs_NTtimeToUnix(info->LastWriteTime); + + if (data->adjust_tz) { + fattr->cf_ctime.tv_sec += tcon->ses->server->timeAdj; + fattr->cf_mtime.tv_sec += tcon->ses->server->timeAdj; + } + + fattr->cf_eof = le64_to_cpu(info->EndOfFile); + fattr->cf_bytes = le64_to_cpu(info->AllocationSize); + fattr->cf_createtime = le64_to_cpu(info->CreationTime); + fattr->cf_nlink = le32_to_cpu(info->NumberOfLinks); + fattr->cf_uid = cifs_sb->ctx->linux_uid; + fattr->cf_gid = cifs_sb->ctx->linux_gid; + + fattr->cf_mode = cifs_sb->ctx->file_mode; + if (cifs_open_data_reparse(data) && + cifs_reparse_point_to_fattr(cifs_sb, fattr, data)) + goto out_reparse; + + if (fattr->cf_cifsattrs & ATTR_DIRECTORY) { + fattr->cf_mode = S_IFDIR | cifs_sb->ctx->dir_mode; + fattr->cf_dtype = DT_DIR; + /* + * Server can return wrong NumberOfLinks value for directories + * when Unix extensions are disabled - fake it. + */ + if (!tcon->unix_ext) + fattr->cf_flags |= CIFS_FATTR_UNKNOWN_NLINK; + } else { + fattr->cf_mode = S_IFREG | cifs_sb->ctx->file_mode; + fattr->cf_dtype = DT_REG; + + /* + * Don't accept zero nlink from non-unix servers unless + * delete is pending. Instead mark it as unknown. + */ + if ((fattr->cf_nlink < 1) && !tcon->unix_ext && + !info->DeletePending) { + cifs_dbg(VFS, "bogus file nlink value %u\n", + fattr->cf_nlink); + fattr->cf_flags |= CIFS_FATTR_UNKNOWN_NLINK; + } + } + + /* clear write bits if ATTR_READONLY is set */ + if (fattr->cf_cifsattrs & ATTR_READONLY) + fattr->cf_mode &= ~(S_IWUGO); + +out_reparse: + if (S_ISLNK(fattr->cf_mode)) { + if (likely(data->symlink_target)) + fattr->cf_eof = strnlen(data->symlink_target, PATH_MAX); + fattr->cf_symlink_target = data->symlink_target; + data->symlink_target = NULL; + } +} + +static int +cifs_get_file_info(struct file *filp) +{ + int rc; + unsigned int xid; + struct cifs_open_info_data data = {}; + struct cifs_fattr fattr; + struct inode *inode = file_inode(filp); + struct cifsFileInfo *cfile = filp->private_data; + struct cifs_tcon *tcon = tlink_tcon(cfile->tlink); + struct TCP_Server_Info *server = tcon->ses->server; + struct dentry *dentry = filp->f_path.dentry; + void *page = alloc_dentry_path(); + const unsigned char *path; + + if (!server->ops->query_file_info) { + free_dentry_path(page); + return -ENOSYS; + } + + xid = get_xid(); + rc = server->ops->query_file_info(xid, tcon, cfile, &data); + switch (rc) { + case 0: + /* TODO: add support to query reparse tag */ + data.adjust_tz = false; + if (data.symlink_target) { + data.reparse_point = true; + data.reparse.tag = IO_REPARSE_TAG_SYMLINK; + } + path = build_path_from_dentry(dentry, page); + if (IS_ERR(path)) { + rc = PTR_ERR(path); + goto cgfi_exit; + } + cifs_open_info_to_fattr(&fattr, &data, inode->i_sb); + if (fattr.cf_flags & CIFS_FATTR_DELETE_PENDING) + cifs_mark_open_handles_for_deleted_file(inode, path); + break; + case -EREMOTE: + cifs_create_junction_fattr(&fattr, inode->i_sb); + break; + case -EOPNOTSUPP: + case -EINVAL: + /* + * FIXME: legacy server -- fall back to path-based call? + * for now, just skip revalidating and mark inode for + * immediate reval. + */ + rc = 0; + CIFS_I(inode)->time = 0; + goto cgfi_exit; + default: + goto cgfi_exit; + } + + /* + * don't bother with SFU junk here -- just mark inode as needing + * revalidation. + */ + fattr.cf_uniqueid = CIFS_I(inode)->uniqueid; + fattr.cf_flags |= CIFS_FATTR_NEED_REVAL; + /* if filetype is different, return error */ + rc = cifs_fattr_to_inode(inode, &fattr, false); +cgfi_exit: + cifs_free_open_info(&data); + free_dentry_path(page); + free_xid(xid); + return rc; +} + +/* Simple function to return a 64 bit hash of string. Rarely called */ +static __u64 simple_hashstr(const char *str) +{ + const __u64 hash_mult = 1125899906842597ULL; /* a big enough prime */ + __u64 hash = 0; + + while (*str) + hash = (hash + (__u64) *str++) * hash_mult; + + return hash; +} + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY +/** + * cifs_backup_query_path_info - SMB1 fallback code to get ino + * + * Fallback code to get file metadata when we don't have access to + * full_path (EACCES) and have backup creds. + * + * @xid: transaction id used to identify original request in logs + * @tcon: information about the server share we have mounted + * @sb: the superblock stores info such as disk space available + * @full_path: name of the file we are getting the metadata for + * @resp_buf: will be set to cifs resp buf and needs to be freed with + * cifs_buf_release() when done with @data + * @data: will be set to search info result buffer + */ +static int +cifs_backup_query_path_info(int xid, + struct cifs_tcon *tcon, + struct super_block *sb, + const char *full_path, + void **resp_buf, + FILE_ALL_INFO **data) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + struct cifs_search_info info = {0}; + u16 flags; + int rc; + + *resp_buf = NULL; + info.endOfSearch = false; + if (tcon->unix_ext) + info.info_level = SMB_FIND_FILE_UNIX; + else if ((tcon->ses->capabilities & + tcon->ses->server->vals->cap_nt_find) == 0) + info.info_level = SMB_FIND_FILE_INFO_STANDARD; + else if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SERVER_INUM) + info.info_level = SMB_FIND_FILE_ID_FULL_DIR_INFO; + else /* no srvino useful for fallback to some netapp */ + info.info_level = SMB_FIND_FILE_DIRECTORY_INFO; + + flags = CIFS_SEARCH_CLOSE_ALWAYS | + CIFS_SEARCH_CLOSE_AT_END | + CIFS_SEARCH_BACKUP_SEARCH; + + rc = CIFSFindFirst(xid, tcon, full_path, + cifs_sb, NULL, flags, &info, false); + if (rc) + return rc; + + *resp_buf = (void *)info.ntwrk_buf_start; + *data = (FILE_ALL_INFO *)info.srch_entries_start; + return 0; +} +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + +static void cifs_set_fattr_ino(int xid, struct cifs_tcon *tcon, struct super_block *sb, + struct inode **inode, const char *full_path, + struct cifs_open_info_data *data, struct cifs_fattr *fattr) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + struct TCP_Server_Info *server = tcon->ses->server; + int rc; + + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SERVER_INUM)) { + if (*inode) + fattr->cf_uniqueid = CIFS_I(*inode)->uniqueid; + else + fattr->cf_uniqueid = iunique(sb, ROOT_I); + return; + } + + /* + * If we have an inode pass a NULL tcon to ensure we don't + * make a round trip to the server. This only works for SMB2+. + */ + rc = server->ops->get_srv_inum(xid, *inode ? NULL : tcon, cifs_sb, full_path, + &fattr->cf_uniqueid, data); + if (rc) { + /* + * If that fails reuse existing ino or generate one + * and disable server ones + */ + if (*inode) + fattr->cf_uniqueid = CIFS_I(*inode)->uniqueid; + else { + fattr->cf_uniqueid = iunique(sb, ROOT_I); + cifs_autodisable_serverino(cifs_sb, "Cannot retrieve inode number via get_srv_inum", rc); + } + return; + } + + /* If no errors, check for zero root inode (invalid) */ + if (fattr->cf_uniqueid == 0 && strlen(full_path) == 0) { + cifs_dbg(FYI, "Invalid (0) inodenum\n"); + if (*inode) { + /* reuse */ + fattr->cf_uniqueid = CIFS_I(*inode)->uniqueid; + } else { + /* make an ino by hashing the UNC */ + fattr->cf_flags |= CIFS_FATTR_FAKE_ROOT_INO; + fattr->cf_uniqueid = simple_hashstr(tcon->tree_name); + } + } +} + +static inline bool is_inode_cache_good(struct inode *ino) +{ + return ino && CIFS_CACHE_READ(CIFS_I(ino)) && CIFS_I(ino)->time != 0; +} + +static int reparse_info_to_fattr(struct cifs_open_info_data *data, + struct super_block *sb, + const unsigned int xid, + struct cifs_tcon *tcon, + const char *full_path, + struct cifs_fattr *fattr) +{ + struct TCP_Server_Info *server = tcon->ses->server; + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + struct kvec rsp_iov, *iov = NULL; + int rsp_buftype = CIFS_NO_BUFFER; + u32 tag = data->reparse.tag; + int rc = 0; + + if (!tag && server->ops->query_reparse_point) { + rc = server->ops->query_reparse_point(xid, tcon, cifs_sb, + full_path, &tag, + &rsp_iov, &rsp_buftype); + if (!rc) + iov = &rsp_iov; + } else if (data->reparse.io.buftype != CIFS_NO_BUFFER && + data->reparse.io.iov.iov_base) { + iov = &data->reparse.io.iov; + } + + rc = -EOPNOTSUPP; + data->reparse.tag = tag; + if (!data->reparse.tag) { + if (server->ops->query_symlink) { + rc = server->ops->query_symlink(xid, tcon, + cifs_sb, full_path, + &data->symlink_target); + } + if (rc == -EOPNOTSUPP) + data->reparse.tag = IO_REPARSE_TAG_INTERNAL; + } + + switch (data->reparse.tag) { + case 0: /* SMB1 symlink */ + break; + case IO_REPARSE_TAG_INTERNAL: + rc = 0; + if (le32_to_cpu(data->fi.Attributes) & ATTR_DIRECTORY) { + cifs_create_junction_fattr(fattr, sb); + goto out; + } + break; + default: + /* Check for cached reparse point data */ + if (data->symlink_target || data->reparse.buf) { + rc = 0; + } else if (iov && server->ops->get_reparse_point_buffer) { + struct reparse_data_buffer *reparse_buf; + u32 reparse_len; + + reparse_buf = server->ops->get_reparse_point_buffer(iov, &reparse_len); + rc = parse_reparse_point(reparse_buf, reparse_len, + cifs_sb, full_path, data); + /* + * If the reparse point was not handled but it is the + * name surrogate which points to directory, then treat + * is as a new mount point. Name surrogate reparse point + * represents another named entity in the system. + */ + if (rc == -EOPNOTSUPP && + IS_REPARSE_TAG_NAME_SURROGATE(data->reparse.tag) && + (le32_to_cpu(data->fi.Attributes) & ATTR_DIRECTORY)) { + rc = 0; + cifs_create_junction_fattr(fattr, sb); + goto out; + } + /* + * If the reparse point is unsupported by the Linux SMB + * client then let it process by the SMB server. So mask + * the -EOPNOTSUPP error code. This will allow Linux SMB + * client to send SMB OPEN request to server. If server + * does not support this reparse point too then server + * will return error during open the path. + */ + if (rc == -EOPNOTSUPP) + rc = 0; + } + + if (data->reparse.tag == IO_REPARSE_TAG_SYMLINK && !rc) { + bool directory = le32_to_cpu(data->fi.Attributes) & ATTR_DIRECTORY; + rc = smb2_fix_symlink_target_type(&data->symlink_target, directory, cifs_sb); + } + break; + } + + if (tcon->posix_extensions) + smb311_posix_info_to_fattr(fattr, data, sb); + else + cifs_open_info_to_fattr(fattr, data, sb); +out: + fattr->cf_cifstag = data->reparse.tag; + free_rsp_buf(rsp_buftype, rsp_iov.iov_base); + return rc; +} + +static int cifs_get_fattr(struct cifs_open_info_data *data, + struct super_block *sb, int xid, + const struct cifs_fid *fid, + struct cifs_fattr *fattr, + struct inode **inode, + const char *full_path) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + struct cifs_open_info_data tmp_data = {}; + void *smb1_backup_rsp_buf = NULL; + struct TCP_Server_Info *server; + struct cifs_tcon *tcon; + struct tcon_link *tlink; + unsigned int sbflags; + int tmprc = 0; + int rc = 0; + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) + return PTR_ERR(tlink); + tcon = tlink_tcon(tlink); + server = tcon->ses->server; + + /* + * 1. Fetch file metadata if not provided (data) + */ + + if (!data) { + rc = server->ops->query_path_info(xid, tcon, cifs_sb, + full_path, &tmp_data); + data = &tmp_data; + } + + /* + * 2. Convert it to internal cifs metadata (fattr) + */ + + switch (rc) { + case 0: + /* + * If the file is a reparse point, it is more complicated + * since we have to check if its reparse tag matches a known + * special file type e.g. symlink or fifo or char etc. + */ + if (cifs_open_data_reparse(data)) { + rc = reparse_info_to_fattr(data, sb, xid, tcon, + full_path, fattr); + } else { + cifs_open_info_to_fattr(fattr, data, sb); + } + if (!rc && *inode && + (fattr->cf_flags & CIFS_FATTR_DELETE_PENDING)) + cifs_mark_open_handles_for_deleted_file(*inode, full_path); + break; + case -EREMOTE: + /* DFS link, no metadata available on this server */ + cifs_create_junction_fattr(fattr, sb); + rc = 0; + break; + case -EACCES: +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + /* + * perm errors, try again with backup flags if possible + * + * For SMB2 and later the backup intent flag + * is already sent if needed on open and there + * is no path based FindFirst operation to use + * to retry with + */ + if (backup_cred(cifs_sb) && is_smb1_server(server)) { + /* for easier reading */ + FILE_ALL_INFO *fi; + FILE_DIRECTORY_INFO *fdi; + FILE_ID_FULL_DIR_INFO *si; + + rc = cifs_backup_query_path_info(xid, tcon, sb, + full_path, + &smb1_backup_rsp_buf, + &fi); + if (rc) + goto out; + + move_cifs_info_to_smb2(&data->fi, fi); + fdi = (FILE_DIRECTORY_INFO *)fi; + si = (FILE_ID_FULL_DIR_INFO *)fi; + + cifs_dir_info_to_fattr(fattr, fdi, cifs_sb); + fattr->cf_uniqueid = le64_to_cpu(si->UniqueId); + /* uniqueid set, skip get inum step */ + goto handle_mnt_opt; + } else { + /* nothing we can do, bail out */ + goto out; + } +#else + goto out; +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + break; + default: + cifs_dbg(FYI, "%s: unhandled err rc %d\n", __func__, rc); + goto out; + } + + /* + * 3. Get or update inode number (fattr->cf_uniqueid) + */ + + cifs_set_fattr_ino(xid, tcon, sb, inode, full_path, data, fattr); + + /* + * 4. Tweak fattr based on mount options + */ +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY +handle_mnt_opt: +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + sbflags = cifs_sb_flags(cifs_sb); + /* query for SFU type info if supported and needed */ + if ((fattr->cf_cifsattrs & ATTR_SYSTEM) && + (sbflags & CIFS_MOUNT_UNX_EMUL)) { + tmprc = cifs_sfu_type(fattr, full_path, cifs_sb, xid); + if (tmprc) + cifs_dbg(FYI, "cifs_sfu_type failed: %d\n", tmprc); + } + + /* fill in 0777 bits from ACL */ + if (sbflags & CIFS_MOUNT_MODE_FROM_SID) { + rc = cifs_acl_to_fattr(cifs_sb, fattr, *inode, + true, full_path, fid); + if (rc == -EREMOTE) + rc = 0; + if (rc) { + cifs_dbg(FYI, "%s: Get mode from SID failed. rc=%d\n", + __func__, rc); + goto out; + } + } else if (sbflags & CIFS_MOUNT_CIFS_ACL) { + rc = cifs_acl_to_fattr(cifs_sb, fattr, *inode, + false, full_path, fid); + if (rc == -EREMOTE) + rc = 0; + if (rc) { + cifs_dbg(FYI, "%s: Getting ACL failed with error: %d\n", + __func__, rc); + goto out; + } + } else if (sbflags & CIFS_MOUNT_UNX_EMUL) + /* fill in remaining high mode bits e.g. SUID, VTX */ + cifs_sfu_mode(fattr, full_path, cifs_sb, xid); + else if (!(tcon->posix_extensions)) + /* clear write bits if ATTR_READONLY is set */ + if (fattr->cf_cifsattrs & ATTR_READONLY) + fattr->cf_mode &= ~(S_IWUGO); + + + /* check for Minshall+French symlinks */ + if (sbflags & CIFS_MOUNT_MF_SYMLINKS) { + tmprc = check_mf_symlink(xid, tcon, cifs_sb, fattr, full_path); + cifs_dbg(FYI, "check_mf_symlink: %d\n", tmprc); + } + +out: + cifs_buf_release(smb1_backup_rsp_buf); + cifs_put_tlink(tlink); + cifs_free_open_info(&tmp_data); + return rc; +} + +int cifs_get_inode_info(struct inode **inode, + const char *full_path, + struct cifs_open_info_data *data, + struct super_block *sb, int xid, + const struct cifs_fid *fid) +{ + struct cifs_fattr fattr = {}; + int rc; + + if (!data && is_inode_cache_good(*inode)) { + cifs_dbg(FYI, "No need to revalidate cached inode sizes\n"); + return 0; + } + + rc = cifs_get_fattr(data, sb, xid, fid, &fattr, inode, full_path); + if (rc) + goto out; + + rc = update_inode_info(sb, &fattr, inode); +out: + kfree(fattr.cf_symlink_target); + return rc; +} + +static int smb311_posix_get_fattr(struct cifs_open_info_data *data, + struct cifs_fattr *fattr, + const char *full_path, + struct super_block *sb, + const unsigned int xid) +{ + struct cifs_open_info_data tmp_data = {}; + struct TCP_Server_Info *server; + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + struct cifs_tcon *tcon; + struct tcon_link *tlink; + int tmprc; + int rc = 0; + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) + return PTR_ERR(tlink); + tcon = tlink_tcon(tlink); + server = tcon->ses->server; + + /* + * 1. Fetch file metadata if not provided (data) + */ + if (!data) { + rc = server->ops->query_path_info(xid, tcon, cifs_sb, + full_path, &tmp_data); + data = &tmp_data; + } + + /* + * 2. Convert it to internal cifs metadata (fattr) + */ + + switch (rc) { + case 0: + if (cifs_open_data_reparse(data)) { + rc = reparse_info_to_fattr(data, sb, xid, tcon, + full_path, fattr); + } else { + smb311_posix_info_to_fattr(fattr, data, sb); + } + break; + case -EREMOTE: + /* DFS link, no metadata available on this server */ + cifs_create_junction_fattr(fattr, sb); + rc = 0; + break; + case -EACCES: + /* + * For SMB2 and later the backup intent flag + * is already sent if needed on open and there + * is no path based FindFirst operation to use + * to retry with so nothing we can do, bail out + */ + goto out; + default: + cifs_dbg(FYI, "%s: unhandled err rc %d\n", __func__, rc); + goto out; + } + + /* + * 3. Tweak fattr based on mount options + */ + /* check for Minshall+French symlinks */ + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MF_SYMLINKS) { + tmprc = check_mf_symlink(xid, tcon, cifs_sb, fattr, full_path); + cifs_dbg(FYI, "check_mf_symlink: %d\n", tmprc); + } + +out: + cifs_put_tlink(tlink); + cifs_free_open_info(data); + return rc; +} + +int smb311_posix_get_inode_info(struct inode **inode, + const char *full_path, + struct cifs_open_info_data *data, + struct super_block *sb, + const unsigned int xid) +{ + struct cifs_fattr fattr = {}; + int rc; + + if (!data && is_inode_cache_good(*inode)) { + cifs_dbg(FYI, "No need to revalidate cached inode sizes\n"); + return 0; + } + + rc = smb311_posix_get_fattr(data, &fattr, full_path, sb, xid); + if (rc) + goto out; + + rc = update_inode_info(sb, &fattr, inode); + if (!rc && fattr.cf_flags & CIFS_FATTR_DELETE_PENDING) + cifs_mark_open_handles_for_deleted_file(*inode, full_path); +out: + kfree(fattr.cf_symlink_target); + return rc; +} + +static const struct inode_operations cifs_ipc_inode_ops = { + .lookup = cifs_lookup, +}; + +static int +cifs_find_inode(struct inode *inode, void *opaque) +{ + struct cifs_fattr *fattr = opaque; + + /* [!] The compared values must be the same in struct cifs_fscache_inode_key. */ + + /* don't match inode with different uniqueid */ + if (CIFS_I(inode)->uniqueid != fattr->cf_uniqueid) + return 0; + + /* use createtime like an i_generation field */ + if (CIFS_I(inode)->createtime != fattr->cf_createtime) + return 0; + + /* don't match inode of different type */ + if (inode_wrong_type(inode, fattr->cf_mode)) + return 0; + + /* if it's not a directory or has no dentries, then flag it */ + if (S_ISDIR(inode->i_mode) && !hlist_empty(&inode->i_dentry)) + fattr->cf_flags |= CIFS_FATTR_INO_COLLISION; + + return 1; +} + +static int +cifs_init_inode(struct inode *inode, void *opaque) +{ + struct cifs_fattr *fattr = opaque; + + CIFS_I(inode)->uniqueid = fattr->cf_uniqueid; + CIFS_I(inode)->createtime = fattr->cf_createtime; + return 0; +} + +/* + * walk dentry list for an inode and report whether it has aliases that + * are hashed. We use this to determine if a directory inode can actually + * be used. + */ +static bool +inode_has_hashed_dentries(struct inode *inode) +{ + struct dentry *dentry; + + spin_lock(&inode->i_lock); + for_each_alias(dentry, inode) { + if (!d_unhashed(dentry) || IS_ROOT(dentry)) { + spin_unlock(&inode->i_lock); + return true; + } + } + spin_unlock(&inode->i_lock); + return false; +} + +/* Given fattrs, get a corresponding inode */ +struct inode * +cifs_iget(struct super_block *sb, struct cifs_fattr *fattr) +{ + unsigned long hash; + struct inode *inode; + +retry_iget5_locked: + cifs_dbg(FYI, "looking for uniqueid=%llu\n", fattr->cf_uniqueid); + + /* hash down to 32-bits on 32-bit arch */ + hash = cifs_uniqueid_to_ino_t(fattr->cf_uniqueid); + + inode = iget5_locked(sb, hash, cifs_find_inode, cifs_init_inode, fattr); + if (inode) { + /* was there a potentially problematic inode collision? */ + if (fattr->cf_flags & CIFS_FATTR_INO_COLLISION) { + fattr->cf_flags &= ~CIFS_FATTR_INO_COLLISION; + + if (inode_has_hashed_dentries(inode)) { + cifs_autodisable_serverino(CIFS_SB(sb), "Inode number collision detected", 0); + iput(inode); + fattr->cf_uniqueid = iunique(sb, ROOT_I); + goto retry_iget5_locked; + } + } + + /* can't fail - see cifs_find_inode() */ + cifs_fattr_to_inode(inode, fattr, false); + if (sb->s_flags & SB_NOATIME) + inode->i_flags |= S_NOATIME | S_NOCMTIME; + if (inode_state_read_once(inode) & I_NEW) { + inode->i_ino = hash; + cifs_fscache_get_inode_cookie(inode); + unlock_new_inode(inode); + } + } + + return inode; +} + +/* gets root inode */ +struct inode *cifs_root_iget(struct super_block *sb) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + struct cifs_fattr fattr = {}; + struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb); + struct inode *inode = NULL; + unsigned int xid; + char *path = NULL; + int len; + int rc; + + if ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_USE_PREFIX_PATH) + && cifs_sb->prepath) { + len = strlen(cifs_sb->prepath); + path = kzalloc(len + 2 /* leading sep + null */, GFP_KERNEL); + if (path == NULL) + return ERR_PTR(-ENOMEM); + path[0] = '/'; + memcpy(path+1, cifs_sb->prepath, len); + } else { + path = kstrdup("", GFP_KERNEL); + if (path == NULL) + return ERR_PTR(-ENOMEM); + } + + xid = get_xid(); + if (tcon->unix_ext) { + rc = cifs_get_unix_fattr(path, sb, &fattr, &inode, xid); + /* some servers mistakenly claim POSIX support */ + if (rc != -EOPNOTSUPP) + goto iget_root; + cifs_dbg(VFS, "server does not support POSIX extensions\n"); + tcon->unix_ext = false; + } + + convert_delimiter(path, CIFS_DIR_SEP(cifs_sb)); + if (tcon->posix_extensions) + rc = smb311_posix_get_fattr(NULL, &fattr, path, sb, xid); + else + rc = cifs_get_fattr(NULL, sb, xid, NULL, &fattr, &inode, path); + +iget_root: + if (!rc) { + if (fattr.cf_flags & CIFS_FATTR_JUNCTION) { + cifs_dbg(VFS, "Removing junction mark and disabling 'serverino' to prevent inode collisions\n"); + fattr.cf_flags &= ~CIFS_FATTR_JUNCTION; + cifs_autodisable_serverino(cifs_sb, "Cannot retrieve attributes for junction point", rc); + } + inode = cifs_iget(sb, &fattr); + } + + if (!inode) { + inode = ERR_PTR(rc); + goto out; + } + + if (!rc && fattr.cf_flags & CIFS_FATTR_DELETE_PENDING) + cifs_mark_open_handles_for_deleted_file(inode, path); + + if (rc && tcon->pipe) { + cifs_dbg(FYI, "ipc connection - fake read inode\n"); + spin_lock(&inode->i_lock); + inode->i_mode |= S_IFDIR; + set_nlink(inode, 2); + inode->i_op = &cifs_ipc_inode_ops; + inode->i_fop = &simple_dir_operations; + inode->i_uid = cifs_sb->ctx->linux_uid; + inode->i_gid = cifs_sb->ctx->linux_gid; + spin_unlock(&inode->i_lock); + } else if (rc) { + iget_failed(inode); + inode = ERR_PTR(rc); + } + +out: + kfree(path); + free_xid(xid); + kfree(fattr.cf_symlink_target); + return inode; +} + +int +cifs_set_file_info(struct inode *inode, struct iattr *attrs, unsigned int xid, + const char *full_path, __u32 dosattr) +{ + bool set_time = false; + struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); + struct TCP_Server_Info *server; + FILE_BASIC_INFO info_buf; + + if (attrs == NULL) + return -EINVAL; + + server = cifs_sb_master_tcon(cifs_sb)->ses->server; + if (!server->ops->set_file_info) + return -ENOSYS; + + info_buf.Pad = 0; + + if (attrs->ia_valid & ATTR_ATIME) { + set_time = true; + info_buf.LastAccessTime = + cpu_to_le64(cifs_UnixTimeToNT(attrs->ia_atime)); + } else + info_buf.LastAccessTime = 0; + + if (attrs->ia_valid & ATTR_MTIME) { + set_time = true; + info_buf.LastWriteTime = + cpu_to_le64(cifs_UnixTimeToNT(attrs->ia_mtime)); + } else + info_buf.LastWriteTime = 0; + + /* + * Samba throws this field away, but windows may actually use it. + * Do not set ctime unless other time stamps are changed explicitly + * (i.e. by utimes()) since we would then have a mix of client and + * server times. + */ + if (set_time && (attrs->ia_valid & ATTR_CTIME)) { + cifs_dbg(FYI, "CIFS - CTIME changed\n"); + info_buf.ChangeTime = + cpu_to_le64(cifs_UnixTimeToNT(attrs->ia_ctime)); + } else + info_buf.ChangeTime = 0; + + info_buf.CreationTime = 0; /* don't change */ + info_buf.Attributes = cpu_to_le32(dosattr); + + return server->ops->set_file_info(inode, full_path, &info_buf, xid); +} + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY +/* + * Open the given file (if it isn't already), set the DELETE_ON_CLOSE bit + * and rename it to a random name that hopefully won't conflict with + * anything else. + */ +int +cifs_rename_pending_delete(const char *full_path, struct dentry *dentry, + const unsigned int xid) +{ + int oplock = 0; + int rc; + struct cifs_fid fid; + struct cifs_open_parms oparms; + struct inode *inode = d_inode(dentry); + struct cifsInodeInfo *cifsInode = CIFS_I(inode); + struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); + struct tcon_link *tlink; + struct cifs_tcon *tcon; + __u32 dosattr, origattr; + FILE_BASIC_INFO *info_buf = NULL; + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) + return PTR_ERR(tlink); + tcon = tlink_tcon(tlink); + + /* + * We cannot rename the file if the server doesn't support + * CAP_INFOLEVEL_PASSTHRU + */ + if (!(tcon->ses->capabilities & CAP_INFOLEVEL_PASSTHRU)) { + rc = -EBUSY; + goto out; + } + + oparms = (struct cifs_open_parms) { + .tcon = tcon, + .cifs_sb = cifs_sb, + .desired_access = DELETE | FILE_WRITE_ATTRIBUTES, + .create_options = cifs_create_options(cifs_sb, CREATE_NOT_DIR), + .disposition = FILE_OPEN, + .path = full_path, + .fid = &fid, + }; + + rc = CIFS_open(xid, &oparms, &oplock, NULL); + if (rc != 0) + goto out; + + origattr = cifsInode->cifsAttrs; + if (origattr == 0) + origattr |= ATTR_NORMAL; + + dosattr = origattr & ~ATTR_READONLY; + if (dosattr == 0) + dosattr |= ATTR_NORMAL; + dosattr |= ATTR_HIDDEN; + + /* set ATTR_HIDDEN and clear ATTR_READONLY, but only if needed */ + if (dosattr != origattr) { + info_buf = kzalloc_obj(*info_buf); + if (info_buf == NULL) { + rc = -ENOMEM; + goto out_close; + } + info_buf->Attributes = cpu_to_le32(dosattr); + rc = CIFSSMBSetFileInfo(xid, tcon, info_buf, fid.netfid, + current->tgid); + /* although we would like to mark the file hidden + if that fails we will still try to rename it */ + if (!rc) + cifsInode->cifsAttrs = dosattr; + else + dosattr = origattr; /* since not able to change them */ + } + + /* rename the file */ + rc = CIFSSMBRenameOpenFile(xid, tcon, fid.netfid, NULL, + cifs_sb->local_nls, + cifs_remap(cifs_sb)); + if (rc != 0) { + rc = -EBUSY; + goto undo_setattr; + } + + /* try to set DELETE_ON_CLOSE */ + if (!test_bit(CIFS_INO_DELETE_PENDING, &cifsInode->flags)) { + rc = CIFSSMBSetFileDisposition(xid, tcon, true, fid.netfid, + current->tgid); + /* + * some samba versions return -ENOENT when we try to set the + * file disposition here. Likely a samba bug, but work around + * it for now. This means that some cifsXXX files may hang + * around after they shouldn't. + * + * BB: remove this hack after more servers have the fix + */ + if (rc == -ENOENT) + rc = 0; + else if (rc != 0) { + rc = -EBUSY; + goto undo_rename; + } + set_bit(CIFS_INO_DELETE_PENDING, &cifsInode->flags); + } + +out_close: + CIFSSMBClose(xid, tcon, fid.netfid); +out: + kfree(info_buf); + cifs_put_tlink(tlink); + return rc; + + /* + * reset everything back to the original state. Don't bother + * dealing with errors here since we can't do anything about + * them anyway. + */ +undo_rename: + CIFSSMBRenameOpenFile(xid, tcon, fid.netfid, dentry->d_name.name, + cifs_sb->local_nls, cifs_remap(cifs_sb)); +undo_setattr: + if (dosattr != origattr) { + info_buf->Attributes = cpu_to_le32(origattr); + if (!CIFSSMBSetFileInfo(xid, tcon, info_buf, fid.netfid, + current->tgid)) + cifsInode->cifsAttrs = origattr; + } + + goto out_close; +} +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + +/* copied from fs/nfs/dir.c with small changes */ +static void +cifs_drop_nlink(struct inode *inode) +{ + spin_lock(&inode->i_lock); + if (inode->i_nlink > 0) + drop_nlink(inode); + spin_unlock(&inode->i_lock); +} + +/* + * If d_inode(dentry) is null (usually meaning the cached dentry + * is a negative dentry) then we would attempt a standard SMB delete, but + * if that fails we can not attempt the fall back mechanisms on EACCES + * but will return the EACCES to the caller. Note that the VFS does not call + * unlink on negative dentries currently. + */ +static int __cifs_unlink(struct inode *dir, struct dentry *dentry, bool sillyrename) +{ + int rc = 0; + unsigned int xid; + const char *full_path; + void *page; + struct inode *inode = d_inode(dentry); + struct cifsInodeInfo *cifs_inode; + struct super_block *sb = dir->i_sb; + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + struct tcon_link *tlink; + struct cifs_tcon *tcon; + __u32 dosattr = 0, origattr = 0; + struct TCP_Server_Info *server; + struct iattr *attrs = NULL; + bool rehash = false; + + cifs_dbg(FYI, "cifs_unlink, dir=0x%p, dentry=0x%p\n", dir, dentry); + + if (unlikely(cifs_forced_shutdown(cifs_sb))) + return smb_EIO(smb_eio_trace_forced_shutdown); + + /* Unhash dentry in advance to prevent any concurrent opens */ + spin_lock(&dentry->d_lock); + if (!d_unhashed(dentry)) { + __d_drop(dentry); + rehash = true; + } + spin_unlock(&dentry->d_lock); + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) + return PTR_ERR(tlink); + tcon = tlink_tcon(tlink); + server = tcon->ses->server; + + xid = get_xid(); + page = alloc_dentry_path(); + + if (tcon->nodelete) { + rc = -EACCES; + goto unlink_out; + } + + /* Unlink can be called from rename so we can not take the + * sb->s_vfs_rename_mutex here */ + full_path = build_path_from_dentry(dentry, page); + if (IS_ERR(full_path)) { + rc = PTR_ERR(full_path); + goto unlink_out; + } + + netfs_wait_for_outstanding_io(inode); + cifs_close_deferred_file_under_dentry(tcon, dentry); +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + if (cap_unix(tcon->ses) && (CIFS_UNIX_POSIX_PATH_OPS_CAP & + le64_to_cpu(tcon->fsUnixInfo.Capability))) { + rc = CIFSPOSIXDelFile(xid, tcon, full_path, + SMB_POSIX_UNLINK_FILE_TARGET, cifs_sb->local_nls, + cifs_remap(cifs_sb)); + cifs_dbg(FYI, "posix del rc %d\n", rc); + if ((rc == 0) || (rc == -ENOENT)) + goto psx_del_no_retry; + } +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + +retry_std_delete: + if (!server->ops->unlink) { + rc = -ENOSYS; + goto psx_del_no_retry; + } + + /* For SMB2+, if the file is open, we always perform a silly rename. + * + * We check for d_count() right after calling + * cifs_close_deferred_file_under_dentry() to make sure that the + * dentry's refcount gets dropped in case the file had any deferred + * close. + */ + if (!sillyrename && server->vals->protocol_id > SMB10_PROT_ID) { + spin_lock(&dentry->d_lock); + if (d_count(dentry) > 1) + sillyrename = true; + spin_unlock(&dentry->d_lock); + } + + if (sillyrename) + rc = -EBUSY; + else + rc = server->ops->unlink(xid, tcon, full_path, cifs_sb, dentry); + +psx_del_no_retry: + if (!rc) { + if (inode) { + cifs_mark_open_handles_for_deleted_file(inode, full_path); + cifs_drop_nlink(inode); + } + } else if (rc == -ENOENT) { + if (simple_positive(dentry)) + d_delete(dentry); + } else if (rc == -EBUSY) { + if (server->ops->rename_pending_delete) { + rc = server->ops->rename_pending_delete(full_path, + dentry, xid); + if (rc == 0) { + cifs_mark_open_handles_for_deleted_file(inode, full_path); + cifs_drop_nlink(inode); + } + } + } else if ((rc == -EACCES) && (dosattr == 0) && inode) { + attrs = kzalloc_obj(*attrs); + if (attrs == NULL) { + rc = -ENOMEM; + goto out_reval; + } + + /* try to reset dos attributes */ + cifs_inode = CIFS_I(inode); + origattr = cifs_inode->cifsAttrs; + if (origattr == 0) + origattr |= ATTR_NORMAL; + dosattr = origattr & ~ATTR_READONLY; + if (dosattr == 0) + dosattr |= ATTR_NORMAL; + dosattr |= ATTR_HIDDEN; + + rc = cifs_set_file_info(inode, attrs, xid, full_path, dosattr); + if (rc != 0) + goto out_reval; + + goto retry_std_delete; + } + + /* undo the setattr if we errored out and it's needed */ + if (rc != 0 && dosattr != 0) + cifs_set_file_info(inode, attrs, xid, full_path, origattr); + +out_reval: + if (!rc && dentry->d_parent) + cifs_invalidate_cached_dir(tcon, dentry->d_parent); + + if (inode) { + cifs_inode = CIFS_I(inode); + cifs_inode->time = 0; /* will force revalidate to get info + when needed */ + inode_set_ctime_current(inode); + } + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + cifs_inode = CIFS_I(dir); + CIFS_I(dir)->time = 0; /* force revalidate of dir as well */ +unlink_out: + free_dentry_path(page); + kfree(attrs); + free_xid(xid); + cifs_put_tlink(tlink); + if (rehash) + d_rehash(dentry); + return rc; +} + +int cifs_unlink(struct inode *dir, struct dentry *dentry) +{ + return __cifs_unlink(dir, dentry, false); +} + +static int +cifs_mkdir_qinfo(struct inode *parent, struct dentry *dentry, umode_t mode, + const char *full_path, struct cifs_sb_info *cifs_sb, + struct cifs_tcon *tcon, const unsigned int xid) +{ + struct inode *inode = NULL; + unsigned int sbflags; + int rc = 0; + + if (tcon->posix_extensions) { + rc = smb311_posix_get_inode_info(&inode, full_path, + NULL, parent->i_sb, xid); +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + } else if (tcon->unix_ext) { + rc = cifs_get_inode_info_unix(&inode, full_path, parent->i_sb, + xid); +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + } else { + rc = cifs_get_inode_info(&inode, full_path, NULL, parent->i_sb, + xid, NULL); + } + + if (rc) + return rc; + + if (!S_ISDIR(inode->i_mode)) { + /* + * mkdir succeeded, but another client has managed to remove the + * sucker and replace it with non-directory. Return success, + * but don't leave the child in dcache. + */ + iput(inode); + d_drop(dentry); + return 0; + } + /* + * setting nlink not necessary except in cases where we failed to get it + * from the server or was set bogus. Also, since this is a brand new + * inode, no need to grab the i_lock before setting the i_nlink. + */ + if (inode->i_nlink < 2) + set_nlink(inode, 2); + mode &= ~current_umask(); + /* must turn on setgid bit if parent dir has it */ + if (parent->i_mode & S_ISGID) + mode |= S_ISGID; + + sbflags = cifs_sb_flags(cifs_sb); +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + if (tcon->unix_ext) { + struct cifs_unix_set_info_args args = { + .mode = mode, + .ctime = NO_CHANGE_64, + .atime = NO_CHANGE_64, + .mtime = NO_CHANGE_64, + .device = 0, + }; + if (sbflags & CIFS_MOUNT_SET_UID) { + args.uid = current_fsuid(); + if (parent->i_mode & S_ISGID) + args.gid = parent->i_gid; + else + args.gid = current_fsgid(); + } else { + args.uid = INVALID_UID; /* no change */ + args.gid = INVALID_GID; /* no change */ + } + CIFSSMBUnixSetPathInfo(xid, tcon, full_path, &args, + cifs_sb->local_nls, + cifs_remap(cifs_sb)); + } else { +#else + { +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + struct TCP_Server_Info *server = tcon->ses->server; + if (!(sbflags & CIFS_MOUNT_CIFS_ACL) && + (mode & S_IWUGO) == 0 && server->ops->mkdir_setinfo) + server->ops->mkdir_setinfo(inode, full_path, cifs_sb, + tcon, xid); + if (sbflags & CIFS_MOUNT_DYNPERM) + inode->i_mode = (mode | S_IFDIR); + + if (sbflags & CIFS_MOUNT_SET_UID) { + inode->i_uid = current_fsuid(); + if (inode->i_mode & S_ISGID) + inode->i_gid = parent->i_gid; + else + inode->i_gid = current_fsgid(); + } + } + d_instantiate(dentry, inode); + return 0; +} + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY +static int +cifs_posix_mkdir(struct inode *inode, struct dentry *dentry, umode_t mode, + const char *full_path, struct cifs_sb_info *cifs_sb, + struct cifs_tcon *tcon, const unsigned int xid) +{ + int rc = 0; + u32 oplock = 0; + FILE_UNIX_BASIC_INFO *info = NULL; + struct inode *newinode = NULL; + struct cifs_fattr fattr; + + info = kzalloc_obj(FILE_UNIX_BASIC_INFO); + if (info == NULL) { + rc = -ENOMEM; + goto posix_mkdir_out; + } + + mode &= ~current_umask(); + rc = CIFSPOSIXCreate(xid, tcon, SMB_O_DIRECTORY | SMB_O_CREAT, mode, + NULL /* netfid */, info, &oplock, full_path, + cifs_sb->local_nls, cifs_remap(cifs_sb)); + if (rc == -EOPNOTSUPP) + goto posix_mkdir_out; + else if (rc) { + cifs_dbg(FYI, "posix mkdir returned 0x%x\n", rc); + d_drop(dentry); + goto posix_mkdir_out; + } + + if (info->Type == cpu_to_le32(-1)) + /* no return info, go query for it */ + goto posix_mkdir_get_info; + /* + * BB check (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_SET_UID ) to see if + * need to set uid/gid. + */ + + cifs_unix_basic_to_fattr(&fattr, info, cifs_sb); + cifs_fill_uniqueid(inode->i_sb, &fattr); + newinode = cifs_iget(inode->i_sb, &fattr); + if (!newinode) + goto posix_mkdir_get_info; + + d_instantiate(dentry, newinode); + +#ifdef CONFIG_CIFS_DEBUG2 + cifs_dbg(FYI, "instantiated dentry %p %pd to inode %p\n", + dentry, dentry, newinode); + + if (newinode->i_nlink != 2) + cifs_dbg(FYI, "unexpected number of links %d\n", + newinode->i_nlink); +#endif + +posix_mkdir_out: + kfree(info); + return rc; +posix_mkdir_get_info: + rc = cifs_mkdir_qinfo(inode, dentry, mode, full_path, cifs_sb, tcon, + xid); + goto posix_mkdir_out; +} +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + +struct dentry *cifs_mkdir(struct mnt_idmap *idmap, struct inode *inode, + struct dentry *direntry, umode_t mode) +{ + int rc = 0; + unsigned int xid; + struct cifs_sb_info *cifs_sb; + struct tcon_link *tlink; + struct cifs_tcon *tcon; + struct TCP_Server_Info *server; + const char *full_path; + void *page; + + /* + * vfs_mkdir() now passes S_IFDIR in @mode, but @mode is forwarded + * verbatim to the server and in the past only contained permission + * bits. Strip the type bit until SMB is verified to deal with it. + */ + mode &= ~S_IFDIR; + + cifs_dbg(FYI, "In cifs_mkdir, mode = %04ho inode = 0x%p\n", + mode, inode); + + cifs_sb = CIFS_SB(inode->i_sb); + if (unlikely(cifs_forced_shutdown(cifs_sb))) + return ERR_PTR(smb_EIO(smb_eio_trace_forced_shutdown)); + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) + return ERR_CAST(tlink); + tcon = tlink_tcon(tlink); + + xid = get_xid(); + + page = alloc_dentry_path(); + full_path = build_path_from_dentry(direntry, page); + if (IS_ERR(full_path)) { + rc = PTR_ERR(full_path); + goto mkdir_out; + } + + server = tcon->ses->server; + + if ((server->ops->posix_mkdir) && (tcon->posix_extensions)) { + rc = server->ops->posix_mkdir(xid, inode, mode, tcon, full_path, + cifs_sb); + d_drop(direntry); /* for time being always refresh inode info */ + goto mkdir_out; + } + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + if (cap_unix(tcon->ses) && (CIFS_UNIX_POSIX_PATH_OPS_CAP & + le64_to_cpu(tcon->fsUnixInfo.Capability))) { + rc = cifs_posix_mkdir(inode, direntry, mode, full_path, cifs_sb, + tcon, xid); + if (rc != -EOPNOTSUPP) + goto mkdir_out; + } +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + + if (!server->ops->mkdir) { + rc = -ENOSYS; + goto mkdir_out; + } + + /* BB add setting the equivalent of mode via CreateX w/ACLs */ + rc = server->ops->mkdir(xid, inode, mode, tcon, full_path, cifs_sb); + if (rc) { + cifs_dbg(FYI, "cifs_mkdir returned 0x%x\n", rc); + d_drop(direntry); + goto mkdir_out; + } + + /* TODO: skip this for smb2/smb3 */ + rc = cifs_mkdir_qinfo(inode, direntry, mode, full_path, cifs_sb, tcon, + xid); +mkdir_out: + /* + * Force revalidate to get parent dir info when needed since cached + * attributes are invalid now. + */ + CIFS_I(inode)->time = 0; + free_dentry_path(page); + free_xid(xid); + cifs_put_tlink(tlink); + return ERR_PTR(rc); +} + +int cifs_rmdir(struct inode *inode, struct dentry *direntry) +{ + int rc = 0; + unsigned int xid; + struct cifs_sb_info *cifs_sb; + struct tcon_link *tlink; + struct cifs_tcon *tcon; + struct TCP_Server_Info *server; + const char *full_path; + void *page = alloc_dentry_path(); + struct cifsInodeInfo *cifsInode; + + cifs_dbg(FYI, "cifs_rmdir, inode = 0x%p\n", inode); + + xid = get_xid(); + + full_path = build_path_from_dentry(direntry, page); + if (IS_ERR(full_path)) { + rc = PTR_ERR(full_path); + goto rmdir_exit; + } + + cifs_sb = CIFS_SB(inode->i_sb); + if (unlikely(cifs_forced_shutdown(cifs_sb))) { + rc = smb_EIO(smb_eio_trace_forced_shutdown); + goto rmdir_exit; + } + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) { + rc = PTR_ERR(tlink); + goto rmdir_exit; + } + tcon = tlink_tcon(tlink); + server = tcon->ses->server; + + if (!server->ops->rmdir) { + rc = -ENOSYS; + cifs_put_tlink(tlink); + goto rmdir_exit; + } + + if (tcon->nodelete) { + rc = -EACCES; + cifs_put_tlink(tlink); + goto rmdir_exit; + } + + rc = server->ops->rmdir(xid, tcon, full_path, cifs_sb); + + cifsInode = CIFS_I(d_inode(direntry)); + + if (!rc) { + set_bit(CIFS_INO_DELETE_PENDING, &cifsInode->flags); + spin_lock(&d_inode(direntry)->i_lock); + i_size_write(d_inode(direntry), 0); + clear_nlink(d_inode(direntry)); + spin_unlock(&d_inode(direntry)->i_lock); + if (direntry->d_parent) + cifs_invalidate_cached_dir(tcon, direntry->d_parent); + } + + /* force revalidate to go get info when needed */ + cifsInode->time = 0; + + cifsInode = CIFS_I(inode); + /* + * Force revalidate to get parent dir info when needed since cached + * attributes are invalid now. + */ + cifsInode->time = 0; + + inode_set_ctime_current(d_inode(direntry)); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + cifs_put_tlink(tlink); + +rmdir_exit: + free_dentry_path(page); + free_xid(xid); + return rc; +} + +static int +cifs_do_rename(const unsigned int xid, struct dentry *from_dentry, + const char *from_path, struct dentry *to_dentry, + const char *to_path) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(from_dentry->d_sb); + struct tcon_link *tlink; + struct cifs_tcon *tcon; + struct TCP_Server_Info *server; +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + struct cifs_fid fid; + struct cifs_open_parms oparms; + int oplock; +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + int rc; + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) + return PTR_ERR(tlink); + tcon = tlink_tcon(tlink); + server = tcon->ses->server; + + if (!server->ops->rename) { + rc = -ENOSYS; + goto do_rename_exit; + } + + /* try path-based rename first */ + rc = server->ops->rename(xid, tcon, from_dentry, + from_path, to_path, cifs_sb); + + /* + * Don't bother with rename by filehandle unless file is busy and + * source. Note that cross directory moves do not work with + * rename by filehandle to various Windows servers. + */ + if (rc == 0 || rc != -EBUSY) + goto do_rename_exit; + + /* Don't fall back to using SMB on SMB 2+ mount */ + if (server->vals->protocol_id != 0) + goto do_rename_exit; + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + /* open-file renames don't work across directories */ + if (to_dentry->d_parent != from_dentry->d_parent) + goto do_rename_exit; + + /* + * CIFSSMBRenameOpenFile() uses SMB_SET_FILE_RENAME_INFORMATION + * which is SMB PASSTHROUGH level. + */ + if (!(tcon->ses->capabilities & CAP_INFOLEVEL_PASSTHRU)) + goto do_rename_exit; + + oparms = (struct cifs_open_parms) { + .tcon = tcon, + .cifs_sb = cifs_sb, + /* open the file to be renamed -- we need DELETE perms */ + .desired_access = DELETE, + .create_options = cifs_create_options(cifs_sb, CREATE_NOT_DIR), + .disposition = FILE_OPEN, + .path = from_path, + .fid = &fid, + }; + + rc = CIFS_open(xid, &oparms, &oplock, NULL); + if (rc == 0) { + rc = CIFSSMBRenameOpenFile(xid, tcon, fid.netfid, + (const char *) to_dentry->d_name.name, + cifs_sb->local_nls, cifs_remap(cifs_sb)); + CIFSSMBClose(xid, tcon, fid.netfid); + } +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ +do_rename_exit: + if (rc == 0) + d_move(from_dentry, to_dentry); + cifs_put_tlink(tlink); + return rc; +} + +int +cifs_rename2(struct mnt_idmap *idmap, struct inode *source_dir, + struct dentry *source_dentry, struct inode *target_dir, + struct dentry *target_dentry, unsigned int flags) +{ + const char *from_name, *to_name; + struct TCP_Server_Info *server; + void *page1, *page2; + struct cifs_sb_info *cifs_sb; + struct tcon_link *tlink; + struct cifs_tcon *tcon; + bool rehash = false; + unsigned int xid; + int rc, tmprc; + int retry_count = 0; + FILE_UNIX_BASIC_INFO *info_buf_source = NULL; +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + FILE_UNIX_BASIC_INFO *info_buf_target; +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + + if (flags & ~RENAME_NOREPLACE) + return -EINVAL; + + cifs_sb = CIFS_SB(source_dir->i_sb); + if (unlikely(cifs_forced_shutdown(cifs_sb))) + return smb_EIO(smb_eio_trace_forced_shutdown); + + /* + * Prevent any concurrent opens on the target by unhashing the dentry. + * VFS already unhashes the target when renaming directories. + */ + if (d_is_positive(target_dentry) && !d_is_dir(target_dentry)) { + if (!d_unhashed(target_dentry)) { + d_drop(target_dentry); + rehash = true; + } + } + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) + return PTR_ERR(tlink); + tcon = tlink_tcon(tlink); + server = tcon->ses->server; + + page1 = alloc_dentry_path(); + page2 = alloc_dentry_path(); + xid = get_xid(); + + from_name = build_path_from_dentry(source_dentry, page1); + if (IS_ERR(from_name)) { + rc = PTR_ERR(from_name); + goto cifs_rename_exit; + } + + to_name = build_path_from_dentry(target_dentry, page2); + if (IS_ERR(to_name)) { + rc = PTR_ERR(to_name); + goto cifs_rename_exit; + } + + cifs_close_deferred_file_under_dentry(tcon, source_dentry); + if (d_inode(target_dentry) != NULL) { + netfs_wait_for_outstanding_io(d_inode(target_dentry)); + cifs_close_deferred_file_under_dentry(tcon, target_dentry); + } + + rc = cifs_do_rename(xid, source_dentry, from_name, target_dentry, + to_name); + + if (rc == -EACCES) { + while (retry_count < 3) { + cifs_close_all_deferred_files(tcon); + rc = cifs_do_rename(xid, source_dentry, from_name, target_dentry, + to_name); + if (rc != -EACCES) + break; + retry_count++; + } + } + + if (!rc) + rehash = false; + /* + * No-replace is the natural behavior for CIFS, so skip unlink hacks. + */ + if (flags & RENAME_NOREPLACE) + goto cifs_rename_exit; + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + if (rc == -EEXIST && tcon->unix_ext) { + /* + * Are src and dst hardlinks of same inode? We can only tell + * with unix extensions enabled. + */ + info_buf_source = + kmalloc_objs(FILE_UNIX_BASIC_INFO, 2); + if (info_buf_source == NULL) { + rc = -ENOMEM; + goto cifs_rename_exit; + } + + info_buf_target = info_buf_source + 1; + tmprc = CIFSSMBUnixQPathInfo(xid, tcon, from_name, + info_buf_source, + cifs_sb->local_nls, + cifs_remap(cifs_sb)); + if (tmprc != 0) + goto unlink_target; + + tmprc = CIFSSMBUnixQPathInfo(xid, tcon, to_name, + info_buf_target, + cifs_sb->local_nls, + cifs_remap(cifs_sb)); + + if (tmprc == 0 && (info_buf_source->UniqueId == + info_buf_target->UniqueId)) { + /* same file, POSIX says that this is a noop */ + rc = 0; + goto cifs_rename_exit; + } + } + /* + * else ... BB we could add the same check for Windows by + * checking the UniqueId via FILE_INTERNAL_INFO + */ + +unlink_target: +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + if (d_really_is_positive(target_dentry)) { + if (!rc) { + struct inode *inode = d_inode(target_dentry); + /* + * Samba and ksmbd servers allow renaming a target + * directory that is open, so make sure to update + * ->i_nlink and then mark it as delete pending. + */ + if (S_ISDIR(inode->i_mode)) { + drop_cached_dir_by_name(xid, tcon, to_name, cifs_sb); + spin_lock(&inode->i_lock); + i_size_write(inode, 0); + clear_nlink(inode); + spin_unlock(&inode->i_lock); + set_bit(CIFS_INO_DELETE_PENDING, &CIFS_I(inode)->flags); + CIFS_I(inode)->time = 0; /* force reval */ + inode_set_ctime_current(inode); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + } + } else if (rc == -EACCES || rc == -EEXIST) { + /* + * Rename failed, possibly due to a busy target. + * Retry it by unliking the target first. + */ + if (d_is_dir(target_dentry)) { + tmprc = cifs_rmdir(target_dir, target_dentry); + } else { + tmprc = __cifs_unlink(target_dir, target_dentry, + server->vals->protocol_id > SMB10_PROT_ID); + } + if (tmprc) { + /* + * Some servers will return STATUS_ACCESS_DENIED + * or STATUS_DIRECTORY_NOT_EMPTY when failing to + * rename a non-empty directory. Make sure to + * propagate the appropriate error back to + * userspace. + */ + if (tmprc == -EEXIST || tmprc == -ENOTEMPTY) + rc = tmprc; + goto cifs_rename_exit; + } + rc = cifs_do_rename(xid, source_dentry, from_name, + target_dentry, to_name); + if (!rc) + rehash = false; + } + } + + /* force revalidate to go get info when needed */ + if (!rc) { + cifs_invalidate_cached_dir(tcon, source_dentry->d_parent); + if (target_dentry->d_parent != source_dentry->d_parent) + cifs_invalidate_cached_dir(tcon, target_dentry->d_parent); + } + + CIFS_I(source_dir)->time = CIFS_I(target_dir)->time = 0; + +cifs_rename_exit: + if (rehash) + d_rehash(target_dentry); + kfree(info_buf_source); + free_dentry_path(page2); + free_dentry_path(page1); + free_xid(xid); + cifs_put_tlink(tlink); + return rc; +} + +static bool +cifs_dentry_needs_reval(struct dentry *dentry) +{ + struct inode *inode = d_inode(dentry); + struct cifsInodeInfo *cifs_i = CIFS_I(inode); + struct cifs_sb_info *cifs_sb = CIFS_SB(inode); + struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb); + struct cached_fid *cfid = NULL; + + if (test_bit(CIFS_INO_DELETE_PENDING, &cifs_i->flags) || + test_bit(CIFS_INO_TMPFILE, &cifs_i->flags)) + return false; + if (cifs_i->time == 0) + return true; + + if (CIFS_CACHE_READ(cifs_i)) + return false; + + if (!lookupCacheEnabled) + return true; + + if (!open_cached_dir_by_dentry(tcon, dentry->d_parent, &cfid)) { + if (cifs_i->time > cfid->time) { + close_cached_dir(cfid); + return false; + } + close_cached_dir(cfid); + } + /* + * depending on inode type, check if attribute caching disabled for + * files or directories + */ + if (S_ISDIR(inode->i_mode)) { + if (!cifs_sb->ctx->acdirmax) + return true; + if (!time_in_range(jiffies, cifs_i->time, + cifs_i->time + cifs_sb->ctx->acdirmax)) + return true; + } else { /* file */ + if (!cifs_sb->ctx->acregmax) + return true; + if (!time_in_range(jiffies, cifs_i->time, + cifs_i->time + cifs_sb->ctx->acregmax)) + return true; + } + + /* hardlinked files w/ noserverino get "special" treatment */ + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SERVER_INUM) && + S_ISREG(inode->i_mode) && inode->i_nlink != 1) + return true; + + return false; +} + +/** + * cifs_wait_bit_killable - helper for functions that are sleeping on bit locks + * + * @key: currently unused + * @mode: the task state to sleep in + */ +static int +cifs_wait_bit_killable(struct wait_bit_key *key, int mode) +{ + schedule(); + if (signal_pending_state(mode, current)) + return -ERESTARTSYS; + return 0; +} + +int +cifs_revalidate_mapping(struct inode *inode) +{ + struct cifsInodeInfo *cifs_inode = CIFS_I(inode); + struct cifs_sb_info *cifs_sb = CIFS_SB(inode); + unsigned long *flags = &cifs_inode->flags; + int rc; + + /* swapfiles are not supposed to be shared */ + if (IS_SWAPFILE(inode)) + return 0; + + rc = wait_on_bit_lock_action(flags, CIFS_INO_LOCK, cifs_wait_bit_killable, + TASK_KILLABLE|TASK_FREEZABLE_UNSAFE); + if (rc) + return rc; + + if (test_and_clear_bit(CIFS_INO_INVALID_MAPPING, flags)) { + /* for cache=singleclient, do not invalidate */ + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_RW_CACHE) + goto skip_invalidate; + + spin_lock(&inode->i_lock); + netfs_write_zero_point(inode, netfs_inode(inode)->_remote_i_size); + spin_unlock(&inode->i_lock); + rc = filemap_invalidate_inode(inode, true, 0, LLONG_MAX); + if (rc) { + cifs_dbg(VFS, "%s: invalidate inode %p failed with rc %d\n", + __func__, inode, rc); + set_bit(CIFS_INO_INVALID_MAPPING, flags); + } + } + +skip_invalidate: + clear_and_wake_up_bit(CIFS_INO_LOCK, flags); + + return rc; +} + +int +cifs_zap_mapping(struct inode *inode) +{ + set_bit(CIFS_INO_INVALID_MAPPING, &CIFS_I(inode)->flags); + return cifs_revalidate_mapping(inode); +} + +int cifs_revalidate_file_attr(struct file *filp) +{ + int rc = 0; + struct dentry *dentry = file_dentry(filp); +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + struct cifsFileInfo *cfile = (struct cifsFileInfo *) filp->private_data; +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + + if (!cifs_dentry_needs_reval(dentry)) + return rc; + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + if (tlink_tcon(cfile->tlink)->unix_ext) + rc = cifs_get_file_info_unix(filp); + else +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + rc = cifs_get_file_info(filp); + + return rc; +} + +int cifs_revalidate_dentry_attr(struct dentry *dentry) +{ + unsigned int xid; + int rc = 0; + struct inode *inode = d_inode(dentry); + struct super_block *sb = dentry->d_sb; + const char *full_path; + void *page; + int count = 0; + + if (inode == NULL) + return -ENOENT; + + if (!cifs_dentry_needs_reval(dentry)) + return rc; + + xid = get_xid(); + + page = alloc_dentry_path(); + full_path = build_path_from_dentry(dentry, page); + if (IS_ERR(full_path)) { + rc = PTR_ERR(full_path); + goto out; + } + + cifs_dbg(FYI, "Update attributes: %s inode 0x%p count %d dentry: 0x%p d_time %ld jiffies %ld\n", + full_path, inode, icount_read_once(inode), + dentry, cifs_get_time(dentry), jiffies); + +again: + if (cifs_sb_master_tcon(CIFS_SB(sb))->posix_extensions) { + rc = smb311_posix_get_inode_info(&inode, full_path, + NULL, sb, xid); + } else if (cifs_sb_master_tcon(CIFS_SB(sb))->unix_ext) { + rc = cifs_get_inode_info_unix(&inode, full_path, sb, xid); + } else { + rc = cifs_get_inode_info(&inode, full_path, NULL, sb, + xid, NULL); + } + if (rc == -EAGAIN && count++ < 10) + goto again; +out: + free_dentry_path(page); + free_xid(xid); + + return rc; +} + +int cifs_revalidate_file(struct file *filp) +{ + int rc; + struct inode *inode = file_inode(filp); + + rc = cifs_revalidate_file_attr(filp); + if (rc) + return rc; + + return cifs_revalidate_mapping(inode); +} + +/* revalidate a dentry's inode attributes */ +int cifs_revalidate_dentry(struct dentry *dentry) +{ + int rc; + struct inode *inode = d_inode(dentry); + + rc = cifs_revalidate_dentry_attr(dentry); + if (rc) + return rc; + + return cifs_revalidate_mapping(inode); +} + +int cifs_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(path->dentry); + struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb); + struct dentry *dentry = path->dentry; + struct inode *inode = d_inode(dentry); + unsigned int sbflags; + int rc; + + if (unlikely(cifs_forced_shutdown(CIFS_SB(inode->i_sb)))) + return smb_EIO(smb_eio_trace_forced_shutdown); + + /* + * We need to be sure that all dirty pages are written and the server + * has actual ctime, mtime and file length. + */ + if ((request_mask & (STATX_CTIME | STATX_MTIME | STATX_SIZE | STATX_BLOCKS)) && + !CIFS_CACHE_READ(CIFS_I(inode)) && + inode->i_mapping && inode->i_mapping->nrpages != 0) { + rc = filemap_fdatawait(inode->i_mapping); + if (rc) { + mapping_set_error(inode->i_mapping, rc); + return rc; + } + } + + if ((flags & AT_STATX_SYNC_TYPE) == AT_STATX_FORCE_SYNC) + CIFS_I(inode)->time = 0; /* force revalidate */ + + /* + * If the caller doesn't require syncing, only sync if + * necessary (e.g. due to earlier truncate or setattr + * invalidating the cached metadata) + */ + if (((flags & AT_STATX_SYNC_TYPE) != AT_STATX_DONT_SYNC) || + (CIFS_I(inode)->time == 0)) { + rc = cifs_revalidate_dentry_attr(dentry); + if (rc) + return rc; + } + + generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat); + stat->blksize = cifs_sb->ctx->bsize; + stat->ino = CIFS_I(inode)->uniqueid; + + /* old CIFS Unix Extensions doesn't return create time */ + if (CIFS_I(inode)->createtime) { + stat->result_mask |= STATX_BTIME; + stat->btime = + cifs_NTtimeToUnix(cpu_to_le64(CIFS_I(inode)->createtime)); + } + + stat->attributes_mask |= (STATX_ATTR_COMPRESSED | STATX_ATTR_ENCRYPTED); + if (CIFS_I(inode)->cifsAttrs & FILE_ATTRIBUTE_COMPRESSED) + stat->attributes |= STATX_ATTR_COMPRESSED; + if (CIFS_I(inode)->cifsAttrs & FILE_ATTRIBUTE_ENCRYPTED) + stat->attributes |= STATX_ATTR_ENCRYPTED; + + /* + * If on a multiuser mount without unix extensions or cifsacl being + * enabled, and the admin hasn't overridden them, set the ownership + * to the fsuid/fsgid of the current process. + */ + sbflags = cifs_sb_flags(cifs_sb); + if ((sbflags & CIFS_MOUNT_MULTIUSER) && + !(sbflags & CIFS_MOUNT_CIFS_ACL) && + !tcon->unix_ext) { + if (!(sbflags & CIFS_MOUNT_OVERR_UID)) + stat->uid = current_fsuid(); + if (!(sbflags & CIFS_MOUNT_OVERR_GID)) + stat->gid = current_fsgid(); + } + return 0; +} + +int cifs_fiemap(struct inode *inode, struct fiemap_extent_info *fei, u64 start, + u64 len) +{ + struct cifsInodeInfo *cifs_i = CIFS_I(inode); + struct cifs_sb_info *cifs_sb = CIFS_SB(cifs_i->netfs.inode.i_sb); + struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb); + struct TCP_Server_Info *server = tcon->ses->server; + struct cifsFileInfo *cfile; + int rc; + + if (unlikely(cifs_forced_shutdown(cifs_sb))) + return smb_EIO(smb_eio_trace_forced_shutdown); + + /* + * We need to be sure that all dirty pages are written as they + * might fill holes on the server. + */ + if (!CIFS_CACHE_READ(CIFS_I(inode)) && inode->i_mapping && + inode->i_mapping->nrpages != 0) { + rc = filemap_fdatawait(inode->i_mapping); + if (rc) { + mapping_set_error(inode->i_mapping, rc); + return rc; + } + } + + cfile = find_readable_file(cifs_i, FIND_ANY); + if (cfile == NULL) + return -EINVAL; + + if (server->ops->fiemap) { + rc = server->ops->fiemap(tcon, cfile, fei, start, len); + cifsFileInfo_put(cfile); + return rc; + } + + cifsFileInfo_put(cfile); + return -EOPNOTSUPP; +} + +void cifs_setsize(struct inode *inode, loff_t offset) +{ + loff_t old_size; + u64 blocks = CIFS_INO_BLOCKS(offset); + + spin_lock(&inode->i_lock); + old_size = i_size_read(inode); + i_size_write(inode, offset); + + /* + * Extending EOF does not allocate the intervening range. Only clamp + * i_blocks on shrink; allocation growth comes from writes or from the + * server-reported AllocationSize. + */ + if (offset < old_size && (u64)inode->i_blocks > blocks) + inode->i_blocks = blocks; + spin_unlock(&inode->i_lock); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + truncate_pagecache(inode, offset); + netfs_wait_for_outstanding_io(inode); + fscache_resize_cookie(cifs_inode_cookie(inode), offset); +} + +int cifs_file_set_size(const unsigned int xid, struct dentry *dentry, + const char *full_path, struct cifsFileInfo *open_file, + loff_t size) +{ + struct inode *inode = d_inode(dentry); + struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); + struct cifsInodeInfo *cifsInode = CIFS_I(inode); + struct tcon_link *tlink = NULL; + struct cifs_tcon *tcon = NULL; + struct TCP_Server_Info *server; + int rc = -EINVAL; + + /* + * To avoid spurious oplock breaks from server, in the case of + * inodes that we already have open, avoid doing path based + * setting of file size if we can do it by handle. + * This keeps our caching token (oplock) and avoids timeouts + * when the local oplock break takes longer to flush + * writebehind data than the SMB timeout for the SetPathInfo + * request would allow + */ + if (open_file && (OPEN_FMODE(open_file->f_flags) & FMODE_WRITE)) { + tcon = tlink_tcon(open_file->tlink); + server = tcon->ses->server; + rc = server->ops->set_file_size(xid, tcon, + open_file, + size, false); + cifs_dbg(FYI, "%s: set_file_size: rc = %d\n", __func__, rc); + } else { + open_file = find_writable_file(cifsInode, FIND_FSUID_ONLY); + if (open_file) { + tcon = tlink_tcon(open_file->tlink); + server = tcon->ses->server; + rc = server->ops->set_file_size(xid, tcon, + open_file, + size, false); + cifs_dbg(FYI, "%s: set_file_size: rc = %d\n", __func__, rc); + cifsFileInfo_put(open_file); + } + } + + if (!rc) + goto set_size_out; + + if (tcon == NULL) { + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) + return PTR_ERR(tlink); + tcon = tlink_tcon(tlink); + server = tcon->ses->server; + } + + /* + * Set file size by pathname rather than by handle either because no + * valid, writeable file handle for it was found or because there was + * an error setting it by handle. + */ + rc = server->ops->set_path_size(xid, tcon, full_path, size, + cifs_sb, false, dentry); + cifs_dbg(FYI, "%s: SetEOF by path (setattrs) rc = %d\n", __func__, rc); + cifs_put_tlink(tlink); + +set_size_out: + if (rc == 0) { + netfs_resize_file(&cifsInode->netfs, size, true); + cifs_setsize(inode, size); + } + + return rc; +} + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY +static int +cifs_setattr_unix(struct dentry *direntry, struct iattr *attrs) +{ + int rc; + unsigned int xid; + const char *full_path; + void *page = alloc_dentry_path(); + struct inode *inode = d_inode(direntry); + struct cifsInodeInfo *cifsInode = CIFS_I(inode); + struct cifs_sb_info *cifs_sb = CIFS_SB(inode); + struct tcon_link *tlink; + struct cifs_tcon *pTcon; + struct cifs_unix_set_info_args *args = NULL; + struct cifsFileInfo *open_file = NULL; + + cifs_dbg(FYI, "setattr_unix on file %pd attrs->ia_valid=0x%x\n", + direntry, attrs->ia_valid); + + xid = get_xid(); + + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NO_PERM) + attrs->ia_valid |= ATTR_FORCE; + + rc = setattr_prepare(&nop_mnt_idmap, direntry, attrs); + if (rc < 0) + goto out; + + if (attrs->ia_valid & ATTR_FILE) + open_file = attrs->ia_file->private_data; + + full_path = build_path_from_dentry(direntry, page); + if (IS_ERR(full_path)) { + rc = PTR_ERR(full_path); + goto out; + } + + /* + * Attempt to flush data before changing attributes. We need to do + * this for ATTR_SIZE and ATTR_MTIME for sure, and if we change the + * ownership or mode then we may also need to do this. Here, we take + * the safe way out and just do the flush on all setattr requests. If + * the flush returns error, store it to report later and continue. + * + * BB: This should be smarter. Why bother flushing pages that + * will be truncated anyway? Also, should we error out here if + * the flush returns error? + */ + rc = filemap_write_and_wait(inode->i_mapping); + if (is_interrupt_error(rc)) { + rc = -ERESTARTSYS; + goto out; + } + + mapping_set_error(inode->i_mapping, rc); + rc = 0; + + if (attrs->ia_valid & ATTR_SIZE) { + if (attrs->ia_size != i_size_read(inode)) { + /* Stamp before RPC. On failure the stamp remains: restoring a + * stale snapshot could silently erase a concurrent + * _cifsFileInfo_put() close stamp. readdir is suppressed + * until the stamp expires; stat() bypasses this via the + * from_readdir=false path in is_size_safe_to_change() and + * always returns an authoritative QUERY_INFO result. + * Pairs with smp_load_acquire() in is_size_safe_to_change(). + */ + smp_store_release(&cifsInode->time_last_write, jiffies); + } + rc = cifs_file_set_size(xid, direntry, full_path, + open_file, attrs->ia_size); + if (rc != 0) + goto out; + /* + * Avoid setting timestamps on the server for ftruncate(2) to + * prevent it from disabling automatic timestamp updates as per + * MS-FSA 2.1.4.17. + */ + attrs->ia_valid &= ~(ATTR_CTIME | ATTR_MTIME); + } + + /* skip mode change if it's just for clearing setuid/setgid */ + if (attrs->ia_valid & (ATTR_KILL_SUID|ATTR_KILL_SGID)) + attrs->ia_valid &= ~ATTR_MODE; + + args = kmalloc_obj(*args); + if (args == NULL) { + rc = -ENOMEM; + goto out; + } + + /* set up the struct */ + if (attrs->ia_valid & ATTR_MODE) + args->mode = attrs->ia_mode; + else + args->mode = NO_CHANGE_64; + + if (attrs->ia_valid & ATTR_UID) + args->uid = attrs->ia_uid; + else + args->uid = INVALID_UID; /* no change */ + + if (attrs->ia_valid & ATTR_GID) + args->gid = attrs->ia_gid; + else + args->gid = INVALID_GID; /* no change */ + + if (attrs->ia_valid & ATTR_ATIME) + args->atime = cifs_UnixTimeToNT(attrs->ia_atime); + else + args->atime = NO_CHANGE_64; + + if (attrs->ia_valid & ATTR_MTIME) + args->mtime = cifs_UnixTimeToNT(attrs->ia_mtime); + else + args->mtime = NO_CHANGE_64; + + if (attrs->ia_valid & ATTR_CTIME) + args->ctime = cifs_UnixTimeToNT(attrs->ia_ctime); + else + args->ctime = NO_CHANGE_64; + + args->device = 0; + rc = -EINVAL; + if (open_file && (OPEN_FMODE(open_file->f_flags) & FMODE_WRITE)) { + pTcon = tlink_tcon(open_file->tlink); + rc = CIFSSMBUnixSetFileInfo(xid, pTcon, args, + open_file->fid.netfid, + open_file->pid); + } else { + open_file = find_writable_file(cifsInode, FIND_FSUID_ONLY); + if (open_file) { + pTcon = tlink_tcon(open_file->tlink); + rc = CIFSSMBUnixSetFileInfo(xid, pTcon, args, + open_file->fid.netfid, + open_file->pid); + cifsFileInfo_put(open_file); + } + } + + if (rc) { + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) { + rc = PTR_ERR(tlink); + goto out; + } + pTcon = tlink_tcon(tlink); + rc = CIFSSMBUnixSetPathInfo(xid, pTcon, full_path, args, + cifs_sb->local_nls, + cifs_remap(cifs_sb)); + cifs_put_tlink(tlink); + } + + if (rc) + goto out; + + if ((attrs->ia_valid & ATTR_SIZE) && + attrs->ia_size != i_size_read(inode)) { + truncate_setsize(inode, attrs->ia_size); + netfs_resize_file(&cifsInode->netfs, attrs->ia_size, true); + fscache_resize_cookie(cifs_inode_cookie(inode), attrs->ia_size); + } + + setattr_copy(&nop_mnt_idmap, inode, attrs); + mark_inode_dirty(inode); + + /* force revalidate when any of these times are set since some + of the fs types (eg ext3, fat) do not have fine enough + time granularity to match protocol, and we do not have a + a way (yet) to query the server fs's time granularity (and + whether it rounds times down). + */ + if (attrs->ia_valid & (ATTR_MTIME | ATTR_CTIME)) + cifsInode->time = 0; +out: + kfree(args); + free_dentry_path(page); + free_xid(xid); + return rc; +} +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + +static int +cifs_setattr_nounix(struct dentry *direntry, struct iattr *attrs) +{ + struct inode *inode = d_inode(direntry); + struct cifsInodeInfo *cifsInode = CIFS_I(inode); + struct cifs_sb_info *cifs_sb = CIFS_SB(inode); + unsigned int sbflags = cifs_sb_flags(cifs_sb); + struct cifsFileInfo *cfile = NULL; + void *page = alloc_dentry_path(); + __u64 mode = NO_CHANGE_64; + kuid_t uid = INVALID_UID; + kgid_t gid = INVALID_GID; + const char *full_path; + __u32 dosattr = 0; + int rc = -EACCES; + unsigned int xid; + + xid = get_xid(); + + cifs_dbg(FYI, "setattr on file %pd attrs->ia_valid 0x%x\n", + direntry, attrs->ia_valid); + + if (sbflags & CIFS_MOUNT_NO_PERM) + attrs->ia_valid |= ATTR_FORCE; + + rc = setattr_prepare(&nop_mnt_idmap, direntry, attrs); + if (rc < 0) + goto cifs_setattr_exit; + + if (attrs->ia_valid & ATTR_FILE) + cfile = attrs->ia_file->private_data; + + full_path = build_path_from_dentry(direntry, page); + if (IS_ERR(full_path)) { + rc = PTR_ERR(full_path); + goto cifs_setattr_exit; + } + + /* + * Attempt to flush data before changing attributes. We need to do + * this for ATTR_SIZE and ATTR_MTIME. If the flush of the data + * returns error, store it to report later and continue. + * + * BB: This should be smarter. Why bother flushing pages that + * will be truncated anyway? Also, should we error out here if + * the flush returns error? Do we need to check for ATTR_MTIME_SET flag? + */ + if (attrs->ia_valid & (ATTR_MTIME | ATTR_SIZE | ATTR_CTIME)) { + rc = filemap_write_and_wait(inode->i_mapping); + if (is_interrupt_error(rc)) { + rc = -ERESTARTSYS; + goto cifs_setattr_exit; + } + mapping_set_error(inode->i_mapping, rc); + } + + rc = 0; + + if (attrs->ia_valid & ATTR_MTIME) { + rc = cifs_file_flush(xid, inode, cfile); + if (rc) + goto cifs_setattr_exit; + } + + if (attrs->ia_valid & ATTR_SIZE) { + if (attrs->ia_size != i_size_read(inode)) { + /* Stamp before RPC. On failure the stamp remains: restoring a + * stale snapshot could silently erase a concurrent + * _cifsFileInfo_put() close stamp. readdir is suppressed + * until the stamp expires; stat() bypasses this via the + * from_readdir=false path in is_size_safe_to_change() and + * always returns an authoritative QUERY_INFO result. + * Pairs with smp_load_acquire() in is_size_safe_to_change(). + */ + smp_store_release(&cifsInode->time_last_write, jiffies); + } + rc = cifs_file_set_size(xid, direntry, full_path, + cfile, attrs->ia_size); + if (rc != 0) + goto cifs_setattr_exit; + /* + * Avoid setting timestamps on the server for ftruncate(2) to + * prevent it from disabling automatic timestamp updates as per + * MS-FSA 2.1.4.17. + */ + attrs->ia_valid &= ~(ATTR_CTIME | ATTR_MTIME); + } + + if (attrs->ia_valid & ATTR_UID) + uid = attrs->ia_uid; + + if (attrs->ia_valid & ATTR_GID) + gid = attrs->ia_gid; + + if ((sbflags & (CIFS_MOUNT_CIFS_ACL | CIFS_MOUNT_MODE_FROM_SID)) || + cifs_sb_master_tcon(cifs_sb)->posix_extensions) { + if (uid_valid(uid) || gid_valid(gid)) { + mode = NO_CHANGE_64; + rc = id_mode_to_cifs_acl(inode, full_path, &mode, + uid, gid); + if (rc) { + cifs_dbg(FYI, "%s: Setting id failed with error: %d\n", + __func__, rc); + goto cifs_setattr_exit; + } + } + } else if (!(sbflags & CIFS_MOUNT_SET_UID)) { + attrs->ia_valid &= ~(ATTR_UID | ATTR_GID); + } + + /* skip mode change if it's just for clearing setuid/setgid */ + if (attrs->ia_valid & (ATTR_KILL_SUID|ATTR_KILL_SGID)) + attrs->ia_valid &= ~ATTR_MODE; + + if (attrs->ia_valid & ATTR_MODE) { + mode = attrs->ia_mode; + rc = 0; + if ((sbflags & (CIFS_MOUNT_CIFS_ACL | CIFS_MOUNT_MODE_FROM_SID)) || + cifs_sb_master_tcon(cifs_sb)->posix_extensions) { + rc = id_mode_to_cifs_acl(inode, full_path, &mode, + INVALID_UID, INVALID_GID); + if (rc) { + cifs_dbg(FYI, "%s: Setting ACL failed with error: %d\n", + __func__, rc); + goto cifs_setattr_exit; + } + + /* + * In case of CIFS_MOUNT_CIFS_ACL, we cannot support all modes. + * Pick up the actual mode bits that were set. + */ + if (mode != attrs->ia_mode) + attrs->ia_mode = mode; + } else + if (((mode & S_IWUGO) == 0) && + (cifsInode->cifsAttrs & ATTR_READONLY) == 0) { + + dosattr = cifsInode->cifsAttrs | ATTR_READONLY; + + /* fix up mode if we're not using dynperm */ + if ((sbflags & CIFS_MOUNT_DYNPERM) == 0) + attrs->ia_mode = inode->i_mode & ~S_IWUGO; + } else if ((mode & S_IWUGO) && + (cifsInode->cifsAttrs & ATTR_READONLY)) { + + dosattr = cifsInode->cifsAttrs & ~ATTR_READONLY; + /* Attributes of 0 are ignored */ + if (dosattr == 0) + dosattr |= ATTR_NORMAL; + + /* reset local inode permissions to normal */ + if (!(sbflags & CIFS_MOUNT_DYNPERM)) { + attrs->ia_mode &= ~(S_IALLUGO); + if (S_ISDIR(inode->i_mode)) + attrs->ia_mode |= + cifs_sb->ctx->dir_mode; + else + attrs->ia_mode |= + cifs_sb->ctx->file_mode; + } + } else if (!(sbflags & CIFS_MOUNT_DYNPERM)) { + /* ignore mode change - ATTR_READONLY hasn't changed */ + attrs->ia_valid &= ~ATTR_MODE; + } + } + + if (attrs->ia_valid & (ATTR_MTIME|ATTR_ATIME|ATTR_CTIME) || + ((attrs->ia_valid & ATTR_MODE) && dosattr)) { + rc = cifs_set_file_info(inode, attrs, xid, full_path, dosattr); + /* BB: check for rc = -EOPNOTSUPP and switch to legacy mode */ + + /* Even if error on time set, no sense failing the call if + the server would set the time to a reasonable value anyway, + and this check ensures that we are not being called from + sys_utimes in which case we ought to fail the call back to + the user when the server rejects the call */ + if ((rc) && (attrs->ia_valid & + (ATTR_MODE | ATTR_GID | ATTR_UID | ATTR_SIZE))) + rc = 0; + } + + /* do not need local check to inode_check_ok since the server does + that */ + if (rc) + goto cifs_setattr_exit; + + if ((attrs->ia_valid & ATTR_SIZE) && + attrs->ia_size != i_size_read(inode)) { + truncate_setsize(inode, attrs->ia_size); + netfs_resize_file(&cifsInode->netfs, attrs->ia_size, true); + fscache_resize_cookie(cifs_inode_cookie(inode), attrs->ia_size); + } + + setattr_copy(&nop_mnt_idmap, inode, attrs); + mark_inode_dirty(inode); + +cifs_setattr_exit: + free_xid(xid); + free_dentry_path(page); + return rc; +} + +int +cifs_setattr(struct mnt_idmap *idmap, struct dentry *direntry, + struct iattr *attrs) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(direntry->d_sb); + int rc, retries = 0; +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + struct cifs_tcon *pTcon = cifs_sb_master_tcon(cifs_sb); +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + + if (unlikely(cifs_forced_shutdown(cifs_sb))) + return smb_EIO(smb_eio_trace_forced_shutdown); + /* + * Avoid setting [cm]time with O_TRUNC to prevent the server from + * disabling automatic timestamp updates as specified in + * MS-FSA 2.1.4.17. + */ + if (attrs->ia_valid & ATTR_OPEN) + return 0; + + do { +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + if (pTcon->unix_ext) + rc = cifs_setattr_unix(direntry, attrs); + else +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + rc = cifs_setattr_nounix(direntry, attrs); + retries++; + } while (is_retryable_error(rc) && retries < 2); + + /* BB: add cifs_setattr_legacy for really old servers */ + return rc; +} diff --git a/fs/smb/client/ioctl.c b/fs/smb/client/ioctl.c new file mode 100644 index 000000000..9fa743be3 --- /dev/null +++ b/fs/smb/client/ioctl.c @@ -0,0 +1,723 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * vfs operations that deal with io control + * + * Copyright (C) International Business Machines Corp., 2005,2013 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ + +#include +#include +#include +#include +#include +#include "cifsglob.h" +#include "cifsproto.h" +#include "cifs_debug.h" +#include "cifsfs.h" +#include "cifs_ioctl.h" +#include "smb2proto.h" +#include "smb2glob.h" +#include + +static long cifs_ioctl_query_info(unsigned int xid, struct file *filep, + unsigned long p) +{ + struct inode *inode = file_inode(filep); + struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); + struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb); + struct dentry *dentry = filep->f_path.dentry; + const unsigned char *path; + void *page = alloc_dentry_path(); + __le16 *utf16_path = NULL, root_path; + int rc = 0; + + path = build_path_from_dentry(dentry, page); + if (IS_ERR(path)) { + free_dentry_path(page); + return PTR_ERR(path); + } + + cifs_dbg(FYI, "%s %s\n", __func__, path); + + if (!path[0]) { + root_path = 0; + utf16_path = &root_path; + } else { + utf16_path = cifs_convert_path_to_utf16(path + 1, cifs_sb); + if (!utf16_path) { + rc = -ENOMEM; + goto ici_exit; + } + } + + if (tcon->ses->server->ops->ioctl_query_info) + rc = tcon->ses->server->ops->ioctl_query_info( + xid, tcon, cifs_sb, utf16_path, + filep->private_data ? 0 : 1, p); + else + rc = -EOPNOTSUPP; + + ici_exit: + if (utf16_path != &root_path) + kfree(utf16_path); + free_dentry_path(page); + return rc; +} + +static int cifs_set_compression_by_path(unsigned int xid, struct file *filep, + struct cifs_tcon *tcon, + __u16 compression_state) +{ + struct inode *inode = file_inode(filep); + struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); + struct TCP_Server_Info *server = tcon->ses->server; + struct cifs_open_parms oparms; + struct cifs_open_info_data data = {}; + struct cifsFileInfo *tmp_cfile = NULL; + struct cifs_fid fid = {}; + const char *full_path; + __u32 oplock = 0; + u64 uniqueid; + void *page; + int rc; + + if (!server->ops->open || !server->ops->close || + !server->ops->query_file_info) + return -EOPNOTSUPP; + + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SERVER_INUM) || + cifs_sb->mnt_cifs_serverino_autodisabled) + return -EOPNOTSUPP; + + if (d_unhashed(filep->f_path.dentry)) + return -ESTALE; + + page = alloc_dentry_path(); + full_path = build_path_from_dentry(filep->f_path.dentry, page); + if (IS_ERR(full_path)) { + free_dentry_path(page); + return PTR_ERR(full_path); + } + + oparms = CIFS_OPARMS(cifs_sb, tcon, full_path, FILE_WRITE_DATA | + FILE_READ_ATTRIBUTES, + FILE_OPEN, 0, ACL_NO_MODE); + oparms.fid = &fid; + + rc = server->ops->open(xid, &oparms, &oplock, NULL); + if (rc) + goto out; + + tmp_cfile = kzalloc_obj(*tmp_cfile); + if (!tmp_cfile) { + rc = -ENOMEM; + goto close; + } + + tmp_cfile->fid = fid; + rc = server->ops->query_file_info(xid, tcon, tmp_cfile, &data); + if (rc) + goto close; + + uniqueid = le64_to_cpu(data.fi.IndexNumber); + if (uniqueid != CIFS_I(inode)->uniqueid) { + rc = -ESTALE; + goto close; + } + + rc = server->ops->set_compression(xid, tcon, tmp_cfile, + compression_state); + +close: + server->ops->close(xid, tcon, &fid); + if (tmp_cfile) + kfree(tmp_cfile); + cifs_free_open_info(&data); +out: + free_dentry_path(page); + return rc; +} + +static int cifs_ioctl_set_compression(unsigned int xid, struct file *filep, + struct cifs_tcon *tcon, + struct cifsFileInfo *cfile, + __u16 compression_state) +{ + struct cifsFileInfo *wfile; + struct cifs_tcon *wtcon; + struct inode *inode = file_inode(filep); + int rc; + + if (!tcon->ses->server->ops->set_compression) + return -EOPNOTSUPP; + + if (cfile && (cfile->fid.access & FILE_WRITE_DATA)) { + rc = tcon->ses->server->ops->set_compression(xid, tcon, cfile, + compression_state); + if (rc != -EACCES) + return rc; + } + + rc = cifs_get_writable_file(CIFS_I(inode), FIND_FSUID_ONLY, &wfile); + if (!rc) { + wtcon = tlink_tcon(wfile->tlink); + rc = wtcon->ses->server->ops->set_compression(xid, wtcon, wfile, + compression_state); + cifsFileInfo_put(wfile); + if (rc != -EACCES) + return rc; + } else if (rc != -EBADF) { + return rc; + } + + return cifs_set_compression_by_path(xid, filep, tcon, + compression_state); +} + +static long cifs_ioctl_copychunk(unsigned int xid, struct file *dst_file, + unsigned long srcfd) +{ + int rc; + struct inode *src_inode; + + cifs_dbg(FYI, "ioctl copychunk range\n"); + /* the destination must be opened for writing */ + if (!(dst_file->f_mode & FMODE_WRITE)) { + cifs_dbg(FYI, "file target not open for write\n"); + return -EINVAL; + } + + /* check if target volume is readonly and take reference */ + rc = mnt_want_write_file(dst_file); + if (rc) { + cifs_dbg(FYI, "mnt_want_write failed with rc %d\n", rc); + return rc; + } + + CLASS(fd, src_file)(srcfd); + if (fd_empty(src_file)) { + rc = -EBADF; + goto out_drop_write; + } + + if (fd_file(src_file)->f_op->unlocked_ioctl != cifs_ioctl) { + rc = -EBADF; + cifs_dbg(VFS, "src file seems to be from a different filesystem type\n"); + goto out_drop_write; + } + + src_inode = file_inode(fd_file(src_file)); + rc = -EINVAL; + if (S_ISDIR(src_inode->i_mode)) + goto out_drop_write; + + rc = cifs_file_copychunk_range(xid, fd_file(src_file), 0, dst_file, 0, + src_inode->i_size, 0); + if (rc > 0) + rc = 0; +out_drop_write: + mnt_drop_write_file(dst_file); + return rc; +} + +static long smb_mnt_get_tcon_info(struct cifs_tcon *tcon, void __user *arg) +{ + int rc = 0; + struct smb_mnt_tcon_info tcon_inf; + + tcon_inf.tid = tcon->tid; + tcon_inf.session_id = tcon->ses->Suid; + + if (copy_to_user(arg, &tcon_inf, sizeof(struct smb_mnt_tcon_info))) + rc = -EFAULT; + + return rc; +} + +static long smb_mnt_get_fsinfo(unsigned int xid, struct cifs_tcon *tcon, + void __user *arg) +{ + int rc = 0; + struct smb_mnt_fs_info *fsinf; + + fsinf = kzalloc_obj(struct smb_mnt_fs_info); + if (fsinf == NULL) + return -ENOMEM; + + fsinf->version = 1; + fsinf->protocol_id = tcon->ses->server->vals->protocol_id; + fsinf->tcon_flags = tcon->Flags; + fsinf->device_characteristics = + le32_to_cpu(tcon->fsDevInfo.DeviceCharacteristics); + fsinf->device_type = le32_to_cpu(tcon->fsDevInfo.DeviceType); + fsinf->fs_attributes = le32_to_cpu(tcon->fsAttrInfo.Attributes); + fsinf->max_path_component = + le32_to_cpu(tcon->fsAttrInfo.MaxPathNameComponentLength); + fsinf->vol_serial_number = tcon->vol_serial_number; + fsinf->vol_create_time = le64_to_cpu(tcon->vol_create_time); + fsinf->share_flags = tcon->share_flags; + fsinf->share_caps = le32_to_cpu(tcon->capabilities); + fsinf->sector_flags = tcon->ss_flags; + fsinf->optimal_sector_size = tcon->perf_sector_size; + fsinf->max_bytes_chunk = tcon->max_bytes_chunk; + fsinf->maximal_access = tcon->maximal_access; + fsinf->cifs_posix_caps = le64_to_cpu(tcon->fsUnixInfo.Capability); + + if (copy_to_user(arg, fsinf, sizeof(struct smb_mnt_fs_info))) + rc = -EFAULT; + + kfree(fsinf); + return rc; +} + +static int cifs_shutdown(struct super_block *sb, unsigned long arg) +{ + struct cifs_sb_info *sbi = CIFS_SB(sb); + struct tcon_link *tlink; + struct cifs_tcon *tcon; + __u32 flags; + int rc; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (get_user(flags, (__u32 __user *)arg)) + return -EFAULT; + + tlink = cifs_sb_tlink(sbi); + if (IS_ERR(tlink)) + return PTR_ERR(tlink); + tcon = tlink_tcon(tlink); + + trace_smb3_shutdown_enter(flags, tcon->tid); + if (flags > CIFS_GOING_FLAGS_NOLOGFLUSH) { + rc = -EINVAL; + goto shutdown_out_err; + } + + if (cifs_forced_shutdown(sbi)) + goto shutdown_good; + + cifs_dbg(VFS, "shut down requested (%d)", flags); + + /* + * see: + * https://man7.org/linux/man-pages/man2/ioctl_xfs_goingdown.2.html + * for more information and description of original intent of the flags + */ + switch (flags) { + /* + * We could add support later for default flag which requires: + * "Flush all dirty data and metadata to disk" + * would need to call syncfs or equivalent to flush page cache for + * the mount and then issue fsync to server (if nostrictsync not set) + */ + case CIFS_GOING_FLAGS_DEFAULT: + cifs_dbg(FYI, "shutdown with default flag not supported\n"); + rc = -EINVAL; + goto shutdown_out_err; + /* + * FLAGS_LOGFLUSH is easy since it asks to write out metadata (not + * data) but metadata writes are not cached on the client, so can treat + * it similarly to NOLOGFLUSH + */ + case CIFS_GOING_FLAGS_LOGFLUSH: + case CIFS_GOING_FLAGS_NOLOGFLUSH: + atomic_or(CIFS_MOUNT_SHUTDOWN, &sbi->mnt_cifs_flags); + goto shutdown_good; + default: + rc = -EINVAL; + goto shutdown_out_err; + } + +shutdown_good: + trace_smb3_shutdown_done(flags, tcon->tid); + cifs_put_tlink(tlink); + return 0; +shutdown_out_err: + trace_smb3_shutdown_err(rc, flags, tcon->tid); + cifs_put_tlink(tlink); + return rc; +} + +static int cifs_dump_full_key(struct cifs_tcon *tcon, struct smb3_full_key_debug_info __user *in) +{ + struct smb3_full_key_debug_info out; + struct cifs_ses *ses; + int rc = 0; + bool found = false; + u8 __user *end; + + if (!smb3_encryption_required(tcon)) { + rc = -EOPNOTSUPP; + goto out; + } + + /* copy user input into our output buffer */ + if (copy_from_user(&out, in, sizeof(out))) { + rc = -EINVAL; + goto out; + } + + if (!out.session_id) { + /* if ses id is 0, use current user session */ + ses = tcon->ses; + } else { + /* otherwise if a session id is given, look for it in all our sessions */ + struct cifs_ses *ses_it = NULL; + struct TCP_Server_Info *server_it = NULL; + + spin_lock(&cifs_tcp_ses_lock); + list_for_each_entry(server_it, &cifs_tcp_ses_list, tcp_ses_list) { + list_for_each_entry(ses_it, &server_it->smb_ses_list, smb_ses_list) { + spin_lock(&ses_it->ses_lock); + if (ses_it->ses_status != SES_EXITING && + ses_it->Suid == out.session_id) { + ses = ses_it; + /* + * since we are using the session outside the crit + * section, we need to make sure it won't be released + * so increment its refcount + */ + cifs_smb_ses_inc_refcount(ses); + spin_unlock(&ses_it->ses_lock); + found = true; + goto search_end; + } + spin_unlock(&ses_it->ses_lock); + } + } +search_end: + spin_unlock(&cifs_tcp_ses_lock); + if (!found) { + rc = -ENOENT; + goto out; + } + } + + switch (ses->server->cipher_type) { + case SMB2_ENCRYPTION_AES128_CCM: + case SMB2_ENCRYPTION_AES128_GCM: + out.session_key_length = CIFS_SESS_KEY_SIZE; + out.server_in_key_length = out.server_out_key_length = SMB3_GCM128_CRYPTKEY_SIZE; + break; + case SMB2_ENCRYPTION_AES256_CCM: + case SMB2_ENCRYPTION_AES256_GCM: + out.session_key_length = ses->auth_key.len; + out.server_in_key_length = out.server_out_key_length = SMB3_GCM256_CRYPTKEY_SIZE; + break; + default: + rc = -EOPNOTSUPP; + goto out; + } + + /* check if user buffer is big enough to store all the keys */ + if (out.in_size < sizeof(out) + out.session_key_length + out.server_in_key_length + + out.server_out_key_length) { + rc = -ENOBUFS; + goto out; + } + + out.session_id = ses->Suid; + out.cipher_type = le16_to_cpu(ses->server->cipher_type); + + /* overwrite user input with our output */ + if (copy_to_user(in, &out, sizeof(out))) { + rc = -EINVAL; + goto out; + } + + /* append all the keys at the end of the user buffer */ + end = in->data; + if (copy_to_user(end, ses->auth_key.response, out.session_key_length)) { + rc = -EINVAL; + goto out; + } + end += out.session_key_length; + + if (copy_to_user(end, ses->smb3encryptionkey, out.server_in_key_length)) { + rc = -EINVAL; + goto out; + } + end += out.server_in_key_length; + + if (copy_to_user(end, ses->smb3decryptionkey, out.server_out_key_length)) { + rc = -EINVAL; + goto out; + } + +out: + if (found) + cifs_put_smb_ses(ses); + return rc; +} + +long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg) +{ + struct inode *inode = file_inode(filep); + struct smb3_key_debug_info pkey_inf; + int rc = -ENOTTY; /* strange error - but the precedent */ + unsigned int xid; + struct cifsFileInfo *pSMBFile = filep->private_data; + struct cifs_tcon *tcon; + struct tcon_link *tlink; + struct cifs_sb_info *cifs_sb; + __u64 ExtAttrBits = 0; + bool enable_compression; + __u16 compression_state; +#ifdef CONFIG_CIFS_POSIX +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + __u64 caps; +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ +#endif /* CONFIG_CIFS_POSIX */ + + xid = get_xid(); + + cifs_dbg(FYI, "cifs ioctl 0x%x\n", command); + if (pSMBFile == NULL) + trace_smb3_ioctl(xid, 0, command); + else + trace_smb3_ioctl(xid, pSMBFile->fid.persistent_fid, command); + + switch (command) { + case FS_IOC_GETFLAGS: + if (pSMBFile == NULL) + break; + tcon = tlink_tcon(pSMBFile->tlink); +#ifdef CONFIG_CIFS_POSIX +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + caps = le64_to_cpu(tcon->fsUnixInfo.Capability); + if (CIFS_UNIX_EXTATTR_CAP & caps) { + __u64 ExtAttrMask = 0; + rc = CIFSGetExtAttr(xid, tcon, + pSMBFile->fid.netfid, + &ExtAttrBits, &ExtAttrMask); + if (rc == 0) + rc = put_user(ExtAttrBits & + FS_FL_USER_VISIBLE, + (int __user *)arg); + if (rc != -EOPNOTSUPP) + break; + } +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ +#endif /* CONFIG_CIFS_POSIX */ + if (CIFS_I(inode)->cifsAttrs & FILE_ATTRIBUTE_COMPRESSED) + ExtAttrBits |= FS_COMPR_FL; + + rc = put_user(ExtAttrBits & FS_FL_USER_VISIBLE, + (int __user *)arg); + break; + case FS_IOC_SETFLAGS: + if (pSMBFile == NULL) + break; + tcon = tlink_tcon(pSMBFile->tlink); + /* caps = le64_to_cpu(tcon->fsUnixInfo.Capability); */ + + if (get_user(ExtAttrBits, (int __user *)arg)) { + rc = -EFAULT; + break; + } + + /* + * if (CIFS_UNIX_EXTATTR_CAP & caps) + * rc = CIFSSetExtAttr(xid, tcon, + * pSMBFile->fid.netfid, + * extAttrBits, + * &ExtAttrMask); + * if (rc != -EOPNOTSUPP) + * break; + */ + + /* Currently only flag we can set or clear is compressed. */ + if (ExtAttrBits & ~FS_COMPR_FL) { + rc = -EOPNOTSUPP; + break; + } + + enable_compression = ExtAttrBits & FS_COMPR_FL; + compression_state = enable_compression ? + COMPRESSION_FORMAT_DEFAULT : + COMPRESSION_FORMAT_NONE; + + rc = cifs_ioctl_set_compression(xid, filep, tcon, + pSMBFile, + compression_state); + if (rc == 0) { + spin_lock(&inode->i_lock); + if (enable_compression) + CIFS_I(inode)->cifsAttrs |= + FILE_ATTRIBUTE_COMPRESSED; + else + CIFS_I(inode)->cifsAttrs &= + ~FILE_ATTRIBUTE_COMPRESSED; + spin_unlock(&inode->i_lock); + } + cifs_dbg(FYI, "set compress flag rc %d\n", rc); + break; + case CIFS_IOC_COPYCHUNK_FILE: + rc = cifs_ioctl_copychunk(xid, filep, arg); + break; + case CIFS_QUERY_INFO: + rc = cifs_ioctl_query_info(xid, filep, arg); + break; + case CIFS_IOC_SET_INTEGRITY: + if (pSMBFile == NULL) + break; + tcon = tlink_tcon(pSMBFile->tlink); + if (tcon->ses->server->ops->set_integrity) + rc = tcon->ses->server->ops->set_integrity(xid, + tcon, pSMBFile); + else + rc = -EOPNOTSUPP; + break; + case CIFS_IOC_GET_MNT_INFO: + if (pSMBFile == NULL) + break; + tcon = tlink_tcon(pSMBFile->tlink); + rc = smb_mnt_get_fsinfo(xid, tcon, (void __user *)arg); + break; + case CIFS_IOC_GET_TCON_INFO: + cifs_sb = CIFS_SB(inode->i_sb); + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) { + rc = PTR_ERR(tlink); + break; + } + tcon = tlink_tcon(tlink); + rc = smb_mnt_get_tcon_info(tcon, (void __user *)arg); + cifs_put_tlink(tlink); + break; + case CIFS_ENUMERATE_SNAPSHOTS: + if (pSMBFile == NULL) + break; + if (arg == 0) { + rc = -EINVAL; + goto cifs_ioc_exit; + } + tcon = tlink_tcon(pSMBFile->tlink); + if (tcon->ses->server->ops->enum_snapshots) + rc = tcon->ses->server->ops->enum_snapshots(xid, tcon, + pSMBFile, (void __user *)arg); + else + rc = -EOPNOTSUPP; + break; + case CIFS_DUMP_KEY: + /* + * Dump encryption keys. This is an old ioctl that only + * handles AES-128-{CCM,GCM}. + */ + if (!capable(CAP_SYS_ADMIN)) { + rc = -EACCES; + break; + } + + cifs_sb = CIFS_SB(inode->i_sb); + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) { + rc = PTR_ERR(tlink); + break; + } + tcon = tlink_tcon(tlink); + if (!smb3_encryption_required(tcon)) { + rc = -EOPNOTSUPP; + cifs_put_tlink(tlink); + break; + } + pkey_inf.cipher_type = + le16_to_cpu(tcon->ses->server->cipher_type); + pkey_inf.Suid = tcon->ses->Suid; + memcpy(pkey_inf.auth_key, tcon->ses->auth_key.response, + SMB2_NTLMV2_SESSKEY_SIZE); + memcpy(pkey_inf.smb3decryptionkey, + tcon->ses->smb3decryptionkey, SMB3_SIGN_KEY_SIZE); + memcpy(pkey_inf.smb3encryptionkey, + tcon->ses->smb3encryptionkey, SMB3_SIGN_KEY_SIZE); + if (copy_to_user((void __user *)arg, &pkey_inf, + sizeof(struct smb3_key_debug_info))) + rc = -EFAULT; + else + rc = 0; + cifs_put_tlink(tlink); + break; + case CIFS_DUMP_FULL_KEY: + /* + * Dump encryption keys (handles any key sizes) + */ + if (pSMBFile == NULL) + break; + if (!capable(CAP_SYS_ADMIN)) { + rc = -EACCES; + break; + } + cifs_sb = CIFS_SB(inode->i_sb); + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) { + rc = PTR_ERR(tlink); + break; + } + + tcon = tlink_tcon(tlink); + rc = cifs_dump_full_key(tcon, (void __user *)arg); + cifs_put_tlink(tlink); + break; + case CIFS_IOC_NOTIFY: + if (!S_ISDIR(inode->i_mode)) { + /* Notify can only be done on directories */ + rc = -EOPNOTSUPP; + break; + } + cifs_sb = CIFS_SB(inode->i_sb); + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) { + rc = PTR_ERR(tlink); + break; + } + tcon = tlink_tcon(tlink); + if (tcon && tcon->ses->server->ops->notify) { + rc = tcon->ses->server->ops->notify(xid, + filep, (void __user *)arg, + false /* no ret data */); + cifs_dbg(FYI, "ioctl notify rc %d\n", rc); + } else + rc = -EOPNOTSUPP; + cifs_put_tlink(tlink); + break; + case CIFS_IOC_NOTIFY_INFO: + if (!S_ISDIR(inode->i_mode)) { + /* Notify can only be done on directories */ + rc = -EOPNOTSUPP; + break; + } + cifs_sb = CIFS_SB(inode->i_sb); + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) { + rc = PTR_ERR(tlink); + break; + } + tcon = tlink_tcon(tlink); + if (tcon && tcon->ses->server->ops->notify) { + rc = tcon->ses->server->ops->notify(xid, + filep, (void __user *)arg, + true /* return details */); + cifs_dbg(FYI, "ioctl notify info rc %d\n", rc); + } else + rc = -EOPNOTSUPP; + cifs_put_tlink(tlink); + break; + case CIFS_IOC_SHUTDOWN: + rc = cifs_shutdown(inode->i_sb, arg); + break; + default: + cifs_dbg(FYI, "unsupported ioctl\n"); + trace_smb3_unsupported_ioctl(xid, + pSMBFile ? pSMBFile->fid.persistent_fid : 0, + command); + break; + } +cifs_ioc_exit: + free_xid(xid); + return rc; +} diff --git a/fs/smb/client/link.c b/fs/smb/client/link.c new file mode 100644 index 000000000..8d5d6aca7 --- /dev/null +++ b/fs/smb/client/link.c @@ -0,0 +1,642 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * Copyright (C) International Business Machines Corp., 2002,2008 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ +#include +#include +#include +#include +#include +#include "cifsfs.h" +#include "cifsglob.h" +#include "cifsproto.h" +#include "cifs_debug.h" +#include "cifs_fs_sb.h" +#include "cifs_unicode.h" +#include "smb2proto.h" +#include "cifs_ioctl.h" +#include "fs_context.h" +#include "reparse.h" + +/* + * M-F Symlink Functions - Begin + */ + +#define CIFS_MF_SYMLINK_LEN_OFFSET (4+1) +#define CIFS_MF_SYMLINK_MD5_OFFSET (CIFS_MF_SYMLINK_LEN_OFFSET+(4+1)) +#define CIFS_MF_SYMLINK_LINK_OFFSET (CIFS_MF_SYMLINK_MD5_OFFSET+(32+1)) +#define CIFS_MF_SYMLINK_LINK_MAXLEN (1024) +#define CIFS_MF_SYMLINK_FILE_SIZE \ + (CIFS_MF_SYMLINK_LINK_OFFSET + CIFS_MF_SYMLINK_LINK_MAXLEN) + +#define CIFS_MF_SYMLINK_LEN_FORMAT "XSym\n%04u\n" +#define CIFS_MF_SYMLINK_MD5_FORMAT "%16phN\n" +#define CIFS_MF_SYMLINK_MD5_ARGS(md5_hash) md5_hash + +static int +parse_mf_symlink(const u8 *buf, unsigned int buf_len, unsigned int *_link_len, + char **_link_str) +{ + int rc; + unsigned int link_len; + const char *md5_str1; + const char *link_str; + u8 md5_hash[16]; + char md5_str2[34]; + + if (buf_len != CIFS_MF_SYMLINK_FILE_SIZE) + return -EINVAL; + + md5_str1 = (const char *)&buf[CIFS_MF_SYMLINK_MD5_OFFSET]; + link_str = (const char *)&buf[CIFS_MF_SYMLINK_LINK_OFFSET]; + + rc = sscanf(buf, CIFS_MF_SYMLINK_LEN_FORMAT, &link_len); + if (rc != 1) + return -EINVAL; + + if (link_len > CIFS_MF_SYMLINK_LINK_MAXLEN) + return -EINVAL; + + md5(link_str, link_len, md5_hash); + + scnprintf(md5_str2, sizeof(md5_str2), + CIFS_MF_SYMLINK_MD5_FORMAT, + CIFS_MF_SYMLINK_MD5_ARGS(md5_hash)); + + if (strncmp(md5_str1, md5_str2, 17) != 0) + return -EINVAL; + + if (_link_str) { + *_link_str = kstrndup(link_str, link_len, GFP_KERNEL); + if (!*_link_str) + return -ENOMEM; + } + + *_link_len = link_len; + return 0; +} + +static int +format_mf_symlink(u8 *buf, unsigned int buf_len, const char *link_str) +{ + unsigned int link_len; + unsigned int ofs; + u8 md5_hash[16]; + + if (buf_len != CIFS_MF_SYMLINK_FILE_SIZE) + return -EINVAL; + + link_len = strlen(link_str); + + if (link_len > CIFS_MF_SYMLINK_LINK_MAXLEN) + return -ENAMETOOLONG; + + md5(link_str, link_len, md5_hash); + + scnprintf(buf, buf_len, + CIFS_MF_SYMLINK_LEN_FORMAT CIFS_MF_SYMLINK_MD5_FORMAT, + link_len, + CIFS_MF_SYMLINK_MD5_ARGS(md5_hash)); + + ofs = CIFS_MF_SYMLINK_LINK_OFFSET; + memcpy(buf + ofs, link_str, link_len); + + ofs += link_len; + if (ofs < CIFS_MF_SYMLINK_FILE_SIZE) { + buf[ofs] = '\n'; + ofs++; + } + + while (ofs < CIFS_MF_SYMLINK_FILE_SIZE) { + buf[ofs] = ' '; + ofs++; + } + + return 0; +} + +bool +couldbe_mf_symlink(const struct cifs_fattr *fattr) +{ + if (!S_ISREG(fattr->cf_mode)) + /* it's not a symlink */ + return false; + + if (fattr->cf_eof != CIFS_MF_SYMLINK_FILE_SIZE) + /* it's not a symlink */ + return false; + + return true; +} + +static int +create_mf_symlink(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, const char *fromName, + const char *toName) +{ + int rc; + u8 *buf; + unsigned int bytes_written = 0; + + buf = kmalloc(CIFS_MF_SYMLINK_FILE_SIZE, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + rc = format_mf_symlink(buf, CIFS_MF_SYMLINK_FILE_SIZE, toName); + if (rc) + goto out; + + if (tcon->ses->server->ops->create_mf_symlink) + rc = tcon->ses->server->ops->create_mf_symlink(xid, tcon, + cifs_sb, fromName, buf, &bytes_written); + else + rc = -EOPNOTSUPP; + + if (rc) + goto out; + + if (bytes_written != CIFS_MF_SYMLINK_FILE_SIZE) + rc = smb_EIO2(smb_eio_trace_symlink_file_size, + bytes_written, CIFS_MF_SYMLINK_FILE_SIZE); +out: + kfree(buf); + return rc; +} + +int +check_mf_symlink(unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, struct cifs_fattr *fattr, + const unsigned char *path) +{ + int rc; + u8 *buf = NULL; + unsigned int link_len = 0; + unsigned int bytes_read = 0; + char *symlink = NULL; + + if (!couldbe_mf_symlink(fattr)) + /* it's not a symlink */ + return 0; + + buf = kmalloc(CIFS_MF_SYMLINK_FILE_SIZE, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + if (tcon->ses->server->ops->query_mf_symlink) + rc = tcon->ses->server->ops->query_mf_symlink(xid, tcon, + cifs_sb, path, buf, &bytes_read); + else + rc = -ENOSYS; + + if (rc) + goto out; + + if (bytes_read == 0) /* not a symlink */ + goto out; + + rc = parse_mf_symlink(buf, bytes_read, &link_len, &symlink); + if (rc == -EINVAL) { + /* it's not a symlink */ + rc = 0; + goto out; + } + + if (rc != 0) + goto out; + + /* it is a symlink */ + fattr->cf_eof = link_len; + fattr->cf_mode &= ~S_IFMT; + fattr->cf_mode |= S_IFLNK | S_IRWXU | S_IRWXG | S_IRWXO; + fattr->cf_dtype = DT_LNK; + fattr->cf_symlink_target = symlink; +out: + kfree(buf); + return rc; +} + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY +/* + * SMB 1.0 Protocol specific functions + */ + +int +cifs_query_mf_symlink(unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, const unsigned char *path, + char *pbuf, unsigned int *pbytes_read) +{ + int rc; + int oplock = 0; + struct cifs_fid fid; + struct cifs_open_parms oparms; + struct cifs_io_parms io_parms = {0}; + int buf_type = CIFS_NO_BUFFER; + struct cifs_open_info_data query_data = {}; + + oparms = (struct cifs_open_parms) { + .tcon = tcon, + .cifs_sb = cifs_sb, + .desired_access = GENERIC_READ, + .create_options = cifs_create_options(cifs_sb, CREATE_NOT_DIR), + .disposition = FILE_OPEN, + .path = path, + .fid = &fid, + }; + + rc = tcon->ses->server->ops->open(xid, &oparms, &oplock, &query_data); + if (rc) + return rc; + + if (query_data.fi.EndOfFile != cpu_to_le64(CIFS_MF_SYMLINK_FILE_SIZE)) { + rc = -ENOENT; + /* it's not a symlink */ + goto out; + } + + io_parms.netfid = fid.netfid; + io_parms.pid = current->tgid; + io_parms.tcon = tcon; + io_parms.offset = 0; + io_parms.length = CIFS_MF_SYMLINK_FILE_SIZE; + + rc = CIFSSMBRead(xid, &io_parms, pbytes_read, &pbuf, &buf_type); +out: + CIFSSMBClose(xid, tcon, fid.netfid); + return rc; +} + +int +cifs_create_mf_symlink(unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, const unsigned char *path, + char *pbuf, unsigned int *pbytes_written) +{ + int rc; + int oplock = 0; + struct cifs_fid fid; + struct cifs_open_parms oparms; + struct cifs_io_parms io_parms = {0}; + + oparms = (struct cifs_open_parms) { + .tcon = tcon, + .cifs_sb = cifs_sb, + .desired_access = GENERIC_WRITE, + .create_options = cifs_create_options(cifs_sb, CREATE_NOT_DIR), + .disposition = FILE_CREATE, + .path = path, + .fid = &fid, + }; + + rc = tcon->ses->server->ops->open(xid, &oparms, &oplock, NULL); + if (rc) + return rc; + + io_parms.netfid = fid.netfid; + io_parms.pid = current->tgid; + io_parms.tcon = tcon; + io_parms.offset = 0; + io_parms.length = CIFS_MF_SYMLINK_FILE_SIZE; + + rc = CIFSSMBWrite(xid, &io_parms, pbytes_written, pbuf); + CIFSSMBClose(xid, tcon, fid.netfid); + return rc; +} +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + +/* + * SMB 2.1/SMB3 Protocol specific functions + */ +int +smb3_query_mf_symlink(unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, const unsigned char *path, + char *pbuf, unsigned int *pbytes_read) +{ + int rc; + struct cifs_fid fid; + struct cifs_open_parms oparms; + struct cifs_io_parms io_parms = {0}; + int buf_type = CIFS_NO_BUFFER; + __le16 *utf16_path; + __u8 oplock = SMB2_OPLOCK_LEVEL_NONE; + struct cifs_open_info_data data = {}; + + oparms = (struct cifs_open_parms) { + .tcon = tcon, + .cifs_sb = cifs_sb, + .path = path, + .desired_access = GENERIC_READ, + .create_options = cifs_create_options(cifs_sb, CREATE_NOT_DIR), + .disposition = FILE_OPEN, + .fid = &fid, + }; + + utf16_path = cifs_convert_path_to_utf16(path, cifs_sb); + if (utf16_path == NULL) + return -ENOMEM; + + rc = SMB2_open(xid, &oparms, utf16_path, &oplock, &data, NULL, + NULL, NULL); + if (rc) + goto qmf_out_open_fail; + + if (data.fi.EndOfFile != cpu_to_le64(CIFS_MF_SYMLINK_FILE_SIZE)) { + /* it's not a symlink */ + rc = -ENOENT; /* Is there a better rc to return? */ + goto qmf_out; + } + + io_parms.netfid = fid.netfid; + io_parms.pid = current->tgid; + io_parms.tcon = tcon; + io_parms.offset = 0; + io_parms.length = CIFS_MF_SYMLINK_FILE_SIZE; + io_parms.persistent_fid = fid.persistent_fid; + io_parms.volatile_fid = fid.volatile_fid; + rc = SMB2_read(xid, &io_parms, pbytes_read, &pbuf, &buf_type); +qmf_out: + SMB2_close(xid, tcon, fid.persistent_fid, fid.volatile_fid); +qmf_out_open_fail: + kfree(utf16_path); + return rc; +} + +int +smb3_create_mf_symlink(unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, const unsigned char *path, + char *pbuf, unsigned int *pbytes_written) +{ + int rc; + struct cifs_fid fid; + struct cifs_open_parms oparms; + struct cifs_io_parms io_parms = {0}; + __le16 *utf16_path; + __u8 oplock = SMB2_OPLOCK_LEVEL_NONE; + struct kvec iov[2]; + + cifs_dbg(FYI, "%s: path: %s\n", __func__, path); + + utf16_path = cifs_convert_path_to_utf16(path, cifs_sb); + if (!utf16_path) + return -ENOMEM; + + oparms = (struct cifs_open_parms) { + .tcon = tcon, + .cifs_sb = cifs_sb, + .path = path, + .desired_access = GENERIC_WRITE, + .create_options = cifs_create_options(cifs_sb, CREATE_NOT_DIR), + .disposition = FILE_CREATE, + .fid = &fid, + .mode = 0644, + }; + + rc = SMB2_open(xid, &oparms, utf16_path, &oplock, NULL, NULL, + NULL, NULL); + if (rc) { + kfree(utf16_path); + return rc; + } + + io_parms.netfid = fid.netfid; + io_parms.pid = current->tgid; + io_parms.tcon = tcon; + io_parms.offset = 0; + io_parms.length = CIFS_MF_SYMLINK_FILE_SIZE; + io_parms.persistent_fid = fid.persistent_fid; + io_parms.volatile_fid = fid.volatile_fid; + + /* iov[0] is reserved for smb header */ + iov[1].iov_base = pbuf; + iov[1].iov_len = CIFS_MF_SYMLINK_FILE_SIZE; + + rc = SMB2_write(xid, &io_parms, pbytes_written, iov, 1); + + /* Make sure we wrote all of the symlink data */ + if ((rc == 0) && (*pbytes_written != CIFS_MF_SYMLINK_FILE_SIZE)) + rc = smb_EIO2(smb_eio_trace_short_symlink_write, + *pbytes_written, CIFS_MF_SYMLINK_FILE_SIZE); + + SMB2_close(xid, tcon, fid.persistent_fid, fid.volatile_fid); + + kfree(utf16_path); + return rc; +} + +/* + * M-F Symlink Functions - End + */ + +int +cifs_hardlink(struct dentry *old_file, struct inode *inode, + struct dentry *direntry) +{ + int rc = -EACCES; + unsigned int xid; + const char *from_name, *to_name; + void *page1, *page2; + struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); + struct tcon_link *tlink; + struct cifs_tcon *tcon; + struct TCP_Server_Info *server; + struct cifsInodeInfo *cifsInode; + + if (unlikely(cifs_forced_shutdown(cifs_sb))) + return smb_EIO(smb_eio_trace_forced_shutdown); + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) + return PTR_ERR(tlink); + tcon = tlink_tcon(tlink); + + xid = get_xid(); + page1 = alloc_dentry_path(); + page2 = alloc_dentry_path(); + + from_name = build_path_from_dentry(old_file, page1); + if (IS_ERR(from_name)) { + rc = PTR_ERR(from_name); + goto cifs_hl_exit; + } + to_name = build_path_from_dentry(direntry, page2); + if (IS_ERR(to_name)) { + rc = PTR_ERR(to_name); + goto cifs_hl_exit; + } + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + if (tcon->unix_ext) + rc = CIFSUnixCreateHardLink(xid, tcon, from_name, to_name, + cifs_sb->local_nls, + cifs_remap(cifs_sb)); + else { +#else + { +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + server = tcon->ses->server; + if (!server->ops->create_hardlink) { + rc = -ENOSYS; + goto cifs_hl_exit; + } + rc = server->ops->create_hardlink(xid, tcon, old_file, + from_name, to_name, cifs_sb); + if ((rc == -EIO) || (rc == -EINVAL)) + rc = -EOPNOTSUPP; + } + + d_drop(direntry); /* force new lookup from server of target */ + + /* + * if source file is cached (oplocked) revalidate will not go to server + * until the file is closed or oplock broken so update nlinks locally + */ + if (d_really_is_positive(old_file)) { + cifsInode = CIFS_I(d_inode(old_file)); + if (rc == 0) { + clear_bit(CIFS_INO_TMPFILE, &cifsInode->flags); + spin_lock(&d_inode(old_file)->i_lock); + inc_nlink(d_inode(old_file)); + spin_unlock(&d_inode(old_file)->i_lock); + + /* + * parent dir timestamps will update from srv within a + * second, would it really be worth it to set the parent + * dir cifs inode time to zero to force revalidate + * (faster) for it too? + */ + } + /* + * if not oplocked will force revalidate to get info on source + * file from srv. Note Samba server prior to 4.2 has bug - + * not updating src file ctime on hardlinks but Windows servers + * handle it properly + */ + cifsInode->time = 0; + + /* + * Will update parent dir timestamps from srv within a second. + * Would it really be worth it to set the parent dir (cifs + * inode) time field to zero to force revalidate on parent + * directory faster ie + * + * CIFS_I(inode)->time = 0; + */ + } + +cifs_hl_exit: + free_dentry_path(page1); + free_dentry_path(page2); + free_xid(xid); + cifs_put_tlink(tlink); + return rc; +} + +int +cifs_symlink(struct mnt_idmap *idmap, struct inode *inode, + struct dentry *direntry, const char *symname) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(inode); + struct inode *newinode = NULL; + struct tcon_link *tlink; + struct cifs_tcon *pTcon; + const char *full_path; + int rc = -EOPNOTSUPP; + unsigned int sbflags; + unsigned int xid; + void *page; + + if (unlikely(cifs_forced_shutdown(cifs_sb))) + return smb_EIO(smb_eio_trace_forced_shutdown); + + page = alloc_dentry_path(); + if (!page) + return -ENOMEM; + + xid = get_xid(); + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) { + rc = PTR_ERR(tlink); + /* BB could be clearer if skipped put_tlink on error here, but harmless */ + goto symlink_exit; + } + pTcon = tlink_tcon(tlink); + + full_path = build_path_from_dentry(direntry, page); + if (IS_ERR(full_path)) { + rc = PTR_ERR(full_path); + goto symlink_exit; + } + + cifs_dbg(FYI, "Full path: %s\n", full_path); + cifs_dbg(FYI, "symname is %s\n", symname); + + /* BB what if DFS and this volume is on different share? BB */ + sbflags = cifs_sb_flags(cifs_sb); + rc = -EOPNOTSUPP; + switch (cifs_symlink_type(cifs_sb)) { + case CIFS_SYMLINK_TYPE_UNIX: +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + if (pTcon->unix_ext) { + rc = CIFSUnixCreateSymLink(xid, pTcon, full_path, + symname, + cifs_sb->local_nls, + cifs_remap(cifs_sb)); + } +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + break; + + case CIFS_SYMLINK_TYPE_MFSYMLINKS: + if (sbflags & CIFS_MOUNT_MF_SYMLINKS) { + rc = create_mf_symlink(xid, pTcon, cifs_sb, + full_path, symname); + } + break; + + case CIFS_SYMLINK_TYPE_SFU: + if (sbflags & CIFS_MOUNT_UNX_EMUL) { + rc = __cifs_sfu_make_node(xid, inode, direntry, pTcon, + full_path, S_IFLNK, + 0, symname); + } + break; + + case CIFS_SYMLINK_TYPE_NATIVE: + case CIFS_SYMLINK_TYPE_NFS: + case CIFS_SYMLINK_TYPE_WSL: + if (CIFS_REPARSE_SUPPORT(pTcon)) { + rc = create_reparse_symlink(xid, inode, direntry, pTcon, + full_path, symname); + goto symlink_exit; + } + break; + default: + break; + } + + if (rc == 0) { + if (pTcon->posix_extensions) { + rc = smb311_posix_get_inode_info(&newinode, full_path, + NULL, inode->i_sb, xid); + } else if (pTcon->unix_ext) { + rc = cifs_get_inode_info_unix(&newinode, full_path, + inode->i_sb, xid); + } else { + rc = cifs_get_inode_info(&newinode, full_path, NULL, + inode->i_sb, xid, NULL); + } + + if (rc != 0) { + cifs_dbg(FYI, "Create symlink ok, getinodeinfo fail rc = %d\n", + rc); + } else { + d_instantiate(direntry, newinode); + } + } +symlink_exit: + free_dentry_path(page); + cifs_put_tlink(tlink); + free_xid(xid); + return rc; +} diff --git a/fs/smb/client/misc.c b/fs/smb/client/misc.c new file mode 100644 index 000000000..46e1382e8 --- /dev/null +++ b/fs/smb/client/misc.c @@ -0,0 +1,1112 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * Copyright (C) International Business Machines Corp., 2002,2008 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ + +#include +#include +#include +#include +#include "cifsglob.h" +#include "cifsproto.h" +#include "cifs_debug.h" +#include "smberr.h" +#include "nterr.h" +#include "cifs_unicode.h" +#include "smb2pdu.h" +#include "smb2proto.h" +#include "smb1proto.h" +#include "cifsfs.h" +#ifdef CONFIG_CIFS_DFS_UPCALL +#include "dns_resolve.h" +#include "dfs_cache.h" +#include "dfs.h" +#endif +#include "fs_context.h" +#include "cached_dir.h" + +struct tcon_list { + struct list_head entry; + struct cifs_tcon *tcon; +}; + +/* The xid serves as a useful identifier for each incoming vfs request, + in a similar way to the mid which is useful to track each sent smb, + and CurrentXid can also provide a running counter (although it + will eventually wrap past zero) of the total vfs operations handled + since the cifs fs was mounted */ + +unsigned int +_get_xid(void) +{ + unsigned int xid; + + spin_lock(&GlobalMid_Lock); + GlobalTotalActiveXid++; + + /* keep high water mark for number of simultaneous ops in filesystem */ + if (GlobalTotalActiveXid > GlobalMaxActiveXid) + GlobalMaxActiveXid = GlobalTotalActiveXid; + if (GlobalTotalActiveXid > 65000) + cifs_dbg(FYI, "warning: more than 65000 requests active\n"); + xid = GlobalCurrentXid++; + spin_unlock(&GlobalMid_Lock); + return xid; +} + +void +_free_xid(unsigned int xid) +{ + spin_lock(&GlobalMid_Lock); + /* if (GlobalTotalActiveXid == 0) + BUG(); */ + GlobalTotalActiveXid--; + spin_unlock(&GlobalMid_Lock); +} + +struct cifs_ses * +sesInfoAlloc(void) +{ + struct cifs_ses *ret_buf; + + ret_buf = kzalloc_obj(struct cifs_ses); + if (ret_buf) { + atomic_inc(&sesInfoAllocCount); + spin_lock_init(&ret_buf->ses_lock); + ret_buf->ses_status = SES_NEW; + ++ret_buf->ses_count; + INIT_LIST_HEAD(&ret_buf->smb_ses_list); + INIT_LIST_HEAD(&ret_buf->tcon_list); + mutex_init(&ret_buf->session_mutex); + spin_lock_init(&ret_buf->iface_lock); + INIT_LIST_HEAD(&ret_buf->iface_list); + spin_lock_init(&ret_buf->chan_lock); + } + return ret_buf; +} + +void +sesInfoFree(struct cifs_ses *buf_to_free) +{ + struct cifs_server_iface *iface = NULL, *niface = NULL; + + if (buf_to_free == NULL) { + cifs_dbg(FYI, "Null buffer passed to sesInfoFree\n"); + return; + } + + unload_nls(buf_to_free->local_nls); + atomic_dec(&sesInfoAllocCount); + kfree(buf_to_free->serverOS); + kfree(buf_to_free->serverDomain); + kfree(buf_to_free->serverNOS); + kfree_sensitive(buf_to_free->password); + kfree_sensitive(buf_to_free->password2); + kfree(buf_to_free->user_name); + kfree(buf_to_free->domainName); + kfree(buf_to_free->dns_dom); + kfree_sensitive(buf_to_free->auth_key.response); + spin_lock(&buf_to_free->iface_lock); + list_for_each_entry_safe(iface, niface, &buf_to_free->iface_list, + iface_head) + kref_put(&iface->refcount, release_iface); + spin_unlock(&buf_to_free->iface_lock); + kfree_sensitive(buf_to_free); +} + +struct cifs_tcon * +tcon_info_alloc(bool dir_leases_enabled, enum smb3_tcon_ref_trace trace) +{ + struct cifs_tcon *ret_buf; + static atomic_t tcon_debug_id; + + ret_buf = kzalloc_obj(*ret_buf); + if (!ret_buf) + return NULL; + + if (dir_leases_enabled == true) { + ret_buf->cfids = init_cached_dirs(); + if (!ret_buf->cfids) { + kfree(ret_buf); + return NULL; + } + } + /* else ret_buf->cfids is already set to NULL above */ + + atomic_inc(&tconInfoAllocCount); + ret_buf->status = TID_NEW; + ret_buf->debug_id = atomic_inc_return(&tcon_debug_id); + ret_buf->tc_count = 1; + spin_lock_init(&ret_buf->tc_lock); + INIT_LIST_HEAD(&ret_buf->openFileList); + INIT_LIST_HEAD(&ret_buf->tcon_list); + INIT_LIST_HEAD(&ret_buf->cifs_sb_list); + spin_lock_init(&ret_buf->open_file_lock); + spin_lock_init(&ret_buf->stat_lock); + spin_lock_init(&ret_buf->sb_list_lock); + atomic_set(&ret_buf->num_local_opens, 0); + atomic_set(&ret_buf->num_remote_opens, 0); + ret_buf->stats_from_time = ktime_get_real_seconds(); +#ifdef CONFIG_CIFS_FSCACHE + mutex_init(&ret_buf->fscache_lock); +#endif + trace_smb3_tcon_ref(ret_buf->debug_id, ret_buf->tc_count, trace); +#ifdef CONFIG_CIFS_DFS_UPCALL + INIT_LIST_HEAD(&ret_buf->dfs_ses_list); +#endif + INIT_LIST_HEAD(&ret_buf->pending_opens); + INIT_DELAYED_WORK(&ret_buf->query_interfaces, + smb2_query_server_interfaces); +#ifdef CONFIG_CIFS_DFS_UPCALL + INIT_DELAYED_WORK(&ret_buf->dfs_cache_work, dfs_cache_refresh); +#endif + + return ret_buf; +} + +void +tconInfoFree(struct cifs_tcon *tcon, enum smb3_tcon_ref_trace trace) +{ + if (tcon == NULL) { + cifs_dbg(FYI, "Null buffer passed to tconInfoFree\n"); + return; + } + trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, trace); + free_cached_dirs(tcon->cfids); + atomic_dec(&tconInfoAllocCount); + kfree(tcon->nativeFileSystem); + kfree_sensitive(tcon->password); + kfree(tcon->origin_fullpath); + kfree(tcon); +} + +void * +cifs_buf_get(void) +{ + void *ret_buf = NULL; + /* + * SMB2 header is bigger than CIFS one - no problems to clean some + * more bytes for CIFS. + */ + size_t buf_size = sizeof(struct smb2_hdr); + + /* + * We could use negotiated size instead of max_msgsize - + * but it may be more efficient to always alloc same size + * albeit slightly larger than necessary and maxbuffersize + * defaults to this and can not be bigger. + */ + ret_buf = mempool_alloc(cifs_req_poolp, GFP_NOFS); + + /* clear the first few header bytes */ + /* for most paths, more is cleared in header_assemble */ + memset(ret_buf, 0, buf_size + 3); + atomic_inc(&buf_alloc_count); +#ifdef CONFIG_CIFS_STATS2 + atomic_inc(&total_buf_alloc_count); +#endif /* CONFIG_CIFS_STATS2 */ + + return ret_buf; +} + +void +cifs_buf_release(void *buf_to_free) +{ + if (buf_to_free == NULL) { + /* cifs_dbg(FYI, "Null buffer passed to cifs_buf_release\n");*/ + return; + } + mempool_free(buf_to_free, cifs_req_poolp); + + atomic_dec(&buf_alloc_count); + return; +} + +void * +cifs_small_buf_get(void) +{ + void *ret_buf = NULL; + +/* We could use negotiated size instead of max_msgsize - + but it may be more efficient to always alloc same size + albeit slightly larger than necessary and maxbuffersize + defaults to this and can not be bigger */ + ret_buf = mempool_alloc(cifs_sm_req_poolp, GFP_NOFS); + /* No need to clear memory here, cleared in header assemble */ + atomic_inc(&small_buf_alloc_count); +#ifdef CONFIG_CIFS_STATS2 + atomic_inc(&total_small_buf_alloc_count); +#endif /* CONFIG_CIFS_STATS2 */ + + return ret_buf; +} + +void +cifs_small_buf_release(void *buf_to_free) +{ + + if (buf_to_free == NULL) { + cifs_dbg(FYI, "Null buffer passed to cifs_small_buf_release\n"); + return; + } + mempool_free(buf_to_free, cifs_sm_req_poolp); + + atomic_dec(&small_buf_alloc_count); + return; +} + +void +free_rsp_buf(int resp_buftype, void *rsp) +{ + if (resp_buftype == CIFS_SMALL_BUFFER) + cifs_small_buf_release(rsp); + else if (resp_buftype == CIFS_LARGE_BUFFER) + cifs_buf_release(rsp); +} + +void +dump_smb(void *buf, int smb_buf_length) +{ + if (traceSMB == 0) + return; + + print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_NONE, 8, 2, buf, + smb_buf_length, true); +} + +void +cifs_autodisable_serverino(struct cifs_sb_info *cifs_sb, const char *reason, int rc) +{ + unsigned int sbflags = cifs_sb_flags(cifs_sb); + + if (sbflags & CIFS_MOUNT_SERVER_INUM) { + struct cifs_tcon *tcon = NULL; + + if (cifs_sb->master_tlink) + tcon = cifs_sb_master_tcon(cifs_sb); + + atomic_andnot(CIFS_MOUNT_SERVER_INUM, &cifs_sb->mnt_cifs_flags); + cifs_sb->mnt_cifs_serverino_autodisabled = true; + if (rc) + cifs_dbg(VFS, "%s: %d\n", reason, rc); + else + cifs_dbg(VFS, "%s\n", reason); + cifs_dbg(VFS, "Autodisabling the use of server inode numbers on %s\n", + tcon ? tcon->tree_name : "new server"); + cifs_dbg(VFS, "The server doesn't seem to support them properly or the files might be on different servers (DFS)\n"); + cifs_dbg(VFS, "Hardlinks will not be recognized on this mount. Consider mounting with the \"noserverino\" option to silence this message.\n"); + + } +} + +void cifs_set_oplock_level(struct cifsInodeInfo *cinode, __u32 oplock) +{ + oplock &= 0xF; + + if (oplock == OPLOCK_EXCLUSIVE) { + cinode->oplock = CIFS_CACHE_WRITE_FLG | CIFS_CACHE_READ_FLG; + cifs_dbg(FYI, "Exclusive Oplock granted on inode %p\n", + &cinode->netfs.inode); + } else if (oplock == OPLOCK_READ) { + cinode->oplock = CIFS_CACHE_READ_FLG; + cifs_dbg(FYI, "Level II Oplock granted on inode %p\n", + &cinode->netfs.inode); + } else + cinode->oplock = 0; +} + +/* + * We wait for oplock breaks to be processed before we attempt to perform + * writes. + */ +int cifs_get_writer(struct cifsInodeInfo *cinode) +{ + int rc; + +start: + rc = wait_on_bit(&cinode->flags, CIFS_INODE_PENDING_OPLOCK_BREAK, + TASK_KILLABLE); + if (rc) + return rc; + + spin_lock(&cinode->writers_lock); + if (!cinode->writers) + set_bit(CIFS_INODE_PENDING_WRITERS, &cinode->flags); + cinode->writers++; + /* Check to see if we have started servicing an oplock break */ + if (test_bit(CIFS_INODE_PENDING_OPLOCK_BREAK, &cinode->flags)) { + cinode->writers--; + if (cinode->writers == 0) { + clear_bit(CIFS_INODE_PENDING_WRITERS, &cinode->flags); + wake_up_bit(&cinode->flags, CIFS_INODE_PENDING_WRITERS); + } + spin_unlock(&cinode->writers_lock); + goto start; + } + spin_unlock(&cinode->writers_lock); + return 0; +} + +void cifs_put_writer(struct cifsInodeInfo *cinode) +{ + spin_lock(&cinode->writers_lock); + cinode->writers--; + if (cinode->writers == 0) { + clear_bit(CIFS_INODE_PENDING_WRITERS, &cinode->flags); + wake_up_bit(&cinode->flags, CIFS_INODE_PENDING_WRITERS); + } + spin_unlock(&cinode->writers_lock); +} + +/** + * cifs_queue_oplock_break - queue the oplock break handler for cfile + * @cfile: The file to break the oplock on + * + * This function is called from the demultiplex thread when it + * receives an oplock break for @cfile. + * + * Assumes the tcon->open_file_lock is held. + * Assumes cfile->file_info_lock is NOT held. + */ +void cifs_queue_oplock_break(struct cifsFileInfo *cfile) +{ + /* + * Bump the handle refcount now while we hold the + * open_file_lock to enforce the validity of it for the oplock + * break handler. The matching put is done at the end of the + * handler. + */ + cifsFileInfo_get(cfile); + + queue_work(cifsoplockd_wq, &cfile->oplock_break); +} + +void cifs_done_oplock_break(struct cifsInodeInfo *cinode) +{ + clear_bit(CIFS_INODE_PENDING_OPLOCK_BREAK, &cinode->flags); + wake_up_bit(&cinode->flags, CIFS_INODE_PENDING_OPLOCK_BREAK); +} + +bool +backup_cred(struct cifs_sb_info *cifs_sb) +{ + unsigned int sbflags = cifs_sb_flags(cifs_sb); + + if (sbflags & CIFS_MOUNT_CIFS_BACKUPUID) { + if (uid_eq(cifs_sb->ctx->backupuid, current_fsuid())) + return true; + } + if (sbflags & CIFS_MOUNT_CIFS_BACKUPGID) { + if (in_group_p(cifs_sb->ctx->backupgid)) + return true; + } + + return false; +} + +void +cifs_del_pending_open(struct cifs_pending_open *open) +{ + spin_lock(&tlink_tcon(open->tlink)->open_file_lock); + list_del(&open->olist); + spin_unlock(&tlink_tcon(open->tlink)->open_file_lock); +} + +void +cifs_add_pending_open_locked(struct cifs_fid *fid, struct tcon_link *tlink, + struct cifs_pending_open *open) +{ + memcpy(open->lease_key, fid->lease_key, SMB2_LEASE_KEY_SIZE); + open->oplock = CIFS_OPLOCK_NO_CHANGE; + open->tlink = tlink; + fid->pending_open = open; + list_add_tail(&open->olist, &tlink_tcon(tlink)->pending_opens); +} + +void +cifs_add_pending_open(struct cifs_fid *fid, struct tcon_link *tlink, + struct cifs_pending_open *open) +{ + spin_lock(&tlink_tcon(tlink)->open_file_lock); + cifs_add_pending_open_locked(fid, tlink, open); + spin_unlock(&tlink_tcon(open->tlink)->open_file_lock); +} + +/* + * Critical section which runs after acquiring deferred_lock. + * As there is no reference count on cifs_deferred_close, pdclose + * should not be used outside deferred_lock. + */ +bool +cifs_is_deferred_close(struct cifsFileInfo *cfile, struct cifs_deferred_close **pdclose) +{ + struct cifs_deferred_close *dclose; + + list_for_each_entry(dclose, &CIFS_I(d_inode(cfile->dentry))->deferred_closes, dlist) { + if ((dclose->netfid == cfile->fid.netfid) && + (dclose->persistent_fid == cfile->fid.persistent_fid) && + (dclose->volatile_fid == cfile->fid.volatile_fid)) { + *pdclose = dclose; + return true; + } + } + return false; +} + +/* + * Critical section which runs after acquiring deferred_lock. + */ +void +cifs_add_deferred_close(struct cifsFileInfo *cfile, struct cifs_deferred_close *dclose) +{ + bool is_deferred = false; + struct cifs_deferred_close *pdclose; + + is_deferred = cifs_is_deferred_close(cfile, &pdclose); + if (is_deferred) { + kfree(dclose); + return; + } + + dclose->tlink = cfile->tlink; + dclose->netfid = cfile->fid.netfid; + dclose->persistent_fid = cfile->fid.persistent_fid; + dclose->volatile_fid = cfile->fid.volatile_fid; + list_add_tail(&dclose->dlist, &CIFS_I(d_inode(cfile->dentry))->deferred_closes); +} + +/* + * Critical section which runs after acquiring deferred_lock. + */ +void +cifs_del_deferred_close(struct cifsFileInfo *cfile) +{ + bool is_deferred = false; + struct cifs_deferred_close *dclose; + + is_deferred = cifs_is_deferred_close(cfile, &dclose); + if (!is_deferred) + return; + list_del(&dclose->dlist); + kfree(dclose); +} + +void +cifs_close_deferred_file(struct cifsInodeInfo *cifs_inode) +{ + struct cifsFileInfo *cfile = NULL, *failed_cfile = NULL; + struct file_list *tmp_list, *tmp_next_list; + LIST_HEAD(file_head); + + if (cifs_inode == NULL) + return; + + spin_lock(&cifs_inode->open_file_lock); + list_for_each_entry(cfile, &cifs_inode->openFileList, flist) { + if (delayed_work_pending(&cfile->deferred)) { + if (cancel_delayed_work(&cfile->deferred)) { + spin_lock(&cifs_inode->deferred_lock); + cifs_del_deferred_close(cfile); + spin_unlock(&cifs_inode->deferred_lock); + + tmp_list = kmalloc_obj(struct file_list, + GFP_ATOMIC); + if (tmp_list == NULL) { + failed_cfile = cfile; + break; + } + tmp_list->cfile = cfile; + list_add_tail(&tmp_list->list, &file_head); + } + } + } + spin_unlock(&cifs_inode->open_file_lock); + + if (failed_cfile) + _cifsFileInfo_put(failed_cfile, false, false); + + list_for_each_entry_safe(tmp_list, tmp_next_list, &file_head, list) { + _cifsFileInfo_put(tmp_list->cfile, false, false); + list_del(&tmp_list->list); + kfree(tmp_list); + } +} + +void +cifs_close_all_deferred_files(struct cifs_tcon *tcon) +{ + struct cifsFileInfo *cfile, *failed_cfile = NULL; + struct file_list *tmp_list, *tmp_next_list; + LIST_HEAD(file_head); + + spin_lock(&tcon->open_file_lock); + list_for_each_entry(cfile, &tcon->openFileList, tlist) { + if (delayed_work_pending(&cfile->deferred)) { + if (cancel_delayed_work(&cfile->deferred)) { + spin_lock(&CIFS_I(d_inode(cfile->dentry))->deferred_lock); + cifs_del_deferred_close(cfile); + spin_unlock(&CIFS_I(d_inode(cfile->dentry))->deferred_lock); + + tmp_list = kmalloc_obj(struct file_list, + GFP_ATOMIC); + if (tmp_list == NULL) { + failed_cfile = cfile; + break; + } + tmp_list->cfile = cfile; + list_add_tail(&tmp_list->list, &file_head); + } + } + } + spin_unlock(&tcon->open_file_lock); + + if (failed_cfile) + _cifsFileInfo_put(failed_cfile, true, false); + + list_for_each_entry_safe(tmp_list, tmp_next_list, &file_head, list) { + _cifsFileInfo_put(tmp_list->cfile, true, false); + list_del(&tmp_list->list); + kfree(tmp_list); + } +} + +void cifs_close_all_deferred_files_sb(struct cifs_sb_info *cifs_sb) +{ + struct rb_root *root = &cifs_sb->tlink_tree; + struct rb_node *node; + struct cifs_tcon *tcon; + struct tcon_link *tlink; + struct tcon_list *tmp_list, *q; + LIST_HEAD(tcon_head); + + spin_lock(&cifs_sb->tlink_tree_lock); + for (node = rb_first(root); node; node = rb_next(node)) { + tlink = rb_entry(node, struct tcon_link, tl_rbnode); + tcon = tlink_tcon(tlink); + if (IS_ERR(tcon)) + continue; + tmp_list = kmalloc_obj(struct tcon_list, GFP_ATOMIC); + if (tmp_list == NULL) + break; + tmp_list->tcon = tcon; + /* Take a reference on tcon to prevent it from being freed */ + spin_lock(&tcon->tc_lock); + ++tcon->tc_count; + trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, + netfs_trace_tcon_ref_get_close_defer_files); + spin_unlock(&tcon->tc_lock); + list_add_tail(&tmp_list->entry, &tcon_head); + } + spin_unlock(&cifs_sb->tlink_tree_lock); + + list_for_each_entry_safe(tmp_list, q, &tcon_head, entry) { + cifs_close_all_deferred_files(tmp_list->tcon); + list_del(&tmp_list->entry); + cifs_put_tcon(tmp_list->tcon, netfs_trace_tcon_ref_put_close_defer_files); + kfree(tmp_list); + } +} + +void cifs_close_deferred_file_under_dentry(struct cifs_tcon *tcon, + struct dentry *dentry) +{ + struct file_list *tmp_list, *tmp_next_list; + struct cifsFileInfo *cfile, *failed_cfile = NULL; + LIST_HEAD(file_head); + + spin_lock(&tcon->open_file_lock); + list_for_each_entry(cfile, &tcon->openFileList, tlist) { + if ((cfile->dentry == dentry) && + delayed_work_pending(&cfile->deferred) && + cancel_delayed_work(&cfile->deferred)) { + spin_lock(&CIFS_I(d_inode(cfile->dentry))->deferred_lock); + cifs_del_deferred_close(cfile); + spin_unlock(&CIFS_I(d_inode(cfile->dentry))->deferred_lock); + + tmp_list = kmalloc_obj(struct file_list, GFP_ATOMIC); + if (tmp_list == NULL) { + failed_cfile = cfile; + break; + } + tmp_list->cfile = cfile; + list_add_tail(&tmp_list->list, &file_head); + } + } + spin_unlock(&tcon->open_file_lock); + + if (failed_cfile) + _cifsFileInfo_put(failed_cfile, true, false); + + list_for_each_entry_safe(tmp_list, tmp_next_list, &file_head, list) { + _cifsFileInfo_put(tmp_list->cfile, true, false); + list_del(&tmp_list->list); + kfree(tmp_list); + } +} + +/* + * If a dentry has been deleted, all corresponding open handles should know that + * so that we do not defer close them. + */ +void cifs_mark_open_handles_for_deleted_file(struct inode *inode, + const char *path) +{ + struct cifsFileInfo *cfile; + void *page; + const char *full_path; + struct cifsInodeInfo *cinode = CIFS_I(inode); + + page = alloc_dentry_path(); + spin_lock(&cinode->open_file_lock); + + /* + * note: we need to construct path from dentry and compare only if the + * inode has any hardlinks. When number of hardlinks is 1, we can just + * mark all open handles since they are going to be from the same file. + */ + if (inode->i_nlink > 1) { + list_for_each_entry(cfile, &cinode->openFileList, flist) { + full_path = build_path_from_dentry(cfile->dentry, page); + if (!IS_ERR(full_path) && strcmp(full_path, path) == 0) + cfile->status_file_deleted = true; + } + } else { + list_for_each_entry(cfile, &cinode->openFileList, flist) + cfile->status_file_deleted = true; + } + spin_unlock(&cinode->open_file_lock); + free_dentry_path(page); +} + +/* parses DFS referral V3 structure + * caller is responsible for freeing target_nodes + * returns: + * - on success - 0 + * - on failure - errno + */ +int +parse_dfs_referrals(struct get_dfs_referral_rsp *rsp, u32 rsp_size, + unsigned int *num_of_nodes, + struct dfs_info3_param **target_nodes, + const struct nls_table *nls_codepage, int remap, + const char *searchName, bool is_unicode) +{ + int i, rc = 0; + char *data_end; + struct dfs_referral_level_3 *ref; + unsigned int path_consumed; + size_t search_name_len; + + if (rsp_size < sizeof(*rsp)) { + cifs_dbg(VFS | ONCE, + "%s: header is malformed (size is %u, must be %zu)\n", + __func__, rsp_size, sizeof(*rsp)); + rc = -EINVAL; + goto parse_DFS_referrals_exit; + } + + *num_of_nodes = le16_to_cpu(rsp->NumberOfReferrals); + + if (*num_of_nodes < 1) { + cifs_dbg(VFS | ONCE, "%s: [path=%s] num_referrals must be at least > 0, but we got %d\n", + __func__, searchName, *num_of_nodes); + rc = -ENOENT; + goto parse_DFS_referrals_exit; + } + + if (sizeof(*rsp) + *num_of_nodes * sizeof(REFERRAL3) > rsp_size) { + cifs_dbg(VFS | ONCE, + "%s: malformed buffer (size is %u, must be at least %zu)\n", + __func__, rsp_size, + sizeof(*rsp) + *num_of_nodes * sizeof(REFERRAL3)); + rc = -EINVAL; + goto parse_DFS_referrals_exit; + } + + ref = (struct dfs_referral_level_3 *) &(rsp->referrals); + if (ref->VersionNumber != cpu_to_le16(3)) { + cifs_dbg(VFS, "Referrals of V%d version are not supported, should be V3\n", + le16_to_cpu(ref->VersionNumber)); + rc = -EINVAL; + goto parse_DFS_referrals_exit; + } + + /* get the upper boundary of the resp buffer */ + data_end = (char *)rsp + rsp_size; + + cifs_dbg(FYI, "num_referrals: %d dfs flags: 0x%x ...\n", + *num_of_nodes, le32_to_cpu(rsp->DFSFlags)); + + *target_nodes = kzalloc_objs(struct dfs_info3_param, *num_of_nodes); + if (*target_nodes == NULL) { + rc = -ENOMEM; + goto parse_DFS_referrals_exit; + } + search_name_len = strlen(searchName); + + /* collect necessary data from referrals */ + for (i = 0; i < *num_of_nodes; i++) { + char *temp; + int max_len; + struct dfs_info3_param *node = (*target_nodes)+i; + + node->flags = le32_to_cpu(rsp->DFSFlags); + path_consumed = le16_to_cpu(rsp->PathConsumed); + if (is_unicode) { + size_t search_name_utf16_len = search_name_len * 2 + 2; + __le16 *tmp; + + if (path_consumed > search_name_utf16_len) { + rc = -EINVAL; + goto parse_DFS_referrals_exit; + } + + tmp = kmalloc(search_name_utf16_len, GFP_KERNEL); + if (!tmp) { + rc = -ENOMEM; + goto parse_DFS_referrals_exit; + } + cifsConvertToUTF16((__le16 *)tmp, searchName, + PATH_MAX, nls_codepage, remap); + node->path_consumed = cifs_utf16_bytes(tmp, path_consumed, + nls_codepage); + kfree(tmp); + } else { + if (path_consumed > search_name_len) { + rc = -EINVAL; + goto parse_DFS_referrals_exit; + } + + node->path_consumed = path_consumed; + } + + node->server_type = le16_to_cpu(ref->ServerType); + node->ref_flag = le16_to_cpu(ref->ReferralEntryFlags); + + /* copy DfsPath */ + if (le16_to_cpu(ref->DfsPathOffset) > data_end - (char *)ref) { + rc = -EINVAL; + goto parse_DFS_referrals_exit; + } + temp = (char *)ref + le16_to_cpu(ref->DfsPathOffset); + max_len = data_end - temp; + node->path_name = cifs_strndup_from_utf16(temp, max_len, + is_unicode, nls_codepage); + if (!node->path_name) { + rc = -ENOMEM; + goto parse_DFS_referrals_exit; + } + + /* copy link target UNC */ + if (le16_to_cpu(ref->NetworkAddressOffset) > data_end - (char *)ref) { + rc = -EINVAL; + goto parse_DFS_referrals_exit; + } + temp = (char *)ref + le16_to_cpu(ref->NetworkAddressOffset); + max_len = data_end - temp; + node->node_name = cifs_strndup_from_utf16(temp, max_len, + is_unicode, nls_codepage); + if (!node->node_name) { + rc = -ENOMEM; + goto parse_DFS_referrals_exit; + } + + node->ttl = le32_to_cpu(ref->TimeToLive); + + ref++; + } + +parse_DFS_referrals_exit: + if (rc) { + free_dfs_info_array(*target_nodes, *num_of_nodes); + *target_nodes = NULL; + *num_of_nodes = 0; + } + return rc; +} + +void extract_unc_hostname(const char *unc, const char **h, size_t *len) +{ + const char *end; + + /* skip initial slashes */ + while (*unc && (*unc == '\\' || *unc == '/')) + unc++; + + end = unc; + + while (*end && !(*end == '\\' || *end == '/')) + end++; + + *h = unc; + *len = end - unc; +} + +/** + * copy_path_name - copy src path to dst, possibly truncating + * @dst: The destination buffer + * @src: The source name + * + * returns number of bytes written (including trailing nul) + */ +int copy_path_name(char *dst, const char *src) +{ + int name_len; + + /* + * PATH_MAX includes nul, so if strlen(src) >= PATH_MAX it + * will truncate and strlen(dst) will be PATH_MAX-1 + */ + name_len = strscpy(dst, src, PATH_MAX); + if (WARN_ON_ONCE(name_len < 0)) + name_len = PATH_MAX-1; + + /* we count the trailing nul */ + name_len++; + return name_len; +} + +struct super_cb_data { + void *data; + struct super_block *sb; +}; + +static void tcon_super_cb(struct super_block *sb, void *arg) +{ + struct super_cb_data *sd = arg; + struct cifs_sb_info *cifs_sb; + struct cifs_tcon *t1 = sd->data, *t2; + + if (sd->sb) + return; + + cifs_sb = CIFS_SB(sb); + t2 = cifs_sb_master_tcon(cifs_sb); + + spin_lock(&t2->tc_lock); + if ((t1->ses == t2->ses || + t1->ses->dfs_root_ses == t2->ses->dfs_root_ses) && + t1->ses->server == t2->ses->server && + t2->origin_fullpath && + dfs_src_pathname_equal(t2->origin_fullpath, t1->origin_fullpath)) + sd->sb = sb; + spin_unlock(&t2->tc_lock); +} + +static struct super_block *__cifs_get_super(void (*f)(struct super_block *, void *), + void *data) +{ + struct super_cb_data sd = { + .data = data, + .sb = NULL, + }; + struct file_system_type **fs_type = (struct file_system_type *[]) { + &cifs_fs_type, &smb3_fs_type, NULL, + }; + + for (; *fs_type; fs_type++) { + iterate_supers_type(*fs_type, f, &sd); + if (sd.sb) { + /* + * Grab an active reference in order to prevent automounts (DFS links) + * of expiring and then freeing up our cifs superblock pointer while + * we're doing failover. + */ + cifs_sb_active(sd.sb); + return sd.sb; + } + } + pr_warn_once("%s: could not find dfs superblock\n", __func__); + return ERR_PTR(-EINVAL); +} + +static void __cifs_put_super(struct super_block *sb) +{ + if (!IS_ERR_OR_NULL(sb)) + cifs_sb_deactive(sb); +} + +struct super_block *cifs_get_dfs_tcon_super(struct cifs_tcon *tcon) +{ + spin_lock(&tcon->tc_lock); + if (!tcon->origin_fullpath) { + spin_unlock(&tcon->tc_lock); + return ERR_PTR(-ENOENT); + } + spin_unlock(&tcon->tc_lock); + return __cifs_get_super(tcon_super_cb, tcon); +} + +void cifs_put_tcp_super(struct super_block *sb) +{ + __cifs_put_super(sb); +} + +#ifdef CONFIG_CIFS_DFS_UPCALL +int match_target_ip(struct TCP_Server_Info *server, + const char *host, size_t hostlen, + bool *result) +{ + struct sockaddr_storage ss; + int rc; + + cifs_dbg(FYI, "%s: hostname=%.*s\n", __func__, (int)hostlen, host); + + *result = false; + + rc = dns_resolve_name(server->dns_dom, host, hostlen, + (struct sockaddr *)&ss); + if (rc < 0) + return rc; + + spin_lock(&server->srv_lock); + *result = cifs_match_ipaddr((struct sockaddr *)&server->dstaddr, (struct sockaddr *)&ss); + spin_unlock(&server->srv_lock); + cifs_dbg(FYI, "%s: ip addresses matched: %s\n", __func__, str_yes_no(*result)); + return 0; +} + +int cifs_update_super_prepath(struct cifs_sb_info *cifs_sb, char *prefix) +{ + int rc; + + kfree(cifs_sb->prepath); + cifs_sb->prepath = NULL; + + if (prefix && *prefix) { + cifs_sb->prepath = cifs_sanitize_prepath(prefix, GFP_ATOMIC); + if (IS_ERR(cifs_sb->prepath)) { + rc = PTR_ERR(cifs_sb->prepath); + cifs_sb->prepath = NULL; + return rc; + } + if (cifs_sb->prepath) + convert_delimiter(cifs_sb->prepath, CIFS_DIR_SEP(cifs_sb)); + } + + atomic_or(CIFS_MOUNT_USE_PREFIX_PATH, &cifs_sb->mnt_cifs_flags); + return 0; +} + +/* + * Handle weird Windows SMB server behaviour. It responds with + * STATUS_OBJECT_NAME_INVALID code to SMB2 QUERY_INFO request for + * "\\\" DFS reference, where contains + * non-ASCII unicode symbols. + */ +int cifs_inval_name_dfs_link_error(const unsigned int xid, + struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, + const char *full_path, + bool *islink) +{ + struct TCP_Server_Info *server = tcon->ses->server; + struct cifs_ses *ses = tcon->ses; + size_t len; + char *path; + char *ref_path; + + *islink = false; + + /* + * Fast path - skip check when @full_path doesn't have a prefix path to + * look up or tcon is not DFS. + */ + if (strlen(full_path) < 2 || !cifs_sb || + (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NO_DFS) || + !is_tcon_dfs(tcon)) + return 0; + + spin_lock(&server->srv_lock); + if (!server->leaf_fullpath) { + spin_unlock(&server->srv_lock); + return 0; + } + spin_unlock(&server->srv_lock); + + /* + * Slow path - tcon is DFS and @full_path has prefix path, so attempt + * to get a referral to figure out whether it is an DFS link. + */ + len = strnlen(tcon->tree_name, MAX_TREE_SIZE + 1) + strlen(full_path) + 1; + path = kmalloc(len, GFP_KERNEL); + if (!path) + return -ENOMEM; + + scnprintf(path, len, "%s%s", tcon->tree_name, full_path); + ref_path = dfs_cache_canonical_path(path + 1, cifs_sb->local_nls, + cifs_remap(cifs_sb)); + kfree(path); + + if (IS_ERR(ref_path)) { + if (PTR_ERR(ref_path) != -EINVAL) + return PTR_ERR(ref_path); + } else { + struct dfs_info3_param *refs = NULL; + int num_refs = 0; + + /* + * XXX: we are not using dfs_cache_find() here because we might + * end up filling all the DFS cache and thus potentially + * removing cached DFS targets that the client would eventually + * need during failover. + */ + ses = CIFS_DFS_ROOT_SES(ses); + if (ses->server->ops->get_dfs_refer && + !ses->server->ops->get_dfs_refer(xid, ses, ref_path, &refs, + &num_refs, cifs_sb->local_nls, + cifs_remap(cifs_sb))) + *islink = refs[0].server_type == DFS_TYPE_LINK; + free_dfs_info_array(refs, num_refs); + kfree(ref_path); + } + return 0; +} +#endif + +int cifs_wait_for_server_reconnect(struct TCP_Server_Info *server, bool retry) +{ + int timeout = 10; + int rc; + + spin_lock(&server->srv_lock); + if (server->tcpStatus != CifsNeedReconnect) { + spin_unlock(&server->srv_lock); + return 0; + } + timeout *= server->nr_targets; + spin_unlock(&server->srv_lock); + + /* + * Give demultiplex thread up to 10 seconds to each target available for + * reconnect -- should be greater than cifs socket timeout which is 7 + * seconds. + * + * On "soft" mounts we wait once. Hard mounts keep retrying until + * process is killed or server comes back on-line. + */ + do { + rc = wait_event_interruptible_timeout(server->response_q, + (server->tcpStatus != CifsNeedReconnect), + timeout * HZ); + if (rc < 0) { + cifs_dbg(FYI, "%s: aborting reconnect due to received signal\n", + __func__); + return -ERESTARTSYS; + } + + /* are we still trying to reconnect? */ + spin_lock(&server->srv_lock); + if (server->tcpStatus != CifsNeedReconnect) { + spin_unlock(&server->srv_lock); + return 0; + } + spin_unlock(&server->srv_lock); + } while (retry); + + cifs_dbg(FYI, "%s: gave up waiting on reconnect\n", __func__); + return -EHOSTDOWN; +} diff --git a/fs/smb/client/namespace.c b/fs/smb/client/namespace.c new file mode 100644 index 000000000..52a51b032 --- /dev/null +++ b/fs/smb/client/namespace.c @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Contains mounting routines used for handling traversal via SMB junctions. + * + * Copyright (c) 2007 Igor Mammedov + * Copyright (C) International Business Machines Corp., 2008 + * Author(s): Igor Mammedov (niallain@gmail.com) + * Steve French (sfrench@us.ibm.com) + * Copyright (c) 2023 Paulo Alcantara + */ + +#include +#include +#include +#include +#include +#include +#include +#include "cifsglob.h" +#include "cifsproto.h" +#include "cifsfs.h" +#include "cifs_debug.h" +#include "fs_context.h" + +static LIST_HEAD(cifs_automount_list); + +static void cifs_expire_automounts(struct work_struct *work); +static DECLARE_DELAYED_WORK(cifs_automount_task, + cifs_expire_automounts); +static int cifs_mountpoint_expiry_timeout = 500 * HZ; + +static void cifs_expire_automounts(struct work_struct *work) +{ + struct list_head *list = &cifs_automount_list; + + mark_mounts_for_expiry(list); + if (!list_empty(list)) + schedule_delayed_work(&cifs_automount_task, + cifs_mountpoint_expiry_timeout); +} + +void cifs_release_automount_timer(void) +{ + if (WARN_ON(!list_empty(&cifs_automount_list))) + return; + cancel_delayed_work_sync(&cifs_automount_task); +} + +/** + * cifs_build_devname - build a devicename from a UNC and optional prepath + * @nodename: pointer to UNC string + * @prepath: pointer to prefixpath (or NULL if there isn't one) + * + * Build a new cifs devicename after chasing a DFS referral. Allocate a buffer + * big enough to hold the final thing. Copy the UNC from the nodename, and + * concatenate the prepath onto the end of it if there is one. + * + * Returns pointer to the built string, or a ERR_PTR. Caller is responsible + * for freeing the returned string. + */ +char * +cifs_build_devname(char *nodename, const char *prepath) +{ + size_t pplen; + size_t unclen; + char *dev; + char *pos; + + /* skip over any preceding delimiters */ + nodename += strspn(nodename, "\\"); + if (!*nodename) + return ERR_PTR(-EINVAL); + + /* get length of UNC and set pos to last char */ + unclen = strlen(nodename); + pos = nodename + unclen - 1; + + /* trim off any trailing delimiters */ + while (*pos == '\\') { + --pos; + --unclen; + } + + /* allocate a buffer: + * +2 for preceding "//" + * +1 for delimiter between UNC and prepath + * +1 for trailing NULL + */ + pplen = prepath ? strlen(prepath) : 0; + dev = kmalloc(2 + unclen + 1 + pplen + 1, GFP_KERNEL); + if (!dev) + return ERR_PTR(-ENOMEM); + + pos = dev; + /* add the initial "//" */ + *pos = '/'; + ++pos; + *pos = '/'; + ++pos; + + /* copy in the UNC portion from referral */ + memcpy(pos, nodename, unclen); + pos += unclen; + + /* copy the prefixpath remainder (if there is one) */ + if (pplen) { + *pos = '/'; + ++pos; + memcpy(pos, prepath, pplen); + pos += pplen; + } + + /* NULL terminator */ + *pos = '\0'; + + convert_delimiter(dev, '/'); + return dev; +} + +static bool is_dfs_mount(struct dentry *dentry) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(dentry->d_sb); + struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb); + bool ret; + + spin_lock(&tcon->tc_lock); + ret = !!tcon->origin_fullpath; + spin_unlock(&tcon->tc_lock); + return ret; +} + +/* Return full path out of a dentry set for automount */ +static char *automount_fullpath(struct dentry *dentry, void *page) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(dentry->d_sb); + struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb); + size_t len; + char *s; + + spin_lock(&tcon->tc_lock); + if (!tcon->origin_fullpath) { + spin_unlock(&tcon->tc_lock); + return build_path_from_dentry_optional_prefix(dentry, + page, + true); + } + spin_unlock(&tcon->tc_lock); + + if (unlikely(!page)) + return ERR_PTR(-ENOMEM); + + s = dentry_path_raw(dentry, page, PATH_MAX); + if (IS_ERR(s)) + return s; + /* for root, we want "" */ + if (!s[1]) + s++; + + spin_lock(&tcon->tc_lock); + len = strlen(tcon->origin_fullpath); + if (s < (char *)page + len) { + spin_unlock(&tcon->tc_lock); + return ERR_PTR(-ENAMETOOLONG); + } + + s -= len; + memcpy(s, tcon->origin_fullpath, len); + spin_unlock(&tcon->tc_lock); + convert_delimiter(s, '/'); + + return s; +} + +static void fs_context_set_ids(struct smb3_fs_context *ctx) +{ + kuid_t uid = current_fsuid(); + kgid_t gid = current_fsgid(); + + if (ctx->multiuser) { + if (!ctx->uid_specified) + ctx->linux_uid = uid; + if (!ctx->gid_specified) + ctx->linux_gid = gid; + } + if (!ctx->cruid_specified) + ctx->cred_uid = uid; +} + +/* + * Create a vfsmount that we can automount + */ +static struct vfsmount *cifs_do_automount(struct path *path) +{ + int rc; + struct dentry *mntpt = path->dentry; + struct fs_context *fc; + void *page = NULL; + struct smb3_fs_context *ctx, *cur_ctx; + struct smb3_fs_context tmp; + char *full_path; + struct vfsmount *mnt; + struct cifs_sb_info *mntpt_sb; + struct cifs_ses *ses; + + if (IS_ROOT(mntpt)) + return ERR_PTR(-ESTALE); + + mntpt_sb = CIFS_SB(mntpt->d_sb); + ses = cifs_sb_master_tcon(mntpt_sb)->ses; + cur_ctx = mntpt_sb->ctx; + + /* + * At this point, the root session should be in the mntpt sb. We should + * bring the sb context passwords in sync with the root session's + * passwords. This would help prevent unnecessary retries and password + * swaps for automounts. + */ + mutex_lock(&ses->session_mutex); + rc = smb3_sync_session_ctx_passwords(mntpt_sb, ses); + mutex_unlock(&ses->session_mutex); + + if (rc) + return ERR_PTR(rc); + + fc = fs_context_for_submount(path->mnt->mnt_sb->s_type, mntpt); + if (IS_ERR(fc)) + return ERR_CAST(fc); + + ctx = smb3_fc2context(fc); + + page = alloc_dentry_path(); + full_path = automount_fullpath(mntpt, page); + if (IS_ERR(full_path)) { + mnt = ERR_CAST(full_path); + goto out; + } + + tmp = *cur_ctx; + tmp.source = NULL; + tmp.leaf_fullpath = NULL; + tmp.UNC = tmp.prepath = NULL; + tmp.dfs_root_ses = NULL; + fs_context_set_ids(&tmp); + + rc = smb3_fs_context_dup(ctx, &tmp); + if (rc) { + mnt = ERR_PTR(rc); + goto out; + } + + rc = smb3_parse_devname(full_path, ctx); + if (rc) { + mnt = ERR_PTR(rc); + goto out; + } + + ctx->source = smb3_fs_context_fullpath(ctx, '/'); + if (IS_ERR(ctx->source)) { + mnt = ERR_CAST(ctx->source); + ctx->source = NULL; + goto out; + } + ctx->dfs_automount = ctx->dfs_conn = is_dfs_mount(mntpt); + cifs_dbg(FYI, "%s: ctx: source=%s UNC=%s prepath=%s dfs_automount=%d\n", + __func__, ctx->source, ctx->UNC, ctx->prepath, ctx->dfs_automount); + + mnt = fc_mount(fc); +out: + put_fs_context(fc); + free_dentry_path(page); + return mnt; +} + +/* + * Attempt to automount the referral + */ +struct vfsmount *cifs_d_automount(struct path *path) +{ + struct vfsmount *newmnt; + + cifs_dbg(FYI, "%s: %pd\n", __func__, path->dentry); + + newmnt = cifs_do_automount(path); + if (IS_ERR(newmnt)) { + cifs_dbg(FYI, "leaving %s [automount failed]\n" , __func__); + return newmnt; + } + + mnt_set_expiry(newmnt, &cifs_automount_list); + schedule_delayed_work(&cifs_automount_task, + cifs_mountpoint_expiry_timeout); + cifs_dbg(FYI, "leaving %s [ok]\n" , __func__); + return newmnt; +} + +const struct inode_operations cifs_namespace_inode_operations = { + .fileattr_get = cifs_fileattr_get, +}; diff --git a/fs/smb/client/netlink.c b/fs/smb/client/netlink.c new file mode 100644 index 000000000..0dd10913c --- /dev/null +++ b/fs/smb/client/netlink.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Netlink routines for CIFS + * + * Copyright (c) 2020 Samuel Cabrero + */ + +#include +#include + +#include "netlink.h" +#include "cifsglob.h" +#include "cifs_debug.h" +#include "cifs_swn.h" + +static const struct nla_policy cifs_genl_policy[CIFS_GENL_ATTR_MAX + 1] = { + [CIFS_GENL_ATTR_SWN_REGISTRATION_ID] = { .type = NLA_U32 }, + [CIFS_GENL_ATTR_SWN_NET_NAME] = { .type = NLA_STRING }, + [CIFS_GENL_ATTR_SWN_SHARE_NAME] = { .type = NLA_STRING }, + [CIFS_GENL_ATTR_SWN_IP] = { .len = sizeof(struct sockaddr_storage) }, + [CIFS_GENL_ATTR_SWN_NET_NAME_NOTIFY] = { .type = NLA_FLAG }, + [CIFS_GENL_ATTR_SWN_SHARE_NAME_NOTIFY] = { .type = NLA_FLAG }, + [CIFS_GENL_ATTR_SWN_IP_NOTIFY] = { .type = NLA_FLAG }, + [CIFS_GENL_ATTR_SWN_KRB_AUTH] = { .type = NLA_FLAG }, + [CIFS_GENL_ATTR_SWN_USER_NAME] = { .type = NLA_STRING }, + [CIFS_GENL_ATTR_SWN_PASSWORD] = { .type = NLA_STRING }, + [CIFS_GENL_ATTR_SWN_DOMAIN_NAME] = { .type = NLA_STRING }, + [CIFS_GENL_ATTR_SWN_NOTIFICATION_TYPE] = { .type = NLA_U32 }, + [CIFS_GENL_ATTR_SWN_RESOURCE_STATE] = { .type = NLA_U32 }, + [CIFS_GENL_ATTR_SWN_RESOURCE_NAME] = { .type = NLA_STRING}, +}; + +static const struct genl_ops cifs_genl_ops[] = { + { + .cmd = CIFS_GENL_CMD_SWN_NOTIFY, + .flags = GENL_ADMIN_PERM, + .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP, + .doit = cifs_swn_notify, + }, +}; + +static const struct genl_multicast_group cifs_genl_mcgrps[] = { + [CIFS_GENL_MCGRP_SWN] = { + .name = CIFS_GENL_MCGRP_SWN_NAME, + .flags = GENL_MCAST_CAP_NET_ADMIN, + }, +}; + +struct genl_family cifs_genl_family = { + .name = CIFS_GENL_NAME, + .version = CIFS_GENL_VERSION, + .hdrsize = 0, + .maxattr = CIFS_GENL_ATTR_MAX, + .module = THIS_MODULE, + .policy = cifs_genl_policy, + .ops = cifs_genl_ops, + .n_ops = ARRAY_SIZE(cifs_genl_ops), + .resv_start_op = CIFS_GENL_CMD_SWN_NOTIFY + 1, + .mcgrps = cifs_genl_mcgrps, + .n_mcgrps = ARRAY_SIZE(cifs_genl_mcgrps), +}; + +/** + * cifs_genl_init - Register generic netlink family + * + * Return zero if initialized successfully, otherwise non-zero. + */ +int cifs_genl_init(void) +{ + int ret; + + ret = genl_register_family(&cifs_genl_family); + if (ret < 0) { + cifs_dbg(VFS, "%s: failed to register netlink family\n", + __func__); + return ret; + } + + return 0; +} + +/** + * cifs_genl_exit - Unregister generic netlink family + */ +void cifs_genl_exit(void) +{ + int ret; + + ret = genl_unregister_family(&cifs_genl_family); + if (ret < 0) { + cifs_dbg(VFS, "%s: failed to unregister netlink family\n", + __func__); + } +} diff --git a/fs/smb/client/netlink.h b/fs/smb/client/netlink.h new file mode 100644 index 000000000..d35eef981 --- /dev/null +++ b/fs/smb/client/netlink.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Netlink routines for CIFS + * + * Copyright (c) 2020 Samuel Cabrero + */ + +#ifndef _CIFS_NETLINK_H +#define _CIFS_NETLINK_H + +extern struct genl_family cifs_genl_family; + +int cifs_genl_init(void); +void cifs_genl_exit(void); + +#endif /* _CIFS_NETLINK_H */ diff --git a/fs/smb/client/netmisc.c b/fs/smb/client/netmisc.c new file mode 100644 index 000000000..bddadee82 --- /dev/null +++ b/fs/smb/client/netmisc.c @@ -0,0 +1,212 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * + * Copyright (c) International Business Machines Corp., 2002,2008 + * Author(s): Steve French (sfrench@us.ibm.com) + * + * Error mapping routines from Samba libsmb/errormap.c + * Copyright (C) Andrew Tridgell 2001 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "cifsfs.h" +#include "cifsglob.h" +#include "cifsproto.h" +#include "smb1proto.h" +#include "smberr.h" +#include "cifs_debug.h" +#include "nterr.h" + +/* + * Convert a string containing text IPv4 or IPv6 address to binary form. + * + * Returns 0 on failure. + */ +static int +cifs_inet_pton(const int address_family, const char *cp, int len, void *dst) +{ + int ret = 0; + + /* calculate length by finding first slash or NULL */ + if (address_family == AF_INET) + ret = in4_pton(cp, len, dst, '\\', NULL); + else if (address_family == AF_INET6) + ret = in6_pton(cp, len, dst , '\\', NULL); + + cifs_dbg(NOISY, "address conversion returned %d for %*.*s\n", + ret, len, len, cp); + if (ret > 0) + ret = 1; + return ret; +} + +/* + * Try to convert a string to an IPv4 address and then attempt to convert + * it to an IPv6 address if that fails. Set the family field if either + * succeeds. If it's an IPv6 address and it has a '%' sign in it, try to + * treat the part following it as a numeric sin6_scope_id. + * + * Returns 0 on failure. + */ +int +cifs_convert_address(struct sockaddr *dst, const char *src, int len) +{ + int rc, alen, slen; + const char *pct; + char scope_id[13]; + struct sockaddr_in *s4 = (struct sockaddr_in *) dst; + struct sockaddr_in6 *s6 = (struct sockaddr_in6 *) dst; + + /* IPv4 address */ + if (cifs_inet_pton(AF_INET, src, len, &s4->sin_addr.s_addr)) { + s4->sin_family = AF_INET; + return 1; + } + + /* attempt to exclude the scope ID from the address part */ + pct = memchr(src, '%', len); + alen = pct ? pct - src : len; + + rc = cifs_inet_pton(AF_INET6, src, alen, &s6->sin6_addr.s6_addr); + if (!rc) + return rc; + + s6->sin6_family = AF_INET6; + if (pct) { + /* grab the scope ID */ + slen = len - (alen + 1); + if (slen <= 0 || slen > 12) + return 0; + memcpy(scope_id, pct + 1, slen); + scope_id[slen] = '\0'; + + rc = kstrtouint(scope_id, 0, &s6->sin6_scope_id); + rc = (rc == 0) ? 1 : 0; + } + + return rc; +} + +void +cifs_set_port(struct sockaddr *addr, const unsigned short int port) +{ + switch (addr->sa_family) { + case AF_INET: + ((struct sockaddr_in *)addr)->sin_port = htons(port); + break; + case AF_INET6: + ((struct sockaddr_in6 *)addr)->sin6_port = htons(port); + break; + } +} + +/* The following are taken from fs/ntfs/util.c */ + +#define NTFS_TIME_OFFSET ((u64)(369*365 + 89) * 24 * 3600 * 10000000) + +/* + * Convert the NT UTC (based 1601-01-01, in hundred nanosecond units) + * into Unix UTC (based 1970-01-01, in seconds). + */ +struct timespec64 +cifs_NTtimeToUnix(__le64 ntutc) +{ + struct timespec64 ts; + /* BB what about the timezone? BB */ + + /* Subtract the NTFS time offset, then convert to 1s intervals. */ + s64 t = le64_to_cpu(ntutc) - NTFS_TIME_OFFSET; + u64 abs_t; + + /* + * Unfortunately can not use normal 64 bit division on 32 bit arch, but + * the alternative, do_div, does not work with negative numbers so have + * to special case them + */ + if (t < 0) { + abs_t = -t; + ts.tv_nsec = (time64_t)(do_div(abs_t, 10000000) * 100); + ts.tv_nsec = -ts.tv_nsec; + ts.tv_sec = -abs_t; + } else { + abs_t = t; + ts.tv_nsec = (time64_t)do_div(abs_t, 10000000) * 100; + ts.tv_sec = abs_t; + } + + return ts; +} + +/* Convert the Unix UTC into NT UTC. */ +u64 +cifs_UnixTimeToNT(struct timespec64 t) +{ + /* Convert to 100ns intervals and then add the NTFS time offset. */ + return (u64) t.tv_sec * 10000000 + t.tv_nsec/100 + NTFS_TIME_OFFSET; +} + +static const int total_days_of_prev_months[] = { + 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334 +}; + +struct timespec64 cnvrtDosUnixTm(__le16 le_date, __le16 le_time, int offset) +{ + struct timespec64 ts; + time64_t sec, days; + int min, day, month, year; + u16 date = le16_to_cpu(le_date); + u16 time = le16_to_cpu(le_time); + SMB_TIME *st = (SMB_TIME *)&time; + SMB_DATE *sd = (SMB_DATE *)&date; + + cifs_dbg(FYI, "date %d time %d\n", date, time); + + sec = 2 * st->TwoSeconds; + min = st->Minutes; + if ((sec > 59) || (min > 59)) + cifs_dbg(VFS, "Invalid time min %d sec %lld\n", min, sec); + sec += (min * 60); + sec += 60 * 60 * st->Hours; + if (st->Hours > 24) + cifs_dbg(VFS, "Invalid hours %d\n", st->Hours); + day = sd->Day; + month = sd->Month; + if (day < 1 || day > 31 || month < 1 || month > 12) { + cifs_dbg(VFS, "Invalid date, month %d day: %d\n", month, day); + day = clamp(day, 1, 31); + month = clamp(month, 1, 12); + } + month -= 1; + days = day + total_days_of_prev_months[month]; + days += 3652; /* account for difference in days between 1980 and 1970 */ + year = sd->Year; + days += year * 365; + days += (year/4); /* leap year */ + /* generalized leap year calculation is more complex, ie no leap year + for years/100 except for years/400, but since the maximum number for DOS + year is 2**7, the last year is 1980+127, which means we need only + consider 2 special case years, ie the years 2000 and 2100, and only + adjust for the lack of leap year for the year 2100, as 2000 was a + leap year (divisible by 400) */ + if (year >= 120) /* the year 2100 */ + days = days - 1; /* do not count leap year for the year 2100 */ + + /* adjust for leap year where we are still before leap day */ + if (year != 120) + days -= ((year & 0x03) == 0) && (month < 2 ? 1 : 0); + sec += 24 * 60 * 60 * days; + + ts.tv_sec = sec + offset; + + /* cifs_dbg(FYI, "sec after cnvrt dos to unix time %d\n",sec); */ + + ts.tv_nsec = 0; + return ts; +} diff --git a/fs/smb/client/nterr.h b/fs/smb/client/nterr.h new file mode 100644 index 000000000..a542029c6 --- /dev/null +++ b/fs/smb/client/nterr.h @@ -0,0 +1,575 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + Unix SMB/Netbios implementation. + Version 1.9. + NT error code constants + Copyright (C) Andrew Tridgell 1992-2000 + Copyright (C) John H Terpstra 1996-2000 + Copyright (C) Luke Kenneth Casson Leighton 1996-2000 + Copyright (C) Paul Ashton 1998-2000 + +*/ + + + +#ifndef _NTERR_H +#define _NTERR_H + +/* NT status -> dos error map */ +struct ntstatus_to_dos_err { + __u8 dos_class; + __u16 dos_code; + __u32 ntstatus; + const char *nt_errstr; +}; + +/* Win32 Error Codes. */ +#define NT_ERROR_INVALID_PARAMETER 0x0057 +#define NT_ERROR_INSUFFICIENT_BUFFER 0x007a +#define NT_ERROR_INVALID_DATATYPE 0x070c + +/* + * NTSTATUS Values extracted using a loop in smbclient then printing a netmon + * sniff to a file. + * + * The comment at the end of each definition indicates `dos_class` + * and `dos_code` fields of the `ntstatus_to_dos_map` array; it is + * used to generate the `smb1_mapping_table.c` file. + */ + +#define NT_STATUS_OK 0x0000 // SUCCESS, 0 +#define NT_STATUS_PENDING 0x0103 // ERRHRD, ERRgeneral +#define NT_STATUS_MORE_ENTRIES 0x0105 // ERRHRD, ERRgeneral +#define NT_STATUS_SOME_NOT_MAPPED 0x0107 // ERRHRD, ERRgeneral +#define NT_STATUS_NOTIFY_ENUM_DIR 0x010c // ERRSRV, ERR_NOTIFY_ENUM_DIR +#define NT_STATUS_BUFFER_OVERFLOW 0x80000005 // ERRDOS, ERRmoredata +#define NT_STATUS_NO_MORE_ENTRIES 0x8000001a // ERRHRD, ERRgeneral +#define NT_STATUS_MEDIA_CHANGED 0x8000001c // ERRHRD, ERRgeneral +#define NT_STATUS_END_OF_MEDIA 0x8000001e // ERRHRD, ERRgeneral +#define NT_STATUS_MEDIA_CHECK 0x80000020 // ERRHRD, ERRgeneral +#define NT_STATUS_NO_DATA_DETECTED 0x80000022 // ERRHRD, ERRgeneral +#define NT_STATUS_STOPPED_ON_SYMLINK 0x8000002d // ERRDOS, ERRsymlink +#define NT_STATUS_DEVICE_REQUIRES_CLEANING 0x80000288 // ERRHRD, ERRgeneral +#define NT_STATUS_DEVICE_DOOR_OPEN 0x80000289 // ERRHRD, ERRgeneral +#define NT_STATUS_UNSUCCESSFUL (0xC0000000 | 0x0001) // ERRDOS, ERRgeneral +#define NT_STATUS_NOT_IMPLEMENTED (0xC0000000 | 0x0002) // ERRDOS, ERRbadfunc +#define NT_STATUS_INVALID_INFO_CLASS (0xC0000000 | 0x0003) // ERRDOS, ERRbadpipe +#define NT_STATUS_INFO_LENGTH_MISMATCH (0xC0000000 | 0x0004) // ERRDOS, 24 +#define NT_STATUS_ACCESS_VIOLATION (0xC0000000 | 0x0005) // ERRHRD, ERRgeneral +#define NT_STATUS_IN_PAGE_ERROR (0xC0000000 | 0x0006) // ERRHRD, ERRgeneral +#define NT_STATUS_PAGEFILE_QUOTA (0xC0000000 | 0x0007) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_HANDLE (0xC0000000 | 0x0008) // ERRDOS, ERRbadfid +#define NT_STATUS_BAD_INITIAL_STACK (0xC0000000 | 0x0009) // ERRHRD, ERRgeneral +#define NT_STATUS_BAD_INITIAL_PC (0xC0000000 | 0x000a) // ERRDOS, 193 +#define NT_STATUS_INVALID_CID (0xC0000000 | 0x000b) // ERRDOS, 87 +#define NT_STATUS_TIMER_NOT_CANCELED (0xC0000000 | 0x000c) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_PARAMETER (0xC0000000 | 0x000d) // ERRDOS, 87 +#define NT_STATUS_NO_SUCH_DEVICE (0xC0000000 | 0x000e) // ERRDOS, ERRbadfile +#define NT_STATUS_NO_SUCH_FILE (0xC0000000 | 0x000f) // ERRDOS, ERRbadfile +#define NT_STATUS_INVALID_DEVICE_REQUEST (0xC0000000 | 0x0010) // ERRDOS, ERRbadfunc +#define NT_STATUS_END_OF_FILE (0xC0000000 | 0x0011) // ERRDOS, 38 +#define NT_STATUS_WRONG_VOLUME (0xC0000000 | 0x0012) // ERRDOS, 34 +#define NT_STATUS_NO_MEDIA_IN_DEVICE (0xC0000000 | 0x0013) // ERRDOS, 21 +#define NT_STATUS_UNRECOGNIZED_MEDIA (0xC0000000 | 0x0014) // ERRHRD, ERRgeneral +#define NT_STATUS_NONEXISTENT_SECTOR (0xC0000000 | 0x0015) // ERRDOS, 27 +#define NT_STATUS_MORE_PROCESSING_REQUIRED (0xC0000000 | 0x0016) // ERRDOS, ERRmoredata +#define NT_STATUS_NO_MEMORY (0xC0000000 | 0x0017) // ERRDOS, ERRnomem +#define NT_STATUS_CONFLICTING_ADDRESSES (0xC0000000 | 0x0018) // ERRDOS, 487 +#define NT_STATUS_NOT_MAPPED_VIEW (0xC0000000 | 0x0019) // ERRDOS, 487 +#define NT_STATUS_UNABLE_TO_FREE_VM (0xC0000000 | 0x001a) // ERRDOS, 87 +#define NT_STATUS_UNABLE_TO_DELETE_SECTION (0xC0000000 | 0x001b) // ERRDOS, 87 +#define NT_STATUS_INVALID_SYSTEM_SERVICE (0xC0000000 | 0x001c) // ERRDOS, 2142 +#define NT_STATUS_ILLEGAL_INSTRUCTION (0xC0000000 | 0x001d) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_LOCK_SEQUENCE (0xC0000000 | 0x001e) // ERRDOS, ERRnoaccess +#define NT_STATUS_INVALID_VIEW_SIZE (0xC0000000 | 0x001f) // ERRDOS, ERRnoaccess +#define NT_STATUS_INVALID_FILE_FOR_SECTION (0xC0000000 | 0x0020) // ERRDOS, 193 +#define NT_STATUS_ALREADY_COMMITTED (0xC0000000 | 0x0021) // ERRDOS, ERRnoaccess +#define NT_STATUS_ACCESS_DENIED (0xC0000000 | 0x0022) // ERRDOS, ERRnoaccess +#define NT_STATUS_BUFFER_TOO_SMALL (0xC0000000 | 0x0023) // ERRDOS, 111 +#define NT_STATUS_OBJECT_TYPE_MISMATCH (0xC0000000 | 0x0024) // ERRDOS, ERRbadfid +#define NT_STATUS_NONCONTINUABLE_EXCEPTION (0xC0000000 | 0x0025) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_DISPOSITION (0xC0000000 | 0x0026) // ERRHRD, ERRgeneral +#define NT_STATUS_UNWIND (0xC0000000 | 0x0027) // ERRHRD, ERRgeneral +#define NT_STATUS_BAD_STACK (0xC0000000 | 0x0028) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_UNWIND_TARGET (0xC0000000 | 0x0029) // ERRHRD, ERRgeneral +#define NT_STATUS_NOT_LOCKED (0xC0000000 | 0x002a) // ERRDOS, 158 +#define NT_STATUS_PARITY_ERROR (0xC0000000 | 0x002b) // ERRHRD, ERRgeneral +#define NT_STATUS_UNABLE_TO_DECOMMIT_VM (0xC0000000 | 0x002c) // ERRDOS, 487 +#define NT_STATUS_NOT_COMMITTED (0xC0000000 | 0x002d) // ERRDOS, 487 +#define NT_STATUS_INVALID_PORT_ATTRIBUTES (0xC0000000 | 0x002e) // ERRHRD, ERRgeneral +#define NT_STATUS_PORT_MESSAGE_TOO_LONG (0xC0000000 | 0x002f) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_PARAMETER_MIX (0xC0000000 | 0x0030) // ERRDOS, 87 +#define NT_STATUS_INVALID_QUOTA_LOWER (0xC0000000 | 0x0031) // ERRHRD, ERRgeneral +#define NT_STATUS_DISK_CORRUPT_ERROR (0xC0000000 | 0x0032) // ERRHRD, ERRgeneral +#define NT_STATUS_OBJECT_NAME_INVALID (0xC0000000 | 0x0033) // ERRDOS, ERRbadfile +#define NT_STATUS_OBJECT_NAME_NOT_FOUND (0xC0000000 | 0x0034) // ERRDOS, ERRbadfile +#define NT_STATUS_OBJECT_NAME_COLLISION (0xC0000000 | 0x0035) // ERRDOS, ERRalreadyexists +#define NT_STATUS_HANDLE_NOT_WAITABLE (0xC0000000 | 0x0036) // ERRHRD, ERRgeneral +#define NT_STATUS_PORT_DISCONNECTED (0xC0000000 | 0x0037) // ERRDOS, ERRbadfid +#define NT_STATUS_DEVICE_ALREADY_ATTACHED (0xC0000000 | 0x0038) // ERRHRD, ERRgeneral +#define NT_STATUS_OBJECT_PATH_INVALID (0xC0000000 | 0x0039) // ERRDOS, 161 +#define NT_STATUS_OBJECT_PATH_NOT_FOUND (0xC0000000 | 0x003a) // ERRDOS, ERRbadpath +#define NT_STATUS_OBJECT_PATH_SYNTAX_BAD (0xC0000000 | 0x003b) // ERRDOS, 161 +#define NT_STATUS_DATA_OVERRUN (0xC0000000 | 0x003c) // ERRHRD, ERRgeneral +#define NT_STATUS_DATA_LATE_ERROR (0xC0000000 | 0x003d) // ERRHRD, ERRgeneral +#define NT_STATUS_DATA_ERROR (0xC0000000 | 0x003e) // ERRDOS, 23 +#define NT_STATUS_CRC_ERROR (0xC0000000 | 0x003f) // ERRDOS, 23 +#define NT_STATUS_SECTION_TOO_BIG (0xC0000000 | 0x0040) // ERRDOS, ERRnomem +#define NT_STATUS_PORT_CONNECTION_REFUSED (0xC0000000 | 0x0041) // ERRDOS, ERRnoaccess +#define NT_STATUS_INVALID_PORT_HANDLE (0xC0000000 | 0x0042) // ERRDOS, ERRbadfid +#define NT_STATUS_SHARING_VIOLATION (0xC0000000 | 0x0043) // ERRDOS, ERRbadshare +#define NT_STATUS_QUOTA_EXCEEDED (0xC0000000 | 0x0044) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_PAGE_PROTECTION (0xC0000000 | 0x0045) // ERRDOS, 87 +#define NT_STATUS_MUTANT_NOT_OWNED (0xC0000000 | 0x0046) // ERRDOS, 288 +#define NT_STATUS_SEMAPHORE_LIMIT_EXCEEDED (0xC0000000 | 0x0047) // ERRDOS, 298 +#define NT_STATUS_PORT_ALREADY_SET (0xC0000000 | 0x0048) // ERRDOS, 87 +#define NT_STATUS_SECTION_NOT_IMAGE (0xC0000000 | 0x0049) // ERRDOS, 87 +#define NT_STATUS_SUSPEND_COUNT_EXCEEDED (0xC0000000 | 0x004a) // ERRDOS, 156 +#define NT_STATUS_THREAD_IS_TERMINATING (0xC0000000 | 0x004b) // ERRDOS, ERRnoaccess +#define NT_STATUS_BAD_WORKING_SET_LIMIT (0xC0000000 | 0x004c) // ERRDOS, 87 +#define NT_STATUS_INCOMPATIBLE_FILE_MAP (0xC0000000 | 0x004d) // ERRDOS, 87 +#define NT_STATUS_SECTION_PROTECTION (0xC0000000 | 0x004e) // ERRDOS, 87 +#define NT_STATUS_EAS_NOT_SUPPORTED (0xC0000000 | 0x004f) // ERRDOS, ERReasnotsupported +#define NT_STATUS_EA_TOO_LARGE (0xC0000000 | 0x0050) // ERRDOS, 255 +#define NT_STATUS_NONEXISTENT_EA_ENTRY (0xC0000000 | 0x0051) // ERRHRD, ERRgeneral +#define NT_STATUS_NO_EAS_ON_FILE (0xC0000000 | 0x0052) // ERRHRD, ERRgeneral +#define NT_STATUS_EA_CORRUPT_ERROR (0xC0000000 | 0x0053) // ERRHRD, ERRgeneral +#define NT_STATUS_FILE_LOCK_CONFLICT (0xC0000000 | 0x0054) // ERRDOS, ERRlock +#define NT_STATUS_LOCK_NOT_GRANTED (0xC0000000 | 0x0055) // ERRDOS, ERRlock +#define NT_STATUS_DELETE_PENDING (0xC0000000 | 0x0056) // ERRDOS, ERRbadfile +#define NT_STATUS_CTL_FILE_NOT_SUPPORTED (0xC0000000 | 0x0057) // ERRDOS, ERRunsup +#define NT_STATUS_UNKNOWN_REVISION (0xC0000000 | 0x0058) // ERRHRD, ERRgeneral +#define NT_STATUS_REVISION_MISMATCH (0xC0000000 | 0x0059) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_OWNER (0xC0000000 | 0x005a) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_PRIMARY_GROUP (0xC0000000 | 0x005b) // ERRHRD, ERRgeneral +#define NT_STATUS_NO_IMPERSONATION_TOKEN (0xC0000000 | 0x005c) // ERRHRD, ERRgeneral +#define NT_STATUS_CANT_DISABLE_MANDATORY (0xC0000000 | 0x005d) // ERRHRD, ERRgeneral +#define NT_STATUS_NO_LOGON_SERVERS (0xC0000000 | 0x005e) // ERRDOS, 2215 +#define NT_STATUS_NO_SUCH_LOGON_SESSION (0xC0000000 | 0x005f) // ERRHRD, ERRgeneral +#define NT_STATUS_NO_SUCH_PRIVILEGE (0xC0000000 | 0x0060) // ERRHRD, ERRgeneral +#define NT_STATUS_PRIVILEGE_NOT_HELD (0xC0000000 | 0x0061) // ERRDOS, ERRnoaccess +#define NT_STATUS_INVALID_ACCOUNT_NAME (0xC0000000 | 0x0062) // ERRHRD, ERRgeneral +#define NT_STATUS_USER_EXISTS (0xC0000000 | 0x0063) // ERRHRD, ERRgeneral +#define NT_STATUS_NO_SUCH_USER (0xC0000000 | 0x0064) // ERRDOS, ERRnoaccess +#define NT_STATUS_GROUP_EXISTS (0xC0000000 | 0x0065) // ERRHRD, ERRgeneral +#define NT_STATUS_NO_SUCH_GROUP (0xC0000000 | 0x0066) // ERRHRD, ERRgeneral +#define NT_STATUS_MEMBER_IN_GROUP (0xC0000000 | 0x0067) // ERRHRD, ERRgeneral +#define NT_STATUS_MEMBER_NOT_IN_GROUP (0xC0000000 | 0x0068) // ERRHRD, ERRgeneral +#define NT_STATUS_LAST_ADMIN (0xC0000000 | 0x0069) // ERRHRD, ERRgeneral +#define NT_STATUS_WRONG_PASSWORD (0xC0000000 | 0x006a) // ERRSRV, ERRbadpw +#define NT_STATUS_ILL_FORMED_PASSWORD (0xC0000000 | 0x006b) // ERRHRD, ERRgeneral +#define NT_STATUS_PASSWORD_RESTRICTION (0xC0000000 | 0x006c) // ERRHRD, ERRgeneral +#define NT_STATUS_LOGON_FAILURE (0xC0000000 | 0x006d) // ERRDOS, ERRnoaccess +#define NT_STATUS_ACCOUNT_RESTRICTION (0xC0000000 | 0x006e) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_LOGON_HOURS (0xC0000000 | 0x006f) // ERRSRV, ERRbadLogonTime +#define NT_STATUS_INVALID_WORKSTATION (0xC0000000 | 0x0070) // ERRSRV, ERRbadclient +#define NT_STATUS_PASSWORD_EXPIRED (0xC0000000 | 0x0071) // ERRSRV, ERRpasswordExpired +#define NT_STATUS_ACCOUNT_DISABLED (0xC0000000 | 0x0072) // ERRSRV, ERRaccountexpired +#define NT_STATUS_NONE_MAPPED (0xC0000000 | 0x0073) // ERRHRD, ERRgeneral +#define NT_STATUS_TOO_MANY_LUIDS_REQUESTED (0xC0000000 | 0x0074) // ERRHRD, ERRgeneral +#define NT_STATUS_LUIDS_EXHAUSTED (0xC0000000 | 0x0075) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_SUB_AUTHORITY (0xC0000000 | 0x0076) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_ACL (0xC0000000 | 0x0077) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_SID (0xC0000000 | 0x0078) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_SECURITY_DESCR (0xC0000000 | 0x0079) // ERRHRD, ERRgeneral +#define NT_STATUS_PROCEDURE_NOT_FOUND (0xC0000000 | 0x007a) // ERRDOS, 127 +#define NT_STATUS_INVALID_IMAGE_FORMAT (0xC0000000 | 0x007b) // ERRDOS, 193 +#define NT_STATUS_NO_TOKEN (0xC0000000 | 0x007c) // ERRHRD, ERRgeneral +#define NT_STATUS_BAD_INHERITANCE_ACL (0xC0000000 | 0x007d) // ERRHRD, ERRgeneral +#define NT_STATUS_RANGE_NOT_LOCKED (0xC0000000 | 0x007e) // ERRDOS, 158 +#define NT_STATUS_DISK_FULL (0xC0000000 | 0x007f) // ERRDOS, 112 +#define NT_STATUS_SERVER_DISABLED (0xC0000000 | 0x0080) // ERRHRD, ERRgeneral +#define NT_STATUS_SERVER_NOT_DISABLED (0xC0000000 | 0x0081) // ERRHRD, ERRgeneral +#define NT_STATUS_TOO_MANY_GUIDS_REQUESTED (0xC0000000 | 0x0082) // ERRDOS, 68 +#define NT_STATUS_GUIDS_EXHAUSTED (0xC0000000 | 0x0083) // ERRDOS, 259 +#define NT_STATUS_INVALID_ID_AUTHORITY (0xC0000000 | 0x0084) // ERRHRD, ERRgeneral +#define NT_STATUS_AGENTS_EXHAUSTED (0xC0000000 | 0x0085) // ERRDOS, 259 +#define NT_STATUS_INVALID_VOLUME_LABEL (0xC0000000 | 0x0086) // ERRDOS, 154 +#define NT_STATUS_SECTION_NOT_EXTENDED (0xC0000000 | 0x0087) // ERRDOS, 14 +#define NT_STATUS_NOT_MAPPED_DATA (0xC0000000 | 0x0088) // ERRDOS, 487 +#define NT_STATUS_RESOURCE_DATA_NOT_FOUND (0xC0000000 | 0x0089) // ERRHRD, ERRgeneral +#define NT_STATUS_RESOURCE_TYPE_NOT_FOUND (0xC0000000 | 0x008a) // ERRHRD, ERRgeneral +#define NT_STATUS_RESOURCE_NAME_NOT_FOUND (0xC0000000 | 0x008b) // ERRHRD, ERRgeneral +#define NT_STATUS_ARRAY_BOUNDS_EXCEEDED (0xC0000000 | 0x008c) // ERRHRD, ERRgeneral +#define NT_STATUS_FLOAT_DENORMAL_OPERAND (0xC0000000 | 0x008d) // ERRHRD, ERRgeneral +#define NT_STATUS_FLOAT_DIVIDE_BY_ZERO (0xC0000000 | 0x008e) // ERRHRD, ERRgeneral +#define NT_STATUS_FLOAT_INEXACT_RESULT (0xC0000000 | 0x008f) // ERRHRD, ERRgeneral +#define NT_STATUS_FLOAT_INVALID_OPERATION (0xC0000000 | 0x0090) // ERRHRD, ERRgeneral +#define NT_STATUS_FLOAT_OVERFLOW (0xC0000000 | 0x0091) // ERRHRD, ERRgeneral +#define NT_STATUS_FLOAT_STACK_CHECK (0xC0000000 | 0x0092) // ERRHRD, ERRgeneral +#define NT_STATUS_FLOAT_UNDERFLOW (0xC0000000 | 0x0093) // ERRHRD, ERRgeneral +#define NT_STATUS_INTEGER_DIVIDE_BY_ZERO (0xC0000000 | 0x0094) // ERRHRD, ERRgeneral +#define NT_STATUS_INTEGER_OVERFLOW (0xC0000000 | 0x0095) // ERRDOS, 534 +#define NT_STATUS_PRIVILEGED_INSTRUCTION (0xC0000000 | 0x0096) // ERRHRD, ERRgeneral +#define NT_STATUS_TOO_MANY_PAGING_FILES (0xC0000000 | 0x0097) // ERRDOS, ERRnomem +#define NT_STATUS_FILE_INVALID (0xC0000000 | 0x0098) // ERRHRD, ERRgeneral +#define NT_STATUS_ALLOTTED_SPACE_EXCEEDED (0xC0000000 | 0x0099) // ERRHRD, ERRgeneral +#define NT_STATUS_INSUFFICIENT_RESOURCES (0xC0000000 | 0x009a) // ERRDOS, ERRnoresource +#define NT_STATUS_DFS_EXIT_PATH_FOUND (0xC0000000 | 0x009b) // ERRDOS, ERRbadpath +#define NT_STATUS_DEVICE_DATA_ERROR (0xC0000000 | 0x009c) // ERRDOS, 23 +#define NT_STATUS_DEVICE_NOT_CONNECTED (0xC0000000 | 0x009d) // ERRHRD, ERRgeneral +#define NT_STATUS_DEVICE_POWER_FAILURE (0xC0000000 | 0x009e) // ERRDOS, 21 +#define NT_STATUS_FREE_VM_NOT_AT_BASE (0xC0000000 | 0x009f) // ERRDOS, 487 +#define NT_STATUS_MEMORY_NOT_ALLOCATED (0xC0000000 | 0x00a0) // ERRDOS, 487 +#define NT_STATUS_WORKING_SET_QUOTA (0xC0000000 | 0x00a1) // ERRHRD, ERRgeneral +#define NT_STATUS_MEDIA_WRITE_PROTECTED (0xC0000000 | 0x00a2) // ERRDOS, 19 +#define NT_STATUS_DEVICE_NOT_READY (0xC0000000 | 0x00a3) // ERRDOS, 21 +#define NT_STATUS_INVALID_GROUP_ATTRIBUTES (0xC0000000 | 0x00a4) // ERRHRD, ERRgeneral +#define NT_STATUS_BAD_IMPERSONATION_LEVEL (0xC0000000 | 0x00a5) // ERRHRD, ERRgeneral +#define NT_STATUS_CANT_OPEN_ANONYMOUS (0xC0000000 | 0x00a6) // ERRHRD, ERRgeneral +#define NT_STATUS_BAD_VALIDATION_CLASS (0xC0000000 | 0x00a7) // ERRHRD, ERRgeneral +#define NT_STATUS_BAD_TOKEN_TYPE (0xC0000000 | 0x00a8) // ERRHRD, ERRgeneral +#define NT_STATUS_BAD_MASTER_BOOT_RECORD (0xC0000000 | 0x00a9) // ERRDOS, 87 +#define NT_STATUS_INSTRUCTION_MISALIGNMENT (0xC0000000 | 0x00aa) // ERRHRD, ERRgeneral +#define NT_STATUS_INSTANCE_NOT_AVAILABLE (0xC0000000 | 0x00ab) // ERRDOS, ERRpipebusy +#define NT_STATUS_PIPE_NOT_AVAILABLE (0xC0000000 | 0x00ac) // ERRDOS, ERRpipebusy +#define NT_STATUS_INVALID_PIPE_STATE (0xC0000000 | 0x00ad) // ERRDOS, ERRbadpipe +#define NT_STATUS_PIPE_BUSY (0xC0000000 | 0x00ae) // ERRDOS, ERRpipebusy +#define NT_STATUS_ILLEGAL_FUNCTION (0xC0000000 | 0x00af) // ERRDOS, ERRbadfunc +#define NT_STATUS_PIPE_DISCONNECTED (0xC0000000 | 0x00b0) // ERRDOS, ERRnotconnected +#define NT_STATUS_PIPE_CLOSING (0xC0000000 | 0x00b1) // ERRDOS, ERRpipeclosing +#define NT_STATUS_PIPE_CONNECTED (0xC0000000 | 0x00b2) // ERRHRD, ERRgeneral +#define NT_STATUS_PIPE_LISTENING (0xC0000000 | 0x00b3) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_READ_MODE (0xC0000000 | 0x00b4) // ERRDOS, ERRbadpipe +#define NT_STATUS_IO_TIMEOUT (0xC0000000 | 0x00b5) // ERRDOS, 121 +#define NT_STATUS_FILE_FORCED_CLOSED (0xC0000000 | 0x00b6) // ERRDOS, 38 +#define NT_STATUS_PROFILING_NOT_STARTED (0xC0000000 | 0x00b7) // ERRHRD, ERRgeneral +#define NT_STATUS_PROFILING_NOT_STOPPED (0xC0000000 | 0x00b8) // ERRHRD, ERRgeneral +#define NT_STATUS_COULD_NOT_INTERPRET (0xC0000000 | 0x00b9) // ERRHRD, ERRgeneral +#define NT_STATUS_FILE_IS_A_DIRECTORY (0xC0000000 | 0x00ba) // ERRDOS, ERRnoaccess +#define NT_STATUS_NOT_SUPPORTED (0xC0000000 | 0x00bb) // ERRDOS, ERRunsup +#define NT_STATUS_REMOTE_NOT_LISTENING (0xC0000000 | 0x00bc) // ERRDOS, 51 +#define NT_STATUS_DUPLICATE_NAME (0xC0000000 | 0x00bd) // ERRDOS, 52 +#define NT_STATUS_BAD_NETWORK_PATH (0xC0000000 | 0x00be) // ERRDOS, 53 +#define NT_STATUS_NETWORK_BUSY (0xC0000000 | 0x00bf) // ERRDOS, 54 +#define NT_STATUS_DEVICE_DOES_NOT_EXIST (0xC0000000 | 0x00c0) // ERRDOS, 55 +#define NT_STATUS_TOO_MANY_COMMANDS (0xC0000000 | 0x00c1) // ERRDOS, 56 +#define NT_STATUS_ADAPTER_HARDWARE_ERROR (0xC0000000 | 0x00c2) // ERRDOS, 57 +#define NT_STATUS_INVALID_NETWORK_RESPONSE (0xC0000000 | 0x00c3) // ERRDOS, 58 +#define NT_STATUS_UNEXPECTED_NETWORK_ERROR (0xC0000000 | 0x00c4) // ERRDOS, 59 +#define NT_STATUS_BAD_REMOTE_ADAPTER (0xC0000000 | 0x00c5) // ERRDOS, 60 +#define NT_STATUS_PRINT_QUEUE_FULL (0xC0000000 | 0x00c6) // ERRDOS, 61 +#define NT_STATUS_NO_SPOOL_SPACE (0xC0000000 | 0x00c7) // ERRDOS, 62 +#define NT_STATUS_PRINT_CANCELLED (0xC0000000 | 0x00c8) // ERRDOS, 63 +#define NT_STATUS_NETWORK_NAME_DELETED (0xC0000000 | 0x00c9) // ERRDOS, 64 +#define NT_STATUS_NETWORK_ACCESS_DENIED (0xC0000000 | 0x00ca) // ERRDOS, 65 +#define NT_STATUS_BAD_DEVICE_TYPE (0xC0000000 | 0x00cb) // ERRDOS, 66 +#define NT_STATUS_BAD_NETWORK_NAME (0xC0000000 | 0x00cc) // ERRDOS, ERRnosuchshare +#define NT_STATUS_TOO_MANY_NAMES (0xC0000000 | 0x00cd) // ERRDOS, 68 +#define NT_STATUS_TOO_MANY_SESSIONS (0xC0000000 | 0x00ce) // ERRDOS, 69 +#define NT_STATUS_SHARING_PAUSED (0xC0000000 | 0x00cf) // ERRDOS, 70 +#define NT_STATUS_REQUEST_NOT_ACCEPTED (0xC0000000 | 0x00d0) // ERRDOS, 71 +#define NT_STATUS_REDIRECTOR_PAUSED (0xC0000000 | 0x00d1) // ERRDOS, 72 +#define NT_STATUS_NET_WRITE_FAULT (0xC0000000 | 0x00d2) // ERRDOS, 88 +#define NT_STATUS_PROFILING_AT_LIMIT (0xC0000000 | 0x00d3) // ERRHRD, ERRgeneral +#define NT_STATUS_NOT_SAME_DEVICE (0xC0000000 | 0x00d4) // ERRDOS, ERRdiffdevice +#define NT_STATUS_FILE_RENAMED (0xC0000000 | 0x00d5) // ERRDOS, ERRnoaccess +#define NT_STATUS_VIRTUAL_CIRCUIT_CLOSED (0xC0000000 | 0x00d6) // ERRDOS, 240 +#define NT_STATUS_NO_SECURITY_ON_OBJECT (0xC0000000 | 0x00d7) // ERRHRD, ERRgeneral +#define NT_STATUS_CANT_WAIT (0xC0000000 | 0x00d8) // ERRHRD, ERRgeneral +#define NT_STATUS_PIPE_EMPTY (0xC0000000 | 0x00d9) // ERRDOS, ERRpipeclosing +#define NT_STATUS_CANT_ACCESS_DOMAIN_INFO (0xC0000000 | 0x00da) // ERRHRD, ERRgeneral +#define NT_STATUS_CANT_TERMINATE_SELF (0xC0000000 | 0x00db) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_SERVER_STATE (0xC0000000 | 0x00dc) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_DOMAIN_STATE (0xC0000000 | 0x00dd) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_DOMAIN_ROLE (0xC0000000 | 0x00de) // ERRHRD, ERRgeneral +#define NT_STATUS_NO_SUCH_DOMAIN (0xC0000000 | 0x00df) // ERRHRD, ERRgeneral +#define NT_STATUS_DOMAIN_EXISTS (0xC0000000 | 0x00e0) // ERRHRD, ERRgeneral +#define NT_STATUS_DOMAIN_LIMIT_EXCEEDED (0xC0000000 | 0x00e1) // ERRHRD, ERRgeneral +#define NT_STATUS_OPLOCK_NOT_GRANTED (0xC0000000 | 0x00e2) // ERRDOS, 300 +#define NT_STATUS_INVALID_OPLOCK_PROTOCOL (0xC0000000 | 0x00e3) // ERRDOS, 301 +#define NT_STATUS_INTERNAL_DB_CORRUPTION (0xC0000000 | 0x00e4) // ERRHRD, ERRgeneral +#define NT_STATUS_INTERNAL_ERROR (0xC0000000 | 0x00e5) // ERRHRD, ERRgeneral +#define NT_STATUS_GENERIC_NOT_MAPPED (0xC0000000 | 0x00e6) // ERRHRD, ERRgeneral +#define NT_STATUS_BAD_DESCRIPTOR_FORMAT (0xC0000000 | 0x00e7) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_USER_BUFFER (0xC0000000 | 0x00e8) // ERRHRD, ERRgeneral +#define NT_STATUS_UNEXPECTED_IO_ERROR (0xC0000000 | 0x00e9) // ERRHRD, ERRgeneral +#define NT_STATUS_UNEXPECTED_MM_CREATE_ERR (0xC0000000 | 0x00ea) // ERRHRD, ERRgeneral +#define NT_STATUS_UNEXPECTED_MM_MAP_ERROR (0xC0000000 | 0x00eb) // ERRHRD, ERRgeneral +#define NT_STATUS_UNEXPECTED_MM_EXTEND_ERR (0xC0000000 | 0x00ec) // ERRHRD, ERRgeneral +#define NT_STATUS_NOT_LOGON_PROCESS (0xC0000000 | 0x00ed) // ERRHRD, ERRgeneral +#define NT_STATUS_LOGON_SESSION_EXISTS (0xC0000000 | 0x00ee) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_PARAMETER_1 (0xC0000000 | 0x00ef) // ERRDOS, 87 +#define NT_STATUS_INVALID_PARAMETER_2 (0xC0000000 | 0x00f0) // ERRDOS, 87 +#define NT_STATUS_INVALID_PARAMETER_3 (0xC0000000 | 0x00f1) // ERRDOS, 87 +#define NT_STATUS_INVALID_PARAMETER_4 (0xC0000000 | 0x00f2) // ERRDOS, 87 +#define NT_STATUS_INVALID_PARAMETER_5 (0xC0000000 | 0x00f3) // ERRDOS, 87 +#define NT_STATUS_INVALID_PARAMETER_6 (0xC0000000 | 0x00f4) // ERRDOS, 87 +#define NT_STATUS_INVALID_PARAMETER_7 (0xC0000000 | 0x00f5) // ERRDOS, 87 +#define NT_STATUS_INVALID_PARAMETER_8 (0xC0000000 | 0x00f6) // ERRDOS, 87 +#define NT_STATUS_INVALID_PARAMETER_9 (0xC0000000 | 0x00f7) // ERRDOS, 87 +#define NT_STATUS_INVALID_PARAMETER_10 (0xC0000000 | 0x00f8) // ERRDOS, 87 +#define NT_STATUS_INVALID_PARAMETER_11 (0xC0000000 | 0x00f9) // ERRDOS, 87 +#define NT_STATUS_INVALID_PARAMETER_12 (0xC0000000 | 0x00fa) // ERRDOS, 87 +#define NT_STATUS_REDIRECTOR_NOT_STARTED (0xC0000000 | 0x00fb) // ERRDOS, ERRbadpath +#define NT_STATUS_REDIRECTOR_STARTED (0xC0000000 | 0x00fc) // ERRHRD, ERRgeneral +#define NT_STATUS_STACK_OVERFLOW (0xC0000000 | 0x00fd) // ERRHRD, ERRgeneral +#define NT_STATUS_NO_SUCH_PACKAGE (0xC0000000 | 0x00fe) // ERRHRD, ERRgeneral +#define NT_STATUS_BAD_FUNCTION_TABLE (0xC0000000 | 0x00ff) // ERRHRD, ERRgeneral +#define NT_STATUS_VARIABLE_NOT_FOUND (0xC0000000 | 0x0100) // ERRDOS, 203 +#define NT_STATUS_DIRECTORY_NOT_EMPTY (0xC0000000 | 0x0101) // ERRDOS, 145 +#define NT_STATUS_FILE_CORRUPT_ERROR (0xC0000000 | 0x0102) // ERRHRD, ERRgeneral +#define NT_STATUS_NOT_A_DIRECTORY (0xC0000000 | 0x0103) // ERRDOS, 267 +#define NT_STATUS_BAD_LOGON_SESSION_STATE (0xC0000000 | 0x0104) // ERRHRD, ERRgeneral +#define NT_STATUS_LOGON_SESSION_COLLISION (0xC0000000 | 0x0105) // ERRHRD, ERRgeneral +#define NT_STATUS_NAME_TOO_LONG (0xC0000000 | 0x0106) // ERRDOS, 206 +#define NT_STATUS_FILES_OPEN (0xC0000000 | 0x0107) // ERRDOS, 2401 +#define NT_STATUS_CONNECTION_IN_USE (0xC0000000 | 0x0108) // ERRDOS, 2404 +#define NT_STATUS_MESSAGE_NOT_FOUND (0xC0000000 | 0x0109) // ERRHRD, ERRgeneral +#define NT_STATUS_PROCESS_IS_TERMINATING (0xC0000000 | 0x010a) // ERRDOS, ERRnoaccess +#define NT_STATUS_INVALID_LOGON_TYPE (0xC0000000 | 0x010b) // ERRHRD, ERRgeneral +#define NT_STATUS_NO_GUID_TRANSLATION (0xC0000000 | 0x010c) // ERRHRD, ERRgeneral +#define NT_STATUS_CANNOT_IMPERSONATE (0xC0000000 | 0x010d) // ERRHRD, ERRgeneral +#define NT_STATUS_IMAGE_ALREADY_LOADED (0xC0000000 | 0x010e) // ERRHRD, ERRgeneral +#define NT_STATUS_ABIOS_NOT_PRESENT (0xC0000000 | 0x010f) // ERRHRD, ERRgeneral +#define NT_STATUS_ABIOS_LID_NOT_EXIST (0xC0000000 | 0x0110) // ERRHRD, ERRgeneral +#define NT_STATUS_ABIOS_LID_ALREADY_OWNED (0xC0000000 | 0x0111) // ERRHRD, ERRgeneral +#define NT_STATUS_ABIOS_NOT_LID_OWNER (0xC0000000 | 0x0112) // ERRHRD, ERRgeneral +#define NT_STATUS_ABIOS_INVALID_COMMAND (0xC0000000 | 0x0113) // ERRHRD, ERRgeneral +#define NT_STATUS_ABIOS_INVALID_LID (0xC0000000 | 0x0114) // ERRHRD, ERRgeneral +#define NT_STATUS_ABIOS_SELECTOR_NOT_AVAILABLE (0xC0000000 | 0x0115) // ERRHRD, ERRgeneral +#define NT_STATUS_ABIOS_INVALID_SELECTOR (0xC0000000 | 0x0116) // ERRHRD, ERRgeneral +#define NT_STATUS_NO_LDT (0xC0000000 | 0x0117) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_LDT_SIZE (0xC0000000 | 0x0118) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_LDT_OFFSET (0xC0000000 | 0x0119) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_LDT_DESCRIPTOR (0xC0000000 | 0x011a) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_IMAGE_NE_FORMAT (0xC0000000 | 0x011b) // ERRDOS, 193 +#define NT_STATUS_RXACT_INVALID_STATE (0xC0000000 | 0x011c) // ERRHRD, ERRgeneral +#define NT_STATUS_RXACT_COMMIT_FAILURE (0xC0000000 | 0x011d) // ERRHRD, ERRgeneral +#define NT_STATUS_MAPPED_FILE_SIZE_ZERO (0xC0000000 | 0x011e) // ERRHRD, ERRgeneral +#define NT_STATUS_TOO_MANY_OPENED_FILES (0xC0000000 | 0x011f) // ERRDOS, ERRnofids +#define NT_STATUS_CANCELLED (0xC0000000 | 0x0120) // ERRHRD, ERRgeneral +#define NT_STATUS_CANNOT_DELETE (0xC0000000 | 0x0121) // ERRDOS, ERRnoaccess +#define NT_STATUS_INVALID_COMPUTER_NAME (0xC0000000 | 0x0122) // ERRHRD, ERRgeneral +#define NT_STATUS_FILE_DELETED (0xC0000000 | 0x0123) // ERRDOS, ERRnoaccess +#define NT_STATUS_SPECIAL_ACCOUNT (0xC0000000 | 0x0124) // ERRHRD, ERRgeneral +#define NT_STATUS_SPECIAL_GROUP (0xC0000000 | 0x0125) // ERRHRD, ERRgeneral +#define NT_STATUS_SPECIAL_USER (0xC0000000 | 0x0126) // ERRHRD, ERRgeneral +#define NT_STATUS_MEMBERS_PRIMARY_GROUP (0xC0000000 | 0x0127) // ERRHRD, ERRgeneral +#define NT_STATUS_FILE_CLOSED (0xC0000000 | 0x0128) // ERRDOS, ERRbadfid +#define NT_STATUS_TOO_MANY_THREADS (0xC0000000 | 0x0129) // ERRHRD, ERRgeneral +#define NT_STATUS_THREAD_NOT_IN_PROCESS (0xC0000000 | 0x012a) // ERRHRD, ERRgeneral +#define NT_STATUS_TOKEN_ALREADY_IN_USE (0xC0000000 | 0x012b) // ERRHRD, ERRgeneral +#define NT_STATUS_PAGEFILE_QUOTA_EXCEEDED (0xC0000000 | 0x012c) // ERRHRD, ERRgeneral +#define NT_STATUS_COMMITMENT_LIMIT (0xC0000000 | 0x012d) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_IMAGE_LE_FORMAT (0xC0000000 | 0x012e) // ERRDOS, 193 +#define NT_STATUS_INVALID_IMAGE_NOT_MZ (0xC0000000 | 0x012f) // ERRDOS, 193 +#define NT_STATUS_INVALID_IMAGE_PROTECT (0xC0000000 | 0x0130) // ERRDOS, 193 +#define NT_STATUS_INVALID_IMAGE_WIN_16 (0xC0000000 | 0x0131) // ERRDOS, 193 +#define NT_STATUS_LOGON_SERVER_CONFLICT (0xC0000000 | 0x0132) // ERRHRD, ERRgeneral +#define NT_STATUS_TIME_DIFFERENCE_AT_DC (0xC0000000 | 0x0133) // ERRHRD, ERRgeneral +#define NT_STATUS_SYNCHRONIZATION_REQUIRED (0xC0000000 | 0x0134) // ERRHRD, ERRgeneral +#define NT_STATUS_DLL_NOT_FOUND (0xC0000000 | 0x0135) // ERRDOS, 126 +#define NT_STATUS_OPEN_FAILED (0xC0000000 | 0x0136) // ERRHRD, ERRgeneral +#define NT_STATUS_IO_PRIVILEGE_FAILED (0xC0000000 | 0x0137) // ERRHRD, ERRgeneral +#define NT_STATUS_ORDINAL_NOT_FOUND (0xC0000000 | 0x0138) // ERRDOS, 182 +#define NT_STATUS_ENTRYPOINT_NOT_FOUND (0xC0000000 | 0x0139) // ERRDOS, 127 +#define NT_STATUS_CONTROL_C_EXIT (0xC0000000 | 0x013a) // ERRHRD, ERRgeneral +#define NT_STATUS_LOCAL_DISCONNECT (0xC0000000 | 0x013b) // ERRDOS, 64 +#define NT_STATUS_REMOTE_DISCONNECT (0xC0000000 | 0x013c) // ERRDOS, 64 +#define NT_STATUS_REMOTE_RESOURCES (0xC0000000 | 0x013d) // ERRDOS, 51 +#define NT_STATUS_LINK_FAILED (0xC0000000 | 0x013e) // ERRDOS, 59 +#define NT_STATUS_LINK_TIMEOUT (0xC0000000 | 0x013f) // ERRDOS, 59 +#define NT_STATUS_INVALID_CONNECTION (0xC0000000 | 0x0140) // ERRDOS, 59 +#define NT_STATUS_INVALID_ADDRESS (0xC0000000 | 0x0141) // ERRDOS, 59 +#define NT_STATUS_DLL_INIT_FAILED (0xC0000000 | 0x0142) // ERRHRD, ERRgeneral +#define NT_STATUS_MISSING_SYSTEMFILE (0xC0000000 | 0x0143) // ERRHRD, ERRgeneral +#define NT_STATUS_UNHANDLED_EXCEPTION (0xC0000000 | 0x0144) // ERRHRD, ERRgeneral +#define NT_STATUS_APP_INIT_FAILURE (0xC0000000 | 0x0145) // ERRHRD, ERRgeneral +#define NT_STATUS_PAGEFILE_CREATE_FAILED (0xC0000000 | 0x0146) // ERRHRD, ERRgeneral +#define NT_STATUS_NO_PAGEFILE (0xC0000000 | 0x0147) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_LEVEL (0xC0000000 | 0x0148) // ERRDOS, 124 +#define NT_STATUS_WRONG_PASSWORD_CORE (0xC0000000 | 0x0149) // ERRDOS, 86 +#define NT_STATUS_ILLEGAL_FLOAT_CONTEXT (0xC0000000 | 0x014a) // ERRHRD, ERRgeneral +#define NT_STATUS_PIPE_BROKEN (0xC0000000 | 0x014b) // ERRDOS, 109 +#define NT_STATUS_REGISTRY_CORRUPT (0xC0000000 | 0x014c) // ERRHRD, ERRgeneral +#define NT_STATUS_REGISTRY_IO_FAILED (0xC0000000 | 0x014d) // ERRHRD, ERRgeneral +#define NT_STATUS_NO_EVENT_PAIR (0xC0000000 | 0x014e) // ERRHRD, ERRgeneral +#define NT_STATUS_UNRECOGNIZED_VOLUME (0xC0000000 | 0x014f) // ERRHRD, ERRgeneral +#define NT_STATUS_SERIAL_NO_DEVICE_INITED (0xC0000000 | 0x0150) // ERRHRD, ERRgeneral +#define NT_STATUS_NO_SUCH_ALIAS (0xC0000000 | 0x0151) // ERRHRD, ERRgeneral +#define NT_STATUS_MEMBER_NOT_IN_ALIAS (0xC0000000 | 0x0152) // ERRHRD, ERRgeneral +#define NT_STATUS_MEMBER_IN_ALIAS (0xC0000000 | 0x0153) // ERRHRD, ERRgeneral +#define NT_STATUS_ALIAS_EXISTS (0xC0000000 | 0x0154) // ERRHRD, ERRgeneral +#define NT_STATUS_LOGON_NOT_GRANTED (0xC0000000 | 0x0155) // ERRHRD, ERRgeneral +#define NT_STATUS_TOO_MANY_SECRETS (0xC0000000 | 0x0156) // ERRHRD, ERRgeneral +#define NT_STATUS_SECRET_TOO_LONG (0xC0000000 | 0x0157) // ERRHRD, ERRgeneral +#define NT_STATUS_INTERNAL_DB_ERROR (0xC0000000 | 0x0158) // ERRHRD, ERRgeneral +#define NT_STATUS_FULLSCREEN_MODE (0xC0000000 | 0x0159) // ERRHRD, ERRgeneral +#define NT_STATUS_TOO_MANY_CONTEXT_IDS (0xC0000000 | 0x015a) // ERRHRD, ERRgeneral +#define NT_STATUS_LOGON_TYPE_NOT_GRANTED (0xC0000000 | 0x015b) // ERRDOS, ERRnoaccess +#define NT_STATUS_NOT_REGISTRY_FILE (0xC0000000 | 0x015c) // ERRHRD, ERRgeneral +#define NT_STATUS_NT_CROSS_ENCRYPTION_REQUIRED (0xC0000000 | 0x015d) // ERRHRD, ERRgeneral +#define NT_STATUS_DOMAIN_CTRLR_CONFIG_ERROR (0xC0000000 | 0x015e) // ERRHRD, ERRgeneral +#define NT_STATUS_FT_MISSING_MEMBER (0xC0000000 | 0x015f) // ERRHRD, ERRgeneral +#define NT_STATUS_ILL_FORMED_SERVICE_ENTRY (0xC0000000 | 0x0160) // ERRHRD, ERRgeneral +#define NT_STATUS_ILLEGAL_CHARACTER (0xC0000000 | 0x0161) // ERRHRD, ERRgeneral +#define NT_STATUS_UNMAPPABLE_CHARACTER (0xC0000000 | 0x0162) // ERRHRD, ERRgeneral +#define NT_STATUS_UNDEFINED_CHARACTER (0xC0000000 | 0x0163) // ERRHRD, ERRgeneral +#define NT_STATUS_FLOPPY_VOLUME (0xC0000000 | 0x0164) // ERRHRD, ERRgeneral +#define NT_STATUS_FLOPPY_ID_MARK_NOT_FOUND (0xC0000000 | 0x0165) // ERRHRD, ERRgeneral +#define NT_STATUS_FLOPPY_WRONG_CYLINDER (0xC0000000 | 0x0166) // ERRHRD, ERRgeneral +#define NT_STATUS_FLOPPY_UNKNOWN_ERROR (0xC0000000 | 0x0167) // ERRHRD, ERRgeneral +#define NT_STATUS_FLOPPY_BAD_REGISTERS (0xC0000000 | 0x0168) // ERRHRD, ERRgeneral +#define NT_STATUS_DISK_RECALIBRATE_FAILED (0xC0000000 | 0x0169) // ERRHRD, ERRgeneral +#define NT_STATUS_DISK_OPERATION_FAILED (0xC0000000 | 0x016a) // ERRHRD, ERRgeneral +#define NT_STATUS_DISK_RESET_FAILED (0xC0000000 | 0x016b) // ERRHRD, ERRgeneral +#define NT_STATUS_SHARED_IRQ_BUSY (0xC0000000 | 0x016c) // ERRHRD, ERRgeneral +#define NT_STATUS_FT_ORPHANING (0xC0000000 | 0x016d) // ERRHRD, ERRgeneral +#define NT_STATUS_BIOS_FAILED_TO_CONNECT_INTERRUPT (0xC0000000 | 0x016e) // ERRHRD, ERRgeneral +#define NT_STATUS_PARTITION_FAILURE (0xC0000000 | 0x0172) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_BLOCK_LENGTH (0xC0000000 | 0x0173) // ERRHRD, ERRgeneral +#define NT_STATUS_DEVICE_NOT_PARTITIONED (0xC0000000 | 0x0174) // ERRHRD, ERRgeneral +#define NT_STATUS_UNABLE_TO_LOCK_MEDIA (0xC0000000 | 0x0175) // ERRHRD, ERRgeneral +#define NT_STATUS_UNABLE_TO_UNLOAD_MEDIA (0xC0000000 | 0x0176) // ERRHRD, ERRgeneral +#define NT_STATUS_EOM_OVERFLOW (0xC0000000 | 0x0177) // ERRHRD, ERRgeneral +#define NT_STATUS_NO_MEDIA (0xC0000000 | 0x0178) // ERRHRD, ERRgeneral +#define NT_STATUS_NO_SUCH_MEMBER (0xC0000000 | 0x017a) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_MEMBER (0xC0000000 | 0x017b) // ERRHRD, ERRgeneral +#define NT_STATUS_KEY_DELETED (0xC0000000 | 0x017c) // ERRHRD, ERRgeneral +#define NT_STATUS_NO_LOG_SPACE (0xC0000000 | 0x017d) // ERRHRD, ERRgeneral +#define NT_STATUS_TOO_MANY_SIDS (0xC0000000 | 0x017e) // ERRHRD, ERRgeneral +#define NT_STATUS_LM_CROSS_ENCRYPTION_REQUIRED (0xC0000000 | 0x017f) // ERRHRD, ERRgeneral +#define NT_STATUS_KEY_HAS_CHILDREN (0xC0000000 | 0x0180) // ERRHRD, ERRgeneral +#define NT_STATUS_CHILD_MUST_BE_VOLATILE (0xC0000000 | 0x0181) // ERRHRD, ERRgeneral +#define NT_STATUS_DEVICE_CONFIGURATION_ERROR (0xC0000000 | 0x0182) // ERRDOS, 87 +#define NT_STATUS_DRIVER_INTERNAL_ERROR (0xC0000000 | 0x0183) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_DEVICE_STATE (0xC0000000 | 0x0184) // ERRDOS, 22 +#define NT_STATUS_IO_DEVICE_ERROR (0xC0000000 | 0x0185) // ERRHRD, ERRgeneral +#define NT_STATUS_DEVICE_PROTOCOL_ERROR (0xC0000000 | 0x0186) // ERRHRD, ERRgeneral +#define NT_STATUS_BACKUP_CONTROLLER (0xC0000000 | 0x0187) // ERRHRD, ERRgeneral +#define NT_STATUS_LOG_FILE_FULL (0xC0000000 | 0x0188) // ERRHRD, ERRgeneral +#define NT_STATUS_TOO_LATE (0xC0000000 | 0x0189) // ERRDOS, 19 +#define NT_STATUS_NO_TRUST_LSA_SECRET (0xC0000000 | 0x018a) // ERRDOS, ERRnoaccess +#define NT_STATUS_NO_TRUST_SAM_ACCOUNT (0xC0000000 | 0x018b) // ERRDOS, ERRnoaccess +#define NT_STATUS_TRUSTED_DOMAIN_FAILURE (0xC0000000 | 0x018c) // ERRDOS, ERRnoaccess +#define NT_STATUS_TRUSTED_RELATIONSHIP_FAILURE (0xC0000000 | 0x018d) // ERRDOS, ERRnoaccess +#define NT_STATUS_EVENTLOG_FILE_CORRUPT (0xC0000000 | 0x018e) // ERRHRD, ERRgeneral +#define NT_STATUS_EVENTLOG_CANT_START (0xC0000000 | 0x018f) // ERRHRD, ERRgeneral +#define NT_STATUS_TRUST_FAILURE (0xC0000000 | 0x0190) // ERRDOS, ERRnoaccess +#define NT_STATUS_MUTANT_LIMIT_EXCEEDED (0xC0000000 | 0x0191) // ERRHRD, ERRgeneral +#define NT_STATUS_NETLOGON_NOT_STARTED (0xC0000000 | 0x0192) // ERRDOS, ERRnetlogonNotStarted +#define NT_STATUS_ACCOUNT_EXPIRED (0xC0000000 | 0x0193) // ERRSRV, ERRaccountexpired +#define NT_STATUS_POSSIBLE_DEADLOCK (0xC0000000 | 0x0194) // ERRHRD, ERRgeneral +#define NT_STATUS_NETWORK_CREDENTIAL_CONFLICT (0xC0000000 | 0x0195) // ERRHRD, ERRgeneral +#define NT_STATUS_REMOTE_SESSION_LIMIT (0xC0000000 | 0x0196) // ERRHRD, ERRgeneral +#define NT_STATUS_EVENTLOG_FILE_CHANGED (0xC0000000 | 0x0197) // ERRHRD, ERRgeneral +#define NT_STATUS_NOLOGON_INTERDOMAIN_TRUST_ACCOUNT (0xC0000000 | 0x0198) // ERRDOS, ERRnoaccess +#define NT_STATUS_NOLOGON_WORKSTATION_TRUST_ACCOUNT (0xC0000000 | 0x0199) // ERRDOS, ERRnoaccess +#define NT_STATUS_NOLOGON_SERVER_TRUST_ACCOUNT (0xC0000000 | 0x019a) // ERRDOS, ERRnoaccess +#define NT_STATUS_DOMAIN_TRUST_INCONSISTENT (0xC0000000 | 0x019b) // ERRDOS, ERRnoaccess +#define NT_STATUS_FS_DRIVER_REQUIRED (0xC0000000 | 0x019c) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_LOCK_RANGE (0xC0000000 | 0x01a1) // ERRHRD, ERRgeneral +#define NT_STATUS_NO_USER_SESSION_KEY (0xC0000000 | 0x0202) // ERRHRD, ERRgeneral +#define NT_STATUS_USER_SESSION_DELETED (0xC0000000 | 0x0203) // ERRDOS, 59 +#define NT_STATUS_RESOURCE_LANG_NOT_FOUND (0xC0000000 | 0x0204) // ERRHRD, ERRgeneral +#define NT_STATUS_INSUFF_SERVER_RESOURCES (0xC0000000 | 0x0205) // ERRDOS, ERRnoresource +#define NT_STATUS_INVALID_BUFFER_SIZE (0xC0000000 | 0x0206) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_ADDRESS_COMPONENT (0xC0000000 | 0x0207) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_ADDRESS_WILDCARD (0xC0000000 | 0x0208) // ERRHRD, ERRgeneral +#define NT_STATUS_TOO_MANY_ADDRESSES (0xC0000000 | 0x0209) // ERRDOS, 68 +#define NT_STATUS_ADDRESS_ALREADY_EXISTS (0xC0000000 | 0x020a) // ERRDOS, 52 +#define NT_STATUS_ADDRESS_CLOSED (0xC0000000 | 0x020b) // ERRDOS, 64 +#define NT_STATUS_CONNECTION_DISCONNECTED (0xC0000000 | 0x020c) // ERRDOS, 64 +#define NT_STATUS_CONNECTION_RESET (0xC0000000 | 0x020d) // ERRDOS, 64 +#define NT_STATUS_TOO_MANY_NODES (0xC0000000 | 0x020e) // ERRDOS, 68 +#define NT_STATUS_TRANSACTION_ABORTED (0xC0000000 | 0x020f) // ERRDOS, 59 +#define NT_STATUS_TRANSACTION_TIMED_OUT (0xC0000000 | 0x0210) // ERRDOS, 59 +#define NT_STATUS_TRANSACTION_NO_RELEASE (0xC0000000 | 0x0211) // ERRDOS, 59 +#define NT_STATUS_TRANSACTION_NO_MATCH (0xC0000000 | 0x0212) // ERRDOS, 59 +#define NT_STATUS_TRANSACTION_RESPONDED (0xC0000000 | 0x0213) // ERRDOS, 59 +#define NT_STATUS_TRANSACTION_INVALID_ID (0xC0000000 | 0x0214) // ERRDOS, 59 +#define NT_STATUS_TRANSACTION_INVALID_TYPE (0xC0000000 | 0x0215) // ERRDOS, 59 +#define NT_STATUS_NOT_SERVER_SESSION (0xC0000000 | 0x0216) // ERRDOS, ERRunsup +#define NT_STATUS_NOT_CLIENT_SESSION (0xC0000000 | 0x0217) // ERRDOS, ERRunsup +#define NT_STATUS_CANNOT_LOAD_REGISTRY_FILE (0xC0000000 | 0x0218) // ERRHRD, ERRgeneral +#define NT_STATUS_DEBUG_ATTACH_FAILED (0xC0000000 | 0x0219) // ERRHRD, ERRgeneral +#define NT_STATUS_SYSTEM_PROCESS_TERMINATED (0xC0000000 | 0x021a) // ERRHRD, ERRgeneral +#define NT_STATUS_DATA_NOT_ACCEPTED (0xC0000000 | 0x021b) // ERRHRD, ERRgeneral +#define NT_STATUS_NO_BROWSER_SERVERS_FOUND (0xC0000000 | 0x021c) // ERRHRD, ERRgeneral +#define NT_STATUS_VDM_HARD_ERROR (0xC0000000 | 0x021d) // ERRHRD, ERRgeneral +#define NT_STATUS_DRIVER_CANCEL_TIMEOUT (0xC0000000 | 0x021e) // ERRHRD, ERRgeneral +#define NT_STATUS_REPLY_MESSAGE_MISMATCH (0xC0000000 | 0x021f) // ERRHRD, ERRgeneral +#define NT_STATUS_MAPPED_ALIGNMENT (0xC0000000 | 0x0220) // ERRHRD, ERRgeneral +#define NT_STATUS_IMAGE_CHECKSUM_MISMATCH (0xC0000000 | 0x0221) // ERRDOS, 193 +#define NT_STATUS_LOST_WRITEBEHIND_DATA (0xC0000000 | 0x0222) // ERRHRD, ERRgeneral +#define NT_STATUS_CLIENT_SERVER_PARAMETERS_INVALID (0xC0000000 | 0x0223) // ERRHRD, ERRgeneral +#define NT_STATUS_PASSWORD_MUST_CHANGE (0xC0000000 | 0x0224) // ERRSRV, ERRpasswordExpired +#define NT_STATUS_NOT_FOUND (0xC0000000 | 0x0225) // ERRHRD, ERRgeneral +#define NT_STATUS_NOT_TINY_STREAM (0xC0000000 | 0x0226) // ERRHRD, ERRgeneral +#define NT_STATUS_RECOVERY_FAILURE (0xC0000000 | 0x0227) // ERRHRD, ERRgeneral +#define NT_STATUS_STACK_OVERFLOW_READ (0xC0000000 | 0x0228) // ERRHRD, ERRgeneral +#define NT_STATUS_FAIL_CHECK (0xC0000000 | 0x0229) // ERRHRD, ERRgeneral +#define NT_STATUS_DUPLICATE_OBJECTID (0xC0000000 | 0x022a) // ERRHRD, ERRgeneral +#define NT_STATUS_OBJECTID_EXISTS (0xC0000000 | 0x022b) // ERRHRD, ERRgeneral +#define NT_STATUS_CONVERT_TO_LARGE (0xC0000000 | 0x022c) // ERRHRD, ERRgeneral +#define NT_STATUS_RETRY (0xC0000000 | 0x022d) // ERRHRD, ERRgeneral +#define NT_STATUS_FOUND_OUT_OF_SCOPE (0xC0000000 | 0x022e) // ERRHRD, ERRgeneral +#define NT_STATUS_ALLOCATE_BUCKET (0xC0000000 | 0x022f) // ERRHRD, ERRgeneral +#define NT_STATUS_PROPSET_NOT_FOUND (0xC0000000 | 0x0230) // ERRHRD, ERRgeneral +#define NT_STATUS_MARSHALL_OVERFLOW (0xC0000000 | 0x0231) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_VARIANT (0xC0000000 | 0x0232) // ERRHRD, ERRgeneral +#define NT_STATUS_DOMAIN_CONTROLLER_NOT_FOUND (0xC0000000 | 0x0233) // ERRHRD, ERRgeneral +#define NT_STATUS_ACCOUNT_LOCKED_OUT (0xC0000000 | 0x0234) // ERRDOS, ERRnoaccess +#define NT_STATUS_HANDLE_NOT_CLOSABLE (0xC0000000 | 0x0235) // ERRDOS, ERRbadfid +#define NT_STATUS_CONNECTION_REFUSED (0xC0000000 | 0x0236) // ERRHRD, ERRgeneral +#define NT_STATUS_GRACEFUL_DISCONNECT (0xC0000000 | 0x0237) // ERRHRD, ERRgeneral +#define NT_STATUS_ADDRESS_ALREADY_ASSOCIATED (0xC0000000 | 0x0238) // ERRHRD, ERRgeneral +#define NT_STATUS_ADDRESS_NOT_ASSOCIATED (0xC0000000 | 0x0239) // ERRHRD, ERRgeneral +#define NT_STATUS_CONNECTION_INVALID (0xC0000000 | 0x023a) // ERRHRD, ERRgeneral +#define NT_STATUS_CONNECTION_ACTIVE (0xC0000000 | 0x023b) // ERRHRD, ERRgeneral +#define NT_STATUS_NETWORK_UNREACHABLE (0xC0000000 | 0x023c) // ERRHRD, ERRgeneral +#define NT_STATUS_HOST_UNREACHABLE (0xC0000000 | 0x023d) // ERRHRD, ERRgeneral +#define NT_STATUS_PROTOCOL_UNREACHABLE (0xC0000000 | 0x023e) // ERRHRD, ERRgeneral +#define NT_STATUS_PORT_UNREACHABLE (0xC0000000 | 0x023f) // ERRHRD, ERRgeneral +#define NT_STATUS_REQUEST_ABORTED (0xC0000000 | 0x0240) // ERRHRD, ERRgeneral +#define NT_STATUS_CONNECTION_ABORTED (0xC0000000 | 0x0241) // ERRHRD, ERRgeneral +#define NT_STATUS_BAD_COMPRESSION_BUFFER (0xC0000000 | 0x0242) // ERRHRD, ERRgeneral +#define NT_STATUS_USER_MAPPED_FILE (0xC0000000 | 0x0243) // ERRHRD, ERRgeneral +#define NT_STATUS_AUDIT_FAILED (0xC0000000 | 0x0244) // ERRHRD, ERRgeneral +#define NT_STATUS_TIMER_RESOLUTION_NOT_SET (0xC0000000 | 0x0245) // ERRHRD, ERRgeneral +#define NT_STATUS_CONNECTION_COUNT_LIMIT (0xC0000000 | 0x0246) // ERRHRD, ERRgeneral +#define NT_STATUS_LOGIN_TIME_RESTRICTION (0xC0000000 | 0x0247) // ERRHRD, ERRgeneral +#define NT_STATUS_LOGIN_WKSTA_RESTRICTION (0xC0000000 | 0x0248) // ERRHRD, ERRgeneral +#define NT_STATUS_IMAGE_MP_UP_MISMATCH (0xC0000000 | 0x0249) // ERRDOS, 193 +#define NT_STATUS_INSUFFICIENT_LOGON_INFO (0xC0000000 | 0x0250) // ERRHRD, ERRgeneral +#define NT_STATUS_BAD_DLL_ENTRYPOINT (0xC0000000 | 0x0251) // ERRHRD, ERRgeneral +#define NT_STATUS_BAD_SERVICE_ENTRYPOINT (0xC0000000 | 0x0252) // ERRHRD, ERRgeneral +#define NT_STATUS_LPC_REPLY_LOST (0xC0000000 | 0x0253) // ERRHRD, ERRgeneral +#define NT_STATUS_IP_ADDRESS_CONFLICT1 (0xC0000000 | 0x0254) // ERRHRD, ERRgeneral +#define NT_STATUS_IP_ADDRESS_CONFLICT2 (0xC0000000 | 0x0255) // ERRHRD, ERRgeneral +#define NT_STATUS_REGISTRY_QUOTA_LIMIT (0xC0000000 | 0x0256) // ERRHRD, ERRgeneral +#define NT_STATUS_PATH_NOT_COVERED (0xC0000000 | 0x0257) // ERRSRV, 3 +#define NT_STATUS_NO_CALLBACK_ACTIVE (0xC0000000 | 0x0258) // ERRHRD, ERRgeneral +#define NT_STATUS_LICENSE_QUOTA_EXCEEDED (0xC0000000 | 0x0259) // ERRHRD, ERRgeneral +#define NT_STATUS_PWD_TOO_SHORT (0xC0000000 | 0x025a) // ERRHRD, ERRgeneral +#define NT_STATUS_PWD_TOO_RECENT (0xC0000000 | 0x025b) // ERRHRD, ERRgeneral +#define NT_STATUS_PWD_HISTORY_CONFLICT (0xC0000000 | 0x025c) // ERRHRD, ERRgeneral +#define NT_STATUS_PLUGPLAY_NO_DEVICE (0xC0000000 | 0x025e) // ERRHRD, ERRgeneral +#define NT_STATUS_UNSUPPORTED_COMPRESSION (0xC0000000 | 0x025f) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_HW_PROFILE (0xC0000000 | 0x0260) // ERRHRD, ERRgeneral +#define NT_STATUS_INVALID_PLUGPLAY_DEVICE_PATH (0xC0000000 | 0x0261) // ERRHRD, ERRgeneral +#define NT_STATUS_DRIVER_ORDINAL_NOT_FOUND (0xC0000000 | 0x0262) // ERRDOS, 182 +#define NT_STATUS_DRIVER_ENTRYPOINT_NOT_FOUND (0xC0000000 | 0x0263) // ERRDOS, 127 +#define NT_STATUS_RESOURCE_NOT_OWNED (0xC0000000 | 0x0264) // ERRDOS, 288 +#define NT_STATUS_TOO_MANY_LINKS (0xC0000000 | 0x0265) // ERRDOS, ErrTooManyLinks +#define NT_STATUS_QUOTA_LIST_INCONSISTENT (0xC0000000 | 0x0266) // ERRHRD, ERRgeneral +#define NT_STATUS_FILE_IS_OFFLINE (0xC0000000 | 0x0267) // ERRHRD, ERRgeneral +#define NT_STATUS_VOLUME_DISMOUNTED (0xC0000000 | 0x026e) // ERRDOS, 21 +#define NT_STATUS_NOT_A_REPARSE_POINT (0xC0000000 | 0x0275) // ERRHRD, ERRgeneral +#define NT_STATUS_DIRECTORY_IS_A_REPARSE_POINT (0xC0000000 | 0x0281) // ERRDOS, 161 +#define NT_STATUS_ENCRYPTION_FAILED (0xC0000000 | 0x028a) // ERRDOS, ERRnoaccess +#define NT_STATUS_DECRYPTION_FAILED (0xC0000000 | 0x028b) // ERRDOS, ERRnoaccess +#define NT_STATUS_RANGE_NOT_FOUND (0xC0000000 | 0x028c) // ERRHRD, ERRgeneral +#define NT_STATUS_NO_RECOVERY_POLICY (0xC0000000 | 0x028d) // ERRDOS, ERRnoaccess +#define NT_STATUS_NO_EFS (0xC0000000 | 0x028e) // ERRDOS, ERRnoaccess +#define NT_STATUS_WRONG_EFS (0xC0000000 | 0x028f) // ERRDOS, ERRnoaccess +#define NT_STATUS_NO_USER_KEYS (0xC0000000 | 0x0290) // ERRDOS, ERRnoaccess +#define NT_STATUS_VOLUME_NOT_UPGRADED (0xC0000000 | 0x029c) // ERRDOS, ERRbadfunc +#define NT_STATUS_NETWORK_SESSION_EXPIRED (0xC0000000 | 0x035c) // ERRHRD, ERRgeneral +/* scheduler */ +#define NT_STATUS_NO_SUCH_JOB (0xC0000000 | 0xEDE) // ERRHRD, ERRgeneral +#define NT_STATUS_NO_PREAUTH_INTEGRITY_HASH_OVERLAP (0xC0000000 | 0x5D0000) // ERRHRD, ERRgeneral +#define NT_STATUS_OS2_INVALID_LEVEL 0x007c0001 // ERRDOS, ERRunknownlevel + +#endif /* _NTERR_H */ diff --git a/fs/smb/client/ntlmssp.h b/fs/smb/client/ntlmssp.h new file mode 100644 index 000000000..be0365f08 --- /dev/null +++ b/fs/smb/client/ntlmssp.h @@ -0,0 +1,158 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +/* + * + * Copyright (c) International Business Machines Corp., 2002,2007 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ + +#define NTLMSSP_SIGNATURE "NTLMSSP" +/* Message Types */ +#define NtLmNegotiate cpu_to_le32(1) +#define NtLmChallenge cpu_to_le32(2) +#define NtLmAuthenticate cpu_to_le32(3) +#define UnknownMessage cpu_to_le32(8) + +/* Negotiate Flags */ +#define NTLMSSP_NEGOTIATE_UNICODE 0x01 /* Text strings are unicode */ +#define NTLMSSP_NEGOTIATE_OEM 0x02 /* Text strings are in OEM */ +#define NTLMSSP_REQUEST_TARGET 0x04 /* Srv returns its auth realm */ +/* define reserved9 0x08 */ +#define NTLMSSP_NEGOTIATE_SIGN 0x0010 /* Request signing capability */ +#define NTLMSSP_NEGOTIATE_SEAL 0x0020 /* Request confidentiality */ +#define NTLMSSP_NEGOTIATE_DGRAM 0x0040 +#define NTLMSSP_NEGOTIATE_LM_KEY 0x0080 /* Use LM session key */ +/* defined reserved 8 0x0100 */ +#define NTLMSSP_NEGOTIATE_NTLM 0x0200 /* NTLM authentication */ +#define NTLMSSP_NEGOTIATE_NT_ONLY 0x0400 /* Lanman not allowed */ +#define NTLMSSP_ANONYMOUS 0x0800 +#define NTLMSSP_NEGOTIATE_DOMAIN_SUPPLIED 0x1000 /* reserved6 */ +#define NTLMSSP_NEGOTIATE_WORKSTATION_SUPPLIED 0x2000 +#define NTLMSSP_NEGOTIATE_LOCAL_CALL 0x4000 /* client/server same machine */ +#define NTLMSSP_NEGOTIATE_ALWAYS_SIGN 0x8000 /* Sign. All security levels */ +#define NTLMSSP_TARGET_TYPE_DOMAIN 0x10000 +#define NTLMSSP_TARGET_TYPE_SERVER 0x20000 +#define NTLMSSP_TARGET_TYPE_SHARE 0x40000 +#define NTLMSSP_NEGOTIATE_EXTENDED_SEC 0x80000 /* NB:not related to NTLMv2 pwd*/ +/* #define NTLMSSP_REQUEST_INIT_RESP 0x100000 */ +#define NTLMSSP_NEGOTIATE_IDENTIFY 0x100000 +#define NTLMSSP_REQUEST_ACCEPT_RESP 0x200000 /* reserved5 */ +#define NTLMSSP_REQUEST_NON_NT_KEY 0x400000 +#define NTLMSSP_NEGOTIATE_TARGET_INFO 0x800000 +/* #define reserved4 0x1000000 */ +#define NTLMSSP_NEGOTIATE_VERSION 0x2000000 /* we only set for SMB2+ */ +/* #define reserved3 0x4000000 */ +/* #define reserved2 0x8000000 */ +/* #define reserved1 0x10000000 */ +#define NTLMSSP_NEGOTIATE_128 0x20000000 +#define NTLMSSP_NEGOTIATE_KEY_XCH 0x40000000 +#define NTLMSSP_NEGOTIATE_56 0x80000000 + +/* Define AV Pair Field IDs */ +enum av_field_type { + NTLMSSP_AV_EOL = 0, + NTLMSSP_AV_NB_COMPUTER_NAME, + NTLMSSP_AV_NB_DOMAIN_NAME, + NTLMSSP_AV_DNS_COMPUTER_NAME, + NTLMSSP_AV_DNS_DOMAIN_NAME, + NTLMSSP_AV_DNS_TREE_NAME, + NTLMSSP_AV_FLAGS, + NTLMSSP_AV_TIMESTAMP, + NTLMSSP_AV_RESTRICTION, + NTLMSSP_AV_TARGET_NAME, + NTLMSSP_AV_CHANNEL_BINDINGS +}; + +/* Although typedefs are not commonly used for structure definitions */ +/* in the Linux kernel, in this particular case they are useful */ +/* to more closely match the standards document for NTLMSSP from */ +/* OpenGroup and to make the code more closely match the standard in */ +/* appearance */ + +typedef struct _SECURITY_BUFFER { + __le16 Length; + __le16 MaximumLength; + __le32 BufferOffset; /* offset to buffer */ +} __packed SECURITY_BUFFER; + +typedef struct _NEGOTIATE_MESSAGE { + __u8 Signature[sizeof(NTLMSSP_SIGNATURE)]; + __le32 MessageType; /* NtLmNegotiate = 1 */ + __le32 NegotiateFlags; + SECURITY_BUFFER DomainName; /* RFC 1001 style and ASCII */ + SECURITY_BUFFER WorkstationName; /* RFC 1001 and ASCII */ + /* SECURITY_BUFFER for version info not present since we + do not set the version is present flag */ + char DomainString[]; + /* followed by WorkstationString */ +} __packed NEGOTIATE_MESSAGE, *PNEGOTIATE_MESSAGE; + +#define NTLMSSP_REVISION_W2K3 0x0F + +/* See MS-NLMP section 2.2.2.10 */ +struct ntlmssp_version { + __u8 ProductMajorVersion; + __u8 ProductMinorVersion; + __le16 ProductBuild; /* we send the cifs.ko module version here */ + __u8 Reserved[3]; + __u8 NTLMRevisionCurrent; /* currently 0x0F */ +} __packed; + +/* see MS-NLMP section 2.2.1.1 */ +struct negotiate_message { + __u8 Signature[sizeof(NTLMSSP_SIGNATURE)]; + __le32 MessageType; /* NtLmNegotiate = 1 */ + __le32 NegotiateFlags; + SECURITY_BUFFER DomainName; /* RFC 1001 style and ASCII */ + SECURITY_BUFFER WorkstationName; /* RFC 1001 and ASCII */ + struct ntlmssp_version Version; + /* SECURITY_BUFFER */ + char DomainString[]; + /* followed by WorkstationString */ +} __packed; + +typedef struct _CHALLENGE_MESSAGE { + __u8 Signature[sizeof(NTLMSSP_SIGNATURE)]; + __le32 MessageType; /* NtLmChallenge = 2 */ + SECURITY_BUFFER TargetName; + __le32 NegotiateFlags; + __u8 Challenge[CIFS_CRYPTO_KEY_SIZE]; + __u8 Reserved[8]; + SECURITY_BUFFER TargetInfoArray; + /* SECURITY_BUFFER for version info not present since we + do not set the version is present flag */ +} __packed CHALLENGE_MESSAGE, *PCHALLENGE_MESSAGE; + +typedef struct _AUTHENTICATE_MESSAGE { + __u8 Signature[sizeof(NTLMSSP_SIGNATURE)]; + __le32 MessageType; /* NtLmsAuthenticate = 3 */ + SECURITY_BUFFER LmChallengeResponse; + SECURITY_BUFFER NtChallengeResponse; + SECURITY_BUFFER DomainName; + SECURITY_BUFFER UserName; + SECURITY_BUFFER WorkstationName; + SECURITY_BUFFER SessionKey; + __le32 NegotiateFlags; + struct ntlmssp_version Version; + /* SECURITY_BUFFER */ + char UserString[]; +} __packed AUTHENTICATE_MESSAGE, *PAUTHENTICATE_MESSAGE; + +/* + * Size of the session key (crypto key encrypted with the password + */ + +int decode_ntlmssp_challenge(char *bcc_ptr, int blob_len, + struct cifs_ses *ses); +int build_ntlmssp_negotiate_blob(unsigned char **pbuffer, u16 *buflen, + struct cifs_ses *ses, + struct TCP_Server_Info *server, + const struct nls_table *nls_cp); +int build_ntlmssp_smb3_negotiate_blob(unsigned char **pbuffer, u16 *buflen, + struct cifs_ses *ses, + struct TCP_Server_Info *server, + const struct nls_table *nls_cp); +int build_ntlmssp_auth_blob(unsigned char **pbuffer, u16 *buflen, + struct cifs_ses *ses, + struct TCP_Server_Info *server, + const struct nls_table *nls_cp); diff --git a/fs/smb/client/readdir.c b/fs/smb/client/readdir.c new file mode 100644 index 000000000..32a75afca --- /dev/null +++ b/fs/smb/client/readdir.c @@ -0,0 +1,1235 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * Directory search handling + * + * Copyright (C) International Business Machines Corp., 2004, 2008 + * Copyright (C) Red Hat, Inc., 2011 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ +#include +#include +#include +#include +#include +#include "cifsglob.h" +#include "cifsproto.h" +#include "cifs_unicode.h" +#include "cifs_debug.h" +#include "cifs_fs_sb.h" +#include "cifsfs.h" +#include "smb2proto.h" +#include "fs_context.h" +#include "cached_dir.h" +#include "reparse.h" + +/* + * To be safe - for UCS to UTF-8 with strings loaded with the rare long + * characters alloc more to account for such multibyte target UTF-8 + * characters. + */ +#define UNICODE_NAME_MAX ((4 * NAME_MAX) + 2) + +#ifdef CONFIG_CIFS_DEBUG2 +static void dump_cifs_file_struct(struct file *file, char *label) +{ + struct cifsFileInfo *cf; + + if (file) { + cf = file->private_data; + if (cf == NULL) { + cifs_dbg(FYI, "empty cifs private file data\n"); + return; + } + if (cf->invalidHandle) + cifs_dbg(FYI, "Invalid handle\n"); + if (cf->srch_inf.endOfSearch) + cifs_dbg(FYI, "end of search\n"); + if (cf->srch_inf.emptyDir) + cifs_dbg(FYI, "empty dir\n"); + } +} +#else +static inline void dump_cifs_file_struct(struct file *file, char *label) +{ +} +#endif /* DEBUG2 */ + +/* + * Attempt to preload the dcache with the results from the FIND_FIRST/NEXT + * + * Find the dentry that matches "name". If there isn't one, create one. If it's + * a negative dentry or the uniqueid or filetype(mode) changed, + * then drop it and recreate it. + */ +static void +cifs_prime_dcache(struct dentry *parent, struct qstr *name, + struct cifs_fattr *fattr) +{ + struct dentry *dentry, *alias; + struct inode *inode; + struct super_block *sb = parent->d_sb; + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + bool posix = cifs_sb_master_tcon(cifs_sb)->posix_extensions; + bool reparse_need_reval = false; + int rc; + + cifs_dbg(FYI, "%s: for %s\n", __func__, name->name); + + dentry = try_lookup_noperm(name, parent); + if (!dentry) { + /* + * If we know that the inode will need to be revalidated + * immediately, then don't create a new dentry for it. + * We'll end up doing an on the wire call either way and + * this spares us an invalidation. + */ +retry: + if (posix) { + switch (fattr->cf_mode & S_IFMT) { + case S_IFLNK: + case S_IFBLK: + case S_IFCHR: + reparse_need_reval = true; + break; + default: + break; + } + } else if (fattr->cf_cifsattrs & ATTR_REPARSE_POINT) { + reparse_need_reval = true; + } + + if (reparse_need_reval || + (fattr->cf_flags & CIFS_FATTR_NEED_REVAL)) + return; + + dentry = d_alloc_parallel(parent, name); + } + if (IS_ERR(dentry)) + return; + if (!d_in_lookup(dentry)) { + inode = d_inode(dentry); + if (inode) { + if (d_mountpoint(dentry)) { + dput(dentry); + return; + } + /* + * If we're generating inode numbers, then we don't + * want to clobber the existing one with the one that + * the readdir code created. + */ + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SERVER_INUM)) + fattr->cf_uniqueid = CIFS_I(inode)->uniqueid; + + /* + * Update inode in place if both i_ino and i_mode didn't + * change. + */ + if (CIFS_I(inode)->uniqueid == fattr->cf_uniqueid) { + /* + * Query dir responses don't provide enough + * information about reparse points other than + * their reparse tags. Save an invalidation by + * not clobbering some existing attributes when + * reparse tag and ctime haven't changed. + */ + rc = 0; + if (fattr->cf_cifsattrs & ATTR_REPARSE_POINT) { + if (likely(reparse_inode_match(inode, fattr))) { + fattr->cf_mode = inode->i_mode; + fattr->cf_rdev = inode->i_rdev; + fattr->cf_uid = inode->i_uid; + fattr->cf_gid = inode->i_gid; + fattr->cf_eof = + netfs_read_remote_i_size(inode); + fattr->cf_symlink_target = NULL; + } else { + CIFS_I(inode)->time = 0; + rc = -ESTALE; + } + } + if (!rc && !cifs_fattr_to_inode(inode, fattr, true)) { + dput(dentry); + return; + } + } + } + d_invalidate(dentry); + dput(dentry); + goto retry; + } else { + inode = cifs_iget(sb, fattr); + if (!inode) + inode = ERR_PTR(-ENOMEM); + alias = d_splice_alias(inode, dentry); + d_lookup_done(dentry); + if (alias && !IS_ERR(alias)) + dput(alias); + } + dput(dentry); +} + +static void +cifs_fill_common_info(struct cifs_fattr *fattr, struct cifs_sb_info *cifs_sb) +{ + struct cifs_open_info_data data = { + .reparse = { .tag = fattr->cf_cifstag, }, + }; + unsigned int sbflags; + + fattr->cf_uid = cifs_sb->ctx->linux_uid; + fattr->cf_gid = cifs_sb->ctx->linux_gid; + + /* + * The IO_REPARSE_TAG_LX_ tags originally were used by WSL but they + * are preferred by the Linux client in some cases since, unlike + * the NFS reparse tag (or EAs), they don't require an extra query + * to determine which type of special file they represent. + * TODO: go through all documented reparse tags to see if we can + * reasonably map some of them to directories vs. files vs. symlinks + */ + if ((fattr->cf_cifsattrs & ATTR_REPARSE_POINT) && + cifs_reparse_point_to_fattr(cifs_sb, fattr, &data)) + goto out_reparse; + + if (fattr->cf_cifsattrs & ATTR_DIRECTORY) { + fattr->cf_mode = S_IFDIR | cifs_sb->ctx->dir_mode; + fattr->cf_dtype = DT_DIR; + } else { + fattr->cf_mode = S_IFREG | cifs_sb->ctx->file_mode; + fattr->cf_dtype = DT_REG; + } + +out_reparse: + /* non-unix readdir doesn't provide nlink */ + fattr->cf_flags |= CIFS_FATTR_UNKNOWN_NLINK; + + if (fattr->cf_cifsattrs & ATTR_READONLY) + fattr->cf_mode &= ~S_IWUGO; + + /* + * We of course don't get ACL info in FIND_FIRST/NEXT results, so + * mark it for revalidation so that "ls -l" will look right. It might + * be super-slow, but if we don't do this then the ownership of files + * may look wrong since the inodes may not have timed out by the time + * "ls" does a stat() call on them. + */ + sbflags = cifs_sb_flags(cifs_sb); + if (sbflags & (CIFS_MOUNT_CIFS_ACL | CIFS_MOUNT_MODE_FROM_SID)) + fattr->cf_flags |= CIFS_FATTR_NEED_REVAL; + + if ((sbflags & CIFS_MOUNT_UNX_EMUL) && + (fattr->cf_cifsattrs & ATTR_SYSTEM)) { + if (fattr->cf_eof == 0) { + fattr->cf_mode &= ~S_IFMT; + fattr->cf_mode |= S_IFIFO; + fattr->cf_dtype = DT_FIFO; + } else { + /* + * trying to get the type and mode via SFU can be slow, + * so just call those regular files for now, and mark + * for reval + */ + fattr->cf_flags |= CIFS_FATTR_NEED_REVAL; + } + } +} + +/* Fill a cifs_fattr struct with info from SMB_FIND_FILE_POSIX_INFO. */ +static void +cifs_posix_to_fattr(struct cifs_fattr *fattr, struct smb2_posix_info *info, + struct cifs_sb_info *cifs_sb) +{ + struct smb2_posix_info_parsed parsed; + + posix_info_parse(info, NULL, &parsed); + + memset(fattr, 0, sizeof(*fattr)); + fattr->cf_uniqueid = le64_to_cpu(info->Inode); + fattr->cf_bytes = le64_to_cpu(info->AllocationSize); + fattr->cf_eof = le64_to_cpu(info->EndOfFile); + + fattr->cf_atime = cifs_NTtimeToUnix(info->LastAccessTime); + fattr->cf_mtime = cifs_NTtimeToUnix(info->LastWriteTime); + fattr->cf_ctime = cifs_NTtimeToUnix(info->CreationTime); + + fattr->cf_nlink = le32_to_cpu(info->HardLinks); + fattr->cf_cifsattrs = le32_to_cpu(info->DosAttributes); + + if (fattr->cf_cifsattrs & ATTR_REPARSE_POINT) + fattr->cf_cifstag = le32_to_cpu(info->ReparseTag); + + /* The Mode field in the response can now include the file type as well */ + fattr->cf_mode = wire_mode_to_posix(le32_to_cpu(info->Mode), + fattr->cf_cifsattrs & ATTR_DIRECTORY); + fattr->cf_dtype = S_DT(fattr->cf_mode); + + switch (fattr->cf_mode & S_IFMT) { + case S_IFLNK: + case S_IFBLK: + case S_IFCHR: + fattr->cf_flags |= CIFS_FATTR_NEED_REVAL; + break; + default: + break; + } + + cifs_dbg(FYI, "posix fattr: dev %d, reparse %d, mode %o\n", + le32_to_cpu(info->DeviceId), + le32_to_cpu(info->ReparseTag), + le32_to_cpu(info->Mode)); + + sid_to_id(cifs_sb, &parsed.owner, fattr, SIDOWNER); + sid_to_id(cifs_sb, &parsed.group, fattr, SIDGROUP); +} + +static void __dir_info_to_fattr(struct cifs_fattr *fattr, const void *info) +{ + const FILE_DIRECTORY_INFO *fi = info; + + memset(fattr, 0, sizeof(*fattr)); + fattr->cf_cifsattrs = le32_to_cpu(fi->ExtFileAttributes); + fattr->cf_eof = le64_to_cpu(fi->EndOfFile); + fattr->cf_bytes = le64_to_cpu(fi->AllocationSize); + fattr->cf_createtime = le64_to_cpu(fi->CreationTime); + fattr->cf_atime = cifs_NTtimeToUnix(fi->LastAccessTime); + fattr->cf_ctime = cifs_NTtimeToUnix(fi->ChangeTime); + fattr->cf_mtime = cifs_NTtimeToUnix(fi->LastWriteTime); +} + +void +cifs_dir_info_to_fattr(struct cifs_fattr *fattr, FILE_DIRECTORY_INFO *info, + struct cifs_sb_info *cifs_sb) +{ + __dir_info_to_fattr(fattr, info); + cifs_fill_common_info(fattr, cifs_sb); +} + +static void cifs_fulldir_info_to_fattr(struct cifs_fattr *fattr, + const void *info, + struct cifs_sb_info *cifs_sb) +{ + const FILE_FULL_DIRECTORY_INFO *di = info; + + __dir_info_to_fattr(fattr, info); + + /* See MS-FSCC 2.4.14, 2.4.19 */ + if (fattr->cf_cifsattrs & ATTR_REPARSE_POINT) + fattr->cf_cifstag = le32_to_cpu(di->EaSize); + cifs_fill_common_info(fattr, cifs_sb); +} + +static void +cifs_std_info_to_fattr(struct cifs_fattr *fattr, FIND_FILE_STANDARD_INFO *info, + struct cifs_sb_info *cifs_sb) +{ + int offset = cifs_sb_master_tcon(cifs_sb)->ses->server->timeAdj; + + memset(fattr, 0, sizeof(*fattr)); + fattr->cf_atime = cnvrtDosUnixTm(info->LastAccessDate, + info->LastAccessTime, offset); + fattr->cf_ctime = cnvrtDosUnixTm(info->LastWriteDate, + info->LastWriteTime, offset); + fattr->cf_mtime = cnvrtDosUnixTm(info->LastWriteDate, + info->LastWriteTime, offset); + + fattr->cf_cifsattrs = le16_to_cpu(info->Attributes); + fattr->cf_bytes = le32_to_cpu(info->AllocationSize); + fattr->cf_eof = le32_to_cpu(info->DataSize); + + cifs_fill_common_info(fattr, cifs_sb); +} + +static int +_initiate_cifs_search(const unsigned int xid, struct file *file, + const char *full_path) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(file); + struct tcon_link *tlink = NULL; + struct TCP_Server_Info *server; + struct cifsFileInfo *cifsFile; + struct cifs_tcon *tcon; + unsigned int sbflags; + __u16 search_flags; + int rc = 0; + + if (file->private_data == NULL) { + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) + return PTR_ERR(tlink); + + cifsFile = kzalloc_obj(struct cifsFileInfo); + if (cifsFile == NULL) { + rc = -ENOMEM; + goto error_exit; + } + spin_lock_init(&cifsFile->file_info_lock); + file->private_data = cifsFile; + cifsFile->tlink = cifs_get_tlink(tlink); + tcon = tlink_tcon(tlink); + } else { + cifsFile = file->private_data; + tcon = tlink_tcon(cifsFile->tlink); + } + + server = tcon->ses->server; + + if (!server->ops->query_dir_first) { + rc = -ENOSYS; + goto error_exit; + } + + cifsFile->invalidHandle = true; + cifsFile->srch_inf.endOfSearch = false; + + cifs_dbg(FYI, "Full path: %s start at: %lld\n", full_path, file->f_pos); + +ffirst_retry: + sbflags = cifs_sb_flags(cifs_sb); + /* test for Unix extensions */ + /* but now check for them on the share/mount not on the SMB session */ + /* if (cap_unix(tcon->ses) { */ + if (tcon->unix_ext) + cifsFile->srch_inf.info_level = SMB_FIND_FILE_UNIX; + else if (tcon->posix_extensions) + cifsFile->srch_inf.info_level = SMB_FIND_FILE_POSIX_INFO; + else if ((tcon->ses->capabilities & + tcon->ses->server->vals->cap_nt_find) == 0) { + cifsFile->srch_inf.info_level = SMB_FIND_FILE_INFO_STANDARD; + } else if (sbflags & CIFS_MOUNT_SERVER_INUM) { + cifsFile->srch_inf.info_level = SMB_FIND_FILE_ID_FULL_DIR_INFO; + } else /* not srvinos - BB fixme add check for backlevel? */ { + cifsFile->srch_inf.info_level = SMB_FIND_FILE_FULL_DIRECTORY_INFO; + } + + search_flags = CIFS_SEARCH_CLOSE_AT_END | CIFS_SEARCH_RETURN_RESUME; + if (backup_cred(cifs_sb)) + search_flags |= CIFS_SEARCH_BACKUP_SEARCH; + + rc = server->ops->query_dir_first(xid, tcon, full_path, cifs_sb, + &cifsFile->fid, search_flags, + &cifsFile->srch_inf); + + if (rc == 0) { + cifsFile->invalidHandle = false; + } else if (rc == -EOPNOTSUPP && (sbflags & CIFS_MOUNT_SERVER_INUM)) { + cifs_autodisable_serverino(cifs_sb, "Cannot retrieve inode number via query_dir_first", rc); + goto ffirst_retry; + } +error_exit: + cifs_put_tlink(tlink); + return rc; +} + +static int +initiate_cifs_search(const unsigned int xid, struct file *file, + const char *full_path) +{ + int rc, retry_count = 0; + + do { + rc = _initiate_cifs_search(xid, file, full_path); + /* + * If we don't have enough credits to start reading the + * directory just try again after short wait. + */ + if (rc != -EDEADLK) + break; + + usleep_range(512, 2048); + } while (retry_count++ < 5); + + return rc; +} + +/* return length of unicode string in bytes */ +static int cifs_unicode_bytelen(const char *str) +{ + int len; + const __le16 *ustr = (const __le16 *)str; + + for (len = 0; len <= PATH_MAX; len++) { + if (ustr[len] == 0) + return len << 1; + } + cifs_dbg(FYI, "Unicode string longer than PATH_MAX found\n"); + return len << 1; +} + +static char *nxt_dir_entry(char *old_entry, char *end_of_smb, int level) +{ + char *new_entry; + FILE_DIRECTORY_INFO *pDirInfo = (FILE_DIRECTORY_INFO *)old_entry; + + if (level == SMB_FIND_FILE_INFO_STANDARD) { + FIND_FILE_STANDARD_INFO *pfData; + pfData = (FIND_FILE_STANDARD_INFO *)pDirInfo; + + new_entry = old_entry + sizeof(FIND_FILE_STANDARD_INFO) + 1 + + pfData->FileNameLength; + } else { + u32 next_offset = le32_to_cpu(pDirInfo->NextEntryOffset); + + if (old_entry + next_offset < old_entry) { + cifs_dbg(VFS, "Invalid offset %u\n", next_offset); + return NULL; + } + new_entry = old_entry + next_offset; + } + cifs_dbg(FYI, "new entry %p old entry %p\n", new_entry, old_entry); + /* validate that new_entry is not past end of SMB */ + if (new_entry >= end_of_smb) { + cifs_dbg(VFS, "search entry %p began after end of SMB %p old entry %p\n", + new_entry, end_of_smb, old_entry); + return NULL; + } else if (((level == SMB_FIND_FILE_INFO_STANDARD) && + (new_entry + sizeof(FIND_FILE_STANDARD_INFO) + 1 > end_of_smb)) + || ((level != SMB_FIND_FILE_INFO_STANDARD) && + (new_entry + sizeof(FILE_DIRECTORY_INFO) + 1 > end_of_smb))) { + cifs_dbg(VFS, "search entry %p extends after end of SMB %p\n", + new_entry, end_of_smb); + return NULL; + } else + return new_entry; + +} + +struct cifs_dirent { + const char *name; + size_t namelen; + u32 resume_key; + u64 ino; +}; + +static void cifs_fill_dirent_posix(struct cifs_dirent *de, + const struct smb2_posix_info *info) +{ + struct smb2_posix_info_parsed parsed; + + /* payload should have already been checked at this point */ + if (posix_info_parse(info, NULL, &parsed) < 0) { + cifs_dbg(VFS, "Invalid POSIX info payload\n"); + return; + } + + de->name = parsed.name; + de->namelen = parsed.name_len; + de->resume_key = info->Ignored; + de->ino = le64_to_cpu(info->Inode); +} + +static void cifs_fill_dirent_unix(struct cifs_dirent *de, + const FILE_UNIX_INFO *info, bool is_unicode) +{ + de->name = &info->FileName[0]; + if (is_unicode) + de->namelen = cifs_unicode_bytelen(de->name); + else + de->namelen = strnlen(de->name, PATH_MAX); + de->resume_key = info->ResumeKey; + de->ino = le64_to_cpu(info->basic.UniqueId); +} + +static void cifs_fill_dirent_dir(struct cifs_dirent *de, + const FILE_DIRECTORY_INFO *info) +{ + de->name = &info->FileName[0]; + de->namelen = le32_to_cpu(info->FileNameLength); + de->resume_key = info->FileIndex; +} + +static void cifs_fill_dirent_full(struct cifs_dirent *de, + const FILE_FULL_DIRECTORY_INFO *info) +{ + de->name = &info->FileName[0]; + de->namelen = le32_to_cpu(info->FileNameLength); + de->resume_key = info->FileIndex; +} + +static void cifs_fill_dirent_search(struct cifs_dirent *de, + const FILE_ID_FULL_DIR_INFO *info) +{ + de->name = &info->FileName[0]; + de->namelen = le32_to_cpu(info->FileNameLength); + de->resume_key = info->FileIndex; + de->ino = le64_to_cpu(info->UniqueId); +} + +static void cifs_fill_dirent_both(struct cifs_dirent *de, + const FILE_BOTH_DIRECTORY_INFO *info) +{ + de->name = &info->FileName[0]; + de->namelen = le32_to_cpu(info->FileNameLength); + de->resume_key = info->FileIndex; +} + +static void cifs_fill_dirent_std(struct cifs_dirent *de, + const FIND_FILE_STANDARD_INFO *info) +{ + de->name = &info->FileName[0]; + /* one byte length, no endianness conversion */ + de->namelen = info->FileNameLength; + de->resume_key = info->ResumeKey; +} + +static int cifs_fill_dirent(struct cifs_dirent *de, const void *info, + u16 level, bool is_unicode) +{ + memset(de, 0, sizeof(*de)); + + switch (level) { + case SMB_FIND_FILE_POSIX_INFO: + cifs_fill_dirent_posix(de, info); + break; + case SMB_FIND_FILE_UNIX: + cifs_fill_dirent_unix(de, info, is_unicode); + break; + case SMB_FIND_FILE_DIRECTORY_INFO: + cifs_fill_dirent_dir(de, info); + break; + case SMB_FIND_FILE_FULL_DIRECTORY_INFO: + cifs_fill_dirent_full(de, info); + break; + case SMB_FIND_FILE_ID_FULL_DIR_INFO: + cifs_fill_dirent_search(de, info); + break; + case SMB_FIND_FILE_BOTH_DIRECTORY_INFO: + cifs_fill_dirent_both(de, info); + break; + case SMB_FIND_FILE_INFO_STANDARD: + cifs_fill_dirent_std(de, info); + break; + default: + cifs_dbg(FYI, "Unknown findfirst level %d\n", level); + return -EINVAL; + } + + return 0; +} + +#define UNICODE_DOT cpu_to_le16(0x2e) + +/* return 0 if no match and 1 for . (current directory) and 2 for .. (parent) */ +static int cifs_entry_is_dot(struct cifs_dirent *de, bool is_unicode) +{ + int rc = 0; + + if (!de->name) + return 0; + + if (is_unicode) { + __le16 *ufilename = (__le16 *)de->name; + if (de->namelen == 2) { + /* check for . */ + if (ufilename[0] == UNICODE_DOT) + rc = 1; + } else if (de->namelen == 4) { + /* check for .. */ + if (ufilename[0] == UNICODE_DOT && + ufilename[1] == UNICODE_DOT) + rc = 2; + } + } else /* ASCII */ { + if (de->namelen == 1) { + if (de->name[0] == '.') + rc = 1; + } else if (de->namelen == 2) { + if (de->name[0] == '.' && de->name[1] == '.') + rc = 2; + } + } + + return rc; +} + +/* Check if directory that we are searching has changed so we can decide + whether we can use the cached search results from the previous search */ +static int is_dir_changed(struct file *file) +{ + struct inode *inode = file_inode(file); + struct cifsInodeInfo *cifs_inode_info = CIFS_I(inode); + + if (cifs_inode_info->time == 0) + return 1; /* directory was changed, e.g. unlink or new file */ + else + return 0; + +} + +static int cifs_save_resume_key(const char *current_entry, + struct cifsFileInfo *file_info) +{ + struct cifs_dirent de; + int rc; + + rc = cifs_fill_dirent(&de, current_entry, file_info->srch_inf.info_level, + file_info->srch_inf.unicode); + if (!rc) { + file_info->srch_inf.presume_name = de.name; + file_info->srch_inf.resume_name_len = de.namelen; + file_info->srch_inf.resume_key = de.resume_key; + } + return rc; +} + +/* + * Find the corresponding entry in the search. Note that the SMB server returns + * search entries for . and .. which complicates logic here if we choose to + * parse for them and we do not assume that they are located in the findfirst + * return buffer. We start counting in the buffer with entry 2 and increment for + * every entry (do not increment for . or .. entry). + */ +static int +find_cifs_entry(const unsigned int xid, struct cifs_tcon *tcon, loff_t pos, + struct file *file, const char *full_path, + char **current_entry, int *num_to_ret) +{ + __u16 search_flags; + int rc = 0; + int pos_in_buf = 0; + loff_t first_entry_in_buffer; + loff_t index_to_find = pos; + struct cifsFileInfo *cfile = file->private_data; + struct cifs_sb_info *cifs_sb = CIFS_SB(file); + struct TCP_Server_Info *server = tcon->ses->server; + /* check if index in the buffer */ + + if (!server->ops->query_dir_first || !server->ops->query_dir_next) + return -ENOSYS; + + if ((cfile == NULL) || (current_entry == NULL) || (num_to_ret == NULL)) + return -ENOENT; + + *current_entry = NULL; + first_entry_in_buffer = cfile->srch_inf.index_of_last_entry - + cfile->srch_inf.entries_in_buffer; + + /* + * If first entry in buf is zero then is first buffer + * in search response data which means it is likely . and .. + * will be in this buffer, although some servers do not return + * . and .. for the root of a drive and for those we need + * to start two entries earlier. + */ + + dump_cifs_file_struct(file, "In fce "); + if (((index_to_find < cfile->srch_inf.index_of_last_entry) && + is_dir_changed(file)) || (index_to_find < first_entry_in_buffer)) { + /* close and restart search */ + cifs_dbg(FYI, "search backing up - close and restart search\n"); + spin_lock(&cfile->file_info_lock); + if (server->ops->dir_needs_close(cfile)) { + cfile->invalidHandle = true; + spin_unlock(&cfile->file_info_lock); + if (server->ops->close_dir) + server->ops->close_dir(xid, tcon, &cfile->fid); + } else + spin_unlock(&cfile->file_info_lock); + if (cfile->srch_inf.ntwrk_buf_start) { + cifs_dbg(FYI, "freeing SMB ff cache buf on search rewind\n"); + if (cfile->srch_inf.smallBuf) + cifs_small_buf_release(cfile->srch_inf. + ntwrk_buf_start); + else if (cfile->srch_inf.is_dynamic_buf) + kfree(cfile->srch_inf.ntwrk_buf_start); + else + cifs_buf_release(cfile->srch_inf. + ntwrk_buf_start); + /* Reset all pointers to the network buffer to prevent stale references */ + cfile->srch_inf.ntwrk_buf_start = NULL; + cfile->srch_inf.srch_entries_start = NULL; + cfile->srch_inf.last_entry = NULL; + } + rc = initiate_cifs_search(xid, file, full_path); + if (rc) { + cifs_dbg(FYI, "error %d reinitiating a search on rewind\n", + rc); + return rc; + } + /* FindFirst/Next set last_entry to NULL on malformed reply */ + if (cfile->srch_inf.last_entry) + cifs_save_resume_key(cfile->srch_inf.last_entry, cfile); + } + + search_flags = CIFS_SEARCH_CLOSE_AT_END | CIFS_SEARCH_RETURN_RESUME; + if (backup_cred(cifs_sb)) + search_flags |= CIFS_SEARCH_BACKUP_SEARCH; + + while ((index_to_find >= cfile->srch_inf.index_of_last_entry) && + (rc == 0) && !cfile->srch_inf.endOfSearch) { + cifs_dbg(FYI, "calling findnext2\n"); + rc = server->ops->query_dir_next(xid, tcon, &cfile->fid, + search_flags, + &cfile->srch_inf); + if (rc) + return -ENOENT; + /* FindFirst/Next set last_entry to NULL on malformed reply */ + if (cfile->srch_inf.last_entry) + cifs_save_resume_key(cfile->srch_inf.last_entry, cfile); + } + if (index_to_find < cfile->srch_inf.index_of_last_entry) { + /* we found the buffer that contains the entry */ + /* scan and find it */ + int i; + char *cur_ent; + char *end_of_smb; + + if (cfile->srch_inf.ntwrk_buf_start == NULL) { + cifs_dbg(VFS, "ntwrk_buf_start is NULL during readdir\n"); + return smb_EIO(smb_eio_trace_null_pointers); + } + + end_of_smb = cfile->srch_inf.ntwrk_buf_start + + server->ops->calc_smb_size( + cfile->srch_inf.ntwrk_buf_start); + + cur_ent = cfile->srch_inf.srch_entries_start; + first_entry_in_buffer = cfile->srch_inf.index_of_last_entry + - cfile->srch_inf.entries_in_buffer; + pos_in_buf = index_to_find - first_entry_in_buffer; + cifs_dbg(FYI, "found entry - pos_in_buf %d\n", pos_in_buf); + + for (i = 0; (i < (pos_in_buf)) && (cur_ent != NULL); i++) { + /* go entry by entry figuring out which is first */ + cur_ent = nxt_dir_entry(cur_ent, end_of_smb, + cfile->srch_inf.info_level); + } + if ((cur_ent == NULL) && (i < pos_in_buf)) { + /* BB fixme - check if we should flag this error */ + cifs_dbg(VFS, "reached end of buf searching for pos in buf %d index to find %lld rc %d\n", + pos_in_buf, index_to_find, rc); + } + rc = 0; + *current_entry = cur_ent; + } else { + cifs_dbg(FYI, "index not in buffer - could not findnext into it\n"); + return 0; + } + + if (pos_in_buf >= cfile->srch_inf.entries_in_buffer) { + cifs_dbg(FYI, "can not return entries pos_in_buf beyond last\n"); + *num_to_ret = 0; + } else + *num_to_ret = cfile->srch_inf.entries_in_buffer - pos_in_buf; + + return rc; +} + +static bool emit_cached_dirents(struct cached_dirents *cde, + struct dir_context *ctx) +{ + struct cached_dirent *dirent; + bool rc; + + list_for_each_entry(dirent, &cde->entries, entry) { + /* + * Skip all early entries prior to the current lseek() + * position. + */ + if (ctx->pos > dirent->pos) + continue; + /* + * We recorded the current ->pos value for the dirent + * when we stored it in the cache. + * However, this sequence of ->pos values may have holes + * in it, for example dot-dirs returned from the server + * are suppressed. + * Handle this by forcing ctx->pos to be the same as the + * ->pos of the current dirent we emit from the cache. + * This means that when we emit these entries from the cache + * we now emit them with the same ->pos value as in the + * initial scan. + */ + ctx->pos = dirent->pos; + rc = dir_emit(ctx, dirent->name, dirent->namelen, + dirent->fattr.cf_uniqueid, + dirent->fattr.cf_dtype); + if (!rc) + return rc; + ctx->pos++; + } + return true; +} + +static void update_cached_dirents_count(struct cached_dirents *cde, + struct file *file) +{ + if (cde->file != file) + return; + if (cde->is_valid || cde->is_failed) + return; + + cde->pos++; +} + +static void finished_cached_dirents_count(struct cached_dirents *cde, + struct dir_context *ctx, struct file *file) +{ + if (cde->file != file) + return; + if (cde->is_valid || cde->is_failed) + return; + if (ctx->pos != cde->pos) + return; + + cde->is_valid = 1; +} + +static bool add_cached_dirent(struct cached_dirents *cde, + struct dir_context *ctx, const char *name, + int namelen, struct cifs_fattr *fattr, + struct file *file) +{ + struct cached_dirent *de; + + if (cde->file != file) + return false; + if (cde->is_valid || cde->is_failed) + return false; + if (ctx->pos != cde->pos) { + cde->is_failed = 1; + return false; + } + de = kzalloc_obj(*de, GFP_ATOMIC); + if (de == NULL) { + cde->is_failed = 1; + return false; + } + de->namelen = namelen; + de->name = kstrndup(name, namelen, GFP_ATOMIC); + if (de->name == NULL) { + kfree(de); + cde->is_failed = 1; + return false; + } + de->pos = ctx->pos; + + memcpy(&de->fattr, fattr, sizeof(struct cifs_fattr)); + + list_add_tail(&de->entry, &cde->entries); + /* update accounting */ + cde->entries_count++; + cde->bytes_used += sizeof(*de) + (size_t)namelen + 1; + return true; +} + +static bool cifs_dir_emit(struct dir_context *ctx, + const char *name, int namelen, + struct cifs_fattr *fattr, + struct cached_fid *cfid, + struct file *file) +{ + size_t delta_bytes = 0; + bool rc, added = false; + ino_t ino = cifs_uniqueid_to_ino_t(fattr->cf_uniqueid); + + rc = dir_emit(ctx, name, namelen, ino, fattr->cf_dtype); + if (!rc) + return rc; + + if (cfid) { + /* Cost of this entry */ + delta_bytes = sizeof(struct cached_dirent) + (size_t)namelen + 1; + + mutex_lock(&cfid->dirents.de_mutex); + added = add_cached_dirent(&cfid->dirents, ctx, name, namelen, + fattr, file); + mutex_unlock(&cfid->dirents.de_mutex); + + if (added) { + /* per-tcon then global for consistency with free path */ + atomic64_add((long long)delta_bytes, &cfid->cfids->total_dirents_bytes); + atomic_long_inc(&cfid->cfids->total_dirents_entries); + atomic64_add((long long)delta_bytes, &cifs_dircache_bytes_used); + } + } + + return rc; +} + +static int cifs_filldir(char *find_entry, struct file *file, + struct dir_context *ctx, + char *scratch_buf, unsigned int max_len, + char *end_of_smb, struct cached_fid *cfid) +{ + struct cifsFileInfo *file_info = file->private_data; + struct super_block *sb = file_inode(file)->i_sb; + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + struct cifs_dirent de = { NULL, }; + struct cifs_fattr fattr; + unsigned int sbflags; + struct qstr name; + int rc = 0; + + rc = cifs_fill_dirent(&de, find_entry, file_info->srch_inf.info_level, + file_info->srch_inf.unicode); + if (rc) + return rc; + + if (de.namelen > max_len) { + cifs_dbg(VFS, "bad search response length %zd past smb end\n", + de.namelen); + return -EINVAL; + } + + if (de.name + de.namelen > end_of_smb) { + cifs_dbg(VFS, "search entry name extends past end of SMB\n"); + return -EINVAL; + } + + /* skip . and .. since we added them first */ + if (cifs_entry_is_dot(&de, file_info->srch_inf.unicode)) + return 0; + + if (file_info->srch_inf.unicode) { + struct nls_table *nlt = cifs_sb->local_nls; + int map_type; + + map_type = cifs_remap(cifs_sb); + name.name = scratch_buf; + name.len = + cifs_from_utf16((char *)name.name, (__le16 *)de.name, + UNICODE_NAME_MAX, + min_t(size_t, de.namelen, + (size_t)max_len), nlt, map_type); + name.len -= nls_nullsize(nlt); + } else { + name.name = de.name; + name.len = de.namelen; + } + + switch (file_info->srch_inf.info_level) { + case SMB_FIND_FILE_POSIX_INFO: + cifs_posix_to_fattr(&fattr, + (struct smb2_posix_info *)find_entry, + cifs_sb); + break; + case SMB_FIND_FILE_UNIX: + cifs_unix_basic_to_fattr(&fattr, + &((FILE_UNIX_INFO *)find_entry)->basic, + cifs_sb); + if (S_ISLNK(fattr.cf_mode)) + fattr.cf_flags |= CIFS_FATTR_NEED_REVAL; + break; + case SMB_FIND_FILE_INFO_STANDARD: + cifs_std_info_to_fattr(&fattr, + (FIND_FILE_STANDARD_INFO *)find_entry, + cifs_sb); + break; + case SMB_FIND_FILE_FULL_DIRECTORY_INFO: + case SMB_FIND_FILE_ID_FULL_DIR_INFO: + cifs_fulldir_info_to_fattr(&fattr, find_entry, cifs_sb); + break; + default: + cifs_dir_info_to_fattr(&fattr, + (FILE_DIRECTORY_INFO *)find_entry, + cifs_sb); + break; + } + + sbflags = cifs_sb_flags(cifs_sb); + if (de.ino && (sbflags & CIFS_MOUNT_SERVER_INUM)) { + fattr.cf_uniqueid = de.ino; + } else { + fattr.cf_uniqueid = iunique(sb, ROOT_I); + cifs_autodisable_serverino(cifs_sb, "Cannot retrieve inode number from readdir", 0); + } + + if ((sbflags & CIFS_MOUNT_MF_SYMLINKS) && couldbe_mf_symlink(&fattr)) + /* + * trying to get the type and mode can be slow, + * so just call those regular files for now, and mark + * for reval + */ + fattr.cf_flags |= CIFS_FATTR_NEED_REVAL; + + cifs_prime_dcache(file_dentry(file), &name, &fattr); + + return !cifs_dir_emit(ctx, name.name, name.len, + &fattr, cfid, file); +} + + +int cifs_readdir(struct file *file, struct dir_context *ctx) +{ + int rc = 0; + unsigned int xid; + int i; + struct tcon_link *tlink = NULL; + struct cifs_tcon *tcon; + struct cifsFileInfo *cifsFile; + char *current_entry; + int num_to_fill = 0; + char *tmp_buf = NULL; + char *end_of_smb; + unsigned int max_len; + const char *full_path; + void *page = alloc_dentry_path(); + struct cached_fid *cfid = NULL; + struct cifs_sb_info *cifs_sb = CIFS_SB(file); + + xid = get_xid(); + + full_path = build_path_from_dentry(file_dentry(file), page); + if (IS_ERR(full_path)) { + rc = PTR_ERR(full_path); + goto rddir2_exit; + } + + if (file->private_data == NULL) { + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) + goto cache_not_found; + tcon = tlink_tcon(tlink); + } else { + cifsFile = file->private_data; + tcon = tlink_tcon(cifsFile->tlink); + } + + rc = open_cached_dir(xid, tcon, full_path, cifs_sb, false, &cfid); + cifs_put_tlink(tlink); + if (rc) + goto cache_not_found; + + mutex_lock(&cfid->dirents.de_mutex); + /* + * If this was reading from the start of the directory + * we need to initialize scanning and storing the + * directory content. + */ + if (ctx->pos == 0 && cfid->dirents.file == NULL) { + cfid->dirents.file = file; + cfid->dirents.pos = 2; + } + /* + * If we already have the entire directory cached then + * we can just serve the cache. + */ + if (cfid->dirents.is_valid) { + if (!dir_emit_dots(file, ctx)) { + mutex_unlock(&cfid->dirents.de_mutex); + goto rddir2_exit; + } + emit_cached_dirents(&cfid->dirents, ctx); + mutex_unlock(&cfid->dirents.de_mutex); + goto rddir2_exit; + } + mutex_unlock(&cfid->dirents.de_mutex); + + /* Drop the cache while calling initiate_cifs_search and + * find_cifs_entry in case there will be reconnects during + * query_directory. + */ + close_cached_dir(cfid); + cfid = NULL; + + cache_not_found: + /* + * Ensure FindFirst doesn't fail before doing filldir() for '.' and + * '..'. Otherwise we won't be able to notify VFS in case of failure. + */ + if (file->private_data == NULL) { + rc = initiate_cifs_search(xid, file, full_path); + cifs_dbg(FYI, "initiate cifs search rc %d\n", rc); + if (rc) + goto rddir2_exit; + } + + if (!dir_emit_dots(file, ctx)) + goto rddir2_exit; + + /* 1) If search is active, + is in current search buffer? + if it before then restart search + if after then keep searching till find it */ + cifsFile = file->private_data; + if (cifsFile->srch_inf.endOfSearch) { + if (cifsFile->srch_inf.emptyDir) { + cifs_dbg(FYI, "End of search, empty dir\n"); + rc = 0; + goto rddir2_exit; + } + } /* else { + cifsFile->invalidHandle = true; + tcon->ses->server->close(xid, tcon, &cifsFile->fid); + } */ + + tcon = tlink_tcon(cifsFile->tlink); + rc = find_cifs_entry(xid, tcon, ctx->pos, file, full_path, + ¤t_entry, &num_to_fill); + open_cached_dir(xid, tcon, full_path, cifs_sb, false, &cfid); + if (rc) { + cifs_dbg(FYI, "fce error %d\n", rc); + goto rddir2_exit; + } else if (current_entry != NULL) { + cifs_dbg(FYI, "entry %lld found\n", ctx->pos); + } else { + if (cfid) { + mutex_lock(&cfid->dirents.de_mutex); + finished_cached_dirents_count(&cfid->dirents, ctx, file); + mutex_unlock(&cfid->dirents.de_mutex); + } + cifs_dbg(FYI, "Could not find entry\n"); + goto rddir2_exit; + } + cifs_dbg(FYI, "loop through %d times filling dir for net buf %p\n", + num_to_fill, cifsFile->srch_inf.ntwrk_buf_start); + max_len = tcon->ses->server->ops->calc_smb_size( + cifsFile->srch_inf.ntwrk_buf_start); + end_of_smb = cifsFile->srch_inf.ntwrk_buf_start + max_len; + + tmp_buf = kmalloc(UNICODE_NAME_MAX, GFP_KERNEL); + if (tmp_buf == NULL) { + rc = -ENOMEM; + goto rddir2_exit; + } + + for (i = 0; i < num_to_fill; i++) { + if (current_entry == NULL) { + /* evaluate whether this case is an error */ + cifs_dbg(VFS, "past SMB end, num to fill %d i %d\n", + num_to_fill, i); + break; + } + /* + * if buggy server returns . and .. late do we want to + * check for that here? + */ + *tmp_buf = 0; + rc = cifs_filldir(current_entry, file, ctx, + tmp_buf, max_len, end_of_smb, cfid); + if (rc) { + if (rc > 0) + rc = 0; + break; + } + + ctx->pos++; + if (cfid) { + mutex_lock(&cfid->dirents.de_mutex); + update_cached_dirents_count(&cfid->dirents, file); + mutex_unlock(&cfid->dirents.de_mutex); + } + + if (ctx->pos == + cifsFile->srch_inf.index_of_last_entry) { + cifs_dbg(FYI, "last entry in buf at pos %lld %s\n", + ctx->pos, tmp_buf); + cifs_save_resume_key(current_entry, cifsFile); + break; + } + current_entry = + nxt_dir_entry(current_entry, end_of_smb, + cifsFile->srch_inf.info_level); + } + kfree(tmp_buf); + +rddir2_exit: + if (cfid) + close_cached_dir(cfid); + free_dentry_path(page); + free_xid(xid); + return rc; +} diff --git a/fs/smb/client/reparse.c b/fs/smb/client/reparse.c new file mode 100644 index 000000000..5cc5b0410 --- /dev/null +++ b/fs/smb/client/reparse.c @@ -0,0 +1,1289 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2024 Paulo Alcantara + */ + +#include +#include +#include +#include "cifsglob.h" +#include "smb2proto.h" +#include "cifsproto.h" +#include "cifs_unicode.h" +#include "cifs_debug.h" +#include "fs_context.h" +#include "reparse.h" + +static int mknod_nfs(unsigned int xid, struct inode *inode, + struct dentry *dentry, struct cifs_tcon *tcon, + const char *full_path, umode_t mode, dev_t dev, + const char *symname); + +static int mknod_wsl(unsigned int xid, struct inode *inode, + struct dentry *dentry, struct cifs_tcon *tcon, + const char *full_path, umode_t mode, dev_t dev, + const char *symname); + +static int create_native_symlink(const unsigned int xid, struct inode *inode, + struct dentry *dentry, struct cifs_tcon *tcon, + const char *full_path, const char *symname); + +static int detect_directory_symlink_target(struct cifs_sb_info *cifs_sb, + const unsigned int xid, + const char *full_path, + const char *symname, + bool *directory); + +int create_reparse_symlink(const unsigned int xid, struct inode *inode, + struct dentry *dentry, struct cifs_tcon *tcon, + const char *full_path, const char *symname) +{ + switch (cifs_symlink_type(CIFS_SB(inode->i_sb))) { + case CIFS_SYMLINK_TYPE_NATIVE: + return create_native_symlink(xid, inode, dentry, tcon, full_path, symname); + case CIFS_SYMLINK_TYPE_NFS: + return mknod_nfs(xid, inode, dentry, tcon, full_path, S_IFLNK, 0, symname); + case CIFS_SYMLINK_TYPE_WSL: + return mknod_wsl(xid, inode, dentry, tcon, full_path, S_IFLNK, 0, symname); + default: + return -EOPNOTSUPP; + } +} + +static int create_native_symlink(const unsigned int xid, struct inode *inode, + struct dentry *dentry, struct cifs_tcon *tcon, + const char *full_path, const char *symname) +{ + struct reparse_symlink_data_buffer *buf = NULL; + struct cifs_sb_info *cifs_sb = CIFS_SB(inode); + const char *symroot = cifs_sb->ctx->symlinkroot; + struct cifs_open_info_data data = {}; + char sep = CIFS_DIR_SEP(cifs_sb); + char *symlink_target = NULL; + u16 len, plen, poff, slen; + unsigned int sbflags; + __le16 *path = NULL; + struct inode *new; + char *sym = NULL; + struct kvec iov; + bool directory; + int path_len; + int rc = 0; + + if (strlen(symname) > REPARSE_SYM_PATH_MAX) + return -ENAMETOOLONG; + + symlink_target = kstrdup(symname, GFP_KERNEL); + if (!symlink_target) { + rc = -ENOMEM; + goto out; + } + + data = (struct cifs_open_info_data) { + .reparse_point = true, + .reparse = { .tag = IO_REPARSE_TAG_SYMLINK, }, + .symlink_target = symlink_target, + }; + + sbflags = cifs_sb_flags(cifs_sb); + if (!(sbflags & CIFS_MOUNT_POSIX_PATHS) && symroot && symname[0] == '/') { + /* + * This is a request to create an absolute symlink on the server + * which does not support POSIX paths, and expects symlink in + * NT-style path. So convert absolute Linux symlink target path + * to the absolute NT-style path. Root of the NT-style path for + * symlinks is specified in "symlinkroot" mount option. This will + * ensure compatibility of this symlink stored in absolute form + * on the SMB server. + */ + if (!strstarts(symname, symroot)) { + /* + * If the absolute Linux symlink target path is not + * inside "symlinkroot" location then there is no way + * to convert such Linux symlink to NT-style path. + */ + cifs_dbg(VFS, + "absolute symlink '%s' cannot be converted to NT format " + "because it is outside of symlinkroot='%s'\n", + symname, symroot); + rc = -EINVAL; + goto out; + } + len = strlen(symroot); + if (symroot[len - 1] != '/') + len++; + if (symname[len] >= 'a' && symname[len] <= 'z' && + (symname[len+1] == '/' || symname[len+1] == '\0')) { + /* + * Symlink points to Linux target /symlinkroot/x/path/... + * where 'x' is the lowercase local Windows drive. + * NT-style path for 'x' has common form \??\X:\path\... + * with uppercase local Windows drive. + */ + int common_path_len = strlen(symname+len+1)+1; + sym = kzalloc(6+common_path_len, GFP_KERNEL); + if (!sym) { + rc = -ENOMEM; + goto out; + } + memcpy(sym, "\\??\\", 4); + sym[4] = symname[len] - ('a'-'A'); + sym[5] = ':'; + memcpy(sym+6, symname+len+1, common_path_len); + } else { + /* Unhandled absolute symlink. Report an error. */ + cifs_dbg( + VFS, + "absolute symlink '%s' cannot be converted to NT format " + "because it points to unknown target\n", + symname); + rc = -EINVAL; + goto out; + } + } else { + /* + * This is request to either create an absolute symlink on + * server which expects POSIX paths or it is an request to + * create a relative symlink from the current directory. + * These paths have same format as relative SMB symlinks, + * so no conversion is needed. So just take symname as-is. + */ + sym = kstrdup(symname, GFP_KERNEL); + if (!sym) { + rc = -ENOMEM; + goto out; + } + } + + if (sep == '\\') + convert_delimiter(sym, sep); + + /* + * For absolute NT symlinks it is required to pass also leading + * backslash and to not mangle NT object prefix "\\??\\" and not to + * mangle colon in drive letter. But cifs_convert_path_to_utf16() + * removes leading backslash and replaces '?' and ':'. So temporary + * mask these characters in NT object prefix by '_' and then change + * them back. + */ + if (!(sbflags & CIFS_MOUNT_POSIX_PATHS) && symname[0] == '/') + sym[0] = sym[1] = sym[2] = sym[5] = '_'; + + /* + * On a POSIX paths mount the symlink target is stored verbatim, so + * convert it with cifs_strndup_to_utf16(). cifs_convert_path_to_utf16() + * must not be used here: it strips a leading path separator (it is + * meant for share-relative SMB paths), which would corrupt an absolute + * POSIX symlink target such as "/foo/bar". Using NO_MAP_UNI_RSVD also + * matches the readback path in smb2_parse_native_symlink(). + */ + if (sbflags & CIFS_MOUNT_POSIX_PATHS) + path = cifs_strndup_to_utf16(sym, strlen(sym), &path_len, + cifs_sb->local_nls, + NO_MAP_UNI_RSVD); + else + path = cifs_convert_path_to_utf16(sym, cifs_sb); + + if (!path) { + rc = -ENOMEM; + goto out; + } + + if (!(sbflags & CIFS_MOUNT_POSIX_PATHS) && symname[0] == '/') { + sym[0] = '\\'; + sym[1] = sym[2] = '?'; + sym[5] = ':'; + path[0] = cpu_to_le16('\\'); + path[1] = path[2] = cpu_to_le16('?'); + path[5] = cpu_to_le16(':'); + } + + /* + * SMB distinguish between symlink to directory and symlink to file. + * They cannot be exchanged (symlink of file type which points to + * directory cannot be resolved and vice-versa). Try to detect if + * the symlink target could be a directory or not. When detection + * fails then treat symlink as a file (non-directory) symlink. + */ + directory = false; + rc = detect_directory_symlink_target(cifs_sb, xid, full_path, symname, &directory); + if (rc < 0) + goto out; + + slen = 2 * UniStrnlen((wchar_t *)path, REPARSE_SYM_PATH_MAX); + poff = 0; + plen = slen; + if (!(sbflags & CIFS_MOUNT_POSIX_PATHS) && symname[0] == '/') { + /* + * For absolute NT symlinks skip leading "\\??\\" in PrintName as + * PrintName is user visible location in DOS/Win32 format (not in NT format). + */ + poff = 4; + plen -= 2 * poff; + } + len = sizeof(*buf) + plen + slen; + buf = kzalloc(len, GFP_KERNEL); + if (!buf) { + rc = -ENOMEM; + goto out; + } + + buf->ReparseTag = cpu_to_le32(IO_REPARSE_TAG_SYMLINK); + buf->ReparseDataLength = cpu_to_le16(len - sizeof(struct reparse_data_buffer)); + + buf->SubstituteNameOffset = cpu_to_le16(plen); + buf->SubstituteNameLength = cpu_to_le16(slen); + memcpy(&buf->PathBuffer[plen], path, slen); + + buf->PrintNameOffset = 0; + buf->PrintNameLength = cpu_to_le16(plen); + memcpy(buf->PathBuffer, path+poff, plen); + + buf->Flags = cpu_to_le32(*symname != '/' ? SYMLINK_FLAG_RELATIVE : 0); + + iov.iov_base = buf; + iov.iov_len = len; + new = tcon->ses->server->ops->create_reparse_inode( + &data, inode->i_sb, xid, + tcon, full_path, directory, + &iov, NULL); + if (!IS_ERR(new)) + d_instantiate(dentry, new); + else + rc = PTR_ERR(new); +out: + kfree(sym); + kfree(path); + cifs_free_open_info(&data); + kfree(buf); + return rc; +} + +static int detect_directory_symlink_target(struct cifs_sb_info *cifs_sb, + const unsigned int xid, + const char *full_path, + const char *symname, + bool *directory) +{ + char sep = CIFS_DIR_SEP(cifs_sb); + struct cifs_open_parms oparms; + struct tcon_link *tlink; + struct cifs_tcon *tcon; + const char *basename; + struct cifs_fid fid; + char *resolved_path; + int full_path_len; + int basename_len; + int symname_len; + char *path_sep; + __u32 oplock; + int open_rc; + + /* + * First do some simple check. If the original Linux symlink target ends + * with slash, or last path component is dot or dot-dot then it is for + * sure symlink to the directory. + */ + basename = kbasename(symname); + basename_len = strlen(basename); + if (basename_len == 0 || /* symname ends with slash */ + (basename_len == 1 && basename[0] == '.') || /* last component is "." */ + (basename_len == 2 && basename[0] == '.' && basename[1] == '.')) { /* or ".." */ + *directory = true; + return 0; + } + + /* + * For absolute symlinks it is not possible to determine + * if it should point to directory or file. + */ + if (symname[0] == '/') { + cifs_dbg(FYI, + "%s: cannot determinate if the symlink target path '%s' " + "is directory or not, creating '%s' as file symlink\n", + __func__, symname, full_path); + return 0; + } + + /* + * If it was not detected as directory yet and the symlink is relative + * then try to resolve the path on the SMB server, check if the path + * exists and determinate if it is a directory or not. + */ + + full_path_len = strlen(full_path); + symname_len = strlen(symname); + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) + return PTR_ERR(tlink); + + resolved_path = kzalloc(full_path_len + symname_len + 1, GFP_KERNEL); + if (!resolved_path) { + cifs_put_tlink(tlink); + return -ENOMEM; + } + + /* + * Compose the resolved SMB symlink path from the SMB full path + * and Linux target symlink path. + */ + memcpy(resolved_path, full_path, full_path_len+1); + path_sep = strrchr(resolved_path, sep); + if (path_sep) + path_sep++; + else + path_sep = resolved_path; + memcpy(path_sep, symname, symname_len+1); + if (sep == '\\') + convert_delimiter(path_sep, sep); + + tcon = tlink_tcon(tlink); + oparms = CIFS_OPARMS(cifs_sb, tcon, resolved_path, + FILE_READ_ATTRIBUTES, FILE_OPEN, 0, ACL_NO_MODE); + oparms.fid = &fid; + + /* Try to open as a directory (NOT_FILE) */ + oplock = 0; + oparms.create_options = cifs_create_options(cifs_sb, + CREATE_NOT_FILE | OPEN_REPARSE_POINT); + open_rc = tcon->ses->server->ops->open(xid, &oparms, &oplock, NULL); + if (open_rc == 0) { + /* Successful open means that the target path is definitely a directory. */ + *directory = true; + tcon->ses->server->ops->close(xid, tcon, &fid); + } else if (open_rc == -ENOTDIR) { + /* -ENOTDIR means that the target path is definitely a file. */ + *directory = false; + } else if (open_rc == -ENOENT) { + /* -ENOENT means that the target path does not exist. */ + cifs_dbg(FYI, + "%s: symlink target path '%s' does not exist, " + "creating '%s' as file symlink\n", + __func__, symname, full_path); + } else { + /* Try to open as a file (NOT_DIR) */ + oplock = 0; + oparms.create_options = cifs_create_options(cifs_sb, + CREATE_NOT_DIR | OPEN_REPARSE_POINT); + open_rc = tcon->ses->server->ops->open(xid, &oparms, &oplock, NULL); + if (open_rc == 0) { + /* Successful open means that the target path is definitely a file. */ + *directory = false; + tcon->ses->server->ops->close(xid, tcon, &fid); + } else if (open_rc == -EISDIR) { + /* -EISDIR means that the target path is definitely a directory. */ + *directory = true; + } else { + /* + * This code branch is called when we do not have a permission to + * open the resolved_path or some other client/process denied + * opening the resolved_path. + * + * TODO: Try to use ops->query_dir_first on the parent directory + * of resolved_path, search for basename of resolved_path and + * check if the ATTR_DIRECTORY is set in fi.Attributes. In some + * case this could work also when opening of the path is denied. + */ + cifs_dbg(FYI, + "%s: cannot determinate if the symlink target path '%s' " + "is directory or not, creating '%s' as file symlink\n", + __func__, symname, full_path); + } + } + + kfree(resolved_path); + cifs_put_tlink(tlink); + return 0; +} + +static int create_native_socket(const unsigned int xid, struct inode *inode, + struct dentry *dentry, struct cifs_tcon *tcon, + const char *full_path) +{ + struct reparse_data_buffer buf = { + .ReparseTag = cpu_to_le32(IO_REPARSE_TAG_AF_UNIX), + .ReparseDataLength = cpu_to_le16(0), + }; + struct cifs_open_info_data data = { + .reparse_point = true, + .reparse = { .tag = IO_REPARSE_TAG_AF_UNIX, .buf = &buf, }, + }; + struct kvec iov = { + .iov_base = &buf, + .iov_len = sizeof(buf), + }; + struct inode *new; + int rc = 0; + + new = tcon->ses->server->ops->create_reparse_inode( + &data, inode->i_sb, xid, + tcon, full_path, false, &iov, NULL); + if (!IS_ERR(new)) + d_instantiate(dentry, new); + else + rc = PTR_ERR(new); + cifs_free_open_info(&data); + return rc; +} + +static int nfs_set_reparse_buf(struct reparse_nfs_data_buffer *buf, + mode_t mode, dev_t dev, + __le16 *symname_utf16, + int symname_utf16_len, + struct kvec *iov) +{ + u64 type; + u16 len, dlen; + + len = sizeof(*buf); + + switch ((type = reparse_mode_nfs_type(mode))) { + case NFS_SPECFILE_BLK: + case NFS_SPECFILE_CHR: + dlen = 2 * sizeof(__le32); + ((__le32 *)buf->DataBuffer)[0] = cpu_to_le32(MAJOR(dev)); + ((__le32 *)buf->DataBuffer)[1] = cpu_to_le32(MINOR(dev)); + break; + case NFS_SPECFILE_LNK: + dlen = symname_utf16_len; + memcpy(buf->DataBuffer, symname_utf16, symname_utf16_len); + break; + case NFS_SPECFILE_FIFO: + case NFS_SPECFILE_SOCK: + dlen = 0; + break; + default: + return -EOPNOTSUPP; + } + + buf->ReparseTag = cpu_to_le32(IO_REPARSE_TAG_NFS); + buf->Reserved = 0; + buf->InodeType = cpu_to_le64(type); + buf->ReparseDataLength = cpu_to_le16(len + dlen - + sizeof(struct reparse_data_buffer)); + iov->iov_base = buf; + iov->iov_len = len + dlen; + return 0; +} + +static int mknod_nfs(unsigned int xid, struct inode *inode, + struct dentry *dentry, struct cifs_tcon *tcon, + const char *full_path, umode_t mode, dev_t dev, + const char *symname) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); + struct cifs_open_info_data data; + struct reparse_nfs_data_buffer *p = NULL; + __le16 *symname_utf16 = NULL; + int symname_utf16_len = 0; + struct inode *new; + struct kvec iov; + __u8 buf[sizeof(*p) + sizeof(__le64)]; + int rc; + + if (S_ISLNK(mode)) { + symname_utf16 = cifs_strndup_to_utf16(symname, strlen(symname), + &symname_utf16_len, + cifs_sb->local_nls, + NO_MAP_UNI_RSVD); + if (!symname_utf16) { + rc = -ENOMEM; + goto out; + } + symname_utf16_len -= 2; /* symlink is without trailing wide-nul */ + p = kzalloc(sizeof(*p) + symname_utf16_len, GFP_KERNEL); + if (!p) { + rc = -ENOMEM; + goto out; + } + } else { + p = (struct reparse_nfs_data_buffer *)buf; + } + rc = nfs_set_reparse_buf(p, mode, dev, symname_utf16, symname_utf16_len, &iov); + if (rc) + goto out; + + data = (struct cifs_open_info_data) { + .reparse_point = true, + .reparse = { .tag = IO_REPARSE_TAG_NFS, .buf = (struct reparse_data_buffer *)p, }, + .symlink_target = kstrdup(symname, GFP_KERNEL), + }; + + new = tcon->ses->server->ops->create_reparse_inode( + &data, inode->i_sb, xid, + tcon, full_path, false, &iov, NULL); + if (!IS_ERR(new)) + d_instantiate(dentry, new); + else + rc = PTR_ERR(new); + cifs_free_open_info(&data); +out: + if (S_ISLNK(mode)) { + kfree(symname_utf16); + kfree(p); + } + return rc; +} + +static int wsl_set_reparse_buf(struct reparse_data_buffer **buf, + mode_t mode, const char *symname, + struct cifs_sb_info *cifs_sb, + struct kvec *iov) +{ + struct reparse_wsl_symlink_data_buffer *symlink_buf; + __le16 *symname_utf16; + int symname_utf16_len; + int symname_utf8_maxlen; + int symname_utf8_len; + size_t buf_len; + u32 tag; + + switch ((tag = reparse_mode_wsl_tag(mode))) { + case IO_REPARSE_TAG_LX_BLK: + case IO_REPARSE_TAG_LX_CHR: + case IO_REPARSE_TAG_LX_FIFO: + case IO_REPARSE_TAG_AF_UNIX: + buf_len = sizeof(struct reparse_data_buffer); + *buf = kzalloc(buf_len, GFP_KERNEL); + if (!*buf) + return -ENOMEM; + break; + case IO_REPARSE_TAG_LX_SYMLINK: + symname_utf16 = cifs_strndup_to_utf16(symname, strlen(symname), + &symname_utf16_len, + cifs_sb->local_nls, + NO_MAP_UNI_RSVD); + if (!symname_utf16) + return -ENOMEM; + symname_utf8_maxlen = symname_utf16_len/2*3; + symlink_buf = kzalloc(sizeof(struct reparse_wsl_symlink_data_buffer) + + symname_utf8_maxlen, GFP_KERNEL); + if (!symlink_buf) { + kfree(symname_utf16); + return -ENOMEM; + } + /* Version field must be set to 2 (MS-FSCC 2.1.2.7) */ + symlink_buf->Version = cpu_to_le32(2); + /* Target for Version 2 is in UTF-8 but without trailing null-term byte */ + symname_utf8_len = utf16s_to_utf8s((wchar_t *)symname_utf16, symname_utf16_len/2, + UTF16_LITTLE_ENDIAN, + symlink_buf->Target, + symname_utf8_maxlen); + *buf = (struct reparse_data_buffer *)symlink_buf; + buf_len = sizeof(struct reparse_wsl_symlink_data_buffer) + symname_utf8_len; + kfree(symname_utf16); + break; + default: + return -EOPNOTSUPP; + } + + (*buf)->ReparseTag = cpu_to_le32(tag); + (*buf)->Reserved = 0; + (*buf)->ReparseDataLength = cpu_to_le16(buf_len - sizeof(struct reparse_data_buffer)); + iov->iov_base = *buf; + iov->iov_len = buf_len; + return 0; +} + +static struct smb2_create_ea_ctx *ea_create_context(u32 dlen, size_t *cc_len) +{ + struct smb2_create_ea_ctx *cc; + + *cc_len = round_up(sizeof(*cc) + dlen, 8); + cc = kzalloc(*cc_len, GFP_KERNEL); + if (!cc) + return ERR_PTR(-ENOMEM); + + cc->ctx.NameOffset = cpu_to_le16(offsetof(struct smb2_create_ea_ctx, + name)); + cc->ctx.NameLength = cpu_to_le16(4); + memcpy(cc->name, SMB2_CREATE_EA_BUFFER, strlen(SMB2_CREATE_EA_BUFFER)); + cc->ctx.DataOffset = cpu_to_le16(offsetof(struct smb2_create_ea_ctx, ea)); + cc->ctx.DataLength = cpu_to_le32(dlen); + return cc; +} + +struct wsl_xattr { + const char *name; + __le64 value; + u16 size; + u32 next; +}; + +static int wsl_set_xattrs(struct inode *inode, umode_t _mode, + dev_t _dev, struct kvec *iov) +{ + struct smb2_file_full_ea_info *ea; + struct smb2_create_ea_ctx *cc; + struct smb3_fs_context *ctx = CIFS_SB(inode->i_sb)->ctx; + __le64 uid = cpu_to_le64(from_kuid(current_user_ns(), ctx->linux_uid)); + __le64 gid = cpu_to_le64(from_kgid(current_user_ns(), ctx->linux_gid)); + __le64 dev = cpu_to_le64(((u64)MINOR(_dev) << 32) | MAJOR(_dev)); + __le64 mode = cpu_to_le64(_mode); + struct wsl_xattr xattrs[] = { + { .name = SMB2_WSL_XATTR_UID, .value = uid, .size = SMB2_WSL_XATTR_UID_SIZE, }, + { .name = SMB2_WSL_XATTR_GID, .value = gid, .size = SMB2_WSL_XATTR_GID_SIZE, }, + { .name = SMB2_WSL_XATTR_MODE, .value = mode, .size = SMB2_WSL_XATTR_MODE_SIZE, }, + { .name = SMB2_WSL_XATTR_DEV, .value = dev, .size = SMB2_WSL_XATTR_DEV_SIZE, }, + }; + size_t cc_len; + u32 dlen = 0, next = 0; + int i, num_xattrs; + u8 name_size = SMB2_WSL_XATTR_NAME_LEN + 1; + + memset(iov, 0, sizeof(*iov)); + + /* Exclude $LXDEV xattr for non-device files */ + if (!S_ISBLK(_mode) && !S_ISCHR(_mode)) + num_xattrs = ARRAY_SIZE(xattrs) - 1; + else + num_xattrs = ARRAY_SIZE(xattrs); + + for (i = 0; i < num_xattrs; i++) { + xattrs[i].next = ALIGN(sizeof(*ea) + name_size + + xattrs[i].size, 4); + dlen += xattrs[i].next; + } + + cc = ea_create_context(dlen, &cc_len); + if (IS_ERR(cc)) + return PTR_ERR(cc); + + ea = &cc->ea; + for (i = 0; i < num_xattrs; i++) { + ea = (void *)((u8 *)ea + next); + next = xattrs[i].next; + ea->next_entry_offset = cpu_to_le32(next); + + ea->ea_name_length = name_size - 1; + ea->ea_value_length = cpu_to_le16(xattrs[i].size); + memcpy(ea->ea_data, xattrs[i].name, name_size); + memcpy(&ea->ea_data[name_size], + &xattrs[i].value, xattrs[i].size); + } + ea->next_entry_offset = 0; + + iov->iov_base = cc; + iov->iov_len = cc_len; + return 0; +} + +static int mknod_wsl(unsigned int xid, struct inode *inode, + struct dentry *dentry, struct cifs_tcon *tcon, + const char *full_path, umode_t mode, dev_t dev, + const char *symname) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); + struct cifs_open_info_data data; + struct reparse_data_buffer *buf; + struct smb2_create_ea_ctx *cc; + struct inode *new; + unsigned int len; + struct kvec reparse_iov, xattr_iov; + int rc; + + rc = wsl_set_reparse_buf(&buf, mode, symname, cifs_sb, &reparse_iov); + if (rc) + return rc; + + rc = wsl_set_xattrs(inode, mode, dev, &xattr_iov); + if (rc) { + kfree(buf); + return rc; + } + + data = (struct cifs_open_info_data) { + .reparse_point = true, + .reparse = { .tag = le32_to_cpu(buf->ReparseTag), .buf = buf, }, + .symlink_target = kstrdup(symname, GFP_KERNEL), + }; + + cc = xattr_iov.iov_base; + len = le32_to_cpu(cc->ctx.DataLength); + memcpy(data.wsl.eas, &cc->ea, len); + data.wsl.eas_len = len; + + new = tcon->ses->server->ops->create_reparse_inode( + &data, inode->i_sb, + xid, tcon, full_path, false, + &reparse_iov, &xattr_iov); + if (!IS_ERR(new)) + d_instantiate(dentry, new); + else + rc = PTR_ERR(new); + cifs_free_open_info(&data); + kfree(xattr_iov.iov_base); + kfree(buf); + return rc; +} + +int mknod_reparse(unsigned int xid, struct inode *inode, + struct dentry *dentry, struct cifs_tcon *tcon, + const char *full_path, umode_t mode, dev_t dev) +{ + struct smb3_fs_context *ctx = CIFS_SB(inode->i_sb)->ctx; + + if (S_ISSOCK(mode) && !ctx->nonativesocket && ctx->reparse_type != CIFS_REPARSE_TYPE_NONE) + return create_native_socket(xid, inode, dentry, tcon, full_path); + + switch (ctx->reparse_type) { + case CIFS_REPARSE_TYPE_NFS: + return mknod_nfs(xid, inode, dentry, tcon, full_path, mode, dev, NULL); + case CIFS_REPARSE_TYPE_WSL: + return mknod_wsl(xid, inode, dentry, tcon, full_path, mode, dev, NULL); + default: + return -EOPNOTSUPP; + } +} + +/* See MS-FSCC 2.1.2.6 for the 'NFS' style reparse tags */ +static int parse_reparse_nfs(struct reparse_nfs_data_buffer *buf, + struct cifs_sb_info *cifs_sb, + struct cifs_open_info_data *data) +{ + unsigned int len; + u64 type; + + len = le16_to_cpu(buf->ReparseDataLength); + if (len < sizeof(buf->InodeType)) { + cifs_dbg(VFS, "srv returned malformed nfs buffer\n"); + return smb_EIO2(smb_eio_trace_reparse_nfs_too_short, + len, sizeof(buf->InodeType)); + } + + len -= sizeof(buf->InodeType); + + switch ((type = le64_to_cpu(buf->InodeType))) { + case NFS_SPECFILE_LNK: + if (len == 0 || (len % 2)) { + cifs_dbg(VFS, "srv returned malformed nfs symlink buffer\n"); + return smb_EIO1(smb_eio_trace_reparse_nfs_symbuf, len); + } + /* + * Check that buffer does not contain UTF-16 null codepoint + * because Linux cannot process symlink with null byte. + */ + if (UniStrnlen((wchar_t *)buf->DataBuffer, len/2) != len/2) { + cifs_dbg(VFS, "srv returned null byte in nfs symlink target location\n"); + return smb_EIO1(smb_eio_trace_reparse_nfs_nul, len); + } + data->symlink_target = cifs_strndup_from_utf16(buf->DataBuffer, + len, true, + cifs_sb->local_nls); + if (!data->symlink_target) + return -ENOMEM; + cifs_dbg(FYI, "%s: target path: %s\n", + __func__, data->symlink_target); + break; + case NFS_SPECFILE_CHR: + case NFS_SPECFILE_BLK: + /* DataBuffer for block and char devices contains two 32-bit numbers */ + if (len != 8) { + cifs_dbg(VFS, "srv returned malformed nfs buffer for type: 0x%llx\n", type); + return smb_EIO1(smb_eio_trace_reparse_nfs_dev, len); + } + break; + case NFS_SPECFILE_FIFO: + case NFS_SPECFILE_SOCK: + /* DataBuffer for fifos and sockets is empty */ + if (len != 0) { + cifs_dbg(VFS, "srv returned malformed nfs buffer for type: 0x%llx\n", type); + return smb_EIO1(smb_eio_trace_reparse_nfs_sockfifo, len); + } + break; + default: + cifs_dbg(VFS, "%s: unhandled inode type: 0x%llx\n", + __func__, type); + return -EOPNOTSUPP; + } + return 0; +} + +int smb2_parse_native_symlink(char **target, const char *buf, unsigned int len, + bool relative, + const char *full_path, + struct cifs_sb_info *cifs_sb) +{ + const char *symroot = cifs_sb->ctx->symlinkroot; + char sep = CIFS_DIR_SEP(cifs_sb); + char *linux_target = NULL; + char *smb_target = NULL; + int symlinkroot_len; + int abs_path_len; + char *abs_path; + int levels; + int rc, ulen; + int i; + + /* Check that length it valid */ + if (!len || (len % 2)) { + cifs_dbg(VFS, "srv returned malformed symlink buffer\n"); + rc = smb_EIO1(smb_eio_trace_reparse_native_nul, len); + goto out; + } + + /* + * Check that buffer does not contain UTF-16 null codepoint + * because Linux cannot process symlink with null byte. + */ + ulen = UniStrnlen((wchar_t *)buf, len/2); + if (ulen != len/2) { + cifs_dbg(VFS, "srv returned null byte in native symlink target location\n"); + rc = smb_EIO2(smb_eio_trace_reparse_native_nul, ulen, len); + goto out; + } + + smb_target = cifs_strndup_from_utf16(buf, len, true, cifs_sb->local_nls); + if (!smb_target) { + rc = -ENOMEM; + goto out; + } + + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_POSIX_PATHS) && + symroot && !relative) { + /* + * This is an absolute symlink from the server which does not + * support POSIX paths, so the symlink is in NT-style path. + * So convert it to absolute Linux symlink target path. Root of + * the NT-style path for symlinks is specified in "symlinkroot" + * mount option. + * + * Root of the DOS and Win32 paths is at NT path \??\ + * It means that DOS/Win32 path C:\folder\file.txt is + * NT path \??\C:\folder\file.txt + * + * NT systems have some well-known object symlinks in their NT + * hierarchy, which is needed to take into account when resolving + * other symlinks. Most commonly used symlink paths are: + * \?? -> \GLOBAL?? + * \DosDevices -> \?? + * \GLOBAL??\GLOBALROOT -> \ + * \GLOBAL??\Global -> \GLOBAL?? + * \GLOBAL??\NUL -> \Device\Null + * \GLOBAL??\UNC -> \Device\Mup + * \GLOBAL??\PhysicalDrive0 -> \Device\Harddisk0\DR0 (for each harddisk) + * \GLOBAL??\A: -> \Device\Floppy0 (if A: is the first floppy) + * \GLOBAL??\C: -> \Device\HarddiskVolume1 (if C: is the first harddisk) + * \GLOBAL??\D: -> \Device\CdRom0 (if D: is first cdrom) + * \SystemRoot -> \Device\Harddisk0\Partition1\WINDOWS (or where is NT system installed) + * \Volume{...} -> \Device\HarddiskVolume1 (where ... is system generated guid) + * + * In most common cases, absolute NT symlinks points to path on + * DOS/Win32 drive letter, system-specific Volume or on UNC share. + * Here are few examples of commonly used absolute NT symlinks + * created by mklink.exe tool: + * \??\C:\folder\file.txt + * \??\\C:\folder\file.txt + * \??\UNC\server\share\file.txt + * \??\\UNC\server\share\file.txt + * \??\Volume{b75e2c83-0000-0000-0000-602f00000000}\folder\file.txt + * + * It means that the most common path prefix \??\ is also NT path + * symlink (to \GLOBAL??). It is less common that second path + * separator is double backslash, but it is valid. + * + * Volume guid is randomly generated by the target system and so + * only the target system knows the mapping between guid and the + * hardisk number. Over SMB it is not possible to resolve this + * mapping, therefore symlinks pointing to target location of + * volume guids are totally unusable over SMB. + * + * For now parse only symlink paths available for DOS and Win32. + * Those are paths with \??\ prefix or paths which points to \??\ + * via other NT symlink (\DosDevices\, \GLOBAL??\, ...). + */ + abs_path = smb_target; +globalroot: + if (strstarts(abs_path, "\\??\\")) + abs_path += sizeof("\\??\\")-1; + else if (strstarts(abs_path, "\\DosDevices\\")) + abs_path += sizeof("\\DosDevices\\")-1; + else if (strstarts(abs_path, "\\GLOBAL??\\")) + abs_path += sizeof("\\GLOBAL??\\")-1; + else + goto out_unhandled_target; + + /* Sometimes path separator after \?? is double backslash */ + if (abs_path[0] == '\\') + abs_path++; + + while (strstarts(abs_path, "Global\\")) + abs_path += sizeof("Global\\")-1; + + if (strstarts(abs_path, "GLOBALROOT\\")) { + /* Label globalroot requires path with leading '\\', so do not trim '\\' */ + abs_path += sizeof("GLOBALROOT")-1; + goto globalroot; + } + + /* For now parse only paths to drive letters */ + if (((abs_path[0] >= 'A' && abs_path[0] <= 'Z') || + (abs_path[0] >= 'a' && abs_path[0] <= 'z')) && + abs_path[1] == ':' && + (abs_path[2] == '\\' || abs_path[2] == '\0')) { + /* Convert drive letter to lowercase and drop colon */ + char drive_letter = abs_path[0]; + if (drive_letter >= 'A' && drive_letter <= 'Z') + drive_letter += 'a'-'A'; + abs_path++; + abs_path[0] = drive_letter; + } else { + goto out_unhandled_target; + } + + abs_path_len = strlen(abs_path)+1; + symlinkroot_len = strlen(symroot); + if (symroot[symlinkroot_len - 1] == '/') + symlinkroot_len--; + linux_target = kmalloc(symlinkroot_len + 1 + abs_path_len, GFP_KERNEL); + if (!linux_target) { + rc = -ENOMEM; + goto out; + } + memcpy(linux_target, symroot, symlinkroot_len); + linux_target[symlinkroot_len] = '/'; + memcpy(linux_target + symlinkroot_len + 1, abs_path, abs_path_len); + } else if (smb_target[0] == sep && relative) { + /* + * This is a relative SMB symlink from the top of the share, + * which is the top level directory of the Linux mount point. + * Linux does not support such relative symlinks, so convert + * it to the relative symlink from the current directory. + * full_path is the SMB path to the symlink (from which is + * extracted current directory) and smb_target is the SMB path + * where symlink points, therefore full_path must always be on + * the SMB share. + */ + int smb_target_len = strlen(smb_target)+1; + levels = 0; + for (i = 1; full_path[i]; i++) { /* i=1 to skip leading sep */ + if (full_path[i] == sep) + levels++; + } + linux_target = kmalloc(levels*3 + smb_target_len, GFP_KERNEL); + if (!linux_target) { + rc = -ENOMEM; + goto out; + } + for (i = 0; i < levels; i++) { + linux_target[i*3 + 0] = '.'; + linux_target[i*3 + 1] = '.'; + linux_target[i*3 + 2] = sep; + } + memcpy(linux_target + levels*3, smb_target+1, smb_target_len); /* +1 to skip leading sep */ + } else { + /* + * This is either an absolute symlink in POSIX-style format + * or relative SMB symlink from the current directory. + * These paths have same format as Linux symlinks, so no + * conversion is needed. + */ +out_unhandled_target: + linux_target = smb_target; + smb_target = NULL; + } + + if (sep == '\\') + convert_delimiter(linux_target, '/'); + + rc = 0; + *target = linux_target; + + cifs_dbg(FYI, "%s: symlink target: %s\n", __func__, *target); + +out: + if (rc != 0) + kfree(linux_target); + kfree(smb_target); + return rc; +} + +static int parse_reparse_native_symlink(struct reparse_symlink_data_buffer *sym, + u32 plen, + struct cifs_sb_info *cifs_sb, + const char *full_path, + struct cifs_open_info_data *data) +{ + unsigned int len; + unsigned int offs; + + /* We handle Symbolic Link reparse tag here. See: MS-FSCC 2.1.2.4 */ + + offs = le16_to_cpu(sym->SubstituteNameOffset); + len = le16_to_cpu(sym->SubstituteNameLength); + if (offs + 20 > plen || offs + len + 20 > plen) { + cifs_dbg(VFS, "srv returned malformed symlink buffer\n"); + return smb_EIO2(smb_eio_trace_reparse_native_sym_len, + offs << 16 | len, plen); + } + + return smb2_parse_native_symlink(&data->symlink_target, + sym->PathBuffer + offs, + len, + le32_to_cpu(sym->Flags) & SYMLINK_FLAG_RELATIVE, + full_path, + cifs_sb); +} + +static int parse_reparse_wsl_symlink(struct reparse_wsl_symlink_data_buffer *buf, + struct cifs_sb_info *cifs_sb, + struct cifs_open_info_data *data) +{ + int len = le16_to_cpu(buf->ReparseDataLength); + int data_offset = offsetof(typeof(*buf), Target) - offsetof(typeof(*buf), Version); + int symname_utf8_len; + __le16 *symname_utf16; + int symname_utf16_len; + + if (len <= data_offset) { + cifs_dbg(VFS, "srv returned malformed wsl symlink buffer\n"); + return smb_EIO2(smb_eio_trace_reparse_wsl_symbuf, + len, data_offset); + } + + /* MS-FSCC 2.1.2.7 defines layout of the Target field only for Version 2. */ + u32 version = le32_to_cpu(buf->Version); + + if (version != 2) { + cifs_dbg(VFS, "srv returned unsupported wsl symlink version %u\n", version); + return smb_EIO1(smb_eio_trace_reparse_wsl_ver, version); + } + + /* Target for Version 2 is in UTF-8 but without trailing null-term byte */ + symname_utf8_len = len - data_offset; + /* + * Check that buffer does not contain null byte + * because Linux cannot process symlink with null byte. + */ + size_t ulen = strnlen(buf->Target, symname_utf8_len); + + if (ulen != symname_utf8_len) { + cifs_dbg(VFS, "srv returned null byte in wsl symlink target location\n"); + return smb_EIO2(smb_eio_trace_reparse_wsl_ver, + ulen, symname_utf8_len); + } + symname_utf16 = kzalloc(symname_utf8_len * 2, GFP_KERNEL); + if (!symname_utf16) + return -ENOMEM; + symname_utf16_len = utf8s_to_utf16s(buf->Target, symname_utf8_len, + UTF16_LITTLE_ENDIAN, + (wchar_t *) symname_utf16, symname_utf8_len * 2); + if (symname_utf16_len < 0) { + kfree(symname_utf16); + return symname_utf16_len; + } + symname_utf16_len *= 2; /* utf8s_to_utf16s() returns number of u16 items, not byte length */ + + data->symlink_target = cifs_strndup_from_utf16((u8 *)symname_utf16, + symname_utf16_len, true, + cifs_sb->local_nls); + kfree(symname_utf16); + if (!data->symlink_target) + return -ENOMEM; + + return 0; +} + +int parse_reparse_point(struct reparse_data_buffer *buf, + u32 plen, struct cifs_sb_info *cifs_sb, + const char *full_path, + struct cifs_open_info_data *data) +{ + data->reparse.buf = buf; + + /* See MS-FSCC 2.1.2 */ + switch (le32_to_cpu(buf->ReparseTag)) { + case IO_REPARSE_TAG_NFS: + return parse_reparse_nfs((struct reparse_nfs_data_buffer *)buf, + cifs_sb, data); + case IO_REPARSE_TAG_SYMLINK: + return parse_reparse_native_symlink( + (struct reparse_symlink_data_buffer *)buf, + plen, cifs_sb, full_path, data); + case IO_REPARSE_TAG_LX_SYMLINK: + return parse_reparse_wsl_symlink( + (struct reparse_wsl_symlink_data_buffer *)buf, + cifs_sb, data); + case IO_REPARSE_TAG_AF_UNIX: + case IO_REPARSE_TAG_LX_FIFO: + case IO_REPARSE_TAG_LX_CHR: + case IO_REPARSE_TAG_LX_BLK: { + u16 dlen = le16_to_cpu(buf->ReparseDataLength); + + if (dlen != 0) { + u32 rtag = le32_to_cpu(buf->ReparseTag); + cifs_dbg(VFS, "srv returned malformed buffer for reparse point: 0x%08x\n", + rtag); + return smb_EIO2(smb_eio_trace_reparse_data_len, dlen, rtag); + } + return 0; + } + default: + return -EOPNOTSUPP; + } +} + +struct reparse_data_buffer *smb2_get_reparse_point_buffer(const struct kvec *rsp_iov, + u32 *plen) +{ + struct smb2_ioctl_rsp *io = rsp_iov->iov_base; + *plen = le32_to_cpu(io->OutputCount); + return (struct reparse_data_buffer *)((u8 *)io + + le32_to_cpu(io->OutputOffset)); +} + +static bool wsl_to_fattr(struct cifs_open_info_data *data, + struct cifs_sb_info *cifs_sb, + u32 tag, struct cifs_fattr *fattr) +{ + struct smb2_file_full_ea_info *ea; + bool have_xattr_dev = false; + u32 next = 0; + + switch (tag) { + case IO_REPARSE_TAG_LX_SYMLINK: + fattr->cf_mode |= S_IFLNK; + break; + case IO_REPARSE_TAG_LX_FIFO: + fattr->cf_mode |= S_IFIFO; + break; + case IO_REPARSE_TAG_AF_UNIX: + fattr->cf_mode |= S_IFSOCK; + break; + case IO_REPARSE_TAG_LX_CHR: + fattr->cf_mode |= S_IFCHR; + break; + case IO_REPARSE_TAG_LX_BLK: + fattr->cf_mode |= S_IFBLK; + break; + } + + if (!data->wsl.eas_len) + goto out; + + ea = (struct smb2_file_full_ea_info *)data->wsl.eas; + do { + const char *name; + void *v; + u8 nlen; + + ea = (void *)((u8 *)ea + next); + next = le32_to_cpu(ea->next_entry_offset); + if (!le16_to_cpu(ea->ea_value_length)) + continue; + + name = ea->ea_data; + nlen = ea->ea_name_length; + v = (void *)((u8 *)ea->ea_data + ea->ea_name_length + 1); + + if (!strncmp(name, SMB2_WSL_XATTR_UID, nlen)) + fattr->cf_uid = wsl_make_kuid(cifs_sb, v); + else if (!strncmp(name, SMB2_WSL_XATTR_GID, nlen)) + fattr->cf_gid = wsl_make_kgid(cifs_sb, v); + else if (!strncmp(name, SMB2_WSL_XATTR_MODE, nlen)) { + /* File type in reparse point tag and in xattr mode must match. */ + if (S_DT(fattr->cf_mode) != S_DT(le32_to_cpu(*(__le32 *)v))) + return false; + fattr->cf_mode = (umode_t)le32_to_cpu(*(__le32 *)v); + } else if (!strncmp(name, SMB2_WSL_XATTR_DEV, nlen)) { + fattr->cf_rdev = reparse_mkdev(v); + have_xattr_dev = true; + } + } while (next); +out: + + /* Major and minor numbers for char and block devices are mandatory. */ + if (!have_xattr_dev && (tag == IO_REPARSE_TAG_LX_CHR || tag == IO_REPARSE_TAG_LX_BLK)) + return false; + + return true; +} + +static bool posix_reparse_to_fattr(struct cifs_sb_info *cifs_sb, + struct cifs_fattr *fattr, + struct cifs_open_info_data *data) +{ + struct reparse_nfs_data_buffer *buf = (struct reparse_nfs_data_buffer *)data->reparse.buf; + + if (buf == NULL) + return true; + + if (le16_to_cpu(buf->ReparseDataLength) < sizeof(buf->InodeType)) { + WARN_ON_ONCE(1); + return false; + } + + switch (le64_to_cpu(buf->InodeType)) { + case NFS_SPECFILE_CHR: + if (le16_to_cpu(buf->ReparseDataLength) != sizeof(buf->InodeType) + 8) { + WARN_ON_ONCE(1); + return false; + } + fattr->cf_mode |= S_IFCHR; + fattr->cf_rdev = reparse_mkdev(buf->DataBuffer); + break; + case NFS_SPECFILE_BLK: + if (le16_to_cpu(buf->ReparseDataLength) != sizeof(buf->InodeType) + 8) { + WARN_ON_ONCE(1); + return false; + } + fattr->cf_mode |= S_IFBLK; + fattr->cf_rdev = reparse_mkdev(buf->DataBuffer); + break; + case NFS_SPECFILE_FIFO: + fattr->cf_mode |= S_IFIFO; + break; + case NFS_SPECFILE_SOCK: + fattr->cf_mode |= S_IFSOCK; + break; + case NFS_SPECFILE_LNK: + fattr->cf_mode |= S_IFLNK; + break; + default: + WARN_ON_ONCE(1); + return false; + } + return true; +} + +bool cifs_reparse_point_to_fattr(struct cifs_sb_info *cifs_sb, + struct cifs_fattr *fattr, + struct cifs_open_info_data *data) +{ + u32 tag = data->reparse.tag; + bool ok; + + switch (tag) { + case IO_REPARSE_TAG_LX_SYMLINK: + case IO_REPARSE_TAG_LX_FIFO: + case IO_REPARSE_TAG_AF_UNIX: + case IO_REPARSE_TAG_LX_CHR: + case IO_REPARSE_TAG_LX_BLK: + ok = wsl_to_fattr(data, cifs_sb, tag, fattr); + if (!ok) + return false; + break; + case IO_REPARSE_TAG_NFS: + ok = posix_reparse_to_fattr(cifs_sb, fattr, data); + if (!ok) + return false; + break; + case 0: /* SMB1 symlink */ + case IO_REPARSE_TAG_SYMLINK: + fattr->cf_mode |= S_IFLNK; + break; + default: + if (!(fattr->cf_cifsattrs & ATTR_DIRECTORY)) + return false; + if (!IS_REPARSE_TAG_NAME_SURROGATE(tag) && + tag != IO_REPARSE_TAG_INTERNAL) + return false; + /* See cifs_create_junction_fattr() */ + fattr->cf_mode = S_IFDIR | 0711; + break; + } + + fattr->cf_dtype = S_DT(fattr->cf_mode); + return true; +} diff --git a/fs/smb/client/reparse.h b/fs/smb/client/reparse.h new file mode 100644 index 000000000..0164dc47b --- /dev/null +++ b/fs/smb/client/reparse.h @@ -0,0 +1,142 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2024 Paulo Alcantara + */ + +#ifndef _CIFS_REPARSE_H +#define _CIFS_REPARSE_H + +#include +#include +#include +#include "fs_context.h" +#include "cifsglob.h" +#include "../common/smbfsctl.h" + +#define REPARSE_SYM_PATH_MAX 4060 + +/* + * Used only by cifs.ko to ignore reparse points from files when client or + * server doesn't support FSCTL_GET_REPARSE_POINT. + */ +#define IO_REPARSE_TAG_INTERNAL ((__u32)~0U) + +static inline dev_t reparse_mkdev(void *ptr) +{ + u64 v = le64_to_cpu(*(__le64 *)ptr); + + return MKDEV(v & 0xffffffff, v >> 32); +} + +static inline kuid_t wsl_make_kuid(struct cifs_sb_info *cifs_sb, + void *ptr) +{ + u32 uid = le32_to_cpu(*(__le32 *)ptr); + + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_OVERR_UID) + return cifs_sb->ctx->linux_uid; + return make_kuid(current_user_ns(), uid); +} + +static inline kgid_t wsl_make_kgid(struct cifs_sb_info *cifs_sb, + void *ptr) +{ + u32 gid = le32_to_cpu(*(__le32 *)ptr); + + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_OVERR_GID) + return cifs_sb->ctx->linux_gid; + return make_kgid(current_user_ns(), gid); +} + +static inline u64 reparse_mode_nfs_type(mode_t mode) +{ + switch (mode & S_IFMT) { + case S_IFLNK: return NFS_SPECFILE_LNK; + case S_IFBLK: return NFS_SPECFILE_BLK; + case S_IFCHR: return NFS_SPECFILE_CHR; + case S_IFIFO: return NFS_SPECFILE_FIFO; + case S_IFSOCK: return NFS_SPECFILE_SOCK; + } + return 0; +} + +static inline u32 reparse_mode_wsl_tag(mode_t mode) +{ + switch (mode & S_IFMT) { + case S_IFLNK: return IO_REPARSE_TAG_LX_SYMLINK; + case S_IFBLK: return IO_REPARSE_TAG_LX_BLK; + case S_IFCHR: return IO_REPARSE_TAG_LX_CHR; + case S_IFIFO: return IO_REPARSE_TAG_LX_FIFO; + case S_IFSOCK: return IO_REPARSE_TAG_AF_UNIX; + } + return 0; +} + +/* + * Match a reparse point inode if reparse tag and ctime haven't changed. + * + * Windows Server updates ctime of reparse points when their data have changed. + * The server doesn't allow changing reparse tags from existing reparse points, + * though it's worth checking. + */ +static inline bool reparse_inode_match(struct inode *inode, + struct cifs_fattr *fattr) +{ + struct cifsInodeInfo *cinode = CIFS_I(inode); + struct timespec64 ctime = inode_get_ctime(inode); + + /* + * Do not match reparse tags when client or server doesn't support + * FSCTL_GET_REPARSE_POINT. @fattr->cf_cifstag should contain correct + * reparse tag from query dir response but the client won't be able to + * read the reparse point data anyway. This spares us a revalidation. + */ + if (cinode->reparse_tag != IO_REPARSE_TAG_INTERNAL && + cinode->reparse_tag != fattr->cf_cifstag) + return false; + return (cinode->cifsAttrs & ATTR_REPARSE_POINT) && + timespec64_equal(&ctime, &fattr->cf_ctime); +} + +static inline bool cifs_open_data_reparse(struct cifs_open_info_data *data) +{ + u32 attrs; + bool ret; + + if (data->contains_posix_file_info) { + struct smb311_posix_qinfo *fi = &data->posix_fi; + + attrs = le32_to_cpu(fi->DosAttributes); + if (data->reparse_point) { + attrs |= ATTR_REPARSE_POINT; + fi->DosAttributes = cpu_to_le32(attrs); + } + + } else { + struct smb2_file_all_info *fi = &data->fi; + + attrs = le32_to_cpu(fi->Attributes); + if (data->reparse_point) { + attrs |= ATTR_REPARSE_POINT; + fi->Attributes = cpu_to_le32(attrs); + } + } + + ret = attrs & ATTR_REPARSE_POINT; + + return ret; +} + +bool cifs_reparse_point_to_fattr(struct cifs_sb_info *cifs_sb, + struct cifs_fattr *fattr, + struct cifs_open_info_data *data); +int create_reparse_symlink(const unsigned int xid, struct inode *inode, + struct dentry *dentry, struct cifs_tcon *tcon, + const char *full_path, const char *symname); +int mknod_reparse(unsigned int xid, struct inode *inode, struct dentry *dentry, + struct cifs_tcon *tcon, const char *full_path, umode_t mode, + dev_t dev); +struct reparse_data_buffer *smb2_get_reparse_point_buffer(const struct kvec *rsp_iov, + u32 *plen); + +#endif /* _CIFS_REPARSE_H */ diff --git a/fs/smb/client/rfc1002pdu.h b/fs/smb/client/rfc1002pdu.h new file mode 100644 index 000000000..f5b143088 --- /dev/null +++ b/fs/smb/client/rfc1002pdu.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +/* + * + * Protocol Data Unit definitions for RFC 1001/1002 support + * + * Copyright (c) International Business Machines Corp., 2004 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ + +/* NB: unlike smb/cifs packets, the RFC1002 structures are big endian */ + + /* RFC 1002 session packet types */ +#define RFC1002_SESSION_MESSAGE 0x00 +#define RFC1002_SESSION_REQUEST 0x81 +#define RFC1002_POSITIVE_SESSION_RESPONSE 0x82 +#define RFC1002_NEGATIVE_SESSION_RESPONSE 0x83 +#define RFC1002_RETARGET_SESSION_RESPONSE 0x84 +#define RFC1002_SESSION_KEEP_ALIVE 0x85 + + /* RFC 1002 flags (only one defined */ +#define RFC1002_LENGTH_EXTEND 0x80 /* high order bit of length (ie +64K) */ + +struct rfc1002_session_packet { + __u8 type; + __u8 flags; + __be16 length; + union { + struct { + __u8 called_len; + __u8 called_name[32]; + __u8 scope1; /* null */ + __u8 calling_len; + __u8 calling_name[32]; + __u8 scope2; /* null */ + } __packed session_req; + struct { + __be32 retarget_ip_addr; + __be16 port; + } __packed retarget_resp; + __u8 neg_ses_resp_error_code; + /* POSITIVE_SESSION_RESPONSE packet does not include trailer. + SESSION_KEEP_ALIVE packet also does not include a trailer. + Trailer for the SESSION_MESSAGE packet is SMB/CIFS header */ + } __packed trailer; +} __packed; + +/* Negative Session Response error codes */ +#define RFC1002_NOT_LISTENING_CALLED 0x80 /* not listening on called name */ +#define RFC1002_NOT_LISTENING_CALLING 0x81 /* not listening on calling name */ +#define RFC1002_NOT_PRESENT 0x82 /* called name not present */ +#define RFC1002_INSUFFICIENT_RESOURCE 0x83 +#define RFC1002_UNSPECIFIED_ERROR 0x8F + +/* RFC 1002 Datagram service packets are not defined here as they +are not needed for the network filesystem client unless we plan on +implementing broadcast resolution of the server ip address (from +server netbios name). Currently server names are resolved only via DNS +(tcp name) or ip address or an /etc/hosts equivalent mapping to ip address.*/ + +#define DEFAULT_CIFS_CALLED_NAME "*SMBSERVER " diff --git a/fs/smb/client/sess.c b/fs/smb/client/sess.c new file mode 100644 index 000000000..7cf7dd104 --- /dev/null +++ b/fs/smb/client/sess.c @@ -0,0 +1,1039 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * SMB/CIFS session setup handling routines + * + * Copyright (c) International Business Machines Corp., 2006, 2009 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ + +#include "cifsglob.h" +#include "cifsproto.h" +#include "cifs_unicode.h" +#include "cifs_debug.h" +#include "ntlmssp.h" +#include "nterr.h" +#include +#include +#include +#include "cifsfs.h" +#include "cifs_spnego.h" +#include "smb2proto.h" +#include "fs_context.h" + +static int +cifs_ses_add_channel(struct cifs_ses *ses, + struct cifs_server_iface *iface); + +bool is_ses_using_iface(struct cifs_ses *ses, struct cifs_server_iface *iface) +{ + int i; + + spin_lock(&ses->chan_lock); + for (i = 0; i < ses->chan_count; i++) { + if (ses->chans[i].iface == iface) { + spin_unlock(&ses->chan_lock); + return true; + } + } + spin_unlock(&ses->chan_lock); + return false; +} + +/* channel helper functions. assumed that chan_lock is held by caller. */ + +int +cifs_ses_get_chan_index(struct cifs_ses *ses, + struct TCP_Server_Info *server) +{ + unsigned int i; + + /* if the channel is waiting for termination */ + if (server && server->terminate) + return CIFS_INVAL_CHAN_INDEX; + + for (i = 0; i < ses->chan_count; i++) { + if (ses->chans[i].server == server) + return i; + } + + /* If we didn't find the channel, it is likely a bug */ + if (server) + cifs_dbg(VFS, "unable to get chan index for server: 0x%llx", + server->conn_id); + return CIFS_INVAL_CHAN_INDEX; +} + +void +cifs_chan_set_in_reconnect(struct cifs_ses *ses, + struct TCP_Server_Info *server) +{ + int chan_index = cifs_ses_get_chan_index(ses, server); + + if (chan_index == CIFS_INVAL_CHAN_INDEX) + return; + + ses->chans[chan_index].in_reconnect = true; +} + +void +cifs_chan_clear_in_reconnect(struct cifs_ses *ses, + struct TCP_Server_Info *server) +{ + unsigned int chan_index = cifs_ses_get_chan_index(ses, server); + + if (chan_index == CIFS_INVAL_CHAN_INDEX) + return; + + ses->chans[chan_index].in_reconnect = false; +} + +void +cifs_chan_set_need_reconnect(struct cifs_ses *ses, + struct TCP_Server_Info *server) +{ + unsigned int chan_index = cifs_ses_get_chan_index(ses, server); + + if (chan_index == CIFS_INVAL_CHAN_INDEX) + return; + + set_bit(chan_index, &ses->chans_need_reconnect); + cifs_dbg(FYI, "Set reconnect bitmask for chan %u; now 0x%lx\n", + chan_index, ses->chans_need_reconnect); +} + +void +cifs_chan_clear_need_reconnect(struct cifs_ses *ses, + struct TCP_Server_Info *server) +{ + unsigned int chan_index = cifs_ses_get_chan_index(ses, server); + + if (chan_index == CIFS_INVAL_CHAN_INDEX) + return; + + clear_bit(chan_index, &ses->chans_need_reconnect); + cifs_dbg(FYI, "Cleared reconnect bitmask for chan %u; now 0x%lx\n", + chan_index, ses->chans_need_reconnect); +} + +bool +cifs_chan_needs_reconnect(struct cifs_ses *ses, + struct TCP_Server_Info *server) +{ + unsigned int chan_index = cifs_ses_get_chan_index(ses, server); + + if (chan_index == CIFS_INVAL_CHAN_INDEX) + return true; /* err on the safer side */ + + return CIFS_CHAN_NEEDS_RECONNECT(ses, chan_index); +} + +bool +cifs_chan_is_iface_active(struct cifs_ses *ses, + struct TCP_Server_Info *server) +{ + unsigned int chan_index = cifs_ses_get_chan_index(ses, server); + + if (chan_index == CIFS_INVAL_CHAN_INDEX) + return true; /* err on the safer side */ + + return ses->chans[chan_index].iface && + ses->chans[chan_index].iface->is_active; +} + +/* returns number of channels added */ +int cifs_try_adding_channels(struct cifs_ses *ses) +{ + struct TCP_Server_Info *server = ses->server; + int old_chan_count, new_chan_count; + int left; + int rc = 0; + int tries = 0; + size_t iface_weight = 0, iface_min_speed = 0; + struct cifs_server_iface *iface = NULL, *niface = NULL; + struct cifs_server_iface *last_iface = NULL; + + spin_lock(&ses->chan_lock); + + new_chan_count = old_chan_count = ses->chan_count; + left = ses->chan_max - ses->chan_count; + + if (left <= 0) { + spin_unlock(&ses->chan_lock); + cifs_dbg(FYI, + "ses already at max_channels (%zu), nothing to open\n", + ses->chan_max); + return 0; + } + + if (server->dialect < SMB30_PROT_ID) { + spin_unlock(&ses->chan_lock); + cifs_dbg(VFS, "multichannel is not supported on this protocol version, use 3.0 or above\n"); + return 0; + } + + if (!(server->capabilities & SMB2_GLOBAL_CAP_MULTI_CHANNEL)) { + spin_unlock(&ses->chan_lock); + cifs_server_dbg(VFS, "no multichannel support\n"); + return 0; + } + spin_unlock(&ses->chan_lock); + + while (left > 0) { + + tries++; + if (tries > 3*ses->chan_max) { + cifs_dbg(VFS, "too many channel open attempts (%d channels left to open)\n", + left); + break; + } + + spin_lock(&ses->iface_lock); + if (!ses->iface_count) { + spin_unlock(&ses->iface_lock); + cifs_dbg(ONCE, "server %s does not advertise interfaces\n", + ses->server->hostname); + break; + } + + if (!iface) + iface = list_first_entry(&ses->iface_list, struct cifs_server_iface, + iface_head); + last_iface = list_last_entry(&ses->iface_list, struct cifs_server_iface, + iface_head); + iface_min_speed = last_iface->speed; + + list_for_each_entry_safe_from(iface, niface, &ses->iface_list, + iface_head) { + /* do not mix rdma and non-rdma interfaces */ + if (iface->rdma_capable != ses->server->rdma) + continue; + + /* skip ifaces that are unusable */ + if (!iface->is_active || + (is_ses_using_iface(ses, iface) && + !iface->rss_capable)) + continue; + + /* check if we already allocated enough channels */ + iface_weight = iface->speed / iface_min_speed; + + if (iface->weight_fulfilled >= iface_weight) + continue; + + /* take ref before unlock */ + kref_get(&iface->refcount); + + spin_unlock(&ses->iface_lock); + rc = cifs_ses_add_channel(ses, iface); + spin_lock(&ses->iface_lock); + + if (rc) { + cifs_dbg(VFS, "failed to open extra channel on iface:%pIS rc=%d\n", + &iface->sockaddr, + rc); + /* failure to add chan should increase weight */ + iface->weight_fulfilled++; + kref_put(&iface->refcount, release_iface); + continue; + } + + iface->num_channels++; + iface->weight_fulfilled++; + cifs_info("successfully opened new channel on iface:%pIS\n", + &iface->sockaddr); + break; + } + + /* reached end of list. reset weight_fulfilled and start over */ + if (list_entry_is_head(iface, &ses->iface_list, iface_head)) { + list_for_each_entry(iface, &ses->iface_list, iface_head) + iface->weight_fulfilled = 0; + spin_unlock(&ses->iface_lock); + iface = NULL; + continue; + } + spin_unlock(&ses->iface_lock); + + left--; + new_chan_count++; + } + + return new_chan_count - old_chan_count; +} + +/* + * cifs_decrease_secondary_channels - Reduce the number of active secondary channels + * @ses: pointer to the CIFS session structure + * @disable_mchan: if true, reduce to a single channel; if false, reduce to chan_max + * + * This function disables and cleans up extra secondary channels for a CIFS session. + * If called during reconfiguration, it reduces the channel count to the new maximum (chan_max). + * Otherwise, it disables all but the primary channel. + */ +void +cifs_decrease_secondary_channels(struct cifs_ses *ses, bool disable_mchan) +{ + int i, chan_count; + struct TCP_Server_Info *server; + struct cifs_server_iface *iface; + + spin_lock(&ses->chan_lock); + chan_count = ses->chan_count; + if (chan_count == 1) + goto done; + + /* Update the chan_count to the new maximum */ + if (disable_mchan) { + cifs_dbg(FYI, "server does not support multichannel anymore.\n"); + ses->chan_count = 1; + } else { + ses->chan_count = ses->chan_max; + } + + /* Disable all secondary channels beyond the new chan_count */ + for (i = ses->chan_count ; i < chan_count; i++) { + iface = ses->chans[i].iface; + server = ses->chans[i].server; + + /* + * remove these references first, since we need to unlock + * the chan_lock here, since iface_lock is a higher lock + */ + ses->chans[i].iface = NULL; + ses->chans[i].server = NULL; + spin_unlock(&ses->chan_lock); + + if (iface) { + spin_lock(&ses->iface_lock); + iface->num_channels--; + if (iface->weight_fulfilled) + iface->weight_fulfilled--; + kref_put(&iface->refcount, release_iface); + spin_unlock(&ses->iface_lock); + } + + if (server) { + if (!server->terminate) { + server->terminate = true; + cifs_signal_cifsd_for_reconnect(server, false); + } + cifs_put_tcp_session(server, false); + } + + spin_lock(&ses->chan_lock); + } + + /* For extra secondary channels, reset the need reconnect bit */ + if (ses->chan_count == 1) { + cifs_dbg(VFS, "Disable all secondary channels\n"); + ses->chans_need_reconnect &= 1; + } else { + cifs_dbg(VFS, "Disable extra secondary channels\n"); + ses->chans_need_reconnect &= ((1UL << ses->chan_max) - 1); + } + +done: + spin_unlock(&ses->chan_lock); +} + +/* update the iface for the channel if necessary. */ +void +cifs_chan_update_iface(struct cifs_ses *ses, struct TCP_Server_Info *server) +{ + unsigned int chan_index; + size_t iface_weight = 0, iface_min_speed = 0; + struct cifs_server_iface *iface = NULL; + struct cifs_server_iface *old_iface = NULL; + struct cifs_server_iface *last_iface = NULL; + struct sockaddr_storage ss; + int retry = 0; + + spin_lock(&ses->chan_lock); + chan_index = cifs_ses_get_chan_index(ses, server); + if (chan_index == CIFS_INVAL_CHAN_INDEX) { + spin_unlock(&ses->chan_lock); + return; + } + + if (ses->chans[chan_index].iface) { + old_iface = ses->chans[chan_index].iface; + if (old_iface->is_active) { + spin_unlock(&ses->chan_lock); + return; + } + } + spin_unlock(&ses->chan_lock); + + spin_lock(&server->srv_lock); + ss = server->dstaddr; + spin_unlock(&server->srv_lock); + + spin_lock(&ses->iface_lock); + if (!ses->iface_count) { + spin_unlock(&ses->iface_lock); + cifs_dbg(ONCE, "server %s does not advertise interfaces\n", ses->server->hostname); + return; + } + +try_again: + last_iface = list_last_entry(&ses->iface_list, struct cifs_server_iface, + iface_head); + iface_min_speed = last_iface->speed; + + /* then look for a new one */ + list_for_each_entry(iface, &ses->iface_list, iface_head) { + if (!chan_index) { + /* if we're trying to get the updated iface for primary channel */ + if (!cifs_match_ipaddr((struct sockaddr *) &ss, + (struct sockaddr *) &iface->sockaddr)) + continue; + + kref_get(&iface->refcount); + break; + } + + /* do not mix rdma and non-rdma interfaces */ + if (iface->rdma_capable != server->rdma) + continue; + + if (!iface->is_active || + (is_ses_using_iface(ses, iface) && + !iface->rss_capable)) { + continue; + } + + /* check if we already allocated enough channels */ + iface_weight = iface->speed / iface_min_speed; + + if (iface->weight_fulfilled >= iface_weight) + continue; + + kref_get(&iface->refcount); + break; + } + + if (list_entry_is_head(iface, &ses->iface_list, iface_head)) { + list_for_each_entry(iface, &ses->iface_list, iface_head) + iface->weight_fulfilled = 0; + + /* see if it can be satisfied in second attempt */ + if (!retry++) + goto try_again; + + iface = NULL; + cifs_dbg(FYI, "unable to find a suitable iface\n"); + } + + if (!iface) { + if (!chan_index) + cifs_dbg(FYI, "unable to get the interface matching: %pIS\n", + &ss); + else { + cifs_dbg(FYI, "unable to find another interface to replace: %pIS\n", + &old_iface->sockaddr); + } + + spin_unlock(&ses->iface_lock); + return; + } + + /* now drop the ref to the current iface */ + if (old_iface) { + cifs_dbg(FYI, "replacing iface: %pIS with %pIS\n", + &old_iface->sockaddr, + &iface->sockaddr); + + old_iface->num_channels--; + if (old_iface->weight_fulfilled) + old_iface->weight_fulfilled--; + iface->num_channels++; + iface->weight_fulfilled++; + + kref_put(&old_iface->refcount, release_iface); + } else if (!chan_index) { + /* special case: update interface for primary channel */ + cifs_dbg(FYI, "referencing primary channel iface: %pIS\n", + &iface->sockaddr); + iface->num_channels++; + iface->weight_fulfilled++; + } + spin_unlock(&ses->iface_lock); + + spin_lock(&ses->chan_lock); + chan_index = cifs_ses_get_chan_index(ses, server); + if (chan_index == CIFS_INVAL_CHAN_INDEX) { + spin_unlock(&ses->chan_lock); + return; + } + + ses->chans[chan_index].iface = iface; + spin_unlock(&ses->chan_lock); + + spin_lock(&server->srv_lock); + memcpy(&server->dstaddr, &iface->sockaddr, sizeof(server->dstaddr)); + spin_unlock(&server->srv_lock); +} + +static int +cifs_ses_add_channel(struct cifs_ses *ses, + struct cifs_server_iface *iface) +{ + struct TCP_Server_Info *chan_server; + struct cifs_chan *chan; + struct smb3_fs_context *ctx; + static const char unc_fmt[] = "\\%s\\foo"; + struct sockaddr_in *ipv4 = (struct sockaddr_in *)&iface->sockaddr; + struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)&iface->sockaddr; + size_t len; + int rc; + unsigned int xid = get_xid(); + + if (iface->sockaddr.ss_family == AF_INET) + cifs_dbg(FYI, "adding channel to ses %p (speed:%zu bps rdma:%s ip:%pI4)\n", + ses, iface->speed, str_yes_no(iface->rdma_capable), + &ipv4->sin_addr); + else + cifs_dbg(FYI, "adding channel to ses %p (speed:%zu bps rdma:%s ip:%pI6)\n", + ses, iface->speed, str_yes_no(iface->rdma_capable), + &ipv6->sin6_addr); + + /* + * Setup a ctx with mostly the same info as the existing + * session and overwrite it with the requested iface data. + * + * We need to setup at least the fields used for negprot and + * sesssetup. + * + * We only need the ctx here, so we can reuse memory from + * the session and server without caring about memory + * management. + */ + ctx = kzalloc_obj(*ctx); + if (!ctx) { + rc = -ENOMEM; + goto out_free_xid; + } + + /* Always make new connection for now (TODO?) */ + ctx->nosharesock = true; + + /* Auth */ + ctx->domainauto = ses->domainAuto; + ctx->domainname = ses->domainName; + + ctx->server_hostname = ses->server->hostname; + + ctx->username = ses->user_name; + ctx->password = ses->password; + ctx->sectype = ses->sectype; + ctx->sign = ses->sign; + ctx->unicode = ses->unicode; + + /* UNC and paths */ + /* XXX: Use ses->server->hostname? */ + len = sizeof(unc_fmt) + SERVER_NAME_LEN_WITH_NULL; + ctx->UNC = kzalloc(len, GFP_KERNEL); + if (!ctx->UNC) { + rc = -ENOMEM; + goto out_free_ctx; + } + scnprintf(ctx->UNC, len, unc_fmt, ses->ip_addr); + ctx->prepath = ""; + + /* Reuse same version as master connection */ + ctx->vals = ses->server->vals; + ctx->ops = ses->server->ops; + + ctx->noblocksnd = ses->server->noblocksnd; + ctx->noautotune = ses->server->noautotune; + ctx->sockopt_tcp_nodelay = ses->server->tcp_nodelay; + ctx->echo_interval = ses->server->echo_interval / HZ; + ctx->max_credits = ses->server->max_credits; + ctx->min_offload = ses->server->min_offload; + ctx->compress = ses->server->compression.requested; + ctx->dfs_conn = ses->server->dfs_conn; + ctx->ignore_signature = ses->server->ignore_signature; + ctx->leaf_fullpath = ses->server->leaf_fullpath; + ctx->rootfs = ses->server->noblockcnt; + ctx->retrans = ses->server->retrans; + + /* + * This will be used for encoding/decoding user/domain/pw + * during sess setup auth. + */ + ctx->local_nls = ses->local_nls; + + /* Use RDMA if possible */ + ctx->rdma = iface->rdma_capable; + memcpy(&ctx->dstaddr, &iface->sockaddr, sizeof(ctx->dstaddr)); + + /* reuse master con client guid */ + memcpy(&ctx->client_guid, ses->server->client_guid, + sizeof(ctx->client_guid)); + ctx->use_client_guid = true; + + chan_server = cifs_get_tcp_session(ctx, ses->server); + + spin_lock(&ses->chan_lock); + chan = &ses->chans[ses->chan_count]; + chan->server = chan_server; + if (IS_ERR(chan->server)) { + rc = PTR_ERR(chan->server); + chan->server = NULL; + spin_unlock(&ses->chan_lock); + goto out; + } + chan->iface = iface; + ses->chan_count++; + atomic_set(&ses->chan_seq, 0); + + /* Mark this channel as needing connect/setup */ + cifs_chan_set_need_reconnect(ses, chan->server); + + spin_unlock(&ses->chan_lock); + + mutex_lock(&ses->session_mutex); + + rc = cifs_negotiate_protocol(xid, ses, chan->server); + if (!rc) + rc = cifs_setup_session(xid, ses, chan->server, ses->local_nls); + + mutex_unlock(&ses->session_mutex); + +out: + if (rc && chan->server) { + cifs_put_tcp_session(chan->server, 0); + + spin_lock(&ses->chan_lock); + + /* we rely on all bits beyond chan_count to be clear */ + cifs_chan_clear_need_reconnect(ses, chan->server); + ses->chan_count--; + /* + * chan_count should never reach 0 as at least the primary + * channel is always allocated + */ + WARN_ON(ses->chan_count < 1); + spin_unlock(&ses->chan_lock); + } + + kfree(ctx->UNC); +out_free_ctx: + kfree(ctx); +out_free_xid: + free_xid(xid); + return rc; +} + + +int decode_ntlmssp_challenge(char *bcc_ptr, int blob_len, + struct cifs_ses *ses) +{ + unsigned int tioffset; /* challenge message target info area */ + unsigned int tilen; /* challenge message target info area length */ + CHALLENGE_MESSAGE *pblob = (CHALLENGE_MESSAGE *)bcc_ptr; + __u32 server_flags; + + if (blob_len < sizeof(CHALLENGE_MESSAGE)) { + cifs_dbg(VFS, "challenge blob len %d too small\n", blob_len); + return -EINVAL; + } + + if (memcmp(pblob->Signature, "NTLMSSP", 8)) { + cifs_dbg(VFS, "blob signature incorrect %s\n", + pblob->Signature); + return -EINVAL; + } + if (pblob->MessageType != NtLmChallenge) { + cifs_dbg(VFS, "Incorrect message type %d\n", + pblob->MessageType); + return -EINVAL; + } + + server_flags = le32_to_cpu(pblob->NegotiateFlags); + cifs_dbg(FYI, "%s: negotiate=0x%08x challenge=0x%08x\n", __func__, + ses->ntlmssp->client_flags, server_flags); + + if ((ses->ntlmssp->client_flags & (NTLMSSP_NEGOTIATE_SEAL | NTLMSSP_NEGOTIATE_SIGN)) && + (!(server_flags & NTLMSSP_NEGOTIATE_56) && !(server_flags & NTLMSSP_NEGOTIATE_128))) { + cifs_dbg(VFS, "%s: requested signing/encryption but server did not return either 56-bit or 128-bit session key size\n", + __func__); + return -EINVAL; + } + if (!(server_flags & NTLMSSP_NEGOTIATE_NTLM) && !(server_flags & NTLMSSP_NEGOTIATE_EXTENDED_SEC)) { + cifs_dbg(VFS, "%s: server does not seem to support either NTLMv1 or NTLMv2\n", __func__); + return -EINVAL; + } + if (ses->server->sign && !(server_flags & NTLMSSP_NEGOTIATE_SIGN)) { + cifs_dbg(VFS, "%s: forced packet signing but server does not seem to support it\n", + __func__); + return -EOPNOTSUPP; + } + if ((ses->ntlmssp->client_flags & NTLMSSP_NEGOTIATE_KEY_XCH) && + !(server_flags & NTLMSSP_NEGOTIATE_KEY_XCH)) + pr_warn_once("%s: authentication has been weakened as server does not support key exchange\n", + __func__); + + ses->ntlmssp->server_flags = server_flags; + + memcpy(ses->ntlmssp->cryptkey, pblob->Challenge, CIFS_CRYPTO_KEY_SIZE); + /* + * In particular we can examine sign flags + * + * BB spec says that if AvId field of MsvAvTimestamp is populated then + * we must set the MIC field of the AUTHENTICATE_MESSAGE + */ + + tioffset = le32_to_cpu(pblob->TargetInfoArray.BufferOffset); + tilen = le16_to_cpu(pblob->TargetInfoArray.Length); + if (tioffset > blob_len || tioffset + tilen > blob_len) { + cifs_dbg(VFS, "tioffset + tilen too high %u + %u\n", + tioffset, tilen); + return -EINVAL; + } + if (tilen) { + kfree_sensitive(ses->auth_key.response); + ses->auth_key.response = kmemdup(bcc_ptr + tioffset, tilen, + GFP_KERNEL); + if (!ses->auth_key.response) { + cifs_dbg(VFS, "Challenge target info alloc failure\n"); + return -ENOMEM; + } + ses->auth_key.len = tilen; + } + + return 0; +} + +static int size_of_ntlmssp_blob(struct cifs_ses *ses, int base_size) +{ + int sz = base_size + ses->auth_key.len + - CIFS_SESS_KEY_SIZE + CIFS_CPHTXT_SIZE + 2; + + if (ses->domainName) + sz += sizeof(__le16) * strnlen(ses->domainName, CIFS_MAX_DOMAINNAME_LEN); + else + sz += sizeof(__le16); + + if (ses->user_name) + sz += sizeof(__le16) * strnlen(ses->user_name, CIFS_MAX_USERNAME_LEN); + else + sz += sizeof(__le16); + + if (ses->workstation_name[0]) + sz += sizeof(__le16) * strnlen(ses->workstation_name, + ntlmssp_workstation_name_size(ses)); + else + sz += sizeof(__le16); + + return sz; +} + +static inline void cifs_security_buffer_from_str(SECURITY_BUFFER *pbuf, + char *str_value, + int str_length, + unsigned char *pstart, + unsigned char **pcur, + const struct nls_table *nls_cp) +{ + unsigned char *tmp = pstart; + int len; + + if (!pbuf) + return; + + if (!pcur) + pcur = &tmp; + + if (!str_value) { + pbuf->BufferOffset = cpu_to_le32(*pcur - pstart); + pbuf->Length = 0; + pbuf->MaximumLength = 0; + *pcur += sizeof(__le16); + } else { + len = cifs_strtoUTF16((__le16 *)*pcur, + str_value, + str_length, + nls_cp); + len *= sizeof(__le16); + pbuf->BufferOffset = cpu_to_le32(*pcur - pstart); + pbuf->Length = cpu_to_le16(len); + pbuf->MaximumLength = cpu_to_le16(len); + *pcur += len; + } +} + +/* BB Move to ntlmssp.c eventually */ + +int build_ntlmssp_negotiate_blob(unsigned char **pbuffer, + u16 *buflen, + struct cifs_ses *ses, + struct TCP_Server_Info *server, + const struct nls_table *nls_cp) +{ + int rc = 0; + NEGOTIATE_MESSAGE *sec_blob; + __u32 flags; + unsigned char *tmp; + int len; + + len = size_of_ntlmssp_blob(ses, sizeof(NEGOTIATE_MESSAGE)); + *pbuffer = kmalloc(len, GFP_KERNEL); + if (!*pbuffer) { + rc = -ENOMEM; + cifs_dbg(VFS, "Error %d during NTLMSSP allocation\n", rc); + *buflen = 0; + goto setup_ntlm_neg_ret; + } + sec_blob = (NEGOTIATE_MESSAGE *)*pbuffer; + + memset(*pbuffer, 0, sizeof(NEGOTIATE_MESSAGE)); + memcpy(sec_blob->Signature, NTLMSSP_SIGNATURE, 8); + sec_blob->MessageType = NtLmNegotiate; + + /* BB is NTLMV2 session security format easier to use here? */ + flags = NTLMSSP_NEGOTIATE_56 | NTLMSSP_REQUEST_TARGET | + NTLMSSP_NEGOTIATE_128 | NTLMSSP_NEGOTIATE_UNICODE | + NTLMSSP_NEGOTIATE_NTLM | NTLMSSP_NEGOTIATE_EXTENDED_SEC | + NTLMSSP_NEGOTIATE_ALWAYS_SIGN | NTLMSSP_NEGOTIATE_SEAL | + NTLMSSP_NEGOTIATE_SIGN; + if (!server->session_estab || ses->ntlmssp->sesskey_per_smbsess) + flags |= NTLMSSP_NEGOTIATE_KEY_XCH; + + tmp = *pbuffer + sizeof(NEGOTIATE_MESSAGE); + ses->ntlmssp->client_flags = flags; + sec_blob->NegotiateFlags = cpu_to_le32(flags); + + /* these fields should be null in negotiate phase MS-NLMP 3.1.5.1.1 */ + cifs_security_buffer_from_str(&sec_blob->DomainName, + NULL, + CIFS_MAX_DOMAINNAME_LEN, + *pbuffer, &tmp, + nls_cp); + + cifs_security_buffer_from_str(&sec_blob->WorkstationName, + NULL, + CIFS_MAX_WORKSTATION_LEN, + *pbuffer, &tmp, + nls_cp); + + *buflen = tmp - *pbuffer; +setup_ntlm_neg_ret: + return rc; +} + +/* + * Build ntlmssp blob with additional fields, such as version, + * supported by modern servers. For safety limit to SMB3 or later + * See notes in MS-NLMP Section 2.2.2.1 e.g. + */ +int build_ntlmssp_smb3_negotiate_blob(unsigned char **pbuffer, + u16 *buflen, + struct cifs_ses *ses, + struct TCP_Server_Info *server, + const struct nls_table *nls_cp) +{ + int rc = 0; + struct negotiate_message *sec_blob; + __u32 flags; + unsigned char *tmp; + int len; + + len = size_of_ntlmssp_blob(ses, sizeof(struct negotiate_message)); + *pbuffer = kmalloc(len, GFP_KERNEL); + if (!*pbuffer) { + rc = -ENOMEM; + cifs_dbg(VFS, "Error %d during NTLMSSP allocation\n", rc); + *buflen = 0; + goto setup_ntlm_smb3_neg_ret; + } + sec_blob = (struct negotiate_message *)*pbuffer; + + memset(*pbuffer, 0, sizeof(struct negotiate_message)); + memcpy(sec_blob->Signature, NTLMSSP_SIGNATURE, 8); + sec_blob->MessageType = NtLmNegotiate; + + /* BB is NTLMV2 session security format easier to use here? */ + flags = NTLMSSP_NEGOTIATE_56 | NTLMSSP_REQUEST_TARGET | + NTLMSSP_NEGOTIATE_128 | NTLMSSP_NEGOTIATE_UNICODE | + NTLMSSP_NEGOTIATE_NTLM | NTLMSSP_NEGOTIATE_EXTENDED_SEC | + NTLMSSP_NEGOTIATE_ALWAYS_SIGN | NTLMSSP_NEGOTIATE_SEAL | + NTLMSSP_NEGOTIATE_SIGN | NTLMSSP_NEGOTIATE_VERSION; + if (!server->session_estab || ses->ntlmssp->sesskey_per_smbsess) + flags |= NTLMSSP_NEGOTIATE_KEY_XCH; + + sec_blob->Version.ProductMajorVersion = LINUX_VERSION_MAJOR; + sec_blob->Version.ProductMinorVersion = LINUX_VERSION_PATCHLEVEL; + sec_blob->Version.ProductBuild = cpu_to_le16(SMB3_PRODUCT_BUILD); + sec_blob->Version.NTLMRevisionCurrent = NTLMSSP_REVISION_W2K3; + + tmp = *pbuffer + sizeof(struct negotiate_message); + ses->ntlmssp->client_flags = flags; + sec_blob->NegotiateFlags = cpu_to_le32(flags); + + /* these fields should be null in negotiate phase MS-NLMP 3.1.5.1.1 */ + cifs_security_buffer_from_str(&sec_blob->DomainName, + NULL, + CIFS_MAX_DOMAINNAME_LEN, + *pbuffer, &tmp, + nls_cp); + + cifs_security_buffer_from_str(&sec_blob->WorkstationName, + NULL, + CIFS_MAX_WORKSTATION_LEN, + *pbuffer, &tmp, + nls_cp); + + *buflen = tmp - *pbuffer; +setup_ntlm_smb3_neg_ret: + return rc; +} + + +/* See MS-NLMP 2.2.1.3 */ +int build_ntlmssp_auth_blob(unsigned char **pbuffer, + u16 *buflen, + struct cifs_ses *ses, + struct TCP_Server_Info *server, + const struct nls_table *nls_cp) +{ + int rc; + AUTHENTICATE_MESSAGE *sec_blob; + __u32 flags; + unsigned char *tmp; + int len; + + rc = setup_ntlmv2_rsp(ses, nls_cp); + if (rc) { + cifs_dbg(VFS, "Error %d during NTLMSSP authentication\n", rc); + *buflen = 0; + goto setup_ntlmv2_ret; + } + + len = size_of_ntlmssp_blob(ses, sizeof(AUTHENTICATE_MESSAGE)); + *pbuffer = kmalloc(len, GFP_KERNEL); + if (!*pbuffer) { + rc = -ENOMEM; + cifs_dbg(VFS, "Error %d during NTLMSSP allocation\n", rc); + *buflen = 0; + goto setup_ntlmv2_ret; + } + sec_blob = (AUTHENTICATE_MESSAGE *)*pbuffer; + + memcpy(sec_blob->Signature, NTLMSSP_SIGNATURE, 8); + sec_blob->MessageType = NtLmAuthenticate; + + /* send version information in ntlmssp authenticate also */ + flags = ses->ntlmssp->server_flags | NTLMSSP_REQUEST_TARGET | + NTLMSSP_NEGOTIATE_TARGET_INFO | NTLMSSP_NEGOTIATE_VERSION | + NTLMSSP_NEGOTIATE_WORKSTATION_SUPPLIED; + + sec_blob->Version.ProductMajorVersion = LINUX_VERSION_MAJOR; + sec_blob->Version.ProductMinorVersion = LINUX_VERSION_PATCHLEVEL; + sec_blob->Version.ProductBuild = cpu_to_le16(SMB3_PRODUCT_BUILD); + sec_blob->Version.NTLMRevisionCurrent = NTLMSSP_REVISION_W2K3; + + tmp = *pbuffer + sizeof(AUTHENTICATE_MESSAGE); + sec_blob->NegotiateFlags = cpu_to_le32(flags); + + sec_blob->LmChallengeResponse.BufferOffset = + cpu_to_le32(sizeof(AUTHENTICATE_MESSAGE)); + sec_blob->LmChallengeResponse.Length = 0; + sec_blob->LmChallengeResponse.MaximumLength = 0; + + sec_blob->NtChallengeResponse.BufferOffset = + cpu_to_le32(tmp - *pbuffer); + if (ses->user_name != NULL) { + memcpy(tmp, ses->auth_key.response + CIFS_SESS_KEY_SIZE, + ses->auth_key.len - CIFS_SESS_KEY_SIZE); + tmp += ses->auth_key.len - CIFS_SESS_KEY_SIZE; + + sec_blob->NtChallengeResponse.Length = + cpu_to_le16(ses->auth_key.len - CIFS_SESS_KEY_SIZE); + sec_blob->NtChallengeResponse.MaximumLength = + cpu_to_le16(ses->auth_key.len - CIFS_SESS_KEY_SIZE); + } else { + /* + * don't send an NT Response for anonymous access + */ + sec_blob->NtChallengeResponse.Length = 0; + sec_blob->NtChallengeResponse.MaximumLength = 0; + } + + cifs_security_buffer_from_str(&sec_blob->DomainName, + ses->domainName, + CIFS_MAX_DOMAINNAME_LEN, + *pbuffer, &tmp, + nls_cp); + + cifs_security_buffer_from_str(&sec_blob->UserName, + ses->user_name, + CIFS_MAX_USERNAME_LEN, + *pbuffer, &tmp, + nls_cp); + + cifs_security_buffer_from_str(&sec_blob->WorkstationName, + ses->workstation_name, + ntlmssp_workstation_name_size(ses), + *pbuffer, &tmp, + nls_cp); + + if ((ses->ntlmssp->server_flags & NTLMSSP_NEGOTIATE_KEY_XCH) && + (!ses->server->session_estab || ses->ntlmssp->sesskey_per_smbsess) && + !calc_seckey(ses)) { + memcpy(tmp, ses->ntlmssp->ciphertext, CIFS_CPHTXT_SIZE); + sec_blob->SessionKey.BufferOffset = cpu_to_le32(tmp - *pbuffer); + sec_blob->SessionKey.Length = cpu_to_le16(CIFS_CPHTXT_SIZE); + sec_blob->SessionKey.MaximumLength = + cpu_to_le16(CIFS_CPHTXT_SIZE); + tmp += CIFS_CPHTXT_SIZE; + } else { + sec_blob->SessionKey.BufferOffset = cpu_to_le32(tmp - *pbuffer); + sec_blob->SessionKey.Length = 0; + sec_blob->SessionKey.MaximumLength = 0; + } + + *buflen = tmp - *pbuffer; +setup_ntlmv2_ret: + return rc; +} + +enum securityEnum +cifs_select_sectype(struct TCP_Server_Info *server, enum securityEnum requested) +{ + switch (server->negflavor) { + case CIFS_NEGFLAVOR_EXTENDED: + switch (requested) { + case Kerberos: + case RawNTLMSSP: + case IAKerb: + return requested; + case Unspecified: + if (server->sec_ntlmssp && + (global_secflags & CIFSSEC_MAY_NTLMSSP)) + return RawNTLMSSP; + if ((server->sec_kerberos || server->sec_mskerberos || server->sec_iakerb) && + (global_secflags & CIFSSEC_MAY_KRB5)) + return Kerberos; + fallthrough; + default: + return Unspecified; + } + case CIFS_NEGFLAVOR_UNENCAP: + switch (requested) { + case NTLMv2: + return requested; + case Unspecified: + if (global_secflags & CIFSSEC_MAY_NTLMV2) + return NTLMv2; + break; + default: + break; + } + fallthrough; + default: + return Unspecified; + } +} diff --git a/fs/smb/client/smb1debug.c b/fs/smb/client/smb1debug.c new file mode 100644 index 000000000..e2d013e75 --- /dev/null +++ b/fs/smb/client/smb1debug.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * + * Copyright (C) International Business Machines Corp., 2000,2005 + * + * Modified by Steve French (sfrench@us.ibm.com) + */ +#include "cifsproto.h" +#include "smb1proto.h" +#include "cifs_debug.h" + +void cifs_dump_detail(void *buf, size_t buf_len, struct TCP_Server_Info *server) +{ +#ifdef CONFIG_CIFS_DEBUG2 + struct smb_hdr *smb = buf; + + cifs_dbg(VFS, "Cmd: %d Err: 0x%x Flags: 0x%x Flgs2: 0x%x Mid: %d Pid: %d Wct: %d\n", + smb->Command, smb->Status.CifsError, smb->Flags, + smb->Flags2, smb->Mid, smb->Pid, smb->WordCount); + if (!server->ops->check_message(buf, buf_len, server->total_read, server)) { + cifs_dbg(VFS, "smb buf %p len %u\n", smb, + server->ops->calc_smb_size(smb)); + } +#endif /* CONFIG_CIFS_DEBUG2 */ +} diff --git a/fs/smb/client/smb1encrypt.c b/fs/smb/client/smb1encrypt.c new file mode 100644 index 000000000..bf10fdeee --- /dev/null +++ b/fs/smb/client/smb1encrypt.c @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * Encryption and hashing operations relating to NTLM, NTLMv2. See MS-NLMP + * for more detailed information + * + * Copyright (C) International Business Machines Corp., 2005,2013 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ + +#include +#include +#include +#include "cifsproto.h" +#include "smb1proto.h" +#include "cifs_debug.h" + +/* + * Calculate and return the CIFS signature based on the mac key and SMB PDU. + * The 16 byte signature must be allocated by the caller. Note we only use the + * 1st eight bytes and that the smb header signature field on input contains + * the sequence number before this function is called. Also, this function + * should be called with the server->srv_mutex held. + */ +static int cifs_calc_signature(struct smb_rqst *rqst, + struct TCP_Server_Info *server, char *signature) +{ + struct md5_ctx ctx; + + if (!rqst->rq_iov || !signature || !server) + return -EINVAL; + if (fips_enabled) { + cifs_dbg(VFS, + "MD5 signature support is disabled due to FIPS\n"); + return -EOPNOTSUPP; + } + + md5_init(&ctx); + md5_update(&ctx, server->session_key.response, server->session_key.len); + + return __cifs_calc_signature( + rqst, server, signature, + &(struct cifs_calc_sig_ctx){ .md5 = &ctx }); +} + +/* must be called with server->srv_mutex held */ +int cifs_sign_rqst(struct smb_rqst *rqst, struct TCP_Server_Info *server, + __u32 *pexpected_response_sequence_number) +{ + int rc = 0; + char smb_signature[20]; + struct smb_hdr *cifs_pdu = (struct smb_hdr *)rqst->rq_iov[0].iov_base; + + if ((cifs_pdu == NULL) || (server == NULL)) + return -EINVAL; + + spin_lock(&server->srv_lock); + if (!(cifs_pdu->Flags2 & SMBFLG2_SECURITY_SIGNATURE) || + server->tcpStatus == CifsNeedNegotiate) { + spin_unlock(&server->srv_lock); + return rc; + } + spin_unlock(&server->srv_lock); + + if (!server->session_estab) { + memcpy(cifs_pdu->Signature.SecuritySignature, "BSRSPYL", 8); + return rc; + } + + cifs_pdu->Signature.Sequence.SequenceNumber = + cpu_to_le32(server->sequence_number); + cifs_pdu->Signature.Sequence.Reserved = 0; + + *pexpected_response_sequence_number = ++server->sequence_number; + ++server->sequence_number; + + rc = cifs_calc_signature(rqst, server, smb_signature); + if (rc) + memset(cifs_pdu->Signature.SecuritySignature, 0, 8); + else + memcpy(cifs_pdu->Signature.SecuritySignature, smb_signature, 8); + + return rc; +} + +int cifs_verify_signature(struct smb_rqst *rqst, + struct TCP_Server_Info *server, + __u32 expected_sequence_number) +{ + unsigned int rc; + char server_response_sig[8]; + char what_we_think_sig_should_be[20]; + struct smb_hdr *cifs_pdu = (struct smb_hdr *)rqst->rq_iov[0].iov_base; + + if (cifs_pdu == NULL || server == NULL) + return -EINVAL; + + if (!server->session_estab) + return 0; + + if (cifs_pdu->Command == SMB_COM_LOCKING_ANDX) { + struct smb_com_lock_req *pSMB = + (struct smb_com_lock_req *)cifs_pdu; + if (pSMB->LockType & LOCKING_ANDX_OPLOCK_RELEASE) + return 0; + } + + /* BB what if signatures are supposed to be on for session but + server does not send one? BB */ + + /* Do not need to verify session setups with signature "BSRSPYL " */ + if (memcmp(cifs_pdu->Signature.SecuritySignature, "BSRSPYL ", 8) == 0) + cifs_dbg(FYI, "dummy signature received for smb command 0x%x\n", + cifs_pdu->Command); + + /* save off the original signature so we can modify the smb and check + its signature against what the server sent */ + memcpy(server_response_sig, cifs_pdu->Signature.SecuritySignature, 8); + + cifs_pdu->Signature.Sequence.SequenceNumber = + cpu_to_le32(expected_sequence_number); + cifs_pdu->Signature.Sequence.Reserved = 0; + + cifs_server_lock(server); + rc = cifs_calc_signature(rqst, server, what_we_think_sig_should_be); + cifs_server_unlock(server); + + if (rc) + return rc; + +/* cifs_dump_mem("what we think it should be: ", + what_we_think_sig_should_be, 16); */ + + if (crypto_memneq(server_response_sig, what_we_think_sig_should_be, 8)) + return -EACCES; + else + return 0; + +} diff --git a/fs/smb/client/smb1maperror.c b/fs/smb/client/smb1maperror.c new file mode 100644 index 000000000..395299f91 --- /dev/null +++ b/fs/smb/client/smb1maperror.c @@ -0,0 +1,282 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * + * Copyright (c) International Business Machines Corp., 2002,2008 + * Author(s): Steve French (sfrench@us.ibm.com) + * + * Error mapping routines from Samba libsmb/errormap.c + * Copyright (C) Andrew Tridgell 2001 + * Copyright (C) Luke Kenneth Casson Leighton 1997-2001. + */ + +#include + +#include + +#include "cifsproto.h" +#include "smb1proto.h" +#include "smberr.h" +#include "nterr.h" +#include "cifs_debug.h" + +static __always_inline int smb1_posix_error_cmp(const void *_key, const void *_pivot) +{ + __u16 key = *(__u16 *)_key; + const struct smb_to_posix_error *pivot = _pivot; + + if (key < pivot->smb_err) + return -1; + if (key > pivot->smb_err) + return 1; + return 0; +} + +static const struct smb_to_posix_error mapping_table_ERRDOS[] = { +/* + * Automatically generated by the `gen_smb1_mapping` script, + * sorted by DOS error code (ascending). + */ +#include "smb1_err_dos_map.c" +}; + +static const struct smb_to_posix_error mapping_table_ERRSRV[] = { +/* + * Automatically generated by the `gen_smb1_mapping` script, + * sorted by SRV error code (ascending). + */ +#include "smb1_err_srv_map.c" +}; + +/***************************************************************************** + *convert a NT status code to a dos class/code + *****************************************************************************/ + +static __always_inline int ntstatus_to_dos_cmp(const void *_key, const void *_pivot) +{ + __u32 key = *(__u32 *)_key; + const struct ntstatus_to_dos_err *pivot = _pivot; + + if (key < pivot->ntstatus) + return -1; + if (key > pivot->ntstatus) + return 1; + return 0; +} + +/* NT status -> dos error map */ +static const struct ntstatus_to_dos_err ntstatus_to_dos_map[] = { +/* + * Automatically generated by the `gen_smb1_mapping` script, + * sorted by NT status code (ascending). + */ +#include "smb1_mapping_table.c" +}; + +static const struct ntstatus_to_dos_err * +search_ntstatus_to_dos_map(__u32 ntstatus) +{ + return __inline_bsearch(&ntstatus, ntstatus_to_dos_map, + ARRAY_SIZE(ntstatus_to_dos_map), + sizeof(struct ntstatus_to_dos_err), + ntstatus_to_dos_cmp); +} + +static const struct smb_to_posix_error * +search_mapping_table_ERRDOS(__u16 smb_err) +{ + return __inline_bsearch(&smb_err, mapping_table_ERRDOS, + ARRAY_SIZE(mapping_table_ERRDOS), + sizeof(struct smb_to_posix_error), + smb1_posix_error_cmp); +} + +static const struct smb_to_posix_error * +search_mapping_table_ERRSRV(__u16 smb_err) +{ + return __inline_bsearch(&smb_err, mapping_table_ERRSRV, + ARRAY_SIZE(mapping_table_ERRSRV), + sizeof(struct smb_to_posix_error), + smb1_posix_error_cmp); +} + +int +map_smb_to_linux_error(char *buf, bool logErr) +{ + struct smb_hdr *smb = (struct smb_hdr *)buf; + int rc = -EIO; /* if transport error smb error may not be set */ + __u8 smberrclass; + __u16 smberrcode; + const struct smb_to_posix_error *err_map = NULL; + + /* BB if NT Status codes - map NT BB */ + + /* old style smb error codes */ + if (smb->Status.CifsError == 0) + return 0; + + if (smb->Flags2 & SMBFLG2_ERR_STATUS) { + /* translate the newer STATUS codes to old style SMB errors + * and then to POSIX errors */ + __u32 err = le32_to_cpu(smb->Status.CifsError); + const struct ntstatus_to_dos_err *map = search_ntstatus_to_dos_map(err); + + if (map) { + if ((logErr && err != NT_STATUS_MORE_PROCESSING_REQUIRED) || + (cifsFYI & CIFS_RC)) + pr_notice("Status code returned 0x%08x %s\n", + map->ntstatus, map->nt_errstr); + + smberrclass = map->dos_class; + smberrcode = map->dos_code; + } else { + smberrclass = ERRHRD; + smberrcode = ERRgeneral; + } + } else { + smberrclass = smb->Status.DosError.ErrorClass; + smberrcode = le16_to_cpu(smb->Status.DosError.Error); + } + + /* old style errors */ + + if (smberrclass == ERRDOS) { + /* DOS class smb error codes - map DOS */ + /* 1 byte field no need to byte reverse */ + err_map = search_mapping_table_ERRDOS(smberrcode); + } else if (smberrclass == ERRSRV) { + /* server class of error codes */ + err_map = search_mapping_table_ERRSRV(smberrcode); + } + if (err_map) + rc = err_map->posix_code; + /* else ERRHRD class errors or junk - return EIO */ + + /* special cases for NT status codes which cannot be translated to DOS codes */ + if (smb->Flags2 & SMBFLG2_ERR_STATUS) { + __u32 err = le32_to_cpu(smb->Status.CifsError); + if (err == (NT_STATUS_NOT_A_REPARSE_POINT)) + rc = -ENODATA; + else if (err == (NT_STATUS_PRIVILEGE_NOT_HELD)) + rc = -EPERM; + } + + cifs_dbg(FYI, "Mapping smb error code 0x%x to POSIX err %d\n", + le32_to_cpu(smb->Status.CifsError), rc); + + /* generic corrective action e.g. reconnect SMB session on + * ERRbaduid could be added */ + + if (rc == -EIO) + smb_EIO2(smb_eio_trace_smb1_received_error, + le32_to_cpu(smb->Status.CifsError), + le16_to_cpu(smb->Flags2)); + return rc; +} + +int +map_and_check_smb_error(struct TCP_Server_Info *server, + struct mid_q_entry *mid, bool logErr) +{ + int rc; + struct smb_hdr *smb = (struct smb_hdr *)mid->resp_buf; + + rc = map_smb_to_linux_error((char *)smb, logErr); + if (rc == -EACCES && !(smb->Flags2 & SMBFLG2_ERR_STATUS)) { + /* possible ERRBaduid */ + __u8 class = smb->Status.DosError.ErrorClass; + __u16 code = le16_to_cpu(smb->Status.DosError.Error); + + /* switch can be used to handle different errors */ + if (class == ERRSRV && code == ERRbaduid) { + cifs_dbg(FYI, "Server returned 0x%x, reconnecting session...\n", + code); + cifs_signal_cifsd_for_reconnect(server, false); + } + } + + return rc; +} + +#define DEFINE_CHECK_SORT_FUNC(__array, __field) \ +static int __init __array ## _is_sorted(void) \ +{ \ + unsigned int i; \ + \ + /* Check whether the array is sorted in ascending order */ \ + for (i = 1; i < ARRAY_SIZE(__array); i++) { \ + if (__array[i].__field >= \ + __array[i - 1].__field) \ + continue; \ + \ + pr_err(#__array " array order is incorrect\n"); \ + return -EINVAL; \ + } \ + \ + return 0; \ +} + +/* ntstatus_to_dos_map_is_sorted */ +DEFINE_CHECK_SORT_FUNC(ntstatus_to_dos_map, ntstatus); +/* mapping_table_ERRDOS_is_sorted */ +DEFINE_CHECK_SORT_FUNC(mapping_table_ERRDOS, smb_err); +/* mapping_table_ERRSRV_is_sorted */ +DEFINE_CHECK_SORT_FUNC(mapping_table_ERRSRV, smb_err); + +int __init smb1_init_maperror(void) +{ + int rc; + + rc = ntstatus_to_dos_map_is_sorted(); + if (rc) + return rc; + + rc = mapping_table_ERRDOS_is_sorted(); + if (rc) + return rc; + + return mapping_table_ERRSRV_is_sorted(); +} + +#if IS_ENABLED(CONFIG_SMB1_KUNIT_TESTS) +const struct ntstatus_to_dos_err * +search_ntstatus_to_dos_map_test(__u32 ntstatus) +{ + return search_ntstatus_to_dos_map(ntstatus); +} +EXPORT_SYMBOL_IF_KUNIT(search_ntstatus_to_dos_map_test); + +const struct ntstatus_to_dos_err * +ntstatus_to_dos_map_test = ntstatus_to_dos_map; +EXPORT_SYMBOL_IF_KUNIT(ntstatus_to_dos_map_test); + +unsigned int ntstatus_to_dos_num = ARRAY_SIZE(ntstatus_to_dos_map); +EXPORT_SYMBOL_IF_KUNIT(ntstatus_to_dos_num); + +const struct smb_to_posix_error * +search_mapping_table_ERRDOS_test(__u16 smb_err) +{ + return search_mapping_table_ERRDOS(smb_err); +} +EXPORT_SYMBOL_IF_KUNIT(search_mapping_table_ERRDOS_test); + +const struct smb_to_posix_error * +mapping_table_ERRDOS_test = mapping_table_ERRDOS; +EXPORT_SYMBOL_IF_KUNIT(mapping_table_ERRDOS_test); + +unsigned int mapping_table_ERRDOS_num = ARRAY_SIZE(mapping_table_ERRDOS); +EXPORT_SYMBOL_IF_KUNIT(mapping_table_ERRDOS_num); + +const struct smb_to_posix_error * +search_mapping_table_ERRSRV_test(__u16 smb_err) +{ + return search_mapping_table_ERRSRV(smb_err); +} +EXPORT_SYMBOL_IF_KUNIT(search_mapping_table_ERRSRV_test); + +const struct smb_to_posix_error * +mapping_table_ERRSRV_test = mapping_table_ERRSRV; +EXPORT_SYMBOL_IF_KUNIT(mapping_table_ERRSRV_test); + +unsigned int mapping_table_ERRSRV_num = ARRAY_SIZE(mapping_table_ERRSRV); +EXPORT_SYMBOL_IF_KUNIT(mapping_table_ERRSRV_num); +#endif diff --git a/fs/smb/client/smb1maperror_test.c b/fs/smb/client/smb1maperror_test.c new file mode 100644 index 000000000..903c46f71 --- /dev/null +++ b/fs/smb/client/smb1maperror_test.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * + * KUnit tests of SMB1 maperror + * + * Copyright (C) 2026 KylinSoft Co., Ltd. All rights reserved. + * Author(s): Youling Tang + * ChenXiaoSong + * + */ + +#include +#include "smb1proto.h" +#include "nterr.h" +#include "smberr.h" + +#define DEFINE_CHECK_SEARCH_FUNC(__struct_name, __field, \ + __array, __num) \ +static void check_search_ ## __array(struct kunit *test) \ +{ \ + unsigned int i; \ + const struct __struct_name *expect, *result; \ + \ + for (i = 0; i < __num; i++) { \ + expect = &__array ## _test[i]; \ + result = search_ ## __array ## _test(expect->__field); \ + KUNIT_ASSERT_NOT_NULL(test, result); \ + test_cmp_ ## __struct_name(test, expect, result); \ + } \ +} + +static void +test_cmp_ntstatus_to_dos_err(struct kunit *test, + const struct ntstatus_to_dos_err *expect, + const struct ntstatus_to_dos_err *result) +{ + KUNIT_EXPECT_EQ(test, expect->dos_class, result->dos_class); + KUNIT_EXPECT_EQ(test, expect->dos_code, result->dos_code); + KUNIT_EXPECT_EQ(test, expect->ntstatus, result->ntstatus); + KUNIT_EXPECT_STREQ(test, expect->nt_errstr, result->nt_errstr); +} + +static void +test_cmp_smb_to_posix_error(struct kunit *test, + const struct smb_to_posix_error *expect, + const struct smb_to_posix_error *result) +{ + KUNIT_EXPECT_EQ(test, expect->smb_err, result->smb_err); + KUNIT_EXPECT_EQ(test, expect->posix_code, result->posix_code); +} + +/* check_search_ntstatus_to_dos_map */ +DEFINE_CHECK_SEARCH_FUNC(ntstatus_to_dos_err, ntstatus, ntstatus_to_dos_map, + ntstatus_to_dos_num); +/* check_search_mapping_table_ERRDOS */ +DEFINE_CHECK_SEARCH_FUNC(smb_to_posix_error, smb_err, mapping_table_ERRDOS, + mapping_table_ERRDOS_num); +/* check_search_mapping_table_ERRSRV */ +DEFINE_CHECK_SEARCH_FUNC(smb_to_posix_error, smb_err, mapping_table_ERRSRV, + mapping_table_ERRSRV_num); + +static struct kunit_case maperror_test_cases[] = { + KUNIT_CASE(check_search_ntstatus_to_dos_map), + KUNIT_CASE(check_search_mapping_table_ERRDOS), + KUNIT_CASE(check_search_mapping_table_ERRSRV), + {} +}; + +static struct kunit_suite maperror_suite = { + .name = "smb1_maperror", + .test_cases = maperror_test_cases, +}; + +kunit_test_suite(maperror_suite); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("KUnit tests of SMB1 maperror"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/fs/smb/client/smb1misc.c b/fs/smb/client/smb1misc.c new file mode 100644 index 000000000..ba5602301 --- /dev/null +++ b/fs/smb/client/smb1misc.c @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * Copyright (C) International Business Machines Corp., 2002,2008 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ + +#include "smb1proto.h" +#include "smberr.h" +#include "nterr.h" +#include "cifs_debug.h" + +/* NB: MID can not be set if treeCon not passed in, in that + case it is responsibility of caller to set the mid */ +unsigned int +header_assemble(struct smb_hdr *buffer, char smb_command, + const struct cifs_tcon *treeCon, int word_count + /* length of fixed section (word count) in two byte units */) +{ + unsigned int in_len; + char *temp = (char *) buffer; + + memset(temp, 0, 256); /* bigger than MAX_CIFS_HDR_SIZE */ + + in_len = (2 * word_count) + sizeof(struct smb_hdr) + + 2 /* for bcc field itself */; + + buffer->Protocol[0] = 0xFF; + buffer->Protocol[1] = 'S'; + buffer->Protocol[2] = 'M'; + buffer->Protocol[3] = 'B'; + buffer->Command = smb_command; + buffer->Flags = 0x00; /* case sensitive */ + buffer->Flags2 = SMBFLG2_KNOWS_LONG_NAMES; + buffer->Pid = cpu_to_le16((__u16)current->tgid); + buffer->PidHigh = cpu_to_le16((__u16)(current->tgid >> 16)); + if (treeCon) { + buffer->Tid = treeCon->tid; + if (treeCon->ses) { + if (treeCon->ses->capabilities & CAP_UNICODE) + buffer->Flags2 |= SMBFLG2_UNICODE; + if (treeCon->ses->capabilities & CAP_STATUS32) + buffer->Flags2 |= SMBFLG2_ERR_STATUS; + + /* Uid is not converted */ + buffer->Uid = treeCon->ses->Suid; + if (treeCon->ses->server) + buffer->Mid = get_next_mid(treeCon->ses->server); + } + if (treeCon->Flags & SMB_SHARE_IS_IN_DFS) + buffer->Flags2 |= SMBFLG2_DFS; + if (treeCon->nocase) + buffer->Flags |= SMBFLG_CASELESS; + if ((treeCon->ses) && (treeCon->ses->server)) + if (treeCon->ses->server->sign) + buffer->Flags2 |= SMBFLG2_SECURITY_SIGNATURE; + } + +/* endian conversion of flags is now done just before sending */ + buffer->WordCount = (char) word_count; + return in_len; +} + +bool +is_valid_oplock_break(char *buffer, struct TCP_Server_Info *srv) +{ + struct smb_hdr *buf = (struct smb_hdr *)buffer; + struct smb_com_lock_req *pSMB = (struct smb_com_lock_req *)buf; + struct TCP_Server_Info *pserver; + struct cifs_ses *ses; + struct cifs_tcon *tcon; + struct cifsInodeInfo *pCifsInode; + struct cifsFileInfo *netfile; + + cifs_dbg(FYI, "Checking for oplock break or dnotify response\n"); + if ((pSMB->hdr.Command == SMB_COM_NT_TRANSACT) && + (pSMB->hdr.Flags & SMBFLG_RESPONSE)) { + struct smb_com_transaction_change_notify_rsp *pSMBr = + (struct smb_com_transaction_change_notify_rsp *)buf; + struct file_notify_information *pnotify; + __u32 data_offset = 0; + size_t len = srv->total_read - srv->pdu_size; + + if (get_bcc(buf) > sizeof(struct file_notify_information)) { + data_offset = le32_to_cpu(pSMBr->DataOffset); + + if (data_offset > + len - sizeof(struct file_notify_information)) { + cifs_dbg(FYI, "Invalid data_offset %u\n", + data_offset); + return true; + } + pnotify = (struct file_notify_information *) + ((char *)&pSMBr->hdr.Protocol + data_offset); + cifs_dbg(FYI, "dnotify on %s Action: 0x%x\n", + pnotify->FileName, pnotify->Action); + /* cifs_dump_mem("Rcvd notify Data: ",buf, + sizeof(struct smb_hdr)+60); */ + return true; + } + if (pSMBr->hdr.Status.CifsError) { + cifs_dbg(FYI, "notify err 0x%x\n", + pSMBr->hdr.Status.CifsError); + return true; + } + return false; + } + if (pSMB->hdr.Command != SMB_COM_LOCKING_ANDX) + return false; + if (pSMB->hdr.Flags & SMBFLG_RESPONSE) { + /* no sense logging error on invalid handle on oplock + break - harmless race between close request and oplock + break response is expected from time to time writing out + large dirty files cached on the client */ + if ((NT_STATUS_INVALID_HANDLE) == + le32_to_cpu(pSMB->hdr.Status.CifsError)) { + cifs_dbg(FYI, "Invalid handle on oplock break\n"); + return true; + } else if (ERRbadfid == + le16_to_cpu(pSMB->hdr.Status.DosError.Error)) { + return true; + } else { + return false; /* on valid oplock brk we get "request" */ + } + } + if (pSMB->hdr.WordCount != 8) + return false; + + cifs_dbg(FYI, "oplock type 0x%x level 0x%x\n", + pSMB->LockType, pSMB->OplockLevel); + if (!(pSMB->LockType & LOCKING_ANDX_OPLOCK_RELEASE)) + return false; + + /* If server is a channel, select the primary channel */ + pserver = SERVER_IS_CHAN(srv) ? srv->primary_server : srv; + + /* look up tcon based on tid & uid */ + spin_lock(&cifs_tcp_ses_lock); + list_for_each_entry(ses, &pserver->smb_ses_list, smb_ses_list) { + if (cifs_ses_exiting(ses)) + continue; + list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { + if (tcon->tid != buf->Tid) + continue; + + cifs_stats_inc(&tcon->stats.cifs_stats.num_oplock_brks); + spin_lock(&tcon->open_file_lock); + list_for_each_entry(netfile, &tcon->openFileList, tlist) { + if (pSMB->Fid != netfile->fid.netfid) + continue; + + cifs_dbg(FYI, "file id match, oplock break\n"); + pCifsInode = CIFS_I(d_inode(netfile->dentry)); + + set_bit(CIFS_INODE_PENDING_OPLOCK_BREAK, + &pCifsInode->flags); + + netfile->oplock_epoch = 0; + netfile->oplock_level = pSMB->OplockLevel; + netfile->oplock_break_cancelled = false; + cifs_queue_oplock_break(netfile); + + spin_unlock(&tcon->open_file_lock); + spin_unlock(&cifs_tcp_ses_lock); + return true; + } + spin_unlock(&tcon->open_file_lock); + spin_unlock(&cifs_tcp_ses_lock); + cifs_dbg(FYI, "No matching file for oplock break\n"); + return true; + } + } + spin_unlock(&cifs_tcp_ses_lock); + cifs_dbg(FYI, "Can not process oplock break for non-existent connection\n"); + return true; +} + +/* + * calculate the size of the SMB message based on the fixed header + * portion, the number of word parameters and the data portion of the message + */ +unsigned int +smbCalcSize(void *buf) +{ + struct smb_hdr *ptr = buf; + return (sizeof(struct smb_hdr) + (2 * ptr->WordCount) + + 2 /* size of the bcc field */ + get_bcc(ptr)); +} diff --git a/fs/smb/client/smb1ops.c b/fs/smb/client/smb1ops.c new file mode 100644 index 000000000..dc5a8c1da --- /dev/null +++ b/fs/smb/client/smb1ops.c @@ -0,0 +1,1498 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * SMB1 (CIFS) version specific operations + * + * Copyright (c) 2012, Jeff Layton + */ + +#include +#include +#include +#include +#include "cifsglob.h" +#include "cifsproto.h" +#include "cifs_debug.h" +#include "cifs_unicode.h" +#include "fs_context.h" +#include "nterr.h" +#include "smberr.h" +#include "reparse.h" + +void reset_cifs_unix_caps(unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, struct smb3_fs_context *ctx) +{ + /* + * If we are reconnecting then should we check to see if + * any requested capabilities changed locally e.g. via + * remount but we can not do much about it here + * if they have (even if we could detect it by the following) + * Perhaps we could add a backpointer to array of sb from tcon + * or if we change to make all sb to same share the same + * sb as NFS - then we only have one backpointer to sb. + * What if we wanted to mount the server share twice once with + * and once without posixacls or posix paths? + */ + __u64 saved_cap = le64_to_cpu(tcon->fsUnixInfo.Capability); + + if (ctx && ctx->no_linux_ext) { + tcon->fsUnixInfo.Capability = 0; + tcon->unix_ext = 0; /* Unix Extensions disabled */ + cifs_dbg(FYI, "Linux protocol extensions disabled\n"); + return; + } else if (ctx) + tcon->unix_ext = 1; /* Unix Extensions supported */ + + if (!tcon->unix_ext) { + cifs_dbg(FYI, "Unix extensions disabled so not set on reconnect\n"); + return; + } + + if (!CIFSSMBQFSUnixInfo(xid, tcon)) { + __u64 cap = le64_to_cpu(tcon->fsUnixInfo.Capability); + + cifs_dbg(FYI, "unix caps which server supports %lld\n", cap); + /* + * check for reconnect case in which we do not + * want to change the mount behavior if we can avoid it + */ + if (ctx == NULL) { + /* + * turn off POSIX ACL and PATHNAMES if not set + * originally at mount time + */ + if ((saved_cap & CIFS_UNIX_POSIX_ACL_CAP) == 0) + cap &= ~CIFS_UNIX_POSIX_ACL_CAP; + if ((saved_cap & CIFS_UNIX_POSIX_PATHNAMES_CAP) == 0) { + if (cap & CIFS_UNIX_POSIX_PATHNAMES_CAP) + cifs_dbg(VFS, "POSIXPATH support change\n"); + cap &= ~CIFS_UNIX_POSIX_PATHNAMES_CAP; + } else if ((cap & CIFS_UNIX_POSIX_PATHNAMES_CAP) == 0) { + cifs_dbg(VFS, "possible reconnect error\n"); + cifs_dbg(VFS, "server disabled POSIX path support\n"); + } + } + + if (cap & CIFS_UNIX_TRANSPORT_ENCRYPTION_MANDATORY_CAP) + cifs_dbg(VFS, "per-share encryption not supported yet\n"); + + cap &= CIFS_UNIX_CAP_MASK; + if (ctx && ctx->no_psx_acl) + cap &= ~CIFS_UNIX_POSIX_ACL_CAP; + else if (CIFS_UNIX_POSIX_ACL_CAP & cap) { + cifs_dbg(FYI, "negotiated posix acl support\n"); + if (cifs_sb) { + atomic_or(CIFS_MOUNT_POSIXACL, + &cifs_sb->mnt_cifs_flags); + } + } + + if (ctx && ctx->posix_paths == 0) + cap &= ~CIFS_UNIX_POSIX_PATHNAMES_CAP; + else if (cap & CIFS_UNIX_POSIX_PATHNAMES_CAP) { + cifs_dbg(FYI, "negotiate posix pathnames\n"); + if (cifs_sb) { + atomic_or(CIFS_MOUNT_POSIX_PATHS, + &cifs_sb->mnt_cifs_flags); + } + } + + cifs_dbg(FYI, "Negotiate caps 0x%x\n", (int)cap); +#ifdef CONFIG_CIFS_DEBUG2 + if (cap & CIFS_UNIX_FCNTL_CAP) + cifs_dbg(FYI, "FCNTL cap\n"); + if (cap & CIFS_UNIX_EXTATTR_CAP) + cifs_dbg(FYI, "EXTATTR cap\n"); + if (cap & CIFS_UNIX_POSIX_PATHNAMES_CAP) + cifs_dbg(FYI, "POSIX path cap\n"); + if (cap & CIFS_UNIX_XATTR_CAP) + cifs_dbg(FYI, "XATTR cap\n"); + if (cap & CIFS_UNIX_POSIX_ACL_CAP) + cifs_dbg(FYI, "POSIX ACL cap\n"); + if (cap & CIFS_UNIX_LARGE_READ_CAP) + cifs_dbg(FYI, "very large read cap\n"); + if (cap & CIFS_UNIX_LARGE_WRITE_CAP) + cifs_dbg(FYI, "very large write cap\n"); + if (cap & CIFS_UNIX_TRANSPORT_ENCRYPTION_CAP) + cifs_dbg(FYI, "transport encryption cap\n"); + if (cap & CIFS_UNIX_TRANSPORT_ENCRYPTION_MANDATORY_CAP) + cifs_dbg(FYI, "mandatory transport encryption cap\n"); +#endif /* CIFS_DEBUG2 */ + if (CIFSSMBSetFSUnixInfo(xid, tcon, cap)) { + if (ctx == NULL) + cifs_dbg(FYI, "resetting capabilities failed\n"); + else + cifs_dbg(VFS, "Negotiating Unix capabilities with the server failed. Consider mounting with the Unix Extensions disabled if problems are found by specifying the nounix mount option.\n"); + + } + } +} + +/* + * An NT cancel request header looks just like the original request except: + * + * The Command is SMB_COM_NT_CANCEL + * The WordCount is zeroed out + * The ByteCount is zeroed out + * + * This function mangles an existing request buffer into a + * SMB_COM_NT_CANCEL request and then sends it. + */ +static int +send_nt_cancel(struct cifs_ses *ses, struct TCP_Server_Info *server, + struct smb_rqst *rqst, struct mid_q_entry *mid, + unsigned int xid) +{ + struct smb_hdr *in_buf = (struct smb_hdr *)rqst->rq_iov[0].iov_base; + struct kvec iov[1]; + struct smb_rqst crqst = { .rq_iov = iov, .rq_nvec = 1 }; + int rc = 0; + + /* +2 for BCC field */ + in_buf->Command = SMB_COM_NT_CANCEL; + in_buf->WordCount = 0; + put_bcc(0, in_buf); + + iov[0].iov_base = in_buf; + iov[0].iov_len = sizeof(struct smb_hdr) + 2; + + cifs_server_lock(server); + rc = cifs_sign_rqst(&crqst, server, &mid->sequence_number); + if (rc) { + cifs_server_unlock(server); + return rc; + } + + /* + * The response to this call was already factored into the sequence + * number when the call went out, so we must adjust it back downward + * after signing here. + */ + --server->sequence_number; + rc = __smb_send_rqst(server, 1, &crqst); + if (rc < 0) + server->sequence_number--; + + cifs_server_unlock(server); + + cifs_dbg(FYI, "issued NT_CANCEL for mid %u, rc = %d\n", + get_mid(in_buf), rc); + + return rc; +} + +/* + * Send a LOCKINGX_CANCEL_LOCK to cause the Windows blocking lock to + * return. + */ +static int +send_lock_cancel(struct cifs_ses *ses, struct TCP_Server_Info *server, + struct smb_rqst *rqst, struct mid_q_entry *mid, + unsigned int xid) +{ + struct smb_hdr *in_buf = (struct smb_hdr *)rqst->rq_iov[0].iov_base; + unsigned int in_len = rqst->rq_iov[0].iov_len; + LOCK_REQ *pSMB = (LOCK_REQ *)in_buf; + int rc; + + /* We just modify the current in_buf to change + * the type of lock from LOCKING_ANDX_SHARED_LOCK + * or LOCKING_ANDX_EXCLUSIVE_LOCK to + * LOCKING_ANDX_CANCEL_LOCK. + */ + pSMB->LockType = LOCKING_ANDX_CANCEL_LOCK|LOCKING_ANDX_LARGE_FILES; + pSMB->Timeout = 0; + pSMB->hdr.Mid = get_next_mid(ses->server); + + rc = SendReceive(xid, ses, in_buf, in_len, NULL, NULL, 0); + if (rc == -ENOLCK) + rc = 0; /* If we get back -ENOLCK, it probably means we managed + * to cancel the lock command before it took effect. + */ + return rc; +} + +static int cifs_send_cancel(struct cifs_ses *ses, struct TCP_Server_Info *server, + struct smb_rqst *rqst, struct mid_q_entry *mid, + unsigned int xid) +{ + if (mid->sr_flags & CIFS_WINDOWS_LOCK) + return send_lock_cancel(ses, server, rqst, mid, xid); + return send_nt_cancel(ses, server, rqst, mid, xid); +} + +static bool +cifs_compare_fids(struct cifsFileInfo *ob1, struct cifsFileInfo *ob2) +{ + return ob1->fid.netfid == ob2->fid.netfid; +} + +static unsigned int +cifs_read_data_offset(char *buf) +{ + READ_RSP *rsp = (READ_RSP *)buf; + return le16_to_cpu(rsp->DataOffset); +} + +static unsigned int +cifs_read_data_length(char *buf, bool in_remaining) +{ + READ_RSP *rsp = (READ_RSP *)buf; + /* It's a bug reading remaining data for SMB1 packets */ + WARN_ON(in_remaining); + return (le16_to_cpu(rsp->DataLengthHigh) << 16) + + le16_to_cpu(rsp->DataLength); +} + +static struct mid_q_entry * +cifs_find_mid(struct TCP_Server_Info *server, char *buffer) +{ + struct smb_hdr *buf = (struct smb_hdr *)buffer; + struct mid_q_entry *mid; + + spin_lock(&server->mid_queue_lock); + list_for_each_entry(mid, &server->pending_mid_q, qhead) { + if (compare_mid(mid->mid, buf) && + mid->mid_state == MID_REQUEST_SUBMITTED && + le16_to_cpu(mid->command) == buf->Command) { + smb_get_mid(mid); + spin_unlock(&server->mid_queue_lock); + return mid; + } + } + spin_unlock(&server->mid_queue_lock); + return NULL; +} + +static void +cifs_add_credits(struct TCP_Server_Info *server, + struct cifs_credits *credits, const int optype) +{ + spin_lock(&server->req_lock); + server->credits += credits->value; + server->in_flight--; + spin_unlock(&server->req_lock); + wake_up(&server->request_q); +} + +static void +cifs_set_credits(struct TCP_Server_Info *server, const int val) +{ + spin_lock(&server->req_lock); + server->credits = val; + server->oplocks = val > 1 ? enable_oplocks : false; + spin_unlock(&server->req_lock); +} + +static int * +cifs_get_credits_field(struct TCP_Server_Info *server, const int optype) +{ + return &server->credits; +} + +static unsigned int +cifs_get_credits(struct mid_q_entry *mid) +{ + return 1; +} + +/* + * Find a free multiplex id (SMB mid). Otherwise there could be + * mid collisions which might cause problems, demultiplexing the + * wrong response to this request. Multiplex ids could collide if + * one of a series requests takes much longer than the others, or + * if a very large number of long lived requests (byte range + * locks or FindNotify requests) are pending. No more than + * 64K-1 requests can be outstanding at one time. If no + * mids are available, return zero. A future optimization + * could make the combination of mids and uid the key we use + * to demultiplex on (rather than mid alone). + * In addition to the above check, the cifs demultiplex + * code already used the command code as a secondary + * check of the frame and if signing is negotiated the + * response would be discarded if the mid were the same + * but the signature was wrong. Since the mid is not put in the + * pending queue until later (when it is about to be dispatched) + * we do have to limit the number of outstanding requests + * to somewhat less than 64K-1 although it is hard to imagine + * so many threads being in the vfs at one time. + */ +static __u64 +cifs_get_next_mid(struct TCP_Server_Info *server) +{ + __u64 mid = 0; + __u16 last_mid, cur_mid; + bool collision, reconnect = false; + + spin_lock(&server->mid_counter_lock); + /* mid is 16 bit only for CIFS/SMB */ + cur_mid = (__u16)((server->current_mid) & 0xffff); + /* we do not want to loop forever */ + last_mid = cur_mid; + cur_mid++; + /* avoid 0xFFFF MID */ + if (cur_mid == 0xffff) + cur_mid++; + + /* + * This nested loop looks more expensive than it is. + * In practice the list of pending requests is short, + * fewer than 50, and the mids are likely to be unique + * on the first pass through the loop unless some request + * takes longer than the 64 thousand requests before it + * (and it would also have to have been a request that + * did not time out). + */ + while (cur_mid != last_mid) { + struct mid_q_entry *mid_entry; + unsigned int num_mids; + + collision = false; + if (cur_mid == 0) + cur_mid++; + + num_mids = 0; + spin_lock(&server->mid_queue_lock); + list_for_each_entry(mid_entry, &server->pending_mid_q, qhead) { + ++num_mids; + if (mid_entry->mid == cur_mid && + mid_entry->mid_state == MID_REQUEST_SUBMITTED) { + /* This mid is in use, try a different one */ + collision = true; + break; + } + } + spin_unlock(&server->mid_queue_lock); + + /* + * if we have more than 32k mids in the list, then something + * is very wrong. Possibly a local user is trying to DoS the + * box by issuing long-running calls and SIGKILL'ing them. If + * we get to 2^16 mids then we're in big trouble as this + * function could loop forever. + * + * Go ahead and assign out the mid in this situation, but force + * an eventual reconnect to clean out the pending_mid_q. + */ + if (num_mids > 32768) + reconnect = true; + + if (!collision) { + mid = (__u64)cur_mid; + server->current_mid = mid; + break; + } + cur_mid++; + } + spin_unlock(&server->mid_counter_lock); + + if (reconnect) { + cifs_signal_cifsd_for_reconnect(server, false); + } + + return mid; +} + +static void +cifs_downgrade_oplock(struct TCP_Server_Info *server, + struct cifsInodeInfo *cinode, __u32 oplock, + __u16 epoch, bool *purge_cache) +{ + lockdep_assert_held(&cinode->open_file_lock); + cifs_set_oplock_level(cinode, oplock); +} + +static bool +cifs_need_neg(struct TCP_Server_Info *server) +{ + return server->maxBuf == 0; +} + +static int +cifs_negotiate(const unsigned int xid, + struct cifs_ses *ses, + struct TCP_Server_Info *server) +{ + int rc; + rc = CIFSSMBNegotiate(xid, ses, server); + return rc; +} + +static unsigned int +smb1_negotiate_wsize(struct cifs_tcon *tcon, struct smb3_fs_context *ctx) +{ + __u64 unix_cap = le64_to_cpu(tcon->fsUnixInfo.Capability); + struct TCP_Server_Info *server = tcon->ses->server; + unsigned int wsize; + + /* start with specified wsize, or default */ + if (ctx->got_wsize) + wsize = ctx->vol_wsize; + else if (tcon->unix_ext && (unix_cap & CIFS_UNIX_LARGE_WRITE_CAP)) + wsize = CIFS_DEFAULT_IOSIZE; + else + wsize = CIFS_DEFAULT_NON_POSIX_WSIZE; + + /* can server support 24-bit write sizes? (via UNIX extensions) */ + if (!tcon->unix_ext || !(unix_cap & CIFS_UNIX_LARGE_WRITE_CAP)) + wsize = min_t(unsigned int, wsize, CIFS_MAX_RFC1002_WSIZE); + + /* + * no CAP_LARGE_WRITE_X or is signing enabled without CAP_UNIX set? + * Limit it to max buffer offered by the server, minus the size of the + * WRITEX header, not including the 4 byte RFC1001 length. + */ + if (!(server->capabilities & CAP_LARGE_WRITE_X) || + (!(server->capabilities & CAP_UNIX) && server->sign)) + wsize = min_t(unsigned int, wsize, + server->maxBuf - sizeof(WRITE_REQ)); + + /* hard limit of CIFS_MAX_WSIZE */ + wsize = min_t(unsigned int, wsize, CIFS_MAX_WSIZE); + + return wsize; +} + +static unsigned int +smb1_negotiate_rsize(struct cifs_tcon *tcon, struct smb3_fs_context *ctx) +{ + __u64 unix_cap = le64_to_cpu(tcon->fsUnixInfo.Capability); + struct TCP_Server_Info *server = tcon->ses->server; + unsigned int rsize, defsize; + + /* + * Set default value... + * + * HACK alert! Ancient servers have very small buffers. Even though + * MS-CIFS indicates that servers are only limited by the client's + * bufsize for reads, testing against win98se shows that it throws + * INVALID_PARAMETER errors if you try to request too large a read. + * OS/2 just sends back short reads. + * + * If the server doesn't advertise CAP_LARGE_READ_X, then assume that + * it can't handle a read request larger than its MaxBufferSize either. + */ + if (tcon->unix_ext && (unix_cap & CIFS_UNIX_LARGE_READ_CAP)) + defsize = CIFS_DEFAULT_IOSIZE; + else if (server->capabilities & CAP_LARGE_READ_X) + defsize = CIFS_DEFAULT_NON_POSIX_RSIZE; + else + defsize = server->maxBuf - sizeof(READ_RSP); + + rsize = ctx->got_rsize ? ctx->vol_rsize : defsize; + + /* + * no CAP_LARGE_READ_X? Then MS-CIFS states that we must limit this to + * the client's MaxBufferSize. + */ + if (!(server->capabilities & CAP_LARGE_READ_X)) + rsize = min_t(unsigned int, CIFSMaxBufSize, rsize); + + /* hard limit of CIFS_MAX_RSIZE */ + rsize = min_t(unsigned int, rsize, CIFS_MAX_RSIZE); + + return rsize; +} + +static void +cifs_qfs_tcon(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb) +{ + CIFSSMBQFSDeviceInfo(xid, tcon); + CIFSSMBQFSAttributeInfo(xid, tcon); +} + +static int +cifs_is_path_accessible(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, const char *full_path) +{ + int rc = -EOPNOTSUPP; + FILE_ALL_INFO file_info; + + if (tcon->ses->capabilities & CAP_NT_SMBS) + rc = CIFSSMBQPathInfo(xid, tcon, full_path, &file_info, + 0 /* not legacy */, cifs_sb->local_nls, + cifs_remap(cifs_sb)); + + /* + * Non-UNICODE variant of fallback functions below expands wildcards, + * so they cannot be used for querying paths with wildcard characters. + * Therefore for such paths returns -ENOENT as they cannot exist. + */ + if ((rc == -EOPNOTSUPP || rc == -EINVAL) && + !(tcon->ses->capabilities & CAP_UNICODE) && + strpbrk(full_path, "*?\"><")) + rc = -ENOENT; + + if (rc == -EOPNOTSUPP || rc == -EINVAL) + rc = SMBQueryInformation(xid, tcon, full_path, &file_info, + cifs_sb->local_nls, cifs_remap(cifs_sb)); + return rc; +} + +static int cifs_query_path_info(const unsigned int xid, + struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, + const char *full_path, + struct cifs_open_info_data *data) +{ + int rc = -EOPNOTSUPP; + FILE_ALL_INFO fi = {}; + struct cifs_search_info search_info = {}; + bool non_unicode_wildcard = false; + + data->reparse_point = false; + data->adjust_tz = false; + + /* + * First try CIFSSMBQPathInfo() function which returns more info + * (NumberOfLinks) than CIFSFindFirst() fallback function. + * Some servers like Win9x do not support SMB_QUERY_FILE_ALL_INFO over + * TRANS2_QUERY_PATH_INFORMATION, but supports it with filehandle over + * TRANS2_QUERY_FILE_INFORMATION (function CIFSSMBQFileInfo(). But SMB + * Open command on non-NT servers works only for files, does not work + * for directories. And moreover Win9x SMB server returns bogus data in + * SMB_QUERY_FILE_ALL_INFO Attributes field. So for non-NT servers, + * do not even use CIFSSMBQPathInfo() or CIFSSMBQFileInfo() function. + */ + if (tcon->ses->capabilities & CAP_NT_SMBS) + rc = CIFSSMBQPathInfo(xid, tcon, full_path, &fi, 0 /* not legacy */, + cifs_sb->local_nls, cifs_remap(cifs_sb)); + + /* + * Non-UNICODE variant of fallback functions below expands wildcards, + * so they cannot be used for querying paths with wildcard characters. + */ + if (rc && !(tcon->ses->capabilities & CAP_UNICODE) && strpbrk(full_path, "*?\"><")) + non_unicode_wildcard = true; + + /* + * Then fallback to CIFSFindFirst() which works also with non-NT servers + * but does not does not provide NumberOfLinks. + */ + if ((rc == -EOPNOTSUPP || rc == -EINVAL) && + !non_unicode_wildcard) { + if (!(tcon->ses->capabilities & tcon->ses->server->vals->cap_nt_find)) + search_info.info_level = SMB_FIND_FILE_INFO_STANDARD; + else + search_info.info_level = SMB_FIND_FILE_FULL_DIRECTORY_INFO; + rc = CIFSFindFirst(xid, tcon, full_path, cifs_sb, NULL, + CIFS_SEARCH_CLOSE_ALWAYS | CIFS_SEARCH_CLOSE_AT_END, + &search_info, false); + if (rc == 0) { + if (!(tcon->ses->capabilities & tcon->ses->server->vals->cap_nt_find)) { + FIND_FILE_STANDARD_INFO *di; + int offset = tcon->ses->server->timeAdj; + + di = (FIND_FILE_STANDARD_INFO *)search_info.srch_entries_start; + fi.CreationTime = cpu_to_le64(cifs_UnixTimeToNT(cnvrtDosUnixTm( + di->CreationDate, di->CreationTime, offset))); + fi.LastAccessTime = cpu_to_le64(cifs_UnixTimeToNT(cnvrtDosUnixTm( + di->LastAccessDate, di->LastAccessTime, offset))); + fi.LastWriteTime = cpu_to_le64(cifs_UnixTimeToNT(cnvrtDosUnixTm( + di->LastWriteDate, di->LastWriteTime, offset))); + fi.ChangeTime = fi.LastWriteTime; + fi.Attributes = cpu_to_le32(le16_to_cpu(di->Attributes)); + fi.AllocationSize = cpu_to_le64(le32_to_cpu(di->AllocationSize)); + fi.EndOfFile = cpu_to_le64(le32_to_cpu(di->DataSize)); + } else { + FILE_FULL_DIRECTORY_INFO *di; + + di = (FILE_FULL_DIRECTORY_INFO *)search_info.srch_entries_start; + fi.CreationTime = di->CreationTime; + fi.LastAccessTime = di->LastAccessTime; + fi.LastWriteTime = di->LastWriteTime; + fi.ChangeTime = di->ChangeTime; + fi.Attributes = di->ExtFileAttributes; + fi.AllocationSize = di->AllocationSize; + fi.EndOfFile = di->EndOfFile; + fi.EASize = di->EaSize; + } + fi.NumberOfLinks = cpu_to_le32(1); + fi.DeletePending = 0; + fi.Directory = !!(le32_to_cpu(fi.Attributes) & ATTR_DIRECTORY); + cifs_buf_release(search_info.ntwrk_buf_start); + } else if (!full_path[0]) { + /* + * CIFSFindFirst() does not work on root path if the + * root path was exported on the server from the top + * level path (drive letter). + */ + rc = -EOPNOTSUPP; + } + } + + /* + * If everything failed then fallback to the legacy SMB command + * SMB_COM_QUERY_INFORMATION which works with all servers, but + * provide just few information. + */ + if ((rc == -EOPNOTSUPP || rc == -EINVAL) && !non_unicode_wildcard) { + rc = SMBQueryInformation(xid, tcon, full_path, &fi, cifs_sb->local_nls, + cifs_remap(cifs_sb)); + data->adjust_tz = true; + } else if ((rc == -EOPNOTSUPP || rc == -EINVAL) && non_unicode_wildcard) { + /* Path with non-UNICODE wildcard character cannot exist. */ + rc = -ENOENT; + } + + if (!rc) { + move_cifs_info_to_smb2(&data->fi, &fi); + data->reparse_point = le32_to_cpu(fi.Attributes) & ATTR_REPARSE_POINT; + } + +#ifdef CONFIG_CIFS_XATTR + /* + * For non-symlink WSL reparse points it is required to fetch + * EA $LXMOD which contains in its S_DT part the mandatory file type. + */ + if (!rc && data->reparse_point) { + struct smb2_file_full_ea_info *ea; + u32 next = 0; + + ea = (struct smb2_file_full_ea_info *)data->wsl.eas; + do { + ea = (void *)((u8 *)ea + next); + next = le32_to_cpu(ea->next_entry_offset); + } while (next); + if (le16_to_cpu(ea->ea_value_length)) { + ea->next_entry_offset = cpu_to_le32(ALIGN(sizeof(*ea) + + ea->ea_name_length + 1 + + le16_to_cpu(ea->ea_value_length), 4)); + ea = (void *)((u8 *)ea + le32_to_cpu(ea->next_entry_offset)); + } + + rc = CIFSSMBQAllEAs(xid, tcon, full_path, SMB2_WSL_XATTR_MODE, + &ea->ea_data[SMB2_WSL_XATTR_NAME_LEN + 1], + SMB2_WSL_XATTR_MODE_SIZE, cifs_sb); + if (rc == SMB2_WSL_XATTR_MODE_SIZE) { + ea->next_entry_offset = cpu_to_le32(0); + ea->flags = 0; + ea->ea_name_length = SMB2_WSL_XATTR_NAME_LEN; + ea->ea_value_length = cpu_to_le16(SMB2_WSL_XATTR_MODE_SIZE); + memcpy(&ea->ea_data[0], SMB2_WSL_XATTR_MODE, SMB2_WSL_XATTR_NAME_LEN + 1); + data->wsl.eas_len += ALIGN(sizeof(*ea) + SMB2_WSL_XATTR_NAME_LEN + 1 + + SMB2_WSL_XATTR_MODE_SIZE, 4); + rc = 0; + } else if (rc >= 0) { + /* It is an error if EA $LXMOD has wrong size. */ + rc = -EINVAL; + } else { + /* + * In all other cases ignore error if fetching + * of EA $LXMOD failed. It is needed only for + * non-symlink WSL reparse points and wsl_to_fattr() + * handle the case when EA is missing. + */ + rc = 0; + } + } + + /* + * For WSL CHR and BLK reparse points it is required to fetch + * EA $LXDEV which contains major and minor device numbers. + */ + if (!rc && data->reparse_point) { + struct smb2_file_full_ea_info *ea; + u32 next = 0; + + ea = (struct smb2_file_full_ea_info *)data->wsl.eas; + do { + ea = (void *)((u8 *)ea + next); + next = le32_to_cpu(ea->next_entry_offset); + } while (next); + if (le16_to_cpu(ea->ea_value_length)) { + ea->next_entry_offset = cpu_to_le32(ALIGN(sizeof(*ea) + + ea->ea_name_length + 1 + + le16_to_cpu(ea->ea_value_length), 4)); + ea = (void *)((u8 *)ea + le32_to_cpu(ea->next_entry_offset)); + } + + rc = CIFSSMBQAllEAs(xid, tcon, full_path, SMB2_WSL_XATTR_DEV, + &ea->ea_data[SMB2_WSL_XATTR_NAME_LEN + 1], + SMB2_WSL_XATTR_DEV_SIZE, cifs_sb); + if (rc == SMB2_WSL_XATTR_DEV_SIZE) { + ea->next_entry_offset = cpu_to_le32(0); + ea->flags = 0; + ea->ea_name_length = SMB2_WSL_XATTR_NAME_LEN; + ea->ea_value_length = cpu_to_le16(SMB2_WSL_XATTR_DEV_SIZE); + memcpy(&ea->ea_data[0], SMB2_WSL_XATTR_DEV, SMB2_WSL_XATTR_NAME_LEN + 1); + data->wsl.eas_len += ALIGN(sizeof(*ea) + SMB2_WSL_XATTR_NAME_LEN + 1 + + SMB2_WSL_XATTR_MODE_SIZE, 4); + rc = 0; + } else if (rc >= 0) { + /* It is an error if EA $LXDEV has wrong size. */ + rc = -EINVAL; + } else { + /* + * In all other cases ignore error if fetching + * of EA $LXDEV failed. It is needed only for + * WSL CHR and BLK reparse points and wsl_to_fattr() + * handle the case when EA is missing. + */ + rc = 0; + } + } +#endif + + return rc; +} + +static int cifs_get_srv_inum(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, const char *full_path, + u64 *uniqueid, struct cifs_open_info_data *unused) +{ + /* + * We can not use the IndexNumber field by default from Windows or + * Samba (in ALL_INFO buf) but we can request it explicitly. The SNIA + * CIFS spec claims that this value is unique within the scope of a + * share, and the windows docs hint that it's actually unique + * per-machine. + * + * There may be higher info levels that work but are there Windows + * server or network appliances for which IndexNumber field is not + * guaranteed unique? + * + * CIFSGetSrvInodeNumber() uses SMB_QUERY_FILE_INTERNAL_INFO + * which is SMB PASSTHROUGH level therefore check for capability. + * Note that this function can be called with tcon == NULL. + */ + if (tcon && !(tcon->ses->capabilities & CAP_INFOLEVEL_PASSTHRU)) + return -EOPNOTSUPP; + return CIFSGetSrvInodeNumber(xid, tcon, full_path, uniqueid, + cifs_sb->local_nls, + cifs_remap(cifs_sb)); +} + +static int cifs_query_file_info(const unsigned int xid, struct cifs_tcon *tcon, + struct cifsFileInfo *cfile, struct cifs_open_info_data *data) +{ + int rc; + FILE_ALL_INFO fi = {}; + + /* + * CIFSSMBQFileInfo() for non-NT servers returns bogus data in + * Attributes fields. So do not use this command for non-NT servers. + */ + if (!(tcon->ses->capabilities & CAP_NT_SMBS)) + return -EOPNOTSUPP; + + if (cfile->symlink_target) { + data->symlink_target = kstrdup(cfile->symlink_target, GFP_KERNEL); + if (!data->symlink_target) + return -ENOMEM; + } + + rc = CIFSSMBQFileInfo(xid, tcon, cfile->fid.netfid, &fi); + if (!rc) + move_cifs_info_to_smb2(&data->fi, &fi); + return rc; +} + +static void +cifs_clear_stats(struct cifs_tcon *tcon) +{ + atomic_set(&tcon->stats.cifs_stats.num_writes, 0); + atomic_set(&tcon->stats.cifs_stats.num_reads, 0); + atomic_set(&tcon->stats.cifs_stats.num_flushes, 0); + atomic_set(&tcon->stats.cifs_stats.num_oplock_brks, 0); + atomic_set(&tcon->stats.cifs_stats.num_opens, 0); + atomic_set(&tcon->stats.cifs_stats.num_posixopens, 0); + atomic_set(&tcon->stats.cifs_stats.num_posixmkdirs, 0); + atomic_set(&tcon->stats.cifs_stats.num_closes, 0); + atomic_set(&tcon->stats.cifs_stats.num_deletes, 0); + atomic_set(&tcon->stats.cifs_stats.num_mkdirs, 0); + atomic_set(&tcon->stats.cifs_stats.num_rmdirs, 0); + atomic_set(&tcon->stats.cifs_stats.num_renames, 0); + atomic_set(&tcon->stats.cifs_stats.num_t2renames, 0); + atomic_set(&tcon->stats.cifs_stats.num_ffirst, 0); + atomic_set(&tcon->stats.cifs_stats.num_fnext, 0); + atomic_set(&tcon->stats.cifs_stats.num_fclose, 0); + atomic_set(&tcon->stats.cifs_stats.num_hardlinks, 0); + atomic_set(&tcon->stats.cifs_stats.num_symlinks, 0); + atomic_set(&tcon->stats.cifs_stats.num_locks, 0); + atomic_set(&tcon->stats.cifs_stats.num_acl_get, 0); + atomic_set(&tcon->stats.cifs_stats.num_acl_set, 0); +} + +static void +cifs_print_stats(struct seq_file *m, struct cifs_tcon *tcon) +{ + seq_printf(m, " Oplocks breaks: %d", + atomic_read(&tcon->stats.cifs_stats.num_oplock_brks)); + seq_printf(m, "\nReads: %d Bytes: %llu", + atomic_read(&tcon->stats.cifs_stats.num_reads), + (long long)(tcon->bytes_read)); + seq_printf(m, "\nWrites: %d Bytes: %llu", + atomic_read(&tcon->stats.cifs_stats.num_writes), + (long long)(tcon->bytes_written)); + seq_printf(m, "\nFlushes: %d", + atomic_read(&tcon->stats.cifs_stats.num_flushes)); + seq_printf(m, "\nLocks: %d HardLinks: %d Symlinks: %d", + atomic_read(&tcon->stats.cifs_stats.num_locks), + atomic_read(&tcon->stats.cifs_stats.num_hardlinks), + atomic_read(&tcon->stats.cifs_stats.num_symlinks)); + seq_printf(m, "\nOpens: %d Closes: %d Deletes: %d", + atomic_read(&tcon->stats.cifs_stats.num_opens), + atomic_read(&tcon->stats.cifs_stats.num_closes), + atomic_read(&tcon->stats.cifs_stats.num_deletes)); + seq_printf(m, "\nPosix Opens: %d Posix Mkdirs: %d", + atomic_read(&tcon->stats.cifs_stats.num_posixopens), + atomic_read(&tcon->stats.cifs_stats.num_posixmkdirs)); + seq_printf(m, "\nMkdirs: %d Rmdirs: %d", + atomic_read(&tcon->stats.cifs_stats.num_mkdirs), + atomic_read(&tcon->stats.cifs_stats.num_rmdirs)); + seq_printf(m, "\nRenames: %d T2 Renames %d", + atomic_read(&tcon->stats.cifs_stats.num_renames), + atomic_read(&tcon->stats.cifs_stats.num_t2renames)); + seq_printf(m, "\nFindFirst: %d FNext %d FClose %d", + atomic_read(&tcon->stats.cifs_stats.num_ffirst), + atomic_read(&tcon->stats.cifs_stats.num_fnext), + atomic_read(&tcon->stats.cifs_stats.num_fclose)); +} + +static void +cifs_mkdir_setinfo(struct inode *inode, const char *full_path, + struct cifs_sb_info *cifs_sb, struct cifs_tcon *tcon, + const unsigned int xid) +{ + FILE_BASIC_INFO info; + struct cifsInodeInfo *cifsInode; + u32 dosattrs; + int rc; + + memset(&info, 0, sizeof(info)); + cifsInode = CIFS_I(inode); + dosattrs = cifsInode->cifsAttrs|ATTR_READONLY; + info.Attributes = cpu_to_le32(dosattrs); + rc = CIFSSMBSetPathInfo(xid, tcon, full_path, &info, cifs_sb->local_nls, + cifs_sb); + if (rc == -EOPNOTSUPP || rc == -EINVAL) + rc = SMBSetInformation(xid, tcon, full_path, + info.Attributes, + 0 /* do not change write time */, + cifs_sb->local_nls, cifs_sb); + if (rc == 0) + cifsInode->cifsAttrs = dosattrs; +} + +static int cifs_open_file(const unsigned int xid, struct cifs_open_parms *oparms, __u32 *oplock, + void *buf) +{ + struct cifs_open_info_data *data = buf; + FILE_ALL_INFO fi = {}; + int rc; + + if (!(oparms->tcon->ses->capabilities & CAP_NT_SMBS)) + rc = SMBLegacyOpen(xid, oparms->tcon, oparms->path, + oparms->disposition, + oparms->desired_access, + oparms->create_options, + &oparms->fid->netfid, oplock, &fi, + oparms->cifs_sb->local_nls, + cifs_remap(oparms->cifs_sb)); + else + rc = CIFS_open(xid, oparms, oplock, &fi); + + if (!rc && data) + move_cifs_info_to_smb2(&data->fi, &fi); + + return rc; +} + +static void +cifs_set_fid(struct cifsFileInfo *cfile, struct cifs_fid *fid, __u32 oplock) +{ + struct cifsInodeInfo *cinode = CIFS_I(d_inode(cfile->dentry)); + + lockdep_assert_held(&cinode->open_file_lock); + + cfile->fid.netfid = fid->netfid; + cifs_set_oplock_level(cinode, oplock); + cinode->can_cache_brlcks = CIFS_CACHE_WRITE(cinode); +} + +static int +cifs_close_file(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_fid *fid) +{ + return CIFSSMBClose(xid, tcon, fid->netfid); +} + +static int +cifs_flush_file(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_fid *fid) +{ + return CIFSSMBFlush(xid, tcon, fid->netfid); +} + +static int +cifs_sync_read(const unsigned int xid, struct cifs_fid *pfid, + struct cifs_io_parms *parms, unsigned int *bytes_read, + char **buf, int *buf_type) +{ + parms->netfid = pfid->netfid; + return CIFSSMBRead(xid, parms, bytes_read, buf, buf_type); +} + +static int +cifs_sync_write(const unsigned int xid, struct cifs_fid *pfid, + struct cifs_io_parms *parms, unsigned int *written, + struct kvec *iov, unsigned long nr_segs) +{ + + parms->netfid = pfid->netfid; + return CIFSSMBWrite2(xid, parms, written, iov, nr_segs); +} + +static int +smb_set_file_info(struct inode *inode, const char *full_path, + FILE_BASIC_INFO *buf, const unsigned int xid) +{ + int oplock = 0; + int rc; + __u32 netpid; + struct cifs_fid fid; + struct cifs_open_parms oparms; + struct cifsFileInfo *open_file; + FILE_BASIC_INFO new_buf; + struct cifs_open_info_data query_data = {}; + __le64 write_time = buf->LastWriteTime; + struct cifsInodeInfo *cinode = CIFS_I(inode); + struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); + struct tcon_link *tlink = NULL; + struct cifs_tcon *tcon; + + /* if the file is already open for write, just use that fileid */ + open_file = find_writable_file(cinode, FIND_FSUID_ONLY); + + if (open_file) { + fid.netfid = open_file->fid.netfid; + netpid = open_file->pid; + tcon = tlink_tcon(open_file->tlink); + } else { + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) { + rc = PTR_ERR(tlink); + tlink = NULL; + goto out; + } + tcon = tlink_tcon(tlink); + } + + /* + * Non-NT servers interprets zero time value in SMB_SET_FILE_BASIC_INFO + * over TRANS2_SET_FILE_INFORMATION as a valid time value. NT servers + * interprets zero time value as do not change existing value on server. + * API of ->set_file_info() callback expects that zero time value has + * the NT meaning - do not change. Therefore if server is non-NT and + * some time values in "buf" are zero, then fetch missing time values. + */ + if (!(tcon->ses->capabilities & CAP_NT_SMBS) && + (!buf->CreationTime || !buf->LastAccessTime || + !buf->LastWriteTime || !buf->ChangeTime)) { + rc = cifs_query_path_info(xid, tcon, cifs_sb, full_path, &query_data); + if (rc) { + if (open_file) { + cifsFileInfo_put(open_file); + open_file = NULL; + } + goto out; + } + /* + * Original write_time from buf->LastWriteTime is preserved + * as SMBSetInformation() interprets zero as do not change. + */ + new_buf = *buf; + buf = &new_buf; + if (!buf->CreationTime) + buf->CreationTime = query_data.fi.CreationTime; + if (!buf->LastAccessTime) + buf->LastAccessTime = query_data.fi.LastAccessTime; + if (!buf->LastWriteTime) + buf->LastWriteTime = query_data.fi.LastWriteTime; + if (!buf->ChangeTime) + buf->ChangeTime = query_data.fi.ChangeTime; + } + + if (open_file) + goto set_via_filehandle; + + rc = CIFSSMBSetPathInfo(xid, tcon, full_path, buf, cifs_sb->local_nls, + cifs_sb); + if (rc == 0) { + cinode->cifsAttrs = le32_to_cpu(buf->Attributes); + goto out; + } else if (rc != -EOPNOTSUPP && rc != -EINVAL) { + goto out; + } + + oparms = (struct cifs_open_parms) { + .tcon = tcon, + .cifs_sb = cifs_sb, + .desired_access = SYNCHRONIZE | FILE_WRITE_ATTRIBUTES, + .create_options = cifs_create_options(cifs_sb, 0), + .disposition = FILE_OPEN, + .path = full_path, + .fid = &fid, + }; + + if (S_ISDIR(inode->i_mode) && !(tcon->ses->capabilities & CAP_NT_SMBS)) { + /* Opening directory path is not possible on non-NT servers. */ + rc = -EOPNOTSUPP; + } else { + /* + * Use cifs_open_file() instead of CIFS_open() as the + * cifs_open_file() selects the correct function which + * works also on non-NT servers. + */ + rc = cifs_open_file(xid, &oparms, &oplock, NULL); + /* + * Opening path for writing on non-NT servers is not + * possible when the read-only attribute is already set. + * Non-NT server in this case returns -EACCES. For those + * servers the only possible way how to clear the read-only + * bit is via SMB_COM_SETATTR command. + */ + if (rc == -EACCES && + (cinode->cifsAttrs & ATTR_READONLY) && + le32_to_cpu(buf->Attributes) != 0 && /* 0 = do not change attrs */ + !(le32_to_cpu(buf->Attributes) & ATTR_READONLY) && + !(tcon->ses->capabilities & CAP_NT_SMBS)) + rc = -EOPNOTSUPP; + } + + /* Fallback to SMB_COM_SETATTR command when absolutely needed. */ + if (rc == -EOPNOTSUPP) { + cifs_dbg(FYI, "calling SetInformation since SetPathInfo for attrs/times not supported by this server\n"); + rc = SMBSetInformation(xid, tcon, full_path, + buf->Attributes != 0 ? buf->Attributes : cpu_to_le32(cinode->cifsAttrs), + write_time, + cifs_sb->local_nls, cifs_sb); + if (rc == 0) + cinode->cifsAttrs = le32_to_cpu(buf->Attributes); + else + rc = -EACCES; + goto out; + } + + if (rc != 0) { + if (rc == -EIO) + rc = -EINVAL; + goto out; + } + + netpid = current->tgid; + cifs_dbg(FYI, "calling SetFileInfo since SetPathInfo for attrs/times not supported by this server\n"); + +set_via_filehandle: + rc = CIFSSMBSetFileInfo(xid, tcon, buf, fid.netfid, netpid); + if (!rc) + cinode->cifsAttrs = le32_to_cpu(buf->Attributes); + + if (open_file == NULL) + CIFSSMBClose(xid, tcon, fid.netfid); + else + cifsFileInfo_put(open_file); + + /* + * Setting the read-only bit is not honored on non-NT servers when done + * via open-semantics. So for setting it, use SMB_COM_SETATTR command. + * This command works only after the file is closed, so use it only when + * operation was called without the filehandle. + */ + if (open_file == NULL && + !(tcon->ses->capabilities & CAP_NT_SMBS) && + le32_to_cpu(buf->Attributes) & ATTR_READONLY) { + SMBSetInformation(xid, tcon, full_path, + buf->Attributes, + 0 /* do not change write time */, + cifs_sb->local_nls, cifs_sb); + } +out: + if (tlink != NULL) + cifs_put_tlink(tlink); + return rc; +} + +static int +cifs_set_compression(const unsigned int xid, struct cifs_tcon *tcon, + struct cifsFileInfo *cfile, __u16 compression_state) +{ + return CIFSSMB_set_compression(xid, tcon, cfile->fid.netfid, + compression_state); +} + +static int +cifs_query_dir_first(const unsigned int xid, struct cifs_tcon *tcon, + const char *path, struct cifs_sb_info *cifs_sb, + struct cifs_fid *fid, __u16 search_flags, + struct cifs_search_info *srch_inf) +{ + int rc; + + rc = CIFSFindFirst(xid, tcon, path, cifs_sb, + &fid->netfid, search_flags, srch_inf, true); + if (rc) + cifs_dbg(FYI, "find first failed=%d\n", rc); + return rc; +} + +static int +cifs_query_dir_next(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_fid *fid, __u16 search_flags, + struct cifs_search_info *srch_inf) +{ + return CIFSFindNext(xid, tcon, fid->netfid, search_flags, srch_inf); +} + +static int +cifs_close_dir(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_fid *fid) +{ + return CIFSFindClose(xid, tcon, fid->netfid); +} + +static int cifs_oplock_response(struct cifs_tcon *tcon, __u64 persistent_fid, + __u64 volatile_fid, __u16 net_fid, + struct cifsInodeInfo *cinode, unsigned int oplock) +{ + unsigned int sbflags = cifs_sb_flags(CIFS_SB(cinode)); + __u8 op; + + op = !!((oplock & CIFS_CACHE_READ_FLG) || (sbflags & CIFS_MOUNT_RO_CACHE)); + return CIFSSMBLock(0, tcon, net_fid, current->tgid, 0, 0, 0, 0, + LOCKING_ANDX_OPLOCK_RELEASE, false, op); +} + +static int +cifs_queryfs(const unsigned int xid, struct cifs_tcon *tcon, + const char *path, struct cifs_sb_info *cifs_sb, struct kstatfs *buf) +{ + int rc = -EOPNOTSUPP; + + buf->f_type = CIFS_SUPER_MAGIC; + + /* + * We could add a second check for a QFS Unix capability bit + */ + if ((tcon->ses->capabilities & CAP_UNIX) && + (CIFS_POSIX_EXTENSIONS & le64_to_cpu(tcon->fsUnixInfo.Capability))) + rc = CIFSSMBQFSPosixInfo(xid, tcon, buf); + + /* + * Only need to call the old QFSInfo if failed on newer one, + * e.g. by OS/2. + **/ + if (rc && (tcon->ses->capabilities & CAP_NT_SMBS)) + rc = CIFSSMBQFSInfo(xid, tcon, buf); + + /* + * Some old Windows servers also do not support level 103, retry with + * older level one if old server failed the previous call or we + * bypassed it because we detected that this was an older LANMAN sess + */ + if (rc) + rc = SMBOldQFSInfo(xid, tcon, buf); + return rc; +} + +static int +cifs_mand_lock(const unsigned int xid, struct cifsFileInfo *cfile, __u64 offset, + __u64 length, __u32 type, int lock, int unlock, bool wait) +{ + return CIFSSMBLock(xid, tlink_tcon(cfile->tlink), cfile->fid.netfid, + current->tgid, length, offset, unlock, lock, + (__u8)type, wait, 0); +} + +static int +cifs_unix_dfs_readlink(const unsigned int xid, struct cifs_tcon *tcon, + const unsigned char *searchName, char **symlinkinfo, + const struct nls_table *nls_codepage) +{ +#ifdef CONFIG_CIFS_DFS_UPCALL + int rc; + struct dfs_info3_param referral = {0}; + + rc = get_dfs_path(xid, tcon->ses, searchName, nls_codepage, &referral, + 0); + + if (!rc) { + *symlinkinfo = kstrdup(referral.node_name, GFP_KERNEL); + free_dfs_info_param(&referral); + if (!*symlinkinfo) + rc = -ENOMEM; + } + return rc; +#else /* No DFS support */ + return -EREMOTE; +#endif +} + +static int cifs_query_symlink(const unsigned int xid, + struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, + const char *full_path, + char **target_path) +{ + int rc; + + cifs_tcon_dbg(FYI, "%s: path=%s\n", __func__, full_path); + + if (!cap_unix(tcon->ses)) + return -EOPNOTSUPP; + + rc = CIFSSMBUnixQuerySymLink(xid, tcon, full_path, target_path, + cifs_sb->local_nls, cifs_remap(cifs_sb)); + if (rc == -EREMOTE) + rc = cifs_unix_dfs_readlink(xid, tcon, full_path, + target_path, cifs_sb->local_nls); + return rc; +} + +static struct reparse_data_buffer *cifs_get_reparse_point_buffer(const struct kvec *rsp_iov, + u32 *plen) +{ + TRANSACT_IOCTL_RSP *io = rsp_iov->iov_base; + *plen = le16_to_cpu(io->ByteCount); + return (struct reparse_data_buffer *)((__u8 *)&io->hdr.Protocol + + le32_to_cpu(io->DataOffset)); +} + +static bool +cifs_is_read_op(__u32 oplock) +{ + return oplock == OPLOCK_READ; +} + +static unsigned int +cifs_wp_retry_size(struct inode *inode) +{ + return CIFS_SB(inode->i_sb)->ctx->wsize; +} + +static bool +cifs_dir_needs_close(struct cifsFileInfo *cfile) +{ + return !cfile->srch_inf.endOfSearch && !cfile->invalidHandle; +} + +static bool +cifs_can_echo(struct TCP_Server_Info *server) +{ + if (server->tcpStatus == CifsGood) + return true; + + return false; +} + +static int +cifs_make_node(unsigned int xid, struct inode *inode, + struct dentry *dentry, struct cifs_tcon *tcon, + const char *full_path, umode_t mode, dev_t dev) +{ + struct cifs_sb_info *cifs_sb = CIFS_SB(inode); + unsigned int sbflags = cifs_sb_flags(cifs_sb); + struct inode *newinode = NULL; + int rc; + + if (tcon->unix_ext) { + /* + * SMB1 Unix Extensions: requires server support but + * works with all special files + */ + struct cifs_unix_set_info_args args = { + .mode = mode & ~current_umask(), + .ctime = NO_CHANGE_64, + .atime = NO_CHANGE_64, + .mtime = NO_CHANGE_64, + .device = dev, + }; + if (sbflags & CIFS_MOUNT_SET_UID) { + args.uid = current_fsuid(); + args.gid = current_fsgid(); + } else { + args.uid = INVALID_UID; /* no change */ + args.gid = INVALID_GID; /* no change */ + } + rc = CIFSSMBUnixSetPathInfo(xid, tcon, full_path, &args, + cifs_sb->local_nls, + cifs_remap(cifs_sb)); + if (rc) + return rc; + + rc = cifs_get_inode_info_unix(&newinode, full_path, + inode->i_sb, xid); + + if (rc == 0) + d_instantiate(dentry, newinode); + return rc; + } else if (sbflags & CIFS_MOUNT_UNX_EMUL) { + /* + * Check if mounted with mount parm 'sfu' mount parm. + * SFU emulation should work with all servers + * and was used by default in earlier versions of Windows. + */ + return cifs_sfu_make_node(xid, inode, dentry, tcon, + full_path, mode, dev); + } else if (CIFS_REPARSE_SUPPORT(tcon)) { + /* + * mknod via reparse points requires server support for + * storing reparse points, which is available since + * Windows 2000, but was not widely used until release + * of Windows Server 2012 by the Windows NFS server. + */ + return mknod_reparse(xid, inode, dentry, tcon, + full_path, mode, dev); + } else { + return -EOPNOTSUPP; + } +} + +static bool +cifs_is_network_name_deleted(char *buf, struct TCP_Server_Info *server) +{ + struct smb_hdr *shdr = (struct smb_hdr *)buf; + struct TCP_Server_Info *pserver; + struct cifs_ses *ses; + struct cifs_tcon *tcon; + + if (shdr->Flags2 & SMBFLG2_ERR_STATUS) { + if (shdr->Status.CifsError != cpu_to_le32(NT_STATUS_NETWORK_NAME_DELETED)) + return false; + } else { + if (shdr->Status.DosError.ErrorClass != ERRSRV || + shdr->Status.DosError.Error != cpu_to_le16(ERRinvtid)) + return false; + } + + /* If server is a channel, select the primary channel */ + pserver = SERVER_IS_CHAN(server) ? server->primary_server : server; + + spin_lock(&cifs_tcp_ses_lock); + list_for_each_entry(ses, &pserver->smb_ses_list, smb_ses_list) { + if (cifs_ses_exiting(ses)) + continue; + list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { + if (tcon->tid == shdr->Tid) { + spin_lock(&tcon->tc_lock); + tcon->need_reconnect = true; + spin_unlock(&tcon->tc_lock); + spin_unlock(&cifs_tcp_ses_lock); + pr_warn_once("Server share %s deleted.\n", + tcon->tree_name); + return true; + } + } + } + spin_unlock(&cifs_tcp_ses_lock); + + return false; +} + +struct smb_version_operations smb1_operations = { + .send_cancel = cifs_send_cancel, + .compare_fids = cifs_compare_fids, + .setup_request = cifs_setup_request, + .setup_async_request = cifs_setup_async_request, + .check_receive = cifs_check_receive, + .add_credits = cifs_add_credits, + .set_credits = cifs_set_credits, + .get_credits_field = cifs_get_credits_field, + .get_credits = cifs_get_credits, + .wait_mtu_credits = cifs_wait_mtu_credits, + .get_next_mid = cifs_get_next_mid, + .read_data_offset = cifs_read_data_offset, + .read_data_length = cifs_read_data_length, + .map_error = map_smb_to_linux_error, + .find_mid = cifs_find_mid, + .check_message = checkSMB, + .dump_detail = cifs_dump_detail, + .clear_stats = cifs_clear_stats, + .print_stats = cifs_print_stats, + .is_oplock_break = is_valid_oplock_break, + .downgrade_oplock = cifs_downgrade_oplock, + .check_trans2 = cifs_check_trans2, + .need_neg = cifs_need_neg, + .negotiate = cifs_negotiate, + .negotiate_wsize = smb1_negotiate_wsize, + .negotiate_rsize = smb1_negotiate_rsize, + .sess_setup = CIFS_SessSetup, + .logoff = CIFSSMBLogoff, + .tree_connect = CIFSTCon, + .tree_disconnect = CIFSSMBTDis, + .get_dfs_refer = CIFSGetDFSRefer, + .qfs_tcon = cifs_qfs_tcon, + .is_path_accessible = cifs_is_path_accessible, + .can_echo = cifs_can_echo, + .query_path_info = cifs_query_path_info, + .query_reparse_point = cifs_query_reparse_point, + .query_file_info = cifs_query_file_info, + .get_srv_inum = cifs_get_srv_inum, + .set_path_size = CIFSSMBSetEOF, + .set_file_size = CIFSSMBSetFileSize, + .set_file_info = smb_set_file_info, + .set_compression = cifs_set_compression, + .echo = CIFSSMBEcho, + .mkdir = CIFSSMBMkDir, + .mkdir_setinfo = cifs_mkdir_setinfo, + .rmdir = CIFSSMBRmDir, + .unlink = CIFSSMBDelFile, + .rename_pending_delete = cifs_rename_pending_delete, + .rename = CIFSSMBRename, + .create_hardlink = CIFSCreateHardLink, + .query_symlink = cifs_query_symlink, + .get_reparse_point_buffer = cifs_get_reparse_point_buffer, + .create_reparse_inode = cifs_create_reparse_inode, + .open = cifs_open_file, + .set_fid = cifs_set_fid, + .close = cifs_close_file, + .flush = cifs_flush_file, + .async_readv = cifs_async_readv, + .async_writev = cifs_async_writev, + .sync_read = cifs_sync_read, + .sync_write = cifs_sync_write, + .query_dir_first = cifs_query_dir_first, + .query_dir_next = cifs_query_dir_next, + .close_dir = cifs_close_dir, + .calc_smb_size = smbCalcSize, + .oplock_response = cifs_oplock_response, + .queryfs = cifs_queryfs, + .mand_lock = cifs_mand_lock, + .mand_unlock_range = cifs_unlock_range, + .push_mand_locks = cifs_push_mandatory_locks, + .query_mf_symlink = cifs_query_mf_symlink, + .create_mf_symlink = cifs_create_mf_symlink, + .is_read_op = cifs_is_read_op, + .wp_retry_size = cifs_wp_retry_size, + .dir_needs_close = cifs_dir_needs_close, + .select_sectype = cifs_select_sectype, +#ifdef CONFIG_CIFS_XATTR + .query_all_EAs = CIFSSMBQAllEAs, + .set_EA = CIFSSMBSetEA, +#endif /* CIFS_XATTR */ + .get_acl = get_cifs_acl, + .get_acl_by_fid = get_cifs_acl_by_fid, + .set_acl = set_cifs_acl, + .make_node = cifs_make_node, + .is_network_name_deleted = cifs_is_network_name_deleted, +}; + +struct smb_version_values smb1_values = { + .version_string = SMB1_VERSION_STRING, + .protocol_id = SMB10_PROT_ID, + .large_lock_type = LOCKING_ANDX_LARGE_FILES, + .exclusive_lock_type = 0, + .shared_lock_type = LOCKING_ANDX_SHARED_LOCK, + .unlock_lock_type = 0, + .header_size = sizeof(struct smb_hdr), + .max_header_size = MAX_CIFS_HDR_SIZE, + .read_rsp_size = sizeof(READ_RSP), + .lock_cmd = cpu_to_le16(SMB_COM_LOCKING_ANDX), + .cap_unix = CAP_UNIX, + .cap_nt_find = CAP_NT_SMBS | CAP_NT_FIND, + .cap_large_files = CAP_LARGE_FILES, + .cap_unicode = CAP_UNICODE, + .signing_enabled = SECMODE_SIGN_ENABLED, + .signing_required = SECMODE_SIGN_REQUIRED, +}; diff --git a/fs/smb/client/smb1pdu.h b/fs/smb/client/smb1pdu.h new file mode 100644 index 000000000..087094914 --- /dev/null +++ b/fs/smb/client/smb1pdu.h @@ -0,0 +1,2340 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +/* + * + * Copyright (c) International Business Machines Corp., 2002,2009 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ + +#ifndef _SMB1PDU_H +#define _SMB1PDU_H + +#include "../common/smb1pdu.h" + +#define CIFS_PROT 0 +#define POSIX_PROT (CIFS_PROT+1) +#define BAD_PROT 0xFFFF + +/* SMB command codes: + * See MS-CIFS 2.2.2.1 + * Note some commands have minimal (wct=0,bcc=0), or uninteresting, responses + * (ie which include no useful data other than the SMB error code itself). + * This can allow us to avoid response buffer allocations and copy in some cases + */ +#define SMB_COM_CREATE_DIRECTORY 0x00 /* trivial response */ +#define SMB_COM_DELETE_DIRECTORY 0x01 /* trivial response */ +#define SMB_COM_CLOSE 0x04 /* triv req/rsp, timestamp ignored */ +#define SMB_COM_FLUSH 0x05 /* triv req/rsp */ +#define SMB_COM_DELETE 0x06 /* trivial response */ +#define SMB_COM_RENAME 0x07 /* trivial response */ +#define SMB_COM_QUERY_INFORMATION 0x08 /* aka getattr */ +#define SMB_COM_SETATTR 0x09 /* trivial response */ +#define SMB_COM_LOCKING_ANDX 0x24 /* trivial response */ +#define SMB_COM_COPY 0x29 /* trivial rsp, fail filename ignrd*/ +#define SMB_COM_ECHO 0x2B /* echo request */ +#define SMB_COM_OPEN_ANDX 0x2D /* Legacy open for old servers */ +#define SMB_COM_READ_ANDX 0x2E +#define SMB_COM_WRITE_ANDX 0x2F +#define SMB_COM_TRANSACTION2 0x32 +#define SMB_COM_TRANSACTION2_SECONDARY 0x33 +#define SMB_COM_FIND_CLOSE2 0x34 /* trivial response */ +#define SMB_COM_TREE_DISCONNECT 0x71 /* trivial response */ +#define SMB_COM_NEGOTIATE 0x72 +#define SMB_COM_SESSION_SETUP_ANDX 0x73 +#define SMB_COM_LOGOFF_ANDX 0x74 /* trivial response */ +#define SMB_COM_TREE_CONNECT_ANDX 0x75 +#define SMB_COM_NT_TRANSACT 0xA0 +#define SMB_COM_NT_TRANSACT_SECONDARY 0xA1 +#define SMB_COM_NT_CREATE_ANDX 0xA2 +#define SMB_COM_NT_CANCEL 0xA4 /* no response */ +#define SMB_COM_NT_RENAME 0xA5 /* trivial response */ + +/* Transact2 subcommand codes */ +#define TRANS2_OPEN 0x00 +#define TRANS2_FIND_FIRST 0x01 +#define TRANS2_FIND_NEXT 0x02 +#define TRANS2_QUERY_FS_INFORMATION 0x03 +#define TRANS2_SET_FS_INFORMATION 0x04 +#define TRANS2_QUERY_PATH_INFORMATION 0x05 +#define TRANS2_SET_PATH_INFORMATION 0x06 +#define TRANS2_QUERY_FILE_INFORMATION 0x07 +#define TRANS2_SET_FILE_INFORMATION 0x08 +#define TRANS2_GET_DFS_REFERRAL 0x10 +#define TRANS2_REPORT_DFS_INCOSISTENCY 0x11 + +/* SMB Transact (Named Pipe) subcommand codes */ +#define TRANS_SET_NMPIPE_STATE 0x0001 +#define TRANS_RAW_READ_NMPIPE 0x0011 +#define TRANS_QUERY_NMPIPE_STATE 0x0021 +#define TRANS_QUERY_NMPIPE_INFO 0x0022 +#define TRANS_PEEK_NMPIPE 0x0023 +#define TRANS_TRANSACT_NMPIPE 0x0026 +#define TRANS_RAW_WRITE_NMPIPE 0x0031 +#define TRANS_READ_NMPIPE 0x0036 +#define TRANS_WRITE_NMPIPE 0x0037 +#define TRANS_WAIT_NMPIPE 0x0053 +#define TRANS_CALL_NMPIPE 0x0054 + +/* NT Transact subcommand codes */ +#define NT_TRANSACT_CREATE 0x01 +#define NT_TRANSACT_IOCTL 0x02 +#define NT_TRANSACT_SET_SECURITY_DESC 0x03 +#define NT_TRANSACT_NOTIFY_CHANGE 0x04 +#define NT_TRANSACT_RENAME 0x05 +#define NT_TRANSACT_QUERY_SECURITY_DESC 0x06 +#define NT_TRANSACT_GET_USER_QUOTA 0x07 +#define NT_TRANSACT_SET_USER_QUOTA 0x08 + +/* future chained NTCreateXReadX bigger, but for time being NTCreateX biggest */ +/* among the requests (NTCreateX response is bigger with wct of 34) */ +#define MAX_CIFS_HDR_SIZE 0x54 /* 32 hdr + (2*24 wct) + 2 bct + 2 pad */ +#define CIFS_SMALL_PATH 120 /* allows for (448-88)/3 */ + +/* internal cifs vfs structures */ +/***************************************************************** + * All constants go here + ***************************************************************** + */ + +/* + * Starting value for maximum SMB size negotiation + */ +#define CIFS_MAX_MSGSIZE (4*4096) + +/* + * Size of encrypted user password in bytes + */ +#define CIFS_ENCPWD_SIZE (16) + +/* + * Size of the crypto key returned on the negotiate SMB in bytes + */ +#define CIFS_CRYPTO_KEY_SIZE (8) + +/* + * Size of the ntlm client response + */ +#define CIFS_AUTH_RESP_SIZE (24) + +/* + * Size of the session key (crypto key encrypted with the password + */ +#define CIFS_SESS_KEY_SIZE (16) + +#define CIFS_SERVER_CHALLENGE_SIZE (8) +#define CIFS_HMAC_MD5_HASH_SIZE (16) +#define CIFS_CPHTXT_SIZE (16) +#define CIFS_NTHASH_SIZE (16) + +/* + * Maximum user name length + */ +#define CIFS_UNLEN (20) + +/* + * Flags on SMB open + */ +#define SMBOPEN_WRITE_THROUGH 0x4000 +#define SMBOPEN_DENY_ALL 0x0010 +#define SMBOPEN_DENY_WRITE 0x0020 +#define SMBOPEN_DENY_READ 0x0030 +#define SMBOPEN_DENY_NONE 0x0040 +#define SMBOPEN_READ 0x0000 +#define SMBOPEN_WRITE 0x0001 +#define SMBOPEN_READWRITE 0x0002 +#define SMBOPEN_EXECUTE 0x0003 + +#define SMBOPEN_OCREATE 0x0010 +#define SMBOPEN_OTRUNC 0x0002 +#define SMBOPEN_OAPPEND 0x0001 + +/* + * SMB flag definitions + * See MS-CIFS 2.2.3.1 + */ +#define SMBFLG_EXTD_LOCK 0x01 /* server supports lock-read write-unlock smb */ +#define SMBFLG_RCV_POSTED 0x02 /* obsolete */ +#define SMBFLG_RSVD 0x04 +#define SMBFLG_CASELESS 0x08 /* all pathnames treated as caseless (off + implies case sensitive file handling request) */ +#define SMBFLG_CANONICAL_PATH_FORMAT 0x10 /* obsolete */ +#define SMBFLG_OLD_OPLOCK 0x20 /* obsolete */ +#define SMBFLG_OLD_OPLOCK_NOTIFY 0x40 /* obsolete */ +#define SMBFLG_RESPONSE 0x80 /* this PDU is a response from server */ + +/* + * SMB flag2 definitions + * See MS-CIFS 2.2.3.1 + * MS-SMB 2.2.3.1 + */ +#define SMBFLG2_KNOWS_LONG_NAMES cpu_to_le16(1) /* can send long (non-8.3) + path names in response */ +#define SMBFLG2_KNOWS_EAS cpu_to_le16(2) +#define SMBFLG2_SECURITY_SIGNATURE cpu_to_le16(4) +#define SMBFLG2_COMPRESSED (8) +#define SMBFLG2_SECURITY_SIGNATURE_REQUIRED (0x10) +#define SMBFLG2_IS_LONG_NAME cpu_to_le16(0x40) +#define SMBFLG2_REPARSE_PATH (0x400) +#define SMBFLG2_EXT_SEC cpu_to_le16(0x800) +#define SMBFLG2_DFS cpu_to_le16(0x1000) +#define SMBFLG2_PAGING_IO cpu_to_le16(0x2000) +#define SMBFLG2_ERR_STATUS cpu_to_le16(0x4000) +#define SMBFLG2_UNICODE cpu_to_le16(0x8000) + +/* Combinations of file access permission bits */ +#define SET_FILE_READ_RIGHTS (FILE_READ_DATA | FILE_READ_EA | FILE_WRITE_EA \ + | FILE_READ_ATTRIBUTES \ + | FILE_WRITE_ATTRIBUTES \ + | DELETE | READ_CONTROL | WRITE_DAC \ + | WRITE_OWNER | SYNCHRONIZE) +#define SET_FILE_WRITE_RIGHTS (FILE_WRITE_DATA | FILE_APPEND_DATA \ + | FILE_READ_EA | FILE_WRITE_EA \ + | FILE_READ_ATTRIBUTES \ + | FILE_WRITE_ATTRIBUTES \ + | DELETE | READ_CONTROL | WRITE_DAC \ + | WRITE_OWNER | SYNCHRONIZE) + +/* + * Invalid readdir handle + */ +#define CIFS_NO_HANDLE 0xFFFF + +#define NO_CHANGE_64 0xFFFFFFFFFFFFFFFFULL + +/* IPC$ in ASCII */ +#define CIFS_IPC_RESOURCE "\x49\x50\x43\x24" + +/* IPC$ in Unicode */ +#define CIFS_IPC_UNICODE_RESOURCE "\x00\x49\x00\x50\x00\x43\x00\x24\x00\x00" + +/* Unicode Null terminate 2 bytes of 0 */ +#define UNICODE_NULL "\x00\x00" +#define ASCII_NULL 0x00 + +/* + * Server type values (returned on EnumServer API + */ +#define CIFS_SV_TYPE_DC 0x00000008 +#define CIFS_SV_TYPE_BACKDC 0x00000010 + +/* + * Alias type flags (From EnumAlias API call + */ +#define CIFS_ALIAS_TYPE_FILE 0x0001 +#define CIFS_SHARE_TYPE_FILE 0x0000 + +/* + * File Attribute flags + */ +#define ATTR_READONLY 0x0001 /* See MS-CIFS 2.2.1.2.3 */ +#define ATTR_HIDDEN 0x0002 /* See MS-CIFS 2.2.1.2.3 */ +#define ATTR_SYSTEM 0x0004 /* See MS-CIFS 2.2.1.2.3 */ +#define ATTR_VOLUME 0x0008 +#define ATTR_DIRECTORY 0x0010 /* See MS-CIFS 2.2.1.2.3 */ +#define ATTR_ARCHIVE 0x0020 /* See MS-CIFS 2.2.1.2.3 */ +#define ATTR_DEVICE 0x0040 +#define ATTR_NORMAL 0x0080 /* See MS-CIFS 2.2.1.2.3 */ +#define ATTR_TEMPORARY 0x0100 /* See MS-CIFS 2.2.1.2.3 */ +#define ATTR_SPARSE 0x0200 /* See MS-SMB 2.2.1.2.1 */ +#define ATTR_REPARSE_POINT 0x0400 /* See MS-SMB 2.2.1.2.1 */ +#define ATTR_COMPRESSED 0x0800 /* See MS-CIFS 2.2.1.2.3 */ +#define ATTR_OFFLINE 0x1000 /* See MS-SMB 2.2.1.2.1 + ie file not immediately available - + on offline storage */ +#define ATTR_NOT_CONTENT_INDEXED 0x2000 /* See MS-SMB 2.2.1.2.1 */ +#define ATTR_ENCRYPTED 0x4000 /* See MS-SMB 2.2.1.2.1 */ +#define ATTR_POSIX_SEMANTICS 0x0100000 /* See MS-CIFS 2.2.1.2.3 */ +#define ATTR_BACKUP_SEMANTICS 0x0200000 /* See MS-CIFS 2.2.1.2.3 */ +#define ATTR_DELETE_ON_CLOSE 0x0400000 /* See MS-CIFS 2.2.1.2.3 */ +#define ATTR_SEQUENTIAL_SCAN 0x0800000 /* See MS-CIFS 2.2.1.2.3 */ +#define ATTR_RANDOM_ACCESS 0x1000000 /* See MS-CIFS 2.2.1.2.3 */ +#define ATTR_NO_BUFFERING 0x2000000 /* See MS-CIFS 2.2.1.2.3 */ +#define ATTR_WRITE_THROUGH 0x8000000 /* See MS-CIFS 2.2.1.2.3 */ + +/* ShareAccess flags */ +#define FILE_NO_SHARE 0x00000000 +#define FILE_SHARE_READ 0x00000001 +#define FILE_SHARE_WRITE 0x00000002 +#define FILE_SHARE_DELETE 0x00000004 +#define FILE_SHARE_ALL 0x00000007 + +/* CreateDisposition flags, similar to CreateAction as well */ +#define FILE_SUPERSEDE 0x00000000 +#define FILE_OPEN 0x00000001 +#define FILE_CREATE 0x00000002 +#define FILE_OPEN_IF 0x00000003 +#define FILE_OVERWRITE 0x00000004 +#define FILE_OVERWRITE_IF 0x00000005 + +/* CreateOptions */ +#define CREATE_NOT_FILE 0x00000001 /* if set must not be file */ +#define CREATE_WRITE_THROUGH 0x00000002 +#define CREATE_SEQUENTIAL 0x00000004 +#define CREATE_NO_BUFFER 0x00000008 /* should not buffer on srv */ +#define CREATE_SYNC_ALERT 0x00000010 /* MBZ */ +#define CREATE_ASYNC_ALERT 0x00000020 /* MBZ */ +#define CREATE_NOT_DIR 0x00000040 /* if set must not be directory */ +#define CREATE_TREE_CONNECTION 0x00000080 /* should be zero */ +#define CREATE_COMPLETE_IF_OPLK 0x00000100 /* should be zero */ +#define CREATE_NO_EA_KNOWLEDGE 0x00000200 +#define CREATE_EIGHT_DOT_THREE 0x00000400 /* doc says this is obsolete + "open for recovery" flag should + be zero in any case */ +#define CREATE_OPEN_FOR_RECOVERY 0x00000400 +#define CREATE_RANDOM_ACCESS 0x00000800 +#define CREATE_DELETE_ON_CLOSE 0x00001000 +#define CREATE_OPEN_BY_ID 0x00002000 +#define CREATE_OPEN_BACKUP_INTENT 0x00004000 +#define CREATE_NO_COMPRESSION 0x00008000 +#define CREATE_RESERVE_OPFILTER 0x00100000 /* should be zero */ +#define OPEN_REPARSE_POINT 0x00200000 +#define OPEN_NO_RECALL 0x00400000 +#define OPEN_FREE_SPACE_QUERY 0x00800000 /* should be zero */ +#define CREATE_OPTIONS_MASK 0x007FFFFF +#define CREATE_OPTION_READONLY 0x10000000 +#define CREATE_OPTION_SPECIAL 0x20000000 /* system. NB not sent over wire */ + +/* ImpersonationLevel flags */ +#define SECURITY_ANONYMOUS 0 +#define SECURITY_IDENTIFICATION 1 +#define SECURITY_IMPERSONATION 2 +#define SECURITY_DELEGATION 3 + +/* SecurityFlags */ +#define SECURITY_CONTEXT_TRACKING 0x01 +#define SECURITY_EFFECTIVE_ONLY 0x02 + +/* + * Default PID value, used in all SMBs where the PID is not important + */ +#define CIFS_DFT_PID 0x1234 + +/* + * We use the same routine for Copy and Move SMBs. This flag is used to + * distinguish + */ +#define CIFS_COPY_OP 1 +#define CIFS_RENAME_OP 2 + +/* + * Computer Name Length (since Netbios name was length 16 with last byte 0x20) + * No longer as important, now that TCP names are more commonly used to + * resolve hosts. + */ +#define CNLEN 15 + +/* + * Share Name Length (SNLEN) + * Note: This length was limited by the SMB used to get + * the Share info. NetShareEnum only returned 13 + * chars, including the null termination. + * This was removed because it no longer is limiting. + */ + +/* + * Comment Length + */ +#define MAXCOMMENTLEN 40 + +/* + * The OS/2 maximum path name + */ +#define MAX_PATHCONF 256 + +/* + * SMB frame definitions (following must be packed structs) + * See the SNIA CIFS Specification for details. + * + * The Naming convention is the lower case version of the + * smb command code name for the struct and this is typedef to the + * uppercase version of the same name with the prefix SMB_ removed + * for brevity. Although typedefs are not commonly used for + * structure definitions in the Linux kernel, their use in the + * CIFS standards document, which this code is based on, may + * make this one of the cases where typedefs for structures make + * sense to improve readability for readers of the standards doc. + * Typedefs can always be removed later if they are too distracting + * and they are only used for the CIFSs PDUs themselves, not + * internal cifs vfs structures + * + */ + +#define MIN_TZ_ADJ (15 * 60) /* minimum grid for timezones in seconds */ + +#define READ_RAW_ENABLE 1 +#define WRITE_RAW_ENABLE 2 +#define RAW_ENABLE (READ_RAW_ENABLE | WRITE_RAW_ENABLE) +#define SMB1_CLIENT_GUID_SIZE (16) + +/* See MS-CIFS 2.2.4.52.2 */ +typedef struct smb_negotiate_rsp { + struct smb_hdr hdr; /* wct = 17 */ + __le16 DialectIndex; /* 0xFFFF = no dialect acceptable */ + __u8 SecurityMode; + __le16 MaxMpxCount; + __le16 MaxNumberVcs; + __le32 MaxBufferSize; + __le32 MaxRawSize; + __le32 SessionKey; + __le32 Capabilities; /* see below */ + __le32 SystemTimeLow; + __le32 SystemTimeHigh; + __le16 ServerTimeZone; + __u8 EncryptionKeyLength; + __u16 ByteCount; + union { + /* cap extended security off */ + DECLARE_FLEX_ARRAY(unsigned char, EncryptionKey); + /* followed by Domain name - if extended security is off */ + /* followed by 16 bytes of server GUID */ + /* then security blob if cap_extended_security negotiated */ + struct { + unsigned char GUID[SMB1_CLIENT_GUID_SIZE]; + unsigned char SecurityBlob[]; + } __packed extended_response; + } __packed u; +} __packed SMB_NEGOTIATE_RSP; + +/* SecurityMode bits */ +#define SECMODE_USER 0x01 /* off indicates share level security */ +#define SECMODE_PW_ENCRYPT 0x02 +#define SECMODE_SIGN_ENABLED 0x04 /* SMB security signatures enabled */ +#define SECMODE_SIGN_REQUIRED 0x08 /* SMB security signatures required */ + +/* Negotiate response Capabilities */ +#define CAP_RAW_MODE 0x00000001 +#define CAP_MPX_MODE 0x00000002 +#define CAP_UNICODE 0x00000004 +#define CAP_LARGE_FILES 0x00000008 +#define CAP_NT_SMBS 0x00000010 /* implies CAP_NT_FIND */ +#define CAP_RPC_REMOTE_APIS 0x00000020 +#define CAP_STATUS32 0x00000040 +#define CAP_LEVEL_II_OPLOCKS 0x00000080 +#define CAP_LOCK_AND_READ 0x00000100 +#define CAP_NT_FIND 0x00000200 +#define CAP_DFS 0x00001000 +#define CAP_INFOLEVEL_PASSTHRU 0x00002000 +#define CAP_LARGE_READ_X 0x00004000 +#define CAP_LARGE_WRITE_X 0x00008000 +#define CAP_LWIO 0x00010000 /* support fctl_srv_req_resume_key */ +#define CAP_UNIX 0x00800000 +#define CAP_COMPRESSED_DATA 0x02000000 +#define CAP_DYNAMIC_REAUTH 0x20000000 +#define CAP_PERSISTENT_HANDLES 0x40000000 +#define CAP_EXTENDED_SECURITY 0x80000000 + +typedef union smb_com_session_setup_andx { + struct { /* request format */ + struct smb_hdr hdr; /* wct = 12 */ + __u8 AndXCommand; + __u8 AndXReserved; + __le16 AndXOffset; + __le16 MaxBufferSize; + __le16 MaxMpxCount; + __le16 VcNumber; + __le32 SessionKey; + __le16 SecurityBlobLength; + __u32 Reserved; + __le32 Capabilities; /* see below */ + __le16 ByteCount; + unsigned char SecurityBlob[]; /* followed by */ + /* STRING NativeOS */ + /* STRING NativeLanMan */ + } __packed req; /* NTLM request format (with + extended security */ + + struct { /* request format */ + struct smb_hdr hdr; /* wct = 13 */ + __u8 AndXCommand; + __u8 AndXReserved; + __le16 AndXOffset; + __le16 MaxBufferSize; + __le16 MaxMpxCount; + __le16 VcNumber; + __le32 SessionKey; + __le16 CaseInsensitivePasswordLength; /* ASCII password len */ + __le16 CaseSensitivePasswordLength; /* Unicode password length*/ + __u32 Reserved; /* see below */ + __le32 Capabilities; + __le16 ByteCount; + unsigned char CaseInsensitivePassword[]; /* followed by: */ + /* unsigned char * CaseSensitivePassword; */ + /* STRING AccountName */ + /* STRING PrimaryDomain */ + /* STRING NativeOS */ + /* STRING NativeLanMan */ + } __packed req_no_secext; /* NTLM request format (without + extended security */ + + struct { /* default (NTLM) response format */ + struct smb_hdr hdr; /* wct = 4 */ + __u8 AndXCommand; + __u8 AndXReserved; + __le16 AndXOffset; + __le16 Action; /* see below */ + __le16 SecurityBlobLength; + __u16 ByteCount; + unsigned char SecurityBlob[]; /* followed by */ +/* unsigned char * NativeOS; */ +/* unsigned char * NativeLanMan; */ +/* unsigned char * PrimaryDomain; */ + } __packed resp; /* NTLM response + (with or without extended sec) */ + + struct { /* request format */ + struct smb_hdr hdr; /* wct = 10 */ + __u8 AndXCommand; + __u8 AndXReserved; + __le16 AndXOffset; + __le16 MaxBufferSize; + __le16 MaxMpxCount; + __le16 VcNumber; + __le32 SessionKey; + __le16 PasswordLength; + __u32 Reserved; /* encrypt key len and offset */ + __le16 ByteCount; + unsigned char AccountPassword[]; /* followed by */ + /* STRING AccountName */ + /* STRING PrimaryDomain */ + /* STRING NativeOS */ + /* STRING NativeLanMan */ + } __packed old_req; /* pre-NTLM (LANMAN2.1) req format */ + + struct { /* default (NTLM) response format */ + struct smb_hdr hdr; /* wct = 3 */ + __u8 AndXCommand; + __u8 AndXReserved; + __le16 AndXOffset; + __le16 Action; /* see below */ + __u16 ByteCount; + unsigned char NativeOS[]; /* followed by */ +/* unsigned char * NativeLanMan; */ +/* unsigned char * PrimaryDomain; */ + } __packed old_resp; /* pre-NTLM (LANMAN2.1) response */ +} __packed SESSION_SETUP_ANDX; + +/* format of NLTMv2 Response ie "case sensitive password" hash when NTLMv2 */ + +#define NTLMSSP_SERVER_TYPE 1 +#define NTLMSSP_DOMAIN_TYPE 2 +#define NTLMSSP_FQ_DOMAIN_TYPE 3 +#define NTLMSSP_DNS_DOMAIN_TYPE 4 +#define NTLMSSP_DNS_PARENT_TYPE 5 + +struct ntlmssp2_name { + __le16 type; + __le16 length; + __u8 data[]; +} __packed; + +struct ntlmv2_resp { + union { + char ntlmv2_hash[CIFS_ENCPWD_SIZE]; + struct { + __u8 reserved[8]; + __u8 key[CIFS_SERVER_CHALLENGE_SIZE]; + } __packed challenge; + } __packed; + __le32 blob_signature; + __u32 reserved; + __le64 time; + __u64 client_chal; /* random */ + __u32 reserved2; + /* array of name entries could follow ending in minimum 4 byte struct */ +} __packed; + + +#define CIFS_NETWORK_OPSYS "CIFS VFS Client for Linux" + + +/* + * Capabilities bits (for NTLM SessSetup request) + * See MS-CIFS 2.2.4.52.2 + * MS-SMB 2.2.4.5.2.1 + */ +#define CAP_UNICODE 0x00000004 +#define CAP_LARGE_FILES 0x00000008 +#define CAP_NT_SMBS 0x00000010 +#define CAP_STATUS32 0x00000040 +#define CAP_LEVEL_II_OPLOCKS 0x00000080 +#define CAP_NT_FIND 0x00000200 /* reserved should be zero + (because NT_SMBs implies the same thing?) */ +#define CAP_BULK_TRANSFER 0x00000400 +#define CAP_EXTENDED_SECURITY 0x80000000 + +/* Action bits */ +#define GUEST_LOGIN 1 + +typedef struct smb_com_tconx_req { + struct smb_hdr hdr; /* wct = 4 */ + __u8 AndXCommand; + __u8 AndXReserved; + __le16 AndXOffset; + __le16 Flags; /* see below */ + __le16 PasswordLength; + __le16 ByteCount; + unsigned char Password[]; /* followed by */ +/* STRING Path *//* \\server\share name */ + /* STRING Service */ +} __packed TCONX_REQ; + +typedef struct smb_com_tconx_rsp { + struct smb_hdr hdr; /* wct = 3 , not extended response */ + __u8 AndXCommand; + __u8 AndXReserved; + __le16 AndXOffset; + __le16 OptionalSupport; /* see below */ + __u16 ByteCount; + unsigned char Service[]; /* always ASCII, not Unicode */ + /* STRING NativeFileSystem */ +} __packed TCONX_RSP; + +typedef struct smb_com_tconx_rsp_ext { + struct smb_hdr hdr; /* wct = 7, extended response */ + __u8 AndXCommand; + __u8 AndXReserved; + __le16 AndXOffset; + __le16 OptionalSupport; /* see below */ + __le32 MaximalShareAccessRights; + __le32 GuestMaximalShareAccessRights; + __u16 ByteCount; + unsigned char Service[]; /* always ASCII, not Unicode */ + /* STRING NativeFileSystem */ +} __packed TCONX_RSP_EXT; + + +/* tree connect Flags */ +#define DISCONNECT_TID 0x0001 +#define TCON_EXTENDED_SIGNATURES 0x0004 +#define TCON_EXTENDED_SECINFO 0x0008 + +/* OptionalSupport bits */ +#define SMB_SUPPORT_SEARCH_BITS 0x0001 /* "must have" directory search bits + (exclusive searches supported) */ +#define SMB_SHARE_IS_IN_DFS 0x0002 +#define SMB_CSC_MASK 0x000C +/* CSC flags defined as follows */ +#define SMB_CSC_CACHE_MANUAL_REINT 0x0000 +#define SMB_CSC_CACHE_AUTO_REINT 0x0004 +#define SMB_CSC_CACHE_VDO 0x0008 +#define SMB_CSC_NO_CACHING 0x000C +#define SMB_UNIQUE_FILE_NAME 0x0010 +#define SMB_EXTENDED_SIGNATURES 0x0020 + +/* services + * + * A: ie disk + * LPT1: ie printer + * IPC ie named pipe + * COMM + * ????? ie any type + * + */ + +typedef struct smb_com_echo_req { + struct smb_hdr hdr; + __le16 EchoCount; + __le16 ByteCount; + char Data[]; +} __packed ECHO_REQ; + +typedef struct smb_com_echo_rsp { + struct smb_hdr hdr; + __le16 SequenceNumber; + __le16 ByteCount; + char Data[]; +} __packed ECHO_RSP; + +typedef struct smb_com_logoff_andx_req { + struct smb_hdr hdr; /* wct = 2 */ + __u8 AndXCommand; + __u8 AndXReserved; + __u16 AndXOffset; + __u16 ByteCount; +} __packed LOGOFF_ANDX_REQ; + +typedef struct smb_com_logoff_andx_rsp { + struct smb_hdr hdr; /* wct = 2 */ + __u8 AndXCommand; + __u8 AndXReserved; + __u16 AndXOffset; + __u16 ByteCount; +} __packed LOGOFF_ANDX_RSP; + +typedef union smb_com_tree_disconnect { /* as an alternative can use flag on + tree_connect PDU to effect disconnect */ + /* tdis is probably simplest SMB PDU */ + struct { + struct smb_hdr hdr; /* wct = 0 */ + __u16 ByteCount; /* bcc = 0 */ + } __packed req; + struct { + struct smb_hdr hdr; /* wct = 0 */ + __u16 ByteCount; /* bcc = 0 */ + } __packed resp; +} __packed TREE_DISCONNECT; + +typedef struct smb_com_close_req { + struct smb_hdr hdr; /* wct = 3 */ + __u16 FileID; + __u32 LastWriteTime; /* should be zero or -1 */ + __u16 ByteCount; /* 0 */ +} __packed CLOSE_REQ; + +typedef struct smb_com_close_rsp { + struct smb_hdr hdr; /* wct = 0 */ + __u16 ByteCount; /* bct = 0 */ +} __packed CLOSE_RSP; + +typedef struct smb_com_flush_req { + struct smb_hdr hdr; /* wct = 1 */ + __u16 FileID; + __u16 ByteCount; /* 0 */ +} __packed FLUSH_REQ; + +typedef struct smb_com_findclose_req { + struct smb_hdr hdr; /* wct = 1 */ + __u16 FileID; + __u16 ByteCount; /* 0 */ +} __packed FINDCLOSE_REQ; + +/* OpenFlags */ +#define REQ_MORE_INFO 0x00000001 /* legacy (OPEN_AND_X) only */ +#define REQ_OPLOCK 0x00000002 +#define REQ_BATCHOPLOCK 0x00000004 +#define REQ_OPENDIRONLY 0x00000008 +#define REQ_EXTENDED_INFO 0x00000010 + +/* File type */ +#define DISK_TYPE 0x0000 +#define BYTE_PIPE_TYPE 0x0001 +#define MESSAGE_PIPE_TYPE 0x0002 +#define PRINTER_TYPE 0x0003 +#define COMM_DEV_TYPE 0x0004 +#define UNKNOWN_TYPE 0xFFFF + +/* Device Type or File Status Flags */ +#define NO_EAS 0x0001 +#define NO_SUBSTREAMS 0x0002 +#define NO_REPARSETAG 0x0004 +/* following flags can apply if pipe */ +#define ICOUNT_MASK 0x00FF +#define PIPE_READ_MODE 0x0100 +#define NAMED_PIPE_TYPE 0x0400 +#define PIPE_END_POINT 0x4000 +#define BLOCKING_NAMED_PIPE 0x8000 + +typedef struct smb_com_open_req { /* also handles create */ + struct smb_hdr hdr; /* wct = 24 */ + __u8 AndXCommand; + __u8 AndXReserved; + __le16 AndXOffset; + __u8 Reserved; /* Must Be Zero */ + __le16 NameLength; + __le32 OpenFlags; + __u32 RootDirectoryFid; + __le32 DesiredAccess; + __le64 AllocationSize; + __le32 FileAttributes; + __le32 ShareAccess; + __le32 CreateDisposition; + __le32 CreateOptions; + __le32 ImpersonationLevel; + __u8 SecurityFlags; + __le16 ByteCount; + char fileName[]; +} __packed OPEN_REQ; + +/* open response: oplock levels */ +#define OPLOCK_NONE 0 +#define OPLOCK_EXCLUSIVE 1 +#define OPLOCK_BATCH 2 +#define OPLOCK_READ 3 /* level 2 oplock */ + +/* open response for CreateAction shifted left */ +#define CIFS_CREATE_ACTION 0x20000 /* file created */ + +typedef struct smb_com_open_rsp { + struct smb_hdr hdr; /* wct = 34 BB */ + __u8 AndXCommand; + __u8 AndXReserved; + __le16 AndXOffset; + __u8 OplockLevel; + __u16 Fid; + __le32 CreateAction; + struct_group_attr(common_attributes, __packed, + __le64 CreationTime; + __le64 LastAccessTime; + __le64 LastWriteTime; + __le64 ChangeTime; + __le32 FileAttributes; + ); + __le64 AllocationSize; + __le64 EndOfFile; + __le16 FileType; + __le16 DeviceState; + __u8 DirectoryFlag; + __u16 ByteCount; /* bct = 0 */ +} __packed OPEN_RSP; + +typedef struct smb_com_open_rsp_ext { + struct smb_hdr hdr; /* wct = 42 but meaningless due to MS bug? */ + __u8 AndXCommand; + __u8 AndXReserved; + __le16 AndXOffset; + __u8 OplockLevel; + __u16 Fid; + __le32 CreateAction; + __le64 CreationTime; + __le64 LastAccessTime; + __le64 LastWriteTime; + __le64 ChangeTime; + __le32 FileAttributes; + __le64 AllocationSize; + __le64 EndOfFile; + __le16 FileType; + __le16 DeviceState; + __u8 DirectoryFlag; + __u8 VolumeGUID[16]; + __u64 FileId; /* note no endian conversion - is opaque UniqueID */ + __le32 MaximalAccessRights; + __le32 GuestMaximalAccessRights; + __u16 ByteCount; /* bct = 0 */ +} __packed OPEN_RSP_EXT; + + +/* format of legacy open request */ +typedef struct smb_com_openx_req { + struct smb_hdr hdr; /* wct = 15 */ + __u8 AndXCommand; + __u8 AndXReserved; + __le16 AndXOffset; + __le16 OpenFlags; + __le16 Mode; + __le16 Sattr; /* search attributes */ + __le16 FileAttributes; /* dos attrs */ + __le32 CreateTime; /* os2 format */ + __le16 OpenFunction; + __le32 EndOfFile; + __le32 Timeout; + __le32 Reserved; + __le16 ByteCount; /* file name follows */ + char fileName[]; +} __packed OPENX_REQ; + +typedef struct smb_com_openx_rsp { + struct smb_hdr hdr; /* wct = 15 */ + __u8 AndXCommand; + __u8 AndXReserved; + __le16 AndXOffset; + __u16 Fid; + __le16 FileAttributes; + __le32 LastWriteTime; /* os2 format */ + __le32 EndOfFile; + __le16 Access; + __le16 FileType; + __le16 IPCState; + __le16 Action; + __u32 FileId; + __u16 Reserved; + __u16 ByteCount; +} __packed OPENX_RSP; + +/* For encoding of POSIX Open Request - see trans2 function 0x209 data struct */ + +/* Legacy write request for older servers */ +typedef struct smb_com_writex_req { + struct smb_hdr hdr; /* wct = 12 */ + __u8 AndXCommand; + __u8 AndXReserved; + __le16 AndXOffset; + __u16 Fid; + __le32 OffsetLow; + __u32 Reserved; /* Timeout */ + __le16 WriteMode; /* 1 = write through */ + __le16 Remaining; + __le16 Reserved2; + __le16 DataLengthLow; + __le16 DataOffset; + __le16 ByteCount; + __u8 Pad; /* BB check for whether padded to DWORD + boundary and optimum performance here */ + char Data[]; +} __packed WRITEX_REQ; + +typedef struct smb_com_write_req { + struct smb_hdr hdr; /* wct = 14 */ + __u8 AndXCommand; + __u8 AndXReserved; + __le16 AndXOffset; + __u16 Fid; + __le32 OffsetLow; + __u32 Reserved; + __le16 WriteMode; + __le16 Remaining; + __le16 DataLengthHigh; + __le16 DataLengthLow; + __le16 DataOffset; + __le32 OffsetHigh; + __le16 ByteCount; + __u8 Pad; /* BB check for whether padded to DWORD + boundary and optimum performance here */ + char Data[]; +} __packed WRITE_REQ; + +typedef struct smb_com_write_rsp { + struct smb_hdr hdr; /* wct = 6 */ + __u8 AndXCommand; + __u8 AndXReserved; + __le16 AndXOffset; + __le16 Count; + __le16 Remaining; + __le16 CountHigh; + __u16 Reserved; + __u16 ByteCount; +} __packed WRITE_RSP; + +/* legacy read request for older servers */ +typedef struct smb_com_readx_req { + struct smb_hdr hdr; /* wct = 10 */ + __u8 AndXCommand; + __u8 AndXReserved; + __le16 AndXOffset; + __u16 Fid; + __le32 OffsetLow; + __le16 MaxCount; + __le16 MinCount; /* obsolete */ + __le32 Reserved; + __le16 Remaining; + __le16 ByteCount; +} __packed READX_REQ; + +typedef struct smb_com_read_req { + struct smb_hdr hdr; /* wct = 12 */ + __u8 AndXCommand; + __u8 AndXReserved; + __le16 AndXOffset; + __u16 Fid; + __le32 OffsetLow; + __le16 MaxCount; + __le16 MinCount; /* obsolete */ + __le32 MaxCountHigh; + __le16 Remaining; + __le32 OffsetHigh; + __le16 ByteCount; +} __packed READ_REQ; + +typedef struct smb_com_read_rsp { + struct smb_hdr hdr; /* wct = 12 */ + __u8 AndXCommand; + __u8 AndXReserved; + __le16 AndXOffset; + __le16 Remaining; + __le16 DataCompactionMode; + __le16 Reserved; + __le16 DataLength; + __le16 DataOffset; + __le16 DataLengthHigh; + __u64 Reserved2; + __u16 ByteCount; + /* read response data immediately follows */ +} __packed READ_RSP; + +typedef struct locking_andx_range { + __le16 Pid; + __le16 Pad; + __le32 OffsetHigh; + __le32 OffsetLow; + __le32 LengthHigh; + __le32 LengthLow; +} __packed LOCKING_ANDX_RANGE; + +#define LOCKING_ANDX_SHARED_LOCK 0x01 +#define LOCKING_ANDX_OPLOCK_RELEASE 0x02 +#define LOCKING_ANDX_CHANGE_LOCKTYPE 0x04 +#define LOCKING_ANDX_CANCEL_LOCK 0x08 +#define LOCKING_ANDX_LARGE_FILES 0x10 /* always on for us */ + +typedef struct smb_com_lock_req { + struct smb_hdr hdr; /* wct = 8 */ + __u8 AndXCommand; + __u8 AndXReserved; + __le16 AndXOffset; + __u16 Fid; + __u8 LockType; + __u8 OplockLevel; + __le32 Timeout; + __le16 NumberOfUnlocks; + __le16 NumberOfLocks; + __le16 ByteCount; + LOCKING_ANDX_RANGE Locks[]; +} __packed LOCK_REQ; + +/* lock type */ +#define CIFS_RDLCK 0 +#define CIFS_WRLCK 1 +#define CIFS_UNLCK 2 +typedef struct cifs_posix_lock { + __le16 lock_type; /* 0 = Read, 1 = Write, 2 = Unlock */ + __le16 lock_flags; /* 1 = Wait (only valid for setlock) */ + __le32 pid; + __le64 start; + __le64 length; + /* BB what about additional owner info to identify network client */ +} __packed CIFS_POSIX_LOCK; + +typedef struct smb_com_lock_rsp { + struct smb_hdr hdr; /* wct = 2 */ + __u8 AndXCommand; + __u8 AndXReserved; + __le16 AndXOffset; + __u16 ByteCount; +} __packed LOCK_RSP; + +typedef struct smb_com_rename_req { + struct smb_hdr hdr; /* wct = 1 */ + __le16 SearchAttributes; /* target file attributes */ + __le16 ByteCount; + __u8 BufferFormat; /* 4 = ASCII or Unicode */ + unsigned char OldFileName[]; + /* followed by __u8 BufferFormat2 */ + /* followed by NewFileName */ +} __packed RENAME_REQ; + + /* copy request flags */ +#define COPY_MUST_BE_FILE 0x0001 +#define COPY_MUST_BE_DIR 0x0002 +#define COPY_TARGET_MODE_ASCII 0x0004 /* if not set, binary */ +#define COPY_SOURCE_MODE_ASCII 0x0008 /* if not set, binary */ +#define COPY_VERIFY_WRITES 0x0010 +#define COPY_TREE 0x0020 + +typedef struct smb_com_copy_req { + struct smb_hdr hdr; /* wct = 3 */ + __u16 Tid2; + __le16 OpenFunction; + __le16 Flags; + __le16 ByteCount; + __u8 BufferFormat; /* 4 = ASCII or Unicode */ + unsigned char OldFileName[]; + /* followed by __u8 BufferFormat2 */ + /* followed by NewFileName string */ +} __packed COPY_REQ; + +typedef struct smb_com_copy_rsp { + struct smb_hdr hdr; /* wct = 1 */ + __le16 CopyCount; /* number of files copied */ + __u16 ByteCount; /* may be zero */ + __u8 BufferFormat; /* 0x04 - only present if errored file follows */ + unsigned char ErrorFileName[]; /* only present if error in copy */ +} __packed COPY_RSP; + +#define CREATE_HARD_LINK 0x103 +#define MOVEFILE_COPY_ALLOWED 0x0002 +#define MOVEFILE_REPLACE_EXISTING 0x0001 + +typedef struct smb_com_nt_rename_req { /* A5 - also used for create hardlink */ + struct smb_hdr hdr; /* wct = 4 */ + __le16 SearchAttributes; /* target file attributes */ + __le16 Flags; /* spec says Information Level */ + __le32 ClusterCount; + __le16 ByteCount; + __u8 BufferFormat; /* 4 = ASCII or Unicode */ + unsigned char OldFileName[]; + /* followed by __u8 BufferFormat2 */ + /* followed by NewFileName */ +} __packed NT_RENAME_REQ; + +typedef struct smb_com_rename_rsp { + struct smb_hdr hdr; /* wct = 0 */ + __u16 ByteCount; /* bct = 0 */ +} __packed RENAME_RSP; + +typedef struct smb_com_delete_file_req { + struct smb_hdr hdr; /* wct = 1 */ + __le16 SearchAttributes; + __le16 ByteCount; + __u8 BufferFormat; /* 4 = ASCII */ + unsigned char fileName[]; +} __packed DELETE_FILE_REQ; + +typedef struct smb_com_delete_file_rsp { + struct smb_hdr hdr; /* wct = 0 */ + __u16 ByteCount; /* bct = 0 */ +} __packed DELETE_FILE_RSP; + +typedef struct smb_com_delete_directory_req { + struct smb_hdr hdr; /* wct = 0 */ + __le16 ByteCount; + __u8 BufferFormat; /* 4 = ASCII */ + unsigned char DirName[]; +} __packed DELETE_DIRECTORY_REQ; + +typedef struct smb_com_delete_directory_rsp { + struct smb_hdr hdr; /* wct = 0 */ + __u16 ByteCount; /* bct = 0 */ +} __packed DELETE_DIRECTORY_RSP; + +typedef struct smb_com_create_directory_req { + struct smb_hdr hdr; /* wct = 0 */ + __le16 ByteCount; + __u8 BufferFormat; /* 4 = ASCII */ + unsigned char DirName[]; +} __packed CREATE_DIRECTORY_REQ; + +typedef struct smb_com_create_directory_rsp { + struct smb_hdr hdr; /* wct = 0 */ + __u16 ByteCount; /* bct = 0 */ +} __packed CREATE_DIRECTORY_RSP; + +typedef struct smb_com_query_information_req { + struct smb_hdr hdr; /* wct = 0 */ + __le16 ByteCount; /* 1 + namelen + 1 */ + __u8 BufferFormat; /* 4 = ASCII */ + unsigned char FileName[]; +} __packed QUERY_INFORMATION_REQ; + +typedef struct smb_com_query_information_rsp { + struct smb_hdr hdr; /* wct = 10 */ + __le16 attr; + __le32 last_write_time; + __le32 size; + __u16 reserved[5]; + __le16 ByteCount; /* bcc = 0 */ +} __packed QUERY_INFORMATION_RSP; + +typedef struct smb_com_setattr_req { + struct smb_hdr hdr; /* wct = 8 */ + __le16 attr; + __le32 last_write_time; + __le16 reserved[5]; /* must be zero */ + __le16 ByteCount; + __u8 BufferFormat; /* 4 = ASCII */ + unsigned char fileName[]; +} __packed SETATTR_REQ; + +typedef struct smb_com_setattr_rsp { + struct smb_hdr hdr; /* wct = 0 */ + __u16 ByteCount; /* bct = 0 */ +} __packed SETATTR_RSP; + +/* empty wct response to setattr */ + +/*******************************************************/ +/* NT Transact structure definitions follow */ +/* Currently only ioctl, acl (get security descriptor) */ +/* and notify are implemented */ +/*******************************************************/ +typedef struct smb_com_ntransact_req { + struct smb_hdr hdr; /* wct >= 19 */ + __u8 MaxSetupCount; + __u16 Reserved; + __le32 TotalParameterCount; + __le32 TotalDataCount; + __le32 MaxParameterCount; + __le32 MaxDataCount; + __le32 ParameterCount; + __le32 ParameterOffset; + __le32 DataCount; + __le32 DataOffset; + __u8 SetupCount; /* four setup words follow subcommand */ + /* SNIA spec incorrectly included spurious pad here */ + __le16 SubCommand; /* 2 = IOCTL/FSCTL */ + /* SetupCount words follow then */ + __le16 ByteCount; + __u8 Pad[3]; + __u8 Parms[]; +} __packed NTRANSACT_REQ; + +typedef struct smb_com_ntransact_rsp { + struct smb_hdr hdr; /* wct = 18 */ + __u8 Reserved[3]; + __le32 TotalParameterCount; + __le32 TotalDataCount; + __le32 ParameterCount; + __le32 ParameterOffset; + __le32 ParameterDisplacement; + __le32 DataCount; + __le32 DataOffset; + __le32 DataDisplacement; + __u8 SetupCount; /* 0 */ + __u16 ByteCount; + /* __u8 Pad[3]; */ + /* parms and data follow */ +} __packed NTRANSACT_RSP; + +typedef struct smb_com_transaction_ioctl_req { + struct smb_hdr hdr; /* wct = 23 */ + __u8 MaxSetupCount; + __u16 Reserved; + __le32 TotalParameterCount; + __le32 TotalDataCount; + __le32 MaxParameterCount; + __le32 MaxDataCount; + __le32 ParameterCount; + __le32 ParameterOffset; + __le32 DataCount; + __le32 DataOffset; + __u8 SetupCount; /* four setup words follow subcommand */ + /* SNIA spec incorrectly included spurious pad here */ + __le16 SubCommand; /* 2 = IOCTL/FSCTL */ + __le32 FunctionCode; + __u16 Fid; + __u8 IsFsctl; /* 1 = File System Control 0 = device control (IOCTL) */ + __u8 IsRootFlag; /* 1 = apply command to root of share (must be DFS) */ + __le16 ByteCount; + __u8 Pad[3]; + __u8 Data[]; +} __packed TRANSACT_IOCTL_REQ; + +typedef struct smb_com_transaction_compr_ioctl_req { + struct smb_hdr hdr; /* wct = 23 */ + __u8 MaxSetupCount; + __u16 Reserved; + __le32 TotalParameterCount; + __le32 TotalDataCount; + __le32 MaxParameterCount; + __le32 MaxDataCount; + __le32 ParameterCount; + __le32 ParameterOffset; + __le32 DataCount; + __le32 DataOffset; + __u8 SetupCount; /* four setup words follow subcommand */ + /* SNIA spec incorrectly included spurious pad here */ + __le16 SubCommand; /* 2 = IOCTL/FSCTL */ + __le32 FunctionCode; + __u16 Fid; + __u8 IsFsctl; /* 1 = File System Control 0 = device control (IOCTL) */ + __u8 IsRootFlag; /* 1 = apply command to root of share (must be DFS) */ + __le16 ByteCount; + __u8 Pad[3]; + __le16 compression_state; /* See below for valid flags */ +} __packed TRANSACT_COMPR_IOCTL_REQ; + +typedef struct smb_com_transaction_ioctl_rsp { + struct smb_hdr hdr; /* wct = 19 */ + __u8 Reserved[3]; + __le32 TotalParameterCount; + __le32 TotalDataCount; + __le32 ParameterCount; + __le32 ParameterOffset; + __le32 ParameterDisplacement; + __le32 DataCount; + __le32 DataOffset; + __le32 DataDisplacement; + __u8 SetupCount; /* 1 */ + __le16 ReturnedDataLen; + __le16 ByteCount; +} __packed TRANSACT_IOCTL_RSP; + +#define CIFS_ACL_OWNER 1 +#define CIFS_ACL_GROUP 2 +#define CIFS_ACL_DACL 4 +#define CIFS_ACL_SACL 8 + +typedef struct smb_com_transaction_qsec_req { + struct smb_hdr hdr; /* wct = 19 */ + __u8 MaxSetupCount; + __u16 Reserved; + __le32 TotalParameterCount; + __le32 TotalDataCount; + __le32 MaxParameterCount; + __le32 MaxDataCount; + __le32 ParameterCount; + __le32 ParameterOffset; + __le32 DataCount; + __le32 DataOffset; + __u8 SetupCount; /* no setup words follow subcommand */ + /* SNIA spec incorrectly included spurious pad here */ + __le16 SubCommand; /* 6 = QUERY_SECURITY_DESC */ + __le16 ByteCount; /* bcc = 3 + 8 */ + __u8 Pad[3]; + __u16 Fid; + __u16 Reserved2; + __le32 AclFlags; +} __packed QUERY_SEC_DESC_REQ; + + +typedef struct smb_com_transaction_ssec_req { + struct smb_hdr hdr; /* wct = 19 */ + __u8 MaxSetupCount; + __u16 Reserved; + __le32 TotalParameterCount; + __le32 TotalDataCount; + __le32 MaxParameterCount; + __le32 MaxDataCount; + __le32 ParameterCount; + __le32 ParameterOffset; + __le32 DataCount; + __le32 DataOffset; + __u8 SetupCount; /* no setup words follow subcommand */ + /* SNIA spec incorrectly included spurious pad here */ + __le16 SubCommand; /* 3 = SET_SECURITY_DESC */ + __le16 ByteCount; /* bcc = 3 + 8 */ + __u8 Pad[3]; + __u16 Fid; + __u16 Reserved2; + __le32 AclFlags; +} __packed SET_SEC_DESC_REQ; + +typedef struct smb_com_transaction_change_notify_req { + struct smb_hdr hdr; /* wct = 23 */ + __u8 MaxSetupCount; + __u16 Reserved; + __le32 TotalParameterCount; + __le32 TotalDataCount; + __le32 MaxParameterCount; + __le32 MaxDataCount; + __le32 ParameterCount; + __le32 ParameterOffset; + __le32 DataCount; + __le32 DataOffset; + __u8 SetupCount; /* four setup words follow subcommand */ + /* SNIA spec incorrectly included spurious pad here */ + __le16 SubCommand;/* 4 = Change Notify */ + __le32 CompletionFilter; /* operation to monitor */ + __u16 Fid; + __u8 WatchTree; /* 1 = Monitor subdirectories */ + __u8 Reserved2; + __le16 ByteCount; +/* __u8 Pad[3];*/ +/* __u8 Data[];*/ +} __packed TRANSACT_CHANGE_NOTIFY_REQ; + +/* BB eventually change to use generic ntransact rsp struct + and validation routine */ +typedef struct smb_com_transaction_change_notify_rsp { + struct smb_hdr hdr; /* wct = 18 */ + __u8 Reserved[3]; + __le32 TotalParameterCount; + __le32 TotalDataCount; + __le32 ParameterCount; + __le32 ParameterOffset; + __le32 ParameterDisplacement; + __le32 DataCount; + __le32 DataOffset; + __le32 DataDisplacement; + __u8 SetupCount; /* 0 */ + __u16 ByteCount; + /* __u8 Pad[3]; */ +} __packed TRANSACT_CHANGE_NOTIFY_RSP; + +struct cifs_quota_data { + __u32 rsrvd1; /* 0 */ + __u32 sid_size; + __u64 rsrvd2; /* 0 */ + __u64 space_used; + __u64 soft_limit; + __u64 hard_limit; + char sid[]; /* variable size? */ +} __packed; + +/* quota sub commands */ +#define QUOTA_LIST_CONTINUE 0 +#define QUOTA_LIST_START 0x100 +#define QUOTA_FOR_SID 0x101 + +struct trans2_req { + /* struct smb_hdr hdr precedes. Set wct = 14+ */ + __le16 TotalParameterCount; + __le16 TotalDataCount; + __le16 MaxParameterCount; + __le16 MaxDataCount; + __u8 MaxSetupCount; + __u8 Reserved; + __le16 Flags; + __le32 Timeout; + __u16 Reserved2; + __le16 ParameterCount; + __le16 ParameterOffset; + __le16 DataCount; + __le16 DataOffset; + __u8 SetupCount; + __u8 Reserved3; + __le16 SubCommand; /* 1st setup word - SetupCount words follow */ + __le16 ByteCount; +} __packed; + +struct smb_t2_req { + struct smb_hdr hdr; + struct trans2_req t2_req; +} __packed; + +struct trans2_resp { + /* struct smb_hdr hdr precedes. Note wct = 10 + setup count */ + __le16 TotalParameterCount; + __le16 TotalDataCount; + __u16 Reserved; + __le16 ParameterCount; + __le16 ParameterOffset; + __le16 ParameterDisplacement; + __le16 DataCount; + __le16 DataOffset; + __le16 DataDisplacement; + __u8 SetupCount; + __u8 Reserved1; + /* SetupWords[SetupCount]; + __u16 ByteCount; + __u16 Reserved2;*/ + /* data area follows */ +} __packed; + +struct smb_t2_rsp { + struct smb_hdr hdr; + struct trans2_resp t2_rsp; +} __packed; + +/* PathInfo/FileInfo infolevels */ +#define SMB_INFO_STANDARD 1 +#define SMB_SET_FILE_EA 2 +#define SMB_QUERY_FILE_EA_SIZE 2 +#define SMB_INFO_QUERY_EAS_FROM_LIST 3 +#define SMB_INFO_QUERY_ALL_EAS 4 +#define SMB_INFO_IS_NAME_VALID 6 +#define SMB_QUERY_FILE_BASIC_INFO 0x101 +#define SMB_QUERY_FILE_STANDARD_INFO 0x102 +#define SMB_QUERY_FILE_EA_INFO 0x103 +#define SMB_QUERY_FILE_NAME_INFO 0x104 +#define SMB_QUERY_FILE_ALLOCATION_INFO 0x105 +#define SMB_QUERY_FILE_END_OF_FILEINFO 0x106 +#define SMB_QUERY_FILE_ALL_INFO 0x107 +#define SMB_QUERY_ALT_NAME_INFO 0x108 +#define SMB_QUERY_FILE_STREAM_INFO 0x109 +#define SMB_QUERY_FILE_COMPRESSION_INFO 0x10B +#define SMB_QUERY_FILE_UNIX_BASIC 0x200 +#define SMB_QUERY_FILE_UNIX_LINK 0x201 +#define SMB_QUERY_POSIX_ACL 0x204 +#define SMB_QUERY_XATTR 0x205 /* e.g. system EA name space */ +#define SMB_QUERY_ATTR_FLAGS 0x206 /* append,immutable etc. */ +#define SMB_QUERY_POSIX_PERMISSION 0x207 +#define SMB_QUERY_POSIX_LOCK 0x208 +/* #define SMB_POSIX_OPEN 0x209 */ +/* #define SMB_POSIX_UNLINK 0x20a */ +#define SMB_QUERY_FILE__UNIX_INFO2 0x20b +#define SMB_QUERY_FILE_INTERNAL_INFO 0x3ee +#define SMB_QUERY_FILE_ACCESS_INFO 0x3f0 +#define SMB_QUERY_FILE_NAME_INFO2 0x3f1 /* 0x30 bytes */ +#define SMB_QUERY_FILE_POSITION_INFO 0x3f6 +#define SMB_QUERY_FILE_MODE_INFO 0x3f8 +#define SMB_QUERY_FILE_ALGN_INFO 0x3f9 + + +#define SMB_SET_FILE_BASIC_INFO 0x101 +#define SMB_SET_FILE_DISPOSITION_INFO 0x102 +#define SMB_SET_FILE_ALLOCATION_INFO 0x103 +#define SMB_SET_FILE_END_OF_FILE_INFO 0x104 +#define SMB_SET_FILE_UNIX_BASIC 0x200 +#define SMB_SET_FILE_UNIX_LINK 0x201 +#define SMB_SET_FILE_UNIX_HLINK 0x203 +#define SMB_SET_POSIX_ACL 0x204 +#define SMB_SET_XATTR 0x205 +#define SMB_SET_ATTR_FLAGS 0x206 /* append, immutable etc. */ +#define SMB_SET_POSIX_LOCK 0x208 +#define SMB_POSIX_OPEN 0x209 +#define SMB_POSIX_UNLINK 0x20a +#define SMB_SET_FILE_UNIX_INFO2 0x20b +#define SMB_SET_FILE_BASIC_INFO2 0x3ec +#define SMB_SET_FILE_RENAME_INFORMATION 0x3f2 /* BB check if qpathinfo too */ +#define SMB_FILE_ALL_INFO2 0x3fa +#define SMB_SET_FILE_ALLOCATION_INFO2 0x3fb +#define SMB_SET_FILE_END_OF_FILE_INFO2 0x3fc +#define SMB_FILE_MOVE_CLUSTER_INFO 0x407 +#define SMB_FILE_QUOTA_INFO 0x408 +#define SMB_FILE_REPARSEPOINT_INFO 0x409 +#define SMB_FILE_MAXIMUM_INFO 0x40d + +/* Find File infolevels */ +#define SMB_FIND_FILE_INFO_STANDARD 0x001 +#define SMB_FIND_FILE_QUERY_EA_SIZE 0x002 +#define SMB_FIND_FILE_QUERY_EAS_FROM_LIST 0x003 +#define SMB_FIND_FILE_DIRECTORY_INFO 0x101 +#define SMB_FIND_FILE_FULL_DIRECTORY_INFO 0x102 +#define SMB_FIND_FILE_NAMES_INFO 0x103 +#define SMB_FIND_FILE_BOTH_DIRECTORY_INFO 0x104 +#define SMB_FIND_FILE_ID_FULL_DIR_INFO 0x105 +#define SMB_FIND_FILE_ID_BOTH_DIR_INFO 0x106 +#define SMB_FIND_FILE_UNIX 0x202 +/* #define SMB_FIND_FILE_POSIX_INFO 0x064 */ + +typedef struct smb_com_transaction2_qpi_req { + struct smb_hdr hdr; /* wct = 14+ */ + __le16 TotalParameterCount; + __le16 TotalDataCount; + __le16 MaxParameterCount; + __le16 MaxDataCount; + __u8 MaxSetupCount; + __u8 Reserved; + __le16 Flags; + __le32 Timeout; + __u16 Reserved2; + __le16 ParameterCount; + __le16 ParameterOffset; + __le16 DataCount; + __le16 DataOffset; + __u8 SetupCount; + __u8 Reserved3; + __le16 SubCommand; /* one setup word */ + __le16 ByteCount; + __u8 Pad; + __le16 InformationLevel; + __u32 Reserved4; + char FileName[]; +} __packed TRANSACTION2_QPI_REQ; + +typedef struct smb_com_transaction2_qpi_rsp { + struct smb_hdr hdr; /* wct = 10 + SetupCount */ + struct trans2_resp t2; + __u16 ByteCount; + __u16 Reserved2; /* parameter word is present for infolevels > 100 */ +} __packed TRANSACTION2_QPI_RSP; + +typedef struct smb_com_transaction2_spi_req { + struct smb_hdr hdr; /* wct = 15 */ + __le16 TotalParameterCount; + __le16 TotalDataCount; + __le16 MaxParameterCount; + __le16 MaxDataCount; + __u8 MaxSetupCount; + __u8 Reserved; + __le16 Flags; + __le32 Timeout; + __u16 Reserved2; + __le16 ParameterCount; + __le16 ParameterOffset; + __le16 DataCount; + __le16 DataOffset; + __u8 SetupCount; + __u8 Reserved3; + __le16 SubCommand; /* one setup word */ + __le16 ByteCount; + __u8 Pad; + __u16 Pad1; + __le16 InformationLevel; + __u32 Reserved4; + char FileName[]; +} __packed TRANSACTION2_SPI_REQ; + +typedef struct smb_com_transaction2_spi_rsp { + struct smb_hdr hdr; /* wct = 10 + SetupCount */ + struct trans2_resp t2; + __u16 ByteCount; + __u16 Reserved2; /* parameter word is present for infolevels > 100 */ +} __packed TRANSACTION2_SPI_RSP; + +struct set_file_rename { + __le32 overwrite; /* 1 = overwrite dest */ + __u32 root_fid; /* zero */ + __le32 target_name_len; + char target_name[]; /* Must be unicode */ +} __packed; + +struct smb_com_transaction2_sfi_req { + struct smb_hdr hdr; /* wct = 15 */ + __le16 TotalParameterCount; + __le16 TotalDataCount; + __le16 MaxParameterCount; + __le16 MaxDataCount; + __u8 MaxSetupCount; + __u8 Reserved; + __le16 Flags; + __le32 Timeout; + __u16 Reserved2; + __le16 ParameterCount; + __le16 ParameterOffset; + __le16 DataCount; + __le16 DataOffset; + __u8 SetupCount; + __u8 Reserved3; + __le16 SubCommand; /* one setup word */ + __le16 ByteCount; + __u8 Pad; + __u16 Pad1; + __u16 Fid; + __le16 InformationLevel; + __u16 Reserved4; + __u8 payload[]; +} __packed; + +struct smb_com_transaction2_sfi_rsp { + struct smb_hdr hdr; /* wct = 10 + SetupCount */ + struct trans2_resp t2; + __u16 ByteCount; + __u16 Reserved2; /* parameter word reserved - present for infolevels > 100 */ +} __packed; + +struct smb_t2_qfi_req { + struct smb_hdr hdr; + struct trans2_req t2; + __u8 Pad; + __u16 Fid; + __le16 InformationLevel; +} __packed; + +struct smb_t2_qfi_rsp { + struct smb_hdr hdr; /* wct = 10 + SetupCount */ + struct trans2_resp t2; + __u16 ByteCount; + __u16 Reserved2; /* parameter word reserved - present for infolevels > 100 */ +} __packed; + +/* + * Flags on T2 FINDFIRST and FINDNEXT + */ +#define CIFS_SEARCH_CLOSE_ALWAYS 0x0001 +#define CIFS_SEARCH_CLOSE_AT_END 0x0002 +#define CIFS_SEARCH_RETURN_RESUME 0x0004 +#define CIFS_SEARCH_CONTINUE_FROM_LAST 0x0008 +#define CIFS_SEARCH_BACKUP_SEARCH 0x0010 + +/* + * Size of the resume key on FINDFIRST and FINDNEXT calls + */ +#define CIFS_SMB_RESUME_KEY_SIZE 4 + +typedef struct smb_com_transaction2_ffirst_req { + struct smb_hdr hdr; /* wct = 15 */ + __le16 TotalParameterCount; + __le16 TotalDataCount; + __le16 MaxParameterCount; + __le16 MaxDataCount; + __u8 MaxSetupCount; + __u8 Reserved; + __le16 Flags; + __le32 Timeout; + __u16 Reserved2; + __le16 ParameterCount; + __le16 ParameterOffset; + __le16 DataCount; + __le16 DataOffset; + __u8 SetupCount; /* one */ + __u8 Reserved3; + __le16 SubCommand; /* TRANS2_FIND_FIRST */ + __le16 ByteCount; + __u8 Pad; + __le16 SearchAttributes; + __le16 SearchCount; + __le16 SearchFlags; + __le16 InformationLevel; + __le32 SearchStorageType; + char FileName[]; +} __packed TRANSACTION2_FFIRST_REQ; + +typedef struct smb_com_transaction2_ffirst_rsp { + struct smb_hdr hdr; /* wct = 10 */ + struct trans2_resp t2; + __u16 ByteCount; +} __packed TRANSACTION2_FFIRST_RSP; + +typedef struct smb_com_transaction2_ffirst_rsp_parms { + __u16 SearchHandle; + __le16 SearchCount; + __le16 EndofSearch; + __le16 EAErrorOffset; + __le16 LastNameOffset; +} __packed T2_FFIRST_RSP_PARMS; + +typedef struct smb_com_transaction2_fnext_req { + struct smb_hdr hdr; /* wct = 15 */ + __le16 TotalParameterCount; + __le16 TotalDataCount; + __le16 MaxParameterCount; + __le16 MaxDataCount; + __u8 MaxSetupCount; + __u8 Reserved; + __le16 Flags; + __le32 Timeout; + __u16 Reserved2; + __le16 ParameterCount; + __le16 ParameterOffset; + __le16 DataCount; + __le16 DataOffset; + __u8 SetupCount; /* one */ + __u8 Reserved3; + __le16 SubCommand; /* TRANS2_FIND_NEXT */ + __le16 ByteCount; + __u8 Pad; + __u16 SearchHandle; + __le16 SearchCount; + __le16 InformationLevel; + __u32 ResumeKey; + __le16 SearchFlags; + char ResumeFileName[]; +} __packed TRANSACTION2_FNEXT_REQ; + +typedef struct smb_com_transaction2_fnext_rsp { + struct smb_hdr hdr; /* wct = 10 */ + struct trans2_resp t2; + __u16 ByteCount; +} __packed TRANSACTION2_FNEXT_RSP; + +typedef struct smb_com_transaction2_fnext_rsp_parms { + __le16 SearchCount; + __le16 EndofSearch; + __le16 EAErrorOffset; + __le16 LastNameOffset; +} __packed T2_FNEXT_RSP_PARMS; + +/* QFSInfo Levels */ +#define SMB_INFO_ALLOCATION 1 +#define SMB_INFO_VOLUME 2 +#define SMB_QUERY_FS_VOLUME_INFO 0x102 +#define SMB_QUERY_FS_SIZE_INFO 0x103 +#define SMB_QUERY_FS_DEVICE_INFO 0x104 +#define SMB_QUERY_FS_ATTRIBUTE_INFO 0x105 +#define SMB_QUERY_CIFS_UNIX_INFO 0x200 +#define SMB_QUERY_POSIX_FS_INFO 0x201 +#define SMB_QUERY_POSIX_WHO_AM_I 0x202 +#define SMB_REQUEST_TRANSPORT_ENCRYPTION 0x203 +#define SMB_QUERY_FS_PROXY 0x204 /* WAFS enabled. Returns structure + FILE_SYSTEM__UNIX_INFO to tell + whether new NTIOCTL available + (0xACE) for WAN friendly SMB + operations to be carried */ +#define SMB_QUERY_LABEL_INFO 0x3ea +#define SMB_QUERY_FS_QUOTA_INFO 0x3ee +#define SMB_QUERY_FS_FULL_SIZE_INFO 0x3ef +#define SMB_QUERY_OBJECTID_INFO 0x3f0 + +typedef struct smb_com_transaction2_qfsi_req { + struct smb_hdr hdr; /* wct = 14+ */ + __le16 TotalParameterCount; + __le16 TotalDataCount; + __le16 MaxParameterCount; + __le16 MaxDataCount; + __u8 MaxSetupCount; + __u8 Reserved; + __le16 Flags; + __le32 Timeout; + __u16 Reserved2; + __le16 ParameterCount; + __le16 ParameterOffset; + __le16 DataCount; + __le16 DataOffset; + __u8 SetupCount; + __u8 Reserved3; + __le16 SubCommand; /* one setup word */ + __le16 ByteCount; + __u8 Pad; + __le16 InformationLevel; +} __packed TRANSACTION2_QFSI_REQ; + +typedef struct smb_com_transaction_qfsi_rsp { + struct smb_hdr hdr; /* wct = 10 + SetupCount */ + struct trans2_resp t2; + __u16 ByteCount; + __u8 Pad; /* may be three bytes? *//* followed by data area */ +} __packed TRANSACTION2_QFSI_RSP; + +typedef struct whoami_rsp_data { /* Query level 0x202 */ + __u32 flags; /* 0 = Authenticated user 1 = GUEST */ + __u32 mask; /* which flags bits server understands ie 0x0001 */ + __u64 unix_user_id; + __u64 unix_user_gid; + __u32 number_of_supplementary_gids; /* may be zero */ + __u32 number_of_sids; /* may be zero */ + __u32 length_of_sid_array; /* in bytes - may be zero */ + __u32 pad; /* reserved - MBZ */ + /* __u64 gid_array[0]; */ /* may be empty */ + /* __u8 * psid_list */ /* may be empty */ +} __packed WHOAMI_RSP_DATA; + +/* SETFSInfo Levels */ +#define SMB_SET_CIFS_UNIX_INFO 0x200 +/* level 0x203 is defined above in list of QFS info levels */ +/* #define SMB_REQUEST_TRANSPORT_ENCRYPTION 0x203 */ + +/* Level 0x200 request structure follows */ +typedef struct smb_com_transaction2_setfsi_req { + struct smb_hdr hdr; /* wct = 15 */ + __le16 TotalParameterCount; + __le16 TotalDataCount; + __le16 MaxParameterCount; + __le16 MaxDataCount; + __u8 MaxSetupCount; + __u8 Reserved; + __le16 Flags; + __le32 Timeout; + __u16 Reserved2; + __le16 ParameterCount; /* 4 */ + __le16 ParameterOffset; + __le16 DataCount; /* 12 */ + __le16 DataOffset; + __u8 SetupCount; /* one */ + __u8 Reserved3; + __le16 SubCommand; /* TRANS2_SET_FS_INFORMATION */ + __le16 ByteCount; + __u8 Pad; + __u16 FileNum; /* Parameters start. */ + __le16 InformationLevel;/* Parameters end. */ + __le16 ClientUnixMajor; /* Data start. */ + __le16 ClientUnixMinor; + __le64 ClientUnixCap; /* Data end */ +} __packed TRANSACTION2_SETFSI_REQ; + +/* level 0x203 request structure follows */ +typedef struct smb_com_transaction2_setfs_enc_req { + struct smb_hdr hdr; /* wct = 15 */ + __le16 TotalParameterCount; + __le16 TotalDataCount; + __le16 MaxParameterCount; + __le16 MaxDataCount; + __u8 MaxSetupCount; + __u8 Reserved; + __le16 Flags; + __le32 Timeout; + __u16 Reserved2; + __le16 ParameterCount; /* 4 */ + __le16 ParameterOffset; + __le16 DataCount; /* 12 */ + __le16 DataOffset; + __u8 SetupCount; /* one */ + __u8 Reserved3; + __le16 SubCommand; /* TRANS2_SET_FS_INFORMATION */ + __le16 ByteCount; + __u8 Pad; + __u16 Reserved4; /* Parameters start. */ + __le16 InformationLevel;/* Parameters end. */ + /* NTLMSSP Blob, Data start. */ +} __packed TRANSACTION2_SETFSI_ENC_REQ; + +/* response for setfsinfo levels 0x200 and 0x203 */ +typedef struct smb_com_transaction2_setfsi_rsp { + struct smb_hdr hdr; /* wct = 10 */ + struct trans2_resp t2; + __u16 ByteCount; +} __packed TRANSACTION2_SETFSI_RSP; + +typedef struct smb_com_transaction2_get_dfs_refer_req { + struct smb_hdr hdr; /* wct = 15 */ + __le16 TotalParameterCount; + __le16 TotalDataCount; + __le16 MaxParameterCount; + __le16 MaxDataCount; + __u8 MaxSetupCount; + __u8 Reserved; + __le16 Flags; + __le32 Timeout; + __u16 Reserved2; + __le16 ParameterCount; + __le16 ParameterOffset; + __le16 DataCount; + __le16 DataOffset; + __u8 SetupCount; + __u8 Reserved3; + __le16 SubCommand; /* one setup word */ + __le16 ByteCount; + __u8 Pad[3]; /* Win2K has sent 0x0F01 (max response length + perhaps?) followed by one byte pad - doesn't + seem to matter though */ + __le16 MaxReferralLevel; + char RequestFileName[]; +} __packed TRANSACTION2_GET_DFS_REFER_REQ; + +#define DFS_VERSION cpu_to_le16(0x0003) + +/* DFS server target type */ +#define DFS_TYPE_LINK 0x0000 /* also for sysvol targets */ +#define DFS_TYPE_ROOT 0x0001 + +/* Referral Entry Flags */ +#define DFS_NAME_LIST_REF 0x0200 /* set for domain or DC referral responses */ +#define DFS_TARGET_SET_BOUNDARY 0x0400 /* only valid with version 4 dfs req */ + +typedef struct dfs_referral_level_3 { /* version 4 is same, + one flag bit */ + __le16 VersionNumber; /* must be 3 or 4 */ + __le16 Size; + __le16 ServerType; /* 0x0001 = root targets; 0x0000 = link targets */ + __le16 ReferralEntryFlags; + __le32 TimeToLive; + __le16 DfsPathOffset; + __le16 DfsAlternatePathOffset; + __le16 NetworkAddressOffset; /* offset of the link target */ + __u8 ServiceSiteGuid[16]; /* MBZ, ignored */ +} __packed REFERRAL3; + +struct get_dfs_referral_rsp { + __le16 PathConsumed; + __le16 NumberOfReferrals; + __le32 DFSFlags; + REFERRAL3 referrals[]; /* array of level 3 dfs_referral structures */ + /* followed by the strings pointed to by the referral structures */ +} __packed; + +typedef struct smb_com_transaction_get_dfs_refer_rsp { + struct smb_hdr hdr; /* wct = 10 */ + struct trans2_resp t2; + __u16 ByteCount; + __u8 Pad; + struct get_dfs_referral_rsp dfs_data; +} __packed TRANSACTION2_GET_DFS_REFER_RSP; + +/* DFS Flags */ +#define DFSREF_REFERRAL_SERVER 0x00000001 /* all targets are DFS roots */ +#define DFSREF_STORAGE_SERVER 0x00000002 /* no further ref requests needed */ +#define DFSREF_TARGET_FAILBACK 0x00000004 /* only for DFS referral version 4 */ + +/* + ************************************************************************ + * All structs for everything above the SMB PDUs themselves + * (such as the T2 level specific data) go here + ************************************************************************ + */ + +/* + * Information on a server + */ + +struct serverInfo { + char name[16]; + unsigned char versionMajor; + unsigned char versionMinor; + unsigned long type; + unsigned int commentOffset; +} __packed; + +/* + * The following structure is the format of the data returned on a NetShareEnum + * with level "90" (x5A) + */ + +struct shareInfo { + char shareName[13]; + char pad; + unsigned short type; + unsigned int commentOffset; +} __packed; + +struct aliasInfo { + char aliasName[9]; + char pad; + unsigned int commentOffset; + unsigned char type[2]; +} __packed; + +struct aliasInfo92 { + int aliasNameOffset; + int serverNameOffset; + int shareNameOffset; +} __packed; + +typedef struct { + __le32 fsid; + __le32 SectorsPerAllocationUnit; + __le32 TotalAllocationUnits; + __le32 FreeAllocationUnits; + __le16 BytesPerSector; +} __packed FILE_SYSTEM_ALLOC_INFO; + +typedef struct { + __le16 MajorVersionNumber; + __le16 MinorVersionNumber; + __le64 Capability; +} __packed FILE_SYSTEM_UNIX_INFO; /* Unix extension level 0x200*/ + +/* Version numbers for CIFS UNIX major and minor. */ +#define CIFS_UNIX_MAJOR_VERSION 1 +#define CIFS_UNIX_MINOR_VERSION 0 + +/* Linux/Unix extensions capability flags */ +#define CIFS_UNIX_FCNTL_CAP 0x00000001 /* support for fcntl locks */ +#define CIFS_UNIX_POSIX_ACL_CAP 0x00000002 /* support getfacl/setfacl */ +#define CIFS_UNIX_XATTR_CAP 0x00000004 /* support new namespace */ +#define CIFS_UNIX_EXTATTR_CAP 0x00000008 /* support chattr/chflag */ +#define CIFS_UNIX_POSIX_PATHNAMES_CAP 0x00000010 /* Allow POSIX path chars */ +#define CIFS_UNIX_POSIX_PATH_OPS_CAP 0x00000020 /* Allow new POSIX path based + calls including posix open + and posix unlink */ +#define CIFS_UNIX_LARGE_READ_CAP 0x00000040 /* support reads >128K (up to 0xFFFF00 */ +#define CIFS_UNIX_LARGE_WRITE_CAP 0x00000080 +#define CIFS_UNIX_TRANSPORT_ENCRYPTION_CAP 0x00000100 /* can do SPNEGO crypt */ +#define CIFS_UNIX_TRANSPORT_ENCRYPTION_MANDATORY_CAP 0x00000200 /* must do */ +#define CIFS_UNIX_PROXY_CAP 0x00000400 /* Proxy cap: 0xACE ioctl and QFS PROXY call */ +#ifdef CONFIG_CIFS_POSIX +/* presumably don't need the 0x20 POSIX_PATH_OPS_CAP since we never send + LockingX instead of posix locking call on unix sess (and we do not expect + LockingX to use different (ie Windows) semantics than posix locking on + the same session (if WINE needs to do this later, we can add this cap + back in later */ +/* #define CIFS_UNIX_CAP_MASK 0x000000fb */ +#define CIFS_UNIX_CAP_MASK 0x000003db +#else +#define CIFS_UNIX_CAP_MASK 0x00000013 +#endif /* CONFIG_CIFS_POSIX */ + + +#define CIFS_POSIX_EXTENSIONS 0x00000010 /* support for new QFSInfo */ + +/******************************************************************************/ +/* QueryFileInfo/QueryPathinfo (also for SetPath/SetFile) data buffer formats */ +/******************************************************************************/ +typedef struct { /* data block encoding of response to level 263 QPathInfo */ + struct_group_attr(common_attributes, __packed, + __le64 CreationTime; + __le64 LastAccessTime; + __le64 LastWriteTime; + __le64 ChangeTime; + __le32 Attributes; + ); + __u32 Pad1; + __le64 AllocationSize; + __le64 EndOfFile; /* size ie offset to first free byte in file */ + __le32 NumberOfLinks; /* hard links */ + __u8 DeletePending; + __u8 Directory; + __u16 Pad2; + __le32 EASize; + __le32 FileNameLength; + union { + char __pad; + DECLARE_FLEX_ARRAY(char, FileName); + }; +} __packed FILE_ALL_INFO; /* level 0x107 QPathInfo */ + +typedef struct { + __le64 AllocationSize; + __le64 EndOfFile; /* size ie offset to first free byte in file */ + __le32 NumberOfLinks; /* hard links */ + __u8 DeletePending; + __u8 Directory; + __u16 Pad; +} __packed FILE_STANDARD_INFO; /* level 0x102 QPathInfo */ + + +/* defines for enumerating possible values of the Unix type field below */ +#define UNIX_FILE 0 +#define UNIX_DIR 1 +#define UNIX_SYMLINK 2 +#define UNIX_CHARDEV 3 +#define UNIX_BLOCKDEV 4 +#define UNIX_FIFO 5 +#define UNIX_SOCKET 6 +typedef struct { + __le64 EndOfFile; + __le64 NumOfBytes; + __le64 LastStatusChange; /*SNIA specs DCE time for the 3 time fields */ + __le64 LastAccessTime; + __le64 LastModificationTime; + __le64 Uid; + __le64 Gid; + __le32 Type; + __le64 DevMajor; + __le64 DevMinor; + __le64 UniqueId; + __le64 Permissions; + __le64 Nlinks; +} __packed FILE_UNIX_BASIC_INFO; /* level 0x200 QPathInfo */ + +typedef struct { + DECLARE_FLEX_ARRAY(char, LinkDest); +} __packed FILE_UNIX_LINK_INFO; /* level 0x201 QPathInfo */ + +/* The following three structures are needed only for + setting time to NT4 and some older servers via + the primitive DOS time format */ +typedef struct { + __u16 Day:5; + __u16 Month:4; + __u16 Year:7; +} __packed SMB_DATE; + +typedef struct { + __u16 TwoSeconds:5; + __u16 Minutes:6; + __u16 Hours:5; +} __packed SMB_TIME; + +typedef struct { + __le16 CreationDate; /* SMB Date see above */ + __le16 CreationTime; /* SMB Time */ + __le16 LastAccessDate; + __le16 LastAccessTime; + __le16 LastWriteDate; + __le16 LastWriteTime; + __le32 DataSize; /* File Size (EOF) */ + __le32 AllocationSize; + __le16 Attributes; /* verify not u32 */ + __le32 EASize; +} __packed FILE_INFO_STANDARD; /* level 1 SetPath/FileInfo */ + +struct file_allocation_info { + __le64 AllocationSize; /* Note old Samba srvr rounds this up too much */ +} __packed; /* size used on disk, for level 0x103 for set, 0x105 for query */ + +struct file_end_of_file_info { + __le64 FileSize; /* offset to end of file */ +} __packed; /* size info, level 0x104 for set, 0x106 for query */ + +struct file_alt_name_info { + DECLARE_FLEX_ARRAY(__u8, alt_name); +} __packed; /* level 0x0108 */ + +struct file_stream_info { + __le32 number_of_streams; /* BB check sizes and verify location */ + /* followed by info on streams themselves + u64 size; + u64 allocation_size + stream info */ +}; /* level 0x109 */ + +struct file_compression_info { + __le64 compressed_size; + __le16 format; + __u8 unit_shift; + __u8 ch_shift; + __u8 cl_shift; + __u8 pad[3]; +} __packed; /* level 0x10b */ + +/* POSIX ACL set/query path info structures */ +#define CIFS_ACL_VERSION 1 +struct cifs_posix_ace { /* access control entry (ACE) */ + __u8 cifs_e_tag; + __u8 cifs_e_perm; + __le64 cifs_uid; /* or gid */ +} __packed; + +struct cifs_posix_acl { /* access control list (ACL) */ + __le16 version; + __le16 access_entry_count; /* access ACL - count of entries */ + __le16 default_entry_count; /* default ACL - count of entries */ + struct cifs_posix_ace ace_array[]; + /* followed by struct cifs_posix_ace default_ace_array[] */ +} __packed; /* level 0x204 */ + +/* types of access control entries already defined in posix_acl.h */ +/* #define CIFS_POSIX_ACL_USER_OBJ 0x01 +#define CIFS_POSIX_ACL_USER 0x02 +#define CIFS_POSIX_ACL_GROUP_OBJ 0x04 +#define CIFS_POSIX_ACL_GROUP 0x08 +#define CIFS_POSIX_ACL_MASK 0x10 +#define CIFS_POSIX_ACL_OTHER 0x20 */ + +/* types of perms */ +/* #define CIFS_POSIX_ACL_EXECUTE 0x01 +#define CIFS_POSIX_ACL_WRITE 0x02 +#define CIFS_POSIX_ACL_READ 0x04 */ + +/* end of POSIX ACL definitions */ + +/* POSIX Open Flags */ +#define SMB_O_RDONLY 0x1 +#define SMB_O_WRONLY 0x2 +#define SMB_O_RDWR 0x4 +#define SMB_O_CREAT 0x10 +#define SMB_O_EXCL 0x20 +#define SMB_O_TRUNC 0x40 +#define SMB_O_APPEND 0x80 +#define SMB_O_SYNC 0x100 +#define SMB_O_DIRECTORY 0x200 +#define SMB_O_NOFOLLOW 0x400 +#define SMB_O_DIRECT 0x800 + +typedef struct { + __le32 OpenFlags; /* same as NT CreateX */ + __le32 PosixOpenFlags; + __le64 Permissions; + __le16 Level; /* reply level requested (see QPathInfo levels) */ +} __packed OPEN_PSX_REQ; /* level 0x209 SetPathInfo data */ + +typedef struct { + __le16 OplockFlags; + __u16 Fid; + __le32 CreateAction; + __le16 ReturnedLevel; + __le16 Pad; + /* struct following varies based on requested level */ +} __packed OPEN_PSX_RSP; /* level 0x209 SetPathInfo data */ + +#define SMB_POSIX_UNLINK_FILE_TARGET 0 +#define SMB_POSIX_UNLINK_DIRECTORY_TARGET 1 + +struct unlink_psx_rq { /* level 0x20a SetPathInfo */ + __le16 type; +} __packed; + +struct file_internal_info { + __le64 UniqueId; /* inode number */ +} __packed; /* level 0x3ee */ + +struct file_mode_info { + __le32 Mode; +} __packed; /* level 0x3f8 */ + +struct file_attrib_tag { + __le32 Attribute; + __le32 ReparseTag; +} __packed; /* level 0x40b */ + + +/********************************************************/ +/* FindFirst/FindNext transact2 data buffer formats */ +/********************************************************/ + +typedef struct { + __le32 NextEntryOffset; + __u32 ResumeKey; /* as with FileIndex - no need to convert */ + FILE_UNIX_BASIC_INFO basic; + union { + char __pad; + DECLARE_FLEX_ARRAY(char, FileName); + }; +} __packed FILE_UNIX_INFO; /* level 0x202 */ + +typedef struct { + __u32 ResumeKey; + __le16 CreationDate; /* SMB Date */ + __le16 CreationTime; /* SMB Time */ + __le16 LastAccessDate; + __le16 LastAccessTime; + __le16 LastWriteDate; + __le16 LastWriteTime; + __le32 DataSize; /* File Size (EOF) */ + __le32 AllocationSize; + __le16 Attributes; /* verify not u32 */ + __u8 FileNameLength; + char FileName[]; +} __packed FIND_FILE_STANDARD_INFO; /* level 0x1 FF resp data */ + + +struct fea { + unsigned char EA_flags; + __u8 name_len; + __le16 value_len; + char name[]; + /* optionally followed by value */ +} __packed; +/* flags for _FEA.fEA */ +#define FEA_NEEDEA 0x80 /* need EA bit */ + +struct fealist { + __le32 list_len; + struct fea list; +} __packed; + +/* used to hold an arbitrary blob of data */ +struct data_blob { + __u8 *data; + size_t length; + void (*free) (struct data_blob *data_blob); +} __packed; + + +#ifdef CONFIG_CIFS_POSIX +/* + For better POSIX semantics from Linux client, (even better + than the existing CIFS Unix Extensions) we need updated PDUs for: + + 1) PosixCreateX - to set and return the mode, inode#, device info and + perhaps add a CreateDevice - to create Pipes and other special .inodes + Also note POSIX open flags + 2) Close - to return the last write time to do cache across close + more safely + 3) FindFirst return unique inode number - what about resume key, two + forms short (matches readdir) and full (enough info to cache inodes) + 4) Mkdir - set mode + + And under consideration: + 5) FindClose2 (return nanosecond timestamp ??) + 6) Use nanosecond timestamps throughout all time fields if + corresponding attribute flag is set + 7) sendfile - handle based copy + + what about fixing 64 bit alignment + + There are also various legacy SMB/CIFS requests used as is + + From existing Lanman and NTLM dialects: + -------------------------------------- + NEGOTIATE + SESSION_SETUP_ANDX (BB which?) + TREE_CONNECT_ANDX (BB which wct?) + TREE_DISCONNECT (BB add volume timestamp on response) + LOGOFF_ANDX + DELETE (note delete open file behavior) + DELETE_DIRECTORY + READ_AND_X + WRITE_AND_X + LOCKING_AND_X (note posix lock semantics) + RENAME (note rename across dirs and open file rename posix behaviors) + NT_RENAME (for hardlinks) Is this good enough for all features? + FIND_CLOSE2 + TRANSACTION2 (18 cases) + SMB_SET_FILE_END_OF_FILE_INFO2 SMB_SET_PATH_END_OF_FILE_INFO2 + (BB verify that never need to set allocation size) + SMB_SET_FILE_BASIC_INFO2 (setting times - BB can it be done via + Unix ext?) + + COPY (note support for copy across directories) - FUTURE, OPTIONAL + setting/getting OS/2 EAs - FUTURE (BB can this handle + setting Linux xattrs perfectly) - OPTIONAL + dnotify - FUTURE, OPTIONAL + quota - FUTURE, OPTIONAL + + Note that various requests implemented for NT interop such as + NT_TRANSACT (IOCTL) QueryReparseInfo + are unneeded to servers compliant with the CIFS POSIX extensions + + From CIFS Unix Extensions: + ------------------------- + T2 SET_PATH_INFO (SMB_SET_FILE_UNIX_LINK) for symlinks + T2 SET_PATH_INFO (SMB_SET_FILE_BASIC_INFO2) + T2 QUERY_PATH_INFO (SMB_QUERY_FILE_UNIX_LINK) + T2 QUERY_PATH_INFO (SMB_QUERY_FILE_UNIX_BASIC) BB check for missing + inode fields + Actually a need QUERY_FILE_UNIX_INFO + since has inode num + BB what about a) blksize/blkbits/blocks + b) i_version + c) i_rdev + d) notify mask? + e) generation + f) size_seqcount + T2 FIND_FIRST/FIND_NEXT FIND_FILE_UNIX + TRANS2_GET_DFS_REFERRAL - OPTIONAL but recommended + T2_QFS_INFO QueryDevice/AttributeInfo - OPTIONAL + */ + +/* xsymlink is a symlink format (used by MacOS) that can be used + to save symlink info in a regular file when + mounted to operating systems that do not + support the cifs Unix extensions or EAs (for xattr + based symlinks). For such a file to be recognized + as containing symlink data: + + 1) file size must be 1067, + 2) signature must begin file data, + 3) length field must be set to ASCII representation + of a number which is less than or equal to 1024, + 4) md5 must match that of the path data */ + +struct xsymlink { + /* 1067 bytes */ + char signature[4]; /* XSym */ /* not null terminated */ + char cr0; /* \n */ +/* ASCII representation of length (4 bytes decimal) terminated by \n not null */ + char length[4]; + char cr1; /* \n */ +/* md5 of valid subset of path ie path[0] through path[length-1] */ + __u8 md5[32]; + char cr2; /* \n */ +/* if room left, then end with \n then 0x20s by convention but not required */ + char path[1024]; +} __packed; + +typedef struct file_xattr_info { + /* BB do we need another field for flags? BB */ + __u32 xattr_name_len; + __u32 xattr_value_len; + char xattr_name[]; + /* followed by xattr_value[xattr_value_len], no pad */ +} __packed FILE_XATTR_INFO; /* extended attribute info level 0x205 */ + +/* flags for lsattr and chflags commands removed arein uapi/linux/fs.h */ + +typedef struct file_chattr_info { + __le64 mask; /* list of all possible attribute bits */ + __le64 mode; /* list of actual attribute bits on this inode */ +} __packed FILE_CHATTR_INFO; /* ext attributes (chattr, chflags) level 0x206 */ +#endif /* POSIX */ + +#endif /* _SMB1PDU_H */ diff --git a/fs/smb/client/smb1proto.h b/fs/smb/client/smb1proto.h new file mode 100644 index 000000000..80eaeb3dd --- /dev/null +++ b/fs/smb/client/smb1proto.h @@ -0,0 +1,350 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +/* + * + * Copyright (c) International Business Machines Corp., 2002,2008 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ +#ifndef _SMB1PROTO_H +#define _SMB1PROTO_H + +#include +#include +#include "../common/smb2pdu.h" +#include "cifsglob.h" + +struct cifs_unix_set_info_args { + __u64 ctime; + __u64 atime; + __u64 mtime; + __u64 mode; + kuid_t uid; + kgid_t gid; + dev_t device; +}; + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + +/* + * cifssmb.c + */ +int small_smb_init_no_tc(const int smb_command, const int wct, + struct cifs_ses *ses, void **request_buf); +int CIFSSMBNegotiate(const unsigned int xid, struct cifs_ses *ses, + struct TCP_Server_Info *server); +int CIFSTCon(const unsigned int xid, struct cifs_ses *ses, const char *tree, + struct cifs_tcon *tcon, const struct nls_table *nls_codepage); +int CIFSSMBTDis(const unsigned int xid, struct cifs_tcon *tcon); +int CIFSSMBEcho(struct TCP_Server_Info *server); +int CIFSSMBLogoff(const unsigned int xid, struct cifs_ses *ses); +int CIFSPOSIXDelFile(const unsigned int xid, struct cifs_tcon *tcon, + const char *fileName, __u16 type, + const struct nls_table *nls_codepage, int remap); +int CIFSSMBDelFile(const unsigned int xid, struct cifs_tcon *tcon, + const char *name, struct cifs_sb_info *cifs_sb, + struct dentry *dentry); +int CIFSSMBRmDir(const unsigned int xid, struct cifs_tcon *tcon, + const char *name, struct cifs_sb_info *cifs_sb); +int CIFSSMBMkDir(const unsigned int xid, struct inode *inode, umode_t mode, + struct cifs_tcon *tcon, const char *name, + struct cifs_sb_info *cifs_sb); +int CIFSPOSIXCreate(const unsigned int xid, struct cifs_tcon *tcon, + __u32 posix_flags, __u64 mode, __u16 *netfid, + FILE_UNIX_BASIC_INFO *pRetData, __u32 *pOplock, + const char *name, const struct nls_table *nls_codepage, + int remap); +int SMBLegacyOpen(const unsigned int xid, struct cifs_tcon *tcon, + const char *fileName, const int openDisposition, + const int access_flags, const int create_options, + __u16 *netfid, int *pOplock, FILE_ALL_INFO *pfile_info, + const struct nls_table *nls_codepage, int remap); +int CIFS_open(const unsigned int xid, struct cifs_open_parms *oparms, + int *oplock, FILE_ALL_INFO *buf); +int cifs_async_readv(struct cifs_io_subrequest *rdata); +int CIFSSMBRead(const unsigned int xid, struct cifs_io_parms *io_parms, + unsigned int *nbytes, char **buf, int *pbuf_type); +int CIFSSMBWrite(const unsigned int xid, struct cifs_io_parms *io_parms, + unsigned int *nbytes, const char *buf); +void cifs_async_writev(struct cifs_io_subrequest *wdata); +int CIFSSMBWrite2(const unsigned int xid, struct cifs_io_parms *io_parms, + unsigned int *nbytes, struct kvec *iov, int n_vec); +int cifs_lockv(const unsigned int xid, struct cifs_tcon *tcon, + const __u16 netfid, const __u8 lock_type, + const __u32 num_unlock, const __u32 num_lock, + LOCKING_ANDX_RANGE *buf); +int CIFSSMBLock(const unsigned int xid, struct cifs_tcon *tcon, + const __u16 smb_file_id, const __u32 netpid, const __u64 len, + const __u64 offset, const __u32 numUnlock, const __u32 numLock, + const __u8 lockType, const bool waitFlag, + const __u8 oplock_level); +int CIFSSMBPosixLock(const unsigned int xid, struct cifs_tcon *tcon, + const __u16 smb_file_id, const __u32 netpid, + const loff_t start_offset, const __u64 len, + struct file_lock *pLockData, const __u16 lock_type, + const bool waitFlag); +int CIFSSMBClose(const unsigned int xid, struct cifs_tcon *tcon, + int smb_file_id); +int CIFSSMBFlush(const unsigned int xid, struct cifs_tcon *tcon, + int smb_file_id); +int CIFSSMBRename(const unsigned int xid, struct cifs_tcon *tcon, + struct dentry *source_dentry, const char *from_name, + const char *to_name, struct cifs_sb_info *cifs_sb); +int CIFSSMBRenameOpenFile(const unsigned int xid, struct cifs_tcon *pTcon, + int netfid, const char *target_name, + const struct nls_table *nls_codepage, int remap); +int CIFSUnixCreateSymLink(const unsigned int xid, struct cifs_tcon *tcon, + const char *fromName, const char *toName, + const struct nls_table *nls_codepage, int remap); +int CIFSUnixCreateHardLink(const unsigned int xid, struct cifs_tcon *tcon, + const char *fromName, const char *toName, + const struct nls_table *nls_codepage, int remap); +int CIFSCreateHardLink(const unsigned int xid, struct cifs_tcon *tcon, + struct dentry *source_dentry, const char *from_name, + const char *to_name, struct cifs_sb_info *cifs_sb); +int CIFSSMBUnixQuerySymLink(const unsigned int xid, struct cifs_tcon *tcon, + const unsigned char *searchName, + char **symlinkinfo, + const struct nls_table *nls_codepage, int remap); +int cifs_query_reparse_point(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, + const char *full_path, u32 *tag, struct kvec *rsp, + int *rsp_buftype); +struct inode *cifs_create_reparse_inode(struct cifs_open_info_data *data, + struct super_block *sb, + const unsigned int xid, + struct cifs_tcon *tcon, + const char *full_path, bool directory, + struct kvec *reparse_iov, + struct kvec *xattr_iov); +int CIFSSMB_set_compression(const unsigned int xid, struct cifs_tcon *tcon, + __u16 fid, __u16 compression_state); +int cifs_do_get_acl(const unsigned int xid, struct cifs_tcon *tcon, + const unsigned char *searchName, struct posix_acl **acl, + const int acl_type, const struct nls_table *nls_codepage, + int remap); +int cifs_do_set_acl(const unsigned int xid, struct cifs_tcon *tcon, + const unsigned char *fileName, const struct posix_acl *acl, + const int acl_type, const struct nls_table *nls_codepage, + int remap); +int CIFSGetExtAttr(const unsigned int xid, struct cifs_tcon *tcon, + const int netfid, __u64 *pExtAttrBits, __u64 *pMask); +int CIFSSMBGetCIFSACL(const unsigned int xid, struct cifs_tcon *tcon, + __u16 fid, struct smb_ntsd **acl_inf, __u32 *pbuflen, + __u32 info); +int CIFSSMBSetCIFSACL(const unsigned int xid, struct cifs_tcon *tcon, + __u16 fid, struct smb_ntsd *pntsd, __u32 acllen, + int aclflag); +int SMBQueryInformation(const unsigned int xid, struct cifs_tcon *tcon, + const char *search_name, FILE_ALL_INFO *data, + const struct nls_table *nls_codepage, int remap); +int CIFSSMBQFileInfo(const unsigned int xid, struct cifs_tcon *tcon, + u16 netfid, FILE_ALL_INFO *pFindData); +int CIFSSMBQPathInfo(const unsigned int xid, struct cifs_tcon *tcon, + const char *search_name, FILE_ALL_INFO *data, + int legacy /* old style infolevel */, + const struct nls_table *nls_codepage, int remap); +int CIFSSMBUnixQFileInfo(const unsigned int xid, struct cifs_tcon *tcon, + u16 netfid, FILE_UNIX_BASIC_INFO *pFindData); +int CIFSSMBUnixQPathInfo(const unsigned int xid, struct cifs_tcon *tcon, + const unsigned char *searchName, + FILE_UNIX_BASIC_INFO *pFindData, + const struct nls_table *nls_codepage, int remap); +int CIFSFindFirst(const unsigned int xid, struct cifs_tcon *tcon, + const char *searchName, struct cifs_sb_info *cifs_sb, + __u16 *pnetfid, __u16 search_flags, + struct cifs_search_info *psrch_inf, bool msearch); +int CIFSFindNext(const unsigned int xid, struct cifs_tcon *tcon, + __u16 searchHandle, __u16 search_flags, + struct cifs_search_info *psrch_inf); +int CIFSFindClose(const unsigned int xid, struct cifs_tcon *tcon, + const __u16 searchHandle); +int CIFSGetSrvInodeNumber(const unsigned int xid, struct cifs_tcon *tcon, + const char *search_name, __u64 *inode_number, + const struct nls_table *nls_codepage, int remap); +int CIFSGetDFSRefer(const unsigned int xid, struct cifs_ses *ses, + const char *search_name, + struct dfs_info3_param **target_nodes, + unsigned int *num_of_nodes, + const struct nls_table *nls_codepage, int remap); +int SMBOldQFSInfo(const unsigned int xid, struct cifs_tcon *tcon, + struct kstatfs *FSData); +int CIFSSMBQFSInfo(const unsigned int xid, struct cifs_tcon *tcon, + struct kstatfs *FSData); +int CIFSSMBQFSAttributeInfo(const unsigned int xid, struct cifs_tcon *tcon); +int CIFSSMBQFSDeviceInfo(const unsigned int xid, struct cifs_tcon *tcon); +int CIFSSMBQFSUnixInfo(const unsigned int xid, struct cifs_tcon *tcon); +int CIFSSMBSetFSUnixInfo(const unsigned int xid, struct cifs_tcon *tcon, + __u64 cap); +int CIFSSMBQFSPosixInfo(const unsigned int xid, struct cifs_tcon *tcon, + struct kstatfs *FSData); +int CIFSSMBSetEOF(const unsigned int xid, struct cifs_tcon *tcon, + const char *file_name, __u64 size, + struct cifs_sb_info *cifs_sb, bool set_allocation, + struct dentry *dentry); +int CIFSSMBSetFileSize(const unsigned int xid, struct cifs_tcon *tcon, + struct cifsFileInfo *cfile, __u64 size, + bool set_allocation); +int SMBSetInformation(const unsigned int xid, struct cifs_tcon *tcon, + const char *fileName, __le32 attributes, + __le64 write_time, const struct nls_table *nls_codepage, + struct cifs_sb_info *cifs_sb); +int CIFSSMBSetFileInfo(const unsigned int xid, struct cifs_tcon *tcon, + const FILE_BASIC_INFO *data, __u16 fid, + __u32 pid_of_opener); +int CIFSSMBSetFileDisposition(const unsigned int xid, struct cifs_tcon *tcon, + bool delete_file, __u16 fid, + __u32 pid_of_opener); +int CIFSSMBSetPathInfo(const unsigned int xid, struct cifs_tcon *tcon, + const char *fileName, const FILE_BASIC_INFO *data, + const struct nls_table *nls_codepage, + struct cifs_sb_info *cifs_sb); +int CIFSSMBUnixSetFileInfo(const unsigned int xid, struct cifs_tcon *tcon, + const struct cifs_unix_set_info_args *args, u16 fid, + u32 pid_of_opener); +int CIFSSMBUnixSetPathInfo(const unsigned int xid, struct cifs_tcon *tcon, + const char *file_name, + const struct cifs_unix_set_info_args *args, + const struct nls_table *nls_codepage, int remap); +ssize_t CIFSSMBQAllEAs(const unsigned int xid, struct cifs_tcon *tcon, + const unsigned char *searchName, + const unsigned char *ea_name, char *EAData, + size_t buf_size, struct cifs_sb_info *cifs_sb); +int CIFSSMBSetEA(const unsigned int xid, struct cifs_tcon *tcon, + const char *fileName, const char *ea_name, + const void *ea_value, const __u16 ea_value_len, + const struct nls_table *nls_codepage, + struct cifs_sb_info *cifs_sb); + +/* + * smb1debug.c + */ +void cifs_dump_detail(void *buf, size_t buf_len, + struct TCP_Server_Info *server); + +/* + * smb1encrypt.c + */ +int cifs_sign_rqst(struct smb_rqst *rqst, struct TCP_Server_Info *server, + __u32 *pexpected_response_sequence_number); +int cifs_verify_signature(struct smb_rqst *rqst, + struct TCP_Server_Info *server, + __u32 expected_sequence_number); + +/* + * smb1maperror.c + */ +int map_smb_to_linux_error(char *buf, bool logErr); +int smb1_init_maperror(void); +int map_and_check_smb_error(struct TCP_Server_Info *server, + struct mid_q_entry *mid, bool logErr); +#if IS_ENABLED(CONFIG_SMB1_KUNIT_TESTS) +extern const struct ntstatus_to_dos_err *ntstatus_to_dos_map_test; +extern unsigned int ntstatus_to_dos_num; +const struct ntstatus_to_dos_err * +search_ntstatus_to_dos_map_test(__u32 ntstatus); +extern const struct smb_to_posix_error *mapping_table_ERRDOS_test; +extern unsigned int mapping_table_ERRDOS_num; +const struct smb_to_posix_error * +search_mapping_table_ERRDOS_test(__u16 smb_err); +extern const struct smb_to_posix_error *mapping_table_ERRSRV_test; +extern unsigned int mapping_table_ERRSRV_num; +const struct smb_to_posix_error * +search_mapping_table_ERRSRV_test(__u16 smb_err); +#endif + +/* + * smb1misc.c + */ +unsigned int header_assemble(struct smb_hdr *buffer, char smb_command, + const struct cifs_tcon *treeCon, int word_count); +bool is_valid_oplock_break(char *buffer, struct TCP_Server_Info *srv); +unsigned int smbCalcSize(void *buf); + +/* + * smb1ops.c + */ +extern struct smb_version_operations smb1_operations; +extern struct smb_version_values smb1_values; + +void reset_cifs_unix_caps(unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, + struct smb3_fs_context *ctx); + +/* + * smb1session.c + */ +int CIFS_SessSetup(const unsigned int xid, struct cifs_ses *ses, + struct TCP_Server_Info *server, + const struct nls_table *nls_cp); + +/* + * smb1transport.c + */ +struct mid_q_entry *cifs_setup_async_request(struct TCP_Server_Info *server, + struct smb_rqst *rqst); +int SendReceiveNoRsp(const unsigned int xid, struct cifs_ses *ses, + char *in_buf, unsigned int in_len, int flags); +int cifs_check_receive(struct mid_q_entry *mid, struct TCP_Server_Info *server, + bool log_error); +struct mid_q_entry *cifs_setup_request(struct cifs_ses *ses, + struct TCP_Server_Info *server, + struct smb_rqst *rqst); +int SendReceive2(const unsigned int xid, struct cifs_ses *ses, + struct kvec *iov, int n_vec, int *resp_buf_type /* ret */, + const int flags, struct kvec *resp_iov); +int SendReceive(const unsigned int xid, struct cifs_ses *ses, + struct smb_hdr *in_buf, unsigned int in_len, + struct smb_hdr *out_buf, int *pbytes_returned, + const int flags); +bool cifs_check_trans2(struct mid_q_entry *mid, struct TCP_Server_Info *server, + char *buf, int malformed); +int checkSMB(char *buf, unsigned int pdu_len, unsigned int total_read, + struct TCP_Server_Info *server); + + +static inline __u16 +get_mid(const struct smb_hdr *smb) +{ + return le16_to_cpu(smb->Mid); +} + +static inline bool +compare_mid(__u16 mid, const struct smb_hdr *smb) +{ + return mid == le16_to_cpu(smb->Mid); +} + +#define GETU16(var) (*((__u16 *)var)) /* BB check for endian issues */ +#define GETU32(var) (*((__u32 *)var)) /* BB check for endian issues */ + +/* given a pointer to an smb_hdr, retrieve a void pointer to the ByteCount */ +static inline void * +BCC(struct smb_hdr *smb) +{ + return (void *)smb + sizeof(*smb) + 2 * smb->WordCount; +} + +/* given a pointer to an smb_hdr retrieve the pointer to the byte area */ +#define pByteArea(smb_var) (BCC(smb_var) + 2) + +/* get the unconverted ByteCount for a SMB packet and return it */ +static inline __u16 +get_bcc(struct smb_hdr *hdr) +{ + __le16 *bc_ptr = (__le16 *)BCC(hdr); + + return get_unaligned_le16(bc_ptr); +} + +/* set the ByteCount for a SMB packet in little-endian */ +static inline void +put_bcc(__u16 count, struct smb_hdr *hdr) +{ + __le16 *bc_ptr = (__le16 *)BCC(hdr); + + put_unaligned_le16(count, bc_ptr); +} + +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ + +#endif /* _SMB1PROTO_H */ diff --git a/fs/smb/client/smb1session.c b/fs/smb/client/smb1session.c new file mode 100644 index 000000000..83bfbf0c0 --- /dev/null +++ b/fs/smb/client/smb1session.c @@ -0,0 +1,995 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * SMB/CIFS session setup handling routines + * + * Copyright (c) International Business Machines Corp., 2006, 2009 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ + +#include "cifsproto.h" +#include "smb1proto.h" +#include "ntlmssp.h" +#include "nterr.h" +#include "cifs_spnego.h" +#include "cifs_unicode.h" +#include "cifs_debug.h" + +struct sess_data { + unsigned int xid; + struct cifs_ses *ses; + struct TCP_Server_Info *server; + struct nls_table *nls_cp; + void (*func)(struct sess_data *); + int result; + unsigned int in_len; + + /* we will send the SMB in three pieces: + * a fixed length beginning part, an optional + * SPNEGO blob (which can be zero length), and a + * last part which will include the strings + * and rest of bcc area. This allows us to avoid + * a large buffer 17K allocation + */ + int buf0_type; + struct kvec iov[3]; +}; + +static __u32 cifs_ssetup_hdr(struct cifs_ses *ses, + struct TCP_Server_Info *server, + SESSION_SETUP_ANDX *pSMB) +{ + __u32 capabilities = 0; + + /* init fields common to all four types of SessSetup */ + /* Note that offsets for first seven fields in req struct are same */ + /* in CIFS Specs so does not matter which of 3 forms of struct */ + /* that we use in next few lines */ + /* Note that header is initialized to zero in header_assemble */ + pSMB->req.AndXCommand = 0xFF; + pSMB->req.MaxBufferSize = cpu_to_le16(min_t(u32, + CIFSMaxBufSize + MAX_CIFS_HDR_SIZE - 4, + USHRT_MAX)); + pSMB->req.MaxMpxCount = cpu_to_le16(server->maxReq); + pSMB->req.VcNumber = cpu_to_le16(1); + pSMB->req.SessionKey = server->session_key_id; + + /* Now no need to set SMBFLG_CASELESS or obsolete CANONICAL PATH */ + + /* BB verify whether signing required on neg or just auth frame (and NTLM case) */ + + capabilities = CAP_LARGE_FILES | CAP_NT_SMBS | CAP_LEVEL_II_OPLOCKS | + CAP_LARGE_WRITE_X | CAP_LARGE_READ_X; + + if (server->sign) + pSMB->req.hdr.Flags2 |= SMBFLG2_SECURITY_SIGNATURE; + + if (ses->capabilities & CAP_UNICODE) { + pSMB->req.hdr.Flags2 |= SMBFLG2_UNICODE; + capabilities |= CAP_UNICODE; + } + if (ses->capabilities & CAP_STATUS32) { + pSMB->req.hdr.Flags2 |= SMBFLG2_ERR_STATUS; + capabilities |= CAP_STATUS32; + } + if (ses->capabilities & CAP_DFS) { + pSMB->req.hdr.Flags2 |= SMBFLG2_DFS; + capabilities |= CAP_DFS; + } + if (ses->capabilities & CAP_UNIX) + capabilities |= CAP_UNIX; + + return capabilities; +} + +static void +unicode_oslm_strings(char **pbcc_area, const struct nls_table *nls_cp) +{ + char *bcc_ptr = *pbcc_area; + int bytes_ret = 0; + + /* Copy OS version */ + bytes_ret = cifs_strtoUTF16((__le16 *)bcc_ptr, "Linux version ", 32, + nls_cp); + bcc_ptr += 2 * bytes_ret; + bytes_ret = cifs_strtoUTF16((__le16 *) bcc_ptr, init_utsname()->release, + 32, nls_cp); + bcc_ptr += 2 * bytes_ret; + bcc_ptr += 2; /* trailing null */ + + bytes_ret = cifs_strtoUTF16((__le16 *) bcc_ptr, CIFS_NETWORK_OPSYS, + 32, nls_cp); + bcc_ptr += 2 * bytes_ret; + bcc_ptr += 2; /* trailing null */ + + *pbcc_area = bcc_ptr; +} + +static void +ascii_oslm_strings(char **pbcc_area, const struct nls_table *nls_cp) +{ + char *bcc_ptr = *pbcc_area; + + strcpy(bcc_ptr, "Linux version "); + bcc_ptr += strlen("Linux version "); + strcpy(bcc_ptr, init_utsname()->release); + bcc_ptr += strlen(init_utsname()->release) + 1; + + strcpy(bcc_ptr, CIFS_NETWORK_OPSYS); + bcc_ptr += strlen(CIFS_NETWORK_OPSYS) + 1; + + *pbcc_area = bcc_ptr; +} + +static void unicode_domain_string(char **pbcc_area, struct cifs_ses *ses, + const struct nls_table *nls_cp) +{ + char *bcc_ptr = *pbcc_area; + int bytes_ret = 0; + + /* copy domain */ + if (ses->domainName == NULL) { + /* + * Sending null domain better than using a bogus domain name (as + * we did briefly in 2.6.18) since server will use its default + */ + *bcc_ptr = 0; + *(bcc_ptr+1) = 0; + bytes_ret = 0; + } else + bytes_ret = cifs_strtoUTF16((__le16 *) bcc_ptr, ses->domainName, + CIFS_MAX_DOMAINNAME_LEN, nls_cp); + bcc_ptr += 2 * bytes_ret; + bcc_ptr += 2; /* account for null terminator */ + + *pbcc_area = bcc_ptr; +} + +static void ascii_domain_string(char **pbcc_area, struct cifs_ses *ses, + const struct nls_table *nls_cp) +{ + char *bcc_ptr = *pbcc_area; + int len; + + /* copy domain */ + if (ses->domainName != NULL) { + len = strscpy(bcc_ptr, ses->domainName, CIFS_MAX_DOMAINNAME_LEN); + if (WARN_ON_ONCE(len < 0)) + len = CIFS_MAX_DOMAINNAME_LEN - 1; + bcc_ptr += len; + } /* else we send a null domain name so server will default to its own domain */ + *bcc_ptr = 0; + bcc_ptr++; + + *pbcc_area = bcc_ptr; +} + +static void unicode_ssetup_strings(char **pbcc_area, struct cifs_ses *ses, + const struct nls_table *nls_cp) +{ + char *bcc_ptr = *pbcc_area; + int bytes_ret = 0; + + /* BB FIXME add check that strings less than 335 or will need to send as arrays */ + + /* copy user */ + if (ses->user_name == NULL) { + /* null user mount */ + *bcc_ptr = 0; + *(bcc_ptr+1) = 0; + } else { + bytes_ret = cifs_strtoUTF16((__le16 *) bcc_ptr, ses->user_name, + CIFS_MAX_USERNAME_LEN, nls_cp); + } + bcc_ptr += 2 * bytes_ret; + bcc_ptr += 2; /* account for null termination */ + + unicode_domain_string(&bcc_ptr, ses, nls_cp); + unicode_oslm_strings(&bcc_ptr, nls_cp); + + *pbcc_area = bcc_ptr; +} + +static void ascii_ssetup_strings(char **pbcc_area, struct cifs_ses *ses, + const struct nls_table *nls_cp) +{ + char *bcc_ptr = *pbcc_area; + int len; + + /* copy user */ + /* BB what about null user mounts - check that we do this BB */ + /* copy user */ + if (ses->user_name != NULL) { + len = strscpy(bcc_ptr, ses->user_name, CIFS_MAX_USERNAME_LEN); + if (WARN_ON_ONCE(len < 0)) + len = CIFS_MAX_USERNAME_LEN - 1; + bcc_ptr += len; + } + /* else null user mount */ + *bcc_ptr = 0; + bcc_ptr++; /* account for null termination */ + + /* BB check for overflow here */ + + ascii_domain_string(&bcc_ptr, ses, nls_cp); + ascii_oslm_strings(&bcc_ptr, nls_cp); + + *pbcc_area = bcc_ptr; +} + +static void +decode_unicode_ssetup(char **pbcc_area, int bleft, struct cifs_ses *ses, + const struct nls_table *nls_cp) +{ + int len; + char *data = *pbcc_area; + + cifs_dbg(FYI, "bleft %d\n", bleft); + + kfree(ses->serverOS); + ses->serverOS = cifs_strndup_from_utf16(data, bleft, true, nls_cp); + cifs_dbg(FYI, "serverOS=%s\n", ses->serverOS); + len = (UniStrnlen((wchar_t *) data, bleft / 2) * 2) + 2; + data += len; + bleft -= len; + if (bleft <= 0) + return; + + kfree(ses->serverNOS); + ses->serverNOS = cifs_strndup_from_utf16(data, bleft, true, nls_cp); + cifs_dbg(FYI, "serverNOS=%s\n", ses->serverNOS); + len = (UniStrnlen((wchar_t *) data, bleft / 2) * 2) + 2; + data += len; + bleft -= len; + if (bleft <= 0) + return; + + kfree(ses->serverDomain); + ses->serverDomain = cifs_strndup_from_utf16(data, bleft, true, nls_cp); + cifs_dbg(FYI, "serverDomain=%s\n", ses->serverDomain); + + return; +} + +static void decode_ascii_ssetup(char **pbcc_area, __u16 bleft, + struct cifs_ses *ses, + const struct nls_table *nls_cp) +{ + int len; + char *bcc_ptr = *pbcc_area; + + cifs_dbg(FYI, "decode sessetup ascii. bleft %d\n", bleft); + + len = strnlen(bcc_ptr, bleft); + if (len >= bleft) + return; + + kfree(ses->serverOS); + + ses->serverOS = kmalloc(len + 1, GFP_KERNEL); + if (ses->serverOS) { + memcpy(ses->serverOS, bcc_ptr, len); + ses->serverOS[len] = 0; + if (strncmp(ses->serverOS, "OS/2", 4) == 0) + cifs_dbg(FYI, "OS/2 server\n"); + } + + bcc_ptr += len + 1; + bleft -= len + 1; + + len = strnlen(bcc_ptr, bleft); + if (len >= bleft) + return; + + kfree(ses->serverNOS); + + ses->serverNOS = kmalloc(len + 1, GFP_KERNEL); + if (ses->serverNOS) { + memcpy(ses->serverNOS, bcc_ptr, len); + ses->serverNOS[len] = 0; + } + + bcc_ptr += len + 1; + bleft -= len + 1; + + len = strnlen(bcc_ptr, bleft); + if (len > bleft) + return; + + /* + * No domain field in LANMAN case. Domain is + * returned by old servers in the SMB negprot response + * + * BB For newer servers which do not support Unicode, + * but thus do return domain here, we could add parsing + * for it later, but it is not very important + */ + cifs_dbg(FYI, "ascii: bytes left %d\n", bleft); +} + +static int +sess_alloc_buffer(struct sess_data *sess_data, int wct) +{ + int rc; + struct cifs_ses *ses = sess_data->ses; + struct smb_hdr *smb_buf; + + rc = small_smb_init_no_tc(SMB_COM_SESSION_SETUP_ANDX, wct, ses, + (void **)&smb_buf); + + if (rc < 0) + return rc; + + sess_data->in_len = rc; + sess_data->iov[0].iov_base = (char *)smb_buf; + sess_data->iov[0].iov_len = sess_data->in_len; + /* + * This variable will be used to clear the buffer + * allocated above in case of any error in the calling function. + */ + sess_data->buf0_type = CIFS_SMALL_BUFFER; + + /* 2000 big enough to fit max user, domain, NOS name etc. */ + sess_data->iov[2].iov_base = kmalloc(2000, GFP_KERNEL); + if (!sess_data->iov[2].iov_base) { + rc = -ENOMEM; + goto out_free_smb_buf; + } + + return 0; + +out_free_smb_buf: + cifs_small_buf_release(smb_buf); + sess_data->iov[0].iov_base = NULL; + sess_data->iov[0].iov_len = 0; + sess_data->buf0_type = CIFS_NO_BUFFER; + return rc; +} + +static void +sess_free_buffer(struct sess_data *sess_data) +{ + struct kvec *iov = sess_data->iov; + + /* + * Zero the session data before freeing, as it might contain sensitive info (keys, etc). + * Note that iov[1] is already freed by caller. + */ + if (sess_data->buf0_type != CIFS_NO_BUFFER && iov[0].iov_base) + memzero_explicit(iov[0].iov_base, iov[0].iov_len); + + free_rsp_buf(sess_data->buf0_type, iov[0].iov_base); + sess_data->buf0_type = CIFS_NO_BUFFER; + kfree_sensitive(iov[2].iov_base); +} + +static int +sess_establish_session(struct sess_data *sess_data) +{ + struct cifs_ses *ses = sess_data->ses; + struct TCP_Server_Info *server = sess_data->server; + + cifs_server_lock(server); + if (!server->session_estab) { + if (server->sign) { + server->session_key.response = + kmemdup(ses->auth_key.response, + ses->auth_key.len, GFP_KERNEL); + if (!server->session_key.response) { + cifs_server_unlock(server); + return -ENOMEM; + } + server->session_key.len = + ses->auth_key.len; + } + server->sequence_number = 0x2; + server->session_estab = true; + } + cifs_server_unlock(server); + + cifs_dbg(FYI, "CIFS session established successfully\n"); + return 0; +} + +static int +sess_sendreceive(struct sess_data *sess_data) +{ + int rc; + struct smb_hdr *smb_buf = (struct smb_hdr *) sess_data->iov[0].iov_base; + __u16 count; + struct kvec rsp_iov = { NULL, 0 }; + + count = sess_data->iov[1].iov_len + sess_data->iov[2].iov_len; + sess_data->in_len += count; + put_bcc(count, smb_buf); + + rc = SendReceive2(sess_data->xid, sess_data->ses, + sess_data->iov, 3 /* num_iovecs */, + &sess_data->buf0_type, + CIFS_LOG_ERROR, &rsp_iov); + cifs_small_buf_release(sess_data->iov[0].iov_base); + memcpy(&sess_data->iov[0], &rsp_iov, sizeof(struct kvec)); + + return rc; +} + +static void +sess_auth_ntlmv2(struct sess_data *sess_data) +{ + int rc = 0; + struct smb_hdr *smb_buf; + SESSION_SETUP_ANDX *pSMB; + char *bcc_ptr; + struct cifs_ses *ses = sess_data->ses; + struct TCP_Server_Info *server = sess_data->server; + __u32 capabilities; + __u16 bytes_remaining; + + /* old style NTLM sessionsetup */ + /* wct = 13 */ + rc = sess_alloc_buffer(sess_data, 13); + if (rc) + goto out; + + pSMB = (SESSION_SETUP_ANDX *)sess_data->iov[0].iov_base; + bcc_ptr = sess_data->iov[2].iov_base; + capabilities = cifs_ssetup_hdr(ses, server, pSMB); + + pSMB->req_no_secext.Capabilities = cpu_to_le32(capabilities); + + /* LM2 password would be here if we supported it */ + pSMB->req_no_secext.CaseInsensitivePasswordLength = 0; + + if (ses->user_name != NULL) { + /* calculate nlmv2 response and session key */ + rc = setup_ntlmv2_rsp(ses, sess_data->nls_cp); + if (rc) { + cifs_dbg(VFS, "Error %d during NTLMv2 authentication\n", rc); + goto out; + } + + memcpy(bcc_ptr, ses->auth_key.response + CIFS_SESS_KEY_SIZE, + ses->auth_key.len - CIFS_SESS_KEY_SIZE); + bcc_ptr += ses->auth_key.len - CIFS_SESS_KEY_SIZE; + + /* set case sensitive password length after tilen may get + * assigned, tilen is 0 otherwise. + */ + pSMB->req_no_secext.CaseSensitivePasswordLength = + cpu_to_le16(ses->auth_key.len - CIFS_SESS_KEY_SIZE); + } else { + pSMB->req_no_secext.CaseSensitivePasswordLength = 0; + } + + if (ses->capabilities & CAP_UNICODE) { + if (!IS_ALIGNED(sess_data->iov[0].iov_len, 2)) { + *bcc_ptr = 0; + bcc_ptr++; + } + unicode_ssetup_strings(&bcc_ptr, ses, sess_data->nls_cp); + } else { + ascii_ssetup_strings(&bcc_ptr, ses, sess_data->nls_cp); + } + + + sess_data->iov[2].iov_len = (long) bcc_ptr - + (long) sess_data->iov[2].iov_base; + + rc = sess_sendreceive(sess_data); + if (rc) + goto out; + + pSMB = (SESSION_SETUP_ANDX *)sess_data->iov[0].iov_base; + smb_buf = (struct smb_hdr *)sess_data->iov[0].iov_base; + + if (smb_buf->WordCount != 3) { + rc = smb_EIO1(smb_eio_trace_sess_nl2_wcc, smb_buf->WordCount); + cifs_dbg(VFS, "bad word count %d\n", smb_buf->WordCount); + goto out; + } + + if (le16_to_cpu(pSMB->resp.Action) & GUEST_LOGIN) + cifs_dbg(FYI, "Guest login\n"); /* BB mark SesInfo struct? */ + + ses->Suid = smb_buf->Uid; /* UID left in wire format (le) */ + cifs_dbg(FYI, "UID = %llu\n", ses->Suid); + + bytes_remaining = get_bcc(smb_buf); + bcc_ptr = pByteArea(smb_buf); + + /* BB check if Unicode and decode strings */ + if (bytes_remaining == 0) { + /* no string area to decode, do nothing */ + } else if (smb_buf->Flags2 & SMBFLG2_UNICODE) { + /* unicode string area must be word-aligned */ + if (!IS_ALIGNED((unsigned long)bcc_ptr - (unsigned long)smb_buf, 2)) { + ++bcc_ptr; + --bytes_remaining; + } + decode_unicode_ssetup(&bcc_ptr, bytes_remaining, ses, + sess_data->nls_cp); + } else { + decode_ascii_ssetup(&bcc_ptr, bytes_remaining, ses, + sess_data->nls_cp); + } + + rc = sess_establish_session(sess_data); +out: + sess_data->result = rc; + sess_data->func = NULL; + sess_free_buffer(sess_data); + kfree_sensitive(ses->auth_key.response); + ses->auth_key.response = NULL; +} + +#ifdef CONFIG_CIFS_UPCALL +static void +sess_auth_kerberos(struct sess_data *sess_data) +{ + int rc = 0; + struct smb_hdr *smb_buf; + SESSION_SETUP_ANDX *pSMB; + char *bcc_ptr; + struct cifs_ses *ses = sess_data->ses; + struct TCP_Server_Info *server = sess_data->server; + __u32 capabilities; + __u16 bytes_remaining; + struct key *spnego_key = NULL; + struct cifs_spnego_msg *msg; + u16 blob_len; + + /* extended security */ + /* wct = 12 */ + rc = sess_alloc_buffer(sess_data, 12); + if (rc) + goto out; + + pSMB = (SESSION_SETUP_ANDX *)sess_data->iov[0].iov_base; + bcc_ptr = sess_data->iov[2].iov_base; + capabilities = cifs_ssetup_hdr(ses, server, pSMB); + + spnego_key = cifs_get_spnego_key(ses, server); + if (IS_ERR(spnego_key)) { + rc = PTR_ERR(spnego_key); + spnego_key = NULL; + goto out; + } + + msg = spnego_key->payload.data[0]; + /* + * check version field to make sure that cifs.upcall is + * sending us a response in an expected form + */ + if (msg->version != CIFS_SPNEGO_UPCALL_VERSION) { + cifs_dbg(VFS, "incorrect version of cifs.upcall (expected %d but got %d)\n", + CIFS_SPNEGO_UPCALL_VERSION, msg->version); + rc = -EKEYREJECTED; + goto out_put_spnego_key; + } + + kfree_sensitive(ses->auth_key.response); + ses->auth_key.response = kmemdup(msg->data, msg->sesskey_len, + GFP_KERNEL); + if (!ses->auth_key.response) { + cifs_dbg(VFS, "Kerberos can't allocate (%u bytes) memory\n", + msg->sesskey_len); + rc = -ENOMEM; + goto out_put_spnego_key; + } + ses->auth_key.len = msg->sesskey_len; + + pSMB->req.hdr.Flags2 |= SMBFLG2_EXT_SEC; + capabilities |= CAP_EXTENDED_SECURITY; + pSMB->req.Capabilities = cpu_to_le32(capabilities); + sess_data->iov[1].iov_base = msg->data + msg->sesskey_len; + sess_data->iov[1].iov_len = msg->secblob_len; + pSMB->req.SecurityBlobLength = cpu_to_le16(sess_data->iov[1].iov_len); + + if (pSMB->req.hdr.Flags2 & SMBFLG2_UNICODE) { + /* unicode strings must be word aligned */ + if (!IS_ALIGNED(sess_data->iov[0].iov_len + sess_data->iov[1].iov_len, 2)) { + *bcc_ptr = 0; + bcc_ptr++; + } + unicode_oslm_strings(&bcc_ptr, sess_data->nls_cp); + unicode_domain_string(&bcc_ptr, ses, sess_data->nls_cp); + } else { + ascii_oslm_strings(&bcc_ptr, sess_data->nls_cp); + ascii_domain_string(&bcc_ptr, ses, sess_data->nls_cp); + } + + sess_data->iov[2].iov_len = (long) bcc_ptr - + (long) sess_data->iov[2].iov_base; + + rc = sess_sendreceive(sess_data); + if (rc) + goto out_put_spnego_key; + + pSMB = (SESSION_SETUP_ANDX *)sess_data->iov[0].iov_base; + smb_buf = (struct smb_hdr *)sess_data->iov[0].iov_base; + + if (smb_buf->WordCount != 4) { + rc = smb_EIO1(smb_eio_trace_sess_krb_wcc, smb_buf->WordCount); + cifs_dbg(VFS, "bad word count %d\n", smb_buf->WordCount); + goto out_put_spnego_key; + } + + if (le16_to_cpu(pSMB->resp.Action) & GUEST_LOGIN) + cifs_dbg(FYI, "Guest login\n"); /* BB mark SesInfo struct? */ + + ses->Suid = smb_buf->Uid; /* UID left in wire format (le) */ + cifs_dbg(FYI, "UID = %llu\n", ses->Suid); + + bytes_remaining = get_bcc(smb_buf); + bcc_ptr = pByteArea(smb_buf); + + blob_len = le16_to_cpu(pSMB->resp.SecurityBlobLength); + if (blob_len > bytes_remaining) { + cifs_dbg(VFS, "bad security blob length %d\n", + blob_len); + rc = -EINVAL; + goto out_put_spnego_key; + } + bcc_ptr += blob_len; + bytes_remaining -= blob_len; + + /* BB check if Unicode and decode strings */ + if (bytes_remaining == 0) { + /* no string area to decode, do nothing */ + } else if (smb_buf->Flags2 & SMBFLG2_UNICODE) { + /* unicode string area must be word-aligned */ + if (!IS_ALIGNED((unsigned long)bcc_ptr - (unsigned long)smb_buf, 2)) { + ++bcc_ptr; + --bytes_remaining; + } + decode_unicode_ssetup(&bcc_ptr, bytes_remaining, ses, + sess_data->nls_cp); + } else { + decode_ascii_ssetup(&bcc_ptr, bytes_remaining, ses, + sess_data->nls_cp); + } + + rc = sess_establish_session(sess_data); +out_put_spnego_key: + key_invalidate(spnego_key); + key_put(spnego_key); +out: + sess_data->result = rc; + sess_data->func = NULL; + sess_free_buffer(sess_data); + kfree_sensitive(ses->auth_key.response); + ses->auth_key.response = NULL; +} + +#endif /* ! CONFIG_CIFS_UPCALL */ + +/* + * The required kvec buffers have to be allocated before calling this + * function. + */ +static int +_sess_auth_rawntlmssp_assemble_req(struct sess_data *sess_data) +{ + SESSION_SETUP_ANDX *pSMB; + struct cifs_ses *ses = sess_data->ses; + struct TCP_Server_Info *server = sess_data->server; + __u32 capabilities; + char *bcc_ptr; + + pSMB = (SESSION_SETUP_ANDX *)sess_data->iov[0].iov_base; + + capabilities = cifs_ssetup_hdr(ses, server, pSMB); + pSMB->req.hdr.Flags2 |= SMBFLG2_EXT_SEC; + capabilities |= CAP_EXTENDED_SECURITY; + pSMB->req.Capabilities |= cpu_to_le32(capabilities); + + bcc_ptr = sess_data->iov[2].iov_base; + + if (pSMB->req.hdr.Flags2 & SMBFLG2_UNICODE) { + /* unicode strings must be word aligned */ + if (!IS_ALIGNED(sess_data->iov[0].iov_len + sess_data->iov[1].iov_len, 2)) { + *bcc_ptr = 0; + bcc_ptr++; + } + unicode_oslm_strings(&bcc_ptr, sess_data->nls_cp); + } else { + ascii_oslm_strings(&bcc_ptr, sess_data->nls_cp); + } + + sess_data->iov[2].iov_len = (long) bcc_ptr - + (long) sess_data->iov[2].iov_base; + + return 0; +} + +static void +sess_auth_rawntlmssp_authenticate(struct sess_data *sess_data); + +static void +sess_auth_rawntlmssp_negotiate(struct sess_data *sess_data) +{ + int rc; + struct smb_hdr *smb_buf; + SESSION_SETUP_ANDX *pSMB; + struct cifs_ses *ses = sess_data->ses; + struct TCP_Server_Info *server = sess_data->server; + __u16 bytes_remaining; + char *bcc_ptr; + unsigned char *ntlmsspblob = NULL; + u16 blob_len; + + cifs_dbg(FYI, "rawntlmssp session setup negotiate phase\n"); + + /* + * if memory allocation is successful, caller of this function + * frees it. + */ + ses->ntlmssp = kmalloc_obj(struct ntlmssp_auth); + if (!ses->ntlmssp) { + rc = -ENOMEM; + goto out; + } + ses->ntlmssp->sesskey_per_smbsess = false; + + /* wct = 12 */ + rc = sess_alloc_buffer(sess_data, 12); + if (rc) + goto out; + + pSMB = (SESSION_SETUP_ANDX *)sess_data->iov[0].iov_base; + + /* Build security blob before we assemble the request */ + rc = build_ntlmssp_negotiate_blob(&ntlmsspblob, + &blob_len, ses, server, + sess_data->nls_cp); + if (rc) + goto out_free_ntlmsspblob; + + sess_data->iov[1].iov_len = blob_len; + sess_data->iov[1].iov_base = ntlmsspblob; + pSMB->req.SecurityBlobLength = cpu_to_le16(blob_len); + + rc = _sess_auth_rawntlmssp_assemble_req(sess_data); + if (rc) + goto out_free_ntlmsspblob; + + rc = sess_sendreceive(sess_data); + + pSMB = (SESSION_SETUP_ANDX *)sess_data->iov[0].iov_base; + smb_buf = (struct smb_hdr *)sess_data->iov[0].iov_base; + + /* If true, rc here is expected and not an error */ + if (sess_data->buf0_type != CIFS_NO_BUFFER && + smb_buf->Status.CifsError == + cpu_to_le32(NT_STATUS_MORE_PROCESSING_REQUIRED)) + rc = 0; + + if (rc) + goto out_free_ntlmsspblob; + + cifs_dbg(FYI, "rawntlmssp session setup challenge phase\n"); + + if (smb_buf->WordCount != 4) { + rc = smb_EIO1(smb_eio_trace_sess_rawnl_neg_wcc, smb_buf->WordCount); + cifs_dbg(VFS, "bad word count %d\n", smb_buf->WordCount); + goto out_free_ntlmsspblob; + } + + ses->Suid = smb_buf->Uid; /* UID left in wire format (le) */ + cifs_dbg(FYI, "UID = %llu\n", ses->Suid); + + bytes_remaining = get_bcc(smb_buf); + bcc_ptr = pByteArea(smb_buf); + + blob_len = le16_to_cpu(pSMB->resp.SecurityBlobLength); + if (blob_len > bytes_remaining) { + cifs_dbg(VFS, "bad security blob length %d\n", + blob_len); + rc = -EINVAL; + goto out_free_ntlmsspblob; + } + + rc = decode_ntlmssp_challenge(bcc_ptr, blob_len, ses); + +out_free_ntlmsspblob: + kfree_sensitive(ntlmsspblob); +out: + sess_free_buffer(sess_data); + + if (!rc) { + sess_data->func = sess_auth_rawntlmssp_authenticate; + return; + } + + /* Else error. Cleanup */ + kfree_sensitive(ses->auth_key.response); + ses->auth_key.response = NULL; + kfree_sensitive(ses->ntlmssp); + ses->ntlmssp = NULL; + + sess_data->func = NULL; + sess_data->result = rc; +} + +static void +sess_auth_rawntlmssp_authenticate(struct sess_data *sess_data) +{ + int rc; + struct smb_hdr *smb_buf; + SESSION_SETUP_ANDX *pSMB; + struct cifs_ses *ses = sess_data->ses; + struct TCP_Server_Info *server = sess_data->server; + __u16 bytes_remaining; + char *bcc_ptr; + unsigned char *ntlmsspblob = NULL; + u16 blob_len; + + cifs_dbg(FYI, "rawntlmssp session setup authenticate phase\n"); + + /* wct = 12 */ + rc = sess_alloc_buffer(sess_data, 12); + if (rc) + goto out; + + /* Build security blob before we assemble the request */ + pSMB = (SESSION_SETUP_ANDX *)sess_data->iov[0].iov_base; + smb_buf = (struct smb_hdr *)pSMB; + rc = build_ntlmssp_auth_blob(&ntlmsspblob, + &blob_len, ses, server, + sess_data->nls_cp); + if (rc) + goto out_free_ntlmsspblob; + sess_data->iov[1].iov_len = blob_len; + sess_data->iov[1].iov_base = ntlmsspblob; + pSMB->req.SecurityBlobLength = cpu_to_le16(blob_len); + /* + * Make sure that we tell the server that we are using + * the uid that it just gave us back on the response + * (challenge) + */ + smb_buf->Uid = ses->Suid; + + rc = _sess_auth_rawntlmssp_assemble_req(sess_data); + if (rc) + goto out_free_ntlmsspblob; + + rc = sess_sendreceive(sess_data); + if (rc) + goto out_free_ntlmsspblob; + + pSMB = (SESSION_SETUP_ANDX *)sess_data->iov[0].iov_base; + smb_buf = (struct smb_hdr *)sess_data->iov[0].iov_base; + if (smb_buf->WordCount != 4) { + rc = smb_EIO1(smb_eio_trace_sess_rawnl_auth_wcc, smb_buf->WordCount); + cifs_dbg(VFS, "bad word count %d\n", smb_buf->WordCount); + goto out_free_ntlmsspblob; + } + + if (le16_to_cpu(pSMB->resp.Action) & GUEST_LOGIN) + cifs_dbg(FYI, "Guest login\n"); /* BB mark SesInfo struct? */ + + if (ses->Suid != smb_buf->Uid) { + ses->Suid = smb_buf->Uid; + cifs_dbg(FYI, "UID changed! new UID = %llu\n", ses->Suid); + } + + bytes_remaining = get_bcc(smb_buf); + bcc_ptr = pByteArea(smb_buf); + blob_len = le16_to_cpu(pSMB->resp.SecurityBlobLength); + if (blob_len > bytes_remaining) { + cifs_dbg(VFS, "bad security blob length %d\n", + blob_len); + rc = -EINVAL; + goto out_free_ntlmsspblob; + } + bcc_ptr += blob_len; + bytes_remaining -= blob_len; + + + /* BB check if Unicode and decode strings */ + if (bytes_remaining == 0) { + /* no string area to decode, do nothing */ + } else if (smb_buf->Flags2 & SMBFLG2_UNICODE) { + /* unicode string area must be word-aligned */ + if (!IS_ALIGNED((unsigned long)bcc_ptr - (unsigned long)smb_buf, 2)) { + ++bcc_ptr; + --bytes_remaining; + } + decode_unicode_ssetup(&bcc_ptr, bytes_remaining, ses, + sess_data->nls_cp); + } else { + decode_ascii_ssetup(&bcc_ptr, bytes_remaining, ses, + sess_data->nls_cp); + } + +out_free_ntlmsspblob: + kfree_sensitive(ntlmsspblob); +out: + sess_free_buffer(sess_data); + + if (!rc) + rc = sess_establish_session(sess_data); + + /* Cleanup */ + kfree_sensitive(ses->auth_key.response); + ses->auth_key.response = NULL; + kfree_sensitive(ses->ntlmssp); + ses->ntlmssp = NULL; + + sess_data->func = NULL; + sess_data->result = rc; +} + +static int select_sec(struct sess_data *sess_data) +{ + int type; + struct cifs_ses *ses = sess_data->ses; + struct TCP_Server_Info *server = sess_data->server; + + type = cifs_select_sectype(server, ses->sectype); + cifs_dbg(FYI, "sess setup type %d\n", type); + if (type == Unspecified) { + cifs_dbg(VFS, "Unable to select appropriate authentication method!\n"); + return -EINVAL; + } + + switch (type) { + case NTLMv2: + sess_data->func = sess_auth_ntlmv2; + break; + case Kerberos: +#ifdef CONFIG_CIFS_UPCALL + sess_data->func = sess_auth_kerberos; + break; +#else + cifs_dbg(VFS, "Kerberos negotiated but upcall support disabled!\n"); + return -ENOSYS; +#endif /* CONFIG_CIFS_UPCALL */ + case RawNTLMSSP: + sess_data->func = sess_auth_rawntlmssp_negotiate; + break; + default: + cifs_dbg(VFS, "secType %d not supported!\n", type); + return -ENOSYS; + } + + return 0; +} + +int CIFS_SessSetup(const unsigned int xid, struct cifs_ses *ses, + struct TCP_Server_Info *server, + const struct nls_table *nls_cp) +{ + int rc = 0; + struct sess_data *sess_data; + + if (ses == NULL) { + WARN(1, "%s: ses == NULL!", __func__); + return -EINVAL; + } + + sess_data = kzalloc_obj(struct sess_data); + if (!sess_data) + return -ENOMEM; + + sess_data->xid = xid; + sess_data->ses = ses; + sess_data->server = server; + sess_data->buf0_type = CIFS_NO_BUFFER; + sess_data->nls_cp = (struct nls_table *) nls_cp; + + rc = select_sec(sess_data); + if (rc) + goto out; + + while (sess_data->func) + sess_data->func(sess_data); + + /* Store result before we free sess_data */ + rc = sess_data->result; + +out: + kfree_sensitive(sess_data); + return rc; +} diff --git a/fs/smb/client/smb1transport.c b/fs/smb/client/smb1transport.c new file mode 100644 index 000000000..966f2cf83 --- /dev/null +++ b/fs/smb/client/smb1transport.c @@ -0,0 +1,584 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * Copyright (C) International Business Machines Corp., 2002,2008 + * Author(s): Steve French (sfrench@us.ibm.com) + * Jeremy Allison (jra@samba.org) 2006. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "cifsglob.h" +#include "cifsproto.h" +#include "smb1proto.h" +#include "smb2proto.h" +#include "cifs_debug.h" +#include "smbdirect.h" +#include "compress.h" + +/* Max number of iovectors we can use off the stack when sending requests. */ +#define CIFS_MAX_IOV_SIZE 8 + +static struct mid_q_entry * +alloc_mid(const struct smb_hdr *smb_buffer, struct TCP_Server_Info *server) +{ + struct mid_q_entry *temp; + + if (server == NULL) { + cifs_dbg(VFS, "%s: null TCP session\n", __func__); + return NULL; + } + + temp = mempool_alloc(&cifs_mid_pool, GFP_NOFS); + memset(temp, 0, sizeof(struct mid_q_entry)); + refcount_set(&temp->refcount, 1); + spin_lock_init(&temp->mid_lock); + temp->mid = get_mid(smb_buffer); + temp->pid = current->pid; + temp->command = cpu_to_le16(smb_buffer->Command); + cifs_dbg(FYI, "For smb_command %d\n", smb_buffer->Command); + /* easier to use jiffies */ + /* when mid allocated can be before when sent */ + temp->when_alloc = jiffies; + + /* + * The default is for the mid to be synchronous, so the + * default callback just wakes up the current task. + */ + get_task_struct(current); + temp->creator = current; + temp->callback = cifs_wake_up_task; + temp->callback_data = current; + + atomic_inc(&mid_count); + temp->mid_state = MID_REQUEST_ALLOCATED; + return temp; +} + +static int allocate_mid(struct cifs_ses *ses, struct smb_hdr *in_buf, + struct mid_q_entry **ppmidQ) +{ + spin_lock(&ses->ses_lock); + if (ses->ses_status == SES_NEW) { + if ((in_buf->Command != SMB_COM_SESSION_SETUP_ANDX) && + (in_buf->Command != SMB_COM_NEGOTIATE)) { + spin_unlock(&ses->ses_lock); + return -EAGAIN; + } + /* else ok - we are setting up session */ + } + + if (ses->ses_status == SES_EXITING) { + /* check if SMB session is bad because we are setting it up */ + if (in_buf->Command != SMB_COM_LOGOFF_ANDX) { + spin_unlock(&ses->ses_lock); + return -EAGAIN; + } + /* else ok - we are shutting down session */ + } + spin_unlock(&ses->ses_lock); + + *ppmidQ = alloc_mid(in_buf, ses->server); + if (*ppmidQ == NULL) + return -ENOMEM; + spin_lock(&ses->server->mid_queue_lock); + list_add_tail(&(*ppmidQ)->qhead, &ses->server->pending_mid_q); + spin_unlock(&ses->server->mid_queue_lock); + return 0; +} + +struct mid_q_entry * +cifs_setup_async_request(struct TCP_Server_Info *server, struct smb_rqst *rqst) +{ + int rc; + struct smb_hdr *hdr = (struct smb_hdr *)rqst->rq_iov[0].iov_base; + struct mid_q_entry *mid; + + /* enable signing if server requires it */ + if (server->sign) + hdr->Flags2 |= SMBFLG2_SECURITY_SIGNATURE; + + mid = alloc_mid(hdr, server); + if (mid == NULL) + return ERR_PTR(-ENOMEM); + + rc = cifs_sign_rqst(rqst, server, &mid->sequence_number); + if (rc) { + release_mid(server, mid); + return ERR_PTR(rc); + } + + return mid; +} + +/* + * + * Send an SMB Request. No response info (other than return code) + * needs to be parsed. + * + * flags indicate the type of request buffer and how long to wait + * and whether to log NT STATUS code (error) before mapping it to POSIX error + * + */ +int +SendReceiveNoRsp(const unsigned int xid, struct cifs_ses *ses, + char *in_buf, unsigned int in_len, int flags) +{ + int rc; + struct kvec iov[1]; + struct kvec rsp_iov; + int resp_buf_type; + + iov[0].iov_base = in_buf; + iov[0].iov_len = in_len; + flags |= CIFS_NO_RSP_BUF; + rc = SendReceive2(xid, ses, iov, 1, &resp_buf_type, flags, &rsp_iov); + cifs_dbg(NOISY, "SendRcvNoRsp flags %d rc %d\n", flags, rc); + + return rc; +} + +int +cifs_check_receive(struct mid_q_entry *mid, struct TCP_Server_Info *server, + bool log_error) +{ + unsigned int len = mid->response_pdu_len; + + dump_smb(mid->resp_buf, min_t(u32, 92, len)); + + /* convert the length into a more usable form */ + if (server->sign) { + struct kvec iov[1]; + int rc = 0; + struct smb_rqst rqst = { .rq_iov = iov, + .rq_nvec = ARRAY_SIZE(iov) }; + + iov[0].iov_base = mid->resp_buf; + iov[0].iov_len = len; + + rc = cifs_verify_signature(&rqst, server, + mid->sequence_number); + if (rc) { + cifs_server_dbg(VFS, "SMB signature verification returned error = %d\n", + rc); + + if (!(server->sec_mode & SECMODE_SIGN_REQUIRED)) { + cifs_reconnect(server, true); + return rc; + } + } + } + + /* BB special case reconnect tid and uid here? */ + return map_and_check_smb_error(server, mid, log_error); +} + +struct mid_q_entry * +cifs_setup_request(struct cifs_ses *ses, struct TCP_Server_Info *server, + struct smb_rqst *rqst) +{ + int rc; + struct smb_hdr *hdr = (struct smb_hdr *)rqst->rq_iov[0].iov_base; + struct mid_q_entry *mid; + + rc = allocate_mid(ses, hdr, &mid); + if (rc) + return ERR_PTR(rc); + rc = cifs_sign_rqst(rqst, server, &mid->sequence_number); + if (rc) { + delete_mid(server, mid); + return ERR_PTR(rc); + } + return mid; +} + +int +SendReceive2(const unsigned int xid, struct cifs_ses *ses, + struct kvec *iov, int n_vec, int *resp_buf_type /* ret */, + const int flags, struct kvec *resp_iov) +{ + struct smb_rqst rqst = { + .rq_iov = iov, + .rq_nvec = n_vec, + }; + + return cifs_send_recv(xid, ses, ses->server, + &rqst, resp_buf_type, flags, resp_iov); +} + +int +SendReceive(const unsigned int xid, struct cifs_ses *ses, + struct smb_hdr *in_buf, unsigned int in_len, + struct smb_hdr *out_buf, int *pbytes_returned, const int flags) +{ + struct TCP_Server_Info *server; + struct kvec resp_iov = {}; + struct kvec iov = { .iov_base = in_buf, .iov_len = in_len }; + struct smb_rqst rqst = { .rq_iov = &iov, .rq_nvec = 1 }; + int resp_buf_type; + int rc = 0; + + if (WARN_ON_ONCE(in_len > 0xffffff)) + return smb_EIO1(smb_eio_trace_tx_too_long, in_len); + if (ses == NULL) { + cifs_dbg(VFS, "Null smb session\n"); + return smb_EIO(smb_eio_trace_null_pointers); + } + server = ses->server; + if (server == NULL) { + cifs_dbg(VFS, "Null tcp session\n"); + return smb_EIO(smb_eio_trace_null_pointers); + } + + /* Ensure that we do not send more than 50 overlapping requests + to the same server. We may make this configurable later or + use ses->maxReq */ + + if (in_len > CIFSMaxBufSize + MAX_CIFS_HDR_SIZE) { + cifs_server_dbg(VFS, "Invalid length, greater than maximum frame, %d\n", + in_len); + return smb_EIO1(smb_eio_trace_tx_too_long, in_len); + } + + rc = cifs_send_recv(xid, ses, ses->server, + &rqst, &resp_buf_type, flags, &resp_iov); + if (rc < 0) + goto out; + + if (out_buf) { + /* Use smbCalcSize() for both single- and multi-part T2 responses, + * both here and in coalesce_t2(). + */ + unsigned int copy_len; + if (WARN_ON_ONCE(!resp_iov.iov_base)) { + rc = -EIO; + goto out; + } + copy_len = smbCalcSize(resp_iov.iov_base); + if (copy_len > CIFSMaxBufSize + MAX_CIFS_HDR_SIZE) { + cifs_dbg(VFS, "response size %u exceeds buffer\n", + copy_len); + rc = -ENOBUFS; + goto out; + } + *pbytes_returned = copy_len; + memcpy(out_buf, resp_iov.iov_base, copy_len); + } + +out: + free_rsp_buf(resp_buf_type, resp_iov.iov_base); + return rc; +} + +/* + return codes: + 0 not a transact2, or all data present + >0 transact2 with that much data missing + -EINVAL invalid transact2 + */ +static int +check2ndT2(char *buf) +{ + struct smb_hdr *pSMB = (struct smb_hdr *)buf; + struct smb_t2_rsp *pSMBt; + int remaining; + __u16 total_data_size, data_in_this_rsp; + + if (pSMB->Command != SMB_COM_TRANSACTION2) + return 0; + + /* check for plausible wct, bcc and t2 data and parm sizes */ + /* check for parm and data offset going beyond end of smb */ + if (pSMB->WordCount != 10) { /* coalesce_t2 depends on this */ + cifs_dbg(FYI, "Invalid transact2 word count\n"); + return -EINVAL; + } + + pSMBt = (struct smb_t2_rsp *)pSMB; + + total_data_size = get_unaligned_le16(&pSMBt->t2_rsp.TotalDataCount); + data_in_this_rsp = get_unaligned_le16(&pSMBt->t2_rsp.DataCount); + + if (total_data_size == data_in_this_rsp) + return 0; + else if (total_data_size < data_in_this_rsp) { + cifs_dbg(FYI, "total data %d smaller than data in frame %d\n", + total_data_size, data_in_this_rsp); + return -EINVAL; + } + + remaining = total_data_size - data_in_this_rsp; + + cifs_dbg(FYI, "missing %d bytes from transact2, check next response\n", + remaining); + if (total_data_size > CIFSMaxBufSize) { + cifs_dbg(VFS, "TotalDataSize %d is over maximum buffer %d\n", + total_data_size, CIFSMaxBufSize); + return -EINVAL; + } + return remaining; +} + +static int +coalesce_t2(char *second_buf, struct smb_hdr *target_hdr, unsigned int *pdu_len) +{ + struct smb_t2_rsp *pSMBs = (struct smb_t2_rsp *)second_buf; + struct smb_t2_rsp *pSMBt = (struct smb_t2_rsp *)target_hdr; + char *data_area_of_tgt; + char *data_area_of_src; + int remaining; + unsigned int byte_count, total_in_tgt; + __u16 tgt_total_cnt, src_total_cnt, total_in_src; + + src_total_cnt = get_unaligned_le16(&pSMBs->t2_rsp.TotalDataCount); + tgt_total_cnt = get_unaligned_le16(&pSMBt->t2_rsp.TotalDataCount); + + if (tgt_total_cnt != src_total_cnt) + cifs_dbg(FYI, "total data count of primary and secondary t2 differ source=%hu target=%hu\n", + src_total_cnt, tgt_total_cnt); + + total_in_tgt = get_unaligned_le16(&pSMBt->t2_rsp.DataCount); + + remaining = tgt_total_cnt - total_in_tgt; + + if (remaining < 0) { + cifs_dbg(FYI, "Server sent too much data. tgt_total_cnt=%hu total_in_tgt=%u\n", + tgt_total_cnt, total_in_tgt); + return -EPROTO; + } + + if (remaining == 0) { + /* nothing to do, ignore */ + cifs_dbg(FYI, "no more data remains\n"); + return 0; + } + + total_in_src = get_unaligned_le16(&pSMBs->t2_rsp.DataCount); + if (remaining < total_in_src) + cifs_dbg(FYI, "transact2 2nd response contains too much data\n"); + + /* find end of first SMB data area */ + data_area_of_tgt = (char *)&pSMBt->hdr.Protocol + + get_unaligned_le16(&pSMBt->t2_rsp.DataOffset); + + /* validate target area */ + data_area_of_src = (char *)&pSMBs->hdr.Protocol + + get_unaligned_le16(&pSMBs->t2_rsp.DataOffset); + + data_area_of_tgt += total_in_tgt; + + total_in_tgt += total_in_src; + /* is the result too big for the field? */ + if (total_in_tgt > USHRT_MAX) { + cifs_dbg(FYI, "coalesced DataCount too large (%u)\n", + total_in_tgt); + return -EPROTO; + } + put_unaligned_le16(total_in_tgt, &pSMBt->t2_rsp.DataCount); + + /* fix up the BCC */ + byte_count = get_bcc(target_hdr); + byte_count += total_in_src; + /* is the result too big for the field? */ + if (byte_count > USHRT_MAX) { + cifs_dbg(FYI, "coalesced BCC too large (%u)\n", byte_count); + return -EPROTO; + } + put_bcc(byte_count, target_hdr); + + /* use smbCalcSize() rather than *pdu_len: the demux loop resets + * *pdu_len to each secondary's pdu_length, making it unreliable. + */ + byte_count = smbCalcSize(target_hdr); + /* don't allow buffer to overflow */ + if (byte_count > CIFSMaxBufSize + MAX_CIFS_HDR_SIZE) { + cifs_dbg(FYI, "coalesced size exceeds buffer size (%u)\n", + byte_count); + return -ENOBUFS; + } + *pdu_len = byte_count; + + /* copy second buffer into end of first buffer */ + memcpy(data_area_of_tgt, data_area_of_src, total_in_src); + + if (remaining != total_in_src) { + /* more responses to go */ + cifs_dbg(FYI, "waiting for more secondary responses\n"); + return 1; + } + + /* we are done */ + cifs_dbg(FYI, "found the last secondary response\n"); + return 0; +} + +bool +cifs_check_trans2(struct mid_q_entry *mid, struct TCP_Server_Info *server, + char *buf, int malformed) +{ + if (malformed) + return false; + if (check2ndT2(buf) <= 0) + return false; + mid->multiRsp = true; + if (mid->resp_buf) { + /* merge response - fix up 1st*/ + malformed = coalesce_t2(buf, mid->resp_buf, &mid->response_pdu_len); + if (malformed > 0) + return true; + /* All parts received or packet is malformed. */ + mid->multiEnd = true; + dequeue_mid(server, mid, malformed); + return true; + } + if (!server->large_buf) { + /*FIXME: switch to already allocated largebuf?*/ + cifs_dbg(VFS, "1st trans2 resp needs bigbuf\n"); + } else { + /* Have first buffer */ + mid->resp_buf = buf; + mid->large_buf = true; + server->bigbuf = NULL; + } + return true; +} + +static int +check_smb_hdr(struct smb_hdr *smb) +{ + /* does it have the right SMB "signature" ? */ + if (*(__le32 *) smb->Protocol != SMB1_PROTO_NUMBER) { + cifs_dbg(VFS, "Bad protocol string signature header 0x%x\n", + *(unsigned int *)smb->Protocol); + return 1; + } + + /* if it's a response then accept */ + if (smb->Flags & SMBFLG_RESPONSE) + return 0; + + /* only one valid case where server sends us request */ + if (smb->Command == SMB_COM_LOCKING_ANDX) + return 0; + + /* + * Windows NT server returns error response (e.g. STATUS_DELETE_PENDING + * or STATUS_OBJECT_NAME_NOT_FOUND or ERRDOS/ERRbadfile or any other) + * for some TRANS2 requests without the RESPONSE flag set in header. + */ + if (smb->Command == SMB_COM_TRANSACTION2 && smb->Status.CifsError != 0) + return 0; + + cifs_dbg(VFS, "Server sent request, not response. mid=%u\n", + get_mid(smb)); + return 1; +} + +int +checkSMB(char *buf, unsigned int pdu_len, unsigned int total_read, + struct TCP_Server_Info *server) +{ + struct smb_hdr *smb = (struct smb_hdr *)buf; + __u32 rfclen = pdu_len; + __u32 clc_len; /* calculated length */ + cifs_dbg(FYI, "checkSMB Length: 0x%x, smb_buf_length: 0x%x\n", + total_read, rfclen); + + /* is this frame too small to even get to a BCC? */ + if (total_read < 2 + sizeof(struct smb_hdr)) { + if ((total_read >= sizeof(struct smb_hdr) - 1) + && (smb->Status.CifsError != 0)) { + /* it's an error return */ + smb->WordCount = 0; + /* some error cases do not return wct and bcc */ + return 0; + } else if ((total_read == sizeof(struct smb_hdr) + 1) && + (smb->WordCount == 0)) { + char *tmp = (char *)smb; + /* Need to work around a bug in two servers here */ + /* First, check if the part of bcc they sent was zero */ + if (tmp[sizeof(struct smb_hdr)] == 0) { + /* some servers return only half of bcc + * on simple responses (wct, bcc both zero) + * in particular have seen this on + * ulogoffX and FindClose. This leaves + * one byte of bcc potentially uninitialized + */ + /* zero rest of bcc */ + tmp[sizeof(struct smb_hdr)+1] = 0; + return 0; + } + cifs_dbg(VFS, "rcvd invalid byte count (bcc)\n"); + return smb_EIO1(smb_eio_trace_rx_inv_bcc, tmp[sizeof(struct smb_hdr)]); + } else { + cifs_dbg(VFS, "Length less than smb header size\n"); + return smb_EIO2(smb_eio_trace_rx_too_short, + total_read, smb->WordCount); + } + } else if (total_read < sizeof(*smb) + 2 * smb->WordCount) { + cifs_dbg(VFS, "%s: can't read BCC due to invalid WordCount(%u)\n", + __func__, smb->WordCount); + return smb_EIO2(smb_eio_trace_rx_check_rsp, + total_read, 2 + sizeof(struct smb_hdr)); + } + + /* otherwise, there is enough to get to the BCC */ + if (check_smb_hdr(smb)) + return smb_EIO1(smb_eio_trace_rx_rfc1002_magic, *(u32 *)smb->Protocol); + clc_len = smbCalcSize(smb); + + if (rfclen != total_read) { + cifs_dbg(VFS, "Length read does not match RFC1001 length %d/%d\n", + rfclen, total_read); + return smb_EIO2(smb_eio_trace_rx_check_rsp, + total_read, rfclen); + } + + if (rfclen != clc_len) { + __u16 mid = get_mid(smb); + /* check if bcc wrapped around for large read responses */ + if ((rfclen > 64 * 1024) && (rfclen > clc_len)) { + /* check if lengths match mod 64K */ + if (((rfclen) & 0xFFFF) == (clc_len & 0xFFFF)) + return 0; /* bcc wrapped */ + } + cifs_dbg(FYI, "Calculated size %u vs length %u mismatch for mid=%u\n", + clc_len, rfclen, mid); + + if (rfclen < clc_len) { + cifs_dbg(VFS, "RFC1001 size %u smaller than SMB for mid=%u\n", + rfclen, mid); + return smb_EIO2(smb_eio_trace_rx_calc_len_too_big, + rfclen, clc_len); + } else if (rfclen > clc_len + 512) { + /* + * Some servers (Windows XP in particular) send more + * data than the lengths in the SMB packet would + * indicate on certain calls (byte range locks and + * trans2 find first calls in particular). While the + * client can handle such a frame by ignoring the + * trailing data, we choose limit the amount of extra + * data to 512 bytes. + */ + cifs_dbg(VFS, "RFC1001 size %u more than 512 bytes larger than SMB for mid=%u\n", + rfclen, mid); + return smb_EIO2(smb_eio_trace_rx_overlong, + rfclen, clc_len + 512); + } + } + return 0; +} diff --git a/fs/smb/client/smb2file.c b/fs/smb/client/smb2file.c new file mode 100644 index 000000000..f35b6488d --- /dev/null +++ b/fs/smb/client/smb2file.c @@ -0,0 +1,467 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * Copyright (C) International Business Machines Corp., 2002, 2011 + * Author(s): Steve French (sfrench@us.ibm.com), + * Pavel Shilovsky ((pshilovsky@samba.org) 2012 + * + */ +#include +#include +#include +#include +#include +#include +#include "cifsfs.h" +#include "cifsglob.h" +#include "cifsproto.h" +#include "cifs_debug.h" +#include "cifs_fs_sb.h" +#include "cifs_unicode.h" +#include "fscache.h" +#include "smb2proto.h" +#include "../common/smb2status.h" +#include "../common/smbfsctl.h" + +static struct smb2_symlink_err_rsp *symlink_data(const struct kvec *iov) +{ + struct smb2_err_rsp *err = iov->iov_base; + struct smb2_symlink_err_rsp *sym = ERR_PTR(-EINVAL); + u8 *end = (u8 *)err + iov->iov_len; + u32 len; + + /* + * Per [MS-SMB2] section 2.2.2, a STATUS_STOPPED_ON_SYMLINK response has to + * carry a Symbolic Link Error Response, so ByteCount cannot be zero. Some + * servers (e.g. the macOS built-in SMB server) violate this and return an + * empty error response, with both ErrorContextCount and ByteCount set to + * zero, i.e. without the symlink target. Detect this and return -ENODATA + * so that callers can tell "server did not send the target" apart from a + * malformed response, and retrieve the target with FSCTL_GET_REPARSE_POINT + * instead. + */ + if (!err->ErrorContextCount && !le32_to_cpu(err->ByteCount)) + return ERR_PTR(-ENODATA); + + if (err->ErrorContextCount) { + struct smb2_error_context_rsp *p; + + len = (u32)err->ErrorContextCount * (offsetof(struct smb2_error_context_rsp, + ErrorContextData) + + sizeof(struct smb2_symlink_err_rsp)); + if (le32_to_cpu(err->ByteCount) < len || iov->iov_len < len + sizeof(*err) + 1) + return ERR_PTR(-EINVAL); + + p = (struct smb2_error_context_rsp *)err->ErrorData; + while ((u8 *)p + sizeof(*p) <= end) { + if (le32_to_cpu(p->ErrorId) == SMB2_ERROR_ID_DEFAULT) { + sym = (struct smb2_symlink_err_rsp *)p->ErrorContextData; + break; + } + cifs_dbg(FYI, "%s: skipping unhandled error context: 0x%x\n", + __func__, le32_to_cpu(p->ErrorId)); + + len = ALIGN(le32_to_cpu(p->ErrorDataLength), 8); + if (len > end - ((u8 *)p + sizeof(*p))) + return ERR_PTR(-EINVAL); + + p = (struct smb2_error_context_rsp *)(p->ErrorContextData + len); + } + } else if (le32_to_cpu(err->ByteCount) >= sizeof(*sym) && + iov->iov_len >= SMB2_SYMLINK_STRUCT_SIZE) { + sym = (struct smb2_symlink_err_rsp *)err->ErrorData; + } + + if (!IS_ERR(sym) && + ((u8 *)sym + sizeof(*sym) > end || + le32_to_cpu(sym->SymLinkErrorTag) != SYMLINK_ERROR_TAG || + le32_to_cpu(sym->ReparseTag) != IO_REPARSE_TAG_SYMLINK)) + sym = ERR_PTR(-EINVAL); + + return sym; +} + +int smb2_fix_symlink_target_type(char **target, bool directory, struct cifs_sb_info *cifs_sb) +{ + char *buf; + int len; + + /* + * POSIX server does not distinguish between symlinks to file and + * symlink directory. So nothing is needed to fix on the client side. + */ + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_POSIX_PATHS) + return 0; + + if (!*target) + return smb_EIO(smb_eio_trace_null_pointers); + + len = strlen(*target); + if (!len) + return smb_EIO1(smb_eio_trace_sym_target_len, len); + + /* + * If this is directory symlink and it does not have trailing slash then + * append it. Trailing slash simulates Windows/SMB behavior which do not + * allow resolving directory symlink to file. + */ + if (directory && (*target)[len-1] != '/') { + buf = krealloc(*target, len+2, GFP_KERNEL); + if (!buf) + return -ENOMEM; + buf[len] = '/'; + buf[len+1] = '\0'; + *target = buf; + len++; + } + + /* + * If this is a file (non-directory) symlink and it points to path name + * with trailing slash then this is an invalid symlink because file name + * cannot contain slash character. File name with slash is invalid on + * both Windows and Linux systems. So return an error for such symlink. + */ + if (!directory && (*target)[len-1] == '/') + return smb_EIO(smb_eio_trace_sym_slash); + + return 0; +} + +int smb2_parse_symlink_response(struct cifs_sb_info *cifs_sb, const struct kvec *iov, + const char *full_path, char **path) +{ + struct smb2_symlink_err_rsp *sym; + unsigned int sub_offs, sub_len; + unsigned int print_offs, print_len; + + if (!cifs_sb || !iov || !iov->iov_base || !iov->iov_len || !path) + return -EINVAL; + + sym = symlink_data(iov); + if (IS_ERR(sym)) + return PTR_ERR(sym); + + sub_len = le16_to_cpu(sym->SubstituteNameLength); + sub_offs = le16_to_cpu(sym->SubstituteNameOffset); + print_len = le16_to_cpu(sym->PrintNameLength); + print_offs = le16_to_cpu(sym->PrintNameOffset); + + if ((char *)sym->PathBuffer + sub_offs + sub_len > + (char *)iov->iov_base + iov->iov_len || + (char *)sym->PathBuffer + print_offs + print_len > + (char *)iov->iov_base + iov->iov_len) + return -EINVAL; + + return smb2_parse_native_symlink(path, + (char *)sym->PathBuffer + sub_offs, + sub_len, + le32_to_cpu(sym->Flags) & SYMLINK_FLAG_RELATIVE, + full_path, + cifs_sb); +} + +int smb2_open_file(const unsigned int xid, struct cifs_open_parms *oparms, + __u32 *oplock, void *buf) +{ + int rc; + __le16 *smb2_path; + __u8 smb2_oplock; + struct cifs_open_info_data *data = buf; + struct kvec err_iov = {}; + int err_buftype = CIFS_NO_BUFFER; + struct cifs_fid *fid = oparms->fid; + struct network_resiliency_req nr_ioctl_req; + bool retry_without_read_attributes = false; + + smb2_path = cifs_convert_path_to_utf16(oparms->path, oparms->cifs_sb); + if (smb2_path == NULL) + return -ENOMEM; + + /* + * GENERIC_READ, GENERIC_EXECUTE, GENERIC_ALL and MAXIMUM_ALLOWED + * contains also FILE_READ_ATTRIBUTES access right. So do not append + * FILE_READ_ATTRIBUTES when not needed and prevent calling code path + * for retry_without_read_attributes. + */ + if (!(oparms->desired_access & FILE_READ_ATTRIBUTES) && + !(oparms->desired_access & GENERIC_READ) && + !(oparms->desired_access & GENERIC_EXECUTE) && + !(oparms->desired_access & GENERIC_ALL) && + !(oparms->desired_access & MAXIMUM_ALLOWED)) { + oparms->desired_access |= FILE_READ_ATTRIBUTES; + retry_without_read_attributes = true; + } + smb2_oplock = SMB2_OPLOCK_LEVEL_BATCH; + + rc = SMB2_open(xid, oparms, smb2_path, &smb2_oplock, data, NULL, &err_iov, + &err_buftype); + if (rc == -EACCES && retry_without_read_attributes) { + free_rsp_buf(err_buftype, err_iov.iov_base); + memset(&err_iov, 0, sizeof(err_iov)); + err_buftype = CIFS_NO_BUFFER; + oparms->desired_access &= ~FILE_READ_ATTRIBUTES; + rc = SMB2_open(xid, oparms, smb2_path, &smb2_oplock, data, NULL, &err_iov, + &err_buftype); + } + if (rc && data) { + struct smb2_hdr *hdr = err_iov.iov_base; + + if (unlikely(!err_iov.iov_base || err_buftype == CIFS_NO_BUFFER)) + goto out; + if (hdr->Status == STATUS_STOPPED_ON_SYMLINK) { + rc = smb2_parse_symlink_response(oparms->cifs_sb, &err_iov, + oparms->path, + &data->symlink_target); + /* + * If smb2_parse_symlink_response returned -ENODATA then the + * symlink_target was not sent. Treat this as if the SMB2_open() + * failed with STATUS_IO_REPARSE_TAG_NOT_HANDLED status, which is + * indicated by the -EIO errno. + */ + if (rc == -ENODATA) + rc = -EIO; + if (!rc) { + memset(&data->fi, 0, sizeof(data->fi)); + oparms->create_options |= OPEN_REPARSE_POINT; + rc = SMB2_open(xid, oparms, smb2_path, &smb2_oplock, data, + NULL, NULL, NULL); + oparms->create_options &= ~OPEN_REPARSE_POINT; + } + if (!rc) { + bool directory = le32_to_cpu(data->fi.Attributes) & ATTR_DIRECTORY; + rc = smb2_fix_symlink_target_type(&data->symlink_target, + directory, oparms->cifs_sb); + } + } + } + + if (rc) + goto out; + + if (oparms->tcon->use_resilient) { + /* default timeout is 0, servers pick default (120 seconds) */ + nr_ioctl_req.Timeout = + cpu_to_le32(oparms->tcon->handle_timeout); + nr_ioctl_req.Reserved = 0; + rc = SMB2_ioctl(xid, oparms->tcon, fid->persistent_fid, + fid->volatile_fid, FSCTL_LMR_REQUEST_RESILIENCY, + (char *)&nr_ioctl_req, sizeof(nr_ioctl_req), + CIFSMaxBufSize, NULL, NULL /* no return info */); + if (rc == -EOPNOTSUPP) { + cifs_dbg(VFS, + "resiliency not supported by server, disabling\n"); + oparms->tcon->use_resilient = false; + } else if (rc) + cifs_dbg(FYI, "error %d setting resiliency\n", rc); + + rc = 0; + } + + if (data) { + /* if open response does not have IndexNumber field - get it */ + if (data->fi.IndexNumber == 0) { + rc = SMB2_get_srv_num(xid, oparms->tcon, + fid->persistent_fid, + fid->volatile_fid, + &data->fi.IndexNumber); + if (rc) { + /* + * let get_inode_info disable server inode + * numbers + */ + data->fi.IndexNumber = 0; + rc = 0; + } + } + } + + *oplock = smb2_oplock; +out: + free_rsp_buf(err_buftype, err_iov.iov_base); + kfree(smb2_path); + return rc; +} + +int +smb2_unlock_range(struct cifsFileInfo *cfile, struct file_lock *flock, + const unsigned int xid) +{ + int rc = 0, stored_rc; + unsigned int max_num, num = 0, max_buf; + struct smb2_lock_element *buf, *cur; + struct cifs_tcon *tcon = tlink_tcon(cfile->tlink); + struct cifsInodeInfo *cinode = CIFS_I(d_inode(cfile->dentry)); + struct cifsLockInfo *li, *tmp; + __u64 length = 1 + flock->fl_end - flock->fl_start; + LIST_HEAD(tmp_llist); + + /* + * Accessing maxBuf is racy with cifs_reconnect - need to store value + * and check it before using. + */ + max_buf = tcon->ses->server->maxBuf; + if (max_buf < sizeof(struct smb2_lock_element)) + return -EINVAL; + + BUILD_BUG_ON(sizeof(struct smb2_lock_element) > PAGE_SIZE); + max_buf = min_t(unsigned int, max_buf, PAGE_SIZE); + max_num = max_buf / sizeof(struct smb2_lock_element); + buf = kzalloc_objs(struct smb2_lock_element, max_num); + if (!buf) + return -ENOMEM; + + cur = buf; + + cifs_down_write(&cinode->lock_sem); + list_for_each_entry_safe(li, tmp, &cfile->llist->locks, llist) { + if (flock->fl_start > li->offset || + (flock->fl_start + length) < + (li->offset + li->length)) + continue; + if (current->tgid != li->pid) + /* + * flock and OFD lock are associated with an open + * file description, not the process. + */ + if (!(flock->c.flc_flags & (FL_FLOCK | FL_OFDLCK))) + continue; + if (cinode->can_cache_brlcks) { + /* + * We can cache brlock requests - simply remove a lock + * from the file's list. + */ + list_del(&li->llist); + cifs_del_lock_waiters(li); + kfree(li); + continue; + } + cur->Length = cpu_to_le64(li->length); + cur->Offset = cpu_to_le64(li->offset); + cur->Flags = cpu_to_le32(SMB2_LOCKFLAG_UNLOCK); + /* + * We need to save a lock here to let us add it again to the + * file's list if the unlock range request fails on the server. + */ + list_move(&li->llist, &tmp_llist); + if (++num == max_num) { + stored_rc = smb2_lockv(xid, tcon, + cfile->fid.persistent_fid, + cfile->fid.volatile_fid, + current->tgid, num, buf); + if (stored_rc) { + /* + * We failed on the unlock range request - add + * all locks from the tmp list to the head of + * the file's list. + */ + cifs_move_llist(&tmp_llist, + &cfile->llist->locks); + rc = stored_rc; + } else + /* + * The unlock range request succeed - free the + * tmp list. + */ + cifs_free_llist(&tmp_llist); + cur = buf; + num = 0; + } else + cur++; + } + if (num) { + stored_rc = smb2_lockv(xid, tcon, cfile->fid.persistent_fid, + cfile->fid.volatile_fid, current->tgid, + num, buf); + if (stored_rc) { + cifs_move_llist(&tmp_llist, &cfile->llist->locks); + rc = stored_rc; + } else + cifs_free_llist(&tmp_llist); + } + up_write(&cinode->lock_sem); + + kfree(buf); + return rc; +} + +static int +smb2_push_mand_fdlocks(struct cifs_fid_locks *fdlocks, const unsigned int xid, + struct smb2_lock_element *buf, unsigned int max_num) +{ + int rc = 0, stored_rc; + struct cifsFileInfo *cfile = fdlocks->cfile; + struct cifsLockInfo *li; + unsigned int num = 0; + struct smb2_lock_element *cur = buf; + struct cifs_tcon *tcon = tlink_tcon(cfile->tlink); + + list_for_each_entry(li, &fdlocks->locks, llist) { + cur->Length = cpu_to_le64(li->length); + cur->Offset = cpu_to_le64(li->offset); + cur->Flags = cpu_to_le32(li->type | + SMB2_LOCKFLAG_FAIL_IMMEDIATELY); + if (++num == max_num) { + stored_rc = smb2_lockv(xid, tcon, + cfile->fid.persistent_fid, + cfile->fid.volatile_fid, + current->tgid, num, buf); + if (stored_rc) + rc = stored_rc; + cur = buf; + num = 0; + } else + cur++; + } + if (num) { + stored_rc = smb2_lockv(xid, tcon, + cfile->fid.persistent_fid, + cfile->fid.volatile_fid, + current->tgid, num, buf); + if (stored_rc) + rc = stored_rc; + } + + return rc; +} + +int +smb2_push_mandatory_locks(struct cifsFileInfo *cfile) +{ + int rc = 0, stored_rc; + unsigned int xid; + unsigned int max_num, max_buf; + struct smb2_lock_element *buf; + struct cifsInodeInfo *cinode = CIFS_I(d_inode(cfile->dentry)); + struct cifs_fid_locks *fdlocks; + + xid = get_xid(); + + /* + * Accessing maxBuf is racy with cifs_reconnect - need to store value + * and check it for zero before using. + */ + max_buf = tlink_tcon(cfile->tlink)->ses->server->maxBuf; + if (max_buf < sizeof(struct smb2_lock_element)) { + free_xid(xid); + return -EINVAL; + } + + BUILD_BUG_ON(sizeof(struct smb2_lock_element) > PAGE_SIZE); + max_buf = min_t(unsigned int, max_buf, PAGE_SIZE); + max_num = max_buf / sizeof(struct smb2_lock_element); + buf = kzalloc_objs(struct smb2_lock_element, max_num); + if (!buf) { + free_xid(xid); + return -ENOMEM; + } + + list_for_each_entry(fdlocks, &cinode->llist, llist) { + stored_rc = smb2_push_mand_fdlocks(fdlocks, xid, buf, max_num); + if (stored_rc) + rc = stored_rc; + } + + kfree(buf); + free_xid(xid); + return rc; +} diff --git a/fs/smb/client/smb2glob.h b/fs/smb/client/smb2glob.h new file mode 100644 index 000000000..19da74b1e --- /dev/null +++ b/fs/smb/client/smb2glob.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +/* + * + * Definitions for various global variables and structures + * + * Copyright (C) International Business Machines Corp., 2002, 2011 + * Etersoft, 2012 + * Author(s): Steve French (sfrench@us.ibm.com) + * Jeremy Allison (jra@samba.org) + * Pavel Shilovsky (pshilovsky@samba.org) 2012 + * + */ +#ifndef _SMB2_GLOB_H +#define _SMB2_GLOB_H + +/* + ***************************************************************** + * Constants go here + ***************************************************************** + */ + +/* + * Identifiers for functions that use the open, operation, close pattern + * in smb2inode.c:smb2_compound_op() + */ +enum smb2_compound_ops { + SMB2_OP_SET_DELETE = 1, + SMB2_OP_SET_INFO, + SMB2_OP_QUERY_INFO, + SMB2_OP_QUERY_DIR, + SMB2_OP_MKDIR, + SMB2_OP_RENAME, + SMB2_OP_HARDLINK, + SMB2_OP_SET_EOF, + SMB2_OP_UNLINK, + SMB2_OP_POSIX_QUERY_INFO, + SMB2_OP_SET_REPARSE, + SMB2_OP_GET_REPARSE, + SMB2_OP_QUERY_WSL_EA, + SMB2_OP_OPEN_QUERY, +}; + +/* Used when constructing chained read requests. */ +#define CHAINED_REQUEST 1 +#define START_OF_CHAIN 2 +#define END_OF_CHAIN 4 +#define RELATED_REQUEST 8 + +/* + ***************************************************************** + * Struct definitions go here + ***************************************************************** + */ + +struct status_to_posix_error { + __u32 smb2_status; + int posix_error; + char *status_string; +}; + +#endif /* _SMB2_GLOB_H */ diff --git a/fs/smb/client/smb2inode.c b/fs/smb/client/smb2inode.c new file mode 100644 index 000000000..213bc298c --- /dev/null +++ b/fs/smb/client/smb2inode.c @@ -0,0 +1,1581 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * Copyright (C) International Business Machines Corp., 2002, 2011 + * Etersoft, 2012 + * Author(s): Pavel Shilovsky (pshilovsky@samba.org), + * Steve French (sfrench@us.ibm.com) + * + */ +#include +#include +#include +#include +#include +#include "cifsfs.h" +#include "cifsglob.h" +#include "cifsproto.h" +#include "cifs_debug.h" +#include "cifs_fs_sb.h" +#include "cifs_unicode.h" +#include "fscache.h" +#include "smb2glob.h" +#include "smb2proto.h" +#include "cached_dir.h" +#include "../common/smb2status.h" +#include "../common/smbfsctl.h" + +static struct reparse_data_buffer *reparse_buf_ptr(struct kvec *iov) +{ + struct reparse_data_buffer *buf; + struct smb2_ioctl_rsp *io = iov->iov_base; + u32 off, count, len; + u16 rdlen; + + count = le32_to_cpu(io->OutputCount); + off = le32_to_cpu(io->OutputOffset); + if (check_add_overflow(off, count, &len) || len > iov->iov_len) + return ERR_PTR(smb_EIO2(smb_eio_trace_reparse_overlong, + off, count)); + + buf = (struct reparse_data_buffer *)((u8 *)io + off); + len = sizeof(*buf); + rdlen = le16_to_cpu(buf->ReparseDataLength); + + if (count < len || count < rdlen + len) + return ERR_PTR(smb_EIO2(smb_eio_trace_reparse_rdlen, count, rdlen)); + return buf; +} + +static inline __u32 file_create_options(struct dentry *dentry) +{ + struct cifsInodeInfo *ci; + + if (dentry) { + ci = CIFS_I(d_inode(dentry)); + if (ci->cifsAttrs & ATTR_REPARSE_POINT) + return OPEN_REPARSE_POINT; + } + return 0; +} + +/* Parse owner and group from SMB3.1.1 POSIX query info */ +static int parse_posix_sids(struct cifs_open_info_data *data, + struct kvec *rsp_iov) +{ + struct smb2_query_info_rsp *qi = rsp_iov->iov_base; + unsigned int out_len = le32_to_cpu(qi->OutputBufferLength); + unsigned int qi_len = sizeof(data->posix_fi); + int owner_len, group_len; + u8 *sidsbuf, *sidsbuf_end; + + if (out_len <= qi_len) + return -EINVAL; + + sidsbuf = (u8 *)qi + le16_to_cpu(qi->OutputBufferOffset) + qi_len; + sidsbuf_end = sidsbuf + out_len - qi_len; + + owner_len = posix_info_sid_size(sidsbuf, sidsbuf_end); + if (owner_len == -1) + return -EINVAL; + + memcpy(&data->posix_owner, sidsbuf, owner_len); + group_len = posix_info_sid_size(sidsbuf + owner_len, sidsbuf_end); + if (group_len == -1) + return -EINVAL; + + memcpy(&data->posix_group, sidsbuf + owner_len, group_len); + return 0; +} + +struct wsl_query_ea { + __le32 next; + __u8 name_len; + __u8 name[SMB2_WSL_XATTR_NAME_LEN + 1]; +} __packed; + +#define NEXT_OFF cpu_to_le32(sizeof(struct wsl_query_ea)) + +static const struct wsl_query_ea wsl_query_eas[] = { + { .next = NEXT_OFF, .name_len = SMB2_WSL_XATTR_NAME_LEN, .name = SMB2_WSL_XATTR_UID, }, + { .next = NEXT_OFF, .name_len = SMB2_WSL_XATTR_NAME_LEN, .name = SMB2_WSL_XATTR_GID, }, + { .next = NEXT_OFF, .name_len = SMB2_WSL_XATTR_NAME_LEN, .name = SMB2_WSL_XATTR_MODE, }, + { .next = 0, .name_len = SMB2_WSL_XATTR_NAME_LEN, .name = SMB2_WSL_XATTR_DEV, }, +}; + +static int check_wsl_eas(struct kvec *rsp_iov) +{ + struct smb2_file_full_ea_info *ea; + struct smb2_query_info_rsp *rsp = rsp_iov->iov_base; + unsigned long addr; + u32 outlen, next; + u16 vlen; + u8 nlen; + u8 *ea_end, *iov_end; + + outlen = le32_to_cpu(rsp->OutputBufferLength); + if (outlen < SMB2_WSL_MIN_QUERY_EA_RESP_SIZE || + outlen > SMB2_WSL_MAX_QUERY_EA_RESP_SIZE) + return -EINVAL; + + ea = (void *)((u8 *)rsp_iov->iov_base + + le16_to_cpu(rsp->OutputBufferOffset)); + ea_end = (u8 *)ea + outlen; + iov_end = (u8 *)rsp_iov->iov_base + rsp_iov->iov_len; + if (ea_end > iov_end) + return -EINVAL; + + for (;;) { + if ((u8 *)ea > ea_end - sizeof(*ea)) + return -EINVAL; + + nlen = ea->ea_name_length; + vlen = le16_to_cpu(ea->ea_value_length); + if (nlen != SMB2_WSL_XATTR_NAME_LEN || + (u8 *)ea->ea_data + nlen + 1 + vlen > ea_end) + return -EINVAL; + + switch (vlen) { + case 4: + if (strncmp(ea->ea_data, SMB2_WSL_XATTR_UID, nlen) && + strncmp(ea->ea_data, SMB2_WSL_XATTR_GID, nlen) && + strncmp(ea->ea_data, SMB2_WSL_XATTR_MODE, nlen)) + return -EINVAL; + break; + case 8: + if (strncmp(ea->ea_data, SMB2_WSL_XATTR_DEV, nlen)) + return -EINVAL; + break; + case 0: + if (!strncmp(ea->ea_data, SMB2_WSL_XATTR_UID, nlen) || + !strncmp(ea->ea_data, SMB2_WSL_XATTR_GID, nlen) || + !strncmp(ea->ea_data, SMB2_WSL_XATTR_MODE, nlen) || + !strncmp(ea->ea_data, SMB2_WSL_XATTR_DEV, nlen)) + break; + fallthrough; + default: + return -EINVAL; + } + + next = le32_to_cpu(ea->next_entry_offset); + if (!next) + break; + if (!IS_ALIGNED(next, 4) || + check_add_overflow((unsigned long)ea, next, &addr)) + return -EINVAL; + ea = (void *)addr; + } + return 0; +} + +/* + * If @cfile is NULL, then need to account for trailing CLOSE request in the + * compound chain. + */ +static void set_next_compound(struct cifs_tcon *tcon, + struct cifsFileInfo *cfile, + int i, int num_cmds, + struct smb_rqst *rqst, int *num_rqst) +{ + int k = !cfile ? 1 : 0; + + if (i + 1 < num_cmds + k) + smb2_set_next_command(tcon, &rqst[*num_rqst]); + if (i + k > 0) + smb2_set_related(&rqst[*num_rqst]); + (*num_rqst)++; +} + +#define COMP_PID(cfile) ((cfile) ? (cfile)->fid.persistent_fid : COMPOUND_FID) +#define COMP_VID(cfile) ((cfile) ? (cfile)->fid.volatile_fid : COMPOUND_FID) + +/* + * note: If cfile is passed, the reference to it is dropped here. + * So make sure that you do not reuse cfile after return from this func. + * + * If passing @out_iov and @out_buftype, ensure to make them both large enough + * (>= 3) to hold all compounded responses. Caller is also responsible for + * freeing them up with free_rsp_buf(). + */ +static int smb2_compound_op(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, const char *full_path, + struct cifs_open_parms *oparms, struct kvec *in_iov, + int *cmds, int num_cmds, struct cifsFileInfo *cfile, + struct kvec *out_iov, int *out_buftype, struct dentry *dentry) +{ + + struct smb2_create_rsp *create_rsp = NULL; + struct smb2_query_info_rsp *qi_rsp = NULL; + struct smb2_compound_vars *vars = NULL; + __u8 oplock = SMB2_OPLOCK_LEVEL_NONE; + struct cifs_open_info_data *idata; + struct cifs_ses *ses = tcon->ses; + struct reparse_data_buffer *rbuf; + struct TCP_Server_Info *server; + int resp_buftype[MAX_COMPOUND]; + int retries = 0, cur_sleep = 0; + __u8 delete_pending[8] = {1,}; + struct kvec *rsp_iov, *iov; + struct inode *inode = NULL; + __le16 *utf16_path = NULL; + struct smb_rqst *rqst; + unsigned int size[2]; + struct cifs_fid fid; + int num_rqst = 0, i; + unsigned int len; + int tmp_rc, rc; + int flags = 0; + void *data[2]; + +replay_again: + /* reinitialize for possible replay */ + flags = 0; + oplock = SMB2_OPLOCK_LEVEL_NONE; + num_rqst = 0; + server = cifs_pick_channel(ses); + + vars = kzalloc_obj(*vars, GFP_KERNEL); + if (vars == NULL) { + rc = -ENOMEM; + goto out; + } + rqst = &vars->rqst[0]; + rsp_iov = &vars->rsp_iov[0]; + + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; + + for (i = 0; i < ARRAY_SIZE(resp_buftype); i++) + resp_buftype[i] = CIFS_NO_BUFFER; + + /* We already have a handle so we can skip the open */ + if (cfile) + goto after_open; + + /* Open */ + utf16_path = cifs_convert_path_to_utf16(full_path, cifs_sb); + if (!utf16_path) { + rc = -ENOMEM; + goto finished; + } + + /* if there is an existing lease, reuse it */ + + /* + * note: files with hardlinks cause unexpected behaviour. As per MS-SMB2, + * lease keys are associated with the filepath. We are maintaining lease keys + * with the inode on the client. If the file has hardlinks, it is possible + * that the lease for a file be reused for an operation on its hardlink or + * vice versa. + * As a workaround, send request using an existing lease key and if the server + * returns STATUS_INVALID_PARAMETER, which maps to EINVAL, send the request + * again without the lease. + */ + if (dentry) { + inode = d_inode(dentry); + if (CIFS_I(inode)->lease_granted && server->ops->get_lease_key) { + oplock = SMB2_OPLOCK_LEVEL_LEASE; + server->ops->get_lease_key(inode, &fid); + } + } + + vars->oparms = *oparms; + vars->oparms.fid = &fid; + + rqst[num_rqst].rq_iov = &vars->open_iov[0]; + rqst[num_rqst].rq_nvec = SMB2_CREATE_IOV_SIZE; + rc = SMB2_open_init(tcon, server, + &rqst[num_rqst], &oplock, &vars->oparms, + utf16_path); + kfree(utf16_path); + if (rc) + goto finished; + + smb2_set_next_command(tcon, &rqst[num_rqst]); + after_open: + num_rqst++; + rc = 0; + + i = 0; + + /* Skip the leading explicit OPEN operation */ + if (num_cmds > 0 && cmds[0] == SMB2_OP_OPEN_QUERY) + i++; + + for (; i < num_cmds; i++) { + /* Operation */ + switch (cmds[i]) { + case SMB2_OP_QUERY_INFO: + rqst[num_rqst].rq_iov = &vars->qi_iov; + rqst[num_rqst].rq_nvec = 1; + + rc = SMB2_query_info_init(tcon, server, + &rqst[num_rqst], + COMP_PID(cfile), COMP_VID(cfile), + FILE_ALL_INFORMATION, + SMB2_O_INFO_FILE, 0, + sizeof(struct smb2_file_all_info) + + PATH_MAX * 2, 0, NULL); + if (rc) + goto finished; + set_next_compound(tcon, cfile, i, num_cmds, rqst, &num_rqst); + trace_smb3_query_info_compound_enter(xid, tcon->tid, + ses->Suid, full_path); + break; + case SMB2_OP_POSIX_QUERY_INFO: + rqst[num_rqst].rq_iov = &vars->qi_iov; + rqst[num_rqst].rq_nvec = 1; + + /* TBD: fix following to allow for longer SIDs */ + rc = SMB2_query_info_init(tcon, server, + &rqst[num_rqst], + COMP_PID(cfile), COMP_VID(cfile), + SMB_FIND_FILE_POSIX_INFO, + SMB2_O_INFO_FILE, 0, + sizeof(struct smb311_posix_qinfo) + + (PATH_MAX * 2) + + (sizeof(struct smb_sid) * 2), 0, NULL); + if (rc) + goto finished; + set_next_compound(tcon, cfile, i, num_cmds, rqst, &num_rqst); + trace_smb3_posix_query_info_compound_enter(xid, tcon->tid, + ses->Suid, full_path); + break; + case SMB2_OP_MKDIR: + /* + * Directories are created through parameters in the + * SMB2_open() call. + */ + trace_smb3_mkdir_enter(xid, tcon->tid, ses->Suid, full_path); + break; + case SMB2_OP_UNLINK: + rqst[num_rqst].rq_iov = vars->unlink_iov; + rqst[num_rqst].rq_nvec = 1; + + size[0] = 1; /* sizeof __u8 See MS-FSCC section 2.4.11 */ + data[0] = &delete_pending[0]; + + rc = SMB2_set_info_init(tcon, server, + &rqst[num_rqst], + COMP_PID(cfile), COMP_VID(cfile), + current->tgid, FILE_DISPOSITION_INFORMATION, + SMB2_O_INFO_FILE, 0, + data, size); + if (rc) + goto finished; + set_next_compound(tcon, cfile, i, num_cmds, rqst, &num_rqst); + trace_smb3_unlink_enter(xid, tcon->tid, ses->Suid, full_path); + break; + case SMB2_OP_SET_EOF: + rqst[num_rqst].rq_iov = &vars->si_iov[0]; + rqst[num_rqst].rq_nvec = 1; + + size[0] = in_iov[i].iov_len; + data[0] = in_iov[i].iov_base; + + rc = SMB2_set_info_init(tcon, server, + &rqst[num_rqst], + COMP_PID(cfile), COMP_VID(cfile), + current->tgid, FILE_END_OF_FILE_INFORMATION, + SMB2_O_INFO_FILE, 0, + data, size); + if (rc) + goto finished; + set_next_compound(tcon, cfile, i, num_cmds, rqst, &num_rqst); + trace_smb3_set_eof_enter(xid, tcon->tid, ses->Suid, full_path); + break; + case SMB2_OP_SET_INFO: + rqst[num_rqst].rq_iov = &vars->si_iov[0]; + rqst[num_rqst].rq_nvec = 1; + + size[0] = in_iov[i].iov_len; + data[0] = in_iov[i].iov_base; + + rc = SMB2_set_info_init(tcon, server, + &rqst[num_rqst], + COMP_PID(cfile), COMP_VID(cfile), + current->tgid, FILE_BASIC_INFORMATION, + SMB2_O_INFO_FILE, 0, data, size); + if (rc) + goto finished; + set_next_compound(tcon, cfile, i, num_cmds, rqst, &num_rqst); + trace_smb3_set_info_compound_enter(xid, tcon->tid, + ses->Suid, full_path); + break; + case SMB2_OP_RENAME: + rqst[num_rqst].rq_iov = vars->rename_iov; + rqst[num_rqst].rq_nvec = 2; + + len = in_iov[i].iov_len; + + vars->rename_info.ReplaceIfExists = 1; + vars->rename_info.RootDirectory = 0; + vars->rename_info.FileNameLength = cpu_to_le32(len); + + size[0] = sizeof(struct smb2_file_rename_info); + data[0] = &vars->rename_info; + + size[1] = len + 2 /* null */; + data[1] = in_iov[i].iov_base; + + rc = SMB2_set_info_init(tcon, server, + &rqst[num_rqst], + COMP_PID(cfile), COMP_VID(cfile), + current->tgid, FILE_RENAME_INFORMATION, + SMB2_O_INFO_FILE, 0, data, size); + + if (rc) + goto finished; + set_next_compound(tcon, cfile, i, num_cmds, rqst, &num_rqst); + trace_smb3_rename_enter(xid, tcon->tid, ses->Suid, full_path); + break; + case SMB2_OP_HARDLINK: + rqst[num_rqst].rq_iov = vars->hl_iov; + rqst[num_rqst].rq_nvec = 2; + + len = in_iov[i].iov_len; + + vars->link_info.ReplaceIfExists = 0; + vars->link_info.RootDirectory = 0; + vars->link_info.FileNameLength = cpu_to_le32(len); + + size[0] = sizeof(struct smb2_file_link_info); + data[0] = &vars->link_info; + + size[1] = len + 2 /* null */; + data[1] = in_iov[i].iov_base; + + rc = SMB2_set_info_init(tcon, server, + &rqst[num_rqst], + COMP_PID(cfile), COMP_VID(cfile), + current->tgid, FILE_LINK_INFORMATION, + SMB2_O_INFO_FILE, 0, data, size); + if (rc) + goto finished; + set_next_compound(tcon, cfile, i, num_cmds, rqst, &num_rqst); + trace_smb3_hardlink_enter(xid, tcon->tid, ses->Suid, full_path); + break; + case SMB2_OP_SET_REPARSE: + rqst[num_rqst].rq_iov = vars->io_iov; + rqst[num_rqst].rq_nvec = ARRAY_SIZE(vars->io_iov); + + rc = SMB2_ioctl_init(tcon, server, &rqst[num_rqst], + COMP_PID(cfile), COMP_VID(cfile), + FSCTL_SET_REPARSE_POINT, + in_iov[i].iov_base, + in_iov[i].iov_len, 0); + if (rc) + goto finished; + set_next_compound(tcon, cfile, i, num_cmds, rqst, &num_rqst); + trace_smb3_set_reparse_compound_enter(xid, tcon->tid, + ses->Suid, full_path); + break; + case SMB2_OP_GET_REPARSE: + rqst[num_rqst].rq_iov = vars->io_iov; + rqst[num_rqst].rq_nvec = ARRAY_SIZE(vars->io_iov); + + rc = SMB2_ioctl_init(tcon, server, &rqst[num_rqst], + COMP_PID(cfile), COMP_VID(cfile), + FSCTL_GET_REPARSE_POINT, + NULL, 0, CIFSMaxBufSize); + if (rc) + goto finished; + set_next_compound(tcon, cfile, i, num_cmds, rqst, &num_rqst); + trace_smb3_get_reparse_compound_enter(xid, tcon->tid, + ses->Suid, full_path); + break; + case SMB2_OP_QUERY_WSL_EA: + rqst[num_rqst].rq_iov = &vars->ea_iov; + rqst[num_rqst].rq_nvec = 1; + + rc = SMB2_query_info_init(tcon, server, + &rqst[num_rqst], + COMP_PID(cfile), COMP_VID(cfile), + FILE_FULL_EA_INFORMATION, + SMB2_O_INFO_FILE, 0, + SMB2_WSL_MAX_QUERY_EA_RESP_SIZE, + sizeof(wsl_query_eas), + (void *)wsl_query_eas); + if (rc) + goto finished; + set_next_compound(tcon, cfile, i, num_cmds, rqst, &num_rqst); + trace_smb3_query_wsl_ea_compound_enter(xid, tcon->tid, + ses->Suid, full_path); + break; + default: + cifs_dbg(VFS, "Invalid command\n"); + rc = -EINVAL; + } + } + if (rc) + goto finished; + + /* We already have a handle so we can skip the close */ + if (cfile) + goto after_close; + /* Close */ + flags |= CIFS_CP_CREATE_CLOSE_OP; + rqst[num_rqst].rq_iov = &vars->close_iov; + rqst[num_rqst].rq_nvec = 1; + rc = SMB2_close_init(tcon, server, + &rqst[num_rqst], COMPOUND_FID, + COMPOUND_FID, false); + smb2_set_related(&rqst[num_rqst]); + if (rc) + goto finished; + after_close: + num_rqst++; + + if (cfile) { + if (retries) { + /* Back-off before retry */ + if (cur_sleep) + msleep(cur_sleep); + for (i = 1; i < num_rqst - 2; i++) + smb2_set_replay(server, &rqst[i]); + } + + rc = compound_send_recv(xid, ses, server, + flags, num_rqst - 2, + &rqst[1], &resp_buftype[1], + &rsp_iov[1]); + } else { + if (retries) { + /* Back-off before retry */ + if (cur_sleep) + msleep(cur_sleep); + for (i = 0; i < num_rqst; i++) + smb2_set_replay(server, &rqst[i]); + } + + rc = compound_send_recv(xid, ses, server, + flags, num_rqst, + rqst, resp_buftype, + rsp_iov); + } + +finished: + num_rqst = 0; + SMB2_open_free(&rqst[num_rqst++]); + if (rc == -EREMCHG) { + pr_warn_once("server share %s deleted\n", tcon->tree_name); + tcon->need_reconnect = true; + } + + tmp_rc = rc; + + if (rc == 0 && num_cmds > 0 && cmds[0] == SMB2_OP_OPEN_QUERY) { + create_rsp = rsp_iov[0].iov_base; + idata = in_iov[0].iov_base; + idata->fi.CreationTime = create_rsp->CreationTime; + idata->fi.LastAccessTime = create_rsp->LastAccessTime; + idata->fi.LastWriteTime = create_rsp->LastWriteTime; + idata->fi.ChangeTime = create_rsp->ChangeTime; + idata->fi.Attributes = create_rsp->FileAttributes; + idata->fi.AllocationSize = create_rsp->AllocationSize; + idata->fi.EndOfFile = create_rsp->EndofFile; + if (le32_to_cpu(idata->fi.NumberOfLinks) == 0) + idata->fi.NumberOfLinks = cpu_to_le32(1); /* dummy value */ + idata->fi.DeletePending = 0; /* successful open = not delete pending */ + idata->fi.Directory = !!(le32_to_cpu(create_rsp->FileAttributes) & ATTR_DIRECTORY); + + /* smb2_parse_contexts() fills idata->fi.IndexNumber */ + rc = smb2_parse_contexts(server, &rsp_iov[0], &oparms->fid->epoch, + oparms->fid->lease_key, &oplock, &idata->fi, NULL); + if (rc) + cifs_dbg(VFS, "rc: %d parsing context of compound op\n", rc); + } + + for (i = 0; i < num_cmds; i++) { + char *buf = rsp_iov[i + 1].iov_base; + + if (buf && resp_buftype[i + 1] != CIFS_NO_BUFFER) + rc = server->ops->map_error(buf, false); + else + rc = tmp_rc; + switch (cmds[i]) { + case SMB2_OP_QUERY_INFO: + idata = in_iov[i].iov_base; + idata->contains_posix_file_info = false; + if (rc == 0 && cfile && cfile->symlink_target) { + idata->symlink_target = kstrdup(cfile->symlink_target, GFP_KERNEL); + if (!idata->symlink_target) + rc = -ENOMEM; + } + if (rc == 0) { + qi_rsp = (struct smb2_query_info_rsp *) + rsp_iov[i + 1].iov_base; + rc = smb2_validate_and_copy_iov( + le16_to_cpu(qi_rsp->OutputBufferOffset), + le32_to_cpu(qi_rsp->OutputBufferLength), + &rsp_iov[i + 1], sizeof(idata->fi), (char *)&idata->fi); + } + SMB2_query_info_free(&rqst[num_rqst++]); + if (rc) + trace_smb3_query_info_compound_err(xid, tcon->tid, + ses->Suid, rc); + else + trace_smb3_query_info_compound_done(xid, tcon->tid, + ses->Suid); + break; + case SMB2_OP_POSIX_QUERY_INFO: + idata = in_iov[i].iov_base; + idata->contains_posix_file_info = true; + if (rc == 0 && cfile && cfile->symlink_target) { + idata->symlink_target = kstrdup(cfile->symlink_target, GFP_KERNEL); + if (!idata->symlink_target) + rc = -ENOMEM; + } + if (rc == 0) { + qi_rsp = (struct smb2_query_info_rsp *) + rsp_iov[i + 1].iov_base; + rc = smb2_validate_and_copy_iov( + le16_to_cpu(qi_rsp->OutputBufferOffset), + le32_to_cpu(qi_rsp->OutputBufferLength), + &rsp_iov[i + 1], sizeof(idata->posix_fi) /* add SIDs */, + (char *)&idata->posix_fi); + } + if (rc == 0) + rc = parse_posix_sids(idata, &rsp_iov[i + 1]); + + SMB2_query_info_free(&rqst[num_rqst++]); + if (rc) + trace_smb3_posix_query_info_compound_err(xid, tcon->tid, + ses->Suid, rc); + else + trace_smb3_posix_query_info_compound_done(xid, tcon->tid, + ses->Suid); + break; + case SMB2_OP_MKDIR: + if (rc) + trace_smb3_mkdir_err(xid, tcon->tid, ses->Suid, rc); + else + trace_smb3_mkdir_done(xid, tcon->tid, ses->Suid); + break; + case SMB2_OP_HARDLINK: + if (rc) + trace_smb3_hardlink_err(xid, tcon->tid, ses->Suid, rc); + else + trace_smb3_hardlink_done(xid, tcon->tid, ses->Suid); + SMB2_set_info_free(&rqst[num_rqst++]); + break; + case SMB2_OP_RENAME: + if (rc) + trace_smb3_rename_err(xid, tcon->tid, ses->Suid, rc); + else + trace_smb3_rename_done(xid, tcon->tid, ses->Suid); + SMB2_set_info_free(&rqst[num_rqst++]); + break; + case SMB2_OP_UNLINK: + if (!rc) + trace_smb3_unlink_done(xid, tcon->tid, ses->Suid); + else + trace_smb3_unlink_err(xid, tcon->tid, ses->Suid, rc); + SMB2_set_info_free(&rqst[num_rqst++]); + break; + case SMB2_OP_SET_EOF: + if (rc) + trace_smb3_set_eof_err(xid, tcon->tid, ses->Suid, rc); + else + trace_smb3_set_eof_done(xid, tcon->tid, ses->Suid); + SMB2_set_info_free(&rqst[num_rqst++]); + break; + case SMB2_OP_SET_INFO: + if (rc) + trace_smb3_set_info_compound_err(xid, tcon->tid, + ses->Suid, rc); + else + trace_smb3_set_info_compound_done(xid, tcon->tid, + ses->Suid); + SMB2_set_info_free(&rqst[num_rqst++]); + break; + case SMB2_OP_SET_REPARSE: + if (rc) { + trace_smb3_set_reparse_compound_err(xid, tcon->tid, + ses->Suid, rc); + } else { + trace_smb3_set_reparse_compound_done(xid, tcon->tid, + ses->Suid); + } + SMB2_ioctl_free(&rqst[num_rqst++]); + break; + case SMB2_OP_GET_REPARSE: + if (!rc) { + iov = &rsp_iov[i + 1]; + idata = in_iov[i].iov_base; + idata->reparse.io.iov = *iov; + idata->reparse.io.buftype = resp_buftype[i + 1]; + idata->contains_posix_file_info = false; /* BB VERIFY */ + rbuf = reparse_buf_ptr(iov); + if (IS_ERR(rbuf)) { + rc = PTR_ERR(rbuf); + trace_smb3_get_reparse_compound_err(xid, tcon->tid, + ses->Suid, rc); + } else { + idata->reparse.tag = le32_to_cpu(rbuf->ReparseTag); + trace_smb3_get_reparse_compound_done(xid, tcon->tid, + ses->Suid); + } + memset(iov, 0, sizeof(*iov)); + resp_buftype[i + 1] = CIFS_NO_BUFFER; + } else { + trace_smb3_get_reparse_compound_err(xid, tcon->tid, + ses->Suid, rc); + } + SMB2_ioctl_free(&rqst[num_rqst++]); + break; + case SMB2_OP_QUERY_WSL_EA: + if (!rc) { + idata = in_iov[i].iov_base; + idata->contains_posix_file_info = false; + qi_rsp = rsp_iov[i + 1].iov_base; + data[0] = (u8 *)qi_rsp + le16_to_cpu(qi_rsp->OutputBufferOffset); + size[0] = le32_to_cpu(qi_rsp->OutputBufferLength); + rc = check_wsl_eas(&rsp_iov[i + 1]); + if (!rc) { + memcpy(idata->wsl.eas, data[0], size[0]); + idata->wsl.eas_len = size[0]; + } + } + if (!rc) { + trace_smb3_query_wsl_ea_compound_done(xid, tcon->tid, + ses->Suid); + } else { + trace_smb3_query_wsl_ea_compound_err(xid, tcon->tid, + ses->Suid, rc); + } + SMB2_query_info_free(&rqst[num_rqst++]); + break; + } + } + SMB2_close_free(&rqst[num_rqst]); + rc = tmp_rc; + + num_cmds += 2; + if (out_iov && out_buftype) { + memcpy(out_iov, rsp_iov, num_cmds * sizeof(*out_iov)); + memcpy(out_buftype, resp_buftype, + num_cmds * sizeof(*out_buftype)); + } else { + for (i = 0; i < num_cmds; i++) + free_rsp_buf(resp_buftype[i], rsp_iov[i].iov_base); + } + num_cmds -= 2; /* correct num_cmds as there could be a retry */ + kfree(vars); + + if (is_replayable_error(rc) && + smb2_should_replay(tcon, &retries, &cur_sleep)) + goto replay_again; + +out: + if (cfile) + cifsFileInfo_put(cfile); + + return rc; +} + +static int parse_create_response(struct cifs_open_info_data *data, + struct cifs_sb_info *cifs_sb, + const char *full_path, + const struct kvec *iov) +{ + struct smb2_create_rsp *rsp = iov->iov_base; + bool reparse_point = false; + u32 tag = 0; + int rc = 0; + + switch (rsp->hdr.Status) { + case STATUS_IO_REPARSE_TAG_NOT_HANDLED: + reparse_point = true; + break; + case STATUS_STOPPED_ON_SYMLINK: + rc = smb2_parse_symlink_response(cifs_sb, iov, + full_path, + &data->symlink_target); + if (rc != 0 && rc != -ENODATA) + return rc; + /* + * -ENODATA means that the response was parsed but did not contain + * the symlink target at all (see symlink_data()). Treat it like + * STATUS_IO_REPARSE_TAG_NOT_HANDLED, which does not contain it + * either: leave the tag unset and clear rc, so that the caller + * retrieves the target with SMB2_OP_GET_REPARSE. + */ + if (rc == -ENODATA) + rc = 0; + else + tag = IO_REPARSE_TAG_SYMLINK; + reparse_point = true; + break; + case STATUS_SUCCESS: + reparse_point = !!(rsp->Flags & SMB2_CREATE_FLAG_REPARSEPOINT); + break; + } + data->reparse_point = reparse_point; + data->reparse.tag = tag; + return rc; +} + +/* Check only if SMB2_OP_QUERY_WSL_EA command failed in the compound chain */ +static bool ea_unsupported(int *cmds, int num_cmds, + struct kvec *out_iov, int *out_buftype) +{ + int i; + + if (cmds[num_cmds - 1] != SMB2_OP_QUERY_WSL_EA) + return false; + + for (i = 1; i < num_cmds - 1; i++) { + struct smb2_hdr *hdr = out_iov[i].iov_base; + + if (out_buftype[i] == CIFS_NO_BUFFER || !hdr || + hdr->Status != STATUS_SUCCESS) + return false; + } + return true; +} + +static inline void free_rsp_iov(struct kvec *iovs, int *buftype, int count) +{ + int i; + + for (i = 0; i < count; i++) { + free_rsp_buf(buftype[i], iovs[i].iov_base); + memset(&iovs[i], 0, sizeof(*iovs)); + buftype[i] = CIFS_NO_BUFFER; + } +} + +int smb2_query_path_info(const unsigned int xid, + struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, + const char *full_path, + struct cifs_open_info_data *data) +{ + struct kvec in_iov[3], out_iov[5] = {}; + struct cached_fid *cfid = NULL; + struct cifs_open_parms oparms; + struct cifsFileInfo *cfile; + __u32 create_options = 0; + int out_buftype[5] = {}; + struct smb2_hdr *hdr; + int num_cmds = 0; + int cmds[3]; + bool islink; + int rc, rc2; + + data->adjust_tz = false; + data->reparse_point = false; + + /* + * BB TODO: Add support for using cached root handle in SMB3.1.1 POSIX. + * Create SMB2_query_posix_info worker function to do non-compounded + * query when we already have an open file handle for this. For now this + * is fast enough (always using the compounded version). + */ + if (!tcon->posix_extensions) { + if (*full_path) { + rc = -ENOENT; + } else { + rc = open_cached_dir(xid, tcon, full_path, + cifs_sb, false, &cfid); + } + /* If it is a root and its handle is cached then use it */ + if (!rc) { + if (cfid->file_all_info_is_valid) { + memcpy(&data->fi, &cfid->file_all_info, + sizeof(data->fi)); + } else { + rc = SMB2_query_info(xid, tcon, + cfid->fid.persistent_fid, + cfid->fid.volatile_fid, + &data->fi); + } + close_cached_dir(cfid); + return rc; + } + cmds[num_cmds++] = SMB2_OP_QUERY_INFO; + } else { + cmds[num_cmds++] = SMB2_OP_POSIX_QUERY_INFO; + } + + in_iov[0].iov_base = data; + in_iov[0].iov_len = sizeof(*data); + in_iov[1] = in_iov[0]; + in_iov[2] = in_iov[0]; + + cifs_get_readable_path(tcon, full_path, &cfile); + oparms = CIFS_OPARMS(cifs_sb, tcon, full_path, FILE_READ_ATTRIBUTES, + FILE_OPEN, create_options, ACL_NO_MODE); + rc = smb2_compound_op(xid, tcon, cifs_sb, full_path, + &oparms, in_iov, cmds, num_cmds, + cfile, out_iov, out_buftype, NULL); + hdr = out_iov[0].iov_base; + /* + * If first iov is unset, then SMB session was dropped or we've got a + * cached open file (@cfile). + */ + if (!hdr || out_buftype[0] == CIFS_NO_BUFFER) + goto out; + + switch (rc) { + case 0: + rc = parse_create_response(data, cifs_sb, full_path, &out_iov[0]); + break; + case -EACCES: + /* + * If SMB2_OP_QUERY_INFO (called when POSIX extensions are not used) failed with + * STATUS_ACCESS_DENIED then it means that caller does not have permission to + * open the path with FILE_READ_ATTRIBUTES access and therefore cannot issue + * SMB2_OP_QUERY_INFO command. + * + * There is an alternative way how to query limited information about path but still + * suitable for stat() syscall. SMB2 OPEN/CREATE operation returns in its successful + * response subset of query information. + * + * So try to open the path without FILE_READ_ATTRIBUTES but with MAXIMUM_ALLOWED + * access which will grant the maximum possible access to the file and the response + * will contain required query information for stat() syscall. + */ + + if (tcon->posix_extensions) + break; + + num_cmds = 1; + cmds[0] = SMB2_OP_OPEN_QUERY; + in_iov[0].iov_base = data; + in_iov[0].iov_len = sizeof(*data); + oparms = CIFS_OPARMS(cifs_sb, tcon, full_path, MAXIMUM_ALLOWED, + FILE_OPEN, create_options, ACL_NO_MODE); + free_rsp_iov(out_iov, out_buftype, ARRAY_SIZE(out_iov)); + rc = smb2_compound_op(xid, tcon, cifs_sb, full_path, + &oparms, in_iov, cmds, num_cmds, + cfile, out_iov, out_buftype, NULL); + + hdr = out_iov[0].iov_base; + if (!hdr || out_buftype[0] == CIFS_NO_BUFFER) + goto out; + + if (!rc) + rc = parse_create_response(data, cifs_sb, full_path, &out_iov[0]); + break; + case -EOPNOTSUPP: + /* + * BB TODO: When support for special files added to Samba + * re-verify this path. + */ + rc = parse_create_response(data, cifs_sb, full_path, &out_iov[0]); + if (rc || !data->reparse_point) + goto out; + + /* + * Skip SMB2_OP_GET_REPARSE if symlink already parsed in create + * response. + */ + if (data->reparse.tag != IO_REPARSE_TAG_SYMLINK) { + cmds[num_cmds++] = SMB2_OP_GET_REPARSE; + if (!tcon->posix_extensions) + cmds[num_cmds++] = SMB2_OP_QUERY_WSL_EA; + } + + oparms = CIFS_OPARMS(cifs_sb, tcon, full_path, + FILE_READ_ATTRIBUTES | + FILE_READ_EA | SYNCHRONIZE, + FILE_OPEN, create_options | + OPEN_REPARSE_POINT, ACL_NO_MODE); + cifs_get_readable_path(tcon, full_path, &cfile); + free_rsp_iov(out_iov, out_buftype, ARRAY_SIZE(out_iov)); + rc = smb2_compound_op(xid, tcon, cifs_sb, full_path, + &oparms, in_iov, cmds, num_cmds, + cfile, out_iov, out_buftype, NULL); + if (rc && ea_unsupported(cmds, num_cmds, + out_iov, out_buftype)) { + if (data->reparse.tag != IO_REPARSE_TAG_LX_BLK && + data->reparse.tag != IO_REPARSE_TAG_LX_CHR) + rc = 0; + else + rc = -EOPNOTSUPP; + } + + /* + * If the symlink was already parsed in create response then it is needed to fix + * its type now (after the second call with OPEN_REPARSE_POINT which filled the + * data->fi.Attributes). If the symlink was not parsed in create response then + * the data->symlink_target was not filled yet and then the type will be fixed + * later after data->symlink_target is filled. + */ + if (data->reparse.tag == IO_REPARSE_TAG_SYMLINK && !rc && data->symlink_target) { + bool directory = le32_to_cpu(data->fi.Attributes) & ATTR_DIRECTORY; + rc = smb2_fix_symlink_target_type(&data->symlink_target, directory, cifs_sb); + } + break; + case -EREMOTE: + break; + default: + if (hdr->Status != STATUS_OBJECT_NAME_INVALID) + break; + rc2 = cifs_inval_name_dfs_link_error(xid, tcon, cifs_sb, + full_path, &islink); + if (rc2) { + rc = rc2; + goto out; + } + if (islink) + rc = -EREMOTE; + } + +out: + free_rsp_iov(out_iov, out_buftype, ARRAY_SIZE(out_iov)); + return rc; +} + +int +smb2_mkdir(const unsigned int xid, struct inode *parent_inode, umode_t mode, + struct cifs_tcon *tcon, const char *name, + struct cifs_sb_info *cifs_sb) +{ + struct cifs_open_parms oparms; + + oparms = CIFS_OPARMS(cifs_sb, tcon, name, FILE_WRITE_ATTRIBUTES, + FILE_CREATE, CREATE_NOT_FILE, mode); + return smb2_compound_op(xid, tcon, cifs_sb, + name, &oparms, NULL, + &(int){SMB2_OP_MKDIR}, 1, + NULL, NULL, NULL, NULL); +} + +void +smb2_mkdir_setinfo(struct inode *inode, const char *name, + struct cifs_sb_info *cifs_sb, struct cifs_tcon *tcon, + const unsigned int xid) +{ + struct cifs_open_parms oparms; + FILE_BASIC_INFO data = {}; + struct cifsInodeInfo *cifs_i; + struct cifsFileInfo *cfile; + struct kvec in_iov; + u32 dosattrs; + int tmprc; + + in_iov.iov_base = &data; + in_iov.iov_len = sizeof(data); + cifs_i = CIFS_I(inode); + dosattrs = cifs_i->cifsAttrs | ATTR_READONLY; + data.Attributes = cpu_to_le32(dosattrs); + cifs_get_writable_path(tcon, name, inode, FIND_ANY, &cfile); + oparms = CIFS_OPARMS(cifs_sb, tcon, name, FILE_WRITE_ATTRIBUTES, + FILE_CREATE, CREATE_NOT_FILE, ACL_NO_MODE); + tmprc = smb2_compound_op(xid, tcon, cifs_sb, name, + &oparms, &in_iov, + &(int){SMB2_OP_SET_INFO}, 1, + cfile, NULL, NULL, NULL); + if (tmprc == 0) + cifs_i->cifsAttrs = dosattrs; +} + +int +smb2_rmdir(const unsigned int xid, struct cifs_tcon *tcon, const char *name, + struct cifs_sb_info *cifs_sb) +{ + struct cifs_open_parms oparms; + + drop_cached_dir_by_name(xid, tcon, name, cifs_sb); + oparms = CIFS_OPARMS(cifs_sb, tcon, name, DELETE, + FILE_OPEN, CREATE_NOT_FILE, ACL_NO_MODE); + return smb2_compound_op(xid, tcon, cifs_sb, + name, &oparms, NULL, + &(int){SMB2_OP_UNLINK}, 1, + NULL, NULL, NULL, NULL); +} + +int +smb2_unlink(const unsigned int xid, struct cifs_tcon *tcon, const char *name, + struct cifs_sb_info *cifs_sb, struct dentry *dentry) +{ + struct kvec open_iov[SMB2_CREATE_IOV_SIZE]; + __le16 *utf16_path __free(kfree) = NULL; + int retries = 0, cur_sleep = 0; + struct TCP_Server_Info *server; + struct cifs_open_parms oparms; + struct smb2_create_req *creq; + struct inode *inode = NULL; + struct smb_rqst rqst[2]; + struct kvec rsp_iov[2]; + struct kvec close_iov; + int resp_buftype[2]; + struct cifs_fid fid; + int flags = 0; + __u8 oplock; + int rc; + + utf16_path = cifs_convert_path_to_utf16(name, cifs_sb); + if (!utf16_path) + return -ENOMEM; + + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; +again: + oplock = SMB2_OPLOCK_LEVEL_NONE; + server = cifs_pick_channel(tcon->ses); + + memset(rqst, 0, sizeof(rqst)); + memset(resp_buftype, 0, sizeof(resp_buftype)); + memset(rsp_iov, 0, sizeof(rsp_iov)); + + memset(open_iov, 0, sizeof(open_iov)); + rqst[0].rq_iov = open_iov; + rqst[0].rq_nvec = ARRAY_SIZE(open_iov); + + oparms = CIFS_OPARMS(cifs_sb, tcon, name, DELETE | FILE_READ_ATTRIBUTES, + FILE_OPEN, CREATE_DELETE_ON_CLOSE | + OPEN_REPARSE_POINT, ACL_NO_MODE); + oparms.fid = &fid; + + if (dentry) { + inode = d_inode(dentry); + if (CIFS_I(inode)->lease_granted && server->ops->get_lease_key) { + oplock = SMB2_OPLOCK_LEVEL_LEASE; + server->ops->get_lease_key(inode, &fid); + } + } + + rc = SMB2_open_init(tcon, server, + &rqst[0], &oplock, &oparms, utf16_path); + if (rc) + goto err_free; + smb2_set_next_command(tcon, &rqst[0]); + creq = rqst[0].rq_iov[0].iov_base; + creq->ShareAccess = FILE_SHARE_DELETE_LE; + + memset(&close_iov, 0, sizeof(close_iov)); + rqst[1].rq_iov = &close_iov; + rqst[1].rq_nvec = 1; + + rc = SMB2_close_init(tcon, server, &rqst[1], + COMPOUND_FID, COMPOUND_FID, false); + if (rc) + goto err_free; + smb2_set_related(&rqst[1]); + + if (retries) { + /* Back-off before retry */ + if (cur_sleep) + msleep(cur_sleep); + for (int i = 0; i < ARRAY_SIZE(rqst); i++) + smb2_set_replay(server, &rqst[i]); + } + + rc = compound_send_recv(xid, tcon->ses, server, flags, + ARRAY_SIZE(rqst), rqst, + resp_buftype, rsp_iov); + SMB2_open_free(&rqst[0]); + SMB2_close_free(&rqst[1]); + free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base); + free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base); + + if (is_replayable_error(rc) && + smb2_should_replay(tcon, &retries, &cur_sleep)) + goto again; + + /* Retry compound request without lease */ + if (rc == -EINVAL && dentry) { + dentry = NULL; + retries = 0; + cur_sleep = 0; + goto again; + } + /* + * If dentry (hence, inode) is NULL, lease break is going to + * take care of degrading leases on handles for deleted files. + */ + if (!rc && inode) + cifs_mark_open_handles_for_deleted_file(inode, name); + + return rc; + +err_free: + SMB2_open_free(&rqst[0]); + SMB2_close_free(&rqst[1]); + free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base); + free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base); + return rc; +} + +static int smb2_set_path_attr(const unsigned int xid, struct cifs_tcon *tcon, + const char *from_name, const char *to_name, + struct cifs_sb_info *cifs_sb, + __u32 create_options, __u32 access, + int command, struct cifsFileInfo *cfile, + struct dentry *dentry) +{ + struct cifs_open_parms oparms; + struct kvec in_iov; + __le16 *smb2_to_name = NULL; + int rc; + + smb2_to_name = cifs_convert_path_to_utf16(to_name, cifs_sb); + if (smb2_to_name == NULL) { + rc = -ENOMEM; + if (cfile) + cifsFileInfo_put(cfile); + goto smb2_rename_path; + } + in_iov.iov_base = smb2_to_name; + in_iov.iov_len = 2 * UniStrnlen((wchar_t *)smb2_to_name, PATH_MAX); + oparms = CIFS_OPARMS(cifs_sb, tcon, from_name, access, FILE_OPEN, + create_options, ACL_NO_MODE); + rc = smb2_compound_op(xid, tcon, cifs_sb, from_name, + &oparms, &in_iov, &command, 1, + cfile, NULL, NULL, dentry); +smb2_rename_path: + kfree(smb2_to_name); + return rc; +} + +int smb2_rename_path(const unsigned int xid, + struct cifs_tcon *tcon, + struct dentry *source_dentry, + const char *from_name, const char *to_name, + struct cifs_sb_info *cifs_sb) +{ + struct inode *inode = source_dentry ? d_inode(source_dentry) : NULL; + struct cifsFileInfo *cfile; + __u32 co = file_create_options(source_dentry); + + drop_cached_dir_by_name(xid, tcon, from_name, cifs_sb); + cifs_get_writable_path(tcon, from_name, inode, + FIND_WITH_DELETE, &cfile); + + int rc = smb2_set_path_attr(xid, tcon, from_name, to_name, cifs_sb, + co, DELETE, SMB2_OP_RENAME, cfile, source_dentry); + if (rc == -EINVAL) { + cifs_dbg(FYI, "invalid lease key, resending request without lease"); + cifs_get_writable_path(tcon, from_name, inode, + FIND_WITH_DELETE, &cfile); + rc = smb2_set_path_attr(xid, tcon, from_name, to_name, cifs_sb, + co, DELETE, SMB2_OP_RENAME, cfile, NULL); + } + return rc; +} + +static int clear_tmpfile_attr(const unsigned int xid, struct cifs_tcon *tcon, + struct inode *inode, const char *full_path) +{ + struct TCP_Server_Info *server = cifs_pick_channel(tcon->ses); + struct cifsInodeInfo *cinode = CIFS_I(inode); + FILE_BASIC_INFO fi; + + cinode->cifsAttrs &= ~(ATTR_TEMPORARY | ATTR_HIDDEN); + fi = (FILE_BASIC_INFO) { + .Attributes = cpu_to_le32(cinode->cifsAttrs), + }; + return server->ops->set_file_info(inode, full_path, &fi, xid); +} + +int smb2_create_hardlink(const unsigned int xid, + struct cifs_tcon *tcon, + struct dentry *source_dentry, + const char *from_name, const char *to_name, + struct cifs_sb_info *cifs_sb) +{ + struct inode *inode = source_dentry ? d_inode(source_dentry) : NULL; + __u32 co = file_create_options(source_dentry); + struct cifsFileInfo *cfile; + int rc; + + if (inode && test_bit(CIFS_INO_TMPFILE, &CIFS_I(inode)->flags)) { + rc = clear_tmpfile_attr(xid, tcon, inode, from_name); + if (rc) + return rc; + } + + cifs_get_writable_path(tcon, from_name, inode, + FIND_WITH_DELETE, &cfile); + return smb2_set_path_attr(xid, tcon, from_name, to_name, + cifs_sb, co, FILE_READ_ATTRIBUTES, + SMB2_OP_HARDLINK, cfile, NULL); +} + +int +smb2_set_path_size(const unsigned int xid, struct cifs_tcon *tcon, + const char *full_path, __u64 size, + struct cifs_sb_info *cifs_sb, bool set_alloc, + struct dentry *dentry) +{ + struct inode *inode = dentry ? d_inode(dentry) : NULL; + __le64 eof = cpu_to_le64(size); + struct cifs_open_parms oparms; + struct cifsFileInfo *cfile; + struct kvec in_iov; + int rc; + + in_iov.iov_base = &eof; + in_iov.iov_len = sizeof(eof); + cifs_get_writable_path(tcon, full_path, inode, FIND_ANY, &cfile); + + oparms = CIFS_OPARMS(cifs_sb, tcon, full_path, FILE_WRITE_DATA, + FILE_OPEN, 0, ACL_NO_MODE); + rc = smb2_compound_op(xid, tcon, cifs_sb, + full_path, &oparms, &in_iov, + &(int){SMB2_OP_SET_EOF}, 1, + cfile, NULL, NULL, dentry); + if (rc == -EINVAL) { + cifs_dbg(FYI, "invalid lease key, resending request without lease"); + cifs_get_writable_path(tcon, full_path, + inode, FIND_ANY, &cfile); + rc = smb2_compound_op(xid, tcon, cifs_sb, + full_path, &oparms, &in_iov, + &(int){SMB2_OP_SET_EOF}, 1, + cfile, NULL, NULL, NULL); + } + return rc; +} + +int +smb2_set_file_info(struct inode *inode, const char *full_path, + FILE_BASIC_INFO *buf, const unsigned int xid) +{ + struct kvec in_iov = { .iov_base = buf, .iov_len = sizeof(*buf), }; + struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); + struct cifsFileInfo *cfile = NULL; + struct cifs_open_parms oparms; + struct tcon_link *tlink; + struct cifs_tcon *tcon; + int rc = 0; + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) + return PTR_ERR(tlink); + tcon = tlink_tcon(tlink); + + if ((buf->CreationTime == 0) && (buf->LastAccessTime == 0) && + (buf->LastWriteTime == 0) && (buf->ChangeTime == 0)) { + if (buf->Attributes == 0) + goto out; /* would be a no op, no sense sending this */ + cifs_get_writable_path(tcon, full_path, + inode, FIND_ANY, &cfile); + } + + oparms = CIFS_OPARMS(cifs_sb, tcon, full_path, FILE_WRITE_ATTRIBUTES, + FILE_OPEN, 0, ACL_NO_MODE); + rc = smb2_compound_op(xid, tcon, cifs_sb, + full_path, &oparms, &in_iov, + &(int){SMB2_OP_SET_INFO}, 1, + cfile, NULL, NULL, NULL); +out: + cifs_put_tlink(tlink); + return rc; +} + +struct inode *smb2_create_reparse_inode(struct cifs_open_info_data *data, + struct super_block *sb, + const unsigned int xid, + struct cifs_tcon *tcon, + const char *full_path, + bool directory, + struct kvec *reparse_iov, + struct kvec *xattr_iov) +{ + struct cifs_open_parms oparms; + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + struct cifsFileInfo *cfile; + struct inode *new = NULL; + int out_buftype[4] = {}; + struct kvec out_iov[4] = {}; + struct kvec in_iov[2]; + int cmds[2]; + int rc; + int i; + + /* + * If server filesystem does not support reparse points then do not + * attempt to create reparse point. This will prevent creating unusable + * empty object on the server. + */ + if (!CIFS_REPARSE_SUPPORT(tcon)) + return ERR_PTR(-EOPNOTSUPP); + + oparms = CIFS_OPARMS(cifs_sb, tcon, full_path, + SYNCHRONIZE | DELETE | + FILE_READ_ATTRIBUTES | + FILE_WRITE_ATTRIBUTES, + FILE_CREATE, + (directory ? CREATE_NOT_FILE : CREATE_NOT_DIR) | OPEN_REPARSE_POINT, + ACL_NO_MODE); + if (xattr_iov) + oparms.ea_cctx = xattr_iov; + + cmds[0] = SMB2_OP_SET_REPARSE; + in_iov[0] = *reparse_iov; + in_iov[1].iov_base = data; + in_iov[1].iov_len = sizeof(*data); + + if (tcon->posix_extensions) { + cmds[1] = SMB2_OP_POSIX_QUERY_INFO; + cifs_get_writable_path(tcon, full_path, NULL, FIND_ANY, &cfile); + rc = smb2_compound_op(xid, tcon, cifs_sb, full_path, &oparms, + in_iov, cmds, 2, cfile, out_iov, out_buftype, NULL); + if (!rc) { + rc = smb311_posix_get_inode_info(&new, full_path, + data, sb, xid); + } + } else { + cmds[1] = SMB2_OP_QUERY_INFO; + cifs_get_writable_path(tcon, full_path, NULL, FIND_ANY, &cfile); + rc = smb2_compound_op(xid, tcon, cifs_sb, full_path, &oparms, + in_iov, cmds, 2, cfile, out_iov, out_buftype, NULL); + if (!rc) { + rc = cifs_get_inode_info(&new, full_path, + data, sb, xid, NULL); + } + } + + + /* + * If CREATE was successful but SMB2_OP_SET_REPARSE failed then + * remove the intermediate object created by CREATE. Otherwise + * empty object stay on the server when reparse call failed. + */ + if (rc && + out_iov[0].iov_base != NULL && out_buftype[0] != CIFS_NO_BUFFER && + ((struct smb2_hdr *)out_iov[0].iov_base)->Status == STATUS_SUCCESS && + (out_iov[1].iov_base == NULL || out_buftype[1] == CIFS_NO_BUFFER || + ((struct smb2_hdr *)out_iov[1].iov_base)->Status != STATUS_SUCCESS)) + smb2_unlink(xid, tcon, full_path, cifs_sb, NULL); + + for (i = 0; i < ARRAY_SIZE(out_buftype); i++) + free_rsp_buf(out_buftype[i], out_iov[i].iov_base); + + return rc ? ERR_PTR(rc) : new; +} + +int smb2_query_reparse_point(const unsigned int xid, + struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, + const char *full_path, + u32 *tag, struct kvec *rsp, + int *rsp_buftype) +{ + struct cifs_open_parms oparms; + struct cifs_open_info_data data = {}; + struct cifsFileInfo *cfile; + struct kvec in_iov = { .iov_base = &data, .iov_len = sizeof(data), }; + int rc; + + cifs_dbg(FYI, "%s: path: %s\n", __func__, full_path); + + cifs_get_readable_path(tcon, full_path, &cfile); + oparms = CIFS_OPARMS(cifs_sb, tcon, full_path, + FILE_READ_ATTRIBUTES | FILE_READ_EA | SYNCHRONIZE, + FILE_OPEN, OPEN_REPARSE_POINT, ACL_NO_MODE); + rc = smb2_compound_op(xid, tcon, cifs_sb, + full_path, &oparms, &in_iov, + &(int){SMB2_OP_GET_REPARSE}, 1, + cfile, NULL, NULL, NULL); + if (rc) + goto out; + + *tag = data.reparse.tag; + *rsp = data.reparse.io.iov; + *rsp_buftype = data.reparse.io.buftype; + memset(&data.reparse.io.iov, 0, sizeof(data.reparse.io.iov)); + data.reparse.io.buftype = CIFS_NO_BUFFER; +out: + cifs_free_open_info(&data); + return rc; +} + +static inline __le16 *utf16_smb2_path(struct cifs_sb_info *cifs_sb, + const char *name, size_t namelen) +{ + int len; + + if (*name == '\\' || + (cifs_sb_master_tlink(cifs_sb) && + cifs_sb_master_tcon(cifs_sb)->posix_extensions && *name == '/')) + name++; + return cifs_strndup_to_utf16(name, namelen, &len, + cifs_sb->local_nls, + cifs_remap(cifs_sb)); +} + +int smb2_rename_pending_delete(const char *full_path, + struct dentry *dentry, + const unsigned int xid) +{ + struct cifsInodeInfo *cinode = CIFS_I(d_inode(dentry)); + struct cifs_sb_info *cifs_sb = CIFS_SB(dentry); + __le16 *utf16_path __free(kfree) = NULL; + __u32 co = file_create_options(dentry); + int cmds[] = { + SMB2_OP_SET_INFO, + SMB2_OP_RENAME, + SMB2_OP_UNLINK, + }; + const int num_cmds = ARRAY_SIZE(cmds); + char *to_name __free(kfree) = NULL; + __u32 attrs = cinode->cifsAttrs; + struct cifs_open_parms oparms; + struct cifsFileInfo *cfile; + struct tcon_link *tlink; + struct cifs_tcon *tcon; + struct kvec iov[2]; + int rc; + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) + return PTR_ERR(tlink); + tcon = tlink_tcon(tlink); + + to_name = cifs_silly_fullpath(dentry); + if (IS_ERR(to_name)) { + rc = PTR_ERR(to_name); + to_name = NULL; + goto out; + } + + utf16_path = utf16_smb2_path(cifs_sb, to_name, strlen(to_name)); + if (!utf16_path) { + rc = -ENOMEM; + goto out; + } + + drop_cached_dir_by_name(xid, tcon, full_path, cifs_sb); + oparms = CIFS_OPARMS(cifs_sb, tcon, full_path, + DELETE | FILE_WRITE_ATTRIBUTES, + FILE_OPEN, co, ACL_NO_MODE); + + attrs &= ~ATTR_READONLY; + if (!attrs) + attrs = ATTR_NORMAL; + if (d_inode(dentry)->i_nlink <= 1) + attrs |= ATTR_HIDDEN; + iov[0].iov_base = &(FILE_BASIC_INFO) { + .Attributes = cpu_to_le32(attrs), + }; + iov[0].iov_len = sizeof(FILE_BASIC_INFO); + iov[1].iov_base = utf16_path; + iov[1].iov_len = sizeof(*utf16_path) * UniStrlen((wchar_t *)utf16_path); + + cifs_get_writable_path(tcon, full_path, d_inode(dentry), + FIND_WITH_DELETE, &cfile); + rc = smb2_compound_op(xid, tcon, cifs_sb, full_path, &oparms, iov, + cmds, num_cmds, cfile, NULL, NULL, dentry); + if (rc == -EINVAL) { + cifs_dbg(FYI, "invalid lease key, resending request without lease\n"); + cifs_get_writable_path(tcon, full_path, d_inode(dentry), + FIND_WITH_DELETE, &cfile); + rc = smb2_compound_op(xid, tcon, cifs_sb, full_path, &oparms, iov, + cmds, num_cmds, cfile, NULL, NULL, NULL); + } + if (!rc) { + set_bit(CIFS_INO_DELETE_PENDING, &cinode->flags); + } else { + cifs_tcon_dbg(FYI, "%s: failed to rename '%s' to '%s': %d\n", + __func__, full_path, to_name, rc); + rc = smb_EIO1(smb_eio_trace_pend_del_fail, rc); + } +out: + cifs_put_tlink(tlink); + return rc; +} diff --git a/fs/smb/client/smb2maperror.c b/fs/smb/client/smb2maperror.c new file mode 100644 index 000000000..d86f2460d --- /dev/null +++ b/fs/smb/client/smb2maperror.c @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * Functions which do error mapping of SMB2 status codes to POSIX errors + * + * Copyright (C) International Business Machines Corp., 2009 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ +#include + +#include + +#include "cifsproto.h" +#include "cifs_debug.h" +#include "smb2proto.h" +#include "smb2glob.h" +#include "../common/smb2status.h" +#include "trace.h" + +static const struct status_to_posix_error smb2_error_map_table[] = { +/* + * Automatically generated by the `gen_smb2_mapping` script, + * sorted by NT status code (cpu-endian, ascending) + */ +#include "smb2_mapping_table.c" +}; + +static __always_inline int cmp_smb2_status(const void *_key, const void *_pivot) +{ + __u32 key = *(__u32 *)_key; + const struct status_to_posix_error *pivot = _pivot; + + if (key < pivot->smb2_status) + return -1; + if (key > pivot->smb2_status) + return 1; + return 0; +} + +static const struct status_to_posix_error *smb2_get_err_map(__u32 smb2_status) +{ + const struct status_to_posix_error *err_map; + + err_map = __inline_bsearch(&smb2_status, smb2_error_map_table, + ARRAY_SIZE(smb2_error_map_table), + sizeof(struct status_to_posix_error), + cmp_smb2_status); + return err_map; +} + +int +map_smb2_to_linux_error(char *buf, bool log_err) +{ + struct smb2_hdr *shdr = (struct smb2_hdr *)buf; + int rc = -EIO; + __le32 smb2err = shdr->Status; + const struct status_to_posix_error *err_map; + + if (smb2err == 0) { + trace_smb3_cmd_done(le32_to_cpu(shdr->Id.SyncId.TreeId), + le64_to_cpu(shdr->SessionId), + le16_to_cpu(shdr->Command), + le64_to_cpu(shdr->MessageId)); + return 0; + } + + log_err = (log_err && (smb2err != STATUS_MORE_PROCESSING_REQUIRED) && + (smb2err != STATUS_END_OF_FILE)) || + (cifsFYI & CIFS_RC); + + err_map = smb2_get_err_map(le32_to_cpu(smb2err)); + if (!err_map) + goto out; + + rc = err_map->posix_error; + if (log_err) + pr_notice("Status code returned 0x%08x %s\n", + err_map->smb2_status, err_map->status_string); + +out: + /* on error mapping not found - return EIO */ + + cifs_dbg(FYI, "Mapping SMB2 status code 0x%08x to POSIX err %d\n", + le32_to_cpu(smb2err), rc); + + trace_smb3_cmd_err(le32_to_cpu(shdr->Id.SyncId.TreeId), + le64_to_cpu(shdr->SessionId), + le16_to_cpu(shdr->Command), + le64_to_cpu(shdr->MessageId), + le32_to_cpu(smb2err), rc); + if (rc == -EIO) + smb_EIO1(smb_eio_trace_smb2_received_error, le32_to_cpu(smb2err)); + return rc; +} + +int __init smb2_init_maperror(void) +{ + unsigned int i; + + /* Check whether the array is sorted in ascending order */ + for (i = 1; i < ARRAY_SIZE(smb2_error_map_table); i++) { + if (smb2_error_map_table[i].smb2_status >= + smb2_error_map_table[i - 1].smb2_status) + continue; + + pr_err("smb2_error_map_table array order is incorrect\n"); + return -EINVAL; + } + + return 0; +} + +#if IS_ENABLED(CONFIG_SMB_KUNIT_TESTS) +const struct status_to_posix_error *smb2_get_err_map_test(__u32 smb2_status) +{ + return smb2_get_err_map(smb2_status); +} +EXPORT_SYMBOL_IF_KUNIT(smb2_get_err_map_test); + +const struct status_to_posix_error *smb2_error_map_table_test = smb2_error_map_table; +EXPORT_SYMBOL_IF_KUNIT(smb2_error_map_table_test); + +unsigned int smb2_error_map_num = ARRAY_SIZE(smb2_error_map_table); +EXPORT_SYMBOL_IF_KUNIT(smb2_error_map_num); +#endif diff --git a/fs/smb/client/smb2maperror_test.c b/fs/smb/client/smb2maperror_test.c new file mode 100644 index 000000000..44dc5e899 --- /dev/null +++ b/fs/smb/client/smb2maperror_test.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * KUnit tests of SMB2 maperror + * + * Copyright (C) 2025 KylinSoft Co., Ltd. All rights reserved. + * Author(s): ChenXiaoSong + * + */ + +#include +#include "cifsglob.h" +#include "smb2glob.h" +#include "smb2proto.h" + +static void +test_cmp_map(struct kunit *test, const struct status_to_posix_error *expect) +{ + const struct status_to_posix_error *result; + + result = smb2_get_err_map_test(expect->smb2_status); + KUNIT_ASSERT_NOT_NULL(test, result); + KUNIT_EXPECT_EQ(test, expect->smb2_status, result->smb2_status); + KUNIT_EXPECT_EQ(test, expect->posix_error, result->posix_error); + KUNIT_EXPECT_STREQ(test, expect->status_string, result->status_string); +} + +static void maperror_test_check_search(struct kunit *test) +{ + unsigned int i; + + for (i = 0; i < smb2_error_map_num; i++) + test_cmp_map(test, &smb2_error_map_table_test[i]); +} + +static struct kunit_case maperror_test_cases[] = { + KUNIT_CASE(maperror_test_check_search), + {} +}; + +static struct kunit_suite maperror_suite = { + .name = "smb2_maperror", + .test_cases = maperror_test_cases, +}; + +kunit_test_suite(maperror_suite); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("KUnit tests of SMB2 maperror"); +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); diff --git a/fs/smb/client/smb2misc.c b/fs/smb/client/smb2misc.c new file mode 100644 index 000000000..9068175e5 --- /dev/null +++ b/fs/smb/client/smb2misc.c @@ -0,0 +1,957 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * Copyright (C) International Business Machines Corp., 2002,2011 + * Etersoft, 2012 + * Author(s): Steve French (sfrench@us.ibm.com) + * Pavel Shilovsky (pshilovsky@samba.org) 2012 + * + */ +#include +#include +#include "cifsglob.h" +#include "cifsproto.h" +#include "smb2proto.h" +#include "cifs_debug.h" +#include "cifs_unicode.h" +#include "../common/smb2status.h" +#include "smb2glob.h" +#include "nterr.h" +#include "cached_dir.h" + +static unsigned int __smb2_calc_size(void *buf, bool *have_data, + bool *data_area_overlap); + +static int +check_smb2_hdr(struct smb2_hdr *shdr, __u64 mid) +{ + __u64 wire_mid = le64_to_cpu(shdr->MessageId); + + /* + * Make sure that this really is an SMB, that it is a response, + * and that the message ids match. + */ + if ((shdr->ProtocolId == SMB2_PROTO_NUMBER) && + (mid == wire_mid)) { + if (shdr->Flags & SMB2_FLAGS_SERVER_TO_REDIR) + return 0; + else { + /* only one valid case where server sends us request */ + if (shdr->Command == SMB2_OPLOCK_BREAK) + return 0; + else + cifs_dbg(VFS, "Received Request not response\n"); + } + } else { /* bad signature or mid */ + if (shdr->ProtocolId != SMB2_PROTO_NUMBER) + cifs_dbg(VFS, "Bad protocol string signature header %x\n", + le32_to_cpu(shdr->ProtocolId)); + if (mid != wire_mid) + cifs_dbg(VFS, "Mids do not match: %llu and %llu\n", + mid, wire_mid); + } + cifs_dbg(VFS, "Bad SMB detected. The Mid=%llu\n", wire_mid); + return 1; +} + +/* + * The following table defines the expected "StructureSize" of SMB2 responses + * in order by SMB2 command. This is similar to "wct" in SMB/CIFS responses. + * + * Note that commands are defined in smb2pdu.h in le16 but the array below is + * indexed by command in host byte order + */ +static const __le16 smb2_rsp_struct_sizes[NUMBER_OF_SMB2_COMMANDS] = { + /* SMB2_NEGOTIATE */ cpu_to_le16(65), + /* SMB2_SESSION_SETUP */ cpu_to_le16(9), + /* SMB2_LOGOFF */ cpu_to_le16(4), + /* SMB2_TREE_CONNECT */ cpu_to_le16(16), + /* SMB2_TREE_DISCONNECT */ cpu_to_le16(4), + /* SMB2_CREATE */ cpu_to_le16(89), + /* SMB2_CLOSE */ cpu_to_le16(60), + /* SMB2_FLUSH */ cpu_to_le16(4), + /* SMB2_READ */ cpu_to_le16(17), + /* SMB2_WRITE */ cpu_to_le16(17), + /* SMB2_LOCK */ cpu_to_le16(4), + /* SMB2_IOCTL */ cpu_to_le16(49), + /* BB CHECK this ... not listed in documentation */ + /* SMB2_CANCEL */ cpu_to_le16(0), + /* SMB2_ECHO */ cpu_to_le16(4), + /* SMB2_QUERY_DIRECTORY */ cpu_to_le16(9), + /* SMB2_CHANGE_NOTIFY */ cpu_to_le16(9), + /* SMB2_QUERY_INFO */ cpu_to_le16(9), + /* SMB2_SET_INFO */ cpu_to_le16(2), + /* BB FIXME can also be 44 for lease break */ + /* SMB2_OPLOCK_BREAK */ cpu_to_le16(24) +}; + +#define SMB311_NEGPROT_BASE_SIZE (sizeof(struct smb2_hdr) + sizeof(struct smb2_negotiate_rsp)) + +static __u32 get_neg_ctxt_len(struct smb2_hdr *hdr, __u32 len, + __u32 non_ctxlen) +{ + __u16 neg_count; + __u32 nc_offset, size_of_pad_before_neg_ctxts; + struct smb2_negotiate_rsp *pneg_rsp = (struct smb2_negotiate_rsp *)hdr; + + /* Negotiate contexts are only valid for latest dialect SMB3.11 */ + neg_count = le16_to_cpu(pneg_rsp->NegotiateContextCount); + if ((neg_count == 0) || + (pneg_rsp->DialectRevision != cpu_to_le16(SMB311_PROT_ID))) + return 0; + + /* + * if SPNEGO blob present (ie the RFC2478 GSS info which indicates + * which security mechanisms the server supports) make sure that + * the negotiate contexts start after it + */ + nc_offset = le32_to_cpu(pneg_rsp->NegotiateContextOffset); + /* + * non_ctxlen is at least shdr->StructureSize + pdu->StructureSize2 + * and the latter is 1 byte bigger than the fix-sized area of the + * NEGOTIATE response + */ + if (nc_offset + 1 < non_ctxlen) { + pr_warn_once("Invalid negotiate context offset %d\n", nc_offset); + return 0; + } else if (nc_offset + 1 == non_ctxlen) { + cifs_dbg(FYI, "no SPNEGO security blob in negprot rsp\n"); + size_of_pad_before_neg_ctxts = 0; + } else if (non_ctxlen == SMB311_NEGPROT_BASE_SIZE + 1) + /* has padding, but no SPNEGO blob */ + size_of_pad_before_neg_ctxts = nc_offset - non_ctxlen + 1; + else + size_of_pad_before_neg_ctxts = nc_offset - non_ctxlen; + + /* Verify that at least minimal negotiate contexts fit within frame */ + if (len < nc_offset + (neg_count * sizeof(struct smb2_neg_context))) { + pr_warn_once("negotiate context goes beyond end\n"); + return 0; + } + + cifs_dbg(FYI, "length of negcontexts %d pad %d\n", + len - nc_offset, size_of_pad_before_neg_ctxts); + + /* length of negcontexts including pad from end of sec blob to them */ + return (len - nc_offset) + size_of_pad_before_neg_ctxts; +} + +int +smb2_check_message(char *buf, unsigned int pdu_len, unsigned int len, + struct TCP_Server_Info *server) +{ + struct TCP_Server_Info *pserver; + struct smb2_hdr *shdr = (struct smb2_hdr *)buf; + struct smb2_pdu *pdu = (struct smb2_pdu *)shdr; + int hdr_size = sizeof(struct smb2_hdr); + int pdu_size = sizeof(struct smb2_pdu); + int command; + __u32 calc_len; /* calculated length */ + __u64 mid; + bool have_data; + bool data_area_overlap; + + /* If server is a channel, select the primary channel */ + pserver = SERVER_IS_CHAN(server) ? server->primary_server : server; + + /* + * Add function to do table lookup of StructureSize by command + * ie Validate the wct via smb2_struct_sizes table above + */ + if (shdr->ProtocolId == SMB2_TRANSFORM_PROTO_NUM) { + struct smb2_transform_hdr *thdr = + (struct smb2_transform_hdr *)buf; + struct cifs_ses *ses = NULL; + struct cifs_ses *iter; + + /* decrypt frame now that it is completely read in */ + spin_lock(&cifs_tcp_ses_lock); + list_for_each_entry(iter, &pserver->smb_ses_list, smb_ses_list) { + if (iter->Suid == le64_to_cpu(thdr->SessionId)) { + ses = iter; + break; + } + } + spin_unlock(&cifs_tcp_ses_lock); + if (!ses) { + cifs_dbg(VFS, "no decryption - session id not found\n"); + return 1; + } + } + + mid = le64_to_cpu(shdr->MessageId); + if (check_smb2_hdr(shdr, mid)) + return 1; + + if (shdr->StructureSize != SMB2_HEADER_STRUCTURE_SIZE) { + cifs_dbg(VFS, "Invalid structure size %u\n", + le16_to_cpu(shdr->StructureSize)); + return 1; + } + + command = le16_to_cpu(shdr->Command); + if (command >= NUMBER_OF_SMB2_COMMANDS) { + cifs_dbg(VFS, "Invalid SMB2 command %d\n", command); + return 1; + } + + if (len < pdu_size) { + if ((len >= hdr_size) + && (shdr->Status != 0)) { + pdu->StructureSize2 = 0; + /* + * As with SMB/CIFS, on some error cases servers may + * not return wct properly + */ + return 0; + } else { + cifs_dbg(VFS, "Length less than SMB header size\n"); + } + return 1; + } + if (len > CIFSMaxBufSize + MAX_SMB2_HDR_SIZE) { + cifs_dbg(VFS, "SMB length greater than maximum, mid=%llu\n", + mid); + return 1; + } + + if (smb2_rsp_struct_sizes[command] != pdu->StructureSize2) { + if (command != SMB2_OPLOCK_BREAK_HE && (shdr->Status == 0 || + pdu->StructureSize2 != SMB2_ERROR_STRUCTURE_SIZE2_LE)) { + /* error packets have 9 byte structure size */ + cifs_dbg(VFS, "Invalid response size %u for command %d\n", + le16_to_cpu(pdu->StructureSize2), command); + return 1; + } else if (command == SMB2_OPLOCK_BREAK_HE + && (shdr->Status == 0) + && (le16_to_cpu(pdu->StructureSize2) != 44) + && (le16_to_cpu(pdu->StructureSize2) != 36)) { + /* special case for SMB2.1 lease break message */ + cifs_dbg(VFS, "Invalid response size %d for oplock break\n", + le16_to_cpu(pdu->StructureSize2)); + return 1; + } + } + + have_data = false; + data_area_overlap = false; + calc_len = __smb2_calc_size(buf, &have_data, &data_area_overlap); + + /* Reject responses whose data area overlaps the fixed area. */ + if (data_area_overlap) + return 1; + + /* For SMB2_IOCTL, OutputOffset and OutputLength are optional, so might + * be 0, and not a real miscalculation */ + if (command == SMB2_IOCTL_HE && calc_len == 0) + return 0; + + if (command == SMB2_NEGOTIATE_HE) + calc_len += get_neg_ctxt_len(shdr, len, calc_len); + + if (len != calc_len) { + /* create failed on symlink */ + if (command == SMB2_CREATE_HE && + shdr->Status == STATUS_STOPPED_ON_SYMLINK && + len > calc_len) + return 0; + /* Windows 7 server returns 24 bytes more */ + if (calc_len + 24 == len && command == SMB2_OPLOCK_BREAK_HE) + return 0; + /* + * Server can return one byte more due to implied bcc[0]. + * Allow it only when there is no data area; if data_length > 0 + * the +1 gap indicates an overreported data length rather than + * the bcc[0] omission. + */ + if (calc_len == len + 1 && !have_data) + return 0; + + /* + * Some windows servers (win2016) will pad also the final + * PDU in a compound to 8 bytes. + */ + if (ALIGN(calc_len, 8) == len) + return 0; + + /* + * MacOS server pads after SMB2.1 write response with 3 bytes + * of junk. Other servers match RFC1001 len to actual + * SMB2/SMB3 frame length (header + smb2 response specific data) + * Some windows servers also pad up to 8 bytes when compounding. + */ + if (calc_len < len) + return 0; + + /* Only log a message if len was really miscalculated */ + if (unlikely(cifsFYI)) + cifs_dbg(FYI, "Server response too short: calculated " + "length %u doesn't match read length %u (cmd=%d, mid=%llu)\n", + calc_len, len, command, mid); + else + pr_warn("Server response too short: calculated length " + "%u doesn't match read length %u (cmd=%d, mid=%llu)\n", + calc_len, len, command, mid); + + return 1; + } + return 0; +} + +/* + * The size of the variable area depends on the offset and length fields + * located in different fields for various SMB2 responses. SMB2 responses + * with no variable length info, show an offset of zero for the offset field. + */ +static const bool has_smb2_data_area[NUMBER_OF_SMB2_COMMANDS] = { + /* SMB2_NEGOTIATE */ true, + /* SMB2_SESSION_SETUP */ true, + /* SMB2_LOGOFF */ false, + /* SMB2_TREE_CONNECT */ false, + /* SMB2_TREE_DISCONNECT */ false, + /* SMB2_CREATE */ true, + /* SMB2_CLOSE */ false, + /* SMB2_FLUSH */ false, + /* SMB2_READ */ true, + /* SMB2_WRITE */ false, + /* SMB2_LOCK */ false, + /* SMB2_IOCTL */ true, + /* SMB2_CANCEL */ false, /* BB CHECK this not listed in documentation */ + /* SMB2_ECHO */ false, + /* SMB2_QUERY_DIRECTORY */ true, + /* SMB2_CHANGE_NOTIFY */ true, + /* SMB2_QUERY_INFO */ true, + /* SMB2_SET_INFO */ false, + /* SMB2_OPLOCK_BREAK */ false +}; + +/* + * Returns the pointer to the beginning of the data area. Length of the data + * area and the offset to it (from the beginning of the smb are also returned. + */ +char * +smb2_get_data_area_len(int *off, int *len, struct smb2_hdr *shdr) +{ + const int max_off = 4096; + const int max_len = 128 * 1024; + + *off = 0; + *len = 0; + + /* error responses do not have data area */ + if (shdr->Status && shdr->Status != STATUS_MORE_PROCESSING_REQUIRED && + (((struct smb2_err_rsp *)shdr)->StructureSize) == + SMB2_ERROR_STRUCTURE_SIZE2_LE) + return NULL; + + /* + * Following commands have data areas so we have to get the location + * of the data buffer offset and data buffer length for the particular + * command. + */ + switch (shdr->Command) { + case SMB2_NEGOTIATE: + *off = le16_to_cpu( + ((struct smb2_negotiate_rsp *)shdr)->SecurityBufferOffset); + *len = le16_to_cpu( + ((struct smb2_negotiate_rsp *)shdr)->SecurityBufferLength); + break; + case SMB2_SESSION_SETUP: + *off = le16_to_cpu( + ((struct smb2_sess_setup_rsp *)shdr)->SecurityBufferOffset); + *len = le16_to_cpu( + ((struct smb2_sess_setup_rsp *)shdr)->SecurityBufferLength); + break; + case SMB2_CREATE: + *off = le32_to_cpu( + ((struct smb2_create_rsp *)shdr)->CreateContextsOffset); + *len = le32_to_cpu( + ((struct smb2_create_rsp *)shdr)->CreateContextsLength); + break; + case SMB2_QUERY_INFO: + *off = le16_to_cpu( + ((struct smb2_query_info_rsp *)shdr)->OutputBufferOffset); + *len = le32_to_cpu( + ((struct smb2_query_info_rsp *)shdr)->OutputBufferLength); + break; + case SMB2_READ: + /* TODO: is this a bug ? */ + *off = ((struct smb2_read_rsp *)shdr)->DataOffset; + *len = le32_to_cpu(((struct smb2_read_rsp *)shdr)->DataLength); + break; + case SMB2_QUERY_DIRECTORY: + *off = le16_to_cpu( + ((struct smb2_query_directory_rsp *)shdr)->OutputBufferOffset); + *len = le32_to_cpu( + ((struct smb2_query_directory_rsp *)shdr)->OutputBufferLength); + break; + case SMB2_IOCTL: + *off = le32_to_cpu( + ((struct smb2_ioctl_rsp *)shdr)->OutputOffset); + *len = le32_to_cpu( + ((struct smb2_ioctl_rsp *)shdr)->OutputCount); + break; + case SMB2_CHANGE_NOTIFY: + *off = le16_to_cpu( + ((struct smb2_change_notify_rsp *)shdr)->OutputBufferOffset); + *len = le32_to_cpu( + ((struct smb2_change_notify_rsp *)shdr)->OutputBufferLength); + break; + default: + cifs_dbg(VFS, "no length check for command %d\n", le16_to_cpu(shdr->Command)); + break; + } + + /* + * Invalid length or offset probably means data area is invalid, but + * we have little choice but to ignore the data area in this case. + */ + if (unlikely(*off < 0 || *off > max_off || + *len < 0 || *len > max_len)) { + cifs_dbg(VFS, "%s: invalid data area (off=%d len=%d)\n", + __func__, *off, *len); + *off = 0; + *len = 0; + } else if (*off == 0) { + *len = 0; + } + + /* return pointer to beginning of data area, ie offset from SMB start */ + if (*off > 0 && *len > 0) + return (char *)shdr + *off; + return NULL; +} + +/* + * Calculate the size of the SMB message based on the fixed header, fixed + * parameter area, and variable data area. + * + * If have_data is not NULL, it is set when a non-empty data area is found. + * If data_area_overlap is not NULL, it is set when the data area overlaps + * the fixed area. + */ +static unsigned int +__smb2_calc_size(void *buf, bool *have_data, bool *data_area_overlap) +{ + struct smb2_pdu *pdu = buf; + struct smb2_hdr *shdr = &pdu->hdr; + int offset; /* the offset from the beginning of SMB to data area */ + int data_length = 0; /* the length of the variable length data area */ + /* Structure Size has already been checked to make sure it is 64 */ + int len = le16_to_cpu(shdr->StructureSize); + + if (have_data) + *have_data = false; + if (data_area_overlap) + *data_area_overlap = false; + + /* + * StructureSize2, ie length of fixed parameter area has already + * been checked to make sure it is the correct length. + */ + len += le16_to_cpu(pdu->StructureSize2); + + if (has_smb2_data_area[le16_to_cpu(shdr->Command)] == false) + goto calc_size_exit; + + smb2_get_data_area_len(&offset, &data_length, shdr); + cifs_dbg(FYI, "SMB2 data length %d offset %d\n", data_length, offset); + + if (data_length > 0) { + /* + * Check to make sure that data area begins after fixed area, + * Note that last byte of the fixed area is part of data area + * for some commands, typically those with odd StructureSize, + * so we must add one to the calculation. + */ + if (offset + 1 < len) { + cifs_dbg(VFS, "data area offset %d overlaps SMB2 header %d\n", + offset + 1, len); + if (data_area_overlap) + *data_area_overlap = true; + data_length = 0; + goto calc_size_exit; + } else { + len = offset + data_length; + } + } +calc_size_exit: + cifs_dbg(FYI, "SMB2 len %d\n", len); + if (have_data) + *have_data = (data_length > 0); + return len; +} + +unsigned int +smb2_calc_size(void *buf) +{ + return __smb2_calc_size(buf, NULL, NULL); +} + +/* Note: caller must free return buffer */ +__le16 * +cifs_convert_path_to_utf16(const char *from, struct cifs_sb_info *cifs_sb) +{ + const char *start_of_path; + int len; + + /* Windows doesn't allow paths beginning with \ */ + if (from[0] == '\\') + start_of_path = from + 1; + + /* SMB311 POSIX extensions paths do not include leading slash */ + else if (cifs_sb_master_tlink(cifs_sb) && + cifs_sb_master_tcon(cifs_sb)->posix_extensions && + (from[0] == '/')) { + start_of_path = from + 1; + } else + start_of_path = from; + + return cifs_strndup_to_utf16(start_of_path, PATH_MAX, &len, + cifs_sb->local_nls, cifs_remap(cifs_sb)); +} + +__le32 smb2_get_lease_state(struct cifsInodeInfo *cinode, unsigned int oplock) +{ + unsigned int sbflags = cifs_sb_flags(CIFS_SB(cinode)); + __le32 lease = 0; + + if ((oplock & CIFS_CACHE_WRITE_FLG) || (sbflags & CIFS_MOUNT_RW_CACHE)) + lease |= SMB2_LEASE_WRITE_CACHING_LE; + if (oplock & CIFS_CACHE_HANDLE_FLG) + lease |= SMB2_LEASE_HANDLE_CACHING_LE; + if ((oplock & CIFS_CACHE_READ_FLG) || (sbflags & CIFS_MOUNT_RO_CACHE)) + lease |= SMB2_LEASE_READ_CACHING_LE; + return lease; +} + +struct smb2_lease_break_work { + struct work_struct lease_break; + struct tcon_link *tlink; + __u8 lease_key[16]; + __le32 lease_state; +}; + +static void +cifs_ses_oplock_break(struct work_struct *work) +{ + struct smb2_lease_break_work *lw = container_of(work, + struct smb2_lease_break_work, lease_break); + int rc = 0; + + rc = SMB2_lease_break(0, tlink_tcon(lw->tlink), lw->lease_key, + lw->lease_state); + + cifs_dbg(FYI, "Lease release rc %d\n", rc); + cifs_put_tlink(lw->tlink); + kfree(lw); +} + +static void +smb2_queue_pending_open_break(struct tcon_link *tlink, __u8 *lease_key, + __le32 new_lease_state) +{ + struct smb2_lease_break_work *lw; + + lw = kmalloc_obj(struct smb2_lease_break_work); + if (!lw) { + cifs_put_tlink(tlink); + return; + } + + INIT_WORK(&lw->lease_break, cifs_ses_oplock_break); + lw->tlink = tlink; + lw->lease_state = new_lease_state; + memcpy(lw->lease_key, lease_key, SMB2_LEASE_KEY_SIZE); + queue_work(cifsiod_wq, &lw->lease_break); +} + +static bool +smb2_tcon_has_lease(struct cifs_tcon *tcon, struct smb2_lease_break *rsp) +{ + __u8 lease_state; + struct cifsFileInfo *cfile; + struct cifsInodeInfo *cinode; + int ack_req = le32_to_cpu(rsp->Flags & + SMB2_NOTIFY_BREAK_LEASE_FLAG_ACK_REQUIRED); + + lease_state = le32_to_cpu(rsp->NewLeaseState); + + list_for_each_entry(cfile, &tcon->openFileList, tlist) { + cinode = CIFS_I(d_inode(cfile->dentry)); + + if (memcmp(cinode->lease_key, rsp->LeaseKey, + SMB2_LEASE_KEY_SIZE)) + continue; + + cifs_dbg(FYI, "found in the open list\n"); + cifs_dbg(FYI, "lease key match, lease break 0x%x\n", + lease_state); + + if (ack_req) + cfile->oplock_break_cancelled = false; + else + cfile->oplock_break_cancelled = true; + + set_bit(CIFS_INODE_PENDING_OPLOCK_BREAK, &cinode->flags); + + cfile->oplock_epoch = le16_to_cpu(rsp->Epoch); + cfile->oplock_level = lease_state; + + cifs_queue_oplock_break(cfile); + return true; + } + + return false; +} + +static struct cifs_pending_open * +smb2_tcon_find_pending_open_lease(struct cifs_tcon *tcon, + struct smb2_lease_break *rsp) +{ + __u8 lease_state = le32_to_cpu(rsp->NewLeaseState); + int ack_req = le32_to_cpu(rsp->Flags & + SMB2_NOTIFY_BREAK_LEASE_FLAG_ACK_REQUIRED); + struct cifs_pending_open *open; + struct cifs_pending_open *found = NULL; + + list_for_each_entry(open, &tcon->pending_opens, olist) { + if (memcmp(open->lease_key, rsp->LeaseKey, + SMB2_LEASE_KEY_SIZE)) + continue; + + if (!found && ack_req) { + found = open; + } + + cifs_dbg(FYI, "found in the pending open list\n"); + cifs_dbg(FYI, "lease key match, lease break 0x%x\n", + lease_state); + + open->oplock = lease_state; + } + + return found; +} + +static bool +smb2_is_valid_lease_break(char *buffer, struct TCP_Server_Info *server) +{ + struct smb2_lease_break *rsp = (struct smb2_lease_break *)buffer; + struct TCP_Server_Info *pserver; + struct cifs_ses *ses; + struct cifs_tcon *tcon; + struct cifs_pending_open *open; + + /* Trace receipt of lease break request from server */ + trace_smb3_lease_break_enter(le32_to_cpu(rsp->CurrentLeaseState), + le32_to_cpu(rsp->Flags), + le16_to_cpu(rsp->Epoch), + le32_to_cpu(rsp->hdr.Id.SyncId.TreeId), + le64_to_cpu(rsp->hdr.SessionId), + *((u64 *)rsp->LeaseKey), + *((u64 *)&rsp->LeaseKey[8])); + + cifs_dbg(FYI, "Checking for lease break\n"); + + /* If server is a channel, select the primary channel */ + pserver = SERVER_IS_CHAN(server) ? server->primary_server : server; + + /* look up tcon based on tid & uid */ + spin_lock(&cifs_tcp_ses_lock); + list_for_each_entry(ses, &pserver->smb_ses_list, smb_ses_list) { + if (cifs_ses_exiting(ses)) + continue; + list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { + spin_lock(&tcon->open_file_lock); + cifs_stats_inc( + &tcon->stats.cifs_stats.num_oplock_brks); + if (smb2_tcon_has_lease(tcon, rsp)) { + spin_unlock(&tcon->open_file_lock); + spin_unlock(&cifs_tcp_ses_lock); + return true; + } + open = smb2_tcon_find_pending_open_lease(tcon, + rsp); + if (open) { + __u8 lease_key[SMB2_LEASE_KEY_SIZE]; + struct tcon_link *tlink; + + tlink = cifs_get_tlink(open->tlink); + memcpy(lease_key, open->lease_key, + SMB2_LEASE_KEY_SIZE); + spin_unlock(&tcon->open_file_lock); + spin_unlock(&cifs_tcp_ses_lock); + smb2_queue_pending_open_break(tlink, + lease_key, + rsp->NewLeaseState); + return true; + } + spin_unlock(&tcon->open_file_lock); + + if (cached_dir_lease_break(tcon, rsp->LeaseKey)) { + spin_unlock(&cifs_tcp_ses_lock); + return true; + } + } + } + spin_unlock(&cifs_tcp_ses_lock); + cifs_dbg(FYI, "Can not process lease break - no lease matched\n"); + trace_smb3_lease_not_found(le32_to_cpu(rsp->CurrentLeaseState), + le32_to_cpu(rsp->Flags), + le16_to_cpu(rsp->Epoch), + le32_to_cpu(rsp->hdr.Id.SyncId.TreeId), + le64_to_cpu(rsp->hdr.SessionId), + *((u64 *)rsp->LeaseKey), + *((u64 *)&rsp->LeaseKey[8])); + + return false; +} + +bool +smb2_is_valid_oplock_break(char *buffer, struct TCP_Server_Info *server) +{ + struct smb2_oplock_break *rsp = (struct smb2_oplock_break *)buffer; + struct TCP_Server_Info *pserver; + struct cifs_ses *ses; + struct cifs_tcon *tcon; + struct cifsInodeInfo *cinode; + struct cifsFileInfo *cfile; + + cifs_dbg(FYI, "Checking for oplock break\n"); + + if (rsp->hdr.Command != SMB2_OPLOCK_BREAK) + return false; + + if (rsp->StructureSize != + smb2_rsp_struct_sizes[SMB2_OPLOCK_BREAK_HE]) { + if (le16_to_cpu(rsp->StructureSize) == 44) + return smb2_is_valid_lease_break(buffer, server); + else + return false; + } + + cifs_dbg(FYI, "oplock level 0x%x\n", rsp->OplockLevel); + + /* If server is a channel, select the primary channel */ + pserver = SERVER_IS_CHAN(server) ? server->primary_server : server; + + /* look up tcon based on tid & uid */ + spin_lock(&cifs_tcp_ses_lock); + list_for_each_entry(ses, &pserver->smb_ses_list, smb_ses_list) { + if (cifs_ses_exiting(ses)) + continue; + list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { + + spin_lock(&tcon->open_file_lock); + list_for_each_entry(cfile, &tcon->openFileList, tlist) { + if (rsp->PersistentFid != + cfile->fid.persistent_fid || + rsp->VolatileFid != + cfile->fid.volatile_fid) + continue; + + cifs_dbg(FYI, "file id match, oplock break\n"); + cifs_stats_inc( + &tcon->stats.cifs_stats.num_oplock_brks); + cinode = CIFS_I(d_inode(cfile->dentry)); + spin_lock(&cfile->file_info_lock); + if (!CIFS_CACHE_WRITE(cinode) && + rsp->OplockLevel == SMB2_OPLOCK_LEVEL_NONE) + cfile->oplock_break_cancelled = true; + else + cfile->oplock_break_cancelled = false; + + set_bit(CIFS_INODE_PENDING_OPLOCK_BREAK, + &cinode->flags); + + cfile->oplock_epoch = 0; + cfile->oplock_level = rsp->OplockLevel; + + spin_unlock(&cfile->file_info_lock); + + cifs_queue_oplock_break(cfile); + + spin_unlock(&tcon->open_file_lock); + spin_unlock(&cifs_tcp_ses_lock); + return true; + } + spin_unlock(&tcon->open_file_lock); + } + } + spin_unlock(&cifs_tcp_ses_lock); + cifs_dbg(FYI, "No file id matched, oplock break ignored\n"); + trace_smb3_oplock_not_found(0 /* no xid */, rsp->PersistentFid, + le32_to_cpu(rsp->hdr.Id.SyncId.TreeId), + le64_to_cpu(rsp->hdr.SessionId)); + + return true; +} + +void +smb2_cancelled_close_fid(struct work_struct *work) +{ + struct close_cancelled_open *cancelled = container_of(work, + struct close_cancelled_open, work); + struct cifs_tcon *tcon = cancelled->tcon; + int rc; + + if (cancelled->mid) + cifs_tcon_dbg(VFS, "Close unmatched open for MID:%llu\n", + cancelled->mid); + else + cifs_tcon_dbg(VFS, "Close interrupted close\n"); + + rc = SMB2_close(0, tcon, cancelled->fid.persistent_fid, + cancelled->fid.volatile_fid); + if (rc) + cifs_tcon_dbg(VFS, "Close cancelled mid failed rc:%d\n", rc); + + cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_cancelled_close_fid); + kfree(cancelled); +} + +/* + * Caller should already has an extra reference to @tcon + * This function is used to queue work to close a handle to prevent leaks + * on the server. + * We handle two cases. If an open was interrupted after we sent the + * SMB2_CREATE to the server but before we processed the reply, and second + * if a close was interrupted before we sent the SMB2_CLOSE to the server. + */ +static int +__smb2_handle_cancelled_cmd(struct cifs_tcon *tcon, __u16 cmd, __u64 mid, + __u64 persistent_fid, __u64 volatile_fid) +{ + struct close_cancelled_open *cancelled; + + cancelled = kzalloc_obj(*cancelled); + if (!cancelled) + return -ENOMEM; + + cancelled->fid.persistent_fid = persistent_fid; + cancelled->fid.volatile_fid = volatile_fid; + cancelled->tcon = tcon; + cancelled->cmd = cmd; + cancelled->mid = mid; + INIT_WORK(&cancelled->work, smb2_cancelled_close_fid); + WARN_ON(queue_work(cifsiod_wq, &cancelled->work) == false); + + return 0; +} + +int +smb2_handle_cancelled_close(struct cifs_tcon *tcon, __u64 persistent_fid, + __u64 volatile_fid) +{ + int rc; + + cifs_dbg(FYI, "%s: tc_count=%d\n", __func__, tcon->tc_count); + spin_lock(&tcon->tc_lock); + if (tcon->tc_count <= 0) { + struct TCP_Server_Info *server = NULL; + + trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, + netfs_trace_tcon_ref_see_cancelled_close); + WARN_ONCE(tcon->tc_count < 0, "tcon refcount is negative"); + spin_unlock(&tcon->tc_lock); + + if (tcon->ses) { + server = tcon->ses->server; + cifs_server_dbg(FYI, + "tid=0x%x: tcon is closing, skipping async close retry of fid %llu %llu\n", + tcon->tid, persistent_fid, volatile_fid); + } + + return 0; + } + tcon->tc_count++; + trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, + netfs_trace_tcon_ref_get_cancelled_close); + spin_unlock(&tcon->tc_lock); + + rc = __smb2_handle_cancelled_cmd(tcon, SMB2_CLOSE_HE, 0, + persistent_fid, volatile_fid); + if (rc) + cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_cancelled_close); + + return rc; +} + +int +smb2_handle_cancelled_mid(struct mid_q_entry *mid, struct TCP_Server_Info *server) +{ + struct smb2_hdr *hdr = mid->resp_buf; + struct smb2_create_rsp *rsp = mid->resp_buf; + struct cifs_tcon *tcon; + int rc; + + if ((mid->optype & CIFS_CP_CREATE_CLOSE_OP) || hdr->Command != SMB2_CREATE || + hdr->Status != STATUS_SUCCESS) + return 0; + + tcon = smb2_find_smb_tcon(server, le64_to_cpu(hdr->SessionId), + le32_to_cpu(hdr->Id.SyncId.TreeId)); + if (!tcon) + return -ENOENT; + + rc = __smb2_handle_cancelled_cmd(tcon, + le16_to_cpu(hdr->Command), + le64_to_cpu(hdr->MessageId), + rsp->PersistentFileId, + rsp->VolatileFileId); + if (rc) + cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_cancelled_mid); + + return rc; +} + +/** + * smb311_update_preauth_hash - update @ses hash with the packet data in @iov + * + * Assumes @iov does not contain the rfc1002 length and iov[0] has the + * SMB2 header. + * + * @ses: server session structure + * @server: pointer to server info + * @iov: array containing the SMB request we will send to the server + * @nvec: number of array entries for the iov + */ +void +smb311_update_preauth_hash(struct cifs_ses *ses, struct TCP_Server_Info *server, + struct kvec *iov, int nvec) +{ + int i; + struct smb2_hdr *hdr; + struct sha512_ctx sha_ctx; + + hdr = (struct smb2_hdr *)iov[0].iov_base; + /* neg prot are always taken */ + if (hdr->Command == SMB2_NEGOTIATE) + goto ok; + + /* + * If we process a command which wasn't a negprot it means the + * neg prot was already done, so the server dialect was set + * and we can test it. Preauth requires 3.1.1 for now. + */ + if (server->dialect != SMB311_PROT_ID) + return; + + if (hdr->Command != SMB2_SESSION_SETUP) + return; + + /* skip last sess setup response */ + if ((hdr->Flags & SMB2_FLAGS_SERVER_TO_REDIR) + && (hdr->Status == NT_STATUS_OK + || (hdr->Status != + cpu_to_le32(NT_STATUS_MORE_PROCESSING_REQUIRED)))) + return; + +ok: + sha512_init(&sha_ctx); + sha512_update(&sha_ctx, ses->preauth_sha_hash, SMB2_PREAUTH_HASH_SIZE); + for (i = 0; i < nvec; i++) + sha512_update(&sha_ctx, iov[i].iov_base, iov[i].iov_len); + sha512_final(&sha_ctx, ses->preauth_sha_hash); +} diff --git a/fs/smb/client/smb2ops.c b/fs/smb/client/smb2ops.c new file mode 100644 index 000000000..192649fec --- /dev/null +++ b/fs/smb/client/smb2ops.c @@ -0,0 +1,6120 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * SMB2 version specific operations + * + * Copyright (c) 2012, Jeff Layton + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "cifsfs.h" +#include "cifsglob.h" +#include "cifsproto.h" +#include "smb2proto.h" +#include "smb2pdu.h" +#include "cifs_debug.h" +#include "cifs_unicode.h" +#include "../common/smb2status.h" +#include "smb2glob.h" +#include "cifs_ioctl.h" +#include "smbdirect.h" +#include "fscache.h" +#include "fs_context.h" +#include "cached_dir.h" +#include "reparse.h" + +/* Change credits for different ops and return the total number of credits */ +static int +change_conf(struct TCP_Server_Info *server) +{ + server->credits += server->echo_credits + server->oplock_credits; + if (server->credits > server->max_credits) + server->credits = server->max_credits; + server->oplock_credits = server->echo_credits = 0; + switch (server->credits) { + case 0: + return 0; + case 1: + server->echoes = false; + server->oplocks = false; + break; + case 2: + server->echoes = true; + server->oplocks = false; + server->echo_credits = 1; + break; + default: + server->echoes = true; + if (enable_oplocks) { + server->oplocks = true; + server->oplock_credits = 1; + } else + server->oplocks = false; + + server->echo_credits = 1; + } + server->credits -= server->echo_credits + server->oplock_credits; + return server->credits + server->echo_credits + server->oplock_credits; +} + +static void +smb2_add_credits(struct TCP_Server_Info *server, + struct cifs_credits *credits, const int optype) +{ + int *val, rc = -1; + int scredits, in_flight; + unsigned int add = credits->value; + unsigned int instance = credits->instance; + bool reconnect_detected = false; + bool reconnect_with_invalid_credits = false; + + spin_lock(&server->req_lock); + val = server->ops->get_credits_field(server, optype); + + /* eg found case where write overlapping reconnect messed up credits */ + if (((optype & CIFS_OP_MASK) == CIFS_NEG_OP) && (*val != 0)) + reconnect_with_invalid_credits = true; + + if ((instance == 0) || (instance == server->reconnect_instance)) + *val += add; + else + reconnect_detected = true; + + if (*val > 65000) { + *val = 65000; /* Don't get near 64K credits, avoid srv bugs */ + pr_warn_once("server overflowed SMB3 credits\n"); + trace_smb3_overflow_credits(server->current_mid, + server->conn_id, server->hostname, *val, + add, server->in_flight); + } + if (credits->in_flight_check > 1) { + pr_warn_once("rreq R=%08x[%x] Credits not in flight\n", + credits->rreq_debug_id, credits->rreq_debug_index); + } else { + credits->in_flight_check = 2; + } + if (WARN_ON_ONCE(server->in_flight == 0)) { + pr_warn_once("rreq R=%08x[%x] Zero in_flight\n", + credits->rreq_debug_id, credits->rreq_debug_index); + trace_smb3_rw_credits(credits->rreq_debug_id, + credits->rreq_debug_index, + credits->value, + server->credits, server->in_flight, 0, + cifs_trace_rw_credits_zero_in_flight); + } + server->in_flight--; + + /* + * Rebalance credits when an op drains in_flight. For session setup, + * do this only when the total accumulated credits are high enough (>2) + * so that a newly established secondary channel can reserve credits for + * echoes and oplocks. We expect this to happen at the end of the final + * session setup response. + */ + if (server->in_flight == 0 && + ((optype & CIFS_OP_MASK) != CIFS_NEG_OP) && + ((optype & CIFS_OP_MASK) != CIFS_SESS_OP)) + rc = change_conf(server); + else if (server->in_flight == 0 && + ((optype & CIFS_OP_MASK) == CIFS_SESS_OP) && *val > 2) + rc = change_conf(server); + /* + * Sometimes server returns 0 credits on oplock break ack - we need to + * rebalance credits in this case. + */ + else if (server->in_flight > 0 && server->oplock_credits == 0 && + server->oplocks) { + if (server->credits > 1) { + server->credits--; + server->oplock_credits++; + } + } else if ((server->in_flight > 0) && (server->oplock_credits > 3) && + ((optype & CIFS_OP_MASK) == CIFS_OBREAK_OP)) + /* if now have too many oplock credits, rebalance so don't starve normal ops */ + change_conf(server); + + scredits = *val; + in_flight = server->in_flight; + spin_unlock(&server->req_lock); + wake_up(&server->request_q); + + if (reconnect_detected) { + trace_smb3_reconnect_detected(server->current_mid, + server->conn_id, server->hostname, scredits, add, in_flight); + + cifs_dbg(FYI, "trying to put %d credits from the old server instance %d\n", + add, instance); + } + + if (reconnect_with_invalid_credits) { + trace_smb3_reconnect_with_invalid_credits(server->current_mid, + server->conn_id, server->hostname, scredits, add, in_flight); + cifs_dbg(FYI, "Negotiate operation when server credits is non-zero. Optype: %d, server credits: %d, credits added: %d\n", + optype, scredits, add); + } + + spin_lock(&server->srv_lock); + if (server->tcpStatus == CifsNeedReconnect + || server->tcpStatus == CifsExiting) { + spin_unlock(&server->srv_lock); + return; + } + spin_unlock(&server->srv_lock); + + switch (rc) { + case -1: + /* change_conf hasn't been executed */ + break; + case 0: + cifs_server_dbg(VFS, "Possible client or server bug - zero credits\n"); + break; + case 1: + cifs_server_dbg(VFS, "disabling echoes and oplocks\n"); + break; + case 2: + cifs_dbg(FYI, "disabling oplocks\n"); + break; + default: + /* change_conf rebalanced credits for different types */ + break; + } + + trace_smb3_add_credits(server->current_mid, + server->conn_id, server->hostname, scredits, add, in_flight); + cifs_dbg(FYI, "%s: added %u credits total=%d\n", __func__, add, scredits); +} + +static void +smb2_set_credits(struct TCP_Server_Info *server, const int val) +{ + int scredits, in_flight; + + spin_lock(&server->req_lock); + server->credits = val; + if (val == 1) { + server->reconnect_instance++; + /* + * ChannelSequence updated for all channels in primary channel so that consistent + * across SMB3 requests sent on any channel. See MS-SMB2 3.2.4.1 and 3.2.7.1 + */ + if (SERVER_IS_CHAN(server)) + server->primary_server->channel_sequence_num++; + else + server->channel_sequence_num++; + } + scredits = server->credits; + in_flight = server->in_flight; + spin_unlock(&server->req_lock); + + trace_smb3_set_credits(server->current_mid, + server->conn_id, server->hostname, scredits, val, in_flight); + cifs_dbg(FYI, "%s: set %u credits\n", __func__, val); + + /* don't log while holding the lock */ + if (val == 1) + cifs_dbg(FYI, "set credits to 1 due to smb2 reconnect\n"); +} + +static int * +smb2_get_credits_field(struct TCP_Server_Info *server, const int optype) +{ + switch (optype) { + case CIFS_ECHO_OP: + return &server->echo_credits; + case CIFS_OBREAK_OP: + return &server->oplock_credits; + default: + return &server->credits; + } +} + +static unsigned int +smb2_get_credits(struct mid_q_entry *mid) +{ + return mid->credits_received; +} + +static int +smb2_wait_mtu_credits(struct TCP_Server_Info *server, size_t size, + size_t *num, struct cifs_credits *credits) +{ + int rc = 0; + unsigned int scredits, in_flight; + + spin_lock(&server->req_lock); + while (1) { + spin_unlock(&server->req_lock); + + spin_lock(&server->srv_lock); + if (server->tcpStatus == CifsExiting) { + spin_unlock(&server->srv_lock); + return -ENOENT; + } + spin_unlock(&server->srv_lock); + + spin_lock(&server->req_lock); + if (server->credits <= 0) { + spin_unlock(&server->req_lock); + cifs_num_waiters_inc(server); + rc = wait_event_killable(server->request_q, + has_credits(server, &server->credits, 1)); + cifs_num_waiters_dec(server); + if (rc) + return rc; + spin_lock(&server->req_lock); + } else { + scredits = server->credits; + /* can deadlock with reopen */ + if (scredits <= 8) { + *num = SMB2_MAX_BUFFER_SIZE; + credits->value = 0; + credits->instance = 0; + break; + } + + /* leave some credits for reopen and other ops */ + scredits -= 8; + *num = min_t(unsigned int, size, + scredits * SMB2_MAX_BUFFER_SIZE); + + credits->value = + DIV_ROUND_UP(*num, SMB2_MAX_BUFFER_SIZE); + credits->instance = server->reconnect_instance; + server->credits -= credits->value; + server->in_flight++; + if (server->in_flight > server->max_in_flight) + server->max_in_flight = server->in_flight; + break; + } + } + scredits = server->credits; + in_flight = server->in_flight; + spin_unlock(&server->req_lock); + + trace_smb3_wait_credits(server->current_mid, + server->conn_id, server->hostname, scredits, -(credits->value), in_flight); + cifs_dbg(FYI, "%s: removed %u credits total=%d\n", + __func__, credits->value, scredits); + + return rc; +} + +static int +smb2_adjust_credits(struct TCP_Server_Info *server, + struct cifs_io_subrequest *subreq, + unsigned int /*enum smb3_rw_credits_trace*/ trace) +{ + struct cifs_credits *credits = &subreq->credits; + int new_val = DIV_ROUND_UP(subreq->subreq.len - subreq->subreq.transferred, + SMB2_MAX_BUFFER_SIZE); + int scredits, in_flight; + + if (!credits->value || credits->value == new_val) + return 0; + + if (credits->value < new_val) { + trace_smb3_rw_credits(subreq->rreq->debug_id, + subreq->subreq.debug_index, + credits->value, + server->credits, server->in_flight, + new_val - credits->value, + cifs_trace_rw_credits_no_adjust_up); + trace_smb3_too_many_credits(server->current_mid, + server->conn_id, server->hostname, 0, credits->value - new_val, 0); + cifs_server_dbg(VFS, "R=%x[%x] request has less credits (%d) than required (%d)", + subreq->rreq->debug_id, subreq->subreq.debug_index, + credits->value, new_val); + + return -EOPNOTSUPP; + } + + spin_lock(&server->req_lock); + + if (server->reconnect_instance != credits->instance) { + scredits = server->credits; + in_flight = server->in_flight; + spin_unlock(&server->req_lock); + + trace_smb3_rw_credits(subreq->rreq->debug_id, + subreq->subreq.debug_index, + credits->value, + server->credits, server->in_flight, + new_val - credits->value, + cifs_trace_rw_credits_old_session); + trace_smb3_reconnect_detected(server->current_mid, + server->conn_id, server->hostname, scredits, + credits->value - new_val, in_flight); + cifs_server_dbg(VFS, "R=%x[%x] trying to return %d credits to old session\n", + subreq->rreq->debug_id, subreq->subreq.debug_index, + credits->value - new_val); + return -EAGAIN; + } + + trace_smb3_rw_credits(subreq->rreq->debug_id, + subreq->subreq.debug_index, + credits->value, + server->credits, server->in_flight, + new_val - credits->value, trace); + server->credits += credits->value - new_val; + scredits = server->credits; + in_flight = server->in_flight; + spin_unlock(&server->req_lock); + wake_up(&server->request_q); + + trace_smb3_adj_credits(server->current_mid, + server->conn_id, server->hostname, scredits, + credits->value - new_val, in_flight); + cifs_dbg(FYI, "%s: adjust added %u credits total=%d\n", + __func__, credits->value - new_val, scredits); + + credits->value = new_val; + + return 0; +} + +static __u64 +smb2_get_next_mid(struct TCP_Server_Info *server) +{ + __u64 mid; + /* for SMB2 we need the current value */ + spin_lock(&server->mid_counter_lock); + mid = server->current_mid++; + spin_unlock(&server->mid_counter_lock); + return mid; +} + +static void +smb2_revert_current_mid(struct TCP_Server_Info *server, const unsigned int val) +{ + spin_lock(&server->mid_counter_lock); + if (server->current_mid >= val) + server->current_mid -= val; + spin_unlock(&server->mid_counter_lock); +} + +static struct mid_q_entry * +__smb2_find_mid(struct TCP_Server_Info *server, char *buf, bool dequeue) +{ + struct mid_q_entry *mid; + struct smb2_hdr *shdr = (struct smb2_hdr *)buf; + __u64 wire_mid = le64_to_cpu(shdr->MessageId); + + if (shdr->ProtocolId == SMB2_TRANSFORM_PROTO_NUM) { + cifs_server_dbg(VFS, "Encrypted frame parsing not supported yet\n"); + return NULL; + } + + spin_lock(&server->mid_queue_lock); + list_for_each_entry(mid, &server->pending_mid_q, qhead) { + if ((mid->mid == wire_mid) && + (mid->mid_state == MID_REQUEST_SUBMITTED) && + (mid->command == shdr->Command)) { + smb_get_mid(mid); + if (dequeue) { + list_del_init(&mid->qhead); + mid->deleted_from_q = true; + } + spin_unlock(&server->mid_queue_lock); + return mid; + } + } + spin_unlock(&server->mid_queue_lock); + return NULL; +} + +static struct mid_q_entry * +smb2_find_mid(struct TCP_Server_Info *server, char *buf) +{ + return __smb2_find_mid(server, buf, false); +} + +static struct mid_q_entry * +smb2_find_dequeue_mid(struct TCP_Server_Info *server, char *buf) +{ + return __smb2_find_mid(server, buf, true); +} + +static void +smb2_dump_detail(void *buf, size_t buf_len, struct TCP_Server_Info *server) +{ +#ifdef CONFIG_CIFS_DEBUG2 + struct smb2_hdr *shdr = (struct smb2_hdr *)buf; + + cifs_server_dbg(VFS, "Cmd: %d Err: 0x%x Flags: 0x%x Mid: %llu Pid: %d\n", + shdr->Command, shdr->Status, shdr->Flags, shdr->MessageId, + shdr->Id.SyncId.ProcessId); + if (!server->ops->check_message(buf, buf_len, server->total_read, server)) { + cifs_server_dbg(VFS, "smb buf %p len %u\n", buf, + server->ops->calc_smb_size(buf)); + } +#endif +} + +static bool +smb2_need_neg(struct TCP_Server_Info *server) +{ + return server->max_read == 0; +} + +static int +smb2_negotiate(const unsigned int xid, + struct cifs_ses *ses, + struct TCP_Server_Info *server) +{ + int rc; + + spin_lock(&server->mid_counter_lock); + server->current_mid = 0; + spin_unlock(&server->mid_counter_lock); + rc = SMB2_negotiate(xid, ses, server); + return rc; +} + +static inline unsigned int +prevent_zero_iosize(unsigned int size, const char *type) +{ + if (size == 0) { + cifs_dbg(VFS, "SMB: Zero %ssize calculated, using minimum value %u\n", + type, CIFS_MIN_DEFAULT_IOSIZE); + return CIFS_MIN_DEFAULT_IOSIZE; + } + return size; +} + +static unsigned int +smb2_negotiate_wsize(struct cifs_tcon *tcon, struct smb3_fs_context *ctx) +{ + struct TCP_Server_Info *server = tcon->ses->server; + unsigned int wsize; + + /* start with specified wsize, or default */ + wsize = ctx->got_wsize ? ctx->vol_wsize : CIFS_DEFAULT_IOSIZE; + wsize = min_t(unsigned int, wsize, server->max_write); + if (!(server->capabilities & SMB2_GLOBAL_CAP_LARGE_MTU)) + wsize = min_t(unsigned int, wsize, SMB2_MAX_BUFFER_SIZE); + + return prevent_zero_iosize(wsize, "w"); +} + +static unsigned int +smb3_negotiate_wsize(struct cifs_tcon *tcon, struct smb3_fs_context *ctx) +{ + struct TCP_Server_Info *server = tcon->ses->server; + unsigned int wsize; + + /* start with specified wsize, or default */ + wsize = ctx->got_wsize ? ctx->vol_wsize : SMB3_DEFAULT_IOSIZE; + wsize = min_t(unsigned int, wsize, server->max_write); +#ifdef CONFIG_CIFS_SMB_DIRECT + if (server->rdma) { + const struct smbdirect_socket_parameters *sp = + smbd_get_parameters(server->smbd_conn); + + if (server->sign) + /* + * Account for SMB2 data transfer packet header and + * possible encryption header + */ + wsize = min_t(unsigned int, + wsize, + sp->max_fragmented_send_size - + SMB2_READWRITE_PDU_HEADER_SIZE - + sizeof(struct smb2_transform_hdr)); + else + wsize = min_t(unsigned int, + wsize, sp->max_read_write_size); + } +#endif + if (!(server->capabilities & SMB2_GLOBAL_CAP_LARGE_MTU)) + wsize = min_t(unsigned int, wsize, SMB2_MAX_BUFFER_SIZE); + + return prevent_zero_iosize(wsize, "w"); +} + +static unsigned int +smb2_negotiate_rsize(struct cifs_tcon *tcon, struct smb3_fs_context *ctx) +{ + struct TCP_Server_Info *server = tcon->ses->server; + unsigned int rsize; + + /* start with specified rsize, or default */ + rsize = ctx->got_rsize ? ctx->vol_rsize : CIFS_DEFAULT_IOSIZE; + rsize = min_t(unsigned int, rsize, server->max_read); + + if (!(server->capabilities & SMB2_GLOBAL_CAP_LARGE_MTU)) + rsize = min_t(unsigned int, rsize, SMB2_MAX_BUFFER_SIZE); + + return prevent_zero_iosize(rsize, "r"); +} + +static unsigned int +smb3_negotiate_rsize(struct cifs_tcon *tcon, struct smb3_fs_context *ctx) +{ + struct TCP_Server_Info *server = tcon->ses->server; + unsigned int rsize; + + /* start with specified rsize, or default */ + rsize = ctx->got_rsize ? ctx->vol_rsize : SMB3_DEFAULT_IOSIZE; + rsize = min_t(unsigned int, rsize, server->max_read); +#ifdef CONFIG_CIFS_SMB_DIRECT + if (server->rdma) { + const struct smbdirect_socket_parameters *sp = + smbd_get_parameters(server->smbd_conn); + + if (server->sign) + /* + * Account for SMB2 data transfer packet header and + * possible encryption header + */ + rsize = min_t(unsigned int, + rsize, + sp->max_fragmented_recv_size - + SMB2_READWRITE_PDU_HEADER_SIZE - + sizeof(struct smb2_transform_hdr)); + else + rsize = min_t(unsigned int, + rsize, sp->max_read_write_size); + } +#endif + + if (!(server->capabilities & SMB2_GLOBAL_CAP_LARGE_MTU)) + rsize = min_t(unsigned int, rsize, SMB2_MAX_BUFFER_SIZE); + + return prevent_zero_iosize(rsize, "r"); +} + +/* + * compare two interfaces a and b + * return 0 if everything matches. + * return 1 if a is rdma capable, or rss capable, or has higher link speed + * return -1 otherwise. + */ +static int +iface_cmp(struct cifs_server_iface *a, struct cifs_server_iface *b) +{ + int cmp_ret = 0; + + WARN_ON(!a || !b); + if (a->rdma_capable == b->rdma_capable) { + if (a->rss_capable == b->rss_capable) { + if (a->speed == b->speed) { + cmp_ret = cifs_ipaddr_cmp((struct sockaddr *) &a->sockaddr, + (struct sockaddr *) &b->sockaddr); + if (!cmp_ret) + return 0; + else if (cmp_ret > 0) + return 1; + else + return -1; + } else if (a->speed > b->speed) + return 1; + else + return -1; + } else if (a->rss_capable > b->rss_capable) + return 1; + else + return -1; + } else if (a->rdma_capable > b->rdma_capable) + return 1; + else + return -1; +} + +static int +parse_server_interfaces(struct network_interface_info_ioctl_rsp *buf, + size_t buf_len, struct cifs_ses *ses, bool in_mount) +{ + struct network_interface_info_ioctl_rsp *p; + struct sockaddr_in *addr4; + struct sockaddr_in6 *addr6; + struct smb_sockaddr_in *p4; + struct smb_sockaddr_in6 *p6; + struct cifs_server_iface *info = NULL, *iface = NULL, *niface = NULL; + struct cifs_server_iface tmp_iface; + __be16 port; + ssize_t bytes_left; + size_t next = 0; + int nb_iface = 0; + int rc = 0, ret = 0; + + bytes_left = buf_len; + p = buf; + + spin_lock(&ses->iface_lock); + + /* + * Go through iface_list and mark them as inactive + */ + list_for_each_entry_safe(iface, niface, &ses->iface_list, + iface_head) + iface->is_active = 0; + + spin_unlock(&ses->iface_lock); + + /* + * Samba server e.g. can return an empty interface list in some cases, + * which would only be a problem if we were requesting multichannel + */ + if (bytes_left == 0) { + /* avoid spamming logs every 10 minutes, so log only in mount */ + if ((ses->chan_max > 1) && in_mount) + cifs_dbg(VFS, + "multichannel not available\n" + "Empty network interface list returned by server %s\n", + ses->server->hostname); + rc = -EOPNOTSUPP; + goto out; + } + + spin_lock(&ses->server->srv_lock); + if (ses->server->dstaddr.ss_family == AF_INET) + port = ((struct sockaddr_in *)&ses->server->dstaddr)->sin_port; + else if (ses->server->dstaddr.ss_family == AF_INET6) + port = ((struct sockaddr_in6 *)&ses->server->dstaddr)->sin6_port; + else + port = cpu_to_be16(CIFS_PORT); + spin_unlock(&ses->server->srv_lock); + + while (bytes_left >= (ssize_t)sizeof(*p)) { + memset(&tmp_iface, 0, sizeof(tmp_iface)); + /* default to 1Gbps when link speed is unset */ + tmp_iface.speed = le64_to_cpu(p->LinkSpeed) ?: 1000000000; + tmp_iface.rdma_capable = le32_to_cpu(p->Capability & RDMA_CAPABLE) ? 1 : 0; + tmp_iface.rss_capable = le32_to_cpu(p->Capability & RSS_CAPABLE) ? 1 : 0; + + switch (p->Family) { + /* + * The kernel and wire socket structures have the same + * layout and use network byte order but make the + * conversion explicit in case either one changes. + */ + case INTERNETWORK: + addr4 = (struct sockaddr_in *)&tmp_iface.sockaddr; + p4 = (struct smb_sockaddr_in *)p->Buffer; + addr4->sin_family = AF_INET; + memcpy(&addr4->sin_addr, &p4->IPv4Address, 4); + + /* [MS-SMB2] 2.2.32.5.1.1 Clients MUST ignore these */ + addr4->sin_port = port; + + cifs_dbg(FYI, "%s: ipv4 %pI4\n", __func__, + &addr4->sin_addr); + break; + case INTERNETWORKV6: + addr6 = (struct sockaddr_in6 *)&tmp_iface.sockaddr; + p6 = (struct smb_sockaddr_in6 *)p->Buffer; + addr6->sin6_family = AF_INET6; + memcpy(&addr6->sin6_addr, &p6->IPv6Address, 16); + + /* [MS-SMB2] 2.2.32.5.1.2 Clients MUST ignore these */ + addr6->sin6_flowinfo = 0; + addr6->sin6_scope_id = 0; + addr6->sin6_port = port; + + cifs_dbg(FYI, "%s: ipv6 %pI6\n", __func__, + &addr6->sin6_addr); + break; + default: + cifs_dbg(VFS, + "%s: skipping unsupported socket family\n", + __func__); + goto next_iface; + } + + /* + * The iface_list is assumed to be sorted by speed. + * Check if the new interface exists in that list. + * NEVER change iface. it could be in use. + * Add a new one instead + */ + spin_lock(&ses->iface_lock); + list_for_each_entry_safe(iface, niface, &ses->iface_list, + iface_head) { + ret = iface_cmp(iface, &tmp_iface); + if (!ret) { + iface->is_active = 1; + spin_unlock(&ses->iface_lock); + goto next_iface; + } else if (ret < 0) { + /* all remaining ifaces are slower */ + kref_get(&iface->refcount); + break; + } + } + spin_unlock(&ses->iface_lock); + + /* no match. insert the entry in the list */ + info = kmalloc_obj(struct cifs_server_iface); + if (!info) { + rc = -ENOMEM; + goto out; + } + memcpy(info, &tmp_iface, sizeof(tmp_iface)); + + /* add this new entry to the list */ + kref_init(&info->refcount); + info->is_active = 1; + + cifs_dbg(FYI, "%s: adding iface %zu\n", __func__, ses->iface_count); + cifs_dbg(FYI, "%s: speed %zu bps\n", __func__, info->speed); + cifs_dbg(FYI, "%s: capabilities 0x%08x\n", __func__, + le32_to_cpu(p->Capability)); + + spin_lock(&ses->iface_lock); + if (!list_entry_is_head(iface, &ses->iface_list, iface_head)) { + list_add_tail(&info->iface_head, &iface->iface_head); + kref_put(&iface->refcount, release_iface); + } else + list_add_tail(&info->iface_head, &ses->iface_list); + + ses->iface_count++; + spin_unlock(&ses->iface_lock); +next_iface: + nb_iface++; + next = le32_to_cpu(p->Next); + if (!next) { + bytes_left -= sizeof(*p); + break; + } + /* Validate that Next doesn't point beyond the buffer */ + if (next > bytes_left) { + cifs_dbg(VFS, "%s: invalid Next pointer %zu > %zd\n", + __func__, next, bytes_left); + rc = -EINVAL; + goto out; + } + p = (struct network_interface_info_ioctl_rsp *)((u8 *)p+next); + bytes_left -= next; + } + + if (!nb_iface) { + cifs_dbg(VFS, "%s: malformed interface info\n", __func__); + rc = -EINVAL; + goto out; + } + + /* Azure rounds the buffer size up 8, to a 16 byte boundary */ + if ((bytes_left > 8) || + (bytes_left >= offsetof(struct network_interface_info_ioctl_rsp, Next) + + sizeof(p->Next) && p->Next)) + cifs_dbg(VFS, "%s: incomplete interface info\n", __func__); + +out: + /* + * Go through the list again and put the inactive entries + */ + spin_lock(&ses->iface_lock); + list_for_each_entry_safe(iface, niface, &ses->iface_list, + iface_head) { + if (!iface->is_active) { + list_del(&iface->iface_head); + kref_put(&iface->refcount, release_iface); + ses->iface_count--; + } + } + spin_unlock(&ses->iface_lock); + + return rc; +} + +int +SMB3_request_interfaces(const unsigned int xid, struct cifs_tcon *tcon, bool in_mount) +{ + int rc; + unsigned int ret_data_len = 0; + struct network_interface_info_ioctl_rsp *out_buf = NULL; + struct cifs_ses *ses = tcon->ses; + struct TCP_Server_Info *pserver; + + /* do not query too frequently */ + spin_lock(&ses->iface_lock); + if (ses->iface_last_update && + time_before(jiffies, ses->iface_last_update + + (SMB_INTERFACE_POLL_INTERVAL * HZ))) { + spin_unlock(&ses->iface_lock); + return 0; + } + + ses->iface_last_update = jiffies; + + spin_unlock(&ses->iface_lock); + + rc = SMB2_ioctl(xid, tcon, NO_FILE_ID, NO_FILE_ID, + FSCTL_QUERY_NETWORK_INTERFACE_INFO, + NULL /* no data input */, 0 /* no data input */, + CIFSMaxBufSize, (char **)&out_buf, &ret_data_len); + if (rc == -EOPNOTSUPP) { + cifs_dbg(FYI, + "server does not support query network interfaces\n"); + ret_data_len = 0; + } else if (rc != 0) { + cifs_tcon_dbg(VFS, "error %d on ioctl to get interface list\n", rc); + goto out; + } + + rc = parse_server_interfaces(out_buf, ret_data_len, ses, in_mount); + if (rc) + goto out; + + /* check if iface is still active */ + spin_lock(&ses->chan_lock); + pserver = ses->chans[0].server; + if (pserver && !cifs_chan_is_iface_active(ses, pserver)) { + spin_unlock(&ses->chan_lock); + cifs_chan_update_iface(ses, pserver); + spin_lock(&ses->chan_lock); + } + spin_unlock(&ses->chan_lock); + +out: + kfree(out_buf); + return rc; +} + +static void +smb3_qfs_tcon(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb) +{ + int rc; + __le16 srch_path = 0; /* Null - open root of share */ + u8 oplock = SMB2_OPLOCK_LEVEL_NONE; + struct cifs_open_parms oparms; + struct cifs_fid fid; + struct cached_fid *cfid = NULL; + + oparms = (struct cifs_open_parms) { + .tcon = tcon, + .path = "", + .desired_access = FILE_READ_ATTRIBUTES, + .disposition = FILE_OPEN, + .create_options = cifs_create_options(cifs_sb, 0), + .fid = &fid, + }; + + rc = open_cached_dir(xid, tcon, "", cifs_sb, false, &cfid); + if (rc == 0) + memcpy(&fid, &cfid->fid, sizeof(struct cifs_fid)); + else + rc = SMB2_open(xid, &oparms, &srch_path, &oplock, NULL, NULL, + NULL, NULL); + if (rc) + return; + + SMB3_request_interfaces(xid, tcon, true /* called during mount */); + + SMB2_QFS_attr(xid, tcon, fid.persistent_fid, fid.volatile_fid, + FS_ATTRIBUTE_INFORMATION); + SMB2_QFS_attr(xid, tcon, fid.persistent_fid, fid.volatile_fid, + FS_DEVICE_INFORMATION); + SMB2_QFS_attr(xid, tcon, fid.persistent_fid, fid.volatile_fid, + FS_VOLUME_INFORMATION); + SMB2_QFS_attr(xid, tcon, fid.persistent_fid, fid.volatile_fid, + FS_SECTOR_SIZE_INFORMATION); /* SMB3 specific */ + if (cfid == NULL) + SMB2_close(xid, tcon, fid.persistent_fid, fid.volatile_fid); + else + close_cached_dir(cfid); +} + +static void +smb2_qfs_tcon(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb) +{ + int rc; + __le16 srch_path = 0; /* Null - open root of share */ + u8 oplock = SMB2_OPLOCK_LEVEL_NONE; + struct cifs_open_parms oparms; + struct cifs_fid fid; + + oparms = (struct cifs_open_parms) { + .tcon = tcon, + .path = "", + .desired_access = FILE_READ_ATTRIBUTES, + .disposition = FILE_OPEN, + .create_options = cifs_create_options(cifs_sb, 0), + .fid = &fid, + }; + + rc = SMB2_open(xid, &oparms, &srch_path, &oplock, NULL, NULL, + NULL, NULL); + if (rc) + return; + + SMB2_QFS_attr(xid, tcon, fid.persistent_fid, fid.volatile_fid, + FS_ATTRIBUTE_INFORMATION); + SMB2_QFS_attr(xid, tcon, fid.persistent_fid, fid.volatile_fid, + FS_DEVICE_INFORMATION); + SMB2_close(xid, tcon, fid.persistent_fid, fid.volatile_fid); +} + +static int +smb2_is_path_accessible(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, const char *full_path) +{ + __le16 *utf16_path; + __u8 oplock = SMB2_OPLOCK_LEVEL_NONE; + int err_buftype = CIFS_NO_BUFFER; + struct cifs_open_parms oparms; + struct kvec err_iov = {}; + struct cifs_fid fid; + struct cached_fid *cfid; + bool islink; + int rc, rc2; + + rc = open_cached_dir(xid, tcon, full_path, cifs_sb, true, &cfid); + if (!rc) { + close_cached_dir(cfid); + return 0; + } + + utf16_path = cifs_convert_path_to_utf16(full_path, cifs_sb); + if (!utf16_path) + return -ENOMEM; + + oparms = (struct cifs_open_parms) { + .tcon = tcon, + .path = full_path, + .desired_access = FILE_READ_ATTRIBUTES, + .disposition = FILE_OPEN, + .create_options = cifs_create_options(cifs_sb, 0), + .fid = &fid, + }; + + rc = SMB2_open(xid, &oparms, utf16_path, &oplock, NULL, NULL, + &err_iov, &err_buftype); + if (rc) { + struct smb2_hdr *hdr = err_iov.iov_base; + + if (unlikely(!hdr || err_buftype == CIFS_NO_BUFFER)) + goto out; + + if (rc != -EREMOTE && hdr->Status == STATUS_OBJECT_NAME_INVALID) { + rc2 = cifs_inval_name_dfs_link_error(xid, tcon, cifs_sb, + full_path, &islink); + if (rc2) { + rc = rc2; + goto out; + } + if (islink) + rc = -EREMOTE; + } + if (rc == -EREMOTE && IS_ENABLED(CONFIG_CIFS_DFS_UPCALL) && + (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NO_DFS)) + rc = -EOPNOTSUPP; + goto out; + } + + rc = SMB2_close(xid, tcon, fid.persistent_fid, fid.volatile_fid); + +out: + free_rsp_buf(err_buftype, err_iov.iov_base); + kfree(utf16_path); + return rc; +} + +static int smb2_get_srv_inum(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, const char *full_path, + u64 *uniqueid, struct cifs_open_info_data *data) +{ + *uniqueid = le64_to_cpu(data->fi.IndexNumber); + return 0; +} + +static int smb2_query_file_info(const unsigned int xid, struct cifs_tcon *tcon, + struct cifsFileInfo *cfile, struct cifs_open_info_data *data) +{ + struct cifs_fid *fid = &cfile->fid; + + if (cfile->symlink_target) { + data->symlink_target = kstrdup(cfile->symlink_target, GFP_KERNEL); + if (!data->symlink_target) + return -ENOMEM; + } + data->contains_posix_file_info = false; + return SMB2_query_info(xid, tcon, fid->persistent_fid, fid->volatile_fid, &data->fi); +} + +#ifdef CONFIG_CIFS_XATTR +static ssize_t +move_smb2_ea_to_cifs(char *dst, size_t dst_size, + struct smb2_file_full_ea_info *src, size_t src_size, + const unsigned char *ea_name) +{ + int rc = 0; + unsigned int ea_name_len = ea_name ? strlen(ea_name) : 0; + char *name, *value; + size_t buf_size = dst_size; + size_t name_len, value_len, user_name_len; + + while (src_size > 0) { + name_len = (size_t)src->ea_name_length; + value_len = (size_t)le16_to_cpu(src->ea_value_length); + + if (name_len == 0) + break; + + if (src_size < 8 + name_len + 1 + value_len) { + cifs_dbg(FYI, "EA entry goes beyond length of list\n"); + rc = smb_EIO2(smb_eio_trace_ea_overrun, + src_size, 8 + name_len + 1 + value_len); + goto out; + } + + name = &src->ea_data[0]; + value = &src->ea_data[src->ea_name_length + 1]; + + if (ea_name) { + if (ea_name_len == name_len && + memcmp(ea_name, name, name_len) == 0) { + rc = value_len; + if (dst_size == 0) + goto out; + if (dst_size < value_len) { + rc = -ERANGE; + goto out; + } + memcpy(dst, value, value_len); + goto out; + } + } else { + /* 'user.' plus a terminating null */ + user_name_len = 5 + 1 + name_len; + + if (buf_size == 0) { + /* skip copy - calc size only */ + rc += user_name_len; + } else if (dst_size >= user_name_len) { + dst_size -= user_name_len; + memcpy(dst, "user.", 5); + dst += 5; + memcpy(dst, src->ea_data, name_len); + dst += name_len; + *dst = 0; + ++dst; + rc += user_name_len; + } else { + /* stop before overrun buffer */ + rc = -ERANGE; + break; + } + } + + if (!src->next_entry_offset) + break; + + if (src_size < le32_to_cpu(src->next_entry_offset)) { + /* stop before overrun buffer */ + rc = -ERANGE; + break; + } + src_size -= le32_to_cpu(src->next_entry_offset); + src = (void *)((char *)src + + le32_to_cpu(src->next_entry_offset)); + } + + /* didn't find the named attribute */ + if (ea_name) + rc = -ENODATA; + +out: + return (ssize_t)rc; +} + +static ssize_t +smb2_query_eas(const unsigned int xid, struct cifs_tcon *tcon, + const unsigned char *path, const unsigned char *ea_name, + char *ea_data, size_t buf_size, + struct cifs_sb_info *cifs_sb) +{ + int rc; + struct kvec rsp_iov = {NULL, 0}; + int buftype = CIFS_NO_BUFFER; + struct smb2_query_info_rsp *rsp; + struct smb2_file_full_ea_info *info = NULL; + + rc = smb2_query_info_compound(xid, tcon, path, + FILE_READ_EA, + FILE_FULL_EA_INFORMATION, + SMB2_O_INFO_FILE, + CIFSMaxBufSize - + MAX_SMB2_CREATE_RESPONSE_SIZE - + MAX_SMB2_CLOSE_RESPONSE_SIZE, + &rsp_iov, &buftype, cifs_sb); + if (rc) { + /* + * If ea_name is NULL (listxattr) and there are no EAs, + * return 0 as it's not an error. Otherwise, the specified + * ea_name was not found. + */ + if (!ea_name && rc == -ENODATA) + rc = 0; + goto qeas_exit; + } + + rsp = (struct smb2_query_info_rsp *)rsp_iov.iov_base; + rc = smb2_validate_iov(le16_to_cpu(rsp->OutputBufferOffset), + le32_to_cpu(rsp->OutputBufferLength), + &rsp_iov, + sizeof(struct smb2_file_full_ea_info)); + if (rc) + goto qeas_exit; + + info = (struct smb2_file_full_ea_info *)( + le16_to_cpu(rsp->OutputBufferOffset) + (char *)rsp); + rc = move_smb2_ea_to_cifs(ea_data, buf_size, info, + le32_to_cpu(rsp->OutputBufferLength), ea_name); + + qeas_exit: + free_rsp_buf(buftype, rsp_iov.iov_base); + return rc; +} + +static int +smb2_set_ea(const unsigned int xid, struct cifs_tcon *tcon, + const char *path, const char *ea_name, const void *ea_value, + const __u16 ea_value_len, const struct nls_table *nls_codepage, + struct cifs_sb_info *cifs_sb) +{ + struct smb2_compound_vars *vars; + struct cifs_ses *ses = tcon->ses; + struct TCP_Server_Info *server; + struct smb_rqst *rqst; + struct kvec *rsp_iov; + __le16 *utf16_path = NULL; + int ea_name_len = strlen(ea_name); + int flags = CIFS_CP_CREATE_CLOSE_OP; + int len; + int resp_buftype[3]; + struct cifs_open_parms oparms; + __u8 oplock = SMB2_OPLOCK_LEVEL_NONE; + struct cifs_fid fid; + unsigned int size[1]; + void *data[1]; + struct smb2_file_full_ea_info *ea; + struct smb2_query_info_rsp *rsp; + int rc, used_len = 0; + int retries = 0, cur_sleep = 0; + +replay_again: + /* reinitialize for possible replay */ + used_len = 0; + flags = CIFS_CP_CREATE_CLOSE_OP; + oplock = SMB2_OPLOCK_LEVEL_NONE; + server = cifs_pick_channel(ses); + + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; + + if (ea_name_len > 255) + return -EINVAL; + + utf16_path = cifs_convert_path_to_utf16(path, cifs_sb); + if (!utf16_path) + return -ENOMEM; + + ea = NULL; + resp_buftype[0] = resp_buftype[1] = resp_buftype[2] = CIFS_NO_BUFFER; + vars = kzalloc_obj(*vars); + if (!vars) { + rc = -ENOMEM; + goto out_free_path; + } + rqst = vars->rqst; + rsp_iov = vars->rsp_iov; + + if (ses->server->ops->query_all_EAs) { + if (!ea_value) { + rc = ses->server->ops->query_all_EAs(xid, tcon, path, + ea_name, NULL, 0, + cifs_sb); + if (rc == -ENODATA) + goto sea_exit; + } else { + /* If we are adding a attribute we should first check + * if there will be enough space available to store + * the new EA. If not we should not add it since we + * would not be able to even read the EAs back. + */ + rc = smb2_query_info_compound(xid, tcon, path, + FILE_READ_EA, + FILE_FULL_EA_INFORMATION, + SMB2_O_INFO_FILE, + CIFSMaxBufSize - + MAX_SMB2_CREATE_RESPONSE_SIZE - + MAX_SMB2_CLOSE_RESPONSE_SIZE, + &rsp_iov[1], &resp_buftype[1], cifs_sb); + if (rc == 0) { + rsp = (struct smb2_query_info_rsp *)rsp_iov[1].iov_base; + used_len = le32_to_cpu(rsp->OutputBufferLength); + } + free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base); + resp_buftype[1] = CIFS_NO_BUFFER; + memset(&rsp_iov[1], 0, sizeof(rsp_iov[1])); + rc = 0; + + /* Use a fudge factor of 256 bytes in case we collide + * with a different set_EAs command. + */ + if (CIFSMaxBufSize - MAX_SMB2_CREATE_RESPONSE_SIZE - + MAX_SMB2_CLOSE_RESPONSE_SIZE - 256 < + used_len + ea_name_len + ea_value_len + 1) { + rc = -ENOSPC; + goto sea_exit; + } + } + } + + /* Open */ + rqst[0].rq_iov = vars->open_iov; + rqst[0].rq_nvec = SMB2_CREATE_IOV_SIZE; + + oparms = (struct cifs_open_parms) { + .tcon = tcon, + .path = path, + .desired_access = FILE_WRITE_EA, + .disposition = FILE_OPEN, + .create_options = cifs_create_options(cifs_sb, 0), + .fid = &fid, + .replay = !!(retries), + }; + + rc = SMB2_open_init(tcon, server, + &rqst[0], &oplock, &oparms, utf16_path); + if (rc) + goto sea_exit; + smb2_set_next_command(tcon, &rqst[0]); + + + /* Set Info */ + rqst[1].rq_iov = vars->si_iov; + rqst[1].rq_nvec = 1; + + len = sizeof(*ea) + ea_name_len + ea_value_len + 1; + ea = kzalloc(len, GFP_KERNEL); + if (ea == NULL) { + rc = -ENOMEM; + goto sea_exit; + } + + ea->ea_name_length = ea_name_len; + ea->ea_value_length = cpu_to_le16(ea_value_len); + memcpy(ea->ea_data, ea_name, ea_name_len + 1); + memcpy(ea->ea_data + ea_name_len + 1, ea_value, ea_value_len); + + size[0] = len; + data[0] = ea; + + rc = SMB2_set_info_init(tcon, server, + &rqst[1], COMPOUND_FID, + COMPOUND_FID, current->tgid, + FILE_FULL_EA_INFORMATION, + SMB2_O_INFO_FILE, 0, data, size); + if (rc) + goto sea_exit; + smb2_set_next_command(tcon, &rqst[1]); + smb2_set_related(&rqst[1]); + + /* Close */ + rqst[2].rq_iov = &vars->close_iov; + rqst[2].rq_nvec = 1; + rc = SMB2_close_init(tcon, server, + &rqst[2], COMPOUND_FID, COMPOUND_FID, false); + if (rc) + goto sea_exit; + smb2_set_related(&rqst[2]); + + if (retries) { + /* Back-off before retry */ + if (cur_sleep) + msleep(cur_sleep); + smb2_set_replay(server, &rqst[0]); + smb2_set_replay(server, &rqst[1]); + smb2_set_replay(server, &rqst[2]); + } + + rc = compound_send_recv(xid, ses, server, + flags, 3, rqst, + resp_buftype, rsp_iov); + /* no need to bump num_remote_opens because handle immediately closed */ + + sea_exit: + kfree(ea); + SMB2_open_free(&rqst[0]); + SMB2_set_info_free(&rqst[1]); + SMB2_close_free(&rqst[2]); + free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base); + free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base); + free_rsp_buf(resp_buftype[2], rsp_iov[2].iov_base); + kfree(vars); +out_free_path: + kfree(utf16_path); + + if (is_replayable_error(rc) && + smb2_should_replay(tcon, &retries, &cur_sleep)) + goto replay_again; + + return rc; +} +#endif + +static bool +smb2_can_echo(struct TCP_Server_Info *server) +{ + return server->echoes; +} + +static void +smb2_clear_stats(struct cifs_tcon *tcon) +{ + int i; + + for (i = 0; i < NUMBER_OF_SMB2_COMMANDS; i++) { + atomic_set(&tcon->stats.smb2_stats.smb2_com_sent[i], 0); + atomic_set(&tcon->stats.smb2_stats.smb2_com_failed[i], 0); + } +} + +static void +smb2_dump_share_caps(struct seq_file *m, struct cifs_tcon *tcon) +{ + seq_puts(m, "\n\tShare Capabilities:"); + if (tcon->capabilities & SMB2_SHARE_CAP_DFS) + seq_puts(m, " DFS,"); + if (tcon->capabilities & SMB2_SHARE_CAP_CONTINUOUS_AVAILABILITY) + seq_puts(m, " CONTINUOUS AVAILABILITY,"); + if (tcon->capabilities & SMB2_SHARE_CAP_SCALEOUT) + seq_puts(m, " SCALEOUT,"); + if (tcon->capabilities & SMB2_SHARE_CAP_CLUSTER) + seq_puts(m, " CLUSTER,"); + if (tcon->capabilities & SMB2_SHARE_CAP_ASYMMETRIC) + seq_puts(m, " ASYMMETRIC,"); + if (tcon->capabilities == 0) + seq_puts(m, " None"); + if (tcon->ss_flags & SSINFO_FLAGS_ALIGNED_DEVICE) + seq_puts(m, " Aligned,"); + if (tcon->ss_flags & SSINFO_FLAGS_PARTITION_ALIGNED_ON_DEVICE) + seq_puts(m, " Partition Aligned,"); + if (tcon->ss_flags & SSINFO_FLAGS_NO_SEEK_PENALTY) + seq_puts(m, " SSD,"); + if (tcon->ss_flags & SSINFO_FLAGS_TRIM_ENABLED) + seq_puts(m, " TRIM-support,"); + + seq_printf(m, "\tShare Flags: 0x%x", tcon->share_flags); + seq_printf(m, "\n\ttid: 0x%x", tcon->tid); + if (tcon->perf_sector_size) + seq_printf(m, "\tOptimal sector size: 0x%x", + tcon->perf_sector_size); + seq_printf(m, "\tMaximal Access: 0x%x", tcon->maximal_access); +} + +static void +smb2_print_stats(struct seq_file *m, struct cifs_tcon *tcon) +{ + atomic_t *sent = tcon->stats.smb2_stats.smb2_com_sent; + atomic_t *failed = tcon->stats.smb2_stats.smb2_com_failed; + + /* + * Can't display SMB2_NEGOTIATE, SESSION_SETUP, LOGOFF, CANCEL and ECHO + * totals (requests sent) since those SMBs are per-session not per tcon + */ + seq_printf(m, "\nBytes read: %llu Bytes written: %llu", + (long long)(tcon->bytes_read), + (long long)(tcon->bytes_written)); + seq_printf(m, "\nOpen files: %d total (local), %d open on server", + atomic_read(&tcon->num_local_opens), + atomic_read(&tcon->num_remote_opens)); + seq_printf(m, "\nTreeConnects: %d total %d failed", + atomic_read(&sent[SMB2_TREE_CONNECT_HE]), + atomic_read(&failed[SMB2_TREE_CONNECT_HE])); + seq_printf(m, "\nTreeDisconnects: %d total %d failed", + atomic_read(&sent[SMB2_TREE_DISCONNECT_HE]), + atomic_read(&failed[SMB2_TREE_DISCONNECT_HE])); + seq_printf(m, "\nCreates: %d total %d failed", + atomic_read(&sent[SMB2_CREATE_HE]), + atomic_read(&failed[SMB2_CREATE_HE])); + seq_printf(m, "\nCloses: %d total %d failed", + atomic_read(&sent[SMB2_CLOSE_HE]), + atomic_read(&failed[SMB2_CLOSE_HE])); + seq_printf(m, "\nFlushes: %d total %d failed", + atomic_read(&sent[SMB2_FLUSH_HE]), + atomic_read(&failed[SMB2_FLUSH_HE])); + seq_printf(m, "\nReads: %d total %d failed", + atomic_read(&sent[SMB2_READ_HE]), + atomic_read(&failed[SMB2_READ_HE])); + seq_printf(m, "\nWrites: %d total %d failed", + atomic_read(&sent[SMB2_WRITE_HE]), + atomic_read(&failed[SMB2_WRITE_HE])); + seq_printf(m, "\nLocks: %d total %d failed", + atomic_read(&sent[SMB2_LOCK_HE]), + atomic_read(&failed[SMB2_LOCK_HE])); + seq_printf(m, "\nIOCTLs: %d total %d failed", + atomic_read(&sent[SMB2_IOCTL_HE]), + atomic_read(&failed[SMB2_IOCTL_HE])); + seq_printf(m, "\nQueryDirectories: %d total %d failed", + atomic_read(&sent[SMB2_QUERY_DIRECTORY_HE]), + atomic_read(&failed[SMB2_QUERY_DIRECTORY_HE])); + seq_printf(m, "\nChangeNotifies: %d total %d failed", + atomic_read(&sent[SMB2_CHANGE_NOTIFY_HE]), + atomic_read(&failed[SMB2_CHANGE_NOTIFY_HE])); + seq_printf(m, "\nQueryInfos: %d total %d failed", + atomic_read(&sent[SMB2_QUERY_INFO_HE]), + atomic_read(&failed[SMB2_QUERY_INFO_HE])); + seq_printf(m, "\nSetInfos: %d total %d failed", + atomic_read(&sent[SMB2_SET_INFO_HE]), + atomic_read(&failed[SMB2_SET_INFO_HE])); + seq_printf(m, "\nOplockBreaks: %d sent %d failed", + atomic_read(&sent[SMB2_OPLOCK_BREAK_HE]), + atomic_read(&failed[SMB2_OPLOCK_BREAK_HE])); +} + +static void +smb2_set_fid(struct cifsFileInfo *cfile, struct cifs_fid *fid, __u32 oplock) +{ + struct cifsInodeInfo *cinode = CIFS_I(d_inode(cfile->dentry)); + struct TCP_Server_Info *server = tlink_tcon(cfile->tlink)->ses->server; + + lockdep_assert_held(&cinode->open_file_lock); + + cfile->fid.persistent_fid = fid->persistent_fid; + cfile->fid.volatile_fid = fid->volatile_fid; + cfile->fid.access = fid->access; +#ifdef CONFIG_CIFS_DEBUG2 + cfile->fid.mid = fid->mid; +#endif /* CIFS_DEBUG2 */ + server->ops->set_oplock_level(cinode, oplock, fid->epoch, + &fid->purge_cache); + cinode->can_cache_brlcks = CIFS_CACHE_WRITE(cinode); + memcpy(cfile->fid.create_guid, fid->create_guid, 16); +} + +static int +smb2_close_file(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_fid *fid) +{ + return SMB2_close(xid, tcon, fid->persistent_fid, fid->volatile_fid); +} + +static int +smb2_close_getattr(const unsigned int xid, struct cifs_tcon *tcon, + struct cifsFileInfo *cfile) +{ + struct smb2_file_network_open_info file_inf; + struct inode *inode; + u64 asize; + int rc; + + rc = __SMB2_close(xid, tcon, cfile->fid.persistent_fid, + cfile->fid.volatile_fid, &file_inf); + if (rc) + return rc; + + inode = d_inode(cfile->dentry); + + spin_lock(&inode->i_lock); + CIFS_I(inode)->time = jiffies; + + /* Creation time should not need to be updated on close */ + if (file_inf.LastWriteTime) + inode_set_mtime_to_ts(inode, + cifs_NTtimeToUnix(file_inf.LastWriteTime)); + if (file_inf.ChangeTime) + inode_set_ctime_to_ts(inode, + cifs_NTtimeToUnix(file_inf.ChangeTime)); + if (file_inf.LastAccessTime) + inode_set_atime_to_ts(inode, + cifs_NTtimeToUnix(file_inf.LastAccessTime)); + + asize = le64_to_cpu(file_inf.AllocationSize); + if (asize > 4096) + inode->i_blocks = CIFS_INO_BLOCKS(asize); + + /* End of file and Attributes should not have to be updated on close */ + spin_unlock(&inode->i_lock); + return rc; +} + +static int +SMB2_request_res_key(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, + struct copychunk_ioctl_req *pcchunk) +{ + int rc; + unsigned int ret_data_len; + struct resume_key_ioctl_rsp *res_key; + + rc = SMB2_ioctl(xid, tcon, persistent_fid, volatile_fid, + FSCTL_SRV_REQUEST_RESUME_KEY, NULL, 0 /* no input */, + CIFSMaxBufSize, (char **)&res_key, &ret_data_len); + + if (rc == -EOPNOTSUPP) { + pr_warn_once("Server share %s does not support copy range\n", tcon->tree_name); + goto req_res_key_exit; + } else if (rc) { + cifs_tcon_dbg(VFS, "refcpy ioctl error %d getting resume key\n", rc); + goto req_res_key_exit; + } + if (ret_data_len < sizeof(struct resume_key_ioctl_rsp)) { + cifs_tcon_dbg(VFS, "Invalid refcopy resume key length\n"); + rc = -EINVAL; + goto req_res_key_exit; + } + memcpy(pcchunk->SourceKey, res_key->ResumeKey, COPY_CHUNK_RES_KEY_SIZE); + +req_res_key_exit: + kfree(res_key); + return rc; +} + +static int +smb2_ioctl_query_info(const unsigned int xid, + struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, + __le16 *path, int is_dir, + unsigned long p) +{ + struct smb2_compound_vars *vars; + struct smb_rqst *rqst; + struct kvec *rsp_iov; + struct cifs_ses *ses = tcon->ses; + struct TCP_Server_Info *server; + char __user *arg = (char __user *)p; + struct smb_query_info qi; + struct smb_query_info __user *pqi; + int rc = 0; + int flags = CIFS_CP_CREATE_CLOSE_OP; + struct smb2_query_info_rsp *qi_rsp = NULL; + struct smb2_ioctl_rsp *io_rsp = NULL; + void *buffer = NULL; + int resp_buftype[3]; + struct cifs_open_parms oparms; + u8 oplock = SMB2_OPLOCK_LEVEL_NONE; + struct cifs_fid fid; + unsigned int size[2]; + void *data[2]; + int create_options = is_dir ? CREATE_NOT_FILE : CREATE_NOT_DIR; + void (*free_req1_func)(struct smb_rqst *r); + int retries = 0, cur_sleep = 0; + +replay_again: + /* reinitialize for possible replay */ + buffer = NULL; + flags = CIFS_CP_CREATE_CLOSE_OP; + oplock = SMB2_OPLOCK_LEVEL_NONE; + server = cifs_pick_channel(ses); + + vars = kzalloc_obj(*vars, GFP_ATOMIC); + if (vars == NULL) + return -ENOMEM; + rqst = &vars->rqst[0]; + rsp_iov = &vars->rsp_iov[0]; + + resp_buftype[0] = resp_buftype[1] = resp_buftype[2] = CIFS_NO_BUFFER; + + if (copy_from_user(&qi, arg, sizeof(struct smb_query_info))) { + rc = -EFAULT; + goto free_vars; + } + if (qi.output_buffer_length > 1024) { + rc = -EINVAL; + goto free_vars; + } + + if (!ses || !server) { + rc = smb_EIO(smb_eio_trace_null_pointers); + goto free_vars; + } + + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; + + if (qi.output_buffer_length) { + buffer = memdup_user(arg + sizeof(struct smb_query_info), qi.output_buffer_length); + if (IS_ERR(buffer)) { + rc = PTR_ERR(buffer); + goto free_vars; + } + } + + /* Open */ + rqst[0].rq_iov = &vars->open_iov[0]; + rqst[0].rq_nvec = SMB2_CREATE_IOV_SIZE; + + oparms = (struct cifs_open_parms) { + .tcon = tcon, + .disposition = FILE_OPEN, + .create_options = cifs_create_options(cifs_sb, create_options), + .fid = &fid, + .replay = !!(retries), + }; + + if (qi.flags & PASSTHRU_FSCTL) { + switch (qi.info_type & FSCTL_DEVICE_ACCESS_MASK) { + case FSCTL_DEVICE_ACCESS_FILE_READ_WRITE_ACCESS: + oparms.desired_access = FILE_READ_DATA | FILE_WRITE_DATA | FILE_READ_ATTRIBUTES | SYNCHRONIZE; + break; + case FSCTL_DEVICE_ACCESS_FILE_ANY_ACCESS: + oparms.desired_access = GENERIC_ALL; + break; + case FSCTL_DEVICE_ACCESS_FILE_READ_ACCESS: + oparms.desired_access = GENERIC_READ; + break; + case FSCTL_DEVICE_ACCESS_FILE_WRITE_ACCESS: + oparms.desired_access = GENERIC_WRITE; + break; + } + } else if (qi.flags & PASSTHRU_SET_INFO) { + oparms.desired_access = GENERIC_WRITE; + } else { + oparms.desired_access = FILE_READ_ATTRIBUTES | READ_CONTROL; + } + + rc = SMB2_open_init(tcon, server, + &rqst[0], &oplock, &oparms, path); + if (rc) + goto free_output_buffer; + smb2_set_next_command(tcon, &rqst[0]); + + /* Query */ + if (qi.flags & PASSTHRU_FSCTL) { + /* Can eventually relax perm check since server enforces too */ + if (!capable(CAP_SYS_ADMIN)) { + rc = -EPERM; + goto free_open_req; + } + rqst[1].rq_iov = &vars->io_iov[0]; + rqst[1].rq_nvec = SMB2_IOCTL_IOV_SIZE; + + rc = SMB2_ioctl_init(tcon, server, &rqst[1], COMPOUND_FID, COMPOUND_FID, + qi.info_type, buffer, qi.output_buffer_length, + CIFSMaxBufSize - MAX_SMB2_CREATE_RESPONSE_SIZE - + MAX_SMB2_CLOSE_RESPONSE_SIZE); + free_req1_func = SMB2_ioctl_free; + } else if (qi.flags == PASSTHRU_SET_INFO) { + /* Can eventually relax perm check since server enforces too */ + if (!capable(CAP_SYS_ADMIN)) { + rc = -EPERM; + goto free_open_req; + } + if (qi.output_buffer_length < 8) { + rc = -EINVAL; + goto free_open_req; + } + rqst[1].rq_iov = vars->si_iov; + rqst[1].rq_nvec = 1; + + /* MS-FSCC 2.4.13 FileEndOfFileInformation */ + size[0] = 8; + data[0] = buffer; + + rc = SMB2_set_info_init(tcon, server, &rqst[1], COMPOUND_FID, COMPOUND_FID, + current->tgid, FILE_END_OF_FILE_INFORMATION, + SMB2_O_INFO_FILE, 0, data, size); + free_req1_func = SMB2_set_info_free; + } else if (qi.flags == PASSTHRU_QUERY_INFO) { + rqst[1].rq_iov = &vars->qi_iov; + rqst[1].rq_nvec = 1; + + rc = SMB2_query_info_init(tcon, server, + &rqst[1], COMPOUND_FID, + COMPOUND_FID, qi.file_info_class, + qi.info_type, qi.additional_information, + qi.input_buffer_length, + qi.output_buffer_length, buffer); + free_req1_func = SMB2_query_info_free; + } else { /* unknown flags */ + cifs_tcon_dbg(VFS, "Invalid passthru query flags: 0x%x\n", + qi.flags); + rc = -EINVAL; + } + + if (rc) + goto free_open_req; + smb2_set_next_command(tcon, &rqst[1]); + smb2_set_related(&rqst[1]); + + /* Close */ + rqst[2].rq_iov = &vars->close_iov; + rqst[2].rq_nvec = 1; + + rc = SMB2_close_init(tcon, server, + &rqst[2], COMPOUND_FID, COMPOUND_FID, false); + if (rc) + goto free_req_1; + smb2_set_related(&rqst[2]); + + if (retries) { + /* Back-off before retry */ + if (cur_sleep) + msleep(cur_sleep); + smb2_set_replay(server, &rqst[0]); + smb2_set_replay(server, &rqst[1]); + smb2_set_replay(server, &rqst[2]); + } + + rc = compound_send_recv(xid, ses, server, + flags, 3, rqst, + resp_buftype, rsp_iov); + if (rc) + goto out; + + /* No need to bump num_remote_opens since handle immediately closed */ + if (qi.flags & PASSTHRU_FSCTL) { + pqi = (struct smb_query_info __user *)arg; + io_rsp = (struct smb2_ioctl_rsp *)rsp_iov[1].iov_base; + if (le32_to_cpu(io_rsp->OutputCount) < qi.input_buffer_length) + qi.input_buffer_length = le32_to_cpu(io_rsp->OutputCount); + if (qi.input_buffer_length > 0 && + size_add(le32_to_cpu(io_rsp->OutputOffset), + qi.input_buffer_length) > rsp_iov[1].iov_len) { + rc = -EFAULT; + goto out; + } + + if (copy_to_user(&pqi->input_buffer_length, + &qi.input_buffer_length, + sizeof(qi.input_buffer_length))) { + rc = -EFAULT; + goto out; + } + + if (copy_to_user((void __user *)pqi + sizeof(struct smb_query_info), + (const void *)io_rsp + le32_to_cpu(io_rsp->OutputOffset), + qi.input_buffer_length)) + rc = -EFAULT; + } else { + pqi = (struct smb_query_info __user *)arg; + qi_rsp = (struct smb2_query_info_rsp *)rsp_iov[1].iov_base; + if (le32_to_cpu(qi_rsp->OutputBufferLength) < qi.input_buffer_length) + qi.input_buffer_length = le32_to_cpu(qi_rsp->OutputBufferLength); + if (qi.input_buffer_length > 0 && + struct_size(qi_rsp, Buffer, qi.input_buffer_length) > + rsp_iov[1].iov_len) { + rc = -EFAULT; + goto out; + } + if (copy_to_user(&pqi->input_buffer_length, + &qi.input_buffer_length, + sizeof(qi.input_buffer_length))) { + rc = -EFAULT; + goto out; + } + + if (copy_to_user(pqi + 1, qi_rsp->Buffer, + qi.input_buffer_length)) + rc = -EFAULT; + } + +out: + free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base); + free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base); + free_rsp_buf(resp_buftype[2], rsp_iov[2].iov_base); + SMB2_close_free(&rqst[2]); +free_req_1: + free_req1_func(&rqst[1]); +free_open_req: + SMB2_open_free(&rqst[0]); +free_output_buffer: + kfree(buffer); +free_vars: + kfree(vars); + + if (is_replayable_error(rc) && + smb2_should_replay(tcon, &retries, &cur_sleep)) + goto replay_again; + + return rc; +} + +/** + * calc_chunk_count - calculates the number chunks to be filled in the Chunks[] + * array of struct copychunk_ioctl + * + * @tcon: destination file tcon + * @bytes_left: how many bytes are left to copy + * + * Return: maximum number of chunks with which Chunks[] can be filled. + */ +static inline u32 +calc_chunk_count(struct cifs_tcon *tcon, u64 bytes_left) +{ + u32 max_chunks = READ_ONCE(tcon->max_chunks); + u32 max_bytes_copy = READ_ONCE(tcon->max_bytes_copy); + u32 max_bytes_chunk = READ_ONCE(tcon->max_bytes_chunk); + u64 need; + u32 allowed; + + if (!max_bytes_chunk || !max_bytes_copy || !max_chunks) + return 0; + + /* chunks needed for the remaining bytes */ + need = DIV_ROUND_UP_ULL(bytes_left, max_bytes_chunk); + /* chunks allowed per cc request */ + allowed = DIV_ROUND_UP(max_bytes_copy, max_bytes_chunk); + + return (u32)umin(need, umin(max_chunks, allowed)); +} + +/** + * smb2_copychunk_range - server-side copy of data range + * + * @xid: transaction id + * @src_file: source file + * @dst_file: destination file + * @src_off: source file byte offset + * @len: number of bytes to copy + * @dst_off: destination file byte offset + * + * Obtains a resume key for @src_file and issues FSCTL_SRV_COPYCHUNK_WRITE + * IOCTLs, splitting the request into chunks limited by tcon->max_*. + * + * Return: @len on success; negative errno on failure. + */ +static ssize_t +smb2_copychunk_range(const unsigned int xid, + struct cifsFileInfo *src_file, + struct cifsFileInfo *dst_file, + u64 src_off, + u64 len, + u64 dst_off) +{ + int rc = 0; + unsigned int ret_data_len = 0; + struct copychunk_ioctl_req *cc_req = NULL; + struct copychunk_ioctl_rsp *cc_rsp = NULL; + struct cifs_tcon *tcon; + struct srv_copychunk *chunk; + u32 chunks, chunk_count, chunk_bytes; + u32 copy_bytes, copy_bytes_left; + u32 chunks_written, bytes_written; + u64 total_bytes_left = len; + u64 src_off_prev, dst_off_prev; + u32 retries = 0; + + tcon = tlink_tcon(dst_file->tlink); + + trace_smb3_copychunk_enter(xid, src_file->fid.volatile_fid, + dst_file->fid.volatile_fid, tcon->tid, + tcon->ses->Suid, src_off, dst_off, len); + +retry: + chunk_count = calc_chunk_count(tcon, total_bytes_left); + if (!chunk_count) { + rc = -EOPNOTSUPP; + goto out; + } + + cc_req = kzalloc_flex(*cc_req, Chunks, chunk_count); + if (!cc_req) { + rc = -ENOMEM; + goto out; + } + + /* Request a key from the server to identify the source of the copy */ + rc = SMB2_request_res_key(xid, + tlink_tcon(src_file->tlink), + src_file->fid.persistent_fid, + src_file->fid.volatile_fid, + cc_req); + + /* Note: request_res_key sets res_key null only if rc != 0 */ + if (rc) + goto out; + + while (total_bytes_left > 0) { + + /* Store previous offsets to allow rewind */ + src_off_prev = src_off; + dst_off_prev = dst_off; + + /* + * __counted_by_le(ChunkCount): set to allocated chunks before + * populating Chunks[] + */ + cc_req->ChunkCount = cpu_to_le32(chunk_count); + + chunks = 0; + copy_bytes = 0; + copy_bytes_left = umin(total_bytes_left, tcon->max_bytes_copy); + while (copy_bytes_left > 0 && chunks < chunk_count) { + chunk = &cc_req->Chunks[chunks++]; + + chunk->SourceOffset = cpu_to_le64(src_off); + chunk->TargetOffset = cpu_to_le64(dst_off); + + chunk_bytes = umin(copy_bytes_left, tcon->max_bytes_chunk); + + chunk->Length = cpu_to_le32(chunk_bytes); + /* Buffer is zeroed, no need to set chunk->Reserved = 0 */ + + src_off += chunk_bytes; + dst_off += chunk_bytes; + + copy_bytes_left -= chunk_bytes; + copy_bytes += chunk_bytes; + } + + cc_req->ChunkCount = cpu_to_le32(chunks); + /* Buffer is zeroed, no need to set cc_req->Reserved = 0 */ + + /* Request server copy to target from src identified by key */ + kfree(cc_rsp); + cc_rsp = NULL; + rc = SMB2_ioctl(xid, tcon, dst_file->fid.persistent_fid, + dst_file->fid.volatile_fid, FSCTL_SRV_COPYCHUNK_WRITE, + (char *)cc_req, struct_size(cc_req, Chunks, chunks), + CIFSMaxBufSize, (char **)&cc_rsp, &ret_data_len); + + if (rc && rc != -EINVAL) + goto out; + + if (unlikely(ret_data_len != sizeof(*cc_rsp))) { + cifs_tcon_dbg(VFS, "Copychunk invalid response: size %u/%zu\n", + ret_data_len, sizeof(*cc_rsp)); + rc = smb_EIO1(smb_eio_trace_copychunk_inv_rsp, ret_data_len); + goto out; + } + + bytes_written = le32_to_cpu(cc_rsp->TotalBytesWritten); + chunks_written = le32_to_cpu(cc_rsp->ChunksWritten); + chunk_bytes = le32_to_cpu(cc_rsp->ChunkBytesWritten); + + if (rc == 0) { + /* Check if server claimed to write more than we asked */ + if (unlikely(!bytes_written || bytes_written > copy_bytes)) { + cifs_tcon_dbg(VFS, "Copychunk invalid response: bytes written %u/%u\n", + bytes_written, copy_bytes); + rc = smb_EIO2(smb_eio_trace_copychunk_overcopy_b, + bytes_written, copy_bytes); + goto out; + } + if (unlikely(!chunks_written || chunks_written > chunks)) { + cifs_tcon_dbg(VFS, "Copychunk invalid response: chunks written %u/%u\n", + chunks_written, chunks); + rc = smb_EIO2(smb_eio_trace_copychunk_overcopy_c, + chunks_written, chunks); + goto out; + } + + /* Partial write: rewind */ + if (bytes_written < copy_bytes) { + u32 delta = copy_bytes - bytes_written; + + src_off -= delta; + dst_off -= delta; + } + + total_bytes_left -= bytes_written; + continue; + } + + /* + * Check if server is not asking us to reduce size. + * + * Note: As per MS-SMB2 2.2.32.1, the values returned + * in cc_rsp are not strictly lower than what existed + * before. + */ + if (bytes_written < tcon->max_bytes_copy) { + cifs_tcon_dbg(FYI, "Copychunk MaxBytesCopy updated: %u -> %u\n", + tcon->max_bytes_copy, bytes_written); + tcon->max_bytes_copy = bytes_written; + } + + if (chunks_written < tcon->max_chunks) { + cifs_tcon_dbg(FYI, "Copychunk MaxChunks updated: %u -> %u\n", + tcon->max_chunks, chunks_written); + tcon->max_chunks = chunks_written; + } + + if (chunk_bytes < tcon->max_bytes_chunk) { + cifs_tcon_dbg(FYI, "Copychunk MaxBytesChunk updated: %u -> %u\n", + tcon->max_bytes_chunk, chunk_bytes); + tcon->max_bytes_chunk = chunk_bytes; + } + + /* reset to last offsets */ + if (retries++ < 2) { + src_off = src_off_prev; + dst_off = dst_off_prev; + kfree(cc_req); + cc_req = NULL; + goto retry; + } + + break; + } + +out: + kfree(cc_req); + kfree(cc_rsp); + if (rc) { + trace_smb3_copychunk_err(xid, src_file->fid.volatile_fid, + dst_file->fid.volatile_fid, tcon->tid, + tcon->ses->Suid, src_off, dst_off, len, rc); + return rc; + } else { + trace_smb3_copychunk_done(xid, src_file->fid.volatile_fid, + dst_file->fid.volatile_fid, tcon->tid, + tcon->ses->Suid, src_off, dst_off, len); + return len; + } +} + +static int +smb2_flush_file(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_fid *fid) +{ + return SMB2_flush(xid, tcon, fid->persistent_fid, fid->volatile_fid); +} + +static unsigned int +smb2_read_data_offset(char *buf) +{ + struct smb2_read_rsp *rsp = (struct smb2_read_rsp *)buf; + + return rsp->DataOffset; +} + +static unsigned int +smb2_read_data_length(char *buf, bool in_remaining) +{ + struct smb2_read_rsp *rsp = (struct smb2_read_rsp *)buf; + + if (in_remaining) + return le32_to_cpu(rsp->DataRemaining); + + return le32_to_cpu(rsp->DataLength); +} + + +static int +smb2_sync_read(const unsigned int xid, struct cifs_fid *pfid, + struct cifs_io_parms *parms, unsigned int *bytes_read, + char **buf, int *buf_type) +{ + parms->persistent_fid = pfid->persistent_fid; + parms->volatile_fid = pfid->volatile_fid; + return SMB2_read(xid, parms, bytes_read, buf, buf_type); +} + +static int +smb2_sync_write(const unsigned int xid, struct cifs_fid *pfid, + struct cifs_io_parms *parms, unsigned int *written, + struct kvec *iov, unsigned long nr_segs) +{ + + parms->persistent_fid = pfid->persistent_fid; + parms->volatile_fid = pfid->volatile_fid; + return SMB2_write(xid, parms, written, iov, nr_segs); +} + +/* Set or clear the SPARSE_FILE attribute based on value passed in setsparse */ +static int smb2_set_sparse(const unsigned int xid, struct cifs_tcon *tcon, + struct cifsFileInfo *cfile, struct inode *inode, + __u8 setsparse) +{ + struct cifsInodeInfo *cifsi; + int rc; + + cifsi = CIFS_I(inode); + + /* if file already sparse don't bother setting sparse again */ + if ((cifsi->cifsAttrs & FILE_ATTRIBUTE_SPARSE_FILE) && setsparse) + return 0; /* already sparse */ + + if (!(cifsi->cifsAttrs & FILE_ATTRIBUTE_SPARSE_FILE) && !setsparse) + return 0; /* already not sparse */ + + /* + * Can't check for sparse support on share the usual way via the + * FS attribute info (FILE_SUPPORTS_SPARSE_FILES) on the share + * since Samba server doesn't set the flag on the share, yet + * supports the set sparse FSCTL and returns sparse correctly + * in the file attributes. If the server returns EOPNOTSUPP, mark + * that sparse files are not supported on this share to avoid + * repeatedly sending the unsupported FSCTL. + */ + if (tcon->broken_sparse_sup) + return -EOPNOTSUPP; + + rc = SMB2_ioctl(xid, tcon, cfile->fid.persistent_fid, + cfile->fid.volatile_fid, FSCTL_SET_SPARSE, + &setsparse, 1, CIFSMaxBufSize, NULL, NULL); + if (rc) { + if (rc == -EOPNOTSUPP) + tcon->broken_sparse_sup = true; + cifs_dbg(FYI, "set sparse rc = %d\n", rc); + return rc; + } + + if (setsparse) + cifsi->cifsAttrs |= FILE_ATTRIBUTE_SPARSE_FILE; + else + cifsi->cifsAttrs &= (~FILE_ATTRIBUTE_SPARSE_FILE); + + return 0; +} + +static int +smb2_set_file_size(const unsigned int xid, struct cifs_tcon *tcon, + struct cifsFileInfo *cfile, __u64 size, bool set_alloc) +{ + struct inode *inode; + + /* + * If extending file more than one page make sparse. Many Linux fs + * make files sparse by default when extending via ftruncate + */ + inode = d_inode(cfile->dentry); + + if (!set_alloc && (size > inode->i_size + 8192)) { + __u8 set_sparse = 1; + + /* whether set sparse succeeds or not, extend the file */ + smb2_set_sparse(xid, tcon, cfile, inode, set_sparse); + } + + return SMB2_set_eof(xid, tcon, cfile->fid.persistent_fid, + cfile->fid.volatile_fid, cfile->pid, size); +} + +static int +smb2_duplicate_extents(const unsigned int xid, + struct cifsFileInfo *srcfile, + struct cifsFileInfo *trgtfile, u64 src_off, + u64 len, u64 dest_off) +{ + int rc; + int qrc; + unsigned int ret_data_len; + struct inode *inode; + struct smb2_file_all_info file_inf; + struct duplicate_extents_to_file dup_ext_buf; + struct timespec64 ts; + struct cifs_tcon *tcon = tlink_tcon(trgtfile->tlink); + u64 asize; + + /* server fileays advertise duplicate extent support with this flag */ + if ((le32_to_cpu(tcon->fsAttrInfo.Attributes) & + FILE_SUPPORTS_BLOCK_REFCOUNTING) == 0) + return -EOPNOTSUPP; + + dup_ext_buf.VolatileFileHandle = srcfile->fid.volatile_fid; + dup_ext_buf.PersistentFileHandle = srcfile->fid.persistent_fid; + dup_ext_buf.SourceFileOffset = cpu_to_le64(src_off); + dup_ext_buf.TargetFileOffset = cpu_to_le64(dest_off); + dup_ext_buf.ByteCount = cpu_to_le64(len); + cifs_dbg(FYI, "Duplicate extents: src off %lld dst off %lld len %lld\n", + src_off, dest_off, len); + trace_smb3_clone_enter(xid, srcfile->fid.volatile_fid, + trgtfile->fid.volatile_fid, tcon->tid, + tcon->ses->Suid, src_off, dest_off, len); + inode = d_inode(trgtfile->dentry); + if (inode->i_size < dest_off + len) { + rc = smb2_set_file_size(xid, tcon, trgtfile, dest_off + len, false); + if (rc) + goto duplicate_extents_out; + netfs_resize_file(netfs_inode(inode), dest_off + len, true); + cifs_setsize(inode, dest_off + len); + } + rc = SMB2_ioctl(xid, tcon, trgtfile->fid.persistent_fid, + trgtfile->fid.volatile_fid, + FSCTL_DUPLICATE_EXTENTS_TO_FILE, + (char *)&dup_ext_buf, + sizeof(struct duplicate_extents_to_file), + CIFSMaxBufSize, NULL, + &ret_data_len); + + if (ret_data_len > 0) + cifs_dbg(FYI, "Non-zero response length in duplicate extents\n"); + + if (rc == 0) { + qrc = SMB2_query_info(xid, tcon, trgtfile->fid.persistent_fid, + trgtfile->fid.volatile_fid, &file_inf); + spin_lock(&inode->i_lock); + if (qrc == 0) { + asize = le64_to_cpu(file_inf.AllocationSize); + CIFS_I(inode)->time = jiffies; + if (file_inf.LastWriteTime) { + ts = cifs_NTtimeToUnix(file_inf.LastWriteTime); + inode_set_mtime_to_ts(inode, ts); + } + if (file_inf.ChangeTime) { + ts = cifs_NTtimeToUnix(file_inf.ChangeTime); + inode_set_ctime_to_ts(inode, ts); + } + if (file_inf.LastAccessTime) { + ts = cifs_NTtimeToUnix(file_inf.LastAccessTime); + inode_set_atime_to_ts(inode, ts); + } + inode->i_blocks = CIFS_INO_BLOCKS(asize); + } else { + CIFS_I(inode)->time = 0; /* force reval */ + } + spin_unlock(&inode->i_lock); + } + +duplicate_extents_out: + if (rc) + trace_smb3_clone_err(xid, srcfile->fid.volatile_fid, + trgtfile->fid.volatile_fid, + tcon->tid, tcon->ses->Suid, src_off, + dest_off, len, rc); + else + trace_smb3_clone_done(xid, srcfile->fid.volatile_fid, + trgtfile->fid.volatile_fid, tcon->tid, + tcon->ses->Suid, src_off, dest_off, len); + return rc; +} + +static int +smb2_set_compression(const unsigned int xid, struct cifs_tcon *tcon, + struct cifsFileInfo *cfile, __u16 compression_state) +{ + return SMB2_set_compression(xid, tcon, cfile->fid.persistent_fid, + cfile->fid.volatile_fid, compression_state); +} + +static int +smb3_set_integrity(const unsigned int xid, struct cifs_tcon *tcon, + struct cifsFileInfo *cfile) +{ + struct fsctl_set_integrity_information_req integr_info; + unsigned int ret_data_len; + + integr_info.ChecksumAlgorithm = cpu_to_le16(CHECKSUM_TYPE_UNCHANGED); + integr_info.Flags = 0; + integr_info.Reserved = 0; + + return SMB2_ioctl(xid, tcon, cfile->fid.persistent_fid, + cfile->fid.volatile_fid, + FSCTL_SET_INTEGRITY_INFORMATION, + (char *)&integr_info, + sizeof(struct fsctl_set_integrity_information_req), + CIFSMaxBufSize, NULL, + &ret_data_len); + +} + +/* GMT Token is @GMT-YYYY.MM.DD-HH.MM.SS Unicode which is 48 bytes + null */ +#define GMT_TOKEN_SIZE 50 + +#define MIN_SNAPSHOT_ARRAY_SIZE 16 /* See MS-SMB2 section 3.3.5.15.1 */ + +/* + * Input buffer contains (empty) struct smb_snapshot array with size filled in + * For output see struct SRV_SNAPSHOT_ARRAY in MS-SMB2 section 2.2.32.2 + */ +static int +smb3_enum_snapshots(const unsigned int xid, struct cifs_tcon *tcon, + struct cifsFileInfo *cfile, void __user *ioc_buf) +{ + char *retbuf = NULL; + unsigned int ret_data_len = 0; + int rc; + u32 max_response_size; + struct smb_snapshot_array snapshot_in; + + /* + * On the first query to enumerate the list of snapshots available + * for this volume the buffer begins with 0 (number of snapshots + * which can be returned is zero since at that point we do not know + * how big the buffer needs to be). On the second query, + * it (ret_data_len) is set to number of snapshots so we can + * know to set the maximum response size larger (see below). + */ + if (get_user(ret_data_len, (unsigned int __user *)ioc_buf)) + return -EFAULT; + + /* + * Note that for snapshot queries that servers like Azure expect that + * the first query be minimal size (and just used to get the number/size + * of previous versions) so response size must be specified as EXACTLY + * sizeof(struct snapshot_array) which is 16 when rounded up to multiple + * of eight bytes. + */ + if (ret_data_len == 0) + max_response_size = MIN_SNAPSHOT_ARRAY_SIZE; + else + max_response_size = CIFSMaxBufSize; + + rc = SMB2_ioctl(xid, tcon, cfile->fid.persistent_fid, + cfile->fid.volatile_fid, + FSCTL_SRV_ENUMERATE_SNAPSHOTS, + NULL, 0 /* no input data */, max_response_size, + (char **)&retbuf, + &ret_data_len); + cifs_dbg(FYI, "enum snapshots ioctl returned %d and ret buflen is %d\n", + rc, ret_data_len); + if (rc) + return rc; + + if (ret_data_len && (ioc_buf != NULL) && (retbuf != NULL)) { + /* Fixup buffer */ + if (copy_from_user(&snapshot_in, ioc_buf, + sizeof(struct smb_snapshot_array))) { + rc = -EFAULT; + kfree(retbuf); + return rc; + } + + /* + * Check for min size, ie not large enough to fit even one GMT + * token (snapshot). On the first ioctl some users may pass in + * smaller size (or zero) to simply get the size of the array + * so the user space caller can allocate sufficient memory + * and retry the ioctl again with larger array size sufficient + * to hold all of the snapshot GMT tokens on the second try. + */ + if (snapshot_in.snapshot_array_size < GMT_TOKEN_SIZE) + ret_data_len = sizeof(struct smb_snapshot_array); + + /* + * We return struct SRV_SNAPSHOT_ARRAY, followed by + * the snapshot array (of 50 byte GMT tokens) each + * representing an available previous version of the data + */ + if (ret_data_len > (snapshot_in.snapshot_array_size + + sizeof(struct smb_snapshot_array))) + ret_data_len = snapshot_in.snapshot_array_size + + sizeof(struct smb_snapshot_array); + + if (copy_to_user(ioc_buf, retbuf, ret_data_len)) + rc = -EFAULT; + } + + kfree(retbuf); + return rc; +} + + + +static int +smb3_notify(const unsigned int xid, struct file *pfile, + void __user *ioc_buf, bool return_changes) +{ + struct smb3_notify_info notify; + struct smb3_notify_info __user *pnotify_buf; + struct dentry *dentry = pfile->f_path.dentry; + struct inode *inode = file_inode(pfile); + struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); + struct cifs_open_parms oparms; + struct cifs_fid fid; + struct cifs_tcon *tcon; + const unsigned char *path; + char *returned_ioctl_info = NULL; + void *page = alloc_dentry_path(); + __le16 *utf16_path = NULL; + u8 oplock = SMB2_OPLOCK_LEVEL_NONE; + int rc = 0; + __u32 ret_len = 0; + + path = build_path_from_dentry(dentry, page); + if (IS_ERR(path)) { + rc = PTR_ERR(path); + goto notify_exit; + } + + utf16_path = cifs_convert_path_to_utf16(path, cifs_sb); + if (utf16_path == NULL) { + rc = -ENOMEM; + goto notify_exit; + } + + if (return_changes) { + if (copy_from_user(¬ify, ioc_buf, sizeof(struct smb3_notify_info))) { + rc = -EFAULT; + goto notify_exit; + } + } else { + if (copy_from_user(¬ify, ioc_buf, sizeof(struct smb3_notify))) { + rc = -EFAULT; + goto notify_exit; + } + notify.data_len = 0; + } + + tcon = cifs_sb_master_tcon(cifs_sb); + oparms = (struct cifs_open_parms) { + .tcon = tcon, + .path = path, + .desired_access = FILE_READ_ATTRIBUTES | FILE_READ_DATA, + .disposition = FILE_OPEN, + .create_options = cifs_create_options(cifs_sb, 0), + .fid = &fid, + }; + + rc = SMB2_open(xid, &oparms, utf16_path, &oplock, NULL, NULL, NULL, + NULL); + if (rc) + goto notify_exit; + + rc = SMB2_change_notify(xid, tcon, fid.persistent_fid, fid.volatile_fid, + notify.watch_tree, notify.completion_filter, + notify.data_len, &returned_ioctl_info, &ret_len); + + SMB2_close(xid, tcon, fid.persistent_fid, fid.volatile_fid); + + cifs_dbg(FYI, "change notify for path %s rc %d\n", path, rc); + if (return_changes && (ret_len > 0) && (notify.data_len > 0)) { + if (ret_len > notify.data_len) + ret_len = notify.data_len; + pnotify_buf = (struct smb3_notify_info __user *)ioc_buf; + if (copy_to_user(pnotify_buf->notify_data, returned_ioctl_info, ret_len)) + rc = -EFAULT; + else if (copy_to_user(&pnotify_buf->data_len, &ret_len, sizeof(ret_len))) + rc = -EFAULT; + } + kfree(returned_ioctl_info); +notify_exit: + free_dentry_path(page); + kfree(utf16_path); + return rc; +} + +static int +smb2_query_dir_first(const unsigned int xid, struct cifs_tcon *tcon, + const char *path, struct cifs_sb_info *cifs_sb, + struct cifs_fid *fid, __u16 search_flags, + struct cifs_search_info *srch_inf) +{ + __le16 *utf16_path; + struct smb_rqst rqst[2]; + struct kvec rsp_iov[2]; + int resp_buftype[2]; + struct kvec open_iov[SMB2_CREATE_IOV_SIZE]; + struct kvec qd_iov[SMB2_QUERY_DIRECTORY_IOV_SIZE]; + int rc, flags = 0; + u8 oplock = SMB2_OPLOCK_LEVEL_NONE; + struct cifs_open_parms oparms; + struct smb2_query_directory_rsp *qd_rsp = NULL; + struct smb2_create_rsp *op_rsp = NULL; + struct TCP_Server_Info *server; + int retries = 0, cur_sleep = 0; + +replay_again: + /* reinitialize for possible replay */ + flags = 0; + oplock = SMB2_OPLOCK_LEVEL_NONE; + server = cifs_pick_channel(tcon->ses); + + utf16_path = cifs_convert_path_to_utf16(path, cifs_sb); + if (!utf16_path) + return -ENOMEM; + + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; + + memset(rqst, 0, sizeof(rqst)); + resp_buftype[0] = resp_buftype[1] = CIFS_NO_BUFFER; + memset(rsp_iov, 0, sizeof(rsp_iov)); + + /* Open */ + memset(&open_iov, 0, sizeof(open_iov)); + rqst[0].rq_iov = open_iov; + rqst[0].rq_nvec = SMB2_CREATE_IOV_SIZE; + + oparms = (struct cifs_open_parms) { + .tcon = tcon, + .path = path, + .desired_access = FILE_READ_ATTRIBUTES | FILE_READ_DATA, + .disposition = FILE_OPEN, + .create_options = cifs_create_options(cifs_sb, 0), + .fid = fid, + .replay = !!(retries), + }; + + rc = SMB2_open_init(tcon, server, + &rqst[0], &oplock, &oparms, utf16_path); + if (rc) + goto qdf_free; + smb2_set_next_command(tcon, &rqst[0]); + + /* Query directory */ + srch_inf->entries_in_buffer = 0; + srch_inf->index_of_last_entry = 2; + + memset(&qd_iov, 0, sizeof(qd_iov)); + rqst[1].rq_iov = qd_iov; + rqst[1].rq_nvec = SMB2_QUERY_DIRECTORY_IOV_SIZE; + + rc = SMB2_query_directory_init(xid, tcon, server, + &rqst[1], + COMPOUND_FID, COMPOUND_FID, + 0, srch_inf->info_level); + if (rc) + goto qdf_free; + + smb2_set_related(&rqst[1]); + + if (retries) { + /* Back-off before retry */ + if (cur_sleep) + msleep(cur_sleep); + smb2_set_replay(server, &rqst[0]); + smb2_set_replay(server, &rqst[1]); + } + + rc = compound_send_recv(xid, tcon->ses, server, + flags, 2, rqst, + resp_buftype, rsp_iov); + + /* If the open failed there is nothing to do */ + op_rsp = (struct smb2_create_rsp *)rsp_iov[0].iov_base; + if (op_rsp == NULL || op_rsp->hdr.Status != STATUS_SUCCESS) { + cifs_dbg(FYI, "query_dir_first: open failed rc=%d\n", rc); + goto qdf_free; + } + fid->persistent_fid = op_rsp->PersistentFileId; + fid->volatile_fid = op_rsp->VolatileFileId; + + /* Anything else than ENODATA means a genuine error */ + if (rc && rc != -ENODATA) { + SMB2_close(xid, tcon, fid->persistent_fid, fid->volatile_fid); + cifs_dbg(FYI, "query_dir_first: query directory failed rc=%d\n", rc); + trace_smb3_query_dir_err(xid, fid->persistent_fid, + tcon->tid, tcon->ses->Suid, 0, 0, rc); + goto qdf_free; + } + + atomic_inc(&tcon->num_remote_opens); + + qd_rsp = (struct smb2_query_directory_rsp *)rsp_iov[1].iov_base; + if (qd_rsp->hdr.Status == STATUS_NO_MORE_FILES) { + trace_smb3_query_dir_done(xid, fid->persistent_fid, + tcon->tid, tcon->ses->Suid, 0, 0); + srch_inf->endOfSearch = true; + rc = 0; + goto qdf_free; + } + + rc = smb2_parse_query_directory(tcon, &rsp_iov[1], resp_buftype[1], + srch_inf); + if (rc) { + trace_smb3_query_dir_err(xid, fid->persistent_fid, tcon->tid, + tcon->ses->Suid, 0, 0, rc); + goto qdf_free; + } + resp_buftype[1] = CIFS_NO_BUFFER; + + trace_smb3_query_dir_done(xid, fid->persistent_fid, tcon->tid, + tcon->ses->Suid, 0, srch_inf->entries_in_buffer); + + qdf_free: + kfree(utf16_path); + SMB2_open_free(&rqst[0]); + SMB2_query_directory_free(&rqst[1]); + free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base); + free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base); + + if (is_replayable_error(rc) && + smb2_should_replay(tcon, &retries, &cur_sleep)) + goto replay_again; + + return rc; +} + +static int +smb2_query_dir_next(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_fid *fid, __u16 search_flags, + struct cifs_search_info *srch_inf) +{ + return SMB2_query_directory(xid, tcon, fid->persistent_fid, + fid->volatile_fid, 0, srch_inf); +} + +static int +smb2_close_dir(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_fid *fid) +{ + return SMB2_close(xid, tcon, fid->persistent_fid, fid->volatile_fid); +} + +/* + * If we negotiate SMB2 protocol and get STATUS_PENDING - update + * the number of credits and return true. Otherwise - return false. + */ +static bool +smb2_is_status_pending(char *buf, struct TCP_Server_Info *server) +{ + struct smb2_hdr *shdr = (struct smb2_hdr *)buf; + int scredits, in_flight; + + if (shdr->Status != STATUS_PENDING) + return false; + + if (shdr->CreditRequest) { + spin_lock(&server->req_lock); + server->credits += le16_to_cpu(shdr->CreditRequest); + scredits = server->credits; + in_flight = server->in_flight; + spin_unlock(&server->req_lock); + wake_up(&server->request_q); + + trace_smb3_pend_credits(server->current_mid, + server->conn_id, server->hostname, scredits, + le16_to_cpu(shdr->CreditRequest), in_flight); + cifs_dbg(FYI, "%s: status pending add %u credits total=%d\n", + __func__, le16_to_cpu(shdr->CreditRequest), scredits); + } + + return true; +} + +static bool +smb2_is_session_expired(char *buf) +{ + struct smb2_hdr *shdr = (struct smb2_hdr *)buf; + + if (shdr->Status != STATUS_NETWORK_SESSION_EXPIRED && + shdr->Status != STATUS_USER_SESSION_DELETED) + return false; + + trace_smb3_ses_expired(le32_to_cpu(shdr->Id.SyncId.TreeId), + le64_to_cpu(shdr->SessionId), + le16_to_cpu(shdr->Command), + le64_to_cpu(shdr->MessageId)); + cifs_dbg(FYI, "Session expired or deleted\n"); + + return true; +} + +static bool +smb2_is_status_io_timeout(char *buf) +{ + struct smb2_hdr *shdr = (struct smb2_hdr *)buf; + + if (shdr->Status == STATUS_IO_TIMEOUT) + return true; + else + return false; +} + +static bool +smb2_is_network_name_deleted(char *buf, struct TCP_Server_Info *server) +{ + struct smb2_hdr *shdr = (struct smb2_hdr *)buf; + struct TCP_Server_Info *pserver; + struct cifs_ses *ses; + struct cifs_tcon *tcon; + + if (shdr->Status != STATUS_NETWORK_NAME_DELETED) + return false; + + /* If server is a channel, select the primary channel */ + pserver = SERVER_IS_CHAN(server) ? server->primary_server : server; + + spin_lock(&cifs_tcp_ses_lock); + list_for_each_entry(ses, &pserver->smb_ses_list, smb_ses_list) { + if (cifs_ses_exiting(ses)) + continue; + list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { + if (tcon->tid == le32_to_cpu(shdr->Id.SyncId.TreeId)) { + spin_lock(&tcon->tc_lock); + tcon->need_reconnect = true; + spin_unlock(&tcon->tc_lock); + spin_unlock(&cifs_tcp_ses_lock); + pr_warn_once("Server share %s deleted.\n", + tcon->tree_name); + return true; + } + } + } + spin_unlock(&cifs_tcp_ses_lock); + + return false; +} + +static int smb2_oplock_response(struct cifs_tcon *tcon, __u64 persistent_fid, + __u64 volatile_fid, __u16 net_fid, + struct cifsInodeInfo *cinode, unsigned int oplock) +{ + unsigned int sbflags = cifs_sb_flags(CIFS_SB(cinode)); + __u8 op; + + if (tcon->ses->server->capabilities & SMB2_GLOBAL_CAP_LEASING) + return SMB2_lease_break(0, tcon, cinode->lease_key, + smb2_get_lease_state(cinode, oplock)); + + op = !!((oplock & CIFS_CACHE_READ_FLG) || (sbflags & CIFS_MOUNT_RO_CACHE)); + return SMB2_oplock_break(0, tcon, persistent_fid, volatile_fid, op); +} + +void +smb2_set_replay(struct TCP_Server_Info *server, struct smb_rqst *rqst) +{ + struct smb2_hdr *shdr; + + if (server->dialect < SMB30_PROT_ID) + return; + + shdr = (struct smb2_hdr *)(rqst->rq_iov[0].iov_base); + if (shdr == NULL) { + cifs_dbg(FYI, "shdr NULL in smb2_set_related\n"); + return; + } + shdr->Flags |= SMB2_FLAGS_REPLAY_OPERATION; +} + +void +smb2_set_related(struct smb_rqst *rqst) +{ + struct smb2_hdr *shdr; + + shdr = (struct smb2_hdr *)(rqst->rq_iov[0].iov_base); + if (shdr == NULL) { + cifs_dbg(FYI, "shdr NULL in smb2_set_related\n"); + return; + } + shdr->Flags |= SMB2_FLAGS_RELATED_OPERATIONS; +} + +char smb2_padding[7] = {0, 0, 0, 0, 0, 0, 0}; + +void +smb2_set_next_command(struct cifs_tcon *tcon, struct smb_rqst *rqst) +{ + struct smb2_hdr *shdr; + struct cifs_ses *ses = tcon->ses; + struct TCP_Server_Info *server = ses->server; + unsigned long len = smb_rqst_len(server, rqst); + int num_padding; + + shdr = (struct smb2_hdr *)(rqst->rq_iov[0].iov_base); + if (shdr == NULL) { + cifs_dbg(FYI, "shdr NULL in smb2_set_next_command\n"); + return; + } + + /* SMB headers in a compound are 8 byte aligned. */ + if (IS_ALIGNED(len, 8)) + goto out; + + num_padding = 8 - (len & 7); + if (smb3_encryption_required(tcon)) { + int i; + + /* + * Flatten request into a single buffer with required padding as + * the encryption layer can't handle the padding iovs. + */ + for (i = 1; i < rqst->rq_nvec; i++) { + memcpy(rqst->rq_iov[0].iov_base + + rqst->rq_iov[0].iov_len, + rqst->rq_iov[i].iov_base, + rqst->rq_iov[i].iov_len); + rqst->rq_iov[0].iov_len += rqst->rq_iov[i].iov_len; + } + memset(rqst->rq_iov[0].iov_base + rqst->rq_iov[0].iov_len, + 0, num_padding); + rqst->rq_iov[0].iov_len += num_padding; + rqst->rq_nvec = 1; + } else { + rqst->rq_iov[rqst->rq_nvec].iov_base = smb2_padding; + rqst->rq_iov[rqst->rq_nvec].iov_len = num_padding; + rqst->rq_nvec++; + } + len += num_padding; +out: + shdr->NextCommand = cpu_to_le32(len); +} + +/* + * helper function for exponential backoff and check if replayable + */ +bool smb2_should_replay(struct cifs_tcon *tcon, + int *pretries, + int *pcur_sleep) +{ + if (!pretries || !pcur_sleep) + return false; + + if (tcon->retry || (*pretries)++ < tcon->ses->server->retrans) { + /* Update sleep time for exponential backoff */ + if (!(*pcur_sleep)) + (*pcur_sleep) = 1; + else { + (*pcur_sleep) = ((*pcur_sleep) << 1); + if ((*pcur_sleep) > CIFS_MAX_SLEEP) + (*pcur_sleep) = CIFS_MAX_SLEEP; + } + return true; + } + + return false; +} + +/* + * Passes the query info response back to the caller on success. + * Caller need to free this with free_rsp_buf(). + */ +int +smb2_query_info_compound(const unsigned int xid, struct cifs_tcon *tcon, + const char *path, u32 desired_access, + u32 class, u32 type, u32 output_len, + struct kvec *rsp, int *buftype, + struct cifs_sb_info *cifs_sb) +{ + struct smb2_compound_vars *vars; + struct cifs_ses *ses = tcon->ses; + struct TCP_Server_Info *server; + int flags = CIFS_CP_CREATE_CLOSE_OP; + struct smb_rqst *rqst; + int resp_buftype[3]; + struct kvec *rsp_iov; + u8 oplock = SMB2_OPLOCK_LEVEL_NONE; + struct cifs_open_parms oparms; + struct cifs_fid fid; + int rc; + __le16 *utf16_path; + struct cached_fid *cfid; + int retries = 0, cur_sleep = 0; + +replay_again: + /* reinitialize for possible replay */ + cfid = NULL; + flags = CIFS_CP_CREATE_CLOSE_OP; + oplock = SMB2_OPLOCK_LEVEL_NONE; + server = cifs_pick_channel(ses); + + if (!path) + path = ""; + utf16_path = cifs_convert_path_to_utf16(path, cifs_sb); + if (!utf16_path) + return -ENOMEM; + + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; + + resp_buftype[0] = resp_buftype[1] = resp_buftype[2] = CIFS_NO_BUFFER; + vars = kzalloc_obj(*vars); + if (!vars) { + rc = -ENOMEM; + goto out_free_path; + } + rqst = vars->rqst; + rsp_iov = vars->rsp_iov; + + /* + * We can only call this for things we know are directories. + */ + if (!strcmp(path, "")) + open_cached_dir(xid, tcon, path, cifs_sb, false, + &cfid); /* cfid null if open dir failed */ + + rqst[0].rq_iov = vars->open_iov; + rqst[0].rq_nvec = SMB2_CREATE_IOV_SIZE; + + oparms = (struct cifs_open_parms) { + .tcon = tcon, + .path = path, + .desired_access = desired_access, + .disposition = FILE_OPEN, + .create_options = cifs_create_options(cifs_sb, 0), + .fid = &fid, + .replay = !!(retries), + }; + + rc = SMB2_open_init(tcon, server, + &rqst[0], &oplock, &oparms, utf16_path); + if (rc) + goto qic_exit; + smb2_set_next_command(tcon, &rqst[0]); + + rqst[1].rq_iov = &vars->qi_iov; + rqst[1].rq_nvec = 1; + + if (cfid) { + rc = SMB2_query_info_init(tcon, server, + &rqst[1], + cfid->fid.persistent_fid, + cfid->fid.volatile_fid, + class, type, 0, + output_len, 0, + NULL); + } else { + rc = SMB2_query_info_init(tcon, server, + &rqst[1], + COMPOUND_FID, + COMPOUND_FID, + class, type, 0, + output_len, 0, + NULL); + } + if (rc) + goto qic_exit; + if (!cfid) { + smb2_set_next_command(tcon, &rqst[1]); + smb2_set_related(&rqst[1]); + } + + rqst[2].rq_iov = &vars->close_iov; + rqst[2].rq_nvec = 1; + + rc = SMB2_close_init(tcon, server, + &rqst[2], COMPOUND_FID, COMPOUND_FID, false); + if (rc) + goto qic_exit; + smb2_set_related(&rqst[2]); + + if (retries) { + /* Back-off before retry */ + if (cur_sleep) + msleep(cur_sleep); + if (!cfid) { + smb2_set_replay(server, &rqst[0]); + smb2_set_replay(server, &rqst[2]); + } + smb2_set_replay(server, &rqst[1]); + } + + if (cfid) { + rc = compound_send_recv(xid, ses, server, + flags, 1, &rqst[1], + &resp_buftype[1], &rsp_iov[1]); + } else { + rc = compound_send_recv(xid, ses, server, + flags, 3, rqst, + resp_buftype, rsp_iov); + } + if (rc) { + free_rsp_buf(resp_buftype[1], rsp_iov[1].iov_base); + if (rc == -EREMCHG) { + tcon->need_reconnect = true; + pr_warn_once("server share %s deleted\n", + tcon->tree_name); + } + goto qic_exit; + } + *rsp = rsp_iov[1]; + *buftype = resp_buftype[1]; + + qic_exit: + SMB2_open_free(&rqst[0]); + SMB2_query_info_free(&rqst[1]); + SMB2_close_free(&rqst[2]); + free_rsp_buf(resp_buftype[0], rsp_iov[0].iov_base); + free_rsp_buf(resp_buftype[2], rsp_iov[2].iov_base); + if (cfid) + close_cached_dir(cfid); + kfree(vars); +out_free_path: + kfree(utf16_path); + + if (is_replayable_error(rc) && + smb2_should_replay(tcon, &retries, &cur_sleep)) + goto replay_again; + + return rc; +} + +static int +smb2_queryfs(const unsigned int xid, struct cifs_tcon *tcon, + const char *path, struct cifs_sb_info *cifs_sb, struct kstatfs *buf) +{ + struct smb2_query_info_rsp *rsp; + struct smb2_fs_full_size_info *info = NULL; + struct kvec rsp_iov = {NULL, 0}; + int buftype = CIFS_NO_BUFFER; + int rc; + + + rc = smb2_query_info_compound(xid, tcon, path, + FILE_READ_ATTRIBUTES, + FS_FULL_SIZE_INFORMATION, + SMB2_O_INFO_FILESYSTEM, + sizeof(struct smb2_fs_full_size_info), + &rsp_iov, &buftype, cifs_sb); + if (rc) + goto qfs_exit; + + rsp = (struct smb2_query_info_rsp *)rsp_iov.iov_base; + buf->f_type = SMB2_SUPER_MAGIC; + info = (struct smb2_fs_full_size_info *)( + le16_to_cpu(rsp->OutputBufferOffset) + (char *)rsp); + rc = smb2_validate_iov(le16_to_cpu(rsp->OutputBufferOffset), + le32_to_cpu(rsp->OutputBufferLength), + &rsp_iov, + sizeof(struct smb2_fs_full_size_info)); + if (!rc) + smb2_copy_fs_info_to_kstatfs(info, buf); + +qfs_exit: + trace_smb3_qfs_done(xid, tcon->tid, tcon->ses->Suid, tcon->tree_name, rc); + free_rsp_buf(buftype, rsp_iov.iov_base); + return rc; +} + +static int +smb311_queryfs(const unsigned int xid, struct cifs_tcon *tcon, + const char *path, struct cifs_sb_info *cifs_sb, struct kstatfs *buf) +{ + int rc; + __le16 *utf16_path = NULL; + u8 oplock = SMB2_OPLOCK_LEVEL_NONE; + struct cifs_open_parms oparms; + struct cifs_fid fid; + + if (!tcon->posix_extensions) + return smb2_queryfs(xid, tcon, path, cifs_sb, buf); + + oparms = (struct cifs_open_parms) { + .tcon = tcon, + .path = path, + .desired_access = FILE_READ_ATTRIBUTES, + .disposition = FILE_OPEN, + .create_options = cifs_create_options(cifs_sb, 0), + .fid = &fid, + }; + + utf16_path = cifs_convert_path_to_utf16(path, cifs_sb); + if (utf16_path == NULL) + return -ENOMEM; + + rc = SMB2_open(xid, &oparms, utf16_path, &oplock, NULL, NULL, + NULL, NULL); + kfree(utf16_path); + if (rc) + return rc; + + rc = SMB311_posix_qfs_info(xid, tcon, fid.persistent_fid, + fid.volatile_fid, buf); + buf->f_type = SMB2_SUPER_MAGIC; + SMB2_close(xid, tcon, fid.persistent_fid, fid.volatile_fid); + return rc; +} + +static bool +smb2_compare_fids(struct cifsFileInfo *ob1, struct cifsFileInfo *ob2) +{ + return ob1->fid.persistent_fid == ob2->fid.persistent_fid && + ob1->fid.volatile_fid == ob2->fid.volatile_fid; +} + +static int +smb2_mand_lock(const unsigned int xid, struct cifsFileInfo *cfile, __u64 offset, + __u64 length, __u32 type, int lock, int unlock, bool wait) +{ + if (unlock && !lock) + type = SMB2_LOCKFLAG_UNLOCK; + return SMB2_lock(xid, tlink_tcon(cfile->tlink), + cfile->fid.persistent_fid, cfile->fid.volatile_fid, + current->tgid, length, offset, type, wait); +} + +static void +smb2_get_lease_key(struct inode *inode, struct cifs_fid *fid) +{ + memcpy(fid->lease_key, CIFS_I(inode)->lease_key, SMB2_LEASE_KEY_SIZE); +} + +static void +smb2_set_lease_key(struct inode *inode, struct cifs_fid *fid) +{ + memcpy(CIFS_I(inode)->lease_key, fid->lease_key, SMB2_LEASE_KEY_SIZE); +} + +static void +smb2_new_lease_key(struct cifs_fid *fid) +{ + generate_random_uuid(fid->lease_key); +} + +static int +smb2_get_dfs_refer(const unsigned int xid, struct cifs_ses *ses, + const char *search_name, + struct dfs_info3_param **target_nodes, + unsigned int *num_of_nodes, + const struct nls_table *nls_codepage, int remap) +{ + int rc; + __le16 *utf16_path = NULL; + int utf16_path_len = 0; + struct cifs_tcon *tcon; + struct fsctl_get_dfs_referral_req *dfs_req = NULL; + struct get_dfs_referral_rsp *dfs_rsp = NULL; + u32 dfs_req_size = 0, dfs_rsp_size = 0; + int retry_once = 0; + + cifs_dbg(FYI, "%s: path: %s\n", __func__, search_name); + + /* + * Try to use the IPC tcon, otherwise just use any + */ + tcon = ses->tcon_ipc; + if (tcon == NULL) { + spin_lock(&cifs_tcp_ses_lock); + tcon = list_first_entry_or_null(&ses->tcon_list, + struct cifs_tcon, + tcon_list); + if (tcon) { + spin_lock(&tcon->tc_lock); + tcon->tc_count++; + spin_unlock(&tcon->tc_lock); + trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, + netfs_trace_tcon_ref_get_dfs_refer); + } + spin_unlock(&cifs_tcp_ses_lock); + } + + if (tcon == NULL) { + cifs_dbg(VFS, "session %p has no tcon available for a dfs referral request\n", + ses); + rc = -ENOTCONN; + goto out; + } + + utf16_path = cifs_strndup_to_utf16(search_name, PATH_MAX, + &utf16_path_len, + nls_codepage, remap); + if (!utf16_path) { + rc = -ENOMEM; + goto out; + } + + dfs_req_size = sizeof(*dfs_req) + utf16_path_len; + dfs_req = kzalloc(dfs_req_size, GFP_KERNEL); + if (!dfs_req) { + rc = -ENOMEM; + goto out; + } + + /* Highest DFS referral version understood */ + dfs_req->MaxReferralLevel = DFS_VERSION; + + /* Path to resolve in an UTF-16 null-terminated string */ + memcpy(dfs_req->RequestFileName, utf16_path, utf16_path_len); + + for (;;) { + rc = SMB2_ioctl(xid, tcon, NO_FILE_ID, NO_FILE_ID, + FSCTL_DFS_GET_REFERRALS, + (char *)dfs_req, dfs_req_size, CIFSMaxBufSize, + (char **)&dfs_rsp, &dfs_rsp_size); + if (fatal_signal_pending(current)) { + rc = -EINTR; + break; + } + if (!is_retryable_error(rc) || retry_once++) + break; + usleep_range(512, 2048); + } + + if (!rc && !dfs_rsp) + rc = smb_EIO(smb_eio_trace_dfsref_no_rsp); + if (rc) { + if (!is_retryable_error(rc) && rc != -ENOENT && rc != -EOPNOTSUPP) + cifs_tcon_dbg(FYI, "%s: ioctl error: rc=%d\n", __func__, rc); + goto out; + } + + rc = parse_dfs_referrals(dfs_rsp, dfs_rsp_size, + num_of_nodes, target_nodes, + nls_codepage, remap, search_name, + true /* is_unicode */); + if (rc && rc != -ENOENT) { + cifs_tcon_dbg(VFS, "%s: failed to parse DFS referral %s: %d\n", + __func__, search_name, rc); + } + + out: + if (tcon && !tcon->ipc) { + /* ipc tcons are not refcounted */ + cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_dfs_refer); + } + kfree(utf16_path); + kfree(dfs_req); + kfree(dfs_rsp); + return rc; +} + +static struct smb_ntsd * +get_smb2_acl_by_fid(struct cifs_sb_info *cifs_sb, + const struct cifs_fid *cifsfid, u32 *pacllen, u32 info) +{ + struct smb_ntsd *pntsd = NULL; + unsigned int xid; + int rc = -EOPNOTSUPP; + struct tcon_link *tlink = cifs_sb_tlink(cifs_sb); + + if (IS_ERR(tlink)) + return ERR_CAST(tlink); + + xid = get_xid(); + cifs_dbg(FYI, "trying to get acl\n"); + + rc = SMB2_query_acl(xid, tlink_tcon(tlink), cifsfid->persistent_fid, + cifsfid->volatile_fid, (void **)&pntsd, pacllen, + info); + free_xid(xid); + + cifs_put_tlink(tlink); + + cifs_dbg(FYI, "%s: rc = %d ACL len %d\n", __func__, rc, *pacllen); + if (rc) + return ERR_PTR(rc); + return pntsd; + +} + +static struct smb_ntsd * +get_smb2_acl_by_path(struct cifs_sb_info *cifs_sb, + const char *path, u32 *pacllen, u32 info) +{ + struct smb_ntsd *pntsd = NULL; + u8 oplock = SMB2_OPLOCK_LEVEL_NONE; + unsigned int xid; + int rc; + struct cifs_tcon *tcon; + struct tcon_link *tlink = cifs_sb_tlink(cifs_sb); + struct cifs_fid fid; + struct cifs_open_parms oparms; + __le16 *utf16_path; + + cifs_dbg(FYI, "get smb3 acl for path %s\n", path); + if (IS_ERR(tlink)) + return ERR_CAST(tlink); + + tcon = tlink_tcon(tlink); + xid = get_xid(); + + utf16_path = cifs_convert_path_to_utf16(path, cifs_sb); + if (!utf16_path) { + rc = -ENOMEM; + goto put_tlink; + } + + oparms = (struct cifs_open_parms) { + .tcon = tcon, + .path = path, + .desired_access = READ_CONTROL, + .disposition = FILE_OPEN, + /* + * When querying an ACL, even if the file is a symlink + * we want to open the source not the target, and so + * the protocol requires that the client specify this + * flag when opening a reparse point + */ + .create_options = cifs_create_options(cifs_sb, 0) | + OPEN_REPARSE_POINT, + .fid = &fid, + }; + + if (info & SACL_SECINFO) + oparms.desired_access |= SYSTEM_SECURITY; + + rc = SMB2_open(xid, &oparms, utf16_path, &oplock, NULL, NULL, NULL, + NULL); + kfree(utf16_path); + if (!rc) { + rc = SMB2_query_acl(xid, tlink_tcon(tlink), fid.persistent_fid, + fid.volatile_fid, (void **)&pntsd, pacllen, + info); + SMB2_close(xid, tcon, fid.persistent_fid, fid.volatile_fid); + } + +put_tlink: + cifs_put_tlink(tlink); + free_xid(xid); + + cifs_dbg(FYI, "%s: rc = %d ACL len %d\n", __func__, rc, *pacllen); + if (rc) + return ERR_PTR(rc); + return pntsd; +} + +static int +set_smb2_acl(struct smb_ntsd *pnntsd, __u32 acllen, + struct inode *inode, const char *path, int aclflag) +{ + u8 oplock = SMB2_OPLOCK_LEVEL_NONE; + unsigned int xid; + int rc, access_flags = 0; + struct cifs_tcon *tcon; + struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); + struct tcon_link *tlink = cifs_sb_tlink(cifs_sb); + struct cifs_fid fid; + struct cifs_open_parms oparms; + __le16 *utf16_path; + + cifs_dbg(FYI, "set smb3 acl for path %s\n", path); + if (IS_ERR(tlink)) + return PTR_ERR(tlink); + + tcon = tlink_tcon(tlink); + xid = get_xid(); + + if (aclflag & CIFS_ACL_OWNER || aclflag & CIFS_ACL_GROUP) + access_flags |= WRITE_OWNER; + if (aclflag & CIFS_ACL_SACL) + access_flags |= SYSTEM_SECURITY; + if (aclflag & CIFS_ACL_DACL) + access_flags |= WRITE_DAC; + + utf16_path = cifs_convert_path_to_utf16(path, cifs_sb); + if (!utf16_path) { + rc = -ENOMEM; + goto put_tlink; + } + + oparms = (struct cifs_open_parms) { + .tcon = tcon, + .desired_access = access_flags, + .create_options = cifs_create_options(cifs_sb, 0), + .disposition = FILE_OPEN, + .path = path, + .fid = &fid, + }; + + rc = SMB2_open(xid, &oparms, utf16_path, &oplock, NULL, NULL, + NULL, NULL); + kfree(utf16_path); + if (!rc) { + rc = SMB2_set_acl(xid, tlink_tcon(tlink), fid.persistent_fid, + fid.volatile_fid, pnntsd, acllen, aclflag); + SMB2_close(xid, tcon, fid.persistent_fid, fid.volatile_fid); + } + +put_tlink: + cifs_put_tlink(tlink); + free_xid(xid); + return rc; +} + +/* Retrieve an ACL from the server */ +static struct smb_ntsd * +get_smb2_acl(struct cifs_sb_info *cifs_sb, + struct inode *inode, const char *path, + u32 *pacllen, u32 info) +{ + struct smb_ntsd *pntsd = NULL; + struct cifsFileInfo *open_file = NULL; + + if (inode && !(info & SACL_SECINFO)) + open_file = find_readable_file(CIFS_I(inode), FIND_FSUID_ONLY); + if (!open_file || (info & SACL_SECINFO)) + return get_smb2_acl_by_path(cifs_sb, path, pacllen, info); + + pntsd = get_smb2_acl_by_fid(cifs_sb, &open_file->fid, pacllen, info); + cifsFileInfo_put(open_file); + return pntsd; +} + +static long smb3_zero_data(struct file *file, struct cifs_tcon *tcon, + loff_t offset, loff_t len, unsigned int xid) +{ + struct cifsFileInfo *cfile = file->private_data; + struct file_zero_data_information fsctl_buf; + + cifs_dbg(FYI, "Offset %lld len %lld\n", offset, len); + + fsctl_buf.FileOffset = cpu_to_le64(offset); + fsctl_buf.BeyondFinalZero = cpu_to_le64(offset + len); + + return SMB2_ioctl(xid, tcon, cfile->fid.persistent_fid, + cfile->fid.volatile_fid, FSCTL_SET_ZERO_DATA, + (char *)&fsctl_buf, + sizeof(struct file_zero_data_information), + 0, NULL, NULL); +} + +static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon, + unsigned long long offset, unsigned long long len, + bool keep_size) +{ + struct cifs_ses *ses = tcon->ses; + struct inode *inode = file_inode(file); + struct cifsInodeInfo *cifsi = CIFS_I(inode); + struct cifsFileInfo *cfile = file->private_data; + unsigned long long i_size, new_size, remote_i_size, zero_point; + long rc; + unsigned int xid; + + xid = get_xid(); + + trace_smb3_zero_enter(xid, cfile->fid.persistent_fid, tcon->tid, + ses->Suid, offset, len); + + filemap_invalidate_lock(inode->i_mapping); + + netfs_read_sizes(inode, &i_size, &remote_i_size, &zero_point); + if (offset + len >= remote_i_size && offset < i_size) { + unsigned long long top = umin(offset + len, i_size); + + rc = filemap_write_and_wait_range(inode->i_mapping, offset, top - 1); + if (rc < 0) + goto zero_range_exit; + } + + /* + * We zero the range through ioctl, so we need remove the page caches + * first, otherwise the data may be inconsistent with the server. + */ + truncate_pagecache_range(inode, offset, offset + len - 1); + netfs_wait_for_outstanding_io(inode); + + /* if file not oplocked can't be sure whether asking to extend size */ + rc = -EOPNOTSUPP; + if (keep_size == false && !CIFS_CACHE_READ(cifsi)) + goto zero_range_exit; + + rc = smb3_zero_data(file, tcon, offset, len, xid); + if (rc < 0) + goto zero_range_exit; + + /* + * do we also need to change the size of the file? + */ + new_size = offset + len; + if (keep_size == false && (unsigned long long)i_size_read(inode) < new_size) { + rc = SMB2_set_eof(xid, tcon, cfile->fid.persistent_fid, + cfile->fid.volatile_fid, cfile->pid, new_size); + if (rc >= 0) { + truncate_setsize(inode, new_size); + spin_lock(&inode->i_lock); + netfs_resize_file(&cifsi->netfs, new_size, true); + if (offset < cifsi->netfs._zero_point) + netfs_write_zero_point(inode, offset); + spin_unlock(&inode->i_lock); + fscache_resize_cookie(cifs_inode_cookie(inode), new_size); + } + } + + zero_range_exit: + filemap_invalidate_unlock(inode->i_mapping); + free_xid(xid); + if (rc) + trace_smb3_zero_err(xid, cfile->fid.persistent_fid, tcon->tid, + ses->Suid, offset, len, rc); + else + trace_smb3_zero_done(xid, cfile->fid.persistent_fid, tcon->tid, + ses->Suid, offset, len); + return rc; +} + +static long smb3_punch_hole(struct file *file, struct cifs_tcon *tcon, + loff_t offset, loff_t len) +{ + struct inode *inode = file_inode(file); + struct cifsFileInfo *cfile = file->private_data; + struct file_zero_data_information fsctl_buf; + unsigned long long end = offset + len, i_size, remote_i_size, zero_point; + long rc; + unsigned int xid; + __u8 set_sparse = 1; + + xid = get_xid(); + + /* Need to make file sparse, if not already, before freeing range. */ + /* Consider adding equivalent for compressed since it could also work */ + rc = smb2_set_sparse(xid, tcon, cfile, inode, set_sparse); + if (rc) + goto out; + + filemap_invalidate_lock(inode->i_mapping); + /* + * Flush dirty data first, otherwise a dirty folio spanning the punched + * range may be written back after the ioctl and refill the hole. + */ + rc = filemap_write_and_wait_range(inode->i_mapping, offset, + offset + len - 1); + if (rc < 0) + goto unlock; + + /* + * We implement the punch hole through ioctl, so we need remove the page + * caches first, otherwise the data may be inconsistent with the server. + */ + truncate_pagecache_range(inode, offset, offset + len - 1); + netfs_wait_for_outstanding_io(inode); + + cifs_dbg(FYI, "Offset %lld len %lld\n", offset, len); + + fsctl_buf.FileOffset = cpu_to_le64(offset); + fsctl_buf.BeyondFinalZero = cpu_to_le64(offset + len); + + rc = SMB2_ioctl(xid, tcon, cfile->fid.persistent_fid, + cfile->fid.volatile_fid, FSCTL_SET_ZERO_DATA, + (char *)&fsctl_buf, + sizeof(struct file_zero_data_information), + CIFSMaxBufSize, NULL, NULL); + + if (rc) + goto unlock; + + /* If there's dirty data in the buffer that would extend the EOF if it + * were written, then we need to move the EOF marker over to the lower + * of the high end of the hole and the proposed EOF. The problem is + * that we locally hole-punch the tail of the dirty data, the proposed + * EOF update will end up in the wrong place. + */ + netfs_read_sizes(inode, &i_size, &remote_i_size, &zero_point); + + if (end > remote_i_size && i_size > remote_i_size) { + unsigned long long extend_to = umin(end, i_size); + rc = SMB2_set_eof(xid, tcon, cfile->fid.persistent_fid, + cfile->fid.volatile_fid, cfile->pid, extend_to); + if (rc >= 0) { + spin_lock(&inode->i_lock); + netfs_write_remote_i_size(inode, extend_to); + spin_unlock(&inode->i_lock); + } + } + +unlock: + filemap_invalidate_unlock(inode->i_mapping); +out: + free_xid(xid); + return rc; +} + +static int smb3_simple_fallocate_write_range(unsigned int xid, + struct cifs_tcon *tcon, + struct cifsFileInfo *cfile, + loff_t off, loff_t len, + char *buf) +{ + struct cifs_io_parms io_parms = {0}; + unsigned int nbytes; + int rc = 0; + struct kvec iov[2]; + + io_parms.netfid = cfile->fid.netfid; + io_parms.pid = current->tgid; + io_parms.tcon = tcon; + io_parms.persistent_fid = cfile->fid.persistent_fid; + io_parms.volatile_fid = cfile->fid.volatile_fid; + + while (len) { + io_parms.offset = off; + io_parms.length = len; + if (io_parms.length > SMB2_MAX_BUFFER_SIZE) + io_parms.length = SMB2_MAX_BUFFER_SIZE; + /* iov[0] is reserved for smb header */ + iov[1].iov_base = buf; + iov[1].iov_len = io_parms.length; + rc = SMB2_write(xid, &io_parms, &nbytes, iov, 1); + if (rc) + break; + if (!nbytes) + return -EIO; + if (nbytes > len) + return -EINVAL; + off += nbytes; + len -= nbytes; + } + return rc; +} + +static int smb3_simple_fallocate_range(unsigned int xid, + struct cifs_tcon *tcon, + struct cifsFileInfo *cfile, + loff_t off, loff_t len) +{ + struct file_allocated_range_buffer in_data, *out_data = NULL, *tmp_data; + struct inode *inode = d_inode(cfile->dentry); + u32 out_data_len; + char *buf = NULL; + u64 range_start, range_len, range_end; + loff_t l; + int rc; + + buf = kvzalloc(min_t(loff_t, len, SMB2_MAX_BUFFER_SIZE), GFP_KERNEL); + if (!buf) { + rc = -ENOMEM; + goto out; + } + + if (off >= i_size_read(inode)) { + rc = smb3_simple_fallocate_write_range(xid, tcon, cfile, + off, len, buf); + goto out; + } + + in_data.file_offset = cpu_to_le64(off); + in_data.length = cpu_to_le64(len); + rc = SMB2_ioctl(xid, tcon, cfile->fid.persistent_fid, + cfile->fid.volatile_fid, + FSCTL_QUERY_ALLOCATED_RANGES, + (char *)&in_data, sizeof(in_data), + 1024 * sizeof(struct file_allocated_range_buffer), + (char **)&out_data, &out_data_len); + if (rc) + goto out; + + tmp_data = out_data; + while (len) { + /* + * The rest of the region is unmapped so write it all. + */ + if (out_data_len == 0) { + rc = smb3_simple_fallocate_write_range(xid, tcon, + cfile, off, len, buf); + goto out; + } + + if (out_data_len < sizeof(struct file_allocated_range_buffer)) { + rc = -EINVAL; + goto out; + } + + range_start = le64_to_cpu(tmp_data->file_offset); + range_len = le64_to_cpu(tmp_data->length); + if (check_add_overflow(range_start, range_len, &range_end) || + range_end > S64_MAX) { + rc = -EINVAL; + goto out; + } + + if (off < range_start) { + /* + * We are at a hole. Write until the end of the region + * or until the next allocated data, + * whichever comes next. + */ + l = range_start - off; + if (len < l) + l = len; + rc = smb3_simple_fallocate_write_range(xid, tcon, + cfile, off, l, buf); + if (rc) + goto out; + off = off + l; + len = len - l; + if (len == 0) + goto out; + } + /* + * We are at a section of allocated data, just skip forward + * until the end of the data or the end of the region + * we are supposed to fallocate, whichever comes first. + */ + if (off < range_end) { + l = range_end - off; + if (len < l) + l = len; + off += l; + len -= l; + } + + tmp_data = &tmp_data[1]; + out_data_len -= sizeof(struct file_allocated_range_buffer); + } + + out: + kfree(out_data); + kvfree(buf); + return rc; +} + + +static long smb3_simple_falloc(struct file *file, struct cifs_tcon *tcon, + loff_t off, loff_t len, bool keep_size) +{ + struct inode *inode; + struct cifsInodeInfo *cifsi; + struct cifsFileInfo *cfile = file->private_data; + long rc = -EOPNOTSUPP; + unsigned int xid; + loff_t old_eof, new_eof; + struct smb2_file_all_info file_inf; + u64 asize; + int qrc; + + xid = get_xid(); + + inode = d_inode(cfile->dentry); + cifsi = CIFS_I(inode); + old_eof = i_size_read(inode); + + trace_smb3_falloc_enter(xid, cfile->fid.persistent_fid, tcon->tid, + tcon->ses->Suid, off, len); + /* if file not oplocked can't be sure whether asking to extend size */ + if (!CIFS_CACHE_READ(cifsi)) + if (!keep_size) { + trace_smb3_falloc_err(xid, cfile->fid.persistent_fid, + tcon->tid, tcon->ses->Suid, off, len, rc); + free_xid(xid); + return rc; + } + + /* + * Extending the file + */ + if (!keep_size && old_eof < off + len) { + rc = inode_newsize_ok(inode, off + len); + if (rc) + goto out; + + /* + * A small range at or beyond EOF can be allocated by writing + * zeroes. For off > old_eof, this preserves the intervening + * hole instead of allocating from offset 0. + */ + if (off > old_eof || + (off == old_eof && old_eof != 0 && + (cifsi->cifsAttrs & FILE_ATTRIBUTE_SPARSE_FILE))) { + if (len > 1024 * 1024) { + rc = -EOPNOTSUPP; + goto out; + } + + rc = smb3_simple_fallocate_range(xid, tcon, cfile, + off, len); + if (rc) { + spin_lock(&inode->i_lock); + cifsi->time = 0; + spin_unlock(&inode->i_lock); + goto out; + } + + new_eof = off + len; + netfs_resize_file(&cifsi->netfs, new_eof, true); + cifs_setsize(inode, new_eof); + + qrc = SMB2_query_info(xid, tcon, + cfile->fid.persistent_fid, + cfile->fid.volatile_fid, &file_inf); + spin_lock(&inode->i_lock); + if (qrc == 0) { + asize = le64_to_cpu(file_inf.AllocationSize); + inode->i_blocks = CIFS_INO_BLOCKS(asize); + } else { + cifsi->time = 0; + } + spin_unlock(&inode->i_lock); + goto out; + } + + if (cifsi->cifsAttrs & FILE_ATTRIBUTE_SPARSE_FILE) + smb2_set_sparse(xid, tcon, cfile, inode, false); + + new_eof = off + len; + + qrc = SMB2_query_info(xid, tcon, + cfile->fid.persistent_fid, + cfile->fid.volatile_fid, &file_inf); + if (qrc == 0) + asize = le64_to_cpu(file_inf.AllocationSize); + + /* + * FILE_ALLOCATION_INFORMATION can only describe allocation up to + * new_eof. Some servers may accept it without allocating blocks, + * so refresh AllocationSize before updating i_blocks. + */ + if (off == 0 || off == old_eof) { + if (qrc || asize < new_eof) { + rc = SMB2_set_allocation(xid, tcon, + cfile->fid.persistent_fid, + cfile->fid.volatile_fid, + cfile->pid, new_eof); + if (rc) + goto out; + } + } + + rc = SMB2_set_eof(xid, tcon, cfile->fid.persistent_fid, + cfile->fid.volatile_fid, cfile->pid, new_eof); + if (rc) + goto out; + + netfs_resize_file(&cifsi->netfs, new_eof, true); + cifs_setsize(inode, new_eof); + + qrc = SMB2_query_info(xid, tcon, + cfile->fid.persistent_fid, + cfile->fid.volatile_fid, &file_inf); + spin_lock(&inode->i_lock); + if (qrc == 0) { + asize = le64_to_cpu(file_inf.AllocationSize); + if (asize >= new_eof) + inode->i_blocks = CIFS_INO_BLOCKS(asize); + } else { + cifsi->time = 0; + } + spin_unlock(&inode->i_lock); + goto out; + } + + /* + * Files are non-sparse by default so falloc may be a no-op + * Must check if file sparse. If not sparse, and since we are not + * extending then no need to do anything since file already allocated + */ + if ((cifsi->cifsAttrs & FILE_ATTRIBUTE_SPARSE_FILE) == 0) { + rc = 0; + goto out; + } + + if (keep_size == true) { + /* + * We can not preallocate pages beyond the end of the file + * in SMB2 + */ + if (off >= i_size_read(inode)) { + rc = 0; + goto out; + } + /* + * For fallocates that are partially beyond the end of file, + * clamp len so we only fallocate up to the end of file. + */ + if (off + len > i_size_read(inode)) { + len = i_size_read(inode) - off; + } + } + + if ((keep_size == true) || (i_size_read(inode) >= off + len)) { + /* + * At this point, we are trying to fallocate an internal + * regions of a sparse file. Since smb2 does not have a + * fallocate command we have two options on how to emulate this. + * We can either turn the entire file to become non-sparse + * which we only do if the fallocate is for virtually + * the whole file, or we can overwrite the region with zeroes + * using SMB2_write, which could be prohibitevly expensive + * if len is large. + */ + /* + * We are only trying to fallocate a small region so + * just write it with zero. + */ + if (len <= 1024 * 1024) { + rc = smb3_simple_fallocate_range(xid, tcon, cfile, + off, len); + goto out; + } + + /* + * Check if falloc starts within first few pages of file + * and ends within a few pages of the end of file to + * ensure that most of file is being forced to be + * fallocated now. If so then setting whole file sparse + * ie potentially making a few extra pages at the beginning + * or end of the file non-sparse via set_sparse is harmless. + */ + if ((off > 8192) || (off + len + 8192 < i_size_read(inode))) { + rc = -EOPNOTSUPP; + goto out; + } + } + + smb2_set_sparse(xid, tcon, cfile, inode, false); + rc = 0; + +out: + if (rc) + trace_smb3_falloc_err(xid, cfile->fid.persistent_fid, tcon->tid, + tcon->ses->Suid, off, len, rc); + else + trace_smb3_falloc_done(xid, cfile->fid.persistent_fid, tcon->tid, + tcon->ses->Suid, off, len); + + free_xid(xid); + return rc; +} + +static long smb3_collapse_range(struct file *file, struct cifs_tcon *tcon, + loff_t off, loff_t len) +{ + int rc; + unsigned int xid; + struct inode *inode = file_inode(file); + struct cifsInodeInfo *cifsi = CIFS_I(inode); + struct cifsFileInfo *cfile = file->private_data; + loff_t old_eof, new_eof; + + xid = get_xid(); + + old_eof = i_size_read(inode); + if ((off >= old_eof) || + off + len >= old_eof) { + rc = -EINVAL; + goto out; + } + + filemap_invalidate_lock(inode->i_mapping); + rc = filemap_write_and_wait_range(inode->i_mapping, off, old_eof - 1); + if (rc < 0) + goto out_2; + + truncate_pagecache_range(inode, off, old_eof); + spin_lock(&inode->i_lock); + netfs_write_zero_point(inode, old_eof); + spin_unlock(&inode->i_lock); + netfs_wait_for_outstanding_io(inode); + + rc = smb2_copychunk_range(xid, cfile, cfile, off + len, + old_eof - off - len, off); + if (rc < 0) + goto out_2; + + new_eof = old_eof - len; + rc = SMB2_set_eof(xid, tcon, cfile->fid.persistent_fid, + cfile->fid.volatile_fid, cfile->pid, new_eof); + if (rc < 0) + goto out_2; + + rc = 0; + + truncate_setsize(inode, new_eof); + spin_lock(&inode->i_lock); + netfs_resize_file(&cifsi->netfs, new_eof, true); + netfs_write_zero_point(inode, new_eof); + spin_unlock(&inode->i_lock); + fscache_resize_cookie(cifs_inode_cookie(inode), new_eof); +out_2: + filemap_invalidate_unlock(inode->i_mapping); +out: + free_xid(xid); + return rc; +} + +static long smb3_insert_range(struct file *file, struct cifs_tcon *tcon, + loff_t off, loff_t len) +{ + int rc; + unsigned int xid; + struct cifsFileInfo *cfile = file->private_data; + struct inode *inode = file_inode(file); + struct cifsInodeInfo *cifsi = CIFS_I(inode); + __u64 count, old_eof, new_eof; + + xid = get_xid(); + + old_eof = i_size_read(inode); + if (off >= old_eof) { + rc = -EINVAL; + goto out; + } + + count = old_eof - off; + new_eof = old_eof + len; + + filemap_invalidate_lock(inode->i_mapping); + rc = filemap_write_and_wait_range(inode->i_mapping, off, new_eof - 1); + if (rc < 0) + goto out_2; + truncate_pagecache_range(inode, off, old_eof); + netfs_wait_for_outstanding_io(inode); + + rc = SMB2_set_eof(xid, tcon, cfile->fid.persistent_fid, + cfile->fid.volatile_fid, cfile->pid, new_eof); + if (rc < 0) + goto out_2; + + truncate_setsize(inode, new_eof); + spin_lock(&inode->i_lock); + netfs_resize_file(&cifsi->netfs, i_size_read(inode), true); + spin_unlock(&inode->i_lock); + fscache_resize_cookie(cifs_inode_cookie(inode), i_size_read(inode)); + + rc = smb2_copychunk_range(xid, cfile, cfile, off, count, off + len); + if (rc < 0) + goto out_2; + spin_lock(&inode->i_lock); + netfs_write_zero_point(inode, new_eof); + spin_unlock(&inode->i_lock); + + rc = smb3_zero_data(file, tcon, off, len, xid); + if (rc < 0) + goto out_2; + + rc = 0; +out_2: + filemap_invalidate_unlock(inode->i_mapping); +out: + free_xid(xid); + return rc; +} + +static loff_t smb3_llseek(struct file *file, struct cifs_tcon *tcon, loff_t offset, int whence) +{ + struct cifsFileInfo *wrcfile, *cfile = file->private_data; + struct cifsInodeInfo *cifsi; + struct inode *inode; + int rc = 0; + struct file_allocated_range_buffer in_data, *out_data = NULL; + u32 out_data_len; + unsigned int xid; + + if (whence != SEEK_HOLE && whence != SEEK_DATA) + return generic_file_llseek(file, offset, whence); + + inode = d_inode(cfile->dentry); + cifsi = CIFS_I(inode); + + if (offset < 0 || offset >= i_size_read(inode)) + return -ENXIO; + + xid = get_xid(); + /* + * We need to be sure that all dirty pages are written as they + * might fill holes on the server. + * Note that we also MUST flush any written pages since at least + * some servers (Windows2016) will not reflect recent writes in + * QUERY_ALLOCATED_RANGES until SMB2_flush is called. + */ + wrcfile = find_writable_file(cifsi, FIND_ANY); + if (wrcfile) { + filemap_write_and_wait(inode->i_mapping); + smb2_flush_file(xid, tcon, &wrcfile->fid); + cifsFileInfo_put(wrcfile); + } + + if (!(cifsi->cifsAttrs & FILE_ATTRIBUTE_SPARSE_FILE)) { + if (whence == SEEK_HOLE) + offset = i_size_read(inode); + goto lseek_exit; + } + + in_data.file_offset = cpu_to_le64(offset); + in_data.length = cpu_to_le64(i_size_read(inode)); + + rc = SMB2_ioctl(xid, tcon, cfile->fid.persistent_fid, + cfile->fid.volatile_fid, + FSCTL_QUERY_ALLOCATED_RANGES, + (char *)&in_data, sizeof(in_data), + sizeof(struct file_allocated_range_buffer), + (char **)&out_data, &out_data_len); + if (rc == -E2BIG) + rc = 0; + if (rc) + goto lseek_exit; + + if (whence == SEEK_HOLE && out_data_len == 0) + goto lseek_exit; + + if (whence == SEEK_DATA && out_data_len == 0) { + rc = -ENXIO; + goto lseek_exit; + } + + if (out_data_len < sizeof(struct file_allocated_range_buffer)) { + rc = -EINVAL; + goto lseek_exit; + } + if (whence == SEEK_DATA) { + offset = le64_to_cpu(out_data->file_offset); + goto lseek_exit; + } + if (offset < le64_to_cpu(out_data->file_offset)) + goto lseek_exit; + + offset = le64_to_cpu(out_data->file_offset) + le64_to_cpu(out_data->length); + + lseek_exit: + free_xid(xid); + kfree(out_data); + if (!rc) + return vfs_setpos(file, offset, inode->i_sb->s_maxbytes); + else + return rc; +} + +static int smb3_fiemap(struct cifs_tcon *tcon, + struct cifsFileInfo *cfile, + struct fiemap_extent_info *fei, u64 start, u64 len) +{ + unsigned int xid; + struct file_allocated_range_buffer in_data, *out_data; + u32 out_data_len; + int i, num, rc, flags, last_blob; + u64 next; + + rc = fiemap_prep(d_inode(cfile->dentry), fei, start, &len, 0); + if (rc) + return rc; + + xid = get_xid(); + again: + in_data.file_offset = cpu_to_le64(start); + in_data.length = cpu_to_le64(len); + + rc = SMB2_ioctl(xid, tcon, cfile->fid.persistent_fid, + cfile->fid.volatile_fid, + FSCTL_QUERY_ALLOCATED_RANGES, + (char *)&in_data, sizeof(in_data), + 1024 * sizeof(struct file_allocated_range_buffer), + (char **)&out_data, &out_data_len); + if (rc == -E2BIG) { + last_blob = 0; + rc = 0; + } else + last_blob = 1; + if (rc) + goto out; + + if (out_data_len && out_data_len < sizeof(struct file_allocated_range_buffer)) { + rc = -EINVAL; + goto out; + } + if (out_data_len % sizeof(struct file_allocated_range_buffer)) { + rc = -EINVAL; + goto out; + } + + num = out_data_len / sizeof(struct file_allocated_range_buffer); + for (i = 0; i < num; i++) { + flags = 0; + if (i == num - 1 && last_blob) + flags |= FIEMAP_EXTENT_LAST; + + rc = fiemap_fill_next_extent(fei, + le64_to_cpu(out_data[i].file_offset), + le64_to_cpu(out_data[i].file_offset), + le64_to_cpu(out_data[i].length), + flags); + if (rc < 0) + goto out; + if (rc == 1) { + rc = 0; + goto out; + } + } + + if (!last_blob) { + next = le64_to_cpu(out_data[num - 1].file_offset) + + le64_to_cpu(out_data[num - 1].length); + len = len - (next - start); + start = next; + goto again; + } + + out: + free_xid(xid); + kfree(out_data); + return rc; +} + +static long smb3_fallocate(struct file *file, struct cifs_tcon *tcon, int mode, + loff_t off, loff_t len) +{ + /* KEEP_SIZE already checked for by do_fallocate */ + if (mode & FALLOC_FL_PUNCH_HOLE) + return smb3_punch_hole(file, tcon, off, len); + else if (mode & FALLOC_FL_ZERO_RANGE) { + if (mode & FALLOC_FL_KEEP_SIZE) + return smb3_zero_range(file, tcon, off, len, true); + return smb3_zero_range(file, tcon, off, len, false); + } else if (mode == FALLOC_FL_KEEP_SIZE) + return smb3_simple_falloc(file, tcon, off, len, true); + else if (mode == FALLOC_FL_COLLAPSE_RANGE) + return smb3_collapse_range(file, tcon, off, len); + else if (mode == FALLOC_FL_INSERT_RANGE) + return smb3_insert_range(file, tcon, off, len); + else if (mode == FALLOC_FL_ALLOCATE_RANGE) + return smb3_simple_falloc(file, tcon, off, len, false); + + return -EOPNOTSUPP; +} + +static void +smb2_downgrade_oplock(struct TCP_Server_Info *server, + struct cifsInodeInfo *cinode, __u32 oplock, + __u16 epoch, bool *purge_cache) +{ + lockdep_assert_held(&cinode->open_file_lock); + server->ops->set_oplock_level(cinode, oplock, 0, NULL); +} + +static void +smb21_set_oplock_level(struct cifsInodeInfo *cinode, __u32 oplock, + __u16 epoch, bool *purge_cache); + +static void +smb3_downgrade_oplock(struct TCP_Server_Info *server, + struct cifsInodeInfo *cinode, __u32 oplock, + __u16 epoch, bool *purge_cache) +{ + unsigned int old_state = cinode->oplock; + __u16 old_epoch = cinode->epoch; + unsigned int new_state; + + if (epoch > old_epoch) { + smb21_set_oplock_level(cinode, oplock, 0, NULL); + cinode->epoch = epoch; + } + + new_state = cinode->oplock; + *purge_cache = false; + + if ((old_state & CIFS_CACHE_READ_FLG) != 0 && + (new_state & CIFS_CACHE_READ_FLG) == 0) + *purge_cache = true; + else if (old_state == new_state && (epoch - old_epoch > 1)) + *purge_cache = true; +} + +static void +smb2_set_oplock_level(struct cifsInodeInfo *cinode, __u32 oplock, + __u16 epoch, bool *purge_cache) +{ + oplock &= 0xFF; + cinode->lease_granted = false; + if (oplock == SMB2_OPLOCK_LEVEL_NOCHANGE) + return; + if (oplock == SMB2_OPLOCK_LEVEL_BATCH) { + WRITE_ONCE(cinode->oplock, CIFS_CACHE_RHW_FLG); + cifs_dbg(FYI, "Batch Oplock granted on inode %p\n", + &cinode->netfs.inode); + } else if (oplock == SMB2_OPLOCK_LEVEL_EXCLUSIVE) { + WRITE_ONCE(cinode->oplock, CIFS_CACHE_RW_FLG); + cifs_dbg(FYI, "Exclusive Oplock granted on inode %p\n", + &cinode->netfs.inode); + } else if (oplock == SMB2_OPLOCK_LEVEL_II) { + WRITE_ONCE(cinode->oplock, CIFS_CACHE_READ_FLG); + cifs_dbg(FYI, "Level II Oplock granted on inode %p\n", + &cinode->netfs.inode); + } else + WRITE_ONCE(cinode->oplock, 0); +} + +static void +smb21_set_oplock_level(struct cifsInodeInfo *cinode, __u32 oplock, + __u16 epoch, bool *purge_cache) +{ + char message[5] = {0}; + unsigned int new_oplock = 0; + + oplock &= 0xFF; + cinode->lease_granted = true; + if (oplock == SMB2_OPLOCK_LEVEL_NOCHANGE) + return; + + /* Check if the server granted an oplock rather than a lease */ + if (oplock & SMB2_OPLOCK_LEVEL_EXCLUSIVE) + return smb2_set_oplock_level(cinode, oplock, epoch, + purge_cache); + + if (oplock & SMB2_LEASE_READ_CACHING_HE) { + new_oplock |= CIFS_CACHE_READ_FLG; + strcat(message, "R"); + } + if (oplock & SMB2_LEASE_HANDLE_CACHING_HE) { + new_oplock |= CIFS_CACHE_HANDLE_FLG; + strcat(message, "H"); + } + if (oplock & SMB2_LEASE_WRITE_CACHING_HE) { + new_oplock |= CIFS_CACHE_WRITE_FLG; + strcat(message, "W"); + } + if (!new_oplock) + strscpy(message, "None"); + + WRITE_ONCE(cinode->oplock, new_oplock); + cifs_dbg(FYI, "%s Lease granted on inode %p\n", message, + &cinode->netfs.inode); +} + +static void +smb3_set_oplock_level(struct cifsInodeInfo *cinode, __u32 oplock, + __u16 epoch, bool *purge_cache) +{ + unsigned int old_oplock = READ_ONCE(cinode->oplock); + unsigned int new_oplock; + + smb21_set_oplock_level(cinode, oplock, epoch, purge_cache); + new_oplock = READ_ONCE(cinode->oplock); + + if (purge_cache) { + *purge_cache = false; + if (old_oplock == CIFS_CACHE_READ_FLG) { + if (new_oplock == CIFS_CACHE_READ_FLG && + (epoch - cinode->epoch > 0)) + *purge_cache = true; + else if (new_oplock == CIFS_CACHE_RH_FLG && + (epoch - cinode->epoch > 1)) + *purge_cache = true; + else if (new_oplock == CIFS_CACHE_RHW_FLG && + (epoch - cinode->epoch > 1)) + *purge_cache = true; + else if (new_oplock == 0 && + (epoch - cinode->epoch > 0)) + *purge_cache = true; + } else if (old_oplock == CIFS_CACHE_RH_FLG) { + if (new_oplock == CIFS_CACHE_RH_FLG && + (epoch - cinode->epoch > 0)) + *purge_cache = true; + else if (new_oplock == CIFS_CACHE_RHW_FLG && + (epoch - cinode->epoch > 1)) + *purge_cache = true; + } + cinode->epoch = epoch; + } +} + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY +static bool +smb2_is_read_op(__u32 oplock) +{ + return oplock == SMB2_OPLOCK_LEVEL_II; +} +#endif /* CIFS_ALLOW_INSECURE_LEGACY */ + +static bool +smb21_is_read_op(__u32 oplock) +{ + return (oplock & SMB2_LEASE_READ_CACHING_HE) && + !(oplock & SMB2_LEASE_WRITE_CACHING_HE); +} + +static __le32 +map_oplock_to_lease(u8 oplock) +{ + if (oplock == SMB2_OPLOCK_LEVEL_EXCLUSIVE) + return SMB2_LEASE_WRITE_CACHING_LE | SMB2_LEASE_READ_CACHING_LE; + else if (oplock == SMB2_OPLOCK_LEVEL_II) + return SMB2_LEASE_READ_CACHING_LE | SMB2_LEASE_HANDLE_CACHING_LE; + else if (oplock == SMB2_OPLOCK_LEVEL_BATCH) + return SMB2_LEASE_HANDLE_CACHING_LE | SMB2_LEASE_READ_CACHING_LE | + SMB2_LEASE_WRITE_CACHING_LE; + return 0; +} + +static char * +smb2_create_lease_buf(u8 *lease_key, u8 oplock, u8 *parent_lease_key, __le32 flags) +{ + struct create_lease *buf; + + buf = kzalloc_obj(struct create_lease); + if (!buf) + return NULL; + + memcpy(&buf->lcontext.LeaseKey, lease_key, SMB2_LEASE_KEY_SIZE); + buf->lcontext.LeaseState = map_oplock_to_lease(oplock); + + buf->ccontext.DataOffset = cpu_to_le16(offsetof + (struct create_lease, lcontext)); + buf->ccontext.DataLength = cpu_to_le32(sizeof(struct lease_context)); + buf->ccontext.NameOffset = cpu_to_le16(offsetof + (struct create_lease, Name)); + buf->ccontext.NameLength = cpu_to_le16(4); + /* SMB2_CREATE_REQUEST_LEASE is "RqLs" */ + buf->Name[0] = 'R'; + buf->Name[1] = 'q'; + buf->Name[2] = 'L'; + buf->Name[3] = 's'; + return (char *)buf; +} + +static char * +smb3_create_lease_buf(u8 *lease_key, u8 oplock, u8 *parent_lease_key, __le32 flags) +{ + struct create_lease_v2 *buf; + + buf = kzalloc_obj(struct create_lease_v2); + if (!buf) + return NULL; + + memcpy(&buf->lcontext.LeaseKey, lease_key, SMB2_LEASE_KEY_SIZE); + buf->lcontext.LeaseState = map_oplock_to_lease(oplock); + buf->lcontext.LeaseFlags = flags; + if (flags & SMB2_LEASE_FLAG_PARENT_LEASE_KEY_SET_LE) + memcpy(&buf->lcontext.ParentLeaseKey, parent_lease_key, SMB2_LEASE_KEY_SIZE); + + buf->ccontext.DataOffset = cpu_to_le16(offsetof + (struct create_lease_v2, lcontext)); + buf->ccontext.DataLength = cpu_to_le32(sizeof(struct lease_context_v2)); + buf->ccontext.NameOffset = cpu_to_le16(offsetof + (struct create_lease_v2, Name)); + buf->ccontext.NameLength = cpu_to_le16(4); + /* SMB2_CREATE_REQUEST_LEASE is "RqLs" */ + buf->Name[0] = 'R'; + buf->Name[1] = 'q'; + buf->Name[2] = 'L'; + buf->Name[3] = 's'; + return (char *)buf; +} + +static __u8 +smb2_parse_lease_buf(void *buf, __u16 *epoch, char *lease_key) +{ + struct create_lease *lc = (struct create_lease *)buf; + + *epoch = 0; /* not used */ + if (lc->lcontext.LeaseFlags & SMB2_LEASE_FLAG_BREAK_IN_PROGRESS_LE) + return SMB2_OPLOCK_LEVEL_NOCHANGE; + return le32_to_cpu(lc->lcontext.LeaseState); +} + +static __u8 +smb3_parse_lease_buf(void *buf, __u16 *epoch, char *lease_key) +{ + struct create_lease_v2 *lc = (struct create_lease_v2 *)buf; + + *epoch = le16_to_cpu(lc->lcontext.Epoch); + if (lc->lcontext.LeaseFlags & SMB2_LEASE_FLAG_BREAK_IN_PROGRESS_LE) + return SMB2_OPLOCK_LEVEL_NOCHANGE; + if (lease_key) + memcpy(lease_key, &lc->lcontext.LeaseKey, SMB2_LEASE_KEY_SIZE); + return le32_to_cpu(lc->lcontext.LeaseState); +} + +static unsigned int +smb2_wp_retry_size(struct inode *inode) +{ + return min_t(unsigned int, CIFS_SB(inode->i_sb)->ctx->wsize, + SMB2_MAX_BUFFER_SIZE); +} + +static bool +smb2_dir_needs_close(struct cifsFileInfo *cfile) +{ + return !cfile->invalidHandle; +} + +static void +fill_transform_hdr(struct smb2_transform_hdr *tr_hdr, unsigned int orig_len, + struct smb_rqst *old_rq, __le16 cipher_type) +{ + struct smb2_hdr *shdr = + (struct smb2_hdr *)old_rq->rq_iov[0].iov_base; + + memset(tr_hdr, 0, sizeof(struct smb2_transform_hdr)); + tr_hdr->ProtocolId = SMB2_TRANSFORM_PROTO_NUM; + tr_hdr->OriginalMessageSize = cpu_to_le32(orig_len); + tr_hdr->Flags = cpu_to_le16(0x01); + if ((cipher_type == SMB2_ENCRYPTION_AES128_GCM) || + (cipher_type == SMB2_ENCRYPTION_AES256_GCM)) + get_random_bytes(&tr_hdr->Nonce, SMB3_AES_GCM_NONCE); + else + get_random_bytes(&tr_hdr->Nonce, SMB3_AES_CCM_NONCE); + memcpy(&tr_hdr->SessionId, &shdr->SessionId, 8); +} + +static void *smb2_aead_req_alloc(struct crypto_aead *tfm, const struct smb_rqst *rqst, + int num_rqst, const u8 *sig, u8 **iv, + struct aead_request **req, struct sg_table *sgt, + unsigned int *num_sgs) +{ + unsigned int req_size = sizeof(**req) + crypto_aead_reqsize(tfm); + unsigned int iv_size = crypto_aead_ivsize(tfm); + unsigned int len; + int ret; + u8 *p; + + ret = cifs_get_num_sgs(rqst, num_rqst, sig); + if (ret < 0) + return ERR_PTR(ret); + *num_sgs = ret; + + len = iv_size; + len += crypto_aead_alignmask(tfm) & ~(crypto_tfm_ctx_alignment() - 1); + len = ALIGN(len, crypto_tfm_ctx_alignment()); + len += req_size; + len = ALIGN(len, __alignof__(struct scatterlist)); + len += array_size(*num_sgs, sizeof(struct scatterlist)); + + p = kzalloc(len, GFP_NOFS); + if (!p) + return ERR_PTR(-ENOMEM); + + *iv = (u8 *)PTR_ALIGN(p, crypto_aead_alignmask(tfm) + 1); + *req = (struct aead_request *)PTR_ALIGN(*iv + iv_size, + crypto_tfm_ctx_alignment()); + sgt->sgl = (struct scatterlist *)PTR_ALIGN((u8 *)*req + req_size, + __alignof__(struct scatterlist)); + return p; +} + +static void *smb2_get_aead_req(struct crypto_aead *tfm, struct smb_rqst *rqst, + int num_rqst, const u8 *sig, u8 **iv, + struct aead_request **req, struct scatterlist **sgl) +{ + struct sg_table sgtable = {}; + unsigned int skip, num_sgs, i, j; + ssize_t rc; + void *p; + + p = smb2_aead_req_alloc(tfm, rqst, num_rqst, sig, iv, req, &sgtable, &num_sgs); + if (IS_ERR(p)) + return ERR_CAST(p); + + sg_init_marker(sgtable.sgl, num_sgs); + + /* + * The first rqst has a transform header where the + * first 20 bytes are not part of the encrypted blob. + */ + skip = 20; + + for (i = 0; i < num_rqst; i++) { + struct iov_iter *iter = &rqst[i].rq_iter; + size_t count = iov_iter_count(iter); + + for (j = 0; j < rqst[i].rq_nvec; j++) { + cifs_sg_set_buf(&sgtable, + rqst[i].rq_iov[j].iov_base + skip, + rqst[i].rq_iov[j].iov_len - skip); + + /* See the above comment on the 'skip' assignment */ + skip = 0; + } + sgtable.orig_nents = sgtable.nents; + + rc = extract_iter_to_sg(iter, count, &sgtable, + num_sgs - sgtable.nents, 0); + iov_iter_revert(iter, rc); + sgtable.orig_nents = sgtable.nents; + } + + cifs_sg_set_buf(&sgtable, sig, SMB2_SIGNATURE_SIZE); + sg_mark_end(&sgtable.sgl[sgtable.nents - 1]); + *sgl = sgtable.sgl; + return p; +} + +static int +smb2_get_enc_key(struct TCP_Server_Info *server, __u64 ses_id, int enc, u8 *key) +{ + struct TCP_Server_Info *pserver; + struct cifs_ses *ses; + u8 *ses_enc_key; + + /* If server is a channel, select the primary channel */ + pserver = SERVER_IS_CHAN(server) ? server->primary_server : server; + + spin_lock(&cifs_tcp_ses_lock); + list_for_each_entry(ses, &pserver->smb_ses_list, smb_ses_list) { + if (ses->Suid == ses_id) { + spin_lock(&ses->ses_lock); + ses_enc_key = enc ? ses->smb3encryptionkey : + ses->smb3decryptionkey; + memcpy(key, ses_enc_key, SMB3_ENC_DEC_KEY_SIZE); + spin_unlock(&ses->ses_lock); + spin_unlock(&cifs_tcp_ses_lock); + return 0; + } + } + spin_unlock(&cifs_tcp_ses_lock); + + trace_smb3_ses_not_found(ses_id); + + return -EAGAIN; +} +/* + * Encrypt or decrypt @rqst message. @rqst[0] has the following format: + * iov[0] - transform header (associate data), + * iov[1-N] - SMB2 header and pages - data to encrypt. + * On success return encrypted data in iov[1-N] and pages, leave iov[0] + * untouched. + */ +static int +crypt_message(struct TCP_Server_Info *server, int num_rqst, + struct smb_rqst *rqst, int enc, struct crypto_aead *tfm) +{ + struct smb2_transform_hdr *tr_hdr = + (struct smb2_transform_hdr *)rqst[0].rq_iov[0].iov_base; + unsigned int assoc_data_len = sizeof(struct smb2_transform_hdr) - 20; + int rc = 0; + struct scatterlist *sg; + u8 sign[SMB2_SIGNATURE_SIZE] = {}; + u8 key[SMB3_ENC_DEC_KEY_SIZE]; + struct aead_request *req; + u8 *iv; + DECLARE_CRYPTO_WAIT(wait); + unsigned int crypt_len = le32_to_cpu(tr_hdr->OriginalMessageSize); + void *creq; + + rc = smb2_get_enc_key(server, le64_to_cpu(tr_hdr->SessionId), enc, key); + if (rc) { + cifs_server_dbg(FYI, "%s: Could not get %scryption key. sid: 0x%llx\n", __func__, + enc ? "en" : "de", le64_to_cpu(tr_hdr->SessionId)); + return rc; + } + + if ((server->cipher_type == SMB2_ENCRYPTION_AES256_CCM) || + (server->cipher_type == SMB2_ENCRYPTION_AES256_GCM)) + rc = crypto_aead_setkey(tfm, key, SMB3_GCM256_CRYPTKEY_SIZE); + else + rc = crypto_aead_setkey(tfm, key, SMB3_GCM128_CRYPTKEY_SIZE); + + if (rc) { + cifs_server_dbg(VFS, "%s: Failed to set aead key %d\n", __func__, rc); + return rc; + } + + rc = crypto_aead_setauthsize(tfm, SMB2_SIGNATURE_SIZE); + if (rc) { + cifs_server_dbg(VFS, "%s: Failed to set authsize %d\n", __func__, rc); + return rc; + } + + creq = smb2_get_aead_req(tfm, rqst, num_rqst, sign, &iv, &req, &sg); + if (IS_ERR(creq)) + return PTR_ERR(creq); + + if (!enc) { + memcpy(sign, &tr_hdr->Signature, SMB2_SIGNATURE_SIZE); + crypt_len += SMB2_SIGNATURE_SIZE; + } + + if ((server->cipher_type == SMB2_ENCRYPTION_AES128_GCM) || + (server->cipher_type == SMB2_ENCRYPTION_AES256_GCM)) + memcpy(iv, (char *)tr_hdr->Nonce, SMB3_AES_GCM_NONCE); + else { + iv[0] = 3; + memcpy(iv + 1, (char *)tr_hdr->Nonce, SMB3_AES_CCM_NONCE); + } + + aead_request_set_tfm(req, tfm); + aead_request_set_crypt(req, sg, sg, crypt_len, iv); + aead_request_set_ad(req, assoc_data_len); + + aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + + rc = crypto_wait_req(enc ? crypto_aead_encrypt(req) + : crypto_aead_decrypt(req), &wait); + + if (!rc && enc) + memcpy(&tr_hdr->Signature, sign, SMB2_SIGNATURE_SIZE); + + kfree_sensitive(creq); + return rc; +} + +/* + * Copy data from an iterator to the folios in a folio queue buffer. + */ +static bool cifs_copy_iter_to_folioq(struct iov_iter *iter, size_t size, + struct folio_queue *buffer) +{ + for (; buffer; buffer = buffer->next) { + for (int s = 0; s < folioq_count(buffer); s++) { + struct folio *folio = folioq_folio(buffer, s); + size_t part = folioq_folio_size(buffer, s); + + part = umin(part, size); + + if (copy_folio_from_iter(folio, 0, part, iter) != part) + return false; + size -= part; + } + } + return true; +} + +void +smb3_free_compound_rqst(int num_rqst, struct smb_rqst *rqst) +{ + for (int i = 0; i < num_rqst; i++) + netfs_free_folioq_buffer(rqst[i].rq_buffer); +} + +/* + * This function will initialize new_rq and encrypt the content. + * The first entry, new_rq[0], only contains a single iov which contains + * a smb2_transform_hdr and is pre-allocated by the caller. + * This function then populates new_rq[1+] with the content from olq_rq[0+]. + * + * The end result is an array of smb_rqst structures where the first structure + * only contains a single iov for the transform header which we then can pass + * to crypt_message(). + * + * new_rq[0].rq_iov[0] : smb2_transform_hdr pre-allocated by the caller + * new_rq[1+].rq_iov[*] == old_rq[0+].rq_iov[*] : SMB2/3 requests + */ +static int +smb3_init_transform_rq(struct TCP_Server_Info *server, int num_rqst, + struct smb_rqst *new_rq, struct smb_rqst *old_rq) +{ + struct smb2_transform_hdr *tr_hdr = new_rq[0].rq_iov[0].iov_base; + unsigned int orig_len = 0; + int rc = -ENOMEM; + + for (int i = 1; i < num_rqst; i++) { + struct smb_rqst *old = &old_rq[i - 1]; + struct smb_rqst *new = &new_rq[i]; + struct folio_queue *buffer = NULL; + size_t size = iov_iter_count(&old->rq_iter); + + orig_len += smb_rqst_len(server, old); + new->rq_iov = old->rq_iov; + new->rq_nvec = old->rq_nvec; + + if (size > 0) { + size_t cur_size = 0; + rc = netfs_alloc_folioq_buffer(NULL, &buffer, &cur_size, + size, GFP_NOFS); + new->rq_buffer = buffer; + if (rc < 0) + goto err_free; + + iov_iter_folio_queue(&new->rq_iter, ITER_SOURCE, + buffer, 0, 0, size); + + if (!cifs_copy_iter_to_folioq(&old->rq_iter, size, buffer)) { + rc = smb_EIO1(smb_eio_trace_tx_copy_iter_to_buf, size); + goto err_free; + } + } + } + + /* fill the 1st iov with a transform header */ + fill_transform_hdr(tr_hdr, orig_len, old_rq, server->cipher_type); + + rc = crypt_message(server, num_rqst, new_rq, 1, server->secmech.enc); + cifs_dbg(FYI, "Encrypt message returned %d\n", rc); + if (rc) + goto err_free; + + return rc; + +err_free: + smb3_free_compound_rqst(num_rqst - 1, &new_rq[1]); + return rc; +} + +static int +smb3_is_transform_hdr(void *buf) +{ + struct smb2_transform_hdr *trhdr = buf; + + return trhdr->ProtocolId == SMB2_TRANSFORM_PROTO_NUM; +} + +static int +decrypt_raw_data(struct TCP_Server_Info *server, char *buf, + unsigned int buf_data_size, struct iov_iter *iter, + bool is_offloaded) +{ + struct crypto_aead *tfm; + struct smb_rqst rqst = {NULL}; + struct kvec iov[2]; + size_t iter_size = 0; + int rc; + + iov[0].iov_base = buf; + iov[0].iov_len = sizeof(struct smb2_transform_hdr); + iov[1].iov_base = buf + sizeof(struct smb2_transform_hdr); + iov[1].iov_len = buf_data_size; + + rqst.rq_iov = iov; + rqst.rq_nvec = 2; + if (iter) { + rqst.rq_iter = *iter; + iter_size = iov_iter_count(iter); + } + + if (is_offloaded) { + if ((server->cipher_type == SMB2_ENCRYPTION_AES128_GCM) || + (server->cipher_type == SMB2_ENCRYPTION_AES256_GCM)) + tfm = crypto_alloc_aead("gcm(aes)", 0, 0); + else + tfm = crypto_alloc_aead("ccm(aes)", 0, 0); + if (IS_ERR(tfm)) { + rc = PTR_ERR(tfm); + cifs_server_dbg(VFS, "%s: Failed alloc decrypt TFM, rc=%d\n", __func__, rc); + + return rc; + } + } else { + rc = smb3_crypto_aead_allocate(server); + if (unlikely(rc)) + return rc; + tfm = server->secmech.dec; + } + + rc = crypt_message(server, 1, &rqst, 0, tfm); + cifs_dbg(FYI, "Decrypt message returned %d\n", rc); + + if (is_offloaded) + crypto_free_aead(tfm); + + if (rc) + return rc; + + memmove(buf, iov[1].iov_base, buf_data_size); + + if (!is_offloaded) + server->total_read = buf_data_size + iter_size; + + return rc; +} + +static int +cifs_copy_folioq_to_iter(struct folio_queue *folioq, size_t data_size, + size_t skip, struct iov_iter *iter) +{ + for (; folioq; folioq = folioq->next) { + for (int s = 0; s < folioq_count(folioq); s++) { + struct folio *folio; + size_t fsize, n, len; + + if (data_size == 0) + return 0; + + folio = folioq_folio(folioq, s); + fsize = folio_size(folio); + len = umin(fsize - skip, data_size); + + n = copy_folio_to_iter(folio, skip, len, iter); + if (n != len) { + cifs_dbg(VFS, "%s: something went wrong\n", __func__); + return smb_EIO2(smb_eio_trace_rx_copy_to_iter, + n, len); + } + data_size -= n; + skip = 0; + } + } + + if (data_size != 0) { + cifs_dbg(VFS, "%s: short copy, %zu bytes missing\n", + __func__, data_size); + return smb_EIO2(smb_eio_trace_rx_copy_to_iter, 0, data_size); + } + + return 0; +} + +static int +handle_read_data(struct TCP_Server_Info *server, struct mid_q_entry *mid, + char *buf, unsigned int buf_len, struct folio_queue *buffer, + unsigned int buffer_len, bool is_offloaded) +{ + unsigned int data_offset; + unsigned int data_len; + unsigned int end_off; + unsigned int cur_off; + unsigned int cur_page_idx; + unsigned int pad_len; + struct cifs_io_subrequest *rdata = mid->callback_data; + struct smb2_hdr *shdr = (struct smb2_hdr *)buf; + size_t copied; + bool use_rdma_mr = false; + + if (shdr->Command != SMB2_READ) { + cifs_server_dbg(VFS, "only big read responses are supported\n"); + return -EOPNOTSUPP; + } + + if (server->ops->is_session_expired && + server->ops->is_session_expired(buf)) { + if (!is_offloaded) + cifs_reconnect(server, true); + return -1; + } + + if (server->ops->is_status_pending && + server->ops->is_status_pending(buf, server)) + return -1; + + /* set up first two iov to get credits */ + rdata->iov[0].iov_base = buf; + rdata->iov[0].iov_len = 0; + rdata->iov[1].iov_base = buf; + rdata->iov[1].iov_len = + min_t(unsigned int, buf_len, server->vals->read_rsp_size); + cifs_dbg(FYI, "0: iov_base=%p iov_len=%zu\n", + rdata->iov[0].iov_base, rdata->iov[0].iov_len); + cifs_dbg(FYI, "1: iov_base=%p iov_len=%zu\n", + rdata->iov[1].iov_base, rdata->iov[1].iov_len); + + rdata->result = server->ops->map_error(buf, true); + if (rdata->result != 0) { + cifs_dbg(FYI, "%s: server returned error %d\n", + __func__, rdata->result); + /* normal error on read response */ + if (is_offloaded) + mid->mid_state = MID_RESPONSE_RECEIVED; + else + dequeue_mid(server, mid, false); + return 0; + } + + data_offset = server->ops->read_data_offset(buf); +#ifdef CONFIG_CIFS_SMB_DIRECT + use_rdma_mr = rdata->mr; +#endif + data_len = server->ops->read_data_length(buf, use_rdma_mr); + + if (data_offset < server->vals->read_rsp_size) { + /* + * win2k8 sometimes sends an offset of 0 when the read + * is beyond the EOF. Treat it as if the data starts just after + * the header. + */ + cifs_dbg(FYI, "%s: data offset (%u) inside read response header\n", + __func__, data_offset); + data_offset = server->vals->read_rsp_size; + } else if (data_offset > MAX_CIFS_SMALL_BUFFER_SIZE) { + /* data_offset is beyond the end of smallbuf */ + cifs_dbg(FYI, "%s: data offset (%u) beyond end of smallbuf\n", + __func__, data_offset); + rdata->result = smb_EIO1(smb_eio_trace_rx_overlong, data_offset); + if (is_offloaded) + mid->mid_state = MID_RESPONSE_MALFORMED; + else + dequeue_mid(server, mid, rdata->result); + return 0; + } + + pad_len = data_offset - server->vals->read_rsp_size; + + if (buf_len <= data_offset) { + /* read response payload is in pages */ + cur_page_idx = pad_len / PAGE_SIZE; + cur_off = pad_len % PAGE_SIZE; + + if (cur_page_idx != 0) { + /* data offset is beyond the 1st page of response */ + cifs_dbg(FYI, "%s: data offset (%u) beyond 1st page of response\n", + __func__, data_offset); + rdata->result = smb_EIO1(smb_eio_trace_rx_overpage, data_offset); + if (is_offloaded) + mid->mid_state = MID_RESPONSE_MALFORMED; + else + dequeue_mid(server, mid, rdata->result); + return 0; + } + + if (data_len > buffer_len - pad_len) { + /* data_len is corrupt -- discard frame */ + rdata->result = smb_EIO1(smb_eio_trace_rx_bad_datalen, data_len); + if (is_offloaded) + mid->mid_state = MID_RESPONSE_MALFORMED; + else + dequeue_mid(server, mid, rdata->result); + return 0; + } + + /* Copy the data to the output I/O iterator. */ + rdata->result = cifs_copy_folioq_to_iter(buffer, data_len, + cur_off, &rdata->subreq.io_iter); + if (rdata->result != 0) { + if (is_offloaded) + mid->mid_state = MID_RESPONSE_MALFORMED; + else + dequeue_mid(server, mid, rdata->result); + return 0; + } + rdata->got_bytes = data_len; + + } else if (!check_add_overflow(data_offset, data_len, &end_off) && + buf_len >= end_off) { + /* read response payload is in buf */ + WARN_ONCE(buffer, "read data can be either in buf or in buffer"); + copied = copy_to_iter(buf + data_offset, data_len, &rdata->subreq.io_iter); + if (copied == 0) + return smb_EIO2(smb_eio_trace_rx_copy_to_iter, copied, data_len); + rdata->got_bytes = copied; + } else { + /* read response payload cannot be in both buf and pages */ + WARN_ONCE(1, "buf can not contain only a part of read data"); + rdata->result = smb_EIO(smb_eio_trace_rx_both_buf); + if (is_offloaded) + mid->mid_state = MID_RESPONSE_MALFORMED; + else + dequeue_mid(server, mid, rdata->result); + return 0; + } + + if (is_offloaded) + mid->mid_state = MID_RESPONSE_RECEIVED; + else + dequeue_mid(server, mid, false); + return 0; +} + +struct smb2_decrypt_work { + struct work_struct decrypt; + struct TCP_Server_Info *server; + struct folio_queue *buffer; + char *buf; + unsigned int len; +}; + + +static void smb2_decrypt_offload(struct work_struct *work) +{ + struct smb2_decrypt_work *dw = container_of(work, + struct smb2_decrypt_work, decrypt); + int rc; + struct mid_q_entry *mid; + struct iov_iter iter; + + iov_iter_folio_queue(&iter, ITER_DEST, dw->buffer, 0, 0, dw->len); + rc = decrypt_raw_data(dw->server, dw->buf, dw->server->vals->read_rsp_size, + &iter, true); + if (rc) { + cifs_dbg(VFS, "error decrypting rc=%d\n", rc); + goto free_pages; + } + + dw->server->lstrp = jiffies; + mid = smb2_find_dequeue_mid(dw->server, dw->buf); + if (mid == NULL) + cifs_dbg(FYI, "mid not found\n"); + else { + mid->decrypted = true; + rc = handle_read_data(dw->server, mid, dw->buf, + dw->server->vals->read_rsp_size, + dw->buffer, dw->len, + true); + if (rc >= 0) { +#ifdef CONFIG_CIFS_STATS2 + mid->when_received = jiffies; +#endif + if (dw->server->ops->is_network_name_deleted) + dw->server->ops->is_network_name_deleted(dw->buf, + dw->server); + + mid_execute_callback(dw->server, mid); + } else { + spin_lock(&dw->server->srv_lock); + if (dw->server->tcpStatus == CifsNeedReconnect) { + spin_lock(&dw->server->mid_queue_lock); + mid->mid_state = MID_RETRY_NEEDED; + spin_unlock(&dw->server->mid_queue_lock); + spin_unlock(&dw->server->srv_lock); + mid_execute_callback(dw->server, mid); + } else { + spin_lock(&dw->server->mid_queue_lock); + mid->mid_state = MID_REQUEST_SUBMITTED; + mid->deleted_from_q = false; + list_add_tail(&mid->qhead, + &dw->server->pending_mid_q); + spin_unlock(&dw->server->mid_queue_lock); + spin_unlock(&dw->server->srv_lock); + } + } + release_mid(dw->server, mid); + } + +free_pages: + netfs_free_folioq_buffer(dw->buffer); + cifs_small_buf_release(dw->buf); + kfree(dw); +} + + +static int +receive_encrypted_read(struct TCP_Server_Info *server, struct mid_q_entry **mid, + int *num_mids) +{ + char *buf = server->smallbuf; + struct smb2_transform_hdr *tr_hdr = (struct smb2_transform_hdr *)buf; + struct iov_iter iter; + unsigned int len; + unsigned int buflen = server->pdu_size; + int rc; + struct smb2_decrypt_work *dw; + + dw = kzalloc_obj(struct smb2_decrypt_work); + if (!dw) + return -ENOMEM; + INIT_WORK(&dw->decrypt, smb2_decrypt_offload); + dw->server = server; + + *num_mids = 1; + len = min_t(unsigned int, buflen, server->vals->read_rsp_size + + sizeof(struct smb2_transform_hdr)) - HEADER_SIZE(server) + 1; + + rc = cifs_read_from_socket(server, buf + HEADER_SIZE(server) - 1, len); + if (rc < 0) + goto free_dw; + server->total_read += rc; + + if (le32_to_cpu(tr_hdr->OriginalMessageSize) < + server->vals->read_rsp_size) { + cifs_server_dbg(VFS, "OriginalMessageSize %u too small for read response (%zu)\n", + le32_to_cpu(tr_hdr->OriginalMessageSize), + server->vals->read_rsp_size); + rc = -EINVAL; + goto discard_data; + } + len = le32_to_cpu(tr_hdr->OriginalMessageSize) - + server->vals->read_rsp_size; + dw->len = len; + len = round_up(dw->len, PAGE_SIZE); + + size_t cur_size = 0; + rc = netfs_alloc_folioq_buffer(NULL, &dw->buffer, &cur_size, len, GFP_NOFS); + if (rc < 0) + goto discard_data; + + iov_iter_folio_queue(&iter, ITER_DEST, dw->buffer, 0, 0, len); + + /* Read the data into the buffer and clear excess bufferage. */ + rc = cifs_read_iter_from_socket(server, &iter, dw->len); + if (rc < 0) + goto discard_data; + + server->total_read += rc; + if (rc < len) { + struct iov_iter tmp = iter; + + iov_iter_advance(&tmp, rc); + iov_iter_zero(len - rc, &tmp); + } + iov_iter_truncate(&iter, dw->len); + + rc = cifs_discard_remaining_data(server); + if (rc) + goto free_pages; + + /* + * For large reads, offload to different thread for better performance, + * use more cores decrypting which can be expensive + */ + + if ((server->min_offload) && (server->in_flight > 1) && + (server->pdu_size >= server->min_offload)) { + dw->buf = server->smallbuf; + server->smallbuf = (char *)cifs_small_buf_get(); + + queue_work(decrypt_wq, &dw->decrypt); + *num_mids = 0; /* worker thread takes care of finding mid */ + return -1; + } + + rc = decrypt_raw_data(server, buf, server->vals->read_rsp_size, + &iter, false); + if (rc) + goto free_pages; + + *mid = smb2_find_mid(server, buf); + if (*mid == NULL) { + cifs_dbg(FYI, "mid not found\n"); + } else { + cifs_dbg(FYI, "mid found\n"); + (*mid)->decrypted = true; + rc = handle_read_data(server, *mid, buf, + server->vals->read_rsp_size, + dw->buffer, dw->len, false); + if (rc >= 0) { + if (server->ops->is_network_name_deleted) { + server->ops->is_network_name_deleted(buf, + server); + } + } + } + +free_pages: + netfs_free_folioq_buffer(dw->buffer); +free_dw: + kfree(dw); + return rc; +discard_data: + cifs_discard_remaining_data(server); + goto free_pages; +} + +static int +receive_encrypted_standard(struct TCP_Server_Info *server, + struct mid_q_entry **mids, char **bufs, + int *num_mids) +{ + int ret, length; + char *buf = server->smallbuf; + struct smb2_hdr *shdr; + unsigned int pdu_length = server->pdu_size; + unsigned int buf_size; + unsigned int next_cmd; + struct mid_q_entry *mid_entry; + int next_is_large; + char *next_buffer = NULL; + + *num_mids = 0; + + /* switch to large buffer if too big for a small one */ + if (pdu_length > MAX_CIFS_SMALL_BUFFER_SIZE) { + server->large_buf = true; + memcpy(server->bigbuf, buf, server->total_read); + buf = server->bigbuf; + } + + /* now read the rest */ + length = cifs_read_from_socket(server, buf + HEADER_SIZE(server) - 1, + pdu_length - HEADER_SIZE(server) + 1); + if (length < 0) + return length; + server->total_read += length; + + buf_size = pdu_length - sizeof(struct smb2_transform_hdr); + length = decrypt_raw_data(server, buf, buf_size, NULL, false); + if (length) + return length; + + next_is_large = server->large_buf; +one_more: + shdr = (struct smb2_hdr *)buf; + next_cmd = le32_to_cpu(shdr->NextCommand); + + if (*num_mids >= MAX_COMPOUND) { + cifs_server_dbg(VFS, "too many PDUs in compound\n"); + return -1; + } + + if (next_cmd) { + if (WARN_ON_ONCE(next_cmd > pdu_length)) + return -1; + if (next_is_large) + next_buffer = (char *)cifs_buf_get(); + else + next_buffer = (char *)cifs_small_buf_get(); + if (!next_buffer) { + cifs_server_dbg(VFS, "No memory for (large) SMB response\n"); + return -1; + } + memcpy(next_buffer, buf + next_cmd, pdu_length - next_cmd); + } + + mid_entry = smb2_find_mid(server, buf); + if (mid_entry == NULL) + cifs_dbg(FYI, "mid not found\n"); + else { + cifs_dbg(FYI, "mid found\n"); + mid_entry->decrypted = true; + mid_entry->resp_buf_size = server->pdu_size; + } + + bufs[*num_mids] = buf; + mids[(*num_mids)++] = mid_entry; + + if (mid_entry && mid_entry->handle) + ret = mid_entry->handle(server, mid_entry); + else + ret = cifs_handle_standard(server, mid_entry); + + if (ret == 0 && next_cmd) { + pdu_length -= next_cmd; + server->large_buf = next_is_large; + if (next_is_large) + server->bigbuf = buf = next_buffer; + else + server->smallbuf = buf = next_buffer; + goto one_more; + } else if (ret != 0) { + /* + * ret != 0 here means that we didn't get to handle_mid() thus + * server->smallbuf and server->bigbuf are still valid. We need + * to free next_buffer because it is not going to be used + * anywhere. + */ + if (next_is_large) + free_rsp_buf(CIFS_LARGE_BUFFER, next_buffer); + else + free_rsp_buf(CIFS_SMALL_BUFFER, next_buffer); + } + + return ret; +} + +static int +smb3_receive_transform(struct TCP_Server_Info *server, + struct mid_q_entry **mids, char **bufs, int *num_mids) +{ + char *buf = server->smallbuf; + unsigned int pdu_length = server->pdu_size; + struct smb2_transform_hdr *tr_hdr = (struct smb2_transform_hdr *)buf; + unsigned int orig_len = le32_to_cpu(tr_hdr->OriginalMessageSize); + + if (pdu_length < sizeof(struct smb2_transform_hdr) + + sizeof(struct smb2_hdr)) { + cifs_server_dbg(VFS, "Transform message is too small (%u)\n", + pdu_length); + cifs_reconnect(server, true); + return -ECONNABORTED; + } + + if (pdu_length < orig_len + sizeof(struct smb2_transform_hdr)) { + cifs_server_dbg(VFS, "Transform message is broken\n"); + cifs_reconnect(server, true); + return -ECONNABORTED; + } + + /* TODO: add support for compounds containing READ. */ + if (pdu_length > CIFSMaxBufSize + MAX_HEADER_SIZE(server)) { + return receive_encrypted_read(server, &mids[0], num_mids); + } + + return receive_encrypted_standard(server, mids, bufs, num_mids); +} + +int +smb3_handle_read_data(struct TCP_Server_Info *server, struct mid_q_entry *mid) +{ + char *buf = server->large_buf ? server->bigbuf : server->smallbuf; + + return handle_read_data(server, mid, buf, server->pdu_size, + NULL, 0, false); +} + +static int smb2_next_header(struct TCP_Server_Info *server, char *buf, + unsigned int *noff) +{ + struct smb2_hdr *hdr = (struct smb2_hdr *)buf; + struct smb2_transform_hdr *t_hdr = (struct smb2_transform_hdr *)buf; + + if (hdr->ProtocolId == SMB2_TRANSFORM_PROTO_NUM) { + *noff = le32_to_cpu(t_hdr->OriginalMessageSize); + if (unlikely(check_add_overflow(*noff, sizeof(*t_hdr), noff))) + return -EINVAL; + } else { + *noff = le32_to_cpu(hdr->NextCommand); + } + if (unlikely(*noff && *noff < MID_HEADER_SIZE(server))) + return -EINVAL; + return 0; +} + +int __cifs_sfu_make_node(unsigned int xid, struct inode *inode, + struct dentry *dentry, struct cifs_tcon *tcon, + const char *full_path, umode_t mode, dev_t dev, + const char *symname) +{ + struct TCP_Server_Info *server = tcon->ses->server; + struct cifs_open_parms oparms; + struct cifs_open_info_data idata = {}; + struct cifs_io_parms io_parms = {}; + struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); + struct cifs_fid fid; + unsigned int bytes_written; + u8 type[8]; + int type_len = 0; + struct { + __le64 major; + __le64 minor; + } __packed pdev = {}; + __le16 *symname_utf16 = NULL; + u8 *data = NULL; + int data_len = 0; + struct kvec iov[3]; + __u32 oplock = server->oplocks ? REQ_OPLOCK : 0; + int rc; + + switch (mode & S_IFMT) { + case S_IFCHR: + type_len = 8; + memcpy(type, "IntxCHR\0", type_len); + pdev.major = cpu_to_le64(MAJOR(dev)); + pdev.minor = cpu_to_le64(MINOR(dev)); + data = (u8 *)&pdev; + data_len = sizeof(pdev); + break; + case S_IFBLK: + type_len = 8; + memcpy(type, "IntxBLK\0", type_len); + pdev.major = cpu_to_le64(MAJOR(dev)); + pdev.minor = cpu_to_le64(MINOR(dev)); + data = (u8 *)&pdev; + data_len = sizeof(pdev); + break; + case S_IFLNK: + type_len = 8; + memcpy(type, "IntxLNK\1", type_len); + symname_utf16 = cifs_strndup_to_utf16(symname, strlen(symname), + &data_len, cifs_sb->local_nls, + NO_MAP_UNI_RSVD); + if (!symname_utf16) { + rc = -ENOMEM; + goto out; + } + data_len -= 2; /* symlink is without trailing wide-nul */ + data = (u8 *)symname_utf16; + break; + case S_IFSOCK: + /* SFU socket is system file with one zero byte */ + type_len = 1; + type[0] = '\0'; + break; + case S_IFIFO: + /* SFU fifo is system file which is empty */ + type_len = 0; + break; + default: + rc = -EPERM; + goto out; + } + + oparms = CIFS_OPARMS(cifs_sb, tcon, full_path, GENERIC_WRITE, + FILE_CREATE, CREATE_NOT_DIR | + CREATE_OPTION_SPECIAL, ACL_NO_MODE); + oparms.fid = &fid; + idata.contains_posix_file_info = false; + rc = server->ops->open(xid, &oparms, &oplock, &idata); + if (rc) + goto out; + + /* + * Check if the server honored ATTR_SYSTEM flag by CREATE_OPTION_SPECIAL + * option. If not then server does not support ATTR_SYSTEM and newly + * created file is not SFU compatible, which means that the call failed. + */ + if (!(le32_to_cpu(idata.fi.Attributes) & ATTR_SYSTEM)) { + rc = -EOPNOTSUPP; + goto out_close; + } + + if (type_len + data_len > 0) { + io_parms.pid = current->tgid; + io_parms.tcon = tcon; + io_parms.length = type_len + data_len; + iov[1].iov_base = type; + iov[1].iov_len = type_len; + iov[2].iov_base = data; + iov[2].iov_len = data_len; + + rc = server->ops->sync_write(xid, &fid, &io_parms, + &bytes_written, + iov, ARRAY_SIZE(iov)-1); + } + +out_close: + server->ops->close(xid, tcon, &fid); + + /* + * If CREATE was successful but either setting ATTR_SYSTEM failed or + * writing type/data information failed then remove the intermediate + * object created by CREATE. Otherwise intermediate empty object stay + * on the server. + */ + if (rc) + server->ops->unlink(xid, tcon, full_path, cifs_sb, NULL); + +out: + kfree(symname_utf16); + return rc; +} + +int cifs_sfu_make_node(unsigned int xid, struct inode *inode, + struct dentry *dentry, struct cifs_tcon *tcon, + const char *full_path, umode_t mode, dev_t dev) +{ + struct inode *new = NULL; + int rc; + + rc = __cifs_sfu_make_node(xid, inode, dentry, tcon, + full_path, mode, dev, NULL); + if (rc) + return rc; + + if (tcon->posix_extensions) { + rc = smb311_posix_get_inode_info(&new, full_path, NULL, + inode->i_sb, xid); + } else if (tcon->unix_ext) { + rc = cifs_get_inode_info_unix(&new, full_path, + inode->i_sb, xid); + } else { + rc = cifs_get_inode_info(&new, full_path, NULL, + inode->i_sb, xid, NULL); + } + if (!rc) + d_instantiate(dentry, new); + return rc; +} + +static int smb2_make_node(unsigned int xid, struct inode *inode, + struct dentry *dentry, struct cifs_tcon *tcon, + const char *full_path, umode_t mode, dev_t dev) +{ + unsigned int sbflags = cifs_sb_flags(CIFS_SB(inode)); + int rc = -EOPNOTSUPP; + + /* + * Check if mounted with mount parm 'sfu' mount parm. + * SFU emulation should work with all servers, but only + * supports block and char device, socket & fifo, + * and was used by default in earlier versions of Windows + */ + if (sbflags & CIFS_MOUNT_UNX_EMUL) { + rc = cifs_sfu_make_node(xid, inode, dentry, tcon, + full_path, mode, dev); + } else if (CIFS_REPARSE_SUPPORT(tcon)) { + rc = mknod_reparse(xid, inode, dentry, tcon, + full_path, mode, dev); + } + return rc; +} + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY +struct smb_version_operations smb20_operations = { + .compare_fids = smb2_compare_fids, + .setup_request = smb2_setup_request, + .setup_async_request = smb2_setup_async_request, + .check_receive = smb2_check_receive, + .add_credits = smb2_add_credits, + .set_credits = smb2_set_credits, + .get_credits_field = smb2_get_credits_field, + .get_credits = smb2_get_credits, + .wait_mtu_credits = cifs_wait_mtu_credits, + .get_next_mid = smb2_get_next_mid, + .revert_current_mid = smb2_revert_current_mid, + .read_data_offset = smb2_read_data_offset, + .read_data_length = smb2_read_data_length, + .map_error = map_smb2_to_linux_error, + .find_mid = smb2_find_mid, + .check_message = smb2_check_message, + .dump_detail = smb2_dump_detail, + .clear_stats = smb2_clear_stats, + .print_stats = smb2_print_stats, + .is_oplock_break = smb2_is_valid_oplock_break, + .handle_cancelled_mid = smb2_handle_cancelled_mid, + .downgrade_oplock = smb2_downgrade_oplock, + .need_neg = smb2_need_neg, + .negotiate = smb2_negotiate, + .negotiate_wsize = smb2_negotiate_wsize, + .negotiate_rsize = smb2_negotiate_rsize, + .sess_setup = SMB2_sess_setup, + .logoff = SMB2_logoff, + .tree_connect = SMB2_tcon, + .tree_disconnect = SMB2_tdis, + .qfs_tcon = smb2_qfs_tcon, + .is_path_accessible = smb2_is_path_accessible, + .can_echo = smb2_can_echo, + .echo = SMB2_echo, + .query_path_info = smb2_query_path_info, + .query_reparse_point = smb2_query_reparse_point, + .get_srv_inum = smb2_get_srv_inum, + .query_file_info = smb2_query_file_info, + .set_path_size = smb2_set_path_size, + .set_file_size = smb2_set_file_size, + .set_file_info = smb2_set_file_info, + .set_compression = smb2_set_compression, + .mkdir = smb2_mkdir, + .mkdir_setinfo = smb2_mkdir_setinfo, + .rmdir = smb2_rmdir, + .unlink = smb2_unlink, + .rename = smb2_rename_path, + .create_hardlink = smb2_create_hardlink, + .get_reparse_point_buffer = smb2_get_reparse_point_buffer, + .query_mf_symlink = smb3_query_mf_symlink, + .create_mf_symlink = smb3_create_mf_symlink, + .create_reparse_inode = smb2_create_reparse_inode, + .open = smb2_open_file, + .set_fid = smb2_set_fid, + .close = smb2_close_file, + .flush = smb2_flush_file, + .async_readv = smb2_async_readv, + .async_writev = smb2_async_writev, + .sync_read = smb2_sync_read, + .sync_write = smb2_sync_write, + .query_dir_first = smb2_query_dir_first, + .query_dir_next = smb2_query_dir_next, + .close_dir = smb2_close_dir, + .calc_smb_size = smb2_calc_size, + .is_status_pending = smb2_is_status_pending, + .is_session_expired = smb2_is_session_expired, + .oplock_response = smb2_oplock_response, + .queryfs = smb2_queryfs, + .mand_lock = smb2_mand_lock, + .mand_unlock_range = smb2_unlock_range, + .push_mand_locks = smb2_push_mandatory_locks, + .get_lease_key = smb2_get_lease_key, + .set_lease_key = smb2_set_lease_key, + .new_lease_key = smb2_new_lease_key, + .is_read_op = smb2_is_read_op, + .set_oplock_level = smb2_set_oplock_level, + .create_lease_buf = smb2_create_lease_buf, + .parse_lease_buf = smb2_parse_lease_buf, + .copychunk_range = smb2_copychunk_range, + .wp_retry_size = smb2_wp_retry_size, + .dir_needs_close = smb2_dir_needs_close, + .get_dfs_refer = smb2_get_dfs_refer, + .select_sectype = smb2_select_sectype, +#ifdef CONFIG_CIFS_XATTR + .query_all_EAs = smb2_query_eas, + .set_EA = smb2_set_ea, +#endif /* CIFS_XATTR */ + .get_acl = get_smb2_acl, + .get_acl_by_fid = get_smb2_acl_by_fid, + .set_acl = set_smb2_acl, + .next_header = smb2_next_header, + .ioctl_query_info = smb2_ioctl_query_info, + .make_node = smb2_make_node, + .fiemap = smb3_fiemap, + .llseek = smb3_llseek, + .is_status_io_timeout = smb2_is_status_io_timeout, + .is_network_name_deleted = smb2_is_network_name_deleted, + .rename_pending_delete = smb2_rename_pending_delete, +}; +#endif /* CIFS_ALLOW_INSECURE_LEGACY */ + +struct smb_version_operations smb21_operations = { + .compare_fids = smb2_compare_fids, + .setup_request = smb2_setup_request, + .setup_async_request = smb2_setup_async_request, + .check_receive = smb2_check_receive, + .add_credits = smb2_add_credits, + .set_credits = smb2_set_credits, + .get_credits_field = smb2_get_credits_field, + .get_credits = smb2_get_credits, + .wait_mtu_credits = smb2_wait_mtu_credits, + .adjust_credits = smb2_adjust_credits, + .get_next_mid = smb2_get_next_mid, + .revert_current_mid = smb2_revert_current_mid, + .read_data_offset = smb2_read_data_offset, + .read_data_length = smb2_read_data_length, + .map_error = map_smb2_to_linux_error, + .find_mid = smb2_find_mid, + .check_message = smb2_check_message, + .dump_detail = smb2_dump_detail, + .clear_stats = smb2_clear_stats, + .print_stats = smb2_print_stats, + .is_oplock_break = smb2_is_valid_oplock_break, + .handle_cancelled_mid = smb2_handle_cancelled_mid, + .downgrade_oplock = smb2_downgrade_oplock, + .need_neg = smb2_need_neg, + .negotiate = smb2_negotiate, + .negotiate_wsize = smb2_negotiate_wsize, + .negotiate_rsize = smb2_negotiate_rsize, + .sess_setup = SMB2_sess_setup, + .logoff = SMB2_logoff, + .tree_connect = SMB2_tcon, + .tree_disconnect = SMB2_tdis, + .qfs_tcon = smb2_qfs_tcon, + .is_path_accessible = smb2_is_path_accessible, + .can_echo = smb2_can_echo, + .echo = SMB2_echo, + .query_path_info = smb2_query_path_info, + .query_reparse_point = smb2_query_reparse_point, + .get_srv_inum = smb2_get_srv_inum, + .query_file_info = smb2_query_file_info, + .set_path_size = smb2_set_path_size, + .set_file_size = smb2_set_file_size, + .set_file_info = smb2_set_file_info, + .set_compression = smb2_set_compression, + .mkdir = smb2_mkdir, + .mkdir_setinfo = smb2_mkdir_setinfo, + .rmdir = smb2_rmdir, + .unlink = smb2_unlink, + .rename = smb2_rename_path, + .create_hardlink = smb2_create_hardlink, + .get_reparse_point_buffer = smb2_get_reparse_point_buffer, + .query_mf_symlink = smb3_query_mf_symlink, + .create_mf_symlink = smb3_create_mf_symlink, + .create_reparse_inode = smb2_create_reparse_inode, + .open = smb2_open_file, + .set_fid = smb2_set_fid, + .close = smb2_close_file, + .flush = smb2_flush_file, + .async_readv = smb2_async_readv, + .async_writev = smb2_async_writev, + .sync_read = smb2_sync_read, + .sync_write = smb2_sync_write, + .query_dir_first = smb2_query_dir_first, + .query_dir_next = smb2_query_dir_next, + .close_dir = smb2_close_dir, + .calc_smb_size = smb2_calc_size, + .is_status_pending = smb2_is_status_pending, + .is_session_expired = smb2_is_session_expired, + .oplock_response = smb2_oplock_response, + .queryfs = smb2_queryfs, + .mand_lock = smb2_mand_lock, + .mand_unlock_range = smb2_unlock_range, + .push_mand_locks = smb2_push_mandatory_locks, + .get_lease_key = smb2_get_lease_key, + .set_lease_key = smb2_set_lease_key, + .new_lease_key = smb2_new_lease_key, + .is_read_op = smb21_is_read_op, + .set_oplock_level = smb21_set_oplock_level, + .create_lease_buf = smb2_create_lease_buf, + .parse_lease_buf = smb2_parse_lease_buf, + .copychunk_range = smb2_copychunk_range, + .wp_retry_size = smb2_wp_retry_size, + .dir_needs_close = smb2_dir_needs_close, + .enum_snapshots = smb3_enum_snapshots, + .notify = smb3_notify, + .get_dfs_refer = smb2_get_dfs_refer, + .select_sectype = smb2_select_sectype, +#ifdef CONFIG_CIFS_XATTR + .query_all_EAs = smb2_query_eas, + .set_EA = smb2_set_ea, +#endif /* CIFS_XATTR */ + .get_acl = get_smb2_acl, + .get_acl_by_fid = get_smb2_acl_by_fid, + .set_acl = set_smb2_acl, + .next_header = smb2_next_header, + .ioctl_query_info = smb2_ioctl_query_info, + .make_node = smb2_make_node, + .fiemap = smb3_fiemap, + .llseek = smb3_llseek, + .is_status_io_timeout = smb2_is_status_io_timeout, + .is_network_name_deleted = smb2_is_network_name_deleted, + .rename_pending_delete = smb2_rename_pending_delete, +}; + +struct smb_version_operations smb30_operations = { + .compare_fids = smb2_compare_fids, + .setup_request = smb2_setup_request, + .setup_async_request = smb2_setup_async_request, + .check_receive = smb2_check_receive, + .add_credits = smb2_add_credits, + .set_credits = smb2_set_credits, + .get_credits_field = smb2_get_credits_field, + .get_credits = smb2_get_credits, + .wait_mtu_credits = smb2_wait_mtu_credits, + .adjust_credits = smb2_adjust_credits, + .get_next_mid = smb2_get_next_mid, + .revert_current_mid = smb2_revert_current_mid, + .read_data_offset = smb2_read_data_offset, + .read_data_length = smb2_read_data_length, + .map_error = map_smb2_to_linux_error, + .find_mid = smb2_find_mid, + .check_message = smb2_check_message, + .dump_detail = smb2_dump_detail, + .clear_stats = smb2_clear_stats, + .print_stats = smb2_print_stats, + .dump_share_caps = smb2_dump_share_caps, + .is_oplock_break = smb2_is_valid_oplock_break, + .handle_cancelled_mid = smb2_handle_cancelled_mid, + .downgrade_oplock = smb3_downgrade_oplock, + .need_neg = smb2_need_neg, + .negotiate = smb2_negotiate, + .negotiate_wsize = smb3_negotiate_wsize, + .negotiate_rsize = smb3_negotiate_rsize, + .sess_setup = SMB2_sess_setup, + .logoff = SMB2_logoff, + .tree_connect = SMB2_tcon, + .tree_disconnect = SMB2_tdis, + .qfs_tcon = smb3_qfs_tcon, + .query_server_interfaces = SMB3_request_interfaces, + .is_path_accessible = smb2_is_path_accessible, + .can_echo = smb2_can_echo, + .echo = SMB2_echo, + .query_path_info = smb2_query_path_info, + /* WSL tags introduced long after smb2.1, enable for SMB3, 3.11 only */ + .query_reparse_point = smb2_query_reparse_point, + .get_srv_inum = smb2_get_srv_inum, + .query_file_info = smb2_query_file_info, + .set_path_size = smb2_set_path_size, + .set_file_size = smb2_set_file_size, + .set_file_info = smb2_set_file_info, + .set_compression = smb2_set_compression, + .mkdir = smb2_mkdir, + .mkdir_setinfo = smb2_mkdir_setinfo, + .rmdir = smb2_rmdir, + .unlink = smb2_unlink, + .rename = smb2_rename_path, + .create_hardlink = smb2_create_hardlink, + .get_reparse_point_buffer = smb2_get_reparse_point_buffer, + .query_mf_symlink = smb3_query_mf_symlink, + .create_mf_symlink = smb3_create_mf_symlink, + .create_reparse_inode = smb2_create_reparse_inode, + .open = smb2_open_file, + .set_fid = smb2_set_fid, + .close = smb2_close_file, + .close_getattr = smb2_close_getattr, + .flush = smb2_flush_file, + .async_readv = smb2_async_readv, + .async_writev = smb2_async_writev, + .sync_read = smb2_sync_read, + .sync_write = smb2_sync_write, + .query_dir_first = smb2_query_dir_first, + .query_dir_next = smb2_query_dir_next, + .close_dir = smb2_close_dir, + .calc_smb_size = smb2_calc_size, + .is_status_pending = smb2_is_status_pending, + .is_session_expired = smb2_is_session_expired, + .oplock_response = smb2_oplock_response, + .queryfs = smb2_queryfs, + .mand_lock = smb2_mand_lock, + .mand_unlock_range = smb2_unlock_range, + .push_mand_locks = smb2_push_mandatory_locks, + .get_lease_key = smb2_get_lease_key, + .set_lease_key = smb2_set_lease_key, + .new_lease_key = smb2_new_lease_key, + .generate_signingkey = generate_smb30signingkey, + .set_integrity = smb3_set_integrity, + .is_read_op = smb21_is_read_op, + .set_oplock_level = smb3_set_oplock_level, + .create_lease_buf = smb3_create_lease_buf, + .parse_lease_buf = smb3_parse_lease_buf, + .copychunk_range = smb2_copychunk_range, + .duplicate_extents = smb2_duplicate_extents, + .validate_negotiate = smb3_validate_negotiate, + .wp_retry_size = smb2_wp_retry_size, + .dir_needs_close = smb2_dir_needs_close, + .fallocate = smb3_fallocate, + .enum_snapshots = smb3_enum_snapshots, + .notify = smb3_notify, + .init_transform_rq = smb3_init_transform_rq, + .is_transform_hdr = smb3_is_transform_hdr, + .receive_transform = smb3_receive_transform, + .get_dfs_refer = smb2_get_dfs_refer, + .select_sectype = smb2_select_sectype, +#ifdef CONFIG_CIFS_XATTR + .query_all_EAs = smb2_query_eas, + .set_EA = smb2_set_ea, +#endif /* CIFS_XATTR */ + .get_acl = get_smb2_acl, + .get_acl_by_fid = get_smb2_acl_by_fid, + .set_acl = set_smb2_acl, + .next_header = smb2_next_header, + .ioctl_query_info = smb2_ioctl_query_info, + .make_node = smb2_make_node, + .fiemap = smb3_fiemap, + .llseek = smb3_llseek, + .is_status_io_timeout = smb2_is_status_io_timeout, + .is_network_name_deleted = smb2_is_network_name_deleted, + .rename_pending_delete = smb2_rename_pending_delete, +}; + +struct smb_version_operations smb311_operations = { + .compare_fids = smb2_compare_fids, + .setup_request = smb2_setup_request, + .setup_async_request = smb2_setup_async_request, + .check_receive = smb2_check_receive, + .add_credits = smb2_add_credits, + .set_credits = smb2_set_credits, + .get_credits_field = smb2_get_credits_field, + .get_credits = smb2_get_credits, + .wait_mtu_credits = smb2_wait_mtu_credits, + .adjust_credits = smb2_adjust_credits, + .get_next_mid = smb2_get_next_mid, + .revert_current_mid = smb2_revert_current_mid, + .read_data_offset = smb2_read_data_offset, + .read_data_length = smb2_read_data_length, + .map_error = map_smb2_to_linux_error, + .find_mid = smb2_find_mid, + .check_message = smb2_check_message, + .dump_detail = smb2_dump_detail, + .clear_stats = smb2_clear_stats, + .print_stats = smb2_print_stats, + .dump_share_caps = smb2_dump_share_caps, + .is_oplock_break = smb2_is_valid_oplock_break, + .handle_cancelled_mid = smb2_handle_cancelled_mid, + .downgrade_oplock = smb3_downgrade_oplock, + .need_neg = smb2_need_neg, + .negotiate = smb2_negotiate, + .negotiate_wsize = smb3_negotiate_wsize, + .negotiate_rsize = smb3_negotiate_rsize, + .sess_setup = SMB2_sess_setup, + .logoff = SMB2_logoff, + .tree_connect = SMB2_tcon, + .tree_disconnect = SMB2_tdis, + .qfs_tcon = smb3_qfs_tcon, + .query_server_interfaces = SMB3_request_interfaces, + .is_path_accessible = smb2_is_path_accessible, + .can_echo = smb2_can_echo, + .echo = SMB2_echo, + .query_path_info = smb2_query_path_info, + .query_reparse_point = smb2_query_reparse_point, + .get_srv_inum = smb2_get_srv_inum, + .query_file_info = smb2_query_file_info, + .set_path_size = smb2_set_path_size, + .set_file_size = smb2_set_file_size, + .set_file_info = smb2_set_file_info, + .set_compression = smb2_set_compression, + .mkdir = smb2_mkdir, + .mkdir_setinfo = smb2_mkdir_setinfo, + .posix_mkdir = smb311_posix_mkdir, + .rmdir = smb2_rmdir, + .unlink = smb2_unlink, + .rename = smb2_rename_path, + .create_hardlink = smb2_create_hardlink, + .get_reparse_point_buffer = smb2_get_reparse_point_buffer, + .query_mf_symlink = smb3_query_mf_symlink, + .create_mf_symlink = smb3_create_mf_symlink, + .create_reparse_inode = smb2_create_reparse_inode, + .open = smb2_open_file, + .set_fid = smb2_set_fid, + .close = smb2_close_file, + .close_getattr = smb2_close_getattr, + .flush = smb2_flush_file, + .async_readv = smb2_async_readv, + .async_writev = smb2_async_writev, + .sync_read = smb2_sync_read, + .sync_write = smb2_sync_write, + .query_dir_first = smb2_query_dir_first, + .query_dir_next = smb2_query_dir_next, + .close_dir = smb2_close_dir, + .calc_smb_size = smb2_calc_size, + .is_status_pending = smb2_is_status_pending, + .is_session_expired = smb2_is_session_expired, + .oplock_response = smb2_oplock_response, + .queryfs = smb311_queryfs, + .mand_lock = smb2_mand_lock, + .mand_unlock_range = smb2_unlock_range, + .push_mand_locks = smb2_push_mandatory_locks, + .get_lease_key = smb2_get_lease_key, + .set_lease_key = smb2_set_lease_key, + .new_lease_key = smb2_new_lease_key, + .generate_signingkey = generate_smb311signingkey, + .set_integrity = smb3_set_integrity, + .is_read_op = smb21_is_read_op, + .set_oplock_level = smb3_set_oplock_level, + .create_lease_buf = smb3_create_lease_buf, + .parse_lease_buf = smb3_parse_lease_buf, + .copychunk_range = smb2_copychunk_range, + .duplicate_extents = smb2_duplicate_extents, +/* .validate_negotiate = smb3_validate_negotiate, */ /* not used in 3.11 */ + .wp_retry_size = smb2_wp_retry_size, + .dir_needs_close = smb2_dir_needs_close, + .fallocate = smb3_fallocate, + .enum_snapshots = smb3_enum_snapshots, + .notify = smb3_notify, + .init_transform_rq = smb3_init_transform_rq, + .is_transform_hdr = smb3_is_transform_hdr, + .receive_transform = smb3_receive_transform, + .get_dfs_refer = smb2_get_dfs_refer, + .select_sectype = smb2_select_sectype, +#ifdef CONFIG_CIFS_XATTR + .query_all_EAs = smb2_query_eas, + .set_EA = smb2_set_ea, +#endif /* CIFS_XATTR */ + .get_acl = get_smb2_acl, + .get_acl_by_fid = get_smb2_acl_by_fid, + .set_acl = set_smb2_acl, + .next_header = smb2_next_header, + .ioctl_query_info = smb2_ioctl_query_info, + .make_node = smb2_make_node, + .fiemap = smb3_fiemap, + .llseek = smb3_llseek, + .is_status_io_timeout = smb2_is_status_io_timeout, + .is_network_name_deleted = smb2_is_network_name_deleted, + .rename_pending_delete = smb2_rename_pending_delete, +}; + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY +struct smb_version_values smb20_values = { + .version_string = SMB20_VERSION_STRING, + .protocol_id = SMB20_PROT_ID, + .req_capabilities = 0, /* MBZ */ + .large_lock_type = 0, + .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE, + .shared_lock_type = SMB2_LOCKFLAG_SHARED, + .unlock_lock_type = SMB2_LOCKFLAG_UNLOCK, + .header_size = sizeof(struct smb2_hdr), + .max_header_size = MAX_SMB2_HDR_SIZE, + .read_rsp_size = sizeof(struct smb2_read_rsp), + .lock_cmd = SMB2_LOCK, + .cap_unix = 0, + .cap_nt_find = SMB2_NT_FIND, + .cap_large_files = SMB2_LARGE_FILES, + .signing_enabled = SMB2_NEGOTIATE_SIGNING_ENABLED | SMB2_NEGOTIATE_SIGNING_REQUIRED, + .signing_required = SMB2_NEGOTIATE_SIGNING_REQUIRED, + .create_lease_size = sizeof(struct create_lease), +}; +#endif /* ALLOW_INSECURE_LEGACY */ + +struct smb_version_values smb21_values = { + .version_string = SMB21_VERSION_STRING, + .protocol_id = SMB21_PROT_ID, + .req_capabilities = 0, /* MBZ on negotiate req until SMB3 dialect */ + .large_lock_type = 0, + .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE, + .shared_lock_type = SMB2_LOCKFLAG_SHARED, + .unlock_lock_type = SMB2_LOCKFLAG_UNLOCK, + .header_size = sizeof(struct smb2_hdr), + .max_header_size = MAX_SMB2_HDR_SIZE, + .read_rsp_size = sizeof(struct smb2_read_rsp), + .lock_cmd = SMB2_LOCK, + .cap_unix = 0, + .cap_nt_find = SMB2_NT_FIND, + .cap_large_files = SMB2_LARGE_FILES, + .signing_enabled = SMB2_NEGOTIATE_SIGNING_ENABLED | SMB2_NEGOTIATE_SIGNING_REQUIRED, + .signing_required = SMB2_NEGOTIATE_SIGNING_REQUIRED, + .create_lease_size = sizeof(struct create_lease), +}; + +struct smb_version_values smb3any_values = { + .version_string = SMB3ANY_VERSION_STRING, + .protocol_id = SMB302_PROT_ID, /* doesn't matter, send protocol array */ + .req_capabilities = SMB2_GLOBAL_CAP_DFS | SMB2_GLOBAL_CAP_LEASING | SMB2_GLOBAL_CAP_LARGE_MTU | SMB2_GLOBAL_CAP_PERSISTENT_HANDLES | SMB2_GLOBAL_CAP_ENCRYPTION | SMB2_GLOBAL_CAP_DIRECTORY_LEASING, + .large_lock_type = 0, + .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE, + .shared_lock_type = SMB2_LOCKFLAG_SHARED, + .unlock_lock_type = SMB2_LOCKFLAG_UNLOCK, + .header_size = sizeof(struct smb2_hdr), + .max_header_size = MAX_SMB2_HDR_SIZE, + .read_rsp_size = sizeof(struct smb2_read_rsp), + .lock_cmd = SMB2_LOCK, + .cap_unix = 0, + .cap_nt_find = SMB2_NT_FIND, + .cap_large_files = SMB2_LARGE_FILES, + .signing_enabled = SMB2_NEGOTIATE_SIGNING_ENABLED | SMB2_NEGOTIATE_SIGNING_REQUIRED, + .signing_required = SMB2_NEGOTIATE_SIGNING_REQUIRED, + .create_lease_size = sizeof(struct create_lease_v2), +}; + +struct smb_version_values smbdefault_values = { + .version_string = SMBDEFAULT_VERSION_STRING, + .protocol_id = SMB302_PROT_ID, /* doesn't matter, send protocol array */ + .req_capabilities = SMB2_GLOBAL_CAP_DFS | SMB2_GLOBAL_CAP_LEASING | SMB2_GLOBAL_CAP_LARGE_MTU | SMB2_GLOBAL_CAP_PERSISTENT_HANDLES | SMB2_GLOBAL_CAP_ENCRYPTION | SMB2_GLOBAL_CAP_DIRECTORY_LEASING, + .large_lock_type = 0, + .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE, + .shared_lock_type = SMB2_LOCKFLAG_SHARED, + .unlock_lock_type = SMB2_LOCKFLAG_UNLOCK, + .header_size = sizeof(struct smb2_hdr), + .max_header_size = MAX_SMB2_HDR_SIZE, + .read_rsp_size = sizeof(struct smb2_read_rsp), + .lock_cmd = SMB2_LOCK, + .cap_unix = 0, + .cap_nt_find = SMB2_NT_FIND, + .cap_large_files = SMB2_LARGE_FILES, + .signing_enabled = SMB2_NEGOTIATE_SIGNING_ENABLED | SMB2_NEGOTIATE_SIGNING_REQUIRED, + .signing_required = SMB2_NEGOTIATE_SIGNING_REQUIRED, + .create_lease_size = sizeof(struct create_lease_v2), +}; + +struct smb_version_values smb30_values = { + .version_string = SMB30_VERSION_STRING, + .protocol_id = SMB30_PROT_ID, + .req_capabilities = SMB2_GLOBAL_CAP_DFS | SMB2_GLOBAL_CAP_LEASING | SMB2_GLOBAL_CAP_LARGE_MTU | SMB2_GLOBAL_CAP_PERSISTENT_HANDLES | SMB2_GLOBAL_CAP_ENCRYPTION | SMB2_GLOBAL_CAP_DIRECTORY_LEASING, + .large_lock_type = 0, + .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE, + .shared_lock_type = SMB2_LOCKFLAG_SHARED, + .unlock_lock_type = SMB2_LOCKFLAG_UNLOCK, + .header_size = sizeof(struct smb2_hdr), + .max_header_size = MAX_SMB2_HDR_SIZE, + .read_rsp_size = sizeof(struct smb2_read_rsp), + .lock_cmd = SMB2_LOCK, + .cap_unix = 0, + .cap_nt_find = SMB2_NT_FIND, + .cap_large_files = SMB2_LARGE_FILES, + .signing_enabled = SMB2_NEGOTIATE_SIGNING_ENABLED | SMB2_NEGOTIATE_SIGNING_REQUIRED, + .signing_required = SMB2_NEGOTIATE_SIGNING_REQUIRED, + .create_lease_size = sizeof(struct create_lease_v2), +}; + +struct smb_version_values smb302_values = { + .version_string = SMB302_VERSION_STRING, + .protocol_id = SMB302_PROT_ID, + .req_capabilities = SMB2_GLOBAL_CAP_DFS | SMB2_GLOBAL_CAP_LEASING | SMB2_GLOBAL_CAP_LARGE_MTU | SMB2_GLOBAL_CAP_PERSISTENT_HANDLES | SMB2_GLOBAL_CAP_ENCRYPTION | SMB2_GLOBAL_CAP_DIRECTORY_LEASING, + .large_lock_type = 0, + .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE, + .shared_lock_type = SMB2_LOCKFLAG_SHARED, + .unlock_lock_type = SMB2_LOCKFLAG_UNLOCK, + .header_size = sizeof(struct smb2_hdr), + .max_header_size = MAX_SMB2_HDR_SIZE, + .read_rsp_size = sizeof(struct smb2_read_rsp), + .lock_cmd = SMB2_LOCK, + .cap_unix = 0, + .cap_nt_find = SMB2_NT_FIND, + .cap_large_files = SMB2_LARGE_FILES, + .signing_enabled = SMB2_NEGOTIATE_SIGNING_ENABLED | SMB2_NEGOTIATE_SIGNING_REQUIRED, + .signing_required = SMB2_NEGOTIATE_SIGNING_REQUIRED, + .create_lease_size = sizeof(struct create_lease_v2), +}; + +struct smb_version_values smb311_values = { + .version_string = SMB311_VERSION_STRING, + .protocol_id = SMB311_PROT_ID, + .req_capabilities = SMB2_GLOBAL_CAP_DFS | SMB2_GLOBAL_CAP_LEASING | SMB2_GLOBAL_CAP_LARGE_MTU | SMB2_GLOBAL_CAP_PERSISTENT_HANDLES | SMB2_GLOBAL_CAP_ENCRYPTION | SMB2_GLOBAL_CAP_DIRECTORY_LEASING, + .large_lock_type = 0, + .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE, + .shared_lock_type = SMB2_LOCKFLAG_SHARED, + .unlock_lock_type = SMB2_LOCKFLAG_UNLOCK, + .header_size = sizeof(struct smb2_hdr), + .max_header_size = MAX_SMB2_HDR_SIZE, + .read_rsp_size = sizeof(struct smb2_read_rsp), + .lock_cmd = SMB2_LOCK, + .cap_unix = 0, + .cap_nt_find = SMB2_NT_FIND, + .cap_large_files = SMB2_LARGE_FILES, + .signing_enabled = SMB2_NEGOTIATE_SIGNING_ENABLED | SMB2_NEGOTIATE_SIGNING_REQUIRED, + .signing_required = SMB2_NEGOTIATE_SIGNING_REQUIRED, + .create_lease_size = sizeof(struct create_lease_v2), +}; diff --git a/fs/smb/client/smb2pdu.c b/fs/smb/client/smb2pdu.c new file mode 100644 index 000000000..4ce165e40 --- /dev/null +++ b/fs/smb/client/smb2pdu.c @@ -0,0 +1,6465 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * Copyright (C) International Business Machines Corp., 2009, 2013 + * Etersoft, 2012 + * Author(s): Steve French (sfrench@us.ibm.com) + * Pavel Shilovsky (pshilovsky@samba.org) 2012 + * + * Contains the routines for constructing the SMB2 PDUs themselves + * + */ + + /* SMB2 PDU handling routines here - except for leftovers (eg session setup) */ + /* Note that there are handle based routines which must be */ + /* treated slightly differently for reconnection purposes since we never */ + /* want to reuse a stale file handle and only the caller knows the file info */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "cifsglob.h" +#include "cifsproto.h" +#include "cifsacl.h" +#include "smb2proto.h" +#include "cifs_unicode.h" +#include "cifs_debug.h" +#include "ntlmssp.h" +#include "../common/smbfsctl.h" +#include "../common/smb2status.h" +#include "smb2glob.h" +#include "cifs_spnego.h" +#include "smbdirect.h" +#include "trace.h" +#ifdef CONFIG_CIFS_DFS_UPCALL +#include "dfs_cache.h" +#endif +#include "cached_dir.h" +#include "compress.h" +#include "fs_context.h" + +/* + * The following table defines the expected "StructureSize" of SMB2 requests + * in order by SMB2 command. This is similar to "wct" in SMB/CIFS requests. + * + * Note that commands are defined in smb2pdu.h in le16 but the array below is + * indexed by command in host byte order. + */ +static const int smb2_req_struct_sizes[NUMBER_OF_SMB2_COMMANDS] = { + /* SMB2_NEGOTIATE */ 36, + /* SMB2_SESSION_SETUP */ 25, + /* SMB2_LOGOFF */ 4, + /* SMB2_TREE_CONNECT */ 9, + /* SMB2_TREE_DISCONNECT */ 4, + /* SMB2_CREATE */ 57, + /* SMB2_CLOSE */ 24, + /* SMB2_FLUSH */ 24, + /* SMB2_READ */ 49, + /* SMB2_WRITE */ 49, + /* SMB2_LOCK */ 48, + /* SMB2_IOCTL */ 57, + /* SMB2_CANCEL */ 4, + /* SMB2_ECHO */ 4, + /* SMB2_QUERY_DIRECTORY */ 33, + /* SMB2_CHANGE_NOTIFY */ 32, + /* SMB2_QUERY_INFO */ 41, + /* SMB2_SET_INFO */ 33, + /* SMB2_OPLOCK_BREAK */ 24 /* BB this is 36 for LEASE_BREAK variant */ +}; + +int smb3_encryption_required(const struct cifs_tcon *tcon) +{ + if (!tcon || !tcon->ses) + return 0; + if ((tcon->ses->session_flags & SMB2_SESSION_FLAG_ENCRYPT_DATA) || + (tcon->share_flags & SHI1005_FLAGS_ENCRYPT_DATA)) + return 1; + if (tcon->seal && + (tcon->ses->server->capabilities & SMB2_GLOBAL_CAP_ENCRYPTION)) + return 1; + if (((global_secflags & CIFSSEC_MUST_SEAL) == CIFSSEC_MUST_SEAL) && + (tcon->ses->server->capabilities & SMB2_GLOBAL_CAP_ENCRYPTION)) + return 1; + return 0; +} + +static void +smb2_hdr_assemble(struct smb2_hdr *shdr, __le16 smb2_cmd, + const struct cifs_tcon *tcon, + struct TCP_Server_Info *server) +{ + struct smb3_hdr_req *smb3_hdr; + + shdr->ProtocolId = SMB2_PROTO_NUMBER; + shdr->StructureSize = cpu_to_le16(64); + shdr->Command = smb2_cmd; + + if (server) { + /* After reconnect SMB3 must set ChannelSequence on subsequent reqs */ + if (server->dialect >= SMB30_PROT_ID) { + smb3_hdr = (struct smb3_hdr_req *)shdr; + /* + * if primary channel is not set yet, use default + * channel for chan sequence num + */ + if (SERVER_IS_CHAN(server)) + smb3_hdr->ChannelSequence = + cpu_to_le16(server->primary_server->channel_sequence_num); + else + smb3_hdr->ChannelSequence = + cpu_to_le16(server->channel_sequence_num); + } + spin_lock(&server->req_lock); + /* Request up to 10 credits but don't go over the limit. */ + if (server->credits >= server->max_credits) + shdr->CreditRequest = cpu_to_le16(0); + else + shdr->CreditRequest = cpu_to_le16( + min_t(int, server->max_credits - + server->credits, 10)); + spin_unlock(&server->req_lock); + } else { + shdr->CreditRequest = cpu_to_le16(2); + } + shdr->Id.SyncId.ProcessId = cpu_to_le32((__u16)current->tgid); + + if (!tcon) + goto out; + + /* GLOBAL_CAP_LARGE_MTU will only be set if dialect > SMB2.02 */ + /* See sections 2.2.4 and 3.2.4.1.5 of MS-SMB2 */ + if (server && (server->capabilities & SMB2_GLOBAL_CAP_LARGE_MTU)) + shdr->CreditCharge = cpu_to_le16(1); + /* else CreditCharge MBZ */ + + shdr->Id.SyncId.TreeId = cpu_to_le32(tcon->tid); + /* Uid is not converted */ + if (tcon->ses) + shdr->SessionId = cpu_to_le64(tcon->ses->Suid); + + /* + * If we would set SMB2_FLAGS_DFS_OPERATIONS on open we also would have + * to pass the path on the Open SMB prefixed by \\server\share. + * Not sure when we would need to do the augmented path (if ever) and + * setting this flag breaks the SMB2 open operation since it is + * illegal to send an empty path name (without \\server\share prefix) + * when the DFS flag is set in the SMB open header. We could + * consider setting the flag on all operations other than open + * but it is safer to net set it for now. + */ +/* if (tcon->share_flags & SHI1005_FLAGS_DFS) + shdr->Flags |= SMB2_FLAGS_DFS_OPERATIONS; */ + + if (server && server->sign && !smb3_encryption_required(tcon)) + shdr->Flags |= SMB2_FLAGS_SIGNED; +out: + return; +} + +/* helper function for code reuse */ +static int +cifs_chan_skip_or_disable(struct cifs_ses *ses, + struct TCP_Server_Info *server, + bool from_reconnect, bool disable_mchan) +{ + struct TCP_Server_Info *pserver; + unsigned int chan_index; + + if (SERVER_IS_CHAN(server)) { + cifs_dbg(VFS, + "server %s does not support multichannel anymore. Skip secondary channel\n", + ses->server->hostname); + + spin_lock(&ses->chan_lock); + chan_index = cifs_ses_get_chan_index(ses, server); + if (chan_index == CIFS_INVAL_CHAN_INDEX) { + spin_unlock(&ses->chan_lock); + goto skip_terminate; + } + + ses->chans[chan_index].server = NULL; + server->terminate = true; + spin_unlock(&ses->chan_lock); + + /* + * the above reference of server by channel + * needs to be dropped without holding chan_lock + * as cifs_put_tcp_session takes a higher lock + * i.e. cifs_tcp_ses_lock + */ + cifs_put_tcp_session(server, from_reconnect); + + cifs_signal_cifsd_for_reconnect(server, false); + + /* mark primary server as needing reconnect */ + pserver = server->primary_server; + cifs_signal_cifsd_for_reconnect(pserver, false); +skip_terminate: + return -EHOSTDOWN; + } + + cifs_decrease_secondary_channels(ses, disable_mchan); + + return 0; +} + +/* + * smb3_update_ses_channels - Synchronize session channels with new configuration + * @ses: pointer to the CIFS session structure + * @server: pointer to the TCP server info structure + * @from_reconnect: indicates if called from reconnect context + * @disable_mchan: indicates if called from reconnect to disable multichannel + * + * Returns 0 on success or error code on failure. + * + * Outside of reconfigure, this function is called from cifs_mount() during mount + * and from reconnect scenarios to adjust channel count when the + * server's multichannel support changes. + */ +int smb3_update_ses_channels(struct cifs_ses *ses, struct TCP_Server_Info *server, + bool from_reconnect, bool disable_mchan) +{ + int rc = 0; + /* + * Manage session channels based on current count vs max: + * - If disable requested, skip or disable the channel + * - If below max channels, attempt to add more + * - If above max channels, skip or disable excess channels + */ + if (disable_mchan) + rc = cifs_chan_skip_or_disable(ses, server, from_reconnect, disable_mchan); + else { + if (ses->chan_count < ses->chan_max) + rc = cifs_try_adding_channels(ses); + else if (ses->chan_count > ses->chan_max) + rc = cifs_chan_skip_or_disable(ses, server, from_reconnect, disable_mchan); + } + + return rc; +} + +static int +smb2_reconnect(__le16 smb2_command, struct cifs_tcon *tcon, + struct TCP_Server_Info *server, bool from_reconnect) +{ + struct cifs_ses *ses; + int xid; + int rc = 0; + + /* + * SMB2s NegProt, SessSetup, Logoff do not have tcon yet so + * check for tcp and smb session status done differently + * for those three - in the calling routine. + */ + if (tcon == NULL) + return 0; + + if (smb2_command == SMB2_TREE_CONNECT) + return 0; + + spin_lock(&tcon->tc_lock); + if (tcon->status == TID_EXITING) { + /* + * only tree disconnect allowed when disconnecting ... + */ + if (smb2_command != SMB2_TREE_DISCONNECT) { + spin_unlock(&tcon->tc_lock); + cifs_tcon_dbg(FYI, "can not send cmd %d while umounting\n", + smb2_command); + return -ENODEV; + } + } + spin_unlock(&tcon->tc_lock); + + ses = tcon->ses; + if (!ses) + return smb_EIO(smb_eio_trace_null_pointers); + spin_lock(&ses->ses_lock); + if (ses->ses_status == SES_EXITING) { + spin_unlock(&ses->ses_lock); + return smb_EIO(smb_eio_trace_sess_exiting); + } + spin_unlock(&ses->ses_lock); + if (!ses->server || !server) + return smb_EIO(smb_eio_trace_null_pointers); + + spin_lock(&server->srv_lock); + if (server->tcpStatus == CifsNeedReconnect) { + /* + * Return to caller for TREE_DISCONNECT and LOGOFF and CLOSE + * here since they are implicitly done when session drops. + */ + switch (smb2_command) { + /* + * BB Should we keep oplock break and add flush to exceptions? + */ + case SMB2_TREE_DISCONNECT: + case SMB2_CANCEL: + case SMB2_CLOSE: + case SMB2_OPLOCK_BREAK: + spin_unlock(&server->srv_lock); + return -EAGAIN; + } + } + + /* if server is marked for termination, cifsd will cleanup */ + if (server->terminate) { + spin_unlock(&server->srv_lock); + return -EHOSTDOWN; + } + spin_unlock(&server->srv_lock); + +again: + rc = cifs_wait_for_server_reconnect(server, tcon->retry); + if (rc) + return rc; + + spin_lock(&ses->chan_lock); + if (!cifs_chan_needs_reconnect(ses, server) && !tcon->need_reconnect) { + spin_unlock(&ses->chan_lock); + return 0; + } + spin_unlock(&ses->chan_lock); + cifs_tcon_dbg(FYI, "sess reconnect mask: 0x%lx, tcon reconnect: %d\n", + tcon->ses->chans_need_reconnect, + tcon->need_reconnect); + + mutex_lock(&ses->session_mutex); + /* + * Handle the case where a concurrent thread failed to negotiate or + * killed a channel. + */ + spin_lock(&server->srv_lock); + switch (server->tcpStatus) { + case CifsExiting: + spin_unlock(&server->srv_lock); + mutex_unlock(&ses->session_mutex); + return -EHOSTDOWN; + case CifsNeedReconnect: + spin_unlock(&server->srv_lock); + mutex_unlock(&ses->session_mutex); + if (!tcon->retry) + return -EHOSTDOWN; + goto again; + default: + break; + } + spin_unlock(&server->srv_lock); + + /* + * need to prevent multiple threads trying to simultaneously + * reconnect the same SMB session + */ + spin_lock(&ses->ses_lock); + spin_lock(&ses->chan_lock); + if (!cifs_chan_needs_reconnect(ses, server) && + ses->ses_status == SES_GOOD) { + spin_unlock(&ses->chan_lock); + spin_unlock(&ses->ses_lock); + /* this means that we only need to tree connect */ + if (tcon->need_reconnect) + goto skip_sess_setup; + + mutex_unlock(&ses->session_mutex); + goto out; + } + spin_unlock(&ses->chan_lock); + spin_unlock(&ses->ses_lock); + + rc = cifs_negotiate_protocol(0, ses, server); + if (rc) { + mutex_unlock(&ses->session_mutex); + if (!tcon->retry) + return -EHOSTDOWN; + goto again; + } + /* + * if server stopped supporting multichannel + * and the first channel reconnected, disable all the others. + */ + if (ses->chan_count > 1 && + !(server->capabilities & SMB2_GLOBAL_CAP_MULTI_CHANNEL)) { + rc = smb3_update_ses_channels(ses, server, + from_reconnect, true /* disable_mchan */); + if (rc) { + mutex_unlock(&ses->session_mutex); + goto out; + } + } + + rc = cifs_setup_session(0, ses, server, ses->local_nls); + if ((rc == -EACCES) || (rc == -EKEYEXPIRED) || (rc == -EKEYREVOKED)) { + /* + * Try alternate password for next reconnect (key rotation + * could be enabled on the server e.g.) if an alternate + * password is available and the current password is expired, + * but do not swap on non pwd related errors like host down + */ + if (ses->password2) + swap(ses->password2, ses->password); + } + if (rc) { + mutex_unlock(&ses->session_mutex); + if (rc == -EACCES && !tcon->retry) + return -EHOSTDOWN; + goto out; + } + +skip_sess_setup: + if (!tcon->need_reconnect) { + mutex_unlock(&ses->session_mutex); + goto out; + } + cifs_mark_open_files_invalid(tcon); + if (tcon->use_persistent) + tcon->need_reopen_files = true; + + rc = cifs_tree_connect(0, tcon); + + cifs_tcon_dbg(FYI, "reconnect tcon rc = %d\n", rc); + if (rc) { + /* If sess reconnected but tcon didn't, something strange ... */ + mutex_unlock(&ses->session_mutex); + cifs_tcon_dbg(VFS, "reconnect tcon failed rc = %d\n", rc); + goto out; + } + + spin_lock(&ses->ses_lock); + if (ses->flags & CIFS_SES_FLAG_SCALE_CHANNELS) { + spin_unlock(&ses->ses_lock); + mutex_unlock(&ses->session_mutex); + goto skip_add_channels; + } + ses->flags |= CIFS_SES_FLAG_SCALE_CHANNELS; + spin_unlock(&ses->ses_lock); + + if (!rc && + (server->capabilities & SMB2_GLOBAL_CAP_MULTI_CHANNEL) && + server->ops->query_server_interfaces) { + /* + * query server network interfaces, in case they change. + * Also mark the session as pending this update while the query + * is in progress. This will be used to avoid calling + * smb2_reconnect recursively. + */ + ses->flags |= CIFS_SES_FLAGS_PENDING_QUERY_INTERFACES; + xid = get_xid(); + rc = server->ops->query_server_interfaces(xid, tcon, false); + free_xid(xid); + ses->flags &= ~CIFS_SES_FLAGS_PENDING_QUERY_INTERFACES; + + if (!tcon->ipc && !tcon->dummy) + queue_delayed_work(cifsiod_wq, &tcon->query_interfaces, + (SMB_INTERFACE_POLL_INTERVAL * HZ)); + + mutex_unlock(&ses->session_mutex); + + if (rc == -EOPNOTSUPP && ses->chan_count > 1) { + /* + * some servers like Azure SMB server do not advertise + * that multichannel has been disabled with server + * capabilities, rather return STATUS_NOT_IMPLEMENTED. + * treat this as server not supporting multichannel + */ + + rc = smb3_update_ses_channels(ses, server, + from_reconnect, + true /* disable_mchan */); + goto skip_add_channels; + } else if (rc) + cifs_tcon_dbg(FYI, "%s: failed to query server interfaces: %d\n", + __func__, rc); + + if (ses->chan_max > ses->chan_count && + ses->iface_count && + !SERVER_IS_CHAN(server)) { + if (ses->chan_count == 1) + cifs_server_dbg(VFS, "supports multichannel now\n"); + + smb3_update_ses_channels(ses, server, from_reconnect, + false /* disable_mchan */); + } + } else { + mutex_unlock(&ses->session_mutex); + } + +skip_add_channels: + spin_lock(&ses->ses_lock); + ses->flags &= ~CIFS_SES_FLAG_SCALE_CHANNELS; + spin_unlock(&ses->ses_lock); + + if (smb2_command != SMB2_INTERNAL_CMD) + cifs_queue_server_reconn(server); + + atomic_inc(&tconInfoReconnectCount); +out: + /* + * Check if handle based operation so we know whether we can continue + * or not without returning to caller to reset file handle. + */ + /* + * BB Is flush done by server on drop of tcp session? Should we special + * case it and skip above? + */ + switch (smb2_command) { + case SMB2_FLUSH: + case SMB2_READ: + case SMB2_WRITE: + case SMB2_LOCK: + case SMB2_QUERY_DIRECTORY: + case SMB2_CHANGE_NOTIFY: + case SMB2_QUERY_INFO: + case SMB2_SET_INFO: + case SMB2_IOCTL: + rc = -EAGAIN; + } + return rc; +} + +static void +fill_small_buf(__le16 smb2_command, struct cifs_tcon *tcon, + struct TCP_Server_Info *server, + void *buf, + unsigned int *total_len) +{ + struct smb2_pdu *spdu = buf; + /* lookup word count ie StructureSize from table */ + __u16 parmsize = smb2_req_struct_sizes[le16_to_cpu(smb2_command)]; + + /* + * smaller than SMALL_BUFFER_SIZE but bigger than fixed area of + * largest operations (Create) + */ + memset(buf, 0, 256); + + smb2_hdr_assemble(&spdu->hdr, smb2_command, tcon, server); + spdu->StructureSize2 = cpu_to_le16(parmsize); + + *total_len = parmsize + sizeof(struct smb2_hdr); +} + +/* + * Allocate and return pointer to an SMB request hdr, and set basic + * SMB information in the SMB header. If the return code is zero, this + * function must have filled in request_buf pointer. + */ +static int __smb2_plain_req_init(__le16 smb2_command, struct cifs_tcon *tcon, + struct TCP_Server_Info *server, + void **request_buf, unsigned int *total_len) +{ + /* BB eventually switch this to SMB2 specific small buf size */ + switch (smb2_command) { + case SMB2_SET_INFO: + case SMB2_QUERY_INFO: + *request_buf = cifs_buf_get(); + break; + default: + *request_buf = cifs_small_buf_get(); + break; + } + if (*request_buf == NULL) { + /* BB should we add a retry in here if not a writepage? */ + return -ENOMEM; + } + + fill_small_buf(smb2_command, tcon, server, + (struct smb2_hdr *)(*request_buf), + total_len); + + if (tcon != NULL) { + uint16_t com_code = le16_to_cpu(smb2_command); + cifs_stats_inc(&tcon->stats.smb2_stats.smb2_com_sent[com_code]); + cifs_stats_inc(&tcon->num_smbs_sent); + } + + return 0; +} + +static int smb2_plain_req_init(__le16 smb2_command, struct cifs_tcon *tcon, + struct TCP_Server_Info *server, + void **request_buf, unsigned int *total_len) +{ + int rc; + + rc = smb2_reconnect(smb2_command, tcon, server, false); + if (rc) + return rc; + + return __smb2_plain_req_init(smb2_command, tcon, server, request_buf, + total_len); +} + +static int smb2_ioctl_req_init(u32 opcode, struct cifs_tcon *tcon, + struct TCP_Server_Info *server, + void **request_buf, unsigned int *total_len) +{ + /* + * Skip reconnect in one of the following cases: + * 1. For FSCTL_VALIDATE_NEGOTIATE_INFO IOCTLs + * 2. For FSCTL_QUERY_NETWORK_INTERFACE_INFO IOCTL when called from + * smb2_reconnect (indicated by CIFS_SES_FLAG_SCALE_CHANNELS ses flag) + */ + if (opcode == FSCTL_VALIDATE_NEGOTIATE_INFO || + (opcode == FSCTL_QUERY_NETWORK_INTERFACE_INFO && + (tcon->ses->flags & CIFS_SES_FLAGS_PENDING_QUERY_INTERFACES))) + return __smb2_plain_req_init(SMB2_IOCTL, tcon, server, + request_buf, total_len); + + return smb2_plain_req_init(SMB2_IOCTL, tcon, server, + request_buf, total_len); +} + +/* For explanation of negotiate contexts see MS-SMB2 section 2.2.3.1 */ + +static void +build_preauth_ctxt(struct smb2_preauth_neg_context *pneg_ctxt) +{ + pneg_ctxt->ContextType = SMB2_PREAUTH_INTEGRITY_CAPABILITIES; + pneg_ctxt->DataLength = cpu_to_le16(38); + pneg_ctxt->HashAlgorithmCount = cpu_to_le16(1); + pneg_ctxt->SaltLength = cpu_to_le16(SMB311_SALT_SIZE); + get_random_bytes(pneg_ctxt->Salt, SMB311_SALT_SIZE); + pneg_ctxt->HashAlgorithms = SMB2_PREAUTH_INTEGRITY_SHA512; +} + +static void +build_compression_ctxt(struct smb2_compression_capabilities_context *pneg_ctxt) +{ + pneg_ctxt->ContextType = SMB2_COMPRESSION_CAPABILITIES; + pneg_ctxt->DataLength = + cpu_to_le16(sizeof(struct smb2_compression_capabilities_context) + - sizeof(struct smb2_neg_context)); + /* + * Pattern_V1 is useful only as part of a chained transform. LZ77 remains + * the preferred general-purpose algorithm selected by this client. + */ + pneg_ctxt->CompressionAlgorithmCount = cpu_to_le16(4); + pneg_ctxt->Flags = SMB2_COMPRESSION_CAPABILITIES_FLAG_CHAINED; + pneg_ctxt->CompressionAlgorithms[0] = SMB3_COMPRESS_LZ77; + pneg_ctxt->CompressionAlgorithms[1] = SMB3_COMPRESS_LZ77_HUFF; + pneg_ctxt->CompressionAlgorithms[2] = SMB3_COMPRESS_LZNT1; + pneg_ctxt->CompressionAlgorithms[3] = SMB3_COMPRESS_PATTERN; +} + +static unsigned int +build_signing_ctxt(struct smb2_signing_capabilities *pneg_ctxt) +{ + unsigned int ctxt_len = sizeof(struct smb2_signing_capabilities); + unsigned short num_algs = 1; /* number of signing algorithms sent */ + + pneg_ctxt->ContextType = SMB2_SIGNING_CAPABILITIES; + /* + * Context Data length must be rounded to multiple of 8 for some servers + */ + pneg_ctxt->DataLength = cpu_to_le16(ALIGN(sizeof(struct smb2_signing_capabilities) - + sizeof(struct smb2_neg_context) + + (num_algs * sizeof(u16)), 8)); + pneg_ctxt->SigningAlgorithmCount = cpu_to_le16(num_algs); + pneg_ctxt->SigningAlgorithms[0] = cpu_to_le16(SIGNING_ALG_AES_CMAC); + + ctxt_len += sizeof(__le16) * num_algs; + ctxt_len = ALIGN(ctxt_len, 8); + return ctxt_len; + /* TBD add SIGNING_ALG_AES_GMAC and/or SIGNING_ALG_HMAC_SHA256 */ +} + +static void +build_encrypt_ctxt(struct smb2_encryption_neg_context *pneg_ctxt) +{ + pneg_ctxt->ContextType = SMB2_ENCRYPTION_CAPABILITIES; + if (require_gcm_256) { + pneg_ctxt->DataLength = cpu_to_le16(4); /* Cipher Count + 1 cipher */ + pneg_ctxt->CipherCount = cpu_to_le16(1); + pneg_ctxt->Ciphers[0] = SMB2_ENCRYPTION_AES256_GCM; + } else if (enable_gcm_256) { + pneg_ctxt->DataLength = cpu_to_le16(8); /* Cipher Count + 3 ciphers */ + pneg_ctxt->CipherCount = cpu_to_le16(3); + pneg_ctxt->Ciphers[0] = SMB2_ENCRYPTION_AES128_GCM; + pneg_ctxt->Ciphers[1] = SMB2_ENCRYPTION_AES256_GCM; + pneg_ctxt->Ciphers[2] = SMB2_ENCRYPTION_AES128_CCM; + } else { + pneg_ctxt->DataLength = cpu_to_le16(6); /* Cipher Count + 2 ciphers */ + pneg_ctxt->CipherCount = cpu_to_le16(2); + pneg_ctxt->Ciphers[0] = SMB2_ENCRYPTION_AES128_GCM; + pneg_ctxt->Ciphers[1] = SMB2_ENCRYPTION_AES128_CCM; + } +} + +static unsigned int +build_netname_ctxt(struct smb2_netname_neg_context *pneg_ctxt, char *hostname) +{ + struct nls_table *cp = load_nls_default(); + + pneg_ctxt->ContextType = SMB2_NETNAME_NEGOTIATE_CONTEXT_ID; + + /* copy up to max of first 100 bytes of server name to NetName field */ + pneg_ctxt->DataLength = cpu_to_le16(2 * cifs_strtoUTF16(pneg_ctxt->NetName, hostname, 100, cp)); + /* context size is DataLength + minimal smb2_neg_context */ + return ALIGN(le16_to_cpu(pneg_ctxt->DataLength) + sizeof(struct smb2_neg_context), 8); +} + +static void +build_posix_ctxt(struct smb2_posix_neg_context *pneg_ctxt) +{ + pneg_ctxt->ContextType = SMB2_POSIX_EXTENSIONS_AVAILABLE; + pneg_ctxt->DataLength = cpu_to_le16(POSIX_CTXT_DATA_LEN); + /* SMB2_CREATE_TAG_POSIX is "0x93AD25509CB411E7B42383DE968BCD7C" */ + pneg_ctxt->Name[0] = 0x93; + pneg_ctxt->Name[1] = 0xAD; + pneg_ctxt->Name[2] = 0x25; + pneg_ctxt->Name[3] = 0x50; + pneg_ctxt->Name[4] = 0x9C; + pneg_ctxt->Name[5] = 0xB4; + pneg_ctxt->Name[6] = 0x11; + pneg_ctxt->Name[7] = 0xE7; + pneg_ctxt->Name[8] = 0xB4; + pneg_ctxt->Name[9] = 0x23; + pneg_ctxt->Name[10] = 0x83; + pneg_ctxt->Name[11] = 0xDE; + pneg_ctxt->Name[12] = 0x96; + pneg_ctxt->Name[13] = 0x8B; + pneg_ctxt->Name[14] = 0xCD; + pneg_ctxt->Name[15] = 0x7C; +} + +static void +assemble_neg_contexts(struct smb2_negotiate_req *req, + struct TCP_Server_Info *server, unsigned int *total_len) +{ + unsigned int ctxt_len, neg_context_count; + struct TCP_Server_Info *pserver; + char *pneg_ctxt; + char *hostname; + + if (*total_len > 200) { + /* In case length corrupted don't want to overrun smb buffer */ + cifs_server_dbg(VFS, "Bad frame length assembling neg contexts\n"); + return; + } + + /* + * round up total_len of fixed part of SMB3 negotiate request to 8 + * byte boundary before adding negotiate contexts + */ + *total_len = ALIGN(*total_len, 8); + + pneg_ctxt = (*total_len) + (char *)req; + req->NegotiateContextOffset = cpu_to_le32(*total_len); + + build_preauth_ctxt((struct smb2_preauth_neg_context *)pneg_ctxt); + ctxt_len = ALIGN(sizeof(struct smb2_preauth_neg_context), 8); + *total_len += ctxt_len; + pneg_ctxt += ctxt_len; + + build_encrypt_ctxt((struct smb2_encryption_neg_context *)pneg_ctxt); + ctxt_len = ALIGN(sizeof(struct smb2_encryption_neg_context), 8); + *total_len += ctxt_len; + pneg_ctxt += ctxt_len; + + /* + * secondary channels don't have the hostname field populated + * use the hostname field in the primary channel instead + */ + pserver = SERVER_IS_CHAN(server) ? server->primary_server : server; + cifs_server_lock(pserver); + hostname = pserver->hostname; + if (hostname && (hostname[0] != 0)) { + ctxt_len = build_netname_ctxt((struct smb2_netname_neg_context *)pneg_ctxt, + hostname); + *total_len += ctxt_len; + pneg_ctxt += ctxt_len; + neg_context_count = 3; + } else + neg_context_count = 2; + cifs_server_unlock(pserver); + + build_posix_ctxt((struct smb2_posix_neg_context *)pneg_ctxt); + *total_len += sizeof(struct smb2_posix_neg_context); + pneg_ctxt += sizeof(struct smb2_posix_neg_context); + neg_context_count++; + + if (server->compression.requested) { + build_compression_ctxt((struct smb2_compression_capabilities_context *) + pneg_ctxt); + ctxt_len = ALIGN(sizeof(struct smb2_compression_capabilities_context), 8); + *total_len += ctxt_len; + pneg_ctxt += ctxt_len; + neg_context_count++; + } + + if (enable_negotiate_signing) { + ctxt_len = build_signing_ctxt((struct smb2_signing_capabilities *) + pneg_ctxt); + *total_len += ctxt_len; + pneg_ctxt += ctxt_len; + neg_context_count++; + } + + /* check for and add transport_capabilities and signing capabilities */ + req->NegotiateContextCount = cpu_to_le16(neg_context_count); + +} + +/* If invalid preauth context warn but use what we requested, SHA-512 */ +static void decode_preauth_context(struct smb2_preauth_neg_context *ctxt) +{ + unsigned int len = le16_to_cpu(ctxt->DataLength); + + /* + * Caller checked that DataLength remains within SMB boundary. We still + * need to confirm that one HashAlgorithms member is accounted for. + */ + if (len < MIN_PREAUTH_CTXT_DATA_LEN) { + pr_warn_once("server sent bad preauth context\n"); + return; + } else if (len < MIN_PREAUTH_CTXT_DATA_LEN + le16_to_cpu(ctxt->SaltLength)) { + pr_warn_once("server sent invalid SaltLength\n"); + return; + } + if (le16_to_cpu(ctxt->HashAlgorithmCount) != 1) + pr_warn_once("Invalid SMB3 hash algorithm count\n"); + if (ctxt->HashAlgorithms != SMB2_PREAUTH_INTEGRITY_SHA512) + pr_warn_once("unknown SMB3 hash algorithm\n"); +} + +static void decode_compress_ctx(struct TCP_Server_Info *server, + struct smb2_compression_capabilities_context *ctxt) +{ + unsigned int len = le16_to_cpu(ctxt->DataLength); + unsigned int count, i; + + server->compression.enabled = false; + server->compression.chained = false; + server->compression.pattern = false; + server->compression.alg = SMB3_COMPRESS_NONE; + + /* + * Caller checked that DataLength remains within SMB boundary. We still + * need to confirm that one CompressionAlgorithms member is accounted + * for. + */ + if (len < 10) { + pr_warn_once("server sent bad compression cntxt\n"); + return; + } + + count = le16_to_cpu(ctxt->CompressionAlgorithmCount); + if (!count || count > ARRAY_SIZE(ctxt->CompressionAlgorithms) || + len < 8 + count * sizeof(__le16)) { + pr_warn_once("invalid SMB3 compress algorithm count\n"); + return; + } + + if (ctxt->Flags != SMB2_COMPRESSION_CAPABILITIES_FLAG_NONE && + ctxt->Flags != SMB2_COMPRESSION_CAPABILITIES_FLAG_CHAINED) { + pr_warn_once("invalid SMB3 compression flags\n"); + return; + } + + for (i = 0; i < count; i++) { + /* Record the intersection supported by the shared SMB codec. */ + if (ctxt->CompressionAlgorithms[i] == SMB3_COMPRESS_LZ77) + server->compression.alg = SMB3_COMPRESS_LZ77; + else if (ctxt->CompressionAlgorithms[i] == SMB3_COMPRESS_PATTERN) + server->compression.pattern = true; + } + if (server->compression.alg != SMB3_COMPRESS_LZ77) + return; + + /* + * Pattern_V1 cannot appear in an unchained transform even if a broken + * peer lists it in the algorithm array. + */ + server->compression.chained = + ctxt->Flags == SMB2_COMPRESSION_CAPABILITIES_FLAG_CHAINED; + if (!server->compression.chained) + server->compression.pattern = false; + server->compression.enabled = true; +} + +static int decode_encrypt_ctx(struct TCP_Server_Info *server, + struct smb2_encryption_neg_context *ctxt) +{ + unsigned int len = le16_to_cpu(ctxt->DataLength); + + cifs_dbg(FYI, "decode SMB3.11 encryption neg context of len %d\n", len); + /* + * Caller checked that DataLength remains within SMB boundary. We still + * need to confirm that one Cipher flexible array member is accounted + * for. + */ + if (len < MIN_ENCRYPT_CTXT_DATA_LEN) { + pr_warn_once("server sent bad crypto ctxt len\n"); + return -EINVAL; + } + + if (le16_to_cpu(ctxt->CipherCount) != 1) { + pr_warn_once("Invalid SMB3.11 cipher count\n"); + return -EINVAL; + } + cifs_dbg(FYI, "SMB311 cipher type:%d\n", le16_to_cpu(ctxt->Ciphers[0])); + if (require_gcm_256) { + if (ctxt->Ciphers[0] != SMB2_ENCRYPTION_AES256_GCM) { + cifs_dbg(VFS, "Server does not support requested encryption type (AES256 GCM)\n"); + return -EOPNOTSUPP; + } + } else if (ctxt->Ciphers[0] == 0) { + /* + * e.g. if server only supported AES256_CCM (very unlikely) + * or server supported no encryption types or had all disabled. + * Since GLOBAL_CAP_ENCRYPTION will be not set, in the case + * in which mount requested encryption ("seal") checks later + * on during tree connection will return proper rc, but if + * seal not requested by client, since server is allowed to + * return 0 to indicate no supported cipher, we can't fail here + */ + server->cipher_type = 0; + server->capabilities &= ~SMB2_GLOBAL_CAP_ENCRYPTION; + pr_warn_once("Server does not support requested encryption types\n"); + return 0; + } else if ((ctxt->Ciphers[0] != SMB2_ENCRYPTION_AES128_CCM) && + (ctxt->Ciphers[0] != SMB2_ENCRYPTION_AES128_GCM) && + (ctxt->Ciphers[0] != SMB2_ENCRYPTION_AES256_GCM)) { + /* server returned a cipher we didn't ask for */ + pr_warn_once("Invalid SMB3.11 cipher returned\n"); + return -EINVAL; + } + server->cipher_type = ctxt->Ciphers[0]; + server->capabilities |= SMB2_GLOBAL_CAP_ENCRYPTION; + return 0; +} + +static void decode_signing_ctx(struct TCP_Server_Info *server, + struct smb2_signing_capabilities *pctxt) +{ + unsigned int len = le16_to_cpu(pctxt->DataLength); + + /* + * Caller checked that DataLength remains within SMB boundary. We still + * need to confirm that one SigningAlgorithms flexible array member is + * accounted for. + */ + if ((len < 4) || (len > 16)) { + pr_warn_once("server sent bad signing negcontext\n"); + return; + } + if (le16_to_cpu(pctxt->SigningAlgorithmCount) != 1) { + pr_warn_once("Invalid signing algorithm count\n"); + return; + } + if (le16_to_cpu(pctxt->SigningAlgorithms[0]) > 2) { + pr_warn_once("unknown signing algorithm\n"); + return; + } + + server->signing_negotiated = true; + server->signing_algorithm = le16_to_cpu(pctxt->SigningAlgorithms[0]); + cifs_dbg(FYI, "signing algorithm %d chosen\n", + server->signing_algorithm); +} + + +static int smb311_decode_neg_context(struct smb2_negotiate_rsp *rsp, + struct TCP_Server_Info *server, + unsigned int len_of_smb) +{ + struct smb2_neg_context *pctx; + unsigned int offset = le32_to_cpu(rsp->NegotiateContextOffset); + unsigned int ctxt_cnt = le16_to_cpu(rsp->NegotiateContextCount); + unsigned int len_of_ctxts, i; + int rc = 0; + + cifs_dbg(FYI, "decoding %d negotiate contexts\n", ctxt_cnt); + if (len_of_smb <= offset) { + cifs_server_dbg(VFS, "Invalid response: negotiate context offset\n"); + return -EINVAL; + } + + len_of_ctxts = len_of_smb - offset; + + for (i = 0; i < ctxt_cnt; i++) { + int clen; + /* check that offset is not beyond end of SMB */ + if (len_of_ctxts < sizeof(struct smb2_neg_context)) + break; + + pctx = (struct smb2_neg_context *)(offset + (char *)rsp); + clen = sizeof(struct smb2_neg_context) + + le16_to_cpu(pctx->DataLength); + /* + * 2.2.4 SMB2 NEGOTIATE Response + * Subsequent negotiate contexts MUST appear at the first 8-byte + * aligned offset following the previous negotiate context. + */ + if (i + 1 != ctxt_cnt) + clen = ALIGN(clen, 8); + if (clen > len_of_ctxts) + break; + + if (pctx->ContextType == SMB2_PREAUTH_INTEGRITY_CAPABILITIES) + decode_preauth_context( + (struct smb2_preauth_neg_context *)pctx); + else if (pctx->ContextType == SMB2_ENCRYPTION_CAPABILITIES) + rc = decode_encrypt_ctx(server, + (struct smb2_encryption_neg_context *)pctx); + else if (pctx->ContextType == SMB2_COMPRESSION_CAPABILITIES) + decode_compress_ctx(server, + (struct smb2_compression_capabilities_context *)pctx); + else if (pctx->ContextType == SMB2_POSIX_EXTENSIONS_AVAILABLE) + server->posix_ext_supported = true; + else if (pctx->ContextType == SMB2_SIGNING_CAPABILITIES) + decode_signing_ctx(server, + (struct smb2_signing_capabilities *)pctx); + else + cifs_server_dbg(VFS, "unknown negcontext of type %d ignored\n", + le16_to_cpu(pctx->ContextType)); + if (rc) + break; + + offset += clen; + len_of_ctxts -= clen; + } + return rc; +} + +static struct create_posix * +create_posix_buf(umode_t mode) +{ + struct create_posix *buf; + + buf = kzalloc_obj(struct create_posix); + if (!buf) + return NULL; + + buf->ccontext.DataOffset = + cpu_to_le16(offsetof(struct create_posix, Mode)); + buf->ccontext.DataLength = cpu_to_le32(4); + buf->ccontext.NameOffset = + cpu_to_le16(offsetof(struct create_posix, Name)); + buf->ccontext.NameLength = cpu_to_le16(16); + + /* SMB2_CREATE_TAG_POSIX is "0x93AD25509CB411E7B42383DE968BCD7C" */ + buf->Name[0] = 0x93; + buf->Name[1] = 0xAD; + buf->Name[2] = 0x25; + buf->Name[3] = 0x50; + buf->Name[4] = 0x9C; + buf->Name[5] = 0xB4; + buf->Name[6] = 0x11; + buf->Name[7] = 0xE7; + buf->Name[8] = 0xB4; + buf->Name[9] = 0x23; + buf->Name[10] = 0x83; + buf->Name[11] = 0xDE; + buf->Name[12] = 0x96; + buf->Name[13] = 0x8B; + buf->Name[14] = 0xCD; + buf->Name[15] = 0x7C; + buf->Mode = cpu_to_le32(mode); + cifs_dbg(FYI, "mode on posix create 0%o\n", mode); + return buf; +} + +static int +add_posix_context(struct kvec *iov, unsigned int *num_iovec, umode_t mode) +{ + unsigned int num = *num_iovec; + + iov[num].iov_base = create_posix_buf(mode); + if (mode == ACL_NO_MODE) + cifs_dbg(FYI, "%s: no mode\n", __func__); + if (iov[num].iov_base == NULL) + return -ENOMEM; + iov[num].iov_len = sizeof(struct create_posix); + *num_iovec = num + 1; + return 0; +} + + +/* + * + * SMB2 Worker functions follow: + * + * The general structure of the worker functions is: + * 1) Call smb2_init (assembles SMB2 header) + * 2) Initialize SMB2 command specific fields in fixed length area of SMB + * 3) Call smb_sendrcv2 (sends request on socket and waits for response) + * 4) Decode SMB2 command specific fields in the fixed length area + * 5) Decode variable length data area (if any for this SMB2 command type) + * 6) Call free smb buffer + * 7) return + * + */ + +int +SMB2_negotiate(const unsigned int xid, + struct cifs_ses *ses, + struct TCP_Server_Info *server) +{ + struct smb_rqst rqst; + struct smb2_negotiate_req *req; + struct smb2_negotiate_rsp *rsp; + struct kvec iov[1]; + struct kvec rsp_iov; + int rc; + int resp_buftype; + int blob_offset, blob_length; + char *security_blob; + int flags = CIFS_NEG_OP; + unsigned int total_len; + + cifs_dbg(FYI, "Negotiate protocol\n"); + + if (!server) { + WARN(1, "%s: server is NULL!\n", __func__); + return smb_EIO(smb_eio_trace_null_pointers); + } + + rc = smb2_plain_req_init(SMB2_NEGOTIATE, NULL, server, + (void **) &req, &total_len); + if (rc) + return rc; + + req->hdr.SessionId = 0; + + memset(server->preauth_sha_hash, 0, SMB2_PREAUTH_HASH_SIZE); + memset(ses->preauth_sha_hash, 0, SMB2_PREAUTH_HASH_SIZE); + + if (strcmp(server->vals->version_string, + SMB3ANY_VERSION_STRING) == 0) { + req->Dialects[0] = cpu_to_le16(SMB30_PROT_ID); + req->Dialects[1] = cpu_to_le16(SMB302_PROT_ID); + req->Dialects[2] = cpu_to_le16(SMB311_PROT_ID); + req->DialectCount = cpu_to_le16(3); + total_len += 6; + } else if (strcmp(server->vals->version_string, + SMBDEFAULT_VERSION_STRING) == 0) { + req->Dialects[0] = cpu_to_le16(SMB21_PROT_ID); + req->Dialects[1] = cpu_to_le16(SMB30_PROT_ID); + req->Dialects[2] = cpu_to_le16(SMB302_PROT_ID); + req->Dialects[3] = cpu_to_le16(SMB311_PROT_ID); + req->DialectCount = cpu_to_le16(4); + total_len += 8; + } else { + /* otherwise send specific dialect */ + req->Dialects[0] = cpu_to_le16(server->vals->protocol_id); + req->DialectCount = cpu_to_le16(1); + total_len += 2; + } + + /* only one of SMB2 signing flags may be set in SMB2 request */ + if (ses->sign) + req->SecurityMode = cpu_to_le16(SMB2_NEGOTIATE_SIGNING_REQUIRED); + else if (global_secflags & CIFSSEC_MAY_SIGN) + req->SecurityMode = cpu_to_le16(SMB2_NEGOTIATE_SIGNING_ENABLED); + else + req->SecurityMode = 0; + + req->Capabilities = cpu_to_le32(server->vals->req_capabilities); + req->Capabilities |= cpu_to_le32(SMB2_GLOBAL_CAP_MULTI_CHANNEL); + + /* ClientGUID must be zero for SMB2.02 dialect */ + if (server->vals->protocol_id == SMB20_PROT_ID) + memset(req->ClientGUID, 0, SMB2_CLIENT_GUID_SIZE); + else { + memcpy(req->ClientGUID, server->client_guid, + SMB2_CLIENT_GUID_SIZE); + if ((server->vals->protocol_id == SMB311_PROT_ID) || + (strcmp(server->vals->version_string, + SMB3ANY_VERSION_STRING) == 0) || + (strcmp(server->vals->version_string, + SMBDEFAULT_VERSION_STRING) == 0)) + assemble_neg_contexts(req, server, &total_len); + } + iov[0].iov_base = (char *)req; + iov[0].iov_len = total_len; + + memset(&rqst, 0, sizeof(struct smb_rqst)); + rqst.rq_iov = iov; + rqst.rq_nvec = 1; + + rc = cifs_send_recv(xid, ses, server, + &rqst, &resp_buftype, flags, &rsp_iov); + cifs_small_buf_release(req); + rsp = (struct smb2_negotiate_rsp *)rsp_iov.iov_base; + /* + * No tcon so can't do + * cifs_stats_inc(&tcon->stats.smb2_stats.smb2_com_fail[SMB2...]); + */ + if (rc == -EOPNOTSUPP) { + cifs_server_dbg(VFS, "Dialect not supported by server. Consider specifying vers=1.0 or vers=2.0 on mount for accessing older servers\n"); + goto neg_exit; + } else if (rc != 0) + goto neg_exit; + + u16 dialect = le16_to_cpu(rsp->DialectRevision); + if (strcmp(server->vals->version_string, + SMB3ANY_VERSION_STRING) == 0) { + switch (dialect) { + case SMB20_PROT_ID: + cifs_server_dbg(VFS, + "SMB2 dialect returned but not requested\n"); + rc = smb_EIO2(smb_eio_trace_neg_unreq_dialect, dialect, 3); + goto neg_exit; + case SMB21_PROT_ID: + cifs_server_dbg(VFS, + "SMB2.1 dialect returned but not requested\n"); + rc = smb_EIO2(smb_eio_trace_neg_unreq_dialect, dialect, 3); + goto neg_exit; + case SMB311_PROT_ID: + /* ops set to 3.0 by default for default so update */ + server->ops = &smb311_operations; + server->vals = &smb311_values; + break; + default: + break; + } + } else if (strcmp(server->vals->version_string, + SMBDEFAULT_VERSION_STRING) == 0) { + switch (dialect) { + case SMB20_PROT_ID: + cifs_server_dbg(VFS, + "SMB2 dialect returned but not requested\n"); + rc = smb_EIO2(smb_eio_trace_neg_unreq_dialect, dialect, 0); + goto neg_exit; + case SMB21_PROT_ID: + /* ops set to 3.0 by default for default so update */ + server->ops = &smb21_operations; + server->vals = &smb21_values; + break; + case SMB311_PROT_ID: + server->ops = &smb311_operations; + server->vals = &smb311_values; + break; + default: + break; + } + } else if (dialect != server->vals->protocol_id) { + /* if requested single dialect ensure returned dialect matched */ + cifs_server_dbg(VFS, "Invalid 0x%x dialect returned: not requested\n", + dialect); + rc = smb_EIO2(smb_eio_trace_neg_unreq_dialect, + dialect, server->vals->protocol_id); + goto neg_exit; + } + + cifs_dbg(FYI, "mode 0x%x\n", rsp->SecurityMode); + + switch (dialect) { + case SMB20_PROT_ID: + cifs_dbg(FYI, "negotiated smb2.0 dialect\n"); + break; + case SMB21_PROT_ID: + cifs_dbg(FYI, "negotiated smb2.1 dialect\n"); + break; + case SMB30_PROT_ID: + cifs_dbg(FYI, "negotiated smb3.0 dialect\n"); + break; + case SMB302_PROT_ID: + cifs_dbg(FYI, "negotiated smb3.02 dialect\n"); + break; + case SMB311_PROT_ID: + cifs_dbg(FYI, "negotiated smb3.1.1 dialect\n"); + break; + default: + cifs_server_dbg(VFS, "Invalid dialect returned by server 0x%x\n", + dialect); + rc = smb_EIO1(smb_eio_trace_neg_inval_dialect, dialect); + goto neg_exit; + } + + rc = 0; + server->dialect = dialect; + + /* + * Keep a copy of the hash after negprot. This hash will be + * the starting hash value for all sessions made from this + * server. + */ + memcpy(server->preauth_sha_hash, ses->preauth_sha_hash, + SMB2_PREAUTH_HASH_SIZE); + + /* SMB2 only has an extended negflavor */ + server->negflavor = CIFS_NEGFLAVOR_EXTENDED; + /* set it to the maximum buffer size value we can send with 1 credit */ + server->maxBuf = min_t(unsigned int, le32_to_cpu(rsp->MaxTransactSize), + SMB2_MAX_BUFFER_SIZE); + server->max_read = le32_to_cpu(rsp->MaxReadSize); + server->max_write = le32_to_cpu(rsp->MaxWriteSize); + server->sec_mode = le16_to_cpu(rsp->SecurityMode); + if ((server->sec_mode & SMB2_SEC_MODE_FLAGS_ALL) != server->sec_mode) + cifs_dbg(FYI, "Server returned unexpected security mode 0x%x\n", + server->sec_mode); + server->capabilities = le32_to_cpu(rsp->Capabilities); + /* Internal types */ + server->capabilities |= SMB2_NT_FIND | SMB2_LARGE_FILES; + + /* + * SMB3.0 supports only 1 cipher and doesn't have a encryption neg context + * Set the cipher type manually. + */ + if ((server->dialect == SMB30_PROT_ID || + server->dialect == SMB302_PROT_ID) && + (server->capabilities & SMB2_GLOBAL_CAP_ENCRYPTION)) + server->cipher_type = SMB2_ENCRYPTION_AES128_CCM; + + security_blob = smb2_get_data_area_len(&blob_offset, &blob_length, + (struct smb2_hdr *)rsp); + /* + * See MS-SMB2 section 2.2.4: if no blob, client picks default which + * for us will be + * ses->sectype = RawNTLMSSP; + * but for time being this is our only auth choice so doesn't matter. + * We just found a server which sets blob length to zero expecting raw. + */ + if (blob_length == 0) { + cifs_dbg(FYI, "missing security blob on negprot\n"); + server->sec_ntlmssp = true; + } + + rc = cifs_enable_signing(server, ses->sign); + if (rc) + goto neg_exit; + if (blob_length) { + rc = decode_negTokenInit(security_blob, blob_length, server); + if (rc == 1) + rc = 0; + else if (rc == 0) + rc = smb_EIO1(smb_eio_trace_neg_decode_token, rc); + } + + if (server->dialect == SMB311_PROT_ID) { + if (rsp->NegotiateContextCount) + rc = smb311_decode_neg_context(rsp, server, + rsp_iov.iov_len); + else + cifs_server_dbg(VFS, "Missing expected negotiate contexts\n"); + } + + if (server->cipher_type && !rc) + rc = smb3_crypto_aead_allocate(server); +neg_exit: + free_rsp_buf(resp_buftype, rsp); + return rc; +} + +int smb3_validate_negotiate(const unsigned int xid, struct cifs_tcon *tcon) +{ + int rc; + struct validate_negotiate_info_req *pneg_inbuf; + struct validate_negotiate_info_rsp *pneg_rsp = NULL; + u32 rsplen; + u32 inbuflen; /* max of 4 dialects */ + struct TCP_Server_Info *server = tcon->ses->server; + + cifs_dbg(FYI, "validate negotiate\n"); + + /* In SMB3.11 preauth integrity supersedes validate negotiate */ + if (server->dialect == SMB311_PROT_ID) + return 0; + + /* + * validation ioctl must be signed, so no point sending this if we + * can not sign it (ie are not known user). Even if signing is not + * required (enabled but not negotiated), in those cases we selectively + * sign just this, the first and only signed request on a connection. + * Having validation of negotiate info helps reduce attack vectors. + */ + if (tcon->ses->session_flags & SMB2_SESSION_FLAG_IS_GUEST) + return 0; /* validation requires signing */ + + if (tcon->ses->user_name == NULL) { + cifs_dbg(FYI, "Can't validate negotiate: null user mount\n"); + return 0; /* validation requires signing */ + } + + if (tcon->ses->session_flags & SMB2_SESSION_FLAG_IS_NULL) + cifs_tcon_dbg(VFS, "Unexpected null user (anonymous) auth flag sent by server\n"); + + pneg_inbuf = kmalloc_obj(*pneg_inbuf, GFP_NOFS); + if (!pneg_inbuf) + return -ENOMEM; + + pneg_inbuf->Capabilities = + cpu_to_le32(server->vals->req_capabilities); + pneg_inbuf->Capabilities |= cpu_to_le32(SMB2_GLOBAL_CAP_MULTI_CHANNEL); + + memcpy(pneg_inbuf->Guid, server->client_guid, + SMB2_CLIENT_GUID_SIZE); + + if (tcon->ses->sign) + pneg_inbuf->SecurityMode = + cpu_to_le16(SMB2_NEGOTIATE_SIGNING_REQUIRED); + else if (global_secflags & CIFSSEC_MAY_SIGN) + pneg_inbuf->SecurityMode = + cpu_to_le16(SMB2_NEGOTIATE_SIGNING_ENABLED); + else + pneg_inbuf->SecurityMode = 0; + + + if (strcmp(server->vals->version_string, + SMB3ANY_VERSION_STRING) == 0) { + pneg_inbuf->Dialects[0] = cpu_to_le16(SMB30_PROT_ID); + pneg_inbuf->Dialects[1] = cpu_to_le16(SMB302_PROT_ID); + pneg_inbuf->Dialects[2] = cpu_to_le16(SMB311_PROT_ID); + pneg_inbuf->DialectCount = cpu_to_le16(3); + /* SMB 2.1 not included so subtract one dialect from len */ + inbuflen = sizeof(*pneg_inbuf) - + (sizeof(pneg_inbuf->Dialects[0])); + } else if (strcmp(server->vals->version_string, + SMBDEFAULT_VERSION_STRING) == 0) { + pneg_inbuf->Dialects[0] = cpu_to_le16(SMB21_PROT_ID); + pneg_inbuf->Dialects[1] = cpu_to_le16(SMB30_PROT_ID); + pneg_inbuf->Dialects[2] = cpu_to_le16(SMB302_PROT_ID); + pneg_inbuf->Dialects[3] = cpu_to_le16(SMB311_PROT_ID); + pneg_inbuf->DialectCount = cpu_to_le16(4); + /* structure is big enough for 4 dialects */ + inbuflen = sizeof(*pneg_inbuf); + } else { + /* otherwise specific dialect was requested */ + pneg_inbuf->Dialects[0] = + cpu_to_le16(server->vals->protocol_id); + pneg_inbuf->DialectCount = cpu_to_le16(1); + /* structure is big enough for 4 dialects, sending only 1 */ + inbuflen = sizeof(*pneg_inbuf) - + sizeof(pneg_inbuf->Dialects[0]) * 3; + } + + rc = SMB2_ioctl(xid, tcon, NO_FILE_ID, NO_FILE_ID, + FSCTL_VALIDATE_NEGOTIATE_INFO, + (char *)pneg_inbuf, inbuflen, CIFSMaxBufSize, + (char **)&pneg_rsp, &rsplen); + if (rc == -EOPNOTSUPP) { + /* + * Old Windows versions or Netapp SMB server can return + * not supported error. Client should accept it. + */ + cifs_tcon_dbg(VFS, "Server does not support validate negotiate\n"); + rc = 0; + goto out_free_inbuf; + } else if (rc != 0) { + cifs_tcon_dbg(VFS, "validate protocol negotiate failed: %d\n", + rc); + rc = smb_EIO1(smb_eio_trace_neg_info_fail, rc); + goto out_free_inbuf; + } + + if (rsplen != sizeof(*pneg_rsp)) { + cifs_tcon_dbg(VFS, "Invalid protocol negotiate response size: %d\n", + rsplen); + + /* relax check since Mac returns max bufsize allowed on ioctl */ + if (rsplen > CIFSMaxBufSize || rsplen < sizeof(*pneg_rsp)) { + rc = smb_EIO1(smb_eio_trace_neg_bad_rsplen, rsplen); + goto out_free_rsp; + } + } + + /* check validate negotiate info response matches what we got earlier */ + u16 dialect = le16_to_cpu(pneg_rsp->Dialect); + + if (dialect != server->dialect) { + rc = smb_EIO2(smb_eio_trace_neg_info_dialect, + dialect, server->dialect); + goto vneg_out; + } + + u16 sec_mode = le16_to_cpu(pneg_rsp->SecurityMode); + + if (sec_mode != server->sec_mode) { + rc = smb_EIO2(smb_eio_trace_neg_info_sec_mode, + sec_mode, server->sec_mode); + goto vneg_out; + } + + /* do not validate server guid because not saved at negprot time yet */ + u32 caps = le32_to_cpu(pneg_rsp->Capabilities); + + if ((caps | SMB2_NT_FIND | + SMB2_LARGE_FILES) != server->capabilities) { + rc = smb_EIO2(smb_eio_trace_neg_info_caps, + caps, server->capabilities); + goto vneg_out; + } + + /* validate negotiate successful */ + rc = 0; + cifs_dbg(FYI, "validate negotiate info successful\n"); + goto out_free_rsp; + +vneg_out: + cifs_tcon_dbg(VFS, "protocol revalidation - security settings mismatch\n"); +out_free_rsp: + kfree(pneg_rsp); +out_free_inbuf: + kfree(pneg_inbuf); + return rc; +} + +enum securityEnum +smb2_select_sectype(struct TCP_Server_Info *server, enum securityEnum requested) +{ + switch (requested) { + case Kerberos: + case RawNTLMSSP: + return requested; + case NTLMv2: + return RawNTLMSSP; + case Unspecified: + if (server->sec_ntlmssp && + (global_secflags & CIFSSEC_MAY_NTLMSSP)) + return RawNTLMSSP; + if ((server->sec_kerberos || server->sec_mskerberos || server->sec_iakerb) && + (global_secflags & CIFSSEC_MAY_KRB5)) + return Kerberos; + fallthrough; + default: + return Unspecified; + } +} + +struct SMB2_sess_data { + unsigned int xid; + struct cifs_ses *ses; + struct TCP_Server_Info *server; + struct nls_table *nls_cp; + void (*func)(struct SMB2_sess_data *); + int result; + u64 previous_session; + + /* we will send the SMB in three pieces: + * a fixed length beginning part, an optional + * SPNEGO blob (which can be zero length), and a + * last part which will include the strings + * and rest of bcc area. This allows us to avoid + * a large buffer 17K allocation + */ + int buf0_type; + struct kvec iov[2]; +}; + +static int +SMB2_sess_alloc_buffer(struct SMB2_sess_data *sess_data) +{ + int rc; + struct cifs_ses *ses = sess_data->ses; + struct TCP_Server_Info *server = sess_data->server; + struct smb2_sess_setup_req *req; + unsigned int total_len; + bool is_binding = false; + + rc = smb2_plain_req_init(SMB2_SESSION_SETUP, NULL, server, + (void **) &req, + &total_len); + if (rc) + return rc; + + spin_lock(&ses->ses_lock); + is_binding = (ses->ses_status == SES_GOOD); + spin_unlock(&ses->ses_lock); + + if (is_binding) { + req->hdr.SessionId = cpu_to_le64(ses->Suid); + req->hdr.Flags |= SMB2_FLAGS_SIGNED; + req->PreviousSessionId = 0; + req->Flags = SMB2_SESSION_REQ_FLAG_BINDING; + cifs_dbg(FYI, "Binding to sess id: %llx\n", ses->Suid); + } else { + /* First session, not a reauthenticate */ + req->hdr.SessionId = 0; + /* + * if reconnect, we need to send previous sess id + * otherwise it is 0 + */ + req->PreviousSessionId = cpu_to_le64(sess_data->previous_session); + req->Flags = 0; /* MBZ */ + cifs_dbg(FYI, "Fresh session. Previous: %llx\n", + sess_data->previous_session); + } + + /* enough to enable echos and oplocks and one max size write */ + if (server->credits >= server->max_credits) + req->hdr.CreditRequest = cpu_to_le16(0); + else + req->hdr.CreditRequest = cpu_to_le16( + min_t(int, server->max_credits - + server->credits, 130)); + + /* only one of SMB2 signing flags may be set in SMB2 request */ + if (server->sign) + req->SecurityMode = SMB2_NEGOTIATE_SIGNING_REQUIRED; + else if (global_secflags & CIFSSEC_MAY_SIGN) /* one flag unlike MUST_ */ + req->SecurityMode = SMB2_NEGOTIATE_SIGNING_ENABLED; + else + req->SecurityMode = 0; + +#ifdef CONFIG_CIFS_DFS_UPCALL + req->Capabilities = cpu_to_le32(SMB2_GLOBAL_CAP_DFS); +#else + req->Capabilities = 0; +#endif /* DFS_UPCALL */ + + req->Channel = 0; /* MBZ */ + + sess_data->iov[0].iov_base = (char *)req; + /* 1 for pad */ + sess_data->iov[0].iov_len = total_len - 1; + /* + * This variable will be used to clear the buffer + * allocated above in case of any error in the calling function. + */ + sess_data->buf0_type = CIFS_SMALL_BUFFER; + + return 0; +} + +static void +SMB2_sess_free_buffer(struct SMB2_sess_data *sess_data) +{ + struct kvec *iov = sess_data->iov; + + /* iov[1] is already freed by caller */ + if (sess_data->buf0_type != CIFS_NO_BUFFER && iov[0].iov_base) + memzero_explicit(iov[0].iov_base, iov[0].iov_len); + + free_rsp_buf(sess_data->buf0_type, iov[0].iov_base); + sess_data->buf0_type = CIFS_NO_BUFFER; +} + +static int +SMB2_sess_sendreceive(struct SMB2_sess_data *sess_data) +{ + int rc; + struct smb_rqst rqst; + struct smb2_sess_setup_req *req = sess_data->iov[0].iov_base; + struct kvec rsp_iov = { NULL, 0 }; + + /* Testing shows that buffer offset must be at location of Buffer[0] */ + req->SecurityBufferOffset = + cpu_to_le16(sizeof(struct smb2_sess_setup_req)); + req->SecurityBufferLength = cpu_to_le16(sess_data->iov[1].iov_len); + + memset(&rqst, 0, sizeof(struct smb_rqst)); + rqst.rq_iov = sess_data->iov; + rqst.rq_nvec = 2; + + /* BB add code to build os and lm fields */ + rc = cifs_send_recv(sess_data->xid, sess_data->ses, + sess_data->server, + &rqst, + &sess_data->buf0_type, + CIFS_LOG_ERROR | CIFS_SESS_OP, &rsp_iov); + cifs_small_buf_release(sess_data->iov[0].iov_base); + if (rc == 0) + sess_data->ses->expired_pwd = false; + else if ((rc == -EACCES) || (rc == -EKEYEXPIRED) || (rc == -EKEYREVOKED)) { + if (sess_data->ses->expired_pwd == false) + trace_smb3_key_expired(sess_data->server->hostname, + sess_data->ses->user_name, + sess_data->server->conn_id, + &sess_data->server->dstaddr, rc); + sess_data->ses->expired_pwd = true; + } + + memcpy(&sess_data->iov[0], &rsp_iov, sizeof(struct kvec)); + + return rc; +} + +static int +SMB2_sess_establish_session(struct SMB2_sess_data *sess_data) +{ + int rc = 0; + struct cifs_ses *ses = sess_data->ses; + struct TCP_Server_Info *server = sess_data->server; + + cifs_server_lock(server); + if (server->ops->generate_signingkey) { + rc = server->ops->generate_signingkey(ses, server); + if (rc) { + cifs_dbg(FYI, + "SMB3 session key generation failed\n"); + cifs_server_unlock(server); + return rc; + } + } + if (!server->session_estab) { + server->sequence_number = 0x2; + server->session_estab = true; + } + cifs_server_unlock(server); + + cifs_dbg(FYI, "SMB2/3 session established successfully\n"); + return rc; +} + +#ifdef CONFIG_CIFS_UPCALL +static void +SMB2_auth_kerberos(struct SMB2_sess_data *sess_data) +{ + int rc; + struct cifs_ses *ses = sess_data->ses; + struct TCP_Server_Info *server = sess_data->server; + struct cifs_spnego_msg *msg; + struct key *spnego_key = NULL; + struct smb2_sess_setup_rsp *rsp = NULL; + bool is_binding = false; + + rc = SMB2_sess_alloc_buffer(sess_data); + if (rc) + goto out; + + spnego_key = cifs_get_spnego_key(ses, server); + if (IS_ERR(spnego_key)) { + rc = PTR_ERR(spnego_key); + spnego_key = NULL; + goto out; + } + + msg = spnego_key->payload.data[0]; + /* + * check version field to make sure that cifs.upcall is + * sending us a response in an expected form + */ + if (msg->version != CIFS_SPNEGO_UPCALL_VERSION) { + cifs_dbg(VFS, "bad cifs.upcall version. Expected %d got %d\n", + CIFS_SPNEGO_UPCALL_VERSION, msg->version); + rc = -EKEYREJECTED; + goto out_put_spnego_key; + } + + spin_lock(&ses->ses_lock); + is_binding = (ses->ses_status == SES_GOOD); + spin_unlock(&ses->ses_lock); + + /* + * Per MS-SMB2 3.2.5.3, Session.SessionKey is the first 16 bytes of the + * GSS cryptographic key, right-padded with zero bytes if shorter. + * Allocate at least SMB2_NTLMV2_SESSKEY_SIZE bytes (zeroed) so the KDF + * input buffer is always valid for HMAC-SHA256 even with deprecated + * Kerberos enctypes that return a short session key. + */ + if (unlikely(msg->sesskey_len < SMB2_NTLMV2_SESSKEY_SIZE)) + cifs_dbg(VFS, + "short GSS session key (%u bytes); zero-padding per MS-SMB2 3.2.5.3\n", + msg->sesskey_len); + + kfree_sensitive(ses->auth_key.response); + ses->auth_key.len = max_t(unsigned int, msg->sesskey_len, + SMB2_NTLMV2_SESSKEY_SIZE); + ses->auth_key.response = kzalloc(ses->auth_key.len, GFP_KERNEL); + if (!ses->auth_key.response) { + cifs_dbg(VFS, "%s: can't allocate (%u bytes) memory\n", + __func__, ses->auth_key.len); + ses->auth_key.len = 0; + rc = -ENOMEM; + goto out_put_spnego_key; + } + memcpy(ses->auth_key.response, msg->data, msg->sesskey_len); + + sess_data->iov[1].iov_base = msg->data + msg->sesskey_len; + sess_data->iov[1].iov_len = msg->secblob_len; + + rc = SMB2_sess_sendreceive(sess_data); + if (rc) + goto out_put_spnego_key; + + rsp = (struct smb2_sess_setup_rsp *)sess_data->iov[0].iov_base; + /* keep session id and flags if binding */ + if (!is_binding) { + ses->Suid = le64_to_cpu(rsp->hdr.SessionId); + ses->session_flags = le16_to_cpu(rsp->SessionFlags); + } + + rc = SMB2_sess_establish_session(sess_data); +out_put_spnego_key: + key_invalidate(spnego_key); + key_put(spnego_key); + if (rc) { + kfree_sensitive(ses->auth_key.response); + ses->auth_key.response = NULL; + ses->auth_key.len = 0; + } +out: + sess_data->result = rc; + sess_data->func = NULL; + SMB2_sess_free_buffer(sess_data); +} +#else +static void +SMB2_auth_kerberos(struct SMB2_sess_data *sess_data) +{ + cifs_dbg(VFS, "Kerberos negotiated but upcall support disabled!\n"); + sess_data->result = -EOPNOTSUPP; + sess_data->func = NULL; +} +#endif + +static void +SMB2_sess_auth_rawntlmssp_authenticate(struct SMB2_sess_data *sess_data); + +static void +SMB2_sess_auth_rawntlmssp_negotiate(struct SMB2_sess_data *sess_data) +{ + int rc; + struct cifs_ses *ses = sess_data->ses; + struct TCP_Server_Info *server = sess_data->server; + struct smb2_sess_setup_rsp *rsp = NULL; + unsigned char *ntlmssp_blob = NULL; + bool use_spnego = false; /* else use raw ntlmssp */ + u16 blob_length = 0; + bool is_binding = false; + + /* + * If memory allocation is successful, caller of this function + * frees it. + */ + ses->ntlmssp = kmalloc_obj(struct ntlmssp_auth); + if (!ses->ntlmssp) { + rc = -ENOMEM; + goto out_err; + } + ses->ntlmssp->sesskey_per_smbsess = true; + + rc = SMB2_sess_alloc_buffer(sess_data); + if (rc) + goto out_err; + + rc = build_ntlmssp_smb3_negotiate_blob(&ntlmssp_blob, + &blob_length, ses, server, + sess_data->nls_cp); + if (rc) + goto out; + + if (use_spnego) { + /* BB eventually need to add this */ + cifs_dbg(VFS, "spnego not supported for SMB2 yet\n"); + rc = -EOPNOTSUPP; + goto out; + } + sess_data->iov[1].iov_base = ntlmssp_blob; + sess_data->iov[1].iov_len = blob_length; + + rc = SMB2_sess_sendreceive(sess_data); + rsp = (struct smb2_sess_setup_rsp *)sess_data->iov[0].iov_base; + + /* If true, rc here is expected and not an error */ + if (sess_data->buf0_type != CIFS_NO_BUFFER && + rsp->hdr.Status == STATUS_MORE_PROCESSING_REQUIRED) + rc = 0; + + if (rc) + goto out; + + u16 boff = le16_to_cpu(rsp->SecurityBufferOffset); + + if (offsetof(struct smb2_sess_setup_rsp, Buffer) != boff) { + cifs_dbg(VFS, "Invalid security buffer offset %d\n", boff); + rc = smb_EIO1(smb_eio_trace_sess_buf_off, boff); + goto out; + } + rc = decode_ntlmssp_challenge(rsp->Buffer, + le16_to_cpu(rsp->SecurityBufferLength), ses); + if (rc) + goto out; + + cifs_dbg(FYI, "rawntlmssp session setup challenge phase\n"); + + spin_lock(&ses->ses_lock); + is_binding = (ses->ses_status == SES_GOOD); + spin_unlock(&ses->ses_lock); + + /* keep existing ses id and flags if binding */ + if (!is_binding) { + ses->Suid = le64_to_cpu(rsp->hdr.SessionId); + ses->session_flags = le16_to_cpu(rsp->SessionFlags); + } + +out: + kfree_sensitive(ntlmssp_blob); + SMB2_sess_free_buffer(sess_data); + if (!rc) { + sess_data->result = 0; + sess_data->func = SMB2_sess_auth_rawntlmssp_authenticate; + return; + } +out_err: + kfree_sensitive(ses->ntlmssp); + ses->ntlmssp = NULL; + sess_data->result = rc; + sess_data->func = NULL; +} + +static void +SMB2_sess_auth_rawntlmssp_authenticate(struct SMB2_sess_data *sess_data) +{ + int rc; + struct cifs_ses *ses = sess_data->ses; + struct TCP_Server_Info *server = sess_data->server; + struct smb2_sess_setup_req *req; + struct smb2_sess_setup_rsp *rsp = NULL; + unsigned char *ntlmssp_blob = NULL; + bool use_spnego = false; /* else use raw ntlmssp */ + u16 blob_length = 0; + bool is_binding = false; + + rc = SMB2_sess_alloc_buffer(sess_data); + if (rc) + goto out; + + req = (struct smb2_sess_setup_req *) sess_data->iov[0].iov_base; + req->hdr.SessionId = cpu_to_le64(ses->Suid); + + rc = build_ntlmssp_auth_blob(&ntlmssp_blob, &blob_length, + ses, server, + sess_data->nls_cp); + if (rc) { + cifs_dbg(FYI, "build_ntlmssp_auth_blob failed %d\n", rc); + goto out; + } + + if (use_spnego) { + /* BB eventually need to add this */ + cifs_dbg(VFS, "spnego not supported for SMB2 yet\n"); + rc = -EOPNOTSUPP; + goto out; + } + sess_data->iov[1].iov_base = ntlmssp_blob; + sess_data->iov[1].iov_len = blob_length; + + rc = SMB2_sess_sendreceive(sess_data); + if (rc) + goto out; + + rsp = (struct smb2_sess_setup_rsp *)sess_data->iov[0].iov_base; + + spin_lock(&ses->ses_lock); + is_binding = (ses->ses_status == SES_GOOD); + spin_unlock(&ses->ses_lock); + + /* keep existing ses id and flags if binding */ + if (!is_binding) { + ses->Suid = le64_to_cpu(rsp->hdr.SessionId); + ses->session_flags = le16_to_cpu(rsp->SessionFlags); + } + + rc = SMB2_sess_establish_session(sess_data); +#ifdef CONFIG_CIFS_DEBUG_DUMP_KEYS + if (ses->server->dialect < SMB30_PROT_ID) { + cifs_dbg(VFS, "%s: dumping generated SMB2 session keys\n", __func__); + /* + * The session id is opaque in terms of endianness, so we can't + * print it as a long long. we dump it as we got it on the wire + */ + cifs_dbg(VFS, "Session Id %*ph\n", (int)sizeof(ses->Suid), + &ses->Suid); + cifs_dbg(VFS, "Session Key %*ph\n", + SMB2_NTLMV2_SESSKEY_SIZE, ses->auth_key.response); + cifs_dbg(VFS, "Signing Key %*ph\n", + SMB3_SIGN_KEY_SIZE, ses->auth_key.response); + } +#endif +out: + kfree_sensitive(ntlmssp_blob); + SMB2_sess_free_buffer(sess_data); + kfree_sensitive(ses->ntlmssp); + ses->ntlmssp = NULL; + sess_data->result = rc; + sess_data->func = NULL; +} + +static int +SMB2_select_sec(struct SMB2_sess_data *sess_data) +{ + int type; + struct cifs_ses *ses = sess_data->ses; + struct TCP_Server_Info *server = sess_data->server; + + type = smb2_select_sectype(server, ses->sectype); + cifs_dbg(FYI, "sess setup type %d\n", type); + if (type == Unspecified) { + cifs_dbg(VFS, "Unable to select appropriate authentication method!\n"); + return -EINVAL; + } + + switch (type) { + case Kerberos: + sess_data->func = SMB2_auth_kerberos; + break; + case RawNTLMSSP: + sess_data->func = SMB2_sess_auth_rawntlmssp_negotiate; + break; + default: + cifs_dbg(VFS, "secType %d not supported!\n", type); + return -EOPNOTSUPP; + } + + return 0; +} + +int +SMB2_sess_setup(const unsigned int xid, struct cifs_ses *ses, + struct TCP_Server_Info *server, + const struct nls_table *nls_cp) +{ + int rc = 0; + struct SMB2_sess_data *sess_data; + + cifs_dbg(FYI, "Session Setup\n"); + + if (!server) { + WARN(1, "%s: server is NULL!\n", __func__); + return smb_EIO(smb_eio_trace_null_pointers); + } + + sess_data = kzalloc_obj(struct SMB2_sess_data); + if (!sess_data) + return -ENOMEM; + + sess_data->xid = xid; + sess_data->ses = ses; + sess_data->server = server; + sess_data->buf0_type = CIFS_NO_BUFFER; + sess_data->nls_cp = (struct nls_table *) nls_cp; + sess_data->previous_session = ses->Suid; + + rc = SMB2_select_sec(sess_data); + if (rc) + goto out; + + /* + * Initialize the session hash with the server one. + */ + memcpy(ses->preauth_sha_hash, server->preauth_sha_hash, + SMB2_PREAUTH_HASH_SIZE); + + while (sess_data->func) + sess_data->func(sess_data); + + if ((ses->session_flags & SMB2_SESSION_FLAG_IS_GUEST) && (ses->sign)) + cifs_server_dbg(VFS, "signing requested but authenticated as guest\n"); + rc = sess_data->result; +out: + kfree_sensitive(sess_data); + return rc; +} + +int +SMB2_logoff(const unsigned int xid, struct cifs_ses *ses) +{ + struct smb_rqst rqst; + struct smb2_logoff_req *req; /* response is also trivial struct */ + int rc = 0; + struct TCP_Server_Info *server; + int flags = 0; + unsigned int total_len; + struct kvec iov[1]; + struct kvec rsp_iov; + int resp_buf_type; + + cifs_dbg(FYI, "disconnect session %p\n", ses); + + if (!ses || !ses->server) + return smb_EIO(smb_eio_trace_null_pointers); + server = ses->server; + + /* no need to send SMB logoff if uid already closed due to reconnect */ + spin_lock(&ses->chan_lock); + if (CIFS_ALL_CHANS_NEED_RECONNECT(ses)) { + spin_unlock(&ses->chan_lock); + goto smb2_session_already_dead; + } + spin_unlock(&ses->chan_lock); + + rc = smb2_plain_req_init(SMB2_LOGOFF, NULL, ses->server, + (void **) &req, &total_len); + if (rc) + return rc; + + /* since no tcon, smb2_init can not do this, so do here */ + req->hdr.SessionId = cpu_to_le64(ses->Suid); + + if (ses->session_flags & SMB2_SESSION_FLAG_ENCRYPT_DATA) + flags |= CIFS_TRANSFORM_REQ; + else if (server->sign) + req->hdr.Flags |= SMB2_FLAGS_SIGNED; + + flags |= CIFS_NO_RSP_BUF; + + iov[0].iov_base = (char *)req; + iov[0].iov_len = total_len; + + memset(&rqst, 0, sizeof(struct smb_rqst)); + rqst.rq_iov = iov; + rqst.rq_nvec = 1; + + rc = cifs_send_recv(xid, ses, ses->server, + &rqst, &resp_buf_type, flags, &rsp_iov); + cifs_small_buf_release(req); + /* + * No tcon so can't do + * cifs_stats_inc(&tcon->stats.smb2_stats.smb2_com_fail[SMB2...]); + */ + +smb2_session_already_dead: + return rc; +} + +static inline void cifs_stats_fail_inc(struct cifs_tcon *tcon, uint16_t code) +{ + cifs_stats_inc(&tcon->stats.smb2_stats.smb2_com_failed[code]); +} + +#define MAX_SHARENAME_LENGTH (255 /* server */ + 80 /* share */ + 1 /* NULL */) + +/* These are similar values to what Windows uses */ +static inline void init_copy_chunk_defaults(struct cifs_tcon *tcon) +{ + tcon->max_chunks = 256; + tcon->max_bytes_chunk = 1048576; + tcon->max_bytes_copy = 16777216; +} + +int +SMB2_tcon(const unsigned int xid, struct cifs_ses *ses, const char *tree, + struct cifs_tcon *tcon, const struct nls_table *cp) +{ + struct smb_rqst rqst; + struct smb2_tree_connect_req *req; + struct smb2_tree_connect_rsp *rsp = NULL; + struct kvec iov[2]; + struct kvec rsp_iov = { NULL, 0 }; + int rc = 0; + int resp_buftype; + int unc_path_len; + __le16 *unc_path = NULL; + int flags = 0; + unsigned int total_len; + struct TCP_Server_Info *server = cifs_pick_channel(ses); + + cifs_dbg(FYI, "TCON\n"); + + if (!server || !tree) + return smb_EIO(smb_eio_trace_null_pointers); + + unc_path = kmalloc(MAX_SHARENAME_LENGTH * 2, GFP_KERNEL); + if (unc_path == NULL) + return -ENOMEM; + + unc_path_len = cifs_strtoUTF16(unc_path, tree, strlen(tree), cp); + if (unc_path_len <= 0) { + rc = -EINVAL; + goto free_unc_path; + } + unc_path_len *= 2; + + /* SMB2 TREE_CONNECT request must be called with TreeId == 0 */ + tcon->tid = 0; + atomic_set(&tcon->num_remote_opens, 0); + rc = smb2_plain_req_init(SMB2_TREE_CONNECT, tcon, server, + (void **) &req, &total_len); + if (rc) + goto free_unc_path; + + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; + + iov[0].iov_base = (char *)req; + /* 1 for pad */ + iov[0].iov_len = total_len - 1; + + /* Testing shows that buffer offset must be at location of Buffer[0] */ + req->PathOffset = cpu_to_le16(sizeof(struct smb2_tree_connect_req)); + req->PathLength = cpu_to_le16(unc_path_len); + iov[1].iov_base = unc_path; + iov[1].iov_len = unc_path_len; + + /* + * 3.11 tcon req must be signed if not encrypted. See MS-SMB2 3.2.4.1.1 + * unless it is guest or anonymous user. See MS-SMB2 3.2.5.3.1 + * (Samba servers don't always set the flag so also check if null user) + */ + if ((server->dialect == SMB311_PROT_ID) && + !smb3_encryption_required(tcon) && + !(ses->session_flags & + (SMB2_SESSION_FLAG_IS_GUEST|SMB2_SESSION_FLAG_IS_NULL)) && + ((ses->user_name != NULL) || (ses->sectype == Kerberos))) + req->hdr.Flags |= SMB2_FLAGS_SIGNED; + + memset(&rqst, 0, sizeof(struct smb_rqst)); + rqst.rq_iov = iov; + rqst.rq_nvec = 2; + + /* Need 64 for max size write so ask for more in case not there yet */ + if (server->credits >= server->max_credits) + req->hdr.CreditRequest = cpu_to_le16(0); + else + req->hdr.CreditRequest = cpu_to_le16( + min_t(int, server->max_credits - + server->credits, 64)); + + rc = cifs_send_recv(xid, ses, server, + &rqst, &resp_buftype, flags, &rsp_iov); + cifs_small_buf_release(req); + rsp = (struct smb2_tree_connect_rsp *)rsp_iov.iov_base; + trace_smb3_tcon(xid, tcon->tid, ses->Suid, tree, rc); + if ((rc != 0) || (rsp == NULL)) { + cifs_stats_fail_inc(tcon, SMB2_TREE_CONNECT_HE); + tcon->need_reconnect = true; + goto tcon_error_exit; + } + + switch (rsp->ShareType) { + case SMB2_SHARE_TYPE_DISK: + cifs_dbg(FYI, "connection to disk share\n"); + break; + case SMB2_SHARE_TYPE_PIPE: + tcon->pipe = true; + cifs_dbg(FYI, "connection to pipe share\n"); + break; + case SMB2_SHARE_TYPE_PRINT: + tcon->print = true; + cifs_dbg(FYI, "connection to printer\n"); + break; + default: + cifs_server_dbg(VFS, "unknown share type %d\n", rsp->ShareType); + rc = -EOPNOTSUPP; + goto tcon_error_exit; + } + + tcon->share_flags = le32_to_cpu(rsp->ShareFlags); + tcon->capabilities = rsp->Capabilities; /* we keep caps little endian */ + tcon->maximal_access = le32_to_cpu(rsp->MaximalAccess); + tcon->tid = le32_to_cpu(rsp->hdr.Id.SyncId.TreeId); + strscpy(tcon->tree_name, tree, sizeof(tcon->tree_name)); + + if ((rsp->Capabilities & SMB2_SHARE_CAP_DFS) && + ((tcon->share_flags & SHI1005_FLAGS_DFS) == 0)) + cifs_tcon_dbg(VFS, "DFS capability contradicts DFS flag\n"); + + if (tcon->seal && + !(server->capabilities & SMB2_GLOBAL_CAP_ENCRYPTION)) + cifs_tcon_dbg(VFS, "Encryption is requested but not supported\n"); + + init_copy_chunk_defaults(tcon); + if (server->ops->validate_negotiate) + rc = server->ops->validate_negotiate(xid, tcon); + if (rc == 0) /* See MS-SMB2 2.2.10 and 3.2.5.5 */ + if (tcon->share_flags & SMB2_SHAREFLAG_ISOLATED_TRANSPORT) + server->nosharesock = true; +tcon_exit: + + free_rsp_buf(resp_buftype, rsp); +free_unc_path: + kfree(unc_path); + return rc; + +tcon_error_exit: + if (rsp && rsp->hdr.Status == STATUS_BAD_NETWORK_NAME) + cifs_dbg(VFS | ONCE, "BAD_NETWORK_NAME: %s\n", tree); + goto tcon_exit; +} + +int +SMB2_tdis(const unsigned int xid, struct cifs_tcon *tcon) +{ + struct smb_rqst rqst; + struct smb2_tree_disconnect_req *req; /* response is trivial */ + int rc = 0; + struct cifs_ses *ses = tcon->ses; + struct TCP_Server_Info *server = cifs_pick_channel(ses); + int flags = 0; + unsigned int total_len; + struct kvec iov[1]; + struct kvec rsp_iov; + int resp_buf_type; + + cifs_dbg(FYI, "Tree Disconnect\n"); + + if (!ses || !(ses->server)) + return smb_EIO(smb_eio_trace_null_pointers); + + trace_smb3_tdis_enter(xid, tcon->tid, ses->Suid, tcon->tree_name); + spin_lock(&ses->chan_lock); + if ((tcon->need_reconnect) || + (CIFS_ALL_CHANS_NEED_RECONNECT(tcon->ses))) { + spin_unlock(&ses->chan_lock); + return 0; + } + spin_unlock(&ses->chan_lock); + + invalidate_all_cached_dirs(tcon, true); + + rc = smb2_plain_req_init(SMB2_TREE_DISCONNECT, tcon, server, + (void **) &req, + &total_len); + if (rc) + return rc; + + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; + + flags |= CIFS_NO_RSP_BUF; + + iov[0].iov_base = (char *)req; + iov[0].iov_len = total_len; + + memset(&rqst, 0, sizeof(struct smb_rqst)); + rqst.rq_iov = iov; + rqst.rq_nvec = 1; + + rc = cifs_send_recv(xid, ses, server, + &rqst, &resp_buf_type, flags, &rsp_iov); + cifs_small_buf_release(req); + if (rc) { + cifs_stats_fail_inc(tcon, SMB2_TREE_DISCONNECT_HE); + trace_smb3_tdis_err(xid, tcon->tid, ses->Suid, rc); + } + trace_smb3_tdis_done(xid, tcon->tid, ses->Suid); + + return rc; +} + +static create_durable_req_t * +create_durable_buf(void) +{ + create_durable_req_t *buf; + + buf = kzalloc_obj(create_durable_req_t); + if (!buf) + return NULL; + + buf->ccontext.DataOffset = cpu_to_le16(offsetof + (create_durable_req_t, Data)); + buf->ccontext.DataLength = cpu_to_le32(16); + buf->ccontext.NameOffset = cpu_to_le16(offsetof + (create_durable_req_t, Name)); + buf->ccontext.NameLength = cpu_to_le16(4); + /* SMB2_CREATE_DURABLE_HANDLE_REQUEST is "DHnQ" */ + buf->Name[0] = 'D'; + buf->Name[1] = 'H'; + buf->Name[2] = 'n'; + buf->Name[3] = 'Q'; + return buf; +} + +static create_durable_req_t * +create_reconnect_durable_buf(struct cifs_fid *fid) +{ + create_durable_req_t *buf; + + buf = kzalloc_obj(create_durable_req_t); + if (!buf) + return NULL; + + buf->ccontext.DataOffset = cpu_to_le16(offsetof + (create_durable_req_t, Data)); + buf->ccontext.DataLength = cpu_to_le32(16); + buf->ccontext.NameOffset = cpu_to_le16(offsetof + (create_durable_req_t, Name)); + buf->ccontext.NameLength = cpu_to_le16(4); + buf->Data.Fid.PersistentFileId = fid->persistent_fid; + buf->Data.Fid.VolatileFileId = fid->volatile_fid; + /* SMB2_CREATE_DURABLE_HANDLE_RECONNECT is "DHnC" */ + buf->Name[0] = 'D'; + buf->Name[1] = 'H'; + buf->Name[2] = 'n'; + buf->Name[3] = 'C'; + return buf; +} + +static void +parse_query_id_ctxt(struct create_context *cc, struct smb2_file_all_info *buf) +{ + struct create_disk_id_rsp *pdisk_id = (struct create_disk_id_rsp *)cc; + + cifs_dbg(FYI, "parse query id context 0x%llx 0x%llx\n", + pdisk_id->DiskFileId, pdisk_id->VolumeId); + buf->IndexNumber = pdisk_id->DiskFileId; +} + +static void +parse_posix_ctxt(struct create_context *cc, struct smb2_file_all_info *info, + struct create_posix_rsp *posix) +{ + int sid_len; + u8 *beg = (u8 *)cc + le16_to_cpu(cc->DataOffset); + u8 *end = beg + le32_to_cpu(cc->DataLength); + u8 *sid; + + memset(posix, 0, sizeof(*posix)); + + posix->nlink = get_unaligned_le32(beg); + posix->reparse_tag = get_unaligned_le32(beg + 4); + posix->mode = get_unaligned_le32(beg + 8); + + sid = beg + 12; + sid_len = posix_info_sid_size(sid, end); + if (sid_len < 0) { + cifs_dbg(VFS, "bad owner sid in posix create response\n"); + return; + } + memcpy(&posix->owner, sid, sid_len); + + sid = sid + sid_len; + sid_len = posix_info_sid_size(sid, end); + if (sid_len < 0) { + cifs_dbg(VFS, "bad group sid in posix create response\n"); + return; + } + memcpy(&posix->group, sid, sid_len); + + cifs_dbg(FYI, "nlink=%d mode=%o reparse_tag=%x\n", + posix->nlink, posix->mode, posix->reparse_tag); +} + +int smb2_parse_contexts(struct TCP_Server_Info *server, + struct kvec *rsp_iov, + __u16 *epoch, + char *lease_key, __u8 *oplock, + struct smb2_file_all_info *buf, + struct create_posix_rsp *posix) +{ + struct smb2_create_rsp *rsp = rsp_iov->iov_base; + struct create_context *cc; + size_t rem, off, len; + size_t doff, dlen; + size_t noff, nlen; + char *name; + static const char smb3_create_tag_posix[] = { + 0x93, 0xAD, 0x25, 0x50, 0x9C, + 0xB4, 0x11, 0xE7, 0xB4, 0x23, 0x83, + 0xDE, 0x96, 0x8B, 0xCD, 0x7C + }; + + *oplock = 0; + + off = le32_to_cpu(rsp->CreateContextsOffset); + rem = le32_to_cpu(rsp->CreateContextsLength); + if (check_add_overflow(off, rem, &len) || len > rsp_iov->iov_len) + return -EINVAL; + cc = (struct create_context *)((u8 *)rsp + off); + + /* Initialize inode number to 0 in case no valid data in qfid context */ + if (buf) + buf->IndexNumber = 0; + + while (rem >= sizeof(*cc)) { + doff = le16_to_cpu(cc->DataOffset); + dlen = le32_to_cpu(cc->DataLength); + if (check_add_overflow(doff, dlen, &len) || len > rem) + return -EINVAL; + + noff = le16_to_cpu(cc->NameOffset); + nlen = le16_to_cpu(cc->NameLength); + if (noff + nlen > doff) + return -EINVAL; + + name = (char *)cc + noff; + switch (nlen) { + case 4: + if (!strncmp(name, SMB2_CREATE_REQUEST_LEASE, 4)) { + *oplock = server->ops->parse_lease_buf(cc, epoch, + lease_key); + } else if (buf && + !strncmp(name, SMB2_CREATE_QUERY_ON_DISK_ID, 4)) { + parse_query_id_ctxt(cc, buf); + } + break; + case 16: + if (posix && !memcmp(name, smb3_create_tag_posix, 16)) + parse_posix_ctxt(cc, buf, posix); + break; + default: + cifs_dbg(FYI, "%s: unhandled context (nlen=%zu dlen=%zu)\n", + __func__, nlen, dlen); + if (IS_ENABLED(CONFIG_CIFS_DEBUG2)) + cifs_dump_mem("context data: ", cc, dlen); + break; + } + + off = le32_to_cpu(cc->Next); + if (!off) + break; + if (check_sub_overflow(rem, off, &rem)) + return -EINVAL; + cc = (struct create_context *)((u8 *)cc + off); + } + + if (rsp->OplockLevel != SMB2_OPLOCK_LEVEL_LEASE) + *oplock = rsp->OplockLevel; + + return 0; +} + +static int +add_lease_context(struct TCP_Server_Info *server, + struct smb2_create_req *req, + struct kvec *iov, + unsigned int *num_iovec, + u8 *lease_key, + __u8 *oplock, + u8 *parent_lease_key, + __le32 flags) +{ + unsigned int num = *num_iovec; + + iov[num].iov_base = server->ops->create_lease_buf(lease_key, *oplock, + parent_lease_key, flags); + if (iov[num].iov_base == NULL) + return -ENOMEM; + iov[num].iov_len = server->vals->create_lease_size; + req->RequestedOplockLevel = SMB2_OPLOCK_LEVEL_LEASE; + *num_iovec = num + 1; + return 0; +} + +static struct create_durable_req_v2 * +create_durable_v2_buf(struct cifs_open_parms *oparms) +{ + struct cifs_fid *pfid = oparms->fid; + struct create_durable_req_v2 *buf; + + buf = kzalloc_obj(struct create_durable_req_v2); + if (!buf) + return NULL; + + buf->ccontext.DataOffset = cpu_to_le16(offsetof + (struct create_durable_req_v2, dcontext)); + buf->ccontext.DataLength = cpu_to_le32(sizeof(struct durable_context_v2_req)); + buf->ccontext.NameOffset = cpu_to_le16(offsetof + (struct create_durable_req_v2, Name)); + buf->ccontext.NameLength = cpu_to_le16(4); + + /* + * NB: Handle timeout defaults to 0, which allows server to choose + * (most servers default to 120 seconds) and most clients default to 0. + * This can be overridden at mount ("handletimeout=") if the user wants + * a different persistent (or resilient) handle timeout for all opens + * on a particular SMB3 mount. + */ + buf->dcontext.Timeout = cpu_to_le32(oparms->tcon->handle_timeout); + buf->dcontext.Flags = cpu_to_le32(SMB2_DHANDLE_FLAG_PERSISTENT); + + /* for replay, we should not overwrite the existing create guid */ + if (!oparms->replay) { + generate_random_uuid(buf->dcontext.CreateGuid); + memcpy(pfid->create_guid, buf->dcontext.CreateGuid, 16); + } else + memcpy(buf->dcontext.CreateGuid, pfid->create_guid, 16); + + /* SMB2_CREATE_DURABLE_HANDLE_REQUEST is "DH2Q" */ + buf->Name[0] = 'D'; + buf->Name[1] = 'H'; + buf->Name[2] = '2'; + buf->Name[3] = 'Q'; + return buf; +} + +static struct create_durable_handle_reconnect_v2 * +create_reconnect_durable_v2_buf(struct cifs_fid *fid) +{ + struct create_durable_handle_reconnect_v2 *buf; + + buf = kzalloc_obj(struct create_durable_handle_reconnect_v2); + if (!buf) + return NULL; + + buf->ccontext.DataOffset = + cpu_to_le16(offsetof(struct create_durable_handle_reconnect_v2, + dcontext)); + buf->ccontext.DataLength = + cpu_to_le32(sizeof(struct durable_reconnect_context_v2)); + buf->ccontext.NameOffset = + cpu_to_le16(offsetof(struct create_durable_handle_reconnect_v2, + Name)); + buf->ccontext.NameLength = cpu_to_le16(4); + + buf->dcontext.Fid.PersistentFileId = fid->persistent_fid; + buf->dcontext.Fid.VolatileFileId = fid->volatile_fid; + buf->dcontext.Flags = cpu_to_le32(SMB2_DHANDLE_FLAG_PERSISTENT); + memcpy(buf->dcontext.CreateGuid, fid->create_guid, 16); + + /* SMB2_CREATE_DURABLE_HANDLE_RECONNECT_V2 is "DH2C" */ + buf->Name[0] = 'D'; + buf->Name[1] = 'H'; + buf->Name[2] = '2'; + buf->Name[3] = 'C'; + return buf; +} + +static int +add_durable_v2_context(struct kvec *iov, unsigned int *num_iovec, + struct cifs_open_parms *oparms) +{ + unsigned int num = *num_iovec; + + iov[num].iov_base = create_durable_v2_buf(oparms); + if (iov[num].iov_base == NULL) + return -ENOMEM; + iov[num].iov_len = sizeof(struct create_durable_req_v2); + *num_iovec = num + 1; + return 0; +} + +static int +add_durable_reconnect_v2_context(struct kvec *iov, unsigned int *num_iovec, + struct cifs_open_parms *oparms) +{ + unsigned int num = *num_iovec; + + /* indicate that we don't need to relock the file */ + oparms->reconnect = false; + + iov[num].iov_base = create_reconnect_durable_v2_buf(oparms->fid); + if (iov[num].iov_base == NULL) + return -ENOMEM; + iov[num].iov_len = sizeof(struct create_durable_handle_reconnect_v2); + *num_iovec = num + 1; + return 0; +} + +static int +add_durable_context(struct kvec *iov, unsigned int *num_iovec, + struct cifs_open_parms *oparms, bool use_persistent) +{ + unsigned int num = *num_iovec; + + if (use_persistent) { + if (oparms->reconnect) + return add_durable_reconnect_v2_context(iov, num_iovec, + oparms); + else + return add_durable_v2_context(iov, num_iovec, oparms); + } + + if (oparms->reconnect) { + iov[num].iov_base = create_reconnect_durable_buf(oparms->fid); + /* indicate that we don't need to relock the file */ + oparms->reconnect = false; + } else + iov[num].iov_base = create_durable_buf(); + if (iov[num].iov_base == NULL) + return -ENOMEM; + iov[num].iov_len = sizeof(create_durable_req_t); + *num_iovec = num + 1; + return 0; +} + +/* See MS-SMB2 2.2.13.2.7 */ +static struct crt_twarp_ctxt * +create_twarp_buf(__u64 timewarp) +{ + struct crt_twarp_ctxt *buf; + + buf = kzalloc_obj(struct crt_twarp_ctxt); + if (!buf) + return NULL; + + buf->ccontext.DataOffset = cpu_to_le16(offsetof + (struct crt_twarp_ctxt, Timestamp)); + buf->ccontext.DataLength = cpu_to_le32(8); + buf->ccontext.NameOffset = cpu_to_le16(offsetof + (struct crt_twarp_ctxt, Name)); + buf->ccontext.NameLength = cpu_to_le16(4); + /* SMB2_CREATE_TIMEWARP_TOKEN is "TWrp" */ + buf->Name[0] = 'T'; + buf->Name[1] = 'W'; + buf->Name[2] = 'r'; + buf->Name[3] = 'p'; + buf->Timestamp = cpu_to_le64(timewarp); + return buf; +} + +/* See MS-SMB2 2.2.13.2.7 */ +static int +add_twarp_context(struct kvec *iov, unsigned int *num_iovec, __u64 timewarp) +{ + unsigned int num = *num_iovec; + + iov[num].iov_base = create_twarp_buf(timewarp); + if (iov[num].iov_base == NULL) + return -ENOMEM; + iov[num].iov_len = sizeof(struct crt_twarp_ctxt); + *num_iovec = num + 1; + return 0; +} + +/* See http://technet.microsoft.com/en-us/library/hh509017(v=ws.10).aspx */ +static void setup_owner_group_sids(char *buf) +{ + struct owner_group_sids *sids = (struct owner_group_sids *)buf; + + /* Populate the user ownership fields S-1-5-88-1 */ + sids->owner.Revision = 1; + sids->owner.NumAuth = 3; + sids->owner.Authority[5] = 5; + sids->owner.SubAuthorities[0] = cpu_to_le32(88); + sids->owner.SubAuthorities[1] = cpu_to_le32(1); + sids->owner.SubAuthorities[2] = cpu_to_le32(current_fsuid().val); + + /* Populate the group ownership fields S-1-5-88-2 */ + sids->group.Revision = 1; + sids->group.NumAuth = 3; + sids->group.Authority[5] = 5; + sids->group.SubAuthorities[0] = cpu_to_le32(88); + sids->group.SubAuthorities[1] = cpu_to_le32(2); + sids->group.SubAuthorities[2] = cpu_to_le32(current_fsgid().val); + + cifs_dbg(FYI, "owner S-1-5-88-1-%d, group S-1-5-88-2-%d\n", current_fsuid().val, current_fsgid().val); +} + +/* See MS-SMB2 2.2.13.2.2 and MS-DTYP 2.4.6 */ +static struct crt_sd_ctxt * +create_sd_buf(umode_t mode, bool set_owner, unsigned int *len) +{ + struct crt_sd_ctxt *buf; + __u8 *ptr, *aclptr; + unsigned int acelen, acl_size, ace_count; + unsigned int owner_offset = 0; + unsigned int group_offset = 0; + struct smb3_acl acl = {}; + + *len = round_up(sizeof(struct crt_sd_ctxt) + (sizeof(struct smb_ace) * 4), 8); + + if (set_owner) { + /* sizeof(struct owner_group_sids) is already multiple of 8 so no need to round */ + *len += sizeof(struct owner_group_sids); + } + + buf = kzalloc(*len, GFP_KERNEL); + if (buf == NULL) + return buf; + + ptr = (__u8 *)&buf[1]; + if (set_owner) { + /* offset fields are from beginning of security descriptor not of create context */ + owner_offset = ptr - (__u8 *)&buf->sd; + buf->sd.OffsetOwner = cpu_to_le32(owner_offset); + group_offset = owner_offset + offsetof(struct owner_group_sids, group); + buf->sd.OffsetGroup = cpu_to_le32(group_offset); + + setup_owner_group_sids(ptr); + ptr += sizeof(struct owner_group_sids); + } else { + buf->sd.OffsetOwner = 0; + buf->sd.OffsetGroup = 0; + } + + buf->ccontext.DataOffset = cpu_to_le16(offsetof(struct crt_sd_ctxt, sd)); + buf->ccontext.NameOffset = cpu_to_le16(offsetof(struct crt_sd_ctxt, Name)); + buf->ccontext.NameLength = cpu_to_le16(4); + /* SMB2_CREATE_SD_BUFFER_TOKEN is "SecD" */ + buf->Name[0] = 'S'; + buf->Name[1] = 'e'; + buf->Name[2] = 'c'; + buf->Name[3] = 'D'; + buf->sd.Revision = 1; /* Must be one see MS-DTYP 2.4.6 */ + + /* + * ACL is "self relative" ie ACL is stored in contiguous block of memory + * and "DP" ie the DACL is present + */ + buf->sd.Control = cpu_to_le16(ACL_CONTROL_SR | ACL_CONTROL_DP); + + /* offset owner, group and Sbz1 and SACL are all zero */ + buf->sd.OffsetDacl = cpu_to_le32(ptr - (__u8 *)&buf->sd); + /* Ship the ACL for now. we will copy it into buf later. */ + aclptr = ptr; + ptr += sizeof(struct smb3_acl); + + /* create one ACE to hold the mode embedded in reserved special SID */ + acelen = setup_special_mode_ACE((struct smb_ace *)ptr, false, (__u64)mode); + ptr += acelen; + acl_size = acelen + sizeof(struct smb3_acl); + ace_count = 1; + + if (set_owner) { + /* we do not need to reallocate buffer to add the two more ACEs. plenty of space */ + acelen = setup_special_user_owner_ACE((struct smb_ace *)ptr); + ptr += acelen; + acl_size += acelen; + ace_count += 1; + } + + /* and one more ACE to allow access for authenticated users */ + acelen = setup_authusers_ACE((struct smb_ace *)ptr); + ptr += acelen; + acl_size += acelen; + ace_count += 1; + + acl.AclRevision = ACL_REVISION; /* See 2.4.4.1 of MS-DTYP */ + acl.AclSize = cpu_to_le16(acl_size); + acl.AceCount = cpu_to_le16(ace_count); + /* acl.Sbz1 and Sbz2 MBZ so are not set here, but initialized above */ + memcpy(aclptr, &acl, sizeof(struct smb3_acl)); + + buf->ccontext.DataLength = cpu_to_le32(ptr - (__u8 *)&buf->sd); + *len = round_up((unsigned int)(ptr - (__u8 *)buf), 8); + + return buf; +} + +static int +add_sd_context(struct kvec *iov, unsigned int *num_iovec, umode_t mode, bool set_owner) +{ + unsigned int num = *num_iovec; + unsigned int len = 0; + + iov[num].iov_base = create_sd_buf(mode, set_owner, &len); + if (iov[num].iov_base == NULL) + return -ENOMEM; + iov[num].iov_len = len; + *num_iovec = num + 1; + return 0; +} + +static struct crt_query_id_ctxt * +create_query_id_buf(void) +{ + struct crt_query_id_ctxt *buf; + + buf = kzalloc_obj(struct crt_query_id_ctxt); + if (!buf) + return NULL; + + buf->ccontext.DataOffset = cpu_to_le16(0); + buf->ccontext.DataLength = cpu_to_le32(0); + buf->ccontext.NameOffset = cpu_to_le16(offsetof + (struct crt_query_id_ctxt, Name)); + buf->ccontext.NameLength = cpu_to_le16(4); + /* SMB2_CREATE_QUERY_ON_DISK_ID is "QFid" */ + buf->Name[0] = 'Q'; + buf->Name[1] = 'F'; + buf->Name[2] = 'i'; + buf->Name[3] = 'd'; + return buf; +} + +/* See MS-SMB2 2.2.13.2.9 */ +static int +add_query_id_context(struct kvec *iov, unsigned int *num_iovec) +{ + unsigned int num = *num_iovec; + + iov[num].iov_base = create_query_id_buf(); + if (iov[num].iov_base == NULL) + return -ENOMEM; + iov[num].iov_len = sizeof(struct crt_query_id_ctxt); + *num_iovec = num + 1; + return 0; +} + +static void add_ea_context(struct cifs_open_parms *oparms, + struct kvec *rq_iov, unsigned int *num_iovs) +{ + struct kvec *iov = oparms->ea_cctx; + + if (iov && iov->iov_base && iov->iov_len) { + rq_iov[(*num_iovs)++] = *iov; + memset(iov, 0, sizeof(*iov)); + } +} + +static int +alloc_path_with_tree_prefix(__le16 **out_path, int *out_size, int *out_len, + const char *treename, const __le16 *path) +{ + int treename_len, path_len; + struct nls_table *cp; + const __le16 sep[] = {cpu_to_le16('\\'), cpu_to_le16(0x0000)}; + + /* + * skip leading "\\" + */ + treename_len = strlen(treename); + if (treename_len < 2 || !(treename[0] == '\\' && treename[1] == '\\')) + return -EINVAL; + + treename += 2; + treename_len -= 2; + + path_len = UniStrnlen((wchar_t *)path, PATH_MAX); + + /* make room for one path separator only if @path isn't empty */ + *out_len = treename_len + (path[0] ? 1 : 0) + path_len; + + /* + * final path needs to be 8-byte aligned as specified in + * MS-SMB2 2.2.13 SMB2 CREATE Request. + */ + *out_size = round_up(*out_len * sizeof(__le16), 8); + *out_path = kzalloc(*out_size + sizeof(__le16) /* null */, GFP_KERNEL); + if (!*out_path) + return -ENOMEM; + + cp = load_nls_default(); + cifs_strtoUTF16(*out_path, treename, treename_len, cp); + + /* Do not append the separator if the path is empty */ + if (path[0] != cpu_to_le16(0x0000)) { + UniStrcat((wchar_t *)*out_path, (wchar_t *)sep); + UniStrcat((wchar_t *)*out_path, (wchar_t *)path); + } + + unload_nls(cp); + + return 0; +} + +int smb311_posix_mkdir(const unsigned int xid, struct inode *inode, + umode_t mode, struct cifs_tcon *tcon, + const char *full_path, + struct cifs_sb_info *cifs_sb) +{ + struct smb_rqst rqst; + struct smb2_create_req *req; + struct smb2_create_rsp *rsp = NULL; + struct cifs_ses *ses = tcon->ses; + struct kvec iov[3]; /* make sure at least one for each open context */ + struct kvec rsp_iov = {NULL, 0}; + int resp_buftype; + int uni_path_len; + __le16 *copy_path = NULL; + int copy_size; + int rc = 0; + unsigned int n_iov = 2; + __u32 file_attributes = 0; + char *pc_buf = NULL; + int flags = 0; + unsigned int total_len; + __le16 *utf16_path = NULL; + struct TCP_Server_Info *server; + int retries = 0, cur_sleep = 0; + +replay_again: + /* reinitialize for possible replay */ + pc_buf = NULL; + flags = 0; + n_iov = 2; + server = cifs_pick_channel(ses); + + cifs_dbg(FYI, "mkdir\n"); + + /* resource #1: path allocation */ + utf16_path = cifs_convert_path_to_utf16(full_path, cifs_sb); + if (!utf16_path) + return -ENOMEM; + + if (!ses || !server) { + rc = smb_EIO(smb_eio_trace_null_pointers); + goto err_free_path; + } + + /* resource #2: request */ + rc = smb2_plain_req_init(SMB2_CREATE, tcon, server, + (void **) &req, &total_len); + if (rc) + goto err_free_path; + + + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; + + req->ImpersonationLevel = IL_IMPERSONATION; + req->DesiredAccess = cpu_to_le32(FILE_WRITE_ATTRIBUTES); + /* File attributes ignored on open (used in create though) */ + req->FileAttributes = cpu_to_le32(file_attributes); + req->ShareAccess = FILE_SHARE_ALL_LE; + req->CreateDisposition = cpu_to_le32(FILE_CREATE); + req->CreateOptions = cpu_to_le32(CREATE_NOT_FILE); + + iov[0].iov_base = (char *)req; + /* -1 since last byte is buf[0] which is sent below (path) */ + iov[0].iov_len = total_len - 1; + + req->NameOffset = cpu_to_le16(sizeof(struct smb2_create_req)); + + /* [MS-SMB2] 2.2.13 NameOffset: + * If SMB2_FLAGS_DFS_OPERATIONS is set in the Flags field of + * the SMB2 header, the file name includes a prefix that will + * be processed during DFS name normalization as specified in + * section 3.3.5.9. Otherwise, the file name is relative to + * the share that is identified by the TreeId in the SMB2 + * header. + */ + if (tcon->share_flags & SHI1005_FLAGS_DFS) { + int name_len; + + req->hdr.Flags |= SMB2_FLAGS_DFS_OPERATIONS; + rc = alloc_path_with_tree_prefix(©_path, ©_size, + &name_len, + tcon->tree_name, utf16_path); + if (rc) + goto err_free_req; + + req->NameLength = cpu_to_le16(name_len * 2); + uni_path_len = copy_size; + /* free before overwriting resource */ + kfree(utf16_path); + utf16_path = copy_path; + } else { + uni_path_len = (2 * UniStrnlen((wchar_t *)utf16_path, PATH_MAX)) + 2; + /* MUST set path len (NameLength) to 0 opening root of share */ + req->NameLength = cpu_to_le16(uni_path_len - 2); + if (uni_path_len % 8 != 0) { + copy_size = roundup(uni_path_len, 8); + copy_path = kzalloc(copy_size, GFP_KERNEL); + if (!copy_path) { + rc = -ENOMEM; + goto err_free_req; + } + memcpy((char *)copy_path, (const char *)utf16_path, + uni_path_len); + uni_path_len = copy_size; + /* free before overwriting resource */ + kfree(utf16_path); + utf16_path = copy_path; + } + } + + iov[1].iov_len = uni_path_len; + iov[1].iov_base = utf16_path; + req->RequestedOplockLevel = SMB2_OPLOCK_LEVEL_NONE; + + if (tcon->posix_extensions) { + /* resource #3: posix buf */ + rc = add_posix_context(iov, &n_iov, mode); + if (rc) + goto err_free_req; + req->CreateContextsOffset = cpu_to_le32( + sizeof(struct smb2_create_req) + + iov[1].iov_len); + le32_add_cpu(&req->CreateContextsLength, iov[n_iov-1].iov_len); + pc_buf = iov[n_iov-1].iov_base; + } + + + memset(&rqst, 0, sizeof(struct smb_rqst)); + rqst.rq_iov = iov; + rqst.rq_nvec = n_iov; + + /* no need to inc num_remote_opens because we close it just below */ + trace_smb3_posix_mkdir_enter(xid, tcon->tid, ses->Suid, full_path, CREATE_NOT_FILE, + FILE_WRITE_ATTRIBUTES); + + if (retries) { + /* Back-off before retry */ + if (cur_sleep) + msleep(cur_sleep); + smb2_set_replay(server, &rqst); + } + + /* resource #4: response buffer */ + rc = cifs_send_recv(xid, ses, server, + &rqst, &resp_buftype, flags, &rsp_iov); + if (rc) { + cifs_stats_fail_inc(tcon, SMB2_CREATE_HE); + trace_smb3_posix_mkdir_err(xid, tcon->tid, ses->Suid, + CREATE_NOT_FILE, + FILE_WRITE_ATTRIBUTES, rc); + goto err_free_rsp_buf; + } + + /* + * Although unlikely to be possible for rsp to be null and rc not set, + * adding check below is slightly safer long term (and quiets Coverity + * warning) + */ + rsp = (struct smb2_create_rsp *)rsp_iov.iov_base; + if (rsp == NULL) { + rc = smb_EIO(smb_eio_trace_mkdir_no_rsp); + kfree(pc_buf); + goto err_free_req; + } + + trace_smb3_posix_mkdir_done(xid, rsp->PersistentFileId, tcon->tid, ses->Suid, + CREATE_NOT_FILE, FILE_WRITE_ATTRIBUTES, + rsp->OplockLevel); + + SMB2_close(xid, tcon, rsp->PersistentFileId, rsp->VolatileFileId); + + /* Eventually save off posix specific response info and timestamps */ + +err_free_rsp_buf: + free_rsp_buf(resp_buftype, rsp_iov.iov_base); + kfree(pc_buf); +err_free_req: + cifs_small_buf_release(req); +err_free_path: + kfree(utf16_path); + + if (is_replayable_error(rc) && + smb2_should_replay(tcon, &retries, &cur_sleep)) + goto replay_again; + + return rc; +} + +int +SMB2_open_init(struct cifs_tcon *tcon, struct TCP_Server_Info *server, + struct smb_rqst *rqst, __u8 *oplock, + struct cifs_open_parms *oparms, __le16 *path) +{ + struct smb2_create_req *req; + unsigned int n_iov = 2; + __u32 file_attributes = 0; + int copy_size; + int uni_path_len; + unsigned int total_len; + struct kvec *iov = rqst->rq_iov; + __le16 *copy_path; + int rc; + + rc = smb2_plain_req_init(SMB2_CREATE, tcon, server, + (void **) &req, &total_len); + if (rc) + return rc; + + iov[0].iov_base = (char *)req; + /* -1 since last byte is buf[0] which is sent below (path) */ + iov[0].iov_len = total_len - 1; + + if (oparms->create_options & CREATE_OPTION_READONLY) + file_attributes |= ATTR_READONLY; + if (oparms->create_options & CREATE_OPTION_SPECIAL) + file_attributes |= ATTR_SYSTEM; + + req->ImpersonationLevel = IL_IMPERSONATION; + req->DesiredAccess = cpu_to_le32(oparms->desired_access); + /* File attributes ignored on open (used in create though) */ + req->FileAttributes = cpu_to_le32(file_attributes); + req->ShareAccess = FILE_SHARE_ALL_LE; + + req->CreateDisposition = cpu_to_le32(oparms->disposition); + req->CreateOptions = cpu_to_le32(oparms->create_options & CREATE_OPTIONS_MASK); + req->NameOffset = cpu_to_le16(sizeof(struct smb2_create_req)); + + /* [MS-SMB2] 2.2.13 NameOffset: + * If SMB2_FLAGS_DFS_OPERATIONS is set in the Flags field of + * the SMB2 header, the file name includes a prefix that will + * be processed during DFS name normalization as specified in + * section 3.3.5.9. Otherwise, the file name is relative to + * the share that is identified by the TreeId in the SMB2 + * header. + */ + if (tcon->share_flags & SHI1005_FLAGS_DFS) { + int name_len; + + req->hdr.Flags |= SMB2_FLAGS_DFS_OPERATIONS; + rc = alloc_path_with_tree_prefix(©_path, ©_size, + &name_len, + tcon->tree_name, path); + if (rc) + return rc; + req->NameLength = cpu_to_le16(name_len * 2); + uni_path_len = copy_size; + path = copy_path; + } else { + uni_path_len = (2 * UniStrnlen((wchar_t *)path, PATH_MAX)) + 2; + /* MUST set path len (NameLength) to 0 opening root of share */ + req->NameLength = cpu_to_le16(uni_path_len - 2); + copy_size = round_up(uni_path_len, 8); + copy_path = kzalloc(copy_size, GFP_KERNEL); + if (!copy_path) + return -ENOMEM; + memcpy((char *)copy_path, (const char *)path, + uni_path_len); + uni_path_len = copy_size; + path = copy_path; + } + + iov[1].iov_len = uni_path_len; + iov[1].iov_base = path; + + if ((!server->oplocks) || (tcon->no_lease)) + *oplock = SMB2_OPLOCK_LEVEL_NONE; + + if (!(server->capabilities & SMB2_GLOBAL_CAP_LEASING) || + *oplock == SMB2_OPLOCK_LEVEL_NONE) + req->RequestedOplockLevel = *oplock; + else if (!(server->capabilities & SMB2_GLOBAL_CAP_DIRECTORY_LEASING) && + (oparms->create_options & CREATE_NOT_FILE)) + req->RequestedOplockLevel = *oplock; /* no srv lease support */ + else { + rc = add_lease_context(server, req, iov, &n_iov, + oparms->fid->lease_key, oplock, + oparms->fid->parent_lease_key, + oparms->lease_flags); + if (rc) + return rc; + } + + if (*oplock == SMB2_OPLOCK_LEVEL_BATCH) { + rc = add_durable_context(iov, &n_iov, oparms, + tcon->use_persistent); + if (rc) + return rc; + } + + if (tcon->posix_extensions) { + rc = add_posix_context(iov, &n_iov, oparms->mode); + if (rc) + return rc; + } + + if (tcon->snapshot_time) { + cifs_dbg(FYI, "adding snapshot context\n"); + rc = add_twarp_context(iov, &n_iov, tcon->snapshot_time); + if (rc) + return rc; + } + + if ((oparms->disposition != FILE_OPEN) && (oparms->cifs_sb)) { + unsigned int sbflags = cifs_sb_flags(oparms->cifs_sb); + bool set_mode; + bool set_owner; + + if ((sbflags & CIFS_MOUNT_MODE_FROM_SID) && + oparms->mode != ACL_NO_MODE) { + set_mode = true; + } else { + set_mode = false; + oparms->mode = ACL_NO_MODE; + } + + set_owner = sbflags & CIFS_MOUNT_UID_FROM_ACL; + if (set_owner | set_mode) { + cifs_dbg(FYI, "add sd with mode 0x%x\n", oparms->mode); + rc = add_sd_context(iov, &n_iov, oparms->mode, set_owner); + if (rc) + return rc; + } + } + + add_query_id_context(iov, &n_iov); + add_ea_context(oparms, iov, &n_iov); + + if (n_iov > 2) { + /* + * We have create contexts behind iov[1] (the file + * name), point at them from the main create request + */ + req->CreateContextsOffset = cpu_to_le32( + sizeof(struct smb2_create_req) + + iov[1].iov_len); + req->CreateContextsLength = 0; + + for (unsigned int i = 2; i < (n_iov-1); i++) { + struct kvec *v = &iov[i]; + size_t len = v->iov_len; + struct create_context *cctx = + (struct create_context *)v->iov_base; + + cctx->Next = cpu_to_le32(len); + le32_add_cpu(&req->CreateContextsLength, len); + } + le32_add_cpu(&req->CreateContextsLength, + iov[n_iov-1].iov_len); + } + + rqst->rq_nvec = n_iov; + return 0; +} + +/* rq_iov[0] is the request and is released by cifs_small_buf_release(). + * All other vectors are freed by kfree(). + */ +void +SMB2_open_free(struct smb_rqst *rqst) +{ + int i; + + if (rqst && rqst->rq_iov) { + cifs_small_buf_release(rqst->rq_iov[0].iov_base); + for (i = 1; i < rqst->rq_nvec; i++) + if (rqst->rq_iov[i].iov_base != smb2_padding) + kfree(rqst->rq_iov[i].iov_base); + } +} + +int +SMB2_open(const unsigned int xid, struct cifs_open_parms *oparms, __le16 *path, + __u8 *oplock, struct cifs_open_info_data *buf, + struct create_posix_rsp *posix, + struct kvec *err_iov, int *buftype) +{ + struct smb_rqst rqst; + struct smb2_create_rsp *rsp = NULL; + struct cifs_tcon *tcon = oparms->tcon; + struct cifs_ses *ses = tcon->ses; + struct TCP_Server_Info *server; + struct kvec iov[SMB2_CREATE_IOV_SIZE]; + struct kvec rsp_iov = {NULL, 0}; + int resp_buftype = CIFS_NO_BUFFER; + int rc = 0; + int flags = 0; + int retries = 0, cur_sleep = 0; + struct smb2_file_all_info *file_info = buf ? &buf->fi : NULL; + +replay_again: + /* reinitialize for possible replay */ + resp_buftype = CIFS_NO_BUFFER; + memset(&rsp_iov, 0, sizeof(rsp_iov)); + flags = 0; + server = cifs_pick_channel(ses); + oparms->replay = !!(retries); + + cifs_dbg(FYI, "create/open\n"); + if (!ses || !server) + return smb_EIO(smb_eio_trace_null_pointers); + + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; + + memset(&rqst, 0, sizeof(struct smb_rqst)); + memset(&iov, 0, sizeof(iov)); + rqst.rq_iov = iov; + rqst.rq_nvec = SMB2_CREATE_IOV_SIZE; + + rc = SMB2_open_init(tcon, server, + &rqst, oplock, oparms, path); + if (rc) + goto creat_exit; + + trace_smb3_open_enter(xid, tcon->tid, tcon->ses->Suid, oparms->path, + oparms->create_options, oparms->desired_access); + + if (retries) { + /* Back-off before retry */ + if (cur_sleep) + msleep(cur_sleep); + smb2_set_replay(server, &rqst); + } + + rc = cifs_send_recv(xid, ses, server, + &rqst, &resp_buftype, flags, + &rsp_iov); + rsp = (struct smb2_create_rsp *)rsp_iov.iov_base; + + if (rc != 0) { + cifs_stats_fail_inc(tcon, SMB2_CREATE_HE); + if (err_iov && rsp) { + *err_iov = rsp_iov; + *buftype = resp_buftype; + resp_buftype = CIFS_NO_BUFFER; + rsp = NULL; + } + trace_smb3_open_err(xid, tcon->tid, ses->Suid, + oparms->create_options, oparms->desired_access, rc); + if (rc == -EREMCHG) { + pr_warn_once("server share %s deleted\n", + tcon->tree_name); + tcon->need_reconnect = true; + } + goto creat_exit; + } else if (rsp == NULL) /* unlikely to happen, but safer to check */ + goto creat_exit; + + atomic_inc(&tcon->num_remote_opens); + oparms->fid->persistent_fid = rsp->PersistentFileId; + oparms->fid->volatile_fid = rsp->VolatileFileId; + oparms->fid->access = oparms->desired_access; +#ifdef CONFIG_CIFS_DEBUG2 + oparms->fid->mid = le64_to_cpu(rsp->hdr.MessageId); +#endif /* CIFS_DEBUG2 */ + + if (file_info) { + file_info->CreationTime = rsp->CreationTime; + file_info->LastAccessTime = rsp->LastAccessTime; + file_info->LastWriteTime = rsp->LastWriteTime; + file_info->ChangeTime = rsp->ChangeTime; + file_info->AllocationSize = rsp->AllocationSize; + file_info->EndOfFile = rsp->EndofFile; + file_info->Attributes = rsp->FileAttributes; + file_info->NumberOfLinks = cpu_to_le32(1); + buf->unknown_nlink = true; + file_info->DeletePending = 0; /* successful open = not delete pending */ + } + + + rc = smb2_parse_contexts(server, &rsp_iov, &oparms->fid->epoch, + oparms->fid->lease_key, oplock, file_info, posix); + + trace_smb3_open_done(xid, rsp->PersistentFileId, tcon->tid, ses->Suid, + oparms->create_options, oparms->desired_access, + *oplock); +creat_exit: + SMB2_open_free(&rqst); + free_rsp_buf(resp_buftype, rsp); + + if (is_replayable_error(rc) && + smb2_should_replay(tcon, &retries, &cur_sleep)) + goto replay_again; + + return rc; +} + +int +SMB2_ioctl_init(struct cifs_tcon *tcon, struct TCP_Server_Info *server, + struct smb_rqst *rqst, + u64 persistent_fid, u64 volatile_fid, u32 opcode, + char *in_data, u32 indatalen, + __u32 max_response_size) +{ + struct smb2_ioctl_req *req; + struct kvec *iov = rqst->rq_iov; + unsigned int total_len; + int rc; + char *in_data_buf; + + rc = smb2_ioctl_req_init(opcode, tcon, server, + (void **) &req, &total_len); + if (rc) + return rc; + + if (indatalen) { + /* + * indatalen is usually small at a couple of bytes max, so + * just allocate through generic pool + */ + in_data_buf = kmemdup(in_data, indatalen, GFP_NOFS); + if (!in_data_buf) { + cifs_small_buf_release(req); + return -ENOMEM; + } + } + + req->CtlCode = cpu_to_le32(opcode); + req->PersistentFileId = persistent_fid; + req->VolatileFileId = volatile_fid; + + iov[0].iov_base = (char *)req; + /* + * If no input data, the size of ioctl struct in + * protocol spec still includes a 1 byte data buffer, + * but if input data passed to ioctl, we do not + * want to double count this, so we do not send + * the dummy one byte of data in iovec[0] if sending + * input data (in iovec[1]). + */ + if (indatalen) { + req->InputCount = cpu_to_le32(indatalen); + /* do not set InputOffset if no input data */ + req->InputOffset = + cpu_to_le32(offsetof(struct smb2_ioctl_req, Buffer)); + rqst->rq_nvec = 2; + iov[0].iov_len = total_len - 1; + iov[1].iov_base = in_data_buf; + iov[1].iov_len = indatalen; + } else { + rqst->rq_nvec = 1; + iov[0].iov_len = total_len; + } + + req->OutputOffset = 0; + req->OutputCount = 0; /* MBZ */ + + /* + * In most cases max_response_size is set to 16K (CIFSMaxBufSize) + * We Could increase default MaxOutputResponse, but that could require + * more credits. Windows typically sets this smaller, but for some + * ioctls it may be useful to allow server to send more. No point + * limiting what the server can send as long as fits in one credit + * We can not handle more than CIFS_MAX_BUF_SIZE yet but may want + * to increase this limit up in the future. + * Note that for snapshot queries that servers like Azure expect that + * the first query be minimal size (and just used to get the number/size + * of previous versions) so response size must be specified as EXACTLY + * sizeof(struct snapshot_array) which is 16 when rounded up to multiple + * of eight bytes. Currently that is the only case where we set max + * response size smaller. + */ + req->MaxOutputResponse = cpu_to_le32(max_response_size); + req->hdr.CreditCharge = + cpu_to_le16(DIV_ROUND_UP(max(indatalen, max_response_size), + SMB2_MAX_BUFFER_SIZE)); + /* always an FSCTL (for now) */ + req->Flags = cpu_to_le32(SMB2_0_IOCTL_IS_FSCTL); + + /* validate negotiate request must be signed - see MS-SMB2 3.2.5.5 */ + if (opcode == FSCTL_VALIDATE_NEGOTIATE_INFO) + req->hdr.Flags |= SMB2_FLAGS_SIGNED; + + return 0; +} + +void +SMB2_ioctl_free(struct smb_rqst *rqst) +{ + int i; + + if (rqst && rqst->rq_iov) { + cifs_small_buf_release(rqst->rq_iov[0].iov_base); /* request */ + for (i = 1; i < rqst->rq_nvec; i++) + if (rqst->rq_iov[i].iov_base != smb2_padding) + kfree(rqst->rq_iov[i].iov_base); + } +} + + +/* + * SMB2 IOCTL is used for both IOCTLs and FSCTLs + */ +int +SMB2_ioctl(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid, + u64 volatile_fid, u32 opcode, char *in_data, u32 indatalen, + u32 max_out_data_len, char **out_data, + u32 *plen /* returned data len */) +{ + struct smb_rqst rqst; + struct smb2_ioctl_rsp *rsp = NULL; + struct cifs_ses *ses; + struct TCP_Server_Info *server; + struct kvec iov[SMB2_IOCTL_IOV_SIZE]; + struct kvec rsp_iov = {NULL, 0}; + int resp_buftype = CIFS_NO_BUFFER; + int rc = 0; + int flags = 0; + int retries = 0, cur_sleep = 0; + + if (!tcon) + return smb_EIO(smb_eio_trace_null_pointers); + + ses = tcon->ses; + if (!ses) + return smb_EIO(smb_eio_trace_null_pointers); + +replay_again: + /* reinitialize for possible replay */ + resp_buftype = CIFS_NO_BUFFER; + memset(&rsp_iov, 0, sizeof(rsp_iov)); + flags = 0; + server = cifs_pick_channel(ses); + + if (!server) + return smb_EIO(smb_eio_trace_null_pointers); + + cifs_dbg(FYI, "SMB2 IOCTL\n"); + + if (out_data != NULL) + *out_data = NULL; + + /* zero out returned data len, in case of error */ + if (plen) + *plen = 0; + + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; + + memset(&rqst, 0, sizeof(struct smb_rqst)); + memset(&iov, 0, sizeof(iov)); + rqst.rq_iov = iov; + rqst.rq_nvec = SMB2_IOCTL_IOV_SIZE; + + rc = SMB2_ioctl_init(tcon, server, + &rqst, persistent_fid, volatile_fid, opcode, + in_data, indatalen, max_out_data_len); + if (rc) + goto ioctl_exit; + + if (retries) { + /* Back-off before retry */ + if (cur_sleep) + msleep(cur_sleep); + smb2_set_replay(server, &rqst); + } + + rc = cifs_send_recv(xid, ses, server, + &rqst, &resp_buftype, flags, + &rsp_iov); + rsp = (struct smb2_ioctl_rsp *)rsp_iov.iov_base; + + if (rc != 0) + trace_smb3_fsctl_err(xid, persistent_fid, tcon->tid, + ses->Suid, 0, opcode, rc); + + if ((rc != 0) && (rc != -EINVAL) && (rc != -E2BIG)) { + cifs_stats_fail_inc(tcon, SMB2_IOCTL_HE); + goto ioctl_exit; + } else if (rc == -EINVAL) { + if ((opcode != FSCTL_SRV_COPYCHUNK_WRITE) && + (opcode != FSCTL_SRV_COPYCHUNK)) { + cifs_stats_fail_inc(tcon, SMB2_IOCTL_HE); + goto ioctl_exit; + } + } else if (rc == -E2BIG) { + if (opcode != FSCTL_QUERY_ALLOCATED_RANGES) { + cifs_stats_fail_inc(tcon, SMB2_IOCTL_HE); + goto ioctl_exit; + } + } + + /* check if caller wants to look at return data or just return rc */ + if ((plen == NULL) || (out_data == NULL)) + goto ioctl_exit; + + /* + * Although unlikely to be possible for rsp to be null and rc not set, + * adding check below is slightly safer long term (and quiets Coverity + * warning) + */ + if (rsp == NULL) { + rc = smb_EIO(smb_eio_trace_ioctl_no_rsp); + goto ioctl_exit; + } + + *plen = le32_to_cpu(rsp->OutputCount); + + /* We check for obvious errors in the output buffer length and offset */ + if (*plen == 0) + goto ioctl_exit; /* server returned no data */ + else if (*plen > rsp_iov.iov_len || *plen > 0xFF00) { + cifs_tcon_dbg(VFS, "srv returned invalid ioctl length: %d\n", *plen); + rc = smb_EIO2(smb_eio_trace_ioctl_data_len, *plen, rsp_iov.iov_len); + *plen = 0; + goto ioctl_exit; + } + + u32 outoff = le32_to_cpu(rsp->OutputOffset); + + if (rsp_iov.iov_len - *plen < outoff) { + cifs_tcon_dbg(VFS, "Malformed ioctl resp: len %d offset %d\n", + *plen, outoff); + rc = smb_EIO2(smb_eio_trace_ioctl_out_off, rsp_iov.iov_len - *plen, outoff); + *plen = 0; + goto ioctl_exit; + } + + *out_data = kmemdup((char *)rsp + le32_to_cpu(rsp->OutputOffset), + *plen, GFP_KERNEL); + if (*out_data == NULL) { + rc = -ENOMEM; + goto ioctl_exit; + } + +ioctl_exit: + SMB2_ioctl_free(&rqst); + free_rsp_buf(resp_buftype, rsp); + + if (is_replayable_error(rc) && + smb2_should_replay(tcon, &retries, &cur_sleep)) + goto replay_again; + + return rc; +} + +/* + * Individual callers to ioctl worker function follow + */ + +int +SMB2_set_compression(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, + __u16 compression_state) +{ + int rc; + struct compress_ioctl fsctl_input; + char *ret_data = NULL; + + fsctl_input.CompressionState = cpu_to_le16(compression_state); + + rc = SMB2_ioctl(xid, tcon, persistent_fid, volatile_fid, + FSCTL_SET_COMPRESSION, + (char *)&fsctl_input /* data input */, + 2 /* in data len */, CIFSMaxBufSize /* max out data */, + &ret_data /* out data */, NULL); + + cifs_dbg(FYI, "set compression rc %d\n", rc); + + return rc; +} + +int +SMB2_close_init(struct cifs_tcon *tcon, struct TCP_Server_Info *server, + struct smb_rqst *rqst, + u64 persistent_fid, u64 volatile_fid, bool query_attrs) +{ + struct smb2_close_req *req; + struct kvec *iov = rqst->rq_iov; + unsigned int total_len; + int rc; + + rc = smb2_plain_req_init(SMB2_CLOSE, tcon, server, + (void **) &req, &total_len); + if (rc) + return rc; + + req->PersistentFileId = persistent_fid; + req->VolatileFileId = volatile_fid; + if (query_attrs) + req->Flags = SMB2_CLOSE_FLAG_POSTQUERY_ATTRIB; + else + req->Flags = 0; + iov[0].iov_base = (char *)req; + iov[0].iov_len = total_len; + + return 0; +} + +void +SMB2_close_free(struct smb_rqst *rqst) +{ + if (rqst && rqst->rq_iov) + cifs_small_buf_release(rqst->rq_iov[0].iov_base); /* request */ +} + +int +__SMB2_close(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, + struct smb2_file_network_open_info *pbuf) +{ + struct smb_rqst rqst; + struct smb2_close_rsp *rsp = NULL; + struct cifs_ses *ses = tcon->ses; + struct TCP_Server_Info *server; + struct kvec iov[1]; + struct kvec rsp_iov; + int resp_buftype = CIFS_NO_BUFFER; + int rc = 0; + int flags = 0; + bool query_attrs = false; + int retries = 0, cur_sleep = 0; + +replay_again: + /* reinitialize for possible replay */ + resp_buftype = CIFS_NO_BUFFER; + memset(&rsp_iov, 0, sizeof(rsp_iov)); + flags = 0; + query_attrs = false; + server = cifs_pick_channel(ses); + + cifs_dbg(FYI, "Close\n"); + + if (!ses || !server) + return smb_EIO(smb_eio_trace_null_pointers); + + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; + + memset(&rqst, 0, sizeof(struct smb_rqst)); + memset(&iov, 0, sizeof(iov)); + rqst.rq_iov = iov; + rqst.rq_nvec = 1; + + /* check if need to ask server to return timestamps in close response */ + if (pbuf) + query_attrs = true; + + trace_smb3_close_enter(xid, persistent_fid, tcon->tid, ses->Suid); + rc = SMB2_close_init(tcon, server, + &rqst, persistent_fid, volatile_fid, + query_attrs); + if (rc) + goto close_exit; + + if (retries) { + /* Back-off before retry */ + if (cur_sleep) + msleep(cur_sleep); + smb2_set_replay(server, &rqst); + } + + rc = cifs_send_recv(xid, ses, server, + &rqst, &resp_buftype, flags, &rsp_iov); + rsp = (struct smb2_close_rsp *)rsp_iov.iov_base; + + if (rc != 0) { + cifs_stats_fail_inc(tcon, SMB2_CLOSE_HE); + trace_smb3_close_err(xid, persistent_fid, tcon->tid, ses->Suid, + rc); + goto close_exit; + } else { + trace_smb3_close_done(xid, persistent_fid, tcon->tid, + ses->Suid); + if (pbuf) + memcpy(&pbuf->network_open_info, + &rsp->network_open_info, + sizeof(pbuf->network_open_info)); + atomic_dec(&tcon->num_remote_opens); + } + +close_exit: + SMB2_close_free(&rqst); + free_rsp_buf(resp_buftype, rsp); + + /* retry close in a worker thread if this one is interrupted */ + if (is_interrupt_error(rc)) { + int tmp_rc; + + tmp_rc = smb2_handle_cancelled_close(tcon, persistent_fid, + volatile_fid); + if (tmp_rc) + cifs_dbg(VFS, "handle cancelled close fid 0x%llx returned error %d\n", + persistent_fid, tmp_rc); + } + + if (is_replayable_error(rc) && + smb2_should_replay(tcon, &retries, &cur_sleep)) + goto replay_again; + + return rc; +} + +int +SMB2_close(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid) +{ + return __SMB2_close(xid, tcon, persistent_fid, volatile_fid, NULL); +} + +int +smb2_validate_iov(unsigned int offset, unsigned int buffer_length, + struct kvec *iov, unsigned int min_buf_size) +{ + unsigned int smb_len = iov->iov_len; + char *end_of_smb = smb_len + (char *)iov->iov_base; + char *begin_of_buf = offset + (char *)iov->iov_base; + char *end_of_buf = begin_of_buf + buffer_length; + + + if (buffer_length < min_buf_size) { + cifs_dbg(VFS, "buffer length %d smaller than minimum size %d\n", + buffer_length, min_buf_size); + return -EINVAL; + } + + /* check if beyond RFC1001 maximum length */ + if ((smb_len > 0x7FFFFF) || (buffer_length > 0x7FFFFF)) { + cifs_dbg(VFS, "buffer length %d or smb length %d too large\n", + buffer_length, smb_len); + return -EINVAL; + } + + if ((begin_of_buf > end_of_smb) || (end_of_buf > end_of_smb)) { + cifs_dbg(VFS, "Invalid server response, bad offset to data\n"); + return -EINVAL; + } + + return 0; +} + +/* + * If SMB buffer fields are valid, copy into temporary buffer to hold result. + * Caller must free buffer. + */ +int +smb2_validate_and_copy_iov(unsigned int offset, unsigned int buffer_length, + struct kvec *iov, unsigned int minbufsize, + char *data) +{ + char *begin_of_buf = offset + (char *)iov->iov_base; + int rc; + + if (!data) + return -EINVAL; + + rc = smb2_validate_iov(offset, buffer_length, iov, minbufsize); + if (rc) + return rc; + + memcpy(data, begin_of_buf, minbufsize); + + return 0; +} + +int +SMB2_query_info_init(struct cifs_tcon *tcon, struct TCP_Server_Info *server, + struct smb_rqst *rqst, + u64 persistent_fid, u64 volatile_fid, + u8 info_class, u8 info_type, u32 additional_info, + size_t output_len, size_t input_len, void *input) +{ + struct smb2_query_info_req *req; + struct kvec *iov = rqst->rq_iov; + unsigned int total_len; + size_t len; + int rc; + + if (unlikely(check_add_overflow(input_len, sizeof(*req), &len) || + len > CIFSMaxBufSize)) + return -EINVAL; + + rc = smb2_plain_req_init(SMB2_QUERY_INFO, tcon, server, + (void **) &req, &total_len); + if (rc) + return rc; + + req->InfoType = info_type; + req->FileInfoClass = info_class; + req->PersistentFileId = persistent_fid; + req->VolatileFileId = volatile_fid; + req->AdditionalInformation = cpu_to_le32(additional_info); + + req->OutputBufferLength = cpu_to_le32(output_len); + if (input_len) { + req->InputBufferLength = cpu_to_le32(input_len); + /* total_len for smb query request never close to le16 max */ + req->InputBufferOffset = cpu_to_le16(total_len - 1); + memcpy(req->Buffer, input, input_len); + } + + iov[0].iov_base = (char *)req; + /* 1 for Buffer */ + iov[0].iov_len = len; + return 0; +} + +void +SMB2_query_info_free(struct smb_rqst *rqst) +{ + if (rqst && rqst->rq_iov) + cifs_buf_release(rqst->rq_iov[0].iov_base); /* request */ +} + +static int +query_info(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, u8 info_class, u8 info_type, + u32 additional_info, size_t output_len, size_t min_len, void **data, + u32 *dlen) +{ + struct smb_rqst rqst; + struct smb2_query_info_rsp *rsp = NULL; + struct kvec iov[1]; + struct kvec rsp_iov; + int rc = 0; + int resp_buftype = CIFS_NO_BUFFER; + struct cifs_ses *ses = tcon->ses; + struct TCP_Server_Info *server; + int flags = 0; + bool allocated = false; + int retries = 0, cur_sleep = 0; + + cifs_dbg(FYI, "Query Info\n"); + + if (!ses) + return smb_EIO(smb_eio_trace_null_pointers); + +replay_again: + /* reinitialize for possible replay */ + resp_buftype = CIFS_NO_BUFFER; + memset(&rsp_iov, 0, sizeof(rsp_iov)); + flags = 0; + allocated = false; + server = cifs_pick_channel(ses); + + if (!server) + return smb_EIO(smb_eio_trace_null_pointers); + + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; + + memset(&rqst, 0, sizeof(struct smb_rqst)); + memset(&iov, 0, sizeof(iov)); + rqst.rq_iov = iov; + rqst.rq_nvec = 1; + + rc = SMB2_query_info_init(tcon, server, + &rqst, persistent_fid, volatile_fid, + info_class, info_type, additional_info, + output_len, 0, NULL); + if (rc) + goto qinf_exit; + + trace_smb3_query_info_enter(xid, persistent_fid, tcon->tid, + ses->Suid, info_class, (__u32)info_type); + + if (retries) { + /* Back-off before retry */ + if (cur_sleep) + msleep(cur_sleep); + smb2_set_replay(server, &rqst); + } + + rc = cifs_send_recv(xid, ses, server, + &rqst, &resp_buftype, flags, &rsp_iov); + rsp = (struct smb2_query_info_rsp *)rsp_iov.iov_base; + + if (rc) { + cifs_stats_fail_inc(tcon, SMB2_QUERY_INFO_HE); + trace_smb3_query_info_err(xid, persistent_fid, tcon->tid, + ses->Suid, info_class, (__u32)info_type, rc); + goto qinf_exit; + } + + trace_smb3_query_info_done(xid, persistent_fid, tcon->tid, + ses->Suid, info_class, (__u32)info_type); + + if (dlen) { + *dlen = le32_to_cpu(rsp->OutputBufferLength); + if (!*data) { + *data = kmalloc(*dlen, GFP_KERNEL); + if (!*data) { + cifs_tcon_dbg(VFS, + "Error %d allocating memory for acl\n", + rc); + *dlen = 0; + rc = -ENOMEM; + goto qinf_exit; + } + allocated = true; + } + } + + rc = smb2_validate_and_copy_iov(le16_to_cpu(rsp->OutputBufferOffset), + le32_to_cpu(rsp->OutputBufferLength), + &rsp_iov, dlen ? *dlen : min_len, *data); + if (rc && allocated) { + kfree(*data); + *data = NULL; + *dlen = 0; + } + +qinf_exit: + SMB2_query_info_free(&rqst); + free_rsp_buf(resp_buftype, rsp); + + if (is_replayable_error(rc) && + smb2_should_replay(tcon, &retries, &cur_sleep)) + goto replay_again; + + return rc; +} + +int SMB2_query_info(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, struct smb2_file_all_info *data) +{ + return query_info(xid, tcon, persistent_fid, volatile_fid, + FILE_ALL_INFORMATION, SMB2_O_INFO_FILE, 0, + sizeof(struct smb2_file_all_info) + PATH_MAX * 2, + sizeof(struct smb2_file_all_info), (void **)&data, + NULL); +} + +int +SMB2_query_acl(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, + void **data, u32 *plen, u32 extra_info) +{ + *plen = 0; + + return query_info(xid, tcon, persistent_fid, volatile_fid, + 0, SMB2_O_INFO_SECURITY, extra_info, + SMB2_MAX_BUFFER_SIZE, MIN_SEC_DESC_LEN, data, plen); +} + +int +SMB2_get_srv_num(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, __le64 *uniqueid) +{ + return query_info(xid, tcon, persistent_fid, volatile_fid, + FILE_INTERNAL_INFORMATION, SMB2_O_INFO_FILE, 0, + sizeof(struct smb2_file_internal_info), + sizeof(struct smb2_file_internal_info), + (void **)&uniqueid, NULL); +} + +/* + * CHANGE_NOTIFY Request is sent to get notifications on changes to a directory + * See MS-SMB2 2.2.35 and 2.2.36 + */ + +static int +SMB2_notify_init(const unsigned int xid, struct smb_rqst *rqst, + struct cifs_tcon *tcon, struct TCP_Server_Info *server, + u64 persistent_fid, u64 volatile_fid, + u32 completion_filter, bool watch_tree) +{ + struct smb2_change_notify_req *req; + struct kvec *iov = rqst->rq_iov; + unsigned int total_len; + int rc; + + rc = smb2_plain_req_init(SMB2_CHANGE_NOTIFY, tcon, server, + (void **) &req, &total_len); + if (rc) + return rc; + + req->PersistentFileId = persistent_fid; + req->VolatileFileId = volatile_fid; + /* See note 354 of MS-SMB2, 64K max */ + req->OutputBufferLength = + cpu_to_le32(SMB2_MAX_BUFFER_SIZE - MAX_SMB2_HDR_SIZE); + req->CompletionFilter = cpu_to_le32(completion_filter); + if (watch_tree) + req->Flags = cpu_to_le16(SMB2_WATCH_TREE); + else + req->Flags = 0; + + iov[0].iov_base = (char *)req; + iov[0].iov_len = total_len; + + return 0; +} + +int +SMB2_change_notify(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, bool watch_tree, + u32 completion_filter, u32 max_out_data_len, char **out_data, + u32 *plen /* returned data len */) +{ + struct cifs_ses *ses = tcon->ses; + struct TCP_Server_Info *server; + struct smb_rqst rqst; + struct smb2_change_notify_rsp *smb_rsp; + struct kvec iov[1]; + struct kvec rsp_iov = {NULL, 0}; + int resp_buftype = CIFS_NO_BUFFER; + int flags = 0; + int rc = 0; + int retries = 0, cur_sleep = 0; + +replay_again: + /* reinitialize for possible replay */ + resp_buftype = CIFS_NO_BUFFER; + memset(&rsp_iov, 0, sizeof(rsp_iov)); + flags = 0; + server = cifs_pick_channel(ses); + + cifs_dbg(FYI, "change notify\n"); + if (!ses || !server) + return smb_EIO(smb_eio_trace_null_pointers); + + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; + + memset(&rqst, 0, sizeof(struct smb_rqst)); + memset(&iov, 0, sizeof(iov)); + if (plen) + *plen = 0; + + rqst.rq_iov = iov; + rqst.rq_nvec = 1; + + rc = SMB2_notify_init(xid, &rqst, tcon, server, + persistent_fid, volatile_fid, + completion_filter, watch_tree); + if (rc) + goto cnotify_exit; + + trace_smb3_notify_enter(xid, persistent_fid, tcon->tid, ses->Suid, + (u8)watch_tree, completion_filter); + + if (retries) { + /* Back-off before retry */ + if (cur_sleep) + msleep(cur_sleep); + smb2_set_replay(server, &rqst); + } + + rc = cifs_send_recv(xid, ses, server, + &rqst, &resp_buftype, flags, &rsp_iov); + + if (rc != 0) { + cifs_stats_fail_inc(tcon, SMB2_CHANGE_NOTIFY_HE); + trace_smb3_notify_err(xid, persistent_fid, tcon->tid, ses->Suid, + (u8)watch_tree, completion_filter, rc); + } else { + trace_smb3_notify_done(xid, persistent_fid, tcon->tid, + ses->Suid, (u8)watch_tree, completion_filter); + /* validate that notify information is plausible */ + if ((rsp_iov.iov_base == NULL) || + (rsp_iov.iov_len < sizeof(struct smb2_change_notify_rsp) + 1)) + goto cnotify_exit; + + smb_rsp = (struct smb2_change_notify_rsp *)rsp_iov.iov_base; + + rc = smb2_validate_iov(le16_to_cpu(smb_rsp->OutputBufferOffset), + le32_to_cpu(smb_rsp->OutputBufferLength), + &rsp_iov, + sizeof(struct file_notify_information)); + if (rc) + goto cnotify_exit; + + *out_data = kmemdup((char *)smb_rsp + le16_to_cpu(smb_rsp->OutputBufferOffset), + le32_to_cpu(smb_rsp->OutputBufferLength), GFP_KERNEL); + if (*out_data == NULL) { + rc = -ENOMEM; + goto cnotify_exit; + } else if (plen) + *plen = le32_to_cpu(smb_rsp->OutputBufferLength); + } + + cnotify_exit: + if (rqst.rq_iov) + cifs_small_buf_release(rqst.rq_iov[0].iov_base); /* request */ + free_rsp_buf(resp_buftype, rsp_iov.iov_base); + + if (is_replayable_error(rc) && + smb2_should_replay(tcon, &retries, &cur_sleep)) + goto replay_again; + + return rc; +} + + + +/* + * This is a no-op for now. We're not really interested in the reply, but + * rather in the fact that the server sent one and that server->lstrp + * gets updated. + * + * FIXME: maybe we should consider checking that the reply matches request? + */ +static void +smb2_echo_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid) +{ + struct smb2_echo_rsp *rsp = (struct smb2_echo_rsp *)mid->resp_buf; + struct cifs_credits credits = { .value = 0, .instance = 0 }; + + if (mid->mid_state == MID_RESPONSE_RECEIVED + || mid->mid_state == MID_RESPONSE_MALFORMED) { + credits.value = le16_to_cpu(rsp->hdr.CreditRequest); + credits.instance = server->reconnect_instance; + } + + release_mid(server, mid); + add_credits(server, &credits, CIFS_ECHO_OP); +} + +static void cifs_renegotiate_iosize(struct TCP_Server_Info *server, + struct cifs_tcon *tcon) +{ + struct cifs_sb_info *cifs_sb; + + if (server == NULL || tcon == NULL) + return; + + spin_lock(&tcon->sb_list_lock); + list_for_each_entry(cifs_sb, &tcon->cifs_sb_list, tcon_sb_link) + cifs_negotiate_iosize(server, cifs_sb->ctx, tcon); + spin_unlock(&tcon->sb_list_lock); +} + +void smb2_reconnect_server(struct work_struct *work) +{ + struct TCP_Server_Info *server = container_of(work, + struct TCP_Server_Info, reconnect.work); + struct TCP_Server_Info *pserver; + struct cifs_ses *ses, *ses2; + struct cifs_tcon *tcon, *tcon2; + struct list_head tmp_list, tmp_ses_list; + bool ses_exist = false; + bool tcon_selected = false; + int rc; + bool resched = false; + + /* first check if ref count has reached 0, if not inc ref count */ + spin_lock(&cifs_tcp_ses_lock); + if (!server->srv_count) { + spin_unlock(&cifs_tcp_ses_lock); + return; + } + server->srv_count++; + spin_unlock(&cifs_tcp_ses_lock); + + /* If server is a channel, select the primary channel */ + pserver = SERVER_IS_CHAN(server) ? server->primary_server : server; + + /* Prevent simultaneous reconnects that can corrupt tcon->rlist list */ + mutex_lock(&pserver->reconnect_mutex); + + /* if the server is marked for termination, drop the ref count here */ + if (server->terminate) { + cifs_put_tcp_session(server, true); + mutex_unlock(&pserver->reconnect_mutex); + return; + } + + INIT_LIST_HEAD(&tmp_list); + INIT_LIST_HEAD(&tmp_ses_list); + cifs_dbg(FYI, "Reconnecting tcons and channels\n"); + + spin_lock(&cifs_tcp_ses_lock); + list_for_each_entry(ses, &pserver->smb_ses_list, smb_ses_list) { + spin_lock(&ses->ses_lock); + if (ses->ses_status == SES_EXITING) { + spin_unlock(&ses->ses_lock); + continue; + } + spin_unlock(&ses->ses_lock); + + tcon_selected = false; + + list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { + if (tcon->need_reconnect || tcon->need_reopen_files) { + spin_lock(&tcon->tc_lock); + tcon->tc_count++; + spin_unlock(&tcon->tc_lock); + trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, + netfs_trace_tcon_ref_get_reconnect_server); + list_add_tail(&tcon->rlist, &tmp_list); + tcon_selected = true; + } + } + /* + * IPC has the same lifetime as its session and uses its + * refcount. + */ + if (ses->tcon_ipc && ses->tcon_ipc->need_reconnect) { + list_add_tail(&ses->tcon_ipc->rlist, &tmp_list); + tcon_selected = true; + cifs_smb_ses_inc_refcount(ses); + } + /* + * handle the case where channel needs to reconnect + * binding session, but tcon is healthy (some other channel + * is active) + */ + spin_lock(&ses->chan_lock); + if (!tcon_selected && cifs_chan_needs_reconnect(ses, server)) { + list_add_tail(&ses->rlist, &tmp_ses_list); + ses_exist = true; + cifs_smb_ses_inc_refcount(ses); + } + spin_unlock(&ses->chan_lock); + } + spin_unlock(&cifs_tcp_ses_lock); + + list_for_each_entry_safe(tcon, tcon2, &tmp_list, rlist) { + rc = smb2_reconnect(SMB2_INTERNAL_CMD, tcon, server, true); + if (!rc) { + cifs_renegotiate_iosize(server, tcon); + cifs_reopen_persistent_handles(tcon); + } else + resched = true; + list_del_init(&tcon->rlist); + if (tcon->ipc) + cifs_put_smb_ses(tcon->ses); + else + cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_reconnect_server); + } + + if (!ses_exist) + goto done; + + /* allocate a dummy tcon struct used for reconnect */ + tcon = tcon_info_alloc(false, netfs_trace_tcon_ref_new_reconnect_server); + if (!tcon) { + resched = true; + list_for_each_entry_safe(ses, ses2, &tmp_ses_list, rlist) { + list_del_init(&ses->rlist); + cifs_put_smb_ses(ses); + } + goto done; + } + tcon->status = TID_GOOD; + tcon->dummy = true; + + /* now reconnect sessions for necessary channels */ + list_for_each_entry_safe(ses, ses2, &tmp_ses_list, rlist) { + tcon->ses = ses; + rc = smb2_reconnect(SMB2_INTERNAL_CMD, tcon, server, true); + if (rc) + resched = true; + list_del_init(&ses->rlist); + cifs_put_smb_ses(ses); + } + tconInfoFree(tcon, netfs_trace_tcon_ref_free_reconnect_server); + +done: + cifs_dbg(FYI, "Reconnecting tcons and channels finished\n"); + if (resched) + cifs_requeue_server_reconn(server); + mutex_unlock(&pserver->reconnect_mutex); + + /* now we can safely release srv struct */ + cifs_put_tcp_session(server, true); +} + +int +SMB2_echo(struct TCP_Server_Info *server) +{ + struct smb2_echo_req *req; + int rc = 0; + struct kvec iov[1]; + struct smb_rqst rqst = { .rq_iov = iov, + .rq_nvec = 1 }; + unsigned int total_len; + + cifs_dbg(FYI, "In echo request for conn_id %lld\n", server->conn_id); + + spin_lock(&server->srv_lock); + if (server->ops->need_neg && + server->ops->need_neg(server)) { + spin_unlock(&server->srv_lock); + /* No need to send echo on newly established connections */ + cifs_queue_server_reconn(server); + return rc; + } + spin_unlock(&server->srv_lock); + + rc = smb2_plain_req_init(SMB2_ECHO, NULL, server, + (void **)&req, &total_len); + if (rc) + return rc; + + req->hdr.CreditRequest = cpu_to_le16(1); + + iov[0].iov_len = total_len; + iov[0].iov_base = (char *)req; + + rc = cifs_call_async(server, &rqst, NULL, smb2_echo_callback, NULL, + server, CIFS_ECHO_OP, NULL); + if (rc) + cifs_dbg(FYI, "Echo request failed: %d\n", rc); + + cifs_small_buf_release(req); + return rc; +} + +void +SMB2_flush_free(struct smb_rqst *rqst) +{ + if (rqst && rqst->rq_iov) + cifs_small_buf_release(rqst->rq_iov[0].iov_base); /* request */ +} + +int +SMB2_flush_init(const unsigned int xid, struct smb_rqst *rqst, + struct cifs_tcon *tcon, struct TCP_Server_Info *server, + u64 persistent_fid, u64 volatile_fid) +{ + struct smb2_flush_req *req; + struct kvec *iov = rqst->rq_iov; + unsigned int total_len; + int rc; + + rc = smb2_plain_req_init(SMB2_FLUSH, tcon, server, + (void **) &req, &total_len); + if (rc) + return rc; + + req->PersistentFileId = persistent_fid; + req->VolatileFileId = volatile_fid; + + iov[0].iov_base = (char *)req; + iov[0].iov_len = total_len; + + return 0; +} + +int +SMB2_flush(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid, + u64 volatile_fid) +{ + struct cifs_ses *ses = tcon->ses; + struct smb_rqst rqst; + struct kvec iov[1]; + struct kvec rsp_iov = {NULL, 0}; + struct TCP_Server_Info *server; + int resp_buftype = CIFS_NO_BUFFER; + int flags = 0; + int rc = 0; + int retries = 0, cur_sleep = 0; + +replay_again: + /* reinitialize for possible replay */ + resp_buftype = CIFS_NO_BUFFER; + memset(&rsp_iov, 0, sizeof(rsp_iov)); + flags = 0; + server = cifs_pick_channel(ses); + + cifs_dbg(FYI, "flush\n"); + if (!ses || !(ses->server)) + return smb_EIO(smb_eio_trace_null_pointers); + + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; + + memset(&rqst, 0, sizeof(struct smb_rqst)); + memset(&iov, 0, sizeof(iov)); + rqst.rq_iov = iov; + rqst.rq_nvec = 1; + + rc = SMB2_flush_init(xid, &rqst, tcon, server, + persistent_fid, volatile_fid); + if (rc) + goto flush_exit; + + trace_smb3_flush_enter(xid, persistent_fid, tcon->tid, ses->Suid); + + if (retries) { + /* Back-off before retry */ + if (cur_sleep) + msleep(cur_sleep); + smb2_set_replay(server, &rqst); + } + + rc = cifs_send_recv(xid, ses, server, + &rqst, &resp_buftype, flags, &rsp_iov); + + if (rc != 0) { + cifs_stats_fail_inc(tcon, SMB2_FLUSH_HE); + trace_smb3_flush_err(xid, persistent_fid, tcon->tid, ses->Suid, + rc); + } else + trace_smb3_flush_done(xid, persistent_fid, tcon->tid, + ses->Suid); + + flush_exit: + SMB2_flush_free(&rqst); + free_rsp_buf(resp_buftype, rsp_iov.iov_base); + + if (is_replayable_error(rc) && + smb2_should_replay(tcon, &retries, &cur_sleep)) + goto replay_again; + + return rc; +} + +#ifdef CONFIG_CIFS_SMB_DIRECT +static inline bool smb3_use_rdma_offload(struct cifs_io_parms *io_parms) +{ + struct TCP_Server_Info *server = io_parms->server; + struct cifs_tcon *tcon = io_parms->tcon; + + /* we can only offload if we're connected */ + if (!server || !tcon) + return false; + + /* we can only offload on an rdma connection */ + if (!server->rdma || !server->smbd_conn) + return false; + + /* we don't support signed offload yet */ + if (server->sign) + return false; + + /* we don't support encrypted offload yet */ + if (smb3_encryption_required(tcon)) + return false; + + /* offload also has its overhead, so only do it if desired */ + if (io_parms->length < server->rdma_readwrite_threshold) + return false; + + return true; +} +#endif /* CONFIG_CIFS_SMB_DIRECT */ + +/* + * To form a chain of read requests, any read requests after the first should + * have the end_of_chain boolean set to true. + */ +static int +smb2_new_read_req(void **buf, unsigned int *total_len, + struct cifs_io_parms *io_parms, struct cifs_io_subrequest *rdata, + unsigned int remaining_bytes, int request_type) +{ + int rc = -EACCES; + struct smb2_read_req *req = NULL; + struct smb2_hdr *shdr; + struct TCP_Server_Info *server = io_parms->server; + + rc = smb2_plain_req_init(SMB2_READ, io_parms->tcon, server, + (void **) &req, total_len); + if (rc) + return rc; + + if (server == NULL) + return -ECONNABORTED; + + shdr = &req->hdr; + shdr->Id.SyncId.ProcessId = cpu_to_le32(io_parms->pid); + + req->PersistentFileId = io_parms->persistent_fid; + req->VolatileFileId = io_parms->volatile_fid; + req->ReadChannelInfoOffset = 0; /* reserved */ + req->ReadChannelInfoLength = 0; /* reserved */ + req->Channel = 0; /* reserved */ + req->MinimumCount = 0; + req->Length = cpu_to_le32(io_parms->length); + req->Offset = cpu_to_le64(io_parms->offset); + + trace_smb3_read_enter(rdata ? rdata->rreq->debug_id : 0, + rdata ? rdata->subreq.debug_index : 0, + rdata ? rdata->xid : 0, + io_parms->persistent_fid, + io_parms->tcon->tid, io_parms->tcon->ses->Suid, + io_parms->offset, io_parms->length); +#ifdef CONFIG_CIFS_SMB_DIRECT + /* + * If we want to do a RDMA write, fill in and append + * smbdirect_buffer_descriptor_v1 to the end of read request + */ + if (rdata && smb3_use_rdma_offload(io_parms)) { + struct smbdirect_buffer_descriptor_v1 *v1; + bool need_invalidate = server->dialect == SMB30_PROT_ID; + + rdata->mr = smbd_register_mr(server->smbd_conn, &rdata->subreq.io_iter, + true, need_invalidate); + if (!rdata->mr) + return -EAGAIN; + + req->Channel = SMB2_CHANNEL_RDMA_V1_INVALIDATE; + if (need_invalidate) + req->Channel = SMB2_CHANNEL_RDMA_V1; + req->ReadChannelInfoOffset = + cpu_to_le16(offsetof(struct smb2_read_req, Buffer)); + req->ReadChannelInfoLength = + cpu_to_le16(sizeof(struct smbdirect_buffer_descriptor_v1)); + v1 = (struct smbdirect_buffer_descriptor_v1 *) &req->Buffer[0]; + smbd_mr_fill_buffer_descriptor(rdata->mr, v1); + + *total_len += sizeof(*v1) - 1; + } +#endif + if (request_type & CHAINED_REQUEST) { + if (!(request_type & END_OF_CHAIN)) { + /* next 8-byte aligned request */ + *total_len = ALIGN(*total_len, 8); + shdr->NextCommand = cpu_to_le32(*total_len); + } else /* END_OF_CHAIN */ + shdr->NextCommand = 0; + if (request_type & RELATED_REQUEST) { + shdr->Flags |= SMB2_FLAGS_RELATED_OPERATIONS; + /* + * Related requests use info from previous read request + * in chain. + */ + shdr->SessionId = cpu_to_le64(0xFFFFFFFFFFFFFFFF); + shdr->Id.SyncId.TreeId = cpu_to_le32(0xFFFFFFFF); + req->PersistentFileId = (u64)-1; + req->VolatileFileId = (u64)-1; + } + } + if (remaining_bytes > io_parms->length) + req->RemainingBytes = cpu_to_le32(remaining_bytes); + else + req->RemainingBytes = 0; + + *buf = req; + return rc; +} + +static void +smb2_readv_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid) +{ + struct cifs_io_subrequest *rdata = mid->callback_data; + struct netfs_inode *ictx = netfs_inode(rdata->rreq->inode); + struct cifs_tcon *tcon = tlink_tcon(rdata->req->cfile->tlink); + struct smb2_hdr *shdr = (struct smb2_hdr *)rdata->iov[0].iov_base; + struct inode *inode = &ictx->inode; + struct cifs_credits credits = { + .value = 0, + .instance = 0, + .rreq_debug_id = rdata->rreq->debug_id, + .rreq_debug_index = rdata->subreq.debug_index, + }; + struct smb_rqst rqst = { .rq_iov = &rdata->iov[0], .rq_nvec = 1 }; + unsigned int rreq_debug_id = rdata->rreq->debug_id; + unsigned int subreq_debug_index = rdata->subreq.debug_index; + + if (rdata->got_bytes) { + rqst.rq_iter = rdata->subreq.io_iter; + } + + WARN_ONCE(rdata->server != server, + "rdata server %p != mid server %p", + rdata->server, server); + + cifs_dbg(FYI, "%s: mid=%llu state=%d result=%d bytes=%zu/%zu\n", + __func__, mid->mid, mid->mid_state, rdata->result, + rdata->got_bytes, rdata->subreq.len - rdata->subreq.transferred); + + switch (mid->mid_state) { + case MID_RESPONSE_RECEIVED: + credits.value = le16_to_cpu(shdr->CreditRequest); + credits.instance = server->reconnect_instance; + /* result already set, check signature */ + if (server->sign && !mid->decrypted) { + int rc; + + iov_iter_truncate(&rqst.rq_iter, rdata->got_bytes); + rc = smb2_verify_signature(&rqst, server); + if (rc) { + cifs_tcon_dbg(VFS, "SMB signature verification returned error = %d\n", + rc); + rdata->subreq.error = rc; + rdata->result = rc; + + if (is_replayable_error(rc)) { + trace_netfs_sreq(&rdata->subreq, netfs_sreq_trace_io_retry_needed); + __set_bit(NETFS_SREQ_NEED_RETRY, &rdata->subreq.flags); + } else + trace_netfs_sreq(&rdata->subreq, netfs_sreq_trace_io_bad); + } else + trace_netfs_sreq(&rdata->subreq, netfs_sreq_trace_io_progress); + } + /* FIXME: should this be counted toward the initiating task? */ + task_io_account_read(rdata->got_bytes); + cifs_stats_bytes_read(tcon, rdata->got_bytes); + break; + case MID_REQUEST_SUBMITTED: + trace_netfs_sreq(&rdata->subreq, netfs_sreq_trace_io_req_submitted); + goto do_retry; + case MID_RETRY_NEEDED: + trace_netfs_sreq(&rdata->subreq, netfs_sreq_trace_io_retry_needed); +do_retry: + __set_bit(NETFS_SREQ_NEED_RETRY, &rdata->subreq.flags); + rdata->result = -EAGAIN; + if (server->sign && rdata->got_bytes) + /* reset bytes number since we can not check a sign */ + rdata->got_bytes = 0; + /* FIXME: should this be counted toward the initiating task? */ + task_io_account_read(rdata->got_bytes); + cifs_stats_bytes_read(tcon, rdata->got_bytes); + break; + case MID_RESPONSE_MALFORMED: + trace_netfs_sreq(&rdata->subreq, netfs_sreq_trace_io_malformed); + credits.value = le16_to_cpu(shdr->CreditRequest); + credits.instance = server->reconnect_instance; + rdata->result = smb_EIO(smb_eio_trace_read_rsp_malformed); + break; + default: + trace_netfs_sreq(&rdata->subreq, netfs_sreq_trace_io_unknown); + rdata->result = smb_EIO1(smb_eio_trace_read_mid_state_unknown, + mid->mid_state); + break; + } +#ifdef CONFIG_CIFS_SMB_DIRECT + /* + * If this rdata has a memory registered, the MR can be freed + * MR needs to be freed as soon as I/O finishes to prevent deadlock + * because they have limited number and are used for future I/Os + */ + if (rdata->mr) { + smbd_deregister_mr(rdata->mr); + rdata->mr = NULL; + } +#endif + if (rdata->result && rdata->result != -ENODATA) { + cifs_stats_fail_inc(tcon, SMB2_READ_HE); + trace_smb3_read_err(rdata->rreq->debug_id, + rdata->subreq.debug_index, + rdata->xid, + rdata->req->cfile->fid.persistent_fid, + tcon->tid, tcon->ses->Suid, + rdata->subreq.start + rdata->subreq.transferred, + rdata->subreq.len - rdata->subreq.transferred, + rdata->result); + } else + trace_smb3_read_done(rdata->rreq->debug_id, + rdata->subreq.debug_index, + rdata->xid, + rdata->req->cfile->fid.persistent_fid, + tcon->tid, tcon->ses->Suid, + rdata->subreq.start + rdata->subreq.transferred, + rdata->got_bytes); + + if (rdata->result == -ENODATA) { + __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags); + rdata->result = 0; + } else { + size_t trans = rdata->subreq.transferred + rdata->got_bytes; + if (trans < rdata->subreq.len && + rdata->subreq.start + trans >= netfs_read_remote_i_size(inode)) { + __set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags); + rdata->result = 0; + } + if (rdata->got_bytes) + __set_bit(NETFS_SREQ_MADE_PROGRESS, &rdata->subreq.flags); + } + + /* see if we need to retry */ + if (is_replayable_error(rdata->result) && + smb2_should_replay(tcon, + &rdata->retries, + &rdata->cur_sleep)) + rdata->replay = true; + + trace_smb3_rw_credits(rreq_debug_id, subreq_debug_index, rdata->credits.value, + server->credits, server->in_flight, + 0, cifs_trace_rw_credits_read_response_clear); + rdata->credits.value = 0; + rdata->subreq.error = rdata->result; + rdata->subreq.transferred += rdata->got_bytes; + trace_netfs_sreq(&rdata->subreq, netfs_sreq_trace_io_progress); + netfs_read_subreq_terminated(&rdata->subreq); + release_mid(server, mid); + trace_smb3_rw_credits(rreq_debug_id, subreq_debug_index, 0, + server->credits, server->in_flight, + credits.value, cifs_trace_rw_credits_read_response_add); + add_credits(server, &credits, 0); +} + +/* smb2_async_readv - send an async read, and set up mid to handle result */ +int +smb2_async_readv(struct cifs_io_subrequest *rdata) +{ + int rc, flags = 0; + char *buf; + struct netfs_io_subrequest *subreq = &rdata->subreq; + struct smb2_hdr *shdr; + struct cifs_io_parms io_parms; + struct smb_rqst rqst = { .rq_iov = rdata->iov, + .rq_nvec = 1 }; + struct TCP_Server_Info *server; + struct cifs_tcon *tcon = tlink_tcon(rdata->req->cfile->tlink); + unsigned int total_len; + int credit_request; + + cifs_dbg(FYI, "%s: offset=%llu bytes=%zu\n", + __func__, subreq->start, subreq->len); + + if (!rdata->server) + rdata->server = cifs_pick_channel(tcon->ses); + + io_parms.tcon = tlink_tcon(rdata->req->cfile->tlink); + io_parms.server = server = rdata->server; + io_parms.offset = subreq->start + subreq->transferred; + io_parms.length = subreq->len - subreq->transferred; + io_parms.persistent_fid = rdata->req->cfile->fid.persistent_fid; + io_parms.volatile_fid = rdata->req->cfile->fid.volatile_fid; + io_parms.pid = rdata->req->pid; + + rc = smb2_new_read_req( + (void **) &buf, &total_len, &io_parms, rdata, 0, 0); + if (rc) + goto out; + + if (smb3_encryption_required(io_parms.tcon)) + flags |= CIFS_TRANSFORM_REQ; + + rdata->iov[0].iov_base = buf; + rdata->iov[0].iov_len = total_len; + rdata->got_bytes = 0; + rdata->result = 0; + + shdr = (struct smb2_hdr *)buf; + + if (rdata->replay) { + /* Back-off before retry */ + if (rdata->cur_sleep) + msleep(rdata->cur_sleep); + smb2_set_replay(server, &rqst); + } + + if (rdata->credits.value > 0) { + shdr->CreditCharge = cpu_to_le16(DIV_ROUND_UP(io_parms.length, + SMB2_MAX_BUFFER_SIZE)); + credit_request = le16_to_cpu(shdr->CreditCharge) + 8; + if (server->credits >= server->max_credits) + shdr->CreditRequest = cpu_to_le16(0); + else + shdr->CreditRequest = cpu_to_le16( + min_t(int, server->max_credits - + server->credits, credit_request)); + + rc = adjust_credits(server, rdata, cifs_trace_rw_credits_call_readv_adjust); + if (rc) + goto async_readv_out; + + flags |= CIFS_HAS_CREDITS; + } + + rc = cifs_call_async(server, &rqst, + cifs_readv_receive, smb2_readv_callback, + smb3_handle_read_data, rdata, flags, + &rdata->credits); + if (rc) { + cifs_stats_fail_inc(io_parms.tcon, SMB2_READ_HE); + trace_smb3_read_err(rdata->rreq->debug_id, + subreq->debug_index, + rdata->xid, io_parms.persistent_fid, + io_parms.tcon->tid, + io_parms.tcon->ses->Suid, + io_parms.offset, + subreq->len - subreq->transferred, rc); + } + +async_readv_out: + cifs_small_buf_release(buf); + +out: + /* if the send error is retryable, let netfs know about it */ + if (is_replayable_error(rc) && + smb2_should_replay(tcon, + &rdata->retries, + &rdata->cur_sleep)) { + trace_netfs_sreq(&rdata->subreq, netfs_sreq_trace_io_retry_needed); + __set_bit(NETFS_SREQ_NEED_RETRY, &rdata->subreq.flags); + } + + return rc; +} + +int +SMB2_read(const unsigned int xid, struct cifs_io_parms *io_parms, + unsigned int *nbytes, char **buf, int *buf_type) +{ + struct smb_rqst rqst; + int resp_buftype, rc; + struct smb2_read_req *req = NULL; + struct smb2_read_rsp *rsp = NULL; + struct kvec iov[1]; + struct kvec rsp_iov; + unsigned int total_len; + int flags = CIFS_LOG_ERROR; + struct cifs_ses *ses = io_parms->tcon->ses; + + if (!io_parms->server) + io_parms->server = cifs_pick_channel(io_parms->tcon->ses); + + *nbytes = 0; + rc = smb2_new_read_req((void **)&req, &total_len, io_parms, NULL, 0, 0); + if (rc) + return rc; + + if (smb3_encryption_required(io_parms->tcon)) + flags |= CIFS_TRANSFORM_REQ; + + iov[0].iov_base = (char *)req; + iov[0].iov_len = total_len; + + memset(&rqst, 0, sizeof(struct smb_rqst)); + rqst.rq_iov = iov; + rqst.rq_nvec = 1; + + rc = cifs_send_recv(xid, ses, io_parms->server, + &rqst, &resp_buftype, flags, &rsp_iov); + rsp = (struct smb2_read_rsp *)rsp_iov.iov_base; + + if (rc) { + if (rc != -ENODATA) { + cifs_stats_fail_inc(io_parms->tcon, SMB2_READ_HE); + cifs_dbg(VFS, "Send error in read = %d\n", rc); + trace_smb3_read_err(0, 0, xid, + req->PersistentFileId, + io_parms->tcon->tid, ses->Suid, + io_parms->offset, io_parms->length, + rc); + } else + trace_smb3_read_done(0, 0, xid, + req->PersistentFileId, io_parms->tcon->tid, + ses->Suid, io_parms->offset, 0); + free_rsp_buf(resp_buftype, rsp_iov.iov_base); + cifs_small_buf_release(req); + return rc == -ENODATA ? 0 : rc; + } else + trace_smb3_read_done(0, 0, xid, + req->PersistentFileId, + io_parms->tcon->tid, ses->Suid, + io_parms->offset, io_parms->length); + + cifs_small_buf_release(req); + + *nbytes = le32_to_cpu(rsp->DataLength); + if ((*nbytes > CIFS_MAX_MSGSIZE) || + (*nbytes > io_parms->length)) { + cifs_dbg(FYI, "bad length %d for count %d\n", + *nbytes, io_parms->length); + rc = smb_EIO2(smb_eio_trace_read_overlarge, + *nbytes, io_parms->length); + *nbytes = 0; + } + + if (*buf) { + memcpy(*buf, (char *)rsp + rsp->DataOffset, *nbytes); + free_rsp_buf(resp_buftype, rsp_iov.iov_base); + } else if (resp_buftype != CIFS_NO_BUFFER) { + *buf = rsp_iov.iov_base; + if (resp_buftype == CIFS_SMALL_BUFFER) + *buf_type = CIFS_SMALL_BUFFER; + else if (resp_buftype == CIFS_LARGE_BUFFER) + *buf_type = CIFS_LARGE_BUFFER; + } + return rc; +} + +/* + * Check the mid_state and signature on received buffer (if any), and queue the + * workqueue completion task. + */ +static void +smb2_writev_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid) +{ + struct cifs_io_subrequest *wdata = mid->callback_data; + struct cifs_tcon *tcon = tlink_tcon(wdata->req->cfile->tlink); + struct smb2_write_rsp *rsp = (struct smb2_write_rsp *)mid->resp_buf; + struct cifs_credits credits = { + .value = 0, + .instance = 0, + .rreq_debug_id = wdata->rreq->debug_id, + .rreq_debug_index = wdata->subreq.debug_index, + }; + unsigned int rreq_debug_id = wdata->rreq->debug_id; + unsigned int subreq_debug_index = wdata->subreq.debug_index; + ssize_t result = 0; + size_t written = 0; + + WARN_ONCE(wdata->server != server, + "wdata server %p != mid server %p", + wdata->server, server); + + switch (mid->mid_state) { + case MID_RESPONSE_RECEIVED: + credits.value = le16_to_cpu(rsp->hdr.CreditRequest); + credits.instance = server->reconnect_instance; + result = smb2_check_receive(mid, server, 0); + if (result != 0) { + if (is_replayable_error(result)) { + trace_netfs_sreq(&wdata->subreq, netfs_sreq_trace_io_retry_needed); + __set_bit(NETFS_SREQ_NEED_RETRY, &wdata->subreq.flags); + } else { + wdata->subreq.error = result; + trace_netfs_sreq(&wdata->subreq, netfs_sreq_trace_io_bad); + } + break; + } + trace_netfs_sreq(&wdata->subreq, netfs_sreq_trace_io_progress); + + written = le32_to_cpu(rsp->DataLength); + /* + * Mask off high 16 bits when bytes written as returned + * by the server is greater than bytes requested by the + * client. OS/2 servers are known to set incorrect + * CountHigh values. + */ + if (written > wdata->subreq.len) + written &= 0xFFFF; + + cifs_stats_bytes_written(tcon, written); + + if (written < wdata->subreq.len) { + result = -ENOSPC; + } else if (written > 0) { + wdata->subreq.len = written; + __set_bit(NETFS_SREQ_MADE_PROGRESS, &wdata->subreq.flags); + } + break; + case MID_REQUEST_SUBMITTED: + trace_netfs_sreq(&wdata->subreq, netfs_sreq_trace_io_req_submitted); + __set_bit(NETFS_SREQ_NEED_RETRY, &wdata->subreq.flags); + result = -EAGAIN; + break; + case MID_RETRY_NEEDED: + trace_netfs_sreq(&wdata->subreq, netfs_sreq_trace_io_retry_needed); + __set_bit(NETFS_SREQ_NEED_RETRY, &wdata->subreq.flags); + result = -EAGAIN; + break; + case MID_RESPONSE_MALFORMED: + trace_netfs_sreq(&wdata->subreq, netfs_sreq_trace_io_malformed); + credits.value = le16_to_cpu(rsp->hdr.CreditRequest); + credits.instance = server->reconnect_instance; + result = smb_EIO(smb_eio_trace_write_rsp_malformed); + break; + default: + trace_netfs_sreq(&wdata->subreq, netfs_sreq_trace_io_unknown); + result = smb_EIO1(smb_eio_trace_write_mid_state_unknown, + mid->mid_state); + break; + } +#ifdef CONFIG_CIFS_SMB_DIRECT + /* + * If this wdata has a memory registered, the MR can be freed + * The number of MRs available is limited, it's important to recover + * used MR as soon as I/O is finished. Hold MR longer in the later + * I/O process can possibly result in I/O deadlock due to lack of MR + * to send request on I/O retry + */ + if (wdata->mr) { + smbd_deregister_mr(wdata->mr); + wdata->mr = NULL; + } +#endif + if (result) { + wdata->result = result; + cifs_stats_fail_inc(tcon, SMB2_WRITE_HE); + trace_smb3_write_err(wdata->rreq->debug_id, + wdata->subreq.debug_index, + wdata->xid, + wdata->req->cfile->fid.persistent_fid, + tcon->tid, tcon->ses->Suid, wdata->subreq.start, + wdata->subreq.len, wdata->result); + if (wdata->result == -ENOSPC) + pr_warn_once("Out of space writing to %s\n", + tcon->tree_name); + } else + trace_smb3_write_done(wdata->rreq->debug_id, + wdata->subreq.debug_index, + wdata->xid, + wdata->req->cfile->fid.persistent_fid, + tcon->tid, tcon->ses->Suid, + wdata->subreq.start, wdata->subreq.len); + + trace_smb3_rw_credits(rreq_debug_id, subreq_debug_index, wdata->credits.value, + server->credits, server->in_flight, + 0, cifs_trace_rw_credits_write_response_clear); + wdata->credits.value = 0; + + /* see if we need to retry */ + if (is_replayable_error(wdata->result) && + smb2_should_replay(tcon, + &wdata->retries, + &wdata->cur_sleep)) + wdata->replay = true; + + cifs_write_subrequest_terminated(wdata, result ?: written); + release_mid(server, mid); + trace_smb3_rw_credits(rreq_debug_id, subreq_debug_index, 0, + server->credits, server->in_flight, + credits.value, cifs_trace_rw_credits_write_response_add); + add_credits(server, &credits, 0); +} + +/* smb2_async_writev - send an async write, and set up mid to handle result */ +void +smb2_async_writev(struct cifs_io_subrequest *wdata) +{ + int rc = -EACCES, flags = 0; + struct smb2_write_req *req = NULL; + struct smb2_hdr *shdr; + struct cifs_tcon *tcon = tlink_tcon(wdata->req->cfile->tlink); + struct TCP_Server_Info *server = wdata->server; + struct kvec iov[1]; + struct smb_rqst rqst = { }; + unsigned int total_len, xid = wdata->xid; + struct cifs_io_parms _io_parms; + struct cifs_io_parms *io_parms = NULL; + int credit_request; + + /* + * in future we may get cifs_io_parms passed in from the caller, + * but for now we construct it here... + */ + _io_parms = (struct cifs_io_parms) { + .tcon = tcon, + .server = server, + .offset = wdata->subreq.start, + .length = wdata->subreq.len, + .persistent_fid = wdata->req->cfile->fid.persistent_fid, + .volatile_fid = wdata->req->cfile->fid.volatile_fid, + .pid = wdata->req->pid, + }; + io_parms = &_io_parms; + + rc = smb2_plain_req_init(SMB2_WRITE, tcon, server, + (void **) &req, &total_len); + if (rc) + goto out; + + rqst.rq_iov = iov; + rqst.rq_iter = wdata->subreq.io_iter; + + rqst.rq_iov[0].iov_len = total_len - 1; + rqst.rq_iov[0].iov_base = (char *)req; + rqst.rq_nvec += 1; + + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; + + shdr = (struct smb2_hdr *)req; + shdr->Id.SyncId.ProcessId = cpu_to_le32(io_parms->pid); + + req->PersistentFileId = io_parms->persistent_fid; + req->VolatileFileId = io_parms->volatile_fid; + req->WriteChannelInfoOffset = 0; + req->WriteChannelInfoLength = 0; + req->Channel = SMB2_CHANNEL_NONE; + req->Length = cpu_to_le32(io_parms->length); + req->Offset = cpu_to_le64(io_parms->offset); + req->DataOffset = cpu_to_le16( + offsetof(struct smb2_write_req, Buffer)); + req->RemainingBytes = 0; + + trace_smb3_write_enter(wdata->rreq->debug_id, + wdata->subreq.debug_index, + wdata->xid, + io_parms->persistent_fid, + io_parms->tcon->tid, + io_parms->tcon->ses->Suid, + io_parms->offset, + io_parms->length); + +#ifdef CONFIG_CIFS_SMB_DIRECT + /* + * If we want to do a server RDMA read, fill in and append + * smbdirect_buffer_descriptor_v1 to the end of write request + */ + if (smb3_use_rdma_offload(io_parms)) { + struct smbdirect_buffer_descriptor_v1 *v1; + bool need_invalidate = server->dialect == SMB30_PROT_ID; + + wdata->mr = smbd_register_mr(server->smbd_conn, &wdata->subreq.io_iter, + false, need_invalidate); + if (!wdata->mr) { + rc = -EAGAIN; + goto async_writev_out; + } + /* For RDMA read, I/O size is in RemainingBytes not in Length */ + req->RemainingBytes = req->Length; + req->Length = 0; + req->DataOffset = 0; + req->Channel = SMB2_CHANNEL_RDMA_V1_INVALIDATE; + if (need_invalidate) + req->Channel = SMB2_CHANNEL_RDMA_V1; + req->WriteChannelInfoOffset = + cpu_to_le16(offsetof(struct smb2_write_req, Buffer)); + req->WriteChannelInfoLength = + cpu_to_le16(sizeof(struct smbdirect_buffer_descriptor_v1)); + v1 = (struct smbdirect_buffer_descriptor_v1 *) &req->Buffer[0]; + smbd_mr_fill_buffer_descriptor(wdata->mr, v1); + + rqst.rq_iov[0].iov_len += sizeof(*v1); + + /* + * We keep wdata->subreq.io_iter, + * but we have to truncate rqst.rq_iter + */ + iov_iter_truncate(&rqst.rq_iter, 0); + } +#endif + + if (wdata->replay) { + /* Back-off before retry */ + if (wdata->cur_sleep) + msleep(wdata->cur_sleep); + smb2_set_replay(server, &rqst); + } + + cifs_dbg(FYI, "async write at %llu %u bytes iter=%zx\n", + io_parms->offset, io_parms->length, iov_iter_count(&wdata->subreq.io_iter)); + + if (wdata->credits.value > 0) { + shdr->CreditCharge = cpu_to_le16(DIV_ROUND_UP(wdata->subreq.len, + SMB2_MAX_BUFFER_SIZE)); + credit_request = le16_to_cpu(shdr->CreditCharge) + 8; + if (server->credits >= server->max_credits) + shdr->CreditRequest = cpu_to_le16(0); + else + shdr->CreditRequest = cpu_to_le16( + min_t(int, server->max_credits - + server->credits, credit_request)); + + rc = adjust_credits(server, wdata, cifs_trace_rw_credits_call_writev_adjust); + if (rc) + goto async_writev_out; + + flags |= CIFS_HAS_CREDITS; + } + + /* XXX: compression + encryption is unsupported for now */ + if (((flags & CIFS_TRANSFORM_REQ) != CIFS_TRANSFORM_REQ) && should_compress(tcon, &rqst)) + flags |= CIFS_COMPRESS_REQ; + + rc = cifs_call_async(server, &rqst, NULL, smb2_writev_callback, NULL, + wdata, flags, &wdata->credits); + /* Can't touch wdata if rc == 0 */ + if (rc) { + trace_smb3_write_err(wdata->rreq->debug_id, + wdata->subreq.debug_index, + xid, + io_parms->persistent_fid, + io_parms->tcon->tid, + io_parms->tcon->ses->Suid, + io_parms->offset, + io_parms->length, + rc); + cifs_stats_fail_inc(tcon, SMB2_WRITE_HE); + } + +async_writev_out: + cifs_small_buf_release(req); +out: + /* if the send error is retryable, let netfs know about it */ + if (is_replayable_error(rc) && + smb2_should_replay(tcon, + &wdata->retries, + &wdata->cur_sleep)) { + wdata->replay = true; + trace_netfs_sreq(&wdata->subreq, netfs_sreq_trace_io_retry_needed); + __set_bit(NETFS_SREQ_NEED_RETRY, &wdata->subreq.flags); + } + + if (rc) { + trace_smb3_rw_credits(wdata->rreq->debug_id, + wdata->subreq.debug_index, + wdata->credits.value, + server->credits, server->in_flight, + -(int)wdata->credits.value, + cifs_trace_rw_credits_write_response_clear); + add_credits_and_wake_if(wdata->server, &wdata->credits, 0); + cifs_write_subrequest_terminated(wdata, rc); + } +} + +/* + * SMB2_write function gets iov pointer to kvec array with n_vec as a length. + * The length field from io_parms must be at least 1 and indicates a number of + * elements with data to write that begins with position 1 in iov array. All + * data length is specified by count. + */ +int +SMB2_write(const unsigned int xid, struct cifs_io_parms *io_parms, + unsigned int *nbytes, struct kvec *iov, int n_vec) +{ + struct smb_rqst rqst; + int rc = 0; + struct smb2_write_req *req = NULL; + struct smb2_write_rsp *rsp = NULL; + int resp_buftype; + struct kvec rsp_iov; + int flags = 0; + unsigned int total_len; + struct TCP_Server_Info *server; + int retries = 0, cur_sleep = 0; + +replay_again: + /* reinitialize for possible replay */ + flags = 0; + *nbytes = 0; + if (!io_parms->server) + io_parms->server = cifs_pick_channel(io_parms->tcon->ses); + server = io_parms->server; + if (server == NULL) + return -ECONNABORTED; + + if (n_vec < 1) + return rc; + + rc = smb2_plain_req_init(SMB2_WRITE, io_parms->tcon, server, + (void **) &req, &total_len); + if (rc) + return rc; + + if (smb3_encryption_required(io_parms->tcon)) + flags |= CIFS_TRANSFORM_REQ; + + req->hdr.Id.SyncId.ProcessId = cpu_to_le32(io_parms->pid); + + req->PersistentFileId = io_parms->persistent_fid; + req->VolatileFileId = io_parms->volatile_fid; + req->WriteChannelInfoOffset = 0; + req->WriteChannelInfoLength = 0; + req->Channel = 0; + req->Length = cpu_to_le32(io_parms->length); + req->Offset = cpu_to_le64(io_parms->offset); + req->DataOffset = cpu_to_le16( + offsetof(struct smb2_write_req, Buffer)); + req->RemainingBytes = 0; + + trace_smb3_write_enter(0, 0, xid, io_parms->persistent_fid, + io_parms->tcon->tid, io_parms->tcon->ses->Suid, + io_parms->offset, io_parms->length); + + iov[0].iov_base = (char *)req; + /* 1 for Buffer */ + iov[0].iov_len = total_len - 1; + + memset(&rqst, 0, sizeof(struct smb_rqst)); + rqst.rq_iov = iov; + /* iov[0] is the SMB header; move payload to rq_iter for encryption safety */ + rqst.rq_nvec = 1; + iov_iter_kvec(&rqst.rq_iter, ITER_SOURCE, &iov[1], n_vec, + io_parms->length); + + if (retries) { + /* Back-off before retry */ + if (cur_sleep) + msleep(cur_sleep); + smb2_set_replay(server, &rqst); + } + + rc = cifs_send_recv(xid, io_parms->tcon->ses, server, + &rqst, + &resp_buftype, flags, &rsp_iov); + rsp = (struct smb2_write_rsp *)rsp_iov.iov_base; + + if (rc) { + trace_smb3_write_err(0, 0, xid, + req->PersistentFileId, + io_parms->tcon->tid, + io_parms->tcon->ses->Suid, + io_parms->offset, io_parms->length, rc); + cifs_stats_fail_inc(io_parms->tcon, SMB2_WRITE_HE); + cifs_dbg(VFS, "Send error in write = %d\n", rc); + } else { + *nbytes = le32_to_cpu(rsp->DataLength); + cifs_stats_bytes_written(io_parms->tcon, *nbytes); + trace_smb3_write_done(0, 0, xid, + req->PersistentFileId, + io_parms->tcon->tid, + io_parms->tcon->ses->Suid, + io_parms->offset, *nbytes); + } + + cifs_small_buf_release(req); + free_rsp_buf(resp_buftype, rsp); + + if (is_replayable_error(rc) && + smb2_should_replay(io_parms->tcon, &retries, &cur_sleep)) + goto replay_again; + + return rc; +} + +int posix_info_sid_size(const void *beg, const void *end) +{ + size_t subauth; + int total; + + if (beg + 2 > end) + return -1; + + subauth = *(u8 *)(beg+1); + if (subauth < 1 || subauth > 15) + return -1; + + total = 1 + 1 + 6 + 4*subauth; + if (beg + total > end) + return -1; + + return total; +} + +int posix_info_parse(const void *beg, const void *end, + struct smb2_posix_info_parsed *out) + +{ + int total_len = 0; + int owner_len, group_len; + int name_len; + const void *owner_sid; + const void *group_sid; + const void *name; + + /* if no end bound given, assume payload to be correct */ + if (!end) { + const struct smb2_posix_info *p = beg; + + end = beg + le32_to_cpu(p->NextEntryOffset); + /* last element will have a 0 offset, pick a sensible bound */ + if (end == beg) + end += 0xFFFF; + } + + /* check base buf */ + if (beg + sizeof(struct smb2_posix_info) > end) + return -1; + total_len = sizeof(struct smb2_posix_info); + + /* check owner sid */ + owner_sid = beg + total_len; + owner_len = posix_info_sid_size(owner_sid, end); + if (owner_len < 0) + return -1; + total_len += owner_len; + + /* check group sid */ + group_sid = beg + total_len; + group_len = posix_info_sid_size(group_sid, end); + if (group_len < 0) + return -1; + total_len += group_len; + + /* check name len */ + if (beg + total_len + 4 > end) + return -1; + name_len = le32_to_cpu(*(__le32 *)(beg + total_len)); + if (name_len < 1 || name_len > 0xFFFF) + return -1; + total_len += 4; + + /* check name */ + name = beg + total_len; + if (name + name_len > end) + return -1; + total_len += name_len; + + if (out) { + out->base = beg; + out->size = total_len; + out->name_len = name_len; + out->name = name; + memcpy(&out->owner, owner_sid, owner_len); + memcpy(&out->group, group_sid, group_len); + } + return total_len; +} + +static int posix_info_extra_size(const void *beg, const void *end) +{ + int len = posix_info_parse(beg, end, NULL); + + if (len < 0) + return -1; + return len - sizeof(struct smb2_posix_info); +} + +static unsigned int +num_entries(int infotype, char *bufstart, char *end_of_buf, char **lastentry, + size_t size) +{ + int len; + unsigned int entrycount = 0; + unsigned int next_offset = 0; + char *entryptr; + FILE_DIRECTORY_INFO *dir_info; + + if (bufstart == NULL) + return 0; + + entryptr = bufstart; + + while (1) { + if (entryptr + next_offset < entryptr || + entryptr + next_offset > end_of_buf || + entryptr + next_offset + size > end_of_buf) { + cifs_dbg(VFS, "malformed search entry would overflow\n"); + break; + } + + entryptr = entryptr + next_offset; + dir_info = (FILE_DIRECTORY_INFO *)entryptr; + + if (infotype == SMB_FIND_FILE_POSIX_INFO) + len = posix_info_extra_size(entryptr, end_of_buf); + else + len = le32_to_cpu(dir_info->FileNameLength); + + if (len < 0 || + entryptr + len < entryptr || + entryptr + len > end_of_buf || + entryptr + len + size > end_of_buf) { + cifs_dbg(VFS, "directory entry name would overflow frame end of buf %p\n", + end_of_buf); + break; + } + + *lastentry = entryptr; + entrycount++; + + next_offset = le32_to_cpu(dir_info->NextEntryOffset); + if (!next_offset) + break; + } + + return entrycount; +} + +/* + * Readdir/FindFirst + */ +int SMB2_query_directory_init(const unsigned int xid, + struct cifs_tcon *tcon, + struct TCP_Server_Info *server, + struct smb_rqst *rqst, + u64 persistent_fid, u64 volatile_fid, + int index, int info_level) +{ + struct smb2_query_directory_req *req; + unsigned char *bufptr; + __le16 asteriks = cpu_to_le16('*'); + unsigned int output_size = CIFSMaxBufSize - + MAX_SMB2_CREATE_RESPONSE_SIZE - + MAX_SMB2_CLOSE_RESPONSE_SIZE; + unsigned int total_len; + struct kvec *iov = rqst->rq_iov; + int len, rc; + + rc = smb2_plain_req_init(SMB2_QUERY_DIRECTORY, tcon, server, + (void **) &req, &total_len); + if (rc) + return rc; + + switch (info_level) { + case SMB_FIND_FILE_DIRECTORY_INFO: + req->FileInformationClass = FILE_DIRECTORY_INFORMATION; + break; + case SMB_FIND_FILE_ID_FULL_DIR_INFO: + req->FileInformationClass = FILEID_FULL_DIRECTORY_INFORMATION; + break; + case SMB_FIND_FILE_POSIX_INFO: + req->FileInformationClass = SMB_FIND_FILE_POSIX_INFO; + break; + case SMB_FIND_FILE_FULL_DIRECTORY_INFO: + req->FileInformationClass = FILE_FULL_DIRECTORY_INFORMATION; + break; + default: + cifs_tcon_dbg(VFS, "info level %u isn't supported\n", + info_level); + return -EINVAL; + } + + req->FileIndex = cpu_to_le32(index); + req->PersistentFileId = persistent_fid; + req->VolatileFileId = volatile_fid; + + len = 0x2; + bufptr = req->Buffer; + memcpy(bufptr, &asteriks, len); + + req->FileNameOffset = + cpu_to_le16(sizeof(struct smb2_query_directory_req)); + req->FileNameLength = cpu_to_le16(len); + /* + * BB could be 30 bytes or so longer if we used SMB2 specific + * buffer lengths, but this is safe and close enough. + */ + output_size = min_t(unsigned int, output_size, server->maxBuf); + output_size = min_t(unsigned int, output_size, 2 << 15); + req->OutputBufferLength = cpu_to_le32(output_size); + + iov[0].iov_base = (char *)req; + /* 1 for Buffer */ + iov[0].iov_len = total_len - 1; + + iov[1].iov_base = (char *)(req->Buffer); + iov[1].iov_len = len; + + trace_smb3_query_dir_enter(xid, persistent_fid, tcon->tid, + tcon->ses->Suid, index, output_size); + + return 0; +} + +void SMB2_query_directory_free(struct smb_rqst *rqst) +{ + if (rqst && rqst->rq_iov) { + cifs_small_buf_release(rqst->rq_iov[0].iov_base); /* request */ + } +} + +int +smb2_parse_query_directory(struct cifs_tcon *tcon, + struct kvec *rsp_iov, + int resp_buftype, + struct cifs_search_info *srch_inf) +{ + struct smb2_query_directory_rsp *rsp; + size_t info_buf_size; + char *end_of_smb; + int rc; + + rsp = (struct smb2_query_directory_rsp *)rsp_iov->iov_base; + + switch (srch_inf->info_level) { + case SMB_FIND_FILE_DIRECTORY_INFO: + info_buf_size = sizeof(FILE_DIRECTORY_INFO); + break; + case SMB_FIND_FILE_ID_FULL_DIR_INFO: + info_buf_size = sizeof(FILE_ID_FULL_DIR_INFO); + break; + case SMB_FIND_FILE_POSIX_INFO: + /* note that posix payload are variable size */ + info_buf_size = sizeof(struct smb2_posix_info); + break; + case SMB_FIND_FILE_FULL_DIRECTORY_INFO: + info_buf_size = sizeof(FILE_FULL_DIRECTORY_INFO); + break; + default: + cifs_tcon_dbg(VFS, "info level %u isn't supported\n", + srch_inf->info_level); + return -EINVAL; + } + + rc = smb2_validate_iov(le16_to_cpu(rsp->OutputBufferOffset), + le32_to_cpu(rsp->OutputBufferLength), rsp_iov, + info_buf_size); + if (rc) { + cifs_tcon_dbg(VFS, "bad info payload"); + return rc; + } + + srch_inf->unicode = true; + + if (srch_inf->ntwrk_buf_start) { + if (srch_inf->smallBuf) + cifs_small_buf_release(srch_inf->ntwrk_buf_start); + else if (srch_inf->is_dynamic_buf) + kfree(srch_inf->ntwrk_buf_start); + else + cifs_buf_release(srch_inf->ntwrk_buf_start); + } + srch_inf->ntwrk_buf_start = (char *)rsp; + srch_inf->srch_entries_start = srch_inf->last_entry = + (char *)rsp + le16_to_cpu(rsp->OutputBufferOffset); + end_of_smb = rsp_iov->iov_len + (char *)rsp; + + srch_inf->entries_in_buffer = num_entries( + srch_inf->info_level, + srch_inf->srch_entries_start, + end_of_smb, + &srch_inf->last_entry, + info_buf_size); + + srch_inf->index_of_last_entry += srch_inf->entries_in_buffer; + cifs_dbg(FYI, "num entries %d last_index %lld srch start %p srch end %p\n", + srch_inf->entries_in_buffer, srch_inf->index_of_last_entry, + srch_inf->srch_entries_start, srch_inf->last_entry); + if (resp_buftype == CIFS_LARGE_BUFFER) { + srch_inf->smallBuf = false; + srch_inf->is_dynamic_buf = false; + } else if (resp_buftype == CIFS_SMALL_BUFFER) { + srch_inf->smallBuf = true; + srch_inf->is_dynamic_buf = false; + } else if (resp_buftype == CIFS_DYNAMIC_BUFFER) { + srch_inf->smallBuf = false; + srch_inf->is_dynamic_buf = true; + } else { + cifs_tcon_dbg(VFS, "Invalid search buffer type\n"); + } + + return 0; +} + +int +SMB2_query_directory(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, int index, + struct cifs_search_info *srch_inf) +{ + struct smb_rqst rqst; + struct kvec iov[SMB2_QUERY_DIRECTORY_IOV_SIZE]; + struct smb2_query_directory_rsp *rsp = NULL; + int resp_buftype = CIFS_NO_BUFFER; + struct kvec rsp_iov; + int rc = 0; + struct cifs_ses *ses = tcon->ses; + struct TCP_Server_Info *server; + int flags = 0; + int retries = 0, cur_sleep = 0; + +replay_again: + /* reinitialize for possible replay */ + resp_buftype = CIFS_NO_BUFFER; + memset(&rsp_iov, 0, sizeof(rsp_iov)); + flags = 0; + server = cifs_pick_channel(ses); + + if (!ses || !(ses->server)) + return smb_EIO(smb_eio_trace_null_pointers); + + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; + + memset(&rqst, 0, sizeof(struct smb_rqst)); + memset(&iov, 0, sizeof(iov)); + rqst.rq_iov = iov; + rqst.rq_nvec = SMB2_QUERY_DIRECTORY_IOV_SIZE; + + rc = SMB2_query_directory_init(xid, tcon, server, + &rqst, persistent_fid, + volatile_fid, index, + srch_inf->info_level); + if (rc) + goto qdir_exit; + + if (retries) { + /* Back-off before retry */ + if (cur_sleep) + msleep(cur_sleep); + smb2_set_replay(server, &rqst); + } + + rc = cifs_send_recv(xid, ses, server, + &rqst, &resp_buftype, flags, &rsp_iov); + rsp = (struct smb2_query_directory_rsp *)rsp_iov.iov_base; + + if (rc) { + if (rc == -ENODATA && + rsp->hdr.Status == STATUS_NO_MORE_FILES) { + trace_smb3_query_dir_done(xid, persistent_fid, + tcon->tid, tcon->ses->Suid, index, 0); + srch_inf->endOfSearch = true; + rc = 0; + } else { + trace_smb3_query_dir_err(xid, persistent_fid, tcon->tid, + tcon->ses->Suid, index, 0, rc); + cifs_stats_fail_inc(tcon, SMB2_QUERY_DIRECTORY_HE); + } + goto qdir_exit; + } + + rc = smb2_parse_query_directory(tcon, &rsp_iov, resp_buftype, + srch_inf); + if (rc) { + trace_smb3_query_dir_err(xid, persistent_fid, tcon->tid, + tcon->ses->Suid, index, 0, rc); + goto qdir_exit; + } + resp_buftype = CIFS_NO_BUFFER; + + trace_smb3_query_dir_done(xid, persistent_fid, tcon->tid, + tcon->ses->Suid, index, srch_inf->entries_in_buffer); + +qdir_exit: + SMB2_query_directory_free(&rqst); + free_rsp_buf(resp_buftype, rsp); + + if (is_replayable_error(rc) && + smb2_should_replay(tcon, &retries, &cur_sleep)) + goto replay_again; + + return rc; +} + +int +SMB2_set_info_init(struct cifs_tcon *tcon, struct TCP_Server_Info *server, + struct smb_rqst *rqst, + u64 persistent_fid, u64 volatile_fid, u32 pid, + u8 info_class, u8 info_type, u32 additional_info, + void **data, unsigned int *size) +{ + struct smb2_set_info_req *req; + struct kvec *iov = rqst->rq_iov; + unsigned int i, total_len; + int rc; + + rc = smb2_plain_req_init(SMB2_SET_INFO, tcon, server, + (void **) &req, &total_len); + if (rc) + return rc; + + req->hdr.Id.SyncId.ProcessId = cpu_to_le32(pid); + req->InfoType = info_type; + req->FileInfoClass = info_class; + req->PersistentFileId = persistent_fid; + req->VolatileFileId = volatile_fid; + req->AdditionalInformation = cpu_to_le32(additional_info); + + req->BufferOffset = cpu_to_le16(sizeof(struct smb2_set_info_req)); + req->BufferLength = cpu_to_le32(*size); + + memcpy(req->Buffer, *data, *size); + total_len += *size; + + iov[0].iov_base = (char *)req; + /* 1 for Buffer */ + iov[0].iov_len = total_len - 1; + + for (i = 1; i < rqst->rq_nvec; i++) { + le32_add_cpu(&req->BufferLength, size[i]); + iov[i].iov_base = (char *)data[i]; + iov[i].iov_len = size[i]; + } + + return 0; +} + +void +SMB2_set_info_free(struct smb_rqst *rqst) +{ + if (rqst && rqst->rq_iov) + cifs_buf_release(rqst->rq_iov[0].iov_base); /* request */ +} + +static int +send_set_info(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, u32 pid, u8 info_class, + u8 info_type, u32 additional_info, unsigned int num, + void **data, unsigned int *size) +{ + struct smb_rqst rqst; + struct smb2_set_info_rsp *rsp = NULL; + struct kvec *iov; + struct kvec rsp_iov; + int rc = 0; + int resp_buftype; + struct cifs_ses *ses = tcon->ses; + struct TCP_Server_Info *server; + int flags = 0; + int retries = 0, cur_sleep = 0; + +replay_again: + /* reinitialize for possible replay */ + flags = 0; + server = cifs_pick_channel(ses); + + if (!ses || !server) + return smb_EIO(smb_eio_trace_null_pointers); + + if (!num) + return -EINVAL; + + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; + + iov = kmalloc_objs(struct kvec, num); + if (!iov) + return -ENOMEM; + + memset(&rqst, 0, sizeof(struct smb_rqst)); + rqst.rq_iov = iov; + rqst.rq_nvec = num; + + rc = SMB2_set_info_init(tcon, server, + &rqst, persistent_fid, volatile_fid, pid, + info_class, info_type, additional_info, + data, size); + if (rc) { + kfree(iov); + return rc; + } + + if (retries) { + /* Back-off before retry */ + if (cur_sleep) + msleep(cur_sleep); + smb2_set_replay(server, &rqst); + } + + rc = cifs_send_recv(xid, ses, server, + &rqst, &resp_buftype, flags, + &rsp_iov); + SMB2_set_info_free(&rqst); + rsp = (struct smb2_set_info_rsp *)rsp_iov.iov_base; + + if (rc != 0) { + cifs_stats_fail_inc(tcon, SMB2_SET_INFO_HE); + trace_smb3_set_info_err(xid, persistent_fid, tcon->tid, + ses->Suid, info_class, (__u32)info_type, rc); + } + + free_rsp_buf(resp_buftype, rsp); + kfree(iov); + + if (is_replayable_error(rc) && + smb2_should_replay(tcon, &retries, &cur_sleep)) + goto replay_again; + + return rc; +} + +int +SMB2_set_eof(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid, + u64 volatile_fid, u32 pid, loff_t new_eof) +{ + struct smb2_file_eof_info info; + void *data; + unsigned int size; + + info.EndOfFile = cpu_to_le64(new_eof); + + data = &info; + size = sizeof(struct smb2_file_eof_info); + + trace_smb3_set_eof(xid, persistent_fid, tcon->tid, tcon->ses->Suid, new_eof); + + return send_set_info(xid, tcon, persistent_fid, volatile_fid, + pid, FILE_END_OF_FILE_INFORMATION, SMB2_O_INFO_FILE, + 0, 1, &data, &size); +} + +int +SMB2_set_allocation(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, u32 pid, + loff_t allocation_size) +{ + struct smb2_file_alloc_info info; + void *data; + unsigned int size; + + info.AllocationSize = cpu_to_le64(allocation_size); + + data = &info; + size = sizeof(struct smb2_file_alloc_info); + + return send_set_info(xid, tcon, persistent_fid, volatile_fid, + pid, FILE_ALLOCATION_INFORMATION, SMB2_O_INFO_FILE, + 0, 1, &data, &size); +} + +int +SMB2_set_acl(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, + struct smb_ntsd *pnntsd, int pacllen, int aclflag) +{ + return send_set_info(xid, tcon, persistent_fid, volatile_fid, + current->tgid, 0, SMB2_O_INFO_SECURITY, aclflag, + 1, (void **)&pnntsd, &pacllen); +} + +int +SMB2_set_ea(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, + struct smb2_file_full_ea_info *buf, int len) +{ + return send_set_info(xid, tcon, persistent_fid, volatile_fid, + current->tgid, FILE_FULL_EA_INFORMATION, SMB2_O_INFO_FILE, + 0, 1, (void **)&buf, &len); +} + +int +SMB2_oplock_break(const unsigned int xid, struct cifs_tcon *tcon, + const u64 persistent_fid, const u64 volatile_fid, + __u8 oplock_level) +{ + struct smb_rqst rqst; + int rc; + struct smb2_oplock_break *req = NULL; + struct cifs_ses *ses = tcon->ses; + struct TCP_Server_Info *server; + int flags = CIFS_OBREAK_OP; + unsigned int total_len; + struct kvec iov[1]; + struct kvec rsp_iov; + int resp_buf_type; + int retries = 0, cur_sleep = 0; + +replay_again: + /* reinitialize for possible replay */ + flags = CIFS_OBREAK_OP; + server = cifs_pick_channel(ses); + + cifs_dbg(FYI, "SMB2_oplock_break\n"); + rc = smb2_plain_req_init(SMB2_OPLOCK_BREAK, tcon, server, + (void **) &req, &total_len); + if (rc) + return rc; + + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; + + req->VolatileFid = volatile_fid; + req->PersistentFid = persistent_fid; + req->OplockLevel = oplock_level; + req->hdr.CreditRequest = cpu_to_le16(1); + + flags |= CIFS_NO_RSP_BUF; + + iov[0].iov_base = (char *)req; + iov[0].iov_len = total_len; + + memset(&rqst, 0, sizeof(struct smb_rqst)); + rqst.rq_iov = iov; + rqst.rq_nvec = 1; + + if (retries) { + /* Back-off before retry */ + if (cur_sleep) + msleep(cur_sleep); + smb2_set_replay(server, &rqst); + } + + rc = cifs_send_recv(xid, ses, server, + &rqst, &resp_buf_type, flags, &rsp_iov); + cifs_small_buf_release(req); + if (rc) { + cifs_stats_fail_inc(tcon, SMB2_OPLOCK_BREAK_HE); + cifs_dbg(FYI, "Send error in Oplock Break = %d\n", rc); + } + + if (is_replayable_error(rc) && + smb2_should_replay(tcon, &retries, &cur_sleep)) + goto replay_again; + + return rc; +} + +void +smb2_copy_fs_info_to_kstatfs(struct smb2_fs_full_size_info *pfs_inf, + struct kstatfs *kst) +{ + kst->f_bsize = le32_to_cpu(pfs_inf->BytesPerSector) * + le32_to_cpu(pfs_inf->SectorsPerAllocationUnit); + kst->f_blocks = le64_to_cpu(pfs_inf->TotalAllocationUnits); + kst->f_bfree = kst->f_bavail = + le64_to_cpu(pfs_inf->CallerAvailableAllocationUnits); + return; +} + +static void +copy_posix_fs_info_to_kstatfs(FILE_SYSTEM_POSIX_INFO *response_data, + struct kstatfs *kst) +{ + kst->f_bsize = le32_to_cpu(response_data->BlockSize); + kst->f_blocks = le64_to_cpu(response_data->TotalBlocks); + kst->f_bfree = le64_to_cpu(response_data->BlocksAvail); + if (response_data->UserBlocksAvail == cpu_to_le64(-1)) + kst->f_bavail = kst->f_bfree; + else + kst->f_bavail = le64_to_cpu(response_data->UserBlocksAvail); + if (response_data->TotalFileNodes != cpu_to_le64(-1)) + kst->f_files = le64_to_cpu(response_data->TotalFileNodes); + if (response_data->FreeFileNodes != cpu_to_le64(-1)) + kst->f_ffree = le64_to_cpu(response_data->FreeFileNodes); + + return; +} + +static int +build_qfs_info_req(struct kvec *iov, struct cifs_tcon *tcon, + struct TCP_Server_Info *server, + int level, int outbuf_len, u64 persistent_fid, + u64 volatile_fid) +{ + int rc; + struct smb2_query_info_req *req; + unsigned int total_len; + + cifs_dbg(FYI, "Query FSInfo level %d\n", level); + + if ((tcon->ses == NULL) || server == NULL) + return smb_EIO(smb_eio_trace_null_pointers); + + rc = smb2_plain_req_init(SMB2_QUERY_INFO, tcon, server, + (void **) &req, &total_len); + if (rc) + return rc; + + req->InfoType = SMB2_O_INFO_FILESYSTEM; + req->FileInfoClass = level; + req->PersistentFileId = persistent_fid; + req->VolatileFileId = volatile_fid; + /* 1 for pad */ + req->InputBufferOffset = + cpu_to_le16(sizeof(struct smb2_query_info_req)); + req->OutputBufferLength = cpu_to_le32( + outbuf_len + sizeof(struct smb2_query_info_rsp)); + + iov->iov_base = (char *)req; + iov->iov_len = total_len; + return 0; +} + +static inline void free_qfs_info_req(struct kvec *iov) +{ + cifs_buf_release(iov->iov_base); +} + +int +SMB311_posix_qfs_info(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, struct kstatfs *fsdata) +{ + struct smb_rqst rqst; + struct smb2_query_info_rsp *rsp = NULL; + struct kvec iov; + struct kvec rsp_iov; + int rc = 0; + int resp_buftype; + struct cifs_ses *ses = tcon->ses; + struct TCP_Server_Info *server; + FILE_SYSTEM_POSIX_INFO *info = NULL; + int flags = 0; + int retries = 0, cur_sleep = 0; + +replay_again: + /* reinitialize for possible replay */ + flags = 0; + server = cifs_pick_channel(ses); + + rc = build_qfs_info_req(&iov, tcon, server, + FS_POSIX_INFORMATION, + sizeof(FILE_SYSTEM_POSIX_INFO), + persistent_fid, volatile_fid); + if (rc) + return rc; + + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; + + memset(&rqst, 0, sizeof(struct smb_rqst)); + rqst.rq_iov = &iov; + rqst.rq_nvec = 1; + + if (retries) { + /* Back-off before retry */ + if (cur_sleep) + msleep(cur_sleep); + smb2_set_replay(server, &rqst); + } + + rc = cifs_send_recv(xid, ses, server, + &rqst, &resp_buftype, flags, &rsp_iov); + free_qfs_info_req(&iov); + if (rc) { + cifs_stats_fail_inc(tcon, SMB2_QUERY_INFO_HE); + goto posix_qfsinf_exit; + } + rsp = (struct smb2_query_info_rsp *)rsp_iov.iov_base; + + info = (FILE_SYSTEM_POSIX_INFO *)( + le16_to_cpu(rsp->OutputBufferOffset) + (char *)rsp); + rc = smb2_validate_iov(le16_to_cpu(rsp->OutputBufferOffset), + le32_to_cpu(rsp->OutputBufferLength), &rsp_iov, + sizeof(FILE_SYSTEM_POSIX_INFO)); + if (!rc) + copy_posix_fs_info_to_kstatfs(info, fsdata); + +posix_qfsinf_exit: + free_rsp_buf(resp_buftype, rsp_iov.iov_base); + + if (is_replayable_error(rc) && + smb2_should_replay(tcon, &retries, &cur_sleep)) + goto replay_again; + + return rc; +} + +int +SMB2_QFS_attr(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, int level) +{ + struct smb_rqst rqst; + struct smb2_query_info_rsp *rsp = NULL; + struct kvec iov; + struct kvec rsp_iov; + int rc = 0; + int resp_buftype, max_len, min_len; + struct cifs_ses *ses = tcon->ses; + struct TCP_Server_Info *server; + unsigned int rsp_len, offset; + int flags = 0; + int retries = 0, cur_sleep = 0; + +replay_again: + /* reinitialize for possible replay */ + flags = 0; + server = cifs_pick_channel(ses); + + if (level == FS_DEVICE_INFORMATION) { + max_len = sizeof(FILE_SYSTEM_DEVICE_INFO); + min_len = sizeof(FILE_SYSTEM_DEVICE_INFO); + } else if (level == FS_ATTRIBUTE_INFORMATION) { + max_len = sizeof(FILE_SYSTEM_ATTRIBUTE_INFO) + MAX_FS_NAME_LEN; + min_len = sizeof(FILE_SYSTEM_ATTRIBUTE_INFO); + } else if (level == FS_SECTOR_SIZE_INFORMATION) { + max_len = sizeof(struct smb3_fs_ss_info); + min_len = sizeof(struct smb3_fs_ss_info); + } else if (level == FS_VOLUME_INFORMATION) { + max_len = sizeof(struct filesystem_vol_info) + MAX_VOL_LABEL_LEN; + min_len = sizeof(struct filesystem_vol_info); + } else { + cifs_dbg(FYI, "Invalid qfsinfo level %d\n", level); + return -EINVAL; + } + + rc = build_qfs_info_req(&iov, tcon, server, + level, max_len, + persistent_fid, volatile_fid); + if (rc) + return rc; + + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; + + memset(&rqst, 0, sizeof(struct smb_rqst)); + rqst.rq_iov = &iov; + rqst.rq_nvec = 1; + + if (retries) { + /* Back-off before retry */ + if (cur_sleep) + msleep(cur_sleep); + smb2_set_replay(server, &rqst); + } + + rc = cifs_send_recv(xid, ses, server, + &rqst, &resp_buftype, flags, &rsp_iov); + free_qfs_info_req(&iov); + if (rc) { + cifs_stats_fail_inc(tcon, SMB2_QUERY_INFO_HE); + goto qfsattr_exit; + } + rsp = (struct smb2_query_info_rsp *)rsp_iov.iov_base; + + rsp_len = le32_to_cpu(rsp->OutputBufferLength); + offset = le16_to_cpu(rsp->OutputBufferOffset); + rc = smb2_validate_iov(offset, rsp_len, &rsp_iov, min_len); + if (rc) + goto qfsattr_exit; + + if (level == FS_ATTRIBUTE_INFORMATION) + memcpy(&tcon->fsAttrInfo, offset + + (char *)rsp, min_t(unsigned int, + rsp_len, min_len)); + else if (level == FS_DEVICE_INFORMATION) + memcpy(&tcon->fsDevInfo, offset + + (char *)rsp, sizeof(FILE_SYSTEM_DEVICE_INFO)); + else if (level == FS_SECTOR_SIZE_INFORMATION) { + struct smb3_fs_ss_info *ss_info = (struct smb3_fs_ss_info *) + (offset + (char *)rsp); + tcon->ss_flags = le32_to_cpu(ss_info->Flags); + tcon->perf_sector_size = + le32_to_cpu(ss_info->PhysicalBytesPerSectorForPerf); + } else if (level == FS_VOLUME_INFORMATION) { + struct filesystem_vol_info *vol_info = (struct filesystem_vol_info *) + (offset + (char *)rsp); + tcon->vol_serial_number = le32_to_cpu(vol_info->VolumeSerialNumber); + tcon->vol_create_time = vol_info->VolumeCreationTime; + } + +qfsattr_exit: + free_rsp_buf(resp_buftype, rsp_iov.iov_base); + + if (is_replayable_error(rc) && + smb2_should_replay(tcon, &retries, &cur_sleep)) + goto replay_again; + + return rc; +} + +int +smb2_lockv(const unsigned int xid, struct cifs_tcon *tcon, + const __u64 persist_fid, const __u64 volatile_fid, const __u32 pid, + const __u32 num_lock, struct smb2_lock_element *buf) +{ + struct smb_rqst rqst; + int rc = 0; + struct smb2_lock_req *req = NULL; + struct kvec iov[2]; + struct kvec rsp_iov; + int resp_buf_type; + unsigned int count; + int flags = CIFS_NO_RSP_BUF; + unsigned int total_len; + struct TCP_Server_Info *server; + int retries = 0, cur_sleep = 0; + +replay_again: + /* reinitialize for possible replay */ + flags = CIFS_NO_RSP_BUF; + server = cifs_pick_channel(tcon->ses); + + cifs_dbg(FYI, "smb2_lockv num lock %d\n", num_lock); + + rc = smb2_plain_req_init(SMB2_LOCK, tcon, server, + (void **) &req, &total_len); + if (rc) + return rc; + + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; + + req->hdr.Id.SyncId.ProcessId = cpu_to_le32(pid); + req->LockCount = cpu_to_le16(num_lock); + + req->PersistentFileId = persist_fid; + req->VolatileFileId = volatile_fid; + + count = num_lock * sizeof(struct smb2_lock_element); + + iov[0].iov_base = (char *)req; + iov[0].iov_len = total_len - sizeof(struct smb2_lock_element); + iov[1].iov_base = (char *)buf; + iov[1].iov_len = count; + + cifs_stats_inc(&tcon->stats.cifs_stats.num_locks); + + memset(&rqst, 0, sizeof(struct smb_rqst)); + rqst.rq_iov = iov; + rqst.rq_nvec = 2; + + if (retries) { + /* Back-off before retry */ + if (cur_sleep) + msleep(cur_sleep); + smb2_set_replay(server, &rqst); + } + + trace_smb3_lock_enter(xid, persist_fid, tcon->tid, tcon->ses->Suid, + le64_to_cpu(buf[0].Offset), + le64_to_cpu(buf[0].Length), + le32_to_cpu(buf[0].Flags), num_lock, 0); + + rc = cifs_send_recv(xid, tcon->ses, server, + &rqst, &resp_buf_type, flags, + &rsp_iov); + cifs_small_buf_release(req); + if (rc) { + cifs_dbg(FYI, "Send error in smb2_lockv = %d\n", rc); + cifs_stats_fail_inc(tcon, SMB2_LOCK_HE); + trace_smb3_lock_err(xid, persist_fid, tcon->tid, + tcon->ses->Suid, + le64_to_cpu(buf[0].Offset), + le64_to_cpu(buf[0].Length), + le32_to_cpu(buf[0].Flags), num_lock, rc); + } else { + trace_smb3_lock_done(xid, persist_fid, tcon->tid, tcon->ses->Suid, + le64_to_cpu(buf[0].Offset), + le64_to_cpu(buf[0].Length), + le32_to_cpu(buf[0].Flags), num_lock, 0); + } + + if (is_replayable_error(rc) && + smb2_should_replay(tcon, &retries, &cur_sleep)) + goto replay_again; + + return rc; +} + +int +SMB2_lock(const unsigned int xid, struct cifs_tcon *tcon, + const __u64 persist_fid, const __u64 volatile_fid, const __u32 pid, + const __u64 length, const __u64 offset, const __u32 lock_flags, + const bool wait) +{ + struct smb2_lock_element lock; + + lock.Offset = cpu_to_le64(offset); + lock.Length = cpu_to_le64(length); + lock.Flags = cpu_to_le32(lock_flags); + if (!wait && lock_flags != SMB2_LOCKFLAG_UNLOCK) + lock.Flags |= cpu_to_le32(SMB2_LOCKFLAG_FAIL_IMMEDIATELY); + + return smb2_lockv(xid, tcon, persist_fid, volatile_fid, pid, 1, &lock); +} + +int +SMB2_lease_break(const unsigned int xid, struct cifs_tcon *tcon, + __u8 *lease_key, const __le32 lease_state) +{ + struct smb_rqst rqst; + int rc; + struct smb2_lease_ack *req = NULL; + struct cifs_ses *ses = tcon->ses; + int flags = CIFS_OBREAK_OP; + unsigned int total_len; + struct kvec iov[1]; + struct kvec rsp_iov; + int resp_buf_type; + __u64 *please_key_high; + __u64 *please_key_low; + struct TCP_Server_Info *server = cifs_pick_channel(tcon->ses); + + cifs_dbg(FYI, "SMB2_lease_break\n"); + rc = smb2_plain_req_init(SMB2_OPLOCK_BREAK, tcon, server, + (void **) &req, &total_len); + if (rc) + return rc; + + if (smb3_encryption_required(tcon)) + flags |= CIFS_TRANSFORM_REQ; + + req->hdr.CreditRequest = cpu_to_le16(1); + req->StructureSize = cpu_to_le16(36); + total_len += 12; + + memcpy(req->LeaseKey, lease_key, SMB2_LEASE_KEY_SIZE); + req->LeaseState = lease_state; + + flags |= CIFS_NO_RSP_BUF; + + iov[0].iov_base = (char *)req; + iov[0].iov_len = total_len; + + memset(&rqst, 0, sizeof(struct smb_rqst)); + rqst.rq_iov = iov; + rqst.rq_nvec = 1; + + rc = cifs_send_recv(xid, ses, server, + &rqst, &resp_buf_type, flags, &rsp_iov); + cifs_small_buf_release(req); + + please_key_low = (__u64 *)lease_key; + please_key_high = (__u64 *)(lease_key+8); + if (rc) { + cifs_stats_fail_inc(tcon, SMB2_OPLOCK_BREAK_HE); + trace_smb3_lease_ack_err(le32_to_cpu(lease_state), tcon->tid, + ses->Suid, *please_key_low, *please_key_high, rc); + cifs_dbg(FYI, "Send error in Lease Break = %d\n", rc); + } else + trace_smb3_lease_ack_done(le32_to_cpu(lease_state), tcon->tid, + ses->Suid, *please_key_low, *please_key_high); + + return rc; +} diff --git a/fs/smb/client/smb2pdu.h b/fs/smb/client/smb2pdu.h new file mode 100644 index 000000000..ab6c667be --- /dev/null +++ b/fs/smb/client/smb2pdu.h @@ -0,0 +1,315 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +/* + * + * Copyright (c) International Business Machines Corp., 2009, 2013 + * Etersoft, 2012 + * Author(s): Steve French (sfrench@us.ibm.com) + * Pavel Shilovsky (pshilovsky@samba.org) 2012 + * + */ + +#ifndef _SMB2PDU_H +#define _SMB2PDU_H + +#include +#include "cifsacl.h" + +/* 52 transform hdr + 64 hdr + 88 create rsp */ +#define SMB2_TRANSFORM_HEADER_SIZE 52 +#define MAX_SMB2_HDR_SIZE 204 + +/* The total header size for SMB2 read and write */ +#define SMB2_READWRITE_PDU_HEADER_SIZE (48 + sizeof(struct smb2_hdr)) + +/* + * Definitions for SMB2 Protocol Data Units (network frames) + * + * See MS-SMB2.PDF specification for protocol details. + * The Naming convention is the lower case version of the SMB2 + * command code name for the struct. Note that structures must be packed. + * + */ + +#define COMPOUND_FID 0xFFFFFFFFFFFFFFFFULL + +#define SMB2_SYMLINK_STRUCT_SIZE \ + (sizeof(struct smb2_err_rsp) + sizeof(struct smb2_symlink_err_rsp)) + +#define SYMLINK_ERROR_TAG 0x4c4d5953 + +struct smb2_symlink_err_rsp { + __le32 SymLinkLength; + __le32 SymLinkErrorTag; + __le32 ReparseTag; + __le16 ReparseDataLength; + __le16 UnparsedPathLength; + __le16 SubstituteNameOffset; + __le16 SubstituteNameLength; + __le16 PrintNameOffset; + __le16 PrintNameLength; + __le32 Flags; + __u8 PathBuffer[]; +} __packed; + +/* SMB 3.1.1 and later dialects. See MS-SMB2 section 2.2.2.1 */ +struct smb2_error_context_rsp { + __le32 ErrorDataLength; + __le32 ErrorId; + __u8 ErrorContextData[] __counted_by_le(ErrorDataLength); +} __packed; + +/* ErrorId values */ +#define SMB2_ERROR_ID_DEFAULT 0x00000000 +#define SMB2_ERROR_ID_SHARE_REDIRECT cpu_to_le32(0x72645253) /* "rdRS" */ + +/* Defines for Type field below (see MS-SMB2 2.2.2.2.2.1) */ +#define MOVE_DST_IPADDR_V4 cpu_to_le32(0x00000001) +#define MOVE_DST_IPADDR_V6 cpu_to_le32(0x00000002) + +struct move_dst_ipaddr { + __le32 Type; + __u32 Reserved; + __u8 address[16]; /* IPv4 followed by 12 bytes rsvd or IPv6 address */ +} __packed; + +struct share_redirect_error_context_rsp { + __le32 StructureSize; + __le32 NotificationType; + __le32 ResourceNameOffset; + __le32 ResourceNameLength; + __le16 Reserved; + __le16 TargetType; + __le32 IPAddrCount; + struct move_dst_ipaddr IpAddrMoveList[]; + /* __u8 ResourceName[] */ /* Name of share as counted Unicode string */ +} __packed; + +/* + * Maximum number of iovs we need for an open/create request. + * [0] : struct smb2_create_req + * [1] : path + * [2] : lease context + * [3] : durable context + * [4] : posix context + * [5] : time warp context + * [6] : query id context + * [7] : create ea context + * [8] : compound padding + */ +#define SMB2_CREATE_IOV_SIZE 9 + +/* + * Maximum size of a SMB2_CREATE response is 64 (smb2 header) + + * 88 (fixed part of create response) + 520 (path) + 208 (contexts) + + * 2 bytes of padding. + */ +#define MAX_SMB2_CREATE_RESPONSE_SIZE 880 + +#define SMB2_LEASE_READ_CACHING_HE 0x01 +#define SMB2_LEASE_HANDLE_CACHING_HE 0x02 +#define SMB2_LEASE_WRITE_CACHING_HE 0x04 + +/* See MS-SMB2 2.2.13.2.5 */ +struct crt_twarp_ctxt { + struct create_context_hdr ccontext; + __u8 Name[8]; + __le64 Timestamp; + +} __packed; + +/* See MS-SMB2 2.2.13.2.9 */ +struct crt_query_id_ctxt { + struct create_context_hdr ccontext; + __u8 Name[8]; +} __packed; + +struct crt_sd_ctxt { + struct create_context_hdr ccontext; + __u8 Name[8]; + struct smb3_sd sd; +} __packed; + +/* See MS-FSCC 2.3.29 and 2.3.30 */ +struct get_retrieval_pointer_count_req { + __le64 StartingVcn; /* virtual cluster number (signed) */ +} __packed; + +struct get_retrieval_pointer_count_rsp { + __le32 ExtentCount; +} __packed; + +/* + * See MS-FSCC 2.3.33 and 2.3.34 + * request is the same as get_retrieval_point_count_req struct above + */ +struct smb3_extents { + __le64 NextVcn; + __le64 Lcn; /* logical cluster number */ +} __packed; + +struct get_retrieval_pointers_refcount_rsp { + __le32 ExtentCount; + __u32 Reserved; + __le64 StartingVcn; + struct smb3_extents extents[]; +} __packed; + +/* See MS-DFSC 2.2.2 */ +struct fsctl_get_dfs_referral_req { + __le16 MaxReferralLevel; + __u8 RequestFileName[]; +} __packed; + +/* DFS response is struct get_dfs_refer_rsp */ + +/* See MS-SMB2 2.2.31.3 */ +struct network_resiliency_req { + __le32 Timeout; + __le32 Reserved; +} __packed; +/* There is no buffer for the response ie no struct network_resiliency_rsp */ + +#define NO_FILE_ID 0xFFFFFFFFFFFFFFFFULL /* general ioctls to srv not to file */ + +/* + * Maximum number of iovs we need for an ioctl request. + * [0] : struct smb2_ioctl_req + * [1] : in_data + */ +#define SMB2_IOCTL_IOV_SIZE 2 + +/* + * PDU query infolevel structure definitions + * BB consider moving to a different header + */ + +struct smb2_file_full_ea_info { /* encoding of response for level 15 */ + __le32 next_entry_offset; + __u8 flags; + __u8 ea_name_length; + __le16 ea_value_length; + char ea_data[]; /* \0 terminated name plus value */ +} __packed; /* level 15 Set */ + +struct smb2_file_reparse_point_info { + __le64 IndexNumber; + __le32 Tag; +} __packed; + +/* See MS-FSCC 2.4.26 */ +struct smb2_file_id_information { + __le64 VolumeSerialNumber; + __u64 PersistentFileId; /* opaque endianness */ + __u64 VolatileFileId; /* opaque endianness */ +} __packed; /* level 59 */ + +/* See MS-FSCC 2.4.18 */ +struct smb2_file_id_extd_directory_info { + __le32 NextEntryOffset; + __u32 FileIndex; + __le64 CreationTime; + __le64 LastAccessTime; + __le64 LastWriteTime; + __le64 ChangeTime; + __le64 EndOfFile; + __le64 AllocationSize; + __le32 FileAttributes; + __le32 FileNameLength; + __le32 EaSize; /* EA size */ + __le32 ReparsePointTag; /* valid if FILE_ATTR_REPARSE_POINT set in FileAttributes */ + __le64 UniqueId; /* inode num - le since Samba puts ino in low 32 bit */ + char FileName[]; +} __packed; /* level 60 */ + +extern char smb2_padding[7]; + +/* + * See POSIX-SMB2 2.2.14.2.16 + * Link: https://gitlab.com/samba-team/smb3-posix-spec/-/blob/master/smb3_posix_extensions.md + */ +struct create_posix_rsp { + u32 nlink; + u32 reparse_tag; + u32 mode; + struct smb_sid owner; /* var-sized on the wire */ + struct smb_sid group; /* var-sized on the wire */ +} __packed; + +#define SMB2_QUERY_DIRECTORY_IOV_SIZE 2 + +/* + * SMB2-only POSIX info level for query dir + * + * See posix_info_sid_size(), posix_info_extra_size() and + * posix_info_parse() to help with the handling of this struct. + */ +struct smb2_posix_info { + __le32 NextEntryOffset; + __u32 Ignored; + __le64 CreationTime; + __le64 LastAccessTime; + __le64 LastWriteTime; + __le64 ChangeTime; + __le64 EndOfFile; + __le64 AllocationSize; + __le32 DosAttributes; + __le64 Inode; + __le32 DeviceId; + __le32 Zero; + /* beginning of POSIX Create Context Response */ + __le32 HardLinks; + __le32 ReparseTag; + __le32 Mode; + /* + * var sized owner SID + * var sized group SID + * le32 filenamelength + * u8 filename[] + */ +} __packed; + +/* + * Parsed version of the above struct. Allows direct access to the + * variable length fields + */ +struct smb2_posix_info_parsed { + const struct smb2_posix_info *base; + size_t size; + struct smb_sid owner; + struct smb_sid group; + int name_len; + const u8 *name; +}; + +struct smb2_create_ea_ctx { + struct create_context_hdr ctx; + __u8 name[8]; + struct smb2_file_full_ea_info ea; +} __packed; + +#define SMB2_WSL_XATTR_UID "$LXUID" +#define SMB2_WSL_XATTR_GID "$LXGID" +#define SMB2_WSL_XATTR_MODE "$LXMOD" +#define SMB2_WSL_XATTR_DEV "$LXDEV" +#define SMB2_WSL_XATTR_NAME_LEN 6 +#define SMB2_WSL_NUM_XATTRS 4 + +#define SMB2_WSL_XATTR_UID_SIZE 4 +#define SMB2_WSL_XATTR_GID_SIZE 4 +#define SMB2_WSL_XATTR_MODE_SIZE 4 +#define SMB2_WSL_XATTR_DEV_SIZE 8 + +#define SMB2_WSL_MIN_QUERY_EA_RESP_SIZE \ + (ALIGN((SMB2_WSL_NUM_XATTRS - 1) * \ + (SMB2_WSL_XATTR_NAME_LEN + 1 + \ + sizeof(struct smb2_file_full_ea_info)), 4) + \ + SMB2_WSL_XATTR_NAME_LEN + 1 + sizeof(struct smb2_file_full_ea_info)) + +#define SMB2_WSL_MAX_QUERY_EA_RESP_SIZE \ + (ALIGN(SMB2_WSL_MIN_QUERY_EA_RESP_SIZE + \ + SMB2_WSL_XATTR_UID_SIZE + \ + SMB2_WSL_XATTR_GID_SIZE + \ + SMB2_WSL_XATTR_MODE_SIZE + \ + SMB2_WSL_XATTR_DEV_SIZE, 4)) + +#endif /* _SMB2PDU_H */ diff --git a/fs/smb/client/smb2proto.h b/fs/smb/client/smb2proto.h new file mode 100644 index 000000000..2e9f70096 --- /dev/null +++ b/fs/smb/client/smb2proto.h @@ -0,0 +1,278 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +/* + * + * Copyright (c) International Business Machines Corp., 2002, 2011 + * Etersoft, 2012 + * Author(s): Steve French (sfrench@us.ibm.com) + * Pavel Shilovsky (pshilovsky@samba.org) 2012 + * + */ +#ifndef _SMB2PROTO_H +#define _SMB2PROTO_H + +#include +#include +#include "cached_dir.h" + +struct smb_rqst; + +/* + ***************************************************************** + * All Prototypes + ***************************************************************** + */ +int map_smb2_to_linux_error(char *buf, bool log_err); +int smb2_init_maperror(void); +#if IS_ENABLED(CONFIG_SMB_KUNIT_TESTS) +const struct status_to_posix_error *smb2_get_err_map_test(__u32 smb2_status); +extern const struct status_to_posix_error *smb2_error_map_table_test; +extern unsigned int smb2_error_map_num; +#endif + +int smb2_check_message(char *buf, unsigned int pdu_len, unsigned int len, + struct TCP_Server_Info *server); +unsigned int smb2_calc_size(void *buf); +char *smb2_get_data_area_len(int *off, int *len, struct smb2_hdr *shdr); +__le16 *cifs_convert_path_to_utf16(const char *from, + struct cifs_sb_info *cifs_sb); + +int smb2_verify_signature(struct smb_rqst *rqst, + struct TCP_Server_Info *server); +int smb2_check_receive(struct mid_q_entry *mid, struct TCP_Server_Info *server, + bool log_error); +struct mid_q_entry *smb2_setup_request(struct cifs_ses *ses, + struct TCP_Server_Info *server, + struct smb_rqst *rqst); +struct mid_q_entry *smb2_setup_async_request(struct TCP_Server_Info *server, + struct smb_rqst *rqst); +struct cifs_tcon *smb2_find_smb_tcon(struct TCP_Server_Info *server, + __u64 ses_id, __u32 tid); +__le32 smb2_get_lease_state(struct cifsInodeInfo *cinode, unsigned int oplock); +bool smb2_is_valid_oplock_break(char *buffer, struct TCP_Server_Info *server); +int smb3_handle_read_data(struct TCP_Server_Info *server, + struct mid_q_entry *mid); +struct inode *smb2_create_reparse_inode(struct cifs_open_info_data *data, + struct super_block *sb, + const unsigned int xid, + struct cifs_tcon *tcon, + const char *full_path, bool directory, + struct kvec *reparse_iov, + struct kvec *xattr_iov); +int smb2_query_reparse_point(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, + const char *full_path, u32 *tag, struct kvec *rsp, + int *rsp_buftype); +int smb2_query_path_info(const unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, const char *full_path, + struct cifs_open_info_data *data); +int smb2_set_path_size(const unsigned int xid, struct cifs_tcon *tcon, + const char *full_path, __u64 size, + struct cifs_sb_info *cifs_sb, bool set_alloc, + struct dentry *dentry); +int smb2_set_file_info(struct inode *inode, const char *full_path, + FILE_BASIC_INFO *buf, const unsigned int xid); +int smb311_posix_mkdir(const unsigned int xid, struct inode *inode, + umode_t mode, struct cifs_tcon *tcon, + const char *full_path, struct cifs_sb_info *cifs_sb); +int smb2_mkdir(const unsigned int xid, struct inode *parent_inode, + umode_t mode, struct cifs_tcon *tcon, const char *name, + struct cifs_sb_info *cifs_sb); +void smb2_mkdir_setinfo(struct inode *inode, const char *name, + struct cifs_sb_info *cifs_sb, struct cifs_tcon *tcon, + const unsigned int xid); +int smb2_rmdir(const unsigned int xid, struct cifs_tcon *tcon, + const char *name, struct cifs_sb_info *cifs_sb); +int smb2_unlink(const unsigned int xid, struct cifs_tcon *tcon, + const char *name, struct cifs_sb_info *cifs_sb, + struct dentry *dentry); +int smb2_rename_path(const unsigned int xid, struct cifs_tcon *tcon, + struct dentry *source_dentry, const char *from_name, + const char *to_name, struct cifs_sb_info *cifs_sb); +int smb2_create_hardlink(const unsigned int xid, struct cifs_tcon *tcon, + struct dentry *source_dentry, const char *from_name, + const char *to_name, struct cifs_sb_info *cifs_sb); +int smb3_create_mf_symlink(unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, + const unsigned char *path, char *pbuf, + unsigned int *pbytes_written); +int smb3_query_mf_symlink(unsigned int xid, struct cifs_tcon *tcon, + struct cifs_sb_info *cifs_sb, + const unsigned char *path, char *pbuf, + unsigned int *pbytes_read); +int smb2_fix_symlink_target_type(char **target, bool directory, + struct cifs_sb_info *cifs_sb); +int smb2_parse_native_symlink(char **target, const char *buf, unsigned int len, + bool relative, const char *full_path, + struct cifs_sb_info *cifs_sb); +int smb2_parse_symlink_response(struct cifs_sb_info *cifs_sb, + const struct kvec *iov, const char *full_path, + char **path); +int smb2_open_file(const unsigned int xid, struct cifs_open_parms *oparms, + __u32 *oplock, void *buf); +int smb2_unlock_range(struct cifsFileInfo *cfile, struct file_lock *flock, + const unsigned int xid); +int smb2_push_mandatory_locks(struct cifsFileInfo *cfile); +void smb2_reconnect_server(struct work_struct *work); +int smb3_crypto_aead_allocate(struct TCP_Server_Info *server); +unsigned long smb_rqst_len(struct TCP_Server_Info *server, + struct smb_rqst *rqst); +void smb2_set_next_command(struct cifs_tcon *tcon, struct smb_rqst *rqst); +void smb2_set_related(struct smb_rqst *rqst); +void smb2_set_replay(struct TCP_Server_Info *server, struct smb_rqst *rqst); +bool smb2_should_replay(struct cifs_tcon *tcon, int *pretries, + int *pcur_sleep); + +/* + * SMB2 Worker functions - most of protocol specific implementation details + * are contained within these calls. + */ +int SMB2_negotiate(const unsigned int xid, struct cifs_ses *ses, + struct TCP_Server_Info *server); +int SMB2_sess_setup(const unsigned int xid, struct cifs_ses *ses, + struct TCP_Server_Info *server, + const struct nls_table *nls_cp); +int SMB2_logoff(const unsigned int xid, struct cifs_ses *ses); +int SMB2_tcon(const unsigned int xid, struct cifs_ses *ses, const char *tree, + struct cifs_tcon *tcon, const struct nls_table *cp); +int SMB2_tdis(const unsigned int xid, struct cifs_tcon *tcon); +int SMB2_open(const unsigned int xid, struct cifs_open_parms *oparms, + __le16 *path, __u8 *oplock, struct cifs_open_info_data *buf, + struct create_posix_rsp *posix, struct kvec *err_iov, + int *buftype); +int SMB2_open_init(struct cifs_tcon *tcon, struct TCP_Server_Info *server, + struct smb_rqst *rqst, __u8 *oplock, + struct cifs_open_parms *oparms, __le16 *path); +void SMB2_open_free(struct smb_rqst *rqst); +int SMB2_ioctl(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, u32 opcode, char *in_data, + u32 indatalen, u32 max_out_data_len, char **out_data, + u32 *plen /* returned data len */); +int SMB2_ioctl_init(struct cifs_tcon *tcon, struct TCP_Server_Info *server, + struct smb_rqst *rqst, u64 persistent_fid, + u64 volatile_fid, u32 opcode, char *in_data, u32 indatalen, + __u32 max_response_size); +void SMB2_ioctl_free(struct smb_rqst *rqst); +int SMB2_change_notify(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, bool watch_tree, + u32 completion_filter, u32 max_out_data_len, + char **out_data, u32 *plen /* returned data len */); + +int __SMB2_close(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, + struct smb2_file_network_open_info *pbuf); +int SMB2_close(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid); +int SMB2_close_init(struct cifs_tcon *tcon, struct TCP_Server_Info *server, + struct smb_rqst *rqst, u64 persistent_fid, + u64 volatile_fid, bool query_attrs); +void SMB2_close_free(struct smb_rqst *rqst); +int SMB2_flush(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid); +int SMB2_flush_init(const unsigned int xid, struct smb_rqst *rqst, + struct cifs_tcon *tcon, struct TCP_Server_Info *server, + u64 persistent_fid, u64 volatile_fid); +void SMB2_flush_free(struct smb_rqst *rqst); +int SMB2_query_info(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, + struct smb2_file_all_info *data); +int SMB2_query_info_init(struct cifs_tcon *tcon, + struct TCP_Server_Info *server, struct smb_rqst *rqst, + u64 persistent_fid, u64 volatile_fid, u8 info_class, + u8 info_type, u32 additional_info, size_t output_len, + size_t input_len, void *input); +void SMB2_query_info_free(struct smb_rqst *rqst); +int SMB2_query_acl(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, void **data, + u32 *plen, u32 extra_info); +int SMB2_get_srv_num(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, __le64 *uniqueid); +int smb2_async_readv(struct cifs_io_subrequest *rdata); +int SMB2_read(const unsigned int xid, struct cifs_io_parms *io_parms, + unsigned int *nbytes, char **buf, int *buf_type); +void smb2_async_writev(struct cifs_io_subrequest *wdata); +int SMB2_write(const unsigned int xid, struct cifs_io_parms *io_parms, + unsigned int *nbytes, struct kvec *iov, int n_vec); +int SMB2_echo(struct TCP_Server_Info *server); +int SMB2_query_directory(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, int index, + struct cifs_search_info *srch_inf); +int SMB2_query_directory_init(const unsigned int xid, struct cifs_tcon *tcon, + struct TCP_Server_Info *server, + struct smb_rqst *rqst, u64 persistent_fid, + u64 volatile_fid, int index, int info_level); +void SMB2_query_directory_free(struct smb_rqst *rqst); +int SMB2_set_eof(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, u32 pid, + loff_t new_eof); +int SMB2_set_allocation(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, u32 pid, + loff_t allocation_size); +int SMB2_set_info_init(struct cifs_tcon *tcon, struct TCP_Server_Info *server, + struct smb_rqst *rqst, u64 persistent_fid, + u64 volatile_fid, u32 pid, u8 info_class, u8 info_type, + u32 additional_info, void **data, unsigned int *size); +void SMB2_set_info_free(struct smb_rqst *rqst); +int SMB2_set_acl(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, struct smb_ntsd *pnntsd, + int pacllen, int aclflag); +int SMB2_set_ea(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, + struct smb2_file_full_ea_info *buf, int len); +int SMB2_set_compression(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, + __u16 compression_state); +int SMB2_oplock_break(const unsigned int xid, struct cifs_tcon *tcon, + const u64 persistent_fid, const u64 volatile_fid, + __u8 oplock_level); +int smb2_handle_cancelled_close(struct cifs_tcon *tcon, __u64 persistent_fid, + __u64 volatile_fid); +int smb2_handle_cancelled_mid(struct mid_q_entry *mid, + struct TCP_Server_Info *server); +void smb2_cancelled_close_fid(struct work_struct *work); +int SMB311_posix_qfs_info(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, + struct kstatfs *fsdata); +int SMB2_QFS_attr(const unsigned int xid, struct cifs_tcon *tcon, + u64 persistent_fid, u64 volatile_fid, int level); +int SMB2_lock(const unsigned int xid, struct cifs_tcon *tcon, + const __u64 persist_fid, const __u64 volatile_fid, + const __u32 pid, const __u64 length, const __u64 offset, + const __u32 lock_flags, const bool wait); +int smb2_lockv(const unsigned int xid, struct cifs_tcon *tcon, + const __u64 persist_fid, const __u64 volatile_fid, + const __u32 pid, const __u32 num_lock, + struct smb2_lock_element *buf); +int SMB2_lease_break(const unsigned int xid, struct cifs_tcon *tcon, + __u8 *lease_key, const __le32 lease_state); +int smb3_validate_negotiate(const unsigned int xid, struct cifs_tcon *tcon); + +enum securityEnum smb2_select_sectype(struct TCP_Server_Info *server, + enum securityEnum requested); +int smb2_parse_contexts(struct TCP_Server_Info *server, struct kvec *rsp_iov, + __u16 *epoch, char *lease_key, __u8 *oplock, + struct smb2_file_all_info *buf, + struct create_posix_rsp *posix); + +int smb3_encryption_required(const struct cifs_tcon *tcon); +int smb2_validate_iov(unsigned int offset, unsigned int buffer_length, + struct kvec *iov, unsigned int min_buf_size); +int smb2_validate_and_copy_iov(unsigned int offset, unsigned int buffer_length, + struct kvec *iov, unsigned int minbufsize, + char *data); +void smb2_copy_fs_info_to_kstatfs(struct smb2_fs_full_size_info *pfs_inf, + struct kstatfs *kst); +void smb311_update_preauth_hash(struct cifs_ses *ses, + struct TCP_Server_Info *server, + struct kvec *iov, int nvec); +int smb2_query_info_compound(const unsigned int xid, struct cifs_tcon *tcon, + const char *path, u32 desired_access, u32 class, + u32 type, u32 output_len, struct kvec *rsp, + int *buftype, struct cifs_sb_info *cifs_sb); +/* query path info from the server using SMB311 POSIX extensions*/ +int posix_info_parse(const void *beg, const void *end, + struct smb2_posix_info_parsed *out); +int posix_info_sid_size(const void *beg, const void *end); +int smb2_rename_pending_delete(const char *full_path, struct dentry *dentry, + const unsigned int xid); + +#endif /* _SMB2PROTO_H */ diff --git a/fs/smb/client/smb2transport.c b/fs/smb/client/smb2transport.c new file mode 100644 index 000000000..d23566da2 --- /dev/null +++ b/fs/smb/client/smb2transport.c @@ -0,0 +1,816 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * Copyright (C) International Business Machines Corp., 2002, 2011 + * Etersoft, 2012 + * Author(s): Steve French (sfrench@us.ibm.com) + * Jeremy Allison (jra@samba.org) 2006 + * Pavel Shilovsky (pshilovsky@samba.org) 2012 + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "cifsglob.h" +#include "cifsproto.h" +#include "smb2proto.h" +#include "cifs_debug.h" +#include "../common/smb2status.h" +#include "smb2glob.h" + +static +int smb3_get_sign_key(__u64 ses_id, struct TCP_Server_Info *server, u8 *key) +{ + struct cifs_chan *chan; + struct TCP_Server_Info *pserver; + struct cifs_ses *ses = NULL; + int i; + int rc = 0; + bool is_binding = false; + + spin_lock(&cifs_tcp_ses_lock); + + /* If server is a channel, select the primary channel */ + pserver = SERVER_IS_CHAN(server) ? server->primary_server : server; + + list_for_each_entry(ses, &pserver->smb_ses_list, smb_ses_list) { + if (ses->Suid == ses_id) + goto found; + } + trace_smb3_ses_not_found(ses_id); + cifs_server_dbg(FYI, "%s: Could not find session 0x%llx\n", + __func__, ses_id); + rc = -ENOENT; + goto out; + +found: + spin_lock(&ses->ses_lock); + spin_lock(&ses->chan_lock); + + is_binding = (cifs_chan_needs_reconnect(ses, server) && + ses->ses_status == SES_GOOD); + if (is_binding) { + /* + * If we are in the process of binding a new channel + * to an existing session, use the master connection + * session key + */ + memcpy(key, ses->smb3signingkey, SMB3_SIGN_KEY_SIZE); + spin_unlock(&ses->chan_lock); + spin_unlock(&ses->ses_lock); + goto out; + } + + /* + * Otherwise, use the channel key. + */ + + for (i = 0; i < ses->chan_count; i++) { + chan = ses->chans + i; + if (chan->server == server) { + memcpy(key, chan->signkey, SMB3_SIGN_KEY_SIZE); + spin_unlock(&ses->chan_lock); + spin_unlock(&ses->ses_lock); + goto out; + } + } + spin_unlock(&ses->chan_lock); + spin_unlock(&ses->ses_lock); + + cifs_dbg(VFS, + "%s: Could not find channel signing key for session 0x%llx\n", + __func__, ses_id); + rc = -ENOENT; + +out: + spin_unlock(&cifs_tcp_ses_lock); + return rc; +} + +static struct cifs_ses * +smb2_find_smb_ses_unlocked(struct TCP_Server_Info *server, __u64 ses_id) +{ + struct TCP_Server_Info *pserver; + struct cifs_ses *ses; + + /* If server is a channel, select the primary channel */ + pserver = SERVER_IS_CHAN(server) ? server->primary_server : server; + + list_for_each_entry(ses, &pserver->smb_ses_list, smb_ses_list) { + if (ses->Suid != ses_id) + continue; + + spin_lock(&ses->ses_lock); + if (ses->ses_status == SES_EXITING) { + spin_unlock(&ses->ses_lock); + continue; + } + cifs_smb_ses_inc_refcount(ses); + spin_unlock(&ses->ses_lock); + return ses; + } + + return NULL; +} + +static int smb2_get_sign_key(struct TCP_Server_Info *server, + __u64 ses_id, u8 *key) +{ + struct cifs_ses *ses; + int rc = -ENOENT; + + if (SERVER_IS_CHAN(server)) + server = server->primary_server; + + spin_lock(&cifs_tcp_ses_lock); + list_for_each_entry(ses, &server->smb_ses_list, smb_ses_list) { + if (ses->Suid != ses_id) + continue; + + rc = 0; + spin_lock(&ses->ses_lock); + switch (ses->ses_status) { + case SES_EXITING: /* SMB2_LOGOFF */ + case SES_GOOD: + if (likely(ses->auth_key.response)) { + memcpy(key, ses->auth_key.response, + SMB2_NTLMV2_SESSKEY_SIZE); + } else { + rc = smb_EIO(smb_eio_trace_no_auth_key); + } + break; + default: + rc = -EAGAIN; + break; + } + spin_unlock(&ses->ses_lock); + break; + } + spin_unlock(&cifs_tcp_ses_lock); + return rc; +} + +static struct cifs_tcon * +smb2_find_smb_sess_tcon_unlocked(struct cifs_ses *ses, __u32 tid) +{ + struct cifs_tcon *tcon; + + list_for_each_entry(tcon, &ses->tcon_list, tcon_list) { + if (tcon->tid != tid) + continue; + spin_lock(&tcon->tc_lock); + ++tcon->tc_count; + spin_unlock(&tcon->tc_lock); + trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count, + netfs_trace_tcon_ref_get_find_sess_tcon); + return tcon; + } + + return NULL; +} + +/* + * Obtain tcon corresponding to the tid in the given + * cifs_ses + */ + +struct cifs_tcon * +smb2_find_smb_tcon(struct TCP_Server_Info *server, __u64 ses_id, __u32 tid) +{ + struct cifs_ses *ses; + struct cifs_tcon *tcon; + + spin_lock(&cifs_tcp_ses_lock); + ses = smb2_find_smb_ses_unlocked(server, ses_id); + if (!ses) { + spin_unlock(&cifs_tcp_ses_lock); + return NULL; + } + tcon = smb2_find_smb_sess_tcon_unlocked(ses, tid); + spin_unlock(&cifs_tcp_ses_lock); + /* tcon already has a ref to ses, so we don't need ses anymore */ + cifs_put_smb_ses(ses); + + return tcon; +} + +static int +smb2_calc_signature(struct smb_rqst *rqst, struct TCP_Server_Info *server) +{ + int rc; + unsigned char smb2_signature[SMB2_HMACSHA256_SIZE]; + struct kvec *iov = rqst->rq_iov; + struct smb2_hdr *shdr = (struct smb2_hdr *)iov[0].iov_base; + struct hmac_sha256_ctx hmac_ctx; + struct smb_rqst drqst; + __u64 sid = le64_to_cpu(shdr->SessionId); + u8 key[SMB2_NTLMV2_SESSKEY_SIZE]; + + rc = smb2_get_sign_key(server, sid, key); + if (unlikely(rc)) { + cifs_server_dbg(FYI, "%s: [sesid=0x%llx] couldn't find signing key: %d\n", + __func__, sid, rc); + return rc; + } + + memset(smb2_signature, 0x0, SMB2_HMACSHA256_SIZE); + memset(shdr->Signature, 0x0, SMB2_SIGNATURE_SIZE); + + hmac_sha256_init_usingrawkey(&hmac_ctx, key, sizeof(key)); + + /* + * For SMB2+, __cifs_calc_signature() expects to sign only the actual + * data, that is, iov[0] should not contain a rfc1002 length. + * + * Sign the rfc1002 length prior to passing the data (iov[1-N]) down to + * __cifs_calc_signature(). + */ + drqst = *rqst; + if (drqst.rq_nvec >= 2 && iov[0].iov_len == 4) { + hmac_sha256_update(&hmac_ctx, iov[0].iov_base, iov[0].iov_len); + drqst.rq_iov++; + drqst.rq_nvec--; + } + + rc = __cifs_calc_signature( + &drqst, server, smb2_signature, + &(struct cifs_calc_sig_ctx){ .hmac = &hmac_ctx }); + if (!rc) + memcpy(shdr->Signature, smb2_signature, SMB2_SIGNATURE_SIZE); + + return rc; +} + +static void generate_key(struct cifs_ses *ses, struct kvec label, + struct kvec context, __u8 *key, unsigned int key_size, + unsigned int full_key_size) +{ + unsigned char zero = 0x0; + __u8 i[4] = {0, 0, 0, 1}; + __u8 L128[4] = {0, 0, 0, 128}; + __u8 L256[4] = {0, 0, 1, 0}; + unsigned char prfhash[SMB2_HMACSHA256_SIZE]; + struct TCP_Server_Info *server = ses->server; + struct hmac_sha256_ctx hmac_ctx; + + memset(prfhash, 0x0, SMB2_HMACSHA256_SIZE); + memset(key, 0x0, key_size); + + hmac_sha256_init_usingrawkey(&hmac_ctx, ses->auth_key.response, + full_key_size); + hmac_sha256_update(&hmac_ctx, i, 4); + hmac_sha256_update(&hmac_ctx, label.iov_base, label.iov_len); + hmac_sha256_update(&hmac_ctx, &zero, 1); + hmac_sha256_update(&hmac_ctx, context.iov_base, context.iov_len); + + if ((server->cipher_type == SMB2_ENCRYPTION_AES256_CCM) || + (server->cipher_type == SMB2_ENCRYPTION_AES256_GCM)) { + hmac_sha256_update(&hmac_ctx, L256, 4); + } else { + hmac_sha256_update(&hmac_ctx, L128, 4); + } + hmac_sha256_final(&hmac_ctx, prfhash); + + memcpy(key, prfhash, key_size); +} + +struct derivation { + struct kvec label; + struct kvec context; +}; + +struct derivation_triplet { + struct derivation signing; + struct derivation encryption; + struct derivation decryption; +}; + +static int +generate_smb3signingkey(struct cifs_ses *ses, + struct TCP_Server_Info *server, + const struct derivation_triplet *ptriplet) +{ + unsigned int full_key_size = SMB2_NTLMV2_SESSKEY_SIZE; + bool is_binding = false; + int chan_index = 0; + + spin_lock(&ses->ses_lock); + spin_lock(&ses->chan_lock); + is_binding = (cifs_chan_needs_reconnect(ses, server) && + ses->ses_status == SES_GOOD); + + chan_index = cifs_ses_get_chan_index(ses, server); + if (chan_index == CIFS_INVAL_CHAN_INDEX) { + spin_unlock(&ses->chan_lock); + spin_unlock(&ses->ses_lock); + + return -EINVAL; + } + + spin_unlock(&ses->chan_lock); + spin_unlock(&ses->ses_lock); + + /* + * All channels use the same encryption/decryption keys but + * they have their own signing key. + * + * When we generate the keys, check if it is for a new channel + * (binding) in which case we only need to generate a signing + * key and store it in the channel as to not overwrite the + * master connection signing key stored in the session + */ + + if (is_binding) { + generate_key(ses, ptriplet->signing.label, + ptriplet->signing.context, + ses->chans[chan_index].signkey, SMB3_SIGN_KEY_SIZE, + SMB2_NTLMV2_SESSKEY_SIZE); + } else { + generate_key(ses, ptriplet->signing.label, + ptriplet->signing.context, ses->smb3signingkey, + SMB3_SIGN_KEY_SIZE, SMB2_NTLMV2_SESSKEY_SIZE); + + /* + * Per MS-SMB2 3.2.5.3.1, signing key always uses Session.SessionKey + * (first 16 bytes). Encryption/decryption keys use + * Session.FullSessionKey when dialect is 3.1.1 and cipher is + * AES-256-CCM or AES-256-GCM, otherwise Session.SessionKey. + */ + + if (server->dialect == SMB311_PROT_ID && + (server->cipher_type == SMB2_ENCRYPTION_AES256_CCM || + server->cipher_type == SMB2_ENCRYPTION_AES256_GCM)) + full_key_size = ses->auth_key.len; + + /* safe to access primary channel, since it will never go away */ + spin_lock(&ses->chan_lock); + memcpy(ses->chans[chan_index].signkey, ses->smb3signingkey, + SMB3_SIGN_KEY_SIZE); + spin_unlock(&ses->chan_lock); + + generate_key(ses, ptriplet->encryption.label, + ptriplet->encryption.context, + ses->smb3encryptionkey, SMB3_ENC_DEC_KEY_SIZE, + full_key_size); + + generate_key(ses, ptriplet->decryption.label, + ptriplet->decryption.context, + ses->smb3decryptionkey, SMB3_ENC_DEC_KEY_SIZE, + full_key_size); + } + +#ifdef CONFIG_CIFS_DEBUG_DUMP_KEYS + cifs_dbg(VFS, "%s: dumping generated AES session keys\n", __func__); + /* + * The session id is opaque in terms of endianness, so we can't + * print it as a long long. we dump it as we got it on the wire + */ + cifs_dbg(VFS, "Session Id %*ph\n", (int)sizeof(ses->Suid), + &ses->Suid); + cifs_dbg(VFS, "Cipher type %d\n", server->cipher_type); + cifs_dbg(VFS, "Session Key %*ph\n", + (int)ses->auth_key.len, ses->auth_key.response); + cifs_dbg(VFS, "Signing Key %*ph\n", + SMB3_SIGN_KEY_SIZE, ses->smb3signingkey); + if ((server->cipher_type == SMB2_ENCRYPTION_AES256_CCM) || + (server->cipher_type == SMB2_ENCRYPTION_AES256_GCM)) { + cifs_dbg(VFS, "ServerIn Key %*ph\n", + SMB3_GCM256_CRYPTKEY_SIZE, ses->smb3encryptionkey); + cifs_dbg(VFS, "ServerOut Key %*ph\n", + SMB3_GCM256_CRYPTKEY_SIZE, ses->smb3decryptionkey); + } else { + cifs_dbg(VFS, "ServerIn Key %*ph\n", + SMB3_GCM128_CRYPTKEY_SIZE, ses->smb3encryptionkey); + cifs_dbg(VFS, "ServerOut Key %*ph\n", + SMB3_GCM128_CRYPTKEY_SIZE, ses->smb3decryptionkey); + } +#endif + return 0; +} + +int +generate_smb30signingkey(struct cifs_ses *ses, + struct TCP_Server_Info *server) + +{ + struct derivation_triplet triplet; + struct derivation *d; + + d = &triplet.signing; + d->label.iov_base = "SMB2AESCMAC"; + d->label.iov_len = 12; + d->context.iov_base = "SmbSign"; + d->context.iov_len = 8; + + d = &triplet.encryption; + d->label.iov_base = "SMB2AESCCM"; + d->label.iov_len = 11; + d->context.iov_base = "ServerIn "; + d->context.iov_len = 10; + + d = &triplet.decryption; + d->label.iov_base = "SMB2AESCCM"; + d->label.iov_len = 11; + d->context.iov_base = "ServerOut"; + d->context.iov_len = 10; + + return generate_smb3signingkey(ses, server, &triplet); +} + +int +generate_smb311signingkey(struct cifs_ses *ses, + struct TCP_Server_Info *server) + +{ + struct derivation_triplet triplet; + struct derivation *d; + + d = &triplet.signing; + d->label.iov_base = "SMBSigningKey"; + d->label.iov_len = 14; + d->context.iov_base = ses->preauth_sha_hash; + d->context.iov_len = 64; + + d = &triplet.encryption; + d->label.iov_base = "SMBC2SCipherKey"; + d->label.iov_len = 16; + d->context.iov_base = ses->preauth_sha_hash; + d->context.iov_len = 64; + + d = &triplet.decryption; + d->label.iov_base = "SMBS2CCipherKey"; + d->label.iov_len = 16; + d->context.iov_base = ses->preauth_sha_hash; + d->context.iov_len = 64; + + return generate_smb3signingkey(ses, server, &triplet); +} + +static int +smb3_calc_signature(struct smb_rqst *rqst, struct TCP_Server_Info *server) +{ + int rc; + unsigned char smb3_signature[SMB2_CMACAES_SIZE]; + struct kvec *iov = rqst->rq_iov; + struct smb2_hdr *shdr = (struct smb2_hdr *)iov[0].iov_base; + struct aes_cmac_key cmac_key __cleanup(aes_cmac_zeroize_key); + struct aes_cmac_ctx cmac_ctx __cleanup(aes_cmac_zeroize_ctx); + struct smb_rqst drqst; + u8 key[SMB3_SIGN_KEY_SIZE]; + + if (server->vals->protocol_id <= SMB21_PROT_ID) + return smb2_calc_signature(rqst, server); + + rc = smb3_get_sign_key(le64_to_cpu(shdr->SessionId), server, key); + if (unlikely(rc)) { + cifs_server_dbg(FYI, "%s: Could not get signing key\n", __func__); + return rc; + } + + memset(smb3_signature, 0x0, SMB2_CMACAES_SIZE); + memset(shdr->Signature, 0x0, SMB2_SIGNATURE_SIZE); + + rc = aes_cmac_preparekey(&cmac_key, key, SMB2_CMACAES_SIZE); + if (rc) { + cifs_server_dbg(VFS, "%s: Could not set key for cmac aes\n", __func__); + return rc; + } + + aes_cmac_init(&cmac_ctx, &cmac_key); + + /* + * For SMB2+, __cifs_calc_signature() expects to sign only the actual + * data, that is, iov[0] should not contain a rfc1002 length. + * + * Sign the rfc1002 length prior to passing the data (iov[1-N]) down to + * __cifs_calc_signature(). + */ + drqst = *rqst; + if (drqst.rq_nvec >= 2 && iov[0].iov_len == 4) { + aes_cmac_update(&cmac_ctx, iov[0].iov_base, iov[0].iov_len); + drqst.rq_iov++; + drqst.rq_nvec--; + } + + rc = __cifs_calc_signature( + &drqst, server, smb3_signature, + &(struct cifs_calc_sig_ctx){ .cmac = &cmac_ctx }); + if (!rc) + memcpy(shdr->Signature, smb3_signature, SMB2_SIGNATURE_SIZE); + return rc; +} + +/* must be called with server->srv_mutex held */ +static int +smb2_sign_rqst(struct smb_rqst *rqst, struct TCP_Server_Info *server) +{ + struct smb2_hdr *shdr; + struct smb2_sess_setup_req *ssr; + bool is_binding; + bool is_signed; + + shdr = (struct smb2_hdr *)rqst->rq_iov[0].iov_base; + ssr = (struct smb2_sess_setup_req *)shdr; + + is_binding = shdr->Command == SMB2_SESSION_SETUP && + (ssr->Flags & SMB2_SESSION_REQ_FLAG_BINDING); + is_signed = shdr->Flags & SMB2_FLAGS_SIGNED; + + if (!is_signed) + return 0; + spin_lock(&server->srv_lock); + if (server->ops->need_neg && + server->ops->need_neg(server)) { + spin_unlock(&server->srv_lock); + return 0; + } + spin_unlock(&server->srv_lock); + if (!is_binding && !server->session_estab) { + strscpy(shdr->Signature, "BSRSPYL"); + return 0; + } + + return smb3_calc_signature(rqst, server); +} + +int +smb2_verify_signature(struct smb_rqst *rqst, struct TCP_Server_Info *server) +{ + unsigned int rc; + char server_response_sig[SMB2_SIGNATURE_SIZE]; + struct smb2_hdr *shdr = + (struct smb2_hdr *)rqst->rq_iov[0].iov_base; + + if ((shdr->Command == SMB2_NEGOTIATE) || + (shdr->Command == SMB2_SESSION_SETUP) || + (shdr->Command == SMB2_OPLOCK_BREAK) || + server->ignore_signature || + (!server->session_estab)) + return 0; + + /* + * BB what if signatures are supposed to be on for session but + * server does not send one? BB + */ + + /* Do not need to verify session setups with signature "BSRSPYL " */ + if (memcmp(shdr->Signature, "BSRSPYL ", 8) == 0) + cifs_dbg(FYI, "dummy signature received for smb command 0x%x\n", + shdr->Command); + + /* + * Save off the original signature so we can modify the smb and check + * our calculated signature against what the server sent. + */ + memcpy(server_response_sig, shdr->Signature, SMB2_SIGNATURE_SIZE); + + memset(shdr->Signature, 0, SMB2_SIGNATURE_SIZE); + + rc = smb3_calc_signature(rqst, server); + + if (rc) + return rc; + + if (crypto_memneq(server_response_sig, shdr->Signature, + SMB2_SIGNATURE_SIZE)) { + cifs_dbg(VFS, "sign fail cmd 0x%x message id 0x%llx\n", + shdr->Command, shdr->MessageId); + return -EACCES; + } else + return 0; +} + +/* + * Set message id for the request. Should be called after wait_for_free_request + * and when srv_mutex is held. + */ +static inline void +smb2_seq_num_into_buf(struct TCP_Server_Info *server, + struct smb2_hdr *shdr) +{ + unsigned int i, num = le16_to_cpu(shdr->CreditCharge); + + shdr->MessageId = get_next_mid64(server); + /* skip message numbers according to CreditCharge field */ + for (i = 1; i < num; i++) + get_next_mid(server); +} + +static struct mid_q_entry * +smb2_mid_entry_alloc(const struct smb2_hdr *shdr, + struct TCP_Server_Info *server) +{ + struct mid_q_entry *temp; + unsigned int credits = le16_to_cpu(shdr->CreditCharge); + + if (server == NULL) { + cifs_dbg(VFS, "Null TCP session in smb2_mid_entry_alloc\n"); + return NULL; + } + + temp = mempool_alloc(&cifs_mid_pool, GFP_NOFS); + memset(temp, 0, sizeof(struct mid_q_entry)); + refcount_set(&temp->refcount, 1); + spin_lock_init(&temp->mid_lock); + temp->mid = le64_to_cpu(shdr->MessageId); + temp->credits = credits > 0 ? credits : 1; + temp->pid = current->pid; + temp->command = shdr->Command; /* Always LE */ + temp->when_alloc = jiffies; + + /* + * The default is for the mid to be synchronous, so the + * default callback just wakes up the current task. + */ + get_task_struct(current); + temp->creator = current; + temp->callback = cifs_wake_up_task; + temp->callback_data = current; + + atomic_inc(&mid_count); + temp->mid_state = MID_REQUEST_ALLOCATED; + trace_smb3_cmd_enter(le32_to_cpu(shdr->Id.SyncId.TreeId), + le64_to_cpu(shdr->SessionId), + le16_to_cpu(shdr->Command), temp->mid); + return temp; +} + +static int +smb2_get_mid_entry(struct cifs_ses *ses, struct TCP_Server_Info *server, + struct smb2_hdr *shdr, struct mid_q_entry **mid) +{ + switch (READ_ONCE(server->tcpStatus)) { + case CifsExiting: + return -ENOENT; + case CifsNeedReconnect: + cifs_dbg(FYI, "tcp session dead - return to caller to retry\n"); + return -EAGAIN; + case CifsNeedNegotiate: + if (shdr->Command != SMB2_NEGOTIATE) + return -EAGAIN; + break; + default: + break; + } + + switch (READ_ONCE(ses->ses_status)) { + case SES_NEW: + if (shdr->Command != SMB2_SESSION_SETUP && + shdr->Command != SMB2_NEGOTIATE) + return -EAGAIN; + /* else ok - we are setting up session */ + break; + case SES_EXITING: + if (shdr->Command != SMB2_LOGOFF) + return -EAGAIN; + /* else ok - we are shutting down the session */ + break; + default: + break; + } + + *mid = smb2_mid_entry_alloc(shdr, server); + if (*mid == NULL) + return -ENOMEM; + spin_lock(&server->mid_queue_lock); + list_add_tail(&(*mid)->qhead, &server->pending_mid_q); + spin_unlock(&server->mid_queue_lock); + + return 0; +} + +int +smb2_check_receive(struct mid_q_entry *mid, struct TCP_Server_Info *server, + bool log_error) +{ + unsigned int len = mid->resp_buf_size; + struct kvec iov[1]; + struct smb_rqst rqst = { .rq_iov = iov, + .rq_nvec = 1 }; + + iov[0].iov_base = (char *)mid->resp_buf; + iov[0].iov_len = len; + + dump_smb(mid->resp_buf, min_t(u32, 80, len)); + /* convert the length into a more usable form */ + if (len > 24 && server->sign && !mid->decrypted) { + int rc; + + rc = smb2_verify_signature(&rqst, server); + if (rc) + cifs_server_dbg(VFS, "SMB signature verification returned error = %d\n", + rc); + } + + return map_smb2_to_linux_error(mid->resp_buf, log_error); +} + +struct mid_q_entry * +smb2_setup_request(struct cifs_ses *ses, struct TCP_Server_Info *server, + struct smb_rqst *rqst) +{ + int rc; + struct smb2_hdr *shdr = + (struct smb2_hdr *)rqst->rq_iov[0].iov_base; + struct mid_q_entry *mid; + + smb2_seq_num_into_buf(server, shdr); + + rc = smb2_get_mid_entry(ses, server, shdr, &mid); + if (rc) { + revert_current_mid_from_hdr(server, shdr); + return ERR_PTR(rc); + } + + rc = smb2_sign_rqst(rqst, server); + if (rc) { + revert_current_mid_from_hdr(server, shdr); + delete_mid(server, mid); + return ERR_PTR(rc); + } + + return mid; +} + +struct mid_q_entry * +smb2_setup_async_request(struct TCP_Server_Info *server, struct smb_rqst *rqst) +{ + int rc; + struct smb2_hdr *shdr = + (struct smb2_hdr *)rqst->rq_iov[0].iov_base; + struct mid_q_entry *mid; + + spin_lock(&server->srv_lock); + if (server->tcpStatus == CifsNeedNegotiate && + shdr->Command != SMB2_NEGOTIATE) { + spin_unlock(&server->srv_lock); + return ERR_PTR(-EAGAIN); + } + spin_unlock(&server->srv_lock); + + smb2_seq_num_into_buf(server, shdr); + + mid = smb2_mid_entry_alloc(shdr, server); + if (mid == NULL) { + revert_current_mid_from_hdr(server, shdr); + return ERR_PTR(-ENOMEM); + } + + rc = smb2_sign_rqst(rqst, server); + if (rc) { + revert_current_mid_from_hdr(server, shdr); + release_mid(server, mid); + return ERR_PTR(rc); + } + + return mid; +} + +int +smb3_crypto_aead_allocate(struct TCP_Server_Info *server) +{ + struct crypto_aead *tfm; + + if (!server->secmech.enc) { + if ((server->cipher_type == SMB2_ENCRYPTION_AES128_GCM) || + (server->cipher_type == SMB2_ENCRYPTION_AES256_GCM)) + tfm = crypto_alloc_aead("gcm(aes)", 0, 0); + else + tfm = crypto_alloc_aead("ccm(aes)", 0, 0); + if (IS_ERR(tfm)) { + cifs_server_dbg(VFS, "%s: Failed alloc encrypt aead\n", + __func__); + return PTR_ERR(tfm); + } + server->secmech.enc = tfm; + } + + if (!server->secmech.dec) { + if ((server->cipher_type == SMB2_ENCRYPTION_AES128_GCM) || + (server->cipher_type == SMB2_ENCRYPTION_AES256_GCM)) + tfm = crypto_alloc_aead("gcm(aes)", 0, 0); + else + tfm = crypto_alloc_aead("ccm(aes)", 0, 0); + if (IS_ERR(tfm)) { + crypto_free_aead(server->secmech.enc); + server->secmech.enc = NULL; + cifs_server_dbg(VFS, "%s: Failed to alloc decrypt aead\n", + __func__); + return PTR_ERR(tfm); + } + server->secmech.dec = tfm; + } + + return 0; +} diff --git a/fs/smb/client/smbdirect.c b/fs/smb/client/smbdirect.c new file mode 100644 index 000000000..563ef488a --- /dev/null +++ b/fs/smb/client/smbdirect.c @@ -0,0 +1,561 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017, Microsoft Corporation. + * + * Author(s): Long Li + */ + +#include "smbdirect.h" +#include "cifs_debug.h" +#include "cifsproto.h" +#include "smb2proto.h" + +/* Port numbers for SMBD transport */ +#define SMB_PORT 445 +#define SMBD_PORT 5445 + +/* Address lookup and resolve timeout in ms */ +#define RDMA_RESOLVE_TIMEOUT 5000 + +/* SMBD negotiation timeout in seconds */ +#define SMBD_NEGOTIATE_TIMEOUT 120 + +/* The timeout to wait for a keepalive message from peer in seconds */ +#define KEEPALIVE_RECV_TIMEOUT 5 + +/* + * Default maximum number of RDMA read/write outstanding on this connection + * This value is possibly decreased during QP creation on hardware limit + */ +#define SMBD_CM_RESPONDER_RESOURCES 32 + +/* + * User configurable initial values per SMBD transport connection + * as defined in [MS-SMBD] 3.1.1.1 + * Those may change after a SMBD negotiation + */ +/* The local peer's maximum number of credits to grant to the peer */ +int smbd_receive_credit_max = 255; + +/* The remote peer's credit request of local peer */ +int smbd_send_credit_target = 255; + +/* The maximum single message size can be sent to remote peer */ +int smbd_max_send_size = 1364; + +/* + * The maximum fragmented upper-layer payload receive size supported + * + * Assume max_payload_per_credit is + * smbd_max_receive_size - 24 = 1340 + * + * The maximum number would be + * smbd_receive_credit_max * max_payload_per_credit + * + * 1340 * 255 = 341700 (0x536C4) + * + * The minimum value from the spec is 131072 (0x20000) + * + * For now we use the logic we used in ksmbd before: + * (1364 * 255) / 2 = 173910 (0x2A756) + */ +int smbd_max_fragmented_recv_size = (1364 * 255) / 2; + +/* The maximum single-message size which can be received */ +int smbd_max_receive_size = 1364; + +/* The timeout to initiate send of a keepalive message on idle */ +int smbd_keep_alive_interval = 120; + +/* + * User configurable initial values for RDMA transport + * The actual values used may be lower and are limited to hardware capabilities + */ +/* Default maximum number of pages in a single RDMA write/read */ +int smbd_max_frmr_depth = 2048; + +/* If payload is less than this byte, use RDMA send/recv not read/write */ +int rdma_readwrite_threshold = 4096; + +/* Transport logging functions + * Logging are defined as classes. They can be OR'ed to define the actual + * logging level via module parameter smbd_logging_class + * e.g. cifs.smbd_logging_class=0xa0 will log all log_rdma_recv() and + * log_rdma_event() + */ +#define LOG_OUTGOING 0x1 +#define LOG_INCOMING 0x2 +#define LOG_READ 0x4 +#define LOG_WRITE 0x8 +#define LOG_RDMA_SEND 0x10 +#define LOG_RDMA_RECV 0x20 +#define LOG_KEEP_ALIVE 0x40 +#define LOG_RDMA_EVENT 0x80 +#define LOG_RDMA_MR 0x100 +static unsigned int smbd_logging_class; +module_param(smbd_logging_class, uint, 0644); +MODULE_PARM_DESC(smbd_logging_class, + "Logging class for SMBD transport 0x0 to 0x100"); + +#define ERR 0x0 +#define INFO 0x1 +static unsigned int smbd_logging_level = ERR; +module_param(smbd_logging_level, uint, 0644); +MODULE_PARM_DESC(smbd_logging_level, + "Logging level for SMBD transport, 0 (default): error, 1: info"); + +static bool smbd_logging_needed(struct smbdirect_socket *sc, + void *private_ptr, + unsigned int lvl, + unsigned int cls) +{ +#define BUILD_BUG_SAME(x) BUILD_BUG_ON(x != SMBDIRECT_LOG_ ##x) + BUILD_BUG_SAME(ERR); + BUILD_BUG_SAME(INFO); +#undef BUILD_BUG_SAME +#define BUILD_BUG_SAME(x) BUILD_BUG_ON(x != SMBDIRECT_ ##x) + BUILD_BUG_SAME(LOG_OUTGOING); + BUILD_BUG_SAME(LOG_INCOMING); + BUILD_BUG_SAME(LOG_READ); + BUILD_BUG_SAME(LOG_WRITE); + BUILD_BUG_SAME(LOG_RDMA_SEND); + BUILD_BUG_SAME(LOG_RDMA_RECV); + BUILD_BUG_SAME(LOG_KEEP_ALIVE); + BUILD_BUG_SAME(LOG_RDMA_EVENT); + BUILD_BUG_SAME(LOG_RDMA_MR); +#undef BUILD_BUG_SAME + + if (lvl <= smbd_logging_level || cls & smbd_logging_class) + return true; + return false; +} + +static void smbd_logging_vaprintf(struct smbdirect_socket *sc, + const char *func, + unsigned int line, + void *private_ptr, + unsigned int lvl, + unsigned int cls, + struct va_format *vaf) +{ + cifs_dbg(VFS, "%s:%u %pV", func, line, vaf); +} + +#define log_rdma(level, class, fmt, args...) \ +do { \ + if (level <= smbd_logging_level || class & smbd_logging_class) \ + cifs_dbg(VFS, "%s:%d " fmt, __func__, __LINE__, ##args);\ +} while (0) + +#define log_outgoing(level, fmt, args...) \ + log_rdma(level, LOG_OUTGOING, fmt, ##args) +#define log_incoming(level, fmt, args...) \ + log_rdma(level, LOG_INCOMING, fmt, ##args) +#define log_read(level, fmt, args...) log_rdma(level, LOG_READ, fmt, ##args) +#define log_write(level, fmt, args...) log_rdma(level, LOG_WRITE, fmt, ##args) +#define log_rdma_send(level, fmt, args...) \ + log_rdma(level, LOG_RDMA_SEND, fmt, ##args) +#define log_rdma_recv(level, fmt, args...) \ + log_rdma(level, LOG_RDMA_RECV, fmt, ##args) +#define log_keep_alive(level, fmt, args...) \ + log_rdma(level, LOG_KEEP_ALIVE, fmt, ##args) +#define log_rdma_event(level, fmt, args...) \ + log_rdma(level, LOG_RDMA_EVENT, fmt, ##args) +#define log_rdma_mr(level, fmt, args...) \ + log_rdma(level, LOG_RDMA_MR, fmt, ##args) + +static int smbd_post_send_full_iter(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch, + struct iov_iter *iter, + u32 remaining_data_length) +{ + int bytes = 0; + + /* + * smbdirect_connection_send_single_iter() respects the + * negotiated max_send_size, so we need to + * loop until the full iter is posted + */ + + while (iov_iter_count(iter) > 0) { + int rc; + + rc = smbdirect_connection_send_single_iter(sc, + batch, + iter, + 0, /* flags */ + remaining_data_length); + if (rc < 0) + return rc; + remaining_data_length -= rc; + bytes += rc; + } + + return bytes; +} + +/* + * Destroy the transport and related RDMA and memory resources + * Need to go through all the pending counters and make sure on one is using + * the transport while it is destroyed + */ +void smbd_destroy(struct TCP_Server_Info *server) +{ + struct smbd_connection *info = server->smbd_conn; + + if (!info) { + log_rdma_event(INFO, "rdma session already destroyed\n"); + return; + } + + smbdirect_socket_release(info->socket); + + kfree(info); + server->smbd_conn = NULL; +} + +/* + * Reconnect this SMBD connection, called from upper layer + * return value: 0 on success, or actual error code + */ +int smbd_reconnect(struct TCP_Server_Info *server) +{ + log_rdma_event(INFO, "reconnecting rdma session\n"); + + if (!server->smbd_conn) { + log_rdma_event(INFO, "rdma session already destroyed\n"); + goto create_conn; + } + + /* + * This is possible if transport is disconnected and we haven't received + * notification from RDMA, but upper layer has detected timeout + */ + log_rdma_event(INFO, "disconnecting transport\n"); + smbd_destroy(server); + +create_conn: + log_rdma_event(INFO, "creating rdma session\n"); + server->smbd_conn = smbd_get_connection( + server, (struct sockaddr *) &server->dstaddr); + + if (server->smbd_conn) { + cifs_dbg(VFS, "RDMA transport re-established\n"); + trace_smb3_smbd_connect_done(server->hostname, server->conn_id, &server->dstaddr); + return 0; + } + trace_smb3_smbd_connect_err(server->hostname, server->conn_id, &server->dstaddr); + return -ENOENT; +} + +/* Create a SMBD connection, called by upper layer */ +static struct smbd_connection *_smbd_get_connection( + struct TCP_Server_Info *server, struct sockaddr *dstaddr, int port) +{ + struct net *net = cifs_net_ns(server); + struct smbd_connection *info; + struct smbdirect_socket *sc; + struct smbdirect_socket_parameters init_params = {}; + struct smbdirect_socket_parameters *sp; + __be16 *sport; + u64 port_flags = 0; + int ret; + + switch (port) { + case SMBD_PORT: + /* + * only allow iWarp devices + * for port 5445. + */ + port_flags |= SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW; + break; + case SMB_PORT: + /* + * only allow InfiniBand, RoCEv1 or RoCEv2 + * devices for port 445. + * + * (Basically don't allow iWarp devices) + */ + port_flags |= SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB; + break; + } + + /* + * Create the initial parameters + */ + sp = &init_params; + sp->flags = port_flags; + sp->resolve_addr_timeout_msec = RDMA_RESOLVE_TIMEOUT; + sp->resolve_route_timeout_msec = RDMA_RESOLVE_TIMEOUT; + sp->rdma_connect_timeout_msec = RDMA_RESOLVE_TIMEOUT; + sp->negotiate_timeout_msec = SMBD_NEGOTIATE_TIMEOUT * 1000; + sp->initiator_depth = 1; + sp->responder_resources = SMBD_CM_RESPONDER_RESOURCES; + sp->recv_credit_max = smbd_receive_credit_max; + sp->send_credit_target = smbd_send_credit_target; + sp->max_send_size = smbd_max_send_size; + sp->max_fragmented_recv_size = smbd_max_fragmented_recv_size; + sp->max_recv_size = smbd_max_receive_size; + sp->max_frmr_depth = smbd_max_frmr_depth; + sp->keepalive_interval_msec = smbd_keep_alive_interval * 1000; + sp->keepalive_timeout_msec = KEEPALIVE_RECV_TIMEOUT * 1000; + + info = kzalloc_obj(*info); + if (!info) + return NULL; + ret = smbdirect_socket_create_kern(net, &sc); + if (ret) + goto socket_init_failed; + smbdirect_socket_set_logging(sc, NULL, smbd_logging_needed, smbd_logging_vaprintf); + ret = smbdirect_socket_set_initial_parameters(sc, sp); + if (ret) + goto set_params_failed; + ret = smbdirect_socket_set_kernel_settings(sc, IB_POLL_SOFTIRQ, GFP_KERNEL); + if (ret) + goto set_settings_failed; + + if (dstaddr->sa_family == AF_INET6) + sport = &((struct sockaddr_in6 *)dstaddr)->sin6_port; + else + sport = &((struct sockaddr_in *)dstaddr)->sin_port; + + *sport = htons(port); + + ret = smbdirect_connect_sync(sc, dstaddr); + if (ret) { + log_rdma_event(ERR, "connect to %pISpsfc failed: %1pe\n", + dstaddr, ERR_PTR(ret)); + goto connect_failed; + } + + info->socket = sc; + return info; + +connect_failed: +set_settings_failed: +set_params_failed: + smbdirect_socket_release(sc); +socket_init_failed: + kfree(info); + return NULL; +} + +const struct smbdirect_socket_parameters *smbd_get_parameters(struct smbd_connection *conn) +{ + if (unlikely(!conn->socket)) { + static const struct smbdirect_socket_parameters zero_params; + + return &zero_params; + } + + return smbdirect_socket_get_current_parameters(conn->socket); +} + +struct smbd_connection *smbd_get_connection( + struct TCP_Server_Info *server, struct sockaddr *dstaddr) +{ + struct smbd_connection *ret; + const struct smbdirect_socket_parameters *sp; + int port = SMBD_PORT; + +try_again: + ret = _smbd_get_connection(server, dstaddr, port); + + /* Try SMB_PORT if SMBD_PORT doesn't work */ + if (!ret && port == SMBD_PORT) { + port = SMB_PORT; + goto try_again; + } + if (!ret) + return NULL; + + sp = smbd_get_parameters(ret); + + server->rdma_readwrite_threshold = + rdma_readwrite_threshold > sp->max_fragmented_send_size ? + sp->max_fragmented_send_size : + rdma_readwrite_threshold; + + return ret; +} + +/* + * Receive data from the transport's receive reassembly queue + * All the incoming data packets are placed in reassembly queue + * iter: the buffer to read data into + * size: the length of data to read + * return value: actual data read + * + * Note: this implementation copies the data from reassembly queue to receive + * buffers used by upper layer. This is not the optimal code path. A better way + * to do it is to not have upper layer allocate its receive buffers but rather + * borrow the buffer from reassembly queue, and return it after data is + * consumed. But this will require more changes to upper layer code, and also + * need to consider packet boundaries while they still being reassembled. + */ +int smbd_recv(struct smbd_connection *info, struct msghdr *msg) +{ + struct smbdirect_socket *sc = info->socket; + + if (!smbdirect_connection_is_connected(sc)) + return -ENOTCONN; + + return smbdirect_connection_recvmsg(sc, msg, 0); +} + +/* + * Send data to transport + * Each rqst is transported as a SMBDirect payload + * rqst: the data to write + * return value: 0 if successfully write, otherwise error code + */ +int smbd_send(struct TCP_Server_Info *server, + int num_rqst, struct smb_rqst *rqst_array) +{ + struct smbd_connection *info = server->smbd_conn; + struct smbdirect_socket *sc = info->socket; + const struct smbdirect_socket_parameters *sp = smbd_get_parameters(info); + struct smb_rqst *rqst; + struct iov_iter iter; + struct smbdirect_send_batch_storage bstorage; + struct smbdirect_send_batch *batch; + unsigned int remaining_data_length, klen; + int rc, i, rqst_idx; + int error = 0; + + if (!smbdirect_connection_is_connected(sc)) + return -EAGAIN; + + /* + * Add in the page array if there is one. The caller needs to set + * rq_tailsz to PAGE_SIZE when the buffer has multiple pages and + * ends at page boundary + */ + remaining_data_length = 0; + for (i = 0; i < num_rqst; i++) + remaining_data_length += smb_rqst_len(server, &rqst_array[i]); + + if (unlikely(remaining_data_length > sp->max_fragmented_send_size)) { + /* assertion: payload never exceeds negotiated maximum */ + log_write(ERR, "payload size %d > max size %d\n", + remaining_data_length, sp->max_fragmented_send_size); + return -EINVAL; + } + + log_write(INFO, "num_rqst=%d total length=%u\n", + num_rqst, remaining_data_length); + + rqst_idx = 0; + batch = smbdirect_init_send_batch_storage(&bstorage, false, 0); + do { + rqst = &rqst_array[rqst_idx]; + + cifs_dbg(FYI, "Sending smb (RDMA): idx=%d smb_len=%lu\n", + rqst_idx, smb_rqst_len(server, rqst)); + for (i = 0; i < rqst->rq_nvec; i++) + dump_smb(rqst->rq_iov[i].iov_base, rqst->rq_iov[i].iov_len); + + log_write(INFO, "RDMA-WR[%u] nvec=%d len=%u iter=%zu rqlen=%lu\n", + rqst_idx, rqst->rq_nvec, remaining_data_length, + iov_iter_count(&rqst->rq_iter), smb_rqst_len(server, rqst)); + + /* Send the metadata pages. */ + klen = 0; + for (i = 0; i < rqst->rq_nvec; i++) + klen += rqst->rq_iov[i].iov_len; + iov_iter_kvec(&iter, ITER_SOURCE, rqst->rq_iov, rqst->rq_nvec, klen); + + rc = smbd_post_send_full_iter(sc, batch, &iter, remaining_data_length); + if (rc < 0) { + error = rc; + break; + } + remaining_data_length -= rc; + + if (iov_iter_count(&rqst->rq_iter) > 0) { + /* And then the data pages if there are any */ + rc = smbd_post_send_full_iter(sc, batch, &rqst->rq_iter, + remaining_data_length); + if (rc < 0) { + error = rc; + break; + } + remaining_data_length -= rc; + } + + } while (++rqst_idx < num_rqst); + + rc = smbdirect_connection_send_batch_flush(sc, batch, true); + if (unlikely(!rc && error)) + rc = error; + + /* + * As an optimization, we don't wait for individual I/O to finish + * before sending the next one. + * Send them all and wait for pending send count to get to 0 + * that means all the I/Os have been out and we are good to return + */ + + error = rc; + rc = smbdirect_connection_send_wait_zero_pending(sc); + if (unlikely(rc && !error)) + error = -EAGAIN; + + if (unlikely(error)) + return error; + + return 0; +} + +/* + * Register memory for RDMA read/write + * iter: the buffer to register memory with + * writing: true if this is a RDMA write (SMB read), false for RDMA read + * need_invalidate: true if this MR needs to be locally invalidated after I/O + * return value: the MR registered, NULL if failed. + */ +struct smbdirect_mr_io *smbd_register_mr(struct smbd_connection *info, + struct iov_iter *iter, + bool writing, bool need_invalidate) +{ + struct smbdirect_socket *sc = info->socket; + + if (!smbdirect_connection_is_connected(sc)) + return NULL; + + return smbdirect_connection_register_mr_io(sc, iter, writing, need_invalidate); +} + +void smbd_mr_fill_buffer_descriptor(struct smbdirect_mr_io *mr, + struct smbdirect_buffer_descriptor_v1 *v1) +{ + smbdirect_mr_io_fill_buffer_descriptor(mr, v1); +} + +/* + * Deregister a MR after I/O is done + * This function may wait if remote invalidation is not used + * and we have to locally invalidate the buffer to prevent data is being + * modified by remote peer after upper layer consumes it + */ +void smbd_deregister_mr(struct smbdirect_mr_io *mr) +{ + smbdirect_connection_deregister_mr_io(mr); +} + +void smbd_debug_proc_show(struct TCP_Server_Info *server, struct seq_file *m) +{ + if (!server->rdma) + return; + + if (!server->smbd_conn) { + seq_puts(m, "\nSMBDirect transport not available"); + return; + } + + smbdirect_connection_legacy_debug_proc_show(server->smbd_conn->socket, + server->rdma_readwrite_threshold, + m); +} + +MODULE_IMPORT_NS("SMBDIRECT"); diff --git a/fs/smb/client/smbdirect.h b/fs/smb/client/smbdirect.h new file mode 100644 index 000000000..be205ec02 --- /dev/null +++ b/fs/smb/client/smbdirect.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2017, Microsoft Corporation. + * + * Author(s): Long Li + */ +#ifndef _SMBDIRECT_H +#define _SMBDIRECT_H + +#ifdef CONFIG_CIFS_SMB_DIRECT +#define cifs_rdma_enabled(server) ((server)->rdma) + +#include "cifsglob.h" + +#include + +extern int rdma_readwrite_threshold; +extern int smbd_max_frmr_depth; +extern int smbd_keep_alive_interval; +extern int smbd_max_receive_size; +extern int smbd_max_fragmented_recv_size; +extern int smbd_max_send_size; +extern int smbd_send_credit_target; +extern int smbd_receive_credit_max; + +struct smbd_connection { + struct smbdirect_socket *socket; +}; + +/* Create a SMBDirect session */ +struct smbd_connection *smbd_get_connection( + struct TCP_Server_Info *server, struct sockaddr *dstaddr); + +const struct smbdirect_socket_parameters *smbd_get_parameters(struct smbd_connection *conn); + +/* Reconnect SMBDirect session */ +int smbd_reconnect(struct TCP_Server_Info *server); +/* Destroy SMBDirect session */ +void smbd_destroy(struct TCP_Server_Info *server); + +/* Interface for carrying upper layer I/O through send/recv */ +int smbd_recv(struct smbd_connection *info, struct msghdr *msg); +int smbd_send(struct TCP_Server_Info *server, + int num_rqst, struct smb_rqst *rqst); + +/* Interfaces to register and deregister MR for RDMA read/write */ +struct smbdirect_mr_io *smbd_register_mr( + struct smbd_connection *info, struct iov_iter *iter, + bool writing, bool need_invalidate); +void smbd_mr_fill_buffer_descriptor(struct smbdirect_mr_io *mr, + struct smbdirect_buffer_descriptor_v1 *v1); +void smbd_deregister_mr(struct smbdirect_mr_io *mr); + +void smbd_debug_proc_show(struct TCP_Server_Info *server, struct seq_file *m); + +#else +#define cifs_rdma_enabled(server) 0 +struct smbd_connection {}; +static inline void *smbd_get_connection( + struct TCP_Server_Info *server, struct sockaddr *dstaddr) {return NULL;} +static inline int smbd_reconnect(struct TCP_Server_Info *server) {return -1; } +static inline void smbd_destroy(struct TCP_Server_Info *server) {} +static inline int smbd_recv(struct smbd_connection *info, struct msghdr *msg) {return -1; } +static inline int smbd_send(struct TCP_Server_Info *server, int num_rqst, struct smb_rqst *rqst) {return -1; } +#endif + +#endif diff --git a/fs/smb/client/smbencrypt.c b/fs/smb/client/smbencrypt.c new file mode 100644 index 000000000..094b8296d --- /dev/null +++ b/fs/smb/client/smbencrypt.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + Unix SMB/Netbios implementation. + Version 1.9. + SMB parameters and setup + Copyright (C) Andrew Tridgell 1992-2000 + Copyright (C) Luke Kenneth Casson Leighton 1996-2000 + Modified by Jeremy Allison 1995. + Copyright (C) Andrew Bartlett 2002-2003 + Modified by Steve French (sfrench@us.ibm.com) 2002-2003 + +*/ + +#include +#include +#include +#include +#include +#include +#include +#include "cifs_fs_sb.h" +#include "cifs_unicode.h" +#include "cifsglob.h" +#include "cifs_debug.h" +#include "cifsproto.h" +#include "../common/md4.h" + +/* following came from the other byteorder.h to avoid include conflicts */ +#define CVAL(buf,pos) (((unsigned char *)(buf))[pos]) +#define SSVALX(buf,pos,val) (CVAL(buf,pos)=(val)&0xFF,CVAL(buf,pos+1)=(val)>>8) +#define SSVAL(buf,pos,val) SSVALX((buf),(pos),((__u16)(val))) + +/* produce a md4 message digest from data of length n bytes */ +static int +mdfour(unsigned char *md4_hash, unsigned char *link_str, int link_len) +{ + int rc; + struct md4_ctx mctx; + + rc = cifs_md4_init(&mctx); + if (rc) { + cifs_dbg(VFS, "%s: Could not init MD4\n", __func__); + goto mdfour_err; + } + rc = cifs_md4_update(&mctx, link_str, link_len); + if (rc) { + cifs_dbg(VFS, "%s: Could not update MD4\n", __func__); + goto mdfour_err; + } + rc = cifs_md4_final(&mctx, md4_hash); + if (rc) + cifs_dbg(VFS, "%s: Could not finalize MD4\n", __func__); + + +mdfour_err: + return rc; +} + +/* + * Creates the MD4 Hash of the users password in NT UNICODE. + */ + +int +E_md4hash(const unsigned char *passwd, unsigned char *p16, + const struct nls_table *codepage) +{ + int rc; + int len; + __le16 wpwd[129]; + + /* Password cannot be longer than 128 characters */ + if (passwd) /* Password must be converted to NT unicode */ + len = cifs_strtoUTF16(wpwd, passwd, 128, codepage); + else { + len = 0; + *wpwd = 0; /* Ensure string is null terminated */ + } + + rc = mdfour(p16, (unsigned char *) wpwd, len * sizeof(__le16)); + memzero_explicit(wpwd, sizeof(wpwd)); + + return rc; +} diff --git a/fs/smb/client/smberr.h b/fs/smb/client/smberr.h new file mode 100644 index 000000000..5c3415bf1 --- /dev/null +++ b/fs/smb/client/smberr.h @@ -0,0 +1,298 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +/* + * + * Copyright (c) International Business Machines Corp., 2002,2004 + * Author(s): Steve French (sfrench@us.ibm.com) + * + * See Error Codes section of the SNIA CIFS Specification + * for more information + * + */ + +struct smb_to_posix_error { + __u16 smb_err; + int posix_code; +}; + +/* The request was successful. */ +#define SUCCESS 0x00 +/* Error is from the core DOS operating system set */ +#define ERRDOS 0x01 +/* Error is generated by the file server daemon */ +#define ERRSRV 0x02 +/* Error is a hardware error. */ +#define ERRHRD 0x03 +/* Command was not in the "SMB" format. */ +#define ERRCMD 0xFF + +/* The following error codes may be generated with the SUCCESS error class.*/ + +/*#define SUCCESS 0 The request was successful. */ + +/* + * The following error codes may be generated with the ERRDOS error class. + * The comment at the end of each definition indicates the POSIX error + * code; it is used to generate the `mapping_table_ERRDOS` array. + */ + +/* + * Invalid function. The server did not + * recognize or could not perform a + * system call generated by the server, + * e.g., set the DIRECTORY attribute on + * a data file, invalid seek mode. + */ +#define ERRbadfunc 1 // -EINVAL +/* + * File not found. The last component + * of a file's pathname could not be + * found. + */ +#define ERRbadfile 2 // -ENOENT +/* + * Directory invalid. A directory + * component in a pathname could not be + * found. + */ +#define ERRbadpath 3 // -ENOTDIR +/* + * Too many open files. The server has + * no file handles available. + */ +#define ERRnofids 4 // -EMFILE +/* + * Access denied, the client's context + * does not permit the requested + * function. This includes the + * following conditions: invalid rename + * command, write to Fid open for read + * only, read on Fid open for write + * only, attempt to delete a non-empty + * directory + */ +#define ERRnoaccess 5 // -EACCES +/* + * Invalid file handle. The file handle + * specified was not recognized by the + * server. + */ +#define ERRbadfid 6 // -EBADF +/* Memory control blocks destroyed. */ +#define ERRbadmcb 7 // -EIO +/* + * Insufficient server memory to + * perform the requested function. + */ +#define ERRnomem 8 // -EREMOTEIO +/* Invalid memory block address. */ +#define ERRbadmem 9 // -EFAULT +/* Invalid environment. */ +#define ERRbadenv 10 // -EFAULT +/* Invalid format. */ +#define ERRbadformat 11 // -EINVAL +/* Invalid open mode. */ +#define ERRbadaccess 12 // -EACCES +/* + * Invalid data (generated only by + * IOCTL calls within the server). + */ +#define ERRbaddata 13 // -EIO +/* Invalid drive specified. */ +#define ERRbaddrive 15 // -ENXIO +/* + * A Delete Directory request attempted + * to remove the server's current + * directory. + */ +#define ERRremcd 16 // -EACCES +/* + * Not same device (e.g., a cross + * volume rename was attempted + */ +#define ERRdiffdevice 17 // -EXDEV +/* + * A File Search command can find no + * more files matching the specified + * criteria. + */ +#define ERRnofiles 18 // -ENOENT +/* media is write protected */ +#define ERRwriteprot 19 // -EROFS +#define ERRgeneral 31 +/* + * The sharing mode specified for an + * Open conflicts with existing FIDs on + * the file. + */ +#define ERRbadshare 32 // -EBUSY +/* + * A Lock request conflicted with an + * existing lock or specified an + * invalid mode, or an Unlock requested + * attempted to remove a lock held by + * another process. + */ +#define ERRlock 33 // -EACCES +#define ERRunsup 50 // -EINVAL +#define ERRnosuchshare 67 // -ENXIO +/* + * The file named in the request + * already exists. + */ +#define ERRfilexists 80 // -EEXIST +#define ERRinvparm 87 // -EINVAL +#define ERRdiskfull 112 // -ENOSPC +#define ERRinvname 123 // -ENOENT +#define ERRunknownlevel 124 // -EOPNOTSUPP +#define ERRdirnotempty 145 // -ENOTEMPTY +#define ERRnotlocked 158 // -ENOLCK +#define ERRcancelviolation 173 // -ENOLCK +#define ERRalreadyexists 183 // -EEXIST +#define ERRbadpipe 230 +#define ERRpipebusy 231 +#define ERRpipeclosing 232 +#define ERRnotconnected 233 +#define ERRmoredata 234 // -EOVERFLOW +#define ERReasnotsupported 282 // -EOPNOTSUPP +/* + * The operation would cause a quota + * limit to be exceeded. + */ +#define ErrQuota 0x200 // -EDQUOT +/* + * A link operation was performed on a + * pathname that was not a link. + */ +#define ErrNotALink 0x201 // -ENOLINK +#define ERRnetlogonNotStarted 2455 // -ENOPROTOOPT + +/* Below errors are used internally (do not come over the wire) for passthrough + from STATUS codes to POSIX only */ +#define ERRsymlink 0xFFFD // -EOPNOTSUPP +#define ErrTooManyLinks 0xFFFE // -EMLINK + +/* + * The following error codes may be generated with the ERRSRV error class. + * The comment at the end of each definition indicates the POSIX error + * code; it is used to generate the `mapping_table_ERRSRV` array. + */ + +/* + * Non-specific error code. It is + * returned under the following + * conditions: resource other than disk + * space exhausted (e.g. TIDs), first + * SMB command was not negotiate, + * multiple negotiates attempted, and + * internal server error. + */ +#define ERRerror 1 // -EIO +/* + * Bad password - name/password pair in + * a TreeConnect or Session Setup are + * invalid. + */ +#define ERRbadpw 2 // -EACCES +/* + * used for indicating DFS referral + * needed + */ +#define ERRbadtype 3 // -EREMOTE +/* + * The client does not have the + * necessary access rights within the + * specified context for requested + * function. + */ +#define ERRaccess 4 // -EACCES +/* + * The Tid specified in a command was + * invalid. + */ +#define ERRinvtid 5 // -ENXIO +/* + * Invalid network name in tree + * connect. + */ +#define ERRinvnetname 6 // -ENXIO +/* + * Invalid device - printer request + * made to non-printer connection or + * non-printer request made to printer + * connection. + */ +#define ERRinvdevice 7 // -ENXIO +/* + * Print queue full (files) -- returned + * by open print file. + */ +#define ERRqfull 49 // -ENOSPC +/* Print queue full -- no space. */ +#define ERRqtoobig 50 // -ENOSPC +/* EOF on print queue dump */ +#define ERRqeof 51 // -EIO +/* Invalid print file FID. */ +#define ERRinvpfid 52 // -EBADF +/* + * The server did not recognize the + * command received. + */ +#define ERRsmbcmd 64 // -EBADRQC +/* + * The server encountered an internal + * error, e.g., system file + * unavailable. + */ +#define ERRsrverror 65 // -EIO +/* (obsolete) */ +#define ERRbadBID 66 // -EIO +/* + * The Fid and pathname parameters + * contained an invalid combination of + * values. + */ +#define ERRfilespecs 67 // -EINVAL +/* (obsolete) */ +#define ERRbadLink 68 // -EIO +/* + * The access permissions specified for + * a file or directory are not a valid + * combination. + */ +#define ERRbadpermits 69 // -EINVAL +#define ERRbadPID 70 // -ESRCH +/* attribute (mode) is invalid */ +#define ERRsetattrmode 71 // -EINVAL +/* Server is paused */ +#define ERRpaused 81 // -EHOSTDOWN +/* reserved - messaging off */ +#define ERRmsgoff 82 // -EHOSTDOWN +/* reserved - no room for message */ +#define ERRnoroom 83 // -ENOSPC +/* reserved - too many remote names */ +#define ERRrmuns 87 // -EUSERS +/* operation timed out */ +#define ERRtimeout 88 // -ETIME +/* No resources available for request */ +#define ERRnoresource 89 // -EREMOTEIO +/* Too many UIDs active on this session */ +#define ERRtoomanyuids 90 // -EUSERS +/* The UID is not known as a valid user */ +#define ERRbaduid 91 // -EACCES +/* temporarily unable to use raw */ +#define ERRusempx 250 // -EIO +/* + * temporarily unable to use either raw + * or mpx + */ +#define ERRusestd 251 // -EIO +#define ERR_NOTIFY_ENUM_DIR 1024 // -ENOBUFS +/* user account does not exist */ +#define ERRnoSuchUser 2238 // -EACCES +#define ERRaccountexpired 2239 // -EKEYEXPIRED +/* can not logon from this client */ +#define ERRbadclient 2240 // -EACCES +/* logon hours do not allow this */ +#define ERRbadLogonTime 2241 // -EACCES +#define ERRpasswordExpired 2242 // -EKEYEXPIRED +#define ERRnosupport 0xFFFF // -EINVAL diff --git a/fs/smb/client/trace.c b/fs/smb/client/trace.c new file mode 100644 index 000000000..8a99b68d0 --- /dev/null +++ b/fs/smb/client/trace.c @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2018, Microsoft Corporation. + * + * Author(s): Steve French + */ +#include "cifsglob.h" +#include "cifs_spnego.h" +#define CREATE_TRACE_POINTS +#include "trace.h" diff --git a/fs/smb/client/trace.h b/fs/smb/client/trace.h new file mode 100644 index 000000000..5b21ad3c1 --- /dev/null +++ b/fs/smb/client/trace.h @@ -0,0 +1,1970 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2018, Microsoft Corporation. + * + * Author(s): Steve French + * + * Please use this 3-part article as a reference for writing new tracepoints: + * https://lwn.net/Articles/379903/ + */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM cifs + +#if !defined(_CIFS_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _CIFS_TRACE_H + +#include +#include +#include + +/* + * Specify enums for tracing information. + */ +#define smb_eio_traces \ + EM(smb_eio_trace_compress_copy, "compress_copy") \ + EM(smb_eio_trace_copychunk_inv_rsp, "copychunk_inv_rsp") \ + EM(smb_eio_trace_copychunk_overcopy_b, "copychunk_overcopy_b") \ + EM(smb_eio_trace_copychunk_overcopy_c, "copychunk_overcopy_c") \ + EM(smb_eio_trace_create_rsp_too_small, "create_rsp_too_small") \ + EM(smb_eio_trace_dfsref_no_rsp, "dfsref_no_rsp") \ + EM(smb_eio_trace_ea_overrun, "ea_overrun") \ + EM(smb_eio_trace_extract_will_pin, "extract_will_pin") \ + EM(smb_eio_trace_forced_shutdown, "forced_shutdown") \ + EM(smb_eio_trace_getacl_bcc_too_small, "getacl_bcc_too_small") \ + EM(smb_eio_trace_getcifsacl_param_count, "getcifsacl_param_count") \ + EM(smb_eio_trace_getdfsrefer_bcc_too_small, "getdfsrefer_bcc_too_small") \ + EM(smb_eio_trace_getextattr_bcc_too_small, "getextattr_bcc_too_small") \ + EM(smb_eio_trace_getextattr_inv_size, "getextattr_inv_size") \ + EM(smb_eio_trace_getsrvinonum_bcc_too_small, "getsrvinonum_bcc_too_small") \ + EM(smb_eio_trace_getsrvinonum_size, "getsrvinonum_size") \ + EM(smb_eio_trace_ioctl_data_len, "ioctl_data_len") \ + EM(smb_eio_trace_ioctl_no_rsp, "ioctl_no_rsp") \ + EM(smb_eio_trace_ioctl_out_off, "ioctl_out_off") \ + EM(smb_eio_trace_lock_bcc_too_small, "lock_bcc_too_small") \ + EM(smb_eio_trace_lock_data_too_small, "lock_data_too_small") \ + EM(smb_eio_trace_malformed_ksid_key, "malformed_ksid_key") \ + EM(smb_eio_trace_malformed_sid_key, "malformed_sid_key") \ + EM(smb_eio_trace_mkdir_no_rsp, "mkdir_no_rsp") \ + EM(smb_eio_trace_neg_bad_rsplen, "neg_bad_rsplen") \ + EM(smb_eio_trace_neg_decode_token, "neg_decode_token") \ + EM(smb_eio_trace_neg_info_caps, "neg_info_caps") \ + EM(smb_eio_trace_neg_info_dialect, "neg_info_dialect") \ + EM(smb_eio_trace_neg_info_fail, "neg_info_fail") \ + EM(smb_eio_trace_neg_info_sec_mode, "neg_info_sec_mode") \ + EM(smb_eio_trace_neg_inval_dialect, "neg_inval_dialect") \ + EM(smb_eio_trace_neg_no_crypt_key, "neg_no_crypt_key") \ + EM(smb_eio_trace_neg_sec_blob_too_small, "neg_sec_blob_too_small") \ + EM(smb_eio_trace_neg_unreq_dialect, "neg_unreq_dialect") \ + EM(smb_eio_trace_no_auth_key, "no_auth_key") \ + EM(smb_eio_trace_no_lease_key, "no_lease_key") \ + EM(smb_eio_trace_not_netfs_writeback, "not_netfs_writeback") \ + EM(smb_eio_trace_null_pointers, "null_pointers") \ + EM(smb_eio_trace_oldqfsinfo_bcc_too_small, "oldqfsinfo_bcc_too_small") \ + EM(smb_eio_trace_pend_del_fail, "pend_del_fail") \ + EM(smb_eio_trace_qalleas_bcc_too_small, "qalleas_bcc_too_small") \ + EM(smb_eio_trace_qalleas_ea_overlong, "qalleas_ea_overlong") \ + EM(smb_eio_trace_qalleas_overlong, "qalleas_overlong") \ + EM(smb_eio_trace_qfileinfo_bcc_too_small, "qfileinfo_bcc_too_small") \ + EM(smb_eio_trace_qfileinfo_invalid, "qfileinfo_invalid") \ + EM(smb_eio_trace_qfsattrinfo_bcc_too_small, "qfsattrinfo_bcc_too_small") \ + EM(smb_eio_trace_qfsdevinfo_bcc_too_small, "qfsdevinfo_bcc_too_small") \ + EM(smb_eio_trace_qfsinfo_bcc_too_small, "qfsinfo_bcc_too_small") \ + EM(smb_eio_trace_qfsposixinfo_bcc_too_small, "qfsposixinfo_bcc_too_small") \ + EM(smb_eio_trace_qfsunixinfo_bcc_too_small, "qfsunixinfo_bcc_too_small") \ + EM(smb_eio_trace_qpathinfo_bcc_too_small, "qpathinfo_bcc_too_small") \ + EM(smb_eio_trace_qpathinfo_invalid, "qpathinfo_invalid") \ + EM(smb_eio_trace_qreparse_data_area, "qreparse_data_area") \ + EM(smb_eio_trace_qreparse_rep_datalen, "qreparse_rep_datalen") \ + EM(smb_eio_trace_qreparse_ret_datalen, "qreparse_ret_datalen") \ + EM(smb_eio_trace_qreparse_setup_count, "qreparse_setup_count") \ + EM(smb_eio_trace_qreparse_sizes_wrong, "qreparse_sizes_wrong") \ + EM(smb_eio_trace_qsym_bcc_too_small, "qsym_bcc_too_small") \ + EM(smb_eio_trace_read_mid_state_unknown, "read_mid_state_unknown") \ + EM(smb_eio_trace_read_overlarge, "read_overlarge") \ + EM(smb_eio_trace_read_rsp_malformed, "read_rsp_malformed") \ + EM(smb_eio_trace_read_rsp_short, "read_rsp_short") \ + EM(smb_eio_trace_read_too_far, "read_too_far") \ + EM(smb_eio_trace_reparse_data_len, "reparse_data_len") \ + EM(smb_eio_trace_reparse_native_len, "reparse_native_len") \ + EM(smb_eio_trace_reparse_native_nul, "reparse_native_nul") \ + EM(smb_eio_trace_reparse_native_sym_len, "reparse_native_sym_len") \ + EM(smb_eio_trace_reparse_nfs_dev, "reparse_nfs_dev") \ + EM(smb_eio_trace_reparse_nfs_nul, "reparse_nfs_nul") \ + EM(smb_eio_trace_reparse_nfs_sockfifo, "reparse_nfs_sockfifo") \ + EM(smb_eio_trace_reparse_nfs_symbuf, "reparse_nfs_symbuf") \ + EM(smb_eio_trace_reparse_nfs_too_short, "reparse_nfs_too_short") \ + EM(smb_eio_trace_reparse_overlong, "reparse_overlong") \ + EM(smb_eio_trace_reparse_rdlen, "reparse_rdlen") \ + EM(smb_eio_trace_reparse_wsl_nul, "reparse_wsl_nul") \ + EM(smb_eio_trace_reparse_wsl_symbuf, "reparse_wsl_symbuf") \ + EM(smb_eio_trace_reparse_wsl_ver, "reparse_wsl_ver") \ + EM(smb_eio_trace_rx_b_read_short, "rx_b_read_short") \ + EM(smb_eio_trace_rx_bad_datalen, "rx_bad_datalen") \ + EM(smb_eio_trace_rx_both_buf, "rx_both_buf") \ + EM(smb_eio_trace_rx_calc_len_too_big, "rx_calc_len_too_big") \ + EM(smb_eio_trace_rx_check_rsp, "rx_check_rsp") \ + EM(smb_eio_trace_rx_copy_to_iter, "rx_copy_to_iter") \ + EM(smb_eio_trace_rx_insuff_res, "rx_insuff_res") \ + EM(smb_eio_trace_rx_inv_bcc, "rx_inv_bcc") \ + EM(smb_eio_trace_rx_mid_unready, "rx_mid_unready") \ + EM(smb_eio_trace_rx_neg_sess_resp, "rx_neg_sess_resp") \ + EM(smb_eio_trace_rx_overlong, "rx_overlong") \ + EM(smb_eio_trace_rx_overpage, "rx_overpage") \ + EM(smb_eio_trace_rx_pos_sess_resp, "rx_pos_sess_resp") \ + EM(smb_eio_trace_rx_rfc1002_magic, "rx_rfc1002_magic") \ + EM(smb_eio_trace_rx_sync_mid_invalid, "rx_sync_mid_invalid") \ + EM(smb_eio_trace_rx_sync_mid_malformed, "rx_sync_mid_malformed") \ + EM(smb_eio_trace_rx_too_short, "rx_too_short") \ + EM(smb_eio_trace_rx_trans2_extract, "rx_trans2_extract") \ + EM(smb_eio_trace_rx_unknown_resp, "rx_unknown_resp") \ + EM(smb_eio_trace_rx_unspec_error, "rx_unspec_error") \ + EM(smb_eio_trace_sess_buf_off, "sess_buf_off") \ + EM(smb_eio_trace_sess_exiting, "sess_exiting") \ + EM(smb_eio_trace_sess_krb_wcc, "sess_krb_wcc") \ + EM(smb_eio_trace_sess_nl2_wcc, "sess_nl2_wcc") \ + EM(smb_eio_trace_sess_rawnl_auth_wcc, "sess_rawnl_auth_wcc") \ + EM(smb_eio_trace_sess_rawnl_neg_wcc, "sess_rawnl_neg_wcc") \ + EM(smb_eio_trace_short_symlink_write, "short_symlink_write") \ + EM(smb_eio_trace_sid_too_many_auth, "sid_too_many_auth") \ + EM(smb_eio_trace_sig_data_too_small, "sig_data_too_small") \ + EM(smb_eio_trace_sig_iter, "sig_iter") \ + EM(smb_eio_trace_smb1_received_error, "smb1_received_error") \ + EM(smb_eio_trace_smb2_received_error, "smb2_received_error") \ + EM(smb_eio_trace_sym_slash, "sym_slash") \ + EM(smb_eio_trace_sym_target_len, "sym_target_len") \ + EM(smb_eio_trace_symlink_file_size, "symlink_file_size") \ + EM(smb_eio_trace_tdis_in_reconnect, "tdis_in_reconnect") \ + EM(smb_eio_trace_tx_chained_async, "tx_chained_async") \ + EM(smb_eio_trace_tx_compress_failed, "tx_compress_failed") \ + EM(smb_eio_trace_tx_copy_iter_to_buf, "tx_copy_iter_to_buf") \ + EM(smb_eio_trace_tx_copy_to_buf, "tx_copy_to_buf") \ + EM(smb_eio_trace_tx_max_compound, "tx_max_compound") \ + EM(smb_eio_trace_tx_miscopy_to_buf, "tx_miscopy_to_buf") \ + EM(smb_eio_trace_tx_need_transform, "tx_need_transform") \ + EM(smb_eio_trace_tx_too_long, "sr_too_long") \ + EM(smb_eio_trace_unixqfileinfo_bcc_too_small, "unixqfileinfo_bcc_too_small") \ + EM(smb_eio_trace_unixqpathinfo_bcc_too_small, "unixqpathinfo_bcc_too_small") \ + EM(smb_eio_trace_user_iter, "user_iter") \ + EM(smb_eio_trace_write_bad_buf_type, "write_bad_buf_type") \ + EM(smb_eio_trace_write_mid_state_unknown, "write_mid_state_unknown") \ + EM(smb_eio_trace_write_rsp_malformed, "write_rsp_malformed") \ + E_(smb_eio_trace_write_too_far, "write_too_far") + +#define smb3_rw_credits_traces \ + EM(cifs_trace_rw_credits_call_readv_adjust, "rd-call-adj") \ + EM(cifs_trace_rw_credits_call_writev_adjust, "wr-call-adj") \ + EM(cifs_trace_rw_credits_free_subreq, "free-subreq") \ + EM(cifs_trace_rw_credits_issue_read_adjust, "rd-issu-adj") \ + EM(cifs_trace_rw_credits_issue_write_adjust, "wr-issu-adj") \ + EM(cifs_trace_rw_credits_no_adjust_up, "no-adj-up ") \ + EM(cifs_trace_rw_credits_old_session, "old-session") \ + EM(cifs_trace_rw_credits_read_response_add, "rd-resp-add") \ + EM(cifs_trace_rw_credits_read_response_clear, "rd-resp-clr") \ + EM(cifs_trace_rw_credits_read_resubmit, "rd-resubmit") \ + EM(cifs_trace_rw_credits_read_submit, "rd-submit ") \ + EM(cifs_trace_rw_credits_write_prepare, "wr-prepare ") \ + EM(cifs_trace_rw_credits_write_response_add, "wr-resp-add") \ + EM(cifs_trace_rw_credits_write_response_clear, "wr-resp-clr") \ + E_(cifs_trace_rw_credits_zero_in_flight, "ZERO-IN-FLT") + +#define smb3_tcon_ref_traces \ + EM(netfs_trace_tcon_ref_free, "FRE ") \ + EM(netfs_trace_tcon_ref_free_fail, "FRE Fail ") \ + EM(netfs_trace_tcon_ref_free_ipc, "FRE Ipc ") \ + EM(netfs_trace_tcon_ref_free_ipc_fail, "FRE Ipc-F ") \ + EM(netfs_trace_tcon_ref_free_reconnect_server, "FRE Reconn") \ + EM(netfs_trace_tcon_ref_get_cached_laundromat, "GET Ch-Lau") \ + EM(netfs_trace_tcon_ref_get_cached_lease_break, "GET Ch-Lea") \ + EM(netfs_trace_tcon_ref_get_cancelled_close, "GET Cn-Cls") \ + EM(netfs_trace_tcon_ref_get_close_defer_files, "GET Cl-Def") \ + EM(netfs_trace_tcon_ref_get_dfs_refer, "GET DfsRef") \ + EM(netfs_trace_tcon_ref_get_find, "GET Find ") \ + EM(netfs_trace_tcon_ref_get_find_sess_tcon, "GET FndSes") \ + EM(netfs_trace_tcon_ref_get_reconnect_server, "GET Reconn") \ + EM(netfs_trace_tcon_ref_get_swn_notify, "GET SwnNot") \ + EM(netfs_trace_tcon_ref_new, "NEW ") \ + EM(netfs_trace_tcon_ref_new_ipc, "NEW Ipc ") \ + EM(netfs_trace_tcon_ref_new_reconnect_server, "NEW Reconn") \ + EM(netfs_trace_tcon_ref_put_cached_close, "PUT Ch-Cls") \ + EM(netfs_trace_tcon_ref_put_cancelled_close, "PUT Cn-Cls") \ + EM(netfs_trace_tcon_ref_put_cancelled_close_fid, "PUT Cn-Fid") \ + EM(netfs_trace_tcon_ref_put_cancelled_mid, "PUT Cn-Mid") \ + EM(netfs_trace_tcon_ref_put_close_defer_files, "PUT Cl-Def") \ + EM(netfs_trace_tcon_ref_put_mnt_ctx, "PUT MntCtx") \ + EM(netfs_trace_tcon_ref_put_dfs_refer, "PUT DfsRfr") \ + EM(netfs_trace_tcon_ref_put_reconnect_server, "PUT Reconn") \ + EM(netfs_trace_tcon_ref_put_swn_notify, "PUT SwnNot") \ + EM(netfs_trace_tcon_ref_put_tlink, "PUT Tlink ") \ + EM(netfs_trace_tcon_ref_see_cancelled_close, "SEE Cn-Cls") \ + EM(netfs_trace_tcon_ref_see_fscache_collision, "SEE FV-CO!") \ + EM(netfs_trace_tcon_ref_see_fscache_okay, "SEE FV-Ok ") \ + EM(netfs_trace_tcon_ref_see_fscache_relinq, "SEE FV-Rlq") \ + E_(netfs_trace_tcon_ref_see_umount, "SEE Umount") + +#undef EM +#undef E_ + +/* + * Define those tracing enums. + */ +#ifndef __SMB3_DECLARE_TRACE_ENUMS_ONCE_ONLY +#define __SMB3_DECLARE_TRACE_ENUMS_ONCE_ONLY + +#define EM(a, b) a, +#define E_(a, b) a + +enum smb_eio_trace { smb_eio_traces } __mode(byte); +enum smb3_rw_credits_trace { smb3_rw_credits_traces } __mode(byte); +enum smb3_tcon_ref_trace { smb3_tcon_ref_traces } __mode(byte); + +#undef EM +#undef E_ +#endif + +/* + * Export enum symbols via userspace. + */ +#define EM(a, b) TRACE_DEFINE_ENUM(a); +#define E_(a, b) TRACE_DEFINE_ENUM(a); + +smb_eio_traces; +smb3_rw_credits_traces; +smb3_tcon_ref_traces; + +#undef EM +#undef E_ + +/* + * Now redefine the EM() and E_() macros to map the enums to the strings that + * will be printed in the output. + */ +#define EM(a, b) { a, b }, +#define E_(a, b) { a, b } + +/* For logging errors in read or write */ +DECLARE_EVENT_CLASS(smb3_rw_err_class, + TP_PROTO(unsigned int rreq_debug_id, + unsigned int rreq_debug_index, + unsigned int xid, + __u64 fid, + __u32 tid, + __u64 sesid, + __u64 offset, + __u32 len, + int rc), + TP_ARGS(rreq_debug_id, rreq_debug_index, + xid, fid, tid, sesid, offset, len, rc), + TP_STRUCT__entry( + __field(unsigned int, rreq_debug_id) + __field(unsigned int, rreq_debug_index) + __field(unsigned int, xid) + __field(__u64, fid) + __field(__u32, tid) + __field(__u64, sesid) + __field(__u64, offset) + __field(__u32, len) + __field(int, rc) + ), + TP_fast_assign( + __entry->rreq_debug_id = rreq_debug_id; + __entry->rreq_debug_index = rreq_debug_index; + __entry->xid = xid; + __entry->fid = fid; + __entry->tid = tid; + __entry->sesid = sesid; + __entry->offset = offset; + __entry->len = len; + __entry->rc = rc; + ), + TP_printk("R=%08x[%x] xid=%u sid=0x%llx tid=0x%x fid=0x%llx offset=0x%llx len=0x%x rc=%d", + __entry->rreq_debug_id, __entry->rreq_debug_index, + __entry->xid, __entry->sesid, __entry->tid, __entry->fid, + __entry->offset, __entry->len, __entry->rc) +) + +#define DEFINE_SMB3_RW_ERR_EVENT(name) \ +DEFINE_EVENT(smb3_rw_err_class, smb3_##name, \ + TP_PROTO(unsigned int rreq_debug_id, \ + unsigned int rreq_debug_index, \ + unsigned int xid, \ + __u64 fid, \ + __u32 tid, \ + __u64 sesid, \ + __u64 offset, \ + __u32 len, \ + int rc), \ + TP_ARGS(rreq_debug_id, rreq_debug_index, xid, fid, tid, sesid, offset, len, rc)) + +DEFINE_SMB3_RW_ERR_EVENT(read_err); +DEFINE_SMB3_RW_ERR_EVENT(write_err); + +/* For logging errors in other file I/O ops */ +DECLARE_EVENT_CLASS(smb3_other_err_class, + TP_PROTO(unsigned int xid, + __u64 fid, + __u32 tid, + __u64 sesid, + __u64 offset, + __u32 len, + int rc), + TP_ARGS(xid, fid, tid, sesid, offset, len, rc), + TP_STRUCT__entry( + __field(unsigned int, xid) + __field(__u64, fid) + __field(__u32, tid) + __field(__u64, sesid) + __field(__u64, offset) + __field(__u32, len) + __field(int, rc) + ), + TP_fast_assign( + __entry->xid = xid; + __entry->fid = fid; + __entry->tid = tid; + __entry->sesid = sesid; + __entry->offset = offset; + __entry->len = len; + __entry->rc = rc; + ), + TP_printk("xid=%u sid=0x%llx tid=0x%x fid=0x%llx offset=0x%llx len=0x%x rc=%d", + __entry->xid, __entry->sesid, __entry->tid, __entry->fid, + __entry->offset, __entry->len, __entry->rc) +) + +#define DEFINE_SMB3_OTHER_ERR_EVENT(name) \ +DEFINE_EVENT(smb3_other_err_class, smb3_##name, \ + TP_PROTO(unsigned int xid, \ + __u64 fid, \ + __u32 tid, \ + __u64 sesid, \ + __u64 offset, \ + __u32 len, \ + int rc), \ + TP_ARGS(xid, fid, tid, sesid, offset, len, rc)) + +DEFINE_SMB3_OTHER_ERR_EVENT(query_dir_err); +DEFINE_SMB3_OTHER_ERR_EVENT(zero_err); +DEFINE_SMB3_OTHER_ERR_EVENT(falloc_err); + +/* + * For logging errors in reflink and copy_range ops e.g. smb2_copychunk_range + * and smb2_duplicate_extents + */ +DECLARE_EVENT_CLASS(smb3_copy_range_err_class, + TP_PROTO(unsigned int xid, + __u64 src_fid, + __u64 target_fid, + __u32 tid, + __u64 sesid, + __u64 src_offset, + __u64 target_offset, + __u32 len, + int rc), + TP_ARGS(xid, src_fid, target_fid, tid, sesid, src_offset, target_offset, len, rc), + TP_STRUCT__entry( + __field(unsigned int, xid) + __field(__u64, src_fid) + __field(__u64, target_fid) + __field(__u32, tid) + __field(__u64, sesid) + __field(__u64, src_offset) + __field(__u64, target_offset) + __field(__u32, len) + __field(int, rc) + ), + TP_fast_assign( + __entry->xid = xid; + __entry->src_fid = src_fid; + __entry->target_fid = target_fid; + __entry->tid = tid; + __entry->sesid = sesid; + __entry->src_offset = src_offset; + __entry->target_offset = target_offset; + __entry->len = len; + __entry->rc = rc; + ), + TP_printk("xid=%u sid=0x%llx tid=0x%x source fid=0x%llx source offset=0x%llx target fid=0x%llx target offset=0x%llx len=0x%x rc=%d", + __entry->xid, __entry->sesid, __entry->tid, __entry->target_fid, + __entry->src_offset, __entry->target_fid, __entry->target_offset, __entry->len, __entry->rc) +) + +#define DEFINE_SMB3_COPY_RANGE_ERR_EVENT(name) \ +DEFINE_EVENT(smb3_copy_range_err_class, smb3_##name, \ + TP_PROTO(unsigned int xid, \ + __u64 src_fid, \ + __u64 target_fid, \ + __u32 tid, \ + __u64 sesid, \ + __u64 src_offset, \ + __u64 target_offset, \ + __u32 len, \ + int rc), \ + TP_ARGS(xid, src_fid, target_fid, tid, sesid, src_offset, target_offset, len, rc)) + +DEFINE_SMB3_COPY_RANGE_ERR_EVENT(clone_err); +DEFINE_SMB3_COPY_RANGE_ERR_EVENT(copychunk_err); + +DECLARE_EVENT_CLASS(smb3_copy_range_done_class, + TP_PROTO(unsigned int xid, + __u64 src_fid, + __u64 target_fid, + __u32 tid, + __u64 sesid, + __u64 src_offset, + __u64 target_offset, + __u32 len), + TP_ARGS(xid, src_fid, target_fid, tid, sesid, src_offset, target_offset, len), + TP_STRUCT__entry( + __field(unsigned int, xid) + __field(__u64, src_fid) + __field(__u64, target_fid) + __field(__u32, tid) + __field(__u64, sesid) + __field(__u64, src_offset) + __field(__u64, target_offset) + __field(__u32, len) + ), + TP_fast_assign( + __entry->xid = xid; + __entry->src_fid = src_fid; + __entry->target_fid = target_fid; + __entry->tid = tid; + __entry->sesid = sesid; + __entry->src_offset = src_offset; + __entry->target_offset = target_offset; + __entry->len = len; + ), + TP_printk("xid=%u sid=0x%llx tid=0x%x source fid=0x%llx source offset=0x%llx target fid=0x%llx target offset=0x%llx len=0x%x", + __entry->xid, __entry->sesid, __entry->tid, __entry->target_fid, + __entry->src_offset, __entry->target_fid, __entry->target_offset, __entry->len) +) + +#define DEFINE_SMB3_COPY_RANGE_DONE_EVENT(name) \ +DEFINE_EVENT(smb3_copy_range_done_class, smb3_##name, \ + TP_PROTO(unsigned int xid, \ + __u64 src_fid, \ + __u64 target_fid, \ + __u32 tid, \ + __u64 sesid, \ + __u64 src_offset, \ + __u64 target_offset, \ + __u32 len), \ + TP_ARGS(xid, src_fid, target_fid, tid, sesid, src_offset, target_offset, len)) + +DEFINE_SMB3_COPY_RANGE_DONE_EVENT(copychunk_enter); +DEFINE_SMB3_COPY_RANGE_DONE_EVENT(clone_enter); +DEFINE_SMB3_COPY_RANGE_DONE_EVENT(copychunk_done); +DEFINE_SMB3_COPY_RANGE_DONE_EVENT(clone_done); + + +/* For logging successful read or write */ +DECLARE_EVENT_CLASS(smb3_rw_done_class, + TP_PROTO(unsigned int rreq_debug_id, + unsigned int rreq_debug_index, + unsigned int xid, + __u64 fid, + __u32 tid, + __u64 sesid, + __u64 offset, + __u32 len), + TP_ARGS(rreq_debug_id, rreq_debug_index, + xid, fid, tid, sesid, offset, len), + TP_STRUCT__entry( + __field(unsigned int, rreq_debug_id) + __field(unsigned int, rreq_debug_index) + __field(unsigned int, xid) + __field(__u64, fid) + __field(__u32, tid) + __field(__u64, sesid) + __field(__u64, offset) + __field(__u32, len) + ), + TP_fast_assign( + __entry->rreq_debug_id = rreq_debug_id; + __entry->rreq_debug_index = rreq_debug_index; + __entry->xid = xid; + __entry->fid = fid; + __entry->tid = tid; + __entry->sesid = sesid; + __entry->offset = offset; + __entry->len = len; + ), + TP_printk("R=%08x[%x] xid=%u sid=0x%llx tid=0x%x fid=0x%llx offset=0x%llx len=0x%x", + __entry->rreq_debug_id, __entry->rreq_debug_index, + __entry->xid, __entry->sesid, __entry->tid, __entry->fid, + __entry->offset, __entry->len) +) + +#define DEFINE_SMB3_RW_DONE_EVENT(name) \ +DEFINE_EVENT(smb3_rw_done_class, smb3_##name, \ + TP_PROTO(unsigned int rreq_debug_id, \ + unsigned int rreq_debug_index, \ + unsigned int xid, \ + __u64 fid, \ + __u32 tid, \ + __u64 sesid, \ + __u64 offset, \ + __u32 len), \ + TP_ARGS(rreq_debug_id, rreq_debug_index, xid, fid, tid, sesid, offset, len)) + +DEFINE_SMB3_RW_DONE_EVENT(read_enter); +DEFINE_SMB3_RW_DONE_EVENT(read_done); +DEFINE_SMB3_RW_DONE_EVENT(write_enter); +DEFINE_SMB3_RW_DONE_EVENT(write_done); + +/* For logging successful other op */ +DECLARE_EVENT_CLASS(smb3_other_done_class, + TP_PROTO(unsigned int xid, + __u64 fid, + __u32 tid, + __u64 sesid, + __u64 offset, + __u32 len), + TP_ARGS(xid, fid, tid, sesid, offset, len), + TP_STRUCT__entry( + __field(unsigned int, xid) + __field(__u64, fid) + __field(__u32, tid) + __field(__u64, sesid) + __field(__u64, offset) + __field(__u32, len) + ), + TP_fast_assign( + __entry->xid = xid; + __entry->fid = fid; + __entry->tid = tid; + __entry->sesid = sesid; + __entry->offset = offset; + __entry->len = len; + ), + TP_printk("xid=%u sid=0x%llx tid=0x%x fid=0x%llx offset=0x%llx len=0x%x", + __entry->xid, __entry->sesid, __entry->tid, __entry->fid, + __entry->offset, __entry->len) +) + +#define DEFINE_SMB3_OTHER_DONE_EVENT(name) \ +DEFINE_EVENT(smb3_other_done_class, smb3_##name, \ + TP_PROTO(unsigned int xid, \ + __u64 fid, \ + __u32 tid, \ + __u64 sesid, \ + __u64 offset, \ + __u32 len), \ + TP_ARGS(xid, fid, tid, sesid, offset, len)) + +DEFINE_SMB3_OTHER_DONE_EVENT(query_dir_enter); +DEFINE_SMB3_OTHER_DONE_EVENT(zero_enter); +DEFINE_SMB3_OTHER_DONE_EVENT(falloc_enter); +DEFINE_SMB3_OTHER_DONE_EVENT(query_dir_done); +DEFINE_SMB3_OTHER_DONE_EVENT(zero_done); +DEFINE_SMB3_OTHER_DONE_EVENT(falloc_done); + +/* For logging successful set EOF (truncate) */ +DECLARE_EVENT_CLASS(smb3_eof_class, + TP_PROTO(unsigned int xid, + __u64 fid, + __u32 tid, + __u64 sesid, + __u64 offset), + TP_ARGS(xid, fid, tid, sesid, offset), + TP_STRUCT__entry( + __field(unsigned int, xid) + __field(__u64, fid) + __field(__u32, tid) + __field(__u64, sesid) + __field(__u64, offset) + ), + TP_fast_assign( + __entry->xid = xid; + __entry->fid = fid; + __entry->tid = tid; + __entry->sesid = sesid; + __entry->offset = offset; + ), + TP_printk("xid=%u sid=0x%llx tid=0x%x fid=0x%llx offset=0x%llx", + __entry->xid, __entry->sesid, __entry->tid, __entry->fid, + __entry->offset) +) + +#define DEFINE_SMB3_EOF_EVENT(name) \ +DEFINE_EVENT(smb3_eof_class, smb3_##name, \ + TP_PROTO(unsigned int xid, \ + __u64 fid, \ + __u32 tid, \ + __u64 sesid, \ + __u64 offset), \ + TP_ARGS(xid, fid, tid, sesid, offset)) + +DEFINE_SMB3_EOF_EVENT(set_eof); + +/* + * For handle based calls other than read and write, and get/set info + */ +DECLARE_EVENT_CLASS(smb3_fd_class, + TP_PROTO(unsigned int xid, + __u64 fid, + __u32 tid, + __u64 sesid), + TP_ARGS(xid, fid, tid, sesid), + TP_STRUCT__entry( + __field(unsigned int, xid) + __field(__u64, fid) + __field(__u32, tid) + __field(__u64, sesid) + ), + TP_fast_assign( + __entry->xid = xid; + __entry->fid = fid; + __entry->tid = tid; + __entry->sesid = sesid; + ), + TP_printk("xid=%u sid=0x%llx tid=0x%x fid=0x%llx", + __entry->xid, __entry->sesid, __entry->tid, __entry->fid) +) + +#define DEFINE_SMB3_FD_EVENT(name) \ +DEFINE_EVENT(smb3_fd_class, smb3_##name, \ + TP_PROTO(unsigned int xid, \ + __u64 fid, \ + __u32 tid, \ + __u64 sesid), \ + TP_ARGS(xid, fid, tid, sesid)) + +DEFINE_SMB3_FD_EVENT(flush_enter); +DEFINE_SMB3_FD_EVENT(flush_done); +DEFINE_SMB3_FD_EVENT(close_enter); +DEFINE_SMB3_FD_EVENT(close_done); +DEFINE_SMB3_FD_EVENT(oplock_not_found); + +DECLARE_EVENT_CLASS(smb3_fd_err_class, + TP_PROTO(unsigned int xid, + __u64 fid, + __u32 tid, + __u64 sesid, + int rc), + TP_ARGS(xid, fid, tid, sesid, rc), + TP_STRUCT__entry( + __field(unsigned int, xid) + __field(__u64, fid) + __field(__u32, tid) + __field(__u64, sesid) + __field(int, rc) + ), + TP_fast_assign( + __entry->xid = xid; + __entry->fid = fid; + __entry->tid = tid; + __entry->sesid = sesid; + __entry->rc = rc; + ), + TP_printk("xid=%u sid=0x%llx tid=0x%x fid=0x%llx rc=%d", + __entry->xid, __entry->sesid, __entry->tid, __entry->fid, + __entry->rc) +) + +#define DEFINE_SMB3_FD_ERR_EVENT(name) \ +DEFINE_EVENT(smb3_fd_err_class, smb3_##name, \ + TP_PROTO(unsigned int xid, \ + __u64 fid, \ + __u32 tid, \ + __u64 sesid, \ + int rc), \ + TP_ARGS(xid, fid, tid, sesid, rc)) + +DEFINE_SMB3_FD_ERR_EVENT(flush_err); +DEFINE_SMB3_FD_ERR_EVENT(close_err); + +DECLARE_EVENT_CLASS(smb3_lock_class, + TP_PROTO(unsigned int xid, + __u64 fid, + __u32 tid, + __u64 sesid, + __u64 offset, + __u64 len, + __u32 flags, + __u32 num_lock, + int rc), + TP_ARGS(xid, fid, tid, sesid, offset, len, flags, num_lock, rc), + TP_STRUCT__entry( + __field(unsigned int, xid) + __field(__u64, fid) + __field(__u32, tid) + __field(__u64, sesid) + __field(__u64, offset) + __field(__u64, len) + __field(__u32, flags) + __field(__u32, num_lock) + __field(int, rc) + ), + TP_fast_assign( + __entry->xid = xid; + __entry->fid = fid; + __entry->tid = tid; + __entry->sesid = sesid; + __entry->offset = offset; + __entry->len = len; + __entry->flags = flags; + __entry->num_lock = num_lock; + __entry->rc = rc; + ), + TP_printk("xid=%u sid=0x%llx tid=0x%x fid=0x%llx offset=0x%llx len=0x%llx flags=0x%x num_lock=%u rc=%d", + __entry->xid, __entry->sesid, __entry->tid, __entry->fid, + __entry->offset, __entry->len, __entry->flags, __entry->num_lock, + __entry->rc) +) + +#define DEFINE_SMB3_LOCK_EVENT(name) \ +DEFINE_EVENT(smb3_lock_class, smb3_##name, \ + TP_PROTO(unsigned int xid, \ + __u64 fid, \ + __u32 tid, \ + __u64 sesid, \ + __u64 offset, \ + __u64 len, \ + __u32 flags, \ + __u32 num_lock, \ + int rc), \ + TP_ARGS(xid, fid, tid, sesid, offset, len, flags, num_lock, rc)) + +DEFINE_SMB3_LOCK_EVENT(lock_enter); +DEFINE_SMB3_LOCK_EVENT(lock_done); +DEFINE_SMB3_LOCK_EVENT(lock_err); +DEFINE_SMB3_LOCK_EVENT(lock_cached); + +TRACE_EVENT(smb3_lock_conflict, + TP_PROTO(__u64 fid, + __u64 req_offset, + __u64 req_len, + __u8 req_type, + __u64 conf_offset, + __u64 conf_len, + __u16 conf_type, + __u32 conf_pid), + TP_ARGS(fid, req_offset, req_len, req_type, conf_offset, conf_len, conf_type, conf_pid), + TP_STRUCT__entry( + __field(__u64, fid) + __field(__u64, req_offset) + __field(__u64, req_len) + __field(__u8, req_type) + __field(__u64, conf_offset) + __field(__u64, conf_len) + __field(__u16, conf_type) + __field(__u32, conf_pid) + ), + TP_fast_assign( + __entry->fid = fid; + __entry->req_offset = req_offset; + __entry->req_len = req_len; + __entry->req_type = req_type; + __entry->conf_offset = conf_offset; + __entry->conf_len = conf_len; + __entry->conf_type = conf_type; + __entry->conf_pid = conf_pid; + ), + TP_printk("fid=0x%llx req=[0x%llx:0x%llx] type=0x%x conflicts with [0x%llx:0x%llx] type=0x%x pid=%u", + __entry->fid, __entry->req_offset, __entry->req_len, __entry->req_type, + __entry->conf_offset, __entry->conf_len, __entry->conf_type, __entry->conf_pid) +); + +/* + * For handle based query/set info calls + */ +DECLARE_EVENT_CLASS(smb3_inf_enter_class, + TP_PROTO(unsigned int xid, + __u64 fid, + __u32 tid, + __u64 sesid, + __u8 infclass, + __u32 type), + TP_ARGS(xid, fid, tid, sesid, infclass, type), + TP_STRUCT__entry( + __field(unsigned int, xid) + __field(__u64, fid) + __field(__u32, tid) + __field(__u64, sesid) + __field(__u8, infclass) + __field(__u32, type) + ), + TP_fast_assign( + __entry->xid = xid; + __entry->fid = fid; + __entry->tid = tid; + __entry->sesid = sesid; + __entry->infclass = infclass; + __entry->type = type; + ), + TP_printk("xid=%u sid=0x%llx tid=0x%x fid=0x%llx class=%u type=0x%x", + __entry->xid, __entry->sesid, __entry->tid, __entry->fid, + __entry->infclass, __entry->type) +) + +#define DEFINE_SMB3_INF_ENTER_EVENT(name) \ +DEFINE_EVENT(smb3_inf_enter_class, smb3_##name, \ + TP_PROTO(unsigned int xid, \ + __u64 fid, \ + __u32 tid, \ + __u64 sesid, \ + __u8 infclass, \ + __u32 type), \ + TP_ARGS(xid, fid, tid, sesid, infclass, type)) + +DEFINE_SMB3_INF_ENTER_EVENT(query_info_enter); +DEFINE_SMB3_INF_ENTER_EVENT(query_info_done); +DEFINE_SMB3_INF_ENTER_EVENT(notify_enter); +DEFINE_SMB3_INF_ENTER_EVENT(notify_done); + +DECLARE_EVENT_CLASS(smb3_inf_err_class, + TP_PROTO(unsigned int xid, + __u64 fid, + __u32 tid, + __u64 sesid, + __u8 infclass, + __u32 type, + int rc), + TP_ARGS(xid, fid, tid, sesid, infclass, type, rc), + TP_STRUCT__entry( + __field(unsigned int, xid) + __field(__u64, fid) + __field(__u32, tid) + __field(__u64, sesid) + __field(__u8, infclass) + __field(__u32, type) + __field(int, rc) + ), + TP_fast_assign( + __entry->xid = xid; + __entry->fid = fid; + __entry->tid = tid; + __entry->sesid = sesid; + __entry->infclass = infclass; + __entry->type = type; + __entry->rc = rc; + ), + TP_printk("xid=%u sid=0x%llx tid=0x%x fid=0x%llx class=%u type=0x%x rc=%d", + __entry->xid, __entry->sesid, __entry->tid, __entry->fid, + __entry->infclass, __entry->type, __entry->rc) +) + +#define DEFINE_SMB3_INF_ERR_EVENT(name) \ +DEFINE_EVENT(smb3_inf_err_class, smb3_##name, \ + TP_PROTO(unsigned int xid, \ + __u64 fid, \ + __u32 tid, \ + __u64 sesid, \ + __u8 infclass, \ + __u32 type, \ + int rc), \ + TP_ARGS(xid, fid, tid, sesid, infclass, type, rc)) + +DEFINE_SMB3_INF_ERR_EVENT(query_info_err); +DEFINE_SMB3_INF_ERR_EVENT(set_info_err); +DEFINE_SMB3_INF_ERR_EVENT(notify_err); +DEFINE_SMB3_INF_ERR_EVENT(fsctl_err); + +DECLARE_EVENT_CLASS(smb3_inf_compound_enter_class, + TP_PROTO(unsigned int xid, + __u32 tid, + __u64 sesid, + const char *full_path), + TP_ARGS(xid, tid, sesid, full_path), + TP_STRUCT__entry( + __field(unsigned int, xid) + __field(__u32, tid) + __field(__u64, sesid) + __string(path, full_path) + ), + TP_fast_assign( + __entry->xid = xid; + __entry->tid = tid; + __entry->sesid = sesid; + __assign_str(path); + ), + TP_printk("xid=%u sid=0x%llx tid=0x%x path=%s", + __entry->xid, __entry->sesid, __entry->tid, + __get_str(path)) +) + +#define DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(name) \ +DEFINE_EVENT(smb3_inf_compound_enter_class, smb3_##name, \ + TP_PROTO(unsigned int xid, \ + __u32 tid, \ + __u64 sesid, \ + const char *full_path), \ + TP_ARGS(xid, tid, sesid, full_path)) + +DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(query_info_compound_enter); +DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(posix_query_info_compound_enter); +DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(hardlink_enter); +DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(rename_enter); +DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(unlink_enter); +DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(set_eof_enter); +DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(set_info_compound_enter); +DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(set_reparse_compound_enter); +DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(get_reparse_compound_enter); +DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(query_wsl_ea_compound_enter); +DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(mkdir_enter); +DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(tdis_enter); +DEFINE_SMB3_INF_COMPOUND_ENTER_EVENT(mknod_enter); + +DECLARE_EVENT_CLASS(smb3_inf_compound_done_class, + TP_PROTO(unsigned int xid, + __u32 tid, + __u64 sesid), + TP_ARGS(xid, tid, sesid), + TP_STRUCT__entry( + __field(unsigned int, xid) + __field(__u32, tid) + __field(__u64, sesid) + ), + TP_fast_assign( + __entry->xid = xid; + __entry->tid = tid; + __entry->sesid = sesid; + ), + TP_printk("xid=%u sid=0x%llx tid=0x%x", + __entry->xid, __entry->sesid, __entry->tid) +) + +#define DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(name) \ +DEFINE_EVENT(smb3_inf_compound_done_class, smb3_##name, \ + TP_PROTO(unsigned int xid, \ + __u32 tid, \ + __u64 sesid), \ + TP_ARGS(xid, tid, sesid)) + +DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(query_info_compound_done); +DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(posix_query_info_compound_done); +DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(hardlink_done); +DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(rename_done); +DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(unlink_done); +DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(set_eof_done); +DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(set_info_compound_done); +DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(set_reparse_compound_done); +DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(get_reparse_compound_done); +DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(query_wsl_ea_compound_done); +DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(mkdir_done); +DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(tdis_done); +DEFINE_SMB3_INF_COMPOUND_DONE_EVENT(mknod_done); + +DECLARE_EVENT_CLASS(smb3_inf_compound_err_class, + TP_PROTO(unsigned int xid, + __u32 tid, + __u64 sesid, + int rc), + TP_ARGS(xid, tid, sesid, rc), + TP_STRUCT__entry( + __field(unsigned int, xid) + __field(__u32, tid) + __field(__u64, sesid) + __field(int, rc) + ), + TP_fast_assign( + __entry->xid = xid; + __entry->tid = tid; + __entry->sesid = sesid; + __entry->rc = rc; + ), + TP_printk("xid=%u sid=0x%llx tid=0x%x rc=%d", + __entry->xid, __entry->sesid, __entry->tid, + __entry->rc) +) + +#define DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(name) \ +DEFINE_EVENT(smb3_inf_compound_err_class, smb3_##name, \ + TP_PROTO(unsigned int xid, \ + __u32 tid, \ + __u64 sesid, \ + int rc), \ + TP_ARGS(xid, tid, sesid, rc)) + +DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(query_info_compound_err); +DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(posix_query_info_compound_err); +DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(hardlink_err); +DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(rename_err); +DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(unlink_err); +DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(set_eof_err); +DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(set_info_compound_err); +DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(set_reparse_compound_err); +DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(get_reparse_compound_err); +DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(query_wsl_ea_compound_err); +DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(mkdir_err); +DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(tdis_err); +DEFINE_SMB3_INF_COMPOUND_ERR_EVENT(mknod_err); + +/* + * For logging SMB3 Status code and Command for responses which return errors + */ +DECLARE_EVENT_CLASS(smb3_cmd_err_class, + TP_PROTO(__u32 tid, + __u64 sesid, + __u16 cmd, + __u64 mid, + __u32 status, + int rc), + TP_ARGS(tid, sesid, cmd, mid, status, rc), + TP_STRUCT__entry( + __field(__u32, tid) + __field(__u64, sesid) + __field(__u16, cmd) + __field(__u64, mid) + __field(__u32, status) + __field(int, rc) + ), + TP_fast_assign( + __entry->tid = tid; + __entry->sesid = sesid; + __entry->cmd = cmd; + __entry->mid = mid; + __entry->status = status; + __entry->rc = rc; + ), + TP_printk("sid=0x%llx tid=0x%x cmd=%u mid=%llu status=0x%x rc=%d", + __entry->sesid, __entry->tid, __entry->cmd, __entry->mid, + __entry->status, __entry->rc) +) + +#define DEFINE_SMB3_CMD_ERR_EVENT(name) \ +DEFINE_EVENT(smb3_cmd_err_class, smb3_##name, \ + TP_PROTO(__u32 tid, \ + __u64 sesid, \ + __u16 cmd, \ + __u64 mid, \ + __u32 status, \ + int rc), \ + TP_ARGS(tid, sesid, cmd, mid, status, rc)) + +DEFINE_SMB3_CMD_ERR_EVENT(cmd_err); + +DECLARE_EVENT_CLASS(smb3_cmd_done_class, + TP_PROTO(__u32 tid, + __u64 sesid, + __u16 cmd, + __u64 mid), + TP_ARGS(tid, sesid, cmd, mid), + TP_STRUCT__entry( + __field(__u32, tid) + __field(__u64, sesid) + __field(__u16, cmd) + __field(__u64, mid) + ), + TP_fast_assign( + __entry->tid = tid; + __entry->sesid = sesid; + __entry->cmd = cmd; + __entry->mid = mid; + ), + TP_printk("sid=0x%llx tid=0x%x cmd=%u mid=%llu", + __entry->sesid, __entry->tid, + __entry->cmd, __entry->mid) +) + +#define DEFINE_SMB3_CMD_DONE_EVENT(name) \ +DEFINE_EVENT(smb3_cmd_done_class, smb3_##name, \ + TP_PROTO(__u32 tid, \ + __u64 sesid, \ + __u16 cmd, \ + __u64 mid), \ + TP_ARGS(tid, sesid, cmd, mid)) + +DEFINE_SMB3_CMD_DONE_EVENT(cmd_enter); +DEFINE_SMB3_CMD_DONE_EVENT(cmd_done); +DEFINE_SMB3_CMD_DONE_EVENT(ses_expired); + +DECLARE_EVENT_CLASS(smb3_mid_class, + TP_PROTO(__u16 cmd, + __u64 mid, + __u32 pid, + unsigned long when_sent, + unsigned long when_received), + TP_ARGS(cmd, mid, pid, when_sent, when_received), + TP_STRUCT__entry( + __field(__u16, cmd) + __field(__u64, mid) + __field(__u32, pid) + __field(unsigned long, when_sent) + __field(unsigned long, when_received) + ), + TP_fast_assign( + __entry->cmd = cmd; + __entry->mid = mid; + __entry->pid = pid; + __entry->when_sent = when_sent; + __entry->when_received = when_received; + ), + TP_printk("cmd=%u mid=%llu pid=%u, when_sent=%lu when_rcv=%lu", + __entry->cmd, __entry->mid, __entry->pid, __entry->when_sent, + __entry->when_received) +) + +#define DEFINE_SMB3_MID_EVENT(name) \ +DEFINE_EVENT(smb3_mid_class, smb3_##name, \ + TP_PROTO(__u16 cmd, \ + __u64 mid, \ + __u32 pid, \ + unsigned long when_sent, \ + unsigned long when_received), \ + TP_ARGS(cmd, mid, pid, when_sent, when_received)) + +DEFINE_SMB3_MID_EVENT(slow_rsp); + +DECLARE_EVENT_CLASS(smb3_exit_err_class, + TP_PROTO(unsigned int xid, + const char *func_name, + int rc), + TP_ARGS(xid, func_name, rc), + TP_STRUCT__entry( + __field(unsigned int, xid) + __string(func_name, func_name) + __field(int, rc) + ), + TP_fast_assign( + __entry->xid = xid; + __assign_str(func_name); + __entry->rc = rc; + ), + TP_printk("%s: xid=%u rc=%d", + __get_str(func_name), __entry->xid, __entry->rc) +) + +#define DEFINE_SMB3_EXIT_ERR_EVENT(name) \ +DEFINE_EVENT(smb3_exit_err_class, smb3_##name, \ + TP_PROTO(unsigned int xid, \ + const char *func_name, \ + int rc), \ + TP_ARGS(xid, func_name, rc)) + +DEFINE_SMB3_EXIT_ERR_EVENT(exit_err); + + +DECLARE_EVENT_CLASS(smb3_sync_err_class, + TP_PROTO(unsigned long ino, + int rc), + TP_ARGS(ino, rc), + TP_STRUCT__entry( + __field(unsigned long, ino) + __field(int, rc) + ), + TP_fast_assign( + __entry->ino = ino; + __entry->rc = rc; + ), + TP_printk("ino=%lu rc=%d", + __entry->ino, __entry->rc) +) + +#define DEFINE_SMB3_SYNC_ERR_EVENT(name) \ +DEFINE_EVENT(smb3_sync_err_class, cifs_##name, \ + TP_PROTO(unsigned long ino, \ + int rc), \ + TP_ARGS(ino, rc)) + +DEFINE_SMB3_SYNC_ERR_EVENT(fsync_err); +DEFINE_SMB3_SYNC_ERR_EVENT(flush_err); + + +DECLARE_EVENT_CLASS(smb3_enter_exit_class, + TP_PROTO(unsigned int xid, + const char *func_name), + TP_ARGS(xid, func_name), + TP_STRUCT__entry( + __field(unsigned int, xid) + __string(func_name, func_name) + ), + TP_fast_assign( + __entry->xid = xid; + __assign_str(func_name); + ), + TP_printk("%s: xid=%u", + __get_str(func_name), __entry->xid) +) + +#define DEFINE_SMB3_ENTER_EXIT_EVENT(name) \ +DEFINE_EVENT(smb3_enter_exit_class, smb3_##name, \ + TP_PROTO(unsigned int xid, \ + const char *func_name), \ + TP_ARGS(xid, func_name)) + +DEFINE_SMB3_ENTER_EXIT_EVENT(enter); +DEFINE_SMB3_ENTER_EXIT_EVENT(exit_done); + +/* + * For SMB2/SMB3 tree connect + */ + +DECLARE_EVENT_CLASS(smb3_tcon_class, + TP_PROTO(unsigned int xid, + __u32 tid, + __u64 sesid, + const char *unc_name, + int rc), + TP_ARGS(xid, tid, sesid, unc_name, rc), + TP_STRUCT__entry( + __field(unsigned int, xid) + __field(__u32, tid) + __field(__u64, sesid) + __string(name, unc_name) + __field(int, rc) + ), + TP_fast_assign( + __entry->xid = xid; + __entry->tid = tid; + __entry->sesid = sesid; + __assign_str(name); + __entry->rc = rc; + ), + TP_printk("xid=%u sid=0x%llx tid=0x%x unc_name=%s rc=%d", + __entry->xid, __entry->sesid, __entry->tid, + __get_str(name), __entry->rc) +) + +#define DEFINE_SMB3_TCON_EVENT(name) \ +DEFINE_EVENT(smb3_tcon_class, smb3_##name, \ + TP_PROTO(unsigned int xid, \ + __u32 tid, \ + __u64 sesid, \ + const char *unc_name, \ + int rc), \ + TP_ARGS(xid, tid, sesid, unc_name, rc)) + +DEFINE_SMB3_TCON_EVENT(tcon); +DEFINE_SMB3_TCON_EVENT(qfs_done); + +/* + * For smb2/smb3 open (including create and mkdir) calls + */ + +DECLARE_EVENT_CLASS(smb3_open_enter_class, + TP_PROTO(unsigned int xid, + __u32 tid, + __u64 sesid, + const char *full_path, + int create_options, + int desired_access), + TP_ARGS(xid, tid, sesid, full_path, create_options, desired_access), + TP_STRUCT__entry( + __field(unsigned int, xid) + __field(__u32, tid) + __field(__u64, sesid) + __string(path, full_path) + __field(int, create_options) + __field(int, desired_access) + ), + TP_fast_assign( + __entry->xid = xid; + __entry->tid = tid; + __entry->sesid = sesid; + __assign_str(path); + __entry->create_options = create_options; + __entry->desired_access = desired_access; + ), + TP_printk("xid=%u sid=0x%llx tid=0x%x path=%s cr_opts=0x%x des_access=0x%x", + __entry->xid, __entry->sesid, __entry->tid, __get_str(path), + __entry->create_options, __entry->desired_access) +) + +#define DEFINE_SMB3_OPEN_ENTER_EVENT(name) \ +DEFINE_EVENT(smb3_open_enter_class, smb3_##name, \ + TP_PROTO(unsigned int xid, \ + __u32 tid, \ + __u64 sesid, \ + const char *full_path, \ + int create_options, \ + int desired_access), \ + TP_ARGS(xid, tid, sesid, full_path, create_options, desired_access)) + +DEFINE_SMB3_OPEN_ENTER_EVENT(open_enter); +DEFINE_SMB3_OPEN_ENTER_EVENT(posix_mkdir_enter); + +DECLARE_EVENT_CLASS(smb3_open_err_class, + TP_PROTO(unsigned int xid, + __u32 tid, + __u64 sesid, + int create_options, + int desired_access, + int rc), + TP_ARGS(xid, tid, sesid, create_options, desired_access, rc), + TP_STRUCT__entry( + __field(unsigned int, xid) + __field(__u32, tid) + __field(__u64, sesid) + __field(int, create_options) + __field(int, desired_access) + __field(int, rc) + ), + TP_fast_assign( + __entry->xid = xid; + __entry->tid = tid; + __entry->sesid = sesid; + __entry->create_options = create_options; + __entry->desired_access = desired_access; + __entry->rc = rc; + ), + TP_printk("xid=%u sid=0x%llx tid=0x%x cr_opts=0x%x des_access=0x%x rc=%d", + __entry->xid, __entry->sesid, __entry->tid, + __entry->create_options, __entry->desired_access, __entry->rc) +) + +#define DEFINE_SMB3_OPEN_ERR_EVENT(name) \ +DEFINE_EVENT(smb3_open_err_class, smb3_##name, \ + TP_PROTO(unsigned int xid, \ + __u32 tid, \ + __u64 sesid, \ + int create_options, \ + int desired_access, \ + int rc), \ + TP_ARGS(xid, tid, sesid, create_options, desired_access, rc)) + +DEFINE_SMB3_OPEN_ERR_EVENT(open_err); +DEFINE_SMB3_OPEN_ERR_EVENT(posix_mkdir_err); + +DECLARE_EVENT_CLASS(smb3_open_done_class, + TP_PROTO(unsigned int xid, + __u64 fid, + __u32 tid, + __u64 sesid, + int create_options, + int desired_access, + __u8 oplock), + TP_ARGS(xid, fid, tid, sesid, create_options, desired_access, oplock), + TP_STRUCT__entry( + __field(unsigned int, xid) + __field(__u64, fid) + __field(__u32, tid) + __field(__u64, sesid) + __field(int, create_options) + __field(int, desired_access) + __field(__u8, oplock) + ), + TP_fast_assign( + __entry->xid = xid; + __entry->fid = fid; + __entry->tid = tid; + __entry->sesid = sesid; + __entry->create_options = create_options; + __entry->desired_access = desired_access; + __entry->oplock = oplock; + ), + TP_printk("xid=%u sid=0x%llx tid=0x%x fid=0x%llx cr_opts=0x%x des_access=0x%x oplock=0x%x", + __entry->xid, __entry->sesid, __entry->tid, __entry->fid, + __entry->create_options, __entry->desired_access, __entry->oplock) +) + +#define DEFINE_SMB3_OPEN_DONE_EVENT(name) \ +DEFINE_EVENT(smb3_open_done_class, smb3_##name, \ + TP_PROTO(unsigned int xid, \ + __u64 fid, \ + __u32 tid, \ + __u64 sesid, \ + int create_options, \ + int desired_access, \ + __u8 oplock), \ + TP_ARGS(xid, fid, tid, sesid, create_options, desired_access, oplock)) + +DEFINE_SMB3_OPEN_DONE_EVENT(open_done); +DEFINE_SMB3_OPEN_DONE_EVENT(posix_mkdir_done); + +TRACE_EVENT(smb3_open_cached, + TP_PROTO(unsigned int xid, + __u32 tid, + __u64 sesid, + __u64 fid, + unsigned int oflags, + unsigned int cflags), + TP_ARGS(xid, tid, sesid, fid, oflags, cflags), + TP_STRUCT__entry( + __field(unsigned int, xid) + __field(__u32, tid) + __field(__u64, sesid) + __field(__u64, fid) + __field(unsigned int, oflags) + __field(unsigned int, cflags) + ), + TP_fast_assign( + __entry->xid = xid; + __entry->tid = tid; + __entry->sesid = sesid; + __entry->fid = fid; + __entry->oflags = oflags; + __entry->cflags = cflags; + ), + TP_printk("xid=%u sid=0x%llx tid=0x%x fid=0x%llx oflags=0x%x cflags=0x%x", + __entry->xid, __entry->sesid, __entry->tid, __entry->fid, + __entry->oflags, __entry->cflags) +); + +TRACE_EVENT(smb3_close_cached, + TP_PROTO(__u32 tid, + __u64 sesid, + __u64 fid, + unsigned long delay_jiffies), + TP_ARGS(tid, sesid, fid, delay_jiffies), + TP_STRUCT__entry( + __field(__u32, tid) + __field(__u64, sesid) + __field(__u64, fid) + __field(unsigned long, delay_jiffies) + ), + TP_fast_assign( + __entry->tid = tid; + __entry->sesid = sesid; + __entry->fid = fid; + __entry->delay_jiffies = delay_jiffies; + ), + TP_printk("sid=0x%llx tid=0x%x fid=0x%llx delay_jiffies=%lu", + __entry->sesid, __entry->tid, __entry->fid, __entry->delay_jiffies) +); + + +DECLARE_EVENT_CLASS(smb3_lease_done_class, + TP_PROTO(__u32 lease_state, + __u32 tid, + __u64 sesid, + __u64 lease_key_low, + __u64 lease_key_high), + TP_ARGS(lease_state, tid, sesid, lease_key_low, lease_key_high), + TP_STRUCT__entry( + __field(__u32, lease_state) + __field(__u32, tid) + __field(__u64, sesid) + __field(__u64, lease_key_low) + __field(__u64, lease_key_high) + ), + TP_fast_assign( + __entry->lease_state = lease_state; + __entry->tid = tid; + __entry->sesid = sesid; + __entry->lease_key_low = lease_key_low; + __entry->lease_key_high = lease_key_high; + ), + TP_printk("sid=0x%llx tid=0x%x lease_key=0x%llx%llx lease_state=0x%x", + __entry->sesid, __entry->tid, __entry->lease_key_high, + __entry->lease_key_low, __entry->lease_state) +) + +#define DEFINE_SMB3_LEASE_DONE_EVENT(name) \ +DEFINE_EVENT(smb3_lease_done_class, smb3_##name, \ + TP_PROTO(__u32 lease_state, \ + __u32 tid, \ + __u64 sesid, \ + __u64 lease_key_low, \ + __u64 lease_key_high), \ + TP_ARGS(lease_state, tid, sesid, lease_key_low, lease_key_high)) + +DEFINE_SMB3_LEASE_DONE_EVENT(lease_ack_done); +/* Tracepoint when a lease break request is received/entered (includes epoch and flags) */ +DECLARE_EVENT_CLASS(smb3_lease_enter_class, + TP_PROTO(__u32 lease_state, + __u32 flags, + __u16 epoch, + __u32 tid, + __u64 sesid, + __u64 lease_key_low, + __u64 lease_key_high), + TP_ARGS(lease_state, flags, epoch, tid, sesid, lease_key_low, lease_key_high), + TP_STRUCT__entry( + __field(__u32, lease_state) + __field(__u32, flags) + __field(__u16, epoch) + __field(__u32, tid) + __field(__u64, sesid) + __field(__u64, lease_key_low) + __field(__u64, lease_key_high) + ), + TP_fast_assign( + __entry->lease_state = lease_state; + __entry->flags = flags; + __entry->epoch = epoch; + __entry->tid = tid; + __entry->sesid = sesid; + __entry->lease_key_low = lease_key_low; + __entry->lease_key_high = lease_key_high; + ), + TP_printk("sid=0x%llx tid=0x%x lease_key=0x%llx%llx lease_state=0x%x flags=0x%x epoch=%u", + __entry->sesid, __entry->tid, __entry->lease_key_high, + __entry->lease_key_low, __entry->lease_state, __entry->flags, __entry->epoch) +) + +#define DEFINE_SMB3_LEASE_ENTER_EVENT(name) \ +DEFINE_EVENT(smb3_lease_enter_class, smb3_##name, \ + TP_PROTO(__u32 lease_state, \ + __u32 flags, \ + __u16 epoch, \ + __u32 tid, \ + __u64 sesid, \ + __u64 lease_key_low, \ + __u64 lease_key_high), \ + TP_ARGS(lease_state, flags, epoch, tid, sesid, lease_key_low, lease_key_high)) + +DEFINE_SMB3_LEASE_ENTER_EVENT(lease_break_enter); +/* Lease not found: reuse lease_enter payload (includes epoch and flags) */ +DEFINE_SMB3_LEASE_ENTER_EVENT(lease_not_found); + +DECLARE_EVENT_CLASS(smb3_lease_err_class, + TP_PROTO(__u32 lease_state, + __u32 tid, + __u64 sesid, + __u64 lease_key_low, + __u64 lease_key_high, + int rc), + TP_ARGS(lease_state, tid, sesid, lease_key_low, lease_key_high, rc), + TP_STRUCT__entry( + __field(__u32, lease_state) + __field(__u32, tid) + __field(__u64, sesid) + __field(__u64, lease_key_low) + __field(__u64, lease_key_high) + __field(int, rc) + ), + TP_fast_assign( + __entry->lease_state = lease_state; + __entry->tid = tid; + __entry->sesid = sesid; + __entry->lease_key_low = lease_key_low; + __entry->lease_key_high = lease_key_high; + __entry->rc = rc; + ), + TP_printk("sid=0x%llx tid=0x%x lease_key=0x%llx%llx lease_state=0x%x rc=%d", + __entry->sesid, __entry->tid, __entry->lease_key_high, + __entry->lease_key_low, __entry->lease_state, __entry->rc) +) + +#define DEFINE_SMB3_LEASE_ERR_EVENT(name) \ +DEFINE_EVENT(smb3_lease_err_class, smb3_##name, \ + TP_PROTO(__u32 lease_state, \ + __u32 tid, \ + __u64 sesid, \ + __u64 lease_key_low, \ + __u64 lease_key_high, \ + int rc), \ + TP_ARGS(lease_state, tid, sesid, lease_key_low, lease_key_high, rc)) + +DEFINE_SMB3_LEASE_ERR_EVENT(lease_ack_err); + +DECLARE_EVENT_CLASS(smb3_connect_class, + TP_PROTO(char *hostname, + __u64 conn_id, + const struct __kernel_sockaddr_storage *dst_addr), + TP_ARGS(hostname, conn_id, dst_addr), + TP_STRUCT__entry( + __string(hostname, hostname) + __field(__u64, conn_id) + __array(__u8, dst_addr, sizeof(struct sockaddr_storage)) + ), + TP_fast_assign( + struct sockaddr_storage *pss = NULL; + + __entry->conn_id = conn_id; + pss = (struct sockaddr_storage *)__entry->dst_addr; + *pss = *dst_addr; + __assign_str(hostname); + ), + TP_printk("conn_id=0x%llx server=%s addr=%pISpsfc", + __entry->conn_id, + __get_str(hostname), + __entry->dst_addr) +) + +#define DEFINE_SMB3_CONNECT_EVENT(name) \ +DEFINE_EVENT(smb3_connect_class, smb3_##name, \ + TP_PROTO(char *hostname, \ + __u64 conn_id, \ + const struct __kernel_sockaddr_storage *addr), \ + TP_ARGS(hostname, conn_id, addr)) + +DEFINE_SMB3_CONNECT_EVENT(connect_done); +DEFINE_SMB3_CONNECT_EVENT(smbd_connect_done); +DEFINE_SMB3_CONNECT_EVENT(smbd_connect_err); + +DECLARE_EVENT_CLASS(smb3_connect_err_class, + TP_PROTO(char *hostname, __u64 conn_id, + const struct __kernel_sockaddr_storage *dst_addr, int rc), + TP_ARGS(hostname, conn_id, dst_addr, rc), + TP_STRUCT__entry( + __string(hostname, hostname) + __field(__u64, conn_id) + __array(__u8, dst_addr, sizeof(struct sockaddr_storage)) + __field(int, rc) + ), + TP_fast_assign( + struct sockaddr_storage *pss = NULL; + + __entry->conn_id = conn_id; + __entry->rc = rc; + pss = (struct sockaddr_storage *)__entry->dst_addr; + *pss = *dst_addr; + __assign_str(hostname); + ), + TP_printk("rc=%d conn_id=0x%llx server=%s addr=%pISpsfc", + __entry->rc, + __entry->conn_id, + __get_str(hostname), + __entry->dst_addr) +) + +#define DEFINE_SMB3_CONNECT_ERR_EVENT(name) \ +DEFINE_EVENT(smb3_connect_err_class, smb3_##name, \ + TP_PROTO(char *hostname, \ + __u64 conn_id, \ + const struct __kernel_sockaddr_storage *addr, \ + int rc), \ + TP_ARGS(hostname, conn_id, addr, rc)) + +DEFINE_SMB3_CONNECT_ERR_EVENT(connect_err); + +DECLARE_EVENT_CLASS(smb3_sess_setup_err_class, + TP_PROTO(char *hostname, char *username, __u64 conn_id, + const struct __kernel_sockaddr_storage *dst_addr, int rc), + TP_ARGS(hostname, username, conn_id, dst_addr, rc), + TP_STRUCT__entry( + __string(hostname, hostname) + __string(username, username) + __field(__u64, conn_id) + __array(__u8, dst_addr, sizeof(struct sockaddr_storage)) + __field(int, rc) + ), + TP_fast_assign( + struct sockaddr_storage *pss = NULL; + + __entry->conn_id = conn_id; + __entry->rc = rc; + pss = (struct sockaddr_storage *)__entry->dst_addr; + *pss = *dst_addr; + __assign_str(hostname); + __assign_str(username); + ), + TP_printk("rc=%d user=%s conn_id=0x%llx server=%s addr=%pISpsfc", + __entry->rc, + __get_str(username), + __entry->conn_id, + __get_str(hostname), + __entry->dst_addr) +) + +#define DEFINE_SMB3_SES_SETUP_ERR_EVENT(name) \ +DEFINE_EVENT(smb3_sess_setup_err_class, smb3_##name, \ + TP_PROTO(char *hostname, \ + char *username, \ + __u64 conn_id, \ + const struct __kernel_sockaddr_storage *addr, \ + int rc), \ + TP_ARGS(hostname, username, conn_id, addr, rc)) + +DEFINE_SMB3_SES_SETUP_ERR_EVENT(key_expired); + +DECLARE_EVENT_CLASS(smb3_reconnect_class, + TP_PROTO(__u64 currmid, + __u64 conn_id, + char *hostname), + TP_ARGS(currmid, conn_id, hostname), + TP_STRUCT__entry( + __field(__u64, currmid) + __field(__u64, conn_id) + __string(hostname, hostname) + ), + TP_fast_assign( + __entry->currmid = currmid; + __entry->conn_id = conn_id; + __assign_str(hostname); + ), + TP_printk("conn_id=0x%llx server=%s current_mid=%llu", + __entry->conn_id, + __get_str(hostname), + __entry->currmid) +) + +#define DEFINE_SMB3_RECONNECT_EVENT(name) \ +DEFINE_EVENT(smb3_reconnect_class, smb3_##name, \ + TP_PROTO(__u64 currmid, \ + __u64 conn_id, \ + char *hostname), \ + TP_ARGS(currmid, conn_id, hostname)) + +DEFINE_SMB3_RECONNECT_EVENT(reconnect); +DEFINE_SMB3_RECONNECT_EVENT(partial_send_reconnect); + +DECLARE_EVENT_CLASS(smb3_ses_class, + TP_PROTO(__u64 sesid), + TP_ARGS(sesid), + TP_STRUCT__entry( + __field(__u64, sesid) + ), + TP_fast_assign( + __entry->sesid = sesid; + ), + TP_printk("sid=0x%llx", + __entry->sesid) +) + +#define DEFINE_SMB3_SES_EVENT(name) \ +DEFINE_EVENT(smb3_ses_class, smb3_##name, \ + TP_PROTO(__u64 sesid), \ + TP_ARGS(sesid)) + +DEFINE_SMB3_SES_EVENT(ses_not_found); + +DECLARE_EVENT_CLASS(smb3_ioctl_class, + TP_PROTO(unsigned int xid, + __u64 fid, + unsigned int command), + TP_ARGS(xid, fid, command), + TP_STRUCT__entry( + __field(unsigned int, xid) + __field(__u64, fid) + __field(unsigned int, command) + ), + TP_fast_assign( + __entry->xid = xid; + __entry->fid = fid; + __entry->command = command; + ), + TP_printk("xid=%u fid=0x%llx ioctl cmd=0x%x", + __entry->xid, __entry->fid, __entry->command) +) + +#define DEFINE_SMB3_IOCTL_EVENT(name) \ +DEFINE_EVENT(smb3_ioctl_class, smb3_##name, \ + TP_PROTO(unsigned int xid, \ + __u64 fid, \ + unsigned int command), \ + TP_ARGS(xid, fid, command)) + +DEFINE_SMB3_IOCTL_EVENT(ioctl); +DEFINE_SMB3_IOCTL_EVENT(unsupported_ioctl); + +DECLARE_EVENT_CLASS(smb3_shutdown_class, + TP_PROTO(__u32 flags, + __u32 tid), + TP_ARGS(flags, tid), + TP_STRUCT__entry( + __field(__u32, flags) + __field(__u32, tid) + ), + TP_fast_assign( + __entry->flags = flags; + __entry->tid = tid; + ), + TP_printk("flags=0x%x tid=0x%x", + __entry->flags, __entry->tid) +) + +#define DEFINE_SMB3_SHUTDOWN_EVENT(name) \ +DEFINE_EVENT(smb3_shutdown_class, smb3_##name, \ + TP_PROTO(__u32 flags, \ + __u32 tid), \ + TP_ARGS(flags, tid)) + +DEFINE_SMB3_SHUTDOWN_EVENT(shutdown_enter); +DEFINE_SMB3_SHUTDOWN_EVENT(shutdown_done); + +DECLARE_EVENT_CLASS(smb3_shutdown_err_class, + TP_PROTO(int rc, + __u32 flags, + __u32 tid), + TP_ARGS(rc, flags, tid), + TP_STRUCT__entry( + __field(int, rc) + __field(__u32, flags) + __field(__u32, tid) + ), + TP_fast_assign( + __entry->rc = rc; + __entry->flags = flags; + __entry->tid = tid; + ), + TP_printk("rc=%d flags=0x%x tid=0x%x", + __entry->rc, __entry->flags, __entry->tid) +) + +#define DEFINE_SMB3_SHUTDOWN_ERR_EVENT(name) \ +DEFINE_EVENT(smb3_shutdown_err_class, smb3_##name, \ + TP_PROTO(int rc, \ + __u32 flags, \ + __u32 tid), \ + TP_ARGS(rc, flags, tid)) + +DEFINE_SMB3_SHUTDOWN_ERR_EVENT(shutdown_err); + +DECLARE_EVENT_CLASS(smb3_credit_class, + TP_PROTO(__u64 currmid, + __u64 conn_id, + char *hostname, + int credits, + int credits_to_add, + int in_flight), + TP_ARGS(currmid, conn_id, hostname, credits, credits_to_add, in_flight), + TP_STRUCT__entry( + __field(__u64, currmid) + __field(__u64, conn_id) + __string(hostname, hostname) + __field(int, credits) + __field(int, credits_to_add) + __field(int, in_flight) + ), + TP_fast_assign( + __entry->currmid = currmid; + __entry->conn_id = conn_id; + __assign_str(hostname); + __entry->credits = credits; + __entry->credits_to_add = credits_to_add; + __entry->in_flight = in_flight; + ), + TP_printk("conn_id=0x%llx server=%s current_mid=%llu " + "credits=%d credit_change=%d in_flight=%d", + __entry->conn_id, + __get_str(hostname), + __entry->currmid, + __entry->credits, + __entry->credits_to_add, + __entry->in_flight) +) + +#define DEFINE_SMB3_CREDIT_EVENT(name) \ +DEFINE_EVENT(smb3_credit_class, smb3_##name, \ + TP_PROTO(__u64 currmid, \ + __u64 conn_id, \ + char *hostname, \ + int credits, \ + int credits_to_add, \ + int in_flight), \ + TP_ARGS(currmid, conn_id, hostname, credits, credits_to_add, in_flight)) + +DEFINE_SMB3_CREDIT_EVENT(reconnect_with_invalid_credits); +DEFINE_SMB3_CREDIT_EVENT(reconnect_detected); +DEFINE_SMB3_CREDIT_EVENT(credit_timeout); +DEFINE_SMB3_CREDIT_EVENT(insufficient_credits); +DEFINE_SMB3_CREDIT_EVENT(too_many_credits); +DEFINE_SMB3_CREDIT_EVENT(add_credits); +DEFINE_SMB3_CREDIT_EVENT(adj_credits); +DEFINE_SMB3_CREDIT_EVENT(hdr_credits); +DEFINE_SMB3_CREDIT_EVENT(nblk_credits); +DEFINE_SMB3_CREDIT_EVENT(pend_credits); +DEFINE_SMB3_CREDIT_EVENT(wait_credits); +DEFINE_SMB3_CREDIT_EVENT(waitff_credits); +DEFINE_SMB3_CREDIT_EVENT(overflow_credits); +DEFINE_SMB3_CREDIT_EVENT(set_credits); + +TRACE_EVENT(smb3_kerberos_auth, + TP_PROTO(struct TCP_Server_Info *server, + struct cifs_ses *ses, + int rc), + TP_ARGS(server, ses, rc), + TP_STRUCT__entry( + __field(pid_t, pid) + __field(uid_t, uid) + __field(uid_t, cruid) + __string(host, server->hostname) + __string(user, ses->user_name) + __array(__u8, addr, sizeof(struct sockaddr_storage)) + __array(char, sec, sizeof("ntlmsspi")) + __array(char, upcall_target, sizeof("mount")) + __field(int, rc) + ), + TP_fast_assign( + __entry->pid = current->pid; + __entry->uid = from_kuid_munged(&init_user_ns, ses->linux_uid); + __entry->cruid = from_kuid_munged(&init_user_ns, ses->cred_uid); + __assign_str(host); + __assign_str(user); + memcpy(__entry->addr, &server->dstaddr, sizeof(__entry->addr)); + + if (server->sec_kerberos) + memcpy(__entry->sec, "krb5", sizeof("krb5")); + else if (server->sec_mskerberos) + memcpy(__entry->sec, "mskrb5", sizeof("mskrb5")); + else if (server->sec_iakerb) + memcpy(__entry->sec, "iakerb", sizeof("iakerb")); + else + memcpy(__entry->sec, "krb5", sizeof("krb5")); + + if (ses->upcall_target == UPTARGET_MOUNT) + memcpy(__entry->upcall_target, "mount", sizeof("mount")); + else + memcpy(__entry->upcall_target, "app", sizeof("app")); + __entry->rc = rc; + ), + TP_printk("vers=%d host=%s ip=%pISpsfc sec=%s uid=%d cruid=%d user=%s pid=%d upcall_target=%s err=%d", + CIFS_SPNEGO_UPCALL_VERSION, __get_str(host), __entry->addr, + __entry->sec, __entry->uid, __entry->cruid, __get_str(user), + __entry->pid, __entry->upcall_target, __entry->rc)) + +TRACE_EVENT(smb3_tcon_ref, + TP_PROTO(unsigned int tcon_debug_id, int ref, + enum smb3_tcon_ref_trace trace), + TP_ARGS(tcon_debug_id, ref, trace), + TP_STRUCT__entry( + __field(unsigned int, tcon) + __field(int, ref) + __field(enum smb3_tcon_ref_trace, trace) + ), + TP_fast_assign( + __entry->tcon = tcon_debug_id; + __entry->ref = ref; + __entry->trace = trace; + ), + TP_printk("TC=%08x %s r=%u", + __entry->tcon, + __print_symbolic(__entry->trace, smb3_tcon_ref_traces), + __entry->ref) + ); + +TRACE_EVENT(smb3_rw_credits, + TP_PROTO(unsigned int rreq_debug_id, + unsigned int subreq_debug_index, + unsigned int subreq_credits, + unsigned int server_credits, + int server_in_flight, + int credit_change, + enum smb3_rw_credits_trace trace), + TP_ARGS(rreq_debug_id, subreq_debug_index, subreq_credits, + server_credits, server_in_flight, credit_change, trace), + TP_STRUCT__entry( + __field(unsigned int, rreq_debug_id) + __field(unsigned int, subreq_debug_index) + __field(unsigned int, subreq_credits) + __field(unsigned int, server_credits) + __field(int, in_flight) + __field(int, credit_change) + __field(enum smb3_rw_credits_trace, trace) + ), + TP_fast_assign( + __entry->rreq_debug_id = rreq_debug_id; + __entry->subreq_debug_index = subreq_debug_index; + __entry->subreq_credits = subreq_credits; + __entry->server_credits = server_credits; + __entry->in_flight = server_in_flight; + __entry->credit_change = credit_change; + __entry->trace = trace; + ), + TP_printk("R=%08x[%x] %s cred=%u chg=%d pool=%u ifl=%d", + __entry->rreq_debug_id, __entry->subreq_debug_index, + __print_symbolic(__entry->trace, smb3_rw_credits_traces), + __entry->subreq_credits, __entry->credit_change, + __entry->server_credits, __entry->in_flight) + ); + +TRACE_EVENT(smb3_eio, + TP_PROTO(enum smb_eio_trace trace, unsigned long info, unsigned long info2), + TP_ARGS(trace, info, info2), + TP_STRUCT__entry( + __field(enum smb_eio_trace, trace) + __field(unsigned long, info) + __field(unsigned long, info2) + ), + TP_fast_assign( + __entry->trace = trace; + __entry->info = info; + __entry->info2 = info2; + ), + TP_printk("%s info=%lx,%lx", + __print_symbolic(__entry->trace, smb_eio_traces), + __entry->info, __entry->info2) + ); + +#undef EM +#undef E_ +#endif /* _CIFS_TRACE_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE trace +#include diff --git a/fs/smb/client/transport.c b/fs/smb/client/transport.c new file mode 100644 index 000000000..fdf4e50c2 --- /dev/null +++ b/fs/smb/client/transport.c @@ -0,0 +1,1291 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * Copyright (C) International Business Machines Corp., 2002,2008 + * Author(s): Steve French (sfrench@us.ibm.com) + * Jeremy Allison (jra@samba.org) 2006. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "cifsglob.h" +#include "cifsproto.h" +#include "cifs_debug.h" +#include "smb2proto.h" +#include "smbdirect.h" +#include "compress.h" + +void +cifs_wake_up_task(struct TCP_Server_Info *server, struct mid_q_entry *mid) +{ + if (mid->mid_state == MID_RESPONSE_RECEIVED) + mid->mid_state = MID_RESPONSE_READY; + wake_up_process(mid->callback_data); +} + +void __release_mid(struct TCP_Server_Info *server, struct mid_q_entry *midEntry) +{ +#ifdef CONFIG_CIFS_STATS2 + __le16 command = server->vals->lock_cmd; + __u16 smb_cmd = le16_to_cpu(midEntry->command); + unsigned long now; + unsigned long roundtrip_time; +#endif + + if (midEntry->resp_buf && (midEntry->wait_cancelled) && + (midEntry->mid_state == MID_RESPONSE_RECEIVED || + midEntry->mid_state == MID_RESPONSE_READY) && + server->ops->handle_cancelled_mid) + server->ops->handle_cancelled_mid(midEntry, server); + + midEntry->mid_state = MID_FREE; + atomic_dec(&mid_count); + if (midEntry->large_buf) + cifs_buf_release(midEntry->resp_buf); + else + cifs_small_buf_release(midEntry->resp_buf); +#ifdef CONFIG_CIFS_STATS2 + now = jiffies; + if (now < midEntry->when_alloc) + cifs_server_dbg(VFS, "Invalid mid allocation time\n"); + roundtrip_time = now - midEntry->when_alloc; + + if (smb_cmd < NUMBER_OF_SMB2_COMMANDS) { + if (atomic_read(&server->num_cmds[smb_cmd]) == 0) { + server->slowest_cmd[smb_cmd] = roundtrip_time; + server->fastest_cmd[smb_cmd] = roundtrip_time; + } else { + if (server->slowest_cmd[smb_cmd] < roundtrip_time) + server->slowest_cmd[smb_cmd] = roundtrip_time; + else if (server->fastest_cmd[smb_cmd] > roundtrip_time) + server->fastest_cmd[smb_cmd] = roundtrip_time; + } + cifs_stats_inc(&server->num_cmds[smb_cmd]); + server->time_per_cmd[smb_cmd] += roundtrip_time; + } + /* + * commands taking longer than one second (default) can be indications + * that something is wrong, unless it is quite a slow link or a very + * busy server. Note that this calc is unlikely or impossible to wrap + * as long as slow_rsp_threshold is not set way above recommended max + * value (32767 ie 9 hours) and is generally harmless even if wrong + * since only affects debug counters - so leaving the calc as simple + * comparison rather than doing multiple conversions and overflow + * checks + */ + if ((slow_rsp_threshold != 0) && + time_after(now, midEntry->when_alloc + (slow_rsp_threshold * HZ)) && + (midEntry->command != command)) { + /* + * smb2slowcmd[NUMBER_OF_SMB2_COMMANDS] counts by command + * NB: le16_to_cpu returns unsigned so can not be negative below + */ + if (smb_cmd < NUMBER_OF_SMB2_COMMANDS) + cifs_stats_inc(&server->smb2slowcmd[smb_cmd]); + + trace_smb3_slow_rsp(smb_cmd, midEntry->mid, midEntry->pid, + midEntry->when_sent, midEntry->when_received); + if (cifsFYI & CIFS_TIMER) { + pr_debug("slow rsp: cmd %d mid %llu", + midEntry->command, midEntry->mid); + cifs_info("A: 0x%lx S: 0x%lx R: 0x%lx\n", + now - midEntry->when_alloc, + now - midEntry->when_sent, + now - midEntry->when_received); + } + } +#endif + put_task_struct(midEntry->creator); + + mempool_free(midEntry, &cifs_mid_pool); +} + +void +delete_mid(struct TCP_Server_Info *server, struct mid_q_entry *mid) +{ + spin_lock(&server->mid_queue_lock); + + if (!mid->deleted_from_q) { + list_del_init(&mid->qhead); + mid->deleted_from_q = true; + } + spin_unlock(&server->mid_queue_lock); + + release_mid(server, mid); +} + +/* + * smb_send_kvec - send an array of kvecs to the server + * @server: Server to send the data to + * @smb_msg: Message to send + * @sent: amount of data sent on socket is stored here + * + * Our basic "send data to server" function. Should be called with srv_mutex + * held. The caller is responsible for handling the results. + */ +int +smb_send_kvec(struct TCP_Server_Info *server, struct msghdr *smb_msg, + size_t *sent) +{ + int rc = 0; + int retries = 0; + struct socket *ssocket = server->ssocket; + + *sent = 0; + + if (server->noblocksnd) + smb_msg->msg_flags = MSG_DONTWAIT + MSG_NOSIGNAL; + else + smb_msg->msg_flags = MSG_NOSIGNAL; + + while (msg_data_left(smb_msg)) { + /* + * If blocking send, we try 3 times, since each can block + * for 5 seconds. For nonblocking we have to try more + * but wait increasing amounts of time allowing time for + * socket to clear. The overall time we wait in either + * case to send on the socket is about 15 seconds. + * Similarly we wait for 15 seconds for a response from + * the server in SendReceive[2] for the server to send + * a response back for most types of requests (except + * SMB Write past end of file which can be slow, and + * blocking lock operations). NFS waits slightly longer + * than CIFS, but this can make it take longer for + * nonresponsive servers to be detected and 15 seconds + * is more than enough time for modern networks to + * send a packet. In most cases if we fail to send + * after the retries we will kill the socket and + * reconnect which may clear the network problem. + * + * Even if regular signals are masked, EINTR might be + * propagated from sk_stream_wait_memory() to here when + * TIF_NOTIFY_SIGNAL is used for task work. For example, + * certain io_uring completions will use that. Treat + * having EINTR with pending task work the same as EAGAIN + * to avoid unnecessary reconnects. + */ + rc = sock_sendmsg(ssocket, smb_msg); + if (rc == -EAGAIN || unlikely(rc == -EINTR && task_work_pending(current))) { + retries++; + if (retries >= 14 || + (!server->noblocksnd && (retries > 2))) { + cifs_server_dbg(VFS, "sends on sock %p stuck for 15 seconds\n", + ssocket); + return -EAGAIN; + } + msleep(1 << retries); + continue; + } + + if (rc < 0) + return rc; + + if (rc == 0) { + /* should never happen, letting socket clear before + retrying is our only obvious option here */ + cifs_server_dbg(VFS, "tcp sent no data\n"); + msleep(500); + continue; + } + + /* send was at least partially successful */ + *sent += rc; + retries = 0; /* in case we get ENOSPC on the next send */ + } + return 0; +} + +unsigned long +smb_rqst_len(struct TCP_Server_Info *server, struct smb_rqst *rqst) +{ + unsigned int i; + struct kvec *iov; + int nvec; + unsigned long buflen = 0; + + if (!is_smb1(server) && rqst->rq_nvec >= 2 && + rqst->rq_iov[0].iov_len == 4) { + iov = &rqst->rq_iov[1]; + nvec = rqst->rq_nvec - 1; + } else { + iov = rqst->rq_iov; + nvec = rqst->rq_nvec; + } + + /* total up iov array first */ + for (i = 0; i < nvec; i++) + buflen += iov[i].iov_len; + + buflen += iov_iter_count(&rqst->rq_iter); + return buflen; +} + +int __smb_send_rqst(struct TCP_Server_Info *server, int num_rqst, + struct smb_rqst *rqst) +{ + int rc; + struct kvec *iov; + int n_vec; + unsigned int send_length = 0; + unsigned int i, j; + sigset_t mask, oldmask; + size_t total_len = 0, sent, size; + struct socket *ssocket = server->ssocket; + struct msghdr smb_msg = {}; + __be32 rfc1002_marker; + + cifs_in_send_inc(server); + if (cifs_rdma_enabled(server)) { + /* return -EAGAIN when connecting or reconnecting */ + rc = -EAGAIN; + if (server->smbd_conn) + rc = smbd_send(server, num_rqst, rqst); + goto smbd_done; + } + + rc = -EAGAIN; + if (ssocket == NULL) + goto out; + + rc = -ERESTARTSYS; + if (fatal_signal_pending(current)) { + cifs_dbg(FYI, "signal pending before send request\n"); + goto out; + } + + rc = 0; + /* cork the socket */ + tcp_sock_set_cork(ssocket->sk, true); + + for (j = 0; j < num_rqst; j++) + send_length += smb_rqst_len(server, &rqst[j]); + rfc1002_marker = cpu_to_be32(send_length); + + /* + * We should not allow signals to interrupt the network send because + * any partial send will cause session reconnects thus increasing + * latency of system calls and overload a server with unnecessary + * requests. + */ + + sigfillset(&mask); + sigprocmask(SIG_BLOCK, &mask, &oldmask); + + /* Generate a rfc1002 marker */ + { + struct kvec hiov = { + .iov_base = &rfc1002_marker, + .iov_len = 4 + }; + iov_iter_kvec(&smb_msg.msg_iter, ITER_SOURCE, &hiov, 1, 4); + rc = smb_send_kvec(server, &smb_msg, &sent); + if (rc < 0) + goto unmask; + + total_len += sent; + send_length += 4; + } + + cifs_dbg(FYI, "Sending smb: smb_len=%u\n", send_length); + + for (j = 0; j < num_rqst; j++) { + iov = rqst[j].rq_iov; + n_vec = rqst[j].rq_nvec; + + size = 0; + for (i = 0; i < n_vec; i++) { + dump_smb(iov[i].iov_base, iov[i].iov_len); + size += iov[i].iov_len; + } + + iov_iter_kvec(&smb_msg.msg_iter, ITER_SOURCE, iov, n_vec, size); + + rc = smb_send_kvec(server, &smb_msg, &sent); + if (rc < 0) + goto unmask; + + total_len += sent; + + if (iov_iter_count(&rqst[j].rq_iter) > 0) { + smb_msg.msg_iter = rqst[j].rq_iter; + rc = smb_send_kvec(server, &smb_msg, &sent); + if (rc < 0) + break; + total_len += sent; + } + } + +unmask: + sigprocmask(SIG_SETMASK, &oldmask, NULL); + + /* + * If signal is pending but we have already sent the whole packet to + * the server we need to return success status to allow a corresponding + * mid entry to be kept in the pending requests queue thus allowing + * to handle responses from the server by the client. + * + * If only part of the packet has been sent there is no need to hide + * interrupt because the session will be reconnected anyway, so there + * won't be any response from the server to handle. + */ + + if (signal_pending(current) && (total_len != send_length)) { + cifs_dbg(FYI, "signal is pending after attempt to send\n"); + rc = -ERESTARTSYS; + } + + /* uncork it */ + tcp_sock_set_cork(ssocket->sk, false); + + if ((total_len > 0) && (total_len != send_length)) { + cifs_dbg(FYI, "partial send (wanted=%u sent=%zu): terminating session\n", + send_length, total_len); + /* + * If we have only sent part of an SMB then the next SMB could + * be taken as the remainder of this one. We need to kill the + * socket so the server throws away the partial SMB + */ + cifs_signal_cifsd_for_reconnect(server, false); + trace_smb3_partial_send_reconnect(server->current_mid, + server->conn_id, server->hostname); + } +smbd_done: + /* + * there's hardly any use for the layers above to know the + * actual error code here. All they should do at this point is + * to retry the connection and hope it goes away. + */ + if (rc < 0 && rc != -EINTR && rc != -EAGAIN) { + cifs_server_dbg(VFS, "Error %d sending data on socket to server\n", + rc); + rc = -ECONNABORTED; + cifs_signal_cifsd_for_reconnect(server, false); + } else if (rc > 0) + rc = 0; +out: + cifs_in_send_dec(server); + return rc; +} + +static int +smb_send_rqst(struct TCP_Server_Info *server, int num_rqst, + struct smb_rqst *rqst, int flags) +{ + struct smb2_transform_hdr tr_hdr; + struct smb_rqst new_rqst[MAX_COMPOUND] = {}; + struct kvec iov = { + .iov_base = &tr_hdr, + .iov_len = sizeof(tr_hdr), + }; + int rc; + + if (flags & CIFS_COMPRESS_REQ) + return smb_compress(server, &rqst[0], __smb_send_rqst); + + if (!(flags & CIFS_TRANSFORM_REQ)) + return __smb_send_rqst(server, num_rqst, rqst); + + if (WARN_ON_ONCE(num_rqst > MAX_COMPOUND - 1)) + return smb_EIO1(smb_eio_trace_tx_max_compound, num_rqst); + + if (!server->ops->init_transform_rq) { + cifs_server_dbg(VFS, "Encryption requested but transform callback is missing\n"); + return smb_EIO(smb_eio_trace_tx_need_transform); + } + + new_rqst[0].rq_iov = &iov; + new_rqst[0].rq_nvec = 1; + + rc = server->ops->init_transform_rq(server, num_rqst + 1, + new_rqst, rqst); + if (!rc) { + rc = __smb_send_rqst(server, num_rqst + 1, new_rqst); + smb3_free_compound_rqst(num_rqst, &new_rqst[1]); + } + return rc; +} + +static int +wait_for_free_credits(struct TCP_Server_Info *server, const int num_credits, + const int timeout, const int flags, + unsigned int *instance) +{ + long rc; + int *credits; + int optype; + long int t; + int scredits, in_flight; + + if (timeout < 0) + t = MAX_JIFFY_OFFSET; + else + t = msecs_to_jiffies(timeout); + + optype = flags & CIFS_OP_MASK; + + *instance = 0; + + credits = server->ops->get_credits_field(server, optype); + /* Since an echo is already inflight, no need to wait to send another */ + if (*credits <= 0 && optype == CIFS_ECHO_OP) + return -EAGAIN; + + spin_lock(&server->req_lock); + if ((flags & CIFS_TIMEOUT_MASK) == CIFS_NON_BLOCKING) { + /* oplock breaks must not be held up */ + server->in_flight++; + if (server->in_flight > server->max_in_flight) + server->max_in_flight = server->in_flight; + *credits -= 1; + *instance = server->reconnect_instance; + scredits = *credits; + in_flight = server->in_flight; + spin_unlock(&server->req_lock); + + trace_smb3_nblk_credits(server->current_mid, + server->conn_id, server->hostname, scredits, -1, in_flight); + cifs_dbg(FYI, "%s: remove %u credits total=%d\n", + __func__, 1, scredits); + + return 0; + } + + while (1) { + spin_unlock(&server->req_lock); + + spin_lock(&server->srv_lock); + if (server->tcpStatus == CifsExiting) { + spin_unlock(&server->srv_lock); + return -ENOENT; + } + spin_unlock(&server->srv_lock); + + spin_lock(&server->req_lock); + if (*credits < num_credits) { + scredits = *credits; + spin_unlock(&server->req_lock); + + cifs_num_waiters_inc(server); + rc = wait_event_killable_timeout(server->request_q, + has_credits(server, credits, num_credits), t); + cifs_num_waiters_dec(server); + if (!rc) { + spin_lock(&server->req_lock); + scredits = *credits; + in_flight = server->in_flight; + spin_unlock(&server->req_lock); + + trace_smb3_credit_timeout(server->current_mid, + server->conn_id, server->hostname, scredits, + num_credits, in_flight); + cifs_server_dbg(VFS, "wait timed out after %d ms\n", + timeout); + return -EBUSY; + } + if (rc == -ERESTARTSYS) + return -ERESTARTSYS; + spin_lock(&server->req_lock); + } else { + /* + * For normal commands, reserve the last MAX_COMPOUND + * credits to compound requests. + * Otherwise these compounds could be permanently + * starved for credits by single-credit requests. + * + * To prevent spinning CPU, block this thread until + * there are >MAX_COMPOUND credits available. + * But only do this is we already have a lot of + * credits in flight to avoid triggering this check + * for servers that are slow to hand out credits on + * new sessions. + */ + if (!optype && num_credits == 1 && + server->in_flight > 2 * MAX_COMPOUND && + *credits <= MAX_COMPOUND) { + spin_unlock(&server->req_lock); + + cifs_num_waiters_inc(server); + rc = wait_event_killable_timeout( + server->request_q, + has_credits(server, credits, + MAX_COMPOUND + 1), + t); + cifs_num_waiters_dec(server); + if (!rc) { + spin_lock(&server->req_lock); + scredits = *credits; + in_flight = server->in_flight; + spin_unlock(&server->req_lock); + + trace_smb3_credit_timeout( + server->current_mid, + server->conn_id, server->hostname, + scredits, num_credits, in_flight); + cifs_server_dbg(VFS, "wait timed out after %d ms\n", + timeout); + return -EBUSY; + } + if (rc == -ERESTARTSYS) + return -ERESTARTSYS; + spin_lock(&server->req_lock); + continue; + } + + /* + * Can not count locking commands against total + * as they are allowed to block on server. + */ + + /* update # of requests on the wire to server */ + if ((flags & CIFS_TIMEOUT_MASK) != CIFS_BLOCKING_OP) { + *credits -= num_credits; + server->in_flight += num_credits; + if (server->in_flight > server->max_in_flight) + server->max_in_flight = server->in_flight; + *instance = server->reconnect_instance; + } + scredits = *credits; + in_flight = server->in_flight; + spin_unlock(&server->req_lock); + + trace_smb3_waitff_credits(server->current_mid, + server->conn_id, server->hostname, scredits, + -(num_credits), in_flight); + cifs_dbg(FYI, "%s: remove %u credits total=%d\n", + __func__, num_credits, scredits); + break; + } + } + return 0; +} + +int wait_for_free_request(struct TCP_Server_Info *server, const int flags, + unsigned int *instance) +{ + return wait_for_free_credits(server, 1, -1, flags, + instance); +} + +static int +wait_for_compound_request(struct TCP_Server_Info *server, int num, + const int flags, unsigned int *instance) +{ + int *credits; + int scredits, in_flight; + + credits = server->ops->get_credits_field(server, flags & CIFS_OP_MASK); + + spin_lock(&server->req_lock); + scredits = *credits; + in_flight = server->in_flight; + + if (*credits < num) { + /* + * If the server is tight on resources or just gives us less + * credits for other reasons (e.g. requests are coming out of + * order and the server delays granting more credits until it + * processes a missing mid) and we exhausted most available + * credits there may be situations when we try to send + * a compound request but we don't have enough credits. At this + * point the client needs to decide if it should wait for + * additional credits or fail the request. If at least one + * request is in flight there is a high probability that the + * server will return enough credits to satisfy this compound + * request. + * + * Return immediately if no requests in flight since we will be + * stuck on waiting for credits. + */ + if (server->in_flight == 0) { + spin_unlock(&server->req_lock); + trace_smb3_insufficient_credits(server->current_mid, + server->conn_id, server->hostname, scredits, + num, in_flight); + cifs_dbg(FYI, "%s: %d requests in flight, needed %d total=%d\n", + __func__, in_flight, num, scredits); + return -EDEADLK; + } + } + spin_unlock(&server->req_lock); + + return wait_for_free_credits(server, num, 60000, flags, + instance); +} + +int +cifs_wait_mtu_credits(struct TCP_Server_Info *server, size_t size, + size_t *num, struct cifs_credits *credits) +{ + *num = size; + credits->value = 0; + credits->instance = server->reconnect_instance; + return 0; +} + +int wait_for_response(struct TCP_Server_Info *server, struct mid_q_entry *mid) +{ + unsigned int sleep_state = TASK_KILLABLE; + int error; + + if (mid->sr_flags & CIFS_INTERRUPTIBLE_WAIT) + sleep_state = TASK_INTERRUPTIBLE; + + error = wait_event_state(server->response_q, + mid->mid_state != MID_REQUEST_SUBMITTED && + mid->mid_state != MID_RESPONSE_RECEIVED, + (sleep_state | TASK_FREEZABLE_UNSAFE)); + if (error < 0) + return -ERESTARTSYS; + + return 0; +} + +/* + * Send a SMB request and set the callback function in the mid to handle + * the result. Caller is responsible for dealing with timeouts. + */ +int +cifs_call_async(struct TCP_Server_Info *server, struct smb_rqst *rqst, + mid_receive_t receive, mid_callback_t callback, + mid_handle_t handle, void *cbdata, const int flags, + const struct cifs_credits *exist_credits) +{ + int rc; + struct mid_q_entry *mid; + struct cifs_credits credits = { .value = 0, .instance = 0 }; + unsigned int instance; + int optype; + + optype = flags & CIFS_OP_MASK; + + if ((flags & CIFS_HAS_CREDITS) == 0) { + rc = wait_for_free_request(server, flags, &instance); + if (rc) + return rc; + credits.value = 1; + credits.instance = instance; + } else + instance = exist_credits->instance; + + cifs_server_lock(server); + + /* + * We can't use credits obtained from the previous session to send this + * request. Check if there were reconnects after we obtained credits and + * return -EAGAIN in such cases to let callers handle it. + */ + if (instance != server->reconnect_instance) { + cifs_server_unlock(server); + add_credits_and_wake_if(server, &credits, optype); + return -EAGAIN; + } + + mid = server->ops->setup_async_request(server, rqst); + if (IS_ERR(mid)) { + cifs_server_unlock(server); + add_credits_and_wake_if(server, &credits, optype); + return PTR_ERR(mid); + } + + mid->sr_flags = flags; + mid->receive = receive; + mid->callback = callback; + mid->callback_data = cbdata; + mid->handle = handle; + mid->mid_state = MID_REQUEST_SUBMITTED; + + /* put it on the pending_mid_q */ + spin_lock(&server->mid_queue_lock); + list_add_tail(&mid->qhead, &server->pending_mid_q); + spin_unlock(&server->mid_queue_lock); + + /* + * Need to store the time in mid before calling I/O. For call_async, + * I/O response may come back and free the mid entry on another thread. + */ + cifs_save_when_sent(mid); + rc = smb_send_rqst(server, 1, rqst, flags); + + if (rc < 0) { + revert_current_mid(server, mid->credits); + server->sequence_number -= 2; + delete_mid(server, mid); + } + + cifs_server_unlock(server); + + if (rc == 0) + return 0; + + add_credits_and_wake_if(server, &credits, optype); + return rc; +} + +int cifs_sync_mid_result(struct mid_q_entry *mid, struct TCP_Server_Info *server) +{ + int rc = 0; + + cifs_dbg(FYI, "%s: cmd=%d mid=%llu state=%d\n", + __func__, le16_to_cpu(mid->command), mid->mid, mid->mid_state); + + spin_lock(&server->mid_queue_lock); + switch (mid->mid_state) { + case MID_RESPONSE_READY: + spin_unlock(&server->mid_queue_lock); + return rc; + case MID_RETRY_NEEDED: + rc = -EAGAIN; + break; + case MID_RESPONSE_MALFORMED: + rc = smb_EIO(smb_eio_trace_rx_sync_mid_malformed); + break; + case MID_SHUTDOWN: + rc = -EHOSTDOWN; + break; + case MID_RC: + rc = mid->mid_rc; + break; + default: + if (mid->deleted_from_q == false) { + list_del_init(&mid->qhead); + mid->deleted_from_q = true; + } + spin_unlock(&server->mid_queue_lock); + cifs_server_dbg(VFS, "%s: invalid mid state mid=%llu state=%d\n", + __func__, mid->mid, mid->mid_state); + rc = smb_EIO1(smb_eio_trace_rx_sync_mid_invalid, mid->mid_state); + goto sync_mid_done; + } + spin_unlock(&server->mid_queue_lock); + +sync_mid_done: + release_mid(server, mid); + return rc; +} + +static void +cifs_compound_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid) +{ + struct cifs_credits credits = { + .value = server->ops->get_credits(mid), + .instance = server->reconnect_instance, + }; + + add_credits(server, &credits, mid->optype); + + if (mid->mid_state == MID_RESPONSE_RECEIVED) + mid->mid_state = MID_RESPONSE_READY; +} + +static void +cifs_compound_last_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid) +{ + cifs_compound_callback(server, mid); + cifs_wake_up_task(server, mid); +} + +static void +cifs_cancelled_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid) +{ + cifs_compound_callback(server, mid); + release_mid(server, mid); +} + +/* + * cifs_pick_channel - pick an eligible channel for network operations + * + * @ses: session reference + * + * Select an eligible channel (not terminating and not marked as needing + * reconnect), preferring the least loaded one. If no eligible channel is + * found, fall back to the primary channel (index 0). + * + * Return: TCP_Server_Info pointer for the chosen channel, or NULL if @ses is + * NULL. + */ +struct TCP_Server_Info *cifs_pick_channel(struct cifs_ses *ses) +{ + uint index = 0; + unsigned int min_in_flight = UINT_MAX; + struct TCP_Server_Info *server = NULL; + int i, start, cur; + + if (!ses) + return NULL; + + spin_lock(&ses->chan_lock); + start = atomic_inc_return(&ses->chan_seq); + for (i = 0; i < ses->chan_count; i++) { + cur = (start + i) % ses->chan_count; + server = ses->chans[cur].server; + if (!server || server->terminate) + continue; + + if (CIFS_CHAN_NEEDS_RECONNECT(ses, cur)) + continue; + + /* + * strictly speaking, we should pick up req_lock to read + * server->in_flight. But it shouldn't matter much here if we + * race while reading this data. The worst that can happen is + * that we could use a channel that's not least loaded. Avoiding + * taking the lock could help reduce wait time, which is + * important for this function + */ + if (server->in_flight < min_in_flight) { + min_in_flight = server->in_flight; + index = cur; + } + } + + server = ses->chans[index].server; + spin_unlock(&ses->chan_lock); + + return server; +} + +int +compound_send_recv(const unsigned int xid, struct cifs_ses *ses, + struct TCP_Server_Info *server, + const int flags, const int num_rqst, struct smb_rqst *rqst, + int *resp_buf_type, struct kvec *resp_iov) +{ + int i, j, optype, rc = 0; + struct mid_q_entry *mid[MAX_COMPOUND]; + bool cancelled_mid[MAX_COMPOUND] = {false}; + struct cifs_credits credits[MAX_COMPOUND] = { + { .value = 0, .instance = 0 } + }; + unsigned int instance; + char *buf; + + optype = flags & CIFS_OP_MASK; + + for (i = 0; i < num_rqst; i++) + resp_buf_type[i] = CIFS_NO_BUFFER; /* no response buf yet */ + + if (!ses || !ses->server || !server) { + cifs_dbg(VFS, "Null session\n"); + return smb_EIO(smb_eio_trace_null_pointers); + } + + spin_lock(&server->srv_lock); + if (server->tcpStatus == CifsExiting) { + spin_unlock(&server->srv_lock); + return -ENOENT; + } + spin_unlock(&server->srv_lock); + + /* + * Wait for all the requests to become available. + * This approach still leaves the possibility to be stuck waiting for + * credits if the server doesn't grant credits to the outstanding + * requests and if the client is completely idle, not generating any + * other requests. + * This can be handled by the eventual session reconnect. + */ + rc = wait_for_compound_request(server, num_rqst, flags, + &instance); + if (rc) + return rc; + + for (i = 0; i < num_rqst; i++) { + credits[i].value = 1; + credits[i].instance = instance; + } + + /* + * Make sure that we sign in the same order that we send on this socket + * and avoid races inside tcp sendmsg code that could cause corruption + * of smb data. + */ + + cifs_server_lock(server); + + /* + * All the parts of the compound chain belong obtained credits from the + * same session. We can not use credits obtained from the previous + * session to send this request. Check if there were reconnects after + * we obtained credits and return -EAGAIN in such cases to let callers + * handle it. + */ + if (instance != server->reconnect_instance) { + cifs_server_unlock(server); + for (j = 0; j < num_rqst; j++) + add_credits(server, &credits[j], optype); + return -EAGAIN; + } + + for (i = 0; i < num_rqst; i++) { + mid[i] = server->ops->setup_request(ses, server, &rqst[i]); + if (IS_ERR(mid[i])) { + revert_current_mid(server, i); + for (j = 0; j < i; j++) + delete_mid(server, mid[j]); + cifs_server_unlock(server); + + /* Update # of requests on wire to server */ + for (j = 0; j < num_rqst; j++) + add_credits(server, &credits[j], optype); + return PTR_ERR(mid[i]); + } + + mid[i]->sr_flags = flags; + mid[i]->mid_state = MID_REQUEST_SUBMITTED; + mid[i]->optype = optype; + /* + * Invoke callback for every part of the compound chain + * to calculate credits properly. Wake up this thread only when + * the last element is received. + */ + if (i < num_rqst - 1) + mid[i]->callback = cifs_compound_callback; + else + mid[i]->callback = cifs_compound_last_callback; + } + rc = smb_send_rqst(server, num_rqst, rqst, flags); + + for (i = 0; i < num_rqst; i++) + cifs_save_when_sent(mid[i]); + + if (rc < 0) { + revert_current_mid(server, num_rqst); + server->sequence_number -= 2; + } + + cifs_server_unlock(server); + + /* + * If sending failed for some reason or it is an oplock break that we + * will not receive a response to - return credits back + */ + if (rc < 0 || (flags & CIFS_NO_SRV_RSP)) { + for (i = 0; i < num_rqst; i++) + add_credits(server, &credits[i], optype); + goto out; + } + + /* + * At this point the request is passed to the network stack - we assume + * that any credits taken from the server structure on the client have + * been spent and we can't return them back. Once we receive responses + * we will collect credits granted by the server in the mid callbacks + * and add those credits to the server structure. + */ + + /* + * Compounding is never used during session establish. + */ + spin_lock(&ses->ses_lock); + if ((ses->ses_status == SES_NEW) || (optype & CIFS_NEG_OP) || (optype & CIFS_SESS_OP)) { + spin_unlock(&ses->ses_lock); + + if (WARN_ON_ONCE(num_rqst != 1 || !resp_iov)) + return -EINVAL; + + cifs_server_lock(server); + smb311_update_preauth_hash(ses, server, rqst[0].rq_iov, rqst[0].rq_nvec); + cifs_server_unlock(server); + + spin_lock(&ses->ses_lock); + } + spin_unlock(&ses->ses_lock); + + for (i = 0; i < num_rqst; i++) { + rc = wait_for_response(server, mid[i]); + if (rc != 0) + break; + } + if (rc != 0) { + for (; i < num_rqst; i++) { + cifs_server_dbg(FYI, "Cancelling wait for mid %llu cmd: %d\n", + mid[i]->mid, le16_to_cpu(mid[i]->command)); + send_cancel(ses, server, &rqst[i], mid[i], xid); + spin_lock(&mid[i]->mid_lock); + mid[i]->wait_cancelled = true; + if (mid[i]->mid_state == MID_REQUEST_SUBMITTED || + mid[i]->mid_state == MID_RESPONSE_RECEIVED) { + mid[i]->callback = cifs_cancelled_callback; + cancelled_mid[i] = true; + credits[i].value = 0; + } + spin_unlock(&mid[i]->mid_lock); + } + } + + for (i = 0; i < num_rqst; i++) { + if (rc < 0) + goto out; + + rc = cifs_sync_mid_result(mid[i], server); + if (rc != 0) { + /* mark this mid as cancelled to not free it below */ + cancelled_mid[i] = true; + goto out; + } + + if (!mid[i]->resp_buf || + mid[i]->mid_state != MID_RESPONSE_READY) { + rc = smb_EIO1(smb_eio_trace_rx_mid_unready, mid[i]->mid_state); + cifs_dbg(FYI, "Bad MID state?\n"); + goto out; + } + + rc = server->ops->check_receive(mid[i], server, + flags & CIFS_LOG_ERROR); + + if (resp_iov) { + buf = (char *)mid[i]->resp_buf; + resp_iov[i].iov_base = buf; + resp_iov[i].iov_len = mid[i]->resp_buf_size; + + if (mid[i]->large_buf) + resp_buf_type[i] = CIFS_LARGE_BUFFER; + else + resp_buf_type[i] = CIFS_SMALL_BUFFER; + + /* mark it so buf will not be freed by delete_mid */ + if ((flags & CIFS_NO_RSP_BUF) == 0) + mid[i]->resp_buf = NULL; + } + } + + /* + * Compounding is never used during session establish. + */ + spin_lock(&ses->ses_lock); + if ((ses->ses_status == SES_NEW) || (optype & CIFS_NEG_OP) || (optype & CIFS_SESS_OP)) { + struct kvec iov = { + .iov_base = resp_iov[0].iov_base, + .iov_len = resp_iov[0].iov_len + }; + spin_unlock(&ses->ses_lock); + cifs_server_lock(server); + smb311_update_preauth_hash(ses, server, &iov, 1); + cifs_server_unlock(server); + spin_lock(&ses->ses_lock); + } + spin_unlock(&ses->ses_lock); + +out: + /* + * This will dequeue all mids. After this it is important that the + * demultiplex_thread will not process any of these mids any further. + * This is prevented above by using a noop callback that will not + * wake this thread except for the very last PDU. + */ + for (i = 0; i < num_rqst; i++) { + if (!cancelled_mid[i]) + delete_mid(server, mid[i]); + } + + return rc; +} + +int +cifs_send_recv(const unsigned int xid, struct cifs_ses *ses, + struct TCP_Server_Info *server, + struct smb_rqst *rqst, int *resp_buf_type, const int flags, + struct kvec *resp_iov) +{ + return compound_send_recv(xid, ses, server, flags, 1, + rqst, resp_buf_type, resp_iov); +} + + +/* + * Discard any remaining data in the current SMB. To do this, we borrow the + * current bigbuf. + */ +int +cifs_discard_remaining_data(struct TCP_Server_Info *server) +{ + unsigned int rfclen = server->pdu_size; + size_t remaining = rfclen - server->total_read; + + while (remaining > 0) { + ssize_t length; + + length = cifs_discard_from_socket(server, + min_t(size_t, remaining, + CIFSMaxBufSize + MAX_HEADER_SIZE(server))); + if (length < 0) + return length; + server->total_read += length; + remaining -= length; + } + + return 0; +} + +static int +__cifs_readv_discard(struct TCP_Server_Info *server, struct mid_q_entry *mid, + bool malformed) +{ + int length; + + length = cifs_discard_remaining_data(server); + dequeue_mid(server, mid, malformed); + mid->resp_buf = server->smallbuf; + server->smallbuf = NULL; + return length; +} + +static int +cifs_readv_discard(struct TCP_Server_Info *server, struct mid_q_entry *mid) +{ + struct cifs_io_subrequest *rdata = mid->callback_data; + + return __cifs_readv_discard(server, mid, rdata->result); +} + +int +cifs_readv_receive(struct TCP_Server_Info *server, struct mid_q_entry *mid) +{ + int length, len; + unsigned int data_offset, data_len, end_off; + struct cifs_io_subrequest *rdata = mid->callback_data; + char *buf = server->smallbuf; + unsigned int buflen = server->pdu_size; + bool use_rdma_mr = false; + + cifs_dbg(FYI, "%s: mid=%llu offset=%llu bytes=%zu\n", + __func__, mid->mid, rdata->subreq.start, rdata->subreq.len); + + /* + * read the rest of READ_RSP header (sans Data array), or whatever we + * can if there's not enough data. At this point, we've read down to + * the Mid. + */ + len = min_t(unsigned int, buflen, server->vals->read_rsp_size) - + HEADER_SIZE(server) + 1; + + length = cifs_read_from_socket(server, + buf + HEADER_SIZE(server) - 1, len); + if (length < 0) + return length; + server->total_read += length; + + if (server->ops->is_session_expired && + server->ops->is_session_expired(buf)) { + cifs_reconnect(server, true); + return -1; + } + + if (server->ops->is_status_pending && + server->ops->is_status_pending(buf, server)) { + cifs_discard_remaining_data(server); + return -1; + } + + /* set up first two iov for signature check and to get credits */ + rdata->iov[0].iov_base = buf; + rdata->iov[0].iov_len = server->total_read; + cifs_dbg(FYI, "0: iov_base=%p iov_len=%zu\n", + rdata->iov[0].iov_base, rdata->iov[0].iov_len); + + /* Was the SMB read successful? */ + rdata->result = server->ops->map_error(buf, false); + if (rdata->result != 0) { + cifs_dbg(FYI, "%s: server returned error %d\n", + __func__, rdata->result); + /* normal error on read response */ + return __cifs_readv_discard(server, mid, false); + } + + /* Is there enough to get to the rest of the READ_RSP header? */ + if (server->total_read < server->vals->read_rsp_size) { + cifs_dbg(FYI, "%s: server returned short header. got=%u expected=%zu\n", + __func__, server->total_read, + server->vals->read_rsp_size); + rdata->result = smb_EIO2(smb_eio_trace_read_rsp_short, + server->total_read, server->vals->read_rsp_size); + return cifs_readv_discard(server, mid); + } + + data_offset = server->ops->read_data_offset(buf); + if (data_offset < server->total_read) { + /* + * win2k8 sometimes sends an offset of 0 when the read + * is beyond the EOF. Treat it as if the data starts just after + * the header. + */ + cifs_dbg(FYI, "%s: data offset (%u) inside read response header\n", + __func__, data_offset); + data_offset = server->total_read; + } else if (data_offset > MAX_CIFS_SMALL_BUFFER_SIZE) { + /* data_offset is beyond the end of smallbuf */ + cifs_dbg(FYI, "%s: data offset (%u) beyond end of smallbuf\n", + __func__, data_offset); + rdata->result = smb_EIO1(smb_eio_trace_read_overlarge, + data_offset); + return cifs_readv_discard(server, mid); + } + + cifs_dbg(FYI, "%s: total_read=%u data_offset=%u\n", + __func__, server->total_read, data_offset); + + len = data_offset - server->total_read; + if (len > 0) { + /* read any junk before data into the rest of smallbuf */ + length = cifs_read_from_socket(server, + buf + server->total_read, len); + if (length < 0) + return length; + server->total_read += length; + rdata->iov[0].iov_len = server->total_read; + } + + /* how much data is in the response? */ +#ifdef CONFIG_CIFS_SMB_DIRECT + use_rdma_mr = rdata->mr; +#endif + data_len = server->ops->read_data_length(buf, use_rdma_mr); + if (!use_rdma_mr) { + if (check_add_overflow(data_offset, data_len, &end_off) || + end_off > buflen) { + /* data_len is corrupt -- discard frame */ + rdata->result = smb_EIO2(smb_eio_trace_read_rsp_malformed, + end_off, buflen); + return cifs_readv_discard(server, mid); + } + } + +#ifdef CONFIG_CIFS_SMB_DIRECT + if (rdata->mr) + length = data_len; /* An RDMA read is already done. */ + else +#endif + length = cifs_read_iter_from_socket(server, &rdata->subreq.io_iter, + data_len); + if (length > 0) + rdata->got_bytes += length; + server->total_read += length; + + cifs_dbg(FYI, "total_read=%u buflen=%u remaining=%u\n", + server->total_read, buflen, data_len); + + /* discard anything left over */ + if (server->total_read < buflen) + return cifs_readv_discard(server, mid); + + dequeue_mid(server, mid, false); + mid->resp_buf = server->smallbuf; + server->smallbuf = NULL; + return length; +} diff --git a/fs/smb/client/unc.c b/fs/smb/client/unc.c new file mode 100644 index 000000000..f6fc5e343 --- /dev/null +++ b/fs/smb/client/unc.c @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2020, Microsoft Corporation. + * + * Author(s): Steve French + * Suresh Jayaraman + * Jeff Layton + */ + +#include +#include +#include +#include +#include "cifsglob.h" +#include "cifsproto.h" + +/* extract the host portion of the UNC string */ +char *extract_hostname(const char *unc) +{ + const char *src; + char *dst, *delim; + unsigned int len; + + /* skip double chars at beginning of string */ + /* BB: check validity of these bytes? */ + if (strlen(unc) < 3) + return ERR_PTR(-EINVAL); + for (src = unc; *src && *src == '\\'; src++) + ; + if (!*src) + return ERR_PTR(-EINVAL); + + /* delimiter between hostname and sharename is always '\\' now */ + delim = strchr(src, '\\'); + if (!delim) + return ERR_PTR(-EINVAL); + + len = delim - src; + dst = kmalloc((len + 1), GFP_KERNEL); + if (dst == NULL) + return ERR_PTR(-ENOMEM); + + memcpy(dst, src, len); + dst[len] = '\0'; + + return dst; +} + +char *extract_sharename(const char *unc) +{ + const char *src; + char *delim, *dst; + + /* skip double chars at the beginning */ + src = unc + 2; + + /* share name is always preceded by '\\' now */ + delim = strchr(src, '\\'); + if (!delim) + return ERR_PTR(-EINVAL); + delim++; + + /* caller has to free the memory */ + dst = kstrdup(delim, GFP_KERNEL); + if (!dst) + return ERR_PTR(-ENOMEM); + + return dst; +} diff --git a/fs/smb/client/winucase.c b/fs/smb/client/winucase.c new file mode 100644 index 000000000..2f075b5b5 --- /dev/null +++ b/fs/smb/client/winucase.c @@ -0,0 +1,649 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * + * Copyright (c) Jeffrey Layton , 2013 + * + * The const tables in this file were converted from the following info + * provided by Microsoft: + * + * 3.1.5.3 Mapping UTF-16 Strings to Upper Case: + * + * https://msdn.microsoft.com/en-us/library/hh877830.aspx + * http://www.microsoft.com/en-us/download/details.aspx?displaylang=en&id=10921 + * + * In particular, the table in "Windows 8 Upper Case Mapping Table.txt" was + * post-processed using the winucase_convert.pl script. + */ + +#include + +wchar_t cifs_toupper(wchar_t in); /* quiet sparse */ + +static const wchar_t t2_00[256] = { + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, + 0x0048, 0x0049, 0x004a, 0x004b, 0x004c, 0x004d, 0x004e, 0x004f, + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, + 0x0058, 0x0059, 0x005a, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00c6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, + 0x00d0, 0x00d1, 0x00d2, 0x00d3, 0x00d4, 0x00d5, 0x00d6, 0x0000, + 0x00d8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00de, 0x0178, +}; + +static const wchar_t t2_01[256] = { + 0x0000, 0x0100, 0x0000, 0x0102, 0x0000, 0x0104, 0x0000, 0x0106, + 0x0000, 0x0108, 0x0000, 0x010a, 0x0000, 0x010c, 0x0000, 0x010e, + 0x0000, 0x0110, 0x0000, 0x0112, 0x0000, 0x0114, 0x0000, 0x0116, + 0x0000, 0x0118, 0x0000, 0x011a, 0x0000, 0x011c, 0x0000, 0x011e, + 0x0000, 0x0120, 0x0000, 0x0122, 0x0000, 0x0124, 0x0000, 0x0126, + 0x0000, 0x0128, 0x0000, 0x012a, 0x0000, 0x012c, 0x0000, 0x012e, + 0x0000, 0x0000, 0x0000, 0x0132, 0x0000, 0x0134, 0x0000, 0x0136, + 0x0000, 0x0000, 0x0139, 0x0000, 0x013b, 0x0000, 0x013d, 0x0000, + 0x013f, 0x0000, 0x0141, 0x0000, 0x0143, 0x0000, 0x0145, 0x0000, + 0x0147, 0x0000, 0x0000, 0x014a, 0x0000, 0x014c, 0x0000, 0x014e, + 0x0000, 0x0150, 0x0000, 0x0152, 0x0000, 0x0154, 0x0000, 0x0156, + 0x0000, 0x0158, 0x0000, 0x015a, 0x0000, 0x015c, 0x0000, 0x015e, + 0x0000, 0x0160, 0x0000, 0x0162, 0x0000, 0x0164, 0x0000, 0x0166, + 0x0000, 0x0168, 0x0000, 0x016a, 0x0000, 0x016c, 0x0000, 0x016e, + 0x0000, 0x0170, 0x0000, 0x0172, 0x0000, 0x0174, 0x0000, 0x0176, + 0x0000, 0x0000, 0x0179, 0x0000, 0x017b, 0x0000, 0x017d, 0x0000, + 0x0243, 0x0000, 0x0000, 0x0182, 0x0000, 0x0184, 0x0000, 0x0000, + 0x0187, 0x0000, 0x0000, 0x0000, 0x018b, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0191, 0x0000, 0x0000, 0x01f6, 0x0000, 0x0000, + 0x0000, 0x0198, 0x023d, 0x0000, 0x0000, 0x0000, 0x0220, 0x0000, + 0x0000, 0x01a0, 0x0000, 0x01a2, 0x0000, 0x01a4, 0x0000, 0x0000, + 0x01a7, 0x0000, 0x0000, 0x0000, 0x0000, 0x01ac, 0x0000, 0x0000, + 0x01af, 0x0000, 0x0000, 0x0000, 0x01b3, 0x0000, 0x01b5, 0x0000, + 0x0000, 0x01b8, 0x0000, 0x0000, 0x0000, 0x01bc, 0x0000, 0x01f7, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x01c4, 0x0000, + 0x0000, 0x01c7, 0x0000, 0x0000, 0x01ca, 0x0000, 0x01cd, 0x0000, + 0x01cf, 0x0000, 0x01d1, 0x0000, 0x01d3, 0x0000, 0x01d5, 0x0000, + 0x01d7, 0x0000, 0x01d9, 0x0000, 0x01db, 0x018e, 0x0000, 0x01de, + 0x0000, 0x01e0, 0x0000, 0x01e2, 0x0000, 0x01e4, 0x0000, 0x01e6, + 0x0000, 0x01e8, 0x0000, 0x01ea, 0x0000, 0x01ec, 0x0000, 0x01ee, + 0x0000, 0x0000, 0x0000, 0x01f1, 0x0000, 0x01f4, 0x0000, 0x0000, + 0x0000, 0x01f8, 0x0000, 0x01fa, 0x0000, 0x01fc, 0x0000, 0x01fe, +}; + +static const wchar_t t2_02[256] = { + 0x0000, 0x0200, 0x0000, 0x0202, 0x0000, 0x0204, 0x0000, 0x0206, + 0x0000, 0x0208, 0x0000, 0x020a, 0x0000, 0x020c, 0x0000, 0x020e, + 0x0000, 0x0210, 0x0000, 0x0212, 0x0000, 0x0214, 0x0000, 0x0216, + 0x0000, 0x0218, 0x0000, 0x021a, 0x0000, 0x021c, 0x0000, 0x021e, + 0x0000, 0x0000, 0x0000, 0x0222, 0x0000, 0x0224, 0x0000, 0x0226, + 0x0000, 0x0228, 0x0000, 0x022a, 0x0000, 0x022c, 0x0000, 0x022e, + 0x0000, 0x0230, 0x0000, 0x0232, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x023b, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0241, 0x0000, 0x0000, 0x0000, 0x0000, 0x0246, + 0x0000, 0x0248, 0x0000, 0x024a, 0x0000, 0x024c, 0x0000, 0x024e, + 0x2c6f, 0x2c6d, 0x0000, 0x0181, 0x0186, 0x0000, 0x0189, 0x018a, + 0x0000, 0x018f, 0x0000, 0x0190, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0193, 0x0000, 0x0000, 0x0194, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0197, 0x0196, 0x0000, 0x2c62, 0x0000, 0x0000, 0x0000, 0x019c, + 0x0000, 0x2c6e, 0x019d, 0x0000, 0x0000, 0x019f, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x2c64, 0x0000, 0x0000, + 0x01a6, 0x0000, 0x0000, 0x01a9, 0x0000, 0x0000, 0x0000, 0x0000, + 0x01ae, 0x0244, 0x01b1, 0x01b2, 0x0245, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x01b7, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, +}; + +static const wchar_t t2_03[256] = { + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0370, 0x0000, 0x0372, 0x0000, 0x0000, 0x0000, 0x0376, + 0x0000, 0x0000, 0x0000, 0x03fd, 0x03fe, 0x03ff, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0386, 0x0388, 0x0389, 0x038a, + 0x0000, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, + 0x0398, 0x0399, 0x039a, 0x039b, 0x039c, 0x039d, 0x039e, 0x039f, + 0x03a0, 0x03a1, 0x0000, 0x03a3, 0x03a4, 0x03a5, 0x03a6, 0x03a7, + 0x03a8, 0x03a9, 0x03aa, 0x03ab, 0x038c, 0x038e, 0x038f, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x03cf, + 0x0000, 0x03d8, 0x0000, 0x03da, 0x0000, 0x03dc, 0x0000, 0x03de, + 0x0000, 0x03e0, 0x0000, 0x03e2, 0x0000, 0x03e4, 0x0000, 0x03e6, + 0x0000, 0x03e8, 0x0000, 0x03ea, 0x0000, 0x03ec, 0x0000, 0x03ee, + 0x0000, 0x0000, 0x03f9, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x03f7, 0x0000, 0x0000, 0x03fa, 0x0000, 0x0000, 0x0000, 0x0000, +}; + +static const wchar_t t2_04[256] = { + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, + 0x0418, 0x0419, 0x041a, 0x041b, 0x041c, 0x041d, 0x041e, 0x041f, + 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, + 0x0428, 0x0429, 0x042a, 0x042b, 0x042c, 0x042d, 0x042e, 0x042f, + 0x0400, 0x0401, 0x0402, 0x0403, 0x0404, 0x0405, 0x0406, 0x0407, + 0x0408, 0x0409, 0x040a, 0x040b, 0x040c, 0x040d, 0x040e, 0x040f, + 0x0000, 0x0460, 0x0000, 0x0462, 0x0000, 0x0464, 0x0000, 0x0466, + 0x0000, 0x0468, 0x0000, 0x046a, 0x0000, 0x046c, 0x0000, 0x046e, + 0x0000, 0x0470, 0x0000, 0x0472, 0x0000, 0x0474, 0x0000, 0x0476, + 0x0000, 0x0478, 0x0000, 0x047a, 0x0000, 0x047c, 0x0000, 0x047e, + 0x0000, 0x0480, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x048a, 0x0000, 0x048c, 0x0000, 0x048e, + 0x0000, 0x0490, 0x0000, 0x0492, 0x0000, 0x0494, 0x0000, 0x0496, + 0x0000, 0x0498, 0x0000, 0x049a, 0x0000, 0x049c, 0x0000, 0x049e, + 0x0000, 0x04a0, 0x0000, 0x04a2, 0x0000, 0x04a4, 0x0000, 0x04a6, + 0x0000, 0x04a8, 0x0000, 0x04aa, 0x0000, 0x04ac, 0x0000, 0x04ae, + 0x0000, 0x04b0, 0x0000, 0x04b2, 0x0000, 0x04b4, 0x0000, 0x04b6, + 0x0000, 0x04b8, 0x0000, 0x04ba, 0x0000, 0x04bc, 0x0000, 0x04be, + 0x0000, 0x0000, 0x04c1, 0x0000, 0x04c3, 0x0000, 0x04c5, 0x0000, + 0x04c7, 0x0000, 0x04c9, 0x0000, 0x04cb, 0x0000, 0x04cd, 0x04c0, + 0x0000, 0x04d0, 0x0000, 0x04d2, 0x0000, 0x04d4, 0x0000, 0x04d6, + 0x0000, 0x04d8, 0x0000, 0x04da, 0x0000, 0x04dc, 0x0000, 0x04de, + 0x0000, 0x04e0, 0x0000, 0x04e2, 0x0000, 0x04e4, 0x0000, 0x04e6, + 0x0000, 0x04e8, 0x0000, 0x04ea, 0x0000, 0x04ec, 0x0000, 0x04ee, + 0x0000, 0x04f0, 0x0000, 0x04f2, 0x0000, 0x04f4, 0x0000, 0x04f6, + 0x0000, 0x04f8, 0x0000, 0x04fa, 0x0000, 0x04fc, 0x0000, 0x04fe, +}; + +static const wchar_t t2_05[256] = { + 0x0000, 0x0500, 0x0000, 0x0502, 0x0000, 0x0504, 0x0000, 0x0506, + 0x0000, 0x0508, 0x0000, 0x050a, 0x0000, 0x050c, 0x0000, 0x050e, + 0x0000, 0x0510, 0x0000, 0x0512, 0x0000, 0x0514, 0x0000, 0x0516, + 0x0000, 0x0518, 0x0000, 0x051a, 0x0000, 0x051c, 0x0000, 0x051e, + 0x0000, 0x0520, 0x0000, 0x0522, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0531, 0x0532, 0x0533, 0x0534, 0x0535, 0x0536, 0x0537, + 0x0538, 0x0539, 0x053a, 0x053b, 0x053c, 0x053d, 0x053e, 0x053f, + 0x0540, 0x0541, 0x0542, 0x0543, 0x0544, 0x0545, 0x0546, 0x0547, + 0x0548, 0x0549, 0x054a, 0x054b, 0x054c, 0x054d, 0x054e, 0x054f, + 0x0550, 0x0551, 0x0552, 0x0553, 0x0554, 0x0555, 0x0556, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, +}; + +static const wchar_t t2_1d[256] = { + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0xa77d, 0x0000, 0x0000, 0x0000, 0x2c63, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, +}; + +static const wchar_t t2_1e[256] = { + 0x0000, 0x1e00, 0x0000, 0x1e02, 0x0000, 0x1e04, 0x0000, 0x1e06, + 0x0000, 0x1e08, 0x0000, 0x1e0a, 0x0000, 0x1e0c, 0x0000, 0x1e0e, + 0x0000, 0x1e10, 0x0000, 0x1e12, 0x0000, 0x1e14, 0x0000, 0x1e16, + 0x0000, 0x1e18, 0x0000, 0x1e1a, 0x0000, 0x1e1c, 0x0000, 0x1e1e, + 0x0000, 0x1e20, 0x0000, 0x1e22, 0x0000, 0x1e24, 0x0000, 0x1e26, + 0x0000, 0x1e28, 0x0000, 0x1e2a, 0x0000, 0x1e2c, 0x0000, 0x1e2e, + 0x0000, 0x1e30, 0x0000, 0x1e32, 0x0000, 0x1e34, 0x0000, 0x1e36, + 0x0000, 0x1e38, 0x0000, 0x1e3a, 0x0000, 0x1e3c, 0x0000, 0x1e3e, + 0x0000, 0x1e40, 0x0000, 0x1e42, 0x0000, 0x1e44, 0x0000, 0x1e46, + 0x0000, 0x1e48, 0x0000, 0x1e4a, 0x0000, 0x1e4c, 0x0000, 0x1e4e, + 0x0000, 0x1e50, 0x0000, 0x1e52, 0x0000, 0x1e54, 0x0000, 0x1e56, + 0x0000, 0x1e58, 0x0000, 0x1e5a, 0x0000, 0x1e5c, 0x0000, 0x1e5e, + 0x0000, 0x1e60, 0x0000, 0x1e62, 0x0000, 0x1e64, 0x0000, 0x1e66, + 0x0000, 0x1e68, 0x0000, 0x1e6a, 0x0000, 0x1e6c, 0x0000, 0x1e6e, + 0x0000, 0x1e70, 0x0000, 0x1e72, 0x0000, 0x1e74, 0x0000, 0x1e76, + 0x0000, 0x1e78, 0x0000, 0x1e7a, 0x0000, 0x1e7c, 0x0000, 0x1e7e, + 0x0000, 0x1e80, 0x0000, 0x1e82, 0x0000, 0x1e84, 0x0000, 0x1e86, + 0x0000, 0x1e88, 0x0000, 0x1e8a, 0x0000, 0x1e8c, 0x0000, 0x1e8e, + 0x0000, 0x1e90, 0x0000, 0x1e92, 0x0000, 0x1e94, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x1ea0, 0x0000, 0x1ea2, 0x0000, 0x1ea4, 0x0000, 0x1ea6, + 0x0000, 0x1ea8, 0x0000, 0x1eaa, 0x0000, 0x1eac, 0x0000, 0x1eae, + 0x0000, 0x1eb0, 0x0000, 0x1eb2, 0x0000, 0x1eb4, 0x0000, 0x1eb6, + 0x0000, 0x1eb8, 0x0000, 0x1eba, 0x0000, 0x1ebc, 0x0000, 0x1ebe, + 0x0000, 0x1ec0, 0x0000, 0x1ec2, 0x0000, 0x1ec4, 0x0000, 0x1ec6, + 0x0000, 0x1ec8, 0x0000, 0x1eca, 0x0000, 0x1ecc, 0x0000, 0x1ece, + 0x0000, 0x1ed0, 0x0000, 0x1ed2, 0x0000, 0x1ed4, 0x0000, 0x1ed6, + 0x0000, 0x1ed8, 0x0000, 0x1eda, 0x0000, 0x1edc, 0x0000, 0x1ede, + 0x0000, 0x1ee0, 0x0000, 0x1ee2, 0x0000, 0x1ee4, 0x0000, 0x1ee6, + 0x0000, 0x1ee8, 0x0000, 0x1eea, 0x0000, 0x1eec, 0x0000, 0x1eee, + 0x0000, 0x1ef0, 0x0000, 0x1ef2, 0x0000, 0x1ef4, 0x0000, 0x1ef6, + 0x0000, 0x1ef8, 0x0000, 0x1efa, 0x0000, 0x1efc, 0x0000, 0x1efe, +}; + +static const wchar_t t2_1f[256] = { + 0x1f08, 0x1f09, 0x1f0a, 0x1f0b, 0x1f0c, 0x1f0d, 0x1f0e, 0x1f0f, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x1f18, 0x1f19, 0x1f1a, 0x1f1b, 0x1f1c, 0x1f1d, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x1f28, 0x1f29, 0x1f2a, 0x1f2b, 0x1f2c, 0x1f2d, 0x1f2e, 0x1f2f, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x1f38, 0x1f39, 0x1f3a, 0x1f3b, 0x1f3c, 0x1f3d, 0x1f3e, 0x1f3f, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x1f48, 0x1f49, 0x1f4a, 0x1f4b, 0x1f4c, 0x1f4d, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x1f59, 0x0000, 0x1f5b, 0x0000, 0x1f5d, 0x0000, 0x1f5f, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x1f68, 0x1f69, 0x1f6a, 0x1f6b, 0x1f6c, 0x1f6d, 0x1f6e, 0x1f6f, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x1fba, 0x1fbb, 0x1fc8, 0x1fc9, 0x1fca, 0x1fcb, 0x1fda, 0x1fdb, + 0x1ff8, 0x1ff9, 0x1fea, 0x1feb, 0x1ffa, 0x1ffb, 0x0000, 0x0000, + 0x1f88, 0x1f89, 0x1f8a, 0x1f8b, 0x1f8c, 0x1f8d, 0x1f8e, 0x1f8f, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x1f98, 0x1f99, 0x1f9a, 0x1f9b, 0x1f9c, 0x1f9d, 0x1f9e, 0x1f9f, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x1fa8, 0x1fa9, 0x1faa, 0x1fab, 0x1fac, 0x1fad, 0x1fae, 0x1faf, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x1fb8, 0x1fb9, 0x0000, 0x1fbc, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x1fcc, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x1fd8, 0x1fd9, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x1fe8, 0x1fe9, 0x0000, 0x0000, 0x0000, 0x1fec, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x1ffc, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, +}; + +static const wchar_t t2_21[256] = { + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x2132, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x2160, 0x2161, 0x2162, 0x2163, 0x2164, 0x2165, 0x2166, 0x2167, + 0x2168, 0x2169, 0x216a, 0x216b, 0x216c, 0x216d, 0x216e, 0x216f, + 0x0000, 0x0000, 0x0000, 0x0000, 0x2183, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, +}; + +static const wchar_t t2_24[256] = { + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x24b6, 0x24b7, 0x24b8, 0x24b9, 0x24ba, 0x24bb, 0x24bc, 0x24bd, + 0x24be, 0x24bf, 0x24c0, 0x24c1, 0x24c2, 0x24c3, 0x24c4, 0x24c5, + 0x24c6, 0x24c7, 0x24c8, 0x24c9, 0x24ca, 0x24cb, 0x24cc, 0x24cd, + 0x24ce, 0x24cf, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, +}; + +static const wchar_t t2_2c[256] = { + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x2c00, 0x2c01, 0x2c02, 0x2c03, 0x2c04, 0x2c05, 0x2c06, 0x2c07, + 0x2c08, 0x2c09, 0x2c0a, 0x2c0b, 0x2c0c, 0x2c0d, 0x2c0e, 0x2c0f, + 0x2c10, 0x2c11, 0x2c12, 0x2c13, 0x2c14, 0x2c15, 0x2c16, 0x2c17, + 0x2c18, 0x2c19, 0x2c1a, 0x2c1b, 0x2c1c, 0x2c1d, 0x2c1e, 0x2c1f, + 0x2c20, 0x2c21, 0x2c22, 0x2c23, 0x2c24, 0x2c25, 0x2c26, 0x2c27, + 0x2c28, 0x2c29, 0x2c2a, 0x2c2b, 0x2c2c, 0x2c2d, 0x2c2e, 0x0000, + 0x0000, 0x2c60, 0x0000, 0x0000, 0x0000, 0x023a, 0x023e, 0x0000, + 0x2c67, 0x0000, 0x2c69, 0x0000, 0x2c6b, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x2c72, 0x0000, 0x0000, 0x2c75, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x2c80, 0x0000, 0x2c82, 0x0000, 0x2c84, 0x0000, 0x2c86, + 0x0000, 0x2c88, 0x0000, 0x2c8a, 0x0000, 0x2c8c, 0x0000, 0x2c8e, + 0x0000, 0x2c90, 0x0000, 0x2c92, 0x0000, 0x2c94, 0x0000, 0x2c96, + 0x0000, 0x2c98, 0x0000, 0x2c9a, 0x0000, 0x2c9c, 0x0000, 0x2c9e, + 0x0000, 0x2ca0, 0x0000, 0x2ca2, 0x0000, 0x2ca4, 0x0000, 0x2ca6, + 0x0000, 0x2ca8, 0x0000, 0x2caa, 0x0000, 0x2cac, 0x0000, 0x2cae, + 0x0000, 0x2cb0, 0x0000, 0x2cb2, 0x0000, 0x2cb4, 0x0000, 0x2cb6, + 0x0000, 0x2cb8, 0x0000, 0x2cba, 0x0000, 0x2cbc, 0x0000, 0x2cbe, + 0x0000, 0x2cc0, 0x0000, 0x2cc2, 0x0000, 0x2cc4, 0x0000, 0x2cc6, + 0x0000, 0x2cc8, 0x0000, 0x2cca, 0x0000, 0x2ccc, 0x0000, 0x2cce, + 0x0000, 0x2cd0, 0x0000, 0x2cd2, 0x0000, 0x2cd4, 0x0000, 0x2cd6, + 0x0000, 0x2cd8, 0x0000, 0x2cda, 0x0000, 0x2cdc, 0x0000, 0x2cde, + 0x0000, 0x2ce0, 0x0000, 0x2ce2, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, +}; + +static const wchar_t t2_2d[256] = { + 0x10a0, 0x10a1, 0x10a2, 0x10a3, 0x10a4, 0x10a5, 0x10a6, 0x10a7, + 0x10a8, 0x10a9, 0x10aa, 0x10ab, 0x10ac, 0x10ad, 0x10ae, 0x10af, + 0x10b0, 0x10b1, 0x10b2, 0x10b3, 0x10b4, 0x10b5, 0x10b6, 0x10b7, + 0x10b8, 0x10b9, 0x10ba, 0x10bb, 0x10bc, 0x10bd, 0x10be, 0x10bf, + 0x10c0, 0x10c1, 0x10c2, 0x10c3, 0x10c4, 0x10c5, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, +}; + +static const wchar_t t2_a6[256] = { + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0xa640, 0x0000, 0xa642, 0x0000, 0xa644, 0x0000, 0xa646, + 0x0000, 0xa648, 0x0000, 0xa64a, 0x0000, 0xa64c, 0x0000, 0xa64e, + 0x0000, 0xa650, 0x0000, 0xa652, 0x0000, 0xa654, 0x0000, 0xa656, + 0x0000, 0xa658, 0x0000, 0xa65a, 0x0000, 0xa65c, 0x0000, 0xa65e, + 0x0000, 0x0000, 0x0000, 0xa662, 0x0000, 0xa664, 0x0000, 0xa666, + 0x0000, 0xa668, 0x0000, 0xa66a, 0x0000, 0xa66c, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0xa680, 0x0000, 0xa682, 0x0000, 0xa684, 0x0000, 0xa686, + 0x0000, 0xa688, 0x0000, 0xa68a, 0x0000, 0xa68c, 0x0000, 0xa68e, + 0x0000, 0xa690, 0x0000, 0xa692, 0x0000, 0xa694, 0x0000, 0xa696, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, +}; + +static const wchar_t t2_a7[256] = { + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0xa722, 0x0000, 0xa724, 0x0000, 0xa726, + 0x0000, 0xa728, 0x0000, 0xa72a, 0x0000, 0xa72c, 0x0000, 0xa72e, + 0x0000, 0x0000, 0x0000, 0xa732, 0x0000, 0xa734, 0x0000, 0xa736, + 0x0000, 0xa738, 0x0000, 0xa73a, 0x0000, 0xa73c, 0x0000, 0xa73e, + 0x0000, 0xa740, 0x0000, 0xa742, 0x0000, 0xa744, 0x0000, 0xa746, + 0x0000, 0xa748, 0x0000, 0xa74a, 0x0000, 0xa74c, 0x0000, 0xa74e, + 0x0000, 0xa750, 0x0000, 0xa752, 0x0000, 0xa754, 0x0000, 0xa756, + 0x0000, 0xa758, 0x0000, 0xa75a, 0x0000, 0xa75c, 0x0000, 0xa75e, + 0x0000, 0xa760, 0x0000, 0xa762, 0x0000, 0xa764, 0x0000, 0xa766, + 0x0000, 0xa768, 0x0000, 0xa76a, 0x0000, 0xa76c, 0x0000, 0xa76e, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0xa779, 0x0000, 0xa77b, 0x0000, 0x0000, 0xa77e, + 0x0000, 0xa780, 0x0000, 0xa782, 0x0000, 0xa784, 0x0000, 0xa786, + 0x0000, 0x0000, 0x0000, 0x0000, 0xa78b, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, +}; + +static const wchar_t t2_ff[256] = { + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0xff21, 0xff22, 0xff23, 0xff24, 0xff25, 0xff26, 0xff27, + 0xff28, 0xff29, 0xff2a, 0xff2b, 0xff2c, 0xff2d, 0xff2e, 0xff2f, + 0xff30, 0xff31, 0xff32, 0xff33, 0xff34, 0xff35, 0xff36, 0xff37, + 0xff38, 0xff39, 0xff3a, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, +}; + +static const wchar_t *const toplevel[256] = { + t2_00, t2_01, t2_02, t2_03, t2_04, t2_05, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, t2_1d, t2_1e, t2_1f, + NULL, t2_21, NULL, NULL, t2_24, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, t2_2c, t2_2d, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, t2_a6, t2_a7, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, t2_ff, +}; + +/** + * cifs_toupper - convert a wchar_t from lower to uppercase + * @in: character to convert from lower to uppercase + * + * This function consults the static tables above to convert a wchar_t from + * lower to uppercase. In the event that there is no mapping, the original + * "in" character is returned. + */ +wchar_t +cifs_toupper(wchar_t in) +{ + unsigned char idx; + const wchar_t *tbl; + wchar_t out; + + /* grab upper byte */ + idx = (in & 0xff00) >> 8; + + /* find pointer to 2nd layer table */ + tbl = toplevel[idx]; + if (!tbl) + return in; + + /* grab lower byte */ + idx = in & 0xff; + + /* look up character in table */ + out = tbl[idx]; + if (out) + return out; + + return in; +} diff --git a/fs/smb/client/xattr.c b/fs/smb/client/xattr.c new file mode 100644 index 000000000..5091f6c0d --- /dev/null +++ b/fs/smb/client/xattr.c @@ -0,0 +1,535 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * Copyright (c) International Business Machines Corp., 2003, 2007 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ + +#include +#include +#include +#include +#include "cifsfs.h" +#include "cifsglob.h" +#include "cifsproto.h" +#include "cifs_debug.h" +#include "cifs_fs_sb.h" +#include "cifs_unicode.h" +#include "cifs_ioctl.h" + +#define MAX_EA_VALUE_SIZE CIFSMaxBufSize +#define CIFS_XATTR_CIFS_ACL "system.cifs_acl" /* DACL only */ +#define CIFS_XATTR_CIFS_NTSD "system.cifs_ntsd" /* owner plus DACL */ +#define CIFS_XATTR_CIFS_NTSD_FULL "system.cifs_ntsd_full" /* owner/DACL/SACL */ +#define CIFS_XATTR_ATTRIB "cifs.dosattrib" /* full name: user.cifs.dosattrib */ +#define CIFS_XATTR_CREATETIME "cifs.creationtime" /* user.cifs.creationtime */ +/* + * Although these three are just aliases for the above, need to move away from + * confusing users and using the 20+ year old term 'cifs' when it is no longer + * secure, replaced by SMB2 (then even more highly secure SMB3) many years ago + */ +#define SMB3_XATTR_CIFS_ACL "system.smb3_acl" /* DACL only */ +#define SMB3_XATTR_CIFS_NTSD_SACL "system.smb3_ntsd_sacl" /* SACL only */ +#define SMB3_XATTR_CIFS_NTSD_OWNER "system.smb3_ntsd_owner" /* owner only */ +#define SMB3_XATTR_CIFS_NTSD "system.smb3_ntsd" /* owner plus DACL */ +#define SMB3_XATTR_CIFS_NTSD_FULL "system.smb3_ntsd_full" /* owner/DACL/SACL */ +#define SMB3_XATTR_ATTRIB "smb3.dosattrib" /* full name: user.smb3.dosattrib */ +#define SMB3_XATTR_CREATETIME "smb3.creationtime" /* user.smb3.creationtime */ +/* BB need to add server (Samba e.g) support for security and trusted prefix */ + +enum { XATTR_USER, XATTR_CIFS_ACL, XATTR_ACL_ACCESS, XATTR_ACL_DEFAULT, + XATTR_CIFS_NTSD_SACL, XATTR_CIFS_NTSD_OWNER, + XATTR_CIFS_NTSD, XATTR_CIFS_NTSD_FULL }; + +static int cifs_attrib_set(unsigned int xid, struct cifs_tcon *pTcon, + struct inode *inode, const char *full_path, + const void *value, size_t size) +{ + ssize_t rc = -EOPNOTSUPP; + __u32 *pattrib = (__u32 *)value; + __u32 attrib; + FILE_BASIC_INFO info_buf; + + if ((value == NULL) || (size != sizeof(__u32))) + return -ERANGE; + + memset(&info_buf, 0, sizeof(info_buf)); + attrib = *pattrib; + info_buf.Attributes = cpu_to_le32(attrib); + if (pTcon->ses->server->ops->set_file_info) + rc = pTcon->ses->server->ops->set_file_info(inode, full_path, + &info_buf, xid); + if (rc == 0) + CIFS_I(inode)->cifsAttrs = attrib; + + return rc; +} + +static int cifs_creation_time_set(unsigned int xid, struct cifs_tcon *pTcon, + struct inode *inode, const char *full_path, + const void *value, size_t size) +{ + ssize_t rc = -EOPNOTSUPP; + __u64 *pcreation_time = (__u64 *)value; + __u64 creation_time; + FILE_BASIC_INFO info_buf; + + if ((value == NULL) || (size != sizeof(__u64))) + return -ERANGE; + + memset(&info_buf, 0, sizeof(info_buf)); + creation_time = *pcreation_time; + info_buf.CreationTime = cpu_to_le64(creation_time); + if (pTcon->ses->server->ops->set_file_info) + rc = pTcon->ses->server->ops->set_file_info(inode, full_path, + &info_buf, xid); + if (rc == 0) + CIFS_I(inode)->createtime = creation_time; + + return rc; +} + +static int cifs_xattr_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *dentry, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +{ + int rc = -EOPNOTSUPP; + unsigned int xid; + struct super_block *sb = dentry->d_sb; + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + struct tcon_link *tlink; + struct cifs_tcon *pTcon; + const char *full_path; + void *page; + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) + return PTR_ERR(tlink); + pTcon = tlink_tcon(tlink); + + xid = get_xid(); + page = alloc_dentry_path(); + + full_path = build_path_from_dentry(dentry, page); + if (IS_ERR(full_path)) { + rc = PTR_ERR(full_path); + goto out; + } + /* return dos attributes as pseudo xattr */ + /* return alt name if available as pseudo attr */ + + /* if proc/fs/cifs/streamstoxattr is set then + search server for EAs or streams to + returns as xattrs */ + if (size > MAX_EA_VALUE_SIZE) { + cifs_dbg(FYI, "size of EA value too large\n"); + rc = -EOPNOTSUPP; + goto out; + } + + switch (handler->flags) { + case XATTR_USER: + cifs_dbg(FYI, "%s:setting user xattr %s\n", __func__, name); + if ((strcmp(name, CIFS_XATTR_ATTRIB) == 0) || + (strcmp(name, SMB3_XATTR_ATTRIB) == 0)) { + rc = cifs_attrib_set(xid, pTcon, inode, full_path, + value, size); + if (rc == 0) /* force revalidate of the inode */ + CIFS_I(inode)->time = 0; + break; + } else if ((strcmp(name, CIFS_XATTR_CREATETIME) == 0) || + (strcmp(name, SMB3_XATTR_CREATETIME) == 0)) { + rc = cifs_creation_time_set(xid, pTcon, inode, + full_path, value, size); + if (rc == 0) /* force revalidate of the inode */ + CIFS_I(inode)->time = 0; + break; + } + + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NO_XATTR) + goto out; + + if (pTcon->ses->server->ops->set_EA) { + rc = pTcon->ses->server->ops->set_EA(xid, pTcon, + full_path, name, value, (__u16)size, + cifs_sb->local_nls, cifs_sb); + if (rc == 0) + inode_set_ctime_current(inode); + } + break; + + case XATTR_CIFS_ACL: + case XATTR_CIFS_NTSD_SACL: + case XATTR_CIFS_NTSD_OWNER: + case XATTR_CIFS_NTSD: + case XATTR_CIFS_NTSD_FULL: { + struct smb_ntsd *pacl; + + if (!value) + goto out; + pacl = kmalloc(size, GFP_KERNEL); + if (!pacl) { + rc = -ENOMEM; + } else { + memcpy(pacl, value, size); + if (pTcon->ses->server->ops->set_acl) { + int aclflags = 0; + + switch (handler->flags) { + case XATTR_CIFS_NTSD_FULL: + aclflags = (CIFS_ACL_OWNER | + CIFS_ACL_GROUP | + CIFS_ACL_DACL | + CIFS_ACL_SACL); + break; + case XATTR_CIFS_NTSD: + aclflags = (CIFS_ACL_OWNER | + CIFS_ACL_GROUP | + CIFS_ACL_DACL); + break; + case XATTR_CIFS_NTSD_OWNER: + aclflags = (CIFS_ACL_OWNER | + CIFS_ACL_GROUP); + break; + case XATTR_CIFS_NTSD_SACL: + aclflags = CIFS_ACL_SACL; + break; + case XATTR_CIFS_ACL: + default: + aclflags = CIFS_ACL_DACL; + } + + rc = pTcon->ses->server->ops->set_acl(pacl, + size, inode, full_path, aclflags); + } else { + rc = -EOPNOTSUPP; + } + if (rc == 0) /* force revalidate of the inode */ + CIFS_I(inode)->time = 0; + kfree(pacl); + } + break; + } + } + +out: + free_dentry_path(page); + free_xid(xid); + cifs_put_tlink(tlink); + return rc; +} + +static int cifs_attrib_get(struct dentry *dentry, + struct inode *inode, void *value, + size_t size) +{ + ssize_t rc; + __u32 *pattribute; + + rc = cifs_revalidate_dentry_attr(dentry); + + if (rc) + return rc; + + if ((value == NULL) || (size == 0)) + return sizeof(__u32); + else if (size < sizeof(__u32)) + return -ERANGE; + + /* return dos attributes as pseudo xattr */ + pattribute = (__u32 *)value; + *pattribute = CIFS_I(inode)->cifsAttrs; + + return sizeof(__u32); +} + +static int cifs_creation_time_get(struct dentry *dentry, struct inode *inode, + void *value, size_t size) +{ + ssize_t rc; + __u64 *pcreatetime; + + rc = cifs_revalidate_dentry_attr(dentry); + if (rc) + return rc; + + if ((value == NULL) || (size == 0)) + return sizeof(__u64); + else if (size < sizeof(__u64)) + return -ERANGE; + + /* return dos attributes as pseudo xattr */ + pcreatetime = (__u64 *)value; + *pcreatetime = CIFS_I(inode)->createtime; + return sizeof(__u64); +} + + +static int cifs_xattr_get(const struct xattr_handler *handler, + struct dentry *dentry, struct inode *inode, + const char *name, void *value, size_t size) +{ + int rc = -EOPNOTSUPP; + unsigned int xid; + struct super_block *sb = dentry->d_sb; + struct cifs_sb_info *cifs_sb = CIFS_SB(sb); + struct tcon_link *tlink; + struct cifs_tcon *pTcon; + const char *full_path; + void *page; + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) + return PTR_ERR(tlink); + pTcon = tlink_tcon(tlink); + + xid = get_xid(); + page = alloc_dentry_path(); + + full_path = build_path_from_dentry(dentry, page); + if (IS_ERR(full_path)) { + rc = PTR_ERR(full_path); + goto out; + } + + /* return alt name if available as pseudo attr */ + switch (handler->flags) { + case XATTR_USER: + cifs_dbg(FYI, "%s:querying user xattr %s\n", __func__, name); + if ((strcmp(name, CIFS_XATTR_ATTRIB) == 0) || + (strcmp(name, SMB3_XATTR_ATTRIB) == 0)) { + rc = cifs_attrib_get(dentry, inode, value, size); + break; + } else if ((strcmp(name, CIFS_XATTR_CREATETIME) == 0) || + (strcmp(name, SMB3_XATTR_CREATETIME) == 0)) { + rc = cifs_creation_time_get(dentry, inode, value, size); + break; + } + + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NO_XATTR) + goto out; + + if (pTcon->ses->server->ops->query_all_EAs) + rc = pTcon->ses->server->ops->query_all_EAs(xid, pTcon, + full_path, name, value, size, cifs_sb); + break; + + case XATTR_CIFS_ACL: + case XATTR_CIFS_NTSD_SACL: + case XATTR_CIFS_NTSD_OWNER: + case XATTR_CIFS_NTSD: + case XATTR_CIFS_NTSD_FULL: { + /* + * fetch owner, DACL, and SACL if asked for full descriptor, + * fetch owner and DACL otherwise + */ + u32 acllen, extra_info; + struct smb_ntsd *pacl; + + if (pTcon->ses->server->ops->get_acl == NULL) + goto out; /* rc already EOPNOTSUPP */ + + switch (handler->flags) { + case XATTR_CIFS_NTSD_FULL: + extra_info = OWNER_SECINFO | GROUP_SECINFO | DACL_SECINFO | SACL_SECINFO; + break; + case XATTR_CIFS_NTSD: + extra_info = OWNER_SECINFO | GROUP_SECINFO | DACL_SECINFO; + break; + case XATTR_CIFS_NTSD_OWNER: + extra_info = OWNER_SECINFO | GROUP_SECINFO; + break; + case XATTR_CIFS_NTSD_SACL: + extra_info = SACL_SECINFO; + break; + case XATTR_CIFS_ACL: + default: + extra_info = DACL_SECINFO; + break; + } + pacl = pTcon->ses->server->ops->get_acl(cifs_sb, + inode, full_path, &acllen, extra_info); + if (IS_ERR(pacl)) { + rc = PTR_ERR(pacl); + cifs_dbg(VFS, "%s: error %d getting sec desc\n", + __func__, rc); + } else { + if (value) { + if (acllen > size) + acllen = -ERANGE; + else + memcpy(value, pacl, acllen); + } + rc = acllen; + kfree(pacl); + } + break; + } + } + + /* We could add an additional check for streams ie + if proc/fs/cifs/streamstoxattr is set then + search server for EAs or streams to + returns as xattrs */ + + if (rc == -EINVAL) + rc = -EOPNOTSUPP; + +out: + free_dentry_path(page); + free_xid(xid); + cifs_put_tlink(tlink); + return rc; +} + +ssize_t cifs_listxattr(struct dentry *direntry, char *data, size_t buf_size) +{ + ssize_t rc = -EOPNOTSUPP; + unsigned int xid; + struct cifs_sb_info *cifs_sb = CIFS_SB(direntry->d_sb); + struct tcon_link *tlink; + struct cifs_tcon *pTcon; + const char *full_path; + void *page; + + if (unlikely(cifs_forced_shutdown(cifs_sb))) + return smb_EIO(smb_eio_trace_forced_shutdown); + + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NO_XATTR) + return -EOPNOTSUPP; + + tlink = cifs_sb_tlink(cifs_sb); + if (IS_ERR(tlink)) + return PTR_ERR(tlink); + pTcon = tlink_tcon(tlink); + + xid = get_xid(); + page = alloc_dentry_path(); + + full_path = build_path_from_dentry(direntry, page); + if (IS_ERR(full_path)) { + rc = PTR_ERR(full_path); + goto list_ea_exit; + } + /* return dos attributes as pseudo xattr */ + /* return alt name if available as pseudo attr */ + + /* if proc/fs/cifs/streamstoxattr is set then + search server for EAs or streams to + returns as xattrs */ + + if (pTcon->ses->server->ops->query_all_EAs) + rc = pTcon->ses->server->ops->query_all_EAs(xid, pTcon, + full_path, NULL, data, buf_size, cifs_sb); +list_ea_exit: + free_dentry_path(page); + free_xid(xid); + cifs_put_tlink(tlink); + return rc; +} + +static const struct xattr_handler cifs_user_xattr_handler = { + .prefix = XATTR_USER_PREFIX, + .flags = XATTR_USER, + .get = cifs_xattr_get, + .set = cifs_xattr_set, +}; + +/* os2.* attributes are treated like user.* attributes */ +static const struct xattr_handler cifs_os2_xattr_handler = { + .prefix = XATTR_OS2_PREFIX, + .flags = XATTR_USER, + .get = cifs_xattr_get, + .set = cifs_xattr_set, +}; + +static const struct xattr_handler cifs_cifs_acl_xattr_handler = { + .name = CIFS_XATTR_CIFS_ACL, + .flags = XATTR_CIFS_ACL, + .get = cifs_xattr_get, + .set = cifs_xattr_set, +}; + +/* + * Although this is just an alias for the above, need to move away from + * confusing users and using the 20 year old term 'cifs' when it is no + * longer secure and was replaced by SMB2/SMB3 a long time ago, and + * SMB3 and later are highly secure. + */ +static const struct xattr_handler smb3_acl_xattr_handler = { + .name = SMB3_XATTR_CIFS_ACL, + .flags = XATTR_CIFS_ACL, + .get = cifs_xattr_get, + .set = cifs_xattr_set, +}; + +static const struct xattr_handler smb3_ntsd_sacl_xattr_handler = { + .name = SMB3_XATTR_CIFS_NTSD_SACL, + .flags = XATTR_CIFS_NTSD_SACL, + .get = cifs_xattr_get, + .set = cifs_xattr_set, +}; + +static const struct xattr_handler smb3_ntsd_owner_xattr_handler = { + .name = SMB3_XATTR_CIFS_NTSD_OWNER, + .flags = XATTR_CIFS_NTSD_OWNER, + .get = cifs_xattr_get, + .set = cifs_xattr_set, +}; + +static const struct xattr_handler cifs_cifs_ntsd_xattr_handler = { + .name = CIFS_XATTR_CIFS_NTSD, + .flags = XATTR_CIFS_NTSD, + .get = cifs_xattr_get, + .set = cifs_xattr_set, +}; + +/* + * Although this is just an alias for the above, need to move away from + * confusing users and using the 20 year old term 'cifs' when it is no + * longer secure and was replaced by SMB2/SMB3 a long time ago, and + * SMB3 and later are highly secure. + */ +static const struct xattr_handler smb3_ntsd_xattr_handler = { + .name = SMB3_XATTR_CIFS_NTSD, + .flags = XATTR_CIFS_NTSD, + .get = cifs_xattr_get, + .set = cifs_xattr_set, +}; + +static const struct xattr_handler cifs_cifs_ntsd_full_xattr_handler = { + .name = CIFS_XATTR_CIFS_NTSD_FULL, + .flags = XATTR_CIFS_NTSD_FULL, + .get = cifs_xattr_get, + .set = cifs_xattr_set, +}; + +/* + * Although this is just an alias for the above, need to move away from + * confusing users and using the 20 year old term 'cifs' when it is no + * longer secure and was replaced by SMB2/SMB3 a long time ago, and + * SMB3 and later are highly secure. + */ +static const struct xattr_handler smb3_ntsd_full_xattr_handler = { + .name = SMB3_XATTR_CIFS_NTSD_FULL, + .flags = XATTR_CIFS_NTSD_FULL, + .get = cifs_xattr_get, + .set = cifs_xattr_set, +}; + +const struct xattr_handler * const cifs_xattr_handlers[] = { + &cifs_user_xattr_handler, + &cifs_os2_xattr_handler, + &cifs_cifs_acl_xattr_handler, + &smb3_acl_xattr_handler, /* alias for above since avoiding "cifs" */ + &smb3_ntsd_sacl_xattr_handler, + &smb3_ntsd_owner_xattr_handler, + &cifs_cifs_ntsd_xattr_handler, + &smb3_ntsd_xattr_handler, /* alias for above since avoiding "cifs" */ + &cifs_cifs_ntsd_full_xattr_handler, + &smb3_ntsd_full_xattr_handler, /* alias for above since avoiding "cifs" */ + NULL +}; diff --git a/fs/smb/common/Makefile b/fs/smb/common/Makefile new file mode 100644 index 000000000..f2c6e09d4 --- /dev/null +++ b/fs/smb/common/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for Linux filesystem routines that are shared by client and server. +# + +obj-$(CONFIG_SMBFS) += cifs_md4.o +obj-$(CONFIG_SMBFS) += smb_compress.o + +smb_compress-y := compress/compress.o compress/lz77.o diff --git a/fs/smb/common/cifs_md4.c b/fs/smb/common/cifs_md4.c new file mode 100644 index 000000000..7ee7f4dad --- /dev/null +++ b/fs/smb/common/cifs_md4.c @@ -0,0 +1,198 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Cryptographic API. + * + * MD4 Message Digest Algorithm (RFC1320). + * + * Implementation derived from Andrew Tridgell and Steve French's + * CIFS MD4 implementation, and the cryptoapi implementation + * originally based on the public domain implementation written + * by Colin Plumb in 1993. + * + * Copyright (c) Andrew Tridgell 1997-1998. + * Modified by Steve French (sfrench@us.ibm.com) 2002 + * Copyright (c) Cryptoapi developers. + * Copyright (c) 2002 David S. Miller (davem@redhat.com) + * Copyright (c) 2002 James Morris + * + */ +#include +#include +#include +#include +#include +#include +#include "md4.h" + +MODULE_DESCRIPTION("MD4 Message Digest Algorithm (RFC1320)"); +MODULE_LICENSE("GPL"); + +static inline u32 lshift(u32 x, unsigned int s) +{ + x &= 0xFFFFFFFF; + return ((x << s) & 0xFFFFFFFF) | (x >> (32 - s)); +} + +static inline u32 F(u32 x, u32 y, u32 z) +{ + return (x & y) | ((~x) & z); +} + +static inline u32 G(u32 x, u32 y, u32 z) +{ + return (x & y) | (x & z) | (y & z); +} + +static inline u32 H(u32 x, u32 y, u32 z) +{ + return x ^ y ^ z; +} + +#define ROUND1(a,b,c,d,k,s) (a = lshift(a + F(b,c,d) + k, s)) +#define ROUND2(a,b,c,d,k,s) (a = lshift(a + G(b,c,d) + k + (u32)0x5A827999,s)) +#define ROUND3(a,b,c,d,k,s) (a = lshift(a + H(b,c,d) + k + (u32)0x6ED9EBA1,s)) + +static void md4_transform(u32 *hash, u32 const *in) +{ + u32 a, b, c, d; + + a = hash[0]; + b = hash[1]; + c = hash[2]; + d = hash[3]; + + ROUND1(a, b, c, d, in[0], 3); + ROUND1(d, a, b, c, in[1], 7); + ROUND1(c, d, a, b, in[2], 11); + ROUND1(b, c, d, a, in[3], 19); + ROUND1(a, b, c, d, in[4], 3); + ROUND1(d, a, b, c, in[5], 7); + ROUND1(c, d, a, b, in[6], 11); + ROUND1(b, c, d, a, in[7], 19); + ROUND1(a, b, c, d, in[8], 3); + ROUND1(d, a, b, c, in[9], 7); + ROUND1(c, d, a, b, in[10], 11); + ROUND1(b, c, d, a, in[11], 19); + ROUND1(a, b, c, d, in[12], 3); + ROUND1(d, a, b, c, in[13], 7); + ROUND1(c, d, a, b, in[14], 11); + ROUND1(b, c, d, a, in[15], 19); + + ROUND2(a, b, c, d, in[0], 3); + ROUND2(d, a, b, c, in[4], 5); + ROUND2(c, d, a, b, in[8], 9); + ROUND2(b, c, d, a, in[12], 13); + ROUND2(a, b, c, d, in[1], 3); + ROUND2(d, a, b, c, in[5], 5); + ROUND2(c, d, a, b, in[9], 9); + ROUND2(b, c, d, a, in[13], 13); + ROUND2(a, b, c, d, in[2], 3); + ROUND2(d, a, b, c, in[6], 5); + ROUND2(c, d, a, b, in[10], 9); + ROUND2(b, c, d, a, in[14], 13); + ROUND2(a, b, c, d, in[3], 3); + ROUND2(d, a, b, c, in[7], 5); + ROUND2(c, d, a, b, in[11], 9); + ROUND2(b, c, d, a, in[15], 13); + + ROUND3(a, b, c, d, in[0], 3); + ROUND3(d, a, b, c, in[8], 9); + ROUND3(c, d, a, b, in[4], 11); + ROUND3(b, c, d, a, in[12], 15); + ROUND3(a, b, c, d, in[2], 3); + ROUND3(d, a, b, c, in[10], 9); + ROUND3(c, d, a, b, in[6], 11); + ROUND3(b, c, d, a, in[14], 15); + ROUND3(a, b, c, d, in[1], 3); + ROUND3(d, a, b, c, in[9], 9); + ROUND3(c, d, a, b, in[5], 11); + ROUND3(b, c, d, a, in[13], 15); + ROUND3(a, b, c, d, in[3], 3); + ROUND3(d, a, b, c, in[11], 9); + ROUND3(c, d, a, b, in[7], 11); + ROUND3(b, c, d, a, in[15], 15); + + hash[0] += a; + hash[1] += b; + hash[2] += c; + hash[3] += d; +} + +static inline void md4_transform_helper(struct md4_ctx *ctx) +{ + le32_to_cpu_array(ctx->block, ARRAY_SIZE(ctx->block)); + md4_transform(ctx->hash, ctx->block); +} + +int cifs_md4_init(struct md4_ctx *mctx) +{ + memset(mctx, 0, sizeof(struct md4_ctx)); + mctx->hash[0] = 0x67452301; + mctx->hash[1] = 0xefcdab89; + mctx->hash[2] = 0x98badcfe; + mctx->hash[3] = 0x10325476; + mctx->byte_count = 0; + + return 0; +} +EXPORT_SYMBOL_GPL(cifs_md4_init); + +int cifs_md4_update(struct md4_ctx *mctx, const u8 *data, unsigned int len) +{ + const u32 avail = sizeof(mctx->block) - (mctx->byte_count & 0x3f); + + mctx->byte_count += len; + + if (avail > len) { + memcpy((char *)mctx->block + (sizeof(mctx->block) - avail), + data, len); + return 0; + } + + memcpy((char *)mctx->block + (sizeof(mctx->block) - avail), + data, avail); + + md4_transform_helper(mctx); + data += avail; + len -= avail; + + while (len >= sizeof(mctx->block)) { + memcpy(mctx->block, data, sizeof(mctx->block)); + md4_transform_helper(mctx); + data += sizeof(mctx->block); + len -= sizeof(mctx->block); + } + + memcpy(mctx->block, data, len); + + return 0; +} +EXPORT_SYMBOL_GPL(cifs_md4_update); + +int cifs_md4_final(struct md4_ctx *mctx, u8 *out) +{ + const unsigned int offset = mctx->byte_count & 0x3f; + char *p = (char *)mctx->block + offset; + int padding = 56 - (offset + 1); + + *p++ = 0x80; + if (padding < 0) { + memset(p, 0x00, padding + sizeof(u64)); + md4_transform_helper(mctx); + p = (char *)mctx->block; + padding = 56; + } + + memset(p, 0, padding); + mctx->block[14] = mctx->byte_count << 3; + mctx->block[15] = mctx->byte_count >> 29; + le32_to_cpu_array(mctx->block, (sizeof(mctx->block) - + sizeof(u64)) / sizeof(u32)); + md4_transform(mctx->hash, mctx->block); + cpu_to_le32_array(mctx->hash, ARRAY_SIZE(mctx->hash)); + memcpy(out, mctx->hash, sizeof(mctx->hash)); + memset(mctx, 0, sizeof(*mctx)); + + return 0; +} +EXPORT_SYMBOL_GPL(cifs_md4_final); diff --git a/fs/smb/common/compress/compress.c b/fs/smb/common/compress/compress.c new file mode 100644 index 000000000..a4123c8f1 --- /dev/null +++ b/fs/smb/common/compress/compress.c @@ -0,0 +1,404 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * SMB2 compression transform helpers. + * + * Copyright (C) 2026 Namjae Jeon + */ +#include +#include +#include +#include + +#include "compress.h" +#include "lz77.h" + +#define SMB2_COMPRESSION_CHAINED_HDR_LEN \ + offsetof(struct smb2_compression_hdr, CompressionAlgorithm) +#define SMB2_COMPRESSION_PAYLOAD_BASE_LEN \ + (sizeof(struct smb2_compression_payload_hdr) - sizeof(__le32)) + +/* + * A NONE payload carries bytes verbatim. Keep both cursors and remaining + * lengths together so every chained payload handler applies identical bounds + * accounting. + */ +static int smb_decompress_none(const u8 **src, u32 *slen, u8 **dst, u32 *dlen, + u32 len) +{ + if (len > *slen || len > *dlen) + return -EINVAL; + + memcpy(*dst, *src, len); + *src += len; + *slen -= len; + *dst += len; + *dlen -= len; + return 0; +} + +/* + * Pattern_V1 represents a run of one byte. Its wire payload is always the + * fixed-size smb2_compression_pattern_v1 structure. + */ +static int smb_decompress_pattern(const u8 **src, u32 *slen, u8 **dst, + u32 *dlen, u32 len) +{ + const struct smb2_compression_pattern_v1 *pattern; + u32 repetitions; + + if (len != sizeof(*pattern) || len > *slen) + return -EINVAL; + + pattern = (const struct smb2_compression_pattern_v1 *)*src; + repetitions = le32_to_cpu(pattern->Repetitions); + if (repetitions > *dlen) + return -EINVAL; + + memset(*dst, pattern->Pattern, repetitions); + *src += len; + *slen -= len; + *dst += repetitions; + *dlen -= repetitions; + return 0; +} + +/* + * LZ77 payload Length includes the four-byte OriginalPayloadSize field. + * Consume that field before passing the compressed stream to the raw codec. + */ +static int smb_decompress_lz77_payload(const u8 **src, u32 *slen, u8 **dst, + u32 *dlen, u32 len) +{ + u32 orig_size; + int rc; + + if (len < sizeof(__le32) || len > *slen) + return -EINVAL; + + orig_size = get_unaligned_le32(*src); + if (orig_size > *dlen) + return -EINVAL; + + *src += sizeof(__le32); + *slen -= sizeof(__le32); + len -= sizeof(__le32); + + rc = smb_lz77_decompress(*src, len, *dst, orig_size); + if (rc) + return rc; + + *src += len; + *slen -= len; + *dst += orig_size; + *dlen -= orig_size; + return 0; +} + +static int smb_decompress_chained(__le16 alg, bool allow_chained, + bool allow_pattern, + const struct smb2_compression_hdr *hdr, + u32 slen, void *dst, u32 dlen) +{ + const struct smb2_compression_payload_hdr *payload; + const u8 *src = (const u8 *)hdr + SMB2_COMPRESSION_CHAINED_HDR_LEN; + u32 orig_size = le32_to_cpu(hdr->OriginalCompressedSegmentSize); + u32 remaining = slen - SMB2_COMPRESSION_CHAINED_HDR_LEN; + u8 *out = dst; + u32 out_remaining = dlen; + bool first = true; + int rc; + + if (!allow_chained || orig_size != dlen) + return -EINVAL; + + /* + * The chained transform has an eight-byte top-level header. The next + * bytes are a sequence of payload headers whose Length fields account + * for payload data, including OriginalPayloadSize where applicable. + */ + while (remaining) { + __le16 payload_alg; + __le16 flags; + u32 len; + + if (remaining < SMB2_COMPRESSION_PAYLOAD_BASE_LEN) + return -EINVAL; + + payload = (const struct smb2_compression_payload_hdr *)src; + payload_alg = payload->CompressionAlgorithm; + flags = payload->Flags; + len = le32_to_cpu(payload->Length); + + /* + * CHAINED marks only the first payload. Requiring NONE on every + * later payload rejects ambiguous or independently chained data. + */ + if ((first && flags != cpu_to_le16(SMB2_COMPRESSION_FLAG_CHAINED)) || + (!first && flags != cpu_to_le16(SMB2_COMPRESSION_FLAG_NONE))) + return -EINVAL; + + src += SMB2_COMPRESSION_PAYLOAD_BASE_LEN; + remaining -= SMB2_COMPRESSION_PAYLOAD_BASE_LEN; + + if (payload_alg == SMB3_COMPRESS_NONE) { + rc = smb_decompress_none(&src, &remaining, &out, + &out_remaining, len); + } else if (payload_alg == SMB3_COMPRESS_PATTERN) { + if (!allow_pattern) + return -EINVAL; + rc = smb_decompress_pattern(&src, &remaining, &out, + &out_remaining, len); + } else if (payload_alg == alg && alg == SMB3_COMPRESS_LZ77) { + rc = smb_decompress_lz77_payload(&src, &remaining, &out, + &out_remaining, len); + } else { + return -EINVAL; + } + if (rc) + return rc; + first = false; + } + + return out_remaining ? -EINVAL : 0; +} + +static int smb_decompress_unchained(__le16 alg, + const struct smb2_compression_hdr *hdr, + u32 slen, void *dst, u32 dlen) +{ + u32 orig_size, offset, comp_size; + + if (hdr->CompressionAlgorithm != alg || + !smb_compress_alg_valid(hdr->CompressionAlgorithm, false)) + return -EINVAL; + + orig_size = le32_to_cpu(hdr->OriginalCompressedSegmentSize); + offset = le32_to_cpu(hdr->Offset); + if (offset > slen - sizeof(*hdr) || offset > dlen || + orig_size > dlen - offset || orig_size + offset != dlen) + return -EINVAL; + + memcpy(dst, (const u8 *)hdr + sizeof(*hdr), offset); + comp_size = slen - sizeof(*hdr) - offset; + return smb_lz77_decompress((const u8 *)hdr + sizeof(*hdr) + offset, + comp_size, (u8 *)dst + offset, orig_size); +} + +/** + * smb_compression_decompress() - decode an SMB2 compression transform + * @alg: negotiated general-purpose compression algorithm + * @allow_chained: whether chained transforms were negotiated + * @allow_pattern: whether Pattern_V1 payloads were negotiated + * @src: transform header followed by compressed payload data + * @slen: total number of bytes available at @src + * @dst: output buffer for the reconstructed SMB2 message + * @dlen: exact expected size of the reconstructed SMB2 message + * + * Validate the transform type and negotiated capabilities before dispatching + * to the chained or unchained decoder. The caller supplies the expected output + * size after applying its transport-specific message size limits. + * + * Return: 0 on success, otherwise a negative errno. + */ +int smb_compression_decompress(__le16 alg, bool allow_chained, + bool allow_pattern, const void *src, u32 slen, + void *dst, u32 dlen) +{ + const struct smb2_compression_hdr *hdr = src; + + if (!src || !dst || slen < sizeof(*hdr) || + hdr->ProtocolId != SMB2_COMPRESSION_TRANSFORM_ID || + alg == SMB3_COMPRESS_NONE) + return -EINVAL; + + if (hdr->Flags == cpu_to_le16(SMB2_COMPRESSION_FLAG_CHAINED)) + return smb_decompress_chained(alg, allow_chained, allow_pattern, + hdr, slen, dst, dlen); + + if (hdr->Flags != cpu_to_le16(SMB2_COMPRESSION_FLAG_NONE)) + return -EINVAL; + + return smb_decompress_unchained(alg, hdr, slen, dst, dlen); +} +EXPORT_SYMBOL_GPL(smb_compression_decompress); + +struct smb_compression_builder { + u8 *pos; + u32 remaining; + bool first; +}; + +/* + * Reserve one chained payload header and initialize its common fields. + * OriginalPayloadSize is present only for LZNT1/LZ77/LZ77+Huffman payloads. + */ +static struct smb2_compression_payload_hdr * +smb_compression_add_payload(struct smb_compression_builder *builder, + __le16 alg, u32 payload_len, bool orig_size) +{ + struct smb2_compression_payload_hdr *payload; + u32 hdr_len = SMB2_COMPRESSION_PAYLOAD_BASE_LEN; + u32 total_len; + + if (orig_size) + hdr_len += sizeof(payload->OriginalPayloadSize); + if (check_add_overflow(hdr_len, payload_len, &total_len) || + total_len > builder->remaining) + return NULL; + + payload = (struct smb2_compression_payload_hdr *)builder->pos; + payload->CompressionAlgorithm = alg; + payload->Flags = cpu_to_le16(builder->first ? + SMB2_COMPRESSION_FLAG_CHAINED : SMB2_COMPRESSION_FLAG_NONE); + payload->Length = cpu_to_le32(payload_len + + (orig_size ? sizeof(payload->OriginalPayloadSize) : 0)); + + builder->pos += hdr_len; + builder->remaining -= hdr_len; + builder->first = false; + return payload; +} + +static int smb_compression_add_pattern(struct smb_compression_builder *builder, + u8 pattern, u32 repetitions) +{ + struct smb2_compression_pattern_v1 *payload; + + if (!smb_compression_add_payload(builder, SMB3_COMPRESS_PATTERN, + sizeof(*payload), false)) + return -ENOSPC; + + payload = (struct smb2_compression_pattern_v1 *)builder->pos; + payload->Pattern = pattern; + payload->Reserved1 = 0; + payload->Reserved2 = 0; + payload->Repetitions = cpu_to_le32(repetitions); + builder->pos += sizeof(*payload); + builder->remaining -= sizeof(*payload); + return 0; +} + +static int smb_compression_add_none(struct smb_compression_builder *builder, + const u8 *src, u32 len) +{ + if (!smb_compression_add_payload(builder, SMB3_COMPRESS_NONE, len, false)) + return -ENOSPC; + + memcpy(builder->pos, src, len); + builder->pos += len; + builder->remaining -= len; + return 0; +} + +static int smb_compression_add_lz77(struct smb_compression_builder *builder, + const u8 *src, u32 len) +{ + struct smb2_compression_payload_hdr *payload; + u32 comp_len; + int rc; + + if (builder->remaining <= sizeof(*payload)) + return -ENOSPC; + + comp_len = builder->remaining - sizeof(*payload); + payload = smb_compression_add_payload(builder, SMB3_COMPRESS_LZ77, + comp_len, true); + if (!payload) + return -ENOSPC; + + rc = smb_lz77_compress(src, len, builder->pos, &comp_len); + if (rc) + return rc; + + payload->Length = cpu_to_le32(comp_len + + sizeof(payload->OriginalPayloadSize)); + payload->OriginalPayloadSize = cpu_to_le32(len); + builder->pos += comp_len; + builder->remaining -= comp_len; + return 0; +} + +/** + * smb_compression_compress_chained() - build a chained SMB2 transform + * @alg: negotiated general-purpose compression algorithm + * @allow_pattern: whether Pattern_V1 was negotiated + * @src: complete uncompressed SMB2 message + * @slen: size of @src + * @dst: output buffer for the transform + * @dlen: input capacity of @dst and output transform size + * + * Following the algorithm in [MS-SMB2] 3.1.4.4, encode sufficiently long + * repeated runs at the front and back as Pattern_V1 payloads. Compress a + * middle region larger than 1 KiB with LZ77; smaller middle regions are + * represented by a chained NONE payload. + * + * This helper does not decide whether the final transform is smaller than the + * original message. The transport caller owns that policy decision. + * + * Return: 0 on success, otherwise a negative errno. + */ +int smb_compression_compress_chained(__le16 alg, bool allow_pattern, + const void *src, u32 slen, + void *dst, u32 *dlen) +{ + struct smb2_compression_hdr *hdr = dst; + struct smb_compression_builder builder; + const u8 *input = src; + u32 forward = 0, backward = 0, middle_len; + int rc; + + if (!src || !dst || !dlen || alg != SMB3_COMPRESS_LZ77 || + *dlen <= SMB2_COMPRESSION_CHAINED_HDR_LEN || !slen) + return -EINVAL; + + hdr->ProtocolId = SMB2_COMPRESSION_TRANSFORM_ID; + hdr->OriginalCompressedSegmentSize = cpu_to_le32(slen); + builder.pos = (u8 *)dst + SMB2_COMPRESSION_CHAINED_HDR_LEN; + builder.remaining = *dlen - SMB2_COMPRESSION_CHAINED_HDR_LEN; + builder.first = true; + + if (allow_pattern && slen > 32) { + for (forward = 1; forward < slen; forward++) { + if (input[forward] != input[0]) + break; + } + if (forward <= 32) + forward = 0; + + for (backward = 1; backward < slen - forward; backward++) { + if (input[slen - backward - 1] != input[slen - 1]) + break; + } + if (backward <= 32) + backward = 0; + } + + if (forward) { + rc = smb_compression_add_pattern(&builder, input[0], forward); + if (rc) + return rc; + } + + middle_len = slen - forward - backward; + if (middle_len > 1024) + rc = smb_compression_add_lz77(&builder, input + forward, + middle_len); + else if (middle_len) + rc = smb_compression_add_none(&builder, + input + forward, middle_len); + else + rc = 0; + if (rc) + return rc; + + if (backward) { + rc = smb_compression_add_pattern(&builder, input[slen - 1], + backward); + if (rc) + return rc; + } + + *dlen = builder.pos - (u8 *)dst; + return 0; +} +EXPORT_SYMBOL_GPL(smb_compression_compress_chained); diff --git a/fs/smb/common/compress/compress.h b/fs/smb/common/compress/compress.h new file mode 100644 index 000000000..d6916669f --- /dev/null +++ b/fs/smb/common/compress/compress.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2026 Namjae Jeon + */ +#ifndef _COMMON_SMB_COMPRESS_H +#define _COMMON_SMB_COMPRESS_H + +#include "../smb2pdu.h" + +/* + * SMB3_COMPRESS_NONE is valid only in chained payload headers. It is never + * negotiated as a compression algorithm. + */ +static __always_inline bool smb_compress_alg_valid(__le16 alg, bool valid_none) +{ + if (alg == SMB3_COMPRESS_NONE) + return valid_none; + + return alg == SMB3_COMPRESS_LZ77 || alg == SMB3_COMPRESS_PATTERN; +} + +int smb_compression_decompress(__le16 alg, bool allow_chained, + bool allow_pattern, const void *src, u32 slen, + void *dst, u32 dlen); +int smb_compression_compress_chained(__le16 alg, bool allow_pattern, + const void *src, u32 slen, + void *dst, u32 *dlen); + +#endif /* _COMMON_SMB_COMPRESS_H */ diff --git a/fs/smb/common/compress/lz77.c b/fs/smb/common/compress/lz77.c new file mode 100644 index 000000000..9216d973d --- /dev/null +++ b/fs/smb/common/compress/lz77.c @@ -0,0 +1,461 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2024-2026, SUSE LLC + * Copyright (C) 2026 Namjae Jeon + * + * Authors: Enzo Matsumiya + * Namjae Jeon + * + * Implementation of the LZ77 "plain" compression algorithm, as per MS-XCA spec. + */ +#include +#include +#include +#include +#include +#include + +#include "lz77.h" + +/* + * Compression parameters. + * + * LZ77_MATCH_MAX_DIST: Farthest back a match can be from current position (can be 1 - 8K). + * LZ77_HASH_LOG: + * LZ77_HASH_SIZE: ilog2 hash size (recommended to be 13 - 18, default 15 (hash size + * 32k)). + * LZ77_RSTEP_SIZE: Number of bytes to read from input buffer for hashing and initial + * match check (default 4 bytes, this effectivelly makes this the min + * match len). + * LZ77_MSTEP_SIZE: Number of bytes to extend-compare a found match (default 8 bytes). + * LZ77_SKIP_TRIGGER: ilog2 value for adaptive skipping, i.e. to progressively skip input + * bytes when we can't find matches. Default is 4. + * Higher values (>0) will decrease compression time, but will result + * in worse compression ratio. Lower values will give better + * compression ratio (more matches found), but will increase time. + */ +#define LZ77_MATCH_MAX_DIST SZ_8K +#define LZ77_HASH_LOG 15 +#define LZ77_HASH_SIZE BIT(LZ77_HASH_LOG) +#define LZ77_RSTEP_SIZE sizeof(u32) +#define LZ77_MSTEP_SIZE sizeof(u64) +#define LZ77_SKIP_TRIGGER 4 + +#define LZ77_PREFETCH(ptr) __builtin_prefetch((ptr), 0, 3) +#define LZ77_FLAG_MAX 32 + +static __always_inline u8 lz77_read8(const u8 *ptr) +{ + return get_unaligned(ptr); +} + +static __always_inline u32 lz77_read32(const u32 *ptr) +{ + return get_unaligned(ptr); +} + +static __always_inline u64 lz77_read64(const u64 *ptr) +{ + return get_unaligned(ptr); +} + +static __always_inline void lz77_write8(u8 *ptr, u8 v) +{ + put_unaligned(v, ptr); +} + +static __always_inline void lz77_write16(u16 *ptr, u16 v) +{ + put_unaligned_le16(v, ptr); +} + +static __always_inline void lz77_write32(u32 *ptr, u32 v) +{ + put_unaligned_le32(v, ptr); +} + +static __always_inline u32 lz77_match_len(const void *match, const void *cur, const void *end) +{ + const void *start = cur; + + /* Safe for a do/while because otherwise we wouldn't reach here from the main loop. */ + do { + const u64 diff = lz77_read64(cur) ^ lz77_read64(match); + + if (!diff) { + cur += LZ77_MSTEP_SIZE; + match += LZ77_MSTEP_SIZE; + + continue; + } + + /* This computes the number of common bytes in @diff. */ + cur += count_trailing_zeros(diff) >> 3; + + return (cur - start); + } while (likely(cur + LZ77_MSTEP_SIZE <= end)); + + /* Fallback to byte-by-byte comparison for last <8 bytes. */ + while (cur < end && lz77_read8(cur) == lz77_read8(match)) { + cur++; + match++; + } + + return (cur - start); +} + +/** + * lz77_encode_match() - Match encoding. + * @dst: compressed buffer + * @nib: pointer to an address in @dst + * @dist: match distance + * @len: match length + * + * Assumes all args were previously checked. + * + * Return: @dst advanced to new position + * + * Ref: MS-XCA 2.3.4 "Plain LZ77 Compression Algorithm Details" - "Processing" + */ +static __always_inline void *lz77_encode_match(void *dst, void **nib, u16 dist, u32 len) +{ + len -= 3; + dist--; + dist <<= 3; + + if (len < 7) { + lz77_write16(dst, dist + len); + + return dst + sizeof(u16); + } + + dist |= 7; + lz77_write16(dst, dist); + dst += sizeof(u16); + len -= 7; + + if (!*nib) { + lz77_write8(dst, umin(len, 15)); + *nib = dst; + dst++; + } else { + u8 *b = *nib; + + lz77_write8(b, *b | umin(len, 15) << 4); + *nib = NULL; + } + + if (len < 15) + return dst; + + len -= 15; + if (len < 255) { + lz77_write8(dst, len); + + return dst + 1; + } + + lz77_write8(dst, 0xff); + dst++; + len += 7 + 15; + if (len <= 0xffff) { + lz77_write16(dst, len); + + return dst + sizeof(u16); + } + + lz77_write16(dst, 0); + dst += sizeof(u16); + lz77_write32(dst, len); + + return dst + sizeof(u32); +} + +/** + * lz77_encode_literals() - Literals encoding. + * @start: where to start copying literals (uncompressed buffer) + * @end: when to stop copying (uncompressed buffer) + * @dst: compressed buffer + * @f: pointer to current flag value + * @fc: pointer to current flag count + * @fp: pointer to current flag address + * + * Batch copy literals from @start to @dst, updating flag values accordingly. + * Assumes all args were previously checked. + * + * Return: @dst advanced to new position + * + * MS-XCA 2.3.4 "Plain LZ77 Compression Algorithm Details" - "Processing" + */ +static __always_inline void *lz77_encode_literals(const void *start, const void *end, void *dst, + long *f, u32 *fc, void **fp) +{ + if (start >= end) + return dst; + + do { + const u32 len = umin(end - start, LZ77_FLAG_MAX - *fc); + + memcpy(dst, start, len); + + dst += len; + start += len; + + *f <<= len; + *fc += len; + if (*fc == LZ77_FLAG_MAX) { + lz77_write32(*fp, *f); + *fc = 0; + *fp = dst; + dst += sizeof(u32); + } + } while (start < end); + + return dst; +} + +static __always_inline u32 lz77_hash(const u32 v) +{ + return ((v ^ 0x9E3779B9) * 0x85EBCA6B) >> (32 - LZ77_HASH_LOG); +} + +noinline int smb_lz77_compress(const void *src, const u32 slen, + void *dst, u32 *dlen) +{ + const void *srcp, *rlim, *end, *anchor; + u32 *htable, hash, flag_count = 0; + void *dstp, *nib, *flag_pos; + long flag = 0; + + /* This is probably a bug, so throw a warning. */ + if (WARN_ON_ONCE(*dlen < smb_lz77_compressed_alloc_size(slen))) + return -EINVAL; + + srcp = src; + anchor = src; + end = srcp + slen; /* absolute end */ + rlim = end - LZ77_MSTEP_SIZE; /* read limit (for lz77_match_len()) */ + dstp = dst; + flag_pos = dstp; + dstp += sizeof(u32); + nib = NULL; + + htable = kvcalloc(LZ77_HASH_SIZE, sizeof(*htable), GFP_KERNEL); + if (!htable) + return -ENOMEM; + + LZ77_PREFETCH(srcp + LZ77_RSTEP_SIZE); + + /* + * Adjust @srcp so we don't get a false positive match on first iteration. + * Then prepare hash for first loop iteration (don't advance @srcp again). + */ + hash = lz77_hash(lz77_read32(srcp++)); + htable[hash] = 0; + hash = lz77_hash(lz77_read32(srcp)); + + /* + * Main loop. + * + * @dlen is >= smb_lz77_compressed_alloc_size(), so run without + * bound-checking @dstp. + * + * This code was crafted in a way to best utilise fetch-decode-execute CPU flow. + * Any attempt to optimize it, or even organize it, can lead to huge performance loss. + */ + do { + const void *match, *next = srcp; + u32 len, step = 1, skip = 1U << LZ77_SKIP_TRIGGER; + + /* Match finding (hot path -- don't change the read/check/write order). */ + do { + const u32 cur_hash = hash; + + srcp = next; + next += step; + + /* + * Adaptive skipping. + * + * Increment @step every (1 << LZ77_SKIP_TRIGGER, 16 in our case) bytes + * without a match. + * Reset to 1 when a match is found. + */ + step = (skip++ >> LZ77_SKIP_TRIGGER); + if (unlikely(next > rlim)) + goto out; + + hash = lz77_hash(lz77_read32(next)); + match = src + htable[cur_hash]; + htable[cur_hash] = srcp - src; + } while (likely(match + LZ77_MATCH_MAX_DIST < srcp) || + lz77_read32(match) != lz77_read32(srcp)); + + /* + * Match found. Warm/cold path; begin parsing @srcp and writing to @dstp: + * - flush literals + * - compute match length (*) + * - encode match + * + * (*) Current minimum match length is defined by the memory read size above, so + * here we already know that we have 4 matching bytes, but it's just faster to + * redundantly compute it again in lz77_match_len() than to adjust pointers/len. + */ + dstp = lz77_encode_literals(anchor, srcp, dstp, &flag, &flag_count, &flag_pos); + len = lz77_match_len(match, srcp, end); + dstp = lz77_encode_match(dstp, &nib, srcp - match, len); + srcp += len; + anchor = srcp; + + LZ77_PREFETCH(srcp); + + flag = (flag << 1) | 1; + flag_count++; + if (flag_count == LZ77_FLAG_MAX) { + lz77_write32(flag_pos, flag); + flag_count = 0; + flag_pos = dstp; + dstp += sizeof(u32); + } + + if (unlikely(srcp > rlim)) + break; + + /* Prepare for next loop. */ + hash = lz77_hash(lz77_read32(srcp)); + } while (srcp < end); +out: + dstp = lz77_encode_literals(anchor, end, dstp, &flag, &flag_count, &flag_pos); + + flag_count = LZ77_FLAG_MAX - flag_count; + flag <<= flag_count; + flag |= (1UL << flag_count) - 1; + lz77_write32(flag_pos, flag); + + *dlen = dstp - dst; + kvfree(htable); + + if (*dlen < slen) + return 0; + + return -EMSGSIZE; +} +EXPORT_SYMBOL_GPL(smb_lz77_compress); + +static int lz77_decode_match_len(const u8 **src, const u8 *end, u16 token, + u8 *nibble, bool *have_nibble, u32 *len) +{ + u8 extra; + + *len = (token & 0x7) + 3; + if ((token & 0x7) != 0x7) + return 0; + + if (!*have_nibble) { + if (*src >= end) + return -EINVAL; + *nibble = *(*src)++; + extra = *nibble & 0xf; + *have_nibble = true; + } else { + extra = *nibble >> 4; + *have_nibble = false; + } + + *len += extra; + if (extra == 0xf) { + u8 b; + + if (*src >= end) + return -EINVAL; + b = *(*src)++; + if (b != 0xff) { + *len += b; + } else { + u16 w; + + if (end - *src < 2) + return -EINVAL; + w = get_unaligned_le16(*src); + *src += 2; + if (w) { + *len = w + 3; + } else { + u32 long_len; + + if (end - *src < 4) + return -EINVAL; + long_len = get_unaligned_le32(*src); + *src += 4; + if (check_add_overflow(long_len, 3, len)) + return -EINVAL; + } + } + } + + return 0; +} + +int smb_lz77_decompress(const void *src, const u32 slen, void *dst, + const u32 dlen) +{ + const u8 *sp = src, *send = sp + slen; + u8 *dp = dst, *dend = dp + dlen; + u32 flags = 0; + int flag_count = 0; + u8 nibble = 0; + bool have_nibble = false; + + while (dp < dend) { + u32 len, dist; + u16 token; + + if (!flag_count) { + if (send - sp < 4) + return -EINVAL; + flags = get_unaligned_le32(sp); + sp += 4; + flag_count = 32; + } + + if (!(flags & 0x80000000)) { + if (sp >= send) + return -EINVAL; + *dp++ = *sp++; + flags <<= 1; + flag_count--; + continue; + } + + flags <<= 1; + flag_count--; + + if (send - sp < 2) + return -EINVAL; + + token = get_unaligned_le16(sp); + sp += 2; + + dist = (token >> 3) + 1; + if (dist > dp - (u8 *)dst) + return -EINVAL; + + if (lz77_decode_match_len(&sp, send, token, &nibble, + &have_nibble, &len)) + return -EINVAL; + + if (len > dend - dp) + return -EINVAL; + + while (len--) { + *dp = *(dp - dist); + dp++; + } + } + + return 0; +} +EXPORT_SYMBOL_GPL(smb_lz77_decompress); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SMB plain LZ77 compression"); diff --git a/fs/smb/common/compress/lz77.h b/fs/smb/common/compress/lz77.h new file mode 100644 index 000000000..e032c0f1b --- /dev/null +++ b/fs/smb/common/compress/lz77.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2024-2026, SUSE LLC + * + * Authors: Enzo Matsumiya + * + * Implementation of the LZ77 "plain" compression algorithm, as per MS-XCA spec. + */ +#ifndef _SMB_COMPRESS_LZ77_H +#define _SMB_COMPRESS_LZ77_H + +#include + +/** + * smb_lz77_compressed_alloc_size() - Compute compressed buffer size. + * @size: uncompressed (src) size + * + * Compute allocation size for the compressed buffer based on uncompressed size. + * Accounts for metadata and overprovision for the worst case scenario. + * + * LZ77 metadata is a 4-byte flag that is written: + * - on dst begin (pos 0) + * - every 32 literals or matches + * - on end-of-stream (possibly, if last write was another flag) + * + * Worst case scenario is an all-literal compression, which means: + * metadata bytes = 4 + ((@size / 32) * 4) + 4, or, simplified, (@size >> 3) + 8 + * + * The worst case scenario rarely happens, but such overprovisioning also + * allows smb_lz77_compress() main loop to run without ever bound checking dst, + * which is a huge perf improvement, while also being safe when compression goes + * bad. + * + * Return: required (*) allocation size for compressed buffer. + * + * (*) checked once in the beginning of smb_lz77_compress() + */ +static __always_inline u32 smb_lz77_compressed_alloc_size(const u32 size) +{ + return size + (size >> 3) + 8; +} + +int smb_lz77_compress(const void *src, const u32 slen, void *dst, u32 *dlen); +int smb_lz77_decompress(const void *src, const u32 slen, void *dst, + const u32 dlen); +#endif /* _SMB_COMPRESS_LZ77_H */ diff --git a/fs/smb/common/fscc.h b/fs/smb/common/fscc.h new file mode 100644 index 000000000..e46d3379b --- /dev/null +++ b/fs/smb/common/fscc.h @@ -0,0 +1,604 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +/* + * + * Copyright (c) International Business Machines Corp., 2009, 2013 + * Etersoft, 2012 + * 2018 Samsung Electronics Co., Ltd. + * Author(s): Steve French (sfrench@us.ibm.com) + * Pavel Shilovsky (pshilovsky@samba.org) 2012 + * Namjae Jeon (linkinjeon@kernel.org) + * + */ +#ifndef _COMMON_SMB_FSCC_H +#define _COMMON_SMB_FSCC_H + +/* Reparse structures - see MS-FSCC 2.1.2 */ + +/* struct fsctl_reparse_info_req is empty, only response structs (see below) */ +struct reparse_data_buffer { + __le32 ReparseTag; + __le16 ReparseDataLength; + __u16 Reserved; + __u8 DataBuffer[]; /* Variable Length */ +} __packed; + +struct reparse_guid_data_buffer { + __le32 ReparseTag; + __le16 ReparseDataLength; + __u16 Reserved; + __u8 ReparseGuid[16]; + __u8 DataBuffer[]; /* Variable Length */ +} __packed; + +struct reparse_mount_point_data_buffer { + __le32 ReparseTag; + __le16 ReparseDataLength; + __u16 Reserved; + __le16 SubstituteNameOffset; + __le16 SubstituteNameLength; + __le16 PrintNameOffset; + __le16 PrintNameLength; + __u8 PathBuffer[]; /* Variable Length */ +} __packed; + +#define SYMLINK_FLAG_RELATIVE 0x00000001 + +struct reparse_symlink_data_buffer { + __le32 ReparseTag; + __le16 ReparseDataLength; + __u16 Reserved; + __le16 SubstituteNameOffset; + __le16 SubstituteNameLength; + __le16 PrintNameOffset; + __le16 PrintNameLength; + __le32 Flags; + __u8 PathBuffer[]; /* Variable Length */ +} __packed; + +/* For IO_REPARSE_TAG_NFS - see MS-FSCC 2.1.2.6 */ +#define NFS_SPECFILE_LNK 0x00000000014B4E4C +#define NFS_SPECFILE_CHR 0x0000000000524843 +#define NFS_SPECFILE_BLK 0x00000000004B4C42 +#define NFS_SPECFILE_FIFO 0x000000004F464946 +#define NFS_SPECFILE_SOCK 0x000000004B434F53 +struct reparse_nfs_data_buffer { + __le32 ReparseTag; + __le16 ReparseDataLength; + __u16 Reserved; + __le64 InodeType; /* NFS_SPECFILE_* */ + __u8 DataBuffer[]; +} __packed; + +/* For IO_REPARSE_TAG_LX_SYMLINK - see MS-FSCC 2.1.2.7 */ +struct reparse_wsl_symlink_data_buffer { + __le32 ReparseTag; + __le16 ReparseDataLength; + __u16 Reserved; + __le32 Version; /* Always 2 */ + __u8 Target[]; /* Variable Length UTF-8 string without nul-term */ +} __packed; + +/* See MS-FSCC 2.3.7 */ +struct duplicate_extents_to_file { + __u64 PersistentFileHandle; /* source file handle, opaque endianness */ + __u64 VolatileFileHandle; + __le64 SourceFileOffset; + __le64 TargetFileOffset; + __le64 ByteCount; /* Bytes to be copied */ +} __packed; + +/* See MS-FSCC 2.3.9 */ +#define DUPLICATE_EXTENTS_DATA_EX_SOURCE_ATOMIC 0x00000001 +struct duplicate_extents_to_file_ex { + __le64 StructureSize; /* MUST be set to 0x30 */ + __u64 PersistentFileHandle; /* source file handle, opaque endianness */ + __u64 VolatileFileHandle; + __le64 SourceFileOffset; + __le64 TargetFileOffset; + __le64 ByteCount; /* Bytes to be copied */ + __le32 Flags; + __le32 Reserved; +} __packed; + +/* + * compression state flags + * See MS-FSCC 2.3.18 + * MS-FSCC 2.3.67 + * MS-FSCC 2.4.9 + */ +#define COMPRESSION_FORMAT_NONE 0x0000 +#define COMPRESSION_FORMAT_DEFAULT 0x0001 +#define COMPRESSION_FORMAT_LZNT1 0x0002 + +/* + * See MS-FSCC 2.3.18 + * MS-FSCC 2.3.67 + */ +struct compress_ioctl { + __le16 CompressionState; +} __packed; + +/* See MS-FSCC 2.3.20 */ +struct fsctl_get_integrity_information_rsp { + __le16 ChecksumAlgorithm; + __le16 Reserved; + __le32 Flags; + __le32 ChecksumChunkSizeInBytes; + __le32 ClusterSizeInBytes; +} __packed; + +/* See MS-FSCC 2.3.52 */ +struct file_allocated_range_buffer { + __le64 file_offset; + __le64 length; +} __packed; + +/* See MS-FSCC 2.3.55 */ +struct fsctl_query_file_regions_req { + __le64 FileOffset; + __le64 Length; + __le32 DesiredUsage; + __le32 Reserved; +} __packed; + +/* DesiredUsage flags see MS-FSCC 2.3.56.1 */ +#define FILE_USAGE_INVALID_RANGE 0x00000000 +#define FILE_USAGE_VALID_CACHED_DATA 0x00000001 +#define FILE_USAGE_NONCACHED_DATA 0x00000002 +struct file_region_info { + __le64 FileOffset; + __le64 Length; + __le32 DesiredUsage; + __le32 Reserved; +} __packed; + +/* See MS-FSCC 2.3.56 */ +struct fsctl_query_file_region_rsp { + __le32 Flags; + __le32 TotalRegionEntryCount; + __le32 RegionEntryCount; + __u32 Reserved; + struct file_region_info Regions[]; +} __packed; + +/* See MS-FSCC 2.3.58 */ +struct fsctl_query_on_disk_vol_info_rsp { + __le64 DirectoryCount; + __le64 FileCount; + __le16 FsFormatMajVersion; + __le16 FsFormatMinVersion; + __u8 FsFormatName[24]; + __le64 FormatTime; + __le64 LastUpdateTime; + __u8 CopyrightInfo[68]; + __u8 AbstractInfo[68]; + __u8 FormatImplInfo[68]; + __u8 LastModifyImplInfo[68]; +} __packed; + +/* See MS-FSCC 2.3.73 */ +struct fsctl_set_integrity_information_req { + __le16 ChecksumAlgorithm; + __le16 Reserved; + __le32 Flags; +} __packed; + +/* See MS-FSCC 2.3.75 */ +struct fsctl_set_integrity_info_ex_req { + __u8 EnableIntegrity; + __u8 KeepState; + __u16 Reserved; + __le32 Flags; + __u8 Version; + __u8 Reserved2[7]; +} __packed; + +/* + * this goes in the ioctl buffer when doing FSCTL_SET_ZERO_DATA + * See MS-FSCC 2.3.85 + */ +struct file_zero_data_information { + __le64 FileOffset; + __le64 BeyondFinalZero; +} __packed; + +struct file_level_trim_range { + __le64 Offset; + __le64 Length; +} __packed; + +struct file_level_trim { + __le32 Key; + __le32 NumRanges; + struct file_level_trim_range Ranges[]; +} __packed; + +struct file_level_trim_output { + __le32 NumRangesProcessed; +} __packed; + +/* + * This level 18, although with struct with same name is different from cifs + * level 0x107. Level 0x107 has an extra u64 between AccessFlags and + * CurrentByteOffset. + * See MS-FSCC 2.4.2 + */ +struct smb2_file_all_info { /* data block encoding of response to level 18 */ + __le64 CreationTime; /* Beginning of FILE_BASIC_INFO equivalent */ + __le64 LastAccessTime; + __le64 LastWriteTime; + __le64 ChangeTime; + __le32 Attributes; + __u32 Pad1; /* End of FILE_BASIC_INFO_INFO equivalent */ + __le64 AllocationSize; /* Beginning of FILE_STANDARD_INFO equivalent */ + __le64 EndOfFile; /* size ie offset to first free byte in file */ + __le32 NumberOfLinks; /* hard links */ + __u8 DeletePending; + __u8 Directory; + __u16 Pad2; /* End of FILE_STANDARD_INFO equivalent */ + __le64 IndexNumber; + __le32 EASize; + __le32 AccessFlags; + __le64 CurrentByteOffset; + __le32 Mode; + __le32 AlignmentRequirement; + __le32 FileNameLength; + union { + char __pad; /* Legacy structure padding */ + DECLARE_FLEX_ARRAY(char, FileName); + }; +} __packed; /* level 18 Query */ + +/* See MS-FSCC 2.4.7 */ +typedef struct file_basic_info { /* data block encoding of response to level 18 */ + __le64 CreationTime; + __le64 LastAccessTime; + __le64 LastWriteTime; + __le64 ChangeTime; + __le32 Attributes; + __u32 Pad; +} __packed FILE_BASIC_INFO; /* size info, level 0x101 */ + +/* See MS-FSCC 2.4.8 */ +typedef struct { + __le32 NextEntryOffset; + __u32 FileIndex; + __le64 CreationTime; + __le64 LastAccessTime; + __le64 LastWriteTime; + __le64 ChangeTime; + __le64 EndOfFile; + __le64 AllocationSize; + __le32 ExtFileAttributes; + __le32 FileNameLength; + __le32 EaSize; /* length of the xattrs */ + __u8 ShortNameLength; + __u8 Reserved; + __u8 ShortName[24]; + char FileName[]; +} __packed FILE_BOTH_DIRECTORY_INFO; /* level 0x104 FFrsp data */ + +/* See MS-FSCC 2.4.10 */ +typedef struct { + __le32 NextEntryOffset; + __u32 FileIndex; + __le64 CreationTime; + __le64 LastAccessTime; + __le64 LastWriteTime; + __le64 ChangeTime; + __le64 EndOfFile; + __le64 AllocationSize; + __le32 ExtFileAttributes; + __le32 FileNameLength; + char FileName[]; +} __packed FILE_DIRECTORY_INFO; /* level 0x101 FF resp data */ + +/* See MS-FSCC 2.4.14 */ +struct smb2_file_eof_info { /* encoding of request for level 10 */ + __le64 EndOfFile; /* new end of file value */ +} __packed; /* level 20 Set */ + +/* See MS-FSCC 2.4.4 */ +struct smb2_file_alloc_info { /* encoding of request for level 19 */ + __le64 AllocationSize; +} __packed; + +/* See MS-FSCC 2.4.15 */ +typedef struct { + __le32 NextEntryOffset; + __u32 FileIndex; + __le64 CreationTime; + __le64 LastAccessTime; + __le64 LastWriteTime; + __le64 ChangeTime; + __le64 EndOfFile; + __le64 AllocationSize; + __le32 ExtFileAttributes; + __le32 FileNameLength; + __le32 EaSize; /* length of the xattrs */ + char FileName[]; +} __packed FILE_FULL_DIRECTORY_INFO; /* level 0x102 rsp data */ + +/* See MS-FSCC 2.4.24 */ +typedef struct { + __le32 NextEntryOffset; + __u32 FileIndex; + __le64 CreationTime; + __le64 LastAccessTime; + __le64 LastWriteTime; + __le64 ChangeTime; + __le64 EndOfFile; + __le64 AllocationSize; + __le32 ExtFileAttributes; + __le32 FileNameLength; + __le32 EaSize; /* EA size */ + __le32 Reserved; + __le64 UniqueId; /* inode num - le since Samba puts ino in low 32 bit*/ + char FileName[]; +} __packed FILE_ID_FULL_DIR_INFO; /* level 0x105 FF rsp data */ + +/* See MS-FSCC 2.4.27 */ +struct smb2_file_internal_info { + __le64 IndexNumber; +} __packed; /* level 6 Query */ + +/* See MS-FSCC 2.4.28.2 */ +struct smb2_file_link_info { /* encoding of request for level 11 */ + /* New members MUST be added within the struct_group() macro below. */ + __struct_group(smb2_file_link_info_hdr, __hdr, __packed, + __u8 ReplaceIfExists; /* 1 = replace existing link with new */ + /* 0 = fail if link already exists */ + __u8 Reserved[7]; + __u64 RootDirectory; /* MBZ for network operations (why says spec?) */ + __le32 FileNameLength; + ); + char FileName[]; /* Name to be assigned to new link */ +} __packed; /* level 11 Set */ +static_assert(offsetof(struct smb2_file_link_info, FileName) == sizeof(struct smb2_file_link_info_hdr), + "struct member likely outside of __struct_group()"); + +/* See MS-FSCC 2.4.34 */ +struct smb2_file_network_open_info { + struct_group_attr(network_open_info, __packed, + __le64 CreationTime; + __le64 LastAccessTime; + __le64 LastWriteTime; + __le64 ChangeTime; + __le64 AllocationSize; + __le64 EndOfFile; + __le32 Attributes; + ); + __le32 Reserved; +} __packed; /* level 34 Query also similar returned in close rsp and open rsp */ + +/* See MS-FSCC 2.4.42.2 */ +struct smb2_file_rename_info { /* encoding of request for level 10 */ + /* New members MUST be added within the struct_group() macro below. */ + __struct_group(smb2_file_rename_info_hdr, __hdr, __packed, + __u8 ReplaceIfExists; /* 1 = replace existing target with new */ + /* 0 = fail if target already exists */ + __u8 Reserved[7]; + __u64 RootDirectory; /* MBZ for network operations (why says spec?) */ + __le32 FileNameLength; + ); + char FileName[]; /* New name to be assigned */ + /* padding - overall struct size must be >= 24 so filename + pad >= 6 */ +} __packed; /* level 10 Set */ +static_assert(offsetof(struct smb2_file_rename_info, FileName) == sizeof(struct smb2_file_rename_info_hdr), + "struct member likely outside of __struct_group()"); + +/* File System Information Classes */ +/* See MS-FSCC 2.5 */ +#define FS_VOLUME_INFORMATION 1 /* Query */ +#define FS_LABEL_INFORMATION 2 /* Set */ +#define FS_SIZE_INFORMATION 3 /* Query */ +#define FS_DEVICE_INFORMATION 4 /* Query */ +#define FS_ATTRIBUTE_INFORMATION 5 /* Query */ +#define FS_CONTROL_INFORMATION 6 /* Query, Set */ +#define FS_FULL_SIZE_INFORMATION 7 /* Query */ +#define FS_OBJECT_ID_INFORMATION 8 /* Query, Set */ +#define FS_DRIVER_PATH_INFORMATION 9 /* Query */ +#define FS_SECTOR_SIZE_INFORMATION 11 /* SMB3 or later. Query */ +/* See POSIX Extensions to MS-FSCC 2.3.1.1 */ +#define FS_POSIX_INFORMATION 100 /* SMB3.1.1 POSIX. Query */ + +/* See MS-FSCC 2.5.1 */ +#define MAX_FS_NAME_LEN 52 +typedef struct { + __le32 Attributes; + __le32 MaxPathNameComponentLength; + __le32 FileSystemNameLen; + __le16 FileSystemName[]; /* do not have to save this - get subset? */ +} __packed FILE_SYSTEM_ATTRIBUTE_INFO; + +/* List of FileSystemAttributes - see MS-FSCC 2.5.1 */ +#define FILE_SUPPORTS_SPARSE_VDL 0x10000000 /* faster nonsparse extend */ +#define FILE_SUPPORTS_BLOCK_REFCOUNTING 0x08000000 /* allow ioctl dup extents */ +#define FILE_SUPPORT_INTEGRITY_STREAMS 0x04000000 +#define FILE_SUPPORTS_USN_JOURNAL 0x02000000 +#define FILE_SUPPORTS_OPEN_BY_FILE_ID 0x01000000 +#define FILE_SUPPORTS_EXTENDED_ATTRIBUTES 0x00800000 +#define FILE_SUPPORTS_HARD_LINKS 0x00400000 +#define FILE_SUPPORTS_TRANSACTIONS 0x00200000 +#define FILE_SEQUENTIAL_WRITE_ONCE 0x00100000 +#define FILE_READ_ONLY_VOLUME 0x00080000 +#define FILE_NAMED_STREAMS 0x00040000 +#define FILE_SUPPORTS_ENCRYPTION 0x00020000 +#define FILE_SUPPORTS_OBJECT_IDS 0x00010000 +#define FILE_VOLUME_IS_COMPRESSED 0x00008000 +#define FILE_SUPPORTS_POSIX_UNLINK_RENAME 0x00000400 +#define FILE_RETURNS_CLEANUP_RESULT_INFO 0x00000200 +#define FILE_SUPPORTS_REMOTE_STORAGE 0x00000100 +#define FILE_SUPPORTS_REPARSE_POINTS 0x00000080 +#define FILE_SUPPORTS_SPARSE_FILES 0x00000040 +#define FILE_VOLUME_QUOTAS 0x00000020 +#define FILE_FILE_COMPRESSION 0x00000010 +#define FILE_PERSISTENT_ACLS 0x00000008 +#define FILE_UNICODE_ON_DISK 0x00000004 +#define FILE_CASE_PRESERVED_NAMES 0x00000002 +#define FILE_CASE_SENSITIVE_SEARCH 0x00000001 + +/* + * File System Control Information + * See MS-FSCC 2.5.2 + */ +struct smb2_fs_control_info { + __le64 FreeSpaceStartFiltering; + __le64 FreeSpaceThreshold; + __le64 FreeSpaceStopFiltering; + __le64 DefaultQuotaThreshold; + __le64 DefaultQuotaLimit; + __le32 FileSystemControlFlags; + __le32 Padding; +} __packed; + +/* See MS-FSCC 2.5.4 */ +struct smb2_fs_full_size_info { + __le64 TotalAllocationUnits; + __le64 CallerAvailableAllocationUnits; + __le64 ActualAvailableAllocationUnits; + __le32 SectorsPerAllocationUnit; + __le32 BytesPerSector; +} __packed; + +/* See MS-FSCC 2.5.7 */ +#define SSINFO_FLAGS_ALIGNED_DEVICE 0x00000001 +#define SSINFO_FLAGS_PARTITION_ALIGNED_ON_DEVICE 0x00000002 +#define SSINFO_FLAGS_NO_SEEK_PENALTY 0x00000004 +#define SSINFO_FLAGS_TRIM_ENABLED 0x00000008 +/* sector size info struct */ +struct smb3_fs_ss_info { + __le32 LogicalBytesPerSector; + __le32 PhysicalBytesPerSectorForAtomicity; + __le32 PhysicalBytesPerSectorForPerf; + __le32 FSEffPhysicalBytesPerSectorForAtomicity; + __le32 Flags; + __le32 ByteOffsetForSectorAlignment; + __le32 ByteOffsetForPartitionAlignment; +} __packed; + +/* See MS-FSCC 2.5.8 */ +typedef struct { + __le64 TotalAllocationUnits; + __le64 AvailableAllocationUnits; + __le32 SectorsPerAllocationUnit; + __le32 BytesPerSector; +} __packed FILE_SYSTEM_SIZE_INFO; /* size info, level 0x103 */ + +/* volume info struct - see MS-FSCC 2.5.9 */ +#define MAX_VOL_LABEL_LEN 32 +struct filesystem_vol_info { + __le64 VolumeCreationTime; + __le32 VolumeSerialNumber; + __le32 VolumeLabelLength; /* includes trailing null */ + __u8 SupportsObjects; /* True if eg like NTFS, supports objects */ + __u8 Reserved; + __u8 VolumeLabel[]; /* variable len */ +} __packed; + +/* See MS-FSCC 2.5.10 */ +typedef struct { + __le32 DeviceType; + __le32 DeviceCharacteristics; +} __packed FILE_SYSTEM_DEVICE_INFO; /* device info level 0x104 */ + +/* + * File Attributes + * See MS-FSCC 2.6 + */ +#define FILE_ATTRIBUTE_READONLY 0x00000001 +#define FILE_ATTRIBUTE_HIDDEN 0x00000002 +#define FILE_ATTRIBUTE_SYSTEM 0x00000004 +#define FILE_ATTRIBUTE_DIRECTORY 0x00000010 +#define FILE_ATTRIBUTE_ARCHIVE 0x00000020 +#define FILE_ATTRIBUTE_NORMAL 0x00000080 +#define FILE_ATTRIBUTE_TEMPORARY 0x00000100 +#define FILE_ATTRIBUTE_SPARSE_FILE 0x00000200 +#define FILE_ATTRIBUTE_REPARSE_POINT 0x00000400 +#define FILE_ATTRIBUTE_COMPRESSED 0x00000800 +#define FILE_ATTRIBUTE_OFFLINE 0x00001000 +#define FILE_ATTRIBUTE_NOT_CONTENT_INDEXED 0x00002000 +#define FILE_ATTRIBUTE_ENCRYPTED 0x00004000 +#define FILE_ATTRIBUTE_INTEGRITY_STREAM 0x00008000 +#define FILE_ATTRIBUTE_NO_SCRUB_DATA 0x00020000 +#define FILE_ATTRIBUTE_MASK (FILE_ATTRIBUTE_READONLY | FILE_ATTRIBUTE_HIDDEN | \ + FILE_ATTRIBUTE_SYSTEM | FILE_ATTRIBUTE_DIRECTORY | \ + FILE_ATTRIBUTE_ARCHIVE | FILE_ATTRIBUTE_NORMAL | \ + FILE_ATTRIBUTE_TEMPORARY | FILE_ATTRIBUTE_SPARSE_FILE | \ + FILE_ATTRIBUTE_REPARSE_POINT | FILE_ATTRIBUTE_COMPRESSED | \ + FILE_ATTRIBUTE_OFFLINE | FILE_ATTRIBUTE_NOT_CONTENT_INDEXED | \ + FILE_ATTRIBUTE_ENCRYPTED | FILE_ATTRIBUTE_INTEGRITY_STREAM | \ + FILE_ATTRIBUTE_NO_SCRUB_DATA) + +#define FILE_ATTRIBUTE_READONLY_LE cpu_to_le32(FILE_ATTRIBUTE_READONLY) +#define FILE_ATTRIBUTE_HIDDEN_LE cpu_to_le32(FILE_ATTRIBUTE_HIDDEN) +#define FILE_ATTRIBUTE_SYSTEM_LE cpu_to_le32(FILE_ATTRIBUTE_SYSTEM) +#define FILE_ATTRIBUTE_DIRECTORY_LE cpu_to_le32(FILE_ATTRIBUTE_DIRECTORY) +#define FILE_ATTRIBUTE_ARCHIVE_LE cpu_to_le32(FILE_ATTRIBUTE_ARCHIVE) +#define FILE_ATTRIBUTE_NORMAL_LE cpu_to_le32(FILE_ATTRIBUTE_NORMAL) +#define FILE_ATTRIBUTE_TEMPORARY_LE cpu_to_le32(FILE_ATTRIBUTE_TEMPORARY) +#define FILE_ATTRIBUTE_SPARSE_FILE_LE cpu_to_le32(FILE_ATTRIBUTE_SPARSE_FILE) +#define FILE_ATTRIBUTE_REPARSE_POINT_LE cpu_to_le32(FILE_ATTRIBUTE_REPARSE_POINT) +#define FILE_ATTRIBUTE_COMPRESSED_LE cpu_to_le32(FILE_ATTRIBUTE_COMPRESSED) +#define FILE_ATTRIBUTE_OFFLINE_LE cpu_to_le32(FILE_ATTRIBUTE_OFFLINE) +#define FILE_ATTRIBUTE_NOT_CONTENT_INDEXED_LE cpu_to_le32(FILE_ATTRIBUTE_NOT_CONTENT_INDEXED) +#define FILE_ATTRIBUTE_ENCRYPTED_LE cpu_to_le32(FILE_ATTRIBUTE_ENCRYPTED) +#define FILE_ATTRIBUTE_INTEGRITY_STREAM_LE cpu_to_le32(FILE_ATTRIBUTE_INTEGRITY_STREAM) +#define FILE_ATTRIBUTE_NO_SCRUB_DATA_LE cpu_to_le32(FILE_ATTRIBUTE_NO_SCRUB_DATA) +#define FILE_ATTRIBUTE_MASK_LE cpu_to_le32(FILE_ATTRIBUTE_MASK) + +/* + * SMB2 Notify Action Flags + * See MS-FSCC 2.7.1 + */ +#define FILE_ACTION_ADDED 0x00000001 +#define FILE_ACTION_REMOVED 0x00000002 +#define FILE_ACTION_MODIFIED 0x00000003 +#define FILE_ACTION_RENAMED_OLD_NAME 0x00000004 +#define FILE_ACTION_RENAMED_NEW_NAME 0x00000005 +#define FILE_ACTION_ADDED_STREAM 0x00000006 +#define FILE_ACTION_REMOVED_STREAM 0x00000007 +#define FILE_ACTION_MODIFIED_STREAM 0x00000008 +#define FILE_ACTION_REMOVED_BY_DELETE 0x00000009 +#define FILE_ACTION_ID_NOT_TUNNELLED 0x0000000A +#define FILE_ACTION_TUNNELLED_ID_COLLISION 0x0000000B + +/* + * Response contains array of the following structures + * See MS-FSCC 2.7.1 + */ +struct file_notify_information { + __le32 NextEntryOffset; + __le32 Action; + __le32 FileNameLength; + __u8 FileName[]; +} __packed; + +/* + * See POSIX Extensions to MS-FSCC 2.3.2.1 + * Link: https://gitlab.com/samba-team/smb3-posix-spec/-/blob/master/fscc_posix_extensions.md + */ +typedef struct { + /* For undefined recommended transfer size return -1 in that field */ + __le32 OptimalTransferSize; /* bsize on some os, iosize on other os */ + __le32 BlockSize; + /* The next three fields are in terms of the block size. + * (above). If block size is unknown, 4096 would be a + * reasonable block size for a server to report. + * Note that returning the blocks/blocksavail removes need + * to make a second call (to QFSInfo level 0x103 to get this info. + * UserBlockAvail is typically less than or equal to BlocksAvail, + * if no distinction is made return the same value in each + */ + __le64 TotalBlocks; + __le64 BlocksAvail; /* bfree */ + __le64 UserBlocksAvail; /* bavail */ + /* For undefined Node fields or FSID return -1 */ + __le64 TotalFileNodes; + __le64 FreeFileNodes; + __le64 FileSysIdentifier; /* fsid */ + /* NB Namelen comes from FILE_SYSTEM_ATTRIBUTE_INFO call */ + /* NB flags can come from FILE_SYSTEM_DEVICE_INFO call */ +} __packed FILE_SYSTEM_POSIX_INFO; + +#endif /* _COMMON_SMB_FSCC_H */ diff --git a/fs/smb/common/md4.h b/fs/smb/common/md4.h new file mode 100644 index 000000000..5337becc6 --- /dev/null +++ b/fs/smb/common/md4.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Common values for ARC4 Cipher Algorithm + */ + +#ifndef _CIFS_MD4_H +#define _CIFS_MD4_H + +#include + +#define MD4_DIGEST_SIZE 16 +#define MD4_HMAC_BLOCK_SIZE 64 +#define MD4_BLOCK_WORDS 16 +#define MD4_HASH_WORDS 4 + +struct md4_ctx { + u32 hash[MD4_HASH_WORDS]; + u32 block[MD4_BLOCK_WORDS]; + u64 byte_count; +}; + + +int cifs_md4_init(struct md4_ctx *mctx); +int cifs_md4_update(struct md4_ctx *mctx, const u8 *data, unsigned int len); +int cifs_md4_final(struct md4_ctx *mctx, u8 *out); + +#endif /* _CIFS_MD4_H */ diff --git a/fs/smb/common/smb1pdu.h b/fs/smb/common/smb1pdu.h new file mode 100644 index 000000000..df6d4e11a --- /dev/null +++ b/fs/smb/common/smb1pdu.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +/* + * + * Copyright (C) International Business Machines Corp., 2002,2009 + * 2018 Samsung Electronics Co., Ltd. + * Author(s): Steve French + * Namjae Jeon + * + */ + +#ifndef _COMMON_SMB1_PDU_H +#define _COMMON_SMB1_PDU_H + +#define SMB1_PROTO_NUMBER cpu_to_le32(0x424d53ff) + +/* + * See MS-CIFS 2.2.3.1 + * MS-SMB 2.2.3.1 + */ +struct smb_hdr { + __u8 Protocol[4]; + __u8 Command; + union { + struct { + __u8 ErrorClass; + __u8 Reserved; + __le16 Error; + } __packed DosError; + __le32 CifsError; + } __packed Status; + __u8 Flags; + __le16 Flags2; /* note: le */ + __le16 PidHigh; + union { + struct { + __le32 SequenceNumber; /* le */ + __u32 Reserved; /* zero */ + } __packed Sequence; + __u8 SecuritySignature[8]; /* le */ + } __packed Signature; + __u8 pad[2]; + __u16 Tid; + __le16 Pid; + __u16 Uid; + __le16 Mid; + __u8 WordCount; +} __packed; + +/* See MS-CIFS 2.2.4.52.1 */ +typedef struct smb_negotiate_req { + struct smb_hdr hdr; /* wct = 0 */ + __le16 ByteCount; + unsigned char DialectsArray[]; +} __packed SMB_NEGOTIATE_REQ; + +#endif /* _COMMON_SMB1_PDU_H */ diff --git a/fs/smb/common/smb2pdu.h b/fs/smb/common/smb2pdu.h new file mode 100644 index 000000000..f9a8862cb --- /dev/null +++ b/fs/smb/common/smb2pdu.h @@ -0,0 +1,1836 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +#ifndef _COMMON_SMB2PDU_H +#define _COMMON_SMB2PDU_H + +#include +#include + +/* + * Note that, due to trying to use names similar to the protocol specifications, + * there are many mixed case field names in the structures below. Although + * this does not match typical Linux kernel style, it is necessary to be + * able to match against the protocol specification. + * + * SMB2 commands + * Some commands have minimal (wct=0,bcc=0), or uninteresting, responses + * (ie no useful data other than the SMB error code itself) and are marked such. + * Knowing this helps avoid response buffer allocations and copy in some cases. + */ + +/* List of commands in host endian */ +#define SMB2_NEGOTIATE_HE 0x0000 +#define SMB2_SESSION_SETUP_HE 0x0001 +#define SMB2_LOGOFF_HE 0x0002 /* trivial request/resp */ +#define SMB2_TREE_CONNECT_HE 0x0003 +#define SMB2_TREE_DISCONNECT_HE 0x0004 /* trivial req/resp */ +#define SMB2_CREATE_HE 0x0005 +#define SMB2_CLOSE_HE 0x0006 +#define SMB2_FLUSH_HE 0x0007 /* trivial resp */ +#define SMB2_READ_HE 0x0008 +#define SMB2_WRITE_HE 0x0009 +#define SMB2_LOCK_HE 0x000A +#define SMB2_IOCTL_HE 0x000B +#define SMB2_CANCEL_HE 0x000C +#define SMB2_ECHO_HE 0x000D +#define SMB2_QUERY_DIRECTORY_HE 0x000E +#define SMB2_CHANGE_NOTIFY_HE 0x000F +#define SMB2_QUERY_INFO_HE 0x0010 +#define SMB2_SET_INFO_HE 0x0011 +#define SMB2_OPLOCK_BREAK_HE 0x0012 +#define SMB2_SERVER_TO_CLIENT_NOTIFICATION 0x0013 + +/* The same list in little endian */ +#define SMB2_NEGOTIATE cpu_to_le16(SMB2_NEGOTIATE_HE) +#define SMB2_SESSION_SETUP cpu_to_le16(SMB2_SESSION_SETUP_HE) +#define SMB2_LOGOFF cpu_to_le16(SMB2_LOGOFF_HE) +#define SMB2_TREE_CONNECT cpu_to_le16(SMB2_TREE_CONNECT_HE) +#define SMB2_TREE_DISCONNECT cpu_to_le16(SMB2_TREE_DISCONNECT_HE) +#define SMB2_CREATE cpu_to_le16(SMB2_CREATE_HE) +#define SMB2_CLOSE cpu_to_le16(SMB2_CLOSE_HE) +#define SMB2_FLUSH cpu_to_le16(SMB2_FLUSH_HE) +#define SMB2_READ cpu_to_le16(SMB2_READ_HE) +#define SMB2_WRITE cpu_to_le16(SMB2_WRITE_HE) +#define SMB2_LOCK cpu_to_le16(SMB2_LOCK_HE) +#define SMB2_IOCTL cpu_to_le16(SMB2_IOCTL_HE) +#define SMB2_CANCEL cpu_to_le16(SMB2_CANCEL_HE) +#define SMB2_ECHO cpu_to_le16(SMB2_ECHO_HE) +#define SMB2_QUERY_DIRECTORY cpu_to_le16(SMB2_QUERY_DIRECTORY_HE) +#define SMB2_CHANGE_NOTIFY cpu_to_le16(SMB2_CHANGE_NOTIFY_HE) +#define SMB2_QUERY_INFO cpu_to_le16(SMB2_QUERY_INFO_HE) +#define SMB2_SET_INFO cpu_to_le16(SMB2_SET_INFO_HE) +#define SMB2_OPLOCK_BREAK cpu_to_le16(SMB2_OPLOCK_BREAK_HE) + +#define SMB2_INTERNAL_CMD cpu_to_le16(0xFFFF) + +#define NUMBER_OF_SMB2_COMMANDS 0x0013 + +/* + * Size of the session key (crypto key encrypted with the password + */ +#define SMB2_NTLMV2_SESSKEY_SIZE 16 +#define SMB2_SIGNATURE_SIZE 16 +#define SMB2_HMACSHA256_SIZE 32 +#define SMB2_CMACAES_SIZE 16 +#define SMB3_GCM128_CRYPTKEY_SIZE 16 +#define SMB3_GCM256_CRYPTKEY_SIZE 32 + +/* + * Size of the smb3 encryption/decryption keys + * This size is big enough to store any cipher key types. + */ +#define SMB3_ENC_DEC_KEY_SIZE 32 + +/* + * Size of the smb3 signing key + */ +#define SMB3_SIGN_KEY_SIZE 16 + +#define CIFS_CLIENT_CHALLENGE_SIZE 8 + +/* Maximum buffer size value we can send with 1 credit */ +#define SMB2_MAX_BUFFER_SIZE 65536 + +/* + * The default wsize is 1M for SMB2 (and for some CIFS cases). + * find_get_pages seems to return a maximum of 256 + * pages in a single call. With PAGE_SIZE == 4k, this means we can + * fill a single wsize request with a single call. + */ +#define SMB3_DEFAULT_IOSIZE (4 * 1024 * 1024) + +/* According to MS-SMB2 specification The minimum recommended value is 65536.*/ +#define CIFS_MIN_DEFAULT_IOSIZE (65536) + +/* + * SMB2 Header Definition + * + * "MBZ" : Must be Zero + * "BB" : BugBug, Something to check/review/analyze later + * "PDU" : "Protocol Data Unit" (ie a network "frame") + * + */ + +#define __SMB2_HEADER_STRUCTURE_SIZE 64 +#define SMB2_HEADER_STRUCTURE_SIZE \ + cpu_to_le16(__SMB2_HEADER_STRUCTURE_SIZE) + +#define SMB2_PROTO_NUMBER cpu_to_le32(0x424d53fe) +#define SMB2_TRANSFORM_PROTO_NUM cpu_to_le32(0x424d53fd) +#define SMB2_COMPRESSION_TRANSFORM_ID cpu_to_le32(0x424d53fc) + +/* + * SMB2 flag definitions + */ +#define SMB2_FLAGS_SERVER_TO_REDIR cpu_to_le32(0x00000001) +#define SMB2_FLAGS_ASYNC_COMMAND cpu_to_le32(0x00000002) +#define SMB2_FLAGS_RELATED_OPERATIONS cpu_to_le32(0x00000004) +#define SMB2_FLAGS_SIGNED cpu_to_le32(0x00000008) +#define SMB2_FLAGS_PRIORITY_MASK cpu_to_le32(0x00000070) /* SMB3.1.1 */ +#define SMB2_FLAGS_DFS_OPERATIONS cpu_to_le32(0x10000000) +#define SMB2_FLAGS_REPLAY_OPERATION cpu_to_le32(0x20000000) /* SMB3 & up */ + +/* + * Definitions for SMB2 Protocol Data Units (network frames) + * + * See MS-SMB2.PDF specification for protocol details. + * The Naming convention is the lower case version of the SMB2 + * command code name for the struct. Note that structures must be packed. + * + */ + +/* See MS-SMB2 section 2.2.1 */ +struct smb2_hdr { + __le32 ProtocolId; /* 0xFE 'S' 'M' 'B' */ + __le16 StructureSize; /* 64 */ + __le16 CreditCharge; /* MBZ */ + __le32 Status; /* Error from server */ + __le16 Command; + __le16 CreditRequest; /* CreditResponse */ + __le32 Flags; + __le32 NextCommand; + __le64 MessageId; + union { + struct { + __le32 ProcessId; + __le32 TreeId; + } __packed SyncId; + __le64 AsyncId; + } __packed Id; + __le64 SessionId; + __u8 Signature[16]; +} __packed; + +struct smb3_hdr_req { + __le32 ProtocolId; /* 0xFE 'S' 'M' 'B' */ + __le16 StructureSize; /* 64 */ + __le16 CreditCharge; /* MBZ */ + __le16 ChannelSequence; /* See MS-SMB2 3.2.4.1 and 3.2.7.1 */ + __le16 Reserved; + __le16 Command; + __le16 CreditRequest; /* CreditResponse */ + __le32 Flags; + __le32 NextCommand; + __le64 MessageId; + union { + struct { + __le32 ProcessId; + __le32 TreeId; + } __packed SyncId; + __le64 AsyncId; + } __packed Id; + __le64 SessionId; + __u8 Signature[16]; +} __packed; + +struct smb2_pdu { + struct smb2_hdr hdr; + __le16 StructureSize2; /* size of wct area (varies, request specific) */ +} __packed; + +#define SMB2_ERROR_STRUCTURE_SIZE2 9 +#define SMB2_ERROR_STRUCTURE_SIZE2_LE cpu_to_le16(SMB2_ERROR_STRUCTURE_SIZE2) + +struct smb2_err_rsp { + struct smb2_hdr hdr; + __le16 StructureSize; + __u8 ErrorContextCount; + __u8 Reserved; + __le32 ByteCount; /* even if zero, at least one byte follows */ + __u8 ErrorData[]; /* variable length */ +} __packed; + +#define SMB3_AES_CCM_NONCE 11 +#define SMB3_AES_GCM_NONCE 12 + +/* Transform flags (for 3.0 dialect this flag indicates CCM */ +#define TRANSFORM_FLAG_ENCRYPTED 0x0001 +struct smb2_transform_hdr { + __le32 ProtocolId; /* 0xFD 'S' 'M' 'B' */ + __u8 Signature[16]; + __u8 Nonce[16]; + __le32 OriginalMessageSize; + __u16 Reserved1; + __le16 Flags; /* EncryptionAlgorithm for 3.0, enc enabled for 3.1.1 */ + __le64 SessionId; +} __packed; + +/* + * These are simplified versions from the spec, as we don't need a fully fledged + * form of both unchained and chained structs. + * + * For chained payloads, only the first 8 bytes belong to the transform header. + * CompressionAlgorithm, Flags and Offset below overlay the first chained + * payload header, where Offset represents Length. + * + * See MS-SMB2 2.2.42 for more details. + */ +#define SMB2_COMPRESSION_FLAG_NONE 0x0000 +#define SMB2_COMPRESSION_FLAG_CHAINED 0x0001 + +struct smb2_compression_hdr { + __le32 ProtocolId; /* 0xFC 'S' 'M' 'B' */ + __le32 OriginalCompressedSegmentSize; + __le16 CompressionAlgorithm; + __le16 Flags; + __le32 Offset; /* this is the size of the uncompressed SMB2 header below */ + /* uncompressed SMB2 header (READ or WRITE) goes here */ + /* compressed data goes here */ +} __packed; + +/* + * ... OTOH, set compression payload header to always have OriginalPayloadSize + * as it's easier to pass the struct size minus sizeof(OriginalPayloadSize) + * than to juggle around the header/data memory. + */ +struct smb2_compression_payload_hdr { + __le16 CompressionAlgorithm; + __le16 Flags; + __le32 Length; /* length of compressed playload including field below if present */ + __le32 OriginalPayloadSize; /* accounted when LZNT1, LZ77, LZ77+Huffman */ +} __packed; + +struct smb2_compression_pattern_v1 { + __u8 Pattern; + __u8 Reserved1; + __le16 Reserved2; + __le32 Repetitions; +} __packed; + +/* See MS-SMB2 section 2.2.9.2 */ +/* Context Types */ +#define SMB2_RESERVED_TREE_CONNECT_CONTEXT_ID 0x0000 +#define SMB2_REMOTED_IDENTITY_TREE_CONNECT_CONTEXT_ID cpu_to_le16(0x0001) + +struct tree_connect_contexts { + __le16 ContextType; + __le16 DataLength; + __le32 Reserved; + __u8 Data[]; +} __packed; + +/* Remoted identity tree connect context structures - see MS-SMB2 2.2.9.2.1 */ +struct smb3_blob_data { + __le16 BlobSize; + __u8 BlobData[]; +} __packed; + +/* Valid values for Attr */ +#define SE_GROUP_MANDATORY 0x00000001 +#define SE_GROUP_ENABLED_BY_DEFAULT 0x00000002 +#define SE_GROUP_ENABLED 0x00000004 +#define SE_GROUP_OWNER 0x00000008 +#define SE_GROUP_USE_FOR_DENY_ONLY 0x00000010 +#define SE_GROUP_INTEGRITY 0x00000020 +#define SE_GROUP_INTEGRITY_ENABLED 0x00000040 +#define SE_GROUP_RESOURCE 0x20000000 +#define SE_GROUP_LOGON_ID 0xC0000000 + +struct sid_array_data { + __le16 SidAttrCount; + /* SidAttrList - array of sid_attr_data structs */ +} __packed; + +/* struct sid_attr_data is SidData array in BlobData format then le32 Attr */ +struct sid_attr_data { + __le16 BlobSize; + __u8 BlobData[]; + /* __le32 Attr */ +} __packed; + +/* + * struct privilege_data is the same as BLOB_DATA - see MS-SMB2 2.2.9.2.1.5 + * but with size of LUID_ATTR_DATA struct and BlobData set to LUID_ATTR DATA + */ + +struct privilege_array_data { + __le16 PrivilegeCount; + /* array of privilege_data structs */ +} __packed; + +struct remoted_identity_tcon_context { + __le16 TicketType; /* must be 0x0001 */ + __le16 TicketSize; /* total size of this struct */ + __le16 User; /* offset to SID_ATTR_DATA struct with user info */ + __le16 UserName; /* offset to null terminated Unicode username string */ + __le16 Domain; /* offset to null terminated Unicode domain name */ + __le16 Groups; /* offset to SID_ARRAY_DATA struct with group info */ + __le16 RestrictedGroups; /* similar to above */ + __le16 Privileges; /* offset to PRIVILEGE_ARRAY_DATA struct */ + __le16 PrimaryGroup; /* offset to SID_ARRAY_DATA struct */ + __le16 Owner; /* offset to BLOB_DATA struct */ + __le16 DefaultDacl; /* offset to BLOB_DATA struct */ + __le16 DeviceGroups; /* offset to SID_ARRAY_DATA struct */ + __le16 UserClaims; /* offset to BLOB_DATA struct */ + __le16 DeviceClaims; /* offset to BLOB_DATA struct */ + __u8 TicketInfo[]; /* variable length buf - remoted identity data */ +} __packed; + +struct smb2_tree_connect_req_extension { + __le32 TreeConnectContextOffset; + __le16 TreeConnectContextCount; + __u8 Reserved[10]; + __u8 PathName[]; /* variable sized array */ + /* followed by array of TreeConnectContexts */ +} __packed; + +/* Flags/Reserved for SMB3.1.1 */ +#define SMB2_TREE_CONNECT_FLAG_CLUSTER_RECONNECT cpu_to_le16(0x0001) +#define SMB2_TREE_CONNECT_FLAG_REDIRECT_TO_OWNER cpu_to_le16(0x0002) +#define SMB2_TREE_CONNECT_FLAG_EXTENSION_PRESENT cpu_to_le16(0x0004) + +struct smb2_tree_connect_req { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 9 */ + __le16 Flags; /* Flags in SMB3.1.1 */ + __le16 PathOffset; + __le16 PathLength; + __u8 Buffer[]; /* variable length */ +} __packed; + +/* Possible ShareType values */ +#define SMB2_SHARE_TYPE_DISK 0x01 +#define SMB2_SHARE_TYPE_PIPE 0x02 +#define SMB2_SHARE_TYPE_PRINT 0x03 + +/* + * Possible ShareFlags - exactly one and only one of the first 4 caching flags + * must be set (any of the remaining, SHI1005, flags may be set individually + * or in combination. + */ +#define SMB2_SHAREFLAG_MANUAL_CACHING 0x00000000 +#define SMB2_SHAREFLAG_AUTO_CACHING 0x00000010 +#define SMB2_SHAREFLAG_VDO_CACHING 0x00000020 +#define SMB2_SHAREFLAG_NO_CACHING 0x00000030 +#define SHI1005_FLAGS_DFS 0x00000001 +#define SHI1005_FLAGS_DFS_ROOT 0x00000002 +#define SMB2_SHAREFLAG_RESTRICT_EXCLUSIVE_OPENS 0x00000100 +#define SMB2_SHAREFLAG_FORCE_SHARED_DELETE 0x00000200 +#define SMB2_SHAREFLAG_ALLOW_NAMESPACE_CACHING 0x00000400 +#define SMB2_SHAREFLAG_ACCESS_BASED_DIRECTORY_ENUM 0x00000800 +#define SMB2_SHAREFLAG_FORCE_LEVELII_OPLOCK 0x00001000 +#define SMB2_SHAREFLAG_ENABLE_HASH_V1 0x00002000 +#define SMB2_SHAREFLAG_ENABLE_HASH_V2 0x00004000 +#define SMB2_SHAREFLAG_ENCRYPT_DATA 0x00008000 +#define SHI1005_FLAGS_ENCRYPT_DATA SMB2_SHAREFLAG_ENCRYPT_DATA +#define SMB2_SHAREFLAG_IDENTITY_REMOTING 0x00040000 /* 3.1.1 */ +#define SMB2_SHAREFLAG_COMPRESS_DATA 0x00100000 /* 3.1.1 */ +#define SMB2_SHAREFLAG_ISOLATED_TRANSPORT 0x00200000 +#define SHI1005_FLAGS_ALL 0x0034FF33 + +/* Possible share capabilities */ +#define SMB2_SHARE_CAP_DFS cpu_to_le32(0x00000008) /* all dialects */ +#define SMB2_SHARE_CAP_CONTINUOUS_AVAILABILITY cpu_to_le32(0x00000010) /* 3.0 */ +#define SMB2_SHARE_CAP_SCALEOUT cpu_to_le32(0x00000020) /* 3.0 */ +#define SMB2_SHARE_CAP_CLUSTER cpu_to_le32(0x00000040) /* 3.0 */ +#define SMB2_SHARE_CAP_ASYMMETRIC cpu_to_le32(0x00000080) /* 3.02 */ +#define SMB2_SHARE_CAP_REDIRECT_TO_OWNER cpu_to_le32(0x00000100) /* 3.1.1 */ + +struct smb2_tree_connect_rsp { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 16 */ + __u8 ShareType; /* see below */ + __u8 Reserved; + __le32 ShareFlags; /* see below */ + __le32 Capabilities; /* see below */ + __le32 MaximalAccess; +} __packed; + +struct smb2_tree_disconnect_req { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 4 */ + __le16 Reserved; +} __packed; + +struct smb2_tree_disconnect_rsp { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 4 */ + __le16 Reserved; +} __packed; + + +/* + * SMB2_NEGOTIATE_PROTOCOL See MS-SMB2 section 2.2.3 + */ +/* SecurityMode flags */ +#define SMB2_NEGOTIATE_SIGNING_ENABLED 0x0001 +#define SMB2_NEGOTIATE_SIGNING_ENABLED_LE cpu_to_le16(0x0001) +#define SMB2_NEGOTIATE_SIGNING_REQUIRED 0x0002 +#define SMB2_NEGOTIATE_SIGNING_REQUIRED_LE cpu_to_le16(0x0002) +#define SMB2_SEC_MODE_FLAGS_ALL 0x0003 + +/* Capabilities flags */ +#define SMB2_GLOBAL_CAP_DFS 0x00000001 +#define SMB2_GLOBAL_CAP_LEASING 0x00000002 /* Resp only New to SMB2.1 */ +#define SMB2_GLOBAL_CAP_LARGE_MTU 0x00000004 /* Resp only New to SMB2.1 */ +#define SMB2_GLOBAL_CAP_MULTI_CHANNEL 0x00000008 /* New to SMB3 */ +#define SMB2_GLOBAL_CAP_PERSISTENT_HANDLES 0x00000010 /* New to SMB3 */ +#define SMB2_GLOBAL_CAP_DIRECTORY_LEASING 0x00000020 /* New to SMB3 */ +#define SMB2_GLOBAL_CAP_ENCRYPTION 0x00000040 /* New to SMB3 */ +#define SMB2_GLOBAL_CAP_NOTIFICATIONS 0x00000080 /* New to SMB3.1.1 */ +/* Internal types */ +#define SMB2_NT_FIND 0x00100000 +#define SMB2_LARGE_FILES 0x00200000 + +#define SMB2_CLIENT_GUID_SIZE 16 +#define SMB2_CREATE_GUID_SIZE 16 + +/* Dialects */ +#define SMB10_PROT_ID 0x0000 /* local only, not sent on wire w/CIFS negprot */ +#define SMB20_PROT_ID 0x0202 +#define SMB21_PROT_ID 0x0210 +#define SMB2X_PROT_ID 0x02FF +#define SMB30_PROT_ID 0x0300 +#define SMB302_PROT_ID 0x0302 +#define SMB311_PROT_ID 0x0311 +#define BAD_PROT_ID 0xFFFF + +#define SMB311_SALT_SIZE 32 +/* Hash Algorithm Types */ +#define SMB2_PREAUTH_INTEGRITY_SHA512 cpu_to_le16(0x0001) +#define SMB2_PREAUTH_HASH_SIZE 64 + +/* Negotiate Contexts - ContextTypes. See MS-SMB2 section 2.2.3.1 for details */ +#define SMB2_PREAUTH_INTEGRITY_CAPABILITIES cpu_to_le16(1) +#define SMB2_ENCRYPTION_CAPABILITIES cpu_to_le16(2) +#define SMB2_COMPRESSION_CAPABILITIES cpu_to_le16(3) +#define SMB2_NETNAME_NEGOTIATE_CONTEXT_ID cpu_to_le16(5) +#define SMB2_TRANSPORT_CAPABILITIES cpu_to_le16(6) +#define SMB2_RDMA_TRANSFORM_CAPABILITIES cpu_to_le16(7) +#define SMB2_SIGNING_CAPABILITIES cpu_to_le16(8) +#define SMB2_POSIX_EXTENSIONS_AVAILABLE cpu_to_le16(0x100) + +struct smb2_neg_context { + __le16 ContextType; + __le16 DataLength; + __le32 Reserved; + /* Followed by array of data. NOTE: some servers require padding to 8 byte boundary */ +} __packed; + +/* + * SaltLength that the server send can be zero, so the only three required + * fields (all __le16) end up six bytes total, so the minimum context data len + * in the response is six bytes which accounts for + * + * HashAlgorithmCount, SaltLength, and 1 HashAlgorithm. + */ +#define MIN_PREAUTH_CTXT_DATA_LEN 6 + +struct smb2_preauth_neg_context { + __le16 ContextType; /* 1 */ + __le16 DataLength; + __le32 Reserved; + __le16 HashAlgorithmCount; /* 1 */ + __le16 SaltLength; + __le16 HashAlgorithms; /* HashAlgorithms[0] since only one defined */ + __u8 Salt[SMB311_SALT_SIZE]; +} __packed; + +/* Encryption Algorithms Ciphers */ +#define SMB2_ENCRYPTION_AES128_CCM cpu_to_le16(0x0001) +#define SMB2_ENCRYPTION_AES128_GCM cpu_to_le16(0x0002) +#define SMB2_ENCRYPTION_AES256_CCM cpu_to_le16(0x0003) +#define SMB2_ENCRYPTION_AES256_GCM cpu_to_le16(0x0004) + +/* Min encrypt context data is one cipher so 2 bytes + 2 byte count field */ +#define MIN_ENCRYPT_CTXT_DATA_LEN 4 +struct smb2_encryption_neg_context { + __le16 ContextType; /* 2 */ + __le16 DataLength; + __le32 Reserved; + /* CipherCount usually 2, but can be 3 when AES256-GCM enabled */ + __le16 CipherCount; /* AES128-GCM and AES128-CCM by default */ + __le16 Ciphers[]; +} __packed; + +/* See MS-SMB2 2.2.3.1.3 */ +#define SMB3_COMPRESS_NONE cpu_to_le16(0x0000) +#define SMB3_COMPRESS_LZNT1 cpu_to_le16(0x0001) +#define SMB3_COMPRESS_LZ77 cpu_to_le16(0x0002) +#define SMB3_COMPRESS_LZ77_HUFF cpu_to_le16(0x0003) +/* Pattern scanning algorithm See MS-SMB2 3.1.4.4.1 */ +#define SMB3_COMPRESS_PATTERN cpu_to_le16(0x0004) /* Pattern_V1 */ +#define SMB3_COMPRESS_LZ4 cpu_to_le16(0x0005) +/* Account for NONE for easier array indexing */ +#define SMB3_COMPRESS_MAX_ALGS 6 + +/* Compression Flags */ +#define SMB2_COMPRESSION_CAPABILITIES_FLAG_NONE cpu_to_le32(0x00000000) +#define SMB2_COMPRESSION_CAPABILITIES_FLAG_CHAINED cpu_to_le32(0x00000001) + +struct smb2_compression_capabilities_context { + __le16 ContextType; /* 3 */ + __le16 DataLength; + __le32 Reserved; + __le16 CompressionAlgorithmCount; + __le16 Padding; + __le32 Flags; + __le16 CompressionAlgorithms[4]; +} __packed; + +/* + * For smb2_netname_negotiate_context_id See MS-SMB2 2.2.3.1.4. + * Its struct simply contains NetName, an array of Unicode characters + */ +struct smb2_netname_neg_context { + __le16 ContextType; /* 5 */ + __le16 DataLength; + __le32 Reserved; + __le16 NetName[]; /* hostname of target converted to UCS-2 */ +} __packed; + +/* + * For smb2_transport_capabilities context see MS-SMB2 2.2.3.1.5 + * and 2.2.4.1.5 + */ + +/* Flags */ +#define SMB2_ACCEPT_TRANSPORT_LEVEL_SECURITY 0x00000001 + +struct smb2_transport_capabilities_context { + __le16 ContextType; /* 6 */ + __le16 DataLength; + __u32 Reserved; + __le32 Flags; + __u32 Pad; +} __packed; + +/* + * For rdma transform capabilities context see MS-SMB2 2.2.3.1.6 + * and 2.2.4.1.6 + */ + +/* RDMA Transform IDs */ +#define SMB2_RDMA_TRANSFORM_NONE 0x0000 +#define SMB2_RDMA_TRANSFORM_ENCRYPTION 0x0001 +#define SMB2_RDMA_TRANSFORM_SIGNING 0x0002 + +struct smb2_rdma_transform_capabilities_context { + __le16 ContextType; /* 7 */ + __le16 DataLength; + __u32 Reserved; + __le16 TransformCount; + __u16 Reserved1; + __u32 Reserved2; + __le16 RDMATransformIds[]; +} __packed; + +/* + * For signing capabilities context see MS-SMB2 2.2.3.1.7 + * and 2.2.4.1.7 + */ + +/* Signing algorithms */ +#define SIGNING_ALG_HMAC_SHA256 0 +#define SIGNING_ALG_HMAC_SHA256_LE cpu_to_le16(0) +#define SIGNING_ALG_AES_CMAC 1 +#define SIGNING_ALG_AES_CMAC_LE cpu_to_le16(1) +#define SIGNING_ALG_AES_GMAC 2 +#define SIGNING_ALG_AES_GMAC_LE cpu_to_le16(2) + +struct smb2_signing_capabilities { + __le16 ContextType; /* 8 */ + __le16 DataLength; + __le32 Reserved; + __le16 SigningAlgorithmCount; + __le16 SigningAlgorithms[]; + /* Followed by padding to 8 byte boundary (required by some servers) */ +} __packed; + +#define POSIX_CTXT_DATA_LEN 16 +struct smb2_posix_neg_context { + __le16 ContextType; /* 0x100 */ + __le16 DataLength; + __le32 Reserved; + __u8 Name[16]; /* POSIX ctxt GUID 93AD25509CB411E7B42383DE968BCD7C */ +} __packed; + +struct smb2_negotiate_req { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 36 */ + __le16 DialectCount; + __le16 SecurityMode; + __le16 Reserved; /* MBZ */ + __le32 Capabilities; + __u8 ClientGUID[SMB2_CLIENT_GUID_SIZE]; + /* In SMB3.02 and earlier next three were MBZ le64 ClientStartTime */ + __le32 NegotiateContextOffset; /* SMB3.1.1 only. MBZ earlier */ + __le16 NegotiateContextCount; /* SMB3.1.1 only. MBZ earlier */ + __le16 Reserved2; + __le16 Dialects[]; +} __packed; + +struct smb2_negotiate_rsp { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 65 */ + __le16 SecurityMode; + __le16 DialectRevision; + __le16 NegotiateContextCount; /* Prior to SMB3.1.1 was Reserved & MBZ */ + __u8 ServerGUID[16]; + __le32 Capabilities; + __le32 MaxTransactSize; + __le32 MaxReadSize; + __le32 MaxWriteSize; + __le64 SystemTime; /* MBZ */ + __le64 ServerStartTime; + __le16 SecurityBufferOffset; + __le16 SecurityBufferLength; + __le32 NegotiateContextOffset; /* Pre:SMB3.1.1 was reserved/ignored */ + __u8 Buffer[]; /* variable length GSS security buffer */ +} __packed; + + +/* + * SMB2_SESSION_SETUP See MS-SMB2 section 2.2.5 + */ +/* Flags */ +#define SMB2_SESSION_REQ_FLAG_BINDING 0x01 +#define SMB2_SESSION_REQ_FLAG_ENCRYPT_DATA 0x04 + +struct smb2_sess_setup_req { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 25 */ + __u8 Flags; + __u8 SecurityMode; + __le32 Capabilities; + __le32 Channel; + __le16 SecurityBufferOffset; + __le16 SecurityBufferLength; + __le64 PreviousSessionId; + __u8 Buffer[]; /* variable length GSS security buffer */ +} __packed; + +/* Currently defined SessionFlags */ +#define SMB2_SESSION_FLAG_IS_GUEST 0x0001 +#define SMB2_SESSION_FLAG_IS_GUEST_LE cpu_to_le16(0x0001) +#define SMB2_SESSION_FLAG_IS_NULL 0x0002 +#define SMB2_SESSION_FLAG_IS_NULL_LE cpu_to_le16(0x0002) +#define SMB2_SESSION_FLAG_ENCRYPT_DATA 0x0004 +#define SMB2_SESSION_FLAG_ENCRYPT_DATA_LE cpu_to_le16(0x0004) + +struct smb2_sess_setup_rsp { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 9 */ + __le16 SessionFlags; + __le16 SecurityBufferOffset; + __le16 SecurityBufferLength; + __u8 Buffer[]; /* variable length GSS security buffer */ +} __packed; + + +/* + * SMB2_LOGOFF See MS-SMB2 section 2.2.7 + */ +struct smb2_logoff_req { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 4 */ + __le16 Reserved; +} __packed; + +struct smb2_logoff_rsp { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 4 */ + __le16 Reserved; +} __packed; + + +/* + * SMB2_CLOSE See MS-SMB2 section 2.2.15 + */ +/* Currently defined values for close flags */ +#define SMB2_CLOSE_FLAG_POSTQUERY_ATTRIB cpu_to_le16(0x0001) +struct smb2_close_req { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 24 */ + __le16 Flags; + __le32 Reserved; + __u64 PersistentFileId; /* opaque endianness */ + __u64 VolatileFileId; /* opaque endianness */ +} __packed; + +/* + * Maximum size of a SMB2_CLOSE response is 64 (smb2 header) + 60 (data) + */ +#define MAX_SMB2_CLOSE_RESPONSE_SIZE 124 + +struct smb2_close_rsp { + struct smb2_hdr hdr; + __le16 StructureSize; /* 60 */ + __le16 Flags; + __le32 Reserved; + struct_group_attr(network_open_info, __packed, + __le64 CreationTime; + __le64 LastAccessTime; + __le64 LastWriteTime; + __le64 ChangeTime; + /* Beginning of FILE_STANDARD_INFO equivalent */ + __le64 AllocationSize; + __le64 EndOfFile; + __le32 Attributes; + ); +} __packed; + + +/* + * SMB2_READ See MS-SMB2 section 2.2.19 + */ +/* For read request Flags field below, following flag is defined for SMB3.02 */ +#define SMB2_READFLAG_READ_UNBUFFERED 0x01 +#define SMB2_READFLAG_REQUEST_COMPRESSED 0x02 /* See MS-SMB2 2.2.19 */ + +/* Channel field for read and write: exactly one of following flags can be set*/ +#define SMB2_CHANNEL_NONE cpu_to_le32(0x00000000) +#define SMB2_CHANNEL_RDMA_V1 cpu_to_le32(0x00000001) +#define SMB2_CHANNEL_RDMA_V1_INVALIDATE cpu_to_le32(0x00000002) +#define SMB2_CHANNEL_RDMA_TRANSFORM cpu_to_le32(0x00000003) + +/* See MS-SMB2 2.2.43. */ +struct smb2_rdma_transform { + __le16 RdmaDescriptorOffset; + __le16 RdmaDescriptorLength; + __le32 Channel; + __le16 TransformCount; + __le16 Reserved1; + __le32 Reserved2; +} __packed; + +#define SMB2_RDMA_TRANSFORM_TYPE_ENCRYPTION 0x0001 +#define SMB2_RDMA_TRANSFORM_TYPE_SIGNING 0x0002 + +struct smb2_rdma_crypto_transform { + __le16 TransformType; + __le16 SignatureLength; + __le16 NonceLength; + __le16 Reserved; + __u8 Signature[]; + /* Followed by Nonce[] and optional alignment padding. */ +} __packed; + +/* SMB2 read request without RFC1001 length at the beginning */ +struct smb2_read_req { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 49 */ + __u8 Padding; /* offset from start of SMB2 header to place read */ + __u8 Flags; /* MBZ unless SMB3.02 or later */ + __le32 Length; + __le64 Offset; + __u64 PersistentFileId; + __u64 VolatileFileId; + __le32 MinimumCount; + __le32 Channel; /* MBZ except for SMB3 or later */ + __le32 RemainingBytes; + __le16 ReadChannelInfoOffset; + __le16 ReadChannelInfoLength; + __u8 Buffer[]; +} __packed; + +/* Read flags */ +#define SMB2_READFLAG_RESPONSE_NONE cpu_to_le32(0x00000000) +#define SMB2_READFLAG_RESPONSE_RDMA_TRANSFORM cpu_to_le32(0x00000001) + +struct smb2_read_rsp { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 17 */ + __u8 DataOffset; + __u8 Reserved; + __le32 DataLength; + __le32 DataRemaining; + __le32 Flags; + __u8 Buffer[]; +} __packed; + + +/* + * SMB2_WRITE See MS-SMB2 section 2.2.21 + */ +/* For write request Flags field below the following flags are defined: */ +#define SMB2_WRITEFLAG_WRITE_THROUGH 0x00000001 /* SMB2.1 or later */ +#define SMB2_WRITEFLAG_WRITE_UNBUFFERED 0x00000002 /* SMB3.02 or later */ + +struct smb2_write_req { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 49 */ + __le16 DataOffset; /* offset from start of SMB2 header to write data */ + __le32 Length; + __le64 Offset; + __u64 PersistentFileId; /* opaque endianness */ + __u64 VolatileFileId; /* opaque endianness */ + __le32 Channel; /* MBZ unless SMB3.02 or later */ + __le32 RemainingBytes; + __le16 WriteChannelInfoOffset; + __le16 WriteChannelInfoLength; + __le32 Flags; + __u8 Buffer[]; +} __packed; + +struct smb2_write_rsp { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 17 */ + __u8 DataOffset; + __u8 Reserved; + __le32 DataLength; + __le32 DataRemaining; + __u32 Reserved2; + __u8 Buffer[]; +} __packed; + + +/* + * SMB2_FLUSH See MS-SMB2 section 2.2.17 + */ +struct smb2_flush_req { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 24 */ + __le16 Reserved1; + __le32 Reserved2; + __u64 PersistentFileId; + __u64 VolatileFileId; +} __packed; + +struct smb2_flush_rsp { + struct smb2_hdr hdr; + __le16 StructureSize; + __le16 Reserved; +} __packed; + +#define SMB2_LOCKFLAG_SHARED 0x0001 +#define SMB2_LOCKFLAG_EXCLUSIVE 0x0002 +#define SMB2_LOCKFLAG_UNLOCK 0x0004 +#define SMB2_LOCKFLAG_FAIL_IMMEDIATELY 0x0010 +#define SMB2_LOCKFLAG_MASK 0x0007 + +struct smb2_lock_element { + __le64 Offset; + __le64 Length; + __le32 Flags; + __le32 Reserved; +} __packed; + +struct smb2_lock_req { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 48 */ + __le16 LockCount; + /* + * The least significant four bits are the lock sequence number. The + * other 28 bits are the index (0 to 64). See MS-SMB2 2.2.26. + */ + __le32 LockSequenceNumber; + __u64 PersistentFileId; + __u64 VolatileFileId; + /* Followed by at least one */ + union { + struct smb2_lock_element lock; + DECLARE_FLEX_ARRAY(struct smb2_lock_element, locks); + }; +} __packed; + +struct smb2_lock_rsp { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 4 */ + __le16 Reserved; +} __packed; + +struct smb2_echo_req { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 4 */ + __u16 Reserved; +} __packed; + +struct smb2_echo_rsp { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 4 */ + __u16 Reserved; +} __packed; + +/* + * Valid FileInformation classes for query directory + * + * Note that these are a subset of the (file) QUERY_INFO levels defined + * later in this file (but since QUERY_DIRECTORY uses equivalent numbers + * we do not redefine them here) + * + * FileDirectoryInfomation 0x01 + * FileFullDirectoryInformation 0x02 + * FileIdFullDirectoryInformation 0x26 + * FileBothDirectoryInformation 0x03 + * FileIdBothDirectoryInformation 0x25 + * FileNamesInformation 0x0C + * FileIdExtdDirectoryInformation 0x3C + */ + +/* search (query_directory) Flags field */ +#define SMB2_RESTART_SCANS 0x01 +#define SMB2_RETURN_SINGLE_ENTRY 0x02 +#define SMB2_INDEX_SPECIFIED 0x04 +#define SMB2_REOPEN 0x10 + +struct smb2_query_directory_req { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 33 */ + __u8 FileInformationClass; + __u8 Flags; + __le32 FileIndex; + __u64 PersistentFileId; + __u64 VolatileFileId; + __le16 FileNameOffset; + __le16 FileNameLength; + __le32 OutputBufferLength; + __u8 Buffer[]; +} __packed; + +struct smb2_query_directory_rsp { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 9 */ + __le16 OutputBufferOffset; + __le32 OutputBufferLength; + __u8 Buffer[]; +} __packed; + +/* DeviceType Flags */ +#define FILE_DEVICE_CD_ROM 0x00000002 +#define FILE_DEVICE_CD_ROM_FILE_SYSTEM 0x00000003 +#define FILE_DEVICE_DFS 0x00000006 +#define FILE_DEVICE_DISK 0x00000007 +#define FILE_DEVICE_DISK_FILE_SYSTEM 0x00000008 +#define FILE_DEVICE_FILE_SYSTEM 0x00000009 +#define FILE_DEVICE_NAMED_PIPE 0x00000011 +#define FILE_DEVICE_NETWORK 0x00000012 +#define FILE_DEVICE_NETWORK_FILE_SYSTEM 0x00000014 +#define FILE_DEVICE_NULL 0x00000015 +#define FILE_DEVICE_PARALLEL_PORT 0x00000016 +#define FILE_DEVICE_PRINTER 0x00000018 +#define FILE_DEVICE_SERIAL_PORT 0x0000001b +#define FILE_DEVICE_STREAMS 0x0000001e +#define FILE_DEVICE_TAPE 0x0000001f +#define FILE_DEVICE_TAPE_FILE_SYSTEM 0x00000020 +#define FILE_DEVICE_VIRTUAL_DISK 0x00000024 +#define FILE_DEVICE_NETWORK_REDIRECTOR 0x00000028 + +/* Device Characteristics */ +#define FILE_REMOVABLE_MEDIA 0x00000001 +#define FILE_READ_ONLY_DEVICE 0x00000002 +#define FILE_FLOPPY_DISKETTE 0x00000004 +#define FILE_WRITE_ONCE_MEDIA 0x00000008 +#define FILE_REMOTE_DEVICE 0x00000010 +#define FILE_DEVICE_IS_MOUNTED 0x00000020 +#define FILE_VIRTUAL_VOLUME 0x00000040 +#define FILE_DEVICE_SECURE_OPEN 0x00000100 +#define FILE_CHARACTERISTIC_TS_DEVICE 0x00001000 +#define FILE_CHARACTERISTIC_WEBDAV_DEVICE 0x00002000 +#define FILE_PORTABLE_DEVICE 0x00004000 +#define FILE_DEVICE_ALLOW_APPCONTAINER_TRAVERSAL 0x00020000 + +/* + * Maximum number of iovs we need for a set-info request. + * The largest one is rename/hardlink + * [0] : struct smb2_set_info_req + smb2_file_[rename|link]_info + * [1] : path + * [2] : compound padding + */ +#define SMB2_SET_INFO_IOV_SIZE 3 + +struct smb2_set_info_req { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 33 */ + __u8 InfoType; + __u8 FileInfoClass; + __le32 BufferLength; + __le16 BufferOffset; + __u16 Reserved; + __le32 AdditionalInformation; + __u64 PersistentFileId; + __u64 VolatileFileId; + __u8 Buffer[]; +} __packed; + +struct smb2_set_info_rsp { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 2 */ +} __packed; + +/* + * SMB2_NOTIFY See MS-SMB2 section 2.2.35 + */ +/* notify flags */ +#define SMB2_WATCH_TREE 0x0001 + +/* notify completion filter flags. See MS-FSCC 2.6 and MS-SMB2 2.2.35 */ +#define FILE_NOTIFY_CHANGE_FILE_NAME 0x00000001 +#define FILE_NOTIFY_CHANGE_DIR_NAME 0x00000002 +#define FILE_NOTIFY_CHANGE_NAME 0x00000003 +#define FILE_NOTIFY_CHANGE_ATTRIBUTES 0x00000004 +#define FILE_NOTIFY_CHANGE_SIZE 0x00000008 +#define FILE_NOTIFY_CHANGE_LAST_WRITE 0x00000010 +#define FILE_NOTIFY_CHANGE_LAST_ACCESS 0x00000020 +#define FILE_NOTIFY_CHANGE_CREATION 0x00000040 +#define FILE_NOTIFY_CHANGE_EA 0x00000080 +#define FILE_NOTIFY_CHANGE_SECURITY 0x00000100 +#define FILE_NOTIFY_CHANGE_STREAM_NAME 0x00000200 +#define FILE_NOTIFY_CHANGE_STREAM_SIZE 0x00000400 +#define FILE_NOTIFY_CHANGE_STREAM_WRITE 0x00000800 + +/* See MS-SMB2 2.2.35 */ +struct smb2_change_notify_req { + struct smb2_hdr hdr; + __le16 StructureSize; + __le16 Flags; + __le32 OutputBufferLength; + __u64 PersistentFileId; /* opaque endianness */ + __u64 VolatileFileId; /* opaque endianness */ + __le32 CompletionFilter; + __u32 Reserved; +} __packed; + +/* See MS-SMB2 2.2.36 */ +struct smb2_change_notify_rsp { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 9 */ + __le16 OutputBufferOffset; + __le32 OutputBufferLength; + __u8 Buffer[]; /* array of file notify structs */ +} __packed; + +/* + * SMB2_SERVER_TO_CLIENT_NOTIFICATION: See MS-SMB2 section 2.2.44 + */ + +#define SMB2_NOTIFY_SESSION_CLOSED 0x0000 + +struct smb2_server_client_notification { + struct smb2_hdr hdr; + __le16 StructureSize; + __u16 Reserved; /* MBZ */ + __le32 NotificationType; + __u8 NotificationBuffer[4]; /* MBZ */ +} __packed; + +/* + * SMB2_CREATE See MS-SMB2 section 2.2.13 + */ +/* Oplock levels */ +#define SMB2_OPLOCK_LEVEL_NONE 0x00 +#define SMB2_OPLOCK_LEVEL_II 0x01 +#define SMB2_OPLOCK_LEVEL_EXCLUSIVE 0x08 +#define SMB2_OPLOCK_LEVEL_BATCH 0x09 +#define SMB2_OPLOCK_LEVEL_LEASE 0xFF +/* Non-spec internal type */ +#define SMB2_OPLOCK_LEVEL_NOCHANGE 0x99 + +/* Impersonation Levels. See MS-WPO section 9.7 and MSDN-IMPERS */ +#define IL_ANONYMOUS cpu_to_le32(0x00000000) +#define IL_IDENTIFICATION cpu_to_le32(0x00000001) +#define IL_IMPERSONATION cpu_to_le32(0x00000002) +#define IL_DELEGATE cpu_to_le32(0x00000003) + +/* Desired Access Flags */ +#define FILE_READ_DATA_LE cpu_to_le32(0x00000001) +#define FILE_LIST_DIRECTORY_LE cpu_to_le32(0x00000001) +#define FILE_WRITE_DATA_LE cpu_to_le32(0x00000002) +#define FILE_APPEND_DATA_LE cpu_to_le32(0x00000004) +#define FILE_ADD_SUBDIRECTORY_LE cpu_to_le32(0x00000004) +#define FILE_READ_EA_LE cpu_to_le32(0x00000008) +#define FILE_WRITE_EA_LE cpu_to_le32(0x00000010) +#define FILE_EXECUTE_LE cpu_to_le32(0x00000020) +#define FILE_DELETE_CHILD_LE cpu_to_le32(0x00000040) +#define FILE_READ_ATTRIBUTES_LE cpu_to_le32(0x00000080) +#define FILE_WRITE_ATTRIBUTES_LE cpu_to_le32(0x00000100) +#define FILE_DELETE_LE cpu_to_le32(0x00010000) +#define FILE_READ_CONTROL_LE cpu_to_le32(0x00020000) +#define FILE_WRITE_DAC_LE cpu_to_le32(0x00040000) +#define FILE_WRITE_OWNER_LE cpu_to_le32(0x00080000) +#define FILE_SYNCHRONIZE_LE cpu_to_le32(0x00100000) +#define FILE_ACCESS_SYSTEM_SECURITY_LE cpu_to_le32(0x01000000) +#define FILE_MAXIMAL_ACCESS_LE cpu_to_le32(0x02000000) +#define FILE_GENERIC_ALL_LE cpu_to_le32(0x10000000) +#define FILE_GENERIC_EXECUTE_LE cpu_to_le32(0x20000000) +#define FILE_GENERIC_WRITE_LE cpu_to_le32(0x40000000) +#define FILE_GENERIC_READ_LE cpu_to_le32(0x80000000) +#define DESIRED_ACCESS_MASK cpu_to_le32(0xF21F01FF) + + +#define FILE_READ_DESIRED_ACCESS_LE (FILE_READ_DATA_LE | \ + FILE_READ_EA_LE | \ + FILE_GENERIC_READ_LE) +#define FILE_WRITE_DESIRE_ACCESS_LE (FILE_WRITE_DATA_LE | \ + FILE_APPEND_DATA_LE | \ + FILE_WRITE_EA_LE | \ + FILE_WRITE_ATTRIBUTES_LE | \ + FILE_GENERIC_WRITE_LE) + +/* ShareAccess Flags */ +#define FILE_SHARE_READ_LE cpu_to_le32(0x00000001) +#define FILE_SHARE_WRITE_LE cpu_to_le32(0x00000002) +#define FILE_SHARE_DELETE_LE cpu_to_le32(0x00000004) +#define FILE_SHARE_ALL_LE cpu_to_le32(0x00000007) + +/* CreateDisposition Flags */ +#define FILE_SUPERSEDE_LE cpu_to_le32(0x00000000) +#define FILE_OPEN_LE cpu_to_le32(0x00000001) +#define FILE_CREATE_LE cpu_to_le32(0x00000002) +#define FILE_OPEN_IF_LE cpu_to_le32(0x00000003) +#define FILE_OVERWRITE_LE cpu_to_le32(0x00000004) +#define FILE_OVERWRITE_IF_LE cpu_to_le32(0x00000005) +#define FILE_CREATE_MASK_LE cpu_to_le32(0x00000007) + +/* CreateOptions Flags */ +#define FILE_DIRECTORY_FILE_LE cpu_to_le32(0x00000001) +/* same as #define CREATE_NOT_FILE_LE cpu_to_le32(0x00000001) */ +#define FILE_WRITE_THROUGH_LE cpu_to_le32(0x00000002) +#define FILE_SEQUENTIAL_ONLY_LE cpu_to_le32(0x00000004) +#define FILE_NO_INTERMEDIATE_BUFFERING_LE cpu_to_le32(0x00000008) +/* FILE_SYNCHRONOUS_IO_ALERT_LE cpu_to_le32(0x00000010) should be zero, ignored */ +/* FILE_SYNCHRONOUS_IO_NONALERT cpu_to_le32(0x00000020) should be zero, ignored */ +#define FILE_NON_DIRECTORY_FILE_LE cpu_to_le32(0x00000040) +#define FILE_COMPLETE_IF_OPLOCKED_LE cpu_to_le32(0x00000100) +#define FILE_NO_EA_KNOWLEDGE_LE cpu_to_le32(0x00000200) +/* FILE_OPEN_REMOTE_INSTANCE cpu_to_le32(0x00000400) should be zero, ignored */ +#define FILE_RANDOM_ACCESS_LE cpu_to_le32(0x00000800) +#define FILE_DELETE_ON_CLOSE_LE cpu_to_le32(0x00001000) /* MBZ */ +#define FILE_OPEN_BY_FILE_ID_LE cpu_to_le32(0x00002000) +#define FILE_OPEN_FOR_BACKUP_INTENT_LE cpu_to_le32(0x00004000) +#define FILE_NO_COMPRESSION_LE cpu_to_le32(0x00008000) +/* FILE_OPEN_REQUIRING_OPLOCK cpu_to_le32(0x00010000) should be zero, ignored */ +/* FILE_DISALLOW_EXCLUSIVE cpu_to_le32(0x00020000) should be zero, ignored */ +/* FILE_RESERVE_OPFILTER cpu_to_le32(0x00100000) MBZ */ +#define FILE_OPEN_REPARSE_POINT_LE cpu_to_le32(0x00200000) +#define FILE_OPEN_NO_RECALL_LE cpu_to_le32(0x00400000) +/* #define FILE_OPEN_FOR_FREE_SPACE_QUERY cpu_to_le32(0x00800000) should be zero, ignored */ +#define CREATE_OPTIONS_MASK_LE cpu_to_le32(0x00FFFFFF) + +#define FILE_READ_RIGHTS_LE (FILE_READ_DATA_LE | FILE_READ_EA_LE \ + | FILE_READ_ATTRIBUTES_LE) +#define FILE_WRITE_RIGHTS_LE (FILE_WRITE_DATA_LE | FILE_APPEND_DATA_LE \ + | FILE_WRITE_EA_LE | FILE_WRITE_ATTRIBUTES_LE) +#define FILE_EXEC_RIGHTS_LE (FILE_EXECUTE_LE) + +/* Create Context Values */ +#define SMB2_CREATE_EA_BUFFER "ExtA" /* extended attributes */ +#define SMB2_CREATE_SD_BUFFER "SecD" /* security descriptor */ +#define SMB2_CREATE_DURABLE_HANDLE_REQUEST "DHnQ" +#define SMB2_CREATE_DURABLE_HANDLE_RECONNECT "DHnC" +#define SMB2_CREATE_ALLOCATION_SIZE "AlSi" +#define SMB2_CREATE_QUERY_MAXIMAL_ACCESS_REQUEST "MxAc" +#define SMB2_CREATE_TIMEWARP_REQUEST "TWrp" +#define SMB2_CREATE_QUERY_ON_DISK_ID "QFid" +#define SMB2_CREATE_REQUEST_LEASE "RqLs" +#define SMB2_CREATE_DURABLE_HANDLE_REQUEST_V2 "DH2Q" +#define SMB2_CREATE_DURABLE_HANDLE_RECONNECT_V2 "DH2C" +#define SMB2_CREATE_TAG_POSIX "\x93\xAD\x25\x50\x9C\xB4\x11\xE7\xB4\x23\x83\xDE\x96\x8B\xCD\x7C" +#define SMB2_CREATE_APP_INSTANCE_ID "\x45\xBC\xA6\x6A\xEF\xA7\xF7\x4A\x90\x08\xFA\x46\x2E\x14\x4D\x74" +#define SMB2_CREATE_APP_INSTANCE_VERSION "\xB9\x82\xD0\xB7\x3B\x56\x07\x4F\xA0\x7B\x52\x4A\x81\x16\xA0\x10" +#define SVHDX_OPEN_DEVICE_CONTEXT "\x9C\xCB\xCF\x9E\x04\xC1\xE6\x43\x98\x0E\x15\x8D\xA1\xF6\xEC\x83" +#define SMB2_CREATE_TAG_AAPL "AAPL" + +/* Flag (SMB3 open response) values */ +#define SMB2_CREATE_FLAG_REPARSEPOINT 0x01 + +struct create_context { + /* New members must be added within the struct_group() macro below. */ + __struct_group(create_context_hdr, hdr, __packed, + __le32 Next; + __le16 NameOffset; + __le16 NameLength; + __le16 Reserved; + __le16 DataOffset; + __le32 DataLength; + ); + __u8 Buffer[]; +} __packed; +static_assert(offsetof(struct create_context, Buffer) == sizeof(struct create_context_hdr), + "struct member likely outside of __struct_group()"); + +struct smb2_create_req { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 57 */ + __u8 SecurityFlags; + __u8 RequestedOplockLevel; + __le32 ImpersonationLevel; + __le64 SmbCreateFlags; + __le64 Reserved; + __le32 DesiredAccess; + __le32 FileAttributes; + __le32 ShareAccess; + __le32 CreateDisposition; + __le32 CreateOptions; + __le16 NameOffset; + __le16 NameLength; + __le32 CreateContextsOffset; + __le32 CreateContextsLength; + __u8 Buffer[]; +} __packed; + +struct smb2_create_rsp { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 89 */ + __u8 OplockLevel; + __u8 Flags; /* 0x01 if reparse point */ + __le32 CreateAction; + __le64 CreationTime; + __le64 LastAccessTime; + __le64 LastWriteTime; + __le64 ChangeTime; + __le64 AllocationSize; + __le64 EndofFile; + __le32 FileAttributes; + __le32 Reserved2; + __u64 PersistentFileId; + __u64 VolatileFileId; + __le32 CreateContextsOffset; + __le32 CreateContextsLength; + __u8 Buffer[]; +} __packed; + +struct create_posix { + struct create_context_hdr ccontext; + __u8 Name[16]; + __le32 Mode; + __u32 Reserved; +} __packed; + +/* See MS-SMB2 2.2.13.2.3 and MS-SMB2 2.2.13.2.4 */ +typedef struct { + struct create_context_hdr ccontext; + __u8 Name[8]; + union { + __u8 Reserved[16]; + struct { + __u64 PersistentFileId; + __u64 VolatileFileId; + } Fid; + } Data; +} __packed create_durable_req_t, create_durable_reconn_t; + +/* See MS-SMB2 2.2.13.2.5 */ +struct create_mxac_req { + struct create_context_hdr ccontext; + __u8 Name[8]; + __le64 Timestamp; +} __packed; + +/* + * AAPL flags. See Samba libcli/smb/smb2_create_ctx.h + */ + +/* "AAPL" Context Command Codes */ +#define SMB2_CRTCTX_AAPL_SERVER_QUERY 1 +#define SMB2_CRTCTX_AAPL_RESOLVE_ID 2 + +/* "AAPL" Server Query request/response bitmap */ +#define SMB2_CRTCTX_AAPL_SERVER_CAPS 1 +#define SMB2_CRTCTX_AAPL_VOLUME_CAPS 2 +#define SMB2_CRTCTX_AAPL_MODEL_INFO 4 + +/* "AAPL" Client/Server Capabilities bitmap */ +#define SMB2_CRTCTX_AAPL_SUPPORTS_READ_DIR_ATTR 1 +#define SMB2_CRTCTX_AAPL_SUPPORTS_OSX_COPYFILE 2 +#define SMB2_CRTCTX_AAPL_UNIX_BASED 4 +#define SMB2_CRTCTX_AAPL_SUPPORTS_NFS_ACE 8 +/* + * V2 extends the same inline-FinderInfo mechanism as + * SMB2_CRTCTX_AAPL_SUPPORTS_READ_DIR_ATTR with an added flags field, + * confirmed byte-identical to V1 otherwise against AAPL's actual + * public client behavior. Mutually exclusive with the V1 bit on + * the wire, not both set together. + */ +#define SMB2_CRTCTX_AAPL_SUPPORTS_READ_DIR_ATTR_V2 16 + +/* "AAPL" Volume Capabilities bitmap */ +#define SMB2_CRTCTX_AAPL_SUPPORT_RESOLVE_ID 1 +#define SMB2_CRTCTX_AAPL_CASE_SENSITIVE 2 +#define SMB2_CRTCTX_AAPL_FULL_SYNC 4 + +/* + * Flags + * See MS-SMB2 2.2.13.2.11 + * MS-SMB2 2.2.13.2.12 + * MS-SMB2 2.2.14.2.12 + */ +#define SMB2_DHANDLE_FLAG_PERSISTENT 0x00000002 + +/* See MS-SMB2 2.2.13.2.11 */ +struct durable_context_v2_req { + __le32 Timeout; + __le32 Flags; /* see SMB2_DHANDLE_FLAG_PERSISTENT */ + __u64 Reserved; + __u8 CreateGuid[16]; +} __packed; + +struct create_durable_req_v2 { + struct create_context_hdr ccontext; + __u8 Name[8]; + struct durable_context_v2_req dcontext; +} __packed; + +/* See MS-SMB2 2.2.13.2.12 */ +struct durable_reconnect_context_v2 { + struct { + __u64 PersistentFileId; + __u64 VolatileFileId; + } Fid; + __u8 CreateGuid[16]; + __le32 Flags; /* see SMB2_DHANDLE_FLAG_PERSISTENT */ +} __packed; + +struct create_durable_handle_reconnect_v2 { + struct create_context_hdr ccontext; + __u8 Name[8]; + struct durable_reconnect_context_v2 dcontext; + __u8 Pad[4]; +} __packed; + +/* See MS-SMB2 2.2.14.2.12 */ +struct durable_context_v2_rsp { + __le32 Timeout; + __le32 Flags; /* see SMB2_DHANDLE_FLAG_PERSISTENT */ +} __packed; + +struct create_durable_rsp_v2 { + struct create_context_hdr ccontext; + __u8 Name[8]; + struct durable_context_v2_rsp dcontext; +} __packed; + +/* See MS-SMB2 2.2.14.2.5 */ +struct create_mxac_rsp { + struct create_context_hdr ccontext; + __u8 Name[8]; + __le32 QueryStatus; + __le32 MaximalAccess; +} __packed; + +#define SMB2_LEASE_NONE_LE cpu_to_le32(0x00) +#define SMB2_LEASE_READ_CACHING_LE cpu_to_le32(0x01) +#define SMB2_LEASE_HANDLE_CACHING_LE cpu_to_le32(0x02) +#define SMB2_LEASE_WRITE_CACHING_LE cpu_to_le32(0x04) + +#define SMB2_LEASE_FLAG_BREAK_IN_PROGRESS_LE cpu_to_le32(0x02) +#define SMB2_LEASE_FLAG_PARENT_LEASE_KEY_SET_LE cpu_to_le32(0x04) + +#define SMB2_LEASE_KEY_SIZE 16 + +/* See MS-SMB2 2.2.13.2.8 */ +struct lease_context { + __u8 LeaseKey[SMB2_LEASE_KEY_SIZE]; + __le32 LeaseState; + __le32 LeaseFlags; + __le64 LeaseDuration; +} __packed; + +/* See MS-SMB2 2.2.13.2.10 */ +struct lease_context_v2 { + __u8 LeaseKey[SMB2_LEASE_KEY_SIZE]; + __le32 LeaseState; + __le32 LeaseFlags; + __le64 LeaseDuration; + __u8 ParentLeaseKey[SMB2_LEASE_KEY_SIZE]; + __le16 Epoch; + __le16 Reserved; +} __packed; + +struct create_lease { + struct create_context_hdr ccontext; + __u8 Name[8]; + struct lease_context lcontext; +} __packed; + +struct create_lease_v2 { + struct create_context_hdr ccontext; + __u8 Name[8]; + struct lease_context_v2 lcontext; + __u8 Pad[4]; +} __packed; + +/* See MS-SMB2 2.2.14.2.9 */ +struct create_disk_id_rsp { + struct create_context_hdr ccontext; + __u8 Name[8]; + __le64 DiskFileId; + __le64 VolumeId; + __u8 Reserved[16]; +} __packed; + +/* See MS-SMB2 2.2.13.2.13 */ +struct create_app_inst_id { + struct create_context_hdr ccontext; + __u8 Name[16]; + __le32 StructureSize; /* Must be 20 */ + __u16 Reserved; + __u8 AppInstanceId[16]; +} __packed; + +/* See MS-SMB2 2.2.13.2.15 */ +struct create_app_inst_id_vers { + struct create_context_hdr ccontext; + __u8 Name[16]; + __le32 StructureSize; /* Must be 24 */ + __u16 Reserved; + __u32 Padding; + __le64 AppInstanceVersionHigh; + __le64 AppInstanceVersionLow; +} __packed; + +/* See MS-SMB2 2.2.31 and 2.2.32 */ +struct smb2_ioctl_req { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 57 */ + __le16 Reserved; /* offset from start of SMB2 header to write data */ + __le32 CtlCode; + __u64 PersistentFileId; + __u64 VolatileFileId; + __le32 InputOffset; /* Reserved MBZ */ + __le32 InputCount; + __le32 MaxInputResponse; + __le32 OutputOffset; + __le32 OutputCount; + __le32 MaxOutputResponse; + __le32 Flags; + __le32 Reserved2; + __u8 Buffer[]; +} __packed; + +/* See MS-SMB2 2.2.31.1.1 */ +struct srv_copychunk { + __le64 SourceOffset; + __le64 TargetOffset; + __le32 Length; + __le32 Reserved; +} __packed; + +#define COPY_CHUNK_RES_KEY_SIZE 24 + +/* See MS-SMB2 2.2.31.1 */ +/* this goes in the ioctl buffer when doing a copychunk request */ +struct copychunk_ioctl_req { + union { + char SourceKey[COPY_CHUNK_RES_KEY_SIZE]; + __le64 SourceKeyU64[3]; + }; + __le32 ChunkCount; + __le32 Reserved; + struct srv_copychunk Chunks[] __counted_by_le(ChunkCount); +} __packed; + +/* See MS-SMB2 2.2.32.1 */ +struct copychunk_ioctl_rsp { + __le32 ChunksWritten; + __le32 ChunkBytesWritten; + __le32 TotalBytesWritten; +} __packed; + +/* See MS-SMB2 2.2.32.3 */ +struct resume_key_ioctl_rsp { + union { + char ResumeKey[COPY_CHUNK_RES_KEY_SIZE]; + __u64 ResumeKeyU64[3]; + }; + __le32 ContextLength; /* MBZ */ + char Context[4]; /* ignored, Windows sets to 4 bytes of zero */ +} __packed; + +struct smb2_ioctl_rsp { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 49 */ + __le16 Reserved; + __le32 CtlCode; + __u64 PersistentFileId; + __u64 VolatileFileId; + __le32 InputOffset; /* Reserved MBZ */ + __le32 InputCount; + __le32 OutputOffset; + __le32 OutputCount; + __le32 Flags; + __le32 Reserved2; + __u8 Buffer[]; +} __packed; + +/* See MS-SMB2 2.2.32.5.1.1 */ +struct smb_sockaddr_in { + __be16 Port; + __be32 IPv4Address; + __u8 Reserved[8]; +} __packed; + +/* See MS-SMB2 2.2.32.5.1.2 */ +struct smb_sockaddr_in6 { + __be16 Port; + __be32 FlowInfo; + __u8 IPv6Address[16]; + __be32 ScopeId; +} __packed; + +/* See MS-SMB2 2.2.32.5 and MS-SMB2 2.2.32.5.1 */ +#define RSS_CAPABLE cpu_to_le32(0x00000001) +#define RDMA_CAPABLE cpu_to_le32(0x00000002) +#define INTERNETWORK cpu_to_le16(0x0002) +#define INTERNETWORKV6 cpu_to_le16(0x0017) +struct network_interface_info_ioctl_rsp { + __le32 Next; /* next interface. zero if this is last one */ + __le32 IfIndex; + __le32 Capability; /* RSS or RDMA Capable */ + __le32 Reserved; + __le64 LinkSpeed; + union { + char SockAddr_Storage[128]; + struct { + __le16 Family; + __u8 Buffer[126]; + }; + }; +} __packed; + +/* Integrity ChecksumAlgorithm choices for above */ +#define CHECKSUM_TYPE_NONE 0x0000 +#define CHECKSUM_TYPE_CRC64 0x0002 +#define CHECKSUM_TYPE_UNCHANGED 0xFFFF /* set only */ + +/* Integrity flags for above */ +#define FSCTL_INTEGRITY_FLAG_CHECKSUM_ENFORCEMENT_OFF 0x00000001 + +struct validate_negotiate_info_req { + __le32 Capabilities; + __u8 Guid[SMB2_CLIENT_GUID_SIZE]; + __le16 SecurityMode; + __le16 DialectCount; + __le16 Dialects[4]; /* BB expand this if autonegotiate > 4 dialects */ +} __packed; + +struct validate_negotiate_info_rsp { + __le32 Capabilities; + __u8 Guid[SMB2_CLIENT_GUID_SIZE]; + __le16 SecurityMode; + __le16 Dialect; /* Dialect in use for the connection */ +} __packed; + + +/* Possible InfoType values */ +#define SMB2_O_INFO_FILE 0x01 +#define SMB2_O_INFO_FILESYSTEM 0x02 +#define SMB2_O_INFO_SECURITY 0x03 +#define SMB2_O_INFO_QUOTA 0x04 + +/* SMB2 Query Info see MS-SMB2 (2.2.37) or MS-DTYP */ + +/* List of QUERY INFO levels (those also valid for QUERY_DIR are noted below */ +#define FILE_DIRECTORY_INFORMATION 1 /* also for QUERY_DIR */ +#define FILE_FULL_DIRECTORY_INFORMATION 2 /* also for QUERY_DIR */ +#define FILE_BOTH_DIRECTORY_INFORMATION 3 /* also for QUERY_DIR */ +#define FILE_BASIC_INFORMATION 4 +#define FILE_STANDARD_INFORMATION 5 +#define FILE_INTERNAL_INFORMATION 6 +#define FILE_EA_INFORMATION 7 +#define FILE_ACCESS_INFORMATION 8 +#define FILE_NAME_INFORMATION 9 +#define FILE_RENAME_INFORMATION 10 +#define FILE_LINK_INFORMATION 11 +#define FILE_NAMES_INFORMATION 12 /* also for QUERY_DIR */ +#define FILE_DISPOSITION_INFORMATION 13 +#define FILE_POSITION_INFORMATION 14 +#define FILE_FULL_EA_INFORMATION 15 +#define FILE_MODE_INFORMATION 16 +#define FILE_ALIGNMENT_INFORMATION 17 +#define FILE_ALL_INFORMATION 18 +#define FILE_ALLOCATION_INFORMATION 19 +#define FILE_END_OF_FILE_INFORMATION 20 +#define FILE_ALTERNATE_NAME_INFORMATION 21 +#define FILE_STREAM_INFORMATION 22 +#define FILE_PIPE_INFORMATION 23 +#define FILE_PIPE_LOCAL_INFORMATION 24 +#define FILE_PIPE_REMOTE_INFORMATION 25 +#define FILE_MAILSLOT_QUERY_INFORMATION 26 +#define FILE_MAILSLOT_SET_INFORMATION 27 +#define FILE_COMPRESSION_INFORMATION 28 +#define FILE_OBJECT_ID_INFORMATION 29 +/* Number 30 not defined in documents */ +#define FILE_MOVE_CLUSTER_INFORMATION 31 +#define FILE_QUOTA_INFORMATION 32 +#define FILE_REPARSE_POINT_INFORMATION 33 +#define FILE_NETWORK_OPEN_INFORMATION 34 +#define FILE_ATTRIBUTE_TAG_INFORMATION 35 +#define FILE_TRACKING_INFORMATION 36 +#define FILEID_BOTH_DIRECTORY_INFORMATION 37 /* also for QUERY_DIR */ +#define FILEID_FULL_DIRECTORY_INFORMATION 38 /* also for QUERY_DIR */ +#define FILE_VALID_DATA_LENGTH_INFORMATION 39 +#define FILE_SHORT_NAME_INFORMATION 40 +#define FILE_SFIO_RESERVE_INFORMATION 44 +#define FILE_SFIO_VOLUME_INFORMATION 45 +#define FILE_HARD_LINK_INFORMATION 46 +#define FILE_NORMALIZED_NAME_INFORMATION 48 +#define FILEID_GLOBAL_TX_DIRECTORY_INFORMATION 50 +#define FILE_STANDARD_LINK_INFORMATION 54 +#define FILE_ID_INFORMATION 59 +#define FILE_ID_EXTD_DIRECTORY_INFORMATION 60 /* also for QUERY_DIR */ +#define FileId64ExtdDirectoryInformation 78 /* also for QUERY_DIR */ +#define FileId64ExtdBothDirectoryInformation 79 /* also for QUERY_DIR */ +#define FileIdAllExtdDirectoryInformation 80 /* also for QUERY_DIR */ +#define FileIdAllExtdBothDirectoryInformation 81 /* also for QUERY_DIR */ +/* Used for Query Info and Find File POSIX Info for SMB3.1.1 and SMB1 */ +#define SMB_FIND_FILE_POSIX_INFO 0x064 + +/* Security info type additionalinfo flags. */ +#define OWNER_SECINFO 0x00000001 +#define GROUP_SECINFO 0x00000002 +#define DACL_SECINFO 0x00000004 +#define SACL_SECINFO 0x00000008 +#define LABEL_SECINFO 0x00000010 +#define ATTRIBUTE_SECINFO 0x00000020 +#define SCOPE_SECINFO 0x00000040 +#define BACKUP_SECINFO 0x00010000 +#define UNPROTECTED_SACL_SECINFO 0x10000000 +#define UNPROTECTED_DACL_SECINFO 0x20000000 +#define PROTECTED_SACL_SECINFO 0x40000000 +#define PROTECTED_DACL_SECINFO 0x80000000 + +/* Flags used for FileFullEAinfo */ +#define SL_RESTART_SCAN 0x00000001 +#define SL_RETURN_SINGLE_ENTRY 0x00000002 +#define SL_INDEX_SPECIFIED 0x00000004 + +struct smb2_query_info_req { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 41 */ + __u8 InfoType; + __u8 FileInfoClass; + __le32 OutputBufferLength; + __le16 InputBufferOffset; + __u16 Reserved; + __le32 InputBufferLength; + __le32 AdditionalInformation; + __le32 Flags; + __u64 PersistentFileId; + __u64 VolatileFileId; + __u8 Buffer[]; +} __packed; + +struct smb2_query_info_rsp { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 9 */ + __le16 OutputBufferOffset; + __le32 OutputBufferLength; + __u8 Buffer[]; +} __packed; + +/* Level 100 query info */ +struct smb311_posix_qinfo { + __le64 CreationTime; + __le64 LastAccessTime; + __le64 LastWriteTime; + __le64 ChangeTime; + __le64 EndOfFile; + __le64 AllocationSize; + __le32 DosAttributes; + __le64 Inode; + __le32 DeviceId; + __le32 Zero; + /* beginning of POSIX Create Context Response */ + __le32 HardLinks; + __le32 ReparseTag; + __le32 Mode; + u8 Sids[]; + /* + * var sized owner SID + * var sized group SID + * le32 filenamelength + * u8 filename[] + */ +} __packed; + +/* See MS-SMB2 2.2.23 through 2.2.25 */ +struct smb2_oplock_break { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 24 */ + __u8 OplockLevel; + __u8 Reserved; + __le32 Reserved2; + __u64 PersistentFid; + __u64 VolatileFid; +} __packed; + +#define SMB2_NOTIFY_BREAK_LEASE_FLAG_ACK_REQUIRED cpu_to_le32(0x01) + +struct smb2_lease_break { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 44 */ + __le16 Epoch; + __le32 Flags; + __u8 LeaseKey[16]; + __le32 CurrentLeaseState; + __le32 NewLeaseState; + __le32 BreakReason; + __le32 AccessMaskHint; + __le32 ShareMaskHint; +} __packed; + +struct smb2_lease_ack { + struct smb2_hdr hdr; + __le16 StructureSize; /* Must be 36 */ + __le16 Reserved; + __le32 Flags; + __u8 LeaseKey[16]; + __le32 LeaseState; + __le64 LeaseDuration; +} __packed; + +#define OP_BREAK_STRUCT_SIZE_20 24 +#define OP_BREAK_STRUCT_SIZE_21 36 + +/* + * See MS-SMB2 2.2.13.1.1 + * MS-SMB 2.2.1.4.1 + * These are the file access permission bits defined in CIFS for the + * NTCreateAndX as well as the level 0x107 + * TRANS2_QUERY_PATH_INFORMATION API. The level 0x107, SMB_QUERY_FILE_ALL_INFO + * responds with the AccessFlags. + * The AccessFlags specifies the access permissions a caller has to the + * file and can have any suitable combination of the following values: + */ +#define FILE_READ_DATA 0x00000001 /* Data can be read from the file */ + /* or directory child entries can */ + /* be listed together with the */ + /* associated child attributes */ + /* (so the FILE_READ_ATTRIBUTES on */ + /* the child entry is not needed) */ +#define FILE_WRITE_DATA 0x00000002 /* Data can be written to the file */ + /* or new file can be created in */ + /* the directory */ +#define FILE_APPEND_DATA 0x00000004 /* Data can be appended to the file */ + /* (for non-local files over SMB it */ + /* is same as FILE_WRITE_DATA) */ + /* or new subdirectory can be */ + /* created in the directory */ +#define FILE_READ_EA 0x00000008 /* Extended attributes associated */ + /* with the file can be read */ +#define FILE_WRITE_EA 0x00000010 /* Extended attributes associated */ + /* with the file can be written */ +#define FILE_EXECUTE 0x00000020 /*Data can be read into memory from */ + /* the file using system paging I/O */ + /* for executing the file / script */ + /* or right to traverse directory */ + /* (but by default all users have */ + /* directory bypass traverse */ + /* privilege and do not need this */ + /* permission on directories at all)*/ +#define FILE_DELETE_CHILD 0x00000040 /* Child entry can be deleted from */ + /* the directory (so the DELETE on */ + /* the child entry is not needed) */ +#define FILE_READ_ATTRIBUTES 0x00000080 /* Attributes associated with the */ + /* file or directory can be read */ +#define FILE_WRITE_ATTRIBUTES 0x00000100 /* Attributes associated with the */ + /* file or directory can be written */ +#define DELETE 0x00010000 /* The file or dir can be deleted */ +#define READ_CONTROL 0x00020000 /* The discretionary access control */ + /* list and ownership associated */ + /* with the file or dir can be read */ +#define WRITE_DAC 0x00040000 /* The discretionary access control */ + /* list associated with the file or */ + /* directory can be written */ +#define WRITE_OWNER 0x00080000 /* Ownership information associated */ + /* with the file/dir can be written */ +#define SYNCHRONIZE 0x00100000 /* The file handle can waited on to */ + /* synchronize with the completion */ + /* of an input/output request */ +#define SYSTEM_SECURITY 0x01000000 /* The system access control list */ + /* associated with the file or */ + /* directory can be read or written */ + /* (cannot be in DACL, can in SACL) */ +#define MAXIMUM_ALLOWED 0x02000000 /* Maximal subset of GENERIC_ALL */ + /* permissions which can be granted */ + /* (cannot be in DACL nor SACL) */ +#define GENERIC_ALL 0x10000000 /* Same as: GENERIC_EXECUTE | */ + /* GENERIC_WRITE | */ + /* GENERIC_READ | */ + /* FILE_DELETE_CHILD | */ + /* DELETE | */ + /* WRITE_DAC | */ + /* WRITE_OWNER */ + /* So GENERIC_ALL contains all bits */ + /* mentioned above except these two */ + /* SYSTEM_SECURITY MAXIMUM_ALLOWED */ +#define GENERIC_EXECUTE 0x20000000 /* Same as: FILE_EXECUTE | */ + /* FILE_READ_ATTRIBUTES | */ + /* READ_CONTROL | */ + /* SYNCHRONIZE */ +#define GENERIC_WRITE 0x40000000 /* Same as: FILE_WRITE_DATA | */ + /* FILE_APPEND_DATA | */ + /* FILE_WRITE_EA | */ + /* FILE_WRITE_ATTRIBUTES | */ + /* READ_CONTROL | */ + /* SYNCHRONIZE */ +#define GENERIC_READ 0x80000000 /* Same as: FILE_READ_DATA | */ + /* FILE_READ_EA | */ + /* FILE_READ_ATTRIBUTES | */ + /* READ_CONTROL | */ + /* SYNCHRONIZE */ + +/* Combinations of file access permission bits */ +#define FILE_READ_RIGHTS (FILE_READ_DATA | FILE_READ_EA | FILE_READ_ATTRIBUTES) +#define FILE_WRITE_RIGHTS (FILE_WRITE_DATA | FILE_APPEND_DATA \ + | FILE_WRITE_EA | FILE_WRITE_ATTRIBUTES) +#define FILE_EXEC_RIGHTS (FILE_EXECUTE) +#define SET_FILE_EXEC_RIGHTS (FILE_READ_EA | FILE_WRITE_EA | FILE_EXECUTE \ + | FILE_READ_ATTRIBUTES \ + | FILE_WRITE_ATTRIBUTES \ + | DELETE | READ_CONTROL | WRITE_DAC \ + | WRITE_OWNER | SYNCHRONIZE) +#define SET_MINIMUM_RIGHTS (FILE_READ_EA | FILE_READ_ATTRIBUTES \ + | READ_CONTROL | SYNCHRONIZE) + +#endif /* _COMMON_SMB2PDU_H */ diff --git a/fs/smb/common/smb2status.h b/fs/smb/common/smb2status.h new file mode 100644 index 000000000..2989c3a5c --- /dev/null +++ b/fs/smb/common/smb2status.h @@ -0,0 +1,1790 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +/* + * + * SMB2 Status code (network error) definitions + * Definitions are from MS-ERREF + * + * Copyright (c) International Business Machines Corp., 2009,2011 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ + +#ifndef FS_SMB_COMMON_SMB2STATUS_H +#define FS_SMB_COMMON_SMB2STATUS_H + +/* + * 0 1 2 3 4 5 6 7 8 9 0 A B C D E F 0 1 2 3 4 5 6 7 8 9 A B C D E F + * SEV C N <-------Facility--------> <------Error Status Code------> + * + * C is set if "customer defined" error, N bit is reserved and MBZ + */ + +#define STATUS_SEVERITY_SUCCESS __constant_cpu_to_le32(0x0000) +#define STATUS_SEVERITY_INFORMATIONAL cpu_to_le32(0x0001) +#define STATUS_SEVERITY_WARNING cpu_to_le32(0x0002) +#define STATUS_SEVERITY_ERROR cpu_to_le32(0x0003) + +struct ntstatus { + /* Facility is the high 12 bits of the following field */ + __le32 Facility; /* low 2 bits Severity, next is Customer, then rsrvd */ + __le32 Code; +}; + +/* + * The comment at the end of each definition indicates `posix_error` + * field of `struct status_to_posix_error`, it is used to generate the + * `smb2_error_map_table` array. + */ + +#define STATUS_SUCCESS cpu_to_le32(0x00000000) // 0 +#define STATUS_WAIT_0 cpu_to_le32(0x00000000) // 0 +#define STATUS_WAIT_1 cpu_to_le32(0x00000001) // -EIO +#define STATUS_WAIT_2 cpu_to_le32(0x00000002) // -EIO +#define STATUS_WAIT_3 cpu_to_le32(0x00000003) // -EIO +#define STATUS_WAIT_63 cpu_to_le32(0x0000003F) // -EIO +#define STATUS_ABANDONED cpu_to_le32(0x00000080) // -EIO +#define STATUS_ABANDONED_WAIT_0 cpu_to_le32(0x00000080) // -EIO +#define STATUS_ABANDONED_WAIT_63 cpu_to_le32(0x000000BF) // -EIO +#define STATUS_USER_APC cpu_to_le32(0x000000C0) // -EIO +#define STATUS_KERNEL_APC cpu_to_le32(0x00000100) // -EIO +#define STATUS_ALERTED cpu_to_le32(0x00000101) // -EIO +#define STATUS_TIMEOUT cpu_to_le32(0x00000102) // -ETIMEDOUT +#define STATUS_PENDING cpu_to_le32(0x00000103) // -EIO +#define STATUS_REPARSE cpu_to_le32(0x00000104) // -EIO +#define STATUS_MORE_ENTRIES cpu_to_le32(0x00000105) // -EIO +#define STATUS_NOT_ALL_ASSIGNED cpu_to_le32(0x00000106) // -EIO +#define STATUS_SOME_NOT_MAPPED cpu_to_le32(0x00000107) // -EIO +#define STATUS_OPLOCK_BREAK_IN_PROGRESS cpu_to_le32(0x00000108) // -EIO +#define STATUS_VOLUME_MOUNTED cpu_to_le32(0x00000109) // -EIO +#define STATUS_RXACT_COMMITTED cpu_to_le32(0x0000010A) // -EIO +#define STATUS_NOTIFY_CLEANUP cpu_to_le32(0x0000010B) // -EIO +#define STATUS_NOTIFY_ENUM_DIR cpu_to_le32(0x0000010C) // -EIO +#define STATUS_NO_QUOTAS_FOR_ACCOUNT cpu_to_le32(0x0000010D) // -EIO +#define STATUS_PRIMARY_TRANSPORT_CONNECT_FAILED cpu_to_le32(0x0000010E) // -EIO +#define STATUS_PAGE_FAULT_TRANSITION cpu_to_le32(0x00000110) // -EIO +#define STATUS_PAGE_FAULT_DEMAND_ZERO cpu_to_le32(0x00000111) // -EIO +#define STATUS_PAGE_FAULT_COPY_ON_WRITE cpu_to_le32(0x00000112) // -EIO +#define STATUS_PAGE_FAULT_GUARD_PAGE cpu_to_le32(0x00000113) // -EIO +#define STATUS_PAGE_FAULT_PAGING_FILE cpu_to_le32(0x00000114) // -EIO +#define STATUS_CACHE_PAGE_LOCKED cpu_to_le32(0x00000115) // -EIO +#define STATUS_CRASH_DUMP cpu_to_le32(0x00000116) // -EIO +#define STATUS_BUFFER_ALL_ZEROS cpu_to_le32(0x00000117) // -EIO +#define STATUS_REPARSE_OBJECT cpu_to_le32(0x00000118) // -EIO +#define STATUS_RESOURCE_REQUIREMENTS_CHANGED cpu_to_le32(0x00000119) // -EIO +#define STATUS_TRANSLATION_COMPLETE cpu_to_le32(0x00000120) // -EIO +#define STATUS_DS_MEMBERSHIP_EVALUATED_LOCALLY cpu_to_le32(0x00000121) // -EIO +#define STATUS_NOTHING_TO_TERMINATE cpu_to_le32(0x00000122) // -EIO +#define STATUS_PROCESS_NOT_IN_JOB cpu_to_le32(0x00000123) // -EIO +#define STATUS_PROCESS_IN_JOB cpu_to_le32(0x00000124) // -EIO +#define STATUS_VOLSNAP_HIBERNATE_READY cpu_to_le32(0x00000125) // -EIO +#define STATUS_FSFILTER_OP_COMPLETED_SUCCESSFULLY cpu_to_le32(0x00000126) // -EIO +#define STATUS_INTERRUPT_VECTOR_ALREADY_CONNECTED cpu_to_le32(0x00000127) // -EIO +#define STATUS_INTERRUPT_STILL_CONNECTED cpu_to_le32(0x00000128) // -EIO +#define STATUS_PROCESS_CLONED cpu_to_le32(0x00000129) // -EIO +#define STATUS_FILE_LOCKED_WITH_ONLY_READERS cpu_to_le32(0x0000012A) // -EIO +#define STATUS_FILE_LOCKED_WITH_WRITERS cpu_to_le32(0x0000012B) // -EIO +#define STATUS_RESOURCEMANAGER_READ_ONLY cpu_to_le32(0x00000202) // -EROFS +#define STATUS_WAIT_FOR_OPLOCK cpu_to_le32(0x00000367) // -EIO +#define DBG_EXCEPTION_HANDLED cpu_to_le32(0x00010001) // -EIO +#define DBG_CONTINUE cpu_to_le32(0x00010002) // -EIO +#define STATUS_FLT_IO_COMPLETE cpu_to_le32(0x001C0001) // -EIO +#define STATUS_OBJECT_NAME_EXISTS cpu_to_le32(0x40000000) // -EIO +#define STATUS_THREAD_WAS_SUSPENDED cpu_to_le32(0x40000001) // -EIO +#define STATUS_WORKING_SET_LIMIT_RANGE cpu_to_le32(0x40000002) // -EIO +#define STATUS_IMAGE_NOT_AT_BASE cpu_to_le32(0x40000003) // -EIO +#define STATUS_RXACT_STATE_CREATED cpu_to_le32(0x40000004) // -EIO +#define STATUS_SEGMENT_NOTIFICATION cpu_to_le32(0x40000005) // -EIO +#define STATUS_LOCAL_USER_SESSION_KEY cpu_to_le32(0x40000006) // -EIO +#define STATUS_BAD_CURRENT_DIRECTORY cpu_to_le32(0x40000007) // -EIO +#define STATUS_SERIAL_MORE_WRITES cpu_to_le32(0x40000008) // -EIO +#define STATUS_REGISTRY_RECOVERED cpu_to_le32(0x40000009) // -EIO +#define STATUS_FT_READ_RECOVERY_FROM_BACKUP cpu_to_le32(0x4000000A) // -EIO +#define STATUS_FT_WRITE_RECOVERY cpu_to_le32(0x4000000B) // -EIO +#define STATUS_SERIAL_COUNTER_TIMEOUT cpu_to_le32(0x4000000C) // -ETIMEDOUT +#define STATUS_NULL_LM_PASSWORD cpu_to_le32(0x4000000D) // -EIO +#define STATUS_IMAGE_MACHINE_TYPE_MISMATCH cpu_to_le32(0x4000000E) // -EIO +#define STATUS_RECEIVE_PARTIAL cpu_to_le32(0x4000000F) // -EIO +#define STATUS_RECEIVE_EXPEDITED cpu_to_le32(0x40000010) // -EIO +#define STATUS_RECEIVE_PARTIAL_EXPEDITED cpu_to_le32(0x40000011) // -EIO +#define STATUS_EVENT_DONE cpu_to_le32(0x40000012) // -EIO +#define STATUS_EVENT_PENDING cpu_to_le32(0x40000013) // -EIO +#define STATUS_CHECKING_FILE_SYSTEM cpu_to_le32(0x40000014) // -EIO +#define STATUS_FATAL_APP_EXIT cpu_to_le32(0x40000015) // -EIO +#define STATUS_PREDEFINED_HANDLE cpu_to_le32(0x40000016) // -EIO +#define STATUS_WAS_UNLOCKED cpu_to_le32(0x40000017) // -EIO +#define STATUS_SERVICE_NOTIFICATION cpu_to_le32(0x40000018) // -EIO +#define STATUS_WAS_LOCKED cpu_to_le32(0x40000019) // -EIO +#define STATUS_LOG_HARD_ERROR cpu_to_le32(0x4000001A) // -EIO +#define STATUS_ALREADY_WIN32 cpu_to_le32(0x4000001B) // -EIO +#define STATUS_WX86_UNSIMULATE cpu_to_le32(0x4000001C) // -EIO +#define STATUS_WX86_CONTINUE cpu_to_le32(0x4000001D) // -EIO +#define STATUS_WX86_SINGLE_STEP cpu_to_le32(0x4000001E) // -EIO +#define STATUS_WX86_BREAKPOINT cpu_to_le32(0x4000001F) // -EIO +#define STATUS_WX86_EXCEPTION_CONTINUE cpu_to_le32(0x40000020) // -EIO +#define STATUS_WX86_EXCEPTION_LASTCHANCE cpu_to_le32(0x40000021) // -EIO +#define STATUS_WX86_EXCEPTION_CHAIN cpu_to_le32(0x40000022) // -EIO +#define STATUS_IMAGE_MACHINE_TYPE_MISMATCH_EXE cpu_to_le32(0x40000023) // -EIO +#define STATUS_NO_YIELD_PERFORMED cpu_to_le32(0x40000024) // -EIO +#define STATUS_TIMER_RESUME_IGNORED cpu_to_le32(0x40000025) // -EIO +#define STATUS_ARBITRATION_UNHANDLED cpu_to_le32(0x40000026) // -EIO +#define STATUS_CARDBUS_NOT_SUPPORTED cpu_to_le32(0x40000027) // -ENOSYS +#define STATUS_WX86_CREATEWX86TIB cpu_to_le32(0x40000028) // -EIO +#define STATUS_MP_PROCESSOR_MISMATCH cpu_to_le32(0x40000029) // -EIO +#define STATUS_HIBERNATED cpu_to_le32(0x4000002A) // -EIO +#define STATUS_RESUME_HIBERNATION cpu_to_le32(0x4000002B) // -EIO +#define STATUS_FIRMWARE_UPDATED cpu_to_le32(0x4000002C) // -EIO +#define STATUS_DRIVERS_LEAKING_LOCKED_PAGES cpu_to_le32(0x4000002D) // -EIO +#define STATUS_MESSAGE_RETRIEVED cpu_to_le32(0x4000002E) // -EIO +#define STATUS_SYSTEM_POWERSTATE_TRANSITION cpu_to_le32(0x4000002F) // -EIO +#define STATUS_ALPC_CHECK_COMPLETION_LIST cpu_to_le32(0x40000030) // -EIO +#define STATUS_SYSTEM_POWERSTATE_COMPLEX_TRANSITION cpu_to_le32(0x40000031) // -EIO +#define STATUS_ACCESS_AUDIT_BY_POLICY cpu_to_le32(0x40000032) // -EIO +#define STATUS_ABANDON_HIBERFILE cpu_to_le32(0x40000033) // -EIO +#define STATUS_BIZRULES_NOT_ENABLED cpu_to_le32(0x40000034) // -EIO +#define STATUS_WAKE_SYSTEM cpu_to_le32(0x40000294) // -EIO +#define STATUS_DS_SHUTTING_DOWN cpu_to_le32(0x40000370) // -EIO +#define DBG_REPLY_LATER cpu_to_le32(0x40010001) // -EIO +#define DBG_UNABLE_TO_PROVIDE_HANDLE cpu_to_le32(0x40010002) // -EIO +#define DBG_TERMINATE_THREAD cpu_to_le32(0x40010003) // -EIO +#define DBG_TERMINATE_PROCESS cpu_to_le32(0x40010004) // -EIO +#define DBG_CONTROL_C cpu_to_le32(0x40010005) // -EIO +#define DBG_PRINTEXCEPTION_C cpu_to_le32(0x40010006) // -EIO +#define DBG_RIPEXCEPTION cpu_to_le32(0x40010007) // -EIO +#define DBG_CONTROL_BREAK cpu_to_le32(0x40010008) // -EIO +#define DBG_COMMAND_EXCEPTION cpu_to_le32(0x40010009) // -EIO +#define RPC_NT_UUID_LOCAL_ONLY cpu_to_le32(0x40020056) // -EIO +#define RPC_NT_SEND_INCOMPLETE cpu_to_le32(0x400200AF) // -EIO +#define STATUS_CTX_CDM_CONNECT cpu_to_le32(0x400A0004) // -EIO +#define STATUS_CTX_CDM_DISCONNECT cpu_to_le32(0x400A0005) // -EIO +#define STATUS_SXS_RELEASE_ACTIVATION_CONTEXT cpu_to_le32(0x4015000D) // -EIO +#define STATUS_RECOVERY_NOT_NEEDED cpu_to_le32(0x40190034) // -EIO +#define STATUS_RM_ALREADY_STARTED cpu_to_le32(0x40190035) // -EIO +#define STATUS_LOG_NO_RESTART cpu_to_le32(0x401A000C) // -EIO +#define STATUS_VIDEO_DRIVER_DEBUG_REPORT_REQUEST cpu_to_le32(0x401B00EC) // -EIO +#define STATUS_GRAPHICS_PARTIAL_DATA_POPULATED cpu_to_le32(0x401E000A) // -EIO +#define STATUS_GRAPHICS_DRIVER_MISMATCH cpu_to_le32(0x401E0117) // -EIO +#define STATUS_GRAPHICS_MODE_NOT_PINNED cpu_to_le32(0x401E0307) // -EIO +#define STATUS_GRAPHICS_NO_PREFERRED_MODE cpu_to_le32(0x401E031E) // -EIO +#define STATUS_GRAPHICS_DATASET_IS_EMPTY cpu_to_le32(0x401E034B) // -EIO +#define STATUS_GRAPHICS_NO_MORE_ELEMENTS_IN_DATASET cpu_to_le32(0x401E034C) // -EIO +#define STATUS_GRAPHICS_PATH_CONTENT_GEOMETRY_TRANSFORMATION_NOT_PINNED cpu_to_le32(0x401E0351) // -EIO +#define STATUS_GRAPHICS_UNKNOWN_CHILD_STATUS cpu_to_le32(0x401E042F) // -EIO +#define STATUS_GRAPHICS_LEADLINK_START_DEFERRED cpu_to_le32(0x401E0437) // -EIO +#define STATUS_GRAPHICS_POLLING_TOO_FREQUENTLY cpu_to_le32(0x401E0439) // -EIO +#define STATUS_GRAPHICS_START_DEFERRED cpu_to_le32(0x401E043A) // -EIO +#define STATUS_NDIS_INDICATION_REQUIRED cpu_to_le32(0x40230001) // -EIO +#define STATUS_GUARD_PAGE_VIOLATION cpu_to_le32(0x80000001) // -EIO +#define STATUS_DATATYPE_MISALIGNMENT cpu_to_le32(0x80000002) // -EIO +#define STATUS_BREAKPOINT cpu_to_le32(0x80000003) // -EIO +#define STATUS_SINGLE_STEP cpu_to_le32(0x80000004) // -EIO +#define STATUS_BUFFER_OVERFLOW cpu_to_le32(0x80000005) // -E2BIG +#define STATUS_NO_MORE_FILES cpu_to_le32(0x80000006) // -ENODATA +#define STATUS_WAKE_SYSTEM_DEBUGGER cpu_to_le32(0x80000007) // -EIO +#define STATUS_HANDLES_CLOSED cpu_to_le32(0x8000000A) // -EIO +#define STATUS_NO_INHERITANCE cpu_to_le32(0x8000000B) // -EIO +#define STATUS_GUID_SUBSTITUTION_MADE cpu_to_le32(0x8000000C) // -EIO +#define STATUS_PARTIAL_COPY cpu_to_le32(0x8000000D) // -EIO +#define STATUS_DEVICE_PAPER_EMPTY cpu_to_le32(0x8000000E) // -EIO +#define STATUS_DEVICE_POWERED_OFF cpu_to_le32(0x8000000F) // -EIO +#define STATUS_DEVICE_OFF_LINE cpu_to_le32(0x80000010) // -EIO +#define STATUS_DEVICE_BUSY cpu_to_le32(0x80000011) // -EBUSY +#define STATUS_NO_MORE_EAS cpu_to_le32(0x80000012) // -EIO +#define STATUS_INVALID_EA_NAME cpu_to_le32(0x80000013) // -EINVAL +#define STATUS_EA_LIST_INCONSISTENT cpu_to_le32(0x80000014) // -EIO +#define STATUS_INVALID_EA_FLAG cpu_to_le32(0x80000015) // -EINVAL +#define STATUS_VERIFY_REQUIRED cpu_to_le32(0x80000016) // -EIO +#define STATUS_EXTRANEOUS_INFORMATION cpu_to_le32(0x80000017) // -EIO +#define STATUS_RXACT_COMMIT_NECESSARY cpu_to_le32(0x80000018) // -EIO +#define STATUS_NO_MORE_ENTRIES cpu_to_le32(0x8000001A) // -EIO +#define STATUS_FILEMARK_DETECTED cpu_to_le32(0x8000001B) // -EIO +#define STATUS_MEDIA_CHANGED cpu_to_le32(0x8000001C) // -EIO +#define STATUS_BUS_RESET cpu_to_le32(0x8000001D) // -EIO +#define STATUS_END_OF_MEDIA cpu_to_le32(0x8000001E) // -EIO +#define STATUS_BEGINNING_OF_MEDIA cpu_to_le32(0x8000001F) // -EIO +#define STATUS_MEDIA_CHECK cpu_to_le32(0x80000020) // -EIO +#define STATUS_SETMARK_DETECTED cpu_to_le32(0x80000021) // -EIO +#define STATUS_NO_DATA_DETECTED cpu_to_le32(0x80000022) // -EIO +#define STATUS_REDIRECTOR_HAS_OPEN_HANDLES cpu_to_le32(0x80000023) // -EIO +#define STATUS_SERVER_HAS_OPEN_HANDLES cpu_to_le32(0x80000024) // -EIO +#define STATUS_ALREADY_DISCONNECTED cpu_to_le32(0x80000025) // -EIO +#define STATUS_LONGJUMP cpu_to_le32(0x80000026) // -EIO +#define STATUS_CLEANER_CARTRIDGE_INSTALLED cpu_to_le32(0x80000027) // -EIO +#define STATUS_PLUGPLAY_QUERY_VETOED cpu_to_le32(0x80000028) // -EIO +#define STATUS_UNWIND_CONSOLIDATE cpu_to_le32(0x80000029) // -EIO +#define STATUS_REGISTRY_HIVE_RECOVERED cpu_to_le32(0x8000002A) // -EIO +#define STATUS_DLL_MIGHT_BE_INSECURE cpu_to_le32(0x8000002B) // -EIO +#define STATUS_DLL_MIGHT_BE_INCOMPATIBLE cpu_to_le32(0x8000002C) // -EIO +#define STATUS_STOPPED_ON_SYMLINK cpu_to_le32(0x8000002D) // -EOPNOTSUPP +#define STATUS_DEVICE_REQUIRES_CLEANING cpu_to_le32(0x80000288) // -EIO +#define STATUS_DEVICE_DOOR_OPEN cpu_to_le32(0x80000289) // -EIO +#define STATUS_DATA_LOST_REPAIR cpu_to_le32(0x80000803) // -EIO +#define DBG_EXCEPTION_NOT_HANDLED cpu_to_le32(0x80010001) // -EIO +#define STATUS_CLUSTER_NODE_ALREADY_UP cpu_to_le32(0x80130001) // -EIO +#define STATUS_CLUSTER_NODE_ALREADY_DOWN cpu_to_le32(0x80130002) // -EIO +#define STATUS_CLUSTER_NETWORK_ALREADY_ONLINE cpu_to_le32(0x80130003) // -EIO +#define STATUS_CLUSTER_NETWORK_ALREADY_OFFLINE cpu_to_le32(0x80130004) // -EIO +#define STATUS_CLUSTER_NODE_ALREADY_MEMBER cpu_to_le32(0x80130005) // -EIO +#define STATUS_COULD_NOT_RESIZE_LOG cpu_to_le32(0x80190009) // -EIO +#define STATUS_NO_TXF_METADATA cpu_to_le32(0x80190029) // -EIO +#define STATUS_CANT_RECOVER_WITH_HANDLE_OPEN cpu_to_le32(0x80190031) // -EIO +#define STATUS_TXF_METADATA_ALREADY_PRESENT cpu_to_le32(0x80190041) // -EIO +#define STATUS_TRANSACTION_SCOPE_CALLBACKS_NOT_SET cpu_to_le32(0x80190042) // -EIO +#define STATUS_VIDEO_HUNG_DISPLAY_DRIVER_THREAD_RECOVERED cpu_to_le32(0x801B00EB) // -EIO +#define STATUS_FLT_BUFFER_TOO_SMALL cpu_to_le32(0x801C0001) // -ENOBUFS +#define STATUS_FVE_PARTIAL_METADATA cpu_to_le32(0x80210001) // -EIO +#define STATUS_UNSUCCESSFUL cpu_to_le32(0xC0000001) // -EIO +#define STATUS_NOT_IMPLEMENTED cpu_to_le32(0xC0000002) // -EOPNOTSUPP +#define STATUS_INVALID_INFO_CLASS cpu_to_le32(0xC0000003) // -EIO +#define STATUS_INFO_LENGTH_MISMATCH cpu_to_le32(0xC0000004) // -EIO +#define STATUS_ACCESS_VIOLATION cpu_to_le32(0xC0000005) // -EACCES +#define STATUS_IN_PAGE_ERROR cpu_to_le32(0xC0000006) // -EFAULT +#define STATUS_PAGEFILE_QUOTA cpu_to_le32(0xC0000007) // -EDQUOT +#define STATUS_INVALID_HANDLE cpu_to_le32(0xC0000008) // -EBADF +#define STATUS_BAD_INITIAL_STACK cpu_to_le32(0xC0000009) // -EIO +#define STATUS_BAD_INITIAL_PC cpu_to_le32(0xC000000A) // -EIO +#define STATUS_INVALID_CID cpu_to_le32(0xC000000B) // -EIO +#define STATUS_TIMER_NOT_CANCELED cpu_to_le32(0xC000000C) // -EIO +#define STATUS_INVALID_PARAMETER cpu_to_le32(0xC000000D) // -EINVAL +#define STATUS_NO_SUCH_DEVICE cpu_to_le32(0xC000000E) // -ENODEV +#define STATUS_NO_SUCH_FILE cpu_to_le32(0xC000000F) // -ENOENT +#define STATUS_INVALID_DEVICE_REQUEST cpu_to_le32(0xC0000010) // -EOPNOTSUPP +#define STATUS_END_OF_FILE cpu_to_le32(0xC0000011) // -ENODATA +#define STATUS_WRONG_VOLUME cpu_to_le32(0xC0000012) // -EIO +#define STATUS_NO_MEDIA_IN_DEVICE cpu_to_le32(0xC0000013) // -EIO +#define STATUS_UNRECOGNIZED_MEDIA cpu_to_le32(0xC0000014) // -EIO +#define STATUS_NONEXISTENT_SECTOR cpu_to_le32(0xC0000015) // -EIO +#define STATUS_MORE_PROCESSING_REQUIRED cpu_to_le32(0xC0000016) // -EIO +#define STATUS_NO_MEMORY cpu_to_le32(0xC0000017) // -EREMOTEIO +#define STATUS_CONFLICTING_ADDRESSES cpu_to_le32(0xC0000018) // -EADDRINUSE +#define STATUS_NOT_MAPPED_VIEW cpu_to_le32(0xC0000019) // -EIO +#define STATUS_UNABLE_TO_FREE_VM cpu_to_le32(0xC000001A) // -EIO +#define STATUS_UNABLE_TO_DELETE_SECTION cpu_to_le32(0xC000001B) // -EIO +#define STATUS_INVALID_SYSTEM_SERVICE cpu_to_le32(0xC000001C) // -EIO +#define STATUS_ILLEGAL_INSTRUCTION cpu_to_le32(0xC000001D) // -EIO +#define STATUS_INVALID_LOCK_SEQUENCE cpu_to_le32(0xC000001E) // -EIO +#define STATUS_INVALID_VIEW_SIZE cpu_to_le32(0xC000001F) // -EIO +#define STATUS_INVALID_FILE_FOR_SECTION cpu_to_le32(0xC0000020) // -EIO +#define STATUS_ALREADY_COMMITTED cpu_to_le32(0xC0000021) // -EIO +#define STATUS_ACCESS_DENIED cpu_to_le32(0xC0000022) // -EACCES +#define STATUS_BUFFER_TOO_SMALL cpu_to_le32(0xC0000023) // -EIO +#define STATUS_OBJECT_TYPE_MISMATCH cpu_to_le32(0xC0000024) // -EIO +#define STATUS_NONCONTINUABLE_EXCEPTION cpu_to_le32(0xC0000025) // -EIO +#define STATUS_INVALID_DISPOSITION cpu_to_le32(0xC0000026) // -EIO +#define STATUS_UNWIND cpu_to_le32(0xC0000027) // -EIO +#define STATUS_BAD_STACK cpu_to_le32(0xC0000028) // -EIO +#define STATUS_INVALID_UNWIND_TARGET cpu_to_le32(0xC0000029) // -EIO +#define STATUS_NOT_LOCKED cpu_to_le32(0xC000002A) // -EIO +#define STATUS_PARITY_ERROR cpu_to_le32(0xC000002B) // -EIO +#define STATUS_UNABLE_TO_DECOMMIT_VM cpu_to_le32(0xC000002C) // -EIO +#define STATUS_NOT_COMMITTED cpu_to_le32(0xC000002D) // -EIO +#define STATUS_INVALID_PORT_ATTRIBUTES cpu_to_le32(0xC000002E) // -EIO +#define STATUS_PORT_MESSAGE_TOO_LONG cpu_to_le32(0xC000002F) // -EIO +#define STATUS_INVALID_PARAMETER_MIX cpu_to_le32(0xC0000030) // -EINVAL +#define STATUS_INVALID_QUOTA_LOWER cpu_to_le32(0xC0000031) // -EIO +#define STATUS_DISK_CORRUPT_ERROR cpu_to_le32(0xC0000032) // -EIO +#define STATUS_OBJECT_NAME_INVALID cpu_to_le32(0xC0000033) // -ENOENT +#define STATUS_OBJECT_NAME_NOT_FOUND cpu_to_le32(0xC0000034) // -ENOENT +#define STATUS_OBJECT_NAME_COLLISION cpu_to_le32(0xC0000035) // -EEXIST +#define STATUS_PORT_DISCONNECTED cpu_to_le32(0xC0000037) // -EIO +#define STATUS_DEVICE_ALREADY_ATTACHED cpu_to_le32(0xC0000038) // -EIO +#define STATUS_OBJECT_PATH_INVALID cpu_to_le32(0xC0000039) // -ENOTDIR +#define STATUS_OBJECT_PATH_NOT_FOUND cpu_to_le32(0xC000003A) // -ENOENT +#define STATUS_OBJECT_PATH_SYNTAX_BAD cpu_to_le32(0xC000003B) // -EIO +#define STATUS_DATA_OVERRUN cpu_to_le32(0xC000003C) // -EIO +#define STATUS_DATA_LATE_ERROR cpu_to_le32(0xC000003D) // -EIO +#define STATUS_DATA_ERROR cpu_to_le32(0xC000003E) // -EIO +#define STATUS_CRC_ERROR cpu_to_le32(0xC000003F) // -EIO +#define STATUS_SECTION_TOO_BIG cpu_to_le32(0xC0000040) // -EIO +#define STATUS_PORT_CONNECTION_REFUSED cpu_to_le32(0xC0000041) // -ECONNREFUSED +#define STATUS_INVALID_PORT_HANDLE cpu_to_le32(0xC0000042) // -EIO +#define STATUS_SHARING_VIOLATION cpu_to_le32(0xC0000043) // -EBUSY +#define STATUS_QUOTA_EXCEEDED cpu_to_le32(0xC0000044) // -EDQUOT +#define STATUS_INVALID_PAGE_PROTECTION cpu_to_le32(0xC0000045) // -EIO +#define STATUS_MUTANT_NOT_OWNED cpu_to_le32(0xC0000046) // -EIO +#define STATUS_SEMAPHORE_LIMIT_EXCEEDED cpu_to_le32(0xC0000047) // -EIO +#define STATUS_PORT_ALREADY_SET cpu_to_le32(0xC0000048) // -EIO +#define STATUS_SECTION_NOT_IMAGE cpu_to_le32(0xC0000049) // -EIO +#define STATUS_SUSPEND_COUNT_EXCEEDED cpu_to_le32(0xC000004A) // -EIO +#define STATUS_THREAD_IS_TERMINATING cpu_to_le32(0xC000004B) // -EIO +#define STATUS_BAD_WORKING_SET_LIMIT cpu_to_le32(0xC000004C) // -EIO +#define STATUS_INCOMPATIBLE_FILE_MAP cpu_to_le32(0xC000004D) // -EIO +#define STATUS_SECTION_PROTECTION cpu_to_le32(0xC000004E) // -EIO +#define STATUS_EAS_NOT_SUPPORTED cpu_to_le32(0xC000004F) // -EOPNOTSUPP +#define STATUS_EA_TOO_LARGE cpu_to_le32(0xC0000050) // -EIO +#define STATUS_NONEXISTENT_EA_ENTRY cpu_to_le32(0xC0000051) // -EIO +#define STATUS_NO_EAS_ON_FILE cpu_to_le32(0xC0000052) // -ENODATA +#define STATUS_EA_CORRUPT_ERROR cpu_to_le32(0xC0000053) // -EIO +#define STATUS_FILE_LOCK_CONFLICT cpu_to_le32(0xC0000054) // -EACCES +#define STATUS_LOCK_NOT_GRANTED cpu_to_le32(0xC0000055) // -EACCES +#define STATUS_DELETE_PENDING cpu_to_le32(0xC0000056) // -ENOENT +#define STATUS_CTL_FILE_NOT_SUPPORTED cpu_to_le32(0xC0000057) // -ENOSYS +#define STATUS_UNKNOWN_REVISION cpu_to_le32(0xC0000058) // -EIO +#define STATUS_REVISION_MISMATCH cpu_to_le32(0xC0000059) // -EIO +#define STATUS_INVALID_OWNER cpu_to_le32(0xC000005A) // -EIO +#define STATUS_INVALID_PRIMARY_GROUP cpu_to_le32(0xC000005B) // -EIO +#define STATUS_NO_IMPERSONATION_TOKEN cpu_to_le32(0xC000005C) // -EIO +#define STATUS_CANT_DISABLE_MANDATORY cpu_to_le32(0xC000005D) // -EIO +#define STATUS_NO_LOGON_SERVERS cpu_to_le32(0xC000005E) // -EIO +#define STATUS_NO_SUCH_LOGON_SESSION cpu_to_le32(0xC000005F) // -EIO +#define STATUS_NO_SUCH_PRIVILEGE cpu_to_le32(0xC0000060) // -EIO +#define STATUS_PRIVILEGE_NOT_HELD cpu_to_le32(0xC0000061) // -EPERM +#define STATUS_INVALID_ACCOUNT_NAME cpu_to_le32(0xC0000062) // -EIO +#define STATUS_USER_EXISTS cpu_to_le32(0xC0000063) // -EIO +#define STATUS_NO_SUCH_USER cpu_to_le32(0xC0000064) // -EIO +#define STATUS_GROUP_EXISTS cpu_to_le32(0xC0000065) // -EIO +#define STATUS_NO_SUCH_GROUP cpu_to_le32(0xC0000066) // -EIO +#define STATUS_MEMBER_IN_GROUP cpu_to_le32(0xC0000067) // -EIO +#define STATUS_MEMBER_NOT_IN_GROUP cpu_to_le32(0xC0000068) // -EIO +#define STATUS_LAST_ADMIN cpu_to_le32(0xC0000069) // -EIO +#define STATUS_WRONG_PASSWORD cpu_to_le32(0xC000006A) // -EACCES +#define STATUS_ILL_FORMED_PASSWORD cpu_to_le32(0xC000006B) // -EINVAL +#define STATUS_PASSWORD_RESTRICTION cpu_to_le32(0xC000006C) // -EACCES +#define STATUS_LOGON_FAILURE cpu_to_le32(0xC000006D) // -EACCES +#define STATUS_ACCOUNT_RESTRICTION cpu_to_le32(0xC000006E) // -EACCES +#define STATUS_INVALID_LOGON_HOURS cpu_to_le32(0xC000006F) // -EACCES +#define STATUS_INVALID_WORKSTATION cpu_to_le32(0xC0000070) // -EACCES +#define STATUS_PASSWORD_EXPIRED cpu_to_le32(0xC0000071) // -EKEYEXPIRED +#define STATUS_ACCOUNT_DISABLED cpu_to_le32(0xC0000072) // -EKEYREVOKED +#define STATUS_NONE_MAPPED cpu_to_le32(0xC0000073) // -EIO +#define STATUS_TOO_MANY_LUIDS_REQUESTED cpu_to_le32(0xC0000074) // -EIO +#define STATUS_LUIDS_EXHAUSTED cpu_to_le32(0xC0000075) // -EIO +#define STATUS_INVALID_SUB_AUTHORITY cpu_to_le32(0xC0000076) // -EIO +#define STATUS_INVALID_ACL cpu_to_le32(0xC0000077) // -EIO +#define STATUS_INVALID_SID cpu_to_le32(0xC0000078) // -EIO +#define STATUS_INVALID_SECURITY_DESCR cpu_to_le32(0xC0000079) // -EIO +#define STATUS_PROCEDURE_NOT_FOUND cpu_to_le32(0xC000007A) // -EIO +#define STATUS_INVALID_IMAGE_FORMAT cpu_to_le32(0xC000007B) // -EIO +#define STATUS_NO_TOKEN cpu_to_le32(0xC000007C) // -EIO +#define STATUS_BAD_INHERITANCE_ACL cpu_to_le32(0xC000007D) // -EIO +#define STATUS_RANGE_NOT_LOCKED cpu_to_le32(0xC000007E) // -EIO +#define STATUS_DISK_FULL cpu_to_le32(0xC000007F) // -ENOSPC +#define STATUS_SERVER_DISABLED cpu_to_le32(0xC0000080) // -EIO +#define STATUS_SERVER_NOT_DISABLED cpu_to_le32(0xC0000081) // -EIO +#define STATUS_TOO_MANY_GUIDS_REQUESTED cpu_to_le32(0xC0000082) // -EIO +#define STATUS_GUIDS_EXHAUSTED cpu_to_le32(0xC0000083) // -EIO +#define STATUS_INVALID_ID_AUTHORITY cpu_to_le32(0xC0000084) // -EIO +#define STATUS_AGENTS_EXHAUSTED cpu_to_le32(0xC0000085) // -EIO +#define STATUS_INVALID_VOLUME_LABEL cpu_to_le32(0xC0000086) // -EIO +#define STATUS_SECTION_NOT_EXTENDED cpu_to_le32(0xC0000087) // -EIO +#define STATUS_NOT_MAPPED_DATA cpu_to_le32(0xC0000088) // -EIO +#define STATUS_RESOURCE_DATA_NOT_FOUND cpu_to_le32(0xC0000089) // -EIO +#define STATUS_RESOURCE_TYPE_NOT_FOUND cpu_to_le32(0xC000008A) // -EIO +#define STATUS_RESOURCE_NAME_NOT_FOUND cpu_to_le32(0xC000008B) // -EIO +#define STATUS_ARRAY_BOUNDS_EXCEEDED cpu_to_le32(0xC000008C) // -EIO +#define STATUS_FLOAT_DENORMAL_OPERAND cpu_to_le32(0xC000008D) // -EIO +#define STATUS_FLOAT_DIVIDE_BY_ZERO cpu_to_le32(0xC000008E) // -EIO +#define STATUS_FLOAT_INEXACT_RESULT cpu_to_le32(0xC000008F) // -EIO +#define STATUS_FLOAT_INVALID_OPERATION cpu_to_le32(0xC0000090) // -EIO +#define STATUS_FLOAT_OVERFLOW cpu_to_le32(0xC0000091) // -EIO +#define STATUS_FLOAT_STACK_CHECK cpu_to_le32(0xC0000092) // -EIO +#define STATUS_FLOAT_UNDERFLOW cpu_to_le32(0xC0000093) // -EIO +#define STATUS_INTEGER_DIVIDE_BY_ZERO cpu_to_le32(0xC0000094) // -EIO +#define STATUS_INTEGER_OVERFLOW cpu_to_le32(0xC0000095) // -EIO +#define STATUS_PRIVILEGED_INSTRUCTION cpu_to_le32(0xC0000096) // -EIO +#define STATUS_TOO_MANY_PAGING_FILES cpu_to_le32(0xC0000097) // -EIO +#define STATUS_FILE_INVALID cpu_to_le32(0xC0000098) // -EIO +#define STATUS_ALLOTTED_SPACE_EXCEEDED cpu_to_le32(0xC0000099) // -EIO +#define STATUS_INSUFFICIENT_RESOURCES cpu_to_le32(0xC000009A) // -EAGAIN +#define STATUS_DFS_EXIT_PATH_FOUND cpu_to_le32(0xC000009B) // -EIO +#define STATUS_DEVICE_DATA_ERROR cpu_to_le32(0xC000009C) // -EIO +#define STATUS_DEVICE_NOT_CONNECTED cpu_to_le32(0xC000009D) // -EIO +#define STATUS_DEVICE_POWER_FAILURE cpu_to_le32(0xC000009E) // -EIO +#define STATUS_FREE_VM_NOT_AT_BASE cpu_to_le32(0xC000009F) // -EIO +#define STATUS_MEMORY_NOT_ALLOCATED cpu_to_le32(0xC00000A0) // -EFAULT +#define STATUS_WORKING_SET_QUOTA cpu_to_le32(0xC00000A1) // -EIO +#define STATUS_MEDIA_WRITE_PROTECTED cpu_to_le32(0xC00000A2) // -EROFS +#define STATUS_DEVICE_NOT_READY cpu_to_le32(0xC00000A3) // -EIO +#define STATUS_INVALID_GROUP_ATTRIBUTES cpu_to_le32(0xC00000A4) // -EIO +#define STATUS_BAD_IMPERSONATION_LEVEL cpu_to_le32(0xC00000A5) // -EIO +#define STATUS_CANT_OPEN_ANONYMOUS cpu_to_le32(0xC00000A6) // -EIO +#define STATUS_BAD_VALIDATION_CLASS cpu_to_le32(0xC00000A7) // -EIO +#define STATUS_BAD_TOKEN_TYPE cpu_to_le32(0xC00000A8) // -EIO +#define STATUS_BAD_MASTER_BOOT_RECORD cpu_to_le32(0xC00000A9) // -EIO +#define STATUS_INSTRUCTION_MISALIGNMENT cpu_to_le32(0xC00000AA) // -EIO +#define STATUS_INSTANCE_NOT_AVAILABLE cpu_to_le32(0xC00000AB) // -EIO +#define STATUS_PIPE_NOT_AVAILABLE cpu_to_le32(0xC00000AC) // -EIO +#define STATUS_INVALID_PIPE_STATE cpu_to_le32(0xC00000AD) // -EIO +#define STATUS_PIPE_BUSY cpu_to_le32(0xC00000AE) // -EBUSY +#define STATUS_ILLEGAL_FUNCTION cpu_to_le32(0xC00000AF) // -EIO +#define STATUS_PIPE_DISCONNECTED cpu_to_le32(0xC00000B0) // -EPIPE +#define STATUS_PIPE_CLOSING cpu_to_le32(0xC00000B1) // -EIO +#define STATUS_PIPE_CONNECTED cpu_to_le32(0xC00000B2) // -EIO +#define STATUS_PIPE_LISTENING cpu_to_le32(0xC00000B3) // -EIO +#define STATUS_INVALID_READ_MODE cpu_to_le32(0xC00000B4) // -EIO +#define STATUS_IO_TIMEOUT cpu_to_le32(0xC00000B5) // -EAGAIN +#define STATUS_FILE_FORCED_CLOSED cpu_to_le32(0xC00000B6) // -EIO +#define STATUS_PROFILING_NOT_STARTED cpu_to_le32(0xC00000B7) // -EIO +#define STATUS_PROFILING_NOT_STOPPED cpu_to_le32(0xC00000B8) // -EIO +#define STATUS_COULD_NOT_INTERPRET cpu_to_le32(0xC00000B9) // -EIO +#define STATUS_FILE_IS_A_DIRECTORY cpu_to_le32(0xC00000BA) // -EISDIR +#define STATUS_NOT_SUPPORTED cpu_to_le32(0xC00000BB) // -EOPNOTSUPP +#define STATUS_REMOTE_NOT_LISTENING cpu_to_le32(0xC00000BC) // -EHOSTDOWN +#define STATUS_DUPLICATE_NAME cpu_to_le32(0xC00000BD) // -ENOTUNIQ +#define STATUS_BAD_NETWORK_PATH cpu_to_le32(0xC00000BE) // -EINVAL +#define STATUS_NETWORK_BUSY cpu_to_le32(0xC00000BF) // -EBUSY +#define STATUS_DEVICE_DOES_NOT_EXIST cpu_to_le32(0xC00000C0) // -ENODEV +#define STATUS_TOO_MANY_COMMANDS cpu_to_le32(0xC00000C1) // -EIO +#define STATUS_ADAPTER_HARDWARE_ERROR cpu_to_le32(0xC00000C2) // -EIO +#define STATUS_INVALID_NETWORK_RESPONSE cpu_to_le32(0xC00000C3) // -EIO +#define STATUS_UNEXPECTED_NETWORK_ERROR cpu_to_le32(0xC00000C4) // -EIO +#define STATUS_BAD_REMOTE_ADAPTER cpu_to_le32(0xC00000C5) // -EIO +#define STATUS_PRINT_QUEUE_FULL cpu_to_le32(0xC00000C6) // -EIO +#define STATUS_NO_SPOOL_SPACE cpu_to_le32(0xC00000C7) // -EIO +#define STATUS_PRINT_CANCELLED cpu_to_le32(0xC00000C8) // -EIO +#define STATUS_NETWORK_NAME_DELETED cpu_to_le32(0xC00000C9) // -EREMCHG +#define STATUS_NETWORK_ACCESS_DENIED cpu_to_le32(0xC00000CA) // -EACCES +#define STATUS_BAD_DEVICE_TYPE cpu_to_le32(0xC00000CB) // -EIO +#define STATUS_BAD_NETWORK_NAME cpu_to_le32(0xC00000CC) // -ENOENT +#define STATUS_TOO_MANY_NAMES cpu_to_le32(0xC00000CD) // -EIO +#define STATUS_TOO_MANY_SESSIONS cpu_to_le32(0xC00000CE) // -EIO +#define STATUS_SHARING_PAUSED cpu_to_le32(0xC00000CF) // -EIO +#define STATUS_REQUEST_NOT_ACCEPTED cpu_to_le32(0xC00000D0) // -EIO +#define STATUS_REDIRECTOR_PAUSED cpu_to_le32(0xC00000D1) // -EIO +#define STATUS_NET_WRITE_FAULT cpu_to_le32(0xC00000D2) // -EIO +#define STATUS_PROFILING_AT_LIMIT cpu_to_le32(0xC00000D3) // -EIO +#define STATUS_NOT_SAME_DEVICE cpu_to_le32(0xC00000D4) // -EXDEV +#define STATUS_FILE_RENAMED cpu_to_le32(0xC00000D5) // -EIO +#define STATUS_VIRTUAL_CIRCUIT_CLOSED cpu_to_le32(0xC00000D6) // -EIO +#define STATUS_NO_SECURITY_ON_OBJECT cpu_to_le32(0xC00000D7) // -EIO +#define STATUS_CANT_WAIT cpu_to_le32(0xC00000D8) // -EIO +#define STATUS_PIPE_EMPTY cpu_to_le32(0xC00000D9) // -EIO +#define STATUS_CANT_ACCESS_DOMAIN_INFO cpu_to_le32(0xC00000DA) // -EIO +#define STATUS_CANT_TERMINATE_SELF cpu_to_le32(0xC00000DB) // -EIO +#define STATUS_INVALID_SERVER_STATE cpu_to_le32(0xC00000DC) // -EIO +#define STATUS_INVALID_DOMAIN_STATE cpu_to_le32(0xC00000DD) // -EIO +#define STATUS_INVALID_DOMAIN_ROLE cpu_to_le32(0xC00000DE) // -EIO +#define STATUS_NO_SUCH_DOMAIN cpu_to_le32(0xC00000DF) // -EIO +#define STATUS_DOMAIN_EXISTS cpu_to_le32(0xC00000E0) // -EIO +#define STATUS_DOMAIN_LIMIT_EXCEEDED cpu_to_le32(0xC00000E1) // -EIO +#define STATUS_OPLOCK_NOT_GRANTED cpu_to_le32(0xC00000E2) // -EIO +#define STATUS_INVALID_OPLOCK_PROTOCOL cpu_to_le32(0xC00000E3) // -EIO +#define STATUS_INTERNAL_DB_CORRUPTION cpu_to_le32(0xC00000E4) // -EIO +#define STATUS_INTERNAL_ERROR cpu_to_le32(0xC00000E5) // -EIO +#define STATUS_GENERIC_NOT_MAPPED cpu_to_le32(0xC00000E6) // -EIO +#define STATUS_BAD_DESCRIPTOR_FORMAT cpu_to_le32(0xC00000E7) // -EIO +#define STATUS_INVALID_USER_BUFFER cpu_to_le32(0xC00000E8) // -EIO +#define STATUS_UNEXPECTED_IO_ERROR cpu_to_le32(0xC00000E9) // -EIO +#define STATUS_UNEXPECTED_MM_CREATE_ERR cpu_to_le32(0xC00000EA) // -EIO +#define STATUS_UNEXPECTED_MM_MAP_ERROR cpu_to_le32(0xC00000EB) // -EIO +#define STATUS_UNEXPECTED_MM_EXTEND_ERR cpu_to_le32(0xC00000EC) // -EIO +#define STATUS_NOT_LOGON_PROCESS cpu_to_le32(0xC00000ED) // -EIO +#define STATUS_LOGON_SESSION_EXISTS cpu_to_le32(0xC00000EE) // -EIO +#define STATUS_INVALID_PARAMETER_1 cpu_to_le32(0xC00000EF) // -EINVAL +#define STATUS_INVALID_PARAMETER_2 cpu_to_le32(0xC00000F0) // -EINVAL +#define STATUS_INVALID_PARAMETER_3 cpu_to_le32(0xC00000F1) // -EINVAL +#define STATUS_INVALID_PARAMETER_4 cpu_to_le32(0xC00000F2) // -EINVAL +#define STATUS_INVALID_PARAMETER_5 cpu_to_le32(0xC00000F3) // -EINVAL +#define STATUS_INVALID_PARAMETER_6 cpu_to_le32(0xC00000F4) // -EINVAL +#define STATUS_INVALID_PARAMETER_7 cpu_to_le32(0xC00000F5) // -EINVAL +#define STATUS_INVALID_PARAMETER_8 cpu_to_le32(0xC00000F6) // -EINVAL +#define STATUS_INVALID_PARAMETER_9 cpu_to_le32(0xC00000F7) // -EINVAL +#define STATUS_INVALID_PARAMETER_10 cpu_to_le32(0xC00000F8) // -EINVAL +#define STATUS_INVALID_PARAMETER_11 cpu_to_le32(0xC00000F9) // -EINVAL +#define STATUS_INVALID_PARAMETER_12 cpu_to_le32(0xC00000FA) // -EINVAL +#define STATUS_REDIRECTOR_NOT_STARTED cpu_to_le32(0xC00000FB) // -EIO +#define STATUS_REDIRECTOR_STARTED cpu_to_le32(0xC00000FC) // -EIO +#define STATUS_STACK_OVERFLOW cpu_to_le32(0xC00000FD) // -EIO +#define STATUS_NO_SUCH_PACKAGE cpu_to_le32(0xC00000FE) // -EIO +#define STATUS_BAD_FUNCTION_TABLE cpu_to_le32(0xC00000FF) // -EIO +#define STATUS_VARIABLE_NOT_FOUND cpu_to_le32(0xC0000100) // -EIO +#define STATUS_DIRECTORY_NOT_EMPTY cpu_to_le32(0xC0000101) // -ENOTEMPTY +#define STATUS_FILE_CORRUPT_ERROR cpu_to_le32(0xC0000102) // -EIO +#define STATUS_NOT_A_DIRECTORY cpu_to_le32(0xC0000103) // -ENOTDIR +#define STATUS_BAD_LOGON_SESSION_STATE cpu_to_le32(0xC0000104) // -EIO +#define STATUS_LOGON_SESSION_COLLISION cpu_to_le32(0xC0000105) // -EIO +#define STATUS_NAME_TOO_LONG cpu_to_le32(0xC0000106) // -ENAMETOOLONG +#define STATUS_FILES_OPEN cpu_to_le32(0xC0000107) // -EIO +#define STATUS_CONNECTION_IN_USE cpu_to_le32(0xC0000108) // -EIO +#define STATUS_MESSAGE_NOT_FOUND cpu_to_le32(0xC0000109) // -EIO +#define STATUS_PROCESS_IS_TERMINATING cpu_to_le32(0xC000010A) // -EIO +#define STATUS_INVALID_LOGON_TYPE cpu_to_le32(0xC000010B) // -EIO +#define STATUS_NO_GUID_TRANSLATION cpu_to_le32(0xC000010C) // -EIO +#define STATUS_CANNOT_IMPERSONATE cpu_to_le32(0xC000010D) // -EIO +#define STATUS_IMAGE_ALREADY_LOADED cpu_to_le32(0xC000010E) // -EIO +#define STATUS_ABIOS_NOT_PRESENT cpu_to_le32(0xC000010F) // -EIO +#define STATUS_ABIOS_LID_NOT_EXIST cpu_to_le32(0xC0000110) // -EIO +#define STATUS_ABIOS_LID_ALREADY_OWNED cpu_to_le32(0xC0000111) // -EIO +#define STATUS_ABIOS_NOT_LID_OWNER cpu_to_le32(0xC0000112) // -EIO +#define STATUS_ABIOS_INVALID_COMMAND cpu_to_le32(0xC0000113) // -EIO +#define STATUS_ABIOS_INVALID_LID cpu_to_le32(0xC0000114) // -EIO +#define STATUS_ABIOS_SELECTOR_NOT_AVAILABLE cpu_to_le32(0xC0000115) // -EIO +#define STATUS_ABIOS_INVALID_SELECTOR cpu_to_le32(0xC0000116) // -EIO +#define STATUS_NO_LDT cpu_to_le32(0xC0000117) // -EIO +#define STATUS_INVALID_LDT_SIZE cpu_to_le32(0xC0000118) // -EIO +#define STATUS_INVALID_LDT_OFFSET cpu_to_le32(0xC0000119) // -EIO +#define STATUS_INVALID_LDT_DESCRIPTOR cpu_to_le32(0xC000011A) // -EIO +#define STATUS_INVALID_IMAGE_NE_FORMAT cpu_to_le32(0xC000011B) // -EIO +#define STATUS_RXACT_INVALID_STATE cpu_to_le32(0xC000011C) // -EIO +#define STATUS_RXACT_COMMIT_FAILURE cpu_to_le32(0xC000011D) // -EIO +#define STATUS_MAPPED_FILE_SIZE_ZERO cpu_to_le32(0xC000011E) // -EIO +#define STATUS_TOO_MANY_OPENED_FILES cpu_to_le32(0xC000011F) // -EMFILE +#define STATUS_CANCELLED cpu_to_le32(0xC0000120) // -EIO +#define STATUS_CANNOT_DELETE cpu_to_le32(0xC0000121) // -EACCES +#define STATUS_INVALID_COMPUTER_NAME cpu_to_le32(0xC0000122) // -EIO +#define STATUS_FILE_DELETED cpu_to_le32(0xC0000123) // -EIO +#define STATUS_SPECIAL_ACCOUNT cpu_to_le32(0xC0000124) // -EIO +#define STATUS_SPECIAL_GROUP cpu_to_le32(0xC0000125) // -EIO +#define STATUS_SPECIAL_USER cpu_to_le32(0xC0000126) // -EIO +#define STATUS_MEMBERS_PRIMARY_GROUP cpu_to_le32(0xC0000127) // -EIO +#define STATUS_FILE_CLOSED cpu_to_le32(0xC0000128) // -EBADF +#define STATUS_TOO_MANY_THREADS cpu_to_le32(0xC0000129) // -EIO +#define STATUS_THREAD_NOT_IN_PROCESS cpu_to_le32(0xC000012A) // -EIO +#define STATUS_TOKEN_ALREADY_IN_USE cpu_to_le32(0xC000012B) // -EIO +#define STATUS_PAGEFILE_QUOTA_EXCEEDED cpu_to_le32(0xC000012C) // -EDQUOT +#define STATUS_COMMITMENT_LIMIT cpu_to_le32(0xC000012D) // -EIO +#define STATUS_INVALID_IMAGE_LE_FORMAT cpu_to_le32(0xC000012E) // -EIO +#define STATUS_INVALID_IMAGE_NOT_MZ cpu_to_le32(0xC000012F) // -EIO +#define STATUS_INVALID_IMAGE_PROTECT cpu_to_le32(0xC0000130) // -EIO +#define STATUS_INVALID_IMAGE_WIN_16 cpu_to_le32(0xC0000131) // -EIO +#define STATUS_LOGON_SERVER_CONFLICT cpu_to_le32(0xC0000132) // -EIO +#define STATUS_TIME_DIFFERENCE_AT_DC cpu_to_le32(0xC0000133) // -EIO +#define STATUS_SYNCHRONIZATION_REQUIRED cpu_to_le32(0xC0000134) // -EIO +#define STATUS_DLL_NOT_FOUND cpu_to_le32(0xC0000135) // -ENOENT +#define STATUS_OPEN_FAILED cpu_to_le32(0xC0000136) // -EIO +#define STATUS_IO_PRIVILEGE_FAILED cpu_to_le32(0xC0000137) // -EIO +#define STATUS_ORDINAL_NOT_FOUND cpu_to_le32(0xC0000138) // -EIO +#define STATUS_ENTRYPOINT_NOT_FOUND cpu_to_le32(0xC0000139) // -EIO +#define STATUS_CONTROL_C_EXIT cpu_to_le32(0xC000013A) // -EIO +#define STATUS_LOCAL_DISCONNECT cpu_to_le32(0xC000013B) // -EIO +#define STATUS_REMOTE_DISCONNECT cpu_to_le32(0xC000013C) // -ESHUTDOWN +#define STATUS_REMOTE_RESOURCES cpu_to_le32(0xC000013D) // -EIO +#define STATUS_LINK_FAILED cpu_to_le32(0xC000013E) // -EXDEV +#define STATUS_LINK_TIMEOUT cpu_to_le32(0xC000013F) // -ETIMEDOUT +#define STATUS_INVALID_CONNECTION cpu_to_le32(0xC0000140) // -EIO +#define STATUS_INVALID_ADDRESS cpu_to_le32(0xC0000141) // -EIO +#define STATUS_DLL_INIT_FAILED cpu_to_le32(0xC0000142) // -EIO +#define STATUS_MISSING_SYSTEMFILE cpu_to_le32(0xC0000143) // -EIO +#define STATUS_UNHANDLED_EXCEPTION cpu_to_le32(0xC0000144) // -EIO +#define STATUS_APP_INIT_FAILURE cpu_to_le32(0xC0000145) // -EIO +#define STATUS_PAGEFILE_CREATE_FAILED cpu_to_le32(0xC0000146) // -EIO +#define STATUS_NO_PAGEFILE cpu_to_le32(0xC0000147) // -EIO +#define STATUS_INVALID_LEVEL cpu_to_le32(0xC0000148) // -EIO +#define STATUS_WRONG_PASSWORD_CORE cpu_to_le32(0xC0000149) // -EIO +#define STATUS_ILLEGAL_FLOAT_CONTEXT cpu_to_le32(0xC000014A) // -EIO +#define STATUS_PIPE_BROKEN cpu_to_le32(0xC000014B) // -EPIPE +#define STATUS_REGISTRY_CORRUPT cpu_to_le32(0xC000014C) // -EIO +#define STATUS_REGISTRY_IO_FAILED cpu_to_le32(0xC000014D) // -EIO +#define STATUS_NO_EVENT_PAIR cpu_to_le32(0xC000014E) // -EIO +#define STATUS_UNRECOGNIZED_VOLUME cpu_to_le32(0xC000014F) // -EIO +#define STATUS_SERIAL_NO_DEVICE_INITED cpu_to_le32(0xC0000150) // -EIO +#define STATUS_NO_SUCH_ALIAS cpu_to_le32(0xC0000151) // -EIO +#define STATUS_MEMBER_NOT_IN_ALIAS cpu_to_le32(0xC0000152) // -EIO +#define STATUS_MEMBER_IN_ALIAS cpu_to_le32(0xC0000153) // -EIO +#define STATUS_ALIAS_EXISTS cpu_to_le32(0xC0000154) // -EIO +#define STATUS_LOGON_NOT_GRANTED cpu_to_le32(0xC0000155) // -EIO +#define STATUS_TOO_MANY_SECRETS cpu_to_le32(0xC0000156) // -EIO +#define STATUS_SECRET_TOO_LONG cpu_to_le32(0xC0000157) // -EIO +#define STATUS_INTERNAL_DB_ERROR cpu_to_le32(0xC0000158) // -EIO +#define STATUS_FULLSCREEN_MODE cpu_to_le32(0xC0000159) // -EIO +#define STATUS_TOO_MANY_CONTEXT_IDS cpu_to_le32(0xC000015A) // -EIO +#define STATUS_LOGON_TYPE_NOT_GRANTED cpu_to_le32(0xC000015B) // -EIO +#define STATUS_NOT_REGISTRY_FILE cpu_to_le32(0xC000015C) // -EIO +#define STATUS_NT_CROSS_ENCRYPTION_REQUIRED cpu_to_le32(0xC000015D) // -EIO +#define STATUS_DOMAIN_CTRLR_CONFIG_ERROR cpu_to_le32(0xC000015E) // -EIO +#define STATUS_FT_MISSING_MEMBER cpu_to_le32(0xC000015F) // -EIO +#define STATUS_ILL_FORMED_SERVICE_ENTRY cpu_to_le32(0xC0000160) // -EIO +#define STATUS_ILLEGAL_CHARACTER cpu_to_le32(0xC0000161) // -EIO +#define STATUS_UNMAPPABLE_CHARACTER cpu_to_le32(0xC0000162) // -EIO +#define STATUS_UNDEFINED_CHARACTER cpu_to_le32(0xC0000163) // -EIO +#define STATUS_FLOPPY_VOLUME cpu_to_le32(0xC0000164) // -EIO +#define STATUS_FLOPPY_ID_MARK_NOT_FOUND cpu_to_le32(0xC0000165) // -EIO +#define STATUS_FLOPPY_WRONG_CYLINDER cpu_to_le32(0xC0000166) // -EIO +#define STATUS_FLOPPY_UNKNOWN_ERROR cpu_to_le32(0xC0000167) // -EIO +#define STATUS_FLOPPY_BAD_REGISTERS cpu_to_le32(0xC0000168) // -EIO +#define STATUS_DISK_RECALIBRATE_FAILED cpu_to_le32(0xC0000169) // -EIO +#define STATUS_DISK_OPERATION_FAILED cpu_to_le32(0xC000016A) // -EIO +#define STATUS_DISK_RESET_FAILED cpu_to_le32(0xC000016B) // -EIO +#define STATUS_SHARED_IRQ_BUSY cpu_to_le32(0xC000016C) // -EBUSY +#define STATUS_FT_ORPHANING cpu_to_le32(0xC000016D) // -EIO +#define STATUS_BIOS_FAILED_TO_CONNECT_INTERRUPT cpu_to_le32(0xC000016E) // -EIO +#define STATUS_PARTITION_FAILURE cpu_to_le32(0xC0000172) // -EIO +#define STATUS_INVALID_BLOCK_LENGTH cpu_to_le32(0xC0000173) // -EIO +#define STATUS_DEVICE_NOT_PARTITIONED cpu_to_le32(0xC0000174) // -EIO +#define STATUS_UNABLE_TO_LOCK_MEDIA cpu_to_le32(0xC0000175) // -EIO +#define STATUS_UNABLE_TO_UNLOAD_MEDIA cpu_to_le32(0xC0000176) // -EIO +#define STATUS_EOM_OVERFLOW cpu_to_le32(0xC0000177) // -EIO +#define STATUS_NO_MEDIA cpu_to_le32(0xC0000178) // -EIO +#define STATUS_NO_SUCH_MEMBER cpu_to_le32(0xC000017A) // -EIO +#define STATUS_INVALID_MEMBER cpu_to_le32(0xC000017B) // -EIO +#define STATUS_KEY_DELETED cpu_to_le32(0xC000017C) // -EIO +#define STATUS_NO_LOG_SPACE cpu_to_le32(0xC000017D) // -EIO +#define STATUS_TOO_MANY_SIDS cpu_to_le32(0xC000017E) // -EIO +#define STATUS_LM_CROSS_ENCRYPTION_REQUIRED cpu_to_le32(0xC000017F) // -EIO +#define STATUS_KEY_HAS_CHILDREN cpu_to_le32(0xC0000180) // -EIO +#define STATUS_CHILD_MUST_BE_VOLATILE cpu_to_le32(0xC0000181) // -EIO +#define STATUS_DEVICE_CONFIGURATION_ERROR cpu_to_le32(0xC0000182) // -EIO +#define STATUS_DRIVER_INTERNAL_ERROR cpu_to_le32(0xC0000183) // -EIO +#define STATUS_INVALID_DEVICE_STATE cpu_to_le32(0xC0000184) // -EIO +#define STATUS_IO_DEVICE_ERROR cpu_to_le32(0xC0000185) // -EIO +#define STATUS_DEVICE_PROTOCOL_ERROR cpu_to_le32(0xC0000186) // -EIO +#define STATUS_BACKUP_CONTROLLER cpu_to_le32(0xC0000187) // -EIO +#define STATUS_LOG_FILE_FULL cpu_to_le32(0xC0000188) // -EIO +#define STATUS_TOO_LATE cpu_to_le32(0xC0000189) // -EIO +#define STATUS_NO_TRUST_LSA_SECRET cpu_to_le32(0xC000018A) // -EIO +#define STATUS_NO_TRUST_SAM_ACCOUNT cpu_to_le32(0xC000018B) // -EIO +#define STATUS_TRUSTED_DOMAIN_FAILURE cpu_to_le32(0xC000018C) // -EIO +#define STATUS_TRUSTED_RELATIONSHIP_FAILURE cpu_to_le32(0xC000018D) // -EIO +#define STATUS_EVENTLOG_FILE_CORRUPT cpu_to_le32(0xC000018E) // -EIO +#define STATUS_EVENTLOG_CANT_START cpu_to_le32(0xC000018F) // -EIO +#define STATUS_TRUST_FAILURE cpu_to_le32(0xC0000190) // -EIO +#define STATUS_MUTANT_LIMIT_EXCEEDED cpu_to_le32(0xC0000191) // -EIO +#define STATUS_NETLOGON_NOT_STARTED cpu_to_le32(0xC0000192) // -EIO +#define STATUS_ACCOUNT_EXPIRED cpu_to_le32(0xC0000193) // -EKEYEXPIRED +#define STATUS_POSSIBLE_DEADLOCK cpu_to_le32(0xC0000194) // -EIO +#define STATUS_NETWORK_CREDENTIAL_CONFLICT cpu_to_le32(0xC0000195) // -EIO +#define STATUS_REMOTE_SESSION_LIMIT cpu_to_le32(0xC0000196) // -EIO +#define STATUS_EVENTLOG_FILE_CHANGED cpu_to_le32(0xC0000197) // -EIO +#define STATUS_NOLOGON_INTERDOMAIN_TRUST_ACCOUNT cpu_to_le32(0xC0000198) // -EIO +#define STATUS_NOLOGON_WORKSTATION_TRUST_ACCOUNT cpu_to_le32(0xC0000199) // -EIO +#define STATUS_NOLOGON_SERVER_TRUST_ACCOUNT cpu_to_le32(0xC000019A) // -EIO +#define STATUS_DOMAIN_TRUST_INCONSISTENT cpu_to_le32(0xC000019B) // -EIO +#define STATUS_FS_DRIVER_REQUIRED cpu_to_le32(0xC000019C) // -EOPNOTSUPP +#define STATUS_IMAGE_ALREADY_LOADED_AS_DLL cpu_to_le32(0xC000019D) // -EIO +#define STATUS_INVALID_LOCK_RANGE cpu_to_le32(0xC00001A1) // -EIO +#define STATUS_NETWORK_OPEN_RESTRICTION cpu_to_le32(0xC0000201) // -EIO +#define STATUS_NO_USER_SESSION_KEY cpu_to_le32(0xC0000202) // -EIO +#define STATUS_USER_SESSION_DELETED cpu_to_le32(0xC0000203) // -EIO +#define STATUS_RESOURCE_LANG_NOT_FOUND cpu_to_le32(0xC0000204) // -EIO +#define STATUS_INSUFF_SERVER_RESOURCES cpu_to_le32(0xC0000205) // -EIO +#define STATUS_INVALID_BUFFER_SIZE cpu_to_le32(0xC0000206) // -EIO +#define STATUS_INVALID_ADDRESS_COMPONENT cpu_to_le32(0xC0000207) // -EIO +#define STATUS_INVALID_ADDRESS_WILDCARD cpu_to_le32(0xC0000208) // -EIO +#define STATUS_TOO_MANY_ADDRESSES cpu_to_le32(0xC0000209) // -EIO +#define STATUS_ADDRESS_ALREADY_EXISTS cpu_to_le32(0xC000020A) // -EADDRINUSE +#define STATUS_ADDRESS_CLOSED cpu_to_le32(0xC000020B) // -EIO +#define STATUS_CONNECTION_DISCONNECTED cpu_to_le32(0xC000020C) // -ECONNABORTED +#define STATUS_CONNECTION_RESET cpu_to_le32(0xC000020D) // -ENETRESET +#define STATUS_TOO_MANY_NODES cpu_to_le32(0xC000020E) // -EIO +#define STATUS_TRANSACTION_ABORTED cpu_to_le32(0xC000020F) // -EIO +#define STATUS_TRANSACTION_TIMED_OUT cpu_to_le32(0xC0000210) // -EIO +#define STATUS_TRANSACTION_NO_RELEASE cpu_to_le32(0xC0000211) // -EIO +#define STATUS_TRANSACTION_NO_MATCH cpu_to_le32(0xC0000212) // -EIO +#define STATUS_TRANSACTION_RESPONDED cpu_to_le32(0xC0000213) // -EIO +#define STATUS_TRANSACTION_INVALID_ID cpu_to_le32(0xC0000214) // -EIO +#define STATUS_TRANSACTION_INVALID_TYPE cpu_to_le32(0xC0000215) // -EIO +#define STATUS_NOT_SERVER_SESSION cpu_to_le32(0xC0000216) // -EIO +#define STATUS_NOT_CLIENT_SESSION cpu_to_le32(0xC0000217) // -EIO +#define STATUS_CANNOT_LOAD_REGISTRY_FILE cpu_to_le32(0xC0000218) // -EIO +#define STATUS_DEBUG_ATTACH_FAILED cpu_to_le32(0xC0000219) // -EIO +#define STATUS_SYSTEM_PROCESS_TERMINATED cpu_to_le32(0xC000021A) // -EIO +#define STATUS_DATA_NOT_ACCEPTED cpu_to_le32(0xC000021B) // -EIO +#define STATUS_NO_BROWSER_SERVERS_FOUND cpu_to_le32(0xC000021C) // -EIO +#define STATUS_VDM_HARD_ERROR cpu_to_le32(0xC000021D) // -EIO +#define STATUS_DRIVER_CANCEL_TIMEOUT cpu_to_le32(0xC000021E) // -EIO +#define STATUS_REPLY_MESSAGE_MISMATCH cpu_to_le32(0xC000021F) // -EIO +#define STATUS_MAPPED_ALIGNMENT cpu_to_le32(0xC0000220) // -EIO +#define STATUS_IMAGE_CHECKSUM_MISMATCH cpu_to_le32(0xC0000221) // -EIO +#define STATUS_LOST_WRITEBEHIND_DATA cpu_to_le32(0xC0000222) // -EIO +#define STATUS_CLIENT_SERVER_PARAMETERS_INVALID cpu_to_le32(0xC0000223) // -EIO +#define STATUS_PASSWORD_MUST_CHANGE cpu_to_le32(0xC0000224) // -EIO +#define STATUS_NOT_FOUND cpu_to_le32(0xC0000225) // -ENOENT +#define STATUS_NOT_TINY_STREAM cpu_to_le32(0xC0000226) // -EIO +#define STATUS_RECOVERY_FAILURE cpu_to_le32(0xC0000227) // -EIO +#define STATUS_STACK_OVERFLOW_READ cpu_to_le32(0xC0000228) // -EIO +#define STATUS_FAIL_CHECK cpu_to_le32(0xC0000229) // -EIO +#define STATUS_DUPLICATE_OBJECTID cpu_to_le32(0xC000022A) // -EIO +#define STATUS_OBJECTID_EXISTS cpu_to_le32(0xC000022B) // -EIO +#define STATUS_CONVERT_TO_LARGE cpu_to_le32(0xC000022C) // -EIO +#define STATUS_RETRY cpu_to_le32(0xC000022D) // -EAGAIN +#define STATUS_FOUND_OUT_OF_SCOPE cpu_to_le32(0xC000022E) // -EIO +#define STATUS_ALLOCATE_BUCKET cpu_to_le32(0xC000022F) // -EIO +#define STATUS_PROPSET_NOT_FOUND cpu_to_le32(0xC0000230) // -EIO +#define STATUS_MARSHALL_OVERFLOW cpu_to_le32(0xC0000231) // -EIO +#define STATUS_INVALID_VARIANT cpu_to_le32(0xC0000232) // -EIO +#define STATUS_DOMAIN_CONTROLLER_NOT_FOUND cpu_to_le32(0xC0000233) // -EIO +#define STATUS_ACCOUNT_LOCKED_OUT cpu_to_le32(0xC0000234) // -EACCES +#define STATUS_HANDLE_NOT_CLOSABLE cpu_to_le32(0xC0000235) // -EIO +#define STATUS_CONNECTION_REFUSED cpu_to_le32(0xC0000236) // -EIO +#define STATUS_GRACEFUL_DISCONNECT cpu_to_le32(0xC0000237) // -EIO +#define STATUS_ADDRESS_ALREADY_ASSOCIATED cpu_to_le32(0xC0000238) // -EIO +#define STATUS_ADDRESS_NOT_ASSOCIATED cpu_to_le32(0xC0000239) // -EIO +#define STATUS_CONNECTION_INVALID cpu_to_le32(0xC000023A) // -EIO +#define STATUS_CONNECTION_ACTIVE cpu_to_le32(0xC000023B) // -EIO +#define STATUS_NETWORK_UNREACHABLE cpu_to_le32(0xC000023C) // -ENETUNREACH +#define STATUS_HOST_UNREACHABLE cpu_to_le32(0xC000023D) // -EHOSTDOWN +#define STATUS_PROTOCOL_UNREACHABLE cpu_to_le32(0xC000023E) // -ENETUNREACH +#define STATUS_PORT_UNREACHABLE cpu_to_le32(0xC000023F) // -ENETUNREACH +#define STATUS_REQUEST_ABORTED cpu_to_le32(0xC0000240) // -EIO +#define STATUS_CONNECTION_ABORTED cpu_to_le32(0xC0000241) // -ECONNABORTED +#define STATUS_BAD_COMPRESSION_BUFFER cpu_to_le32(0xC0000242) // -EIO +#define STATUS_USER_MAPPED_FILE cpu_to_le32(0xC0000243) // -EIO +#define STATUS_AUDIT_FAILED cpu_to_le32(0xC0000244) // -EIO +#define STATUS_TIMER_RESOLUTION_NOT_SET cpu_to_le32(0xC0000245) // -EIO +#define STATUS_CONNECTION_COUNT_LIMIT cpu_to_le32(0xC0000246) // -EIO +#define STATUS_LOGIN_TIME_RESTRICTION cpu_to_le32(0xC0000247) // -EACCES +#define STATUS_LOGIN_WKSTA_RESTRICTION cpu_to_le32(0xC0000248) // -EACCES +#define STATUS_IMAGE_MP_UP_MISMATCH cpu_to_le32(0xC0000249) // -EIO +#define STATUS_INSUFFICIENT_LOGON_INFO cpu_to_le32(0xC0000250) // -EIO +#define STATUS_BAD_DLL_ENTRYPOINT cpu_to_le32(0xC0000251) // -EIO +#define STATUS_BAD_SERVICE_ENTRYPOINT cpu_to_le32(0xC0000252) // -EIO +#define STATUS_LPC_REPLY_LOST cpu_to_le32(0xC0000253) // -EIO +#define STATUS_IP_ADDRESS_CONFLICT1 cpu_to_le32(0xC0000254) // -EIO +#define STATUS_IP_ADDRESS_CONFLICT2 cpu_to_le32(0xC0000255) // -EIO +#define STATUS_REGISTRY_QUOTA_LIMIT cpu_to_le32(0xC0000256) // -EDQUOT +#define STATUS_PATH_NOT_COVERED cpu_to_le32(0xC0000257) // -EREMOTE +#define STATUS_NO_CALLBACK_ACTIVE cpu_to_le32(0xC0000258) // -EIO +#define STATUS_LICENSE_QUOTA_EXCEEDED cpu_to_le32(0xC0000259) // -EACCES +#define STATUS_PWD_TOO_SHORT cpu_to_le32(0xC000025A) // -EIO +#define STATUS_PWD_TOO_RECENT cpu_to_le32(0xC000025B) // -EIO +#define STATUS_PWD_HISTORY_CONFLICT cpu_to_le32(0xC000025C) // -EIO +#define STATUS_PLUGPLAY_NO_DEVICE cpu_to_le32(0xC000025E) // -EIO +#define STATUS_UNSUPPORTED_COMPRESSION cpu_to_le32(0xC000025F) // -EIO +#define STATUS_INVALID_HW_PROFILE cpu_to_le32(0xC0000260) // -EIO +#define STATUS_INVALID_PLUGPLAY_DEVICE_PATH cpu_to_le32(0xC0000261) // -EIO +#define STATUS_DRIVER_ORDINAL_NOT_FOUND cpu_to_le32(0xC0000262) // -EIO +#define STATUS_DRIVER_ENTRYPOINT_NOT_FOUND cpu_to_le32(0xC0000263) // -EIO +#define STATUS_RESOURCE_NOT_OWNED cpu_to_le32(0xC0000264) // -EIO +#define STATUS_TOO_MANY_LINKS cpu_to_le32(0xC0000265) // -EMLINK +#define STATUS_QUOTA_LIST_INCONSISTENT cpu_to_le32(0xC0000266) // -EIO +#define STATUS_FILE_IS_OFFLINE cpu_to_le32(0xC0000267) // -EIO +#define STATUS_EVALUATION_EXPIRATION cpu_to_le32(0xC0000268) // -EIO +#define STATUS_ILLEGAL_DLL_RELOCATION cpu_to_le32(0xC0000269) // -EIO +#define STATUS_LICENSE_VIOLATION cpu_to_le32(0xC000026A) // -EIO +#define STATUS_DLL_INIT_FAILED_LOGOFF cpu_to_le32(0xC000026B) // -EIO +#define STATUS_DRIVER_UNABLE_TO_LOAD cpu_to_le32(0xC000026C) // -EIO +#define STATUS_DFS_UNAVAILABLE cpu_to_le32(0xC000026D) // -EIO +#define STATUS_VOLUME_DISMOUNTED cpu_to_le32(0xC000026E) // -EIO +#define STATUS_WX86_INTERNAL_ERROR cpu_to_le32(0xC000026F) // -EIO +#define STATUS_WX86_FLOAT_STACK_CHECK cpu_to_le32(0xC0000270) // -EIO +#define STATUS_VALIDATE_CONTINUE cpu_to_le32(0xC0000271) // -EIO +#define STATUS_NO_MATCH cpu_to_le32(0xC0000272) // -EIO +#define STATUS_NO_MORE_MATCHES cpu_to_le32(0xC0000273) // -EIO +#define STATUS_NOT_A_REPARSE_POINT cpu_to_le32(0xC0000275) // -ENODATA +#define STATUS_IO_REPARSE_TAG_INVALID cpu_to_le32(0xC0000276) // -EIO +#define STATUS_IO_REPARSE_TAG_MISMATCH cpu_to_le32(0xC0000277) // -EIO +#define STATUS_IO_REPARSE_DATA_INVALID cpu_to_le32(0xC0000278) // -EIO +#define STATUS_IO_REPARSE_TAG_NOT_HANDLED cpu_to_le32(0xC0000279) // -EOPNOTSUPP +#define STATUS_REPARSE_POINT_NOT_RESOLVED cpu_to_le32(0xC0000280) // -EIO +#define STATUS_DIRECTORY_IS_A_REPARSE_POINT cpu_to_le32(0xC0000281) // -EIO +#define STATUS_RANGE_LIST_CONFLICT cpu_to_le32(0xC0000282) // -EIO +#define STATUS_SOURCE_ELEMENT_EMPTY cpu_to_le32(0xC0000283) // -EIO +#define STATUS_DESTINATION_ELEMENT_FULL cpu_to_le32(0xC0000284) // -EIO +#define STATUS_ILLEGAL_ELEMENT_ADDRESS cpu_to_le32(0xC0000285) // -EIO +#define STATUS_MAGAZINE_NOT_PRESENT cpu_to_le32(0xC0000286) // -EIO +#define STATUS_REINITIALIZATION_NEEDED cpu_to_le32(0xC0000287) // -EIO +#define STATUS_ENCRYPTION_FAILED cpu_to_le32(0xC000028A) // -EIO +#define STATUS_DECRYPTION_FAILED cpu_to_le32(0xC000028B) // -EIO +#define STATUS_RANGE_NOT_FOUND cpu_to_le32(0xC000028C) // -EIO +#define STATUS_NO_RECOVERY_POLICY cpu_to_le32(0xC000028D) // -EIO +#define STATUS_NO_EFS cpu_to_le32(0xC000028E) // -EIO +#define STATUS_WRONG_EFS cpu_to_le32(0xC000028F) // -EIO +#define STATUS_NO_USER_KEYS cpu_to_le32(0xC0000290) // -EIO +#define STATUS_FILE_NOT_ENCRYPTED cpu_to_le32(0xC0000291) // -EIO +#define STATUS_NOT_EXPORT_FORMAT cpu_to_le32(0xC0000292) // -EIO +#define STATUS_FILE_ENCRYPTED cpu_to_le32(0xC0000293) // -EIO +#define STATUS_WMI_GUID_NOT_FOUND cpu_to_le32(0xC0000295) // -EIO +#define STATUS_WMI_INSTANCE_NOT_FOUND cpu_to_le32(0xC0000296) // -EIO +#define STATUS_WMI_ITEMID_NOT_FOUND cpu_to_le32(0xC0000297) // -EIO +#define STATUS_WMI_TRY_AGAIN cpu_to_le32(0xC0000298) // -EIO +#define STATUS_SHARED_POLICY cpu_to_le32(0xC0000299) // -EIO +#define STATUS_POLICY_OBJECT_NOT_FOUND cpu_to_le32(0xC000029A) // -EIO +#define STATUS_POLICY_ONLY_IN_DS cpu_to_le32(0xC000029B) // -EIO +#define STATUS_VOLUME_NOT_UPGRADED cpu_to_le32(0xC000029C) // -EIO +#define STATUS_REMOTE_STORAGE_NOT_ACTIVE cpu_to_le32(0xC000029D) // -EIO +#define STATUS_REMOTE_STORAGE_MEDIA_ERROR cpu_to_le32(0xC000029E) // -EIO +#define STATUS_NO_TRACKING_SERVICE cpu_to_le32(0xC000029F) // -EIO +#define STATUS_SERVER_SID_MISMATCH cpu_to_le32(0xC00002A0) // -EIO +#define STATUS_DS_NO_ATTRIBUTE_OR_VALUE cpu_to_le32(0xC00002A1) // -EIO +#define STATUS_DS_INVALID_ATTRIBUTE_SYNTAX cpu_to_le32(0xC00002A2) // -EIO +#define STATUS_DS_ATTRIBUTE_TYPE_UNDEFINED cpu_to_le32(0xC00002A3) // -EIO +#define STATUS_DS_ATTRIBUTE_OR_VALUE_EXISTS cpu_to_le32(0xC00002A4) // -EIO +#define STATUS_DS_BUSY cpu_to_le32(0xC00002A5) // -EBUSY +#define STATUS_DS_UNAVAILABLE cpu_to_le32(0xC00002A6) // -EIO +#define STATUS_DS_NO_RIDS_ALLOCATED cpu_to_le32(0xC00002A7) // -EIO +#define STATUS_DS_NO_MORE_RIDS cpu_to_le32(0xC00002A8) // -EIO +#define STATUS_DS_INCORRECT_ROLE_OWNER cpu_to_le32(0xC00002A9) // -EIO +#define STATUS_DS_RIDMGR_INIT_ERROR cpu_to_le32(0xC00002AA) // -EIO +#define STATUS_DS_OBJ_CLASS_VIOLATION cpu_to_le32(0xC00002AB) // -EIO +#define STATUS_DS_CANT_ON_NON_LEAF cpu_to_le32(0xC00002AC) // -EIO +#define STATUS_DS_CANT_ON_RDN cpu_to_le32(0xC00002AD) // -EIO +#define STATUS_DS_CANT_MOD_OBJ_CLASS cpu_to_le32(0xC00002AE) // -EIO +#define STATUS_DS_CROSS_DOM_MOVE_FAILED cpu_to_le32(0xC00002AF) // -EIO +#define STATUS_DS_GC_NOT_AVAILABLE cpu_to_le32(0xC00002B0) // -EIO +#define STATUS_DIRECTORY_SERVICE_REQUIRED cpu_to_le32(0xC00002B1) // -EIO +#define STATUS_REPARSE_ATTRIBUTE_CONFLICT cpu_to_le32(0xC00002B2) // -EIO +#define STATUS_CANT_ENABLE_DENY_ONLY cpu_to_le32(0xC00002B3) // -EIO +#define STATUS_FLOAT_MULTIPLE_FAULTS cpu_to_le32(0xC00002B4) // -EIO +#define STATUS_FLOAT_MULTIPLE_TRAPS cpu_to_le32(0xC00002B5) // -EIO +#define STATUS_DEVICE_REMOVED cpu_to_le32(0xC00002B6) // -EIO +#define STATUS_JOURNAL_DELETE_IN_PROGRESS cpu_to_le32(0xC00002B7) // -EIO +#define STATUS_JOURNAL_NOT_ACTIVE cpu_to_le32(0xC00002B8) // -EIO +#define STATUS_NOINTERFACE cpu_to_le32(0xC00002B9) // -EIO +#define STATUS_DS_ADMIN_LIMIT_EXCEEDED cpu_to_le32(0xC00002C1) // -EIO +#define STATUS_DRIVER_FAILED_SLEEP cpu_to_le32(0xC00002C2) // -EIO +#define STATUS_MUTUAL_AUTHENTICATION_FAILED cpu_to_le32(0xC00002C3) // -EIO +#define STATUS_CORRUPT_SYSTEM_FILE cpu_to_le32(0xC00002C4) // -EIO +#define STATUS_DATATYPE_MISALIGNMENT_ERROR cpu_to_le32(0xC00002C5) // -EIO +#define STATUS_WMI_READ_ONLY cpu_to_le32(0xC00002C6) // -EROFS +#define STATUS_WMI_SET_FAILURE cpu_to_le32(0xC00002C7) // -EIO +#define STATUS_COMMITMENT_MINIMUM cpu_to_le32(0xC00002C8) // -EIO +#define STATUS_REG_NAT_CONSUMPTION cpu_to_le32(0xC00002C9) // -EIO +#define STATUS_TRANSPORT_FULL cpu_to_le32(0xC00002CA) // -EIO +#define STATUS_DS_SAM_INIT_FAILURE cpu_to_le32(0xC00002CB) // -EIO +#define STATUS_ONLY_IF_CONNECTED cpu_to_le32(0xC00002CC) // -EIO +#define STATUS_DS_SENSITIVE_GROUP_VIOLATION cpu_to_le32(0xC00002CD) // -EIO +#define STATUS_PNP_RESTART_ENUMERATION cpu_to_le32(0xC00002CE) // -EIO +#define STATUS_JOURNAL_ENTRY_DELETED cpu_to_le32(0xC00002CF) // -EIO +#define STATUS_DS_CANT_MOD_PRIMARYGROUPID cpu_to_le32(0xC00002D0) // -EIO +#define STATUS_SYSTEM_IMAGE_BAD_SIGNATURE cpu_to_le32(0xC00002D1) // -EIO +#define STATUS_PNP_REBOOT_REQUIRED cpu_to_le32(0xC00002D2) // -EIO +#define STATUS_POWER_STATE_INVALID cpu_to_le32(0xC00002D3) // -EIO +#define STATUS_DS_INVALID_GROUP_TYPE cpu_to_le32(0xC00002D4) // -EIO +#define STATUS_DS_NO_NEST_GLOBALGROUP_IN_MIXEDDOMAIN cpu_to_le32(0xC00002D5) // -EIO +#define STATUS_DS_NO_NEST_LOCALGROUP_IN_MIXEDDOMAIN cpu_to_le32(0xC00002D6) // -EIO +#define STATUS_DS_GLOBAL_CANT_HAVE_LOCAL_MEMBER cpu_to_le32(0xC00002D7) // -EIO +#define STATUS_DS_GLOBAL_CANT_HAVE_UNIVERSAL_MEMBER cpu_to_le32(0xC00002D8) // -EIO +#define STATUS_DS_UNIVERSAL_CANT_HAVE_LOCAL_MEMBER cpu_to_le32(0xC00002D9) // -EIO +#define STATUS_DS_GLOBAL_CANT_HAVE_CROSSDOMAIN_MEMBER cpu_to_le32(0xC00002DA) // -EIO +#define STATUS_DS_LOCAL_CANT_HAVE_CROSSDOMAIN_LOCAL_MEMBER cpu_to_le32(0xC00002DB) // -EIO +#define STATUS_DS_HAVE_PRIMARY_MEMBERS cpu_to_le32(0xC00002DC) // -EIO +#define STATUS_WMI_NOT_SUPPORTED cpu_to_le32(0xC00002DD) // -EOPNOTSUPP +#define STATUS_INSUFFICIENT_POWER cpu_to_le32(0xC00002DE) // -EIO +#define STATUS_SAM_NEED_BOOTKEY_PASSWORD cpu_to_le32(0xC00002DF) // -EIO +#define STATUS_SAM_NEED_BOOTKEY_FLOPPY cpu_to_le32(0xC00002E0) // -EIO +#define STATUS_DS_CANT_START cpu_to_le32(0xC00002E1) // -EIO +#define STATUS_DS_INIT_FAILURE cpu_to_le32(0xC00002E2) // -EIO +#define STATUS_SAM_INIT_FAILURE cpu_to_le32(0xC00002E3) // -EIO +#define STATUS_DS_GC_REQUIRED cpu_to_le32(0xC00002E4) // -EIO +#define STATUS_DS_LOCAL_MEMBER_OF_LOCAL_ONLY cpu_to_le32(0xC00002E5) // -EIO +#define STATUS_DS_NO_FPO_IN_UNIVERSAL_GROUPS cpu_to_le32(0xC00002E6) // -EIO +#define STATUS_DS_MACHINE_ACCOUNT_QUOTA_EXCEEDED cpu_to_le32(0xC00002E7) // -EDQUOT +#define STATUS_MULTIPLE_FAULT_VIOLATION cpu_to_le32(0xC00002E8) // -EIO +#define STATUS_CURRENT_DOMAIN_NOT_ALLOWED cpu_to_le32(0xC00002E9) // -EIO +#define STATUS_CANNOT_MAKE cpu_to_le32(0xC00002EA) // -EIO +#define STATUS_SYSTEM_SHUTDOWN cpu_to_le32(0xC00002EB) // -EIO +#define STATUS_DS_INIT_FAILURE_CONSOLE cpu_to_le32(0xC00002EC) // -EIO +#define STATUS_DS_SAM_INIT_FAILURE_CONSOLE cpu_to_le32(0xC00002ED) // -EIO +#define STATUS_UNFINISHED_CONTEXT_DELETED cpu_to_le32(0xC00002EE) // -EIO +#define STATUS_NO_TGT_REPLY cpu_to_le32(0xC00002EF) // -EIO +#define STATUS_OBJECTID_NOT_FOUND cpu_to_le32(0xC00002F0) // -ENODATA +#define STATUS_NO_IP_ADDRESSES cpu_to_le32(0xC00002F1) // -EIO +#define STATUS_WRONG_CREDENTIAL_HANDLE cpu_to_le32(0xC00002F2) // -EIO +#define STATUS_CRYPTO_SYSTEM_INVALID cpu_to_le32(0xC00002F3) // -EIO +#define STATUS_MAX_REFERRALS_EXCEEDED cpu_to_le32(0xC00002F4) // -EIO +#define STATUS_MUST_BE_KDC cpu_to_le32(0xC00002F5) // -EIO +#define STATUS_STRONG_CRYPTO_NOT_SUPPORTED cpu_to_le32(0xC00002F6) // -EIO +#define STATUS_TOO_MANY_PRINCIPALS cpu_to_le32(0xC00002F7) // -EIO +#define STATUS_NO_PA_DATA cpu_to_le32(0xC00002F8) // -EIO +#define STATUS_PKINIT_NAME_MISMATCH cpu_to_le32(0xC00002F9) // -EIO +#define STATUS_SMARTCARD_LOGON_REQUIRED cpu_to_le32(0xC00002FA) // -EIO +#define STATUS_KDC_INVALID_REQUEST cpu_to_le32(0xC00002FB) // -EIO +#define STATUS_KDC_UNABLE_TO_REFER cpu_to_le32(0xC00002FC) // -EIO +#define STATUS_KDC_UNKNOWN_ETYPE cpu_to_le32(0xC00002FD) // -EIO +#define STATUS_SHUTDOWN_IN_PROGRESS cpu_to_le32(0xC00002FE) // -EIO +#define STATUS_SERVER_SHUTDOWN_IN_PROGRESS cpu_to_le32(0xC00002FF) // -EIO +#define STATUS_NOT_SUPPORTED_ON_SBS cpu_to_le32(0xC0000300) // -EOPNOTSUPP +#define STATUS_WMI_GUID_DISCONNECTED cpu_to_le32(0xC0000301) // -EIO +#define STATUS_WMI_ALREADY_DISABLED cpu_to_le32(0xC0000302) // -EIO +#define STATUS_WMI_ALREADY_ENABLED cpu_to_le32(0xC0000303) // -EIO +#define STATUS_MFT_TOO_FRAGMENTED cpu_to_le32(0xC0000304) // -EIO +#define STATUS_COPY_PROTECTION_FAILURE cpu_to_le32(0xC0000305) // -EIO +#define STATUS_CSS_AUTHENTICATION_FAILURE cpu_to_le32(0xC0000306) // -EIO +#define STATUS_CSS_KEY_NOT_PRESENT cpu_to_le32(0xC0000307) // -EIO +#define STATUS_CSS_KEY_NOT_ESTABLISHED cpu_to_le32(0xC0000308) // -EIO +#define STATUS_CSS_SCRAMBLED_SECTOR cpu_to_le32(0xC0000309) // -EIO +#define STATUS_CSS_REGION_MISMATCH cpu_to_le32(0xC000030A) // -EIO +#define STATUS_CSS_RESETS_EXHAUSTED cpu_to_le32(0xC000030B) // -EIO +#define STATUS_PKINIT_FAILURE cpu_to_le32(0xC0000320) // -EIO +#define STATUS_SMARTCARD_SUBSYSTEM_FAILURE cpu_to_le32(0xC0000321) // -EIO +#define STATUS_NO_KERB_KEY cpu_to_le32(0xC0000322) // -EIO +#define STATUS_HOST_DOWN cpu_to_le32(0xC0000350) // -EIO +#define STATUS_UNSUPPORTED_PREAUTH cpu_to_le32(0xC0000351) // -EIO +#define STATUS_EFS_ALG_BLOB_TOO_BIG cpu_to_le32(0xC0000352) // -EIO +#define STATUS_PORT_NOT_SET cpu_to_le32(0xC0000353) // -EIO +#define STATUS_DEBUGGER_INACTIVE cpu_to_le32(0xC0000354) // -EIO +#define STATUS_DS_VERSION_CHECK_FAILURE cpu_to_le32(0xC0000355) // -EIO +#define STATUS_AUDITING_DISABLED cpu_to_le32(0xC0000356) // -EIO +#define STATUS_PRENT4_MACHINE_ACCOUNT cpu_to_le32(0xC0000357) // -EIO +#define STATUS_DS_AG_CANT_HAVE_UNIVERSAL_MEMBER cpu_to_le32(0xC0000358) // -EIO +#define STATUS_INVALID_IMAGE_WIN_32 cpu_to_le32(0xC0000359) // -EIO +#define STATUS_INVALID_IMAGE_WIN_64 cpu_to_le32(0xC000035A) // -EIO +#define STATUS_BAD_BINDINGS cpu_to_le32(0xC000035B) // -EIO +#define STATUS_NETWORK_SESSION_EXPIRED cpu_to_le32(0xC000035C) // -EIO +#define STATUS_APPHELP_BLOCK cpu_to_le32(0xC000035D) // -EIO +#define STATUS_ALL_SIDS_FILTERED cpu_to_le32(0xC000035E) // -EIO +#define STATUS_NOT_SAFE_MODE_DRIVER cpu_to_le32(0xC000035F) // -EIO +#define STATUS_ACCESS_DISABLED_BY_POLICY_DEFAULT cpu_to_le32(0xC0000361) // -EACCES +#define STATUS_ACCESS_DISABLED_BY_POLICY_PATH cpu_to_le32(0xC0000362) // -EACCES +#define STATUS_ACCESS_DISABLED_BY_POLICY_PUBLISHER cpu_to_le32(0xC0000363) // -EACCES +#define STATUS_ACCESS_DISABLED_BY_POLICY_OTHER cpu_to_le32(0xC0000364) // -EACCES +#define STATUS_FAILED_DRIVER_ENTRY cpu_to_le32(0xC0000365) // -EIO +#define STATUS_DEVICE_ENUMERATION_ERROR cpu_to_le32(0xC0000366) // -EIO +#define STATUS_MOUNT_POINT_NOT_RESOLVED cpu_to_le32(0xC0000368) // -EIO +#define STATUS_INVALID_DEVICE_OBJECT_PARAMETER cpu_to_le32(0xC0000369) // -EIO +/* + * 'OCCURED' is typo in MS-ERREF, it should be 'OCCURRED', + * but we'll keep it consistent with MS-ERREF. + */ +#define STATUS_MCA_OCCURED cpu_to_le32(0xC000036A) // -EIO +#define STATUS_DRIVER_BLOCKED_CRITICAL cpu_to_le32(0xC000036B) // -EIO +#define STATUS_DRIVER_BLOCKED cpu_to_le32(0xC000036C) // -EIO +#define STATUS_DRIVER_DATABASE_ERROR cpu_to_le32(0xC000036D) // -EIO +#define STATUS_SYSTEM_HIVE_TOO_LARGE cpu_to_le32(0xC000036E) // -EIO +#define STATUS_INVALID_IMPORT_OF_NON_DLL cpu_to_le32(0xC000036F) // -EIO +#define STATUS_NO_SECRETS cpu_to_le32(0xC0000371) // -EIO +#define STATUS_ACCESS_DISABLED_NO_SAFER_UI_BY_POLICY cpu_to_le32(0xC0000372) // -EACCES +#define STATUS_FAILED_STACK_SWITCH cpu_to_le32(0xC0000373) // -EIO +#define STATUS_HEAP_CORRUPTION cpu_to_le32(0xC0000374) // -EIO +#define STATUS_SMARTCARD_WRONG_PIN cpu_to_le32(0xC0000380) // -EIO +#define STATUS_SMARTCARD_CARD_BLOCKED cpu_to_le32(0xC0000381) // -EIO +#define STATUS_SMARTCARD_CARD_NOT_AUTHENTICATED cpu_to_le32(0xC0000382) // -EIO +#define STATUS_SMARTCARD_NO_CARD cpu_to_le32(0xC0000383) // -EIO +#define STATUS_SMARTCARD_NO_KEY_CONTAINER cpu_to_le32(0xC0000384) // -EIO +#define STATUS_SMARTCARD_NO_CERTIFICATE cpu_to_le32(0xC0000385) // -EIO +#define STATUS_SMARTCARD_NO_KEYSET cpu_to_le32(0xC0000386) // -EIO +#define STATUS_SMARTCARD_IO_ERROR cpu_to_le32(0xC0000387) // -EIO +#define STATUS_DOWNGRADE_DETECTED cpu_to_le32(0xC0000388) // -EIO +#define STATUS_SMARTCARD_CERT_REVOKED cpu_to_le32(0xC0000389) // -EIO +#define STATUS_ISSUING_CA_UNTRUSTED cpu_to_le32(0xC000038A) // -EIO +#define STATUS_REVOCATION_OFFLINE_C cpu_to_le32(0xC000038B) // -EIO +#define STATUS_PKINIT_CLIENT_FAILURE cpu_to_le32(0xC000038C) // -EIO +#define STATUS_SMARTCARD_CERT_EXPIRED cpu_to_le32(0xC000038D) // -EIO +#define STATUS_DRIVER_FAILED_PRIOR_UNLOAD cpu_to_le32(0xC000038E) // -EIO +#define STATUS_SMARTCARD_SILENT_CONTEXT cpu_to_le32(0xC000038F) // -EIO +#define STATUS_PER_USER_TRUST_QUOTA_EXCEEDED cpu_to_le32(0xC0000401) // -EDQUOT +#define STATUS_ALL_USER_TRUST_QUOTA_EXCEEDED cpu_to_le32(0xC0000402) // -EDQUOT +#define STATUS_USER_DELETE_TRUST_QUOTA_EXCEEDED cpu_to_le32(0xC0000403) // -EDQUOT +#define STATUS_DS_NAME_NOT_UNIQUE cpu_to_le32(0xC0000404) // -EIO +#define STATUS_DS_DUPLICATE_ID_FOUND cpu_to_le32(0xC0000405) // -EIO +#define STATUS_DS_GROUP_CONVERSION_ERROR cpu_to_le32(0xC0000406) // -EIO +#define STATUS_VOLSNAP_PREPARE_HIBERNATE cpu_to_le32(0xC0000407) // -EIO +#define STATUS_USER2USER_REQUIRED cpu_to_le32(0xC0000408) // -EIO +#define STATUS_STACK_BUFFER_OVERRUN cpu_to_le32(0xC0000409) // -EIO +#define STATUS_NO_S4U_PROT_SUPPORT cpu_to_le32(0xC000040A) // -EIO +#define STATUS_CROSSREALM_DELEGATION_FAILURE cpu_to_le32(0xC000040B) // -EIO +#define STATUS_REVOCATION_OFFLINE_KDC cpu_to_le32(0xC000040C) // -EIO +#define STATUS_ISSUING_CA_UNTRUSTED_KDC cpu_to_le32(0xC000040D) // -EIO +#define STATUS_KDC_CERT_EXPIRED cpu_to_le32(0xC000040E) // -EIO +#define STATUS_KDC_CERT_REVOKED cpu_to_le32(0xC000040F) // -EIO +#define STATUS_PARAMETER_QUOTA_EXCEEDED cpu_to_le32(0xC0000410) // -EDQUOT +#define STATUS_HIBERNATION_FAILURE cpu_to_le32(0xC0000411) // -EIO +#define STATUS_DELAY_LOAD_FAILED cpu_to_le32(0xC0000412) // -EIO +#define STATUS_AUTHENTICATION_FIREWALL_FAILED cpu_to_le32(0xC0000413) // -EIO +#define STATUS_VDM_DISALLOWED cpu_to_le32(0xC0000414) // -EIO +#define STATUS_HUNG_DISPLAY_DRIVER_THREAD cpu_to_le32(0xC0000415) // -EIO +#define STATUS_INSUFFICIENT_RESOURCE_FOR_SPECIFIED_SHARED_SECTION_SIZE cpu_to_le32(0xC0000416) // -EIO +#define STATUS_INVALID_CRUNTIME_PARAMETER cpu_to_le32(0xC0000417) // -EIO +#define STATUS_NTLM_BLOCKED cpu_to_le32(0xC0000418) // -EIO +#define STATUS_ASSERTION_FAILURE cpu_to_le32(0xC0000420) // -EIO +#define STATUS_VERIFIER_STOP cpu_to_le32(0xC0000421) // -EIO +#define STATUS_CALLBACK_POP_STACK cpu_to_le32(0xC0000423) // -EIO +#define STATUS_INCOMPATIBLE_DRIVER_BLOCKED cpu_to_le32(0xC0000424) // -EIO +#define STATUS_HIVE_UNLOADED cpu_to_le32(0xC0000425) // -EIO +#define STATUS_COMPRESSION_DISABLED cpu_to_le32(0xC0000426) // -EIO +#define STATUS_FILE_SYSTEM_LIMITATION cpu_to_le32(0xC0000427) // -EIO +#define STATUS_INVALID_IMAGE_HASH cpu_to_le32(0xC0000428) // -EIO +#define STATUS_NOT_CAPABLE cpu_to_le32(0xC0000429) // -EIO +#define STATUS_REQUEST_OUT_OF_SEQUENCE cpu_to_le32(0xC000042A) // -EIO +#define STATUS_IMPLEMENTATION_LIMIT cpu_to_le32(0xC000042B) // -EIO +#define STATUS_ELEVATION_REQUIRED cpu_to_le32(0xC000042C) // -EIO +#define STATUS_BEYOND_VDL cpu_to_le32(0xC0000432) // -EIO +#define STATUS_ENCOUNTERED_WRITE_IN_PROGRESS cpu_to_le32(0xC0000433) // -EIO +#define STATUS_PTE_CHANGED cpu_to_le32(0xC0000434) // -EIO +#define STATUS_PURGE_FAILED cpu_to_le32(0xC0000435) // -EIO +#define STATUS_CRED_REQUIRES_CONFIRMATION cpu_to_le32(0xC0000440) // -EIO +#define STATUS_CS_ENCRYPTION_INVALID_SERVER_RESPONSE cpu_to_le32(0xC0000441) // -EIO +#define STATUS_CS_ENCRYPTION_UNSUPPORTED_SERVER cpu_to_le32(0xC0000442) // -EIO +#define STATUS_CS_ENCRYPTION_EXISTING_ENCRYPTED_FILE cpu_to_le32(0xC0000443) // -EIO +#define STATUS_CS_ENCRYPTION_NEW_ENCRYPTED_FILE cpu_to_le32(0xC0000444) // -EIO +#define STATUS_CS_ENCRYPTION_FILE_NOT_CSE cpu_to_le32(0xC0000445) // -EIO +#define STATUS_INVALID_LABEL cpu_to_le32(0xC0000446) // -EIO +#define STATUS_DRIVER_PROCESS_TERMINATED cpu_to_le32(0xC0000450) // -EIO +#define STATUS_AMBIGUOUS_SYSTEM_DEVICE cpu_to_le32(0xC0000451) // -EIO +#define STATUS_SYSTEM_DEVICE_NOT_FOUND cpu_to_le32(0xC0000452) // -EIO +#define STATUS_RESTART_BOOT_APPLICATION cpu_to_le32(0xC0000453) // -EIO +#define STATUS_INVALID_TASK_NAME cpu_to_le32(0xC0000500) // -EIO +#define STATUS_INVALID_TASK_INDEX cpu_to_le32(0xC0000501) // -EIO +#define STATUS_THREAD_ALREADY_IN_TASK cpu_to_le32(0xC0000502) // -EIO +#define STATUS_CALLBACK_BYPASS cpu_to_le32(0xC0000503) // -EIO +#define STATUS_SERVER_UNAVAILABLE cpu_to_le32(0xC0000466) // -EAGAIN +#define STATUS_FILE_NOT_AVAILABLE cpu_to_le32(0xC0000467) // -EAGAIN +#define STATUS_PORT_CLOSED cpu_to_le32(0xC0000700) // -EIO +#define STATUS_MESSAGE_LOST cpu_to_le32(0xC0000701) // -EIO +#define STATUS_INVALID_MESSAGE cpu_to_le32(0xC0000702) // -EIO +#define STATUS_REQUEST_CANCELED cpu_to_le32(0xC0000703) // -EIO +#define STATUS_RECURSIVE_DISPATCH cpu_to_le32(0xC0000704) // -EIO +#define STATUS_LPC_RECEIVE_BUFFER_EXPECTED cpu_to_le32(0xC0000705) // -EIO +#define STATUS_LPC_INVALID_CONNECTION_USAGE cpu_to_le32(0xC0000706) // -EIO +#define STATUS_LPC_REQUESTS_NOT_ALLOWED cpu_to_le32(0xC0000707) // -EIO +#define STATUS_RESOURCE_IN_USE cpu_to_le32(0xC0000708) // -EIO +#define STATUS_HARDWARE_MEMORY_ERROR cpu_to_le32(0xC0000709) // -EIO +#define STATUS_THREADPOOL_HANDLE_EXCEPTION cpu_to_le32(0xC000070A) // -EIO +#define STATUS_THREADPOOL_SET_EVENT_ON_COMPLETION_FAILED cpu_to_le32(0xC000070B) // -EIO +#define STATUS_THREADPOOL_RELEASE_SEMAPHORE_ON_COMPLETION_FAILED cpu_to_le32(0xC000070C) // -EIO +#define STATUS_THREADPOOL_RELEASE_MUTEX_ON_COMPLETION_FAILED cpu_to_le32(0xC000070D) // -EIO +#define STATUS_THREADPOOL_FREE_LIBRARY_ON_COMPLETION_FAILED cpu_to_le32(0xC000070E) // -EIO +#define STATUS_THREADPOOL_RELEASED_DURING_OPERATION cpu_to_le32(0xC000070F) // -EIO +#define STATUS_CALLBACK_RETURNED_WHILE_IMPERSONATING cpu_to_le32(0xC0000710) // -EIO +#define STATUS_APC_RETURNED_WHILE_IMPERSONATING cpu_to_le32(0xC0000711) // -EIO +#define STATUS_PROCESS_IS_PROTECTED cpu_to_le32(0xC0000712) // -EIO +#define STATUS_MCA_EXCEPTION cpu_to_le32(0xC0000713) // -EIO +#define STATUS_CERTIFICATE_MAPPING_NOT_UNIQUE cpu_to_le32(0xC0000714) // -EIO +#define STATUS_SYMLINK_CLASS_DISABLED cpu_to_le32(0xC0000715) // -EIO +#define STATUS_INVALID_IDN_NORMALIZATION cpu_to_le32(0xC0000716) // -EIO +#define STATUS_NO_UNICODE_TRANSLATION cpu_to_le32(0xC0000717) // -EIO +#define STATUS_ALREADY_REGISTERED cpu_to_le32(0xC0000718) // -EIO +#define STATUS_CONTEXT_MISMATCH cpu_to_le32(0xC0000719) // -EIO +#define STATUS_PORT_ALREADY_HAS_COMPLETION_LIST cpu_to_le32(0xC000071A) // -EIO +#define STATUS_CALLBACK_RETURNED_THREAD_PRIORITY cpu_to_le32(0xC000071B) // -EIO +#define STATUS_INVALID_THREAD cpu_to_le32(0xC000071C) // -EIO +#define STATUS_CALLBACK_RETURNED_TRANSACTION cpu_to_le32(0xC000071D) // -EIO +#define STATUS_CALLBACK_RETURNED_LDR_LOCK cpu_to_le32(0xC000071E) // -EIO +#define STATUS_CALLBACK_RETURNED_LANG cpu_to_le32(0xC000071F) // -EIO +#define STATUS_CALLBACK_RETURNED_PRI_BACK cpu_to_le32(0xC0000720) // -EIO +#define STATUS_CALLBACK_RETURNED_THREAD_AFFINITY cpu_to_le32(0xC0000721) // -EIO +#define STATUS_DISK_REPAIR_DISABLED cpu_to_le32(0xC0000800) // -EIO +#define STATUS_DS_DOMAIN_RENAME_IN_PROGRESS cpu_to_le32(0xC0000801) // -EIO +#define STATUS_DISK_QUOTA_EXCEEDED cpu_to_le32(0xC0000802) // -EDQUOT +#define STATUS_CONTENT_BLOCKED cpu_to_le32(0xC0000804) // -EIO +#define STATUS_BAD_CLUSTERS cpu_to_le32(0xC0000805) // -EIO +#define STATUS_VOLUME_DIRTY cpu_to_le32(0xC0000806) // -EIO +#define STATUS_FILE_CHECKED_OUT cpu_to_le32(0xC0000901) // -EIO +#define STATUS_CHECKOUT_REQUIRED cpu_to_le32(0xC0000902) // -EIO +#define STATUS_BAD_FILE_TYPE cpu_to_le32(0xC0000903) // -EIO +#define STATUS_FILE_TOO_LARGE cpu_to_le32(0xC0000904) // -EIO +#define STATUS_FORMS_AUTH_REQUIRED cpu_to_le32(0xC0000905) // -EIO +#define STATUS_VIRUS_INFECTED cpu_to_le32(0xC0000906) // -EIO +#define STATUS_VIRUS_DELETED cpu_to_le32(0xC0000907) // -EIO +#define STATUS_BAD_MCFG_TABLE cpu_to_le32(0xC0000908) // -EIO +#define STATUS_WOW_ASSERTION cpu_to_le32(0xC0009898) // -EIO +#define STATUS_INVALID_SIGNATURE cpu_to_le32(0xC000A000) // -EIO +#define STATUS_HMAC_NOT_SUPPORTED cpu_to_le32(0xC000A001) // -EIO +#define STATUS_AUTH_TAG_MISMATCH cpu_to_le32(0xC000A002) // -EBADMSG +#define STATUS_IPSEC_QUEUE_OVERFLOW cpu_to_le32(0xC000A010) // -EIO +#define STATUS_ND_QUEUE_OVERFLOW cpu_to_le32(0xC000A011) // -EIO +#define STATUS_HOPLIMIT_EXCEEDED cpu_to_le32(0xC000A012) // -EIO +#define STATUS_PROTOCOL_NOT_SUPPORTED cpu_to_le32(0xC000A013) // -EOPNOTSUPP +#define STATUS_LOST_WRITEBEHIND_DATA_NETWORK_DISCONNECTED cpu_to_le32(0xC000A080) // -EIO +#define STATUS_LOST_WRITEBEHIND_DATA_NETWORK_SERVER_ERROR cpu_to_le32(0xC000A081) // -EIO +#define STATUS_LOST_WRITEBEHIND_DATA_LOCAL_DISK_ERROR cpu_to_le32(0xC000A082) // -EIO +#define STATUS_XML_PARSE_ERROR cpu_to_le32(0xC000A083) // -EIO +#define STATUS_XMLDSIG_ERROR cpu_to_le32(0xC000A084) // -EIO +#define STATUS_WRONG_COMPARTMENT cpu_to_le32(0xC000A085) // -EIO +#define STATUS_AUTHIP_FAILURE cpu_to_le32(0xC000A086) // -EIO +#define DBG_NO_STATE_CHANGE cpu_to_le32(0xC0010001) // -EIO +#define DBG_APP_NOT_IDLE cpu_to_le32(0xC0010002) // -EIO +#define RPC_NT_INVALID_STRING_BINDING cpu_to_le32(0xC0020001) // -EIO +#define RPC_NT_WRONG_KIND_OF_BINDING cpu_to_le32(0xC0020002) // -EIO +#define RPC_NT_INVALID_BINDING cpu_to_le32(0xC0020003) // -EIO +#define RPC_NT_PROTSEQ_NOT_SUPPORTED cpu_to_le32(0xC0020004) // -EOPNOTSUPP +#define RPC_NT_INVALID_RPC_PROTSEQ cpu_to_le32(0xC0020005) // -EIO +#define RPC_NT_INVALID_STRING_UUID cpu_to_le32(0xC0020006) // -EIO +#define RPC_NT_INVALID_ENDPOINT_FORMAT cpu_to_le32(0xC0020007) // -EIO +#define RPC_NT_INVALID_NET_ADDR cpu_to_le32(0xC0020008) // -EIO +#define RPC_NT_NO_ENDPOINT_FOUND cpu_to_le32(0xC0020009) // -EIO +#define RPC_NT_INVALID_TIMEOUT cpu_to_le32(0xC002000A) // -EINVAL +#define RPC_NT_OBJECT_NOT_FOUND cpu_to_le32(0xC002000B) // -ENOENT +#define RPC_NT_ALREADY_REGISTERED cpu_to_le32(0xC002000C) // -EIO +#define RPC_NT_TYPE_ALREADY_REGISTERED cpu_to_le32(0xC002000D) // -EIO +#define RPC_NT_ALREADY_LISTENING cpu_to_le32(0xC002000E) // -EIO +#define RPC_NT_NO_PROTSEQS_REGISTERED cpu_to_le32(0xC002000F) // -EIO +#define RPC_NT_NOT_LISTENING cpu_to_le32(0xC0020010) // -EIO +#define RPC_NT_UNKNOWN_MGR_TYPE cpu_to_le32(0xC0020011) // -EIO +#define RPC_NT_UNKNOWN_IF cpu_to_le32(0xC0020012) // -EIO +#define RPC_NT_NO_BINDINGS cpu_to_le32(0xC0020013) // -EIO +#define RPC_NT_NO_PROTSEQS cpu_to_le32(0xC0020014) // -EIO +#define RPC_NT_CANT_CREATE_ENDPOINT cpu_to_le32(0xC0020015) // -EIO +#define RPC_NT_OUT_OF_RESOURCES cpu_to_le32(0xC0020016) // -EIO +#define RPC_NT_SERVER_UNAVAILABLE cpu_to_le32(0xC0020017) // -EIO +#define RPC_NT_SERVER_TOO_BUSY cpu_to_le32(0xC0020018) // -EBUSY +#define RPC_NT_INVALID_NETWORK_OPTIONS cpu_to_le32(0xC0020019) // -EIO +#define RPC_NT_NO_CALL_ACTIVE cpu_to_le32(0xC002001A) // -EIO +#define RPC_NT_CALL_FAILED cpu_to_le32(0xC002001B) // -EIO +#define RPC_NT_CALL_FAILED_DNE cpu_to_le32(0xC002001C) // -EIO +#define RPC_NT_PROTOCOL_ERROR cpu_to_le32(0xC002001D) // -EIO +#define RPC_NT_UNSUPPORTED_TRANS_SYN cpu_to_le32(0xC002001F) // -EIO +#define RPC_NT_UNSUPPORTED_TYPE cpu_to_le32(0xC0020021) // -EIO +#define RPC_NT_INVALID_TAG cpu_to_le32(0xC0020022) // -EIO +#define RPC_NT_INVALID_BOUND cpu_to_le32(0xC0020023) // -EIO +#define RPC_NT_NO_ENTRY_NAME cpu_to_le32(0xC0020024) // -EIO +#define RPC_NT_INVALID_NAME_SYNTAX cpu_to_le32(0xC0020025) // -EIO +#define RPC_NT_UNSUPPORTED_NAME_SYNTAX cpu_to_le32(0xC0020026) // -EIO +#define RPC_NT_UUID_NO_ADDRESS cpu_to_le32(0xC0020028) // -EIO +#define RPC_NT_DUPLICATE_ENDPOINT cpu_to_le32(0xC0020029) // -ENOTUNIQ +#define RPC_NT_UNKNOWN_AUTHN_TYPE cpu_to_le32(0xC002002A) // -EIO +#define RPC_NT_MAX_CALLS_TOO_SMALL cpu_to_le32(0xC002002B) // -EIO +#define RPC_NT_STRING_TOO_LONG cpu_to_le32(0xC002002C) // -EIO +#define RPC_NT_PROTSEQ_NOT_FOUND cpu_to_le32(0xC002002D) // -EIO +#define RPC_NT_PROCNUM_OUT_OF_RANGE cpu_to_le32(0xC002002E) // -EIO +#define RPC_NT_BINDING_HAS_NO_AUTH cpu_to_le32(0xC002002F) // -EIO +#define RPC_NT_UNKNOWN_AUTHN_SERVICE cpu_to_le32(0xC0020030) // -EIO +#define RPC_NT_UNKNOWN_AUTHN_LEVEL cpu_to_le32(0xC0020031) // -EIO +#define RPC_NT_INVALID_AUTH_IDENTITY cpu_to_le32(0xC0020032) // -EIO +#define RPC_NT_UNKNOWN_AUTHZ_SERVICE cpu_to_le32(0xC0020033) // -EIO +#define EPT_NT_INVALID_ENTRY cpu_to_le32(0xC0020034) // -EIO +#define EPT_NT_CANT_PERFORM_OP cpu_to_le32(0xC0020035) // -EIO +#define EPT_NT_NOT_REGISTERED cpu_to_le32(0xC0020036) // -EIO +#define RPC_NT_NOTHING_TO_EXPORT cpu_to_le32(0xC0020037) // -EIO +#define RPC_NT_INCOMPLETE_NAME cpu_to_le32(0xC0020038) // -EIO +#define RPC_NT_INVALID_VERS_OPTION cpu_to_le32(0xC0020039) // -EIO +#define RPC_NT_NO_MORE_MEMBERS cpu_to_le32(0xC002003A) // -EIO +#define RPC_NT_NOT_ALL_OBJS_UNEXPORTED cpu_to_le32(0xC002003B) // -EIO +#define RPC_NT_INTERFACE_NOT_FOUND cpu_to_le32(0xC002003C) // -EIO +#define RPC_NT_ENTRY_ALREADY_EXISTS cpu_to_le32(0xC002003D) // -EIO +#define RPC_NT_ENTRY_NOT_FOUND cpu_to_le32(0xC002003E) // -EIO +#define RPC_NT_NAME_SERVICE_UNAVAILABLE cpu_to_le32(0xC002003F) // -EIO +#define RPC_NT_INVALID_NAF_ID cpu_to_le32(0xC0020040) // -EIO +#define RPC_NT_CANNOT_SUPPORT cpu_to_le32(0xC0020041) // -EOPNOTSUPP +#define RPC_NT_NO_CONTEXT_AVAILABLE cpu_to_le32(0xC0020042) // -EIO +#define RPC_NT_INTERNAL_ERROR cpu_to_le32(0xC0020043) // -EIO +#define RPC_NT_ZERO_DIVIDE cpu_to_le32(0xC0020044) // -EIO +#define RPC_NT_ADDRESS_ERROR cpu_to_le32(0xC0020045) // -EIO +#define RPC_NT_FP_DIV_ZERO cpu_to_le32(0xC0020046) // -EIO +#define RPC_NT_FP_UNDERFLOW cpu_to_le32(0xC0020047) // -EIO +#define RPC_NT_FP_OVERFLOW cpu_to_le32(0xC0020048) // -EIO +#define RPC_NT_CALL_IN_PROGRESS cpu_to_le32(0xC0020049) // -EIO +#define RPC_NT_NO_MORE_BINDINGS cpu_to_le32(0xC002004A) // -EIO +#define RPC_NT_GROUP_MEMBER_NOT_FOUND cpu_to_le32(0xC002004B) // -EIO +#define EPT_NT_CANT_CREATE cpu_to_le32(0xC002004C) // -EIO +#define RPC_NT_INVALID_OBJECT cpu_to_le32(0xC002004D) // -EIO +#define RPC_NT_NO_INTERFACES cpu_to_le32(0xC002004F) // -EIO +#define RPC_NT_CALL_CANCELLED cpu_to_le32(0xC0020050) // -EIO +#define RPC_NT_BINDING_INCOMPLETE cpu_to_le32(0xC0020051) // -EIO +#define RPC_NT_COMM_FAILURE cpu_to_le32(0xC0020052) // -EIO +#define RPC_NT_UNSUPPORTED_AUTHN_LEVEL cpu_to_le32(0xC0020053) // -EIO +#define RPC_NT_NO_PRINC_NAME cpu_to_le32(0xC0020054) // -EIO +#define RPC_NT_NOT_RPC_ERROR cpu_to_le32(0xC0020055) // -EIO +#define RPC_NT_SEC_PKG_ERROR cpu_to_le32(0xC0020057) // -EIO +#define RPC_NT_NOT_CANCELLED cpu_to_le32(0xC0020058) // -EIO +#define RPC_NT_INVALID_ASYNC_HANDLE cpu_to_le32(0xC0020062) // -EIO +#define RPC_NT_INVALID_ASYNC_CALL cpu_to_le32(0xC0020063) // -EIO +#define RPC_NT_PROXY_ACCESS_DENIED cpu_to_le32(0xC0020064) // -EACCES +#define RPC_NT_NO_MORE_ENTRIES cpu_to_le32(0xC0030001) // -EIO +#define RPC_NT_SS_CHAR_TRANS_OPEN_FAIL cpu_to_le32(0xC0030002) // -EIO +#define RPC_NT_SS_CHAR_TRANS_SHORT_FILE cpu_to_le32(0xC0030003) // -EIO +#define RPC_NT_SS_IN_NULL_CONTEXT cpu_to_le32(0xC0030004) // -EIO +#define RPC_NT_SS_CONTEXT_MISMATCH cpu_to_le32(0xC0030005) // -EIO +#define RPC_NT_SS_CONTEXT_DAMAGED cpu_to_le32(0xC0030006) // -EIO +#define RPC_NT_SS_HANDLES_MISMATCH cpu_to_le32(0xC0030007) // -EIO +#define RPC_NT_SS_CANNOT_GET_CALL_HANDLE cpu_to_le32(0xC0030008) // -EIO +#define RPC_NT_NULL_REF_POINTER cpu_to_le32(0xC0030009) // -EIO +#define RPC_NT_ENUM_VALUE_OUT_OF_RANGE cpu_to_le32(0xC003000A) // -EIO +#define RPC_NT_BYTE_COUNT_TOO_SMALL cpu_to_le32(0xC003000B) // -EIO +#define RPC_NT_BAD_STUB_DATA cpu_to_le32(0xC003000C) // -EIO +#define RPC_NT_INVALID_ES_ACTION cpu_to_le32(0xC0030059) // -EIO +#define RPC_NT_WRONG_ES_VERSION cpu_to_le32(0xC003005A) // -EIO +#define RPC_NT_WRONG_STUB_VERSION cpu_to_le32(0xC003005B) // -EIO +#define RPC_NT_INVALID_PIPE_OBJECT cpu_to_le32(0xC003005C) // -EIO +#define RPC_NT_INVALID_PIPE_OPERATION cpu_to_le32(0xC003005D) // -EIO +#define RPC_NT_WRONG_PIPE_VERSION cpu_to_le32(0xC003005E) // -EIO +#define RPC_NT_PIPE_CLOSED cpu_to_le32(0xC003005F) // -EIO +#define RPC_NT_PIPE_DISCIPLINE_ERROR cpu_to_le32(0xC0030060) // -EIO +#define RPC_NT_PIPE_EMPTY cpu_to_le32(0xC0030061) // -EIO +#define STATUS_PNP_BAD_MPS_TABLE cpu_to_le32(0xC0040035) // -EIO +#define STATUS_PNP_TRANSLATION_FAILED cpu_to_le32(0xC0040036) // -EIO +#define STATUS_PNP_IRQ_TRANSLATION_FAILED cpu_to_le32(0xC0040037) // -EIO +#define STATUS_PNP_INVALID_ID cpu_to_le32(0xC0040038) // -EIO +#define STATUS_IO_REISSUE_AS_CACHED cpu_to_le32(0xC0040039) // -EIO +#define STATUS_CTX_WINSTATION_NAME_INVALID cpu_to_le32(0xC00A0001) // -EIO +#define STATUS_CTX_INVALID_PD cpu_to_le32(0xC00A0002) // -EIO +#define STATUS_CTX_PD_NOT_FOUND cpu_to_le32(0xC00A0003) // -EIO +#define STATUS_CTX_CLOSE_PENDING cpu_to_le32(0xC00A0006) // -EIO +#define STATUS_CTX_NO_OUTBUF cpu_to_le32(0xC00A0007) // -EIO +#define STATUS_CTX_MODEM_INF_NOT_FOUND cpu_to_le32(0xC00A0008) // -EIO +#define STATUS_CTX_INVALID_MODEMNAME cpu_to_le32(0xC00A0009) // -EIO +#define STATUS_CTX_RESPONSE_ERROR cpu_to_le32(0xC00A000A) // -EIO +#define STATUS_CTX_MODEM_RESPONSE_TIMEOUT cpu_to_le32(0xC00A000B) // -ETIMEDOUT +#define STATUS_CTX_MODEM_RESPONSE_NO_CARRIER cpu_to_le32(0xC00A000C) // -EIO +#define STATUS_CTX_MODEM_RESPONSE_NO_DIALTONE cpu_to_le32(0xC00A000D) // -EIO +#define STATUS_CTX_MODEM_RESPONSE_BUSY cpu_to_le32(0xC00A000E) // -EBUSY +#define STATUS_CTX_MODEM_RESPONSE_VOICE cpu_to_le32(0xC00A000F) // -EIO +#define STATUS_CTX_TD_ERROR cpu_to_le32(0xC00A0010) // -EIO +#define STATUS_CTX_LICENSE_CLIENT_INVALID cpu_to_le32(0xC00A0012) // -EIO +#define STATUS_CTX_LICENSE_NOT_AVAILABLE cpu_to_le32(0xC00A0013) // -EIO +#define STATUS_CTX_LICENSE_EXPIRED cpu_to_le32(0xC00A0014) // -EIO +#define STATUS_CTX_WINSTATION_NOT_FOUND cpu_to_le32(0xC00A0015) // -EIO +#define STATUS_CTX_WINSTATION_NAME_COLLISION cpu_to_le32(0xC00A0016) // -EIO +#define STATUS_CTX_WINSTATION_BUSY cpu_to_le32(0xC00A0017) // -EBUSY +#define STATUS_CTX_BAD_VIDEO_MODE cpu_to_le32(0xC00A0018) // -EIO +#define STATUS_CTX_GRAPHICS_INVALID cpu_to_le32(0xC00A0022) // -EIO +#define STATUS_CTX_NOT_CONSOLE cpu_to_le32(0xC00A0024) // -EIO +#define STATUS_CTX_CLIENT_QUERY_TIMEOUT cpu_to_le32(0xC00A0026) // -EIO +#define STATUS_CTX_CONSOLE_DISCONNECT cpu_to_le32(0xC00A0027) // -EIO +#define STATUS_CTX_CONSOLE_CONNECT cpu_to_le32(0xC00A0028) // -EIO +#define STATUS_CTX_SHADOW_DENIED cpu_to_le32(0xC00A002A) // -EIO +#define STATUS_CTX_WINSTATION_ACCESS_DENIED cpu_to_le32(0xC00A002B) // -EACCES +#define STATUS_CTX_INVALID_WD cpu_to_le32(0xC00A002E) // -EIO +#define STATUS_CTX_WD_NOT_FOUND cpu_to_le32(0xC00A002F) // -EIO +#define STATUS_CTX_SHADOW_INVALID cpu_to_le32(0xC00A0030) // -EIO +#define STATUS_CTX_SHADOW_DISABLED cpu_to_le32(0xC00A0031) // -EIO +#define STATUS_RDP_PROTOCOL_ERROR cpu_to_le32(0xC00A0032) // -EIO +#define STATUS_CTX_CLIENT_LICENSE_NOT_SET cpu_to_le32(0xC00A0033) // -EIO +#define STATUS_CTX_CLIENT_LICENSE_IN_USE cpu_to_le32(0xC00A0034) // -EIO +#define STATUS_CTX_SHADOW_ENDED_BY_MODE_CHANGE cpu_to_le32(0xC00A0035) // -EIO +#define STATUS_CTX_SHADOW_NOT_RUNNING cpu_to_le32(0xC00A0036) // -EIO +#define STATUS_CTX_LOGON_DISABLED cpu_to_le32(0xC00A0037) // -EIO +#define STATUS_CTX_SECURITY_LAYER_ERROR cpu_to_le32(0xC00A0038) // -EIO +#define STATUS_TS_INCOMPATIBLE_SESSIONS cpu_to_le32(0xC00A0039) // -EIO +#define STATUS_MUI_FILE_NOT_FOUND cpu_to_le32(0xC00B0001) // -EIO +#define STATUS_MUI_INVALID_FILE cpu_to_le32(0xC00B0002) // -EIO +#define STATUS_MUI_INVALID_RC_CONFIG cpu_to_le32(0xC00B0003) // -EIO +#define STATUS_MUI_INVALID_LOCALE_NAME cpu_to_le32(0xC00B0004) // -EIO +#define STATUS_MUI_INVALID_ULTIMATEFALLBACK_NAME cpu_to_le32(0xC00B0005) // -EIO +#define STATUS_MUI_FILE_NOT_LOADED cpu_to_le32(0xC00B0006) // -EIO +#define STATUS_RESOURCE_ENUM_USER_STOP cpu_to_le32(0xC00B0007) // -EIO +#define STATUS_CLUSTER_INVALID_NODE cpu_to_le32(0xC0130001) // -EIO +#define STATUS_CLUSTER_NODE_EXISTS cpu_to_le32(0xC0130002) // -EIO +#define STATUS_CLUSTER_JOIN_IN_PROGRESS cpu_to_le32(0xC0130003) // -EIO +#define STATUS_CLUSTER_NODE_NOT_FOUND cpu_to_le32(0xC0130004) // -EIO +#define STATUS_CLUSTER_LOCAL_NODE_NOT_FOUND cpu_to_le32(0xC0130005) // -EIO +#define STATUS_CLUSTER_NETWORK_EXISTS cpu_to_le32(0xC0130006) // -EIO +#define STATUS_CLUSTER_NETWORK_NOT_FOUND cpu_to_le32(0xC0130007) // -EIO +#define STATUS_CLUSTER_NETINTERFACE_EXISTS cpu_to_le32(0xC0130008) // -EIO +#define STATUS_CLUSTER_NETINTERFACE_NOT_FOUND cpu_to_le32(0xC0130009) // -EIO +#define STATUS_CLUSTER_INVALID_REQUEST cpu_to_le32(0xC013000A) // -EIO +#define STATUS_CLUSTER_INVALID_NETWORK_PROVIDER cpu_to_le32(0xC013000B) // -EIO +#define STATUS_CLUSTER_NODE_DOWN cpu_to_le32(0xC013000C) // -EIO +#define STATUS_CLUSTER_NODE_UNREACHABLE cpu_to_le32(0xC013000D) // -EIO +#define STATUS_CLUSTER_NODE_NOT_MEMBER cpu_to_le32(0xC013000E) // -EIO +#define STATUS_CLUSTER_JOIN_NOT_IN_PROGRESS cpu_to_le32(0xC013000F) // -EIO +#define STATUS_CLUSTER_INVALID_NETWORK cpu_to_le32(0xC0130010) // -EIO +#define STATUS_CLUSTER_NO_NET_ADAPTERS cpu_to_le32(0xC0130011) // -EIO +#define STATUS_CLUSTER_NODE_UP cpu_to_le32(0xC0130012) // -EIO +#define STATUS_CLUSTER_NODE_PAUSED cpu_to_le32(0xC0130013) // -EIO +#define STATUS_CLUSTER_NODE_NOT_PAUSED cpu_to_le32(0xC0130014) // -EIO +#define STATUS_CLUSTER_NO_SECURITY_CONTEXT cpu_to_le32(0xC0130015) // -EIO +#define STATUS_CLUSTER_NETWORK_NOT_INTERNAL cpu_to_le32(0xC0130016) // -EIO +#define STATUS_CLUSTER_POISONED cpu_to_le32(0xC0130017) // -EIO +#define STATUS_ACPI_INVALID_OPCODE cpu_to_le32(0xC0140001) // -EIO +#define STATUS_ACPI_STACK_OVERFLOW cpu_to_le32(0xC0140002) // -EIO +#define STATUS_ACPI_ASSERT_FAILED cpu_to_le32(0xC0140003) // -EIO +#define STATUS_ACPI_INVALID_INDEX cpu_to_le32(0xC0140004) // -EIO +#define STATUS_ACPI_INVALID_ARGUMENT cpu_to_le32(0xC0140005) // -EIO +#define STATUS_ACPI_FATAL cpu_to_le32(0xC0140006) // -EIO +#define STATUS_ACPI_INVALID_SUPERNAME cpu_to_le32(0xC0140007) // -EIO +#define STATUS_ACPI_INVALID_ARGTYPE cpu_to_le32(0xC0140008) // -EIO +#define STATUS_ACPI_INVALID_OBJTYPE cpu_to_le32(0xC0140009) // -EIO +#define STATUS_ACPI_INVALID_TARGETTYPE cpu_to_le32(0xC014000A) // -EIO +#define STATUS_ACPI_INCORRECT_ARGUMENT_COUNT cpu_to_le32(0xC014000B) // -EIO +#define STATUS_ACPI_ADDRESS_NOT_MAPPED cpu_to_le32(0xC014000C) // -EIO +#define STATUS_ACPI_INVALID_EVENTTYPE cpu_to_le32(0xC014000D) // -EIO +#define STATUS_ACPI_HANDLER_COLLISION cpu_to_le32(0xC014000E) // -EIO +#define STATUS_ACPI_INVALID_DATA cpu_to_le32(0xC014000F) // -EIO +#define STATUS_ACPI_INVALID_REGION cpu_to_le32(0xC0140010) // -EIO +#define STATUS_ACPI_INVALID_ACCESS_SIZE cpu_to_le32(0xC0140011) // -EIO +#define STATUS_ACPI_ACQUIRE_GLOBAL_LOCK cpu_to_le32(0xC0140012) // -EIO +#define STATUS_ACPI_ALREADY_INITIALIZED cpu_to_le32(0xC0140013) // -EIO +#define STATUS_ACPI_NOT_INITIALIZED cpu_to_le32(0xC0140014) // -EIO +#define STATUS_ACPI_INVALID_MUTEX_LEVEL cpu_to_le32(0xC0140015) // -EIO +#define STATUS_ACPI_MUTEX_NOT_OWNED cpu_to_le32(0xC0140016) // -EIO +#define STATUS_ACPI_MUTEX_NOT_OWNER cpu_to_le32(0xC0140017) // -EIO +#define STATUS_ACPI_RS_ACCESS cpu_to_le32(0xC0140018) // -EIO +#define STATUS_ACPI_INVALID_TABLE cpu_to_le32(0xC0140019) // -EIO +#define STATUS_ACPI_REG_HANDLER_FAILED cpu_to_le32(0xC0140020) // -EIO +#define STATUS_ACPI_POWER_REQUEST_FAILED cpu_to_le32(0xC0140021) // -EIO +#define STATUS_SXS_SECTION_NOT_FOUND cpu_to_le32(0xC0150001) // -EIO +#define STATUS_SXS_CANT_GEN_ACTCTX cpu_to_le32(0xC0150002) // -EIO +#define STATUS_SXS_INVALID_ACTCTXDATA_FORMAT cpu_to_le32(0xC0150003) // -EIO +#define STATUS_SXS_ASSEMBLY_NOT_FOUND cpu_to_le32(0xC0150004) // -EIO +#define STATUS_SXS_MANIFEST_FORMAT_ERROR cpu_to_le32(0xC0150005) // -EIO +#define STATUS_SXS_MANIFEST_PARSE_ERROR cpu_to_le32(0xC0150006) // -EIO +#define STATUS_SXS_ACTIVATION_CONTEXT_DISABLED cpu_to_le32(0xC0150007) // -EIO +#define STATUS_SXS_KEY_NOT_FOUND cpu_to_le32(0xC0150008) // -EIO +#define STATUS_SXS_VERSION_CONFLICT cpu_to_le32(0xC0150009) // -EIO +#define STATUS_SXS_WRONG_SECTION_TYPE cpu_to_le32(0xC015000A) // -EIO +#define STATUS_SXS_THREAD_QUERIES_DISABLED cpu_to_le32(0xC015000B) // -EIO +#define STATUS_SXS_ASSEMBLY_MISSING cpu_to_le32(0xC015000C) // -EIO +#define STATUS_SXS_PROCESS_DEFAULT_ALREADY_SET cpu_to_le32(0xC015000E) // -EIO +#define STATUS_SXS_EARLY_DEACTIVATION cpu_to_le32(0xC015000F) // -EIO +#define STATUS_SXS_INVALID_DEACTIVATION cpu_to_le32(0xC0150010) // -EIO +#define STATUS_SXS_MULTIPLE_DEACTIVATION cpu_to_le32(0xC0150011) // -EIO +#define STATUS_SXS_SYSTEM_DEFAULT_ACTIVATION_CONTEXT_EMPTY cpu_to_le32(0xC0150012) // -EIO +#define STATUS_SXS_PROCESS_TERMINATION_REQUESTED cpu_to_le32(0xC0150013) // -EIO +#define STATUS_SXS_CORRUPT_ACTIVATION_STACK cpu_to_le32(0xC0150014) // -EIO +#define STATUS_SXS_CORRUPTION cpu_to_le32(0xC0150015) // -EIO +#define STATUS_SXS_INVALID_IDENTITY_ATTRIBUTE_VALUE cpu_to_le32(0xC0150016) // -EIO +#define STATUS_SXS_INVALID_IDENTITY_ATTRIBUTE_NAME cpu_to_le32(0xC0150017) // -EIO +#define STATUS_SXS_IDENTITY_DUPLICATE_ATTRIBUTE cpu_to_le32(0xC0150018) // -EIO +#define STATUS_SXS_IDENTITY_PARSE_ERROR cpu_to_le32(0xC0150019) // -EIO +#define STATUS_SXS_COMPONENT_STORE_CORRUPT cpu_to_le32(0xC015001A) // -EIO +#define STATUS_SXS_FILE_HASH_MISMATCH cpu_to_le32(0xC015001B) // -EIO +#define STATUS_SXS_MANIFEST_IDENTITY_SAME_BUT_CONTENTS_DIFFERENT cpu_to_le32(0xC015001C) // -EIO +#define STATUS_SXS_IDENTITIES_DIFFERENT cpu_to_le32(0xC015001D) // -EIO +#define STATUS_SXS_ASSEMBLY_IS_NOT_A_DEPLOYMENT cpu_to_le32(0xC015001E) // -EIO +#define STATUS_SXS_FILE_NOT_PART_OF_ASSEMBLY cpu_to_le32(0xC015001F) // -EIO +#define STATUS_ADVANCED_INSTALLER_FAILED cpu_to_le32(0xC0150020) // -EIO +#define STATUS_XML_ENCODING_MISMATCH cpu_to_le32(0xC0150021) // -EIO +#define STATUS_SXS_MANIFEST_TOO_BIG cpu_to_le32(0xC0150022) // -EIO +#define STATUS_SXS_SETTING_NOT_REGISTERED cpu_to_le32(0xC0150023) // -EIO +#define STATUS_SXS_TRANSACTION_CLOSURE_INCOMPLETE cpu_to_le32(0xC0150024) // -EIO +#define STATUS_SMI_PRIMITIVE_INSTALLER_FAILED cpu_to_le32(0xC0150025) // -EIO +#define STATUS_GENERIC_COMMAND_FAILED cpu_to_le32(0xC0150026) // -EIO +#define STATUS_SXS_FILE_HASH_MISSING cpu_to_le32(0xC0150027) // -EIO +#define STATUS_TRANSACTIONAL_CONFLICT cpu_to_le32(0xC0190001) // -EIO +#define STATUS_INVALID_TRANSACTION cpu_to_le32(0xC0190002) // -EIO +#define STATUS_TRANSACTION_NOT_ACTIVE cpu_to_le32(0xC0190003) // -EIO +#define STATUS_TM_INITIALIZATION_FAILED cpu_to_le32(0xC0190004) // -EIO +#define STATUS_RM_NOT_ACTIVE cpu_to_le32(0xC0190005) // -EIO +#define STATUS_RM_METADATA_CORRUPT cpu_to_le32(0xC0190006) // -EIO +#define STATUS_TRANSACTION_NOT_JOINED cpu_to_le32(0xC0190007) // -EIO +#define STATUS_DIRECTORY_NOT_RM cpu_to_le32(0xC0190008) // -EIO +#define STATUS_TRANSACTIONS_UNSUPPORTED_REMOTE cpu_to_le32(0xC019000A) // -EIO +#define STATUS_LOG_RESIZE_INVALID_SIZE cpu_to_le32(0xC019000B) // -EIO +#define STATUS_REMOTE_FILE_VERSION_MISMATCH cpu_to_le32(0xC019000C) // -EIO +#define STATUS_CRM_PROTOCOL_ALREADY_EXISTS cpu_to_le32(0xC019000F) // -EIO +#define STATUS_TRANSACTION_PROPAGATION_FAILED cpu_to_le32(0xC0190010) // -EIO +#define STATUS_CRM_PROTOCOL_NOT_FOUND cpu_to_le32(0xC0190011) // -EIO +#define STATUS_TRANSACTION_SUPERIOR_EXISTS cpu_to_le32(0xC0190012) // -EIO +#define STATUS_TRANSACTION_REQUEST_NOT_VALID cpu_to_le32(0xC0190013) // -EIO +#define STATUS_TRANSACTION_NOT_REQUESTED cpu_to_le32(0xC0190014) // -EIO +#define STATUS_TRANSACTION_ALREADY_ABORTED cpu_to_le32(0xC0190015) // -EIO +#define STATUS_TRANSACTION_ALREADY_COMMITTED cpu_to_le32(0xC0190016) // -EIO +#define STATUS_TRANSACTION_INVALID_MARSHALL_BUFFER cpu_to_le32(0xC0190017) // -EIO +#define STATUS_CURRENT_TRANSACTION_NOT_VALID cpu_to_le32(0xC0190018) // -EIO +#define STATUS_LOG_GROWTH_FAILED cpu_to_le32(0xC0190019) // -EIO +#define STATUS_OBJECT_NO_LONGER_EXISTS cpu_to_le32(0xC0190021) // -EIO +#define STATUS_STREAM_MINIVERSION_NOT_FOUND cpu_to_le32(0xC0190022) // -EIO +#define STATUS_STREAM_MINIVERSION_NOT_VALID cpu_to_le32(0xC0190023) // -EIO +#define STATUS_MINIVERSION_INACCESSIBLE_FROM_SPECIFIED_TRANSACTION cpu_to_le32(0xC0190024) // -EIO +#define STATUS_CANT_OPEN_MINIVERSION_WITH_MODIFY_INTENT cpu_to_le32(0xC0190025) // -EIO +#define STATUS_CANT_CREATE_MORE_STREAM_MINIVERSIONS cpu_to_le32(0xC0190026) // -EIO +#define STATUS_HANDLE_NO_LONGER_VALID cpu_to_le32(0xC0190028) // -EIO +#define STATUS_LOG_CORRUPTION_DETECTED cpu_to_le32(0xC0190030) // -EIO +#define STATUS_RM_DISCONNECTED cpu_to_le32(0xC0190032) // -EIO +#define STATUS_ENLISTMENT_NOT_SUPERIOR cpu_to_le32(0xC0190033) // -EIO +#define STATUS_FILE_IDENTITY_NOT_PERSISTENT cpu_to_le32(0xC0190036) // -EIO +#define STATUS_CANT_BREAK_TRANSACTIONAL_DEPENDENCY cpu_to_le32(0xC0190037) // -EIO +#define STATUS_CANT_CROSS_RM_BOUNDARY cpu_to_le32(0xC0190038) // -EIO +#define STATUS_TXF_DIR_NOT_EMPTY cpu_to_le32(0xC0190039) // -EIO +#define STATUS_INDOUBT_TRANSACTIONS_EXIST cpu_to_le32(0xC019003A) // -EIO +#define STATUS_TM_VOLATILE cpu_to_le32(0xC019003B) // -EIO +#define STATUS_ROLLBACK_TIMER_EXPIRED cpu_to_le32(0xC019003C) // -EIO +#define STATUS_TXF_ATTRIBUTE_CORRUPT cpu_to_le32(0xC019003D) // -EIO +#define STATUS_EFS_NOT_ALLOWED_IN_TRANSACTION cpu_to_le32(0xC019003E) // -EIO +#define STATUS_TRANSACTIONAL_OPEN_NOT_ALLOWED cpu_to_le32(0xC019003F) // -EIO +#define STATUS_TRANSACTED_MAPPING_UNSUPPORTED_REMOTE cpu_to_le32(0xC0190040) // -EIO +#define STATUS_TRANSACTION_REQUIRED_PROMOTION cpu_to_le32(0xC0190043) // -EIO +#define STATUS_CANNOT_EXECUTE_FILE_IN_TRANSACTION cpu_to_le32(0xC0190044) // -EIO +#define STATUS_TRANSACTIONS_NOT_FROZEN cpu_to_le32(0xC0190045) // -EIO +#define STATUS_TRANSACTION_FREEZE_IN_PROGRESS cpu_to_le32(0xC0190046) // -EIO +#define STATUS_NOT_SNAPSHOT_VOLUME cpu_to_le32(0xC0190047) // -EIO +#define STATUS_NO_SAVEPOINT_WITH_OPEN_FILES cpu_to_le32(0xC0190048) // -EIO +#define STATUS_SPARSE_NOT_ALLOWED_IN_TRANSACTION cpu_to_le32(0xC0190049) // -EIO +#define STATUS_TM_IDENTITY_MISMATCH cpu_to_le32(0xC019004A) // -EIO +#define STATUS_FLOATED_SECTION cpu_to_le32(0xC019004B) // -EIO +#define STATUS_CANNOT_ACCEPT_TRANSACTED_WORK cpu_to_le32(0xC019004C) // -EIO +#define STATUS_CANNOT_ABORT_TRANSACTIONS cpu_to_le32(0xC019004D) // -EIO +#define STATUS_TRANSACTION_NOT_FOUND cpu_to_le32(0xC019004E) // -EIO +#define STATUS_RESOURCEMANAGER_NOT_FOUND cpu_to_le32(0xC019004F) // -EIO +#define STATUS_ENLISTMENT_NOT_FOUND cpu_to_le32(0xC0190050) // -EIO +#define STATUS_TRANSACTIONMANAGER_NOT_FOUND cpu_to_le32(0xC0190051) // -EIO +#define STATUS_TRANSACTIONMANAGER_NOT_ONLINE cpu_to_le32(0xC0190052) // -EIO +#define STATUS_TRANSACTIONMANAGER_RECOVERY_NAME_COLLISION cpu_to_le32(0xC0190053) // -EIO +#define STATUS_TRANSACTION_NOT_ROOT cpu_to_le32(0xC0190054) // -EIO +#define STATUS_TRANSACTION_OBJECT_EXPIRED cpu_to_le32(0xC0190055) // -EIO +#define STATUS_COMPRESSION_NOT_ALLOWED_IN_TRANSACTION cpu_to_le32(0xC0190056) // -EIO +#define STATUS_TRANSACTION_RESPONSE_NOT_ENLISTED cpu_to_le32(0xC0190057) // -EIO +#define STATUS_TRANSACTION_RECORD_TOO_LONG cpu_to_le32(0xC0190058) // -EIO +#define STATUS_NO_LINK_TRACKING_IN_TRANSACTION cpu_to_le32(0xC0190059) // -EIO +#define STATUS_OPERATION_NOT_SUPPORTED_IN_TRANSACTION cpu_to_le32(0xC019005A) // -EOPNOTSUPP +#define STATUS_TRANSACTION_INTEGRITY_VIOLATED cpu_to_le32(0xC019005B) // -EIO +#define STATUS_LOG_SECTOR_INVALID cpu_to_le32(0xC01A0001) // -EIO +#define STATUS_LOG_SECTOR_PARITY_INVALID cpu_to_le32(0xC01A0002) // -EIO +#define STATUS_LOG_SECTOR_REMAPPED cpu_to_le32(0xC01A0003) // -EIO +#define STATUS_LOG_BLOCK_INCOMPLETE cpu_to_le32(0xC01A0004) // -EIO +#define STATUS_LOG_INVALID_RANGE cpu_to_le32(0xC01A0005) // -EIO +#define STATUS_LOG_BLOCKS_EXHAUSTED cpu_to_le32(0xC01A0006) // -EIO +#define STATUS_LOG_READ_CONTEXT_INVALID cpu_to_le32(0xC01A0007) // -EIO +#define STATUS_LOG_RESTART_INVALID cpu_to_le32(0xC01A0008) // -EIO +#define STATUS_LOG_BLOCK_VERSION cpu_to_le32(0xC01A0009) // -EIO +#define STATUS_LOG_BLOCK_INVALID cpu_to_le32(0xC01A000A) // -EIO +#define STATUS_LOG_READ_MODE_INVALID cpu_to_le32(0xC01A000B) // -EIO +#define STATUS_LOG_METADATA_CORRUPT cpu_to_le32(0xC01A000D) // -EIO +#define STATUS_LOG_METADATA_INVALID cpu_to_le32(0xC01A000E) // -EIO +#define STATUS_LOG_METADATA_INCONSISTENT cpu_to_le32(0xC01A000F) // -EIO +#define STATUS_LOG_RESERVATION_INVALID cpu_to_le32(0xC01A0010) // -EIO +#define STATUS_LOG_CANT_DELETE cpu_to_le32(0xC01A0011) // -EIO +#define STATUS_LOG_CONTAINER_LIMIT_EXCEEDED cpu_to_le32(0xC01A0012) // -EIO +#define STATUS_LOG_START_OF_LOG cpu_to_le32(0xC01A0013) // -EIO +#define STATUS_LOG_POLICY_ALREADY_INSTALLED cpu_to_le32(0xC01A0014) // -EIO +#define STATUS_LOG_POLICY_NOT_INSTALLED cpu_to_le32(0xC01A0015) // -EIO +#define STATUS_LOG_POLICY_INVALID cpu_to_le32(0xC01A0016) // -EIO +#define STATUS_LOG_POLICY_CONFLICT cpu_to_le32(0xC01A0017) // -EIO +#define STATUS_LOG_PINNED_ARCHIVE_TAIL cpu_to_le32(0xC01A0018) // -EIO +#define STATUS_LOG_RECORD_NONEXISTENT cpu_to_le32(0xC01A0019) // -EIO +#define STATUS_LOG_RECORDS_RESERVED_INVALID cpu_to_le32(0xC01A001A) // -EIO +#define STATUS_LOG_SPACE_RESERVED_INVALID cpu_to_le32(0xC01A001B) // -EIO +#define STATUS_LOG_TAIL_INVALID cpu_to_le32(0xC01A001C) // -EIO +#define STATUS_LOG_FULL cpu_to_le32(0xC01A001D) // -EIO +#define STATUS_LOG_MULTIPLEXED cpu_to_le32(0xC01A001E) // -EIO +#define STATUS_LOG_DEDICATED cpu_to_le32(0xC01A001F) // -EIO +#define STATUS_LOG_ARCHIVE_NOT_IN_PROGRESS cpu_to_le32(0xC01A0020) // -EIO +#define STATUS_LOG_ARCHIVE_IN_PROGRESS cpu_to_le32(0xC01A0021) // -EIO +#define STATUS_LOG_EPHEMERAL cpu_to_le32(0xC01A0022) // -EIO +#define STATUS_LOG_NOT_ENOUGH_CONTAINERS cpu_to_le32(0xC01A0023) // -EIO +#define STATUS_LOG_CLIENT_ALREADY_REGISTERED cpu_to_le32(0xC01A0024) // -EIO +#define STATUS_LOG_CLIENT_NOT_REGISTERED cpu_to_le32(0xC01A0025) // -EIO +#define STATUS_LOG_FULL_HANDLER_IN_PROGRESS cpu_to_le32(0xC01A0026) // -EIO +#define STATUS_LOG_CONTAINER_READ_FAILED cpu_to_le32(0xC01A0027) // -EIO +#define STATUS_LOG_CONTAINER_WRITE_FAILED cpu_to_le32(0xC01A0028) // -EIO +#define STATUS_LOG_CONTAINER_OPEN_FAILED cpu_to_le32(0xC01A0029) // -EIO +#define STATUS_LOG_CONTAINER_STATE_INVALID cpu_to_le32(0xC01A002A) // -EIO +#define STATUS_LOG_STATE_INVALID cpu_to_le32(0xC01A002B) // -EIO +#define STATUS_LOG_PINNED cpu_to_le32(0xC01A002C) // -EIO +#define STATUS_LOG_METADATA_FLUSH_FAILED cpu_to_le32(0xC01A002D) // -EIO +#define STATUS_LOG_INCONSISTENT_SECURITY cpu_to_le32(0xC01A002E) // -EIO +#define STATUS_LOG_APPENDED_FLUSH_FAILED cpu_to_le32(0xC01A002F) // -EIO +#define STATUS_LOG_PINNED_RESERVATION cpu_to_le32(0xC01A0030) // -EIO +#define STATUS_VIDEO_HUNG_DISPLAY_DRIVER_THREAD cpu_to_le32(0xC01B00EA) // -EIO +#define STATUS_FLT_NO_HANDLER_DEFINED cpu_to_le32(0xC01C0001) // -EIO +#define STATUS_FLT_CONTEXT_ALREADY_DEFINED cpu_to_le32(0xC01C0002) // -EIO +#define STATUS_FLT_INVALID_ASYNCHRONOUS_REQUEST cpu_to_le32(0xC01C0003) // -EIO +#define STATUS_FLT_DISALLOW_FAST_IO cpu_to_le32(0xC01C0004) // -EIO +#define STATUS_FLT_INVALID_NAME_REQUEST cpu_to_le32(0xC01C0005) // -EIO +#define STATUS_FLT_NOT_SAFE_TO_POST_OPERATION cpu_to_le32(0xC01C0006) // -EIO +#define STATUS_FLT_NOT_INITIALIZED cpu_to_le32(0xC01C0007) // -EIO +#define STATUS_FLT_FILTER_NOT_READY cpu_to_le32(0xC01C0008) // -EIO +#define STATUS_FLT_POST_OPERATION_CLEANUP cpu_to_le32(0xC01C0009) // -EIO +#define STATUS_FLT_INTERNAL_ERROR cpu_to_le32(0xC01C000A) // -EIO +#define STATUS_FLT_DELETING_OBJECT cpu_to_le32(0xC01C000B) // -EIO +#define STATUS_FLT_MUST_BE_NONPAGED_POOL cpu_to_le32(0xC01C000C) // -EIO +#define STATUS_FLT_DUPLICATE_ENTRY cpu_to_le32(0xC01C000D) // -EIO +#define STATUS_FLT_CBDQ_DISABLED cpu_to_le32(0xC01C000E) // -EIO +#define STATUS_FLT_DO_NOT_ATTACH cpu_to_le32(0xC01C000F) // -EIO +#define STATUS_FLT_DO_NOT_DETACH cpu_to_le32(0xC01C0010) // -EIO +#define STATUS_FLT_INSTANCE_ALTITUDE_COLLISION cpu_to_le32(0xC01C0011) // -EIO +#define STATUS_FLT_INSTANCE_NAME_COLLISION cpu_to_le32(0xC01C0012) // -EIO +#define STATUS_FLT_FILTER_NOT_FOUND cpu_to_le32(0xC01C0013) // -EIO +#define STATUS_FLT_VOLUME_NOT_FOUND cpu_to_le32(0xC01C0014) // -EIO +#define STATUS_FLT_INSTANCE_NOT_FOUND cpu_to_le32(0xC01C0015) // -EIO +#define STATUS_FLT_CONTEXT_ALLOCATION_NOT_FOUND cpu_to_le32(0xC01C0016) // -EIO +#define STATUS_FLT_INVALID_CONTEXT_REGISTRATION cpu_to_le32(0xC01C0017) // -EIO +#define STATUS_FLT_NAME_CACHE_MISS cpu_to_le32(0xC01C0018) // -EIO +#define STATUS_FLT_NO_DEVICE_OBJECT cpu_to_le32(0xC01C0019) // -EIO +#define STATUS_FLT_VOLUME_ALREADY_MOUNTED cpu_to_le32(0xC01C001A) // -EIO +#define STATUS_FLT_ALREADY_ENLISTED cpu_to_le32(0xC01C001B) // -EIO +#define STATUS_FLT_CONTEXT_ALREADY_LINKED cpu_to_le32(0xC01C001C) // -EIO +#define STATUS_FLT_NO_WAITER_FOR_REPLY cpu_to_le32(0xC01C0020) // -EIO +#define STATUS_MONITOR_NO_DESCRIPTOR cpu_to_le32(0xC01D0001) // -EIO +#define STATUS_MONITOR_UNKNOWN_DESCRIPTOR_FORMAT cpu_to_le32(0xC01D0002) // -EIO +#define STATUS_MONITOR_INVALID_DESCRIPTOR_CHECKSUM cpu_to_le32(0xC01D0003) // -EIO +#define STATUS_MONITOR_INVALID_STANDARD_TIMING_BLOCK cpu_to_le32(0xC01D0004) // -EIO +#define STATUS_MONITOR_WMI_DATABLOCK_REGISTRATION_FAILED cpu_to_le32(0xC01D0005) // -EIO +#define STATUS_MONITOR_INVALID_SERIAL_NUMBER_MONDSC_BLOCK cpu_to_le32(0xC01D0006) // -EIO +#define STATUS_MONITOR_INVALID_USER_FRIENDLY_MONDSC_BLOCK cpu_to_le32(0xC01D0007) // -EIO +#define STATUS_MONITOR_NO_MORE_DESCRIPTOR_DATA cpu_to_le32(0xC01D0008) // -EIO +#define STATUS_MONITOR_INVALID_DETAILED_TIMING_BLOCK cpu_to_le32(0xC01D0009) // -EIO +#define STATUS_GRAPHICS_NOT_EXCLUSIVE_MODE_OWNER cpu_to_le32(0xC01E0000) // -EIO +#define STATUS_GRAPHICS_INSUFFICIENT_DMA_BUFFER cpu_to_le32(0xC01E0001) // -EIO +#define STATUS_GRAPHICS_INVALID_DISPLAY_ADAPTER cpu_to_le32(0xC01E0002) // -EIO +#define STATUS_GRAPHICS_ADAPTER_WAS_RESET cpu_to_le32(0xC01E0003) // -EIO +#define STATUS_GRAPHICS_INVALID_DRIVER_MODEL cpu_to_le32(0xC01E0004) // -EIO +#define STATUS_GRAPHICS_PRESENT_MODE_CHANGED cpu_to_le32(0xC01E0005) // -EIO +#define STATUS_GRAPHICS_PRESENT_OCCLUDED cpu_to_le32(0xC01E0006) // -EIO +#define STATUS_GRAPHICS_PRESENT_DENIED cpu_to_le32(0xC01E0007) // -EIO +#define STATUS_GRAPHICS_CANNOTCOLORCONVERT cpu_to_le32(0xC01E0008) // -EIO +#define STATUS_GRAPHICS_NO_VIDEO_MEMORY cpu_to_le32(0xC01E0100) // -EIO +#define STATUS_GRAPHICS_CANT_LOCK_MEMORY cpu_to_le32(0xC01E0101) // -EIO +#define STATUS_GRAPHICS_ALLOCATION_BUSY cpu_to_le32(0xC01E0102) // -EBUSY +#define STATUS_GRAPHICS_TOO_MANY_REFERENCES cpu_to_le32(0xC01E0103) // -EIO +#define STATUS_GRAPHICS_TRY_AGAIN_LATER cpu_to_le32(0xC01E0104) // -EIO +#define STATUS_GRAPHICS_TRY_AGAIN_NOW cpu_to_le32(0xC01E0105) // -EIO +#define STATUS_GRAPHICS_ALLOCATION_INVALID cpu_to_le32(0xC01E0106) // -EIO +#define STATUS_GRAPHICS_UNSWIZZLING_APERTURE_UNAVAILABLE cpu_to_le32(0xC01E0107) // -EIO +#define STATUS_GRAPHICS_UNSWIZZLING_APERTURE_UNSUPPORTED cpu_to_le32(0xC01E0108) // -EIO +#define STATUS_GRAPHICS_CANT_EVICT_PINNED_ALLOCATION cpu_to_le32(0xC01E0109) // -EIO +#define STATUS_GRAPHICS_INVALID_ALLOCATION_USAGE cpu_to_le32(0xC01E0110) // -EIO +#define STATUS_GRAPHICS_CANT_RENDER_LOCKED_ALLOCATION cpu_to_le32(0xC01E0111) // -EIO +#define STATUS_GRAPHICS_ALLOCATION_CLOSED cpu_to_le32(0xC01E0112) // -EIO +#define STATUS_GRAPHICS_INVALID_ALLOCATION_INSTANCE cpu_to_le32(0xC01E0113) // -EIO +#define STATUS_GRAPHICS_INVALID_ALLOCATION_HANDLE cpu_to_le32(0xC01E0114) // -EIO +#define STATUS_GRAPHICS_WRONG_ALLOCATION_DEVICE cpu_to_le32(0xC01E0115) // -EIO +#define STATUS_GRAPHICS_ALLOCATION_CONTENT_LOST cpu_to_le32(0xC01E0116) // -EIO +#define STATUS_GRAPHICS_GPU_EXCEPTION_ON_DEVICE cpu_to_le32(0xC01E0200) // -EIO +#define STATUS_GRAPHICS_INVALID_VIDPN_TOPOLOGY cpu_to_le32(0xC01E0300) // -EIO +#define STATUS_GRAPHICS_VIDPN_TOPOLOGY_NOT_SUPPORTED cpu_to_le32(0xC01E0301) // -EIO +#define STATUS_GRAPHICS_VIDPN_TOPOLOGY_CURRENTLY_NOT_SUPPORTED cpu_to_le32(0xC01E0302) // -EIO +#define STATUS_GRAPHICS_INVALID_VIDPN cpu_to_le32(0xC01E0303) // -EIO +#define STATUS_GRAPHICS_INVALID_VIDEO_PRESENT_SOURCE cpu_to_le32(0xC01E0304) // -EIO +#define STATUS_GRAPHICS_INVALID_VIDEO_PRESENT_TARGET cpu_to_le32(0xC01E0305) // -EIO +#define STATUS_GRAPHICS_VIDPN_MODALITY_NOT_SUPPORTED cpu_to_le32(0xC01E0306) // -EIO +#define STATUS_GRAPHICS_INVALID_VIDPN_SOURCEMODESET cpu_to_le32(0xC01E0308) // -EIO +#define STATUS_GRAPHICS_INVALID_VIDPN_TARGETMODESET cpu_to_le32(0xC01E0309) // -EIO +#define STATUS_GRAPHICS_INVALID_FREQUENCY cpu_to_le32(0xC01E030A) // -EIO +#define STATUS_GRAPHICS_INVALID_ACTIVE_REGION cpu_to_le32(0xC01E030B) // -EIO +#define STATUS_GRAPHICS_INVALID_TOTAL_REGION cpu_to_le32(0xC01E030C) // -EIO +#define STATUS_GRAPHICS_INVALID_VIDEO_PRESENT_SOURCE_MODE cpu_to_le32(0xC01E0310) // -EIO +#define STATUS_GRAPHICS_INVALID_VIDEO_PRESENT_TARGET_MODE cpu_to_le32(0xC01E0311) // -EIO +#define STATUS_GRAPHICS_PINNED_MODE_MUST_REMAIN_IN_SET cpu_to_le32(0xC01E0312) // -EIO +#define STATUS_GRAPHICS_PATH_ALREADY_IN_TOPOLOGY cpu_to_le32(0xC01E0313) // -EIO +#define STATUS_GRAPHICS_MODE_ALREADY_IN_MODESET cpu_to_le32(0xC01E0314) // -EIO +#define STATUS_GRAPHICS_INVALID_VIDEOPRESENTSOURCESET cpu_to_le32(0xC01E0315) // -EIO +#define STATUS_GRAPHICS_INVALID_VIDEOPRESENTTARGETSET cpu_to_le32(0xC01E0316) // -EIO +#define STATUS_GRAPHICS_SOURCE_ALREADY_IN_SET cpu_to_le32(0xC01E0317) // -EIO +#define STATUS_GRAPHICS_TARGET_ALREADY_IN_SET cpu_to_le32(0xC01E0318) // -EIO +#define STATUS_GRAPHICS_INVALID_VIDPN_PRESENT_PATH cpu_to_le32(0xC01E0319) // -EIO +#define STATUS_GRAPHICS_NO_RECOMMENDED_VIDPN_TOPOLOGY cpu_to_le32(0xC01E031A) // -EIO +#define STATUS_GRAPHICS_INVALID_MONITOR_FREQUENCYRANGESET cpu_to_le32(0xC01E031B) // -EIO +#define STATUS_GRAPHICS_INVALID_MONITOR_FREQUENCYRANGE cpu_to_le32(0xC01E031C) // -EIO +#define STATUS_GRAPHICS_FREQUENCYRANGE_NOT_IN_SET cpu_to_le32(0xC01E031D) // -EIO +#define STATUS_GRAPHICS_FREQUENCYRANGE_ALREADY_IN_SET cpu_to_le32(0xC01E031F) // -EIO +#define STATUS_GRAPHICS_STALE_MODESET cpu_to_le32(0xC01E0320) // -EIO +#define STATUS_GRAPHICS_INVALID_MONITOR_SOURCEMODESET cpu_to_le32(0xC01E0321) // -EIO +#define STATUS_GRAPHICS_INVALID_MONITOR_SOURCE_MODE cpu_to_le32(0xC01E0322) // -EIO +#define STATUS_GRAPHICS_NO_RECOMMENDED_FUNCTIONAL_VIDPN cpu_to_le32(0xC01E0323) // -EIO +#define STATUS_GRAPHICS_MODE_ID_MUST_BE_UNIQUE cpu_to_le32(0xC01E0324) // -EIO +#define STATUS_GRAPHICS_EMPTY_ADAPTER_MONITOR_MODE_SUPPORT_INTERSECTION cpu_to_le32(0xC01E0325) // -EIO +#define STATUS_GRAPHICS_VIDEO_PRESENT_TARGETS_LESS_THAN_SOURCES cpu_to_le32(0xC01E0326) // -EIO +#define STATUS_GRAPHICS_PATH_NOT_IN_TOPOLOGY cpu_to_le32(0xC01E0327) // -EIO +#define STATUS_GRAPHICS_ADAPTER_MUST_HAVE_AT_LEAST_ONE_SOURCE cpu_to_le32(0xC01E0328) // -EIO +#define STATUS_GRAPHICS_ADAPTER_MUST_HAVE_AT_LEAST_ONE_TARGET cpu_to_le32(0xC01E0329) // -EIO +#define STATUS_GRAPHICS_INVALID_MONITORDESCRIPTORSET cpu_to_le32(0xC01E032A) // -EIO +#define STATUS_GRAPHICS_INVALID_MONITORDESCRIPTOR cpu_to_le32(0xC01E032B) // -EIO +#define STATUS_GRAPHICS_MONITORDESCRIPTOR_NOT_IN_SET cpu_to_le32(0xC01E032C) // -EIO +#define STATUS_GRAPHICS_MONITORDESCRIPTOR_ALREADY_IN_SET cpu_to_le32(0xC01E032D) // -EIO +#define STATUS_GRAPHICS_MONITORDESCRIPTOR_ID_MUST_BE_UNIQUE cpu_to_le32(0xC01E032E) // -EIO +#define STATUS_GRAPHICS_INVALID_VIDPN_TARGET_SUBSET_TYPE cpu_to_le32(0xC01E032F) // -EIO +#define STATUS_GRAPHICS_RESOURCES_NOT_RELATED cpu_to_le32(0xC01E0330) // -EIO +#define STATUS_GRAPHICS_SOURCE_ID_MUST_BE_UNIQUE cpu_to_le32(0xC01E0331) // -EIO +#define STATUS_GRAPHICS_TARGET_ID_MUST_BE_UNIQUE cpu_to_le32(0xC01E0332) // -EIO +#define STATUS_GRAPHICS_NO_AVAILABLE_VIDPN_TARGET cpu_to_le32(0xC01E0333) // -EIO +#define STATUS_GRAPHICS_MONITOR_COULD_NOT_BE_ASSOCIATED_WITH_ADAPTER cpu_to_le32(0xC01E0334) // -EIO +#define STATUS_GRAPHICS_NO_VIDPNMGR cpu_to_le32(0xC01E0335) // -EIO +#define STATUS_GRAPHICS_NO_ACTIVE_VIDPN cpu_to_le32(0xC01E0336) // -EIO +#define STATUS_GRAPHICS_STALE_VIDPN_TOPOLOGY cpu_to_le32(0xC01E0337) // -EIO +#define STATUS_GRAPHICS_MONITOR_NOT_CONNECTED cpu_to_le32(0xC01E0338) // -EIO +#define STATUS_GRAPHICS_SOURCE_NOT_IN_TOPOLOGY cpu_to_le32(0xC01E0339) // -EIO +#define STATUS_GRAPHICS_INVALID_PRIMARYSURFACE_SIZE cpu_to_le32(0xC01E033A) // -EIO +#define STATUS_GRAPHICS_INVALID_VISIBLEREGION_SIZE cpu_to_le32(0xC01E033B) // -EIO +#define STATUS_GRAPHICS_INVALID_STRIDE cpu_to_le32(0xC01E033C) // -EIO +#define STATUS_GRAPHICS_INVALID_PIXELFORMAT cpu_to_le32(0xC01E033D) // -EIO +#define STATUS_GRAPHICS_INVALID_COLORBASIS cpu_to_le32(0xC01E033E) // -EIO +#define STATUS_GRAPHICS_INVALID_PIXELVALUEACCESSMODE cpu_to_le32(0xC01E033F) // -EIO +#define STATUS_GRAPHICS_TARGET_NOT_IN_TOPOLOGY cpu_to_le32(0xC01E0340) // -EIO +#define STATUS_GRAPHICS_NO_DISPLAY_MODE_MANAGEMENT_SUPPORT cpu_to_le32(0xC01E0341) // -EIO +#define STATUS_GRAPHICS_VIDPN_SOURCE_IN_USE cpu_to_le32(0xC01E0342) // -EIO +#define STATUS_GRAPHICS_CANT_ACCESS_ACTIVE_VIDPN cpu_to_le32(0xC01E0343) // -EIO +#define STATUS_GRAPHICS_INVALID_PATH_IMPORTANCE_ORDINAL cpu_to_le32(0xC01E0344) // -EIO +#define STATUS_GRAPHICS_INVALID_PATH_CONTENT_GEOMETRY_TRANSFORMATION cpu_to_le32(0xC01E0345) // -EIO +#define STATUS_GRAPHICS_PATH_CONTENT_GEOMETRY_TRANSFORMATION_NOT_SUPPORTED cpu_to_le32(0xC01E0346) // -EIO +#define STATUS_GRAPHICS_INVALID_GAMMA_RAMP cpu_to_le32(0xC01E0347) // -EIO +#define STATUS_GRAPHICS_GAMMA_RAMP_NOT_SUPPORTED cpu_to_le32(0xC01E0348) // -EIO +#define STATUS_GRAPHICS_MULTISAMPLING_NOT_SUPPORTED cpu_to_le32(0xC01E0349) // -EIO +#define STATUS_GRAPHICS_MODE_NOT_IN_MODESET cpu_to_le32(0xC01E034A) // -EIO +#define STATUS_GRAPHICS_INVALID_VIDPN_TOPOLOGY_RECOMMENDATION_REASON cpu_to_le32(0xC01E034D) // -EIO +#define STATUS_GRAPHICS_INVALID_PATH_CONTENT_TYPE cpu_to_le32(0xC01E034E) // -EIO +#define STATUS_GRAPHICS_INVALID_COPYPROTECTION_TYPE cpu_to_le32(0xC01E034F) // -EIO +#define STATUS_GRAPHICS_UNASSIGNED_MODESET_ALREADY_EXISTS cpu_to_le32(0xC01E0350) // -EIO +#define STATUS_GRAPHICS_INVALID_SCANLINE_ORDERING cpu_to_le32(0xC01E0352) // -EIO +#define STATUS_GRAPHICS_TOPOLOGY_CHANGES_NOT_ALLOWED cpu_to_le32(0xC01E0353) // -EIO +#define STATUS_GRAPHICS_NO_AVAILABLE_IMPORTANCE_ORDINALS cpu_to_le32(0xC01E0354) // -EIO +#define STATUS_GRAPHICS_INCOMPATIBLE_PRIVATE_FORMAT cpu_to_le32(0xC01E0355) // -EIO +#define STATUS_GRAPHICS_INVALID_MODE_PRUNING_ALGORITHM cpu_to_le32(0xC01E0356) // -EIO +#define STATUS_GRAPHICS_INVALID_MONITOR_CAPABILITY_ORIGIN cpu_to_le32(0xC01E0357) // -EIO +#define STATUS_GRAPHICS_INVALID_MONITOR_FREQUENCYRANGE_CONSTRAINT cpu_to_le32(0xC01E0358) // -EIO +#define STATUS_GRAPHICS_MAX_NUM_PATHS_REACHED cpu_to_le32(0xC01E0359) // -EIO +#define STATUS_GRAPHICS_CANCEL_VIDPN_TOPOLOGY_AUGMENTATION cpu_to_le32(0xC01E035A) // -EIO +#define STATUS_GRAPHICS_INVALID_CLIENT_TYPE cpu_to_le32(0xC01E035B) // -EIO +#define STATUS_GRAPHICS_CLIENTVIDPN_NOT_SET cpu_to_le32(0xC01E035C) // -EIO +#define STATUS_GRAPHICS_SPECIFIED_CHILD_ALREADY_CONNECTED cpu_to_le32(0xC01E0400) // -EIO +#define STATUS_GRAPHICS_CHILD_DESCRIPTOR_NOT_SUPPORTED cpu_to_le32(0xC01E0401) // -EIO +#define STATUS_GRAPHICS_NOT_A_LINKED_ADAPTER cpu_to_le32(0xC01E0430) // -EIO +#define STATUS_GRAPHICS_LEADLINK_NOT_ENUMERATED cpu_to_le32(0xC01E0431) // -EIO +#define STATUS_GRAPHICS_CHAINLINKS_NOT_ENUMERATED cpu_to_le32(0xC01E0432) // -EIO +#define STATUS_GRAPHICS_ADAPTER_CHAIN_NOT_READY cpu_to_le32(0xC01E0433) // -EIO +#define STATUS_GRAPHICS_CHAINLINKS_NOT_STARTED cpu_to_le32(0xC01E0434) // -EIO +#define STATUS_GRAPHICS_CHAINLINKS_NOT_POWERED_ON cpu_to_le32(0xC01E0435) // -EIO +#define STATUS_GRAPHICS_INCONSISTENT_DEVICE_LINK_STATE cpu_to_le32(0xC01E0436) // -EIO +#define STATUS_GRAPHICS_NOT_POST_DEVICE_DRIVER cpu_to_le32(0xC01E0438) // -EIO +#define STATUS_GRAPHICS_ADAPTER_ACCESS_NOT_EXCLUDED cpu_to_le32(0xC01E043B) // -EIO +#define STATUS_GRAPHICS_OPM_PROTECTED_OUTPUT_DOES_NOT_HAVE_COPP_SEMANTICS cpu_to_le32(0xC01E051C) // -EIO +#define STATUS_GRAPHICS_OPM_INVALID_INFORMATION_REQUEST cpu_to_le32(0xC01E051D) // -EIO +#define STATUS_GRAPHICS_OPM_DRIVER_INTERNAL_ERROR cpu_to_le32(0xC01E051E) // -EIO +#define STATUS_GRAPHICS_OPM_PROTECTED_OUTPUT_DOES_NOT_HAVE_OPM_SEMANTICS cpu_to_le32(0xC01E051F) // -EIO +#define STATUS_GRAPHICS_OPM_SIGNALING_NOT_SUPPORTED cpu_to_le32(0xC01E0520) // -EIO +#define STATUS_GRAPHICS_OPM_INVALID_CONFIGURATION_REQUEST cpu_to_le32(0xC01E0521) // -EIO +#define STATUS_GRAPHICS_OPM_NOT_SUPPORTED cpu_to_le32(0xC01E0500) // -EIO +#define STATUS_GRAPHICS_COPP_NOT_SUPPORTED cpu_to_le32(0xC01E0501) // -EIO +#define STATUS_GRAPHICS_UAB_NOT_SUPPORTED cpu_to_le32(0xC01E0502) // -EIO +#define STATUS_GRAPHICS_OPM_INVALID_ENCRYPTED_PARAMETERS cpu_to_le32(0xC01E0503) // -EIO +#define STATUS_GRAPHICS_OPM_PARAMETER_ARRAY_TOO_SMALL cpu_to_le32(0xC01E0504) // -EIO +#define STATUS_GRAPHICS_OPM_NO_PROTECTED_OUTPUTS_EXIST cpu_to_le32(0xC01E0505) // -EIO +#define STATUS_GRAPHICS_PVP_NO_DISPLAY_DEVICE_CORRESPONDS_TO_NAME cpu_to_le32(0xC01E0506) // -EIO +#define STATUS_GRAPHICS_PVP_DISPLAY_DEVICE_NOT_ATTACHED_TO_DESKTOP cpu_to_le32(0xC01E0507) // -EIO +#define STATUS_GRAPHICS_PVP_MIRRORING_DEVICES_NOT_SUPPORTED cpu_to_le32(0xC01E0508) // -EIO +#define STATUS_GRAPHICS_OPM_INVALID_POINTER cpu_to_le32(0xC01E050A) // -EIO +#define STATUS_GRAPHICS_OPM_INTERNAL_ERROR cpu_to_le32(0xC01E050B) // -EIO +#define STATUS_GRAPHICS_OPM_INVALID_HANDLE cpu_to_le32(0xC01E050C) // -EIO +#define STATUS_GRAPHICS_PVP_NO_MONITORS_CORRESPOND_TO_DISPLAY_DEVICE cpu_to_le32(0xC01E050D) // -EIO +#define STATUS_GRAPHICS_PVP_INVALID_CERTIFICATE_LENGTH cpu_to_le32(0xC01E050E) // -EIO +#define STATUS_GRAPHICS_OPM_SPANNING_MODE_ENABLED cpu_to_le32(0xC01E050F) // -EIO +#define STATUS_GRAPHICS_OPM_THEATER_MODE_ENABLED cpu_to_le32(0xC01E0510) // -EIO +#define STATUS_GRAPHICS_PVP_HFS_FAILED cpu_to_le32(0xC01E0511) // -EIO +#define STATUS_GRAPHICS_OPM_INVALID_SRM cpu_to_le32(0xC01E0512) // -EIO +#define STATUS_GRAPHICS_OPM_OUTPUT_DOES_NOT_SUPPORT_HDCP cpu_to_le32(0xC01E0513) // -EIO +#define STATUS_GRAPHICS_OPM_OUTPUT_DOES_NOT_SUPPORT_ACP cpu_to_le32(0xC01E0514) // -EIO +#define STATUS_GRAPHICS_OPM_OUTPUT_DOES_NOT_SUPPORT_CGMSA cpu_to_le32(0xC01E0515) // -EIO +#define STATUS_GRAPHICS_OPM_HDCP_SRM_NEVER_SET cpu_to_le32(0xC01E0516) // -EIO +#define STATUS_GRAPHICS_OPM_RESOLUTION_TOO_HIGH cpu_to_le32(0xC01E0517) // -EIO +#define STATUS_GRAPHICS_OPM_ALL_HDCP_HARDWARE_ALREADY_IN_USE cpu_to_le32(0xC01E0518) // -EIO +#define STATUS_GRAPHICS_OPM_PROTECTED_OUTPUT_NO_LONGER_EXISTS cpu_to_le32(0xC01E051A) // -EIO +#define STATUS_GRAPHICS_OPM_SESSION_TYPE_CHANGE_IN_PROGRESS cpu_to_le32(0xC01E051B) // -EIO +#define STATUS_GRAPHICS_I2C_NOT_SUPPORTED cpu_to_le32(0xC01E0580) // -EIO +#define STATUS_GRAPHICS_I2C_DEVICE_DOES_NOT_EXIST cpu_to_le32(0xC01E0581) // -EIO +#define STATUS_GRAPHICS_I2C_ERROR_TRANSMITTING_DATA cpu_to_le32(0xC01E0582) // -EIO +#define STATUS_GRAPHICS_I2C_ERROR_RECEIVING_DATA cpu_to_le32(0xC01E0583) // -EIO +#define STATUS_GRAPHICS_DDCCI_VCP_NOT_SUPPORTED cpu_to_le32(0xC01E0584) // -EIO +#define STATUS_GRAPHICS_DDCCI_INVALID_DATA cpu_to_le32(0xC01E0585) // -EIO +#define STATUS_GRAPHICS_DDCCI_MONITOR_RETURNED_INVALID_TIMING_STATUS_BYTE cpu_to_le32(0xC01E0586) // -EIO +#define STATUS_GRAPHICS_DDCCI_INVALID_CAPABILITIES_STRING cpu_to_le32(0xC01E0587) // -EIO +#define STATUS_GRAPHICS_MCA_INTERNAL_ERROR cpu_to_le32(0xC01E0588) // -EIO +#define STATUS_GRAPHICS_DDCCI_INVALID_MESSAGE_COMMAND cpu_to_le32(0xC01E0589) // -EIO +#define STATUS_GRAPHICS_DDCCI_INVALID_MESSAGE_LENGTH cpu_to_le32(0xC01E058A) // -EIO +#define STATUS_GRAPHICS_DDCCI_INVALID_MESSAGE_CHECKSUM cpu_to_le32(0xC01E058B) // -EIO +#define STATUS_GRAPHICS_INVALID_PHYSICAL_MONITOR_HANDLE cpu_to_le32(0xC01E058C) // -EIO +#define STATUS_GRAPHICS_MONITOR_NO_LONGER_EXISTS cpu_to_le32(0xC01E058D) // -EIO +#define STATUS_GRAPHICS_ONLY_CONSOLE_SESSION_SUPPORTED cpu_to_le32(0xC01E05E0) // -EIO +#define STATUS_GRAPHICS_NO_DISPLAY_DEVICE_CORRESPONDS_TO_NAME cpu_to_le32(0xC01E05E1) // -EIO +#define STATUS_GRAPHICS_DISPLAY_DEVICE_NOT_ATTACHED_TO_DESKTOP cpu_to_le32(0xC01E05E2) // -EIO +#define STATUS_GRAPHICS_MIRRORING_DEVICES_NOT_SUPPORTED cpu_to_le32(0xC01E05E3) // -EIO +#define STATUS_GRAPHICS_INVALID_POINTER cpu_to_le32(0xC01E05E4) // -EIO +#define STATUS_GRAPHICS_NO_MONITORS_CORRESPOND_TO_DISPLAY_DEVICE cpu_to_le32(0xC01E05E5) // -EIO +#define STATUS_GRAPHICS_PARAMETER_ARRAY_TOO_SMALL cpu_to_le32(0xC01E05E6) // -EIO +#define STATUS_GRAPHICS_INTERNAL_ERROR cpu_to_le32(0xC01E05E7) // -EIO +#define STATUS_GRAPHICS_SESSION_TYPE_CHANGE_IN_PROGRESS cpu_to_le32(0xC01E05E8) // -EIO +#define STATUS_FVE_LOCKED_VOLUME cpu_to_le32(0xC0210000) // -EIO +#define STATUS_FVE_NOT_ENCRYPTED cpu_to_le32(0xC0210001) // -EIO +#define STATUS_FVE_BAD_INFORMATION cpu_to_le32(0xC0210002) // -EIO +#define STATUS_FVE_TOO_SMALL cpu_to_le32(0xC0210003) // -EIO +#define STATUS_FVE_FAILED_WRONG_FS cpu_to_le32(0xC0210004) // -EIO +#define STATUS_FVE_FAILED_BAD_FS cpu_to_le32(0xC0210005) // -EIO +#define STATUS_FVE_FS_NOT_EXTENDED cpu_to_le32(0xC0210006) // -EIO +#define STATUS_FVE_FS_MOUNTED cpu_to_le32(0xC0210007) // -EIO +#define STATUS_FVE_NO_LICENSE cpu_to_le32(0xC0210008) // -EIO +#define STATUS_FVE_ACTION_NOT_ALLOWED cpu_to_le32(0xC0210009) // -EIO +#define STATUS_FVE_BAD_DATA cpu_to_le32(0xC021000A) // -EIO +#define STATUS_FVE_VOLUME_NOT_BOUND cpu_to_le32(0xC021000B) // -EIO +#define STATUS_FVE_NOT_DATA_VOLUME cpu_to_le32(0xC021000C) // -EIO +#define STATUS_FVE_CONV_READ_ERROR cpu_to_le32(0xC021000D) // -EIO +#define STATUS_FVE_CONV_WRITE_ERROR cpu_to_le32(0xC021000E) // -EIO +#define STATUS_FVE_OVERLAPPED_UPDATE cpu_to_le32(0xC021000F) // -EIO +#define STATUS_FVE_FAILED_SECTOR_SIZE cpu_to_le32(0xC0210010) // -EIO +#define STATUS_FVE_FAILED_AUTHENTICATION cpu_to_le32(0xC0210011) // -EIO +#define STATUS_FVE_NOT_OS_VOLUME cpu_to_le32(0xC0210012) // -EIO +#define STATUS_FVE_KEYFILE_NOT_FOUND cpu_to_le32(0xC0210013) // -EIO +#define STATUS_FVE_KEYFILE_INVALID cpu_to_le32(0xC0210014) // -EIO +#define STATUS_FVE_KEYFILE_NO_VMK cpu_to_le32(0xC0210015) // -EIO +#define STATUS_FVE_TPM_DISABLED cpu_to_le32(0xC0210016) // -EIO +#define STATUS_FVE_TPM_SRK_AUTH_NOT_ZERO cpu_to_le32(0xC0210017) // -EIO +#define STATUS_FVE_TPM_INVALID_PCR cpu_to_le32(0xC0210018) // -EIO +#define STATUS_FVE_TPM_NO_VMK cpu_to_le32(0xC0210019) // -EIO +#define STATUS_FVE_PIN_INVALID cpu_to_le32(0xC021001A) // -EIO +#define STATUS_FVE_AUTH_INVALID_APPLICATION cpu_to_le32(0xC021001B) // -EIO +#define STATUS_FVE_AUTH_INVALID_CONFIG cpu_to_le32(0xC021001C) // -EIO +#define STATUS_FVE_DEBUGGER_ENABLED cpu_to_le32(0xC021001D) // -EIO +#define STATUS_FVE_DRY_RUN_FAILED cpu_to_le32(0xC021001E) // -EIO +#define STATUS_FVE_BAD_METADATA_POINTER cpu_to_le32(0xC021001F) // -EIO +#define STATUS_FVE_OLD_METADATA_COPY cpu_to_le32(0xC0210020) // -EIO +#define STATUS_FVE_REBOOT_REQUIRED cpu_to_le32(0xC0210021) // -EIO +#define STATUS_FVE_RAW_ACCESS cpu_to_le32(0xC0210022) // -EIO +#define STATUS_FVE_RAW_BLOCKED cpu_to_le32(0xC0210023) // -EIO +#define STATUS_FWP_CALLOUT_NOT_FOUND cpu_to_le32(0xC0220001) // -EIO +#define STATUS_FWP_CONDITION_NOT_FOUND cpu_to_le32(0xC0220002) // -EIO +#define STATUS_FWP_FILTER_NOT_FOUND cpu_to_le32(0xC0220003) // -EIO +#define STATUS_FWP_LAYER_NOT_FOUND cpu_to_le32(0xC0220004) // -EIO +#define STATUS_FWP_PROVIDER_NOT_FOUND cpu_to_le32(0xC0220005) // -EIO +#define STATUS_FWP_PROVIDER_CONTEXT_NOT_FOUND cpu_to_le32(0xC0220006) // -EIO +#define STATUS_FWP_SUBLAYER_NOT_FOUND cpu_to_le32(0xC0220007) // -EIO +#define STATUS_FWP_NOT_FOUND cpu_to_le32(0xC0220008) // -EIO +#define STATUS_FWP_ALREADY_EXISTS cpu_to_le32(0xC0220009) // -EIO +#define STATUS_FWP_IN_USE cpu_to_le32(0xC022000A) // -EIO +#define STATUS_FWP_DYNAMIC_SESSION_IN_PROGRESS cpu_to_le32(0xC022000B) // -EIO +#define STATUS_FWP_WRONG_SESSION cpu_to_le32(0xC022000C) // -EIO +#define STATUS_FWP_NO_TXN_IN_PROGRESS cpu_to_le32(0xC022000D) // -EIO +#define STATUS_FWP_TXN_IN_PROGRESS cpu_to_le32(0xC022000E) // -EIO +#define STATUS_FWP_TXN_ABORTED cpu_to_le32(0xC022000F) // -EIO +#define STATUS_FWP_SESSION_ABORTED cpu_to_le32(0xC0220010) // -EIO +#define STATUS_FWP_INCOMPATIBLE_TXN cpu_to_le32(0xC0220011) // -EIO +#define STATUS_FWP_TIMEOUT cpu_to_le32(0xC0220012) // -ETIMEDOUT +#define STATUS_FWP_NET_EVENTS_DISABLED cpu_to_le32(0xC0220013) // -EIO +#define STATUS_FWP_INCOMPATIBLE_LAYER cpu_to_le32(0xC0220014) // -EIO +#define STATUS_FWP_KM_CLIENTS_ONLY cpu_to_le32(0xC0220015) // -EIO +#define STATUS_FWP_LIFETIME_MISMATCH cpu_to_le32(0xC0220016) // -EIO +#define STATUS_FWP_BUILTIN_OBJECT cpu_to_le32(0xC0220017) // -EIO +#define STATUS_FWP_TOO_MANY_BOOTTIME_FILTERS cpu_to_le32(0xC0220018) // -EIO +#define STATUS_FWP_TOO_MANY_CALLOUTS cpu_to_le32(0xC0220018) // -EIO +#define STATUS_FWP_NOTIFICATION_DROPPED cpu_to_le32(0xC0220019) // -EIO +#define STATUS_FWP_TRAFFIC_MISMATCH cpu_to_le32(0xC022001A) // -EIO +#define STATUS_FWP_INCOMPATIBLE_SA_STATE cpu_to_le32(0xC022001B) // -EIO +#define STATUS_FWP_NULL_POINTER cpu_to_le32(0xC022001C) // -EIO +#define STATUS_FWP_INVALID_ENUMERATOR cpu_to_le32(0xC022001D) // -EIO +#define STATUS_FWP_INVALID_FLAGS cpu_to_le32(0xC022001E) // -EIO +#define STATUS_FWP_INVALID_NET_MASK cpu_to_le32(0xC022001F) // -EIO +#define STATUS_FWP_INVALID_RANGE cpu_to_le32(0xC0220020) // -EIO +#define STATUS_FWP_INVALID_INTERVAL cpu_to_le32(0xC0220021) // -EIO +#define STATUS_FWP_ZERO_LENGTH_ARRAY cpu_to_le32(0xC0220022) // -EIO +#define STATUS_FWP_NULL_DISPLAY_NAME cpu_to_le32(0xC0220023) // -EIO +#define STATUS_FWP_INVALID_ACTION_TYPE cpu_to_le32(0xC0220024) // -EIO +#define STATUS_FWP_INVALID_WEIGHT cpu_to_le32(0xC0220025) // -EIO +#define STATUS_FWP_MATCH_TYPE_MISMATCH cpu_to_le32(0xC0220026) // -EIO +#define STATUS_FWP_TYPE_MISMATCH cpu_to_le32(0xC0220027) // -EIO +#define STATUS_FWP_OUT_OF_BOUNDS cpu_to_le32(0xC0220028) // -EIO +#define STATUS_FWP_RESERVED cpu_to_le32(0xC0220029) // -EIO +#define STATUS_FWP_DUPLICATE_CONDITION cpu_to_le32(0xC022002A) // -EIO +#define STATUS_FWP_DUPLICATE_KEYMOD cpu_to_le32(0xC022002B) // -EIO +#define STATUS_FWP_ACTION_INCOMPATIBLE_WITH_LAYER cpu_to_le32(0xC022002C) // -EIO +#define STATUS_FWP_ACTION_INCOMPATIBLE_WITH_SUBLAYER cpu_to_le32(0xC022002D) // -EIO +#define STATUS_FWP_CONTEXT_INCOMPATIBLE_WITH_LAYER cpu_to_le32(0xC022002E) // -EIO +#define STATUS_FWP_CONTEXT_INCOMPATIBLE_WITH_CALLOUT cpu_to_le32(0xC022002F) // -EIO +#define STATUS_FWP_INCOMPATIBLE_AUTH_METHOD cpu_to_le32(0xC0220030) // -EIO +#define STATUS_FWP_INCOMPATIBLE_DH_GROUP cpu_to_le32(0xC0220031) // -EIO +#define STATUS_FWP_EM_NOT_SUPPORTED cpu_to_le32(0xC0220032) // -EOPNOTSUPP +#define STATUS_FWP_NEVER_MATCH cpu_to_le32(0xC0220033) // -EIO +#define STATUS_FWP_PROVIDER_CONTEXT_MISMATCH cpu_to_le32(0xC0220034) // -EIO +#define STATUS_FWP_INVALID_PARAMETER cpu_to_le32(0xC0220035) // -EIO +#define STATUS_FWP_TOO_MANY_SUBLAYERS cpu_to_le32(0xC0220036) // -EIO +#define STATUS_FWP_CALLOUT_NOTIFICATION_FAILED cpu_to_le32(0xC0220037) // -EIO +#define STATUS_FWP_INCOMPATIBLE_AUTH_CONFIG cpu_to_le32(0xC0220038) // -EIO +#define STATUS_FWP_INCOMPATIBLE_CIPHER_CONFIG cpu_to_le32(0xC0220039) // -EIO +#define STATUS_FWP_TCPIP_NOT_READY cpu_to_le32(0xC0220100) // -EIO +#define STATUS_FWP_INJECT_HANDLE_CLOSING cpu_to_le32(0xC0220101) // -EIO +#define STATUS_FWP_INJECT_HANDLE_STALE cpu_to_le32(0xC0220102) // -EIO +#define STATUS_FWP_CANNOT_PEND cpu_to_le32(0xC0220103) // -EIO +#define STATUS_NDIS_CLOSING cpu_to_le32(0xC0230002) // -EIO +#define STATUS_NDIS_BAD_VERSION cpu_to_le32(0xC0230004) // -EIO +#define STATUS_NDIS_BAD_CHARACTERISTICS cpu_to_le32(0xC0230005) // -EIO +#define STATUS_NDIS_ADAPTER_NOT_FOUND cpu_to_le32(0xC0230006) // -EIO +#define STATUS_NDIS_OPEN_FAILED cpu_to_le32(0xC0230007) // -EIO +#define STATUS_NDIS_DEVICE_FAILED cpu_to_le32(0xC0230008) // -EIO +#define STATUS_NDIS_MULTICAST_FULL cpu_to_le32(0xC0230009) // -EIO +#define STATUS_NDIS_MULTICAST_EXISTS cpu_to_le32(0xC023000A) // -EIO +#define STATUS_NDIS_MULTICAST_NOT_FOUND cpu_to_le32(0xC023000B) // -EIO +#define STATUS_NDIS_REQUEST_ABORTED cpu_to_le32(0xC023000C) // -EIO +#define STATUS_NDIS_RESET_IN_PROGRESS cpu_to_le32(0xC023000D) // -EIO +#define STATUS_NDIS_INVALID_PACKET cpu_to_le32(0xC023000F) // -EIO +#define STATUS_NDIS_INVALID_DEVICE_REQUEST cpu_to_le32(0xC0230010) // -EIO +#define STATUS_NDIS_ADAPTER_NOT_READY cpu_to_le32(0xC0230011) // -EIO +#define STATUS_NDIS_INVALID_LENGTH cpu_to_le32(0xC0230014) // -EIO +#define STATUS_NDIS_INVALID_DATA cpu_to_le32(0xC0230015) // -EIO +#define STATUS_NDIS_BUFFER_TOO_SHORT cpu_to_le32(0xC0230016) // -ENOBUFS +#define STATUS_NDIS_INVALID_OID cpu_to_le32(0xC0230017) // -EIO +#define STATUS_NDIS_ADAPTER_REMOVED cpu_to_le32(0xC0230018) // -EIO +#define STATUS_NDIS_UNSUPPORTED_MEDIA cpu_to_le32(0xC0230019) // -EIO +#define STATUS_NDIS_GROUP_ADDRESS_IN_USE cpu_to_le32(0xC023001A) // -EIO +#define STATUS_NDIS_FILE_NOT_FOUND cpu_to_le32(0xC023001B) // -EIO +#define STATUS_NDIS_ERROR_READING_FILE cpu_to_le32(0xC023001C) // -EIO +#define STATUS_NDIS_ALREADY_MAPPED cpu_to_le32(0xC023001D) // -EIO +#define STATUS_NDIS_RESOURCE_CONFLICT cpu_to_le32(0xC023001E) // -EIO +#define STATUS_NDIS_MEDIA_DISCONNECTED cpu_to_le32(0xC023001F) // -EIO +#define STATUS_NDIS_INVALID_ADDRESS cpu_to_le32(0xC0230022) // -EIO +#define STATUS_NDIS_PAUSED cpu_to_le32(0xC023002A) // -EIO +#define STATUS_NDIS_INTERFACE_NOT_FOUND cpu_to_le32(0xC023002B) // -EIO +#define STATUS_NDIS_UNSUPPORTED_REVISION cpu_to_le32(0xC023002C) // -EIO +#define STATUS_NDIS_INVALID_PORT cpu_to_le32(0xC023002D) // -EIO +#define STATUS_NDIS_INVALID_PORT_STATE cpu_to_le32(0xC023002E) // -EIO +#define STATUS_NDIS_LOW_POWER_STATE cpu_to_le32(0xC023002F) // -EIO +#define STATUS_NDIS_NOT_SUPPORTED cpu_to_le32(0xC02300BB) // -ENOSYS +#define STATUS_NDIS_DOT11_AUTO_CONFIG_ENABLED cpu_to_le32(0xC0232000) // -EIO +#define STATUS_NDIS_DOT11_MEDIA_IN_USE cpu_to_le32(0xC0232001) // -EIO +#define STATUS_NDIS_DOT11_POWER_STATE_INVALID cpu_to_le32(0xC0232002) // -EIO +#define STATUS_IPSEC_BAD_SPI cpu_to_le32(0xC0360001) // -EIO +#define STATUS_IPSEC_SA_LIFETIME_EXPIRED cpu_to_le32(0xC0360002) // -EIO +#define STATUS_IPSEC_WRONG_SA cpu_to_le32(0xC0360003) // -EIO +#define STATUS_IPSEC_REPLAY_CHECK_FAILED cpu_to_le32(0xC0360004) // -EIO +#define STATUS_IPSEC_INVALID_PACKET cpu_to_le32(0xC0360005) // -EIO +#define STATUS_IPSEC_INTEGRITY_CHECK_FAILED cpu_to_le32(0xC0360006) // -EIO +#define STATUS_IPSEC_CLEAR_TEXT_DROP cpu_to_le32(0xC0360007) // -EIO +/* See MS-SMB2 3.3.5.4 */ +#define STATUS_SMB_NO_PREAUTH_INTEGRITY_HASH_OVERLAP cpu_to_le32(0xC05D0000) // -EIO + +#endif /* FS_SMB_COMMON_SMB2STATUS_H */ diff --git a/fs/smb/common/smbacl.h b/fs/smb/common/smbacl.h new file mode 100644 index 000000000..70bba5ff7 --- /dev/null +++ b/fs/smb/common/smbacl.h @@ -0,0 +1,122 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ +/* + * Copyright (c) International Business Machines Corp., 2007 + * Author(s): Steve French (sfrench@us.ibm.com) + * Modified by Namjae Jeon (linkinjeon@kernel.org) + */ + +#ifndef _COMMON_SMBACL_H +#define _COMMON_SMBACL_H + +#define NUM_AUTHS (6) /* number of authority fields */ +#define SID_MAX_SUB_AUTHORITIES (15) /* max number of sub authority fields */ + +/* ACE types - see MS-DTYP 2.4.4.1 */ +#define ACCESS_ALLOWED_ACE_TYPE 0x00 +#define ACCESS_DENIED_ACE_TYPE 0x01 +#define SYSTEM_AUDIT_ACE_TYPE 0x02 +#define SYSTEM_ALARM_ACE_TYPE 0x03 +#define ACCESS_ALLOWED_COMPOUND_ACE_TYPE 0x04 +#define ACCESS_ALLOWED_OBJECT_ACE_TYPE 0x05 +#define ACCESS_DENIED_OBJECT_ACE_TYPE 0x06 +#define SYSTEM_AUDIT_OBJECT_ACE_TYPE 0x07 +#define SYSTEM_ALARM_OBJECT_ACE_TYPE 0x08 +#define ACCESS_ALLOWED_CALLBACK_ACE_TYPE 0x09 +#define ACCESS_DENIED_CALLBACK_ACE_TYPE 0x0A +#define ACCESS_ALLOWED_CALLBACK_OBJECT_ACE_TYPE 0x0B +#define ACCESS_DENIED_CALLBACK_OBJECT_ACE_TYPE 0x0C +#define SYSTEM_AUDIT_CALLBACK_ACE_TYPE 0x0D +#define SYSTEM_ALARM_CALLBACK_ACE_TYPE 0x0E /* Reserved */ +#define SYSTEM_AUDIT_CALLBACK_OBJECT_ACE_TYPE 0x0F +#define SYSTEM_ALARM_CALLBACK_OBJECT_ACE_TYPE 0x10 /* reserved */ +#define SYSTEM_MANDATORY_LABEL_ACE_TYPE 0x11 +#define SYSTEM_RESOURCE_ATTRIBUTE_ACE_TYPE 0x12 +#define SYSTEM_SCOPED_POLICY_ID_ACE_TYPE 0x13 + +/* ACE flags */ +#define OBJECT_INHERIT_ACE 0x01 +#define CONTAINER_INHERIT_ACE 0x02 +#define NO_PROPAGATE_INHERIT_ACE 0x04 +#define INHERIT_ONLY_ACE 0x08 +#define INHERITED_ACE 0x10 +#define SUCCESSFUL_ACCESS_ACE_FLAG 0x40 +#define FAILED_ACCESS_ACE_FLAG 0x80 + +/* + * Maximum size of a string representation of a SID: + * + * The fields are unsigned values in decimal. So: + * + * u8: max 3 bytes in decimal + * u32: max 10 bytes in decimal + * + * "S-" + 3 bytes for version field + 15 for authority field + NULL terminator + * + * For authority field, max is when all 6 values are non-zero and it must be + * represented in hex. So "-0x" + 12 hex digits. + * + * Add 11 bytes for each subauthority field (10 bytes each + 1 for '-') + */ +#define SID_STRING_BASE_SIZE (2 + 3 + 15 + 1) +#define SID_STRING_SUBAUTH_SIZE (11) /* size of a single subauth string */ + +#define DOMAIN_USER_RID_LE cpu_to_le32(513) + +/* + * ACE types - see MS-DTYP 2.4.4.1 + */ +enum { + ACCESS_ALLOWED, + ACCESS_DENIED, +}; + +/* + * Security ID types + */ +enum { + SIDOWNER = 1, + SIDGROUP, + SIDCREATOR_OWNER, + SIDCREATOR_GROUP, + SIDUNIX_USER, + SIDUNIX_GROUP, + SIDNFS_USER, + SIDNFS_GROUP, + SIDNFS_MODE, +}; + +struct smb_ntsd { + __le16 revision; /* revision level */ + __le16 type; + __le32 osidoffset; + __le32 gsidoffset; + __le32 sacloffset; + __le32 dacloffset; +} __packed; + +struct smb_sid { + __u8 revision; /* revision level */ + __u8 num_subauth; + __u8 authority[NUM_AUTHS]; + __le32 sub_auth[SID_MAX_SUB_AUTHORITIES]; /* sub_auth[num_subauth] */ +} __packed; + +/* size of a struct smb_sid, sans sub_auth array */ +#define CIFS_SID_BASE_SIZE (1 + 1 + NUM_AUTHS) + +struct smb_acl { + __le16 revision; /* revision level */ + __le16 size; + __le16 num_aces; + __le16 reserved; +} __packed; + +struct smb_ace { + __u8 type; /* see above and MS-DTYP 2.4.4.1 */ + __u8 flags; + __le16 size; + __le32 access_req; + struct smb_sid sid; /* ie UUID of user or group who gets these perms */ +} __packed; + +#endif /* _COMMON_SMBACL_H */ diff --git a/fs/smb/common/smbfsctl.h b/fs/smb/common/smbfsctl.h new file mode 100644 index 000000000..b1123b0f7 --- /dev/null +++ b/fs/smb/common/smbfsctl.h @@ -0,0 +1,175 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ +/* + * SMB, CIFS, SMB2 FSCTL definitions + * + * Copyright (c) International Business Machines Corp., 2002,2013 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ + +/* IOCTL information */ +/* + * List of ioctl/fsctl function codes that are or could be useful in the + * future to remote clients like cifs or SMB2/SMB3 client. This is probably + * a slightly larger set of fsctls that NTFS local filesystem could handle, + * including the seven below that we do not have struct definitions for. + * Even with protocol definitions for most of these now available, we still + * need to do some experimentation to identify which are practical to do + * remotely. Some of the following, such as the encryption/compression ones + * could be invoked from tools via a specialized hook into the VFS rather + * than via the standard vfs entry points + * + * See MS-SMB2 Section 2.2.31 (last checked September 2021, all of that list are + * below). Additional detail on less common ones can be found in MS-FSCC + * section 2.3. + */ + +#ifndef __SMBFSCTL_H +#define __SMBFSCTL_H + +/* + * FSCTL values are 32 bits and are constructed as + * + */ +/* Device */ +#define FSCTL_DEVICE_DFS (0x0006 << 16) +#define FSCTL_DEVICE_FILE_SYSTEM (0x0009 << 16) +#define FSCTL_DEVICE_NAMED_PIPE (0x0011 << 16) +#define FSCTL_DEVICE_NETWORK_FILE_SYSTEM (0x0014 << 16) +#define FSCTL_DEVICE_MASK 0xffff0000 +/* Access */ +#define FSCTL_DEVICE_ACCESS_FILE_ANY_ACCESS (0x00 << 14) +#define FSCTL_DEVICE_ACCESS_FILE_READ_ACCESS (0x01 << 14) +#define FSCTL_DEVICE_ACCESS_FILE_WRITE_ACCESS (0x02 << 14) +#define FSCTL_DEVICE_ACCESS_FILE_READ_WRITE_ACCESS (0x03 << 14) +#define FSCTL_DEVICE_ACCESS_MASK 0x0000c000 +/* Function */ +#define FSCTL_DEVICE_FUNCTION_MASK 0x00003ffc +/* Method */ +#define FSCTL_DEVICE_METHOD_BUFFERED 0x00 +#define FSCTL_DEVICE_METHOD_IN_DIRECT 0x01 +#define FSCTL_DEVICE_METHOD_OUT_DIRECT 0x02 +#define FSCTL_DEVICE_METHOD_NEITHER 0x03 +#define FSCTL_DEVICE_METHOD_MASK 0x00000003 + + +#define FSCTL_DFS_GET_REFERRALS 0x00060194 +#define FSCTL_DFS_GET_REFERRALS_EX 0x000601B0 +#define FSCTL_REQUEST_OPLOCK_LEVEL_1 0x00090000 +#define FSCTL_REQUEST_OPLOCK_LEVEL_2 0x00090004 +#define FSCTL_REQUEST_BATCH_OPLOCK 0x00090008 +#define FSCTL_LOCK_VOLUME 0x00090018 +#define FSCTL_UNLOCK_VOLUME 0x0009001C +#define FSCTL_IS_PATHNAME_VALID 0x0009002C /* BB add struct */ +#define FSCTL_GET_COMPRESSION 0x0009003C +#define FSCTL_SET_COMPRESSION 0x0009C040 +#define FSCTL_QUERY_FAT_BPB 0x00090058 /* BB add struct */ +/* Verify the next FSCTL number, we had it as 0x00090090 before */ +#define FSCTL_FILESYSTEM_GET_STATS 0x00090060 /* BB add struct */ +#define FSCTL_GET_NTFS_VOLUME_DATA 0x00090064 /* BB add struct */ +#define FSCTL_GET_RETRIEVAL_POINTERS 0x00090073 /* BB add struct */ +#define FSCTL_IS_VOLUME_DIRTY 0x00090078 /* BB add struct */ +#define FSCTL_ALLOW_EXTENDED_DASD_IO 0x00090083 /* BB add struct */ +#define FSCTL_REQUEST_FILTER_OPLOCK 0x0009008C +#define FSCTL_FIND_FILES_BY_SID 0x0009008F /* BB add struct */ +#define FSCTL_SET_OBJECT_ID 0x00090098 /* BB add struct */ +#define FSCTL_GET_OBJECT_ID 0x0009009C /* BB add struct */ +#define FSCTL_DELETE_OBJECT_ID 0x000900A0 /* BB add struct */ +#define FSCTL_SET_REPARSE_POINT 0x000900A4 /* BB add struct */ +#define FSCTL_GET_REPARSE_POINT 0x000900A8 /* BB add struct */ +#define FSCTL_DELETE_REPARSE_POINT 0x000900AC /* BB add struct */ +#define FSCTL_SET_OBJECT_ID_EXTENDED 0x000900BC /* BB add struct */ +#define FSCTL_CREATE_OR_GET_OBJECT_ID 0x000900C0 /* BB add struct */ +#define FSCTL_SET_SPARSE 0x000900C4 /* BB add struct */ +#define FSCTL_SET_ZERO_DATA 0x000980C8 +#define FSCTL_SET_ENCRYPTION 0x000900D7 /* BB add struct */ +#define FSCTL_ENCRYPTION_FSCTL_IO 0x000900DB /* BB add struct */ +#define FSCTL_WRITE_RAW_ENCRYPTED 0x000900DF /* BB add struct */ +#define FSCTL_READ_RAW_ENCRYPTED 0x000900E3 /* BB add struct */ +#define FSCTL_READ_FILE_USN_DATA 0x000900EB /* BB add struct */ +#define FSCTL_WRITE_USN_CLOSE_RECORD 0x000900EF /* BB add struct */ +#define FSCTL_MARK_HANDLE 0x000900FC /* BB add struct */ +#define FSCTL_SIS_COPYFILE 0x00090100 /* BB add struct */ +#define FSCTL_RECALL_FILE 0x00090117 /* BB add struct */ +#define FSCTL_QUERY_SPARING_INFO 0x00090138 /* BB add struct */ +#define FSCTL_QUERY_ON_DISK_VOLUME_INFO 0x0009013C +#define FSCTL_SET_ZERO_ON_DEALLOC 0x00090194 /* BB add struct */ +#define FSCTL_SET_SHORT_NAME_BEHAVIOR 0x000901B4 /* BB add struct */ +#define FSCTL_GET_INTEGRITY_INFORMATION 0x0009027C +#define FSCTL_QUERY_FILE_REGIONS 0x00090284 +#define FSCTL_GET_REFS_VOLUME_DATA 0x000902D8 /* See MS-FSCC 2.3.24 */ +#define FSCTL_SET_INTEGRITY_INFORMATION_EXT 0x00090380 +#define FSCTL_GET_RETRIEVAL_POINTERS_AND_REFCOUNT 0x000903d3 +#define FSCTL_GET_RETRIEVAL_POINTER_COUNT 0x0009042b +#define FSCTL_REFS_STREAM_SNAPSHOT_MANAGEMENT 0x00090440 +#define FSCTL_QUERY_ALLOCATED_RANGES 0x000940CF +#define FSCTL_OFFLOAD_READ 0x00094264 /* BB add struct */ +#define FSCTL_OFFLOAD_WRITE 0x00098268 /* BB add struct */ +#define FSCTL_SET_DEFECT_MANAGEMENT 0x00098134 /* BB add struct */ +#define FSCTL_FILE_LEVEL_TRIM 0x00098208 /* BB add struct */ +#define FSCTL_DUPLICATE_EXTENTS_TO_FILE 0x00098344 +#define FSCTL_DUPLICATE_EXTENTS_TO_FILE_EX 0x000983E8 +#define FSCTL_SIS_LINK_FILES 0x0009C104 +#define FSCTL_SET_INTEGRITY_INFORMATION 0x0009C280 +#define FSCTL_PIPE_PEEK 0x0011400C /* BB add struct */ +#define FSCTL_PIPE_TRANSCEIVE 0x0011C017 /* BB add struct */ +/* strange that the number for this op is not sequential with previous op */ +#define FSCTL_PIPE_WAIT 0x00110018 /* BB add struct */ +/* Enumerate previous versions of a file */ +#define FSCTL_SRV_ENUMERATE_SNAPSHOTS 0x00144064 +/* Retrieve an opaque file reference for server-side data movement ie copy */ +#define FSCTL_SRV_REQUEST_RESUME_KEY 0x00140078 +#define FSCTL_SRV_ENUM_SNAPS 0x00144064 +#define FSCTL_LMR_REQUEST_RESILIENCY 0x001401D4 +#define FSCTL_LMR_GET_LINK_TRACK_INF 0x001400E8 /* BB add struct */ +#define FSCTL_LMR_SET_LINK_TRACK_INF 0x001400EC /* BB add struct */ +#define FSCTL_VALIDATE_NEGOTIATE_INFO 0x00140204 +/* Perform server-side data movement */ +#define FSCTL_SRV_COPYCHUNK 0x001440F2 +#define FSCTL_SRV_COPYCHUNK_WRITE 0x001480F2 +#define FSCTL_QUERY_NETWORK_INTERFACE_INFO 0x001401FC /* BB add struct */ +#define FSCTL_SRV_READ_HASH 0x001441BB /* BB add struct */ + +/* See FSCC 2.1.2.5 */ +#define IO_REPARSE_TAG_MOUNT_POINT 0xA0000003 +#define IO_REPARSE_TAG_HSM 0xC0000004 +#define IO_REPARSE_TAG_SIS 0x80000007 +#define IO_REPARSE_TAG_HSM2 0x80000006 +#define IO_REPARSE_TAG_DRIVER_EXTENDER 0x80000005 +/* Used by the DFS filter. See MS-DFSC */ +#define IO_REPARSE_TAG_DFS 0x8000000A +/* Used by the DFS filter See MS-DFSC */ +#define IO_REPARSE_TAG_DFSR 0x80000012 +#define IO_REPARSE_TAG_FILTER_MANAGER 0x8000000B +/* Native SMB symlinks since Windows Vista, see MS-FSCC 2.1.2.4 */ +#define IO_REPARSE_TAG_SYMLINK 0xA000000C +#define IO_REPARSE_TAG_DEDUP 0x80000013 +#define IO_REPARSE_APPXSTREAM 0xC0000014 +/* NFS special files used by Windows NFS server since Windows Server 2012, see MS-FSCC 2.1.2.6 */ +#define IO_REPARSE_TAG_NFS 0x80000014 +/* + * AzureFileSync - see + * https://docs.microsoft.com/en-us/azure/storage/files/storage-sync-cloud-tiering + */ +#define IO_REPARSE_TAG_AZ_FILE_SYNC 0x8000001e +/* Native Win32 AF_UNIX sockets since Windows 10 April 2018 Update, used also by WSL */ +#define IO_REPARSE_TAG_AF_UNIX 0x80000023 +/* WSL reparse tags */ +#define IO_REPARSE_TAG_LX_SYMLINK 0xA000001D +#define IO_REPARSE_TAG_LX_FIFO 0x80000024 +#define IO_REPARSE_TAG_LX_CHR 0x80000025 +#define IO_REPARSE_TAG_LX_BLK 0x80000026 + +#define IO_REPARSE_TAG_LX_SYMLINK_LE cpu_to_le32(IO_REPARSE_TAG_LX_SYMLINK) +#define IO_REPARSE_TAG_AF_UNIX_LE cpu_to_le32(IO_REPARSE_TAG_AF_UNIX) +#define IO_REPARSE_TAG_LX_FIFO_LE cpu_to_le32(IO_REPARSE_TAG_LX_FIFO) +#define IO_REPARSE_TAG_LX_CHR_LE cpu_to_le32(IO_REPARSE_TAG_LX_CHR) +#define IO_REPARSE_TAG_LX_BLK_LE cpu_to_le32(IO_REPARSE_TAG_LX_BLK) + +/* If Name Surrogate Bit is set, the file or directory represents another named entity in the system. */ +#define IS_REPARSE_TAG_NAME_SURROGATE(tag) (!!((tag) & 0x20000000)) + +/* fsctl flags */ +/* If Flags is set to this value, the request is an FSCTL not ioctl request */ +#define SMB2_0_IOCTL_IS_FSCTL 0x00000001 +#endif /* __SMBFSCTL_H */ diff --git a/fs/smb/common/smbglob.h b/fs/smb/common/smbglob.h new file mode 100644 index 000000000..d9c7e6e7a --- /dev/null +++ b/fs/smb/common/smbglob.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +/* + * + * Copyright (C) International Business Machines Corp., 2002,2008 + * 2018 Samsung Electronics Co., Ltd. + * Author(s): Steve French (sfrench@us.ibm.com) + * Jeremy Allison (jra@samba.org) + * Namjae Jeon (linkinjeon@kernel.org) + * + */ +#ifndef _COMMON_SMB_GLOB_H +#define _COMMON_SMB_GLOB_H + +struct smb_version_values { + char *version_string; + __u16 protocol_id; + __le16 lock_cmd; + __u32 req_capabilities; + __u32 max_read_size; + __u32 max_write_size; + __u32 max_trans_size; + __u32 max_credits; + __u32 large_lock_type; + __u32 exclusive_lock_type; + __u32 shared_lock_type; + __u32 unlock_lock_type; + size_t header_size; + size_t max_header_size; + size_t read_rsp_size; + unsigned int cap_unix; + unsigned int cap_nt_find; + unsigned int cap_large_files; + unsigned int cap_unicode; + __u16 signing_enabled; + __u16 signing_required; + size_t create_lease_size; + size_t create_durable_size; + size_t create_durable_v2_size; + size_t create_mxac_size; + size_t create_disk_id_size; + size_t create_posix_size; + size_t create_aapl_size; +}; + +static inline unsigned int get_rfc1002_len(void *buf) +{ + return be32_to_cpu(*((__be32 *)buf)) & 0xffffff; +} + +static inline void inc_rfc1001_len(void *buf, int count) +{ + be32_add_cpu((__be32 *)buf, count); +} + +#define SMB1_VERSION_STRING "1.0" +#define SMB20_VERSION_STRING "2.0" +#define SMB21_VERSION_STRING "2.1" +#define SMBDEFAULT_VERSION_STRING "default" +#define SMB3ANY_VERSION_STRING "3" +#define SMB30_VERSION_STRING "3.0" +#define SMB302_VERSION_STRING "3.02" +#define ALT_SMB302_VERSION_STRING "3.0.2" +#define SMB311_VERSION_STRING "3.1.1" +#define ALT_SMB311_VERSION_STRING "3.11" + +#define CIFS_DEFAULT_IOSIZE (1024 * 1024) + +#define MAX_CIFS_SMALL_BUFFER_SIZE 448 /* big enough for most */ + +#endif /* _COMMON_SMB_GLOB_H */ diff --git a/fs/smb/server/Kconfig b/fs/smb/server/Kconfig new file mode 100644 index 000000000..221ec9717 --- /dev/null +++ b/fs/smb/server/Kconfig @@ -0,0 +1,73 @@ +config SMB_SERVER + tristate "SMB3 server support" + depends on INET + depends on MULTIUSER + depends on FILE_LOCKING + select NLS + select NLS_UTF8 + select NLS_UCS2_UTILS + select CRYPTO + select CRYPTO_LIB_AES_CBC_MACS + select CRYPTO_LIB_ARC4 + select CRYPTO_LIB_MD5 + select CRYPTO_LIB_SHA256 + select CRYPTO_LIB_SHA512 + select CRYPTO_LIB_UTILS + select CRYPTO_AEAD2 + select CRYPTO_CCM + select CRYPTO_GCM + select ASN1 + select OID_REGISTRY + select CRC32 + default n + help + Choose Y here if you want to allow SMB3 compliant clients + to access files residing on this system using SMB3 protocol. + To compile the SMB3 server support as a module, + choose M here: the module will be called ksmbd. + + You may choose to use a samba server instead, in which + case you can choose N here. + + You also need to install user space programs which can be found + in ksmbd-tools, available from + https://github.com/cifsd-team/ksmbd-tools. + More detail about how to run the ksmbd kernel server is + available via the README file + (https://github.com/cifsd-team/ksmbd-tools/blob/master/README). + + ksmbd kernel server includes support for auto-negotiation, + Secure negotiate, Pre-authentication integrity, oplock/lease, + compound requests, multi-credit, packet signing, RDMA(smbdirect), + smb3 encryption, copy-offload, secure per-user session + establishment via Kerberos or NTLMv2. + +if SMB_SERVER + +config SMB_SERVER_SMBDIRECT + bool "Support for SMB Direct protocol" + depends on SMB_SERVER && INFINIBAND && INFINIBAND_ADDR_TRANS + depends on SMB_SERVER=m || INFINIBAND=y + select SMBDIRECT + default n + + help + Enables SMB Direct support for SMB 3.0, 3.02 and 3.1.1. + + SMB Direct allows transferring SMB packets over RDMA. If unsure, + say N. + +endif + +config SMB_SERVER_CHECK_CAP_NET_ADMIN + bool "Enable check network administration capability" + depends on SMB_SERVER + default y + + help + Prevent unprivileged processes to start the ksmbd kernel server. + +config SMB_SERVER_KERBEROS5 + bool "Support for Kerberos 5" + depends on SMB_SERVER + default y diff --git a/fs/smb/server/Makefile b/fs/smb/server/Makefile new file mode 100644 index 000000000..a3e930605 --- /dev/null +++ b/fs/smb/server/Makefile @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Makefile for Linux SMB3 kernel server +# +obj-$(CONFIG_SMB_SERVER) += ksmbd.o + +ksmbd-y := unicode.o auth.o vfs.o vfs_cache.o server.o ndr.o \ + misc.o oplock.o connection.o ksmbd_work.o crypto_ctx.o \ + mgmt/ksmbd_ida.o mgmt/user_config.o mgmt/share_config.o \ + mgmt/tree_connect.o mgmt/user_session.o smb_common.o \ + transport_tcp.o transport_ipc.o smbacl.o smb2pdu.o \ + smb2ops.o smb2misc.o ksmbd_spnego_negtokeninit.asn1.o \ + ksmbd_spnego_negtokentarg.asn1.o asn1.o compress.o + +$(obj)/asn1.o: $(obj)/ksmbd_spnego_negtokeninit.asn1.h $(obj)/ksmbd_spnego_negtokentarg.asn1.h + +$(obj)/ksmbd_spnego_negtokeninit.asn1.o: $(obj)/ksmbd_spnego_negtokeninit.asn1.c $(obj)/ksmbd_spnego_negtokeninit.asn1.h +$(obj)/ksmbd_spnego_negtokentarg.asn1.o: $(obj)/ksmbd_spnego_negtokentarg.asn1.c $(obj)/ksmbd_spnego_negtokentarg.asn1.h + +ksmbd-$(CONFIG_SMB_SERVER_SMBDIRECT) += transport_rdma.o +ksmbd-$(CONFIG_PROC_FS) += proc.o diff --git a/fs/smb/server/asn1.c b/fs/smb/server/asn1.c new file mode 100644 index 000000000..5c4c5121f --- /dev/null +++ b/fs/smb/server/asn1.c @@ -0,0 +1,241 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * The ASB.1/BER parsing code is derived from ip_nat_snmp_basic.c which was in + * turn derived from the gxsnmp package by Gregory McLean & Jochen Friedrich + * + * Copyright (c) 2000 RP Internet (www.rpi.net.au). + */ + +#include +#include +#include +#include +#include +#include + +#include "glob.h" + +#include "asn1.h" +#include "connection.h" +#include "auth.h" +#include "ksmbd_spnego_negtokeninit.asn1.h" +#include "ksmbd_spnego_negtokentarg.asn1.h" + +#define NTLMSSP_OID_LEN 10 + +static char NTLMSSP_OID_STR[NTLMSSP_OID_LEN] = { 0x2b, 0x06, 0x01, 0x04, 0x01, + 0x82, 0x37, 0x02, 0x02, 0x0a }; + +int +ksmbd_decode_negTokenInit(unsigned char *security_blob, int length, + struct ksmbd_conn *conn) +{ + return asn1_ber_decoder(&ksmbd_spnego_negtokeninit_decoder, conn, + security_blob, length); +} + +int +ksmbd_decode_negTokenTarg(unsigned char *security_blob, int length, + struct ksmbd_conn *conn) +{ + return asn1_ber_decoder(&ksmbd_spnego_negtokentarg_decoder, conn, + security_blob, length); +} + +static int compute_asn_hdr_len_bytes(int len) +{ + if (len > 0xFFFFFF) + return 4; + else if (len > 0xFFFF) + return 3; + else if (len > 0xFF) + return 2; + else if (len > 0x7F) + return 1; + else + return 0; +} + +static void encode_asn_tag(char *buf, unsigned int *ofs, char tag, char seq, + int length) +{ + int i; + int index = *ofs; + char hdr_len = compute_asn_hdr_len_bytes(length); + int len = length + 2 + hdr_len; + + /* insert tag */ + buf[index++] = tag; + + if (!hdr_len) { + buf[index++] = len; + } else { + buf[index++] = 0x80 | hdr_len; + for (i = hdr_len - 1; i >= 0; i--) + buf[index++] = (len >> (i * 8)) & 0xFF; + } + + /* insert seq */ + len = len - (index - *ofs); + buf[index++] = seq; + + if (!hdr_len) { + buf[index++] = len; + } else { + buf[index++] = 0x80 | hdr_len; + for (i = hdr_len - 1; i >= 0; i--) + buf[index++] = (len >> (i * 8)) & 0xFF; + } + + *ofs += (index - *ofs); +} + +int build_spnego_ntlmssp_neg_blob(unsigned char **pbuffer, u16 *buflen, + char *ntlm_blob, int ntlm_blob_len) +{ + char *buf; + unsigned int ofs = 0; + int neg_result_len = 4 + compute_asn_hdr_len_bytes(1) * 2 + 1; + int oid_len = 4 + compute_asn_hdr_len_bytes(NTLMSSP_OID_LEN) * 2 + + NTLMSSP_OID_LEN; + int ntlmssp_len = 4 + compute_asn_hdr_len_bytes(ntlm_blob_len) * 2 + + ntlm_blob_len; + int total_len = 4 + compute_asn_hdr_len_bytes(neg_result_len + + oid_len + ntlmssp_len) * 2 + + neg_result_len + oid_len + ntlmssp_len; + + buf = kmalloc(total_len, KSMBD_DEFAULT_GFP); + if (!buf) + return -ENOMEM; + + /* insert main gss header */ + encode_asn_tag(buf, &ofs, 0xa1, 0x30, neg_result_len + oid_len + + ntlmssp_len); + + /* insert neg result */ + encode_asn_tag(buf, &ofs, 0xa0, 0x0a, 1); + buf[ofs++] = 1; + + /* insert oid */ + encode_asn_tag(buf, &ofs, 0xa1, 0x06, NTLMSSP_OID_LEN); + memcpy(buf + ofs, NTLMSSP_OID_STR, NTLMSSP_OID_LEN); + ofs += NTLMSSP_OID_LEN; + + /* insert response token - ntlmssp blob */ + encode_asn_tag(buf, &ofs, 0xa2, 0x04, ntlm_blob_len); + memcpy(buf + ofs, ntlm_blob, ntlm_blob_len); + ofs += ntlm_blob_len; + + *pbuffer = buf; + *buflen = total_len; + return 0; +} + +int build_spnego_ntlmssp_auth_blob(unsigned char **pbuffer, u16 *buflen, + int neg_result) +{ + char *buf; + unsigned int ofs = 0; + int neg_result_len = 4 + compute_asn_hdr_len_bytes(1) * 2 + 1; + int total_len = 4 + compute_asn_hdr_len_bytes(neg_result_len) * 2 + + neg_result_len; + + buf = kmalloc(total_len, KSMBD_DEFAULT_GFP); + if (!buf) + return -ENOMEM; + + /* insert main gss header */ + encode_asn_tag(buf, &ofs, 0xa1, 0x30, neg_result_len); + + /* insert neg result */ + encode_asn_tag(buf, &ofs, 0xa0, 0x0a, 1); + if (neg_result) + buf[ofs++] = 2; + else + buf[ofs++] = 0; + + *pbuffer = buf; + *buflen = total_len; + return 0; +} + +int ksmbd_gssapi_this_mech(void *context, size_t hdrlen, unsigned char tag, + const void *value, size_t vlen) +{ + enum OID oid; + + oid = look_up_OID(value, vlen); + if (oid != OID_spnego) { + char buf[50]; + + sprint_oid(value, vlen, buf, sizeof(buf)); + ksmbd_debug(AUTH, "Unexpected OID: %s\n", buf); + return -EBADMSG; + } + + return 0; +} + +int ksmbd_neg_token_init_mech_type(void *context, size_t hdrlen, + unsigned char tag, const void *value, + size_t vlen) +{ + struct ksmbd_conn *conn = context; + enum OID oid; + int mech_type; + + oid = look_up_OID(value, vlen); + if (oid == OID_ntlmssp) { + mech_type = KSMBD_AUTH_NTLMSSP; + } else if (oid == OID_mskrb5) { + mech_type = KSMBD_AUTH_MSKRB5; + } else if (oid == OID_krb5) { + mech_type = KSMBD_AUTH_KRB5; + } else if (oid == OID_krb5u2u) { + mech_type = KSMBD_AUTH_KRB5U2U; + } else { + char buf[50]; + + sprint_oid(value, vlen, buf, sizeof(buf)); + ksmbd_debug(AUTH, "Unexpected OID: %s\n", buf); + return -EBADMSG; + } + + conn->auth_mechs |= mech_type; + if (conn->preferred_auth_mech == 0) + conn->preferred_auth_mech = mech_type; + + return 0; +} + +static int ksmbd_neg_token_alloc(void *context, size_t hdrlen, + unsigned char tag, const void *value, + size_t vlen) +{ + struct ksmbd_conn *conn = context; + + if (!vlen) + return -EINVAL; + + conn->mechToken = kmemdup_nul(value, vlen, KSMBD_DEFAULT_GFP); + if (!conn->mechToken) + return -ENOMEM; + + conn->mechTokenLen = (unsigned int)vlen; + + return 0; +} + +int ksmbd_neg_token_init_mech_token(void *context, size_t hdrlen, + unsigned char tag, const void *value, + size_t vlen) +{ + return ksmbd_neg_token_alloc(context, hdrlen, tag, value, vlen); +} + +int ksmbd_neg_token_targ_resp_token(void *context, size_t hdrlen, + unsigned char tag, const void *value, + size_t vlen) +{ + return ksmbd_neg_token_alloc(context, hdrlen, tag, value, vlen); +} diff --git a/fs/smb/server/asn1.h b/fs/smb/server/asn1.h new file mode 100644 index 000000000..ce105f4ce --- /dev/null +++ b/fs/smb/server/asn1.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * The ASB.1/BER parsing code is derived from ip_nat_snmp_basic.c which was in + * turn derived from the gxsnmp package by Gregory McLean & Jochen Friedrich + * + * Copyright (c) 2000 RP Internet (www.rpi.net.au). + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#ifndef __ASN1_H__ +#define __ASN1_H__ + +int ksmbd_decode_negTokenInit(unsigned char *security_blob, int length, + struct ksmbd_conn *conn); +int ksmbd_decode_negTokenTarg(unsigned char *security_blob, int length, + struct ksmbd_conn *conn); +int build_spnego_ntlmssp_neg_blob(unsigned char **pbuffer, u16 *buflen, + char *ntlm_blob, int ntlm_blob_len); +int build_spnego_ntlmssp_auth_blob(unsigned char **pbuffer, u16 *buflen, + int neg_result); +#endif /* __ASN1_H__ */ diff --git a/fs/smb/server/auth.c b/fs/smb/server/auth.c new file mode 100644 index 000000000..1b24eb3f3 --- /dev/null +++ b/fs/smb/server/auth.c @@ -0,0 +1,1137 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2016 Namjae Jeon + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "auth.h" +#include "glob.h" + +#include +#include + +#include "server.h" +#include "smb_common.h" +#include "connection.h" +#include "mgmt/user_session.h" +#include "mgmt/user_config.h" +#include "crypto_ctx.h" +#include "transport_ipc.h" + +/* + * Fixed format data defining GSS header and fixed string + * "not_defined_in_RFC4178@please_ignore". + * So sec blob data in neg phase could be generated statically. + */ +static char NEGOTIATE_GSS_HEADER[AUTH_GSS_LENGTH] = { +#ifdef CONFIG_SMB_SERVER_KERBEROS5 + 0x60, 0x5e, 0x06, 0x06, 0x2b, 0x06, 0x01, 0x05, + 0x05, 0x02, 0xa0, 0x54, 0x30, 0x52, 0xa0, 0x24, + 0x30, 0x22, 0x06, 0x09, 0x2a, 0x86, 0x48, 0x86, + 0xf7, 0x12, 0x01, 0x02, 0x02, 0x06, 0x09, 0x2a, + 0x86, 0x48, 0x82, 0xf7, 0x12, 0x01, 0x02, 0x02, + 0x06, 0x0a, 0x2b, 0x06, 0x01, 0x04, 0x01, 0x82, + 0x37, 0x02, 0x02, 0x0a, 0xa3, 0x2a, 0x30, 0x28, + 0xa0, 0x26, 0x1b, 0x24, 0x6e, 0x6f, 0x74, 0x5f, + 0x64, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64, 0x5f, + 0x69, 0x6e, 0x5f, 0x52, 0x46, 0x43, 0x34, 0x31, + 0x37, 0x38, 0x40, 0x70, 0x6c, 0x65, 0x61, 0x73, + 0x65, 0x5f, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65 +#else + 0x60, 0x48, 0x06, 0x06, 0x2b, 0x06, 0x01, 0x05, + 0x05, 0x02, 0xa0, 0x3e, 0x30, 0x3c, 0xa0, 0x0e, + 0x30, 0x0c, 0x06, 0x0a, 0x2b, 0x06, 0x01, 0x04, + 0x01, 0x82, 0x37, 0x02, 0x02, 0x0a, 0xa3, 0x2a, + 0x30, 0x28, 0xa0, 0x26, 0x1b, 0x24, 0x6e, 0x6f, + 0x74, 0x5f, 0x64, 0x65, 0x66, 0x69, 0x6e, 0x65, + 0x64, 0x5f, 0x69, 0x6e, 0x5f, 0x52, 0x46, 0x43, + 0x34, 0x31, 0x37, 0x38, 0x40, 0x70, 0x6c, 0x65, + 0x61, 0x73, 0x65, 0x5f, 0x69, 0x67, 0x6e, 0x6f, + 0x72, 0x65 +#endif +}; + +void ksmbd_copy_gss_neg_header(void *buf) +{ + memcpy(buf, NEGOTIATE_GSS_HEADER, AUTH_GSS_LENGTH); +} + +static int calc_ntlmv2_hash(struct ksmbd_conn *conn, struct ksmbd_session *sess, + char *ntlmv2_hash, char *dname) +{ + int ret, len, conv_len; + wchar_t *domain = NULL; + __le16 *uniname = NULL; + struct hmac_md5_ctx ctx; + + hmac_md5_init_usingrawkey(&ctx, user_passkey(sess->user), + CIFS_ENCPWD_SIZE); + + /* convert user_name to unicode */ + len = strlen(user_name(sess->user)); + uniname = kzalloc(2 + UNICODE_LEN(len), KSMBD_DEFAULT_GFP); + if (!uniname) { + ret = -ENOMEM; + goto out; + } + + conv_len = smb_strtoUTF16(uniname, user_name(sess->user), len, + conn->local_nls); + if (conv_len < 0 || conv_len > len) { + ret = -EINVAL; + goto out; + } + UniStrupr(uniname); + + hmac_md5_update(&ctx, (const u8 *)uniname, UNICODE_LEN(conv_len)); + + /* Convert domain name or conn name to unicode and uppercase */ + len = strlen(dname); + domain = kzalloc(2 + UNICODE_LEN(len), KSMBD_DEFAULT_GFP); + if (!domain) { + ret = -ENOMEM; + goto out; + } + + conv_len = smb_strtoUTF16((__le16 *)domain, dname, len, + conn->local_nls); + if (conv_len < 0 || conv_len > len) { + ret = -EINVAL; + goto out; + } + + hmac_md5_update(&ctx, (const u8 *)domain, UNICODE_LEN(conv_len)); + hmac_md5_final(&ctx, ntlmv2_hash); + ret = 0; +out: + kfree(uniname); + kfree(domain); + if (ret) /* Done by hmac_md5_final() already if ret == 0 */ + memzero_explicit(&ctx, sizeof(ctx)); + return ret; +} + +/** + * ksmbd_auth_ntlmv2() - NTLMv2 authentication handler + * @conn: connection + * @sess: session of connection + * @ntlmv2: NTLMv2 challenge response + * @blen: NTLMv2 blob length + * @domain_name: domain name + * @cryptkey: session crypto key + * @sess_key: derived session key output buffer + * + * Return: 0 on success, error number on error + */ +int ksmbd_auth_ntlmv2(struct ksmbd_conn *conn, struct ksmbd_session *sess, + struct ntlmv2_resp *ntlmv2, int blen, char *domain_name, + char *cryptkey, char *sess_key) +{ + char ntlmv2_hash[CIFS_ENCPWD_SIZE]; + char ntlmv2_rsp[CIFS_HMAC_MD5_HASH_SIZE]; + char base_key[SMB2_NTLMV2_SESSKEY_SIZE]; + struct hmac_md5_ctx ctx; + int rc; + + if (fips_enabled) { + ksmbd_debug(AUTH, "NTLMv2 support is disabled due to FIPS\n"); + return -EOPNOTSUPP; + } + + rc = calc_ntlmv2_hash(conn, sess, ntlmv2_hash, domain_name); + if (rc) { + ksmbd_debug(AUTH, "could not get v2 hash rc %d\n", rc); + return rc; + } + + hmac_md5_init_usingrawkey(&ctx, ntlmv2_hash, CIFS_HMAC_MD5_HASH_SIZE); + hmac_md5_update(&ctx, cryptkey, CIFS_CRYPTO_KEY_SIZE); + hmac_md5_update(&ctx, (const u8 *)&ntlmv2->blob_signature, blen); + hmac_md5_final(&ctx, ntlmv2_rsp); + + /* Generate the session key */ + hmac_md5_usingrawkey(ntlmv2_hash, CIFS_HMAC_MD5_HASH_SIZE, + ntlmv2_rsp, CIFS_HMAC_MD5_HASH_SIZE, + base_key); + + if (crypto_memneq(ntlmv2->ntlmv2_hash, ntlmv2_rsp, + CIFS_HMAC_MD5_HASH_SIZE)) { + rc = -EINVAL; + goto out; + } + + memcpy(sess_key, base_key, sizeof(base_key)); + rc = 0; +out: + memzero_explicit(ntlmv2_hash, sizeof(ntlmv2_hash)); + memzero_explicit(ntlmv2_rsp, sizeof(ntlmv2_rsp)); + memzero_explicit(base_key, sizeof(base_key)); + return rc; +} + +/** + * ksmbd_decode_ntlmssp_auth_blob() - helper function to construct + * authenticate blob + * @authblob: authenticate blob source pointer + * @blob_len: length of the @authblob message + * @conn: connection + * @sess: session of connection + * @sess_key: derived session key output buffer + * + * Return: 0 on success, error number on error + */ +int ksmbd_decode_ntlmssp_auth_blob(struct authenticate_message *authblob, + int blob_len, struct ksmbd_conn *conn, + struct ksmbd_session *sess, char *sess_key) +{ + char *domain_name; + unsigned int nt_off, dn_off; + unsigned short nt_len, dn_len; + int ret; + + if (blob_len < sizeof(struct authenticate_message)) { + ksmbd_debug(AUTH, "negotiate blob len %d too small\n", + blob_len); + return -EINVAL; + } + + if (memcmp(authblob->Signature, "NTLMSSP", 8)) { + ksmbd_debug(AUTH, "blob signature incorrect %s\n", + authblob->Signature); + return -EINVAL; + } + + nt_off = le32_to_cpu(authblob->NtChallengeResponse.BufferOffset); + nt_len = le16_to_cpu(authblob->NtChallengeResponse.Length); + dn_off = le32_to_cpu(authblob->DomainName.BufferOffset); + dn_len = le16_to_cpu(authblob->DomainName.Length); + + if (blob_len < (u64)dn_off + dn_len || blob_len < (u64)nt_off + nt_len || + nt_len < CIFS_ENCPWD_SIZE) + return -EINVAL; + + /* TODO : use domain name that imported from configuration file */ + domain_name = smb_strndup_from_utf16((const char *)authblob + dn_off, + dn_len, true, conn->local_nls); + if (IS_ERR(domain_name)) + return PTR_ERR(domain_name); + + /* process NTLMv2 authentication */ + ksmbd_debug(AUTH, "decode_ntlmssp_authenticate_blob dname%s\n", + domain_name); + ret = ksmbd_auth_ntlmv2(conn, sess, + (struct ntlmv2_resp *)((char *)authblob + nt_off), + nt_len - CIFS_ENCPWD_SIZE, + domain_name, conn->ntlmssp.cryptkey, sess_key); + kfree(domain_name); + if (ret) + return ret; + + /* The recovered secondary session key */ + if (conn->ntlmssp.client_flags & NTLMSSP_NEGOTIATE_KEY_XCH) { + struct arc4_ctx *ctx_arc4; + unsigned int sess_key_off, sess_key_len; + + sess_key_off = le32_to_cpu(authblob->SessionKey.BufferOffset); + sess_key_len = le16_to_cpu(authblob->SessionKey.Length); + + if (blob_len < (u64)sess_key_off + sess_key_len) + return -EINVAL; + + if (sess_key_len > CIFS_KEY_SIZE) + return -EINVAL; + + ctx_arc4 = kmalloc_obj(*ctx_arc4, KSMBD_DEFAULT_GFP); + if (!ctx_arc4) + return -ENOMEM; + + arc4_setkey(ctx_arc4, sess_key, SMB2_NTLMV2_SESSKEY_SIZE); + arc4_crypt(ctx_arc4, sess_key, + (char *)authblob + sess_key_off, sess_key_len); + kfree_sensitive(ctx_arc4); + } + + return ret; +} + +/** + * ksmbd_decode_ntlmssp_neg_blob() - helper function to construct + * negotiate blob + * @negblob: negotiate blob source pointer + * @blob_len: length of the @authblob message + * @conn: connection + * + */ +int ksmbd_decode_ntlmssp_neg_blob(struct negotiate_message *negblob, + int blob_len, struct ksmbd_conn *conn) +{ + if (blob_len < sizeof(struct negotiate_message)) { + ksmbd_debug(AUTH, "negotiate blob len %d too small\n", + blob_len); + return -EINVAL; + } + + if (memcmp(negblob->Signature, "NTLMSSP", 8)) { + ksmbd_debug(AUTH, "blob signature incorrect %s\n", + negblob->Signature); + return -EINVAL; + } + + conn->ntlmssp.client_flags = le32_to_cpu(negblob->NegotiateFlags); + return 0; +} + +/** + * ksmbd_build_ntlmssp_challenge_blob() - helper function to construct + * challenge blob + * @chgblob: challenge blob source pointer to initialize + * @conn: connection + * + */ +unsigned int +ksmbd_build_ntlmssp_challenge_blob(struct challenge_message *chgblob, + struct ksmbd_conn *conn) +{ + struct target_info *tinfo; + wchar_t *name; + __u8 *target_name; + unsigned int flags, blob_off, blob_len, type, target_info_len = 0; + int len, uni_len, conv_len; + int cflags = conn->ntlmssp.client_flags; + + memcpy(chgblob->Signature, NTLMSSP_SIGNATURE, 8); + chgblob->MessageType = NtLmChallenge; + + flags = NTLMSSP_NEGOTIATE_UNICODE | + NTLMSSP_NEGOTIATE_NTLM | NTLMSSP_TARGET_TYPE_SERVER | + NTLMSSP_NEGOTIATE_TARGET_INFO; + + if (cflags & NTLMSSP_NEGOTIATE_SIGN) { + flags |= NTLMSSP_NEGOTIATE_SIGN; + flags |= cflags & (NTLMSSP_NEGOTIATE_128 | + NTLMSSP_NEGOTIATE_56); + } + + if (cflags & NTLMSSP_NEGOTIATE_SEAL && smb3_encryption_negotiated(conn)) + flags |= NTLMSSP_NEGOTIATE_SEAL; + + if (cflags & NTLMSSP_NEGOTIATE_ALWAYS_SIGN) + flags |= NTLMSSP_NEGOTIATE_ALWAYS_SIGN; + + if (cflags & NTLMSSP_REQUEST_TARGET) + flags |= NTLMSSP_REQUEST_TARGET; + + if (conn->use_spnego && + (cflags & NTLMSSP_NEGOTIATE_EXTENDED_SEC)) + flags |= NTLMSSP_NEGOTIATE_EXTENDED_SEC; + + if (cflags & NTLMSSP_NEGOTIATE_KEY_XCH) + flags |= NTLMSSP_NEGOTIATE_KEY_XCH; + + chgblob->NegotiateFlags = cpu_to_le32(flags); + len = strlen(ksmbd_netbios_name()); + name = kmalloc(2 + UNICODE_LEN(len), KSMBD_DEFAULT_GFP); + if (!name) + return -ENOMEM; + + conv_len = smb_strtoUTF16((__le16 *)name, ksmbd_netbios_name(), len, + conn->local_nls); + if (conv_len < 0 || conv_len > len) { + kfree(name); + return -EINVAL; + } + + uni_len = UNICODE_LEN(conv_len); + + blob_off = sizeof(struct challenge_message); + blob_len = blob_off + uni_len; + + chgblob->TargetName.Length = cpu_to_le16(uni_len); + chgblob->TargetName.MaximumLength = cpu_to_le16(uni_len); + chgblob->TargetName.BufferOffset = cpu_to_le32(blob_off); + + /* Initialize random conn challenge */ + get_random_bytes(conn->ntlmssp.cryptkey, sizeof(__u64)); + memcpy(chgblob->Challenge, conn->ntlmssp.cryptkey, + CIFS_CRYPTO_KEY_SIZE); + + /* Add Target Information to security buffer */ + chgblob->TargetInfoArray.BufferOffset = cpu_to_le32(blob_len); + + target_name = (__u8 *)chgblob + blob_off; + memcpy(target_name, name, uni_len); + tinfo = (struct target_info *)(target_name + uni_len); + + chgblob->TargetInfoArray.Length = 0; + /* Add target info list for NetBIOS/DNS settings */ + for (type = NTLMSSP_AV_NB_COMPUTER_NAME; + type <= NTLMSSP_AV_DNS_DOMAIN_NAME; type++) { + tinfo->Type = cpu_to_le16(type); + tinfo->Length = cpu_to_le16(uni_len); + memcpy(tinfo->Content, name, uni_len); + tinfo = (struct target_info *)((char *)tinfo + 4 + uni_len); + target_info_len += 4 + uni_len; + } + + /* Add terminator subblock */ + tinfo->Type = 0; + tinfo->Length = 0; + target_info_len += 4; + + chgblob->TargetInfoArray.Length = cpu_to_le16(target_info_len); + chgblob->TargetInfoArray.MaximumLength = cpu_to_le16(target_info_len); + blob_len += target_info_len; + kfree(name); + ksmbd_debug(AUTH, "NTLMSSP SecurityBufferLength %d\n", blob_len); + return blob_len; +} + +#ifdef CONFIG_SMB_SERVER_KERBEROS5 +int ksmbd_krb5_authenticate(struct ksmbd_session *sess, char *in_blob, + int in_len, char *out_blob, int *out_len, + char *sess_key) +{ + struct ksmbd_spnego_authen_response *resp; + struct ksmbd_login_response_ext *resp_ext = NULL; + struct ksmbd_user *user = NULL; + int retval; + + resp = ksmbd_ipc_spnego_authen_request(in_blob, in_len); + if (!resp) { + ksmbd_debug(AUTH, "SPNEGO_AUTHEN_REQUEST failure\n"); + return -EINVAL; + } + + if (!(resp->login_response.status & KSMBD_USER_FLAG_OK)) { + ksmbd_debug(AUTH, "krb5 authentication failure\n"); + retval = -EPERM; + goto out; + } + + if (*out_len <= resp->spnego_blob_len) { + ksmbd_debug(AUTH, "buf len %d, but blob len %d\n", + *out_len, resp->spnego_blob_len); + retval = -EINVAL; + goto out; + } + + if (resp->session_key_len > sizeof(sess->sess_key)) { + ksmbd_debug(AUTH, "session key is too long\n"); + retval = -EINVAL; + goto out; + } + + if (resp->login_response.status & KSMBD_USER_FLAG_EXTENSION) + resp_ext = ksmbd_ipc_login_request_ext(resp->login_response.account); + + user = ksmbd_alloc_user(&resp->login_response, resp_ext); + kvfree(resp_ext); + if (!user) { + ksmbd_debug(AUTH, "login failure\n"); + retval = -ENOMEM; + goto out; + } + + if (!sess->user) { + /* First successful authentication */ + sess->user = user; + } else { + if (!ksmbd_compare_user(sess->user, user)) { + ksmbd_debug(AUTH, "different user tried to reuse session\n"); + retval = -EKEYREJECTED; + ksmbd_free_user(user); + goto out; + } + ksmbd_free_user(user); + } + + memcpy(sess_key, resp->payload, resp->session_key_len); + memcpy(out_blob, resp->payload + resp->session_key_len, + resp->spnego_blob_len); + *out_len = resp->spnego_blob_len; + sess->kerberos_expiry = resp->session_expiry; + retval = 0; +out: + kvfree_sensitive(resp, sizeof(*resp) + resp->session_key_len + + resp->spnego_blob_len); + return retval; +} +#else +int ksmbd_krb5_authenticate(struct ksmbd_session *sess, char *in_blob, + int in_len, char *out_blob, int *out_len, + char *sess_key) +{ + return -EOPNOTSUPP; +} +#endif + +/** + * ksmbd_sign_smb2_pdu() - function to generate packet signing + * @conn: connection + * @key: signing key + * @iov: buffer iov array + * @n_vec: number of iovecs + * @sig: signature value generated for client request packet + * + */ +void ksmbd_sign_smb2_pdu(struct ksmbd_conn *conn, char *key, struct kvec *iov, + int n_vec, char *sig) +{ + struct hmac_sha256_ctx ctx; + int i; + + hmac_sha256_init_usingrawkey(&ctx, key, SMB2_NTLMV2_SESSKEY_SIZE); + for (i = 0; i < n_vec; i++) + hmac_sha256_update(&ctx, iov[i].iov_base, iov[i].iov_len); + hmac_sha256_final(&ctx, sig); +} + +/** + * ksmbd_sign_smb3_pdu() - function to generate packet signing + * @conn: connection + * @key: signing key + * @iov: buffer iov array + * @n_vec: number of iovecs + * @sig: signature value generated for client request packet + * + */ +void ksmbd_sign_smb3_pdu(struct ksmbd_conn *conn, char *key, struct kvec *iov, + int n_vec, char *sig) +{ + struct aes_cmac_key cmac_key __cleanup(aes_cmac_zeroize_key); + struct aes_cmac_ctx cmac_ctx; + int i; + + /* This cannot fail, since we always pass a valid key length. */ + static_assert(SMB2_CMACAES_SIZE == AES_KEYSIZE_128); + aes_cmac_preparekey(&cmac_key, key, SMB2_CMACAES_SIZE); + + aes_cmac_init(&cmac_ctx, &cmac_key); + for (i = 0; i < n_vec; i++) + aes_cmac_update(&cmac_ctx, iov[i].iov_base, iov[i].iov_len); + aes_cmac_final(&cmac_ctx, sig); +} + +struct derivation { + struct kvec label; + struct kvec context; + bool binding; +}; + +static void generate_key(struct ksmbd_conn *conn, const char *sess_key, + struct kvec label, struct kvec context, __u8 *key, + unsigned int key_size) +{ + unsigned char zero = 0x0; + __u8 i[4] = {0, 0, 0, 1}; + __u8 L128[4] = {0, 0, 0, 128}; + __u8 L256[4] = {0, 0, 1, 0}; + unsigned char prfhash[SMB2_HMACSHA256_SIZE]; + struct hmac_sha256_ctx ctx; + + hmac_sha256_init_usingrawkey(&ctx, sess_key, + SMB2_NTLMV2_SESSKEY_SIZE); + hmac_sha256_update(&ctx, i, 4); + hmac_sha256_update(&ctx, label.iov_base, label.iov_len); + hmac_sha256_update(&ctx, &zero, 1); + hmac_sha256_update(&ctx, context.iov_base, context.iov_len); + + if (key_size == SMB3_ENC_DEC_KEY_SIZE && + (conn->cipher_type == SMB2_ENCRYPTION_AES256_CCM || + conn->cipher_type == SMB2_ENCRYPTION_AES256_GCM)) + hmac_sha256_update(&ctx, L256, 4); + else + hmac_sha256_update(&ctx, L128, 4); + + hmac_sha256_final(&ctx, prfhash); + memcpy(key, prfhash, key_size); + memzero_explicit(prfhash, sizeof(prfhash)); +} + +static int generate_smb3signingkey(struct ksmbd_session *sess, + struct ksmbd_conn *conn, + const struct derivation *signing) +{ + struct channel *chann; + char *key, *sess_key; + + chann = lookup_chann_list(sess, conn); + if (!chann) + return 0; + + if (conn->dialect >= SMB30_PROT_ID && signing->binding) { + key = chann->smb3signingkey; + sess_key = chann->sess_key; + } else { + key = sess->smb3signingkey; + sess_key = sess->sess_key; + } + + generate_key(conn, sess_key, signing->label, signing->context, key, + SMB3_SIGN_KEY_SIZE); + + if (!(conn->dialect >= SMB30_PROT_ID && signing->binding)) + memcpy(chann->smb3signingkey, key, SMB3_SIGN_KEY_SIZE); + + ksmbd_debug(AUTH, "generated SMB3 signing key\n"); + ksmbd_debug(AUTH, "Session Id %llu\n", sess->id); + return 0; +} + +int ksmbd_gen_smb30_signingkey(struct ksmbd_session *sess, + struct ksmbd_conn *conn) +{ + struct derivation d; + + d.label.iov_base = "SMB2AESCMAC"; + d.label.iov_len = 12; + d.context.iov_base = "SmbSign"; + d.context.iov_len = 8; + d.binding = conn->binding; + + return generate_smb3signingkey(sess, conn, &d); +} + +int ksmbd_gen_smb311_signingkey(struct ksmbd_session *sess, + struct ksmbd_conn *conn) +{ + struct derivation d; + + d.label.iov_base = "SMBSigningKey"; + d.label.iov_len = 14; + if (conn->binding) { + struct preauth_session *preauth_sess; + + preauth_sess = ksmbd_preauth_session_lookup(conn, sess->id); + if (!preauth_sess) + return -ENOENT; + d.context.iov_base = preauth_sess->Preauth_HashValue; + } else { + d.context.iov_base = sess->Preauth_HashValue; + } + d.context.iov_len = 64; + d.binding = conn->binding; + + return generate_smb3signingkey(sess, conn, &d); +} + +struct derivation_twin { + struct derivation encryption; + struct derivation decryption; +}; + +static void generate_smb3encryptionkey(struct ksmbd_conn *conn, + struct ksmbd_session *sess, + const struct derivation_twin *ptwin) +{ + generate_key(conn, sess->sess_key, ptwin->encryption.label, + ptwin->encryption.context, sess->smb3encryptionkey, + SMB3_ENC_DEC_KEY_SIZE); + + generate_key(conn, sess->sess_key, ptwin->decryption.label, + ptwin->decryption.context, + sess->smb3decryptionkey, SMB3_ENC_DEC_KEY_SIZE); + + ksmbd_debug(AUTH, "generated SMB3 encryption/decryption keys\n"); + ksmbd_debug(AUTH, "Cipher type %d\n", conn->cipher_type); + ksmbd_debug(AUTH, "Session Id %llu\n", sess->id); +} + +void ksmbd_gen_smb30_encryptionkey(struct ksmbd_conn *conn, + struct ksmbd_session *sess) +{ + struct derivation_twin twin; + struct derivation *d; + + d = &twin.encryption; + d->label.iov_base = "SMB2AESCCM"; + d->label.iov_len = 11; + d->context.iov_base = "ServerOut"; + d->context.iov_len = 10; + + d = &twin.decryption; + d->label.iov_base = "SMB2AESCCM"; + d->label.iov_len = 11; + d->context.iov_base = "ServerIn "; + d->context.iov_len = 10; + + generate_smb3encryptionkey(conn, sess, &twin); +} + +void ksmbd_gen_smb311_encryptionkey(struct ksmbd_conn *conn, + struct ksmbd_session *sess) +{ + struct derivation_twin twin; + struct derivation *d; + + d = &twin.encryption; + d->label.iov_base = "SMBS2CCipherKey"; + d->label.iov_len = 16; + d->context.iov_base = sess->Preauth_HashValue; + d->context.iov_len = 64; + + d = &twin.decryption; + d->label.iov_base = "SMBC2SCipherKey"; + d->label.iov_len = 16; + d->context.iov_base = sess->Preauth_HashValue; + d->context.iov_len = 64; + + generate_smb3encryptionkey(conn, sess, &twin); +} + +int ksmbd_gen_preauth_integrity_hash(struct ksmbd_conn *conn, char *buf, + __u8 *pi_hash) +{ + struct smb2_hdr *rcv_hdr = smb_get_msg(buf); + char *all_bytes_msg = (char *)&rcv_hdr->ProtocolId; + int msg_size = get_rfc1002_len(buf); + struct sha512_ctx sha_ctx; + + if (conn->preauth_info->Preauth_HashId != + SMB2_PREAUTH_INTEGRITY_SHA512) + return -EINVAL; + + sha512_init(&sha_ctx); + sha512_update(&sha_ctx, pi_hash, 64); + sha512_update(&sha_ctx, all_bytes_msg, msg_size); + sha512_final(&sha_ctx, pi_hash); + return 0; +} + +static int ksmbd_get_encryption_key(struct ksmbd_work *work, __u64 ses_id, + int enc, u8 *key) +{ + struct ksmbd_session *sess; + u8 *ses_enc_key; + + if (enc) + sess = work->sess; + else { + /* + * A previous-session replacement leaves the old encryption key in + * place. Use it to authenticate an encrypted request, then let + * session validation reject the expired session. This preserves the + * encrypted STATUS_USER_SESSION_DELETED response without reviving + * the session. + */ + sess = ksmbd_session_lookup_all_states(work->conn, ses_id); + if (sess && sess->state != SMB2_SESSION_VALID && + (sess->state != SMB2_SESSION_EXPIRED || !sess->enc)) { + ksmbd_user_session_put(sess); + sess = NULL; + } + } + if (!sess) + return -EINVAL; + + ses_enc_key = enc ? sess->smb3encryptionkey : + sess->smb3decryptionkey; + memcpy(key, ses_enc_key, SMB3_ENC_DEC_KEY_SIZE); + if (!enc) + ksmbd_user_session_put(sess); + + return 0; +} + +static inline void smb2_sg_set_buf(struct scatterlist *sg, const void *buf, + unsigned int buflen) +{ + void *addr; + + if (is_vmalloc_addr(buf)) + addr = vmalloc_to_page(buf); + else + addr = virt_to_page(buf); + sg_set_page(sg, addr, buflen, offset_in_page(buf)); +} + +static struct scatterlist *ksmbd_init_sg(struct kvec *iov, unsigned int nvec, + u8 *sign) +{ + struct scatterlist *sg; + unsigned int assoc_data_len = sizeof(struct smb2_transform_hdr) - 20; + int i, *nr_entries, total_entries = 0, sg_idx = 0; + + if (!nvec) + return NULL; + + nr_entries = kzalloc_objs(int, nvec, KSMBD_DEFAULT_GFP); + if (!nr_entries) + return NULL; + + for (i = 0; i < nvec - 1; i++) { + unsigned long kaddr = (unsigned long)iov[i + 1].iov_base; + + if (is_vmalloc_addr(iov[i + 1].iov_base)) { + nr_entries[i] = ((kaddr + iov[i + 1].iov_len + + PAGE_SIZE - 1) >> PAGE_SHIFT) - + (kaddr >> PAGE_SHIFT); + } else { + nr_entries[i]++; + } + total_entries += nr_entries[i]; + } + + /* Add two entries for transform header and signature */ + total_entries += 2; + + sg = kmalloc_objs(struct scatterlist, total_entries, KSMBD_DEFAULT_GFP); + if (!sg) { + kfree(nr_entries); + return NULL; + } + + sg_init_table(sg, total_entries); + smb2_sg_set_buf(&sg[sg_idx++], iov[0].iov_base + 24, assoc_data_len); + for (i = 0; i < nvec - 1; i++) { + void *data = iov[i + 1].iov_base; + int len = iov[i + 1].iov_len; + + if (is_vmalloc_addr(data)) { + int j, offset = offset_in_page(data); + + for (j = 0; j < nr_entries[i]; j++) { + unsigned int bytes = PAGE_SIZE - offset; + + if (!len) + break; + + if (bytes > len) + bytes = len; + + sg_set_page(&sg[sg_idx++], + vmalloc_to_page(data), bytes, + offset_in_page(data)); + + data += bytes; + len -= bytes; + offset = 0; + } + } else { + sg_set_page(&sg[sg_idx++], virt_to_page(data), len, + offset_in_page(data)); + } + } + smb2_sg_set_buf(&sg[sg_idx], sign, SMB2_SIGNATURE_SIZE); + kfree(nr_entries); + return sg; +} + +/** + * ksmbd_init_rdma_sg() - build an AEAD scatterlist for an RDMA payload + * @buf: payload buffer + * @buflen: payload length + * @tag: authentication tag buffer + * @taglen: authentication tag length + * + * Split vmalloc-backed payloads at page boundaries and append the detached + * authentication tag as the final scatterlist entry. + * + * Return: allocated scatterlist, or NULL on allocation failure + */ +static struct scatterlist *ksmbd_init_rdma_sg(void *buf, + unsigned int buflen, + u8 *tag, + unsigned int taglen) +{ + struct scatterlist *sg; + unsigned int nr_data = 1, nr_entries, i = 0; + void *data = buf; + int len = buflen; + + if (is_vmalloc_addr(buf)) + nr_data = DIV_ROUND_UP(offset_in_page(buf) + buflen, PAGE_SIZE); + nr_entries = nr_data + 1; + + sg = kmalloc_objs(struct scatterlist, nr_entries, KSMBD_DEFAULT_GFP); + if (!sg) + return NULL; + + sg_init_table(sg, nr_entries); + if (!is_vmalloc_addr(buf)) { + smb2_sg_set_buf(&sg[i++], buf, buflen); + } else { + while (len) { + unsigned int bytes = min_t(unsigned int, + PAGE_SIZE - offset_in_page(data), len); + + sg_set_page(&sg[i++], vmalloc_to_page(data), bytes, + offset_in_page(data)); + data += bytes; + len -= bytes; + } + } + smb2_sg_set_buf(&sg[i], tag, taglen); + return sg; +} + +/** + * ksmbd_crypt_rdma() - encrypt or decrypt an SMB Direct data buffer + * @conn: connection containing the negotiated cipher + * @key: session encryption or decryption key + * @buf: RDMA payload, transformed in place + * @buflen: payload length (the authentication tag is carried out of band) + * @nonce: transform nonce + * @nonce_len: nonce length + * @tag: authentication tag output for encryption, input for decryption + * @tag_len: authentication tag length + * @enc: true to encrypt, false to decrypt + * + * SMB2_RDMA_CRYPTO_TRANSFORM carries the nonce and authentication tag in the + * SMB2 message while only the payload is transferred through RDMA. Therefore + * this uses AEAD without the normal SMB3 transform header as associated data. + * + * Return: 0 on success, otherwise a negative errno + */ +int ksmbd_crypt_rdma(struct ksmbd_conn *conn, const u8 *key, + void *buf, unsigned int buflen, const u8 *nonce, + unsigned int nonce_len, u8 *tag, unsigned int tag_len, + bool enc) +{ + struct ksmbd_crypto_ctx *ctx; + struct crypto_aead *tfm; + struct aead_request *req = NULL; + struct scatterlist *sg = NULL; + unsigned int iv_len, crypt_len; + u8 auth_tag[SMB2_SIGNATURE_SIZE] = {}; + u8 *iv = NULL; + u16 cipher = le16_to_cpu(conn->cipher_type); + int rc; + DECLARE_CRYPTO_WAIT(wait); + + if (!buflen || !tag_len || tag_len > SMB2_SIGNATURE_SIZE) { + pr_err("RDMA %s rejected: cipher=0x%04x payload=%u nonce=%u tag=%u\n", + enc ? "encryption" : "decryption", cipher, buflen, + nonce_len, tag_len); + return -EINVAL; + } + if (!enc) + memcpy(auth_tag, tag, tag_len); + + if (conn->cipher_type == SMB2_ENCRYPTION_AES128_GCM || + conn->cipher_type == SMB2_ENCRYPTION_AES256_GCM) { + if (nonce_len != SMB3_AES_GCM_NONCE) { + pr_err("RDMA %s rejected: cipher=0x%04x invalid nonce=%u expected=%u\n", + enc ? "encryption" : "decryption", cipher, + nonce_len, SMB3_AES_GCM_NONCE); + return -EINVAL; + } + ctx = ksmbd_crypto_ctx_find_gcm(); + } else { + if (nonce_len != SMB3_AES_CCM_NONCE) { + pr_err("RDMA %s rejected: cipher=0x%04x invalid nonce=%u expected=%u\n", + enc ? "encryption" : "decryption", cipher, + nonce_len, SMB3_AES_CCM_NONCE); + return -EINVAL; + } + ctx = ksmbd_crypto_ctx_find_ccm(); + } + if (!ctx) { + pr_err("RDMA %s failed: cipher=0x%04x crypto context unavailable\n", + enc ? "encryption" : "decryption", cipher); + return -ENOMEM; + } + + tfm = (conn->cipher_type == SMB2_ENCRYPTION_AES128_GCM || + conn->cipher_type == SMB2_ENCRYPTION_AES256_GCM) ? + CRYPTO_GCM(ctx) : CRYPTO_CCM(ctx); + if (conn->cipher_type == SMB2_ENCRYPTION_AES256_CCM || + conn->cipher_type == SMB2_ENCRYPTION_AES256_GCM) + rc = crypto_aead_setkey(tfm, key, SMB3_GCM256_CRYPTKEY_SIZE); + else + rc = crypto_aead_setkey(tfm, key, SMB3_GCM128_CRYPTKEY_SIZE); + if (rc) + goto out; + + rc = crypto_aead_setauthsize(tfm, tag_len); + if (rc) + goto out; + + req = aead_request_alloc(tfm, KSMBD_DEFAULT_GFP); + if (!req) { + rc = -ENOMEM; + goto out; + } + + sg = ksmbd_init_rdma_sg(buf, buflen, auth_tag, tag_len); + if (!sg) { + rc = -ENOMEM; + goto out; + } + + iv_len = crypto_aead_ivsize(tfm); + iv = kzalloc(iv_len, KSMBD_DEFAULT_GFP); + if (!iv) { + rc = -ENOMEM; + goto out; + } + if (conn->cipher_type == SMB2_ENCRYPTION_AES128_GCM || + conn->cipher_type == SMB2_ENCRYPTION_AES256_GCM) { + memcpy(iv, nonce, nonce_len); + } else { + iv[0] = 3; + memcpy(iv + 1, nonce, nonce_len); + } + + crypt_len = buflen + (enc ? 0 : tag_len); + aead_request_set_crypt(req, sg, sg, crypt_len, iv); + aead_request_set_ad(req, 0); + aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP, + crypto_req_done, &wait); + rc = crypto_wait_req(enc ? crypto_aead_encrypt(req) : + crypto_aead_decrypt(req), &wait); + if (!rc && enc) + memcpy(tag, auth_tag, tag_len); +out: + kfree(iv); + kfree(sg); + aead_request_free(req); + ksmbd_release_crypto_ctx(ctx); + if (rc) + pr_err("RDMA %s failed: cipher=0x%04x payload=%u nonce=%u tag=%u rc=%d\n", + enc ? "encryption" : "decryption", cipher, buflen, + nonce_len, tag_len, rc); + else + ksmbd_debug(RDMA, + "RDMA %s completed: cipher=0x%04x payload=%u nonce=%u tag=%u\n", + enc ? "encryption" : "decryption", cipher, buflen, + nonce_len, tag_len); + return rc; +} + +int ksmbd_crypt_message(struct ksmbd_work *work, struct kvec *iov, + unsigned int nvec, int enc) +{ + struct ksmbd_conn *conn = work->conn; + struct smb2_transform_hdr *tr_hdr = smb_get_msg(iov[0].iov_base); + unsigned int assoc_data_len = sizeof(struct smb2_transform_hdr) - 20; + int rc; + DECLARE_CRYPTO_WAIT(wait); + struct scatterlist *sg; + u8 sign[SMB2_SIGNATURE_SIZE] = {}; + u8 key[SMB3_ENC_DEC_KEY_SIZE]; + struct aead_request *req; + char *iv; + unsigned int iv_len; + struct crypto_aead *tfm; + unsigned int crypt_len = le32_to_cpu(tr_hdr->OriginalMessageSize); + struct ksmbd_crypto_ctx *ctx; + + rc = ksmbd_get_encryption_key(work, + le64_to_cpu(tr_hdr->SessionId), + enc, + key); + if (rc) { + pr_err("Could not get %scryption key\n", enc ? "en" : "de"); + return rc; + } + + if (conn->cipher_type == SMB2_ENCRYPTION_AES128_GCM || + conn->cipher_type == SMB2_ENCRYPTION_AES256_GCM) + ctx = ksmbd_crypto_ctx_find_gcm(); + else + ctx = ksmbd_crypto_ctx_find_ccm(); + if (!ctx) { + pr_err("crypto alloc failed\n"); + rc = -ENOMEM; + goto zeroize_key; + } + + if (conn->cipher_type == SMB2_ENCRYPTION_AES128_GCM || + conn->cipher_type == SMB2_ENCRYPTION_AES256_GCM) + tfm = CRYPTO_GCM(ctx); + else + tfm = CRYPTO_CCM(ctx); + + if (conn->cipher_type == SMB2_ENCRYPTION_AES256_CCM || + conn->cipher_type == SMB2_ENCRYPTION_AES256_GCM) + rc = crypto_aead_setkey(tfm, key, SMB3_GCM256_CRYPTKEY_SIZE); + else + rc = crypto_aead_setkey(tfm, key, SMB3_GCM128_CRYPTKEY_SIZE); + if (rc) { + pr_err("Failed to set aead key %d\n", rc); + goto free_ctx; + } + + rc = crypto_aead_setauthsize(tfm, SMB2_SIGNATURE_SIZE); + if (rc) { + pr_err("Failed to set authsize %d\n", rc); + goto free_ctx; + } + + req = aead_request_alloc(tfm, KSMBD_DEFAULT_GFP); + if (!req) { + rc = -ENOMEM; + goto free_ctx; + } + + if (!enc) { + memcpy(sign, &tr_hdr->Signature, SMB2_SIGNATURE_SIZE); + crypt_len += SMB2_SIGNATURE_SIZE; + } + + sg = ksmbd_init_sg(iov, nvec, sign); + if (!sg) { + pr_err("Failed to init sg\n"); + rc = -ENOMEM; + goto free_req; + } + + iv_len = crypto_aead_ivsize(tfm); + iv = kzalloc(iv_len, KSMBD_DEFAULT_GFP); + if (!iv) { + rc = -ENOMEM; + goto free_sg; + } + + if (conn->cipher_type == SMB2_ENCRYPTION_AES128_GCM || + conn->cipher_type == SMB2_ENCRYPTION_AES256_GCM) { + memcpy(iv, (char *)tr_hdr->Nonce, SMB3_AES_GCM_NONCE); + } else { + iv[0] = 3; + memcpy(iv + 1, (char *)tr_hdr->Nonce, SMB3_AES_CCM_NONCE); + } + + aead_request_set_crypt(req, sg, sg, crypt_len, iv); + aead_request_set_ad(req, assoc_data_len); + aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP, + crypto_req_done, &wait); + + rc = crypto_wait_req(enc ? crypto_aead_encrypt(req) : + crypto_aead_decrypt(req), &wait); + if (rc) + goto free_iv; + + if (enc) + memcpy(&tr_hdr->Signature, sign, SMB2_SIGNATURE_SIZE); + +free_iv: + kfree(iv); +free_sg: + kfree(sg); +free_req: + aead_request_free(req); +free_ctx: + ksmbd_release_crypto_ctx(ctx); +zeroize_key: + memzero_explicit(key, sizeof(key)); + memzero_explicit(sign, sizeof(sign)); + return rc; +} diff --git a/fs/smb/server/auth.h b/fs/smb/server/auth.h new file mode 100644 index 000000000..7ce9c42d5 --- /dev/null +++ b/fs/smb/server/auth.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#ifndef __AUTH_H__ +#define __AUTH_H__ + +#include "ntlmssp.h" + +#ifdef CONFIG_SMB_SERVER_KERBEROS5 +#define AUTH_GSS_LENGTH 96 +#define AUTH_GSS_PADDING 0 +#else +#define AUTH_GSS_LENGTH 74 +#define AUTH_GSS_PADDING 6 +#endif + +#define CIFS_HMAC_MD5_HASH_SIZE (16) +#define CIFS_NTHASH_SIZE (16) + +/* + * Size of the ntlm client response + */ +#define CIFS_AUTH_RESP_SIZE 24 +#define CIFS_SMB1_SIGNATURE_SIZE 8 +#define CIFS_SMB1_SESSKEY_SIZE 16 + +#define KSMBD_AUTH_NTLMSSP 0x0001 +#define KSMBD_AUTH_KRB5 0x0002 +#define KSMBD_AUTH_MSKRB5 0x0004 +#define KSMBD_AUTH_KRB5U2U 0x0008 + +struct ksmbd_session; +struct ksmbd_conn; +struct ksmbd_work; +struct kvec; + +int ksmbd_crypt_message(struct ksmbd_work *work, struct kvec *iov, + unsigned int nvec, int enc); +int ksmbd_crypt_rdma(struct ksmbd_conn *conn, const u8 *key, + void *buf, unsigned int buflen, const u8 *nonce, + unsigned int nonce_len, u8 *tag, unsigned int tag_len, + bool enc); +void ksmbd_copy_gss_neg_header(void *buf); +int ksmbd_auth_ntlmv2(struct ksmbd_conn *conn, struct ksmbd_session *sess, + struct ntlmv2_resp *ntlmv2, int blen, char *domain_name, + char *cryptkey, char *sess_key); +int ksmbd_decode_ntlmssp_auth_blob(struct authenticate_message *authblob, + int blob_len, struct ksmbd_conn *conn, + struct ksmbd_session *sess, char *sess_key); +int ksmbd_decode_ntlmssp_neg_blob(struct negotiate_message *negblob, + int blob_len, struct ksmbd_conn *conn); +unsigned int +ksmbd_build_ntlmssp_challenge_blob(struct challenge_message *chgblob, + struct ksmbd_conn *conn); +int ksmbd_krb5_authenticate(struct ksmbd_session *sess, char *in_blob, + int in_len, char *out_blob, int *out_len, + char *sess_key); +void ksmbd_sign_smb2_pdu(struct ksmbd_conn *conn, char *key, struct kvec *iov, + int n_vec, char *sig); +void ksmbd_sign_smb3_pdu(struct ksmbd_conn *conn, char *key, struct kvec *iov, + int n_vec, char *sig); +int ksmbd_gen_smb30_signingkey(struct ksmbd_session *sess, + struct ksmbd_conn *conn); +int ksmbd_gen_smb311_signingkey(struct ksmbd_session *sess, + struct ksmbd_conn *conn); +void ksmbd_gen_smb30_encryptionkey(struct ksmbd_conn *conn, + struct ksmbd_session *sess); +void ksmbd_gen_smb311_encryptionkey(struct ksmbd_conn *conn, + struct ksmbd_session *sess); +int ksmbd_gen_preauth_integrity_hash(struct ksmbd_conn *conn, char *buf, + __u8 *pi_hash); +#endif diff --git a/fs/smb/server/compress.c b/fs/smb/server/compress.c new file mode 100644 index 000000000..5162fb84c --- /dev/null +++ b/fs/smb/server/compress.c @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * SMB2 compression support for ksmbd. + * + * Receive and send SMB 3.1.1 compression transforms using the common helpers. + * + * Copyright (C) 2026 Namjae Jeon + */ +#include + +#include "compress.h" +#include "smb_common.h" +#include "../common/compress/lz77.h" + +#define SMB_COMPRESS_MIN_LEN PAGE_SIZE + +static int __ksmbd_decompress_request(struct ksmbd_conn *conn, + void *request_buf, void **out_buf) +{ + struct smb2_compression_hdr *hdr; + unsigned int pdu_size = get_rfc1002_len(request_buf); + u32 orig_size, offset, out_size; + u32 max_allowed_pdu_size; + char *out; + int rc; + + if (pdu_size < sizeof(struct smb2_compression_hdr)) + return -EINVAL; + + if (conn->dialect != SMB311_PROT_ID || + conn->compress_algorithm == SMB3_COMPRESS_NONE) + return -EINVAL; + + hdr = smb_get_msg(request_buf); + if (hdr->ProtocolId != SMB2_COMPRESSION_TRANSFORM_ID) + return -EINVAL; + + orig_size = le32_to_cpu(hdr->OriginalCompressedSegmentSize); + /* + * For chained transforms the top-level header is only eight bytes; the + * Flags field overlays the first payload header. Reject unknown Flags + * and unnegotiated chained mode before allocating the output buffer. + */ + if (hdr->Flags == cpu_to_le16(SMB2_COMPRESSION_FLAG_CHAINED)) { + if (!conn->compress_chained) + return -EINVAL; + out_size = orig_size; + } else if (hdr->Flags == cpu_to_le16(SMB2_COMPRESSION_FLAG_NONE)) { + offset = le32_to_cpu(hdr->Offset); + if (offset > pdu_size - sizeof(*hdr) || + check_add_overflow(orig_size, offset, &out_size)) + return -EINVAL; + } else { + return -EINVAL; + } + + max_allowed_pdu_size = ksmbd_max_allowed_pdu_size(conn); + if (out_size < sizeof(struct smb2_pdu) || + out_size > max_allowed_pdu_size || + out_size > MAX_STREAM_PROT_LEN) + return -EINVAL; + + out = kvmalloc(out_size + 4 + 1, KSMBD_DEFAULT_GFP); + if (!out) + return -ENOMEM; + + *(__be32 *)out = cpu_to_be32(out_size); + rc = smb_compression_decompress(conn->compress_algorithm, + conn->compress_chained, + conn->compress_pattern, + (char *)hdr, pdu_size, out + 4, out_size); + if (rc) { + kvfree(out); + return rc; + } + + *out_buf = out; + return 0; +} + +/** + * ksmbd_decompress_request() - replace a compressed request with its SMB2 PDU + * @conn: connection which owns the current RFC1002 request buffer + * + * Derive the uncompressed size from the transform variant, enforce ksmbd's + * normal message limits, and ask the common decoder to validate every payload. + * On success, replace conn->request_buf with a regular RFC1002-framed SMB2 + * message so the rest of the request path needs no compression awareness. + * + * Return: 0 on success, otherwise a negative errno. + */ +int ksmbd_decompress_request(struct ksmbd_conn *conn) +{ + void *out_buf; + int rc; + + rc = __ksmbd_decompress_request(conn, conn->request_buf, &out_buf); + if (rc) + return rc; + + kvfree(conn->request_buf); + conn->request_buf = out_buf; + return 0; +} + +/** + * ksmbd_decompress_work_request() - decompress an encrypted work request + * @work: work item whose request buffer contains a compression transform + * + * SMB3 encrypts a compressed message by applying compression first and + * encryption second. The receive loop can therefore only decode the + * compression transform before work allocation for an unencrypted request; + * an encrypted request must be decompressed after its encryption layer has + * been removed. + * + * Return: 0 on success, otherwise a negative errno. + */ +int ksmbd_decompress_work_request(struct ksmbd_work *work) +{ + void *out_buf; + int rc; + + rc = __ksmbd_decompress_request(work->conn, work->request_buf, + &out_buf); + if (rc) + return rc; + + kvfree(work->request_buf); + work->request_buf = out_buf; + return 0; +} + +/** + * ksmbd_compress_response() - compress an eligible ksmbd response + * @work: request work item containing the response iov + * + * Compression transforms describe one contiguous SMB2 message, while ksmbd + * builds responses from multiple iov entries. Flatten the response first, + * produce the negotiated transform, and replace the response iov only when the + * result is smaller than the original message. + * + * Encrypted and compound responses are intentionally left unchanged. The + * caller may still continue sending the original response when this function + * returns zero. + * + * Return: 1 if the response was replaced, 0 if compression was skipped, or a + * negative errno on failure. + */ +int ksmbd_compress_response(struct ksmbd_work *work) +{ + struct smb2_compression_hdr *chdr; + struct smb2_hdr *req_hdr; + u32 src_len, dst_len, compressed_pdu_len, max_dst_len; + u8 *src = NULL, *out = NULL, *p; + int i, rc; + + if (!work->compress_response || work->encrypted || + work->conn->compress_algorithm != SMB3_COMPRESS_LZ77) + return 0; + + req_hdr = smb_get_msg(work->request_buf); + if (req_hdr->NextCommand || work->next_smb2_rcv_hdr_off || + work->next_smb2_rsp_hdr_off) + return 0; + + src_len = get_rfc1002_len(work->iov[0].iov_base); + if (src_len < SMB_COMPRESS_MIN_LEN) + return 0; + + src = kvmalloc(src_len, KSMBD_DEFAULT_GFP); + if (!src) + return -ENOMEM; + + p = src; + /* iov[0] contains only the RFC1002 length; the SMB2 PDU starts at iov[1]. */ + for (i = 1; i < work->iov_cnt; i++) { + if (work->iov[i].iov_len > src + src_len - p) { + rc = -EINVAL; + goto out; + } + memcpy(p, work->iov[i].iov_base, work->iov[i].iov_len); + p += work->iov[i].iov_len; + } + if (p != src + src_len) { + rc = -EINVAL; + goto out; + } + + max_dst_len = smb_lz77_compressed_alloc_size(src_len) + + sizeof(struct smb2_compression_hdr) + + 3 * sizeof(struct smb2_compression_payload_hdr) + + 2 * sizeof(struct smb2_compression_pattern_v1); + out = kvzalloc(sizeof(__be32) + max_dst_len, + KSMBD_DEFAULT_GFP); + if (!out) { + rc = -ENOMEM; + goto out; + } + + if (work->conn->compress_chained) { + dst_len = max_dst_len; + rc = smb_compression_compress_chained(SMB3_COMPRESS_LZ77, + work->conn->compress_pattern, + src, src_len, + out + sizeof(__be32), + &dst_len); + if (rc == -EMSGSIZE || dst_len >= src_len) { + rc = 0; + goto out; + } + if (rc) + goto out; + compressed_pdu_len = dst_len; + } else { + /* + * Peers which did not negotiate chained compression still use + * the original 16-byte unchained transform format. + */ + dst_len = smb_lz77_compressed_alloc_size(src_len); + rc = smb_lz77_compress(src, src_len, + out + sizeof(__be32) + sizeof(*chdr), + &dst_len); + if (rc == -EMSGSIZE || + dst_len + sizeof(*chdr) >= src_len) { + rc = 0; + goto out; + } + if (rc) + goto out; + + compressed_pdu_len = sizeof(*chdr) + dst_len; + chdr = (struct smb2_compression_hdr *)(out + sizeof(__be32)); + chdr->ProtocolId = SMB2_COMPRESSION_TRANSFORM_ID; + chdr->OriginalCompressedSegmentSize = cpu_to_le32(src_len); + chdr->CompressionAlgorithm = SMB3_COMPRESS_LZ77; + chdr->Flags = cpu_to_le16(SMB2_COMPRESSION_FLAG_NONE); + chdr->Offset = 0; + } + + *(__be32 *)out = cpu_to_be32(compressed_pdu_len); + + /* + * Keep the transform in work->compress_buf until send completion. + * Existing response iovs can then be replaced without changing their + * individual ownership rules. + */ + work->compress_buf = out; + work->iov[0].iov_base = out; + work->iov[0].iov_len = sizeof(__be32); + work->iov[1].iov_base = out + sizeof(__be32); + work->iov[1].iov_len = compressed_pdu_len; + work->iov_cnt = 2; + work->iov_idx = 1; + out = NULL; + rc = 1; +out: + kvfree(out); + kvfree(src); + return rc; +} diff --git a/fs/smb/server/compress.h b/fs/smb/server/compress.h new file mode 100644 index 000000000..13df2eb22 --- /dev/null +++ b/fs/smb/server/compress.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * SMB2 compression support for ksmbd. + * + * Copyright (C) 2026 Namjae Jeon + */ +#ifndef __KSMBD_COMPRESS_H__ +#define __KSMBD_COMPRESS_H__ + +#include "connection.h" +#include "../common/compress/compress.h" + +int ksmbd_decompress_request(struct ksmbd_conn *conn); +int ksmbd_decompress_work_request(struct ksmbd_work *work); +int ksmbd_compress_response(struct ksmbd_work *work); + +#endif /* __KSMBD_COMPRESS_H__ */ diff --git a/fs/smb/server/connection.c b/fs/smb/server/connection.c new file mode 100644 index 000000000..91fdd1ddc --- /dev/null +++ b/fs/smb/server/connection.c @@ -0,0 +1,860 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2016 Namjae Jeon + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#include +#include +#include + +#include "server.h" +#include "smb_common.h" +#include "mgmt/ksmbd_ida.h" +#include "mgmt/user_session.h" +#include "connection.h" +#include "compress.h" +#include "transport_tcp.h" +#include "transport_rdma.h" +#include "misc.h" + +static DEFINE_MUTEX(init_lock); + +static struct ksmbd_conn_ops default_conn_ops; + +DEFINE_HASHTABLE(conn_list, CONN_HASH_BITS); +DECLARE_RWSEM(conn_list_lock); + +#ifdef CONFIG_PROC_FS +static struct proc_dir_entry *proc_clients; + +static const char *ksmbd_conn_state_string(struct ksmbd_conn *conn) +{ + switch (READ_ONCE(conn->status)) { + case KSMBD_SESS_NEW: + return "new"; + case KSMBD_SESS_GOOD: + return "good"; + case KSMBD_SESS_EXITING: + return "exiting"; + case KSMBD_SESS_NEED_RECONNECT: + return "reconnect"; + case KSMBD_SESS_NEED_NEGOTIATE: + return "negotiate"; + case KSMBD_SESS_NEED_SETUP: + return "setup"; + case KSMBD_SESS_RELEASING: + return "releasing"; + default: + return "unknown"; + } +} + +static const char *ksmbd_conn_transport_string(struct ksmbd_conn *conn) +{ + if (conn->transport->ops->rdma_read || conn->transport->ops->rdma_write) + return "smbdirect"; + return "tcp"; +} + +static void proc_show_conn_feature(struct seq_file *m, bool *separator, + bool enabled, const char *name) +{ + if (!enabled) + return; + seq_printf(m, "%s%s", *separator ? "," : "", name); + *separator = true; +} + +static void proc_show_conn_features(struct seq_file *m, + struct ksmbd_conn *conn) +{ + bool separator = false; + + proc_show_conn_feature(m, &separator, + conn->sign || conn->signing_negotiated, "sign"); + proc_show_conn_feature(m, &separator, conn->cipher_type, "encrypt"); + proc_show_conn_feature(m, &separator, + conn->compress_algorithm != SMB3_COMPRESS_NONE, + "compress"); + proc_show_conn_feature(m, &separator, conn->rdma_transform_ids, + "rdma-transform"); + proc_show_conn_feature(m, &separator, conn->posix_ext_supported, "posix"); + if (!separator) + seq_puts(m, "none"); +} + +static int proc_show_clients(struct seq_file *m, void *v) +{ + struct ksmbd_conn *conn; + struct timespec64 now, t; + int i; + + down_read(&conn_list_lock); + hash_for_each(conn_list, i, conn, hlist) { + unsigned int outstanding_credits, total_credits; + unsigned long id; + void *entry; + unsigned int sessions = 0; + + jiffies_to_timespec64(jiffies - conn->last_active, &t); + ktime_get_real_ts64(&now); + t = timespec64_sub(now, t); + + spin_lock(&conn->credits_lock); + outstanding_credits = conn->outstanding_credits; + total_credits = conn->total_credits; + spin_unlock(&conn->credits_lock); + + rcu_read_lock(); + xa_for_each(&conn->sessions, id, entry) + sessions++; + rcu_read_unlock(); +#if IS_ENABLED(CONFIG_IPV6) + if (!conn->inet_addr) + seq_printf(m, "client:\t%pI6c\n", &conn->inet6_addr); + else +#endif + seq_printf(m, "client:\t%pI4\n", &conn->inet_addr); + seq_printf(m, "transport:\t%s\n", ksmbd_conn_transport_string(conn)); + seq_printf(m, "state:\t%s\n", ksmbd_conn_state_string(conn)); + seq_printf(m, "dialect:\t0x%04x\n", conn->dialect); + seq_printf(m, "credits:\t%u/%u\n", outstanding_credits, + total_credits); + seq_printf(m, "sessions:\t%u\n", sessions); + seq_printf(m, "open_files:\t%d\n", + atomic_read(&conn->stats.open_files_count)); + seq_printf(m, "requests:\t%lld\n", + atomic64_read(&conn->stats.request_served)); + seq_puts(m, "features:\t"); + proc_show_conn_features(m, conn); + seq_printf(m, "\nlast_active:\t%ptT\n\n", &t); + } + up_read(&conn_list_lock); + return 0; +} + +static int create_proc_clients(void) +{ + proc_clients = ksmbd_proc_create("clients", + proc_show_clients, NULL); + if (!proc_clients) + return -ENOMEM; + return 0; +} + +static void delete_proc_clients(void) +{ + if (proc_clients) { + proc_remove(proc_clients); + proc_clients = NULL; + } +} +#else +static int create_proc_clients(void) { return 0; } +static void delete_proc_clients(void) {} +#endif + +static struct workqueue_struct *ksmbd_conn_wq; + +int ksmbd_conn_wq_init(void) +{ + ksmbd_conn_wq = alloc_workqueue("ksmbd-conn-release", + WQ_UNBOUND | WQ_MEM_RECLAIM, 0); + if (!ksmbd_conn_wq) + return -ENOMEM; + return 0; +} + +void ksmbd_conn_wq_destroy(void) +{ + if (ksmbd_conn_wq) { + destroy_workqueue(ksmbd_conn_wq); + ksmbd_conn_wq = NULL; + } +} + +/* + * __ksmbd_conn_release_work() - perform the final, once-per-struct cleanup + * of a ksmbd_conn whose refcount has just dropped to zero. + * + * This is the common release path used by ksmbd_conn_put() for the embedded + * state that outlives the connection thread: async_ida and the attached + * transport (which owns the socket and iov for TCP). Called from a workqueue + * so that sleep-allowed teardown (sock_release -> tcp_close -> + * lock_sock_nested) never runs from an RCU softirq callback (free_opinfo_rcu) + * or any other non-sleeping putter context. + */ +static void __ksmbd_conn_release_work(struct work_struct *work) +{ + struct ksmbd_conn *conn = + container_of(work, struct ksmbd_conn, release_work); + + ida_destroy(&conn->async_ida); + conn->transport->ops->free_transport(conn->transport); + kfree_sensitive(conn); +} + +/** + * ksmbd_conn_get() - take a reference on @conn and return it. + * + * @conn: connection instance to get a reference to + * + * Returns @conn unchanged so callers can write + * "fp->conn = ksmbd_conn_get(work->conn);" in one expression. Returns NULL + * if @conn is NULL. + */ +struct ksmbd_conn *ksmbd_conn_get(struct ksmbd_conn *conn) +{ + if (!conn) + return NULL; + + atomic_inc(&conn->refcnt); + return conn; +} + +/** + * ksmbd_conn_put() - drop a reference and, if it was the last, queue the + * release onto ksmbd_conn_wq so it runs from process context. + * + * @conn: connection instance to put a reference to + * + * Callable from any context including RCU softirq callbacks and non-sleeping + * locks; the actual release is deferred to the workqueue. ksmbd_conn_wq is + * created in ksmbd_server_init() before any conn can be allocated and is + * destroyed in ksmbd_server_exit() after rcu_barrier(), so it is always + * non-NULL while a conn reference is held. + */ +void ksmbd_conn_put(struct ksmbd_conn *conn) +{ + if (!conn) + return; + + if (atomic_dec_and_test(&conn->refcnt)) + queue_work(ksmbd_conn_wq, &conn->release_work); +} + +/** + * ksmbd_conn_free() - free resources of the connection instance + * + * @conn: connection instance to be cleaned up + * + * During the thread termination, the corresponding conn instance + * resources(sock/memory) are released and finally the conn object is freed. + */ +void ksmbd_conn_free(struct ksmbd_conn *conn) +{ + down_write(&conn_list_lock); + hash_del(&conn->hlist); + up_write(&conn_list_lock); + + /* + * request_buf / preauth_info / mechToken are only ever accessed by the + * connection handler thread that owns @conn. ksmbd_conn_free() is + * called from the transport free_transport() path when that thread is + * exiting, so it is safe to release them unconditionally even when + * ksmbd_conn_put() below is not the final putter (oplock / ksmbd_file + * holders only retain the conn pointer, not these per-thread buffers). + */ + xa_destroy(&conn->sessions); + kvfree(conn->request_buf); + kfree_sensitive(conn->preauth_info); + kfree(conn->mechToken); + ksmbd_preauth_session_destroy(conn); + ksmbd_conn_put(conn); +} + +/** + * ksmbd_conn_alloc() - initialize a new connection instance + * + * Return: ksmbd_conn struct on success, otherwise NULL + */ +struct ksmbd_conn *ksmbd_conn_alloc(void) +{ + struct ksmbd_conn *conn; + + conn = kzalloc_obj(struct ksmbd_conn, KSMBD_DEFAULT_GFP); + if (!conn) + return NULL; + + conn->need_neg = true; + ksmbd_conn_set_new(conn); + conn->local_nls = load_nls("utf8"); + if (!conn->local_nls) + conn->local_nls = load_nls_default(); + if (IS_ENABLED(CONFIG_UNICODE)) + conn->um = utf8_load(UNICODE_AGE(12, 1, 0)); + else + conn->um = ERR_PTR(-EOPNOTSUPP); + if (IS_ERR(conn->um)) + conn->um = NULL; + INIT_WORK(&conn->release_work, __ksmbd_conn_release_work); + atomic_set(&conn->req_running, 0); + atomic_set(&conn->r_count, 0); + atomic_set(&conn->refcnt, 1); + conn->total_credits = 1; + conn->outstanding_credits = 0; + + /* + * The command sequence window starts as the set { 0 } when the + * connection is established. + */ + conn->seq_low = 0; + conn->seq_high = 1; + __set_bit(0, conn->seq_bitmap); + + init_waitqueue_head(&conn->req_running_q); + init_waitqueue_head(&conn->r_count_q); + INIT_LIST_HEAD(&conn->requests); + INIT_LIST_HEAD(&conn->async_requests); + INIT_LIST_HEAD(&conn->preauth_sess_table); + spin_lock_init(&conn->request_lock); + spin_lock_init(&conn->credits_lock); + ida_init(&conn->async_ida); + xa_init(&conn->sessions); + + spin_lock_init(&conn->llist_lock); + INIT_LIST_HEAD(&conn->lock_list); + + init_rwsem(&conn->session_lock); + + return conn; +} + +bool ksmbd_conn_lookup_dialect(struct ksmbd_conn *c) +{ + struct ksmbd_conn *t; + int bkt; + bool ret = false; + + down_read(&conn_list_lock); + hash_for_each(conn_list, bkt, t, hlist) { + if (memcmp(t->ClientGUID, c->ClientGUID, SMB2_CLIENT_GUID_SIZE)) + continue; + + ret = true; + break; + } + up_read(&conn_list_lock); + return ret; +} + +void ksmbd_conn_enqueue_request(struct ksmbd_work *work) +{ + struct ksmbd_conn *conn = work->conn; + struct list_head *requests_queue = NULL; + + if (conn->ops->get_cmd_val(work) != SMB2_CANCEL_HE) + requests_queue = &conn->requests; + + atomic_inc(&conn->req_running); + if (requests_queue) { + spin_lock(&conn->request_lock); + list_add_tail(&work->request_entry, requests_queue); + spin_unlock(&conn->request_lock); + } +} + +void ksmbd_conn_try_dequeue_request(struct ksmbd_work *work) +{ + struct ksmbd_conn *conn = work->conn; + + atomic_dec(&conn->req_running); + if (waitqueue_active(&conn->req_running_q)) + wake_up(&conn->req_running_q); + + if (list_empty(&work->request_entry) && + list_empty(&work->async_request_entry)) + return; + + spin_lock(&conn->request_lock); + list_del_init(&work->request_entry); + spin_unlock(&conn->request_lock); + if (work->asynchronous) + release_async_work(work); + + wake_up_all(&conn->req_running_q); +} + +static void ksmbd_conn_cancel_async_requests(struct ksmbd_conn *conn) +{ + struct ksmbd_work *work, *tmp; + + ksmbd_debug(CONN, "Cancel pending async requests on releasing connection\n"); + spin_lock(&conn->request_lock); + list_for_each_entry_safe(work, tmp, &conn->async_requests, + async_request_entry) { + if (work->state != KSMBD_WORK_ACTIVE) + continue; + + ksmbd_debug(CONN, "Cancel async request id %d\n", + work->async_id); + work->state = KSMBD_WORK_CANCELLED; + if (work->cancel_fn) + work->cancel_fn(work->cancel_argv); + } + spin_unlock(&conn->request_lock); +} + +void ksmbd_conn_lock(struct ksmbd_conn *conn) +{ + mutex_lock(&conn->srv_mutex); +} + +void ksmbd_conn_unlock(struct ksmbd_conn *conn) +{ + mutex_unlock(&conn->srv_mutex); +} + +static bool ksmbd_session_is_bound_to_conn(struct ksmbd_session *sess, + struct ksmbd_conn *conn) +{ + bool found; + + rcu_read_lock(); + found = xa_load(&conn->sessions, sess->id) == sess; + rcu_read_unlock(); + if (found) + return true; + + down_read(&sess->chann_lock); + found = xa_load(&sess->ksmbd_chann_list, (long)conn); + up_read(&sess->chann_lock); + return found; +} + +void ksmbd_all_conn_set_status(struct ksmbd_session *sess, u32 status) +{ + struct ksmbd_conn *conn; + int bkt; + + down_read(&conn_list_lock); + hash_for_each(conn_list, bkt, conn, hlist) { + if (ksmbd_session_is_bound_to_conn(sess, conn)) { + spin_lock(&conn->request_lock); + if (!ksmbd_conn_exiting(conn) && + !ksmbd_conn_releasing(conn)) + WRITE_ONCE(conn->status, status); + spin_unlock(&conn->request_lock); + } + } + up_read(&conn_list_lock); +} + +void ksmbd_conn_abort(struct ksmbd_conn *conn) +{ + bool shutdown = false; + + spin_lock(&conn->request_lock); + if (!ksmbd_conn_exiting(conn) && !ksmbd_conn_releasing(conn)) { + ksmbd_conn_set_exiting(conn); + shutdown = true; + } + spin_unlock(&conn->request_lock); + wake_up_all(&conn->req_running_q); + + if (shutdown && conn->transport->ops->shutdown) + conn->transport->ops->shutdown(conn->transport); +} + +void ksmbd_conn_wait_idle(struct ksmbd_conn *conn) +{ + wait_event(conn->req_running_q, atomic_read(&conn->req_running) < 2); +} + +int ksmbd_conn_wait_idle_sess(struct ksmbd_conn *curr_conn, + struct ksmbd_session *sess) +{ + struct ksmbd_conn *conn; + int rc, retry_count = 0, max_timeout = 120; + int rcount, bkt; + +retry_idle: + if (retry_count >= max_timeout) + return -EIO; + + down_read(&conn_list_lock); + hash_for_each(conn_list, bkt, conn, hlist) { + if (ksmbd_session_is_bound_to_conn(sess, conn)) { + rcount = (conn == curr_conn) ? 2 : 1; + if (atomic_read(&conn->req_running) >= rcount) { + rc = wait_event_timeout(conn->req_running_q, + atomic_read(&conn->req_running) < rcount, + HZ); + if (!rc) { + up_read(&conn_list_lock); + retry_count++; + goto retry_idle; + } + } + } + } + up_read(&conn_list_lock); + + return 0; +} + +static int __ksmbd_conn_write(struct ksmbd_work *work, + struct ksmbd_transport_write *tx) +{ + struct ksmbd_conn *conn = work->conn; + int sent; + + if (!work->response_buf) { + pr_err("NULL response header\n"); + return -EINVAL; + } + + if (work->send_no_response) + return 0; + + if (!work->iov_idx) + return -EINVAL; + + tx->iov = work->iov; + tx->iov_cnt = work->iov_cnt; + tx->size = get_rfc1002_len(work->iov[0].iov_base) + 4; + tx->need_invalidate_rkey = work->need_invalidate_rkey; + tx->remote_key = work->remote_key; + + ksmbd_conn_lock(conn); + sent = conn->transport->ops->writev(conn->transport, tx); + ksmbd_conn_unlock(conn); + + if (sent < 0) { + pr_err("Failed to send message: %d\n", sent); + return sent; + } + + return 0; +} + +int ksmbd_conn_write(struct ksmbd_work *work) +{ + struct ksmbd_transport_write tx = {}; + + return __ksmbd_conn_write(work, &tx); +} + +int ksmbd_conn_write_eor(struct ksmbd_work *work) +{ + struct ksmbd_transport_write tx = { + .msg_flags = MSG_EOR, + }; + + return __ksmbd_conn_write(work, &tx); +} + +int ksmbd_conn_rdma_read(struct ksmbd_conn *conn, + void *buf, unsigned int buflen, + struct smbdirect_buffer_descriptor_v1 *desc, + unsigned int desc_len) +{ + int ret = -EINVAL; + + if (conn->transport->ops->rdma_read) + ret = conn->transport->ops->rdma_read(conn->transport, + buf, buflen, + desc, desc_len); + return ret; +} + +int ksmbd_conn_rdma_write(struct ksmbd_conn *conn, + void *buf, unsigned int buflen, + struct smbdirect_buffer_descriptor_v1 *desc, + unsigned int desc_len) +{ + int ret = -EINVAL; + + if (conn->transport->ops->rdma_write) + ret = conn->transport->ops->rdma_write(conn->transport, + buf, buflen, + desc, desc_len); + return ret; +} + +bool ksmbd_conn_alive(struct ksmbd_conn *conn) +{ + if (!ksmbd_server_running()) + return false; + + if (ksmbd_conn_exiting(conn)) + return false; + + if (kthread_should_stop()) + return false; + + if (atomic_read(&conn->stats.open_files_count) > 0) + return true; + + /* + * Stop current session if the time that get last request from client + * is bigger than deadtime user configured and opening file count is + * zero. + */ + if (server_conf.deadtime > 0 && + time_after(jiffies, conn->last_active + server_conf.deadtime)) { + ksmbd_debug(CONN, "No response from client in %lu minutes\n", + server_conf.deadtime / SMB_ECHO_INTERVAL); + return false; + } + return true; +} + +/* "+2" for BCC field (ByteCount, 2 bytes) */ +#define SMB1_MIN_SUPPORTED_PDU_SIZE (sizeof(struct smb_hdr) + 2) +#define SMB2_MIN_SUPPORTED_PDU_SIZE (sizeof(struct smb2_pdu)) +#define SMB2_TRANSFORM_MIN_SUPPORTED_PDU_SIZE \ + (sizeof(struct smb2_transform_hdr) + sizeof(struct smb2_hdr)) + +/** + * ksmbd_conn_handler_loop() - session thread to listen on new smb requests + * @p: connection instance + * + * One thread each per connection + * + * Return: 0 on success + */ +int ksmbd_conn_handler_loop(void *p) +{ + struct ksmbd_conn *conn = (struct ksmbd_conn *)p; + struct ksmbd_transport *t = conn->transport; + unsigned int pdu_size, max_allowed_pdu_size, max_req; + __le32 proto; + char hdr_buf[4] = {0,}; + int size; + + mutex_init(&conn->srv_mutex); + __module_get(THIS_MODULE); + + max_req = server_conf.max_inflight_req; + conn->last_active = jiffies; + set_freezable(); + while (ksmbd_conn_alive(conn)) { + if (try_to_freeze()) + continue; + + kvfree(conn->request_buf); + conn->request_buf = NULL; + +recheck: + if (atomic_read(&conn->req_running) + 1 > max_req) { + wait_event_interruptible(conn->req_running_q, + atomic_read(&conn->req_running) < max_req); + goto recheck; + } + + size = t->ops->read(t, hdr_buf, sizeof(hdr_buf), -1); + if (size != sizeof(hdr_buf)) + break; + + pdu_size = get_rfc1002_len(hdr_buf); + ksmbd_debug(CONN, "RFC1002 header %u bytes\n", pdu_size); + + max_allowed_pdu_size = ksmbd_max_allowed_pdu_size(conn); + + if (pdu_size > max_allowed_pdu_size) { + pr_err_ratelimited("PDU length(%u) exceeded maximum allowed pdu size(%u) on connection(%d)\n", + pdu_size, max_allowed_pdu_size, + READ_ONCE(conn->status)); + break; + } + + /* + * Check maximum pdu size(0x00FFFFFF). + */ + if (pdu_size > MAX_STREAM_PROT_LEN) + break; + + if (pdu_size < SMB1_MIN_SUPPORTED_PDU_SIZE) + break; + + /* 4 for rfc1002 length field */ + /* 1 for implied bcc[0] */ + size = pdu_size + 4 + 1; + conn->request_buf = kvmalloc(size, KSMBD_DEFAULT_GFP); + if (!conn->request_buf) + break; + + memcpy(conn->request_buf, hdr_buf, sizeof(hdr_buf)); + + /* + * We already read 4 bytes to find out PDU size, now + * read in PDU + */ + size = t->ops->read(t, conn->request_buf + 4, pdu_size, 2); + if (size < 0) { + pr_err("sock_read failed: %d\n", size); + break; + } + + if (size != pdu_size) { + pr_err("PDU error. Read: %d, Expected: %d\n", + size, pdu_size); + continue; + } + + if (((struct smb2_hdr *)smb_get_msg(conn->request_buf))->ProtocolId == + SMB2_COMPRESSION_TRANSFORM_ID) { + /* + * Convert the transform into a normal RFC1002-framed SMB2 + * request before protocol validation and work allocation. + */ + if (ksmbd_decompress_request(conn)) + break; + pdu_size = get_rfc1002_len(conn->request_buf); + } + + if (!ksmbd_smb_request(conn)) + break; + + proto = *(__le32 *)smb_get_msg(conn->request_buf); + if (proto == SMB2_PROTO_NUMBER && + pdu_size < SMB2_MIN_SUPPORTED_PDU_SIZE) + break; + + if (proto == SMB2_TRANSFORM_PROTO_NUM && + pdu_size < SMB2_TRANSFORM_MIN_SUPPORTED_PDU_SIZE) + break; + + if (!default_conn_ops.process_fn) { + pr_err("No connection request callback\n"); + break; + } + + if (default_conn_ops.process_fn(conn)) { + pr_err("Cannot handle request\n"); + break; + } + } + + ksmbd_conn_set_releasing(conn); + ksmbd_conn_cancel_async_requests(conn); + /* Wait till all reference dropped to the Server object*/ + ksmbd_debug(CONN, "Wait for all pending requests(%d)\n", atomic_read(&conn->r_count)); + wait_event(conn->r_count_q, atomic_read(&conn->r_count) == 0); + + if (IS_ENABLED(CONFIG_UNICODE)) + utf8_unload(conn->um); + unload_nls(conn->local_nls); + if (default_conn_ops.terminate_fn) + default_conn_ops.terminate_fn(conn); + t->ops->disconnect(t); + module_put(THIS_MODULE); + return 0; +} + +void ksmbd_conn_init_server_callbacks(struct ksmbd_conn_ops *ops) +{ + default_conn_ops.process_fn = ops->process_fn; + default_conn_ops.terminate_fn = ops->terminate_fn; +} + +void ksmbd_conn_r_count_inc(struct ksmbd_conn *conn) +{ + atomic_inc(&conn->r_count); +} + +void ksmbd_conn_r_count_dec(struct ksmbd_conn *conn) +{ + /* + * Checking waitqueue to dropping pending requests on + * disconnection. waitqueue_active is safe because it + * uses atomic operation for condition. + */ + atomic_inc(&conn->refcnt); + if (!atomic_dec_return(&conn->r_count) && waitqueue_active(&conn->r_count_q)) + wake_up(&conn->r_count_q); + + ksmbd_conn_put(conn); +} + +int ksmbd_conn_transport_init(void) +{ + int ret; + + mutex_lock(&init_lock); + ret = ksmbd_tcp_init(); + if (ret) { + pr_err("Failed to init TCP subsystem: %d\n", ret); + goto out; + } + + ret = ksmbd_rdma_init(); + if (ret) { + pr_err("Failed to init RDMA subsystem: %d\n", ret); + goto out; + } +out: + mutex_unlock(&init_lock); + if (create_proc_clients()) + pr_warn("Unable to create clients procfs entry\n"); + return ret; +} + +static void stop_sessions(void) +{ + struct ksmbd_conn *conn, *target; + struct ksmbd_transport *t; + bool any; + int bkt; + + /* + * Serialised via init_lock; no concurrent stop_sessions() can + * touch conn->stop_called, so writing it under the read lock is + * safe. + */ +again: + target = NULL; + any = false; + down_read(&conn_list_lock); + hash_for_each(conn_list, bkt, conn, hlist) { + any = true; + if (conn->stop_called) + continue; + atomic_inc(&conn->refcnt); + conn->stop_called = true; + /* + * Mark the connection EXITING while still holding the + * read lock so the selection and the status transition + * happen together. Do not regress a connection that has + * already advanced to RELEASING on its own (e.g. the + * handler exited its receive loop for an unrelated + * reason). + */ + spin_lock(&conn->request_lock); + if (!ksmbd_conn_releasing(conn)) + ksmbd_conn_set_exiting(conn); + spin_unlock(&conn->request_lock); + target = conn; + break; + } + up_read(&conn_list_lock); + + if (target) { + t = target->transport; + if (t->ops->shutdown) + t->ops->shutdown(t); + if (atomic_dec_and_test(&target->refcnt)) { + ida_destroy(&target->async_ida); + t->ops->free_transport(t); + kfree_sensitive(target); + } + goto again; + } + + if (any) { + msleep(100); + goto again; + } +} + +void ksmbd_conn_transport_destroy(void) +{ + delete_proc_clients(); + mutex_lock(&init_lock); + ksmbd_tcp_destroy(); + ksmbd_rdma_stop_listening(); + stop_sessions(); + mutex_unlock(&init_lock); +} diff --git a/fs/smb/server/connection.h b/fs/smb/server/connection.h new file mode 100644 index 000000000..63484c8ef --- /dev/null +++ b/fs/smb/server/connection.h @@ -0,0 +1,320 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#ifndef __KSMBD_CONNECTION_H__ +#define __KSMBD_CONNECTION_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "smb_common.h" +#include "ksmbd_work.h" + +struct smbdirect_buffer_descriptor_v1; +struct ksmbd_session; + +#define KSMBD_SOCKET_BACKLOG 16 + +/* + * Size of the per-connection SMB2 command sequence window. This mirrors + * SMB2_MAX_CREDITS, the maximum number of credits (and therefore the + * maximum number of outstanding sequence numbers) that can be granted on + * a connection. It must be a power of two so the window can be indexed as + * a ring. + */ +#define KSMBD_CMD_SEQ_WINDOW 8192 + +enum { + KSMBD_SESS_NEW = 0, + KSMBD_SESS_GOOD, + KSMBD_SESS_EXITING, + KSMBD_SESS_NEED_RECONNECT, + KSMBD_SESS_NEED_NEGOTIATE, + KSMBD_SESS_NEED_SETUP, + KSMBD_SESS_RELEASING +}; + +struct ksmbd_conn_stats { + atomic_t open_files_count; + atomic64_t request_served; +}; + +struct ksmbd_transport; + +struct ksmbd_conn { + struct smb_version_values *vals; + struct smb_version_ops *ops; + struct smb_version_cmds *cmds; + unsigned int max_cmds; + struct mutex srv_mutex; + int status; + unsigned int cli_cap; + bool stop_called; + union { + __be32 inet_addr; +#if IS_ENABLED(CONFIG_IPV6) + u8 inet6_addr[16]; +#endif + }; + unsigned int inet_hash; + char *request_buf; + struct ksmbd_transport *transport; + struct nls_table *local_nls; + struct unicode_map *um; + struct hlist_node hlist; + struct rw_semaphore session_lock; + /* smb session 1 per user */ + struct xarray sessions; + unsigned long last_active; + /* How many request are running currently */ + atomic_t req_running; + /* References which are made for this Server object*/ + atomic_t r_count; + unsigned int total_credits; + unsigned int outstanding_credits; + spinlock_t credits_lock; + /* + * Connection command sequence window. [seq_low, seq_high) is the + * range of granted sequence numbers (message IDs). seq_bitmap marks + * the ones in that range that have been granted but + * not yet consumed by a received request. All three are protected by + * credits_lock. + */ + u64 seq_low; + u64 seq_high; + DECLARE_BITMAP(seq_bitmap, KSMBD_CMD_SEQ_WINDOW); + wait_queue_head_t req_running_q; + wait_queue_head_t r_count_q; + /* Lock to protect requests list*/ + spinlock_t request_lock; + struct list_head requests; + struct list_head async_requests; + int connection_type; + struct ksmbd_conn_stats stats; + char ClientGUID[SMB2_CLIENT_GUID_SIZE]; + struct ntlmssp_auth ntlmssp; + + spinlock_t llist_lock; + struct list_head lock_list; + + struct preauth_integrity_info *preauth_info; + + bool need_neg; + unsigned int auth_mechs; + unsigned int preferred_auth_mech; + bool sign; + bool use_spnego:1; + __u16 cli_sec_mode; + __u16 srv_sec_mode; + /* dialect index that server chose */ + __u16 dialect; + + char *mechToken; + unsigned int mechTokenLen; + + struct ksmbd_conn_ops *conn_ops; + + /* Preauth Session Table */ + struct list_head preauth_sess_table; + + struct sockaddr_storage peer_addr; + + /* Identifier for async message */ + struct ida async_ida; + + __le16 cipher_type; + __le16 compress_algorithm; + /* Negotiated SMB 3.1.1 compression capabilities. */ + bool compress_chained; + bool compress_pattern; + /* Bitmap indexed by SMB2_RDMA_TRANSFORM_* IDs. */ + unsigned long rdma_transform_ids; + bool rdma_transform_negotiated; + bool posix_ext_supported; + bool signing_negotiated; + __le16 signing_algorithm; + bool binding; + atomic_t refcnt; + bool is_aapl; + bool aapl_readdir_attr; /* READDIR_ATTR negotiated */ + bool aapl_readdir_attr_v2; /* V2 specifically */ + struct work_struct release_work; +}; + +struct ksmbd_conn_ops { + int (*process_fn)(struct ksmbd_conn *conn); + int (*terminate_fn)(struct ksmbd_conn *conn); +}; + +struct ksmbd_transport_write { + struct kvec *iov; + int iov_cnt; + int size; + bool need_invalidate_rkey; + unsigned int remote_key; + int msg_flags; +}; + +struct ksmbd_transport_ops { + void (*disconnect)(struct ksmbd_transport *t); + void (*shutdown)(struct ksmbd_transport *t); + int (*read)(struct ksmbd_transport *t, char *buf, + unsigned int size, int max_retries); + int (*writev)(struct ksmbd_transport *t, + const struct ksmbd_transport_write *tx); + int (*rdma_read)(struct ksmbd_transport *t, + void *buf, unsigned int len, + struct smbdirect_buffer_descriptor_v1 *desc, + unsigned int desc_len); + int (*rdma_write)(struct ksmbd_transport *t, + void *buf, unsigned int len, + struct smbdirect_buffer_descriptor_v1 *desc, + unsigned int desc_len); + void (*free_transport)(struct ksmbd_transport *kt); +}; + +struct ksmbd_transport { + struct ksmbd_conn *conn; + const struct ksmbd_transport_ops *ops; +}; + +#define KSMBD_TCP_RECV_TIMEOUT (7 * HZ) +#define KSMBD_TCP_SEND_TIMEOUT (5 * HZ) +#define KSMBD_TCP_PEER_SOCKADDR(c) ((struct sockaddr *)&((c)->peer_addr)) + +#define CONN_HASH_BITS 12 +extern DECLARE_HASHTABLE(conn_list, CONN_HASH_BITS); +extern struct rw_semaphore conn_list_lock; + +bool ksmbd_conn_alive(struct ksmbd_conn *conn); +void ksmbd_conn_wait_idle(struct ksmbd_conn *conn); +int ksmbd_conn_wait_idle_sess(struct ksmbd_conn *curr_conn, + struct ksmbd_session *sess); +struct ksmbd_conn *ksmbd_conn_alloc(void); +void ksmbd_conn_free(struct ksmbd_conn *conn); +struct ksmbd_conn *ksmbd_conn_get(struct ksmbd_conn *conn); +void ksmbd_conn_put(struct ksmbd_conn *conn); +void ksmbd_conn_abort(struct ksmbd_conn *conn); +int ksmbd_conn_wq_init(void); +void ksmbd_conn_wq_destroy(void); +bool ksmbd_conn_lookup_dialect(struct ksmbd_conn *c); +int ksmbd_conn_write(struct ksmbd_work *work); +int ksmbd_conn_write_eor(struct ksmbd_work *work); +int ksmbd_conn_rdma_read(struct ksmbd_conn *conn, + void *buf, unsigned int buflen, + struct smbdirect_buffer_descriptor_v1 *desc, + unsigned int desc_len); +int ksmbd_conn_rdma_write(struct ksmbd_conn *conn, + void *buf, unsigned int buflen, + struct smbdirect_buffer_descriptor_v1 *desc, + unsigned int desc_len); +void ksmbd_conn_enqueue_request(struct ksmbd_work *work); +void ksmbd_conn_try_dequeue_request(struct ksmbd_work *work); +void ksmbd_conn_init_server_callbacks(struct ksmbd_conn_ops *ops); +int ksmbd_conn_handler_loop(void *p); +int ksmbd_conn_transport_init(void); +void ksmbd_conn_transport_destroy(void); +void ksmbd_conn_lock(struct ksmbd_conn *conn); +void ksmbd_conn_unlock(struct ksmbd_conn *conn); +void ksmbd_conn_r_count_inc(struct ksmbd_conn *conn); +void ksmbd_conn_r_count_dec(struct ksmbd_conn *conn); + +/* + * WARNING + * + * This is a hack. We will move status to a proper place once we land + * a multi-sessions support. + */ +static inline bool ksmbd_conn_new(struct ksmbd_conn *conn) +{ + return READ_ONCE(conn->status) == KSMBD_SESS_NEW; +} + +static inline bool ksmbd_conn_good(struct ksmbd_conn *conn) +{ + return READ_ONCE(conn->status) == KSMBD_SESS_GOOD; +} + +static inline unsigned int +ksmbd_max_allowed_pdu_size(struct ksmbd_conn *conn) +{ + if (ksmbd_conn_good(conn)) + return SMB3_MAX_MSGSIZE + conn->vals->max_write_size; + + return SMB3_MAX_MSGSIZE; +} + +static inline bool ksmbd_conn_need_negotiate(struct ksmbd_conn *conn) +{ + return READ_ONCE(conn->status) == KSMBD_SESS_NEED_NEGOTIATE; +} + +static inline bool ksmbd_conn_need_setup(struct ksmbd_conn *conn) +{ + return READ_ONCE(conn->status) == KSMBD_SESS_NEED_SETUP; +} + +static inline bool ksmbd_conn_need_reconnect(struct ksmbd_conn *conn) +{ + return READ_ONCE(conn->status) == KSMBD_SESS_NEED_RECONNECT; +} + +static inline bool ksmbd_conn_exiting(struct ksmbd_conn *conn) +{ + return READ_ONCE(conn->status) == KSMBD_SESS_EXITING; +} + +static inline bool ksmbd_conn_releasing(struct ksmbd_conn *conn) +{ + return READ_ONCE(conn->status) == KSMBD_SESS_RELEASING; +} + +static inline void ksmbd_conn_set_new(struct ksmbd_conn *conn) +{ + WRITE_ONCE(conn->status, KSMBD_SESS_NEW); +} + +static inline void ksmbd_conn_set_good(struct ksmbd_conn *conn) +{ + WRITE_ONCE(conn->status, KSMBD_SESS_GOOD); +} + +static inline void ksmbd_conn_set_need_negotiate(struct ksmbd_conn *conn) +{ + WRITE_ONCE(conn->status, KSMBD_SESS_NEED_NEGOTIATE); +} + +static inline void ksmbd_conn_set_need_setup(struct ksmbd_conn *conn) +{ + WRITE_ONCE(conn->status, KSMBD_SESS_NEED_SETUP); +} + +static inline void ksmbd_conn_set_need_reconnect(struct ksmbd_conn *conn) +{ + WRITE_ONCE(conn->status, KSMBD_SESS_NEED_RECONNECT); +} + +static inline void ksmbd_conn_set_exiting(struct ksmbd_conn *conn) +{ + WRITE_ONCE(conn->status, KSMBD_SESS_EXITING); +} + +static inline void ksmbd_conn_set_releasing(struct ksmbd_conn *conn) +{ + WRITE_ONCE(conn->status, KSMBD_SESS_RELEASING); +} + +void ksmbd_all_conn_set_status(struct ksmbd_session *sess, u32 status); +#endif /* __CONNECTION_H__ */ diff --git a/fs/smb/server/crypto_ctx.c b/fs/smb/server/crypto_ctx.c new file mode 100644 index 000000000..2fe7d3300 --- /dev/null +++ b/fs/smb/server/crypto_ctx.c @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2019 Samsung Electronics Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include + +#include "glob.h" +#include "crypto_ctx.h" + +struct crypto_ctx_list { + spinlock_t ctx_lock; + int avail_ctx; + struct list_head idle_ctx; + wait_queue_head_t ctx_wait; +}; + +static struct crypto_ctx_list ctx_list; + +static inline void free_aead(struct crypto_aead *aead) +{ + if (aead) + crypto_free_aead(aead); +} + +static struct crypto_aead *alloc_aead(int id) +{ + struct crypto_aead *tfm = NULL; + + switch (id) { + case CRYPTO_AEAD_AES_GCM: + tfm = crypto_alloc_aead("gcm(aes)", 0, 0); + break; + case CRYPTO_AEAD_AES_CCM: + tfm = crypto_alloc_aead("ccm(aes)", 0, 0); + break; + default: + pr_err("Does not support encrypt ahead(id : %d)\n", id); + return NULL; + } + + if (IS_ERR(tfm)) { + pr_err("Failed to alloc encrypt aead : %ld\n", PTR_ERR(tfm)); + return NULL; + } + + return tfm; +} + +static void ctx_free(struct ksmbd_crypto_ctx *ctx) +{ + int i; + + for (i = 0; i < CRYPTO_AEAD_MAX; i++) + free_aead(ctx->ccmaes[i]); + kfree(ctx); +} + +static struct ksmbd_crypto_ctx *ksmbd_find_crypto_ctx(void) +{ + struct ksmbd_crypto_ctx *ctx; + + while (1) { + spin_lock(&ctx_list.ctx_lock); + if (!list_empty(&ctx_list.idle_ctx)) { + ctx = list_entry(ctx_list.idle_ctx.next, + struct ksmbd_crypto_ctx, + list); + list_del(&ctx->list); + spin_unlock(&ctx_list.ctx_lock); + return ctx; + } + + if (ctx_list.avail_ctx > num_online_cpus()) { + spin_unlock(&ctx_list.ctx_lock); + wait_event(ctx_list.ctx_wait, + !list_empty(&ctx_list.idle_ctx)); + continue; + } + + ctx_list.avail_ctx++; + spin_unlock(&ctx_list.ctx_lock); + + ctx = kzalloc_obj(struct ksmbd_crypto_ctx, KSMBD_DEFAULT_GFP); + if (!ctx) { + spin_lock(&ctx_list.ctx_lock); + ctx_list.avail_ctx--; + spin_unlock(&ctx_list.ctx_lock); + wait_event(ctx_list.ctx_wait, + !list_empty(&ctx_list.idle_ctx)); + continue; + } + break; + } + return ctx; +} + +void ksmbd_release_crypto_ctx(struct ksmbd_crypto_ctx *ctx) +{ + if (!ctx) + return; + + spin_lock(&ctx_list.ctx_lock); + if (ctx_list.avail_ctx <= num_online_cpus()) { + list_add(&ctx->list, &ctx_list.idle_ctx); + spin_unlock(&ctx_list.ctx_lock); + wake_up(&ctx_list.ctx_wait); + return; + } + + ctx_list.avail_ctx--; + spin_unlock(&ctx_list.ctx_lock); + ctx_free(ctx); +} + +static struct ksmbd_crypto_ctx *____crypto_aead_ctx_find(int id) +{ + struct ksmbd_crypto_ctx *ctx; + + if (id >= CRYPTO_AEAD_MAX) + return NULL; + + ctx = ksmbd_find_crypto_ctx(); + if (ctx->ccmaes[id]) + return ctx; + + ctx->ccmaes[id] = alloc_aead(id); + if (ctx->ccmaes[id]) + return ctx; + ksmbd_release_crypto_ctx(ctx); + return NULL; +} + +struct ksmbd_crypto_ctx *ksmbd_crypto_ctx_find_gcm(void) +{ + return ____crypto_aead_ctx_find(CRYPTO_AEAD_AES_GCM); +} + +struct ksmbd_crypto_ctx *ksmbd_crypto_ctx_find_ccm(void) +{ + return ____crypto_aead_ctx_find(CRYPTO_AEAD_AES_CCM); +} + +void ksmbd_crypto_destroy(void) +{ + struct ksmbd_crypto_ctx *ctx; + + while (!list_empty(&ctx_list.idle_ctx)) { + ctx = list_entry(ctx_list.idle_ctx.next, + struct ksmbd_crypto_ctx, + list); + list_del(&ctx->list); + ctx_free(ctx); + } +} + +int ksmbd_crypto_create(void) +{ + struct ksmbd_crypto_ctx *ctx; + + spin_lock_init(&ctx_list.ctx_lock); + INIT_LIST_HEAD(&ctx_list.idle_ctx); + init_waitqueue_head(&ctx_list.ctx_wait); + ctx_list.avail_ctx = 1; + + ctx = kzalloc_obj(struct ksmbd_crypto_ctx, KSMBD_DEFAULT_GFP); + if (!ctx) + return -ENOMEM; + list_add(&ctx->list, &ctx_list.idle_ctx); + return 0; +} diff --git a/fs/smb/server/crypto_ctx.h b/fs/smb/server/crypto_ctx.h new file mode 100644 index 000000000..b22c6e086 --- /dev/null +++ b/fs/smb/server/crypto_ctx.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2019 Samsung Electronics Co., Ltd. + */ + +#ifndef __CRYPTO_CTX_H__ +#define __CRYPTO_CTX_H__ + +#include + +enum { + CRYPTO_AEAD_AES_GCM = 16, + CRYPTO_AEAD_AES_CCM, + CRYPTO_AEAD_MAX, +}; + +struct ksmbd_crypto_ctx { + struct list_head list; + + struct crypto_aead *ccmaes[CRYPTO_AEAD_MAX]; +}; + +#define CRYPTO_GCM(c) ((c)->ccmaes[CRYPTO_AEAD_AES_GCM]) +#define CRYPTO_CCM(c) ((c)->ccmaes[CRYPTO_AEAD_AES_CCM]) + +void ksmbd_release_crypto_ctx(struct ksmbd_crypto_ctx *ctx); +struct ksmbd_crypto_ctx *ksmbd_crypto_ctx_find_gcm(void); +struct ksmbd_crypto_ctx *ksmbd_crypto_ctx_find_ccm(void); +void ksmbd_crypto_destroy(void); +int ksmbd_crypto_create(void); + +#endif /* __CRYPTO_CTX_H__ */ diff --git a/fs/smb/server/glob.h b/fs/smb/server/glob.h new file mode 100644 index 000000000..4ea187af2 --- /dev/null +++ b/fs/smb/server/glob.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2016 Namjae Jeon + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#ifndef __KSMBD_GLOB_H +#define __KSMBD_GLOB_H + +#include + +#include "unicode.h" +#include "vfs_cache.h" + +extern int ksmbd_debug_types; + +#define KSMBD_DEBUG_SMB BIT(0) +#define KSMBD_DEBUG_AUTH BIT(1) +#define KSMBD_DEBUG_VFS BIT(2) +#define KSMBD_DEBUG_OPLOCK BIT(3) +#define KSMBD_DEBUG_IPC BIT(4) +#define KSMBD_DEBUG_CONN BIT(5) +#define KSMBD_DEBUG_RDMA BIT(6) +#define KSMBD_DEBUG_ALL (KSMBD_DEBUG_SMB | KSMBD_DEBUG_AUTH | \ + KSMBD_DEBUG_VFS | KSMBD_DEBUG_OPLOCK | \ + KSMBD_DEBUG_IPC | KSMBD_DEBUG_CONN | \ + KSMBD_DEBUG_RDMA) + +#ifdef pr_fmt +#undef pr_fmt +#endif + +#ifdef SUBMOD_NAME +#define pr_fmt(fmt) "ksmbd: " SUBMOD_NAME ": " fmt +#else +#define pr_fmt(fmt) "ksmbd: " fmt +#endif + +#define ksmbd_debug(type, fmt, ...) \ + do { \ + if (ksmbd_debug_types & KSMBD_DEBUG_##type) \ + pr_info(fmt, ##__VA_ARGS__); \ + } while (0) + +#define UNICODE_LEN(x) ((x) * 2) + +#define KSMBD_DEFAULT_GFP (GFP_KERNEL | __GFP_RETRY_MAYFAIL) + +#endif /* __KSMBD_GLOB_H */ diff --git a/fs/smb/server/ksmbd_netlink.h b/fs/smb/server/ksmbd_netlink.h new file mode 100644 index 000000000..2673522c7 --- /dev/null +++ b/fs/smb/server/ksmbd_netlink.h @@ -0,0 +1,443 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + * + * linux-ksmbd-devel@lists.sourceforge.net + */ + +#ifndef _LINUX_KSMBD_SERVER_H +#define _LINUX_KSMBD_SERVER_H + +#include + +/* + * This is a userspace ABI to communicate data between ksmbd and user IPC + * daemon using netlink. This is added to track and cache user account DB + * and share configuration info from userspace. + * + * - KSMBD_EVENT_HEARTBEAT_REQUEST(ksmbd_heartbeat) + * This event is to check whether user IPC daemon is alive. If user IPC + * daemon is dead, ksmbd keep existing connection till disconnecting and + * new connection will be denied. + * + * - KSMBD_EVENT_STARTING_UP(ksmbd_startup_request) + * This event is to receive the information that initializes the ksmbd + * server from the user IPC daemon and to start the server. The global + * section parameters are given from smb.conf as initialization + * information. + * + * - KSMBD_EVENT_SHUTTING_DOWN(ksmbd_shutdown_request) + * This event is to shutdown ksmbd server. + * + * - KSMBD_EVENT_LOGIN_REQUEST/RESPONSE(ksmbd_login_request/response) + * This event is to get user account info to user IPC daemon. + * + * - KSMBD_EVENT_SHARE_CONFIG_REQUEST/RESPONSE(ksmbd_share_config_request/response) + * This event is to get net share configuration info. + * + * - KSMBD_EVENT_TREE_CONNECT_REQUEST/RESPONSE(ksmbd_tree_connect_request/response) + * This event is to get session and tree connect info. + * + * - KSMBD_EVENT_TREE_DISCONNECT_REQUEST(ksmbd_tree_disconnect_request) + * This event is to send tree disconnect info to user IPC daemon. + * + * - KSMBD_EVENT_LOGOUT_REQUEST(ksmbd_logout_request) + * This event is to send logout request to user IPC daemon. + * + * - KSMBD_EVENT_RPC_REQUEST/RESPONSE(ksmbd_rpc_command) + * This event is to make DCE/RPC request like srvsvc, wkssvc, lsarpc, + * samr to be processed in userspace. + * + * - KSMBD_EVENT_SPNEGO_AUTHEN_REQUEST/RESPONSE(ksmbd_spnego_authen_request/response) + * This event is to make kerberos authentication to be processed in + * userspace. + * + * - KSMBD_EVENT_LOGIN_REQUEST_EXT/RESPONSE_EXT(ksmbd_login_request_ext/response_ext) + * This event is to get user account extension info to user IPC daemon. + */ + +#define KSMBD_GENL_NAME "SMBD_GENL" +#define KSMBD_GENL_VERSION 0x01 + +#define KSMBD_REQ_MAX_ACCOUNT_NAME_SZ 48 +#define KSMBD_REQ_MAX_HASH_SZ 18 +#define KSMBD_REQ_MAX_SHARE_NAME 64 + +/* + * IPC heartbeat frame to check whether user IPC daemon is alive. + */ +struct ksmbd_heartbeat { + __u32 handle; +}; + +/* + * Global config flags. + */ +#define KSMBD_GLOBAL_FLAG_INVALID (0) +#define KSMBD_GLOBAL_FLAG_SMB2_LEASES BIT(0) +#define KSMBD_GLOBAL_FLAG_SMB2_ENCRYPTION BIT(1) +#define KSMBD_GLOBAL_FLAG_SMB3_MULTICHANNEL BIT(2) +#define KSMBD_GLOBAL_FLAG_SMB2_ENCRYPTION_OFF BIT(3) +#define KSMBD_GLOBAL_FLAG_DURABLE_HANDLE BIT(4) + +/* + * IPC request for ksmbd server startup + */ +struct ksmbd_startup_request { + __u32 flags; /* Flags for global config */ + __s32 signing; /* Signing enabled */ + __s8 min_prot[16]; /* The minimum SMB protocol version */ + __s8 max_prot[16]; /* The maximum SMB protocol version */ + __s8 netbios_name[16]; + __s8 work_group[64]; /* Workgroup */ + __s8 server_string[64]; /* Server string */ + __u16 tcp_port; /* tcp port */ + __u16 ipc_timeout; /* + * specifies the number of seconds + * server will wait for the userspace to + * reply to heartbeat frames. + */ + __u32 deadtime; /* Number of minutes of inactivity */ + __u32 file_max; /* Limits the maximum number of open files */ + __u32 smb2_max_write; /* MAX write size */ + __u32 smb2_max_read; /* MAX read size */ + __u32 smb2_max_trans; /* MAX trans size */ + __u32 share_fake_fscaps; /* + * Support some special application that + * makes QFSINFO calls to check whether + * we set the SPARSE_FILES bit (0x40). + */ + __u32 sub_auth[3]; /* Subauth value for Security ID */ + __u32 smb2_max_credits; /* MAX credits */ + __u32 smbd_max_io_size; /* smbd read write size */ + __u32 max_connections; /* Number of maximum simultaneous connections */ + __s8 bind_interfaces_only; + __u32 max_ip_connections; /* Number of maximum connection per ip address */ + __s8 aapl_model[32]; /* AAPL model string for Finder icon, e.g. "Xserve" */ + __s8 reserved[467]; /* Reserved room */ + __u32 ifc_list_sz; /* interfaces list size */ + __s8 ____payload[]; +} __packed; + +#define KSMBD_STARTUP_CONFIG_INTERFACES(s) ((s)->____payload) + +/* + * IPC request to shutdown ksmbd server. + */ +struct ksmbd_shutdown_request { + __s32 reserved[16]; +}; + +/* + * IPC user login request. + */ +struct ksmbd_login_request { + __u32 handle; + __s8 account[KSMBD_REQ_MAX_ACCOUNT_NAME_SZ]; /* user account name */ + __u32 reserved[16]; /* Reserved room */ +}; + +/* + * IPC user login response. + */ +struct ksmbd_login_response { + __u32 handle; + __u32 gid; /* group id */ + __u32 uid; /* user id */ + __s8 account[KSMBD_REQ_MAX_ACCOUNT_NAME_SZ]; /* user account name */ + __u16 status; + __u16 hash_sz; /* hash size */ + __s8 hash[KSMBD_REQ_MAX_HASH_SZ]; /* password hash */ + __u32 reserved[16]; /* Reserved room */ +}; + +/* + * IPC user login response extension. + */ +struct ksmbd_login_response_ext { + __u32 handle; + __s32 ngroups; /* supplementary group count */ + __s8 reserved[128]; /* Reserved room */ + __s8 ____payload[]; +}; + +/* + * IPC request to fetch net share config. + */ +struct ksmbd_share_config_request { + __u32 handle; + __s8 share_name[KSMBD_REQ_MAX_SHARE_NAME]; /* share name */ + __u32 reserved[16]; /* Reserved room */ +}; + +/* + * IPC response to the net share config request. + */ +struct ksmbd_share_config_response { + __u32 handle; + __u32 flags; + __u16 create_mask; + __u16 directory_mask; + __u16 force_create_mode; + __u16 force_directory_mode; + __u16 force_uid; + __u16 force_gid; + __s8 share_name[KSMBD_REQ_MAX_SHARE_NAME]; + __u32 reserved[111]; /* Reserved room */ + __u32 payload_sz; + __u32 veto_list_sz; + __s8 ____payload[]; +}; + +#define KSMBD_SHARE_CONFIG_VETO_LIST(s) ((s)->____payload) + +static inline char * +ksmbd_share_config_path(struct ksmbd_share_config_response *sc) +{ + char *p = sc->____payload; + + if (sc->veto_list_sz) + p += sc->veto_list_sz + 1; + + return p; +} + +/* + * IPC request for tree connection. This request include session and tree + * connect info from client. + */ +struct ksmbd_tree_connect_request { + __u32 handle; + __u16 account_flags; + __u16 flags; + __u64 session_id; + __u64 connect_id; + __s8 account[KSMBD_REQ_MAX_ACCOUNT_NAME_SZ]; + __s8 share[KSMBD_REQ_MAX_SHARE_NAME]; + __s8 peer_addr[64]; + __u32 reserved[16]; /* Reserved room */ +}; + +/* + * IPC Response structure for tree connection. + */ +struct ksmbd_tree_connect_response { + __u32 handle; + __u16 status; + __u16 connection_flags; + __u32 reserved[16]; /* Reserved room */ +}; + +/* + * IPC Request structure to disconnect tree connection. + */ +struct ksmbd_tree_disconnect_request { + __u64 session_id; /* session id */ + __u64 connect_id; /* tree connection id */ + __u32 reserved[16]; /* Reserved room */ +}; + +/* + * IPC Response structure to logout user account. + */ +struct ksmbd_logout_request { + __s8 account[KSMBD_REQ_MAX_ACCOUNT_NAME_SZ]; /* user account name */ + __u32 account_flags; + __u32 reserved[16]; /* Reserved room */ +}; + +/* + * RPC command structure to send rpc request like srvsvc or wkssvc to + * IPC user daemon. + */ +struct ksmbd_rpc_command { + __u32 handle; + __u32 flags; + __u32 payload_sz; + __u8 payload[]; +}; + +/* + * IPC Request Kerberos authentication + */ +struct ksmbd_spnego_authen_request { + __u32 handle; + __u16 spnego_blob_len; /* the length of spnego_blob */ + __u8 spnego_blob[]; /* + * the GSS token from SecurityBuffer of + * SMB2 SESSION SETUP request + */ +}; + +/* + * Response data which includes the GSS token and the session key generated by + * user daemon. + */ +struct ksmbd_spnego_authen_response { + __u32 handle; + struct ksmbd_login_response login_response; /* + * the login response with + * a user identified by the + * GSS token from a client + */ + __u16 session_key_len; /* the length of the session key */ + __u16 spnego_blob_len; /* + * the length of the GSS token which will be + * stored in SecurityBuffer of SMB2 SESSION + * SETUP response + */ + __u64 session_expiry; /* Kerberos ticket expiry time */ + __u8 payload[]; /* session key + AP_REP */ +}; + +/* + * This also used as NETLINK attribute type value. + * + * NOTE: + * Response message type value should be equal to + * request message type value + 1. + */ +enum ksmbd_event { + KSMBD_EVENT_UNSPEC = 0, + KSMBD_EVENT_HEARTBEAT_REQUEST, + + KSMBD_EVENT_STARTING_UP, + KSMBD_EVENT_SHUTTING_DOWN, + + KSMBD_EVENT_LOGIN_REQUEST, + KSMBD_EVENT_LOGIN_RESPONSE = 5, + + KSMBD_EVENT_SHARE_CONFIG_REQUEST, + KSMBD_EVENT_SHARE_CONFIG_RESPONSE, + + KSMBD_EVENT_TREE_CONNECT_REQUEST, + KSMBD_EVENT_TREE_CONNECT_RESPONSE, + + KSMBD_EVENT_TREE_DISCONNECT_REQUEST = 10, + + KSMBD_EVENT_LOGOUT_REQUEST, + + KSMBD_EVENT_RPC_REQUEST, + KSMBD_EVENT_RPC_RESPONSE, + + KSMBD_EVENT_SPNEGO_AUTHEN_REQUEST, + KSMBD_EVENT_SPNEGO_AUTHEN_RESPONSE = 15, + + KSMBD_EVENT_LOGIN_REQUEST_EXT, + KSMBD_EVENT_LOGIN_RESPONSE_EXT, + + __KSMBD_EVENT_MAX, + KSMBD_EVENT_MAX = __KSMBD_EVENT_MAX - 1 +}; + +/* + * Enumeration for IPC tree connect status. + */ +enum KSMBD_TREE_CONN_STATUS { + KSMBD_TREE_CONN_STATUS_OK = 0, + KSMBD_TREE_CONN_STATUS_NOMEM, + KSMBD_TREE_CONN_STATUS_NO_SHARE, + KSMBD_TREE_CONN_STATUS_NO_USER, + KSMBD_TREE_CONN_STATUS_INVALID_USER, + KSMBD_TREE_CONN_STATUS_HOST_DENIED = 5, + KSMBD_TREE_CONN_STATUS_CONN_EXIST, + KSMBD_TREE_CONN_STATUS_TOO_MANY_CONNS, + KSMBD_TREE_CONN_STATUS_TOO_MANY_SESSIONS, + KSMBD_TREE_CONN_STATUS_ERROR, +}; + +/* + * User config flags. + */ +#define KSMBD_USER_FLAG_INVALID (0) +#define KSMBD_USER_FLAG_OK BIT(0) +#define KSMBD_USER_FLAG_BAD_PASSWORD BIT(1) +#define KSMBD_USER_FLAG_BAD_UID BIT(2) +#define KSMBD_USER_FLAG_BAD_USER BIT(3) +#define KSMBD_USER_FLAG_GUEST_ACCOUNT BIT(4) +#define KSMBD_USER_FLAG_DELAY_SESSION BIT(5) +#define KSMBD_USER_FLAG_EXTENSION BIT(6) + +/* + * Share config flags. + */ +#define KSMBD_SHARE_FLAG_INVALID (0) +#define KSMBD_SHARE_FLAG_AVAILABLE BIT(0) +#define KSMBD_SHARE_FLAG_BROWSEABLE BIT(1) +#define KSMBD_SHARE_FLAG_WRITEABLE BIT(2) +#define KSMBD_SHARE_FLAG_READONLY BIT(3) +#define KSMBD_SHARE_FLAG_GUEST_OK BIT(4) +#define KSMBD_SHARE_FLAG_GUEST_ONLY BIT(5) +#define KSMBD_SHARE_FLAG_STORE_DOS_ATTRS BIT(6) +#define KSMBD_SHARE_FLAG_OPLOCKS BIT(7) +#define KSMBD_SHARE_FLAG_PIPE BIT(8) +#define KSMBD_SHARE_FLAG_HIDE_DOT_FILES BIT(9) +#define KSMBD_SHARE_FLAG_INHERIT_OWNER BIT(10) +#define KSMBD_SHARE_FLAG_STREAMS BIT(11) +#define KSMBD_SHARE_FLAG_FOLLOW_SYMLINKS BIT(12) +#define KSMBD_SHARE_FLAG_ACL_XATTR BIT(13) +#define KSMBD_SHARE_FLAG_UPDATE BIT(14) +#define KSMBD_SHARE_FLAG_CROSSMNT BIT(15) +#define KSMBD_SHARE_FLAG_CONTINUOUS_AVAILABILITY BIT(16) +#define KSMBD_SHARE_FLAG_HIDE_UNREADABLE BIT(17) +#define KSMBD_SHARE_FLAG_TIME_MACHINE BIT(18) +/* Keep BIT(19) reserved for the existing ksmbd-tools WIDE_LINKS flag. */ +#define KSMBD_SHARE_FLAG_ENCRYPT_DATA BIT(20) + +/* + * Tree connect request flags. + */ +#define KSMBD_TREE_CONN_FLAG_REQUEST_SMB1 (0) +#define KSMBD_TREE_CONN_FLAG_REQUEST_IPV6 BIT(0) +#define KSMBD_TREE_CONN_FLAG_REQUEST_SMB2 BIT(1) + +/* + * Tree connect flags. + */ +#define KSMBD_TREE_CONN_FLAG_GUEST_ACCOUNT BIT(0) +#define KSMBD_TREE_CONN_FLAG_READ_ONLY BIT(1) +#define KSMBD_TREE_CONN_FLAG_WRITABLE BIT(2) +#define KSMBD_TREE_CONN_FLAG_ADMIN_ACCOUNT BIT(3) +#define KSMBD_TREE_CONN_FLAG_UPDATE BIT(4) + +/* + * RPC over IPC. + */ +#define KSMBD_RPC_METHOD_RETURN BIT(0) +#define KSMBD_RPC_SRVSVC_METHOD_INVOKE BIT(1) +#define KSMBD_RPC_SRVSVC_METHOD_RETURN (KSMBD_RPC_SRVSVC_METHOD_INVOKE | KSMBD_RPC_METHOD_RETURN) +#define KSMBD_RPC_WKSSVC_METHOD_INVOKE BIT(2) +#define KSMBD_RPC_WKSSVC_METHOD_RETURN (KSMBD_RPC_WKSSVC_METHOD_INVOKE | KSMBD_RPC_METHOD_RETURN) +#define KSMBD_RPC_IOCTL_METHOD (BIT(3) | KSMBD_RPC_METHOD_RETURN) +#define KSMBD_RPC_OPEN_METHOD BIT(4) +#define KSMBD_RPC_WRITE_METHOD BIT(5) +#define KSMBD_RPC_READ_METHOD (BIT(6) | KSMBD_RPC_METHOD_RETURN) +#define KSMBD_RPC_CLOSE_METHOD BIT(7) +#define KSMBD_RPC_RAP_METHOD (BIT(8) | KSMBD_RPC_METHOD_RETURN) +#define KSMBD_RPC_RESTRICTED_CONTEXT BIT(9) +#define KSMBD_RPC_SAMR_METHOD_INVOKE BIT(10) +#define KSMBD_RPC_SAMR_METHOD_RETURN (KSMBD_RPC_SAMR_METHOD_INVOKE | KSMBD_RPC_METHOD_RETURN) +#define KSMBD_RPC_LSARPC_METHOD_INVOKE BIT(11) +#define KSMBD_RPC_LSARPC_METHOD_RETURN (KSMBD_RPC_LSARPC_METHOD_INVOKE | KSMBD_RPC_METHOD_RETURN) + +/* + * RPC status definitions. + */ +#define KSMBD_RPC_OK 0 +#define KSMBD_RPC_EBAD_FUNC 0x00000001 +#define KSMBD_RPC_EACCESS_DENIED 0x00000005 +#define KSMBD_RPC_EBAD_FID 0x00000006 +#define KSMBD_RPC_ENOMEM 0x00000008 +#define KSMBD_RPC_EBAD_DATA 0x0000000D +#define KSMBD_RPC_ENOTIMPLEMENTED 0x00000040 +#define KSMBD_RPC_EINVALID_PARAMETER 0x00000057 +#define KSMBD_RPC_EMORE_DATA 0x000000EA +#define KSMBD_RPC_EINVALID_LEVEL 0x0000007C +#define KSMBD_RPC_SOME_NOT_MAPPED 0x00000107 + +#define KSMBD_CONFIG_OPT_DISABLED 0 +#define KSMBD_CONFIG_OPT_ENABLED 1 +#define KSMBD_CONFIG_OPT_AUTO 2 +#define KSMBD_CONFIG_OPT_MANDATORY 3 + +#endif /* _LINUX_KSMBD_SERVER_H */ diff --git a/fs/smb/server/ksmbd_spnego_negtokeninit.asn1 b/fs/smb/server/ksmbd_spnego_negtokeninit.asn1 new file mode 100644 index 000000000..001513806 --- /dev/null +++ b/fs/smb/server/ksmbd_spnego_negtokeninit.asn1 @@ -0,0 +1,39 @@ +-- SPDX-License-Identifier: BSD-3-Clause +-- +-- Copyright (C) 1998, 2000 IETF Trust and the persons identified as authors +-- of the code +-- +-- https://www.rfc-editor.org/rfc/rfc2478#section-3.2.1 +-- https://www.rfc-editor.org/rfc/rfc2743#section-3.1 + +GSSAPI ::= + [APPLICATION 0] IMPLICIT SEQUENCE { + thisMech + OBJECT IDENTIFIER ({ksmbd_gssapi_this_mech}), + negotiationToken + NegotiationToken + } + +MechType ::= OBJECT IDENTIFIER ({ksmbd_neg_token_init_mech_type}) + +MechTypeList ::= SEQUENCE OF MechType + +NegTokenInit ::= + SEQUENCE { + mechTypes + [0] MechTypeList, + reqFlags + [1] BIT STRING OPTIONAL, + mechToken + [2] OCTET STRING OPTIONAL ({ksmbd_neg_token_init_mech_token}), + mechListMIC + [3] OCTET STRING OPTIONAL + } + +NegotiationToken ::= + CHOICE { + negTokenInit + [0] NegTokenInit, + negTokenTarg + [1] ANY + } diff --git a/fs/smb/server/ksmbd_spnego_negtokentarg.asn1 b/fs/smb/server/ksmbd_spnego_negtokentarg.asn1 new file mode 100644 index 000000000..797e485d5 --- /dev/null +++ b/fs/smb/server/ksmbd_spnego_negtokentarg.asn1 @@ -0,0 +1,26 @@ +-- SPDX-License-Identifier: BSD-3-Clause +-- +-- Copyright (C) 1998 IETF Trust and the persons identified as authors +-- of the code +-- +-- https://www.rfc-editor.org/rfc/rfc2478#section-3.2.1 + +GSSAPI ::= + CHOICE { + negTokenInit + [0] ANY, + negTokenTarg + [1] NegTokenTarg + } + +NegTokenTarg ::= + SEQUENCE { + negResult + [0] ENUMERATED OPTIONAL, + supportedMech + [1] OBJECT IDENTIFIER OPTIONAL, + responseToken + [2] OCTET STRING OPTIONAL ({ksmbd_neg_token_targ_resp_token}), + mechListMIC + [3] OCTET STRING OPTIONAL + } diff --git a/fs/smb/server/ksmbd_work.c b/fs/smb/server/ksmbd_work.c new file mode 100644 index 000000000..f35335307 --- /dev/null +++ b/fs/smb/server/ksmbd_work.c @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2019 Samsung Electronics Co., Ltd. + */ + +#include +#include +#include +#include + +#include "server.h" +#include "connection.h" +#include "ksmbd_work.h" +#include "vfs_cache.h" +#include "mgmt/ksmbd_ida.h" + +static struct kmem_cache *work_cache; +static struct workqueue_struct *ksmbd_wq; + +static int ksmbd_reserve_iov(struct ksmbd_work *work, int need_iov_cnt) +{ + struct kvec *new; + int new_alloc_cnt = work->iov_alloc_cnt; + + if (work->iov_alloc_cnt >= work->iov_cnt + need_iov_cnt) + return 0; + + do { + new_alloc_cnt += KSMBD_WORK_INLINE_IOVS; + } while (new_alloc_cnt < work->iov_cnt + need_iov_cnt); + + if (work->iov == work->iov_inline) { + new = kcalloc(new_alloc_cnt, sizeof(*new), KSMBD_DEFAULT_GFP); + if (!new) + return -ENOMEM; + + memcpy(new, work->iov_inline, sizeof(work->iov_inline)); + } else { + new = krealloc(work->iov, sizeof(*new) * new_alloc_cnt, + KSMBD_DEFAULT_GFP | __GFP_ZERO); + if (!new) + return -ENOMEM; + } + + work->iov = new; + work->iov_alloc_cnt = new_alloc_cnt; + return 0; +} + +struct ksmbd_work *ksmbd_alloc_work_struct(void) +{ + struct ksmbd_work *work = kmem_cache_zalloc(work_cache, KSMBD_DEFAULT_GFP); + + if (work) { + work->compound_fid = KSMBD_NO_FID; + work->compound_pfid = KSMBD_NO_FID; + INIT_LIST_HEAD(&work->request_entry); + INIT_LIST_HEAD(&work->async_request_entry); + INIT_LIST_HEAD(&work->fp_entry); + INIT_LIST_HEAD(&work->notify_entry); + INIT_LIST_HEAD(&work->aux_read_list); + work->iov_alloc_cnt = ARRAY_SIZE(work->iov_inline); + work->iov = work->iov_inline; + } + return work; +} + +void ksmbd_free_work_struct(struct ksmbd_work *work) +{ + struct aux_read *ar, *tmp; + + WARN_ON(work->saved_cred != NULL); + + kvfree(work->response_buf); + + list_for_each_entry_safe(ar, tmp, &work->aux_read_list, entry) { + kvfree(ar->buf); + list_del(&ar->entry); + kfree(ar); + } + + kfree(work->tr_buf); + kvfree(work->compress_buf); + kvfree(work->request_buf); + if (work->iov != work->iov_inline) + kfree(work->iov); + + if (work->async_id) + ksmbd_release_id(&work->conn->async_ida, work->async_id); + if (work->owns_conn_ref) + ksmbd_conn_put(work->conn); + ksmbd_fd_put(work, work->request_open); + kmem_cache_free(work_cache, work); +} + +void ksmbd_work_pool_destroy(void) +{ + kmem_cache_destroy(work_cache); +} + +int ksmbd_work_pool_init(void) +{ + work_cache = kmem_cache_create("ksmbd_work_cache", + sizeof(struct ksmbd_work), 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!work_cache) + return -ENOMEM; + return 0; +} + +int ksmbd_workqueue_init(void) +{ + ksmbd_wq = alloc_workqueue("ksmbd-io", WQ_PERCPU, 0); + if (!ksmbd_wq) + return -ENOMEM; + return 0; +} + +void ksmbd_workqueue_destroy(void) +{ + destroy_workqueue(ksmbd_wq); + ksmbd_wq = NULL; +} + +bool ksmbd_queue_work(struct ksmbd_work *work) +{ + return queue_work(ksmbd_wq, &work->work); +} + +static inline void __ksmbd_iov_pin(struct ksmbd_work *work, void *ib, + unsigned int ib_len) +{ + work->iov[++work->iov_idx].iov_base = ib; + work->iov[work->iov_idx].iov_len = ib_len; + work->iov_cnt++; +} + +static int __ksmbd_iov_pin_rsp(struct ksmbd_work *work, void *ib, int len, + void *aux_buf, unsigned int aux_size) +{ + struct aux_read *ar = NULL; + int need_iov_cnt = 1; + + if (aux_size) { + need_iov_cnt++; + ar = kmalloc_obj(struct aux_read, KSMBD_DEFAULT_GFP); + if (!ar) + return -ENOMEM; + } + + if (ksmbd_reserve_iov(work, need_iov_cnt)) { + kfree(ar); + return -ENOMEM; + } + + /* Plus rfc_length size on first iov */ + if (!work->iov_idx) { + work->iov[work->iov_idx].iov_base = work->response_buf; + *(__be32 *)work->iov[0].iov_base = 0; + work->iov[work->iov_idx].iov_len = 4; + work->iov_cnt++; + } + + __ksmbd_iov_pin(work, ib, len); + inc_rfc1001_len(work->iov[0].iov_base, len); + + if (aux_size) { + __ksmbd_iov_pin(work, aux_buf, aux_size); + inc_rfc1001_len(work->iov[0].iov_base, aux_size); + + ar->buf = aux_buf; + list_add(&ar->entry, &work->aux_read_list); + } + + return 0; +} + +int ksmbd_iov_pin_rsp(struct ksmbd_work *work, void *ib, int len) +{ + return __ksmbd_iov_pin_rsp(work, ib, len, NULL, 0); +} + +int ksmbd_iov_pin_rsp_read(struct ksmbd_work *work, void *ib, int len, + void *aux_buf, unsigned int aux_size) +{ + return __ksmbd_iov_pin_rsp(work, ib, len, aux_buf, aux_size); +} + +int allocate_interim_rsp_buf(struct ksmbd_work *work) +{ + work->response_buf = kzalloc(MAX_CIFS_SMALL_BUFFER_SIZE, KSMBD_DEFAULT_GFP); + if (!work->response_buf) + return -ENOMEM; + work->response_sz = MAX_CIFS_SMALL_BUFFER_SIZE; + return 0; +} diff --git a/fs/smb/server/ksmbd_work.h b/fs/smb/server/ksmbd_work.h new file mode 100644 index 000000000..5f1d3ebab --- /dev/null +++ b/fs/smb/server/ksmbd_work.h @@ -0,0 +1,163 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2019 Samsung Electronics Co., Ltd. + */ + +#ifndef __KSMBD_WORK_H__ +#define __KSMBD_WORK_H__ + +#include +#include + +struct ksmbd_conn; +struct ksmbd_session; +struct ksmbd_tree_connect; +struct ksmbd_file; + +#define KSMBD_WORK_INLINE_IOVS 4 + +enum { + KSMBD_WORK_ACTIVE = 0, + KSMBD_WORK_CANCELLED, + KSMBD_WORK_CLOSED, +}; + +struct aux_read { + void *buf; + struct list_head entry; +}; + +/* one of these for every pending CIFS request at the connection */ +struct ksmbd_work { + /* Server corresponding to this mid */ + struct ksmbd_conn *conn; + struct ksmbd_session *sess; + struct ksmbd_tree_connect *tcon; + + /* Pointer to received SMB header */ + void *request_buf; + /* Response buffer */ + void *response_buf; + + struct list_head aux_read_list; + + struct kvec *iov; + int iov_alloc_cnt; + int iov_cnt; + int iov_idx; + struct kvec iov_inline[KSMBD_WORK_INLINE_IOVS]; + + /* Next cmd hdr in compound req buf*/ + int next_smb2_rcv_hdr_off; + /* Next cmd hdr in compound rsp buf*/ + int next_smb2_rsp_hdr_off; + /* Current cmd hdr in compound rsp buf*/ + int curr_smb2_rsp_hdr_off; + + /* + * Current Local FID assigned compound response if SMB2 CREATE + * command is present in compound request + */ + u64 compound_fid; + u64 compound_pfid; + u64 compound_sid; + __le32 compound_status; + + const struct cred *saved_cred; + + /* Number of granted credits */ + unsigned int credits_granted; + + /* + * Credit charge added to conn->outstanding_credits at receive time + * for the SMB2 PDU currently being processed, pending release. Zero + * once the charge has been returned (on the response or error path). + */ + unsigned short credit_charge; + + /* response smb header size */ + unsigned int response_sz; + + void *tr_buf; + /* Contiguous SMB2 compression transform owned by this work item. */ + void *compress_buf; + + unsigned char state; + /* No response for cancelled request */ + bool send_no_response:1; + /* Request is encrypted */ + bool encrypted:1; + /* READ response should be wrapped in a compression transform. */ + bool compress_response:1; + /* Is this SYNC or ASYNC ksmbd_work */ + bool asynchronous:1; + /* Work owns a reference to @conn. */ + bool owns_conn_ref:1; + bool need_invalidate_rkey:1; + bool request_open_chseq_tracked:1; + bool session_setup_reauth:1; + + unsigned int remote_key; + /* cancel works */ + int async_id; + void **cancel_argv; + void (*cancel_fn)(void **argv); + + /* + * Refcounted open associated with the SMB2 command currently being + * processed. + */ + struct ksmbd_file *request_open; + __le16 request_open_chseq; + + struct work_struct work; + /* List head at conn->requests */ + struct list_head request_entry; + /* List head at conn->async_requests */ + struct list_head async_request_entry; + struct list_head fp_entry; + /* List head at ksmbd_file->notify_pendings */ + struct list_head notify_entry; +}; + +/** + * ksmbd_resp_buf_next - Get next buffer on compound response. + * @work: smb work containing response buffer + */ +static inline void *ksmbd_resp_buf_next(struct ksmbd_work *work) +{ + return work->response_buf + work->next_smb2_rsp_hdr_off + 4; +} + +/** + * ksmbd_resp_buf_curr - Get current buffer on compound response. + * @work: smb work containing response buffer + */ +static inline void *ksmbd_resp_buf_curr(struct ksmbd_work *work) +{ + return work->response_buf + work->curr_smb2_rsp_hdr_off + 4; +} + +/** + * ksmbd_req_buf_next - Get next buffer on compound request. + * @work: smb work containing response buffer + */ +static inline void *ksmbd_req_buf_next(struct ksmbd_work *work) +{ + return work->request_buf + work->next_smb2_rcv_hdr_off + 4; +} + +struct ksmbd_work *ksmbd_alloc_work_struct(void); +void ksmbd_free_work_struct(struct ksmbd_work *work); + +void ksmbd_work_pool_destroy(void); +int ksmbd_work_pool_init(void); + +int ksmbd_workqueue_init(void); +void ksmbd_workqueue_destroy(void); +bool ksmbd_queue_work(struct ksmbd_work *work); +int ksmbd_iov_pin_rsp_read(struct ksmbd_work *work, void *ib, int len, + void *aux_buf, unsigned int aux_size); +int ksmbd_iov_pin_rsp(struct ksmbd_work *work, void *ib, int len); +int allocate_interim_rsp_buf(struct ksmbd_work *work); +#endif /* __KSMBD_WORK_H__ */ diff --git a/fs/smb/server/mgmt/ksmbd_ida.c b/fs/smb/server/mgmt/ksmbd_ida.c new file mode 100644 index 000000000..0e2ae994a --- /dev/null +++ b/fs/smb/server/mgmt/ksmbd_ida.c @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#include "ksmbd_ida.h" +#include "../glob.h" + +int ksmbd_acquire_smb2_tid(struct ida *ida) +{ + return ida_alloc_range(ida, 1, 0xFFFFFFFE, KSMBD_DEFAULT_GFP); +} + +int ksmbd_acquire_smb2_uid(struct ida *ida) +{ + int id; + + id = ida_alloc_min(ida, 1, KSMBD_DEFAULT_GFP); + if (id == 0xFFFE) + id = ida_alloc_min(ida, 1, KSMBD_DEFAULT_GFP); + + return id; +} + +int ksmbd_acquire_async_msg_id(struct ida *ida) +{ + return ida_alloc_min(ida, 1, KSMBD_DEFAULT_GFP); +} + +int ksmbd_acquire_id(struct ida *ida) +{ + return ida_alloc(ida, KSMBD_DEFAULT_GFP); +} + +void ksmbd_release_id(struct ida *ida, int id) +{ + ida_free(ida, id); +} diff --git a/fs/smb/server/mgmt/ksmbd_ida.h b/fs/smb/server/mgmt/ksmbd_ida.h new file mode 100644 index 000000000..2bc07b16c --- /dev/null +++ b/fs/smb/server/mgmt/ksmbd_ida.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#ifndef __KSMBD_IDA_MANAGEMENT_H__ +#define __KSMBD_IDA_MANAGEMENT_H__ + +#include +#include + +/* + * 2.2.1.6.7 TID Generation + * The value 0xFFFF MUST NOT be used as a valid TID. All other + * possible values for TID, including zero (0x0000), are valid. + * The value 0xFFFF is used to specify all TIDs or no TID, + * depending upon the context in which it is used. + */ +int ksmbd_acquire_smb2_tid(struct ida *ida); + +/* + * 2.2.1.6.8 UID Generation + * The value 0xFFFE was declared reserved in the LAN Manager 1.0 + * documentation, so a value of 0xFFFE SHOULD NOT be used as a + * valid UID.<21> All other possible values for a UID, excluding + * zero (0x0000), are valid. + */ +int ksmbd_acquire_smb2_uid(struct ida *ida); +int ksmbd_acquire_async_msg_id(struct ida *ida); + +int ksmbd_acquire_id(struct ida *ida); + +void ksmbd_release_id(struct ida *ida, int id); +#endif /* __KSMBD_IDA_MANAGEMENT_H__ */ diff --git a/fs/smb/server/mgmt/share_config.c b/fs/smb/server/mgmt/share_config.c new file mode 100644 index 000000000..b2d9580bd --- /dev/null +++ b/fs/smb/server/mgmt/share_config.c @@ -0,0 +1,321 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "share_config.h" +#include "user_config.h" +#include "user_session.h" +#include "../connection.h" +#include "../transport_ipc.h" +#include "../misc.h" + +#define SHARE_HASH_BITS 12 +static DEFINE_HASHTABLE(shares_table, SHARE_HASH_BITS); +static DECLARE_RWSEM(shares_table_lock); + +struct ksmbd_veto_pattern { + char *pattern; + struct list_head list; +}; + +#ifdef CONFIG_PROC_FS +static const struct ksmbd_const_name ksmbd_share_flag_names[] = { + {KSMBD_SHARE_FLAG_AVAILABLE, "available"}, + {KSMBD_SHARE_FLAG_BROWSEABLE, "browseable"}, + {KSMBD_SHARE_FLAG_WRITEABLE, "writeable"}, + {KSMBD_SHARE_FLAG_READONLY, "read-only"}, + {KSMBD_SHARE_FLAG_GUEST_OK, "guest-ok"}, + {KSMBD_SHARE_FLAG_GUEST_ONLY, "guest-only"}, + {KSMBD_SHARE_FLAG_STORE_DOS_ATTRS, "store-dos-attrs"}, + {KSMBD_SHARE_FLAG_OPLOCKS, "oplocks"}, + {KSMBD_SHARE_FLAG_PIPE, "pipe"}, + {KSMBD_SHARE_FLAG_HIDE_DOT_FILES, "hide-dot-files"}, + {KSMBD_SHARE_FLAG_INHERIT_OWNER, "inherit-owner"}, + {KSMBD_SHARE_FLAG_STREAMS, "streams"}, + {KSMBD_SHARE_FLAG_FOLLOW_SYMLINKS, "follow-symlinks"}, + {KSMBD_SHARE_FLAG_ACL_XATTR, "acl-xattr"}, + {KSMBD_SHARE_FLAG_UPDATE, "update"}, + {KSMBD_SHARE_FLAG_CROSSMNT, "crossmnt"}, + {KSMBD_SHARE_FLAG_CONTINUOUS_AVAILABILITY, "continuous-availability"}, + {KSMBD_SHARE_FLAG_ENCRYPT_DATA, "encrypt-data"}, +}; + +static int proc_show_shares(struct seq_file *m, void *v) +{ + struct ksmbd_share_config *share; + int i; + + down_read(&shares_table_lock); + hash_for_each(shares_table, i, share, hlist) { + seq_printf(m, "name:\t%s\n", share->name); + seq_printf(m, "type:\t%s\n", + test_share_config_flag(share, KSMBD_SHARE_FLAG_PIPE) ? + "pipe" : "disk"); + seq_printf(m, "tree_connects:\t%d\n", + atomic_read(&share->tree_connections)); + seq_printf(m, "file_mask:\t0%07o\n", share->create_mask); + seq_printf(m, "directory_mask:\t0%07o\n", share->directory_mask); + seq_puts(m, "flags:\t"); + ksmbd_proc_show_flag_names(m, ksmbd_share_flag_names, + ARRAY_SIZE(ksmbd_share_flag_names), + share->flags); + seq_puts(m, "\n\n"); + } + up_read(&shares_table_lock); + return 0; +} + +int create_proc_shares(void) +{ + if (!ksmbd_proc_create("shares", proc_show_shares, NULL)) + return -ENOMEM; + return 0; +} +#else +int create_proc_shares(void) { return 0; } +#endif + +static unsigned int share_name_hash(const char *name) +{ + return jhash(name, strlen(name), 0); +} + +static void kill_share(struct ksmbd_share_config *share) +{ + while (!list_empty(&share->veto_list)) { + struct ksmbd_veto_pattern *p; + + p = list_entry(share->veto_list.next, + struct ksmbd_veto_pattern, + list); + list_del(&p->list); + kfree(p->pattern); + kfree(p); + } + + if (share->path) + path_put(&share->vfs_path); + kfree(share->name); + kfree(share->path); + kfree(share); +} + +void ksmbd_share_config_del(struct ksmbd_share_config *share) +{ + down_write(&shares_table_lock); + hash_del(&share->hlist); + up_write(&shares_table_lock); +} + +void __ksmbd_share_config_put(struct ksmbd_share_config *share) +{ + ksmbd_share_config_del(share); + kill_share(share); +} + +static struct ksmbd_share_config * +__get_share_config(struct ksmbd_share_config *share) +{ + if (!atomic_inc_not_zero(&share->refcount)) + return NULL; + return share; +} + +static struct ksmbd_share_config *__share_lookup(const char *name) +{ + struct ksmbd_share_config *share; + unsigned int key = share_name_hash(name); + + hash_for_each_possible(shares_table, share, hlist, key) { + if (!strcmp(name, share->name)) + return share; + } + return NULL; +} + +static int parse_veto_list(struct ksmbd_share_config *share, + char *veto_list, + int veto_list_sz) +{ + int sz = 0; + + if (!veto_list_sz) + return 0; + + while (veto_list_sz > 0) { + struct ksmbd_veto_pattern *p; + + sz = strlen(veto_list); + if (!sz) + break; + + p = kzalloc_obj(struct ksmbd_veto_pattern, KSMBD_DEFAULT_GFP); + if (!p) + return -ENOMEM; + + p->pattern = kstrdup(veto_list, KSMBD_DEFAULT_GFP); + if (!p->pattern) { + kfree(p); + return -ENOMEM; + } + + list_add(&p->list, &share->veto_list); + + veto_list += sz + 1; + veto_list_sz -= (sz + 1); + } + + return 0; +} + +static struct ksmbd_share_config *share_config_request(struct ksmbd_work *work, + const char *name) +{ + struct ksmbd_share_config_response *resp; + struct ksmbd_share_config *share = NULL; + struct ksmbd_share_config *lookup; + struct unicode_map *um = work->conn->um; + int ret; + + resp = ksmbd_ipc_share_config_request(name); + if (!resp) + return NULL; + + if (resp->flags == KSMBD_SHARE_FLAG_INVALID) + goto out; + + if (*resp->share_name) { + char *cf_resp_name; + bool equal; + + cf_resp_name = ksmbd_casefold_sharename(um, resp->share_name); + if (IS_ERR(cf_resp_name)) + goto out; + equal = !strcmp(cf_resp_name, name); + kfree(cf_resp_name); + if (!equal) + goto out; + } + + share = kzalloc_obj(struct ksmbd_share_config, KSMBD_DEFAULT_GFP); + if (!share) + goto out; + + share->flags = resp->flags; + atomic_set(&share->refcount, 1); + ksmbd_share_tree_conn_init(share); + INIT_LIST_HEAD(&share->veto_list); + share->name = kstrdup(name, KSMBD_DEFAULT_GFP); + if (!share->name) { + kill_share(share); + share = NULL; + goto out; + } + + if (!test_share_config_flag(share, KSMBD_SHARE_FLAG_PIPE)) { + int path_len = PATH_MAX; + + if (resp->payload_sz) + path_len = resp->payload_sz - resp->veto_list_sz; + + share->path = kstrndup(ksmbd_share_config_path(resp), path_len, + KSMBD_DEFAULT_GFP); + if (!share->path) { + ret = -ENOMEM; + } else { + ret = 0; + share->path_sz = strlen(share->path); + while (share->path_sz > 1 && + share->path[share->path_sz - 1] == '/') + share->path[--share->path_sz] = '\0'; + } + share->create_mask = resp->create_mask; + share->directory_mask = resp->directory_mask; + share->force_create_mode = resp->force_create_mode; + share->force_directory_mode = resp->force_directory_mode; + share->force_uid = resp->force_uid; + share->force_gid = resp->force_gid; + if (!ret) + ret = parse_veto_list(share, + KSMBD_SHARE_CONFIG_VETO_LIST(resp), + resp->veto_list_sz); + if (!ret && share->path) { + if (__ksmbd_override_fsids(work, share)) { + kill_share(share); + share = NULL; + goto out; + } + + scoped_with_init_fs() + ret = kern_path(share->path, 0, &share->vfs_path); + ksmbd_revert_fsids(work); + if (ret) { + ksmbd_debug(SMB, "failed to access '%s'\n", + share->path); + /* Avoid put_path() */ + kfree(share->path); + share->path = NULL; + } + } + if (ret) { + kill_share(share); + share = NULL; + goto out; + } + } + + down_write(&shares_table_lock); + lookup = __share_lookup(name); + if (lookup) + lookup = __get_share_config(lookup); + if (!lookup) { + hash_add(shares_table, &share->hlist, share_name_hash(name)); + } else { + kill_share(share); + share = lookup; + } + up_write(&shares_table_lock); + +out: + kvfree(resp); + return share; +} + +struct ksmbd_share_config *ksmbd_share_config_get(struct ksmbd_work *work, + const char *name) +{ + struct ksmbd_share_config *share; + + down_read(&shares_table_lock); + share = __share_lookup(name); + if (share) + share = __get_share_config(share); + up_read(&shares_table_lock); + + if (share) + return share; + return share_config_request(work, name); +} + +bool ksmbd_share_veto_filename(struct ksmbd_share_config *share, + const char *filename) +{ + struct ksmbd_veto_pattern *p; + + list_for_each_entry(p, &share->veto_list, list) { + if (match_wildcard(p->pattern, filename)) + return true; + } + return false; +} diff --git a/fs/smb/server/mgmt/share_config.h b/fs/smb/server/mgmt/share_config.h new file mode 100644 index 000000000..d157545fe --- /dev/null +++ b/fs/smb/server/mgmt/share_config.h @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#ifndef __SHARE_CONFIG_MANAGEMENT_H__ +#define __SHARE_CONFIG_MANAGEMENT_H__ + +#include +#include +#include +#include + +struct ksmbd_work; + +struct ksmbd_share_config { + char *name; + char *path; + + unsigned int path_sz; + unsigned int flags; + struct list_head veto_list; + + struct path vfs_path; + + atomic_t refcount; +#ifdef CONFIG_PROC_FS + atomic_t tree_connections; +#endif + struct hlist_node hlist; + unsigned short create_mask; + unsigned short directory_mask; + unsigned short force_create_mode; + unsigned short force_directory_mode; + unsigned short force_uid; + unsigned short force_gid; +}; + +#define KSMBD_SHARE_INVALID_UID ((__u16)-1) +#define KSMBD_SHARE_INVALID_GID ((__u16)-1) + +static inline umode_t +share_config_create_mode(struct ksmbd_share_config *share, + umode_t posix_mode) +{ + umode_t mode = (posix_mode ?: (umode_t)-1) & share->create_mask; + + return mode | share->force_create_mode; +} + +static inline umode_t +share_config_directory_mode(struct ksmbd_share_config *share, + umode_t posix_mode) +{ + umode_t mode = (posix_mode ?: (umode_t)-1) & share->directory_mask; + + return mode | share->force_directory_mode; +} + +static inline int test_share_config_flag(struct ksmbd_share_config *share, + int flag) +{ + return share->flags & flag; +} + +#ifdef CONFIG_PROC_FS +static inline void ksmbd_share_tree_conn_init(struct ksmbd_share_config *share) +{ + atomic_set(&share->tree_connections, 0); +} + +static inline void ksmbd_share_tree_conn_inc(struct ksmbd_share_config *share) +{ + atomic_inc(&share->tree_connections); +} + +static inline void ksmbd_share_tree_conn_dec(struct ksmbd_share_config *share) +{ + atomic_dec(&share->tree_connections); +} +#else +static inline void ksmbd_share_tree_conn_init(struct ksmbd_share_config *share) {} +static inline void ksmbd_share_tree_conn_inc(struct ksmbd_share_config *share) {} +static inline void ksmbd_share_tree_conn_dec(struct ksmbd_share_config *share) {} +#endif + +void ksmbd_share_config_del(struct ksmbd_share_config *share); +void __ksmbd_share_config_put(struct ksmbd_share_config *share); + +static inline void ksmbd_share_config_put(struct ksmbd_share_config *share) +{ + if (!atomic_dec_and_test(&share->refcount)) + return; + __ksmbd_share_config_put(share); +} + +struct ksmbd_share_config *ksmbd_share_config_get(struct ksmbd_work *work, + const char *name); +bool ksmbd_share_veto_filename(struct ksmbd_share_config *share, + const char *filename); +int create_proc_shares(void); +#endif /* __SHARE_CONFIG_MANAGEMENT_H__ */ diff --git a/fs/smb/server/mgmt/tree_connect.c b/fs/smb/server/mgmt/tree_connect.c new file mode 100644 index 000000000..5f63e2362 --- /dev/null +++ b/fs/smb/server/mgmt/tree_connect.c @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#include +#include +#include + +#include "../transport_ipc.h" +#include "../connection.h" +#include "../stats.h" + +#include "tree_connect.h" +#include "user_config.h" +#include "share_config.h" +#include "user_session.h" + +struct ksmbd_tree_conn_status +ksmbd_tree_conn_connect(struct ksmbd_work *work, const char *share_name) +{ + struct ksmbd_tree_conn_status status = {-ENOENT, NULL}; + struct ksmbd_tree_connect_response *resp = NULL; + struct ksmbd_share_config *sc; + struct ksmbd_tree_connect *tree_conn = NULL; + struct sockaddr *peer_addr; + struct ksmbd_conn *conn = work->conn; + struct ksmbd_session *sess = work->sess; + int ret; + + sc = ksmbd_share_config_get(work, share_name); + if (!sc) + return status; + + tree_conn = kzalloc_obj(struct ksmbd_tree_connect, KSMBD_DEFAULT_GFP); + if (!tree_conn) { + status.ret = -ENOMEM; + goto out_error; + } + + tree_conn->id = ksmbd_acquire_tree_conn_id(sess); + if (tree_conn->id < 0) { + status.ret = -EINVAL; + goto out_error; + } + + peer_addr = KSMBD_TCP_PEER_SOCKADDR(conn); + resp = ksmbd_ipc_tree_connect_request(sess, + sc, + tree_conn, + peer_addr); + if (!resp) { + status.ret = -EINVAL; + goto out_error; + } + + status.ret = resp->status; + if (status.ret != KSMBD_TREE_CONN_STATUS_OK) + goto out_error; + + tree_conn->flags = resp->connection_flags; + if (test_tree_conn_flag(tree_conn, KSMBD_TREE_CONN_FLAG_UPDATE)) { + struct ksmbd_share_config *new_sc; + + ksmbd_share_config_del(sc); + new_sc = ksmbd_share_config_get(work, share_name); + if (!new_sc) { + pr_err("Failed to update stale share config\n"); + status.ret = -ESTALE; + goto out_error; + } + ksmbd_share_config_put(sc); + sc = new_sc; + } + + tree_conn->user = sess->user; + tree_conn->share_conf = sc; + tree_conn->t_state = TREE_NEW; + status.tree_conn = tree_conn; + atomic_set(&tree_conn->refcount, 1); + + down_write(&sess->tree_conns_lock); + ret = xa_err(xa_store(&sess->tree_conns, tree_conn->id, tree_conn, + KSMBD_DEFAULT_GFP)); + up_write(&sess->tree_conns_lock); + if (ret) { + status.ret = -ENOMEM; + goto out_error; + } + ksmbd_counter_inc(KSMBD_COUNTER_TREE_CONNS); + ksmbd_share_tree_conn_inc(sc); + kvfree(resp); + return status; + +out_error: + if (tree_conn) + ksmbd_release_tree_conn_id(sess, tree_conn->id); + ksmbd_share_config_put(sc); + kfree(tree_conn); + kvfree(resp); + return status; +} + +void ksmbd_tree_connect_put(struct ksmbd_tree_connect *tcon) +{ + if (atomic_dec_and_test(&tcon->refcount)) { + ksmbd_share_config_put(tcon->share_conf); + kfree(tcon); + } +} + +static int __ksmbd_tree_conn_disconnect(struct ksmbd_session *sess, + struct ksmbd_tree_connect *tree_conn) +{ + int ret; + + ret = ksmbd_ipc_tree_disconnect_request(sess->id, tree_conn->id); + ksmbd_release_tree_conn_id(sess, tree_conn->id); + ksmbd_counter_dec(KSMBD_COUNTER_TREE_CONNS); + ksmbd_share_tree_conn_dec(tree_conn->share_conf); + if (atomic_dec_and_test(&tree_conn->refcount)) { + ksmbd_share_config_put(tree_conn->share_conf); + kfree(tree_conn); + } + return ret; +} + +int ksmbd_tree_conn_disconnect(struct ksmbd_session *sess, + struct ksmbd_tree_connect *tree_conn) +{ + down_write(&sess->tree_conns_lock); + xa_erase(&sess->tree_conns, tree_conn->id); + up_write(&sess->tree_conns_lock); + + return __ksmbd_tree_conn_disconnect(sess, tree_conn); +} + +struct ksmbd_tree_connect *ksmbd_tree_conn_lookup(struct ksmbd_session *sess, + unsigned int id) +{ + struct ksmbd_tree_connect *tcon; + + down_read(&sess->tree_conns_lock); + tcon = xa_load(&sess->tree_conns, id); + if (tcon) { + if (tcon->t_state != TREE_CONNECTED) + tcon = NULL; + else if (!atomic_inc_not_zero(&tcon->refcount)) + tcon = NULL; + } + up_read(&sess->tree_conns_lock); + + return tcon; +} + +int ksmbd_tree_conn_session_logoff(struct ksmbd_session *sess) +{ + int ret = 0; + struct ksmbd_tree_connect *tc; + unsigned long id; + + if (!sess) + return -EINVAL; + + down_write(&sess->tree_conns_lock); + xa_for_each(&sess->tree_conns, id, tc) { + if (tc->t_state == TREE_DISCONNECTED) { + ret = -ENOENT; + continue; + } + tc->t_state = TREE_DISCONNECTED; + + xa_erase(&sess->tree_conns, tc->id); + ret |= __ksmbd_tree_conn_disconnect(sess, tc); + } + xa_destroy(&sess->tree_conns); + up_write(&sess->tree_conns_lock); + + return ret; +} diff --git a/fs/smb/server/mgmt/tree_connect.h b/fs/smb/server/mgmt/tree_connect.h new file mode 100644 index 000000000..f0023d867 --- /dev/null +++ b/fs/smb/server/mgmt/tree_connect.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#ifndef __TREE_CONNECT_MANAGEMENT_H__ +#define __TREE_CONNECT_MANAGEMENT_H__ + +#include + +#include "../ksmbd_netlink.h" + +struct ksmbd_share_config; +struct ksmbd_user; +struct ksmbd_conn; +struct ksmbd_work; + +enum { + TREE_NEW = 0, + TREE_CONNECTED, + TREE_DISCONNECTED +}; + +struct ksmbd_tree_connect { + int id; + + unsigned int flags; + struct ksmbd_share_config *share_conf; + struct ksmbd_user *user; + + struct list_head list; + + int maximal_access; + bool posix_extensions; + atomic_t refcount; + unsigned int t_state; +}; + +struct ksmbd_tree_conn_status { + unsigned int ret; + struct ksmbd_tree_connect *tree_conn; +}; + +static inline int test_tree_conn_flag(struct ksmbd_tree_connect *tree_conn, + int flag) +{ + return tree_conn->flags & flag; +} + +struct ksmbd_session; + +struct ksmbd_tree_conn_status +ksmbd_tree_conn_connect(struct ksmbd_work *work, const char *share_name); +void ksmbd_tree_connect_put(struct ksmbd_tree_connect *tcon); + +int ksmbd_tree_conn_disconnect(struct ksmbd_session *sess, + struct ksmbd_tree_connect *tree_conn); + +struct ksmbd_tree_connect *ksmbd_tree_conn_lookup(struct ksmbd_session *sess, + unsigned int id); + +int ksmbd_tree_conn_session_logoff(struct ksmbd_session *sess); + +#endif /* __TREE_CONNECT_MANAGEMENT_H__ */ diff --git a/fs/smb/server/mgmt/user_config.c b/fs/smb/server/mgmt/user_config.c new file mode 100644 index 000000000..5efc3d745 --- /dev/null +++ b/fs/smb/server/mgmt/user_config.c @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#include +#include + +#include "user_config.h" +#include "../transport_ipc.h" + +struct ksmbd_user *ksmbd_login_user(const char *account) +{ + struct ksmbd_login_response *resp; + struct ksmbd_login_response_ext *resp_ext = NULL; + struct ksmbd_user *user = NULL; + + resp = ksmbd_ipc_login_request(account); + if (!resp) + return NULL; + + if (!(resp->status & KSMBD_USER_FLAG_OK)) + goto out; + + if (resp->status & KSMBD_USER_FLAG_EXTENSION) + resp_ext = ksmbd_ipc_login_request_ext(account); + + user = ksmbd_alloc_user(resp, resp_ext); + kvfree(resp_ext); +out: + kvfree_sensitive(resp, sizeof(*resp)); + return user; +} + +struct ksmbd_user *ksmbd_alloc_user(struct ksmbd_login_response *resp, + struct ksmbd_login_response_ext *resp_ext) +{ + struct ksmbd_user *user; + + /* + * resp->hash_sz is a __u16 taken from the mountd IPC login response but + * resp->hash[] is only KSMBD_REQ_MAX_HASH_SZ bytes. A malformed or + * malicious response can set hash_sz far beyond that (up to 65535), + * making the memcpy() below read past the response object + * (slab-out-of-bounds in ksmbd_alloc_user()). Reject any oversized + * hash rather than trust the length. + */ + if (resp->hash_sz > sizeof(resp->hash)) + return NULL; + + user = kmalloc_obj(struct ksmbd_user, KSMBD_DEFAULT_GFP); + if (!user) + return NULL; + + user->name = kstrdup(resp->account, KSMBD_DEFAULT_GFP); + user->flags = resp->status; + user->gid = resp->gid; + user->uid = resp->uid; + user->passkey_sz = resp->hash_sz; + user->passkey = kmalloc(resp->hash_sz, KSMBD_DEFAULT_GFP); + if (user->passkey) + memcpy(user->passkey, resp->hash, resp->hash_sz); + + user->ngroups = 0; + user->sgid = NULL; + + if (!user->name || !user->passkey) + goto err_free; + + if (resp_ext) { + user->sgid = kmemdup(resp_ext->____payload, + resp_ext->ngroups * sizeof(gid_t), + KSMBD_DEFAULT_GFP); + if (!user->sgid) + goto err_free; + + user->ngroups = resp_ext->ngroups; + ksmbd_debug(SMB, "supplementary groups : %d\n", user->ngroups); + } + + return user; + +err_free: + kfree(user->name); + kfree_sensitive(user->passkey); + kfree(user); + return NULL; +} + +void ksmbd_free_user(struct ksmbd_user *user) +{ + ksmbd_ipc_logout_request(user->name, user->flags); + kfree(user->sgid); + kfree(user->name); + kfree_sensitive(user->passkey); + kfree(user); +} + +bool ksmbd_anonymous_user(struct ksmbd_user *user) +{ + return user->name[0] == '\0'; +} + +bool ksmbd_compare_user(struct ksmbd_user *u1, struct ksmbd_user *u2) +{ + if (strcmp(u1->name, u2->name)) + return false; + if (memcmp(u1->passkey, u2->passkey, u1->passkey_sz)) + return false; + + return true; +} diff --git a/fs/smb/server/mgmt/user_config.h b/fs/smb/server/mgmt/user_config.h new file mode 100644 index 000000000..cc460b4ff --- /dev/null +++ b/fs/smb/server/mgmt/user_config.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#ifndef __USER_CONFIG_MANAGEMENT_H__ +#define __USER_CONFIG_MANAGEMENT_H__ + +#include "../glob.h" + +struct ksmbd_user { + unsigned short flags; + + unsigned int uid; + unsigned int gid; + + char *name; + + size_t passkey_sz; + char *passkey; + int ngroups; + gid_t *sgid; +}; + +static inline bool user_guest(struct ksmbd_user *user) +{ + return user->flags & KSMBD_USER_FLAG_GUEST_ACCOUNT; +} + +static inline void set_user_flag(struct ksmbd_user *user, int flag) +{ + user->flags |= flag; +} + +static inline int test_user_flag(struct ksmbd_user *user, int flag) +{ + return user->flags & flag; +} + +static inline void set_user_guest(struct ksmbd_user *user) +{ +} + +static inline char *user_passkey(struct ksmbd_user *user) +{ + return user->passkey; +} + +static inline char *user_name(struct ksmbd_user *user) +{ + return user->name; +} + +static inline unsigned int user_uid(struct ksmbd_user *user) +{ + return user->uid; +} + +static inline unsigned int user_gid(struct ksmbd_user *user) +{ + return user->gid; +} + +struct ksmbd_user *ksmbd_login_user(const char *account); +struct ksmbd_user *ksmbd_alloc_user(struct ksmbd_login_response *resp, + struct ksmbd_login_response_ext *resp_ext); +void ksmbd_free_user(struct ksmbd_user *user); +bool ksmbd_anonymous_user(struct ksmbd_user *user); +bool ksmbd_compare_user(struct ksmbd_user *u1, struct ksmbd_user *u2); +#endif /* __USER_CONFIG_MANAGEMENT_H__ */ diff --git a/fs/smb/server/mgmt/user_session.c b/fs/smb/server/mgmt/user_session.c new file mode 100644 index 000000000..7022d5d65 --- /dev/null +++ b/fs/smb/server/mgmt/user_session.c @@ -0,0 +1,780 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#include +#include +#include +#include + +#include "ksmbd_ida.h" +#include "user_session.h" +#include "user_config.h" +#include "tree_connect.h" +#include "share_config.h" +#include "../transport_ipc.h" +#include "../connection.h" +#include "../vfs_cache.h" +#include "../misc.h" +#include "../stats.h" + +static DEFINE_IDA(session_ida); + +#define SESSION_HASH_BITS 12 +static DEFINE_HASHTABLE(sessions_table, SESSION_HASH_BITS); +static DECLARE_RWSEM(sessions_table_lock); + +struct ksmbd_session_rpc { + int id; + unsigned int method; +}; + +#ifdef CONFIG_PROC_FS + +static const struct ksmbd_const_name ksmbd_sess_cap_const_names[] = { + {SMB2_GLOBAL_CAP_DFS, "dfs"}, + {SMB2_GLOBAL_CAP_LEASING, "lease"}, + {SMB2_GLOBAL_CAP_LARGE_MTU, "large-mtu"}, + {SMB2_GLOBAL_CAP_MULTI_CHANNEL, "multi-channel"}, + {SMB2_GLOBAL_CAP_PERSISTENT_HANDLES, "persistent-handles"}, + {SMB2_GLOBAL_CAP_DIRECTORY_LEASING, "dir-lease"}, + {SMB2_GLOBAL_CAP_ENCRYPTION, "encryption"} +}; + +static const struct ksmbd_const_name ksmbd_cipher_const_names[] = { + {le16_to_cpu(SMB2_ENCRYPTION_AES128_CCM), "aes128-ccm"}, + {le16_to_cpu(SMB2_ENCRYPTION_AES128_GCM), "aes128-gcm"}, + {le16_to_cpu(SMB2_ENCRYPTION_AES256_CCM), "aes256-ccm"}, + {le16_to_cpu(SMB2_ENCRYPTION_AES256_GCM), "aes256-gcm"}, +}; + +static const struct ksmbd_const_name ksmbd_signing_const_names[] = { + {SIGNING_ALG_HMAC_SHA256, "hmac-sha256"}, + {SIGNING_ALG_AES_CMAC, "aes-cmac"}, + {SIGNING_ALG_AES_GMAC, "aes-gmac"}, +}; + +static const char *session_state_string(struct ksmbd_session *session) +{ + switch (session->state) { + case SMB2_SESSION_VALID: + return "valid"; + case SMB2_SESSION_IN_PROGRESS: + return "progress"; + case SMB2_SESSION_EXPIRED: + return "expired"; + default: + return ""; + } +} + +static const char *session_user_name(struct ksmbd_session *session) +{ + if (user_guest(session->user)) + return "(Guest)"; + else if (ksmbd_anonymous_user(session->user)) + return "(Anonymous)"; + return session->user->name; +} + +static const char *session_account_type(struct ksmbd_session *session) +{ + if (user_guest(session->user)) + return "guest"; + if (ksmbd_anonymous_user(session->user)) + return "anonymous"; + return "user"; +} + +static unsigned int session_open_file_count(struct ksmbd_session *session) +{ + struct ksmbd_file *fp; + unsigned int count = 0; + unsigned int id; + + read_lock(&session->file_table.lock); + idr_for_each_entry(session->file_table.idr, fp, id) + count++; + read_unlock(&session->file_table.lock); + return count; +} + +static int show_proc_session(struct seq_file *m, void *v) +{ + struct ksmbd_session *sess; + struct ksmbd_tree_connect *tree_conn; + struct ksmbd_share_config *share_conf; + struct channel *chan; + unsigned long id; + int i = 0; + + sess = (struct ksmbd_session *)m->private; + ksmbd_user_session_get(sess); + + seq_printf(m, "user:\t%s\n", session_user_name(sess)); + seq_printf(m, "account_type:\t%s\n", + session_account_type(sess)); + seq_printf(m, "id:\t%llu\n", sess->id); + seq_printf(m, "state:\t%s\n", session_state_string(sess)); + seq_printf(m, "dialect:\t0x%04x\n", sess->dialect); + seq_printf(m, "last_active_seconds:\t%lu\n", + jiffies_to_msecs(jiffies - sess->last_active) / MSEC_PER_SEC); + seq_printf(m, "open_files:\t%u\n", + session_open_file_count(sess)); + + i = 0; + down_read(&sess->chann_lock); + xa_for_each(&sess->ksmbd_chann_list, id, chan) { + const char *name; + +#if IS_ENABLED(CONFIG_IPV6) + if (chan->conn->inet_addr) + seq_printf(m, "client:\t%pI4\n", + &chan->conn->inet_addr); + else + seq_printf(m, "client:\t%pI6c\n", + &chan->conn->inet6_addr); +#else + seq_printf(m, "client:\t%pI4\n", + &chan->conn->inet_addr); +#endif + seq_puts(m, "capabilities:\t"); + ksmbd_proc_show_flag_names(m, + ksmbd_sess_cap_const_names, + ARRAY_SIZE(ksmbd_sess_cap_const_names), + chan->conn->vals->req_capabilities); + seq_putc(m, '\n'); + seq_printf(m, "posix_extensions:\t%s\n", + chan->conn->posix_ext_supported ? "yes" : "no"); + + if (sess->sign) { + unsigned int algorithm = + le16_to_cpu(chan->conn->signing_algorithm); + + name = ksmbd_proc_const_name(ksmbd_signing_const_names, + ARRAY_SIZE(ksmbd_signing_const_names), + algorithm); + if (name) + seq_printf(m, "signing:\t%s\n", name); + else + seq_printf(m, "signing:\t0x%04x\n", + algorithm); + } + if (sess->enc) { + unsigned int cipher = le16_to_cpu(chan->conn->cipher_type); + + name = ksmbd_proc_const_name(ksmbd_cipher_const_names, + ARRAY_SIZE(ksmbd_cipher_const_names), + cipher); + if (name) + seq_printf(m, "encryption:\t%s\n", name); + else + seq_printf(m, "encryption:\t0x%04x\n", + cipher); + } + i++; + } + up_read(&sess->chann_lock); + + seq_printf(m, "channels:\t%d\n", i); + + i = 0; + down_read(&sess->tree_conns_lock); + xa_for_each(&sess->tree_conns, id, tree_conn) { + share_conf = tree_conn->share_conf; + seq_printf(m, "share:\t%s\n", share_conf->name); + seq_printf(m, "tree_id:\t%d\n", tree_conn->id); + seq_printf(m, "share_type:\t%s\n", + test_share_config_flag(share_conf, KSMBD_SHARE_FLAG_PIPE) ? + "pipe" : "disk"); + i++; + } + up_read(&sess->tree_conns_lock); + seq_printf(m, "tree_connects:\t%d\n", i); + + ksmbd_user_session_put(sess); + return 0; +} + +static int create_proc_session(struct ksmbd_session *sess) +{ + char name[30]; + + snprintf(name, sizeof(name), "sessions/%llu", sess->id); + sess->proc_entry = ksmbd_proc_create(name, + show_proc_session, sess); + if (!sess->proc_entry) + return -ENOMEM; + return 0; +} + +static void delete_proc_session(struct ksmbd_session *sess) +{ + if (sess->proc_entry) + proc_remove(sess->proc_entry); +} + +static int show_proc_sessions(struct seq_file *m, void *v) +{ + struct ksmbd_session *session; + struct channel *chan; + int i; + unsigned long id; + + down_read(&sessions_table_lock); + hash_for_each(sessions_table, i, session, hlist) { + down_read(&session->chann_lock); + xa_for_each(&session->ksmbd_chann_list, id, chan) { + down_read(&chan->conn->session_lock); + ksmbd_user_session_get(session); + +#if IS_ENABLED(CONFIG_IPV6) + if (!chan->conn->inet_addr) + seq_printf(m, "client:\t%pI6c\n", &chan->conn->inet6_addr); + else +#endif + seq_printf(m, "client:\t%pI4\n", &chan->conn->inet_addr); + seq_printf(m, "user:\t%s\n", session_user_name(session)); + seq_printf(m, "id:\t%llu\n", session->id); + seq_printf(m, "state:\t%s\n\n", + session_state_string(session)); + + ksmbd_user_session_put(session); + up_read(&chan->conn->session_lock); + } + up_read(&session->chann_lock); + } + up_read(&sessions_table_lock); + return 0; +} + +int create_proc_sessions(void) +{ + if (!ksmbd_proc_create("sessions/sessions", + show_proc_sessions, NULL)) + return -ENOMEM; + return 0; +} +#else +int create_proc_sessions(void) { return 0; } +static int create_proc_session(struct ksmbd_session *sess) { return 0; } +static void delete_proc_session(struct ksmbd_session *sess) {} +#endif + +static void free_channel_list(struct ksmbd_session *sess) +{ + struct channel *chann; + unsigned long index; + + down_write(&sess->chann_lock); + xa_for_each(&sess->ksmbd_chann_list, index, chann) { + xa_erase(&sess->ksmbd_chann_list, index); + kfree_sensitive(chann); + } + + xa_destroy(&sess->ksmbd_chann_list); + up_write(&sess->chann_lock); +} + +static void __session_rpc_close(struct ksmbd_session *sess, + struct ksmbd_session_rpc *entry) +{ + struct ksmbd_rpc_command *resp; + + resp = ksmbd_rpc_close(sess, entry->id); + if (!resp) + pr_err("Unable to close RPC pipe %d\n", entry->id); + + kvfree(resp); + ksmbd_rpc_id_free(entry->id); + kfree(entry); +} + +static void ksmbd_session_rpc_clear_list(struct ksmbd_session *sess) +{ + struct ksmbd_session_rpc *entry; + long index; + + down_write(&sess->rpc_lock); + xa_for_each(&sess->rpc_handle_list, index, entry) { + xa_erase(&sess->rpc_handle_list, index); + __session_rpc_close(sess, entry); + } + up_write(&sess->rpc_lock); + + xa_destroy(&sess->rpc_handle_list); +} + +static int __rpc_method(char *rpc_name) +{ + if (!strcmp(rpc_name, "\\srvsvc") || !strcmp(rpc_name, "srvsvc")) + return KSMBD_RPC_SRVSVC_METHOD_INVOKE; + + if (!strcmp(rpc_name, "\\wkssvc") || !strcmp(rpc_name, "wkssvc")) + return KSMBD_RPC_WKSSVC_METHOD_INVOKE; + + if (!strcmp(rpc_name, "LANMAN") || !strcmp(rpc_name, "lanman")) + return KSMBD_RPC_RAP_METHOD; + + if (!strcmp(rpc_name, "\\samr") || !strcmp(rpc_name, "samr")) + return KSMBD_RPC_SAMR_METHOD_INVOKE; + + if (!strcmp(rpc_name, "\\lsarpc") || !strcmp(rpc_name, "lsarpc")) + return KSMBD_RPC_LSARPC_METHOD_INVOKE; + + if (!strcmp(rpc_name, "\\mdssvc") || !strcmp(rpc_name, "mdssvc")) + return -ENOENT; + + pr_err("Unsupported RPC: %s\n", rpc_name); + return -ENOENT; +} + +int ksmbd_session_rpc_open(struct ksmbd_session *sess, char *rpc_name) +{ + struct ksmbd_session_rpc *entry, *old; + struct ksmbd_rpc_command *resp; + int method, id; + + method = __rpc_method(rpc_name); + if (method < 0) + return method; + + entry = kzalloc_obj(struct ksmbd_session_rpc, KSMBD_DEFAULT_GFP); + if (!entry) + return -ENOMEM; + + entry->method = method; + entry->id = id = ksmbd_ipc_id_alloc(); + if (id < 0) + goto free_entry; + + down_write(&sess->rpc_lock); + old = xa_store(&sess->rpc_handle_list, id, entry, KSMBD_DEFAULT_GFP); + if (xa_is_err(old)) { + up_write(&sess->rpc_lock); + goto free_id; + } + + resp = ksmbd_rpc_open(sess, id); + if (!resp) { + xa_erase(&sess->rpc_handle_list, entry->id); + up_write(&sess->rpc_lock); + goto free_id; + } + + up_write(&sess->rpc_lock); + kvfree(resp); + return id; +free_id: + ksmbd_rpc_id_free(entry->id); +free_entry: + kfree(entry); + return -EINVAL; +} + +void ksmbd_session_rpc_close(struct ksmbd_session *sess, int id) +{ + struct ksmbd_session_rpc *entry; + + down_write(&sess->rpc_lock); + entry = xa_erase(&sess->rpc_handle_list, id); + if (entry) + __session_rpc_close(sess, entry); + up_write(&sess->rpc_lock); +} + +int ksmbd_session_rpc_method(struct ksmbd_session *sess, int id) +{ + struct ksmbd_session_rpc *entry; + + lockdep_assert_held(&sess->rpc_lock); + entry = xa_load(&sess->rpc_handle_list, id); + + return entry ? entry->method : 0; +} + +void ksmbd_session_destroy(struct ksmbd_session *sess) +{ + if (!sess) + return; + + delete_proc_session(sess); + ksmbd_tree_conn_session_logoff(sess); + ksmbd_destroy_file_table(sess); + if (sess->user) + ksmbd_free_user(sess->user); + ksmbd_launch_ksmbd_durable_scavenger(); + ksmbd_session_rpc_clear_list(sess); + free_channel_list(sess); + kfree_sensitive(sess->Preauth_HashValue); + ksmbd_release_id(&session_ida, sess->id); + ida_destroy(&sess->tree_conn_ida); + kfree_sensitive(sess); +} + +static void ksmbd_session_remove_from_table(struct ksmbd_session *sess) +{ + hash_del(&sess->hlist); + ksmbd_counter_dec(KSMBD_COUNTER_SESSIONS); +} + +struct ksmbd_session *__session_lookup(unsigned long long id) +{ + struct ksmbd_session *sess; + + hash_for_each_possible(sessions_table, sess, hlist, id) { + if (id == sess->id) { + sess->last_active = jiffies; + return sess; + } + } + return NULL; +} + +static void ksmbd_expire_session(struct ksmbd_conn *conn) +{ + unsigned long id; + struct ksmbd_session *sess; + + down_write(&sessions_table_lock); + down_write(&conn->session_lock); + xa_for_each(&conn->sessions, id, sess) { + if (atomic_read(&sess->refcnt) <= 1 && + (sess->state != SMB2_SESSION_VALID || + time_after(jiffies, + sess->last_active + SMB2_SESSION_TIMEOUT))) { + xa_erase(&conn->sessions, sess->id); + ksmbd_session_remove_from_table(sess); + ksmbd_session_destroy(sess); + continue; + } + } + up_write(&conn->session_lock); + up_write(&sessions_table_lock); +} + +int ksmbd_session_register(struct ksmbd_conn *conn, + struct ksmbd_session *sess) +{ + int ret; + + sess->dialect = conn->dialect; + memcpy(sess->ClientGUID, conn->ClientGUID, SMB2_CLIENT_GUID_SIZE); + ksmbd_expire_session(conn); + ret = xa_err(xa_store(&conn->sessions, sess->id, sess, + KSMBD_DEFAULT_GFP)); + if (ret) { + down_write(&sessions_table_lock); + ksmbd_session_remove_from_table(sess); + up_write(&sessions_table_lock); + ksmbd_user_session_put(sess); + } + + return ret; +} + +static int ksmbd_chann_del(struct ksmbd_conn *conn, struct ksmbd_session *sess) +{ + struct channel *chann; + + down_write(&sess->chann_lock); + chann = xa_erase(&sess->ksmbd_chann_list, (long)conn); + up_write(&sess->chann_lock); + if (!chann) + return -ENOENT; + + kfree_sensitive(chann); + return 0; +} + +void ksmbd_sessions_deregister(struct ksmbd_conn *conn) +{ + struct ksmbd_session *sess; + unsigned long id; + struct hlist_node *tmp; + int bkt; + + down_write(&sessions_table_lock); + hash_for_each_safe(sessions_table, bkt, tmp, sess, hlist) { + if (!ksmbd_chann_del(conn, sess) && + xa_empty(&sess->ksmbd_chann_list)) { + ksmbd_session_remove_from_table(sess); + down_write(&conn->session_lock); + xa_erase(&conn->sessions, sess->id); + up_write(&conn->session_lock); + if (atomic_dec_and_test(&sess->refcnt)) + ksmbd_session_destroy(sess); + } + } + + down_write(&conn->session_lock); + xa_for_each(&conn->sessions, id, sess) { + ksmbd_chann_del(conn, sess); + if (xa_empty(&sess->ksmbd_chann_list)) { + xa_erase(&conn->sessions, sess->id); + ksmbd_session_remove_from_table(sess); + if (atomic_dec_and_test(&sess->refcnt)) + ksmbd_session_destroy(sess); + } + } + up_write(&conn->session_lock); + up_write(&sessions_table_lock); +} + +bool is_ksmbd_session_in_connection(struct ksmbd_conn *conn, + unsigned long long id) +{ + struct ksmbd_session *sess; + + down_read(&conn->session_lock); + sess = xa_load(&conn->sessions, id); + if (sess) { + up_read(&conn->session_lock); + return true; + } + up_read(&conn->session_lock); + + return false; +} + +struct ksmbd_session *ksmbd_session_lookup(struct ksmbd_conn *conn, + unsigned long long id) +{ + struct ksmbd_session *sess; + + down_read(&conn->session_lock); + sess = xa_load(&conn->sessions, id); + if (sess) { + sess->last_active = jiffies; + ksmbd_user_session_get(sess); + } + up_read(&conn->session_lock); + return sess; +} + +struct ksmbd_session *ksmbd_session_lookup_slowpath(unsigned long long id) +{ + struct ksmbd_session *sess; + + down_read(&sessions_table_lock); + sess = __session_lookup(id); + if (sess) + ksmbd_user_session_get(sess); + up_read(&sessions_table_lock); + + return sess; +} + +struct ksmbd_session *ksmbd_session_lookup_all_states(struct ksmbd_conn *conn, + unsigned long long id) +{ + struct ksmbd_session *sess; + bool channel_found; + + sess = ksmbd_session_lookup(conn, id); + if (!sess) { + sess = ksmbd_session_lookup_slowpath(id); + if (!sess) + return NULL; + + down_read(&sess->chann_lock); + channel_found = xa_load(&sess->ksmbd_chann_list, (long)conn); + up_read(&sess->chann_lock); + if (!channel_found) { + ksmbd_user_session_put(sess); + sess = NULL; + } + } + return sess; +} + +struct ksmbd_session *ksmbd_session_lookup_all(struct ksmbd_conn *conn, + unsigned long long id) +{ + struct ksmbd_session *sess; + + sess = ksmbd_session_lookup_all_states(conn, id); + if (sess && sess->state != SMB2_SESSION_VALID) { + ksmbd_user_session_put(sess); + sess = NULL; + } + return sess; +} + +void ksmbd_user_session_get(struct ksmbd_session *sess) +{ + atomic_inc(&sess->refcnt); +} + +void ksmbd_user_session_put(struct ksmbd_session *sess) +{ + if (!sess) + return; + + if (atomic_read(&sess->refcnt) <= 0) + WARN_ON(1); + else if (atomic_dec_and_test(&sess->refcnt)) + ksmbd_session_destroy(sess); +} + +struct preauth_session *ksmbd_preauth_session_alloc(struct ksmbd_conn *conn, + u64 sess_id) +{ + struct preauth_session *sess; + + sess = kmalloc_obj(struct preauth_session, KSMBD_DEFAULT_GFP); + if (!sess) + return NULL; + + sess->id = sess_id; + memcpy(sess->Preauth_HashValue, conn->preauth_info->Preauth_HashValue, + PREAUTH_HASHVALUE_SIZE); + list_add(&sess->preauth_entry, &conn->preauth_sess_table); + + return sess; +} + +void ksmbd_preauth_session_destroy(struct ksmbd_conn *conn) +{ + struct preauth_session *sess, *tmp; + + list_for_each_entry_safe(sess, tmp, &conn->preauth_sess_table, + preauth_entry) { + list_del(&sess->preauth_entry); + kfree(sess); + } +} + +void destroy_previous_session(struct ksmbd_conn *conn, + struct ksmbd_user *user, u64 id) +{ + struct ksmbd_session *prev_sess; + struct ksmbd_user *prev_user; + int err; + + down_write(&sessions_table_lock); + down_write(&conn->session_lock); + prev_sess = __session_lookup(id); + if (!prev_sess || prev_sess->state == SMB2_SESSION_EXPIRED) + goto out; + + prev_user = prev_sess->user; + if (!prev_user || + strcmp(user->name, prev_user->name) || + user->passkey_sz != prev_user->passkey_sz || + memcmp(user->passkey, prev_user->passkey, user->passkey_sz)) + goto out; + + ksmbd_all_conn_set_status(prev_sess, KSMBD_SESS_NEED_RECONNECT); + err = ksmbd_conn_wait_idle_sess(conn, prev_sess); + if (err) { + ksmbd_all_conn_set_status(prev_sess, KSMBD_SESS_NEED_SETUP); + goto out; + } + + ksmbd_destroy_file_table(prev_sess); + prev_sess->kerberos_expiry = 0; + prev_sess->state = SMB2_SESSION_EXPIRED; + ksmbd_all_conn_set_status(prev_sess, KSMBD_SESS_NEED_SETUP); + ksmbd_launch_ksmbd_durable_scavenger(); +out: + up_write(&conn->session_lock); + up_write(&sessions_table_lock); +} + +static bool ksmbd_preauth_session_id_match(struct preauth_session *sess, + unsigned long long id) +{ + return sess->id == id; +} + +struct preauth_session *ksmbd_preauth_session_lookup(struct ksmbd_conn *conn, + unsigned long long id) +{ + struct preauth_session *sess = NULL; + + list_for_each_entry(sess, &conn->preauth_sess_table, preauth_entry) { + if (ksmbd_preauth_session_id_match(sess, id)) + return sess; + } + return NULL; +} + +static int __init_smb2_session(struct ksmbd_session *sess) +{ + int id = ksmbd_acquire_smb2_uid(&session_ida); + + if (id < 0) + return -EINVAL; + sess->id = id; + return 0; +} + +static struct ksmbd_session *__session_create(int protocol) +{ + struct ksmbd_session *sess; + int ret; + + if (protocol != CIFDS_SESSION_FLAG_SMB2) + return NULL; + + sess = kzalloc_obj(struct ksmbd_session, KSMBD_DEFAULT_GFP); + if (!sess) + return NULL; + + ida_init(&sess->tree_conn_ida); + + if (ksmbd_init_file_table(&sess->file_table)) + goto error; + + sess->last_active = jiffies; + sess->state = SMB2_SESSION_IN_PROGRESS; + set_session_flag(sess, protocol); + xa_init(&sess->tree_conns); + xa_init(&sess->ksmbd_chann_list); + xa_init(&sess->rpc_handle_list); + sess->sequence_number = 1; + atomic_set(&sess->refcnt, 2); + init_rwsem(&sess->tree_conns_lock); + init_rwsem(&sess->rpc_lock); + init_rwsem(&sess->chann_lock); + + ret = __init_smb2_session(sess); + if (ret) + goto error; + + down_write(&sessions_table_lock); + hash_add(sessions_table, &sess->hlist, sess->id); + ksmbd_counter_inc(KSMBD_COUNTER_SESSIONS); + up_write(&sessions_table_lock); + + if (create_proc_session(sess)) + pr_warn_ratelimited("Unable to create session %llu procfs entry\n", sess->id); + return sess; + +error: + ksmbd_session_destroy(sess); + return NULL; +} + +struct ksmbd_session *ksmbd_smb2_session_create(void) +{ + return __session_create(CIFDS_SESSION_FLAG_SMB2); +} + +int ksmbd_acquire_tree_conn_id(struct ksmbd_session *sess) +{ + int id = -EINVAL; + + if (test_session_flag(sess, CIFDS_SESSION_FLAG_SMB2)) + id = ksmbd_acquire_smb2_tid(&sess->tree_conn_ida); + + return id; +} + +void ksmbd_release_tree_conn_id(struct ksmbd_session *sess, int id) +{ + if (id >= 0) + ksmbd_release_id(&sess->tree_conn_ida, id); +} diff --git a/fs/smb/server/mgmt/user_session.h b/fs/smb/server/mgmt/user_session.h new file mode 100644 index 000000000..f8a24c33f --- /dev/null +++ b/fs/smb/server/mgmt/user_session.h @@ -0,0 +1,123 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#ifndef __USER_SESSION_MANAGEMENT_H__ +#define __USER_SESSION_MANAGEMENT_H__ + +#include +#include + +#include "../smb_common.h" +#include "../ntlmssp.h" + +#define CIFDS_SESSION_FLAG_SMB2 BIT(1) + +#define PREAUTH_HASHVALUE_SIZE 64 + +struct ksmbd_file_table; + +struct channel { + char sess_key[CIFS_KEY_SIZE]; + __u8 smb3signingkey[SMB3_SIGN_KEY_SIZE]; + struct ksmbd_conn *conn; +}; + +struct preauth_session { + __u8 Preauth_HashValue[PREAUTH_HASHVALUE_SIZE]; + u64 id; + struct list_head preauth_entry; +}; + +struct ksmbd_session { + u64 id; + + __u16 dialect; + char ClientGUID[SMB2_CLIENT_GUID_SIZE]; + + struct ksmbd_user *user; + unsigned int sequence_number; + unsigned int flags; + + bool sign; + bool enc; + + int state; + __u8 *Preauth_HashValue; + + char sess_key[CIFS_KEY_SIZE]; + u64 kerberos_expiry; + + struct hlist_node hlist; + struct rw_semaphore chann_lock; + struct xarray ksmbd_chann_list; + struct xarray tree_conns; + struct ida tree_conn_ida; + struct xarray rpc_handle_list; + + __u8 smb3encryptionkey[SMB3_ENC_DEC_KEY_SIZE]; + __u8 smb3decryptionkey[SMB3_ENC_DEC_KEY_SIZE]; + __u8 smb3signingkey[SMB3_SIGN_KEY_SIZE]; + + struct ksmbd_file_table file_table; + unsigned long last_active; + struct rw_semaphore tree_conns_lock; + +#ifdef CONFIG_PROC_FS + struct proc_dir_entry *proc_entry; +#endif + atomic_t refcnt; + struct rw_semaphore rpc_lock; +}; + +static inline int test_session_flag(struct ksmbd_session *sess, int bit) +{ + return sess->flags & bit; +} + +static inline void set_session_flag(struct ksmbd_session *sess, int bit) +{ + sess->flags |= bit; +} + +static inline void clear_session_flag(struct ksmbd_session *sess, int bit) +{ + sess->flags &= ~bit; +} + +struct ksmbd_session *ksmbd_smb2_session_create(void); + +void ksmbd_session_destroy(struct ksmbd_session *sess); + +struct ksmbd_session *ksmbd_session_lookup_slowpath(unsigned long long id); +struct ksmbd_session *ksmbd_session_lookup(struct ksmbd_conn *conn, + unsigned long long id); +bool is_ksmbd_session_in_connection(struct ksmbd_conn *conn, + unsigned long long id); +int ksmbd_session_register(struct ksmbd_conn *conn, + struct ksmbd_session *sess); +void ksmbd_sessions_deregister(struct ksmbd_conn *conn); +struct ksmbd_session *__session_lookup(unsigned long long id); +struct ksmbd_session *ksmbd_session_lookup_all(struct ksmbd_conn *conn, + unsigned long long id); +struct ksmbd_session *ksmbd_session_lookup_all_states(struct ksmbd_conn *conn, + unsigned long long id); +void destroy_previous_session(struct ksmbd_conn *conn, + struct ksmbd_user *user, u64 id); +struct preauth_session *ksmbd_preauth_session_alloc(struct ksmbd_conn *conn, + u64 sess_id); +void ksmbd_preauth_session_destroy(struct ksmbd_conn *conn); +struct preauth_session *ksmbd_preauth_session_lookup(struct ksmbd_conn *conn, + unsigned long long id); + +int ksmbd_acquire_tree_conn_id(struct ksmbd_session *sess); +void ksmbd_release_tree_conn_id(struct ksmbd_session *sess, int id); + +int ksmbd_session_rpc_open(struct ksmbd_session *sess, char *rpc_name); +void ksmbd_session_rpc_close(struct ksmbd_session *sess, int id); +int ksmbd_session_rpc_method(struct ksmbd_session *sess, int id); +void ksmbd_user_session_get(struct ksmbd_session *sess); +void ksmbd_user_session_put(struct ksmbd_session *sess); +int create_proc_sessions(void); +#endif /* __USER_SESSION_MANAGEMENT_H__ */ diff --git a/fs/smb/server/misc.c b/fs/smb/server/misc.c new file mode 100644 index 000000000..2dd91c9e9 --- /dev/null +++ b/fs/smb/server/misc.c @@ -0,0 +1,363 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2016 Namjae Jeon + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#include +#include +#include +#include + +#include "misc.h" +#include "smb_common.h" +#include "connection.h" +#include "vfs.h" + +#include "mgmt/share_config.h" + +/** + * match_pattern() - compare a string with a pattern which might include + * wildcard '*' and '?' + * TODO : implement consideration about DOS_DOT, DOS_QM and DOS_STAR + * + * @str: string to compare with a pattern + * @len: string length + * @pattern: pattern string which might include wildcard '*' and '?' + * + * Return: 0 if pattern matched with the string, otherwise non zero value + */ +int match_pattern(const char *str, size_t len, const char *pattern) +{ + const char *s = str; + const char *p = pattern; + bool star = false; + + while (*s && len) { + switch (*p) { + case '?': + s++; + len--; + p++; + break; + case '*': + star = true; + str = s; + if (!*++p) + return true; + pattern = p; + break; + default: + if (tolower(*s) == tolower(*p)) { + s++; + len--; + p++; + } else { + if (!star) + return false; + str++; + s = str; + p = pattern; + } + break; + } + } + + if (*p == '*') + ++p; + return !*p; +} + +/* + * is_char_allowed() - check for valid character + * @ch: input character to be checked + * + * Return: 1 if char is allowed, otherwise 0 + */ +static inline int is_char_allowed(char ch) +{ + /* check for control chars, wildcards etc. */ + if (!(ch & 0x80) && + (ch <= 0x1f || + ch == '?' || ch == '"' || ch == '<' || + ch == '>' || ch == '|' || ch == '*')) + return 0; + + return 1; +} + +int ksmbd_validate_filename(char *filename) +{ + while (*filename) { + char c = *filename; + + filename++; + if (!is_char_allowed(c)) { + ksmbd_debug(VFS, "File name validation failed: 0x%x\n", c); + return -ENOENT; + } + } + + return 0; +} + +static int ksmbd_validate_stream_name(char *stream_name) +{ + while (*stream_name) { + char c = *stream_name; + + stream_name++; + if (c == '/' || c == ':' || c == '\\') { + pr_err("Stream name validation failed: %c\n", c); + return -ENOENT; + } + } + + return 0; +} + +int parse_stream_name(char *filename, char **stream_name, int *s_type) +{ + char *stream_type; + char *s_name; + int rc = 0; + bool has_stream_type = false; + + *stream_name = NULL; + s_name = filename; + filename = strsep(&s_name, ":"); + ksmbd_debug(SMB, "filename : %s, streams : %s\n", filename, s_name); + if (strchr(s_name, ':')) { + stream_type = s_name; + s_name = strsep(&stream_type, ":"); + + rc = ksmbd_validate_stream_name(s_name); + if (rc < 0) { + rc = -ENOENT; + goto out; + } + + ksmbd_debug(SMB, "stream name : %s, stream type : %s\n", s_name, + stream_type); + if (!strncasecmp("$data", stream_type, 5)) { + *s_type = DATA_STREAM; + has_stream_type = true; + } else if (!strncasecmp("$index_allocation", stream_type, 17)) { + *s_type = DIR_STREAM; + has_stream_type = true; + } else { + rc = -ENOENT; + } + } + + if (has_stream_type && !s_name[0] && *s_type == DATA_STREAM) + goto out; + + *stream_name = s_name; +out: + return rc; +} + +/** + * convert_to_nt_pathname() - extract and return windows path string + * whose share directory prefix was removed from file path + * @share: ksmbd_share_config pointer + * @path: path to report + * + * Return : windows path string or error + */ + +char *convert_to_nt_pathname(struct ksmbd_share_config *share, + const struct path *path) +{ + char *pathname, *ab_pathname, *nt_pathname; + int share_path_len = share->path_sz; + size_t ab_pathname_len; + int prefix; + + pathname = kmalloc(PATH_MAX, KSMBD_DEFAULT_GFP); + if (!pathname) + return ERR_PTR(-EACCES); + + ab_pathname = d_path(path, pathname, PATH_MAX); + if (IS_ERR(ab_pathname)) { + nt_pathname = ERR_PTR(-EACCES); + goto free_pathname; + } + + if (strncmp(ab_pathname, share->path, share_path_len)) { + nt_pathname = ERR_PTR(-EACCES); + goto free_pathname; + } + + ab_pathname_len = strlen(&ab_pathname[share_path_len]); + prefix = ab_pathname[share_path_len] == '\0' ? 1 : 0; + nt_pathname = kmalloc(prefix + ab_pathname_len + 1, KSMBD_DEFAULT_GFP); + if (!nt_pathname) { + nt_pathname = ERR_PTR(-ENOMEM); + goto free_pathname; + } + + if (prefix) + *nt_pathname = '/'; + memcpy(nt_pathname + prefix, &ab_pathname[share_path_len], + ab_pathname_len + 1); + + ksmbd_conv_path_to_windows(nt_pathname); + +free_pathname: + kfree(pathname); + return nt_pathname; +} + +int get_nlink(struct kstat *st) +{ + int nlink; + + nlink = st->nlink; + if (S_ISDIR(st->mode)) + nlink--; + + return nlink; +} + +void ksmbd_conv_path_to_unix(char *path) +{ + strreplace(path, '\\', '/'); +} + +void ksmbd_strip_last_slash(char *path) +{ + int len = strlen(path); + + while (len && path[len - 1] == '/') { + path[len - 1] = '\0'; + len--; + } +} + +void ksmbd_conv_path_to_windows(char *path) +{ + strreplace(path, '/', '\\'); +} + +char *ksmbd_casefold_sharename(struct unicode_map *um, const char *name) +{ + char *cf_name; + int cf_len; + + cf_name = kzalloc(KSMBD_REQ_MAX_SHARE_NAME, KSMBD_DEFAULT_GFP); + if (!cf_name) + return ERR_PTR(-ENOMEM); + + if (IS_ENABLED(CONFIG_UNICODE) && um) { + const struct qstr q_name = {.name = name, .len = strlen(name)}; + + cf_len = utf8_casefold(um, &q_name, cf_name, + KSMBD_REQ_MAX_SHARE_NAME); + if (cf_len < 0) + goto out_ascii; + + return cf_name; + } + +out_ascii: + cf_len = strscpy(cf_name, name, KSMBD_REQ_MAX_SHARE_NAME); + if (cf_len < 0) { + kfree(cf_name); + return ERR_PTR(-E2BIG); + } + + for (; *cf_name; ++cf_name) + *cf_name = isascii(*cf_name) ? tolower(*cf_name) : *cf_name; + return cf_name - cf_len; +} + +/** + * ksmbd_extract_sharename() - get share name from tree connect request + * @um: pointer to a unicode_map structure for character encoding handling + * @treename: buffer containing tree name and share name + * + * Return: share name on success, otherwise error + */ +char *ksmbd_extract_sharename(struct unicode_map *um, const char *treename) +{ + const char *name = treename, *pos = strrchr(name, '\\'); + + if (pos) + name = (pos + 1); + + /* caller has to free the memory */ + return ksmbd_casefold_sharename(um, name); +} + +char *ksmbd_convert_dir_info_name(struct ksmbd_dir_info *d_info, + const struct nls_table *local_nls, + int *conv_len) +{ + char *conv; + int sz = min(4 * d_info->name_len, PATH_MAX); + + if (!sz) + return NULL; + + conv = kmalloc(sz, KSMBD_DEFAULT_GFP); + if (!conv) + return NULL; + + /* XXX */ + *conv_len = smbConvertToUTF16((__le16 *)conv, d_info->name, + d_info->name_len, local_nls, 0); + *conv_len *= 2; + + /* We allocate buffer twice bigger than needed. */ + conv[*conv_len] = 0x00; + conv[*conv_len + 1] = 0x00; + return conv; +} + +/* + * Convert the NT UTC (based 1601-01-01, in hundred nanosecond units) + * into Unix UTC (based 1970-01-01, in seconds). + */ +struct timespec64 ksmbd_NTtimeToUnix(__le64 ntutc) +{ + struct timespec64 ts; + + /* Subtract the NTFS time offset, then convert to 1s intervals. */ + s64 t = le64_to_cpu(ntutc) - NTFS_TIME_OFFSET; + u64 abs_t; + + /* + * Unfortunately can not use normal 64 bit division on 32 bit arch, but + * the alternative, do_div, does not work with negative numbers so have + * to special case them + */ + if (t < 0) { + abs_t = -t; + ts.tv_nsec = do_div(abs_t, 10000000) * 100; + ts.tv_nsec = -ts.tv_nsec; + ts.tv_sec = -abs_t; + } else { + abs_t = t; + ts.tv_nsec = do_div(abs_t, 10000000) * 100; + ts.tv_sec = abs_t; + } + + return ts; +} + +/* Convert the Unix UTC into NT UTC. */ +inline u64 ksmbd_UnixTimeToNT(struct timespec64 t) +{ + /* Convert to 100ns intervals and then add the NTFS time offset. */ + return (u64)t.tv_sec * 10000000 + t.tv_nsec / 100 + NTFS_TIME_OFFSET; +} + +inline long long ksmbd_systime(void) +{ + struct timespec64 ts; + + ktime_get_real_ts64(&ts); + return ksmbd_UnixTimeToNT(ts); +} diff --git a/fs/smb/server/misc.h b/fs/smb/server/misc.h new file mode 100644 index 000000000..1faaddd0f --- /dev/null +++ b/fs/smb/server/misc.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#ifndef __KSMBD_MISC_H__ +#define __KSMBD_MISC_H__ + +#ifdef CONFIG_PROC_FS +#include +#endif +struct ksmbd_share_config; +struct nls_table; +struct kstat; +struct ksmbd_file; + +int match_pattern(const char *str, size_t len, const char *pattern); +int ksmbd_validate_filename(char *filename); +int parse_stream_name(char *filename, char **stream_name, int *s_type); +char *convert_to_nt_pathname(struct ksmbd_share_config *share, + const struct path *path); +int get_nlink(struct kstat *st); +void ksmbd_conv_path_to_unix(char *path); +void ksmbd_strip_last_slash(char *path); +void ksmbd_conv_path_to_windows(char *path); +char *ksmbd_casefold_sharename(struct unicode_map *um, const char *name); +char *ksmbd_extract_sharename(struct unicode_map *um, const char *treename); + +#define KSMBD_DIR_INFO_ALIGNMENT 8 +struct ksmbd_dir_info; +char *ksmbd_convert_dir_info_name(struct ksmbd_dir_info *d_info, + const struct nls_table *local_nls, + int *conv_len); + +#define NTFS_TIME_OFFSET ((u64)(369 * 365 + 89) * 24 * 3600 * 10000000) +struct timespec64 ksmbd_NTtimeToUnix(__le64 ntutc); +u64 ksmbd_UnixTimeToNT(struct timespec64 t); +long long ksmbd_systime(void); + +#ifdef CONFIG_PROC_FS +struct ksmbd_const_name { + unsigned int const_value; + const char *name; +}; + +int ksmbd_proc_init(void); +void ksmbd_proc_cleanup(void); +void ksmbd_proc_reset(void); +struct proc_dir_entry *ksmbd_proc_create(const char *name, + int (*show)(struct seq_file *m, void *v), + void *v); +void ksmbd_proc_show_flag_names(struct seq_file *m, + const struct ksmbd_const_name *table, + int count, + unsigned int flags); +const char *ksmbd_proc_const_name(const struct ksmbd_const_name *table, + int count, unsigned int const_value); +#else +static inline int ksmbd_proc_init(void) { return 0; } +static inline void ksmbd_proc_cleanup(void) {} +static inline void ksmbd_proc_reset(void) {} +#endif +#endif /* __KSMBD_MISC_H__ */ diff --git a/fs/smb/server/ndr.c b/fs/smb/server/ndr.c new file mode 100644 index 000000000..58d71560f --- /dev/null +++ b/fs/smb/server/ndr.c @@ -0,0 +1,514 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2021 Samsung Electronics Co., Ltd. + * Author(s): Namjae Jeon + */ + +#include + +#include "glob.h" +#include "ndr.h" + +static inline char *ndr_get_field(struct ndr *n) +{ + return n->data + n->offset; +} + +static int try_to_realloc_ndr_blob(struct ndr *n, size_t sz) +{ + char *data; + + data = krealloc(n->data, n->offset + sz + 1024, KSMBD_DEFAULT_GFP); + if (!data) + return -ENOMEM; + + n->data = data; + n->length += 1024; + memset(n->data + n->offset, 0, 1024); + return 0; +} + +static int ndr_write_int16(struct ndr *n, __u16 value) +{ + if (n->length <= n->offset + sizeof(value)) { + int ret; + + ret = try_to_realloc_ndr_blob(n, sizeof(value)); + if (ret) + return ret; + } + + *(__le16 *)ndr_get_field(n) = cpu_to_le16(value); + n->offset += sizeof(value); + return 0; +} + +static int ndr_write_int32(struct ndr *n, __u32 value) +{ + if (n->length <= n->offset + sizeof(value)) { + int ret; + + ret = try_to_realloc_ndr_blob(n, sizeof(value)); + if (ret) + return ret; + } + + *(__le32 *)ndr_get_field(n) = cpu_to_le32(value); + n->offset += sizeof(value); + return 0; +} + +static int ndr_write_int64(struct ndr *n, __u64 value) +{ + if (n->length <= n->offset + sizeof(value)) { + int ret; + + ret = try_to_realloc_ndr_blob(n, sizeof(value)); + if (ret) + return ret; + } + + *(__le64 *)ndr_get_field(n) = cpu_to_le64(value); + n->offset += sizeof(value); + return 0; +} + +static int ndr_write_bytes(struct ndr *n, void *value, size_t sz) +{ + if (n->length <= n->offset + sz) { + int ret; + + ret = try_to_realloc_ndr_blob(n, sz); + if (ret) + return ret; + } + + memcpy(ndr_get_field(n), value, sz); + n->offset += sz; + return 0; +} + +static int ndr_write_string(struct ndr *n, char *value) +{ + size_t sz; + + sz = strlen(value) + 1; + if (n->length <= n->offset + sz) { + int ret; + + ret = try_to_realloc_ndr_blob(n, sz); + if (ret) + return ret; + } + + memcpy(ndr_get_field(n), value, sz); + n->offset += sz; + n->offset = ALIGN(n->offset, 2); + return 0; +} + +static int ndr_read_string(struct ndr *n, void *value, size_t sz) +{ + int len; + + if (n->offset + sz > n->length) + return -EINVAL; + + len = strnlen(ndr_get_field(n), sz); + if (value) + memcpy(value, ndr_get_field(n), len); + len++; + n->offset += len; + n->offset = ALIGN(n->offset, 2); + return 0; +} + +static int ndr_read_bytes(struct ndr *n, void *value, size_t sz) +{ + if (n->offset + sz > n->length) + return -EINVAL; + + if (value) + memcpy(value, ndr_get_field(n), sz); + n->offset += sz; + return 0; +} + +static int ndr_read_int16(struct ndr *n, __u16 *value) +{ + if (n->offset + sizeof(__u16) > n->length) + return -EINVAL; + + if (value) + *value = le16_to_cpu(*(__le16 *)ndr_get_field(n)); + n->offset += sizeof(__u16); + return 0; +} + +static int ndr_read_int32(struct ndr *n, __u32 *value) +{ + if (n->offset + sizeof(__u32) > n->length) + return -EINVAL; + + if (value) + *value = le32_to_cpu(*(__le32 *)ndr_get_field(n)); + n->offset += sizeof(__u32); + return 0; +} + +static int ndr_read_int64(struct ndr *n, __u64 *value) +{ + if (n->offset + sizeof(__u64) > n->length) + return -EINVAL; + + if (value) + *value = le64_to_cpu(*(__le64 *)ndr_get_field(n)); + n->offset += sizeof(__u64); + return 0; +} + +int ndr_encode_dos_attr(struct ndr *n, struct xattr_dos_attrib *da) +{ + char hex_attr[12] = {0}; + int ret; + + n->offset = 0; + n->length = 1024; + n->data = kzalloc(n->length, KSMBD_DEFAULT_GFP); + if (!n->data) + return -ENOMEM; + + if (da->version == 3) { + snprintf(hex_attr, 10, "0x%x", da->attr); + ret = ndr_write_string(n, hex_attr); + } else { + ret = ndr_write_string(n, ""); + } + if (ret) + return ret; + + ret = ndr_write_int16(n, da->version); + if (ret) + return ret; + + ret = ndr_write_int32(n, da->version); + if (ret) + return ret; + + ret = ndr_write_int32(n, da->flags); + if (ret) + return ret; + + ret = ndr_write_int32(n, da->attr); + if (ret) + return ret; + + if (da->version == 3) { + ret = ndr_write_int32(n, da->ea_size); + if (ret) + return ret; + ret = ndr_write_int64(n, da->size); + if (ret) + return ret; + ret = ndr_write_int64(n, da->alloc_size); + } else { + ret = ndr_write_int64(n, da->itime); + } + if (ret) + return ret; + + ret = ndr_write_int64(n, da->create_time); + if (ret) + return ret; + + if (da->version == 3) + ret = ndr_write_int64(n, da->change_time); + return ret; +} + +int ndr_decode_dos_attr(struct ndr *n, struct xattr_dos_attrib *da) +{ + char hex_attr[12]; + unsigned int version2; + int ret; + + n->offset = 0; + ret = ndr_read_string(n, hex_attr, sizeof(hex_attr)); + if (ret) + return ret; + + ret = ndr_read_int16(n, &da->version); + if (ret) + return ret; + + if (da->version != 3 && da->version != 4) { + ksmbd_debug(VFS, "v%d version is not supported\n", da->version); + return -EINVAL; + } + + ret = ndr_read_int32(n, &version2); + if (ret) + return ret; + + if (da->version != version2) { + ksmbd_debug(VFS, "ndr version mismatched(version: %d, version2: %d)\n", + da->version, version2); + return -EINVAL; + } + + ret = ndr_read_int32(n, NULL); + if (ret) + return ret; + + ret = ndr_read_int32(n, &da->attr); + if (ret) + return ret; + + if (da->version == 4) { + ret = ndr_read_int64(n, &da->itime); + if (ret) + return ret; + + ret = ndr_read_int64(n, &da->create_time); + } else { + ret = ndr_read_int32(n, NULL); + if (ret) + return ret; + + ret = ndr_read_int64(n, NULL); + if (ret) + return ret; + + ret = ndr_read_int64(n, NULL); + if (ret) + return ret; + + ret = ndr_read_int64(n, &da->create_time); + if (ret) + return ret; + + ret = ndr_read_int64(n, NULL); + } + + return ret; +} + +static int ndr_encode_posix_acl_entry(struct ndr *n, struct xattr_smb_acl *acl) +{ + int i, ret; + + ret = ndr_write_int32(n, acl->count); + if (ret) + return ret; + + n->offset = ALIGN(n->offset, 8); + ret = ndr_write_int32(n, acl->count); + if (ret) + return ret; + + ret = ndr_write_int32(n, 0); + if (ret) + return ret; + + for (i = 0; i < acl->count; i++) { + n->offset = ALIGN(n->offset, 8); + ret = ndr_write_int16(n, acl->entries[i].type); + if (ret) + return ret; + + ret = ndr_write_int16(n, acl->entries[i].type); + if (ret) + return ret; + + if (acl->entries[i].type == SMB_ACL_USER) { + n->offset = ALIGN(n->offset, 8); + ret = ndr_write_int64(n, acl->entries[i].uid); + } else if (acl->entries[i].type == SMB_ACL_GROUP) { + n->offset = ALIGN(n->offset, 8); + ret = ndr_write_int64(n, acl->entries[i].gid); + } + if (ret) + return ret; + + /* push permission */ + ret = ndr_write_int32(n, acl->entries[i].perm); + } + + return ret; +} + +int ndr_encode_posix_acl(struct ndr *n, + struct mnt_idmap *idmap, + struct inode *inode, + struct xattr_smb_acl *acl, + struct xattr_smb_acl *def_acl) +{ + unsigned int ref_id = 0x00020000; + int ret; + vfsuid_t vfsuid; + vfsgid_t vfsgid; + + n->offset = 0; + n->length = 1024; + n->data = kzalloc(n->length, KSMBD_DEFAULT_GFP); + if (!n->data) + return -ENOMEM; + + if (acl) { + /* ACL ACCESS */ + ret = ndr_write_int32(n, ref_id); + ref_id += 4; + } else { + ret = ndr_write_int32(n, 0); + } + if (ret) + return ret; + + if (def_acl) { + /* DEFAULT ACL ACCESS */ + ret = ndr_write_int32(n, ref_id); + ref_id += 4; + } else { + ret = ndr_write_int32(n, 0); + } + if (ret) + return ret; + + vfsuid = i_uid_into_vfsuid(idmap, inode); + ret = ndr_write_int64(n, from_kuid(&init_user_ns, vfsuid_into_kuid(vfsuid))); + if (ret) + return ret; + vfsgid = i_gid_into_vfsgid(idmap, inode); + ret = ndr_write_int64(n, from_kgid(&init_user_ns, vfsgid_into_kgid(vfsgid))); + if (ret) + return ret; + ret = ndr_write_int32(n, inode->i_mode); + if (ret) + return ret; + + if (acl) { + ret = ndr_encode_posix_acl_entry(n, acl); + if (def_acl && !ret) + ret = ndr_encode_posix_acl_entry(n, def_acl); + } + return ret; +} + +int ndr_encode_v4_ntacl(struct ndr *n, struct xattr_ntacl *acl) +{ + unsigned int ref_id = 0x00020004; + int ret; + + n->offset = 0; + n->length = 2048; + n->data = kzalloc(n->length, KSMBD_DEFAULT_GFP); + if (!n->data) + return -ENOMEM; + + ret = ndr_write_int16(n, acl->version); + if (ret) + return ret; + + ret = ndr_write_int32(n, acl->version); + if (ret) + return ret; + + ret = ndr_write_int16(n, 2); + if (ret) + return ret; + + ret = ndr_write_int32(n, ref_id); + if (ret) + return ret; + + /* push hash type and hash 64bytes */ + ret = ndr_write_int16(n, acl->hash_type); + if (ret) + return ret; + + ret = ndr_write_bytes(n, acl->hash, XATTR_SD_HASH_SIZE); + if (ret) + return ret; + + ret = ndr_write_bytes(n, acl->desc, acl->desc_len); + if (ret) + return ret; + + ret = ndr_write_int64(n, acl->current_time); + if (ret) + return ret; + + ret = ndr_write_bytes(n, acl->posix_acl_hash, XATTR_SD_HASH_SIZE); + if (ret) + return ret; + + /* push ndr for security descriptor */ + ret = ndr_write_bytes(n, acl->sd_buf, acl->sd_size); + return ret; +} + +int ndr_decode_v4_ntacl(struct ndr *n, struct xattr_ntacl *acl) +{ + unsigned int version2; + int ret; + + n->offset = 0; + ret = ndr_read_int16(n, &acl->version); + if (ret) + return ret; + if (acl->version != 4) { + ksmbd_debug(VFS, "v%d version is not supported\n", acl->version); + return -EINVAL; + } + + ret = ndr_read_int32(n, &version2); + if (ret) + return ret; + if (acl->version != version2) { + ksmbd_debug(VFS, "ndr version mismatched(version: %d, version2: %d)\n", + acl->version, version2); + return -EINVAL; + } + + /* Read Level */ + ret = ndr_read_int16(n, NULL); + if (ret) + return ret; + + /* Read Ref Id */ + ret = ndr_read_int32(n, NULL); + if (ret) + return ret; + + ret = ndr_read_int16(n, &acl->hash_type); + if (ret) + return ret; + + ret = ndr_read_bytes(n, acl->hash, XATTR_SD_HASH_SIZE); + if (ret) + return ret; + + ndr_read_bytes(n, acl->desc, 10); + if (strncmp(acl->desc, "posix_acl", 9)) { + pr_err("Invalid acl description : %s\n", acl->desc); + return -EINVAL; + } + + /* Read Time */ + ret = ndr_read_int64(n, NULL); + if (ret) + return ret; + + /* Read Posix ACL hash */ + ret = ndr_read_bytes(n, acl->posix_acl_hash, XATTR_SD_HASH_SIZE); + if (ret) + return ret; + + acl->sd_size = n->length - n->offset; + acl->sd_buf = kzalloc(acl->sd_size, KSMBD_DEFAULT_GFP); + if (!acl->sd_buf) + return -ENOMEM; + + ret = ndr_read_bytes(n, acl->sd_buf, acl->sd_size); + return ret; +} diff --git a/fs/smb/server/ndr.h b/fs/smb/server/ndr.h new file mode 100644 index 000000000..f3c108c8c --- /dev/null +++ b/fs/smb/server/ndr.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020 Samsung Electronics Co., Ltd. + * Author(s): Namjae Jeon + */ + +struct ndr { + char *data; + int offset; + int length; +}; + +#define NDR_NTSD_OFFSETOF 0xA0 + +int ndr_encode_dos_attr(struct ndr *n, struct xattr_dos_attrib *da); +int ndr_decode_dos_attr(struct ndr *n, struct xattr_dos_attrib *da); +int ndr_encode_posix_acl(struct ndr *n, struct mnt_idmap *idmap, + struct inode *inode, struct xattr_smb_acl *acl, + struct xattr_smb_acl *def_acl); +int ndr_encode_v4_ntacl(struct ndr *n, struct xattr_ntacl *acl); +int ndr_encode_v3_ntacl(struct ndr *n, struct xattr_ntacl *acl); +int ndr_decode_v4_ntacl(struct ndr *n, struct xattr_ntacl *acl); diff --git a/fs/smb/server/ntlmssp.h b/fs/smb/server/ntlmssp.h new file mode 100644 index 000000000..f13153c18 --- /dev/null +++ b/fs/smb/server/ntlmssp.h @@ -0,0 +1,169 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ +/* + * Copyright (c) International Business Machines Corp., 2002,2007 + * Author(s): Steve French (sfrench@us.ibm.com) + */ + +#ifndef __KSMBD_NTLMSSP_H +#define __KSMBD_NTLMSSP_H + +#define NTLMSSP_SIGNATURE "NTLMSSP" + +/* Security blob target info data */ +#define TGT_Name "KSMBD" + +/* + * Size of the crypto key returned on the negotiate SMB in bytes + */ +#define CIFS_CRYPTO_KEY_SIZE (8) +#define CIFS_KEY_SIZE (40) + +/* + * Size of encrypted user password in bytes + */ +#define CIFS_ENCPWD_SIZE (16) +#define CIFS_CPHTXT_SIZE (16) + +/* Message Types */ +#define NtLmNegotiate cpu_to_le32(1) +#define NtLmChallenge cpu_to_le32(2) +#define NtLmAuthenticate cpu_to_le32(3) +#define UnknownMessage cpu_to_le32(8) + +/* Negotiate Flags */ +#define NTLMSSP_NEGOTIATE_UNICODE 0x01 /* Text strings are unicode */ +#define NTLMSSP_NEGOTIATE_OEM 0x02 /* Text strings are in OEM */ +#define NTLMSSP_REQUEST_TARGET 0x04 /* Srv returns its auth realm */ +/* define reserved9 0x08 */ +#define NTLMSSP_NEGOTIATE_SIGN 0x0010 /* Request signing capability */ +#define NTLMSSP_NEGOTIATE_SEAL 0x0020 /* Request confidentiality */ +#define NTLMSSP_NEGOTIATE_DGRAM 0x0040 +#define NTLMSSP_NEGOTIATE_LM_KEY 0x0080 /* Use LM session key */ +/* defined reserved 8 0x0100 */ +#define NTLMSSP_NEGOTIATE_NTLM 0x0200 /* NTLM authentication */ +#define NTLMSSP_NEGOTIATE_NT_ONLY 0x0400 /* Lanman not allowed */ +#define NTLMSSP_ANONYMOUS 0x0800 +#define NTLMSSP_NEGOTIATE_DOMAIN_SUPPLIED 0x1000 /* reserved6 */ +#define NTLMSSP_NEGOTIATE_WORKSTATION_SUPPLIED 0x2000 +#define NTLMSSP_NEGOTIATE_LOCAL_CALL 0x4000 /* client/server same machine */ +#define NTLMSSP_NEGOTIATE_ALWAYS_SIGN 0x8000 /* Sign. All security levels */ +#define NTLMSSP_TARGET_TYPE_DOMAIN 0x10000 +#define NTLMSSP_TARGET_TYPE_SERVER 0x20000 +#define NTLMSSP_TARGET_TYPE_SHARE 0x40000 +#define NTLMSSP_NEGOTIATE_EXTENDED_SEC 0x80000 /* NB:not related to NTLMv2 pwd*/ +/* #define NTLMSSP_REQUEST_INIT_RESP 0x100000 */ +#define NTLMSSP_NEGOTIATE_IDENTIFY 0x100000 +#define NTLMSSP_REQUEST_ACCEPT_RESP 0x200000 /* reserved5 */ +#define NTLMSSP_REQUEST_NON_NT_KEY 0x400000 +#define NTLMSSP_NEGOTIATE_TARGET_INFO 0x800000 +/* #define reserved4 0x1000000 */ +#define NTLMSSP_NEGOTIATE_VERSION 0x2000000 /* we do not set */ +/* #define reserved3 0x4000000 */ +/* #define reserved2 0x8000000 */ +/* #define reserved1 0x10000000 */ +#define NTLMSSP_NEGOTIATE_128 0x20000000 +#define NTLMSSP_NEGOTIATE_KEY_XCH 0x40000000 +#define NTLMSSP_NEGOTIATE_56 0x80000000 + +/* Define AV Pair Field IDs */ +enum av_field_type { + NTLMSSP_AV_EOL = 0, + NTLMSSP_AV_NB_COMPUTER_NAME, + NTLMSSP_AV_NB_DOMAIN_NAME, + NTLMSSP_AV_DNS_COMPUTER_NAME, + NTLMSSP_AV_DNS_DOMAIN_NAME, + NTLMSSP_AV_DNS_TREE_NAME, + NTLMSSP_AV_FLAGS, + NTLMSSP_AV_TIMESTAMP, + NTLMSSP_AV_RESTRICTION, + NTLMSSP_AV_TARGET_NAME, + NTLMSSP_AV_CHANNEL_BINDINGS +}; + +/* Although typedefs are not commonly used for structure definitions */ +/* in the Linux kernel, in this particular case they are useful */ +/* to more closely match the standards document for NTLMSSP from */ +/* OpenGroup and to make the code more closely match the standard in */ +/* appearance */ + +struct security_buffer { + __le16 Length; + __le16 MaximumLength; + __le32 BufferOffset; /* offset to buffer */ +} __packed; + +struct target_info { + __le16 Type; + __le16 Length; + __u8 Content[]; +} __packed; + +struct negotiate_message { + __u8 Signature[sizeof(NTLMSSP_SIGNATURE)]; + __le32 MessageType; /* NtLmNegotiate = 1 */ + __le32 NegotiateFlags; + struct security_buffer DomainName; /* RFC 1001 style and ASCII */ + struct security_buffer WorkstationName; /* RFC 1001 and ASCII */ + /* + * struct security_buffer for version info not present since we + * do not set the version is present flag + */ + char DomainString[]; + /* followed by WorkstationString */ +} __packed; + +struct challenge_message { + __u8 Signature[sizeof(NTLMSSP_SIGNATURE)]; + __le32 MessageType; /* NtLmChallenge = 2 */ + struct security_buffer TargetName; + __le32 NegotiateFlags; + __u8 Challenge[CIFS_CRYPTO_KEY_SIZE]; + __u8 Reserved[8]; + struct security_buffer TargetInfoArray; + /* + * struct security_buffer for version info not present since we + * do not set the version is present flag + */ +} __packed; + +struct authenticate_message { + __u8 Signature[sizeof(NTLMSSP_SIGNATURE)]; + __le32 MessageType; /* NtLmsAuthenticate = 3 */ + struct security_buffer LmChallengeResponse; + struct security_buffer NtChallengeResponse; + struct security_buffer DomainName; + struct security_buffer UserName; + struct security_buffer WorkstationName; + struct security_buffer SessionKey; + __le32 NegotiateFlags; + /* + * struct security_buffer for version info not present since we + * do not set the version is present flag + */ + char UserString[]; +} __packed; + +struct ntlmv2_resp { + char ntlmv2_hash[CIFS_ENCPWD_SIZE]; + __le32 blob_signature; + __u32 reserved; + __le64 time; + __u64 client_chal; /* random */ + __u32 reserved2; + /* array of name entries could follow ending in minimum 4 byte struct */ +} __packed; + +/* per smb session structure/fields */ +struct ntlmssp_auth { + /* whether session key is per smb session */ + bool sesskey_per_smbsess; + /* sent by client in type 1 ntlmsssp exchange */ + __u32 client_flags; + /* sent by server in type 2 ntlmssp exchange */ + __u32 conn_flags; + /* sent to server */ + unsigned char ciphertext[CIFS_CPHTXT_SIZE]; + /* used by ntlmssp */ + char cryptkey[CIFS_CRYPTO_KEY_SIZE]; +}; +#endif /* __KSMBD_NTLMSSP_H */ diff --git a/fs/smb/server/oplock.c b/fs/smb/server/oplock.c new file mode 100644 index 000000000..58af0fddf --- /dev/null +++ b/fs/smb/server/oplock.c @@ -0,0 +1,2503 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2016 Namjae Jeon + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#include +#include + +#include "glob.h" +#include "oplock.h" + +#include "smb_common.h" +#include "../common/smb2status.h" +#include "connection.h" +#include "mgmt/user_session.h" +#include "mgmt/share_config.h" +#include "mgmt/tree_connect.h" +#include "server.h" + +static LIST_HEAD(lease_table_list); +static DEFINE_RWLOCK(lease_list_lock); + +#define SMB2_LEASE_STATE_MASK_LE (SMB2_LEASE_READ_CACHING_LE | \ + SMB2_LEASE_HANDLE_CACHING_LE | \ + SMB2_LEASE_WRITE_CACHING_LE) + +static bool lease_state_valid(__le32 state) +{ + return !(state & ~SMB2_LEASE_STATE_MASK_LE); +} + +static __le32 lease_state_grantable(__le32 state) +{ + if (state == SMB2_LEASE_READ_CACHING_LE || + state == (SMB2_LEASE_READ_CACHING_LE | SMB2_LEASE_HANDLE_CACHING_LE) || + state == (SMB2_LEASE_READ_CACHING_LE | SMB2_LEASE_WRITE_CACHING_LE) || + state == SMB2_LEASE_STATE_MASK_LE) + return state; + + return 0; +} + +static bool lease_v2_flags_valid(__le32 flags) +{ + return !(flags & ~SMB2_LEASE_FLAG_PARENT_LEASE_KEY_SET_LE); +} + +static bool lease_has_parent_key(struct lease *lease) +{ + return lease->flags & SMB2_LEASE_FLAG_PARENT_LEASE_KEY_SET_LE; +} + +static bool lease_break_in_progress(struct lease *lease) +{ + struct oplock_info *opinfo; + bool ret = false; + + spin_lock(&lease->lock); + list_for_each_entry(opinfo, &lease->open_list, lease_entry) { + if (opinfo->op_state == OPLOCK_ACK_WAIT) { + ret = true; + break; + } + } + spin_unlock(&lease->lock); + + return ret; +} + +/** + * alloc_opinfo() - allocate a new opinfo object for oplock info + * @work: smb work + * @id: fid of open file + * @Tid: tree id of connection + * + * Return: allocated opinfo object on success, otherwise NULL + */ +static struct oplock_info *alloc_opinfo(struct ksmbd_work *work, + u64 id, __u16 Tid) +{ + struct ksmbd_session *sess = work->sess; + struct oplock_info *opinfo; + + opinfo = kzalloc_obj(struct oplock_info, KSMBD_DEFAULT_GFP); + if (!opinfo) + return NULL; + + opinfo->sess = sess; + opinfo->conn = ksmbd_conn_get(work->conn); + opinfo->level = SMB2_OPLOCK_LEVEL_NONE; + opinfo->op_state = OPLOCK_STATE_NONE; + spin_lock_init(&opinfo->state_lock); + opinfo->pending_break = 0; + opinfo->fid = id; + opinfo->Tid = Tid; + INIT_LIST_HEAD(&opinfo->op_entry); + INIT_LIST_HEAD(&opinfo->lease_entry); + init_waitqueue_head(&opinfo->oplock_q); + init_waitqueue_head(&opinfo->oplock_brk); + atomic_set(&opinfo->refcount, 1); + atomic_set(&opinfo->breaking_cnt, 0); + + return opinfo; +} + +static void lease_get(struct lease *lease) +{ + atomic_inc(&lease->refcount); +} + +static void lease_put(struct lease *lease) +{ + if (lease && atomic_dec_and_test(&lease->refcount)) + kfree(lease); +} + +static void lease_add_table(struct lease *lease, struct lease_table *lb) +{ + lease_get(lease); + lease->l_lb = lb; + spin_lock(&lb->lb_lock); + list_add_rcu(&lease->l_entry, &lb->lease_list); + spin_unlock(&lb->lb_lock); +} + +static void lease_del_table(struct lease *lease) +{ + struct lease_table *lb = lease->l_lb; + + if (!lb) + return; + + spin_lock(&lb->lb_lock); + if (list_empty(&lease->l_entry)) { + spin_unlock(&lb->lb_lock); + return; + } + + list_del_init(&lease->l_entry); + lease->l_lb = NULL; + spin_unlock(&lb->lb_lock); + + lease_put(lease); +} + +static struct lease_table *alloc_lease_table(struct oplock_info *opinfo) +{ + struct lease_table *lb; + + lb = kmalloc_obj(struct lease_table, KSMBD_DEFAULT_GFP); + if (!lb) + return NULL; + + memcpy(lb->client_guid, opinfo->conn->ClientGUID, + SMB2_CLIENT_GUID_SIZE); + lb->conn = ksmbd_conn_get(opinfo->conn); + INIT_LIST_HEAD(&lb->lease_list); + spin_lock_init(&lb->lb_lock); + return lb; +} + +static void free_lease_table(struct lease_table *lb) +{ + if (!lb) + return; + + ksmbd_conn_put(lb->conn); + kfree(lb); +} + +static struct lease *alloc_lease(struct lease_ctx_info *lctx, + struct ksmbd_inode *ci) +{ + struct lease *lease; + + lease = kmalloc_obj(struct lease, KSMBD_DEFAULT_GFP); + if (!lease) + return NULL; + + memcpy(lease->lease_key, lctx->lease_key, SMB2_LEASE_KEY_SIZE); + lease->state = lctx->req_state; + lease->new_state = 0; + lease->flags = lctx->flags; + lease->duration = lctx->duration; + lease->is_dir = lctx->is_dir; + memcpy(lease->parent_lease_key, lctx->parent_lease_key, SMB2_LEASE_KEY_SIZE); + lease->version = lctx->version; + lease->epoch = lctx->version == 2 ? le16_to_cpu(lctx->epoch) + 1 : 0; + lease->ci = ci; + lease->reuse_epoch = false; + lease->l_lb = NULL; + INIT_LIST_HEAD(&lease->l_entry); + INIT_LIST_HEAD(&lease->open_list); + spin_lock_init(&lease->lock); + atomic_set(&lease->refcount, 1); + + return lease; +} + +static void lease_add_open(struct lease *lease, struct oplock_info *opinfo) +{ + spin_lock(&lease->lock); + list_add(&opinfo->lease_entry, &lease->open_list); + spin_unlock(&lease->lock); +} + +static void lease_del_open(struct oplock_info *opinfo) +{ + struct lease *lease = opinfo->o_lease; + bool remove_table = false; + + if (!lease) + return; + + spin_lock(&lease->lock); + if (!list_empty(&opinfo->lease_entry)) { + list_del_init(&opinfo->lease_entry); + remove_table = list_empty(&lease->open_list); + } + spin_unlock(&lease->lock); + + if (remove_table) { + write_lock(&lease_list_lock); + lease_del_table(lease); + write_unlock(&lease_list_lock); + } +} + +static void free_lease(struct oplock_info *opinfo) +{ + lease_put(opinfo->o_lease); +} + +static void __free_opinfo(struct oplock_info *opinfo) +{ + if (opinfo->is_lease) + free_lease(opinfo); + ksmbd_conn_put(opinfo->conn); + kfree(opinfo); +} + +static void free_opinfo_rcu(struct rcu_head *rcu) +{ + struct oplock_info *opinfo = container_of(rcu, struct oplock_info, rcu); + + __free_opinfo(opinfo); +} + +static void free_opinfo(struct oplock_info *opinfo) +{ + call_rcu(&opinfo->rcu, free_opinfo_rcu); +} + +void lease_update_oplock_levels(struct lease *lease) +{ + struct oplock_info *opinfo; + __u8 level; + + if (!lease) + return; + + level = smb2_map_lease_to_oplock(lease->state); + spin_lock(&lease->lock); + list_for_each_entry(opinfo, &lease->open_list, lease_entry) + opinfo->level = level; + spin_unlock(&lease->lock); +} + +struct oplock_info *opinfo_get(struct ksmbd_file *fp) +{ + struct oplock_info *opinfo; + + rcu_read_lock(); + opinfo = rcu_dereference(fp->f_opinfo); + if (opinfo && !atomic_inc_not_zero(&opinfo->refcount)) + opinfo = NULL; + rcu_read_unlock(); + + return opinfo; +} + +struct oplock_snapshot { + bool durable_open; + bool durable_detached; + unsigned long long fid; +}; + +static struct oplock_info *opinfo_get_list(struct ksmbd_inode *ci, + struct ksmbd_file *skip_fp, + struct oplock_snapshot *snapshot) +{ + struct oplock_info *opinfo; + + if (snapshot) { + snapshot->durable_open = false; + snapshot->durable_detached = false; + snapshot->fid = KSMBD_NO_FID; + } + + down_read(&ci->m_lock); + opinfo = list_first_entry_or_null(&ci->m_op_list, struct oplock_info, + op_entry); + if (opinfo) { + if (opinfo->conn == NULL || + !atomic_inc_not_zero(&opinfo->refcount)) + opinfo = NULL; + else { + if (ksmbd_conn_releasing(opinfo->conn)) { + atomic_dec(&opinfo->refcount); + opinfo = NULL; + } + } + + if (opinfo && snapshot && opinfo->o_fp && + opinfo->o_fp != skip_fp && + READ_ONCE(opinfo->o_fp->is_durable)) { + snapshot->durable_open = true; + snapshot->durable_detached = + !READ_ONCE(opinfo->o_fp->conn) || + !READ_ONCE(opinfo->o_fp->tcon); + snapshot->fid = opinfo->fid; + } + } + up_read(&ci->m_lock); + + return opinfo; +} + +void opinfo_put(struct oplock_info *opinfo) +{ + if (!opinfo) + return; + + if (!atomic_dec_and_test(&opinfo->refcount)) + return; + + free_opinfo(opinfo); +} + +static bool ksmbd_inode_has_lease(struct ksmbd_inode *ci) +{ + struct oplock_info *opinfo = opinfo_get_list(ci, NULL, NULL); + bool is_lease; + + if (!opinfo) + return false; + is_lease = opinfo->is_lease; + opinfo_put(opinfo); + return is_lease; +} + +static void opinfo_add(struct oplock_info *opinfo, struct ksmbd_file *fp) +{ + struct ksmbd_inode *ci = fp->f_ci; + + down_write(&ci->m_lock); + list_add(&opinfo->op_entry, &ci->m_op_list); + up_write(&ci->m_lock); +} + +static void opinfo_del(struct oplock_info *opinfo) +{ + struct ksmbd_inode *ci = opinfo->o_fp->f_ci; + + if (opinfo->is_lease) + lease_del_open(opinfo); + + down_write(&ci->m_lock); + list_del(&opinfo->op_entry); + up_write(&ci->m_lock); +} + +static unsigned long opinfo_count(struct ksmbd_file *fp) +{ + if (ksmbd_stream_fd(fp)) + return atomic_read(&fp->f_ci->sop_count); + else + return atomic_read(&fp->f_ci->op_count); +} + +static void opinfo_count_inc(struct ksmbd_file *fp) +{ + if (ksmbd_stream_fd(fp)) + return atomic_inc(&fp->f_ci->sop_count); + else + return atomic_inc(&fp->f_ci->op_count); +} + +static void opinfo_count_dec(struct ksmbd_file *fp) +{ + if (ksmbd_stream_fd(fp)) + return atomic_dec(&fp->f_ci->sop_count); + else + return atomic_dec(&fp->f_ci->op_count); +} + +/** + * opinfo_write_to_read() - convert a write oplock to read oplock + * @opinfo: current oplock info + * + * Return: 0 on success, otherwise -EINVAL + */ +int opinfo_write_to_read(struct oplock_info *opinfo) +{ + struct lease *lease = opinfo->o_lease; + + if (!(opinfo->level == SMB2_OPLOCK_LEVEL_BATCH || + opinfo->level == SMB2_OPLOCK_LEVEL_EXCLUSIVE)) { + pr_err("bad oplock(0x%x)\n", opinfo->level); + if (opinfo->is_lease) + pr_err("lease state(0x%x)\n", lease->state); + return -EINVAL; + } + opinfo->level = SMB2_OPLOCK_LEVEL_II; + + if (opinfo->is_lease) { + lease->state = lease->new_state; + lease_update_oplock_levels(lease); + } + return 0; +} + +/** + * opinfo_read_handle_to_read() - convert a read/handle oplock to read oplock + * @opinfo: current oplock info + * + * Return: 0 on success, otherwise -EINVAL + */ +int opinfo_read_handle_to_read(struct oplock_info *opinfo) +{ + struct lease *lease = opinfo->o_lease; + + lease->state = lease->new_state; + lease_update_oplock_levels(lease); + return 0; +} + +/** + * opinfo_write_to_none() - convert a write oplock to none + * @opinfo: current oplock info + * + * Return: 0 on success, otherwise -EINVAL + */ +int opinfo_write_to_none(struct oplock_info *opinfo) +{ + struct lease *lease = opinfo->o_lease; + + if (!(opinfo->level == SMB2_OPLOCK_LEVEL_BATCH || + opinfo->level == SMB2_OPLOCK_LEVEL_EXCLUSIVE)) { + pr_err("bad oplock(0x%x)\n", opinfo->level); + if (opinfo->is_lease) + pr_err("lease state(0x%x)\n", lease->state); + return -EINVAL; + } + opinfo->level = SMB2_OPLOCK_LEVEL_NONE; + if (opinfo->is_lease) { + lease->state = lease->new_state; + lease_update_oplock_levels(lease); + } + return 0; +} + +/** + * opinfo_read_to_none() - convert a write read to none + * @opinfo: current oplock info + * + * Return: 0 on success, otherwise -EINVAL + */ +int opinfo_read_to_none(struct oplock_info *opinfo) +{ + struct lease *lease = opinfo->o_lease; + + if (opinfo->level != SMB2_OPLOCK_LEVEL_II) { + pr_err("bad oplock(0x%x)\n", opinfo->level); + if (opinfo->is_lease) + pr_err("lease state(0x%x)\n", lease->state); + return -EINVAL; + } + opinfo->level = SMB2_OPLOCK_LEVEL_NONE; + if (opinfo->is_lease) { + lease->state = lease->new_state; + lease_update_oplock_levels(lease); + } + return 0; +} + +/** + * lease_read_to_write() - upgrade lease state from read to write + * @opinfo: current lease info + * + * Return: 0 on success, otherwise -EINVAL + */ +int lease_read_to_write(struct oplock_info *opinfo) +{ + struct lease *lease = opinfo->o_lease; + + if (!(lease->state & SMB2_LEASE_READ_CACHING_LE)) { + ksmbd_debug(OPLOCK, "bad lease state(0x%x)\n", lease->state); + return -EINVAL; + } + + lease->new_state = SMB2_LEASE_NONE_LE; + lease->state |= SMB2_LEASE_WRITE_CACHING_LE; + lease_update_oplock_levels(lease); + return 0; +} + +/** + * lease_none_upgrade() - upgrade lease state from none + * @opinfo: current lease info + * @new_state: new lease state + * + * Return: 0 on success, otherwise -EINVAL + */ +static int lease_none_upgrade(struct oplock_info *opinfo, __le32 new_state) +{ + struct lease *lease = opinfo->o_lease; + + if (!(lease->state == SMB2_LEASE_NONE_LE)) { + ksmbd_debug(OPLOCK, "bad lease state(0x%x)\n", lease->state); + return -EINVAL; + } + + lease->new_state = SMB2_LEASE_NONE_LE; + lease->state = new_state; + lease_update_oplock_levels(lease); + + return 0; +} + +/** + * close_id_del_oplock() - release oplock object at file close time + * @fp: ksmbd file pointer + */ +void close_id_del_oplock(struct ksmbd_file *fp) +{ + struct oplock_info *opinfo; + + if (fp->reserve_lease_break) + smb_lazy_parent_lease_break_close(fp); + + opinfo = opinfo_get(fp); + if (!opinfo) + return; + + opinfo_del(opinfo); + + rcu_assign_pointer(fp->f_opinfo, NULL); + spin_lock(&opinfo->state_lock); + if (opinfo->op_state == OPLOCK_ACK_WAIT && opinfo->is_lease) + atomic_set(&opinfo->breaking_cnt, 0); + /* + * An opinfo that has been removed from the inode list is terminal. Keep + * this transition and releasing pending_break under state_lock. a breaker + * takes the same lock before it acquires pending_break or sets ACK_WAIT. + */ + opinfo->op_state = OPLOCK_CLOSING; + clear_bit_unlock(0, &opinfo->pending_break); + spin_unlock(&opinfo->state_lock); + wake_up_interruptible_all(&opinfo->oplock_q); + if (opinfo->is_lease) + wake_up_interruptible_all(&opinfo->oplock_brk); + /* memory barrier is needed for wake_up_bit() */ + smp_mb__after_atomic(); + wake_up_bit(&opinfo->pending_break, 0); + + opinfo_count_dec(fp); + atomic_dec(&opinfo->refcount); + opinfo_put(opinfo); +} + +/** + * grant_write_oplock() - grant exclusive/batch oplock or write lease + * @opinfo_new: new oplock info object + * @req_oplock: request oplock + * @lctx: lease context information + * + * Return: 0 + */ +static void grant_write_oplock(struct oplock_info *opinfo_new, int req_oplock, + struct lease_ctx_info *lctx) +{ + struct lease *lease = opinfo_new->o_lease; + + if (req_oplock == SMB2_OPLOCK_LEVEL_BATCH) + opinfo_new->level = SMB2_OPLOCK_LEVEL_BATCH; + else + opinfo_new->level = SMB2_OPLOCK_LEVEL_EXCLUSIVE; + + if (lctx) { + lease->state = lctx->req_state; + memcpy(lease->lease_key, lctx->lease_key, SMB2_LEASE_KEY_SIZE); + } +} + +/** + * grant_read_oplock() - grant level2 oplock or read lease + * @opinfo_new: new oplock info object + * @lctx: lease context information + * + * Return: 0 + */ +static void grant_read_oplock(struct oplock_info *opinfo_new, + struct lease_ctx_info *lctx) +{ + struct lease *lease = opinfo_new->o_lease; + + opinfo_new->level = SMB2_OPLOCK_LEVEL_II; + + if (lctx) { + lease->state = SMB2_LEASE_READ_CACHING_LE; + if (lctx->req_state & SMB2_LEASE_HANDLE_CACHING_LE) + lease->state |= SMB2_LEASE_HANDLE_CACHING_LE; + memcpy(lease->lease_key, lctx->lease_key, SMB2_LEASE_KEY_SIZE); + } +} + +/** + * grant_none_oplock() - grant none oplock or none lease + * @opinfo_new: new oplock info object + * @lctx: lease context information + * + * Return: 0 + */ +static void grant_none_oplock(struct oplock_info *opinfo_new, + struct lease_ctx_info *lctx) +{ + struct lease *lease = opinfo_new->o_lease; + + opinfo_new->level = SMB2_OPLOCK_LEVEL_NONE; + + if (lctx) { + lease->state = 0; + memcpy(lease->lease_key, lctx->lease_key, SMB2_LEASE_KEY_SIZE); + } +} + +static inline int compare_guid_key(struct oplock_info *opinfo, + const char *guid1, const char *key1) +{ + const char *guid2, *key2; + struct ksmbd_conn *conn; + + conn = READ_ONCE(opinfo->conn); + if (!conn) + return 0; + guid2 = conn->ClientGUID; + key2 = opinfo->o_lease->lease_key; + if (!memcmp(guid1, guid2, SMB2_CLIENT_GUID_SIZE) && + !memcmp(key1, key2, SMB2_LEASE_KEY_SIZE)) + return 1; + + return 0; +} + +/** + * same_client_has_lease() - check whether current lease request is + * from lease owner of file + * @ci: master file pointer + * @client_guid: Client GUID + * @lctx: lease context information + * + * Return: oplock(lease) object on success, otherwise NULL + */ +static struct oplock_info *same_client_has_lease(struct ksmbd_inode *ci, + const char *client_guid, + struct lease_ctx_info *lctx) +{ + int ret; + struct lease *lease; + struct oplock_info *opinfo; + struct oplock_info *m_opinfo = NULL; + + if (!lctx) + return NULL; + + /* + * Compare lease key and client_guid to know request from same owner + * of same client + */ + down_read(&ci->m_lock); + list_for_each_entry(opinfo, &ci->m_op_list, op_entry) { + if (!opinfo->is_lease || !opinfo->conn) + continue; + lease = opinfo->o_lease; + + ret = compare_guid_key(opinfo, client_guid, lctx->lease_key); + if (ret) { + if (!atomic_inc_not_zero(&opinfo->refcount)) + continue; + if (m_opinfo) + opinfo_put(m_opinfo); + m_opinfo = opinfo; + + /* skip upgrading lease about breaking lease */ + if (atomic_read(&opinfo->breaking_cnt)) + continue; + + /* upgrading lease */ + if ((atomic_read(&ci->op_count) + + atomic_read(&ci->sop_count)) == 1) { + if (lease->state != SMB2_LEASE_NONE_LE && + lease->state == (lctx->req_state & lease->state)) { + lease->epoch++; + lease->state |= lctx->req_state; + if (lctx->req_state & + SMB2_LEASE_WRITE_CACHING_LE) + lease_read_to_write(opinfo); + + } + } else if ((atomic_read(&ci->op_count) + + atomic_read(&ci->sop_count)) > 1) { + if (lctx->req_state == + (SMB2_LEASE_READ_CACHING_LE | + SMB2_LEASE_HANDLE_CACHING_LE)) { + if (lease->state != lctx->req_state) { + lease->epoch++; + lease->state = lctx->req_state; + lease_update_oplock_levels(lease); + } + } + } + + if (lctx->req_state && lease->state == + SMB2_LEASE_NONE_LE) { + lease->epoch++; + lease_none_upgrade(opinfo, lctx->req_state); + } + } + } + up_read(&ci->m_lock); + + return m_opinfo; +} + +static bool wait_for_break_ack(struct oplock_info *opinfo) +{ + int rc = 0; + + rc = wait_event_interruptible_timeout(opinfo->oplock_q, + opinfo->op_state == OPLOCK_STATE_NONE || + opinfo->op_state == OPLOCK_CLOSING, + OPLOCK_WAIT_TIME); + + /* is this a timeout ? */ + if (!rc) { + spin_lock(&opinfo->state_lock); + if (opinfo->op_state == OPLOCK_CLOSING) { + spin_unlock(&opinfo->state_lock); + return false; + } + if (opinfo->is_lease) { + opinfo->o_lease->state = SMB2_LEASE_NONE_LE; + lease_update_oplock_levels(opinfo->o_lease); + } + opinfo->level = SMB2_OPLOCK_LEVEL_NONE; + opinfo->op_state = OPLOCK_STATE_NONE; + spin_unlock(&opinfo->state_lock); + return true; + } + + return false; +} + +static void wake_up_oplock_break(struct oplock_info *opinfo) +{ + clear_bit_unlock(0, &opinfo->pending_break); + /* memory barrier is needed for wake_up_bit() */ + smp_mb__after_atomic(); + wake_up_bit(&opinfo->pending_break, 0); +} + +static bool oplock_break_set_ack_wait(struct oplock_info *opinfo) +{ + bool ret = false; + + spin_lock(&opinfo->state_lock); + if (opinfo->op_state != OPLOCK_CLOSING) { + opinfo->op_state = OPLOCK_ACK_WAIT; + ret = true; + } + spin_unlock(&opinfo->state_lock); + + return ret; +} + +static int oplock_break_pending(struct oplock_info *opinfo, int req_op_level) +{ + for (;;) { + bool closing; + + spin_lock(&opinfo->state_lock); + closing = opinfo->op_state == OPLOCK_CLOSING; + if (!closing && !test_and_set_bit(0, &opinfo->pending_break)) { + spin_unlock(&opinfo->state_lock); + break; + } + spin_unlock(&opinfo->state_lock); + if (closing) + return -ENOENT; + + if (opinfo->is_lease) + opinfo->o_lease->reuse_epoch = true; + + wait_on_bit(&opinfo->pending_break, 0, TASK_UNINTERRUPTIBLE); + + /* Not immediately break to none. */ + opinfo->open_trunc = 0; + + spin_lock(&opinfo->state_lock); + closing = opinfo->op_state == OPLOCK_CLOSING; + spin_unlock(&opinfo->state_lock); + if (closing) + return -ENOENT; + if (opinfo->level <= req_op_level) { + if (opinfo->is_lease == false) + return 1; + + if (opinfo->o_lease->state != + (SMB2_LEASE_HANDLE_CACHING_LE | + SMB2_LEASE_READ_CACHING_LE)) + return 1; + } + } + + if (opinfo->level <= req_op_level) { + if (opinfo->is_lease == false) { + wake_up_oplock_break(opinfo); + return 1; + } + if (opinfo->o_lease->state != + (SMB2_LEASE_HANDLE_CACHING_LE | + SMB2_LEASE_READ_CACHING_LE)) { + wake_up_oplock_break(opinfo); + return 1; + } + } + return 0; +} + +static bool lease_break_needed(struct oplock_info *opinfo, int req_op_level, + bool open_trunc) +{ + struct lease *lease = opinfo->o_lease; + + if (open_trunc) + return lease->state != SMB2_LEASE_NONE_LE; + + return opinfo->level > req_op_level; +} + +/** + * __smb2_oplock_break_noti() - send smb2 oplock break cmd from conn + * to client + * @wk: smb work object + * + * There are two ways this function can be called. 1- while file open we break + * from exclusive/batch lock to levelII oplock and 2- while file write/truncate + * we break from levelII oplock no oplock. + * work->request_buf contains oplock_info. + */ +static void __smb2_oplock_break_noti(struct work_struct *wk) +{ + struct smb2_oplock_break *rsp = NULL; + struct ksmbd_work *work = container_of(wk, struct ksmbd_work, work); + struct ksmbd_conn *conn = work->conn; + struct oplock_break_info *br_info = work->request_buf; + struct smb2_hdr *rsp_hdr; + struct ksmbd_file *fp; + + fp = ksmbd_lookup_global_fd(br_info->fid); + if (!fp) + goto out; + + if (allocate_interim_rsp_buf(work)) { + pr_err("smb2_allocate_rsp_buf failed! "); + ksmbd_fd_put(work, fp); + goto out; + } + + rsp_hdr = smb_get_msg(work->response_buf); + memset(rsp_hdr, 0, sizeof(struct smb2_hdr) + 2); + rsp_hdr->ProtocolId = SMB2_PROTO_NUMBER; + rsp_hdr->StructureSize = SMB2_HEADER_STRUCTURE_SIZE; + rsp_hdr->CreditRequest = cpu_to_le16(0); + rsp_hdr->Command = SMB2_OPLOCK_BREAK; + rsp_hdr->Flags = (SMB2_FLAGS_SERVER_TO_REDIR); + rsp_hdr->NextCommand = 0; + rsp_hdr->MessageId = cpu_to_le64(-1); + rsp_hdr->Id.SyncId.ProcessId = 0; + rsp_hdr->Id.SyncId.TreeId = 0; + rsp_hdr->SessionId = 0; + memset(rsp_hdr->Signature, 0, 16); + + rsp = smb_get_msg(work->response_buf); + + rsp->StructureSize = cpu_to_le16(24); + if (!br_info->open_trunc && + (br_info->level == SMB2_OPLOCK_LEVEL_BATCH || + br_info->level == SMB2_OPLOCK_LEVEL_EXCLUSIVE)) + rsp->OplockLevel = SMB2_OPLOCK_LEVEL_II; + else + rsp->OplockLevel = SMB2_OPLOCK_LEVEL_NONE; + rsp->Reserved = 0; + rsp->Reserved2 = 0; + rsp->PersistentFid = fp->persistent_id; + rsp->VolatileFid = fp->volatile_id; + + ksmbd_fd_put(work, fp); + if (ksmbd_iov_pin_rsp(work, (void *)rsp, + sizeof(struct smb2_oplock_break))) + goto out; + + ksmbd_debug(OPLOCK, + "sending oplock break v_id %llu p_id = %llu lock level = %d\n", + rsp->VolatileFid, rsp->PersistentFid, rsp->OplockLevel); + + ksmbd_conn_write(work); + +out: + ksmbd_free_work_struct(work); + ksmbd_conn_r_count_dec(conn); + ksmbd_conn_put(conn); +} + +/** + * smb2_oplock_break_noti() - send smb2 exclusive/batch to level2 oplock + * break command from server to client + * @opinfo: oplock info object + * + * Return: 0 on success, otherwise error + */ +static int smb2_oplock_break_noti(struct oplock_info *opinfo) +{ + struct ksmbd_conn *conn; + struct oplock_break_info *br_info; + int ret = 0; + struct ksmbd_work *work; + + conn = READ_ONCE(opinfo->conn); + if (!conn) + return ksmbd_invalidate_durable_fd(opinfo->fid); + + work = ksmbd_alloc_work_struct(); + if (!work) + return -ENOMEM; + + br_info = kmalloc_obj(struct oplock_break_info, KSMBD_DEFAULT_GFP); + if (!br_info) { + ksmbd_free_work_struct(work); + return -ENOMEM; + } + + br_info->level = opinfo->level; + br_info->fid = opinfo->fid; + br_info->open_trunc = opinfo->open_trunc; + + work->request_buf = (char *)br_info; + work->conn = ksmbd_conn_get(conn); + work->sess = opinfo->sess; + + ksmbd_conn_r_count_inc(conn); + if (opinfo->op_state == OPLOCK_ACK_WAIT) { + INIT_WORK(&work->work, __smb2_oplock_break_noti); + ksmbd_queue_work(work); + + if (wait_for_break_ack(opinfo)) + ret = ksmbd_invalidate_durable_fd(opinfo->fid); + } else { + __smb2_oplock_break_noti(&work->work); + if (opinfo->level == SMB2_OPLOCK_LEVEL_II) + opinfo->level = SMB2_OPLOCK_LEVEL_NONE; + } + return ret; +} + +/** + * __smb2_lease_break_noti() - send lease break command from server + * to client + * @wk: smb work object + */ +static void __smb2_lease_break_noti(struct work_struct *wk) +{ + struct smb2_lease_break *rsp = NULL; + struct ksmbd_work *work = container_of(wk, struct ksmbd_work, work); + struct ksmbd_conn *conn = work->conn; + struct lease_break_info *br_info = work->request_buf; + struct smb2_hdr *rsp_hdr; + + if (allocate_interim_rsp_buf(work)) { + ksmbd_debug(OPLOCK, "smb2_allocate_rsp_buf failed! "); + goto out; + } + + rsp_hdr = smb_get_msg(work->response_buf); + memset(rsp_hdr, 0, sizeof(struct smb2_hdr) + 2); + rsp_hdr->ProtocolId = SMB2_PROTO_NUMBER; + rsp_hdr->StructureSize = SMB2_HEADER_STRUCTURE_SIZE; + rsp_hdr->CreditRequest = cpu_to_le16(0); + rsp_hdr->Command = SMB2_OPLOCK_BREAK; + rsp_hdr->Flags = (SMB2_FLAGS_SERVER_TO_REDIR); + rsp_hdr->NextCommand = 0; + rsp_hdr->MessageId = cpu_to_le64(-1); + rsp_hdr->Id.SyncId.ProcessId = 0; + rsp_hdr->Id.SyncId.TreeId = 0; + rsp_hdr->SessionId = 0; + memset(rsp_hdr->Signature, 0, 16); + + rsp = smb_get_msg(work->response_buf); + rsp->StructureSize = cpu_to_le16(44); + rsp->Epoch = br_info->epoch; + rsp->Flags = 0; + + if (br_info->curr_state & (SMB2_LEASE_WRITE_CACHING_LE | + SMB2_LEASE_HANDLE_CACHING_LE)) + rsp->Flags = SMB2_NOTIFY_BREAK_LEASE_FLAG_ACK_REQUIRED; + + memcpy(rsp->LeaseKey, br_info->lease_key, SMB2_LEASE_KEY_SIZE); + rsp->CurrentLeaseState = br_info->curr_state; + rsp->NewLeaseState = br_info->new_state; + rsp->BreakReason = 0; + rsp->AccessMaskHint = 0; + rsp->ShareMaskHint = 0; + + if (ksmbd_iov_pin_rsp(work, (void *)rsp, + sizeof(struct smb2_lease_break))) + goto out; + + ksmbd_conn_write(work); + +out: + ksmbd_free_work_struct(work); + ksmbd_conn_r_count_dec(conn); + ksmbd_conn_put(conn); +} + +/* + * Select and pin the connection used for a lease break before doing any + * allocations which may sleep. opinfo->conn is cleared under ci->m_lock, + * while lease->l_lb and the lease table lifetime are protected by + * lease_list_lock. + */ +static struct ksmbd_conn *smb2_lease_break_conn_get(struct oplock_info *opinfo) +{ + struct lease *lease = opinfo->o_lease; + struct lease_table *lb; + struct ksmbd_conn *conn; + + /* Keep the connection which owns the open, when it is still active. */ + down_read(&lease->ci->m_lock); + conn = READ_ONCE(opinfo->conn); + if (conn && !ksmbd_conn_releasing(conn)) + conn = ksmbd_conn_get(conn); + else + conn = NULL; + up_read(&lease->ci->m_lock); + + if (conn || lease->version != 2) + return conn; + + /* Otherwise route v2 lease breaks through the shared lease channel. */ + read_lock(&lease_list_lock); + lb = lease->l_lb; + if (lb && lb->conn && !ksmbd_conn_releasing(lb->conn)) + conn = ksmbd_conn_get(lb->conn); + read_unlock(&lease_list_lock); + + return conn; +} + +/** + * smb2_lease_break_noti() - break lease when a new client request + * write lease + * @opinfo: contains lease state information + * @sync: send the lease break notification synchronously + * @inc_epoch: increment the lease epoch before sending the break + * + * Return: 0 on success, otherwise error + */ +static int smb2_lease_break_noti(struct oplock_info *opinfo, bool sync, + bool inc_epoch) +{ + struct ksmbd_conn *conn; + struct ksmbd_work *work; + struct lease_break_info *br_info; + struct lease *lease = opinfo->o_lease; + + conn = smb2_lease_break_conn_get(opinfo); + if (!conn) + return ksmbd_invalidate_durable_fd(opinfo->fid); + + work = ksmbd_alloc_work_struct(); + if (!work) { + ksmbd_conn_put(conn); + return -ENOMEM; + } + + br_info = kmalloc_obj(struct lease_break_info, KSMBD_DEFAULT_GFP); + if (!br_info) { + ksmbd_free_work_struct(work); + ksmbd_conn_put(conn); + return -ENOMEM; + } + + br_info->curr_state = lease->state; + br_info->new_state = lease->new_state; + if (lease->version == 2) { + if (inc_epoch) + lease->epoch++; + br_info->epoch = cpu_to_le16(lease->epoch); + } else { + br_info->epoch = 0; + } + memcpy(br_info->lease_key, lease->lease_key, SMB2_LEASE_KEY_SIZE); + + work->request_buf = (char *)br_info; + /* Transfer the reference acquired by smb2_lease_break_conn_get(). */ + work->conn = conn; + work->sess = opinfo->sess; + + ksmbd_conn_r_count_inc(conn); + if (opinfo->op_state == OPLOCK_ACK_WAIT) { + if (sync) { + __smb2_lease_break_noti(&work->work); + } else { + INIT_WORK(&work->work, __smb2_lease_break_noti); + ksmbd_queue_work(work); + } + } else { + __smb2_lease_break_noti(&work->work); + if (opinfo->o_lease->new_state == SMB2_LEASE_NONE_LE) { + opinfo->o_lease->state = SMB2_LEASE_NONE_LE; + lease_update_oplock_levels(opinfo->o_lease); + } + } + return 0; +} + +static void wait_lease_breaking(struct oplock_info *opinfo) +{ + if (!opinfo->is_lease) + return; + + wake_up_interruptible_all(&opinfo->oplock_brk); + if (atomic_read(&opinfo->breaking_cnt)) { + int ret = 0; + + ret = wait_event_interruptible_timeout(opinfo->oplock_brk, + atomic_read(&opinfo->breaking_cnt) == 0, + HZ); + if (!ret) + atomic_set(&opinfo->breaking_cnt, 0); + } +} + +static int oplock_break(struct oplock_info *brk_opinfo, int req_op_level, + struct ksmbd_work *in_work, bool share_break, + bool sync_lease_break) +{ + int err = 0; + bool sent_interim = false; + + /* Need to break exclusive/batch oplock, write lease or overwrite_if */ + ksmbd_debug(OPLOCK, + "request to send oplock(level : 0x%x) break notification\n", + brk_opinfo->level); + + if (brk_opinfo->is_lease) { + struct lease *lease = brk_opinfo->o_lease; + bool open_trunc = brk_opinfo->open_trunc; + bool was_pending = test_bit(0, &brk_opinfo->pending_break); + bool wait_ack; + bool inc_epoch = true; + + if (in_work && was_pending) { + setup_async_work(in_work, NULL, NULL); + smb2_send_interim_resp(in_work, STATUS_PENDING); + release_async_work(in_work); + sent_interim = true; + } + + err = oplock_break_pending(brk_opinfo, req_op_level); + if (err) + return err < 0 ? err : 0; + if (was_pending) + open_trunc = brk_opinfo->open_trunc; + +again: + atomic_inc(&brk_opinfo->breaking_cnt); + if (open_trunc) { + /* + * Create overwrite break trigger the lease break to + * none. + */ + lease->new_state = SMB2_LEASE_NONE_LE; + } else if (share_break && + lease->state & SMB2_LEASE_HANDLE_CACHING_LE) { + lease->new_state = + lease->state & ~SMB2_LEASE_HANDLE_CACHING_LE; + } else { + if (lease->state & SMB2_LEASE_WRITE_CACHING_LE) { + if (lease->state & SMB2_LEASE_HANDLE_CACHING_LE) + lease->new_state = + SMB2_LEASE_READ_CACHING_LE | + SMB2_LEASE_HANDLE_CACHING_LE; + else + lease->new_state = + SMB2_LEASE_READ_CACHING_LE; + } else { + if (lease->state & SMB2_LEASE_HANDLE_CACHING_LE && + !lease->is_dir) + lease->new_state = + SMB2_LEASE_READ_CACHING_LE; + else + lease->new_state = SMB2_LEASE_NONE_LE; + } + } + + if (lease->state & (SMB2_LEASE_WRITE_CACHING_LE | + SMB2_LEASE_HANDLE_CACHING_LE)) { + if (!oplock_break_set_ack_wait(brk_opinfo)) { + atomic_dec_if_positive(&brk_opinfo->breaking_cnt); + wake_up_oplock_break(brk_opinfo); + return -ENOENT; + } + } else + atomic_dec(&brk_opinfo->breaking_cnt); + + wait_ack = !(open_trunc && + lease->state == (SMB2_LEASE_READ_CACHING_LE | + SMB2_LEASE_HANDLE_CACHING_LE)); + if (lease->reuse_epoch) { + inc_epoch = false; + lease->reuse_epoch = false; + } + err = smb2_lease_break_noti(brk_opinfo, sync_lease_break, inc_epoch); + inc_epoch = false; + if (in_work && !sent_interim) { + setup_async_work(in_work, NULL, NULL); + smb2_send_interim_resp(in_work, STATUS_PENDING); + release_async_work(in_work); + sent_interim = true; + } + if (wait_ack && !err && wait_for_break_ack(brk_opinfo)) + err = ksmbd_invalidate_durable_fd(brk_opinfo->fid); + + ksmbd_debug(OPLOCK, "oplock granted = %d\n", brk_opinfo->level); + if (brk_opinfo->op_state == OPLOCK_CLOSING) + err = -ENOENT; + + if (wait_ack) + wait_lease_breaking(brk_opinfo); + /* + * A share-mode conflict break only drops the conflicting + * caching bit; the triggering open fails with a sharing + * violation, so keep it to a single break. + * + * Otherwise chain another break while the lease is still + * incompatible with this open (req_op_level), or while a + * truncating waiter that arrived during the break still needs + * the lease dropped to none. open_trunc snapshotted for this + * break stays cleared, so the next state is computed from the + * lease state and the cascade steps down (e.g. RH->R->none) + * instead of collapsing straight to none. + */ + if (wait_ack && !err && !share_break && + (lease_break_needed(brk_opinfo, req_op_level, open_trunc) || + (brk_opinfo->open_trunc && + lease->state != SMB2_LEASE_NONE_LE))) + goto again; + + wake_up_oplock_break(brk_opinfo); + return err; + } else { + err = oplock_break_pending(brk_opinfo, req_op_level); + if (err) + return err < 0 ? err : 0; + + if (brk_opinfo->level == SMB2_OPLOCK_LEVEL_BATCH || + brk_opinfo->level == SMB2_OPLOCK_LEVEL_EXCLUSIVE) { + if (!oplock_break_set_ack_wait(brk_opinfo)) { + wake_up_oplock_break(brk_opinfo); + return -ENOENT; + } + } + + /* + * Keep a conflicting CREATE asynchronous while waiting for an + * oplock-break acknowledgement. Besides avoiding a blocked client + * request, this lets a replay arrive while the original CREATE is + * still pending and be rejected with FILE_NOT_AVAILABLE. + */ + if (in_work) { + setup_async_work(in_work, NULL, NULL); + smb2_send_interim_resp(in_work, STATUS_PENDING); + release_async_work(in_work); + } + } + + err = smb2_oplock_break_noti(brk_opinfo); + + ksmbd_debug(OPLOCK, "oplock granted = %d\n", brk_opinfo->level); + if (brk_opinfo->op_state == OPLOCK_CLOSING) + err = -EAGAIN; + wake_up_oplock_break(brk_opinfo); + + return err; +} + +struct oplock_break_entry { + struct list_head list; + struct oplock_info *opinfo; +}; + +static int oplock_break_add(struct list_head *head, struct oplock_info *opinfo) +{ + struct oplock_break_entry *ent; + + ent = kmalloc_obj(struct oplock_break_entry, KSMBD_DEFAULT_GFP); + if (!ent) + return -ENOMEM; + + ent->opinfo = opinfo; + list_add_tail(&ent->list, head); + return 0; +} + +static void oplock_break_drain_none(struct list_head *head) +{ + struct oplock_break_entry *ent, *tmp; + + list_for_each_entry_safe(ent, tmp, head, list) { + oplock_break(ent->opinfo, SMB2_OPLOCK_LEVEL_NONE, NULL, false, + false); + list_del(&ent->list); + opinfo_put(ent->opinfo); + kfree(ent); + } +} + +void destroy_lease_table(struct ksmbd_conn *conn) +{ + struct lease_table *lb, *lbtmp; + struct lease *lease, *ltmp; + + write_lock(&lease_list_lock); + if (list_empty(&lease_table_list)) { + write_unlock(&lease_list_lock); + return; + } + + list_for_each_entry_safe(lb, lbtmp, &lease_table_list, l_entry) { + if (conn && memcmp(lb->client_guid, conn->ClientGUID, + SMB2_CLIENT_GUID_SIZE)) + continue; + list_for_each_entry_safe(lease, ltmp, &lb->lease_list, l_entry) + lease_del_table(lease); + list_del(&lb->l_entry); + free_lease_table(lb); + } + write_unlock(&lease_list_lock); +} + +int find_same_lease_key(struct ksmbd_conn *conn, struct ksmbd_inode *ci, + struct lease_ctx_info *lctx) +{ + struct lease *lease; + int err = 0; + struct lease_table *lb; + + if (!lctx) + return err; + + read_lock(&lease_list_lock); + if (list_empty(&lease_table_list)) { + read_unlock(&lease_list_lock); + return 0; + } + + list_for_each_entry(lb, &lease_table_list, l_entry) { + if (!memcmp(lb->client_guid, conn->ClientGUID, + SMB2_CLIENT_GUID_SIZE)) + goto found; + } + read_unlock(&lease_list_lock); + + return 0; + +found: + list_for_each_entry(lease, &lb->lease_list, l_entry) { + if (lease->ci == ci) + continue; + if (!memcmp(lease->lease_key, lctx->lease_key, + SMB2_LEASE_KEY_SIZE)) { + err = -EINVAL; + ksmbd_debug(OPLOCK, + "found same lease key is already used in other files\n"); + goto out; + } + } + +out: + read_unlock(&lease_list_lock); + return err; +} + +static void add_lease_global_list(struct lease *lease, struct ksmbd_conn *conn, + struct lease_table *new_lb) +{ + struct lease_table *lb; + + write_lock(&lease_list_lock); + list_for_each_entry(lb, &lease_table_list, l_entry) { + if (!memcmp(lb->client_guid, conn->ClientGUID, + SMB2_CLIENT_GUID_SIZE)) { + lease_add_table(lease, lb); + write_unlock(&lease_list_lock); + free_lease_table(new_lb); + return; + } + } + + lease_add_table(lease, new_lb); + list_add(&new_lb->l_entry, &lease_table_list); + write_unlock(&lease_list_lock); +} + +static void set_oplock_level(struct oplock_info *opinfo, int level, + struct lease_ctx_info *lctx) +{ + switch (level) { + case SMB2_OPLOCK_LEVEL_BATCH: + case SMB2_OPLOCK_LEVEL_EXCLUSIVE: + grant_write_oplock(opinfo, level, lctx); + break; + case SMB2_OPLOCK_LEVEL_II: + grant_read_oplock(opinfo, lctx); + break; + default: + grant_none_oplock(opinfo, lctx); + break; + } +} + +void smb_send_parent_lease_break_noti(struct ksmbd_file *fp, + struct lease_ctx_info *lctx) +{ + struct oplock_info *opinfo; + struct ksmbd_inode *p_ci = NULL; + LIST_HEAD(brk_list); + + if (lctx && lctx->version != 2) + return; + + p_ci = ksmbd_inode_lookup_lock(fp->filp->f_path.dentry->d_parent); + if (!p_ci) + return; + + down_read(&p_ci->m_lock); + list_for_each_entry(opinfo, &p_ci->m_op_list, op_entry) { + if (opinfo->conn == NULL || !opinfo->is_lease) + continue; + + if (opinfo->o_lease->state != SMB2_OPLOCK_LEVEL_NONE && + (!lctx || + (!(lctx->flags & SMB2_LEASE_FLAG_PARENT_LEASE_KEY_SET_LE) || + !compare_guid_key(opinfo, fp->conn->ClientGUID, + lctx->parent_lease_key)))) { + if (!atomic_inc_not_zero(&opinfo->refcount)) + continue; + + if (ksmbd_conn_releasing(opinfo->conn)) { + opinfo_put(opinfo); + continue; + } + + if (oplock_break_add(&brk_list, opinfo)) + opinfo_put(opinfo); + } + } + up_read(&p_ci->m_lock); + + oplock_break_drain_none(&brk_list); + + ksmbd_inode_put(p_ci); +} + +void smb_lazy_parent_lease_break_close(struct ksmbd_file *fp) +{ + struct oplock_info *opinfo; + struct ksmbd_inode *p_ci = NULL; + LIST_HEAD(brk_list); + + rcu_read_lock(); + opinfo = rcu_dereference(fp->f_opinfo); + + if (!opinfo || !opinfo->is_lease || opinfo->o_lease->version != 2) { + rcu_read_unlock(); + return; + } + rcu_read_unlock(); + + p_ci = ksmbd_inode_lookup_lock(fp->filp->f_path.dentry->d_parent); + if (!p_ci) + return; + + down_read(&p_ci->m_lock); + list_for_each_entry(opinfo, &p_ci->m_op_list, op_entry) { + if (opinfo->conn == NULL || !opinfo->is_lease) + continue; + + if (opinfo->o_lease->state != SMB2_OPLOCK_LEVEL_NONE) { + if (!atomic_inc_not_zero(&opinfo->refcount)) + continue; + + if (ksmbd_conn_releasing(opinfo->conn)) { + opinfo_put(opinfo); + continue; + } + + if (oplock_break_add(&brk_list, opinfo)) + opinfo_put(opinfo); + } + } + up_read(&p_ci->m_lock); + + oplock_break_drain_none(&brk_list); + + ksmbd_inode_put(p_ci); +} + +/** + * smb_grant_oplock() - handle oplock/lease request on file open + * @work: smb work + * @req_op_level: oplock level + * @pid: id of open file + * @fp: ksmbd file pointer + * @tid: Tree id of connection + * @lctx: lease context information on file open + * @share_ret: share mode + * @replay: whether this is a replayed CREATE request + * + * Return: 0 on success, otherwise error + */ +int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid, + struct ksmbd_file *fp, __u16 tid, + struct lease_ctx_info *lctx, int share_ret, bool replay) +{ + int err = 0; + int break_level = SMB2_OPLOCK_LEVEL_II; + struct oplock_info *opinfo = NULL, *prev_opinfo = NULL; + struct ksmbd_inode *ci = fp->f_ci; + struct lease_table *new_lb = NULL; + struct oplock_snapshot prev_op_snapshot; + bool prev_op_has_lease; + bool prev_durable_open = false; + bool prev_durable_detached = false; + unsigned long long prev_fid = KSMBD_NO_FID; + bool new_lease = false; + bool break_needed; + __le32 prev_op_state = 0; + + /* Only v2 leases handle the directory */ + if (S_ISDIR(file_inode(fp->filp)->i_mode)) { + if (!lctx || lctx->version != 2) + return 0; + } + + opinfo = alloc_opinfo(work, pid, tid); + if (!opinfo) + return -ENOMEM; + + if (lctx) { + opinfo->o_lease = alloc_lease(lctx, ci); + if (!opinfo->o_lease) { + err = -ENOMEM; + goto err_out; + } + opinfo->is_lease = 1; + new_lease = true; + } + + /* ci does not have any oplock */ + if (!opinfo_count(fp)) + goto set_lev; + + /* + * A stat open that only requests metadata access must not break the + * existing caching state. READ_CONTROL (reading the security + * descriptor) does not conflict with a lease, but it does conflict + * with an oplock, so only treat a read-control-only open as a stat + * open when the existing holder is a lease. + */ + if (fp->cdoption != FILE_OVERWRITE_IF_LE && + fp->cdoption != FILE_OVERWRITE_LE && + fp->cdoption != FILE_SUPERSEDE_LE && + (fp->attrib_only || + (!(fp->daccess & ~(FILE_READ_ATTRIBUTES_LE | + FILE_WRITE_ATTRIBUTES_LE | + FILE_SYNCHRONIZE_LE | + FILE_READ_CONTROL_LE)) && + ksmbd_inode_has_lease(ci)))) { + req_op_level = SMB2_OPLOCK_LEVEL_NONE; + goto set_lev; + } + + if (lctx) { + struct oplock_info *m_opinfo; + + /* is lease already granted ? */ + m_opinfo = same_client_has_lease(ci, work->conn->ClientGUID, + lctx); + if (m_opinfo) { + lease_put(opinfo->o_lease); + lease_get(m_opinfo->o_lease); + opinfo->o_lease = m_opinfo->o_lease; + opinfo->level = m_opinfo->level; + new_lease = false; + opinfo_put(m_opinfo); + goto out; + } + } + prev_opinfo = opinfo_get_list(ci, fp, &prev_op_snapshot); + if (!prev_opinfo || + (prev_opinfo->level == SMB2_OPLOCK_LEVEL_NONE && lctx)) { + opinfo_put(prev_opinfo); + goto set_lev; + } + prev_op_has_lease = prev_opinfo->is_lease; + if (prev_op_has_lease) + prev_op_state = prev_opinfo->o_lease->state; + /* + * A replay received while this open is waiting for an oplock or lease + * break must not observe an intermediate level and proceed as a new + * open. This check has to precede break_needed. an oplock may already + * have been downgraded from Batch to II while its acknowledgement is + * still pending. + */ + if (replay && + (test_bit(0, &prev_opinfo->pending_break) || + prev_opinfo->op_state == OPLOCK_ACK_WAIT)) { + err = -EINPROGRESS; + opinfo_put(prev_opinfo); + goto err_out; + } + + if (share_ret < 0 && + prev_opinfo->level == SMB2_OPLOCK_LEVEL_EXCLUSIVE) { + err = share_ret; + opinfo_put(prev_opinfo); + goto err_out; + } + + break_needed = prev_opinfo->level == SMB2_OPLOCK_LEVEL_BATCH || + prev_opinfo->level == SMB2_OPLOCK_LEVEL_EXCLUSIVE || + (share_ret < 0 && prev_op_has_lease && + (prev_op_state & SMB2_LEASE_HANDLE_CACHING_LE)); + if (!break_needed) { + opinfo_put(prev_opinfo); + goto op_break_not_needed; + } + + prev_durable_open = prev_op_snapshot.durable_open; + prev_durable_detached = prev_op_snapshot.durable_detached; + prev_fid = prev_op_snapshot.fid; + + err = oplock_break(prev_opinfo, break_level, work, + share_ret < 0 && prev_opinfo->is_lease, false); + if (prev_durable_detached || (prev_durable_open && err == -ENOENT)) + ksmbd_invalidate_durable_fd(prev_fid); + opinfo_put(prev_opinfo); + if (err == -EAGAIN) { + share_ret = ksmbd_smb_check_shared_mode(fp->filp, fp); + if (share_ret < 0) { + err = share_ret; + goto err_out; + } + goto set_lev; + } + if (err == -ENOENT) { + /* + * A pending durable CREATE can lose the previous oplock when + * its holder closes the file. In that case grant the original + * request its full caching state. Other opens still need the + * normal shared-open downgrade below. + */ + if (!prev_durable_open && + req_op_level != SMB2_OPLOCK_LEVEL_NONE) + req_op_level = SMB2_OPLOCK_LEVEL_II; + goto set_lev; + } + /* Check all oplock was freed by close */ + else if (err < 0) + goto err_out; + +op_break_not_needed: + if (share_ret < 0) { + err = share_ret; + goto err_out; + } + + if (req_op_level != SMB2_OPLOCK_LEVEL_NONE) + req_op_level = SMB2_OPLOCK_LEVEL_II; + + /* grant fixed oplock on stacked locking between lease and oplock */ + if (prev_op_has_lease && !lctx) + if (prev_op_state & SMB2_LEASE_HANDLE_CACHING_LE) + req_op_level = SMB2_OPLOCK_LEVEL_NONE; + + if (!prev_op_has_lease && lctx) { + req_op_level = SMB2_OPLOCK_LEVEL_II; + lctx->req_state = SMB2_LEASE_READ_CACHING_LE; + } + +set_lev: + set_oplock_level(opinfo, req_op_level, lctx); + +out: + /* + * Keep the original publication order so concurrent opens can + * still observe the in-flight grant via ci->m_op_list, but make + * everything after opinfo_add() no-fail by preallocating any new + * lease_table first. + */ + opinfo->o_fp = fp; + if (new_lease) { + new_lb = alloc_lease_table(opinfo); + if (!new_lb) { + err = -ENOMEM; + goto err_out; + } + } + + opinfo_count_inc(fp); + opinfo_add(opinfo, fp); + + if (new_lease) + add_lease_global_list(opinfo->o_lease, opinfo->conn, new_lb); + if (opinfo->is_lease) + lease_add_open(opinfo->o_lease, opinfo); + + rcu_assign_pointer(fp->f_opinfo, opinfo); + + return 0; +err_out: + kfree(new_lb); + opinfo_put(opinfo); + return err; +} + +/** + * smb_break_all_write_oplock() - break batch/exclusive oplock to level2 + * @work: smb work + * @fp: ksmbd file pointer + * @is_trunc: truncate on open + */ +static bool smb_break_all_write_oplock(struct ksmbd_work *work, + struct ksmbd_file *fp, int is_trunc) +{ + struct oplock_info *brk_opinfo; + bool sent_break = false; + + brk_opinfo = opinfo_get_list(fp->f_ci, NULL, NULL); + if (!brk_opinfo) + return false; + if (brk_opinfo->level != SMB2_OPLOCK_LEVEL_BATCH && + brk_opinfo->level != SMB2_OPLOCK_LEVEL_EXCLUSIVE) { + opinfo_put(brk_opinfo); + return false; + } + + brk_opinfo->open_trunc = is_trunc; + oplock_break(brk_opinfo, SMB2_OPLOCK_LEVEL_II, work, false, false); + sent_break = true; + opinfo_put(brk_opinfo); + + return sent_break; +} + +/** + * __smb_break_all_levII_oplock() - send level2 oplock or read lease break command + * from server to client + * @work: smb work + * @fp: ksmbd file pointer + * @is_trunc: truncate on open + * @send_interim: send interim response to the client + * @send_oplock_break: send oplock break notification to the client + * @sync_lease_break: send the lease break notification synchronously + */ +static void __smb_break_all_levII_oplock(struct ksmbd_work *work, + struct ksmbd_file *fp, int is_trunc, + bool send_interim, bool send_oplock_break, + bool sync_lease_break) +{ + struct oplock_info *op, *brk_op; + struct oplock_break_entry *ent, *tmp; + struct ksmbd_inode *ci; + struct ksmbd_conn *conn = work->conn; + bool sent_interim = false; + LIST_HEAD(brk_list); + + if (!test_share_config_flag(work->tcon->share_conf, + KSMBD_SHARE_FLAG_OPLOCKS)) + return; + + ci = fp->f_ci; + op = opinfo_get(fp); + + down_read(&ci->m_lock); + list_for_each_entry(brk_op, &ci->m_op_list, op_entry) { + if (brk_op->conn == NULL) + continue; + + if (!atomic_inc_not_zero(&brk_op->refcount)) + continue; + + if (ksmbd_conn_releasing(brk_op->conn)) { + opinfo_put(brk_op); + continue; + } + + if (!brk_op->is_lease && + brk_op->level != SMB2_OPLOCK_LEVEL_II) { + ksmbd_debug(OPLOCK, "unexpected oplock(0x%x)\n", + brk_op->level); + goto next; + } + + /* Skip oplock being break to none */ + if (brk_op->is_lease && + brk_op->o_lease->new_state == SMB2_LEASE_NONE_LE && + atomic_read(&brk_op->breaking_cnt)) + goto next; + + if (op && op->is_lease && brk_op->is_lease && + !memcmp(conn->ClientGUID, brk_op->conn->ClientGUID, + SMB2_CLIENT_GUID_SIZE) && + !memcmp(op->o_lease->lease_key, brk_op->o_lease->lease_key, + SMB2_LEASE_KEY_SIZE)) + goto next; + brk_op->open_trunc = is_trunc; + + /* + * Defer the break until ci->m_lock is released: oplock_break() + * may block waiting for the lease break acknowledgment, and the + * close that wakes that wait needs ci->m_lock for write. + */ + if (!oplock_break_add(&brk_list, brk_op)) + continue; +next: + opinfo_put(brk_op); + } + up_read(&ci->m_lock); + + list_for_each_entry_safe(ent, tmp, &brk_list, list) { + brk_op = ent->opinfo; + + if (!brk_op->is_lease && !send_oplock_break) { + brk_op->level = SMB2_OPLOCK_LEVEL_NONE; + spin_lock(&brk_op->state_lock); + if (brk_op->op_state != OPLOCK_CLOSING) + brk_op->op_state = OPLOCK_STATE_NONE; + spin_unlock(&brk_op->state_lock); + } else { + oplock_break(brk_op, + brk_op->is_lease && !is_trunc ? + SMB2_OPLOCK_LEVEL_II : SMB2_OPLOCK_LEVEL_NONE, + send_interim && !sent_interim ? work : NULL, + false, sync_lease_break); + } + sent_interim = true; + list_del(&ent->list); + opinfo_put(brk_op); + kfree(ent); + } + + if (op) + opinfo_put(op); +} + +void smb_break_all_levII_oplock(struct ksmbd_work *work, struct ksmbd_file *fp, + int is_trunc) +{ + __smb_break_all_levII_oplock(work, fp, is_trunc, true, true, false); +} + +void smb_break_all_levII_oplock_rename(struct ksmbd_work *work, struct ksmbd_file *fp) +{ + __smb_break_all_levII_oplock(work, fp, 0, true, true, true); +} + +void smb_break_all_levII_oplock_no_interim(struct ksmbd_work *work, + struct ksmbd_file *fp, int is_trunc) +{ + __smb_break_all_levII_oplock(work, fp, is_trunc, false, true, false); +} + +void smb_break_all_levII_oplock_for_delete(struct ksmbd_work *work, + struct ksmbd_file *fp) +{ + __smb_break_all_levII_oplock(work, fp, 0, false, false, false); +} + +/** + * smb_break_all_oplock() - break both batch/exclusive and level2 oplock + * @work: smb work + * @fp: ksmbd file pointer + */ +void smb_break_all_oplock(struct ksmbd_work *work, struct ksmbd_file *fp) +{ + bool sent_break; + + if (!test_share_config_flag(work->tcon->share_conf, + KSMBD_SHARE_FLAG_OPLOCKS)) + return; + + sent_break = smb_break_all_write_oplock(work, fp, 1); + __smb_break_all_levII_oplock(work, fp, 1, !sent_break, true, false); +} + +/** + * smb2_map_lease_to_oplock() - map lease state to corresponding oplock type + * @lease_state: lease type + * + * Return: 0 if no mapping, otherwise corresponding oplock type + */ +__u8 smb2_map_lease_to_oplock(__le32 lease_state) +{ + if ((lease_state & SMB2_LEASE_WRITE_CACHING_LE) && + (lease_state & SMB2_LEASE_HANDLE_CACHING_LE)) { + return SMB2_OPLOCK_LEVEL_BATCH; + } else if (lease_state & SMB2_LEASE_WRITE_CACHING_LE) { + return SMB2_OPLOCK_LEVEL_EXCLUSIVE; + } else if (lease_state & (SMB2_LEASE_READ_CACHING_LE | + SMB2_LEASE_HANDLE_CACHING_LE)) { + return SMB2_OPLOCK_LEVEL_II; + } + return 0; +} + +/** + * create_lease_buf() - create lease context for open cmd response + * @rbuf: buffer to create lease context response + * @lease: buffer to stored parsed lease state information + */ +void create_lease_buf(u8 *rbuf, struct lease *lease) +{ + if (lease->version == 2) { + struct create_lease_v2 *buf = (struct create_lease_v2 *)rbuf; + __le32 flags = 0; + + memset(buf, 0, sizeof(struct create_lease_v2)); + memcpy(buf->lcontext.LeaseKey, lease->lease_key, + SMB2_LEASE_KEY_SIZE); + if (lease_has_parent_key(lease)) + flags |= SMB2_LEASE_FLAG_PARENT_LEASE_KEY_SET_LE; + if (lease_break_in_progress(lease)) + flags |= SMB2_LEASE_FLAG_BREAK_IN_PROGRESS_LE; + buf->lcontext.LeaseFlags = flags; + buf->lcontext.Epoch = cpu_to_le16(lease->epoch); + buf->lcontext.LeaseState = lease->state; + if (lease_has_parent_key(lease)) + memcpy(buf->lcontext.ParentLeaseKey, lease->parent_lease_key, + SMB2_LEASE_KEY_SIZE); + buf->ccontext.DataOffset = cpu_to_le16(offsetof + (struct create_lease_v2, lcontext)); + buf->ccontext.DataLength = cpu_to_le32(sizeof(struct lease_context_v2)); + buf->ccontext.NameOffset = cpu_to_le16(offsetof + (struct create_lease_v2, Name)); + buf->ccontext.NameLength = cpu_to_le16(4); + buf->Name[0] = 'R'; + buf->Name[1] = 'q'; + buf->Name[2] = 'L'; + buf->Name[3] = 's'; + } else { + struct create_lease *buf = (struct create_lease *)rbuf; + + memset(buf, 0, sizeof(struct create_lease)); + memcpy(buf->lcontext.LeaseKey, lease->lease_key, SMB2_LEASE_KEY_SIZE); + if (lease_break_in_progress(lease)) + buf->lcontext.LeaseFlags = + SMB2_LEASE_FLAG_BREAK_IN_PROGRESS_LE; + buf->lcontext.LeaseState = lease->state; + buf->ccontext.DataOffset = cpu_to_le16(offsetof + (struct create_lease, lcontext)); + buf->ccontext.DataLength = cpu_to_le32(sizeof(struct lease_context)); + buf->ccontext.NameOffset = cpu_to_le16(offsetof + (struct create_lease, Name)); + buf->ccontext.NameLength = cpu_to_le16(4); + buf->Name[0] = 'R'; + buf->Name[1] = 'q'; + buf->Name[2] = 'L'; + buf->Name[3] = 's'; + } +} + +/** + * parse_lease_state() - parse lease context contained in file open request + * @open_req: buffer containing smb2 file open(create) request + * + * Return: allocated lease context object on success, otherwise NULL + */ +struct lease_ctx_info *parse_lease_state(void *open_req) +{ + struct create_context *cc; + struct smb2_create_req *req = (struct smb2_create_req *)open_req; + struct lease_ctx_info *lreq; + + cc = smb2_find_context_vals(req, SMB2_CREATE_REQUEST_LEASE, 4); + if (IS_ERR(cc)) + return ERR_CAST(cc); + if (!cc) + return NULL; + + lreq = kzalloc_obj(struct lease_ctx_info, KSMBD_DEFAULT_GFP); + if (!lreq) + return ERR_PTR(-ENOMEM); + + if (sizeof(struct lease_context_v2) == le32_to_cpu(cc->DataLength)) { + struct create_lease_v2 *lc = (struct create_lease_v2 *)cc; + + if (le16_to_cpu(cc->DataOffset) + le32_to_cpu(cc->DataLength) < + sizeof(struct create_lease_v2) - 4) + goto err_out; + + memcpy(lreq->lease_key, lc->lcontext.LeaseKey, SMB2_LEASE_KEY_SIZE); + lreq->req_state = lc->lcontext.LeaseState; + lreq->flags = lc->lcontext.LeaseFlags; + lreq->epoch = lc->lcontext.Epoch; + lreq->duration = lc->lcontext.LeaseDuration; + if (!lease_state_valid(lreq->req_state) || + !lease_v2_flags_valid(lreq->flags)) + goto err_out; + lreq->req_state = lease_state_grantable(lreq->req_state); + if (lreq->flags == SMB2_LEASE_FLAG_PARENT_LEASE_KEY_SET_LE) + memcpy(lreq->parent_lease_key, lc->lcontext.ParentLeaseKey, + SMB2_LEASE_KEY_SIZE); + lreq->version = 2; + } else if (sizeof(struct lease_context) == le32_to_cpu(cc->DataLength)) { + struct create_lease *lc = (struct create_lease *)cc; + + if (le16_to_cpu(cc->DataOffset) + le32_to_cpu(cc->DataLength) < + sizeof(struct create_lease)) + goto err_out; + + memcpy(lreq->lease_key, lc->lcontext.LeaseKey, SMB2_LEASE_KEY_SIZE); + lreq->req_state = lc->lcontext.LeaseState; + lreq->flags = 0; + lreq->duration = lc->lcontext.LeaseDuration; + if (!lease_state_valid(lreq->req_state)) + goto err_out; + lreq->req_state = lease_state_grantable(lreq->req_state); + lreq->version = 1; + } else + goto err_out; + return lreq; +err_out: + kfree(lreq); + return ERR_PTR(-EINVAL); +} + +/** + * smb2_find_context_vals() - find a particular context info in open request + * @open_req: buffer containing smb2 file open(create) request + * @tag: context name to search for + * @tag_len: the length of tag + * + * Return: pointer to requested context, NULL if @str context not found + * or error pointer if name length is invalid. + */ +struct create_context *smb2_find_context_vals(void *open_req, const char *tag, int tag_len) +{ + struct create_context *cc; + unsigned int next = 0; + char *name; + struct smb2_create_req *req = (struct smb2_create_req *)open_req; + unsigned int remain_len, name_off, name_len, value_off, value_len, + cc_len; + + /* + * CreateContextsOffset and CreateContextsLength are guaranteed to + * be valid because of ksmbd_smb2_check_message(). + */ + if (!req->CreateContextsOffset || !req->CreateContextsLength) + return NULL; + + cc = (struct create_context *)((char *)req + + le32_to_cpu(req->CreateContextsOffset)); + remain_len = le32_to_cpu(req->CreateContextsLength); + do { + cc = (struct create_context *)((char *)cc + next); + if (remain_len < offsetof(struct create_context, Buffer)) + return ERR_PTR(-EINVAL); + + next = le32_to_cpu(cc->Next); + name_off = le16_to_cpu(cc->NameOffset); + name_len = le16_to_cpu(cc->NameLength); + value_off = le16_to_cpu(cc->DataOffset); + value_len = le32_to_cpu(cc->DataLength); + cc_len = next ? next : remain_len; + + if ((next & 0x7) != 0 || + next > remain_len || + name_off != offsetof(struct create_context, Buffer) || + name_len < 4 || + name_off + name_len > cc_len || + (value_off & 0x7) != 0 || + (value_len && value_off < name_off + (name_len < 8 ? 8 : name_len)) || + ((u64)value_off + value_len > cc_len)) + return ERR_PTR(-EINVAL); + + name = (char *)cc + name_off; + if (name_len == tag_len && !memcmp(name, tag, name_len)) + return cc; + + remain_len -= next; + } while (next != 0); + + return NULL; +} + +/** + * create_durable_rsp_buf() - create durable handle context + * @cc: buffer to create durable context response + */ +void create_durable_rsp_buf(char *cc) +{ + struct create_durable_rsp *buf; + + buf = (struct create_durable_rsp *)cc; + memset(buf, 0, sizeof(struct create_durable_rsp)); + buf->ccontext.DataOffset = cpu_to_le16(offsetof + (struct create_durable_rsp, Data)); + buf->ccontext.DataLength = cpu_to_le32(8); + buf->ccontext.NameOffset = cpu_to_le16(offsetof + (struct create_durable_rsp, Name)); + buf->ccontext.NameLength = cpu_to_le16(4); + /* SMB2_CREATE_DURABLE_HANDLE_RESPONSE is "DHnQ" */ + buf->Name[0] = 'D'; + buf->Name[1] = 'H'; + buf->Name[2] = 'n'; + buf->Name[3] = 'Q'; +} + +/** + * create_durable_v2_rsp_buf() - create durable handle v2 context + * @cc: buffer to create durable context response + * @fp: ksmbd file pointer + */ +void create_durable_v2_rsp_buf(char *cc, struct ksmbd_file *fp) +{ + struct create_durable_rsp_v2 *buf; + + buf = (struct create_durable_rsp_v2 *)cc; + memset(buf, 0, sizeof(*buf)); + buf->ccontext.DataOffset = cpu_to_le16(offsetof + (struct create_durable_rsp_v2, dcontext)); + buf->ccontext.DataLength = cpu_to_le32(8); + buf->ccontext.NameOffset = cpu_to_le16(offsetof + (struct create_durable_rsp_v2, Name)); + buf->ccontext.NameLength = cpu_to_le16(4); + /* SMB2_CREATE_DURABLE_HANDLE_RESPONSE_V2 is "DH2Q" */ + buf->Name[0] = 'D'; + buf->Name[1] = 'H'; + buf->Name[2] = '2'; + buf->Name[3] = 'Q'; + + buf->dcontext.Timeout = cpu_to_le32(fp->durable_timeout); + if (fp->is_persistent) + buf->dcontext.Flags = cpu_to_le32(SMB2_DHANDLE_FLAG_PERSISTENT); +} + +/** + * create_mxac_rsp_buf() - create query maximal access context + * @cc: buffer to create maximal access context response + * @maximal_access: maximal access + */ +void create_mxac_rsp_buf(char *cc, int maximal_access) +{ + struct create_mxac_rsp *buf; + + buf = (struct create_mxac_rsp *)cc; + memset(buf, 0, sizeof(struct create_mxac_rsp)); + buf->ccontext.DataOffset = cpu_to_le16(offsetof + (struct create_mxac_rsp, QueryStatus)); + buf->ccontext.DataLength = cpu_to_le32(8); + buf->ccontext.NameOffset = cpu_to_le16(offsetof + (struct create_mxac_rsp, Name)); + buf->ccontext.NameLength = cpu_to_le16(4); + /* SMB2_CREATE_QUERY_MAXIMAL_ACCESS_RESPONSE is "MxAc" */ + buf->Name[0] = 'M'; + buf->Name[1] = 'x'; + buf->Name[2] = 'A'; + buf->Name[3] = 'c'; + + buf->QueryStatus = STATUS_SUCCESS; + buf->MaximalAccess = cpu_to_le32(maximal_access); +} + +void create_disk_id_rsp_buf(char *cc, __u64 file_id, __u64 vol_id) +{ + struct create_disk_id_rsp *buf; + + buf = (struct create_disk_id_rsp *)cc; + memset(buf, 0, sizeof(struct create_disk_id_rsp)); + buf->ccontext.DataOffset = cpu_to_le16(offsetof + (struct create_disk_id_rsp, DiskFileId)); + buf->ccontext.DataLength = cpu_to_le32(32); + buf->ccontext.NameOffset = cpu_to_le16(offsetof + (struct create_mxac_rsp, Name)); + buf->ccontext.NameLength = cpu_to_le16(4); + /* SMB2_CREATE_QUERY_ON_DISK_ID_RESPONSE is "QFid" */ + buf->Name[0] = 'Q'; + buf->Name[1] = 'F'; + buf->Name[2] = 'i'; + buf->Name[3] = 'd'; + + buf->DiskFileId = cpu_to_le64(file_id); + buf->VolumeId = cpu_to_le64(vol_id); +} + +/** + * create_posix_rsp_buf() - create posix extension context + * @cc: buffer to create posix on posix response + * @fp: ksmbd file pointer + */ +void create_posix_rsp_buf(char *cc, struct ksmbd_file *fp) +{ + struct create_posix_rsp *buf; + struct inode *inode = file_inode(fp->filp); + struct mnt_idmap *idmap = file_mnt_idmap(fp->filp); + vfsuid_t vfsuid = i_uid_into_vfsuid(idmap, inode); + vfsgid_t vfsgid = i_gid_into_vfsgid(idmap, inode); + + buf = (struct create_posix_rsp *)cc; + memset(buf, 0, sizeof(struct create_posix_rsp)); + buf->ccontext.DataOffset = cpu_to_le16(offsetof + (struct create_posix_rsp, nlink)); + /* + * DataLength = nlink(4) + reparse_tag(4) + mode(4) + + * domain sid(28) + unix group sid(16). + */ + buf->ccontext.DataLength = cpu_to_le32(56); + buf->ccontext.NameOffset = cpu_to_le16(offsetof + (struct create_posix_rsp, Name)); + buf->ccontext.NameLength = cpu_to_le16(POSIX_CTXT_DATA_LEN); + /* SMB2_CREATE_TAG_POSIX is "0x93AD25509CB411E7B42383DE968BCD7C" */ + buf->Name[0] = 0x93; + buf->Name[1] = 0xAD; + buf->Name[2] = 0x25; + buf->Name[3] = 0x50; + buf->Name[4] = 0x9C; + buf->Name[5] = 0xB4; + buf->Name[6] = 0x11; + buf->Name[7] = 0xE7; + buf->Name[8] = 0xB4; + buf->Name[9] = 0x23; + buf->Name[10] = 0x83; + buf->Name[11] = 0xDE; + buf->Name[12] = 0x96; + buf->Name[13] = 0x8B; + buf->Name[14] = 0xCD; + buf->Name[15] = 0x7C; + + buf->nlink = cpu_to_le32(inode->i_nlink); + buf->reparse_tag = cpu_to_le32(fp->volatile_id); + buf->mode = cpu_to_le32(inode->i_mode & 0777); + /* + * SidBuffer(44) contain two sids(Domain sid(28), UNIX group sid(16)). + * Domain sid(28) = revision(1) + num_subauth(1) + authority(6) + + * sub_auth(4 * 4(num_subauth)) + RID(4). + * UNIX group id(16) = revision(1) + num_subauth(1) + authority(6) + + * sub_auth(4 * 1(num_subauth)) + RID(4). + */ + id_to_sid(from_kuid_munged(&init_user_ns, vfsuid_into_kuid(vfsuid)), + SIDOWNER, (struct smb_sid *)&buf->SidBuffer[0]); + id_to_sid(from_kgid_munged(&init_user_ns, vfsgid_into_kgid(vfsgid)), + SIDUNIX_GROUP, (struct smb_sid *)&buf->SidBuffer[28]); +} + +/** + * create_aapl_rsp_buf() - build AAPL kAAPL_SERVER_QUERY response + * @cc: buffer to write the create context into (AAPL_RSP_MAX_SIZE bytes) + * @vol_caps: volume capability flags (SMB2_CRTCTX_AAPL_* volume bits) + * @req_bitmap: the client's request bitmap, echoed back in reply_bitmap + * + * Response format follows the layout observed from macOS's own smbd, and + * matches the client-side parsing in AAPL's published public client kernel + * source (public client behavior reference, kAAPL_SERVER_QUERY + * case): reply_bitmap, then server_caps/vol_caps/model-info fields present + * only when their reply_bitmap bit is set: + * reply_bitmap = req_bitmap masked to the fields we support + * server_caps = AAPL_SERVER_CAPS_KSMBD when requested + * vol_caps = caller-supplied + * model string = server_conf.aapl_model (default "Xserve") in UTF-16LE, + * when SMB2_CRTCTX_AAPL_MODEL_INFO requested + * + * Sending reply_bitmap with MODEL_INFO set but no model string causes + * smbfs.kext to enter a broken disconnect path requiring a macOS reboot. + * @readdir_attr_v2: advertise SMB2_CRTCTX_AAPL_SUPPORTS_READ_DIR_ATTR_V2 + * instead of the V1 bit + */ +void create_aapl_rsp_buf(char *cc, __u64 vol_caps, __u64 req_bitmap, + bool readdir_attr_v2) +{ + struct create_aapl_rsp *buf; + u64 reply_bitmap; + u64 server_caps; + u32 data_len; + + buf = (struct create_aapl_rsp *)cc; + memset(buf, 0, AAPL_RSP_MAX_SIZE); + + reply_bitmap = req_bitmap & (SMB2_CRTCTX_AAPL_SERVER_CAPS | + SMB2_CRTCTX_AAPL_VOLUME_CAPS | + SMB2_CRTCTX_AAPL_MODEL_INFO); + + /* base data: cmd(4)+reserved(4)+reply_bitmap(8)+server_caps(8)+vol_caps(8) */ + data_len = 32; + if (reply_bitmap & SMB2_CRTCTX_AAPL_MODEL_INFO) + data_len += 4 + 4 + AAPL_MODEL_UTF16_BYTES; /* pad2+model_bytes+string */ + + buf->ccontext.DataOffset = cpu_to_le16(offsetof(struct create_aapl_rsp, cmd)); + buf->ccontext.DataLength = cpu_to_le32(data_len); + buf->ccontext.NameOffset = cpu_to_le16(offsetof(struct create_aapl_rsp, Name)); + buf->ccontext.NameLength = cpu_to_le16(SMB2_CREATE_AAPL_LEN); + buf->Name[0] = 'A'; + buf->Name[1] = 'A'; + buf->Name[2] = 'P'; + buf->Name[3] = 'L'; + + buf->cmd = cpu_to_le32(SMB2_CRTCTX_AAPL_SERVER_QUERY); + buf->reply_bitmap = cpu_to_le64(reply_bitmap); + server_caps = AAPL_SERVER_CAPS_KSMBD; + if (readdir_attr_v2) + server_caps = (server_caps & ~SMB2_CRTCTX_AAPL_SUPPORTS_READ_DIR_ATTR) | + SMB2_CRTCTX_AAPL_SUPPORTS_READ_DIR_ATTR_V2; + buf->server_caps = (reply_bitmap & SMB2_CRTCTX_AAPL_SERVER_CAPS) ? + cpu_to_le64(server_caps) : 0; + buf->vol_caps = (reply_bitmap & SMB2_CRTCTX_AAPL_VOLUME_CAPS) ? + cpu_to_le64(vol_caps) : 0; + + if (reply_bitmap & SMB2_CRTCTX_AAPL_MODEL_INFO) { + __le32 *p = (__le32 *)((u8 *)buf + sizeof(*buf)); + __le16 *model_str = (__le16 *)(p + 2); + const char *src = server_conf.aapl_model[0] ? + server_conf.aapl_model : "Xserve"; + int i, model_bytes = 0; + + /* Convert ASCII model string to UTF-16LE in-place */ + for (i = 0; src[i] && i < AAPL_MODEL_MAX_CHARS; i++) { + model_str[i] = cpu_to_le16((unsigned char)src[i]); + model_bytes += 2; + } + + p[0] = 0; /* pad2 */ + p[1] = cpu_to_le32(model_bytes); + + /* Update DataLength to reflect actual model string size */ + buf->ccontext.DataLength = + cpu_to_le32(data_len - AAPL_MODEL_UTF16_BYTES + model_bytes); + } +} + +/* + * Find lease object(opinfo) for given lease key/fid from lease + * break/file close path. + */ +/** + * lookup_lease_in_table() - find a matching lease info object + * @conn: connection instance + * @lease_key: lease key to be searched for + * + * Return: opinfo if found matching opinfo, otherwise NULL + */ +struct oplock_info *lookup_lease_in_table(struct ksmbd_conn *conn, + char *lease_key) +{ + struct oplock_info *opinfo = NULL, *ret_op = NULL; + struct lease *lease; + struct lease_table *lt; + + read_lock(&lease_list_lock); + list_for_each_entry(lt, &lease_table_list, l_entry) { + if (!memcmp(lt->client_guid, conn->ClientGUID, + SMB2_CLIENT_GUID_SIZE)) + goto found; + } + + read_unlock(&lease_list_lock); + return NULL; + +found: + list_for_each_entry(lease, <->lease_list, l_entry) { + if (memcmp(lease->lease_key, lease_key, SMB2_LEASE_KEY_SIZE)) + continue; + if (!(lease->state & (SMB2_LEASE_HANDLE_CACHING_LE | + SMB2_LEASE_WRITE_CACHING_LE))) + break; + + spin_lock(&lease->lock); + list_for_each_entry(opinfo, &lease->open_list, lease_entry) { + if (!opinfo->op_state || + opinfo->op_state == OPLOCK_CLOSING) + continue; + if (!atomic_inc_not_zero(&opinfo->refcount)) + continue; + ret_op = opinfo; + } + spin_unlock(&lease->lock); + if (ret_op) { + ksmbd_debug(OPLOCK, "found opinfo\n"); + goto out; + } + break; + } + +out: + read_unlock(&lease_list_lock); + return ret_op; +} + +int smb2_check_durable_oplock(struct ksmbd_conn *conn, + struct ksmbd_share_config *share, + struct ksmbd_file *fp, + struct lease_ctx_info *lctx, + struct ksmbd_user *user, + char *name) +{ + struct oplock_info *opinfo = opinfo_get(fp); + int ret = 0; + + if (!opinfo) + return 0; + + if (ksmbd_has_other_active_fd(fp)) { + ksmbd_debug(SMB, "Durable handle reconnect failed: competing open\n"); + ret = -EBADF; + goto out; + } + + if (ksmbd_vfs_compare_durable_owner(fp, user) == false) { + ksmbd_debug(SMB, "Durable handle reconnect failed: owner mismatch\n"); + ret = -EBADF; + goto out; + } + + if (opinfo->is_lease == false) { + if (lctx) { + pr_err("create context include lease\n"); + ret = -EBADF; + goto out; + } + + if (opinfo->level != SMB2_OPLOCK_LEVEL_BATCH) { + pr_err("oplock level is not equal to SMB2_OPLOCK_LEVEL_BATCH\n"); + ret = -EBADF; + } + + goto out; + } + + if (memcmp(conn->ClientGUID, fp->client_guid, + SMB2_CLIENT_GUID_SIZE)) { + ksmbd_debug(SMB, "Client guid of fp is not equal to the one of connection\n"); + ret = -EBADF; + goto out; + } + + if (!lctx) { + ksmbd_debug(SMB, "create context does not include lease\n"); + ret = -EBADF; + goto out; + } + + if (memcmp(opinfo->o_lease->lease_key, lctx->lease_key, + SMB2_LEASE_KEY_SIZE)) { + ksmbd_debug(SMB, + "lease key of fp does not match lease key in create context\n"); + ret = -EBADF; + goto out; + } + + if (!(opinfo->o_lease->state & SMB2_LEASE_HANDLE_CACHING_LE)) { + ksmbd_debug(SMB, "lease state does not contain SMB2_LEASE_HANDLE_CACHING\n"); + ret = -EBADF; + goto out; + } + + if (opinfo->o_lease->version != lctx->version) { + ksmbd_debug(SMB, + "lease version of fp does not match the one in create context\n"); + ret = -EBADF; + goto out; + } + + if (!ksmbd_inode_pending_delete(fp)) + ret = ksmbd_validate_name_reconnect(share, fp, name); +out: + opinfo_put(opinfo); + return ret; +} diff --git a/fs/smb/server/oplock.h b/fs/smb/server/oplock.h new file mode 100644 index 000000000..b08d21758 --- /dev/null +++ b/fs/smb/server/oplock.h @@ -0,0 +1,147 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2016 Namjae Jeon + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#ifndef __KSMBD_OPLOCK_H +#define __KSMBD_OPLOCK_H + +#include "smb_common.h" + +#define OPLOCK_WAIT_TIME (35 * HZ) + +/* Oplock states */ +#define OPLOCK_STATE_NONE 0x00 +#define OPLOCK_ACK_WAIT 0x01 +#define OPLOCK_CLOSING 0x02 + +#define OPLOCK_WRITE_TO_READ 0x01 +#define OPLOCK_READ_HANDLE_TO_READ 0x02 +#define OPLOCK_WRITE_TO_NONE 0x04 +#define OPLOCK_READ_TO_NONE 0x08 + +struct lease_ctx_info { + __u8 lease_key[SMB2_LEASE_KEY_SIZE]; + __le32 req_state; + __le32 flags; + __le64 duration; + __u8 parent_lease_key[SMB2_LEASE_KEY_SIZE]; + __le16 epoch; + int version; + bool is_dir; +}; + +struct lease_table { + char client_guid[SMB2_CLIENT_GUID_SIZE]; + struct ksmbd_conn *conn; + struct list_head lease_list; + struct list_head l_entry; + spinlock_t lb_lock; +}; + +struct lease { + __u8 lease_key[SMB2_LEASE_KEY_SIZE]; + __le32 state; + __le32 new_state; + __le32 flags; + __le64 duration; + __u8 parent_lease_key[SMB2_LEASE_KEY_SIZE]; + int version; + unsigned short epoch; + bool is_dir; + bool reuse_epoch; + struct ksmbd_inode *ci; + struct lease_table *l_lb; + struct list_head l_entry; + struct list_head open_list; + spinlock_t lock; + atomic_t refcount; +}; + +struct oplock_info { + struct ksmbd_conn *conn; + struct ksmbd_session *sess; + struct ksmbd_work *work; + struct ksmbd_file *o_fp; + int level; + int op_state; + spinlock_t state_lock; + unsigned long pending_break; + u64 fid; + atomic_t breaking_cnt; + atomic_t refcount; + __u16 Tid; + bool is_lease; + bool open_trunc; /* truncate on open */ + struct lease *o_lease; + struct list_head op_entry; + struct list_head lease_entry; + wait_queue_head_t oplock_q; /* Other server threads */ + wait_queue_head_t oplock_brk; /* oplock breaking wait */ + struct rcu_head rcu; +}; + +struct lease_break_info { + __le32 curr_state; + __le32 new_state; + __le16 epoch; + char lease_key[SMB2_LEASE_KEY_SIZE]; +}; + +struct oplock_break_info { + int level; + int open_trunc; + int fid; +}; + +int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, + u64 pid, struct ksmbd_file *fp, __u16 tid, + struct lease_ctx_info *lctx, int share_ret, bool replay); +void smb_break_all_levII_oplock(struct ksmbd_work *work, + struct ksmbd_file *fp, int is_trunc); +void smb_break_all_levII_oplock_rename(struct ksmbd_work *work, struct ksmbd_file *fp); +void smb_break_all_levII_oplock_no_interim(struct ksmbd_work *work, + struct ksmbd_file *fp, int is_trunc); +void smb_break_all_levII_oplock_for_delete(struct ksmbd_work *work, + struct ksmbd_file *fp); +int opinfo_write_to_read(struct oplock_info *opinfo); +int opinfo_read_handle_to_read(struct oplock_info *opinfo); +int opinfo_write_to_none(struct oplock_info *opinfo); +int opinfo_read_to_none(struct oplock_info *opinfo); +void close_id_del_oplock(struct ksmbd_file *fp); +void smb_break_all_oplock(struct ksmbd_work *work, struct ksmbd_file *fp); +struct oplock_info *opinfo_get(struct ksmbd_file *fp); +void opinfo_put(struct oplock_info *opinfo); + +/* Lease related functions */ +void create_lease_buf(u8 *rbuf, struct lease *lease); +struct lease_ctx_info *parse_lease_state(void *open_req); +__u8 smb2_map_lease_to_oplock(__le32 lease_state); +void lease_update_oplock_levels(struct lease *lease); +int lease_read_to_write(struct oplock_info *opinfo); + +/* Durable related functions */ +void create_durable_rsp_buf(char *cc); +void create_durable_v2_rsp_buf(char *cc, struct ksmbd_file *fp); +void create_mxac_rsp_buf(char *cc, int maximal_access); +void create_disk_id_rsp_buf(char *cc, __u64 file_id, __u64 vol_id); +void create_posix_rsp_buf(char *cc, struct ksmbd_file *fp); +void create_aapl_rsp_buf(char *cc, __u64 vol_caps, __u64 req_bitmap, + bool readdir_attr_v2); +struct create_context *smb2_find_context_vals(void *open_req, const char *tag, int tag_len); +struct oplock_info *lookup_lease_in_table(struct ksmbd_conn *conn, + char *lease_key); +int find_same_lease_key(struct ksmbd_conn *conn, struct ksmbd_inode *ci, + struct lease_ctx_info *lctx); +void destroy_lease_table(struct ksmbd_conn *conn); +void smb_send_parent_lease_break_noti(struct ksmbd_file *fp, + struct lease_ctx_info *lctx); +void smb_lazy_parent_lease_break_close(struct ksmbd_file *fp); +int smb2_check_durable_oplock(struct ksmbd_conn *conn, + struct ksmbd_share_config *share, + struct ksmbd_file *fp, + struct lease_ctx_info *lctx, + struct ksmbd_user *user, + char *name); +#endif /* __KSMBD_OPLOCK_H */ diff --git a/fs/smb/server/proc.c b/fs/smb/server/proc.c new file mode 100644 index 000000000..826353ed0 --- /dev/null +++ b/fs/smb/server/proc.c @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2025, LG Electronics. + * Author(s): Hyunchul Lee + * Copyright (C) 2025, Samsung Electronics. + * Author(s): Vedansh Bhardwaj + */ + +#include +#include +#include + +#include "misc.h" +#include "connection.h" +#include "server.h" +#include "stats.h" +#include "smb_common.h" +#include "smb2pdu.h" +#include "vfs_cache.h" + +static struct proc_dir_entry *ksmbd_proc_fs; +struct ksmbd_counters ksmbd_counters; + +struct proc_dir_entry *ksmbd_proc_create(const char *name, + int (*show)(struct seq_file *m, void *v), + void *v) +{ + return proc_create_single_data(name, 0400, ksmbd_proc_fs, + show, v); +} + +void ksmbd_proc_show_flag_names(struct seq_file *m, + const struct ksmbd_const_name *table, + int count, unsigned int flags) +{ + unsigned int remaining = flags; + bool separator = false; + int i; + + for (i = 0; i < count; i++) { + unsigned int flag = table[i].const_value; + + if (!flag || (remaining & flag) != flag) + continue; + seq_printf(m, "%s%s", separator ? "," : "", table[i].name); + separator = true; + remaining &= ~flag; + } + + if (remaining) + seq_printf(m, "%s0x%08x", separator ? "," : "", remaining); + else if (!separator) + seq_puts(m, "none"); +} + +const char *ksmbd_proc_const_name(const struct ksmbd_const_name *table, + int count, unsigned int const_value) +{ + int i; + + for (i = 0; i < count; i++) { + if (table[i].const_value == const_value) + return table[i].name; + } + return NULL; +} + +struct ksmbd_const_smb2_process_req { + unsigned int const_value; + const char *name; +}; + +static const struct ksmbd_const_smb2_process_req smb2_process_req[KSMBD_COUNTER_MAX_REQS] = { + {le16_to_cpu(SMB2_NEGOTIATE), "SMB2_NEGOTIATE"}, + {le16_to_cpu(SMB2_SESSION_SETUP), "SMB2_SESSION_SETUP"}, + {le16_to_cpu(SMB2_LOGOFF), "SMB2_LOGOFF"}, + {le16_to_cpu(SMB2_TREE_CONNECT), "SMB2_TREE_CONNECT"}, + {le16_to_cpu(SMB2_TREE_DISCONNECT), "SMB2_TREE_DISCONNECT"}, + {le16_to_cpu(SMB2_CREATE), "SMB2_CREATE"}, + {le16_to_cpu(SMB2_CLOSE), "SMB2_CLOSE"}, + {le16_to_cpu(SMB2_FLUSH), "SMB2_FLUSH"}, + {le16_to_cpu(SMB2_READ), "SMB2_READ"}, + {le16_to_cpu(SMB2_WRITE), "SMB2_WRITE"}, + {le16_to_cpu(SMB2_LOCK), "SMB2_LOCK"}, + {le16_to_cpu(SMB2_IOCTL), "SMB2_IOCTL"}, + {le16_to_cpu(SMB2_CANCEL), "SMB2_CANCEL"}, + {le16_to_cpu(SMB2_ECHO), "SMB2_ECHO"}, + {le16_to_cpu(SMB2_QUERY_DIRECTORY), "SMB2_QUERY_DIRECTORY"}, + {le16_to_cpu(SMB2_CHANGE_NOTIFY), "SMB2_CHANGE_NOTIFY"}, + {le16_to_cpu(SMB2_QUERY_INFO), "SMB2_QUERY_INFO"}, + {le16_to_cpu(SMB2_SET_INFO), "SMB2_SET_INFO"}, + {le16_to_cpu(SMB2_OPLOCK_BREAK), "SMB2_OPLOCK_BREAK"}, +}; + +static const char *ksmbd_server_state_string(void) +{ + switch (READ_ONCE(server_conf.state)) { + case SERVER_STATE_STARTING_UP: + return "starting"; + case SERVER_STATE_RUNNING: + return "running"; + case SERVER_STATE_RESETTING: + return "resetting"; + case SERVER_STATE_SHUTTING_DOWN: + return "shutdown"; + default: + return "unknown"; + } +} + +static const char *ksmbd_signing_mode_string(void) +{ + switch (server_conf.signing) { + case KSMBD_CONFIG_OPT_DISABLED: + return "disabled"; + case KSMBD_CONFIG_OPT_MANDATORY: + return "mandatory"; + case KSMBD_CONFIG_OPT_AUTO: + return "auto"; + default: + return "unknown"; + } +} + +static void proc_show_runtime_totals(struct seq_file *m) +{ + struct ksmbd_conn *conn; + unsigned int clients = 0; + unsigned int open_files = 0; + int i; + + down_read(&conn_list_lock); + hash_for_each(conn_list, i, conn, hlist) { + clients++; + open_files += atomic_read(&conn->stats.open_files_count); + } + up_read(&conn_list_lock); + + seq_printf(m, "clients:\t%u\n", clients); + seq_printf(m, "open_files:\t%u\n", open_files); +} + +static int proc_show_ksmbd_stats(struct seq_file *m, void *v) +{ + int i; + + seq_puts(m, "Server\n"); + seq_printf(m, "state:\t%s\n", ksmbd_server_state_string()); + seq_printf(m, "name:\t%s\n", ksmbd_server_string()); + seq_printf(m, "netbios:\t%s\n", ksmbd_netbios_name()); + seq_printf(m, "work_group:\t%s\n", ksmbd_work_group()); + seq_printf(m, "min_protocol:\t%s\n", ksmbd_get_protocol_string(server_conf.min_protocol)); + seq_printf(m, "max_protocol:\t%s\n", ksmbd_get_protocol_string(server_conf.max_protocol)); + seq_printf(m, "flags:\t0x%08x\n", server_conf.flags); + seq_printf(m, "tcp_port:\t%u\n", server_conf.tcp_port); + seq_printf(m, "signing:\t%s\n", ksmbd_signing_mode_string()); + seq_printf(m, "signing_enforced:\t%s\n", + server_conf.enforced_signing ? "yes" : "no"); + seq_printf(m, "bind_interfaces_only:\t%s\n", + server_conf.bind_interfaces_only ? "yes" : "no"); + seq_printf(m, "max_connections:\t%u\n", server_conf.max_connections); + seq_printf(m, "max_connections_per_ip:\t%u\n", + server_conf.max_ip_connections); + seq_printf(m, "max_inflight_requests:\t%u\n", + server_conf.max_inflight_req); + seq_printf(m, "deadtime_seconds:\t%lu\n", server_conf.deadtime / HZ); + seq_printf(m, "ipc_timeout_seconds:\t%u\n", server_conf.ipc_timeout / HZ); + if (server_conf.ipc_last_active) + seq_printf(m, "ipc_last_active_seconds:\t%lu\n", + jiffies_to_msecs(jiffies - server_conf.ipc_last_active) / + MSEC_PER_SEC); + else + seq_puts(m, "ipc_last_active_seconds:\tnever\n"); + seq_printf(m, "durable_scavenger:\t%s\n", + ksmbd_durable_scavenger_active() ? "running" : "stopped"); + seq_printf(m, "share_fake_fscaps:\t0x%08x\n", + server_conf.share_fake_fscaps); + proc_show_runtime_totals(m); + seq_printf(m, "sessions:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_SESSIONS)); + seq_printf(m, "tree_connects:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_TREE_CONNS)); + seq_printf(m, "requests:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_REQUESTS)); + seq_printf(m, "read_bytes:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_READ_BYTES)); + seq_printf(m, "written_bytes:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_WRITE_BYTES)); + + seq_puts(m, "\nSMB2\n"); + for (i = 0; i < KSMBD_COUNTER_MAX_REQS; i++) + seq_printf(m, "%s:\t%lld\n", smb2_process_req[i].name, + ksmbd_counter_sum(KSMBD_COUNTER_FIRST_REQ + i)); + + seq_puts(m, "\nSMB2 status\n"); + seq_printf(m, "success:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_STATUS_SUCCESS)); + seq_printf(m, "informational:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_STATUS_INFORMATIONAL)); + seq_printf(m, "warning:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_STATUS_WARNING)); + seq_printf(m, "error:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_STATUS_ERROR)); + seq_printf(m, "access_denied:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_ERROR_ACCESS_DENIED)); + seq_printf(m, "not_found:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_ERROR_NOT_FOUND)); + seq_printf(m, "invalid_parameter:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_ERROR_INVALID_PARAMETER)); + seq_printf(m, "sharing_violation:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_ERROR_SHARING_VIOLATION)); + seq_printf(m, "not_supported:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_ERROR_NOT_SUPPORTED)); + seq_printf(m, "other:\t%lld\n", + ksmbd_counter_sum(KSMBD_COUNTER_ERROR_OTHER)); + return 0; +} + +void ksmbd_proc_cleanup(void) +{ + int i; + + if (!ksmbd_proc_fs) + return; + + proc_remove(ksmbd_proc_fs); + + for (i = 0; i < ARRAY_SIZE(ksmbd_counters.counters); i++) + percpu_counter_destroy(&ksmbd_counters.counters[i]); + + ksmbd_proc_fs = NULL; +} + +void ksmbd_proc_reset(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(ksmbd_counters.counters); i++) + percpu_counter_set(&ksmbd_counters.counters[i], 0); +} + +int ksmbd_proc_init(void) +{ + int i; + int retval = -ENOMEM; + + ksmbd_proc_fs = proc_mkdir("fs/ksmbd", NULL); + if (!ksmbd_proc_fs) + return retval; + + if (!proc_mkdir_mode("sessions", 0400, ksmbd_proc_fs)) + goto err_out; + + for (i = 0; i < ARRAY_SIZE(ksmbd_counters.counters); i++) { + retval = percpu_counter_init(&ksmbd_counters.counters[i], 0, GFP_KERNEL); + if (retval) + goto err_out; + } + + if (!ksmbd_proc_create("server", proc_show_ksmbd_stats, NULL)) { + retval = -ENOMEM; + goto err_out; + } + + ksmbd_proc_reset(); + return 0; +err_out: + ksmbd_proc_cleanup(); + return retval; +} diff --git a/fs/smb/server/server.c b/fs/smb/server/server.c new file mode 100644 index 000000000..0069d4e6a --- /dev/null +++ b/fs/smb/server/server.c @@ -0,0 +1,760 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2016 Namjae Jeon + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#include "glob.h" +#include "oplock.h" +#include "misc.h" +#include +#include +#include +#include +#include + +#include "server.h" +#include "smb_common.h" +#include "smb2pdu.h" +#include "../common/smb2status.h" +#include "connection.h" +#include "transport_ipc.h" +#include "mgmt/user_session.h" +#include "crypto_ctx.h" +#include "auth.h" +#include "stats.h" +#include "compress.h" +#include "mgmt/share_config.h" +#include "mgmt/tree_connect.h" + +int ksmbd_debug_types; + +struct ksmbd_server_config server_conf; + +enum SERVER_CTRL_TYPE { + SERVER_CTRL_TYPE_INIT, + SERVER_CTRL_TYPE_RESET, +}; + +struct server_ctrl_struct { + int type; + struct work_struct ctrl_work; +}; + +static DEFINE_MUTEX(ctrl_lock); + +static int ___server_conf_set(int idx, char *val) +{ + if (idx >= ARRAY_SIZE(server_conf.conf)) + return -EINVAL; + + if (!val || val[0] == 0x00) + return -EINVAL; + + kfree(server_conf.conf[idx]); + server_conf.conf[idx] = kstrdup(val, KSMBD_DEFAULT_GFP); + if (!server_conf.conf[idx]) + return -ENOMEM; + return 0; +} + +int ksmbd_set_netbios_name(char *v) +{ + return ___server_conf_set(SERVER_CONF_NETBIOS_NAME, v); +} + +int ksmbd_set_server_string(char *v) +{ + return ___server_conf_set(SERVER_CONF_SERVER_STRING, v); +} + +int ksmbd_set_work_group(char *v) +{ + return ___server_conf_set(SERVER_CONF_WORK_GROUP, v); +} + +char *ksmbd_netbios_name(void) +{ + return server_conf.conf[SERVER_CONF_NETBIOS_NAME]; +} + +char *ksmbd_server_string(void) +{ + return server_conf.conf[SERVER_CONF_SERVER_STRING]; +} + +char *ksmbd_work_group(void) +{ + return server_conf.conf[SERVER_CONF_WORK_GROUP]; +} + +/** + * check_conn_state() - check state of server thread connection + * @work: smb work containing server thread information + * + * Return: 0 on valid connection, otherwise 1 to reconnect + */ +static inline int check_conn_state(struct ksmbd_work *work) +{ + struct smb_hdr *rsp_hdr; + + if (ksmbd_conn_exiting(work->conn) || + ksmbd_conn_need_reconnect(work->conn)) { + rsp_hdr = smb_get_msg(work->response_buf); + rsp_hdr->Status.CifsError = STATUS_CONNECTION_DISCONNECTED; + return 1; + } + return 0; +} + +#define SERVER_HANDLER_CONTINUE 0 +#define SERVER_HANDLER_ABORT 1 + +static int __process_request(struct ksmbd_work *work, struct ksmbd_conn *conn, + u16 *cmd) +{ + struct smb_version_cmds *cmds; + u16 command; + bool signed_req; + int ret; + + if (check_conn_state(work)) + return SERVER_HANDLER_CONTINUE; + + if (ksmbd_verify_smb_message(work)) { + conn->ops->set_rsp_status(work, STATUS_INVALID_PARAMETER); + return SERVER_HANDLER_ABORT; + } + + command = conn->ops->get_cmd_val(work); + *cmd = command; + +andx_again: + if (command >= conn->max_cmds) { + conn->ops->set_rsp_status(work, STATUS_INVALID_PARAMETER); + return SERVER_HANDLER_ABORT; + } + + cmds = &conn->cmds[command]; + if (!cmds->proc) { + ksmbd_debug(SMB, "*** not implemented yet cmd = %x\n", command); + conn->ops->set_rsp_status(work, STATUS_NOT_IMPLEMENTED); + return SERVER_HANDLER_ABORT; + } + + signed_req = conn->ops->is_sign_req && conn->ops->is_sign_req(work, command); + if (work->sess && work->sess->sign && !work->encrypted && + !signed_req) { + conn->ops->set_rsp_status(work, STATUS_ACCESS_DENIED); + return SERVER_HANDLER_ABORT; + } + + if (work->sess && signed_req) { + ret = conn->ops->check_sign_req(work); + if (!ret) { + conn->ops->set_rsp_status(work, STATUS_ACCESS_DENIED); + return SERVER_HANDLER_ABORT; + } + } + + ret = cmds->proc(work); + if (conn->ops->inc_reqs) { + struct smb2_hdr *rsp = ksmbd_resp_buf_curr(work); + + conn->ops->inc_reqs(command, rsp->Status); + } + + if (ret < 0) + ksmbd_debug(CONN, "Failed to process %u [%d]\n", command, ret); + /* AndX commands - chained request can return positive values */ + else if (ret > 0) { + command = ret; + *cmd = command; + goto andx_again; + } + + if (work->send_no_response) + return SERVER_HANDLER_ABORT; + return SERVER_HANDLER_CONTINUE; +} + +static void __handle_ksmbd_work(struct ksmbd_work *work, + struct ksmbd_conn *conn) +{ + u16 command = 0; + int rc; + bool is_chained = false; + + if (conn->ops->is_transform_hdr && + conn->ops->is_transform_hdr(work->request_buf)) { + rc = conn->ops->decrypt_req(work); + if (rc < 0) { + ksmbd_conn_abort(conn); + return; + } + work->encrypted = true; + + /* + * SMB3 applies compression before encryption. The receive loop + * handles a plain compression transform before allocating work, but + * an encrypted request exposes that transform only after decryption. + */ + if (((struct smb2_hdr *)smb_get_msg(work->request_buf))->ProtocolId == + SMB2_COMPRESSION_TRANSFORM_ID) { + rc = ksmbd_decompress_work_request(work); + if (rc < 0) { + ksmbd_conn_abort(conn); + return; + } + } + + /* The decrypted payload must now be a complete SMB2 request. */ + if (((struct smb2_hdr *)smb_get_msg(work->request_buf))->ProtocolId != + SMB2_PROTO_NUMBER || + get_rfc1002_len(work->request_buf) < sizeof(struct smb2_pdu)) { + ksmbd_conn_abort(conn); + return; + } + } + + if (conn->ops->allocate_rsp_buf(work)) + return; + + rc = conn->ops->init_rsp_hdr(work); + if (rc) { + /* either uid or tid is not correct */ + conn->ops->set_rsp_status(work, STATUS_INVALID_HANDLE); + goto send; + } + + do { + if (conn->ops->check_user_session) { + rc = conn->ops->check_user_session(work); + if (rc < 0) { + if (rc == -EINVAL) + conn->ops->set_rsp_status(work, + STATUS_INVALID_PARAMETER); + else if (rc == -EKEYEXPIRED) + conn->ops->set_rsp_status(work, + STATUS_NETWORK_SESSION_EXPIRED); + else + conn->ops->set_rsp_status(work, + STATUS_USER_SESSION_DELETED); + if (conn->ops->is_sign_req(work, conn->ops->get_cmd_val(work))) { + struct smb2_hdr *rsp_hdr; + + rsp_hdr = ksmbd_resp_buf_curr(work); + if (rc == -EKEYEXPIRED && work->sess && + conn->ops->set_sign_rsp) + conn->ops->set_sign_rsp(work); + else + rsp_hdr->Flags |= SMB2_FLAGS_SIGNED; + } + goto send; + } else if (rc > 0) { + rc = conn->ops->get_ksmbd_tcon(work); + if (rc < 0) { + if (rc == -EINVAL) + conn->ops->set_rsp_status(work, + STATUS_INVALID_PARAMETER); + else + conn->ops->set_rsp_status(work, + STATUS_NETWORK_NAME_DELETED); + goto send; + } + + if (work->tcon && + test_share_config_flag(work->tcon->share_conf, + KSMBD_SHARE_FLAG_ENCRYPT_DATA) && + !work->encrypted) { + conn->ops->set_rsp_status(work, + STATUS_ACCESS_DENIED); + goto send; + } + } + } + + rc = __process_request(work, conn, &command); + if (rc == SERVER_HANDLER_ABORT) { + smb2_complete_request_open(work); + break; + } + + /* + * Call smb2_set_rsp_credits() function to set number of credits + * granted in hdr of smb2 response. + */ + if (conn->ops->set_rsp_credits) { + spin_lock(&conn->credits_lock); + rc = conn->ops->set_rsp_credits(work); + spin_unlock(&conn->credits_lock); + if (rc < 0) { + conn->ops->set_rsp_status(work, + STATUS_INVALID_PARAMETER); + smb2_complete_request_open(work); + goto send; + } + } + + smb2_complete_request_open(work); + + is_chained = is_chained_smb2_message(work); + + if (work->sess && + (work->sess->sign || smb3_11_final_sess_setup_resp(work) || + conn->ops->is_sign_req(work, command))) { + if (command == SMB2_SESSION_SETUP_HE && + work->sess->dialect >= SMB30_PROT_ID && + conn->dialect < SMB30_PROT_ID) + smb3_set_sign_rsp(work); + else + conn->ops->set_sign_rsp(work); + } + } while (is_chained == true); + +send: + smb2_complete_request_open(work); + /* + * Release any credit charge still outstanding for this request. On + * the normal path smb2_set_rsp_credits() already returned it, but the + * abort, error and send-no-response paths skip that call, so the + * charge would otherwise leak and eventually exhaust the connection's + * outstanding credit window. + */ + if (work->credit_charge) { + spin_lock(&conn->credits_lock); + conn->outstanding_credits -= work->credit_charge; + work->credit_charge = 0; + spin_unlock(&conn->credits_lock); + } + + if (work->tcon) + ksmbd_tree_connect_put(work->tcon); + smb3_preauth_hash_rsp(work); + /* + * Preauthentication hashes cover the original SMB2 response. Apply the + * transport compression wrapper only after updating the hash. + */ + if (work->compress_response) { + rc = ksmbd_compress_response(work); + if (rc < 0) + ksmbd_debug(CONN, "Failed to compress response: %d\n", rc); + } + if (work->sess && work->sess->enc && work->encrypted && + conn->ops->encrypt_resp) { + rc = conn->ops->encrypt_resp(work); + if (rc < 0) + conn->ops->set_rsp_status(work, STATUS_DATA_ERROR); + } + if (work->sess) + ksmbd_user_session_put(work->sess); + + ksmbd_conn_write(work); +} + +/** + * handle_ksmbd_work() - process pending smb work requests + * @wk: smb work containing request command buffer + * + * called by kworker threads to processing remaining smb work requests + */ +static void handle_ksmbd_work(struct work_struct *wk) +{ + struct ksmbd_work *work = container_of(wk, struct ksmbd_work, work); + struct ksmbd_conn *conn = work->conn; + + atomic64_inc(&conn->stats.request_served); + + __handle_ksmbd_work(work, conn); + + ksmbd_conn_try_dequeue_request(work); + ksmbd_free_work_struct(work); + ksmbd_conn_r_count_dec(conn); +} + +/** + * queue_ksmbd_work() - queue a smb request to worker thread queue + * for processing smb command and sending response + * @conn: connection instance + * + * read remaining data from socket create and submit work. + */ +static int queue_ksmbd_work(struct ksmbd_conn *conn) +{ + struct ksmbd_work *work; + int err; + + err = ksmbd_init_smb_server(conn); + if (err) + return 0; + + work = ksmbd_alloc_work_struct(); + if (!work) { + pr_err("allocation for work failed\n"); + return -ENOMEM; + } + + work->conn = conn; + work->request_buf = conn->request_buf; + conn->request_buf = NULL; + + ksmbd_conn_enqueue_request(work); + ksmbd_conn_r_count_inc(conn); + /* update activity on connection */ + conn->last_active = jiffies; + INIT_WORK(&work->work, handle_ksmbd_work); + ksmbd_queue_work(work); + return 0; +} + +static int ksmbd_server_process_request(struct ksmbd_conn *conn) +{ + return queue_ksmbd_work(conn); +} + +static int ksmbd_server_terminate_conn(struct ksmbd_conn *conn) +{ + ksmbd_sessions_deregister(conn); + destroy_lease_table(conn); + return 0; +} + +static void ksmbd_server_tcp_callbacks_init(void) +{ + struct ksmbd_conn_ops ops; + + ops.process_fn = ksmbd_server_process_request; + ops.terminate_fn = ksmbd_server_terminate_conn; + + ksmbd_conn_init_server_callbacks(&ops); +} + +static void server_conf_free(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(server_conf.conf); i++) { + kfree(server_conf.conf[i]); + server_conf.conf[i] = NULL; + } +} + +static int server_conf_init(void) +{ + WRITE_ONCE(server_conf.state, SERVER_STATE_STARTING_UP); + server_conf.enforced_signing = 0; + server_conf.min_protocol = ksmbd_min_protocol(); + server_conf.max_protocol = ksmbd_max_protocol(); + server_conf.auth_mechs = KSMBD_AUTH_NTLMSSP; +#ifdef CONFIG_SMB_SERVER_KERBEROS5 + server_conf.auth_mechs |= KSMBD_AUTH_KRB5 | + KSMBD_AUTH_MSKRB5; +#endif + server_conf.max_inflight_req = SMB2_MAX_CREDITS; + return 0; +} + +static void server_ctrl_handle_init(struct server_ctrl_struct *ctrl) +{ + int ret; + + ksmbd_proc_reset(); + ret = ksmbd_conn_transport_init(); + if (ret) { + server_queue_ctrl_reset_work(); + return; + } + + pr_info("running\n"); + WRITE_ONCE(server_conf.state, SERVER_STATE_RUNNING); +} + +static void server_ctrl_handle_reset(struct server_ctrl_struct *ctrl) +{ + ksmbd_ipc_soft_reset(); + ksmbd_conn_transport_destroy(); + ksmbd_stop_durable_scavenger(); + server_conf_free(); + server_conf_init(); + WRITE_ONCE(server_conf.state, SERVER_STATE_STARTING_UP); +} + +static void server_ctrl_handle_work(struct work_struct *work) +{ + struct server_ctrl_struct *ctrl; + + ctrl = container_of(work, struct server_ctrl_struct, ctrl_work); + + mutex_lock(&ctrl_lock); + switch (ctrl->type) { + case SERVER_CTRL_TYPE_INIT: + server_ctrl_handle_init(ctrl); + break; + case SERVER_CTRL_TYPE_RESET: + server_ctrl_handle_reset(ctrl); + break; + default: + pr_err("Unknown server work type: %d\n", ctrl->type); + } + mutex_unlock(&ctrl_lock); + kfree(ctrl); + module_put(THIS_MODULE); +} + +static int __queue_ctrl_work(int type) +{ + struct server_ctrl_struct *ctrl; + + ctrl = kmalloc_obj(struct server_ctrl_struct, KSMBD_DEFAULT_GFP); + if (!ctrl) + return -ENOMEM; + + __module_get(THIS_MODULE); + ctrl->type = type; + INIT_WORK(&ctrl->ctrl_work, server_ctrl_handle_work); + queue_work(system_long_wq, &ctrl->ctrl_work); + return 0; +} + +int server_queue_ctrl_init_work(void) +{ + return __queue_ctrl_work(SERVER_CTRL_TYPE_INIT); +} + +int server_queue_ctrl_reset_work(void) +{ + return __queue_ctrl_work(SERVER_CTRL_TYPE_RESET); +} + +static ssize_t stats_show(const struct class *class, const struct class_attribute *attr, + char *buf) +{ + /* + * Inc this each time you change stats output format, + * so user space will know what to do. + */ + static int stats_version = 2; + static const char * const state[] = { + "startup", + "running", + "reset", + "shutdown" + }; + return sysfs_emit(buf, "%d %s %d %lu\n", stats_version, + state[server_conf.state], server_conf.tcp_port, + server_conf.ipc_last_active / HZ); +} + +static ssize_t kill_server_store(const struct class *class, + const struct class_attribute *attr, const char *buf, + size_t len) +{ + if (!sysfs_streq(buf, "hard")) + return len; + + pr_info("kill command received\n"); + mutex_lock(&ctrl_lock); + WRITE_ONCE(server_conf.state, SERVER_STATE_RESETTING); + __module_get(THIS_MODULE); + server_ctrl_handle_reset(NULL); + module_put(THIS_MODULE); + mutex_unlock(&ctrl_lock); + return len; +} + +static const char * const debug_type_strings[] = {"smb", "auth", "vfs", + "oplock", "ipc", "conn", + "rdma"}; + +static ssize_t debug_show(const struct class *class, const struct class_attribute *attr, + char *buf) +{ + ssize_t sz = 0; + int i, pos = 0; + + for (i = 0; i < ARRAY_SIZE(debug_type_strings); i++) { + if ((ksmbd_debug_types >> i) & 1) { + pos = sysfs_emit_at(buf, sz, "[%s] ", debug_type_strings[i]); + } else { + pos = sysfs_emit_at(buf, sz, "%s ", debug_type_strings[i]); + } + sz += pos; + } + sz += sysfs_emit_at(buf, sz, "\n"); + return sz; +} + +static ssize_t debug_store(const struct class *class, const struct class_attribute *attr, + const char *buf, size_t len) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(debug_type_strings); i++) { + if (sysfs_streq(buf, "all")) { + if (ksmbd_debug_types == KSMBD_DEBUG_ALL) + ksmbd_debug_types = 0; + else + ksmbd_debug_types = KSMBD_DEBUG_ALL; + break; + } + + if (sysfs_streq(buf, debug_type_strings[i])) { + if (ksmbd_debug_types & (1 << i)) + ksmbd_debug_types &= ~(1 << i); + else + ksmbd_debug_types |= (1 << i); + break; + } + } + + return len; +} + +static CLASS_ATTR_RO(stats); +static CLASS_ATTR_WO(kill_server); +static CLASS_ATTR_RW(debug); + +static struct attribute *ksmbd_control_class_attrs[] = { + &class_attr_stats.attr, + &class_attr_kill_server.attr, + &class_attr_debug.attr, + NULL, +}; +ATTRIBUTE_GROUPS(ksmbd_control_class); + +static struct class ksmbd_control_class = { + .name = "ksmbd-control", + .class_groups = ksmbd_control_class_groups, +}; + +static int ksmbd_server_shutdown(void) +{ + WRITE_ONCE(server_conf.state, SERVER_STATE_SHUTTING_DOWN); + + class_unregister(&ksmbd_control_class); + ksmbd_workqueue_destroy(); + ksmbd_ipc_release(); + ksmbd_conn_transport_destroy(); + /* + * ksmbd_conn_transport_destroy() calls delete_proc_clients() and destroys + * sessions. ksmbd_session_destroy() removes each session's proc entry. + * Keep the procfs tree alive until these entries have been removed. + */ + ksmbd_proc_cleanup(); + ksmbd_crypto_destroy(); + ksmbd_free_global_file_table(); + destroy_lease_table(NULL); + ksmbd_work_pool_destroy(); + ksmbd_exit_file_cache(); + server_conf_free(); + return 0; +} + +static int __init ksmbd_server_init(void) +{ + int ret; + + ret = class_register(&ksmbd_control_class); + if (ret) { + pr_err("Unable to register ksmbd-control class\n"); + return ret; + } + + ret = ksmbd_proc_init(); + if (ret) + goto err_unregister; + + if (create_proc_sessions()) + pr_warn("Unable to create sessions procfs entry\n"); + + if (create_proc_shares()) + pr_warn("Unable to create shares procfs entry\n"); + + ksmbd_server_tcp_callbacks_init(); + + ret = server_conf_init(); + if (ret) + goto err_proc_cleanup; + + ret = ksmbd_work_pool_init(); + if (ret) + goto err_proc_cleanup; + + ret = ksmbd_init_file_cache(); + if (ret) + goto err_destroy_work_pools; + + ret = ksmbd_ipc_init(); + if (ret) + goto err_exit_file_cache; + + ret = ksmbd_init_global_file_table(); + if (ret) + goto err_ipc_release; + + ret = ksmbd_inode_hash_init(); + if (ret) + goto err_destroy_file_table; + + ret = ksmbd_crypto_create(); + if (ret) + goto err_release_inode_hash; + + ret = ksmbd_workqueue_init(); + if (ret) + goto err_crypto_destroy; + + ret = ksmbd_conn_wq_init(); + if (ret) + goto err_workqueue_destroy; + + return 0; + +err_workqueue_destroy: + ksmbd_workqueue_destroy(); +err_crypto_destroy: + ksmbd_crypto_destroy(); +err_release_inode_hash: + ksmbd_release_inode_hash(); +err_destroy_file_table: + ksmbd_free_global_file_table(); +err_ipc_release: + ksmbd_ipc_release(); +err_exit_file_cache: + ksmbd_exit_file_cache(); +err_destroy_work_pools: + ksmbd_work_pool_destroy(); +err_proc_cleanup: + ksmbd_proc_cleanup(); +err_unregister: + class_unregister(&ksmbd_control_class); + + return ret; +} + +/** + * ksmbd_server_exit() - shutdown forker thread and free memory at module exit + */ +static void __exit ksmbd_server_exit(void) +{ + ksmbd_server_shutdown(); + rcu_barrier(); + /* + * ksmbd_conn_put() defers the final release onto ksmbd_conn_wq, + * so drain it after rcu_barrier() has fired any pending RCU + * callbacks that may have queued a release. + */ + ksmbd_conn_wq_destroy(); + ksmbd_release_inode_hash(); +} + +MODULE_AUTHOR("Namjae Jeon "); +MODULE_DESCRIPTION("Linux kernel CIFS/SMB SERVER"); +MODULE_LICENSE("GPL"); +MODULE_SOFTDEP("pre: nls"); +MODULE_SOFTDEP("pre: aes"); +MODULE_SOFTDEP("pre: aead2"); +MODULE_SOFTDEP("pre: ccm"); +MODULE_SOFTDEP("pre: gcm"); +module_init(ksmbd_server_init) +module_exit(ksmbd_server_exit) diff --git a/fs/smb/server/server.h b/fs/smb/server/server.h new file mode 100644 index 000000000..4d4d268b5 --- /dev/null +++ b/fs/smb/server/server.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#ifndef __SERVER_H__ +#define __SERVER_H__ + +#include "smbacl.h" + +/* + * Server state type + */ +enum { + SERVER_STATE_STARTING_UP, + SERVER_STATE_RUNNING, + SERVER_STATE_RESETTING, + SERVER_STATE_SHUTTING_DOWN, +}; + +/* + * Server global config string index + */ +enum { + SERVER_CONF_NETBIOS_NAME, + SERVER_CONF_SERVER_STRING, + SERVER_CONF_WORK_GROUP, +}; + +struct ksmbd_server_config { + unsigned int flags; + unsigned int state; + short signing; + short enforced_signing; + short min_protocol; + short max_protocol; + unsigned short tcp_port; + unsigned short ipc_timeout; + unsigned long ipc_last_active; + unsigned long deadtime; + unsigned int share_fake_fscaps; + struct smb_sid domain_sid; + unsigned int auth_mechs; + unsigned int max_connections; + unsigned int max_inflight_req; + unsigned int max_ip_connections; + + char *conf[SERVER_CONF_WORK_GROUP + 1]; + struct task_struct *dh_task; + bool bind_interfaces_only; + /* AAPL model string for Finder icon, e.g. "Xserve" */ + char aapl_model[32]; +}; + +extern struct ksmbd_server_config server_conf; + +int ksmbd_set_netbios_name(char *v); +int ksmbd_set_server_string(char *v); +int ksmbd_set_work_group(char *v); + +char *ksmbd_netbios_name(void); +char *ksmbd_server_string(void); +char *ksmbd_work_group(void); + +static inline int ksmbd_server_running(void) +{ + return READ_ONCE(server_conf.state) == SERVER_STATE_RUNNING; +} + +static inline int ksmbd_server_configurable(void) +{ + return READ_ONCE(server_conf.state) < SERVER_STATE_RESETTING; +} + +int server_queue_ctrl_init_work(void); +int server_queue_ctrl_reset_work(void); +#endif /* __SERVER_H__ */ diff --git a/fs/smb/server/smb2misc.c b/fs/smb/server/smb2misc.c new file mode 100644 index 000000000..532dea7be --- /dev/null +++ b/fs/smb/server/smb2misc.c @@ -0,0 +1,564 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2016 Namjae Jeon + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#include "glob.h" +#include "smb_common.h" +#include "../common/smb2status.h" +#include "mgmt/user_session.h" +#include "connection.h" + +static int check_smb2_hdr(struct smb2_hdr *hdr) +{ + /* + * Make sure that this really is an SMB, that it is a response. + */ + if (hdr->Flags & SMB2_FLAGS_SERVER_TO_REDIR) + return 1; + return 0; +} + +/* + * The following table defines the expected "StructureSize" of SMB2 requests + * in order by SMB2 command. This is similar to "wct" in SMB/CIFS requests. + * + * Note that commands are defined in smb2pdu.h in le16 but the array below is + * indexed by command in host byte order + */ +static const __le16 smb2_req_struct_sizes[NUMBER_OF_SMB2_COMMANDS] = { + /* SMB2_NEGOTIATE */ cpu_to_le16(36), + /* SMB2_SESSION_SETUP */ cpu_to_le16(25), + /* SMB2_LOGOFF */ cpu_to_le16(4), + /* SMB2_TREE_CONNECT */ cpu_to_le16(9), + /* SMB2_TREE_DISCONNECT */ cpu_to_le16(4), + /* SMB2_CREATE */ cpu_to_le16(57), + /* SMB2_CLOSE */ cpu_to_le16(24), + /* SMB2_FLUSH */ cpu_to_le16(24), + /* SMB2_READ */ cpu_to_le16(49), + /* SMB2_WRITE */ cpu_to_le16(49), + /* SMB2_LOCK */ cpu_to_le16(48), + /* SMB2_IOCTL */ cpu_to_le16(57), + /* SMB2_CANCEL */ cpu_to_le16(4), + /* SMB2_ECHO */ cpu_to_le16(4), + /* SMB2_QUERY_DIRECTORY */ cpu_to_le16(33), + /* SMB2_CHANGE_NOTIFY */ cpu_to_le16(32), + /* SMB2_QUERY_INFO */ cpu_to_le16(41), + /* SMB2_SET_INFO */ cpu_to_le16(33), + /* use 44 for lease break */ + /* SMB2_OPLOCK_BREAK */ cpu_to_le16(36) +}; + +/* + * The size of the variable area depends on the offset and length fields + * located in different fields for various SMB2 requests. SMB2 requests + * with no variable length info, show an offset of zero for the offset field. + */ +static const bool has_smb2_data_area[NUMBER_OF_SMB2_COMMANDS] = { + /* SMB2_NEGOTIATE */ true, + /* SMB2_SESSION_SETUP */ true, + /* SMB2_LOGOFF */ false, + /* SMB2_TREE_CONNECT */ true, + /* SMB2_TREE_DISCONNECT */ false, + /* SMB2_CREATE */ true, + /* SMB2_CLOSE */ false, + /* SMB2_FLUSH */ false, + /* SMB2_READ */ true, + /* SMB2_WRITE */ true, + /* SMB2_LOCK */ true, + /* SMB2_IOCTL */ true, + /* SMB2_CANCEL */ false, /* BB CHECK this not listed in documentation */ + /* SMB2_ECHO */ false, + /* SMB2_QUERY_DIRECTORY */ true, + /* SMB2_CHANGE_NOTIFY */ false, + /* SMB2_QUERY_INFO */ true, + /* SMB2_SET_INFO */ true, + /* SMB2_OPLOCK_BREAK */ false +}; + +/* + * Set length of the data area and the offset to arguments. + * if they are invalid, return error. + */ +static int smb2_get_data_area_len(unsigned int *off, unsigned int *len, + struct smb2_hdr *hdr) +{ + int ret = 0; + + *off = 0; + *len = 0; + + /* + * Following commands have data areas so we have to get the location + * of the data buffer offset and data buffer length for the particular + * command. + */ + switch (hdr->Command) { + case SMB2_SESSION_SETUP: + *off = le16_to_cpu(((struct smb2_sess_setup_req *)hdr)->SecurityBufferOffset); + *len = le16_to_cpu(((struct smb2_sess_setup_req *)hdr)->SecurityBufferLength); + break; + case SMB2_TREE_CONNECT: + *off = max_t(unsigned short int, + le16_to_cpu(((struct smb2_tree_connect_req *)hdr)->PathOffset), + offsetof(struct smb2_tree_connect_req, Buffer)); + *len = le16_to_cpu(((struct smb2_tree_connect_req *)hdr)->PathLength); + break; + case SMB2_CREATE: + { + unsigned short int name_off = + max_t(unsigned short int, + le16_to_cpu(((struct smb2_create_req *)hdr)->NameOffset), + offsetof(struct smb2_create_req, Buffer)); + unsigned short int name_len = + le16_to_cpu(((struct smb2_create_req *)hdr)->NameLength); + + if (((struct smb2_create_req *)hdr)->CreateContextsLength) { + *off = le32_to_cpu(((struct smb2_create_req *) + hdr)->CreateContextsOffset); + *len = le32_to_cpu(((struct smb2_create_req *) + hdr)->CreateContextsLength); + if (!name_len) + break; + + if (name_off + name_len < (u64)*off + *len) + break; + } + + *off = name_off; + *len = name_len; + break; + } + case SMB2_QUERY_INFO: + *off = max_t(unsigned int, + le16_to_cpu(((struct smb2_query_info_req *)hdr)->InputBufferOffset), + offsetof(struct smb2_query_info_req, Buffer)); + *len = le32_to_cpu(((struct smb2_query_info_req *)hdr)->InputBufferLength); + break; + case SMB2_SET_INFO: + *off = max_t(unsigned int, + le16_to_cpu(((struct smb2_set_info_req *)hdr)->BufferOffset), + offsetof(struct smb2_set_info_req, Buffer)); + *len = le32_to_cpu(((struct smb2_set_info_req *)hdr)->BufferLength); + break; + case SMB2_READ: + *off = le16_to_cpu(((struct smb2_read_req *)hdr)->ReadChannelInfoOffset); + *len = le16_to_cpu(((struct smb2_read_req *)hdr)->ReadChannelInfoLength); + break; + case SMB2_WRITE: + if (((struct smb2_write_req *)hdr)->DataOffset || + ((struct smb2_write_req *)hdr)->Length) { + *off = max_t(unsigned short int, + le16_to_cpu(((struct smb2_write_req *)hdr)->DataOffset), + offsetof(struct smb2_write_req, Buffer)); + *len = le32_to_cpu(((struct smb2_write_req *)hdr)->Length); + break; + } + + *off = le16_to_cpu(((struct smb2_write_req *)hdr)->WriteChannelInfoOffset); + *len = le16_to_cpu(((struct smb2_write_req *)hdr)->WriteChannelInfoLength); + break; + case SMB2_QUERY_DIRECTORY: + *off = max_t(unsigned short int, + le16_to_cpu(((struct smb2_query_directory_req *)hdr)->FileNameOffset), + offsetof(struct smb2_query_directory_req, Buffer)); + *len = le16_to_cpu(((struct smb2_query_directory_req *)hdr)->FileNameLength); + break; + case SMB2_LOCK: + { + unsigned short lock_count; + + lock_count = le16_to_cpu(((struct smb2_lock_req *)hdr)->LockCount); + if (lock_count > 0) { + *off = offsetof(struct smb2_lock_req, locks); + *len = sizeof(struct smb2_lock_element) * lock_count; + } + break; + } + case SMB2_IOCTL: + *off = max_t(unsigned int, + le32_to_cpu(((struct smb2_ioctl_req *)hdr)->InputOffset), + offsetof(struct smb2_ioctl_req, Buffer)); + *len = le32_to_cpu(((struct smb2_ioctl_req *)hdr)->InputCount); + break; + default: + ksmbd_debug(SMB, "no length check for command\n"); + break; + } + + if (*off > 4096) { + ksmbd_debug(SMB, "offset %d too large\n", *off); + ret = -EINVAL; + } else if ((u64)*off + *len > MAX_STREAM_PROT_LEN) { + ksmbd_debug(SMB, "Request is larger than maximum stream protocol length(%u): %llu\n", + MAX_STREAM_PROT_LEN, (u64)*off + *len); + ret = -EINVAL; + } + + return ret; +} + +/* + * Calculate the size of the SMB message based on the fixed header + * portion, the number of word parameters and the data portion of the message. + */ +static int smb2_calc_size(void *buf, unsigned int *len) +{ + struct smb2_pdu *pdu = (struct smb2_pdu *)buf; + struct smb2_hdr *hdr = &pdu->hdr; + unsigned int offset; /* the offset from the beginning of SMB to data area */ + unsigned int data_length; /* the length of the variable length data area */ + int ret; + + /* Structure Size has already been checked to make sure it is 64 */ + *len = le16_to_cpu(hdr->StructureSize); + + /* + * StructureSize2, ie length of fixed parameter area has already + * been checked to make sure it is the correct length. + */ + *len += le16_to_cpu(pdu->StructureSize2); + /* + * StructureSize2 of smb2_lock pdu is set to 48, indicating + * the size of smb2 lock request with single smb2_lock_element + * regardless of number of locks. Subtract single + * smb2_lock_element for correct buffer size check. + */ + if (hdr->Command == SMB2_LOCK) + *len -= sizeof(struct smb2_lock_element); + + if (has_smb2_data_area[le16_to_cpu(hdr->Command)] == false) + goto calc_size_exit; + + ret = smb2_get_data_area_len(&offset, &data_length, hdr); + if (ret) + return ret; + ksmbd_debug(SMB, "SMB2 data length %u offset %u\n", data_length, + offset); + + if (data_length > 0) { + /* + * Check to make sure that data area begins after fixed area, + * Note that last byte of the fixed area is part of data area + * for some commands, typically those with odd StructureSize, + * so we must add one to the calculation. + */ + if (offset + 1 < *len) { + ksmbd_debug(SMB, + "data area offset %d overlaps SMB2 header %u\n", + offset + 1, *len); + return -EINVAL; + } + + *len = offset + data_length; + } + +calc_size_exit: + ksmbd_debug(SMB, "SMB2 len %u\n", *len); + return 0; +} + +static inline int smb2_query_info_req_len(struct smb2_query_info_req *h) +{ + return le32_to_cpu(h->InputBufferLength); +} + +static inline int smb2_query_info_resp_len(struct smb2_query_info_req *h) +{ + return le32_to_cpu(h->OutputBufferLength); +} + +static inline int smb2_set_info_req_len(struct smb2_set_info_req *h) +{ + return le32_to_cpu(h->BufferLength); +} + +static inline int smb2_read_req_len(struct smb2_read_req *h) +{ + return le32_to_cpu(h->Length); +} + +static inline int smb2_write_req_len(struct smb2_write_req *h) +{ + return le32_to_cpu(h->Length); +} + +static inline int smb2_query_dir_req_len(struct smb2_query_directory_req *h) +{ + return le32_to_cpu(h->OutputBufferLength); +} + +static inline int smb2_ioctl_req_len(struct smb2_ioctl_req *h) +{ + return le32_to_cpu(h->InputCount) + + le32_to_cpu(h->OutputCount); +} + +static inline int smb2_ioctl_resp_len(struct smb2_ioctl_req *h) +{ + return le32_to_cpu(h->MaxInputResponse) + + le32_to_cpu(h->MaxOutputResponse); +} + +static int smb2_validate_credit_charge(struct ksmbd_work *work, + struct smb2_hdr *hdr) +{ + struct ksmbd_conn *conn = work->conn; + unsigned int req_len = 0, expect_resp_len = 0, calc_credit_num, max_len; + unsigned short credit_charge = le16_to_cpu(hdr->CreditCharge); + void *__hdr = hdr; + int ret = 0; + + switch (hdr->Command) { + case SMB2_QUERY_INFO: + req_len = smb2_query_info_req_len(__hdr); + expect_resp_len = smb2_query_info_resp_len(__hdr); + break; + case SMB2_SET_INFO: + req_len = smb2_set_info_req_len(__hdr); + break; + case SMB2_READ: + req_len = smb2_read_req_len(__hdr); + break; + case SMB2_WRITE: + req_len = smb2_write_req_len(__hdr); + break; + case SMB2_QUERY_DIRECTORY: + req_len = smb2_query_dir_req_len(__hdr); + break; + case SMB2_IOCTL: + req_len = smb2_ioctl_req_len(__hdr); + expect_resp_len = smb2_ioctl_resp_len(__hdr); + break; + case SMB2_CANCEL: + return 0; + default: + req_len = 1; + break; + } + + credit_charge = max_t(unsigned short, credit_charge, 1); + max_len = max_t(unsigned int, req_len, expect_resp_len); + calc_credit_num = DIV_ROUND_UP(max_len, SMB2_MAX_BUFFER_SIZE); + + if (credit_charge < calc_credit_num) { + ksmbd_debug(SMB, "Insufficient credit charge, given: %d, needed: %d\n", + credit_charge, calc_credit_num); + return 1; + } else if (credit_charge > conn->vals->max_credits) { + ksmbd_debug(SMB, "Too large credit charge: %d\n", credit_charge); + return 1; + } + + spin_lock(&conn->credits_lock); + if (credit_charge > conn->total_credits) { + ksmbd_debug(SMB, "Insufficient credits granted, given: %u, granted: %u\n", + credit_charge, conn->total_credits); + ret = 1; + } + + if ((u64)conn->outstanding_credits + credit_charge > conn->total_credits) { + ksmbd_debug(SMB, "Limits exceeding the maximum allowable outstanding requests, given : %u, pending : %u\n", + credit_charge, conn->outstanding_credits); + ret = 1; + } else { + conn->outstanding_credits += credit_charge; + work->credit_charge = credit_charge; + } + + spin_unlock(&conn->credits_lock); + + return ret; +} + +/* + * Verify that the sequence number(s) consumed by an incoming request fall + * within the connection's command sequence window and are not a replay, then + * remove them from the window. Returns 0 if the request + * may proceed, or 1 if it is invalid and the connection must be torn down. + */ +static int smb2_check_sequence_number(struct ksmbd_work *work, + struct smb2_hdr *hdr) +{ + struct ksmbd_conn *conn = work->conn; + u64 mid = le64_to_cpu(hdr->MessageId); + unsigned short charge; + u64 i; + int ret = 0; + + /* An SMB2 CANCEL consumes no sequence number. */ + if (hdr->Command == SMB2_CANCEL) + return 0; + + /* + * A multi-credit request consumes CreditCharge consecutive sequence + * numbers; every other request consumes exactly one. + */ + charge = le16_to_cpu(hdr->CreditCharge); + if (!(conn->vals->req_capabilities & SMB2_GLOBAL_CAP_LARGE_MTU) || + charge == 0) + charge = 1; + + /* The 64-bit sequence number space must not wrap. */ + if (mid + charge < mid) { + pr_err("SMB2 sequence number wrapped (mid %llu charge %u)\n", + mid, charge); + return 1; + } + + spin_lock(&conn->credits_lock); + + /* The whole range must lie within the granted window... */ + if (mid < conn->seq_low || mid + charge > conn->seq_high) { + ksmbd_debug(SMB, + "MessageId %llu (charge %u) outside command window [%llu, %llu)\n", + mid, charge, conn->seq_low, conn->seq_high); + ret = 1; + goto out; + } + + /* ...and none of it may have been consumed already (replay). */ + for (i = mid; i < mid + charge; i++) { + if (!test_bit(i & (KSMBD_CMD_SEQ_WINDOW - 1), conn->seq_bitmap)) { + ksmbd_debug(SMB, + "replayed sequence number %llu (mid %llu charge %u)\n", + i, mid, charge); + ret = 1; + goto out; + } + } + + /* Consume the sequence numbers and slide the low edge forward. */ + for (i = mid; i < mid + charge; i++) + __clear_bit(i & (KSMBD_CMD_SEQ_WINDOW - 1), conn->seq_bitmap); + while (conn->seq_low < conn->seq_high && + !test_bit(conn->seq_low & (KSMBD_CMD_SEQ_WINDOW - 1), + conn->seq_bitmap)) + conn->seq_low++; +out: + spin_unlock(&conn->credits_lock); + return ret; +} + +int ksmbd_smb2_check_message(struct ksmbd_work *work) +{ + struct smb2_pdu *pdu = ksmbd_req_buf_next(work); + struct smb2_hdr *hdr = &pdu->hdr; + int command; + __u32 clc_len; /* calculated length */ + __u32 len = get_rfc1002_len(work->request_buf); + __u32 req_struct_size, next_cmd = le32_to_cpu(hdr->NextCommand); + + if ((u64)work->next_smb2_rcv_hdr_off + next_cmd > len) { + pr_err("next command(%u) offset exceeds smb msg size\n", + next_cmd); + return 1; + } + + if (next_cmd > 0) + len = next_cmd; + else if (work->next_smb2_rcv_hdr_off) + len -= work->next_smb2_rcv_hdr_off; + + if (check_smb2_hdr(hdr)) + return 1; + + if (hdr->StructureSize != SMB2_HEADER_STRUCTURE_SIZE) { + ksmbd_debug(SMB, "Illegal structure size %u\n", + le16_to_cpu(hdr->StructureSize)); + return 1; + } + + command = le16_to_cpu(hdr->Command); + if (command >= NUMBER_OF_SMB2_COMMANDS) { + ksmbd_debug(SMB, "Illegal SMB2 command %d\n", command); + return 1; + } + + if (len < __SMB2_HEADER_STRUCTURE_SIZE + sizeof(__le16)) { + ksmbd_debug(SMB, "Message is too small for StructureSize2\n"); + return 1; + } + + if (smb2_req_struct_sizes[command] != pdu->StructureSize2) { + if (!(command == SMB2_OPLOCK_BREAK_HE && + (le16_to_cpu(pdu->StructureSize2) == OP_BREAK_STRUCT_SIZE_20 || + le16_to_cpu(pdu->StructureSize2) == OP_BREAK_STRUCT_SIZE_21))) { + /* special case for SMB2.1 lease break message */ + ksmbd_debug(SMB, + "Illegal request size %u for command %d\n", + le16_to_cpu(pdu->StructureSize2), command); + return 1; + } + } + + req_struct_size = le16_to_cpu(pdu->StructureSize2) + + __SMB2_HEADER_STRUCTURE_SIZE; + if (command == SMB2_LOCK_HE) + req_struct_size -= sizeof(struct smb2_lock_element); + + if (req_struct_size > len + 1) + return 1; + + if (smb2_calc_size(hdr, &clc_len)) + return 1; + + if (len != clc_len) { + /* client can return one byte more due to implied bcc[0] */ + if (clc_len == len + 1) + goto validate_credit; + + /* + * Some windows servers (win2016) will pad also the final + * PDU in a compound to 8 bytes. + */ + if (ALIGN(clc_len, 8) == len) + goto validate_credit; + + /* + * SMB2 NEGOTIATE request will be validated when message + * handling proceeds. + */ + if (command == SMB2_NEGOTIATE_HE) + goto validate_credit; + + /* + * Allow a message that padded to 8byte boundary. + * Linux 4.19.217 with smb 3.0.2 are sometimes + * sending messages where the cls_len is exactly + * 8 bytes less than len. + */ + if (clc_len < len && (len - clc_len) <= 8) + goto validate_credit; + + pr_err_ratelimited( + "cli req too short, len %d not %d. cmd:%d mid:%llu\n", + len, clc_len, command, + le64_to_cpu(hdr->MessageId)); + + return 1; + } + +validate_credit: + if ((work->conn->vals->req_capabilities & SMB2_GLOBAL_CAP_LARGE_MTU) && + smb2_validate_credit_charge(work, hdr)) + return 1; + + /* + * A sequence number violation (out of window or a replay) is a + * protocol error. tear the connection down rather than + * keep accepting requests on it. + */ + if (smb2_check_sequence_number(work, hdr)) { + ksmbd_conn_set_exiting(work->conn); + return 1; + } + + return 0; +} + +int smb2_negotiate_request(struct ksmbd_work *work) +{ + return ksmbd_smb_negotiate_common(work, SMB2_NEGOTIATE_HE); +} diff --git a/fs/smb/server/smb2ops.c b/fs/smb/server/smb2ops.c new file mode 100644 index 000000000..4578291fb --- /dev/null +++ b/fs/smb/server/smb2ops.c @@ -0,0 +1,343 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2016 Namjae Jeon + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#include +#include "glob.h" + +#include "auth.h" +#include "connection.h" +#include "smb_common.h" +#include "server.h" +#include "stats.h" + +static struct smb_version_values smb21_server_values = { + .version_string = SMB21_VERSION_STRING, + .protocol_id = SMB21_PROT_ID, + .req_capabilities = SMB2_GLOBAL_CAP_LARGE_MTU, + .max_read_size = SMB21_DEFAULT_IOSIZE, + .max_write_size = SMB21_DEFAULT_IOSIZE, + .max_trans_size = SMB21_DEFAULT_IOSIZE, + .max_credits = SMB2_MAX_CREDITS, + .large_lock_type = 0, + .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE, + .shared_lock_type = SMB2_LOCKFLAG_SHARED, + .unlock_lock_type = SMB2_LOCKFLAG_UNLOCK, + .header_size = sizeof(struct smb2_hdr), + .max_header_size = MAX_SMB2_HDR_SIZE, + .read_rsp_size = sizeof(struct smb2_read_rsp), + .lock_cmd = SMB2_LOCK, + .cap_unix = 0, + .cap_nt_find = SMB2_NT_FIND, + .cap_large_files = SMB2_LARGE_FILES, + .create_lease_size = sizeof(struct create_lease), + .create_durable_size = sizeof(struct create_durable_rsp), + .create_mxac_size = sizeof(struct create_mxac_rsp), + .create_disk_id_size = sizeof(struct create_disk_id_rsp), + .create_posix_size = sizeof(struct create_posix_rsp), + .create_aapl_size = AAPL_RSP_MAX_SIZE, +}; + +static struct smb_version_values smb30_server_values = { + .version_string = SMB30_VERSION_STRING, + .protocol_id = SMB30_PROT_ID, + .req_capabilities = SMB2_GLOBAL_CAP_LARGE_MTU, + .max_read_size = SMB3_DEFAULT_IOSIZE, + .max_write_size = SMB3_DEFAULT_IOSIZE, + .max_trans_size = SMB3_DEFAULT_TRANS_SIZE, + .max_credits = SMB2_MAX_CREDITS, + .large_lock_type = 0, + .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE, + .shared_lock_type = SMB2_LOCKFLAG_SHARED, + .unlock_lock_type = SMB2_LOCKFLAG_UNLOCK, + .header_size = sizeof(struct smb2_hdr), + .max_header_size = MAX_SMB2_HDR_SIZE, + .read_rsp_size = sizeof(struct smb2_read_rsp), + .lock_cmd = SMB2_LOCK, + .cap_unix = 0, + .cap_nt_find = SMB2_NT_FIND, + .cap_large_files = SMB2_LARGE_FILES, + .create_lease_size = sizeof(struct create_lease_v2), + .create_durable_size = sizeof(struct create_durable_rsp), + .create_durable_v2_size = sizeof(struct create_durable_rsp_v2), + .create_mxac_size = sizeof(struct create_mxac_rsp), + .create_disk_id_size = sizeof(struct create_disk_id_rsp), + .create_posix_size = sizeof(struct create_posix_rsp), + .create_aapl_size = AAPL_RSP_MAX_SIZE, +}; + +static struct smb_version_values smb302_server_values = { + .version_string = SMB302_VERSION_STRING, + .protocol_id = SMB302_PROT_ID, + .req_capabilities = SMB2_GLOBAL_CAP_LARGE_MTU, + .max_read_size = SMB3_DEFAULT_IOSIZE, + .max_write_size = SMB3_DEFAULT_IOSIZE, + .max_trans_size = SMB3_DEFAULT_TRANS_SIZE, + .max_credits = SMB2_MAX_CREDITS, + .large_lock_type = 0, + .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE, + .shared_lock_type = SMB2_LOCKFLAG_SHARED, + .unlock_lock_type = SMB2_LOCKFLAG_UNLOCK, + .header_size = sizeof(struct smb2_hdr), + .max_header_size = MAX_SMB2_HDR_SIZE, + .read_rsp_size = sizeof(struct smb2_read_rsp), + .lock_cmd = SMB2_LOCK, + .cap_unix = 0, + .cap_nt_find = SMB2_NT_FIND, + .cap_large_files = SMB2_LARGE_FILES, + .create_lease_size = sizeof(struct create_lease_v2), + .create_durable_size = sizeof(struct create_durable_rsp), + .create_durable_v2_size = sizeof(struct create_durable_rsp_v2), + .create_mxac_size = sizeof(struct create_mxac_rsp), + .create_disk_id_size = sizeof(struct create_disk_id_rsp), + .create_posix_size = sizeof(struct create_posix_rsp), + .create_aapl_size = AAPL_RSP_MAX_SIZE, +}; + +static struct smb_version_values smb311_server_values = { + .version_string = SMB311_VERSION_STRING, + .protocol_id = SMB311_PROT_ID, + .req_capabilities = SMB2_GLOBAL_CAP_LARGE_MTU, + .max_read_size = SMB3_DEFAULT_IOSIZE, + .max_write_size = SMB3_DEFAULT_IOSIZE, + .max_trans_size = SMB3_DEFAULT_TRANS_SIZE, + .max_credits = SMB2_MAX_CREDITS, + .large_lock_type = 0, + .exclusive_lock_type = SMB2_LOCKFLAG_EXCLUSIVE, + .shared_lock_type = SMB2_LOCKFLAG_SHARED, + .unlock_lock_type = SMB2_LOCKFLAG_UNLOCK, + .header_size = sizeof(struct smb2_hdr), + .max_header_size = MAX_SMB2_HDR_SIZE, + .read_rsp_size = sizeof(struct smb2_read_rsp), + .lock_cmd = SMB2_LOCK, + .cap_unix = 0, + .cap_nt_find = SMB2_NT_FIND, + .cap_large_files = SMB2_LARGE_FILES, + .create_lease_size = sizeof(struct create_lease_v2), + .create_durable_size = sizeof(struct create_durable_rsp), + .create_durable_v2_size = sizeof(struct create_durable_rsp_v2), + .create_mxac_size = sizeof(struct create_mxac_rsp), + .create_disk_id_size = sizeof(struct create_disk_id_rsp), + .create_posix_size = sizeof(struct create_posix_rsp), + .create_aapl_size = AAPL_RSP_MAX_SIZE, +}; + +static struct smb_version_ops smb2_0_server_ops = { + .get_cmd_val = get_smb2_cmd_val, + .inc_reqs = ksmbd_counter_inc_reqs, + .init_rsp_hdr = init_smb2_rsp_hdr, + .set_rsp_status = set_smb2_rsp_status, + .allocate_rsp_buf = smb2_allocate_rsp_buf, + .set_rsp_credits = smb2_set_rsp_credits, + .check_user_session = smb2_check_user_session, + .get_ksmbd_tcon = smb2_get_ksmbd_tcon, + .is_sign_req = smb2_is_sign_req, + .check_sign_req = smb2_check_sign_req, + .set_sign_rsp = smb2_set_sign_rsp +}; + +static struct smb_version_ops smb3_0_server_ops = { + .get_cmd_val = get_smb2_cmd_val, + .inc_reqs = ksmbd_counter_inc_reqs, + .init_rsp_hdr = init_smb2_rsp_hdr, + .set_rsp_status = set_smb2_rsp_status, + .allocate_rsp_buf = smb2_allocate_rsp_buf, + .set_rsp_credits = smb2_set_rsp_credits, + .check_user_session = smb2_check_user_session, + .get_ksmbd_tcon = smb2_get_ksmbd_tcon, + .is_sign_req = smb2_is_sign_req, + .check_sign_req = smb3_check_sign_req, + .set_sign_rsp = smb3_set_sign_rsp, + .generate_signingkey = ksmbd_gen_smb30_signingkey, + .generate_encryptionkey = ksmbd_gen_smb30_encryptionkey, + .is_transform_hdr = smb3_is_transform_hdr, + .decrypt_req = smb3_decrypt_req, + .encrypt_resp = smb3_encrypt_resp +}; + +static struct smb_version_ops smb3_11_server_ops = { + .get_cmd_val = get_smb2_cmd_val, + .inc_reqs = ksmbd_counter_inc_reqs, + .init_rsp_hdr = init_smb2_rsp_hdr, + .set_rsp_status = set_smb2_rsp_status, + .allocate_rsp_buf = smb2_allocate_rsp_buf, + .set_rsp_credits = smb2_set_rsp_credits, + .check_user_session = smb2_check_user_session, + .get_ksmbd_tcon = smb2_get_ksmbd_tcon, + .is_sign_req = smb2_is_sign_req, + .check_sign_req = smb3_check_sign_req, + .set_sign_rsp = smb3_set_sign_rsp, + .generate_signingkey = ksmbd_gen_smb311_signingkey, + .generate_encryptionkey = ksmbd_gen_smb311_encryptionkey, + .is_transform_hdr = smb3_is_transform_hdr, + .decrypt_req = smb3_decrypt_req, + .encrypt_resp = smb3_encrypt_resp +}; + +static struct smb_version_cmds smb2_0_server_cmds[NUMBER_OF_SMB2_COMMANDS] = { + [SMB2_NEGOTIATE_HE] = { .proc = smb2_negotiate_request, }, + [SMB2_SESSION_SETUP_HE] = { .proc = smb2_sess_setup, }, + [SMB2_TREE_CONNECT_HE] = { .proc = smb2_tree_connect,}, + [SMB2_TREE_DISCONNECT_HE] = { .proc = smb2_tree_disconnect,}, + [SMB2_LOGOFF_HE] = { .proc = smb2_session_logoff,}, + [SMB2_CREATE_HE] = { .proc = smb2_open}, + [SMB2_QUERY_INFO_HE] = { .proc = smb2_query_info}, + [SMB2_QUERY_DIRECTORY_HE] = { .proc = smb2_query_dir}, + [SMB2_CLOSE_HE] = { .proc = smb2_close}, + [SMB2_ECHO_HE] = { .proc = smb2_echo}, + [SMB2_SET_INFO_HE] = { .proc = smb2_set_info}, + [SMB2_READ_HE] = { .proc = smb2_read}, + [SMB2_WRITE_HE] = { .proc = smb2_write}, + [SMB2_FLUSH_HE] = { .proc = smb2_flush}, + [SMB2_CANCEL_HE] = { .proc = smb2_cancel}, + [SMB2_LOCK_HE] = { .proc = smb2_lock}, + [SMB2_IOCTL_HE] = { .proc = smb2_ioctl}, + [SMB2_OPLOCK_BREAK_HE] = { .proc = smb2_oplock_break}, + [SMB2_CHANGE_NOTIFY_HE] = { .proc = smb2_notify}, +}; + +/** + * init_smb2_1_server() - initialize a smb server connection with smb2.1 + * command dispatcher + * @conn: connection instance + */ +void init_smb2_1_server(struct ksmbd_conn *conn) +{ + conn->vals = &smb21_server_values; + conn->ops = &smb2_0_server_ops; + conn->cmds = smb2_0_server_cmds; + conn->max_cmds = ARRAY_SIZE(smb2_0_server_cmds); + conn->signing_algorithm = SIGNING_ALG_HMAC_SHA256_LE; + + if (server_conf.flags & KSMBD_GLOBAL_FLAG_SMB2_LEASES) + conn->vals->req_capabilities |= SMB2_GLOBAL_CAP_LEASING; +} + +/** + * init_smb3_0_server() - initialize a smb server connection with smb3.0 + * command dispatcher + * @conn: connection instance + */ +void init_smb3_0_server(struct ksmbd_conn *conn) +{ + conn->vals = &smb30_server_values; + conn->ops = &smb3_0_server_ops; + conn->cmds = smb2_0_server_cmds; + conn->max_cmds = ARRAY_SIZE(smb2_0_server_cmds); + conn->signing_algorithm = SIGNING_ALG_AES_CMAC_LE; + + if (server_conf.flags & KSMBD_GLOBAL_FLAG_SMB2_LEASES) + conn->vals->req_capabilities |= SMB2_GLOBAL_CAP_LEASING | + SMB2_GLOBAL_CAP_DIRECTORY_LEASING; + + if (server_conf.flags & KSMBD_GLOBAL_FLAG_SMB2_ENCRYPTION && + conn->cli_cap & SMB2_GLOBAL_CAP_ENCRYPTION) + conn->vals->req_capabilities |= SMB2_GLOBAL_CAP_ENCRYPTION; + + if (server_conf.flags & KSMBD_GLOBAL_FLAG_SMB2_ENCRYPTION || + (!(server_conf.flags & KSMBD_GLOBAL_FLAG_SMB2_ENCRYPTION_OFF) && + conn->cli_cap & SMB2_GLOBAL_CAP_ENCRYPTION)) + conn->vals->req_capabilities |= SMB2_GLOBAL_CAP_ENCRYPTION; + + if (server_conf.flags & KSMBD_GLOBAL_FLAG_SMB3_MULTICHANNEL) + conn->vals->req_capabilities |= SMB2_GLOBAL_CAP_MULTI_CHANNEL; +} + +/** + * init_smb3_02_server() - initialize a smb server connection with smb3.02 + * command dispatcher + * @conn: connection instance + */ +void init_smb3_02_server(struct ksmbd_conn *conn) +{ + conn->vals = &smb302_server_values; + conn->ops = &smb3_0_server_ops; + conn->cmds = smb2_0_server_cmds; + conn->max_cmds = ARRAY_SIZE(smb2_0_server_cmds); + conn->signing_algorithm = SIGNING_ALG_AES_CMAC_LE; + + if (server_conf.flags & KSMBD_GLOBAL_FLAG_SMB2_LEASES) + conn->vals->req_capabilities |= SMB2_GLOBAL_CAP_LEASING | + SMB2_GLOBAL_CAP_DIRECTORY_LEASING; + + if (server_conf.flags & KSMBD_GLOBAL_FLAG_SMB2_ENCRYPTION || + (!(server_conf.flags & KSMBD_GLOBAL_FLAG_SMB2_ENCRYPTION_OFF) && + conn->cli_cap & SMB2_GLOBAL_CAP_ENCRYPTION)) + conn->vals->req_capabilities |= SMB2_GLOBAL_CAP_ENCRYPTION; + + if (server_conf.flags & KSMBD_GLOBAL_FLAG_SMB3_MULTICHANNEL) + conn->vals->req_capabilities |= SMB2_GLOBAL_CAP_MULTI_CHANNEL; + + /* + * Durable handles are in-memory only. Do not advertise persistent + * handles until CA recovery and fencing are implemented. + */ +} + +/** + * init_smb3_11_server() - initialize a smb server connection with smb3.11 + * command dispatcher + * @conn: connection instance + */ +int init_smb3_11_server(struct ksmbd_conn *conn) +{ + conn->vals = &smb311_server_values; + conn->ops = &smb3_11_server_ops; + conn->cmds = smb2_0_server_cmds; + conn->max_cmds = ARRAY_SIZE(smb2_0_server_cmds); + conn->signing_algorithm = SIGNING_ALG_AES_CMAC_LE; + + if (server_conf.flags & KSMBD_GLOBAL_FLAG_SMB2_LEASES) + conn->vals->req_capabilities |= SMB2_GLOBAL_CAP_LEASING | + SMB2_GLOBAL_CAP_DIRECTORY_LEASING; + + if (server_conf.flags & KSMBD_GLOBAL_FLAG_SMB3_MULTICHANNEL) + conn->vals->req_capabilities |= SMB2_GLOBAL_CAP_MULTI_CHANNEL; + + /* See init_smb3_02_server(): persistent handles require CA recovery. */ + return 0; +} + +void init_smb2_max_read_size(unsigned int sz) +{ + sz = clamp_val(sz, SMB3_MIN_IOSIZE, SMB3_MAX_IOSIZE); + smb21_server_values.max_read_size = sz; + smb30_server_values.max_read_size = sz; + smb302_server_values.max_read_size = sz; + smb311_server_values.max_read_size = sz; +} + +void init_smb2_max_write_size(unsigned int sz) +{ + sz = clamp_val(sz, SMB3_MIN_IOSIZE, SMB3_MAX_IOSIZE); + smb21_server_values.max_write_size = sz; + smb30_server_values.max_write_size = sz; + smb302_server_values.max_write_size = sz; + smb311_server_values.max_write_size = sz; +} + +void init_smb2_max_trans_size(unsigned int sz) +{ + sz = clamp_val(sz, SMB3_MIN_IOSIZE, SMB3_MAX_IOSIZE); + smb21_server_values.max_trans_size = sz; + smb30_server_values.max_trans_size = sz; + smb302_server_values.max_trans_size = sz; + smb311_server_values.max_trans_size = sz; +} + +void init_smb2_max_credits(unsigned int sz) +{ + /* + * The command sequence window (and its backing bitmap) can track at + * most SMB2_MAX_CREDITS outstanding sequence numbers, so the number of + * credits granted on a connection must not exceed that. + */ + if (sz > SMB2_MAX_CREDITS) + sz = SMB2_MAX_CREDITS; + smb21_server_values.max_credits = sz; + smb30_server_values.max_credits = sz; + smb302_server_values.max_credits = sz; + smb311_server_values.max_credits = sz; +} diff --git a/fs/smb/server/smb2pdu.c b/fs/smb/server/smb2pdu.c new file mode 100644 index 000000000..a8046f477 --- /dev/null +++ b/fs/smb/server/smb2pdu.c @@ -0,0 +1,12348 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2016 Namjae Jeon + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "glob.h" +#include "../common/smbfsctl.h" +#include "oplock.h" +#include "smbacl.h" + +#include "auth.h" +#include "asn1.h" +#include "connection.h" +#include "transport_ipc.h" +#include "transport_rdma.h" +#include "vfs.h" +#include "vfs_cache.h" +#include "misc.h" + +#include "server.h" +#include "smb_common.h" +#include "../common/smb2status.h" +#include "ksmbd_work.h" +#include "mgmt/user_config.h" +#include "mgmt/share_config.h" +#include "mgmt/tree_connect.h" +#include "mgmt/user_session.h" +#include "mgmt/ksmbd_ida.h" +#include "ndr.h" +#include "stats.h" +#include "transport_tcp.h" +#include "compress.h" + +static void __wbuf(struct ksmbd_work *work, void **req, void **rsp) +{ + if (work->next_smb2_rcv_hdr_off) { + *req = ksmbd_req_buf_next(work); + *rsp = ksmbd_resp_buf_next(work); + } else { + *req = smb_get_msg(work->request_buf); + *rsp = smb_get_msg(work->response_buf); + } +} + +static struct ksmbd_work *smb2_notify_cancel_claim(void **argv); +static void smb2_notify_cancel_fn(void **argv); +static void smb2_complete_notify_cancel(struct ksmbd_work *in_work); + +#define WORK_BUFFERS(w, rq, rs) __wbuf((w), (void **)&(rq), (void **)&(rs)) + +#define SMB2_CREATE_FILE_ATTRIBUTE_MASK \ + (FILE_ATTRIBUTE_MASK & ~(FILE_ATTRIBUTE_INTEGRITY_STREAM | \ + FILE_ATTRIBUTE_NO_SCRUB_DATA)) + +/* Windows reports automatic write-time updates at roughly 15 ms resolution. */ +#define KSMBD_WRITE_TIME_RESOLUTION (15ULL * 10000) + +/* MAXFILESIZE in [MS-FSA] 2.1.5.3 Server Requests a Write. */ +#define SMB2_MAX_FILE_SIZE 0xfffffff0000ULL + +struct channel *lookup_chann_list(struct ksmbd_session *sess, struct ksmbd_conn *conn) +{ + struct channel *chann; + + down_read(&sess->chann_lock); + chann = xa_load(&sess->ksmbd_chann_list, (long)conn); + up_read(&sess->chann_lock); + + return chann; +} + +#define KSMBD_MAX_CHANNELS 32 + +static int register_session_channel(struct ksmbd_session *sess, + struct ksmbd_conn *conn, + const char *sess_key) +{ + struct channel *chann, *old; + unsigned long index; + unsigned int count = 0; + int rc = 0; + + down_write(&sess->chann_lock); + if (xa_load(&sess->ksmbd_chann_list, (long)conn)) + goto out; + + xa_for_each(&sess->ksmbd_chann_list, index, chann) + count++; + if (count >= KSMBD_MAX_CHANNELS) { + rc = -ENOSPC; + goto out; + } + + chann = kmalloc_obj(struct channel, KSMBD_DEFAULT_GFP); + if (!chann) { + rc = -ENOMEM; + goto out; + } + + chann->conn = conn; + memcpy(chann->sess_key, sess_key, sizeof(chann->sess_key)); + old = xa_store(&sess->ksmbd_chann_list, (long)conn, chann, + KSMBD_DEFAULT_GFP); + if (xa_is_err(old)) { + kfree_sensitive(chann); + rc = xa_err(old); + } +out: + up_write(&sess->chann_lock); + return rc; +} + +/** + * smb2_get_ksmbd_tcon() - get tree connection information using a tree id. + * @work: smb work + * + * Return: 0 if there is a tree connection matched or these are + * skipable commands, otherwise error + */ +int smb2_get_ksmbd_tcon(struct ksmbd_work *work) +{ + struct smb2_hdr *req_hdr = ksmbd_req_buf_next(work); + unsigned int cmd = le16_to_cpu(req_hdr->Command); + unsigned int tree_id; + + if (cmd == SMB2_TREE_CONNECT_HE || + cmd == SMB2_CANCEL_HE || + cmd == SMB2_LOGOFF_HE) { + ksmbd_debug(SMB, "skip to check tree connect request\n"); + return 0; + } + + if (xa_empty(&work->sess->tree_conns)) { + ksmbd_debug(SMB, "NO tree connected\n"); + return -ENOENT; + } + + tree_id = le32_to_cpu(req_hdr->Id.SyncId.TreeId); + + /* + * If request is not the first in Compound request, + * Just validate tree id in header with work->tcon->id. + */ + if (work->next_smb2_rcv_hdr_off) { + if (!work->tcon) { + pr_err("The first operation in the compound does not have tcon\n"); + return -EINVAL; + } + if (work->tcon->t_state != TREE_CONNECTED) + return -ENOENT; + if (tree_id != UINT_MAX && work->tcon->id != tree_id) { + pr_err("tree id(%u) is different with id(%u) in first operation\n", + tree_id, work->tcon->id); + return -EINVAL; + } + return 1; + } + + work->tcon = ksmbd_tree_conn_lookup(work->sess, tree_id); + if (!work->tcon) { + pr_err("Invalid tid %d\n", tree_id); + return -ENOENT; + } + + return 1; +} + +/** + * smb2_set_err_rsp() - set error response code on smb response + * @work: smb work containing response buffer + */ +void smb2_set_err_rsp(struct ksmbd_work *work) +{ + struct smb2_err_rsp *err_rsp; + + if (work->next_smb2_rcv_hdr_off) + err_rsp = ksmbd_resp_buf_next(work); + else + err_rsp = smb_get_msg(work->response_buf); + + if (err_rsp->hdr.Status != STATUS_STOPPED_ON_SYMLINK) { + int err; + + err_rsp->StructureSize = SMB2_ERROR_STRUCTURE_SIZE2_LE; + err_rsp->ErrorContextCount = 0; + err_rsp->Reserved = 0; + err_rsp->ByteCount = 0; + err_rsp->ErrorData[0] = 0; + err = ksmbd_iov_pin_rsp(work, (void *)err_rsp, + __SMB2_HEADER_STRUCTURE_SIZE + + SMB2_ERROR_STRUCTURE_SIZE2); + if (err) + work->send_no_response = 1; + } +} + +/** + * is_smb2_neg_cmd() - is it smb2 negotiation command + * @work: smb work containing smb header + * + * Return: true if smb2 negotiation command, otherwise false + */ +bool is_smb2_neg_cmd(struct ksmbd_work *work) +{ + struct smb2_hdr *hdr = smb_get_msg(work->request_buf); + + /* is it SMB2 header ? */ + if (hdr->ProtocolId != SMB2_PROTO_NUMBER) + return false; + + /* make sure it is request not response message */ + if (hdr->Flags & SMB2_FLAGS_SERVER_TO_REDIR) + return false; + + if (hdr->Command != SMB2_NEGOTIATE) + return false; + + return true; +} + +/** + * is_smb2_rsp() - is it smb2 response + * @work: smb work containing smb response buffer + * + * Return: true if smb2 response, otherwise false + */ +bool is_smb2_rsp(struct ksmbd_work *work) +{ + struct smb2_hdr *hdr = smb_get_msg(work->response_buf); + + /* is it SMB2 header ? */ + if (hdr->ProtocolId != SMB2_PROTO_NUMBER) + return false; + + /* make sure it is response not request message */ + if (!(hdr->Flags & SMB2_FLAGS_SERVER_TO_REDIR)) + return false; + + return true; +} + +/** + * get_smb2_cmd_val() - get smb command code from smb header + * @work: smb work containing smb request buffer + * + * Return: smb2 request command value + */ +u16 get_smb2_cmd_val(struct ksmbd_work *work) +{ + struct smb2_hdr *rcv_hdr; + + if (work->next_smb2_rcv_hdr_off) + rcv_hdr = ksmbd_req_buf_next(work); + else + rcv_hdr = smb_get_msg(work->request_buf); + return le16_to_cpu(rcv_hdr->Command); +} + +/** + * set_smb2_rsp_status() - set error response code on smb2 header + * @work: smb work containing response buffer + * @err: error response code + */ +void set_smb2_rsp_status(struct ksmbd_work *work, __le32 err) +{ + struct smb2_hdr *rsp_hdr; + + if (work->next_smb2_rcv_hdr_off) { + rsp_hdr = ksmbd_resp_buf_next(work); + rsp_hdr->Status = err; + smb2_set_err_rsp(work); + return; + } + + rsp_hdr = smb_get_msg(work->response_buf); + rsp_hdr->Status = err; + + work->iov_idx = 0; + work->iov_cnt = 0; + work->next_smb2_rcv_hdr_off = 0; + smb2_set_err_rsp(work); +} + +/** + * init_smb2_neg_rsp() - initialize smb2 response for negotiate command + * @work: smb work containing smb request buffer + * + * smb2 negotiate response is sent in reply of smb1 negotiate command for + * dialect auto-negotiation. + */ +int init_smb2_neg_rsp(struct ksmbd_work *work) +{ + struct smb2_hdr *rsp_hdr; + struct smb2_negotiate_rsp *rsp; + struct ksmbd_conn *conn = work->conn; + int err; + + rsp_hdr = smb_get_msg(work->response_buf); + memset(rsp_hdr, 0, sizeof(struct smb2_hdr) + 2); + rsp_hdr->ProtocolId = SMB2_PROTO_NUMBER; + rsp_hdr->StructureSize = SMB2_HEADER_STRUCTURE_SIZE; + rsp_hdr->CreditRequest = cpu_to_le16(2); + rsp_hdr->Command = SMB2_NEGOTIATE; + rsp_hdr->Flags = (SMB2_FLAGS_SERVER_TO_REDIR); + rsp_hdr->NextCommand = 0; + rsp_hdr->MessageId = 0; + rsp_hdr->Id.SyncId.ProcessId = 0; + rsp_hdr->Id.SyncId.TreeId = 0; + rsp_hdr->SessionId = 0; + memset(rsp_hdr->Signature, 0, 16); + + rsp = smb_get_msg(work->response_buf); + + WARN_ON(ksmbd_conn_good(conn)); + + rsp->StructureSize = cpu_to_le16(65); + ksmbd_debug(SMB, "conn->dialect 0x%x\n", conn->dialect); + rsp->DialectRevision = cpu_to_le16(conn->dialect); + /* Not setting conn guid rsp->ServerGUID, as it + * not used by client for identifying connection + */ + rsp->Capabilities = cpu_to_le32(conn->vals->req_capabilities); + /* Default Max Message Size till SMB2.0, 64K*/ + rsp->MaxTransactSize = cpu_to_le32(conn->vals->max_trans_size); + rsp->MaxReadSize = cpu_to_le32(conn->vals->max_read_size); + rsp->MaxWriteSize = cpu_to_le32(conn->vals->max_write_size); + + rsp->SystemTime = cpu_to_le64(ksmbd_systime()); + rsp->ServerStartTime = 0; + + rsp->SecurityBufferOffset = cpu_to_le16(128); + rsp->SecurityBufferLength = cpu_to_le16(AUTH_GSS_LENGTH); + ksmbd_copy_gss_neg_header((char *)(&rsp->hdr) + + le16_to_cpu(rsp->SecurityBufferOffset)); + rsp->SecurityMode = SMB2_NEGOTIATE_SIGNING_ENABLED_LE; + if (server_conf.signing == KSMBD_CONFIG_OPT_MANDATORY) + rsp->SecurityMode |= SMB2_NEGOTIATE_SIGNING_REQUIRED_LE; + err = ksmbd_iov_pin_rsp(work, rsp, + sizeof(struct smb2_negotiate_rsp) + AUTH_GSS_LENGTH); + if (err) + return err; + conn->use_spnego = true; + + ksmbd_conn_set_need_negotiate(conn); + return 0; +} + +/** + * smb2_set_rsp_credits() - set number of credits in response buffer + * @work: smb work containing smb response buffer + */ +int smb2_set_rsp_credits(struct ksmbd_work *work) +{ + struct smb2_hdr *req_hdr = ksmbd_req_buf_next(work); + struct smb2_hdr *hdr = ksmbd_resp_buf_next(work); + struct ksmbd_conn *conn = work->conn; + unsigned short credits_requested, aux_max; + unsigned short credit_charge, credits_granted = 0; + u64 window_room, i; + + if (work->send_no_response) + return 0; + + hdr->CreditCharge = req_hdr->CreditCharge; + + if (conn->total_credits > conn->vals->max_credits) { + hdr->CreditRequest = 0; + pr_err("Total credits overflow: %d\n", conn->total_credits); + return -EINVAL; + } + + credit_charge = max_t(unsigned short, + le16_to_cpu(req_hdr->CreditCharge), 1); + if (credit_charge > conn->total_credits) { + ksmbd_debug(SMB, "Insufficient credits granted, given: %u, granted: %u\n", + credit_charge, conn->total_credits); + return -EINVAL; + } + + conn->total_credits -= credit_charge; + conn->outstanding_credits -= credit_charge; + work->credit_charge = 0; + credits_requested = max_t(unsigned short, + le16_to_cpu(req_hdr->CreditRequest), 1); + + /* according to smb2.credits smbtorture, Windows server + * 2016 or later grant up to 8192 credits at once. + * + * TODO: Need to adjuct CreditRequest value according to + * current cpu load + */ + if (hdr->Command == SMB2_NEGOTIATE) + aux_max = 1; + else + aux_max = conn->vals->max_credits - conn->total_credits; + + /* + * The command sequence window must not grow beyond + * KSMBD_CMD_SEQ_WINDOW sequence numbers ahead of the oldest one still + * outstanding. Cap the grant by the room left in the window so that + * credits are withheld until the client consumes the low end (and so + * that seq_bitmap stays usable as a ring). + */ + window_room = conn->seq_low + KSMBD_CMD_SEQ_WINDOW - conn->seq_high; + aux_max = min_t(unsigned short, aux_max, window_room); + credits_granted = min_t(unsigned short, credits_requested, aux_max); + + conn->total_credits += credits_granted; + work->credits_granted += credits_granted; + + /* Extend the sequence window to cover the newly granted credits. */ + for (i = conn->seq_high; i < conn->seq_high + credits_granted; i++) + __set_bit(i & (KSMBD_CMD_SEQ_WINDOW - 1), conn->seq_bitmap); + conn->seq_high += credits_granted; + + if (!req_hdr->NextCommand) { + /* Update CreditRequest in last request */ + hdr->CreditRequest = cpu_to_le16(work->credits_granted); + } + ksmbd_debug(SMB, + "credits: requested[%d] granted[%d] total_granted[%d]\n", + credits_requested, credits_granted, + conn->total_credits); + return 0; +} + +/** + * init_chained_smb2_rsp() - initialize smb2 chained response + * @work: smb work containing smb response buffer + */ +static void init_chained_smb2_rsp(struct ksmbd_work *work) +{ + struct smb2_hdr *req = ksmbd_req_buf_next(work); + struct smb2_hdr *rsp = ksmbd_resp_buf_next(work); + struct smb2_hdr *rsp_hdr; + struct smb2_hdr *rcv_hdr; + int next_hdr_offset = 0; + int len, new_len; + + /* Len of this response = updated RFC len - offset of previous cmd + * in the compound rsp + */ + + /* Storing the current local FID which may be needed by subsequent + * command in the compound request + */ + if (req->Command == SMB2_CREATE && rsp->Status == STATUS_SUCCESS) { + work->compound_fid = ((struct smb2_create_rsp *)rsp)->VolatileFileId; + work->compound_pfid = ((struct smb2_create_rsp *)rsp)->PersistentFileId; + work->compound_sid = le64_to_cpu(rsp->SessionId); + work->compound_status = STATUS_SUCCESS; + } else if ((req->Command == SMB2_FLUSH || + req->Command == SMB2_READ || + req->Command == SMB2_WRITE) && + rsp->Status == STATUS_SUCCESS) { + u64 volatile_id = KSMBD_NO_FID; + u64 persistent_id = KSMBD_NO_FID; + + if (req->Command == SMB2_FLUSH) { + struct smb2_flush_req *flush_req = + (struct smb2_flush_req *)req; + + volatile_id = flush_req->VolatileFileId; + persistent_id = flush_req->PersistentFileId; + } else if (req->Command == SMB2_READ) { + struct smb2_read_req *read_req = + (struct smb2_read_req *)req; + + volatile_id = read_req->VolatileFileId; + persistent_id = read_req->PersistentFileId; + } else { + struct smb2_write_req *write_req = + (struct smb2_write_req *)req; + + volatile_id = write_req->VolatileFileId; + persistent_id = write_req->PersistentFileId; + } + + if (has_file_id(volatile_id)) { + work->compound_fid = volatile_id; + work->compound_pfid = persistent_id; + work->compound_sid = le64_to_cpu(rsp->SessionId); + work->compound_status = STATUS_SUCCESS; + } + } else if (req->Command == SMB2_CREATE) { + work->compound_fid = KSMBD_NO_FID; + work->compound_pfid = KSMBD_NO_FID; + work->compound_sid = le64_to_cpu(rsp->SessionId); + work->compound_status = rsp->Status; + } else if (rsp->Status != STATUS_SUCCESS) { + work->compound_sid = le64_to_cpu(rsp->SessionId); + /* + * Only carry the failed status forward when the failing command + * was itself part of the related chain. An unrelated command + * that fails (e.g. a standalone request with a bad session id) + * must not seed the status for a following related command, + * which has to be evaluated on its own (and may legitimately + * fail with a different status such as INVALID_PARAMETER). The + * compound session id is still tracked so a following related + * command can validate it. + */ + if (req->Flags & SMB2_FLAGS_RELATED_OPERATIONS) + work->compound_status = rsp->Status; + } + + len = get_rfc1002_len(work->response_buf) - work->next_smb2_rsp_hdr_off; + next_hdr_offset = le32_to_cpu(req->NextCommand); + + new_len = ALIGN(len, 8); + work->iov[work->iov_idx].iov_len += (new_len - len); + inc_rfc1001_len(work->response_buf, new_len - len); + rsp->NextCommand = cpu_to_le32(new_len); + + work->next_smb2_rcv_hdr_off += next_hdr_offset; + work->curr_smb2_rsp_hdr_off = work->next_smb2_rsp_hdr_off; + work->next_smb2_rsp_hdr_off += new_len; + ksmbd_debug(SMB, + "Compound req new_len = %d rcv off = %d rsp off = %d\n", + new_len, work->next_smb2_rcv_hdr_off, + work->next_smb2_rsp_hdr_off); + + rsp_hdr = ksmbd_resp_buf_next(work); + rcv_hdr = ksmbd_req_buf_next(work); + + if (!(rcv_hdr->Flags & SMB2_FLAGS_RELATED_OPERATIONS)) { + ksmbd_debug(SMB, "related flag should be set\n"); + work->compound_fid = KSMBD_NO_FID; + work->compound_pfid = KSMBD_NO_FID; + work->compound_status = STATUS_SUCCESS; + } + memset((char *)rsp_hdr, 0, sizeof(struct smb2_hdr) + 2); + rsp_hdr->ProtocolId = SMB2_PROTO_NUMBER; + rsp_hdr->StructureSize = SMB2_HEADER_STRUCTURE_SIZE; + rsp_hdr->Command = rcv_hdr->Command; + + /* + * Message is response. We don't grant oplock yet. + */ + rsp_hdr->Flags = (SMB2_FLAGS_SERVER_TO_REDIR | + SMB2_FLAGS_RELATED_OPERATIONS); + if (rcv_hdr->Flags & SMB2_FLAGS_REPLAY_OPERATION) + rsp_hdr->Flags |= SMB2_FLAGS_REPLAY_OPERATION; + rsp_hdr->NextCommand = 0; + rsp_hdr->MessageId = rcv_hdr->MessageId; + rsp_hdr->Id.SyncId.ProcessId = rcv_hdr->Id.SyncId.ProcessId; + rsp_hdr->Id.SyncId.TreeId = rcv_hdr->Id.SyncId.TreeId; + rsp_hdr->SessionId = rcv_hdr->SessionId; + memcpy(rsp_hdr->Signature, rcv_hdr->Signature, 16); +} + +static bool smb2_compound_has_failed(struct ksmbd_work *work, + struct smb2_hdr *rsp) +{ + if (!work->next_smb2_rcv_hdr_off || + has_file_id(work->compound_fid) || + work->compound_status == STATUS_SUCCESS) + return false; + + rsp->Status = work->compound_status; + smb2_set_err_rsp(work); + return true; +} + +/** + * is_chained_smb2_message() - check for chained command + * @work: smb work containing smb request buffer + * + * Return: true if chained request, otherwise false + */ +bool is_chained_smb2_message(struct ksmbd_work *work) +{ + struct smb2_hdr *hdr = smb_get_msg(work->request_buf); + unsigned int len, next_cmd; + + if (hdr->ProtocolId != SMB2_PROTO_NUMBER) + return false; + + hdr = ksmbd_req_buf_next(work); + next_cmd = le32_to_cpu(hdr->NextCommand); + if (next_cmd > 0) { + if ((u64)work->next_smb2_rcv_hdr_off + next_cmd + + __SMB2_HEADER_STRUCTURE_SIZE > + get_rfc1002_len(work->request_buf)) { + pr_err("next command(%u) offset exceeds smb msg size\n", + next_cmd); + return false; + } + + if ((u64)get_rfc1002_len(work->response_buf) + MAX_CIFS_SMALL_BUFFER_SIZE > + work->response_sz) { + pr_err("next response offset exceeds response buffer size\n"); + return false; + } + + ksmbd_debug(SMB, "got SMB2 chained command\n"); + init_chained_smb2_rsp(work); + return true; + } else if (work->next_smb2_rcv_hdr_off) { + /* + * This is last request in chained command, + * align response to 8 byte + */ + len = ALIGN(get_rfc1002_len(work->response_buf), 8); + len = len - get_rfc1002_len(work->response_buf); + if (len) { + ksmbd_debug(SMB, "padding len %u\n", len); + work->iov[work->iov_idx].iov_len += len; + inc_rfc1001_len(work->response_buf, len); + } + work->curr_smb2_rsp_hdr_off = work->next_smb2_rsp_hdr_off; + } + return false; +} + +/** + * init_smb2_rsp_hdr() - initialize smb2 response + * @work: smb work containing smb request buffer + * + * Return: 0 + */ +int init_smb2_rsp_hdr(struct ksmbd_work *work) +{ + struct smb2_hdr *rsp_hdr = smb_get_msg(work->response_buf); + struct smb2_hdr *rcv_hdr = smb_get_msg(work->request_buf); + + memset(rsp_hdr, 0, sizeof(struct smb2_hdr) + 2); + rsp_hdr->ProtocolId = rcv_hdr->ProtocolId; + rsp_hdr->StructureSize = SMB2_HEADER_STRUCTURE_SIZE; + rsp_hdr->Command = rcv_hdr->Command; + + /* + * Message is response. We don't grant oplock yet. + */ + rsp_hdr->Flags = (SMB2_FLAGS_SERVER_TO_REDIR); + if (rcv_hdr->Flags & SMB2_FLAGS_REPLAY_OPERATION) + rsp_hdr->Flags |= SMB2_FLAGS_REPLAY_OPERATION; + rsp_hdr->NextCommand = 0; + rsp_hdr->MessageId = rcv_hdr->MessageId; + rsp_hdr->Id.SyncId.ProcessId = rcv_hdr->Id.SyncId.ProcessId; + rsp_hdr->Id.SyncId.TreeId = rcv_hdr->Id.SyncId.TreeId; + rsp_hdr->SessionId = rcv_hdr->SessionId; + memcpy(rsp_hdr->Signature, rcv_hdr->Signature, 16); + + return 0; +} + +static __le16 smb3_hdr_channel_sequence(struct smb2_hdr *hdr) +{ + return ((struct smb3_hdr_req *)hdr)->ChannelSequence; +} + +static bool smb3_hdr_replay(struct smb2_hdr *hdr) +{ + return hdr->Flags & SMB2_FLAGS_REPLAY_OPERATION; +} + +static int smb3_verify_channel_sequence(struct ksmbd_work *work, + struct ksmbd_file *fp, + struct smb2_hdr *hdr, + bool allow_stale) +{ + __le16 chseq_le; + u16 chseq, old_chseq; + int ret = 0; + + if (work->conn->dialect < SMB30_PROT_ID) + return 0; + + chseq_le = smb3_hdr_channel_sequence(hdr); + chseq = le16_to_cpu(chseq_le); + + spin_lock(&fp->f_lock); + old_chseq = le16_to_cpu(fp->channel_sequence); + if (smb3_hdr_replay(hdr)) { + if (chseq == old_chseq && fp->outstanding_pre_requests == 0) { + fp->outstanding_requests++; + } else if ((u16)(chseq - old_chseq) <= 0x7fff && + fp->outstanding_pre_requests == 0) { + fp->outstanding_pre_requests += fp->outstanding_requests; + fp->outstanding_requests = 1; + fp->channel_sequence = chseq_le; + } else if (allow_stale) { + fp->outstanding_pre_requests++; + } else { + ret = -EAGAIN; + } + } else { + if (chseq == old_chseq) { + fp->outstanding_requests++; + } else if ((u16)(chseq - old_chseq) <= 0x7fff) { + fp->outstanding_pre_requests += fp->outstanding_requests; + fp->outstanding_requests = 1; + fp->channel_sequence = chseq_le; + } else if (allow_stale) { + fp->outstanding_pre_requests++; + } else { + ret = -EAGAIN; + } + } + spin_unlock(&fp->f_lock); + + return ret; +} + +static void smb3_complete_channel_sequence(struct ksmbd_work *work, + struct ksmbd_file *fp, + __le16 chseq_le) +{ + u16 chseq; + + if (work->conn->dialect < SMB30_PROT_ID) + return; + + chseq = le16_to_cpu(chseq_le); + + spin_lock(&fp->f_lock); + if (chseq == le16_to_cpu(fp->channel_sequence)) { + if (fp->outstanding_requests) + fp->outstanding_requests--; + } else { + if (fp->outstanding_pre_requests) + fp->outstanding_pre_requests--; + } + spin_unlock(&fp->f_lock); +} + +static int smb2_set_request_open(struct ksmbd_work *work, struct ksmbd_file *fp, + struct smb2_hdr *hdr, bool verify_chseq, + bool allow_stale_chseq) +{ + struct ksmbd_file *open; + int ret; + + smb2_complete_request_open(work); + + open = ksmbd_file_get(fp); + if (!open) + return -ESTALE; + + if (verify_chseq) { + ret = smb3_verify_channel_sequence(work, fp, hdr, + allow_stale_chseq); + if (ret) { + ksmbd_fd_put(work, open); + return ret; + } + work->request_open_chseq_tracked = true; + } + + work->request_open = open; + work->request_open_chseq = smb3_hdr_channel_sequence(hdr); + return 0; +} + +void smb2_complete_request_open(struct ksmbd_work *work) +{ + struct ksmbd_file *open = work->request_open; + + if (!open) + return; + + if (work->request_open_chseq_tracked) + smb3_complete_channel_sequence(work, open, + work->request_open_chseq); + + work->request_open = NULL; + work->request_open_chseq_tracked = false; + ksmbd_fd_put(work, open); +} + +static bool smb2_lock_sequence_applicable(struct ksmbd_work *work, + struct ksmbd_file *fp) +{ + return fp->is_resilient || fp->is_durable || fp->is_persistent || + (work->conn->dialect >= SMB30_PROT_ID && + (work->conn->vals->req_capabilities & + SMB2_GLOBAL_CAP_MULTI_CHANNEL)); +} + +static bool smb2_verify_lock_sequence(struct ksmbd_work *work, + struct ksmbd_file *fp, + struct smb2_lock_req *req) +{ + u32 val, index; + u8 sequence; + bool replay = false; + + if (work->conn->dialect == SMB20_PROT_ID || + !smb2_lock_sequence_applicable(work, fp)) + return false; + + val = le32_to_cpu(req->LockSequenceNumber); + sequence = val & 0xf; + index = val >> 4; + if (!index || index > KSMBD_LOCK_SEQ_ARRAY_SIZE) + return false; + + spin_lock(&fp->f_lock); + if (fp->lock_seq[index - 1].valid) { + if (fp->lock_seq[index - 1].sequence == sequence) + replay = true; + else + fp->lock_seq[index - 1].valid = false; + } + spin_unlock(&fp->f_lock); + + return replay; +} + +static void smb2_update_lock_sequence(struct ksmbd_work *work, + struct ksmbd_file *fp, + struct smb2_lock_req *req) +{ + u32 val, index; + u8 sequence; + + if (work->conn->dialect == SMB20_PROT_ID || + !smb2_lock_sequence_applicable(work, fp)) + return; + + val = le32_to_cpu(req->LockSequenceNumber); + sequence = val & 0xf; + index = val >> 4; + if (!index || index > KSMBD_LOCK_SEQ_ARRAY_SIZE) + return; + + spin_lock(&fp->f_lock); + fp->lock_seq[index - 1].valid = true; + fp->lock_seq[index - 1].sequence = sequence; + spin_unlock(&fp->f_lock); +} + +/** + * smb2_allocate_rsp_buf() - allocate smb2 response buffer + * @work: smb work containing smb request buffer + * + * Return: 0 on success, otherwise error + */ +int smb2_allocate_rsp_buf(struct ksmbd_work *work) +{ + struct smb2_hdr *hdr = smb_get_msg(work->request_buf); + size_t small_sz = MAX_CIFS_SMALL_BUFFER_SIZE; + size_t large_sz = small_sz + work->conn->vals->max_trans_size; + size_t sz = small_sz; + int cmd = le16_to_cpu(hdr->Command); + + if (cmd == SMB2_IOCTL_HE || cmd == SMB2_QUERY_DIRECTORY_HE) + sz = large_sz; + + if (cmd == SMB2_QUERY_INFO_HE) { + struct smb2_query_info_req *req; + + if (get_rfc1002_len(work->request_buf) < + offsetof(struct smb2_query_info_req, OutputBufferLength)) + return -EINVAL; + + req = smb_get_msg(work->request_buf); + if ((req->InfoType == SMB2_O_INFO_FILE && + (req->FileInfoClass == FILE_FULL_EA_INFORMATION || + req->FileInfoClass == FILE_ALL_INFORMATION)) || + req->InfoType == SMB2_O_INFO_SECURITY) + sz = large_sz; + } + + /* allocate large response buf for chained commands */ + if (le32_to_cpu(hdr->NextCommand) > 0) + sz = large_sz; + + work->response_buf = kvzalloc(sz, KSMBD_DEFAULT_GFP); + if (!work->response_buf) + return -ENOMEM; + + work->response_sz = sz; + return 0; +} + +static bool smb2_session_expired_cmd_allowed(struct ksmbd_work *work, + unsigned int cmd) +{ + struct smb2_lock_req *req; + unsigned int len, lock_count, i; + + if (cmd == SMB2_CANCEL_HE || cmd == SMB2_CLOSE_HE || + cmd == SMB2_LOGOFF_HE) + return true; + if (cmd != SMB2_LOCK_HE) + return false; + + req = ksmbd_req_buf_next(work); + if (req->hdr.NextCommand) + len = le32_to_cpu(req->hdr.NextCommand); + else { + len = get_rfc1002_len(work->request_buf); + if (len < work->next_smb2_rcv_hdr_off) + return false; + len -= work->next_smb2_rcv_hdr_off; + } + + lock_count = le16_to_cpu(req->LockCount); + if (!lock_count || len < offsetof(struct smb2_lock_req, locks) || + lock_count > (len - offsetof(struct smb2_lock_req, locks)) / + sizeof(struct smb2_lock_element)) + return false; + + for (i = 0; i < lock_count; i++) { + if (le32_to_cpu(req->locks[i].Flags) != SMB2_LOCKFLAG_UNLOCK) + return false; + } + return true; +} + +static bool smb2_session_kerberos_expired(struct ksmbd_session *sess) +{ + return sess->kerberos_expiry && + ktime_get_real_seconds() >= sess->kerberos_expiry; +} + +/** + * smb2_check_user_session() - check for valid session for a user + * @work: smb work containing smb request buffer + * + * Return: 0 on success, otherwise error + */ +int smb2_check_user_session(struct ksmbd_work *work) +{ + struct smb2_hdr *req_hdr = ksmbd_req_buf_next(work); + struct ksmbd_conn *conn = work->conn; + unsigned int cmd = le16_to_cpu(req_hdr->Command); + unsigned long long sess_id; + + /* + * SMB2_NEGOTIATE and SMB2_SESSION_SETUP do not require a session id. + * SMB2_ECHO may omit it, but an echo carrying a session id still needs + * the session attached to work so that its signature can be checked and + * the response can be signed, including after Kerberos expiry. + */ + if (cmd == SMB2_NEGOTIATE_HE || cmd == SMB2_SESSION_SETUP_HE) + return 0; + + sess_id = le64_to_cpu(req_hdr->SessionId); + if (cmd == SMB2_ECHO_HE) { + /* + * ECHO remains valid without a live session, including after + * LOGOFF. Attach an existing session only to authenticate a signed + * ECHO and sign its response; a stale SessionId is not an error. + */ + if (!work->next_smb2_rcv_hdr_off && sess_id) + work->sess = ksmbd_session_lookup_all_states(conn, sess_id); + if (work->sess) { + if (smb2_session_kerberos_expired(work->sess)) { + work->sess->state = SMB2_SESSION_EXPIRED; + } else if (work->sess->state != SMB2_SESSION_VALID) { + ksmbd_user_session_put(work->sess); + work->sess = NULL; + } + } + return 0; + } + + if (!ksmbd_conn_good(conn)) + return -EIO; + + /* + * If request is not the first in Compound request, + * Just validate session id in header with work->sess->id. + */ + if (work->next_smb2_rcv_hdr_off) { + if (!work->sess) { + pr_err("The first operation in the compound does not have sess\n"); + return -EINVAL; + } + if (sess_id != ULLONG_MAX && work->sess->id != sess_id) { + pr_err("session id(%llu) is different with the first operation(%lld)\n", + sess_id, work->sess->id); + return -EINVAL; + } + if (smb2_session_kerberos_expired(work->sess)) + work->sess->state = SMB2_SESSION_EXPIRED; + if (work->sess->state != SMB2_SESSION_VALID) { + pr_err("compound request on a non-valid session (state %d)\n", + work->sess->state); + if (smb2_session_kerberos_expired(work->sess) && + smb2_session_expired_cmd_allowed(work, cmd)) + return 1; + return smb2_session_kerberos_expired(work->sess) ? + -EKEYEXPIRED : -EINVAL; + } + return 1; + } + + /* Check for validity of user session */ + work->sess = ksmbd_session_lookup_all_states(conn, sess_id); + if (work->sess) { + if (smb2_session_kerberos_expired(work->sess)) { + work->sess->state = SMB2_SESSION_EXPIRED; + return smb2_session_expired_cmd_allowed(work, cmd) ? + 1 : -EKEYEXPIRED; + } + if (work->sess->state != SMB2_SESSION_VALID) { + /* + * Keep the reference for an encrypted request so the caller can + * return STATUS_USER_SESSION_DELETED encrypted with the old key. + */ + if (work->encrypted && + work->sess->state == SMB2_SESSION_EXPIRED && + work->sess->enc) + return -ENOENT; + ksmbd_user_session_put(work->sess); + work->sess = NULL; + return -ENOENT; + } + return 1; + } + ksmbd_debug(SMB, "Invalid user session, Uid %llu\n", sess_id); + return -ENOENT; +} + +/** + * smb2_get_name() - get filename string from on the wire smb format + * @src: source buffer + * @maxlen: maxlen of source string + * @local_nls: nls_table pointer + * + * Return: matching converted filename on success, otherwise error ptr + */ +static char * +smb2_get_name(const char *src, const int maxlen, struct nls_table *local_nls) +{ + char *name; + + name = smb_strndup_from_utf16(src, maxlen, 1, local_nls); + if (IS_ERR(name)) { + pr_err("failed to get name %ld\n", PTR_ERR(name)); + return name; + } + + if (*name == '\0') { + kfree(name); + return ERR_PTR(-EINVAL); + } + + if (*name == '\\') { + pr_err("not allow directory name included leading slash\n"); + kfree(name); + return ERR_PTR(-EINVAL); + } + + ksmbd_conv_path_to_unix(name); + ksmbd_strip_last_slash(name); + return name; +} + +/* Link a fully initialized async work item unless the connection is closing. */ +static bool ksmbd_conn_link_async_request(struct ksmbd_conn *conn, + struct ksmbd_work *work) +{ + bool linked = false; + + spin_lock(&conn->request_lock); + if (!ksmbd_conn_exiting(conn) && !ksmbd_conn_releasing(conn)) { + if (list_empty(&work->async_request_entry)) + list_add_tail(&work->async_request_entry, + &conn->async_requests); + linked = true; + } + spin_unlock(&conn->request_lock); + + return linked; +} + +int setup_async_work(struct ksmbd_work *work, void (*fn)(void **), void **arg) +{ + struct ksmbd_conn *conn = work->conn; + int id; + + id = ksmbd_acquire_async_msg_id(&conn->async_ida); + if (id < 0) { + pr_err("Failed to alloc async message id\n"); + return id; + } + work->asynchronous = true; + work->async_id = id; + work->cancel_fn = fn; + work->cancel_argv = arg; + + if (!ksmbd_conn_link_async_request(conn, work)) { + work->asynchronous = false; + work->async_id = 0; + work->cancel_fn = NULL; + work->cancel_argv = NULL; + ksmbd_release_id(&conn->async_ida, id); + return -ESHUTDOWN; + } + + ksmbd_debug(SMB, + "Send interim Response to inform async request id : %d\n", + work->async_id); + + return 0; +} + +void release_async_work(struct ksmbd_work *work) +{ + struct ksmbd_conn *conn = work->conn; + + spin_lock(&conn->request_lock); + list_del_init(&work->async_request_entry); + spin_unlock(&conn->request_lock); + + work->asynchronous = 0; + work->cancel_fn = NULL; + kfree(work->cancel_argv); + work->cancel_argv = NULL; + if (work->async_id) { + ksmbd_release_id(&conn->async_ida, work->async_id); + work->async_id = 0; + } +} + +static int smb2_send_interim_work(struct ksmbd_work *in_work, + struct ksmbd_work *work, bool eor) +{ + int err = 0; + + in_work->encrypted = work->encrypted; + if (work->encrypted && work->sess && work->sess->enc && + work->conn->ops->encrypt_resp) { + in_work->sess = work->sess; + err = work->conn->ops->encrypt_resp(in_work); + in_work->sess = NULL; + } + if (err) + return err; + + return eor ? ksmbd_conn_write_eor(in_work) : + ksmbd_conn_write(in_work); +} + +static int smb2_send_interim_prefix_work(struct ksmbd_work *work) +{ + struct ksmbd_work *in_work; + unsigned int len, copied = 0; + char *dst; + int err = -ENOMEM; + int i; + + len = get_rfc1002_len(work->iov[0].iov_base); + in_work = ksmbd_alloc_work_struct(); + if (!in_work) + return err; + + in_work->response_buf = kvzalloc(len + 4, KSMBD_DEFAULT_GFP); + if (!in_work->response_buf) + goto out; + in_work->response_sz = len + 4; + in_work->conn = work->conn; + dst = in_work->response_buf + 4; + for (i = 1; i <= work->iov_idx; i++) { + if (work->iov[i].iov_len > len - copied) { + err = -EINVAL; + goto out; + } + memcpy(dst + copied, work->iov[i].iov_base, + work->iov[i].iov_len); + copied += work->iov[i].iov_len; + } + if (copied != len) { + err = -EINVAL; + goto out; + } + + err = ksmbd_iov_pin_rsp(in_work, dst, len); + if (!err) + err = smb2_send_interim_work(in_work, work, true); +out: + ksmbd_free_work_struct(in_work); + return err; +} + +static void smb2_send_interim_compound_prefix(struct ksmbd_work *work) +{ + struct smb2_hdr *req_hdr; + struct smb2_hdr *rsp_hdr; + int err; + + if (!work->next_smb2_rcv_hdr_off || + !work->next_smb2_rsp_hdr_off || + work->curr_smb2_rsp_hdr_off == work->next_smb2_rsp_hdr_off || + !work->iov_idx) + return; + + req_hdr = ksmbd_req_buf_next(work); + /* Detach only the final async command from the completed prefix. */ + if (req_hdr->NextCommand) + return; + + /* + * The responses before the async command are sent as a standalone + * compound response. The last response in this prefix must terminate + * the chain. + */ + rsp_hdr = ksmbd_resp_buf_curr(work); + rsp_hdr->NextCommand = 0; + if ((rsp_hdr->Flags & SMB2_FLAGS_SIGNED) && work->sess && + work->conn->ops->set_sign_rsp) + work->conn->ops->set_sign_rsp(work); + + err = smb2_send_interim_prefix_work(work); + if (err) + ksmbd_debug(SMB, "failed to send compound interim prefix: %d\n", + err); + + work->iov_idx = 0; + work->iov_cnt = 0; + work->curr_smb2_rsp_hdr_off = work->next_smb2_rsp_hdr_off; + *(__be32 *)work->response_buf = 0; + + rsp_hdr = ksmbd_resp_buf_next(work); + rsp_hdr->Flags &= ~SMB2_FLAGS_RELATED_OPERATIONS; +} + +void smb2_send_interim_resp(struct ksmbd_work *work, __le32 status) +{ + struct smb2_hdr *rsp_hdr; + struct ksmbd_work *in_work = ksmbd_alloc_work_struct(); + + if (!in_work) + return; + + if (allocate_interim_rsp_buf(in_work)) { + pr_err("smb_allocate_rsp_buf failed!\n"); + ksmbd_free_work_struct(in_work); + return; + } + + if (status == STATUS_PENDING) + smb2_send_interim_compound_prefix(work); + + in_work->conn = work->conn; + memcpy(smb_get_msg(in_work->response_buf), ksmbd_resp_buf_next(work), + __SMB2_HEADER_STRUCTURE_SIZE); + + rsp_hdr = smb_get_msg(in_work->response_buf); + rsp_hdr->Flags |= SMB2_FLAGS_ASYNC_COMMAND; + rsp_hdr->Id.AsyncId = cpu_to_le64(work->async_id); + smb2_set_err_rsp(in_work); + rsp_hdr->Status = status; + + if (smb2_send_interim_work(in_work, work, true)) + ksmbd_debug(SMB, "failed to send interim response\n"); + ksmbd_free_work_struct(in_work); +} + +static __le32 smb2_get_reparse_tag_special_file(umode_t mode) +{ + if (S_ISDIR(mode) || S_ISREG(mode)) + return 0; + + if (S_ISLNK(mode)) + return IO_REPARSE_TAG_LX_SYMLINK_LE; + else if (S_ISFIFO(mode)) + return IO_REPARSE_TAG_LX_FIFO_LE; + else if (S_ISSOCK(mode)) + return IO_REPARSE_TAG_AF_UNIX_LE; + else if (S_ISCHR(mode)) + return IO_REPARSE_TAG_LX_CHR_LE; + else if (S_ISBLK(mode)) + return IO_REPARSE_TAG_LX_BLK_LE; + + return 0; +} + +/** + * smb2_get_dos_mode() - get file mode in dos format from unix mode + * @stat: kstat containing file mode + * @attribute: attribute flags + * + * Return: converted dos mode + */ +static int smb2_get_dos_mode(struct kstat *stat, int attribute) +{ + int attr = 0; + + if (S_ISDIR(stat->mode)) { + attr = FILE_ATTRIBUTE_DIRECTORY | + (attribute & (FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_SYSTEM)); + } else { + attr = (attribute & 0x00005137) | FILE_ATTRIBUTE_ARCHIVE; + attr &= ~(FILE_ATTRIBUTE_DIRECTORY); + + if (smb2_get_reparse_tag_special_file(stat->mode)) + attr |= FILE_ATTRIBUTE_REPARSE_POINT; + } + + return attr; +} + +static void build_preauth_ctxt(struct smb2_preauth_neg_context *pneg_ctxt, + __le16 hash_id) +{ + pneg_ctxt->ContextType = SMB2_PREAUTH_INTEGRITY_CAPABILITIES; + pneg_ctxt->DataLength = cpu_to_le16(38); + pneg_ctxt->HashAlgorithmCount = cpu_to_le16(1); + pneg_ctxt->Reserved = cpu_to_le32(0); + pneg_ctxt->SaltLength = cpu_to_le16(SMB311_SALT_SIZE); + get_random_bytes(pneg_ctxt->Salt, SMB311_SALT_SIZE); + pneg_ctxt->HashAlgorithms = hash_id; +} + +static void build_encrypt_ctxt(struct smb2_encryption_neg_context *pneg_ctxt, + __le16 cipher_type) +{ + pneg_ctxt->ContextType = SMB2_ENCRYPTION_CAPABILITIES; + pneg_ctxt->DataLength = cpu_to_le16(4); + pneg_ctxt->Reserved = cpu_to_le32(0); + pneg_ctxt->CipherCount = cpu_to_le16(1); + pneg_ctxt->Ciphers[0] = cipher_type; +} + +static void build_compress_ctxt(struct smb2_compression_capabilities_context *pneg_ctxt, + __le16 compress_algorithm, bool compress_chained, + bool compress_pattern) +{ + /* + * Return only algorithms implemented by ksmbd. Pattern_V1 is advertised + * as a second ID when the client also enabled chained transforms. + */ + pneg_ctxt->ContextType = SMB2_COMPRESSION_CAPABILITIES; + pneg_ctxt->DataLength = cpu_to_le16(compress_pattern ? 12 : 10); + pneg_ctxt->Reserved = cpu_to_le32(0); + pneg_ctxt->CompressionAlgorithmCount = + cpu_to_le16(compress_pattern ? 2 : 1); + pneg_ctxt->Padding = cpu_to_le16(0); + pneg_ctxt->Flags = compress_chained ? + SMB2_COMPRESSION_CAPABILITIES_FLAG_CHAINED : + SMB2_COMPRESSION_CAPABILITIES_FLAG_NONE; + pneg_ctxt->CompressionAlgorithms[0] = compress_algorithm; + pneg_ctxt->CompressionAlgorithms[1] = compress_pattern ? + SMB3_COMPRESS_PATTERN : 0; + pneg_ctxt->CompressionAlgorithms[2] = 0; + pneg_ctxt->CompressionAlgorithms[3] = 0; +} + +/** + * build_rdma_ctx() - build an RDMA transform negotiate response context + * @ctxt: response context header to populate + * @transform_ids: bitmap of transforms common to the client and server + * + * Return: encoded negotiate context length + */ +static int build_rdma_ctx(struct smb2_neg_context *ctxt, + unsigned long transform_ids) +{ + struct smb2_rdma_transform_capabilities_context *pneg_ctxt; + int count = 0; + + pneg_ctxt = (void *)ctxt; + pneg_ctxt->ContextType = SMB2_RDMA_TRANSFORM_CAPABILITIES; + pneg_ctxt->Reserved = 0; + pneg_ctxt->Reserved1 = 0; + pneg_ctxt->Reserved2 = 0; + if (transform_ids & BIT(SMB2_RDMA_TRANSFORM_ENCRYPTION)) + pneg_ctxt->RDMATransformIds[count++] = + cpu_to_le16(SMB2_RDMA_TRANSFORM_ENCRYPTION); + if (!count) + pneg_ctxt->RDMATransformIds[count++] = + cpu_to_le16(SMB2_RDMA_TRANSFORM_NONE); + + pneg_ctxt->TransformCount = cpu_to_le16(count); + pneg_ctxt->DataLength = cpu_to_le16(8 + count * sizeof(__le16)); + return sizeof(struct smb2_neg_context) + + le16_to_cpu(pneg_ctxt->DataLength); +} + +static void build_sign_cap_ctxt(struct smb2_signing_capabilities *pneg_ctxt, + __le16 sign_algo) +{ + pneg_ctxt->ContextType = SMB2_SIGNING_CAPABILITIES; + pneg_ctxt->DataLength = + cpu_to_le16((sizeof(struct smb2_signing_capabilities) + 2) + - sizeof(struct smb2_neg_context)); + pneg_ctxt->Reserved = cpu_to_le32(0); + pneg_ctxt->SigningAlgorithmCount = cpu_to_le16(1); + pneg_ctxt->SigningAlgorithms[0] = sign_algo; +} + +static void build_posix_ctxt(struct smb2_posix_neg_context *pneg_ctxt) +{ + pneg_ctxt->ContextType = SMB2_POSIX_EXTENSIONS_AVAILABLE; + pneg_ctxt->DataLength = cpu_to_le16(POSIX_CTXT_DATA_LEN); + /* SMB2_CREATE_TAG_POSIX is "0x93AD25509CB411E7B42383DE968BCD7C" */ + pneg_ctxt->Name[0] = 0x93; + pneg_ctxt->Name[1] = 0xAD; + pneg_ctxt->Name[2] = 0x25; + pneg_ctxt->Name[3] = 0x50; + pneg_ctxt->Name[4] = 0x9C; + pneg_ctxt->Name[5] = 0xB4; + pneg_ctxt->Name[6] = 0x11; + pneg_ctxt->Name[7] = 0xE7; + pneg_ctxt->Name[8] = 0xB4; + pneg_ctxt->Name[9] = 0x23; + pneg_ctxt->Name[10] = 0x83; + pneg_ctxt->Name[11] = 0xDE; + pneg_ctxt->Name[12] = 0x96; + pneg_ctxt->Name[13] = 0x8B; + pneg_ctxt->Name[14] = 0xCD; + pneg_ctxt->Name[15] = 0x7C; +} + +static unsigned int assemble_neg_contexts(struct ksmbd_conn *conn, + struct smb2_negotiate_rsp *rsp) +{ + char * const pneg_ctxt = (char *)rsp + + le32_to_cpu(rsp->NegotiateContextOffset); + int neg_ctxt_cnt = 1; + int ctxt_size; + + ksmbd_debug(SMB, + "assemble SMB2_PREAUTH_INTEGRITY_CAPABILITIES context\n"); + build_preauth_ctxt((struct smb2_preauth_neg_context *)pneg_ctxt, + conn->preauth_info->Preauth_HashId); + ctxt_size = sizeof(struct smb2_preauth_neg_context); + + if (conn->cipher_type) { + /* Round to 8 byte boundary */ + ctxt_size = round_up(ctxt_size, 8); + ksmbd_debug(SMB, + "assemble SMB2_ENCRYPTION_CAPABILITIES context\n"); + build_encrypt_ctxt((struct smb2_encryption_neg_context *) + (pneg_ctxt + ctxt_size), + conn->cipher_type); + neg_ctxt_cnt++; + ctxt_size += sizeof(struct smb2_encryption_neg_context) + 2; + } + + if (conn->compress_algorithm != SMB3_COMPRESS_NONE) { + ctxt_size = round_up(ctxt_size, 8); + ksmbd_debug(SMB, + "assemble SMB2_COMPRESSION_CAPABILITIES context\n"); + build_compress_ctxt((struct smb2_compression_capabilities_context *) + (pneg_ctxt + ctxt_size), + conn->compress_algorithm, + conn->compress_chained, + conn->compress_pattern); + neg_ctxt_cnt++; + ctxt_size += sizeof(struct smb2_neg_context) + + (conn->compress_pattern ? 12 : 10); + } + + if (conn->rdma_transform_negotiated) { + struct smb2_neg_context *rdma_ctxt; + + ctxt_size = round_up(ctxt_size, 8); + ksmbd_debug(SMB, + "assemble SMB2_RDMA_TRANSFORM_CAPABILITIES context\n"); + rdma_ctxt = (void *)(pneg_ctxt + ctxt_size); + ctxt_size += build_rdma_ctx(rdma_ctxt, + conn->rdma_transform_ids); + neg_ctxt_cnt++; + } + + if (conn->posix_ext_supported) { + ctxt_size = round_up(ctxt_size, 8); + ksmbd_debug(SMB, + "assemble SMB2_POSIX_EXTENSIONS_AVAILABLE context\n"); + build_posix_ctxt((struct smb2_posix_neg_context *) + (pneg_ctxt + ctxt_size)); + neg_ctxt_cnt++; + ctxt_size += sizeof(struct smb2_posix_neg_context); + } + + if (conn->signing_negotiated) { + ctxt_size = round_up(ctxt_size, 8); + ksmbd_debug(SMB, + "assemble SMB2_SIGNING_CAPABILITIES context\n"); + build_sign_cap_ctxt((struct smb2_signing_capabilities *) + (pneg_ctxt + ctxt_size), + conn->signing_algorithm); + neg_ctxt_cnt++; + ctxt_size += sizeof(struct smb2_signing_capabilities) + 2; + } + + rsp->NegotiateContextCount = cpu_to_le16(neg_ctxt_cnt); + return ctxt_size + AUTH_GSS_PADDING; +} + +static __le32 decode_preauth_ctxt(struct ksmbd_conn *conn, + struct smb2_preauth_neg_context *pneg_ctxt, + int ctxt_len) +{ + /* + * sizeof(smb2_preauth_neg_context) assumes SMB311_SALT_SIZE Salt, + * which may not be present. Only check for used HashAlgorithms[1]. + */ + if (ctxt_len < + sizeof(struct smb2_neg_context) + MIN_PREAUTH_CTXT_DATA_LEN) + return STATUS_INVALID_PARAMETER; + + if (pneg_ctxt->HashAlgorithms != SMB2_PREAUTH_INTEGRITY_SHA512) + return STATUS_SMB_NO_PREAUTH_INTEGRITY_HASH_OVERLAP; + + conn->preauth_info->Preauth_HashId = SMB2_PREAUTH_INTEGRITY_SHA512; + return STATUS_SUCCESS; +} + +static void decode_encrypt_ctxt(struct ksmbd_conn *conn, + struct smb2_encryption_neg_context *pneg_ctxt, + int ctxt_len) +{ + int cph_cnt; + int i, cphs_size; + + if (sizeof(struct smb2_encryption_neg_context) > ctxt_len) { + pr_err("Invalid SMB2_ENCRYPTION_CAPABILITIES context size\n"); + return; + } + + conn->cipher_type = 0; + + cph_cnt = le16_to_cpu(pneg_ctxt->CipherCount); + cphs_size = cph_cnt * sizeof(__le16); + + if (sizeof(struct smb2_encryption_neg_context) + cphs_size > + ctxt_len) { + pr_err("Invalid cipher count(%d)\n", cph_cnt); + return; + } + + if (server_conf.flags & KSMBD_GLOBAL_FLAG_SMB2_ENCRYPTION_OFF) + return; + + for (i = 0; i < cph_cnt; i++) { + if (pneg_ctxt->Ciphers[i] == SMB2_ENCRYPTION_AES128_GCM || + pneg_ctxt->Ciphers[i] == SMB2_ENCRYPTION_AES128_CCM || + pneg_ctxt->Ciphers[i] == SMB2_ENCRYPTION_AES256_CCM || + pneg_ctxt->Ciphers[i] == SMB2_ENCRYPTION_AES256_GCM) { + ksmbd_debug(SMB, "Cipher ID = 0x%x\n", + pneg_ctxt->Ciphers[i]); + conn->cipher_type = pneg_ctxt->Ciphers[i]; + break; + } + } +} + +/** + * smb3_encryption_negotiated() - checks if server and client agreed on enabling encryption + * @conn: smb connection + * + * Return: true if connection should be encrypted, else false + */ +bool smb3_encryption_negotiated(struct ksmbd_conn *conn) +{ + if (!conn->ops->generate_encryptionkey) + return false; + + /* + * SMB 3.0 and 3.0.2 dialects use the SMB2_GLOBAL_CAP_ENCRYPTION flag. + * SMB 3.1.1 uses the cipher_type field. + */ + return (conn->vals->req_capabilities & SMB2_GLOBAL_CAP_ENCRYPTION) || + conn->cipher_type; +} + +static __le32 decode_compress_ctxt(struct ksmbd_conn *conn, + struct smb2_compression_capabilities_context *pneg_ctxt, + int ctxt_len) +{ + int alg_cnt, algs_size, i; + __le16 *algs; + + if (sizeof(struct smb2_neg_context) + 10 > ctxt_len) { + pr_err("Invalid SMB2_COMPRESSION_CAPABILITIES context length\n"); + return STATUS_INVALID_PARAMETER; + } + + conn->compress_algorithm = SMB3_COMPRESS_NONE; + conn->compress_chained = false; + conn->compress_pattern = false; + + alg_cnt = le16_to_cpu(pneg_ctxt->CompressionAlgorithmCount); + if (!alg_cnt) + return STATUS_INVALID_PARAMETER; + + if (pneg_ctxt->Flags != SMB2_COMPRESSION_CAPABILITIES_FLAG_NONE && + pneg_ctxt->Flags != SMB2_COMPRESSION_CAPABILITIES_FLAG_CHAINED) + return STATUS_INVALID_PARAMETER; + + algs_size = alg_cnt * sizeof(__le16); + if (sizeof(struct smb2_neg_context) + 8 + algs_size > ctxt_len) { + pr_err("Invalid compression algorithm count(%d)\n", alg_cnt); + return STATUS_INVALID_PARAMETER; + } + + /* + * CompressionAlgorithms[] is declared as a fixed 4-element array, but + * the actual element count is variable (clients such as Windows may + * advertise more). The on-wire length was validated above, so walk the + * algorithms through a pointer to avoid a fixed-array bounds check. + */ + algs = pneg_ctxt->CompressionAlgorithms; + for (i = 0; i < alg_cnt; i++) { + __le16 alg = algs[i]; + + /* + * LZ77 is the required general-purpose codec. Pattern_V1 is an + * optional chained payload type and cannot stand alone. + */ + if (alg == SMB3_COMPRESS_LZ77) { + conn->compress_algorithm = alg; + conn->compress_chained = + pneg_ctxt->Flags == + SMB2_COMPRESSION_CAPABILITIES_FLAG_CHAINED; + ksmbd_debug(SMB, "Compression Algorithm ID = 0x%x\n", + le16_to_cpu(alg)); + } else if (alg == SMB3_COMPRESS_PATTERN) { + conn->compress_pattern = true; + } + } + + if (conn->compress_algorithm == SMB3_COMPRESS_NONE || + !conn->compress_chained) + conn->compress_pattern = false; + + return STATUS_SUCCESS; +} + +static void decode_sign_cap_ctxt(struct ksmbd_conn *conn, + struct smb2_signing_capabilities *pneg_ctxt, + int ctxt_len) +{ + int sign_algo_cnt; + int i, sign_alos_size; + + if (sizeof(struct smb2_signing_capabilities) > ctxt_len) { + pr_err("Invalid SMB2_SIGNING_CAPABILITIES context length\n"); + return; + } + + conn->signing_negotiated = false; + sign_algo_cnt = le16_to_cpu(pneg_ctxt->SigningAlgorithmCount); + sign_alos_size = sign_algo_cnt * sizeof(__le16); + + if (sizeof(struct smb2_signing_capabilities) + sign_alos_size > + ctxt_len) { + pr_err("Invalid signing algorithm count(%d)\n", sign_algo_cnt); + return; + } + + for (i = 0; i < sign_algo_cnt; i++) { + if (pneg_ctxt->SigningAlgorithms[i] == SIGNING_ALG_HMAC_SHA256_LE || + pneg_ctxt->SigningAlgorithms[i] == SIGNING_ALG_AES_CMAC_LE) { + ksmbd_debug(SMB, "Signing Algorithm ID = 0x%x\n", + pneg_ctxt->SigningAlgorithms[i]); + conn->signing_negotiated = true; + conn->signing_algorithm = + pneg_ctxt->SigningAlgorithms[i]; + break; + } + } +} + +/** + * decode_rdma_ctx() - decode an RDMA transform negotiate request context + * @conn: connection being negotiated + * @ctxt: request context header to decode + * @ctxt_len: total context length, including the negotiate context header + * + * Record transforms supported by both peers only for SMB Direct connections. + * + * Return: NT status describing the decode result + */ +static __le32 decode_rdma_ctx(struct ksmbd_conn *conn, + struct smb2_neg_context *ctxt, int ctxt_len) +{ + struct smb2_rdma_transform_capabilities_context *pneg_ctxt; + unsigned int count, i; + + pneg_ctxt = (void *)ctxt; + /* RDMA transforms are a node capability, not just a transport capability. */ + if (!ksmbd_rdma_enabled()) + return STATUS_SUCCESS; + + if (ctxt_len < sizeof(*pneg_ctxt)) + return STATUS_INVALID_PARAMETER; + + count = le16_to_cpu(pneg_ctxt->TransformCount); + if (!count || count > + (ctxt_len - sizeof(*pneg_ctxt)) / sizeof(__le16)) + return STATUS_INVALID_PARAMETER; + + conn->rdma_transform_negotiated = true; + conn->rdma_transform_ids = 0; + for (i = 0; i < count; i++) { + u16 id = le16_to_cpu(pneg_ctxt->RDMATransformIds[i]); + + if (id == SMB2_RDMA_TRANSFORM_ENCRYPTION) + conn->rdma_transform_ids |= BIT(id); + } + return STATUS_SUCCESS; +} + +static __le32 deassemble_neg_contexts(struct ksmbd_conn *conn, + struct smb2_negotiate_req *req, + unsigned int len_of_smb) +{ + /* +4 is to account for the RFC1001 len field */ + struct smb2_neg_context *pctx = (struct smb2_neg_context *)req; + int i = 0, len_of_ctxts; + unsigned int offset = le32_to_cpu(req->NegotiateContextOffset); + unsigned int neg_ctxt_cnt = le16_to_cpu(req->NegotiateContextCount); + __le32 status = STATUS_INVALID_PARAMETER; + int compress_ctxt_cnt = 0, rdma_transform_ctxt_cnt = 0; + + ksmbd_debug(SMB, "decoding %d negotiate contexts\n", neg_ctxt_cnt); + if (len_of_smb <= offset) { + ksmbd_debug(SMB, "Invalid response: negotiate context offset\n"); + return status; + } + + len_of_ctxts = len_of_smb - offset; + + while (i++ < neg_ctxt_cnt) { + int clen, ctxt_len; + + if (len_of_ctxts < (int)sizeof(struct smb2_neg_context)) + break; + + pctx = (struct smb2_neg_context *)((char *)pctx + offset); + clen = le16_to_cpu(pctx->DataLength); + ctxt_len = clen + sizeof(struct smb2_neg_context); + + if (ctxt_len > len_of_ctxts) + break; + + if (pctx->ContextType == SMB2_PREAUTH_INTEGRITY_CAPABILITIES) { + ksmbd_debug(SMB, + "deassemble SMB2_PREAUTH_INTEGRITY_CAPABILITIES context\n"); + if (conn->preauth_info->Preauth_HashId) + break; + + status = decode_preauth_ctxt(conn, + (struct smb2_preauth_neg_context *)pctx, + ctxt_len); + if (status != STATUS_SUCCESS) + break; + } else if (pctx->ContextType == SMB2_ENCRYPTION_CAPABILITIES) { + ksmbd_debug(SMB, + "deassemble SMB2_ENCRYPTION_CAPABILITIES context\n"); + if (conn->cipher_type) + break; + + decode_encrypt_ctxt(conn, + (struct smb2_encryption_neg_context *)pctx, + ctxt_len); + } else if (pctx->ContextType == SMB2_COMPRESSION_CAPABILITIES) { + ksmbd_debug(SMB, + "deassemble SMB2_COMPRESSION_CAPABILITIES context\n"); + if (compress_ctxt_cnt++) { + status = STATUS_INVALID_PARAMETER; + break; + } + + status = decode_compress_ctxt(conn, + (struct smb2_compression_capabilities_context *) + pctx, ctxt_len); + if (status != STATUS_SUCCESS) + break; + } else if (pctx->ContextType == SMB2_NETNAME_NEGOTIATE_CONTEXT_ID) { + ksmbd_debug(SMB, + "deassemble SMB2_NETNAME_NEGOTIATE_CONTEXT_ID context\n"); + } else if (pctx->ContextType == SMB2_RDMA_TRANSFORM_CAPABILITIES) { + ksmbd_debug(SMB, + "deassemble SMB2_RDMA_TRANSFORM_CAPABILITIES context\n"); + if (ksmbd_rdma_enabled() && + rdma_transform_ctxt_cnt++) { + status = STATUS_INVALID_PARAMETER; + break; + } + status = decode_rdma_ctx(conn, pctx, ctxt_len); + if (status != STATUS_SUCCESS) + break; + } else if (pctx->ContextType == SMB2_POSIX_EXTENSIONS_AVAILABLE) { + ksmbd_debug(SMB, + "deassemble SMB2_POSIX_EXTENSIONS_AVAILABLE context\n"); + conn->posix_ext_supported = true; + } else if (pctx->ContextType == SMB2_SIGNING_CAPABILITIES) { + ksmbd_debug(SMB, + "deassemble SMB2_SIGNING_CAPABILITIES context\n"); + + decode_sign_cap_ctxt(conn, + (struct smb2_signing_capabilities *)pctx, + ctxt_len); + } + + /* offsets must be 8 byte aligned */ + offset = (ctxt_len + 7) & ~0x7; + len_of_ctxts -= offset; + } + return status; +} + +/** + * smb2_handle_negotiate() - handler for smb2 negotiate command + * @work: smb work containing smb request buffer + * + * The caller holds conn->srv_mutex. + * + * Return: 0 + */ +int smb2_handle_negotiate(struct ksmbd_work *work) +{ + struct ksmbd_conn *conn = work->conn; + struct smb2_negotiate_req *req = smb_get_msg(work->request_buf); + struct smb2_negotiate_rsp *rsp = smb_get_msg(work->response_buf); + int rc = 0; + unsigned int smb2_buf_len, smb2_neg_size, neg_ctxt_len = 0; + __le32 status; + + ksmbd_debug(SMB, "Received negotiate request\n"); + conn->need_neg = false; + smb2_buf_len = get_rfc1002_len(work->request_buf); + smb2_neg_size = offsetof(struct smb2_negotiate_req, Dialects); + if (smb2_neg_size > smb2_buf_len) { + rsp->hdr.Status = STATUS_INVALID_PARAMETER; + rc = -EINVAL; + goto err_out; + } + + if (req->DialectCount == 0) { + pr_err("malformed packet\n"); + rsp->hdr.Status = STATUS_INVALID_PARAMETER; + rc = -EINVAL; + goto err_out; + } + + if (conn->dialect == SMB311_PROT_ID) { + unsigned int nego_ctxt_off = le32_to_cpu(req->NegotiateContextOffset); + + if (smb2_buf_len < nego_ctxt_off) { + rsp->hdr.Status = STATUS_INVALID_PARAMETER; + rc = -EINVAL; + goto err_out; + } + + if (smb2_neg_size > nego_ctxt_off) { + rsp->hdr.Status = STATUS_INVALID_PARAMETER; + rc = -EINVAL; + goto err_out; + } + + if (smb2_neg_size + le16_to_cpu(req->DialectCount) * sizeof(__le16) > + nego_ctxt_off) { + rsp->hdr.Status = STATUS_INVALID_PARAMETER; + rc = -EINVAL; + goto err_out; + } + } else { + if (smb2_neg_size + le16_to_cpu(req->DialectCount) * sizeof(__le16) > + smb2_buf_len) { + rsp->hdr.Status = STATUS_INVALID_PARAMETER; + rc = -EINVAL; + goto err_out; + } + } + + conn->cli_cap = le32_to_cpu(req->Capabilities); + switch (conn->dialect) { + case SMB311_PROT_ID: + conn->preauth_info = + kzalloc_obj(struct preauth_integrity_info, + KSMBD_DEFAULT_GFP); + if (!conn->preauth_info) { + rc = -ENOMEM; + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + goto err_out; + } + + status = deassemble_neg_contexts(conn, req, + get_rfc1002_len(work->request_buf)); + if (status != STATUS_SUCCESS) { + pr_err("deassemble_neg_contexts error(0x%x)\n", + status); + rsp->hdr.Status = status; + rc = -EINVAL; + kfree(conn->preauth_info); + conn->preauth_info = NULL; + goto err_out; + } + if (!conn->cipher_type) + conn->rdma_transform_ids &= + ~BIT(SMB2_RDMA_TRANSFORM_ENCRYPTION); + ksmbd_debug(RDMA, + "RDMA transform negotiation: transport=%s context=%s encryption=%s cipher=0x%04x\n", + conn->transport->ops->rdma_read ? "rdma" : "tcp", + conn->rdma_transform_negotiated ? "present" : "absent", + conn->rdma_transform_ids & + BIT(SMB2_RDMA_TRANSFORM_ENCRYPTION) ? "enabled" : "disabled", + le16_to_cpu(conn->cipher_type)); + + rc = init_smb3_11_server(conn); + if (rc < 0) { + rsp->hdr.Status = STATUS_INVALID_PARAMETER; + kfree(conn->preauth_info); + conn->preauth_info = NULL; + goto err_out; + } + + ksmbd_gen_preauth_integrity_hash(conn, + work->request_buf, + conn->preauth_info->Preauth_HashValue); + rsp->NegotiateContextOffset = + cpu_to_le32(OFFSET_OF_NEG_CONTEXT); + neg_ctxt_len = assemble_neg_contexts(conn, rsp); + break; + case SMB302_PROT_ID: + init_smb3_02_server(conn); + break; + case SMB30_PROT_ID: + init_smb3_0_server(conn); + break; + case SMB21_PROT_ID: + init_smb2_1_server(conn); + break; + case SMB2X_PROT_ID: + case BAD_PROT_ID: + default: + ksmbd_debug(SMB, "Server dialect :0x%x not supported\n", + conn->dialect); + rsp->hdr.Status = STATUS_NOT_SUPPORTED; + rc = -EINVAL; + goto err_out; + } + rsp->Capabilities = cpu_to_le32(conn->vals->req_capabilities); + + /* For stats */ + conn->connection_type = conn->dialect; + + rsp->MaxTransactSize = cpu_to_le32(conn->vals->max_trans_size); + rsp->MaxReadSize = cpu_to_le32(conn->vals->max_read_size); + rsp->MaxWriteSize = cpu_to_le32(conn->vals->max_write_size); + + memcpy(conn->ClientGUID, req->ClientGUID, + SMB2_CLIENT_GUID_SIZE); + conn->cli_sec_mode = le16_to_cpu(req->SecurityMode); + + rsp->StructureSize = cpu_to_le16(65); + rsp->DialectRevision = cpu_to_le16(conn->dialect); + /* Not setting conn guid rsp->ServerGUID, as it + * not used by client for identifying server + */ + memset(rsp->ServerGUID, 0, SMB2_CLIENT_GUID_SIZE); + + rsp->SystemTime = cpu_to_le64(ksmbd_systime()); + rsp->ServerStartTime = 0; + ksmbd_debug(SMB, "negotiate context offset %d, count %d\n", + le32_to_cpu(rsp->NegotiateContextOffset), + le16_to_cpu(rsp->NegotiateContextCount)); + + rsp->SecurityBufferOffset = cpu_to_le16(128); + rsp->SecurityBufferLength = cpu_to_le16(AUTH_GSS_LENGTH); + ksmbd_copy_gss_neg_header((char *)(&rsp->hdr) + + le16_to_cpu(rsp->SecurityBufferOffset)); + + rsp->SecurityMode = SMB2_NEGOTIATE_SIGNING_ENABLED_LE; + conn->use_spnego = true; + + if (req->SecurityMode & SMB2_NEGOTIATE_SIGNING_REQUIRED_LE) + conn->sign = true; + if (server_conf.signing == KSMBD_CONFIG_OPT_MANDATORY) { + server_conf.enforced_signing = true; + rsp->SecurityMode |= SMB2_NEGOTIATE_SIGNING_REQUIRED_LE; + conn->sign = true; + } + + conn->srv_sec_mode = le16_to_cpu(rsp->SecurityMode); + ksmbd_conn_set_need_setup(conn); + +err_out: + if (rc && rsp->hdr.Status == STATUS_SUCCESS) + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + + if (!rc) + rc = ksmbd_iov_pin_rsp(work, rsp, + sizeof(struct smb2_negotiate_rsp) + + AUTH_GSS_LENGTH + neg_ctxt_len); + if (rc < 0) + smb2_set_err_rsp(work); + return rc; +} + +static int alloc_preauth_hash(struct ksmbd_session *sess, + struct ksmbd_conn *conn) +{ + if (sess->Preauth_HashValue) + return 0; + + if (!conn->preauth_info) + return -ENOMEM; + + sess->Preauth_HashValue = kmemdup(conn->preauth_info->Preauth_HashValue, + PREAUTH_HASHVALUE_SIZE, KSMBD_DEFAULT_GFP); + if (!sess->Preauth_HashValue) + return -ENOMEM; + + return 0; +} + +static int generate_preauth_hash(struct ksmbd_work *work) +{ + struct ksmbd_conn *conn = work->conn; + struct ksmbd_session *sess = work->sess; + u8 *preauth_hash; + + if (conn->dialect != SMB311_PROT_ID) + return 0; + + if (conn->binding) { + struct preauth_session *preauth_sess; + + preauth_sess = ksmbd_preauth_session_lookup(conn, sess->id); + if (!preauth_sess) { + preauth_sess = ksmbd_preauth_session_alloc(conn, sess->id); + if (!preauth_sess) + return -ENOMEM; + } + + preauth_hash = preauth_sess->Preauth_HashValue; + } else { + if (!sess->Preauth_HashValue) + if (alloc_preauth_hash(sess, conn)) + return -ENOMEM; + preauth_hash = sess->Preauth_HashValue; + } + + ksmbd_gen_preauth_integrity_hash(conn, work->request_buf, preauth_hash); + return 0; +} + +static int decode_negotiation_token(struct ksmbd_conn *conn, + struct negotiate_message *negblob, + size_t sz) +{ + if (!conn->use_spnego) + return -EINVAL; + + if (ksmbd_decode_negTokenInit((char *)negblob, sz, conn)) { + if (ksmbd_decode_negTokenTarg((char *)negblob, sz, conn)) { + conn->auth_mechs |= KSMBD_AUTH_NTLMSSP; + conn->preferred_auth_mech = KSMBD_AUTH_NTLMSSP; + conn->use_spnego = false; + } + } + return 0; +} + +static int ntlm_negotiate(struct ksmbd_work *work, + struct negotiate_message *negblob, + size_t negblob_len, struct smb2_sess_setup_rsp *rsp) +{ + struct challenge_message *chgblob; + unsigned char *spnego_blob = NULL; + u16 spnego_blob_len; + char *neg_blob; + int sz, rc; + + ksmbd_debug(SMB, "negotiate phase\n"); + rc = ksmbd_decode_ntlmssp_neg_blob(negblob, negblob_len, work->conn); + if (rc) + return rc; + + sz = le16_to_cpu(rsp->SecurityBufferOffset); + chgblob = (struct challenge_message *)rsp->Buffer; + memset(chgblob, 0, sizeof(struct challenge_message)); + + if (!work->conn->use_spnego) { + sz = ksmbd_build_ntlmssp_challenge_blob(chgblob, work->conn); + if (sz < 0) + return -ENOMEM; + + rsp->SecurityBufferLength = cpu_to_le16(sz); + return 0; + } + + sz = sizeof(struct challenge_message); + sz += (strlen(ksmbd_netbios_name()) * 2 + 1 + 4) * 6; + + neg_blob = kzalloc(sz, KSMBD_DEFAULT_GFP); + if (!neg_blob) + return -ENOMEM; + + chgblob = (struct challenge_message *)neg_blob; + sz = ksmbd_build_ntlmssp_challenge_blob(chgblob, work->conn); + if (sz < 0) { + rc = -ENOMEM; + goto out; + } + + rc = build_spnego_ntlmssp_neg_blob(&spnego_blob, &spnego_blob_len, + neg_blob, sz); + if (rc) { + rc = -ENOMEM; + goto out; + } + + memcpy(rsp->Buffer, spnego_blob, spnego_blob_len); + rsp->SecurityBufferLength = cpu_to_le16(spnego_blob_len); + +out: + kfree(spnego_blob); + kfree(neg_blob); + return rc; +} + +static struct authenticate_message *user_authblob(struct ksmbd_conn *conn, + struct smb2_sess_setup_req *req) +{ + int sz; + + if (conn->use_spnego && conn->mechToken) + return (struct authenticate_message *)conn->mechToken; + + sz = le16_to_cpu(req->SecurityBufferOffset); + return (struct authenticate_message *)((char *)&req->hdr.ProtocolId + + sz); +} + +static struct ksmbd_user *session_user(struct ksmbd_conn *conn, + struct smb2_sess_setup_req *req) +{ + struct authenticate_message *authblob; + struct ksmbd_user *user; + char *name; + unsigned int name_off, name_len, secbuf_len; + + if (conn->use_spnego && conn->mechToken) + secbuf_len = conn->mechTokenLen; + else + secbuf_len = le16_to_cpu(req->SecurityBufferLength); + if (secbuf_len < sizeof(struct authenticate_message)) { + ksmbd_debug(SMB, "blob len %d too small\n", secbuf_len); + return NULL; + } + authblob = user_authblob(conn, req); + name_off = le32_to_cpu(authblob->UserName.BufferOffset); + name_len = le16_to_cpu(authblob->UserName.Length); + + if (secbuf_len < (u64)name_off + name_len) + return NULL; + + name = smb_strndup_from_utf16((const char *)authblob + name_off, + name_len, + true, + conn->local_nls); + if (IS_ERR(name)) { + pr_err("cannot allocate memory\n"); + return NULL; + } + + ksmbd_debug(SMB, "session setup request for user %s\n", name); + user = ksmbd_login_user(name); + kfree(name); + return user; +} + +static int ntlm_authenticate(struct ksmbd_work *work, + struct smb2_sess_setup_req *req, + struct smb2_sess_setup_rsp *rsp) +{ + struct ksmbd_conn *conn = work->conn; + struct ksmbd_session *sess = work->sess; + struct ksmbd_user *user; + char channel_key[CIFS_KEY_SIZE] = {}; + char *auth_key = conn->binding ? channel_key : sess->sess_key; + u64 prev_id; + bool binding = conn->binding; + int sz, rc; + + ksmbd_debug(SMB, "authenticate phase\n"); + if (conn->use_spnego) { + unsigned char *spnego_blob; + u16 spnego_blob_len; + + rc = build_spnego_ntlmssp_auth_blob(&spnego_blob, + &spnego_blob_len, + 0); + if (rc) + return -ENOMEM; + + memcpy(rsp->Buffer, spnego_blob, spnego_blob_len); + rsp->SecurityBufferLength = cpu_to_le16(spnego_blob_len); + kfree(spnego_blob); + } + + user = session_user(conn, req); + if (!user) { + ksmbd_debug(SMB, "Unknown user name or an error\n"); + return -EPERM; + } + + if (sess->state == SMB2_SESSION_VALID) { + /* + * Reuse session if anonymous try to connect + * on reauthetication. + */ + if (conn->binding == false && ksmbd_anonymous_user(user)) { + ksmbd_free_user(user); + return 0; + } + + if (!ksmbd_compare_user(sess->user, user)) { + ksmbd_free_user(user); + return -EKEYREJECTED; + } + ksmbd_free_user(user); + } else { + sess->user = user; + } + + if (conn->binding == false && user_guest(sess->user)) { + rsp->SessionFlags = SMB2_SESSION_FLAG_IS_GUEST_LE; + } else { + struct authenticate_message *authblob; + + authblob = user_authblob(conn, req); + if (conn->use_spnego && conn->mechToken) + sz = conn->mechTokenLen; + else + sz = le16_to_cpu(req->SecurityBufferLength); + rc = ksmbd_decode_ntlmssp_auth_blob(authblob, sz, conn, sess, + auth_key); + if (rc) { + set_user_flag(sess->user, KSMBD_USER_FLAG_BAD_PASSWORD); + ksmbd_debug(SMB, "authentication failed\n"); + rc = -EPERM; + goto out; + } + } + + prev_id = le64_to_cpu(req->PreviousSessionId); + if (prev_id && prev_id != sess->id) + destroy_previous_session(conn, sess->user, prev_id); + + /* + * If session state is SMB2_SESSION_VALID, We can assume + * that it is reauthentication. And the user/password + * has been verified, so return it here. + */ + if (sess->state == SMB2_SESSION_VALID) { + if (conn->binding) + goto binding_session; + return 0; + } + + if ((rsp->SessionFlags != SMB2_SESSION_FLAG_IS_GUEST_LE && + (conn->sign || server_conf.enforced_signing)) || + (req->SecurityMode & SMB2_NEGOTIATE_SIGNING_REQUIRED)) + sess->sign = true; + + if (smb3_encryption_negotiated(conn) && + !(req->Flags & SMB2_SESSION_REQ_FLAG_BINDING)) { + conn->ops->generate_encryptionkey(conn, sess); + sess->enc = true; + if (server_conf.flags & KSMBD_GLOBAL_FLAG_SMB2_ENCRYPTION) + rsp->SessionFlags = SMB2_SESSION_FLAG_ENCRYPT_DATA_LE; + /* + * signing is disable if encryption is enable + * on this session + */ + sess->sign = false; + } + +binding_session: + if (conn->dialect >= SMB30_PROT_ID) { + rc = register_session_channel(sess, conn, auth_key); + if (rc) + goto out; + } + + if (conn->ops->generate_signingkey) { + rc = conn->ops->generate_signingkey(sess, conn); + if (rc) { + ksmbd_debug(SMB, "SMB3 signing key generation failed\n"); + rc = -EINVAL; + goto out; + } + } + + if (!ksmbd_conn_lookup_dialect(conn)) { + pr_err("fail to verify the dialect\n"); + rc = -ENOENT; + goto out; + } + rc = 0; +out: + if (binding) + memzero_explicit(channel_key, sizeof(channel_key)); + return rc; +} + +#ifdef CONFIG_SMB_SERVER_KERBEROS5 +static int krb5_authenticate(struct ksmbd_work *work, + struct smb2_sess_setup_req *req, + struct smb2_sess_setup_rsp *rsp) +{ + struct ksmbd_conn *conn = work->conn; + struct ksmbd_session *sess = work->sess; + char *in_blob, *out_blob; + char channel_key[CIFS_KEY_SIZE] = {}; + char reauth_key[CIFS_KEY_SIZE] = {}; + char *auth_key = conn->binding ? channel_key : + (work->session_setup_reauth ? reauth_key : sess->sess_key); + u64 prev_sess_id; + bool binding = conn->binding; + int in_len, out_len; + int retval; + + in_blob = (char *)&req->hdr.ProtocolId + + le16_to_cpu(req->SecurityBufferOffset); + in_len = le16_to_cpu(req->SecurityBufferLength); + out_blob = (char *)&rsp->hdr.ProtocolId + + le16_to_cpu(rsp->SecurityBufferOffset); + out_len = work->response_sz - work->next_smb2_rsp_hdr_off - + (le16_to_cpu(rsp->SecurityBufferOffset) + 4); + + retval = ksmbd_krb5_authenticate(sess, in_blob, in_len, + out_blob, &out_len, auth_key); + if (retval) { + ksmbd_debug(SMB, "krb5 authentication failed\n"); + if (retval != -EKEYREJECTED) + retval = -EPERM; + goto out; + } + + /* Check previous session */ + prev_sess_id = le64_to_cpu(req->PreviousSessionId); + if (prev_sess_id && prev_sess_id != sess->id) + destroy_previous_session(conn, sess->user, prev_sess_id); + + rsp->SecurityBufferLength = cpu_to_le16(out_len); + + /* + * If session state is SMB2_SESSION_VALID, We can assume + * that it is reauthentication. And the user/password + * has been verified, so return it here. + */ + if (sess->state == SMB2_SESSION_VALID && !work->session_setup_reauth) { + if (conn->binding) + goto binding_session; + return 0; + } + + /* + * Reauthentication verifies the new Kerberos credentials but keeps + * the established SMB session keys. + */ + if (work->session_setup_reauth) { + retval = 0; + goto out; + } + + if ((rsp->SessionFlags != SMB2_SESSION_FLAG_IS_GUEST_LE && + (conn->sign || server_conf.enforced_signing)) || + (req->SecurityMode & SMB2_NEGOTIATE_SIGNING_REQUIRED)) + sess->sign = true; + + if (smb3_encryption_negotiated(conn) && + !(req->Flags & SMB2_SESSION_REQ_FLAG_BINDING)) { + conn->ops->generate_encryptionkey(conn, sess); + sess->enc = true; + if (server_conf.flags & KSMBD_GLOBAL_FLAG_SMB2_ENCRYPTION) + rsp->SessionFlags = SMB2_SESSION_FLAG_ENCRYPT_DATA_LE; + sess->sign = false; + } + +binding_session: + if (conn->dialect >= SMB30_PROT_ID) { + retval = register_session_channel(sess, conn, auth_key); + if (retval) + goto out; + } + + if (conn->ops->generate_signingkey) { + retval = conn->ops->generate_signingkey(sess, conn); + if (retval) { + ksmbd_debug(SMB, "SMB3 signing key generation failed\n"); + retval = -EINVAL; + goto out; + } + } + + if (!ksmbd_conn_lookup_dialect(conn)) { + pr_err("fail to verify the dialect\n"); + retval = -ENOENT; + goto out; + } + retval = 0; +out: + memzero_explicit(reauth_key, sizeof(reauth_key)); + if (binding) + memzero_explicit(channel_key, sizeof(channel_key)); + return retval; +} +#else +static int krb5_authenticate(struct ksmbd_work *work, + struct smb2_sess_setup_req *req, + struct smb2_sess_setup_rsp *rsp) +{ + return -EOPNOTSUPP; +} +#endif + +int smb2_sess_setup(struct ksmbd_work *work) +{ + struct ksmbd_conn *conn = work->conn; + struct smb2_sess_setup_req *req; + struct smb2_sess_setup_rsp *rsp; + struct ksmbd_session *sess; + struct negotiate_message *negblob; + unsigned int negblob_len, negblob_off; + int rc = 0; + + ksmbd_debug(SMB, "Received smb2 session setup request\n"); + + if (!ksmbd_conn_need_setup(conn) && !ksmbd_conn_good(conn)) { + work->send_no_response = 1; + return rc; + } + + WORK_BUFFERS(work, req, rsp); + + rsp->StructureSize = cpu_to_le16(9); + rsp->SessionFlags = 0; + rsp->SecurityBufferOffset = cpu_to_le16(72); + rsp->SecurityBufferLength = 0; + + ksmbd_conn_lock(conn); + if (!req->hdr.SessionId) { + sess = ksmbd_smb2_session_create(); + if (!sess) { + rc = -ENOMEM; + goto out_err; + } + rsp->hdr.SessionId = cpu_to_le64(sess->id); + rc = ksmbd_session_register(conn, sess); + if (rc) + goto out_err; + + conn->binding = false; + } else if (conn->dialect >= SMB30_PROT_ID && + (server_conf.flags & KSMBD_GLOBAL_FLAG_SMB3_MULTICHANNEL) && + req->Flags & SMB2_SESSION_REQ_FLAG_BINDING) { + u64 sess_id = le64_to_cpu(req->hdr.SessionId); + + sess = ksmbd_session_lookup_slowpath(sess_id); + if (!sess) { + rc = -ENOENT; + goto out_err; + } + + if (conn->dialect != sess->dialect) { + rc = -EINVAL; + goto out_err; + } + + if (conn->dialect == SMB311_PROT_ID) { + struct channel *chann; + unsigned long index; + + down_read(&sess->chann_lock); + xa_for_each(&sess->ksmbd_chann_list, index, chann) { + if (conn->cipher_type != chann->conn->cipher_type) + rc = -EINVAL; + break; + } + up_read(&sess->chann_lock); + if (rc) + goto out_err; + } + + if (!(req->hdr.Flags & SMB2_FLAGS_SIGNED)) { + rc = -EINVAL; + goto out_err; + } + + if (memcmp(conn->ClientGUID, sess->ClientGUID, + SMB2_CLIENT_GUID_SIZE)) { + rc = -ENOENT; + goto out_err; + } + + if (sess->state == SMB2_SESSION_IN_PROGRESS) { + rc = -EACCES; + goto out_err; + } + + if (sess->state == SMB2_SESSION_EXPIRED) { + rc = -EFAULT; + goto out_err; + } + + if (ksmbd_conn_need_reconnect(conn)) { + rc = -EFAULT; + ksmbd_user_session_put(sess); + sess = NULL; + goto out_err; + } + + if (is_ksmbd_session_in_connection(conn, sess_id)) { + rc = -EACCES; + goto out_err; + } + + if (user_guest(sess->user)) { + rc = -EOPNOTSUPP; + goto out_err; + } + + conn->binding = true; + } else if ((conn->dialect < SMB30_PROT_ID || + server_conf.flags & KSMBD_GLOBAL_FLAG_SMB3_MULTICHANNEL) && + (req->Flags & SMB2_SESSION_REQ_FLAG_BINDING)) { + sess = ksmbd_session_lookup_slowpath(le64_to_cpu(req->hdr.SessionId)); + if (sess) { + int sign_ret; + + work->sess = sess; + if (sess->dialect >= SMB30_PROT_ID) + sign_ret = smb3_check_sign_req(work); + else + sign_ret = smb2_check_sign_req(work); + if (sess->state != SMB2_SESSION_VALID || + !(req->hdr.Flags & SMB2_FLAGS_SIGNED) || + !sign_ret) { + ksmbd_user_session_put(sess); + work->sess = NULL; + sess = NULL; + } + } + rc = -EACCES; + goto out_err; + } else { + sess = ksmbd_session_lookup(conn, + le64_to_cpu(req->hdr.SessionId)); + if (!sess) { + sess = ksmbd_session_lookup_slowpath(le64_to_cpu(req->hdr.SessionId)); + if (sess && !lookup_chann_list(sess, conn)) { + ksmbd_user_session_put(sess); + sess = NULL; + } + } + if (!sess) { + rc = -ENOENT; + goto out_err; + } + + if (sess->state == SMB2_SESSION_EXPIRED) { + if (sess->kerberos_expiry && + ktime_get_real_seconds() >= sess->kerberos_expiry) { + work->session_setup_reauth = true; + } else { + rc = -EFAULT; + goto out_err; + } + } + + if (ksmbd_conn_need_reconnect(conn)) { + rc = -EFAULT; + ksmbd_user_session_put(sess); + sess = NULL; + goto out_err; + } + + conn->binding = false; + } + work->sess = sess; + + negblob_off = le16_to_cpu(req->SecurityBufferOffset); + negblob_len = le16_to_cpu(req->SecurityBufferLength); + if (negblob_off < offsetof(struct smb2_sess_setup_req, Buffer)) { + rc = -EINVAL; + goto out_err; + } + + negblob = (struct negotiate_message *)((char *)&req->hdr.ProtocolId + + negblob_off); + + if (decode_negotiation_token(conn, negblob, negblob_len) == 0) { + if (conn->mechToken) { + negblob = (struct negotiate_message *)conn->mechToken; + negblob_len = conn->mechTokenLen; + } + } + + if (negblob_len < offsetof(struct negotiate_message, NegotiateFlags)) { + rc = -EINVAL; + goto out_err; + } + + if (server_conf.auth_mechs & conn->auth_mechs) { + rc = generate_preauth_hash(work); + if (rc) + goto out_err; + + if (conn->preferred_auth_mech & + (KSMBD_AUTH_KRB5 | KSMBD_AUTH_MSKRB5)) { + rc = krb5_authenticate(work, req, rsp); + if (rc) + goto out_err; + + if (!ksmbd_conn_need_reconnect(conn)) { + ksmbd_conn_set_good(conn); + sess->state = SMB2_SESSION_VALID; + } + } else if (conn->preferred_auth_mech == KSMBD_AUTH_NTLMSSP) { + if (negblob->MessageType == NtLmNegotiate) { + rc = ntlm_negotiate(work, negblob, negblob_len, rsp); + if (rc) + goto out_err; + rsp->hdr.Status = + STATUS_MORE_PROCESSING_REQUIRED; + } else if (negblob->MessageType == NtLmAuthenticate) { + rc = ntlm_authenticate(work, req, rsp); + if (rc) + goto out_err; + + if (!ksmbd_conn_need_reconnect(conn)) { + ksmbd_conn_set_good(conn); + sess->state = SMB2_SESSION_VALID; + } + if (conn->binding) { + struct preauth_session *preauth_sess; + + preauth_sess = + ksmbd_preauth_session_lookup(conn, sess->id); + if (preauth_sess) { + list_del(&preauth_sess->preauth_entry); + kfree_sensitive(preauth_sess); + } + } + } else { + pr_info_ratelimited("Unknown NTLMSSP message type : 0x%x\n", + le32_to_cpu(negblob->MessageType)); + rc = -EINVAL; + } + } else { + /* TODO: need one more negotiation */ + pr_err("Not support the preferred authentication\n"); + rc = -EINVAL; + } + } else { + pr_err("Not support authentication\n"); + rc = -EINVAL; + } + +out_err: + if (rc == -EINVAL) + rsp->hdr.Status = STATUS_INVALID_PARAMETER; + else if (rc == -ENOENT) + rsp->hdr.Status = STATUS_USER_SESSION_DELETED; + else if (rc == -EACCES) + rsp->hdr.Status = STATUS_REQUEST_NOT_ACCEPTED; + else if (rc == -EFAULT) + rsp->hdr.Status = STATUS_NETWORK_SESSION_EXPIRED; + else if (rc == -ENOMEM || rc == -ENOSPC) + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + else if (rc == -EOPNOTSUPP) + rsp->hdr.Status = STATUS_NOT_SUPPORTED; + else if (rc == -EKEYREJECTED) + rsp->hdr.Status = STATUS_ACCESS_DENIED; + else if (rc) + rsp->hdr.Status = STATUS_LOGON_FAILURE; + if ((rsp->hdr.Status == STATUS_USER_SESSION_DELETED || + (rsp->hdr.Status == STATUS_INVALID_PARAMETER && + (req->Flags & SMB2_SESSION_REQ_FLAG_BINDING))) && + (req->hdr.Flags & SMB2_FLAGS_SIGNED)) + rsp->hdr.Flags |= SMB2_FLAGS_SIGNED; + + if (conn->mechToken) { + kfree(conn->mechToken); + conn->mechToken = NULL; + } + + if (rc < 0) { + if (sess && conn->dialect == SMB311_PROT_ID && + (req->Flags & SMB2_SESSION_REQ_FLAG_BINDING)) { + struct preauth_session *preauth_sess; + + preauth_sess = ksmbd_preauth_session_lookup(conn, sess->id); + if (preauth_sess) { + list_del(&preauth_sess->preauth_entry); + kfree_sensitive(preauth_sess); + } + } + + /* + * SecurityBufferOffset should be set to zero + * in session setup error response. + */ + rsp->SecurityBufferOffset = 0; + + if (sess) { + bool try_delay = false; + + /* + * To avoid dictionary attacks (repeated session setups rapidly sent) to + * connect to server, ksmbd make a delay of a 5 seconds on session setup + * failure to make it harder to send enough random connection requests + * to break into a server. + */ + if (sess->user && sess->user->flags & KSMBD_USER_FLAG_DELAY_SESSION) + try_delay = true; + + /* + * For binding requests, session belongs to another + * connection. Do not expire it. + */ + if (!(req->Flags & SMB2_SESSION_REQ_FLAG_BINDING)) { + sess->last_active = jiffies; + sess->kerberos_expiry = 0; + sess->state = SMB2_SESSION_EXPIRED; + } + /* + * Keep the binding session reference until the response is + * signed and sent. Error responses for a signed binding + * request are signed with the existing session signing key. + */ + if (!(req->Flags & SMB2_SESSION_REQ_FLAG_BINDING) || + work->sess != sess) { + ksmbd_user_session_put(sess); + work->sess = NULL; + } + if (try_delay) { + ksmbd_conn_set_need_reconnect(conn); + ssleep(5); + ksmbd_conn_set_need_setup(conn); + } + } + smb2_set_err_rsp(work); + conn->binding = false; + } else { + unsigned int iov_len; + + if (rsp->SecurityBufferLength) + iov_len = offsetof(struct smb2_sess_setup_rsp, Buffer) + + le16_to_cpu(rsp->SecurityBufferLength); + else + iov_len = sizeof(struct smb2_sess_setup_rsp); + rc = ksmbd_iov_pin_rsp(work, rsp, iov_len); + if (rc) + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + } + + ksmbd_conn_unlock(conn); + return rc; +} + +/** + * smb2_tree_connect() - handler for smb2 tree connect command + * @work: smb work containing smb request buffer + * + * Return: 0 on success, otherwise error + */ +int smb2_tree_connect(struct ksmbd_work *work) +{ + struct ksmbd_conn *conn = work->conn; + struct smb2_tree_connect_req *req; + struct smb2_tree_connect_rsp *rsp; + struct ksmbd_session *sess = work->sess; + char *treename = NULL, *name = NULL; + struct ksmbd_tree_conn_status status; + struct ksmbd_share_config *share = NULL; + int rc = -EINVAL; + + ksmbd_debug(SMB, "Received smb2 tree connect request\n"); + + WORK_BUFFERS(work, req, rsp); + + treename = smb_strndup_from_utf16((char *)req + le16_to_cpu(req->PathOffset), + le16_to_cpu(req->PathLength), true, + conn->local_nls); + if (IS_ERR(treename)) { + pr_err("treename is NULL\n"); + status.ret = KSMBD_TREE_CONN_STATUS_ERROR; + goto out_err1; + } + + name = ksmbd_extract_sharename(conn->um, treename); + if (IS_ERR(name)) { + status.ret = KSMBD_TREE_CONN_STATUS_ERROR; + goto out_err1; + } + + ksmbd_debug(SMB, "tree connect request for tree %s treename %s\n", + name, treename); + + status = ksmbd_tree_conn_connect(work, name); + if (status.ret == KSMBD_TREE_CONN_STATUS_OK) { + rsp->hdr.Id.SyncId.TreeId = cpu_to_le32(status.tree_conn->id); + share = status.tree_conn->share_conf; + + /* A share that requires encryption needs a negotiated SMB3 cipher. */ + if (test_share_config_flag(share, KSMBD_SHARE_FLAG_ENCRYPT_DATA) && + !smb3_encryption_negotiated(conn)) { + ksmbd_tree_conn_disconnect(sess, status.tree_conn); + status.tree_conn = NULL; + share = NULL; + status.ret = KSMBD_TREE_CONN_STATUS_ERROR; + goto out_err1; + } + } else + goto out_err1; + + if (test_share_config_flag(share, KSMBD_SHARE_FLAG_PIPE)) { + ksmbd_debug(SMB, "IPC share path request\n"); + rsp->ShareType = SMB2_SHARE_TYPE_PIPE; + rsp->MaximalAccess = FILE_READ_DATA_LE | FILE_READ_EA_LE | + FILE_EXECUTE_LE | FILE_READ_ATTRIBUTES_LE | + FILE_DELETE_LE | FILE_READ_CONTROL_LE | + FILE_WRITE_DAC_LE | FILE_WRITE_OWNER_LE | + FILE_SYNCHRONIZE_LE; + } else { + rsp->ShareType = SMB2_SHARE_TYPE_DISK; + rsp->MaximalAccess = FILE_READ_DATA_LE | FILE_READ_EA_LE | + FILE_EXECUTE_LE | FILE_READ_ATTRIBUTES_LE; + if (test_tree_conn_flag(status.tree_conn, + KSMBD_TREE_CONN_FLAG_WRITABLE)) { + rsp->MaximalAccess |= FILE_WRITE_DATA_LE | + FILE_APPEND_DATA_LE | FILE_WRITE_EA_LE | + FILE_DELETE_LE | FILE_WRITE_ATTRIBUTES_LE | + FILE_DELETE_CHILD_LE | FILE_READ_CONTROL_LE | + FILE_WRITE_DAC_LE | FILE_WRITE_OWNER_LE | + FILE_SYNCHRONIZE_LE; + } + } + + status.tree_conn->maximal_access = le32_to_cpu(rsp->MaximalAccess); + if (conn->posix_ext_supported) + status.tree_conn->posix_extensions = true; + + down_write(&sess->tree_conns_lock); + status.tree_conn->t_state = TREE_CONNECTED; + up_write(&sess->tree_conns_lock); + rsp->StructureSize = cpu_to_le16(16); +out_err1: + /* + * A configured CA share is not continuously available until persistent + * open recovery, ownership fencing, and failover are implemented. + */ + rsp->Capabilities = 0; + rsp->Reserved = 0; + /* default manual caching */ + rsp->ShareFlags = SMB2_SHAREFLAG_MANUAL_CACHING; + /* Tell the client that READ requests may request compressed responses. */ + if (conn->dialect == SMB311_PROT_ID && + conn->compress_algorithm != SMB3_COMPRESS_NONE) + rsp->ShareFlags |= cpu_to_le32(SMB2_SHAREFLAG_COMPRESS_DATA); + if (share && test_share_config_flag(share, + KSMBD_SHARE_FLAG_HIDE_UNREADABLE)) + rsp->ShareFlags |= + cpu_to_le32(SMB2_SHAREFLAG_ACCESS_BASED_DIRECTORY_ENUM); + if (share && test_share_config_flag(share, + KSMBD_SHARE_FLAG_ENCRYPT_DATA)) + rsp->ShareFlags |= + cpu_to_le32(SMB2_SHAREFLAG_ENCRYPT_DATA); + + rc = ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_tree_connect_rsp)); + if (rc) { + if (status.ret == KSMBD_TREE_CONN_STATUS_OK) { + down_write(&sess->tree_conns_lock); + status.tree_conn->t_state = TREE_DISCONNECTED; + up_write(&sess->tree_conns_lock); + ksmbd_tree_conn_disconnect(sess, status.tree_conn); + status.tree_conn = NULL; + } + status.ret = KSMBD_TREE_CONN_STATUS_NOMEM; + } + + if (!IS_ERR(treename)) + kfree(treename); + if (!IS_ERR(name)) + kfree(name); + + switch (status.ret) { + case KSMBD_TREE_CONN_STATUS_OK: + rsp->hdr.Status = STATUS_SUCCESS; + rc = 0; + break; + case -ESTALE: + case -ENOENT: + case KSMBD_TREE_CONN_STATUS_NO_SHARE: + rsp->hdr.Status = STATUS_BAD_NETWORK_NAME; + break; + case -ENOMEM: + case KSMBD_TREE_CONN_STATUS_NOMEM: + rsp->hdr.Status = STATUS_NO_MEMORY; + break; + case KSMBD_TREE_CONN_STATUS_ERROR: + case KSMBD_TREE_CONN_STATUS_TOO_MANY_CONNS: + case KSMBD_TREE_CONN_STATUS_TOO_MANY_SESSIONS: + rsp->hdr.Status = STATUS_ACCESS_DENIED; + break; + case -EINVAL: + rsp->hdr.Status = STATUS_INVALID_PARAMETER; + break; + default: + rsp->hdr.Status = STATUS_ACCESS_DENIED; + } + + if (status.ret != KSMBD_TREE_CONN_STATUS_OK) + smb2_set_err_rsp(work); + + return rc; +} + +/** + * smb2_create_open_flags() - convert smb open flags to unix open flags + * @file_present: is file already present + * @access: file access flags + * @disposition: file disposition flags + * @may_flags: set with MAY_ flags + * @coptions: file creation options + * @mode: file mode + * + * Return: file open flags + */ +static int smb2_create_open_flags(bool file_present, __le32 access, + __le32 disposition, + int *may_flags, + __le32 coptions, + umode_t mode) +{ + int oflags = O_NONBLOCK | O_LARGEFILE; + + if (coptions & FILE_DIRECTORY_FILE_LE || S_ISDIR(mode)) { + access &= ~FILE_WRITE_DESIRE_ACCESS_LE; + ksmbd_debug(SMB, "Discard write access to a directory\n"); + } + + if (access & FILE_READ_DESIRED_ACCESS_LE && + access & FILE_WRITE_DESIRE_ACCESS_LE) { + oflags |= O_RDWR; + *may_flags = MAY_OPEN | MAY_READ | MAY_WRITE; + } else if (access & FILE_WRITE_DESIRE_ACCESS_LE) { + oflags |= O_WRONLY; + *may_flags = MAY_OPEN | MAY_WRITE; + } else { + oflags |= O_RDONLY; + *may_flags = MAY_OPEN | MAY_READ; + } + + if (access == FILE_READ_ATTRIBUTES_LE || S_ISBLK(mode) || S_ISCHR(mode)) + oflags |= O_PATH; + + if (file_present) { + switch (disposition & FILE_CREATE_MASK_LE) { + case FILE_OPEN_LE: + case FILE_CREATE_LE: + break; + case FILE_SUPERSEDE_LE: + case FILE_OVERWRITE_LE: + case FILE_OVERWRITE_IF_LE: + oflags |= O_TRUNC; + break; + default: + break; + } + } else { + switch (disposition & FILE_CREATE_MASK_LE) { + case FILE_SUPERSEDE_LE: + case FILE_CREATE_LE: + case FILE_OPEN_IF_LE: + case FILE_OVERWRITE_IF_LE: + oflags |= O_CREAT; + break; + case FILE_OPEN_LE: + case FILE_OVERWRITE_LE: + oflags &= ~O_CREAT; + break; + default: + break; + } + } + + return oflags; +} + +/** + * smb2_tree_disconnect() - handler for smb tree connect request + * @work: smb work containing request buffer + * + * Return: 0 on success, otherwise error + */ +int smb2_tree_disconnect(struct ksmbd_work *work) +{ + struct smb2_tree_disconnect_rsp *rsp; + struct smb2_tree_disconnect_req *req; + struct ksmbd_session *sess = work->sess; + struct ksmbd_tree_connect *tcon = work->tcon; + int err; + + ksmbd_debug(SMB, "Received smb2 tree disconnect request\n"); + + WORK_BUFFERS(work, req, rsp); + + if (!tcon) { + ksmbd_debug(SMB, "Invalid tid %d\n", req->hdr.Id.SyncId.TreeId); + + rsp->hdr.Status = STATUS_NETWORK_NAME_DELETED; + err = -ENOENT; + goto err_out; + } + + ksmbd_close_tree_conn_fds(work); + + down_write(&sess->tree_conns_lock); + if (tcon->t_state == TREE_DISCONNECTED) { + up_write(&sess->tree_conns_lock); + rsp->hdr.Status = STATUS_NETWORK_NAME_DELETED; + err = -ENOENT; + goto err_out; + } + + tcon->t_state = TREE_DISCONNECTED; + up_write(&sess->tree_conns_lock); + + err = ksmbd_tree_conn_disconnect(sess, tcon); + if (err) { + rsp->hdr.Status = STATUS_NETWORK_NAME_DELETED; + goto err_out; + } + + rsp->StructureSize = cpu_to_le16(4); + err = ksmbd_iov_pin_rsp(work, rsp, + sizeof(struct smb2_tree_disconnect_rsp)); + if (err) { + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + goto err_out; + } + + return 0; + +err_out: + smb2_set_err_rsp(work); + return err; + +} + +/** + * smb2_session_logoff() - handler for session log off request + * @work: smb work containing request buffer + * + * Return: 0 on success, otherwise error + */ +int smb2_session_logoff(struct ksmbd_work *work) +{ + struct ksmbd_conn *conn = work->conn; + struct ksmbd_session *sess = work->sess; + struct smb2_logoff_req *req; + struct smb2_logoff_rsp *rsp; + int err; + + WORK_BUFFERS(work, req, rsp); + + ksmbd_debug(SMB, "Received smb2 session logoff request\n"); + + ksmbd_conn_lock(conn); + if (!ksmbd_conn_good(conn)) { + ksmbd_conn_unlock(conn); + rsp->hdr.Status = STATUS_NETWORK_NAME_DELETED; + smb2_set_err_rsp(work); + return -ENOENT; + } + ksmbd_all_conn_set_status(sess, KSMBD_SESS_NEED_RECONNECT); + ksmbd_conn_unlock(conn); + + ksmbd_close_session_fds(work); + ksmbd_conn_wait_idle(conn); + + if (ksmbd_tree_conn_session_logoff(sess)) { + ksmbd_debug(SMB, "Invalid tid %d\n", req->hdr.Id.SyncId.TreeId); + rsp->hdr.Status = STATUS_NETWORK_NAME_DELETED; + smb2_set_err_rsp(work); + return -ENOENT; + } + + down_write(&conn->session_lock); + sess->kerberos_expiry = 0; + sess->state = SMB2_SESSION_EXPIRED; + up_write(&conn->session_lock); + + ksmbd_all_conn_set_status(sess, KSMBD_SESS_NEED_SETUP); + + rsp->StructureSize = cpu_to_le16(4); + err = ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_logoff_rsp)); + if (err) { + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + smb2_set_err_rsp(work); + return err; + } + return 0; +} + +/** + * create_smb2_pipe() - create IPC pipe + * @work: smb work containing request buffer + * + * Return: 0 on success, otherwise error + */ +static noinline int create_smb2_pipe(struct ksmbd_work *work) +{ + struct smb2_create_rsp *rsp; + struct smb2_create_req *req; + int id = -1; + int err; + char *name; + + WORK_BUFFERS(work, req, rsp); + + name = smb_strndup_from_utf16(req->Buffer, le16_to_cpu(req->NameLength), + 1, work->conn->local_nls); + if (IS_ERR(name)) { + rsp->hdr.Status = STATUS_NO_MEMORY; + err = PTR_ERR(name); + goto out; + } + + id = ksmbd_session_rpc_open(work->sess, name); + if (id < 0) { + /* + * mdssvc (Spotlight) is a routine, expected probe from macOS + * that we deliberately don't support -- it's disabled at the + * __rpc_method() level (mgmt/user_session.c), but this + * generic failure log would otherwise still fire on every + * single probe regardless. + */ + if (!(id == -ENOENT && (!strcmp(name, "\\mdssvc") || + !strcmp(name, "mdssvc")))) + pr_err("Unable to open RPC pipe: %d\n", id); + err = id; + goto out; + } + + rsp->hdr.Status = STATUS_SUCCESS; + rsp->StructureSize = cpu_to_le16(89); + rsp->OplockLevel = SMB2_OPLOCK_LEVEL_NONE; + rsp->Flags = 0; + rsp->CreateAction = cpu_to_le32(FILE_OPENED); + + rsp->CreationTime = cpu_to_le64(0); + rsp->LastAccessTime = cpu_to_le64(0); + rsp->ChangeTime = cpu_to_le64(0); + rsp->AllocationSize = cpu_to_le64(0); + rsp->EndofFile = cpu_to_le64(0); + rsp->FileAttributes = FILE_ATTRIBUTE_NORMAL_LE; + rsp->Reserved2 = 0; + rsp->VolatileFileId = id; + rsp->PersistentFileId = 0; + rsp->CreateContextsOffset = 0; + rsp->CreateContextsLength = 0; + + err = ksmbd_iov_pin_rsp(work, rsp, offsetof(struct smb2_create_rsp, Buffer)); + if (err) + goto out; + + kfree(name); + return 0; + +out: + switch (err) { + case -EINVAL: + rsp->hdr.Status = STATUS_INVALID_PARAMETER; + break; + case -ENOENT: + rsp->hdr.Status = STATUS_OBJECT_NAME_NOT_FOUND; + break; + case -ENOSPC: + case -ENOMEM: + rsp->hdr.Status = STATUS_NO_MEMORY; + break; + } + + if (id >= 0) + ksmbd_session_rpc_close(work->sess, id); + + if (!IS_ERR(name)) + kfree(name); + + smb2_set_err_rsp(work); + return err; +} + +static bool smb2_is_private_ea(const char *name, size_t name_len) +{ + if (name_len == SD_PREFIX_LEN && + !strncasecmp(name, SD_PREFIX, SD_PREFIX_LEN)) + return true; + if (name_len == DOS_ATTRIBUTE_PREFIX_LEN && + !strncasecmp(name, DOS_ATTRIBUTE_PREFIX, + DOS_ATTRIBUTE_PREFIX_LEN)) + return true; + if (name_len >= STREAM_PREFIX_LEN && + !strncasecmp(name, STREAM_PREFIX, STREAM_PREFIX_LEN)) + return true; + + return false; +} + +/** + * smb2_set_ea() - handler for setting extended attributes using set + * info command + * @eabuf: set info command buffer + * @buf_len: set info command buffer length + * @path: dentry path for get ea + * @get_write: get write access to a mount + * + * Return: 0 on success, otherwise error + */ +static int smb2_set_ea(struct smb2_ea_info *eabuf, unsigned int buf_len, + const struct path *path, bool get_write) +{ + struct mnt_idmap *idmap = mnt_idmap(path->mnt); + char *attr_name = NULL, *value; + int rc = 0; + unsigned int next = 0; + + if (buf_len < sizeof(struct smb2_ea_info) + eabuf->EaNameLength + 1 + + le16_to_cpu(eabuf->EaValueLength)) + return -EINVAL; + + attr_name = kmalloc(XATTR_NAME_MAX + 1, KSMBD_DEFAULT_GFP); + if (!attr_name) + return -ENOMEM; + + do { + if (!eabuf->EaNameLength) + goto next; + + ksmbd_debug(SMB, + "name : <%s>, name_len : %u, value_len : %u, next : %u\n", + eabuf->name, eabuf->EaNameLength, + le16_to_cpu(eabuf->EaValueLength), + le32_to_cpu(eabuf->NextEntryOffset)); + + if (eabuf->EaNameLength > + (XATTR_NAME_MAX - XATTR_USER_PREFIX_LEN)) { + rc = -EINVAL; + break; + } + if (smb2_is_private_ea(eabuf->name, eabuf->EaNameLength)) { + rc = -EACCES; + break; + } + + memcpy(attr_name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN); + memcpy(&attr_name[XATTR_USER_PREFIX_LEN], eabuf->name, + eabuf->EaNameLength); + attr_name[XATTR_USER_PREFIX_LEN + eabuf->EaNameLength] = '\0'; + value = (char *)&eabuf->name + eabuf->EaNameLength + 1; + + if (!eabuf->EaValueLength) { + rc = ksmbd_vfs_casexattr_len(idmap, + path->dentry, + attr_name, + XATTR_USER_PREFIX_LEN + + eabuf->EaNameLength); + + /* delete the EA only when it exits */ + if (rc > 0) { + rc = ksmbd_vfs_remove_xattr(idmap, + path, + attr_name, + get_write); + + if (rc < 0) { + ksmbd_debug(SMB, + "remove xattr failed(%d)\n", + rc); + break; + } + } + + /* if the EA doesn't exist, just do nothing. */ + rc = 0; + } else { + rc = ksmbd_vfs_setxattr(idmap, path, attr_name, value, + le16_to_cpu(eabuf->EaValueLength), + 0, get_write); + if (rc < 0) { + ksmbd_debug(SMB, + "ksmbd_vfs_setxattr is failed(%d)\n", + rc); + break; + } + } + +next: + next = le32_to_cpu(eabuf->NextEntryOffset); + if (next == 0 || buf_len < next) + break; + buf_len -= next; + eabuf = (struct smb2_ea_info *)((char *)eabuf + next); + if (buf_len < sizeof(struct smb2_ea_info)) { + rc = -EINVAL; + break; + } + + if (buf_len < sizeof(struct smb2_ea_info) + eabuf->EaNameLength + 1 + + le16_to_cpu(eabuf->EaValueLength)) { + rc = -EINVAL; + break; + } + } while (next != 0); + + kfree(attr_name); + return rc; +} + +static noinline int smb2_set_stream_name_xattr(const struct path *path, + struct ksmbd_file *fp, + char *stream_name, int s_type) +{ + struct mnt_idmap *idmap = mnt_idmap(path->mnt); + size_t xattr_stream_size; + char *xattr_stream_name; + int rc; + + rc = ksmbd_vfs_xattr_stream_name(stream_name, + &xattr_stream_name, + &xattr_stream_size, + s_type); + if (rc) + return rc; + + fp->stream.name = xattr_stream_name; + fp->stream.size = xattr_stream_size; + + /* Check if there is stream prefix in xattr space */ + rc = ksmbd_vfs_casexattr_len(idmap, + path->dentry, + xattr_stream_name, + xattr_stream_size); + if (rc >= 0) + return 0; + + if (fp->cdoption == FILE_OPEN_LE) { + if (!strcmp(stream_name, "AFP_AfpInfo") && + test_share_config_flag(fp->tcon->share_conf, + KSMBD_SHARE_FLAG_TIME_MACHINE)) { + /* + * Synthesize an empty AFP_AfpInfo xattr on first access. + * type=0/creator=0 tells macOS to use the file extension + * for icon and type detection. + * + * Scoped to TIME_MACHINE shares, matching the rest of + * the AAPL series -- conn->is_aapl alone isn't a safe + * gate here, since the pre-existing narrow UniqueId=0 + * path can also set it on ordinary, non-Time-Machine + * shares whenever a Mac client happens to negotiate + * AAPL there too. + */ + static const u8 afpinfo_empty[60] = { + 0x00, 0x05, 0x16, 0x07, /* magic 0x00051607 BE */ + 0x00, 0x02, 0x00, 0x00, /* version 0x00020000 BE */ + }; + rc = ksmbd_vfs_setxattr(idmap, path, xattr_stream_name, + (void *)afpinfo_empty, + sizeof(afpinfo_empty), 0, false); + return rc < 0 ? rc : 0; + } + ksmbd_debug(SMB, "XATTR stream name lookup failed: %d\n", rc); + return -EBADF; + } + + rc = ksmbd_vfs_setxattr(idmap, path, xattr_stream_name, NULL, 0, 0, false); + if (rc < 0) + pr_err("Failed to store XATTR stream name :%d\n", rc); + return 0; +} + +/* + * fp->stream.size is the byte length of the mangled xattr *name* + * (used as attr_name_len when looking the xattr up), not the size of + * the xattr's value. Reporting it as EndOfFile/AllocationSize for a + * stream handle is wrong -- query the xattr's actual value length + * instead. + */ +static loff_t ksmbd_stream_eof(struct ksmbd_file *fp) +{ + ssize_t slen = ksmbd_vfs_casexattr_len(file_mnt_idmap(fp->filp), + fp->filp->f_path.dentry, + fp->stream.name, + fp->stream.size); + return slen < 0 ? 0 : (loff_t)slen; +} + +static int smb2_remove_smb_xattrs(const struct path *path) +{ + struct mnt_idmap *idmap = mnt_idmap(path->mnt); + char *name, *xattr_list = NULL; + ssize_t xattr_list_len; + int err = 0; + + xattr_list_len = ksmbd_vfs_listxattr(path->dentry, &xattr_list); + if (xattr_list_len < 0) { + goto out; + } else if (!xattr_list_len) { + ksmbd_debug(SMB, "empty xattr in the file\n"); + goto out; + } + + for (name = xattr_list; name - xattr_list < xattr_list_len; + name += strlen(name) + 1) { + ksmbd_debug(SMB, "%s, len %zd\n", name, strlen(name)); + + if (!strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN) && + !strncmp(&name[XATTR_USER_PREFIX_LEN], STREAM_PREFIX, + STREAM_PREFIX_LEN)) { + err = ksmbd_vfs_remove_xattr(idmap, path, + name, true); + if (err) + ksmbd_debug(SMB, "remove xattr failed : %s\n", + name); + } + } +out: + kvfree(xattr_list); + return err; +} + +static int smb2_create_truncate(const struct path *path) +{ + int rc = vfs_truncate(path, 0); + + if (rc) { + pr_err("vfs_truncate failed, rc %d\n", rc); + return rc; + } + + rc = smb2_remove_smb_xattrs(path); + if (rc == -EOPNOTSUPP) + rc = 0; + if (rc) + ksmbd_debug(SMB, + "ksmbd_truncate_stream_name_xattr failed, rc %d\n", + rc); + return rc; +} + +static void smb2_new_xattrs(struct ksmbd_tree_connect *tcon, const struct path *path, + struct ksmbd_file *fp) +{ + struct xattr_dos_attrib da = {0}; + int rc; + + if (!test_share_config_flag(tcon->share_conf, + KSMBD_SHARE_FLAG_STORE_DOS_ATTRS)) + return; + + da.version = 4; + da.attr = le32_to_cpu(fp->f_ci->m_fattr); + da.itime = da.create_time = fp->create_time; + da.flags = XATTR_DOSINFO_ATTRIB | XATTR_DOSINFO_CREATE_TIME | + XATTR_DOSINFO_ITIME; + + rc = ksmbd_vfs_set_dos_attrib_xattr(mnt_idmap(path->mnt), path, &da, true); + if (rc) + ksmbd_debug(SMB, "failed to store file attribute into xattr\n"); +} + +static bool smb2_parent_compressed(struct ksmbd_tree_connect *tcon, + const struct path *path) +{ + struct dentry *parent = dget_parent(path->dentry); + struct file_kattr fa = { .flags_valid = true }; + struct xattr_dos_attrib da; + bool compressed = false; + int rc; + + rc = vfs_fileattr_get(parent, &fa); + if (!rc && fa.flags & FS_COMPR_FL) { + compressed = true; + goto out; + } + + rc = ksmbd_vfs_get_dos_attrib_xattr(mnt_idmap(path->mnt), parent, &da); + if (rc > 0 && da.attr & FILE_ATTRIBUTE_COMPRESSED) + compressed = true; + +out: + dput(parent); + return compressed; +} + +static void smb2_update_xattrs(struct ksmbd_tree_connect *tcon, + const struct path *path, struct ksmbd_file *fp) +{ + struct xattr_dos_attrib da = {}; + bool store_dos_attrs = test_share_config_flag(tcon->share_conf, + KSMBD_SHARE_FLAG_STORE_DOS_ATTRS); + int rc; + + fp->f_ci->m_fattr &= ~(FILE_ATTRIBUTE_HIDDEN_LE | FILE_ATTRIBUTE_SYSTEM_LE); + + /* get FileAttributes from XATTR_NAME_DOS_ATTRIBUTE */ + rc = ksmbd_vfs_get_dos_attrib_xattr(mnt_idmap(path->mnt), + path->dentry, &da); + if (rc > 0) { + if (store_dos_attrs) { + fp->f_ci->m_fattr = cpu_to_le32(da.attr); + fp->create_time = da.create_time; + fp->itime = da.itime; + } else { + fp->f_ci->m_fattr &= + ~(FILE_ATTRIBUTE_COMPRESSED_LE | + FILE_ATTRIBUTE_SPARSE_FILE_LE); + fp->f_ci->m_fattr |= + cpu_to_le32(da.attr & + (FILE_ATTRIBUTE_COMPRESSED | + FILE_ATTRIBUTE_SPARSE_FILE)); + } + } +} + +static int smb2_creat(struct ksmbd_work *work, + struct path *path, char *name, int open_flags, + umode_t posix_mode, bool is_dir) +{ + struct ksmbd_tree_connect *tcon = work->tcon; + struct ksmbd_share_config *share = tcon->share_conf; + umode_t mode; + int rc; + + if (!(open_flags & O_CREAT)) + return -EBADF; + + ksmbd_debug(SMB, "file does not exist, so creating\n"); + if (is_dir == true) { + ksmbd_debug(SMB, "creating directory\n"); + + mode = share_config_directory_mode(share, posix_mode); + rc = ksmbd_vfs_mkdir(work, name, mode); + if (rc) + return rc; + } else { + ksmbd_debug(SMB, "creating regular file\n"); + + mode = share_config_create_mode(share, posix_mode); + rc = ksmbd_vfs_create(work, name, mode); + if (rc) + return rc; + } + + rc = ksmbd_vfs_kern_path(work, name, 0, path, 0); + if (rc) { + pr_err("cannot get linux path (%s), err = %d\n", + name, rc); + return rc; + } + return 0; +} + +static int smb2_create_sd_buffer(struct ksmbd_work *work, + struct smb2_create_req *req, + const struct path *path) +{ + struct create_context *context; + struct create_sd_buf_req *sd_buf; + + if (!req->CreateContextsOffset) + return -ENOENT; + + /* Parse SD BUFFER create contexts */ + context = smb2_find_context_vals(req, SMB2_CREATE_SD_BUFFER, 4); + if (!context) + return -ENOENT; + else if (IS_ERR(context)) + return PTR_ERR(context); + + ksmbd_debug(SMB, + "Set ACLs using SMB2_CREATE_SD_BUFFER context\n"); + sd_buf = (struct create_sd_buf_req *)context; + if (le16_to_cpu(context->DataOffset) + + le32_to_cpu(context->DataLength) < + sizeof(struct create_sd_buf_req)) + return -EINVAL; + return set_info_sec(work->conn, work->tcon, path, &sd_buf->ntsd, + le32_to_cpu(sd_buf->ccontext.DataLength), true, false); +} + +static void ksmbd_acls_fattr(struct smb_fattr *fattr, + struct mnt_idmap *idmap, + struct inode *inode) +{ + vfsuid_t vfsuid = i_uid_into_vfsuid(idmap, inode); + vfsgid_t vfsgid = i_gid_into_vfsgid(idmap, inode); + + fattr->cf_uid = vfsuid_into_kuid(vfsuid); + fattr->cf_gid = vfsgid_into_kgid(vfsgid); + fattr->cf_mode = inode->i_mode; + fattr->cf_acls = NULL; + fattr->cf_dacls = NULL; + + if (IS_ENABLED(CONFIG_FS_POSIX_ACL)) { + fattr->cf_acls = get_inode_acl(inode, ACL_TYPE_ACCESS); + if (S_ISDIR(inode->i_mode)) + fattr->cf_dacls = get_inode_acl(inode, ACL_TYPE_DEFAULT); + } +} + +enum { + DURABLE_RECONN_V2 = 1, + DURABLE_RECONN, + DURABLE_REQ_V2, + DURABLE_REQ, +}; + +struct durable_info { + struct ksmbd_file *fp; + unsigned short int type; + bool persistent; + bool reconnected; + bool replay; + bool replay_consumed; + bool app_instance_id; + bool app_instance_version_valid; + unsigned int timeout; + char *CreateGuid; + char AppInstanceId[SMB2_CREATE_GUID_SIZE]; + u64 app_instance_version_high; + u64 app_instance_version_low; +}; + +static int smb2_check_durable_replay(struct ksmbd_work *work, + struct ksmbd_file *fp, + struct lease_ctx_info *lc, + bool persistent) +{ + struct oplock_info *opinfo; + int ret = 0; + + if (!fp->is_durable && !fp->is_persistent) + return -EACCES; + + if (ksmbd_vfs_compare_durable_owner(fp, work->sess->user) == false) + return -EACCES; + + if (fp->is_persistent && !persistent) + return -EINVAL; + + opinfo = opinfo_get(fp); + if (!opinfo) + return 0; + + if (opinfo->sess && opinfo->sess->id != work->sess->id) { + ret = -ENOEXEC; + goto out; + } + + if (opinfo->is_lease) { + if (!lc || + memcmp(opinfo->o_lease->lease_key, lc->lease_key, + SMB2_LEASE_KEY_SIZE)) { + ret = -EACCES; + goto out; + } + } else { + if (lc) { + ret = -EACCES; + goto out; + } + + if (fp->is_durable && opinfo->level != SMB2_OPLOCK_LEVEL_BATCH) + ret = -EACCES; + } +out: + opinfo_put(opinfo); + return ret; +} + +static bool smb2_durable_replay_consumed(struct ksmbd_file *fp) +{ + bool consumed; + + spin_lock(&fp->f_lock); + consumed = fp->durable_replay_consumed; + spin_unlock(&fp->f_lock); + + return consumed; +} + +static void smb2_mark_durable_replay_consumed(struct ksmbd_file *fp) +{ + spin_lock(&fp->f_lock); + fp->durable_replay_consumed = true; + spin_unlock(&fp->f_lock); +} + +static bool smb2_durable_replay_differs(struct ksmbd_file *fp, + struct smb2_create_req *req) +{ + return fp->cdoption != req->CreateDisposition || + fp->create_file_attributes != req->FileAttributes; +} + +static int parse_durable_handle_context(struct ksmbd_work *work, + struct smb2_create_req *req, + struct lease_ctx_info *lc, + struct durable_info *dh_info) +{ + struct ksmbd_conn *conn = work->conn; + struct create_context *context; + int dh_idx, err = 0; + u64 persistent_id = 0; + int req_op_level; + static const char * const durable_arr[] = {"DH2C", "DHnC", "DH2Q", "DHnQ"}; + + req_op_level = req->RequestedOplockLevel; + for (dh_idx = DURABLE_RECONN_V2; dh_idx <= ARRAY_SIZE(durable_arr); + dh_idx++) { + context = smb2_find_context_vals(req, durable_arr[dh_idx - 1], 4); + if (IS_ERR(context)) { + err = PTR_ERR(context); + goto out; + } + if (!context) + continue; + + switch (dh_idx) { + case DURABLE_RECONN_V2: + { + struct create_durable_handle_reconnect_v2 *recon_v2; + u32 flags; + + if (dh_info->type == DURABLE_RECONN || + dh_info->type == DURABLE_REQ_V2) { + err = -EINVAL; + goto out; + } + + if (le32_to_cpu(context->DataLength) < + sizeof(recon_v2->dcontext)) { + err = -EINVAL; + goto out; + } + + recon_v2 = (struct create_durable_handle_reconnect_v2 *)context; + flags = le32_to_cpu(recon_v2->dcontext.Flags); + if (flags & ~SMB2_DHANDLE_FLAG_PERSISTENT) { + err = -EINVAL; + goto out; + } + dh_info->persistent = flags & SMB2_DHANDLE_FLAG_PERSISTENT; + persistent_id = recon_v2->dcontext.Fid.PersistentFileId; + dh_info->fp = ksmbd_lookup_durable_fd(persistent_id); + if (!dh_info->fp) { + ksmbd_debug(SMB, "Failed to get durable handle state\n"); + err = -EBADF; + goto out; + } + + /* A zero VolatileFileId means that the client did not specify it. */ + if (recon_v2->dcontext.Fid.VolatileFileId && + dh_info->fp->durable_volatile_id != + recon_v2->dcontext.Fid.VolatileFileId) { + err = -EBADF; + ksmbd_put_durable_fd(dh_info->fp); + goto out; + } + + if (memcmp(dh_info->fp->create_guid, recon_v2->dcontext.CreateGuid, + SMB2_CREATE_GUID_SIZE)) { + err = -EBADF; + ksmbd_put_durable_fd(dh_info->fp); + goto out; + } + + /* A persistent reconnect must match the original open type. */ + if (dh_info->fp->is_persistent != dh_info->persistent) { + err = dh_info->persistent ? -EINVAL : -EBADF; + ksmbd_put_durable_fd(dh_info->fp); + goto out; + } + + dh_info->type = dh_idx; + dh_info->reconnected = true; + ksmbd_debug(SMB, + "reconnect v2 Persistent-id from reconnect = %llu\n", + persistent_id); + break; + } + case DURABLE_RECONN: + { + create_durable_reconn_t *recon; + + if (dh_info->type == DURABLE_RECONN_V2 || + dh_info->type == DURABLE_REQ_V2) { + err = -EINVAL; + goto out; + } + + if (le32_to_cpu(context->DataLength) < + sizeof(recon->Data)) { + err = -EINVAL; + goto out; + } + + recon = (create_durable_reconn_t *)context; + persistent_id = recon->Data.Fid.PersistentFileId; + dh_info->fp = ksmbd_lookup_durable_fd(persistent_id); + if (!dh_info->fp) { + ksmbd_debug(SMB, "Failed to get durable handle state\n"); + err = -EBADF; + goto out; + } + + /* A zero VolatileFileId means that the client did not specify it. */ + if (recon->Data.Fid.VolatileFileId && + dh_info->fp->durable_volatile_id != + recon->Data.Fid.VolatileFileId) { + err = -EBADF; + ksmbd_put_durable_fd(dh_info->fp); + goto out; + } + + dh_info->type = dh_idx; + dh_info->reconnected = true; + ksmbd_debug(SMB, "reconnect Persistent-id from reconnect = %llu\n", + persistent_id); + break; + } + case DURABLE_REQ_V2: + { + struct create_durable_req_v2 *durable_v2_blob; + + if (dh_info->type == DURABLE_RECONN || + dh_info->type == DURABLE_RECONN_V2) { + err = -EINVAL; + goto out; + } + + if (le32_to_cpu(context->DataLength) < + sizeof(durable_v2_blob->dcontext)) { + err = -EINVAL; + goto out; + } + + durable_v2_blob = + (struct create_durable_req_v2 *)context; + if (le32_to_cpu(durable_v2_blob->dcontext.Flags) & + ~SMB2_DHANDLE_FLAG_PERSISTENT) { + err = -EINVAL; + goto out; + } + ksmbd_debug(SMB, "Request for durable v2 open\n"); + dh_info->CreateGuid = durable_v2_blob->dcontext.CreateGuid; + dh_info->persistent = + le32_to_cpu(durable_v2_blob->dcontext.Flags) & + SMB2_DHANDLE_FLAG_PERSISTENT; + dh_info->fp = ksmbd_lookup_fd_cguid(durable_v2_blob->dcontext.CreateGuid); + if (dh_info->fp) { + if (!memcmp(conn->ClientGUID, dh_info->fp->client_guid, + SMB2_CLIENT_GUID_SIZE)) { + if (!(req->hdr.Flags & SMB2_FLAGS_REPLAY_OPERATION)) { + err = -ENOEXEC; + ksmbd_put_durable_fd(dh_info->fp); + goto out; + } + + if (dh_info->fp->f_state == FP_NEW) { + /* Original CREATE is still pending. */ + ksmbd_put_durable_fd(dh_info->fp); + err = -EAGAIN; + goto out; + } + + if (!dh_info->fp->is_durable && + !dh_info->fp->is_persistent) { + /* + * A DurableHandleReqV2 CREATE can complete + * without granting durability (for example, if + * it requested no oplock). Its CreateGuid still + * identifies a completed CREATE for replay. + */ + if (dh_info->fp->conn && + ksmbd_vfs_compare_durable_owner( + dh_info->fp, work->sess->user)) { + if (smb2_durable_replay_consumed( + dh_info->fp)) { + ksmbd_put_durable_fd(dh_info->fp); + dh_info->fp = NULL; + dh_info->type = dh_idx; + dh_info->replay_consumed = true; + break; + } + if (smb2_durable_replay_differs( + dh_info->fp, req)) + smb2_mark_durable_replay_consumed( + dh_info->fp); + dh_info->replay = true; + dh_info->type = dh_idx; + goto out; + } + ksmbd_put_durable_fd(dh_info->fp); + err = -EACCES; + goto out; + } + + if (dh_info->fp->conn && + smb2_durable_replay_consumed(dh_info->fp)) { + ksmbd_put_durable_fd(dh_info->fp); + dh_info->fp = NULL; + dh_info->type = dh_idx; + dh_info->replay_consumed = true; + break; + } + + err = smb2_check_durable_replay(work, + dh_info->fp, + lc, + dh_info->persistent); + if (err) { + ksmbd_put_durable_fd(dh_info->fp); + goto out; + } + + if (dh_info->fp->conn) { + if (smb2_durable_replay_differs(dh_info->fp, + req)) + smb2_mark_durable_replay_consumed( + dh_info->fp); + dh_info->replay = true; + } else { + dh_info->reconnected = true; + } + dh_info->type = dh_idx; + goto out; + } + ksmbd_put_durable_fd(dh_info->fp); + dh_info->fp = NULL; + } + + if ((lc && (lc->req_state & SMB2_LEASE_HANDLE_CACHING_LE)) || + req_op_level == SMB2_OPLOCK_LEVEL_BATCH) { + dh_info->timeout = + le32_to_cpu(durable_v2_blob->dcontext.Timeout); + dh_info->type = dh_idx; + } + break; + } + case DURABLE_REQ: + if (dh_info->type == DURABLE_RECONN) + goto out; + if (dh_info->type == DURABLE_RECONN_V2 || + dh_info->type == DURABLE_REQ_V2) { + err = -EINVAL; + goto out; + } + + if ((lc && (lc->req_state & SMB2_LEASE_HANDLE_CACHING_LE)) || + req_op_level == SMB2_OPLOCK_LEVEL_BATCH) { + ksmbd_debug(SMB, "Request for durable open\n"); + dh_info->type = dh_idx; + } + } + } + +out: + return err; +} + +static int parse_app_instance_id(struct smb2_create_req *req, + struct durable_info *dh_info) +{ + struct create_context *context; + char *data; + + context = smb2_find_context_vals(req, SMB2_CREATE_APP_INSTANCE_ID, + SMB2_CREATE_GUID_SIZE); + if (IS_ERR(context)) + return PTR_ERR(context); + if (!context) + return 0; + + if (le32_to_cpu(context->DataLength) < 20) + return -EINVAL; + + data = (char *)context + le16_to_cpu(context->DataOffset); + if (data[0] != 20 || data[1]) + return -EINVAL; + + memcpy(dh_info->AppInstanceId, data + 4, SMB2_CREATE_GUID_SIZE); + dh_info->app_instance_id = true; + return 0; +} + +static int parse_app_instance_version(struct smb2_create_req *req, + struct durable_info *dh_info) +{ + struct create_context *context; + char *data; + + context = smb2_find_context_vals(req, SMB2_CREATE_APP_INSTANCE_VERSION, + SMB2_CREATE_GUID_SIZE); + if (IS_ERR(context)) + return PTR_ERR(context); + if (!context) + return 0; + + if (le32_to_cpu(context->DataLength) < 24) + return -EINVAL; + + data = (char *)context + le16_to_cpu(context->DataOffset); + if (get_unaligned_le16(data) != 24 || + get_unaligned_le16(data + 2) != 0) + return -EINVAL; + + dh_info->app_instance_version_high = get_unaligned_le64(data + 8); + dh_info->app_instance_version_low = get_unaligned_le64(data + 16); + dh_info->app_instance_version_valid = true; + return 0; +} + +static int smb2_handle_app_instance_id(struct smb2_create_rsp *rsp, + struct durable_info *dh_info) +{ + struct ksmbd_file *old_fp; + bool reject = false; + + if (!dh_info->app_instance_id) + return 0; + + old_fp = ksmbd_lookup_fd_app_instance_id(dh_info->AppInstanceId); + if (!old_fp) + return 0; + + if (dh_info->app_instance_version_valid) { + if (old_fp->app_instance_version_valid && + (dh_info->app_instance_version_high < + old_fp->app_instance_version_high || + (dh_info->app_instance_version_high == + old_fp->app_instance_version_high && + dh_info->app_instance_version_low <= + old_fp->app_instance_version_low))) + reject = true; + } else if (old_fp->app_instance_version_valid) { + reject = true; + } + + ksmbd_put_durable_fd(old_fp); + if (reject) { + rsp->hdr.Status = STATUS_FILE_FORCED_CLOSED; + return -EIO; + } + + return ksmbd_close_fd_app_instance_id(dh_info->AppInstanceId); +} + +/** + * smb2_open() - handler for smb file open request + * @work: smb work containing request buffer + * + * Return: 0 on success, otherwise error + */ +int smb2_open(struct ksmbd_work *work) +{ + struct ksmbd_conn *conn = work->conn; + struct ksmbd_session *sess = work->sess; + struct ksmbd_tree_connect *tcon = work->tcon; + struct smb2_create_req *req; + struct smb2_create_rsp *rsp; + struct path path; + struct ksmbd_share_config *share = tcon->share_conf; + struct ksmbd_file *fp = NULL; + struct file *filp = NULL; + struct mnt_idmap *idmap = NULL; + struct kstat stat; + struct create_context *context; + struct lease_ctx_info *lc = NULL; + struct create_ea_buf_req *ea_buf = NULL; + struct oplock_info *opinfo; + struct durable_info dh_info = {0}; + __le32 *next_ptr = NULL; + int req_op_level = 0, open_flags = 0, may_flags = 0, file_info = 0; + int rc = 0; + int contxt_cnt = 0, query_disk_id = 0; + bool maximal_access_ctxt = false, posix_ctxt = false; + bool aapl_ctxt = false; + bool durable_rsp = true; + __u64 aapl_req_bitmap = 0, aapl_client_caps = 0; + int s_type = 0; + int next_off = 0; + char *name = NULL; + char *stream_name = NULL; + bool file_present = false, created = false, already_permitted = false; + int share_ret, need_truncate = 0; + u64 time, alloc_size = 0; + umode_t posix_mode = 0; + __le32 daccess, maximal_access = 0; + u32 dos_attr; + int iov_len = 0; + + ksmbd_debug(SMB, "Received smb2 create request\n"); + + WORK_BUFFERS(work, req, rsp); + + if (req->hdr.NextCommand && !work->next_smb2_rcv_hdr_off && + (req->hdr.Flags & SMB2_FLAGS_RELATED_OPERATIONS)) { + ksmbd_debug(SMB, "invalid flag in chained command\n"); + rsp->hdr.Status = STATUS_INVALID_PARAMETER; + smb2_set_err_rsp(work); + return -EINVAL; + } + + if (test_share_config_flag(share, KSMBD_SHARE_FLAG_PIPE)) { + ksmbd_debug(SMB, "IPC pipe create request\n"); + return create_smb2_pipe(work); + } + + if (req->CreateContextsOffset && tcon->posix_extensions) { + context = smb2_find_context_vals(req, SMB2_CREATE_TAG_POSIX, 16); + if (IS_ERR(context)) { + rc = PTR_ERR(context); + goto err_out2; + } else if (context) { + struct create_posix *posix = (struct create_posix *)context; + + if (le16_to_cpu(context->DataOffset) + + le32_to_cpu(context->DataLength) < + sizeof(struct create_posix) - 4) { + rc = -EINVAL; + goto err_out2; + } + ksmbd_debug(SMB, "get posix context\n"); + + posix_mode = le32_to_cpu(posix->Mode); + posix_ctxt = true; + } + } + + if (req->NameLength) { + name = smb2_get_name((char *)req + le16_to_cpu(req->NameOffset), + le16_to_cpu(req->NameLength), + work->conn->local_nls); + if (IS_ERR(name)) { + rc = PTR_ERR(name); + name = NULL; + goto err_out2; + } + + ksmbd_debug(SMB, "converted name = %s\n", name); + + if (posix_ctxt == false) { + if (strchr(name, ':')) { + if (!test_share_config_flag(work->tcon->share_conf, + KSMBD_SHARE_FLAG_STREAMS)) { + rc = -EBADF; + goto err_out2; + } + rc = parse_stream_name(name, &stream_name, &s_type); + if (rc < 0) + goto err_out2; + } + + rc = ksmbd_validate_filename(name); + if (rc < 0) + goto err_out2; + } + + if (ksmbd_share_veto_filename(share, name)) { + rc = -ENOENT; + ksmbd_debug(SMB, "Reject open(), vetoed file: %s\n", + name); + goto err_out2; + } + } else { + name = kstrdup("", KSMBD_DEFAULT_GFP); + if (!name) { + rc = -ENOMEM; + goto err_out2; + } + } + + req_op_level = req->RequestedOplockLevel; + + if (req->CreateContextsOffset) { + rc = parse_app_instance_id(req, &dh_info); + if (rc) + goto err_out2; + rc = parse_app_instance_version(req, &dh_info); + if (rc) + goto err_out2; + } + + if (server_conf.flags & KSMBD_GLOBAL_FLAG_DURABLE_HANDLE && + req->CreateContextsOffset) { + lc = parse_lease_state(req); + if (IS_ERR(lc)) { + rc = PTR_ERR(lc); + lc = NULL; + goto err_out2; + } + if (lc && lc->version == 2 && conn->dialect < SMB30_PROT_ID) { + kfree(lc); + lc = NULL; + if (req_op_level == SMB2_OPLOCK_LEVEL_LEASE) + req_op_level = SMB2_OPLOCK_LEVEL_NONE; + } + rc = parse_durable_handle_context(work, req, lc, &dh_info); + if (rc) { + ksmbd_debug(SMB, "error parsing durable handle context\n"); + goto err_out2; + } + + if (dh_info.replay == true) { + fp = dh_info.fp; + if (ksmbd_override_fsids(work)) { + rc = -ENOMEM; + goto err_out2; + } + + file_info = FILE_OPENED; + rc = ksmbd_vfs_getattr(&fp->filp->f_path, &stat); + if (rc) + goto err_out2; + + goto reconnected_fp; + } + + if (dh_info.reconnected == true) { + rc = smb2_check_durable_oplock(conn, share, dh_info.fp, + lc, sess->user, name); + if (rc) + goto err_out2; + + rc = ksmbd_reopen_durable_fd(work, dh_info.fp); + if (rc) + goto err_out2; + + fp = dh_info.fp; + + if (ksmbd_override_fsids(work)) { + rc = -ENOMEM; + goto err_out2; + } + + file_info = FILE_OPENED; + + rc = ksmbd_vfs_getattr(&fp->filp->f_path, &stat); + if (rc) + goto err_out2; + + goto reconnected_fp; + } + + } else if (req_op_level == SMB2_OPLOCK_LEVEL_LEASE) { + lc = parse_lease_state(req); + if (IS_ERR(lc)) { + rc = PTR_ERR(lc); + lc = NULL; + goto err_out2; + } + if (lc && lc->version == 2 && conn->dialect < SMB30_PROT_ID) { + kfree(lc); + lc = NULL; + req_op_level = SMB2_OPLOCK_LEVEL_NONE; + } + } + + if (dh_info.app_instance_id && !dh_info.reconnected && + !dh_info.replay) { + rc = smb2_handle_app_instance_id(rsp, &dh_info); + if (rc) + goto err_out2; + } + + if (le32_to_cpu(req->ImpersonationLevel) > le32_to_cpu(IL_DELEGATE)) { + pr_err("Invalid impersonationlevel : 0x%x\n", + le32_to_cpu(req->ImpersonationLevel)); + rc = -EIO; + rsp->hdr.Status = STATUS_BAD_IMPERSONATION_LEVEL; + goto err_out2; + } + + if (req->CreateOptions && !(req->CreateOptions & CREATE_OPTIONS_MASK_LE)) { + pr_err("Invalid create options : 0x%x\n", + le32_to_cpu(req->CreateOptions)); + rc = -EINVAL; + goto err_out2; + } else { + if (req->CreateOptions & FILE_SEQUENTIAL_ONLY_LE && + req->CreateOptions & FILE_RANDOM_ACCESS_LE) + req->CreateOptions &= ~FILE_SEQUENTIAL_ONLY_LE; + + if (req->CreateOptions & + (FILE_OPEN_BY_FILE_ID_LE | CREATE_TREE_CONNECTION | + FILE_RESERVE_OPFILTER_LE)) { + rc = -EOPNOTSUPP; + goto err_out2; + } + + if (req->CreateOptions & FILE_DIRECTORY_FILE_LE) { + if (req->CreateOptions & FILE_NON_DIRECTORY_FILE_LE) { + rc = -EINVAL; + goto err_out2; + } + } + } + + if (le32_to_cpu(req->CreateDisposition) > + le32_to_cpu(FILE_OVERWRITE_IF_LE)) { + pr_err("Invalid create disposition : 0x%x\n", + le32_to_cpu(req->CreateDisposition)); + rc = -EINVAL; + goto err_out2; + } + + if (!(req->DesiredAccess & DESIRED_ACCESS_MASK)) { + pr_err("Invalid desired access : 0x%x\n", + le32_to_cpu(req->DesiredAccess)); + rc = -EACCES; + goto err_out2; + } + + if (req->DesiredAccess == FILE_SYNCHRONIZE_LE && + req->CreateDisposition == FILE_OPEN_IF_LE && + !req->FileAttributes) { + rc = -EACCES; + goto err_out2; + } + + if (req->FileAttributes && + (req->FileAttributes & ~cpu_to_le32(SMB2_CREATE_FILE_ATTRIBUTE_MASK))) { + pr_err("Invalid file attribute : 0x%x\n", + le32_to_cpu(req->FileAttributes)); + rc = -EINVAL; + goto err_out2; + } + + if (req->CreateContextsOffset) { + /* Parse non-durable handle create contexts */ + context = smb2_find_context_vals(req, SMB2_CREATE_EA_BUFFER, 4); + if (IS_ERR(context)) { + rc = PTR_ERR(context); + goto err_out2; + } else if (context) { + ea_buf = (struct create_ea_buf_req *)context; + if (le16_to_cpu(context->DataOffset) + + le32_to_cpu(context->DataLength) < + sizeof(struct create_ea_buf_req)) { + rc = -EINVAL; + goto err_out2; + } + if (req->CreateOptions & FILE_NO_EA_KNOWLEDGE_LE) { + rsp->hdr.Status = STATUS_ACCESS_DENIED; + rc = -EACCES; + goto err_out2; + } + } + + context = smb2_find_context_vals(req, + SMB2_CREATE_QUERY_MAXIMAL_ACCESS_REQUEST, 4); + if (IS_ERR(context)) { + rc = PTR_ERR(context); + goto err_out2; + } else if (context) { + ksmbd_debug(SMB, + "get query maximal access context\n"); + maximal_access_ctxt = 1; + } + + context = smb2_find_context_vals(req, + SMB2_CREATE_TIMEWARP_REQUEST, 4); + if (IS_ERR(context)) { + rc = PTR_ERR(context); + goto err_out2; + } else if (context) { + ksmbd_debug(SMB, "get timewarp context\n"); + rc = -EBADF; + goto err_out2; + } + } + + if (ksmbd_override_fsids(work)) { + rc = -ENOMEM; + goto err_out2; + } + + rc = ksmbd_vfs_kern_path(work, name, LOOKUP_NO_SYMLINKS, + &path, 1); + + /* + * A durable handle opened with delete-on-close is preserved across a + * disconnect so it can be reclaimed by a durable reconnect. When a new + * delete-on-close open for the same name arrives instead, the + * disconnected handle must give way: close it so its delete-on-close + * removes the file, then re-resolve so this open can create a fresh one. + */ + if (!rc && (req->CreateOptions & FILE_DELETE_ON_CLOSE_LE) && + (req->CreateDisposition == FILE_OVERWRITE_IF_LE || + req->CreateDisposition == FILE_OPEN_IF_LE) && + ksmbd_close_disconnected_durable_delete_on_close(path.dentry)) { + path_put(&path); + rc = ksmbd_vfs_kern_path(work, name, LOOKUP_NO_SYMLINKS, + &path, 1); + } + + if (!rc) { + file_present = true; + + if (req->CreateOptions & FILE_DELETE_ON_CLOSE_LE) { + struct xattr_dos_attrib da; + + /* + * If file exists with under flags, return access + * denied error. + */ + if (req->CreateDisposition == FILE_OVERWRITE_IF_LE || + req->CreateDisposition == FILE_OPEN_IF_LE) { + rc = -EACCES; + goto err_out; + } + + if (!test_tree_conn_flag(tcon, KSMBD_TREE_CONN_FLAG_WRITABLE)) { + ksmbd_debug(SMB, + "User does not have write permission\n"); + rc = -EACCES; + goto err_out; + } + + if (test_share_config_flag(tcon->share_conf, + KSMBD_SHARE_FLAG_STORE_DOS_ATTRS) && + ksmbd_vfs_get_dos_attrib_xattr(mnt_idmap(path.mnt), + path.dentry, &da) > 0 && + da.attr & FILE_ATTRIBUTE_READONLY) { + rsp->hdr.Status = STATUS_CANNOT_DELETE; + rc = -EACCES; + goto err_out; + } + } else if (d_is_symlink(path.dentry)) { + rc = -EACCES; + goto err_out; + } + + idmap = mnt_idmap(path.mnt); + } else { + if (rc != -ENOENT) + goto err_out; + ksmbd_debug(SMB, "can not get linux path for %s, rc = %d\n", + name, rc); + rc = 0; + } + + if (!file_present && req->CreateOptions & FILE_DELETE_ON_CLOSE_LE && + req->FileAttributes & FILE_ATTRIBUTE_READONLY_LE) { + rsp->hdr.Status = STATUS_CANNOT_DELETE; + rc = -EACCES; + goto err_out; + } + + /* + * An explicit ::$DATA suffix names the unnamed data stream and is + * canonicalized to a NULL stream name (base file), but the request + * still has to be validated against the data-stream type, e.g. opening + *

::$DATA with FILE_DIRECTORY_FILE must fail with + * STATUS_NOT_A_DIRECTORY. + */ + if (stream_name || s_type == DATA_STREAM) { + if (req->CreateOptions & FILE_DIRECTORY_FILE_LE) { + if (s_type == DATA_STREAM) { + rc = -EIO; + rsp->hdr.Status = STATUS_NOT_A_DIRECTORY; + } + } else { + if (file_present && S_ISDIR(d_inode(path.dentry)->i_mode) && + !stream_name && s_type == DATA_STREAM) { + rc = -EIO; + rsp->hdr.Status = STATUS_FILE_IS_A_DIRECTORY; + } + } + + if (req->CreateOptions & FILE_DIRECTORY_FILE_LE && + req->FileAttributes & FILE_ATTRIBUTE_NORMAL_LE) { + rsp->hdr.Status = STATUS_NOT_A_DIRECTORY; + rc = -EIO; + } + + if (rc < 0) + goto err_out; + } + + if (file_present && req->CreateOptions & FILE_NON_DIRECTORY_FILE_LE && + S_ISDIR(d_inode(path.dentry)->i_mode) && + !(req->CreateOptions & FILE_DELETE_ON_CLOSE_LE)) { + ksmbd_debug(SMB, "open() argument is a directory: %s, %x\n", + name, req->CreateOptions); + rsp->hdr.Status = STATUS_FILE_IS_A_DIRECTORY; + rc = -EIO; + goto err_out; + } + + if (file_present && (req->CreateOptions & FILE_DIRECTORY_FILE_LE) && + !(req->CreateDisposition == FILE_CREATE_LE) && + !S_ISDIR(d_inode(path.dentry)->i_mode)) { + rsp->hdr.Status = STATUS_NOT_A_DIRECTORY; + rc = -EIO; + goto err_out; + } + + if (!stream_name && file_present && + req->CreateDisposition == FILE_CREATE_LE) { + rc = -EEXIST; + goto err_out; + } + + daccess = smb_map_generic_desired_access(req->DesiredAccess); + + if (file_present && !(req->CreateOptions & FILE_DELETE_ON_CLOSE_LE)) { + rc = smb_check_perm_dacl(conn, &path, &daccess, + req->DesiredAccess, + sess->user->uid, false); + if (rc) + goto err_out; + + if (maximal_access_ctxt) { + maximal_access = FILE_MAXIMAL_ACCESS_LE; + rc = smb_check_perm_dacl(conn, &path, &maximal_access, + 0, sess->user->uid, false); + if (rc) + goto err_out; + + /* + * smb_check_perm_dacl() returns success without + * touching *pdaccess when the object has no stored + * NT ACL, leaving maximal_access as the + * FILE_MAXIMAL_ACCESS_LE request sentinel instead of + * a real access mask. + */ + if (maximal_access == FILE_MAXIMAL_ACCESS_LE) + ksmbd_vfs_query_maximal_access(idmap, path.dentry, + &maximal_access); + } + } + + if (daccess & FILE_MAXIMAL_ACCESS_LE) { + if (!file_present) { + daccess = cpu_to_le32(GENERIC_ALL_FLAGS); + } else { + ksmbd_vfs_query_maximal_access(idmap, + path.dentry, + &daccess); + already_permitted = true; + } + maximal_access = daccess; + } + + open_flags = smb2_create_open_flags(file_present, daccess, + req->CreateDisposition, + &may_flags, + req->CreateOptions, + file_present ? d_inode(path.dentry)->i_mode : 0); + + if (!test_tree_conn_flag(tcon, KSMBD_TREE_CONN_FLAG_WRITABLE)) { + if (open_flags & (O_CREAT | O_TRUNC)) { + ksmbd_debug(SMB, + "User does not have write permission\n"); + rc = -EACCES; + goto err_out; + } + } + + /*create file if not present */ + if (!file_present) { + rc = smb2_creat(work, &path, name, open_flags, + posix_mode, + req->CreateOptions & FILE_DIRECTORY_FILE_LE); + if (rc) { + if (rc == -ENOENT) { + rc = -EIO; + rsp->hdr.Status = STATUS_OBJECT_PATH_NOT_FOUND; + } + goto err_out; + } + + created = true; + idmap = mnt_idmap(path.mnt); + if (ea_buf) { + if (le32_to_cpu(ea_buf->ccontext.DataLength) < + sizeof(struct smb2_ea_info)) { + rc = -EINVAL; + goto err_out; + } + + rc = smb2_set_ea(&ea_buf->ea, + le32_to_cpu(ea_buf->ccontext.DataLength), + &path, false); + if (rc == -EOPNOTSUPP) + rc = 0; + else if (rc) + goto err_out; + } + } else if (!already_permitted) { + /* FILE_READ_ATTRIBUTE is allowed without inode_permission, + * because execute(search) permission on a parent directory, + * is already granted. + */ + if (daccess & ~(FILE_READ_ATTRIBUTES_LE | FILE_READ_CONTROL_LE)) { + rc = inode_permission(idmap, + d_inode(path.dentry), + may_flags); + if (rc) + goto err_out; + + if ((daccess & FILE_DELETE_LE) || + (req->CreateOptions & FILE_DELETE_ON_CLOSE_LE)) { + rc = inode_permission(idmap, + d_inode(path.dentry->d_parent), + MAY_EXEC | MAY_WRITE); + if (rc) + goto err_out; + } + } + } + + rc = ksmbd_query_inode_status(path.dentry->d_parent); + if (rc == KSMBD_INODE_STATUS_PENDING_DELETE) { + rc = -EBUSY; + goto err_out; + } + + rc = 0; + filp = dentry_open(&path, open_flags, current_cred()); + if (IS_ERR(filp)) { + rc = PTR_ERR(filp); + pr_err("dentry open for dir failed, rc %d\n", rc); + goto err_out; + } + + if (file_present) { + if (!(open_flags & O_TRUNC)) + file_info = FILE_OPENED; + else + file_info = FILE_OVERWRITTEN; + + if ((req->CreateDisposition & FILE_CREATE_MASK_LE) == + FILE_SUPERSEDE_LE) + file_info = FILE_SUPERSEDED; + } else if (open_flags & O_CREAT) { + file_info = FILE_CREATED; + } + + ksmbd_vfs_set_fadvise(filp, req->CreateOptions); + + /* Obtain Volatile-ID */ + fp = ksmbd_open_fd(work, filp); + if (IS_ERR(fp)) { + fput(filp); + rc = PTR_ERR(fp); + fp = NULL; + goto err_out; + } + + /* Get Persistent-ID */ + ksmbd_open_durable_fd(fp); + if (!has_file_id(fp->persistent_id)) { + rc = -ENOMEM; + goto err_out; + } + + /* + * Publish the client and create GUID before an oplock/lease break can + * make this CREATE pending. A replay of that in-flight CREATE must find + * this FP_NEW handle and fail with STATUS_FILE_NOT_AVAILABLE instead of + * waiting on the same break again. + */ + memcpy(fp->client_guid, conn->ClientGUID, SMB2_CLIENT_GUID_SIZE); + if (dh_info.app_instance_id) { + memcpy(fp->app_instance_id, dh_info.AppInstanceId, + SMB2_CREATE_GUID_SIZE); + fp->has_app_instance_id = true; + } + if (dh_info.app_instance_version_valid) { + fp->app_instance_version_high = + dh_info.app_instance_version_high; + fp->app_instance_version_low = dh_info.app_instance_version_low; + fp->app_instance_version_valid = true; + } + if (dh_info.CreateGuid) { + memcpy(fp->create_guid, dh_info.CreateGuid, SMB2_CREATE_GUID_SIZE); + fp->durable_replay_consumed = dh_info.replay_consumed; + rc = ksmbd_vfs_set_durable_owner(fp, sess->user); + if (rc) + goto err_out; + } + + fp->cdoption = req->CreateDisposition; + fp->create_file_attributes = req->FileAttributes; + fp->daccess = daccess; + fp->saccess = req->ShareAccess; + fp->coption = req->CreateOptions; + + /* Set default windows and posix acls if creating new file */ + if (created) { + int posix_acl_rc; + struct inode *inode = d_inode(path.dentry); + + posix_acl_rc = ksmbd_vfs_inherit_posix_acl(idmap, + &path, + d_inode(path.dentry->d_parent)); + if (posix_acl_rc) + ksmbd_debug(SMB, "inherit posix acl failed : %d\n", posix_acl_rc); + + rc = smb2_create_sd_buffer(work, req, &path); + if (rc && rc != -ENOENT) + goto err_out; + + if (rc == -ENOENT) { + if (test_share_config_flag(work->tcon->share_conf, + KSMBD_SHARE_FLAG_ACL_XATTR)) { + rc = smb_inherit_dacl(conn, &path, sess->user->uid, + sess->user->gid); + } + if (rc) { + if (posix_acl_rc) + ksmbd_vfs_set_init_posix_acl(idmap, + &path); + + if (test_share_config_flag(work->tcon->share_conf, + KSMBD_SHARE_FLAG_ACL_XATTR)) { + struct smb_fattr fattr; + struct smb_ntsd *pntsd; + int pntsd_size; + size_t scratch_len; + + ksmbd_acls_fattr(&fattr, idmap, inode); + scratch_len = smb_acl_sec_desc_scratch_len(&fattr, + NULL, 0, + OWNER_SECINFO | GROUP_SECINFO | + DACL_SECINFO); + if (!scratch_len || scratch_len == SIZE_MAX) { + rc = -EFBIG; + posix_acl_release(fattr.cf_acls); + posix_acl_release(fattr.cf_dacls); + goto err_out; + } + + pntsd = kvzalloc(scratch_len, KSMBD_DEFAULT_GFP); + if (!pntsd) { + rc = -ENOMEM; + posix_acl_release(fattr.cf_acls); + posix_acl_release(fattr.cf_dacls); + goto err_out; + } + + rc = build_sec_desc(idmap, + pntsd, NULL, 0, + OWNER_SECINFO | + GROUP_SECINFO | + DACL_SECINFO, + &pntsd_size, &fattr); + posix_acl_release(fattr.cf_acls); + posix_acl_release(fattr.cf_dacls); + if (rc) { + kvfree(pntsd); + goto err_out; + } + + rc = ksmbd_vfs_set_sd_xattr(conn, + idmap, + &path, + pntsd, + pntsd_size, + false); + kvfree(pntsd); + if (rc) + pr_err("failed to store ntacl in xattr : %d\n", + rc); + } + } + } + rc = 0; + } + + if (stream_name) { + rc = smb2_set_stream_name_xattr(&path, + fp, + stream_name, + s_type); + if (rc) + goto err_out; + file_info = FILE_CREATED; + } + + fp->attrib_only = !(req->DesiredAccess & ~(FILE_READ_ATTRIBUTES_LE | + FILE_WRITE_ATTRIBUTES_LE | FILE_SYNCHRONIZE_LE)); + + fp->is_posix_ctxt = posix_ctxt; + + /* fp should be searchable through ksmbd_inode.m_fp_list + * after daccess, saccess, attrib_only, and stream are + * initialized. + */ + down_write(&fp->f_ci->m_lock); + list_add(&fp->node, &fp->f_ci->m_fp_list); + up_write(&fp->f_ci->m_lock); + + /* Check delete pending among previous fp before oplock break */ + if (ksmbd_inode_pending_delete(fp)) { + rc = -EBUSY; + goto err_out; + } + + if (!stream_name && daccess & FILE_DELETE_LE && + ksmbd_has_stream_without_delete_share(fp)) { + rc = -EPERM; + goto err_out; + } + + if (file_present || created) + path_put(&path); + + if (!S_ISDIR(file_inode(filp)->i_mode) && open_flags & O_TRUNC && + !fp->attrib_only && !stream_name) { + smb_break_all_oplock(work, fp); + need_truncate = 1; + } + + share_ret = ksmbd_smb_check_shared_mode(fp->filp, fp); + if (!test_share_config_flag(work->tcon->share_conf, KSMBD_SHARE_FLAG_OPLOCKS) || + (req_op_level == SMB2_OPLOCK_LEVEL_LEASE && + !(conn->vals->req_capabilities & SMB2_GLOBAL_CAP_LEASING))) { + if (share_ret < 0 && !S_ISDIR(file_inode(fp->filp)->i_mode)) { + rc = share_ret; + goto err_out1; + } + } else { + if (created && !lc) + smb_send_parent_lease_break_noti(fp, NULL); + + if (req_op_level == SMB2_OPLOCK_LEVEL_LEASE && lc) { + if (S_ISDIR(file_inode(filp)->i_mode)) { + lc->req_state &= ~SMB2_LEASE_WRITE_CACHING_LE; + lc->is_dir = true; + } + + /* + * Compare parent lease using parent key. If there is no + * a lease that has same parent key, Send lease break + * notification. + */ + smb_send_parent_lease_break_noti(fp, lc); + + req_op_level = smb2_map_lease_to_oplock(lc->req_state); + ksmbd_debug(SMB, + "lease req for(%s) req oplock state 0x%x, lease state 0x%x\n", + name, req_op_level, lc->req_state); + rc = find_same_lease_key(conn, fp->f_ci, lc); + if (rc) + goto err_out1; + } else if (open_flags == O_RDONLY && + (req_op_level == SMB2_OPLOCK_LEVEL_BATCH || + req_op_level == SMB2_OPLOCK_LEVEL_EXCLUSIVE)) + req_op_level = SMB2_OPLOCK_LEVEL_II; + + rc = smb_grant_oplock(work, req_op_level, + fp->persistent_id, fp, + le32_to_cpu(req->hdr.Id.SyncId.TreeId), + lc, share_ret, + smb3_hdr_replay(&req->hdr)); + if (rc < 0) + goto err_out1; + } + + if (req->CreateOptions & FILE_DELETE_ON_CLOSE_LE) { + smb_break_all_levII_oplock_for_delete(work, fp); + ksmbd_fd_set_delete_on_close(fp, file_info); + } + + if (need_truncate) { + rc = smb2_create_truncate(&fp->filp->f_path); + if (rc) + goto err_out1; + } + + if (req->CreateContextsOffset) { + struct create_alloc_size_req *az_req; + + az_req = (struct create_alloc_size_req *)smb2_find_context_vals(req, + SMB2_CREATE_ALLOCATION_SIZE, 4); + if (IS_ERR(az_req)) { + rc = PTR_ERR(az_req); + goto err_out1; + } else if (az_req) { + int err; + + if (le16_to_cpu(az_req->ccontext.DataOffset) + + le32_to_cpu(az_req->ccontext.DataLength) < + sizeof(struct create_alloc_size_req)) { + rc = -EINVAL; + goto err_out1; + } + alloc_size = le64_to_cpu(az_req->AllocationSize); + fp->allocation_size_set = true; + ksmbd_debug(SMB, + "request smb2 create allocate size : %llu\n", + alloc_size); + /* + * fp->filp is the base file's data fork for a stream + * handle (streams are xattr-backed on the same + * underlying file) -- fallocate has no meaning for a + * stream and would otherwise pre-allocate storage on + * the base file's data instead. + */ + if (!ksmbd_stream_fd(fp)) { + smb_break_all_levII_oplock(work, fp, 1); + err = vfs_fallocate(fp->filp, FALLOC_FL_KEEP_SIZE, 0, + alloc_size); + if (err < 0) + ksmbd_debug(SMB, + "vfs_fallocate is failed : %d\n", + err); + } + } + + context = smb2_find_context_vals(req, SMB2_CREATE_QUERY_ON_DISK_ID, 4); + if (IS_ERR(context)) { + rc = PTR_ERR(context); + goto err_out1; + } else if (context) { + ksmbd_debug(SMB, "get query on disk id context\n"); + query_disk_id = 1; + } + + if (test_share_config_flag(share, KSMBD_SHARE_FLAG_TIME_MACHINE)) { + context = smb2_find_context_vals(req, SMB2_CREATE_AAPL, 4); + if (IS_ERR(context)) { + rc = PTR_ERR(context); + goto err_out1; + } else if (context) { + struct aapl_server_query_req *aapl_req; + + if (le32_to_cpu(context->DataLength) < + sizeof(struct aapl_server_query_req)) { + rc = -EINVAL; + goto err_out1; + } + + aapl_req = (struct aapl_server_query_req *) + ((char *)context + + le16_to_cpu(context->DataOffset)); + if (le32_to_cpu(aapl_req->cmd) == + SMB2_CRTCTX_AAPL_SERVER_QUERY) { + conn->is_aapl = true; + aapl_ctxt = true; + aapl_req_bitmap = le64_to_cpu(aapl_req->req_bitmap); + aapl_client_caps = le64_to_cpu(aapl_req->client_caps); + } + } + } else if (conn->is_aapl == false) { + context = smb2_find_context_vals(req, SMB2_CREATE_AAPL, 4); + if (IS_ERR(context)) { + rc = PTR_ERR(context); + goto err_out1; + } else if (context) + conn->is_aapl = true; + } + } + + rc = ksmbd_vfs_getattr(&path, &stat); + if (rc) + goto err_out1; + + if (stat.result_mask & STATX_BTIME) + fp->create_time = ksmbd_UnixTimeToNT(stat.btime); + else + fp->create_time = ksmbd_UnixTimeToNT(stat.ctime); + fp->change_time = ksmbd_UnixTimeToNT(stat.ctime); + fp->allocation_size = S_ISDIR(stat.mode) ? 0 : + (alloc_size ?: stat.blocks << 9); + if (created || fp->f_ci->m_fattr == 0) + fp->f_ci->m_fattr = + cpu_to_le32(smb2_get_dos_mode(&stat, le32_to_cpu(req->FileAttributes))); + + if (!created) + smb2_update_xattrs(tcon, &path, fp); + if (need_truncate && req->FileAttributes) { + dos_attr = le32_to_cpu(req->FileAttributes); + fp->f_ci->m_fattr = + cpu_to_le32(smb2_get_dos_mode(&stat, dos_attr)); + smb2_new_xattrs(tcon, &path, fp); + } + + ksmbd_vfs_update_compressed_fattr(path.dentry, &fp->f_ci->m_fattr); + + if (created) { + if (fp->coption & FILE_NO_COMPRESSION_LE) { + rc = ksmbd_vfs_set_compression_create(work, fp, + COMPRESSION_FORMAT_NONE); + if (rc) + fp->f_ci->m_fattr &= ~FILE_ATTRIBUTE_COMPRESSED_LE; + rc = 0; + } else if (smb2_parent_compressed(tcon, &path)) { + rc = ksmbd_vfs_set_compression_create(work, fp, + COMPRESSION_FORMAT_LZNT1); + if (rc) + fp->f_ci->m_fattr |= FILE_ATTRIBUTE_COMPRESSED_LE; + rc = 0; + } + } + + if (created) + smb2_new_xattrs(tcon, &path, fp); + + fp->create_action = cpu_to_le32(file_info); + + if (dh_info.type == DURABLE_REQ_V2 || dh_info.type == DURABLE_REQ) { + if (dh_info.type == DURABLE_REQ_V2 && dh_info.persistent && + test_share_config_flag(work->tcon->share_conf, + KSMBD_SHARE_FLAG_CONTINUOUS_AVAILABILITY) && + (conn->vals->req_capabilities & + SMB2_GLOBAL_CAP_PERSISTENT_HANDLES)) { + /* MS-SMB2 3.3.5.9.10: a persistent open is durable too. */ + fp->is_durable = true; + fp->is_persistent = true; + } else { + fp->is_durable = true; + } + if (dh_info.type == DURABLE_REQ_V2) { + if (dh_info.app_instance_id) + memcpy(fp->app_instance_id, + dh_info.AppInstanceId, + SMB2_CREATE_GUID_SIZE); + if (dh_info.timeout) + fp->durable_timeout = + min_t(unsigned int, dh_info.timeout, + DURABLE_HANDLE_MAX_TIMEOUT); + else + fp->durable_timeout = 60000; + } + } + + /* + * conn->is_aapl detection above (this function's create-context + * parsing) is skipped on the reconnect path below, since a + * reconnect always arrives on a fresh connection -- if the client + * cares, it sends its own AAPL context on this same CREATE, which + * this function's normal (non-reconnect) parsing already handles. + */ + reconnected_fp: + if (dh_info.replay) + file_info = le32_to_cpu(fp->create_action); + rsp->StructureSize = cpu_to_le16(89); + opinfo = opinfo_get(fp); + rsp->OplockLevel = opinfo != NULL ? opinfo->level : 0; + /* + * A durable CREATE replay does not modify the existing open. When + * replayed without an oplock, however, its response reflects that + * request and cannot include a new durable-handle response context. + */ + if (dh_info.replay && !lc && + req_op_level == SMB2_OPLOCK_LEVEL_NONE) { + rsp->OplockLevel = SMB2_OPLOCK_LEVEL_NONE; + durable_rsp = false; + } + rsp->Flags = 0; + rsp->CreateAction = cpu_to_le32(file_info); + rsp->CreationTime = cpu_to_le64(fp->create_time); + time = ksmbd_UnixTimeToNT(stat.atime); + rsp->LastAccessTime = cpu_to_le64(time); + time = ksmbd_UnixTimeToNT(stat.mtime); + fp->open_mtime = time; + rsp->LastWriteTime = cpu_to_le64(time); + rsp->ChangeTime = cpu_to_le64(fp->change_time); + /* + * The cached allocation size hides filesystem rounding for the + * requested allocation, but it can go stale when the file grows past + * it via writes (e.g. across a durable reconnect). Refresh it once the + * file exceeds the cached value, rounding the end of file up to the + * volume allocation unit (the filesystem block size, matching the + * SectorsPerAllocationUnit/BytesPerSector ksmbd advertises) rather than + * using the raw on-disk block count, which can include filesystem + * preallocation and metadata rounding. + */ + if (ksmbd_stream_fd(fp)) { + loff_t seof = ksmbd_stream_eof(fp); + + rsp->AllocationSize = cpu_to_le64((u64)seof); + rsp->EndofFile = cpu_to_le64((u64)seof); + } else { + if (!S_ISDIR(stat.mode) && stat.size > fp->allocation_size) + fp->allocation_size = round_up(stat.size, stat.blksize); + rsp->AllocationSize = cpu_to_le64(fp->allocation_size); + rsp->EndofFile = S_ISDIR(stat.mode) ? 0 : cpu_to_le64(stat.size); + } + rsp->FileAttributes = fp->f_ci->m_fattr; + + rsp->Reserved2 = 0; + + rsp->PersistentFileId = fp->persistent_id; + rsp->VolatileFileId = fp->volatile_id; + + rsp->CreateContextsOffset = 0; + rsp->CreateContextsLength = 0; + iov_len = offsetof(struct smb2_create_rsp, Buffer); + + /* If lease is request send lease context response */ + if (opinfo && opinfo->is_lease) { + struct create_context *lease_ccontext; + + ksmbd_debug(SMB, "lease granted on(%s) lease state 0x%x\n", + name, opinfo->o_lease->state); + rsp->OplockLevel = SMB2_OPLOCK_LEVEL_LEASE; + + lease_ccontext = (struct create_context *)rsp->Buffer; + contxt_cnt++; + create_lease_buf(rsp->Buffer, opinfo->o_lease); + le32_add_cpu(&rsp->CreateContextsLength, + conn->vals->create_lease_size); + iov_len += conn->vals->create_lease_size; + next_ptr = &lease_ccontext->Next; + next_off = conn->vals->create_lease_size; + } + opinfo_put(opinfo); + + if (maximal_access_ctxt) { + struct create_context *mxac_ccontext; + + if (maximal_access == 0) + ksmbd_vfs_query_maximal_access(idmap, + path.dentry, + &maximal_access); + mxac_ccontext = (struct create_context *)(rsp->Buffer + + le32_to_cpu(rsp->CreateContextsLength)); + contxt_cnt++; + create_mxac_rsp_buf(rsp->Buffer + + le32_to_cpu(rsp->CreateContextsLength), + le32_to_cpu(maximal_access)); + le32_add_cpu(&rsp->CreateContextsLength, + conn->vals->create_mxac_size); + iov_len += conn->vals->create_mxac_size; + if (next_ptr) + *next_ptr = cpu_to_le32(next_off); + next_ptr = &mxac_ccontext->Next; + next_off = conn->vals->create_mxac_size; + } + + if (query_disk_id) { + struct create_context *disk_id_ccontext; + + disk_id_ccontext = (struct create_context *)(rsp->Buffer + + le32_to_cpu(rsp->CreateContextsLength)); + contxt_cnt++; + create_disk_id_rsp_buf(rsp->Buffer + + le32_to_cpu(rsp->CreateContextsLength), + stat.ino, tcon->id); + le32_add_cpu(&rsp->CreateContextsLength, + conn->vals->create_disk_id_size); + iov_len += conn->vals->create_disk_id_size; + if (next_ptr) + *next_ptr = cpu_to_le32(next_off); + next_ptr = &disk_id_ccontext->Next; + next_off = conn->vals->create_disk_id_size; + } + + if (durable_rsp && + (dh_info.type == DURABLE_REQ || dh_info.type == DURABLE_REQ_V2)) { + struct create_context *durable_ccontext; + + durable_ccontext = (struct create_context *)(rsp->Buffer + + le32_to_cpu(rsp->CreateContextsLength)); + contxt_cnt++; + if (dh_info.type == DURABLE_REQ) { + create_durable_rsp_buf(rsp->Buffer + + le32_to_cpu(rsp->CreateContextsLength)); + le32_add_cpu(&rsp->CreateContextsLength, + conn->vals->create_durable_size); + iov_len += conn->vals->create_durable_size; + } else { + create_durable_v2_rsp_buf(rsp->Buffer + + le32_to_cpu(rsp->CreateContextsLength), + fp); + le32_add_cpu(&rsp->CreateContextsLength, + conn->vals->create_durable_v2_size); + iov_len += conn->vals->create_durable_v2_size; + } + + if (next_ptr) + *next_ptr = cpu_to_le32(next_off); + next_ptr = &durable_ccontext->Next; + next_off = dh_info.type == DURABLE_REQ ? + conn->vals->create_durable_size : + conn->vals->create_durable_v2_size; + } + + if (posix_ctxt) { + struct create_context *posix_ccontext; + + posix_ccontext = (struct create_context *)(rsp->Buffer + + le32_to_cpu(rsp->CreateContextsLength)); + contxt_cnt++; + create_posix_rsp_buf(rsp->Buffer + + le32_to_cpu(rsp->CreateContextsLength), + fp); + le32_add_cpu(&rsp->CreateContextsLength, + conn->vals->create_posix_size); + iov_len += conn->vals->create_posix_size; + if (next_ptr) + *next_ptr = cpu_to_le32(next_off); + next_ptr = &posix_ccontext->Next; + next_off = conn->vals->create_posix_size; + } + + /* + * AAPL create context response: see smb2pdu.h for the capability + * rationale. Scoped to TIME_MACHINE shares only. + */ + if (aapl_ctxt) { + if (aapl_client_caps & SMB2_CRTCTX_AAPL_SUPPORTS_READ_DIR_ATTR) + conn->aapl_readdir_attr = true; + /* + * V2 extends the same inline-FinderInfo mechanism (see + * smb2pdu.h), so a V2-requesting client also gets + * aapl_readdir_attr treatment -- the reply just advertises + * the V2 bit instead of the V1 one (create_aapl_rsp_buf). + */ + if (aapl_client_caps & SMB2_CRTCTX_AAPL_SUPPORTS_READ_DIR_ATTR_V2) { + conn->aapl_readdir_attr = true; + conn->aapl_readdir_attr_v2 = true; + } + + contxt_cnt++; + create_aapl_rsp_buf(rsp->Buffer + + le32_to_cpu(rsp->CreateContextsLength), + SMB2_CRTCTX_AAPL_FULL_SYNC, + aapl_req_bitmap, + conn->aapl_readdir_attr_v2); + le32_add_cpu(&rsp->CreateContextsLength, + conn->vals->create_aapl_size); + iov_len += conn->vals->create_aapl_size; + if (next_ptr) + *next_ptr = cpu_to_le32(next_off); + /* AAPL is last; next_ptr need not be updated */ + } + + if (contxt_cnt > 0) { + rsp->CreateContextsOffset = + cpu_to_le32(offsetof(struct smb2_create_rsp, Buffer)); + } + +err_out: + if (rc && (file_present || created)) + path_put(&path); + +err_out1: + ksmbd_revert_fsids(work); + +err_out2: + if (!rc) { + if (!dh_info.replay) + rc = ksmbd_update_fstate(&work->sess->file_table, fp, + FP_INITED); + if (!rc) + rc = smb2_set_request_open(work, fp, &req->hdr, false, false); + if (!rc) + rc = ksmbd_iov_pin_rsp(work, (void *)rsp, iov_len); + } + if (rc) { + if (rc == -EINVAL) + rsp->hdr.Status = STATUS_INVALID_PARAMETER; + else if (rc == -EOPNOTSUPP) + rsp->hdr.Status = STATUS_NOT_SUPPORTED; + else if ((rc == -EACCES || rc == -ESTALE || rc == -EXDEV) && + !rsp->hdr.Status) { + if (req->DesiredAccess & FILE_ACCESS_SYSTEM_SECURITY_LE) + rsp->hdr.Status = STATUS_PRIVILEGE_NOT_HELD; + else + rsp->hdr.Status = STATUS_ACCESS_DENIED; + } + else if (rc == -ENOENT) + rsp->hdr.Status = STATUS_OBJECT_NAME_INVALID; + else if (rc == -EPERM) + rsp->hdr.Status = STATUS_SHARING_VIOLATION; + else if (rc == -EBUSY) + rsp->hdr.Status = STATUS_DELETE_PENDING; + else if (rc == -EBADF) + rsp->hdr.Status = STATUS_OBJECT_NAME_NOT_FOUND; + else if (rc == -ENOEXEC) + rsp->hdr.Status = STATUS_DUPLICATE_OBJECTID; + else if (rc == -ENXIO) + rsp->hdr.Status = STATUS_NO_SUCH_DEVICE; + else if (rc == -EEXIST) + rsp->hdr.Status = STATUS_OBJECT_NAME_COLLISION; + else if (rc == -EMFILE) + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + else if (rc == -EINPROGRESS) + rsp->hdr.Status = STATUS_FILE_NOT_AVAILABLE; + else if (rc == -EAGAIN) + rsp->hdr.Status = STATUS_FILE_NOT_AVAILABLE; + if (!rsp->hdr.Status) + rsp->hdr.Status = STATUS_UNEXPECTED_IO_ERROR; + + if (fp && !dh_info.replay) + ksmbd_fd_put(work, fp); + smb2_set_err_rsp(work); + ksmbd_debug(SMB, "Error response: %x\n", rsp->hdr.Status); + } + + if (dh_info.replay) + ksmbd_put_durable_fd(dh_info.fp); + + if (dh_info.reconnected) { + /* + * If reconnect succeeded, fp was republished in the + * session file table. On a later error, ksmbd_fd_put() + * above drops the session reference; drop the durable + * lookup reference through the same session-aware path so + * final close removes the volatile id before freeing fp. + */ + if (rc && fp == dh_info.fp) + ksmbd_fd_put(work, dh_info.fp); + else + ksmbd_put_durable_fd(dh_info.fp); + } + + kfree(name); + kfree(lc); + + return rc; +} + +static int readdir_info_level_struct_sz(int info_level) +{ + switch (info_level) { + case FILE_FULL_DIRECTORY_INFORMATION: + return sizeof(FILE_FULL_DIRECTORY_INFO); + case FILE_BOTH_DIRECTORY_INFORMATION: + return sizeof(FILE_BOTH_DIRECTORY_INFO); + case FILE_DIRECTORY_INFORMATION: + return sizeof(FILE_DIRECTORY_INFO); + case FILE_NAMES_INFORMATION: + return sizeof(struct file_names_info); + case FILEID_FULL_DIRECTORY_INFORMATION: + return sizeof(FILE_ID_FULL_DIR_INFO); + case FILEID_BOTH_DIRECTORY_INFORMATION: + return sizeof(struct file_id_both_directory_info); + case SMB_FIND_FILE_POSIX_INFO: + return sizeof(struct smb2_posix_info); + default: + return -EOPNOTSUPP; + } +} + +static int dentry_name(struct ksmbd_dir_info *d_info, int info_level) +{ + switch (info_level) { + case FILE_FULL_DIRECTORY_INFORMATION: + { + FILE_FULL_DIRECTORY_INFO *ffdinfo; + + ffdinfo = (FILE_FULL_DIRECTORY_INFO *)d_info->rptr; + d_info->rptr += le32_to_cpu(ffdinfo->NextEntryOffset); + d_info->name = ffdinfo->FileName; + d_info->name_len = le32_to_cpu(ffdinfo->FileNameLength); + return 0; + } + case FILE_BOTH_DIRECTORY_INFORMATION: + { + FILE_BOTH_DIRECTORY_INFO *fbdinfo; + + fbdinfo = (FILE_BOTH_DIRECTORY_INFO *)d_info->rptr; + d_info->rptr += le32_to_cpu(fbdinfo->NextEntryOffset); + d_info->name = fbdinfo->FileName; + d_info->name_len = le32_to_cpu(fbdinfo->FileNameLength); + return 0; + } + case FILE_DIRECTORY_INFORMATION: + { + FILE_DIRECTORY_INFO *fdinfo; + + fdinfo = (FILE_DIRECTORY_INFO *)d_info->rptr; + d_info->rptr += le32_to_cpu(fdinfo->NextEntryOffset); + d_info->name = fdinfo->FileName; + d_info->name_len = le32_to_cpu(fdinfo->FileNameLength); + return 0; + } + case FILE_NAMES_INFORMATION: + { + struct file_names_info *fninfo; + + fninfo = (struct file_names_info *)d_info->rptr; + d_info->rptr += le32_to_cpu(fninfo->NextEntryOffset); + d_info->name = fninfo->FileName; + d_info->name_len = le32_to_cpu(fninfo->FileNameLength); + return 0; + } + case FILEID_FULL_DIRECTORY_INFORMATION: + { + FILE_ID_FULL_DIR_INFO *dinfo; + + dinfo = (FILE_ID_FULL_DIR_INFO *)d_info->rptr; + d_info->rptr += le32_to_cpu(dinfo->NextEntryOffset); + d_info->name = dinfo->FileName; + d_info->name_len = le32_to_cpu(dinfo->FileNameLength); + return 0; + } + case FILEID_BOTH_DIRECTORY_INFORMATION: + { + struct file_id_both_directory_info *fibdinfo; + + fibdinfo = (struct file_id_both_directory_info *)d_info->rptr; + d_info->rptr += le32_to_cpu(fibdinfo->NextEntryOffset); + d_info->name = fibdinfo->FileName; + d_info->name_len = le32_to_cpu(fibdinfo->FileNameLength); + return 0; + } + case SMB_FIND_FILE_POSIX_INFO: + { + struct smb2_posix_info *posix_info; + + posix_info = (struct smb2_posix_info *)d_info->rptr; + d_info->rptr += le32_to_cpu(posix_info->NextEntryOffset); + d_info->name = posix_info->name; + d_info->name_len = le32_to_cpu(posix_info->name_len); + return 0; + } + default: + return -EINVAL; + } +} + +/** + * smb2_populate_readdir_entry() - encode directory entry in smb2 response + * buffer + * @conn: connection instance + * @info_level: smb information level + * @d_info: structure included variables for query dir + * @ksmbd_kstat: ksmbd wrapper of dirent stat information + * + * if directory has many entries, find first can't read it fully. + * find next might be called multiple times to read remaining dir entries + * + * Return: 0 on success, otherwise error + */ +static int smb2_populate_readdir_entry(struct ksmbd_conn *conn, int info_level, + struct ksmbd_dir_info *d_info, + struct ksmbd_kstat *ksmbd_kstat) +{ + int next_entry_offset = 0; + char *conv_name; + int conv_len; + void *kstat; + int struct_sz, rc = 0; + + conv_name = ksmbd_convert_dir_info_name(d_info, + conn->local_nls, + &conv_len); + if (!conv_name) + return -ENOMEM; + + /* Somehow the name has only terminating NULL bytes */ + if (conv_len < 0) { + rc = -EINVAL; + goto free_conv_name; + } + + struct_sz = readdir_info_level_struct_sz(info_level); + if (struct_sz == -EOPNOTSUPP) { + rc = -EINVAL; + goto free_conv_name; + } + + struct_sz += conv_len; + next_entry_offset = ALIGN(struct_sz, KSMBD_DIR_INFO_ALIGNMENT); + d_info->last_entry_off_align = next_entry_offset - struct_sz; + + if (next_entry_offset > d_info->out_buf_len) { + d_info->out_buf_len = 0; + rc = -ENOSPC; + goto free_conv_name; + } + + kstat = d_info->wptr; + if (info_level != FILE_NAMES_INFORMATION) + kstat = ksmbd_vfs_init_kstat(&d_info->wptr, ksmbd_kstat); + + switch (info_level) { + case FILE_FULL_DIRECTORY_INFORMATION: + { + FILE_FULL_DIRECTORY_INFO *ffdinfo; + + ffdinfo = (FILE_FULL_DIRECTORY_INFO *)kstat; + ffdinfo->FileNameLength = cpu_to_le32(conv_len); + ffdinfo->EaSize = + smb2_get_reparse_tag_special_file(ksmbd_kstat->kstat->mode); + if (ffdinfo->EaSize) + ffdinfo->ExtFileAttributes = FILE_ATTRIBUTE_REPARSE_POINT_LE; + if (d_info->hide_dot_file && d_info->name[0] == '.') + ffdinfo->ExtFileAttributes |= FILE_ATTRIBUTE_HIDDEN_LE; + memcpy(ffdinfo->FileName, conv_name, conv_len); + ffdinfo->NextEntryOffset = cpu_to_le32(next_entry_offset); + break; + } + case FILE_BOTH_DIRECTORY_INFORMATION: + { + FILE_BOTH_DIRECTORY_INFO *fbdinfo; + + fbdinfo = (FILE_BOTH_DIRECTORY_INFO *)kstat; + fbdinfo->FileNameLength = cpu_to_le32(conv_len); + fbdinfo->EaSize = + smb2_get_reparse_tag_special_file(ksmbd_kstat->kstat->mode); + if (fbdinfo->EaSize) + fbdinfo->ExtFileAttributes = FILE_ATTRIBUTE_REPARSE_POINT_LE; + fbdinfo->ShortNameLength = 0; + fbdinfo->Reserved = 0; + if (d_info->hide_dot_file && d_info->name[0] == '.') + fbdinfo->ExtFileAttributes |= FILE_ATTRIBUTE_HIDDEN_LE; + memcpy(fbdinfo->FileName, conv_name, conv_len); + fbdinfo->NextEntryOffset = cpu_to_le32(next_entry_offset); + break; + } + case FILE_DIRECTORY_INFORMATION: + { + FILE_DIRECTORY_INFO *fdinfo; + + fdinfo = (FILE_DIRECTORY_INFO *)kstat; + fdinfo->FileNameLength = cpu_to_le32(conv_len); + if (d_info->hide_dot_file && d_info->name[0] == '.') + fdinfo->ExtFileAttributes |= FILE_ATTRIBUTE_HIDDEN_LE; + memcpy(fdinfo->FileName, conv_name, conv_len); + fdinfo->NextEntryOffset = cpu_to_le32(next_entry_offset); + break; + } + case FILE_NAMES_INFORMATION: + { + struct file_names_info *fninfo; + + fninfo = (struct file_names_info *)kstat; + fninfo->FileNameLength = cpu_to_le32(conv_len); + memcpy(fninfo->FileName, conv_name, conv_len); + fninfo->NextEntryOffset = cpu_to_le32(next_entry_offset); + break; + } + case FILEID_FULL_DIRECTORY_INFORMATION: + { + FILE_ID_FULL_DIR_INFO *dinfo; + + dinfo = (FILE_ID_FULL_DIR_INFO *)kstat; + dinfo->FileNameLength = cpu_to_le32(conv_len); + dinfo->EaSize = + smb2_get_reparse_tag_special_file(ksmbd_kstat->kstat->mode); + if (dinfo->EaSize) + dinfo->ExtFileAttributes = FILE_ATTRIBUTE_REPARSE_POINT_LE; + dinfo->Reserved = 0; + if (conn->is_aapl) + dinfo->UniqueId = 0; + else + dinfo->UniqueId = cpu_to_le64(ksmbd_kstat->kstat->ino); + if (d_info->hide_dot_file && d_info->name[0] == '.') + dinfo->ExtFileAttributes |= FILE_ATTRIBUTE_HIDDEN_LE; + memcpy(dinfo->FileName, conv_name, conv_len); + dinfo->NextEntryOffset = cpu_to_le32(next_entry_offset); + break; + } + case FILEID_BOTH_DIRECTORY_INFORMATION: + { + struct file_id_both_directory_info *fibdinfo; + + fibdinfo = (struct file_id_both_directory_info *)kstat; + fibdinfo->FileNameLength = cpu_to_le32(conv_len); + if (conn->is_aapl) + fibdinfo->UniqueId = 0; + else + fibdinfo->UniqueId = cpu_to_le64(ksmbd_kstat->kstat->ino); + fibdinfo->ShortNameLength = 0; + fibdinfo->Reserved = 0; + if (conn->aapl_readdir_attr) { + /* + * READDIR_ATTR wire format, confirmed against reference server's + * reference implementation marshalling (reference implementation behavior): + * EaSize = max_access (expanded specific + * rights, simplified to "grant all") + * ShortNameLength = 24 (fixed; not 0, despite the spec) + * ShortName[0..7] = resource fork size (uint64 LE, 0 = no rfork) + * ShortName[8..23] = compressed FinderInfo (type+creator+flags+ + * ext_flags+date_added, 16 bytes LE; all + * zeros means type=0/creator=0, i.e. use + * the file extension for icon lookup) + * Reserved2 = Unix mode bits (uint16 LE) + * Reparse-point tag is indicated via ExtFileAttributes, not EaSize. + * + * V2 (conn->aapl_readdir_attr_v2): ShortNameLength+Reserved + * are read as a single flags field instead of being ignored + * -- see smb2pdu.h for the wire-format confirmation and + * AAPL_READDIR_ATTR_V2_NO_XATTR's meaning. + */ + __le32 reparse_tag = + smb2_get_reparse_tag_special_file(ksmbd_kstat->kstat->mode); + + if (reparse_tag) + fibdinfo->ExtFileAttributes = FILE_ATTRIBUTE_REPARSE_POINT_LE; + /* + * FILE_GENERIC_ALL_LE (0x10000000) is the raw + * "generic all" meta-bit -- valid only in a + * client's requested access mask, for the server + * to expand. It has none of the specific FILE_* + * rights bits set (FILE_LIST_DIRECTORY, FILE_TRAVERSE, + * etc.), so reporting it here as max_access would make + * macOS's bit-by-bit access checks fail on every + * entry -> permanent "no entry" badges in Finder. + * Report the actual expanded rights instead, same + * as smb_map_generic_desired_access() does when + * translating a client's GENERIC_ALL request. + */ + fibdinfo->EaSize = cpu_to_le32(GENERIC_ALL_FLAGS); + /* + * The spec says ShortNameLength should be 0 when + * there's no short name; 24 here instead matches + * reference implementation marshalling (reference + * behavior) for server-to-server wire parity. + * V2 repurposes it as a flags field that is + * interpreted; V1 doesn't. Either value is safe + * here, so keep 24 for parity. + */ + if (conn->aapl_readdir_attr_v2) { + /* + * V2 repurposes this field as flags (see comment + * above) -- 24 is a V1-only convention that real + * macOS clients ignore outright, so don't reuse it + * here as a base value for a field V2 clients + * actually interpret. + */ + fibdinfo->ShortNameLength = 0; + if (!ksmbd_kstat->has_ads_stream) + fibdinfo->ShortNameLength = AAPL_READDIR_ATTR_V2_NO_XATTR; + } else { + fibdinfo->ShortNameLength = 24; + } + memset(fibdinfo->ShortName, 0, sizeof(fibdinfo->ShortName)); + fibdinfo->Reserved2 = cpu_to_le16(ksmbd_kstat->kstat->mode & 0xffff); + } else { + fibdinfo->EaSize = + smb2_get_reparse_tag_special_file(ksmbd_kstat->kstat->mode); + if (fibdinfo->EaSize) + fibdinfo->ExtFileAttributes = FILE_ATTRIBUTE_REPARSE_POINT_LE; + fibdinfo->Reserved2 = cpu_to_le16(0); + } + if (d_info->hide_dot_file && d_info->name[0] == '.') + fibdinfo->ExtFileAttributes |= FILE_ATTRIBUTE_HIDDEN_LE; + memcpy(fibdinfo->FileName, conv_name, conv_len); + fibdinfo->NextEntryOffset = cpu_to_le32(next_entry_offset); + break; + } + case SMB_FIND_FILE_POSIX_INFO: + { + struct smb2_posix_info *posix_info; + u64 time; + + posix_info = (struct smb2_posix_info *)kstat; + posix_info->Ignored = 0; + posix_info->CreationTime = cpu_to_le64(ksmbd_kstat->create_time); + time = ksmbd_UnixTimeToNT(ksmbd_kstat->kstat->ctime); + posix_info->ChangeTime = cpu_to_le64(time); + time = ksmbd_UnixTimeToNT(ksmbd_kstat->kstat->atime); + posix_info->LastAccessTime = cpu_to_le64(time); + time = ksmbd_UnixTimeToNT(ksmbd_kstat->kstat->mtime); + posix_info->LastWriteTime = cpu_to_le64(time); + posix_info->EndOfFile = cpu_to_le64(ksmbd_kstat->kstat->size); + posix_info->AllocationSize = cpu_to_le64(ksmbd_kstat->kstat->blocks << 9); + posix_info->DeviceId = cpu_to_le32(ksmbd_kstat->kstat->rdev); + posix_info->HardLinks = cpu_to_le32(ksmbd_kstat->kstat->nlink); + posix_info->Mode = cpu_to_le32(ksmbd_kstat->kstat->mode & 0777); + switch (ksmbd_kstat->kstat->mode & S_IFMT) { + case S_IFDIR: + posix_info->Mode |= cpu_to_le32(POSIX_TYPE_DIR << POSIX_FILETYPE_SHIFT); + break; + case S_IFLNK: + posix_info->Mode |= cpu_to_le32(POSIX_TYPE_SYMLINK << POSIX_FILETYPE_SHIFT); + break; + case S_IFCHR: + posix_info->Mode |= cpu_to_le32(POSIX_TYPE_CHARDEV << POSIX_FILETYPE_SHIFT); + break; + case S_IFBLK: + posix_info->Mode |= cpu_to_le32(POSIX_TYPE_BLKDEV << POSIX_FILETYPE_SHIFT); + break; + case S_IFIFO: + posix_info->Mode |= cpu_to_le32(POSIX_TYPE_FIFO << POSIX_FILETYPE_SHIFT); + break; + case S_IFSOCK: + posix_info->Mode |= cpu_to_le32(POSIX_TYPE_SOCKET << POSIX_FILETYPE_SHIFT); + } + + posix_info->Inode = cpu_to_le64(ksmbd_kstat->kstat->ino); + posix_info->DosAttributes = + S_ISDIR(ksmbd_kstat->kstat->mode) ? + FILE_ATTRIBUTE_DIRECTORY_LE : FILE_ATTRIBUTE_ARCHIVE_LE; + if (d_info->hide_dot_file && d_info->name[0] == '.') + posix_info->DosAttributes |= FILE_ATTRIBUTE_HIDDEN_LE; + /* + * SidBuffer(32) contain two sids(Domain sid(16), UNIX group sid(16)). + * UNIX sid(16) = revision(1) + num_subauth(1) + authority(6) + + * sub_auth(4 * 1(num_subauth)) + RID(4). + */ + id_to_sid(from_kuid_munged(&init_user_ns, ksmbd_kstat->kstat->uid), + SIDUNIX_USER, (struct smb_sid *)&posix_info->SidBuffer[0]); + id_to_sid(from_kgid_munged(&init_user_ns, ksmbd_kstat->kstat->gid), + SIDUNIX_GROUP, (struct smb_sid *)&posix_info->SidBuffer[16]); + memcpy(posix_info->name, conv_name, conv_len); + posix_info->name_len = cpu_to_le32(conv_len); + posix_info->NextEntryOffset = cpu_to_le32(next_entry_offset); + break; + } + + } /* switch (info_level) */ + + d_info->last_entry_offset = d_info->data_count; + d_info->data_count += next_entry_offset; + d_info->out_buf_len -= next_entry_offset; + d_info->wptr += next_entry_offset; + + ksmbd_debug(SMB, + "info_level : %d, buf_len :%d, next_offset : %d, data_count : %d\n", + info_level, d_info->out_buf_len, + next_entry_offset, d_info->data_count); + +free_conv_name: + kfree(conv_name); + return rc; +} + +struct smb2_query_dir_private { + struct ksmbd_work *work; + char *search_pattern; + struct ksmbd_file *dir_fp; + + struct ksmbd_dir_info *d_info; + int info_level; +}; + +static int process_query_dir_entries(struct smb2_query_dir_private *priv) +{ + struct mnt_idmap *idmap = file_mnt_idmap(priv->dir_fp->filp); + struct kstat kstat; + struct ksmbd_kstat ksmbd_kstat; + int rc; + int i; + + for (i = 0; i < priv->d_info->num_entry; i++) { + struct dentry *dent; + struct path path; + + if (dentry_name(priv->d_info, priv->info_level)) + return -EINVAL; + + dent = lookup_one_unlocked(idmap, + &QSTR_LEN(priv->d_info->name, + priv->d_info->name_len), + priv->dir_fp->filp->f_path.dentry); + + if (IS_ERR(dent)) { + ksmbd_debug(SMB, "Cannot lookup `%s' [%ld]\n", + priv->d_info->name, + PTR_ERR(dent)); + continue; + } + if (unlikely(d_is_negative(dent))) { + dput(dent); + ksmbd_debug(SMB, "Negative dentry `%s'\n", + priv->d_info->name); + continue; + } + + if (test_share_config_flag(priv->work->tcon->share_conf, + KSMBD_SHARE_FLAG_HIDE_UNREADABLE)) { + __le32 daccess = FILE_READ_DATA_LE | FILE_READ_EA_LE | + FILE_READ_ATTRIBUTES_LE; + + path.mnt = priv->dir_fp->filp->f_path.mnt; + path.dentry = dent; + rc = smb_check_perm_dacl(priv->work->conn, &path, + &daccess, daccess, + priv->work->sess->user->uid, + true); + if (rc) { + dput(dent); + continue; + } + } + + ksmbd_kstat.kstat = &kstat; + if (priv->info_level != FILE_NAMES_INFORMATION) { + rc = ksmbd_vfs_fill_dentry_attrs(priv->work, + idmap, + dent, + &ksmbd_kstat); + if (rc) { + dput(dent); + continue; + } + } + + rc = smb2_populate_readdir_entry(priv->work->conn, + priv->info_level, + priv->d_info, + &ksmbd_kstat); + dput(dent); + if (rc) + return rc; + } + return 0; +} + +static int reserve_populate_dentry(struct ksmbd_dir_info *d_info, + int info_level) +{ + int struct_sz; + int conv_len; + int next_entry_offset; + + struct_sz = readdir_info_level_struct_sz(info_level); + if (struct_sz == -EOPNOTSUPP) + return -EOPNOTSUPP; + + conv_len = (d_info->name_len + 1) * 2; + next_entry_offset = ALIGN(struct_sz + conv_len, + KSMBD_DIR_INFO_ALIGNMENT); + + if (next_entry_offset > d_info->out_buf_len) { + d_info->out_buf_len = 0; + return -ENOSPC; + } + + switch (info_level) { + case FILE_FULL_DIRECTORY_INFORMATION: + { + FILE_FULL_DIRECTORY_INFO *ffdinfo; + + ffdinfo = (FILE_FULL_DIRECTORY_INFO *)d_info->wptr; + memcpy(ffdinfo->FileName, d_info->name, d_info->name_len); + ffdinfo->FileName[d_info->name_len] = 0x00; + ffdinfo->FileNameLength = cpu_to_le32(d_info->name_len); + ffdinfo->NextEntryOffset = cpu_to_le32(next_entry_offset); + break; + } + case FILE_BOTH_DIRECTORY_INFORMATION: + { + FILE_BOTH_DIRECTORY_INFO *fbdinfo; + + fbdinfo = (FILE_BOTH_DIRECTORY_INFO *)d_info->wptr; + memcpy(fbdinfo->FileName, d_info->name, d_info->name_len); + fbdinfo->FileName[d_info->name_len] = 0x00; + fbdinfo->FileNameLength = cpu_to_le32(d_info->name_len); + fbdinfo->NextEntryOffset = cpu_to_le32(next_entry_offset); + break; + } + case FILE_DIRECTORY_INFORMATION: + { + FILE_DIRECTORY_INFO *fdinfo; + + fdinfo = (FILE_DIRECTORY_INFO *)d_info->wptr; + memcpy(fdinfo->FileName, d_info->name, d_info->name_len); + fdinfo->FileName[d_info->name_len] = 0x00; + fdinfo->FileNameLength = cpu_to_le32(d_info->name_len); + fdinfo->NextEntryOffset = cpu_to_le32(next_entry_offset); + break; + } + case FILE_NAMES_INFORMATION: + { + struct file_names_info *fninfo; + + fninfo = (struct file_names_info *)d_info->wptr; + memcpy(fninfo->FileName, d_info->name, d_info->name_len); + fninfo->FileName[d_info->name_len] = 0x00; + fninfo->FileNameLength = cpu_to_le32(d_info->name_len); + fninfo->NextEntryOffset = cpu_to_le32(next_entry_offset); + break; + } + case FILEID_FULL_DIRECTORY_INFORMATION: + { + FILE_ID_FULL_DIR_INFO *dinfo; + + dinfo = (FILE_ID_FULL_DIR_INFO *)d_info->wptr; + memcpy(dinfo->FileName, d_info->name, d_info->name_len); + dinfo->FileName[d_info->name_len] = 0x00; + dinfo->FileNameLength = cpu_to_le32(d_info->name_len); + dinfo->NextEntryOffset = cpu_to_le32(next_entry_offset); + break; + } + case FILEID_BOTH_DIRECTORY_INFORMATION: + { + struct file_id_both_directory_info *fibdinfo; + + fibdinfo = (struct file_id_both_directory_info *)d_info->wptr; + memcpy(fibdinfo->FileName, d_info->name, d_info->name_len); + fibdinfo->FileName[d_info->name_len] = 0x00; + fibdinfo->FileNameLength = cpu_to_le32(d_info->name_len); + fibdinfo->NextEntryOffset = cpu_to_le32(next_entry_offset); + break; + } + case SMB_FIND_FILE_POSIX_INFO: + { + struct smb2_posix_info *posix_info; + + posix_info = (struct smb2_posix_info *)d_info->wptr; + memcpy(posix_info->name, d_info->name, d_info->name_len); + posix_info->name[d_info->name_len] = 0x00; + posix_info->name_len = cpu_to_le32(d_info->name_len); + posix_info->NextEntryOffset = + cpu_to_le32(next_entry_offset); + break; + } + } /* switch (info_level) */ + + d_info->num_entry++; + d_info->out_buf_len -= next_entry_offset; + d_info->wptr += next_entry_offset; + return 0; +} + +static bool __query_dir(struct dir_context *ctx, const char *name, int namlen, + loff_t offset, u64 ino, unsigned int d_type) +{ + struct ksmbd_readdir_data *buf; + struct smb2_query_dir_private *priv; + struct ksmbd_dir_info *d_info; + int rc; + + buf = container_of(ctx, struct ksmbd_readdir_data, ctx); + priv = buf->private; + d_info = priv->d_info; + + /* dot and dotdot entries are already reserved */ + if (!strcmp(".", name) || !strcmp("..", name)) + return true; + d_info->num_scan++; + if (ksmbd_share_veto_filename(priv->work->tcon->share_conf, name)) + return true; + if (!match_pattern(name, namlen, priv->search_pattern)) + return true; + + d_info->name = name; + d_info->name_len = namlen; + rc = reserve_populate_dentry(d_info, priv->info_level); + if (rc) + return false; + if (d_info->flags & SMB2_RETURN_SINGLE_ENTRY) + d_info->out_buf_len = 0; + return true; +} + +static int verify_info_level(int info_level) +{ + switch (info_level) { + case FILE_FULL_DIRECTORY_INFORMATION: + case FILE_BOTH_DIRECTORY_INFORMATION: + case FILE_DIRECTORY_INFORMATION: + case FILE_NAMES_INFORMATION: + case FILEID_FULL_DIRECTORY_INFORMATION: + case FILEID_BOTH_DIRECTORY_INFORMATION: + case SMB_FIND_FILE_POSIX_INFO: + break; + default: + return -EOPNOTSUPP; + } + + return 0; +} + +static int smb2_resp_buf_len(struct ksmbd_work *work, unsigned short hdr2_len) +{ + int free_len; + + free_len = (int)(work->response_sz - + (get_rfc1002_len(work->response_buf) + 4)) - hdr2_len; + return free_len; +} + +static int smb2_calc_max_out_buf_len(struct ksmbd_work *work, + unsigned short hdr2_len, + unsigned int out_buf_len) +{ + int free_len; + + if (out_buf_len > work->conn->vals->max_trans_size) + return -EINVAL; + + free_len = smb2_resp_buf_len(work, hdr2_len); + if (free_len < 0) + return -EINVAL; + + return min_t(int, out_buf_len, free_len); +} + +int smb2_query_dir(struct ksmbd_work *work) +{ + struct ksmbd_conn *conn = work->conn; + struct smb2_query_directory_req *req; + struct smb2_query_directory_rsp *rsp; + struct ksmbd_share_config *share = work->tcon->share_conf; + struct ksmbd_file *dir_fp = NULL; + struct ksmbd_dir_info d_info; + int rc = 0; + char *srch_ptr = NULL; + unsigned char srch_flag; + int buffer_sz; + struct smb2_query_dir_private query_dir_private = {NULL, }; + unsigned int id = KSMBD_NO_FID, pid = KSMBD_NO_FID; + + ksmbd_debug(SMB, "Received smb2 query directory request\n"); + + WORK_BUFFERS(work, req, rsp); + + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; + + if (work->next_smb2_rcv_hdr_off && + !has_file_id(req->VolatileFileId)) { + ksmbd_debug(SMB, "Compound request set FID = %llu\n", + work->compound_fid); + id = work->compound_fid; + pid = work->compound_pfid; + } + + if (!has_file_id(id)) { + id = req->VolatileFileId; + pid = req->PersistentFileId; + } + + if (ksmbd_override_fsids(work)) { + rsp->hdr.Status = STATUS_NO_MEMORY; + smb2_set_err_rsp(work); + return -ENOMEM; + } + + rc = verify_info_level(req->FileInformationClass); + if (rc) { + rc = -EFAULT; + goto err_out2; + } + + dir_fp = ksmbd_lookup_fd_slow(work, id, pid); + if (!dir_fp) { + rc = -EBADF; + goto err_out2; + } + + if (!(dir_fp->daccess & FILE_LIST_DIRECTORY_LE) || + inode_permission(file_mnt_idmap(dir_fp->filp), + file_inode(dir_fp->filp), + MAY_READ | MAY_EXEC)) { + pr_err("no right to enumerate directory (%pD)\n", dir_fp->filp); + rc = -EACCES; + goto err_out2; + } + + if (!S_ISDIR(file_inode(dir_fp->filp)->i_mode)) { + pr_err("can't do query dir for a file\n"); + rc = -EINVAL; + goto err_out2; + } + + srch_flag = req->Flags; + srch_ptr = smb_strndup_from_utf16((char *)req + le16_to_cpu(req->FileNameOffset), + le16_to_cpu(req->FileNameLength), 1, + conn->local_nls); + if (IS_ERR(srch_ptr)) { + ksmbd_debug(SMB, "Search Pattern not found\n"); + rc = -EINVAL; + goto err_out2; + } else { + ksmbd_debug(SMB, "Search pattern is %s\n", srch_ptr); + } + + mutex_lock(&dir_fp->readdir_lock); + + if (srch_flag & SMB2_REOPEN || srch_flag & SMB2_RESTART_SCANS) { + ksmbd_debug(SMB, "Restart directory scan\n"); + generic_file_llseek(dir_fp->filp, 0, SEEK_SET); + } + + memset(&d_info, 0, sizeof(struct ksmbd_dir_info)); + d_info.wptr = (char *)rsp->Buffer; + d_info.rptr = (char *)rsp->Buffer; + d_info.out_buf_len = + smb2_calc_max_out_buf_len(work, + offsetof(struct smb2_query_directory_rsp, Buffer), + le32_to_cpu(req->OutputBufferLength)); + if (d_info.out_buf_len < 0) { + rc = -EINVAL; + goto err_out; + } + d_info.flags = srch_flag; + + /* + * reserve dot and dotdot entries in head of buffer + * in first response + */ + rc = ksmbd_populate_dot_dotdot_entries(work, req->FileInformationClass, + dir_fp, &d_info, srch_ptr, + smb2_populate_readdir_entry); + if (rc == -ENOSPC) + rc = 0; + else if (rc) + goto err_out; + + if (test_share_config_flag(share, KSMBD_SHARE_FLAG_HIDE_DOT_FILES)) + d_info.hide_dot_file = true; + + buffer_sz = d_info.out_buf_len; + d_info.rptr = d_info.wptr; + query_dir_private.work = work; + query_dir_private.search_pattern = srch_ptr; + query_dir_private.dir_fp = dir_fp; + query_dir_private.d_info = &d_info; + query_dir_private.info_level = req->FileInformationClass; + dir_fp->readdir_data.private = &query_dir_private; + set_ctx_actor(&dir_fp->readdir_data.ctx, __query_dir); +again: + d_info.num_scan = 0; + rc = iterate_dir(dir_fp->filp, &dir_fp->readdir_data.ctx); + /* + * num_entry can be 0 if the directory iteration stops before reaching + * the end of the directory and no file is matched with the search + * pattern. + */ + if (rc >= 0 && !d_info.num_entry && d_info.num_scan && + d_info.out_buf_len > 0) + goto again; + /* + * req->OutputBufferLength is too small to contain even one entry. + * In this case, it immediately returns OutputBufferLength 0 to client. + */ + if (!d_info.out_buf_len && !d_info.num_entry) + goto no_buf_len; + if (rc > 0 || rc == -ENOSPC) + rc = 0; + else if (rc) + goto err_out; + + d_info.wptr = d_info.rptr; + d_info.out_buf_len = buffer_sz; + rc = process_query_dir_entries(&query_dir_private); + if (rc) + goto err_out; + + if (!d_info.data_count && d_info.out_buf_len >= 0) { + if (srch_flag & SMB2_RETURN_SINGLE_ENTRY && !is_asterisk(srch_ptr)) { + rsp->hdr.Status = STATUS_NO_SUCH_FILE; + } else { + dir_fp->dot_dotdot[0] = dir_fp->dot_dotdot[1] = 0; + rsp->hdr.Status = STATUS_NO_MORE_FILES; + } + rsp->StructureSize = cpu_to_le16(9); + rsp->OutputBufferOffset = cpu_to_le16(0); + rsp->OutputBufferLength = cpu_to_le32(0); + rsp->Buffer[0] = 0; + rc = ksmbd_iov_pin_rsp(work, (void *)rsp, + offsetof(struct smb2_query_directory_rsp, Buffer) + + 1); + if (rc) + goto err_out; + } else { +no_buf_len: + ((FILE_DIRECTORY_INFO *) + ((char *)rsp->Buffer + d_info.last_entry_offset)) + ->NextEntryOffset = 0; + if (d_info.data_count >= d_info.last_entry_off_align) + d_info.data_count -= d_info.last_entry_off_align; + + rsp->StructureSize = cpu_to_le16(9); + rsp->OutputBufferOffset = cpu_to_le16(72); + rsp->OutputBufferLength = cpu_to_le32(d_info.data_count); + rc = ksmbd_iov_pin_rsp(work, (void *)rsp, + offsetof(struct smb2_query_directory_rsp, Buffer) + + d_info.data_count); + if (rc) + goto err_out; + } + + mutex_unlock(&dir_fp->readdir_lock); + kfree(srch_ptr); + ksmbd_fd_put(work, dir_fp); + ksmbd_revert_fsids(work); + return 0; + +err_out: + pr_err("error while processing smb2 query dir rc = %d\n", rc); + mutex_unlock(&dir_fp->readdir_lock); + kfree(srch_ptr); + +err_out2: + if (rc == -EINVAL) + rsp->hdr.Status = STATUS_INVALID_PARAMETER; + else if (rc == -EACCES) + rsp->hdr.Status = STATUS_ACCESS_DENIED; + else if (rc == -ENOENT) + rsp->hdr.Status = STATUS_NO_SUCH_FILE; + else if (rc == -EBADF) + rsp->hdr.Status = STATUS_FILE_CLOSED; + else if (rc == -ENOMEM) + rsp->hdr.Status = STATUS_NO_MEMORY; + else if (rc == -EFAULT) + rsp->hdr.Status = STATUS_INVALID_INFO_CLASS; + else if (rc == -EIO) + rsp->hdr.Status = STATUS_FILE_CORRUPT_ERROR; + if (!rsp->hdr.Status) + rsp->hdr.Status = STATUS_UNEXPECTED_IO_ERROR; + + smb2_set_err_rsp(work); + ksmbd_fd_put(work, dir_fp); + ksmbd_revert_fsids(work); + return rc; +} + +/** + * buffer_check_err() - helper function to check buffer errors + * @reqOutputBufferLength: max buffer length expected in command response + * @fixed_len: minimum fixed response length + * @rsp: query info response buffer contains output buffer length + * @rsp_org: base response buffer pointer in case of chained response + * + * Return: 0 on success, otherwise error + */ +static int buffer_check_err(int reqOutputBufferLength, + unsigned int fixed_len, + struct smb2_query_info_rsp *rsp, + void *rsp_org) +{ + unsigned int output_len = le32_to_cpu(rsp->OutputBufferLength); + + if (reqOutputBufferLength < fixed_len) { + pr_err("Invalid Buffer Size Requested\n"); + rsp->hdr.Status = STATUS_INFO_LENGTH_MISMATCH; + *(__be32 *)rsp_org = cpu_to_be32(sizeof(struct smb2_hdr)); + return -EINVAL; + } + + if (reqOutputBufferLength < output_len) { + rsp->hdr.Status = STATUS_BUFFER_OVERFLOW; + rsp->OutputBufferLength = cpu_to_le32(reqOutputBufferLength); + } + return 0; +} + +static void get_standard_info_pipe(struct smb2_query_info_rsp *rsp, + void *rsp_org) +{ + struct smb2_file_standard_info *sinfo; + + sinfo = (struct smb2_file_standard_info *)rsp->Buffer; + + sinfo->AllocationSize = cpu_to_le64(4096); + sinfo->EndOfFile = cpu_to_le64(0); + sinfo->NumberOfLinks = cpu_to_le32(1); + sinfo->DeletePending = 1; + sinfo->Directory = 0; + rsp->OutputBufferLength = + cpu_to_le32(sizeof(struct smb2_file_standard_info)); +} + +static void get_internal_info_pipe(struct smb2_query_info_rsp *rsp, u64 num, + void *rsp_org) +{ + struct smb2_file_internal_info *file_info; + + file_info = (struct smb2_file_internal_info *)rsp->Buffer; + + /* any unique number */ + file_info->IndexNumber = cpu_to_le64(num | (1ULL << 63)); + rsp->OutputBufferLength = + cpu_to_le32(sizeof(struct smb2_file_internal_info)); +} + +static int smb2_get_info_file_pipe(struct ksmbd_session *sess, + struct smb2_query_info_req *req, + struct smb2_query_info_rsp *rsp, + void *rsp_org) +{ + u64 id; + int rc; + + /* + * Windows can sometime send query file info request on + * pipe without opening it, checking error condition here + */ + id = req->VolatileFileId; + + lockdep_assert_not_held(&sess->rpc_lock); + + down_read(&sess->rpc_lock); + if (!ksmbd_session_rpc_method(sess, id)) { + up_read(&sess->rpc_lock); + return -ENOENT; + } + up_read(&sess->rpc_lock); + + ksmbd_debug(SMB, "FileInfoClass %u, FileId 0x%llx\n", + req->FileInfoClass, req->VolatileFileId); + + switch (req->FileInfoClass) { + case FILE_STANDARD_INFORMATION: + get_standard_info_pipe(rsp, rsp_org); + rc = buffer_check_err(le32_to_cpu(req->OutputBufferLength), + le32_to_cpu(rsp->OutputBufferLength), + rsp, rsp_org); + break; + case FILE_INTERNAL_INFORMATION: + get_internal_info_pipe(rsp, id, rsp_org); + rc = buffer_check_err(le32_to_cpu(req->OutputBufferLength), + le32_to_cpu(rsp->OutputBufferLength), + rsp, rsp_org); + break; + default: + ksmbd_debug(SMB, "smb2_info_file_pipe for %u not supported\n", + req->FileInfoClass); + rc = -EOPNOTSUPP; + } + return rc; +} + +/** + * smb2_get_ea() - handler for smb2 get extended attribute command + * @work: smb work containing query info command buffer + * @fp: ksmbd_file pointer + * @req: get extended attribute request + * @rsp: response buffer pointer + * @rsp_org: base response buffer pointer in case of chained response + * + * Return: 0 on success, otherwise error + */ +static int smb2_get_ea(struct ksmbd_work *work, struct ksmbd_file *fp, + struct smb2_query_info_req *req, + struct smb2_query_info_rsp *rsp, void *rsp_org) +{ + struct smb2_ea_info *eainfo, *prev_eainfo; + char *name, *ptr, *xattr_list = NULL, *buf; + int rc, name_len, value_len, xattr_list_len, idx; + ssize_t buf_free_len, alignment_bytes, next_offset, rsp_data_cnt = 0; + struct smb2_ea_info_req *ea_req = NULL; + const struct path *path; + struct mnt_idmap *idmap = file_mnt_idmap(fp->filp); + + if (!(fp->daccess & FILE_READ_EA_LE)) { + pr_err("Not permitted to read ext attr : 0x%x\n", + fp->daccess); + return -EACCES; + } + + path = &fp->filp->f_path; + /* single EA entry is requested with given user.* name */ + if (req->InputBufferLength) { + if (le32_to_cpu(req->InputBufferLength) <= + sizeof(struct smb2_ea_info_req)) + return -EINVAL; + + ea_req = (struct smb2_ea_info_req *)((char *)req + + le16_to_cpu(req->InputBufferOffset)); + + if (le32_to_cpu(req->InputBufferLength) < + offsetof(struct smb2_ea_info_req, name) + + ea_req->EaNameLength) + return -EINVAL; + } else { + /* need to send all EAs, if no specific EA is requested*/ + if (le32_to_cpu(req->Flags) & SL_RETURN_SINGLE_ENTRY) + ksmbd_debug(SMB, + "All EAs are requested but need to send single EA entry in rsp flags 0x%x\n", + le32_to_cpu(req->Flags)); + } + + buf_free_len = + smb2_calc_max_out_buf_len(work, + offsetof(struct smb2_query_info_rsp, Buffer), + le32_to_cpu(req->OutputBufferLength)); + if (buf_free_len < 0) + return -EINVAL; + + rc = ksmbd_vfs_listxattr(path->dentry, &xattr_list); + if (rc < 0) { + rsp->hdr.Status = STATUS_INVALID_HANDLE; + goto out; + } else if (!rc) { /* there is no EA in the file */ + ksmbd_debug(SMB, "no ea data in the file\n"); + goto done; + } + xattr_list_len = rc; + + ptr = (char *)rsp->Buffer; + eainfo = (struct smb2_ea_info *)ptr; + prev_eainfo = eainfo; + idx = 0; + + while (idx < xattr_list_len) { + name = xattr_list + idx; + name_len = strlen(name); + + ksmbd_debug(SMB, "%s, len %d\n", name, name_len); + idx += name_len + 1; + + /* + * CIFS does not support EA other than user.* namespace, + * still keep the framework generic, to list other attrs + * in future. + */ + if (strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN)) + continue; + + if (req->InputBufferLength && + strncmp(&name[XATTR_USER_PREFIX_LEN], ea_req->name, + ea_req->EaNameLength)) + continue; + + if (smb2_is_private_ea(&name[XATTR_USER_PREFIX_LEN], + name_len - XATTR_USER_PREFIX_LEN)) + continue; + + if (!strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN)) + name_len -= XATTR_USER_PREFIX_LEN; + + ptr = eainfo->name + name_len + 1; + buf_free_len -= (offsetof(struct smb2_ea_info, name) + + name_len + 1); + /* bailout if xattr can't fit in buf_free_len */ + value_len = ksmbd_vfs_getxattr(idmap, path->dentry, + name, &buf); + if (value_len <= 0) { + rc = -ENOENT; + rsp->hdr.Status = STATUS_INVALID_HANDLE; + goto out; + } + + buf_free_len -= value_len; + if (buf_free_len < 0) { + kfree(buf); + break; + } + + memcpy(ptr, buf, value_len); + kfree(buf); + + ptr += value_len; + eainfo->Flags = 0; + eainfo->EaNameLength = name_len; + + if (!strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN)) + memcpy(eainfo->name, &name[XATTR_USER_PREFIX_LEN], + name_len); + else + memcpy(eainfo->name, name, name_len); + + eainfo->name[name_len] = '\0'; + eainfo->EaValueLength = cpu_to_le16(value_len); + next_offset = offsetof(struct smb2_ea_info, name) + + name_len + 1 + value_len; + + /* align next xattr entry at 4 byte bundary */ + alignment_bytes = ((next_offset + 3) & ~3) - next_offset; + if (alignment_bytes) { + if (buf_free_len < alignment_bytes) + break; + memset(ptr, '\0', alignment_bytes); + ptr += alignment_bytes; + next_offset += alignment_bytes; + buf_free_len -= alignment_bytes; + } + eainfo->NextEntryOffset = cpu_to_le32(next_offset); + prev_eainfo = eainfo; + eainfo = (struct smb2_ea_info *)ptr; + rsp_data_cnt += next_offset; + + if (req->InputBufferLength) { + ksmbd_debug(SMB, "single entry requested\n"); + break; + } + } + + /* no more ea entries */ + prev_eainfo->NextEntryOffset = 0; +done: + rc = 0; + if (rsp_data_cnt == 0) + rsp->hdr.Status = STATUS_NO_EAS_ON_FILE; + rsp->OutputBufferLength = cpu_to_le32(rsp_data_cnt); +out: + kvfree(xattr_list); + return rc; +} + +static void get_file_access_info(struct smb2_query_info_rsp *rsp, + struct ksmbd_file *fp, void *rsp_org) +{ + struct smb2_file_access_info *file_info; + + file_info = (struct smb2_file_access_info *)rsp->Buffer; + file_info->AccessFlags = fp->daccess; + rsp->OutputBufferLength = + cpu_to_le32(sizeof(struct smb2_file_access_info)); +} + +static int get_file_basic_info(struct smb2_query_info_rsp *rsp, + struct ksmbd_file *fp, void *rsp_org) +{ + struct file_basic_info *basic_info; + struct kstat stat; + u64 time; + int ret; + + if (!(fp->daccess & FILE_READ_ATTRIBUTES_LE)) { + pr_err("no right to read the attributes : 0x%x\n", + fp->daccess); + return -EACCES; + } + + ret = vfs_getattr(&fp->filp->f_path, &stat, STATX_BASIC_STATS, + AT_STATX_SYNC_AS_STAT); + if (ret) + return ret; + + basic_info = (struct file_basic_info *)rsp->Buffer; + basic_info->CreationTime = cpu_to_le64(fp->create_time); + time = ksmbd_UnixTimeToNT(stat.atime); + basic_info->LastAccessTime = cpu_to_le64(time); + time = ksmbd_UnixTimeToNT(stat.mtime); + basic_info->LastWriteTime = cpu_to_le64(time); + basic_info->ChangeTime = cpu_to_le64(fp->change_time); + basic_info->Attributes = fp->f_ci->m_fattr; + basic_info->Pad = 0; + rsp->OutputBufferLength = + cpu_to_le32(sizeof(struct file_basic_info)); + return 0; +} + +static int get_file_allocation_stat(struct ksmbd_file *fp, struct kstat *stat) +{ + int ret; + + /* + * Buffered writes can leave delayed allocation in a state where two + * consecutive queries report different block counts even when the + * second write only overwrites the first one. Complete writeback before + * reporting the filesystem allocation for an ordinary open. + */ + if (!fp->allocation_size_set) { + ret = file_write_and_wait(fp->filp); + if (ret) + return ret; + } + + ret = vfs_getattr(&fp->filp->f_path, stat, STATX_BASIC_STATS, + AT_STATX_SYNC_AS_STAT); + if (!ret && !fp->allocation_size_set) + fp->allocation_size = S_ISDIR(stat->mode) ? 0 : stat->blocks << 9; + + return ret; +} + +static int get_file_standard_info(struct smb2_query_info_rsp *rsp, + struct ksmbd_file *fp, void *rsp_org) +{ + struct smb2_file_standard_info *sinfo; + unsigned int delete_pending; + struct kstat stat; + int ret; + + ret = get_file_allocation_stat(fp, &stat); + if (ret) + return ret; + + sinfo = (struct smb2_file_standard_info *)rsp->Buffer; + delete_pending = ksmbd_inode_pending_delete(fp); + + if (ksmbd_stream_fd(fp) == false) { + sinfo->AllocationSize = cpu_to_le64(fp->allocation_size); + sinfo->EndOfFile = S_ISDIR(stat.mode) ? 0 : cpu_to_le64(stat.size); + } else { + loff_t seof = ksmbd_stream_eof(fp); + + sinfo->AllocationSize = cpu_to_le64((u64)seof); + sinfo->EndOfFile = cpu_to_le64((u64)seof); + } + sinfo->NumberOfLinks = cpu_to_le32(get_nlink(&stat) - delete_pending); + sinfo->DeletePending = delete_pending; + sinfo->Directory = S_ISDIR(stat.mode) ? 1 : 0; + rsp->OutputBufferLength = + cpu_to_le32(sizeof(struct smb2_file_standard_info)); + + return 0; +} + +static void get_file_alignment_info(struct smb2_query_info_rsp *rsp, + void *rsp_org) +{ + struct smb2_file_alignment_info *file_info; + + file_info = (struct smb2_file_alignment_info *)rsp->Buffer; + file_info->AlignmentRequirement = 0; + rsp->OutputBufferLength = + cpu_to_le32(sizeof(struct smb2_file_alignment_info)); +} + +static int get_file_all_info(struct ksmbd_work *work, + struct smb2_query_info_rsp *rsp, + struct ksmbd_file *fp, + void *rsp_org) +{ + struct ksmbd_conn *conn = work->conn; + struct smb2_file_all_info *file_info; + unsigned int delete_pending; + struct kstat stat; + int conv_len; + char *filename; + u64 time; + int ret, buf_free_len, filename_len; + + if (!(fp->daccess & FILE_READ_ATTRIBUTES_LE)) { + ksmbd_debug(SMB, "no right to read the attributes : 0x%x\n", + fp->daccess); + return -EACCES; + } + + filename = convert_to_nt_pathname(work->tcon->share_conf, &fp->filp->f_path); + if (IS_ERR(filename)) + return PTR_ERR(filename); + + filename_len = strlen(filename); + buf_free_len = smb2_resp_buf_len(work, + offsetof(struct smb2_query_info_rsp, Buffer) + + offsetof(struct smb2_file_all_info, FileName)); + if (buf_free_len < (filename_len + 1) * 2) { + kfree(filename); + return -EINVAL; + } + + ret = get_file_allocation_stat(fp, &stat); + if (ret) { + kfree(filename); + return ret; + } + + ksmbd_debug(SMB, "filename = %s\n", filename); + delete_pending = ksmbd_inode_pending_delete(fp); + file_info = (struct smb2_file_all_info *)rsp->Buffer; + + file_info->CreationTime = cpu_to_le64(fp->create_time); + time = ksmbd_UnixTimeToNT(stat.atime); + file_info->LastAccessTime = cpu_to_le64(time); + time = ksmbd_UnixTimeToNT(stat.mtime); + file_info->LastWriteTime = cpu_to_le64(time); + file_info->ChangeTime = cpu_to_le64(fp->change_time); + file_info->Attributes = fp->f_ci->m_fattr; + file_info->Pad1 = 0; + if (ksmbd_stream_fd(fp) == false) { + file_info->AllocationSize = cpu_to_le64(fp->allocation_size); + file_info->EndOfFile = S_ISDIR(stat.mode) ? 0 : cpu_to_le64(stat.size); + } else { + loff_t seof = ksmbd_stream_eof(fp); + + file_info->AllocationSize = cpu_to_le64((u64)seof); + file_info->EndOfFile = cpu_to_le64((u64)seof); + } + file_info->NumberOfLinks = + cpu_to_le32(get_nlink(&stat) - delete_pending); + file_info->DeletePending = delete_pending; + file_info->Directory = S_ISDIR(stat.mode) ? 1 : 0; + file_info->Pad2 = 0; + file_info->IndexNumber = cpu_to_le64(stat.ino); + file_info->EASize = 0; + file_info->AccessFlags = fp->daccess; + if (ksmbd_stream_fd(fp) == false) + file_info->CurrentByteOffset = cpu_to_le64(fp->filp->f_pos); + else + file_info->CurrentByteOffset = cpu_to_le64(fp->stream.pos); + file_info->Mode = fp->coption; + file_info->AlignmentRequirement = 0; + conv_len = smbConvertToUTF16((__le16 *)file_info->FileName, filename, + min(filename_len, PATH_MAX), + conn->local_nls, 0); + conv_len *= 2; + file_info->FileNameLength = cpu_to_le32(conv_len); + rsp->OutputBufferLength = + cpu_to_le32(sizeof(struct smb2_file_all_info) + conv_len - 1); + kfree(filename); + return 0; +} + +static void get_file_alternate_info(struct ksmbd_work *work, + struct smb2_query_info_rsp *rsp, + struct ksmbd_file *fp, + void *rsp_org) +{ + struct ksmbd_conn *conn = work->conn; + struct smb2_file_alt_name_info *file_info; + struct dentry *dentry = fp->filp->f_path.dentry; + int conv_len; + + spin_lock(&dentry->d_lock); + file_info = (struct smb2_file_alt_name_info *)rsp->Buffer; + conv_len = ksmbd_extract_shortname(conn, + dentry->d_name.name, + file_info->FileName); + spin_unlock(&dentry->d_lock); + file_info->FileNameLength = cpu_to_le32(conv_len); + rsp->OutputBufferLength = + cpu_to_le32(struct_size(file_info, FileName, conv_len)); +} + +static char *smb2_get_normalized_stream_name(struct ksmbd_file *fp) +{ + char *name, *stream_name = NULL, *xattr_list = NULL; + ssize_t xattr_list_len; + + if (!ksmbd_stream_fd(fp)) + return NULL; + + xattr_list_len = ksmbd_vfs_listxattr(fp->filp->f_path.dentry, + &xattr_list); + if (xattr_list_len <= 0) + goto out; + + for (name = xattr_list; name - xattr_list < xattr_list_len; + name += strlen(name) + 1) { + char *type; + + if (strlen(name) + 1 != fp->stream.size || + strncasecmp(name, fp->stream.name, fp->stream.size - 1)) + continue; + + name += XATTR_NAME_STREAM_LEN; + type = strrchr(name, ':'); + if (type) + stream_name = kstrndup(name, type - name, + KSMBD_DEFAULT_GFP); + break; + } +out: + kvfree(xattr_list); + return stream_name; +} + +static int get_file_normalized_name_info(struct ksmbd_work *work, + struct smb2_query_info_rsp *rsp, + struct ksmbd_file *fp) +{ + struct smb2_file_alt_name_info *file_info; + char *filename, *normalized, *stream_name; + int conv_len, filename_len; + + if (work->conn->dialect < SMB311_PROT_ID) { + rsp->hdr.Status = STATUS_NOT_SUPPORTED; + return -EOPNOTSUPP; + } + + filename = convert_to_nt_pathname(work->tcon->share_conf, + &fp->filp->f_path); + if (IS_ERR(filename)) + return PTR_ERR(filename); + if (filename[0] == '\\') + memmove(filename, filename + 1, strlen(filename)); + + stream_name = smb2_get_normalized_stream_name(fp); + normalized = kasprintf(KSMBD_DEFAULT_GFP, "%s%s%s", filename, + stream_name ? ":" : "", + stream_name ? stream_name : ""); + kfree(stream_name); + kfree(filename); + if (!normalized) + return -ENOMEM; + + filename_len = strlen(normalized); + file_info = (struct smb2_file_alt_name_info *)rsp->Buffer; + conv_len = smbConvertToUTF16((__le16 *)file_info->FileName, + normalized, filename_len, + work->conn->local_nls, 0); + kfree(normalized); + conv_len *= 2; + file_info->FileNameLength = cpu_to_le32(conv_len); + rsp->OutputBufferLength = cpu_to_le32(sizeof(*file_info) + conv_len); + return 0; +} + +static int get_file_stream_info(struct ksmbd_work *work, + struct smb2_query_info_rsp *rsp, + struct ksmbd_file *fp, + void *rsp_org) +{ + struct ksmbd_conn *conn = work->conn; + struct smb2_file_stream_info *file_info; + char *stream_name, *xattr_list = NULL, *stream_buf; + struct kstat stat; + const struct path *path = &fp->filp->f_path; + ssize_t xattr_list_len; + ssize_t slen; + loff_t ssize; + int nbytes = 0, streamlen, stream_name_len, next, idx = 0; + int buf_free_len; + int ret; + + ret = vfs_getattr(&fp->filp->f_path, &stat, STATX_BASIC_STATS, + AT_STATX_SYNC_AS_STAT); + if (ret) + return ret; + + file_info = (struct smb2_file_stream_info *)rsp->Buffer; + + buf_free_len = smb2_resp_buf_len(work, + offsetof(struct smb2_query_info_rsp, Buffer)); + if (buf_free_len < 0) + goto out; + + xattr_list_len = ksmbd_vfs_listxattr(path->dentry, &xattr_list); + if (xattr_list_len < 0) { + goto out; + } else if (!xattr_list_len) { + ksmbd_debug(SMB, "empty xattr in the file\n"); + goto out; + } + + while (idx < xattr_list_len) { + stream_name = xattr_list + idx; + streamlen = strlen(stream_name); + idx += streamlen + 1; + + ksmbd_debug(SMB, "%s, len %d\n", stream_name, streamlen); + + if (strncmp(&stream_name[XATTR_USER_PREFIX_LEN], + STREAM_PREFIX, STREAM_PREFIX_LEN)) + continue; + + stream_name_len = streamlen - (XATTR_USER_PREFIX_LEN + + STREAM_PREFIX_LEN); + streamlen = stream_name_len; + + /* plus : size */ + streamlen += 1; + stream_buf = kmalloc(streamlen + 1, KSMBD_DEFAULT_GFP); + if (!stream_buf) + break; + + streamlen = snprintf(stream_buf, streamlen + 1, + ":%s", &stream_name[XATTR_NAME_STREAM_LEN]); + + next = sizeof(struct smb2_file_stream_info) + streamlen * 2; + if (next > buf_free_len) { + kfree(stream_buf); + break; + } + + file_info = (struct smb2_file_stream_info *)&rsp->Buffer[nbytes]; + streamlen = smbConvertToUTF16((__le16 *)file_info->StreamName, + stream_buf, streamlen, + conn->local_nls, 0); + streamlen *= 2; + kfree(stream_buf); + file_info->StreamNameLength = cpu_to_le32(streamlen); + /* + * stream_name_len is the byte length of the xattr's *name*, + * not its value -- same class of bug ksmbd_stream_eof() + * (smb2pdu.c) already fixes for EndOfFile/AllocationSize on + * a stream handle; this enumeration path needs the same + * real xattr value length, not the name length reused as a + * size. + */ + slen = ksmbd_vfs_casexattr_len(file_mnt_idmap(fp->filp), + path->dentry, stream_name, + strlen(stream_name) + 1); + ssize = slen < 0 ? 0 : (loff_t)slen; + file_info->StreamSize = cpu_to_le64(ssize); + file_info->StreamAllocationSize = cpu_to_le64(ssize); + + nbytes += next; + buf_free_len -= next; + file_info->NextEntryOffset = cpu_to_le32(next); + } + +out: + if (!S_ISDIR(stat.mode) && + buf_free_len >= sizeof(struct smb2_file_stream_info) + 7 * 2) { + file_info = (struct smb2_file_stream_info *) + &rsp->Buffer[nbytes]; + streamlen = smbConvertToUTF16((__le16 *)file_info->StreamName, + "::$DATA", 7, conn->local_nls, 0); + streamlen *= 2; + file_info->StreamNameLength = cpu_to_le32(streamlen); + file_info->StreamSize = cpu_to_le64(stat.size); + file_info->StreamAllocationSize = cpu_to_le64(stat.blocks << 9); + nbytes += sizeof(struct smb2_file_stream_info) + streamlen; + } + + /* last entry offset should be 0 */ + file_info->NextEntryOffset = 0; + kvfree(xattr_list); + + rsp->OutputBufferLength = cpu_to_le32(nbytes); + + return 0; +} + +static int get_file_internal_info(struct smb2_query_info_rsp *rsp, + struct ksmbd_file *fp, void *rsp_org) +{ + struct smb2_file_internal_info *file_info; + struct kstat stat; + int ret; + + ret = vfs_getattr(&fp->filp->f_path, &stat, STATX_BASIC_STATS, + AT_STATX_SYNC_AS_STAT); + if (ret) + return ret; + + file_info = (struct smb2_file_internal_info *)rsp->Buffer; + file_info->IndexNumber = cpu_to_le64(stat.ino); + rsp->OutputBufferLength = + cpu_to_le32(sizeof(struct smb2_file_internal_info)); + + return 0; +} + +static int get_file_network_open_info(struct smb2_query_info_rsp *rsp, + struct ksmbd_file *fp, void *rsp_org) +{ + struct smb2_file_network_open_info *file_info; + struct kstat stat; + u64 time; + int ret; + + if (!(fp->daccess & FILE_READ_ATTRIBUTES_LE)) { + pr_err("no right to read the attributes : 0x%x\n", + fp->daccess); + return -EACCES; + } + + ret = get_file_allocation_stat(fp, &stat); + if (ret) + return ret; + + file_info = (struct smb2_file_network_open_info *)rsp->Buffer; + + file_info->CreationTime = cpu_to_le64(fp->create_time); + time = ksmbd_UnixTimeToNT(stat.atime); + file_info->LastAccessTime = cpu_to_le64(time); + time = ksmbd_UnixTimeToNT(stat.mtime); + file_info->LastWriteTime = cpu_to_le64(time); + file_info->ChangeTime = cpu_to_le64(fp->change_time); + file_info->Attributes = fp->f_ci->m_fattr; + if (ksmbd_stream_fd(fp) == false) { + file_info->AllocationSize = cpu_to_le64(fp->allocation_size); + file_info->EndOfFile = S_ISDIR(stat.mode) ? 0 : cpu_to_le64(stat.size); + } else { + loff_t seof = ksmbd_stream_eof(fp); + + file_info->AllocationSize = cpu_to_le64((u64)seof); + file_info->EndOfFile = cpu_to_le64((u64)seof); + } + file_info->Reserved = cpu_to_le32(0); + rsp->OutputBufferLength = + cpu_to_le32(sizeof(struct smb2_file_network_open_info)); + return 0; +} + +static void get_file_ea_info(struct smb2_query_info_rsp *rsp, void *rsp_org) +{ + struct smb2_file_ea_info *file_info; + + file_info = (struct smb2_file_ea_info *)rsp->Buffer; + file_info->EASize = 0; + rsp->OutputBufferLength = + cpu_to_le32(sizeof(struct smb2_file_ea_info)); +} + +static void get_file_position_info(struct smb2_query_info_rsp *rsp, + struct ksmbd_file *fp, void *rsp_org) +{ + struct smb2_file_pos_info *file_info; + + file_info = (struct smb2_file_pos_info *)rsp->Buffer; + if (ksmbd_stream_fd(fp) == false) + file_info->CurrentByteOffset = cpu_to_le64(fp->filp->f_pos); + else + file_info->CurrentByteOffset = cpu_to_le64(fp->stream.pos); + + rsp->OutputBufferLength = + cpu_to_le32(sizeof(struct smb2_file_pos_info)); +} + +static void get_file_mode_info(struct smb2_query_info_rsp *rsp, + struct ksmbd_file *fp, void *rsp_org) +{ + struct smb2_file_mode_info *file_info; + + file_info = (struct smb2_file_mode_info *)rsp->Buffer; + file_info->Mode = fp->coption & FILE_MODE_INFO_MASK; + rsp->OutputBufferLength = + cpu_to_le32(sizeof(struct smb2_file_mode_info)); +} + +static int get_file_compression_info(struct smb2_query_info_rsp *rsp, + struct ksmbd_file *fp, void *rsp_org) +{ + struct smb2_file_comp_info *file_info; + struct kstat stat; + u16 fmt; + int ret; + + ret = vfs_getattr(&fp->filp->f_path, &stat, STATX_BASIC_STATS, + AT_STATX_SYNC_AS_STAT); + if (ret) + return ret; + + ret = ksmbd_vfs_get_compression(fp, &fmt); + if (ret) + return ret; + + file_info = (struct smb2_file_comp_info *)rsp->Buffer; + file_info->CompressedFileSize = cpu_to_le64(min_t(u64, stat.blocks << 9, stat.size)); + file_info->CompressionFormat = cpu_to_le16(fmt); + file_info->CompressionUnitShift = 0; + file_info->ChunkShift = 0; + file_info->ClusterShift = 0; + memset(&file_info->Reserved[0], 0, 3); + + rsp->OutputBufferLength = + cpu_to_le32(sizeof(struct smb2_file_comp_info)); + + return 0; +} + +static int get_file_attribute_tag_info(struct smb2_query_info_rsp *rsp, + struct ksmbd_file *fp, void *rsp_org) +{ + struct smb2_file_attr_tag_info *file_info; + + if (!(fp->daccess & FILE_READ_ATTRIBUTES_LE)) { + pr_err("no right to read the attributes : 0x%x\n", + fp->daccess); + return -EACCES; + } + + file_info = (struct smb2_file_attr_tag_info *)rsp->Buffer; + file_info->FileAttributes = fp->f_ci->m_fattr; + file_info->ReparseTag = 0; + rsp->OutputBufferLength = + cpu_to_le32(sizeof(struct smb2_file_attr_tag_info)); + return 0; +} + +static int find_file_posix_info(struct smb2_query_info_rsp *rsp, + struct ksmbd_file *fp, void *rsp_org) +{ + struct smb311_posix_qinfo *file_info; + struct inode *inode = file_inode(fp->filp); + struct mnt_idmap *idmap = file_mnt_idmap(fp->filp); + vfsuid_t vfsuid = i_uid_into_vfsuid(idmap, inode); + vfsgid_t vfsgid = i_gid_into_vfsgid(idmap, inode); + struct kstat stat; + u64 time; + int out_buf_len = sizeof(struct smb311_posix_qinfo) + 32; + int ret; + + if (!(fp->daccess & FILE_READ_ATTRIBUTES_LE)) { + pr_err("no right to read the attributes : 0x%x\n", + fp->daccess); + return -EACCES; + } + + ret = get_file_allocation_stat(fp, &stat); + if (ret) + return ret; + + file_info = (struct smb311_posix_qinfo *)rsp->Buffer; + file_info->CreationTime = cpu_to_le64(fp->create_time); + time = ksmbd_UnixTimeToNT(stat.atime); + file_info->LastAccessTime = cpu_to_le64(time); + time = ksmbd_UnixTimeToNT(stat.mtime); + file_info->LastWriteTime = cpu_to_le64(time); + file_info->ChangeTime = cpu_to_le64(fp->change_time); + file_info->DosAttributes = fp->f_ci->m_fattr; + file_info->Inode = cpu_to_le64(stat.ino); + if (ksmbd_stream_fd(fp) == false) { + file_info->EndOfFile = cpu_to_le64(stat.size); + file_info->AllocationSize = cpu_to_le64(fp->allocation_size); + } else { + loff_t seof = ksmbd_stream_eof(fp); + + file_info->EndOfFile = cpu_to_le64((u64)seof); + file_info->AllocationSize = cpu_to_le64((u64)seof); + } + file_info->HardLinks = cpu_to_le32(stat.nlink); + file_info->Mode = cpu_to_le32(stat.mode & 0777); + switch (stat.mode & S_IFMT) { + case S_IFDIR: + file_info->Mode |= cpu_to_le32(POSIX_TYPE_DIR << POSIX_FILETYPE_SHIFT); + break; + case S_IFLNK: + file_info->Mode |= cpu_to_le32(POSIX_TYPE_SYMLINK << POSIX_FILETYPE_SHIFT); + break; + case S_IFCHR: + file_info->Mode |= cpu_to_le32(POSIX_TYPE_CHARDEV << POSIX_FILETYPE_SHIFT); + break; + case S_IFBLK: + file_info->Mode |= cpu_to_le32(POSIX_TYPE_BLKDEV << POSIX_FILETYPE_SHIFT); + break; + case S_IFIFO: + file_info->Mode |= cpu_to_le32(POSIX_TYPE_FIFO << POSIX_FILETYPE_SHIFT); + break; + case S_IFSOCK: + file_info->Mode |= cpu_to_le32(POSIX_TYPE_SOCKET << POSIX_FILETYPE_SHIFT); + } + + file_info->DeviceId = cpu_to_le32(stat.rdev); + + /* + * Sids(32) contain two sids(Domain sid(16), UNIX group sid(16)). + * UNIX sid(16) = revision(1) + num_subauth(1) + authority(6) + + * sub_auth(4 * 1(num_subauth)) + RID(4). + */ + id_to_sid(from_kuid_munged(&init_user_ns, vfsuid_into_kuid(vfsuid)), + SIDUNIX_USER, (struct smb_sid *)&file_info->Sids[0]); + id_to_sid(from_kgid_munged(&init_user_ns, vfsgid_into_kgid(vfsgid)), + SIDUNIX_GROUP, (struct smb_sid *)&file_info->Sids[16]); + + rsp->OutputBufferLength = cpu_to_le32(out_buf_len); + + return 0; +} + +static int smb2_get_info_file(struct ksmbd_work *work, + struct smb2_query_info_req *req, + struct smb2_query_info_rsp *rsp) +{ + struct ksmbd_file *fp; + int fileinfoclass = 0; + int rc = 0; + unsigned int fixed_len; + unsigned int id = KSMBD_NO_FID, pid = KSMBD_NO_FID; + + if (test_share_config_flag(work->tcon->share_conf, + KSMBD_SHARE_FLAG_PIPE)) { + /* smb2 info file called for pipe */ + rc = smb2_get_info_file_pipe(work->sess, req, rsp, + work->response_buf); + goto iov_pin_out; + } + + if (work->next_smb2_rcv_hdr_off) { + if (!has_file_id(req->VolatileFileId)) { + ksmbd_debug(SMB, "Compound request set FID = %llu\n", + work->compound_fid); + id = work->compound_fid; + pid = work->compound_pfid; + } + } + + if (!has_file_id(id)) { + id = req->VolatileFileId; + pid = req->PersistentFileId; + } + + fp = ksmbd_lookup_fd_slow(work, id, pid); + if (!fp) + return -ENOENT; + + fileinfoclass = req->FileInfoClass; + + switch (fileinfoclass) { + case FILE_ACCESS_INFORMATION: + get_file_access_info(rsp, fp, work->response_buf); + break; + + case FILE_BASIC_INFORMATION: + rc = get_file_basic_info(rsp, fp, work->response_buf); + break; + + case FILE_STANDARD_INFORMATION: + rc = get_file_standard_info(rsp, fp, work->response_buf); + break; + + case FILE_ALIGNMENT_INFORMATION: + get_file_alignment_info(rsp, work->response_buf); + break; + + case FILE_ALL_INFORMATION: + rc = get_file_all_info(work, rsp, fp, work->response_buf); + break; + + case FILE_ALTERNATE_NAME_INFORMATION: + get_file_alternate_info(work, rsp, fp, work->response_buf); + break; + case FILE_NORMALIZED_NAME_INFORMATION: + rc = get_file_normalized_name_info(work, rsp, fp); + break; + + case FILE_STREAM_INFORMATION: + rc = get_file_stream_info(work, rsp, fp, work->response_buf); + break; + + case FILE_INTERNAL_INFORMATION: + rc = get_file_internal_info(rsp, fp, work->response_buf); + break; + + case FILE_NETWORK_OPEN_INFORMATION: + rc = get_file_network_open_info(rsp, fp, work->response_buf); + break; + + case FILE_EA_INFORMATION: + get_file_ea_info(rsp, work->response_buf); + break; + + case FILE_FULL_EA_INFORMATION: + rc = smb2_get_ea(work, fp, req, rsp, work->response_buf); + break; + + case FILE_POSITION_INFORMATION: + get_file_position_info(rsp, fp, work->response_buf); + break; + + case FILE_MODE_INFORMATION: + get_file_mode_info(rsp, fp, work->response_buf); + break; + + case FILE_COMPRESSION_INFORMATION: + rc = get_file_compression_info(rsp, fp, work->response_buf); + break; + + case FILE_ATTRIBUTE_TAG_INFORMATION: + rc = get_file_attribute_tag_info(rsp, fp, work->response_buf); + break; + case SMB_FIND_FILE_POSIX_INFO: + if (!work->tcon->posix_extensions) { + pr_err("client doesn't negotiate with SMB3.1.1 POSIX Extensions\n"); + rc = -EOPNOTSUPP; + } else { + rc = find_file_posix_info(rsp, fp, work->response_buf); + } + break; + default: + ksmbd_debug(SMB, "fileinfoclass %d not supported yet\n", + fileinfoclass); + rc = -EOPNOTSUPP; + } + if (!rc) { + fixed_len = le32_to_cpu(rsp->OutputBufferLength); + switch (fileinfoclass) { + case FILE_ALL_INFORMATION: + fixed_len = FILE_ALL_INFORMATION_SIZE; + break; + case FILE_ALTERNATE_NAME_INFORMATION: + fixed_len = FILE_ALTERNATE_NAME_INFORMATION_SIZE; + break; + case FILE_STREAM_INFORMATION: + fixed_len = FILE_STREAM_INFORMATION_SIZE; + break; + } + rc = buffer_check_err(le32_to_cpu(req->OutputBufferLength), + fixed_len, + rsp, work->response_buf); + } + ksmbd_fd_put(work, fp); + +iov_pin_out: + if (!rc) + rc = ksmbd_iov_pin_rsp(work, (void *)rsp, + offsetof(struct smb2_query_info_rsp, Buffer) + + le32_to_cpu(rsp->OutputBufferLength)); + return rc; +} + +static int smb2_get_info_filesystem(struct ksmbd_work *work, + struct smb2_query_info_req *req, + struct smb2_query_info_rsp *rsp) +{ + struct ksmbd_conn *conn = work->conn; + struct ksmbd_share_config *share = work->tcon->share_conf; + int fsinfoclass = 0; + struct kstatfs stfs; + struct path path; + int rc = 0, len; + unsigned int fixed_len = 0; + + if (!share->path) + return -EIO; + + scoped_with_init_fs() + rc = kern_path(share->path, LOOKUP_NO_SYMLINKS, &path); + if (rc) { + pr_err("cannot create vfs path\n"); + return -EIO; + } + + rc = vfs_statfs(&path, &stfs); + if (rc) { + pr_err("cannot do stat of path %s\n", share->path); + path_put(&path); + return -EIO; + } + + fsinfoclass = req->FileInfoClass; + + switch (fsinfoclass) { + case FS_DEVICE_INFORMATION: + { + FILE_SYSTEM_DEVICE_INFO *info; + + info = (FILE_SYSTEM_DEVICE_INFO *)rsp->Buffer; + + info->DeviceType = cpu_to_le32(FILE_DEVICE_DISK); + info->DeviceCharacteristics = + cpu_to_le32(FILE_DEVICE_IS_MOUNTED); + if (!test_tree_conn_flag(work->tcon, + KSMBD_TREE_CONN_FLAG_WRITABLE)) + info->DeviceCharacteristics |= + cpu_to_le32(FILE_READ_ONLY_DEVICE); + rsp->OutputBufferLength = cpu_to_le32(8); + fixed_len = 8; + break; + } + case FS_ATTRIBUTE_INFORMATION: + { + FILE_SYSTEM_ATTRIBUTE_INFO *info; + struct file_kattr fa = {}; + size_t sz; + u32 attrs; + int err; + + info = (FILE_SYSTEM_ATTRIBUTE_INFO *)rsp->Buffer; + attrs = FILE_SUPPORTS_OBJECT_IDS | + FILE_PERSISTENT_ACLS | + FILE_UNICODE_ON_DISK | + FILE_FILE_COMPRESSION | + FILE_SUPPORTS_SPARSE_FILES | + FILE_SUPPORTS_BLOCK_REFCOUNTING; + + err = vfs_fileattr_get(path.dentry, &fa); + /* + * -EINVAL, -EOPNOTSUPP: ntfs-3g and other FUSE + * filesystems that lack FS_IOC_FSGETXATTR support. + */ + if (err && err != -ENOIOCTLCMD && err != -ENOTTY && + err != -EINVAL && err != -EOPNOTSUPP) { + path_put(&path); + return err; + } + if (!(fa.fsx_xflags & FS_XFLAG_CASEFOLD)) + attrs |= FILE_CASE_SENSITIVE_SEARCH; + if (!(fa.fsx_xflags & FS_XFLAG_CASENONPRESERVING)) + attrs |= FILE_CASE_PRESERVED_NAMES; + + info->Attributes = cpu_to_le32(attrs); + info->Attributes |= cpu_to_le32(server_conf.share_fake_fscaps); + + if (test_share_config_flag(work->tcon->share_conf, + KSMBD_SHARE_FLAG_STREAMS)) + info->Attributes |= cpu_to_le32(FILE_NAMED_STREAMS); + + info->MaxPathNameComponentLength = cpu_to_le32(stfs.f_namelen); + /* + * some application(potableapp) can not run on ksmbd share + * because only NTFS handle security setting on windows. + * So Although local fs(EXT4 or F2fs, etc) is not NTFS, + * ksmbd should show share as NTFS. Later, If needed, we can add + * fs type(s) parameter to change fs type user wanted. + */ + len = smbConvertToUTF16((__le16 *)info->FileSystemName, + "NTFS", PATH_MAX, conn->local_nls, 0); + len = len * 2; + info->FileSystemNameLen = cpu_to_le32(len); + sz = sizeof(FILE_SYSTEM_ATTRIBUTE_INFO) + len; + rsp->OutputBufferLength = cpu_to_le32(sz); + fixed_len = 16; + break; + } + case FS_VOLUME_INFORMATION: + { + struct filesystem_vol_info *info; + size_t sz; + unsigned int serial_crc = 0; + + info = (struct filesystem_vol_info *)(rsp->Buffer); + info->VolumeCreationTime = 0; + serial_crc = crc32_le(serial_crc, share->name, + strlen(share->name)); + serial_crc = crc32_le(serial_crc, share->path, + strlen(share->path)); + serial_crc = crc32_le(serial_crc, ksmbd_netbios_name(), + strlen(ksmbd_netbios_name())); + /* Taking dummy value of serial number*/ + info->VolumeSerialNumber = cpu_to_le32(serial_crc); + len = smbConvertToUTF16((__le16 *)info->VolumeLabel, + share->name, PATH_MAX, + conn->local_nls, 0); + len = len * 2; + info->VolumeLabelLength = cpu_to_le32(len); + info->Reserved = 0; + info->SupportsObjects = 0; + sz = sizeof(struct filesystem_vol_info) + len; + rsp->OutputBufferLength = cpu_to_le32(sz); + fixed_len = 24; + break; + } + case FS_SIZE_INFORMATION: + { + FILE_SYSTEM_SIZE_INFO *info; + + info = (FILE_SYSTEM_SIZE_INFO *)(rsp->Buffer); + info->TotalAllocationUnits = cpu_to_le64(stfs.f_blocks); + info->AvailableAllocationUnits = cpu_to_le64(stfs.f_bfree); + info->SectorsPerAllocationUnit = cpu_to_le32(1); + info->BytesPerSector = cpu_to_le32(stfs.f_bsize); + rsp->OutputBufferLength = cpu_to_le32(24); + fixed_len = 24; + break; + } + case FS_FULL_SIZE_INFORMATION: + { + struct smb2_fs_full_size_info *info; + + info = (struct smb2_fs_full_size_info *)(rsp->Buffer); + info->TotalAllocationUnits = cpu_to_le64(stfs.f_blocks); + info->CallerAvailableAllocationUnits = + cpu_to_le64(stfs.f_bavail); + info->ActualAvailableAllocationUnits = + cpu_to_le64(stfs.f_bfree); + info->SectorsPerAllocationUnit = cpu_to_le32(1); + info->BytesPerSector = cpu_to_le32(stfs.f_bsize); + rsp->OutputBufferLength = cpu_to_le32(32); + fixed_len = 32; + break; + } + case FS_OBJECT_ID_INFORMATION: + { + struct object_id_info *info; + + info = (struct object_id_info *)(rsp->Buffer); + + if (path.mnt->mnt_sb->s_uuid_len == 16) + memcpy(info->objid, path.mnt->mnt_sb->s_uuid.b, + path.mnt->mnt_sb->s_uuid_len); + else + memcpy(info->objid, &stfs.f_fsid, sizeof(stfs.f_fsid)); + + info->extended_info.magic = cpu_to_le32(EXTENDED_INFO_MAGIC); + info->extended_info.version = cpu_to_le32(1); + info->extended_info.release = cpu_to_le32(1); + info->extended_info.rel_date = 0; + memcpy(info->extended_info.version_string, "1.1.0", strlen("1.1.0")); + rsp->OutputBufferLength = cpu_to_le32(64); + fixed_len = 64; + break; + } + case FS_SECTOR_SIZE_INFORMATION: + { + struct smb3_fs_ss_info *info; + unsigned int sector_size = + min_t(unsigned int, path.mnt->mnt_sb->s_blocksize, 4096); + + info = (struct smb3_fs_ss_info *)(rsp->Buffer); + + info->LogicalBytesPerSector = cpu_to_le32(sector_size); + info->PhysicalBytesPerSectorForAtomicity = + cpu_to_le32(sector_size); + info->PhysicalBytesPerSectorForPerf = cpu_to_le32(sector_size); + info->FSEffPhysicalBytesPerSectorForAtomicity = + cpu_to_le32(sector_size); + info->Flags = cpu_to_le32(SSINFO_FLAGS_ALIGNED_DEVICE | + SSINFO_FLAGS_PARTITION_ALIGNED_ON_DEVICE | + SSINFO_FLAGS_TRIM_ENABLED); + info->ByteOffsetForSectorAlignment = 0; + info->ByteOffsetForPartitionAlignment = 0; + rsp->OutputBufferLength = cpu_to_le32(28); + fixed_len = 28; + break; + } + case FS_CONTROL_INFORMATION: + { + /* + * TODO : The current implementation is based on + * test result with win7(NTFS) server. It's need to + * modify this to get valid Quota values + * from Linux kernel + */ + struct smb2_fs_control_info *info; + + info = (struct smb2_fs_control_info *)(rsp->Buffer); + info->FreeSpaceStartFiltering = 0; + info->FreeSpaceThreshold = 0; + info->FreeSpaceStopFiltering = 0; + info->DefaultQuotaThreshold = cpu_to_le64(SMB2_NO_FID); + info->DefaultQuotaLimit = cpu_to_le64(SMB2_NO_FID); + info->Padding = 0; + rsp->OutputBufferLength = cpu_to_le32(48); + fixed_len = 48; + break; + } + case FS_POSIX_INFORMATION: + { + FILE_SYSTEM_POSIX_INFO *info; + + if (!work->tcon->posix_extensions) { + pr_err("client doesn't negotiate with SMB3.1.1 POSIX Extensions\n"); + path_put(&path); + return -EOPNOTSUPP; + } else { + info = (FILE_SYSTEM_POSIX_INFO *)(rsp->Buffer); + info->OptimalTransferSize = cpu_to_le32(stfs.f_bsize); + info->BlockSize = cpu_to_le32(stfs.f_bsize); + info->TotalBlocks = cpu_to_le64(stfs.f_blocks); + info->BlocksAvail = cpu_to_le64(stfs.f_bfree); + info->UserBlocksAvail = cpu_to_le64(stfs.f_bavail); + info->TotalFileNodes = cpu_to_le64(stfs.f_files); + info->FreeFileNodes = cpu_to_le64(stfs.f_ffree); + rsp->OutputBufferLength = cpu_to_le32(56); + fixed_len = 56; + } + break; + } + default: + path_put(&path); + return -EOPNOTSUPP; + } + rc = buffer_check_err(le32_to_cpu(req->OutputBufferLength), + fixed_len, + rsp, work->response_buf); + path_put(&path); + + if (!rc) + rc = ksmbd_iov_pin_rsp(work, (void *)rsp, + offsetof(struct smb2_query_info_rsp, Buffer) + + le32_to_cpu(rsp->OutputBufferLength)); + return rc; +} + +static int smb2_get_info_sec(struct ksmbd_work *work, + struct smb2_query_info_req *req, + struct smb2_query_info_rsp *rsp) +{ + struct ksmbd_file *fp; + struct mnt_idmap *idmap; + struct smb_ntsd *pntsd = NULL, *ppntsd = NULL; + struct smb_fattr fattr = {{0}}; + struct inode *inode; + __u32 secdesclen = 0; + unsigned int id = KSMBD_NO_FID, pid = KSMBD_NO_FID; + int addition_info = le32_to_cpu(req->AdditionalInformation); + int rc = 0, ppntsd_size = 0, max_len; + size_t scratch_len = 0; + + if (addition_info & ~(OWNER_SECINFO | GROUP_SECINFO | DACL_SECINFO | + PROTECTED_DACL_SECINFO | + UNPROTECTED_DACL_SECINFO)) { + ksmbd_debug(SMB, "Unsupported addition info: 0x%x)\n", + addition_info); + + rsp->hdr.Status = STATUS_NOT_SUPPORTED; + return -EINVAL; + } + + if (work->next_smb2_rcv_hdr_off) { + if (!has_file_id(req->VolatileFileId)) { + ksmbd_debug(SMB, "Compound request set FID = %llu\n", + work->compound_fid); + id = work->compound_fid; + pid = work->compound_pfid; + } + } + + if (!has_file_id(id)) { + id = req->VolatileFileId; + pid = req->PersistentFileId; + } + + fp = ksmbd_lookup_fd_slow(work, id, pid); + if (!fp) + return -ENOENT; + + if (addition_info & (OWNER_SECINFO | GROUP_SECINFO | DACL_SECINFO) && + !(fp->daccess & FILE_READ_CONTROL_LE)) { + ksmbd_fd_put(work, fp); + return -EACCES; + } + + if (le32_to_cpu(req->OutputBufferLength) < sizeof(struct smb_ntsd)) { + rsp->hdr.Status = STATUS_BUFFER_TOO_SMALL; + ksmbd_fd_put(work, fp); + return -ENOSPC; + } + + idmap = file_mnt_idmap(fp->filp); + inode = file_inode(fp->filp); + ksmbd_acls_fattr(&fattr, idmap, inode); + + if (test_share_config_flag(work->tcon->share_conf, + KSMBD_SHARE_FLAG_ACL_XATTR)) + ppntsd_size = ksmbd_vfs_get_sd_xattr(work->conn, idmap, + fp->filp->f_path.dentry, + &ppntsd); + + /* Check if sd buffer size exceeds response buffer size */ + max_len = smb2_calc_max_out_buf_len(work, + offsetof(struct smb2_query_info_rsp, Buffer), + le32_to_cpu(req->OutputBufferLength)); + if (max_len < 0) { + rc = -EINVAL; + goto release_acl; + } + + scratch_len = smb_acl_sec_desc_scratch_len(&fattr, ppntsd, + ppntsd_size, addition_info); + if (!scratch_len || scratch_len == SIZE_MAX) { + rc = -EFBIG; + goto release_acl; + } + + pntsd = kvzalloc(scratch_len, KSMBD_DEFAULT_GFP); + if (!pntsd) { + rc = -ENOMEM; + goto release_acl; + } + + rc = build_sec_desc(idmap, pntsd, ppntsd, ppntsd_size, + addition_info, &secdesclen, &fattr); + +release_acl: + posix_acl_release(fattr.cf_acls); + posix_acl_release(fattr.cf_dacls); + kfree(ppntsd); + ksmbd_fd_put(work, fp); + + if (!rc && ALIGN(secdesclen, 8) > scratch_len) + rc = -EFBIG; + if (rc) + goto err_out; + + rsp->OutputBufferLength = cpu_to_le32(secdesclen); + rc = buffer_check_err(le32_to_cpu(req->OutputBufferLength), + le32_to_cpu(rsp->OutputBufferLength), + rsp, work->response_buf); + if (rc) + goto err_out; + + rc = ksmbd_iov_pin_rsp_read(work, (void *)rsp, + offsetof(struct smb2_query_info_rsp, Buffer), + pntsd, secdesclen); +err_out: + if (rc) { + rsp->OutputBufferLength = 0; + kvfree(pntsd); + } + + return rc; +} + +/** + * smb2_query_info() - handler for smb2 query info command + * @work: smb work containing query info request buffer + * + * Return: 0 on success, otherwise error + */ +int smb2_query_info(struct ksmbd_work *work) +{ + struct smb2_query_info_req *req; + struct smb2_query_info_rsp *rsp; + int rc = 0; + + ksmbd_debug(SMB, "Received request smb2 query info request\n"); + + WORK_BUFFERS(work, req, rsp); + + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; + + if (ksmbd_override_fsids(work)) { + rc = -ENOMEM; + goto err_out; + } + + rsp->StructureSize = cpu_to_le16(9); + rsp->OutputBufferOffset = cpu_to_le16(72); + + switch (req->InfoType) { + case SMB2_O_INFO_FILE: + ksmbd_debug(SMB, "GOT SMB2_O_INFO_FILE\n"); + rc = smb2_get_info_file(work, req, rsp); + break; + case SMB2_O_INFO_FILESYSTEM: + ksmbd_debug(SMB, "GOT SMB2_O_INFO_FILESYSTEM\n"); + rc = smb2_get_info_filesystem(work, req, rsp); + break; + case SMB2_O_INFO_SECURITY: + ksmbd_debug(SMB, "GOT SMB2_O_INFO_SECURITY\n"); + rc = smb2_get_info_sec(work, req, rsp); + break; + default: + ksmbd_debug(SMB, "InfoType %d not supported yet\n", + req->InfoType); + rc = -EOPNOTSUPP; + } + ksmbd_revert_fsids(work); + +err_out: + if (rc < 0) { + if (rc == -EACCES) + rsp->hdr.Status = STATUS_ACCESS_DENIED; + else if (rc == -ENOENT) + rsp->hdr.Status = STATUS_FILE_CLOSED; + else if (rc == -EIO) + rsp->hdr.Status = STATUS_UNEXPECTED_IO_ERROR; + else if (rc == -ENOMEM) + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + else if (rc == -EINVAL && rsp->hdr.Status == 0) + rsp->hdr.Status = STATUS_INVALID_PARAMETER; + else if (rsp->hdr.Status == 0) + rsp->hdr.Status = STATUS_INVALID_INFO_CLASS; + smb2_set_err_rsp(work); + + ksmbd_debug(SMB, "error while processing smb2 query rc = %d\n", + rc); + return rc; + } + return 0; +} + +/** + * smb2_close_pipe() - handler for closing IPC pipe + * @work: smb work containing close request buffer + * + * Return: 0 + */ +static noinline int smb2_close_pipe(struct ksmbd_work *work) +{ + u64 id; + struct smb2_close_req *req; + struct smb2_close_rsp *rsp; + + WORK_BUFFERS(work, req, rsp); + + id = req->VolatileFileId; + ksmbd_session_rpc_close(work->sess, id); + + rsp->StructureSize = cpu_to_le16(60); + rsp->Flags = 0; + rsp->Reserved = 0; + rsp->CreationTime = 0; + rsp->LastAccessTime = 0; + rsp->LastWriteTime = 0; + rsp->ChangeTime = 0; + rsp->AllocationSize = 0; + rsp->EndOfFile = 0; + rsp->Attributes = 0; + + return ksmbd_iov_pin_rsp(work, (void *)rsp, + sizeof(struct smb2_close_rsp)); +} + +/** + * smb2_close() - handler for smb2 close file command + * @work: smb work containing close request buffer + * + * Return: 0 on success, otherwise error + */ +int smb2_close(struct ksmbd_work *work) +{ + u64 volatile_id = KSMBD_NO_FID; + u64 sess_id; + struct smb2_close_req *req; + struct smb2_close_rsp *rsp; + struct ksmbd_file *fp; + u64 time; + int err = 0; + + ksmbd_debug(SMB, "Received smb2 close request\n"); + + WORK_BUFFERS(work, req, rsp); + + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; + + if (test_share_config_flag(work->tcon->share_conf, + KSMBD_SHARE_FLAG_PIPE)) { + ksmbd_debug(SMB, "IPC pipe close request\n"); + return smb2_close_pipe(work); + } + + sess_id = le64_to_cpu(req->hdr.SessionId); + if (req->hdr.Flags & SMB2_FLAGS_RELATED_OPERATIONS) + sess_id = work->compound_sid; + + work->compound_sid = 0; + if (work->sess && work->sess->id == sess_id) { + work->compound_sid = sess_id; + } else { + rsp->hdr.Status = STATUS_USER_SESSION_DELETED; + if (req->hdr.Flags & SMB2_FLAGS_RELATED_OPERATIONS) + rsp->hdr.Status = STATUS_INVALID_PARAMETER; + err = -EBADF; + goto out; + } + + if (work->next_smb2_rcv_hdr_off && + !has_file_id(req->VolatileFileId)) { + if (!has_file_id(work->compound_fid)) { + /* file already closed, return FILE_CLOSED */ + ksmbd_debug(SMB, "file already closed\n"); + rsp->hdr.Status = STATUS_FILE_CLOSED; + err = -EBADF; + goto out; + } else { + ksmbd_debug(SMB, + "Compound request set FID = %llu:%llu\n", + work->compound_fid, + work->compound_pfid); + volatile_id = work->compound_fid; + + /* file closed, stored id is not valid anymore */ + work->compound_fid = KSMBD_NO_FID; + work->compound_pfid = KSMBD_NO_FID; + } + } else { + volatile_id = req->VolatileFileId; + } + ksmbd_debug(SMB, "volatile_id = %llu\n", volatile_id); + + rsp->StructureSize = cpu_to_le16(60); + rsp->Reserved = 0; + + if (req->Flags == SMB2_CLOSE_FLAG_POSTQUERY_ATTRIB) { + struct kstat stat; + int ret; + + fp = ksmbd_lookup_fd_fast(work, volatile_id); + if (!fp) { + err = -ENOENT; + goto out; + } + + ret = vfs_getattr(&fp->filp->f_path, &stat, STATX_BASIC_STATS, + AT_STATX_SYNC_AS_STAT); + if (ret) { + ksmbd_fd_put(work, fp); + goto out; + } + + rsp->Flags = SMB2_CLOSE_FLAG_POSTQUERY_ATTRIB; + rsp->AllocationSize = cpu_to_le64(fp->allocation_size); + rsp->EndOfFile = cpu_to_le64(stat.size); + rsp->Attributes = fp->f_ci->m_fattr; + rsp->CreationTime = cpu_to_le64(fp->create_time); + time = ksmbd_UnixTimeToNT(stat.atime); + rsp->LastAccessTime = cpu_to_le64(time); + time = ksmbd_UnixTimeToNT(stat.mtime); + if (time > fp->open_mtime && + time - fp->open_mtime < KSMBD_WRITE_TIME_RESOLUTION) + time = fp->open_mtime; + rsp->LastWriteTime = cpu_to_le64(time); + rsp->ChangeTime = cpu_to_le64(fp->change_time); + ksmbd_fd_put(work, fp); + } else { + rsp->Flags = 0; + rsp->AllocationSize = 0; + rsp->EndOfFile = 0; + rsp->Attributes = 0; + rsp->CreationTime = 0; + rsp->LastAccessTime = 0; + rsp->LastWriteTime = 0; + rsp->ChangeTime = 0; + } + + err = ksmbd_close_fd(work, volatile_id); +out: + if (!err) + err = ksmbd_iov_pin_rsp(work, (void *)rsp, + sizeof(struct smb2_close_rsp)); + + if (err) { + if (rsp->hdr.Status == 0) + rsp->hdr.Status = STATUS_FILE_CLOSED; + smb2_set_err_rsp(work); + } + + return err; +} + +/** + * smb2_echo() - handler for smb2 echo(ping) command + * @work: smb work containing echo request buffer + * + * Return: 0 on success, otherwise error + */ +int smb2_echo(struct ksmbd_work *work) +{ + struct smb2_echo_rsp *rsp = smb_get_msg(work->response_buf); + + ksmbd_debug(SMB, "Received smb2 echo request\n"); + + if (work->next_smb2_rcv_hdr_off) + rsp = ksmbd_resp_buf_next(work); + + rsp->StructureSize = cpu_to_le16(4); + rsp->Reserved = 0; + return ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_echo_rsp)); +} + +static int smb2_rename(struct ksmbd_work *work, + struct ksmbd_file *fp, + struct smb2_file_rename_info *file_info, + struct nls_table *local_nls) +{ + struct ksmbd_share_config *share = fp->tcon->share_conf; + char *new_name = NULL; + int rc, flags = 0; + + ksmbd_debug(SMB, "setting FILE_RENAME_INFO\n"); + new_name = smb2_get_name(file_info->FileName, + le32_to_cpu(file_info->FileNameLength), + local_nls); + if (IS_ERR(new_name)) + return PTR_ERR(new_name); + + if (fp->is_posix_ctxt == false && strchr(new_name, ':')) { + int s_type; + char *xattr_stream_name, *stream_name = NULL; + size_t xattr_stream_size; + int len; + + rc = parse_stream_name(new_name, &stream_name, &s_type); + if (rc < 0) + goto out; + + len = strlen(new_name); + if (len > 0 && new_name[len - 1] != '/') { + pr_err("not allow base filename in rename\n"); + rc = -ESHARE; + goto out; + } + + rc = ksmbd_vfs_xattr_stream_name(stream_name, + &xattr_stream_name, + &xattr_stream_size, + s_type); + if (rc) + goto out; + + rc = ksmbd_vfs_setxattr(file_mnt_idmap(fp->filp), + &fp->filp->f_path, + xattr_stream_name, + NULL, 0, 0, true); + if (rc < 0) { + pr_err("failed to store stream name in xattr: %d\n", + rc); + rc = -EINVAL; + } + kfree(xattr_stream_name); + goto out; + } + + ksmbd_debug(SMB, "new name %s\n", new_name); + if (ksmbd_share_veto_filename(share, new_name)) { + rc = -ENOENT; + ksmbd_debug(SMB, "Can't rename vetoed file: %s\n", new_name); + goto out; + } + + if (!file_info->ReplaceIfExists) + flags = RENAME_NOREPLACE; + + rc = ksmbd_vfs_check_rename_share(work, &fp->filp->f_path); + if (rc) + goto out; + + smb_break_all_levII_oplock_rename(work, fp); + rc = ksmbd_vfs_rename(work, fp, new_name, flags); +out: + kfree(new_name); + return rc; +} + +static int smb2_create_link(struct ksmbd_work *work, + struct ksmbd_share_config *share, + struct smb2_file_link_info *file_info, + unsigned int buf_len, struct file *filp, + struct nls_table *local_nls) +{ + char *link_name = NULL, *target_name = NULL, *pathname = NULL; + struct path path; + int rc; + + if (buf_len < (u64)sizeof(struct smb2_file_link_info) + + le32_to_cpu(file_info->FileNameLength)) + return -EINVAL; + + ksmbd_debug(SMB, "setting FILE_LINK_INFORMATION\n"); + pathname = kmalloc(PATH_MAX, KSMBD_DEFAULT_GFP); + if (!pathname) + return -ENOMEM; + + link_name = smb2_get_name(file_info->FileName, + le32_to_cpu(file_info->FileNameLength), + local_nls); + if (IS_ERR(link_name) || S_ISDIR(file_inode(filp)->i_mode)) { + rc = -EINVAL; + goto out; + } + + ksmbd_debug(SMB, "link name is %s\n", link_name); + target_name = file_path(filp, pathname, PATH_MAX); + if (IS_ERR(target_name)) { + rc = -EINVAL; + goto out; + } + + ksmbd_debug(SMB, "target name is %s\n", target_name); + rc = ksmbd_vfs_kern_path_start_removing(work, link_name, LOOKUP_NO_SYMLINKS, + &path, 0); + if (rc) { + if (rc != -ENOENT) + goto out; + } else { + if (file_info->ReplaceIfExists) { + rc = ksmbd_vfs_remove_file(work, &path); + if (rc) { + rc = -EINVAL; + ksmbd_debug(SMB, "cannot delete %s\n", + link_name); + } + } else { + rc = -EEXIST; + ksmbd_debug(SMB, "link already exists\n"); + } + ksmbd_vfs_kern_path_end_removing(&path); + if (rc) + goto out; + } + rc = ksmbd_vfs_link(work, target_name, link_name); + if (rc) + rc = -EINVAL; +out: + + if (!IS_ERR(link_name)) + kfree(link_name); + kfree(pathname); + return rc; +} + +static int set_file_basic_info(struct ksmbd_file *fp, + struct file_basic_info *file_info, + struct ksmbd_share_config *share) +{ + struct iattr attrs; + struct file *filp; + struct inode *inode; + struct mnt_idmap *idmap; + __le32 attrs_mask = FILE_ATTRIBUTE_DIRECTORY_LE | + FILE_ATTRIBUTE_COMPRESSED_LE; + int rc = 0; + + if (!(fp->daccess & FILE_WRITE_ATTRIBUTES_LE)) + return -EACCES; + + attrs.ia_valid = 0; + filp = fp->filp; + inode = file_inode(filp); + idmap = file_mnt_idmap(filp); + + if (file_info->CreationTime) + fp->create_time = le64_to_cpu(file_info->CreationTime); + + if (file_info->LastAccessTime) { + attrs.ia_atime = ksmbd_NTtimeToUnix(file_info->LastAccessTime); + attrs.ia_valid |= (ATTR_ATIME | ATTR_ATIME_SET); + } + + if (file_info->ChangeTime) { + fp->change_time = le64_to_cpu(file_info->ChangeTime); + inode_set_ctime_to_ts(inode, + ksmbd_NTtimeToUnix(file_info->ChangeTime)); + } + + if (file_info->LastWriteTime) { + attrs.ia_mtime = ksmbd_NTtimeToUnix(file_info->LastWriteTime); + attrs.ia_valid |= (ATTR_MTIME | ATTR_MTIME_SET | ATTR_CTIME); + } + + if (file_info->Attributes) { + if (!S_ISDIR(inode->i_mode) && + file_info->Attributes & FILE_ATTRIBUTE_DIRECTORY_LE) { + pr_err("can't change a file to a directory\n"); + return -EINVAL; + } + + if (!(S_ISDIR(inode->i_mode) && file_info->Attributes == FILE_ATTRIBUTE_NORMAL_LE)) + fp->f_ci->m_fattr = + (file_info->Attributes & ~FILE_ATTRIBUTE_COMPRESSED_LE) | + (fp->f_ci->m_fattr & attrs_mask); + } + + if (test_share_config_flag(share, KSMBD_SHARE_FLAG_STORE_DOS_ATTRS) && + (file_info->CreationTime || file_info->Attributes)) { + struct xattr_dos_attrib da = {0}; + + da.version = 4; + da.itime = fp->itime; + da.create_time = fp->create_time; + da.attr = le32_to_cpu(fp->f_ci->m_fattr); + da.flags = XATTR_DOSINFO_ATTRIB | XATTR_DOSINFO_CREATE_TIME | + XATTR_DOSINFO_ITIME; + + rc = ksmbd_vfs_set_dos_attrib_xattr(idmap, &filp->f_path, &da, + true); + if (rc) + ksmbd_debug(SMB, + "failed to restore file attribute in EA\n"); + rc = 0; + } + + if (attrs.ia_valid) { + struct dentry *dentry = filp->f_path.dentry; + struct inode *inode = d_inode(dentry); + + if (IS_IMMUTABLE(inode) || IS_APPEND(inode)) + return -EACCES; + + inode_lock(inode); + rc = notify_change(idmap, dentry, &attrs, NULL); + inode_unlock(inode); + } + return rc; +} + +static int set_file_allocation_info(struct ksmbd_work *work, + struct ksmbd_file *fp, + struct smb2_file_alloc_info *file_alloc_info) +{ + /* + * TODO : It's working fine only when store dos attributes + * is not yes. need to implement a logic which works + * properly with any smb.conf option + */ + + loff_t alloc_blks; + u64 alloc_size; + struct inode *inode; + struct kstat stat; + int rc; + + if (!(fp->daccess & FILE_WRITE_DATA_LE)) + return -EACCES; + + if (ksmbd_stream_fd(fp) == true) + return 0; + + rc = vfs_getattr(&fp->filp->f_path, &stat, STATX_BASIC_STATS, + AT_STATX_SYNC_AS_STAT); + if (rc) + return rc; + + /* + * AllocationSize is fully client-controlled (the caller only + * validates the fixed 8-byte buffer length). Reject values that + * would overflow the "round up to 512-byte blocks" conversion + * below instead of silently wrapping it to a tiny block count, + * which would truncate the file to a size the client never + * asked for. + */ + alloc_size = le64_to_cpu(file_alloc_info->AllocationSize); + if (alloc_size > MAX_LFS_FILESIZE - 511) + return -EINVAL; + + alloc_blks = (alloc_size + 511) >> 9; + inode = file_inode(fp->filp); + + if (alloc_blks > stat.blocks) { + smb_break_all_levII_oplock(work, fp, 1); + rc = vfs_fallocate(fp->filp, FALLOC_FL_KEEP_SIZE, 0, + alloc_blks * 512); + if (rc && rc != -EOPNOTSUPP) { + pr_err("vfs_fallocate is failed : %d\n", rc); + return rc; + } + } else if (alloc_blks < stat.blocks) { + loff_t size; + + /* + * Allocation size could be smaller than original one + * which means allocated blocks in file should be + * deallocated. use truncate to cut out it, but inode + * size is also updated with truncate offset. + * inode size is retained by backup inode size. + */ + size = i_size_read(inode); + rc = ksmbd_vfs_truncate(work, fp, alloc_blks * 512); + if (rc) { + pr_err("truncate failed!, err %d\n", rc); + return rc; + } + if (size < alloc_blks * 512) + i_size_write(inode, size); + } + + fp->allocation_size = le64_to_cpu(file_alloc_info->AllocationSize); + fp->allocation_size_set = true; + return 0; +} + +static int set_end_of_file_info(struct ksmbd_work *work, struct ksmbd_file *fp, + struct smb2_file_eof_info *file_eof_info) +{ + loff_t newsize; + struct inode *inode; + int rc; + + if (!(fp->daccess & FILE_WRITE_DATA_LE)) + return -EACCES; + + newsize = le64_to_cpu(file_eof_info->EndOfFile); + inode = file_inode(fp->filp); + + /* + * If FILE_END_OF_FILE_INFORMATION of set_info_file is called + * on FAT32 shared device, truncate execution time is too long + * and network error could cause from windows client. because + * truncate of some filesystem like FAT32 fill zero data in + * truncated range. + */ + if (inode->i_sb->s_magic != MSDOS_SUPER_MAGIC && + ksmbd_stream_fd(fp) == false) { + ksmbd_debug(SMB, "truncated to newsize %lld\n", newsize); + rc = ksmbd_vfs_truncate(work, fp, newsize); + if (rc) { + ksmbd_debug(SMB, "truncate failed!, err %d\n", rc); + if (rc != -EAGAIN) + rc = -EBADF; + return rc; + } + } + return 0; +} + +static int set_rename_info(struct ksmbd_work *work, struct ksmbd_file *fp, + struct smb2_file_rename_info *rename_info, + unsigned int buf_len) +{ + if (!(fp->daccess & FILE_DELETE_LE)) { + pr_err("no right to delete : 0x%x\n", fp->daccess); + return -EACCES; + } + + if (buf_len < (u64)sizeof(struct smb2_file_rename_info) + + le32_to_cpu(rename_info->FileNameLength)) + return -EINVAL; + + if (!le32_to_cpu(rename_info->FileNameLength)) + return -EINVAL; + + return smb2_rename(work, fp, rename_info, work->conn->local_nls); +} + +static int set_file_disposition_info(struct ksmbd_work *work, + struct ksmbd_file *fp, + struct smb2_file_disposition_info *file_info) +{ + struct inode *inode; + + if (!(fp->daccess & FILE_DELETE_LE)) { + pr_err("no right to delete : 0x%x\n", fp->daccess); + return -EACCES; + } + + if (fp->f_ci->m_fattr & FILE_ATTRIBUTE_READONLY_LE) + return -EACCES; + + inode = file_inode(fp->filp); + if (file_info->DeletePending) { + if (ksmbd_has_stream_without_delete_share(fp)) + return -ESHARE; + + if (S_ISDIR(inode->i_mode) && !ksmbd_stream_fd(fp) && + ksmbd_vfs_empty_dir(fp) == -ENOTEMPTY) + return -EBUSY; + smb_break_all_levII_oplock_for_delete(work, fp); + ksmbd_fd_set_delete_pending(fp); + } else { + ksmbd_fd_clear_delete_pending(fp); + } + return 0; +} + +static int set_file_position_info(struct ksmbd_file *fp, + struct smb2_file_pos_info *file_info) +{ + loff_t current_byte_offset; + unsigned long sector_size; + struct inode *inode; + + inode = file_inode(fp->filp); + current_byte_offset = le64_to_cpu(file_info->CurrentByteOffset); + sector_size = inode->i_sb->s_blocksize; + + if (current_byte_offset < 0 || + (fp->coption & FILE_NO_INTERMEDIATE_BUFFERING_LE && + current_byte_offset & (sector_size - 1))) { + pr_err("CurrentByteOffset is not valid : %llu\n", + current_byte_offset); + return -EINVAL; + } + + if (ksmbd_stream_fd(fp) == false) + fp->filp->f_pos = current_byte_offset; + else { + if (current_byte_offset > XATTR_SIZE_MAX) + current_byte_offset = XATTR_SIZE_MAX; + fp->stream.pos = current_byte_offset; + } + return 0; +} + +static int set_file_mode_info(struct ksmbd_file *fp, + struct smb2_file_mode_info *file_info) +{ + __le32 mode; + + mode = file_info->Mode; + + if ((mode & ~FILE_MODE_INFO_MASK)) { + pr_err("Mode is not valid : 0x%x\n", le32_to_cpu(mode)); + return -EINVAL; + } + + /* + * TODO : need to implement consideration for + * FILE_SYNCHRONOUS_IO_ALERT and FILE_SYNCHRONOUS_IO_NONALERT + */ + ksmbd_vfs_set_fadvise(fp->filp, mode); + fp->coption = mode; + return 0; +} + +/** + * smb2_set_info_file() - handler for smb2 set info command + * @work: smb work containing set info command buffer + * @fp: ksmbd_file pointer + * @req: request buffer pointer + * @share: ksmbd_share_config pointer + * + * Return: 0 on success, otherwise error + */ +static int smb2_set_info_file(struct ksmbd_work *work, struct ksmbd_file *fp, + struct smb2_set_info_req *req, + struct ksmbd_share_config *share) +{ + unsigned int buf_len = le32_to_cpu(req->BufferLength); + char *buffer = (char *)req + le16_to_cpu(req->BufferOffset); + + switch (req->FileInfoClass) { + case FILE_BASIC_INFORMATION: + { + if (buf_len < sizeof(struct file_basic_info)) + return -EMSGSIZE; + + return set_file_basic_info(fp, (struct file_basic_info *)buffer, share); + } + case FILE_ALLOCATION_INFORMATION: + { + if (buf_len < sizeof(struct smb2_file_alloc_info)) + return -EMSGSIZE; + + return set_file_allocation_info(work, fp, + (struct smb2_file_alloc_info *)buffer); + } + case FILE_END_OF_FILE_INFORMATION: + { + if (buf_len < sizeof(struct smb2_file_eof_info)) + return -EMSGSIZE; + + return set_end_of_file_info(work, fp, + (struct smb2_file_eof_info *)buffer); + } + case FILE_RENAME_INFORMATION: + { + if (buf_len < sizeof(struct smb2_file_rename_info)) + return -EMSGSIZE; + + return set_rename_info(work, fp, + (struct smb2_file_rename_info *)buffer, + buf_len); + } + case FILE_LINK_INFORMATION: + { + struct smb2_file_link_info *file_info; + + if (buf_len < sizeof(struct smb2_file_link_info)) + return -EMSGSIZE; + + file_info = (struct smb2_file_link_info *)buffer; + if (file_info->ReplaceIfExists && !(fp->daccess & FILE_DELETE_LE)) { + pr_err("no right to delete : 0x%x\n", fp->daccess); + return -EACCES; + } + + return smb2_create_link(work, work->tcon->share_conf, file_info, + buf_len, fp->filp, + work->conn->local_nls); + } + case FILE_DISPOSITION_INFORMATION: + { + if (buf_len < sizeof(struct smb2_file_disposition_info)) + return -EMSGSIZE; + + return set_file_disposition_info(work, fp, + (struct smb2_file_disposition_info *)buffer); + } + case FILE_FULL_EA_INFORMATION: + { + if (!(fp->daccess & FILE_WRITE_EA_LE)) { + pr_err("Not permitted to write ext attr: 0x%x\n", + fp->daccess); + return -EACCES; + } + + if (buf_len < sizeof(struct smb2_ea_info)) + return -EMSGSIZE; + + return smb2_set_ea((struct smb2_ea_info *)buffer, + buf_len, &fp->filp->f_path, true); + } + case FILE_POSITION_INFORMATION: + { + if (buf_len < sizeof(struct smb2_file_pos_info)) + return -EMSGSIZE; + + return set_file_position_info(fp, (struct smb2_file_pos_info *)buffer); + } + case FILE_MODE_INFORMATION: + { + if (buf_len < sizeof(struct smb2_file_mode_info)) + return -EMSGSIZE; + + return set_file_mode_info(fp, (struct smb2_file_mode_info *)buffer); + } + } + + pr_err("Unimplemented Fileinfoclass :%d\n", req->FileInfoClass); + return -EOPNOTSUPP; +} + +static int smb2_set_info_sec(struct ksmbd_file *fp, int addition_info, + char *buffer, int buf_len) +{ + struct smb_ntsd *pntsd = (struct smb_ntsd *)buffer; + + fp->saccess |= FILE_SHARE_DELETE_LE; + + if (!(fp->daccess & (FILE_WRITE_DAC_LE | FILE_WRITE_OWNER_LE))) + return -EACCES; + + return set_info_sec(fp->conn, fp->tcon, &fp->filp->f_path, pntsd, + buf_len, false, true); +} + +/** + * smb2_set_info() - handler for smb2 set info command handler + * @work: smb work containing set info request buffer + * + * Return: 0 on success, otherwise error + */ +int smb2_set_info(struct ksmbd_work *work) +{ + const struct cred *saved_cred; + struct smb2_set_info_req *req; + struct smb2_set_info_rsp *rsp; + struct ksmbd_file *fp = NULL; + int rc = 0; + bool chseq_err = false; + unsigned int id = KSMBD_NO_FID, pid = KSMBD_NO_FID; + + ksmbd_debug(SMB, "Received smb2 set info request\n"); + + if (work->next_smb2_rcv_hdr_off) { + req = ksmbd_req_buf_next(work); + rsp = ksmbd_resp_buf_next(work); + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; + if (!has_file_id(req->VolatileFileId)) { + ksmbd_debug(SMB, "Compound request set FID = %llu\n", + work->compound_fid); + id = work->compound_fid; + pid = work->compound_pfid; + } + } else { + req = smb_get_msg(work->request_buf); + rsp = smb_get_msg(work->response_buf); + } + + if (!test_tree_conn_flag(work->tcon, KSMBD_TREE_CONN_FLAG_WRITABLE)) { + ksmbd_debug(SMB, "User does not have write permission\n"); + pr_err("User does not have write permission\n"); + rc = -EACCES; + goto err_out; + } + + if (!has_file_id(id)) { + id = req->VolatileFileId; + pid = req->PersistentFileId; + } + + fp = ksmbd_lookup_fd_slow(work, id, pid); + if (!fp) { + ksmbd_debug(SMB, "Invalid id for close: %u\n", id); + rc = -ENOENT; + goto err_out; + } + + rc = smb2_set_request_open(work, fp, &req->hdr, true, false); + if (rc) { + rsp->hdr.Status = STATUS_FILE_NOT_AVAILABLE; + chseq_err = true; + goto err_out; + } + + saved_cred = override_creds(fp->filp->f_cred); + switch (req->InfoType) { + case SMB2_O_INFO_FILE: + ksmbd_debug(SMB, "GOT SMB2_O_INFO_FILE\n"); + rc = smb2_set_info_file(work, fp, req, work->tcon->share_conf); + break; + case SMB2_O_INFO_SECURITY: + ksmbd_debug(SMB, "GOT SMB2_O_INFO_SECURITY\n"); + rc = smb2_set_info_sec(fp, + le32_to_cpu(req->AdditionalInformation), + (char *)req + le16_to_cpu(req->BufferOffset), + le32_to_cpu(req->BufferLength)); + break; + default: + rc = -EOPNOTSUPP; + } + revert_creds(saved_cred); + + if (rc < 0) + goto err_out; + + rsp->StructureSize = cpu_to_le16(2); + rc = ksmbd_iov_pin_rsp(work, (void *)rsp, + sizeof(struct smb2_set_info_rsp)); + if (rc) + goto err_out; + ksmbd_fd_put(work, fp); + return 0; + +err_out: + if (rc == -EACCES || rc == -EPERM || rc == -EXDEV) { + if (fp && req->InfoType == SMB2_O_INFO_FILE && + req->FileInfoClass == FILE_DISPOSITION_INFORMATION && + fp->f_ci->m_fattr & FILE_ATTRIBUTE_READONLY_LE) + rsp->hdr.Status = STATUS_CANNOT_DELETE; + else + rsp->hdr.Status = STATUS_ACCESS_DENIED; + } + else if (rc == -EINVAL) + rsp->hdr.Status = STATUS_INVALID_PARAMETER; + else if (rc == -EMSGSIZE) + rsp->hdr.Status = STATUS_INFO_LENGTH_MISMATCH; + else if (rc == -ENOSPC || rc == -EFBIG) + rsp->hdr.Status = STATUS_DISK_FULL; + else if (rc == -ESHARE) + rsp->hdr.Status = STATUS_SHARING_VIOLATION; + else if (rc == -ENOENT) + rsp->hdr.Status = STATUS_OBJECT_NAME_INVALID; + else if (rc == -EBUSY || rc == -ENOTEMPTY) + rsp->hdr.Status = STATUS_DIRECTORY_NOT_EMPTY; + else if (rc == -EAGAIN && !chseq_err) + rsp->hdr.Status = STATUS_FILE_LOCK_CONFLICT; + else if (rc == -EBADF || rc == -ESTALE) + rsp->hdr.Status = STATUS_INVALID_HANDLE; + else if (rc == -EEXIST) + rsp->hdr.Status = STATUS_OBJECT_NAME_COLLISION; + else if (rsp->hdr.Status == 0 || rc == -EOPNOTSUPP) + rsp->hdr.Status = STATUS_INVALID_INFO_CLASS; + smb2_set_err_rsp(work); + ksmbd_fd_put(work, fp); + ksmbd_debug(SMB, "error while processing smb2 query rc = %d\n", rc); + return rc; +} + +/** + * smb2_read_pipe() - handler for smb2 read from IPC pipe + * @work: smb work containing read IPC pipe command buffer + * + * Return: 0 on success, otherwise error + */ +static noinline int smb2_read_pipe(struct ksmbd_work *work) +{ + int nbytes = 0, err; + u64 id; + struct ksmbd_rpc_command *rpc_resp; + struct smb2_read_req *req; + struct smb2_read_rsp *rsp; + + WORK_BUFFERS(work, req, rsp); + + id = req->VolatileFileId; + + rpc_resp = ksmbd_rpc_read(work->sess, id); + if (rpc_resp) { + void *aux_payload_buf; + + if (rpc_resp->flags != KSMBD_RPC_OK) { + err = -EINVAL; + goto out; + } + + aux_payload_buf = + kvmalloc(rpc_resp->payload_sz, KSMBD_DEFAULT_GFP); + if (!aux_payload_buf) { + err = -ENOMEM; + goto out; + } + + memcpy(aux_payload_buf, rpc_resp->payload, rpc_resp->payload_sz); + + nbytes = rpc_resp->payload_sz; + err = ksmbd_iov_pin_rsp_read(work, (void *)rsp, + offsetof(struct smb2_read_rsp, Buffer), + aux_payload_buf, nbytes); + if (err) { + kvfree(aux_payload_buf); + goto out; + } + kvfree(rpc_resp); + } else { + err = ksmbd_iov_pin_rsp(work, (void *)rsp, + offsetof(struct smb2_read_rsp, Buffer)); + if (err) + goto out; + } + + rsp->StructureSize = cpu_to_le16(17); + rsp->DataOffset = 80; + rsp->Reserved = 0; + rsp->DataLength = cpu_to_le32(nbytes); + rsp->DataRemaining = 0; + rsp->Flags = 0; + return 0; + +out: + rsp->hdr.Status = STATUS_UNEXPECTED_IO_ERROR; + smb2_set_err_rsp(work); + kvfree(rpc_resp); + return err; +} + +/** + * smb2_set_rdma_key() - validate descriptors and save invalidation state + * @work: request work item + * @desc: first RDMA buffer descriptor + * @Channel: nested RDMA channel type + * @channel_info_len: descriptor array length + * + * Return: 0 on success, otherwise -EINVAL + */ +static int smb2_set_rdma_key(struct ksmbd_work *work, + struct smbdirect_buffer_descriptor_v1 *desc, + __le32 Channel, __le16 channel_info_len) +{ + unsigned int i, ch_count; + + if (Channel != SMB2_CHANNEL_RDMA_V1 && + Channel != SMB2_CHANNEL_RDMA_V1_INVALIDATE) + return -EINVAL; + if (work->conn->dialect == SMB30_PROT_ID && + Channel != SMB2_CHANNEL_RDMA_V1) + return -EINVAL; + if (le16_to_cpu(channel_info_len) % sizeof(*desc)) + return -EINVAL; + + ch_count = le16_to_cpu(channel_info_len) / sizeof(*desc); + if (ksmbd_debug_types & KSMBD_DEBUG_RDMA) { + for (i = 0; i < ch_count; i++) { + pr_info("RDMA r/w request %#x: token %#x, length %#x\n", + i, + le32_to_cpu(desc[i].token), + le32_to_cpu(desc[i].length)); + } + } + if (!ch_count) + return -EINVAL; + + work->need_invalidate_rkey = + (Channel == SMB2_CHANNEL_RDMA_V1_INVALIDATE); + if (Channel == SMB2_CHANNEL_RDMA_V1_INVALIDATE) + work->remote_key = le32_to_cpu(desc->token); + return 0; +} + +/** + * smb2_prep_rdma_read() - transform an RDMA READ payload + * @work: request work item + * @req: READ request controlling encryption or signing + * @rsp: READ response receiving transform metadata + * @data: data that will be transferred through RDMA + * @datalen: data length + * + * Encrypt the payload in place and encode the detached crypto metadata in + * the response buffer. + * + * Return: metadata length, zero when no transform applies, or negative errno + */ +static int smb2_prep_rdma_read(struct ksmbd_work *work, + struct smb2_read_req *req, + struct smb2_read_rsp *rsp, + void *data, unsigned int datalen) +{ + struct ksmbd_conn *conn = work->conn; + struct smb2_rdma_transform *transform; + struct smb2_rdma_crypto_transform *crypto; + u8 *nonce; + unsigned int nonce_len = 0, transform_len; + u16 transform_type; + int err; + + if (!work->encrypted || + !(conn->rdma_transform_ids & BIT(SMB2_RDMA_TRANSFORM_ENCRYPTION))) + return 0; + + transform_type = SMB2_RDMA_TRANSFORM_TYPE_ENCRYPTION; + nonce_len = (conn->cipher_type == SMB2_ENCRYPTION_AES128_GCM || + conn->cipher_type == SMB2_ENCRYPTION_AES256_GCM) ? + SMB3_AES_GCM_NONCE : SMB3_AES_CCM_NONCE; + + transform = (struct smb2_rdma_transform *)rsp->Buffer; + crypto = (struct smb2_rdma_crypto_transform *)(transform + 1); + memset(transform, 0, sizeof(*transform) + sizeof(*crypto) + + SMB2_SIGNATURE_SIZE + nonce_len); + transform->Channel = SMB2_CHANNEL_NONE; + transform->TransformCount = cpu_to_le16(1); + + crypto->TransformType = cpu_to_le16(transform_type); + crypto->SignatureLength = cpu_to_le16(SMB2_SIGNATURE_SIZE); + crypto->NonceLength = cpu_to_le16(nonce_len); + nonce = crypto->Signature + SMB2_SIGNATURE_SIZE; + + get_random_bytes(nonce, nonce_len); + err = ksmbd_crypt_rdma(conn, + work->sess->smb3encryptionkey, + data, datalen, nonce, nonce_len, + crypto->Signature, + SMB2_SIGNATURE_SIZE, true); + if (err) { + pr_err("RDMA READ encryption failed: session=%llu payload=%u rc=%d\n", + work->sess->id, datalen, err); + return err; + } + + transform_len = sizeof(*transform) + sizeof(*crypto) + + SMB2_SIGNATURE_SIZE + nonce_len; + rsp->Flags = SMB2_READFLAG_RESPONSE_RDMA_TRANSFORM; + rsp->DataLength = cpu_to_le32(transform_len); + ksmbd_debug(RDMA, + "RDMA READ encryption prepared: session=%llu cipher=0x%04x payload=%u transform=%u nonce=%u tag=%u\n", + work->sess->id, le16_to_cpu(conn->cipher_type), datalen, + transform_len, nonce_len, SMB2_SIGNATURE_SIZE); + return transform_len; +} + +struct smb2_rdma_write_transform { + struct smbdirect_buffer_descriptor_v1 *desc; + struct smb2_rdma_crypto_transform *crypto; + u8 *nonce; + unsigned int desc_len; + unsigned int nonce_len; + unsigned int signature_len; + u16 type; + __le32 channel; +}; + +/** + * smb2_current_req_len() - return the current compound request element size + * @work: request work item + * @hdr: current SMB2 header + * + * Return: current request element length measured from the SMB2 header + */ +static unsigned int smb2_current_req_len(struct ksmbd_work *work, + struct smb2_hdr *hdr) +{ + if (hdr->NextCommand) + return le32_to_cpu(hdr->NextCommand); + return get_rfc1002_len(work->request_buf) - + work->next_smb2_rcv_hdr_off; +} + +/** + * check_rdma_desc() - validate an RDMA descriptor array + * @desc: descriptor array + * @desc_len: descriptor array length + * @required_len: minimum aggregate buffer length + * + * Return: 0 when the descriptors cover the transfer, otherwise -EINVAL + */ +static int check_rdma_desc(struct smbdirect_buffer_descriptor_v1 *desc, + unsigned int desc_len, + unsigned int required_len) +{ + unsigned int i, count; + u64 described_len = 0; + + if (!desc_len || desc_len % sizeof(*desc)) + return -EINVAL; + count = desc_len / sizeof(*desc); + if (!le32_to_cpu(desc[0].length)) + return -EINVAL; + for (i = 0; i < count; i++) + described_len += le32_to_cpu(desc[i].length); + return described_len < required_len ? -EINVAL : 0; +} + +/** + * smb2_parse_rdma_write_transform() - validate RDMA WRITE transform metadata + * @work: request work item + * @req: WRITE request containing the transform + * @info: parsed transform information + * + * Validate transform counts, crypto fields, descriptor alignment and bounds, + * negotiated algorithms, and the nested RDMA channel. + * + * Return: 0 on success, otherwise a negative errno + */ +static int smb2_parse_rdma_write_transform(struct ksmbd_work *work, + struct smb2_write_req *req, + struct smb2_rdma_write_transform *info) +{ + struct smb2_rdma_transform *transform; + struct smb2_rdma_crypto_transform *crypto; + unsigned int req_len = smb2_current_req_len(work, &req->hdr); + unsigned int offset = le16_to_cpu(req->WriteChannelInfoOffset); + unsigned int length = le16_to_cpu(req->WriteChannelInfoLength); + unsigned int desc_offset, desc_len, crypto_len, expected_desc_offset; + int err; + + if (!work->conn->rdma_transform_ids || + offset < offsetof(struct smb2_write_req, Buffer) || + length < sizeof(*transform) || offset > req_len || + length > req_len - offset) + return -EINVAL; + + transform = (struct smb2_rdma_transform *)((char *)req + offset); + if (le16_to_cpu(transform->TransformCount) != 1 || + (transform->Channel != SMB2_CHANNEL_RDMA_V1 && + transform->Channel != SMB2_CHANNEL_RDMA_V1_INVALIDATE)) + return -EINVAL; + + desc_offset = le16_to_cpu(transform->RdmaDescriptorOffset); + desc_len = le16_to_cpu(transform->RdmaDescriptorLength); + if (!desc_len || desc_len % sizeof(*info->desc) || + desc_offset < sizeof(*transform) || desc_offset > length || + desc_len > length - desc_offset) + return -EINVAL; + + crypto = (struct smb2_rdma_crypto_transform *)(transform + 1); + if (length - sizeof(*transform) < sizeof(*crypto)) + return -EINVAL; + info->type = le16_to_cpu(crypto->TransformType); + info->signature_len = le16_to_cpu(crypto->SignatureLength); + info->nonce_len = le16_to_cpu(crypto->NonceLength); + if (!info->signature_len) + return info->type == SMB2_RDMA_TRANSFORM_TYPE_ENCRYPTION ? + -EBADMSG : -EINVAL; + if (info->signature_len > SMB2_SIGNATURE_SIZE) + return info->type == SMB2_RDMA_TRANSFORM_TYPE_ENCRYPTION ? + -EBADMSG : -EINVAL; + if (info->signature_len > length - sizeof(*transform) - sizeof(*crypto) || + info->nonce_len > length - sizeof(*transform) - sizeof(*crypto) - + info->signature_len) + return -EINVAL; + + crypto_len = sizeof(*crypto) + info->signature_len + info->nonce_len; + expected_desc_offset = ALIGN(sizeof(*transform) + crypto_len, 8); + if (desc_offset != expected_desc_offset) + return -EINVAL; + + if (info->type == SMB2_RDMA_TRANSFORM_TYPE_ENCRYPTION) { + unsigned int expected_nonce_len; + + if (!(work->conn->rdma_transform_ids & + BIT(SMB2_RDMA_TRANSFORM_ENCRYPTION)) || !work->encrypted) + return -EINVAL; + expected_nonce_len = + (work->conn->cipher_type == SMB2_ENCRYPTION_AES128_GCM || + work->conn->cipher_type == SMB2_ENCRYPTION_AES256_GCM) ? + SMB3_AES_GCM_NONCE : SMB3_AES_CCM_NONCE; + if (info->nonce_len != expected_nonce_len) + return -EBADMSG; + } else { + return -EINVAL; + } + + info->desc = (struct smbdirect_buffer_descriptor_v1 *) + ((char *)transform + desc_offset); + info->desc_len = desc_len; + info->crypto = crypto; + info->nonce = crypto->Signature + info->signature_len; + info->channel = transform->Channel; + err = check_rdma_desc(info->desc, info->desc_len, + le32_to_cpu(req->RemainingBytes)); + if (err) + return err; + + ksmbd_debug(RDMA, + "RDMA WRITE encryption metadata: session=%llu cipher=0x%04x payload=%u channel=0x%x descriptors=%zu nonce=%u tag=%u\n", + work->sess->id, le16_to_cpu(work->conn->cipher_type), + le32_to_cpu(req->RemainingBytes), le32_to_cpu(info->channel), + info->desc_len / sizeof(*info->desc), info->nonce_len, + info->signature_len); + return 0; +} + +/** + * smb2_read_rdma() - transfer READ data to client RDMA buffers + * @work: request work item + * @req: READ request containing client descriptors + * @data_buf: data to transfer + * @length: data length + * + * Return: transferred length on success, otherwise a negative errno + */ +static ssize_t smb2_read_rdma(struct ksmbd_work *work, + struct smb2_read_req *req, void *data_buf, + size_t length) +{ + int err; + + err = ksmbd_conn_rdma_write(work->conn, data_buf, length, + (struct smbdirect_buffer_descriptor_v1 *) + ((char *)req + le16_to_cpu(req->ReadChannelInfoOffset)), + le16_to_cpu(req->ReadChannelInfoLength)); + if (err) + return err; + + return length; +} + +/** + * smb2_read() - handler for smb2 read from file + * @work: smb work containing read command buffer + * + * Return: 0 on success, otherwise error + */ +int smb2_read(struct ksmbd_work *work) +{ + struct ksmbd_conn *conn = work->conn; + struct smb2_read_req *req; + struct smb2_read_rsp *rsp; + struct ksmbd_file *fp = NULL; + loff_t offset; + size_t length, mincount; + ssize_t nbytes = 0, remain_bytes = 0; + int err = 0; + int rdma_transform_len = 0; + bool is_rdma_channel = false, async_interim = false; + unsigned int max_read_size = conn->vals->max_read_size; + unsigned int id = KSMBD_NO_FID, pid = KSMBD_NO_FID; + void *aux_payload_buf; + + ksmbd_debug(SMB, "Received smb2 read request\n"); + + if (test_share_config_flag(work->tcon->share_conf, + KSMBD_SHARE_FLAG_PIPE)) { + ksmbd_debug(SMB, "IPC pipe read request\n"); + return smb2_read_pipe(work); + } + + if (work->next_smb2_rcv_hdr_off) { + req = ksmbd_req_buf_next(work); + rsp = ksmbd_resp_buf_next(work); + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; + if (!has_file_id(req->VolatileFileId)) { + ksmbd_debug(SMB, "Compound request set FID = %llu\n", + work->compound_fid); + id = work->compound_fid; + pid = work->compound_pfid; + } + } else { + req = smb_get_msg(work->request_buf); + rsp = smb_get_msg(work->response_buf); + } + + if (!has_file_id(id)) { + id = req->VolatileFileId; + pid = req->PersistentFileId; + } + + if (req->Channel != SMB2_CHANNEL_NONE && + req->Channel != SMB2_CHANNEL_RDMA_V1 && + req->Channel != SMB2_CHANNEL_RDMA_V1_INVALIDATE) { + err = -EINVAL; + goto out; + } + if (req->Channel == SMB2_CHANNEL_RDMA_V1_INVALIDATE || + req->Channel == SMB2_CHANNEL_RDMA_V1) { + is_rdma_channel = true; + max_read_size = get_smbd_max_read_write_size(work->conn->transport); + if (max_read_size == 0) { + err = -EINVAL; + goto out; + } + } + + if (is_rdma_channel == true) { + unsigned int ch_offset = le16_to_cpu(req->ReadChannelInfoOffset); + unsigned int ch_len = le16_to_cpu(req->ReadChannelInfoLength); + unsigned int req_len = smb2_current_req_len(work, &req->hdr); + struct smbdirect_buffer_descriptor_v1 *desc; + + if (!le32_to_cpu(req->Length) || + ch_offset < offsetof(struct smb2_read_req, Buffer) || + ch_offset > req_len || ch_len > req_len - ch_offset) { + err = -EINVAL; + goto out; + } + desc = (struct smbdirect_buffer_descriptor_v1 *) + ((char *)req + ch_offset); + err = check_rdma_desc(desc, ch_len, le32_to_cpu(req->Length)); + if (err) + goto out; + err = smb2_set_rdma_key(work, desc, + req->Channel, + req->ReadChannelInfoLength); + if (err) + goto out; + } + + fp = ksmbd_lookup_fd_slow(work, id, pid); + if (!fp) { + err = -ENOENT; + goto out; + } + + err = smb2_set_request_open(work, fp, &req->hdr, true, true); + if (err) + goto out; + + if (!(fp->daccess & (FILE_READ_DATA_LE | FILE_READ_ATTRIBUTES_LE))) { + pr_err("Not permitted to read : 0x%x\n", fp->daccess); + err = -EACCES; + goto out; + } + + if (work->next_smb2_rcv_hdr_off && !req->hdr.NextCommand) { + err = setup_async_work(work, NULL, NULL); + if (err) + goto out; + smb2_send_interim_resp(work, STATUS_PENDING); + async_interim = true; + } + + offset = le64_to_cpu(req->Offset); + if (offset < 0) { + err = -EINVAL; + goto out; + } + length = le32_to_cpu(req->Length); + mincount = le32_to_cpu(req->MinimumCount); + + if (length > max_read_size) { + ksmbd_debug(SMB, "limiting read size to max size(%u)\n", + max_read_size); + err = -EINVAL; + goto out; + } + + ksmbd_debug(SMB, "filename %pD, offset %lld, len %zu\n", + fp->filp, offset, length); + + aux_payload_buf = kvmalloc(ALIGN(length, 8), KSMBD_DEFAULT_GFP); + if (!aux_payload_buf) { + err = -ENOMEM; + goto out; + } + + nbytes = ksmbd_vfs_read(work, fp, length, &offset, aux_payload_buf); + if (nbytes < 0) { + kvfree(aux_payload_buf); + err = nbytes; + goto out; + } + + /* + * ksmbd_vfs_read() fills only nbytes; the [nbytes, ALIGN(nbytes, 8)) + * tail of the un-zeroed buffer is transmitted as compound-response + * alignment padding, leaking uninitialized kernel memory to the + * client. Zero just that tail. + */ + if (nbytes & 7) + memset(aux_payload_buf + nbytes, 0, ALIGN(nbytes, 8) - nbytes); + + if ((nbytes == 0 && length != 0) || nbytes < mincount) { + kvfree(aux_payload_buf); + rsp->hdr.Status = STATUS_END_OF_FILE; + smb2_set_err_rsp(work); + if (async_interim) + release_async_work(work); + ksmbd_fd_put(work, fp); + return -ENODATA; + } + + ksmbd_debug(SMB, "nbytes %zu, offset %lld mincount %zu\n", + nbytes, offset, mincount); + + if (is_rdma_channel == true) { + rdma_transform_len = smb2_prep_rdma_read(work, req, + rsp, + aux_payload_buf, + nbytes); + if (rdma_transform_len < 0) { + kvfree(aux_payload_buf); + err = rdma_transform_len; + goto out; + } + /* write data to the client using rdma channel */ + remain_bytes = smb2_read_rdma(work, req, + aux_payload_buf, + nbytes); + if (remain_bytes < 0) + pr_err("RDMA READ transfer failed: session=%llu payload=%zu transform=%d rc=%zd\n", + work->sess ? work->sess->id : 0, nbytes, + rdma_transform_len, remain_bytes); + else + ksmbd_debug(RDMA, + "RDMA READ transfer completed: session=%llu payload=%zu transform=%d\n", + work->sess ? work->sess->id : 0, nbytes, + rdma_transform_len); + kvfree(aux_payload_buf); + aux_payload_buf = NULL; + nbytes = 0; + if (remain_bytes < 0) { + err = (int)remain_bytes; + goto out; + } + } + + rsp->StructureSize = cpu_to_le16(17); + rsp->DataOffset = 80; + rsp->Reserved = 0; + rsp->DataLength = cpu_to_le32(rdma_transform_len ?: nbytes); + rsp->DataRemaining = cpu_to_le32(remain_bytes); + rsp->Flags = rdma_transform_len ? + SMB2_READFLAG_RESPONSE_RDMA_TRANSFORM : 0; + err = ksmbd_iov_pin_rsp_read(work, (void *)rsp, + offsetof(struct smb2_read_rsp, Buffer) + + rdma_transform_len, + aux_payload_buf, nbytes); + if (err) { + kvfree(aux_payload_buf); + goto out; + } + if (async_interim) + release_async_work(work); + /* + * RDMA responses are transferred through channel buffers and encrypted + * responses use the encryption transform, so only normal SMB transport + * responses are candidates for compression. + */ + if (!is_rdma_channel && nbytes && + (req->Flags & SMB2_READFLAG_REQUEST_COMPRESSED) && + conn->compress_algorithm != SMB3_COMPRESS_NONE) + work->compress_response = true; + ksmbd_fd_put(work, fp); + return 0; + +out: + if (async_interim) + release_async_work(work); + if (err) { + if (err == -EISDIR) + rsp->hdr.Status = STATUS_INVALID_DEVICE_REQUEST; + else if (err == -EAGAIN) + rsp->hdr.Status = STATUS_FILE_LOCK_CONFLICT; + else if (err == -ENOENT) + rsp->hdr.Status = STATUS_FILE_CLOSED; + else if (err == -EACCES) + rsp->hdr.Status = STATUS_ACCESS_DENIED; + else if (err == -ESHARE) + rsp->hdr.Status = STATUS_SHARING_VIOLATION; + else if (err == -EINVAL) + rsp->hdr.Status = STATUS_INVALID_PARAMETER; + else + rsp->hdr.Status = STATUS_INVALID_HANDLE; + + smb2_set_err_rsp(work); + } + ksmbd_fd_put(work, fp); + return err; +} + +/** + * smb2_write_pipe() - handler for smb2 write on IPC pipe + * @work: smb work containing write IPC pipe command buffer + * + * Return: 0 on success, otherwise error + */ +static noinline int smb2_write_pipe(struct ksmbd_work *work) +{ + struct smb2_write_req *req; + struct smb2_write_rsp *rsp; + struct ksmbd_rpc_command *rpc_resp; + u64 id = 0; + int err = 0, ret = 0; + char *data_buf; + size_t length; + + WORK_BUFFERS(work, req, rsp); + + length = le32_to_cpu(req->Length); + id = req->VolatileFileId; + + if ((u64)le16_to_cpu(req->DataOffset) + length > + get_rfc1002_len(work->request_buf)) { + pr_err("invalid write data offset %u, smb_len %u\n", + le16_to_cpu(req->DataOffset), + get_rfc1002_len(work->request_buf)); + err = -EINVAL; + goto out; + } + + data_buf = (char *)(((char *)&req->hdr.ProtocolId) + + le16_to_cpu(req->DataOffset)); + + rpc_resp = ksmbd_rpc_write(work->sess, id, data_buf, length); + if (rpc_resp) { + if (rpc_resp->flags == KSMBD_RPC_ENOTIMPLEMENTED) { + rsp->hdr.Status = STATUS_NOT_SUPPORTED; + kvfree(rpc_resp); + smb2_set_err_rsp(work); + return -EOPNOTSUPP; + } + if (rpc_resp->flags != KSMBD_RPC_OK) { + rsp->hdr.Status = STATUS_INVALID_HANDLE; + smb2_set_err_rsp(work); + kvfree(rpc_resp); + return ret; + } + kvfree(rpc_resp); + } + + rsp->StructureSize = cpu_to_le16(17); + rsp->DataOffset = 0; + rsp->Reserved = 0; + rsp->DataLength = cpu_to_le32(length); + rsp->DataRemaining = 0; + rsp->Reserved2 = 0; + err = ksmbd_iov_pin_rsp(work, (void *)rsp, + offsetof(struct smb2_write_rsp, Buffer)); +out: + if (err) { + rsp->hdr.Status = STATUS_INVALID_HANDLE; + smb2_set_err_rsp(work); + } + + return err; +} + +/** + * smb2_write_rdma() - receive and store an RDMA WRITE payload + * @work: request work item + * @desc: client RDMA buffer descriptors + * @desc_len: descriptor array length + * @transform: parsed transform, or NULL for an untransformed transfer + * @fp: target open file + * @offset: target file offset + * @length: transfer length + * @sync: request synchronous storage completion + * + * Receive the payload, authenticate or decrypt it when required, and write it + * to the target file. + * + * Return: written byte count on success, otherwise a negative errno + */ +static ssize_t smb2_write_rdma(struct ksmbd_work *work, + struct smbdirect_buffer_descriptor_v1 *desc, + unsigned int desc_len, + struct smb2_rdma_write_transform *transform, + struct ksmbd_file *fp, loff_t offset, + size_t length, bool sync) +{ + char *data_buf; + int ret; + ssize_t nbytes; + + data_buf = kvzalloc(length, KSMBD_DEFAULT_GFP); + if (!data_buf) + return -ENOMEM; + + ret = ksmbd_conn_rdma_read(work->conn, data_buf, length, desc, + desc_len); + if (ret < 0) { + if (transform) + pr_err("RDMA WRITE encrypted transfer failed: session=%llu payload=%zu rdma_read_rc=%d\n", + work->sess->id, length, ret); + kvfree(data_buf); + return ret; + } + if (transform && + transform->type == SMB2_RDMA_TRANSFORM_TYPE_ENCRYPTION) { + ret = ksmbd_crypt_rdma(work->conn, + work->sess->smb3decryptionkey, + data_buf, length, transform->nonce, + transform->nonce_len, + transform->crypto->Signature, + transform->signature_len, false); + if (ret) { + pr_err("RDMA WRITE decryption failed: session=%llu payload=%zu rc=%d\n", + work->sess->id, length, ret); + kvfree(data_buf); + return ret == -ENOMEM ? ret : -EBADMSG; + } + } + + ret = ksmbd_vfs_write(work, fp, data_buf, length, &offset, sync, &nbytes); + kvfree(data_buf); + if (ret < 0) { + if (transform) + pr_err("RDMA WRITE encrypted file write failed: session=%llu payload=%zu rc=%d\n", + work->sess->id, length, ret); + return ret; + } + ksmbd_debug(RDMA, + "RDMA WRITE transfer completed: session=%llu payload=%zu transformed=%u written=%zd\n", + work->sess ? work->sess->id : 0, length, !!transform, nbytes); + + return nbytes; +} + +/** + * smb2_write() - handler for smb2 write from file + * @work: smb work containing write command buffer + * + * Return: 0 on success, otherwise error + */ +int smb2_write(struct ksmbd_work *work) +{ + struct smb2_write_req *req; + struct smb2_write_rsp *rsp; + struct smb2_rdma_write_transform rdma_transform = {}; + struct smb2_rdma_write_transform *rdma_info = NULL; + struct smbdirect_buffer_descriptor_v1 *rdma_desc = NULL; + unsigned int rdma_desc_len = 0; + struct ksmbd_file *fp = NULL; + loff_t offset; + size_t length; + ssize_t nbytes; + char *data_buf; + bool writethrough = false, is_rdma_channel = false; + bool async_interim = false; + bool chseq_err = false; + int err = 0; + unsigned int max_write_size = work->conn->vals->max_write_size; + unsigned int id = KSMBD_NO_FID, pid = KSMBD_NO_FID; + + ksmbd_debug(SMB, "Received smb2 write request\n"); + + WORK_BUFFERS(work, req, rsp); + + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; + + if (work->next_smb2_rcv_hdr_off && + !has_file_id(req->VolatileFileId)) { + ksmbd_debug(SMB, "Compound request set FID = %llu\n", + work->compound_fid); + id = work->compound_fid; + pid = work->compound_pfid; + } + + if (!has_file_id(id)) { + id = req->VolatileFileId; + pid = req->PersistentFileId; + } + + if (test_share_config_flag(work->tcon->share_conf, KSMBD_SHARE_FLAG_PIPE)) { + ksmbd_debug(SMB, "IPC pipe write request\n"); + return smb2_write_pipe(work); + } + + offset = le64_to_cpu(req->Offset); + if (offset < 0) { + err = -EINVAL; + goto out; + } + length = le32_to_cpu(req->Length); + + if (req->Channel != SMB2_CHANNEL_NONE && + req->Channel != SMB2_CHANNEL_RDMA_V1 && + req->Channel != SMB2_CHANNEL_RDMA_V1_INVALIDATE && + req->Channel != SMB2_CHANNEL_RDMA_TRANSFORM) { + err = -EINVAL; + goto out; + } + if (req->Channel == SMB2_CHANNEL_RDMA_TRANSFORM && + work->conn->dialect != SMB311_PROT_ID) { + err = -EINVAL; + goto out; + } + if (req->Channel == SMB2_CHANNEL_RDMA_V1 || + req->Channel == SMB2_CHANNEL_RDMA_V1_INVALIDATE || + req->Channel == SMB2_CHANNEL_RDMA_TRANSFORM) { + is_rdma_channel = true; + max_write_size = get_smbd_max_read_write_size(work->conn->transport); + if (max_write_size == 0) { + err = -EINVAL; + goto out; + } + length = le32_to_cpu(req->RemainingBytes); + } + + if (length) { + u64 end = (u64)offset + length; + + if (end > SMB2_MAX_FILE_SIZE) { + err = -EINVAL; + goto out; + } + if (end == SMB2_MAX_FILE_SIZE) { + err = -EFBIG; + goto out; + } + } + + if (is_rdma_channel == true) { + unsigned int ch_offset = le16_to_cpu(req->WriteChannelInfoOffset); + unsigned int ch_len = le16_to_cpu(req->WriteChannelInfoLength); + unsigned int req_len = smb2_current_req_len(work, &req->hdr); + + if (!length || req->Length != 0 || req->DataOffset != 0 || + ch_offset < offsetof(struct smb2_write_req, Buffer) || + ch_offset > req_len || ch_len > req_len - ch_offset) { + err = -EINVAL; + goto out; + } + if (req->Channel == SMB2_CHANNEL_RDMA_TRANSFORM) { + err = smb2_parse_rdma_write_transform(work, req, + &rdma_transform); + if (err) { + pr_err("RDMA WRITE encryption metadata rejected: session=%llu rc=%d\n", + work->sess ? work->sess->id : 0, err); + goto out; + } + rdma_desc = rdma_transform.desc; + rdma_desc_len = rdma_transform.desc_len; + rdma_info = &rdma_transform; + err = smb2_set_rdma_key(work, rdma_desc, + rdma_transform.channel, + cpu_to_le16(rdma_desc_len)); + } else { + rdma_desc = (struct smbdirect_buffer_descriptor_v1 *) + ((char *)req + ch_offset); + rdma_desc_len = ch_len; + err = check_rdma_desc(rdma_desc, rdma_desc_len, length); + if (err) + goto out; + err = smb2_set_rdma_key(work, rdma_desc, + req->Channel, + req->WriteChannelInfoLength); + } + if (err) + goto out; + } + + if (!test_tree_conn_flag(work->tcon, KSMBD_TREE_CONN_FLAG_WRITABLE)) { + ksmbd_debug(SMB, "User does not have write permission\n"); + err = -EACCES; + goto out; + } + + fp = ksmbd_lookup_fd_slow(work, id, pid); + if (!fp) { + err = -ENOENT; + goto out; + } + + err = smb2_set_request_open(work, fp, &req->hdr, true, false); + if (err) { + rsp->hdr.Status = STATUS_FILE_NOT_AVAILABLE; + chseq_err = true; + goto out; + } + + if (!(fp->daccess & (FILE_WRITE_DATA_LE | FILE_READ_ATTRIBUTES_LE))) { + pr_err("Not permitted to write : 0x%x\n", fp->daccess); + err = -EACCES; + goto out; + } + + if (work->next_smb2_rcv_hdr_off && !req->hdr.NextCommand) { + err = setup_async_work(work, NULL, NULL); + if (err) + goto out; + smb2_send_interim_resp(work, STATUS_PENDING); + async_interim = true; + } + + if (length > max_write_size) { + ksmbd_debug(SMB, "limiting write size to max size(%u)\n", + max_write_size); + err = -EINVAL; + goto out; + } + + ksmbd_debug(SMB, "flags %u\n", le32_to_cpu(req->Flags)); + if (le32_to_cpu(req->Flags) & SMB2_WRITEFLAG_WRITE_THROUGH) + writethrough = true; + + if (is_rdma_channel == false) { + if (le16_to_cpu(req->DataOffset) < + offsetof(struct smb2_write_req, Buffer)) { + err = -EINVAL; + goto out; + } + + data_buf = (char *)(((char *)&req->hdr.ProtocolId) + + le16_to_cpu(req->DataOffset)); + + ksmbd_debug(SMB, "filename %pD, offset %lld, len %zu\n", + fp->filp, offset, length); + err = ksmbd_vfs_write(work, fp, data_buf, length, &offset, + writethrough, &nbytes); + if (err < 0) + goto out; + } else { + /* read data from the client using rdma channel, and + * write the data. + */ + nbytes = smb2_write_rdma(work, rdma_desc, rdma_desc_len, + rdma_info, fp, offset, length, + writethrough); + if (nbytes < 0) { + err = (int)nbytes; + goto out; + } + } + + rsp->StructureSize = cpu_to_le16(17); + rsp->DataOffset = 0; + rsp->Reserved = 0; + rsp->DataLength = cpu_to_le32(nbytes); + rsp->DataRemaining = 0; + rsp->Reserved2 = 0; + err = ksmbd_iov_pin_rsp(work, rsp, offsetof(struct smb2_write_rsp, Buffer)); + if (err) + goto out; + if (async_interim) + release_async_work(work); + ksmbd_fd_put(work, fp); + return 0; + +out: + if (async_interim) + release_async_work(work); + + if (err == -EAGAIN && !chseq_err) + rsp->hdr.Status = STATUS_FILE_LOCK_CONFLICT; + else if (err == -ENOSPC || err == -EFBIG) + rsp->hdr.Status = STATUS_DISK_FULL; + else if (err == -ENOENT) + rsp->hdr.Status = STATUS_FILE_CLOSED; + else if (err == -EACCES) + rsp->hdr.Status = STATUS_ACCESS_DENIED; + else if (err == -ESHARE) + rsp->hdr.Status = STATUS_SHARING_VIOLATION; + else if (err == -EINVAL) + rsp->hdr.Status = STATUS_INVALID_PARAMETER; + else if (err == -EBADMSG) + rsp->hdr.Status = STATUS_AUTH_TAG_MISMATCH; + else if (err == -EKEYREJECTED) + rsp->hdr.Status = STATUS_INVALID_SIGNATURE; + else if (rsp->hdr.Status == 0) + rsp->hdr.Status = STATUS_INVALID_HANDLE; + + smb2_set_err_rsp(work); + ksmbd_fd_put(work, fp); + return err; +} + +/** + * smb2_flush() - handler for smb2 flush file - fsync + * @work: smb work containing flush command buffer + * + * Return: 0 on success, otherwise error + */ +int smb2_flush(struct ksmbd_work *work) +{ + struct smb2_flush_req *req; + struct smb2_flush_rsp *rsp; + u64 id = KSMBD_NO_FID, pid = KSMBD_NO_FID; + int err; + + WORK_BUFFERS(work, req, rsp); + + ksmbd_debug(SMB, "Received smb2 flush request(fid : %llu)\n", req->VolatileFileId); + + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; + + if (work->next_smb2_rcv_hdr_off && + !has_file_id(req->VolatileFileId)) { + ksmbd_debug(SMB, "Compound request set FID = %llu\n", + work->compound_fid); + id = work->compound_fid; + pid = work->compound_pfid; + } + + if (!has_file_id(id)) { + id = req->VolatileFileId; + pid = req->PersistentFileId; + } + + err = ksmbd_vfs_fsync(work, id, pid); + if (err) + goto out; + + rsp->StructureSize = cpu_to_le16(4); + rsp->Reserved = 0; + return ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_flush_rsp)); + +out: + rsp->hdr.Status = STATUS_INVALID_HANDLE; + smb2_set_err_rsp(work); + return err; +} + +/** + * smb2_cancel() - handler for smb2 cancel command + * @work: smb work containing cancel command buffer + * + * Return: 0 on success, otherwise error + */ +int smb2_cancel(struct ksmbd_work *work) +{ + struct ksmbd_conn *conn = work->conn; + struct smb2_hdr *hdr = smb_get_msg(work->request_buf); + struct smb2_hdr *chdr; + struct ksmbd_work *iter; + struct ksmbd_work *cancelled_notify = NULL; + struct list_head *command_list; + + if (work->next_smb2_rcv_hdr_off) + hdr = ksmbd_resp_buf_next(work); + + ksmbd_debug(SMB, "smb2 cancel called on mid %llu, async flags 0x%x\n", + le64_to_cpu(hdr->MessageId), + le32_to_cpu(hdr->Flags)); + + if (hdr->Flags & SMB2_FLAGS_ASYNC_COMMAND) { + command_list = &conn->async_requests; + + spin_lock(&conn->request_lock); + list_for_each_entry(iter, command_list, + async_request_entry) { + chdr = smb_get_msg(iter->request_buf); + + if (iter->async_id != + le64_to_cpu(hdr->Id.AsyncId)) + continue; + + /* + * Only an ACTIVE deferred work may have its cancel_fn + * fired. A CANCELLED or CLOSED work already took the + * smb2_lock() non-ACTIVE early-exit that frees the + * file_lock and skips release_async_work(), so it is + * still on conn->async_requests with a live cancel_fn + * pointing at the freed file_lock. + */ + if (iter->state != KSMBD_WORK_ACTIVE) + break; + + ksmbd_debug(SMB, + "smb2 with AsyncId %llu cancelled command = 0x%x\n", + le64_to_cpu(hdr->Id.AsyncId), + le16_to_cpu(chdr->Command)); + iter->state = KSMBD_WORK_CANCELLED; + if (iter->cancel_fn == smb2_notify_cancel_fn) + cancelled_notify = + smb2_notify_cancel_claim(iter->cancel_argv); + else if (iter->cancel_fn) + iter->cancel_fn(iter->cancel_argv); + break; + } + spin_unlock(&conn->request_lock); + + /* + * Complete a cancelled notify before this CANCEL handler returns. + * Deferring it to the system workqueue lets a following request and + * its response overtake STATUS_CANCELLED, leaving clients waiting + * for the original notify even though the cancellation was accepted. + */ + if (cancelled_notify) + smb2_complete_notify_cancel(cancelled_notify); + } else { + command_list = &conn->requests; + + spin_lock(&conn->request_lock); + list_for_each_entry(iter, command_list, request_entry) { + chdr = smb_get_msg(iter->request_buf); + + if (chdr->MessageId != hdr->MessageId || + iter == work) + continue; + + ksmbd_debug(SMB, + "smb2 with mid %llu cancelled command = 0x%x\n", + le64_to_cpu(hdr->MessageId), + le16_to_cpu(chdr->Command)); + iter->state = KSMBD_WORK_CANCELLED; + break; + } + spin_unlock(&conn->request_lock); + } + + /* For SMB2_CANCEL command itself send no response*/ + work->send_no_response = 1; + return 0; +} + +struct file_lock *smb_flock_init(struct file *f) +{ + struct file_lock *fl; + + fl = locks_alloc_lock(); + if (!fl) + goto out; + + locks_init_lock(fl); + + fl->c.flc_owner = f; + fl->c.flc_pid = current->tgid; + fl->c.flc_file = f; + fl->c.flc_flags = FL_POSIX; + fl->fl_ops = NULL; + fl->fl_lmops = NULL; + +out: + return fl; +} + +static int smb2_set_flock_flags(struct file_lock *flock, int flags) +{ + int cmd = -EINVAL; + + /* Checking for wrong flag combination during lock request*/ + switch (flags) { + case SMB2_LOCKFLAG_SHARED: + ksmbd_debug(SMB, "received shared request\n"); + cmd = F_SETLKW; + flock->c.flc_type = F_RDLCK; + flock->c.flc_flags |= FL_SLEEP; + break; + case SMB2_LOCKFLAG_EXCLUSIVE: + ksmbd_debug(SMB, "received exclusive request\n"); + cmd = F_SETLKW; + flock->c.flc_type = F_WRLCK; + flock->c.flc_flags |= FL_SLEEP; + break; + case SMB2_LOCKFLAG_SHARED | SMB2_LOCKFLAG_FAIL_IMMEDIATELY: + ksmbd_debug(SMB, + "received shared & fail immediately request\n"); + cmd = F_SETLK; + flock->c.flc_type = F_RDLCK; + break; + case SMB2_LOCKFLAG_EXCLUSIVE | SMB2_LOCKFLAG_FAIL_IMMEDIATELY: + ksmbd_debug(SMB, + "received exclusive & fail immediately request\n"); + cmd = F_SETLK; + flock->c.flc_type = F_WRLCK; + break; + case SMB2_LOCKFLAG_UNLOCK: + ksmbd_debug(SMB, "received unlock request\n"); + flock->c.flc_type = F_UNLCK; + cmd = F_SETLK; + break; + } + + return cmd; +} + +static struct ksmbd_lock *smb2_lock_init(struct file_lock *flock, + unsigned int cmd, int flags, bool zero_len, + struct list_head *lock_list) +{ + struct ksmbd_lock *lock; + + lock = kzalloc_obj(struct ksmbd_lock, KSMBD_DEFAULT_GFP); + if (!lock) + return NULL; + + lock->cmd = cmd; + lock->fl = flock; + lock->start = flock->fl_start; + lock->end = flock->fl_end; + lock->flags = flags; + lock->zero_len = zero_len; + INIT_LIST_HEAD(&lock->clist); + INIT_LIST_HEAD(&lock->flist); + INIT_LIST_HEAD(&lock->llist); + list_add_tail(&lock->llist, lock_list); + + return lock; +} + +static void smb2_remove_blocked_lock(void **argv) +{ + struct file_lock *flock = (struct file_lock *)argv[0]; + + ksmbd_vfs_posix_lock_unblock(flock); + locks_wake_up(flock); +} + +static void smb2_free_lock(struct file_lock *flock) +{ + ksmbd_vfs_posix_lock_unblock(flock); + locks_free_lock(flock); +} + +static void smb2_free_blocked_lock(struct file_lock *flock) +{ + ksmbd_vfs_posix_lock_unblock(flock); + locks_wake_up(flock); + locks_free_lock(flock); +} + +static inline bool lock_defer_pending(struct file_lock *fl) +{ + /* check pending lock waiters */ + return waitqueue_active(&fl->c.flc_wait); +} + +/** + * smb2_lock() - handler for smb2 file lock command + * @work: smb work containing lock command buffer + * + * Return: 0 on success, otherwise error + */ +int smb2_lock(struct ksmbd_work *work) +{ + struct smb2_lock_req *req; + struct smb2_lock_rsp *rsp; + struct smb2_lock_element *lock_ele; + struct ksmbd_file *fp = NULL; + struct file_lock *flock = NULL; + struct file *filp = NULL; + int lock_count; + int flags = 0; + int cmd = 0; + int err = -EIO, i, rc = 0; + u64 lock_start, lock_length; + struct ksmbd_lock *smb_lock = NULL, *cmp_lock, *tmp, *tmp2; + struct ksmbd_conn *conn; + int nolock = 0; + LIST_HEAD(lock_list); + LIST_HEAD(rollback_list); + int prior_lock = 0, bkt; + unsigned int id = KSMBD_NO_FID, pid = KSMBD_NO_FID; + bool lock_replayed; + + WORK_BUFFERS(work, req, rsp); + + ksmbd_debug(SMB, "Received smb2 lock request\n"); + + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; + + if (work->next_smb2_rcv_hdr_off && + !has_file_id(req->VolatileFileId)) { + ksmbd_debug(SMB, "Compound request set FID = %llu\n", + work->compound_fid); + id = work->compound_fid; + pid = work->compound_pfid; + } + + if (!has_file_id(id)) { + id = req->VolatileFileId; + pid = req->PersistentFileId; + } + + fp = ksmbd_lookup_fd_slow(work, id, pid); + if (!fp) { + ksmbd_debug(SMB, "Invalid file id for lock : %llu\n", req->VolatileFileId); + err = -ENOENT; + goto out2; + } + + err = smb2_set_request_open(work, fp, &req->hdr, false, false); + if (err) + goto out2; + + lock_replayed = smb2_verify_lock_sequence(work, fp, req); + if (lock_replayed) + goto lock_success; + + filp = fp->filp; + lock_count = le16_to_cpu(req->LockCount); + lock_ele = req->locks; + + ksmbd_debug(SMB, "lock count is %d\n", lock_count); + /* + * Cap lock_count at 64. The MS-SMB2 spec defines Open.LockSequenceArray + * as exactly 64 entries so 64 is the intended ceiling. No real workload + * comes close to this in a single request. + */ + if (!lock_count || lock_count > 64) { + err = -EINVAL; + goto out2; + } + + for (i = 0; i < lock_count; i++) { + flags = le32_to_cpu(lock_ele[i].Flags); + + flock = smb_flock_init(filp); + if (!flock) + goto out; + + cmd = smb2_set_flock_flags(flock, flags); + + lock_start = le64_to_cpu(lock_ele[i].Offset); + lock_length = le64_to_cpu(lock_ele[i].Length); + if (lock_start > OFFSET_MAX || + (lock_length && + lock_length - 1 > OFFSET_MAX - lock_start)) { + pr_err("Invalid lock range requested\n"); + rsp->hdr.Status = STATUS_INVALID_LOCK_RANGE; + locks_free_lock(flock); + goto out; + } + + flock->fl_start = lock_start; + flock->fl_end = lock_length ? + flock->fl_start + lock_length - 1 : flock->fl_start; + + /* Check conflict locks in one request */ + list_for_each_entry(cmp_lock, &lock_list, llist) { + if (cmp_lock->fl->fl_start <= flock->fl_start && + cmp_lock->fl->fl_end >= flock->fl_end) { + if (cmp_lock->fl->c.flc_type != F_UNLCK && + flock->c.flc_type != F_UNLCK) { + pr_err("conflict two locks in one request\n"); + err = -EINVAL; + locks_free_lock(flock); + goto out; + } + } + } + + smb_lock = smb2_lock_init(flock, cmd, flags, !lock_length, + &lock_list); + if (!smb_lock) { + err = -EINVAL; + locks_free_lock(flock); + goto out; + } + } + + list_for_each_entry_safe(smb_lock, tmp, &lock_list, llist) { + if (lock_count > 1 && + !(le32_to_cpu(lock_ele[0].Flags) & SMB2_LOCKFLAG_UNLOCK) && + !(smb_lock->flags & SMB2_LOCKFLAG_FAIL_IMMEDIATELY)) { + err = -EINVAL; + goto out; + } + + if (smb_lock->cmd < 0) { + err = -EINVAL; + goto out; + } + + if (!(smb_lock->flags & SMB2_LOCKFLAG_MASK)) { + err = -EINVAL; + goto out; + } + + if ((prior_lock & (SMB2_LOCKFLAG_EXCLUSIVE | SMB2_LOCKFLAG_SHARED) && + smb_lock->flags & SMB2_LOCKFLAG_UNLOCK) || + (prior_lock == SMB2_LOCKFLAG_UNLOCK && + !(smb_lock->flags & SMB2_LOCKFLAG_UNLOCK))) { + err = -EINVAL; + goto out; + } + + prior_lock = smb_lock->flags; + + if (!(smb_lock->flags & SMB2_LOCKFLAG_UNLOCK) && + !(smb_lock->flags & SMB2_LOCKFLAG_FAIL_IMMEDIATELY)) + goto no_check_cl; + + nolock = 1; + /* check locks in connection list */ + down_read(&conn_list_lock); + hash_for_each(conn_list, bkt, conn, hlist) { + spin_lock(&conn->llist_lock); + list_for_each_entry_safe(cmp_lock, tmp2, &conn->lock_list, clist) { + if (file_inode(cmp_lock->fl->c.flc_file) != + file_inode(smb_lock->fl->c.flc_file)) + continue; + + if (lock_is_unlock(smb_lock->fl)) { + if (cmp_lock->fl->c.flc_file == smb_lock->fl->c.flc_file && + cmp_lock->start == smb_lock->start && + cmp_lock->end == smb_lock->end && + !lock_defer_pending(cmp_lock->fl)) { + nolock = 0; + list_del_init(&cmp_lock->flist); + list_del_init(&cmp_lock->clist); + cmp_lock->conn = NULL; + spin_unlock(&conn->llist_lock); + up_read(&conn_list_lock); + + ksmbd_conn_put(conn); + smb2_free_lock(cmp_lock->fl); + kfree(cmp_lock); + goto out_check_cl; + } + continue; + } + + if (cmp_lock->fl->c.flc_file == smb_lock->fl->c.flc_file) { + if (smb_lock->flags & SMB2_LOCKFLAG_SHARED) + continue; + } else { + if (cmp_lock->flags & SMB2_LOCKFLAG_SHARED) + continue; + } + + /* check zero byte lock range */ + if (cmp_lock->zero_len && !smb_lock->zero_len && + cmp_lock->start > smb_lock->start && + cmp_lock->start <= smb_lock->end) { + spin_unlock(&conn->llist_lock); + up_read(&conn_list_lock); + pr_err("previous lock conflict with zero byte lock range\n"); + goto out; + } + + if (smb_lock->zero_len && !cmp_lock->zero_len && + smb_lock->start > cmp_lock->start && + smb_lock->start <= cmp_lock->end) { + spin_unlock(&conn->llist_lock); + up_read(&conn_list_lock); + pr_err("current lock conflict with zero byte lock range\n"); + goto out; + } + + if (cmp_lock->start <= smb_lock->end && + smb_lock->start <= cmp_lock->end && + !cmp_lock->zero_len && !smb_lock->zero_len) { + spin_unlock(&conn->llist_lock); + up_read(&conn_list_lock); + pr_err("Not allow lock operation on exclusive lock range\n"); + goto out; + } + } + spin_unlock(&conn->llist_lock); + } + up_read(&conn_list_lock); +out_check_cl: + if (lock_is_unlock(smb_lock->fl) && nolock) { + pr_err("Try to unlock nolocked range\n"); + rsp->hdr.Status = STATUS_RANGE_NOT_LOCKED; + goto out; + } + +no_check_cl: + flock = smb_lock->fl; + list_del(&smb_lock->llist); + + if (smb_lock->zero_len) { + err = 0; + goto skip; + } +retry: + rc = vfs_lock_file(filp, smb_lock->cmd, flock, NULL); +skip: + if (smb_lock->flags & SMB2_LOCKFLAG_UNLOCK) { + locks_free_lock(flock); + kfree(smb_lock); + if (!rc) { + ksmbd_debug(SMB, "File unlocked\n"); + } else if (rc == -ENOENT) { + rsp->hdr.Status = STATUS_NOT_LOCKED; + err = rc; + goto out; + } + } else { + if (rc == FILE_LOCK_DEFERRED) { + void **argv; + + ksmbd_debug(SMB, + "would have to wait for getting lock\n"); + + argv = kmalloc(sizeof(void *), KSMBD_DEFAULT_GFP); + if (!argv) { + err = -ENOMEM; + smb2_free_blocked_lock(flock); + kfree(smb_lock); + goto out; + } + argv[0] = flock; + + rc = setup_async_work(work, + smb2_remove_blocked_lock, + argv); + if (rc) { + kfree(argv); + err = -ENOMEM; + smb2_free_blocked_lock(flock); + kfree(smb_lock); + goto out; + } + list_add(&smb_lock->llist, &rollback_list); + spin_lock(&fp->f_lock); + list_add(&work->fp_entry, &fp->blocked_works); + spin_unlock(&fp->f_lock); + + smb2_send_interim_resp(work, STATUS_PENDING); + + ksmbd_vfs_posix_lock_wait(flock); + + spin_lock(&fp->f_lock); + list_del(&work->fp_entry); + spin_unlock(&fp->f_lock); + + list_del(&smb_lock->llist); + + if (work->state == KSMBD_WORK_CANCELLED) { + rsp->hdr.Status = STATUS_CANCELLED; + kfree(smb_lock); + smb2_send_interim_resp(work, + STATUS_CANCELLED); + release_async_work(work); + locks_free_lock(flock); + work->send_no_response = 1; + goto out; + } + + release_async_work(work); + + if (work->state == KSMBD_WORK_ACTIVE) + goto retry; + + locks_free_lock(flock); + + rsp->hdr.Status = + STATUS_RANGE_NOT_LOCKED; + kfree(smb_lock); + /* rollback_list may still hold earlier grants */ + goto out; + } else if (!rc) { + list_add(&smb_lock->llist, &rollback_list); + ksmbd_debug(SMB, "successful in taking lock\n"); + } else { + locks_free_lock(flock); + kfree(smb_lock); + err = rc; + goto out; + } + } + } + + if (atomic_read(&fp->f_ci->op_count) > 1) + smb_break_all_oplock(work, fp); + +lock_success: + rsp->StructureSize = cpu_to_le16(4); + ksmbd_debug(SMB, "successful in taking lock\n"); + rsp->hdr.Status = STATUS_SUCCESS; + rsp->Reserved = 0; + err = ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_lock_rsp)); + if (err) + goto out; + + /* publish only once the whole batch has committed */ + if (!list_empty(&rollback_list)) { + spin_lock(&work->conn->llist_lock); + list_for_each_entry_safe(smb_lock, tmp, &rollback_list, llist) { + list_del_init(&smb_lock->llist); + smb_lock->conn = ksmbd_conn_get(work->conn); + list_add_tail(&smb_lock->clist, + &work->conn->lock_list); + list_add_tail(&smb_lock->flist, + &fp->lock_list); + } + spin_unlock(&work->conn->llist_lock); + } + + if (!lock_replayed) + smb2_update_lock_sequence(work, fp, req); + + ksmbd_fd_put(work, fp); + return 0; + +out: + list_for_each_entry_safe(smb_lock, tmp, &lock_list, llist) { + locks_free_lock(smb_lock->fl); + list_del(&smb_lock->llist); + kfree(smb_lock); + } + + list_for_each_entry_safe(smb_lock, tmp, &rollback_list, llist) { + struct file_lock *rlock = NULL; + + rlock = smb_flock_init(filp); + if (rlock) { + rlock->c.flc_type = F_UNLCK; + rlock->fl_start = smb_lock->start; + rlock->fl_end = smb_lock->end; + + rc = vfs_lock_file(filp, F_SETLK, rlock, NULL); + if (rc) + pr_err("rollback unlock fail : %d\n", rc); + } else { + pr_err("rollback unlock alloc failed\n"); + } + + list_del(&smb_lock->llist); + smb2_free_lock(smb_lock->fl); + if (rlock) + locks_free_lock(rlock); + kfree(smb_lock); + } +out2: + ksmbd_debug(SMB, "failed in taking lock(flags : %x), err : %d\n", flags, err); + + if (!rsp->hdr.Status) { + if (err == -EINVAL) + rsp->hdr.Status = STATUS_INVALID_PARAMETER; + else if (err == -ENOMEM) + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + else if (err == -ENOENT) + rsp->hdr.Status = STATUS_FILE_CLOSED; + else + rsp->hdr.Status = STATUS_LOCK_NOT_GRANTED; + } + + smb2_set_err_rsp(work); + ksmbd_fd_put(work, fp); + return err; +} + +static int fsctl_copychunk(struct ksmbd_work *work, + struct copychunk_ioctl_req *ci_req, + unsigned int cnt_code, + unsigned int input_count, + unsigned long long volatile_id, + unsigned long long persistent_id, + struct smb2_ioctl_rsp *rsp) +{ + struct copychunk_ioctl_rsp *ci_rsp; + struct ksmbd_file *src_fp = NULL, *dst_fp = NULL; + struct srv_copychunk *chunks; + unsigned int i, chunk_count, chunk_count_written = 0; + unsigned int chunk_size_written = 0; + loff_t total_size_written = 0; + int ret = 0; + + ci_rsp = (struct copychunk_ioctl_rsp *)&rsp->Buffer[0]; + + rsp->VolatileFileId = volatile_id; + rsp->PersistentFileId = persistent_id; + ci_rsp->ChunksWritten = + cpu_to_le32(ksmbd_server_side_copy_max_chunk_count()); + ci_rsp->ChunkBytesWritten = + cpu_to_le32(ksmbd_server_side_copy_max_chunk_size()); + ci_rsp->TotalBytesWritten = + cpu_to_le32(ksmbd_server_side_copy_max_total_size()); + + chunk_count = le32_to_cpu(ci_req->ChunkCount); + /* + * ChunkCount=0 is the standard SMB2 "query my copy limits" request + * (no data copied) -- but macOS Finder's Cmd+D duplicate sends + * FSCTL_SRV_COPYCHUNK with ChunkCount=0 meaning "copy the whole + * file", relying on the AAPL-negotiated server to do a full copy + * instead. Keep the standard no-op behavior for everyone else. + * + * Gate on the TIME_MACHINE share flag, not just conn->is_aapl: + * that flag alone has ambiguous provenance -- the pre-existing + * narrow UniqueId=0 path can also set it on ordinary, + * non-Time-Machine shares, and this series' stated design keeps + * every AAPL-driven behavior opt-in per share. + */ + if (chunk_count == 0 && + !(work->conn->is_aapl && + test_share_config_flag(work->tcon->share_conf, + KSMBD_SHARE_FLAG_TIME_MACHINE))) + goto out; + total_size_written = 0; + i = 0; + + if (chunk_count) { + /* verify the SRV_COPYCHUNK_COPY packet */ + if (chunk_count > ksmbd_server_side_copy_max_chunk_count() || + input_count < struct_size(ci_req, Chunks, chunk_count)) { + rsp->hdr.Status = STATUS_INVALID_PARAMETER; + return -EINVAL; + } + + chunks = &ci_req->Chunks[0]; + for (i = 0; i < chunk_count; i++) { + if (le32_to_cpu(chunks[i].Length) == 0 || + le32_to_cpu(chunks[i].Length) > ksmbd_server_side_copy_max_chunk_size()) + break; + total_size_written += le32_to_cpu(chunks[i].Length); + } + } else { + chunks = &ci_req->Chunks[0]; + } + + if (i < chunk_count || + total_size_written > ksmbd_server_side_copy_max_total_size()) { + rsp->hdr.Status = STATUS_INVALID_PARAMETER; + return -EINVAL; + } + + src_fp = ksmbd_lookup_foreign_fd(work, + le64_to_cpu(ci_req->SourceKeyU64[0])); + dst_fp = ksmbd_lookup_fd_slow(work, volatile_id, persistent_id); + ret = -EINVAL; + if (!src_fp || + src_fp->persistent_id != le64_to_cpu(ci_req->SourceKeyU64[1])) { + rsp->hdr.Status = STATUS_OBJECT_NAME_NOT_FOUND; + goto out; + } + + if (!dst_fp) { + rsp->hdr.Status = STATUS_FILE_CLOSED; + goto out; + } + + /* + * FILE_READ_DATA should only be included in + * the FSCTL_SRV_COPYCHUNK case + */ + if (cnt_code == FSCTL_SRV_COPYCHUNK && + !(dst_fp->daccess & (FILE_READ_DATA_LE | FILE_GENERIC_READ_LE))) { + rsp->hdr.Status = STATUS_ACCESS_DENIED; + goto out; + } + + ret = ksmbd_vfs_copy_file_ranges(work, src_fp, dst_fp, + chunks, chunk_count, + &chunk_count_written, + &chunk_size_written, + &total_size_written); + if (ret < 0) { + if (ret == -EACCES) + rsp->hdr.Status = STATUS_ACCESS_DENIED; + else if (ret == -EAGAIN) + rsp->hdr.Status = STATUS_FILE_LOCK_CONFLICT; + else if (ret == -EBADF) + rsp->hdr.Status = STATUS_INVALID_HANDLE; + else if (ret == -EFBIG || ret == -ENOSPC) + rsp->hdr.Status = STATUS_DISK_FULL; + else if (ret == -EINVAL) + rsp->hdr.Status = STATUS_INVALID_PARAMETER; + else if (ret == -EISDIR) + rsp->hdr.Status = STATUS_FILE_IS_A_DIRECTORY; + else if (ret == -E2BIG) + rsp->hdr.Status = STATUS_INVALID_VIEW_SIZE; + else + rsp->hdr.Status = STATUS_UNEXPECTED_IO_ERROR; + } + + ci_rsp->ChunksWritten = cpu_to_le32(chunk_count_written); + ci_rsp->ChunkBytesWritten = cpu_to_le32(chunk_size_written); + ci_rsp->TotalBytesWritten = cpu_to_le32(total_size_written); +out: + ksmbd_fd_put(work, src_fp); + ksmbd_fd_put(work, dst_fp); + return ret; +} + +static __be32 idev_ipv4_address(struct in_device *idev) +{ + __be32 addr = 0; + + struct in_ifaddr *ifa; + + rcu_read_lock(); + in_dev_for_each_ifa_rcu(ifa, idev) { + if (ifa->ifa_flags & IFA_F_SECONDARY) + continue; + + addr = ifa->ifa_address; + break; + } + rcu_read_unlock(); + return addr; +} + +static int fsctl_query_iface_info_ioctl(struct ksmbd_conn *conn, + struct smb2_ioctl_rsp *rsp, + unsigned int out_buf_len) +{ + struct network_interface_info_ioctl_rsp *nii_rsp = NULL; + int nbytes = 0; + struct net_device *netdev; + struct sockaddr_storage_rsp *sockaddr_storage; + unsigned int flags; + unsigned long long speed; + + rtnl_lock(); + for_each_netdev(&init_net, netdev) { + bool ipv4_set = false; + + if (netdev->type == ARPHRD_LOOPBACK) + continue; + + if (!ksmbd_find_netdev_name_iface_list(netdev->name)) + continue; + + flags = netif_get_flags(netdev); + if (!(flags & IFF_RUNNING)) + continue; +ipv6_retry: + if (out_buf_len < + nbytes + sizeof(struct network_interface_info_ioctl_rsp)) { + rtnl_unlock(); + return -ENOSPC; + } + + nii_rsp = (struct network_interface_info_ioctl_rsp *) + &rsp->Buffer[nbytes]; + nii_rsp->IfIndex = cpu_to_le32(netdev->ifindex); + + nii_rsp->Capability = 0; + if (netdev->real_num_tx_queues > 1) + nii_rsp->Capability |= RSS_CAPABLE; + if (ksmbd_rdma_capable_netdev(netdev)) + nii_rsp->Capability |= RDMA_CAPABLE; + + nii_rsp->Next = cpu_to_le32(152); + nii_rsp->Reserved = 0; + + if (netdev->ethtool_ops->get_link_ksettings) { + struct ethtool_link_ksettings cmd; + + netdev->ethtool_ops->get_link_ksettings(netdev, &cmd); + speed = cmd.base.speed; + } else { + ksmbd_debug(SMB, "%s %s\n", netdev->name, + "speed is unknown, defaulting to 1Gb/sec"); + speed = SPEED_1000; + } + + speed *= 1000000; + nii_rsp->LinkSpeed = cpu_to_le64(speed); + + sockaddr_storage = (struct sockaddr_storage_rsp *) + nii_rsp->SockAddr_Storage; + memset(sockaddr_storage, 0, 128); + + if (!ipv4_set) { + struct in_device *idev; + + sockaddr_storage->Family = INTERNETWORK; + sockaddr_storage->addr4.Port = 0; + + idev = __in_dev_get_rtnl(netdev); + if (!idev) + continue; + sockaddr_storage->addr4.IPv4Address = + idev_ipv4_address(idev); + nbytes += sizeof(struct network_interface_info_ioctl_rsp); + ipv4_set = true; + goto ipv6_retry; + } else { + struct inet6_dev *idev6; + struct inet6_ifaddr *ifa; + __u8 *ipv6_addr = sockaddr_storage->addr6.IPv6Address; + + sockaddr_storage->Family = INTERNETWORKV6; + sockaddr_storage->addr6.Port = 0; + sockaddr_storage->addr6.FlowInfo = 0; + + idev6 = __in6_dev_get(netdev); + if (!idev6) + continue; + + list_for_each_entry(ifa, &idev6->addr_list, if_list) { + if (ifa->flags & (IFA_F_TENTATIVE | + IFA_F_DEPRECATED)) + continue; + memcpy(ipv6_addr, ifa->addr.s6_addr, 16); + break; + } + sockaddr_storage->addr6.ScopeId = 0; + nbytes += sizeof(struct network_interface_info_ioctl_rsp); + } + } + rtnl_unlock(); + + /* zero if this is last one */ + if (nii_rsp) + nii_rsp->Next = 0; + + rsp->PersistentFileId = SMB2_NO_FID; + rsp->VolatileFileId = SMB2_NO_FID; + return nbytes; +} + +static int fsctl_validate_negotiate_info(struct ksmbd_conn *conn, + struct validate_negotiate_info_req *neg_req, + struct validate_negotiate_info_rsp *neg_rsp, + unsigned int in_buf_len) +{ + int ret = 0; + int dialect; + + if (in_buf_len < offsetof(struct validate_negotiate_info_req, Dialects) + + le16_to_cpu(neg_req->DialectCount) * sizeof(__le16)) + return -EINVAL; + + dialect = ksmbd_lookup_dialect_by_id(neg_req->Dialects, + neg_req->DialectCount); + if (dialect == BAD_PROT_ID || dialect != conn->dialect) { + ret = -EINVAL; + goto err_out; + } + + if (memcmp(neg_req->Guid, conn->ClientGUID, SMB2_CLIENT_GUID_SIZE)) { + ret = -EINVAL; + goto err_out; + } + + if (le16_to_cpu(neg_req->SecurityMode) != conn->cli_sec_mode) { + ret = -EINVAL; + goto err_out; + } + + if (le32_to_cpu(neg_req->Capabilities) != conn->cli_cap) { + ret = -EINVAL; + goto err_out; + } + + neg_rsp->Capabilities = cpu_to_le32(conn->vals->req_capabilities); + memset(neg_rsp->Guid, 0, SMB2_CLIENT_GUID_SIZE); + neg_rsp->SecurityMode = cpu_to_le16(conn->srv_sec_mode); + neg_rsp->Dialect = cpu_to_le16(conn->dialect); +err_out: + return ret; +} + +static int fsctl_query_allocated_ranges(struct ksmbd_work *work, u64 id, + struct file_allocated_range_buffer *qar_req, + struct file_allocated_range_buffer *qar_rsp, + unsigned int in_count, unsigned int *out_count) +{ + struct ksmbd_file *fp; + loff_t start, length; + int ret = 0; + + *out_count = 0; + + start = le64_to_cpu(qar_req->file_offset); + length = le64_to_cpu(qar_req->length); + + if (start < 0 || length < 0) + return -EINVAL; + + fp = ksmbd_lookup_fd_fast(work, id); + if (!fp) + return -ENOENT; + + if (!(fp->daccess & FILE_READ_DATA_LE)) { + ret = -EACCES; + goto out; + } + + if (!in_count) { + struct file_allocated_range_buffer range; + + ret = ksmbd_vfs_query_allocated_ranges(fp, start, length, + &range, 1, out_count); + if ((!ret || ret == -E2BIG) && *out_count) + ret = -ENOSPC; + *out_count = 0; + } else { + ret = ksmbd_vfs_query_allocated_ranges(fp, start, length, + qar_rsp, in_count, + out_count); + } + if (ret && ret != -E2BIG) + *out_count = 0; + +out: + ksmbd_fd_put(work, fp); + return ret; +} + +static int fsctl_pipe_transceive(struct ksmbd_work *work, u64 id, + unsigned int out_buf_len, + struct smb2_ioctl_req *req, + struct smb2_ioctl_rsp *rsp) +{ + struct ksmbd_rpc_command *rpc_resp; + char *data_buf = (char *)req + le32_to_cpu(req->InputOffset); + int nbytes = 0; + + rpc_resp = ksmbd_rpc_ioctl(work->sess, id, data_buf, + le32_to_cpu(req->InputCount)); + if (rpc_resp) { + if (rpc_resp->flags == KSMBD_RPC_SOME_NOT_MAPPED) { + /* + * set STATUS_SOME_NOT_MAPPED response + * for unknown domain sid. + */ + rsp->hdr.Status = STATUS_SOME_NOT_MAPPED; + } else if (rpc_resp->flags == KSMBD_RPC_ENOTIMPLEMENTED) { + rsp->hdr.Status = STATUS_NOT_SUPPORTED; + goto out; + } else if (rpc_resp->flags != KSMBD_RPC_OK) { + rsp->hdr.Status = STATUS_INVALID_PARAMETER; + goto out; + } + + nbytes = rpc_resp->payload_sz; + if (rpc_resp->payload_sz > out_buf_len) { + rsp->hdr.Status = STATUS_BUFFER_OVERFLOW; + nbytes = out_buf_len; + } + + if (!rpc_resp->payload_sz) { + rsp->hdr.Status = + STATUS_UNEXPECTED_IO_ERROR; + goto out; + } + + memcpy((char *)rsp->Buffer, rpc_resp->payload, nbytes); + } +out: + kvfree(rpc_resp); + return nbytes; +} + +static inline int fsctl_set_sparse(struct ksmbd_work *work, u64 id, + struct file_sparse *sparse) +{ + struct ksmbd_file *fp; + struct mnt_idmap *idmap; + int ret = 0; + __le32 old_fattr; + + if (!test_tree_conn_flag(work->tcon, KSMBD_TREE_CONN_FLAG_WRITABLE)) { + ksmbd_debug(SMB, "User does not have write permission\n"); + return -EACCES; + } + + fp = ksmbd_lookup_fd_fast(work, id); + if (!fp) + return -ENOENT; + + if (S_ISDIR(file_inode(fp->filp)->i_mode)) { + ret = -EINVAL; + goto out; + } + + if (!(fp->daccess & (FILE_WRITE_DATA_LE | FILE_APPEND_DATA_LE | + FILE_WRITE_ATTRIBUTES_LE))) { + ret = -EACCES; + goto out; + } + + idmap = file_mnt_idmap(fp->filp); + + old_fattr = fp->f_ci->m_fattr; + if (!sparse->SetSparse && + (old_fattr & FILE_ATTRIBUTE_SPARSE_FILE_LE)) { + ret = ksmbd_vfs_zero_holes(fp); + if (ret) + goto out; + } + + if (sparse->SetSparse) + fp->f_ci->m_fattr |= FILE_ATTRIBUTE_SPARSE_FILE_LE; + else + fp->f_ci->m_fattr &= ~FILE_ATTRIBUTE_SPARSE_FILE_LE; + + if (fp->f_ci->m_fattr != old_fattr) { + const struct cred *saved_cred; + struct xattr_dos_attrib da = {0}; + + ret = ksmbd_vfs_get_dos_attrib_xattr(idmap, + fp->filp->f_path.dentry, &da); + if (ret <= 0) { + da.version = 4; + da.itime = fp->itime; + da.create_time = fp->create_time; + da.flags = XATTR_DOSINFO_CREATE_TIME | + XATTR_DOSINFO_ITIME; + } + + da.attr = le32_to_cpu(fp->f_ci->m_fattr); + da.flags |= XATTR_DOSINFO_ATTRIB; + saved_cred = override_creds(fp->filp->f_cred); + ret = ksmbd_vfs_set_dos_attrib_xattr(idmap, + &fp->filp->f_path, + &da, true); + revert_creds(saved_cred); + if (ret) + fp->f_ci->m_fattr = old_fattr; + } + +out: + ksmbd_fd_put(work, fp); + return ret; +} + +static int fsctl_request_resume_key(struct ksmbd_work *work, + struct smb2_ioctl_req *req, + struct resume_key_ioctl_rsp *key_rsp) +{ + struct ksmbd_file *fp; + + fp = ksmbd_lookup_fd_slow(work, req->VolatileFileId, req->PersistentFileId); + if (!fp) + return -ENOENT; + + memset(key_rsp, 0, sizeof(*key_rsp)); + key_rsp->ResumeKeyU64[0] = req->VolatileFileId; + key_rsp->ResumeKeyU64[1] = req->PersistentFileId; + ksmbd_fd_put(work, fp); + + return 0; +} + +/** + * smb2_ioctl() - handler for smb2 ioctl command + * @work: smb work containing ioctl command buffer + * + * Return: 0 on success, otherwise error + */ +int smb2_ioctl(struct ksmbd_work *work) +{ + struct smb2_ioctl_req *req; + struct smb2_ioctl_rsp *rsp; + unsigned int cnt_code, nbytes = 0, out_buf_len, in_buf_len; + u64 id = KSMBD_NO_FID, pid = KSMBD_NO_FID; + struct ksmbd_conn *conn = work->conn; + int ret = 0; + char *buffer; + bool no_fileid_ioctl = false; + bool chseq_err = false; + + ksmbd_debug(SMB, "Received smb2 ioctl request\n"); + + if (work->next_smb2_rcv_hdr_off) { + req = ksmbd_req_buf_next(work); + rsp = ksmbd_resp_buf_next(work); + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; + if (!has_file_id(req->VolatileFileId)) { + ksmbd_debug(SMB, "Compound request set FID = %llu\n", + work->compound_fid); + id = work->compound_fid; + pid = work->compound_pfid; + } + } else { + req = smb_get_msg(work->request_buf); + rsp = smb_get_msg(work->response_buf); + } + + if (!has_file_id(id)) { + id = req->VolatileFileId; + pid = req->PersistentFileId; + } + + if (req->Flags != cpu_to_le32(SMB2_0_IOCTL_IS_FSCTL)) { + ret = -EOPNOTSUPP; + goto out; + } + + buffer = (char *)req + le32_to_cpu(req->InputOffset); + + cnt_code = le32_to_cpu(req->CtlCode); + switch (cnt_code) { + case FSCTL_DFS_GET_REFERRALS: + case FSCTL_DFS_GET_REFERRALS_EX: + case FSCTL_QUERY_NETWORK_INTERFACE_INFO: + case FSCTL_VALIDATE_NEGOTIATE_INFO: + case FSCTL_PIPE_WAIT: + case FSCTL_PIPE_TRANSCEIVE: + no_fileid_ioctl = true; + break; + default: + break; + } + + if (!no_fileid_ioctl && has_file_id(id)) { + struct ksmbd_file *fp; + + fp = ksmbd_lookup_fd_slow(work, id, pid); + if (!fp) { + if (cnt_code == FSCTL_DUPLICATE_EXTENTS_TO_FILE) { + rsp->hdr.Status = STATUS_FILE_CLOSED; + goto out2; + } + ret = -ENOENT; + goto out; + } + + ret = smb2_set_request_open(work, fp, &req->hdr, true, false); + ksmbd_fd_put(work, fp); + if (ret) { + rsp->hdr.Status = STATUS_FILE_NOT_AVAILABLE; + chseq_err = true; + goto out; + } + } + + ret = smb2_calc_max_out_buf_len(work, + offsetof(struct smb2_ioctl_rsp, Buffer), + le32_to_cpu(req->MaxOutputResponse)); + if (ret < 0) { + rsp->hdr.Status = STATUS_INVALID_PARAMETER; + goto out; + } + out_buf_len = (unsigned int)ret; + in_buf_len = le32_to_cpu(req->InputCount); + + switch (cnt_code) { + case FSCTL_SRV_ENUM_SNAPS: { + struct srv_snapshot_array *snap_rsp; + struct ksmbd_file *fp; + + if (out_buf_len < sizeof(*snap_rsp)) { + ret = -EINVAL; + goto out; + } + + fp = ksmbd_lookup_fd_fast(work, id); + if (!fp) { + ret = -ENOENT; + goto out; + } + ksmbd_fd_put(work, fp); + + snap_rsp = (struct srv_snapshot_array *)rsp->Buffer; + snap_rsp->NumberOfSnapShots = 0; + snap_rsp->NumberOfSnapShotsReturned = 0; + snap_rsp->SnapShotArraySize = cpu_to_le32(2); + snap_rsp->Reserved = 0; + nbytes = sizeof(*snap_rsp); + break; + } + case FSCTL_DFS_GET_REFERRALS: + case FSCTL_DFS_GET_REFERRALS_EX: + /* Not support DFS yet */ + ret = -EOPNOTSUPP; + rsp->hdr.Status = STATUS_FS_DRIVER_REQUIRED; + goto out2; + case FSCTL_GET_COMPRESSION: { + struct compress_ioctl *cmpr_rsp; + struct ksmbd_file *fp; + u16 fmt; + + if (out_buf_len < sizeof(struct compress_ioctl)) { + ret = -EINVAL; + goto out; + } + + fp = ksmbd_lookup_fd_fast(work, id); + if (!fp) { + ret = -ENOENT; + goto out; + } + + ret = ksmbd_vfs_get_compression(fp, &fmt); + ksmbd_fd_put(work, fp); + if (ret < 0) + goto out; + + cmpr_rsp = (struct compress_ioctl *)&rsp->Buffer[0]; + cmpr_rsp->CompressionState = cpu_to_le16(fmt); + nbytes = sizeof(struct compress_ioctl); + rsp->PersistentFileId = req->PersistentFileId; + rsp->VolatileFileId = req->VolatileFileId; + break; + } + case FSCTL_SET_COMPRESSION: { + struct compress_ioctl *cmpr_req; + struct ksmbd_file *fp; + + if (in_buf_len < sizeof(struct compress_ioctl)) { + ret = -EINVAL; + goto out; + } + + if (!test_tree_conn_flag(work->tcon, KSMBD_TREE_CONN_FLAG_WRITABLE)) { + ksmbd_debug(SMB, "User does not have write permission\n"); + ret = -EACCES; + goto out; + } + + cmpr_req = (struct compress_ioctl *)buffer; + fp = ksmbd_lookup_fd_fast(work, id); + if (!fp) { + ret = -ENOENT; + goto out; + } + + ret = ksmbd_vfs_set_compression(work, fp, le16_to_cpu(cmpr_req->CompressionState)); + ksmbd_fd_put(work, fp); + if (ret) + goto out; + break; + } + case FSCTL_CREATE_OR_GET_OBJECT_ID: + { + struct file_object_buf_type1_ioctl_rsp *obj_buf; + struct ksmbd_file *fp; + + fp = ksmbd_lookup_fd_fast(work, id); + if (!fp) { + ret = -EBADF; + rsp->hdr.Status = STATUS_FILE_CLOSED; + goto out2; + } + + if (out_buf_len < sizeof(struct file_object_buf_type1_ioctl_rsp)) { + ksmbd_fd_put(work, fp); + ret = -EINVAL; + goto out; + } + ksmbd_fd_put(work, fp); + + nbytes = sizeof(struct file_object_buf_type1_ioctl_rsp); + obj_buf = (struct file_object_buf_type1_ioctl_rsp *) + &rsp->Buffer[0]; + + /* + * TODO: This is dummy implementation to pass smbtorture + * Need to check correct response later + */ + memset(obj_buf->ObjectId, 0x0, 16); + memset(obj_buf->BirthVolumeId, 0x0, 16); + memset(obj_buf->BirthObjectId, 0x0, 16); + memset(obj_buf->DomainId, 0x0, 16); + + break; + } + case FSCTL_PIPE_TRANSCEIVE: + out_buf_len = min_t(u32, KSMBD_IPC_MAX_PAYLOAD, out_buf_len); + nbytes = fsctl_pipe_transceive(work, id, out_buf_len, req, rsp); + break; + case FSCTL_VALIDATE_NEGOTIATE_INFO: + if (conn->dialect < SMB30_PROT_ID) { + ret = -EOPNOTSUPP; + goto out; + } + + if (in_buf_len < offsetof(struct validate_negotiate_info_req, + Dialects)) { + ret = -EINVAL; + goto out; + } + + if (out_buf_len < sizeof(struct validate_negotiate_info_rsp)) { + ret = -EINVAL; + goto out; + } + + ret = fsctl_validate_negotiate_info(conn, + (struct validate_negotiate_info_req *)buffer, + (struct validate_negotiate_info_rsp *)&rsp->Buffer[0], + in_buf_len); + if (ret < 0) + goto out; + + nbytes = sizeof(struct validate_negotiate_info_rsp); + rsp->PersistentFileId = SMB2_NO_FID; + rsp->VolatileFileId = SMB2_NO_FID; + break; + case FSCTL_QUERY_NETWORK_INTERFACE_INFO: + if (req->PersistentFileId != SMB2_NO_FID || + req->VolatileFileId != SMB2_NO_FID) { + ret = -EINVAL; + goto out; + } + + ret = fsctl_query_iface_info_ioctl(conn, rsp, out_buf_len); + if (ret < 0) + goto out; + nbytes = ret; + break; + case FSCTL_SRV_REQUEST_RESUME_KEY: + if (out_buf_len < sizeof(struct resume_key_ioctl_rsp)) { + ret = -EINVAL; + goto out; + } + + ret = fsctl_request_resume_key(work, req, + (struct resume_key_ioctl_rsp *)&rsp->Buffer[0]); + if (ret < 0) + goto out; + rsp->PersistentFileId = req->PersistentFileId; + rsp->VolatileFileId = req->VolatileFileId; + nbytes = sizeof(struct resume_key_ioctl_rsp); + break; + case FSCTL_SRV_COPYCHUNK: + case FSCTL_SRV_COPYCHUNK_WRITE: + if (!test_tree_conn_flag(work->tcon, KSMBD_TREE_CONN_FLAG_WRITABLE)) { + ksmbd_debug(SMB, + "User does not have write permission\n"); + ret = -EACCES; + goto out; + } + + if (in_buf_len < offsetof(struct copychunk_ioctl_req, Chunks)) { + ret = -EINVAL; + goto out; + } + + if (out_buf_len < sizeof(struct copychunk_ioctl_rsp)) { + ret = -EINVAL; + goto out; + } + + nbytes = sizeof(struct copychunk_ioctl_rsp); + rsp->VolatileFileId = req->VolatileFileId; + rsp->PersistentFileId = req->PersistentFileId; + fsctl_copychunk(work, + (struct copychunk_ioctl_req *)buffer, + le32_to_cpu(req->CtlCode), + le32_to_cpu(req->InputCount), + req->VolatileFileId, + req->PersistentFileId, + rsp); + break; + case FSCTL_SET_SPARSE: + { + struct file_sparse sparse = {0}; + + if (in_buf_len && in_buf_len < sizeof(struct file_sparse)) { + ret = -EINVAL; + goto out; + } + + *(u8 *)&sparse = 1; + ret = fsctl_set_sparse(work, id, in_buf_len ? + (struct file_sparse *)buffer : &sparse); + if (ret < 0) + goto out; + break; + } + case FSCTL_SET_ZERO_DATA: + { + struct file_zero_data_information *zero_data; + struct ksmbd_file *fp; + loff_t off, len, bfz; + + if (!test_tree_conn_flag(work->tcon, KSMBD_TREE_CONN_FLAG_WRITABLE)) { + ksmbd_debug(SMB, + "User does not have write permission\n"); + ret = -EACCES; + goto out; + } + + if (in_buf_len < sizeof(struct file_zero_data_information)) { + ret = -EINVAL; + goto out; + } + + zero_data = + (struct file_zero_data_information *)buffer; + + off = le64_to_cpu(zero_data->FileOffset); + bfz = le64_to_cpu(zero_data->BeyondFinalZero); + if (off < 0 || bfz < 0 || off > bfz) { + ret = -EINVAL; + goto out; + } + + len = bfz - off; + if (len) { + fp = ksmbd_lookup_fd_fast(work, id); + if (!fp) { + ret = -ENOENT; + goto out; + } + + if (!(fp->daccess & FILE_WRITE_DATA_LE)) { + ksmbd_fd_put(work, fp); + ret = -EACCES; + goto out; + } + + ret = ksmbd_vfs_zero_data(work, fp, off, len); + ksmbd_fd_put(work, fp); + if (ret == -EAGAIN) { + rsp->hdr.Status = STATUS_FILE_LOCK_CONFLICT; + ret = 0; + goto out; + } else if (ret < 0) { + goto out; + } + } + break; + } + case FSCTL_FILE_LEVEL_TRIM: + { + struct file_level_trim *trim_req; + struct file_level_trim_output *trim_rsp; + struct ksmbd_file *fp; + u32 i, num_ranges; + + if (!test_tree_conn_flag(work->tcon, KSMBD_TREE_CONN_FLAG_WRITABLE)) { + ksmbd_debug(SMB, + "User does not have write permission\n"); + ret = -EACCES; + goto out; + } + + if (in_buf_len < offsetof(struct file_level_trim, Ranges)) { + ret = -EINVAL; + goto out; + } + + if (out_buf_len < sizeof(struct file_level_trim_output)) { + ret = -EINVAL; + goto out; + } + + trim_req = (struct file_level_trim *)buffer; + num_ranges = le32_to_cpu(trim_req->NumRanges); + if (num_ranges > + (in_buf_len - offsetof(struct file_level_trim, Ranges)) / + sizeof(struct file_level_trim_range)) { + ret = -EINVAL; + goto out; + } + + fp = ksmbd_lookup_fd_fast(work, id); + if (!fp) { + ret = -ENOENT; + goto out; + } + + if (!(fp->daccess & FILE_WRITE_DATA_LE)) { + ksmbd_fd_put(work, fp); + ret = -EACCES; + goto out; + } + + trim_rsp = (struct file_level_trim_output *)&rsp->Buffer[0]; + trim_rsp->NumRangesProcessed = 0; + for (i = 0; i < num_ranges; i++) { + loff_t off = le64_to_cpu(trim_req->Ranges[i].Offset); + loff_t len = le64_to_cpu(trim_req->Ranges[i].Length); + + if (off < 0 || len < 0) { + ret = -EINVAL; + break; + } + + if (!len) { + trim_rsp->NumRangesProcessed = + cpu_to_le32(i + 1); + continue; + } + + ret = ksmbd_vfs_trim_data(work, fp, off, len); + if (ret) + break; + trim_rsp->NumRangesProcessed = cpu_to_le32(i + 1); + } + ksmbd_fd_put(work, fp); + if (ret == -EAGAIN) { + rsp->hdr.Status = STATUS_FILE_LOCK_CONFLICT; + ret = 0; + goto out; + } else if (ret < 0) { + goto out; + } + + nbytes = sizeof(struct file_level_trim_output); + break; + } + case FSCTL_QUERY_ALLOCATED_RANGES: + if (in_buf_len < sizeof(struct file_allocated_range_buffer)) { + ret = -EINVAL; + goto out; + } + + ret = fsctl_query_allocated_ranges(work, id, + (struct file_allocated_range_buffer *)buffer, + (struct file_allocated_range_buffer *)&rsp->Buffer[0], + out_buf_len / + sizeof(struct file_allocated_range_buffer), &nbytes); + if (ret == -E2BIG) { + rsp->hdr.Status = STATUS_BUFFER_OVERFLOW; + } else if (ret < 0) { + nbytes = 0; + goto out; + } + + nbytes *= sizeof(struct file_allocated_range_buffer); + break; + case FSCTL_GET_REPARSE_POINT: + { + struct reparse_data_buffer *reparse_ptr; + struct ksmbd_file *fp; + + if (out_buf_len < sizeof(struct reparse_data_buffer)) { + ret = -EINVAL; + goto out; + } + + reparse_ptr = (struct reparse_data_buffer *)&rsp->Buffer[0]; + fp = ksmbd_lookup_fd_fast(work, id); + if (!fp) { + pr_err("not found fp!!\n"); + ret = -ENOENT; + goto out; + } + + reparse_ptr->ReparseTag = + smb2_get_reparse_tag_special_file(file_inode(fp->filp)->i_mode); + reparse_ptr->ReparseDataLength = 0; + ksmbd_fd_put(work, fp); + nbytes = sizeof(struct reparse_data_buffer); + break; + } + case FSCTL_DUPLICATE_EXTENTS_TO_FILE: + { + struct ksmbd_file *fp_in, *fp_out = NULL; + struct duplicate_extents_to_file *dup_ext; + loff_t src_off, dst_off, length, cloned; + + if (in_buf_len < sizeof(struct duplicate_extents_to_file)) { + ret = -EINVAL; + goto out; + } + + dup_ext = (struct duplicate_extents_to_file *)buffer; + + fp_in = ksmbd_lookup_fd_slow(work, dup_ext->VolatileFileHandle, + dup_ext->PersistentFileHandle); + if (!fp_in) { + pr_err("not found file handle in duplicate extent to file\n"); + ret = -EBADF; + rsp->hdr.Status = STATUS_INVALID_HANDLE; + goto out2; + } + + fp_out = ksmbd_lookup_fd_fast(work, id); + if (!fp_out) { + pr_err("not found fp\n"); + ret = -EBADF; + rsp->hdr.Status = STATUS_FILE_CLOSED; + ksmbd_fd_put(work, fp_in); + goto out2; + } + + if (!test_tree_conn_flag(work->tcon, + KSMBD_TREE_CONN_FLAG_WRITABLE)) { + ret = -EACCES; + goto dup_ext_out; + } + + if (!(fp_out->daccess & FILE_WRITE_DATA_LE)) { + ret = -EACCES; + goto dup_ext_out; + } + if (!(fp_in->daccess & FILE_READ_DATA_LE)) { + ret = -EACCES; + goto dup_ext_out; + } + + src_off = le64_to_cpu(dup_ext->SourceFileOffset); + dst_off = le64_to_cpu(dup_ext->TargetFileOffset); + length = le64_to_cpu(dup_ext->ByteCount); + if (src_off < 0 || dst_off < 0 || length < 0 || + src_off + length < src_off || dst_off + length < dst_off) { + ret = -EINVAL; + goto dup_ext_out; + } + if (src_off + length > i_size_read(file_inode(fp_in->filp))) { + ret = -EOPNOTSUPP; + goto dup_ext_out; + } + if (dst_off + length > i_size_read(file_inode(fp_out->filp))) + goto dup_ext_out; + if ((fp_in->f_ci->m_fattr & FILE_ATTRIBUTE_SPARSE_FILE_LE) && + !(fp_out->f_ci->m_fattr & FILE_ATTRIBUTE_SPARSE_FILE_LE)) { + ret = -EOPNOTSUPP; + goto dup_ext_out; + } + if (file_inode(fp_in->filp) == file_inode(fp_out->filp) && + dst_off + length > src_off && + dst_off < src_off + length) { + ret = -EOPNOTSUPP; + goto dup_ext_out; + } + + cloned = vfs_clone_file_range(fp_in->filp, src_off, + fp_out->filp, dst_off, length, 0); + if (cloned != length) { + cloned = vfs_copy_file_range(fp_in->filp, src_off, + fp_out->filp, dst_off, + length, 0); + if (cloned != length) { + if (cloned < 0) + ret = cloned; + else + ret = -EINVAL; + } + } + +dup_ext_out: + ksmbd_fd_put(work, fp_in); + ksmbd_fd_put(work, fp_out); + if (ret < 0) + goto out; + break; + } + default: + ksmbd_debug(SMB, "not implemented yet ioctl command 0x%x\n", + cnt_code); + ret = -EOPNOTSUPP; + goto out; + } + + rsp->CtlCode = cpu_to_le32(cnt_code); + rsp->InputCount = cpu_to_le32(0); + rsp->InputOffset = cpu_to_le32(112); + rsp->OutputOffset = cpu_to_le32(112); + rsp->OutputCount = cpu_to_le32(nbytes); + rsp->StructureSize = cpu_to_le16(49); + rsp->Reserved = cpu_to_le16(0); + rsp->Flags = cpu_to_le32(0); + rsp->Reserved2 = cpu_to_le32(0); + ret = ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_ioctl_rsp) + nbytes); + if (!ret) + return ret; + +out: + if (ret == -EACCES) + rsp->hdr.Status = STATUS_ACCESS_DENIED; + else if (ret == -ENOENT) + rsp->hdr.Status = STATUS_OBJECT_NAME_NOT_FOUND; + else if (ret == -EOPNOTSUPP) + rsp->hdr.Status = STATUS_NOT_SUPPORTED; + else if (ret == -ENOSPC) + rsp->hdr.Status = STATUS_BUFFER_TOO_SMALL; + else if (!chseq_err && (ret < 0 || rsp->hdr.Status == 0)) + rsp->hdr.Status = STATUS_INVALID_PARAMETER; + +out2: + smb2_set_err_rsp(work); + return ret; +} + +/** + * smb20_oplock_break_ack() - handler for smb2.0 oplock break command + * @work: smb work containing oplock break command buffer + * + * Return: 0 + */ +static void smb20_oplock_break_ack(struct ksmbd_work *work) +{ + struct smb2_oplock_break *req; + struct smb2_oplock_break *rsp; + struct ksmbd_file *fp; + struct oplock_info *opinfo = NULL; + __le32 status = STATUS_SUCCESS; + int ret; + u64 volatile_id, persistent_id; + char req_oplevel = 0, rsp_oplevel = 0; + + WORK_BUFFERS(work, req, rsp); + + volatile_id = req->VolatileFid; + persistent_id = req->PersistentFid; + req_oplevel = req->OplockLevel; + ksmbd_debug(OPLOCK, "v_id %llu, p_id %llu request oplock level %d\n", + volatile_id, persistent_id, req_oplevel); + + fp = ksmbd_lookup_fd_slow(work, volatile_id, persistent_id); + if (!fp) { + rsp->hdr.Status = STATUS_FILE_CLOSED; + smb2_set_err_rsp(work); + return; + } + + ret = smb2_set_request_open(work, fp, &req->hdr, false, false); + if (ret) { + rsp->hdr.Status = STATUS_FILE_CLOSED; + smb2_set_err_rsp(work); + ksmbd_fd_put(work, fp); + return; + } + + opinfo = opinfo_get(fp); + if (!opinfo) { + pr_err("unexpected null oplock_info\n"); + rsp->hdr.Status = STATUS_INVALID_OPLOCK_PROTOCOL; + smb2_set_err_rsp(work); + ksmbd_fd_put(work, fp); + return; + } + + if (opinfo->op_state != OPLOCK_ACK_WAIT) { + ksmbd_debug(SMB, "unexpected oplock state 0x%x\n", + opinfo->op_state); + if (smb3_hdr_replay(&req->hdr) && + opinfo->op_state == OPLOCK_STATE_NONE) { + rsp->StructureSize = cpu_to_le16(24); + rsp->OplockLevel = opinfo->level; + rsp->Reserved = 0; + rsp->Reserved2 = 0; + rsp->VolatileFid = volatile_id; + rsp->PersistentFid = persistent_id; + ret = ksmbd_iov_pin_rsp(work, rsp, + sizeof(struct smb2_oplock_break)); + if (ret) + ksmbd_debug(SMB, + "failed to pin replayed oplock break response: %d\n", + ret); + goto out_no_state_change; + } + if (opinfo->level == SMB2_OPLOCK_LEVEL_NONE) + status = STATUS_INVALID_OPLOCK_PROTOCOL; + else + status = STATUS_INVALID_DEVICE_STATE; + goto err_out; + } + + if (req_oplevel == SMB2_OPLOCK_LEVEL_LEASE) { + opinfo->level = SMB2_OPLOCK_LEVEL_NONE; + status = STATUS_INVALID_PARAMETER; + goto err_out; + } + + if (opinfo->level == SMB2_OPLOCK_LEVEL_NONE) { + status = STATUS_INVALID_OPLOCK_PROTOCOL; + goto err_out; + } + + if (opinfo->level == SMB2_OPLOCK_LEVEL_EXCLUSIVE && + req_oplevel != SMB2_OPLOCK_LEVEL_II && + req_oplevel != SMB2_OPLOCK_LEVEL_NONE) { + opinfo->level = SMB2_OPLOCK_LEVEL_NONE; + status = STATUS_INVALID_OPLOCK_PROTOCOL; + goto err_out; + } + + if (opinfo->level == SMB2_OPLOCK_LEVEL_BATCH && + req_oplevel != SMB2_OPLOCK_LEVEL_II && + req_oplevel != SMB2_OPLOCK_LEVEL_NONE && + req_oplevel != SMB2_OPLOCK_LEVEL_EXCLUSIVE) { + opinfo->level = SMB2_OPLOCK_LEVEL_NONE; + status = STATUS_INVALID_OPLOCK_PROTOCOL; + goto err_out; + } + + if (opinfo->level == SMB2_OPLOCK_LEVEL_II && + req_oplevel != SMB2_OPLOCK_LEVEL_NONE) { + opinfo->level = SMB2_OPLOCK_LEVEL_NONE; + status = STATUS_INVALID_OPLOCK_PROTOCOL; + goto err_out; + } + + if (req_oplevel == SMB2_OPLOCK_LEVEL_EXCLUSIVE) + rsp_oplevel = SMB2_OPLOCK_LEVEL_NONE; + else + rsp_oplevel = req_oplevel; + + opinfo->level = rsp_oplevel; + + rsp->StructureSize = cpu_to_le16(24); + rsp->OplockLevel = rsp_oplevel; + rsp->Reserved = 0; + rsp->Reserved2 = 0; + rsp->VolatileFid = volatile_id; + rsp->PersistentFid = persistent_id; + ret = ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_oplock_break)); + if (ret) + ksmbd_debug(SMB, "failed to pin oplock break response: %d\n", + ret); + goto out; + +err_out: + rsp->hdr.Status = status; + smb2_set_err_rsp(work); + +out: + spin_lock(&opinfo->state_lock); + if (opinfo->op_state != OPLOCK_CLOSING) + opinfo->op_state = OPLOCK_STATE_NONE; + spin_unlock(&opinfo->state_lock); + wake_up_interruptible_all(&opinfo->oplock_q); +out_no_state_change: + opinfo_put(opinfo); + ksmbd_fd_put(work, fp); +} + +static bool smb2_lease_state_valid(__le32 state) +{ + return !(state & ~(SMB2_LEASE_READ_CACHING_LE | + SMB2_LEASE_HANDLE_CACHING_LE | + SMB2_LEASE_WRITE_CACHING_LE)); +} + +static int check_lease_state(struct lease *lease, __le32 req_state) +{ + if (smb2_lease_state_valid(req_state) && + !(req_state & ~lease->new_state)) + return 0; + + return 1; +} + +/** + * smb21_lease_break_ack() - handler for smb2.1 lease break command + * @work: smb work containing lease break command buffer + * + * Return: 0 + */ +static void smb21_lease_break_ack(struct ksmbd_work *work) +{ + struct ksmbd_conn *conn = work->conn; + struct smb2_lease_ack *req; + struct smb2_lease_ack *rsp; + struct oplock_info *opinfo; + int ret = 0; + __le32 lease_state; + struct lease *lease; + + WORK_BUFFERS(work, req, rsp); + + ksmbd_debug(OPLOCK, "smb21 lease break, lease state(0x%x)\n", + le32_to_cpu(req->LeaseState)); + opinfo = lookup_lease_in_table(conn, req->LeaseKey); + if (!opinfo) { + ksmbd_debug(OPLOCK, "file not opened\n"); + smb2_set_err_rsp(work); + rsp->hdr.Status = STATUS_UNSUCCESSFUL; + return; + } + lease = opinfo->o_lease; + + if (opinfo->op_state == OPLOCK_STATE_NONE) { + pr_err("unexpected lease break state 0x%x\n", + opinfo->op_state); + if (smb3_hdr_replay(&req->hdr)) + goto replay_rsp; + rsp->hdr.Status = STATUS_UNSUCCESSFUL; + goto err_out; + } + + if (!atomic_read(&opinfo->breaking_cnt)) { + if (smb3_hdr_replay(&req->hdr)) + goto replay_rsp; + rsp->hdr.Status = STATUS_UNSUCCESSFUL; + goto err_out; + } + + if (check_lease_state(lease, req->LeaseState)) { + rsp->hdr.Status = STATUS_REQUEST_NOT_ACCEPTED; + ksmbd_debug(OPLOCK, + "req lease state: 0x%x, expected state: 0x%x\n", + req->LeaseState, lease->new_state); + goto err_out; + } + + lease_state = req->LeaseState; + lease->state = lease_state; + lease->new_state = SMB2_LEASE_NONE_LE; + lease_update_oplock_levels(lease); + + rsp->StructureSize = cpu_to_le16(36); + rsp->Reserved = 0; + rsp->Flags = 0; + memcpy(rsp->LeaseKey, req->LeaseKey, 16); + rsp->LeaseState = lease_state; + rsp->LeaseDuration = 0; + ret = ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_lease_ack)); + if (ret) + goto err_out; + + spin_lock(&opinfo->state_lock); + if (opinfo->op_state != OPLOCK_CLOSING) + opinfo->op_state = OPLOCK_STATE_NONE; + spin_unlock(&opinfo->state_lock); + wake_up_interruptible_all(&opinfo->oplock_q); + atomic_dec_if_positive(&opinfo->breaking_cnt); + wake_up_interruptible_all(&opinfo->oplock_brk); + opinfo_put(opinfo); + return; + +replay_rsp: + rsp->StructureSize = cpu_to_le16(36); + rsp->Reserved = 0; + rsp->Flags = 0; + memcpy(rsp->LeaseKey, req->LeaseKey, 16); + rsp->LeaseState = lease->state; + rsp->LeaseDuration = 0; + ret = ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_lease_ack)); + if (ret) + goto err_out; + opinfo_put(opinfo); + return; + +err_out: + smb2_set_err_rsp(work); + opinfo_put(opinfo); + return; +} + +/** + * smb2_oplock_break() - dispatcher for smb2.0 and 2.1 oplock/lease break + * @work: smb work containing oplock/lease break command buffer + * + * Return: 0 on success, otherwise error + */ +int smb2_oplock_break(struct ksmbd_work *work) +{ + struct smb2_oplock_break *req; + struct smb2_oplock_break *rsp; + + ksmbd_debug(SMB, "Received smb2 oplock break acknowledgment request\n"); + + WORK_BUFFERS(work, req, rsp); + + switch (le16_to_cpu(req->StructureSize)) { + case OP_BREAK_STRUCT_SIZE_20: + smb20_oplock_break_ack(work); + break; + case OP_BREAK_STRUCT_SIZE_21: + smb21_lease_break_ack(work); + break; + default: + ksmbd_debug(OPLOCK, "invalid break cmd %d\n", + le16_to_cpu(req->StructureSize)); + rsp->hdr.Status = STATUS_INVALID_PARAMETER; + smb2_set_err_rsp(work); + return -EINVAL; + } + + return 0; +} + +/* + * Cancel handler for a deferred CHANGE_NOTIFY. Races against + * __ksmbd_close_fd()'s notify_pendings drain (vfs_cache.c), which can run + * concurrently on a different connection closing the same handle -- only + * one of the two may claim and free in_work, so both sides check + * list_empty() under fp->f_lock before touching it (list_del_init() + * leaves a node empty, so whichever side removes it first is the owner; + * the loser must not touch in_work again, since the winner may already be + * freeing it). + * + * smb2_cancel() holds conn->request_lock (a spinlock) for the entire + * time it walks conn->async_requests and calls this function -- so this + * runs with preemption disabled and must not sleep or re-acquire that + * same lock. release_async_work() does both (it takes conn->request_lock + * itself, and frees things that can involve sleeping paths), so calling + * it from here would self-deadlock the very thread processing the + * client's CANCEL command. ksmbd_conn_write() can also sleep (it takes + * conn's write mutex). So: do only the non-sleeping, no-relock cleanup + * inline here. smb2_cancel() sends and frees the claimed notify after it + * drops request_lock, preserving response order for a client CANCEL. The + * connection teardown caller has no such post-unlock path, so its wrapper + * defers the send and free to a workqueue. + */ +struct notify_cancel_ctx { + struct work_struct work; + struct ksmbd_work *in_work; +}; + +static void smb2_send_notify_cancelled(struct ksmbd_work *work) +{ + struct smb2_hdr *hdr = smb_get_msg(work->response_buf); + struct ksmbd_conn *conn = work->conn; + struct ksmbd_session *sess; + + sess = ksmbd_session_lookup(conn, le64_to_cpu(hdr->SessionId)); + if (sess) { + work->sess = sess; + if (work->encrypted && sess->enc && conn->ops->encrypt_resp) { + conn->ops->encrypt_resp(work); + } else if (conn->ops->is_sign_req && conn->ops->set_sign_rsp && + conn->ops->is_sign_req(work, + conn->ops->get_cmd_val(work))) { + conn->ops->set_sign_rsp(work); + } + } + + ksmbd_conn_write(work); + if (sess) { + ksmbd_user_session_put(sess); + work->sess = NULL; + } +} + +static void smb2_notify_cancel_deferred(struct work_struct *w) +{ + struct notify_cancel_ctx *ctx = + container_of(w, struct notify_cancel_ctx, work); + struct ksmbd_conn *conn = ctx->in_work->conn; + + smb2_complete_notify_cancel(ctx->in_work); + kfree(ctx); + /* + * The connection teardown waits for r_count before destroying + * connection sessions and their proc entries. + */ + ksmbd_conn_r_count_dec(conn); +} + +static struct ksmbd_work *smb2_notify_cancel_claim(void **argv) +{ + struct ksmbd_work *in_work = (struct ksmbd_work *)argv[0]; + struct ksmbd_file *fp = (struct ksmbd_file *)argv[1]; + bool claimed; + + spin_lock(&fp->f_lock); + claimed = !list_empty(&in_work->notify_entry); + if (claimed) + list_del_init(&in_work->notify_entry); + spin_unlock(&fp->f_lock); + + if (!claimed) + return NULL; + + /* conn->request_lock is held by smb2_cancel() or connection teardown. */ + in_work->cancel_fn = NULL; + kfree(in_work->cancel_argv); + in_work->cancel_argv = NULL; + return in_work; +} + +static void smb2_complete_notify_cancel(struct ksmbd_work *in_work) +{ + struct smb2_hdr *in_hdr = smb_get_msg(in_work->response_buf); + + in_hdr->Status = STATUS_CANCELLED; + smb2_send_notify_cancelled(in_work); + release_async_work(in_work); + ksmbd_free_work_struct(in_work); +} + +static void smb2_notify_cancel_fn(void **argv) +{ + struct ksmbd_work *in_work = smb2_notify_cancel_claim(argv); + struct ksmbd_conn *conn; + struct notify_cancel_ctx *ctx; + + if (!in_work) + return; + conn = in_work->conn; + + ctx = kmalloc(sizeof(*ctx), GFP_ATOMIC); + if (!ctx) { + /* Can't defer the response -- free without sending one. */ + list_del_init(&in_work->async_request_entry); + in_work->asynchronous = false; + if (in_work->async_id) { + ksmbd_release_id(&conn->async_ida, in_work->async_id); + in_work->async_id = 0; + } + ksmbd_free_work_struct(in_work); + return; + } + ctx->in_work = in_work; + INIT_WORK(&ctx->work, smb2_notify_cancel_deferred); + /* + * This deferred work can outlive the connection handler's receive loop. + * Keep teardown from destroying the connection's sessions until the + * deferred response has finished using them. + */ + ksmbd_conn_r_count_inc(conn); + schedule_work(&ctx->work); +} + +/** + * smb2_notify() - handler for smb2 notify request + * @work: smb work containing notify command buffer + * + * Return: 0 on success, otherwise error + */ +int smb2_notify(struct ksmbd_work *work) +{ + struct smb2_change_notify_req *req; + struct smb2_change_notify_rsp *rsp; + struct ksmbd_work *in_work; + struct smb2_hdr *in_hdr; + struct ksmbd_file *fp; + + ksmbd_debug(SMB, "Received smb2 notify\n"); + + WORK_BUFFERS(work, req, rsp); + + if (smb2_compound_has_failed(work, &rsp->hdr)) + return -EACCES; + + if (work->next_smb2_rcv_hdr_off && req->hdr.NextCommand) { + rsp->hdr.Status = STATUS_INTERNAL_ERROR; + smb2_set_err_rsp(work); + return -EIO; + } + + /* + * macOS backupd sends CHANGE_NOTIFY with FileId=FFFF...FFFF (share-root + * sentinel) to watch for changes on the share root without holding an + * open handle. Respond STATUS_PENDING + STATUS_NOTIFY_CLEANUP immediately; + * without this, backupd aborts Time Machine setup on STATUS_FILE_CLOSED. + */ + if (req->VolatileFileId == SMB2_NO_FID && + req->PersistentFileId == SMB2_NO_FID) { + in_work = ksmbd_alloc_work_struct(); + if (!in_work || allocate_interim_rsp_buf(in_work)) { + if (in_work) + ksmbd_free_work_struct(in_work); + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + smb2_set_err_rsp(work); + return 0; + } + if (setup_async_work(work, NULL, NULL)) { + ksmbd_free_work_struct(in_work); + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + smb2_set_err_rsp(work); + return 0; + } + smb2_send_interim_resp(work, STATUS_PENDING); + in_work->conn = work->conn; + in_hdr = smb_get_msg(in_work->response_buf); + memcpy(in_hdr, ksmbd_resp_buf_next(work), + __SMB2_HEADER_STRUCTURE_SIZE); + in_hdr->Flags |= SMB2_FLAGS_ASYNC_COMMAND; + in_hdr->Id.AsyncId = cpu_to_le64(work->async_id); + smb2_set_err_rsp(in_work); + in_hdr->Status = STATUS_NOTIFY_CLEANUP; + in_work->async_id = work->async_id; + work->async_id = 0; + release_async_work(work); + if (smb2_send_interim_work(in_work, work, false)) + ksmbd_debug(SMB, "failed to send notify cleanup\n"); + ksmbd_free_work_struct(in_work); + work->send_no_response = 1; + return 0; + } + + /* + * KSMBD does not implement a real change-notification backend. + * Genuine SMB2 servers (and macOS smbfs) never complete a + * CHANGE_NOTIFY spontaneously: it is satisfied only by a real + * directory change, or with STATUS_NOTIFY_CLEANUP when the watched + * handle is closed. Completing it early (e.g. on a timer) makes + * Finder treat the cleanup as "directory changed" and re-enumerate + * the directory forever, leaving items unopenable. Returning + * STATUS_NOT_IMPLEMENTED here (like stock ksmbd) makes macOS smbfs + * hard-freeze on unmount, so this must stay deferred. + */ + fp = ksmbd_lookup_fd_slow(work, req->VolatileFileId, req->PersistentFileId); + if (!fp) { + rsp->hdr.Status = STATUS_FILE_CLOSED; + smb2_set_err_rsp(work); + return 0; + } + + in_work = ksmbd_alloc_work_struct(); + if (!in_work || allocate_interim_rsp_buf(in_work)) { + if (in_work) + ksmbd_free_work_struct(in_work); + ksmbd_fd_put(work, fp); + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + smb2_set_err_rsp(work); + return 0; + } + /* + * in_work is synthetic (not from the normal request-receiving + * pipeline), so it has no request_buf of its own. It gets registered + * into conn->async_requests below, and smb2_cancel() unconditionally + * computes smb_get_msg(iter->request_buf) for every entry in that + * list while searching for a match -- give it its own small buffer + * (not an alias of response_buf: ksmbd_free_work_struct() kvfree()s + * both separately, so aliasing them would double-free) so that stays + * a harmless read instead of a near-NULL dereference. + */ + in_work->request_buf = kzalloc(MAX_CIFS_SMALL_BUFFER_SIZE, KSMBD_DEFAULT_GFP); + if (!in_work->request_buf) { + ksmbd_free_work_struct(in_work); + ksmbd_fd_put(work, fp); + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + smb2_set_err_rsp(work); + return 0; + } + memcpy(smb_get_msg(in_work->request_buf), req, + __SMB2_HEADER_STRUCTURE_SIZE); + + if (setup_async_work(work, NULL, NULL)) { + ksmbd_free_work_struct(in_work); + ksmbd_fd_put(work, fp); + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; + smb2_set_err_rsp(work); + return 0; + } + + smb2_send_interim_resp(work, STATUS_PENDING); + + /* Keep the async IDA alive until the deferred work is released. */ + in_work->conn = ksmbd_conn_get(work->conn); + in_work->owns_conn_ref = true; + in_work->encrypted = work->encrypted; + in_hdr = smb_get_msg(in_work->response_buf); + memcpy(in_hdr, ksmbd_resp_buf_next(work), __SMB2_HEADER_STRUCTURE_SIZE); + in_hdr->Flags |= SMB2_FLAGS_ASYNC_COMMAND; + in_hdr->Id.AsyncId = cpu_to_le64(work->async_id); + smb2_set_err_rsp(in_work); + in_hdr->Status = STATUS_NOTIFY_CLEANUP; + + /* + * Transfer ownership of the async id to in_work; it stays reserved + * until in_work is freed after the deferred response is sent on + * close, so it can't be reused for an unrelated async response. + */ + in_work->async_id = work->async_id; + work->async_id = 0; + release_async_work(work); + + /* + * work itself is about to be recycled by the normal request-processing + * pipeline, so it can't stay the target of a future CANCEL -- register + * in_work instead, reusing the same async_id, so a client-sent CANCEL + * for this notify actually finds something to cancel instead of + * silently doing nothing until the handle eventually closes. + */ + in_work->asynchronous = true; + in_work->cancel_argv = kmalloc_array(2, sizeof(void *), KSMBD_DEFAULT_GFP); + if (in_work->cancel_argv) { + in_work->cancel_argv[0] = in_work; + in_work->cancel_argv[1] = fp; + in_work->cancel_fn = smb2_notify_cancel_fn; + } + + if (!ksmbd_conn_link_async_request(work->conn, in_work)) { + kfree(in_work->cancel_argv); + in_work->cancel_argv = NULL; + in_work->cancel_fn = NULL; + in_work->asynchronous = false; + ksmbd_fd_put(work, fp); + if (smb2_send_interim_work(in_work, work, false)) + ksmbd_debug(SMB, "failed to send notify cleanup\n"); + ksmbd_free_work_struct(in_work); + work->send_no_response = 1; + return 0; + } + + spin_lock(&fp->f_lock); + list_add_tail(&in_work->notify_entry, &fp->notify_pendings); + spin_unlock(&fp->f_lock); + + ksmbd_fd_put(work, fp); + work->send_no_response = 1; + return 0; +} + +/** + * smb2_is_sign_req() - handler for checking packet signing status + * @work: smb work containing notify command buffer + * @command: SMB2 command id + * + * Return: true if packed is signed, false otherwise + */ +bool smb2_is_sign_req(struct ksmbd_work *work, unsigned int command) +{ + struct smb2_hdr *rcv_hdr2 = smb_get_msg(work->request_buf); + + if ((rcv_hdr2->Flags & SMB2_FLAGS_SIGNED) && + command != SMB2_NEGOTIATE_HE) + return true; + + return false; +} + +/** + * smb2_check_sign_req() - handler for req packet sign processing + * @work: smb work containing notify command buffer + * + * Return: 1 on success, 0 otherwise + */ +int smb2_check_sign_req(struct ksmbd_work *work) +{ + struct smb2_hdr *hdr; + char signature_req[SMB2_SIGNATURE_SIZE]; + char signature[SMB2_HMACSHA256_SIZE]; + struct kvec iov[1]; + size_t len; + + hdr = smb_get_msg(work->request_buf); + if (work->next_smb2_rcv_hdr_off) + hdr = ksmbd_req_buf_next(work); + + if (!hdr->NextCommand && !work->next_smb2_rcv_hdr_off) + len = get_rfc1002_len(work->request_buf); + else if (hdr->NextCommand) + len = le32_to_cpu(hdr->NextCommand); + else + len = get_rfc1002_len(work->request_buf) - + work->next_smb2_rcv_hdr_off; + + memcpy(signature_req, hdr->Signature, SMB2_SIGNATURE_SIZE); + memset(hdr->Signature, 0, SMB2_SIGNATURE_SIZE); + + iov[0].iov_base = (char *)&hdr->ProtocolId; + iov[0].iov_len = len; + + ksmbd_sign_smb2_pdu(work->conn, work->sess->sess_key, iov, 1, + signature); + + if (crypto_memneq(signature, signature_req, SMB2_SIGNATURE_SIZE)) { + pr_err("bad smb2 signature\n"); + return 0; + } + + return 1; +} + +/** + * smb2_get_sign_rsp_iov() - get the iovecs used to sign a response + * @work: work that has the response iovecs + * @hdr: SMB2 header of the response + * @n_vec: set to the number of iovecs to sign + * + * Response data may be in another buffer. In this case, the response uses + * more than one iovec. Find the iovec that starts with @hdr. Sign this + * iovec and all iovecs after it. + * + * Return: The first iovec to sign. + */ +static struct kvec *smb2_get_sign_rsp_iov(struct ksmbd_work *work, + struct smb2_hdr *hdr, int *n_vec) +{ + int i; + + /* + * iov[0] has the RFC1002 message length. It is not part of the SMB2 + * message, so do not sign it. + */ + for (i = 1; i <= work->iov_idx; i++) { + if (work->iov[i].iov_base == hdr) { + *n_vec = work->iov_idx - i + 1; + return &work->iov[i]; + } + } + + WARN_ON_ONCE(1); + *n_vec = 1; + return &work->iov[work->iov_idx]; +} + +/** + * smb2_set_sign_rsp() - handler for rsp packet sign processing + * @work: smb work containing notify command buffer + * + */ +void smb2_set_sign_rsp(struct ksmbd_work *work) +{ + struct smb2_hdr *hdr; + char signature[SMB2_HMACSHA256_SIZE]; + struct kvec *iov; + int n_vec; + + hdr = ksmbd_resp_buf_curr(work); + hdr->Flags |= SMB2_FLAGS_SIGNED; + memset(hdr->Signature, 0, SMB2_SIGNATURE_SIZE); + + iov = smb2_get_sign_rsp_iov(work, hdr, &n_vec); + + ksmbd_sign_smb2_pdu(work->conn, work->sess->sess_key, iov, n_vec, + signature); + memcpy(hdr->Signature, signature, SMB2_SIGNATURE_SIZE); +} + +/** + * smb3_check_sign_req() - handler for req packet sign processing + * @work: smb work containing notify command buffer + * + * Return: 1 on success, 0 otherwise + */ +int smb3_check_sign_req(struct ksmbd_work *work) +{ + struct ksmbd_conn *conn = work->conn; + char *signing_key; + struct smb2_hdr *hdr; + struct channel *chann; + char signature_req[SMB2_SIGNATURE_SIZE]; + char signature[SMB2_CMACAES_SIZE]; + struct kvec iov[1]; + size_t len; + + hdr = smb_get_msg(work->request_buf); + if (work->next_smb2_rcv_hdr_off) + hdr = ksmbd_req_buf_next(work); + + if (!hdr->NextCommand && !work->next_smb2_rcv_hdr_off) + len = get_rfc1002_len(work->request_buf); + else if (hdr->NextCommand) + len = le32_to_cpu(hdr->NextCommand); + else + len = get_rfc1002_len(work->request_buf) - + work->next_smb2_rcv_hdr_off; + + if (le16_to_cpu(hdr->Command) == SMB2_SESSION_SETUP_HE) { + signing_key = work->sess->smb3signingkey; + } else { + chann = lookup_chann_list(work->sess, conn); + if (!chann) { + if (le16_to_cpu(hdr->Command) != SMB2_SESSION_SETUP_HE || + !(hdr->Flags & SMB2_FLAGS_SIGNED)) + return 0; + signing_key = work->sess->smb3signingkey; + } else { + signing_key = chann->smb3signingkey; + } + } + + if (!signing_key) { + pr_err("SMB3 signing key is not generated\n"); + return 0; + } + + memcpy(signature_req, hdr->Signature, SMB2_SIGNATURE_SIZE); + memset(hdr->Signature, 0, SMB2_SIGNATURE_SIZE); + iov[0].iov_base = (char *)&hdr->ProtocolId; + iov[0].iov_len = len; + + ksmbd_sign_smb3_pdu(conn, signing_key, iov, 1, signature); + + if (crypto_memneq(signature, signature_req, SMB2_SIGNATURE_SIZE)) { + pr_err("bad smb2 signature\n"); + return 0; + } + + return 1; +} + +/** + * smb3_set_sign_rsp() - handler for rsp packet sign processing + * @work: smb work containing notify command buffer + * + */ +void smb3_set_sign_rsp(struct ksmbd_work *work) +{ + struct ksmbd_conn *conn = work->conn; + struct smb2_hdr *hdr; + struct channel *chann; + char signature[SMB2_CMACAES_SIZE]; + struct kvec *iov; + u16 command = conn->ops->get_cmd_val(work); + int n_vec; + char *signing_key; + + hdr = ksmbd_resp_buf_curr(work); + + if (command == SMB2_SESSION_SETUP_HE && + (!conn->binding || hdr->Status != STATUS_SUCCESS)) { + signing_key = work->sess->smb3signingkey; + } else { + chann = lookup_chann_list(work->sess, work->conn); + if (!chann) { + return; + } + signing_key = chann->smb3signingkey; + } + + if (!signing_key) + return; + + hdr->Flags |= SMB2_FLAGS_SIGNED; + memset(hdr->Signature, 0, SMB2_SIGNATURE_SIZE); + + iov = smb2_get_sign_rsp_iov(work, hdr, &n_vec); + + ksmbd_sign_smb3_pdu(conn, signing_key, iov, n_vec, signature); + memcpy(hdr->Signature, signature, SMB2_SIGNATURE_SIZE); +} + +/** + * smb3_preauth_hash_rsp() - handler for computing preauth hash on response + * @work: smb work containing response buffer + * + */ +void smb3_preauth_hash_rsp(struct ksmbd_work *work) +{ + struct ksmbd_conn *conn = work->conn; + struct ksmbd_session *sess = work->sess; + struct smb2_hdr *req, *rsp; + + if (conn->dialect != SMB311_PROT_ID) + return; + + WORK_BUFFERS(work, req, rsp); + + if (le16_to_cpu(req->Command) == SMB2_NEGOTIATE_HE) { + ksmbd_conn_lock(conn); + if (conn->preauth_info) + ksmbd_gen_preauth_integrity_hash(conn, work->response_buf, + conn->preauth_info->Preauth_HashValue); + ksmbd_conn_unlock(conn); + } + + if (le16_to_cpu(rsp->Command) == SMB2_SESSION_SETUP_HE && sess) { + ksmbd_conn_lock(conn); + + if (conn->binding) { + struct preauth_session *preauth_sess; + + preauth_sess = ksmbd_preauth_session_lookup(conn, sess->id); + if (preauth_sess) + ksmbd_gen_preauth_integrity_hash(conn, + work->response_buf, + preauth_sess->Preauth_HashValue); + } else if (sess->Preauth_HashValue) { + ksmbd_gen_preauth_integrity_hash(conn, work->response_buf, + sess->Preauth_HashValue); + } + ksmbd_conn_unlock(conn); + } +} + +static void fill_transform_hdr(void *tr_buf, char *old_buf, __le16 cipher_type) +{ + struct smb2_transform_hdr *tr_hdr = tr_buf + 4; + struct smb2_hdr *hdr = smb_get_msg(old_buf); + unsigned int orig_len = get_rfc1002_len(old_buf); + + /* tr_buf must be cleared by the caller */ + tr_hdr->ProtocolId = SMB2_TRANSFORM_PROTO_NUM; + tr_hdr->OriginalMessageSize = cpu_to_le32(orig_len); + tr_hdr->Flags = cpu_to_le16(TRANSFORM_FLAG_ENCRYPTED); + if (cipher_type == SMB2_ENCRYPTION_AES128_GCM || + cipher_type == SMB2_ENCRYPTION_AES256_GCM) + get_random_bytes(&tr_hdr->Nonce, SMB3_AES_GCM_NONCE); + else + get_random_bytes(&tr_hdr->Nonce, SMB3_AES_CCM_NONCE); + memcpy(&tr_hdr->SessionId, &hdr->SessionId, 8); + inc_rfc1001_len(tr_buf, sizeof(struct smb2_transform_hdr)); + inc_rfc1001_len(tr_buf, orig_len); +} + +int smb3_encrypt_resp(struct ksmbd_work *work) +{ + struct kvec *iov = work->iov; + int rc = -ENOMEM; + void *tr_buf; + + tr_buf = kzalloc(sizeof(struct smb2_transform_hdr) + 4, KSMBD_DEFAULT_GFP); + if (!tr_buf) + return rc; + + /* fill transform header */ + fill_transform_hdr(tr_buf, work->response_buf, work->conn->cipher_type); + + iov[0].iov_base = tr_buf; + iov[0].iov_len = sizeof(struct smb2_transform_hdr) + 4; + work->tr_buf = tr_buf; + + return ksmbd_crypt_message(work, iov, work->iov_idx + 1, 1); +} + +bool smb3_is_transform_hdr(void *buf) +{ + struct smb2_transform_hdr *trhdr = smb_get_msg(buf); + + return trhdr->ProtocolId == SMB2_TRANSFORM_PROTO_NUM; +} + +int smb3_decrypt_req(struct ksmbd_work *work) +{ + char *buf = work->request_buf; + unsigned int pdu_length = get_rfc1002_len(buf); + struct kvec iov[2]; + unsigned int buf_data_size; + struct smb2_transform_hdr *tr_hdr = smb_get_msg(buf); + unsigned int original_msg_size; + int rc = 0; + + if (pdu_length < sizeof(struct smb2_transform_hdr)) { + pr_err("Transform message is too small (%u)\n", + pdu_length); + return -ECONNABORTED; + } + + buf_data_size = pdu_length - sizeof(struct smb2_transform_hdr); + original_msg_size = le32_to_cpu(tr_hdr->OriginalMessageSize); + if (buf_data_size < sizeof(struct smb2_compression_hdr) || + original_msg_size < sizeof(struct smb2_compression_hdr)) { + pr_err("Transform message is too small (%u)\n", + pdu_length); + return -ECONNABORTED; + } + + if (buf_data_size < original_msg_size) { + pr_err("Transform message is broken\n"); + return -ECONNABORTED; + } + + iov[0].iov_base = buf; + iov[0].iov_len = sizeof(struct smb2_transform_hdr) + 4; + iov[1].iov_base = buf + sizeof(struct smb2_transform_hdr) + 4; + iov[1].iov_len = buf_data_size; + rc = ksmbd_crypt_message(work, iov, 2, 0); + if (rc) + return rc; + + /* Drop the AEAD authentication tag from the inner RFC1002 frame. */ + memmove(buf + 4, iov[1].iov_base, original_msg_size); + *(__be32 *)buf = cpu_to_be32(original_msg_size); + + return rc; +} + +bool smb3_11_final_sess_setup_resp(struct ksmbd_work *work) +{ + struct ksmbd_conn *conn = work->conn; + struct ksmbd_session *sess = work->sess; + struct smb2_hdr *rsp = smb_get_msg(work->response_buf); + + if (conn->dialect < SMB30_PROT_ID) + return false; + + if (work->next_smb2_rcv_hdr_off) + rsp = ksmbd_resp_buf_next(work); + + if (le16_to_cpu(rsp->Command) == SMB2_SESSION_SETUP_HE && + sess->user && !user_guest(sess->user) && + rsp->Status == STATUS_SUCCESS) + return true; + return false; +} diff --git a/fs/smb/server/smb2pdu.h b/fs/smb/server/smb2pdu.h new file mode 100644 index 000000000..3f08d1ca5 --- /dev/null +++ b/fs/smb/server/smb2pdu.h @@ -0,0 +1,465 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2016 Namjae Jeon + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#ifndef _SMB2PDU_H +#define _SMB2PDU_H + +#include "ntlmssp.h" +#include "smbacl.h" + +/*Create Action Flags*/ +#define FILE_SUPERSEDED 0x00000000 +#define FILE_OPENED 0x00000001 +#define FILE_CREATED 0x00000002 +#define FILE_OVERWRITTEN 0x00000003 + +/* SMB2 Max Credits */ +#define SMB2_MAX_CREDITS 8192 + +/* BB FIXME - analyze following length BB */ +#define MAX_SMB2_HDR_SIZE 0x78 /* 4 len + 64 hdr + (2*24 wct) + 2 bct + 2 pad */ + +#define SMB21_DEFAULT_IOSIZE (1024 * 1024) +#define SMB3_DEFAULT_TRANS_SIZE (4 * 1024 * 1024) +#define SMB3_MIN_IOSIZE (64 * 1024) +#define SMB3_MAX_IOSIZE (8 * 1024 * 1024) +#define SMB3_MAX_MSGSIZE (4 * 4096) + +/* + * Definitions for SMB2 Protocol Data Units (network frames) + * + * See MS-SMB2.PDF specification for protocol details. + * The Naming convention is the lower case version of the SMB2 + * command code name for the struct. Note that structures must be packed. + * + */ + +struct preauth_integrity_info { + /* PreAuth integrity Hash ID */ + __le16 Preauth_HashId; + /* PreAuth integrity Hash Value */ + __u8 Preauth_HashValue[SMB2_PREAUTH_HASH_SIZE]; +}; + +/* offset is sizeof smb2_negotiate_rsp but rounded up to 8 bytes. */ +#ifdef CONFIG_SMB_SERVER_KERBEROS5 +/* sizeof(struct smb2_negotiate_rsp) = + * header(64) + response(64) + GSS_LENGTH(96) + GSS_PADDING(0) + */ +#define OFFSET_OF_NEG_CONTEXT 0xe0 +#else +/* sizeof(struct smb2_negotiate_rsp) = + * header(64) + response(64) + GSS_LENGTH(74) + GSS_PADDING(6) + */ +#define OFFSET_OF_NEG_CONTEXT 0xd0 +#endif + +#define SMB2_SESSION_EXPIRED (0) +#define SMB2_SESSION_IN_PROGRESS BIT(0) +#define SMB2_SESSION_VALID BIT(1) + +#define SMB2_SESSION_TIMEOUT (10 * HZ) + +/* Apple Defined Contexts */ +#define SMB2_CREATE_AAPL "AAPL" + +/* + * AAPL SMB2 extension -- kAAPL_SERVER_QUERY create context. + * + * Command code and bitmap values are the existing + * SMB2_CRTCTX_AAPL_* constants in fs/smb/common/smb2pdu.h. + * + * Omitting the model string when reply_bitmap includes + * SMB2_CRTCTX_AAPL_MODEL_INFO causes smbfs.kext to enter a broken + * disconnect path requiring a reboot. + * + * Layout: ccontext(16) + Name[4] + Pad[4] + cmd(4) + reserved(4) + + * reply_bitmap(8) + server_caps(8) + vol_caps(8) + * When MODEL_INFO requested, appended: pad2(4) + model_bytes(4) + UTF-16LE + */ +#define SMB2_CREATE_AAPL_LEN 4 + +/* + * Server capability flags (server_caps field) -- SMB2_CRTCTX_AAPL_UNIX_BASED: + * prevents macOS Windows-compat mode (question-mark icons). + * SMB2_CRTCTX_AAPL_SUPPORTS_OSX_COPYFILE: enables server-side file copy via + * FSCTL_SRV_COPYCHUNK. SMB2_CRTCTX_AAPL_SUPPORTS_READ_DIR_ATTR: inline + * FinderInfo per FIND entry, set when client also advertises the bit; + * format: EaSize=max_access, ShortName[0..7]=rfork_size, + * ShortName[8..23]=FinderInfo(16B), Reserved2=unix_mode. + */ +#define AAPL_SERVER_CAPS_KSMBD (SMB2_CRTCTX_AAPL_UNIX_BASED | \ + SMB2_CRTCTX_AAPL_SUPPORTS_OSX_COPYFILE | \ + SMB2_CRTCTX_AAPL_SUPPORTS_READ_DIR_ATTR) + +/* + * READDIR_ATTR_V2 (SMB2_CRTCTX_AAPL_SUPPORTS_READ_DIR_ATTR_V2, see + * fs/smb/common/smb2pdu.h) extends the same inline-FinderInfo mechanism + * above with a flags field, confirmed byte-identical to V1 otherwise + * against AAPL's actual public client behavior. When a client's own + * client_caps requests V2, the server advertises V2 instead of V1 in + * its own server_caps reply; V1 and V2 are mutually exclusive on the + * wire, not both set together. The wire format's ShortNameLength+Reserved + * (ignored in V1) become a single flags field in V2 -- + * AAPL_READDIR_ATTR_V2_NO_XATTR is the only flag bit currently defined, + * signaling the item has no xattrs/streams so the client can skip a + * separate query. + */ +#define AAPL_READDIR_ATTR_V2_NO_XATTR 0x01 + +/* Model string: up to 31 ASCII chars */ +#define AAPL_MODEL_MAX_CHARS 31 +#define AAPL_MODEL_UTF16_BYTES (AAPL_MODEL_MAX_CHARS * 2) + +/* + * Max AAPL response: header(24) + base data(32) + pad2(4) + model_bytes(4) + * + model(62), 8-byte aligned: ALIGN(126, 8) = 128 bytes. + */ +#define AAPL_RSP_MAX_SIZE 128 + +/* AAPL server query request (client->server) */ +struct aapl_server_query_req { + __le32 cmd; + __le32 reserved; + __le64 req_bitmap; + __le64 client_caps; +} __packed; + +struct create_aapl_rsp { + struct create_context_hdr ccontext; + __u8 Name[4]; + __u8 Pad[4]; + __le32 cmd; + __le32 reserved; + __le64 reply_bitmap; + __le64 server_caps; + __le64 vol_caps; + /* when MODEL_INFO requested: __le32 pad2; __le32 model_bytes; __le16 model[] */ +} __packed; + +#define DURABLE_HANDLE_MAX_TIMEOUT 300000 + +struct create_alloc_size_req { + struct create_context_hdr ccontext; + __u8 Name[8]; + __le64 AllocationSize; +} __packed; + +struct create_durable_rsp { + struct create_context_hdr ccontext; + __u8 Name[8]; + union { + __u8 Reserved[8]; + __u64 data; + } Data; +} __packed; + +/* + * See POSIX-SMB2 2.2.14.2.16 + * Link: https://gitlab.com/samba-team/smb3-posix-spec/-/blob/master/smb3_posix_extensions.md + */ +struct create_posix_rsp { + struct create_context_hdr ccontext; + __u8 Name[16]; + __le32 nlink; + __le32 reparse_tag; + __le32 mode; + /* SidBuffer contain two sids(Domain sid(28), UNIX group sid(16)) */ + u8 SidBuffer[44]; +} __packed; + +#define SMB2_0_IOCTL_IS_FSCTL 0x00000001 + +struct sockaddr_storage_rsp { + __le16 Family; + union { + struct smb_sockaddr_in addr4; + struct smb_sockaddr_in6 addr6; + }; +} __packed; + +struct file_object_buf_type1_ioctl_rsp { + __u8 ObjectId[16]; + __u8 BirthVolumeId[16]; + __u8 BirthObjectId[16]; + __u8 DomainId[16]; +} __packed; + +struct file_sparse { + __u8 SetSparse; +} __packed; + +/* FILE Info response size */ +#define FILE_DIRECTORY_INFORMATION_SIZE 1 +#define FILE_FULL_DIRECTORY_INFORMATION_SIZE 2 +#define FILE_BOTH_DIRECTORY_INFORMATION_SIZE 3 +#define FILE_BASIC_INFORMATION_SIZE 40 +#define FILE_STANDARD_INFORMATION_SIZE 24 +#define FILE_INTERNAL_INFORMATION_SIZE 8 +#define FILE_EA_INFORMATION_SIZE 4 +#define FILE_ACCESS_INFORMATION_SIZE 4 +#define FILE_NAME_INFORMATION_SIZE 9 +#define FILE_RENAME_INFORMATION_SIZE 10 +#define FILE_LINK_INFORMATION_SIZE 11 +#define FILE_NAMES_INFORMATION_SIZE 12 +#define FILE_DISPOSITION_INFORMATION_SIZE 13 +#define FILE_POSITION_INFORMATION_SIZE 14 +#define FILE_FULL_EA_INFORMATION_SIZE 15 +#define FILE_MODE_INFORMATION_SIZE 4 +#define FILE_ALIGNMENT_INFORMATION_SIZE 4 +#define FILE_ALL_INFORMATION_SIZE 104 +#define FILE_ALLOCATION_INFORMATION_SIZE 19 +#define FILE_END_OF_FILE_INFORMATION_SIZE 20 +#define FILE_ALTERNATE_NAME_INFORMATION_SIZE 8 +#define FILE_STREAM_INFORMATION_SIZE 32 +#define FILE_PIPE_INFORMATION_SIZE 23 +#define FILE_PIPE_LOCAL_INFORMATION_SIZE 24 +#define FILE_PIPE_REMOTE_INFORMATION_SIZE 25 +#define FILE_MAILSLOT_QUERY_INFORMATION_SIZE 26 +#define FILE_MAILSLOT_SET_INFORMATION_SIZE 27 +#define FILE_COMPRESSION_INFORMATION_SIZE 16 +#define FILE_OBJECT_ID_INFORMATION_SIZE 29 +/* Number 30 not defined in documents */ +#define FILE_MOVE_CLUSTER_INFORMATION_SIZE 31 +#define FILE_QUOTA_INFORMATION_SIZE 32 +#define FILE_REPARSE_POINT_INFORMATION_SIZE 33 +#define FILE_NETWORK_OPEN_INFORMATION_SIZE 56 +#define FILE_ATTRIBUTE_TAG_INFORMATION_SIZE 8 + +/* FS Info response size */ +#define FS_DEVICE_INFORMATION_SIZE 8 +#define FS_ATTRIBUTE_INFORMATION_SIZE 16 +#define FS_VOLUME_INFORMATION_SIZE 24 +#define FS_SIZE_INFORMATION_SIZE 24 +#define FS_FULL_SIZE_INFORMATION_SIZE 32 +#define FS_SECTOR_SIZE_INFORMATION_SIZE 28 +#define FS_OBJECT_ID_INFORMATION_SIZE 64 +#define FS_CONTROL_INFORMATION_SIZE 48 +#define FS_POSIX_INFORMATION_SIZE 56 + +/* FS_ATTRIBUTE_File_System_Name */ +#define FS_TYPE_SUPPORT_SIZE 44 +struct fs_type_info { + char *fs_name; + long magic_number; +} __packed; + +/* + * PDU query infolevel structure definitions + * BB consider moving to a different header + */ + +struct smb2_file_access_info { + __le32 AccessFlags; +} __packed; + +struct smb2_file_alignment_info { + __le32 AlignmentRequirement; +} __packed; + +struct smb2_file_alt_name_info { + __le32 FileNameLength; + char FileName[]; +} __packed; + +struct smb2_file_stream_info { + __le32 NextEntryOffset; + __le32 StreamNameLength; + __le64 StreamSize; + __le64 StreamAllocationSize; + char StreamName[]; +} __packed; + +struct srv_snapshot_array { + __le32 NumberOfSnapShots; + __le32 NumberOfSnapShotsReturned; + __le32 SnapShotArraySize; + __le32 Reserved; +} __packed; + +struct smb2_file_standard_info { + __le64 AllocationSize; + __le64 EndOfFile; + __le32 NumberOfLinks; /* hard links */ + __u8 DeletePending; + __u8 Directory; + __le16 Reserved; +} __packed; /* level 18 Query */ + +struct smb2_file_ea_info { + __le32 EASize; +} __packed; + +struct smb2_file_disposition_info { + __u8 DeletePending; +} __packed; + +struct smb2_file_pos_info { + __le64 CurrentByteOffset; +} __packed; + +#define FILE_MODE_INFO_MASK cpu_to_le32(0x0000100e) + +struct smb2_file_mode_info { + __le32 Mode; +} __packed; + +struct smb2_file_comp_info { + __le64 CompressedFileSize; + __le16 CompressionFormat; + __u8 CompressionUnitShift; + __u8 ChunkShift; + __u8 ClusterShift; + __u8 Reserved[3]; +} __packed; + +struct smb2_file_attr_tag_info { + __le32 FileAttributes; + __le32 ReparseTag; +} __packed; + +#define SL_RESTART_SCAN 0x00000001 +#define SL_RETURN_SINGLE_ENTRY 0x00000002 +#define SL_INDEX_SPECIFIED 0x00000004 + +struct smb2_ea_info_req { + __le32 NextEntryOffset; + __u8 EaNameLength; + char name[]; +} __packed; /* level 15 Query */ + +struct smb2_ea_info { + __le32 NextEntryOffset; + __u8 Flags; + __u8 EaNameLength; + __le16 EaValueLength; + char name[]; + /* optionally followed by value */ +} __packed; /* level 15 Query */ + +struct create_ea_buf_req { + struct create_context_hdr ccontext; + __u8 Name[8]; + struct smb2_ea_info ea; +} __packed; + +struct create_sd_buf_req { + struct create_context_hdr ccontext; + __u8 Name[8]; + struct smb_ntsd ntsd; +} __packed; + +struct smb2_posix_info { + __le32 NextEntryOffset; + __u32 Ignored; + __le64 CreationTime; + __le64 LastAccessTime; + __le64 LastWriteTime; + __le64 ChangeTime; + __le64 EndOfFile; + __le64 AllocationSize; + __le32 DosAttributes; + __le64 Inode; + __le32 DeviceId; + __le32 Zero; + /* beginning of POSIX Create Context Response */ + __le32 HardLinks; + __le32 ReparseTag; + __le32 Mode; + /* SidBuffer contain two sids (UNIX user sid(16), UNIX group sid(16)) */ + u8 SidBuffer[32]; + __le32 name_len; + u8 name[]; + /* + * var sized owner SID + * var sized group SID + * le32 filenamelength + * u8 filename[] + */ +} __packed; + +/* functions */ +void init_smb2_1_server(struct ksmbd_conn *conn); +void init_smb3_0_server(struct ksmbd_conn *conn); +void init_smb3_02_server(struct ksmbd_conn *conn); +int init_smb3_11_server(struct ksmbd_conn *conn); + +void init_smb2_max_read_size(unsigned int sz); +void init_smb2_max_write_size(unsigned int sz); +void init_smb2_max_trans_size(unsigned int sz); +void init_smb2_max_credits(unsigned int sz); + +bool is_smb2_neg_cmd(struct ksmbd_work *work); +bool is_smb2_rsp(struct ksmbd_work *work); + +u16 get_smb2_cmd_val(struct ksmbd_work *work); +void set_smb2_rsp_status(struct ksmbd_work *work, __le32 err); +int init_smb2_rsp_hdr(struct ksmbd_work *work); +int smb2_allocate_rsp_buf(struct ksmbd_work *work); +bool is_chained_smb2_message(struct ksmbd_work *work); +int init_smb2_neg_rsp(struct ksmbd_work *work); +void smb2_set_err_rsp(struct ksmbd_work *work); +int smb2_check_user_session(struct ksmbd_work *work); +int smb2_get_ksmbd_tcon(struct ksmbd_work *work); +bool smb2_is_sign_req(struct ksmbd_work *work, unsigned int command); +int smb2_check_sign_req(struct ksmbd_work *work); +void smb2_set_sign_rsp(struct ksmbd_work *work); +int smb3_check_sign_req(struct ksmbd_work *work); +void smb3_set_sign_rsp(struct ksmbd_work *work); +int find_matching_smb2_dialect(int start_index, __le16 *cli_dialects, + __le16 dialects_count); +struct file_lock *smb_flock_init(struct file *f); +int setup_async_work(struct ksmbd_work *work, void (*fn)(void **), + void **arg); +void release_async_work(struct ksmbd_work *work); +void smb2_send_interim_resp(struct ksmbd_work *work, __le32 status); +struct channel *lookup_chann_list(struct ksmbd_session *sess, + struct ksmbd_conn *conn); +void smb3_preauth_hash_rsp(struct ksmbd_work *work); +bool smb3_is_transform_hdr(void *buf); +int smb3_decrypt_req(struct ksmbd_work *work); +int smb3_encrypt_resp(struct ksmbd_work *work); +bool smb3_11_final_sess_setup_resp(struct ksmbd_work *work); +int smb2_set_rsp_credits(struct ksmbd_work *work); +bool smb3_encryption_negotiated(struct ksmbd_conn *conn); + +/* smb2 misc functions */ +int ksmbd_smb2_check_message(struct ksmbd_work *work); +void smb2_complete_request_open(struct ksmbd_work *work); + +/* smb2 command handlers */ +int smb2_handle_negotiate(struct ksmbd_work *work); +int smb2_negotiate_request(struct ksmbd_work *work); +int smb2_sess_setup(struct ksmbd_work *work); +int smb2_tree_connect(struct ksmbd_work *work); +int smb2_tree_disconnect(struct ksmbd_work *work); +int smb2_session_logoff(struct ksmbd_work *work); +int smb2_open(struct ksmbd_work *work); +int smb2_query_info(struct ksmbd_work *work); +int smb2_query_dir(struct ksmbd_work *work); +int smb2_close(struct ksmbd_work *work); +int smb2_echo(struct ksmbd_work *work); +int smb2_set_info(struct ksmbd_work *work); +int smb2_read(struct ksmbd_work *work); +int smb2_write(struct ksmbd_work *work); +int smb2_flush(struct ksmbd_work *work); +int smb2_cancel(struct ksmbd_work *work); +int smb2_lock(struct ksmbd_work *work); +int smb2_ioctl(struct ksmbd_work *work); +int smb2_oplock_break(struct ksmbd_work *work); +int smb2_notify(struct ksmbd_work *ksmbd_work); + +#define POSIX_TYPE_FILE 0 +#define POSIX_TYPE_DIR 1 +#define POSIX_TYPE_SYMLINK 2 +#define POSIX_TYPE_CHARDEV 3 +#define POSIX_TYPE_BLKDEV 4 +#define POSIX_TYPE_FIFO 5 +#define POSIX_TYPE_SOCKET 6 + +#define POSIX_FILETYPE_SHIFT 12 + +#endif /* _SMB2PDU_H */ diff --git a/fs/smb/server/smb_common.c b/fs/smb/server/smb_common.c new file mode 100644 index 000000000..4c2da6551 --- /dev/null +++ b/fs/smb/server/smb_common.c @@ -0,0 +1,879 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + * Copyright (C) 2018 Namjae Jeon + */ + +#include + +#include "smb_common.h" +#include "server.h" +#include "misc.h" +#include "../common/smb2status.h" +#include "connection.h" +#include "ksmbd_work.h" +#include "mgmt/user_session.h" +#include "mgmt/user_config.h" +#include "mgmt/tree_connect.h" +#include "mgmt/share_config.h" + +/*for shortname implementation */ +static const char *basechars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_-!@#$%"; +#define MANGLE_BASE (strlen(basechars) - 1) +#define MAGIC_CHAR '~' +#define PERIOD '.' +#define mangle(V) ((char)(basechars[(V) % MANGLE_BASE])) + +struct smb_protocol { + int index; + char *name; + char *prot; + __u16 prot_id; +}; + +static struct smb_protocol smb1_protos[] = { + { + SMB21_PROT, + "\2SMB 2.1", + "SMB2_10", + SMB21_PROT_ID + }, + { + SMB2X_PROT, + "\2SMB 2.???", + "SMB2_22", + SMB2X_PROT_ID + }, +}; + +static struct smb_protocol smb2_protos[] = { + { + SMB21_PROT, + "\2SMB 2.1", + "SMB2_10", + SMB21_PROT_ID + }, + { + SMB30_PROT, + "\2SMB 3.0", + "SMB3_00", + SMB30_PROT_ID + }, + { + SMB302_PROT, + "\2SMB 3.02", + "SMB3_02", + SMB302_PROT_ID + }, + { + SMB311_PROT, + "\2SMB 3.1.1", + "SMB3_11", + SMB311_PROT_ID + }, +}; + +unsigned int ksmbd_server_side_copy_max_chunk_count(void) +{ + return 256; +} + +unsigned int ksmbd_server_side_copy_max_chunk_size(void) +{ + return (2U << 30) - 1; +} + +unsigned int ksmbd_server_side_copy_max_total_size(void) +{ + return (2U << 30) - 1; +} + +inline int ksmbd_min_protocol(void) +{ + return SMB21_PROT; +} + +inline int ksmbd_max_protocol(void) +{ + return SMB311_PROT; +} + +static const struct { + int version; + const char *string; +} version_strings[] = { + {SMB2_PROT, SMB20_VERSION_STRING}, + {SMB21_PROT, SMB21_VERSION_STRING}, + {SMB30_PROT, SMB30_VERSION_STRING}, + {SMB302_PROT, SMB302_VERSION_STRING}, + {SMB311_PROT, SMB311_VERSION_STRING}, +}; + +const char *ksmbd_get_protocol_string(int version) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(version_strings); i++) { + if (version_strings[i].version == version) + return version_strings[i].string; + } + return ""; +} +int ksmbd_lookup_protocol_idx(char *str) +{ + int offt = ARRAY_SIZE(smb1_protos) - 1; + int len = strlen(str); + + while (offt >= 0) { + if (!strncmp(str, smb1_protos[offt].prot, len)) { + ksmbd_debug(SMB, "selected %s dialect idx = %d\n", + smb1_protos[offt].prot, offt); + return smb1_protos[offt].index; + } + offt--; + } + + offt = ARRAY_SIZE(smb2_protos) - 1; + while (offt >= 0) { + if (!strncmp(str, smb2_protos[offt].prot, len)) { + ksmbd_debug(SMB, "selected %s dialect idx = %d\n", + smb2_protos[offt].prot, offt); + return smb2_protos[offt].index; + } + offt--; + } + return -1; +} + +/** + * ksmbd_verify_smb_message() - check for valid smb2 request header + * @work: smb work + * + * check for valid smb signature and packet direction(request/response) + * + * Return: 0 on success, otherwise -EINVAL + */ +int ksmbd_verify_smb_message(struct ksmbd_work *work) +{ + struct smb2_hdr *smb2_hdr = ksmbd_req_buf_next(work); + struct smb_hdr *hdr; + + if (smb2_hdr->ProtocolId == SMB2_PROTO_NUMBER) + return ksmbd_smb2_check_message(work); + + hdr = smb_get_msg(work->request_buf); + if (*(__le32 *)hdr->Protocol == SMB1_PROTO_NUMBER && + hdr->Command == SMB_COM_NEGOTIATE) { + struct ksmbd_conn *conn = work->conn; + + conn->outstanding_credits++; + /* + * A legacy SMB1 multi-protocol negotiate occupies sequence + * number 0 but does not pass through + * ksmbd_smb2_check_message(). Consume it here so that, after + * the connection is upgraded to SMB2, the command sequence + * window can advance instead of staying pinned at 0. + */ + spin_lock(&conn->credits_lock); + if (conn->seq_low == 0) { + __clear_bit(0, conn->seq_bitmap); + conn->seq_low = 1; + } + spin_unlock(&conn->credits_lock); + return 0; + } + + return -EINVAL; +} + +/** + * ksmbd_smb_request() - check for valid smb request type + * @conn: connection instance + * + * Return: true on success, otherwise false + */ +bool ksmbd_smb_request(struct ksmbd_conn *conn) +{ + __le32 *proto; + + if (conn->request_buf[0] != 0) + return false; + + proto = (__le32 *)smb_get_msg(conn->request_buf); + if (*proto != SMB1_PROTO_NUMBER && + *proto != SMB2_PROTO_NUMBER && + *proto != SMB2_TRANSFORM_PROTO_NUM) + return false; + + return true; +} + +static bool supported_protocol(int idx) +{ + if (idx == SMB2X_PROT && + (server_conf.min_protocol >= SMB21_PROT || + server_conf.max_protocol <= SMB311_PROT)) + return true; + + return (server_conf.min_protocol <= idx && + idx <= server_conf.max_protocol); +} + +static char *next_dialect(char *dialect, int *next_off, int bcount) +{ + dialect = dialect + *next_off; + *next_off = strnlen(dialect, bcount); + if (dialect[*next_off] != '\0') + return NULL; + return dialect; +} + +static int ksmbd_lookup_dialect_by_name(char *cli_dialects, __le16 byte_count) +{ + int i, seq_num, bcount, next; + char *dialect; + + for (i = ARRAY_SIZE(smb1_protos) - 1; i >= 0; i--) { + seq_num = 0; + next = 0; + dialect = cli_dialects; + bcount = le16_to_cpu(byte_count); + do { + dialect = next_dialect(dialect, &next, bcount); + if (!dialect) + break; + ksmbd_debug(SMB, "client requested dialect %s\n", + dialect); + if (!strcmp(dialect, smb1_protos[i].name)) { + if (supported_protocol(smb1_protos[i].index)) { + ksmbd_debug(SMB, + "selected %s dialect\n", + smb1_protos[i].name); + if (smb1_protos[i].index == SMB1_PROT) + return seq_num; + return smb1_protos[i].prot_id; + } + } + seq_num++; + bcount -= (++next); + } while (bcount > 0); + } + + return BAD_PROT_ID; +} + +int ksmbd_lookup_dialect_by_id(__le16 *cli_dialects, __le16 dialects_count) +{ + int i; + int count; + + for (i = ARRAY_SIZE(smb2_protos) - 1; i >= 0; i--) { + count = le16_to_cpu(dialects_count); + while (--count >= 0) { + ksmbd_debug(SMB, "client requested dialect 0x%x\n", + le16_to_cpu(cli_dialects[count])); + if (le16_to_cpu(cli_dialects[count]) != + smb2_protos[i].prot_id) + continue; + + if (supported_protocol(smb2_protos[i].index)) { + ksmbd_debug(SMB, "selected %s dialect\n", + smb2_protos[i].name); + return smb2_protos[i].prot_id; + } + } + } + + return BAD_PROT_ID; +} + +static int ksmbd_negotiate_smb_dialect(void *buf) +{ + int smb_buf_length = get_rfc1002_len(buf); + __le32 proto = ((struct smb2_hdr *)smb_get_msg(buf))->ProtocolId; + + if (proto == SMB2_PROTO_NUMBER) { + struct smb2_negotiate_req *req; + int smb2_neg_size = + offsetof(struct smb2_negotiate_req, Dialects); + + req = (struct smb2_negotiate_req *)smb_get_msg(buf); + if (smb2_neg_size > smb_buf_length) + goto err_out; + + if (struct_size(req, Dialects, le16_to_cpu(req->DialectCount)) > + smb_buf_length) + goto err_out; + + return ksmbd_lookup_dialect_by_id(req->Dialects, + req->DialectCount); + } + + if (proto == SMB1_PROTO_NUMBER) { + struct smb_negotiate_req *req; + + req = (struct smb_negotiate_req *)smb_get_msg(buf); + if (le16_to_cpu(req->ByteCount) < 2) + goto err_out; + + if (offsetof(struct smb_negotiate_req, DialectsArray) + + le16_to_cpu(req->ByteCount) > smb_buf_length) { + goto err_out; + } + + return ksmbd_lookup_dialect_by_name(req->DialectsArray, + req->ByteCount); + } + +err_out: + return BAD_PROT_ID; +} + +#define SMB_COM_NEGOTIATE_EX 0x0 + +/** + * get_smb1_cmd_val() - get smb command value from smb header + * @work: smb work containing smb header + * + * Return: smb command value + */ +static u16 get_smb1_cmd_val(struct ksmbd_work *work) +{ + return SMB_COM_NEGOTIATE_EX; +} + +/** + * init_smb1_rsp_hdr() - initialize smb negotiate response header + * @work: smb work containing smb request + * + * Return: 0 on success, otherwise -EINVAL + */ +static int init_smb1_rsp_hdr(struct ksmbd_work *work) +{ + struct smb_hdr *rsp_hdr = (struct smb_hdr *)smb_get_msg(work->response_buf); + struct smb_hdr *rcv_hdr = (struct smb_hdr *)smb_get_msg(work->request_buf); + + rsp_hdr->Command = SMB_COM_NEGOTIATE; + *(__le32 *)rsp_hdr->Protocol = SMB1_PROTO_NUMBER; + rsp_hdr->Flags = SMBFLG_RESPONSE; + rsp_hdr->Flags2 = SMBFLG2_UNICODE | SMBFLG2_ERR_STATUS | + SMBFLG2_EXT_SEC | SMBFLG2_IS_LONG_NAME; + rsp_hdr->Pid = rcv_hdr->Pid; + rsp_hdr->Mid = rcv_hdr->Mid; + return 0; +} + +/** + * smb1_check_user_session() - check for valid session for a user + * @work: smb work containing smb request buffer + * + * Return: 0 on success, otherwise error + */ +static int smb1_check_user_session(struct ksmbd_work *work) +{ + unsigned int cmd = work->conn->ops->get_cmd_val(work); + + if (cmd == SMB_COM_NEGOTIATE_EX) + return 0; + + return -EINVAL; +} + +/** + * smb1_allocate_rsp_buf() - allocate response buffer for a command + * @work: smb work containing smb request + * + * Return: 0 on success, otherwise -ENOMEM + */ +static int smb1_allocate_rsp_buf(struct ksmbd_work *work) +{ + work->response_buf = kzalloc(MAX_CIFS_SMALL_BUFFER_SIZE, + KSMBD_DEFAULT_GFP); + work->response_sz = MAX_CIFS_SMALL_BUFFER_SIZE; + + if (!work->response_buf) { + pr_err("Failed to allocate %u bytes buffer\n", + MAX_CIFS_SMALL_BUFFER_SIZE); + return -ENOMEM; + } + + return 0; +} + +/** + * set_smb1_rsp_status() - set error type in smb response header + * @work: smb work containing smb response header + * @err: error code to set in response + */ +static void set_smb1_rsp_status(struct ksmbd_work *work, __le32 err) +{ + work->send_no_response = 1; +} + +static struct smb_version_ops smb1_server_ops = { + .get_cmd_val = get_smb1_cmd_val, + .init_rsp_hdr = init_smb1_rsp_hdr, + .allocate_rsp_buf = smb1_allocate_rsp_buf, + .check_user_session = smb1_check_user_session, + .set_rsp_status = set_smb1_rsp_status, +}; + +static struct smb_version_values smb1_server_values = { + .max_credits = SMB2_MAX_CREDITS, +}; + +static int smb1_negotiate(struct ksmbd_work *work) +{ + return ksmbd_smb_negotiate_common(work, SMB_COM_NEGOTIATE); +} + +static struct smb_version_cmds smb1_server_cmds[1] = { + [SMB_COM_NEGOTIATE_EX] = { .proc = smb1_negotiate, }, +}; + +static int init_smb1_server(struct ksmbd_conn *conn) +{ + conn->vals = &smb1_server_values; + conn->ops = &smb1_server_ops; + conn->cmds = smb1_server_cmds; + conn->max_cmds = ARRAY_SIZE(smb1_server_cmds); + return 0; +} + +int ksmbd_init_smb_server(struct ksmbd_conn *conn) +{ + struct smb_hdr *rcv_hdr = (struct smb_hdr *)smb_get_msg(conn->request_buf); + __le32 proto; + + proto = *(__le32 *)rcv_hdr->Protocol; + if (conn->need_neg == false) { + if (proto == SMB1_PROTO_NUMBER) + return -EINVAL; + return 0; + } + + if (proto == SMB1_PROTO_NUMBER) + return init_smb1_server(conn); + return init_smb3_11_server(conn); +} + +int ksmbd_populate_dot_dotdot_entries(struct ksmbd_work *work, int info_level, + struct ksmbd_file *dir, + struct ksmbd_dir_info *d_info, + char *search_pattern, + int (*fn)(struct ksmbd_conn *, int, + struct ksmbd_dir_info *, + struct ksmbd_kstat *)) +{ + int i, rc = 0; + struct ksmbd_conn *conn = work->conn; + struct mnt_idmap *idmap = file_mnt_idmap(dir->filp); + + for (i = 0; i < 2; i++) { + struct kstat kstat; + struct ksmbd_kstat ksmbd_kstat; + struct dentry *dentry; + + if (!dir->dot_dotdot[i]) { /* fill dot entry info */ + if (i == 0) { + d_info->name = "."; + d_info->name_len = 1; + dentry = dir->filp->f_path.dentry; + } else { + d_info->name = ".."; + d_info->name_len = 2; + dentry = dir->filp->f_path.dentry->d_parent; + } + + if (!match_pattern(d_info->name, d_info->name_len, + search_pattern)) { + dir->dot_dotdot[i] = 1; + continue; + } + + ksmbd_kstat.kstat = &kstat; + rc = ksmbd_vfs_fill_dentry_attrs(work, + idmap, + dentry, + &ksmbd_kstat); + if (rc) + break; + + rc = fn(conn, info_level, d_info, &ksmbd_kstat); + if (rc) + break; + if (d_info->out_buf_len <= 0) + break; + + dir->dot_dotdot[i] = 1; + if (d_info->flags & SMB2_RETURN_SINGLE_ENTRY) { + d_info->out_buf_len = 0; + break; + } + } + } + + return rc; +} + +/** + * ksmbd_extract_shortname() - get shortname from long filename + * @conn: connection instance + * @longname: source long filename + * @shortname: destination short filename + * + * Return: shortname length or 0 when source long name is '.' or '..' + * TODO: Though this function conforms the restriction of 8.3 Filename spec, + * but the result is different with Windows 7's one. need to check. + */ +int ksmbd_extract_shortname(struct ksmbd_conn *conn, const char *longname, + char *shortname) +{ + const char *p; + char base[9], extension[4]; + char out[13] = {0}; + int baselen = 0; + int extlen = 0, len = 0; + unsigned int csum = 0; + const unsigned char *ptr; + bool dot_present = true; + + p = longname; + if ((*p == '.') || (!(strcmp(p, "..")))) { + /*no mangling required */ + return 0; + } + + p = strrchr(longname, '.'); + if (p == longname) { /*name starts with a dot*/ + strscpy(extension, "___", sizeof(extension)); + } else { + if (p) { + p++; + while (*p && extlen < 3) { + if (*p != '.') + extension[extlen++] = toupper(*p); + p++; + } + extension[extlen] = '\0'; + } else { + dot_present = false; + } + } + + p = longname; + if (*p == '.') { + p++; + longname++; + } + while (*p && (baselen < 5)) { + if (*p != '.') + base[baselen++] = toupper(*p); + p++; + } + + base[baselen] = MAGIC_CHAR; + memcpy(out, base, baselen + 1); + + ptr = longname; + len = strlen(longname); + for (; len > 0; len--, ptr++) + csum += *ptr; + + csum = csum % (MANGLE_BASE * MANGLE_BASE); + out[baselen + 1] = mangle(csum / MANGLE_BASE); + out[baselen + 2] = mangle(csum); + out[baselen + 3] = PERIOD; + + if (dot_present) + memcpy(out + baselen + 4, extension, 4); + else + out[baselen + 4] = '\0'; + smbConvertToUTF16((__le16 *)shortname, out, PATH_MAX, + conn->local_nls, 0); + len = strlen(out) * 2; + return len; +} + +static int __smb2_negotiate(struct ksmbd_conn *conn) +{ + return (conn->dialect >= SMB20_PROT_ID && + conn->dialect <= SMB311_PROT_ID); +} + +static int smb_handle_negotiate(struct ksmbd_work *work) +{ + struct smb_negotiate_rsp *neg_rsp = smb_get_msg(work->response_buf); + + ksmbd_debug(SMB, "Unsupported SMB1 protocol\n"); + + if (ksmbd_iov_pin_rsp(work, (void *)neg_rsp, + sizeof(struct smb_negotiate_rsp))) + return -ENOMEM; + + neg_rsp->hdr.Status.CifsError = STATUS_SUCCESS; + neg_rsp->hdr.WordCount = 1; + neg_rsp->DialectIndex = cpu_to_le16(work->conn->dialect); + neg_rsp->ByteCount = 0; + return 0; +} + +int ksmbd_smb_negotiate_common(struct ksmbd_work *work, unsigned int command) +{ + struct ksmbd_conn *conn = work->conn; + int ret; + + if (command == SMB2_NEGOTIATE_HE) { + /* + * An SMB2 NEGOTIATE is valid for a new connection, or after an + * SMB1 multi-protocol negotiate has selected SMB2. Do not allow + * a second SMB2 NEGOTIATE to replace connection-wide state + * while a session setup is pending. KSMBD_SESS_NEED_RECONNECT + * is a transient session state and does not restart transport + * negotiation. + */ + ksmbd_conn_lock(conn); + if (!ksmbd_conn_new(conn) && + !ksmbd_conn_need_negotiate(conn)) { + work->send_no_response = 1; + ksmbd_conn_set_exiting(conn); + ksmbd_conn_unlock(conn); + return 0; + } + + conn->dialect = + ksmbd_negotiate_smb_dialect(work->request_buf); + ksmbd_debug(SMB, "conn->dialect 0x%x\n", conn->dialect); + ret = smb2_handle_negotiate(work); + ksmbd_conn_unlock(conn); + return ret; + } + + if (command == SMB_COM_NEGOTIATE) { + ksmbd_conn_lock(conn); + conn->dialect = + ksmbd_negotiate_smb_dialect(work->request_buf); + ksmbd_debug(SMB, "conn->dialect 0x%x\n", conn->dialect); + if (__smb2_negotiate(conn)) { + init_smb3_11_server(conn); + ret = init_smb2_neg_rsp(work); + ksmbd_debug(SMB, "Upgrade to SMB2 negotiation\n"); + } else { + ret = smb_handle_negotiate(work); + } + ksmbd_conn_unlock(conn); + return ret; + } + + pr_err("Unknown SMB negotiation command: %u\n", command); + return -EINVAL; +} + +enum SHARED_MODE_ERRORS { + SHARE_DELETE_ERROR, + SHARE_READ_ERROR, + SHARE_WRITE_ERROR, + FILE_READ_ERROR, + FILE_WRITE_ERROR, + FILE_DELETE_ERROR, +}; + +static const char * const shared_mode_errors[] = { + "Current access mode does not permit SHARE_DELETE", + "Current access mode does not permit SHARE_READ", + "Current access mode does not permit SHARE_WRITE", + "Desired access mode does not permit FILE_READ", + "Desired access mode does not permit FILE_WRITE", + "Desired access mode does not permit FILE_DELETE", +}; + +static void smb_shared_mode_error(int error, struct ksmbd_file *prev_fp, + struct ksmbd_file *curr_fp) +{ + ksmbd_debug(SMB, "%s\n", shared_mode_errors[error]); + ksmbd_debug(SMB, "Current mode: 0x%x Desired mode: 0x%x\n", + prev_fp->saccess, curr_fp->daccess); +} + +int ksmbd_smb_check_shared_mode(struct file *filp, struct ksmbd_file *curr_fp) +{ + int rc = 0; + struct ksmbd_file *prev_fp; + + /* + * Lookup fp in master fp list, and check desired access and + * shared mode between previous open and current open. + */ + down_read(&curr_fp->f_ci->m_lock); + list_for_each_entry(prev_fp, &curr_fp->f_ci->m_fp_list, node) { + if (file_inode(filp) != file_inode(prev_fp->filp)) + continue; + + if (filp == prev_fp->filp) + continue; + + if (ksmbd_stream_fd(prev_fp) && ksmbd_stream_fd(curr_fp)) + if (strcmp(prev_fp->stream.name, curr_fp->stream.name)) + continue; + + if (prev_fp->attrib_only != curr_fp->attrib_only) + continue; + + if (!(prev_fp->saccess & FILE_SHARE_DELETE_LE) && + curr_fp->daccess & FILE_DELETE_LE) { + smb_shared_mode_error(SHARE_DELETE_ERROR, + prev_fp, + curr_fp); + rc = -EPERM; + break; + } + + /* + * Only check FILE_SHARE_DELETE if stream opened and + * normal file opened. + */ + if (ksmbd_stream_fd(prev_fp) && !ksmbd_stream_fd(curr_fp)) + continue; + + if (!(prev_fp->saccess & FILE_SHARE_READ_LE) && + curr_fp->daccess & (FILE_EXECUTE_LE | FILE_READ_DATA_LE)) { + smb_shared_mode_error(SHARE_READ_ERROR, + prev_fp, + curr_fp); + rc = -EPERM; + break; + } + + if (!(prev_fp->saccess & FILE_SHARE_WRITE_LE) && + curr_fp->daccess & (FILE_WRITE_DATA_LE | FILE_APPEND_DATA_LE)) { + smb_shared_mode_error(SHARE_WRITE_ERROR, + prev_fp, + curr_fp); + rc = -EPERM; + break; + } + + if (prev_fp->daccess & (FILE_EXECUTE_LE | FILE_READ_DATA_LE) && + !(curr_fp->saccess & FILE_SHARE_READ_LE)) { + smb_shared_mode_error(FILE_READ_ERROR, + prev_fp, + curr_fp); + rc = -EPERM; + break; + } + + if (prev_fp->daccess & (FILE_WRITE_DATA_LE | FILE_APPEND_DATA_LE) && + !(curr_fp->saccess & FILE_SHARE_WRITE_LE)) { + smb_shared_mode_error(FILE_WRITE_ERROR, + prev_fp, + curr_fp); + rc = -EPERM; + break; + } + + if (prev_fp->daccess & FILE_DELETE_LE && + !(curr_fp->saccess & FILE_SHARE_DELETE_LE)) { + smb_shared_mode_error(FILE_DELETE_ERROR, + prev_fp, + curr_fp); + rc = -EPERM; + break; + } + } + up_read(&curr_fp->f_ci->m_lock); + + return rc; +} + +bool is_asterisk(char *p) +{ + return p && p[0] == '*'; +} + +int __ksmbd_override_fsids(struct ksmbd_work *work, + struct ksmbd_share_config *share) +{ + struct ksmbd_session *sess = work->sess; + struct ksmbd_user *user = sess->user; + struct cred *cred; + struct group_info *gi; + unsigned int uid; + unsigned int gid; + int i; + + uid = user_uid(user); + gid = user_gid(user); + if (share->force_uid != KSMBD_SHARE_INVALID_UID) + uid = share->force_uid; + if (share->force_gid != KSMBD_SHARE_INVALID_GID) + gid = share->force_gid; + + cred = prepare_kernel_cred(&init_task); + if (!cred) + return -ENOMEM; + + cred->fsuid = make_kuid(&init_user_ns, uid); + cred->fsgid = make_kgid(&init_user_ns, gid); + + gi = groups_alloc(user->ngroups); + if (!gi) { + abort_creds(cred); + return -ENOMEM; + } + + for (i = 0; i < user->ngroups; i++) + gi->gid[i] = make_kgid(&init_user_ns, user->sgid[i]); + + if (user->ngroups) + groups_sort(gi); + + set_groups(cred, gi); + put_group_info(gi); + + if (!uid_eq(cred->fsuid, GLOBAL_ROOT_UID)) + cred->cap_effective = cap_drop_fs_set(cred->cap_effective); + + WARN_ON(work->saved_cred); + work->saved_cred = override_creds(cred); + return 0; +} + +int ksmbd_override_fsids(struct ksmbd_work *work) +{ + return __ksmbd_override_fsids(work, work->tcon->share_conf); +} + +void ksmbd_revert_fsids(struct ksmbd_work *work) +{ + const struct cred *cred; + WARN_ON(!work->saved_cred); + + cred = revert_creds(work->saved_cred); + work->saved_cred = NULL; + put_cred(cred); +} + +__le32 smb_map_generic_desired_access(__le32 daccess) +{ + if (daccess & FILE_GENERIC_READ_LE) { + daccess |= cpu_to_le32(GENERIC_READ_FLAGS); + daccess &= ~FILE_GENERIC_READ_LE; + } + + if (daccess & FILE_GENERIC_WRITE_LE) { + daccess |= cpu_to_le32(GENERIC_WRITE_FLAGS); + daccess &= ~FILE_GENERIC_WRITE_LE; + } + + if (daccess & FILE_GENERIC_EXECUTE_LE) { + daccess |= cpu_to_le32(GENERIC_EXECUTE_FLAGS); + daccess &= ~FILE_GENERIC_EXECUTE_LE; + } + + if (daccess & FILE_GENERIC_ALL_LE) { + daccess |= cpu_to_le32(GENERIC_ALL_FLAGS); + daccess &= ~FILE_GENERIC_ALL_LE; + } + + return daccess; +} diff --git a/fs/smb/server/smb_common.h b/fs/smb/server/smb_common.h new file mode 100644 index 000000000..7b9c5cfcb --- /dev/null +++ b/fs/smb/server/smb_common.h @@ -0,0 +1,209 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#ifndef __SMB_SERVER_COMMON_H__ +#define __SMB_SERVER_COMMON_H__ + +#include + +#include "glob.h" +#include "../common/smbglob.h" +#include "../common/smb1pdu.h" +#include "../common/smb2pdu.h" +#include "../common/fscc.h" +#include "smb2pdu.h" + +/* ksmbd's Specific ERRNO */ +#define ESHARE 50000 + +#define SMB1_PROT 0 +#define SMB2_PROT 1 +#define SMB21_PROT 2 +/* multi-protocol negotiate request */ +#define SMB2X_PROT 3 +#define SMB30_PROT 4 +#define SMB302_PROT 5 +#define SMB311_PROT 6 +#define BAD_PROT 0xFFFF + +#define SMB_ECHO_INTERVAL (60 * HZ) + +#define MAX_STREAM_PROT_LEN 0x00FFFFFF + +/* Responses when opening a file. */ +#define F_SUPERSEDED 0 +#define F_OPENED 1 +#define F_CREATED 2 +#define F_OVERWRITTEN 3 + +/* Combinations of file access permission bits */ +#define SET_FILE_READ_RIGHTS (FILE_READ_DATA | FILE_READ_EA \ + | FILE_READ_ATTRIBUTES \ + | DELETE | READ_CONTROL | WRITE_DAC \ + | WRITE_OWNER | SYNCHRONIZE) +#define SET_FILE_WRITE_RIGHTS (FILE_WRITE_DATA | FILE_APPEND_DATA \ + | FILE_WRITE_EA \ + | FILE_DELETE_CHILD \ + | FILE_WRITE_ATTRIBUTES \ + | DELETE | READ_CONTROL | WRITE_DAC \ + | WRITE_OWNER | SYNCHRONIZE) + +/* generic flags for file open */ +#define GENERIC_READ_FLAGS (READ_CONTROL | FILE_READ_DATA | \ + FILE_READ_ATTRIBUTES | \ + FILE_READ_EA | SYNCHRONIZE) + +#define GENERIC_WRITE_FLAGS (READ_CONTROL | FILE_WRITE_DATA | \ + FILE_WRITE_ATTRIBUTES | FILE_WRITE_EA | \ + FILE_APPEND_DATA | SYNCHRONIZE) + +#define GENERIC_EXECUTE_FLAGS (READ_CONTROL | FILE_EXECUTE | \ + FILE_READ_ATTRIBUTES | SYNCHRONIZE) + +#define GENERIC_ALL_FLAGS (DELETE | READ_CONTROL | WRITE_DAC | \ + WRITE_OWNER | SYNCHRONIZE | FILE_READ_DATA | \ + FILE_WRITE_DATA | FILE_APPEND_DATA | \ + FILE_READ_EA | FILE_WRITE_EA | \ + FILE_EXECUTE | FILE_DELETE_CHILD | \ + FILE_READ_ATTRIBUTES | FILE_WRITE_ATTRIBUTES) + +#define SMB_COM_NEGOTIATE 0x72 /* See MS-CIFS 2.2.2.1 */ + +/* See MS-CIFS 2.2.3.1 */ +#define SMBFLG_RESPONSE 0x80 /* this PDU is a response from server */ + +/* + * See MS-CIFS 2.2.3.1 + * MS-SMB 2.2.3.1 + */ +#define SMBFLG2_IS_LONG_NAME cpu_to_le16(0x40) +#define SMBFLG2_EXT_SEC cpu_to_le16(0x800) +#define SMBFLG2_ERR_STATUS cpu_to_le16(0x4000) +#define SMBFLG2_UNICODE cpu_to_le16(0x8000) + +/* See MS-CIFS 2.2.4.52.2 */ +struct smb_negotiate_rsp { + struct smb_hdr hdr; /* wct = 17 */ + __le16 DialectIndex; /* 0xFFFF = no dialect acceptable */ + __le16 ByteCount; +} __packed; + +#define EXTENDED_INFO_MAGIC 0x43667364 /* Cfsd */ +#define STRING_LENGTH 28 + +struct fs_extended_info { + __le32 magic; + __le32 version; + __le32 release; + __u64 rel_date; + char version_string[STRING_LENGTH]; +} __packed; + +struct object_id_info { + char objid[16]; + struct fs_extended_info extended_info; +} __packed; + +struct file_names_info { + __le32 NextEntryOffset; + __u32 FileIndex; + __le32 FileNameLength; + char FileName[]; +} __packed; /* level 0xc FF resp data */ + +struct file_id_both_directory_info { + __le32 NextEntryOffset; + __u32 FileIndex; + __le64 CreationTime; + __le64 LastAccessTime; + __le64 LastWriteTime; + __le64 ChangeTime; + __le64 EndOfFile; + __le64 AllocationSize; + __le32 ExtFileAttributes; + __le32 FileNameLength; + __le32 EaSize; /* length of the xattrs */ + __u8 ShortNameLength; + __u8 Reserved; + __u8 ShortName[24]; + __le16 Reserved2; + __le64 UniqueId; + char FileName[]; +} __packed; + +struct smb_version_ops { + u16 (*get_cmd_val)(struct ksmbd_work *swork); + void (*inc_reqs)(unsigned int cmd, __le32 status); + int (*init_rsp_hdr)(struct ksmbd_work *swork); + void (*set_rsp_status)(struct ksmbd_work *swork, __le32 err); + int (*allocate_rsp_buf)(struct ksmbd_work *work); + int (*set_rsp_credits)(struct ksmbd_work *work); + int (*check_user_session)(struct ksmbd_work *work); + int (*get_ksmbd_tcon)(struct ksmbd_work *work); + bool (*is_sign_req)(struct ksmbd_work *work, unsigned int command); + int (*check_sign_req)(struct ksmbd_work *work); + void (*set_sign_rsp)(struct ksmbd_work *work); + int (*generate_signingkey)(struct ksmbd_session *sess, struct ksmbd_conn *conn); + void (*generate_encryptionkey)(struct ksmbd_conn *conn, struct ksmbd_session *sess); + bool (*is_transform_hdr)(void *buf); + int (*decrypt_req)(struct ksmbd_work *work); + int (*encrypt_resp)(struct ksmbd_work *work); +}; + +struct smb_version_cmds { + int (*proc)(struct ksmbd_work *swork); +}; + +int ksmbd_min_protocol(void); +int ksmbd_max_protocol(void); +const char *ksmbd_get_protocol_string(int version); + +int ksmbd_lookup_protocol_idx(char *str); + +int ksmbd_verify_smb_message(struct ksmbd_work *work); +bool ksmbd_smb_request(struct ksmbd_conn *conn); + +int ksmbd_lookup_dialect_by_id(__le16 *cli_dialects, __le16 dialects_count); + +int ksmbd_init_smb_server(struct ksmbd_conn *conn); + +struct ksmbd_kstat; +int ksmbd_populate_dot_dotdot_entries(struct ksmbd_work *work, + int info_level, + struct ksmbd_file *dir, + struct ksmbd_dir_info *d_info, + char *search_pattern, + int (*fn)(struct ksmbd_conn *, + int, + struct ksmbd_dir_info *, + struct ksmbd_kstat *)); + +int ksmbd_extract_shortname(struct ksmbd_conn *conn, + const char *longname, + char *shortname); + +int ksmbd_smb_negotiate_common(struct ksmbd_work *work, unsigned int command); + +int ksmbd_smb_check_shared_mode(struct file *filp, struct ksmbd_file *curr_fp); +int __ksmbd_override_fsids(struct ksmbd_work *work, + struct ksmbd_share_config *share); +int ksmbd_override_fsids(struct ksmbd_work *work); +void ksmbd_revert_fsids(struct ksmbd_work *work); + +unsigned int ksmbd_server_side_copy_max_chunk_count(void); +unsigned int ksmbd_server_side_copy_max_chunk_size(void); +unsigned int ksmbd_server_side_copy_max_total_size(void); +bool is_asterisk(char *p); +__le32 smb_map_generic_desired_access(__le32 daccess); + +/* + * Get the body of the smb message excluding the 4 byte rfc1002 headers + * from request/response buffer. + */ +static inline void *smb_get_msg(void *buf) +{ + return buf + 4; +} +#endif /* __SMB_SERVER_COMMON_H__ */ diff --git a/fs/smb/server/smbacl.c b/fs/smb/server/smbacl.c new file mode 100644 index 000000000..8ad2e5a5c --- /dev/null +++ b/fs/smb/server/smbacl.c @@ -0,0 +1,1735 @@ +// SPDX-License-Identifier: LGPL-2.1+ +/* + * Copyright (C) International Business Machines Corp., 2007,2008 + * Author(s): Steve French (sfrench@us.ibm.com) + * Copyright (C) 2020 Samsung Electronics Co., Ltd. + * Author(s): Namjae Jeon + */ + +#include +#include +#include +#include + +#include "smbacl.h" +#include "smb_common.h" +#include "server.h" +#include "misc.h" +#include "mgmt/share_config.h" + +static const struct smb_sid domain = {1, 4, {0, 0, 0, 0, 0, 5}, + {cpu_to_le32(21), cpu_to_le32(1), cpu_to_le32(2), cpu_to_le32(3), + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} }; + +/* security id for everyone/world system group */ +static const struct smb_sid creator_owner = { + 1, 1, {0, 0, 0, 0, 0, 3}, {0} }; +/* security id for everyone/world system group */ +static const struct smb_sid creator_group = { + 1, 1, {0, 0, 0, 0, 0, 3}, {cpu_to_le32(1)} }; +/* security id for owner rights */ +static const struct smb_sid sid_owner_rights = { + 1, 1, {0, 0, 0, 0, 0, 3}, {cpu_to_le32(4)} }; + +/* security id for everyone/world system group */ +static const struct smb_sid sid_everyone = { + 1, 1, {0, 0, 0, 0, 0, 1}, {0} }; +/* security id for Authenticated Users system group */ +static const struct smb_sid sid_authusers = { + 1, 1, {0, 0, 0, 0, 0, 5}, {cpu_to_le32(11)} }; + +/* S-1-22-1 Unmapped Unix users */ +static const struct smb_sid sid_unix_users = {1, 1, {0, 0, 0, 0, 0, 22}, + {cpu_to_le32(1), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} }; + +/* S-1-22-2 Unmapped Unix groups */ +static const struct smb_sid sid_unix_groups = { 1, 1, {0, 0, 0, 0, 0, 22}, + {cpu_to_le32(2), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} }; + +/* + * See http://technet.microsoft.com/en-us/library/hh509017(v=ws.10).aspx + */ + +/* S-1-5-88 MS NFS and Apple style UID/GID/mode */ + +/* S-1-5-88-1 Unix uid */ +static const struct smb_sid sid_unix_NFS_users = { 1, 2, {0, 0, 0, 0, 0, 5}, + {cpu_to_le32(88), + cpu_to_le32(1), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} }; + +/* S-1-5-88-2 Unix gid */ +static const struct smb_sid sid_unix_NFS_groups = { 1, 2, {0, 0, 0, 0, 0, 5}, + {cpu_to_le32(88), + cpu_to_le32(2), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} }; + +/* S-1-5-88-3 Unix mode */ +static const struct smb_sid sid_unix_NFS_mode = { 1, 2, {0, 0, 0, 0, 0, 5}, + {cpu_to_le32(88), + cpu_to_le32(3), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} }; + +/* + * if the two SIDs (roughly equivalent to a UUID for a user or group) are + * the same returns zero, if they do not match returns non-zero. + */ +int compare_sids(const struct smb_sid *ctsid, const struct smb_sid *cwsid) +{ + int i; + int num_subauth, num_sat, num_saw; + + if (!ctsid || !cwsid) + return 1; + + /* compare the revision */ + if (ctsid->revision != cwsid->revision) { + if (ctsid->revision > cwsid->revision) + return 1; + else + return -1; + } + + /* compare all of the six auth values */ + for (i = 0; i < NUM_AUTHS; ++i) { + if (ctsid->authority[i] != cwsid->authority[i]) { + if (ctsid->authority[i] > cwsid->authority[i]) + return 1; + else + return -1; + } + } + + /* compare all of the subauth values if any */ + num_sat = ctsid->num_subauth; + num_saw = cwsid->num_subauth; + num_subauth = min(num_sat, num_saw); + if (num_subauth) { + for (i = 0; i < num_subauth; ++i) { + if (ctsid->sub_auth[i] != cwsid->sub_auth[i]) { + if (le32_to_cpu(ctsid->sub_auth[i]) > + le32_to_cpu(cwsid->sub_auth[i])) + return 1; + else + return -1; + } + } + } + + return 0; /* sids compare/match */ +} + +static void smb_copy_sid(struct smb_sid *dst, const struct smb_sid *src) +{ + int i; + + dst->revision = src->revision; + dst->num_subauth = min_t(u8, src->num_subauth, SID_MAX_SUB_AUTHORITIES); + for (i = 0; i < NUM_AUTHS; ++i) + dst->authority[i] = src->authority[i]; + for (i = 0; i < dst->num_subauth; ++i) + dst->sub_auth[i] = src->sub_auth[i]; +} + +/* + * change posix mode to reflect permissions + * pmode is the existing mode (we only want to overwrite part of this + * bits to set can be: S_IRWXU, S_IRWXG or S_IRWXO ie 00700 or 00070 or 00007 + */ +static umode_t access_flags_to_mode(struct smb_fattr *fattr, __le32 ace_flags, + int type) +{ + __u32 flags = le32_to_cpu(ace_flags); + umode_t mode = 0; + + if (flags & GENERIC_ALL) { + mode = 0777; + ksmbd_debug(SMB, "all perms\n"); + return mode; + } + + if ((flags & GENERIC_READ) || (flags & FILE_READ_RIGHTS)) + mode = 0444; + if ((flags & GENERIC_WRITE) || (flags & FILE_WRITE_RIGHTS)) { + mode |= 0222; + if (S_ISDIR(fattr->cf_mode)) + mode |= 0111; + } + if ((flags & GENERIC_EXECUTE) || (flags & FILE_EXEC_RIGHTS)) + mode |= 0111; + + if (type == ACCESS_DENIED_ACE_TYPE || type == ACCESS_DENIED_OBJECT_ACE_TYPE) + mode = ~mode; + + ksmbd_debug(SMB, "access flags 0x%x mode now %04o\n", flags, mode); + + return mode; +} + +/* + * Generate access flags to reflect permissions mode is the existing mode. + * This function is called for every ACE in the DACL whose SID matches + * with either owner or group or everyone. + */ +static void mode_to_access_flags(umode_t mode, umode_t bits_to_use, + __u32 *pace_flags) +{ + /* reset access mask */ + *pace_flags = 0x0; + + /* bits to use are either S_IRWXU or S_IRWXG or S_IRWXO */ + mode &= bits_to_use; + + /* + * check for R/W/X UGO since we do not know whose flags + * is this but we have cleared all the bits sans RWX for + * either user or group or other as per bits_to_use + */ + if (mode & 0444) + *pace_flags |= SET_FILE_READ_RIGHTS; + if (mode & 0222) + *pace_flags |= FILE_WRITE_RIGHTS; + if (mode & 0111) + *pace_flags |= SET_FILE_EXEC_RIGHTS; + + ksmbd_debug(SMB, "mode: %o, access flags now 0x%x\n", + mode, *pace_flags); +} + +static __u16 fill_ace_for_sid(struct smb_ace *pntace, + const struct smb_sid *psid, int type, int flags, + umode_t mode, umode_t bits) +{ + int i; + __u16 size = 0; + __u32 access_req = 0; + + pntace->type = type; + pntace->flags = flags; + mode_to_access_flags(mode, bits, &access_req); + if (!access_req) + access_req = SET_MINIMUM_RIGHTS; + pntace->access_req = cpu_to_le32(access_req); + + pntace->sid.revision = psid->revision; + pntace->sid.num_subauth = psid->num_subauth; + for (i = 0; i < NUM_AUTHS; i++) + pntace->sid.authority[i] = psid->authority[i]; + for (i = 0; i < psid->num_subauth; i++) + pntace->sid.sub_auth[i] = psid->sub_auth[i]; + + size = 1 + 1 + 2 + 4 + 1 + 1 + 6 + (psid->num_subauth * 4); + pntace->size = cpu_to_le16(size); + + return size; +} + +void id_to_sid(unsigned int cid, uint sidtype, struct smb_sid *ssid) +{ + switch (sidtype) { + case SIDOWNER: + smb_copy_sid(ssid, &server_conf.domain_sid); + break; + case SIDUNIX_USER: + smb_copy_sid(ssid, &sid_unix_users); + break; + case SIDUNIX_GROUP: + smb_copy_sid(ssid, &sid_unix_groups); + break; + case SIDCREATOR_OWNER: + smb_copy_sid(ssid, &creator_owner); + return; + case SIDCREATOR_GROUP: + smb_copy_sid(ssid, &creator_group); + return; + case SIDNFS_USER: + smb_copy_sid(ssid, &sid_unix_NFS_users); + break; + case SIDNFS_GROUP: + smb_copy_sid(ssid, &sid_unix_NFS_groups); + break; + case SIDNFS_MODE: + smb_copy_sid(ssid, &sid_unix_NFS_mode); + break; + default: + return; + } + + /* RID */ + ssid->sub_auth[ssid->num_subauth] = cpu_to_le32(cid); + ssid->num_subauth++; +} + +static int sid_to_id(struct mnt_idmap *idmap, + struct smb_sid *psid, uint sidtype, + struct smb_fattr *fattr) +{ + const struct smb_sid *sid_prefix; + int rc = -EINVAL; + + /* + * If we have too many subauthorities, then something is really wrong. + * Just return an error. + */ + if (unlikely(psid->num_subauth > SID_MAX_SUB_AUTHORITIES)) { + pr_err("%s: %u subauthorities is too many!\n", + __func__, psid->num_subauth); + return -EIO; + } + + if (psid->num_subauth == 0) { + pr_err("%s: zero subauthorities!\n", __func__); + return -EIO; + } + + if (sidtype == SIDOWNER) { + kuid_t uid; + uid_t id; + + /* Only the server domain RID has a local uid representation. */ + sid_prefix = &server_conf.domain_sid; + if (psid->num_subauth != sid_prefix->num_subauth + 1 || + compare_sids(psid, sid_prefix)) + return -EINVAL; + + id = le32_to_cpu(psid->sub_auth[psid->num_subauth - 1]); + uid = KUIDT_INIT(id); + uid = from_vfsuid(idmap, &init_user_ns, VFSUIDT_INIT(uid)); + if (uid_valid(uid)) { + fattr->cf_uid = uid; + rc = 0; + } + } else { + kgid_t gid; + gid_t id; + + /* Local gids are represented by S-1-22-2-. */ + sid_prefix = &sid_unix_groups; + if (psid->num_subauth != sid_prefix->num_subauth + 1 || + compare_sids(psid, sid_prefix)) + return -EINVAL; + + id = le32_to_cpu(psid->sub_auth[psid->num_subauth - 1]); + gid = KGIDT_INIT(id); + gid = from_vfsgid(idmap, &init_user_ns, VFSGIDT_INIT(gid)); + if (gid_valid(gid)) { + fattr->cf_gid = gid; + rc = 0; + } + } + + return rc; +} + +void posix_state_to_acl(struct posix_acl_state *state, + struct posix_acl_entry *pace) +{ + int i; + + pace->e_tag = ACL_USER_OBJ; + pace->e_perm = state->owner.allow; + for (i = 0; i < state->users->n; i++) { + pace++; + pace->e_tag = ACL_USER; + pace->e_uid = state->users->aces[i].uid; + pace->e_perm = state->users->aces[i].perms.allow; + } + + pace++; + pace->e_tag = ACL_GROUP_OBJ; + pace->e_perm = state->group.allow; + + for (i = 0; i < state->groups->n; i++) { + pace++; + pace->e_tag = ACL_GROUP; + pace->e_gid = state->groups->aces[i].gid; + pace->e_perm = state->groups->aces[i].perms.allow; + } + + if (state->users->n || state->groups->n) { + pace++; + pace->e_tag = ACL_MASK; + pace->e_perm = state->mask.allow; + } + + pace++; + pace->e_tag = ACL_OTHER; + pace->e_perm = state->other.allow; +} + +int init_acl_state(struct posix_acl_state *state, u16 cnt) +{ + int alloc; + + memset(state, 0, sizeof(struct posix_acl_state)); + /* + * In the worst case, each individual acl could be for a distinct + * named user or group, but we don't know which, so we allocate + * enough space for either: + */ + alloc = sizeof(struct posix_ace_state_array) + + cnt * sizeof(struct posix_user_ace_state); + state->users = kzalloc(alloc, KSMBD_DEFAULT_GFP); + if (!state->users) + return -ENOMEM; + state->groups = kzalloc(alloc, KSMBD_DEFAULT_GFP); + if (!state->groups) { + kfree(state->users); + return -ENOMEM; + } + return 0; +} + +void free_acl_state(struct posix_acl_state *state) +{ + kfree(state->users); + kfree(state->groups); +} + +static void parse_dacl(struct mnt_idmap *idmap, + struct smb_acl *pdacl, char *end_of_acl, + struct smb_sid *pownersid, struct smb_sid *pgrpsid, + struct smb_fattr *fattr) +{ + int i, ret; + u16 num_aces = 0; + u16 dacl_size; + unsigned int acl_size; + char *acl_base; + struct smb_ace **ppace; + struct posix_acl_entry *cf_pace, *cf_pdace; + struct posix_acl_state acl_state, default_acl_state; + umode_t mode = 0, acl_mode; + bool owner_found = false, group_found = false, others_found = false; + + if (!pdacl) + return; + + /* validate that we do not go past end of acl */ + if (end_of_acl < (char *)pdacl + sizeof(struct smb_acl) || + end_of_acl < (char *)pdacl + le16_to_cpu(pdacl->size)) { + pr_err("ACL too small to parse DACL\n"); + return; + } + + ksmbd_debug(SMB, "DACL revision %d size %d num aces %d\n", + le16_to_cpu(pdacl->revision), le16_to_cpu(pdacl->size), + le16_to_cpu(pdacl->num_aces)); + + acl_base = (char *)pdacl; + acl_size = sizeof(struct smb_acl); + + num_aces = le16_to_cpu(pdacl->num_aces); + if (num_aces <= 0) + return; + + dacl_size = le16_to_cpu(pdacl->size); + if (dacl_size < sizeof(struct smb_acl)) + return; + + if (num_aces > (dacl_size - sizeof(struct smb_acl)) / + (offsetof(struct smb_ace, sid) + + offsetof(struct smb_sid, sub_auth) + sizeof(__le16))) + return; + + ret = init_acl_state(&acl_state, num_aces); + if (ret) + return; + ret = init_acl_state(&default_acl_state, num_aces); + if (ret) { + free_acl_state(&acl_state); + return; + } + + ppace = kmalloc_objs(struct smb_ace *, num_aces, KSMBD_DEFAULT_GFP); + if (!ppace) { + free_acl_state(&default_acl_state); + free_acl_state(&acl_state); + return; + } + + /* + * reset rwx permissions for user/group/other. + * Also, if num_aces is 0 i.e. DACL has no ACEs, + * user/group/other have no permissions + */ + for (i = 0; i < num_aces; ++i) { + if (end_of_acl - acl_base < acl_size) + break; + + ppace[i] = (struct smb_ace *)(acl_base + acl_size); + acl_base = (char *)ppace[i]; + acl_size = offsetof(struct smb_ace, sid) + + offsetof(struct smb_sid, sub_auth); + + if (end_of_acl - acl_base < acl_size || + ppace[i]->sid.num_subauth == 0 || + ppace[i]->sid.num_subauth > SID_MAX_SUB_AUTHORITIES || + (end_of_acl - acl_base < + acl_size + sizeof(__le32) * ppace[i]->sid.num_subauth) || + (le16_to_cpu(ppace[i]->size) < + acl_size + sizeof(__le32) * ppace[i]->sid.num_subauth)) + break; + + acl_size = le16_to_cpu(ppace[i]->size); + ppace[i]->access_req = + smb_map_generic_desired_access(ppace[i]->access_req); + + if (ppace[i]->sid.num_subauth >= 3 && + !(compare_sids(&ppace[i]->sid, &sid_unix_NFS_mode))) { + fattr->cf_mode = + le32_to_cpu(ppace[i]->sid.sub_auth[2]); + break; + } else if (!compare_sids(&ppace[i]->sid, pownersid)) { + acl_mode = access_flags_to_mode(fattr, + ppace[i]->access_req, + ppace[i]->type); + acl_mode &= 0700; + + if (!owner_found) { + mode &= ~(0700); + mode |= acl_mode; + } + owner_found = true; + } else if (!compare_sids(&ppace[i]->sid, pgrpsid) || + ppace[i]->sid.sub_auth[ppace[i]->sid.num_subauth - 1] == + DOMAIN_USER_RID_LE) { + acl_mode = access_flags_to_mode(fattr, + ppace[i]->access_req, + ppace[i]->type); + acl_mode &= 0070; + if (!group_found) { + mode &= ~(0070); + mode |= acl_mode; + } + group_found = true; + } else if (!compare_sids(&ppace[i]->sid, &sid_everyone)) { + acl_mode = access_flags_to_mode(fattr, + ppace[i]->access_req, + ppace[i]->type); + acl_mode &= 0007; + if (!others_found) { + mode &= ~(0007); + mode |= acl_mode; + } + others_found = true; + } else if (!compare_sids(&ppace[i]->sid, &creator_owner)) { + continue; + } else if (!compare_sids(&ppace[i]->sid, &creator_group)) { + continue; + } else if (!compare_sids(&ppace[i]->sid, &sid_authusers)) { + continue; + } else { + struct smb_fattr temp_fattr; + + acl_mode = access_flags_to_mode(fattr, ppace[i]->access_req, + ppace[i]->type); + temp_fattr.cf_uid = INVALID_UID; + ret = sid_to_id(idmap, &ppace[i]->sid, SIDOWNER, &temp_fattr); + if (ret || uid_eq(temp_fattr.cf_uid, INVALID_UID)) { + pr_err("%s: Error %d mapping Owner SID to uid\n", + __func__, ret); + continue; + } + + acl_state.owner.allow = ((acl_mode & 0700) >> 6) | 0004; + acl_state.users->aces[acl_state.users->n].uid = + temp_fattr.cf_uid; + acl_state.users->aces[acl_state.users->n++].perms.allow = + ((acl_mode & 0700) >> 6) | 0004; + default_acl_state.owner.allow = ((acl_mode & 0700) >> 6) | 0004; + default_acl_state.users->aces[default_acl_state.users->n].uid = + temp_fattr.cf_uid; + default_acl_state.users->aces[default_acl_state.users->n++].perms.allow = + ((acl_mode & 0700) >> 6) | 0004; + } + } + kfree(ppace); + + if (owner_found) { + /* The owner must be set to at least read-only. */ + acl_state.owner.allow = ((mode & 0700) >> 6) | 0004; + acl_state.users->aces[acl_state.users->n].uid = fattr->cf_uid; + acl_state.users->aces[acl_state.users->n++].perms.allow = + ((mode & 0700) >> 6) | 0004; + default_acl_state.owner.allow = ((mode & 0700) >> 6) | 0004; + default_acl_state.users->aces[default_acl_state.users->n].uid = + fattr->cf_uid; + default_acl_state.users->aces[default_acl_state.users->n++].perms.allow = + ((mode & 0700) >> 6) | 0004; + } + + if (group_found) { + acl_state.group.allow = (mode & 0070) >> 3; + acl_state.groups->aces[acl_state.groups->n].gid = + fattr->cf_gid; + acl_state.groups->aces[acl_state.groups->n++].perms.allow = + (mode & 0070) >> 3; + default_acl_state.group.allow = (mode & 0070) >> 3; + default_acl_state.groups->aces[default_acl_state.groups->n].gid = + fattr->cf_gid; + default_acl_state.groups->aces[default_acl_state.groups->n++].perms.allow = + (mode & 0070) >> 3; + } + + if (others_found) { + fattr->cf_mode &= ~(0007); + fattr->cf_mode |= mode & 0007; + + acl_state.other.allow = mode & 0007; + default_acl_state.other.allow = mode & 0007; + } + + if (acl_state.users->n || acl_state.groups->n) { + acl_state.mask.allow = 0x07; + + if (IS_ENABLED(CONFIG_FS_POSIX_ACL)) { + fattr->cf_acls = + posix_acl_alloc(acl_state.users->n + + acl_state.groups->n + 4, KSMBD_DEFAULT_GFP); + if (fattr->cf_acls) { + cf_pace = fattr->cf_acls->a_entries; + posix_state_to_acl(&acl_state, cf_pace); + } + } + } + + if (default_acl_state.users->n || default_acl_state.groups->n) { + default_acl_state.mask.allow = 0x07; + + if (IS_ENABLED(CONFIG_FS_POSIX_ACL)) { + fattr->cf_dacls = + posix_acl_alloc(default_acl_state.users->n + + default_acl_state.groups->n + 4, KSMBD_DEFAULT_GFP); + if (fattr->cf_dacls) { + cf_pdace = fattr->cf_dacls->a_entries; + posix_state_to_acl(&default_acl_state, cf_pdace); + } + } + } + free_acl_state(&acl_state); + free_acl_state(&default_acl_state); +} + +static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap, + struct smb_ace *pndace, + struct smb_fattr *fattr, u16 *num_aces, + u16 *size, u16 existing_nt_aces, + bool had_nt_aces) +{ + struct posix_acl_entry *pace; + struct smb_sid *sid; + struct smb_ace *ntace; + int i, j; + u16 ace_sz; + + if (!fattr->cf_acls) + goto posix_default_acl; + + pace = fattr->cf_acls->a_entries; + for (i = 0; i < fattr->cf_acls->a_count; i++, pace++) { + int flags = 0; + + sid = kmalloc_obj(struct smb_sid, KSMBD_DEFAULT_GFP); + if (!sid) + break; + + if (pace->e_tag == ACL_USER) { + uid_t uid; + unsigned int sid_type = SIDOWNER; + + uid = posix_acl_uid_translate(idmap, pace); + if (!uid) + sid_type = SIDUNIX_USER; + id_to_sid(uid, sid_type, sid); + } else if (pace->e_tag == ACL_GROUP) { + gid_t gid; + + gid = posix_acl_gid_translate(idmap, pace); + id_to_sid(gid, SIDUNIX_GROUP, sid); + } else if (pace->e_tag == ACL_OTHER && !had_nt_aces) { + smb_copy_sid(sid, &sid_everyone); + } else { + kfree(sid); + continue; + } + ntace = pndace; + for (j = 0; j < existing_nt_aces; j++) { + if (ntace->sid.sub_auth[ntace->sid.num_subauth - 1] == + sid->sub_auth[sid->num_subauth - 1]) + goto pass_same_sid; + ntace = (struct smb_ace *)((char *)ntace + + le16_to_cpu(ntace->size)); + } + + if (S_ISDIR(fattr->cf_mode) && pace->e_tag == ACL_OTHER) + flags = 0x03; + + ntace = (struct smb_ace *)((char *)pndace + *size); + ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, flags, + pace->e_perm, 0777); + if (check_add_overflow(*size, ace_sz, size)) { + *size -= ace_sz; + kfree(sid); + break; + } + (*num_aces)++; + if (pace->e_tag == ACL_USER) + ntace->access_req |= + FILE_DELETE_LE | FILE_DELETE_CHILD_LE; + + if (S_ISDIR(fattr->cf_mode) && + (pace->e_tag == ACL_USER || pace->e_tag == ACL_GROUP)) { + ntace = (struct smb_ace *)((char *)pndace + *size); + ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, + 0x03, pace->e_perm, 0777); + if (check_add_overflow(*size, ace_sz, size)) { + *size -= ace_sz; + kfree(sid); + break; + } + (*num_aces)++; + if (pace->e_tag == ACL_USER) + ntace->access_req |= + FILE_DELETE_LE | FILE_DELETE_CHILD_LE; + } + +pass_same_sid: + kfree(sid); + } + + if (had_nt_aces) + return; + +posix_default_acl: + if (!fattr->cf_dacls) + return; + + pace = fattr->cf_dacls->a_entries; + for (i = 0; i < fattr->cf_dacls->a_count; i++, pace++) { + sid = kmalloc_obj(struct smb_sid, KSMBD_DEFAULT_GFP); + if (!sid) + break; + + if (pace->e_tag == ACL_USER) { + uid_t uid; + + uid = posix_acl_uid_translate(idmap, pace); + id_to_sid(uid, SIDCREATOR_OWNER, sid); + } else if (pace->e_tag == ACL_GROUP) { + gid_t gid; + + gid = posix_acl_gid_translate(idmap, pace); + id_to_sid(gid, SIDCREATOR_GROUP, sid); + } else { + kfree(sid); + continue; + } + + ntace = (struct smb_ace *)((char *)pndace + *size); + ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, 0x0b, + pace->e_perm, 0777); + if (check_add_overflow(*size, ace_sz, size)) { + *size -= ace_sz; + kfree(sid); + break; + } + (*num_aces)++; + if (pace->e_tag == ACL_USER) + ntace->access_req |= + FILE_DELETE_LE | FILE_DELETE_CHILD_LE; + kfree(sid); + } +} + +static void set_ntacl_dacl(struct mnt_idmap *idmap, + struct smb_acl *pndacl, + struct smb_acl *nt_dacl, + unsigned int aces_size, + const struct smb_sid *pownersid, + const struct smb_sid *pgrpsid, + struct smb_fattr *fattr) +{ + struct smb_ace *ntace, *pndace; + u16 nt_num_aces = le16_to_cpu(nt_dacl->num_aces), num_aces = 0; + u16 copied_nt_aces; + unsigned short size = 0; + int i; + + pndace = (struct smb_ace *)((char *)pndacl + sizeof(struct smb_acl)); + if (nt_num_aces) { + ntace = (struct smb_ace *)((char *)nt_dacl + sizeof(struct smb_acl)); + for (i = 0; i < nt_num_aces; i++) { + unsigned short nt_ace_size; + + if (aces_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE) + break; + + nt_ace_size = le16_to_cpu(ntace->size); + if (nt_ace_size > aces_size || + nt_ace_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE) + break; + + if (ntace->sid.num_subauth == 0 || + ntace->sid.num_subauth > SID_MAX_SUB_AUTHORITIES || + nt_ace_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE + + sizeof(__le32) * + ntace->sid.num_subauth) + goto next_ace; + + memcpy((char *)pndace + size, ntace, nt_ace_size); + if (check_add_overflow(size, nt_ace_size, &size)) { + size -= nt_ace_size; + break; + } + num_aces++; + +next_ace: + aces_size -= nt_ace_size; + ntace = (struct smb_ace *)((char *)ntace + nt_ace_size); + } + } + + copied_nt_aces = num_aces; + set_posix_acl_entries_dacl(idmap, pndace, fattr, + &num_aces, &size, copied_nt_aces, + nt_num_aces != 0); + pndacl->num_aces = cpu_to_le16(num_aces); + pndacl->size = cpu_to_le16(le16_to_cpu(pndacl->size) + size); +} + +static void set_mode_dacl(struct mnt_idmap *idmap, + struct smb_acl *pndacl, struct smb_fattr *fattr) +{ + struct smb_ace *pace, *pndace; + u16 num_aces = 0; + u16 size = 0, ace_size = 0; + uid_t uid; + const struct smb_sid *sid; + + pace = pndace = (struct smb_ace *)((char *)pndacl + sizeof(struct smb_acl)); + + if (fattr->cf_acls) { + set_posix_acl_entries_dacl(idmap, pndace, fattr, + &num_aces, &size, num_aces, false); + goto out; + } + + /* owner RID */ + uid = from_kuid(&init_user_ns, fattr->cf_uid); + if (uid) + sid = &server_conf.domain_sid; + else + sid = &sid_unix_users; + ace_size = fill_ace_for_sid(pace, sid, ACCESS_ALLOWED, 0, + fattr->cf_mode, 0700); + pace->sid.sub_auth[pace->sid.num_subauth++] = cpu_to_le32(uid); + pace->size = cpu_to_le16(ace_size + 4); + size += le16_to_cpu(pace->size); + pace = (struct smb_ace *)((char *)pndace + size); + + /* Group RID */ + ace_size = fill_ace_for_sid(pace, &sid_unix_groups, + ACCESS_ALLOWED, 0, fattr->cf_mode, 0070); + pace->sid.sub_auth[pace->sid.num_subauth++] = + cpu_to_le32(from_kgid(&init_user_ns, fattr->cf_gid)); + pace->size = cpu_to_le16(ace_size + 4); + size += le16_to_cpu(pace->size); + pace = (struct smb_ace *)((char *)pndace + size); + num_aces = 3; + + if (S_ISDIR(fattr->cf_mode)) { + pace = (struct smb_ace *)((char *)pndace + size); + + /* creator owner */ + size += fill_ace_for_sid(pace, &creator_owner, ACCESS_ALLOWED, + 0x0b, fattr->cf_mode, 0700); + pace = (struct smb_ace *)((char *)pndace + size); + + /* creator group */ + size += fill_ace_for_sid(pace, &creator_group, ACCESS_ALLOWED, + 0x0b, fattr->cf_mode, 0070); + pace = (struct smb_ace *)((char *)pndace + size); + num_aces = 5; + } + + /* other */ + size += fill_ace_for_sid(pace, &sid_everyone, ACCESS_ALLOWED, 0, + fattr->cf_mode, 0007); + +out: + pndacl->num_aces = cpu_to_le16(num_aces); + pndacl->size = cpu_to_le16(le16_to_cpu(pndacl->size) + size); +} + +static int parse_sid(struct smb_sid *psid, char *end_of_acl) +{ + /* + * validate that we do not go past end of ACL - sid must be at least 8 + * bytes long (assuming no sub-auths - e.g. the null SID + */ + if (end_of_acl < (char *)psid + 8) { + pr_err("ACL too small to parse SID %p\n", psid); + return -EINVAL; + } + + if (!psid->num_subauth) + return 0; + + if (psid->num_subauth > SID_MAX_SUB_AUTHORITIES || + end_of_acl < (char *)psid + 8 + sizeof(__le32) * psid->num_subauth) + return -EINVAL; + + return 0; +} + +/* Convert CIFS ACL to POSIX form */ +int parse_sec_desc(struct mnt_idmap *idmap, struct smb_ntsd *pntsd, + int acl_len, struct smb_fattr *fattr) +{ + int rc = 0; + struct smb_sid *owner_sid_ptr, *group_sid_ptr; + struct smb_acl *dacl_ptr; /* no need for SACL ptr */ + char *end_of_acl = ((char *)pntsd) + acl_len; + __u32 dacloffset; + int pntsd_type; + + if (!pntsd) + return -EIO; + + if (acl_len < sizeof(struct smb_ntsd)) + return -EINVAL; + + owner_sid_ptr = (struct smb_sid *)((char *)pntsd + + le32_to_cpu(pntsd->osidoffset)); + group_sid_ptr = (struct smb_sid *)((char *)pntsd + + le32_to_cpu(pntsd->gsidoffset)); + dacloffset = le32_to_cpu(pntsd->dacloffset); + dacl_ptr = (struct smb_acl *)((char *)pntsd + dacloffset); + ksmbd_debug(SMB, + "revision %d type 0x%x ooffset 0x%x goffset 0x%x sacloffset 0x%x dacloffset 0x%x\n", + pntsd->revision, pntsd->type, le32_to_cpu(pntsd->osidoffset), + le32_to_cpu(pntsd->gsidoffset), + le32_to_cpu(pntsd->sacloffset), dacloffset); + + pntsd_type = le16_to_cpu(pntsd->type); + if (!(pntsd_type & DACL_PRESENT)) { + ksmbd_debug(SMB, "DACL_PRESENT in DACL type is not set\n"); + return rc; + } + + pntsd->type = cpu_to_le16(DACL_PRESENT); + + if (pntsd->osidoffset) { + if (le32_to_cpu(pntsd->osidoffset) < sizeof(struct smb_ntsd)) + return -EINVAL; + + rc = parse_sid(owner_sid_ptr, end_of_acl); + if (rc) { + pr_err("%s: Error %d parsing Owner SID\n", __func__, rc); + return rc; + } + + rc = sid_to_id(idmap, owner_sid_ptr, SIDOWNER, fattr); + if (rc) { + ksmbd_debug(SMB, "Owner SID has no Unix uid mapping\n"); + owner_sid_ptr = NULL; + rc = 0; + } + } + + if (pntsd->gsidoffset) { + if (le32_to_cpu(pntsd->gsidoffset) < sizeof(struct smb_ntsd)) + return -EINVAL; + + rc = parse_sid(group_sid_ptr, end_of_acl); + if (rc) { + pr_err("%s: Error %d mapping Owner SID to gid\n", + __func__, rc); + return rc; + } + rc = sid_to_id(idmap, group_sid_ptr, SIDUNIX_GROUP, fattr); + if (rc) { + ksmbd_debug(SMB, "Group SID has no Unix gid mapping\n"); + group_sid_ptr = NULL; + rc = 0; + } + } + + if ((pntsd_type & (DACL_AUTO_INHERITED | DACL_AUTO_INHERIT_REQ)) == + (DACL_AUTO_INHERITED | DACL_AUTO_INHERIT_REQ)) + pntsd->type |= cpu_to_le16(DACL_AUTO_INHERITED); + if (pntsd_type & DACL_PROTECTED) + pntsd->type |= cpu_to_le16(DACL_PROTECTED); + + if (dacloffset) { + if (dacloffset < sizeof(struct smb_ntsd)) + return -EINVAL; + + parse_dacl(idmap, dacl_ptr, end_of_acl, + owner_sid_ptr, group_sid_ptr, fattr); + } + + return 0; +} + +size_t smb_acl_sec_desc_scratch_len(struct smb_fattr *fattr, + struct smb_ntsd *ppntsd, int ppntsd_size, int addition_info) +{ + size_t len = sizeof(struct smb_ntsd); + size_t tmp; + + if (addition_info & OWNER_SECINFO) + len += sizeof(struct smb_sid); + if (addition_info & GROUP_SECINFO) + len += sizeof(struct smb_sid); + if (!(addition_info & DACL_SECINFO)) + return len; + + len += sizeof(struct smb_acl); + if (ppntsd && ppntsd_size > 0) { + unsigned int dacl_offset = le32_to_cpu(ppntsd->dacloffset); + + if (dacl_offset < ppntsd_size && + check_add_overflow(len, ppntsd_size - dacl_offset, &len)) + return 0; + } + + if (fattr->cf_acls) { + if (check_mul_overflow((size_t)fattr->cf_acls->a_count, + 2 * sizeof(struct smb_ace), &tmp) || + check_add_overflow(len, tmp, &len)) + return 0; + } else { + /* default/minimum DACL */ + if (check_add_overflow(len, 5 * sizeof(struct smb_ace), &len)) + return 0; + } + + if (fattr->cf_dacls) { + if (check_mul_overflow((size_t)fattr->cf_dacls->a_count, + sizeof(struct smb_ace), &tmp) || + check_add_overflow(len, tmp, &len)) + return 0; + } + + return len; +} + +/* Convert permission bits from mode to equivalent CIFS ACL */ +int build_sec_desc(struct mnt_idmap *idmap, + struct smb_ntsd *pntsd, struct smb_ntsd *ppntsd, + int ppntsd_size, int addition_info, __u32 *secdesclen, + struct smb_fattr *fattr) +{ + int rc = 0; + __u32 offset; + struct smb_sid *owner_sid_ptr, *group_sid_ptr; + struct smb_sid *nowner_sid_ptr, *ngroup_sid_ptr; + struct smb_acl *dacl_ptr = NULL; /* no need for SACL ptr */ + uid_t uid; + gid_t gid; + unsigned int sid_type = SIDOWNER; + + nowner_sid_ptr = kmalloc_obj(struct smb_sid, KSMBD_DEFAULT_GFP); + if (!nowner_sid_ptr) + return -ENOMEM; + + uid = from_kuid(&init_user_ns, fattr->cf_uid); + if (!uid) + sid_type = SIDUNIX_USER; + id_to_sid(uid, sid_type, nowner_sid_ptr); + + ngroup_sid_ptr = kmalloc_obj(struct smb_sid, KSMBD_DEFAULT_GFP); + if (!ngroup_sid_ptr) { + kfree(nowner_sid_ptr); + return -ENOMEM; + } + + gid = from_kgid(&init_user_ns, fattr->cf_gid); + id_to_sid(gid, SIDUNIX_GROUP, ngroup_sid_ptr); + + offset = sizeof(struct smb_ntsd); + pntsd->sacloffset = 0; + pntsd->revision = cpu_to_le16(1); + pntsd->type = cpu_to_le16(SELF_RELATIVE); + if (ppntsd) + pntsd->type |= ppntsd->type; + + if (addition_info & OWNER_SECINFO) { + pntsd->osidoffset = cpu_to_le32(offset); + owner_sid_ptr = (struct smb_sid *)((char *)pntsd + offset); + smb_copy_sid(owner_sid_ptr, nowner_sid_ptr); + offset += 1 + 1 + 6 + (nowner_sid_ptr->num_subauth * 4); + } + + if (addition_info & GROUP_SECINFO) { + pntsd->gsidoffset = cpu_to_le32(offset); + group_sid_ptr = (struct smb_sid *)((char *)pntsd + offset); + smb_copy_sid(group_sid_ptr, ngroup_sid_ptr); + offset += 1 + 1 + 6 + (ngroup_sid_ptr->num_subauth * 4); + } + + if (addition_info & DACL_SECINFO) { + pntsd->type |= cpu_to_le16(DACL_PRESENT); + dacl_ptr = (struct smb_acl *)((char *)pntsd + offset); + dacl_ptr->revision = cpu_to_le16(2); + dacl_ptr->size = cpu_to_le16(sizeof(struct smb_acl)); + dacl_ptr->num_aces = 0; + + if (!ppntsd) { + set_mode_dacl(idmap, dacl_ptr, fattr); + } else { + struct smb_acl *ppdacl_ptr; + unsigned int dacl_offset = le32_to_cpu(ppntsd->dacloffset); + int ppdacl_size, ntacl_size = ppntsd_size - dacl_offset; + + if (!dacl_offset || + (dacl_offset + sizeof(struct smb_acl) > ppntsd_size)) + goto out; + + ppdacl_ptr = (struct smb_acl *)((char *)ppntsd + dacl_offset); + ppdacl_size = le16_to_cpu(ppdacl_ptr->size); + if (ppdacl_size > ntacl_size || + ppdacl_size < sizeof(struct smb_acl)) + goto out; + + set_ntacl_dacl(idmap, dacl_ptr, ppdacl_ptr, + ntacl_size - sizeof(struct smb_acl), + nowner_sid_ptr, ngroup_sid_ptr, + fattr); + } + pntsd->dacloffset = cpu_to_le32(offset); + offset += le16_to_cpu(dacl_ptr->size); + } + +out: + kfree(nowner_sid_ptr); + kfree(ngroup_sid_ptr); + *secdesclen = offset; + return rc; +} + +static void smb_set_ace(struct smb_ace *ace, const struct smb_sid *sid, u8 type, + u8 flags, __le32 access_req) +{ + ace->type = type; + ace->flags = flags; + ace->access_req = access_req; + smb_copy_sid(&ace->sid, sid); + ace->size = cpu_to_le16(1 + 1 + 2 + 4 + 1 + 1 + 6 + + (ace->sid.num_subauth * 4)); +} + +static int smb_append_inherited_ace(struct smb_ace **ace, int *nt_size, + u16 *ace_cnt, const struct smb_sid *sid, + u8 type, u8 flags, __le32 access_req) +{ + int ace_size; + + smb_set_ace(*ace, sid, type, flags, access_req); + ace_size = le16_to_cpu((*ace)->size); + /* pdacl->size is __le16 and includes struct smb_acl. */ + if (check_add_overflow(*nt_size, ace_size, nt_size) || + *nt_size > U16_MAX - (int)sizeof(struct smb_acl)) + return -EINVAL; + + (*ace_cnt)++; + *ace = (struct smb_ace *)((char *)*ace + ace_size); + return 0; +} + +static int smb_validate_ntsd_sid(struct smb_ntsd *pntsd, size_t pntsd_size, + unsigned int sid_offset, struct smb_sid **sid, + size_t *sid_size) +{ + size_t sid_end; + + *sid = NULL; + *sid_size = 0; + + if (!sid_offset) + return 0; + + if (sid_offset < sizeof(struct smb_ntsd) || + check_add_overflow(sid_offset, (size_t)CIFS_SID_BASE_SIZE, + &sid_end) || + sid_end > pntsd_size) + return -EINVAL; + + *sid = (struct smb_sid *)((char *)pntsd + sid_offset); + if ((*sid)->num_subauth > SID_MAX_SUB_AUTHORITIES) + return -EINVAL; + + if (check_add_overflow((size_t)CIFS_SID_BASE_SIZE, + sizeof(__le32) * (size_t)(*sid)->num_subauth, + &sid_end)) + return -EINVAL; + + if (sid_offset > pntsd_size || sid_end > pntsd_size - sid_offset) + return -EINVAL; + + *sid_size = sid_end; + return 0; +} + +int smb_inherit_dacl(struct ksmbd_conn *conn, + const struct path *path, + unsigned int uid, unsigned int gid) +{ + const struct smb_sid *psid, *creator = NULL; + struct smb_ace *parent_aces, *aces; + struct smb_acl *parent_pdacl; + struct smb_ntsd *parent_pntsd = NULL; + struct smb_sid owner_sid, group_sid; + struct dentry *parent = path->dentry->d_parent; + struct mnt_idmap *idmap = mnt_idmap(path->mnt); + int inherited_flags = 0, flags = 0, i, nt_size = 0, pdacl_size; + int rc = 0, pntsd_type, ppntsd_size, acl_len, aces_size; + unsigned int dacloffset; + size_t dacl_struct_end; + u16 num_aces, ace_cnt = 0; + char *aces_base; + bool is_dir = S_ISDIR(d_inode(path->dentry)->i_mode); + + ppntsd_size = ksmbd_vfs_get_sd_xattr(conn, idmap, + parent, &parent_pntsd); + if (ppntsd_size <= 0) + return -ENOENT; + + dacloffset = le32_to_cpu(parent_pntsd->dacloffset); + if (!dacloffset || + check_add_overflow(dacloffset, sizeof(struct smb_acl), &dacl_struct_end) || + dacl_struct_end > (size_t)ppntsd_size) { + rc = -EINVAL; + goto free_parent_pntsd; + } + + parent_pdacl = (struct smb_acl *)((char *)parent_pntsd + dacloffset); + acl_len = ppntsd_size - dacloffset; + num_aces = le16_to_cpu(parent_pdacl->num_aces); + pntsd_type = le16_to_cpu(parent_pntsd->type); + pdacl_size = le16_to_cpu(parent_pdacl->size); + + if (pdacl_size > acl_len || pdacl_size < sizeof(struct smb_acl)) { + rc = -EINVAL; + goto free_parent_pntsd; + } + + aces_size = pdacl_size - sizeof(struct smb_acl); + + /* + * Validate num_aces against the DACL payload before allocating. + * Each ACE must be at least as large as its fixed-size header + * (up to the SID base), so num_aces cannot exceed the payload + * divided by the minimum ACE size. This mirrors the existing + * check in parse_dacl(). + */ + if (num_aces > aces_size / (offsetof(struct smb_ace, sid) + + offsetof(struct smb_sid, sub_auth) + + sizeof(__le16))) { + rc = -EINVAL; + goto free_parent_pntsd; + } + + aces_base = kmalloc_array(num_aces * 2, sizeof(struct smb_ace), + KSMBD_DEFAULT_GFP); + if (!aces_base) { + rc = -ENOMEM; + goto free_parent_pntsd; + } + + aces = (struct smb_ace *)aces_base; + parent_aces = (struct smb_ace *)((char *)parent_pdacl + + sizeof(struct smb_acl)); + + if (pntsd_type & DACL_AUTO_INHERITED) + inherited_flags = INHERITED_ACE; + + for (i = 0; i < num_aces; i++) { + int pace_size; + + if (aces_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE) + break; + + pace_size = le16_to_cpu(parent_aces->size); + if (pace_size > aces_size || + pace_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE) + break; + + if (parent_aces->sid.num_subauth > SID_MAX_SUB_AUTHORITIES || + pace_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE + + sizeof(__le32) * parent_aces->sid.num_subauth) + break; + + aces_size -= pace_size; + + flags = parent_aces->flags; + if (!smb_inherit_flags(flags, is_dir)) + goto pass; + if (is_dir) { + flags &= ~(INHERIT_ONLY_ACE | INHERITED_ACE); + if (!(flags & CONTAINER_INHERIT_ACE)) + flags |= INHERIT_ONLY_ACE; + if (flags & NO_PROPAGATE_INHERIT_ACE) + flags = 0; + } else { + flags = 0; + } + + if (!compare_sids(&creator_owner, &parent_aces->sid)) { + creator = &creator_owner; + id_to_sid(uid, SIDOWNER, &owner_sid); + psid = &owner_sid; + } else if (!compare_sids(&creator_group, &parent_aces->sid)) { + creator = &creator_group; + id_to_sid(gid, SIDUNIX_GROUP, &group_sid); + psid = &group_sid; + } else { + creator = NULL; + psid = &parent_aces->sid; + } + + if (is_dir && creator && flags & CONTAINER_INHERIT_ACE) { + rc = smb_append_inherited_ace(&aces, &nt_size, &ace_cnt, + psid, parent_aces->type, + inherited_flags, + parent_aces->access_req); + if (rc) + goto free_aces_base; + flags |= INHERIT_ONLY_ACE; + psid = creator; + } else if (is_dir && !(parent_aces->flags & NO_PROPAGATE_INHERIT_ACE)) { + psid = &parent_aces->sid; + } + + rc = smb_append_inherited_ace(&aces, &nt_size, &ace_cnt, psid, + parent_aces->type, + flags | inherited_flags, + parent_aces->access_req); + if (rc) + goto free_aces_base; +pass: + parent_aces = (struct smb_ace *)((char *)parent_aces + pace_size); + } + + if (nt_size > 0) { + struct smb_ntsd *pntsd; + struct smb_acl *pdacl; + struct smb_sid *powner_sid = NULL, *pgroup_sid = NULL; + size_t powner_sid_size = 0, pgroup_sid_size = 0, pntsd_size; + size_t pntsd_alloc_size; + + rc = smb_validate_ntsd_sid(parent_pntsd, ppntsd_size, + le32_to_cpu(parent_pntsd->osidoffset), + &powner_sid, &powner_sid_size); + if (rc) + goto free_aces_base; + rc = smb_validate_ntsd_sid(parent_pntsd, ppntsd_size, + le32_to_cpu(parent_pntsd->gsidoffset), + &pgroup_sid, &pgroup_sid_size); + if (rc) + goto free_aces_base; + + if (check_add_overflow(sizeof(struct smb_ntsd), + (size_t)powner_sid_size, + &pntsd_alloc_size) || + check_add_overflow(pntsd_alloc_size, + (size_t)pgroup_sid_size, + &pntsd_alloc_size) || + check_add_overflow(pntsd_alloc_size, sizeof(struct smb_acl), + &pntsd_alloc_size) || + check_add_overflow(pntsd_alloc_size, (size_t)nt_size, + &pntsd_alloc_size)) { + rc = -EINVAL; + goto free_aces_base; + } + + pntsd = kzalloc(pntsd_alloc_size, KSMBD_DEFAULT_GFP); + if (!pntsd) { + rc = -ENOMEM; + goto free_aces_base; + } + + pntsd->revision = cpu_to_le16(1); + pntsd->type = cpu_to_le16(SELF_RELATIVE | DACL_PRESENT); + if (le16_to_cpu(parent_pntsd->type) & DACL_AUTO_INHERITED) + pntsd->type |= cpu_to_le16(DACL_AUTO_INHERITED); + pntsd_size = sizeof(struct smb_ntsd); + pntsd->osidoffset = parent_pntsd->osidoffset; + pntsd->gsidoffset = parent_pntsd->gsidoffset; + pntsd->dacloffset = parent_pntsd->dacloffset; + + if ((u64)le32_to_cpu(pntsd->osidoffset) + powner_sid_size > + pntsd_alloc_size) { + rc = -EINVAL; + kfree(pntsd); + goto free_aces_base; + } + + if ((u64)le32_to_cpu(pntsd->gsidoffset) + pgroup_sid_size > + pntsd_alloc_size) { + rc = -EINVAL; + kfree(pntsd); + goto free_aces_base; + } + + if ((u64)le32_to_cpu(pntsd->dacloffset) + sizeof(struct smb_acl) + nt_size > + pntsd_alloc_size) { + rc = -EINVAL; + kfree(pntsd); + goto free_aces_base; + } + + if (pntsd->osidoffset) { + struct smb_sid *owner_sid = (struct smb_sid *)((char *)pntsd + + le32_to_cpu(pntsd->osidoffset)); + memcpy(owner_sid, powner_sid, powner_sid_size); + pntsd_size += powner_sid_size; + } + + if (pntsd->gsidoffset) { + struct smb_sid *group_sid = (struct smb_sid *)((char *)pntsd + + le32_to_cpu(pntsd->gsidoffset)); + memcpy(group_sid, pgroup_sid, pgroup_sid_size); + pntsd_size += pgroup_sid_size; + } + + if (pntsd->dacloffset) { + struct smb_ace *pace; + + pdacl = (struct smb_acl *)((char *)pntsd + le32_to_cpu(pntsd->dacloffset)); + pdacl->revision = cpu_to_le16(2); + pdacl->size = cpu_to_le16(sizeof(struct smb_acl) + nt_size); + pdacl->num_aces = cpu_to_le16(ace_cnt); + pace = (struct smb_ace *)((char *)pdacl + sizeof(struct smb_acl)); + memcpy(pace, aces_base, nt_size); + pntsd_size += sizeof(struct smb_acl) + nt_size; + } + + ksmbd_vfs_set_sd_xattr(conn, idmap, path, pntsd, pntsd_size, false); + kfree(pntsd); + } + +free_aces_base: + kfree(aces_base); +free_parent_pntsd: + kfree(parent_pntsd); + return rc; +} + +bool smb_inherit_flags(int flags, bool is_dir) +{ + if (!is_dir) + return (flags & OBJECT_INHERIT_ACE) != 0; + + if (flags & OBJECT_INHERIT_ACE && !(flags & NO_PROPAGATE_INHERIT_ACE)) + return true; + + if (flags & CONTAINER_INHERIT_ACE) + return true; + return false; +} + +int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, + __le32 *pdaccess, __le32 raw_daccess, int uid, + bool strict) +{ + struct mnt_idmap *idmap = mnt_idmap(path->mnt); + struct smb_ntsd *pntsd = NULL; + struct smb_acl *pdacl; + struct posix_acl *posix_acls; + int rc = 0, pntsd_size, acl_size, aces_size, pdacl_size; + unsigned int dacl_offset; + size_t dacl_struct_end; + struct smb_sid sid; + int requested = le32_to_cpu(*pdaccess & ~FILE_MAXIMAL_ACCESS_LE); + int granted = requested; + struct smb_ace *ace; + int i, found = 0; + unsigned int access_bits = 0, denied = 0; + struct smb_ace *others_ace = NULL; + struct posix_acl_entry *pa_entry; + unsigned int sid_type = SIDOWNER; + unsigned short ace_size; + bool is_owner, owner_rights = false; + vfsuid_t vfsuid; + + ksmbd_debug(SMB, "check permission using windows acl\n"); + pntsd_size = ksmbd_vfs_get_sd_xattr(conn, idmap, + path->dentry, &pntsd); + if (pntsd_size <= 0 || !pntsd) + goto err_out; + + dacl_offset = le32_to_cpu(pntsd->dacloffset); + if (!dacl_offset || + check_add_overflow(dacl_offset, sizeof(struct smb_acl), &dacl_struct_end) || + dacl_struct_end > (size_t)pntsd_size) + goto err_out; + + pdacl = (struct smb_acl *)((char *)pntsd + le32_to_cpu(pntsd->dacloffset)); + acl_size = pntsd_size - dacl_offset; + pdacl_size = le16_to_cpu(pdacl->size); + + if (pdacl_size > acl_size || pdacl_size < sizeof(struct smb_acl)) + goto err_out; + + if (!pdacl->num_aces) { + if (!(pdacl_size - sizeof(struct smb_acl)) && + *pdaccess & ~(FILE_READ_CONTROL_LE | FILE_WRITE_DAC_LE)) { + rc = -EACCES; + goto err_out; + } + goto err_out; + } + + if (!uid) + sid_type = SIDUNIX_USER; + id_to_sid(uid, sid_type, &sid); + vfsuid = i_uid_into_vfsuid(idmap, d_inode(path->dentry)); + is_owner = uid == from_kuid(&init_user_ns, vfsuid_into_kuid(vfsuid)); + + if (*pdaccess & FILE_MAXIMAL_ACCESS_LE) { + ace = (struct smb_ace *)((char *)pdacl + sizeof(struct smb_acl)); + aces_size = pdacl_size - sizeof(struct smb_acl); + for (i = 0; i < le16_to_cpu(pdacl->num_aces); i++) { + if (aces_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE) + break; + ace_size = le16_to_cpu(ace->size); + if (ace_size > aces_size || + ace_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE) + break; + aces_size -= ace_size; + + if (ace->sid.num_subauth > SID_MAX_SUB_AUTHORITIES || + ace_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE + + sizeof(__le32) * ace->sid.num_subauth) + break; + + if (!compare_sids(&sid_owner_rights, &ace->sid)) { + owner_rights = true; + if (!is_owner) + goto next_ace; + } + + if (ace->flags & INHERIT_ONLY_ACE || + (compare_sids(&sid, &ace->sid) && + compare_sids(&sid_unix_NFS_mode, &ace->sid) && + compare_sids(&sid_everyone, &ace->sid) && + compare_sids(&sid_authusers, &ace->sid) && + compare_sids(&sid_owner_rights, &ace->sid))) + goto next_ace; + + switch (ace->type) { + case ACCESS_ALLOWED_ACE_TYPE: + access_bits |= le32_to_cpu(ace->access_req); + break; + case ACCESS_DENIED_ACE_TYPE: + case ACCESS_DENIED_CALLBACK_ACE_TYPE: + denied |= ~access_bits & + le32_to_cpu(ace->access_req); + break; + } +next_ace: + ace = (struct smb_ace *)((char *)ace + le16_to_cpu(ace->size)); + } + if (is_owner && !owner_rights) + access_bits |= READ_CONTROL | WRITE_DAC | + FILE_READ_ATTRIBUTES | DELETE; + access_bits &= ~denied; + if ((raw_daccess & FILE_GENERIC_EXECUTE_LE) && + S_ISREG(d_inode(path->dentry)->i_mode) && + (access_bits & GENERIC_READ_FLAGS) == GENERIC_READ_FLAGS) + access_bits |= FILE_EXECUTE; + granted = requested | access_bits; + } + + ace = (struct smb_ace *)((char *)pdacl + sizeof(struct smb_acl)); + aces_size = pdacl_size - sizeof(struct smb_acl); + for (i = 0; i < le16_to_cpu(pdacl->num_aces); i++) { + if (aces_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE) + break; + ace_size = le16_to_cpu(ace->size); + if (ace_size > aces_size || + ace_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE) + break; + aces_size -= ace_size; + + if (ace->sid.num_subauth > SID_MAX_SUB_AUTHORITIES || + ace_size < offsetof(struct smb_ace, sid) + CIFS_SID_BASE_SIZE + + sizeof(__le32) * ace->sid.num_subauth) + break; + + if (!compare_sids(&sid, &ace->sid) || + !compare_sids(&sid_unix_NFS_mode, &ace->sid)) { + found = 1; + break; + } + if (!compare_sids(&sid_everyone, &ace->sid) || + !compare_sids(&sid_authusers, &ace->sid)) + others_ace = ace; + + ace = (struct smb_ace *)((char *)ace + le16_to_cpu(ace->size)); + } + + if (IS_ENABLED(CONFIG_FS_POSIX_ACL)) { + posix_acls = get_inode_acl(d_inode(path->dentry), ACL_TYPE_ACCESS); + if (!IS_ERR_OR_NULL(posix_acls) && !found && !others_ace) { + unsigned int id = -1; + + pa_entry = posix_acls->a_entries; + for (i = 0; i < posix_acls->a_count; i++, pa_entry++) { + if (pa_entry->e_tag == ACL_USER) + id = posix_acl_uid_translate(idmap, pa_entry); + else if (pa_entry->e_tag == ACL_GROUP) + id = posix_acl_gid_translate(idmap, pa_entry); + else + continue; + + if (id == uid) { + mode_to_access_flags(pa_entry->e_perm, + 0777, + &access_bits); + if (!access_bits) + access_bits = + SET_MINIMUM_RIGHTS; + posix_acl_release(posix_acls); + goto check_access_bits; + } + } + } + if (!IS_ERR_OR_NULL(posix_acls)) + posix_acl_release(posix_acls); + } + + if (!found) { + if (others_ace) { + ace = others_ace; + } else { + ksmbd_debug(SMB, "Can't find corresponding sid\n"); + rc = -EACCES; + goto err_out; + } + } + + if (!(*pdaccess & FILE_MAXIMAL_ACCESS_LE)) { + switch (ace->type) { + case ACCESS_ALLOWED_ACE_TYPE: + access_bits = le32_to_cpu(ace->access_req); + break; + case ACCESS_DENIED_ACE_TYPE: + case ACCESS_DENIED_CALLBACK_ACE_TYPE: + access_bits = le32_to_cpu(~ace->access_req); + break; + } + } + +check_access_bits: + if (strict) { + access_bits &= granted; + } else { + access_bits |= FILE_READ_ATTRIBUTES | READ_CONTROL | + WRITE_DAC | DELETE; + } + + if (granted & ~access_bits) { + ksmbd_debug(SMB, "Access denied with winACL, granted : %x, access_req : %x\n", + granted, le32_to_cpu(ace->access_req)); + rc = -EACCES; + goto err_out; + } + + *pdaccess = cpu_to_le32(granted); +err_out: + kfree(pntsd); + return rc; +} + +int set_info_sec(struct ksmbd_conn *conn, struct ksmbd_tree_connect *tcon, + const struct path *path, struct smb_ntsd *pntsd, int ntsd_len, + bool type_check, bool get_write) +{ + int rc; + struct smb_fattr fattr = {{0}}; + struct inode *inode = d_inode(path->dentry); + struct mnt_idmap *idmap = mnt_idmap(path->mnt); + struct iattr newattrs; + + fattr.cf_uid = INVALID_UID; + fattr.cf_gid = INVALID_GID; + fattr.cf_mode = inode->i_mode; + + rc = parse_sec_desc(idmap, pntsd, ntsd_len, &fattr); + if (rc) + goto out; + + newattrs.ia_valid = ATTR_CTIME; + if (!uid_eq(fattr.cf_uid, INVALID_UID)) { + newattrs.ia_valid |= ATTR_UID; + newattrs.ia_uid = fattr.cf_uid; + } + if (!gid_eq(fattr.cf_gid, INVALID_GID)) { + newattrs.ia_valid |= ATTR_GID; + newattrs.ia_gid = fattr.cf_gid; + } + newattrs.ia_valid |= ATTR_MODE; + newattrs.ia_mode = (inode->i_mode & ~0777) | (fattr.cf_mode & 0777); + + ksmbd_vfs_remove_acl_xattrs(idmap, path); + /* Update posix acls */ + if (IS_ENABLED(CONFIG_FS_POSIX_ACL) && fattr.cf_dacls) { + rc = set_posix_acl(idmap, path->dentry, + ACL_TYPE_ACCESS, fattr.cf_acls); + if (rc < 0) + ksmbd_debug(SMB, + "Set posix acl(ACL_TYPE_ACCESS) failed, rc : %d\n", + rc); + if (S_ISDIR(inode->i_mode) && fattr.cf_dacls) { + rc = set_posix_acl(idmap, path->dentry, + ACL_TYPE_DEFAULT, fattr.cf_dacls); + if (rc) + ksmbd_debug(SMB, + "Set posix acl(ACL_TYPE_DEFAULT) failed, rc : %d\n", + rc); + } + } + + inode_lock(inode); + rc = notify_change(idmap, path->dentry, &newattrs, NULL); + inode_unlock(inode); + if (rc) + goto out; + + /* Check it only calling from SD BUFFER context */ + if (type_check && !(le16_to_cpu(pntsd->type) & DACL_PRESENT)) + goto out; + + if (test_share_config_flag(tcon->share_conf, KSMBD_SHARE_FLAG_ACL_XATTR)) { + /* Update WinACL in xattr */ + ksmbd_vfs_remove_sd_xattrs(idmap, path); + ksmbd_vfs_set_sd_xattr(conn, idmap, path, pntsd, ntsd_len, + get_write); + } + +out: + posix_acl_release(fattr.cf_acls); + posix_acl_release(fattr.cf_dacls); + return rc; +} + +void ksmbd_init_domain(u32 *sub_auth) +{ + int i; + + memcpy(&server_conf.domain_sid, &domain, sizeof(struct smb_sid)); + for (i = 0; i < 3; ++i) + server_conf.domain_sid.sub_auth[i + 1] = cpu_to_le32(sub_auth[i]); +} diff --git a/fs/smb/server/smbacl.h b/fs/smb/server/smbacl.h new file mode 100644 index 000000000..01810c16c --- /dev/null +++ b/fs/smb/server/smbacl.h @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: LGPL-2.1+ */ +/* + * Copyright (c) International Business Machines Corp., 2007 + * Author(s): Steve French (sfrench@us.ibm.com) + * Modified by Namjae Jeon (linkinjeon@kernel.org) + */ + +#ifndef _SMBACL_H +#define _SMBACL_H + +#include "../common/smbacl.h" +#include +#include +#include +#include + +#include "mgmt/tree_connect.h" + +/* Revision for ACLs */ +#define SD_REVISION 1 + +/* Control flags for Security Descriptor */ +#define OWNER_DEFAULTED 0x0001 +#define GROUP_DEFAULTED 0x0002 +#define DACL_PRESENT 0x0004 +#define DACL_DEFAULTED 0x0008 +#define SACL_PRESENT 0x0010 +#define SACL_DEFAULTED 0x0020 +#define DACL_TRUSTED 0x0040 +#define SERVER_SECURITY 0x0080 +#define DACL_AUTO_INHERIT_REQ 0x0100 +#define SACL_AUTO_INHERIT_REQ 0x0200 +#define DACL_AUTO_INHERITED 0x0400 +#define SACL_AUTO_INHERITED 0x0800 +#define DACL_PROTECTED 0x1000 +#define SACL_PROTECTED 0x2000 +#define RM_CONTROL_VALID 0x4000 +#define SELF_RELATIVE 0x8000 + +struct ksmbd_conn; + +struct smb_fattr { + kuid_t cf_uid; + kgid_t cf_gid; + umode_t cf_mode; + __le32 daccess; + struct posix_acl *cf_acls; + struct posix_acl *cf_dacls; +}; + +struct posix_ace_state { + u32 allow; + u32 deny; +}; + +struct posix_user_ace_state { + union { + kuid_t uid; + kgid_t gid; + }; + struct posix_ace_state perms; +}; + +struct posix_ace_state_array { + int n; + struct posix_user_ace_state aces[]; +}; + +/* + * while processing the nfsv4 ace, this maintains the partial permissions + * calculated so far: + */ + +struct posix_acl_state { + struct posix_ace_state owner; + struct posix_ace_state group; + struct posix_ace_state other; + struct posix_ace_state everyone; + struct posix_ace_state mask; /* deny unused in this case */ + struct posix_ace_state_array *users; + struct posix_ace_state_array *groups; +}; + +int parse_sec_desc(struct mnt_idmap *idmap, struct smb_ntsd *pntsd, + int acl_len, struct smb_fattr *fattr); +int build_sec_desc(struct mnt_idmap *idmap, struct smb_ntsd *pntsd, + struct smb_ntsd *ppntsd, int ppntsd_size, int addition_info, + __u32 *secdesclen, struct smb_fattr *fattr); +int init_acl_state(struct posix_acl_state *state, u16 cnt); +void free_acl_state(struct posix_acl_state *state); +void posix_state_to_acl(struct posix_acl_state *state, + struct posix_acl_entry *pace); +int compare_sids(const struct smb_sid *ctsid, const struct smb_sid *cwsid); +bool smb_inherit_flags(int flags, bool is_dir); +int smb_inherit_dacl(struct ksmbd_conn *conn, const struct path *path, + unsigned int uid, unsigned int gid); +int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, + __le32 *pdaccess, __le32 raw_daccess, int uid, + bool strict); +int set_info_sec(struct ksmbd_conn *conn, struct ksmbd_tree_connect *tcon, + const struct path *path, struct smb_ntsd *pntsd, int ntsd_len, + bool type_check, bool get_write); +void id_to_sid(unsigned int cid, uint sidtype, struct smb_sid *ssid); +void ksmbd_init_domain(u32 *sub_auth); +size_t smb_acl_sec_desc_scratch_len(struct smb_fattr *fattr, + struct smb_ntsd *ppntsd, int ppntsd_size, int addition_info); + +static inline uid_t posix_acl_uid_translate(struct mnt_idmap *idmap, + struct posix_acl_entry *pace) +{ + vfsuid_t vfsuid; + + /* If this is an idmapped mount, apply the idmapping. */ + vfsuid = make_vfsuid(idmap, &init_user_ns, pace->e_uid); + + /* Translate the kuid into a userspace id ksmbd would see. */ + return from_kuid(&init_user_ns, vfsuid_into_kuid(vfsuid)); +} + +static inline gid_t posix_acl_gid_translate(struct mnt_idmap *idmap, + struct posix_acl_entry *pace) +{ + vfsgid_t vfsgid; + + /* If this is an idmapped mount, apply the idmapping. */ + vfsgid = make_vfsgid(idmap, &init_user_ns, pace->e_gid); + + /* Translate the kgid into a userspace id ksmbd would see. */ + return from_kgid(&init_user_ns, vfsgid_into_kgid(vfsgid)); +} + +#endif /* _SMBACL_H */ diff --git a/fs/smb/server/stats.h b/fs/smb/server/stats.h new file mode 100644 index 000000000..bc864efa0 --- /dev/null +++ b/fs/smb/server/stats.h @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2025, LG Electronics. + * Author(s): Hyunchul Lee + * Copyright (C) 2025, Samsung Electronics. + * Author(s): Vedansh Bhardwaj + */ + +#ifndef __KSMBD_STATS_H__ +#define __KSMBD_STATS_H__ + +#include "../common/smb2status.h" + +#define KSMBD_COUNTER_MAX_REQS 19 + +enum { + KSMBD_COUNTER_SESSIONS = 0, + KSMBD_COUNTER_TREE_CONNS, + KSMBD_COUNTER_REQUESTS, + KSMBD_COUNTER_STATUS_SUCCESS, + KSMBD_COUNTER_STATUS_INFORMATIONAL, + KSMBD_COUNTER_STATUS_WARNING, + KSMBD_COUNTER_STATUS_ERROR, + KSMBD_COUNTER_ERROR_ACCESS_DENIED, + KSMBD_COUNTER_ERROR_NOT_FOUND, + KSMBD_COUNTER_ERROR_INVALID_PARAMETER, + KSMBD_COUNTER_ERROR_SHARING_VIOLATION, + KSMBD_COUNTER_ERROR_NOT_SUPPORTED, + KSMBD_COUNTER_ERROR_OTHER, + KSMBD_COUNTER_READ_BYTES, + KSMBD_COUNTER_WRITE_BYTES, + KSMBD_COUNTER_FIRST_REQ, + KSMBD_COUNTER_LAST_REQ = KSMBD_COUNTER_FIRST_REQ + + KSMBD_COUNTER_MAX_REQS - 1, + KSMBD_COUNTER_MAX, +}; + +#ifdef CONFIG_PROC_FS +extern struct ksmbd_counters ksmbd_counters; + +struct ksmbd_counters { + struct percpu_counter counters[KSMBD_COUNTER_MAX]; +}; + +static inline void ksmbd_counter_inc(int type) +{ + percpu_counter_inc(&ksmbd_counters.counters[type]); +} + +static inline void ksmbd_counter_dec(int type) +{ + percpu_counter_dec(&ksmbd_counters.counters[type]); +} + +static inline void ksmbd_counter_add(int type, s64 value) +{ + percpu_counter_add(&ksmbd_counters.counters[type], value); +} + +static inline void ksmbd_counter_sub(int type, s64 value) +{ + percpu_counter_sub(&ksmbd_counters.counters[type], value); +} + +static inline void ksmbd_counter_inc_reqs(unsigned int cmd, __le32 status) +{ + unsigned int severity = le32_to_cpu(status) >> 30; + int type; + + switch (severity) { + case 0: + type = KSMBD_COUNTER_STATUS_SUCCESS; + break; + case 1: + type = KSMBD_COUNTER_STATUS_INFORMATIONAL; + break; + case 2: + type = KSMBD_COUNTER_STATUS_WARNING; + break; + default: + type = KSMBD_COUNTER_STATUS_ERROR; + break; + } + percpu_counter_inc(&ksmbd_counters.counters[type]); + + if (severity == 3) { + if (status == STATUS_ACCESS_DENIED) + type = KSMBD_COUNTER_ERROR_ACCESS_DENIED; + else if (status == STATUS_OBJECT_NAME_NOT_FOUND || + status == STATUS_NO_SUCH_FILE) + type = KSMBD_COUNTER_ERROR_NOT_FOUND; + else if (status == STATUS_INVALID_PARAMETER) + type = KSMBD_COUNTER_ERROR_INVALID_PARAMETER; + else if (status == STATUS_SHARING_VIOLATION) + type = KSMBD_COUNTER_ERROR_SHARING_VIOLATION; + else if (status == STATUS_NOT_SUPPORTED || + status == STATUS_NOT_IMPLEMENTED) + type = KSMBD_COUNTER_ERROR_NOT_SUPPORTED; + else + type = KSMBD_COUNTER_ERROR_OTHER; + percpu_counter_inc(&ksmbd_counters.counters[type]); + } + + if (cmd < KSMBD_COUNTER_MAX_REQS) { + percpu_counter_inc(&ksmbd_counters.counters[KSMBD_COUNTER_REQUESTS]); + percpu_counter_inc(&ksmbd_counters.counters[KSMBD_COUNTER_FIRST_REQ + cmd]); + } +} + +static inline s64 ksmbd_counter_sum(int type) +{ + return percpu_counter_sum_positive(&ksmbd_counters.counters[type]); +} +#else + +static inline void ksmbd_counter_inc(int type) {} +static inline void ksmbd_counter_dec(int type) {} +static inline void ksmbd_counter_add(int type, s64 value) {} +static inline void ksmbd_counter_sub(int type, s64 value) {} +static inline void ksmbd_counter_inc_reqs(unsigned int cmd, __le32 status) {} +static inline s64 ksmbd_counter_sum(int type) { return 0; } +#endif + +#endif diff --git a/fs/smb/server/transport_ipc.c b/fs/smb/server/transport_ipc.c new file mode 100644 index 000000000..4b0b572a3 --- /dev/null +++ b/fs/smb/server/transport_ipc.c @@ -0,0 +1,1025 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "vfs_cache.h" +#include "transport_ipc.h" +#include "server.h" +#include "smb_common.h" + +#include "mgmt/user_config.h" +#include "mgmt/share_config.h" +#include "mgmt/user_session.h" +#include "mgmt/tree_connect.h" +#include "mgmt/ksmbd_ida.h" +#include "connection.h" +#include "transport_tcp.h" +#include "transport_rdma.h" + +#define IPC_WAIT_TIMEOUT (2 * HZ) + +#define IPC_MSG_HASH_BITS 3 +static DEFINE_HASHTABLE(ipc_msg_table, IPC_MSG_HASH_BITS); +static DECLARE_RWSEM(ipc_msg_table_lock); +static DEFINE_MUTEX(startup_lock); + +static DEFINE_IDA(ipc_ida); + +static unsigned int ksmbd_tools_pid; + +static bool ksmbd_ipc_validate_version(struct genl_info *m) +{ + if (m->genlhdr->version != KSMBD_GENL_VERSION) { + pr_err("%s. ksmbd: %d, kernel module: %d. %s.\n", + "Daemon and kernel module version mismatch", + m->genlhdr->version, + KSMBD_GENL_VERSION, + "User-space ksmbd should terminate"); + return false; + } + return true; +} + +struct ksmbd_ipc_msg { + unsigned int type; + unsigned int sz; + unsigned char payload[] __counted_by(sz); +}; + +struct ipc_msg_table_entry { + unsigned int handle; + unsigned int type; + wait_queue_head_t wait; + struct hlist_node ipc_table_hlist; + + void *response; + unsigned int msg_sz; +}; + +static struct delayed_work ipc_timer_work; + +static int handle_startup_event(struct sk_buff *skb, struct genl_info *info); +static int handle_unsupported_event(struct sk_buff *skb, struct genl_info *info); +static int handle_generic_event(struct sk_buff *skb, struct genl_info *info); +static int ksmbd_ipc_heartbeat_request(void); + +static const struct nla_policy ksmbd_nl_policy[KSMBD_EVENT_MAX + 1] = { + [KSMBD_EVENT_UNSPEC] = { + .len = 0, + }, + [KSMBD_EVENT_HEARTBEAT_REQUEST] = { + .len = sizeof(struct ksmbd_heartbeat), + }, + [KSMBD_EVENT_STARTING_UP] = { + .len = sizeof(struct ksmbd_startup_request), + }, + [KSMBD_EVENT_SHUTTING_DOWN] = { + .len = sizeof(struct ksmbd_shutdown_request), + }, + [KSMBD_EVENT_LOGIN_REQUEST] = { + .len = sizeof(struct ksmbd_login_request), + }, + [KSMBD_EVENT_LOGIN_RESPONSE] = { + .len = sizeof(struct ksmbd_login_response), + }, + [KSMBD_EVENT_SHARE_CONFIG_REQUEST] = { + .len = sizeof(struct ksmbd_share_config_request), + }, + [KSMBD_EVENT_SHARE_CONFIG_RESPONSE] = { + .len = sizeof(struct ksmbd_share_config_response), + }, + [KSMBD_EVENT_TREE_CONNECT_REQUEST] = { + .len = sizeof(struct ksmbd_tree_connect_request), + }, + [KSMBD_EVENT_TREE_CONNECT_RESPONSE] = { + .len = sizeof(struct ksmbd_tree_connect_response), + }, + [KSMBD_EVENT_TREE_DISCONNECT_REQUEST] = { + .len = sizeof(struct ksmbd_tree_disconnect_request), + }, + [KSMBD_EVENT_LOGOUT_REQUEST] = { + .len = sizeof(struct ksmbd_logout_request), + }, + [KSMBD_EVENT_RPC_REQUEST] = { + }, + [KSMBD_EVENT_RPC_RESPONSE] = { + }, + [KSMBD_EVENT_SPNEGO_AUTHEN_REQUEST] = { + }, + [KSMBD_EVENT_SPNEGO_AUTHEN_RESPONSE] = { + }, + [KSMBD_EVENT_LOGIN_REQUEST_EXT] = { + .len = sizeof(struct ksmbd_login_request), + }, + [KSMBD_EVENT_LOGIN_RESPONSE_EXT] = { + .len = sizeof(struct ksmbd_login_response_ext), + }, +}; + +static struct genl_ops ksmbd_genl_ops[] = { + { + .cmd = KSMBD_EVENT_UNSPEC, + .doit = handle_unsupported_event, + }, + { + .cmd = KSMBD_EVENT_HEARTBEAT_REQUEST, + .doit = handle_unsupported_event, + }, + { + .cmd = KSMBD_EVENT_STARTING_UP, + .doit = handle_startup_event, + }, + { + .cmd = KSMBD_EVENT_SHUTTING_DOWN, + .doit = handle_unsupported_event, + }, + { + .cmd = KSMBD_EVENT_LOGIN_REQUEST, + .doit = handle_unsupported_event, + }, + { + .cmd = KSMBD_EVENT_LOGIN_RESPONSE, + .doit = handle_generic_event, + }, + { + .cmd = KSMBD_EVENT_SHARE_CONFIG_REQUEST, + .doit = handle_unsupported_event, + }, + { + .cmd = KSMBD_EVENT_SHARE_CONFIG_RESPONSE, + .doit = handle_generic_event, + }, + { + .cmd = KSMBD_EVENT_TREE_CONNECT_REQUEST, + .doit = handle_unsupported_event, + }, + { + .cmd = KSMBD_EVENT_TREE_CONNECT_RESPONSE, + .doit = handle_generic_event, + }, + { + .cmd = KSMBD_EVENT_TREE_DISCONNECT_REQUEST, + .doit = handle_unsupported_event, + }, + { + .cmd = KSMBD_EVENT_LOGOUT_REQUEST, + .doit = handle_unsupported_event, + }, + { + .cmd = KSMBD_EVENT_RPC_REQUEST, + .doit = handle_unsupported_event, + }, + { + .cmd = KSMBD_EVENT_RPC_RESPONSE, + .doit = handle_generic_event, + }, + { + .cmd = KSMBD_EVENT_SPNEGO_AUTHEN_REQUEST, + .doit = handle_unsupported_event, + }, + { + .cmd = KSMBD_EVENT_SPNEGO_AUTHEN_RESPONSE, + .doit = handle_generic_event, + }, + { + .cmd = KSMBD_EVENT_LOGIN_REQUEST_EXT, + .doit = handle_unsupported_event, + }, + { + .cmd = KSMBD_EVENT_LOGIN_RESPONSE_EXT, + .doit = handle_generic_event, + }, +}; + +static struct genl_family ksmbd_genl_family = { + .name = KSMBD_GENL_NAME, + .version = KSMBD_GENL_VERSION, + .hdrsize = 0, + .maxattr = KSMBD_EVENT_MAX, + .netnsok = true, + .module = THIS_MODULE, + .ops = ksmbd_genl_ops, + .n_ops = ARRAY_SIZE(ksmbd_genl_ops), + .resv_start_op = KSMBD_EVENT_LOGIN_RESPONSE_EXT + 1, +}; + +static void ksmbd_nl_init_fixup(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(ksmbd_genl_ops); i++) + ksmbd_genl_ops[i].validate = GENL_DONT_VALIDATE_STRICT | + GENL_DONT_VALIDATE_DUMP; + + ksmbd_genl_family.policy = ksmbd_nl_policy; +} + +static int rpc_context_flags(struct ksmbd_session *sess) +{ + if (user_guest(sess->user)) + return KSMBD_RPC_RESTRICTED_CONTEXT; + return 0; +} + +static void ipc_update_last_active(void) +{ + if (server_conf.ipc_timeout) + server_conf.ipc_last_active = jiffies; +} + +static struct ksmbd_ipc_msg *ipc_msg_alloc(size_t sz) +{ + struct ksmbd_ipc_msg *msg; + + msg = kvzalloc_flex(*msg, payload, sz, KSMBD_DEFAULT_GFP); + if (msg) + msg->sz = sz; + return msg; +} + +static void ipc_msg_free(struct ksmbd_ipc_msg *msg) +{ + kvfree(msg); +} + +static void ipc_msg_handle_free(int handle) +{ + if (handle >= 0) + ksmbd_release_id(&ipc_ida, handle); +} + +static int handle_response(int type, void *payload, size_t sz) +{ + unsigned int handle; + struct ipc_msg_table_entry *entry; + int ret = 0; + + /* Prevent 4-byte read beyond declared payload size */ + if (sz < sizeof(unsigned int)) + return -EINVAL; + + handle = *(unsigned int *)payload; + + ipc_update_last_active(); + down_read(&ipc_msg_table_lock); + hash_for_each_possible(ipc_msg_table, entry, ipc_table_hlist, handle) { + if (handle != entry->handle) + continue; + + entry->response = NULL; + /* + * Response message type value should be equal to + * request message type + 1. + */ + if (entry->type + 1 != type) { + pr_err("Waiting for IPC type %d, got %d. Ignore.\n", + entry->type + 1, type); + continue; + } + + entry->response = kvzalloc(sz, KSMBD_DEFAULT_GFP); + if (!entry->response) { + ret = -ENOMEM; + break; + } + + memcpy(entry->response, payload, sz); + entry->msg_sz = sz; + wake_up_interruptible(&entry->wait); + ret = 0; + break; + } + up_read(&ipc_msg_table_lock); + + return ret; +} + +static int ipc_server_config_on_startup(struct ksmbd_startup_request *req) +{ + int ret; + + ksmbd_set_fd_limit(req->file_max); + server_conf.flags = req->flags; + server_conf.signing = req->signing; + server_conf.tcp_port = req->tcp_port; + server_conf.ipc_timeout = req->ipc_timeout * HZ; + if (check_mul_overflow(req->deadtime, SMB_ECHO_INTERVAL, + &server_conf.deadtime)) { + ret = -EINVAL; + goto out; + } + server_conf.share_fake_fscaps = req->share_fake_fscaps; + + /* AAPL model string for Finder icon */ + if (req->aapl_model[0]) + strscpy(server_conf.aapl_model, req->aapl_model, + sizeof(server_conf.aapl_model)); + else + strscpy(server_conf.aapl_model, "Xserve", + sizeof(server_conf.aapl_model)); + + ksmbd_init_domain(req->sub_auth); + + if (req->smb2_max_read) + init_smb2_max_read_size(req->smb2_max_read); + if (req->smb2_max_write) + init_smb2_max_write_size(req->smb2_max_write); + if (req->smb2_max_trans) + init_smb2_max_trans_size(req->smb2_max_trans); + if (req->smb2_max_credits) { + init_smb2_max_credits(req->smb2_max_credits); + server_conf.max_inflight_req = + req->smb2_max_credits; + } + if (req->smbd_max_io_size) + init_smbd_max_io_size(req->smbd_max_io_size); + + if (req->max_connections) + server_conf.max_connections = req->max_connections; + + if (req->max_ip_connections) + server_conf.max_ip_connections = req->max_ip_connections; + + ret = ksmbd_set_netbios_name(req->netbios_name); + ret |= ksmbd_set_server_string(req->server_string); + ret |= ksmbd_set_work_group(req->work_group); + server_conf.bind_interfaces_only = req->bind_interfaces_only; + ret |= ksmbd_tcp_set_interfaces(KSMBD_STARTUP_CONFIG_INTERFACES(req), + req->ifc_list_sz); +out: + if (ret) { + pr_err("Server configuration error: %s %s %s\n", + req->netbios_name, req->server_string, + req->work_group); + return ret; + } + + if (req->min_prot[0]) { + ret = ksmbd_lookup_protocol_idx(req->min_prot); + if (ret >= 0) + server_conf.min_protocol = ret; + } + if (req->max_prot[0]) { + ret = ksmbd_lookup_protocol_idx(req->max_prot); + if (ret >= 0) + server_conf.max_protocol = ret; + } + + if (server_conf.ipc_timeout) + schedule_delayed_work(&ipc_timer_work, server_conf.ipc_timeout); + return 0; +} + +static int handle_startup_event(struct sk_buff *skb, struct genl_info *info) +{ + int ret = 0; + +#ifdef CONFIG_SMB_SERVER_CHECK_CAP_NET_ADMIN + if (!netlink_capable(skb, CAP_NET_ADMIN)) + return -EPERM; +#endif + + if (!ksmbd_ipc_validate_version(info)) + return -EINVAL; + + if (!info->attrs[KSMBD_EVENT_STARTING_UP]) + return -EINVAL; + + mutex_lock(&startup_lock); + if (!ksmbd_server_configurable()) { + mutex_unlock(&startup_lock); + pr_err("Server reset is in progress, can't start daemon\n"); + return -EINVAL; + } + + if (ksmbd_tools_pid) { + if (ksmbd_ipc_heartbeat_request() == 0) { + ret = -EINVAL; + goto out; + } + + pr_err("Reconnect to a new user space daemon\n"); + } else { + struct ksmbd_startup_request *req; + + req = nla_data(info->attrs[info->genlhdr->cmd]); + ret = ipc_server_config_on_startup(req); + if (ret) + goto out; + server_queue_ctrl_init_work(); + } + + ksmbd_tools_pid = info->snd_portid; + ipc_update_last_active(); + +out: + mutex_unlock(&startup_lock); + return ret; +} + +static int handle_unsupported_event(struct sk_buff *skb, struct genl_info *info) +{ + pr_err("Unknown IPC event: %d, ignore.\n", info->genlhdr->cmd); + return -EINVAL; +} + +static int handle_generic_event(struct sk_buff *skb, struct genl_info *info) +{ + void *payload; + int sz; + int type = info->genlhdr->cmd; + +#ifdef CONFIG_SMB_SERVER_CHECK_CAP_NET_ADMIN + if (!netlink_capable(skb, CAP_NET_ADMIN)) + return -EPERM; +#endif + + if (type > KSMBD_EVENT_MAX) { + WARN_ON(1); + return -EINVAL; + } + + if (!ksmbd_ipc_validate_version(info)) + return -EINVAL; + + if (!info->attrs[type]) + return -EINVAL; + + payload = nla_data(info->attrs[info->genlhdr->cmd]); + sz = nla_len(info->attrs[info->genlhdr->cmd]); + return handle_response(type, payload, sz); +} + +static int ipc_msg_send(struct ksmbd_ipc_msg *msg) +{ + struct genlmsghdr *nlh; + struct sk_buff *skb; + int ret = -EINVAL; + + if (!ksmbd_tools_pid) + return ret; + + skb = genlmsg_new(msg->sz, KSMBD_DEFAULT_GFP); + if (!skb) + return -ENOMEM; + + nlh = genlmsg_put(skb, 0, 0, &ksmbd_genl_family, 0, msg->type); + if (!nlh) + goto out; + + ret = nla_put(skb, msg->type, msg->sz, msg->payload); + if (ret) { + genlmsg_cancel(skb, nlh); + goto out; + } + + genlmsg_end(skb, nlh); + ret = genlmsg_unicast(&init_net, skb, ksmbd_tools_pid); + if (!ret) + ipc_update_last_active(); + return ret; + +out: + nlmsg_free(skb); + return ret; +} + +static int ipc_validate_msg(struct ipc_msg_table_entry *entry) +{ + unsigned int msg_sz = entry->msg_sz; + + switch (entry->type) { + case KSMBD_EVENT_RPC_REQUEST: + { + struct ksmbd_rpc_command *resp = entry->response; + + if (entry->msg_sz < sizeof(struct ksmbd_rpc_command)) + return -EINVAL; + + if (check_add_overflow(sizeof(struct ksmbd_rpc_command), + resp->payload_sz, &msg_sz)) + return -EINVAL; + break; + } + case KSMBD_EVENT_SPNEGO_AUTHEN_REQUEST: + { + struct ksmbd_spnego_authen_response *resp = entry->response; + + if (entry->msg_sz < sizeof(struct ksmbd_spnego_authen_response)) + return -EINVAL; + + msg_sz = sizeof(struct ksmbd_spnego_authen_response) + + resp->session_key_len + resp->spnego_blob_len; + break; + } + case KSMBD_EVENT_SHARE_CONFIG_REQUEST: + { + struct ksmbd_share_config_response *resp = entry->response; + + if (entry->msg_sz < sizeof(struct ksmbd_share_config_response)) + return -EINVAL; + + if (resp->payload_sz) { + if (resp->payload_sz < resp->veto_list_sz) + return -EINVAL; + + if (check_add_overflow(sizeof(struct ksmbd_share_config_response), + resp->payload_sz, &msg_sz)) + return -EINVAL; + } + break; + } + case KSMBD_EVENT_LOGIN_REQUEST_EXT: + { + struct ksmbd_login_response_ext *resp = entry->response; + + if (entry->msg_sz < sizeof(struct ksmbd_login_response_ext)) + return -EINVAL; + + if (resp->ngroups) { + if (resp->ngroups < 0 || + resp->ngroups > NGROUPS_MAX) { + pr_err("ngroups(%d) from login response exceeds max groups(%d)\n", + resp->ngroups, NGROUPS_MAX); + return -EINVAL; + } + msg_sz = sizeof(struct ksmbd_login_response_ext) + + resp->ngroups * sizeof(gid_t); + } + } + } + + return entry->msg_sz != msg_sz ? -EINVAL : 0; +} + +static void *ipc_msg_send_request(struct ksmbd_ipc_msg *msg, unsigned int handle) +{ + struct ipc_msg_table_entry entry; + int ret; + + if ((int)handle < 0) + return NULL; + + entry.type = msg->type; + entry.response = NULL; + init_waitqueue_head(&entry.wait); + + down_write(&ipc_msg_table_lock); + entry.handle = handle; + hash_add(ipc_msg_table, &entry.ipc_table_hlist, entry.handle); + up_write(&ipc_msg_table_lock); + + ret = ipc_msg_send(msg); + if (ret) { + down_write(&ipc_msg_table_lock); + goto out; + } + + ret = wait_event_interruptible_timeout(entry.wait, + entry.response != NULL, + IPC_WAIT_TIMEOUT); + + down_write(&ipc_msg_table_lock); + if (entry.response) { + ret = ipc_validate_msg(&entry); + if (ret) { + kvfree(entry.response); + entry.response = NULL; + } + } +out: + hash_del(&entry.ipc_table_hlist); + up_write(&ipc_msg_table_lock); + return entry.response; +} + +static int ksmbd_ipc_heartbeat_request(void) +{ + struct ksmbd_ipc_msg *msg; + int ret; + + msg = ipc_msg_alloc(sizeof(struct ksmbd_heartbeat)); + if (!msg) + return -EINVAL; + + msg->type = KSMBD_EVENT_HEARTBEAT_REQUEST; + ret = ipc_msg_send(msg); + ipc_msg_free(msg); + return ret; +} + +struct ksmbd_login_response *ksmbd_ipc_login_request(const char *account) +{ + struct ksmbd_ipc_msg *msg; + struct ksmbd_login_request *req; + struct ksmbd_login_response *resp; + + if (strlen(account) >= KSMBD_REQ_MAX_ACCOUNT_NAME_SZ) + return NULL; + + msg = ipc_msg_alloc(sizeof(struct ksmbd_login_request)); + if (!msg) + return NULL; + + msg->type = KSMBD_EVENT_LOGIN_REQUEST; + req = (struct ksmbd_login_request *)msg->payload; + req->handle = ksmbd_acquire_id(&ipc_ida); + strscpy(req->account, account, KSMBD_REQ_MAX_ACCOUNT_NAME_SZ); + + resp = ipc_msg_send_request(msg, req->handle); + ipc_msg_handle_free(req->handle); + ipc_msg_free(msg); + return resp; +} + +struct ksmbd_login_response_ext *ksmbd_ipc_login_request_ext(const char *account) +{ + struct ksmbd_ipc_msg *msg; + struct ksmbd_login_request *req; + struct ksmbd_login_response_ext *resp; + + if (strlen(account) >= KSMBD_REQ_MAX_ACCOUNT_NAME_SZ) + return NULL; + + msg = ipc_msg_alloc(sizeof(struct ksmbd_login_request)); + if (!msg) + return NULL; + + msg->type = KSMBD_EVENT_LOGIN_REQUEST_EXT; + req = (struct ksmbd_login_request *)msg->payload; + req->handle = ksmbd_acquire_id(&ipc_ida); + strscpy(req->account, account, KSMBD_REQ_MAX_ACCOUNT_NAME_SZ); + resp = ipc_msg_send_request(msg, req->handle); + ipc_msg_handle_free(req->handle); + ipc_msg_free(msg); + return resp; +} + +struct ksmbd_spnego_authen_response * +ksmbd_ipc_spnego_authen_request(const char *spnego_blob, int blob_len) +{ + struct ksmbd_ipc_msg *msg; + struct ksmbd_spnego_authen_request *req; + struct ksmbd_spnego_authen_response *resp; + + if (blob_len > KSMBD_IPC_MAX_PAYLOAD) + return NULL; + + msg = ipc_msg_alloc(sizeof(struct ksmbd_spnego_authen_request) + + blob_len); + if (!msg) + return NULL; + + msg->type = KSMBD_EVENT_SPNEGO_AUTHEN_REQUEST; + req = (struct ksmbd_spnego_authen_request *)msg->payload; + req->handle = ksmbd_acquire_id(&ipc_ida); + req->spnego_blob_len = blob_len; + memcpy(req->spnego_blob, spnego_blob, blob_len); + + resp = ipc_msg_send_request(msg, req->handle); + ipc_msg_handle_free(req->handle); + ipc_msg_free(msg); + return resp; +} + +struct ksmbd_tree_connect_response * +ksmbd_ipc_tree_connect_request(struct ksmbd_session *sess, + struct ksmbd_share_config *share, + struct ksmbd_tree_connect *tree_conn, + struct sockaddr *peer_addr) +{ + struct ksmbd_ipc_msg *msg; + struct ksmbd_tree_connect_request *req; + struct ksmbd_tree_connect_response *resp; + + if (strlen(user_name(sess->user)) >= KSMBD_REQ_MAX_ACCOUNT_NAME_SZ) + return NULL; + + if (strlen(share->name) >= KSMBD_REQ_MAX_SHARE_NAME) + return NULL; + + msg = ipc_msg_alloc(sizeof(struct ksmbd_tree_connect_request)); + if (!msg) + return NULL; + + msg->type = KSMBD_EVENT_TREE_CONNECT_REQUEST; + req = (struct ksmbd_tree_connect_request *)msg->payload; + + req->handle = ksmbd_acquire_id(&ipc_ida); + req->account_flags = sess->user->flags; + req->session_id = sess->id; + req->connect_id = tree_conn->id; + strscpy(req->account, user_name(sess->user), KSMBD_REQ_MAX_ACCOUNT_NAME_SZ); + strscpy(req->share, share->name, KSMBD_REQ_MAX_SHARE_NAME); + snprintf(req->peer_addr, sizeof(req->peer_addr), "%pIS", peer_addr); + + if (peer_addr->sa_family == AF_INET6) + req->flags |= KSMBD_TREE_CONN_FLAG_REQUEST_IPV6; + if (test_session_flag(sess, CIFDS_SESSION_FLAG_SMB2)) + req->flags |= KSMBD_TREE_CONN_FLAG_REQUEST_SMB2; + + resp = ipc_msg_send_request(msg, req->handle); + ipc_msg_handle_free(req->handle); + ipc_msg_free(msg); + return resp; +} + +int ksmbd_ipc_tree_disconnect_request(unsigned long long session_id, + unsigned long long connect_id) +{ + struct ksmbd_ipc_msg *msg; + struct ksmbd_tree_disconnect_request *req; + int ret; + + msg = ipc_msg_alloc(sizeof(struct ksmbd_tree_disconnect_request)); + if (!msg) + return -ENOMEM; + + msg->type = KSMBD_EVENT_TREE_DISCONNECT_REQUEST; + req = (struct ksmbd_tree_disconnect_request *)msg->payload; + req->session_id = session_id; + req->connect_id = connect_id; + + ret = ipc_msg_send(msg); + ipc_msg_free(msg); + return ret; +} + +int ksmbd_ipc_logout_request(const char *account, int flags) +{ + struct ksmbd_ipc_msg *msg; + struct ksmbd_logout_request *req; + int ret; + + if (strlen(account) >= KSMBD_REQ_MAX_ACCOUNT_NAME_SZ) + return -EINVAL; + + msg = ipc_msg_alloc(sizeof(struct ksmbd_logout_request)); + if (!msg) + return -ENOMEM; + + msg->type = KSMBD_EVENT_LOGOUT_REQUEST; + req = (struct ksmbd_logout_request *)msg->payload; + req->account_flags = flags; + strscpy(req->account, account, KSMBD_REQ_MAX_ACCOUNT_NAME_SZ); + + ret = ipc_msg_send(msg); + ipc_msg_free(msg); + return ret; +} + +struct ksmbd_share_config_response * +ksmbd_ipc_share_config_request(const char *name) +{ + struct ksmbd_ipc_msg *msg; + struct ksmbd_share_config_request *req; + struct ksmbd_share_config_response *resp; + + if (strlen(name) >= KSMBD_REQ_MAX_SHARE_NAME) + return NULL; + + msg = ipc_msg_alloc(sizeof(struct ksmbd_share_config_request)); + if (!msg) + return NULL; + + msg->type = KSMBD_EVENT_SHARE_CONFIG_REQUEST; + req = (struct ksmbd_share_config_request *)msg->payload; + req->handle = ksmbd_acquire_id(&ipc_ida); + strscpy(req->share_name, name, KSMBD_REQ_MAX_SHARE_NAME); + + resp = ipc_msg_send_request(msg, req->handle); + ipc_msg_handle_free(req->handle); + ipc_msg_free(msg); + return resp; +} + +struct ksmbd_rpc_command *ksmbd_rpc_open(struct ksmbd_session *sess, int handle) +{ + struct ksmbd_ipc_msg *msg; + struct ksmbd_rpc_command *req; + struct ksmbd_rpc_command *resp; + + msg = ipc_msg_alloc(sizeof(struct ksmbd_rpc_command)); + if (!msg) + return NULL; + + msg->type = KSMBD_EVENT_RPC_REQUEST; + req = (struct ksmbd_rpc_command *)msg->payload; + req->handle = handle; + req->flags = ksmbd_session_rpc_method(sess, handle); + req->flags |= KSMBD_RPC_OPEN_METHOD; + req->payload_sz = 0; + + resp = ipc_msg_send_request(msg, req->handle); + ipc_msg_free(msg); + return resp; +} + +struct ksmbd_rpc_command *ksmbd_rpc_close(struct ksmbd_session *sess, int handle) +{ + struct ksmbd_ipc_msg *msg; + struct ksmbd_rpc_command *req; + struct ksmbd_rpc_command *resp; + + msg = ipc_msg_alloc(sizeof(struct ksmbd_rpc_command)); + if (!msg) + return NULL; + + msg->type = KSMBD_EVENT_RPC_REQUEST; + req = (struct ksmbd_rpc_command *)msg->payload; + req->handle = handle; + req->flags = ksmbd_session_rpc_method(sess, handle); + req->flags |= KSMBD_RPC_CLOSE_METHOD; + req->payload_sz = 0; + + resp = ipc_msg_send_request(msg, req->handle); + ipc_msg_free(msg); + return resp; +} + +struct ksmbd_rpc_command *ksmbd_rpc_write(struct ksmbd_session *sess, int handle, + void *payload, size_t payload_sz) +{ + struct ksmbd_ipc_msg *msg; + struct ksmbd_rpc_command *req; + struct ksmbd_rpc_command *resp; + + if (payload_sz > KSMBD_IPC_MAX_PAYLOAD) + return NULL; + + msg = ipc_msg_alloc(sizeof(struct ksmbd_rpc_command) + payload_sz); + if (!msg) + return NULL; + + lockdep_assert_not_held(&sess->rpc_lock); + + down_read(&sess->rpc_lock); + msg->type = KSMBD_EVENT_RPC_REQUEST; + req = (struct ksmbd_rpc_command *)msg->payload; + req->handle = handle; + req->flags = ksmbd_session_rpc_method(sess, handle); + req->flags |= rpc_context_flags(sess); + req->flags |= KSMBD_RPC_WRITE_METHOD; + req->payload_sz = payload_sz; + memcpy(req->payload, payload, payload_sz); + up_read(&sess->rpc_lock); + + resp = ipc_msg_send_request(msg, req->handle); + ipc_msg_free(msg); + return resp; +} + +struct ksmbd_rpc_command *ksmbd_rpc_read(struct ksmbd_session *sess, int handle) +{ + struct ksmbd_ipc_msg *msg; + struct ksmbd_rpc_command *req; + struct ksmbd_rpc_command *resp; + + msg = ipc_msg_alloc(sizeof(struct ksmbd_rpc_command)); + if (!msg) + return NULL; + + lockdep_assert_not_held(&sess->rpc_lock); + + down_read(&sess->rpc_lock); + msg->type = KSMBD_EVENT_RPC_REQUEST; + req = (struct ksmbd_rpc_command *)msg->payload; + req->handle = handle; + req->flags = ksmbd_session_rpc_method(sess, handle); + req->flags |= rpc_context_flags(sess); + req->flags |= KSMBD_RPC_READ_METHOD; + req->payload_sz = 0; + up_read(&sess->rpc_lock); + + resp = ipc_msg_send_request(msg, req->handle); + ipc_msg_free(msg); + return resp; +} + +struct ksmbd_rpc_command *ksmbd_rpc_ioctl(struct ksmbd_session *sess, int handle, + void *payload, size_t payload_sz) +{ + struct ksmbd_ipc_msg *msg; + struct ksmbd_rpc_command *req; + struct ksmbd_rpc_command *resp; + + if (payload_sz > KSMBD_IPC_MAX_PAYLOAD) + return NULL; + + msg = ipc_msg_alloc(sizeof(struct ksmbd_rpc_command) + payload_sz); + if (!msg) + return NULL; + + lockdep_assert_not_held(&sess->rpc_lock); + + down_read(&sess->rpc_lock); + msg->type = KSMBD_EVENT_RPC_REQUEST; + req = (struct ksmbd_rpc_command *)msg->payload; + req->handle = handle; + req->flags = ksmbd_session_rpc_method(sess, handle); + req->flags |= rpc_context_flags(sess); + req->flags |= KSMBD_RPC_IOCTL_METHOD; + req->payload_sz = payload_sz; + memcpy(req->payload, payload, payload_sz); + up_read(&sess->rpc_lock); + + resp = ipc_msg_send_request(msg, req->handle); + ipc_msg_free(msg); + return resp; +} + +static int __ipc_heartbeat(void) +{ + unsigned long delta; + + if (!ksmbd_server_running()) + return 0; + + if (time_after(jiffies, server_conf.ipc_last_active)) { + delta = (jiffies - server_conf.ipc_last_active); + } else { + ipc_update_last_active(); + schedule_delayed_work(&ipc_timer_work, + server_conf.ipc_timeout); + return 0; + } + + if (delta < server_conf.ipc_timeout) { + schedule_delayed_work(&ipc_timer_work, + server_conf.ipc_timeout - delta); + return 0; + } + + if (ksmbd_ipc_heartbeat_request() == 0) { + schedule_delayed_work(&ipc_timer_work, + server_conf.ipc_timeout); + return 0; + } + + mutex_lock(&startup_lock); + WRITE_ONCE(server_conf.state, SERVER_STATE_RESETTING); + server_conf.ipc_last_active = 0; + ksmbd_tools_pid = 0; + pr_err("No IPC daemon response for %lus\n", delta / HZ); + mutex_unlock(&startup_lock); + return -EINVAL; +} + +static void ipc_timer_heartbeat(struct work_struct *w) +{ + if (__ipc_heartbeat()) + server_queue_ctrl_reset_work(); +} + +int ksmbd_ipc_id_alloc(void) +{ + return ksmbd_acquire_id(&ipc_ida); +} + +void ksmbd_rpc_id_free(int handle) +{ + ksmbd_release_id(&ipc_ida, handle); +} + +void ksmbd_ipc_release(void) +{ + cancel_delayed_work_sync(&ipc_timer_work); + genl_unregister_family(&ksmbd_genl_family); +} + +void ksmbd_ipc_soft_reset(void) +{ + mutex_lock(&startup_lock); + ksmbd_tools_pid = 0; + cancel_delayed_work_sync(&ipc_timer_work); + mutex_unlock(&startup_lock); +} + +int ksmbd_ipc_init(void) +{ + int ret = 0; + + ksmbd_nl_init_fixup(); + INIT_DELAYED_WORK(&ipc_timer_work, ipc_timer_heartbeat); + + ret = genl_register_family(&ksmbd_genl_family); + if (ret) { + pr_err("Failed to register KSMBD netlink interface %d\n", ret); + cancel_delayed_work_sync(&ipc_timer_work); + } + + return ret; +} diff --git a/fs/smb/server/transport_ipc.h b/fs/smb/server/transport_ipc.h new file mode 100644 index 000000000..e51850f14 --- /dev/null +++ b/fs/smb/server/transport_ipc.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#ifndef __KSMBD_TRANSPORT_IPC_H__ +#define __KSMBD_TRANSPORT_IPC_H__ + +#include + +#define KSMBD_IPC_MAX_PAYLOAD 4096 + +struct ksmbd_login_response * +ksmbd_ipc_login_request(const char *account); +struct ksmbd_login_response_ext * +ksmbd_ipc_login_request_ext(const char *account); + +struct ksmbd_session; +struct ksmbd_share_config; +struct ksmbd_tree_connect; +struct sockaddr; + +struct ksmbd_tree_connect_response * +ksmbd_ipc_tree_connect_request(struct ksmbd_session *sess, + struct ksmbd_share_config *share, + struct ksmbd_tree_connect *tree_conn, + struct sockaddr *peer_addr); +int ksmbd_ipc_tree_disconnect_request(unsigned long long session_id, + unsigned long long connect_id); +int ksmbd_ipc_logout_request(const char *account, int flags); +struct ksmbd_share_config_response * +ksmbd_ipc_share_config_request(const char *name); +struct ksmbd_spnego_authen_response * +ksmbd_ipc_spnego_authen_request(const char *spnego_blob, int blob_len); +int ksmbd_ipc_id_alloc(void); +void ksmbd_rpc_id_free(int handle); +struct ksmbd_rpc_command *ksmbd_rpc_open(struct ksmbd_session *sess, int handle); +struct ksmbd_rpc_command *ksmbd_rpc_close(struct ksmbd_session *sess, int handle); +struct ksmbd_rpc_command *ksmbd_rpc_write(struct ksmbd_session *sess, int handle, + void *payload, size_t payload_sz); +struct ksmbd_rpc_command *ksmbd_rpc_read(struct ksmbd_session *sess, int handle); +struct ksmbd_rpc_command *ksmbd_rpc_ioctl(struct ksmbd_session *sess, int handle, + void *payload, size_t payload_sz); +void ksmbd_ipc_release(void); +void ksmbd_ipc_soft_reset(void); +int ksmbd_ipc_init(void); +#endif /* __KSMBD_TRANSPORT_IPC_H__ */ diff --git a/fs/smb/server/transport_rdma.c b/fs/smb/server/transport_rdma.c new file mode 100644 index 000000000..ee28a4d1c --- /dev/null +++ b/fs/smb/server/transport_rdma.c @@ -0,0 +1,554 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017, Microsoft Corporation. + * Copyright (C) 2018, LG Electronics. + * + * Author(s): Long Li , + * Hyunchul Lee + */ + +#define SUBMOD_NAME "smb_direct" + +#include +#include +#include + +#include "glob.h" +#include "connection.h" +#include "smb_common.h" +#include "../common/smb2status.h" +#include "transport_rdma.h" + + +#define SMB_DIRECT_PORT_IWARP 5445 +#define SMB_DIRECT_PORT_INFINIBAND 445 + +/* SMB_DIRECT negotiation timeout (for the server) in seconds */ +#define SMB_DIRECT_NEGOTIATE_TIMEOUT 5 + +/* The timeout to wait for a keepalive message from peer in seconds */ +#define SMB_DIRECT_KEEPALIVE_SEND_INTERVAL 120 + +/* The timeout to wait for a keepalive message from peer in seconds */ +#define SMB_DIRECT_KEEPALIVE_RECV_TIMEOUT 5 + +/* + * Default maximum number of RDMA read/write outstanding on this connection + * This value is possibly decreased during QP creation on hardware limit + */ +#define SMB_DIRECT_CM_INITIATOR_DEPTH 8 + +/* + * User configurable initial values per SMB_DIRECT transport connection + * as defined in [MS-SMBD] 3.1.1.1 + * Those may change after a SMB_DIRECT negotiation + */ + +/* The local peer's maximum number of credits to grant to the peer */ +static int smb_direct_receive_credit_max = 255; + +/* The remote peer's credit request of local peer */ +static int smb_direct_send_credit_target = 255; + +/* The maximum single message size can be sent to remote peer */ +static int smb_direct_max_send_size = 1364; + +/* + * The maximum fragmented upper-layer payload receive size supported + * + * Assume max_payload_per_credit is + * smb_direct_receive_credit_max - 24 = 1340 + * + * The maximum number would be + * smb_direct_receive_credit_max * max_payload_per_credit + * + * 1340 * 255 = 341700 (0x536C4) + * + * The minimum value from the spec is 131072 (0x20000) + * + * For now we use the logic we used before: + * (1364 * 255) / 2 = 173910 (0x2A756) + */ +static int smb_direct_max_fragmented_recv_size = (1364 * 255) / 2; + +/* The maximum single-message size which can be received */ +static int smb_direct_max_receive_size = 1364; + +static int smb_direct_max_read_write_size = SMBD_DEFAULT_IOSIZE; + +static bool smb_direct_enabled; + +static struct smb_direct_listener { + int port; + + struct task_struct *thread; + + struct smbdirect_socket *socket; +} smb_direct_ib_listener, smb_direct_iw_listener; + +struct smb_direct_transport { + struct ksmbd_transport transport; + + struct smbdirect_socket *socket; +}; + +static bool smb_direct_logging_needed(struct smbdirect_socket *sc, + void *private_ptr, + unsigned int lvl, + unsigned int cls) +{ + if (lvl <= SMBDIRECT_LOG_ERR) + return true; + + if (lvl > SMBDIRECT_LOG_INFO) + return false; + + switch (cls) { + /* + * These were more or less also logged before + * the move to common code. + * + * SMBDIRECT_LOG_RDMA_MR was not used, but + * that's client only code and we should + * notice if it's used on the server... + */ + case SMBDIRECT_LOG_RDMA_EVENT: + case SMBDIRECT_LOG_RDMA_SEND: + case SMBDIRECT_LOG_RDMA_RECV: + case SMBDIRECT_LOG_WRITE: + case SMBDIRECT_LOG_READ: + case SMBDIRECT_LOG_NEGOTIATE: + case SMBDIRECT_LOG_OUTGOING: + case SMBDIRECT_LOG_RDMA_RW: + case SMBDIRECT_LOG_RDMA_MR: + return true; + /* + * These were not logged before the move + * to common code. + */ + case SMBDIRECT_LOG_KEEP_ALIVE: + case SMBDIRECT_LOG_INCOMING: + return false; + } + + /* + * Log all unknown messages + */ + return true; +} + +static void smb_direct_logging_vaprintf(struct smbdirect_socket *sc, + const char *func, + unsigned int line, + void *private_ptr, + unsigned int lvl, + unsigned int cls, + struct va_format *vaf) +{ + if (lvl <= SMBDIRECT_LOG_ERR) + pr_err("%pV", vaf); + else + ksmbd_debug(RDMA, "%pV", vaf); +} + +#define KSMBD_TRANS(t) (&(t)->transport) +#define SMBD_TRANS(t) (container_of(t, \ + struct smb_direct_transport, transport)) + +static const struct ksmbd_transport_ops ksmbd_smb_direct_transport_ops; + +void init_smbd_max_io_size(unsigned int sz) +{ + sz = clamp_val(sz, SMBD_MIN_IOSIZE, SMBD_MAX_IOSIZE); + smb_direct_max_read_write_size = sz; +} + +unsigned int get_smbd_max_read_write_size(struct ksmbd_transport *kt) +{ + struct smb_direct_transport *t; + const struct smbdirect_socket_parameters *sp; + + if (kt->ops != &ksmbd_smb_direct_transport_ops) + return 0; + + t = SMBD_TRANS(kt); + sp = smbdirect_socket_get_current_parameters(t->socket); + + return sp->max_read_write_size; +} + +static struct smb_direct_transport *alloc_transport(struct smbdirect_socket *sc) +{ + struct smb_direct_transport *t; + struct ksmbd_conn *conn; + + t = kzalloc_obj(*t, KSMBD_DEFAULT_GFP); + if (!t) + return NULL; + t->socket = sc; + + conn = ksmbd_conn_alloc(); + if (!conn) + goto conn_alloc_failed; + + down_write(&conn_list_lock); + hash_add(conn_list, &conn->hlist, 0); + up_write(&conn_list_lock); + + conn->transport = KSMBD_TRANS(t); + KSMBD_TRANS(t)->conn = conn; + KSMBD_TRANS(t)->ops = &ksmbd_smb_direct_transport_ops; + + return t; + +conn_alloc_failed: + kfree(t); + return NULL; +} + +static void smb_direct_free_transport(struct ksmbd_transport *kt) +{ + struct smb_direct_transport *t = SMBD_TRANS(kt); + + smbdirect_socket_release(t->socket); + kfree(t); +} + +static void free_transport(struct smb_direct_transport *t) +{ + smbdirect_socket_shutdown(t->socket); + ksmbd_conn_free(KSMBD_TRANS(t)->conn); +} + +static int smb_direct_read(struct ksmbd_transport *t, char *buf, + unsigned int size, int unused) +{ + struct smb_direct_transport *st = SMBD_TRANS(t); + struct smbdirect_socket *sc = st->socket; + struct msghdr msg = { .msg_flags = 0, }; + struct kvec iov = { + .iov_base = buf, + .iov_len = size, + }; + int ret; + + iov_iter_kvec(&msg.msg_iter, ITER_DEST, &iov, 1, size); + + ret = smbdirect_connection_recvmsg(sc, &msg, 0); + if (ret == -ERESTARTSYS) + ret = -EINTR; + return ret; +} + +static int smb_direct_writev(struct ksmbd_transport *t, + const struct ksmbd_transport_write *tx) +{ + struct smb_direct_transport *st = SMBD_TRANS(t); + struct smbdirect_socket *sc = st->socket; + struct iov_iter iter; + + iov_iter_kvec(&iter, ITER_SOURCE, tx->iov, tx->iov_cnt, + tx->size); + + return smbdirect_connection_send_iter(sc, &iter, 0, + tx->need_invalidate_rkey, + tx->remote_key); +} + +static int smb_direct_rdma_write(struct ksmbd_transport *t, + void *buf, unsigned int buflen, + struct smbdirect_buffer_descriptor_v1 *desc, + unsigned int desc_len) +{ + struct smb_direct_transport *st = SMBD_TRANS(t); + struct smbdirect_socket *sc = st->socket; + + return smbdirect_connection_rdma_xmit(sc, buf, buflen, + desc, desc_len, false); +} + +static int smb_direct_rdma_read(struct ksmbd_transport *t, + void *buf, unsigned int buflen, + struct smbdirect_buffer_descriptor_v1 *desc, + unsigned int desc_len) +{ + struct smb_direct_transport *st = SMBD_TRANS(t); + struct smbdirect_socket *sc = st->socket; + + return smbdirect_connection_rdma_xmit(sc, buf, buflen, + desc, desc_len, true); +} + +static void smb_direct_disconnect(struct ksmbd_transport *t) +{ + struct smb_direct_transport *st = SMBD_TRANS(t); + struct smbdirect_socket *sc = st->socket; + + ksmbd_debug(RDMA, "Disconnecting sc=%p\n", sc); + + free_transport(st); +} + +static void smb_direct_shutdown(struct ksmbd_transport *t) +{ + struct smb_direct_transport *st = SMBD_TRANS(t); + struct smbdirect_socket *sc = st->socket; + + ksmbd_debug(RDMA, "smb-direct shutdown sc=%p\n", sc); + + smbdirect_socket_shutdown(sc); +} + +static int smb_direct_new_connection(struct smb_direct_listener *listener, + struct smbdirect_socket *client_sc) +{ + struct smb_direct_transport *t; + struct task_struct *handler; + int ret; + + t = alloc_transport(client_sc); + if (!t) { + smbdirect_socket_release(client_sc); + return -ENOMEM; + } + + handler = kthread_run(ksmbd_conn_handler_loop, + KSMBD_TRANS(t)->conn, "ksmbd:r%u", + listener->port); + if (IS_ERR(handler)) { + ret = PTR_ERR(handler); + pr_err("Can't start thread\n"); + goto out_err; + } + + return 0; +out_err: + free_transport(t); + return ret; +} + +static int smb_direct_listener_kthread_fn(void *p) +{ + struct smb_direct_listener *listener = (struct smb_direct_listener *)p; + struct smbdirect_socket *client_sc = NULL; + + while (!kthread_should_stop()) { + struct proto_accept_arg arg = { .err = -EINVAL, }; + long timeo = MAX_SCHEDULE_TIMEOUT; + + if (!listener->socket) + break; + client_sc = smbdirect_socket_accept(listener->socket, timeo, &arg); + if (!client_sc && arg.err == -EINVAL) + break; + if (!client_sc) + continue; + + ksmbd_debug(CONN, "connect success: accepted new connection\n"); + smb_direct_new_connection(listener, client_sc); + } + + ksmbd_debug(CONN, "releasing socket\n"); + return 0; +} + +static void smb_direct_listener_destroy(struct smb_direct_listener *listener) +{ + int ret; + + if (listener->socket) + smbdirect_socket_shutdown(listener->socket); + + if (listener->thread) { + ret = kthread_stop(listener->thread); + if (ret) + pr_err("failed to stop forker thread\n"); + listener->thread = NULL; + } + + if (listener->socket) { + smbdirect_socket_release(listener->socket); + listener->socket = NULL; + } + + listener->port = 0; +} + +static int smb_direct_listen(struct smb_direct_listener *listener, + int port) +{ + struct net *net = current->nsproxy->net_ns; + struct task_struct *kthread; + struct sockaddr_in sin = { + .sin_family = AF_INET, + .sin_addr.s_addr = htonl(INADDR_ANY), + .sin_port = htons(port), + }; + struct smbdirect_socket_parameters init_params = {}; + struct smbdirect_socket_parameters *sp; + struct smbdirect_socket *sc; + u64 port_flags = 0; + int ret; + + switch (port) { + case SMB_DIRECT_PORT_IWARP: + /* + * only allow iWarp devices + * for port 5445. + */ + port_flags |= SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW; + break; + case SMB_DIRECT_PORT_INFINIBAND: + /* + * only allow InfiniBand, RoCEv1 or RoCEv2 + * devices for port 445. + * + * (Basically don't allow iWarp devices) + */ + port_flags |= SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB; + break; + default: + pr_err("unsupported smbdirect port=%d!\n", port); + return -ENODEV; + } + + ret = smbdirect_socket_create_kern(net, &sc); + if (ret) { + pr_err("smbdirect_socket_create_kern() failed: %d %1pe\n", + ret, ERR_PTR(ret)); + return ret; + } + + /* + * Create the initial parameters + */ + sp = &init_params; + sp->flags |= port_flags; + sp->negotiate_timeout_msec = SMB_DIRECT_NEGOTIATE_TIMEOUT * 1000; + sp->initiator_depth = SMB_DIRECT_CM_INITIATOR_DEPTH; + sp->responder_resources = 1; + sp->recv_credit_max = smb_direct_receive_credit_max; + sp->send_credit_target = smb_direct_send_credit_target; + sp->max_send_size = smb_direct_max_send_size; + sp->max_fragmented_recv_size = smb_direct_max_fragmented_recv_size; + sp->max_recv_size = smb_direct_max_receive_size; + sp->max_read_write_size = smb_direct_max_read_write_size; + sp->keepalive_interval_msec = SMB_DIRECT_KEEPALIVE_SEND_INTERVAL * 1000; + sp->keepalive_timeout_msec = SMB_DIRECT_KEEPALIVE_RECV_TIMEOUT * 1000; + + smbdirect_socket_set_logging(sc, NULL, + smb_direct_logging_needed, + smb_direct_logging_vaprintf); + ret = smbdirect_socket_set_initial_parameters(sc, sp); + if (ret) { + pr_err("Failed smbdirect_socket_set_initial_parameters(): %d %1pe\n", + ret, ERR_PTR(ret)); + goto err; + } + ret = smbdirect_socket_set_kernel_settings(sc, IB_POLL_WORKQUEUE, KSMBD_DEFAULT_GFP); + if (ret) { + pr_err("Failed smbdirect_socket_set_kernel_settings(): %d %1pe\n", + ret, ERR_PTR(ret)); + goto err; + } + + ret = smbdirect_socket_bind(sc, (struct sockaddr *)&sin); + if (ret) { + pr_err("smbdirect_socket_bind() failed: %d %1pe\n", + ret, ERR_PTR(ret)); + goto err; + } + + ret = smbdirect_socket_listen(sc, 10); + if (ret) { + pr_err("Port[%d] smbdirect_socket_listen() failed: %d %1pe\n", + port, ret, ERR_PTR(ret)); + goto err; + } + + listener->port = port; + listener->socket = sc; + + kthread = kthread_run(smb_direct_listener_kthread_fn, + listener, + "ksmbd-smbdirect-listener-%u", port); + if (IS_ERR(kthread)) { + ret = PTR_ERR(kthread); + pr_err("Can't start ksmbd listen kthread: %d %1pe\n", + ret, ERR_PTR(ret)); + goto err; + } + + listener->thread = kthread; + return 0; +err: + smb_direct_listener_destroy(listener); + return ret; +} + +int ksmbd_rdma_init(void) +{ + int ret; + + smb_direct_ib_listener = smb_direct_iw_listener = (struct smb_direct_listener) { + .socket = NULL, + }; + + ret = smb_direct_listen(&smb_direct_ib_listener, + SMB_DIRECT_PORT_INFINIBAND); + if (ret) { + pr_err("Can't listen on InfiniBand/RoCEv1/RoCEv2: %d\n", ret); + goto err; + } + + ksmbd_debug(RDMA, "InfiniBand/RoCEv1/RoCEv2 RDMA listener. socket=%p\n", + smb_direct_ib_listener.socket); + + ret = smb_direct_listen(&smb_direct_iw_listener, + SMB_DIRECT_PORT_IWARP); + if (ret) { + pr_err("Can't listen on iWarp: %d\n", ret); + goto err; + } + + ksmbd_debug(RDMA, "iWarp RDMA listener. socket=%p\n", + smb_direct_iw_listener.socket); + + WRITE_ONCE(smb_direct_enabled, true); + return 0; +err: + WRITE_ONCE(smb_direct_enabled, false); + ksmbd_rdma_stop_listening(); + return ret; +} + +void ksmbd_rdma_stop_listening(void) +{ + WRITE_ONCE(smb_direct_enabled, false); + smb_direct_listener_destroy(&smb_direct_ib_listener); + smb_direct_listener_destroy(&smb_direct_iw_listener); +} + +bool ksmbd_rdma_enabled(void) +{ + return READ_ONCE(smb_direct_enabled); +} + +bool ksmbd_rdma_capable_netdev(struct net_device *netdev) +{ + u8 node_type = smbdirect_netdev_rdma_capable_node_type(netdev); + + return node_type != RDMA_NODE_UNSPECIFIED; +} + +static const struct ksmbd_transport_ops ksmbd_smb_direct_transport_ops = { + .disconnect = smb_direct_disconnect, + .shutdown = smb_direct_shutdown, + .writev = smb_direct_writev, + .read = smb_direct_read, + .rdma_read = smb_direct_rdma_read, + .rdma_write = smb_direct_rdma_write, + .free_transport = smb_direct_free_transport, +}; + +MODULE_IMPORT_NS("SMBDIRECT"); diff --git a/fs/smb/server/transport_rdma.h b/fs/smb/server/transport_rdma.h new file mode 100644 index 000000000..23247713b --- /dev/null +++ b/fs/smb/server/transport_rdma.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2017, Microsoft Corporation. + * Copyright (C) 2018, LG Electronics. + */ + +#ifndef __KSMBD_TRANSPORT_RDMA_H__ +#define __KSMBD_TRANSPORT_RDMA_H__ + +#define SMBD_DEFAULT_IOSIZE (8 * 1024 * 1024) +#define SMBD_MIN_IOSIZE (512 * 1024) +#define SMBD_MAX_IOSIZE (16 * 1024 * 1024) + +#ifdef CONFIG_SMB_SERVER_SMBDIRECT +int ksmbd_rdma_init(void); +void ksmbd_rdma_stop_listening(void); +bool ksmbd_rdma_enabled(void); +bool ksmbd_rdma_capable_netdev(struct net_device *netdev); +void init_smbd_max_io_size(unsigned int sz); +unsigned int get_smbd_max_read_write_size(struct ksmbd_transport *kt); +#else +static inline int ksmbd_rdma_init(void) { return 0; } +static inline void ksmbd_rdma_stop_listening(void) { } +static inline bool ksmbd_rdma_enabled(void) { return false; } +static inline bool ksmbd_rdma_capable_netdev(struct net_device *netdev) { return false; } +static inline void init_smbd_max_io_size(unsigned int sz) { } +static inline unsigned int get_smbd_max_read_write_size(struct ksmbd_transport *kt) { return 0; } +#endif + +#include + +#endif /* __KSMBD_TRANSPORT_RDMA_H__ */ diff --git a/fs/smb/server/transport_tcp.c b/fs/smb/server/transport_tcp.c new file mode 100644 index 000000000..832e93084 --- /dev/null +++ b/fs/smb/server/transport_tcp.c @@ -0,0 +1,703 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2016 Namjae Jeon + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#include + +#include "smb_common.h" +#include "server.h" +#include "auth.h" +#include "connection.h" +#include "transport_tcp.h" + +#define IFACE_STATE_DOWN BIT(0) +#define IFACE_STATE_CONFIGURED BIT(1) + +static atomic_t active_num_conn; + +struct interface { + struct task_struct *ksmbd_kthread; + struct socket *ksmbd_socket; + struct list_head entry; + char *name; + int state; +}; + +static LIST_HEAD(iface_list); + +static int bind_additional_ifaces; + +struct tcp_transport { + struct ksmbd_transport transport; + struct socket *sock; + struct kvec *iov; + unsigned int nr_iov; +}; + +static const struct ksmbd_transport_ops ksmbd_tcp_transport_ops; + +static void tcp_stop_kthread(struct task_struct *kthread); +static struct interface *alloc_iface(char *ifname); +static void ksmbd_tcp_disconnect(struct ksmbd_transport *t); + +#define KSMBD_TRANS(t) (&(t)->transport) +#define TCP_TRANS(t) ((struct tcp_transport *)container_of(t, \ + struct tcp_transport, transport)) + +static inline void ksmbd_tcp_nodelay(struct socket *sock) +{ + tcp_sock_set_nodelay(sock->sk); +} + +static inline void ksmbd_tcp_reuseaddr(struct socket *sock) +{ + sock_set_reuseaddr(sock->sk); +} + +static struct tcp_transport *alloc_transport(struct socket *client_sk) +{ + struct tcp_transport *t; + struct ksmbd_conn *conn; + + t = kzalloc_obj(*t, KSMBD_DEFAULT_GFP); + if (!t) + return NULL; + t->sock = client_sk; + + conn = ksmbd_conn_alloc(); + if (!conn) { + kfree(t); + return NULL; + } + +#if IS_ENABLED(CONFIG_IPV6) + if (client_sk->sk->sk_family == AF_INET6) { + memcpy(&conn->inet6_addr, &client_sk->sk->sk_v6_daddr, 16); + conn->inet_hash = ipv6_addr_hash(&client_sk->sk->sk_v6_daddr); + } else { + conn->inet_addr = inet_sk(client_sk->sk)->inet_daddr; + conn->inet_hash = ipv4_addr_hash(inet_sk(client_sk->sk)->inet_daddr); + } +#else + conn->inet_addr = inet_sk(client_sk->sk)->inet_daddr; + conn->inet_hash = ipv4_addr_hash(inet_sk(client_sk->sk)->inet_daddr); +#endif + down_write(&conn_list_lock); + hash_add(conn_list, &conn->hlist, conn->inet_hash); + up_write(&conn_list_lock); + + conn->transport = KSMBD_TRANS(t); + KSMBD_TRANS(t)->conn = conn; + KSMBD_TRANS(t)->ops = &ksmbd_tcp_transport_ops; + return t; +} + +static void ksmbd_tcp_free_transport(struct ksmbd_transport *kt) +{ + struct tcp_transport *t = TCP_TRANS(kt); + + sock_release(t->sock); + kfree(t->iov); + kfree(t); +} + +static void free_transport(struct tcp_transport *t) +{ + kernel_sock_shutdown(t->sock, SHUT_RDWR); + ksmbd_conn_free(KSMBD_TRANS(t)->conn); +} + +/** + * kvec_array_init() - initialize a IO vector segment + * @new: IO vector to be initialized + * @iov: base IO vector + * @nr_segs: number of segments in base iov + * @bytes: total iovec length so far for read + * + * Return: Number of IO segments + */ +static unsigned int kvec_array_init(struct kvec *new, struct kvec *iov, + unsigned int nr_segs, size_t bytes) +{ + size_t base = 0; + + while (bytes || !iov->iov_len) { + int copy = min(bytes, iov->iov_len); + + bytes -= copy; + base += copy; + if (iov->iov_len == base) { + iov++; + nr_segs--; + base = 0; + } + } + + memcpy(new, iov, sizeof(*iov) * nr_segs); + new->iov_base += base; + new->iov_len -= base; + return nr_segs; +} + +/** + * get_conn_iovec() - get connection iovec for reading from socket + * @t: TCP transport instance + * @nr_segs: number of segments in iov + * + * Return: return existing or newly allocate iovec + */ +static struct kvec *get_conn_iovec(struct tcp_transport *t, unsigned int nr_segs) +{ + struct kvec *new_iov; + + if (t->iov && nr_segs <= t->nr_iov) + return t->iov; + + /* not big enough -- allocate a new one and release the old */ + new_iov = kmalloc_objs(*new_iov, nr_segs, KSMBD_DEFAULT_GFP); + if (new_iov) { + kfree(t->iov); + t->iov = new_iov; + t->nr_iov = nr_segs; + } + return new_iov; +} + +/** + * ksmbd_tcp_new_connection() - create a new tcp session on mount + * @client_sk: socket associated with new connection + * + * whenever a new connection is requested, create a conn thread + * (session thread) to handle new incoming smb requests from the connection + * + * Return: 0 on success, otherwise error + */ +static int ksmbd_tcp_new_connection(struct socket *client_sk) +{ + int rc = 0; + struct tcp_transport *t; + struct task_struct *handler; + + t = alloc_transport(client_sk); + if (!t) { + sock_release(client_sk); + if (server_conf.max_connections) + atomic_dec(&active_num_conn); + return -ENOMEM; + } + +#if IS_ENABLED(CONFIG_IPV6) + if (client_sk->sk->sk_family == AF_INET6) + handler = kthread_run(ksmbd_conn_handler_loop, + KSMBD_TRANS(t)->conn, "ksmbd:%pI6c", + &KSMBD_TRANS(t)->conn->inet6_addr); + else + handler = kthread_run(ksmbd_conn_handler_loop, + KSMBD_TRANS(t)->conn, "ksmbd:%pI4", + &KSMBD_TRANS(t)->conn->inet_addr); +#else + handler = kthread_run(ksmbd_conn_handler_loop, + KSMBD_TRANS(t)->conn, "ksmbd:%pI4", + &KSMBD_TRANS(t)->conn->inet_addr); +#endif + if (IS_ERR(handler)) { + pr_err("cannot start conn thread\n"); + rc = PTR_ERR(handler); + ksmbd_tcp_disconnect(KSMBD_TRANS(t)); + } + return rc; +} + +/** + * ksmbd_kthread_fn() - listen to new SMB connections and callback server + * @p: arguments to forker thread + * + * Return: 0 on success, error number otherwise + */ +static int ksmbd_kthread_fn(void *p) +{ + struct socket *client_sk = NULL; + struct interface *iface = (struct interface *)p; + struct ksmbd_conn *conn; + int ret, inet_hash; + unsigned int max_ip_conns; + + while (!kthread_should_stop()) { + if (!iface->ksmbd_socket) { + break; + } + ret = kernel_accept(iface->ksmbd_socket, &client_sk, 0); + if (ret == -EINVAL) + break; + if (ret) + continue; + + if (!server_conf.max_ip_connections) + goto skip_max_ip_conns_limit; + + /* + * Limits repeated connections from clients with the same IP. + */ +#if IS_ENABLED(CONFIG_IPV6) + if (client_sk->sk->sk_family == AF_INET6) + inet_hash = ipv6_addr_hash(&client_sk->sk->sk_v6_daddr); + else + inet_hash = ipv4_addr_hash(inet_sk(client_sk->sk)->inet_daddr); +#else + inet_hash = ipv4_addr_hash(inet_sk(client_sk->sk)->inet_daddr); +#endif + + max_ip_conns = 0; + down_read(&conn_list_lock); + hash_for_each_possible(conn_list, conn, hlist, inet_hash) { +#if IS_ENABLED(CONFIG_IPV6) + if (client_sk->sk->sk_family == AF_INET6) { + if (memcmp(&client_sk->sk->sk_v6_daddr, + &conn->inet6_addr, 16) == 0) + max_ip_conns++; + } else if (inet_sk(client_sk->sk)->inet_daddr == + conn->inet_addr) + max_ip_conns++; +#else + if (inet_sk(client_sk->sk)->inet_daddr == + conn->inet_addr) + max_ip_conns++; +#endif + if (server_conf.max_ip_connections <= max_ip_conns) { + pr_info_ratelimited("Maximum IP connections exceeded (%u/%u)\n", + max_ip_conns, server_conf.max_ip_connections); + ret = -EAGAIN; + break; + } + } + up_read(&conn_list_lock); + if (ret == -EAGAIN) { + /* Per-IP limit hit: release the just-accepted socket. */ + sock_release(client_sk); + continue; + } + +skip_max_ip_conns_limit: + if (server_conf.max_connections && + atomic_inc_return(&active_num_conn) > server_conf.max_connections) { + pr_info_ratelimited("Limit the maximum number of connections(%u)\n", + atomic_read(&active_num_conn)); + atomic_dec(&active_num_conn); + sock_release(client_sk); + continue; + } + + ksmbd_debug(CONN, "connect success: accepted new connection\n"); + client_sk->sk->sk_rcvtimeo = KSMBD_TCP_RECV_TIMEOUT; + client_sk->sk->sk_sndtimeo = KSMBD_TCP_SEND_TIMEOUT; + /* + * Detect peers that disappear without sending a FIN or RST. + * Otherwise the connection handler can retry receive timeouts + * indefinitely and keep the connection in conn_list. + */ + sock_set_keepalive(client_sk->sk); + + ksmbd_tcp_new_connection(client_sk); + } + + ksmbd_debug(CONN, "releasing socket\n"); + return 0; +} + +/** + * ksmbd_tcp_run_kthread() - start forker thread + * @iface: pointer to struct interface + * + * start forker thread(ksmbd/0) at module init time to listen + * on port 445 for new SMB connection requests. It creates per connection + * server threads(ksmbd/x) + * + * Return: 0 on success or error number + */ +static int ksmbd_tcp_run_kthread(struct interface *iface) +{ + int rc; + struct task_struct *kthread; + + kthread = kthread_run(ksmbd_kthread_fn, (void *)iface, "ksmbd-%s", + iface->name); + if (IS_ERR(kthread)) { + rc = PTR_ERR(kthread); + return rc; + } + iface->ksmbd_kthread = kthread; + + return 0; +} + +/** + * ksmbd_tcp_readv() - read data from socket in given iovec + * @t: TCP transport instance + * @iov_orig: base IO vector + * @nr_segs: number of segments in base iov + * @to_read: number of bytes to read from socket + * @max_retries: maximum retry count + * + * Return: on success return number of bytes read from socket, + * otherwise return error number + */ +static int ksmbd_tcp_readv(struct tcp_transport *t, struct kvec *iov_orig, + unsigned int nr_segs, unsigned int to_read, + int max_retries) +{ + int length = 0; + int total_read; + unsigned int segs; + struct msghdr ksmbd_msg; + struct kvec *iov; + struct ksmbd_conn *conn = KSMBD_TRANS(t)->conn; + + iov = get_conn_iovec(t, nr_segs); + if (!iov) + return -ENOMEM; + + ksmbd_msg.msg_control = NULL; + ksmbd_msg.msg_controllen = 0; + + for (total_read = 0; to_read; total_read += length, to_read -= length) { + try_to_freeze(); + + if (!ksmbd_conn_alive(conn)) { + total_read = -ESHUTDOWN; + break; + } + segs = kvec_array_init(iov, iov_orig, nr_segs, total_read); + + length = kernel_recvmsg(t->sock, &ksmbd_msg, + iov, segs, to_read, 0); + + if (length == -EINTR) { + total_read = -ESHUTDOWN; + break; + } else if (ksmbd_conn_need_reconnect(conn)) { + total_read = -EAGAIN; + break; + } else if (length == -ERESTARTSYS || length == -EAGAIN) { + /* + * If max_retries is negative, Allow unlimited + * retries to keep connection with inactive sessions. + */ + if (max_retries == 0) { + total_read = length; + break; + } else if (max_retries > 0) { + max_retries--; + } + + usleep_range(1000, 2000); + length = 0; + continue; + } else if (length <= 0) { + total_read = length; + break; + } + } + return total_read; +} + +/** + * ksmbd_tcp_read() - read data from socket in given buffer + * @t: TCP transport instance + * @buf: buffer to store read data from socket + * @to_read: number of bytes to read from socket + * @max_retries: number of retries if reading from socket fails + * + * Return: on success return number of bytes read from socket, + * otherwise return error number + */ +static int ksmbd_tcp_read(struct ksmbd_transport *t, char *buf, + unsigned int to_read, int max_retries) +{ + struct kvec iov; + + iov.iov_base = buf; + iov.iov_len = to_read; + + return ksmbd_tcp_readv(TCP_TRANS(t), &iov, 1, to_read, max_retries); +} + +static int ksmbd_tcp_writev(struct ksmbd_transport *t, + const struct ksmbd_transport_write *tx) +{ + struct msghdr smb_msg = { + .msg_flags = MSG_NOSIGNAL | tx->msg_flags, + }; + + return kernel_sendmsg(TCP_TRANS(t)->sock, &smb_msg, tx->iov, + tx->iov_cnt, tx->size); +} + +static void ksmbd_tcp_disconnect(struct ksmbd_transport *t) +{ + free_transport(TCP_TRANS(t)); + if (server_conf.max_connections) + atomic_dec(&active_num_conn); +} + +static void ksmbd_tcp_shutdown(struct ksmbd_transport *t) +{ + kernel_sock_shutdown(TCP_TRANS(t)->sock, SHUT_RDWR); +} + +static void tcp_destroy_socket(struct socket *ksmbd_socket) +{ + int ret; + + if (!ksmbd_socket) + return; + + ret = kernel_sock_shutdown(ksmbd_socket, SHUT_RDWR); + if (ret) + pr_err("Failed to shutdown socket: %d\n", ret); + sock_release(ksmbd_socket); +} + +/** + * create_socket - create socket for ksmbd/0 + * @iface: interface to bind the created socket to + * + * Return: 0 on success, error number otherwise + */ +static int create_socket(struct interface *iface) +{ + int ret; + struct sockaddr_in6 sin6; + struct sockaddr_in sin; + struct socket *ksmbd_socket; + bool ipv4 = false; + + ret = sock_create_kern(current->nsproxy->net_ns, PF_INET6, SOCK_STREAM, + IPPROTO_TCP, &ksmbd_socket); + if (ret) { + if (ret != -EAFNOSUPPORT) + pr_err("Can't create socket for ipv6, fallback to ipv4: %d\n", ret); + ret = sock_create_kern(current->nsproxy->net_ns, PF_INET, + SOCK_STREAM, IPPROTO_TCP, &ksmbd_socket); + if (ret) { + pr_err("Can't create socket for ipv4: %d\n", ret); + goto out_clear; + } + + sin.sin_family = PF_INET; + sin.sin_addr.s_addr = htonl(INADDR_ANY); + sin.sin_port = htons(server_conf.tcp_port); + ipv4 = true; + } else { + sin6.sin6_family = PF_INET6; + sin6.sin6_addr = in6addr_any; + sin6.sin6_port = htons(server_conf.tcp_port); + + lock_sock(ksmbd_socket->sk); + ksmbd_socket->sk->sk_ipv6only = false; + release_sock(ksmbd_socket->sk); + } + + ksmbd_tcp_nodelay(ksmbd_socket); + ksmbd_tcp_reuseaddr(ksmbd_socket); + + ret = sock_setsockopt(ksmbd_socket, + SOL_SOCKET, + SO_BINDTODEVICE, + KERNEL_SOCKPTR(iface->name), + strlen(iface->name)); + if (ret != -ENODEV && ret < 0) { + pr_err("Failed to set SO_BINDTODEVICE: %d\n", ret); + goto out_error; + } + + if (ipv4) + ret = kernel_bind(ksmbd_socket, (struct sockaddr_unsized *)&sin, + sizeof(sin)); + else + ret = kernel_bind(ksmbd_socket, (struct sockaddr_unsized *)&sin6, + sizeof(sin6)); + if (ret) { + pr_err("Failed to bind socket: %d\n", ret); + goto out_error; + } + + /* + * Accepted sockets inherit the listener's net reference. Keep TCP + * timers alive after a kernel socket is released. + */ + sk_net_refcnt_upgrade(ksmbd_socket->sk); + + ret = kernel_listen(ksmbd_socket, KSMBD_SOCKET_BACKLOG); + if (ret) { + pr_err("Port listen() error: %d\n", ret); + goto out_error; + } + + iface->ksmbd_socket = ksmbd_socket; + ret = ksmbd_tcp_run_kthread(iface); + if (ret) { + pr_err("Can't start ksmbd main kthread: %d\n", ret); + goto out_error; + } + iface->state = IFACE_STATE_CONFIGURED; + + return 0; + +out_error: + tcp_destroy_socket(ksmbd_socket); +out_clear: + iface->ksmbd_socket = NULL; + return ret; +} + +struct interface *ksmbd_find_netdev_name_iface_list(char *netdev_name) +{ + struct interface *iface; + + list_for_each_entry(iface, &iface_list, entry) + if (!strcmp(iface->name, netdev_name)) + return iface; + return NULL; +} + +static int ksmbd_netdev_event(struct notifier_block *nb, unsigned long event, + void *ptr) +{ + struct net_device *netdev = netdev_notifier_info_to_dev(ptr); + struct interface *iface; + int ret; + + switch (event) { + case NETDEV_UP: + if (netif_is_bridge_port(netdev)) + return NOTIFY_OK; + + iface = ksmbd_find_netdev_name_iface_list(netdev->name); + if (iface && iface->state == IFACE_STATE_DOWN) { + ksmbd_debug(CONN, "netdev-up event: netdev(%s) is going up\n", + iface->name); + ret = create_socket(iface); + if (ret) + return NOTIFY_OK; + } + if (!iface && bind_additional_ifaces) { + iface = alloc_iface(kstrdup(netdev->name, KSMBD_DEFAULT_GFP)); + if (!iface) + return NOTIFY_OK; + ksmbd_debug(CONN, "netdev-up event: netdev(%s) is going up\n", + iface->name); + ret = create_socket(iface); + if (ret) + break; + } + break; + case NETDEV_DOWN: + iface = ksmbd_find_netdev_name_iface_list(netdev->name); + if (iface && iface->state == IFACE_STATE_CONFIGURED) { + ksmbd_debug(CONN, "netdev-down event: netdev(%s) is going down\n", + iface->name); + kernel_sock_shutdown(iface->ksmbd_socket, SHUT_RDWR); + tcp_stop_kthread(iface->ksmbd_kthread); + iface->ksmbd_kthread = NULL; + sock_release(iface->ksmbd_socket); + iface->ksmbd_socket = NULL; + + iface->state = IFACE_STATE_DOWN; + break; + } + break; + } + + return NOTIFY_DONE; +} + +static struct notifier_block ksmbd_netdev_notifier = { + .notifier_call = ksmbd_netdev_event, +}; + +int ksmbd_tcp_init(void) +{ + register_netdevice_notifier(&ksmbd_netdev_notifier); + + return 0; +} + +static void tcp_stop_kthread(struct task_struct *kthread) +{ + int ret; + + if (!kthread) + return; + + ret = kthread_stop(kthread); + if (ret) + pr_err("failed to stop forker thread\n"); +} + +void ksmbd_tcp_destroy(void) +{ + struct interface *iface, *tmp; + + unregister_netdevice_notifier(&ksmbd_netdev_notifier); + + list_for_each_entry_safe(iface, tmp, &iface_list, entry) { + list_del(&iface->entry); + kfree(iface->name); + kfree(iface); + } +} + +static struct interface *alloc_iface(char *ifname) +{ + struct interface *iface; + + if (!ifname) + return NULL; + + iface = kzalloc_obj(struct interface, KSMBD_DEFAULT_GFP); + if (!iface) { + kfree(ifname); + return NULL; + } + + iface->name = ifname; + iface->state = IFACE_STATE_DOWN; + list_add(&iface->entry, &iface_list); + return iface; +} + +int ksmbd_tcp_set_interfaces(char *ifc_list, int ifc_list_sz) +{ + int sz = 0; + + if (!ifc_list_sz) { + bind_additional_ifaces = 1; + return 0; + } + + while (ifc_list_sz > 0) { + if (!alloc_iface(kstrdup(ifc_list, KSMBD_DEFAULT_GFP))) + return -ENOMEM; + + sz = strlen(ifc_list); + if (!sz) + break; + + ifc_list += sz + 1; + ifc_list_sz -= (sz + 1); + } + + bind_additional_ifaces = 0; + + return 0; +} + +static const struct ksmbd_transport_ops ksmbd_tcp_transport_ops = { + .read = ksmbd_tcp_read, + .writev = ksmbd_tcp_writev, + .disconnect = ksmbd_tcp_disconnect, + .shutdown = ksmbd_tcp_shutdown, + .free_transport = ksmbd_tcp_free_transport, +}; diff --git a/fs/smb/server/transport_tcp.h b/fs/smb/server/transport_tcp.h new file mode 100644 index 000000000..1e51675ee --- /dev/null +++ b/fs/smb/server/transport_tcp.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#ifndef __KSMBD_TRANSPORT_TCP_H__ +#define __KSMBD_TRANSPORT_TCP_H__ + +int ksmbd_tcp_set_interfaces(char *ifc_list, int ifc_list_sz); +struct interface *ksmbd_find_netdev_name_iface_list(char *netdev_name); +void ksmbd_free_transport(struct ksmbd_transport *kt); +int ksmbd_tcp_init(void); +void ksmbd_tcp_destroy(void); + +#endif /* __KSMBD_TRANSPORT_TCP_H__ */ diff --git a/fs/smb/server/unicode.c b/fs/smb/server/unicode.c new file mode 100644 index 000000000..85e679174 --- /dev/null +++ b/fs/smb/server/unicode.c @@ -0,0 +1,454 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Some of the source code in this file came from fs/cifs/cifs_unicode.c + * + * Copyright (c) International Business Machines Corp., 2000,2009 + * Modified by Steve French (sfrench@us.ibm.com) + * Modified by Namjae Jeon (linkinjeon@kernel.org) + */ +#include +#include +#include +#include "glob.h" +#include "unicode.h" +#include "smb_common.h" + +/* + * cifs_mapchar() - convert a host-endian char to proper char in codepage + * @target: where converted character should be copied + * @from: host-endian source string + * @cp: codepage to which character should be converted + * @mapchar: should character be mapped according to mapchars mount option? + * + * This function handles the conversion of a single character. It is the + * responsibility of the caller to ensure that the target buffer is large + * enough to hold the result of the conversion (at least NLS_MAX_CHARSET_SIZE). + * + * Return: string length after conversion + */ +static int +cifs_mapchar(char *target, const __u16 *from, const struct nls_table *cp, + bool mapchar) +{ + int len = 1; + __u16 src_char; + + src_char = *from; + + if (!mapchar) + goto cp_convert; + + /* + * BB: Cannot handle remapping UNI_SLASH until all the calls to + * build_path_from_dentry are modified, as they use slash as + * separator. + */ + switch (src_char) { + case UNI_COLON: + *target = ':'; + break; + case UNI_ASTERISK: + *target = '*'; + break; + case UNI_QUESTION: + *target = '?'; + break; + case UNI_PIPE: + *target = '|'; + break; + case UNI_GRTRTHAN: + *target = '>'; + break; + case UNI_LESSTHAN: + *target = '<'; + break; + default: + goto cp_convert; + } + +out: + return len; + +cp_convert: + len = cp->uni2char(src_char, target, NLS_MAX_CHARSET_SIZE); + if (len <= 0) + goto surrogate_pair; + + goto out; + +surrogate_pair: + /* convert SURROGATE_PAIR and IVS */ + if (strcmp(cp->charset, "utf8")) + goto unknown; + len = utf16s_to_utf8s(from, 3, UTF16_LITTLE_ENDIAN, target, 6); + if (len <= 0) + goto unknown; + return len; + +unknown: + *target = '?'; + len = 1; + goto out; +} + +/* + * smb_utf16_bytes() - compute converted string length + * @from: pointer to input string + * @maxbytes: input string length + * @codepage: destination codepage + * + * Walk a utf16le string and return the number of bytes that the string will + * be after being converted to the given charset, not including any null + * termination required. Don't walk past maxbytes in the source buffer. + * + * Return: string length after conversion + */ +static int smb_utf16_bytes(const __le16 *from, int maxbytes, + const struct nls_table *codepage) +{ + int i, j; + int charlen, outlen = 0; + int maxwords = maxbytes / 2; + char tmp[NLS_MAX_CHARSET_SIZE]; + __u16 ftmp[3]; + + for (i = 0; i < maxwords; i++) { + ftmp[0] = get_unaligned_le16(&from[i]); + if (ftmp[0] == 0) + break; + for (j = 1; j <= 2; j++) { + if (i + j < maxwords) + ftmp[j] = get_unaligned_le16(&from[i + j]); + else + ftmp[j] = 0; + } + + charlen = cifs_mapchar(tmp, ftmp, codepage, 0); + if (charlen > 0) + outlen += charlen; + else + outlen++; + } + + return outlen; +} + +/* + * smb_from_utf16() - convert utf16le string to local charset + * @to: destination buffer + * @from: source buffer + * @tolen: destination buffer size (in bytes) + * @fromlen: source buffer size (in bytes) + * @codepage: codepage to which characters should be converted + * @mapchar: should characters be remapped according to the mapchars option? + * + * Convert a little-endian utf16le string (as sent by the server) to a string + * in the provided codepage. The tolen and fromlen parameters are to ensure + * that the code doesn't walk off of the end of the buffer (which is always + * a danger if the alignment of the source buffer is off). The destination + * string is always properly null terminated and fits in the destination + * buffer. Returns the length of the destination string in bytes (including + * null terminator). + * + * Note that some windows versions actually send multiword UTF-16 characters + * instead of straight UTF16-2. The linux nls routines however aren't able to + * deal with those characters properly. In the event that we get some of + * those characters, they won't be translated properly. + * + * Return: string length after conversion + */ +static int smb_from_utf16(char *to, const __le16 *from, int tolen, int fromlen, + const struct nls_table *codepage, bool mapchar) +{ + int i, j, charlen, safelen; + int outlen = 0; + int nullsize = nls_nullsize(codepage); + int fromwords = fromlen / 2; + char tmp[NLS_MAX_CHARSET_SIZE]; + __u16 ftmp[3]; /* ftmp[3] = 3array x 2bytes = 6bytes UTF-16 */ + + /* + * because the chars can be of varying widths, we need to take care + * not to overflow the destination buffer when we get close to the + * end of it. Until we get to this offset, we don't need to check + * for overflow however. + */ + safelen = tolen - (NLS_MAX_CHARSET_SIZE + nullsize); + + for (i = 0; i < fromwords; i++) { + ftmp[0] = get_unaligned_le16(&from[i]); + if (ftmp[0] == 0) + break; + for (j = 1; j <= 2; j++) { + if (i + j < fromwords) + ftmp[j] = get_unaligned_le16(&from[i + j]); + else + ftmp[j] = 0; + } + + /* + * check to see if converting this character might make the + * conversion bleed into the null terminator + */ + if (outlen >= safelen) { + charlen = cifs_mapchar(tmp, ftmp, codepage, mapchar); + if ((outlen + charlen) > (tolen - nullsize)) + break; + } + + /* put converted char into 'to' buffer */ + charlen = cifs_mapchar(&to[outlen], ftmp, codepage, mapchar); + outlen += charlen; + + /* + * charlen (=bytes of UTF-8 for 1 character) + * 4bytes UTF-8(surrogate pair) is charlen=4 + * (4bytes UTF-16 code) + * 7-8bytes UTF-8(IVS) is charlen=3+4 or 4+4 + * (2 UTF-8 pairs divided to 2 UTF-16 pairs) + */ + if (charlen == 4) + i++; + else if (charlen >= 5) + /* 5-6bytes UTF-8 */ + i += 2; + } + + /* properly null-terminate string */ + for (i = 0; i < nullsize; i++) + to[outlen++] = 0; + + return outlen; +} + +/* + * smb_strtoUTF16() - Convert character string to unicode string + * @to: destination buffer + * @from: source buffer + * @len: destination buffer size (in bytes) + * @codepage: codepage to which characters should be converted + * + * Return: string length after conversion + */ +int smb_strtoUTF16(__le16 *to, const char *from, int len, + const struct nls_table *codepage) +{ + int charlen; + int i; + wchar_t wchar_to; /* needed to quiet sparse */ + + /* special case for utf8 to handle no plane0 chars */ + if (!strcmp(codepage->charset, "utf8")) { + /* + * convert utf8 -> utf16, we assume we have enough space + * as caller should have assumed conversion does not overflow + * in destination len is length in wchar_t units (16bits) + */ + i = utf8s_to_utf16s(from, len, UTF16_LITTLE_ENDIAN, + (wchar_t *)to, len); + + /* if success terminate and exit */ + if (i >= 0) + goto success; + /* + * if fails fall back to UCS encoding as this + * function should not return negative values + * currently can fail only if source contains + * invalid encoded characters + */ + } + + for (i = 0; len > 0 && *from; i++, from += charlen, len -= charlen) { + charlen = codepage->char2uni(from, len, &wchar_to); + if (charlen < 1) { + /* A question mark */ + wchar_to = 0x003f; + charlen = 1; + } + put_unaligned_le16(wchar_to, &to[i]); + } + +success: + put_unaligned_le16(0, &to[i]); + return i; +} + +/* + * smb_strndup_from_utf16() - copy a string from wire format to the local + * codepage + * @src: source string + * @maxlen: don't walk past this many bytes in the source string + * @is_unicode: is this a unicode string? + * @codepage: destination codepage + * + * Take a string given by the server, convert it to the local codepage and + * put it in a new buffer. Returns a pointer to the new string or NULL on + * error. + * + * Return: destination string buffer or error ptr + */ +char *smb_strndup_from_utf16(const char *src, const int maxlen, + const bool is_unicode, + const struct nls_table *codepage) +{ + int len, ret; + char *dst; + + if (is_unicode) { + len = smb_utf16_bytes((__le16 *)src, maxlen, codepage); + len += nls_nullsize(codepage); + dst = kmalloc(len, KSMBD_DEFAULT_GFP); + if (!dst) + return ERR_PTR(-ENOMEM); + ret = smb_from_utf16(dst, (__le16 *)src, len, maxlen, codepage, + false); + if (ret < 0) { + kfree(dst); + return ERR_PTR(-EINVAL); + } + } else { + len = strnlen(src, maxlen); + len++; + dst = kmalloc(len, KSMBD_DEFAULT_GFP); + if (!dst) + return ERR_PTR(-ENOMEM); + strscpy(dst, src, len); + } + + return dst; +} + +/* + * Convert 16 bit Unicode pathname to wire format from string in current code + * page. Conversion may involve remapping up the six characters that are + * only legal in POSIX-like OS (if they are present in the string). Path + * names are little endian 16 bit Unicode on the wire + */ +/* + * smbConvertToUTF16() - convert string from local charset to utf16 + * @target: destination buffer + * @source: source buffer + * @srclen: source buffer size (in bytes) + * @cp: codepage to which characters should be converted + * @mapchar: should characters be remapped according to the mapchars option? + * + * Convert 16 bit Unicode pathname to wire format from string in current code + * page. Conversion may involve remapping up the six characters that are + * only legal in POSIX-like OS (if they are present in the string). Path + * names are little endian 16 bit Unicode on the wire + * + * Return: char length after conversion + */ +int smbConvertToUTF16(__le16 *target, const char *source, int srclen, + const struct nls_table *cp, int mapchars) +{ + int i, j, charlen; + char src_char; + __le16 dst_char; + wchar_t tmp; + wchar_t wchar_to[6]; /* UTF-16 */ + int ret; + unicode_t u; + + if (!mapchars) + return smb_strtoUTF16(target, source, srclen, cp); + + for (i = 0, j = 0; i < srclen; j++) { + src_char = source[i]; + charlen = 1; + switch (src_char) { + case 0: + put_unaligned(0, &target[j]); + return j; + case ':': + dst_char = cpu_to_le16(UNI_COLON); + break; + case '*': + dst_char = cpu_to_le16(UNI_ASTERISK); + break; + case '?': + dst_char = cpu_to_le16(UNI_QUESTION); + break; + case '<': + dst_char = cpu_to_le16(UNI_LESSTHAN); + break; + case '>': + dst_char = cpu_to_le16(UNI_GRTRTHAN); + break; + case '|': + dst_char = cpu_to_le16(UNI_PIPE); + break; + /* + * FIXME: We can not handle remapping backslash (UNI_SLASH) + * until all the calls to build_path_from_dentry are modified, + * as they use backslash as separator. + */ + default: + charlen = cp->char2uni(source + i, srclen - i, &tmp); + dst_char = cpu_to_le16(tmp); + + /* + * if no match, use question mark, which at least in + * some cases serves as wild card + */ + if (charlen > 0) + goto ctoUTF16; + + /* convert SURROGATE_PAIR */ + if (strcmp(cp->charset, "utf8")) + goto unknown; + if (*(source + i) & 0x80) { + charlen = utf8_to_utf32(source + i, 6, &u); + if (charlen < 0) + goto unknown; + } else + goto unknown; + ret = utf8s_to_utf16s(source + i, charlen, + UTF16_LITTLE_ENDIAN, + wchar_to, 6); + if (ret < 0) + goto unknown; + + i += charlen; + dst_char = cpu_to_le16(*wchar_to); + if (charlen <= 3) + /* 1-3bytes UTF-8 to 2bytes UTF-16 */ + put_unaligned(dst_char, &target[j]); + else if (charlen == 4) { + /* + * 4bytes UTF-8(surrogate pair) to 4bytes UTF-16 + * 7-8bytes UTF-8(IVS) divided to 2 UTF-16 + * (charlen=3+4 or 4+4) + */ + put_unaligned(dst_char, &target[j]); + dst_char = cpu_to_le16(*(wchar_to + 1)); + j++; + put_unaligned(dst_char, &target[j]); + } else if (charlen >= 5) { + /* 5-6bytes UTF-8 to 6bytes UTF-16 */ + put_unaligned(dst_char, &target[j]); + dst_char = cpu_to_le16(*(wchar_to + 1)); + j++; + put_unaligned(dst_char, &target[j]); + dst_char = cpu_to_le16(*(wchar_to + 2)); + j++; + put_unaligned(dst_char, &target[j]); + } + continue; + +unknown: + dst_char = cpu_to_le16(0x003f); + charlen = 1; + } + +ctoUTF16: + /* + * character may take more than one byte in the source string, + * but will take exactly two bytes in the target string + */ + i += charlen; + put_unaligned(dst_char, &target[j]); + } + + return j; +} diff --git a/fs/smb/server/unicode.h b/fs/smb/server/unicode.h new file mode 100644 index 000000000..28c7c736f --- /dev/null +++ b/fs/smb/server/unicode.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Some of the source code in this file came from fs/cifs/cifs_unicode.c + * cifs_unicode: Unicode kernel case support + * + * Function: + * Convert a unicode character to upper or lower case using + * compressed tables. + * + * Copyright (c) International Business Machines Corp., 2000,2009 + * + * + * Notes: + * These APIs are based on the C library functions. The semantics + * should match the C functions but with expanded size operands. + * + * The upper/lower functions are based on a table created by mkupr. + * This is a compressed table of upper and lower case conversion. + * + */ +#ifndef _SMB_UNICODE_H +#define _SMB_UNICODE_H + +#include +#include +#include +#include +#include "../../nls/nls_ucs2_utils.h" + +#ifdef __KERNEL__ +int smb_strtoUTF16(__le16 *to, const char *from, int len, + const struct nls_table *codepage); +char *smb_strndup_from_utf16(const char *src, const int maxlen, + const bool is_unicode, + const struct nls_table *codepage); +int smbConvertToUTF16(__le16 *target, const char *source, int srclen, + const struct nls_table *cp, int mapchars); +char *ksmbd_extract_sharename(struct unicode_map *um, const char *treename); +#endif + +#endif /* _SMB_UNICODE_H */ diff --git a/fs/smb/server/vfs.c b/fs/smb/server/vfs.c new file mode 100644 index 000000000..d2b524f79 --- /dev/null +++ b/fs/smb/server/vfs.c @@ -0,0 +1,2411 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2016 Namjae Jeon + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "glob.h" +#include "oplock.h" +#include "connection.h" +#include "vfs.h" +#include "vfs_cache.h" +#include "smbacl.h" +#include "ndr.h" +#include "auth.h" +#include "misc.h" +#include "stats.h" + +#include "smb_common.h" +#include "mgmt/share_config.h" +#include "mgmt/tree_connect.h" +#include "mgmt/user_session.h" +#include "mgmt/user_config.h" + +static void ksmbd_vfs_inherit_owner(struct ksmbd_work *work, + struct inode *parent_inode, + struct inode *inode) +{ + if (!test_share_config_flag(work->tcon->share_conf, + KSMBD_SHARE_FLAG_INHERIT_OWNER)) + return; + + i_uid_write(inode, i_uid_read(parent_inode)); +} + +static int ksmbd_vfs_path_lookup(struct ksmbd_share_config *share_conf, + char *pathname, unsigned int flags, + struct path *path, bool for_remove) +{ + struct qstr last; + const struct path *root_share_path = &share_conf->vfs_path; + int err; + struct dentry *d; + + if (pathname[0] == '\0') { + pathname = share_conf->path; + root_share_path = NULL; + } else { + flags |= LOOKUP_BENEATH; + } + + CLASS(filename_kernel, filename)(pathname); + scoped_with_init_fs() + err = vfs_path_parent_lookup(filename, flags, path, &last, + root_share_path); + if (err) + return err; + + if (for_remove) { + err = mnt_want_write(path->mnt); + if (err) { + path_put(path); + return -ENOENT; + } + + d = start_removing_noperm(path->dentry, &last); + + if (!IS_ERR(d)) { + dput(path->dentry); + path->dentry = d; + return 0; + } + mnt_drop_write(path->mnt); + path_put(path); + return -ENOENT; + } + + d = lookup_noperm_unlocked(&last, path->dentry); + if (!IS_ERR(d) && d_is_negative(d)) { + dput(d); + d = ERR_PTR(-ENOENT); + } + if (IS_ERR(d)) { + path_put(path); + return -ENOENT; + } + dput(path->dentry); + path->dentry = d; + + if (test_share_config_flag(share_conf, KSMBD_SHARE_FLAG_CROSSMNT)) { + err = follow_down(path, 0); + if (err < 0) { + path_put(path); + return -ENOENT; + } + } + return 0; +} + +void ksmbd_vfs_query_maximal_access(struct mnt_idmap *idmap, + struct dentry *dentry, __le32 *daccess) +{ + *daccess = cpu_to_le32(FILE_READ_ATTRIBUTES | READ_CONTROL); + + if (!inode_permission(idmap, d_inode(dentry), MAY_OPEN | MAY_WRITE)) + *daccess |= cpu_to_le32(WRITE_DAC | WRITE_OWNER | SYNCHRONIZE | + FILE_WRITE_DATA | FILE_APPEND_DATA | + FILE_WRITE_EA | FILE_WRITE_ATTRIBUTES | + FILE_DELETE_CHILD); + + if (!inode_permission(idmap, d_inode(dentry), MAY_OPEN | MAY_READ)) + *daccess |= FILE_READ_DATA_LE | FILE_READ_EA_LE; + + if (!inode_permission(idmap, d_inode(dentry), MAY_OPEN | MAY_EXEC)) + *daccess |= FILE_EXECUTE_LE; + + if (!inode_permission(idmap, d_inode(dentry->d_parent), MAY_EXEC | MAY_WRITE)) + *daccess |= FILE_DELETE_LE; +} + +/** + * ksmbd_vfs_create() - vfs helper for smb create file + * @work: work + * @name: file name that is relative to share + * @mode: file create mode + * + * Return: 0 on success, otherwise error + */ +int ksmbd_vfs_create(struct ksmbd_work *work, const char *name, umode_t mode) +{ + struct path path; + struct dentry *dentry; + int err; + + dentry = ksmbd_vfs_kern_path_create(work, name, + LOOKUP_NO_SYMLINKS, &path); + if (IS_ERR(dentry)) { + err = PTR_ERR(dentry); + if (err != -ENOENT) + pr_err("path create failed for %s, err %d\n", + name, err); + return err; + } + + mode |= S_IFREG; + err = vfs_create(mnt_idmap(path.mnt), dentry, mode, NULL); + if (!err) { + ksmbd_vfs_inherit_owner(work, d_inode(path.dentry), + d_inode(dentry)); + } else { + pr_err("File(%s): creation failed (err:%d)\n", name, err); + } + + end_creating_path(&path, dentry); + return err; +} + +/** + * ksmbd_vfs_mkdir() - vfs helper for smb create directory + * @work: work + * @name: directory name that is relative to share + * @mode: directory create mode + * + * Return: 0 on success, otherwise error + */ +int ksmbd_vfs_mkdir(struct ksmbd_work *work, const char *name, umode_t mode) +{ + struct mnt_idmap *idmap; + struct path path; + struct dentry *dentry, *d; + int err = 0; + + dentry = ksmbd_vfs_kern_path_create(work, name, + LOOKUP_NO_SYMLINKS | LOOKUP_DIRECTORY, + &path); + if (IS_ERR(dentry)) { + err = PTR_ERR(dentry); + if (err != -EEXIST) + ksmbd_debug(VFS, "path create failed for %s, err %d\n", + name, err); + return err; + } + + idmap = mnt_idmap(path.mnt); + mode |= S_IFDIR; + d = dentry; + dentry = vfs_mkdir(idmap, d_inode(path.dentry), dentry, mode, NULL); + if (IS_ERR(dentry)) + err = PTR_ERR(dentry); + else if (d_is_negative(dentry)) + err = -ENOENT; + if (!err && dentry != d) + ksmbd_vfs_inherit_owner(work, d_inode(path.dentry), d_inode(dentry)); + + end_creating_path(&path, dentry); + if (err) + pr_err("mkdir(%s): creation failed (err:%d)\n", name, err); + return err; +} + +static ssize_t ksmbd_vfs_getcasexattr(struct mnt_idmap *idmap, + struct dentry *dentry, char *attr_name, + int attr_name_len, char **attr_value) +{ + char *name, *xattr_list = NULL; + ssize_t value_len = -ENOENT, xattr_list_len; + + xattr_list_len = ksmbd_vfs_listxattr(dentry, &xattr_list); + if (xattr_list_len <= 0) + goto out; + + for (name = xattr_list; name - xattr_list < xattr_list_len; + name += strlen(name) + 1) { + ksmbd_debug(VFS, "%s, len %zd\n", name, strlen(name)); + if (strncasecmp(attr_name, name, attr_name_len)) + continue; + + value_len = ksmbd_vfs_getxattr(idmap, + dentry, + name, + attr_value); + if (value_len < 0) + pr_err("failed to get xattr in file\n"); + break; + } + +out: + kvfree(xattr_list); + return value_len; +} + +static int ksmbd_vfs_stream_read(struct ksmbd_file *fp, char *buf, loff_t *pos, + size_t count) +{ + const struct cred *saved_cred; + ssize_t v_len; + char *stream_buf = NULL; + + ksmbd_debug(VFS, "read stream data pos : %llu, count : %zd\n", + *pos, count); + + saved_cred = override_creds(fp->filp->f_cred); + v_len = ksmbd_vfs_getcasexattr(file_mnt_idmap(fp->filp), + fp->filp->f_path.dentry, + fp->stream.name, + fp->stream.size, + &stream_buf); + revert_creds(saved_cred); + if ((int)v_len <= 0) + return (int)v_len; + + if (v_len <= *pos) { + count = -EINVAL; + goto free_buf; + } + + if (v_len - *pos < count) + count = v_len - *pos; + fp->stream.pos = v_len; + + memcpy(buf, &stream_buf[*pos], count); + +free_buf: + kvfree(stream_buf); + return count; +} + +/** + * check_lock_range() - vfs helper for smb byte range file locking + * @filp: the file to apply the lock to + * @start: lock start byte offset + * @end: lock end byte offset + * @type: byte range type read/write + * + * Return: 0 on success, otherwise error + */ +static int check_lock_range(struct file *filp, loff_t start, loff_t end, + unsigned char type) +{ + struct file_lock *flock; + struct file_lock_context *ctx = locks_inode_context(file_inode(filp)); + int error = 0; + + if (!ctx || list_empty_careful(&ctx->flc_posix)) + return 0; + + spin_lock(&ctx->flc_lock); + for_each_file_lock(flock, &ctx->flc_posix) { + /* check conflict locks */ + if (flock->fl_end >= start && end >= flock->fl_start) { + if (lock_is_read(flock)) { + if (type == WRITE) { + pr_err("not allow write by shared lock\n"); + error = 1; + goto out; + } + } else if (lock_is_write(flock)) { + /* check owner in lock */ + if (flock->c.flc_file != filp) { + error = 1; + pr_err("not allow rw access by exclusive lock from other opens\n"); + goto out; + } + } + } + } +out: + spin_unlock(&ctx->flc_lock); + return error; +} + +/** + * ksmbd_vfs_read() - vfs helper for smb file read + * @work: smb work + * @fp: ksmbd file pointer + * @count: read byte count + * @pos: file pos + * @rbuf: read data buffer + * + * Return: number of read bytes on success, otherwise error + */ +int ksmbd_vfs_read(struct ksmbd_work *work, struct ksmbd_file *fp, size_t count, + loff_t *pos, char *rbuf) +{ + struct file *filp = fp->filp; + ssize_t nbytes = 0; + struct inode *inode = file_inode(filp); + + if (S_ISDIR(inode->i_mode) && !ksmbd_stream_fd(fp)) + return -EISDIR; + + if (unlikely(count == 0)) + return 0; + + if (work->conn->connection_type) { + if (!(fp->daccess & (FILE_READ_DATA_LE | FILE_EXECUTE_LE))) { + pr_err("no right to read(%pD)\n", fp->filp); + return -EACCES; + } + } + + if (ksmbd_stream_fd(fp)) + return ksmbd_vfs_stream_read(fp, rbuf, pos, count); + + if (!work->tcon->posix_extensions) { + int ret; + + ret = check_lock_range(filp, *pos, *pos + count - 1, READ); + if (ret) { + pr_err("unable to read due to lock\n"); + return -EAGAIN; + } + } + + nbytes = kernel_read(filp, rbuf, count, pos); + if (nbytes < 0) { + pr_err("smb read failed, err = %zd\n", nbytes); + return nbytes; + } + + filp->f_pos = *pos; + ksmbd_counter_add(KSMBD_COUNTER_READ_BYTES, (s64)nbytes); + return nbytes; +} + +static int ksmbd_vfs_stream_write(struct ksmbd_file *fp, char *buf, loff_t *pos, + size_t count) +{ + const struct cred *saved_cred; + char *stream_buf = NULL, *wbuf; + struct mnt_idmap *idmap = file_mnt_idmap(fp->filp); + size_t size; + ssize_t v_len; + int err = 0; + + ksmbd_debug(VFS, "write stream data pos : %llu, count : %zd\n", + *pos, count); + + if (*pos >= XATTR_SIZE_MAX) { + pr_err("stream write position %lld is out of bounds\n", *pos); + return -EINVAL; + } + + size = *pos + count; + if (size > XATTR_SIZE_MAX) { + size = XATTR_SIZE_MAX; + count = XATTR_SIZE_MAX - *pos; + } + + saved_cred = override_creds(fp->filp->f_cred); + v_len = ksmbd_vfs_getcasexattr(idmap, + fp->filp->f_path.dentry, + fp->stream.name, + fp->stream.size, + &stream_buf); + if (v_len < 0) { + pr_err("not found stream in xattr : %zd\n", v_len); + err = v_len; + goto out_revert; + } + + if (v_len < size) { + wbuf = kvzalloc(size, KSMBD_DEFAULT_GFP); + if (!wbuf) { + err = -ENOMEM; + goto out_revert; + } + + if (v_len > 0) + memcpy(wbuf, stream_buf, v_len); + kvfree(stream_buf); + stream_buf = wbuf; + } + + memcpy(&stream_buf[*pos], buf, count); + + err = ksmbd_vfs_setxattr(idmap, + &fp->filp->f_path, + fp->stream.name, + (void *)stream_buf, + size, + 0, + true); +out_revert: + revert_creds(saved_cred); + if (err < 0) + goto out; + else + fp->stream.pos = size; + err = 0; +out: + kvfree(stream_buf); + return err; +} + +/** + * ksmbd_vfs_write() - vfs helper for smb file write + * @work: work + * @fp: ksmbd file pointer + * @buf: buf containing data for writing + * @count: read byte count + * @pos: file pos + * @sync: fsync after write + * @written: number of bytes written + * + * Return: 0 on success, otherwise error + */ +int ksmbd_vfs_write(struct ksmbd_work *work, struct ksmbd_file *fp, + char *buf, size_t count, loff_t *pos, bool sync, + ssize_t *written) +{ + struct file *filp; + loff_t offset = *pos; + int err = 0; + + if (work->conn->connection_type) { + if (!(fp->daccess & (FILE_WRITE_DATA_LE | FILE_APPEND_DATA_LE)) || + (S_ISDIR(file_inode(fp->filp)->i_mode) && + !ksmbd_stream_fd(fp))) { + pr_err("no right to write(%pD)\n", fp->filp); + err = -EACCES; + goto out; + } + } + + filp = fp->filp; + + if (ksmbd_stream_fd(fp)) { + err = ksmbd_vfs_stream_write(fp, buf, pos, count); + if (!err) + *written = count; + goto out; + } + + if (!work->tcon->posix_extensions) { + err = check_lock_range(filp, *pos, *pos + count - 1, WRITE); + if (err) { + pr_err("unable to write due to lock\n"); + err = -EAGAIN; + goto out; + } + } + + /* Reserve lease break for parent dir at closing time */ + fp->reserve_lease_break = true; + + /* Do we need to break any of a levelII oplock? */ + smb_break_all_levII_oplock(work, fp, 1); + + err = kernel_write(filp, buf, count, pos); + if (err < 0) { + ksmbd_debug(VFS, "smb write failed, err = %d\n", err); + goto out; + } + + filp->f_pos = *pos; + *written = err; + err = 0; + if (sync) { + err = vfs_fsync_range(filp, offset, offset + *written, 0); + if (err < 0) + pr_err("fsync failed for filename = %pD, err = %d\n", + fp->filp, err); + } + ksmbd_counter_add(KSMBD_COUNTER_WRITE_BYTES, (s64)*written); + +out: + return err; +} + +/** + * ksmbd_vfs_getattr() - vfs helper for smb getattr + * @path: path of dentry + * @stat: pointer to returned kernel stat structure + * Return: 0 on success, otherwise error + */ +int ksmbd_vfs_getattr(const struct path *path, struct kstat *stat) +{ + int err; + + err = vfs_getattr(path, stat, STATX_BASIC_STATS | STATX_BTIME, + AT_STATX_SYNC_AS_STAT); + if (err) + pr_err("getattr failed, err %d\n", err); + return err; +} + +/** + * ksmbd_vfs_fsync() - vfs helper for smb fsync + * @work: work + * @fid: file id of open file + * @p_id: persistent file id + * + * Return: 0 on success, otherwise error + */ +int ksmbd_vfs_fsync(struct ksmbd_work *work, u64 fid, u64 p_id) +{ + struct ksmbd_file *fp; + int err; + + fp = ksmbd_lookup_fd_slow(work, fid, p_id); + if (!fp) { + pr_err("failed to get filp for fid %llu\n", fid); + return -ENOENT; + } + err = vfs_fsync(fp->filp, 0); + if (err < 0) + pr_err("smb fsync failed, err = %d\n", err); + ksmbd_fd_put(work, fp); + return err; +} + +/** + * ksmbd_vfs_remove_file() - vfs helper for smb rmdir or unlink + * @work: work + * @path: path of dentry + * + * Return: 0 on success, otherwise error + */ +int ksmbd_vfs_remove_file(struct ksmbd_work *work, const struct path *path) +{ + struct mnt_idmap *idmap; + struct dentry *parent = path->dentry->d_parent; + int err; + + if (ksmbd_override_fsids(work)) + return -ENOMEM; + + if (!d_inode(path->dentry)->i_nlink) { + err = -ENOENT; + goto out_err; + } + + idmap = mnt_idmap(path->mnt); + if (S_ISDIR(d_inode(path->dentry)->i_mode)) { + err = vfs_rmdir(idmap, d_inode(parent), path->dentry, NULL); + if (err && err != -ENOTEMPTY) + ksmbd_debug(VFS, "rmdir failed, err %d\n", err); + } else { + err = vfs_unlink(idmap, d_inode(parent), path->dentry, NULL); + if (err) + ksmbd_debug(VFS, "unlink failed, err %d\n", err); + } + +out_err: + ksmbd_revert_fsids(work); + return err; +} + +/** + * ksmbd_vfs_link() - vfs helper for creating smb hardlink + * @work: work + * @oldname: source file name + * @newname: hardlink name that is relative to share + * + * Return: 0 on success, otherwise error + */ +int ksmbd_vfs_link(struct ksmbd_work *work, const char *oldname, + const char *newname) +{ + struct path oldpath, newpath; + struct dentry *dentry; + int err; + + if (ksmbd_override_fsids(work)) + return -ENOMEM; + + scoped_with_init_fs() + err = kern_path(oldname, LOOKUP_NO_SYMLINKS, &oldpath); + if (err) { + pr_err("cannot get linux path for %s, err = %d\n", + oldname, err); + goto out1; + } + + dentry = ksmbd_vfs_kern_path_create(work, newname, + LOOKUP_NO_SYMLINKS | LOOKUP_REVAL, + &newpath); + if (IS_ERR(dentry)) { + err = PTR_ERR(dentry); + pr_err("path create err for %s, err %d\n", newname, err); + goto out2; + } + + err = -EXDEV; + if (oldpath.mnt != newpath.mnt) { + pr_err("vfs_link failed err %d\n", err); + goto out3; + } + + err = vfs_link(oldpath.dentry, mnt_idmap(newpath.mnt), + d_inode(newpath.dentry), + dentry, NULL); + if (err) + ksmbd_debug(VFS, "vfs_link failed err %d\n", err); + +out3: + end_creating_path(&newpath, dentry); +out2: + path_put(&oldpath); +out1: + ksmbd_revert_fsids(work); + return err; +} + +int ksmbd_vfs_check_rename_share(struct ksmbd_work *work, + const struct path *old_path) +{ + struct ksmbd_file *parent_fp; + int err = 0; + + parent_fp = ksmbd_lookup_fd_inode(old_path->dentry->d_parent); + if (!parent_fp) + return 0; + + if ((parent_fp->daccess & FILE_DELETE_LE) || + (!parent_fp->attrib_only && + !(parent_fp->saccess & FILE_SHARE_DELETE_LE))) { + ksmbd_debug(VFS, "parent dir blocks delete sharing\n"); + err = -ESHARE; + } + ksmbd_fd_put(work, parent_fp); + return err; +} + +int ksmbd_vfs_rename(struct ksmbd_work *work, struct ksmbd_file *old_fp, + char *newname, int flags) +{ + const struct path *old_path = &old_fp->filp->f_path; + struct dentry *old_child = old_path->dentry; + struct path new_path; + struct qstr new_last; + struct renamedata rd; + struct ksmbd_share_config *share_conf = work->tcon->share_conf; + int err, lookup_flags = LOOKUP_NO_SYMLINKS; + + if (ksmbd_override_fsids(work)) + return -ENOMEM; + + CLASS(filename_kernel, to)(newname); + +retry: + err = vfs_path_parent_lookup(to, lookup_flags | LOOKUP_BENEATH, + &new_path, &new_last, &share_conf->vfs_path); + if (err) + goto out1; + + if (old_path->mnt != new_path.mnt) { + err = -EXDEV; + goto out2; + } + + err = mnt_want_write(old_path->mnt); + if (err) + goto out2; + + rd.mnt_idmap = mnt_idmap(old_path->mnt); + rd.old_parent = NULL; + rd.new_parent = new_path.dentry; + rd.flags = flags; + rd.delegated_inode = NULL; + err = start_renaming_dentry(&rd, lookup_flags, old_child, &new_last); + if (err) + goto out_drop_write; + + if (d_is_dir(old_child) && ksmbd_has_nonposix_open_child(old_fp)) { + err = -EACCES; + goto out3; + } + + /* + * See MS-FSA 2.1.5.15.12. + * An overwrite rename must fail with STATUS_ACCESS_DENIED if the + * existing target still has a non-POSIX open. + */ + if (!(flags & (RENAME_NOREPLACE | RENAME_EXCHANGE)) && + d_inode(rd.new_dentry) && + d_inode(rd.new_dentry) != d_inode(old_child) && + ksmbd_has_other_nonposix_open(rd.new_dentry)) { + err = -EACCES; + goto out3; + } + + err = ksmbd_vfs_check_rename_share(work, old_path); + if (err) + goto out3; + + if (d_is_symlink(rd.new_dentry)) { + err = -EACCES; + goto out3; + } + + err = vfs_rename(&rd); + if (err) + ksmbd_debug(VFS, "vfs_rename failed err %d\n", err); + +out3: + end_renaming(&rd); +out_drop_write: + mnt_drop_write(old_path->mnt); +out2: + path_put(&new_path); + + if (retry_estale(err, lookup_flags)) { + lookup_flags |= LOOKUP_REVAL; + goto retry; + } +out1: + ksmbd_revert_fsids(work); + return err; +} + +/** + * ksmbd_vfs_truncate() - vfs helper for smb file truncate + * @work: work + * @fp: ksmbd file pointer + * @size: truncate to given size + * + * Return: 0 on success, otherwise error + */ +int ksmbd_vfs_truncate(struct ksmbd_work *work, + struct ksmbd_file *fp, loff_t size) +{ + int err = 0; + struct file *filp; + + filp = fp->filp; + + /* Do we need to break any of a levelII oplock? */ + smb_break_all_levII_oplock(work, fp, 1); + + if (!work->tcon->posix_extensions) { + struct inode *inode = file_inode(filp); + + if (size < inode->i_size) { + err = check_lock_range(filp, size, + inode->i_size - 1, WRITE); + } else if (size > inode->i_size) { + err = check_lock_range(filp, inode->i_size, + size - 1, WRITE); + } + + if (err) { + pr_err("failed due to lock\n"); + return -EAGAIN; + } + } + + err = vfs_truncate(&filp->f_path, size); + if (err) + pr_err("truncate failed, err %d\n", err); + return err; +} + +/** + * ksmbd_vfs_listxattr() - vfs helper for smb list extended attributes + * @dentry: dentry of file for listing xattrs + * @list: destination buffer + * + * Return: xattr list length on success, otherwise error + */ +ssize_t ksmbd_vfs_listxattr(struct dentry *dentry, char **list) +{ + ssize_t size; + char *vlist = NULL; + + size = vfs_listxattr(dentry, NULL, 0); + if (size <= 0) + return size; + + vlist = kvzalloc(size, KSMBD_DEFAULT_GFP); + if (!vlist) + return -ENOMEM; + + *list = vlist; + size = vfs_listxattr(dentry, vlist, size); + if (size < 0) { + ksmbd_debug(VFS, "listxattr failed\n"); + kvfree(vlist); + *list = NULL; + } + + return size; +} + +static ssize_t ksmbd_vfs_xattr_len(struct mnt_idmap *idmap, + struct dentry *dentry, char *xattr_name) +{ + return vfs_getxattr(idmap, dentry, xattr_name, NULL, 0); +} + +/** + * ksmbd_vfs_getxattr() - vfs helper for smb get extended attributes value + * @idmap: idmap + * @dentry: dentry of file for getting xattrs + * @xattr_name: name of xattr name to query + * @xattr_buf: destination buffer xattr value + * + * Return: read xattr value length on success, otherwise error + */ +ssize_t ksmbd_vfs_getxattr(struct mnt_idmap *idmap, + struct dentry *dentry, + char *xattr_name, char **xattr_buf) +{ + ssize_t xattr_len; + char *buf; + + *xattr_buf = NULL; + xattr_len = ksmbd_vfs_xattr_len(idmap, dentry, xattr_name); + if (xattr_len < 0) + return xattr_len; + + buf = kmalloc(xattr_len + 1, KSMBD_DEFAULT_GFP); + if (!buf) + return -ENOMEM; + + xattr_len = vfs_getxattr(idmap, dentry, xattr_name, + (void *)buf, xattr_len); + if (xattr_len > 0) + *xattr_buf = buf; + else + kfree(buf); + return xattr_len; +} + +/** + * ksmbd_vfs_setxattr() - vfs helper for smb set extended attributes value + * @idmap: idmap of the relevant mount + * @path: path of dentry to set XATTR at + * @attr_name: xattr name for setxattr + * @attr_value: xattr value to set + * @attr_size: size of xattr value + * @flags: destination buffer length + * @get_write: get write access to a mount + * + * Return: 0 on success, otherwise error + */ +int ksmbd_vfs_setxattr(struct mnt_idmap *idmap, + const struct path *path, const char *attr_name, + void *attr_value, size_t attr_size, int flags, + bool get_write) +{ + int err; + + if (get_write == true) { + err = mnt_want_write(path->mnt); + if (err) + return err; + } + + err = vfs_setxattr(idmap, + path->dentry, + attr_name, + attr_value, + attr_size, + flags); + if (err) + ksmbd_debug(VFS, "setxattr failed, err %d\n", err); + if (get_write == true) + mnt_drop_write(path->mnt); + return err; +} + +/** + * ksmbd_vfs_set_fadvise() - convert smb IO caching options to linux options + * @filp: file pointer for IO + * @option: smb IO options + */ +void ksmbd_vfs_set_fadvise(struct file *filp, __le32 option) +{ + struct address_space *mapping; + + mapping = filp->f_mapping; + + if (!option || !mapping) + return; + + if (option & FILE_WRITE_THROUGH_LE) { + filp->f_flags |= O_SYNC; + } else if (option & FILE_SEQUENTIAL_ONLY_LE) { + filp->f_ra.ra_pages = inode_to_bdi(mapping->host)->ra_pages * 2; + spin_lock(&filp->f_lock); + filp->f_mode &= ~FMODE_RANDOM; + spin_unlock(&filp->f_lock); + } else if (option & FILE_RANDOM_ACCESS_LE) { + spin_lock(&filp->f_lock); + filp->f_mode |= FMODE_RANDOM; + spin_unlock(&filp->f_lock); + } +} + +int ksmbd_vfs_zero_data(struct ksmbd_work *work, struct ksmbd_file *fp, + loff_t off, loff_t len) +{ + const struct cred *saved_cred; + loff_t pos = off, size; + char *zero_buf = NULL; + int err; + + smb_break_all_levII_oplock(work, fp, 1); + if (!work->tcon->posix_extensions) { + loff_t size = i_size_read(file_inode(fp->filp)); + + if (off < size) { + err = check_lock_range(fp->filp, off, + min(off + len, size) - 1, + WRITE); + if (err) + return -EAGAIN; + } + } + + saved_cred = override_creds(fp->filp->f_cred); + if (fp->f_ci->m_fattr & FILE_ATTRIBUTE_SPARSE_FILE_LE) { + err = vfs_fallocate(fp->filp, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + off, len); + } else { + size = i_size_read(file_inode(fp->filp)); + if (off >= size) { + err = 0; + goto out; + } + + len = min(len, size - off); + zero_buf = kvzalloc(SZ_64K, GFP_KERNEL); + if (!zero_buf) { + err = -ENOMEM; + goto out; + } + + while (len) { + ssize_t written; + size_t count = min_t(loff_t, len, SZ_64K); + + written = kernel_write(fp->filp, zero_buf, count, &pos); + if (written < 0) { + err = written; + goto out; + } + if (!written) { + err = -EIO; + goto out; + } + len -= written; + } + err = 0; + } +out: + revert_creds(saved_cred); + kvfree(zero_buf); + return err; +} + +int ksmbd_vfs_zero_holes(struct ksmbd_file *fp) +{ + struct file *f = fp->filp; + const struct cred *saved_cred; + loff_t size, pos = 0; + char *zero_buf; + int err; + + err = file_write_and_wait(f); + if (err) + return err; + + size = i_size_read(file_inode(f)); + if (!size) + return 0; + + /* + * FALLOC_FL_ZERO_RANGE may leave unwritten extents, which SEEK_DATA + * reports as holes. Write zeroes into each hole so that clearing the + * sparse attribute leaves the file fully allocated. + */ + zero_buf = kvzalloc(SZ_64K, GFP_KERNEL); + if (!zero_buf) + return -ENOMEM; + + saved_cred = override_creds(f->f_cred); + while (pos < size) { + loff_t data, hole; + + hole = vfs_llseek(f, pos, SEEK_HOLE); + if (hole == -ENXIO || hole >= size) + break; + if (hole < 0) { + err = hole; + goto out; + } + + data = vfs_llseek(f, hole, SEEK_DATA); + if (data == -ENXIO) { + data = size; + } else if (data < 0) { + err = data; + goto out; + } + data = min(data, size); + if (data <= hole) { + err = -EIO; + goto out; + } + + pos = hole; + while (pos < data) { + ssize_t written; + size_t count = min_t(loff_t, data - pos, SZ_64K); + + written = kernel_write(f, zero_buf, count, &pos); + if (written < 0) { + err = written; + goto out; + } + if (!written) { + err = -EIO; + goto out; + } + } + } + err = file_write_and_wait(f); +out: + revert_creds(saved_cred); + kvfree(zero_buf); + return err; +} + +int ksmbd_vfs_trim_data(struct ksmbd_work *work, struct ksmbd_file *fp, + loff_t off, loff_t len) +{ + const struct cred *saved_cred; + int err; + + smb_break_all_levII_oplock(work, fp, 1); + if (!work->tcon->posix_extensions) { + loff_t size = i_size_read(file_inode(fp->filp)); + + if (off < size) { + err = check_lock_range(fp->filp, off, + min(off + len, size) - 1, + WRITE); + if (err) + return -EAGAIN; + } + } + + saved_cred = override_creds(fp->filp->f_cred); + err = vfs_fallocate(fp->filp, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + off, len); + revert_creds(saved_cred); + return err; +} + +int ksmbd_vfs_query_allocated_ranges(struct ksmbd_file *fp, loff_t start, + loff_t length, + struct file_allocated_range_buffer *ranges, + unsigned int in_count, + unsigned int *out_count) +{ + struct file *f = fp->filp; + struct inode *inode = file_inode(fp->filp); + loff_t maxbytes = inode->i_sb->s_maxbytes, size; + loff_t extent_start, extent_end, end; + int ret = 0; + + *out_count = 0; + if (start < 0 || length < 0) + return -EINVAL; + if (start > maxbytes) + return -EFBIG; + if (!in_count) + return 0; + if (length > maxbytes || maxbytes - length < start) + length = maxbytes - start; + size = i_size_read(inode); + if (!length || start >= size) + return 0; + if (length > size - start) + length = size - start; + + end = start + length; + if ((fp->f_ci->m_fattr & FILE_ATTRIBUTE_SPARSE_FILE_LE) && + start < end) { + ret = file_write_and_wait_range(f, start, end - 1); + if (ret) + return ret; + } + + while (start < end && *out_count < in_count) { + extent_start = vfs_llseek(f, start, SEEK_DATA); + if (extent_start < 0) { + if (extent_start != -ENXIO) + ret = extent_start; + break; + } + + if (extent_start >= end) + break; + + extent_end = vfs_llseek(f, extent_start, SEEK_HOLE); + if (extent_end < 0) { + if (extent_end != -ENXIO) + ret = extent_end; + break; + } else if (extent_start >= extent_end) { + break; + } + + ranges[*out_count].file_offset = cpu_to_le64(extent_start); + ranges[(*out_count)++].length = + cpu_to_le64(min(extent_end, end) - extent_start); + start = extent_end; + } + + if (!ret && start < end && *out_count == in_count) + ret = -E2BIG; + + return ret; +} + +int ksmbd_vfs_remove_xattr(struct mnt_idmap *idmap, + const struct path *path, char *attr_name, + bool get_write) +{ + int err; + + if (get_write == true) { + err = mnt_want_write(path->mnt); + if (err) + return err; + } + + err = vfs_removexattr(idmap, path->dentry, attr_name); + + if (get_write == true) + mnt_drop_write(path->mnt); + + return err; +} + +int ksmbd_vfs_unlink(struct file *filp) +{ + const struct cred *saved_cred; + int err = 0; + struct dentry *dir, *dentry = filp->f_path.dentry; + struct mnt_idmap *idmap = file_mnt_idmap(filp); + + saved_cred = override_creds(filp->f_cred); + err = mnt_want_write(filp->f_path.mnt); + if (err) + goto out_revert; + + dir = dget_parent(dentry); + dentry = start_removing_dentry(dir, dentry); + err = PTR_ERR(dentry); + if (IS_ERR(dentry)) + goto out; + + if (S_ISDIR(d_inode(dentry)->i_mode)) + err = vfs_rmdir(idmap, d_inode(dir), dentry, NULL); + else + err = vfs_unlink(idmap, d_inode(dir), dentry, NULL); + + end_removing(dentry); + if (err) + ksmbd_debug(VFS, "failed to delete, err %d\n", err); +out: + dput(dir); + mnt_drop_write(filp->f_path.mnt); +out_revert: + revert_creds(saved_cred); + return err; +} + +static bool __dir_empty(struct dir_context *ctx, const char *name, int namlen, + loff_t offset, u64 ino, unsigned int d_type) +{ + struct ksmbd_readdir_data *buf; + + buf = container_of(ctx, struct ksmbd_readdir_data, ctx); + if (!name_is_dot_dotdot(name, namlen)) + buf->dirent_count++; + + return !buf->dirent_count; +} + +/** + * ksmbd_vfs_empty_dir() - check for empty directory + * @fp: ksmbd file pointer + * + * Return: true if directory empty, otherwise false + */ +int ksmbd_vfs_empty_dir(struct ksmbd_file *fp) +{ + int err; + struct ksmbd_readdir_data readdir_data; + + memset(&readdir_data, 0, sizeof(struct ksmbd_readdir_data)); + + set_ctx_actor(&readdir_data.ctx, __dir_empty); + readdir_data.dirent_count = 0; + + err = iterate_dir(fp->filp, &readdir_data.ctx); + if (readdir_data.dirent_count) + err = -ENOTEMPTY; + else + err = 0; + return err; +} + +static bool __caseless_lookup(struct dir_context *ctx, const char *name, + int namlen, loff_t offset, u64 ino, + unsigned int d_type) +{ + struct ksmbd_readdir_data *buf; + int cmp = -EINVAL; + + buf = container_of(ctx, struct ksmbd_readdir_data, ctx); + + if (buf->used != namlen) + return true; + if (IS_ENABLED(CONFIG_UNICODE) && buf->um) { + const struct qstr q_buf = {.name = buf->private, + .len = buf->used}; + const struct qstr q_name = {.name = name, + .len = namlen}; + + cmp = utf8_strncasecmp(buf->um, &q_buf, &q_name); + } + if (cmp < 0) + cmp = strncasecmp((char *)buf->private, name, namlen); + if (!cmp) { + memcpy((char *)buf->private, name, buf->used); + buf->dirent_count = 1; + return false; + } + return true; +} + +/** + * ksmbd_vfs_lookup_in_dir() - lookup a file in a directory + * @dir: path info + * @name: filename to lookup + * @namelen: filename length + * @um: &struct unicode_map to use + * + * Return: 0 on success, otherwise error + */ +static int ksmbd_vfs_lookup_in_dir(const struct path *dir, char *name, + size_t namelen, struct unicode_map *um) +{ + int ret; + struct file *dfilp; + int flags = O_RDONLY | O_LARGEFILE; + struct ksmbd_readdir_data readdir_data = { + .ctx.actor = __caseless_lookup, + .private = name, + .used = namelen, + .dirent_count = 0, + .um = um, + }; + + dfilp = dentry_open(dir, flags, current_cred()); + if (IS_ERR(dfilp)) + return PTR_ERR(dfilp); + + ret = iterate_dir(dfilp, &readdir_data.ctx); + if (readdir_data.dirent_count > 0) + ret = 0; + fput(dfilp); + return ret; +} + +static +int __ksmbd_vfs_kern_path(struct ksmbd_work *work, char *filepath, + unsigned int flags, + struct path *path, bool caseless, bool for_remove) +{ + struct ksmbd_share_config *share_conf = work->tcon->share_conf; + struct path parent_path; + size_t path_len, remain_len; + int err; + +retry: + err = ksmbd_vfs_path_lookup(share_conf, filepath, flags, path, for_remove); + if (!err || err != -ENOENT || !caseless) + return err; + + path_len = strlen(filepath); + remain_len = path_len; + + parent_path = share_conf->vfs_path; + path_get(&parent_path); + + while (d_can_lookup(parent_path.dentry)) { + char *filename = filepath + path_len - remain_len; + char *next = strchrnul(filename, '/'); + size_t filename_len = next - filename; + bool is_last = !next[0]; + + if (filename_len == 0) + break; + + err = ksmbd_vfs_lookup_in_dir(&parent_path, filename, + filename_len, + work->conn->um); + path_put(&parent_path); + if (err) + goto out; + if (is_last) { + caseless = false; + goto retry; + } + next[0] = '\0'; + + err = vfs_path_lookup(share_conf->vfs_path.dentry, + share_conf->vfs_path.mnt, + filepath, + flags, + &parent_path); + next[0] = '/'; + if (err) + goto out; + + remain_len -= filename_len + 1; + } + + err = -EINVAL; + path_put(&parent_path); +out: + return err; +} + +/** + * ksmbd_vfs_kern_path() - lookup a file and get path info + * @work: work + * @filepath: file path that is relative to share + * @flags: lookup flags + * @path: if lookup succeed, return path info + * @caseless: caseless filename lookup + * + * Perform the lookup, possibly crossing over any mount point. + * On return no locks will be held and write-access to filesystem + * won't have been checked. + * Return: 0 if file was found, otherwise error + */ +int ksmbd_vfs_kern_path(struct ksmbd_work *work, char *filepath, + unsigned int flags, + struct path *path, bool caseless) +{ + return __ksmbd_vfs_kern_path(work, filepath, flags, path, + caseless, false); +} + +/** + * ksmbd_vfs_kern_path_start_removing() - lookup a file and get path info prior to removal + * @work: work + * @filepath: file path that is relative to share + * @flags: lookup flags + * @path: if lookup succeed, return path info + * @caseless: caseless filename lookup + * + * Perform the lookup, but don't cross over any mount point. + * On return the parent of path->dentry will be locked and write-access to + * filesystem will have been gained. + * Return: 0 on if file was found, otherwise error + */ +int ksmbd_vfs_kern_path_start_removing(struct ksmbd_work *work, char *filepath, + unsigned int flags, + struct path *path, bool caseless) +{ + return __ksmbd_vfs_kern_path(work, filepath, flags, path, + caseless, true); +} + +void ksmbd_vfs_kern_path_end_removing(const struct path *path) +{ + end_removing(path->dentry); + mnt_drop_write(path->mnt); + mntput(path->mnt); +} + +struct dentry *ksmbd_vfs_kern_path_create(struct ksmbd_work *work, + const char *name, + unsigned int flags, + struct path *path) +{ + struct ksmbd_share_config *share_conf = work->tcon->share_conf; + struct qstr last; + struct dentry *dent; + int err; + + /* resolve the name beneath the share root so ".." cannot escape */ + CLASS(filename_kernel, filename)(name); + + err = vfs_path_parent_lookup(filename, flags | LOOKUP_BENEATH, + path, &last, &share_conf->vfs_path); + if (err) + return ERR_PTR(err); + + err = mnt_want_write(path->mnt); + if (err) { + path_put(path); + return ERR_PTR(err); + } + + dent = start_creating_noperm(path->dentry, &last); + if (IS_ERR(dent)) { + mnt_drop_write(path->mnt); + path_put(path); + } + return dent; +} + +int ksmbd_vfs_remove_acl_xattrs(struct mnt_idmap *idmap, + const struct path *path) +{ + char *name, *xattr_list = NULL; + ssize_t xattr_list_len; + int err = 0; + + xattr_list_len = ksmbd_vfs_listxattr(path->dentry, &xattr_list); + if (xattr_list_len < 0) { + goto out; + } else if (!xattr_list_len) { + ksmbd_debug(SMB, "empty xattr in the file\n"); + goto out; + } + + err = mnt_want_write(path->mnt); + if (err) + goto out; + + for (name = xattr_list; name - xattr_list < xattr_list_len; + name += strlen(name) + 1) { + ksmbd_debug(SMB, "%s, len %zd\n", name, strlen(name)); + + if (!strncmp(name, XATTR_NAME_POSIX_ACL_ACCESS, + sizeof(XATTR_NAME_POSIX_ACL_ACCESS) - 1) || + !strncmp(name, XATTR_NAME_POSIX_ACL_DEFAULT, + sizeof(XATTR_NAME_POSIX_ACL_DEFAULT) - 1)) { + err = vfs_remove_acl(idmap, path->dentry, name); + if (err) + ksmbd_debug(SMB, + "remove acl xattr failed : %s\n", name); + } + } + mnt_drop_write(path->mnt); + +out: + kvfree(xattr_list); + return err; +} + +int ksmbd_vfs_remove_sd_xattrs(struct mnt_idmap *idmap, const struct path *path) +{ + char *name, *xattr_list = NULL; + ssize_t xattr_list_len; + int err = 0; + + xattr_list_len = ksmbd_vfs_listxattr(path->dentry, &xattr_list); + if (xattr_list_len < 0) { + goto out; + } else if (!xattr_list_len) { + ksmbd_debug(SMB, "empty xattr in the file\n"); + goto out; + } + + for (name = xattr_list; name - xattr_list < xattr_list_len; + name += strlen(name) + 1) { + ksmbd_debug(SMB, "%s, len %zd\n", name, strlen(name)); + + if (!strncmp(name, XATTR_NAME_SD, XATTR_NAME_SD_LEN)) { + err = ksmbd_vfs_remove_xattr(idmap, path, name, true); + if (err) + ksmbd_debug(SMB, "remove xattr failed : %s\n", name); + } + } +out: + kvfree(xattr_list); + return err; +} + +static struct xattr_smb_acl *ksmbd_vfs_make_xattr_posix_acl(struct mnt_idmap *idmap, + struct inode *inode, + int acl_type) +{ + struct xattr_smb_acl *smb_acl = NULL; + struct posix_acl *posix_acls; + struct posix_acl_entry *pa_entry; + struct xattr_acl_entry *xa_entry; + int i; + + if (!IS_ENABLED(CONFIG_FS_POSIX_ACL)) + return NULL; + + posix_acls = get_inode_acl(inode, acl_type); + if (IS_ERR_OR_NULL(posix_acls)) + return NULL; + + smb_acl = kzalloc(sizeof(struct xattr_smb_acl) + + sizeof(struct xattr_acl_entry) * posix_acls->a_count, + KSMBD_DEFAULT_GFP); + if (!smb_acl) + goto out; + + smb_acl->count = posix_acls->a_count; + pa_entry = posix_acls->a_entries; + xa_entry = smb_acl->entries; + for (i = 0; i < posix_acls->a_count; i++, pa_entry++, xa_entry++) { + switch (pa_entry->e_tag) { + case ACL_USER: + xa_entry->type = SMB_ACL_USER; + xa_entry->uid = posix_acl_uid_translate(idmap, pa_entry); + break; + case ACL_USER_OBJ: + xa_entry->type = SMB_ACL_USER_OBJ; + break; + case ACL_GROUP: + xa_entry->type = SMB_ACL_GROUP; + xa_entry->gid = posix_acl_gid_translate(idmap, pa_entry); + break; + case ACL_GROUP_OBJ: + xa_entry->type = SMB_ACL_GROUP_OBJ; + break; + case ACL_OTHER: + xa_entry->type = SMB_ACL_OTHER; + break; + case ACL_MASK: + xa_entry->type = SMB_ACL_MASK; + break; + default: + pr_err("unknown type : 0x%x\n", pa_entry->e_tag); + goto out; + } + + if (pa_entry->e_perm & ACL_READ) + xa_entry->perm |= SMB_ACL_READ; + if (pa_entry->e_perm & ACL_WRITE) + xa_entry->perm |= SMB_ACL_WRITE; + if (pa_entry->e_perm & ACL_EXECUTE) + xa_entry->perm |= SMB_ACL_EXECUTE; + } +out: + posix_acl_release(posix_acls); + return smb_acl; +} + +int ksmbd_vfs_set_sd_xattr(struct ksmbd_conn *conn, + struct mnt_idmap *idmap, + const struct path *path, + struct smb_ntsd *pntsd, int len, + bool get_write) +{ + int rc; + struct ndr sd_ndr = {0}, acl_ndr = {0}; + struct xattr_ntacl acl = {0}; + struct xattr_smb_acl *smb_acl, *def_smb_acl = NULL; + struct dentry *dentry = path->dentry; + struct inode *inode = d_inode(dentry); + + acl.version = 4; + acl.hash_type = XATTR_SD_HASH_TYPE_SHA256; + acl.current_time = ksmbd_UnixTimeToNT(current_time(inode)); + + memcpy(acl.desc, "posix_acl", 9); + acl.desc_len = 10; + + pntsd->osidoffset = + cpu_to_le32(le32_to_cpu(pntsd->osidoffset) + NDR_NTSD_OFFSETOF); + pntsd->gsidoffset = + cpu_to_le32(le32_to_cpu(pntsd->gsidoffset) + NDR_NTSD_OFFSETOF); + pntsd->dacloffset = + cpu_to_le32(le32_to_cpu(pntsd->dacloffset) + NDR_NTSD_OFFSETOF); + + acl.sd_buf = (char *)pntsd; + acl.sd_size = len; + + sha256(acl.sd_buf, acl.sd_size, acl.hash); + + smb_acl = ksmbd_vfs_make_xattr_posix_acl(idmap, inode, + ACL_TYPE_ACCESS); + if (S_ISDIR(inode->i_mode)) + def_smb_acl = ksmbd_vfs_make_xattr_posix_acl(idmap, inode, + ACL_TYPE_DEFAULT); + + rc = ndr_encode_posix_acl(&acl_ndr, idmap, inode, + smb_acl, def_smb_acl); + if (rc) { + pr_err("failed to encode ndr to posix acl\n"); + goto out; + } + + sha256(acl_ndr.data, acl_ndr.offset, acl.posix_acl_hash); + + rc = ndr_encode_v4_ntacl(&sd_ndr, &acl); + if (rc) { + pr_err("failed to encode ndr to posix acl\n"); + goto out; + } + + rc = ksmbd_vfs_setxattr(idmap, path, + XATTR_NAME_SD, sd_ndr.data, + sd_ndr.offset, 0, get_write); + if (rc < 0) + pr_err("Failed to store XATTR ntacl :%d\n", rc); + +out: + kfree(sd_ndr.data); + kfree(acl_ndr.data); + kfree(smb_acl); + kfree(def_smb_acl); + return rc; +} + +int ksmbd_vfs_get_sd_xattr(struct ksmbd_conn *conn, + struct mnt_idmap *idmap, + struct dentry *dentry, + struct smb_ntsd **pntsd) +{ + int rc; + struct ndr n; + struct inode *inode = d_inode(dentry); + struct ndr acl_ndr = {0}; + struct xattr_ntacl acl = {0}; + struct xattr_smb_acl *smb_acl = NULL, *def_smb_acl = NULL; + __u8 cmp_hash[XATTR_SD_HASH_SIZE] = {0}; + + rc = ksmbd_vfs_getxattr(idmap, dentry, XATTR_NAME_SD, &n.data); + if (rc <= 0) + return rc; + + n.length = rc; + rc = ndr_decode_v4_ntacl(&n, &acl); + if (rc) + goto out_free; + + smb_acl = ksmbd_vfs_make_xattr_posix_acl(idmap, inode, + ACL_TYPE_ACCESS); + if (S_ISDIR(inode->i_mode)) + def_smb_acl = ksmbd_vfs_make_xattr_posix_acl(idmap, inode, + ACL_TYPE_DEFAULT); + + rc = ndr_encode_posix_acl(&acl_ndr, idmap, inode, smb_acl, + def_smb_acl); + if (rc) { + pr_err("failed to encode ndr to posix acl\n"); + goto out_free; + } + + sha256(acl_ndr.data, acl_ndr.offset, cmp_hash); + + if (memcmp(cmp_hash, acl.posix_acl_hash, XATTR_SD_HASH_SIZE)) { + pr_err("hash value diff\n"); + rc = -EINVAL; + goto out_free; + } + + *pntsd = acl.sd_buf; + if (acl.sd_size < sizeof(struct smb_ntsd)) { + pr_err("sd size is invalid\n"); + rc = -EINVAL; + goto out_free; + } + + (*pntsd)->osidoffset = cpu_to_le32(le32_to_cpu((*pntsd)->osidoffset) - + NDR_NTSD_OFFSETOF); + (*pntsd)->gsidoffset = cpu_to_le32(le32_to_cpu((*pntsd)->gsidoffset) - + NDR_NTSD_OFFSETOF); + (*pntsd)->dacloffset = cpu_to_le32(le32_to_cpu((*pntsd)->dacloffset) - + NDR_NTSD_OFFSETOF); + + rc = acl.sd_size; +out_free: + kfree(acl_ndr.data); + kfree(smb_acl); + kfree(def_smb_acl); + if (rc < 0) { + kfree(acl.sd_buf); + *pntsd = NULL; + } + kfree(n.data); + return rc; +} + +int ksmbd_vfs_set_dos_attrib_xattr(struct mnt_idmap *idmap, + const struct path *path, + struct xattr_dos_attrib *da, + bool get_write) +{ + struct ndr n; + int err; + + err = ndr_encode_dos_attr(&n, da); + if (err) + goto out; + + err = ksmbd_vfs_setxattr(idmap, path, XATTR_NAME_DOS_ATTRIBUTE, + (void *)n.data, n.offset, 0, get_write); + if (err) + ksmbd_debug(SMB, "failed to store dos attribute in xattr\n"); + +out: + kfree(n.data); + return err; +} + +int ksmbd_vfs_get_dos_attrib_xattr(struct mnt_idmap *idmap, + struct dentry *dentry, + struct xattr_dos_attrib *da) +{ + struct ndr n; + int err; + + err = ksmbd_vfs_getxattr(idmap, dentry, XATTR_NAME_DOS_ATTRIBUTE, + (char **)&n.data); + if (err > 0) { + n.length = err; + if (ndr_decode_dos_attr(&n, da)) + err = -EINVAL; + kfree(n.data); + } else { + ksmbd_debug(SMB, "failed to load dos attribute in xattr\n"); + } + + return err; +} + +/** + * ksmbd_vfs_init_kstat() - convert unix stat information to smb stat format + * @p: destination buffer + * @ksmbd_kstat: ksmbd kstat wrapper + * + * Returns: pointer to the converted &struct file_directory_info + */ +void *ksmbd_vfs_init_kstat(char **p, struct ksmbd_kstat *ksmbd_kstat) +{ + FILE_DIRECTORY_INFO *info = (FILE_DIRECTORY_INFO *)(*p); + struct kstat *kstat = ksmbd_kstat->kstat; + u64 time; + + info->FileIndex = 0; + info->CreationTime = cpu_to_le64(ksmbd_kstat->create_time); + time = ksmbd_UnixTimeToNT(kstat->atime); + info->LastAccessTime = cpu_to_le64(time); + time = ksmbd_UnixTimeToNT(kstat->mtime); + info->LastWriteTime = cpu_to_le64(time); + time = ksmbd_UnixTimeToNT(kstat->ctime); + info->ChangeTime = cpu_to_le64(time); + + if (ksmbd_kstat->file_attributes & FILE_ATTRIBUTE_DIRECTORY_LE) { + info->EndOfFile = 0; + info->AllocationSize = 0; + } else { + info->EndOfFile = cpu_to_le64(kstat->size); + info->AllocationSize = cpu_to_le64(kstat->blocks << 9); + } + info->ExtFileAttributes = ksmbd_kstat->file_attributes; + + return info; +} + +int ksmbd_vfs_fill_dentry_attrs(struct ksmbd_work *work, + struct mnt_idmap *idmap, + struct dentry *dentry, + struct ksmbd_kstat *ksmbd_kstat) +{ + struct ksmbd_share_config *share_conf = work->tcon->share_conf; + u64 time; + int rc; + struct path path = { + .mnt = share_conf->vfs_path.mnt, + .dentry = dentry, + }; + + rc = vfs_getattr(&path, ksmbd_kstat->kstat, + STATX_BASIC_STATS | STATX_BTIME, + AT_STATX_SYNC_AS_STAT); + if (rc) + return rc; + + time = ksmbd_UnixTimeToNT(ksmbd_kstat->kstat->ctime); + ksmbd_kstat->create_time = time; + + /* + * set default value for the case that store dos attributes is not yes + * or that acl is disable in server's filesystem and the config is yes. + */ + if (S_ISDIR(ksmbd_kstat->kstat->mode)) + ksmbd_kstat->file_attributes = FILE_ATTRIBUTE_DIRECTORY_LE; + else + ksmbd_kstat->file_attributes = FILE_ATTRIBUTE_ARCHIVE_LE; + + if (test_share_config_flag(work->tcon->share_conf, + KSMBD_SHARE_FLAG_STORE_DOS_ATTRS)) { + struct xattr_dos_attrib da; + + rc = ksmbd_vfs_get_dos_attrib_xattr(idmap, dentry, &da); + if (rc > 0) { + ksmbd_kstat->file_attributes = cpu_to_le32(da.attr); + ksmbd_kstat->create_time = da.create_time; + } else { + ksmbd_debug(VFS, "fail to load dos attribute.\n"); + } + } + + /* + * Only pay for this when it'll actually be used: AAPL + * READDIR_ATTR_V2's flags field (AAPL_READDIR_ATTR_V2_NO_XATTR) is + * the only consumer. XATTR_NAME_STREAM ("user.DosStream.") is a + * reliable, distinct prefix for genuine ADS/stream xattrs -- unlike + * DOSATTRIB or ACL xattrs, which live under different prefixes, so + * this can't false-positive into telling Finder a file has no extra + * data when it actually does. + */ + ksmbd_kstat->has_ads_stream = false; + if (work->conn->aapl_readdir_attr_v2) { + char *xattr_list = NULL, *name; + ssize_t xattr_list_len; + + xattr_list_len = ksmbd_vfs_listxattr(dentry, &xattr_list); + if (xattr_list_len > 0) { + for (name = xattr_list; + name - xattr_list < xattr_list_len; + name += strlen(name) + 1) { + if (!strncmp(name, XATTR_NAME_STREAM, + XATTR_NAME_STREAM_LEN)) { + ksmbd_kstat->has_ads_stream = true; + break; + } + } + } + kvfree(xattr_list); + } + + return 0; +} + +ssize_t ksmbd_vfs_casexattr_len(struct mnt_idmap *idmap, + struct dentry *dentry, char *attr_name, + int attr_name_len) +{ + char *name, *xattr_list = NULL; + ssize_t value_len = -ENOENT, xattr_list_len; + + xattr_list_len = ksmbd_vfs_listxattr(dentry, &xattr_list); + if (xattr_list_len <= 0) + goto out; + + for (name = xattr_list; name - xattr_list < xattr_list_len; + name += strlen(name) + 1) { + ksmbd_debug(VFS, "%s, len %zd\n", name, strlen(name)); + if (strncasecmp(attr_name, name, attr_name_len)) + continue; + + value_len = ksmbd_vfs_xattr_len(idmap, dentry, name); + break; + } + +out: + kvfree(xattr_list); + return value_len; +} + +int ksmbd_vfs_xattr_stream_name(char *stream_name, char **xattr_stream_name, + size_t *xattr_stream_name_size, int s_type) +{ + char *type, *buf; + + if (s_type == DIR_STREAM) + type = ":$INDEX_ALLOCATION"; + else + type = ":$DATA"; + + buf = kasprintf(KSMBD_DEFAULT_GFP, "%s%s%s", + XATTR_NAME_STREAM, stream_name, type); + if (!buf) + return -ENOMEM; + + *xattr_stream_name = buf; + *xattr_stream_name_size = strlen(buf) + 1; + + return 0; +} + +static ssize_t ksmbd_vfs_copy_file_range_buffered(struct ksmbd_work *work, + struct ksmbd_file *src_fp, + struct ksmbd_file *dst_fp, + loff_t src_off, + loff_t dst_off, size_t len) +{ + size_t buf_size = min_t(size_t, len, SZ_1M); + size_t copied = 0; + char *buf; + ssize_t ret = 0; + + buf = kvmalloc(buf_size, KSMBD_DEFAULT_GFP); + if (!buf) + return -ENOMEM; + + while (copied < len) { + size_t chunk_size = min(buf_size, len - copied); + size_t done = 0; + loff_t src_pos, dst_pos; + + if (dst_off > src_off) { + src_pos = src_off + len - copied - chunk_size; + dst_pos = dst_off + len - copied - chunk_size; + } else { + src_pos = src_off + copied; + dst_pos = dst_off + copied; + } + + while (done < chunk_size) { + loff_t pos = src_pos + done; + + ret = ksmbd_vfs_read(work, src_fp, chunk_size - done, + &pos, buf + done); + if (ret <= 0) { + if (!ret) + ret = -EIO; + goto out; + } + done += ret; + } + + done = 0; + while (done < chunk_size) { + loff_t pos = dst_pos + done; + ssize_t written = 0; + + ret = ksmbd_vfs_write(work, dst_fp, buf + done, + chunk_size - done, &pos, false, + &written); + if (ret < 0) + goto out; + if (!written) { + ret = -EIO; + goto out; + } + done += written; + } + copied += chunk_size; + } + ret = copied; +out: + kvfree(buf); + return ret; +} + +int ksmbd_vfs_copy_file_ranges(struct ksmbd_work *work, + struct ksmbd_file *src_fp, + struct ksmbd_file *dst_fp, + struct srv_copychunk *chunks, + unsigned int chunk_count, + unsigned int *chunk_count_written, + unsigned int *chunk_size_written, + loff_t *total_size_written) +{ + unsigned int i; + loff_t src_off, dst_off, src_file_size; + size_t len; + int ret; + + *chunk_count_written = 0; + *chunk_size_written = 0; + *total_size_written = 0; + + if (!(src_fp->daccess & (FILE_READ_DATA_LE | FILE_EXECUTE_LE))) { + pr_err("no right to read(%pD)\n", src_fp->filp); + return -EACCES; + } + if (!(dst_fp->daccess & (FILE_WRITE_DATA_LE | FILE_APPEND_DATA_LE))) { + pr_err("no right to write(%pD)\n", dst_fp->filp); + return -EACCES; + } + + smb_break_all_levII_oplock(work, dst_fp, 1); + + if (!work->tcon->posix_extensions) { + for (i = 0; i < chunk_count; i++) { + src_off = le64_to_cpu(chunks[i].SourceOffset); + dst_off = le64_to_cpu(chunks[i].TargetOffset); + len = le32_to_cpu(chunks[i].Length); + + if (check_lock_range(src_fp->filp, src_off, + src_off + len - 1, READ)) + return -EAGAIN; + if (check_lock_range(dst_fp->filp, dst_off, + dst_off + len - 1, WRITE)) + return -EAGAIN; + } + } + + if (ksmbd_stream_fd(src_fp)) { + const struct cred *saved_cred; + + saved_cred = override_creds(src_fp->filp->f_cred); + src_file_size = ksmbd_vfs_casexattr_len( + file_mnt_idmap(src_fp->filp), + src_fp->filp->f_path.dentry, + src_fp->stream.name, src_fp->stream.size); + revert_creds(saved_cred); + if (src_file_size < 0) + return src_file_size; + } else { + src_file_size = i_size_read(file_inode(src_fp->filp)); + } + + /* + * macOS Finder's Cmd+D duplicate sends FSCTL_SRV_COPYCHUNK with + * ChunkCount=0 meaning "copy the whole file/stream", not the + * standard SMB2 "query my copy limits, no data" semantics -- + * fsctl_copychunk() only reaches here with chunk_count == 0 for + * AAPL-negotiated connections, so this doesn't affect compliant + * non-AAPL clients. Without this, the destination stays at its + * just-created 0 bytes / empty stream: the for loop below is a + * no-op when chunk_count is 0, since it never has an iteration to + * treat as "copy everything". + */ + if (chunk_count == 0 && work->conn->is_aapl) { + loff_t off = 0; + + while (off < src_file_size) { + size_t remaining = src_file_size - off; + ssize_t copied; + + /* Same source/destination offset here: an in-place, + * same-inode copy at matching offsets is a degenerate + * no-op range, not a real overlap, but vfs_copy_file_range + * still doesn't support streams -- route those (and the + * same-inode case defensively) through the buffered path. + */ + if (ksmbd_stream_fd(src_fp) || ksmbd_stream_fd(dst_fp) || + file_inode(src_fp->filp) == file_inode(dst_fp->filp)) { + copied = ksmbd_vfs_copy_file_range_buffered(work, src_fp, dst_fp, + off, off, remaining); + } else { + copied = vfs_copy_file_range(src_fp->filp, off, + dst_fp->filp, off, + remaining, 0); + if (copied == -EOPNOTSUPP || copied == -EXDEV) + copied = vfs_copy_file_range(src_fp->filp, off, + dst_fp->filp, off, + remaining, + COPY_FILE_SPLICE); + } + if (copied < 0) + return copied; + if (copied == 0) + break; + off += copied; + } + + /* + * This is a synthesized whole-file copy, not a response to + * any chunk descriptor the client actually sent (it sent + * none -- chunk_count is 0). Report zero chunks/chunk-bytes + * rather than inventing a chunk that doesn't correspond to + * anything in the request; only total_size_written (bytes + * actually copied) is meaningful here. + */ + *chunk_count_written = 0; + *chunk_size_written = 0; + *total_size_written = off; + return 0; + } + + for (i = 0; i < chunk_count; i++) { + bool stream_len_mismatch = false; + size_t copy_len; + + src_off = le64_to_cpu(chunks[i].SourceOffset); + dst_off = le64_to_cpu(chunks[i].TargetOffset); + len = le32_to_cpu(chunks[i].Length); + copy_len = len; + + if (src_off < 0) + return -E2BIG; + + if (src_off > src_file_size || len > src_file_size - src_off) { + /* + * macOS can reuse the main file's chunk list when copying + * streams, so the requested range can exceed the size of + * the xattr-backed stream. For an AAPL connection, copy the + * available stream data and report the requested length to + * avoid a copy length mismatch. + */ + if (!work->conn->is_aapl || + !ksmbd_stream_fd(src_fp) || + !ksmbd_stream_fd(dst_fp)) + return -E2BIG; + + stream_len_mismatch = true; + if (src_off < src_file_size) + copy_len = src_file_size - src_off; + else + copy_len = 0; + } + + /* + * vfs_copy_file_range does not support streams or overlapping + * ranges within the same file. + */ + if (!copy_len) { + ret = 0; + } else if (ksmbd_stream_fd(src_fp) || ksmbd_stream_fd(dst_fp) || + (file_inode(src_fp->filp) == file_inode(dst_fp->filp) && + dst_off + copy_len > src_off && + dst_off < src_off + copy_len)) { + ret = ksmbd_vfs_copy_file_range_buffered(work, src_fp, + dst_fp, src_off, + dst_off, copy_len); + } else { + ret = vfs_copy_file_range(src_fp->filp, src_off, + dst_fp->filp, dst_off, copy_len, 0); + if (ret == -EOPNOTSUPP || ret == -EXDEV) + ret = vfs_copy_file_range(src_fp->filp, src_off, + dst_fp->filp, dst_off, + copy_len, + COPY_FILE_SPLICE); + } + if (ret < 0) + return ret; + if (stream_len_mismatch) + ret = len; + + *chunk_count_written += 1; + *total_size_written += ret; + } + return 0; +} + +void ksmbd_vfs_posix_lock_wait(struct file_lock *flock) +{ + wait_event(flock->c.flc_wait, !flock->c.flc_blocker); +} + +void ksmbd_vfs_posix_lock_unblock(struct file_lock *flock) +{ + locks_delete_block(flock); +} + +int ksmbd_vfs_set_init_posix_acl(struct mnt_idmap *idmap, + const struct path *path) +{ + struct posix_acl_state acl_state; + struct posix_acl *acls; + struct dentry *dentry = path->dentry; + struct inode *inode = d_inode(dentry); + int rc; + + if (!IS_ENABLED(CONFIG_FS_POSIX_ACL)) + return -EOPNOTSUPP; + + ksmbd_debug(SMB, "Set posix acls\n"); + rc = init_acl_state(&acl_state, 1); + if (rc) + return rc; + + /* Set default owner group */ + acl_state.owner.allow = (inode->i_mode & 0700) >> 6; + acl_state.group.allow = (inode->i_mode & 0070) >> 3; + acl_state.other.allow = inode->i_mode & 0007; + acl_state.users->aces[acl_state.users->n].uid = inode->i_uid; + acl_state.users->aces[acl_state.users->n++].perms.allow = + acl_state.owner.allow; + acl_state.groups->aces[acl_state.groups->n].gid = inode->i_gid; + acl_state.groups->aces[acl_state.groups->n++].perms.allow = + acl_state.group.allow; + acl_state.mask.allow = 0x07; + + acls = posix_acl_alloc(6, KSMBD_DEFAULT_GFP); + if (!acls) { + free_acl_state(&acl_state); + return -ENOMEM; + } + posix_state_to_acl(&acl_state, acls->a_entries); + + rc = set_posix_acl(idmap, dentry, ACL_TYPE_ACCESS, acls); + if (rc < 0) + ksmbd_debug(SMB, "Set posix acl(ACL_TYPE_ACCESS) failed, rc : %d\n", + rc); + else if (S_ISDIR(inode->i_mode)) { + posix_state_to_acl(&acl_state, acls->a_entries); + rc = set_posix_acl(idmap, dentry, ACL_TYPE_DEFAULT, acls); + if (rc < 0) + ksmbd_debug(SMB, "Set posix acl(ACL_TYPE_DEFAULT) failed, rc : %d\n", + rc); + } + + free_acl_state(&acl_state); + posix_acl_release(acls); + return rc; +} + +int ksmbd_vfs_inherit_posix_acl(struct mnt_idmap *idmap, + const struct path *path, struct inode *parent_inode) +{ + struct posix_acl *acls; + + if (!IS_ENABLED(CONFIG_FS_POSIX_ACL)) + return -EOPNOTSUPP; + + acls = get_inode_acl(parent_inode, ACL_TYPE_DEFAULT); + if (IS_ERR_OR_NULL(acls)) + return -ENOENT; + + posix_acl_release(acls); + return 0; +} + +void ksmbd_vfs_update_compressed_fattr(struct dentry *dentry, __le32 *fattr) +{ + int rc; + struct file_kattr fa = { .flags_valid = true }; + + rc = vfs_fileattr_get(dentry, &fa); + if (rc) + return; + + if (fa.flags & FS_COMPR_FL) + *fattr |= FILE_ATTRIBUTE_COMPRESSED_LE; +} + +int ksmbd_vfs_get_compression(struct ksmbd_file *fp, u16 *fmt) +{ + struct file_kattr fa = { .flags_valid = true }; + int rc; + + rc = vfs_fileattr_get(fp->filp->f_path.dentry, &fa); + if (rc == -ENOIOCTLCMD || rc == -ENOTTY || rc == -EINVAL || + rc == -EOPNOTSUPP) { + if (fp->f_ci->m_fattr & FILE_ATTRIBUTE_COMPRESSED_LE) + *fmt = COMPRESSION_FORMAT_LZNT1; + else + *fmt = COMPRESSION_FORMAT_NONE; + rc = 0; + goto out; + } + if (rc) + goto out; + + if (fp->f_ci->m_fattr & FILE_ATTRIBUTE_COMPRESSED_LE) + *fmt = COMPRESSION_FORMAT_LZNT1; + else + *fmt = COMPRESSION_FORMAT_NONE; + +out: + return rc; +} + +static int __ksmbd_vfs_set_compression(struct ksmbd_work *work, + struct ksmbd_file *fp, u16 fmt, + bool check_access) +{ + const struct cred *saved_cred = NULL; + struct file_kattr fa; + struct dentry *dentry = fp->filp->f_path.dentry; + struct mnt_idmap *idmap = file_mnt_idmap(fp->filp); + u32 flags; + __le32 old_fattr; + int rc; + + if (check_access && !(fp->daccess & FILE_WRITE_DATA_LE)) { + rc = -EACCES; + goto out; + } + + if (fmt != COMPRESSION_FORMAT_NONE && + fmt != COMPRESSION_FORMAT_DEFAULT && + fmt != COMPRESSION_FORMAT_LZNT1) { + rc = -EINVAL; + goto out; + } + + saved_cred = override_creds(fp->filp->f_cred); + rc = vfs_fileattr_get(dentry, &fa); + if (rc == -ENOIOCTLCMD || rc == -ENOTTY || rc == -EINVAL || + rc == -EOPNOTSUPP) + goto update_fattr; + if (rc) + goto out; + + flags = fa.flags; + if (fmt == COMPRESSION_FORMAT_NONE) { + flags &= ~FS_COMPR_FL; + } else if (fmt == COMPRESSION_FORMAT_DEFAULT || + fmt == COMPRESSION_FORMAT_LZNT1) { + flags |= FS_COMPR_FL; + } + + if (flags != fa.flags) { + fileattr_fill_flags(&fa, flags); + rc = mnt_want_write_file(fp->filp); + if (rc) + goto out; + + rc = vfs_fileattr_set(idmap, dentry, &fa); + mnt_drop_write_file(fp->filp); + if (rc == -ENOIOCTLCMD || rc == -ENOTTY || rc == -EINVAL || + rc == -EOPNOTSUPP) + goto update_fattr; + if (rc) + goto out; + } + +update_fattr: + old_fattr = fp->f_ci->m_fattr; + if (fmt == COMPRESSION_FORMAT_NONE) + fp->f_ci->m_fattr &= ~FILE_ATTRIBUTE_COMPRESSED_LE; + else + fp->f_ci->m_fattr |= FILE_ATTRIBUTE_COMPRESSED_LE; + + if (fp->f_ci->m_fattr != old_fattr) { + struct xattr_dos_attrib da = {0}; + + rc = ksmbd_vfs_get_dos_attrib_xattr(idmap, dentry, &da); + if (rc <= 0) { + da.version = 4; + da.itime = fp->itime; + da.create_time = fp->create_time; + da.flags = XATTR_DOSINFO_CREATE_TIME | + XATTR_DOSINFO_ITIME; + } + + da.attr = le32_to_cpu(fp->f_ci->m_fattr); + da.flags |= XATTR_DOSINFO_ATTRIB; + rc = ksmbd_vfs_set_dos_attrib_xattr(idmap, + &fp->filp->f_path, + &da, true); + if (rc) + rc = 0; + } + +out: + if (saved_cred) + revert_creds(saved_cred); + return rc; +} + +int ksmbd_vfs_set_compression(struct ksmbd_work *work, + struct ksmbd_file *fp, u16 fmt) +{ + return __ksmbd_vfs_set_compression(work, fp, fmt, true); +} + +int ksmbd_vfs_set_compression_create(struct ksmbd_work *work, + struct ksmbd_file *fp, u16 fmt) +{ + return __ksmbd_vfs_set_compression(work, fp, fmt, false); +} diff --git a/fs/smb/server/vfs.h b/fs/smb/server/vfs.h new file mode 100644 index 000000000..55d099de7 --- /dev/null +++ b/fs/smb/server/vfs.h @@ -0,0 +1,184 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2016 Namjae Jeon + * Copyright (C) 2018 Samsung Electronics Co., Ltd. + */ + +#ifndef __KSMBD_VFS_H__ +#define __KSMBD_VFS_H__ + +#include +#include +#include +#include +#include +#include + +#include "smbacl.h" +#include "xattr.h" + +/* + * Enumeration for stream type. + */ +enum { + DATA_STREAM = 1, /* type $DATA */ + DIR_STREAM /* type $INDEX_ALLOCATION */ +}; + +/* CreateOptions */ +#define CREATE_TREE_CONNECTION cpu_to_le32(0x00000080) +#define FILE_RESERVE_OPFILTER_LE cpu_to_le32(0x00100000) + +#define CREATE_OPTION_READONLY 0x10000000 +/* system. NB not sent over wire */ +#define CREATE_OPTION_SPECIAL 0x20000000 + +struct ksmbd_work; +struct ksmbd_file; +struct ksmbd_conn; + +struct ksmbd_dir_info { + const char *name; + char *wptr; + char *rptr; + int name_len; + int out_buf_len; + int num_scan; + int num_entry; + int data_count; + int last_entry_offset; + bool hide_dot_file; + int flags; + int last_entry_off_align; +}; + +struct ksmbd_readdir_data { + struct dir_context ctx; + union { + void *private; + char *dirent; + }; + + unsigned int used; + unsigned int dirent_count; + unsigned int file_attr; + struct unicode_map *um; +}; + +/* ksmbd kstat wrapper to get valid create time when reading dir entry */ +struct ksmbd_kstat { + struct kstat *kstat; + unsigned long long create_time; + __le32 file_attributes; + bool has_ads_stream; /* AAPL READDIR_ATTR V2 xattr-presence flag */ +}; + +int ksmbd_vfs_lock_parent(struct dentry *parent, struct dentry *child); +void ksmbd_vfs_query_maximal_access(struct mnt_idmap *idmap, + struct dentry *dentry, __le32 *daccess); +int ksmbd_vfs_create(struct ksmbd_work *work, const char *name, umode_t mode); +int ksmbd_vfs_mkdir(struct ksmbd_work *work, const char *name, umode_t mode); +int ksmbd_vfs_read(struct ksmbd_work *work, struct ksmbd_file *fp, size_t count, + loff_t *pos, char *rbuf); +int ksmbd_vfs_write(struct ksmbd_work *work, struct ksmbd_file *fp, + char *buf, size_t count, loff_t *pos, bool sync, + ssize_t *written); +int ksmbd_vfs_fsync(struct ksmbd_work *work, u64 fid, u64 p_id); +int ksmbd_vfs_remove_file(struct ksmbd_work *work, const struct path *path); +int ksmbd_vfs_link(struct ksmbd_work *work, + const char *oldname, const char *newname); +int ksmbd_vfs_getattr(const struct path *path, struct kstat *stat); +int ksmbd_vfs_rename(struct ksmbd_work *work, struct ksmbd_file *old_fp, + char *newname, int flags); +int ksmbd_vfs_check_rename_share(struct ksmbd_work *work, + const struct path *old_path); +int ksmbd_vfs_truncate(struct ksmbd_work *work, + struct ksmbd_file *fp, loff_t size); +struct srv_copychunk; +int ksmbd_vfs_copy_file_ranges(struct ksmbd_work *work, + struct ksmbd_file *src_fp, + struct ksmbd_file *dst_fp, + struct srv_copychunk *chunks, + unsigned int chunk_count, + unsigned int *chunk_count_written, + unsigned int *chunk_size_written, + loff_t *total_size_written); +ssize_t ksmbd_vfs_listxattr(struct dentry *dentry, char **list); +ssize_t ksmbd_vfs_getxattr(struct mnt_idmap *idmap, + struct dentry *dentry, + char *xattr_name, + char **xattr_buf); +ssize_t ksmbd_vfs_casexattr_len(struct mnt_idmap *idmap, + struct dentry *dentry, char *attr_name, + int attr_name_len); +int ksmbd_vfs_setxattr(struct mnt_idmap *idmap, + const struct path *path, const char *attr_name, + void *attr_value, size_t attr_size, int flags, + bool get_write); +int ksmbd_vfs_xattr_stream_name(char *stream_name, char **xattr_stream_name, + size_t *xattr_stream_name_size, int s_type); +int ksmbd_vfs_remove_xattr(struct mnt_idmap *idmap, + const struct path *path, char *attr_name, + bool get_write); +int ksmbd_vfs_kern_path(struct ksmbd_work *work, char *name, + unsigned int flags, + struct path *path, bool caseless); +int ksmbd_vfs_kern_path_start_removing(struct ksmbd_work *work, char *name, + unsigned int flags, + struct path *path, bool caseless); +void ksmbd_vfs_kern_path_end_removing(const struct path *path); +struct dentry *ksmbd_vfs_kern_path_create(struct ksmbd_work *work, + const char *name, + unsigned int flags, + struct path *path); +int ksmbd_vfs_empty_dir(struct ksmbd_file *fp); +void ksmbd_vfs_set_fadvise(struct file *filp, __le32 option); +int ksmbd_vfs_zero_data(struct ksmbd_work *work, struct ksmbd_file *fp, + loff_t off, loff_t len); +int ksmbd_vfs_zero_holes(struct ksmbd_file *fp); +int ksmbd_vfs_trim_data(struct ksmbd_work *work, struct ksmbd_file *fp, + loff_t off, loff_t len); +struct file_allocated_range_buffer; +int ksmbd_vfs_query_allocated_ranges(struct ksmbd_file *fp, loff_t start, + loff_t length, + struct file_allocated_range_buffer *ranges, + unsigned int in_count, + unsigned int *out_count); +int ksmbd_vfs_unlink(struct file *filp); +void *ksmbd_vfs_init_kstat(char **p, struct ksmbd_kstat *ksmbd_kstat); +int ksmbd_vfs_fill_dentry_attrs(struct ksmbd_work *work, + struct mnt_idmap *idmap, + struct dentry *dentry, + struct ksmbd_kstat *ksmbd_kstat); +void ksmbd_vfs_posix_lock_wait(struct file_lock *flock); +void ksmbd_vfs_posix_lock_unblock(struct file_lock *flock); +int ksmbd_vfs_remove_acl_xattrs(struct mnt_idmap *idmap, + const struct path *path); +int ksmbd_vfs_remove_sd_xattrs(struct mnt_idmap *idmap, const struct path *path); +int ksmbd_vfs_set_sd_xattr(struct ksmbd_conn *conn, + struct mnt_idmap *idmap, + const struct path *path, + struct smb_ntsd *pntsd, int len, + bool get_write); +int ksmbd_vfs_get_sd_xattr(struct ksmbd_conn *conn, + struct mnt_idmap *idmap, + struct dentry *dentry, + struct smb_ntsd **pntsd); +int ksmbd_vfs_set_dos_attrib_xattr(struct mnt_idmap *idmap, + const struct path *path, + struct xattr_dos_attrib *da, + bool get_write); +int ksmbd_vfs_get_dos_attrib_xattr(struct mnt_idmap *idmap, + struct dentry *dentry, + struct xattr_dos_attrib *da); +int ksmbd_vfs_set_init_posix_acl(struct mnt_idmap *idmap, + const struct path *path); +int ksmbd_vfs_inherit_posix_acl(struct mnt_idmap *idmap, + const struct path *path, + struct inode *parent_inode); +void ksmbd_vfs_update_compressed_fattr(struct dentry *dentry, __le32 *fattr); +int ksmbd_vfs_get_compression(struct ksmbd_file *fp, u16 *fmt); +int ksmbd_vfs_set_compression(struct ksmbd_work *work, struct ksmbd_file *fp, u16 fmt); +int ksmbd_vfs_set_compression_create(struct ksmbd_work *work, + struct ksmbd_file *fp, u16 fmt); +#endif /* __KSMBD_VFS_H__ */ diff --git a/fs/smb/server/vfs_cache.c b/fs/smb/server/vfs_cache.c new file mode 100644 index 000000000..81626d204 --- /dev/null +++ b/fs/smb/server/vfs_cache.c @@ -0,0 +1,1995 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2016 Namjae Jeon + * Copyright (C) 2019 Samsung Electronics Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "glob.h" +#include "vfs_cache.h" +#include "oplock.h" +#include "vfs.h" +#include "connection.h" +#include "misc.h" +#include "mgmt/tree_connect.h" +#include "mgmt/user_session.h" +#include "mgmt/user_config.h" +#include "smb_common.h" +#include "server.h" +#include "smb2pdu.h" + +#define S_DEL_PENDING 1 +#define S_DEL_ON_CLS 2 +#define S_DEL_ON_CLS_STREAM 8 + +static unsigned int inode_hash_mask __read_mostly; +static unsigned int inode_hash_shift __read_mostly; +static struct hlist_head *inode_hashtable __read_mostly; +static DEFINE_RWLOCK(inode_hash_lock); + +static struct ksmbd_file_table global_ft; +static atomic_long_t fd_limit; +static struct kmem_cache *filp_cache; + +static int ksmbd_mark_fp_closed(struct ksmbd_file *fp); + +#define OPLOCK_NONE 0 +#define OPLOCK_EXCLUSIVE 1 +#define OPLOCK_BATCH 2 +#define OPLOCK_READ 3 /* level 2 oplock */ + +#ifdef CONFIG_PROC_FS + +static const struct ksmbd_const_name ksmbd_lease_const_names[] = { + {le32_to_cpu(SMB2_LEASE_NONE_LE), "LEASE_NONE"}, + {le32_to_cpu(SMB2_LEASE_READ_CACHING_LE), "LEASE_R"}, + {le32_to_cpu(SMB2_LEASE_HANDLE_CACHING_LE), "LEASE_H"}, + {le32_to_cpu(SMB2_LEASE_WRITE_CACHING_LE), "LEASE_W"}, + {le32_to_cpu(SMB2_LEASE_READ_CACHING_LE | + SMB2_LEASE_HANDLE_CACHING_LE), "LEASE_RH"}, + {le32_to_cpu(SMB2_LEASE_READ_CACHING_LE | + SMB2_LEASE_WRITE_CACHING_LE), "LEASE_RW"}, + {le32_to_cpu(SMB2_LEASE_HANDLE_CACHING_LE | + SMB2_LEASE_WRITE_CACHING_LE), "LEASE_WH"}, + {le32_to_cpu(SMB2_LEASE_READ_CACHING_LE | + SMB2_LEASE_HANDLE_CACHING_LE | + SMB2_LEASE_WRITE_CACHING_LE), "LEASE_RWH"}, +}; + +static const struct ksmbd_const_name ksmbd_oplock_const_names[] = { + {SMB2_OPLOCK_LEVEL_NONE, "OPLOCK_NONE"}, + {SMB2_OPLOCK_LEVEL_II, "OPLOCK_II"}, + {SMB2_OPLOCK_LEVEL_EXCLUSIVE, "OPLOCK_EXCLUSIVE"}, + {SMB2_OPLOCK_LEVEL_BATCH, "OPLOCK_BATCH"}, +}; + +static const struct ksmbd_const_name ksmbd_file_state_names[] = { + {FP_NEW, "new"}, + {FP_INITED, "open"}, + {FP_CLOSED, "closed"}, +}; + +#define KSMBD_PROC_FILE_DURABLE BIT(0) +#define KSMBD_PROC_FILE_PERSISTENT BIT(1) +#define KSMBD_PROC_FILE_RESILIENT BIT(2) +#define KSMBD_PROC_FILE_DELETE_ON_CLOSE BIT(3) +#define KSMBD_PROC_FILE_STREAM BIT(4) +#define KSMBD_PROC_FILE_POSIX BIT(5) +#define KSMBD_PROC_FILE_ATTRIB_ONLY BIT(6) + +static const struct ksmbd_const_name ksmbd_file_flag_names[] = { + {KSMBD_PROC_FILE_DURABLE, "durable"}, + {KSMBD_PROC_FILE_PERSISTENT, "persistent"}, + {KSMBD_PROC_FILE_RESILIENT, "resilient"}, + {KSMBD_PROC_FILE_DELETE_ON_CLOSE, "delete-on-close"}, + {KSMBD_PROC_FILE_STREAM, "stream"}, + {KSMBD_PROC_FILE_POSIX, "posix"}, + {KSMBD_PROC_FILE_ATTRIB_ONLY, "attrib-only"}, +}; + +static unsigned int ksmbd_proc_file_flags(struct ksmbd_file *fp) +{ + unsigned int flags = 0; + + if (fp->is_durable) + flags |= KSMBD_PROC_FILE_DURABLE; + if (fp->is_persistent) + flags |= KSMBD_PROC_FILE_PERSISTENT; + if (fp->is_resilient) + flags |= KSMBD_PROC_FILE_RESILIENT; + if (fp->coption & FILE_DELETE_ON_CLOSE_LE) + flags |= KSMBD_PROC_FILE_DELETE_ON_CLOSE; + if (fp->stream.name) + flags |= KSMBD_PROC_FILE_STREAM; + if (fp->is_posix_ctxt) + flags |= KSMBD_PROC_FILE_POSIX; + if (fp->attrib_only) + flags |= KSMBD_PROC_FILE_ATTRIB_ONLY; + return flags; +} + +static int proc_show_files(struct seq_file *m, void *v) +{ + struct ksmbd_file *fp = NULL; + unsigned int id; + struct oplock_info *opinfo; + + read_lock(&global_ft.lock); + idr_for_each_entry(global_ft.idr, fp, id) { + seq_printf(m, "tree_id:\t0x%x\n", fp->tcon ? fp->tcon->id : 0); + seq_printf(m, "persistent_id:\t0x%llx\n", fp->persistent_id); + seq_printf(m, "volatile_id:\t0x%llx\n", fp->volatile_id); + seq_printf(m, "refcount:\t%d\n", atomic_read(&fp->refcount)); + + rcu_read_lock(); + opinfo = rcu_dereference(fp->f_opinfo); + if (opinfo) { + const struct ksmbd_const_name *const_names; + const char *name; + int count; + unsigned int level; + + if (opinfo->is_lease) { + const_names = ksmbd_lease_const_names; + count = ARRAY_SIZE(ksmbd_lease_const_names); + level = le32_to_cpu(opinfo->o_lease->state); + } else { + const_names = ksmbd_oplock_const_names; + count = ARRAY_SIZE(ksmbd_oplock_const_names); + level = opinfo->level; + } + rcu_read_unlock(); + name = ksmbd_proc_const_name(const_names, count, level); + if (name) + seq_printf(m, "oplock:\t%s\n", name); + else + seq_printf(m, "oplock:\t0x%x\n", level); + } else { + rcu_read_unlock(); + seq_puts(m, "oplock:\tnone\n"); + } + + seq_printf(m, "state:\t%s\n", + ksmbd_proc_const_name(ksmbd_file_state_names, + ARRAY_SIZE(ksmbd_file_state_names), + fp->f_state)); + seq_printf(m, "durable_timeout:\t%u\n", fp->durable_timeout); + seq_printf(m, "create_options:\t0x%08x\n", + le32_to_cpu(fp->coption)); + seq_printf(m, "desired_access:\t0x%08x\n", + le32_to_cpu(fp->daccess)); + seq_printf(m, "share_access:\t0x%08x\n", + le32_to_cpu(fp->saccess)); + seq_puts(m, "flags:\t"); + ksmbd_proc_show_flag_names(m, ksmbd_file_flag_names, + ARRAY_SIZE(ksmbd_file_flag_names), + ksmbd_proc_file_flags(fp)); + seq_printf(m, "\nname:\t%s\n\n", + fp->filp->f_path.dentry->d_name.name); + } + read_unlock(&global_ft.lock); + return 0; +} + +static int create_proc_files(void) +{ + if (!ksmbd_proc_create("files", proc_show_files, NULL)) + return -ENOMEM; + return 0; +} +#else +static int create_proc_files(void) { return 0; } +#endif + +static bool durable_scavenger_running; +static DEFINE_MUTEX(durable_scavenger_lock); +static wait_queue_head_t dh_wq; + +bool ksmbd_durable_scavenger_active(void) +{ + bool active; + + mutex_lock(&durable_scavenger_lock); + active = durable_scavenger_running; + mutex_unlock(&durable_scavenger_lock); + return active; +} + +void ksmbd_set_fd_limit(unsigned long limit) +{ + limit = min(limit, get_max_files()); + atomic_long_set(&fd_limit, limit); +} + +static bool fd_limit_depleted(void) +{ + long v = atomic_long_dec_return(&fd_limit); + + if (v >= 0) + return false; + atomic_long_inc(&fd_limit); + return true; +} + +static void fd_limit_close(void) +{ + atomic_long_inc(&fd_limit); +} + +/* + * INODE hash + */ + +static unsigned long inode_hash(struct super_block *sb, unsigned long hashval) +{ + unsigned long tmp; + + tmp = (hashval * (unsigned long)sb) ^ (GOLDEN_RATIO_PRIME + hashval) / + L1_CACHE_BYTES; + tmp = tmp ^ ((tmp ^ GOLDEN_RATIO_PRIME) >> inode_hash_shift); + return tmp & inode_hash_mask; +} + +static struct ksmbd_inode *__ksmbd_inode_lookup(struct dentry *de) +{ + struct hlist_head *head = inode_hashtable + + inode_hash(d_inode(de)->i_sb, (unsigned long)de); + struct ksmbd_inode *ci = NULL, *ret_ci = NULL; + + hlist_for_each_entry(ci, head, m_hash) { + if (ci->m_de == de) { + if (atomic_inc_not_zero(&ci->m_count)) + ret_ci = ci; + break; + } + } + return ret_ci; +} + +static struct ksmbd_inode *ksmbd_inode_lookup(struct ksmbd_file *fp) +{ + return __ksmbd_inode_lookup(fp->filp->f_path.dentry); +} + +struct ksmbd_inode *ksmbd_inode_lookup_lock(struct dentry *d) +{ + struct ksmbd_inode *ci; + + read_lock(&inode_hash_lock); + ci = __ksmbd_inode_lookup(d); + read_unlock(&inode_hash_lock); + + return ci; +} + +int ksmbd_query_inode_status(struct dentry *dentry) +{ + struct ksmbd_inode *ci; + int ret = KSMBD_INODE_STATUS_UNKNOWN; + + read_lock(&inode_hash_lock); + ci = __ksmbd_inode_lookup(dentry); + read_unlock(&inode_hash_lock); + if (!ci) + return ret; + + down_read(&ci->m_lock); + if (ci->m_flags & S_DEL_PENDING) + ret = KSMBD_INODE_STATUS_PENDING_DELETE; + else + ret = KSMBD_INODE_STATUS_OK; + up_read(&ci->m_lock); + + ksmbd_inode_put(ci); + return ret; +} + +bool ksmbd_inode_pending_delete(struct ksmbd_file *fp) +{ + struct ksmbd_inode *ci = fp->f_ci; + int ret; + + down_read(&ci->m_lock); + ret = (ci->m_flags & S_DEL_PENDING); + up_read(&ci->m_lock); + if (ret || !ksmbd_stream_fd(fp)) + return ret; + + spin_lock(&fp->f_lock); + ret = fp->stream_del_pending; + spin_unlock(&fp->f_lock); + + return ret; +} + +void ksmbd_set_inode_pending_delete(struct ksmbd_file *fp) +{ + struct ksmbd_inode *ci = fp->f_ci; + + down_write(&ci->m_lock); + ci->m_flags |= S_DEL_PENDING; + up_write(&ci->m_lock); +} + +void ksmbd_clear_inode_pending_delete(struct ksmbd_file *fp) +{ + struct ksmbd_inode *ci = fp->f_ci; + + down_write(&ci->m_lock); + ci->m_flags &= ~S_DEL_PENDING; + up_write(&ci->m_lock); +} + +bool ksmbd_has_stream_without_delete_share(struct ksmbd_file *fp) +{ + struct ksmbd_file *prev_fp; + struct ksmbd_inode *ci = fp->f_ci; + bool ret = false; + + if (ksmbd_stream_fd(fp)) + return false; + + down_read(&ci->m_lock); + list_for_each_entry(prev_fp, &ci->m_fp_list, node) { + if (prev_fp == fp || !ksmbd_stream_fd(prev_fp)) + continue; + + if (file_inode(fp->filp) != file_inode(prev_fp->filp)) + continue; + + if (!(prev_fp->saccess & FILE_SHARE_DELETE_LE)) { + ret = true; + break; + } + } + up_read(&ci->m_lock); + + return ret; +} + +void ksmbd_fd_set_delete_on_close(struct ksmbd_file *fp, + int file_info) +{ + struct ksmbd_inode *ci = fp->f_ci; + + down_write(&ci->m_lock); + if (ksmbd_stream_fd(fp)) + ci->m_flags |= S_DEL_ON_CLS_STREAM; + else + ci->m_flags |= S_DEL_ON_CLS; + up_write(&ci->m_lock); +} + +/* + * FileDispositionInformation (SET_INFO) on a stream handle must only + * mark the stream for deletion, not the whole file -- otherwise + * deleting a single alternate data stream (e.g. AFP_AfpInfo) deletes + * the entire file's data along with it. + * + * This is tracked on fp itself (stream_del_pending), not the shared + * ksmbd_inode: the inode-wide S_DEL_ON_CLS_STREAM flag used by + * ksmbd_fd_set_delete_on_close() can't record *which* stream should be + * deleted, so if a different stream handle on the same file closed + * first, it would delete the wrong stream. + */ +void ksmbd_fd_set_delete_pending(struct ksmbd_file *fp) +{ + if (ksmbd_stream_fd(fp)) { + spin_lock(&fp->f_lock); + fp->stream_del_pending = true; + spin_unlock(&fp->f_lock); + } else { + ksmbd_set_inode_pending_delete(fp); + } +} + +void ksmbd_fd_clear_delete_pending(struct ksmbd_file *fp) +{ + if (ksmbd_stream_fd(fp)) { + spin_lock(&fp->f_lock); + fp->stream_del_pending = false; + spin_unlock(&fp->f_lock); + } else { + ksmbd_clear_inode_pending_delete(fp); + } +} + +static void ksmbd_inode_hash(struct ksmbd_inode *ci) +{ + struct hlist_head *b = inode_hashtable + + inode_hash(d_inode(ci->m_de)->i_sb, (unsigned long)ci->m_de); + + hlist_add_head(&ci->m_hash, b); +} + +static void ksmbd_inode_unhash(struct ksmbd_inode *ci) +{ + write_lock(&inode_hash_lock); + hlist_del_init(&ci->m_hash); + write_unlock(&inode_hash_lock); +} + +static int ksmbd_inode_init(struct ksmbd_inode *ci, struct ksmbd_file *fp) +{ + atomic_set(&ci->m_count, 1); + atomic_set(&ci->op_count, 0); + atomic_set(&ci->sop_count, 0); + ci->m_flags = 0; + ci->m_fattr = 0; + INIT_LIST_HEAD(&ci->m_fp_list); + INIT_LIST_HEAD(&ci->m_op_list); + init_rwsem(&ci->m_lock); + ci->m_de = fp->filp->f_path.dentry; + return 0; +} + +static struct ksmbd_inode *ksmbd_inode_get(struct ksmbd_file *fp) +{ + struct ksmbd_inode *ci, *tmpci; + int rc; + + read_lock(&inode_hash_lock); + ci = ksmbd_inode_lookup(fp); + read_unlock(&inode_hash_lock); + if (ci) + return ci; + + ci = kmalloc_obj(struct ksmbd_inode, KSMBD_DEFAULT_GFP); + if (!ci) + return NULL; + + rc = ksmbd_inode_init(ci, fp); + if (rc) { + pr_err("inode initialized failed\n"); + kfree(ci); + return NULL; + } + + write_lock(&inode_hash_lock); + tmpci = ksmbd_inode_lookup(fp); + if (!tmpci) { + ksmbd_inode_hash(ci); + } else { + kfree(ci); + ci = tmpci; + } + write_unlock(&inode_hash_lock); + return ci; +} + +static void ksmbd_inode_free(struct ksmbd_inode *ci) +{ + ksmbd_inode_unhash(ci); + kfree(ci); +} + +void ksmbd_inode_put(struct ksmbd_inode *ci) +{ + if (atomic_dec_and_test(&ci->m_count)) + ksmbd_inode_free(ci); +} + +int __init ksmbd_inode_hash_init(void) +{ + unsigned int loop; + unsigned long numentries = 16384; + unsigned long bucketsize = sizeof(struct hlist_head); + unsigned long size; + + inode_hash_shift = ilog2(numentries); + inode_hash_mask = (1 << inode_hash_shift) - 1; + + size = bucketsize << inode_hash_shift; + + /* init master fp hash table */ + inode_hashtable = vmalloc(size); + if (!inode_hashtable) + return -ENOMEM; + + for (loop = 0; loop < (1U << inode_hash_shift); loop++) + INIT_HLIST_HEAD(&inode_hashtable[loop]); + return 0; +} + +void ksmbd_release_inode_hash(void) +{ + vfree(inode_hashtable); +} + +static void __ksmbd_inode_close(struct ksmbd_file *fp) +{ + struct ksmbd_inode *ci = fp->f_ci; + int err; + struct file *filp; + + filp = fp->filp; + + if (ksmbd_stream_fd(fp)) { + bool remove_stream_xattr = false; + + down_write(&ci->m_lock); + if (ci->m_flags & S_DEL_ON_CLS_STREAM) { + ci->m_flags &= ~S_DEL_ON_CLS_STREAM; + remove_stream_xattr = true; + } + up_write(&ci->m_lock); + + /* + * Per-handle delete-pending from ksmbd_fd_set_delete_pending() + * (FileDispositionInformation on this stream) -- separate from + * the inode-wide flag above, which only ever meant "some + * stream on this file" with no way to say which one. + */ + spin_lock(&fp->f_lock); + if (fp->stream_del_pending) { + fp->stream_del_pending = false; + remove_stream_xattr = true; + } + spin_unlock(&fp->f_lock); + + if (remove_stream_xattr) { + const struct cred *saved_cred; + + saved_cred = override_creds(filp->f_cred); + err = ksmbd_vfs_remove_xattr(file_mnt_idmap(filp), + &filp->f_path, + fp->stream.name, + true); + revert_creds(saved_cred); + if (err) + pr_err("remove xattr failed : %s\n", + fp->stream.name); + } + } + + down_write(&ci->m_lock); + /* Promote S_DEL_ON_CLS to S_DEL_PENDING when close */ + if (ci->m_flags & S_DEL_ON_CLS) { + ci->m_flags &= ~S_DEL_ON_CLS; + ci->m_flags |= S_DEL_PENDING; + } + up_write(&ci->m_lock); + + if (atomic_dec_and_test(&ci->m_count)) { + bool do_unlink = false; + + down_write(&ci->m_lock); + if (ci->m_flags & S_DEL_PENDING) { + ci->m_flags &= ~S_DEL_PENDING; + do_unlink = true; + } + up_write(&ci->m_lock); + + if (do_unlink) + ksmbd_vfs_unlink(filp); + + ksmbd_inode_free(ci); + } +} + +static void __ksmbd_remove_durable_fd(struct ksmbd_file *fp) +{ + if (!has_file_id(fp->persistent_id)) + return; + + idr_remove(global_ft.idr, fp->persistent_id); + /* + * Clear persistent_id so a later __ksmbd_close_fd() that runs from a + * delayed putter (e.g. when a concurrent ksmbd_lookup_fd_inode() + * walker held the final reference) does not re-issue idr_remove() on + * an id that idr_alloc_cyclic() may have already handed out to a new + * durable handle. + */ + fp->persistent_id = KSMBD_NO_FID; +} + +static void ksmbd_remove_durable_fd(struct ksmbd_file *fp) +{ + write_lock(&global_ft.lock); + __ksmbd_remove_durable_fd(fp); + write_unlock(&global_ft.lock); + if (waitqueue_active(&dh_wq)) + wake_up(&dh_wq); +} + +static void __ksmbd_remove_fd(struct ksmbd_file_table *ft, struct ksmbd_file *fp) +{ + down_write(&fp->f_ci->m_lock); + list_del_init(&fp->node); + up_write(&fp->f_ci->m_lock); + + if (!has_file_id(fp->volatile_id)) + return; + + write_lock(&ft->lock); + idr_remove(ft->idr, fp->volatile_id); + write_unlock(&ft->lock); +} + +static void __ksmbd_close_fd(struct ksmbd_file_table *ft, struct ksmbd_file *fp) +{ + struct file *filp; + struct ksmbd_lock *smb_lock, *tmp_lock; + struct ksmbd_work *cn_work; + + fd_limit_close(); + ksmbd_remove_durable_fd(fp); + if (ft) + __ksmbd_remove_fd(ft, fp); + + close_id_del_oplock(fp); + filp = fp->filp; + + __ksmbd_inode_close(fp); + if (!IS_ERR_OR_NULL(filp)) + fput(filp); + + /* + * The zero fp reference count serializes access to fp->lock_list, but + * the VFS may still have blocked requests chained below these locks. + */ + list_for_each_entry_safe(smb_lock, tmp_lock, &fp->lock_list, flist) { + struct ksmbd_conn *conn = smb_lock->conn; + + if (conn) { + spin_lock(&conn->llist_lock); + list_del_init(&smb_lock->clist); + smb_lock->conn = NULL; + spin_unlock(&conn->llist_lock); + ksmbd_conn_put(conn); + } + + list_del_init(&smb_lock->flist); + ksmbd_vfs_posix_lock_unblock(smb_lock->fl); + locks_free_lock(smb_lock->fl); + kfree(smb_lock); + } + + /* + * Complete any CHANGE_NOTIFY left pending on this handle now that + * it is closed. KSMBD never completes CHANGE_NOTIFY spontaneously + * (no real change-notification backend), only on close -- matching + * genuine SMB2/macOS smbfs semantics and avoiding the Finder + * "directory changed, re-enumerate everything" loop. + * + * smb2_notify() on another connection can be adding to + * notify_pendings under fp->f_lock at the same time this handle is + * closed, and a client-sent CANCEL can concurrently be racing to + * claim the same entry via smb2_notify_cancel_fn() (smb2pdu.c). + * Pop one entry at a time under the lock via list_del_init() rather + * than a bulk list_splice_init(): list_del_init() leaves the node + * self-linked ("empty"), which is what the cancel path checks under + * the same lock to tell whether it lost the race -- a bulk splice + * would instead relink every entry into a shared local list, so an + * entry claimed here would still read as "not empty" to a racing + * cancel_fn, and both sides could end up freeing the same work. + * ksmbd_conn_write() can sleep (it takes conn's write mutex), so it + * must not be called while fp->f_lock is held -- release the lock + * before processing each popped entry, then reacquire it for the + * next. + */ + for (;;) { + spin_lock(&fp->f_lock); + if (list_empty(&fp->notify_pendings)) { + spin_unlock(&fp->f_lock); + break; + } + cn_work = list_first_entry(&fp->notify_pendings, + struct ksmbd_work, notify_entry); + list_del_init(&cn_work->notify_entry); + spin_unlock(&fp->f_lock); + + ksmbd_conn_write(cn_work); + /* + * release_async_work() removes cn_work from + * conn->async_requests, frees cancel_argv, and releases+zeroes + * async_id -- all needed before ksmbd_free_work_struct(), which + * only releases async_id itself if still nonzero (i.e. if this + * hadn't already been done). + */ + release_async_work(cn_work); + ksmbd_free_work_struct(cn_work); + } + + /* + * Drop fp's strong reference on conn (taken in ksmbd_open_fd() / + * ksmbd_reopen_durable_fd()). Durable fps that reached the + * scavenger have already had fp->conn cleared by session_fd_check(), + * in which case there is nothing to drop here. + */ + if (fp->conn) { + ksmbd_conn_put(fp->conn); + fp->conn = NULL; + } + + if (ksmbd_stream_fd(fp)) + kfree(fp->stream.name); + kfree(fp->owner.name); + + kmem_cache_free(filp_cache, fp); +} + +/** + * ksmbd_close_disconnected_durable_delete_on_close() - drop a delete-on-close + * file kept present only by disconnected durable handles + * @dentry: dentry of the file being opened + * + * A durable handle opened with delete-on-close is preserved across a + * disconnect so it can be reclaimed by a durable reconnect. When a new + * (non-reconnect) open arrives for the same name instead, the disconnected + * handle has to give way. Close such handles so their delete-on-close is + * applied and the file is removed once the last handle is gone, letting the + * new open create a fresh file. + * + * The caller's inode reference is dropped before closing so that the final + * close can promote S_DEL_ON_CLS to S_DEL_PENDING and unlink the file. + * + * Return: true if a disconnected durable handle was closed. + */ +bool ksmbd_close_disconnected_durable_delete_on_close(struct dentry *dentry) +{ + struct ksmbd_inode *ci; + struct ksmbd_file *fp, *tmp; + LIST_HEAD(dispose); + bool closed = false; + + ci = ksmbd_inode_lookup_lock(dentry); + if (!ci) + return false; + + down_write(&ci->m_lock); + if (ci->m_flags & (S_DEL_ON_CLS | S_DEL_ON_CLS_STREAM | S_DEL_PENDING)) { + list_for_each_entry_safe(fp, tmp, &ci->m_fp_list, node) { + if (fp->conn || !fp->is_durable || + fp->f_state != FP_INITED) + continue; + + /* + * Claim the close before unlinking fp from m_fp_list. + * refcount == 1 means only the durable lifetime ref is + * left. Add a transient ref so final close can drop both. + */ + write_lock(&global_ft.lock); + if (atomic_read(&fp->refcount) == 1) { + atomic_inc(&fp->refcount); + __ksmbd_remove_durable_fd(fp); + ksmbd_mark_fp_closed(fp); + list_move_tail(&fp->node, &dispose); + } + write_unlock(&global_ft.lock); + } + } + up_write(&ci->m_lock); + + /* + * Drop our lookup reference before closing so the last __ksmbd_close_fd() + * can drop m_count to zero and unlink the delete-on-close file. The + * collected handles still hold the transient reference taken above, so + * ci stays valid until they are closed below. + */ + ksmbd_inode_put(ci); + + while (!list_empty(&dispose)) { + fp = list_first_entry(&dispose, struct ksmbd_file, node); + list_del_init(&fp->node); + if (atomic_sub_and_test(2, &fp->refcount)) { + __ksmbd_close_fd(NULL, fp); + closed = true; + } + } + + return closed; +} + +static struct ksmbd_file *ksmbd_fp_get(struct ksmbd_file *fp) +{ + if (fp->f_state != FP_INITED) + return NULL; + + if (!atomic_inc_not_zero(&fp->refcount)) + return NULL; + return fp; +} + +struct ksmbd_file *ksmbd_file_get(struct ksmbd_file *fp) +{ + return ksmbd_fp_get(fp); +} + +static struct ksmbd_file *__ksmbd_lookup_fd(struct ksmbd_file_table *ft, + u64 id) +{ + struct ksmbd_file *fp; + + if (!has_file_id(id)) + return NULL; + + read_lock(&ft->lock); + fp = idr_find(ft->idr, id); + if (fp) + fp = ksmbd_fp_get(fp); + read_unlock(&ft->lock); + return fp; +} + +static void __put_fd_final(struct ksmbd_work *work, struct ksmbd_file *fp) +{ + /* + * Detached durable fp -- session_fd_check() cleared fp->conn at + * preserve, so this fp is no longer tracked by any conn's + * stats.open_files_count. This happens when + * ksmbd_scavenger_dispose_dh() hands the final close off to an + * m_fp_list walker (e.g. ksmbd_lookup_fd_inode()) whose work->conn + * is unrelated to the conn that originally opened the handle; close + * via the NULL-ft path so we do not underflow that unrelated + * counter. + */ + if (!fp->conn) { + __ksmbd_close_fd(NULL, fp); + return; + } + __ksmbd_close_fd(&work->sess->file_table, fp); + atomic_dec(&work->conn->stats.open_files_count); +} + +static void set_close_state_blocked_works(struct ksmbd_file *fp) +{ + struct ksmbd_work *cancel_work; + + spin_lock(&fp->f_lock); + list_for_each_entry(cancel_work, &fp->blocked_works, + fp_entry) { + cancel_work->state = KSMBD_WORK_CLOSED; + cancel_work->cancel_fn(cancel_work->cancel_argv); + } + spin_unlock(&fp->f_lock); +} + +int ksmbd_close_fd(struct ksmbd_work *work, u64 id) +{ + struct ksmbd_file *fp; + struct ksmbd_file_table *ft; + bool closed = false; + + if (!has_file_id(id)) + return 0; + + ft = &work->sess->file_table; + write_lock(&ft->lock); + fp = idr_find(ft->idr, id); + if (fp) { + set_close_state_blocked_works(fp); + + if (fp->f_state != FP_INITED) + fp = NULL; + else { + fp->f_state = FP_CLOSED; + idr_remove(ft->idr, id); + fp->volatile_id = KSMBD_NO_FID; + closed = true; + if (!atomic_dec_and_test(&fp->refcount)) + fp = NULL; + } + } + write_unlock(&ft->lock); + + if (!fp) + return closed ? 0 : -EINVAL; + + __put_fd_final(work, fp); + return 0; +} + +void ksmbd_fd_put(struct ksmbd_work *work, struct ksmbd_file *fp) +{ + if (!fp) + return; + + if (!atomic_dec_and_test(&fp->refcount)) + return; + __put_fd_final(work, fp); +} + +static bool __sanity_check(struct ksmbd_tree_connect *tcon, struct ksmbd_file *fp) +{ + if (!fp) + return false; + if (fp->tcon != tcon) + return false; + return true; +} + +struct ksmbd_file *ksmbd_lookup_foreign_fd(struct ksmbd_work *work, u64 id) +{ + return __ksmbd_lookup_fd(&work->sess->file_table, id); +} + +struct ksmbd_file *ksmbd_lookup_fd_fast(struct ksmbd_work *work, u64 id) +{ + struct ksmbd_file *fp = __ksmbd_lookup_fd(&work->sess->file_table, id); + + if (__sanity_check(work->tcon, fp)) + return fp; + + ksmbd_fd_put(work, fp); + return NULL; +} + +struct ksmbd_file *ksmbd_lookup_fd_slow(struct ksmbd_work *work, u64 id, + u64 pid) +{ + struct ksmbd_file *fp; + + if (!has_file_id(id)) { + id = work->compound_fid; + pid = work->compound_pfid; + } + + fp = __ksmbd_lookup_fd(&work->sess->file_table, id); + if (!__sanity_check(work->tcon, fp)) { + ksmbd_fd_put(work, fp); + return NULL; + } + if (fp->persistent_id != pid) { + ksmbd_fd_put(work, fp); + return NULL; + } + return fp; +} + +struct ksmbd_file *ksmbd_lookup_global_fd(unsigned long long id) +{ + return __ksmbd_lookup_fd(&global_ft, id); +} + +struct ksmbd_file *ksmbd_lookup_durable_fd(unsigned long long id) +{ + struct ksmbd_file *fp; + + fp = __ksmbd_lookup_fd(&global_ft, id); + if (fp && (fp->durable_reconnect_disabled || + fp->conn || + (fp->durable_scavenger_timeout && + (fp->durable_scavenger_timeout < + jiffies_to_msecs(jiffies))))) { + ksmbd_put_durable_fd(fp); + fp = NULL; + } + + return fp; +} + +void ksmbd_put_durable_fd(struct ksmbd_file *fp) +{ + if (!atomic_dec_and_test(&fp->refcount)) + return; + + __ksmbd_close_fd(NULL, fp); +} + +bool ksmbd_has_other_active_fd(struct ksmbd_file *fp) +{ + struct ksmbd_file *lfp; + struct ksmbd_inode *ci = fp->f_ci; + bool ret = false; + + down_read(&ci->m_lock); + list_for_each_entry(lfp, &ci->m_fp_list, node) { + if (lfp == fp) + continue; + + if (lfp->f_state == FP_INITED && + (READ_ONCE(lfp->conn) || READ_ONCE(lfp->tcon))) { + ret = true; + break; + } + } + up_read(&ci->m_lock); + + return ret; +} + +struct ksmbd_file *ksmbd_lookup_fd_app_instance_id(char *app_instance_id) +{ + struct ksmbd_file *fp = NULL; + unsigned int id; + + read_lock(&global_ft.lock); + idr_for_each_entry(global_ft.idr, fp, id) { + if (!fp->has_app_instance_id) + continue; + if (!memcmp(fp->app_instance_id, app_instance_id, + SMB2_CREATE_GUID_SIZE)) { + fp = ksmbd_fp_get(fp); + break; + } + } + read_unlock(&global_ft.lock); + + return fp; +} + +int ksmbd_close_fd_app_instance_id(char *app_instance_id) +{ + struct ksmbd_file_table *ft; + struct ksmbd_file *fp; + struct oplock_info *opinfo; + int n_to_drop = 0; + + fp = ksmbd_lookup_fd_app_instance_id(app_instance_id); + if (!fp) + return 0; + + opinfo = opinfo_get(fp); + if (!opinfo) + goto out; + + down_read(&fp->f_ci->m_lock); + if (!opinfo->conn) { + up_read(&fp->f_ci->m_lock); + goto out; + } + + ft = &opinfo->sess->file_table; + write_lock(&ft->lock); + if (fp->f_state == FP_INITED && has_file_id(fp->volatile_id)) { + idr_remove(ft->idr, fp->volatile_id); + fp->volatile_id = KSMBD_NO_FID; + n_to_drop = ksmbd_mark_fp_closed(fp); + } + write_unlock(&ft->lock); + up_read(&fp->f_ci->m_lock); + opinfo_put(opinfo); + opinfo = NULL; + + if (!n_to_drop) + goto out; + + down_write(&fp->f_ci->m_lock); + list_del_init(&fp->node); + up_write(&fp->f_ci->m_lock); + + if (atomic_sub_and_test(n_to_drop, &fp->refcount)) { + if (fp->conn) + atomic_dec(&fp->conn->stats.open_files_count); + __ksmbd_close_fd(NULL, fp); + } + return 0; + +out: + if (opinfo) + opinfo_put(opinfo); + ksmbd_put_durable_fd(fp); + return 0; +} + +int ksmbd_invalidate_durable_fd(unsigned long long id) +{ + struct ksmbd_file *fp; + + fp = ksmbd_lookup_global_fd(id); + if (!fp) + return -ENOENT; + + fp->durable_reconnect_disabled = true; + + if (fp->conn) { + ksmbd_put_durable_fd(fp); + return -ENOENT; + } + + fp->durable_timeout = 1; + fp->durable_scavenger_timeout = jiffies_to_msecs(jiffies); + ksmbd_put_durable_fd(fp); + if (waitqueue_active(&dh_wq)) + wake_up(&dh_wq); + + return -ENOENT; +} + +struct ksmbd_file *ksmbd_lookup_fd_cguid(char *cguid) +{ + struct ksmbd_file *fp = NULL; + unsigned int id; + + read_lock(&global_ft.lock); + idr_for_each_entry(global_ft.idr, fp, id) { + if (!memcmp(fp->create_guid, + cguid, + SMB2_CREATE_GUID_SIZE)) { + fp = ksmbd_fp_get(fp); + break; + } + } + read_unlock(&global_ft.lock); + + return fp; +} + +struct ksmbd_file *ksmbd_lookup_fd_inode(struct dentry *dentry) +{ + struct ksmbd_file *lfp; + struct ksmbd_inode *ci; + struct inode *inode = d_inode(dentry); + + read_lock(&inode_hash_lock); + ci = __ksmbd_inode_lookup(dentry); + read_unlock(&inode_hash_lock); + if (!ci) + return NULL; + + down_read(&ci->m_lock); + list_for_each_entry(lfp, &ci->m_fp_list, node) { + if (inode == file_inode(lfp->filp)) { + lfp = ksmbd_fp_get(lfp); + up_read(&ci->m_lock); + ksmbd_inode_put(ci); + return lfp; + } + } + up_read(&ci->m_lock); + ksmbd_inode_put(ci); + return NULL; +} + +bool ksmbd_has_other_nonposix_open(struct dentry *dentry) +{ + struct ksmbd_file *fp; + struct inode *inode = d_inode(dentry); + unsigned int id; + bool ret = false; + + if (!inode) + return false; + + read_lock(&global_ft.lock); + idr_for_each_entry(global_ft.idr, fp, id) { + if (READ_ONCE(fp->f_state) != FP_INITED) + continue; + if (inode != file_inode(fp->filp)) + continue; + if (fp->is_posix_ctxt) + continue; + + ret = true; + break; + } + read_unlock(&global_ft.lock); + + return ret; +} + +bool ksmbd_has_nonposix_open_child(struct ksmbd_file *old_fp) +{ + struct dentry *dentry = old_fp->filp->f_path.dentry; + struct ksmbd_file *fp; + unsigned int id; + bool ret = false; + + read_lock(&global_ft.lock); + idr_for_each_entry(global_ft.idr, fp, id) { + struct dentry *fp_dentry = fp->filp->f_path.dentry; + + if (fp->f_state != FP_INITED) + continue; + if (fp_dentry == dentry) + continue; + if (old_fp->is_posix_ctxt && fp->is_posix_ctxt) + continue; + if (is_subdir(fp_dentry, dentry)) { + ret = true; + break; + } + } + read_unlock(&global_ft.lock); + + return ret; +} + +#define OPEN_ID_TYPE_VOLATILE_ID (0) +#define OPEN_ID_TYPE_PERSISTENT_ID (1) + +static void __open_id_set(struct ksmbd_file *fp, u64 id, int type) +{ + if (type == OPEN_ID_TYPE_VOLATILE_ID) + fp->volatile_id = id; + if (type == OPEN_ID_TYPE_PERSISTENT_ID) + fp->persistent_id = id; +} + +static int __open_id(struct ksmbd_file_table *ft, struct ksmbd_file *fp, + int type) +{ + u64 id = 0; + int ret; + + if (type == OPEN_ID_TYPE_VOLATILE_ID && fd_limit_depleted()) { + __open_id_set(fp, KSMBD_NO_FID, type); + return -EMFILE; + } + + idr_preload(KSMBD_DEFAULT_GFP); + write_lock(&ft->lock); + ret = idr_alloc_cyclic(ft->idr, fp, KSMBD_START_FID, INT_MAX - 1, + GFP_NOWAIT); + if (ret >= 0) { + id = ret; + ret = 0; + } else { + id = KSMBD_NO_FID; + fd_limit_close(); + } + + __open_id_set(fp, id, type); + write_unlock(&ft->lock); + idr_preload_end(); + return ret; +} + +unsigned int ksmbd_open_durable_fd(struct ksmbd_file *fp) +{ + __open_id(&global_ft, fp, OPEN_ID_TYPE_PERSISTENT_ID); + return fp->persistent_id; +} + +struct ksmbd_file *ksmbd_open_fd(struct ksmbd_work *work, struct file *filp) +{ + struct ksmbd_file *fp; + int ret; + + fp = kmem_cache_zalloc(filp_cache, KSMBD_DEFAULT_GFP); + if (!fp) { + pr_err("Failed to allocate memory\n"); + return ERR_PTR(-ENOMEM); + } + + INIT_LIST_HEAD(&fp->blocked_works); + INIT_LIST_HEAD(&fp->node); + INIT_LIST_HEAD(&fp->lock_list); + INIT_LIST_HEAD(&fp->notify_pendings); + spin_lock_init(&fp->f_lock); + mutex_init(&fp->readdir_lock); + atomic_set(&fp->refcount, 1); + + fp->filp = filp; + /* + * fp owns a strong reference on fp->conn for as long as fp->conn is + * non-NULL, so session_fd_check() and __ksmbd_close_fd() never + * dereference a dangling pointer. Paired with ksmbd_conn_put() in + * session_fd_check() (durable preserve), in __ksmbd_close_fd() + * (final close), and on the error paths below. + */ + fp->conn = ksmbd_conn_get(work->conn); + fp->tcon = work->tcon; + fp->volatile_id = KSMBD_NO_FID; + fp->persistent_id = KSMBD_NO_FID; + fp->f_state = FP_NEW; + fp->f_ci = ksmbd_inode_get(fp); + + if (!fp->f_ci) { + ret = -ENOMEM; + goto err_out; + } + + ret = __open_id(&work->sess->file_table, fp, OPEN_ID_TYPE_VOLATILE_ID); + if (ret) { + ksmbd_inode_put(fp->f_ci); + goto err_out; + } + + atomic_inc(&work->conn->stats.open_files_count); + return fp; + +err_out: + /* fp->conn was set and refcounted before every branch here. */ + ksmbd_conn_put(fp->conn); + kmem_cache_free(filp_cache, fp); + return ERR_PTR(ret); +} + +/** + * ksmbd_update_fstate() - update an fp state under the file-table lock + * @ft: file table that publishes @fp's volatile id + * @fp: file pointer to update + * @state: new state + * + * Return: 0 on success. The FP_NEW -> FP_INITED transition is special: + * -ENOENT if teardown already unpublished @fp by advancing the state or + * clearing the volatile id. Other state updates preserve the historical + * fire-and-forget behavior. + */ +int ksmbd_update_fstate(struct ksmbd_file_table *ft, struct ksmbd_file *fp, + unsigned int state) +{ + int ret; + + if (!fp) + return -ENOENT; + + write_lock(&ft->lock); + if (state == FP_INITED && + (fp->f_state != FP_NEW || !has_file_id(fp->volatile_id))) { + ret = -ENOENT; + } else { + fp->f_state = state; + ret = 0; + } + write_unlock(&ft->lock); + + return ret; +} + +/* + * ksmbd_mark_fp_closed() - mark fp closed under ft->lock and return how many + * refs the teardown path owns. + * + * FP_INITED has a normal idr-owned reference, so teardown owns both that + * reference and the transient lookup reference. FP_NEW is still owned by the + * in-flight opener/reopener, which will drop the original reference after + * ksmbd_update_fstate(..., FP_INITED) observes the cleared volatile id. + * FP_CLOSED on entry means an earlier ksmbd_close_fd() already consumed the + * idr-owned ref. + */ +static int ksmbd_mark_fp_closed(struct ksmbd_file *fp) +{ + if (fp->f_state == FP_INITED) { + set_close_state_blocked_works(fp); + fp->f_state = FP_CLOSED; + return 2; + } + + return 1; +} + +static int +__close_file_table_ids(struct ksmbd_session *sess, + struct ksmbd_tree_connect *tcon, + bool (*skip)(struct ksmbd_tree_connect *tcon, + struct ksmbd_file *fp, + struct ksmbd_user *user), + bool skip_preserves_fp) +{ + struct ksmbd_file_table *ft = &sess->file_table; + struct ksmbd_file *fp; + unsigned int id = 0; + int num = 0; + + while (1) { + int n_to_drop; + + write_lock(&ft->lock); + fp = idr_get_next(ft->idr, &id); + if (!fp) { + write_unlock(&ft->lock); + break; + } + if (!atomic_inc_not_zero(&fp->refcount)) { + id++; + write_unlock(&ft->lock); + continue; + } + + if (skip_preserves_fp) { + /* + * Session teardown: skip() is session_fd_check(), + * which may sleep and mutates fp->conn / fp->tcon / + * fp->volatile_id when it chooses to preserve fp + * for durable reconnect. Unpublish fp from the + * session idr here, under ft->lock, so that + * __ksmbd_lookup_fd() through this session cannot + * grant a new ksmbd_fp_get() reference to an fp + * whose fields are about to be rewritten outside + * the lock. Durable reconnect still reaches fp via + * global_ft. + */ + idr_remove(ft->idr, id); + fp->durable_volatile_id = fp->volatile_id; + fp->volatile_id = KSMBD_NO_FID; + write_unlock(&ft->lock); + + if (skip(tcon, fp, sess->user)) { + /* + * session_fd_check() has converted fp to + * durable-preserve state and cleared its + * per-conn fields. fp is already unpublished + * above; the original idr-owned ref keeps it + * alive for the durable scavenger. Drop only + * the transient ref. atomic_dec() is safe -- + * atomic_inc_not_zero() succeeded on a + * positive value and we added one more, so + * refcount cannot be zero here. + */ + atomic_dec(&fp->refcount); + id++; + continue; + } + + /* + * Keep the close-state decision under the same lock + * observed by ksmbd_update_fstate(), which is how an + * in-flight FP_NEW opener learns that teardown has + * cleared its volatile id. + */ + write_lock(&ft->lock); + n_to_drop = ksmbd_mark_fp_closed(fp); + write_unlock(&ft->lock); + } else { + /* + * Tree teardown: skip() is tree_conn_fd_check(), a + * cheap pointer compare that doesn't sleep and has + * no side effects, so keep the skip decision plus + * the unpublish-and-mark-closed sequence atomic + * under ft->lock. fps belonging to other tree + * connects (skip() == true) stay fully published in + * the session idr with no lock window. + */ + if (skip(tcon, fp, sess->user)) { + atomic_dec(&fp->refcount); + write_unlock(&ft->lock); + id++; + continue; + } + idr_remove(ft->idr, id); + fp->volatile_id = KSMBD_NO_FID; + n_to_drop = ksmbd_mark_fp_closed(fp); + write_unlock(&ft->lock); + } + + /* + * fp->volatile_id is already cleared to prevent stale idr + * removal from a deferred final close. Remove fp from + * m_fp_list here because __ksmbd_remove_fd() will skip the + * list unlink when volatile_id is KSMBD_NO_FID. + */ + down_write(&fp->f_ci->m_lock); + list_del_init(&fp->node); + up_write(&fp->f_ci->m_lock); + + /* + * Drop the references this iteration owns: + * + * n_to_drop == 2: we observed FP_INITED and committed + * the FP_CLOSED transition ourselves, so we own the + * transient (+1) and the still-intact idr-owned ref. + * + * n_to_drop == 1: either a prior ksmbd_close_fd() + * already consumed the idr-owned ref, or fp was still + * FP_NEW and the in-flight opener/reopener must keep + * the original reference until ksmbd_update_fstate() + * observes the cleared volatile id. + * + * If we end up as the final putter, finalize fp and + * account the open_files_count decrement via the caller's + * atomic_sub(num, ...). Otherwise the remaining user's + * ksmbd_fd_put() reaches __put_fd_final(), which does its + * own atomic_dec(&open_files_count), so we must not count + * this fp here -- doing so would double-decrement the + * connection-wide counter. + */ + if (atomic_sub_and_test(n_to_drop, &fp->refcount)) { + __ksmbd_close_fd(NULL, fp); + num++; + } + id++; + } + + return num; +} + +static inline bool is_reconnectable(struct ksmbd_file *fp) +{ + struct oplock_info *opinfo = opinfo_get(fp); + bool reconn = false; + + if (!opinfo) + return false; + + if (opinfo->op_state != OPLOCK_STATE_NONE) { + opinfo_put(opinfo); + return false; + } + + if (fp->is_resilient || fp->is_persistent) + reconn = true; + else if (fp->is_durable && opinfo->is_lease && + opinfo->o_lease->state & SMB2_LEASE_HANDLE_CACHING_LE) + reconn = true; + + else if (fp->is_durable && opinfo->level == SMB2_OPLOCK_LEVEL_BATCH) + reconn = true; + + opinfo_put(opinfo); + return reconn; +} + +static bool tree_conn_fd_check(struct ksmbd_tree_connect *tcon, + struct ksmbd_file *fp, + struct ksmbd_user *user) +{ + return fp->tcon != tcon; +} + +static bool ksmbd_durable_scavenger_alive(void) +{ + if (!durable_scavenger_running) + return false; + + if (kthread_should_stop()) + return false; + + if (idr_is_empty(global_ft.idr)) + return false; + + return true; +} + +static void ksmbd_scavenger_dispose_dh(struct ksmbd_file *fp) +{ + /* + * Durable-preserved fp can remain linked on f_ci->m_fp_list for + * share-mode checks. Unlink it before final close; fp->node is not + * available as a scavenger-private list node because re-adding it to + * another list corrupts m_fp_list. + */ + down_write(&fp->f_ci->m_lock); + list_del_init(&fp->node); + up_write(&fp->f_ci->m_lock); + + /* + * Drop both the durable lifetime reference and the transient reference + * taken by the scavenger under global_ft.lock. If a concurrent + * ksmbd_lookup_fd_inode() (or any other m_fp_list walker) snatched fp + * before the unlink above, that holder owns the final close via + * ksmbd_fd_put() -> __ksmbd_close_fd(). Otherwise the scavenger is + * the last putter and finalises fp here. + */ + if (atomic_sub_and_test(2, &fp->refcount)) + __ksmbd_close_fd(NULL, fp); +} + +static int ksmbd_durable_scavenger(void *dummy) +{ + struct ksmbd_file *fp = NULL; + struct ksmbd_file *expired_fp; + unsigned int id; + unsigned int min_timeout = 1; + bool found_fp_timeout; + unsigned long remaining_jiffies; + + __module_get(THIS_MODULE); + + set_freezable(); + while (ksmbd_durable_scavenger_alive()) { + if (try_to_freeze()) + continue; + + remaining_jiffies = wait_event_interruptible_timeout(dh_wq, + ksmbd_durable_scavenger_alive() == false, + __msecs_to_jiffies(min_timeout)); + if ((long)remaining_jiffies > 0) + min_timeout = jiffies_to_msecs(remaining_jiffies); + else + min_timeout = DURABLE_HANDLE_MAX_TIMEOUT; + + do { + expired_fp = NULL; + found_fp_timeout = false; + + write_lock(&global_ft.lock); + idr_for_each_entry(global_ft.idr, fp, id) { + unsigned long durable_timeout; + + if (!fp->durable_timeout) + continue; + + if (atomic_read(&fp->refcount) > 1 || + fp->conn) + continue; + + found_fp_timeout = true; + if (fp->durable_scavenger_timeout <= + jiffies_to_msecs(jiffies)) { + __ksmbd_remove_durable_fd(fp); + /* + * Take a transient reference so fp + * cannot be freed by an in-flight + * ksmbd_lookup_fd_inode() that found + * it through f_ci->m_fp_list while we + * drop global_ft.lock and reach the + * m_fp_list unlink in + * ksmbd_scavenger_dispose_dh(). + */ + atomic_inc(&fp->refcount); + expired_fp = fp; + break; + } + + durable_timeout = + fp->durable_scavenger_timeout - + jiffies_to_msecs(jiffies); + + if (min_timeout > durable_timeout) + min_timeout = durable_timeout; + } + write_unlock(&global_ft.lock); + + if (expired_fp) + ksmbd_scavenger_dispose_dh(expired_fp); + } while (expired_fp); + + if (found_fp_timeout == false) + break; + } + + durable_scavenger_running = false; + + module_put(THIS_MODULE); + + return 0; +} + +void ksmbd_launch_ksmbd_durable_scavenger(void) +{ + if (!(server_conf.flags & KSMBD_GLOBAL_FLAG_DURABLE_HANDLE)) + return; + + mutex_lock(&durable_scavenger_lock); + if (durable_scavenger_running == true) { + mutex_unlock(&durable_scavenger_lock); + return; + } + + durable_scavenger_running = true; + + server_conf.dh_task = kthread_run(ksmbd_durable_scavenger, + (void *)NULL, "ksmbd-durable-scavenger"); + if (IS_ERR(server_conf.dh_task)) { + pr_err("cannot start conn thread, err : %ld\n", + PTR_ERR(server_conf.dh_task)); + server_conf.dh_task = NULL; + durable_scavenger_running = false; + } + mutex_unlock(&durable_scavenger_lock); +} + +void ksmbd_stop_durable_scavenger(void) +{ + if (!(server_conf.flags & KSMBD_GLOBAL_FLAG_DURABLE_HANDLE)) + return; + + mutex_lock(&durable_scavenger_lock); + if (!durable_scavenger_running) { + mutex_unlock(&durable_scavenger_lock); + return; + } + + durable_scavenger_running = false; + if (waitqueue_active(&dh_wq)) + wake_up(&dh_wq); + mutex_unlock(&durable_scavenger_lock); + kthread_stop(server_conf.dh_task); +} + +/* + * ksmbd_vfs_set_durable_owner - Store owner info for durable replay/reconnect + * @fp: ksmbd file pointer to store owner info + * @user: user pointer to copy from + * + * This function binds the current user's identity to the file handle + * to satisfy MS-SMB2 Step 8 (SecurityContext matching) during reconnect. + * + * Return: 0 on success, or negative error code on failure + */ +int ksmbd_vfs_set_durable_owner(struct ksmbd_file *fp, + struct ksmbd_user *user) +{ + char *name, *old_name; + + if (!user) + return -EINVAL; + + /* Duplicate the user name to ensure identity persistence */ + name = kstrdup(user->name, GFP_KERNEL); + if (!name) + return -ENOMEM; + + spin_lock(&fp->f_lock); + old_name = fp->owner.name; + fp->owner.uid = user->uid; + fp->owner.gid = user->gid; + fp->owner.name = name; + spin_unlock(&fp->f_lock); + kfree(old_name); + + return 0; +} + +/** + * ksmbd_vfs_compare_durable_owner - Verify if the requester is original owner + * @fp: existing ksmbd file pointer + * @user: user pointer of the reconnect requester + * + * Compares the UID, GID, and name of the current requester against the + * original owner stored in the file handle. + * + * Return: true if the user matches, false otherwise + */ +bool ksmbd_vfs_compare_durable_owner(struct ksmbd_file *fp, + struct ksmbd_user *user) +{ + bool ret = false; + + if (!user) + return false; + + spin_lock(&fp->f_lock); + if (!fp->owner.name) + goto out; + + /* Check if the UID and GID match first (fast path) */ + if (fp->owner.uid != user->uid || fp->owner.gid != user->gid) + goto out; + + /* Validate the account name to ensure the same SecurityContext */ + ret = (strcmp(fp->owner.name, user->name) == 0); +out: + spin_unlock(&fp->f_lock); + return ret; +} + +static bool session_fd_check(struct ksmbd_tree_connect *tcon, + struct ksmbd_file *fp, struct ksmbd_user *user) +{ + struct ksmbd_inode *ci; + struct oplock_info *op; + struct ksmbd_conn *conn; + struct ksmbd_lock *smb_lock, *tmp_lock; + + if (!is_reconnectable(fp)) + return false; + + if (fp->f_state != FP_INITED) + return false; + + if (WARN_ON_ONCE(!fp->conn)) + return false; + + if (ksmbd_vfs_set_durable_owner(fp, user)) + return false; + + /* + * fp owns a strong reference on fp->conn (taken in ksmbd_open_fd() + * / ksmbd_reopen_durable_fd()), so conn stays valid for the whole + * body of this function regardless of any op->conn puts below. + */ + conn = fp->conn; + ci = fp->f_ci; + down_write(&ci->m_lock); + list_for_each_entry_rcu(op, &ci->m_op_list, op_entry, + lockdep_is_held(&ci->m_lock)) { + if (op->conn != conn) + continue; + ksmbd_conn_put(op->conn); + op->conn = NULL; + op->sess = NULL; + } + up_write(&ci->m_lock); + + list_for_each_entry_safe(smb_lock, tmp_lock, &fp->lock_list, flist) { + struct ksmbd_conn *lock_conn = smb_lock->conn; + + if (!lock_conn) + continue; + spin_lock(&lock_conn->llist_lock); + list_del_init(&smb_lock->clist); + smb_lock->conn = NULL; + spin_unlock(&lock_conn->llist_lock); + ksmbd_conn_put(lock_conn); + } + + fp->conn = NULL; + fp->tcon = NULL; + fp->volatile_id = KSMBD_NO_FID; + + if (fp->durable_timeout) + fp->durable_scavenger_timeout = + jiffies_to_msecs(jiffies) + fp->durable_timeout; + + /* Drop fp's own reference on conn. */ + ksmbd_conn_put(conn); + return true; +} + +void ksmbd_close_tree_conn_fds(struct ksmbd_work *work) +{ + int num = __close_file_table_ids(work->sess, + work->tcon, + tree_conn_fd_check, + false); + + atomic_sub(num, &work->conn->stats.open_files_count); +} + +void ksmbd_close_session_fds(struct ksmbd_work *work) +{ + int num = __close_file_table_ids(work->sess, + work->tcon, + session_fd_check, + true); + + atomic_sub(num, &work->conn->stats.open_files_count); +} + +int ksmbd_init_global_file_table(void) +{ + if (create_proc_files()) + pr_warn("Unable to create files procfs entry\n"); + return ksmbd_init_file_table(&global_ft); +} + +void ksmbd_free_global_file_table(void) +{ + struct ksmbd_file *fp = NULL; + unsigned int id; + + idr_for_each_entry(global_ft.idr, fp, id) { + ksmbd_remove_durable_fd(fp); + __ksmbd_close_fd(NULL, fp); + } + + idr_destroy(global_ft.idr); + kfree(global_ft.idr); +} + +int ksmbd_validate_name_reconnect(struct ksmbd_share_config *share, + struct ksmbd_file *fp, char *name) +{ + char *pathname, *ab_pathname; + int ret = 0; + + pathname = kmalloc(PATH_MAX, KSMBD_DEFAULT_GFP); + if (!pathname) + return -EACCES; + + ab_pathname = d_path(&fp->filp->f_path, pathname, PATH_MAX); + if (IS_ERR(ab_pathname)) { + kfree(pathname); + return -EACCES; + } + + if (name && strcmp(&ab_pathname[share->path_sz + 1], name)) { + ksmbd_debug(SMB, "invalid name reconnect %s\n", name); + ret = -EINVAL; + } + + kfree(pathname); + + return ret; +} + +int ksmbd_reopen_durable_fd(struct ksmbd_work *work, struct ksmbd_file *fp) +{ + struct ksmbd_inode *ci; + struct oplock_info *op; + struct ksmbd_conn *conn = work->conn; + struct ksmbd_lock *smb_lock; + unsigned int old_f_state; + + write_lock(&global_ft.lock); + if ((!fp->is_durable && !fp->is_persistent) || fp->conn || fp->tcon) { + write_unlock(&global_ft.lock); + pr_err("Invalid durable fd [%p:%p]\n", fp->conn, fp->tcon); + return -EBADF; + } + + if (has_file_id(fp->volatile_id)) { + write_unlock(&global_ft.lock); + pr_err("Still in use durable fd: %llu\n", fp->volatile_id); + return -EBADF; + } + + /* + * Initialize fp's connection binding before publishing fp into the + * session's file table. If __open_id() is ordered first, a + * concurrent teardown that iterates the table can observe a valid + * volatile_id with fp->conn == NULL and preserve a + * partially-initialized fp. fp owns a strong reference on the new + * conn (see ksmbd_open_fd()); undo it on __open_id() failure. + */ + fp->conn = ksmbd_conn_get(conn); + fp->tcon = work->tcon; + write_unlock(&global_ft.lock); + + old_f_state = fp->f_state; + fp->f_state = FP_NEW; + + __open_id(&work->sess->file_table, fp, OPEN_ID_TYPE_VOLATILE_ID); + if (!has_file_id(fp->volatile_id)) { + write_lock(&global_ft.lock); + fp->conn = NULL; + fp->tcon = NULL; + write_unlock(&global_ft.lock); + ksmbd_conn_put(conn); + fp->f_state = old_f_state; + return -EBADF; + } + + list_for_each_entry(smb_lock, &fp->lock_list, flist) { + smb_lock->conn = ksmbd_conn_get(conn); + spin_lock(&conn->llist_lock); + list_add_tail(&smb_lock->clist, &conn->lock_list); + spin_unlock(&conn->llist_lock); + } + + ci = fp->f_ci; + down_write(&ci->m_lock); + list_for_each_entry_rcu(op, &ci->m_op_list, op_entry, + lockdep_is_held(&ci->m_lock)) { + if (op->conn || op->o_fp != fp) + continue; + op->conn = ksmbd_conn_get(fp->conn); + op->sess = work->sess; + } + up_write(&ci->m_lock); + + spin_lock(&fp->f_lock); + fp->owner.uid = fp->owner.gid = 0; + kfree(fp->owner.name); + fp->owner.name = NULL; + spin_unlock(&fp->f_lock); + + return 0; +} + +int ksmbd_init_file_table(struct ksmbd_file_table *ft) +{ + ft->idr = kzalloc_obj(struct idr, KSMBD_DEFAULT_GFP); + if (!ft->idr) + return -ENOMEM; + + idr_init(ft->idr); + rwlock_init(&ft->lock); + return 0; +} + +void ksmbd_destroy_file_table(struct ksmbd_session *sess) +{ + struct ksmbd_file_table *ft = &sess->file_table; + + if (!ft->idr) + return; + + __close_file_table_ids(sess, NULL, session_fd_check, true); + idr_destroy(ft->idr); + kfree(ft->idr); + ft->idr = NULL; +} + +int ksmbd_init_file_cache(void) +{ + filp_cache = kmem_cache_create("ksmbd_file_cache", + sizeof(struct ksmbd_file), 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!filp_cache) + goto out; + + init_waitqueue_head(&dh_wq); + + return 0; + +out: + pr_err("failed to allocate file cache\n"); + return -ENOMEM; +} + +void ksmbd_exit_file_cache(void) +{ + kmem_cache_destroy(filp_cache); +} diff --git a/fs/smb/server/vfs_cache.h b/fs/smb/server/vfs_cache.h new file mode 100644 index 000000000..502efb16f --- /dev/null +++ b/fs/smb/server/vfs_cache.h @@ -0,0 +1,263 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2019 Samsung Electronics Co., Ltd. + */ + +#ifndef __VFS_CACHE_H__ +#define __VFS_CACHE_H__ + +#include +#include +#include +#include +#include +#include +#include + +#include "vfs.h" +#include "mgmt/share_config.h" + +/* Windows style file permissions for extended response */ +#define FILE_GENERIC_ALL 0x1F01FF +#define FILE_GENERIC_READ 0x120089 +#define FILE_GENERIC_WRITE 0x120116 +#define FILE_GENERIC_EXECUTE 0X1200a0 + +/* + * Start volatile/persistent file id allocation at 1. A file id of 0 yields an + * SMB2 FileId of {0, 0}, which clients (e.g. Windows, Samba) treat as a null + * handle and never close, leaking the open on the server. + */ +#define KSMBD_START_FID 1 +#define KSMBD_NO_FID (INT_MAX) +#define SMB2_NO_FID (0xFFFFFFFFFFFFFFFFULL) + +struct ksmbd_conn; +struct ksmbd_session; + +struct ksmbd_lock { + struct file_lock *fl; + struct ksmbd_conn *conn; + struct list_head clist; + struct list_head flist; + struct list_head llist; + unsigned int flags; + int cmd; + int zero_len; + unsigned long long start; + unsigned long long end; +}; + +struct stream { + char *name; + ssize_t size; + loff_t pos; +}; + +struct ksmbd_inode { + struct rw_semaphore m_lock; + atomic_t m_count; + atomic_t op_count; + /* opinfo count for streams */ + atomic_t sop_count; + struct dentry *m_de; + unsigned int m_flags; + struct hlist_node m_hash; + struct list_head m_fp_list; + struct list_head m_op_list; + struct oplock_info *m_opinfo; + __le32 m_fattr; +}; + +enum { + FP_NEW = 0, + FP_INITED, + FP_CLOSED +}; + +/* Owner information for durable handle reconnect */ +struct durable_owner { + unsigned int uid; + unsigned int gid; + char *name; +}; + +#define KSMBD_LOCK_SEQ_ARRAY_SIZE 64 + +struct ksmbd_lock_sequence { + bool valid; + u8 sequence; +}; + +struct ksmbd_file { + struct file *filp; + u64 persistent_id; + u64 volatile_id; + u64 durable_volatile_id; + + spinlock_t f_lock; + + struct ksmbd_inode *f_ci; + struct ksmbd_inode *f_parent_ci; + struct oplock_info __rcu *f_opinfo; + struct ksmbd_conn *conn; + struct ksmbd_tree_connect *tcon; + + atomic_t refcount; + __le32 daccess; + __le32 saccess; + __le32 coption; + __le32 cdoption; + __le32 create_file_attributes; + __u64 create_time; + __u64 change_time; + __u64 allocation_size; + __u64 itime; + __u64 open_mtime; + + bool is_nt_open; + bool attrib_only; + bool allocation_size_set; + + char client_guid[16]; + char create_guid[16]; + char app_instance_id[16]; + + struct stream stream; + struct list_head node; + struct list_head blocked_works; + struct list_head lock_list; + /* + * Per-handle FileDispositionInformation delete-pending state for a + * stream handle -- separate from ksmbd_inode's inode-wide m_flags, + * which have no way to record which stream on a multi-stream file + * was actually marked for deletion. See ksmbd_fd_set_delete_pending(). + */ + bool stream_del_pending; + + unsigned int durable_timeout; + unsigned int durable_scavenger_timeout; + /* CREATE action returned when this durable handle was established. */ + __le32 create_action; + + /* if ls is happening on directory, below is valid*/ + struct ksmbd_readdir_data readdir_data; + struct mutex readdir_lock; + int dot_dotdot[2]; + unsigned int f_state; + bool reserve_lease_break; + bool is_durable; + bool is_persistent; + bool is_resilient; + bool has_app_instance_id; + bool app_instance_version_valid; + u64 app_instance_version_high; + u64 app_instance_version_low; + bool durable_reconnect_disabled; + bool durable_replay_consumed; + + bool is_posix_ctxt; + struct durable_owner owner; + __le16 channel_sequence; + unsigned int outstanding_requests; + unsigned int outstanding_pre_requests; + struct ksmbd_lock_sequence lock_seq[KSMBD_LOCK_SEQ_ARRAY_SIZE]; + + /* + * Pending CHANGE_NOTIFY completions for this handle, sent with + * STATUS_NOTIFY_CLEANUP when the handle is closed. + */ + struct list_head notify_pendings; +}; + +static inline void set_ctx_actor(struct dir_context *ctx, + filldir_t actor) +{ + ctx->actor = actor; +} + +#define KSMBD_NR_OPEN_DEFAULT BITS_PER_LONG + +struct ksmbd_file_table { + rwlock_t lock; + struct idr *idr; +}; + +static inline bool has_file_id(u64 id) +{ + return id < KSMBD_NO_FID; +} + +static inline bool ksmbd_stream_fd(struct ksmbd_file *fp) +{ + return fp->stream.name != NULL; +} + +int ksmbd_init_file_table(struct ksmbd_file_table *ft); +void ksmbd_destroy_file_table(struct ksmbd_session *sess); +int ksmbd_close_fd(struct ksmbd_work *work, u64 id); +struct ksmbd_file *ksmbd_lookup_fd_fast(struct ksmbd_work *work, u64 id); +struct ksmbd_file *ksmbd_lookup_foreign_fd(struct ksmbd_work *work, u64 id); +struct ksmbd_file *ksmbd_lookup_fd_slow(struct ksmbd_work *work, u64 id, + u64 pid); +int ksmbd_vfs_set_durable_owner(struct ksmbd_file *fp, + struct ksmbd_user *user); +struct ksmbd_file *ksmbd_file_get(struct ksmbd_file *fp); +void ksmbd_fd_put(struct ksmbd_work *work, struct ksmbd_file *fp); +struct ksmbd_inode *ksmbd_inode_lookup_lock(struct dentry *d); +void ksmbd_inode_put(struct ksmbd_inode *ci); +bool ksmbd_close_disconnected_durable_delete_on_close(struct dentry *dentry); +struct ksmbd_file *ksmbd_lookup_global_fd(unsigned long long id); +struct ksmbd_file *ksmbd_lookup_durable_fd(unsigned long long id); +void ksmbd_put_durable_fd(struct ksmbd_file *fp); +int ksmbd_invalidate_durable_fd(unsigned long long id); +bool ksmbd_has_other_active_fd(struct ksmbd_file *fp); +bool ksmbd_has_stream_without_delete_share(struct ksmbd_file *fp); +struct ksmbd_file *ksmbd_lookup_fd_app_instance_id(char *app_instance_id); +int ksmbd_close_fd_app_instance_id(char *app_instance_id); +struct ksmbd_file *ksmbd_lookup_fd_cguid(char *cguid); +struct ksmbd_file *ksmbd_lookup_fd_inode(struct dentry *dentry); +bool ksmbd_has_other_nonposix_open(struct dentry *dentry); +bool ksmbd_has_nonposix_open_child(struct ksmbd_file *old_fp); +unsigned int ksmbd_open_durable_fd(struct ksmbd_file *fp); +struct ksmbd_file *ksmbd_open_fd(struct ksmbd_work *work, struct file *filp); +void ksmbd_launch_ksmbd_durable_scavenger(void); +void ksmbd_stop_durable_scavenger(void); +bool ksmbd_durable_scavenger_active(void); +void ksmbd_close_tree_conn_fds(struct ksmbd_work *work); +void ksmbd_close_session_fds(struct ksmbd_work *work); +int ksmbd_close_inode_fds(struct ksmbd_work *work, struct inode *inode); +int ksmbd_init_global_file_table(void); +void ksmbd_free_global_file_table(void); +void ksmbd_set_fd_limit(unsigned long limit); +int ksmbd_update_fstate(struct ksmbd_file_table *ft, struct ksmbd_file *fp, + unsigned int state); +bool ksmbd_vfs_compare_durable_owner(struct ksmbd_file *fp, + struct ksmbd_user *user); + +/* + * INODE hash + */ +int __init ksmbd_inode_hash_init(void); +void ksmbd_release_inode_hash(void); + +enum KSMBD_INODE_STATUS { + KSMBD_INODE_STATUS_OK, + KSMBD_INODE_STATUS_UNKNOWN, + KSMBD_INODE_STATUS_PENDING_DELETE, +}; + +int ksmbd_query_inode_status(struct dentry *dentry); +bool ksmbd_inode_pending_delete(struct ksmbd_file *fp); +void ksmbd_set_inode_pending_delete(struct ksmbd_file *fp); +void ksmbd_clear_inode_pending_delete(struct ksmbd_file *fp); +void ksmbd_fd_set_delete_on_close(struct ksmbd_file *fp, + int file_info); +void ksmbd_fd_set_delete_pending(struct ksmbd_file *fp); +void ksmbd_fd_clear_delete_pending(struct ksmbd_file *fp); +int ksmbd_reopen_durable_fd(struct ksmbd_work *work, struct ksmbd_file *fp); +int ksmbd_validate_name_reconnect(struct ksmbd_share_config *share, + struct ksmbd_file *fp, char *name); +int ksmbd_init_file_cache(void); +void ksmbd_exit_file_cache(void); +#endif /* __VFS_CACHE_H__ */ diff --git a/fs/smb/server/xattr.h b/fs/smb/server/xattr.h new file mode 100644 index 000000000..505101a81 --- /dev/null +++ b/fs/smb/server/xattr.h @@ -0,0 +1,122 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2021 Samsung Electronics Co., Ltd. + */ + +#ifndef __XATTR_H__ +#define __XATTR_H__ + +/* + * These are on-disk structures to store additional metadata into xattr to + * reproduce windows filesystem semantics. And they are encoded with NDR to + * compatible with samba's xattr meta format. The compatibility with samba + * is important because it can lose the information(file attribute, + * creation time, acls) about the existing files when switching between + * ksmbd and samba. + */ + +/* + * Dos attribute flags used for what variable is valid. + */ +enum { + XATTR_DOSINFO_ATTRIB = 0x00000001, + XATTR_DOSINFO_EA_SIZE = 0x00000002, + XATTR_DOSINFO_SIZE = 0x00000004, + XATTR_DOSINFO_ALLOC_SIZE = 0x00000008, + XATTR_DOSINFO_CREATE_TIME = 0x00000010, + XATTR_DOSINFO_CHANGE_TIME = 0x00000020, + XATTR_DOSINFO_ITIME = 0x00000040 +}; + +/* + * Dos attribute structure which is compatible with samba's one. + * Storing it into the xattr named "DOSATTRIB" separately from inode + * allows ksmbd to faithfully reproduce windows filesystem semantics + * on top of a POSIX filesystem. + */ +struct xattr_dos_attrib { + __u16 version; /* version 3 or version 4 */ + __u32 flags; /* valid flags */ + __u32 attr; /* Dos attribute */ + __u32 ea_size; /* EA size */ + __u64 size; + __u64 alloc_size; + __u64 create_time; /* File creation time */ + __u64 change_time; /* File change time */ + __u64 itime; /* Invented/Initial time */ +}; + +/* + * Enumeration is used for computing posix acl hash. + */ +enum { + SMB_ACL_TAG_INVALID = 0, + SMB_ACL_USER, + SMB_ACL_USER_OBJ, + SMB_ACL_GROUP, + SMB_ACL_GROUP_OBJ, + SMB_ACL_OTHER, + SMB_ACL_MASK +}; + +#define SMB_ACL_READ 4 +#define SMB_ACL_WRITE 2 +#define SMB_ACL_EXECUTE 1 + +struct xattr_acl_entry { + int type; + uid_t uid; + gid_t gid; + mode_t perm; +}; + +/* + * xattr_smb_acl structure is used for computing posix acl hash. + */ +struct xattr_smb_acl { + int count; + int next; + struct xattr_acl_entry entries[] __counted_by(count); +}; + +/* 64bytes hash in xattr_ntacl is computed with sha256 */ +#define XATTR_SD_HASH_TYPE_SHA256 0x1 +#define XATTR_SD_HASH_SIZE 64 + +/* + * xattr_ntacl is used for storing ntacl and hashes. + * Hash is used for checking valid posix acl and ntacl in xattr. + */ +struct xattr_ntacl { + __u16 version; /* version 4*/ + void *sd_buf; + __u32 sd_size; + __u16 hash_type; /* hash type */ + __u8 desc[10]; /* posix_acl description */ + __u16 desc_len; + __u64 current_time; + __u8 hash[XATTR_SD_HASH_SIZE]; /* 64bytes hash for ntacl */ + __u8 posix_acl_hash[XATTR_SD_HASH_SIZE]; /* 64bytes hash for posix acl */ +}; + +/* DOS ATTRIBUTE XATTR PREFIX */ +#define DOS_ATTRIBUTE_PREFIX "DOSATTRIB" +#define DOS_ATTRIBUTE_PREFIX_LEN (sizeof(DOS_ATTRIBUTE_PREFIX) - 1) +#define XATTR_NAME_DOS_ATTRIBUTE (XATTR_USER_PREFIX DOS_ATTRIBUTE_PREFIX) +#define XATTR_NAME_DOS_ATTRIBUTE_LEN \ + (sizeof(XATTR_USER_PREFIX DOS_ATTRIBUTE_PREFIX) - 1) + +/* STREAM XATTR PREFIX */ +#define STREAM_PREFIX "DosStream." +#define STREAM_PREFIX_LEN (sizeof(STREAM_PREFIX) - 1) +#define XATTR_NAME_STREAM (XATTR_USER_PREFIX STREAM_PREFIX) +#define XATTR_NAME_STREAM_LEN (sizeof(XATTR_NAME_STREAM) - 1) + +/* SECURITY DESCRIPTOR(NTACL) XATTR PREFIX */ +#define SD_PREFIX "NTACL" +#define SD_PREFIX_LEN (sizeof(SD_PREFIX) - 1) +#define XATTR_NAME_SD (XATTR_SECURITY_PREFIX SD_PREFIX) +#define XATTR_NAME_SD_LEN \ + (sizeof(XATTR_SECURITY_PREFIX SD_PREFIX) - 1) + +#endif /* __XATTR_H__ */ diff --git a/fs/smb/smbdirect/Kconfig b/fs/smb/smbdirect/Kconfig new file mode 100644 index 000000000..2bf1b3350 --- /dev/null +++ b/fs/smb/smbdirect/Kconfig @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# +# smbdirect configuration + +config SMBDIRECT + def_tristate n + depends on INFINIBAND && INFINIBAND_ADDR_TRANS + depends on m || INFINIBAND=y + select SG_POOL diff --git a/fs/smb/smbdirect/Makefile b/fs/smb/smbdirect/Makefile new file mode 100644 index 000000000..80d602598 --- /dev/null +++ b/fs/smb/smbdirect/Makefile @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Makefile for smbdirect support +# + +obj-$(CONFIG_SMBDIRECT) += smbdirect.o + +smbdirect-y := \ + socket.o \ + connection.o \ + mr.o \ + rw.o \ + debug.o \ + connect.o \ + listen.o \ + accept.o \ + devices.o \ + main.o diff --git a/fs/smb/smbdirect/accept.c b/fs/smb/smbdirect/accept.c new file mode 100644 index 000000000..1b30ca847 --- /dev/null +++ b/fs/smb/smbdirect/accept.c @@ -0,0 +1,868 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017, Microsoft Corporation. + * Copyright (C) 2018, LG Electronics. + * Copyright (c) 2025, Stefan Metzmacher + */ + +#include "internal.h" +#include +#include "../common/smb2status.h" + +static int smbdirect_accept_rdma_event_handler(struct rdma_cm_id *id, + struct rdma_cm_event *event); +static int smbdirect_accept_init_params(struct smbdirect_socket *sc); +static void smbdirect_accept_negotiate_recv_done(struct ib_cq *cq, struct ib_wc *wc); +static void smbdirect_accept_negotiate_send_done(struct ib_cq *cq, struct ib_wc *wc); + +int smbdirect_accept_connect_request(struct smbdirect_socket *sc, + const struct rdma_conn_param *param) +{ + struct smbdirect_socket_parameters *sp = &sc->parameters; + struct smbdirect_recv_io *recv_io; + u8 peer_initiator_depth; + u8 peer_responder_resources; + struct rdma_conn_param conn_param; + __be32 ird_ord_hdr[2]; + int ret; + + if (SMBDIRECT_CHECK_STATUS_WARN(sc, SMBDIRECT_SOCKET_CREATED)) + return -EINVAL; + + /* + * First set what the we as server are able to support + */ + sp->initiator_depth = min3(sp->initiator_depth, + sc->ib.dev->attrs.max_qp_rd_atom, + U8_MAX); + + peer_initiator_depth = param->initiator_depth; + peer_responder_resources = param->responder_resources; + smbdirect_connection_negotiate_rdma_resources(sc, + peer_initiator_depth, + peer_responder_resources, + param); + + ret = smbdirect_accept_init_params(sc); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_accept_init_params() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto init_params_failed; + } + + ret = smbdirect_connection_create_qp(sc); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_create_qp() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto create_qp_failed; + } + + ret = smbdirect_connection_create_mem_pools(sc); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_create_mem_pools() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto create_mem_failed; + } + + recv_io = smbdirect_connection_get_recv_io(sc); + if (WARN_ON_ONCE(!recv_io)) { + ret = -EINVAL; + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_get_recv_io() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto get_recv_io_failed; + } + recv_io->cqe.done = smbdirect_accept_negotiate_recv_done; + + /* + * Now post the recv_io buffer in order to get + * the negotiate request + */ + sc->recv_io.expected = SMBDIRECT_EXPECT_NEGOTIATE_REQ; + ret = smbdirect_connection_post_recv_io(recv_io); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_post_recv_io() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto post_recv_io_failed; + } + /* + * From here recv_io is known to the RDMA QP and needs ib_drain_qp and + * smbdirect_accept_negotiate_recv_done to cleanup... + */ + recv_io = NULL; + + /* already checked with SMBDIRECT_CHECK_STATUS_WARN above */ + WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED); + sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED; + + /* + * We already negotiated sp->initiator_depth + * and sp->responder_resources above. + */ + memset(&conn_param, 0, sizeof(conn_param)); + conn_param.initiator_depth = sp->initiator_depth; + conn_param.responder_resources = sp->responder_resources; + + if (sc->rdma.legacy_iwarp) { + ird_ord_hdr[0] = cpu_to_be32(conn_param.responder_resources); + ird_ord_hdr[1] = cpu_to_be32(conn_param.initiator_depth); + conn_param.private_data = ird_ord_hdr; + conn_param.private_data_len = sizeof(ird_ord_hdr); + } else { + conn_param.private_data = NULL; + conn_param.private_data_len = 0; + } + conn_param.retry_count = SMBDIRECT_RDMA_CM_RETRY; + conn_param.rnr_retry_count = SMBDIRECT_RDMA_CM_RNR_RETRY; + conn_param.flow_control = 0; + + /* explicitly set above */ + WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED); + sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING; + sc->rdma.expected_event = RDMA_CM_EVENT_ESTABLISHED; + sc->rdma.cm_id->event_handler = smbdirect_accept_rdma_event_handler; + ret = rdma_accept(sc->rdma.cm_id, &conn_param); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "rdma_accept() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto rdma_accept_failed; + } + + /* + * start with the negotiate timeout and SMBDIRECT_KEEPALIVE_PENDING + * so that the timer will cause a disconnect. + */ + INIT_DELAYED_WORK(&sc->idle.timer_work, smbdirect_connection_idle_timer_work); + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING; + mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work, + msecs_to_jiffies(sp->negotiate_timeout_msec)); + + return 0; + +rdma_accept_failed: + /* + * The recv_io posted above is now owned by the QP (recv_io was set to + * NULL after a successful post). smbdirect_connection_destroy_qp() + * calls ib_drain_qp(), whose completion + * (smbdirect_accept_negotiate_recv_done) returns the recv_io to the + * free list via smbdirect_connection_put_recv_io(). It therefore MUST + * run BEFORE smbdirect_connection_destroy_mem_pools(): otherwise the + * posted recv_io is still outstanding when kmem_cache_destroy() runs + * ("Slab cache still has objects") and is later freed into an + * already-destroyed mempool (mempool_free_bulk NULL-ptr-deref). + */ + smbdirect_connection_destroy_qp(sc); + smbdirect_connection_destroy_mem_pools(sc); + return ret; +post_recv_io_failed: + /* post failed: recv_io was not accepted by the QP, still in hand */ + if (recv_io) + smbdirect_connection_put_recv_io(recv_io); +get_recv_io_failed: + smbdirect_connection_destroy_mem_pools(sc); +create_mem_failed: + smbdirect_connection_destroy_qp(sc); +create_qp_failed: +init_params_failed: + return ret; +} + +static int smbdirect_accept_init_params(struct smbdirect_socket *sc) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + int max_send_sges; + unsigned int maxpages; + + /* need 3 more sge. because a SMB_DIRECT header, SMB2 header, + * SMB2 response could be mapped. + */ + max_send_sges = DIV_ROUND_UP(sp->max_send_size, PAGE_SIZE) + 3; + if (max_send_sges > SMBDIRECT_SEND_IO_MAX_SGE) { + pr_err("max_send_size %d is too large\n", sp->max_send_size); + return -EINVAL; + } + + /* + * There is only a single batch credit + */ + atomic_set(&sc->send_io.bcredits.count, 1); + + /* + * Initialize the local credits to post + * IB_WR_SEND[_WITH_INV]. + */ + atomic_set(&sc->send_io.lcredits.count, sp->send_credit_target); + + if (sp->max_read_write_size) { + maxpages = DIV_ROUND_UP(sp->max_read_write_size, PAGE_SIZE); + sc->rw_io.credits.max = rdma_rw_mr_factor(sc->ib.dev, + sc->rdma.cm_id->port_num, + maxpages); + sc->rw_io.credits.num_pages = DIV_ROUND_UP(maxpages, sc->rw_io.credits.max); + /* add one extra in order to handle unaligned pages */ + sc->rw_io.credits.max += 1; + } + + sc->recv_io.credits.target = 1; + + atomic_set(&sc->rw_io.credits.count, sc->rw_io.credits.max); + + return 0; +} + +static void smbdirect_accept_negotiate_recv_work(struct work_struct *work); + +static void smbdirect_accept_negotiate_recv_done(struct ib_cq *cq, struct ib_wc *wc) +{ + struct smbdirect_recv_io *recv_io = + container_of(wc->wr_cqe, struct smbdirect_recv_io, cqe); + struct smbdirect_socket *sc = recv_io->socket; + unsigned long flags; + + if (unlikely(wc->status != IB_WC_SUCCESS || WARN_ON_ONCE(wc->opcode != IB_WC_RECV))) { + if (wc->status != IB_WC_WR_FLUSH_ERR) + smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_ERR, + "wc->status=%s (%d) wc->opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + goto error; + } + + smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_INFO, + "smbdirect_recv_io completed. status='%s (%d)', opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + + /* + * This is an internal error! + */ + if (WARN_ON_ONCE(sc->recv_io.expected != SMBDIRECT_EXPECT_NEGOTIATE_REQ)) + goto error; + + /* + * Don't reset timer to the keepalive interval in + * this will be done in smbdirect_accept_direct_negotiate_recv_work. + */ + + ib_dma_sync_single_for_cpu(sc->ib.dev, + recv_io->sge.addr, + recv_io->sge.length, + DMA_FROM_DEVICE); + + /* + * Only remember recv_io if it has enough bytes, + * this gives smbdirect_accept_negotiate_recv_work enough + * information in order to disconnect if it was not + * valid. + */ + sc->recv_io.reassembly.full_packet_received = true; + if (wc->byte_len >= sizeof(struct smbdirect_negotiate_req)) + smbdirect_connection_reassembly_append_recv_io(sc, recv_io, 0); + else + smbdirect_connection_put_recv_io(recv_io); + + /* + * Some drivers (at least mlx5_ib and irdma) might post a + * recv completion before RDMA_CM_EVENT_ESTABLISHED, + * we need to adjust our expectation in that case. + * + * So we defer further processing of the negotiation + * to smbdirect_accept_negotiate_recv_work(). + * + * If we are already in SMBDIRECT_SOCKET_NEGOTIATE_NEEDED + * we queue the work directly otherwise + * smbdirect_accept_rdma_event_handler() will do it, when + * RDMA_CM_EVENT_ESTABLISHED arrived. + */ + spin_lock_irqsave(&sc->connect.lock, flags); + if (!sc->first_error) { + INIT_WORK(&sc->connect.work, smbdirect_accept_negotiate_recv_work); + if (sc->status == SMBDIRECT_SOCKET_NEGOTIATE_NEEDED) + queue_work(sc->workqueues.accept, &sc->connect.work); + } + spin_unlock_irqrestore(&sc->connect.lock, flags); + + return; + +error: + /* + * recv_io.posted.refill_work is still disabled, + * so smbdirect_connection_put_recv_io() won't + * start it. + */ + smbdirect_connection_put_recv_io(recv_io); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); +} + +static void smbdirect_accept_negotiate_recv_work(struct work_struct *work) +{ + struct smbdirect_socket *sc = + container_of(work, struct smbdirect_socket, connect.work); + struct smbdirect_socket_parameters *sp = &sc->parameters; + struct smbdirect_recv_io *recv_io; + struct smbdirect_negotiate_req *nreq; + unsigned long flags; + u16 min_version; + u16 max_version; + u16 credits_requested; + u32 preferred_send_size; + u32 max_receive_size; + u32 max_fragmented_size; + u32 ntstatus; + + if (sc->first_error) + return; + + /* + * make sure we won't start again... + */ + disable_work(work); + + /* + * Reset timer to the keepalive interval in + * order to trigger our next keepalive message. + */ + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; + mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work, + msecs_to_jiffies(sp->keepalive_interval_msec)); + + /* + * If smbdirect_accept_negotiate_recv_done() detected an + * invalid request we want to disconnect. + */ + recv_io = smbdirect_connection_reassembly_first_recv_io(sc); + if (!recv_io) { + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); + sc->recv_io.reassembly.queue_length--; + list_del(&recv_io->list); + spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); + smbdirect_connection_put_recv_io(recv_io); + + if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_NEGOTIATE_NEEDED)) + return; + sc->status = SMBDIRECT_SOCKET_NEGOTIATE_RUNNING; + + /* + * Note recv_io is already part of the free list, + * as we just called smbdirect_connection_put_recv_io(), + * but it won't be reused before we call + * smbdirect_connection_recv_io_refill() below. + */ + + nreq = (struct smbdirect_negotiate_req *)recv_io->packet; + min_version = le16_to_cpu(nreq->min_version); + max_version = le16_to_cpu(nreq->max_version); + credits_requested = le16_to_cpu(nreq->credits_requested); + preferred_send_size = le32_to_cpu(nreq->preferred_send_size); + max_receive_size = le32_to_cpu(nreq->max_receive_size); + max_fragmented_size = le32_to_cpu(nreq->max_fragmented_size); + + smbdirect_log_negotiate(sc, SMBDIRECT_LOG_INFO, + "ReqIn: %s%x, %s%x, %s%u, %s%u, %s%u, %s%u\n", + "MinVersion=0x", + le16_to_cpu(nreq->min_version), + "MaxVersion=0x", + le16_to_cpu(nreq->max_version), + "CreditsRequested=", + le16_to_cpu(nreq->credits_requested), + "PreferredSendSize=", + le32_to_cpu(nreq->preferred_send_size), + "MaxRecvSize=", + le32_to_cpu(nreq->max_receive_size), + "MaxFragmentedSize=", + le32_to_cpu(nreq->max_fragmented_size)); + + if (!(min_version <= SMBDIRECT_V1 && max_version >= SMBDIRECT_V1)) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: min_version=0x%x max_version=0x%x\n", + min_version, max_version); + ntstatus = le32_to_cpu(STATUS_NOT_SUPPORTED); + goto not_supported; + } + + if (credits_requested == 0) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: credits_requested == 0\n"); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + + if (max_receive_size < SMBDIRECT_MIN_RECEIVE_SIZE) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: max_receive_size=%u < %u\n", + max_receive_size, + SMBDIRECT_MIN_RECEIVE_SIZE); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + + if (max_fragmented_size < SMBDIRECT_MIN_FRAGMENTED_SIZE) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: max_fragmented_size=%u < %u\n", + max_fragmented_size, + SMBDIRECT_MIN_FRAGMENTED_SIZE); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + + /* + * At least the value of SMBDIRECT_MIN_RECEIVE_SIZE is used. + */ + sp->max_recv_size = min_t(u32, sp->max_recv_size, preferred_send_size); + sp->max_recv_size = max_t(u32, sp->max_recv_size, SMBDIRECT_MIN_RECEIVE_SIZE); + + /* + * The maximum fragmented upper-layer payload receive size supported + * + * Assume max_payload_per_credit is + * smb_direct_receive_credit_max - 24 = 1340 + * + * The maximum number would be + * smb_direct_receive_credit_max * max_payload_per_credit + * + * 1340 * 255 = 341700 (0x536C4) + * + * The minimum value from the spec is 131072 (0x20000) + * + * For now we use the logic we used in ksmbd before: + * (1364 * 255) / 2 = 173910 (0x2A756) + * + * We need to adjust this here in case the peer + * lowered sp->max_recv_size. + * + * TODO: instead of adjusting max_fragmented_recv_size + * we should adjust the number of available buffers, + * but for now we keep the logic as it was used + * in ksmbd before. + */ + sp->max_fragmented_recv_size = (sp->recv_credit_max * sp->max_recv_size) / 2; + + /* + * We take the value from the peer, which is checked to be higher than 0, + * but we limit it to the max value we support in order to have + * the main logic simpler. + */ + sc->recv_io.credits.target = credits_requested; + sc->recv_io.credits.target = min_t(u16, sc->recv_io.credits.target, + sp->recv_credit_max); + + /* + * Note nreq->max_receive_size was already checked against + * SMBDIRECT_MIN_RECEIVE_SIZE above. + */ + sp->max_send_size = min_t(u32, sp->max_send_size, max_receive_size); + + /* + * Note nreq->max_fragmented_size was already checked against + * SMBDIRECT_MIN_FRAGMENTED_SIZE above. + */ + sp->max_fragmented_send_size = max_fragmented_size; + + if (sc->accept.listener) { + struct smbdirect_socket *lsc = sc->accept.listener; + unsigned long flags; + + spin_lock_irqsave(&lsc->listen.lock, flags); + list_del(&sc->accept.list); + list_add_tail(&sc->accept.list, &lsc->listen.ready); + wake_up(&lsc->listen.wait_queue); + spin_unlock_irqrestore(&lsc->listen.lock, flags); + + /* + * smbdirect_socket_accept() will call + * smbdirect_accept_negotiate_finish(nsc, 0); + * + * So that we don't send the negotiation + * response that grants credits to the peer + * before the socket is accepted by the + * application. + */ + return; + } + + ntstatus = le32_to_cpu(STATUS_SUCCESS); + +not_supported: + smbdirect_accept_negotiate_finish(sc, ntstatus); +} + +void smbdirect_accept_negotiate_finish(struct smbdirect_socket *sc, u32 ntstatus) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + struct smbdirect_recv_io *recv_io; + struct smbdirect_send_io *send_io; + struct smbdirect_negotiate_resp *nrep; + int posted; + u16 new_credits; + int ret; + + if (ntstatus) + goto not_supported; + + /* + * Prepare for receiving data_transfer messages + */ + sc->recv_io.reassembly.full_packet_received = true; + sc->recv_io.expected = SMBDIRECT_EXPECT_DATA_TRANSFER; + list_for_each_entry(recv_io, &sc->recv_io.free.list, list) + recv_io->cqe.done = smbdirect_connection_recv_io_done; + recv_io = NULL; + + /* + * We should at least post 1 smbdirect_recv_io! + */ + posted = smbdirect_connection_recv_io_refill(sc); + if (posted < 1) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_recv_io_refill() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(posted)); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + + /* + * The response will grant credits for all posted + * smbdirect_recv_io messages. + */ + new_credits = smbdirect_connection_grant_recv_credits(sc); + +not_supported: + send_io = smbdirect_connection_alloc_send_io(sc); + if (IS_ERR(send_io)) { + ret = PTR_ERR(send_io); + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_alloc_send_io() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + smbdirect_socket_schedule_cleanup(sc, ret); + return; + } + send_io->cqe.done = smbdirect_accept_negotiate_send_done; + + nrep = (struct smbdirect_negotiate_resp *)send_io->packet; + nrep->min_version = cpu_to_le16(SMBDIRECT_V1); + nrep->max_version = cpu_to_le16(SMBDIRECT_V1); + if (ntstatus == 0) { + nrep->negotiated_version = cpu_to_le16(SMBDIRECT_V1); + nrep->reserved = 0; + nrep->credits_requested = cpu_to_le16(sp->send_credit_target); + nrep->credits_granted = cpu_to_le16(new_credits); + nrep->status = cpu_to_le32(ntstatus); + nrep->max_readwrite_size = cpu_to_le32(sp->max_read_write_size); + nrep->preferred_send_size = cpu_to_le32(sp->max_send_size); + nrep->max_receive_size = cpu_to_le32(sp->max_recv_size); + nrep->max_fragmented_size = cpu_to_le32(sp->max_fragmented_recv_size); + } else { + nrep->negotiated_version = 0; + nrep->reserved = 0; + nrep->credits_requested = 0; + nrep->credits_granted = 0; + nrep->status = cpu_to_le32(ntstatus); + nrep->max_readwrite_size = 0; + nrep->preferred_send_size = 0; + nrep->max_receive_size = 0; + nrep->max_fragmented_size = 0; + } + + smbdirect_log_negotiate(sc, SMBDIRECT_LOG_INFO, + "RepOut: %s%x, %s%x, %s%x, %s%u, %s%u, %s%x, %s%u, %s%u, %s%u, %s%u\n", + "MinVersion=0x", + le16_to_cpu(nrep->min_version), + "MaxVersion=0x", + le16_to_cpu(nrep->max_version), + "NegotiatedVersion=0x", + le16_to_cpu(nrep->negotiated_version), + "CreditsRequested=", + le16_to_cpu(nrep->credits_requested), + "CreditsGranted=", + le16_to_cpu(nrep->credits_granted), + "Status=0x", + le32_to_cpu(nrep->status), + "MaxReadWriteSize=", + le32_to_cpu(nrep->max_readwrite_size), + "PreferredSendSize=", + le32_to_cpu(nrep->preferred_send_size), + "MaxRecvSize=", + le32_to_cpu(nrep->max_receive_size), + "MaxFragmentedSize=", + le32_to_cpu(nrep->max_fragmented_size)); + + send_io->sge[0].addr = ib_dma_map_single(sc->ib.dev, + nrep, + sizeof(*nrep), + DMA_TO_DEVICE); + ret = ib_dma_mapping_error(sc->ib.dev, send_io->sge[0].addr); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "ib_dma_mapping_error() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + smbdirect_connection_free_send_io(send_io); + smbdirect_socket_schedule_cleanup(sc, ret); + return; + } + + send_io->sge[0].length = sizeof(*nrep); + send_io->sge[0].lkey = sc->ib.pd->local_dma_lkey; + send_io->num_sge = 1; + + ib_dma_sync_single_for_device(sc->ib.dev, + send_io->sge[0].addr, + send_io->sge[0].length, + DMA_TO_DEVICE); + + send_io->wr.next = NULL; + send_io->wr.wr_cqe = &send_io->cqe; + send_io->wr.sg_list = send_io->sge; + send_io->wr.num_sge = send_io->num_sge; + send_io->wr.opcode = IB_WR_SEND; + send_io->wr.send_flags = IB_SEND_SIGNALED; + + ret = smbdirect_connection_post_send_wr(sc, &send_io->wr); + if (ret) { + /* if we reach here, post send failed */ + smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_post_send_wr() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + /* + * Note smbdirect_connection_free_send_io() + * does ib_dma_unmap_page() + */ + smbdirect_connection_free_send_io(send_io); + smbdirect_socket_schedule_cleanup(sc, ret); + return; + } + + /* + * smbdirect_accept_negotiate_send_done + * will do all remaining work... + */ +} + +static void smbdirect_accept_negotiate_send_done(struct ib_cq *cq, struct ib_wc *wc) +{ + struct smbdirect_send_io *send_io = + container_of(wc->wr_cqe, struct smbdirect_send_io, cqe); + struct smbdirect_socket *sc = send_io->socket; + struct smbdirect_negotiate_resp *nrep; + u32 ntstatus; + + smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_INFO, + "smbdirect_send_io completed. status='%s (%d)', opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + + nrep = (struct smbdirect_negotiate_resp *)send_io->packet; + ntstatus = le32_to_cpu(nrep->status); + + /* Note this frees wc->wr_cqe, but not wc */ + smbdirect_connection_free_send_io(send_io); + atomic_dec(&sc->send_io.pending.count); + + if (unlikely(wc->status != IB_WC_SUCCESS || WARN_ON_ONCE(wc->opcode != IB_WC_SEND))) { + if (wc->status != IB_WC_WR_FLUSH_ERR) + smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR, + "wc->status=%s (%d) wc->opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + + /* + * If we send a smbdirect_negotiate_resp without NT_STATUS_OK (0) + * we need to disconnect now. + * + * Otherwise smbdirect_connection_negotiation_done() + * will setup all required things and wake up + * the waiter. + */ + if (ntstatus) + smbdirect_socket_schedule_cleanup(sc, -EOPNOTSUPP); + else + smbdirect_connection_negotiation_done(sc); +} + +static int smbdirect_accept_rdma_event_handler(struct rdma_cm_id *id, + struct rdma_cm_event *event) +{ + struct smbdirect_socket *sc = id->context; + unsigned long flags; + + /* + * cma_cm_event_handler() has + * lockdep_assert_held(&id_priv->handler_mutex); + * + * Mutexes are not allowed in interrupts, + * and we rely on not being in an interrupt here, + * as we might sleep. + * + * We didn't timeout so we cancel our idle timer, + * it will be scheduled again if needed. + */ + WARN_ON_ONCE(in_interrupt()); + + if (event->status || event->event != sc->rdma.expected_event) { + int ret = -ECONNABORTED; + + if (event->event == RDMA_CM_EVENT_REJECTED) + ret = -ECONNREFUSED; + if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) + ret = -ENETDOWN; + if (IS_ERR(SMBDIRECT_DEBUG_ERR_PTR(event->status))) + ret = event->status; + + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "%s (first_error=%1pe, expected=%s) => event=%s status=%d => ret=%1pe\n", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), + rdma_event_msg(sc->rdma.expected_event), + rdma_event_msg(event->event), + event->status, + SMBDIRECT_DEBUG_ERR_PTR(ret)); + + smbdirect_socket_schedule_cleanup(sc, ret); + return 0; + } + + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "%s (first_error=%1pe) event=%s\n", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), + rdma_event_msg(event->event)); + + if (sc->first_error) + return 0; + + switch (event->event) { + case RDMA_CM_EVENT_ESTABLISHED: + smbdirect_connection_rdma_established(sc); + + /* + * Some drivers (at least mlx5_ib and irdma) might post a + * recv completion before RDMA_CM_EVENT_ESTABLISHED, + * we need to adjust our expectation in that case. + * + * If smbdirect_accept_negotiate_recv_done was called first + * it initialized sc->connect.work only for us to + * start, so that we turned into + * SMBDIRECT_SOCKET_NEGOTIATE_NEEDED, before + * smbdirect_accept_negotiate_recv_work() runs. + * + * If smbdirect_accept_negotiate_recv_done didn't happen + * yet. sc->connect.work is still be disabled and + * queue_work() is a no-op. + */ + if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING)) + return 0; + sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED; + spin_lock_irqsave(&sc->connect.lock, flags); + if (!sc->first_error) + queue_work(sc->workqueues.accept, &sc->connect.work); + spin_unlock_irqrestore(&sc->connect.lock, flags); + + /* + * wait for smbdirect_accept_negotiate_recv_done() + * to get the negotiate request. + */ + return 0; + + default: + break; + } + + /* + * This is an internal error + */ + WARN_ON_ONCE(sc->rdma.expected_event != RDMA_CM_EVENT_ESTABLISHED); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return 0; +} + +static long smbdirect_socket_wait_for_accept(struct smbdirect_socket *lsc, long timeo) +{ + long ret; + + ret = wait_event_interruptible_timeout(lsc->listen.wait_queue, + !list_empty_careful(&lsc->listen.ready) || + lsc->status != SMBDIRECT_SOCKET_LISTENING || + lsc->first_error, + timeo); + if (lsc->status != SMBDIRECT_SOCKET_LISTENING) + return -EINVAL; + if (lsc->first_error) + return lsc->first_error; + if (!ret) + ret = -ETIMEDOUT; + if (ret < 0) + return ret; + + return 0; +} + +struct smbdirect_socket *smbdirect_socket_accept(struct smbdirect_socket *lsc, + long timeo, + struct proto_accept_arg *arg) +{ + struct smbdirect_socket *nsc; + unsigned long flags; + + if (lsc->status != SMBDIRECT_SOCKET_LISTENING) { + arg->err = -EINVAL; + return NULL; + } + + if (lsc->first_error) { + arg->err = lsc->first_error; + return NULL; + } + + if (list_empty_careful(&lsc->listen.ready)) { + int ret; + + if (timeo == 0) { + arg->err = -EAGAIN; + return NULL; + } + + ret = smbdirect_socket_wait_for_accept(lsc, timeo); + if (ret) { + arg->err = ret; + return NULL; + } + } + + spin_lock_irqsave(&lsc->listen.lock, flags); + nsc = list_first_entry_or_null(&lsc->listen.ready, + struct smbdirect_socket, + accept.list); + if (nsc) { + nsc->accept.listener = NULL; + list_del_init_careful(&nsc->accept.list); + arg->is_empty = list_empty_careful(&lsc->listen.ready); + } + spin_unlock_irqrestore(&lsc->listen.lock, flags); + if (!nsc) { + arg->err = -EAGAIN; + return NULL; + } + + /* + * We did not send the negotiation response + * yet, so we did not grant any credits to the client, + * so it didn't grant any credits to us. + * + * The caller expects a connected socket + * now as there are no credits anyway. + * + * Then we send the negotiation response in + * order to grant credits to the peer. + */ + nsc->status = SMBDIRECT_SOCKET_CONNECTED; + smbdirect_accept_negotiate_finish(nsc, 0); + + return nsc; +} +EXPORT_SYMBOL_GPL(smbdirect_socket_accept); diff --git a/fs/smb/smbdirect/connect.c b/fs/smb/smbdirect/connect.c new file mode 100644 index 000000000..34a3e72c3 --- /dev/null +++ b/fs/smb/smbdirect/connect.c @@ -0,0 +1,926 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2012,2016,2017,2025 Stefan Metzmacher + */ + +#include "internal.h" +#include "../common/smb2status.h" + +static int smbdirect_connect_setup_connection(struct smbdirect_socket *sc); +static int smbdirect_connect_resolve_addr(struct smbdirect_socket *sc, + const struct sockaddr *src, + const struct sockaddr *dst); +static int smbdirect_connect_rdma_event_handler(struct rdma_cm_id *id, + struct rdma_cm_event *event); +static int smbdirect_connect_negotiate_start(struct smbdirect_socket *sc); +static void smbdirect_connect_negotiate_send_done(struct ib_cq *cq, struct ib_wc *wc); +static void smbdirect_connect_negotiate_recv_done(struct ib_cq *cq, struct ib_wc *wc); + +int smbdirect_connect(struct smbdirect_socket *sc, const struct sockaddr *dst) +{ + const struct sockaddr *src = NULL; + union { + struct sockaddr sa; + struct sockaddr_storage ss; + } src_addr = { + .sa = { + .sa_family = AF_UNSPEC, + }, + }; + int ret; + + if (sc->first_error) + return -ENOTCONN; + + if (sc->status != SMBDIRECT_SOCKET_CREATED) + return -EALREADY; + + if (WARN_ON_ONCE(!sc->rdma.cm_id)) + return -EINVAL; + + src_addr.ss = sc->rdma.cm_id->route.addr.src_addr; + if (src_addr.sa.sa_family != AF_UNSPEC) + src = &src_addr.sa; + + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "connect: src: %pISpsfc dst: %pISpsfc\n", + src, dst); + + ret = smbdirect_connect_setup_connection(sc); + if (ret) + return ret; + + ret = smbdirect_connect_resolve_addr(sc, src, dst); + if (ret) + return ret; + + /* + * The rest happens async via smbdirect_connect_rdma_event_handler() + * the caller will decide to wait or not. + */ + return 0; +} +EXPORT_SYMBOL_GPL(smbdirect_connect); + +static int smbdirect_connect_setup_connection(struct smbdirect_socket *sc) +{ + rdma_lock_handler(sc->rdma.cm_id); + sc->rdma.cm_id->event_handler = smbdirect_connect_rdma_event_handler; + rdma_unlock_handler(sc->rdma.cm_id); + + if (SMBDIRECT_CHECK_STATUS_WARN(sc, SMBDIRECT_SOCKET_CREATED)) + return -EINVAL; + sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED; + + return 0; +} + +static int smbdirect_connect_resolve_addr(struct smbdirect_socket *sc, + const struct sockaddr *src, + const struct sockaddr *dst) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + struct sockaddr *src_addr = NULL; + struct sockaddr *dst_addr = NULL; + int ret; + + src_addr = (struct sockaddr *)src; + if (src_addr && src_addr->sa_family == AF_UNSPEC) + src_addr = NULL; + dst_addr = (struct sockaddr *)dst; + + if (SMBDIRECT_CHECK_STATUS_WARN(sc, SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED)) + return -EINVAL; + sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING; + sc->rdma.expected_event = RDMA_CM_EVENT_ADDR_RESOLVED; + ret = rdma_resolve_addr(sc->rdma.cm_id, src_addr, dst_addr, + sp->resolve_addr_timeout_msec); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "rdma_resolve_addr() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + return ret; + } + + return 0; +} + +static int smbdirect_connect_resolve_route(struct smbdirect_socket *sc) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + int ret; + + if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED)) + return sc->first_error; + sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING; + sc->rdma.expected_event = RDMA_CM_EVENT_ROUTE_RESOLVED; + ret = rdma_resolve_route(sc->rdma.cm_id, sp->resolve_route_timeout_msec); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "rdma_resolve_route() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + return ret; + } + + return 0; +} + +static int smbdirect_connect_rdma_connect(struct smbdirect_socket *sc) +{ + struct smbdirect_socket_parameters *sp = &sc->parameters; + struct rdma_conn_param conn_param; + __be32 ird_ord_hdr[2]; + int ret; + + sc->ib.dev = sc->rdma.cm_id->device; + + if (!smbdirect_frwr_is_supported(&sc->ib.dev->attrs)) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "Fast Registration Work Requests (FRWR) is not supported device %.*s\n", + IB_DEVICE_NAME_MAX, + sc->ib.dev->name); + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "Device capability flags = %llx max_fast_reg_page_list_len = %u\n", + sc->ib.dev->attrs.device_cap_flags, + sc->ib.dev->attrs.max_fast_reg_page_list_len); + return -EPROTONOSUPPORT; + } + + if (sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB && + !rdma_ib_or_roce(sc->ib.dev, sc->rdma.cm_id->port_num)) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "Not IB: device: %.*s IW:%u local: %pISpsfc remote: %pISpsfc\n", + IB_DEVICE_NAME_MAX, + sc->ib.dev->name, + rdma_protocol_iwarp(sc->ib.dev, sc->rdma.cm_id->port_num), + &sc->rdma.cm_id->route.addr.src_addr, + &sc->rdma.cm_id->route.addr.dst_addr); + return -EPROTONOSUPPORT; + } + if (sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW && + !rdma_protocol_iwarp(sc->ib.dev, sc->rdma.cm_id->port_num)) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "Not IW: device: %.*s IB:%u local: %pISpsfc remote: %pISpsfc\n", + IB_DEVICE_NAME_MAX, + sc->ib.dev->name, + rdma_ib_or_roce(sc->ib.dev, sc->rdma.cm_id->port_num), + &sc->rdma.cm_id->route.addr.src_addr, + &sc->rdma.cm_id->route.addr.dst_addr); + return -EPROTONOSUPPORT; + } + + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "rdma connect: device: %.*s local: %pISpsfc remote: %pISpsfc\n", + IB_DEVICE_NAME_MAX, + sc->ib.dev->name, + &sc->rdma.cm_id->route.addr.src_addr, + &sc->rdma.cm_id->route.addr.dst_addr); + + sp->max_frmr_depth = min_t(u32, sp->max_frmr_depth, + sc->ib.dev->attrs.max_fast_reg_page_list_len); + sc->mr_io.type = IB_MR_TYPE_MEM_REG; + if (sc->ib.dev->attrs.kernel_cap_flags & IBK_SG_GAPS_REG) + sc->mr_io.type = IB_MR_TYPE_SG_GAPS; + + sp->responder_resources = min3(sp->responder_resources, + sc->ib.dev->attrs.max_qp_rd_atom, + U8_MAX); + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_INFO, + "responder_resources=%d\n", + sp->responder_resources); + + ret = smbdirect_connection_create_qp(sc); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_create_qp() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + return ret; + } + + memset(&conn_param, 0, sizeof(conn_param)); + conn_param.initiator_depth = sp->initiator_depth; + conn_param.responder_resources = sp->responder_resources; + + /* Need to send IRD/ORD in private data for iWARP */ + if (rdma_protocol_iwarp(sc->ib.dev, sc->rdma.cm_id->port_num)) { + ird_ord_hdr[0] = cpu_to_be32(conn_param.responder_resources); + ird_ord_hdr[1] = cpu_to_be32(conn_param.initiator_depth); + conn_param.private_data = ird_ord_hdr; + conn_param.private_data_len = sizeof(ird_ord_hdr); + } else { + conn_param.private_data = NULL; + conn_param.private_data_len = 0; + } + + conn_param.retry_count = SMBDIRECT_RDMA_CM_RETRY; + conn_param.rnr_retry_count = SMBDIRECT_RDMA_CM_RNR_RETRY; + conn_param.flow_control = 0; + + if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED)) + return sc->first_error; + sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING; + sc->rdma.expected_event = RDMA_CM_EVENT_ESTABLISHED; + ret = rdma_connect_locked(sc->rdma.cm_id, &conn_param); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "rdma_connect_locked() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + return ret; + } + + /* + * start with the rdma connect timeout and SMBDIRECT_KEEPALIVE_PENDING + * so that the timer will cause a disconnect. + */ + INIT_DELAYED_WORK(&sc->idle.timer_work, smbdirect_connection_idle_timer_work); + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING; + mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work, + msecs_to_jiffies(sp->rdma_connect_timeout_msec)); + + return 0; +} + +static int smbdirect_connect_rdma_event_handler(struct rdma_cm_id *id, + struct rdma_cm_event *event) +{ + struct smbdirect_socket *sc = id->context; + u8 peer_initiator_depth; + u8 peer_responder_resources; + int ret; + + /* + * cma_cm_event_handler() has + * lockdep_assert_held(&id_priv->handler_mutex); + * + * Mutexes are not allowed in interrupts, + * and we rely on not being in an interrupt here, + * as we might sleep. + * + * We didn't timeout so we cancel our idle timer, + * it will be scheduled again if needed. + */ + WARN_ON_ONCE(in_interrupt()); + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; + cancel_delayed_work_sync(&sc->idle.timer_work); + + if (event->status || event->event != sc->rdma.expected_event) { + int lvl = SMBDIRECT_LOG_ERR; + + ret = -ECONNABORTED; + + if (event->event == RDMA_CM_EVENT_REJECTED) + ret = -ECONNREFUSED; + if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) + ret = -ENETDOWN; + if (IS_ERR(SMBDIRECT_DEBUG_ERR_PTR(event->status))) + ret = event->status; + + if (ret == -ENODEV) + lvl = SMBDIRECT_LOG_INFO; + + smbdirect_log_rdma_event(sc, lvl, + "%s (first_error=%1pe, expected=%s) => event=%s status=%d => ret=%1pe\n", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), + rdma_event_msg(sc->rdma.expected_event), + rdma_event_msg(event->event), + event->status, + SMBDIRECT_DEBUG_ERR_PTR(ret)); + + smbdirect_socket_schedule_cleanup_lvl(sc, + lvl, + ret); + return 0; + } + + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "%s (first_error=%1pe) event=%s\n", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), + rdma_event_msg(event->event)); + + if (sc->first_error) + return 0; + + switch (event->event) { + case RDMA_CM_EVENT_ADDR_RESOLVED: + if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING)) + return 0; + sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED; + + ret = smbdirect_connect_resolve_route(sc); + if (ret) + smbdirect_socket_schedule_cleanup(sc, ret); + return 0; + + case RDMA_CM_EVENT_ROUTE_RESOLVED: + if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING)) + return 0; + sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED; + + ret = smbdirect_connect_rdma_connect(sc); + if (ret) + smbdirect_socket_schedule_cleanup(sc, ret); + return 0; + + case RDMA_CM_EVENT_ESTABLISHED: + smbdirect_connection_rdma_established(sc); + + if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING)) + return 0; + sc->status = SMBDIRECT_SOCKET_NEGOTIATE_NEEDED; + + /* + * Here we work around an inconsistency between + * iWarp and other devices (at least rxe and irdma using RoCEv2) + */ + if (rdma_protocol_iwarp(id->device, id->port_num)) { + /* + * iWarp devices report the peer's values + * with the perspective of the peer here. + * Tested with siw and irdma (in iwarp mode) + * We need to change to our perspective here, + * so we need to switch the values. + */ + peer_initiator_depth = event->param.conn.responder_resources; + peer_responder_resources = event->param.conn.initiator_depth; + } else { + /* + * Non iWarp devices report the peer's values + * already changed to our perspective here. + * Tested with rxe and irdma (in roce mode). + */ + peer_initiator_depth = event->param.conn.initiator_depth; + peer_responder_resources = event->param.conn.responder_resources; + } + smbdirect_connection_negotiate_rdma_resources(sc, + peer_initiator_depth, + peer_responder_resources, + &event->param.conn); + + ret = smbdirect_connect_negotiate_start(sc); + if (ret) + smbdirect_socket_schedule_cleanup(sc, ret); + return 0; + + default: + break; + } + + /* + * This is an internal error + */ + WARN_ON_ONCE(sc->rdma.expected_event != RDMA_CM_EVENT_ESTABLISHED); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return 0; +} + +static int smbdirect_connect_negotiate_start(struct smbdirect_socket *sc) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + struct smbdirect_recv_io *recv_io = NULL; + struct smbdirect_send_io *send_io = NULL; + struct smbdirect_negotiate_req *nreq = NULL; + int ret; + + if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_NEGOTIATE_NEEDED)) + return sc->first_error; + sc->status = SMBDIRECT_SOCKET_NEGOTIATE_RUNNING; + + ret = smbdirect_connection_create_mem_pools(sc); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_create_mem_pools() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto create_mem_pools_failed; + } + + /* + * There is only a single batch credit + */ + atomic_set(&sc->send_io.bcredits.count, 1); + + /* + * Initialize the local credits to post + * IB_WR_SEND[_WITH_INV]. + */ + atomic_set(&sc->send_io.lcredits.count, sp->send_credit_target); + + recv_io = smbdirect_connection_get_recv_io(sc); + if (WARN_ON_ONCE(!recv_io)) { + ret = -EINVAL; + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_get_recv_io() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto get_recv_io_failed; + } + recv_io->cqe.done = smbdirect_connect_negotiate_recv_done; + + send_io = smbdirect_connection_alloc_send_io(sc); + if (IS_ERR(send_io)) { + ret = PTR_ERR(send_io); + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_alloc_send_io() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto alloc_send_io_failed; + } + send_io->cqe.done = smbdirect_connect_negotiate_send_done; + + nreq = (struct smbdirect_negotiate_req *)send_io->packet; + nreq->min_version = cpu_to_le16(SMBDIRECT_V1); + nreq->max_version = cpu_to_le16(SMBDIRECT_V1); + nreq->reserved = 0; + nreq->credits_requested = cpu_to_le16(sp->send_credit_target); + nreq->preferred_send_size = cpu_to_le32(sp->max_send_size); + nreq->max_receive_size = cpu_to_le32(sp->max_recv_size); + nreq->max_fragmented_size = cpu_to_le32(sp->max_fragmented_recv_size); + + smbdirect_log_negotiate(sc, SMBDIRECT_LOG_INFO, + "ReqOut: %s%x, %s%x, %s%u, %s%u, %s%u, %s%u\n", + "MinVersion=0x", + le16_to_cpu(nreq->min_version), + "MaxVersion=0x", + le16_to_cpu(nreq->max_version), + "CreditsRequested=", + le16_to_cpu(nreq->credits_requested), + "PreferredSendSize=", + le32_to_cpu(nreq->preferred_send_size), + "MaxRecvSize=", + le32_to_cpu(nreq->max_receive_size), + "MaxFragmentedSize=", + le32_to_cpu(nreq->max_fragmented_size)); + + send_io->sge[0].addr = ib_dma_map_single(sc->ib.dev, + nreq, + sizeof(*nreq), + DMA_TO_DEVICE); + ret = ib_dma_mapping_error(sc->ib.dev, send_io->sge[0].addr); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "ib_dma_mapping_error() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto dma_mapping_failed; + } + + send_io->sge[0].length = sizeof(*nreq); + send_io->sge[0].lkey = sc->ib.pd->local_dma_lkey; + send_io->num_sge = 1; + + ib_dma_sync_single_for_device(sc->ib.dev, + send_io->sge[0].addr, + send_io->sge[0].length, + DMA_TO_DEVICE); + + smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_INFO, + "sge addr=0x%llx length=%u lkey=0x%x\n", + send_io->sge[0].addr, + send_io->sge[0].length, + send_io->sge[0].lkey); + + /* + * Now post the recv_io buffer in order to get + * the negotiate response + */ + sc->recv_io.expected = SMBDIRECT_EXPECT_NEGOTIATE_REP; + ret = smbdirect_connection_post_recv_io(recv_io); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_post_recv_io() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto post_recv_io_failed; + } + + send_io->wr.next = NULL; + send_io->wr.wr_cqe = &send_io->cqe; + send_io->wr.sg_list = send_io->sge; + send_io->wr.num_sge = send_io->num_sge; + send_io->wr.opcode = IB_WR_SEND; + send_io->wr.send_flags = IB_SEND_SIGNALED; + + ret = smbdirect_connection_post_send_wr(sc, &send_io->wr); + if (ret) { + /* if we reach here, post send failed */ + smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_post_send_wr() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto post_send_wr_failed; + } + + /* + * start with the negotiate timeout and SMBDIRECT_KEEPALIVE_PENDING + * so that the timer will cause a disconnect. + */ + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING; + mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work, + msecs_to_jiffies(sp->negotiate_timeout_msec)); + + return 0; + +post_send_wr_failed: + /* + * ib_dma_unmap_single is called in + * smbdirect_connection_free_send_io() + */ + smbdirect_connection_free_send_io(send_io); + /* + * recv_io is given to the rdma layer, + * we should not put it even on error + * nor call smbdirect_connection_destroy_mem_pools() + * it will be cleaned up during disconnect. + */ + return ret; + +post_recv_io_failed: + /* + * ib_dma_unmap_single is called in + * smbdirect_connection_free_send_io() + */ +dma_mapping_failed: + smbdirect_connection_free_send_io(send_io); + +alloc_send_io_failed: + smbdirect_connection_put_recv_io(recv_io); + +get_recv_io_failed: + smbdirect_connection_destroy_mem_pools(sc); + +create_mem_pools_failed: + return ret; +} + +static void smbdirect_connect_negotiate_send_done(struct ib_cq *cq, struct ib_wc *wc) +{ + struct smbdirect_send_io *send_io = + container_of(wc->wr_cqe, struct smbdirect_send_io, cqe); + struct smbdirect_socket *sc = send_io->socket; + + smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_INFO, + "smbdirect_send_io completed. status='%s (%d)', opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + + /* Note this frees wc->wr_cqe, but not wc */ + smbdirect_connection_free_send_io(send_io); + atomic_dec(&sc->send_io.pending.count); + + if (unlikely(wc->status != IB_WC_SUCCESS || WARN_ON_ONCE(wc->opcode != IB_WC_SEND))) { + if (wc->status != IB_WC_WR_FLUSH_ERR) + smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR, + "wc->status=%s (%d) wc->opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } +} + +static void smbdirect_connect_negotiate_recv_work(struct work_struct *work); + +static void smbdirect_connect_negotiate_recv_done(struct ib_cq *cq, struct ib_wc *wc) +{ + struct smbdirect_recv_io *recv_io = + container_of(wc->wr_cqe, struct smbdirect_recv_io, cqe); + struct smbdirect_socket *sc = recv_io->socket; + unsigned long flags; + + if (unlikely(wc->status != IB_WC_SUCCESS || WARN_ON_ONCE(wc->opcode != IB_WC_RECV))) { + if (wc->status != IB_WC_WR_FLUSH_ERR) + smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_ERR, + "wc->status=%s (%d) wc->opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + goto error; + } + + smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_INFO, + "smbdirect_recv_io completed. status='%s (%d)', opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + + /* + * This is an internal error! + */ + if (WARN_ON_ONCE(sc->recv_io.expected != SMBDIRECT_EXPECT_NEGOTIATE_REP)) + goto error; + + /* + * Don't reset timer to the keepalive interval in + * this will be done in smbdirect_accept_direct_negotiate_recv_work. + */ + + ib_dma_sync_single_for_cpu(sc->ib.dev, + recv_io->sge.addr, + recv_io->sge.length, + DMA_FROM_DEVICE); + + /* + * Only remember recv_io if it has enough bytes, + * this gives smbdirect_accept_negotiate_recv_work enough + * information in order to disconnect if it was not + * valid. + */ + sc->recv_io.reassembly.full_packet_received = true; + if (wc->byte_len >= sizeof(struct smbdirect_negotiate_resp)) + smbdirect_connection_reassembly_append_recv_io(sc, recv_io, 0); + else + smbdirect_connection_put_recv_io(recv_io); + + /* + * We continue via the workqueue as we may have + * complex work that might sleep. + * + * So we defer further processing of the negotiation + * to smbdirect_connect_negotiate_recv_work(). + */ + spin_lock_irqsave(&sc->connect.lock, flags); + if (!sc->first_error) { + INIT_WORK(&sc->connect.work, smbdirect_connect_negotiate_recv_work); + if (sc->status == SMBDIRECT_SOCKET_NEGOTIATE_RUNNING) + queue_work(sc->workqueues.connect, &sc->connect.work); + } + spin_unlock_irqrestore(&sc->connect.lock, flags); + + return; + +error: + /* + * recv_io.posted.refill_work is still disabled, + * so smbdirect_connection_put_recv_io() won't + * start it. + */ + smbdirect_connection_put_recv_io(recv_io); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); +} + +static void smbdirect_connect_negotiate_recv_work(struct work_struct *work) +{ + struct smbdirect_socket *sc = + container_of(work, struct smbdirect_socket, connect.work); + struct smbdirect_socket_parameters *sp = &sc->parameters; + struct smbdirect_recv_io *recv_io; + struct smbdirect_negotiate_resp *nrep; + unsigned long flags; + u16 negotiated_version; + u16 credits_requested; + u16 credits_granted; + u32 status; + u32 max_readwrite_size; + u32 preferred_send_size; + u32 max_receive_size; + u32 max_fragmented_size; + int posted; + int ret; + + if (sc->first_error) + return; + + /* + * make sure we won't start again... + */ + disable_work(work); + + /* + * Reset timer to the keepalive interval in + * order to trigger our next keepalive message. + */ + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; + mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work, + msecs_to_jiffies(sp->keepalive_interval_msec)); + + /* + * If smbdirect_connect_negotiate_recv_done() detected an + * invalid request we want to disconnect. + */ + recv_io = smbdirect_connection_reassembly_first_recv_io(sc); + if (!recv_io) { + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); + sc->recv_io.reassembly.queue_length--; + list_del(&recv_io->list); + spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); + smbdirect_connection_put_recv_io(recv_io); + + if (SMBDIRECT_CHECK_STATUS_DISCONNECT(sc, SMBDIRECT_SOCKET_NEGOTIATE_RUNNING)) + return; + + /* + * Note recv_io is already part of the free list, + * as we just called smbdirect_connection_put_recv_io(), + * but it won't be reused before we call + * smbdirect_connection_recv_io_refill() below. + */ + + nrep = (struct smbdirect_negotiate_resp *)recv_io->packet; + negotiated_version = le16_to_cpu(nrep->negotiated_version); + credits_requested = le16_to_cpu(nrep->credits_requested); + credits_granted = le16_to_cpu(nrep->credits_granted); + status = le32_to_cpu(nrep->status); + max_readwrite_size = le32_to_cpu(nrep->max_readwrite_size); + preferred_send_size = le32_to_cpu(nrep->preferred_send_size); + max_receive_size = le32_to_cpu(nrep->max_receive_size); + max_fragmented_size = le32_to_cpu(nrep->max_fragmented_size); + + smbdirect_log_negotiate(sc, SMBDIRECT_LOG_INFO, + "RepIn: %s%x, %s%x, %s%x, %s%u, %s%u, %s%x, %s%u, %s%u, %s%u, %s%u\n", + "MinVersion=0x", + le16_to_cpu(nrep->min_version), + "MaxVersion=0x", + le16_to_cpu(nrep->max_version), + "NegotiatedVersion=0x", + le16_to_cpu(nrep->negotiated_version), + "CreditsRequested=", + le16_to_cpu(nrep->credits_requested), + "CreditsGranted=", + le16_to_cpu(nrep->credits_granted), + "Status=0x", + le32_to_cpu(nrep->status), + "MaxReadWriteSize=", + le32_to_cpu(nrep->max_readwrite_size), + "PreferredSendSize=", + le32_to_cpu(nrep->preferred_send_size), + "MaxRecvSize=", + le32_to_cpu(nrep->max_receive_size), + "MaxFragmentedSize=", + le32_to_cpu(nrep->max_fragmented_size)); + + if (negotiated_version != SMBDIRECT_V1) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: negotiated_version=0x%x\n", + negotiated_version); + smbdirect_socket_schedule_cleanup(sc, -ECONNREFUSED); + return; + } + + if (status != le32_to_cpu(STATUS_SUCCESS)) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: status=0x%x != 0x0\n", + status); + smbdirect_socket_schedule_cleanup(sc, -ECONNREFUSED); + return; + } + + if (max_receive_size < SMBDIRECT_MIN_RECEIVE_SIZE) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: max_receive_size=%u < %u\n", + max_receive_size, + SMBDIRECT_MIN_RECEIVE_SIZE); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + + if (max_fragmented_size < SMBDIRECT_MIN_FRAGMENTED_SIZE) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: max_fragmented_size=%u < %u\n", + max_fragmented_size, + SMBDIRECT_MIN_FRAGMENTED_SIZE); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + + if (credits_granted == 0) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: credits_granted == 0\n"); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + + if (credits_requested == 0) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: credits_requested == 0\n"); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + + if (preferred_send_size > sp->max_recv_size) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: preferred_send_size=%u > max_recv_size=%u\n", + preferred_send_size, + sp->max_recv_size); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + + /* + * We take the value from the peer, which is checked to be higher than 0, + * but we limit it to the max value we support in order to have + * the main logic simpler. + */ + sc->recv_io.credits.target = credits_requested; + sc->recv_io.credits.target = min_t(u16, sc->recv_io.credits.target, + sp->recv_credit_max); + + /* + * At least the value of SMBDIRECT_MIN_RECEIVE_SIZE is used. + */ + sp->max_recv_size = min_t(u32, sp->max_recv_size, preferred_send_size); + sp->max_recv_size = max_t(u32, sp->max_recv_size, SMBDIRECT_MIN_RECEIVE_SIZE); + + /* + * We already sent our sp->max_fragmented_recv_size + * to the peer, so we can't lower it here any more. + * + * TODO: but if the peer lowered sp->max_recv_size + * we will have to adjust our number of buffers. + * + * But for now we keep it as the cifs.ko code + * worked before. + */ + + /* + * Note nrep->max_receive_size was already checked against + * SMBDIRECT_MIN_RECEIVE_SIZE above. + */ + sp->max_send_size = min_t(u32, sp->max_send_size, max_receive_size); + + /* + * Make sure the resulting max_frmr_depth is at least 1, + * which means max_read_write_size needs to be at least PAGE_SIZE. + */ + sp->max_read_write_size = min_t(u32, sp->max_frmr_depth * PAGE_SIZE, + max_readwrite_size); + if (sp->max_read_write_size < PAGE_SIZE) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: max_readwrite_size=%u < PAGE_SIZE(%lu)\n", + max_readwrite_size, + PAGE_SIZE); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + sp->max_frmr_depth = sp->max_read_write_size / PAGE_SIZE; + + /* + * Note nrep->credits_granted was already checked against 0 above. + */ + atomic_set(&sc->send_io.credits.count, credits_granted); + + /* + * Note nrep->max_fragmented_size was already checked against + * SMBDIRECT_MIN_FRAGMENTED_SIZE above. + */ + sp->max_fragmented_send_size = max_fragmented_size; + + ret = smbdirect_connection_create_mr_list(sc); + if (ret) { + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_create_mr_list() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + smbdirect_socket_schedule_cleanup(sc, ret); + return; + } + + /* + * Prepare for receiving data_transfer messages + */ + sc->recv_io.reassembly.full_packet_received = true; + sc->recv_io.expected = SMBDIRECT_EXPECT_DATA_TRANSFER; + list_for_each_entry(recv_io, &sc->recv_io.free.list, list) + recv_io->cqe.done = smbdirect_connection_recv_io_done; + recv_io = NULL; + + /* + * We should at least post 1 smbdirect_recv_io! + */ + posted = smbdirect_connection_recv_io_refill(sc); + if (posted < 1) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_recv_io_refill() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + + /* + * smbdirect_connection_negotiation_done() + * will setup all required things and wake up + * the waiter. + */ + smbdirect_connection_negotiation_done(sc); +} + +int smbdirect_connect_sync(struct smbdirect_socket *sc, + const struct sockaddr *dst) +{ + int ret; + + ret = smbdirect_connect(sc, dst); + if (ret) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connect(%pISpsfc) failed %1pe\n", + dst, SMBDIRECT_DEBUG_ERR_PTR(ret)); + return ret; + } + + ret = smbdirect_connection_wait_for_connected(sc); + if (ret) { + int lvl = SMBDIRECT_LOG_ERR; + + if (ret == -ENODEV) + lvl = SMBDIRECT_LOG_INFO; + + smbdirect_log_rdma_event(sc, lvl, + "wait for smbdirect_connect(%pISpsfc) failed %1pe\n", + dst, SMBDIRECT_DEBUG_ERR_PTR(ret)); + return ret; + } + + return 0; +} +EXPORT_SYMBOL_GPL(smbdirect_connect_sync); diff --git a/fs/smb/smbdirect/connection.c b/fs/smb/smbdirect/connection.c new file mode 100644 index 000000000..afd31fa12 --- /dev/null +++ b/fs/smb/smbdirect/connection.c @@ -0,0 +1,2190 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017, Microsoft Corporation. + * Copyright (c) 2025, Stefan Metzmacher + */ + +#include "internal.h" +#include + +struct smbdirect_map_sges { + struct ib_sge *sge; + size_t num_sge; + size_t max_sge; + struct ib_device *device; + u32 local_dma_lkey; + enum dma_data_direction direction; +}; + +static ssize_t smbdirect_map_sges_from_iter(struct iov_iter *iter, size_t len, + struct smbdirect_map_sges *state); + +static void smbdirect_connection_recv_io_refill_work(struct work_struct *work); +static void smbdirect_connection_send_immediate_work(struct work_struct *work); + +static void smbdirect_connection_qp_event_handler(struct ib_event *event, void *context) +{ + struct smbdirect_socket *sc = context; + + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "%s on device %.*s socket %p (cm_id=%p) status %s first_error %1pe\n", + ib_event_msg(event->event), + IB_DEVICE_NAME_MAX, + event->device->name, + sc, sc->rdma.cm_id, + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); + + switch (event->event) { + case IB_EVENT_CQ_ERR: + case IB_EVENT_QP_FATAL: + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + break; + + default: + break; + } +} + +static int smbdirect_connection_rdma_event_handler(struct rdma_cm_id *id, + struct rdma_cm_event *event) +{ + struct smbdirect_socket *sc = id->context; + int ret = -ECONNRESET; + + if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) + ret = -ENETDOWN; + if (IS_ERR(SMBDIRECT_DEBUG_ERR_PTR(event->status))) + ret = event->status; + + /* + * cma_cm_event_handler() has + * lockdep_assert_held(&id_priv->handler_mutex); + * + * Mutexes are not allowed in interrupts, + * and we rely on not being in an interrupt here. + */ + WARN_ON_ONCE(in_interrupt()); + + if (event->event != sc->rdma.expected_event) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "%s (first_error=%1pe, expected=%s) => event=%s status=%d => ret=%1pe\n", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), + rdma_event_msg(sc->rdma.expected_event), + rdma_event_msg(event->event), + event->status, + SMBDIRECT_DEBUG_ERR_PTR(ret)); + + /* + * If we get RDMA_CM_EVENT_DEVICE_REMOVAL, + * we should change to SMBDIRECT_SOCKET_DISCONNECTED, + * so that rdma_disconnect() is avoided later via + * smbdirect_socket_schedule_cleanup[_status]() => + * smbdirect_socket_cleanup_work(). + * + * As otherwise we'd set SMBDIRECT_SOCKET_DISCONNECTING, + * but never ever get RDMA_CM_EVENT_DISCONNECTED and + * never reach SMBDIRECT_SOCKET_DISCONNECTED. + */ + if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) + smbdirect_socket_schedule_cleanup_status(sc, + SMBDIRECT_LOG_ERR, + ret, + SMBDIRECT_SOCKET_DISCONNECTED); + else + smbdirect_socket_schedule_cleanup(sc, ret); + if (sc->ib.qp) + ib_drain_qp(sc->ib.qp); + return 0; + } + + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "%s (first_error=%1pe) event=%s\n", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), + rdma_event_msg(event->event)); + + switch (event->event) { + case RDMA_CM_EVENT_DISCONNECTED: + /* + * We need to change to SMBDIRECT_SOCKET_DISCONNECTED, + * so that rdma_disconnect() is avoided later via + * smbdirect_socket_schedule_cleanup_status() => + * smbdirect_socket_cleanup_work(). + * + * As otherwise we'd set SMBDIRECT_SOCKET_DISCONNECTING, + * but never ever get RDMA_CM_EVENT_DISCONNECTED and + * never reach SMBDIRECT_SOCKET_DISCONNECTED. + * + * This is also a normal disconnect so + * SMBDIRECT_LOG_INFO should be good enough + * and avoids spamming the default logs. + */ + smbdirect_socket_schedule_cleanup_status(sc, + SMBDIRECT_LOG_INFO, + ret, + SMBDIRECT_SOCKET_DISCONNECTED); + if (sc->ib.qp) + ib_drain_qp(sc->ib.qp); + return 0; + + default: + break; + } + + /* + * This is an internal error, should be handled above via + * event->event != sc->rdma.expected_event already. + */ + WARN_ON_ONCE(sc->rdma.expected_event != RDMA_CM_EVENT_DISCONNECTED); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return 0; +} + +void smbdirect_connection_rdma_established(struct smbdirect_socket *sc) +{ + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "rdma established: device: %.*s local: %pISpsfc remote: %pISpsfc\n", + IB_DEVICE_NAME_MAX, + sc->ib.dev->name, + &sc->rdma.cm_id->route.addr.src_addr, + &sc->rdma.cm_id->route.addr.dst_addr); + + sc->rdma.cm_id->event_handler = smbdirect_connection_rdma_event_handler; + sc->rdma.expected_event = RDMA_CM_EVENT_DISCONNECTED; +} + +void smbdirect_connection_negotiation_done(struct smbdirect_socket *sc) +{ + if (unlikely(sc->first_error)) + return; + + if (sc->status == SMBDIRECT_SOCKET_CONNECTED) + /* + * This is the accept case where + * smbdirect_socket_accept() already sets + * SMBDIRECT_SOCKET_CONNECTED + */ + goto done; + + if (sc->status != SMBDIRECT_SOCKET_NEGOTIATE_RUNNING) { + /* + * Something went wrong... + */ + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "status=%s first_error=%1pe local: %pISpsfc remote: %pISpsfc\n", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), + &sc->rdma.cm_id->route.addr.src_addr, + &sc->rdma.cm_id->route.addr.dst_addr); + return; + } + + /* + * We are done, so we can wake up the waiter. + */ + WARN_ONCE(sc->status == SMBDIRECT_SOCKET_CONNECTED, + "status=%s first_error=%1pe", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); + sc->status = SMBDIRECT_SOCKET_CONNECTED; + + /* + * We need to setup the refill and send immediate work + * in order to get a working connection. + */ +done: + INIT_WORK(&sc->recv_io.posted.refill_work, smbdirect_connection_recv_io_refill_work); + INIT_WORK(&sc->idle.immediate_work, smbdirect_connection_send_immediate_work); + + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "negotiated: local: %pISpsfc remote: %pISpsfc\n", + &sc->rdma.cm_id->route.addr.src_addr, + &sc->rdma.cm_id->route.addr.dst_addr); + + wake_up(&sc->status_wait); +} + +static u32 smbdirect_rdma_rw_send_wrs(struct ib_device *dev, + const struct ib_qp_init_attr *attr) +{ + /* + * This could be split out of rdma_rw_init_qp() + * and be a helper function next to rdma_rw_mr_factor() + * + * We can't check unlikely(rdma_rw_force_mr) here, + * but that is most likely 0 anyway. + */ + u32 factor; + + WARN_ON_ONCE(attr->port_num == 0); + + /* + * Each context needs at least one RDMA READ or WRITE WR. + * + * For some hardware we might need more, eventually we should ask the + * HCA driver for a multiplier here. + */ + factor = 1; + + /* + * If the device needs MRs to perform RDMA READ or WRITE operations, + * we'll need two additional MRs for the registrations and the + * invalidation. + */ + if (rdma_protocol_iwarp(dev, attr->port_num) || dev->attrs.max_sgl_rd) + factor += 2; /* inv + reg */ + + return factor * attr->cap.max_rdma_ctxs; +} + +int smbdirect_connection_create_qp(struct smbdirect_socket *sc) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + struct ib_qp_init_attr qp_attr; + struct ib_qp_cap qp_cap; + u32 rdma_send_wr; + u32 max_send_wr; + int ret; + + /* + * Note that {rdma,ib}_create_qp() will call + * rdma_rw_init_qp() if max_rdma_ctxs is not 0. + * It will adjust max_send_wr to the required + * number of additional WRs for the RDMA RW operations. + * It will cap max_send_wr to the device limit. + * + * We use allocate sp->responder_resources * 2 MRs + * and each MR needs WRs for REG and INV, so + * we use '* 4'. + * + * +1 for ib_drain_qp() + */ + memset(&qp_cap, 0, sizeof(qp_cap)); + qp_cap.max_send_wr = sp->send_credit_target + sp->responder_resources * 4 + 1; + qp_cap.max_recv_wr = sp->recv_credit_max + 1; + qp_cap.max_send_sge = SMBDIRECT_SEND_IO_MAX_SGE; + qp_cap.max_recv_sge = SMBDIRECT_RECV_IO_MAX_SGE; + qp_cap.max_inline_data = 0; + qp_cap.max_rdma_ctxs = sc->rw_io.credits.max; + + /* + * Find out the number of max_send_wr + * after rdma_rw_init_qp() adjusted it. + * + * We only do it on a temporary variable, + * as rdma_create_qp() will trigger + * rdma_rw_init_qp() again. + */ + memset(&qp_attr, 0, sizeof(qp_attr)); + qp_attr.cap = qp_cap; + qp_attr.port_num = sc->rdma.cm_id->port_num; + rdma_send_wr = smbdirect_rdma_rw_send_wrs(sc->ib.dev, &qp_attr); + max_send_wr = qp_cap.max_send_wr + rdma_send_wr; + + if (qp_cap.max_send_wr > sc->ib.dev->attrs.max_cqe || + qp_cap.max_send_wr > sc->ib.dev->attrs.max_qp_wr) { + pr_err("Possible CQE overrun: max_send_wr %d\n", + qp_cap.max_send_wr); + pr_err("device %.*s reporting max_cqe %u max_qp_wr %u\n", + IB_DEVICE_NAME_MAX, + sc->ib.dev->name, + sc->ib.dev->attrs.max_cqe, + sc->ib.dev->attrs.max_qp_wr); + pr_err("consider lowering send_credit_target = %d\n", + sp->send_credit_target); + return -EINVAL; + } + + if (qp_cap.max_rdma_ctxs && + (max_send_wr >= sc->ib.dev->attrs.max_cqe || + max_send_wr >= sc->ib.dev->attrs.max_qp_wr)) { + pr_err("Possible CQE overrun: rdma_send_wr %d + max_send_wr %d = %d\n", + rdma_send_wr, qp_cap.max_send_wr, max_send_wr); + pr_err("device %.*s reporting max_cqe %u max_qp_wr %u\n", + IB_DEVICE_NAME_MAX, + sc->ib.dev->name, + sc->ib.dev->attrs.max_cqe, + sc->ib.dev->attrs.max_qp_wr); + pr_err("consider lowering send_credit_target = %d, max_rdma_ctxs = %d\n", + sp->send_credit_target, qp_cap.max_rdma_ctxs); + return -EINVAL; + } + + if (qp_cap.max_recv_wr > sc->ib.dev->attrs.max_cqe || + qp_cap.max_recv_wr > sc->ib.dev->attrs.max_qp_wr) { + pr_err("Possible CQE overrun: max_recv_wr %d\n", + qp_cap.max_recv_wr); + pr_err("device %.*s reporting max_cqe %u max_qp_wr %u\n", + IB_DEVICE_NAME_MAX, + sc->ib.dev->name, + sc->ib.dev->attrs.max_cqe, + sc->ib.dev->attrs.max_qp_wr); + pr_err("consider lowering receive_credit_max = %d\n", + sp->recv_credit_max); + return -EINVAL; + } + + if (qp_cap.max_send_sge > sc->ib.dev->attrs.max_send_sge || + qp_cap.max_recv_sge > sc->ib.dev->attrs.max_recv_sge) { + pr_err("device %.*s max_send_sge/max_recv_sge = %u/%u too small\n", + IB_DEVICE_NAME_MAX, + sc->ib.dev->name, + sc->ib.dev->attrs.max_send_sge, + sc->ib.dev->attrs.max_recv_sge); + return -EINVAL; + } + + sc->ib.pd = ib_alloc_pd(sc->ib.dev, 0); + if (IS_ERR(sc->ib.pd)) { + pr_err("Can't create RDMA PD: %1pe\n", sc->ib.pd); + ret = PTR_ERR(sc->ib.pd); + sc->ib.pd = NULL; + return ret; + } + + sc->ib.send_cq = ib_alloc_cq_any(sc->ib.dev, sc, + max_send_wr, + sc->ib.poll_ctx); + if (IS_ERR(sc->ib.send_cq)) { + pr_err("Can't create RDMA send CQ: %1pe\n", sc->ib.send_cq); + ret = PTR_ERR(sc->ib.send_cq); + sc->ib.send_cq = NULL; + goto err; + } + + sc->ib.recv_cq = ib_alloc_cq_any(sc->ib.dev, sc, + qp_cap.max_recv_wr, + sc->ib.poll_ctx); + if (IS_ERR(sc->ib.recv_cq)) { + pr_err("Can't create RDMA recv CQ: %1pe\n", sc->ib.recv_cq); + ret = PTR_ERR(sc->ib.recv_cq); + sc->ib.recv_cq = NULL; + goto err; + } + + /* + * We reset completely here! + * As the above use was just temporary + * to calc max_send_wr and rdma_send_wr. + * + * rdma_create_qp() will trigger rdma_rw_init_qp() + * again if max_rdma_ctxs is not 0. + */ + memset(&qp_attr, 0, sizeof(qp_attr)); + qp_attr.event_handler = smbdirect_connection_qp_event_handler; + qp_attr.qp_context = sc; + qp_attr.cap = qp_cap; + qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR; + qp_attr.qp_type = IB_QPT_RC; + qp_attr.send_cq = sc->ib.send_cq; + qp_attr.recv_cq = sc->ib.recv_cq; + qp_attr.port_num = ~0; + + ret = rdma_create_qp(sc->rdma.cm_id, sc->ib.pd, &qp_attr); + if (ret) { + pr_err("Can't create RDMA QP: %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + goto err; + } + sc->ib.qp = sc->rdma.cm_id->qp; + + return 0; +err: + smbdirect_connection_destroy_qp(sc); + return ret; +} + +void smbdirect_connection_destroy_qp(struct smbdirect_socket *sc) +{ + if (sc->ib.qp) { + ib_drain_qp(sc->ib.qp); + sc->ib.qp = NULL; + rdma_destroy_qp(sc->rdma.cm_id); + } + /* + * These CQs were created with ib_alloc_cq_any(), which arms an internal + * completion handler (ib_cq_poll_work for IB_POLL_WORKQUEUE). They MUST be + * torn down with ib_free_cq(), which cancel_work_sync()es that poll work + * before freeing the CQ. ib_destroy_cq() skips that step, so a completion + * posted late by the (software) provider — e.g. rxe posting an RNR error + * from rxe_receiver after rdma_destroy_qp() — re-queues ib_cq_poll_work on + * an already-freed CQ (KASAN slab-use-after-free in ib_cq_poll_work). + */ + if (sc->ib.recv_cq) { + ib_free_cq(sc->ib.recv_cq); + sc->ib.recv_cq = NULL; + } + if (sc->ib.send_cq) { + ib_free_cq(sc->ib.send_cq); + sc->ib.send_cq = NULL; + } + if (sc->ib.pd) { + ib_dealloc_pd(sc->ib.pd); + sc->ib.pd = NULL; + } +} + +int smbdirect_connection_create_mem_pools(struct smbdirect_socket *sc) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + char name[80]; + size_t i; + + /* + * We use sizeof(struct smbdirect_negotiate_resp) for the + * payload size as it is larger as + * sizeof(struct smbdirect_data_transfer). + * + * This will fit client and server usage for now. + */ + snprintf(name, sizeof(name), "smbdirect_send_io_cache_%p", sc); + struct kmem_cache_args send_io_args = { + .align = __alignof__(struct smbdirect_send_io), + }; + sc->send_io.mem.cache = kmem_cache_create(name, + sizeof(struct smbdirect_send_io) + + sizeof(struct smbdirect_negotiate_resp), + &send_io_args, + SLAB_HWCACHE_ALIGN); + if (!sc->send_io.mem.cache) + goto err; + + sc->send_io.mem.pool = mempool_create_slab_pool(sp->send_credit_target, + sc->send_io.mem.cache); + if (!sc->send_io.mem.pool) + goto err; + + /* + * A payload size of sp->max_recv_size should fit + * any message. + * + * For smbdirect_data_transfer messages the whole + * buffer might be exposed to userspace + * (currently on the client side...) + * The documentation says data_offset = 0 would be + * strange but valid. + */ + snprintf(name, sizeof(name), "smbdirect_recv_io_cache_%p", sc); + struct kmem_cache_args recv_io_args = { + .align = __alignof__(struct smbdirect_recv_io), + .useroffset = sizeof(struct smbdirect_recv_io), + .usersize = sp->max_recv_size, + }; + sc->recv_io.mem.cache = kmem_cache_create(name, + sizeof(struct smbdirect_recv_io) + + sp->max_recv_size, + &recv_io_args, + SLAB_HWCACHE_ALIGN); + if (!sc->recv_io.mem.cache) + goto err; + + sc->recv_io.mem.pool = mempool_create_slab_pool(sp->recv_credit_max, + sc->recv_io.mem.cache); + if (!sc->recv_io.mem.pool) + goto err; + + for (i = 0; i < sp->recv_credit_max; i++) { + struct smbdirect_recv_io *recv_io; + + recv_io = mempool_alloc(sc->recv_io.mem.pool, + sc->recv_io.mem.gfp_mask); + if (!recv_io) + goto err; + recv_io->socket = sc; + recv_io->sge.length = 0; + list_add_tail(&recv_io->list, &sc->recv_io.free.list); + } + + return 0; +err: + smbdirect_connection_destroy_mem_pools(sc); + return -ENOMEM; +} + +void smbdirect_connection_destroy_mem_pools(struct smbdirect_socket *sc) +{ + struct smbdirect_recv_io *recv_io, *next_io; + + list_for_each_entry_safe(recv_io, next_io, &sc->recv_io.free.list, list) { + list_del(&recv_io->list); + mempool_free(recv_io, sc->recv_io.mem.pool); + } + + /* + * Note mempool_destroy() and kmem_cache_destroy() + * work fine with a NULL pointer + */ + + mempool_destroy(sc->recv_io.mem.pool); + sc->recv_io.mem.pool = NULL; + + kmem_cache_destroy(sc->recv_io.mem.cache); + sc->recv_io.mem.cache = NULL; + + mempool_destroy(sc->send_io.mem.pool); + sc->send_io.mem.pool = NULL; + + kmem_cache_destroy(sc->send_io.mem.cache); + sc->send_io.mem.cache = NULL; +} + +struct smbdirect_send_io *smbdirect_connection_alloc_send_io(struct smbdirect_socket *sc) +{ + struct smbdirect_send_io *msg; + + msg = mempool_alloc(sc->send_io.mem.pool, sc->send_io.mem.gfp_mask); + if (!msg) + return ERR_PTR(-ENOMEM); + msg->socket = sc; + INIT_LIST_HEAD(&msg->sibling_list); + msg->num_sge = 0; + + return msg; +} + +void smbdirect_connection_free_send_io(struct smbdirect_send_io *msg) +{ + struct smbdirect_socket *sc = msg->socket; + size_t i; + + /* + * The list needs to be empty! + * The caller should take care of it. + */ + WARN_ON_ONCE(!list_empty(&msg->sibling_list)); + + /* + * Note we call ib_dma_unmap_page(), even if some sges are mapped using + * ib_dma_map_single(). + * + * The difference between _single() and _page() only matters for the + * ib_dma_map_*() case. + * + * For the ib_dma_unmap_*() case it does not matter as both take the + * dma_addr_t and dma_unmap_single_attrs() is just an alias to + * dma_unmap_page_attrs(). + */ + for (i = 0; i < msg->num_sge; i++) + ib_dma_unmap_page(sc->ib.dev, + msg->sge[i].addr, + msg->sge[i].length, + DMA_TO_DEVICE); + + mempool_free(msg, sc->send_io.mem.pool); +} + +struct smbdirect_recv_io *smbdirect_connection_get_recv_io(struct smbdirect_socket *sc) +{ + struct smbdirect_recv_io *msg = NULL; + unsigned long flags; + + spin_lock_irqsave(&sc->recv_io.free.lock, flags); + if (likely(!sc->first_error)) + msg = list_first_entry_or_null(&sc->recv_io.free.list, + struct smbdirect_recv_io, + list); + if (likely(msg)) { + list_del(&msg->list); + sc->statistics.get_receive_buffer++; + } + spin_unlock_irqrestore(&sc->recv_io.free.lock, flags); + + return msg; +} + +void smbdirect_connection_put_recv_io(struct smbdirect_recv_io *msg) +{ + struct smbdirect_socket *sc = msg->socket; + unsigned long flags; + + if (likely(msg->sge.length != 0)) { + ib_dma_unmap_single(sc->ib.dev, + msg->sge.addr, + msg->sge.length, + DMA_FROM_DEVICE); + msg->sge.length = 0; + } + + spin_lock_irqsave(&sc->recv_io.free.lock, flags); + list_add_tail(&msg->list, &sc->recv_io.free.list); + sc->statistics.put_receive_buffer++; + spin_unlock_irqrestore(&sc->recv_io.free.lock, flags); + + queue_work(sc->workqueues.refill, &sc->recv_io.posted.refill_work); +} + +void smbdirect_connection_reassembly_append_recv_io(struct smbdirect_socket *sc, + struct smbdirect_recv_io *msg, + u32 data_length) +{ + unsigned long flags; + + spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); + list_add_tail(&msg->list, &sc->recv_io.reassembly.list); + sc->recv_io.reassembly.queue_length++; + /* + * Make sure reassembly_data_length is updated after list and + * reassembly_queue_length are updated. On the dequeue side + * reassembly_data_length is checked without a lock to determine + * if reassembly_queue_length and list is up to date + */ + virt_wmb(); + sc->recv_io.reassembly.data_length += data_length; + spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); + sc->statistics.enqueue_reassembly_queue++; +} + +struct smbdirect_recv_io * +smbdirect_connection_reassembly_first_recv_io(struct smbdirect_socket *sc) +{ + struct smbdirect_recv_io *msg; + + msg = list_first_entry_or_null(&sc->recv_io.reassembly.list, + struct smbdirect_recv_io, + list); + + return msg; +} + +void smbdirect_connection_negotiate_rdma_resources(struct smbdirect_socket *sc, + u8 peer_initiator_depth, + u8 peer_responder_resources, + const struct rdma_conn_param *param) +{ + struct smbdirect_socket_parameters *sp = &sc->parameters; + + if (rdma_protocol_iwarp(sc->ib.dev, sc->rdma.cm_id->port_num) && + param->private_data_len == 8) { + /* + * Legacy clients with only iWarp MPA v1 support + * need a private blob in order to negotiate + * the IRD/ORD values. + */ + const __be32 *ird_ord_hdr = param->private_data; + u32 ird32 = be32_to_cpu(ird_ord_hdr[0]); + u32 ord32 = be32_to_cpu(ird_ord_hdr[1]); + + /* + * cifs.ko sends the legacy IRD/ORD negotiation + * event if iWarp MPA v2 was used. + * + * Here we check that the values match and only + * mark the client as legacy if they don't match. + */ + if ((u32)param->initiator_depth != ird32 || + (u32)param->responder_resources != ord32) { + /* + * There are broken clients (old cifs.ko) + * using little endian and also + * struct rdma_conn_param only uses u8 + * for initiator_depth and responder_resources, + * so we truncate the value to U8_MAX. + * + * smb_direct_accept_client() will then + * do the real negotiation in order to + * select the minimum between client and + * server. + */ + ird32 = min_t(u32, ird32, U8_MAX); + ord32 = min_t(u32, ord32, U8_MAX); + + sc->rdma.legacy_iwarp = true; + peer_initiator_depth = (u8)ird32; + peer_responder_resources = (u8)ord32; + } + } + + /* + * negotiate the value by using the minimum + * between client and server if the client provided + * non 0 values. + */ + if (peer_initiator_depth != 0) + sp->initiator_depth = min_t(u8, sp->initiator_depth, + peer_initiator_depth); + if (peer_responder_resources != 0) + sp->responder_resources = min_t(u8, sp->responder_resources, + peer_responder_resources); +} + +bool smbdirect_connection_is_connected(struct smbdirect_socket *sc) +{ + if (unlikely(!sc || sc->first_error || sc->status != SMBDIRECT_SOCKET_CONNECTED)) + return false; + return true; +} +EXPORT_SYMBOL_GPL(smbdirect_connection_is_connected); + +int smbdirect_connection_wait_for_connected(struct smbdirect_socket *sc) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + union { + struct sockaddr sa; + struct sockaddr_storage ss; + } src_addr, dst_addr; + const struct sockaddr *src = NULL; + const struct sockaddr *dst = NULL; + char _devname[IB_DEVICE_NAME_MAX] = { 0, }; + const char *devname = NULL; + int ret; + + if (sc->rdma.cm_id) { + src_addr.ss = sc->rdma.cm_id->route.addr.src_addr; + if (src_addr.sa.sa_family != AF_UNSPEC) + src = &src_addr.sa; + dst_addr.ss = sc->rdma.cm_id->route.addr.dst_addr; + if (dst_addr.sa.sa_family != AF_UNSPEC) + dst = &dst_addr.sa; + + if (sc->ib.dev) { + memcpy(_devname, sc->ib.dev->name, IB_DEVICE_NAME_MAX); + devname = _devname; + } + } + + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "waiting for connection: device: %.*s local: %pISpsfc remote: %pISpsfc\n", + IB_DEVICE_NAME_MAX, devname, src, dst); + + ret = wait_event_interruptible_timeout(sc->status_wait, + sc->status == SMBDIRECT_SOCKET_CONNECTED || + sc->first_error, + msecs_to_jiffies(sp->negotiate_timeout_msec)); + if (sc->rdma.cm_id) { + /* + * Maybe src and dev are updated in the meantime. + */ + src_addr.ss = sc->rdma.cm_id->route.addr.src_addr; + if (src_addr.sa.sa_family != AF_UNSPEC) + src = &src_addr.sa; + dst_addr.ss = sc->rdma.cm_id->route.addr.dst_addr; + if (dst_addr.sa.sa_family != AF_UNSPEC) + dst = &dst_addr.sa; + + if (sc->ib.dev) { + memcpy(_devname, sc->ib.dev->name, IB_DEVICE_NAME_MAX); + devname = _devname; + } + } + if (ret == 0) + ret = -ETIMEDOUT; + if (ret < 0) + smbdirect_socket_schedule_cleanup(sc, ret); + if (sc->first_error) { + int lvl = SMBDIRECT_LOG_ERR; + + ret = sc->first_error; + if (ret == -ENODEV) + lvl = SMBDIRECT_LOG_INFO; + + smbdirect_log_rdma_event(sc, lvl, + "connection failed %1pe device: %.*s local: %pISpsfc remote: %pISpsfc\n", + SMBDIRECT_DEBUG_ERR_PTR(ret), + IB_DEVICE_NAME_MAX, devname, src, dst); + return ret; + } + + return 0; +} +EXPORT_SYMBOL_GPL(smbdirect_connection_wait_for_connected); + +void smbdirect_connection_idle_timer_work(struct work_struct *work) +{ + struct smbdirect_socket *sc = + container_of(work, struct smbdirect_socket, idle.timer_work.work); + const struct smbdirect_socket_parameters *sp = &sc->parameters; + + if (sc->idle.keepalive != SMBDIRECT_KEEPALIVE_NONE) { + smbdirect_log_keep_alive(sc, SMBDIRECT_LOG_ERR, + "%s => timeout sc->idle.keepalive=%s\n", + smbdirect_socket_status_string(sc->status), + sc->idle.keepalive == SMBDIRECT_KEEPALIVE_SENT ? + "SENT" : "PENDING"); + smbdirect_socket_schedule_cleanup(sc, -ETIMEDOUT); + return; + } + + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) + return; + + /* + * Now use the keepalive timeout (instead of keepalive interval) + * in order to wait for a response + */ + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_PENDING; + mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work, + msecs_to_jiffies(sp->keepalive_timeout_msec)); + smbdirect_log_keep_alive(sc, SMBDIRECT_LOG_INFO, + "schedule send of empty idle message\n"); + queue_work(sc->workqueues.immediate, &sc->idle.immediate_work); +} + +u16 smbdirect_connection_grant_recv_credits(struct smbdirect_socket *sc) +{ + int missing; + int available; + int new_credits; + + if (atomic_read(&sc->recv_io.credits.count) >= sc->recv_io.credits.target) + return 0; + + missing = (int)sc->recv_io.credits.target - atomic_read(&sc->recv_io.credits.count); + available = atomic_xchg(&sc->recv_io.credits.available, 0); + new_credits = min3((int)U16_MAX, missing, available); + if (new_credits <= 0) { + /* + * If credits are available, but not granted + * we need to re-add them again. + */ + if (available) + atomic_add(available, &sc->recv_io.credits.available); + return 0; + } + + if (new_credits < available) { + /* + * Readd the remaining available again. + */ + available -= new_credits; + atomic_add(available, &sc->recv_io.credits.available); + } + + /* + * Remember we granted the credits + */ + atomic_add(new_credits, &sc->recv_io.credits.count); + return new_credits; +} + +static bool smbdirect_connection_request_keep_alive(struct smbdirect_socket *sc) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + + if (sc->idle.keepalive == SMBDIRECT_KEEPALIVE_PENDING) { + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_SENT; + /* + * Now use the keepalive timeout (instead of keepalive interval) + * in order to wait for a response + */ + mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work, + msecs_to_jiffies(sp->keepalive_timeout_msec)); + return true; + } + + return false; +} + +int smbdirect_connection_post_send_wr(struct smbdirect_socket *sc, + struct ib_send_wr *wr) +{ + int ret; + + if (unlikely(sc->first_error)) + return sc->first_error; + + atomic_inc(&sc->send_io.pending.count); + ret = ib_post_send(sc->ib.qp, wr, NULL); + if (ret) { + atomic_dec(&sc->send_io.pending.count); + smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR, + "ib_post_send() failed %1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + smbdirect_socket_schedule_cleanup(sc, ret); + } + + return ret; +} + +static void smbdirect_connection_send_batch_init(struct smbdirect_send_batch *batch, + bool need_invalidate_rkey, + unsigned int remote_key) +{ + INIT_LIST_HEAD(&batch->msg_list); + batch->wr_cnt = 0; + batch->need_invalidate_rkey = need_invalidate_rkey; + batch->remote_key = remote_key; + batch->credit = 0; +} + +int smbdirect_connection_send_batch_flush(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch, + bool is_last) +{ + struct smbdirect_send_io *first, *last; + int ret = 0; + + if (list_empty(&batch->msg_list)) + goto release_credit; + + first = list_first_entry(&batch->msg_list, + struct smbdirect_send_io, + sibling_list); + last = list_last_entry(&batch->msg_list, + struct smbdirect_send_io, + sibling_list); + + if (batch->need_invalidate_rkey) { + first->wr.opcode = IB_WR_SEND_WITH_INV; + first->wr.ex.invalidate_rkey = batch->remote_key; + batch->need_invalidate_rkey = false; + batch->remote_key = 0; + } + + last->wr.send_flags = IB_SEND_SIGNALED; + last->wr.wr_cqe = &last->cqe; + + /* + * Remove last from send_ctx->msg_list + * and splice the rest of send_ctx->msg_list + * to last->sibling_list. + * + * send_ctx->msg_list is a valid empty list + * at the end. + */ + list_del_init(&last->sibling_list); + list_splice_tail_init(&batch->msg_list, &last->sibling_list); + batch->wr_cnt = 0; + + ret = smbdirect_connection_post_send_wr(sc, &first->wr); + if (ret) { + struct smbdirect_send_io *sibling, *next; + + list_for_each_entry_safe(sibling, next, &last->sibling_list, sibling_list) { + list_del_init(&sibling->sibling_list); + smbdirect_connection_free_send_io(sibling); + } + smbdirect_connection_free_send_io(last); + } + +release_credit: + if (is_last && !ret && batch->credit) { + atomic_add(batch->credit, &sc->send_io.bcredits.count); + batch->credit = 0; + wake_up(&sc->send_io.bcredits.wait_queue); + } + + return ret; +} +EXPORT_SYMBOL_GPL(smbdirect_connection_send_batch_flush); + +struct smbdirect_send_batch * +smbdirect_init_send_batch_storage(struct smbdirect_send_batch_storage *storage, + bool need_invalidate_rkey, + unsigned int remote_key) +{ + struct smbdirect_send_batch *batch = (struct smbdirect_send_batch *)storage; + + memset(storage, 0, sizeof(*storage)); + BUILD_BUG_ON(sizeof(*batch) > sizeof(*storage)); + + smbdirect_connection_send_batch_init(batch, + need_invalidate_rkey, + remote_key); + + return batch; +} +EXPORT_SYMBOL_GPL(smbdirect_init_send_batch_storage); + +static int smbdirect_connection_wait_for_send_bcredit(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch) +{ + int ret; + + if (batch->credit) + return 0; + + ret = smbdirect_socket_wait_for_credits(sc, + SMBDIRECT_SOCKET_CONNECTED, + -ENOTCONN, + &sc->send_io.bcredits.wait_queue, + &sc->send_io.bcredits.count, + 1); + if (ret) + return ret; + + batch->credit = 1; + return 0; +} + +static int smbdirect_connection_wait_for_send_lcredit(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch) +{ + if (batch && atomic_read(&sc->send_io.lcredits.count) <= 1) { + int ret; + + ret = smbdirect_connection_send_batch_flush(sc, batch, false); + if (ret) + return ret; + } + + return smbdirect_socket_wait_for_credits(sc, + SMBDIRECT_SOCKET_CONNECTED, + -ENOTCONN, + &sc->send_io.lcredits.wait_queue, + &sc->send_io.lcredits.count, + 1); +} + +static int smbdirect_connection_wait_for_send_credits(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch) +{ + if (batch && (batch->wr_cnt >= 16 || atomic_read(&sc->send_io.credits.count) <= 1)) { + int ret; + + ret = smbdirect_connection_send_batch_flush(sc, batch, false); + if (ret) + return ret; + } + + return smbdirect_socket_wait_for_credits(sc, + SMBDIRECT_SOCKET_CONNECTED, + -ENOTCONN, + &sc->send_io.credits.wait_queue, + &sc->send_io.credits.count, + 1); +} + +static void smbdirect_connection_send_io_done(struct ib_cq *cq, struct ib_wc *wc); + +static int smbdirect_connection_post_send_io(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch, + struct smbdirect_send_io *msg) +{ + int i; + + for (i = 0; i < msg->num_sge; i++) + ib_dma_sync_single_for_device(sc->ib.dev, + msg->sge[i].addr, msg->sge[i].length, + DMA_TO_DEVICE); + + msg->cqe.done = smbdirect_connection_send_io_done; + msg->wr.wr_cqe = &msg->cqe; + msg->wr.opcode = IB_WR_SEND; + msg->wr.sg_list = &msg->sge[0]; + msg->wr.num_sge = msg->num_sge; + msg->wr.next = NULL; + + if (batch) { + msg->wr.send_flags = 0; + if (!list_empty(&batch->msg_list)) { + struct smbdirect_send_io *last; + + last = list_last_entry(&batch->msg_list, + struct smbdirect_send_io, + sibling_list); + last->wr.next = &msg->wr; + } + list_add_tail(&msg->sibling_list, &batch->msg_list); + batch->wr_cnt++; + return 0; + } + + msg->wr.send_flags = IB_SEND_SIGNALED; + return smbdirect_connection_post_send_wr(sc, &msg->wr); +} + +int smbdirect_connection_send_single_iter(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch, + struct iov_iter *iter, + unsigned int flags, + u32 remaining_data_length) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + struct smbdirect_send_batch _batch; + struct smbdirect_send_io *msg; + struct smbdirect_data_transfer *packet; + size_t header_length; + u16 new_credits = 0; + u32 data_length = 0; + int ret; + + if (WARN_ON_ONCE(flags)) + return -EINVAL; /* no flags support for now */ + + if (iter) { + if (WARN_ON_ONCE(iov_iter_rw(iter) != ITER_SOURCE)) + return -EINVAL; /* It's a bug in upper layer to get there */ + + header_length = sizeof(struct smbdirect_data_transfer); + if (WARN_ON_ONCE(remaining_data_length == 0 || + iov_iter_count(iter) > remaining_data_length)) + return -EINVAL; + } else { + /* If this is a packet without payload, don't send padding */ + header_length = offsetof(struct smbdirect_data_transfer, padding); + if (WARN_ON_ONCE(remaining_data_length)) + return -EINVAL; + } + + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { + smbdirect_log_write(sc, SMBDIRECT_LOG_ERR, + "status=%s first_error=%1pe => %1pe\n", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), + SMBDIRECT_DEBUG_ERR_PTR(-ENOTCONN)); + return -ENOTCONN; + } + + if (!batch) { + smbdirect_connection_send_batch_init(&_batch, false, 0); + batch = &_batch; + } + + ret = smbdirect_connection_wait_for_send_bcredit(sc, batch); + if (ret) + goto bcredit_failed; + + ret = smbdirect_connection_wait_for_send_lcredit(sc, batch); + if (ret) + goto lcredit_failed; + + ret = smbdirect_connection_wait_for_send_credits(sc, batch); + if (ret) + goto credit_failed; + + new_credits = smbdirect_connection_grant_recv_credits(sc); + if (new_credits == 0 && + atomic_read(&sc->send_io.credits.count) == 0 && + atomic_read(&sc->recv_io.credits.count) == 0) { + /* + * queue the refill work in order to + * get some new recv credits we can grant to + * the peer. + */ + queue_work(sc->workqueues.refill, &sc->recv_io.posted.refill_work); + + /* + * wait until either the refill work or the peer + * granted new credits + */ + ret = wait_event_interruptible(sc->send_io.credits.wait_queue, + atomic_read(&sc->send_io.credits.count) >= 1 || + atomic_read(&sc->recv_io.credits.available) >= 1 || + sc->status != SMBDIRECT_SOCKET_CONNECTED); + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) + ret = -ENOTCONN; + if (ret < 0) + goto credit_failed; + + new_credits = smbdirect_connection_grant_recv_credits(sc); + } + + msg = smbdirect_connection_alloc_send_io(sc); + if (IS_ERR(msg)) { + ret = PTR_ERR(msg); + goto alloc_failed; + } + + /* Map the packet to DMA */ + msg->sge[0].addr = ib_dma_map_single(sc->ib.dev, + msg->packet, + header_length, + DMA_TO_DEVICE); + ret = ib_dma_mapping_error(sc->ib.dev, msg->sge[0].addr); + if (ret) + goto err; + + msg->sge[0].length = header_length; + msg->sge[0].lkey = sc->ib.pd->local_dma_lkey; + msg->num_sge = 1; + + if (iter) { + struct smbdirect_map_sges extract = { + .num_sge = msg->num_sge, + .max_sge = ARRAY_SIZE(msg->sge), + .sge = msg->sge, + .device = sc->ib.dev, + .local_dma_lkey = sc->ib.pd->local_dma_lkey, + .direction = DMA_TO_DEVICE, + }; + size_t payload_len = umin(iov_iter_count(iter), + sp->max_send_size - sizeof(*packet)); + + ret = smbdirect_map_sges_from_iter(iter, payload_len, &extract); + if (ret < 0) + goto err; + data_length = ret; + remaining_data_length -= data_length; + msg->num_sge = extract.num_sge; + } + + /* Fill in the packet header */ + packet = (struct smbdirect_data_transfer *)msg->packet; + packet->credits_requested = cpu_to_le16(sp->send_credit_target); + packet->credits_granted = cpu_to_le16(new_credits); + + packet->flags = 0; + if (smbdirect_connection_request_keep_alive(sc)) + packet->flags |= cpu_to_le16(SMBDIRECT_FLAG_RESPONSE_REQUESTED); + + packet->reserved = 0; + if (!data_length) + packet->data_offset = 0; + else + packet->data_offset = cpu_to_le32(24); + packet->data_length = cpu_to_le32(data_length); + packet->remaining_data_length = cpu_to_le32(remaining_data_length); + packet->padding = 0; + + smbdirect_log_outgoing(sc, SMBDIRECT_LOG_INFO, + "DataOut: %s=%u, %s=%u, %s=0x%x, %s=%u, %s=%u, %s=%u\n", + "CreditsRequested", + le16_to_cpu(packet->credits_requested), + "CreditsGranted", + le16_to_cpu(packet->credits_granted), + "Flags", + le16_to_cpu(packet->flags), + "RemainingDataLength", + le32_to_cpu(packet->remaining_data_length), + "DataOffset", + le32_to_cpu(packet->data_offset), + "DataLength", + le32_to_cpu(packet->data_length)); + + ret = smbdirect_connection_post_send_io(sc, batch, msg); + if (ret) + goto err; + + /* + * From here msg is moved to send_ctx + * and we should not free it explicitly. + */ + + if (batch == &_batch) { + ret = smbdirect_connection_send_batch_flush(sc, batch, true); + if (ret) + goto flush_failed; + } + + return data_length; +err: + smbdirect_connection_free_send_io(msg); +flush_failed: +alloc_failed: + atomic_inc(&sc->send_io.credits.count); +credit_failed: + atomic_inc(&sc->send_io.lcredits.count); +lcredit_failed: + atomic_add(batch->credit, &sc->send_io.bcredits.count); + batch->credit = 0; +bcredit_failed: + return ret; +} +EXPORT_SYMBOL_GPL(smbdirect_connection_send_single_iter); + +int smbdirect_connection_send_wait_zero_pending(struct smbdirect_socket *sc) +{ + /* + * As an optimization, we don't wait for individual I/O to finish + * before sending the next one. + * Send them all and wait for pending send count to get to 0 + * that means all the I/Os have been out and we are good to return + */ + + wait_event(sc->send_io.pending.zero_wait_queue, + atomic_read(&sc->send_io.pending.count) == 0 || + sc->status != SMBDIRECT_SOCKET_CONNECTED); + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { + smbdirect_log_write(sc, SMBDIRECT_LOG_ERR, + "status=%s first_error=%1pe => %1pe\n", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), + SMBDIRECT_DEBUG_ERR_PTR(-ENOTCONN)); + return -ENOTCONN; + } + + return 0; +} +EXPORT_SYMBOL_GPL(smbdirect_connection_send_wait_zero_pending); + +int smbdirect_connection_send_iter(struct smbdirect_socket *sc, + struct iov_iter *iter, + unsigned int flags, + bool need_invalidate, + unsigned int remote_key) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + struct smbdirect_send_batch batch; + int total_count = iov_iter_count(iter); + int ret; + int error = 0; + __be32 hdr; + + if (WARN_ONCE(flags, "unexpected flags=0x%x\n", flags)) + return -EINVAL; /* no flags support for now */ + + if (WARN_ON_ONCE(iov_iter_rw(iter) != ITER_SOURCE)) + return -EINVAL; /* It's a bug in upper layer to get there */ + + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { + smbdirect_log_write(sc, SMBDIRECT_LOG_INFO, + "status=%s first_error=%1pe => %1pe\n", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), + SMBDIRECT_DEBUG_ERR_PTR(-ENOTCONN)); + return -ENOTCONN; + } + + /* + * For now we expect the iter to have the full + * message, including a 4 byte length header. + */ + if (iov_iter_count(iter) <= 4) + return -EINVAL; + if (!copy_from_iter_full(&hdr, sizeof(hdr), iter)) + return -EFAULT; + if (iov_iter_count(iter) != be32_to_cpu(hdr)) + return -EINVAL; + + /* + * The size must fit into the negotiated + * fragmented send size. + */ + if (iov_iter_count(iter) > sp->max_fragmented_send_size) + return -EMSGSIZE; + + smbdirect_log_write(sc, SMBDIRECT_LOG_INFO, + "Sending (RDMA): length=%zu\n", + iov_iter_count(iter)); + + smbdirect_connection_send_batch_init(&batch, need_invalidate, remote_key); + while (iov_iter_count(iter)) { + ret = smbdirect_connection_send_single_iter(sc, + &batch, + iter, + flags, + iov_iter_count(iter)); + if (unlikely(ret < 0)) { + error = ret; + break; + } + } + + ret = smbdirect_connection_send_batch_flush(sc, &batch, true); + if (unlikely(ret && !error)) + error = ret; + + /* + * As an optimization, we don't wait for individual I/O to finish + * before sending the next one. + * Send them all and wait for pending send count to get to 0 + * that means all the I/Os have been out and we are good to return + */ + + ret = smbdirect_connection_send_wait_zero_pending(sc); + if (unlikely(ret && !error)) + error = ret; + + if (unlikely(error)) + return error; + + return total_count; +} +EXPORT_SYMBOL_GPL(smbdirect_connection_send_iter); + +static void smbdirect_connection_send_io_done(struct ib_cq *cq, struct ib_wc *wc) +{ + struct smbdirect_send_io *msg = + container_of(wc->wr_cqe, struct smbdirect_send_io, cqe); + struct smbdirect_socket *sc = msg->socket; + struct smbdirect_send_io *sibling, *next; + int lcredits = 0; + + smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_INFO, + "smbdirect_send_io completed. status='%s (%d)', opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + + if (unlikely(!(msg->wr.send_flags & IB_SEND_SIGNALED))) { + /* + * This happens when smbdirect_send_io is a sibling + * before the final message, it is signaled on + * error anyway, so we need to skip + * smbdirect_connection_free_send_io here, + * otherwise is will destroy the memory + * of the siblings too, which will cause + * use after free problems for the others + * triggered from ib_drain_qp(). + */ + if (wc->status != IB_WC_SUCCESS) + goto skip_free; + + /* + * This should not happen! + * But we better just close the + * connection... + */ + smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR, + "unexpected send completion wc->status=%s (%d) wc->opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + + /* + * Free possible siblings and then the main send_io + */ + list_for_each_entry_safe(sibling, next, &msg->sibling_list, sibling_list) { + list_del_init(&sibling->sibling_list); + smbdirect_connection_free_send_io(sibling); + lcredits += 1; + } + /* Note this frees wc->wr_cqe, but not wc */ + smbdirect_connection_free_send_io(msg); + lcredits += 1; + + if (unlikely(wc->status != IB_WC_SUCCESS || WARN_ON_ONCE(wc->opcode != IB_WC_SEND))) { +skip_free: + if (wc->status != IB_WC_WR_FLUSH_ERR) + smbdirect_log_rdma_send(sc, SMBDIRECT_LOG_ERR, + "wc->status=%s (%d) wc->opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + return; + } + + atomic_add(lcredits, &sc->send_io.lcredits.count); + wake_up(&sc->send_io.lcredits.wait_queue); + + if (atomic_dec_and_test(&sc->send_io.pending.count)) + wake_up(&sc->send_io.pending.zero_wait_queue); +} + +static void smbdirect_connection_send_immediate_work(struct work_struct *work) +{ + struct smbdirect_socket *sc = + container_of(work, struct smbdirect_socket, idle.immediate_work); + int ret; + + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) + return; + + smbdirect_log_keep_alive(sc, SMBDIRECT_LOG_INFO, + "send an empty message\n"); + sc->statistics.send_empty++; + ret = smbdirect_connection_send_single_iter(sc, NULL, NULL, 0, 0); + if (ret < 0) { + smbdirect_log_write(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_send_single_iter ret=%1pe\n", + SMBDIRECT_DEBUG_ERR_PTR(ret)); + smbdirect_socket_schedule_cleanup(sc, ret); + } +} + +int smbdirect_connection_post_recv_io(struct smbdirect_recv_io *msg) +{ + struct smbdirect_socket *sc = msg->socket; + const struct smbdirect_socket_parameters *sp = &sc->parameters; + struct ib_recv_wr recv_wr = { + .wr_cqe = &msg->cqe, + .sg_list = &msg->sge, + .num_sge = 1, + }; + int ret; + + if (unlikely(sc->first_error)) + return sc->first_error; + + msg->sge.addr = ib_dma_map_single(sc->ib.dev, + msg->packet, + sp->max_recv_size, + DMA_FROM_DEVICE); + ret = ib_dma_mapping_error(sc->ib.dev, msg->sge.addr); + if (ret) + return ret; + + msg->sge.length = sp->max_recv_size; + msg->sge.lkey = sc->ib.pd->local_dma_lkey; + + ret = ib_post_recv(sc->ib.qp, &recv_wr, NULL); + if (ret) { + smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_ERR, + "ib_post_recv failed ret=%d (%1pe)\n", + ret, SMBDIRECT_DEBUG_ERR_PTR(ret)); + ib_dma_unmap_single(sc->ib.dev, + msg->sge.addr, + msg->sge.length, + DMA_FROM_DEVICE); + msg->sge.length = 0; + smbdirect_socket_schedule_cleanup(sc, ret); + } + + return ret; +} + +void smbdirect_connection_recv_io_done(struct ib_cq *cq, struct ib_wc *wc) +{ + struct smbdirect_recv_io *recv_io = + container_of(wc->wr_cqe, struct smbdirect_recv_io, cqe); + struct smbdirect_socket *sc = recv_io->socket; + const struct smbdirect_socket_parameters *sp = &sc->parameters; + struct smbdirect_data_transfer *data_transfer; + int current_recv_credits; + u16 old_recv_credit_target; + u16 credits_requested; + u16 credits_granted; + u16 flags; + u32 data_offset; + u32 data_length; + u32 remaining_data_length; + + if (unlikely(wc->status != IB_WC_SUCCESS || WARN_ON_ONCE(wc->opcode != IB_WC_RECV))) { + if (wc->status != IB_WC_WR_FLUSH_ERR) + smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_ERR, + "wc->status=%s (%d) wc->opcode=%d\n", + ib_wc_status_msg(wc->status), wc->status, wc->opcode); + goto error; + } + + smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_INFO, + "recv_io=0x%p type=%d wc status=%s wc opcode %d byte_len=%d pkey_index=%u\n", + recv_io, sc->recv_io.expected, + ib_wc_status_msg(wc->status), wc->opcode, + wc->byte_len, wc->pkey_index); + + /* + * Reset timer to the keepalive interval in + * order to trigger our next keepalive message. + */ + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; + mod_delayed_work(sc->workqueues.idle, &sc->idle.timer_work, + msecs_to_jiffies(sp->keepalive_interval_msec)); + + ib_dma_sync_single_for_cpu(sc->ib.dev, + recv_io->sge.addr, + recv_io->sge.length, + DMA_FROM_DEVICE); + + if (unlikely(wc->byte_len < + offsetof(struct smbdirect_data_transfer, padding))) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "wc->byte_len=%u < %zu\n", + wc->byte_len, + offsetof(struct smbdirect_data_transfer, padding)); + goto error; + } + + data_transfer = (struct smbdirect_data_transfer *)recv_io->packet; + credits_requested = le16_to_cpu(data_transfer->credits_requested); + credits_granted = le16_to_cpu(data_transfer->credits_granted); + flags = le16_to_cpu(data_transfer->flags); + remaining_data_length = le32_to_cpu(data_transfer->remaining_data_length); + data_offset = le32_to_cpu(data_transfer->data_offset); + data_length = le32_to_cpu(data_transfer->data_length); + + smbdirect_log_incoming(sc, SMBDIRECT_LOG_INFO, + "DataIn: %s=%u, %s=%u, %s=0x%x, %s=%u, %s=%u, %s=%u\n", + "CreditsRequested", + credits_requested, + "CreditsGranted", + credits_granted, + "Flags", + flags, + "RemainingDataLength", + remaining_data_length, + "DataOffset", + data_offset, + "DataLength", + data_length); + + if (unlikely(credits_requested == 0)) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: credits_requested == 0\n"); + goto error; + } + + if (unlikely(data_offset % 8 != 0)) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "invalid: data_offset=%u (0x%x) not aligned to 8\n", + data_offset, data_offset); + goto error; + } + + if (unlikely(wc->byte_len < data_offset || + (u64)wc->byte_len < (u64)data_offset + data_length)) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "wc->byte_len=%u < date_offset=%u + data_length=%u\n", + wc->byte_len, data_offset, data_length); + goto error; + } + + if (unlikely(remaining_data_length > sp->max_fragmented_recv_size || + data_length > sp->max_fragmented_recv_size || + (u64)remaining_data_length + (u64)data_length > (u64)sp->max_fragmented_recv_size)) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "remaining_data_length=%u + data_length=%u > max_fragmented=%u\n", + remaining_data_length, data_length, sp->max_fragmented_recv_size); + goto error; + } + + if (data_length) { + if (sc->recv_io.reassembly.full_packet_received) + recv_io->first_segment = true; + + if (remaining_data_length) + sc->recv_io.reassembly.full_packet_received = false; + else + sc->recv_io.reassembly.full_packet_received = true; + } + + atomic_dec(&sc->recv_io.posted.count); + current_recv_credits = atomic_dec_return(&sc->recv_io.credits.count); + + /* + * We take the value from the peer, which is checked to be higher than 0, + * but we limit it to the max value we support in order to have + * the main logic simpler. + */ + old_recv_credit_target = sc->recv_io.credits.target; + sc->recv_io.credits.target = credits_requested; + sc->recv_io.credits.target = min_t(u16, sc->recv_io.credits.target, + sp->recv_credit_max); + if (credits_granted) { + atomic_add(credits_granted, &sc->send_io.credits.count); + /* + * We have new send credits granted from remote peer + * If any sender is waiting for credits, unblock it + */ + wake_up(&sc->send_io.credits.wait_queue); + } + + /* Send an immediate response right away if requested */ + if (flags & SMBDIRECT_FLAG_RESPONSE_REQUESTED) { + smbdirect_log_keep_alive(sc, SMBDIRECT_LOG_INFO, + "schedule send of immediate response\n"); + queue_work(sc->workqueues.immediate, &sc->idle.immediate_work); + } + + /* + * If this is a packet with data playload place the data in + * reassembly queue and wake up the reading thread + */ + if (data_length) { + if (current_recv_credits <= (sc->recv_io.credits.target / 4) || + sc->recv_io.credits.target > old_recv_credit_target) + queue_work(sc->workqueues.refill, &sc->recv_io.posted.refill_work); + + smbdirect_connection_reassembly_append_recv_io(sc, recv_io, data_length); + wake_up(&sc->recv_io.reassembly.wait_queue); + } else + smbdirect_connection_put_recv_io(recv_io); + + return; + +error: + /* + * Make sure smbdirect_connection_put_recv_io() does not + * start recv_io.posted.refill_work. + */ + disable_work(&sc->recv_io.posted.refill_work); + smbdirect_connection_put_recv_io(recv_io); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); +} + +int smbdirect_connection_recv_io_refill(struct smbdirect_socket *sc) +{ + int missing; + int posted = 0; + + if (unlikely(sc->first_error)) + return sc->first_error; + + /* + * Find out how much smbdirect_recv_io buffers we should post. + * + * Note that sc->recv_io.credits.target is the value + * from the peer and it can in theory change over time, + * but it is forced to be at least 1 and at max + * sp->recv_credit_max. + * + * So it can happen that missing will be lower than 0, + * which means the peer has recently lowered its desired + * target, while be already granted a higher number of credits. + * + * Note 'posted' is the number of smbdirect_recv_io buffers + * posted within this function, while sc->recv_io.posted.count + * is the overall value of posted smbdirect_recv_io buffers. + * + * We try to post as much buffers as missing, but + * this is limited if a lot of smbdirect_recv_io buffers + * are still in the sc->recv_io.reassembly.list instead of + * the sc->recv_io.free.list. + * + */ + missing = (int)sc->recv_io.credits.target - atomic_read(&sc->recv_io.posted.count); + while (posted < missing) { + struct smbdirect_recv_io *recv_io; + int ret; + + /* + * It's ok if smbdirect_connection_get_recv_io() + * returns NULL, it means smbdirect_recv_io structures + * are still be in the reassembly.list. + */ + recv_io = smbdirect_connection_get_recv_io(sc); + if (!recv_io) + break; + + recv_io->first_segment = false; + + ret = smbdirect_connection_post_recv_io(recv_io); + if (ret) { + smbdirect_log_rdma_recv(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_post_recv_io failed rc=%d (%1pe)\n", + ret, SMBDIRECT_DEBUG_ERR_PTR(ret)); + smbdirect_connection_put_recv_io(recv_io); + return ret; + } + + atomic_inc(&sc->recv_io.posted.count); + posted += 1; + } + + /* If nothing was posted we're done */ + if (posted == 0) + return 0; + + atomic_add(posted, &sc->recv_io.credits.available); + + /* + * If the last send credit is waiting for credits + * it can grant we need to wake it up + */ + if (atomic_read(&sc->send_io.bcredits.count) == 0 && + atomic_read(&sc->send_io.credits.count) == 0) + wake_up(&sc->send_io.credits.wait_queue); + + /* + * If we posted at least one smbdirect_recv_io buffer, + * we need to inform the peer about it and grant + * additional credits. + * + * However there is one case where we don't want to + * do that. + * + * If only a single credit was missing before + * reaching the requested target, we should not + * post an immediate send, as that would cause + * endless ping pong once a keep alive exchange + * is started. + * + * However if sc->recv_io.credits.target is only 1, + * the peer has no credit left and we need to + * grant the credit anyway. + */ + if (missing == 1 && sc->recv_io.credits.target != 1) + return 0; + + return posted; +} + +static void smbdirect_connection_recv_io_refill_work(struct work_struct *work) +{ + struct smbdirect_socket *sc = + container_of(work, struct smbdirect_socket, recv_io.posted.refill_work); + int posted; + + posted = smbdirect_connection_recv_io_refill(sc); + if (unlikely(posted < 0)) { + smbdirect_socket_schedule_cleanup(sc, posted); + return; + } + if (posted > 0) { + smbdirect_log_keep_alive(sc, SMBDIRECT_LOG_INFO, + "schedule send of an empty message\n"); + queue_work(sc->workqueues.immediate, &sc->idle.immediate_work); + } +} + +int smbdirect_connection_recvmsg(struct smbdirect_socket *sc, + struct msghdr *msg, + unsigned int flags) +{ + struct smbdirect_recv_io *response; + struct smbdirect_data_transfer *data_transfer; + size_t size = iov_iter_count(&msg->msg_iter); + int to_copy, to_read, data_read, offset; + u32 data_length, remaining_data_length, data_offset; + int ret; + + if (WARN_ONCE(flags, "unexpected flags=0x%x\n", flags)) + return -EINVAL; /* no flags support for now */ + + if (WARN_ON_ONCE(iov_iter_rw(&msg->msg_iter) != ITER_DEST)) + return -EINVAL; /* It's a bug in upper layer to get there */ + +again: + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { + smbdirect_log_read(sc, SMBDIRECT_LOG_INFO, + "status=%s first_error=%1pe => %1pe\n", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), + SMBDIRECT_DEBUG_ERR_PTR(-ENOTCONN)); + return -ENOTCONN; + } + + /* + * No need to hold the reassembly queue lock all the time as we are + * the only one reading from the front of the queue. The transport + * may add more entries to the back of the queue at the same time + */ + smbdirect_log_read(sc, SMBDIRECT_LOG_INFO, + "size=%zd sc->recv_io.reassembly.data_length=%d\n", + size, sc->recv_io.reassembly.data_length); + if (sc->recv_io.reassembly.data_length >= size) { + int queue_length; + int queue_removed = 0; + unsigned long flags; + + /* + * Need to make sure reassembly_data_length is read before + * reading reassembly_queue_length and calling + * smbdirect_connection_reassembly_first_recv_io. This call is lock free + * as we never read at the end of the queue which are being + * updated in SOFTIRQ as more data is received + */ + virt_rmb(); + queue_length = sc->recv_io.reassembly.queue_length; + data_read = 0; + to_read = size; + offset = sc->recv_io.reassembly.first_entry_offset; + while (data_read < size) { + response = smbdirect_connection_reassembly_first_recv_io(sc); + data_transfer = (void *)response->packet; + data_length = le32_to_cpu(data_transfer->data_length); + remaining_data_length = + le32_to_cpu( + data_transfer->remaining_data_length); + data_offset = le32_to_cpu(data_transfer->data_offset); + + /* + * The upper layer expects RFC1002 length at the + * beginning of the payload. Return it to indicate + * the total length of the packet. This minimize the + * change to upper layer packet processing logic. This + * will be eventually remove when an intermediate + * transport layer is added + */ + if (response->first_segment && size == 4) { + unsigned int rfc1002_len = + data_length + remaining_data_length; + __be32 rfc1002_hdr = cpu_to_be32(rfc1002_len); + + if (copy_to_iter(&rfc1002_hdr, sizeof(rfc1002_hdr), + &msg->msg_iter) != sizeof(rfc1002_hdr)) + return -EFAULT; + data_read = 4; + response->first_segment = false; + smbdirect_log_read(sc, SMBDIRECT_LOG_INFO, + "returning rfc1002 length %d\n", + rfc1002_len); + goto read_rfc1002_done; + } + + to_copy = min_t(int, data_length - offset, to_read); + if (copy_to_iter((u8 *)data_transfer + data_offset + offset, + to_copy, &msg->msg_iter) != to_copy) + return -EFAULT; + + /* move on to the next buffer? */ + if (to_copy == data_length - offset) { + queue_length--; + /* + * No need to lock if we are not at the + * end of the queue + */ + if (queue_length) + list_del(&response->list); + else { + spin_lock_irqsave( + &sc->recv_io.reassembly.lock, flags); + list_del(&response->list); + spin_unlock_irqrestore( + &sc->recv_io.reassembly.lock, flags); + } + queue_removed++; + sc->statistics.dequeue_reassembly_queue++; + smbdirect_connection_put_recv_io(response); + offset = 0; + smbdirect_log_read(sc, SMBDIRECT_LOG_INFO, + "smbdirect_connection_put_recv_io offset=0\n"); + } else + offset += to_copy; + + to_read -= to_copy; + data_read += to_copy; + + smbdirect_log_read(sc, SMBDIRECT_LOG_INFO, + "memcpy %d bytes len-ofs=%u => todo=%u done=%u ofs=%u\n", + to_copy, data_length - offset, + to_read, data_read, offset); + } + + spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); + sc->recv_io.reassembly.data_length -= data_read; + sc->recv_io.reassembly.queue_length -= queue_removed; + spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); + + sc->recv_io.reassembly.first_entry_offset = offset; + smbdirect_log_read(sc, SMBDIRECT_LOG_INFO, + "returning data_read=%d reassembly_length=%d first_ofs=%u\n", + data_read, sc->recv_io.reassembly.data_length, + sc->recv_io.reassembly.first_entry_offset); +read_rfc1002_done: + return data_read; + } + + smbdirect_log_read(sc, SMBDIRECT_LOG_INFO, + "wait_event on more data\n"); + ret = wait_event_interruptible(sc->recv_io.reassembly.wait_queue, + sc->recv_io.reassembly.data_length >= size || + sc->status != SMBDIRECT_SOCKET_CONNECTED); + /* Don't return any data if interrupted */ + if (ret) + return ret; + + goto again; +} +EXPORT_SYMBOL_GPL(smbdirect_connection_recvmsg); + +static bool smbdirect_map_sges_single_page(struct smbdirect_map_sges *state, + struct page *page, size_t off, size_t len) +{ + struct ib_sge *sge; + u64 addr; + + if (state->num_sge >= state->max_sge) + return false; + + addr = ib_dma_map_page(state->device, page, + off, len, state->direction); + if (ib_dma_mapping_error(state->device, addr)) + return false; + + sge = &state->sge[state->num_sge++]; + sge->addr = addr; + sge->length = len; + sge->lkey = state->local_dma_lkey; + + return true; +} + +/* + * Extract page fragments from a BVEC-class iterator and add them to an ib_sge + * list. The pages are not pinned. + */ +static ssize_t smbdirect_map_sges_from_bvec(struct iov_iter *iter, + struct smbdirect_map_sges *state, + ssize_t maxsize) +{ + const struct bio_vec *bv = iter->bvec; + unsigned long start = iter->iov_offset; + unsigned int i; + ssize_t ret = 0; + + for (i = 0; i < iter->nr_segs; i++) { + size_t off, len; + bool ok; + + len = bv[i].bv_len; + if (start >= len) { + start -= len; + continue; + } + + len = min_t(size_t, maxsize, len - start); + off = bv[i].bv_offset + start; + + ok = smbdirect_map_sges_single_page(state, + bv[i].bv_page, + off, + len); + if (!ok) + return -EIO; + + ret += len; + maxsize -= len; + if (state->num_sge >= state->max_sge || maxsize <= 0) + break; + start = 0; + } + + if (ret > 0) + iov_iter_advance(iter, ret); + return ret; +} + +/* + * Extract fragments from a KVEC-class iterator and add them to an ib_sge list. + * This can deal with vmalloc'd buffers as well as kmalloc'd or static buffers. + * The pages are not pinned. + */ +static ssize_t smbdirect_map_sges_from_kvec(struct iov_iter *iter, + struct smbdirect_map_sges *state, + ssize_t maxsize) +{ + const struct kvec *kv = iter->kvec; + unsigned long start = iter->iov_offset; + unsigned int i; + ssize_t ret = 0; + + for (i = 0; i < iter->nr_segs; i++) { + struct page *page; + unsigned long kaddr; + size_t off, len, seg; + + len = kv[i].iov_len; + if (start >= len) { + start -= len; + continue; + } + + kaddr = (unsigned long)kv[i].iov_base + start; + off = kaddr & ~PAGE_MASK; + len = min_t(size_t, maxsize, len - start); + kaddr &= PAGE_MASK; + + maxsize -= len; + do { + bool ok; + + seg = min_t(size_t, len, PAGE_SIZE - off); + + if (is_vmalloc_or_module_addr((void *)kaddr)) + page = vmalloc_to_page((void *)kaddr); + else + page = virt_to_page((void *)kaddr); + + ok = smbdirect_map_sges_single_page(state, page, off, seg); + if (!ok) + return -EIO; + + ret += seg; + len -= seg; + kaddr += PAGE_SIZE; + off = 0; + } while (len > 0 && state->num_sge < state->max_sge); + + if (state->num_sge >= state->max_sge || maxsize <= 0) + break; + start = 0; + } + + if (ret > 0) + iov_iter_advance(iter, ret); + return ret; +} + +/* + * Extract folio fragments from a FOLIOQ-class iterator and add them to an + * ib_sge list. The folios are not pinned. + */ +static ssize_t smbdirect_map_sges_from_folioq(struct iov_iter *iter, + struct smbdirect_map_sges *state, + ssize_t maxsize) +{ + const struct folio_queue *folioq = iter->folioq; + unsigned int slot = iter->folioq_slot; + ssize_t ret = 0; + size_t offset = iter->iov_offset; + + if (WARN_ON_ONCE(!folioq)) + return -EIO; + + if (slot >= folioq_nr_slots(folioq)) { + folioq = folioq->next; + if (WARN_ON_ONCE(!folioq)) + return -EIO; + slot = 0; + } + + do { + struct folio *folio = folioq_folio(folioq, slot); + size_t fsize = folioq_folio_size(folioq, slot); + + if (offset < fsize) { + size_t part = umin(maxsize, fsize - offset); + bool ok; + + ok = smbdirect_map_sges_single_page(state, + folio_page(folio, 0), + offset, + part); + if (!ok) + return -EIO; + + offset += part; + ret += part; + maxsize -= part; + } + + if (offset >= fsize) { + offset = 0; + slot++; + if (slot >= folioq_nr_slots(folioq)) { + if (!folioq->next) { + WARN_ON_ONCE(ret < iter->count); + break; + } + folioq = folioq->next; + slot = 0; + } + } + } while (state->num_sge < state->max_sge && maxsize > 0); + + iter->folioq = folioq; + iter->folioq_slot = slot; + iter->iov_offset = offset; + iter->count -= ret; + return ret; +} + +/* + * Extract page fragments from up to the given amount of the source iterator + * and build up an ib_sge list that refers to all of those bits. The ib_sge list + * is appended to, up to the maximum number of elements set in the parameter + * block. + * + * The extracted page fragments are not pinned or ref'd in any way; if an + * IOVEC/UBUF-type iterator is to be used, it should be converted to a + * BVEC-type iterator and the pages pinned, ref'd or otherwise held in some + * way. + */ +static ssize_t smbdirect_map_sges_from_iter(struct iov_iter *iter, size_t len, + struct smbdirect_map_sges *state) +{ + ssize_t ret; + size_t before = state->num_sge; + + if (WARN_ON_ONCE(iov_iter_rw(iter) != ITER_SOURCE)) + return -EIO; + + switch (iov_iter_type(iter)) { + case ITER_BVEC: + ret = smbdirect_map_sges_from_bvec(iter, state, len); + break; + case ITER_KVEC: + ret = smbdirect_map_sges_from_kvec(iter, state, len); + break; + case ITER_FOLIOQ: + ret = smbdirect_map_sges_from_folioq(iter, state, len); + break; + default: + WARN_ONCE(1, "iov_iter_type[%u]\n", iov_iter_type(iter)); + return -EIO; + } + + if (ret < 0) { + while (state->num_sge > before) { + struct ib_sge *sge = &state->sge[--state->num_sge]; + + ib_dma_unmap_page(state->device, + sge->addr, + sge->length, + state->direction); + } + } + + return ret; +} diff --git a/fs/smb/smbdirect/debug.c b/fs/smb/smbdirect/debug.c new file mode 100644 index 000000000..344584344 --- /dev/null +++ b/fs/smb/smbdirect/debug.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017, Microsoft Corporation. + * Copyright (c) 2025, Stefan Metzmacher + */ + +#include "internal.h" +#include + +void smbdirect_connection_legacy_debug_proc_show(struct smbdirect_socket *sc, + unsigned int rdma_readwrite_threshold, + struct seq_file *m) +{ + const struct smbdirect_socket_parameters *sp; + + if (!sc) + return; + sp = &sc->parameters; + + seq_puts(m, "\n"); + seq_printf(m, "SMBDirect protocol version: 0x%x ", + SMBDIRECT_V1); + seq_printf(m, "transport status: %s (%u)", + smbdirect_socket_status_string(sc->status), + sc->status); + + seq_puts(m, "\n"); + seq_printf(m, "Conn receive_credit_max: %u ", + sp->recv_credit_max); + seq_printf(m, "send_credit_target: %u max_send_size: %u", + sp->send_credit_target, + sp->max_send_size); + + seq_puts(m, "\n"); + seq_printf(m, "Conn max_fragmented_recv_size: %u ", + sp->max_fragmented_recv_size); + seq_printf(m, "max_fragmented_send_size: %u max_receive_size:%u", + sp->max_fragmented_send_size, + sp->max_recv_size); + + seq_puts(m, "\n"); + seq_printf(m, "Conn keep_alive_interval: %u ", + sp->keepalive_interval_msec / 1000); + seq_printf(m, "max_readwrite_size: %u rdma_readwrite_threshold: %u", + sp->max_read_write_size, + rdma_readwrite_threshold); + + seq_puts(m, "\n"); + seq_printf(m, "Debug count_get_receive_buffer: %llu ", + sc->statistics.get_receive_buffer); + seq_printf(m, "count_put_receive_buffer: %llu count_send_empty: %llu", + sc->statistics.put_receive_buffer, + sc->statistics.send_empty); + + seq_puts(m, "\n"); + seq_printf(m, "Read Queue count_enqueue_reassembly_queue: %llu ", + sc->statistics.enqueue_reassembly_queue); + seq_printf(m, "count_dequeue_reassembly_queue: %llu ", + sc->statistics.dequeue_reassembly_queue); + seq_printf(m, "reassembly_data_length: %u ", + sc->recv_io.reassembly.data_length); + seq_printf(m, "reassembly_queue_length: %u", + sc->recv_io.reassembly.queue_length); + + seq_puts(m, "\n"); + seq_printf(m, "Current Credits send_credits: %u ", + atomic_read(&sc->send_io.credits.count)); + seq_printf(m, "receive_credits: %u receive_credit_target: %u", + atomic_read(&sc->recv_io.credits.count), + sc->recv_io.credits.target); + + seq_puts(m, "\n"); + seq_printf(m, "Pending send_pending: %u ", + atomic_read(&sc->send_io.pending.count)); + + seq_puts(m, "\n"); + seq_printf(m, "MR responder_resources: %u ", + sp->responder_resources); + seq_printf(m, "max_frmr_depth: %u mr_type: 0x%x", + sp->max_frmr_depth, + sc->mr_io.type); + + seq_puts(m, "\n"); + seq_printf(m, "MR mr_ready_count: %u mr_used_count: %u", + atomic_read(&sc->mr_io.ready.count), + atomic_read(&sc->mr_io.used.count)); +} +EXPORT_SYMBOL_GPL(smbdirect_connection_legacy_debug_proc_show); diff --git a/fs/smb/smbdirect/devices.c b/fs/smb/smbdirect/devices.c new file mode 100644 index 000000000..7adacbdfe --- /dev/null +++ b/fs/smb/smbdirect/devices.c @@ -0,0 +1,277 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017, Microsoft Corporation. + * Copyright (C) 2018, LG Electronics. + * Copyright (c) 2025 Stefan Metzmacher + */ + +#include "internal.h" + +static u8 smbdirect_ib_device_rdma_capable_node_type(struct ib_device *ib_dev) +{ + if (!smbdirect_frwr_is_supported(&ib_dev->attrs)) + return RDMA_NODE_UNSPECIFIED; + + switch (ib_dev->node_type) { + case RDMA_NODE_IB_CA: /* Infiniband, RoCE v1 and v2 */ + case RDMA_NODE_RNIC: /* iWarp */ + return ib_dev->node_type; + } + + return RDMA_NODE_UNSPECIFIED; +} + +static int smbdirect_ib_client_add(struct ib_device *ib_dev) +{ + u8 node_type = smbdirect_ib_device_rdma_capable_node_type(ib_dev); + struct smbdirect_device *sdev; + const char *node_str; + const char *action; + u32 pidx; + + switch (node_type) { + case RDMA_NODE_IB_CA: + node_str = "IB_CA"; + action = "added"; + break; + case RDMA_NODE_RNIC: + node_str = "RNIC"; + action = "added"; + break; + case RDMA_NODE_UNSPECIFIED: + node_str = "UNSPECIFIED"; + action = "ignored"; + break; + default: + node_str = "UNKNOWN"; + action = "ignored"; + node_type = RDMA_NODE_UNSPECIFIED; + break; + } + + pr_info("ib_dev[%.*s]: %s: %s %s=%u %s=0x%llx %s=0x%llx %s=0x%llx\n", + IB_DEVICE_NAME_MAX, + ib_dev->name, + action, + node_str, + "max_fast_reg_page_list_len", + ib_dev->attrs.max_fast_reg_page_list_len, + "device_cap_flags", + ib_dev->attrs.device_cap_flags, + "kernel_cap_flags", + ib_dev->attrs.kernel_cap_flags, + "page_size_cap", + ib_dev->attrs.page_size_cap); + + if (node_type == RDMA_NODE_UNSPECIFIED) + return 0; + + pr_info("ib_dev[%.*s]: %s=%u %s=%u %s=%u %s=%u %s=%u %s=%u %s=%u %s=%u %s=%u\n", + IB_DEVICE_NAME_MAX, + ib_dev->name, + "num_ports", + rdma_end_port(ib_dev), + "max_qp_rd_atom", + ib_dev->attrs.max_qp_rd_atom, + "max_qp_init_rd_atom", + ib_dev->attrs.max_qp_init_rd_atom, + "max_sgl_rd", + ib_dev->attrs.max_sgl_rd, + "max_sge_rd", + ib_dev->attrs.max_sge_rd, + "max_cqe", + ib_dev->attrs.max_cqe, + "max_qp_wr", + ib_dev->attrs.max_qp_wr, + "max_send_sge", + ib_dev->attrs.max_send_sge, + "max_recv_sge", + ib_dev->attrs.max_recv_sge); + + rdma_for_each_port(ib_dev, pidx) { + const struct ib_port_immutable *ib_pi = + ib_port_immutable_read(ib_dev, pidx); + u32 core_cap_flags = ib_pi ? ib_pi->core_cap_flags : 0; + + pr_info("ib_dev[%.*s]PORT[%u]: %s=%u %s=%u %s=%u %s=%u %s=%u %s=0x%x\n", + IB_DEVICE_NAME_MAX, + ib_dev->name, + pidx, + "iwarp", + rdma_protocol_iwarp(ib_dev, pidx), + "ib", + rdma_protocol_ib(ib_dev, pidx), + "roce", + rdma_protocol_roce(ib_dev, pidx), + "v1", + rdma_protocol_roce_eth_encap(ib_dev, pidx), + "v2", + rdma_protocol_roce_udp_encap(ib_dev, pidx), + "core_cap_flags", + core_cap_flags); + } + + sdev = kzalloc_obj(*sdev); + if (!sdev) + return -ENOMEM; + sdev->ib_dev = ib_dev; + snprintf(sdev->ib_name, ARRAY_SIZE(sdev->ib_name), "%.*s", + IB_DEVICE_NAME_MAX, ib_dev->name); + + write_lock(&smbdirect_globals.devices.lock); + list_add(&sdev->list, &smbdirect_globals.devices.list); + write_unlock(&smbdirect_globals.devices.lock); + + return 0; +} + +static void smbdirect_ib_client_remove(struct ib_device *ib_dev, void *client_data) +{ + struct smbdirect_device *sdev, *tmp; + + write_lock(&smbdirect_globals.devices.lock); + list_for_each_entry_safe(sdev, tmp, &smbdirect_globals.devices.list, list) { + if (sdev->ib_dev == ib_dev) { + list_del(&sdev->list); + pr_info("ib_dev[%.*s] removed\n", + IB_DEVICE_NAME_MAX, sdev->ib_name); + kfree(sdev); + break; + } + } + write_unlock(&smbdirect_globals.devices.lock); +} + +static void smbdirect_ib_client_rename(struct ib_device *ib_dev, void *client_data) +{ + struct smbdirect_device *sdev; + + write_lock(&smbdirect_globals.devices.lock); + list_for_each_entry(sdev, &smbdirect_globals.devices.list, list) { + if (sdev->ib_dev == ib_dev) { + pr_info("ib_dev[%.*s] renamed to [%.*s]\n", + IB_DEVICE_NAME_MAX, sdev->ib_name, + IB_DEVICE_NAME_MAX, ib_dev->name); + snprintf(sdev->ib_name, ARRAY_SIZE(sdev->ib_name), "%.*s", + IB_DEVICE_NAME_MAX, ib_dev->name); + break; + } + } + write_unlock(&smbdirect_globals.devices.lock); +} + +static struct ib_client smbdirect_ib_client = { + .name = "smbdirect_ib_client", + .add = smbdirect_ib_client_add, + .remove = smbdirect_ib_client_remove, + .rename = smbdirect_ib_client_rename, +}; + +static u8 smbdirect_netdev_find_rdma_capable_node_type(struct net_device *netdev) +{ + struct smbdirect_device *sdev; + u8 node_type = RDMA_NODE_UNSPECIFIED; + + read_lock(&smbdirect_globals.devices.lock); + list_for_each_entry(sdev, &smbdirect_globals.devices.list, list) { + u32 pi; + + rdma_for_each_port(sdev->ib_dev, pi) { + struct net_device *ndev; + + ndev = ib_device_get_netdev(sdev->ib_dev, pi); + if (!ndev) + continue; + + if (ndev == netdev) { + dev_put(ndev); + node_type = sdev->ib_dev->node_type; + goto out; + } + dev_put(ndev); + } + } +out: + read_unlock(&smbdirect_globals.devices.lock); + + if (node_type == RDMA_NODE_UNSPECIFIED) { + struct ib_device *ibdev; + + ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_UNKNOWN); + if (ibdev) { + node_type = smbdirect_ib_device_rdma_capable_node_type(ibdev); + ib_device_put(ibdev); + } + } + + return node_type; +} + +/* + * Returns RDMA_NODE_UNSPECIFIED when the netdev has + * no support for smbdirect capable rdma. + * + * Otherwise RDMA_NODE_RNIC is returned for iwarp devices + * and RDMA_NODE_IB_CA or Infiniband and RoCE (v1 and v2) + */ +u8 smbdirect_netdev_rdma_capable_node_type(struct net_device *netdev) +{ + struct net_device *lower_dev; + struct list_head *iter; + u8 node_type = RDMA_NODE_UNSPECIFIED; + + node_type = smbdirect_netdev_find_rdma_capable_node_type(netdev); + if (node_type != RDMA_NODE_UNSPECIFIED) + return node_type; + + /* check if netdev is bridge or VLAN */ + if (netif_is_bridge_master(netdev) || netdev->priv_flags & IFF_802_1Q_VLAN) + netdev_for_each_lower_dev(netdev, lower_dev, iter) { + node_type = smbdirect_netdev_find_rdma_capable_node_type(lower_dev); + if (node_type != RDMA_NODE_UNSPECIFIED) + return node_type; + } + + /* check if netdev is IPoIB safely without layer violation */ + if (netdev->type == ARPHRD_INFINIBAND) + return RDMA_NODE_IB_CA; + + return RDMA_NODE_UNSPECIFIED; +} +EXPORT_SYMBOL_GPL(smbdirect_netdev_rdma_capable_node_type); + +__init int smbdirect_devices_init(void) +{ + int ret; + + rwlock_init(&smbdirect_globals.devices.lock); + INIT_LIST_HEAD(&smbdirect_globals.devices.list); + + ret = ib_register_client(&smbdirect_ib_client); + if (ret) { + pr_crit("failed to ib_register_client: %d %1pe\n", + ret, SMBDIRECT_DEBUG_ERR_PTR(ret)); + return ret; + } + + return 0; +} + +__exit void smbdirect_devices_exit(void) +{ + struct smbdirect_device *sdev, *tmp; + + /* + * On exist we just cleanup so that + * smbdirect_ib_client_remove() won't + * print removals of devices. + */ + write_lock(&smbdirect_globals.devices.lock); + list_for_each_entry_safe(sdev, tmp, &smbdirect_globals.devices.list, list) { + list_del(&sdev->list); + kfree(sdev); + } + write_unlock(&smbdirect_globals.devices.lock); + + ib_unregister_client(&smbdirect_ib_client); +} diff --git a/fs/smb/smbdirect/internal.h b/fs/smb/smbdirect/internal.h new file mode 100644 index 000000000..e9959e6dc --- /dev/null +++ b/fs/smb/smbdirect/internal.h @@ -0,0 +1,141 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2025, Stefan Metzmacher + */ + +#ifndef __FS_SMB_COMMON_SMBDIRECT_INTERNAL_H__ +#define __FS_SMB_COMMON_SMBDIRECT_INTERNAL_H__ + +#define DEFAULT_SYMBOL_NAMESPACE "SMBDIRECT" +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include "pdu.h" + +#include + +struct smbdirect_module_state { + struct mutex mutex; + + struct { + struct workqueue_struct *accept; + struct workqueue_struct *connect; + struct workqueue_struct *idle; + struct workqueue_struct *refill; + struct workqueue_struct *immediate; + struct workqueue_struct *cleanup; + } workqueues; + + struct { + rwlock_t lock; + struct list_head list; + } devices; +}; + +extern struct smbdirect_module_state smbdirect_globals; + +#include "socket.h" + +struct smbdirect_device { + struct list_head list; + struct ib_device *ib_dev; + /* + * copy of ib_dev->name, + * in order to print renames + */ + char ib_name[IB_DEVICE_NAME_MAX]; +}; + +int smbdirect_socket_init_new(struct net *net, struct smbdirect_socket *sc); + +int smbdirect_socket_init_accepting(struct rdma_cm_id *id, struct smbdirect_socket *sc); + +void __smbdirect_socket_schedule_cleanup(struct smbdirect_socket *sc, + const char *macro_name, + unsigned int lvl, + const char *func, + unsigned int line, + int error, + enum smbdirect_socket_status *force_status); +#define smbdirect_socket_schedule_cleanup(__sc, __error) \ + __smbdirect_socket_schedule_cleanup(__sc, \ + "smbdirect_socket_schedule_cleanup", SMBDIRECT_LOG_ERR, \ + __func__, __LINE__, __error, NULL) +#define smbdirect_socket_schedule_cleanup_lvl(__sc, __lvl, __error) \ + __smbdirect_socket_schedule_cleanup(__sc, \ + "smbdirect_socket_schedule_cleanup_lvl", __lvl, \ + __func__, __LINE__, __error, NULL) +#define smbdirect_socket_schedule_cleanup_status(__sc, __lvl, __error, __status) do { \ + enum smbdirect_socket_status __force_status = __status; \ + __smbdirect_socket_schedule_cleanup(__sc, \ + "smbdirect_socket_schedule_cleanup_status", __lvl, \ + __func__, __LINE__, __error, &__force_status); \ +} while (0) + +void smbdirect_socket_destroy_sync(struct smbdirect_socket *sc); + +int smbdirect_socket_wait_for_credits(struct smbdirect_socket *sc, + enum smbdirect_socket_status expected_status, + int unexpected_errno, + wait_queue_head_t *waitq, + atomic_t *total_credits, + int needed); + +void smbdirect_connection_rdma_established(struct smbdirect_socket *sc); + +void smbdirect_connection_negotiation_done(struct smbdirect_socket *sc); + +int smbdirect_connection_create_qp(struct smbdirect_socket *sc); + +void smbdirect_connection_destroy_qp(struct smbdirect_socket *sc); + +int smbdirect_connection_create_mem_pools(struct smbdirect_socket *sc); + +void smbdirect_connection_destroy_mem_pools(struct smbdirect_socket *sc); + +struct smbdirect_send_io *smbdirect_connection_alloc_send_io(struct smbdirect_socket *sc); + +void smbdirect_connection_free_send_io(struct smbdirect_send_io *msg); + +struct smbdirect_recv_io *smbdirect_connection_get_recv_io(struct smbdirect_socket *sc); + +void smbdirect_connection_put_recv_io(struct smbdirect_recv_io *msg); + +void smbdirect_connection_reassembly_append_recv_io(struct smbdirect_socket *sc, + struct smbdirect_recv_io *msg, + u32 data_length); + +struct smbdirect_recv_io * +smbdirect_connection_reassembly_first_recv_io(struct smbdirect_socket *sc); + +void smbdirect_connection_negotiate_rdma_resources(struct smbdirect_socket *sc, + u8 peer_initiator_depth, + u8 peer_responder_resources, + const struct rdma_conn_param *param); + +void smbdirect_connection_idle_timer_work(struct work_struct *work); + +u16 smbdirect_connection_grant_recv_credits(struct smbdirect_socket *sc); + +int smbdirect_connection_post_send_wr(struct smbdirect_socket *sc, + struct ib_send_wr *wr); + +int smbdirect_connection_post_recv_io(struct smbdirect_recv_io *msg); + +void smbdirect_connection_recv_io_done(struct ib_cq *cq, struct ib_wc *wc); + +int smbdirect_connection_recv_io_refill(struct smbdirect_socket *sc); + +int smbdirect_connection_create_mr_list(struct smbdirect_socket *sc); + +void smbdirect_connection_destroy_mr_list(struct smbdirect_socket *sc); + +int smbdirect_accept_connect_request(struct smbdirect_socket *sc, + const struct rdma_conn_param *param); + +void smbdirect_accept_negotiate_finish(struct smbdirect_socket *sc, u32 ntstatus); + +__init int smbdirect_devices_init(void); +__exit void smbdirect_devices_exit(void); + +#endif /* __FS_SMB_COMMON_SMBDIRECT_INTERNAL_H__ */ diff --git a/fs/smb/smbdirect/listen.c b/fs/smb/smbdirect/listen.c new file mode 100644 index 000000000..2f78bcaed --- /dev/null +++ b/fs/smb/smbdirect/listen.c @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017, Microsoft Corporation. + * Copyright (C) 2018, LG Electronics. + * Copyright (c) 2025, Stefan Metzmacher + */ + +#include "internal.h" + +static int smbdirect_listen_rdma_event_handler(struct rdma_cm_id *id, + struct rdma_cm_event *event); + +int smbdirect_socket_listen(struct smbdirect_socket *sc, int backlog) +{ + int ret; + + if (backlog < 0) + return -EINVAL; + if (!backlog) + backlog = 1; /* use 1 as default for now */ + + if (sc->first_error) + return -EINVAL; + + if (sc->status != SMBDIRECT_SOCKET_CREATED) + return -EINVAL; + + if (WARN_ON_ONCE(!sc->rdma.cm_id)) + return -EINVAL; + + if (sc->rdma.cm_id->device) + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "try to listen on addr: %pISpsfc dev: %.*s\n", + &sc->rdma.cm_id->route.addr.src_addr, + IB_DEVICE_NAME_MAX, + sc->rdma.cm_id->device->name); + else + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "try to listen on addr: %pISpsfc\n", + &sc->rdma.cm_id->route.addr.src_addr); + + /* already checked above */ + WARN_ON_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED); + sc->status = SMBDIRECT_SOCKET_LISTENING; + sc->rdma.expected_event = RDMA_CM_EVENT_CONNECT_REQUEST; + rdma_lock_handler(sc->rdma.cm_id); + sc->rdma.cm_id->event_handler = smbdirect_listen_rdma_event_handler; + rdma_unlock_handler(sc->rdma.cm_id); + + ret = rdma_listen(sc->rdma.cm_id, backlog); + if (ret) { + sc->first_error = ret; + sc->status = SMBDIRECT_SOCKET_DISCONNECTED; + if (sc->rdma.cm_id->device) + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "listening failed %1pe on addr: %pISpsfc dev: %.*s\n", + SMBDIRECT_DEBUG_ERR_PTR(ret), + &sc->rdma.cm_id->route.addr.src_addr, + IB_DEVICE_NAME_MAX, + sc->rdma.cm_id->device->name); + else + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "listening failed %1pe on addr: %pISpsfc\n", + SMBDIRECT_DEBUG_ERR_PTR(ret), + &sc->rdma.cm_id->route.addr.src_addr); + return ret; + } + + /* + * This is a value > 0, checked above, + * so we are able to use sc->listen.backlog == -1, + * as indication that the socket was never + * a listener. + */ + sc->listen.backlog = backlog; + + if (sc->rdma.cm_id->device) + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "listening on addr: %pISpsfc dev: %.*s\n", + &sc->rdma.cm_id->route.addr.src_addr, + IB_DEVICE_NAME_MAX, + sc->rdma.cm_id->device->name); + else + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "listening on addr: %pISpsfc\n", + &sc->rdma.cm_id->route.addr.src_addr); + + /* + * The rest happens async via smbdirect_listen_rdma_event_handler() + */ + return 0; +} +EXPORT_SYMBOL_GPL(smbdirect_socket_listen); + +static int smbdirect_new_rdma_event_handler(struct rdma_cm_id *new_id, + struct rdma_cm_event *event) +{ + int ret = -ESTALE; + + /* + * This should be replaced before any real work + * starts! So it should never be called! + */ + + if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) + ret = -ENETDOWN; + if (IS_ERR(SMBDIRECT_DEBUG_ERR_PTR(event->status))) + ret = event->status; + WARN_ONCE(1, + "%s should not be called! event=%s status=%d => ret=%1pe\n", + __func__, + rdma_event_msg(event->event), + event->status, + SMBDIRECT_DEBUG_ERR_PTR(ret)); + return -ESTALE; +} + +static int smbdirect_listen_connect_request(struct smbdirect_socket *lsc, + struct rdma_cm_id *new_id, + const struct rdma_cm_event *event); + +static int smbdirect_listen_rdma_event_handler(struct rdma_cm_id *new_id, + struct rdma_cm_event *event) +{ + struct smbdirect_socket *lsc = new_id->context; + int ret; + + if (event->event == RDMA_CM_EVENT_CONNECT_REQUEST) { + new_id->context = NULL; + new_id->event_handler = smbdirect_new_rdma_event_handler; + } else + new_id = NULL; + + /* + * cma_cm_event_handler() has + * lockdep_assert_held(&id_priv->handler_mutex); + * + * Mutexes are not allowed in interrupts, + * and we rely on not being in an interrupt here, + * as we might sleep. + */ + WARN_ON_ONCE(in_interrupt()); + + if (event->status || event->event != lsc->rdma.expected_event) { + ret = -ECONNABORTED; + + if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) + ret = -ENETDOWN; + if (IS_ERR(SMBDIRECT_DEBUG_ERR_PTR(event->status))) + ret = event->status; + + smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_ERR, + "%s (first_error=%1pe, expected=%s) => event=%s status=%d => ret=%1pe\n", + smbdirect_socket_status_string(lsc->status), + SMBDIRECT_DEBUG_ERR_PTR(lsc->first_error), + rdma_event_msg(lsc->rdma.expected_event), + rdma_event_msg(event->event), + event->status, + SMBDIRECT_DEBUG_ERR_PTR(ret)); + + /* + * In case of error return it and let the caller + * destroy new_id + */ + smbdirect_socket_schedule_cleanup(lsc, ret); + return new_id ? ret : 0; + } + + smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_INFO, + "%s (first_error=%1pe) event=%s\n", + smbdirect_socket_status_string(lsc->status), + SMBDIRECT_DEBUG_ERR_PTR(lsc->first_error), + rdma_event_msg(event->event)); + + /* + * In case of error return it and let the caller + * destroy new_id + */ + if (lsc->first_error) + return new_id ? lsc->first_error : 0; + + switch (event->event) { + case RDMA_CM_EVENT_CONNECT_REQUEST: + WARN_ON_ONCE(lsc->status != SMBDIRECT_SOCKET_LISTENING); + + /* + * In case of error return it and let the caller + * destroy new_id + */ + ret = smbdirect_listen_connect_request(lsc, new_id, event); + if (ret) + return ret; + return 0; + + default: + break; + } + + /* + * This is an internal error + */ + WARN_ON_ONCE(lsc->rdma.expected_event != RDMA_CM_EVENT_CONNECT_REQUEST); + smbdirect_socket_schedule_cleanup(lsc, -EINVAL); + return 0; +} + +static int smbdirect_listen_connect_request(struct smbdirect_socket *lsc, + struct rdma_cm_id *new_id, + const struct rdma_cm_event *event) +{ + const struct smbdirect_socket_parameters *lsp = &lsc->parameters; + struct smbdirect_socket *nsc; + unsigned long flags; + size_t backlog = max_t(size_t, 1, lsc->listen.backlog); + size_t psockets; + size_t rsockets; + int ret; + + if (!smbdirect_frwr_is_supported(&new_id->device->attrs)) { + smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_ERR, + "Fast Registration Work Requests (FRWR) is not supported device %.*s\n", + IB_DEVICE_NAME_MAX, + new_id->device->name); + smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_ERR, + "Device capability flags = %llx max_fast_reg_page_list_len = %u\n", + new_id->device->attrs.device_cap_flags, + new_id->device->attrs.max_fast_reg_page_list_len); + return -EPROTONOSUPPORT; + } + + if (lsp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB && + !rdma_ib_or_roce(new_id->device, new_id->port_num)) { + smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_ERR, + "Not IB: device: %.*s IW:%u local: %pISpsfc remote: %pISpsfc\n", + IB_DEVICE_NAME_MAX, + new_id->device->name, + rdma_protocol_iwarp(new_id->device, new_id->port_num), + &new_id->route.addr.src_addr, + &new_id->route.addr.dst_addr); + return -EPROTONOSUPPORT; + } + if (lsp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW && + !rdma_protocol_iwarp(new_id->device, new_id->port_num)) { + smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_ERR, + "Not IW: device: %.*s IB:%u local: %pISpsfc remote: %pISpsfc\n", + IB_DEVICE_NAME_MAX, + new_id->device->name, + rdma_ib_or_roce(new_id->device, new_id->port_num), + &new_id->route.addr.src_addr, + &new_id->route.addr.dst_addr); + return -EPROTONOSUPPORT; + } + + spin_lock_irqsave(&lsc->listen.lock, flags); + psockets = list_count_nodes(&lsc->listen.pending); + rsockets = list_count_nodes(&lsc->listen.ready); + spin_unlock_irqrestore(&lsc->listen.lock, flags); + + if (psockets > backlog || + rsockets > backlog || + (psockets + rsockets) > backlog) { + smbdirect_log_rdma_event(lsc, SMBDIRECT_LOG_ERR, + "Backlog[%d][%zu] full pending[%zu] ready[%zu]\n", + lsc->listen.backlog, backlog, psockets, rsockets); + return -EBUSY; + } + + ret = smbdirect_socket_create_accepting(new_id, &nsc); + if (ret) + goto socket_init_failed; + + nsc->logging = lsc->logging; + ret = smbdirect_socket_set_initial_parameters(nsc, &lsc->parameters); + if (ret) + goto set_params_failed; + ret = smbdirect_socket_set_kernel_settings(nsc, + lsc->ib.poll_ctx, + lsc->send_io.mem.gfp_mask); + if (ret) + goto set_settings_failed; + + spin_lock_irqsave(&lsc->listen.lock, flags); + list_add_tail(&nsc->accept.list, &lsc->listen.pending); + nsc->accept.listener = lsc; + spin_unlock_irqrestore(&lsc->listen.lock, flags); + + ret = smbdirect_accept_connect_request(nsc, &event->param.conn); + if (ret) + goto accept_connect_failed; + + return 0; + +accept_connect_failed: + spin_lock_irqsave(&lsc->listen.lock, flags); + list_del_init(&nsc->accept.list); + nsc->accept.listener = NULL; + spin_unlock_irqrestore(&lsc->listen.lock, flags); +set_settings_failed: +set_params_failed: + /* + * The caller will destroy new_id + */ + nsc->ib.dev = NULL; + nsc->rdma.cm_id = NULL; + smbdirect_socket_release(nsc); +socket_init_failed: + return ret; +} diff --git a/fs/smb/smbdirect/main.c b/fs/smb/smbdirect/main.c new file mode 100644 index 000000000..606732fef --- /dev/null +++ b/fs/smb/smbdirect/main.c @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2025, Stefan Metzmacher + */ + +#include "internal.h" +#include + +struct smbdirect_module_state smbdirect_globals = { + .mutex = __MUTEX_INITIALIZER(smbdirect_globals.mutex), +}; + +static __init int smbdirect_module_init(void) +{ + int ret = -ENOMEM; + + pr_notice("subsystem loading...\n"); + mutex_lock(&smbdirect_globals.mutex); + + smbdirect_globals.workqueues.accept = alloc_workqueue("smbdirect-accept", + WQ_SYSFS | + WQ_PERCPU | + WQ_POWER_EFFICIENT, + 0); + if (smbdirect_globals.workqueues.accept == NULL) + goto alloc_accept_wq_failed; + + smbdirect_globals.workqueues.connect = alloc_workqueue("smbdirect-connect", + WQ_SYSFS | + WQ_PERCPU | + WQ_POWER_EFFICIENT, + 0); + if (smbdirect_globals.workqueues.connect == NULL) + goto alloc_connect_wq_failed; + + smbdirect_globals.workqueues.idle = alloc_workqueue("smbdirect-idle", + WQ_SYSFS | + WQ_PERCPU | + WQ_POWER_EFFICIENT, + 0); + if (smbdirect_globals.workqueues.idle == NULL) + goto alloc_idle_wq_failed; + + smbdirect_globals.workqueues.refill = alloc_workqueue("smbdirect-refill", + WQ_HIGHPRI | + WQ_SYSFS | + WQ_PERCPU | + WQ_POWER_EFFICIENT, + 0); + if (smbdirect_globals.workqueues.refill == NULL) + goto alloc_refill_wq_failed; + + smbdirect_globals.workqueues.immediate = alloc_workqueue("smbdirect-immediate", + WQ_HIGHPRI | + WQ_SYSFS | + WQ_PERCPU | + WQ_POWER_EFFICIENT, + 0); + if (smbdirect_globals.workqueues.immediate == NULL) + goto alloc_immediate_wq_failed; + + smbdirect_globals.workqueues.cleanup = alloc_workqueue("smbdirect-cleanup", + WQ_MEM_RECLAIM | + WQ_HIGHPRI | + WQ_SYSFS | + WQ_PERCPU | + WQ_POWER_EFFICIENT, + 0); + if (smbdirect_globals.workqueues.cleanup == NULL) + goto alloc_cleanup_wq_failed; + + ret = smbdirect_devices_init(); + if (ret) + goto devices_init_failed; + + mutex_unlock(&smbdirect_globals.mutex); + pr_notice("subsystem loaded\n"); + return 0; + +devices_init_failed: + destroy_workqueue(smbdirect_globals.workqueues.cleanup); +alloc_cleanup_wq_failed: + destroy_workqueue(smbdirect_globals.workqueues.immediate); +alloc_immediate_wq_failed: + destroy_workqueue(smbdirect_globals.workqueues.refill); +alloc_refill_wq_failed: + destroy_workqueue(smbdirect_globals.workqueues.idle); +alloc_idle_wq_failed: + destroy_workqueue(smbdirect_globals.workqueues.connect); +alloc_connect_wq_failed: + destroy_workqueue(smbdirect_globals.workqueues.accept); +alloc_accept_wq_failed: + mutex_unlock(&smbdirect_globals.mutex); + pr_crit("failed to loaded: %d (%1pe)\n", + ret, SMBDIRECT_DEBUG_ERR_PTR(ret)); + return ret; +} + +static __exit void smbdirect_module_exit(void) +{ + pr_notice("subsystem unloading...\n"); + mutex_lock(&smbdirect_globals.mutex); + + smbdirect_devices_exit(); + + destroy_workqueue(smbdirect_globals.workqueues.accept); + destroy_workqueue(smbdirect_globals.workqueues.connect); + destroy_workqueue(smbdirect_globals.workqueues.idle); + destroy_workqueue(smbdirect_globals.workqueues.refill); + destroy_workqueue(smbdirect_globals.workqueues.immediate); + destroy_workqueue(smbdirect_globals.workqueues.cleanup); + + mutex_unlock(&smbdirect_globals.mutex); + pr_notice("subsystem unloaded\n"); +} + +module_init(smbdirect_module_init); +module_exit(smbdirect_module_exit); + +MODULE_DESCRIPTION("smbdirect subsystem"); +MODULE_LICENSE("GPL"); diff --git a/fs/smb/smbdirect/mr.c b/fs/smb/smbdirect/mr.c new file mode 100644 index 000000000..15c6363a2 --- /dev/null +++ b/fs/smb/smbdirect/mr.c @@ -0,0 +1,496 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017, Microsoft Corporation. + * Copyright (c) 2025, Stefan Metzmacher + */ + +#include "internal.h" + +/* + * Allocate MRs used for RDMA read/write + * The number of MRs will not exceed hardware capability in responder_resources + * All MRs are kept in mr_list. The MR can be recovered after it's used + * Recovery is done in smbd_mr_recovery_work. The content of list entry changes + * as MRs are used and recovered for I/O, but the list links will not change + */ +int smbdirect_connection_create_mr_list(struct smbdirect_socket *sc) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + struct smbdirect_mr_io *mr; + int ret; + u32 i; + + if (sp->responder_resources == 0) { + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "responder_resources negotiated as 0\n"); + return -EINVAL; + } + + /* Allocate more MRs (2x) than hardware responder_resources */ + for (i = 0; i < sp->responder_resources * 2; i++) { + mr = kzalloc_obj(*mr); + if (!mr) { + ret = -ENOMEM; + goto kzalloc_mr_failed; + } + + kref_init(&mr->kref); + mutex_init(&mr->mutex); + + mr->mr = ib_alloc_mr(sc->ib.pd, + sc->mr_io.type, + sp->max_frmr_depth); + if (IS_ERR(mr->mr)) { + ret = PTR_ERR(mr->mr); + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "ib_alloc_mr failed ret=%d (%1pe) type=0x%x max_frmr_depth=%u\n", + ret, SMBDIRECT_DEBUG_ERR_PTR(ret), + sc->mr_io.type, sp->max_frmr_depth); + goto ib_alloc_mr_failed; + } + mr->sgt.sgl = kzalloc_objs(struct scatterlist, sp->max_frmr_depth); + if (!mr->sgt.sgl) { + ret = -ENOMEM; + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "failed to allocate sgl, max_frmr_depth=%u\n", + sp->max_frmr_depth); + goto kcalloc_sgl_failed; + } + mr->state = SMBDIRECT_MR_READY; + mr->socket = sc; + + list_add_tail(&mr->list, &sc->mr_io.all.list); + atomic_inc(&sc->mr_io.ready.count); + } + + return 0; + +kcalloc_sgl_failed: + ib_dereg_mr(mr->mr); +ib_alloc_mr_failed: + mutex_destroy(&mr->mutex); + kfree(mr); +kzalloc_mr_failed: + smbdirect_connection_destroy_mr_list(sc); + return ret; +} + +static void smbdirect_mr_io_disable_locked(struct smbdirect_mr_io *mr) +{ + struct smbdirect_socket *sc = mr->socket; + + lockdep_assert_held(&mr->mutex); + + if (mr->state == SMBDIRECT_MR_DISABLED) + return; + + if (mr->mr) + ib_dereg_mr(mr->mr); + if (mr->sgt.nents) + ib_dma_unmap_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir); + kfree(mr->sgt.sgl); + + mr->mr = NULL; + mr->sgt.sgl = NULL; + mr->sgt.nents = 0; + + mr->state = SMBDIRECT_MR_DISABLED; +} + +static void smbdirect_mr_io_free_locked(struct kref *kref) +{ + struct smbdirect_mr_io *mr = + container_of(kref, struct smbdirect_mr_io, kref); + + lockdep_assert_held(&mr->mutex); + + /* + * smbdirect_mr_io_disable_locked() should already be called! + */ + if (WARN_ON_ONCE(mr->state != SMBDIRECT_MR_DISABLED)) + smbdirect_mr_io_disable_locked(mr); + + mutex_unlock(&mr->mutex); + mutex_destroy(&mr->mutex); + kfree(mr); +} + +void smbdirect_connection_destroy_mr_list(struct smbdirect_socket *sc) +{ + struct smbdirect_mr_io *mr, *tmp; + LIST_HEAD(all_list); + unsigned long flags; + + spin_lock_irqsave(&sc->mr_io.all.lock, flags); + list_splice_tail_init(&sc->mr_io.all.list, &all_list); + spin_unlock_irqrestore(&sc->mr_io.all.lock, flags); + + list_for_each_entry_safe(mr, tmp, &all_list, list) { + mutex_lock(&mr->mutex); + + smbdirect_mr_io_disable_locked(mr); + list_del(&mr->list); + mr->socket = NULL; + + /* + * No kref_put_mutex() as it's already locked. + * + * If smbdirect_mr_io_free_locked() is called + * and the mutex is unlocked and mr is gone, + * in that case kref_put() returned 1. + * + * If kref_put() returned 0 we know that + * smbdirect_mr_io_free_locked() didn't + * run. Not by us nor by anyone else, as we + * still hold the mutex, so we need to unlock. + * + * If the mr is still registered it will + * be dangling (detached from the connection + * waiting for smbd_deregister_mr() to be + * called in order to free the memory. + */ + if (!kref_put(&mr->kref, smbdirect_mr_io_free_locked)) + mutex_unlock(&mr->mutex); + } +} + +/* + * Get a MR from mr_list. This function waits until there is at least one MR + * available in the list. There may be several CPUs issuing I/O trying to get MR + * at the same time, mr_list_lock is used to protect this situation. + */ +static struct smbdirect_mr_io * +smbdirect_connection_get_mr_io(struct smbdirect_socket *sc) +{ + struct smbdirect_mr_io *mr; + unsigned long flags; + int ret; + +again: + ret = wait_event_interruptible(sc->mr_io.ready.wait_queue, + atomic_read(&sc->mr_io.ready.count) || + sc->status != SMBDIRECT_SOCKET_CONNECTED); + if (ret) { + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "wait_event_interruptible ret=%d (%1pe)\n", + ret, SMBDIRECT_DEBUG_ERR_PTR(ret)); + return NULL; + } + + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "sc->status=%s sc->first_error=%1pe\n", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); + return NULL; + } + + spin_lock_irqsave(&sc->mr_io.all.lock, flags); + list_for_each_entry(mr, &sc->mr_io.all.list, list) { + if (mr->state == SMBDIRECT_MR_READY) { + mr->state = SMBDIRECT_MR_REGISTERED; + kref_get(&mr->kref); + spin_unlock_irqrestore(&sc->mr_io.all.lock, flags); + atomic_dec(&sc->mr_io.ready.count); + atomic_inc(&sc->mr_io.used.count); + return mr; + } + } + + spin_unlock_irqrestore(&sc->mr_io.all.lock, flags); + /* + * It is possible that we could fail to get MR because other processes may + * try to acquire a MR at the same time. If this is the case, retry it. + */ + goto again; +} + +static void smbdirect_connection_mr_io_register_done(struct ib_cq *cq, struct ib_wc *wc) +{ + struct smbdirect_mr_io *mr = + container_of(wc->wr_cqe, struct smbdirect_mr_io, cqe); + struct smbdirect_socket *sc = mr->socket; + + if (wc->status != IB_WC_SUCCESS) { + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "wc->status=%s opcode=%d\n", + ib_wc_status_msg(wc->status), wc->opcode); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + } +} + +static void smbdirect_connection_mr_io_local_inv_done(struct ib_cq *cq, struct ib_wc *wc) +{ + struct smbdirect_mr_io *mr = + container_of(wc->wr_cqe, struct smbdirect_mr_io, cqe); + struct smbdirect_socket *sc = mr->socket; + + mr->state = SMBDIRECT_MR_INVALIDATED; + if (wc->status != IB_WC_SUCCESS) { + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "invalidate failed status=%s\n", + ib_wc_status_msg(wc->status)); + smbdirect_socket_schedule_cleanup(sc, -ECONNABORTED); + } + complete(&mr->invalidate_done); +} + +/* + * Transcribe the pages from an iterator into an MR scatterlist. + */ +static int smbdirect_iter_to_sgt(struct iov_iter *iter, + struct sg_table *sgt, + unsigned int max_sg) +{ + int ret; + + memset(sgt->sgl, 0, max_sg * sizeof(struct scatterlist)); + + ret = extract_iter_to_sg(iter, iov_iter_count(iter), sgt, max_sg, 0); + WARN_ON(ret < 0); + if (sgt->nents > 0) + sg_mark_end(&sgt->sgl[sgt->nents - 1]); + + return ret; +} + +/* + * Register memory for RDMA read/write + * iter: the buffer to register memory with + * writing: true if this is a RDMA write (SMB read), false for RDMA read + * need_invalidate: true if this MR needs to be locally invalidated after I/O + * return value: the MR registered, NULL if failed. + */ +struct smbdirect_mr_io * +smbdirect_connection_register_mr_io(struct smbdirect_socket *sc, + struct iov_iter *iter, + bool writing, + bool need_invalidate) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + struct smbdirect_mr_io *mr; + int ret, num_pages, num_mapped; + struct ib_reg_wr *reg_wr; + + num_pages = iov_iter_npages(iter, sp->max_frmr_depth + 1); + if (num_pages > sp->max_frmr_depth) { + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "num_pages=%d max_frmr_depth=%d\n", + num_pages, sp->max_frmr_depth); + WARN_ON_ONCE(1); + return NULL; + } + + mr = smbdirect_connection_get_mr_io(sc); + if (!mr) { + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "smbdirect_connection_get_mr_io returning NULL\n"); + return NULL; + } + + mutex_lock(&mr->mutex); + + mr->dir = writing ? DMA_FROM_DEVICE : DMA_TO_DEVICE; + mr->need_invalidate = need_invalidate; + mr->sgt.nents = 0; + mr->sgt.orig_nents = 0; + + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_INFO, + "num_pages=%u count=%zu depth=%u\n", + num_pages, iov_iter_count(iter), sp->max_frmr_depth); + smbdirect_iter_to_sgt(iter, &mr->sgt, sp->max_frmr_depth); + + num_mapped = ib_dma_map_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir); + if (!num_mapped) { + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "ib_dma_map_sg num_pages=%u dir=%x num_mapped=%d\n", + num_pages, mr->dir, num_mapped); + ret = -EIO; + goto dma_map_error; + } + + ret = ib_map_mr_sg(mr->mr, mr->sgt.sgl, num_mapped, NULL, PAGE_SIZE); + if (ret != num_mapped) { + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "ib_map_mr_sg failed ret = %d num_mapped = %u\n", + ret, num_mapped); + if (ret >= 0) + ret = -EIO; + goto map_mr_error; + } + + ib_update_fast_reg_key(mr->mr, ib_inc_rkey(mr->mr->rkey)); + reg_wr = &mr->wr; + reg_wr->wr.opcode = IB_WR_REG_MR; + mr->cqe.done = smbdirect_connection_mr_io_register_done; + reg_wr->wr.wr_cqe = &mr->cqe; + reg_wr->wr.num_sge = 0; + reg_wr->wr.send_flags = IB_SEND_SIGNALED; + reg_wr->mr = mr->mr; + reg_wr->key = mr->mr->rkey; + reg_wr->access = writing ? + IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE : + IB_ACCESS_REMOTE_READ; + + /* + * There is no need for waiting for complemtion on ib_post_send + * on IB_WR_REG_MR. Hardware enforces a barrier and order of execution + * on the next ib_post_send when we actually send I/O to remote peer + */ + ret = ib_post_send(sc->ib.qp, ®_wr->wr, NULL); + if (!ret) { + /* + * smbdirect_connection_get_mr_io() gave us a reference + * via kref_get(&mr->kref), we keep that and let + * the caller use smbdirect_connection_deregister_mr_io() + * to remove it again. + */ + mutex_unlock(&mr->mutex); + return mr; + } + + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "ib_post_send failed ret=%d (%1pe) reg_wr->key=0x%x\n", + ret, SMBDIRECT_DEBUG_ERR_PTR(ret), reg_wr->key); + +map_mr_error: + ib_dma_unmap_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir); + +dma_map_error: + mr->sgt.nents = 0; + mr->state = SMBDIRECT_MR_ERROR; + atomic_dec(&sc->mr_io.used.count); + + smbdirect_socket_schedule_cleanup(sc, ret); + + /* + * smbdirect_connection_get_mr_io() gave us a reference + * via kref_get(&mr->kref), we need to remove it again + * on error. + * + * No kref_put_mutex() as it's already locked. + * + * If smbdirect_mr_io_free_locked() is called + * and the mutex is unlocked and mr is gone, + * in that case kref_put() returned 1. + * + * If kref_put() returned 0 we know that + * smbdirect_mr_io_free_locked() didn't + * run. Not by us nor by anyone else, as we + * still hold the mutex, so we need to unlock. + */ + if (!kref_put(&mr->kref, smbdirect_mr_io_free_locked)) + mutex_unlock(&mr->mutex); + return NULL; +} +EXPORT_SYMBOL_GPL(smbdirect_connection_register_mr_io); + +void smbdirect_mr_io_fill_buffer_descriptor(struct smbdirect_mr_io *mr, + struct smbdirect_buffer_descriptor_v1 *v1) +{ + mutex_lock(&mr->mutex); + if (mr->state == SMBDIRECT_MR_REGISTERED) { + v1->offset = cpu_to_le64(mr->mr->iova); + v1->token = cpu_to_le32(mr->mr->rkey); + v1->length = cpu_to_le32(mr->mr->length); + } else { + v1->offset = cpu_to_le64(U64_MAX); + v1->token = cpu_to_le32(U32_MAX); + v1->length = cpu_to_le32(U32_MAX); + } + mutex_unlock(&mr->mutex); +} +EXPORT_SYMBOL_GPL(smbdirect_mr_io_fill_buffer_descriptor); + +/* + * Deregister a MR after I/O is done + * This function may wait if remote invalidation is not used + * and we have to locally invalidate the buffer to prevent data is being + * modified by remote peer after upper layer consumes it + */ +void smbdirect_connection_deregister_mr_io(struct smbdirect_mr_io *mr) +{ + struct smbdirect_socket *sc = mr->socket; + int ret = 0; + +lock_again: + mutex_lock(&mr->mutex); + if (mr->state == SMBDIRECT_MR_DISABLED) + goto put_kref; + + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { + smbdirect_mr_io_disable_locked(mr); + goto put_kref; + } + + if (mr->need_invalidate) { + struct ib_send_wr *wr = &mr->inv_wr; + + /* Need to finish local invalidation before returning */ + wr->opcode = IB_WR_LOCAL_INV; + mr->cqe.done = smbdirect_connection_mr_io_local_inv_done; + wr->wr_cqe = &mr->cqe; + wr->num_sge = 0; + wr->ex.invalidate_rkey = mr->mr->rkey; + wr->send_flags = IB_SEND_SIGNALED; + + init_completion(&mr->invalidate_done); + ret = ib_post_send(sc->ib.qp, wr, NULL); + if (ret) { + smbdirect_log_rdma_mr(sc, SMBDIRECT_LOG_ERR, + "ib_post_send failed ret=%d (%1pe)\n", + ret, SMBDIRECT_DEBUG_ERR_PTR(ret)); + smbdirect_mr_io_disable_locked(mr); + smbdirect_socket_schedule_cleanup(sc, ret); + goto done; + } + + /* + * We still hold the reference to mr + * so we can unlock while waiting. + */ + mutex_unlock(&mr->mutex); + wait_for_completion(&mr->invalidate_done); + mr->need_invalidate = false; + goto lock_again; + } else + /* + * For remote invalidation, just set it to SMBDIRECT_MR_INVALIDATED + * and defer to mr_recovery_work to recover the MR for next use + */ + mr->state = SMBDIRECT_MR_INVALIDATED; + + if (mr->sgt.nents) { + ib_dma_unmap_sg(sc->ib.dev, mr->sgt.sgl, mr->sgt.nents, mr->dir); + mr->sgt.nents = 0; + } + + WARN_ONCE(mr->state != SMBDIRECT_MR_INVALIDATED, + "mr->state[%u] != SMBDIRECT_MR_INVALIDATED[%u]\n", + mr->state, SMBDIRECT_MR_INVALIDATED); + mr->state = SMBDIRECT_MR_READY; + if (atomic_inc_return(&sc->mr_io.ready.count) == 1) + wake_up(&sc->mr_io.ready.wait_queue); + +done: + atomic_dec(&sc->mr_io.used.count); + +put_kref: + /* + * No kref_put_mutex() as it's already locked. + * + * If smbdirect_mr_io_free_locked() is called + * and the mutex is unlocked and mr is gone, + * in that case kref_put() returned 1. + * + * If kref_put() returned 0 we know that + * smbdirect_mr_io_free_locked() didn't + * run. Not by us nor by anyone else, as we + * still hold the mutex, so we need to unlock + * and keep the mr in SMBDIRECT_MR_READY or + * SMBDIRECT_MR_ERROR state. + */ + if (!kref_put(&mr->kref, smbdirect_mr_io_free_locked)) + mutex_unlock(&mr->mutex); +} +EXPORT_SYMBOL_GPL(smbdirect_connection_deregister_mr_io); diff --git a/fs/smb/smbdirect/pdu.h b/fs/smb/smbdirect/pdu.h new file mode 100644 index 000000000..7693ba337 --- /dev/null +++ b/fs/smb/smbdirect/pdu.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2017 Stefan Metzmacher + */ + +#ifndef __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_PDU_H__ +#define __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_PDU_H__ + +#define SMBDIRECT_V1 0x0100 + +/* SMBD minimum receive size and fragmented sized defined in [MS-SMBD] */ +#define SMBDIRECT_MIN_RECEIVE_SIZE 128 +#define SMBDIRECT_MIN_FRAGMENTED_SIZE 131072 + +/* SMBD negotiation request packet [MS-SMBD] 2.2.1 */ +struct smbdirect_negotiate_req { + __le16 min_version; + __le16 max_version; + __le16 reserved; + __le16 credits_requested; + __le32 preferred_send_size; + __le32 max_receive_size; + __le32 max_fragmented_size; +} __packed; + +/* SMBD negotiation response packet [MS-SMBD] 2.2.2 */ +struct smbdirect_negotiate_resp { + __le16 min_version; + __le16 max_version; + __le16 negotiated_version; + __le16 reserved; + __le16 credits_requested; + __le16 credits_granted; + __le32 status; + __le32 max_readwrite_size; + __le32 preferred_send_size; + __le32 max_receive_size; + __le32 max_fragmented_size; +} __packed; + +#define SMBDIRECT_DATA_MIN_HDR_SIZE 0x14 +#define SMBDIRECT_DATA_OFFSET 0x18 + +#define SMBDIRECT_FLAG_RESPONSE_REQUESTED 0x0001 + +/* SMBD data transfer packet with payload [MS-SMBD] 2.2.3 */ +struct smbdirect_data_transfer { + __le16 credits_requested; + __le16 credits_granted; + __le16 flags; + __le16 reserved; + __le32 remaining_data_length; + __le32 data_offset; + __le32 data_length; + __le32 padding; + __u8 buffer[]; +} __packed; + +#endif /* __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_PDU_H__ */ diff --git a/fs/smb/smbdirect/rw.c b/fs/smb/smbdirect/rw.c new file mode 100644 index 000000000..6fe38042c --- /dev/null +++ b/fs/smb/smbdirect/rw.c @@ -0,0 +1,255 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017, Microsoft Corporation. + * Copyright (C) 2018, LG Electronics. + * Copyright (c) 2025, Stefan Metzmacher + */ + +#include "internal.h" + +static int smbdirect_connection_wait_for_rw_credits(struct smbdirect_socket *sc, + int credits) +{ + return smbdirect_socket_wait_for_credits(sc, + SMBDIRECT_SOCKET_CONNECTED, + -ENOTCONN, + &sc->rw_io.credits.wait_queue, + &sc->rw_io.credits.count, + credits); +} + +static int smbdirect_connection_calc_rw_credits(struct smbdirect_socket *sc, + const void *buf, + size_t len) +{ + return DIV_ROUND_UP(smbdirect_get_buf_page_count(buf, len), + sc->rw_io.credits.num_pages); +} + +static int smbdirect_connection_rdma_get_sg_list(void *buf, + size_t size, + struct scatterlist *sg_list, + size_t nentries) +{ + bool high = is_vmalloc_addr(buf); + struct page *page; + size_t offset, len; + int i = 0; + + if (size == 0 || nentries < smbdirect_get_buf_page_count(buf, size)) + return -EINVAL; + + offset = offset_in_page(buf); + buf -= offset; + while (size > 0) { + len = min_t(size_t, PAGE_SIZE - offset, size); + if (high) + page = vmalloc_to_page(buf); + else + page = kmap_to_page(buf); + + if (!sg_list) + return -EINVAL; + sg_set_page(sg_list, page, len, offset); + sg_list = sg_next(sg_list); + + buf += PAGE_SIZE; + size -= len; + offset = 0; + i++; + } + + return i; +} + +static void smbdirect_connection_rw_io_free(struct smbdirect_rw_io *msg, + enum dma_data_direction dir) +{ + struct smbdirect_socket *sc = msg->socket; + + rdma_rw_ctx_destroy(&msg->rdma_ctx, + sc->ib.qp, + sc->ib.qp->port, + msg->sgt.sgl, + msg->sgt.nents, + dir); + sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE); + kfree(msg); +} + +static void smbdirect_connection_rdma_rw_done(struct ib_cq *cq, struct ib_wc *wc, + enum dma_data_direction dir) +{ + struct smbdirect_rw_io *msg = + container_of(wc->wr_cqe, struct smbdirect_rw_io, cqe); + struct smbdirect_socket *sc = msg->socket; + + if (wc->status != IB_WC_SUCCESS) { + msg->error = -EIO; + pr_err("read/write error. opcode = %d, status = %s(%d)\n", + wc->opcode, ib_wc_status_msg(wc->status), wc->status); + if (wc->status != IB_WC_WR_FLUSH_ERR) + smbdirect_socket_schedule_cleanup(sc, msg->error); + } + + complete(msg->completion); +} + +static void smbdirect_connection_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc) +{ + smbdirect_connection_rdma_rw_done(cq, wc, DMA_FROM_DEVICE); +} + +static void smbdirect_connection_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) +{ + smbdirect_connection_rdma_rw_done(cq, wc, DMA_TO_DEVICE); +} + +int smbdirect_connection_rdma_xmit(struct smbdirect_socket *sc, + void *buf, size_t buf_len, + struct smbdirect_buffer_descriptor_v1 *desc, + size_t desc_len, + bool is_read) +{ + const struct smbdirect_socket_parameters *sp = &sc->parameters; + enum dma_data_direction direction = is_read ? DMA_FROM_DEVICE : DMA_TO_DEVICE; + struct smbdirect_rw_io *msg, *next_msg; + size_t i; + int ret; + DECLARE_COMPLETION_ONSTACK(completion); + struct ib_send_wr *first_wr; + LIST_HEAD(msg_list); + u8 *desc_buf; + int credits_needed; + size_t desc_buf_len, desc_num = 0; + + if (sc->status != SMBDIRECT_SOCKET_CONNECTED) + return -ENOTCONN; + + if (buf_len > sp->max_read_write_size) + return -EINVAL; + + /* calculate needed credits */ + credits_needed = 0; + desc_buf = buf; + for (i = 0; i < desc_len / sizeof(*desc); i++) { + if (!buf_len) + break; + + desc_buf_len = le32_to_cpu(desc[i].length); + if (!desc_buf_len) + return -EINVAL; + + if (desc_buf_len > buf_len) { + desc_buf_len = buf_len; + desc[i].length = cpu_to_le32(desc_buf_len); + buf_len = 0; + } + + credits_needed += smbdirect_connection_calc_rw_credits(sc, + desc_buf, + desc_buf_len); + desc_buf += desc_buf_len; + buf_len -= desc_buf_len; + desc_num++; + } + + smbdirect_log_rdma_rw(sc, SMBDIRECT_LOG_INFO, + "RDMA %s, len %zu, needed credits %d\n", + str_read_write(is_read), buf_len, credits_needed); + + ret = smbdirect_connection_wait_for_rw_credits(sc, credits_needed); + if (ret < 0) + return ret; + + /* build rdma_rw_ctx for each descriptor */ + desc_buf = buf; + for (i = 0; i < desc_num; i++) { + size_t page_count; + + msg = kzalloc_flex(*msg, sg_list, SG_CHUNK_SIZE, + sc->rw_io.mem.gfp_mask); + if (!msg) { + ret = -ENOMEM; + goto out; + } + + desc_buf_len = le32_to_cpu(desc[i].length); + page_count = smbdirect_get_buf_page_count(desc_buf, desc_buf_len); + + msg->socket = sc; + msg->cqe.done = is_read ? + smbdirect_connection_rdma_read_done : + smbdirect_connection_rdma_write_done; + msg->completion = &completion; + + msg->sgt.sgl = &msg->sg_list[0]; + ret = sg_alloc_table_chained(&msg->sgt, + page_count, + msg->sg_list, + SG_CHUNK_SIZE); + if (ret) { + ret = -ENOMEM; + goto free_msg; + } + + ret = smbdirect_connection_rdma_get_sg_list(desc_buf, + desc_buf_len, + msg->sgt.sgl, + msg->sgt.orig_nents); + if (ret < 0) + goto free_table; + + ret = rdma_rw_ctx_init(&msg->rdma_ctx, + sc->ib.qp, + sc->ib.qp->port, + msg->sgt.sgl, + page_count, + 0, + le64_to_cpu(desc[i].offset), + le32_to_cpu(desc[i].token), + direction); + if (ret < 0) { + pr_err("failed to init rdma_rw_ctx: %d\n", ret); + goto free_table; + } + + list_add_tail(&msg->list, &msg_list); + desc_buf += desc_buf_len; + } + + /* concatenate work requests of rdma_rw_ctxs */ + first_wr = NULL; + list_for_each_entry_reverse(msg, &msg_list, list) { + first_wr = rdma_rw_ctx_wrs(&msg->rdma_ctx, + sc->ib.qp, + sc->ib.qp->port, + &msg->cqe, + first_wr); + } + + ret = ib_post_send(sc->ib.qp, first_wr, NULL); + if (ret) { + pr_err("failed to post send wr for RDMA R/W: %d\n", ret); + goto out; + } + + msg = list_last_entry(&msg_list, struct smbdirect_rw_io, list); + wait_for_completion(&completion); + ret = msg->error; +out: + list_for_each_entry_safe(msg, next_msg, &msg_list, list) { + list_del(&msg->list); + smbdirect_connection_rw_io_free(msg, direction); + } + atomic_add(credits_needed, &sc->rw_io.credits.count); + wake_up(&sc->rw_io.credits.wait_queue); + return ret; + +free_table: + sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE); +free_msg: + kfree(msg); + goto out; +} +EXPORT_SYMBOL_GPL(smbdirect_connection_rdma_xmit); diff --git a/fs/smb/smbdirect/socket.c b/fs/smb/smbdirect/socket.c new file mode 100644 index 000000000..bb02df615 --- /dev/null +++ b/fs/smb/smbdirect/socket.c @@ -0,0 +1,791 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017, Microsoft Corporation. + * Copyright (c) 2025, Stefan Metzmacher + */ + +#include "internal.h" + +bool smbdirect_frwr_is_supported(const struct ib_device_attr *attrs) +{ + /* + * Test if FRWR (Fast Registration Work Requests) is supported on the + * device This implementation requires FRWR on RDMA read/write return + * value: true if it is supported + */ + + if (!(attrs->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) + return false; + if (attrs->max_fast_reg_page_list_len == 0) + return false; + return true; +} +EXPORT_SYMBOL_GPL(smbdirect_frwr_is_supported); + +static void smbdirect_socket_cleanup_work(struct work_struct *work); + +static int smbdirect_socket_rdma_event_handler(struct rdma_cm_id *id, + struct rdma_cm_event *event) +{ + struct smbdirect_socket *sc = id->context; + int ret = -ESTALE; + + /* + * This should be replaced before any real work + * starts! So it should never be called! + */ + + if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) + ret = -ENETDOWN; + if (IS_ERR(SMBDIRECT_DEBUG_ERR_PTR(event->status))) + ret = event->status; + pr_err("%s (first_error=%1pe, expected=%s) => event=%s status=%d => ret=%1pe\n", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error), + rdma_event_msg(sc->rdma.expected_event), + rdma_event_msg(event->event), + event->status, + SMBDIRECT_DEBUG_ERR_PTR(ret)); + WARN_ONCE(1, "%s should not be called!\n", __func__); + sc->rdma.cm_id = NULL; + return -ESTALE; +} + +int smbdirect_socket_init_new(struct net *net, struct smbdirect_socket *sc) +{ + struct rdma_cm_id *id; + int ret; + + smbdirect_socket_init(sc); + + id = rdma_create_id(net, + smbdirect_socket_rdma_event_handler, + sc, + RDMA_PS_TCP, + IB_QPT_RC); + if (IS_ERR(id)) { + pr_err("%s: rdma_create_id() failed %1pe\n", __func__, id); + return PTR_ERR(id); + } + + ret = rdma_set_afonly(id, 1); + if (ret) { + rdma_destroy_id(id); + pr_err("%s: rdma_set_afonly() failed %1pe\n", + __func__, SMBDIRECT_DEBUG_ERR_PTR(ret)); + return ret; + } + + sc->rdma.cm_id = id; + + INIT_WORK(&sc->disconnect_work, smbdirect_socket_cleanup_work); + + return 0; +} + +int smbdirect_socket_create_kern(struct net *net, struct smbdirect_socket **_sc) +{ + struct smbdirect_socket *sc; + int ret; + + ret = -ENOMEM; + sc = kzalloc_obj(*sc); + if (!sc) + goto alloc_failed; + + ret = smbdirect_socket_init_new(net, sc); + if (ret) + goto init_failed; + + kref_init(&sc->refs.destroy); + + *_sc = sc; + return 0; + +init_failed: + kfree(sc); +alloc_failed: + return ret; +} +EXPORT_SYMBOL_GPL(smbdirect_socket_create_kern); + +int smbdirect_socket_init_accepting(struct rdma_cm_id *id, struct smbdirect_socket *sc) +{ + smbdirect_socket_init(sc); + + sc->rdma.cm_id = id; + sc->rdma.cm_id->context = sc; + sc->rdma.cm_id->event_handler = smbdirect_socket_rdma_event_handler; + + sc->ib.dev = sc->rdma.cm_id->device; + + INIT_WORK(&sc->disconnect_work, smbdirect_socket_cleanup_work); + + return 0; +} + +int smbdirect_socket_create_accepting(struct rdma_cm_id *id, struct smbdirect_socket **_sc) +{ + struct smbdirect_socket *sc; + int ret; + + ret = -ENOMEM; + sc = kzalloc_obj(*sc); + if (!sc) + goto alloc_failed; + + ret = smbdirect_socket_init_accepting(id, sc); + if (ret) + goto init_failed; + + kref_init(&sc->refs.destroy); + + *_sc = sc; + return 0; + +init_failed: + kfree(sc); +alloc_failed: + return ret; +} +EXPORT_SYMBOL_GPL(smbdirect_socket_create_accepting); + +int smbdirect_socket_set_initial_parameters(struct smbdirect_socket *sc, + const struct smbdirect_socket_parameters *sp) +{ + /* + * This is only allowed before connect or accept + */ + WARN_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED, + "status=%s first_error=%1pe", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); + if (sc->status != SMBDIRECT_SOCKET_CREATED) + return -EINVAL; + + if (sp->flags & ~SMBDIRECT_FLAG_PORT_RANGE_MASK) + return -EINVAL; + + if (sp->initiator_depth > U8_MAX) + return -EINVAL; + if (sp->responder_resources > U8_MAX) + return -EINVAL; + + if (sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB && + sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW) + return -EINVAL; + else if (sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB) + rdma_restrict_node_type(sc->rdma.cm_id, RDMA_NODE_IB_CA); + else if (sp->flags & SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW) + rdma_restrict_node_type(sc->rdma.cm_id, RDMA_NODE_RNIC); + + /* + * Make a copy of the callers parameters + * from here we only work on the copy + * + * TODO: do we want consistency checking? + */ + sc->parameters = *sp; + + return 0; +} +EXPORT_SYMBOL_GPL(smbdirect_socket_set_initial_parameters); + +const struct smbdirect_socket_parameters * +smbdirect_socket_get_current_parameters(struct smbdirect_socket *sc) +{ + return &sc->parameters; +} +EXPORT_SYMBOL_GPL(smbdirect_socket_get_current_parameters); + +int smbdirect_socket_set_kernel_settings(struct smbdirect_socket *sc, + enum ib_poll_context poll_ctx, + gfp_t gfp_mask) +{ + /* + * This is only allowed before connect or accept + */ + WARN_ONCE(sc->status != SMBDIRECT_SOCKET_CREATED, + "status=%s first_error=%1pe", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); + if (sc->status != SMBDIRECT_SOCKET_CREATED) + return -EINVAL; + + sc->ib.poll_ctx = poll_ctx; + + sc->send_io.mem.gfp_mask = gfp_mask; + sc->recv_io.mem.gfp_mask = gfp_mask; + sc->rw_io.mem.gfp_mask = gfp_mask; + + return 0; +} +EXPORT_SYMBOL_GPL(smbdirect_socket_set_kernel_settings); + +void smbdirect_socket_set_logging(struct smbdirect_socket *sc, + void *private_ptr, + bool (*needed)(struct smbdirect_socket *sc, + void *private_ptr, + unsigned int lvl, + unsigned int cls), + void (*vaprintf)(struct smbdirect_socket *sc, + const char *func, + unsigned int line, + void *private_ptr, + unsigned int lvl, + unsigned int cls, + struct va_format *vaf)) +{ + sc->logging.private_ptr = private_ptr; + sc->logging.needed = needed; + sc->logging.vaprintf = vaprintf; +} +EXPORT_SYMBOL_GPL(smbdirect_socket_set_logging); + +static void smbdirect_socket_wake_up_all(struct smbdirect_socket *sc) +{ + /* + * Wake up all waiters in all wait queues + * in order to notice the broken connection. + */ + wake_up_all(&sc->status_wait); + wake_up_all(&sc->listen.wait_queue); + wake_up_all(&sc->send_io.bcredits.wait_queue); + wake_up_all(&sc->send_io.lcredits.wait_queue); + wake_up_all(&sc->send_io.credits.wait_queue); + wake_up_all(&sc->send_io.pending.zero_wait_queue); + wake_up_all(&sc->recv_io.reassembly.wait_queue); + wake_up_all(&sc->rw_io.credits.wait_queue); + wake_up_all(&sc->mr_io.ready.wait_queue); +} + +void __smbdirect_socket_schedule_cleanup(struct smbdirect_socket *sc, + const char *macro_name, + unsigned int lvl, + const char *func, + unsigned int line, + int error, + enum smbdirect_socket_status *force_status) +{ + struct smbdirect_socket *psc, *tsc; + unsigned long flags; + bool was_first = false; + + if (!sc->first_error) { + ___smbdirect_log_generic(sc, func, line, + lvl, + SMBDIRECT_LOG_RDMA_EVENT, + "%s(%1pe%s%s) called from %s in line=%u status=%s\n", + macro_name, + SMBDIRECT_DEBUG_ERR_PTR(error), + force_status ? ", " : "", + force_status ? smbdirect_socket_status_string(*force_status) : "", + func, line, + smbdirect_socket_status_string(sc->status)); + if (error) + sc->first_error = error; + else + sc->first_error = -ECONNABORTED; + was_first = true; + } + + /* + * make sure other work (than disconnect_work) + * is not queued again but here we don't block and avoid + * disable[_delayed]_work_sync() + */ + disable_work(&sc->connect.work); + disable_work(&sc->recv_io.posted.refill_work); + disable_work(&sc->idle.immediate_work); + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; + disable_delayed_work(&sc->idle.timer_work); + + /* + * In case we were a listener we need to + * disconnect all pending and ready sockets + * + * First we move ready sockets to pending again. + * + * Only a socket that was a listener (listen.backlog != -1) owns a + * populated listen.ready/pending list. Guarding on that also keeps + * lockdep quiet: without it, the listener holds sc->listen.lock while + * the loop recurses into each child psc, which takes psc->listen.lock. + * Those are always different instances of the same lock class -- a + * child never listens, so the nesting is strictly listener -> child + * and cannot really deadlock -- but lockdep only sees the class and + * reports "possible recursive locking". A child has empty listen + * lists and nothing to do here, so skipping it loses nothing, and a + * pending child stays on its listener's list for the free path + * (smbdirect_socket_destroy) to reap. + */ + if (sc->listen.backlog != -1) { /* was a listener */ + spin_lock_irqsave(&sc->listen.lock, flags); + list_splice_init(&sc->listen.ready, &sc->listen.pending); + list_for_each_entry_safe(psc, tsc, &sc->listen.pending, accept.list) + smbdirect_socket_schedule_cleanup(psc, sc->first_error); + spin_unlock_irqrestore(&sc->listen.lock, flags); + } + + switch (sc->status) { + case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED: + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED: + case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED: + case SMBDIRECT_SOCKET_NEGOTIATE_FAILED: + case SMBDIRECT_SOCKET_ERROR: + case SMBDIRECT_SOCKET_DISCONNECTING: + case SMBDIRECT_SOCKET_DISCONNECTED: + case SMBDIRECT_SOCKET_DESTROYED: + /* + * Keep the current error status + */ + break; + + case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED: + case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING: + sc->status = SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED; + break; + + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED: + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING: + sc->status = SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED; + break; + + case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED: + case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING: + sc->status = SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED; + break; + + case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED: + case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING: + sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED; + break; + + case SMBDIRECT_SOCKET_CREATED: + case SMBDIRECT_SOCKET_LISTENING: + sc->status = SMBDIRECT_SOCKET_DISCONNECTED; + break; + + case SMBDIRECT_SOCKET_CONNECTED: + sc->status = SMBDIRECT_SOCKET_ERROR; + break; + } + + if (force_status && (was_first || *force_status > sc->status)) + sc->status = *force_status; + + /* + * Wake up all waiters in all wait queues + * in order to notice the broken connection. + */ + smbdirect_socket_wake_up_all(sc); + + queue_work(sc->workqueues.cleanup, &sc->disconnect_work); +} + +static void smbdirect_socket_cleanup_work(struct work_struct *work) +{ + struct smbdirect_socket *sc = + container_of(work, struct smbdirect_socket, disconnect_work); + struct smbdirect_socket *psc, *tsc; + unsigned long flags; + + /* + * This should not never be called in an interrupt! + */ + WARN_ON_ONCE(in_interrupt()); + + if (!sc->first_error) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, + "%s called with first_error==0\n", + smbdirect_socket_status_string(sc->status)); + + sc->first_error = -ECONNABORTED; + } + + /* + * make sure this and other work is not queued again + * but here we don't block and avoid + * disable[_delayed]_work_sync() + */ + disable_work(&sc->disconnect_work); + disable_work(&sc->connect.work); + disable_work(&sc->recv_io.posted.refill_work); + disable_work(&sc->idle.immediate_work); + sc->idle.keepalive = SMBDIRECT_KEEPALIVE_NONE; + disable_delayed_work(&sc->idle.timer_work); + + /* + * In case we were a listener we need to + * disconnect all pending and ready sockets + * + * First we move ready sockets to pending again. + * + * Guarded on listen.backlog != -1 for the same reason as in + * __smbdirect_socket_schedule_cleanup(): only a listener owns a + * populated listen list, and skipping the block for a child avoids + * nesting psc->listen.lock under a listener's listen.lock (different + * instances of one class -- harmless, but lockdep cannot tell). + */ + if (sc->listen.backlog != -1) { /* was a listener */ + spin_lock_irqsave(&sc->listen.lock, flags); + list_splice_init(&sc->listen.ready, &sc->listen.pending); + list_for_each_entry_safe(psc, tsc, &sc->listen.pending, accept.list) + smbdirect_socket_schedule_cleanup(psc, sc->first_error); + spin_unlock_irqrestore(&sc->listen.lock, flags); + } + + switch (sc->status) { + case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED: + case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING: + case SMBDIRECT_SOCKET_NEGOTIATE_FAILED: + case SMBDIRECT_SOCKET_CONNECTED: + case SMBDIRECT_SOCKET_ERROR: + sc->status = SMBDIRECT_SOCKET_DISCONNECTING; + /* + * Make sure we hold the callback lock + * im order to coordinate with the + * rdma_event handlers, typically + * smbdirect_connection_rdma_event_handler(), + * and smbdirect_socket_destroy(). + * + * So that the order of ib_drain_qp() + * and rdma_disconnect() is controlled + * by the mutex. + */ + rdma_lock_handler(sc->rdma.cm_id); + rdma_disconnect(sc->rdma.cm_id); + rdma_unlock_handler(sc->rdma.cm_id); + break; + + case SMBDIRECT_SOCKET_CREATED: + case SMBDIRECT_SOCKET_LISTENING: + case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED: + case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING: + case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED: + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED: + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING: + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED: + case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED: + case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING: + case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED: + /* + * rdma_{accept,connect}() never reached + * RDMA_CM_EVENT_ESTABLISHED + */ + sc->status = SMBDIRECT_SOCKET_DISCONNECTED; + break; + + case SMBDIRECT_SOCKET_DISCONNECTING: + case SMBDIRECT_SOCKET_DISCONNECTED: + case SMBDIRECT_SOCKET_DESTROYED: + break; + } + + /* + * Wake up all waiters in all wait queues + * in order to notice the broken connection. + */ + smbdirect_socket_wake_up_all(sc); +} + +static void smbdirect_socket_destroy(struct smbdirect_socket *sc) +{ + struct smbdirect_socket *psc, *tsc; + size_t psockets; + struct smbdirect_recv_io *recv_io; + struct smbdirect_recv_io *recv_tmp; + LIST_HEAD(all_list); + LIST_HEAD(pending_list); + unsigned long flags; + + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "status=%s first_error=%1pe", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); + + /* + * This should not never be called in an interrupt! + */ + WARN_ON_ONCE(in_interrupt()); + + if (sc->status == SMBDIRECT_SOCKET_DESTROYED) + return; + + WARN_ONCE(sc->status != SMBDIRECT_SOCKET_DISCONNECTED, + "status=%s first_error=%1pe", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); + + /* + * The listener should clear this before we reach this + */ + WARN_ONCE(sc->accept.listener, + "status=%s first_error=%1pe", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); + + /* + * Wake up all waiters in all wait queues + * in order to notice the broken connection. + * + * Most likely this was already called via + * smbdirect_socket_cleanup_work(), but call it again... + */ + smbdirect_socket_wake_up_all(sc); + + disable_work_sync(&sc->disconnect_work); + disable_work_sync(&sc->connect.work); + disable_work_sync(&sc->recv_io.posted.refill_work); + disable_work_sync(&sc->idle.immediate_work); + disable_delayed_work_sync(&sc->idle.timer_work); + + if (sc->rdma.cm_id) + rdma_lock_handler(sc->rdma.cm_id); + + if (sc->ib.qp) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "drain qp\n"); + ib_drain_qp(sc->ib.qp); + } + + /* + * In case we were a listener we need to + * disconnect all pending and ready sockets + * + * We move ready sockets to pending again. + * + * Capture them here -- rdma_lock_handler(sc->rdma.cm_id) is held above, + * so a concurrent CM CONNECT_REQUEST cannot add more; sc->listen.lock + * below only protects the list splice itself -- but DEFER releasing + * them until the listener's cm_id is destroyed: + * + * - smbdirect_socket_release() -> smbdirect_socket_destroy() takes the + * child's own rdma_lock_handler() lock (&id_priv->handler_mutex). + * The listener's and the child's cm_id are always different + * instances, so the nesting cannot really deadlock, but lockdep only + * sees one lock class and reports "possible recursive locking". + * + * - rdma_destroy_id() of a child before the listener's own + * rdma_destroy_id() below lets _cma_cancel_listens() walk the freed + * child id_priv (KASAN slab-use-after-free in __mutex_lock()). + * + * The children are independent sockets whose teardown does not need + * the listener's handler lock. + */ + spin_lock_irqsave(&sc->listen.lock, flags); + list_splice_tail_init(&sc->listen.ready, &pending_list); + list_splice_tail_init(&sc->listen.pending, &pending_list); + spin_unlock_irqrestore(&sc->listen.lock, flags); + + /* It's not possible for upper layer to get to reassembly */ + if (sc->listen.backlog == -1) /* was not a listener */ + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "drain the reassembly queue\n"); + spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); + list_splice_tail_init(&sc->recv_io.reassembly.list, &all_list); + spin_unlock_irqrestore(&sc->recv_io.reassembly.lock, flags); + list_for_each_entry_safe(recv_io, recv_tmp, &all_list, list) + smbdirect_connection_put_recv_io(recv_io); + sc->recv_io.reassembly.data_length = 0; + + if (sc->listen.backlog == -1) /* was not a listener */ + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "freeing mr list\n"); + smbdirect_connection_destroy_mr_list(sc); + + if (sc->listen.backlog == -1) /* was not a listener */ + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "destroying qp\n"); + smbdirect_connection_destroy_qp(sc); + if (sc->rdma.cm_id) { + rdma_unlock_handler(sc->rdma.cm_id); + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "destroying cm_id\n"); + rdma_destroy_id(sc->rdma.cm_id); + sc->rdma.cm_id = NULL; + } + + /* + * The listener's rdma_lock_handler() lock is dropped and its cm_id is + * destroyed, so it is safe to release the child sockets captured + * above: each release recurses into smbdirect_socket_destroy() and + * takes that child's own handler_mutex without nesting it under the + * listener's, and _cma_cancel_listens() can no longer reach them. + */ + psockets = list_count_nodes(&pending_list); + if (sc->listen.backlog != -1) /* was a listener */ + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "release %zu pending sockets\n", psockets); + list_for_each_entry_safe(psc, tsc, &pending_list, accept.list) { + list_del_init(&psc->accept.list); + psc->accept.listener = NULL; + smbdirect_socket_release(psc); + } + if (sc->listen.backlog != -1) /* was a listener */ + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "released %zu pending sockets\n", psockets); + + if (sc->listen.backlog == -1) /* was not a listener */ + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "destroying mem pools\n"); + smbdirect_connection_destroy_mem_pools(sc); + + sc->status = SMBDIRECT_SOCKET_DESTROYED; + + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "rdma session destroyed\n"); +} + +void smbdirect_socket_destroy_sync(struct smbdirect_socket *sc) +{ + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "status=%s first_error=%1pe", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); + + /* + * This should not never be called in an interrupt! + */ + WARN_ON_ONCE(in_interrupt()); + + /* + * First we try to disable the work + * without disable_work_sync() in a + * non blocking way, if it's already + * running it will be handles by + * disable_work_sync() below. + * + * Here we just want to make sure queue_work() in + * smbdirect_socket_schedule_cleanup_lvl() + * is a no-op. + */ + disable_work(&sc->disconnect_work); + + if (!sc->first_error) + /* + * SMBDIRECT_LOG_INFO is enough here + * as this is the typical case where + * we terminate the connection ourself. + */ + smbdirect_socket_schedule_cleanup_lvl(sc, + SMBDIRECT_LOG_INFO, + -ESHUTDOWN); + + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "cancelling and disable disconnect_work\n"); + disable_work_sync(&sc->disconnect_work); + + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "destroying rdma session\n"); + if (sc->status < SMBDIRECT_SOCKET_DISCONNECTING) + smbdirect_socket_cleanup_work(&sc->disconnect_work); + if (sc->status < SMBDIRECT_SOCKET_DISCONNECTED) { + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "wait for transport being disconnected\n"); + wait_event(sc->status_wait, sc->status == SMBDIRECT_SOCKET_DISCONNECTED); + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "waited for transport being disconnected\n"); + } + + /* + * Once we reached SMBDIRECT_SOCKET_DISCONNECTED, + * we should call smbdirect_socket_destroy() + */ + smbdirect_socket_destroy(sc); + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, + "status=%s first_error=%1pe", + smbdirect_socket_status_string(sc->status), + SMBDIRECT_DEBUG_ERR_PTR(sc->first_error)); +} + +int smbdirect_socket_bind(struct smbdirect_socket *sc, struct sockaddr *addr) +{ + int ret; + + if (sc->status != SMBDIRECT_SOCKET_CREATED) + return -EINVAL; + + ret = rdma_bind_addr(sc->rdma.cm_id, addr); + if (ret) + return ret; + + return 0; +} +EXPORT_SYMBOL_GPL(smbdirect_socket_bind); + +void smbdirect_socket_shutdown(struct smbdirect_socket *sc) +{ + smbdirect_socket_schedule_cleanup_lvl(sc, SMBDIRECT_LOG_INFO, -ESHUTDOWN); +} +EXPORT_SYMBOL_GPL(smbdirect_socket_shutdown); + +static void smbdirect_socket_release_disconnect(struct kref *kref) +{ + struct smbdirect_socket *sc = + container_of(kref, struct smbdirect_socket, refs.disconnect); + + /* + * For now do a sync disconnect/destroy + */ + smbdirect_socket_destroy_sync(sc); +} + +static void smbdirect_socket_release_destroy(struct kref *kref) +{ + struct smbdirect_socket *sc = + container_of(kref, struct smbdirect_socket, refs.destroy); + + /* + * Do a sync disconnect/destroy... + * hopefully a no-op, as it should be already + * in DESTROYED state, before we free the memory. + */ + smbdirect_socket_destroy_sync(sc); + kfree(sc); +} + +void smbdirect_socket_release(struct smbdirect_socket *sc) +{ + /* + * We expect only 1 disconnect reference + * and if it is already 0, it's a use after free! + */ + WARN_ON_ONCE(kref_read(&sc->refs.disconnect) != 1); + WARN_ON(!kref_put(&sc->refs.disconnect, smbdirect_socket_release_disconnect)); + + /* + * This may not trigger smbdirect_socket_release_destroy(), + * if struct smbdirect_socket is embedded in another structure + * indicated by REFCOUNT_MAX. + */ + kref_put(&sc->refs.destroy, smbdirect_socket_release_destroy); +} +EXPORT_SYMBOL_GPL(smbdirect_socket_release); + +int smbdirect_socket_wait_for_credits(struct smbdirect_socket *sc, + enum smbdirect_socket_status expected_status, + int unexpected_errno, + wait_queue_head_t *waitq, + atomic_t *total_credits, + int needed) +{ + int ret; + + if (WARN_ON_ONCE(needed < 0)) + return -EINVAL; + + do { + if (atomic_sub_return(needed, total_credits) >= 0) + return 0; + + atomic_add(needed, total_credits); + ret = wait_event_interruptible(*waitq, + atomic_read(total_credits) >= needed || + sc->status != expected_status); + + if (sc->status != expected_status) + return unexpected_errno; + else if (ret < 0) + return ret; + } while (true); +} diff --git a/fs/smb/smbdirect/socket.h b/fs/smb/smbdirect/socket.h new file mode 100644 index 000000000..c09eddd8a --- /dev/null +++ b/fs/smb/smbdirect/socket.h @@ -0,0 +1,832 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2025 Stefan Metzmacher + */ + +#ifndef __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_SOCKET_H__ +#define __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_SOCKET_H__ + +#include +#include +#include +#include +#include +#include +#include +#include + +enum smbdirect_socket_status { + SMBDIRECT_SOCKET_CREATED, + SMBDIRECT_SOCKET_LISTENING, + SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED, + SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING, + SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED, + SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED, + SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING, + SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED, + SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED, + SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING, + SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED, + SMBDIRECT_SOCKET_NEGOTIATE_NEEDED, + SMBDIRECT_SOCKET_NEGOTIATE_RUNNING, + SMBDIRECT_SOCKET_NEGOTIATE_FAILED, + SMBDIRECT_SOCKET_CONNECTED, + SMBDIRECT_SOCKET_ERROR, + SMBDIRECT_SOCKET_DISCONNECTING, + SMBDIRECT_SOCKET_DISCONNECTED, + SMBDIRECT_SOCKET_DESTROYED +}; + +static __always_inline +const char *smbdirect_socket_status_string(enum smbdirect_socket_status status) +{ + switch (status) { + case SMBDIRECT_SOCKET_CREATED: + return "CREATED"; + case SMBDIRECT_SOCKET_LISTENING: + return "LISTENING"; + case SMBDIRECT_SOCKET_RESOLVE_ADDR_NEEDED: + return "RESOLVE_ADDR_NEEDED"; + case SMBDIRECT_SOCKET_RESOLVE_ADDR_RUNNING: + return "RESOLVE_ADDR_RUNNING"; + case SMBDIRECT_SOCKET_RESOLVE_ADDR_FAILED: + return "RESOLVE_ADDR_FAILED"; + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_NEEDED: + return "RESOLVE_ROUTE_NEEDED"; + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_RUNNING: + return "RESOLVE_ROUTE_RUNNING"; + case SMBDIRECT_SOCKET_RESOLVE_ROUTE_FAILED: + return "RESOLVE_ROUTE_FAILED"; + case SMBDIRECT_SOCKET_RDMA_CONNECT_NEEDED: + return "RDMA_CONNECT_NEEDED"; + case SMBDIRECT_SOCKET_RDMA_CONNECT_RUNNING: + return "RDMA_CONNECT_RUNNING"; + case SMBDIRECT_SOCKET_RDMA_CONNECT_FAILED: + return "RDMA_CONNECT_FAILED"; + case SMBDIRECT_SOCKET_NEGOTIATE_NEEDED: + return "NEGOTIATE_NEEDED"; + case SMBDIRECT_SOCKET_NEGOTIATE_RUNNING: + return "NEGOTIATE_RUNNING"; + case SMBDIRECT_SOCKET_NEGOTIATE_FAILED: + return "NEGOTIATE_FAILED"; + case SMBDIRECT_SOCKET_CONNECTED: + return "CONNECTED"; + case SMBDIRECT_SOCKET_ERROR: + return "ERROR"; + case SMBDIRECT_SOCKET_DISCONNECTING: + return "DISCONNECTING"; + case SMBDIRECT_SOCKET_DISCONNECTED: + return "DISCONNECTED"; + case SMBDIRECT_SOCKET_DESTROYED: + return "DESTROYED"; + } + + return ""; +} + +/* + * This can be used with %1pe to print errors as strings or '0' + * And it avoids warnings like: warn: passing zero to 'ERR_PTR' + * from smatch -p=kernel --pedantic + */ +static __always_inline +const void * __must_check SMBDIRECT_DEBUG_ERR_PTR(long error) +{ + if (error == 0) + return NULL; + return ERR_PTR(error); +} + +enum smbdirect_keepalive_status { + SMBDIRECT_KEEPALIVE_NONE, + SMBDIRECT_KEEPALIVE_PENDING, + SMBDIRECT_KEEPALIVE_SENT +}; + +struct smbdirect_socket { + enum smbdirect_socket_status status; + wait_queue_head_t status_wait; + int first_error; + + /* + * This points to the workqueues to + * be used for this socket. + */ + struct { + struct workqueue_struct *accept; + struct workqueue_struct *connect; + struct workqueue_struct *idle; + struct workqueue_struct *refill; + struct workqueue_struct *immediate; + struct workqueue_struct *cleanup; + } workqueues; + + struct work_struct disconnect_work; + + /* + * The reference counts. + */ + struct { + /* + * This holds the references by the + * frontend, typically the smb layer. + * + * It is typically 1 and a disconnect + * will happen if it reaches 0. + */ + struct kref disconnect; + + /* + * This holds the reference by the + * backend, the code that manages + * the lifetime of the whole + * struct smbdirect_socket, + * if this reaches 0 it can will + * be freed. + * + * Can be REFCOUNT_MAX is part + * of another structure. + * + * This is equal or higher than + * the disconnect refcount. + */ + struct kref destroy; + } refs; + + /* RDMA related */ + struct { + struct rdma_cm_id *cm_id; + /* + * The expected event in our current + * cm_id->event_handler, all other events + * are treated as an error. + */ + enum rdma_cm_event_type expected_event; + /* + * This is for iWarp MPA v1 + */ + bool legacy_iwarp; + } rdma; + + /* IB verbs related */ + struct { + struct ib_pd *pd; + enum ib_poll_context poll_ctx; + struct ib_cq *send_cq; + struct ib_cq *recv_cq; + + /* + * shortcuts for rdma.cm_id->{qp,device}; + */ + struct ib_qp *qp; + struct ib_device *dev; + } ib; + + struct smbdirect_socket_parameters parameters; + + /* + * The state for connect/negotiation + */ + struct { + spinlock_t lock; + struct work_struct work; + } connect; + + /* + * The state for keepalive and timeout handling + */ + struct { + enum smbdirect_keepalive_status keepalive; + struct work_struct immediate_work; + struct delayed_work timer_work; + } idle; + + /* + * The state for listen sockets + */ + struct { + spinlock_t lock; + struct list_head pending; + struct list_head ready; + wait_queue_head_t wait_queue; + /* + * This starts as -1 and a value != -1 + * means this socket was in LISTENING state + * before. Note the valid backlog can + * only be > 0. + */ + int backlog; + } listen; + + /* + * The state for sockets waiting + * for accept, either still waiting + * for the negotiation to finish + * or already ready with a usable + * connection. + */ + struct { + struct smbdirect_socket *listener; + struct list_head list; + } accept; + + /* + * The state for posted send buffers + */ + struct { + /* + * Memory pools for preallocating + * smbdirect_send_io buffers + */ + struct { + struct kmem_cache *cache; + mempool_t *pool; + gfp_t gfp_mask; + } mem; + + /* + * This is a coordination for smbdirect_send_batch. + * + * There's only one possible credit, which means + * only one instance is running at a time. + */ + struct { + atomic_t count; + wait_queue_head_t wait_queue; + } bcredits; + + /* + * The local credit state for ib_post_send() + */ + struct { + atomic_t count; + wait_queue_head_t wait_queue; + } lcredits; + + /* + * The remote credit state for the send side + */ + struct { + atomic_t count; + wait_queue_head_t wait_queue; + } credits; + + /* + * The state about posted/pending sends + */ + struct { + atomic_t count; + /* + * woken when count reached zero + */ + wait_queue_head_t zero_wait_queue; + } pending; + } send_io; + + /* + * The state for posted receive buffers + */ + struct { + /* + * The type of PDU we are expecting + */ + enum { + SMBDIRECT_EXPECT_NEGOTIATE_REQ = 1, + SMBDIRECT_EXPECT_NEGOTIATE_REP = 2, + SMBDIRECT_EXPECT_DATA_TRANSFER = 3, + } expected; + + /* + * Memory pools for preallocating + * smbdirect_recv_io buffers + */ + struct { + struct kmem_cache *cache; + mempool_t *pool; + gfp_t gfp_mask; + } mem; + + /* + * The list of free smbdirect_recv_io + * structures + */ + struct { + struct list_head list; + spinlock_t lock; + } free; + + /* + * The state for posted recv_io messages + * and the refill work struct. + */ + struct { + atomic_t count; + struct work_struct refill_work; + } posted; + + /* + * The credit state for the recv side + */ + struct { + u16 target; + atomic_t available; + atomic_t count; + } credits; + + /* + * The list of arrived non-empty smbdirect_recv_io + * structures + * + * This represents the reassembly queue. + */ + struct { + struct list_head list; + spinlock_t lock; + wait_queue_head_t wait_queue; + /* total data length of reassembly queue */ + int data_length; + int queue_length; + /* the offset to first buffer in reassembly queue */ + int first_entry_offset; + /* + * Indicate if we have received a full packet on the + * connection This is used to identify the first SMBD + * packet of a assembled payload (SMB packet) in + * reassembly queue so we can return a RFC1002 length to + * upper layer to indicate the length of the SMB packet + * received + */ + bool full_packet_received; + } reassembly; + } recv_io; + + /* + * The state for Memory registrations on the client + */ + struct { + enum ib_mr_type type; + + /* + * The list of free smbdirect_mr_io + * structures + */ + struct { + struct list_head list; + spinlock_t lock; + } all; + + /* + * The number of available MRs ready for memory registration + */ + struct { + atomic_t count; + wait_queue_head_t wait_queue; + } ready; + + /* + * The number of used MRs + */ + struct { + atomic_t count; + } used; + } mr_io; + + /* + * The state for RDMA read/write requests on the server + */ + struct { + /* + * Memory hints for + * smbdirect_rw_io structs + */ + struct { + gfp_t gfp_mask; + } mem; + + /* + * The credit state for the send side + */ + struct { + /* + * The maximum number of rw credits + */ + size_t max; + /* + * The number of pages per credit + */ + size_t num_pages; + atomic_t count; + wait_queue_head_t wait_queue; + } credits; + } rw_io; + + /* + * For debug purposes + */ + struct { + u64 get_receive_buffer; + u64 put_receive_buffer; + u64 enqueue_reassembly_queue; + u64 dequeue_reassembly_queue; + u64 send_empty; + } statistics; + + struct { + void *private_ptr; + bool (*needed)(struct smbdirect_socket *sc, + void *private_ptr, + unsigned int lvl, + unsigned int cls); + void (*vaprintf)(struct smbdirect_socket *sc, + const char *func, + unsigned int line, + void *private_ptr, + unsigned int lvl, + unsigned int cls, + struct va_format *vaf); + } logging; +}; + +static void __smbdirect_socket_disabled_work(struct work_struct *work) +{ + /* + * Should never be called as disable_[delayed_]work_sync() was used. + */ + WARN_ON_ONCE(1); +} + +static bool __smbdirect_log_needed(struct smbdirect_socket *sc, + void *private_ptr, + unsigned int lvl, + unsigned int cls) +{ + /* + * Should never be called, the caller should + * set it's own functions. + */ + WARN_ON_ONCE(1); + return false; +} + +static void __smbdirect_log_vaprintf(struct smbdirect_socket *sc, + const char *func, + unsigned int line, + void *private_ptr, + unsigned int lvl, + unsigned int cls, + struct va_format *vaf) +{ + /* + * Should never be called, the caller should + * set it's own functions. + */ + WARN_ON_ONCE(1); +} + +__printf(6, 7) +static void __smbdirect_log_printf(struct smbdirect_socket *sc, + const char *func, + unsigned int line, + unsigned int lvl, + unsigned int cls, + const char *fmt, + ...); +__maybe_unused +static void __smbdirect_log_printf(struct smbdirect_socket *sc, + const char *func, + unsigned int line, + unsigned int lvl, + unsigned int cls, + const char *fmt, + ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + sc->logging.vaprintf(sc, + func, + line, + sc->logging.private_ptr, + lvl, + cls, + &vaf); + va_end(args); +} + +#define ___smbdirect_log_generic(sc, func, line, lvl, cls, fmt, args...) do { \ + if (sc->logging.needed(sc, sc->logging.private_ptr, lvl, cls)) { \ + __smbdirect_log_printf(sc, func, line, lvl, cls, fmt, ##args); \ + } \ +} while (0) +#define __smbdirect_log_generic(sc, lvl, cls, fmt, args...) \ + ___smbdirect_log_generic(sc, __func__, __LINE__, lvl, cls, fmt, ##args) + +#define smbdirect_log_outgoing(sc, lvl, fmt, args...) \ + __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_OUTGOING, fmt, ##args) +#define smbdirect_log_incoming(sc, lvl, fmt, args...) \ + __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_INCOMING, fmt, ##args) +#define smbdirect_log_read(sc, lvl, fmt, args...) \ + __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_READ, fmt, ##args) +#define smbdirect_log_write(sc, lvl, fmt, args...) \ + __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_WRITE, fmt, ##args) +#define smbdirect_log_rdma_send(sc, lvl, fmt, args...) \ + __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_RDMA_SEND, fmt, ##args) +#define smbdirect_log_rdma_recv(sc, lvl, fmt, args...) \ + __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_RDMA_RECV, fmt, ##args) +#define smbdirect_log_keep_alive(sc, lvl, fmt, args...) \ + __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_KEEP_ALIVE, fmt, ##args) +#define smbdirect_log_rdma_event(sc, lvl, fmt, args...) \ + __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_RDMA_EVENT, fmt, ##args) +#define smbdirect_log_rdma_mr(sc, lvl, fmt, args...) \ + __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_RDMA_MR, fmt, ##args) +#define smbdirect_log_rdma_rw(sc, lvl, fmt, args...) \ + __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_RDMA_RW, fmt, ##args) +#define smbdirect_log_negotiate(sc, lvl, fmt, args...) \ + __smbdirect_log_generic(sc, lvl, SMBDIRECT_LOG_NEGOTIATE, fmt, ##args) + +static __always_inline void smbdirect_socket_init(struct smbdirect_socket *sc) +{ + /* + * This also sets status = SMBDIRECT_SOCKET_CREATED + */ + BUILD_BUG_ON(SMBDIRECT_SOCKET_CREATED != 0); + memset(sc, 0, sizeof(*sc)); + + init_waitqueue_head(&sc->status_wait); + + sc->workqueues.accept = smbdirect_globals.workqueues.accept; + sc->workqueues.connect = smbdirect_globals.workqueues.connect; + sc->workqueues.idle = smbdirect_globals.workqueues.idle; + sc->workqueues.refill = smbdirect_globals.workqueues.refill; + sc->workqueues.immediate = smbdirect_globals.workqueues.immediate; + sc->workqueues.cleanup = smbdirect_globals.workqueues.cleanup; + + INIT_WORK(&sc->disconnect_work, __smbdirect_socket_disabled_work); + disable_work_sync(&sc->disconnect_work); + + kref_init(&sc->refs.disconnect); + sc->refs.destroy = (struct kref) KREF_INIT(REFCOUNT_MAX); + + sc->rdma.expected_event = RDMA_CM_EVENT_INTERNAL; + + sc->ib.poll_ctx = IB_POLL_UNBOUND_WORKQUEUE; + + spin_lock_init(&sc->connect.lock); + INIT_WORK(&sc->connect.work, __smbdirect_socket_disabled_work); + disable_work_sync(&sc->connect.work); + + INIT_WORK(&sc->idle.immediate_work, __smbdirect_socket_disabled_work); + disable_work_sync(&sc->idle.immediate_work); + INIT_DELAYED_WORK(&sc->idle.timer_work, __smbdirect_socket_disabled_work); + disable_delayed_work_sync(&sc->idle.timer_work); + + spin_lock_init(&sc->listen.lock); + INIT_LIST_HEAD(&sc->listen.pending); + INIT_LIST_HEAD(&sc->listen.ready); + sc->listen.backlog = -1; /* not a listener */ + init_waitqueue_head(&sc->listen.wait_queue); + + INIT_LIST_HEAD(&sc->accept.list); + + sc->send_io.mem.gfp_mask = GFP_KERNEL; + + atomic_set(&sc->send_io.bcredits.count, 0); + init_waitqueue_head(&sc->send_io.bcredits.wait_queue); + + atomic_set(&sc->send_io.lcredits.count, 0); + init_waitqueue_head(&sc->send_io.lcredits.wait_queue); + + atomic_set(&sc->send_io.credits.count, 0); + init_waitqueue_head(&sc->send_io.credits.wait_queue); + + atomic_set(&sc->send_io.pending.count, 0); + init_waitqueue_head(&sc->send_io.pending.zero_wait_queue); + + sc->recv_io.mem.gfp_mask = GFP_KERNEL; + + INIT_LIST_HEAD(&sc->recv_io.free.list); + spin_lock_init(&sc->recv_io.free.lock); + + atomic_set(&sc->recv_io.posted.count, 0); + INIT_WORK(&sc->recv_io.posted.refill_work, __smbdirect_socket_disabled_work); + disable_work_sync(&sc->recv_io.posted.refill_work); + + atomic_set(&sc->recv_io.credits.available, 0); + atomic_set(&sc->recv_io.credits.count, 0); + + INIT_LIST_HEAD(&sc->recv_io.reassembly.list); + spin_lock_init(&sc->recv_io.reassembly.lock); + init_waitqueue_head(&sc->recv_io.reassembly.wait_queue); + + sc->rw_io.mem.gfp_mask = GFP_KERNEL; + atomic_set(&sc->rw_io.credits.count, 0); + init_waitqueue_head(&sc->rw_io.credits.wait_queue); + + spin_lock_init(&sc->mr_io.all.lock); + INIT_LIST_HEAD(&sc->mr_io.all.list); + atomic_set(&sc->mr_io.ready.count, 0); + init_waitqueue_head(&sc->mr_io.ready.wait_queue); + atomic_set(&sc->mr_io.used.count, 0); + + sc->logging.private_ptr = NULL; + sc->logging.needed = __smbdirect_log_needed; + sc->logging.vaprintf = __smbdirect_log_vaprintf; +} + +#define __SMBDIRECT_CHECK_STATUS_FAILED(__sc, __expected_status, __error_cmd, __unexpected_cmd) ({ \ + bool __failed = false; \ + if (unlikely((__sc)->first_error)) { \ + __failed = true; \ + __error_cmd \ + } else if (unlikely((__sc)->status != (__expected_status))) { \ + __failed = true; \ + __unexpected_cmd \ + } \ + __failed; \ +}) + +#define __SMBDIRECT_CHECK_STATUS_WARN(__sc, __expected_status, __unexpected_cmd) \ + __SMBDIRECT_CHECK_STATUS_FAILED(__sc, __expected_status, \ + { \ + const struct sockaddr_storage *__src = NULL; \ + const struct sockaddr_storage *__dst = NULL; \ + if ((__sc)->rdma.cm_id) { \ + __src = &(__sc)->rdma.cm_id->route.addr.src_addr; \ + __dst = &(__sc)->rdma.cm_id->route.addr.dst_addr; \ + } \ + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_INFO, \ + "expected[%s] != %s first_error=%1pe local=%pISpsfc remote=%pISpsfc\n", \ + smbdirect_socket_status_string(__expected_status), \ + smbdirect_socket_status_string((__sc)->status), \ + SMBDIRECT_DEBUG_ERR_PTR((__sc)->first_error), \ + __src, __dst); \ + }, \ + { \ + const struct sockaddr_storage *__src = NULL; \ + const struct sockaddr_storage *__dst = NULL; \ + if ((__sc)->rdma.cm_id) { \ + __src = &(__sc)->rdma.cm_id->route.addr.src_addr; \ + __dst = &(__sc)->rdma.cm_id->route.addr.dst_addr; \ + } \ + smbdirect_log_rdma_event(sc, SMBDIRECT_LOG_ERR, \ + "expected[%s] != %s first_error=%1pe local=%pISpsfc remote=%pISpsfc\n", \ + smbdirect_socket_status_string(__expected_status), \ + smbdirect_socket_status_string((__sc)->status), \ + SMBDIRECT_DEBUG_ERR_PTR((__sc)->first_error), \ + __src, __dst); \ + WARN_ONCE(1, \ + "expected[%s] != %s first_error=%1pe local=%pISpsfc remote=%pISpsfc\n", \ + smbdirect_socket_status_string(__expected_status), \ + smbdirect_socket_status_string((__sc)->status), \ + SMBDIRECT_DEBUG_ERR_PTR((__sc)->first_error), \ + __src, __dst); \ + __unexpected_cmd \ + }) + +#define SMBDIRECT_CHECK_STATUS_WARN(__sc, __expected_status) \ + __SMBDIRECT_CHECK_STATUS_WARN(__sc, __expected_status, /* nothing */) + +#ifndef __SMBDIRECT_SOCKET_DISCONNECT +#define __SMBDIRECT_SOCKET_DISCONNECT(__sc) \ + smbdirect_socket_schedule_cleanup(__sc, -ECONNABORTED) +#endif /* ! __SMBDIRECT_SOCKET_DISCONNECT */ + +#define SMBDIRECT_CHECK_STATUS_DISCONNECT(__sc, __expected_status) \ + __SMBDIRECT_CHECK_STATUS_WARN(__sc, __expected_status, \ + __SMBDIRECT_SOCKET_DISCONNECT(__sc);) + +struct smbdirect_send_io { + struct smbdirect_socket *socket; + struct ib_cqe cqe; + + /* + * The SGE entries for this work request + * + * The first points to the packet header + */ +#define SMBDIRECT_SEND_IO_MAX_SGE 6 + size_t num_sge; + struct ib_sge sge[SMBDIRECT_SEND_IO_MAX_SGE]; + + /* + * Link to the list of sibling smbdirect_send_io + * messages. + */ + struct list_head sibling_list; + struct ib_send_wr wr; + + /* SMBD packet header follows this structure */ + u8 packet[]; +}; + +struct smbdirect_send_batch { + /* + * List of smbdirect_send_io messages + */ + struct list_head msg_list; + /* + * Number of list entries + */ + size_t wr_cnt; + + /* + * Possible remote key invalidation state + */ + bool need_invalidate_rkey; + u32 remote_key; + + int credit; +}; + +struct smbdirect_recv_io { + struct smbdirect_socket *socket; + struct ib_cqe cqe; + + /* + * For now we only use a single SGE + * as we have just one large buffer + * per posted recv. + */ +#define SMBDIRECT_RECV_IO_MAX_SGE 1 + struct ib_sge sge; + + /* Link to free or reassembly list */ + struct list_head list; + + /* Indicate if this is the 1st packet of a payload */ + bool first_segment; + + /* SMBD packet header and payload follows this structure */ + u8 packet[]; +}; + +enum smbdirect_mr_state { + SMBDIRECT_MR_READY, + SMBDIRECT_MR_REGISTERED, + SMBDIRECT_MR_INVALIDATED, + SMBDIRECT_MR_ERROR, + SMBDIRECT_MR_DISABLED +}; + +struct smbdirect_mr_io { + struct smbdirect_socket *socket; + struct ib_cqe cqe; + + /* + * We can have up to two references: + * 1. by the connection + * 2. by the registration + */ + struct kref kref; + struct mutex mutex; + + struct list_head list; + + enum smbdirect_mr_state state; + struct ib_mr *mr; + struct sg_table sgt; + enum dma_data_direction dir; + union { + struct ib_reg_wr wr; + struct ib_send_wr inv_wr; + }; + + bool need_invalidate; + struct completion invalidate_done; +}; + +struct smbdirect_rw_io { + struct smbdirect_socket *socket; + struct ib_cqe cqe; + + struct list_head list; + + int error; + struct completion *completion; + + struct rdma_rw_ctx rdma_ctx; + struct sg_table sgt; + struct scatterlist sg_list[]; +}; + +static inline size_t smbdirect_get_buf_page_count(const void *buf, size_t size) +{ + return DIV_ROUND_UP((uintptr_t)buf + size, PAGE_SIZE) - + (uintptr_t)buf / PAGE_SIZE; +} + +/* + * Maximum number of retries on data transfer operations + */ +#define SMBDIRECT_RDMA_CM_RETRY 6 +/* + * No need to retry on Receiver Not Ready since SMB_DIRECT manages credits + */ +#define SMBDIRECT_RDMA_CM_RNR_RETRY 0 + +#endif /* __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_SOCKET_H__ */ diff --git a/fs/splice.c b/fs/splice.c new file mode 100644 index 000000000..9d8f63e2f --- /dev/null +++ b/fs/splice.c @@ -0,0 +1,1994 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * "splice": joining two ropes together by interweaving their strands. + * + * This is the "extended pipe" functionality, where a pipe is used as + * an arbitrary in-memory buffer. Think of a pipe as a small kernel + * buffer that you can use to transfer data from one end to the other. + * + * The traditional unix read/write is extended with a "splice()" operation + * that transfers data buffers to or from a pipe buffer. + * + * Named by Larry McVoy, original implementation from Linus, extended by + * Jens to support splicing to files, network, direct splicing, etc and + * fixing lots of bugs. + * + * Copyright (C) 2005-2006 Jens Axboe + * Copyright (C) 2005-2006 Linus Torvalds + * Copyright (C) 2006 Ingo Molnar + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +/* + * Splice doesn't support FMODE_NOWAIT. Since pipes may set this flag to + * indicate they support non-blocking reads or writes, we must clear it + * here if set to avoid blocking other users of this pipe if splice is + * being done on it. + */ +static noinline void pipe_clear_nowait(struct file *file) +{ + fmode_t fmode = READ_ONCE(file->f_mode); + + do { + if (!(fmode & FMODE_NOWAIT)) + break; + } while (!try_cmpxchg(&file->f_mode, &fmode, fmode & ~FMODE_NOWAIT)); +} + +/* + * Attempt to steal a page from a pipe buffer. This should perhaps go into + * a vm helper function, it's already simplified quite a bit by the + * addition of remove_mapping(). If success is returned, the caller may + * attempt to reuse this page for another destination. + */ +static bool page_cache_pipe_buf_try_steal(struct pipe_inode_info *pipe, + struct pipe_buffer *buf) +{ + struct folio *folio = page_folio(buf->page); + struct address_space *mapping; + + folio_lock(folio); + + mapping = folio_mapping(folio); + if (mapping) { + WARN_ON(!folio_test_uptodate(folio)); + + /* + * At least for ext2 with nobh option, we need to wait on + * writeback completing on this folio, since we'll remove it + * from the pagecache. Otherwise truncate wont wait on the + * folio, allowing the disk blocks to be reused by someone else + * before we actually wrote our data to them. fs corruption + * ensues. + */ + folio_wait_writeback(folio); + + if (!filemap_release_folio(folio, GFP_KERNEL)) + goto out_unlock; + + /* + * If we succeeded in removing the mapping, set LRU flag + * and return good. + */ + if (remove_mapping(mapping, folio)) { + buf->flags |= PIPE_BUF_FLAG_LRU; + return true; + } + } + + /* + * Raced with truncate or failed to remove folio from current + * address space, unlock and return failure. + */ +out_unlock: + folio_unlock(folio); + return false; +} + +static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe, + struct pipe_buffer *buf) +{ + put_page(buf->page); + buf->flags &= ~PIPE_BUF_FLAG_LRU; +} + +/* + * Check whether the contents of buf is OK to access. Since the content + * is a page cache page, IO may be in flight. + */ +static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe, + struct pipe_buffer *buf) +{ + struct folio *folio = page_folio(buf->page); + int err; + + if (!folio_test_uptodate(folio)) { + folio_lock(folio); + + /* + * Folio got truncated/unhashed. This will cause a 0-byte + * splice, if this is the first page. + */ + if (!folio->mapping) { + err = -ENODATA; + goto error; + } + + /* + * Uh oh, read-error from disk. + */ + if (!folio_test_uptodate(folio)) { + err = -EIO; + goto error; + } + + /* Folio is ok after all, we are done */ + folio_unlock(folio); + } + + return 0; +error: + folio_unlock(folio); + return err; +} + +const struct pipe_buf_operations page_cache_pipe_buf_ops = { + .confirm = page_cache_pipe_buf_confirm, + .release = page_cache_pipe_buf_release, + .try_steal = page_cache_pipe_buf_try_steal, + .get = generic_pipe_buf_get, +}; + +static bool user_page_pipe_buf_try_steal(struct pipe_inode_info *pipe, + struct pipe_buffer *buf) +{ + if (!(buf->flags & PIPE_BUF_FLAG_GIFT)) + return false; + + buf->flags |= PIPE_BUF_FLAG_LRU; + return generic_pipe_buf_try_steal(pipe, buf); +} + +static const struct pipe_buf_operations user_page_pipe_buf_ops = { + .release = page_cache_pipe_buf_release, + .try_steal = user_page_pipe_buf_try_steal, + .get = generic_pipe_buf_get, +}; + +static void wakeup_pipe_readers(struct pipe_inode_info *pipe) +{ + smp_mb(); + if (waitqueue_active(&pipe->rd_wait)) + wake_up_interruptible(&pipe->rd_wait); + kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); +} + +/** + * splice_to_pipe - fill passed data into a pipe + * @pipe: pipe to fill + * @spd: data to fill + * + * Description: + * @spd contains a map of pages and len/offset tuples, along with + * the struct pipe_buf_operations associated with these pages. This + * function will link that data to the pipe. + * + */ +ssize_t splice_to_pipe(struct pipe_inode_info *pipe, + struct splice_pipe_desc *spd) +{ + unsigned int spd_pages = spd->nr_pages; + unsigned int tail = pipe->tail; + unsigned int head = pipe->head; + ssize_t ret = 0; + int page_nr = 0; + + if (!spd_pages) + return 0; + + if (unlikely(!pipe->readers)) { + send_sig(SIGPIPE, current, 0); + ret = -EPIPE; + goto out; + } + + while (!pipe_full(head, tail, pipe->max_usage)) { + struct pipe_buffer *buf = pipe_buf(pipe, head); + + buf->page = spd->pages[page_nr]; + buf->offset = spd->partial[page_nr].offset; + buf->len = spd->partial[page_nr].len; + buf->private = spd->partial[page_nr].private; + buf->ops = spd->ops; + buf->flags = 0; + + head++; + pipe->head = head; + page_nr++; + ret += buf->len; + + if (!--spd->nr_pages) + break; + } + + if (!ret) + ret = -EAGAIN; + +out: + while (page_nr < spd_pages) + spd->spd_release(spd, page_nr++); + + return ret; +} +EXPORT_SYMBOL_GPL(splice_to_pipe); + +ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf) +{ + unsigned int head = pipe->head; + unsigned int tail = pipe->tail; + int ret; + + if (unlikely(!pipe->readers)) { + send_sig(SIGPIPE, current, 0); + ret = -EPIPE; + } else if (pipe_full(head, tail, pipe->max_usage)) { + ret = -EAGAIN; + } else { + *pipe_buf(pipe, head) = *buf; + pipe->head = head + 1; + return buf->len; + } + pipe_buf_release(pipe, buf); + return ret; +} +EXPORT_SYMBOL(add_to_pipe); + +/* + * Check if we need to grow the arrays holding pages and partial page + * descriptions. + */ +int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd) +{ + unsigned int max_usage = READ_ONCE(pipe->max_usage); + + spd->nr_pages_max = max_usage; + if (max_usage <= PIPE_DEF_BUFFERS) + return 0; + + spd->pages = kmalloc_objs(struct page *, max_usage); + spd->partial = kmalloc_objs(struct partial_page, max_usage); + + if (spd->pages && spd->partial) + return 0; + + kfree(spd->pages); + kfree(spd->partial); + return -ENOMEM; +} + +void splice_shrink_spd(struct splice_pipe_desc *spd) +{ + if (spd->nr_pages_max <= PIPE_DEF_BUFFERS) + return; + + kfree(spd->pages); + kfree(spd->partial); +} + +/** + * copy_splice_read - Copy data from a file and splice the copy into a pipe + * @in: The file to read from + * @ppos: Pointer to the file position to read from + * @pipe: The pipe to splice into + * @len: The amount to splice + * @flags: The SPLICE_F_* flags + * + * This function allocates a bunch of pages sufficient to hold the requested + * amount of data (but limited by the remaining pipe capacity), passes it to + * the file's ->read_iter() to read into and then splices the used pages into + * the pipe. + * + * Return: On success, the number of bytes read will be returned and *@ppos + * will be updated if appropriate; 0 will be returned if there is no more data + * to be read; -EAGAIN will be returned if the pipe had no space, and some + * other negative error code will be returned on error. A short read may occur + * if the pipe has insufficient space, we reach the end of the data or we hit a + * hole. + */ +ssize_t copy_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, + size_t len, unsigned int flags) +{ + struct iov_iter to; + struct bio_vec *bv; + struct kiocb kiocb; + struct page **pages; + ssize_t ret; + size_t used, npages, chunk, remain, keep = 0; + int i; + + /* Work out how much data we can actually add into the pipe */ + used = pipe_buf_usage(pipe); + npages = max_t(ssize_t, pipe->max_usage - used, 0); + len = min_t(size_t, len, npages * PAGE_SIZE); + npages = DIV_ROUND_UP(len, PAGE_SIZE); + + bv = kzalloc(array_size(npages, sizeof(bv[0])) + + array_size(npages, sizeof(struct page *)), GFP_KERNEL); + if (!bv) + return -ENOMEM; + + pages = (struct page **)(bv + npages); + npages = alloc_pages_bulk(GFP_USER, npages, pages); + if (!npages) { + kfree(bv); + return -ENOMEM; + } + + remain = len = min_t(size_t, len, npages * PAGE_SIZE); + + for (i = 0; i < npages; i++) { + chunk = min_t(size_t, PAGE_SIZE, remain); + bv[i].bv_page = pages[i]; + bv[i].bv_offset = 0; + bv[i].bv_len = chunk; + remain -= chunk; + } + + /* Do the I/O */ + iov_iter_bvec(&to, ITER_DEST, bv, npages, len); + init_sync_kiocb(&kiocb, in); + kiocb.ki_pos = *ppos; + ret = in->f_op->read_iter(&kiocb, &to); + + if (ret > 0) { + keep = DIV_ROUND_UP(ret, PAGE_SIZE); + *ppos = kiocb.ki_pos; + } + + /* + * Callers of ->splice_read() expect -EAGAIN on "can't put anything in + * there", rather than -EFAULT. + */ + if (ret == -EFAULT) + ret = -EAGAIN; + + /* Free any pages that didn't get touched at all. */ + if (keep < npages) + release_pages(pages + keep, npages - keep); + + /* Push the remaining pages into the pipe. */ + remain = ret; + for (i = 0; i < keep; i++) { + struct pipe_buffer *buf = pipe_head_buf(pipe); + + chunk = min_t(size_t, remain, PAGE_SIZE); + *buf = (struct pipe_buffer) { + .ops = &default_pipe_buf_ops, + .page = bv[i].bv_page, + .offset = 0, + .len = chunk, + }; + pipe->head++; + remain -= chunk; + } + + kfree(bv); + return ret; +} +EXPORT_SYMBOL(copy_splice_read); + +const struct pipe_buf_operations default_pipe_buf_ops = { + .release = generic_pipe_buf_release, + .try_steal = generic_pipe_buf_try_steal, + .get = generic_pipe_buf_get, +}; + +/* Pipe buffer operations for a socket and similar. */ +const struct pipe_buf_operations nosteal_pipe_buf_ops = { + .release = generic_pipe_buf_release, + .get = generic_pipe_buf_get, +}; +EXPORT_SYMBOL(nosteal_pipe_buf_ops); + +static void wakeup_pipe_writers(struct pipe_inode_info *pipe) +{ + smp_mb(); + if (waitqueue_active(&pipe->wr_wait)) + wake_up_interruptible(&pipe->wr_wait); + kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); +} + +/** + * splice_from_pipe_feed - feed available data from a pipe to a file + * @pipe: pipe to splice from + * @sd: information to @actor + * @actor: handler that splices the data + * + * Description: + * This function loops over the pipe and calls @actor to do the + * actual moving of a single struct pipe_buffer to the desired + * destination. It returns when there's no more buffers left in + * the pipe or if the requested number of bytes (@sd->total_len) + * have been copied. It returns a positive number (one) if the + * pipe needs to be filled with more data, zero if the required + * number of bytes have been copied and -errno on error. + * + * This, together with splice_from_pipe_{begin,end,next}, may be + * used to implement the functionality of __splice_from_pipe() when + * locking is required around copying the pipe buffers to the + * destination. + */ +static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd, + splice_actor *actor) +{ + unsigned int head = pipe->head; + unsigned int tail = pipe->tail; + int ret; + + while (!pipe_empty(head, tail)) { + struct pipe_buffer *buf = pipe_buf(pipe, tail); + + sd->len = buf->len; + if (sd->len > sd->total_len) + sd->len = sd->total_len; + + ret = pipe_buf_confirm(pipe, buf); + if (unlikely(ret)) { + if (ret == -ENODATA) + ret = 0; + return ret; + } + + ret = actor(pipe, buf, sd); + if (ret <= 0) + return ret; + + buf->offset += ret; + buf->len -= ret; + + sd->num_spliced += ret; + sd->len -= ret; + sd->pos += ret; + sd->total_len -= ret; + + if (!buf->len) { + pipe_buf_release(pipe, buf); + tail++; + pipe->tail = tail; + if (pipe->files) + sd->need_wakeup = true; + } + + if (!sd->total_len) + return 0; + } + + return 1; +} + +/* We know we have a pipe buffer, but maybe it's empty? */ +static inline bool eat_empty_buffer(struct pipe_inode_info *pipe) +{ + unsigned int tail = pipe->tail; + struct pipe_buffer *buf = pipe_buf(pipe, tail); + + if (unlikely(!buf->len)) { + pipe_buf_release(pipe, buf); + pipe->tail = tail+1; + return true; + } + + return false; +} + +/** + * splice_from_pipe_next - wait for some data to splice from + * @pipe: pipe to splice from + * @sd: information about the splice operation + * + * Description: + * This function will wait for some data and return a positive + * value (one) if pipe buffers are available. It will return zero + * or -errno if no more data needs to be spliced. + */ +static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd) +{ + /* + * Check for signal early to make process killable when there are + * always buffers available + */ + if (signal_pending(current)) + return -ERESTARTSYS; + +repeat: + while (pipe_is_empty(pipe)) { + if (!pipe->writers) + return 0; + + if (sd->num_spliced) + return 0; + + if (sd->flags & SPLICE_F_NONBLOCK) + return -EAGAIN; + + if (signal_pending(current)) + return -ERESTARTSYS; + + if (sd->need_wakeup) { + wakeup_pipe_writers(pipe); + sd->need_wakeup = false; + } + + pipe_wait_readable(pipe); + } + + if (eat_empty_buffer(pipe)) + goto repeat; + + return 1; +} + +/** + * splice_from_pipe_begin - start splicing from pipe + * @sd: information about the splice operation + * + * Description: + * This function should be called before a loop containing + * splice_from_pipe_next() and splice_from_pipe_feed() to + * initialize the necessary fields of @sd. + */ +static void splice_from_pipe_begin(struct splice_desc *sd) +{ + sd->num_spliced = 0; + sd->need_wakeup = false; +} + +/** + * splice_from_pipe_end - finish splicing from pipe + * @pipe: pipe to splice from + * @sd: information about the splice operation + * + * Description: + * This function will wake up pipe writers if necessary. It should + * be called after a loop containing splice_from_pipe_next() and + * splice_from_pipe_feed(). + */ +static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd) +{ + if (sd->need_wakeup) + wakeup_pipe_writers(pipe); +} + +/** + * __splice_from_pipe - splice data from a pipe to given actor + * @pipe: pipe to splice from + * @sd: information to @actor + * @actor: handler that splices the data + * + * Description: + * This function does little more than loop over the pipe and call + * @actor to do the actual moving of a single struct pipe_buffer to + * the desired destination. See pipe_to_file, pipe_to_sendmsg, or + * pipe_to_user. + * + */ +ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd, + splice_actor *actor) +{ + int ret; + + splice_from_pipe_begin(sd); + do { + cond_resched(); + ret = splice_from_pipe_next(pipe, sd); + if (ret > 0) + ret = splice_from_pipe_feed(pipe, sd, actor); + } while (ret > 0); + splice_from_pipe_end(pipe, sd); + + return sd->num_spliced ? sd->num_spliced : ret; +} +EXPORT_SYMBOL(__splice_from_pipe); + +/** + * splice_from_pipe - splice data from a pipe to a file + * @pipe: pipe to splice from + * @out: file to splice to + * @ppos: position in @out + * @len: how many bytes to splice + * @flags: splice modifier flags + * @actor: handler that splices the data + * + * Description: + * See __splice_from_pipe. This function locks the pipe inode, + * otherwise it's identical to __splice_from_pipe(). + * + */ +ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out, + loff_t *ppos, size_t len, unsigned int flags, + splice_actor *actor) +{ + ssize_t ret; + struct splice_desc sd = { + .total_len = len, + .flags = flags, + .pos = *ppos, + .u.file = out, + }; + + pipe_lock(pipe); + ret = __splice_from_pipe(pipe, &sd, actor); + pipe_unlock(pipe); + + return ret; +} + +/** + * iter_file_splice_write - splice data from a pipe to a file + * @pipe: pipe info + * @out: file to write to + * @ppos: position in @out + * @len: number of bytes to splice + * @flags: splice modifier flags + * + * Description: + * Will either move or copy pages (determined by @flags options) from + * the given pipe inode to the given file. + * This one is ->write_iter-based. + * + */ +ssize_t +iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out, + loff_t *ppos, size_t len, unsigned int flags) +{ + struct splice_desc sd = { + .total_len = len, + .flags = flags, + .pos = *ppos, + .u.file = out, + }; + int nbufs = pipe->max_usage; + struct bio_vec *array; + ssize_t ret; + + if (!out->f_op->write_iter) + return -EINVAL; + + array = kzalloc_objs(struct bio_vec, nbufs); + if (unlikely(!array)) + return -ENOMEM; + + pipe_lock(pipe); + + splice_from_pipe_begin(&sd); + while (sd.total_len) { + struct kiocb kiocb; + struct iov_iter from; + unsigned int head, tail; + size_t left; + int n; + + ret = splice_from_pipe_next(pipe, &sd); + if (ret <= 0) + break; + + if (unlikely(nbufs < pipe->max_usage)) { + kfree(array); + nbufs = pipe->max_usage; + array = kzalloc_objs(struct bio_vec, nbufs); + if (!array) { + ret = -ENOMEM; + break; + } + } + + head = pipe->head; + tail = pipe->tail; + + /* build the vector */ + left = sd.total_len; + for (n = 0; !pipe_empty(head, tail) && left && n < nbufs; tail++) { + struct pipe_buffer *buf = pipe_buf(pipe, tail); + size_t this_len = buf->len; + + /* zero-length bvecs are not supported, skip them */ + if (!this_len) + continue; + this_len = min(this_len, left); + + ret = pipe_buf_confirm(pipe, buf); + if (unlikely(ret)) { + if (ret == -ENODATA) + ret = 0; + goto done; + } + + bvec_set_page(&array[n], buf->page, this_len, + buf->offset); + left -= this_len; + n++; + } + + iov_iter_bvec(&from, ITER_SOURCE, array, n, sd.total_len - left); + init_sync_kiocb(&kiocb, out); + kiocb.ki_pos = sd.pos; + ret = out->f_op->write_iter(&kiocb, &from); + sd.pos = kiocb.ki_pos; + if (ret <= 0) + break; + WARN_ONCE(ret > sd.total_len - left, + "Splice Exceeded! ret=%zd tot=%zu left=%zu\n", + ret, sd.total_len, left); + + sd.num_spliced += ret; + sd.total_len -= ret; + *ppos = sd.pos; + + /* dismiss the fully eaten buffers, adjust the partial one */ + tail = pipe->tail; + while (ret) { + struct pipe_buffer *buf = pipe_buf(pipe, tail); + if (ret >= buf->len) { + ret -= buf->len; + buf->len = 0; + pipe_buf_release(pipe, buf); + tail++; + pipe->tail = tail; + if (pipe->files) + sd.need_wakeup = true; + } else { + buf->offset += ret; + buf->len -= ret; + ret = 0; + } + } + } +done: + kfree(array); + splice_from_pipe_end(pipe, &sd); + + pipe_unlock(pipe); + + if (sd.num_spliced) + ret = sd.num_spliced; + + return ret; +} + +EXPORT_SYMBOL(iter_file_splice_write); + +#ifdef CONFIG_NET +/** + * splice_to_socket - splice data from a pipe to a socket + * @pipe: pipe to splice from + * @out: socket to write to + * @ppos: position in @out + * @len: number of bytes to splice + * @flags: splice modifier flags + * + * Description: + * Will send @len bytes from the pipe to a network socket. No data copying + * is involved. + * + */ +ssize_t splice_to_socket(struct pipe_inode_info *pipe, struct file *out, + loff_t *ppos, size_t len, unsigned int flags) +{ + struct socket *sock = sock_from_file(out); + struct bio_vec bvec[16]; + struct msghdr msg = {}; + ssize_t ret = 0; + size_t spliced = 0; + bool need_wakeup = false; + + pipe_lock(pipe); + + while (len > 0) { + unsigned int head, tail, bc = 0; + size_t remain = len; + + /* + * Check for signal early to make process killable when there + * are always buffers available + */ + ret = -ERESTARTSYS; + if (signal_pending(current)) + break; + + while (pipe_is_empty(pipe)) { + ret = 0; + if (!pipe->writers) + goto out; + + if (spliced) + goto out; + + ret = -EAGAIN; + if (flags & SPLICE_F_NONBLOCK) + goto out; + + ret = -ERESTARTSYS; + if (signal_pending(current)) + goto out; + + if (need_wakeup) { + wakeup_pipe_writers(pipe); + need_wakeup = false; + } + + pipe_wait_readable(pipe); + } + + head = pipe->head; + tail = pipe->tail; + + while (!pipe_empty(head, tail)) { + struct pipe_buffer *buf = pipe_buf(pipe, tail); + size_t seg; + + if (!buf->len) { + tail++; + continue; + } + + seg = min_t(size_t, remain, buf->len); + + ret = pipe_buf_confirm(pipe, buf); + if (unlikely(ret)) { + if (ret == -ENODATA) + ret = 0; + break; + } + + bvec_set_page(&bvec[bc++], buf->page, seg, buf->offset); + remain -= seg; + if (remain == 0 || bc >= ARRAY_SIZE(bvec)) + break; + tail++; + } + + if (!bc) + break; + + msg.msg_flags = MSG_SPLICE_PAGES; + if (flags & SPLICE_F_MORE) + msg.msg_flags |= MSG_MORE; + if (remain && pipe_occupancy(pipe->head, tail) > 0) + msg.msg_flags |= MSG_MORE; + if (out->f_flags & O_NONBLOCK) + msg.msg_flags |= MSG_DONTWAIT; + + iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, bvec, bc, + len - remain); + ret = sock_sendmsg(sock, &msg); + if (ret <= 0) + break; + + spliced += ret; + len -= ret; + tail = pipe->tail; + while (ret > 0) { + struct pipe_buffer *buf = pipe_buf(pipe, tail); + size_t seg = min_t(size_t, ret, buf->len); + + buf->offset += seg; + buf->len -= seg; + ret -= seg; + + if (!buf->len) { + pipe_buf_release(pipe, buf); + tail++; + } + } + + if (tail != pipe->tail) { + pipe->tail = tail; + if (pipe->files) + need_wakeup = true; + } + } + +out: + pipe_unlock(pipe); + if (need_wakeup) + wakeup_pipe_writers(pipe); + return spliced ?: ret; +} +#endif + +static int warn_unsupported(struct file *file, const char *op) +{ + pr_debug_ratelimited( + "splice %s not supported for file %pD4 (pid: %d comm: %.20s)\n", + op, file, current->pid, current->comm); + return -EINVAL; +} + +/* + * Attempt to initiate a splice from pipe to file. + */ +static ssize_t do_splice_from(struct pipe_inode_info *pipe, struct file *out, + loff_t *ppos, size_t len, unsigned int flags) +{ + if (unlikely(!out->f_op->splice_write)) + return warn_unsupported(out, "write"); + return out->f_op->splice_write(pipe, out, ppos, len, flags); +} + +/* + * Indicate to the caller that there was a premature EOF when reading from the + * source and the caller didn't indicate they would be sending more data after + * this. + */ +static void do_splice_eof(struct splice_desc *sd) +{ + if (sd->splice_eof) + sd->splice_eof(sd); +} + +/* + * Callers already called rw_verify_area() on the entire range. + * No need to call it for sub ranges. + */ +static ssize_t do_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, + unsigned int flags) +{ + unsigned int p_space; + + if (unlikely(!(in->f_mode & FMODE_READ))) + return -EBADF; + if (!len) + return 0; + + /* Don't try to read more the pipe has space for. */ + p_space = pipe->max_usage - pipe_buf_usage(pipe); + len = min_t(size_t, len, p_space << PAGE_SHIFT); + + if (unlikely(len > MAX_RW_COUNT)) + len = MAX_RW_COUNT; + + if (unlikely(!in->f_op->splice_read)) + return warn_unsupported(in, "read"); + /* + * O_DIRECT and DAX don't deal with the pagecache, so we allocate a + * buffer, copy into it and splice that into the pipe. + */ + if ((in->f_flags & O_DIRECT) || IS_DAX(in->f_mapping->host)) + return copy_splice_read(in, ppos, pipe, len, flags); + return in->f_op->splice_read(in, ppos, pipe, len, flags); +} + +/** + * vfs_splice_read - Read data from a file and splice it into a pipe + * @in: File to splice from + * @ppos: Input file offset + * @pipe: Pipe to splice to + * @len: Number of bytes to splice + * @flags: Splice modifier flags (SPLICE_F_*) + * + * Splice the requested amount of data from the input file to the pipe. This + * is synchronous as the caller must hold the pipe lock across the entire + * operation. + * + * If successful, it returns the amount of data spliced, 0 if it hit the EOF or + * a hole and a negative error code otherwise. + */ +ssize_t vfs_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, size_t len, + unsigned int flags) +{ + ssize_t ret; + + ret = rw_verify_area(READ, in, ppos, len); + if (unlikely(ret < 0)) + return ret; + + return do_splice_read(in, ppos, pipe, len, flags); +} +EXPORT_SYMBOL_GPL(vfs_splice_read); + +/** + * splice_direct_to_actor - splices data directly between two non-pipes + * @in: file to splice from + * @sd: actor information on where to splice to + * @actor: handles the data splicing + * + * Description: + * This is a special case helper to splice directly between two + * points, without requiring an explicit pipe. Internally an allocated + * pipe is cached in the process, and reused during the lifetime of + * that process. + * + */ +ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd, + splice_direct_actor *actor) +{ + struct pipe_inode_info *pipe; + ssize_t ret, bytes; + size_t len; + int i, flags, more; + + /* + * We require the input to be seekable, as we don't want to randomly + * drop data for eg socket -> socket splicing. Use the piped splicing + * for that! + */ + if (unlikely(!(in->f_mode & FMODE_LSEEK))) + return -EINVAL; + + /* + * neither in nor out is a pipe, setup an internal pipe attached to + * 'out' and transfer the wanted data from 'in' to 'out' through that + */ + pipe = current->splice_pipe; + if (unlikely(!pipe)) { + pipe = alloc_pipe_info(); + if (!pipe) + return -ENOMEM; + + /* + * We don't have an immediate reader, but we'll read the stuff + * out of the pipe right after the splice_to_pipe(). So set + * PIPE_READERS appropriately. + */ + pipe->readers = 1; + + current->splice_pipe = pipe; + } + + /* + * Do the splice. + */ + bytes = 0; + len = sd->total_len; + + /* Don't block on output, we have to drain the direct pipe. */ + flags = sd->flags; + sd->flags &= ~SPLICE_F_NONBLOCK; + + /* + * We signal MORE until we've read sufficient data to fulfill the + * request and we keep signalling it if the caller set it. + */ + more = sd->flags & SPLICE_F_MORE; + sd->flags |= SPLICE_F_MORE; + + WARN_ON_ONCE(!pipe_is_empty(pipe)); + + while (len) { + size_t read_len; + loff_t pos = sd->pos, prev_pos = pos; + + ret = do_splice_read(in, &pos, pipe, len, flags); + if (unlikely(ret <= 0)) + goto read_failure; + + read_len = ret; + sd->total_len = read_len; + + /* + * If we now have sufficient data to fulfill the request then + * we clear SPLICE_F_MORE if it was not set initially. + */ + if (read_len >= len && !more) + sd->flags &= ~SPLICE_F_MORE; + + /* + * NOTE: nonblocking mode only applies to the input. We + * must not do the output in nonblocking mode as then we + * could get stuck data in the internal pipe: + */ + ret = actor(pipe, sd); + if (unlikely(ret <= 0)) { + sd->pos = prev_pos; + goto out_release; + } + + bytes += ret; + len -= ret; + sd->pos = pos; + + if (ret < read_len) { + sd->pos = prev_pos + ret; + goto out_release; + } + } + +done: + pipe->tail = pipe->head = 0; + file_accessed(in); + return bytes; + +read_failure: + /* + * If the user did *not* set SPLICE_F_MORE *and* we didn't hit that + * "use all of len" case that cleared SPLICE_F_MORE, *and* we did a + * "->splice_in()" that returned EOF (ie zero) *and* we have sent at + * least 1 byte *then* we will also do the ->splice_eof() call. + */ + if (ret == 0 && !more && len > 0 && bytes) + do_splice_eof(sd); +out_release: + /* + * If we did an incomplete transfer we must release + * the pipe buffers in question: + */ + for (i = 0; i < pipe->ring_size; i++) { + struct pipe_buffer *buf = &pipe->bufs[i]; + + if (buf->ops) + pipe_buf_release(pipe, buf); + } + + if (!bytes) + bytes = ret; + + goto done; +} +EXPORT_SYMBOL(splice_direct_to_actor); + +static int direct_splice_actor(struct pipe_inode_info *pipe, + struct splice_desc *sd) +{ + struct file *file = sd->u.file; + long ret; + + file_start_write(file); + ret = do_splice_from(pipe, file, sd->opos, sd->total_len, sd->flags); + file_end_write(file); + return ret; +} + +static int splice_file_range_actor(struct pipe_inode_info *pipe, + struct splice_desc *sd) +{ + struct file *file = sd->u.file; + + return do_splice_from(pipe, file, sd->opos, sd->total_len, sd->flags); +} + +static void direct_file_splice_eof(struct splice_desc *sd) +{ + struct file *file = sd->u.file; + + if (file->f_op->splice_eof) + file->f_op->splice_eof(file); +} + +static ssize_t do_splice_direct_actor(struct file *in, loff_t *ppos, + struct file *out, loff_t *opos, + size_t len, unsigned int flags, + splice_direct_actor *actor) +{ + struct splice_desc sd = { + .len = len, + .total_len = len, + .flags = flags, + .pos = *ppos, + .u.file = out, + .splice_eof = direct_file_splice_eof, + .opos = opos, + }; + ssize_t ret; + + if (unlikely(!(out->f_mode & FMODE_WRITE))) + return -EBADF; + + if (unlikely(out->f_flags & O_APPEND)) + return -EINVAL; + + ret = splice_direct_to_actor(in, &sd, actor); + if (ret > 0) + *ppos = sd.pos; + + return ret; +} +/** + * do_splice_direct - splices data directly between two files + * @in: file to splice from + * @ppos: input file offset + * @out: file to splice to + * @opos: output file offset + * @len: number of bytes to splice + * @flags: splice modifier flags + * + * Description: + * For use by do_sendfile(). splice can easily emulate sendfile, but + * doing it in the application would incur an extra system call + * (splice in + splice out, as compared to just sendfile()). So this helper + * can splice directly through a process-private pipe. + * + * Callers already called rw_verify_area() on the entire range. + */ +ssize_t do_splice_direct(struct file *in, loff_t *ppos, struct file *out, + loff_t *opos, size_t len, unsigned int flags) +{ + return do_splice_direct_actor(in, ppos, out, opos, len, flags, + direct_splice_actor); +} +EXPORT_SYMBOL(do_splice_direct); + +/** + * splice_file_range - splices data between two files for copy_file_range() + * @in: file to splice from + * @ppos: input file offset + * @out: file to splice to + * @opos: output file offset + * @len: number of bytes to splice + * + * Description: + * For use by ->copy_file_range() methods. + * Like do_splice_direct(), but vfs_copy_file_range() already holds + * start_file_write() on @out file. + * + * Callers already called rw_verify_area() on the entire range. + */ +ssize_t splice_file_range(struct file *in, loff_t *ppos, struct file *out, + loff_t *opos, size_t len) +{ + lockdep_assert(file_write_started(out)); + + return do_splice_direct_actor(in, ppos, out, opos, + min_t(size_t, len, MAX_RW_COUNT), + 0, splice_file_range_actor); +} +EXPORT_SYMBOL(splice_file_range); + +static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags) +{ + for (;;) { + if (unlikely(!pipe->readers)) { + send_sig(SIGPIPE, current, 0); + return -EPIPE; + } + if (!pipe_is_full(pipe)) + return 0; + if (flags & SPLICE_F_NONBLOCK) + return -EAGAIN; + if (signal_pending(current)) + return -ERESTARTSYS; + pipe_wait_writable(pipe); + } +} + +static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, + struct pipe_inode_info *opipe, + size_t len, unsigned int flags); + +ssize_t splice_file_to_pipe(struct file *in, + struct pipe_inode_info *opipe, + loff_t *offset, + size_t len, unsigned int flags) +{ + ssize_t ret; + + pipe_lock(opipe); + ret = wait_for_space(opipe, flags); + if (!ret) + ret = do_splice_read(in, offset, opipe, len, flags); + pipe_unlock(opipe); + if (ret > 0) + wakeup_pipe_readers(opipe); + return ret; +} + +/* + * Determine where to splice to/from. + */ +ssize_t do_splice(struct file *in, loff_t *off_in, struct file *out, + loff_t *off_out, size_t len, unsigned int flags) +{ + struct pipe_inode_info *ipipe; + struct pipe_inode_info *opipe; + loff_t offset; + ssize_t ret; + + if (unlikely(!(in->f_mode & FMODE_READ) || + !(out->f_mode & FMODE_WRITE))) + return -EBADF; + + ipipe = get_pipe_info(in, true); + opipe = get_pipe_info(out, true); + + if (ipipe && opipe) { + if (off_in || off_out) + return -ESPIPE; + + /* Splicing to self would be fun, but... */ + if (ipipe == opipe) + return -EINVAL; + + if ((in->f_flags | out->f_flags) & O_NONBLOCK) + flags |= SPLICE_F_NONBLOCK; + + ret = splice_pipe_to_pipe(ipipe, opipe, len, flags); + } else if (ipipe) { + if (off_in) + return -ESPIPE; + if (off_out) { + if (!(out->f_mode & FMODE_PWRITE)) + return -EINVAL; + offset = *off_out; + } else { + offset = out->f_pos; + } + + if (unlikely(out->f_flags & O_APPEND)) + return -EINVAL; + + ret = rw_verify_area(WRITE, out, &offset, len); + if (unlikely(ret < 0)) + return ret; + + if (in->f_flags & O_NONBLOCK) + flags |= SPLICE_F_NONBLOCK; + + file_start_write(out); + ret = do_splice_from(ipipe, out, &offset, len, flags); + file_end_write(out); + + if (!off_out) + out->f_pos = offset; + else + *off_out = offset; + } else if (opipe) { + if (off_out) + return -ESPIPE; + if (off_in) { + if (!(in->f_mode & FMODE_PREAD)) + return -EINVAL; + offset = *off_in; + } else { + offset = in->f_pos; + } + + ret = rw_verify_area(READ, in, &offset, len); + if (unlikely(ret < 0)) + return ret; + + if (out->f_flags & O_NONBLOCK) + flags |= SPLICE_F_NONBLOCK; + + ret = splice_file_to_pipe(in, opipe, &offset, len, flags); + + if (!off_in) + in->f_pos = offset; + else + *off_in = offset; + } else { + ret = -EINVAL; + } + + if (ret > 0) { + /* + * Generate modify out before access in: + * do_splice_from() may've already sent modify out, + * and this ensures the events get merged. + */ + fsnotify_modify(out); + fsnotify_access(in); + } + + return ret; +} + +static ssize_t __do_splice(struct file *in, loff_t __user *off_in, + struct file *out, loff_t __user *off_out, + size_t len, unsigned int flags) +{ + struct pipe_inode_info *ipipe; + struct pipe_inode_info *opipe; + loff_t offset, *__off_in = NULL, *__off_out = NULL; + ssize_t ret; + + ipipe = get_pipe_info(in, true); + opipe = get_pipe_info(out, true); + + if (ipipe) { + if (off_in) + return -ESPIPE; + pipe_clear_nowait(in); + } + if (opipe) { + if (off_out) + return -ESPIPE; + pipe_clear_nowait(out); + } + + if (off_out) { + if (copy_from_user(&offset, off_out, sizeof(loff_t))) + return -EFAULT; + __off_out = &offset; + } + if (off_in) { + if (copy_from_user(&offset, off_in, sizeof(loff_t))) + return -EFAULT; + __off_in = &offset; + } + + ret = do_splice(in, __off_in, out, __off_out, len, flags); + if (ret < 0) + return ret; + + if (__off_out && copy_to_user(off_out, __off_out, sizeof(loff_t))) + return -EFAULT; + if (__off_in && copy_to_user(off_in, __off_in, sizeof(loff_t))) + return -EFAULT; + + return ret; +} + +static ssize_t iter_to_pipe(struct iov_iter *from, + struct pipe_inode_info *pipe, + unsigned int flags) +{ + struct pipe_buffer buf = { + .ops = &user_page_pipe_buf_ops, + .flags = flags + }; + size_t total = 0; + ssize_t ret = 0; + + while (iov_iter_count(from)) { + struct page *pages[16]; + ssize_t left; + size_t start; + int i, n; + + left = iov_iter_get_pages2(from, pages, ~0UL, 16, &start); + if (left <= 0) { + ret = left; + break; + } + + n = DIV_ROUND_UP(left + start, PAGE_SIZE); + for (i = 0; i < n; i++) { + int size = umin(left, PAGE_SIZE - start); + + buf.page = pages[i]; + buf.offset = start; + buf.len = size; + ret = add_to_pipe(pipe, &buf); + if (unlikely(ret < 0)) { + iov_iter_revert(from, left); + // this one got dropped by add_to_pipe() + while (++i < n) + put_page(pages[i]); + goto out; + } + total += ret; + left -= size; + start = 0; + } + } +out: + return total ? total : ret; +} + +static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf, + struct splice_desc *sd) +{ + int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data); + return n == sd->len ? n : -EFAULT; +} + +/* + * For lack of a better implementation, implement vmsplice() to userspace + * as a simple copy of the pipe's pages to the user iov. + */ +static ssize_t vmsplice_to_user(struct file *file, struct iov_iter *iter, + unsigned int flags) +{ + struct pipe_inode_info *pipe = get_pipe_info(file, true); + struct splice_desc sd = { + .total_len = iov_iter_count(iter), + .flags = flags, + .u.data = iter + }; + ssize_t ret = 0; + + if (!pipe) + return -EBADF; + + pipe_clear_nowait(file); + + if (sd.total_len) { + pipe_lock(pipe); + ret = __splice_from_pipe(pipe, &sd, pipe_to_user); + pipe_unlock(pipe); + } + + if (ret > 0) + fsnotify_access(file); + + return ret; +} + +/* + * vmsplice splices a user address range into a pipe. It can be thought of + * as splice-from-memory, where the regular splice is splice-from-file (or + * to file). In both cases the output is a pipe, naturally. + */ +static ssize_t vmsplice_to_pipe(struct file *file, struct iov_iter *iter, + unsigned int flags) +{ + struct pipe_inode_info *pipe; + ssize_t ret = 0; + unsigned buf_flag = 0; + + if (flags & SPLICE_F_GIFT) + buf_flag = PIPE_BUF_FLAG_GIFT; + + pipe = get_pipe_info(file, true); + if (!pipe) + return -EBADF; + + pipe_clear_nowait(file); + + pipe_lock(pipe); + ret = wait_for_space(pipe, flags); + if (!ret) + ret = iter_to_pipe(iter, pipe, buf_flag); + pipe_unlock(pipe); + if (ret > 0) { + wakeup_pipe_readers(pipe); + fsnotify_modify(file); + } + return ret; +} + +/* + * Note that vmsplice only really supports true splicing _from_ user memory + * to a pipe, not the other way around. Splicing from user memory is a simple + * operation that can be supported without any funky alignment restrictions + * or nasty vm tricks. We simply map in the user memory and fill them into + * a pipe. The reverse isn't quite as easy, though. There are two possible + * solutions for that: + * + * - memcpy() the data internally, at which point we might as well just + * do a regular read() on the buffer anyway. + * - Lots of nasty vm tricks, that are neither fast nor flexible (it + * has restriction limitations on both ends of the pipe). + * + * Currently we punt and implement it as a normal copy, see pipe_to_user(). + * + */ +SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, uiov, + unsigned long, nr_segs, unsigned int, flags) +{ + struct iovec iovstack[UIO_FASTIOV]; + struct iovec *iov = iovstack; + struct iov_iter iter; + ssize_t error; + int type; + + if (unlikely(flags & ~SPLICE_F_ALL)) + return -EINVAL; + + CLASS(fd, f)(fd); + if (fd_empty(f)) + return -EBADF; + if (fd_file(f)->f_mode & FMODE_WRITE) + type = ITER_SOURCE; + else if (fd_file(f)->f_mode & FMODE_READ) + type = ITER_DEST; + else + return -EBADF; + + error = import_iovec(type, uiov, nr_segs, + ARRAY_SIZE(iovstack), &iov, &iter); + if (error < 0) + return error; + + if (!iov_iter_count(&iter)) + error = 0; + else if (type == ITER_SOURCE) + error = vmsplice_to_pipe(fd_file(f), &iter, flags); + else + error = vmsplice_to_user(fd_file(f), &iter, flags); + + kfree(iov); + return error; +} + +SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in, + int, fd_out, loff_t __user *, off_out, + size_t, len, unsigned int, flags) +{ + if (unlikely(!len)) + return 0; + + if (unlikely(flags & ~SPLICE_F_ALL)) + return -EINVAL; + + CLASS(fd, in)(fd_in); + if (fd_empty(in)) + return -EBADF; + + CLASS(fd, out)(fd_out); + if (fd_empty(out)) + return -EBADF; + + return __do_splice(fd_file(in), off_in, fd_file(out), off_out, + len, flags); +} + +/* + * Make sure there's data to read. Wait for input if we can, otherwise + * return an appropriate error. + */ +static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags) +{ + int ret; + + /* + * Check the pipe occupancy without the inode lock first. This function + * is speculative anyways, so missing one is ok. + */ + if (!pipe_is_empty(pipe)) + return 0; + + ret = 0; + pipe_lock(pipe); + + while (pipe_is_empty(pipe)) { + if (signal_pending(current)) { + ret = -ERESTARTSYS; + break; + } + if (!pipe->writers) + break; + if (flags & SPLICE_F_NONBLOCK) { + ret = -EAGAIN; + break; + } + pipe_wait_readable(pipe); + } + + pipe_unlock(pipe); + return ret; +} + +/* + * Make sure there's writeable room. Wait for room if we can, otherwise + * return an appropriate error. + */ +static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags) +{ + int ret; + + /* + * Check pipe occupancy without the inode lock first. This function + * is speculative anyways, so missing one is ok. + */ + if (!pipe_is_full(pipe)) + return 0; + + ret = 0; + pipe_lock(pipe); + + while (pipe_is_full(pipe)) { + if (!pipe->readers) { + send_sig(SIGPIPE, current, 0); + ret = -EPIPE; + break; + } + if (flags & SPLICE_F_NONBLOCK) { + ret = -EAGAIN; + break; + } + if (signal_pending(current)) { + ret = -ERESTARTSYS; + break; + } + pipe_wait_writable(pipe); + } + + pipe_unlock(pipe); + return ret; +} + +/* + * Splice contents of ipipe to opipe. + */ +static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, + struct pipe_inode_info *opipe, + size_t len, unsigned int flags) +{ + struct pipe_buffer *ibuf, *obuf; + unsigned int i_head, o_head; + unsigned int i_tail, o_tail; + int ret = 0; + bool input_wakeup = false; + + +retry: + ret = ipipe_prep(ipipe, flags); + if (ret) + return ret; + + ret = opipe_prep(opipe, flags); + if (ret) + return ret; + + /* + * Potential ABBA deadlock, work around it by ordering lock + * grabbing by pipe info address. Otherwise two different processes + * could deadlock (one doing tee from A -> B, the other from B -> A). + */ + pipe_double_lock(ipipe, opipe); + + i_tail = ipipe->tail; + o_head = opipe->head; + + do { + size_t o_len; + + if (!opipe->readers) { + send_sig(SIGPIPE, current, 0); + if (!ret) + ret = -EPIPE; + break; + } + + i_head = ipipe->head; + o_tail = opipe->tail; + + if (pipe_empty(i_head, i_tail) && !ipipe->writers) + break; + + /* + * Cannot make any progress, because either the input + * pipe is empty or the output pipe is full. + */ + if (pipe_empty(i_head, i_tail) || + pipe_full(o_head, o_tail, opipe->max_usage)) { + /* Already processed some buffers, break */ + if (ret) + break; + + if (flags & SPLICE_F_NONBLOCK) { + ret = -EAGAIN; + break; + } + + /* + * We raced with another reader/writer and haven't + * managed to process any buffers. A zero return + * value means EOF, so retry instead. + */ + pipe_unlock(ipipe); + pipe_unlock(opipe); + goto retry; + } + + ibuf = pipe_buf(ipipe, i_tail); + obuf = pipe_buf(opipe, o_head); + + if (len >= ibuf->len) { + /* + * Simply move the whole buffer from ipipe to opipe + */ + *obuf = *ibuf; + ibuf->ops = NULL; + i_tail++; + ipipe->tail = i_tail; + input_wakeup = true; + o_len = obuf->len; + o_head++; + opipe->head = o_head; + } else { + /* + * Get a reference to this pipe buffer, + * so we can copy the contents over. + */ + if (!pipe_buf_get(ipipe, ibuf)) { + if (ret == 0) + ret = -EFAULT; + break; + } + *obuf = *ibuf; + + /* + * Don't inherit the gift and merge flags, we need to + * prevent multiple steals of this page. + */ + obuf->flags &= ~PIPE_BUF_FLAG_GIFT; + obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE; + + obuf->len = len; + ibuf->offset += len; + ibuf->len -= len; + o_len = len; + o_head++; + opipe->head = o_head; + } + ret += o_len; + len -= o_len; + } while (len); + + pipe_unlock(ipipe); + pipe_unlock(opipe); + + /* + * If we put data in the output pipe, wakeup any potential readers. + */ + if (ret > 0) + wakeup_pipe_readers(opipe); + + if (input_wakeup) + wakeup_pipe_writers(ipipe); + + return ret; +} + +/* + * Link contents of ipipe to opipe. + */ +static ssize_t link_pipe(struct pipe_inode_info *ipipe, + struct pipe_inode_info *opipe, + size_t len, unsigned int flags) +{ + struct pipe_buffer *ibuf, *obuf; + unsigned int i_head, o_head; + unsigned int i_tail, o_tail; + ssize_t ret = 0; + + /* + * Potential ABBA deadlock, work around it by ordering lock + * grabbing by pipe info address. Otherwise two different processes + * could deadlock (one doing tee from A -> B, the other from B -> A). + */ + pipe_double_lock(ipipe, opipe); + + i_tail = ipipe->tail; + o_head = opipe->head; + + do { + if (!opipe->readers) { + send_sig(SIGPIPE, current, 0); + if (!ret) + ret = -EPIPE; + break; + } + + i_head = ipipe->head; + o_tail = opipe->tail; + + /* + * If we have iterated all input buffers or run out of + * output room, break. + */ + if (pipe_empty(i_head, i_tail) || + pipe_full(o_head, o_tail, opipe->max_usage)) + break; + + ibuf = pipe_buf(ipipe, i_tail); + obuf = pipe_buf(opipe, o_head); + + /* + * Get a reference to this pipe buffer, + * so we can copy the contents over. + */ + if (!pipe_buf_get(ipipe, ibuf)) { + if (ret == 0) + ret = -EFAULT; + break; + } + + *obuf = *ibuf; + + /* + * Don't inherit the gift and merge flag, we need to prevent + * multiple steals of this page. + */ + obuf->flags &= ~PIPE_BUF_FLAG_GIFT; + obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE; + + if (obuf->len > len) + obuf->len = len; + ret += obuf->len; + len -= obuf->len; + + o_head++; + opipe->head = o_head; + i_tail++; + } while (len); + + pipe_unlock(ipipe); + pipe_unlock(opipe); + + /* + * If we put data in the output pipe, wakeup any potential readers. + */ + if (ret > 0) + wakeup_pipe_readers(opipe); + + return ret; +} + +/* + * This is a tee(1) implementation that works on pipes. It doesn't copy + * any data, it simply references the 'in' pages on the 'out' pipe. + * The 'flags' used are the SPLICE_F_* variants, currently the only + * applicable one is SPLICE_F_NONBLOCK. + */ +ssize_t do_tee(struct file *in, struct file *out, size_t len, + unsigned int flags) +{ + struct pipe_inode_info *ipipe = get_pipe_info(in, true); + struct pipe_inode_info *opipe = get_pipe_info(out, true); + ssize_t ret = -EINVAL; + + if (unlikely(!(in->f_mode & FMODE_READ) || + !(out->f_mode & FMODE_WRITE))) + return -EBADF; + + /* + * Duplicate the contents of ipipe to opipe without actually + * copying the data. + */ + if (ipipe && opipe && ipipe != opipe) { + if ((in->f_flags | out->f_flags) & O_NONBLOCK) + flags |= SPLICE_F_NONBLOCK; + + /* + * Keep going, unless we encounter an error. The ipipe/opipe + * ordering doesn't really matter. + */ + ret = ipipe_prep(ipipe, flags); + if (!ret) { + ret = opipe_prep(opipe, flags); + if (!ret) + ret = link_pipe(ipipe, opipe, len, flags); + } + } + + if (ret > 0) { + fsnotify_access(in); + fsnotify_modify(out); + } + + return ret; +} + +SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags) +{ + if (unlikely(flags & ~SPLICE_F_ALL)) + return -EINVAL; + + if (unlikely(!len)) + return 0; + + CLASS(fd, in)(fdin); + if (fd_empty(in)) + return -EBADF; + + CLASS(fd, out)(fdout); + if (fd_empty(out)) + return -EBADF; + + return do_tee(fd_file(in), fd_file(out), len, flags); +} diff --git a/fs/squashfs/Kconfig b/fs/squashfs/Kconfig new file mode 100644 index 000000000..a9602aae2 --- /dev/null +++ b/fs/squashfs/Kconfig @@ -0,0 +1,287 @@ +# SPDX-License-Identifier: GPL-2.0-only +config SQUASHFS + tristate "SquashFS 4.0 - Squashed file system support" + depends on BLOCK + help + Saying Y here includes support for SquashFS 4.0 (a Compressed + Read-Only File System). Squashfs is a highly compressed read-only + filesystem for Linux. It uses zlib, lz4, lzo, xz or zstd compression + to compress both files, inodes and directories. Inodes in the system + are very small and all blocks are packed to minimise data overhead. + Block sizes greater than 4K are supported up to a maximum of 1 Mbytes + (default block size 128K). SquashFS 4.0 supports 64 bit filesystems + and files (larger than 4GB), full uid/gid information, hard links and + timestamps. + + Squashfs is intended for general read-only filesystem use, for + archival use (i.e. in cases where a .tar.gz file may be used), and in + embedded systems where low overhead is needed. Further information + and tools are available from github.com/plougher/squashfs-tools. + + If you want to compile this as a module ( = code which can be + inserted in and removed from the running kernel whenever you want), + say M here. The module will be called squashfs. Note that the root + file system (the one containing the directory /) cannot be compiled + as a module. + + If unsure, say N. + +choice + prompt "File decompression options" + depends on SQUASHFS + help + Squashfs now supports two options for decompressing file + data. Traditionally Squashfs has decompressed into an + intermediate buffer and then memcopied it into the page cache. + Squashfs now supports the ability to decompress directly into + the page cache. + + If unsure, select "Decompress file data into an intermediate buffer" + +config SQUASHFS_FILE_CACHE + bool "Decompress file data into an intermediate buffer" + help + Decompress file data into an intermediate buffer and then + memcopy it into the page cache. + +config SQUASHFS_FILE_DIRECT + bool "Decompress files directly into the page cache" + help + Directly decompress file data into the page cache. + Doing so can significantly improve performance because + it eliminates a memcpy and it also removes the lock contention + on the single buffer. + +endchoice + +config SQUASHFS_DECOMP_SINGLE + depends on SQUASHFS + def_bool n + +config SQUASHFS_DECOMP_MULTI + depends on SQUASHFS + def_bool n + +config SQUASHFS_DECOMP_MULTI_PERCPU + depends on SQUASHFS + def_bool n + +config SQUASHFS_CHOICE_DECOMP_BY_MOUNT + bool "Select the parallel decompression mode during mount" + depends on SQUASHFS + default n + select SQUASHFS_DECOMP_SINGLE + select SQUASHFS_DECOMP_MULTI + select SQUASHFS_DECOMP_MULTI_PERCPU + select SQUASHFS_MOUNT_DECOMP_THREADS + help + Compile all parallel decompression modes and specify the + decompression mode by setting "threads=" during mount. + default Decompressor parallelisation is SQUASHFS_DECOMP_SINGLE + +choice + prompt "Select decompression parallel mode at compile time" + depends on SQUASHFS + depends on !SQUASHFS_CHOICE_DECOMP_BY_MOUNT + help + Squashfs now supports three parallelisation options for + decompression. Each one exhibits various trade-offs between + decompression performance and CPU and memory usage. + + If in doubt, select "Single threaded compression" + +config SQUASHFS_COMPILE_DECOMP_SINGLE + bool "Single threaded compression" + select SQUASHFS_DECOMP_SINGLE + help + Traditionally Squashfs has used single-threaded decompression. + Only one block (data or metadata) can be decompressed at any + one time. This limits CPU and memory usage to a minimum. + +config SQUASHFS_COMPILE_DECOMP_MULTI + bool "Use multiple decompressors for parallel I/O" + select SQUASHFS_DECOMP_MULTI + help + By default Squashfs uses a single decompressor but it gives + poor performance on parallel I/O workloads when using multiple CPU + machines due to waiting on decompressor availability. + + If you have a parallel I/O workload and your system has enough memory, + using this option may improve overall I/O performance. + + This decompressor implementation uses up to two parallel + decompressors per core. It dynamically allocates decompressors + on a demand basis. + +config SQUASHFS_COMPILE_DECOMP_MULTI_PERCPU + bool "Use percpu multiple decompressors for parallel I/O" + select SQUASHFS_DECOMP_MULTI_PERCPU + help + By default Squashfs uses a single decompressor but it gives + poor performance on parallel I/O workloads when using multiple CPU + machines due to waiting on decompressor availability. + + This decompressor implementation uses a maximum of one + decompressor per core. It uses percpu variables to ensure + decompression is load-balanced across the cores. +endchoice + +config SQUASHFS_MOUNT_DECOMP_THREADS + bool "Add the mount parameter 'threads=' for squashfs" + depends on SQUASHFS + depends on SQUASHFS_DECOMP_MULTI + default n + help + Use threads= to set the decompression parallel mode and the number of threads. + If SQUASHFS_CHOICE_DECOMP_BY_MOUNT=y + threads= + else + threads=<2|3|...> + The upper limit is num_online_cpus() * 2. + +config SQUASHFS_XATTR + bool "Squashfs XATTR support" + depends on SQUASHFS + help + Saying Y here includes support for extended attributes (xattrs). + Xattrs are name:value pairs associated with inodes by + the kernel or by users (see the attr(5) manual page). + + If unsure, say N. + +config SQUASHFS_COMP_CACHE_FULL + bool "Enable full caching of compressed blocks" + depends on SQUASHFS + default n + help + This option enables caching of all compressed blocks, Without caching, + repeated reads of the same files trigger excessive disk I/O, significantly + reducinng performance in workloads like fio-based benchmarks. + + For example, fio tests (iodepth=1, numjobs=1, ioengine=psync) show: + With caching: IOPS=2223, BW=278MiB/s (291MB/s) + Without caching: IOPS=815, BW=102MiB/s (107MB/s) + + Enabling this option restores performance to pre-regression levels by + caching all compressed blocks in the page cache, reducing disk I/O for + repeated reads. However, this increases memory usage, which may be a + concern in memory-constrained environments. + + Enable this option if your workload involves frequent repeated reads and + memory usage is not a limiting factor. If unsure, say N. + +config SQUASHFS_ZLIB + bool "Include support for ZLIB compressed file systems" + depends on SQUASHFS + select ZLIB_INFLATE + default y + help + ZLIB compression is the standard compression used by Squashfs + file systems. It offers a good trade-off between compression + achieved and the amount of CPU time and memory necessary to + compress and decompress. + + If unsure, say Y. + +config SQUASHFS_LZ4 + bool "Include support for LZ4 compressed file systems" + depends on SQUASHFS + select LZ4_DECOMPRESS + help + Saying Y here includes support for reading Squashfs file systems + compressed with LZ4 compression. LZ4 compression is mainly + aimed at embedded systems with slower CPUs where the overheads + of zlib are too high. + + LZ4 is not the standard compression used in Squashfs and so most + file systems will be readable without selecting this option. + + If unsure, say N. + +config SQUASHFS_LZO + bool "Include support for LZO compressed file systems" + depends on SQUASHFS + select LZO_DECOMPRESS + help + Saying Y here includes support for reading Squashfs file systems + compressed with LZO compression. LZO compression is mainly + aimed at embedded systems with slower CPUs where the overheads + of zlib are too high. + + LZO is not the standard compression used in Squashfs and so most + file systems will be readable without selecting this option. + + If unsure, say N. + +config SQUASHFS_XZ + bool "Include support for XZ compressed file systems" + depends on SQUASHFS + select XZ_DEC + help + Saying Y here includes support for reading Squashfs file systems + compressed with XZ compression. XZ gives better compression than + the default zlib compression, at the expense of greater CPU and + memory overhead. + + XZ is not the standard compression used in Squashfs and so most + file systems will be readable without selecting this option. + + If unsure, say N. + +config SQUASHFS_ZSTD + bool "Include support for ZSTD compressed file systems" + depends on SQUASHFS + select ZSTD_DECOMPRESS + help + Saying Y here includes support for reading Squashfs file systems + compressed with ZSTD compression. ZSTD gives better compression than + the default ZLIB compression, while using less CPU. + + ZSTD is not the standard compression used in Squashfs and so most + file systems will be readable without selecting this option. + + If unsure, say N. + +config SQUASHFS_4K_DEVBLK_SIZE + bool "Use 4K device block size?" + depends on SQUASHFS + help + By default Squashfs sets the dev block size (sb_min_blocksize) + to 1K or the smallest block size supported by the block device + (if larger). This, because blocks are packed together and + unaligned in Squashfs, should reduce latency. + + This, however, gives poor performance on MTD NAND devices where + the optimal I/O size is 4K (even though the devices can support + smaller block sizes). + + Using a 4K device block size may also improve overall I/O + performance for some file access patterns (e.g. sequential + accesses of files in filesystem order) on all media. + + Setting this option will force Squashfs to use a 4K device block + size by default. + + If unsure, say N. + +config SQUASHFS_EMBEDDED + bool "Additional option for memory-constrained systems" + depends on SQUASHFS + help + Saying Y here allows you to specify cache size. + + If unsure, say N. + +config SQUASHFS_FRAGMENT_CACHE_SIZE + int "Number of fragments cached" if SQUASHFS_EMBEDDED + depends on SQUASHFS + default "3" + help + By default SquashFS caches the last 3 fragments read from + the filesystem. Increasing this amount may mean SquashFS + has to re-read fragments less often from disk, at the expense + of extra system memory. Decreasing this amount will mean + SquashFS uses less memory at the expense of extra reads from disk. + + Note there must be at least one cached fragment. Anything + much more than three will probably not make much difference. diff --git a/fs/squashfs/Makefile b/fs/squashfs/Makefile new file mode 100644 index 000000000..477c89a51 --- /dev/null +++ b/fs/squashfs/Makefile @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the linux squashfs routines. +# + +obj-$(CONFIG_SQUASHFS) += squashfs.o +squashfs-y += block.o cache.o dir.o export.o file.o fragment.o id.o inode.o +squashfs-y += namei.o super.o symlink.o decompressor.o page_actor.o +squashfs-$(CONFIG_SQUASHFS_FILE_CACHE) += file_cache.o +squashfs-$(CONFIG_SQUASHFS_FILE_DIRECT) += file_direct.o +squashfs-$(CONFIG_SQUASHFS_DECOMP_SINGLE) += decompressor_single.o +squashfs-$(CONFIG_SQUASHFS_DECOMP_MULTI) += decompressor_multi.o +squashfs-$(CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU) += decompressor_multi_percpu.o +squashfs-$(CONFIG_SQUASHFS_XATTR) += xattr.o xattr_id.o +squashfs-$(CONFIG_SQUASHFS_LZ4) += lz4_wrapper.o +squashfs-$(CONFIG_SQUASHFS_LZO) += lzo_wrapper.o +squashfs-$(CONFIG_SQUASHFS_XZ) += xz_wrapper.o +squashfs-$(CONFIG_SQUASHFS_ZLIB) += zlib_wrapper.o +squashfs-$(CONFIG_SQUASHFS_ZSTD) += zstd_wrapper.o diff --git a/fs/squashfs/block.c b/fs/squashfs/block.c new file mode 100644 index 000000000..2d5850168 --- /dev/null +++ b/fs/squashfs/block.c @@ -0,0 +1,386 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Squashfs - a compressed read only filesystem for Linux + * + * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008 + * Phillip Lougher + * + * block.c + */ + +/* + * This file implements the low-level routines to read and decompress + * datablocks and metadata blocks. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "squashfs_fs.h" +#include "squashfs_fs_sb.h" +#include "squashfs.h" +#include "decompressor.h" +#include "page_actor.h" + +/* + * Returns the amount of bytes copied to the page actor. + */ +static int copy_bio_to_actor(struct bio *bio, + struct squashfs_page_actor *actor, + int offset, int req_length) +{ + void *actor_addr; + struct bvec_iter_all iter_all = {}; + struct bio_vec *bvec = bvec_init_iter_all(&iter_all); + int copied_bytes = 0; + int actor_offset = 0; + + squashfs_actor_nobuff(actor); + actor_addr = squashfs_first_page(actor); + + if (WARN_ON_ONCE(!bio_next_segment(bio, &iter_all))) + return 0; + + while (copied_bytes < req_length) { + int bytes_to_copy = min_t(int, bvec->bv_len - offset, + PAGE_SIZE - actor_offset); + + bytes_to_copy = min_t(int, bytes_to_copy, + req_length - copied_bytes); + if (!IS_ERR(actor_addr)) + memcpy(actor_addr + actor_offset, bvec_virt(bvec) + + offset, bytes_to_copy); + + actor_offset += bytes_to_copy; + copied_bytes += bytes_to_copy; + offset += bytes_to_copy; + + if (actor_offset >= PAGE_SIZE) { + actor_addr = squashfs_next_page(actor); + if (!actor_addr) + break; + actor_offset = 0; + } + if (offset >= bvec->bv_len) { + if (!bio_next_segment(bio, &iter_all)) + break; + offset = 0; + } + } + squashfs_finish_page(actor); + return copied_bytes; +} + +static int squashfs_bio_read_cached(struct bio *fullbio, + struct address_space *cache_mapping, u64 index, int length, + u64 read_start, u64 read_end, int page_count) +{ + struct folio *head_to_cache = NULL, *tail_to_cache = NULL; + struct block_device *bdev = fullbio->bi_bdev; + int start_idx = 0, end_idx = 0; + struct folio_iter fi; + struct bio *bio = NULL; + int idx = 0; + int err = 0; +#ifdef CONFIG_SQUASHFS_COMP_CACHE_FULL + struct folio **cache_folios = kmalloc_objs(*cache_folios, page_count, + GFP_KERNEL | __GFP_ZERO); +#endif + + bio_for_each_folio_all(fi, fullbio) { + struct folio *folio = fi.folio; + + if (folio->mapping == cache_mapping) { + idx++; + continue; + } + + /* + * We only use this when the device block size is the same as + * the page size, so read_start and read_end cover full pages. + * + * Compare these to the original required index and length to + * only cache pages which were requested partially, since these + * are the ones which are likely to be needed when reading + * adjacent blocks. + */ + if (idx == 0 && index != read_start) + head_to_cache = folio; + else if (idx == page_count - 1 && index + length != read_end) + tail_to_cache = folio; +#ifdef CONFIG_SQUASHFS_COMP_CACHE_FULL + /* Cache all pages in the BIO for repeated reads */ + else if (cache_folios) + cache_folios[idx] = folio; +#endif + + if (!bio || idx != end_idx) { + struct bio *new = bio_alloc_clone(bdev, fullbio, + GFP_NOIO, &fs_bio_set); + + if (bio) { + bio_trim(bio, start_idx * PAGE_SECTORS, + (end_idx - start_idx) * PAGE_SECTORS); + bio_chain(bio, new); + submit_bio(bio); + } + + bio = new; + start_idx = idx; + } + + idx++; + end_idx = idx; + } + + if (bio) { + bio_trim(bio, start_idx * PAGE_SECTORS, + (end_idx - start_idx) * PAGE_SECTORS); + err = submit_bio_wait(bio); + bio_put(bio); + } + + if (err) + return err; + + if (head_to_cache) { + int ret = filemap_add_folio(cache_mapping, head_to_cache, + read_start >> PAGE_SHIFT, + GFP_NOIO); + + if (!ret) { + folio_mark_uptodate(head_to_cache); + folio_unlock(head_to_cache); + } + + } + + if (tail_to_cache) { + int ret = filemap_add_folio(cache_mapping, tail_to_cache, + (read_end >> PAGE_SHIFT) - 1, + GFP_NOIO); + + if (!ret) { + folio_mark_uptodate(tail_to_cache); + folio_unlock(tail_to_cache); + } + } + +#ifdef CONFIG_SQUASHFS_COMP_CACHE_FULL + if (!cache_folios) + goto out; + + for (idx = 0; idx < page_count; idx++) { + if (!cache_folios[idx]) + continue; + int ret = filemap_add_folio(cache_mapping, cache_folios[idx], + (read_start >> PAGE_SHIFT) + idx, + GFP_NOIO); + + if (!ret) { + folio_mark_uptodate(cache_folios[idx]); + folio_unlock(cache_folios[idx]); + } + } + kfree(cache_folios); +out: +#endif + return 0; +} + +static struct page *squashfs_get_cache_page(struct address_space *mapping, + pgoff_t index) +{ + struct page *page; + + if (!mapping) + return NULL; + + page = find_get_page(mapping, index); + if (!page) + return NULL; + + if (!PageUptodate(page)) { + put_page(page); + return NULL; + } + + return page; +} + +static int squashfs_bio_read(struct super_block *sb, u64 index, int length, + struct bio **biop, int *block_offset) +{ + struct squashfs_sb_info *msblk = sb->s_fs_info; + struct address_space *cache_mapping = msblk->cache_mapping; + const u64 read_start = round_down(index, msblk->devblksize); + const sector_t block = read_start >> msblk->devblksize_log2; + const u64 read_end = round_up(index + length, msblk->devblksize); + const sector_t block_end = read_end >> msblk->devblksize_log2; + int offset = read_start - round_down(index, PAGE_SIZE); + int total_len = (block_end - block) << msblk->devblksize_log2; + const int page_count = DIV_ROUND_UP(total_len + offset, PAGE_SIZE); + int error, i; + struct bio *bio; + + bio = bio_kmalloc(page_count, GFP_NOIO); + if (!bio) + return -ENOMEM; + bio_init_inline(bio, sb->s_bdev, page_count, REQ_OP_READ); + bio->bi_iter.bi_sector = block * (msblk->devblksize >> SECTOR_SHIFT); + + for (i = 0; i < page_count; ++i) { + unsigned int len = + min_t(unsigned int, PAGE_SIZE - offset, total_len); + pgoff_t index = (read_start >> PAGE_SHIFT) + i; + struct page *page; + + page = squashfs_get_cache_page(cache_mapping, index); + if (!page) + page = alloc_page(GFP_NOIO); + + if (!page) { + error = -ENOMEM; + goto out_free_bio; + } + + /* + * Use the __ version to avoid merging since we need each page + * to be separate when we check for and avoid cached pages. + */ + __bio_add_page(bio, page, len, offset); + offset = 0; + total_len -= len; + } + + if (cache_mapping) + error = squashfs_bio_read_cached(bio, cache_mapping, index, + length, read_start, read_end, + page_count); + else + error = submit_bio_wait(bio); + if (error) + goto out_free_bio; + + *biop = bio; + *block_offset = index & ((1 << msblk->devblksize_log2) - 1); + return 0; + +out_free_bio: + bio_free_pages(bio); + bio_uninit(bio); + kfree(bio); + return error; +} + +/* + * Read and decompress a metadata block or datablock. Length is non-zero + * if a datablock is being read (the size is stored elsewhere in the + * filesystem), otherwise the length is obtained from the first two bytes of + * the metadata block. A bit in the length field indicates if the block + * is stored uncompressed in the filesystem (usually because compression + * generated a larger block - this does occasionally happen with compression + * algorithms). + */ +int squashfs_read_data(struct super_block *sb, u64 index, int length, + u64 *next_index, struct squashfs_page_actor *output) +{ + struct squashfs_sb_info *msblk = sb->s_fs_info; + struct bio *bio = NULL; + int compressed; + int res; + int offset; + + if (length) { + /* + * Datablock. + */ + compressed = SQUASHFS_COMPRESSED_BLOCK(length); + length = SQUASHFS_COMPRESSED_SIZE_BLOCK(length); + TRACE("Block @ 0x%llx, %scompressed size %d, src size %d\n", + index, compressed ? "" : "un", length, output->length); + } else { + /* + * Metadata block. + */ + const u8 *data; + struct bvec_iter_all iter_all = {}; + struct bio_vec *bvec = bvec_init_iter_all(&iter_all); + + if (index + 2 > msblk->bytes_used) { + res = -EIO; + goto out; + } + res = squashfs_bio_read(sb, index, 2, &bio, &offset); + if (res) + goto out; + + if (WARN_ON_ONCE(!bio_next_segment(bio, &iter_all))) { + res = -EIO; + goto out_free_bio; + } + /* Extract the length of the metadata block */ + data = bvec_virt(bvec); + length = data[offset]; + if (offset < bvec->bv_len - 1) { + length |= data[offset + 1] << 8; + } else { + if (WARN_ON_ONCE(!bio_next_segment(bio, &iter_all))) { + res = -EIO; + goto out_free_bio; + } + data = bvec_virt(bvec); + length |= data[0] << 8; + } + bio_free_pages(bio); + bio_uninit(bio); + kfree(bio); + + compressed = SQUASHFS_COMPRESSED(length); + length = SQUASHFS_COMPRESSED_SIZE(length); + index += 2; + + TRACE("Block @ 0x%llx, %scompressed size %d\n", index - 2, + compressed ? "" : "un", length); + } + if (length <= 0 || length > output->length || + (index + length) > msblk->bytes_used) { + res = -EIO; + goto out; + } + + if (next_index) + *next_index = index + length; + + res = squashfs_bio_read(sb, index, length, &bio, &offset); + if (res) + goto out; + + if (compressed) { + if (!msblk->stream) { + res = -EIO; + goto out_free_bio; + } + res = msblk->thread_ops->decompress(msblk, bio, offset, length, output); + } else { + res = copy_bio_to_actor(bio, output, offset, length); + } + +out_free_bio: + bio_free_pages(bio); + bio_uninit(bio); + kfree(bio); +out: + if (res < 0) { + ERROR("Failed to read block 0x%llx: %d\n", index, res); + if (msblk->panic_on_errors) + panic("squashfs read failed"); + } + + return res; +} diff --git a/fs/squashfs/cache.c b/fs/squashfs/cache.c new file mode 100644 index 000000000..c1a83bb0e --- /dev/null +++ b/fs/squashfs/cache.c @@ -0,0 +1,557 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Squashfs - a compressed read only filesystem for Linux + * + * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008 + * Phillip Lougher + * + * cache.c + */ + +/* + * Blocks in Squashfs are compressed. To avoid repeatedly decompressing + * recently accessed data Squashfs uses two small metadata and fragment caches. + * + * This file implements a generic cache implementation used for both caches, + * plus functions layered ontop of the generic cache implementation to + * access the metadata and fragment caches. + * + * To avoid out of memory and fragmentation issues with vmalloc the cache + * uses sequences of kmalloced PAGE_SIZE buffers. + * + * It should be noted that the cache is not used for file datablocks, these + * are decompressed and cached in the page-cache in the normal way. The + * cache is only used to temporarily cache fragment and metadata blocks + * which have been read as as a result of a metadata (i.e. inode or + * directory) or fragment access. Because metadata and fragments are packed + * together into blocks (to gain greater compression) the read of a particular + * piece of metadata or fragment will retrieve other metadata/fragments which + * have been packed with it, these because of locality-of-reference may be read + * in the near future. Temporarily caching them ensures they are available for + * near future access without requiring an additional read and decompress. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "squashfs_fs.h" +#include "squashfs_fs_sb.h" +#include "squashfs.h" +#include "page_actor.h" + +/* + * Waiters on cache->wait_queue are keyed by the block they want, so a wakeup + * can name who it is for. A NULL key is a capacity wakeup: one entry became + * free, so wake one waiter. A block key is a publication wakeup: that block + * now has an entry, so wake every waiter which can share it. + */ +struct squashfs_cache_wait { + wait_queue_entry_t wait; + u64 block; + bool capacity_wake; +}; + +static int squashfs_cache_wake_function(wait_queue_entry_t *wait, + unsigned int mode, int sync, void *key) +{ + struct squashfs_cache_wait *cache_wait = + container_of(wait, struct squashfs_cache_wait, wait); + u64 *block = key; + + if (block && cache_wait->block != *block) + return 0; + + WRITE_ONCE(cache_wait->capacity_wake, !block); + + /* + * Wake and unlink unconditionally instead of using + * autoremove_wake_function(), which unlinks only when it changed the + * task state. A waiter can be made runnable by something which does + * not go through this queue: wake_up_process() takes TASK_NORMAL, and + * a cgroup v2 thaw calls it on every task in the cgroup, as do + * free_pid() on a pid namespace init and a late rcuwait_wake_up(). + * try_to_wake_up() then fails. Leaving such a waiter queued with a + * reason already recorded would let it act on a freed entry it was not + * given, and the failure would not consume the exclusive budget, so a + * second waiter would be woken for the same entry. + * + * list_del_init_careful() must be the last access to @cache_wait: it + * releases the waiter, whose wait structure lives on its stack, and it + * pairs with list_empty_careful() in finish_wait() to publish the + * store above. __wake_up_common() samples ->flags and the next entry + * before calling here, so it does not touch @wait afterwards either. + */ + default_wake_function(wait, mode, sync, key); + list_del_init_careful(&wait->entry); + + return 1; +} + +static void squashfs_cache_wake_block(struct squashfs_cache *cache, u64 block) +{ + /* nr_exclusive == 0: wake every waiter which matches the key. */ + __wake_up(&cache->wait_queue, TASK_NORMAL, 0, &block); +} + +/* + * Look-up block in cache, and increment usage count. If not in cache, read + * and decompress it from disk. + * + * A caller which finds no free entry sleeps on cache->wait_queue as an + * exclusive waiter, so squashfs_cache_put() releasing one entry wakes exactly + * one task. Because a wakee may find its block published in the meantime and + * share that entry rather than claim the free one, a wakee which shares hands + * its wakeup on to the next waiter. + */ +struct squashfs_cache_entry *squashfs_cache_get(struct super_block *sb, + struct squashfs_cache *cache, u64 block, int length) +{ + int i, n; + struct squashfs_cache_entry *entry; + bool capacity_wake = false; + + spin_lock(&cache->lock); + + while (1) { + bool pending, wake_next, wake_block; + + for (i = cache->curr_blk, n = 0; n < cache->entries; n++) { + if (cache->entry[i].block == block) { + cache->curr_blk = i; + break; + } + i = (i + 1) % cache->entries; + } + + if (n == cache->entries) { + /* + * Block not in cache, if all cache entries are used + * go to sleep waiting for one to become available. + */ + if (cache->unused == 0) { + struct squashfs_cache_wait wait = { + .block = block, + .capacity_wake = false, + }; + + init_wait_func(&wait.wait, + squashfs_cache_wake_function); + cache->num_waiters++; + /* + * Enqueue while still holding cache->lock, so + * that a concurrent lookup either sees us + * queued or we see the block it publishes. + */ + prepare_to_wait_exclusive(&cache->wait_queue, + &wait.wait, TASK_UNINTERRUPTIBLE); + spin_unlock(&cache->lock); + schedule(); + finish_wait(&cache->wait_queue, &wait.wait); + capacity_wake = READ_ONCE(wait.capacity_wake); + spin_lock(&cache->lock); + cache->num_waiters--; + continue; + } + + /* + * At least one unused cache entry. A simple + * round-robin strategy is used to choose the entry to + * be evicted from the cache. + */ + i = cache->next_blk; + for (n = 0; n < cache->entries; n++) { + if (cache->entry[i].refcount == 0) + break; + i = (i + 1) % cache->entries; + } + + cache->next_blk = (i + 1) % cache->entries; + entry = &cache->entry[i]; + + /* + * Initialise chosen cache entry, and fill it in from + * disk. + */ + cache->unused--; + entry->block = block; + entry->refcount = 1; + entry->pending = 1; + entry->num_waiters = 0; + entry->error = 0; + wake_block = cache->num_waiters > 0; + spin_unlock(&cache->lock); + + /* + * The entry is now findable, so release everybody + * queued for this block to share it rather than each + * waiting for an entry of their own. They will block + * on entry->wait_queue below until the read completes. + */ + if (wake_block) + squashfs_cache_wake_block(cache, block); + + entry->length = squashfs_read_data(sb, block, length, + &entry->next_index, entry->actor); + + spin_lock(&cache->lock); + + if (entry->length < 0) + entry->error = entry->length; + + entry->pending = 0; + + /* + * While filling this entry one or more other processes + * have looked it up in the cache, and have slept + * waiting for it to become available. + */ + if (entry->num_waiters) { + spin_unlock(&cache->lock); + wake_up_all(&entry->wait_queue); + } else + spin_unlock(&cache->lock); + + goto out; + } + + /* + * Block already in cache. Increment refcount so it doesn't + * get reused until we're finished with it, if it was + * previously unused there's one less cache entry available + * for reuse. + */ + entry = &cache->entry[i]; + if (entry->refcount == 0) { + cache->unused--; + /* This claims the capacity we were woken for. */ + capacity_wake = false; + } + entry->refcount++; + + /* + * If the entry is currently being filled in by another process + * go to sleep waiting for it to become available. + */ + pending = entry->pending; + if (pending) + entry->num_waiters++; + + /* + * We were woken because an entry became free, but shared a + * block instead of claiming it. Hand the wakeup on, otherwise + * the free entry sits unclaimed while others sleep. + */ + wake_next = capacity_wake && cache->unused && cache->num_waiters; + spin_unlock(&cache->lock); + + if (wake_next) + wake_up(&cache->wait_queue); + if (pending) + wait_event(entry->wait_queue, !entry->pending); + + goto out; + } + +out: + TRACE("Got %s %d, start block %lld, refcount %d, error %d\n", + cache->name, i, entry->block, entry->refcount, entry->error); + + if (entry->error) + ERROR("Unable to read %s cache entry [%llx]\n", cache->name, + block); + return entry; +} + + +/* + * Release cache entry, once usage count is zero it can be reused. + */ +void squashfs_cache_put(struct squashfs_cache_entry *entry) +{ + struct squashfs_cache *cache = entry->cache; + + spin_lock(&cache->lock); + entry->refcount--; + if (entry->refcount == 0) { + cache->unused++; + /* + * If there's any processes waiting for a block to become + * available, wake one up. + */ + if (cache->num_waiters) { + spin_unlock(&cache->lock); + wake_up(&cache->wait_queue); + return; + } + } + spin_unlock(&cache->lock); +} + +/* + * Delete cache reclaiming all kmalloced buffers. + */ +void squashfs_cache_delete(struct squashfs_cache *cache) +{ + int i, j; + + if (IS_ERR(cache) || cache == NULL) + return; + + for (i = 0; i < cache->entries; i++) { + if (cache->entry[i].data) { + for (j = 0; j < cache->pages; j++) + kfree(cache->entry[i].data[j]); + kfree(cache->entry[i].data); + } + kfree(cache->entry[i].actor); + } + + kfree(cache->entry); + kfree(cache); +} + + +/* + * Initialise cache allocating the specified number of entries, each of + * size block_size. To avoid vmalloc fragmentation issues each entry + * is allocated as a sequence of kmalloced PAGE_SIZE buffers. + */ +struct squashfs_cache *squashfs_cache_init(char *name, int entries, + int block_size) +{ + int i, j; + struct squashfs_cache *cache; + + if (entries == 0) + return NULL; + + cache = kzalloc_obj(*cache); + if (cache == NULL) { + ERROR("Failed to allocate %s cache\n", name); + return ERR_PTR(-ENOMEM); + } + + cache->entry = kzalloc_objs(*(cache->entry), entries); + if (cache->entry == NULL) { + ERROR("Failed to allocate %s cache\n", name); + goto cleanup; + } + + cache->curr_blk = 0; + cache->next_blk = 0; + cache->unused = entries; + cache->entries = entries; + cache->block_size = block_size; + cache->pages = block_size >> PAGE_SHIFT; + cache->pages = cache->pages ? cache->pages : 1; + cache->name = name; + cache->num_waiters = 0; + spin_lock_init(&cache->lock); + init_waitqueue_head(&cache->wait_queue); + + for (i = 0; i < entries; i++) { + struct squashfs_cache_entry *entry = &cache->entry[i]; + + init_waitqueue_head(&cache->entry[i].wait_queue); + entry->cache = cache; + entry->block = SQUASHFS_INVALID_BLK; + entry->data = kcalloc(cache->pages, sizeof(void *), GFP_KERNEL); + if (entry->data == NULL) { + ERROR("Failed to allocate %s cache entry\n", name); + goto cleanup; + } + + for (j = 0; j < cache->pages; j++) { + entry->data[j] = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (entry->data[j] == NULL) { + ERROR("Failed to allocate %s buffer\n", name); + goto cleanup; + } + } + + entry->actor = squashfs_page_actor_init(entry->data, + cache->pages, 0); + if (entry->actor == NULL) { + ERROR("Failed to allocate %s cache entry\n", name); + goto cleanup; + } + } + + return cache; + +cleanup: + squashfs_cache_delete(cache); + return ERR_PTR(-ENOMEM); +} + + +/* + * Copy up to length bytes from cache entry to buffer starting at offset bytes + * into the cache entry. If there's not length bytes then copy the number of + * bytes available. In all cases return the number of bytes copied. + */ +int squashfs_copy_data(void *buffer, struct squashfs_cache_entry *entry, + int offset, int length) +{ + int remaining = length; + + if (length == 0 || offset < 0) + return 0; + else if (buffer == NULL) + return min(length, entry->length - offset); + + while (offset < entry->length) { + void *buff = entry->data[offset / PAGE_SIZE] + + (offset % PAGE_SIZE); + int bytes = min_t(int, entry->length - offset, + PAGE_SIZE - (offset % PAGE_SIZE)); + + if (bytes >= remaining) { + memcpy(buffer, buff, remaining); + remaining = 0; + break; + } + + memcpy(buffer, buff, bytes); + buffer += bytes; + remaining -= bytes; + offset += bytes; + } + + return length - remaining; +} + + +/* + * Read length bytes from metadata position (block is the + * start of the compressed block on disk, and offset is the offset into + * the block once decompressed). Data is packed into consecutive blocks, + * and length bytes may require reading more than one block. + */ +int squashfs_read_metadata(struct super_block *sb, void *buffer, + u64 *block, int *offset, int length) +{ + struct squashfs_sb_info *msblk = sb->s_fs_info; + int bytes, res = length; + struct squashfs_cache_entry *entry; + + TRACE("Entered squashfs_read_metadata [%llx:%x]\n", *block, *offset); + + if (unlikely(length < 0)) + return -EIO; + + if (unlikely(*offset < 0 || *offset >= SQUASHFS_METADATA_SIZE)) + return -EIO; + + while (length) { + entry = squashfs_cache_get(sb, msblk->block_cache, *block, 0); + if (entry->error) { + res = entry->error; + goto error; + } else if (*offset >= entry->length) { + res = -EIO; + goto error; + } + + bytes = squashfs_copy_data(buffer, entry, *offset, length); + if (buffer) + buffer += bytes; + length -= bytes; + *offset += bytes; + + if (*offset == entry->length) { + *block = entry->next_index; + *offset = 0; + } + + squashfs_cache_put(entry); + } + + return res; + +error: + squashfs_cache_put(entry); + return res; +} + + +/* + * Look-up in the fragmment cache the fragment located at in the + * filesystem. If necessary read and decompress it from disk. + */ +struct squashfs_cache_entry *squashfs_get_fragment(struct super_block *sb, + u64 start_block, int length) +{ + struct squashfs_sb_info *msblk = sb->s_fs_info; + + return squashfs_cache_get(sb, msblk->fragment_cache, start_block, + length); +} + + +/* + * Read and decompress the datablock located at in the + * filesystem. The cache is used here to avoid duplicating locking and + * read/decompress code. + */ +struct squashfs_cache_entry *squashfs_get_datablock(struct super_block *sb, + u64 start_block, int length) +{ + struct squashfs_sb_info *msblk = sb->s_fs_info; + + return squashfs_cache_get(sb, msblk->read_page, start_block, length); +} + + +/* + * Read a filesystem table (uncompressed sequence of bytes) from disk + */ +void *squashfs_read_table(struct super_block *sb, u64 block, int length) +{ + int pages = (length + PAGE_SIZE - 1) >> PAGE_SHIFT; + int i, res; + void *table, *buffer, **data; + struct squashfs_page_actor *actor; + + table = buffer = kmalloc(length, GFP_KERNEL); + if (table == NULL) + return ERR_PTR(-ENOMEM); + + data = kcalloc(pages, sizeof(void *), GFP_KERNEL); + if (data == NULL) { + res = -ENOMEM; + goto failed; + } + + actor = squashfs_page_actor_init(data, pages, length); + if (actor == NULL) { + res = -ENOMEM; + goto failed2; + } + + for (i = 0; i < pages; i++, buffer += PAGE_SIZE) + data[i] = buffer; + + res = squashfs_read_data(sb, block, length | + SQUASHFS_COMPRESSED_BIT_BLOCK, NULL, actor); + + kfree(data); + kfree(actor); + + if (res < 0) + goto failed; + + return table; + +failed2: + kfree(data); +failed: + kfree(table); + return ERR_PTR(res); +} diff --git a/fs/squashfs/decompressor.c b/fs/squashfs/decompressor.c new file mode 100644 index 000000000..a676084be --- /dev/null +++ b/fs/squashfs/decompressor.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Squashfs - a compressed read only filesystem for Linux + * + * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 + * Phillip Lougher + * + * decompressor.c + */ + +#include +#include +#include + +#include "squashfs_fs.h" +#include "squashfs_fs_sb.h" +#include "decompressor.h" +#include "squashfs.h" +#include "page_actor.h" + +/* + * This file (and decompressor.h) implements a decompressor framework for + * Squashfs, allowing multiple decompressors to be easily supported + */ + +static const struct squashfs_decompressor squashfs_lzma_unsupported_comp_ops = { + NULL, NULL, NULL, NULL, LZMA_COMPRESSION, "lzma", 0 +}; + +#ifndef CONFIG_SQUASHFS_LZ4 +static const struct squashfs_decompressor squashfs_lz4_comp_ops = { + NULL, NULL, NULL, NULL, LZ4_COMPRESSION, "lz4", 0 +}; +#endif + +#ifndef CONFIG_SQUASHFS_LZO +static const struct squashfs_decompressor squashfs_lzo_comp_ops = { + NULL, NULL, NULL, NULL, LZO_COMPRESSION, "lzo", 0 +}; +#endif + +#ifndef CONFIG_SQUASHFS_XZ +static const struct squashfs_decompressor squashfs_xz_comp_ops = { + NULL, NULL, NULL, NULL, XZ_COMPRESSION, "xz", 0 +}; +#endif + +#ifndef CONFIG_SQUASHFS_ZLIB +static const struct squashfs_decompressor squashfs_zlib_comp_ops = { + NULL, NULL, NULL, NULL, ZLIB_COMPRESSION, "zlib", 0 +}; +#endif + +#ifndef CONFIG_SQUASHFS_ZSTD +static const struct squashfs_decompressor squashfs_zstd_comp_ops = { + NULL, NULL, NULL, NULL, ZSTD_COMPRESSION, "zstd", 0 +}; +#endif + +static const struct squashfs_decompressor squashfs_unknown_comp_ops = { + NULL, NULL, NULL, NULL, 0, "unknown", 0 +}; + +static const struct squashfs_decompressor *decompressor[] = { + &squashfs_zlib_comp_ops, + &squashfs_lz4_comp_ops, + &squashfs_lzo_comp_ops, + &squashfs_xz_comp_ops, + &squashfs_lzma_unsupported_comp_ops, + &squashfs_zstd_comp_ops, + &squashfs_unknown_comp_ops +}; + + +const struct squashfs_decompressor *squashfs_lookup_decompressor(int id) +{ + int i; + + for (i = 0; decompressor[i]->id; i++) + if (id == decompressor[i]->id) + break; + + return decompressor[i]; +} + + +static void *get_comp_opts(struct super_block *sb, unsigned short flags) +{ + struct squashfs_sb_info *msblk = sb->s_fs_info; + void *buffer = NULL, *comp_opts; + struct squashfs_page_actor *actor = NULL; + int length = 0; + + /* + * Read decompressor specific options from file system if present + */ + if (SQUASHFS_COMP_OPTS(flags)) { + buffer = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (buffer == NULL) { + comp_opts = ERR_PTR(-ENOMEM); + goto out; + } + + actor = squashfs_page_actor_init(&buffer, 1, 0); + if (actor == NULL) { + comp_opts = ERR_PTR(-ENOMEM); + goto out; + } + + length = squashfs_read_data(sb, + sizeof(struct squashfs_super_block), 0, NULL, actor); + + if (length < 0) { + comp_opts = ERR_PTR(length); + goto out; + } + } + + comp_opts = squashfs_comp_opts(msblk, buffer, length); + +out: + kfree(actor); + kfree(buffer); + return comp_opts; +} + + +void *squashfs_decompressor_setup(struct super_block *sb, unsigned short flags) +{ + struct squashfs_sb_info *msblk = sb->s_fs_info; + void *stream, *comp_opts = get_comp_opts(sb, flags); + + if (IS_ERR(comp_opts)) + return comp_opts; + + stream = msblk->thread_ops->create(msblk, comp_opts); + if (IS_ERR(stream)) + kfree(comp_opts); + + return stream; +} diff --git a/fs/squashfs/decompressor.h b/fs/squashfs/decompressor.h new file mode 100644 index 000000000..19ab60834 --- /dev/null +++ b/fs/squashfs/decompressor.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef DECOMPRESSOR_H +#define DECOMPRESSOR_H +/* + * Squashfs - a compressed read only filesystem for Linux + * + * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 + * Phillip Lougher + * + * decompressor.h + */ + +#include + +struct squashfs_decompressor { + void *(*init)(struct squashfs_sb_info *, void *); + void *(*comp_opts)(struct squashfs_sb_info *, void *, int); + void (*free)(void *); + int (*decompress)(struct squashfs_sb_info *, void *, + struct bio *, int, int, struct squashfs_page_actor *); + int id; + char *name; + int alloc_buffer; + int supported; +}; + +static inline void *squashfs_comp_opts(struct squashfs_sb_info *msblk, + void *buff, int length) +{ + return msblk->decompressor->comp_opts ? + msblk->decompressor->comp_opts(msblk, buff, length) : NULL; +} + +#ifdef CONFIG_SQUASHFS_XZ +extern const struct squashfs_decompressor squashfs_xz_comp_ops; +#endif + +#ifdef CONFIG_SQUASHFS_LZ4 +extern const struct squashfs_decompressor squashfs_lz4_comp_ops; +#endif + +#ifdef CONFIG_SQUASHFS_LZO +extern const struct squashfs_decompressor squashfs_lzo_comp_ops; +#endif + +#ifdef CONFIG_SQUASHFS_ZLIB +extern const struct squashfs_decompressor squashfs_zlib_comp_ops; +#endif + +#ifdef CONFIG_SQUASHFS_ZSTD +extern const struct squashfs_decompressor squashfs_zstd_comp_ops; +#endif + +#endif diff --git a/fs/squashfs/decompressor_multi.c b/fs/squashfs/decompressor_multi.c new file mode 100644 index 000000000..5c1aede1e --- /dev/null +++ b/fs/squashfs/decompressor_multi.c @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2013 + * Minchan Kim + */ +#include +#include +#include +#include +#include +#include +#include + +#include "squashfs_fs.h" +#include "squashfs_fs_sb.h" +#include "decompressor.h" +#include "squashfs.h" + +/* + * This file implements multi-threaded decompression in the + * decompressor framework + */ + + +/* + * The reason that multiply two is that a CPU can request new I/O + * while it is waiting previous request. + */ +#define MAX_DECOMPRESSOR (num_online_cpus() * 2) + + +static int squashfs_max_decompressors(void) +{ + return MAX_DECOMPRESSOR; +} + +struct squashfs_stream { + void *comp_opts; + struct list_head strm_list; + struct mutex mutex; + int avail_decomp; + wait_queue_head_t wait; +}; + + +struct decomp_stream { + void *stream; + struct list_head list; +}; + + +static void put_decomp_stream(struct decomp_stream *decomp_strm, + struct squashfs_stream *stream) +{ + mutex_lock(&stream->mutex); + list_add(&decomp_strm->list, &stream->strm_list); + mutex_unlock(&stream->mutex); + wake_up(&stream->wait); +} + +static void *squashfs_decompressor_create(struct squashfs_sb_info *msblk, + void *comp_opts) +{ + struct squashfs_stream *stream; + struct decomp_stream *decomp_strm = NULL; + int err = -ENOMEM; + + stream = kzalloc_obj(*stream); + if (!stream) + goto out; + + stream->comp_opts = comp_opts; + mutex_init(&stream->mutex); + INIT_LIST_HEAD(&stream->strm_list); + init_waitqueue_head(&stream->wait); + + /* + * We should have a decompressor at least as default + * so if we fail to allocate new decompressor dynamically, + * we could always fall back to default decompressor and + * file system works. + */ + decomp_strm = kmalloc_obj(*decomp_strm); + if (!decomp_strm) + goto out; + + decomp_strm->stream = msblk->decompressor->init(msblk, + stream->comp_opts); + if (IS_ERR(decomp_strm->stream)) { + err = PTR_ERR(decomp_strm->stream); + goto out; + } + + list_add(&decomp_strm->list, &stream->strm_list); + stream->avail_decomp = 1; + return stream; + +out: + kfree(decomp_strm); + kfree(stream); + return ERR_PTR(err); +} + + +static void squashfs_decompressor_destroy(struct squashfs_sb_info *msblk) +{ + struct squashfs_stream *stream = msblk->stream; + if (stream) { + struct decomp_stream *decomp_strm; + + while (!list_empty(&stream->strm_list)) { + decomp_strm = list_entry(stream->strm_list.prev, + struct decomp_stream, list); + list_del(&decomp_strm->list); + msblk->decompressor->free(decomp_strm->stream); + kfree(decomp_strm); + stream->avail_decomp--; + } + WARN_ON(stream->avail_decomp); + kfree(stream->comp_opts); + kfree(stream); + } +} + + +static struct decomp_stream *get_decomp_stream(struct squashfs_sb_info *msblk, + struct squashfs_stream *stream) +{ + struct decomp_stream *decomp_strm; + + while (1) { + mutex_lock(&stream->mutex); + + /* There is available decomp_stream */ + if (!list_empty(&stream->strm_list)) { + decomp_strm = list_entry(stream->strm_list.prev, + struct decomp_stream, list); + list_del(&decomp_strm->list); + mutex_unlock(&stream->mutex); + break; + } + + /* + * If there is no available decomp and already full, + * let's wait for releasing decomp from other users. + */ + if (stream->avail_decomp >= msblk->max_thread_num) + goto wait; + + /* Let's allocate new decomp */ + decomp_strm = kmalloc_obj(*decomp_strm); + if (!decomp_strm) + goto wait; + + decomp_strm->stream = msblk->decompressor->init(msblk, + stream->comp_opts); + if (IS_ERR(decomp_strm->stream)) { + kfree(decomp_strm); + goto wait; + } + + stream->avail_decomp++; + WARN_ON(stream->avail_decomp > msblk->max_thread_num); + + mutex_unlock(&stream->mutex); + break; +wait: + /* + * If system memory is tough, let's for other's + * releasing instead of hurting VM because it could + * make page cache thrashing. + */ + mutex_unlock(&stream->mutex); + wait_event(stream->wait, + !list_empty(&stream->strm_list)); + } + + return decomp_strm; +} + + +static int squashfs_decompress(struct squashfs_sb_info *msblk, struct bio *bio, + int offset, int length, + struct squashfs_page_actor *output) +{ + int res; + struct squashfs_stream *stream = msblk->stream; + struct decomp_stream *decomp_stream = get_decomp_stream(msblk, stream); + res = msblk->decompressor->decompress(msblk, decomp_stream->stream, + bio, offset, length, output); + put_decomp_stream(decomp_stream, stream); + if (res < 0) + ERROR("%s decompression failed, data probably corrupt\n", + msblk->decompressor->name); + return res; +} + +const struct squashfs_decompressor_thread_ops squashfs_decompressor_multi = { + .create = squashfs_decompressor_create, + .destroy = squashfs_decompressor_destroy, + .decompress = squashfs_decompress, + .max_decompressors = squashfs_max_decompressors, +}; diff --git a/fs/squashfs/decompressor_multi_percpu.c b/fs/squashfs/decompressor_multi_percpu.c new file mode 100644 index 000000000..e4d7e507b --- /dev/null +++ b/fs/squashfs/decompressor_multi_percpu.c @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2013 + * Phillip Lougher + */ + +#include +#include +#include +#include + +#include "squashfs_fs.h" +#include "squashfs_fs_sb.h" +#include "decompressor.h" +#include "squashfs.h" + +/* + * This file implements multi-threaded decompression using percpu + * variables, one thread per cpu core. + */ + +struct squashfs_stream { + void *stream; + local_lock_t lock; +}; + +static void *squashfs_decompressor_create(struct squashfs_sb_info *msblk, + void *comp_opts) +{ + struct squashfs_stream *stream; + struct squashfs_stream __percpu *percpu; + int err, cpu; + + percpu = alloc_percpu(struct squashfs_stream); + if (percpu == NULL) + return ERR_PTR(-ENOMEM); + + for_each_possible_cpu(cpu) { + stream = per_cpu_ptr(percpu, cpu); + stream->stream = msblk->decompressor->init(msblk, comp_opts); + if (IS_ERR(stream->stream)) { + err = PTR_ERR(stream->stream); + goto out; + } + local_lock_init(&stream->lock); + } + + kfree(comp_opts); + return (void *)(__force unsigned long) percpu; + +out: + for_each_possible_cpu(cpu) { + stream = per_cpu_ptr(percpu, cpu); + if (!IS_ERR_OR_NULL(stream->stream)) + msblk->decompressor->free(stream->stream); + } + free_percpu(percpu); + return ERR_PTR(err); +} + +static void squashfs_decompressor_destroy(struct squashfs_sb_info *msblk) +{ + struct squashfs_stream __percpu *percpu = + (void __percpu *)(unsigned long) msblk->stream; + struct squashfs_stream *stream; + int cpu; + + if (msblk->stream) { + for_each_possible_cpu(cpu) { + stream = per_cpu_ptr(percpu, cpu); + msblk->decompressor->free(stream->stream); + } + free_percpu(percpu); + } +} + +static int squashfs_decompress(struct squashfs_sb_info *msblk, struct bio *bio, + int offset, int length, struct squashfs_page_actor *output) +{ + struct squashfs_stream *stream; + struct squashfs_stream __percpu *percpu = + (void __percpu *)(unsigned long) msblk->stream; + int res; + + local_lock(&percpu->lock); + stream = this_cpu_ptr(percpu); + + res = msblk->decompressor->decompress(msblk, stream->stream, bio, + offset, length, output); + + local_unlock(&percpu->lock); + + if (res < 0) + ERROR("%s decompression failed, data probably corrupt\n", + msblk->decompressor->name); + + return res; +} + +static int squashfs_max_decompressors(void) +{ + return num_possible_cpus(); +} + +const struct squashfs_decompressor_thread_ops squashfs_decompressor_percpu = { + .create = squashfs_decompressor_create, + .destroy = squashfs_decompressor_destroy, + .decompress = squashfs_decompress, + .max_decompressors = squashfs_max_decompressors, +}; diff --git a/fs/squashfs/decompressor_single.c b/fs/squashfs/decompressor_single.c new file mode 100644 index 000000000..48501ba55 --- /dev/null +++ b/fs/squashfs/decompressor_single.c @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2013 + * Phillip Lougher + */ + +#include +#include +#include +#include + +#include "squashfs_fs.h" +#include "squashfs_fs_sb.h" +#include "decompressor.h" +#include "squashfs.h" + +/* + * This file implements single-threaded decompression in the + * decompressor framework + */ + +struct squashfs_stream { + void *stream; + struct mutex mutex; +}; + +static void *squashfs_decompressor_create(struct squashfs_sb_info *msblk, + void *comp_opts) +{ + struct squashfs_stream *stream; + int err = -ENOMEM; + + stream = kmalloc_obj(*stream); + if (stream == NULL) + goto out; + + stream->stream = msblk->decompressor->init(msblk, comp_opts); + if (IS_ERR(stream->stream)) { + err = PTR_ERR(stream->stream); + goto out; + } + + kfree(comp_opts); + mutex_init(&stream->mutex); + return stream; + +out: + kfree(stream); + return ERR_PTR(err); +} + +static void squashfs_decompressor_destroy(struct squashfs_sb_info *msblk) +{ + struct squashfs_stream *stream = msblk->stream; + + if (stream) { + msblk->decompressor->free(stream->stream); + kfree(stream); + } +} + +static int squashfs_decompress(struct squashfs_sb_info *msblk, struct bio *bio, + int offset, int length, + struct squashfs_page_actor *output) +{ + int res; + struct squashfs_stream *stream = msblk->stream; + + mutex_lock(&stream->mutex); + res = msblk->decompressor->decompress(msblk, stream->stream, bio, + offset, length, output); + mutex_unlock(&stream->mutex); + + if (res < 0) + ERROR("%s decompression failed, data probably corrupt\n", + msblk->decompressor->name); + + return res; +} + +static int squashfs_max_decompressors(void) +{ + return 1; +} + +const struct squashfs_decompressor_thread_ops squashfs_decompressor_single = { + .create = squashfs_decompressor_create, + .destroy = squashfs_decompressor_destroy, + .decompress = squashfs_decompress, + .max_decompressors = squashfs_max_decompressors, +}; diff --git a/fs/squashfs/dir.c b/fs/squashfs/dir.c new file mode 100644 index 000000000..cd3598bd0 --- /dev/null +++ b/fs/squashfs/dir.c @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Squashfs - a compressed read only filesystem for Linux + * + * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008 + * Phillip Lougher + * + * dir.c + */ + +/* + * This file implements code to read directories from disk. + * + * See namei.c for a description of directory organisation on disk. + */ + +#include +#include +#include +#include + +#include "squashfs_fs.h" +#include "squashfs_fs_sb.h" +#include "squashfs_fs_i.h" +#include "squashfs.h" + +static const unsigned char squashfs_filetype_table[] = { + DT_UNKNOWN, DT_DIR, DT_REG, DT_LNK, DT_BLK, DT_CHR, DT_FIFO, DT_SOCK +}; + +/* + * Lookup offset (f_pos) in the directory index, returning the + * metadata block containing it. + * + * If we get an error reading the index then return the part of the index + * (if any) we have managed to read - the index isn't essential, just + * quicker. + */ +static int get_dir_index_using_offset(struct super_block *sb, + u64 *next_block, int *next_offset, u64 index_start, int index_offset, + int i_count, u64 f_pos) +{ + struct squashfs_sb_info *msblk = sb->s_fs_info; + int err, i, index, length = 0; + unsigned int size; + struct squashfs_dir_index dir_index; + + TRACE("Entered get_dir_index_using_offset, i_count %d, f_pos %lld\n", + i_count, f_pos); + + /* + * Translate from external f_pos to the internal f_pos. This + * is offset by 3 because we invent "." and ".." entries which are + * not actually stored in the directory. + */ + if (f_pos <= 3) + return f_pos; + f_pos -= 3; + + for (i = 0; i < i_count; i++) { + err = squashfs_read_metadata(sb, &dir_index, &index_start, + &index_offset, sizeof(dir_index)); + if (err < 0) + break; + + index = le32_to_cpu(dir_index.index); + if (index > f_pos) + /* + * Found the index we're looking for. + */ + break; + + size = le32_to_cpu(dir_index.size) + 1; + + /* size should never be larger than SQUASHFS_NAME_LEN */ + if (size > SQUASHFS_NAME_LEN) + break; + + err = squashfs_read_metadata(sb, NULL, &index_start, + &index_offset, size); + if (err < 0) + break; + + length = index; + *next_block = le32_to_cpu(dir_index.start_block) + + msblk->directory_table; + } + + *next_offset = (length + *next_offset) % SQUASHFS_METADATA_SIZE; + + /* + * Translate back from internal f_pos to external f_pos. + */ + return length + 3; +} + + +static int squashfs_readdir(struct file *file, struct dir_context *ctx) +{ + struct inode *inode = file_inode(file); + struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info; + u64 block = squashfs_i(inode)->start + msblk->directory_table; + int offset = squashfs_i(inode)->offset, length, err; + unsigned int inode_number, dir_count, size, type; + struct squashfs_dir_header dirh; + struct squashfs_dir_entry *dire; + + TRACE("Entered squashfs_readdir [%llx:%x]\n", block, offset); + + dire = kmalloc(sizeof(*dire) + SQUASHFS_NAME_LEN + 1, GFP_KERNEL); + if (dire == NULL) { + ERROR("Failed to allocate squashfs_dir_entry\n"); + goto finish; + } + + /* + * Return "." and ".." entries as the first two filenames in the + * directory. To maximise compression these two entries are not + * stored in the directory, and so we invent them here. + * + * It also means that the external f_pos is offset by 3 from the + * on-disk directory f_pos. + */ + while (ctx->pos < 3) { + char *name; + int i_ino; + + if (ctx->pos == 0) { + name = "."; + size = 1; + i_ino = inode->i_ino; + } else { + name = ".."; + size = 2; + i_ino = squashfs_i(inode)->parent; + } + + if (!dir_emit(ctx, name, size, i_ino, + squashfs_filetype_table[1])) + goto finish; + + ctx->pos += size; + } + + length = get_dir_index_using_offset(inode->i_sb, &block, &offset, + squashfs_i(inode)->dir_idx_start, + squashfs_i(inode)->dir_idx_offset, + squashfs_i(inode)->dir_idx_cnt, + ctx->pos); + + while (length < i_size_read(inode)) { + /* + * Read directory header + */ + err = squashfs_read_metadata(inode->i_sb, &dirh, &block, + &offset, sizeof(dirh)); + if (err < 0) + goto failed_read; + + length += sizeof(dirh); + + dir_count = le32_to_cpu(dirh.count) + 1; + + if (dir_count > SQUASHFS_DIR_COUNT) + goto failed_read; + + while (dir_count--) { + /* + * Read directory entry. + */ + err = squashfs_read_metadata(inode->i_sb, dire, &block, + &offset, sizeof(*dire)); + if (err < 0) + goto failed_read; + + size = le16_to_cpu(dire->size) + 1; + + /* size should never be larger than SQUASHFS_NAME_LEN */ + if (size > SQUASHFS_NAME_LEN) + goto failed_read; + + err = squashfs_read_metadata(inode->i_sb, dire->name, + &block, &offset, size); + if (err < 0) + goto failed_read; + + length += sizeof(*dire) + size; + + if (ctx->pos >= length) + continue; + + dire->name[size] = '\0'; + inode_number = le32_to_cpu(dirh.inode_number) + + ((short) le16_to_cpu(dire->inode_number)); + type = le16_to_cpu(dire->type); + + if (type > SQUASHFS_MAX_DIR_TYPE) + goto failed_read; + + if (!dir_emit(ctx, dire->name, size, + inode_number, + squashfs_filetype_table[type])) + goto finish; + + ctx->pos = length; + } + } + +finish: + kfree(dire); + return 0; + +failed_read: + ERROR("Unable to read directory block [%llx:%x]\n", block, offset); + kfree(dire); + return 0; +} + + +const struct file_operations squashfs_dir_ops = { + .read = generic_read_dir, + .iterate_shared = squashfs_readdir, + .llseek = generic_file_llseek, + .setlease = generic_setlease, +}; diff --git a/fs/squashfs/export.c b/fs/squashfs/export.c new file mode 100644 index 000000000..62972f0ff --- /dev/null +++ b/fs/squashfs/export.c @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Squashfs - a compressed read only filesystem for Linux + * + * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008 + * Phillip Lougher + * + * export.c + */ + +/* + * This file implements code to make Squashfs filesystems exportable (NFS etc.) + * + * The export code uses an inode lookup table to map inode numbers passed in + * filehandles to an inode location on disk. This table is stored compressed + * into metadata blocks. A second index table is used to locate these. This + * second index table for speed of access (and because it is small) is read at + * mount time and cached in memory. + * + * The inode lookup table is used only by the export code, inode disk + * locations are directly encoded in directories, enabling direct access + * without an intermediate lookup for all operations except the export ops. + */ + +#include +#include +#include +#include +#include + +#include "squashfs_fs.h" +#include "squashfs_fs_sb.h" +#include "squashfs_fs_i.h" +#include "squashfs.h" + +/* + * Look-up inode number (ino) in table, returning the inode location. + */ +static long long squashfs_inode_lookup(struct super_block *sb, int ino_num) +{ + struct squashfs_sb_info *msblk = sb->s_fs_info; + int blk = SQUASHFS_LOOKUP_BLOCK(ino_num - 1); + int offset = SQUASHFS_LOOKUP_BLOCK_OFFSET(ino_num - 1); + u64 start; + __le64 ino; + int err; + + TRACE("Entered squashfs_inode_lookup, inode_number = %d\n", ino_num); + + if (ino_num == 0 || (ino_num - 1) >= msblk->inodes) + return -EINVAL; + + start = le64_to_cpu(msblk->inode_lookup_table[blk]); + + err = squashfs_read_metadata(sb, &ino, &start, &offset, sizeof(ino)); + if (err < 0) + return err; + + TRACE("squashfs_inode_lookup, inode = 0x%llx\n", + (u64) le64_to_cpu(ino)); + + return le64_to_cpu(ino); +} + + +static struct dentry *squashfs_export_iget(struct super_block *sb, + unsigned int ino_num) +{ + long long ino; + struct dentry *dentry = ERR_PTR(-ENOENT); + + TRACE("Entered squashfs_export_iget\n"); + + ino = squashfs_inode_lookup(sb, ino_num); + if (ino >= 0) + dentry = d_obtain_alias(squashfs_iget(sb, ino, ino_num)); + + return dentry; +} + + +static struct dentry *squashfs_fh_to_dentry(struct super_block *sb, + struct fid *fid, int fh_len, int fh_type) +{ + if ((fh_type != FILEID_INO32_GEN && fh_type != FILEID_INO32_GEN_PARENT) + || fh_len < 2) + return NULL; + + return squashfs_export_iget(sb, fid->i32.ino); +} + + +static struct dentry *squashfs_fh_to_parent(struct super_block *sb, + struct fid *fid, int fh_len, int fh_type) +{ + if (fh_type != FILEID_INO32_GEN_PARENT || fh_len < 4) + return NULL; + + return squashfs_export_iget(sb, fid->i32.parent_ino); +} + + +static struct dentry *squashfs_get_parent(struct dentry *child) +{ + struct inode *inode = d_inode(child); + unsigned int parent_ino = squashfs_i(inode)->parent; + + return squashfs_export_iget(inode->i_sb, parent_ino); +} + + +/* + * Read uncompressed inode lookup table indexes off disk into memory + */ +__le64 *squashfs_read_inode_lookup_table(struct super_block *sb, + u64 lookup_table_start, u64 next_table, unsigned int inodes) +{ + unsigned int length = SQUASHFS_LOOKUP_BLOCK_BYTES(inodes); + unsigned int indexes = SQUASHFS_LOOKUP_BLOCKS(inodes); + int n; + __le64 *table; + u64 start, end; + + TRACE("In read_inode_lookup_table, length %d\n", length); + + /* Sanity check values */ + + /* there should always be at least one inode */ + if (inodes == 0) + return ERR_PTR(-EINVAL); + + /* + * The computed size of the lookup table (length bytes) should exactly + * match the table start and end points + */ + if (length != (next_table - lookup_table_start)) + return ERR_PTR(-EINVAL); + + table = squashfs_read_table(sb, lookup_table_start, length); + if (IS_ERR(table)) + return table; + + /* + * table0], table[1], ... table[indexes - 1] store the locations + * of the compressed inode lookup blocks. Each entry should be + * less than the next (i.e. table[0] < table[1]), and the difference + * between them should be SQUASHFS_METADATA_SIZE or less. + * table[indexes - 1] should be less than lookup_table_start, and + * again the difference should be SQUASHFS_METADATA_SIZE or less + */ + for (n = 0; n < (indexes - 1); n++) { + start = le64_to_cpu(table[n]); + end = le64_to_cpu(table[n + 1]); + + if (start >= end + || (end - start) > + (SQUASHFS_METADATA_SIZE + SQUASHFS_BLOCK_OFFSET)) { + kfree(table); + return ERR_PTR(-EINVAL); + } + } + + start = le64_to_cpu(table[indexes - 1]); + if (start >= lookup_table_start || + (lookup_table_start - start) > + (SQUASHFS_METADATA_SIZE + SQUASHFS_BLOCK_OFFSET)) { + kfree(table); + return ERR_PTR(-EINVAL); + } + + return table; +} + + +const struct export_operations squashfs_export_ops = { + .encode_fh = generic_encode_ino32_fh, + .fh_to_dentry = squashfs_fh_to_dentry, + .fh_to_parent = squashfs_fh_to_parent, + .get_parent = squashfs_get_parent +}; diff --git a/fs/squashfs/file.c b/fs/squashfs/file.c new file mode 100644 index 000000000..d83594ce9 --- /dev/null +++ b/fs/squashfs/file.c @@ -0,0 +1,781 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Squashfs - a compressed read only filesystem for Linux + * + * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008 + * Phillip Lougher + * + * file.c + */ + +/* + * This file contains code for handling regular files. A regular file + * consists of a sequence of contiguous compressed blocks, and/or a + * compressed fragment block (tail-end packed block). The compressed size + * of each datablock is stored in a block list contained within the + * file inode (itself stored in one or more compressed metadata blocks). + * + * To speed up access to datablocks when reading 'large' files (256 Mbytes or + * larger), the code implements an index cache that caches the mapping from + * block index to datablock location on disk. + * + * The index cache allows Squashfs to handle large files (up to 1.75 TiB) while + * retaining a simple and space-efficient block list on disk. The cache + * is split into slots, caching up to eight 224 GiB files (128 KiB blocks). + * Larger files use multiple slots, with 1.75 TiB files using all 8 slots. + * The index cache is designed to be memory efficient, and by default uses + * 16 KiB. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "squashfs_fs.h" +#include "squashfs_fs_sb.h" +#include "squashfs_fs_i.h" +#include "squashfs.h" +#include "page_actor.h" + +/* + * Locate cache slot in range [offset, index] for specified inode. If + * there's more than one return the slot closest to index. + */ +static struct meta_index *locate_meta_index(struct inode *inode, int offset, + int index) +{ + struct meta_index *meta = NULL; + struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info; + int i; + + mutex_lock(&msblk->meta_index_mutex); + + TRACE("locate_meta_index: index %d, offset %d\n", index, offset); + + if (msblk->meta_index == NULL) + goto not_allocated; + + for (i = 0; i < SQUASHFS_META_SLOTS; i++) { + if (msblk->meta_index[i].inode_number == inode->i_ino && + msblk->meta_index[i].offset >= offset && + msblk->meta_index[i].offset <= index && + msblk->meta_index[i].locked == 0) { + TRACE("locate_meta_index: entry %d, offset %d\n", i, + msblk->meta_index[i].offset); + meta = &msblk->meta_index[i]; + offset = meta->offset; + } + } + + if (meta) + meta->locked = 1; + +not_allocated: + mutex_unlock(&msblk->meta_index_mutex); + + return meta; +} + + +/* + * Find and initialise an empty cache slot for index offset. + */ +static struct meta_index *empty_meta_index(struct inode *inode, int offset, + int skip) +{ + struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info; + struct meta_index *meta = NULL; + int i; + + mutex_lock(&msblk->meta_index_mutex); + + TRACE("empty_meta_index: offset %d, skip %d\n", offset, skip); + + if (msblk->meta_index == NULL) { + /* + * First time cache index has been used, allocate and + * initialise. The cache index could be allocated at + * mount time but doing it here means it is allocated only + * if a 'large' file is read. + */ + msblk->meta_index = kzalloc_objs(*(msblk->meta_index), + SQUASHFS_META_SLOTS); + if (msblk->meta_index == NULL) { + ERROR("Failed to allocate meta_index\n"); + goto failed; + } + for (i = 0; i < SQUASHFS_META_SLOTS; i++) { + msblk->meta_index[i].inode_number = 0; + msblk->meta_index[i].locked = 0; + } + msblk->next_meta_index = 0; + } + + for (i = SQUASHFS_META_SLOTS; i && + msblk->meta_index[msblk->next_meta_index].locked; i--) + msblk->next_meta_index = (msblk->next_meta_index + 1) % + SQUASHFS_META_SLOTS; + + if (i == 0) { + TRACE("empty_meta_index: failed!\n"); + goto failed; + } + + TRACE("empty_meta_index: returned meta entry %d, %p\n", + msblk->next_meta_index, + &msblk->meta_index[msblk->next_meta_index]); + + meta = &msblk->meta_index[msblk->next_meta_index]; + msblk->next_meta_index = (msblk->next_meta_index + 1) % + SQUASHFS_META_SLOTS; + + meta->inode_number = inode->i_ino; + meta->offset = offset; + meta->skip = skip; + meta->entries = 0; + meta->locked = 1; + +failed: + mutex_unlock(&msblk->meta_index_mutex); + return meta; +} + + +static void release_meta_index(struct inode *inode, struct meta_index *meta) +{ + struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info; + mutex_lock(&msblk->meta_index_mutex); + meta->locked = 0; + mutex_unlock(&msblk->meta_index_mutex); +} + + +/* + * Read the next n blocks from the block list, starting from + * metadata block . + */ +static long long read_indexes(struct super_block *sb, int n, + u64 *start_block, int *offset) +{ + int err, i; + long long block = 0; + __le32 *blist = kmalloc(PAGE_SIZE, GFP_KERNEL); + + if (blist == NULL) { + ERROR("read_indexes: Failed to allocate block_list\n"); + return -ENOMEM; + } + + while (n) { + int blocks = min_t(int, n, PAGE_SIZE >> 2); + + err = squashfs_read_metadata(sb, blist, start_block, + offset, blocks << 2); + if (err < 0) { + ERROR("read_indexes: reading block [%llx:%x]\n", + *start_block, *offset); + goto failure; + } + + for (i = 0; i < blocks; i++) { + int size = squashfs_block_size(blist[i]); + if (size < 0) { + err = size; + goto failure; + } + block += SQUASHFS_COMPRESSED_SIZE_BLOCK(size); + } + n -= blocks; + } + + kfree(blist); + return block; + +failure: + kfree(blist); + return err; +} + + +/* + * Each cache index slot has SQUASHFS_META_ENTRIES, each of which + * can cache one index -> datablock/blocklist-block mapping. We wish + * to distribute these over the length of the file, entry[0] maps index x, + * entry[1] maps index x + skip, entry[2] maps index x + 2 * skip, and so on. + * The larger the file, the greater the skip factor. The skip factor is + * limited to the size of the metadata cache (SQUASHFS_CACHED_BLKS) to ensure + * the number of metadata blocks that need to be read fits into the cache. + * If the skip factor is limited in this way then the file will use multiple + * slots. + */ +static inline int calculate_skip(u64 blocks) +{ + u64 skip = blocks / ((SQUASHFS_META_ENTRIES + 1) + * SQUASHFS_META_INDEXES); + return min((u64) SQUASHFS_CACHED_BLKS - 1, skip + 1); +} + + +/* + * Search and grow the index cache for the specified inode, returning the + * on-disk locations of the datablock and block list metadata block + * for index (scaled to nearest cache index). + */ +static int fill_meta_index(struct inode *inode, int index, + u64 *index_block, int *index_offset, u64 *data_block) +{ + struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info; + int skip = calculate_skip(i_size_read(inode) >> msblk->block_log); + int offset = 0; + struct meta_index *meta; + struct meta_entry *meta_entry; + u64 cur_index_block = squashfs_i(inode)->block_list_start; + int cur_offset = squashfs_i(inode)->offset; + u64 cur_data_block = squashfs_i(inode)->start; + int err, i; + + /* + * Scale index to cache index (cache slot entry) + */ + index /= SQUASHFS_META_INDEXES * skip; + + while (offset < index) { + meta = locate_meta_index(inode, offset + 1, index); + + if (meta == NULL) { + meta = empty_meta_index(inode, offset + 1, skip); + if (meta == NULL) + goto all_done; + } else { + offset = index < meta->offset + meta->entries ? index : + meta->offset + meta->entries - 1; + meta_entry = &meta->meta_entry[offset - meta->offset]; + cur_index_block = meta_entry->index_block + + msblk->inode_table; + cur_offset = meta_entry->offset; + cur_data_block = meta_entry->data_block; + TRACE("get_meta_index: offset %d, meta->offset %d, " + "meta->entries %d\n", offset, meta->offset, + meta->entries); + TRACE("get_meta_index: index_block 0x%llx, offset 0x%x" + " data_block 0x%llx\n", cur_index_block, + cur_offset, cur_data_block); + } + + /* + * If necessary grow cache slot by reading block list. Cache + * slot is extended up to index or to the end of the slot, in + * which case further slots will be used. + */ + for (i = meta->offset + meta->entries; i <= index && + i < meta->offset + SQUASHFS_META_ENTRIES; i++) { + int blocks = skip * SQUASHFS_META_INDEXES; + long long res = read_indexes(inode->i_sb, blocks, + &cur_index_block, &cur_offset); + + if (res < 0) { + if (meta->entries == 0) + /* + * Don't leave an empty slot on read + * error allocated to this inode... + */ + meta->inode_number = 0; + err = res; + goto failed; + } + + cur_data_block += res; + meta_entry = &meta->meta_entry[i - meta->offset]; + meta_entry->index_block = cur_index_block - + msblk->inode_table; + meta_entry->offset = cur_offset; + meta_entry->data_block = cur_data_block; + meta->entries++; + offset++; + } + + TRACE("get_meta_index: meta->offset %d, meta->entries %d\n", + meta->offset, meta->entries); + + release_meta_index(inode, meta); + } + +all_done: + *index_block = cur_index_block; + *index_offset = cur_offset; + if (data_block) + *data_block = cur_data_block; + + /* + * Scale cache index (cache slot entry) to index + */ + return offset * SQUASHFS_META_INDEXES * skip; + +failed: + release_meta_index(inode, meta); + return err; +} + + +/* + * Get the on-disk location and compressed size of the datablock + * specified by index. Fill_meta_index() does most of the work. + */ +static int read_blocklist_ptrs(struct inode *inode, int index, u64 *start, + int *offset, u64 *block) +{ + long long blks; + __le32 size; + int res = fill_meta_index(inode, index, start, offset, block); + + TRACE("read_blocklist: res %d, index %d, start 0x%llx, offset 0x%x, block 0x%llx\n", + res, index, *start, *offset, block ? *block : 0); + + if (res < 0) + return res; + + /* + * res contains the index of the mapping returned by fill_meta_index(), + * this will likely be less than the desired index (because the + * meta_index cache works at a higher granularity). Read any + * extra block indexes needed. + */ + if (res < index) { + blks = read_indexes(inode->i_sb, index - res, start, offset); + if (blks < 0) + return (int) blks; + if (block) + *block += blks; + } + + /* + * Read length of block specified by index. + */ + res = squashfs_read_metadata(inode->i_sb, &size, start, offset, + sizeof(size)); + if (res < 0) + return res; + return squashfs_block_size(size); +} + +static inline int read_blocklist(struct inode *inode, int index, u64 *block) +{ + u64 start; + int offset; + + return read_blocklist_ptrs(inode, index, &start, &offset, block); +} + +static bool squashfs_fill_page(struct folio *folio, + struct squashfs_cache_entry *buffer, size_t offset, + size_t avail) +{ + size_t copied; + void *pageaddr; + + pageaddr = kmap_local_folio(folio, 0); + copied = squashfs_copy_data(pageaddr, buffer, offset, avail); + memset(pageaddr + copied, 0, PAGE_SIZE - copied); + kunmap_local(pageaddr); + + flush_dcache_folio(folio); + + return copied == avail; +} + +/* Copy data into page cache */ +void squashfs_copy_cache(struct folio *folio, + struct squashfs_cache_entry *buffer, size_t bytes, + size_t offset) +{ + struct address_space *mapping = folio->mapping; + struct inode *inode = mapping->host; + struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info; + int i, mask = (1 << (msblk->block_log - PAGE_SHIFT)) - 1; + int start_index = folio->index & ~mask, end_index = start_index | mask; + + /* + * Loop copying datablock into pages. As the datablock likely covers + * many PAGE_SIZE pages (default block size is 128 KiB) explicitly + * grab the pages from the page cache, except for the page that we've + * been called to fill. + */ + for (i = start_index; i <= end_index && bytes > 0; i++, + bytes -= PAGE_SIZE, offset += PAGE_SIZE) { + struct folio *push_folio; + size_t avail = buffer ? min(bytes, PAGE_SIZE) : 0; + bool updated = false; + + TRACE("bytes %zu, i %d, available_bytes %zu\n", bytes, i, avail); + + push_folio = (i == folio->index) ? folio : + __filemap_get_folio(mapping, i, + FGP_LOCK|FGP_CREAT|FGP_NOFS|FGP_NOWAIT, + mapping_gfp_mask(mapping)); + + if (IS_ERR(push_folio)) + continue; + + if (folio_test_uptodate(push_folio)) + goto skip_folio; + + updated = squashfs_fill_page(push_folio, buffer, offset, avail); +skip_folio: + folio_end_read(push_folio, updated); + if (i != folio->index) + folio_put(push_folio); + } +} + +/* Read datablock stored packed inside a fragment (tail-end packed block) */ +static int squashfs_readpage_fragment(struct folio *folio, int expected) +{ + struct inode *inode = folio->mapping->host; + struct squashfs_cache_entry *buffer = squashfs_get_fragment(inode->i_sb, + squashfs_i(inode)->fragment_block, + squashfs_i(inode)->fragment_size); + int res = buffer->error; + + if (res) + ERROR("Unable to read page, block %llx, size %x\n", + squashfs_i(inode)->fragment_block, + squashfs_i(inode)->fragment_size); + else + squashfs_copy_cache(folio, buffer, expected, + squashfs_i(inode)->fragment_offset); + + squashfs_cache_put(buffer); + return res; +} + +static int squashfs_readpage_sparse(struct folio *folio, int expected) +{ + squashfs_copy_cache(folio, NULL, expected, 0); + return 0; +} + +static int squashfs_read_folio(struct file *file, struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info; + int index = folio->index >> (msblk->block_log - PAGE_SHIFT); + int file_end = i_size_read(inode) >> msblk->block_log; + int expected = index == file_end ? + (i_size_read(inode) & (msblk->block_size - 1)) : + msblk->block_size; + int res = 0; + + TRACE("Entered squashfs_readpage, page index %lx, start block %llx\n", + folio->index, squashfs_i(inode)->start); + + if (folio->index >= ((i_size_read(inode) + PAGE_SIZE - 1) >> + PAGE_SHIFT)) + goto out; + + if (index < file_end || squashfs_i(inode)->fragment_block == + SQUASHFS_INVALID_BLK) { + u64 block = 0; + + res = read_blocklist(inode, index, &block); + if (res < 0) + goto out; + + if (res == 0) + res = squashfs_readpage_sparse(folio, expected); + else + res = squashfs_readpage_block(folio, block, res, expected); + } else + res = squashfs_readpage_fragment(folio, expected); + + if (!res) + return 0; + +out: + folio_zero_segment(folio, 0, folio_size(folio)); + folio_end_read(folio, res == 0); + + return res; +} + +static int squashfs_readahead_fragment(struct inode *inode, struct page **page, + unsigned int pages, unsigned int expected, loff_t start) +{ + struct squashfs_cache_entry *buffer = squashfs_get_fragment(inode->i_sb, + squashfs_i(inode)->fragment_block, + squashfs_i(inode)->fragment_size); + struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info; + int i, bytes, copied; + struct squashfs_page_actor *actor; + unsigned int offset; + void *addr; + struct page *last_page; + + if (buffer->error) + goto out; + + actor = squashfs_page_actor_init_special(msblk, page, pages, + expected, start); + if (!actor) + goto out; + + squashfs_actor_nobuff(actor); + addr = squashfs_first_page(actor); + + for (copied = offset = 0; offset < expected; offset += PAGE_SIZE) { + int avail = min_t(int, expected - offset, PAGE_SIZE); + + if (!IS_ERR(addr)) { + bytes = squashfs_copy_data(addr, buffer, offset + + squashfs_i(inode)->fragment_offset, avail); + + if (bytes != avail) + goto failed; + } + + copied += avail; + addr = squashfs_next_page(actor); + } + + last_page = squashfs_page_actor_free(actor); + + if (copied == expected && !IS_ERR(last_page)) { + /* Last page (if present) may have trailing bytes not filled */ + bytes = copied % PAGE_SIZE; + if (bytes && last_page) + memzero_page(last_page, bytes, PAGE_SIZE - bytes); + + for (i = 0; i < pages; i++) { + flush_dcache_page(page[i]); + SetPageUptodate(page[i]); + } + } + + for (i = 0; i < pages; i++) { + unlock_page(page[i]); + put_page(page[i]); + } + + squashfs_cache_put(buffer); + return 0; + +failed: + squashfs_page_actor_free(actor); + +out: + squashfs_cache_put(buffer); + return 1; +} + +static void squashfs_readahead(struct readahead_control *ractl) +{ + struct inode *inode = ractl->mapping->host; + struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info; + size_t mask = (1UL << msblk->block_log) - 1; + unsigned short shift = msblk->block_log - PAGE_SHIFT; + loff_t start = readahead_pos(ractl) & ~mask; + size_t len = readahead_length(ractl) + readahead_pos(ractl) - start; + struct squashfs_page_actor *actor; + unsigned int nr_pages = 0; + struct page **pages; + int i; + loff_t file_end = i_size_read(inode) >> msblk->block_log; + unsigned int max_pages = 1UL << shift; + + readahead_expand(ractl, start, (len | mask) + 1); + + pages = kmalloc_array(max_pages, sizeof(void *), GFP_KERNEL); + if (!pages) + return; + + for (;;) { + int res, bsize; + u64 block = 0; + unsigned int expected; + struct page *last_page; + + expected = start >> msblk->block_log == file_end ? + (i_size_read(inode) & (msblk->block_size - 1)) : + msblk->block_size; + + max_pages = (expected + PAGE_SIZE - 1) >> PAGE_SHIFT; + + nr_pages = __readahead_batch(ractl, pages, max_pages); + if (!nr_pages) + break; + + if (readahead_pos(ractl) >= i_size_read(inode)) + goto skip_pages; + + if (start >> msblk->block_log == file_end && + squashfs_i(inode)->fragment_block != SQUASHFS_INVALID_BLK) { + res = squashfs_readahead_fragment(inode, pages, + nr_pages, expected, start); + if (res) + goto skip_pages; + continue; + } + + bsize = read_blocklist(inode, start >> msblk->block_log, &block); + if (bsize == 0) + goto skip_pages; + + actor = squashfs_page_actor_init_special(msblk, pages, nr_pages, + expected, start); + if (!actor) + goto skip_pages; + + res = squashfs_read_data(inode->i_sb, block, bsize, NULL, actor); + + last_page = squashfs_page_actor_free(actor); + + if (res == expected && !IS_ERR(last_page)) { + int bytes; + + /* Last page (if present) may have trailing bytes not filled */ + bytes = res % PAGE_SIZE; + if (start >> msblk->block_log == file_end && bytes && last_page) + memzero_page(last_page, bytes, + PAGE_SIZE - bytes); + + for (i = 0; i < nr_pages; i++) { + flush_dcache_page(pages[i]); + SetPageUptodate(pages[i]); + } + } + + for (i = 0; i < nr_pages; i++) { + unlock_page(pages[i]); + put_page(pages[i]); + } + + start += readahead_batch_length(ractl); + } + + kfree(pages); + return; + +skip_pages: + for (i = 0; i < nr_pages; i++) { + unlock_page(pages[i]); + put_page(pages[i]); + } + kfree(pages); +} + +static loff_t seek_hole_data(struct file *file, loff_t offset, int whence) +{ + struct inode *inode = file->f_mapping->host; + struct super_block *sb = inode->i_sb; + struct squashfs_sb_info *msblk = sb->s_fs_info; + u64 start, index = offset >> msblk->block_log; + u64 file_end = (i_size_read(inode) + msblk->block_size - 1) >> msblk->block_log; + int s_offset, length; + __le32 *blist = NULL; + + /* reject offset if negative or beyond file end */ + if ((unsigned long long)offset >= i_size_read(inode)) + return -ENXIO; + + /* is offset within tailend and is tailend packed into a fragment? */ + if (index + 1 == file_end && + squashfs_i(inode)->fragment_block != SQUASHFS_INVALID_BLK) { + if (whence == SEEK_DATA) + return offset; + + /* there is an implicit hole at the end of any file */ + return i_size_read(inode); + } + + length = read_blocklist_ptrs(inode, index, &start, &s_offset, NULL); + if (length < 0) + return length; + + /* nothing more to do if offset matches desired whence value */ + if ((length == 0 && whence == SEEK_HOLE) || + (length && whence == SEEK_DATA)) + return offset; + + /* skip scanning forwards if we're at file end */ + if (++ index == file_end) + goto not_found; + + blist = kmalloc(SQUASHFS_SCAN_INDEXES << 2, GFP_KERNEL); + if (blist == NULL) { + ERROR("%s: Failed to allocate block_list\n", __func__); + return -ENOMEM; + } + + while (index < file_end) { + int i, indexes = min(file_end - index, SQUASHFS_SCAN_INDEXES); + + offset = squashfs_read_metadata(sb, blist, &start, &s_offset, indexes << 2); + if (offset < 0) + goto finished; + + for (i = 0; i < indexes; i++) { + length = squashfs_block_size(blist[i]); + if (length < 0) { + offset = length; + goto finished; + } + + /* does this block match desired whence value? */ + if ((length == 0 && whence == SEEK_HOLE) || + (length && whence == SEEK_DATA)) { + offset = (index + i) << msblk->block_log; + goto finished; + } + } + + index += indexes; + } + +not_found: + /* whence value determines what happens */ + if (whence == SEEK_DATA) + offset = -ENXIO; + else + /* there is an implicit hole at the end of any file */ + offset = i_size_read(inode); + +finished: + kfree(blist); + return offset; +} + +static loff_t squashfs_llseek(struct file *file, loff_t offset, int whence) +{ + struct inode *inode = file->f_mapping->host; + + switch (whence) { + default: + return generic_file_llseek(file, offset, whence); + case SEEK_DATA: + case SEEK_HOLE: + offset = seek_hole_data(file, offset, whence); + break; + } + + if (offset < 0) + return offset; + + return vfs_setpos(file, offset, inode->i_sb->s_maxbytes); +} + +const struct address_space_operations squashfs_aops = { + .read_folio = squashfs_read_folio, + .readahead = squashfs_readahead +}; + +const struct file_operations squashfs_file_operations = { + .llseek = squashfs_llseek, + .read_iter = generic_file_read_iter, + .mmap_prepare = generic_file_readonly_mmap_prepare, + .splice_read = filemap_splice_read, + .setlease = generic_setlease, +}; diff --git a/fs/squashfs/file_cache.c b/fs/squashfs/file_cache.c new file mode 100644 index 000000000..40e59a43d --- /dev/null +++ b/fs/squashfs/file_cache.c @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2013 + * Phillip Lougher + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "squashfs_fs.h" +#include "squashfs_fs_sb.h" +#include "squashfs_fs_i.h" +#include "squashfs.h" + +/* Read separately compressed datablock and memcopy into page cache */ +int squashfs_readpage_block(struct folio *folio, u64 block, int bsize, int expected) +{ + struct inode *i = folio->mapping->host; + struct squashfs_cache_entry *buffer = squashfs_get_datablock(i->i_sb, + block, bsize); + int res = buffer->error; + + if (res) + ERROR("Unable to read page, block %llx, size %x\n", block, + bsize); + else + squashfs_copy_cache(folio, buffer, expected, 0); + + squashfs_cache_put(buffer); + return res; +} diff --git a/fs/squashfs/file_direct.c b/fs/squashfs/file_direct.c new file mode 100644 index 000000000..2c3e809d6 --- /dev/null +++ b/fs/squashfs/file_direct.c @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2013 + * Phillip Lougher + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "squashfs_fs.h" +#include "squashfs_fs_sb.h" +#include "squashfs_fs_i.h" +#include "squashfs.h" +#include "page_actor.h" + +/* Read separately compressed datablock directly into page cache */ +int squashfs_readpage_block(struct folio *folio, u64 block, int bsize, + int expected) +{ + struct page *target_page = &folio->page; + struct inode *inode = folio->mapping->host; + struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info; + loff_t file_end = (i_size_read(inode) - 1) >> PAGE_SHIFT; + int mask = (1 << (msblk->block_log - PAGE_SHIFT)) - 1; + loff_t start_index = folio->index & ~mask; + loff_t end_index = start_index | mask; + loff_t index; + int i, pages, bytes, res = -ENOMEM; + struct page **page, *last_page; + struct squashfs_page_actor *actor; + void *pageaddr; + + if (end_index > file_end) + end_index = file_end; + + pages = end_index - start_index + 1; + + page = kmalloc_array(pages, sizeof(void *), GFP_KERNEL); + if (page == NULL) + return res; + + /* Try to grab all the pages covered by the Squashfs block */ + for (i = 0, index = start_index; index <= end_index; index++) { + page[i] = (index == folio->index) ? target_page : + grab_cache_page_nowait(folio->mapping, index); + + if (page[i] == NULL) + continue; + + if (PageUptodate(page[i])) { + unlock_page(page[i]); + put_page(page[i]); + continue; + } + + i++; + } + + pages = i; + + /* + * Create a "page actor" which will kmap and kunmap the + * page cache pages appropriately within the decompressor + */ + actor = squashfs_page_actor_init_special(msblk, page, pages, expected, + start_index << PAGE_SHIFT); + if (actor == NULL) + goto out; + + /* Decompress directly into the page cache buffers */ + res = squashfs_read_data(inode->i_sb, block, bsize, NULL, actor); + + last_page = squashfs_page_actor_free(actor); + + if (res < 0) + goto mark_errored; + + if (res != expected || IS_ERR(last_page)) { + res = -EIO; + goto mark_errored; + } + + /* Last page (if present) may have trailing bytes not filled */ + bytes = res % PAGE_SIZE; + if (end_index == file_end && last_page && bytes) { + pageaddr = kmap_local_page(last_page); + memset(pageaddr + bytes, 0, PAGE_SIZE - bytes); + kunmap_local(pageaddr); + } + + /* Mark pages as uptodate, unlock and release */ + for (i = 0; i < pages; i++) { + flush_dcache_page(page[i]); + SetPageUptodate(page[i]); + unlock_page(page[i]); + if (page[i] != target_page) + put_page(page[i]); + } + + kfree(page); + + return 0; + +mark_errored: + /* Decompression failed. Target_page is + * dealt with by the caller + */ + for (i = 0; i < pages; i++) { + if (page[i] == NULL || page[i] == target_page) + continue; + flush_dcache_page(page[i]); + unlock_page(page[i]); + put_page(page[i]); + } + +out: + kfree(page); + return res; +} diff --git a/fs/squashfs/fragment.c b/fs/squashfs/fragment.c new file mode 100644 index 000000000..49602b9a4 --- /dev/null +++ b/fs/squashfs/fragment.c @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Squashfs - a compressed read only filesystem for Linux + * + * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008 + * Phillip Lougher + * + * fragment.c + */ + +/* + * This file implements code to handle compressed fragments (tail-end packed + * datablocks). + * + * Regular files contain a fragment index which is mapped to a fragment + * location on disk and compressed size using a fragment lookup table. + * Like everything in Squashfs this fragment lookup table is itself stored + * compressed into metadata blocks. A second index table is used to locate + * these. This second index table for speed of access (and because it + * is small) is read at mount time and cached in memory. + */ + +#include +#include +#include + +#include "squashfs_fs.h" +#include "squashfs_fs_sb.h" +#include "squashfs.h" + +/* + * Look-up fragment using the fragment index table. Return the on disk + * location of the fragment and its compressed size + */ +int squashfs_frag_lookup(struct super_block *sb, unsigned int fragment, + u64 *fragment_block) +{ + struct squashfs_sb_info *msblk = sb->s_fs_info; + int block, offset, size; + struct squashfs_fragment_entry fragment_entry; + u64 start_block; + + if (fragment >= msblk->fragments) + return -EIO; + block = SQUASHFS_FRAGMENT_INDEX(fragment); + offset = SQUASHFS_FRAGMENT_INDEX_OFFSET(fragment); + + start_block = le64_to_cpu(msblk->fragment_index[block]); + + size = squashfs_read_metadata(sb, &fragment_entry, &start_block, + &offset, sizeof(fragment_entry)); + if (size < 0) + return size; + + *fragment_block = le64_to_cpu(fragment_entry.start_block); + return squashfs_block_size(fragment_entry.size); +} + + +/* + * Read the uncompressed fragment lookup table indexes off disk into memory + */ +__le64 *squashfs_read_fragment_index_table(struct super_block *sb, + u64 fragment_table_start, u64 next_table, unsigned int fragments) +{ + unsigned int length = SQUASHFS_FRAGMENT_INDEX_BYTES(fragments); + __le64 *table; + + /* + * Sanity check, length bytes should not extend into the next table - + * this check also traps instances where fragment_table_start is + * incorrectly larger than the next table start + */ + if (fragment_table_start + length > next_table) + return ERR_PTR(-EINVAL); + + table = squashfs_read_table(sb, fragment_table_start, length); + + /* + * table[0] points to the first fragment table metadata block, this + * should be less than fragment_table_start + */ + if (!IS_ERR(table) && le64_to_cpu(table[0]) >= fragment_table_start) { + kfree(table); + return ERR_PTR(-EINVAL); + } + + return table; +} diff --git a/fs/squashfs/id.c b/fs/squashfs/id.c new file mode 100644 index 000000000..ea5387679 --- /dev/null +++ b/fs/squashfs/id.c @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Squashfs - a compressed read only filesystem for Linux + * + * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008 + * Phillip Lougher + * + * id.c + */ + +/* + * This file implements code to handle uids and gids. + * + * For space efficiency regular files store uid and gid indexes, which are + * converted to 32-bit uids/gids using an id look up table. This table is + * stored compressed into metadata blocks. A second index table is used to + * locate these. This second index table for speed of access (and because it + * is small) is read at mount time and cached in memory. + */ + +#include +#include +#include + +#include "squashfs_fs.h" +#include "squashfs_fs_sb.h" +#include "squashfs.h" + +/* + * Map uid/gid index into real 32-bit uid/gid using the id look up table + */ +int squashfs_get_id(struct super_block *sb, unsigned int index, + unsigned int *id) +{ + struct squashfs_sb_info *msblk = sb->s_fs_info; + int block = SQUASHFS_ID_BLOCK(index); + int offset = SQUASHFS_ID_BLOCK_OFFSET(index); + u64 start_block; + __le32 disk_id; + int err; + + if (index >= msblk->ids) + return -EINVAL; + + start_block = le64_to_cpu(msblk->id_table[block]); + + err = squashfs_read_metadata(sb, &disk_id, &start_block, &offset, + sizeof(disk_id)); + if (err < 0) + return err; + + *id = le32_to_cpu(disk_id); + return 0; +} + + +/* + * Read uncompressed id lookup table indexes from disk into memory + */ +__le64 *squashfs_read_id_index_table(struct super_block *sb, + u64 id_table_start, u64 next_table, unsigned short no_ids) +{ + unsigned int length = SQUASHFS_ID_BLOCK_BYTES(no_ids); + unsigned int indexes = SQUASHFS_ID_BLOCKS(no_ids); + int n; + __le64 *table; + u64 start, end; + + TRACE("In read_id_index_table, length %d\n", length); + + /* Sanity check values */ + + /* there should always be at least one id */ + if (no_ids == 0) + return ERR_PTR(-EINVAL); + + /* + * The computed size of the index table (length bytes) should exactly + * match the table start and end points + */ + if (length != (next_table - id_table_start)) + return ERR_PTR(-EINVAL); + + table = squashfs_read_table(sb, id_table_start, length); + if (IS_ERR(table)) + return table; + + /* + * table[0], table[1], ... table[indexes - 1] store the locations + * of the compressed id blocks. Each entry should be less than + * the next (i.e. table[0] < table[1]), and the difference between them + * should be SQUASHFS_METADATA_SIZE or less. table[indexes - 1] + * should be less than id_table_start, and again the difference + * should be SQUASHFS_METADATA_SIZE or less + */ + for (n = 0; n < (indexes - 1); n++) { + start = le64_to_cpu(table[n]); + end = le64_to_cpu(table[n + 1]); + + if (start >= end || (end - start) > + (SQUASHFS_METADATA_SIZE + SQUASHFS_BLOCK_OFFSET)) { + kfree(table); + return ERR_PTR(-EINVAL); + } + } + + start = le64_to_cpu(table[indexes - 1]); + if (start >= id_table_start || (id_table_start - start) > + (SQUASHFS_METADATA_SIZE + SQUASHFS_BLOCK_OFFSET)) { + kfree(table); + return ERR_PTR(-EINVAL); + } + + return table; +} diff --git a/fs/squashfs/inode.c b/fs/squashfs/inode.c new file mode 100644 index 000000000..82b687414 --- /dev/null +++ b/fs/squashfs/inode.c @@ -0,0 +1,456 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Squashfs - a compressed read only filesystem for Linux + * + * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008 + * Phillip Lougher + * + * inode.c + */ + +/* + * This file implements code to create and read inodes from disk. + * + * Inodes in Squashfs are identified by a 48-bit inode which encodes the + * location of the compressed metadata block containing the inode, and the byte + * offset into that block where the inode is placed (). + * + * To maximise compression there are different inodes for each file type + * (regular file, directory, device, etc.), the inode contents and length + * varying with the type. + * + * To further maximise compression, two types of regular file inode and + * directory inode are defined: inodes optimised for frequently occurring + * regular files and directories, and extended types where extra + * information has to be stored. + */ + +#include +#include +#include +#include + +#include "squashfs_fs.h" +#include "squashfs_fs_sb.h" +#include "squashfs_fs_i.h" +#include "squashfs.h" +#include "xattr.h" + +/* + * Initialise VFS inode with the base inode information common to all + * Squashfs inode types. Sqsh_ino contains the unswapped base inode + * off disk. + */ +static int squashfs_new_inode(struct super_block *sb, struct inode *inode, + struct squashfs_base_inode *sqsh_ino) +{ + uid_t i_uid; + gid_t i_gid; + int err; + + inode->i_ino = le32_to_cpu(sqsh_ino->inode_number); + if (inode->i_ino == 0) + return -EINVAL; + + err = squashfs_get_id(sb, le16_to_cpu(sqsh_ino->uid), &i_uid); + if (err) + return err; + + err = squashfs_get_id(sb, le16_to_cpu(sqsh_ino->guid), &i_gid); + if (err) + return err; + + i_uid_write(inode, i_uid); + i_gid_write(inode, i_gid); + inode_set_mtime(inode, le32_to_cpu(sqsh_ino->mtime), 0); + inode_set_atime(inode, inode_get_mtime_sec(inode), 0); + inode_set_ctime(inode, inode_get_mtime_sec(inode), 0); + inode->i_mode = le16_to_cpu(sqsh_ino->mode); + inode->i_size = 0; + + /* File type must not be set at this moment, for it will later be set by the caller. */ + if (inode->i_mode & S_IFMT) + err = -EIO; + + return err; +} + + +struct inode *squashfs_iget(struct super_block *sb, long long ino, + unsigned int ino_number) +{ + struct inode *inode = iget_locked(sb, ino_number); + int err; + + TRACE("Entered squashfs_iget\n"); + + if (!inode) + return ERR_PTR(-ENOMEM); + if (!(inode_state_read_once(inode) & I_NEW)) + return inode; + + err = squashfs_read_inode(inode, ino); + if (err) { + iget_failed(inode); + return ERR_PTR(err); + } + + unlock_new_inode(inode); + return inode; +} + + +/* + * Initialise VFS inode by reading inode from inode table (compressed + * metadata). The format and amount of data read depends on type. + */ +int squashfs_read_inode(struct inode *inode, long long ino) +{ + struct super_block *sb = inode->i_sb; + struct squashfs_sb_info *msblk = sb->s_fs_info; + u64 block = SQUASHFS_INODE_BLK(ino) + msblk->inode_table; + int err, type, offset = SQUASHFS_INODE_OFFSET(ino); + union squashfs_inode squashfs_ino; + struct squashfs_base_inode *sqshb_ino = &squashfs_ino.base; + int xattr_id = SQUASHFS_INVALID_XATTR; + + TRACE("Entered squashfs_read_inode\n"); + + /* + * Read inode base common to all inode types. + */ + err = squashfs_read_metadata(sb, sqshb_ino, &block, + &offset, sizeof(*sqshb_ino)); + if (err < 0) + goto failed_read; + + err = squashfs_new_inode(sb, inode, sqshb_ino); + if (err) + goto failed_read; + + block = SQUASHFS_INODE_BLK(ino) + msblk->inode_table; + offset = SQUASHFS_INODE_OFFSET(ino); + + type = le16_to_cpu(sqshb_ino->inode_type); + switch (type) { + case SQUASHFS_REG_TYPE: { + unsigned int frag_offset, frag; + int frag_size; + u64 frag_blk; + struct squashfs_reg_inode *sqsh_ino = &squashfs_ino.reg; + + err = squashfs_read_metadata(sb, sqsh_ino, &block, &offset, + sizeof(*sqsh_ino)); + if (err < 0) + goto failed_read; + + inode->i_size = le32_to_cpu(sqsh_ino->file_size); + frag = le32_to_cpu(sqsh_ino->fragment); + if (frag != SQUASHFS_INVALID_FRAG) { + /* + * the file cannot have a fragment (tailend) and have a + * file size a multiple of the block size + */ + if ((inode->i_size & (msblk->block_size - 1)) == 0) { + err = -EINVAL; + goto failed_read; + } + frag_offset = le32_to_cpu(sqsh_ino->offset); + frag_size = squashfs_frag_lookup(sb, frag, &frag_blk); + if (frag_size < 0) { + err = frag_size; + goto failed_read; + } + } else { + frag_blk = SQUASHFS_INVALID_BLK; + frag_size = 0; + frag_offset = 0; + } + + set_nlink(inode, 1); + inode->i_fop = &squashfs_file_operations; + inode->i_mode |= S_IFREG; + inode->i_blocks = ((inode->i_size - 1) >> 9) + 1; + squashfs_i(inode)->fragment_block = frag_blk; + squashfs_i(inode)->fragment_size = frag_size; + squashfs_i(inode)->fragment_offset = frag_offset; + squashfs_i(inode)->start = le32_to_cpu(sqsh_ino->start_block); + squashfs_i(inode)->block_list_start = block; + squashfs_i(inode)->offset = offset; + squashfs_i(inode)->parent = 0; + inode->i_data.a_ops = &squashfs_aops; + + TRACE("File inode %x:%x, start_block %llx, block_list_start " + "%llx, offset %x\n", SQUASHFS_INODE_BLK(ino), + offset, squashfs_i(inode)->start, block, offset); + break; + } + case SQUASHFS_LREG_TYPE: { + unsigned int frag_offset, frag; + int frag_size; + u64 frag_blk; + struct squashfs_lreg_inode *sqsh_ino = &squashfs_ino.lreg; + + err = squashfs_read_metadata(sb, sqsh_ino, &block, &offset, + sizeof(*sqsh_ino)); + if (err < 0) + goto failed_read; + + inode->i_size = le64_to_cpu(sqsh_ino->file_size); + if (inode->i_size < 0) { + err = -EINVAL; + goto failed_read; + } + frag = le32_to_cpu(sqsh_ino->fragment); + if (frag != SQUASHFS_INVALID_FRAG) { + /* + * the file cannot have a fragment (tailend) and have a + * file size a multiple of the block size + */ + if ((inode->i_size & (msblk->block_size - 1)) == 0) { + err = -EINVAL; + goto failed_read; + } + frag_offset = le32_to_cpu(sqsh_ino->offset); + frag_size = squashfs_frag_lookup(sb, frag, &frag_blk); + if (frag_size < 0) { + err = frag_size; + goto failed_read; + } + } else { + frag_blk = SQUASHFS_INVALID_BLK; + frag_size = 0; + frag_offset = 0; + } + + xattr_id = le32_to_cpu(sqsh_ino->xattr); + set_nlink(inode, le32_to_cpu(sqsh_ino->nlink)); + inode->i_op = &squashfs_inode_ops; + inode->i_fop = &squashfs_file_operations; + inode->i_mode |= S_IFREG; + inode->i_blocks = (inode->i_size - + le64_to_cpu(sqsh_ino->sparse) + 511) >> 9; + + squashfs_i(inode)->fragment_block = frag_blk; + squashfs_i(inode)->fragment_size = frag_size; + squashfs_i(inode)->fragment_offset = frag_offset; + squashfs_i(inode)->start = le64_to_cpu(sqsh_ino->start_block); + squashfs_i(inode)->block_list_start = block; + squashfs_i(inode)->offset = offset; + squashfs_i(inode)->parent = 0; + inode->i_data.a_ops = &squashfs_aops; + + TRACE("File inode %x:%x, start_block %llx, block_list_start " + "%llx, offset %x\n", SQUASHFS_INODE_BLK(ino), + offset, squashfs_i(inode)->start, block, offset); + break; + } + case SQUASHFS_DIR_TYPE: { + struct squashfs_dir_inode *sqsh_ino = &squashfs_ino.dir; + + err = squashfs_read_metadata(sb, sqsh_ino, &block, &offset, + sizeof(*sqsh_ino)); + if (err < 0) + goto failed_read; + + set_nlink(inode, le32_to_cpu(sqsh_ino->nlink)); + inode->i_size = le16_to_cpu(sqsh_ino->file_size); + inode->i_op = &squashfs_dir_inode_ops; + inode->i_fop = &squashfs_dir_ops; + inode->i_mode |= S_IFDIR; + squashfs_i(inode)->start = le32_to_cpu(sqsh_ino->start_block); + squashfs_i(inode)->offset = le16_to_cpu(sqsh_ino->offset); + squashfs_i(inode)->dir_idx_cnt = 0; + squashfs_i(inode)->parent = le32_to_cpu(sqsh_ino->parent_inode); + + TRACE("Directory inode %x:%x, start_block %llx, offset %x\n", + SQUASHFS_INODE_BLK(ino), offset, + squashfs_i(inode)->start, + le16_to_cpu(sqsh_ino->offset)); + break; + } + case SQUASHFS_LDIR_TYPE: { + struct squashfs_ldir_inode *sqsh_ino = &squashfs_ino.ldir; + + err = squashfs_read_metadata(sb, sqsh_ino, &block, &offset, + sizeof(*sqsh_ino)); + if (err < 0) + goto failed_read; + + xattr_id = le32_to_cpu(sqsh_ino->xattr); + set_nlink(inode, le32_to_cpu(sqsh_ino->nlink)); + inode->i_size = le32_to_cpu(sqsh_ino->file_size); + inode->i_op = &squashfs_dir_inode_ops; + inode->i_fop = &squashfs_dir_ops; + inode->i_mode |= S_IFDIR; + squashfs_i(inode)->start = le32_to_cpu(sqsh_ino->start_block); + squashfs_i(inode)->offset = le16_to_cpu(sqsh_ino->offset); + squashfs_i(inode)->dir_idx_start = block; + squashfs_i(inode)->dir_idx_offset = offset; + squashfs_i(inode)->dir_idx_cnt = le16_to_cpu(sqsh_ino->i_count); + squashfs_i(inode)->parent = le32_to_cpu(sqsh_ino->parent_inode); + + TRACE("Long directory inode %x:%x, start_block %llx, offset " + "%x\n", SQUASHFS_INODE_BLK(ino), offset, + squashfs_i(inode)->start, + le16_to_cpu(sqsh_ino->offset)); + break; + } + case SQUASHFS_SYMLINK_TYPE: + case SQUASHFS_LSYMLINK_TYPE: { + struct squashfs_symlink_inode *sqsh_ino = &squashfs_ino.symlink; + + err = squashfs_read_metadata(sb, sqsh_ino, &block, &offset, + sizeof(*sqsh_ino)); + if (err < 0) + goto failed_read; + + inode->i_size = le32_to_cpu(sqsh_ino->symlink_size); + if (inode->i_size > PAGE_SIZE) { + ERROR("Corrupted symlink\n"); + return -EINVAL; + } + + set_nlink(inode, le32_to_cpu(sqsh_ino->nlink)); + inode->i_op = &squashfs_symlink_inode_ops; + inode_nohighmem(inode); + inode->i_data.a_ops = &squashfs_symlink_aops; + inode->i_mode |= S_IFLNK; + squashfs_i(inode)->start = block; + squashfs_i(inode)->offset = offset; + squashfs_i(inode)->parent = 0; + + if (type == SQUASHFS_LSYMLINK_TYPE) { + __le32 xattr; + + err = squashfs_read_metadata(sb, NULL, &block, + &offset, inode->i_size); + if (err < 0) + goto failed_read; + err = squashfs_read_metadata(sb, &xattr, &block, + &offset, sizeof(xattr)); + if (err < 0) + goto failed_read; + xattr_id = le32_to_cpu(xattr); + } + + TRACE("Symbolic link inode %x:%x, start_block %llx, offset " + "%x\n", SQUASHFS_INODE_BLK(ino), offset, + block, offset); + break; + } + case SQUASHFS_BLKDEV_TYPE: + case SQUASHFS_CHRDEV_TYPE: { + struct squashfs_dev_inode *sqsh_ino = &squashfs_ino.dev; + unsigned int rdev; + + err = squashfs_read_metadata(sb, sqsh_ino, &block, &offset, + sizeof(*sqsh_ino)); + if (err < 0) + goto failed_read; + + if (type == SQUASHFS_CHRDEV_TYPE) + inode->i_mode |= S_IFCHR; + else + inode->i_mode |= S_IFBLK; + set_nlink(inode, le32_to_cpu(sqsh_ino->nlink)); + rdev = le32_to_cpu(sqsh_ino->rdev); + init_special_inode(inode, inode->i_mode, new_decode_dev(rdev)); + squashfs_i(inode)->parent = 0; + + TRACE("Device inode %x:%x, rdev %x\n", + SQUASHFS_INODE_BLK(ino), offset, rdev); + break; + } + case SQUASHFS_LBLKDEV_TYPE: + case SQUASHFS_LCHRDEV_TYPE: { + struct squashfs_ldev_inode *sqsh_ino = &squashfs_ino.ldev; + unsigned int rdev; + + err = squashfs_read_metadata(sb, sqsh_ino, &block, &offset, + sizeof(*sqsh_ino)); + if (err < 0) + goto failed_read; + + if (type == SQUASHFS_LCHRDEV_TYPE) + inode->i_mode |= S_IFCHR; + else + inode->i_mode |= S_IFBLK; + xattr_id = le32_to_cpu(sqsh_ino->xattr); + inode->i_op = &squashfs_inode_ops; + set_nlink(inode, le32_to_cpu(sqsh_ino->nlink)); + rdev = le32_to_cpu(sqsh_ino->rdev); + init_special_inode(inode, inode->i_mode, new_decode_dev(rdev)); + squashfs_i(inode)->parent = 0; + + TRACE("Device inode %x:%x, rdev %x\n", + SQUASHFS_INODE_BLK(ino), offset, rdev); + break; + } + case SQUASHFS_FIFO_TYPE: + case SQUASHFS_SOCKET_TYPE: { + struct squashfs_ipc_inode *sqsh_ino = &squashfs_ino.ipc; + + err = squashfs_read_metadata(sb, sqsh_ino, &block, &offset, + sizeof(*sqsh_ino)); + if (err < 0) + goto failed_read; + + if (type == SQUASHFS_FIFO_TYPE) + inode->i_mode |= S_IFIFO; + else + inode->i_mode |= S_IFSOCK; + set_nlink(inode, le32_to_cpu(sqsh_ino->nlink)); + init_special_inode(inode, inode->i_mode, 0); + squashfs_i(inode)->parent = 0; + break; + } + case SQUASHFS_LFIFO_TYPE: + case SQUASHFS_LSOCKET_TYPE: { + struct squashfs_lipc_inode *sqsh_ino = &squashfs_ino.lipc; + + err = squashfs_read_metadata(sb, sqsh_ino, &block, &offset, + sizeof(*sqsh_ino)); + if (err < 0) + goto failed_read; + + if (type == SQUASHFS_LFIFO_TYPE) + inode->i_mode |= S_IFIFO; + else + inode->i_mode |= S_IFSOCK; + xattr_id = le32_to_cpu(sqsh_ino->xattr); + inode->i_op = &squashfs_inode_ops; + set_nlink(inode, le32_to_cpu(sqsh_ino->nlink)); + init_special_inode(inode, inode->i_mode, 0); + squashfs_i(inode)->parent = 0; + break; + } + default: + ERROR("Unknown inode type %d in squashfs_iget!\n", type); + return -EINVAL; + } + + if (xattr_id != SQUASHFS_INVALID_XATTR && msblk->xattr_id_table) { + err = squashfs_xattr_lookup(sb, xattr_id, + &squashfs_i(inode)->xattr_count, + &squashfs_i(inode)->xattr_size, + &squashfs_i(inode)->xattr); + if (err < 0) + goto failed_read; + inode->i_blocks += ((squashfs_i(inode)->xattr_size - 1) >> 9) + + 1; + } else + squashfs_i(inode)->xattr_count = 0; + + return 0; + +failed_read: + ERROR("Unable to read inode 0x%llx\n", ino); + return err; +} + + +const struct inode_operations squashfs_inode_ops = { + .listxattr = squashfs_listxattr +}; + diff --git a/fs/squashfs/lz4_wrapper.c b/fs/squashfs/lz4_wrapper.c new file mode 100644 index 000000000..1646bbee2 --- /dev/null +++ b/fs/squashfs/lz4_wrapper.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2013, 2014 + * Phillip Lougher + */ + +#include +#include +#include +#include +#include + +#include "squashfs_fs.h" +#include "squashfs_fs_sb.h" +#include "squashfs.h" +#include "decompressor.h" +#include "page_actor.h" + +#define LZ4_LEGACY 1 + +struct lz4_comp_opts { + __le32 version; + __le32 flags; +}; + +struct squashfs_lz4 { + void *input; + void *output; +}; + + +static void *lz4_comp_opts(struct squashfs_sb_info *msblk, + void *buff, int len) +{ + struct lz4_comp_opts *comp_opts = buff; + + /* LZ4 compressed filesystems always have compression options */ + if (comp_opts == NULL || len < sizeof(*comp_opts)) + return ERR_PTR(-EIO); + + if (le32_to_cpu(comp_opts->version) != LZ4_LEGACY) { + /* LZ4 format currently used by the kernel is the 'legacy' + * format */ + ERROR("Unknown LZ4 version\n"); + return ERR_PTR(-EINVAL); + } + + return NULL; +} + + +static void *lz4_init(struct squashfs_sb_info *msblk, void *buff) +{ + int block_size = max_t(int, msblk->block_size, SQUASHFS_METADATA_SIZE); + struct squashfs_lz4 *stream; + + stream = kzalloc_obj(*stream); + if (stream == NULL) + goto failed; + stream->input = vmalloc(block_size); + if (stream->input == NULL) + goto failed2; + stream->output = vmalloc(block_size); + if (stream->output == NULL) + goto failed3; + + return stream; + +failed3: + vfree(stream->input); +failed2: + kfree(stream); +failed: + ERROR("Failed to initialise LZ4 decompressor\n"); + return ERR_PTR(-ENOMEM); +} + + +static void lz4_free(void *strm) +{ + struct squashfs_lz4 *stream = strm; + + if (stream) { + vfree(stream->input); + vfree(stream->output); + } + kfree(stream); +} + + +static int lz4_uncompress(struct squashfs_sb_info *msblk, void *strm, + struct bio *bio, int offset, int length, + struct squashfs_page_actor *output) +{ + struct bvec_iter_all iter_all = {}; + struct bio_vec *bvec = bvec_init_iter_all(&iter_all); + struct squashfs_lz4 *stream = strm; + void *buff = stream->input, *data; + int bytes = length, res; + + while (bio_next_segment(bio, &iter_all)) { + int avail = min(bytes, ((int)bvec->bv_len) - offset); + + data = bvec_virt(bvec); + memcpy(buff, data + offset, avail); + buff += avail; + bytes -= avail; + offset = 0; + } + + res = LZ4_decompress_safe(stream->input, stream->output, + length, output->length); + + if (res < 0) + return -EIO; + + bytes = res; + data = squashfs_first_page(output); + buff = stream->output; + while (data) { + if (bytes <= PAGE_SIZE) { + if (!IS_ERR(data)) + memcpy(data, buff, bytes); + break; + } + if (!IS_ERR(data)) + memcpy(data, buff, PAGE_SIZE); + buff += PAGE_SIZE; + bytes -= PAGE_SIZE; + data = squashfs_next_page(output); + } + squashfs_finish_page(output); + + return res; +} + +const struct squashfs_decompressor squashfs_lz4_comp_ops = { + .init = lz4_init, + .comp_opts = lz4_comp_opts, + .free = lz4_free, + .decompress = lz4_uncompress, + .id = LZ4_COMPRESSION, + .name = "lz4", + .alloc_buffer = 0, + .supported = 1 +}; diff --git a/fs/squashfs/lzo_wrapper.c b/fs/squashfs/lzo_wrapper.c new file mode 100644 index 000000000..be1df7a25 --- /dev/null +++ b/fs/squashfs/lzo_wrapper.c @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Squashfs - a compressed read only filesystem for Linux + * + * Copyright (c) 2010 LG Electronics + * Chan Jeong + * + * lzo_wrapper.c + */ + +#include +#include +#include +#include +#include + +#include "squashfs_fs.h" +#include "squashfs_fs_sb.h" +#include "squashfs.h" +#include "decompressor.h" +#include "page_actor.h" + +struct squashfs_lzo { + void *input; + void *output; +}; + +static void *lzo_init(struct squashfs_sb_info *msblk, void *buff) +{ + int block_size = max_t(int, msblk->block_size, SQUASHFS_METADATA_SIZE); + + struct squashfs_lzo *stream = kzalloc_obj(*stream); + if (stream == NULL) + goto failed; + stream->input = vmalloc(block_size); + if (stream->input == NULL) + goto failed; + stream->output = vmalloc(block_size); + if (stream->output == NULL) + goto failed2; + + return stream; + +failed2: + vfree(stream->input); +failed: + ERROR("Failed to allocate lzo workspace\n"); + kfree(stream); + return ERR_PTR(-ENOMEM); +} + + +static void lzo_free(void *strm) +{ + struct squashfs_lzo *stream = strm; + + if (stream) { + vfree(stream->input); + vfree(stream->output); + } + kfree(stream); +} + + +static int lzo_uncompress(struct squashfs_sb_info *msblk, void *strm, + struct bio *bio, int offset, int length, + struct squashfs_page_actor *output) +{ + struct bvec_iter_all iter_all = {}; + struct bio_vec *bvec = bvec_init_iter_all(&iter_all); + struct squashfs_lzo *stream = strm; + void *buff = stream->input, *data; + int bytes = length, res; + size_t out_len = output->length; + + while (bio_next_segment(bio, &iter_all)) { + int avail = min(bytes, ((int)bvec->bv_len) - offset); + + data = bvec_virt(bvec); + memcpy(buff, data + offset, avail); + buff += avail; + bytes -= avail; + offset = 0; + } + + res = lzo1x_decompress_safe(stream->input, (size_t)length, + stream->output, &out_len); + if (res != LZO_E_OK) + goto failed; + + res = bytes = (int)out_len; + data = squashfs_first_page(output); + buff = stream->output; + while (data) { + if (bytes <= PAGE_SIZE) { + if (!IS_ERR(data)) + memcpy(data, buff, bytes); + break; + } else { + if (!IS_ERR(data)) + memcpy(data, buff, PAGE_SIZE); + buff += PAGE_SIZE; + bytes -= PAGE_SIZE; + data = squashfs_next_page(output); + } + } + squashfs_finish_page(output); + + return res; + +failed: + return -EIO; +} + +const struct squashfs_decompressor squashfs_lzo_comp_ops = { + .init = lzo_init, + .free = lzo_free, + .decompress = lzo_uncompress, + .id = LZO_COMPRESSION, + .name = "lzo", + .alloc_buffer = 0, + .supported = 1 +}; diff --git a/fs/squashfs/namei.c b/fs/squashfs/namei.c new file mode 100644 index 000000000..65aae7e2a --- /dev/null +++ b/fs/squashfs/namei.c @@ -0,0 +1,232 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Squashfs - a compressed read only filesystem for Linux + * + * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008 + * Phillip Lougher + * + * namei.c + */ + +/* + * This file implements code to do filename lookup in directories. + * + * Like inodes, directories are packed into compressed metadata blocks, stored + * in a directory table. Directories are accessed using the start address of + * the metablock containing the directory and the offset into the + * decompressed block (). + * + * Directories are organised in a slightly complex way, and are not simply + * a list of file names. The organisation takes advantage of the + * fact that (in most cases) the inodes of the files will be in the same + * compressed metadata block, and therefore, can share the start block. + * Directories are therefore organised in a two level list, a directory + * header containing the shared start block value, and a sequence of directory + * entries, each of which share the shared start block. A new directory header + * is written once/if the inode start block changes. The directory + * header/directory entry list is repeated as many times as necessary. + * + * Directories are sorted, and can contain a directory index to speed up + * file lookup. Directory indexes store one entry per metablock, each entry + * storing the index/filename mapping to the first directory header + * in each metadata block. Directories are sorted in alphabetical order, + * and at lookup the index is scanned linearly looking for the first filename + * alphabetically larger than the filename being looked up. At this point the + * location of the metadata block the filename is in has been found. + * The general idea of the index is ensure only one metadata block needs to be + * decompressed to do a lookup irrespective of the length of the directory. + * This scheme has the advantage that it doesn't require extra memory overhead + * and doesn't require much extra storage on disk. + */ + +#include +#include +#include +#include +#include +#include + +#include "squashfs_fs.h" +#include "squashfs_fs_sb.h" +#include "squashfs_fs_i.h" +#include "squashfs.h" +#include "xattr.h" + +/* + * Lookup name in the directory index, returning the location of the metadata + * block containing it, and the directory index this represents. + * + * If we get an error reading the index then return the part of the index + * (if any) we have managed to read - the index isn't essential, just + * quicker. + */ +static int get_dir_index_using_name(struct super_block *sb, + u64 *next_block, int *next_offset, u64 index_start, + int index_offset, int i_count, const char *name) +{ + struct squashfs_sb_info *msblk = sb->s_fs_info; + int i, length = 0, err; + unsigned int size; + struct squashfs_dir_index *index; + + TRACE("Entered get_dir_index_using_name, i_count %d\n", i_count); + + index = kmalloc(sizeof(*index) + SQUASHFS_NAME_LEN + 1, GFP_KERNEL); + if (index == NULL) { + ERROR("Failed to allocate squashfs_dir_index\n"); + goto out; + } + + for (i = 0; i < i_count; i++) { + err = squashfs_read_metadata(sb, index, &index_start, + &index_offset, sizeof(*index)); + if (err < 0) + break; + + + size = le32_to_cpu(index->size) + 1; + if (size > SQUASHFS_NAME_LEN) + break; + + err = squashfs_read_metadata(sb, index->name, &index_start, + &index_offset, size); + if (err < 0) + break; + + index->name[size] = '\0'; + + if (strcmp(index->name, name) > 0) + break; + + length = le32_to_cpu(index->index); + *next_block = le32_to_cpu(index->start_block) + + msblk->directory_table; + } + + *next_offset = (length + *next_offset) % SQUASHFS_METADATA_SIZE; + kfree(index); + +out: + /* + * Return index (f_pos) of the looked up metadata block. Translate + * from internal f_pos to external f_pos which is offset by 3 because + * we invent "." and ".." entries which are not actually stored in the + * directory. + */ + return length + 3; +} + + +static struct dentry *squashfs_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + const unsigned char *name = dentry->d_name.name; + int len = dentry->d_name.len; + struct inode *inode = NULL; + struct squashfs_sb_info *msblk = dir->i_sb->s_fs_info; + struct squashfs_dir_header dirh; + struct squashfs_dir_entry *dire; + u64 block = squashfs_i(dir)->start + msblk->directory_table; + int offset = squashfs_i(dir)->offset; + int err, length; + unsigned int dir_count, size; + + TRACE("Entered squashfs_lookup [%llx:%x]\n", block, offset); + + dire = kmalloc(sizeof(*dire) + SQUASHFS_NAME_LEN + 1, GFP_KERNEL); + if (dire == NULL) { + ERROR("Failed to allocate squashfs_dir_entry\n"); + return ERR_PTR(-ENOMEM); + } + + if (len > SQUASHFS_NAME_LEN) { + err = -ENAMETOOLONG; + goto failed; + } + + length = get_dir_index_using_name(dir->i_sb, &block, &offset, + squashfs_i(dir)->dir_idx_start, + squashfs_i(dir)->dir_idx_offset, + squashfs_i(dir)->dir_idx_cnt, name); + + while (length < i_size_read(dir)) { + /* + * Read directory header. + */ + err = squashfs_read_metadata(dir->i_sb, &dirh, &block, + &offset, sizeof(dirh)); + if (err < 0) + goto read_failure; + + length += sizeof(dirh); + + dir_count = le32_to_cpu(dirh.count) + 1; + + if (dir_count > SQUASHFS_DIR_COUNT) + goto data_error; + + while (dir_count--) { + /* + * Read directory entry. + */ + err = squashfs_read_metadata(dir->i_sb, dire, &block, + &offset, sizeof(*dire)); + if (err < 0) + goto read_failure; + + size = le16_to_cpu(dire->size) + 1; + + /* size should never be larger than SQUASHFS_NAME_LEN */ + if (size > SQUASHFS_NAME_LEN) + goto data_error; + + err = squashfs_read_metadata(dir->i_sb, dire->name, + &block, &offset, size); + if (err < 0) + goto read_failure; + + length += sizeof(*dire) + size; + + if (name[0] < dire->name[0]) + goto exit_lookup; + + if (len == size && !strncmp(name, dire->name, len)) { + unsigned int blk, off, ino_num; + long long ino; + blk = le32_to_cpu(dirh.start_block); + off = le16_to_cpu(dire->offset); + ino_num = le32_to_cpu(dirh.inode_number) + + (short) le16_to_cpu(dire->inode_number); + ino = SQUASHFS_MKINODE(blk, off); + + TRACE("calling squashfs_iget for directory " + "entry %s, inode %x:%x, %d\n", name, + blk, off, ino_num); + + inode = squashfs_iget(dir->i_sb, ino, ino_num); + goto exit_lookup; + } + } + } + +exit_lookup: + kfree(dire); + return d_splice_alias(inode, dentry); + +data_error: + err = -EIO; + +read_failure: + ERROR("Unable to read directory block [%llx:%x]\n", + squashfs_i(dir)->start + msblk->directory_table, + squashfs_i(dir)->offset); +failed: + kfree(dire); + return ERR_PTR(err); +} + + +const struct inode_operations squashfs_dir_inode_ops = { + .lookup = squashfs_lookup, + .listxattr = squashfs_listxattr +}; diff --git a/fs/squashfs/page_actor.c b/fs/squashfs/page_actor.c new file mode 100644 index 000000000..79b73e98a --- /dev/null +++ b/fs/squashfs/page_actor.c @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2013 + * Phillip Lougher + */ + +#include +#include +#include +#include "squashfs_fs_sb.h" +#include "decompressor.h" +#include "page_actor.h" + +/* + * This file contains implementations of page_actor for decompressing into + * an intermediate buffer, and for decompressing directly into the + * page cache. + * + * Calling code should avoid sleeping between calls to squashfs_first_page() + * and squashfs_finish_page(). + */ + +/* Implementation of page_actor for decompressing into intermediate buffer */ +static void *cache_first_page(struct squashfs_page_actor *actor) +{ + actor->next_page = 1; + return actor->buffer[0]; +} + +static void *cache_next_page(struct squashfs_page_actor *actor) +{ + if (actor->next_page == actor->pages) + return NULL; + + return actor->buffer[actor->next_page++]; +} + +static void cache_finish_page(struct squashfs_page_actor *actor) +{ + /* empty */ +} + +struct squashfs_page_actor *squashfs_page_actor_init(void **buffer, + int pages, int length) +{ + struct squashfs_page_actor *actor = kmalloc_obj(*actor); + + if (actor == NULL) + return NULL; + + actor->length = length ? : pages * PAGE_SIZE; + actor->buffer = buffer; + actor->pages = pages; + actor->next_page = 0; + actor->tmp_buffer = NULL; + actor->squashfs_first_page = cache_first_page; + actor->squashfs_next_page = cache_next_page; + actor->squashfs_finish_page = cache_finish_page; + return actor; +} + +/* Implementation of page_actor for decompressing directly into page cache. */ +static loff_t page_next_index(struct squashfs_page_actor *actor) +{ + return page_folio(actor->page[actor->next_page])->index; +} + +static void *handle_next_page(struct squashfs_page_actor *actor) +{ + int max_pages = (actor->length + PAGE_SIZE - 1) >> PAGE_SHIFT; + + if (actor->returned_pages == max_pages) + return NULL; + + if ((actor->next_page == actor->pages) || + (actor->next_index != page_next_index(actor))) { + actor->next_index++; + actor->returned_pages++; + actor->last_page = NULL; + return actor->alloc_buffer ? actor->tmp_buffer : ERR_PTR(-ENOMEM); + } + + actor->next_index++; + actor->returned_pages++; + actor->last_page = actor->page[actor->next_page]; + return actor->pageaddr = kmap_local_page(actor->page[actor->next_page++]); +} + +static void *direct_first_page(struct squashfs_page_actor *actor) +{ + return handle_next_page(actor); +} + +static void *direct_next_page(struct squashfs_page_actor *actor) +{ + if (actor->pageaddr) { + kunmap_local(actor->pageaddr); + actor->pageaddr = NULL; + } + + return handle_next_page(actor); +} + +static void direct_finish_page(struct squashfs_page_actor *actor) +{ + if (actor->pageaddr) + kunmap_local(actor->pageaddr); +} + +struct squashfs_page_actor *squashfs_page_actor_init_special(struct squashfs_sb_info *msblk, + struct page **page, int pages, int length, loff_t start_index) +{ + struct squashfs_page_actor *actor = kmalloc_obj(*actor); + + if (actor == NULL) + return NULL; + + if (msblk->decompressor->alloc_buffer) { + actor->tmp_buffer = kmalloc(PAGE_SIZE, GFP_KERNEL); + + if (actor->tmp_buffer == NULL) { + kfree(actor); + return NULL; + } + } else + actor->tmp_buffer = NULL; + + actor->length = length ? : pages * PAGE_SIZE; + actor->page = page; + actor->pages = pages; + actor->next_page = 0; + actor->returned_pages = 0; + actor->next_index = start_index >> PAGE_SHIFT; + actor->pageaddr = NULL; + actor->last_page = NULL; + actor->alloc_buffer = msblk->decompressor->alloc_buffer; + actor->squashfs_first_page = direct_first_page; + actor->squashfs_next_page = direct_next_page; + actor->squashfs_finish_page = direct_finish_page; + return actor; +} diff --git a/fs/squashfs/page_actor.h b/fs/squashfs/page_actor.h new file mode 100644 index 000000000..ffe25eb77 --- /dev/null +++ b/fs/squashfs/page_actor.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef PAGE_ACTOR_H +#define PAGE_ACTOR_H +/* + * Copyright (c) 2013 + * Phillip Lougher + */ + +struct squashfs_page_actor { + union { + void **buffer; + struct page **page; + }; + void *pageaddr; + void *tmp_buffer; + void *(*squashfs_first_page)(struct squashfs_page_actor *); + void *(*squashfs_next_page)(struct squashfs_page_actor *); + void (*squashfs_finish_page)(struct squashfs_page_actor *); + struct page *last_page; + int pages; + int length; + int next_page; + int alloc_buffer; + int returned_pages; + pgoff_t next_index; +}; + +extern struct squashfs_page_actor *squashfs_page_actor_init(void **buffer, + int pages, int length); +extern struct squashfs_page_actor *squashfs_page_actor_init_special( + struct squashfs_sb_info *msblk, + struct page **page, int pages, int length, + loff_t start_index); +static inline struct page *squashfs_page_actor_free(struct squashfs_page_actor *actor) +{ + struct page *last_page = actor->next_page == actor->pages ? actor->last_page : ERR_PTR(-EIO); + + kfree(actor->tmp_buffer); + kfree(actor); + + return last_page; +} +static inline void *squashfs_first_page(struct squashfs_page_actor *actor) +{ + return actor->squashfs_first_page(actor); +} +static inline void *squashfs_next_page(struct squashfs_page_actor *actor) +{ + return actor->squashfs_next_page(actor); +} +static inline void squashfs_finish_page(struct squashfs_page_actor *actor) +{ + actor->squashfs_finish_page(actor); +} +static inline void squashfs_actor_nobuff(struct squashfs_page_actor *actor) +{ + actor->alloc_buffer = 0; +} +#endif diff --git a/fs/squashfs/squashfs.h b/fs/squashfs/squashfs.h new file mode 100644 index 000000000..4851bd964 --- /dev/null +++ b/fs/squashfs/squashfs.h @@ -0,0 +1,120 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Squashfs - a compressed read only filesystem for Linux + * + * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008 + * Phillip Lougher + * + * squashfs.h + */ + +#define TRACE(s, args...) pr_debug("SQUASHFS: "s, ## args) + +#define ERROR(s, args...) pr_err("SQUASHFS error: "s, ## args) + +#define WARNING(s, args...) pr_warn("SQUASHFS: "s, ## args) + +#ifdef CONFIG_SQUASHFS_FILE_CACHE +#define SQUASHFS_READ_PAGES msblk->max_thread_num +#else +#define SQUASHFS_READ_PAGES 0 +#endif + +/* block.c */ +extern int squashfs_read_data(struct super_block *, u64, int, u64 *, + struct squashfs_page_actor *); + +/* cache.c */ +extern struct squashfs_cache *squashfs_cache_init(char *, int, int); +extern void squashfs_cache_delete(struct squashfs_cache *); +extern struct squashfs_cache_entry *squashfs_cache_get(struct super_block *, + struct squashfs_cache *, u64, int); +extern void squashfs_cache_put(struct squashfs_cache_entry *); +extern int squashfs_copy_data(void *, struct squashfs_cache_entry *, int, int); +extern int squashfs_read_metadata(struct super_block *, void *, u64 *, + int *, int); +extern struct squashfs_cache_entry *squashfs_get_fragment(struct super_block *, + u64, int); +extern struct squashfs_cache_entry *squashfs_get_datablock(struct super_block *, + u64, int); +extern void *squashfs_read_table(struct super_block *, u64, int); + +/* decompressor.c */ +extern const struct squashfs_decompressor *squashfs_lookup_decompressor(int); +extern void *squashfs_decompressor_setup(struct super_block *, unsigned short); + +/* decompressor_xxx.c */ + +struct squashfs_decompressor_thread_ops { + void * (*create)(struct squashfs_sb_info *msblk, void *comp_opts); + void (*destroy)(struct squashfs_sb_info *msblk); + int (*decompress)(struct squashfs_sb_info *msblk, struct bio *bio, + int offset, int length, struct squashfs_page_actor *output); + int (*max_decompressors)(void); +}; + +#ifdef CONFIG_SQUASHFS_DECOMP_SINGLE +extern const struct squashfs_decompressor_thread_ops squashfs_decompressor_single; +#endif +#ifdef CONFIG_SQUASHFS_DECOMP_MULTI +extern const struct squashfs_decompressor_thread_ops squashfs_decompressor_multi; +#endif +#ifdef CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU +extern const struct squashfs_decompressor_thread_ops squashfs_decompressor_percpu; +#endif + +/* export.c */ +extern __le64 *squashfs_read_inode_lookup_table(struct super_block *, u64, u64, + unsigned int); + +/* fragment.c */ +extern int squashfs_frag_lookup(struct super_block *, unsigned int, u64 *); +extern __le64 *squashfs_read_fragment_index_table(struct super_block *, + u64, u64, unsigned int); + +/* file.c */ +void squashfs_copy_cache(struct folio *, struct squashfs_cache_entry *, + size_t bytes, size_t offset); + +/* file_xxx.c */ +int squashfs_readpage_block(struct folio *, u64 block, int bsize, int expected); + +/* id.c */ +extern int squashfs_get_id(struct super_block *, unsigned int, unsigned int *); +extern __le64 *squashfs_read_id_index_table(struct super_block *, u64, u64, + unsigned short); + +/* inode.c */ +extern struct inode *squashfs_iget(struct super_block *, long long, + unsigned int); +extern int squashfs_read_inode(struct inode *, long long); + +/* xattr.c */ +extern ssize_t squashfs_listxattr(struct dentry *, char *, size_t); + +/* + * Inodes, files, decompressor and xattr operations + */ + +/* dir.c */ +extern const struct file_operations squashfs_dir_ops; + +/* export.c */ +extern const struct export_operations squashfs_export_ops; + +/* file.c */ +extern const struct address_space_operations squashfs_aops; + +/* inode.c */ +extern const struct inode_operations squashfs_inode_ops; +extern const struct file_operations squashfs_file_operations; + +/* namei.c */ +extern const struct inode_operations squashfs_dir_inode_ops; + +/* symlink.c */ +extern const struct address_space_operations squashfs_symlink_aops; +extern const struct inode_operations squashfs_symlink_inode_ops; + +/* xattr.c */ +extern const struct xattr_handler * const squashfs_xattr_handlers[]; diff --git a/fs/squashfs/squashfs_fs.h b/fs/squashfs/squashfs_fs.h new file mode 100644 index 000000000..a955d9369 --- /dev/null +++ b/fs/squashfs/squashfs_fs.h @@ -0,0 +1,453 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef SQUASHFS_FS +#define SQUASHFS_FS +/* + * Squashfs + * + * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008 + * Phillip Lougher + * + * squashfs_fs.h + */ + +#define SQUASHFS_CACHED_FRAGMENTS CONFIG_SQUASHFS_FRAGMENT_CACHE_SIZE +#define SQUASHFS_MAJOR 4 +#define SQUASHFS_MINOR 0 +#define SQUASHFS_START 0 + +/* size of metadata (inode and directory) blocks */ +#define SQUASHFS_METADATA_SIZE 8192 +#define SQUASHFS_BLOCK_OFFSET 2 + +/* default size of block device I/O */ +#ifdef CONFIG_SQUASHFS_4K_DEVBLK_SIZE +#define SQUASHFS_DEVBLK_SIZE 4096 +#else +#define SQUASHFS_DEVBLK_SIZE 1024 +#endif + +#define SQUASHFS_FILE_MAX_SIZE 1048576 +#define SQUASHFS_FILE_MAX_LOG 20 + +/* Max length of filename (not 255) */ +#define SQUASHFS_NAME_LEN 256 + +/* Max value for directory header count*/ +#define SQUASHFS_DIR_COUNT 256 + +#define SQUASHFS_INVALID_FRAG (0xffffffffU) +#define SQUASHFS_INVALID_XATTR (0xffffffffU) +#define SQUASHFS_INVALID_BLK (-1LL) + +/* Filesystem flags */ +#define SQUASHFS_NOI 0 +#define SQUASHFS_NOD 1 +#define SQUASHFS_NOF 3 +#define SQUASHFS_NO_FRAG 4 +#define SQUASHFS_ALWAYS_FRAG 5 +#define SQUASHFS_DUPLICATE 6 +#define SQUASHFS_EXPORT 7 +#define SQUASHFS_COMP_OPT 10 + +#define SQUASHFS_BIT(flag, bit) ((flag >> bit) & 1) + +#define SQUASHFS_UNCOMPRESSED_INODES(flags) SQUASHFS_BIT(flags, \ + SQUASHFS_NOI) + +#define SQUASHFS_UNCOMPRESSED_DATA(flags) SQUASHFS_BIT(flags, \ + SQUASHFS_NOD) + +#define SQUASHFS_UNCOMPRESSED_FRAGMENTS(flags) SQUASHFS_BIT(flags, \ + SQUASHFS_NOF) + +#define SQUASHFS_NO_FRAGMENTS(flags) SQUASHFS_BIT(flags, \ + SQUASHFS_NO_FRAG) + +#define SQUASHFS_ALWAYS_FRAGMENTS(flags) SQUASHFS_BIT(flags, \ + SQUASHFS_ALWAYS_FRAG) + +#define SQUASHFS_DUPLICATES(flags) SQUASHFS_BIT(flags, \ + SQUASHFS_DUPLICATE) + +#define SQUASHFS_EXPORTABLE(flags) SQUASHFS_BIT(flags, \ + SQUASHFS_EXPORT) + +#define SQUASHFS_COMP_OPTS(flags) SQUASHFS_BIT(flags, \ + SQUASHFS_COMP_OPT) + +/* Inode types including extended types */ +#define SQUASHFS_DIR_TYPE 1 +#define SQUASHFS_REG_TYPE 2 +#define SQUASHFS_SYMLINK_TYPE 3 +#define SQUASHFS_BLKDEV_TYPE 4 +#define SQUASHFS_CHRDEV_TYPE 5 +#define SQUASHFS_FIFO_TYPE 6 +#define SQUASHFS_SOCKET_TYPE 7 +#define SQUASHFS_LDIR_TYPE 8 +#define SQUASHFS_LREG_TYPE 9 +#define SQUASHFS_LSYMLINK_TYPE 10 +#define SQUASHFS_LBLKDEV_TYPE 11 +#define SQUASHFS_LCHRDEV_TYPE 12 +#define SQUASHFS_LFIFO_TYPE 13 +#define SQUASHFS_LSOCKET_TYPE 14 + +/* Max type value stored in directory entry */ +#define SQUASHFS_MAX_DIR_TYPE 7 + +/* Xattr types */ +#define SQUASHFS_XATTR_USER 0 +#define SQUASHFS_XATTR_TRUSTED 1 +#define SQUASHFS_XATTR_SECURITY 2 +#define SQUASHFS_XATTR_VALUE_OOL 256 +#define SQUASHFS_XATTR_PREFIX_MASK 0xff + +/* Flag whether block is compressed or uncompressed, bit is set if block is + * uncompressed */ +#define SQUASHFS_COMPRESSED_BIT (1 << 15) + +#define SQUASHFS_COMPRESSED_SIZE(B) (((B) & ~SQUASHFS_COMPRESSED_BIT) ? \ + (B) & ~SQUASHFS_COMPRESSED_BIT : SQUASHFS_COMPRESSED_BIT) + +#define SQUASHFS_COMPRESSED(B) (!((B) & SQUASHFS_COMPRESSED_BIT)) + +#define SQUASHFS_COMPRESSED_BIT_BLOCK (1 << 24) + +#define SQUASHFS_COMPRESSED_SIZE_BLOCK(B) ((B) & \ + ~SQUASHFS_COMPRESSED_BIT_BLOCK) + +#define SQUASHFS_COMPRESSED_BLOCK(B) (!((B) & SQUASHFS_COMPRESSED_BIT_BLOCK)) + +static inline int squashfs_block_size(__le32 raw) +{ + u32 size = le32_to_cpu(raw); + return (size >> 25) ? -EIO : size; +} + +/* + * Inode number ops. Inodes consist of a compressed block number, and an + * uncompressed offset within that block + */ +#define SQUASHFS_INODE_BLK(A) ((unsigned int) ((A) >> 16)) + +#define SQUASHFS_INODE_OFFSET(A) ((unsigned int) ((A) & 0xffff)) + +#define SQUASHFS_MKINODE(A, B) ((long long)(((long long) (A)\ + << 16) + (B))) + +/* fragment and fragment table defines */ +#define SQUASHFS_FRAGMENT_BYTES(A) \ + ((A) * sizeof(struct squashfs_fragment_entry)) + +#define SQUASHFS_FRAGMENT_INDEX(A) (SQUASHFS_FRAGMENT_BYTES(A) / \ + SQUASHFS_METADATA_SIZE) + +#define SQUASHFS_FRAGMENT_INDEX_OFFSET(A) (SQUASHFS_FRAGMENT_BYTES(A) % \ + SQUASHFS_METADATA_SIZE) + +#define SQUASHFS_FRAGMENT_INDEXES(A) ((SQUASHFS_FRAGMENT_BYTES(A) + \ + SQUASHFS_METADATA_SIZE - 1) / \ + SQUASHFS_METADATA_SIZE) + +#define SQUASHFS_FRAGMENT_INDEX_BYTES(A) (SQUASHFS_FRAGMENT_INDEXES(A) *\ + sizeof(u64)) + +/* inode lookup table defines */ +#define SQUASHFS_LOOKUP_BYTES(A) ((A) * sizeof(u64)) + +#define SQUASHFS_LOOKUP_BLOCK(A) (SQUASHFS_LOOKUP_BYTES(A) / \ + SQUASHFS_METADATA_SIZE) + +#define SQUASHFS_LOOKUP_BLOCK_OFFSET(A) (SQUASHFS_LOOKUP_BYTES(A) % \ + SQUASHFS_METADATA_SIZE) + +#define SQUASHFS_LOOKUP_BLOCKS(A) ((SQUASHFS_LOOKUP_BYTES(A) + \ + SQUASHFS_METADATA_SIZE - 1) / \ + SQUASHFS_METADATA_SIZE) + +#define SQUASHFS_LOOKUP_BLOCK_BYTES(A) (SQUASHFS_LOOKUP_BLOCKS(A) *\ + sizeof(u64)) + +/* uid/gid lookup table defines */ +#define SQUASHFS_ID_BYTES(A) ((A) * sizeof(unsigned int)) + +#define SQUASHFS_ID_BLOCK(A) (SQUASHFS_ID_BYTES(A) / \ + SQUASHFS_METADATA_SIZE) + +#define SQUASHFS_ID_BLOCK_OFFSET(A) (SQUASHFS_ID_BYTES(A) % \ + SQUASHFS_METADATA_SIZE) + +#define SQUASHFS_ID_BLOCKS(A) ((SQUASHFS_ID_BYTES(A) + \ + SQUASHFS_METADATA_SIZE - 1) / \ + SQUASHFS_METADATA_SIZE) + +#define SQUASHFS_ID_BLOCK_BYTES(A) (SQUASHFS_ID_BLOCKS(A) *\ + sizeof(u64)) +/* xattr id lookup table defines */ +#define SQUASHFS_XATTR_BYTES(A) (((u64) (A)) * sizeof(struct squashfs_xattr_id)) + +#define SQUASHFS_XATTR_BLOCK(A) (SQUASHFS_XATTR_BYTES(A) / \ + SQUASHFS_METADATA_SIZE) + +#define SQUASHFS_XATTR_BLOCK_OFFSET(A) (SQUASHFS_XATTR_BYTES(A) % \ + SQUASHFS_METADATA_SIZE) + +#define SQUASHFS_XATTR_BLOCKS(A) ((SQUASHFS_XATTR_BYTES(A) + \ + SQUASHFS_METADATA_SIZE - 1) / \ + SQUASHFS_METADATA_SIZE) + +#define SQUASHFS_XATTR_BLOCK_BYTES(A) (SQUASHFS_XATTR_BLOCKS(A) *\ + sizeof(u64)) +#define SQUASHFS_XATTR_BLK(A) ((unsigned int) ((A) >> 16)) + +#define SQUASHFS_XATTR_OFFSET(A) ((unsigned int) ((A) & 0xffff)) + +/* cached data constants for filesystem */ +#define SQUASHFS_CACHED_BLKS 8 + +/* meta index cache */ +#define SQUASHFS_META_INDEXES (SQUASHFS_METADATA_SIZE / sizeof(unsigned int)) +#define SQUASHFS_META_ENTRIES 127 +#define SQUASHFS_META_SLOTS 8 +#define SQUASHFS_SCAN_INDEXES 1024 + +struct meta_entry { + u64 data_block; + unsigned int index_block; + unsigned short offset; + unsigned short pad; +}; + +struct meta_index { + unsigned int inode_number; + unsigned int offset; + unsigned short entries; + unsigned short skip; + unsigned short locked; + unsigned short pad; + struct meta_entry meta_entry[SQUASHFS_META_ENTRIES]; +}; + + +/* + * definitions for structures on disk + */ +#define ZLIB_COMPRESSION 1 +#define LZMA_COMPRESSION 2 +#define LZO_COMPRESSION 3 +#define XZ_COMPRESSION 4 +#define LZ4_COMPRESSION 5 +#define ZSTD_COMPRESSION 6 + +struct squashfs_super_block { + __le32 s_magic; + __le32 inodes; + __le32 mkfs_time; + __le32 block_size; + __le32 fragments; + __le16 compression; + __le16 block_log; + __le16 flags; + __le16 no_ids; + __le16 s_major; + __le16 s_minor; + __le64 root_inode; + __le64 bytes_used; + __le64 id_table_start; + __le64 xattr_id_table_start; + __le64 inode_table_start; + __le64 directory_table_start; + __le64 fragment_table_start; + __le64 lookup_table_start; +}; + +struct squashfs_dir_index { + __le32 index; + __le32 start_block; + __le32 size; + unsigned char name[]; +}; + +struct squashfs_base_inode { + __le16 inode_type; + __le16 mode; + __le16 uid; + __le16 guid; + __le32 mtime; + __le32 inode_number; +}; + +struct squashfs_ipc_inode { + __le16 inode_type; + __le16 mode; + __le16 uid; + __le16 guid; + __le32 mtime; + __le32 inode_number; + __le32 nlink; +}; + +struct squashfs_lipc_inode { + __le16 inode_type; + __le16 mode; + __le16 uid; + __le16 guid; + __le32 mtime; + __le32 inode_number; + __le32 nlink; + __le32 xattr; +}; + +struct squashfs_dev_inode { + __le16 inode_type; + __le16 mode; + __le16 uid; + __le16 guid; + __le32 mtime; + __le32 inode_number; + __le32 nlink; + __le32 rdev; +}; + +struct squashfs_ldev_inode { + __le16 inode_type; + __le16 mode; + __le16 uid; + __le16 guid; + __le32 mtime; + __le32 inode_number; + __le32 nlink; + __le32 rdev; + __le32 xattr; +}; + +struct squashfs_symlink_inode { + __le16 inode_type; + __le16 mode; + __le16 uid; + __le16 guid; + __le32 mtime; + __le32 inode_number; + __le32 nlink; + __le32 symlink_size; + char symlink[]; +}; + +struct squashfs_reg_inode { + __le16 inode_type; + __le16 mode; + __le16 uid; + __le16 guid; + __le32 mtime; + __le32 inode_number; + __le32 start_block; + __le32 fragment; + __le32 offset; + __le32 file_size; + __le16 block_list[]; +}; + +struct squashfs_lreg_inode { + __le16 inode_type; + __le16 mode; + __le16 uid; + __le16 guid; + __le32 mtime; + __le32 inode_number; + __le64 start_block; + __le64 file_size; + __le64 sparse; + __le32 nlink; + __le32 fragment; + __le32 offset; + __le32 xattr; + __le16 block_list[]; +}; + +struct squashfs_dir_inode { + __le16 inode_type; + __le16 mode; + __le16 uid; + __le16 guid; + __le32 mtime; + __le32 inode_number; + __le32 start_block; + __le32 nlink; + __le16 file_size; + __le16 offset; + __le32 parent_inode; +}; + +struct squashfs_ldir_inode { + __le16 inode_type; + __le16 mode; + __le16 uid; + __le16 guid; + __le32 mtime; + __le32 inode_number; + __le32 nlink; + __le32 file_size; + __le32 start_block; + __le32 parent_inode; + __le16 i_count; + __le16 offset; + __le32 xattr; + struct squashfs_dir_index index[]; +}; + +union squashfs_inode { + struct squashfs_base_inode base; + struct squashfs_dev_inode dev; + struct squashfs_ldev_inode ldev; + struct squashfs_symlink_inode symlink; + struct squashfs_reg_inode reg; + struct squashfs_lreg_inode lreg; + struct squashfs_dir_inode dir; + struct squashfs_ldir_inode ldir; + struct squashfs_ipc_inode ipc; + struct squashfs_lipc_inode lipc; +}; + +struct squashfs_dir_entry { + __le16 offset; + __le16 inode_number; + __le16 type; + __le16 size; + char name[]; +}; + +struct squashfs_dir_header { + __le32 count; + __le32 start_block; + __le32 inode_number; +}; + +struct squashfs_fragment_entry { + __le64 start_block; + __le32 size; + unsigned int unused; +}; + +struct squashfs_xattr_entry { + __le16 type; + __le16 size; + char data[]; +}; + +struct squashfs_xattr_val { + __le32 vsize; + char value[]; +}; + +struct squashfs_xattr_id { + __le64 xattr; + __le32 count; + __le32 size; +}; + +struct squashfs_xattr_id_table { + __le64 xattr_table_start; + __le32 xattr_ids; + __le32 unused; +}; + +#endif diff --git a/fs/squashfs/squashfs_fs_i.h b/fs/squashfs/squashfs_fs_i.h new file mode 100644 index 000000000..8e497ac07 --- /dev/null +++ b/fs/squashfs/squashfs_fs_i.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef SQUASHFS_FS_I +#define SQUASHFS_FS_I +/* + * Squashfs + * + * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008 + * Phillip Lougher + * + * squashfs_fs_i.h + */ + +struct squashfs_inode_info { + u64 start; + int offset; + u64 xattr; + unsigned int xattr_size; + int xattr_count; + int parent; + union { + struct { + u64 fragment_block; + int fragment_size; + int fragment_offset; + u64 block_list_start; + }; + struct { + u64 dir_idx_start; + int dir_idx_offset; + int dir_idx_cnt; + }; + }; + struct inode vfs_inode; +}; + + +static inline struct squashfs_inode_info *squashfs_i(struct inode *inode) +{ + return container_of(inode, struct squashfs_inode_info, vfs_inode); +} +#endif diff --git a/fs/squashfs/squashfs_fs_sb.h b/fs/squashfs/squashfs_fs_sb.h new file mode 100644 index 000000000..b87221ea9 --- /dev/null +++ b/fs/squashfs/squashfs_fs_sb.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef SQUASHFS_FS_SB +#define SQUASHFS_FS_SB +/* + * Squashfs + * + * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008 + * Phillip Lougher + * + * squashfs_fs_sb.h + */ + +#include "squashfs_fs.h" + +/* + * Waiters for a cache entry sleep on wait_queue as exclusive waiters, so + * freeing one entry wakes one task. See squashfs_cache_get(). + * + * num_waiters is only a hint used to skip pointless wakeups: it is + * incremented before a task queues itself and decremented after it is woken, + * so it can transiently exceed the number of queued tasks. It never + * undercounts them, which is what the wakeup paths rely on. + */ +struct squashfs_cache { + char *name; + int entries; + int curr_blk; + int next_blk; + int num_waiters; + int unused; + int block_size; + int pages; + spinlock_t lock; + wait_queue_head_t wait_queue; + struct squashfs_cache_entry *entry; +}; + +struct squashfs_cache_entry { + u64 block; + int length; + int refcount; + u64 next_index; + int pending; + int error; + int num_waiters; + wait_queue_head_t wait_queue; + struct squashfs_cache *cache; + void **data; + struct squashfs_page_actor *actor; +}; + +struct squashfs_sb_info { + const struct squashfs_decompressor *decompressor; + int devblksize; + int devblksize_log2; + struct squashfs_cache *block_cache; + struct squashfs_cache *fragment_cache; + struct squashfs_cache *read_page; + struct address_space *cache_mapping; + int next_meta_index; + __le64 *id_table; + __le64 *fragment_index; + __le64 *xattr_id_table; + struct mutex meta_index_mutex; + struct meta_index *meta_index; + void *stream; + __le64 *inode_lookup_table; + u64 inode_table; + u64 directory_table; + u64 xattr_table; + unsigned int block_size; + unsigned short block_log; + long long bytes_used; + unsigned int inodes; + unsigned int fragments; + unsigned int xattr_ids; + unsigned int ids; + bool panic_on_errors; + const struct squashfs_decompressor_thread_ops *thread_ops; + int max_thread_num; +}; +#endif diff --git a/fs/squashfs/super.c b/fs/squashfs/super.c new file mode 100644 index 000000000..5dabc5770 --- /dev/null +++ b/fs/squashfs/super.c @@ -0,0 +1,706 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Squashfs - a compressed read only filesystem for Linux + * + * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008 + * Phillip Lougher + * + * super.c + */ + +/* + * This file implements code to read the superblock, read and initialise + * in-memory structures at mount time, and all the VFS glue code to register + * the filesystem. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "squashfs_fs.h" +#include "squashfs_fs_sb.h" +#include "squashfs_fs_i.h" +#include "squashfs.h" +#include "decompressor.h" +#include "xattr.h" + +static struct file_system_type squashfs_fs_type; +static const struct super_operations squashfs_super_ops; + +enum Opt_errors { + Opt_errors_continue, + Opt_errors_panic, +}; + +enum squashfs_param { + Opt_errors, + Opt_threads, +}; + +struct squashfs_mount_opts { + enum Opt_errors errors; + const struct squashfs_decompressor_thread_ops *thread_ops; + int thread_num; +}; + +static const struct constant_table squashfs_param_errors[] = { + {"continue", Opt_errors_continue }, + {"panic", Opt_errors_panic }, + {} +}; + +static const struct fs_parameter_spec squashfs_fs_parameters[] = { + fsparam_enum("errors", Opt_errors, squashfs_param_errors), + fsparam_string("threads", Opt_threads), + {} +}; + + +static int squashfs_parse_param_threads_str(const char *str, struct squashfs_mount_opts *opts) +{ +#ifdef CONFIG_SQUASHFS_CHOICE_DECOMP_BY_MOUNT + if (strcmp(str, "single") == 0) { + opts->thread_ops = &squashfs_decompressor_single; + return 0; + } + if (strcmp(str, "multi") == 0) { + opts->thread_ops = &squashfs_decompressor_multi; + return 0; + } + if (strcmp(str, "percpu") == 0) { + opts->thread_ops = &squashfs_decompressor_percpu; + return 0; + } +#endif + return -EINVAL; +} + +static int squashfs_parse_param_threads_num(const char *str, struct squashfs_mount_opts *opts) +{ +#ifdef CONFIG_SQUASHFS_MOUNT_DECOMP_THREADS + int ret; + unsigned long num; + + ret = kstrtoul(str, 0, &num); + if (ret != 0) + return -EINVAL; + if (num > 1) { + opts->thread_ops = &squashfs_decompressor_multi; + if (num > opts->thread_ops->max_decompressors()) + return -EINVAL; + opts->thread_num = (int)num; + return 0; + } +#ifdef CONFIG_SQUASHFS_DECOMP_SINGLE + if (num == 1) { + opts->thread_ops = &squashfs_decompressor_single; + opts->thread_num = 1; + return 0; + } +#endif +#endif /* !CONFIG_SQUASHFS_MOUNT_DECOMP_THREADS */ + return -EINVAL; +} + +static int squashfs_parse_param_threads(const char *str, struct squashfs_mount_opts *opts) +{ + int ret = squashfs_parse_param_threads_str(str, opts); + + if (ret == 0) + return ret; + return squashfs_parse_param_threads_num(str, opts); +} + +static int squashfs_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct squashfs_mount_opts *opts = fc->fs_private; + struct fs_parse_result result; + int opt; + + opt = fs_parse(fc, squashfs_fs_parameters, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_errors: + opts->errors = result.uint_32; + break; + case Opt_threads: + if (squashfs_parse_param_threads(param->string, opts) != 0) + return -EINVAL; + break; + default: + return -EINVAL; + } + + return 0; +} + +static const struct squashfs_decompressor *supported_squashfs_filesystem( + struct fs_context *fc, + short major, short minor, short id) +{ + const struct squashfs_decompressor *decompressor; + + if (major < SQUASHFS_MAJOR) { + errorf(fc, "Major/Minor mismatch, older Squashfs %d.%d " + "filesystems are unsupported", major, minor); + return NULL; + } else if (major > SQUASHFS_MAJOR || minor > SQUASHFS_MINOR) { + errorf(fc, "Major/Minor mismatch, trying to mount newer " + "%d.%d filesystem", major, minor); + errorf(fc, "Please update your kernel"); + return NULL; + } + + decompressor = squashfs_lookup_decompressor(id); + if (!decompressor->supported) { + errorf(fc, "Filesystem uses \"%s\" compression. This is not supported", + decompressor->name); + return NULL; + } + + return decompressor; +} + + +static int squashfs_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct squashfs_mount_opts *opts = fc->fs_private; + struct squashfs_sb_info *msblk; + struct squashfs_super_block *sblk = NULL; + struct inode *root; + long long root_inode; + unsigned short flags; + unsigned int fragments; + u64 lookup_table_start, xattr_id_table_start, next_table; + int err, devblksize = sb_min_blocksize(sb, SQUASHFS_DEVBLK_SIZE); + + TRACE("Entered squashfs_fill_superblock\n"); + + if (!devblksize) { + errorf(fc, "squashfs: unable to set blocksize\n"); + return -EINVAL; + } + + sb->s_fs_info = kzalloc_obj(*msblk); + if (sb->s_fs_info == NULL) { + ERROR("Failed to allocate squashfs_sb_info\n"); + return -ENOMEM; + } + msblk = sb->s_fs_info; + msblk->thread_ops = opts->thread_ops; + + msblk->panic_on_errors = (opts->errors == Opt_errors_panic); + + msblk->devblksize = devblksize; + msblk->devblksize_log2 = ffz(~msblk->devblksize); + + mutex_init(&msblk->meta_index_mutex); + + /* + * msblk->bytes_used is checked in squashfs_read_table to ensure reads + * are not beyond filesystem end. But as we're using + * squashfs_read_table here to read the superblock (including the value + * of bytes_used) we need to set it to an initial sensible dummy value + */ + msblk->bytes_used = sizeof(*sblk); + sblk = squashfs_read_table(sb, SQUASHFS_START, sizeof(*sblk)); + + if (IS_ERR(sblk)) { + errorf(fc, "unable to read squashfs_super_block"); + err = PTR_ERR(sblk); + sblk = NULL; + goto failed_mount; + } + + err = -EINVAL; + + /* Check it is a SQUASHFS superblock */ + sb->s_magic = le32_to_cpu(sblk->s_magic); + if (sb->s_magic != SQUASHFS_MAGIC) { + if (!(fc->sb_flags & SB_SILENT)) + errorf(fc, "Can't find a SQUASHFS superblock on %pg", + sb->s_bdev); + goto failed_mount; + } + + if (opts->thread_num == 0) { + msblk->max_thread_num = msblk->thread_ops->max_decompressors(); + } else { + msblk->max_thread_num = opts->thread_num; + } + + /* Check the MAJOR & MINOR versions and lookup compression type */ + msblk->decompressor = supported_squashfs_filesystem( + fc, + le16_to_cpu(sblk->s_major), + le16_to_cpu(sblk->s_minor), + le16_to_cpu(sblk->compression)); + if (msblk->decompressor == NULL) + goto failed_mount; + + /* Check the filesystem does not extend beyond the end of the + block device */ + msblk->bytes_used = le64_to_cpu(sblk->bytes_used); + if (msblk->bytes_used < 0 || + msblk->bytes_used > bdev_nr_bytes(sb->s_bdev)) + goto failed_mount; + + /* Check block size for sanity */ + msblk->block_size = le32_to_cpu(sblk->block_size); + if (msblk->block_size > SQUASHFS_FILE_MAX_SIZE) + goto insanity; + + /* + * Check the system page size is not larger than the filesystem + * block size (by default 128K). This is currently not supported. + */ + if (PAGE_SIZE > msblk->block_size) { + errorf(fc, "Page size > filesystem block size (%d). This is " + "currently not supported!", msblk->block_size); + goto failed_mount; + } + + /* Check block log for sanity */ + msblk->block_log = le16_to_cpu(sblk->block_log); + if (msblk->block_log > SQUASHFS_FILE_MAX_LOG) + goto failed_mount; + + /* Check that block_size and block_log match */ + if (msblk->block_size != (1 << msblk->block_log)) + goto insanity; + + /* Check the root inode for sanity */ + root_inode = le64_to_cpu(sblk->root_inode); + if (SQUASHFS_INODE_OFFSET(root_inode) > SQUASHFS_METADATA_SIZE) + goto insanity; + + msblk->inode_table = le64_to_cpu(sblk->inode_table_start); + msblk->directory_table = le64_to_cpu(sblk->directory_table_start); + msblk->inodes = le32_to_cpu(sblk->inodes); + msblk->fragments = le32_to_cpu(sblk->fragments); + msblk->ids = le16_to_cpu(sblk->no_ids); + flags = le16_to_cpu(sblk->flags); + + TRACE("Found valid superblock on %pg\n", sb->s_bdev); + TRACE("Inodes are %scompressed\n", SQUASHFS_UNCOMPRESSED_INODES(flags) + ? "un" : ""); + TRACE("Data is %scompressed\n", SQUASHFS_UNCOMPRESSED_DATA(flags) + ? "un" : ""); + TRACE("Filesystem size %lld bytes\n", msblk->bytes_used); + TRACE("Block size %d\n", msblk->block_size); + TRACE("Number of inodes %d\n", msblk->inodes); + TRACE("Number of fragments %d\n", msblk->fragments); + TRACE("Number of ids %d\n", msblk->ids); + TRACE("sblk->inode_table_start %llx\n", msblk->inode_table); + TRACE("sblk->directory_table_start %llx\n", msblk->directory_table); + TRACE("sblk->fragment_table_start %llx\n", + (u64) le64_to_cpu(sblk->fragment_table_start)); + TRACE("sblk->id_table_start %llx\n", + (u64) le64_to_cpu(sblk->id_table_start)); + + sb->s_maxbytes = MAX_LFS_FILESIZE; + sb->s_time_min = 0; + sb->s_time_max = U32_MAX; + sb->s_flags |= SB_RDONLY; + sb->s_op = &squashfs_super_ops; + + msblk->block_cache = squashfs_cache_init("metadata", + SQUASHFS_CACHED_BLKS, SQUASHFS_METADATA_SIZE); + if (IS_ERR(msblk->block_cache)) { + err = PTR_ERR(msblk->block_cache); + goto failed_mount; + } + + /* Allocate read_page block */ + msblk->read_page = squashfs_cache_init("data", + SQUASHFS_READ_PAGES, msblk->block_size); + if (IS_ERR(msblk->read_page)) { + errorf(fc, "Failed to allocate read_page block"); + err = PTR_ERR(msblk->read_page); + goto failed_mount; + } + + if (msblk->devblksize == PAGE_SIZE) { + struct inode *cache = new_inode(sb); + + if (cache == NULL) { + err = -ENOMEM; + goto failed_mount; + } + + set_nlink(cache, 1); + cache->i_size = OFFSET_MAX; + mapping_set_gfp_mask(cache->i_mapping, GFP_NOFS); + + msblk->cache_mapping = cache->i_mapping; + } + + msblk->stream = squashfs_decompressor_setup(sb, flags); + if (IS_ERR(msblk->stream)) { + err = PTR_ERR(msblk->stream); + msblk->stream = NULL; + goto insanity; + } + + /* Handle xattrs */ + sb->s_xattr = squashfs_xattr_handlers; + xattr_id_table_start = le64_to_cpu(sblk->xattr_id_table_start); + if (xattr_id_table_start == SQUASHFS_INVALID_BLK) { + next_table = msblk->bytes_used; + goto allocate_id_index_table; + } + + /* Allocate and read xattr id lookup table */ + msblk->xattr_id_table = squashfs_read_xattr_id_table(sb, + xattr_id_table_start, &msblk->xattr_table, &msblk->xattr_ids); + if (IS_ERR(msblk->xattr_id_table)) { + errorf(fc, "unable to read xattr id index table"); + err = PTR_ERR(msblk->xattr_id_table); + msblk->xattr_id_table = NULL; + if (err != -ENOTSUPP) + goto failed_mount; + } + next_table = msblk->xattr_table; + +allocate_id_index_table: + /* Allocate and read id index table */ + msblk->id_table = squashfs_read_id_index_table(sb, + le64_to_cpu(sblk->id_table_start), next_table, msblk->ids); + if (IS_ERR(msblk->id_table)) { + errorf(fc, "unable to read id index table"); + err = PTR_ERR(msblk->id_table); + msblk->id_table = NULL; + goto failed_mount; + } + next_table = le64_to_cpu(msblk->id_table[0]); + + /* Handle inode lookup table */ + lookup_table_start = le64_to_cpu(sblk->lookup_table_start); + if (lookup_table_start == SQUASHFS_INVALID_BLK) + goto handle_fragments; + + /* Allocate and read inode lookup table */ + msblk->inode_lookup_table = squashfs_read_inode_lookup_table(sb, + lookup_table_start, next_table, msblk->inodes); + if (IS_ERR(msblk->inode_lookup_table)) { + errorf(fc, "unable to read inode lookup table"); + err = PTR_ERR(msblk->inode_lookup_table); + msblk->inode_lookup_table = NULL; + goto failed_mount; + } + next_table = le64_to_cpu(msblk->inode_lookup_table[0]); + + sb->s_export_op = &squashfs_export_ops; + +handle_fragments: + fragments = msblk->fragments; + if (fragments == 0) + goto check_directory_table; + + msblk->fragment_cache = squashfs_cache_init("fragment", + min(SQUASHFS_CACHED_FRAGMENTS, fragments), msblk->block_size); + if (IS_ERR(msblk->fragment_cache)) { + err = PTR_ERR(msblk->fragment_cache); + goto failed_mount; + } + + /* Allocate and read fragment index table */ + msblk->fragment_index = squashfs_read_fragment_index_table(sb, + le64_to_cpu(sblk->fragment_table_start), next_table, fragments); + if (IS_ERR(msblk->fragment_index)) { + errorf(fc, "unable to read fragment index table"); + err = PTR_ERR(msblk->fragment_index); + msblk->fragment_index = NULL; + goto failed_mount; + } + next_table = le64_to_cpu(msblk->fragment_index[0]); + +check_directory_table: + /* Sanity check directory_table */ + if (msblk->directory_table > next_table) { + err = -EINVAL; + goto insanity; + } + + /* Sanity check inode_table */ + if (msblk->inode_table >= msblk->directory_table) { + err = -EINVAL; + goto insanity; + } + + /* allocate root */ + root = new_inode(sb); + if (!root) { + err = -ENOMEM; + goto failed_mount; + } + + err = squashfs_read_inode(root, root_inode); + if (err) { + make_bad_inode(root); + iput(root); + goto failed_mount; + } + insert_inode_hash(root); + + sb->s_root = d_make_root(root); + if (sb->s_root == NULL) { + ERROR("Root inode create failed\n"); + err = -ENOMEM; + goto failed_mount; + } + + TRACE("Leaving squashfs_fill_super\n"); + kfree(sblk); + return 0; + +insanity: + errorf(fc, "squashfs image failed sanity check"); +failed_mount: + squashfs_cache_delete(msblk->block_cache); + squashfs_cache_delete(msblk->fragment_cache); + squashfs_cache_delete(msblk->read_page); + if (msblk->cache_mapping) + iput(msblk->cache_mapping->host); + msblk->thread_ops->destroy(msblk); + kfree(msblk->inode_lookup_table); + kfree(msblk->fragment_index); + kfree(msblk->id_table); + kfree(msblk->xattr_id_table); + kfree(sb->s_fs_info); + sb->s_fs_info = NULL; + kfree(sblk); + return err; +} + +static int squashfs_get_tree(struct fs_context *fc) +{ + return get_tree_bdev(fc, squashfs_fill_super); +} + +static int squashfs_reconfigure(struct fs_context *fc) +{ + struct super_block *sb = fc->root->d_sb; + struct squashfs_sb_info *msblk = sb->s_fs_info; + struct squashfs_mount_opts *opts = fc->fs_private; + + sync_filesystem(fc->root->d_sb); + fc->sb_flags |= SB_RDONLY; + + msblk->panic_on_errors = (opts->errors == Opt_errors_panic); + + return 0; +} + +static void squashfs_free_fs_context(struct fs_context *fc) +{ + kfree(fc->fs_private); +} + +static const struct fs_context_operations squashfs_context_ops = { + .get_tree = squashfs_get_tree, + .free = squashfs_free_fs_context, + .parse_param = squashfs_parse_param, + .reconfigure = squashfs_reconfigure, +}; + +static int squashfs_show_options(struct seq_file *s, struct dentry *root) +{ + struct super_block *sb = root->d_sb; + struct squashfs_sb_info *msblk = sb->s_fs_info; + + if (msblk->panic_on_errors) + seq_puts(s, ",errors=panic"); + else + seq_puts(s, ",errors=continue"); + +#ifdef CONFIG_SQUASHFS_CHOICE_DECOMP_BY_MOUNT + if (msblk->thread_ops == &squashfs_decompressor_single) { + seq_puts(s, ",threads=single"); + return 0; + } + if (msblk->thread_ops == &squashfs_decompressor_percpu) { + seq_puts(s, ",threads=percpu"); + return 0; + } +#endif +#ifdef CONFIG_SQUASHFS_MOUNT_DECOMP_THREADS + seq_printf(s, ",threads=%d", msblk->max_thread_num); +#endif + return 0; +} + +static int squashfs_init_fs_context(struct fs_context *fc) +{ + struct squashfs_mount_opts *opts; + + opts = kzalloc_obj(*opts); + if (!opts) + return -ENOMEM; + +#ifdef CONFIG_SQUASHFS_DECOMP_SINGLE + opts->thread_ops = &squashfs_decompressor_single; +#elif defined(CONFIG_SQUASHFS_DECOMP_MULTI) + opts->thread_ops = &squashfs_decompressor_multi; +#elif defined(CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU) + opts->thread_ops = &squashfs_decompressor_percpu; +#else +#error "fail: unknown squashfs decompression thread mode?" +#endif + opts->thread_num = 0; + fc->fs_private = opts; + fc->ops = &squashfs_context_ops; + return 0; +} + +static int squashfs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct squashfs_sb_info *msblk = dentry->d_sb->s_fs_info; + u64 id = huge_encode_dev(dentry->d_sb->s_bdev->bd_dev); + + TRACE("Entered squashfs_statfs\n"); + + buf->f_type = SQUASHFS_MAGIC; + buf->f_bsize = msblk->block_size; + buf->f_blocks = ((msblk->bytes_used - 1) >> msblk->block_log) + 1; + buf->f_bfree = buf->f_bavail = 0; + buf->f_files = msblk->inodes; + buf->f_ffree = 0; + buf->f_namelen = SQUASHFS_NAME_LEN; + buf->f_fsid = u64_to_fsid(id); + + return 0; +} + + +static void squashfs_put_super(struct super_block *sb) +{ + if (sb->s_fs_info) { + struct squashfs_sb_info *sbi = sb->s_fs_info; + squashfs_cache_delete(sbi->block_cache); + squashfs_cache_delete(sbi->fragment_cache); + squashfs_cache_delete(sbi->read_page); + if (sbi->cache_mapping) + iput(sbi->cache_mapping->host); + sbi->thread_ops->destroy(sbi); + kfree(sbi->id_table); + kfree(sbi->fragment_index); + kfree(sbi->meta_index); + kfree(sbi->inode_lookup_table); + kfree(sbi->xattr_id_table); + kfree(sb->s_fs_info); + sb->s_fs_info = NULL; + } +} + +static struct kmem_cache *squashfs_inode_cachep; + + +static void init_once(void *foo) +{ + struct squashfs_inode_info *ei = foo; + + inode_init_once(&ei->vfs_inode); +} + + +static int __init init_inodecache(void) +{ + squashfs_inode_cachep = kmem_cache_create("squashfs_inode_cache", + sizeof(struct squashfs_inode_info), 0, + SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT|SLAB_ACCOUNT, + init_once); + + return squashfs_inode_cachep ? 0 : -ENOMEM; +} + + +static void destroy_inodecache(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(squashfs_inode_cachep); +} + + +static int __init init_squashfs_fs(void) +{ + int err = init_inodecache(); + + if (err) + return err; + + err = register_filesystem(&squashfs_fs_type); + if (err) { + destroy_inodecache(); + return err; + } + + pr_info("version 4.0 (2009/01/31) Phillip Lougher\n"); + + return 0; +} + + +static void __exit exit_squashfs_fs(void) +{ + unregister_filesystem(&squashfs_fs_type); + destroy_inodecache(); +} + + +static struct inode *squashfs_alloc_inode(struct super_block *sb) +{ + struct squashfs_inode_info *ei = + alloc_inode_sb(sb, squashfs_inode_cachep, GFP_KERNEL); + + return ei ? &ei->vfs_inode : NULL; +} + + +static void squashfs_free_inode(struct inode *inode) +{ + kmem_cache_free(squashfs_inode_cachep, squashfs_i(inode)); +} + +static struct file_system_type squashfs_fs_type = { + .owner = THIS_MODULE, + .name = "squashfs", + .init_fs_context = squashfs_init_fs_context, + .parameters = squashfs_fs_parameters, + .kill_sb = kill_block_super, + .fs_flags = FS_REQUIRES_DEV | FS_ALLOW_IDMAP, +}; +MODULE_ALIAS_FS("squashfs"); + +static const struct super_operations squashfs_super_ops = { + .alloc_inode = squashfs_alloc_inode, + .free_inode = squashfs_free_inode, + .statfs = squashfs_statfs, + .put_super = squashfs_put_super, + .show_options = squashfs_show_options, +}; + +module_init(init_squashfs_fs); +module_exit(exit_squashfs_fs); +MODULE_DESCRIPTION("squashfs 4.0, a compressed read-only filesystem"); +MODULE_AUTHOR("Phillip Lougher "); +MODULE_LICENSE("GPL"); diff --git a/fs/squashfs/symlink.c b/fs/squashfs/symlink.c new file mode 100644 index 000000000..6ef735bd8 --- /dev/null +++ b/fs/squashfs/symlink.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Squashfs - a compressed read only filesystem for Linux + * + * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008 + * Phillip Lougher + * + * symlink.c + */ + +/* + * This file implements code to handle symbolic links. + * + * The data contents of symbolic links are stored inside the symbolic + * link inode within the inode table. This allows the normally small symbolic + * link to be compressed as part of the inode table, achieving much greater + * compression than if the symbolic link was compressed individually. + */ + +#include +#include +#include +#include +#include +#include + +#include "squashfs_fs.h" +#include "squashfs_fs_sb.h" +#include "squashfs_fs_i.h" +#include "squashfs.h" +#include "xattr.h" + +static int squashfs_symlink_read_folio(struct file *file, struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + struct super_block *sb = inode->i_sb; + struct squashfs_sb_info *msblk = sb->s_fs_info; + int index = folio_pos(folio); + u64 block = squashfs_i(inode)->start; + int offset = squashfs_i(inode)->offset; + int length = min_t(int, i_size_read(inode) - index, PAGE_SIZE); + int bytes, copied, error; + void *pageaddr; + struct squashfs_cache_entry *entry; + + TRACE("Entered squashfs_symlink_readpage, page index %ld, start block " + "%llx, offset %x\n", folio->index, block, offset); + + /* + * Skip index bytes into symlink metadata. + */ + if (index) { + bytes = squashfs_read_metadata(sb, NULL, &block, &offset, + index); + if (bytes < 0) { + ERROR("Unable to read symlink [%llx:%x]\n", + squashfs_i(inode)->start, + squashfs_i(inode)->offset); + error = bytes; + goto out; + } + } + + /* + * Read length bytes from symlink metadata. Squashfs_read_metadata + * is not used here because it can sleep and we want to use + * kmap_local to map the folio. Instead call the underlying + * squashfs_cache_get routine. As length bytes may overlap metadata + * blocks, we may need to call squashfs_cache_get multiple times. + */ + for (bytes = 0; bytes < length; offset = 0, bytes += copied) { + entry = squashfs_cache_get(sb, msblk->block_cache, block, 0); + if (entry->error) { + ERROR("Unable to read symlink [%llx:%x]\n", + squashfs_i(inode)->start, + squashfs_i(inode)->offset); + squashfs_cache_put(entry); + error = entry->error; + goto out; + } + + pageaddr = kmap_local_folio(folio, 0); + copied = squashfs_copy_data(pageaddr + bytes, entry, offset, + length - bytes); + if (copied == length - bytes) + memset(pageaddr + length, 0, PAGE_SIZE - length); + else + block = entry->next_index; + kunmap_local(pageaddr); + squashfs_cache_put(entry); + } + + flush_dcache_folio(folio); + error = 0; +out: + folio_end_read(folio, error == 0); + return error; +} + + +const struct address_space_operations squashfs_symlink_aops = { + .read_folio = squashfs_symlink_read_folio +}; + +const struct inode_operations squashfs_symlink_inode_ops = { + .get_link = page_get_link, + .listxattr = squashfs_listxattr +}; + diff --git a/fs/squashfs/xattr.c b/fs/squashfs/xattr.c new file mode 100644 index 000000000..ce6608cab --- /dev/null +++ b/fs/squashfs/xattr.c @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Squashfs - a compressed read only filesystem for Linux + * + * Copyright (c) 2010 + * Phillip Lougher + * + * xattr.c + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "squashfs_fs.h" +#include "squashfs_fs_sb.h" +#include "squashfs_fs_i.h" +#include "squashfs.h" + +static const struct xattr_handler *squashfs_xattr_handler(int); + +ssize_t squashfs_listxattr(struct dentry *d, char *buffer, + size_t buffer_size) +{ + struct inode *inode = d_inode(d); + struct super_block *sb = inode->i_sb; + struct squashfs_sb_info *msblk = sb->s_fs_info; + u64 start = SQUASHFS_XATTR_BLK(squashfs_i(inode)->xattr) + + msblk->xattr_table; + int offset = SQUASHFS_XATTR_OFFSET(squashfs_i(inode)->xattr); + int count = squashfs_i(inode)->xattr_count; + size_t rest = buffer_size; + int err; + + /* check that the file system has xattrs */ + if (msblk->xattr_id_table == NULL) + return -EOPNOTSUPP; + + /* loop reading each xattr name */ + while (count--) { + struct squashfs_xattr_entry entry; + struct squashfs_xattr_val val; + const struct xattr_handler *handler; + int name_size; + + err = squashfs_read_metadata(sb, &entry, &start, &offset, + sizeof(entry)); + if (err < 0) + goto failed; + + name_size = le16_to_cpu(entry.size); + handler = squashfs_xattr_handler(le16_to_cpu(entry.type)); + if (handler && (!handler->list || handler->list(d))) { + const char *prefix = handler->prefix ?: handler->name; + size_t prefix_size = strlen(prefix); + + if (buffer) { + if (prefix_size + name_size + 1 > rest) { + err = -ERANGE; + goto failed; + } + memcpy(buffer, prefix, prefix_size); + buffer += prefix_size; + } + err = squashfs_read_metadata(sb, buffer, &start, + &offset, name_size); + if (err < 0) + goto failed; + if (buffer) { + buffer[name_size] = '\0'; + buffer += name_size + 1; + } + rest -= prefix_size + name_size + 1; + } else { + /* no handler or insuffficient privileges, so skip */ + err = squashfs_read_metadata(sb, NULL, &start, + &offset, name_size); + if (err < 0) + goto failed; + } + + + /* skip remaining xattr entry */ + err = squashfs_read_metadata(sb, &val, &start, &offset, + sizeof(val)); + if (err < 0) + goto failed; + + err = squashfs_read_metadata(sb, NULL, &start, &offset, + le32_to_cpu(val.vsize)); + if (err < 0) + goto failed; + } + err = buffer_size - rest; + +failed: + return err; +} + + +static int squashfs_xattr_get(struct inode *inode, int name_index, + const char *name, void *buffer, size_t buffer_size) +{ + struct super_block *sb = inode->i_sb; + struct squashfs_sb_info *msblk = sb->s_fs_info; + u64 start = SQUASHFS_XATTR_BLK(squashfs_i(inode)->xattr) + + msblk->xattr_table; + int offset = SQUASHFS_XATTR_OFFSET(squashfs_i(inode)->xattr); + int count = squashfs_i(inode)->xattr_count; + int name_len = strlen(name); + int err, vsize; + char *target = kmalloc(name_len, GFP_KERNEL); + + if (target == NULL) + return -ENOMEM; + + /* loop reading each xattr name */ + for (; count; count--) { + struct squashfs_xattr_entry entry; + struct squashfs_xattr_val val; + int type, prefix, name_size; + + err = squashfs_read_metadata(sb, &entry, &start, &offset, + sizeof(entry)); + if (err < 0) + goto failed; + + name_size = le16_to_cpu(entry.size); + type = le16_to_cpu(entry.type); + prefix = type & SQUASHFS_XATTR_PREFIX_MASK; + + if (prefix == name_index && name_size == name_len) + err = squashfs_read_metadata(sb, target, &start, + &offset, name_size); + else + err = squashfs_read_metadata(sb, NULL, &start, + &offset, name_size); + if (err < 0) + goto failed; + + if (prefix == name_index && name_size == name_len && + strncmp(target, name, name_size) == 0) { + /* found xattr */ + if (type & SQUASHFS_XATTR_VALUE_OOL) { + __le64 xattr_val; + u64 xattr; + /* val is a reference to the real location */ + err = squashfs_read_metadata(sb, &val, &start, + &offset, sizeof(val)); + if (err < 0) + goto failed; + err = squashfs_read_metadata(sb, &xattr_val, + &start, &offset, sizeof(xattr_val)); + if (err < 0) + goto failed; + xattr = le64_to_cpu(xattr_val); + start = SQUASHFS_XATTR_BLK(xattr) + + msblk->xattr_table; + offset = SQUASHFS_XATTR_OFFSET(xattr); + } + /* read xattr value */ + err = squashfs_read_metadata(sb, &val, &start, &offset, + sizeof(val)); + if (err < 0) + goto failed; + + vsize = le32_to_cpu(val.vsize); + if (buffer) { + if (vsize > buffer_size) { + err = -ERANGE; + goto failed; + } + err = squashfs_read_metadata(sb, buffer, &start, + &offset, vsize); + if (err < 0) + goto failed; + } + break; + } + + /* no match, skip remaining xattr entry */ + err = squashfs_read_metadata(sb, &val, &start, &offset, + sizeof(val)); + if (err < 0) + goto failed; + err = squashfs_read_metadata(sb, NULL, &start, &offset, + le32_to_cpu(val.vsize)); + if (err < 0) + goto failed; + } + err = count ? vsize : -ENODATA; + +failed: + kfree(target); + return err; +} + + +static int squashfs_xattr_handler_get(const struct xattr_handler *handler, + struct dentry *unused, + struct inode *inode, + const char *name, + void *buffer, size_t size) +{ + return squashfs_xattr_get(inode, handler->flags, name, + buffer, size); +} + +/* + * User namespace support + */ +static const struct xattr_handler squashfs_xattr_user_handler = { + .prefix = XATTR_USER_PREFIX, + .flags = SQUASHFS_XATTR_USER, + .get = squashfs_xattr_handler_get +}; + +/* + * Trusted namespace support + */ +static bool squashfs_trusted_xattr_handler_list(struct dentry *d) +{ + return capable(CAP_SYS_ADMIN); +} + +static const struct xattr_handler squashfs_xattr_trusted_handler = { + .prefix = XATTR_TRUSTED_PREFIX, + .flags = SQUASHFS_XATTR_TRUSTED, + .list = squashfs_trusted_xattr_handler_list, + .get = squashfs_xattr_handler_get +}; + +/* + * Security namespace support + */ +static const struct xattr_handler squashfs_xattr_security_handler = { + .prefix = XATTR_SECURITY_PREFIX, + .flags = SQUASHFS_XATTR_SECURITY, + .get = squashfs_xattr_handler_get +}; + +static const struct xattr_handler *squashfs_xattr_handler(int type) +{ + if (type & ~(SQUASHFS_XATTR_PREFIX_MASK | SQUASHFS_XATTR_VALUE_OOL)) + /* ignore unrecognised type */ + return NULL; + + switch (type & SQUASHFS_XATTR_PREFIX_MASK) { + case SQUASHFS_XATTR_USER: + return &squashfs_xattr_user_handler; + case SQUASHFS_XATTR_TRUSTED: + return &squashfs_xattr_trusted_handler; + case SQUASHFS_XATTR_SECURITY: + return &squashfs_xattr_security_handler; + default: + /* ignore unrecognised type */ + return NULL; + } +} + +const struct xattr_handler * const squashfs_xattr_handlers[] = { + &squashfs_xattr_user_handler, + &squashfs_xattr_trusted_handler, + &squashfs_xattr_security_handler, + NULL +}; + diff --git a/fs/squashfs/xattr.h b/fs/squashfs/xattr.h new file mode 100644 index 000000000..f1a463d8b --- /dev/null +++ b/fs/squashfs/xattr.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Squashfs - a compressed read only filesystem for Linux + * + * Copyright (c) 2010 + * Phillip Lougher + * + * xattr.h + */ + +#ifdef CONFIG_SQUASHFS_XATTR +extern __le64 *squashfs_read_xattr_id_table(struct super_block *, u64, + u64 *, unsigned int *); +extern int squashfs_xattr_lookup(struct super_block *, unsigned int, int *, + unsigned int *, unsigned long long *); +#else +static inline __le64 *squashfs_read_xattr_id_table(struct super_block *sb, + u64 start, u64 *xattr_table_start, unsigned int *xattr_ids) +{ + struct squashfs_xattr_id_table *id_table; + + id_table = squashfs_read_table(sb, start, sizeof(*id_table)); + if (IS_ERR(id_table)) + return (__le64 *) id_table; + + *xattr_table_start = le64_to_cpu(id_table->xattr_table_start); + kfree(id_table); + + ERROR("Xattrs in filesystem, these will be ignored\n"); + return ERR_PTR(-ENOTSUPP); +} + +static inline int squashfs_xattr_lookup(struct super_block *sb, + unsigned int index, int *count, unsigned int *size, + unsigned long long *xattr) +{ + return 0; +} +#define squashfs_listxattr NULL +#define squashfs_xattr_handlers NULL +#endif diff --git a/fs/squashfs/xattr_id.c b/fs/squashfs/xattr_id.c new file mode 100644 index 000000000..c8469c656 --- /dev/null +++ b/fs/squashfs/xattr_id.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Squashfs - a compressed read only filesystem for Linux + * + * Copyright (c) 2010 + * Phillip Lougher + * + * xattr_id.c + */ + +/* + * This file implements code to map the 32-bit xattr id stored in the inode + * into the on disk location of the xattr data. + */ + +#include +#include +#include + +#include "squashfs_fs.h" +#include "squashfs_fs_sb.h" +#include "squashfs.h" +#include "xattr.h" + +/* + * Map xattr id using the xattr id look up table + */ +int squashfs_xattr_lookup(struct super_block *sb, unsigned int index, + int *count, unsigned int *size, unsigned long long *xattr) +{ + struct squashfs_sb_info *msblk = sb->s_fs_info; + int block = SQUASHFS_XATTR_BLOCK(index); + int offset = SQUASHFS_XATTR_BLOCK_OFFSET(index); + u64 start_block; + struct squashfs_xattr_id id; + int err; + + if (index >= msblk->xattr_ids) + return -EINVAL; + + start_block = le64_to_cpu(msblk->xattr_id_table[block]); + + err = squashfs_read_metadata(sb, &id, &start_block, &offset, + sizeof(id)); + if (err < 0) + return err; + + *xattr = le64_to_cpu(id.xattr); + *size = le32_to_cpu(id.size); + *count = le32_to_cpu(id.count); + return 0; +} + + +/* + * Read uncompressed xattr id lookup table indexes from disk into memory + */ +__le64 *squashfs_read_xattr_id_table(struct super_block *sb, u64 table_start, + u64 *xattr_table_start, unsigned int *xattr_ids) +{ + struct squashfs_sb_info *msblk = sb->s_fs_info; + unsigned int len, indexes; + struct squashfs_xattr_id_table *id_table; + __le64 *table; + u64 start, end; + int n; + + id_table = squashfs_read_table(sb, table_start, sizeof(*id_table)); + if (IS_ERR(id_table)) + return (__le64 *) id_table; + + *xattr_table_start = le64_to_cpu(id_table->xattr_table_start); + *xattr_ids = le32_to_cpu(id_table->xattr_ids); + kfree(id_table); + + /* Sanity check values */ + + /* there is always at least one xattr id */ + if (*xattr_ids == 0) + return ERR_PTR(-EINVAL); + + len = SQUASHFS_XATTR_BLOCK_BYTES(*xattr_ids); + indexes = SQUASHFS_XATTR_BLOCKS(*xattr_ids); + + /* + * The computed size of the index table (len bytes) should exactly + * match the table start and end points + */ + start = table_start + sizeof(*id_table); + end = msblk->bytes_used; + + if (len != (end - start)) + return ERR_PTR(-EINVAL); + + table = squashfs_read_table(sb, start, len); + if (IS_ERR(table)) + return table; + + /* table[0], table[1], ... table[indexes - 1] store the locations + * of the compressed xattr id blocks. Each entry should be less than + * the next (i.e. table[0] < table[1]), and the difference between them + * should be SQUASHFS_METADATA_SIZE or less. table[indexes - 1] + * should be less than table_start, and again the difference + * shouls be SQUASHFS_METADATA_SIZE or less. + * + * Finally xattr_table_start should be less than table[0]. + */ + for (n = 0; n < (indexes - 1); n++) { + start = le64_to_cpu(table[n]); + end = le64_to_cpu(table[n + 1]); + + if (start >= end || (end - start) > + (SQUASHFS_METADATA_SIZE + SQUASHFS_BLOCK_OFFSET)) { + kfree(table); + return ERR_PTR(-EINVAL); + } + } + + start = le64_to_cpu(table[indexes - 1]); + if (start >= table_start || (table_start - start) > + (SQUASHFS_METADATA_SIZE + SQUASHFS_BLOCK_OFFSET)) { + kfree(table); + return ERR_PTR(-EINVAL); + } + + if (*xattr_table_start >= le64_to_cpu(table[0])) { + kfree(table); + return ERR_PTR(-EINVAL); + } + + return table; +} diff --git a/fs/squashfs/xz_wrapper.c b/fs/squashfs/xz_wrapper.c new file mode 100644 index 000000000..0a4ff3ec9 --- /dev/null +++ b/fs/squashfs/xz_wrapper.c @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Squashfs - a compressed read only filesystem for Linux + * + * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 + * Phillip Lougher + * + * xz_wrapper.c + */ + + +#include +#include +#include +#include +#include + +#include "squashfs_fs.h" +#include "squashfs_fs_sb.h" +#include "squashfs.h" +#include "decompressor.h" +#include "page_actor.h" + +struct squashfs_xz { + struct xz_dec *state; + struct xz_buf buf; +}; + +struct disk_comp_opts { + __le32 dictionary_size; + __le32 flags; +}; + +struct comp_opts { + int dict_size; +}; + +static void *squashfs_xz_comp_opts(struct squashfs_sb_info *msblk, + void *buff, int len) +{ + struct disk_comp_opts *comp_opts = buff; + struct comp_opts *opts; + int err = 0, n; + + opts = kmalloc_obj(*opts); + if (opts == NULL) { + err = -ENOMEM; + goto out2; + } + + if (comp_opts) { + /* check compressor options are the expected length */ + if (len < sizeof(*comp_opts)) { + err = -EIO; + goto out; + } + + opts->dict_size = le32_to_cpu(comp_opts->dictionary_size); + + /* the dictionary size should be 2^n or 2^n+2^(n+1) */ + n = ffs(opts->dict_size) - 1; + if (opts->dict_size != (1 << n) && opts->dict_size != (1 << n) + + (1 << (n + 1))) { + err = -EIO; + goto out; + } + } else + /* use defaults */ + opts->dict_size = max_t(int, msblk->block_size, + SQUASHFS_METADATA_SIZE); + + return opts; + +out: + kfree(opts); +out2: + return ERR_PTR(err); +} + + +static void *squashfs_xz_init(struct squashfs_sb_info *msblk, void *buff) +{ + struct comp_opts *comp_opts = buff; + struct squashfs_xz *stream; + int err; + + stream = kmalloc_obj(*stream); + if (stream == NULL) { + err = -ENOMEM; + goto failed; + } + + stream->state = xz_dec_init(XZ_PREALLOC, comp_opts->dict_size); + if (stream->state == NULL) { + kfree(stream); + err = -ENOMEM; + goto failed; + } + + return stream; + +failed: + ERROR("Failed to initialise xz decompressor\n"); + return ERR_PTR(err); +} + + +static void squashfs_xz_free(void *strm) +{ + struct squashfs_xz *stream = strm; + + if (stream) { + xz_dec_end(stream->state); + kfree(stream); + } +} + + +static int squashfs_xz_uncompress(struct squashfs_sb_info *msblk, void *strm, + struct bio *bio, int offset, int length, + struct squashfs_page_actor *output) +{ + struct bvec_iter_all iter_all = {}; + struct bio_vec *bvec = bvec_init_iter_all(&iter_all); + int total = 0, error = 0; + struct squashfs_xz *stream = strm; + + xz_dec_reset(stream->state); + stream->buf.in_pos = 0; + stream->buf.in_size = 0; + stream->buf.out_pos = 0; + stream->buf.out_size = PAGE_SIZE; + stream->buf.out = squashfs_first_page(output); + if (IS_ERR(stream->buf.out)) { + error = PTR_ERR(stream->buf.out); + goto finish; + } + + for (;;) { + enum xz_ret xz_err; + + if (stream->buf.in_pos == stream->buf.in_size) { + const void *data; + int avail; + + if (!bio_next_segment(bio, &iter_all)) { + /* XZ_STREAM_END must be reached. */ + error = -EIO; + break; + } + + avail = min(length, ((int)bvec->bv_len) - offset); + data = bvec_virt(bvec); + length -= avail; + stream->buf.in = data + offset; + stream->buf.in_size = avail; + stream->buf.in_pos = 0; + offset = 0; + } + + if (stream->buf.out_pos == stream->buf.out_size) { + stream->buf.out = squashfs_next_page(output); + if (IS_ERR(stream->buf.out)) { + error = PTR_ERR(stream->buf.out); + break; + } else if (stream->buf.out != NULL) { + stream->buf.out_pos = 0; + total += PAGE_SIZE; + } + } + + xz_err = xz_dec_run(stream->state, &stream->buf); + if (xz_err == XZ_STREAM_END) + break; + if (xz_err != XZ_OK) { + error = -EIO; + break; + } + } + +finish: + squashfs_finish_page(output); + + return error ? error : total + stream->buf.out_pos; +} + +const struct squashfs_decompressor squashfs_xz_comp_ops = { + .init = squashfs_xz_init, + .comp_opts = squashfs_xz_comp_opts, + .free = squashfs_xz_free, + .decompress = squashfs_xz_uncompress, + .id = XZ_COMPRESSION, + .name = "xz", + .alloc_buffer = 1, + .supported = 1 +}; diff --git a/fs/squashfs/zlib_wrapper.c b/fs/squashfs/zlib_wrapper.c new file mode 100644 index 000000000..a29001311 --- /dev/null +++ b/fs/squashfs/zlib_wrapper.c @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Squashfs - a compressed read only filesystem for Linux + * + * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 + * Phillip Lougher + * + * zlib_wrapper.c + */ + + +#include +#include +#include +#include +#include + +#include "squashfs_fs.h" +#include "squashfs_fs_sb.h" +#include "squashfs.h" +#include "decompressor.h" +#include "page_actor.h" + +static void *zlib_init(struct squashfs_sb_info *dummy, void *buff) +{ + z_stream *stream = kmalloc_obj(z_stream); + if (stream == NULL) + goto failed; + stream->workspace = vmalloc(zlib_inflate_workspacesize()); + if (stream->workspace == NULL) + goto failed; + + return stream; + +failed: + ERROR("Failed to allocate zlib workspace\n"); + kfree(stream); + return ERR_PTR(-ENOMEM); +} + + +static void zlib_free(void *strm) +{ + z_stream *stream = strm; + + if (stream) + vfree(stream->workspace); + kfree(stream); +} + + +static int zlib_uncompress(struct squashfs_sb_info *msblk, void *strm, + struct bio *bio, int offset, int length, + struct squashfs_page_actor *output) +{ + struct bvec_iter_all iter_all = {}; + struct bio_vec *bvec = bvec_init_iter_all(&iter_all); + int zlib_init = 0, error = 0; + z_stream *stream = strm; + + stream->avail_out = PAGE_SIZE; + stream->next_out = squashfs_first_page(output); + stream->avail_in = 0; + + if (IS_ERR(stream->next_out)) { + error = PTR_ERR(stream->next_out); + goto finish; + } + + for (;;) { + int zlib_err; + + if (stream->avail_in == 0) { + const void *data; + int avail; + + if (!bio_next_segment(bio, &iter_all)) { + /* Z_STREAM_END must be reached. */ + error = -EIO; + break; + } + + avail = min(length, ((int)bvec->bv_len) - offset); + data = bvec_virt(bvec); + length -= avail; + stream->next_in = data + offset; + stream->avail_in = avail; + offset = 0; + } + + if (stream->avail_out == 0) { + stream->next_out = squashfs_next_page(output); + if (IS_ERR(stream->next_out)) { + error = PTR_ERR(stream->next_out); + break; + } else if (stream->next_out != NULL) + stream->avail_out = PAGE_SIZE; + } + + if (!zlib_init) { + zlib_err = zlib_inflateInit(stream); + if (zlib_err != Z_OK) { + error = -EIO; + break; + } + zlib_init = 1; + } + + zlib_err = zlib_inflate(stream, Z_SYNC_FLUSH); + if (zlib_err == Z_STREAM_END) + break; + if (zlib_err != Z_OK) { + error = -EIO; + break; + } + } + +finish: + squashfs_finish_page(output); + + if (!error) + if (zlib_inflateEnd(stream) != Z_OK) + error = -EIO; + + return error ? error : stream->total_out; +} + +const struct squashfs_decompressor squashfs_zlib_comp_ops = { + .init = zlib_init, + .free = zlib_free, + .decompress = zlib_uncompress, + .id = ZLIB_COMPRESSION, + .name = "zlib", + .alloc_buffer = 1, + .supported = 1 +}; + diff --git a/fs/squashfs/zstd_wrapper.c b/fs/squashfs/zstd_wrapper.c new file mode 100644 index 000000000..487fbc51d --- /dev/null +++ b/fs/squashfs/zstd_wrapper.c @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Squashfs - a compressed read only filesystem for Linux + * + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * zstd_wrapper.c + */ + +#include +#include +#include +#include +#include + +#include "squashfs_fs.h" +#include "squashfs_fs_sb.h" +#include "squashfs.h" +#include "decompressor.h" +#include "page_actor.h" + +struct workspace { + void *mem; + size_t mem_size; + size_t window_size; +}; + +static void *zstd_init(struct squashfs_sb_info *msblk, void *buff) +{ + struct workspace *wksp = kmalloc_obj(*wksp); + + if (wksp == NULL) + goto failed; + wksp->window_size = max_t(size_t, + msblk->block_size, SQUASHFS_METADATA_SIZE); + wksp->mem_size = zstd_dstream_workspace_bound(wksp->window_size); + wksp->mem = vmalloc(wksp->mem_size); + if (wksp->mem == NULL) + goto failed; + + return wksp; + +failed: + ERROR("Failed to allocate zstd workspace\n"); + kfree(wksp); + return ERR_PTR(-ENOMEM); +} + + +static void zstd_free(void *strm) +{ + struct workspace *wksp = strm; + + if (wksp) + vfree(wksp->mem); + kfree(wksp); +} + + +static int zstd_uncompress(struct squashfs_sb_info *msblk, void *strm, + struct bio *bio, int offset, int length, + struct squashfs_page_actor *output) +{ + struct workspace *wksp = strm; + zstd_dstream *stream; + size_t total_out = 0; + int error = 0; + zstd_in_buffer in_buf = { NULL, 0, 0 }; + zstd_out_buffer out_buf = { NULL, 0, 0 }; + struct bvec_iter_all iter_all = {}; + struct bio_vec *bvec = bvec_init_iter_all(&iter_all); + + stream = zstd_init_dstream(wksp->window_size, wksp->mem, wksp->mem_size); + + if (!stream) { + ERROR("Failed to initialize zstd decompressor\n"); + return -EIO; + } + + out_buf.size = PAGE_SIZE; + out_buf.dst = squashfs_first_page(output); + if (IS_ERR(out_buf.dst)) { + error = PTR_ERR(out_buf.dst); + goto finish; + } + + for (;;) { + size_t zstd_err; + + if (in_buf.pos == in_buf.size) { + const void *data; + int avail; + + if (!bio_next_segment(bio, &iter_all)) { + error = -EIO; + break; + } + + avail = min(length, ((int)bvec->bv_len) - offset); + data = bvec_virt(bvec); + length -= avail; + in_buf.src = data + offset; + in_buf.size = avail; + in_buf.pos = 0; + offset = 0; + } + + if (out_buf.pos == out_buf.size) { + out_buf.dst = squashfs_next_page(output); + if (IS_ERR(out_buf.dst)) { + error = PTR_ERR(out_buf.dst); + break; + } else if (out_buf.dst == NULL) { + /* Shouldn't run out of pages + * before stream is done. + */ + error = -EIO; + break; + } + out_buf.pos = 0; + out_buf.size = PAGE_SIZE; + } + + total_out -= out_buf.pos; + zstd_err = zstd_decompress_stream(stream, &out_buf, &in_buf); + total_out += out_buf.pos; /* add the additional data produced */ + if (zstd_err == 0) + break; + + if (zstd_is_error(zstd_err)) { + ERROR("zstd decompression error: %d\n", + (int)zstd_get_error_code(zstd_err)); + error = -EIO; + break; + } + } + +finish: + + squashfs_finish_page(output); + + return error ? error : total_out; +} + +const struct squashfs_decompressor squashfs_zstd_comp_ops = { + .init = zstd_init, + .free = zstd_free, + .decompress = zstd_uncompress, + .id = ZSTD_COMPRESSION, + .name = "zstd", + .alloc_buffer = 1, + .supported = 1 +}; diff --git a/fs/stack.c b/fs/stack.c new file mode 100644 index 000000000..d8c782e06 --- /dev/null +++ b/fs/stack.c @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include + +/* does _NOT_ require i_rwsem to be held. + * + * This function cannot be inlined since i_size_{read,write} is rather + * heavy-weight on 32-bit systems + */ +void fsstack_copy_inode_size(struct inode *dst, struct inode *src) +{ + loff_t i_size; + blkcnt_t i_blocks; + + /* + * i_size_read() includes its own seqlocking and protection from + * preemption (see include/linux/fs.h): we need nothing extra for + * that here, and prefer to avoid nesting locks than attempt to keep + * i_size and i_blocks in sync together. + */ + i_size = i_size_read(src); + + /* + * But on 32-bit, we ought to make an effort to keep the two halves of + * i_blocks in sync despite SMP or PREEMPTION - though stat's + * generic_fillattr() doesn't bother, and we won't be applying quotas + * (where i_blocks does become important) at the upper level. + * + * We don't actually know what locking is used at the lower level; + * but if it's a filesystem that supports quotas, it will be using + * i_lock as in inode_add_bytes(). + */ + if (sizeof(i_blocks) > sizeof(long)) + spin_lock(&src->i_lock); + i_blocks = src->i_blocks; + if (sizeof(i_blocks) > sizeof(long)) + spin_unlock(&src->i_lock); + + /* + * If CONFIG_SMP or CONFIG_PREEMPTION on 32-bit, it's vital for + * fsstack_copy_inode_size() to hold some lock around + * i_size_write(), otherwise i_size_read() may spin forever (see + * include/linux/fs.h). We don't necessarily hold i_rwsem when this + * is called, so take i_lock for that case. + * + * And if on 32-bit, continue our effort to keep the two halves of + * i_blocks in sync despite SMP or PREEMPTION: use i_lock for that case + * too, and do both at once by combining the tests. + * + * There is none of this locking overhead in the 64-bit case. + */ + if (sizeof(i_size) > sizeof(long) || sizeof(i_blocks) > sizeof(long)) + spin_lock(&dst->i_lock); + i_size_write(dst, i_size); + dst->i_blocks = i_blocks; + if (sizeof(i_size) > sizeof(long) || sizeof(i_blocks) > sizeof(long)) + spin_unlock(&dst->i_lock); +} +EXPORT_SYMBOL_GPL(fsstack_copy_inode_size); + +/* copy all attributes */ +void fsstack_copy_attr_all(struct inode *dest, const struct inode *src) +{ + dest->i_mode = src->i_mode; + dest->i_uid = src->i_uid; + dest->i_gid = src->i_gid; + dest->i_rdev = src->i_rdev; + inode_set_atime_to_ts(dest, inode_get_atime(src)); + inode_set_mtime_to_ts(dest, inode_get_mtime(src)); + inode_set_ctime_to_ts(dest, inode_get_ctime(src)); + dest->i_blkbits = src->i_blkbits; + dest->i_flags = src->i_flags; + set_nlink(dest, src->i_nlink); +} +EXPORT_SYMBOL_GPL(fsstack_copy_attr_all); diff --git a/fs/stat.c b/fs/stat.c new file mode 100644 index 000000000..c461c3054 --- /dev/null +++ b/fs/stat.c @@ -0,0 +1,968 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/stat.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include "internal.h" +#include "mount.h" + +/** + * fill_mg_cmtime - Fill in the mtime and ctime and flag ctime as QUERIED + * @stat: where to store the resulting values + * @request_mask: STATX_* values requested + * @inode: inode from which to grab the c/mtime + * + * Given @inode, grab the ctime and mtime out if it and store the result + * in @stat. When fetching the value, flag it as QUERIED (if not already) + * so the next write will record a distinct timestamp. + * + * NB: The QUERIED flag is tracked in the ctime, but we set it there even + * if only the mtime was requested, as that ensures that the next mtime + * change will be distinct. + */ +void fill_mg_cmtime(struct kstat *stat, u32 request_mask, struct inode *inode) +{ + atomic_t *pcn = (atomic_t *)&inode->i_ctime_nsec; + + /* If neither time was requested, then don't report them */ + if (!(request_mask & (STATX_CTIME|STATX_MTIME))) { + stat->result_mask &= ~(STATX_CTIME|STATX_MTIME); + return; + } + + stat->mtime = inode_get_mtime(inode); + stat->ctime.tv_sec = inode_get_ctime_sec(inode); + stat->ctime.tv_nsec = (u32)atomic_read(pcn); + if (!(stat->ctime.tv_nsec & I_CTIME_QUERIED)) + stat->ctime.tv_nsec = ((u32)atomic_fetch_or(I_CTIME_QUERIED, pcn)); + stat->ctime.tv_nsec &= ~I_CTIME_QUERIED; + trace_fill_mg_cmtime(inode, &stat->ctime, &stat->mtime); +} +EXPORT_SYMBOL(fill_mg_cmtime); + +/** + * generic_fillattr - Fill in the basic attributes from the inode struct + * @idmap: idmap of the mount the inode was found from + * @request_mask: statx request_mask + * @inode: Inode to use as the source + * @stat: Where to fill in the attributes + * + * Fill in the basic attributes in the kstat structure from data that's to be + * found on the VFS inode structure. This is the default if no getattr inode + * operation is supplied. + * + * If the inode has been found through an idmapped mount the idmap of + * the vfsmount must be passed through @idmap. This function will then + * take care to map the inode according to @idmap before filling in the + * uid and gid filds. On non-idmapped mounts or if permission checking is to be + * performed on the raw inode simply pass @nop_mnt_idmap. + */ +void generic_fillattr(struct mnt_idmap *idmap, u32 request_mask, + struct inode *inode, struct kstat *stat) +{ + vfsuid_t vfsuid = i_uid_into_vfsuid(idmap, inode); + vfsgid_t vfsgid = i_gid_into_vfsgid(idmap, inode); + + stat->dev = inode->i_sb->s_dev; + stat->ino = inode->i_ino; + stat->mode = inode->i_mode; + stat->nlink = inode->i_nlink; + stat->uid = vfsuid_into_kuid(vfsuid); + stat->gid = vfsgid_into_kgid(vfsgid); + stat->rdev = inode->i_rdev; + stat->size = i_size_read(inode); + stat->atime = inode_get_atime(inode); + + if (is_mgtime(inode)) { + fill_mg_cmtime(stat, request_mask, inode); + } else { + stat->ctime = inode_get_ctime(inode); + stat->mtime = inode_get_mtime(inode); + } + + stat->blksize = i_blocksize(inode); + stat->blocks = inode->i_blocks; + + if ((request_mask & STATX_CHANGE_COOKIE) && IS_I_VERSION(inode)) { + stat->result_mask |= STATX_CHANGE_COOKIE; + stat->change_cookie = inode_query_iversion(inode); + } + +} +EXPORT_SYMBOL(generic_fillattr); + +/** + * generic_fill_statx_attr - Fill in the statx attributes from the inode flags + * @inode: Inode to use as the source + * @stat: Where to fill in the attribute flags + * + * Fill in the STATX_ATTR_* flags in the kstat structure for properties of the + * inode that are published on i_flags and enforced by the VFS. + */ +void generic_fill_statx_attr(struct inode *inode, struct kstat *stat) +{ + if (inode->i_flags & S_IMMUTABLE) + stat->attributes |= STATX_ATTR_IMMUTABLE; + if (inode->i_flags & S_APPEND) + stat->attributes |= STATX_ATTR_APPEND; + stat->attributes_mask |= KSTAT_ATTR_VFS_FLAGS; +} +EXPORT_SYMBOL(generic_fill_statx_attr); + +/** + * generic_fill_statx_atomic_writes - Fill in atomic writes statx attributes + * @stat: Where to fill in the attribute flags + * @unit_min: Minimum supported atomic write length in bytes + * @unit_max: Maximum supported atomic write length in bytes + * @unit_max_opt: Optimised maximum supported atomic write length in bytes + * + * Fill in the STATX{_ATTR}_WRITE_ATOMIC flags in the kstat structure from + * atomic write unit_min and unit_max values. + */ +void generic_fill_statx_atomic_writes(struct kstat *stat, + unsigned int unit_min, + unsigned int unit_max, + unsigned int unit_max_opt) +{ + /* Confirm that the request type is known */ + stat->result_mask |= STATX_WRITE_ATOMIC; + + /* Confirm that the file attribute type is known */ + stat->attributes_mask |= STATX_ATTR_WRITE_ATOMIC; + + if (unit_min) { + stat->atomic_write_unit_min = unit_min; + stat->atomic_write_unit_max = unit_max; + stat->atomic_write_unit_max_opt = unit_max_opt; + /* Initially only allow 1x segment */ + stat->atomic_write_segments_max = 1; + + /* Confirm atomic writes are actually supported */ + stat->attributes |= STATX_ATTR_WRITE_ATOMIC; + } +} +EXPORT_SYMBOL_GPL(generic_fill_statx_atomic_writes); + +/** + * vfs_getattr_nosec - getattr without security checks + * @path: file to get attributes from + * @stat: structure to return attributes in + * @request_mask: STATX_xxx flags indicating what the caller wants + * @query_flags: Query mode (AT_STATX_SYNC_TYPE) + * + * Get attributes without calling security_inode_getattr. + * + * Currently the only caller other than vfs_getattr is internal to the + * filehandle lookup code, which uses only the inode number and returns no + * attributes to any user. Any other code probably wants vfs_getattr. + */ +int vfs_getattr_nosec(const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int query_flags) +{ + struct mnt_idmap *idmap; + struct inode *inode = d_backing_inode(path->dentry); + + memset(stat, 0, sizeof(*stat)); + stat->result_mask |= STATX_BASIC_STATS; + query_flags &= AT_STATX_SYNC_TYPE; + + /* allow the fs to override these if it really wants to */ + /* SB_NOATIME means filesystem supplies dummy atime value */ + if (inode->i_sb->s_flags & SB_NOATIME) + stat->result_mask &= ~STATX_ATIME; + + /* + * Note: If you add another clause to set an attribute flag, please + * update attributes_mask below. + */ + if (IS_AUTOMOUNT(inode)) + stat->attributes |= STATX_ATTR_AUTOMOUNT; + + if (IS_DAX(inode)) + stat->attributes |= STATX_ATTR_DAX; + + stat->attributes_mask |= (STATX_ATTR_AUTOMOUNT | + STATX_ATTR_DAX); + + idmap = mnt_idmap(path->mnt); + if (inode->i_op->getattr) { + int ret; + + ret = inode->i_op->getattr(idmap, path, stat, request_mask, + query_flags); + if (ret) + return ret; + } else { + generic_fillattr(idmap, request_mask, inode, stat); + } + + /* + * If this is a block device inode, override the filesystem attributes + * with the block device specific parameters that need to be obtained + * from the bdev backing inode. + */ + if (S_ISBLK(stat->mode)) + bdev_statx(path, stat, request_mask); + + return 0; +} +EXPORT_SYMBOL(vfs_getattr_nosec); + +/* + * vfs_getattr - Get the enhanced basic attributes of a file + * @path: The file of interest + * @stat: Where to return the statistics + * @request_mask: STATX_xxx flags indicating what the caller wants + * @query_flags: Query mode (AT_STATX_SYNC_TYPE) + * + * Ask the filesystem for a file's attributes. The caller must indicate in + * request_mask and query_flags to indicate what they want. + * + * If the file is remote, the filesystem can be forced to update the attributes + * from the backing store by passing AT_STATX_FORCE_SYNC in query_flags or can + * suppress the update by passing AT_STATX_DONT_SYNC. + * + * Bits must have been set in request_mask to indicate which attributes the + * caller wants retrieving. Any such attribute not requested may be returned + * anyway, but the value may be approximate, and, if remote, may not have been + * synchronised with the server. + * + * 0 will be returned on success, and a -ve error code if unsuccessful. + */ +int vfs_getattr(const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int query_flags) +{ + int retval; + + retval = security_inode_getattr(path); + if (unlikely(retval)) + return retval; + return vfs_getattr_nosec(path, stat, request_mask, query_flags); +} +EXPORT_SYMBOL(vfs_getattr); + +/** + * vfs_fstat - Get the basic attributes by file descriptor + * @fd: The file descriptor referring to the file of interest + * @stat: The result structure to fill in. + * + * This function is a wrapper around vfs_getattr(). The main difference is + * that it uses a file descriptor to determine the file location. + * + * 0 will be returned on success, and a -ve error code if unsuccessful. + */ +int vfs_fstat(int fd, struct kstat *stat) +{ + CLASS(fd_raw, f)(fd); + if (fd_empty(f)) + return -EBADF; + return vfs_getattr(&fd_file(f)->f_path, stat, STATX_BASIC_STATS, 0); +} + +static int statx_lookup_flags(int flags) +{ + int lookup_flags = 0; + + if (!(flags & AT_SYMLINK_NOFOLLOW)) + lookup_flags |= LOOKUP_FOLLOW; + if (!(flags & AT_NO_AUTOMOUNT)) + lookup_flags |= LOOKUP_AUTOMOUNT; + + return lookup_flags; +} + +static int vfs_statx_path(const struct path *path, int flags, struct kstat *stat, + u32 request_mask) +{ + int error = vfs_getattr(path, stat, request_mask, flags); + if (error) + return error; + + if (request_mask & STATX_MNT_ID_UNIQUE) { + stat->mnt_id = real_mount(path->mnt)->mnt_id_unique; + stat->result_mask |= STATX_MNT_ID_UNIQUE; + } else { + stat->mnt_id = real_mount(path->mnt)->mnt_id; + stat->result_mask |= STATX_MNT_ID; + } + + if (path_mounted(path)) + stat->attributes |= STATX_ATTR_MOUNT_ROOT; + stat->attributes_mask |= STATX_ATTR_MOUNT_ROOT; + return 0; +} + +static int vfs_statx_fd(int fd, int flags, struct kstat *stat, + u32 request_mask) +{ + CLASS(fd_raw, f)(fd); + if (fd_empty(f)) + return -EBADF; + return vfs_statx_path(&fd_file(f)->f_path, flags, stat, request_mask); +} + +/** + * vfs_statx - Get basic and extra attributes by filename + * @dfd: A file descriptor representing the base dir for a relative filename + * @filename: The name of the file of interest + * @flags: Flags to control the query + * @stat: The result structure to fill in. + * @request_mask: STATX_xxx flags indicating what the caller wants + * + * This function is a wrapper around vfs_getattr(). The main difference is + * that it uses a filename and base directory to determine the file location. + * Additionally, the use of AT_SYMLINK_NOFOLLOW in flags will prevent a symlink + * at the given name from being referenced. + * + * 0 will be returned on success, and a -ve error code if unsuccessful. + */ +static int vfs_statx(int dfd, struct filename *filename, int flags, + struct kstat *stat, u32 request_mask) +{ + struct path path; + unsigned int lookup_flags = statx_lookup_flags(flags); + int error; + + if (flags & ~(AT_SYMLINK_NOFOLLOW | AT_NO_AUTOMOUNT | AT_EMPTY_PATH | + AT_STATX_SYNC_TYPE)) + return -EINVAL; + +retry: + error = filename_lookup(dfd, filename, lookup_flags, &path, NULL); + if (error) + return error; + error = vfs_statx_path(&path, flags, stat, request_mask); + path_put(&path); + if (retry_estale(error, lookup_flags)) { + lookup_flags |= LOOKUP_REVAL; + goto retry; + } + return error; +} + +int vfs_fstatat(int dfd, const char __user *filename, + struct kstat *stat, int flags) +{ + CLASS(filename_maybe_null, name)(filename, flags); + + if (!name && dfd >= 0) + return vfs_fstat(dfd, stat); + + return vfs_statx(dfd, name, flags | AT_NO_AUTOMOUNT, + stat, STATX_BASIC_STATS); +} + +#ifdef __ARCH_WANT_OLD_STAT + +/* + * For backward compatibility? Maybe this should be moved + * into arch/i386 instead? + */ +static int cp_old_stat(struct kstat *stat, struct __old_kernel_stat __user * statbuf) +{ + static int warncount = 5; + struct __old_kernel_stat tmp; + + if (warncount > 0) { + warncount--; + printk(KERN_WARNING "VFS: Warning: %s using old stat() call. Recompile your binary.\n", + current->comm); + } else if (warncount < 0) { + /* it's laughable, but... */ + warncount = 0; + } + + memset(&tmp, 0, sizeof(struct __old_kernel_stat)); + tmp.st_dev = old_encode_dev(stat->dev); + tmp.st_ino = stat->ino; + if (sizeof(tmp.st_ino) < sizeof(stat->ino) && tmp.st_ino != stat->ino) + return -EOVERFLOW; + tmp.st_mode = stat->mode; + tmp.st_nlink = stat->nlink; + if (tmp.st_nlink != stat->nlink) + return -EOVERFLOW; + SET_UID(tmp.st_uid, from_kuid_munged(current_user_ns(), stat->uid)); + SET_GID(tmp.st_gid, from_kgid_munged(current_user_ns(), stat->gid)); + tmp.st_rdev = old_encode_dev(stat->rdev); +#if BITS_PER_LONG == 32 + if (stat->size > MAX_NON_LFS) + return -EOVERFLOW; +#endif + tmp.st_size = stat->size; + tmp.st_atime = stat->atime.tv_sec; + tmp.st_mtime = stat->mtime.tv_sec; + tmp.st_ctime = stat->ctime.tv_sec; + return copy_to_user(statbuf,&tmp,sizeof(tmp)) ? -EFAULT : 0; +} + +SYSCALL_DEFINE2(stat, const char __user *, filename, + struct __old_kernel_stat __user *, statbuf) +{ + struct kstat stat; + int error; + + error = vfs_stat(filename, &stat); + if (unlikely(error)) + return error; + + return cp_old_stat(&stat, statbuf); +} + +SYSCALL_DEFINE2(lstat, const char __user *, filename, + struct __old_kernel_stat __user *, statbuf) +{ + struct kstat stat; + int error; + + error = vfs_lstat(filename, &stat); + if (unlikely(error)) + return error; + + return cp_old_stat(&stat, statbuf); +} + +SYSCALL_DEFINE2(fstat, unsigned int, fd, struct __old_kernel_stat __user *, statbuf) +{ + struct kstat stat; + int error; + + error = vfs_fstat(fd, &stat); + if (unlikely(error)) + return error; + + return cp_old_stat(&stat, statbuf); +} + +#endif /* __ARCH_WANT_OLD_STAT */ + +#ifdef __ARCH_WANT_NEW_STAT + +#ifndef INIT_STRUCT_STAT_PADDING +# define INIT_STRUCT_STAT_PADDING(st) memset(&st, 0, sizeof(st)) +#endif + +static int cp_new_stat(struct kstat *stat, struct stat __user *statbuf) +{ + struct stat tmp; + + if (sizeof(tmp.st_dev) < 4 && !old_valid_dev(stat->dev)) + return -EOVERFLOW; + if (sizeof(tmp.st_rdev) < 4 && !old_valid_dev(stat->rdev)) + return -EOVERFLOW; +#if BITS_PER_LONG == 32 + if (stat->size > MAX_NON_LFS) + return -EOVERFLOW; +#endif + + INIT_STRUCT_STAT_PADDING(tmp); + tmp.st_dev = new_encode_dev(stat->dev); + tmp.st_ino = stat->ino; + if (sizeof(tmp.st_ino) < sizeof(stat->ino) && tmp.st_ino != stat->ino) + return -EOVERFLOW; + tmp.st_mode = stat->mode; + tmp.st_nlink = stat->nlink; + if (tmp.st_nlink != stat->nlink) + return -EOVERFLOW; + SET_UID(tmp.st_uid, from_kuid_munged(current_user_ns(), stat->uid)); + SET_GID(tmp.st_gid, from_kgid_munged(current_user_ns(), stat->gid)); + tmp.st_rdev = new_encode_dev(stat->rdev); + tmp.st_size = stat->size; + tmp.st_atime = stat->atime.tv_sec; + tmp.st_mtime = stat->mtime.tv_sec; + tmp.st_ctime = stat->ctime.tv_sec; +#ifdef STAT_HAVE_NSEC + tmp.st_atime_nsec = stat->atime.tv_nsec; + tmp.st_mtime_nsec = stat->mtime.tv_nsec; + tmp.st_ctime_nsec = stat->ctime.tv_nsec; +#endif + tmp.st_blocks = stat->blocks; + tmp.st_blksize = stat->blksize; + return copy_to_user(statbuf,&tmp,sizeof(tmp)) ? -EFAULT : 0; +} + +SYSCALL_DEFINE2(newstat, const char __user *, filename, + struct stat __user *, statbuf) +{ + struct kstat stat; + int error; + + error = vfs_stat(filename, &stat); + if (unlikely(error)) + return error; + + return cp_new_stat(&stat, statbuf); +} + +SYSCALL_DEFINE2(newlstat, const char __user *, filename, + struct stat __user *, statbuf) +{ + struct kstat stat; + int error; + + error = vfs_lstat(filename, &stat); + if (unlikely(error)) + return error; + + return cp_new_stat(&stat, statbuf); +} + +#if !defined(__ARCH_WANT_STAT64) || defined(__ARCH_WANT_SYS_NEWFSTATAT) +SYSCALL_DEFINE4(newfstatat, int, dfd, const char __user *, filename, + struct stat __user *, statbuf, int, flag) +{ + struct kstat stat; + int error; + + error = vfs_fstatat(dfd, filename, &stat, flag); + if (unlikely(error)) + return error; + + return cp_new_stat(&stat, statbuf); +} +#endif + +SYSCALL_DEFINE2(newfstat, unsigned int, fd, struct stat __user *, statbuf) +{ + struct kstat stat; + int error; + + error = vfs_fstat(fd, &stat); + if (unlikely(error)) + return error; + + return cp_new_stat(&stat, statbuf); +} +#endif + +static int do_readlinkat(int dfd, const char __user *pathname, + char __user *buf, int bufsiz) +{ + struct path path; + int error; + unsigned int lookup_flags = 0; + + if (bufsiz <= 0) + return -EINVAL; + + CLASS(filename_flags, name)(pathname, LOOKUP_EMPTY); +retry: + error = filename_lookup(dfd, name, lookup_flags, &path, NULL); + if (unlikely(error)) + return error; + + /* + * AFS mountpoints allow readlink(2) but are not symlinks + */ + if (d_is_symlink(path.dentry) || + d_backing_inode(path.dentry)->i_op->readlink) { + error = security_inode_readlink(path.dentry); + if (!error) { + touch_atime(&path); + error = vfs_readlink(path.dentry, buf, bufsiz); + } + } else { + error = (name->name[0] == '\0') ? -ENOENT : -EINVAL; + } + path_put(&path); + if (retry_estale(error, lookup_flags)) { + lookup_flags |= LOOKUP_REVAL; + goto retry; + } + return error; +} + +SYSCALL_DEFINE4(readlinkat, int, dfd, const char __user *, pathname, + char __user *, buf, int, bufsiz) +{ + return do_readlinkat(dfd, pathname, buf, bufsiz); +} + +SYSCALL_DEFINE3(readlink, const char __user *, path, char __user *, buf, + int, bufsiz) +{ + return do_readlinkat(AT_FDCWD, path, buf, bufsiz); +} + + +/* ---------- LFS-64 ----------- */ +#if defined(__ARCH_WANT_STAT64) || defined(__ARCH_WANT_COMPAT_STAT64) + +#ifndef INIT_STRUCT_STAT64_PADDING +# define INIT_STRUCT_STAT64_PADDING(st) memset(&st, 0, sizeof(st)) +#endif + +static long cp_new_stat64(struct kstat *stat, struct stat64 __user *statbuf) +{ + struct stat64 tmp; + + INIT_STRUCT_STAT64_PADDING(tmp); +#ifdef CONFIG_MIPS + /* mips has weird padding, so we don't get 64 bits there */ + tmp.st_dev = new_encode_dev(stat->dev); + tmp.st_rdev = new_encode_dev(stat->rdev); +#else + tmp.st_dev = huge_encode_dev(stat->dev); + tmp.st_rdev = huge_encode_dev(stat->rdev); +#endif + tmp.st_ino = stat->ino; + if (sizeof(tmp.st_ino) < sizeof(stat->ino) && tmp.st_ino != stat->ino) + return -EOVERFLOW; +#ifdef STAT64_HAS_BROKEN_ST_INO + tmp.__st_ino = stat->ino; +#endif + tmp.st_mode = stat->mode; + tmp.st_nlink = stat->nlink; + tmp.st_uid = from_kuid_munged(current_user_ns(), stat->uid); + tmp.st_gid = from_kgid_munged(current_user_ns(), stat->gid); + tmp.st_atime = stat->atime.tv_sec; + tmp.st_atime_nsec = stat->atime.tv_nsec; + tmp.st_mtime = stat->mtime.tv_sec; + tmp.st_mtime_nsec = stat->mtime.tv_nsec; + tmp.st_ctime = stat->ctime.tv_sec; + tmp.st_ctime_nsec = stat->ctime.tv_nsec; + tmp.st_size = stat->size; + tmp.st_blocks = stat->blocks; + tmp.st_blksize = stat->blksize; + return copy_to_user(statbuf,&tmp,sizeof(tmp)) ? -EFAULT : 0; +} + +SYSCALL_DEFINE2(stat64, const char __user *, filename, + struct stat64 __user *, statbuf) +{ + struct kstat stat; + int error = vfs_stat(filename, &stat); + + if (!error) + error = cp_new_stat64(&stat, statbuf); + + return error; +} + +SYSCALL_DEFINE2(lstat64, const char __user *, filename, + struct stat64 __user *, statbuf) +{ + struct kstat stat; + int error = vfs_lstat(filename, &stat); + + if (!error) + error = cp_new_stat64(&stat, statbuf); + + return error; +} + +SYSCALL_DEFINE2(fstat64, unsigned long, fd, struct stat64 __user *, statbuf) +{ + struct kstat stat; + int error = vfs_fstat(fd, &stat); + + if (!error) + error = cp_new_stat64(&stat, statbuf); + + return error; +} + +SYSCALL_DEFINE4(fstatat64, int, dfd, const char __user *, filename, + struct stat64 __user *, statbuf, int, flag) +{ + struct kstat stat; + int error; + + error = vfs_fstatat(dfd, filename, &stat, flag); + if (error) + return error; + return cp_new_stat64(&stat, statbuf); +} +#endif /* __ARCH_WANT_STAT64 || __ARCH_WANT_COMPAT_STAT64 */ + +static noinline_for_stack int +cp_statx(const struct kstat *stat, struct statx __user *buffer) +{ + struct statx tmp; + + memset(&tmp, 0, sizeof(tmp)); + + /* STATX_CHANGE_COOKIE is kernel-only for now */ + tmp.stx_mask = stat->result_mask & ~STATX_CHANGE_COOKIE; + tmp.stx_blksize = stat->blksize; + /* STATX_ATTR_CHANGE_MONOTONIC is kernel-only for now */ + tmp.stx_attributes = stat->attributes & ~STATX_ATTR_CHANGE_MONOTONIC; + tmp.stx_nlink = stat->nlink; + tmp.stx_uid = from_kuid_munged(current_user_ns(), stat->uid); + tmp.stx_gid = from_kgid_munged(current_user_ns(), stat->gid); + tmp.stx_mode = stat->mode; + tmp.stx_ino = stat->ino; + tmp.stx_size = stat->size; + tmp.stx_blocks = stat->blocks; + tmp.stx_attributes_mask = stat->attributes_mask; + tmp.stx_atime.tv_sec = stat->atime.tv_sec; + tmp.stx_atime.tv_nsec = stat->atime.tv_nsec; + tmp.stx_btime.tv_sec = stat->btime.tv_sec; + tmp.stx_btime.tv_nsec = stat->btime.tv_nsec; + tmp.stx_ctime.tv_sec = stat->ctime.tv_sec; + tmp.stx_ctime.tv_nsec = stat->ctime.tv_nsec; + tmp.stx_mtime.tv_sec = stat->mtime.tv_sec; + tmp.stx_mtime.tv_nsec = stat->mtime.tv_nsec; + tmp.stx_rdev_major = MAJOR(stat->rdev); + tmp.stx_rdev_minor = MINOR(stat->rdev); + tmp.stx_dev_major = MAJOR(stat->dev); + tmp.stx_dev_minor = MINOR(stat->dev); + tmp.stx_mnt_id = stat->mnt_id; + tmp.stx_dio_mem_align = stat->dio_mem_align; + tmp.stx_dio_offset_align = stat->dio_offset_align; + tmp.stx_dio_read_offset_align = stat->dio_read_offset_align; + tmp.stx_subvol = stat->subvol; + tmp.stx_atomic_write_unit_min = stat->atomic_write_unit_min; + tmp.stx_atomic_write_unit_max = stat->atomic_write_unit_max; + tmp.stx_atomic_write_segments_max = stat->atomic_write_segments_max; + tmp.stx_atomic_write_unit_max_opt = stat->atomic_write_unit_max_opt; + + return copy_to_user(buffer, &tmp, sizeof(tmp)) ? -EFAULT : 0; +} + +int do_statx(int dfd, struct filename *filename, unsigned int flags, + unsigned int mask, struct statx __user *buffer) +{ + struct kstat stat; + int error; + + if (mask & STATX__RESERVED) + return -EINVAL; + if ((flags & AT_STATX_SYNC_TYPE) == AT_STATX_SYNC_TYPE) + return -EINVAL; + + /* + * STATX_CHANGE_COOKIE is kernel-only for now. Ignore requests + * from userland. + */ + mask &= ~STATX_CHANGE_COOKIE; + + error = vfs_statx(dfd, filename, flags, &stat, mask); + if (error) + return error; + + return cp_statx(&stat, buffer); +} + +int do_statx_fd(int fd, unsigned int flags, unsigned int mask, + struct statx __user *buffer) +{ + struct kstat stat; + int error; + + if (mask & STATX__RESERVED) + return -EINVAL; + if ((flags & AT_STATX_SYNC_TYPE) == AT_STATX_SYNC_TYPE) + return -EINVAL; + + /* + * STATX_CHANGE_COOKIE is kernel-only for now. Ignore requests + * from userland. + */ + mask &= ~STATX_CHANGE_COOKIE; + + error = vfs_statx_fd(fd, flags, &stat, mask); + if (error) + return error; + + return cp_statx(&stat, buffer); +} + +/** + * sys_statx - System call to get enhanced stats + * @dfd: Base directory to pathwalk from *or* fd to stat. + * @filename: File to stat or either NULL or "" with AT_EMPTY_PATH + * @flags: AT_* flags to control pathwalk. + * @mask: Parts of statx struct actually required. + * @buffer: Result buffer. + * + * Note that fstat() can be emulated by setting dfd to the fd of interest, + * supplying "" (or preferably NULL) as the filename and setting AT_EMPTY_PATH + * in the flags. + */ +SYSCALL_DEFINE5(statx, + int, dfd, const char __user *, filename, unsigned, flags, + unsigned int, mask, + struct statx __user *, buffer) +{ + CLASS(filename_maybe_null, name)(filename, flags); + + if (!name && dfd >= 0) + return do_statx_fd(dfd, flags & ~AT_NO_AUTOMOUNT, mask, buffer); + + return do_statx(dfd, name, flags, mask, buffer); +} + +#if defined(CONFIG_COMPAT) && defined(__ARCH_WANT_COMPAT_STAT) +static int cp_compat_stat(struct kstat *stat, struct compat_stat __user *ubuf) +{ + struct compat_stat tmp; + + if (sizeof(tmp.st_dev) < 4 && !old_valid_dev(stat->dev)) + return -EOVERFLOW; + if (sizeof(tmp.st_rdev) < 4 && !old_valid_dev(stat->rdev)) + return -EOVERFLOW; + + memset(&tmp, 0, sizeof(tmp)); + tmp.st_dev = new_encode_dev(stat->dev); + tmp.st_ino = stat->ino; + if (sizeof(tmp.st_ino) < sizeof(stat->ino) && tmp.st_ino != stat->ino) + return -EOVERFLOW; + tmp.st_mode = stat->mode; + tmp.st_nlink = stat->nlink; + if (tmp.st_nlink != stat->nlink) + return -EOVERFLOW; + SET_UID(tmp.st_uid, from_kuid_munged(current_user_ns(), stat->uid)); + SET_GID(tmp.st_gid, from_kgid_munged(current_user_ns(), stat->gid)); + tmp.st_rdev = new_encode_dev(stat->rdev); + if ((u64) stat->size > MAX_NON_LFS) + return -EOVERFLOW; + tmp.st_size = stat->size; + tmp.st_atime = stat->atime.tv_sec; + tmp.st_atime_nsec = stat->atime.tv_nsec; + tmp.st_mtime = stat->mtime.tv_sec; + tmp.st_mtime_nsec = stat->mtime.tv_nsec; + tmp.st_ctime = stat->ctime.tv_sec; + tmp.st_ctime_nsec = stat->ctime.tv_nsec; + tmp.st_blocks = stat->blocks; + tmp.st_blksize = stat->blksize; + return copy_to_user(ubuf, &tmp, sizeof(tmp)) ? -EFAULT : 0; +} + +COMPAT_SYSCALL_DEFINE2(newstat, const char __user *, filename, + struct compat_stat __user *, statbuf) +{ + struct kstat stat; + int error; + + error = vfs_stat(filename, &stat); + if (error) + return error; + return cp_compat_stat(&stat, statbuf); +} + +COMPAT_SYSCALL_DEFINE2(newlstat, const char __user *, filename, + struct compat_stat __user *, statbuf) +{ + struct kstat stat; + int error; + + error = vfs_lstat(filename, &stat); + if (error) + return error; + return cp_compat_stat(&stat, statbuf); +} + +#ifndef __ARCH_WANT_STAT64 +COMPAT_SYSCALL_DEFINE4(newfstatat, unsigned int, dfd, + const char __user *, filename, + struct compat_stat __user *, statbuf, int, flag) +{ + struct kstat stat; + int error; + + error = vfs_fstatat(dfd, filename, &stat, flag); + if (error) + return error; + return cp_compat_stat(&stat, statbuf); +} +#endif + +COMPAT_SYSCALL_DEFINE2(newfstat, unsigned int, fd, + struct compat_stat __user *, statbuf) +{ + struct kstat stat; + int error = vfs_fstat(fd, &stat); + + if (!error) + error = cp_compat_stat(&stat, statbuf); + return error; +} +#endif + +/* Caller is here responsible for sufficient locking (ie. inode->i_lock) */ +void __inode_add_bytes(struct inode *inode, loff_t bytes) +{ + inode->i_blocks += bytes >> 9; + bytes &= 511; + inode->i_bytes += bytes; + if (inode->i_bytes >= 512) { + inode->i_blocks++; + inode->i_bytes -= 512; + } +} +EXPORT_SYMBOL(__inode_add_bytes); + +void inode_add_bytes(struct inode *inode, loff_t bytes) +{ + spin_lock(&inode->i_lock); + __inode_add_bytes(inode, bytes); + spin_unlock(&inode->i_lock); +} + +EXPORT_SYMBOL(inode_add_bytes); + +void __inode_sub_bytes(struct inode *inode, loff_t bytes) +{ + inode->i_blocks -= bytes >> 9; + bytes &= 511; + if (inode->i_bytes < bytes) { + inode->i_blocks--; + inode->i_bytes += 512; + } + inode->i_bytes -= bytes; +} + +EXPORT_SYMBOL(__inode_sub_bytes); + +void inode_sub_bytes(struct inode *inode, loff_t bytes) +{ + spin_lock(&inode->i_lock); + __inode_sub_bytes(inode, bytes); + spin_unlock(&inode->i_lock); +} + +EXPORT_SYMBOL(inode_sub_bytes); + +loff_t inode_get_bytes(struct inode *inode) +{ + loff_t ret; + + spin_lock(&inode->i_lock); + ret = __inode_get_bytes(inode); + spin_unlock(&inode->i_lock); + return ret; +} + +EXPORT_SYMBOL(inode_get_bytes); + +void inode_set_bytes(struct inode *inode, loff_t bytes) +{ + /* Caller is here responsible for sufficient locking + * (ie. inode->i_lock) */ + inode->i_blocks = bytes >> 9; + inode->i_bytes = bytes & 511; +} + +EXPORT_SYMBOL(inode_set_bytes); diff --git a/fs/statfs.c b/fs/statfs.c new file mode 100644 index 000000000..377bcef7a --- /dev/null +++ b/fs/statfs.c @@ -0,0 +1,405 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +static int flags_by_mnt(int mnt_flags) +{ + int flags = 0; + + if (mnt_flags & MNT_READONLY) + flags |= ST_RDONLY; + if (mnt_flags & MNT_NOSUID) + flags |= ST_NOSUID; + if (mnt_flags & MNT_NODEV) + flags |= ST_NODEV; + if (mnt_flags & MNT_NOEXEC) + flags |= ST_NOEXEC; + if (mnt_flags & MNT_NOATIME) + flags |= ST_NOATIME; + if (mnt_flags & MNT_NODIRATIME) + flags |= ST_NODIRATIME; + if (mnt_flags & MNT_RELATIME) + flags |= ST_RELATIME; + if (mnt_flags & MNT_NOSYMFOLLOW) + flags |= ST_NOSYMFOLLOW; + return flags; +} + +static int flags_by_sb(int s_flags) +{ + int flags = 0; + if (s_flags & SB_SYNCHRONOUS) + flags |= ST_SYNCHRONOUS; + if (s_flags & SB_MANDLOCK) + flags |= ST_MANDLOCK; + if (s_flags & SB_RDONLY) + flags |= ST_RDONLY; + return flags; +} + +static int calculate_f_flags(struct vfsmount *mnt) +{ + return ST_VALID | flags_by_mnt(mnt->mnt_flags) | + flags_by_sb(mnt->mnt_sb->s_flags); +} + +static int statfs_by_dentry(struct dentry *dentry, struct kstatfs *buf) +{ + int retval; + + if (!dentry->d_sb->s_op->statfs) + return -ENOSYS; + + memset(buf, 0, sizeof(*buf)); + retval = security_sb_statfs(dentry); + if (retval) + return retval; + retval = dentry->d_sb->s_op->statfs(dentry, buf); + if (retval == 0 && buf->f_frsize == 0) + buf->f_frsize = buf->f_bsize; + return retval; +} + +int vfs_get_fsid(struct dentry *dentry, __kernel_fsid_t *fsid) +{ + struct kstatfs st; + int error; + + error = statfs_by_dentry(dentry, &st); + if (error) + return error; + + *fsid = st.f_fsid; + return 0; +} +EXPORT_SYMBOL(vfs_get_fsid); + +int vfs_statfs(const struct path *path, struct kstatfs *buf) +{ + int error; + + error = statfs_by_dentry(path->dentry, buf); + if (!error) + buf->f_flags = calculate_f_flags(path->mnt); + return error; +} +EXPORT_SYMBOL(vfs_statfs); + +int user_statfs(const char __user *pathname, struct kstatfs *st) +{ + struct path path; + int error; + unsigned int lookup_flags = LOOKUP_FOLLOW|LOOKUP_AUTOMOUNT; + CLASS(filename, name)(pathname); +retry: + error = filename_lookup(AT_FDCWD, name, lookup_flags, &path, NULL); + if (!error) { + error = vfs_statfs(&path, st); + path_put(&path); + if (retry_estale(error, lookup_flags)) { + lookup_flags |= LOOKUP_REVAL; + goto retry; + } + } + return error; +} + +int fd_statfs(int fd, struct kstatfs *st) +{ + CLASS(fd_raw, f)(fd); + + if (fd_empty(f)) + return -EBADF; + return vfs_statfs(&fd_file(f)->f_path, st); +} + +static int do_statfs_native(struct kstatfs *st, struct statfs __user *p) +{ + struct statfs buf; + + if (sizeof(buf) == sizeof(*st)) + memcpy(&buf, st, sizeof(*st)); + else { + memset(&buf, 0, sizeof(buf)); + if (sizeof buf.f_blocks == 4) { + if ((st->f_blocks | st->f_bfree | st->f_bavail | + st->f_bsize | st->f_frsize) & + 0xffffffff00000000ULL) + return -EOVERFLOW; + /* + * f_files and f_ffree may be -1; it's okay to stuff + * that into 32 bits + */ + if (st->f_files != -1 && + (st->f_files & 0xffffffff00000000ULL)) + return -EOVERFLOW; + if (st->f_ffree != -1 && + (st->f_ffree & 0xffffffff00000000ULL)) + return -EOVERFLOW; + } + + buf.f_type = st->f_type; + buf.f_bsize = st->f_bsize; + buf.f_blocks = st->f_blocks; + buf.f_bfree = st->f_bfree; + buf.f_bavail = st->f_bavail; + buf.f_files = st->f_files; + buf.f_ffree = st->f_ffree; + buf.f_fsid = st->f_fsid; + buf.f_namelen = st->f_namelen; + buf.f_frsize = st->f_frsize; + buf.f_flags = st->f_flags; + } + if (copy_to_user(p, &buf, sizeof(buf))) + return -EFAULT; + return 0; +} + +static int do_statfs64(struct kstatfs *st, struct statfs64 __user *p) +{ + struct statfs64 buf; + if (sizeof(buf) == sizeof(*st)) + memcpy(&buf, st, sizeof(*st)); + else { + memset(&buf, 0, sizeof(buf)); + buf.f_type = st->f_type; + buf.f_bsize = st->f_bsize; + buf.f_blocks = st->f_blocks; + buf.f_bfree = st->f_bfree; + buf.f_bavail = st->f_bavail; + buf.f_files = st->f_files; + buf.f_ffree = st->f_ffree; + buf.f_fsid = st->f_fsid; + buf.f_namelen = st->f_namelen; + buf.f_frsize = st->f_frsize; + buf.f_flags = st->f_flags; + } + if (copy_to_user(p, &buf, sizeof(buf))) + return -EFAULT; + return 0; +} + +SYSCALL_DEFINE2(statfs, const char __user *, pathname, struct statfs __user *, buf) +{ + struct kstatfs st; + int error = user_statfs(pathname, &st); + if (!error) + error = do_statfs_native(&st, buf); + return error; +} + +SYSCALL_DEFINE3(statfs64, const char __user *, pathname, size_t, sz, struct statfs64 __user *, buf) +{ + struct kstatfs st; + int error; + if (sz != sizeof(*buf)) + return -EINVAL; + error = user_statfs(pathname, &st); + if (!error) + error = do_statfs64(&st, buf); + return error; +} + +SYSCALL_DEFINE2(fstatfs, unsigned int, fd, struct statfs __user *, buf) +{ + struct kstatfs st; + int error = fd_statfs(fd, &st); + if (!error) + error = do_statfs_native(&st, buf); + return error; +} + +SYSCALL_DEFINE3(fstatfs64, unsigned int, fd, size_t, sz, struct statfs64 __user *, buf) +{ + struct kstatfs st; + int error; + + if (sz != sizeof(*buf)) + return -EINVAL; + + error = fd_statfs(fd, &st); + if (!error) + error = do_statfs64(&st, buf); + return error; +} + +static int vfs_ustat(dev_t dev, struct kstatfs *sbuf) +{ + struct super_block *s = user_get_super(dev, false); + int err; + if (!s) + return -EINVAL; + + err = statfs_by_dentry(s->s_root, sbuf); + drop_super(s); + return err; +} + +SYSCALL_DEFINE2(ustat, unsigned, dev, struct ustat __user *, ubuf) +{ + struct ustat tmp; + struct kstatfs sbuf; + int err = vfs_ustat(new_decode_dev(dev), &sbuf); + if (err) + return err; + + memset(&tmp,0,sizeof(struct ustat)); + tmp.f_tfree = sbuf.f_bfree; + if (IS_ENABLED(CONFIG_ARCH_32BIT_USTAT_F_TINODE)) + tmp.f_tinode = min_t(u64, sbuf.f_ffree, UINT_MAX); + else + tmp.f_tinode = sbuf.f_ffree; + + return copy_to_user(ubuf, &tmp, sizeof(struct ustat)) ? -EFAULT : 0; +} + +#ifdef CONFIG_COMPAT +static int put_compat_statfs(struct compat_statfs __user *ubuf, struct kstatfs *kbuf) +{ + struct compat_statfs buf; + if (sizeof ubuf->f_blocks == 4) { + if ((kbuf->f_blocks | kbuf->f_bfree | kbuf->f_bavail | + kbuf->f_bsize | kbuf->f_frsize) & 0xffffffff00000000ULL) + return -EOVERFLOW; + /* f_files and f_ffree may be -1; it's okay + * to stuff that into 32 bits */ + if (kbuf->f_files != 0xffffffffffffffffULL + && (kbuf->f_files & 0xffffffff00000000ULL)) + return -EOVERFLOW; + if (kbuf->f_ffree != 0xffffffffffffffffULL + && (kbuf->f_ffree & 0xffffffff00000000ULL)) + return -EOVERFLOW; + } + memset(&buf, 0, sizeof(struct compat_statfs)); + buf.f_type = kbuf->f_type; + buf.f_bsize = kbuf->f_bsize; + buf.f_blocks = kbuf->f_blocks; + buf.f_bfree = kbuf->f_bfree; + buf.f_bavail = kbuf->f_bavail; + buf.f_files = kbuf->f_files; + buf.f_ffree = kbuf->f_ffree; + buf.f_namelen = kbuf->f_namelen; + buf.f_fsid.val[0] = kbuf->f_fsid.val[0]; + buf.f_fsid.val[1] = kbuf->f_fsid.val[1]; + buf.f_frsize = kbuf->f_frsize; + buf.f_flags = kbuf->f_flags; + if (copy_to_user(ubuf, &buf, sizeof(struct compat_statfs))) + return -EFAULT; + return 0; +} + +/* + * The following statfs calls are copies of code from fs/statfs.c and + * should be checked against those from time to time + */ +COMPAT_SYSCALL_DEFINE2(statfs, const char __user *, pathname, struct compat_statfs __user *, buf) +{ + struct kstatfs tmp; + int error = user_statfs(pathname, &tmp); + if (!error) + error = put_compat_statfs(buf, &tmp); + return error; +} + +COMPAT_SYSCALL_DEFINE2(fstatfs, unsigned int, fd, struct compat_statfs __user *, buf) +{ + struct kstatfs tmp; + int error = fd_statfs(fd, &tmp); + if (!error) + error = put_compat_statfs(buf, &tmp); + return error; +} + +static int put_compat_statfs64(struct compat_statfs64 __user *ubuf, struct kstatfs *kbuf) +{ + struct compat_statfs64 buf; + + if ((kbuf->f_bsize | kbuf->f_frsize) & 0xffffffff00000000ULL) + return -EOVERFLOW; + + memset(&buf, 0, sizeof(struct compat_statfs64)); + buf.f_type = kbuf->f_type; + buf.f_bsize = kbuf->f_bsize; + buf.f_blocks = kbuf->f_blocks; + buf.f_bfree = kbuf->f_bfree; + buf.f_bavail = kbuf->f_bavail; + buf.f_files = kbuf->f_files; + buf.f_ffree = kbuf->f_ffree; + buf.f_namelen = kbuf->f_namelen; + buf.f_fsid.val[0] = kbuf->f_fsid.val[0]; + buf.f_fsid.val[1] = kbuf->f_fsid.val[1]; + buf.f_frsize = kbuf->f_frsize; + buf.f_flags = kbuf->f_flags; + if (copy_to_user(ubuf, &buf, sizeof(struct compat_statfs64))) + return -EFAULT; + return 0; +} + +int kcompat_sys_statfs64(const char __user * pathname, compat_size_t sz, struct compat_statfs64 __user * buf) +{ + struct kstatfs tmp; + int error; + + if (sz != sizeof(*buf)) + return -EINVAL; + + error = user_statfs(pathname, &tmp); + if (!error) + error = put_compat_statfs64(buf, &tmp); + return error; +} + +COMPAT_SYSCALL_DEFINE3(statfs64, const char __user *, pathname, compat_size_t, sz, struct compat_statfs64 __user *, buf) +{ + return kcompat_sys_statfs64(pathname, sz, buf); +} + +int kcompat_sys_fstatfs64(unsigned int fd, compat_size_t sz, struct compat_statfs64 __user * buf) +{ + struct kstatfs tmp; + int error; + + if (sz != sizeof(*buf)) + return -EINVAL; + + error = fd_statfs(fd, &tmp); + if (!error) + error = put_compat_statfs64(buf, &tmp); + return error; +} + +COMPAT_SYSCALL_DEFINE3(fstatfs64, unsigned int, fd, compat_size_t, sz, struct compat_statfs64 __user *, buf) +{ + return kcompat_sys_fstatfs64(fd, sz, buf); +} + +/* + * This is a copy of sys_ustat, just dealing with a structure layout. + * Given how simple this syscall is that apporach is more maintainable + * than the various conversion hacks. + */ +COMPAT_SYSCALL_DEFINE2(ustat, unsigned, dev, struct compat_ustat __user *, u) +{ + struct compat_ustat tmp; + struct kstatfs sbuf; + int err = vfs_ustat(new_decode_dev(dev), &sbuf); + if (err) + return err; + + memset(&tmp, 0, sizeof(struct compat_ustat)); + tmp.f_tfree = sbuf.f_bfree; + tmp.f_tinode = sbuf.f_ffree; + if (copy_to_user(u, &tmp, sizeof(struct compat_ustat))) + return -EFAULT; + return 0; +} +#endif diff --git a/fs/super.c b/fs/super.c new file mode 100644 index 000000000..05e443173 --- /dev/null +++ b/fs/super.c @@ -0,0 +1,2465 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/super.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * super.c contains code to handle: - mount structures + * - super-block tables + * - filesystem drivers list + * - mount system call + * - umount system call + * - ustat system call + * + * GK 2/5/95 - Changed to support mounting the root fs via NFS + * + * Added kerneld support: Jacques Gelinas and Bjorn Ekwall + * Added change_root: Werner Almesberger & Hans Lermen, Feb '96 + * Added options to /proc/mounts: + * Torbjörn Lindh (torbjorn.lindh@gopta.se), April 14, 1996. + * Added devfs support: Richard Gooch , 13-JAN-1998 + * Heavily rewritten for 'one fs - one tree' dcache architecture. AV, Mar 2000 + */ + +#include +#include +#include +#include +#include +#include +#include +#include /* for the emergency remount stuff */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +static int thaw_super_locked(struct super_block *sb, enum freeze_holder who, + const void *freeze_owner); + +static LIST_HEAD(super_blocks); +static DEFINE_SPINLOCK(sb_lock); + +static char *sb_writers_name[SB_FREEZE_LEVELS] = { + "sb_writers", + "sb_pagefaults", + "sb_internal", +}; + +static inline void __super_lock(struct super_block *sb, bool excl) +{ + if (excl) + down_write(&sb->s_umount); + else + down_read(&sb->s_umount); +} + +static inline void super_unlock(struct super_block *sb, bool excl) +{ + if (excl) + up_write(&sb->s_umount); + else + up_read(&sb->s_umount); +} + +static inline void __super_lock_excl(struct super_block *sb) +{ + __super_lock(sb, true); +} + +static inline void super_unlock_excl(struct super_block *sb) +{ + super_unlock(sb, true); +} + +static inline void super_unlock_shared(struct super_block *sb) +{ + super_unlock(sb, false); +} + +static bool super_flags(const struct super_block *sb, unsigned int flags) +{ + /* + * Pairs with smp_store_release() in super_wake() and ensures + * that we see @flags after we're woken. + */ + return smp_load_acquire(&sb->s_flags) & flags; +} + +/** + * super_lock - wait for superblock to become ready and lock it + * @sb: superblock to wait for + * @excl: whether exclusive access is required + * + * If the superblock has neither passed through vfs_get_tree() or + * generic_shutdown_super() yet wait for it to happen. Either superblock + * creation will succeed and SB_BORN is set by vfs_get_tree() or we're + * woken and we'll see SB_DYING. + * + * The caller must have acquired a temporary reference on @sb->s_passive. + * + * Return: The function returns true if SB_BORN was set and with + * s_umount held. The function returns false if SB_DYING was + * set and without s_umount held. + */ +static __must_check bool super_lock(struct super_block *sb, bool excl) +{ + lockdep_assert_not_held(&sb->s_umount); + + /* wait until the superblock is ready or dying */ + wait_var_event(&sb->s_flags, super_flags(sb, SB_BORN | SB_DYING)); + + /* Don't pointlessly acquire s_umount. */ + if (super_flags(sb, SB_DYING)) + return false; + + __super_lock(sb, excl); + + /* + * Has gone through generic_shutdown_super() in the meantime. + * @sb->s_root is NULL and @sb->s_active is 0. No one needs to + * grab a reference to this. Tell them so. + */ + if (sb->s_flags & SB_DYING) { + super_unlock(sb, excl); + return false; + } + + WARN_ON_ONCE(!(sb->s_flags & SB_BORN)); + return true; +} + +/* wait and try to acquire read-side of @sb->s_umount */ +static inline bool super_lock_shared(struct super_block *sb) +{ + return super_lock(sb, false); +} + +/* wait and try to acquire write-side of @sb->s_umount */ +static inline bool super_lock_excl(struct super_block *sb) +{ + return super_lock(sb, true); +} + +/* wake waiters */ +#define SUPER_WAKE_FLAGS (SB_BORN | SB_DYING | SB_DEAD) +static void super_wake(struct super_block *sb, unsigned int flag) +{ + WARN_ON_ONCE((flag & ~SUPER_WAKE_FLAGS)); + WARN_ON_ONCE(hweight32(flag & SUPER_WAKE_FLAGS) > 1); + + /* + * Pairs with smp_load_acquire() in super_lock() to make sure + * all initializations in the superblock are seen by the user + * seeing SB_BORN sent. + */ + smp_store_release(&sb->s_flags, sb->s_flags | flag); + /* + * Pairs with the barrier in prepare_to_wait_event() to make sure + * ___wait_var_event() either sees SB_BORN set or + * waitqueue_active() check in wake_up_var() sees the waiter. + */ + smp_mb(); + wake_up_var(&sb->s_flags); +} + +/* + * The s_op->nr_cached_objects hooks (used for example by btrfs and xfs) + * operate on filesystem-global state and ignore sc->memcg. Driving them + * from per-memcg shrink_slab_memcg() invocations only burns CPU walking + * per-cpu counters and queueing duplicate work: the actual reclaim happens on + * the global path (kswapd or root direct reclaim) regardless. Restrict them + * to that path. + */ +static inline bool super_fs_objects_eligible(struct shrink_control *sc) +{ + return !sc->memcg || mem_cgroup_is_root(sc->memcg); +} + +/* + * One thing we have to be careful of with a per-sb shrinker is that we don't + * drop the last active reference to the superblock from within the shrinker. + * If that happens we could trigger unregistering the shrinker from within the + * shrinker path and that leads to deadlock on the shrinker_mutex. Hence we + * take a passive reference to the superblock to avoid this from occurring. + */ +static unsigned long super_cache_scan(struct shrinker *shrink, + struct shrink_control *sc) +{ + struct super_block *sb; + long fs_objects = 0; + long total_objects; + long freed = 0; + long dentries; + long inodes; + + sb = shrink->private_data; + + /* + * Deadlock avoidance. We may hold various FS locks, and we don't want + * to recurse into the FS that called us in clear_inode() and friends.. + */ + if (!(sc->gfp_mask & __GFP_FS)) + return SHRINK_STOP; + + if (!super_trylock_shared(sb)) + return SHRINK_STOP; + + if (sb->s_op->nr_cached_objects && super_fs_objects_eligible(sc)) + fs_objects = sb->s_op->nr_cached_objects(sb, sc); + + inodes = list_lru_shrink_count(&sb->s_inode_lru, sc); + dentries = list_lru_shrink_count(&sb->s_dentry_lru, sc); + total_objects = dentries + inodes + fs_objects; + if (!total_objects) + total_objects = 1; + + /* proportion the scan between the caches */ + dentries = mult_frac(sc->nr_to_scan, dentries, total_objects); + inodes = mult_frac(sc->nr_to_scan, inodes, total_objects); + fs_objects = mult_frac(sc->nr_to_scan, fs_objects, total_objects); + + /* + * prune the dcache first as the icache is pinned by it, then + * prune the icache, followed by the filesystem specific caches + * + * Ensure that we always scan at least one object - memcg kmem + * accounting uses this to fully empty the caches. + */ + sc->nr_to_scan = dentries + 1; + freed = prune_dcache_sb(sb, sc); + sc->nr_to_scan = inodes + 1; + freed += prune_icache_sb(sb, sc); + + if (fs_objects) { + sc->nr_to_scan = fs_objects + 1; + freed += sb->s_op->free_cached_objects(sb, sc); + } + + super_unlock_shared(sb); + return freed; +} + +static unsigned long super_cache_count(struct shrinker *shrink, + struct shrink_control *sc) +{ + struct super_block *sb; + long total_objects = 0; + + sb = shrink->private_data; + + /* + * We don't call super_trylock_shared() here as it is a scalability + * bottleneck, so we're exposed to partial setup state. The shrinker + * rwsem does not protect filesystem operations backing + * list_lru_shrink_count() or s_op->nr_cached_objects(). Counts can + * change between super_cache_count and super_cache_scan, so we really + * don't need locks here. + * + * However, if we are currently mounting the superblock, the underlying + * filesystem might be in a state of partial construction and hence it + * is dangerous to access it. super_trylock_shared() uses a SB_BORN check + * to avoid this situation, so do the same here. The memory barrier is + * matched with the one in mount_fs() as we don't hold locks here. + */ + if (!(sb->s_flags & SB_BORN)) + return 0; + smp_rmb(); + + if (sb->s_op && sb->s_op->nr_cached_objects && + super_fs_objects_eligible(sc)) + total_objects = sb->s_op->nr_cached_objects(sb, sc); + + total_objects += list_lru_shrink_count(&sb->s_dentry_lru, sc); + total_objects += list_lru_shrink_count(&sb->s_inode_lru, sc); + + if (!total_objects) + return SHRINK_EMPTY; + + total_objects = vfs_pressure_ratio(total_objects); + return total_objects; +} + +static struct super_dev *super_dev_alloc(dev_t dev, struct super_block *sb); + +static void destroy_super_work(struct work_struct *work) +{ + struct super_block *s = container_of(work, struct super_block, + destroy_work); + fsnotify_sb_free(s); + security_sb_free(s); + put_user_ns(s->s_user_ns); + /* Only an unregistered entry is still owned by the superblock. */ + kfree(s->s_super_dev); + kfree(s->s_subtype); + for (int i = 0; i < SB_FREEZE_LEVELS; i++) + percpu_free_rwsem(&s->s_writers.rw_sem[i]); + kfree(s); +} + +static void destroy_super_rcu(struct rcu_head *head) +{ + struct super_block *s = container_of(head, struct super_block, rcu); + INIT_WORK(&s->destroy_work, destroy_super_work); + schedule_work(&s->destroy_work); +} + +/* Free a superblock that has never been seen by anyone */ +static void destroy_unused_super(struct super_block *s) +{ + if (!s) + return; + super_unlock_excl(s); + list_lru_destroy(&s->s_dentry_lru); + list_lru_destroy(&s->s_inode_lru); + shrinker_free(s->s_shrink); + /* no delays needed */ + destroy_super_work(&s->destroy_work); +} + +/** + * alloc_super - create new superblock + * @type: filesystem type superblock should belong to + * @flags: the mount flags + * @user_ns: User namespace for the super_block + * + * Allocates and initializes a new &struct super_block. alloc_super() + * returns a pointer new superblock or %NULL if allocation had failed. + */ +static struct super_block *alloc_super(struct file_system_type *type, int flags, + struct user_namespace *user_ns) +{ + struct super_block *s = kzalloc_obj(struct super_block); + static const struct super_operations default_op; + int i; + + if (!s) + return NULL; + + s->s_user_ns = get_user_ns(user_ns); + init_rwsem(&s->s_umount); + lockdep_set_class(&s->s_umount, &type->s_umount_key); + /* + * sget_fc() can have s_umount recursion. + * + * When it cannot find a suitable sb, it allocates a new + * one (this one), and tries again to find a suitable old + * one. + * + * In case that succeeds, it will acquire the s_umount + * lock of the old one. Since these are clearly distrinct + * locks, and this object isn't exposed yet, there's no + * risk of deadlocks. + * + * Annotate this by putting this lock in a different + * subclass. + */ + down_write_nested(&s->s_umount, SINGLE_DEPTH_NESTING); + + if (security_sb_alloc(s)) + goto fail; + + for (i = 0; i < SB_FREEZE_LEVELS; i++) { + if (__percpu_init_rwsem(&s->s_writers.rw_sem[i], + sb_writers_name[i], + &type->s_writers_key[i])) + goto fail; + } + s->s_bdi = &noop_backing_dev_info; + s->s_flags = flags; + if (s->s_user_ns != &init_user_ns) + s->s_iflags |= SB_I_NODEV; + INIT_HLIST_NODE(&s->s_instances); + INIT_HLIST_BL_HEAD(&s->s_roots); + spin_lock_init(&s->s_roots_lock); + mutex_init(&s->s_sync_lock); + INIT_LIST_HEAD(&s->s_inodes); + spin_lock_init(&s->s_inode_list_lock); + INIT_LIST_HEAD(&s->s_inodes_wb); + spin_lock_init(&s->s_inode_wblist_lock); + fserror_mount(s); + + refcount_set(&s->s_passive, 1); + atomic_set(&s->s_active, 1); + mutex_init(&s->s_vfs_rename_mutex); + lockdep_set_class(&s->s_vfs_rename_mutex, &type->s_vfs_rename_key); + init_rwsem(&s->s_dquot.dqio_sem); + s->s_maxbytes = MAX_NON_LFS; + s->s_op = &default_op; + s->s_time_gran = 1000000000; + s->s_time_min = TIME64_MIN; + s->s_time_max = TIME64_MAX; + + s->s_shrink = shrinker_alloc(SHRINKER_NUMA_AWARE | SHRINKER_MEMCG_AWARE, + "sb-%s", type->name); + if (!s->s_shrink) + goto fail; + + s->s_shrink->scan_objects = super_cache_scan; + s->s_shrink->count_objects = super_cache_count; + s->s_shrink->batch = 1024; + s->s_shrink->private_data = s; + + if (list_lru_init_memcg(&s->s_dentry_lru, s->s_shrink)) + goto fail; + if (list_lru_init_memcg(&s->s_inode_lru, s->s_shrink)) + goto fail; + s->s_super_dev = super_dev_alloc(0, s); + if (!s->s_super_dev) + goto fail; + + s->s_min_writeback_pages = MIN_WRITEBACK_PAGES; + return s; + +fail: + destroy_unused_super(s); + return NULL; +} + +/* Superblock refcounting */ + +/* + * Drop a superblock's passive reference. Must be called WITHOUT sb_lock held; + * put_super() acquires sb_lock itself when the final reference is dropped. + */ +void put_super(struct super_block *s) +{ + if (refcount_dec_and_test(&s->s_passive)) { + + spin_lock(&sb_lock); + list_del_init(&s->s_list); + spin_unlock(&sb_lock); + + WARN_ON(s->s_dentry_lru.node); + WARN_ON(s->s_inode_lru.node); + WARN_ON(s->s_mounts); + call_rcu(&s->rcu, destroy_super_rcu); + } +} + +struct super_dev { + dev_t sd_dev; + struct super_block *sd_sb; + refcount_t sd_ref; + struct rhlist_head sd_node; + struct rcu_head sd_rcu; +}; + +static struct rhltable super_dev_table; +static const struct rhashtable_params super_dev_params = { + .key_len = sizeof(dev_t), + .key_offset = offsetof(struct super_dev, sd_dev), + .head_offset = offsetof(struct super_dev, sd_node), +}; + +static struct super_dev *super_dev_alloc(dev_t dev, struct super_block *sb) +{ + struct super_dev *fsd; + + fsd = kzalloc_obj(*fsd); + if (!fsd) + return NULL; + fsd->sd_dev = dev; + fsd->sd_sb = sb; + refcount_set(&fsd->sd_ref, 1); + return fsd; +} + +static void super_dev_put(struct super_dev *fsd) +{ + /* Unlink only once unpinned, so a cursor never resumes from a removed node. */ + if (fsd && refcount_dec_and_test(&fsd->sd_ref)) { + rhltable_remove(&super_dev_table, &fsd->sd_node, super_dev_params); + put_super(fsd->sd_sb); + kfree_rcu(fsd, sd_rcu); + } +} + +void __init super_dev_init(void) +{ + if (rhltable_init(&super_dev_table, &super_dev_params)) + panic("VFS: Cannot initialise super_dev_table\n"); +} + +static int super_dev_insert(struct super_dev *fsd) +{ + int err; + + err = rhltable_insert(&super_dev_table, &fsd->sd_node, super_dev_params); + if (!err) + refcount_inc(&fsd->sd_sb->s_passive); + return err; +} + +/* Register @sb under @sb->s_dev as the final fallible act of a set callback. */ +static int super_dev_register(struct super_block *sb) +{ + struct super_dev *fsd = sb->s_super_dev; + int err; + + lockdep_assert_held(&sb_lock); + VFS_WARN_ON_ONCE(!sb->s_dev); + VFS_WARN_ON_ONCE(!fsd || fsd->sd_dev); + + fsd->sd_dev = sb->s_dev; + err = super_dev_insert(fsd); + if (err) + fsd->sd_dev = 0; + return err; +} + +static struct super_dev *super_dev_get(struct rhlist_head *pos) +{ + struct super_dev *sb_dev; + + for (; pos; pos = rcu_dereference_all(pos->next)) { + sb_dev = container_of(pos, struct super_dev, sd_node); + if (refcount_inc_not_zero(&sb_dev->sd_ref)) + return sb_dev; + } + return NULL; +} + +static struct super_dev *super_dev_first(dev_t dev) +{ + struct super_dev *sb_dev; + + rcu_read_lock(); + sb_dev = super_dev_get(rhltable_lookup(&super_dev_table, &dev, super_dev_params)); + rcu_read_unlock(); + return sb_dev; +} + +static struct super_dev *super_dev_next(struct super_dev *prev) +{ + struct super_dev *sb_dev; + + rcu_read_lock(); + sb_dev = super_dev_get(rcu_dereference_all(prev->sd_node.next)); + rcu_read_unlock(); + + super_dev_put(prev); + return sb_dev; +} + +static void kill_super_notify(struct super_block *sb) +{ + lockdep_assert_not_held(&sb->s_umount); + + /* already notified earlier */ + if (sb->s_flags & SB_DEAD) + return; + + /* + * Remove it from @fs_supers so it isn't found by new + * sget_fc() walkers anymore. Any concurrent mounter still + * managing to grab a temporary reference is guaranteed to + * already see SB_DYING and will wait until we notify them about + * SB_DEAD. + */ + spin_lock(&sb_lock); + hlist_del_init(&sb->s_instances); + spin_unlock(&sb_lock); + + /* Drop sget_fc()'s claim; a never-registered entry stays with the sb. */ + if (sb->s_super_dev->sd_dev) { + super_dev_put(sb->s_super_dev); + sb->s_super_dev = NULL; + } + + /* + * Let concurrent mounts know that this thing is really dead. + * We don't need @sb->s_umount here as every concurrent caller + * will see SB_DYING and either discard the superblock or wait + * for SB_DEAD. + */ + super_wake(sb, SB_DEAD); +} + +/** + * deactivate_locked_super - drop an active reference to superblock + * @s: superblock to deactivate + * + * Drops an active reference to superblock, converting it into a temporary + * one if there is no other active references left. In that case we + * tell fs driver to shut it down and drop the temporary reference we + * had just acquired. + * + * Caller holds exclusive lock on superblock; that lock is released. + */ +void deactivate_locked_super(struct super_block *s) +{ + struct file_system_type *fs = s->s_type; + if (atomic_dec_and_test(&s->s_active)) { + shrinker_free(s->s_shrink); + fs->kill_sb(s); + + kill_super_notify(s); + + /* list_lru_destroy() may sleep; put_super() callers may not. */ + list_lru_destroy(&s->s_dentry_lru); + list_lru_destroy(&s->s_inode_lru); + + put_filesystem(fs); + put_super(s); + } else { + super_unlock_excl(s); + } +} + +EXPORT_SYMBOL(deactivate_locked_super); + +/** + * deactivate_super - drop an active reference to superblock + * @s: superblock to deactivate + * + * Variant of deactivate_locked_super(), except that superblock is *not* + * locked by caller. If we are going to drop the final active reference, + * lock will be acquired prior to that. + */ +void deactivate_super(struct super_block *s) +{ + if (!atomic_add_unless(&s->s_active, -1, 1)) { + __super_lock_excl(s); + deactivate_locked_super(s); + } +} + +EXPORT_SYMBOL(deactivate_super); + +/** + * grab_super - acquire an active reference to a superblock + * @sb: superblock to acquire + * + * Acquire a temporary reference on a superblock and try to trade it for + * an active reference. This is used in sget_fc() to wait for a + * superblock to either become SB_BORN or for it to pass through + * sb->kill() and be marked as SB_DEAD. + * + * Return: This returns true if an active reference could be acquired, + * false if not. + */ +static bool grab_super(struct super_block *sb) +{ + bool locked; + + refcount_inc(&sb->s_passive); + spin_unlock(&sb_lock); + locked = super_lock_excl(sb); + if (locked) { + if (atomic_inc_not_zero(&sb->s_active)) { + put_super(sb); + return true; + } + super_unlock_excl(sb); + } + wait_var_event(&sb->s_flags, super_flags(sb, SB_DEAD)); + put_super(sb); + return false; +} + +/* + * super_trylock_shared - try to grab ->s_umount shared + * @sb: reference we are trying to grab + * + * Try to prevent fs shutdown. This is used in places where we + * cannot take an active reference but we need to ensure that the + * filesystem is not shut down while we are working on it. It returns + * false if we cannot acquire s_umount or if we lose the race and + * filesystem already got into shutdown, and returns true with the s_umount + * lock held in read mode in case of success. On successful return, + * the caller must drop the s_umount lock when done. + * + * Note that unlike get_super() et.al. this one does *not* bump ->s_passive. + * The reason why it's safe is that we are OK with doing trylock instead + * of down_read(). There's a couple of places that are OK with that, but + * it's very much not a general-purpose interface. + */ +bool super_trylock_shared(struct super_block *sb) +{ + if (down_read_trylock(&sb->s_umount)) { + if (!(sb->s_flags & SB_DYING) && sb->s_root && + (sb->s_flags & SB_BORN)) + return true; + super_unlock_shared(sb); + } + + return false; +} + +/** + * retire_super - prevents superblock from being reused + * @sb: superblock to retire + * + * The function marks superblock to be ignored in superblock test, which + * prevents it from being reused for any new mounts. If the superblock has + * a private bdi, it also unregisters it, but doesn't reduce the refcount + * of the superblock to prevent potential races. The refcount is reduced + * by generic_shutdown_super(). The function can not be called + * concurrently with generic_shutdown_super(). It is safe to call the + * function multiple times, subsequent calls have no effect. + * + * The marker will affect the re-use only for block-device-based + * superblocks. Other superblocks will still get marked if this function + * is used, but that will not affect their reusability. + */ +void retire_super(struct super_block *sb) +{ + WARN_ON(!sb->s_bdev); + __super_lock_excl(sb); + if (sb->s_iflags & SB_I_PERSB_BDI) { + bdi_unregister(sb->s_bdi); + sb->s_iflags &= ~SB_I_PERSB_BDI; + } + sb->s_iflags |= SB_I_RETIRED; + super_unlock_excl(sb); +} +EXPORT_SYMBOL(retire_super); + +/** + * generic_shutdown_super - common helper for ->kill_sb() + * @sb: superblock to kill + * + * generic_shutdown_super() does all fs-independent work on superblock + * shutdown. Typical ->kill_sb() should pick all fs-specific objects + * that need destruction out of superblock, call generic_shutdown_super() + * and release aforementioned objects. Note: dentries and inodes _are_ + * taken care of and do not need specific handling. + * + * Upon calling this function, the filesystem may no longer alter or + * rearrange the set of dentries belonging to this super_block, nor may it + * change the attachments of dentries to inodes. + */ +void generic_shutdown_super(struct super_block *sb) +{ + const struct super_operations *sop = sb->s_op; + + if (sb->s_root) { + fsnotify_sb_delete(sb); + shrink_dcache_for_umount(sb); + sync_filesystem(sb); + sb->s_flags &= ~SB_ACTIVE; + + fserror_unmount(sb); + cgroup_writeback_umount(sb); + + /* Evict all inodes with zero refcount. */ + evict_inodes(sb); + + /* + * Clean up and evict any inodes that still have references due + * to the security policy. + */ + security_sb_delete(sb); + + if (sb->s_dio_done_wq) { + destroy_workqueue(sb->s_dio_done_wq); + sb->s_dio_done_wq = NULL; + } + + if (sop->put_super) + sop->put_super(sb); + + /* + * Now that all potentially-encrypted inodes have been evicted, + * the fscrypt keyring can be destroyed. + */ + fscrypt_destroy_keyring(sb); + + if (CHECK_DATA_CORRUPTION(!list_empty(&sb->s_inodes), NULL, + "VFS: Busy inodes after unmount of %s (%s)", + sb->s_id, sb->s_type->name)) { + /* + * Adding a proper bailout path here would be hard, but + * we can at least make it more likely that a later + * iput_final() or such crashes cleanly. + */ + struct inode *inode; + + spin_lock(&sb->s_inode_list_lock); + list_for_each_entry(inode, &sb->s_inodes, i_sb_list) { + inode->i_op = VFS_PTR_POISON; + inode->i_sb = VFS_PTR_POISON; + inode->i_mapping = VFS_PTR_POISON; + } + spin_unlock(&sb->s_inode_list_lock); + } + } + /* + * Broadcast to everyone that grabbed a temporary reference to this + * superblock before we removed it from @fs_supers that the superblock + * is dying. Every walker of @fs_supers outside of sget_fc() will now + * discard this superblock and treat it as dead. + * + * We leave the superblock on @fs_supers so it can be found by + * sget_fc() until we passed sb->kill_sb(). + */ + super_wake(sb, SB_DYING); + super_unlock_excl(sb); + if (sb->s_bdi != &noop_backing_dev_info) { + if (sb->s_iflags & SB_I_PERSB_BDI) + bdi_unregister(sb->s_bdi); + bdi_put(sb->s_bdi); + sb->s_bdi = &noop_backing_dev_info; + } +} + +EXPORT_SYMBOL(generic_shutdown_super); + +bool mount_capable(struct fs_context *fc) +{ + if (!(fc->fs_type->fs_flags & FS_USERNS_MOUNT)) + return capable(CAP_SYS_ADMIN); + else + return ns_capable(fc->user_ns, CAP_SYS_ADMIN); +} + +/** + * sget_fc - Find or create a superblock + * @fc: Filesystem context. + * @test: Comparison callback + * @set: Setup callback + * + * Create a new superblock or find an existing one. + * + * The @test callback is used to find a matching existing superblock. + * Whether or not the requested parameters in @fc are taken into account + * is specific to the @test callback that is used. They may even be + * completely ignored. + * + * If an extant superblock is matched, it will be returned unless: + * + * (1) the namespace the filesystem context @fc and the extant + * superblock's namespace differ + * + * (2) the filesystem context @fc has requested that reusing an extant + * superblock is not allowed + * + * In both cases EBUSY will be returned. + * + * If no match is made, a new superblock will be allocated and basic + * initialisation will be performed (s_type, s_fs_info and s_id will be + * set and the @set callback will be invoked), the superblock will be + * published and it will be returned in a partially constructed state + * with SB_BORN and SB_ACTIVE as yet unset. + * + * Return: On success, an extant or newly created superblock is + * returned. On failure an error pointer is returned. + */ +struct super_block *sget_fc(struct fs_context *fc, + int (*test)(struct super_block *, struct fs_context *), + int (*set)(struct super_block *, struct fs_context *)) +{ + struct super_block *s = NULL; + struct super_block *old; + struct user_namespace *user_ns = fc->global ? &init_user_ns : fc->user_ns; + int err; + + /* + * Never allow s_user_ns != &init_user_ns when FS_USERNS_MOUNT or + * FS_USERNS_DELEGATABLE is not set, as the filesystem is likely + * unprepared to handle it. This can happen when fsconfig() is called + * from init_user_ns with an fs_fd opened in another user namespace. + */ + if (user_ns != &init_user_ns && + !(fc->fs_type->fs_flags & (FS_USERNS_MOUNT | FS_USERNS_DELEGATABLE))) { + errorfc(fc, "VFS: Mounting from non-initial user namespace is not allowed"); + return ERR_PTR(-EPERM); + } + +retry: + spin_lock(&sb_lock); + if (test) { + hlist_for_each_entry(old, &fc->fs_type->fs_supers, s_instances) { + if (test(old, fc)) + goto share_extant_sb; + } + } + if (!s) { + spin_unlock(&sb_lock); + + s = alloc_super(fc->fs_type, fc->sb_flags, user_ns); + if (!s) + return ERR_PTR(-ENOMEM); + goto retry; + } + + s->s_fs_info = fc->s_fs_info; + err = set(s, fc); + if (err) { + VFS_WARN_ON_ONCE(s->s_super_dev->sd_dev); + s->s_fs_info = NULL; + spin_unlock(&sb_lock); + destroy_unused_super(s); + return ERR_PTR(err); + } + VFS_WARN_ON_ONCE(!s->s_super_dev->sd_dev); + fc->s_fs_info = NULL; + s->s_type = fc->fs_type; + s->s_iflags |= fc->s_iflags; + strscpy(s->s_id, s->s_type->name, sizeof(s->s_id)); + /* + * Make the superblock visible on @super_blocks and @fs_supers. + * It's in a nascent state and users should wait on SB_BORN or + * SB_DYING to be set. + */ + list_add_tail(&s->s_list, &super_blocks); + hlist_add_head(&s->s_instances, &s->s_type->fs_supers); + spin_unlock(&sb_lock); + get_filesystem(s->s_type); + shrinker_register(s->s_shrink); + return s; + +share_extant_sb: + if (user_ns != old->s_user_ns || fc->exclusive) { + spin_unlock(&sb_lock); + destroy_unused_super(s); + if (fc->exclusive) + warnfc(fc, "reusing existing filesystem not allowed"); + else + warnfc(fc, "reusing existing filesystem in another namespace not allowed"); + return ERR_PTR(-EBUSY); + } + if (!grab_super(old)) + goto retry; + destroy_unused_super(s); + return old; +} +EXPORT_SYMBOL(sget_fc); + +void drop_super(struct super_block *sb) +{ + super_unlock_shared(sb); + put_super(sb); +} + +EXPORT_SYMBOL(drop_super); + +void drop_super_exclusive(struct super_block *sb) +{ + super_unlock_excl(sb); + put_super(sb); +} + +enum super_iter_flags_t { + SUPER_ITER_EXCL = (1U << 0), + SUPER_ITER_UNLOCKED = (1U << 1), + SUPER_ITER_REVERSE = (1U << 2), +}; + +static inline struct super_block *first_super(enum super_iter_flags_t flags) +{ + if (flags & SUPER_ITER_REVERSE) + return list_last_entry(&super_blocks, struct super_block, s_list); + return list_first_entry(&super_blocks, struct super_block, s_list); +} + +static inline struct super_block *next_super(struct super_block *sb, + enum super_iter_flags_t flags) +{ + if (flags & SUPER_ITER_REVERSE) + return list_prev_entry(sb, s_list); + return list_next_entry(sb, s_list); +} + +static void __iterate_supers(void (*f)(struct super_block *, void *), void *arg, + enum super_iter_flags_t flags) +{ + struct super_block *sb, *p = NULL; + bool excl = flags & SUPER_ITER_EXCL; + + spin_lock(&sb_lock); + + for (sb = first_super(flags); + !list_entry_is_head(sb, &super_blocks, s_list); + sb = next_super(sb, flags)) { + if (super_flags(sb, SB_DYING)) + continue; + + if (!refcount_inc_not_zero(&sb->s_passive)) + continue; + + spin_unlock(&sb_lock); + + if (flags & SUPER_ITER_UNLOCKED) { + f(sb, arg); + } else if (super_lock(sb, excl)) { + f(sb, arg); + super_unlock(sb, excl); + } + + if (p) + put_super(p); + p = sb; + spin_lock(&sb_lock); + } + spin_unlock(&sb_lock); + if (p) + put_super(p); +} + +void iterate_supers(void (*f)(struct super_block *, void *), void *arg) +{ + __iterate_supers(f, arg, 0); +} + +/** + * iterate_supers_type - call function for superblocks of given type + * @type: fs type + * @f: function to call + * @arg: argument to pass to it + * + * Scans the superblock list and calls given function, passing it + * locked superblock and given argument. + */ +void iterate_supers_type(struct file_system_type *type, + void (*f)(struct super_block *, void *), void *arg) +{ + struct super_block *sb, *p = NULL; + + spin_lock(&sb_lock); + hlist_for_each_entry(sb, &type->fs_supers, s_instances) { + bool locked; + + if (super_flags(sb, SB_DYING)) + continue; + + if (!refcount_inc_not_zero(&sb->s_passive)) + continue; + + spin_unlock(&sb_lock); + + locked = super_lock_shared(sb); + if (locked) { + f(sb, arg); + super_unlock_shared(sb); + } + + if (p) + put_super(p); + p = sb; + spin_lock(&sb_lock); + } + spin_unlock(&sb_lock); + if (p) + put_super(p); +} + +EXPORT_SYMBOL(iterate_supers_type); + +struct super_block *user_get_super(dev_t dev, bool excl) +{ + struct super_dev *sb_dev; + + for (sb_dev = super_dev_first(dev); sb_dev; sb_dev = super_dev_next(sb_dev)) { + struct super_block *sb = sb_dev->sd_sb; + + if (!super_lock(sb, excl)) + continue; + + /* The pinned entry holds a passive reference, take our own. */ + refcount_inc(&sb->s_passive); + super_dev_put(sb_dev); + return sb; + } + return NULL; +} + +/** + * reconfigure_super - asks filesystem to change superblock parameters + * @fc: The superblock and configuration + * + * Alters the configuration parameters of a live superblock. + */ +int reconfigure_super(struct fs_context *fc) +{ + struct super_block *sb = fc->root->d_sb; + int retval; + bool remount_ro = false; + bool remount_rw = false; + bool force = fc->sb_flags & SB_FORCE; + + if (fc->sb_flags_mask & ~MS_RMT_MASK) + return -EINVAL; + if (sb->s_writers.frozen != SB_UNFROZEN) + return -EBUSY; + + retval = security_sb_remount(sb, fc->security); + if (retval) + return retval; + + if (fc->sb_flags_mask & SB_RDONLY) { +#ifdef CONFIG_BLOCK + if (!(fc->sb_flags & SB_RDONLY) && sb->s_bdev && + bdev_read_only(sb->s_bdev)) + return -EACCES; +#endif + remount_rw = !(fc->sb_flags & SB_RDONLY) && sb_rdonly(sb); + remount_ro = (fc->sb_flags & SB_RDONLY) && !sb_rdonly(sb); + } + + if (remount_ro) { + if (!hlist_empty(&sb->s_pins)) { + super_unlock_excl(sb); + group_pin_kill(&sb->s_pins); + __super_lock_excl(sb); + if (!sb->s_root) + return 0; + if (sb->s_writers.frozen != SB_UNFROZEN) + return -EBUSY; + remount_ro = !sb_rdonly(sb); + } + } + shrink_dcache_sb(sb); + + /* If we are reconfiguring to RDONLY and current sb is read/write, + * make sure there are no files open for writing. + */ + if (remount_ro) { + if (force) { + sb_start_ro_state_change(sb); + } else { + retval = sb_prepare_remount_readonly(sb); + if (retval) + return retval; + } + } else if (remount_rw) { + /* + * Protect filesystem's reconfigure code from writes from + * userspace until reconfigure finishes. + */ + sb_start_ro_state_change(sb); + } + + if (fc->ops->reconfigure) { + retval = fc->ops->reconfigure(fc); + if (retval) { + if (!force) + goto cancel_readonly; + /* If forced remount, go ahead despite any errors */ + WARN(1, "forced remount of a %s fs returned %i\n", + sb->s_type->name, retval); + } + } + + WRITE_ONCE(sb->s_flags, ((sb->s_flags & ~fc->sb_flags_mask) | + (fc->sb_flags & fc->sb_flags_mask))); + sb_end_ro_state_change(sb); + + /* + * Some filesystems modify their metadata via some other path than the + * bdev buffer cache (eg. use a private mapping, or directories in + * pagecache, etc). Also file data modifications go via their own + * mappings. So If we try to mount readonly then copy the filesystem + * from bdev, we could get stale data, so invalidate it to give a best + * effort at coherency. + */ + if (remount_ro && sb->s_bdev) + invalidate_bdev(sb->s_bdev); + return 0; + +cancel_readonly: + sb_end_ro_state_change(sb); + return retval; +} + +static void do_emergency_remount_callback(struct super_block *sb, void *unused) +{ + if (sb->s_bdev && !sb_rdonly(sb)) { + struct fs_context *fc; + + fc = fs_context_for_reconfigure(sb->s_root, + SB_RDONLY | SB_FORCE, SB_RDONLY); + if (!IS_ERR(fc)) { + if (parse_monolithic_mount_data(fc, NULL) == 0) + (void)reconfigure_super(fc); + put_fs_context(fc); + } + } +} + +static void do_emergency_remount(struct work_struct *work) +{ + __iterate_supers(do_emergency_remount_callback, NULL, + SUPER_ITER_EXCL | SUPER_ITER_REVERSE); + kfree(work); + printk("Emergency Remount complete\n"); +} + +void emergency_remount(void) +{ + struct work_struct *work; + + work = kmalloc_obj(*work, GFP_ATOMIC); + if (work) { + INIT_WORK(work, do_emergency_remount); + schedule_work(work); + } +} + +static inline bool get_active_super(struct super_block *sb) +{ + bool active = false; + + if (super_lock_excl(sb)) { + active = atomic_inc_not_zero(&sb->s_active); + super_unlock_excl(sb); + } + return active; +} + +static void do_thaw_all_callback(struct super_block *sb, void *unused) +{ + if (!get_active_super(sb)) + return; + + /* fs_bdev_thaw() acquires s_umount so it must not be held here */ + if (IS_ENABLED(CONFIG_BLOCK)) + while (sb->s_bdev && !bdev_thaw(sb->s_bdev)) + pr_warn("Emergency Thaw on %pg\n", sb->s_bdev); + + if (super_lock_excl(sb)) + thaw_super_locked(sb, FREEZE_HOLDER_USERSPACE, NULL); + deactivate_super(sb); +} + +static void do_thaw_all(struct work_struct *work) +{ + __iterate_supers(do_thaw_all_callback, NULL, SUPER_ITER_UNLOCKED); + kfree(work); + printk(KERN_WARNING "Emergency Thaw complete\n"); +} + +/** + * emergency_thaw_all -- forcibly thaw every frozen filesystem + * + * Used for emergency unfreeze of all filesystems via SysRq + */ +void emergency_thaw_all(void) +{ + struct work_struct *work; + + work = kmalloc_obj(*work, GFP_ATOMIC); + if (work) { + INIT_WORK(work, do_thaw_all); + schedule_work(work); + } +} + +static const char *filesystems_freeze_ptr = "filesystems_freeze"; + +static void filesystems_freeze_callback(struct super_block *sb, void *freeze_all_ptr) +{ + if (!sb->s_op->freeze_fs && !sb->s_op->freeze_super) + return; + + if (!freeze_all_ptr && !(sb->s_type->fs_flags & FS_POWER_FREEZE)) + return; + + if (!get_active_super(sb)) + return; + + if (sb->s_op->freeze_super) + sb->s_op->freeze_super(sb, FREEZE_EXCL | FREEZE_HOLDER_KERNEL, + filesystems_freeze_ptr); + else + freeze_super(sb, FREEZE_EXCL | FREEZE_HOLDER_KERNEL, + filesystems_freeze_ptr); + + deactivate_super(sb); +} + +void filesystems_freeze(bool freeze_all) +{ + void *freeze_all_ptr = NULL; + + if (freeze_all) + freeze_all_ptr = &freeze_all; + __iterate_supers(filesystems_freeze_callback, freeze_all_ptr, + SUPER_ITER_UNLOCKED | SUPER_ITER_REVERSE); +} + +static void filesystems_thaw_callback(struct super_block *sb, void *unused) +{ + if (!sb->s_op->freeze_fs && !sb->s_op->freeze_super) + return; + + if (!get_active_super(sb)) + return; + + if (sb->s_op->thaw_super) + sb->s_op->thaw_super(sb, FREEZE_EXCL | FREEZE_HOLDER_KERNEL, + filesystems_freeze_ptr); + else + thaw_super(sb, FREEZE_EXCL | FREEZE_HOLDER_KERNEL, + filesystems_freeze_ptr); + + deactivate_super(sb); +} + +void filesystems_thaw(void) +{ + __iterate_supers(filesystems_thaw_callback, NULL, SUPER_ITER_UNLOCKED); +} + +static DEFINE_IDA(unnamed_dev_ida); + +/** + * get_anon_bdev - Allocate a block device for filesystems which don't have one. + * @p: Pointer to a dev_t. + * + * Filesystems which don't use real block devices can call this function + * to allocate a virtual block device. + * + * Context: Any context. Frequently called while holding sb_lock. + * Return: 0 on success, -EMFILE if there are no anonymous bdevs left + * or -ENOMEM if memory allocation failed. + */ +int get_anon_bdev(dev_t *p) +{ + int dev; + + /* + * Many userspace utilities consider an FSID of 0 invalid. + * Always return at least 1 from get_anon_bdev. + */ + dev = ida_alloc_range(&unnamed_dev_ida, 1, (1 << MINORBITS) - 1, + GFP_ATOMIC); + if (dev == -ENOSPC) + dev = -EMFILE; + if (dev < 0) + return dev; + + *p = MKDEV(0, dev); + return 0; +} +EXPORT_SYMBOL(get_anon_bdev); + +void free_anon_bdev(dev_t dev) +{ + ida_free(&unnamed_dev_ida, MINOR(dev)); +} +EXPORT_SYMBOL(free_anon_bdev); + +int set_anon_super(struct super_block *s, void *data) +{ + int error; + + error = get_anon_bdev(&s->s_dev); + if (error) + return error; + + error = super_dev_register(s); + if (error) + free_anon_bdev(s->s_dev); + return error; +} +EXPORT_SYMBOL(set_anon_super); + +void kill_anon_super(struct super_block *sb) +{ + dev_t dev = sb->s_dev; + generic_shutdown_super(sb); + kill_super_notify(sb); + free_anon_bdev(dev); +} +EXPORT_SYMBOL(kill_anon_super); + +int set_anon_super_fc(struct super_block *sb, struct fs_context *fc) +{ + return set_anon_super(sb, NULL); +} +EXPORT_SYMBOL(set_anon_super_fc); + +static int test_keyed_super(struct super_block *sb, struct fs_context *fc) +{ + return sb->s_fs_info == fc->s_fs_info; +} + +static int test_single_super(struct super_block *s, struct fs_context *fc) +{ + return 1; +} + +static int vfs_get_super(struct fs_context *fc, + int (*test)(struct super_block *, struct fs_context *), + int (*fill_super)(struct super_block *sb, + struct fs_context *fc)) +{ + struct super_block *sb; + int err; + + sb = sget_fc(fc, test, set_anon_super_fc); + if (IS_ERR(sb)) + return PTR_ERR(sb); + + if (!sb->s_root) { + err = fill_super(sb, fc); + if (err) + goto error; + + sb->s_flags |= SB_ACTIVE; + } + + fc->root = dget(sb->s_root); + return 0; + +error: + deactivate_locked_super(sb); + return err; +} + +int get_tree_nodev(struct fs_context *fc, + int (*fill_super)(struct super_block *sb, + struct fs_context *fc)) +{ + return vfs_get_super(fc, NULL, fill_super); +} +EXPORT_SYMBOL(get_tree_nodev); + +int get_tree_single(struct fs_context *fc, + int (*fill_super)(struct super_block *sb, + struct fs_context *fc)) +{ + return vfs_get_super(fc, test_single_super, fill_super); +} +EXPORT_SYMBOL(get_tree_single); + +int get_tree_keyed(struct fs_context *fc, + int (*fill_super)(struct super_block *sb, + struct fs_context *fc), + void *key) +{ + fc->s_fs_info = key; + return vfs_get_super(fc, test_keyed_super, fill_super); +} +EXPORT_SYMBOL(get_tree_keyed); + +static int set_bdev_super(struct super_block *s, void *data) +{ + s->s_dev = *(dev_t *)data; + return super_dev_register(s); +} + +static int super_s_dev_set(struct super_block *s, struct fs_context *fc) +{ + return set_bdev_super(s, fc->sget_key); +} + +static int super_s_dev_test(struct super_block *s, struct fs_context *fc) +{ + return !(s->s_iflags & SB_I_RETIRED) && + s->s_dev == *(dev_t *)fc->sget_key; +} + +/** + * sget_dev - Find or create a superblock by device number + * @fc: Filesystem context. + * @dev: device number + * + * Find or create a superblock using the provided device number that + * will be stored in fc->sget_key. + * + * If an extant superblock is matched, then that will be returned with + * an elevated reference count that the caller must transfer or discard. + * + * If no match is made, a new superblock will be allocated and basic + * initialisation will be performed (s_type, s_fs_info, s_id, s_dev will + * be set). The superblock will be published and it will be returned in + * a partially constructed state with SB_BORN and SB_ACTIVE as yet + * unset. + * + * Return: an existing or newly created superblock on success, an error + * pointer on failure. + */ +struct super_block *sget_dev(struct fs_context *fc, dev_t dev) +{ + fc->sget_key = &dev; + return sget_fc(fc, super_s_dev_test, super_s_dev_set); +} +EXPORT_SYMBOL(sget_dev); + +#ifdef CONFIG_BLOCK +static int fs_super_freeze(struct super_block *sb) +{ + if (sb->s_op->freeze_super) + return sb->s_op->freeze_super(sb, + FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE, NULL); + return freeze_super(sb, FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE, NULL); +} + +static int fs_super_thaw(struct super_block *sb) +{ + if (sb->s_op->thaw_super) + return sb->s_op->thaw_super(sb, + FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE, NULL); + return thaw_super(sb, FREEZE_MAY_NEST | FREEZE_HOLDER_USERSPACE, NULL); +} + +static void fs_bdev_mark_dead(struct block_device *bdev, bool surprise) +{ + struct super_dev *sb_dev; + dev_t dev = bdev->bd_dev; + + mutex_unlock(&bdev->bd_holder_lock); + + for (sb_dev = super_dev_first(dev); sb_dev; sb_dev = super_dev_next(sb_dev)) { + struct super_block *sb = sb_dev->sd_sb; + + if (!super_lock_shared(sb)) + continue; + if (sb->s_root && (sb->s_flags & SB_ACTIVE)) { + if (!sb->s_op->remove_bdev || + sb->s_op->remove_bdev(sb, bdev)) { + if (!surprise) + sync_filesystem(sb); + shrink_dcache_sb(sb); + evict_inodes(sb); + if (sb->s_op->shutdown) + sb->s_op->shutdown(sb); + } + } + super_unlock_shared(sb); + } +} + +static void fs_bdev_sync(struct block_device *bdev) +{ + struct super_dev *sb_dev; + dev_t dev = bdev->bd_dev; + + mutex_unlock(&bdev->bd_holder_lock); + + for (sb_dev = super_dev_first(dev); sb_dev; sb_dev = super_dev_next(sb_dev)) { + struct super_block *sb = sb_dev->sd_sb; + + if (!super_lock_shared(sb)) + continue; + if (sb->s_root && (sb->s_flags & SB_ACTIVE)) + sync_filesystem(sb); + super_unlock_shared(sb); + } +} + +/** + * fs_bdev_freeze - freeze every superblock using a block device + * @bdev: block device + * + * Freeze each live superblock using @bdev. A superblock owning several block + * devices is frozen once per device and stays frozen until all are thawed; the + * block layer nests these freezes so the count stays balanced. + * + * Return: 0, or the error from the one superblock on a single-fs device. When + * several superblocks share @bdev a per-superblock failure is swallowed + * (see below), but a sync_blockdev() failure is always reported. + */ +static int fs_bdev_freeze(struct block_device *bdev) +{ + dev_t dev = bdev->bd_dev; + struct super_dev *sb_dev; + unsigned int count = 0; + int error = 0, err; + + lockdep_assert_held(&bdev->bd_fsfreeze_mutex); + + mutex_unlock(&bdev->bd_holder_lock); + + for (sb_dev = super_dev_first(dev); sb_dev; sb_dev = super_dev_next(sb_dev)) { + if (!get_active_super(sb_dev->sd_sb)) + continue; + err = fs_super_freeze(sb_dev->sd_sb); + if (err && !error) + error = err; + deactivate_super(sb_dev->sd_sb); + count++; + } + + /* + * When several superblocks share the device, keep it frozen even if some + * of them failed to freeze and swallow the error: rolling the rest back + * via thaw_super() can fail too, so neither is a clear win. A single + * filesystem (count == 1) still reports its error. + */ + if (error && count > 1) + error = 0; + if (!error) + error = sync_blockdev(bdev); + return error; +} + +/** + * fs_bdev_thaw - thaw every superblock using a block device + * @bdev: block device + * + * The counterpart to fs_bdev_freeze(): thaw each live superblock using @bdev. + * A zero return does not imply a superblock is fully unfrozen; it may have been + * frozen more than once (by the kernel or via another device). + * + * Return: 0, or the first error on a single-fs device; a shared device swallows + * per-superblock errors, as fs_bdev_freeze() does. + */ +static int fs_bdev_thaw(struct block_device *bdev) +{ + dev_t dev = bdev->bd_dev; + struct super_dev *sb_dev; + unsigned int count = 0; + int error = 0, err; + + lockdep_assert_held(&bdev->bd_fsfreeze_mutex); + + mutex_unlock(&bdev->bd_holder_lock); + + for (sb_dev = super_dev_first(dev); sb_dev; sb_dev = super_dev_next(sb_dev)) { + if (!get_active_super(sb_dev->sd_sb)) + continue; + err = fs_super_thaw(sb_dev->sd_sb); + if (err && !error) + error = err; + deactivate_super(sb_dev->sd_sb); + count++; + } + + /* Shared device: swallow per-superblock errors, like fs_bdev_freeze(). */ + if (error && count > 1) + error = 0; + return error; +} + +static const struct blk_holder_ops fs_holder_ops = { + .mark_dead = fs_bdev_mark_dead, + .sync = fs_bdev_sync, + .freeze = fs_bdev_freeze, + .thaw = fs_bdev_thaw, +}; + +static struct super_dev *super_dev_lookup(dev_t dev, struct super_block *sb) +{ + struct super_dev *it; + struct rhlist_head *list, *pos; + + RCU_LOCKDEP_WARN(!rcu_read_lock_held(), "suspicious super_dev_lookup() usage"); + VFS_WARN_ON_ONCE(!dev); + VFS_WARN_ON_ONCE(!sb); + + list = rhltable_lookup(&super_dev_table, &dev, super_dev_params); + rhl_for_each_entry_rcu(it, pos, list, sd_node) { + if (it->sd_sb == sb) + return it; + } + + return NULL; +} + +static int fs_bdev_register(struct file *bdev_file, struct super_block *sb) +{ + struct super_dev *sb_dev __free(kfree) = NULL; + dev_t dev = file_bdev(bdev_file)->bd_dev; + int err; + + scoped_guard(rcu) { + sb_dev = super_dev_lookup(dev, sb); + if (sb_dev && refcount_inc_not_zero(&sb_dev->sd_ref)) { + retain_and_null_ptr(sb_dev); + return 0; + } + } + + sb_dev = super_dev_alloc(dev, sb); + if (!sb_dev) + return -ENOMEM; + + err = super_dev_insert(sb_dev); + if (err) + return err; + + /* Publish the entry before reading the count; pairs with bdev_freeze(). */ + smp_mb(); + if (atomic_read(&file_bdev(bdev_file)->bd_fsfreeze_count) > 0) { + err = -EBUSY; + super_dev_put(sb_dev); + } + + retain_and_null_ptr(sb_dev); + return err; +} + +/** + * fs_bdev_file_open_by_dev - claim a block device on behalf of a superblock + * @dev: block device number + * @mode: open mode + * @holder: block-layer exclusivity token (a superblock, or the file_system_type + * when the device may be shared by several superblocks of that type) + * @sb: superblock to drive fs_holder_ops events for + * + * Open @dev with &fs_holder_ops and register that @sb uses it, so device + * removal/sync/freeze/thaw are propagated to @sb (and any other superblock + * sharing @dev). Must be paired with fs_bdev_file_release(). + * + * Return: an opened block-device file or an ERR_PTR(). + */ +struct file *fs_bdev_file_open_by_dev(dev_t dev, blk_mode_t mode, void *holder, + struct super_block *sb) +{ + struct file *bdev_file; + int err; + + bdev_file = bdev_file_open_by_dev(dev, mode, holder, &fs_holder_ops); + if (IS_ERR(bdev_file)) + return bdev_file; + + err = fs_bdev_register(bdev_file, sb); + if (err) { + bdev_fput(bdev_file); + return ERR_PTR(err); + } + return bdev_file; +} +EXPORT_SYMBOL_GPL(fs_bdev_file_open_by_dev); + +/** + * fs_bdev_file_open_by_path - claim a block device on behalf of a superblock + * @path: path to the block device + * @mode: open mode + * @holder: block-layer exclusivity token (a superblock, or the file_system_type + * when the device may be shared by several superblocks of that type) + * @sb: superblock to drive fs_holder_ops events for + * + * Open the block device at @path with &fs_holder_ops and register that @sb + * uses it, so device removal/sync/freeze/thaw are propagated to @sb (and any + * other superblock sharing the device). Must be paired with + * fs_bdev_file_release(). + * + * Return: an opened block-device file or an ERR_PTR(). + */ +struct file *fs_bdev_file_open_by_path(const char *path, blk_mode_t mode, + void *holder, struct super_block *sb) +{ + struct file *bdev_file; + int err; + + bdev_file = bdev_file_open_by_path(path, mode, holder, &fs_holder_ops); + if (IS_ERR(bdev_file)) + return bdev_file; + + err = fs_bdev_register(bdev_file, sb); + if (err) { + bdev_fput(bdev_file); + return ERR_PTR(err); + } + return bdev_file; +} +EXPORT_SYMBOL_GPL(fs_bdev_file_open_by_path); + +/** + * fs_bdev_unregister - drop a superblock's claim on a block device + * @bdev_file: file returned by fs_bdev_file_open_by_{dev,path}() + * @sb: superblock the device was claimed for + * + * The inverse of fs_bdev_register(): drop one claim on the {dev, @sb} entry + * (the last claim unregisters it; a pinning cursor defers the actual unlink) + * without closing the device. A caller that must act on the still-open device + * between unregistering and closing - e.g. re-allow freezing one denied for a + * membership change - pairs this with bdev_fput(). fs_bdev_file_release() is + * the common unregister-and-close. + */ +void fs_bdev_unregister(struct file *bdev_file, struct super_block *sb) +{ + dev_t dev = file_bdev(bdev_file)->bd_dev; + struct super_dev *sb_dev; + + rcu_read_lock(); + sb_dev = super_dev_lookup(dev, sb); + rcu_read_unlock(); + super_dev_put(sb_dev); +} +EXPORT_SYMBOL_GPL(fs_bdev_unregister); + +/** + * fs_bdev_file_release - release a block device claimed for a superblock + * @bdev_file: file returned by fs_bdev_file_open_by_{dev,path}() + * @sb: superblock the device was claimed for + * + * Unregister the {dev, @sb} entry, then close the block device. + */ +void fs_bdev_file_release(struct file *bdev_file, struct super_block *sb) +{ + fs_bdev_unregister(bdev_file, sb); + bdev_fput(bdev_file); +} +EXPORT_SYMBOL_GPL(fs_bdev_file_release); + +int setup_bdev_super(struct super_block *sb, int sb_flags, + struct fs_context *fc) +{ + blk_mode_t mode = sb_open_mode(sb_flags); + struct file *bdev_file; + struct block_device *bdev; + + bdev_file = fs_bdev_file_open_by_dev(sb->s_dev, mode, sb, sb); + if (IS_ERR(bdev_file)) { + if (fc) + errorf(fc, "%s: Can't open blockdev", fc->source); + return PTR_ERR(bdev_file); + } + bdev = file_bdev(bdev_file); + + /* + * This really should be in blkdev_get_by_dev, but right now can't due + * to legacy issues that require us to allow opening a block device node + * writable from userspace even for a read-only block device. + */ + if ((mode & BLK_OPEN_WRITE) && bdev_read_only(bdev)) { + fs_bdev_file_release(bdev_file, sb); + return -EACCES; + } + + /* The sget_fc() entry is already published; pairs with bdev_freeze(). */ + smp_mb(); + if (atomic_read(&bdev->bd_fsfreeze_count) > 0) { + if (fc) + warnf(fc, "%pg: Can't mount, blockdev is frozen", bdev); + fs_bdev_file_release(bdev_file, sb); + return -EBUSY; + } + + spin_lock(&sb_lock); + sb->s_bdev_file = bdev_file; + sb->s_bdev = bdev; + sb->s_bdi = bdi_get(bdev->bd_disk->bdi); + if (bdev_stable_writes(bdev)) + sb->s_iflags |= SB_I_STABLE_WRITES; + spin_unlock(&sb_lock); + + snprintf(sb->s_id, sizeof(sb->s_id), "%pg", bdev); + shrinker_debugfs_rename(sb->s_shrink, "sb-%s:%s", sb->s_type->name, + sb->s_id); + sb_set_blocksize(sb, block_size(bdev)); + return 0; +} +EXPORT_SYMBOL_GPL(setup_bdev_super); + +/** + * get_tree_bdev_flags - Get a superblock based on a single block device + * @fc: The filesystem context holding the parameters + * @fill_super: Helper to initialise a new superblock + * @flags: GET_TREE_BDEV_* flags + */ +int get_tree_bdev_flags(struct fs_context *fc, + int (*fill_super)(struct super_block *sb, + struct fs_context *fc), unsigned int flags) +{ + struct super_block *s; + int error = 0; + dev_t dev; + + if (!fc->source) + return invalf(fc, "No source specified"); + + error = lookup_bdev(fc->source, &dev); + if (error) { + if (!(flags & GET_TREE_BDEV_QUIET_LOOKUP)) + errorf(fc, "%s: Can't lookup blockdev", fc->source); + return error; + } + fc->sb_flags |= SB_NOSEC; + s = sget_dev(fc, dev); + if (IS_ERR(s)) + return PTR_ERR(s); + + if (s->s_root) { + /* Don't summarily change the RO/RW state. */ + if ((fc->sb_flags ^ s->s_flags) & SB_RDONLY) { + warnf(fc, "%pg: Can't mount, would change RO state", s->s_bdev); + deactivate_locked_super(s); + return -EBUSY; + } + } else { + error = setup_bdev_super(s, fc->sb_flags, fc); + if (!error) + error = fill_super(s, fc); + if (error) { + deactivate_locked_super(s); + return error; + } + s->s_flags |= SB_ACTIVE; + } + + BUG_ON(fc->root); + fc->root = dget(s->s_root); + return 0; +} +EXPORT_SYMBOL_GPL(get_tree_bdev_flags); + +/** + * get_tree_bdev - Get a superblock based on a single block device + * @fc: The filesystem context holding the parameters + * @fill_super: Helper to initialise a new superblock + */ +int get_tree_bdev(struct fs_context *fc, + int (*fill_super)(struct super_block *, + struct fs_context *)) +{ + return get_tree_bdev_flags(fc, fill_super, 0); +} +EXPORT_SYMBOL(get_tree_bdev); + +void kill_block_super(struct super_block *sb) +{ + struct block_device *bdev = sb->s_bdev; + + generic_shutdown_super(sb); + if (bdev) { + sync_blockdev(bdev); + fs_bdev_file_release(sb->s_bdev_file, sb); + } +} + +EXPORT_SYMBOL(kill_block_super); +#endif + +/** + * vfs_get_tree - Get the mountable root + * @fc: The superblock configuration context. + * + * The filesystem is invoked to get or create a superblock which can then later + * be used for mounting. The filesystem places a pointer to the root to be + * used for mounting in @fc->root. + */ +int vfs_get_tree(struct fs_context *fc) +{ + struct super_block *sb; + int error; + + if (fc->root) + return -EBUSY; + + /* Get the mountable root in fc->root, with a ref on the root and a ref + * on the superblock. + */ + error = fc->ops->get_tree(fc); + if (error < 0) + return error; + + if (!fc->root) { + pr_err("Filesystem %s get_tree() didn't set fc->root, returned %i\n", + fc->fs_type->name, error); + /* We don't know what the locking state of the superblock is - + * if there is a superblock. + */ + BUG(); + } + + sb = fc->root->d_sb; + WARN_ON(!sb->s_bdi); + + /* + * super_wake() contains a memory barrier which also care of + * ordering for super_cache_count(). We place it before setting + * SB_BORN as the data dependency between the two functions is + * the superblock structure contents that we just set up, not + * the SB_BORN flag. + */ + super_wake(sb, SB_BORN); + + error = security_sb_set_mnt_opts(sb, fc->security, 0, NULL); + if (unlikely(error)) { + fc_drop_locked(fc); + return error; + } + + /* + * filesystems should never set s_maxbytes larger than MAX_LFS_FILESIZE + * but s_maxbytes was an unsigned long long for many releases. Throw + * this warning for a little while to try and catch filesystems that + * violate this rule. + */ + WARN((sb->s_maxbytes < 0), "%s set sb->s_maxbytes to " + "negative value (%lld)\n", fc->fs_type->name, sb->s_maxbytes); + + return 0; +} +EXPORT_SYMBOL(vfs_get_tree); + +/* + * Setup private BDI for given superblock. It gets automatically cleaned up + * in generic_shutdown_super(). + */ +int super_setup_bdi_name(struct super_block *sb, char *fmt, ...) +{ + struct backing_dev_info *bdi; + int err; + va_list args; + + bdi = bdi_alloc(NUMA_NO_NODE); + if (!bdi) + return -ENOMEM; + + va_start(args, fmt); + err = bdi_register_va(bdi, fmt, args); + va_end(args); + if (err) { + bdi_put(bdi); + return err; + } + WARN_ON(sb->s_bdi != &noop_backing_dev_info); + sb->s_bdi = bdi; + sb->s_iflags |= SB_I_PERSB_BDI; + + return 0; +} +EXPORT_SYMBOL(super_setup_bdi_name); + +/* + * Setup private BDI for given superblock. I gets automatically cleaned up + * in generic_shutdown_super(). + */ +int super_setup_bdi(struct super_block *sb) +{ + static atomic_long_t bdi_seq = ATOMIC_LONG_INIT(0); + + return super_setup_bdi_name(sb, "%.28s-%ld", sb->s_type->name, + atomic_long_inc_return(&bdi_seq)); +} +EXPORT_SYMBOL(super_setup_bdi); + +/** + * sb_wait_write - wait until all writers to given file system finish + * @sb: the super for which we wait + * @level: type of writers we wait for (normal vs page fault) + * + * This function waits until there are no writers of given type to given file + * system. + */ +static void sb_wait_write(struct super_block *sb, int level) +{ + percpu_down_write(sb->s_writers.rw_sem + level-1); +} + +/* + * We are going to return to userspace and forget about these locks, the + * ownership goes to the caller of thaw_super() which does unlock(). + */ +static void lockdep_sb_freeze_release(struct super_block *sb) +{ + int level; + + for (level = SB_FREEZE_LEVELS - 1; level >= 0; level--) + percpu_rwsem_release(sb->s_writers.rw_sem + level, _THIS_IP_); +} + +/* + * Tell lockdep we are holding these locks before we call ->unfreeze_fs(sb). + */ +static void lockdep_sb_freeze_acquire(struct super_block *sb) +{ + int level; + + for (level = 0; level < SB_FREEZE_LEVELS; ++level) + percpu_rwsem_acquire(sb->s_writers.rw_sem + level, 0, _THIS_IP_); +} + +static void sb_freeze_unlock(struct super_block *sb, int level) +{ + for (level--; level >= 0; level--) + percpu_up_write(sb->s_writers.rw_sem + level); +} + +static int wait_for_partially_frozen(struct super_block *sb) +{ + int ret = 0; + + do { + unsigned short old = sb->s_writers.frozen; + + up_write(&sb->s_umount); + ret = wait_var_event_killable(&sb->s_writers.frozen, + sb->s_writers.frozen != old); + down_write(&sb->s_umount); + } while (ret == 0 && + sb->s_writers.frozen != SB_UNFROZEN && + sb->s_writers.frozen != SB_FREEZE_COMPLETE); + + return ret; +} + +#define FREEZE_HOLDERS (FREEZE_HOLDER_KERNEL | FREEZE_HOLDER_USERSPACE) +#define FREEZE_FLAGS (FREEZE_HOLDERS | FREEZE_MAY_NEST | FREEZE_EXCL) + +static inline int freeze_inc(struct super_block *sb, enum freeze_holder who) +{ + WARN_ON_ONCE((who & ~FREEZE_FLAGS)); + WARN_ON_ONCE(hweight32(who & FREEZE_HOLDERS) > 1); + + if (who & FREEZE_HOLDER_KERNEL) + ++sb->s_writers.freeze_kcount; + if (who & FREEZE_HOLDER_USERSPACE) + ++sb->s_writers.freeze_ucount; + return sb->s_writers.freeze_kcount + sb->s_writers.freeze_ucount; +} + +static inline int freeze_dec(struct super_block *sb, enum freeze_holder who) +{ + WARN_ON_ONCE((who & ~FREEZE_FLAGS)); + WARN_ON_ONCE(hweight32(who & FREEZE_HOLDERS) > 1); + + if ((who & FREEZE_HOLDER_KERNEL) && sb->s_writers.freeze_kcount) + --sb->s_writers.freeze_kcount; + if ((who & FREEZE_HOLDER_USERSPACE) && sb->s_writers.freeze_ucount) + --sb->s_writers.freeze_ucount; + return sb->s_writers.freeze_kcount + sb->s_writers.freeze_ucount; +} + +static inline bool may_freeze(struct super_block *sb, enum freeze_holder who, + const void *freeze_owner) +{ + lockdep_assert_held(&sb->s_umount); + + WARN_ON_ONCE((who & ~FREEZE_FLAGS)); + WARN_ON_ONCE(hweight32(who & FREEZE_HOLDERS) > 1); + + if (who & FREEZE_EXCL) { + if (WARN_ON_ONCE(!(who & FREEZE_HOLDER_KERNEL))) + return false; + if (WARN_ON_ONCE(who & ~(FREEZE_EXCL | FREEZE_HOLDER_KERNEL))) + return false; + if (WARN_ON_ONCE(!freeze_owner)) + return false; + /* This freeze already has a specific owner. */ + if (sb->s_writers.freeze_owner) + return false; + /* + * This is already frozen multiple times so we're just + * going to take a reference count and mark the freeze as + * being owned by the caller. + */ + if (sb->s_writers.freeze_kcount + sb->s_writers.freeze_ucount) + sb->s_writers.freeze_owner = freeze_owner; + return true; + } + + if (who & FREEZE_HOLDER_KERNEL) + return (who & FREEZE_MAY_NEST) || + sb->s_writers.freeze_kcount == 0; + if (who & FREEZE_HOLDER_USERSPACE) + return (who & FREEZE_MAY_NEST) || + sb->s_writers.freeze_ucount == 0; + return false; +} + +static inline bool may_unfreeze(struct super_block *sb, enum freeze_holder who, + const void *freeze_owner) +{ + lockdep_assert_held(&sb->s_umount); + + WARN_ON_ONCE((who & ~FREEZE_FLAGS)); + WARN_ON_ONCE(hweight32(who & FREEZE_HOLDERS) > 1); + + if (who & FREEZE_EXCL) { + if (WARN_ON_ONCE(!(who & FREEZE_HOLDER_KERNEL))) + return false; + if (WARN_ON_ONCE(who & ~(FREEZE_EXCL | FREEZE_HOLDER_KERNEL))) + return false; + if (WARN_ON_ONCE(!freeze_owner)) + return false; + if (WARN_ON_ONCE(sb->s_writers.freeze_kcount == 0)) + return false; + /* This isn't exclusively frozen. */ + if (!sb->s_writers.freeze_owner) + return false; + /* This isn't exclusively frozen by us. */ + if (sb->s_writers.freeze_owner != freeze_owner) + return false; + /* + * This is still frozen multiple times so we're just + * going to drop our reference count and undo our + * exclusive freeze. + */ + if ((sb->s_writers.freeze_kcount + sb->s_writers.freeze_ucount) > 1) + sb->s_writers.freeze_owner = NULL; + return true; + } + + if (who & FREEZE_HOLDER_KERNEL) { + /* + * Someone's trying to steal the reference belonging to + * @sb->s_writers.freeze_owner. + */ + if (sb->s_writers.freeze_kcount == 1 && + sb->s_writers.freeze_owner) + return false; + return sb->s_writers.freeze_kcount > 0; + } + + if (who & FREEZE_HOLDER_USERSPACE) + return sb->s_writers.freeze_ucount > 0; + + return false; +} + +/** + * freeze_super - lock the filesystem and force it into a consistent state + * @sb: the super to lock + * @who: context that wants to freeze + * @freeze_owner: owner of the freeze + * + * Syncs the super to make sure the filesystem is consistent and calls the fs's + * freeze_fs. Subsequent calls to this without first thawing the fs may return + * -EBUSY. + * + * @who should be: + * * %FREEZE_HOLDER_USERSPACE if userspace wants to freeze the fs; + * * %FREEZE_HOLDER_KERNEL if the kernel wants to freeze the fs. + * * %FREEZE_MAY_NEST whether nesting freeze and thaw requests is allowed. + * + * The @who argument distinguishes between the kernel and userspace trying to + * freeze the filesystem. Although there cannot be multiple kernel freezes or + * multiple userspace freezes in effect at any given time, the kernel and + * userspace can both hold a filesystem frozen. The filesystem remains frozen + * until there are no kernel or userspace freezes in effect. + * + * A filesystem may hold multiple devices and thus a filesystems may be + * frozen through the block layer via multiple block devices. In this + * case the request is marked as being allowed to nest by passing + * FREEZE_MAY_NEST. The filesystem remains frozen until all block + * devices are unfrozen. If multiple freezes are attempted without + * FREEZE_MAY_NEST -EBUSY will be returned. + * + * During this function, sb->s_writers.frozen goes through these values: + * + * SB_UNFROZEN: File system is normal, all writes progress as usual. + * + * SB_FREEZE_WRITE: The file system is in the process of being frozen. New + * writes should be blocked, though page faults are still allowed. We wait for + * all writes to complete and then proceed to the next stage. + * + * SB_FREEZE_PAGEFAULT: Freezing continues. Now also page faults are blocked + * but internal fs threads can still modify the filesystem (although they + * should not dirty new pages or inodes), writeback can run etc. After waiting + * for all running page faults we sync the filesystem which will clean all + * dirty pages and inodes (no new dirty pages or inodes can be created when + * sync is running). + * + * SB_FREEZE_FS: The file system is frozen. Now all internal sources of fs + * modification are blocked (e.g. XFS preallocation truncation on inode + * reclaim). This is usually implemented by blocking new transactions for + * filesystems that have them and need this additional guard. After all + * internal writers are finished we call ->freeze_fs() to finish filesystem + * freezing. Then we transition to SB_FREEZE_COMPLETE state. This state is + * mostly auxiliary for filesystems to verify they do not modify frozen fs. + * + * sb->s_writers.frozen is protected by sb->s_umount. + * + * Return: If the freeze was successful zero is returned. If the freeze + * failed a negative error code is returned. + */ +int freeze_super(struct super_block *sb, enum freeze_holder who, const void *freeze_owner) +{ + int ret; + + if (!super_lock_excl(sb)) { + WARN_ONCE(1, "Dying superblock while freezing!"); + return -EINVAL; + } + atomic_inc(&sb->s_active); + +retry: + if (sb->s_writers.frozen == SB_FREEZE_COMPLETE) { + if (may_freeze(sb, who, freeze_owner)) + ret = !!WARN_ON_ONCE(freeze_inc(sb, who) == 1); + else + ret = -EBUSY; + /* All freezers share a single active reference. */ + deactivate_locked_super(sb); + return ret; + } + + if (sb->s_writers.frozen != SB_UNFROZEN) { + ret = wait_for_partially_frozen(sb); + if (ret) { + deactivate_locked_super(sb); + return ret; + } + + goto retry; + } + + if (sb_rdonly(sb)) { + /* Nothing to do really... */ + WARN_ON_ONCE(freeze_inc(sb, who) > 1); + sb->s_writers.freeze_owner = freeze_owner; + sb->s_writers.frozen = SB_FREEZE_COMPLETE; + wake_up_var(&sb->s_writers.frozen); + super_unlock_excl(sb); + return 0; + } + + sb->s_writers.frozen = SB_FREEZE_WRITE; + /* Release s_umount to preserve sb_start_write -> s_umount ordering */ + super_unlock_excl(sb); + sb_wait_write(sb, SB_FREEZE_WRITE); + __super_lock_excl(sb); + + /* Now we go and block page faults... */ + sb->s_writers.frozen = SB_FREEZE_PAGEFAULT; + sb_wait_write(sb, SB_FREEZE_PAGEFAULT); + + /* All writers are done so after syncing there won't be dirty data */ + ret = sync_filesystem(sb); + if (ret) { + sb->s_writers.frozen = SB_UNFROZEN; + sb_freeze_unlock(sb, SB_FREEZE_PAGEFAULT); + wake_up_var(&sb->s_writers.frozen); + deactivate_locked_super(sb); + return ret; + } + + /* Now wait for internal filesystem counter */ + sb->s_writers.frozen = SB_FREEZE_FS; + sb_wait_write(sb, SB_FREEZE_FS); + + if (sb->s_op->freeze_fs) { + ret = sb->s_op->freeze_fs(sb); + if (ret) { + printk(KERN_ERR + "VFS:Filesystem freeze failed\n"); + sb->s_writers.frozen = SB_UNFROZEN; + sb_freeze_unlock(sb, SB_FREEZE_FS); + wake_up_var(&sb->s_writers.frozen); + deactivate_locked_super(sb); + return ret; + } + } + /* + * For debugging purposes so that fs can warn if it sees write activity + * when frozen is set to SB_FREEZE_COMPLETE, and for thaw_super(). + */ + WARN_ON_ONCE(freeze_inc(sb, who) > 1); + sb->s_writers.freeze_owner = freeze_owner; + sb->s_writers.frozen = SB_FREEZE_COMPLETE; + wake_up_var(&sb->s_writers.frozen); + lockdep_sb_freeze_release(sb); + super_unlock_excl(sb); + return 0; +} +EXPORT_SYMBOL(freeze_super); + +/* + * Undoes the effect of a freeze_super_locked call. If the filesystem is + * frozen both by userspace and the kernel, a thaw call from either source + * removes that state without releasing the other state or unlocking the + * filesystem. + */ +static int thaw_super_locked(struct super_block *sb, enum freeze_holder who, + const void *freeze_owner) +{ + int error = -EINVAL; + + if (sb->s_writers.frozen != SB_FREEZE_COMPLETE) + goto out_unlock; + + if (!may_unfreeze(sb, who, freeze_owner)) + goto out_unlock; + + /* + * All freezers share a single active reference. + * So just unlock in case there are any left. + */ + if (freeze_dec(sb, who)) + goto out_unlock; + + if (sb_rdonly(sb)) { + sb->s_writers.frozen = SB_UNFROZEN; + sb->s_writers.freeze_owner = NULL; + wake_up_var(&sb->s_writers.frozen); + goto out_deactivate; + } + + lockdep_sb_freeze_acquire(sb); + + if (sb->s_op->unfreeze_fs) { + error = sb->s_op->unfreeze_fs(sb); + if (error) { + pr_err("VFS: Filesystem thaw failed\n"); + freeze_inc(sb, who); + lockdep_sb_freeze_release(sb); + goto out_unlock; + } + } + + sb->s_writers.frozen = SB_UNFROZEN; + sb->s_writers.freeze_owner = NULL; + wake_up_var(&sb->s_writers.frozen); + sb_freeze_unlock(sb, SB_FREEZE_FS); +out_deactivate: + deactivate_locked_super(sb); + return 0; + +out_unlock: + super_unlock_excl(sb); + return error; +} + +/** + * thaw_super -- unlock filesystem + * @sb: the super to thaw + * @who: context that wants to freeze + * @freeze_owner: owner of the freeze + * + * Unlocks the filesystem and marks it writeable again after freeze_super() + * if there are no remaining freezes on the filesystem. + * + * @who should be: + * * %FREEZE_HOLDER_USERSPACE if userspace wants to thaw the fs; + * * %FREEZE_HOLDER_KERNEL if the kernel wants to thaw the fs. + * * %FREEZE_MAY_NEST whether nesting freeze and thaw requests is allowed + * + * A filesystem may hold multiple devices and thus a filesystems may + * have been frozen through the block layer via multiple block devices. + * The filesystem remains frozen until all block devices are unfrozen. + */ +int thaw_super(struct super_block *sb, enum freeze_holder who, + const void *freeze_owner) +{ + if (!super_lock_excl(sb)) { + WARN_ONCE(1, "Dying superblock while thawing!"); + return -EINVAL; + } + return thaw_super_locked(sb, who, freeze_owner); +} +EXPORT_SYMBOL(thaw_super); + +/* + * Create workqueue for deferred direct IO completions. We allocate the + * workqueue when it's first needed. This avoids creating workqueue for + * filesystems that don't need it and also allows us to create the workqueue + * late enough so the we can include s_id in the name of the workqueue. + */ +int sb_init_dio_done_wq(struct super_block *sb) +{ + struct workqueue_struct *old; + struct workqueue_struct *wq = alloc_workqueue("dio/%s", + WQ_MEM_RECLAIM | WQ_PERCPU, + 0, + sb->s_id); + if (!wq) + return -ENOMEM; + + old = NULL; + /* + * This has to be atomic as more DIOs can race to create the workqueue + */ + if (!try_cmpxchg(&sb->s_dio_done_wq, &old, wq)) { + /* Someone created workqueue before us? Free ours... */ + destroy_workqueue(wq); + } + return 0; +} +EXPORT_SYMBOL_GPL(sb_init_dio_done_wq); diff --git a/fs/sync.c b/fs/sync.c new file mode 100644 index 000000000..4a84dd837 --- /dev/null +++ b/fs/sync.c @@ -0,0 +1,380 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * High-level sync()-related operations + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +#define VALID_FLAGS (SYNC_FILE_RANGE_WAIT_BEFORE|SYNC_FILE_RANGE_WRITE| \ + SYNC_FILE_RANGE_WAIT_AFTER) + +/* + * Write out and wait upon all dirty data associated with this + * superblock. Filesystem data as well as the underlying block + * device. Takes the superblock lock. + */ +int sync_filesystem(struct super_block *sb) +{ + int ret = 0; + + /* + * We need to be protected against the filesystem going from + * r/o to r/w or vice versa. + */ + WARN_ON(!rwsem_is_locked(&sb->s_umount)); + + /* + * No point in syncing out anything if the filesystem is read-only. + */ + if (sb_rdonly(sb)) + return 0; + + /* + * Do the filesystem syncing work. For simple filesystems + * writeback_inodes_sb(sb) just dirties buffers with inodes so we have + * to submit I/O for these buffers via sync_blockdev(). This also + * speeds up the wait == 1 case since in that case write_inode() + * methods call sync_dirty_buffer() and thus effectively write one block + * at a time. + */ + writeback_inodes_sb(sb, WB_REASON_SYNC); + if (sb->s_op->sync_fs) { + ret = sb->s_op->sync_fs(sb, 0); + if (ret) + return ret; + } + ret = sync_blockdev_nowait(sb->s_bdev); + if (ret) + return ret; + + sync_inodes_sb(sb); + if (sb->s_op->sync_fs) { + ret = sb->s_op->sync_fs(sb, 1); + if (ret) + return ret; + } + return sync_blockdev(sb->s_bdev); +} +EXPORT_SYMBOL(sync_filesystem); + +static void sync_inodes_one_sb(struct super_block *sb, void *arg) +{ + if (!sb_rdonly(sb)) + sync_inodes_sb(sb); +} + +static void sync_fs_one_sb(struct super_block *sb, void *arg) +{ + if (!sb_rdonly(sb) && !(sb->s_iflags & SB_I_SKIP_SYNC) && + sb->s_op->sync_fs) + sb->s_op->sync_fs(sb, *(int *)arg); +} + +/* + * Sync everything. We start by waking flusher threads so that most of + * writeback runs on all devices in parallel. Then we sync all inodes reliably + * which effectively also waits for all flusher threads to finish doing + * writeback. At this point all data is on disk so metadata should be stable + * and we tell filesystems to sync their metadata via ->sync_fs() calls. + * Finally, we writeout all block devices because some filesystems (e.g. ext2) + * just write metadata (such as inodes or bitmaps) to block device page cache + * and do not sync it on their own in ->sync_fs(). + */ +void ksys_sync(void) +{ + int nowait = 0, wait = 1; + + wakeup_flusher_threads(WB_REASON_SYNC); + iterate_supers(sync_inodes_one_sb, NULL); + iterate_supers(sync_fs_one_sb, &nowait); + iterate_supers(sync_fs_one_sb, &wait); + sync_bdevs(false); + sync_bdevs(true); +} + +SYSCALL_DEFINE0(sync) +{ + ksys_sync(); + return 0; +} + +static void do_sync_work(struct work_struct *work) +{ + int nowait = 0; + int wait = 1; + + /* + * Sync twice to reduce the possibility we skipped some inodes / pages + * because they were temporarily locked + */ + iterate_supers(sync_inodes_one_sb, NULL); + iterate_supers(sync_fs_one_sb, &nowait); + sync_bdevs(false); + iterate_supers(sync_inodes_one_sb, NULL); + iterate_supers(sync_fs_one_sb, &wait); + sync_bdevs(false); + printk("Emergency Sync complete\n"); + kfree(work); +} + +void emergency_sync(void) +{ + struct work_struct *work; + + work = kmalloc_obj(*work, GFP_ATOMIC); + if (work) { + INIT_WORK(work, do_sync_work); + schedule_work(work); + } +} + +/* + * sync a single super + */ +SYSCALL_DEFINE1(syncfs, int, fd) +{ + CLASS(fd, f)(fd); + struct super_block *sb; + int ret, ret2; + + if (fd_empty(f)) + return -EBADF; + sb = fd_file(f)->f_path.dentry->d_sb; + + down_read(&sb->s_umount); + ret = sync_filesystem(sb); + up_read(&sb->s_umount); + + ret2 = errseq_check_and_advance(&sb->s_wb_err, &fd_file(f)->f_sb_err); + + return ret ? ret : ret2; +} + +/** + * vfs_fsync_range - helper to sync a range of data & metadata to disk + * @file: file to sync + * @start: offset in bytes of the beginning of data range to sync + * @end: offset in bytes of the end of data range (inclusive) + * @datasync: perform only datasync + * + * Write back data in range @start..@end and metadata for @file to disk. If + * @datasync is set only metadata needed to access modified file data is + * written. + */ +int vfs_fsync_range(struct file *file, loff_t start, loff_t end, int datasync) +{ + struct inode *inode = file->f_mapping->host; + + if (!file->f_op->fsync) + return -EINVAL; + if (!datasync) + sync_lazytime(inode); + return file->f_op->fsync(file, start, end, datasync); +} +EXPORT_SYMBOL(vfs_fsync_range); + +/** + * vfs_fsync - perform a fsync or fdatasync on a file + * @file: file to sync + * @datasync: only perform a fdatasync operation + * + * Write back data and metadata for @file to disk. If @datasync is + * set only metadata needed to access modified file data is written. + */ +int vfs_fsync(struct file *file, int datasync) +{ + return vfs_fsync_range(file, 0, LLONG_MAX, datasync); +} +EXPORT_SYMBOL(vfs_fsync); + +static int do_fsync(unsigned int fd, int datasync) +{ + CLASS(fd, f)(fd); + + if (fd_empty(f)) + return -EBADF; + + return vfs_fsync(fd_file(f), datasync); +} + +SYSCALL_DEFINE1(fsync, unsigned int, fd) +{ + return do_fsync(fd, 0); +} + +SYSCALL_DEFINE1(fdatasync, unsigned int, fd) +{ + return do_fsync(fd, 1); +} + +int sync_file_range(struct file *file, loff_t offset, loff_t nbytes, + unsigned int flags) +{ + int ret; + struct address_space *mapping; + loff_t endbyte; /* inclusive */ + umode_t i_mode; + + ret = -EINVAL; + if (flags & ~VALID_FLAGS) + goto out; + + endbyte = offset + nbytes; + + if ((s64)offset < 0) + goto out; + if ((s64)endbyte < 0) + goto out; + if (endbyte < offset) + goto out; + + if (sizeof(pgoff_t) == 4) { + if (offset >= (0x100000000ULL << PAGE_SHIFT)) { + /* + * The range starts outside a 32 bit machine's + * pagecache addressing capabilities. Let it "succeed" + */ + ret = 0; + goto out; + } + if (endbyte >= (0x100000000ULL << PAGE_SHIFT)) { + /* + * Out to EOF + */ + nbytes = 0; + } + } + + if (nbytes == 0) + endbyte = LLONG_MAX; + else + endbyte--; /* inclusive */ + + i_mode = file_inode(file)->i_mode; + ret = -ESPIPE; + if (!S_ISREG(i_mode) && !S_ISBLK(i_mode) && !S_ISDIR(i_mode)) + goto out; + + mapping = file->f_mapping; + ret = 0; + if (flags & SYNC_FILE_RANGE_WAIT_BEFORE) { + ret = file_fdatawait_range(file, offset, endbyte); + if (ret < 0) + goto out; + } + + if (flags & SYNC_FILE_RANGE_WRITE) { + if ((flags & SYNC_FILE_RANGE_WRITE_AND_WAIT) == + SYNC_FILE_RANGE_WRITE_AND_WAIT) + ret = filemap_fdatawrite_range(mapping, offset, + endbyte); + else + ret = filemap_flush_range(mapping, offset, endbyte); + if (ret < 0) + goto out; + } + + if (flags & SYNC_FILE_RANGE_WAIT_AFTER) + ret = file_fdatawait_range(file, offset, endbyte); + +out: + return ret; +} + +/* + * ksys_sync_file_range() permits finely controlled syncing over a segment of + * a file in the range offset .. (offset+nbytes-1) inclusive. If nbytes is + * zero then ksys_sync_file_range() will operate from offset out to EOF. + * + * The flag bits are: + * + * SYNC_FILE_RANGE_WAIT_BEFORE: wait upon writeout of all pages in the range + * before performing the write. + * + * SYNC_FILE_RANGE_WRITE: initiate writeout of all those dirty pages in the + * range which are not presently under writeback. Note that this may block for + * significant periods due to exhaustion of disk request structures. + * + * SYNC_FILE_RANGE_WAIT_AFTER: wait upon writeout of all pages in the range + * after performing the write. + * + * Useful combinations of the flag bits are: + * + * SYNC_FILE_RANGE_WAIT_BEFORE|SYNC_FILE_RANGE_WRITE: ensures that all pages + * in the range which were dirty on entry to ksys_sync_file_range() are placed + * under writeout. This is a start-write-for-data-integrity operation. + * + * SYNC_FILE_RANGE_WRITE: start writeout of all dirty pages in the range which + * are not presently under writeout. This is an asynchronous flush-to-disk + * operation. Not suitable for data integrity operations. + * + * SYNC_FILE_RANGE_WAIT_BEFORE (or SYNC_FILE_RANGE_WAIT_AFTER): wait for + * completion of writeout of all pages in the range. This will be used after an + * earlier SYNC_FILE_RANGE_WAIT_BEFORE|SYNC_FILE_RANGE_WRITE operation to wait + * for that operation to complete and to return the result. + * + * SYNC_FILE_RANGE_WAIT_BEFORE|SYNC_FILE_RANGE_WRITE|SYNC_FILE_RANGE_WAIT_AFTER + * (a.k.a. SYNC_FILE_RANGE_WRITE_AND_WAIT): + * a traditional sync() operation. This is a write-for-data-integrity operation + * which will ensure that all pages in the range which were dirty on entry to + * ksys_sync_file_range() are written to disk. It should be noted that disk + * caches are not flushed by this call, so there are no guarantees here that the + * data will be available on disk after a crash. + * + * + * SYNC_FILE_RANGE_WAIT_BEFORE and SYNC_FILE_RANGE_WAIT_AFTER will detect any + * I/O errors or ENOSPC conditions and will return those to the caller, after + * clearing the EIO and ENOSPC flags in the address_space. + * + * It should be noted that none of these operations write out the file's + * metadata. So unless the application is strictly performing overwrites of + * already-instantiated disk blocks, there are no guarantees here that the data + * will be available after a crash. + */ +int ksys_sync_file_range(int fd, loff_t offset, loff_t nbytes, + unsigned int flags) +{ + CLASS(fd, f)(fd); + + if (fd_empty(f)) + return -EBADF; + + return sync_file_range(fd_file(f), offset, nbytes, flags); +} + +SYSCALL_DEFINE4(sync_file_range, int, fd, loff_t, offset, loff_t, nbytes, + unsigned int, flags) +{ + return ksys_sync_file_range(fd, offset, nbytes, flags); +} + +#if defined(CONFIG_COMPAT) && defined(__ARCH_WANT_COMPAT_SYNC_FILE_RANGE) +COMPAT_SYSCALL_DEFINE6(sync_file_range, int, fd, compat_arg_u64_dual(offset), + compat_arg_u64_dual(nbytes), unsigned int, flags) +{ + return ksys_sync_file_range(fd, compat_arg_u64_glue(offset), + compat_arg_u64_glue(nbytes), flags); +} +#endif + +/* It would be nice if people remember that not all the world's an i386 + when they introduce new system calls */ +SYSCALL_DEFINE4(sync_file_range2, int, fd, unsigned int, flags, + loff_t, offset, loff_t, nbytes) +{ + return ksys_sync_file_range(fd, offset, nbytes, flags); +} diff --git a/fs/sysctls.c b/fs/sysctls.c new file mode 100644 index 000000000..ad429dffe --- /dev/null +++ b/fs/sysctls.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * /proc/sys/fs shared sysctls + * + * These sysctls are shared between different filesystems. + */ +#include +#include + +static const struct ctl_table fs_shared_sysctls[] = { + { + .procname = "overflowuid", + .data = &fs_overflowuid, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_MAXOLDUID, + }, + { + .procname = "overflowgid", + .data = &fs_overflowgid, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_MAXOLDUID, + }, +}; + +static int __init init_fs_sysctls(void) +{ + register_sysctl_init("fs", fs_shared_sysctls); + return 0; +} + +early_initcall(init_fs_sysctls); diff --git a/fs/sysfs/Kconfig b/fs/sysfs/Kconfig new file mode 100644 index 000000000..b0fe1cce3 --- /dev/null +++ b/fs/sysfs/Kconfig @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: GPL-2.0-only +config SYSFS + bool "sysfs file system support" if EXPERT + default y + select KERNFS + help + The sysfs filesystem is a virtual filesystem that the kernel uses to + export internal kernel objects, their attributes, and their + relationships to one another. + + Users can use sysfs to ascertain useful information about the running + kernel, such as the devices the kernel has discovered on each bus and + which driver each is bound to. sysfs can also be used to tune devices + and other kernel subsystems. + + Some system agents rely on the information in sysfs to operate. + /sbin/hotplug uses device and object attributes in sysfs to assist in + delegating policy decisions, like persistently naming devices. + + sysfs is currently used by the block subsystem to mount the root + partition. If sysfs is disabled you must specify the boot device on + the kernel boot command line via its major and minor numbers. For + example, "root=03:01" for /dev/hda1. + + Designers of embedded systems may wish to say N here to conserve space. diff --git a/fs/sysfs/Makefile b/fs/sysfs/Makefile new file mode 100644 index 000000000..0906b9e52 --- /dev/null +++ b/fs/sysfs/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the sysfs virtual filesystem +# + +obj-y := file.o dir.o symlink.o mount.o group.o diff --git a/fs/sysfs/dir.c b/fs/sysfs/dir.c new file mode 100644 index 000000000..ffdcd4153 --- /dev/null +++ b/fs/sysfs/dir.c @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/sysfs/dir.c - sysfs core and dir operation implementation + * + * Copyright (c) 2001-3 Patrick Mochel + * Copyright (c) 2007 SUSE Linux Products GmbH + * Copyright (c) 2007 Tejun Heo + * + * Please see Documentation/filesystems/sysfs.rst for more information. + */ + +#define pr_fmt(fmt) "sysfs: " fmt + +#include +#include +#include +#include "sysfs.h" + +DEFINE_SPINLOCK(sysfs_symlink_target_lock); + +void sysfs_warn_dup(struct kernfs_node *parent, const char *name) +{ + char *buf; + + buf = kzalloc(PATH_MAX, GFP_KERNEL); + if (buf) + kernfs_path(parent, buf, PATH_MAX); + + pr_warn("cannot create duplicate filename '%s/%s'\n", buf, name); + dump_stack(); + + kfree(buf); +} + +/** + * sysfs_create_dir_ns - create a directory for an object with a namespace tag + * @kobj: object we're creating directory for + * @ns: the namespace tag to use + */ +int sysfs_create_dir_ns(struct kobject *kobj, const struct ns_common *ns) +{ + struct kernfs_node *parent, *kn; + kuid_t uid; + kgid_t gid; + + if (WARN_ON(!kobj)) + return -EINVAL; + + if (kobj->parent) + parent = kobj->parent->sd; + else + parent = sysfs_root_kn; + + if (!parent) + return -ENOENT; + + kobject_get_ownership(kobj, &uid, &gid); + + kn = kernfs_create_dir_ns(parent, kobject_name(kobj), 0755, uid, gid, + kobj, ns); + if (IS_ERR(kn)) { + if (PTR_ERR(kn) == -EEXIST) + sysfs_warn_dup(parent, kobject_name(kobj)); + return PTR_ERR(kn); + } + + kobj->sd = kn; + return 0; +} + +/** + * sysfs_remove_dir - remove an object's directory. + * @kobj: object. + * + * The only thing special about this is that we remove any files in + * the directory before we remove the directory, and we've inlined + * what used to be sysfs_rmdir() below, instead of calling separately. + */ +void sysfs_remove_dir(struct kobject *kobj) +{ + struct kernfs_node *kn = kobj->sd; + + /* + * In general, kobject owner is responsible for ensuring removal + * doesn't race with other operations and sysfs doesn't provide any + * protection; however, when @kobj is used as a symlink target, the + * symlinking entity usually doesn't own @kobj and thus has no + * control over removal. @kobj->sd may be removed anytime + * and symlink code may end up dereferencing an already freed node. + * + * sysfs_symlink_target_lock synchronizes @kobj->sd + * disassociation against symlink operations so that symlink code + * can safely dereference @kobj->sd. + */ + spin_lock(&sysfs_symlink_target_lock); + kobj->sd = NULL; + spin_unlock(&sysfs_symlink_target_lock); + + if (kn) { + WARN_ON_ONCE(kernfs_type(kn) != KERNFS_DIR); + kernfs_remove(kn); + } +} + +int sysfs_rename_dir_ns(struct kobject *kobj, const char *new_name, + const struct ns_common *new_ns) +{ + struct kernfs_node *parent; + int ret; + + parent = kernfs_get_parent(kobj->sd); + ret = kernfs_rename_ns(kobj->sd, parent, new_name, new_ns); + kernfs_put(parent); + return ret; +} + +int sysfs_move_dir_ns(struct kobject *kobj, struct kobject *new_parent_kobj, + const struct ns_common *new_ns) +{ + struct kernfs_node *kn = kobj->sd; + struct kernfs_node *new_parent; + + new_parent = new_parent_kobj && new_parent_kobj->sd ? + new_parent_kobj->sd : sysfs_root_kn; + + return kernfs_rename_ns(kn, new_parent, NULL, new_ns); +} + +/** + * sysfs_create_mount_point - create an always empty directory + * @parent_kobj: kobject that will contain this always empty directory + * @name: The name of the always empty directory to add + */ +int sysfs_create_mount_point(struct kobject *parent_kobj, const char *name) +{ + struct kernfs_node *kn, *parent = parent_kobj->sd; + + kn = kernfs_create_empty_dir(parent, name); + if (IS_ERR(kn)) { + if (PTR_ERR(kn) == -EEXIST) + sysfs_warn_dup(parent, name); + return PTR_ERR(kn); + } + + return 0; +} +EXPORT_SYMBOL_GPL(sysfs_create_mount_point); + +/** + * sysfs_remove_mount_point - remove an always empty directory. + * @parent_kobj: kobject that will contain this always empty directory + * @name: The name of the always empty directory to remove + * + */ +void sysfs_remove_mount_point(struct kobject *parent_kobj, const char *name) +{ + struct kernfs_node *parent = parent_kobj->sd; + + kernfs_remove_by_name_ns(parent, name, NULL); +} +EXPORT_SYMBOL_GPL(sysfs_remove_mount_point); diff --git a/fs/sysfs/file.c b/fs/sysfs/file.c new file mode 100644 index 000000000..cd5bb0f9f --- /dev/null +++ b/fs/sysfs/file.c @@ -0,0 +1,821 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/sysfs/file.c - sysfs regular (text) file implementation + * + * Copyright (c) 2001-3 Patrick Mochel + * Copyright (c) 2007 SUSE Linux Products GmbH + * Copyright (c) 2007 Tejun Heo + * + * Please see Documentation/filesystems/sysfs.rst for more information. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "sysfs.h" + +static struct kobject *sysfs_file_kobj(struct kernfs_node *kn) +{ + guard(rcu)(); + return rcu_dereference(kn->__parent)->priv; +} + +/* + * Determine ktype->sysfs_ops for the given kernfs_node. This function + * must be called while holding an active reference. + */ +static const struct sysfs_ops *sysfs_file_ops(struct kernfs_node *kn) +{ + struct kobject *kobj = sysfs_file_kobj(kn); + + if (kn->flags & KERNFS_LOCKDEP) + lockdep_assert_held(kn); + return kobj->ktype ? kobj->ktype->sysfs_ops : NULL; +} + +/* + * Reads on sysfs are handled through seq_file, which takes care of hairy + * details like buffering and seeking. The following function pipes + * sysfs_ops->show() result through seq_file. + */ +static int sysfs_kf_seq_show(struct seq_file *sf, void *v) +{ + struct kernfs_open_file *of = sf->private; + struct kobject *kobj = sysfs_file_kobj(of->kn); + const struct sysfs_ops *ops = sysfs_file_ops(of->kn); + ssize_t count; + char *buf; + + if (WARN_ON_ONCE(!ops->show)) + return -EINVAL; + + /* acquire buffer and ensure that it's >= PAGE_SIZE and clear */ + count = seq_get_buf(sf, &buf); + if (count < PAGE_SIZE) { + seq_commit(sf, -1); + return 0; + } + memset(buf, 0, PAGE_SIZE); + + count = ops->show(kobj, of->kn->priv, buf); + if (count < 0) + return count; + + /* + * The code works fine with PAGE_SIZE return but it's likely to + * indicate truncated result or overflow in normal use cases. + */ + if (count >= PAGE_SIZE) { + WARN(1, "OOB write or bad count %zd at %pS\n", count, ops->show); + /* Try to struggle along */ + count = PAGE_SIZE - 1; + } + seq_commit(sf, count); + return 0; +} + +static ssize_t sysfs_kf_bin_read(struct kernfs_open_file *of, char *buf, + size_t count, loff_t pos) +{ + const struct bin_attribute *battr = of->kn->priv; + struct kobject *kobj = sysfs_file_kobj(of->kn); + loff_t size = file_inode(of->file)->i_size; + + if (!count) + return 0; + + if (size) { + if (pos >= size) + return 0; + if (pos + count > size) + count = size - pos; + } + + if (!battr->read) + return -EIO; + + return battr->read(of->file, kobj, battr, buf, pos, count); +} + +/* kernfs read callback for regular sysfs files with pre-alloc */ +static ssize_t sysfs_kf_read(struct kernfs_open_file *of, char *buf, + size_t count, loff_t pos) +{ + const struct sysfs_ops *ops = sysfs_file_ops(of->kn); + struct kobject *kobj = sysfs_file_kobj(of->kn); + ssize_t len; + + /* + * If buf != of->prealloc_buf, we don't know how + * large it is, so cannot safely pass it to ->show + */ + if (WARN_ON_ONCE(buf != of->prealloc_buf)) + return 0; + len = ops->show(kobj, of->kn->priv, buf); + if (len < 0) + return len; + if (len >= (ssize_t)PAGE_SIZE) { + printk("fill_read_buffer: %pS returned bad count\n", ops->show); + len = PAGE_SIZE - 1; + } + if (pos) { + if (len <= pos) + return 0; + len -= pos; + memmove(buf, buf + pos, len); + } + return min_t(ssize_t, count, len); +} + +/* kernfs write callback for regular sysfs files */ +static ssize_t sysfs_kf_write(struct kernfs_open_file *of, char *buf, + size_t count, loff_t pos) +{ + const struct sysfs_ops *ops = sysfs_file_ops(of->kn); + struct kobject *kobj = sysfs_file_kobj(of->kn); + + if (!count) + return 0; + + return ops->store(kobj, of->kn->priv, buf, count); +} + +/* kernfs write callback for bin sysfs files */ +static ssize_t sysfs_kf_bin_write(struct kernfs_open_file *of, char *buf, + size_t count, loff_t pos) +{ + const struct bin_attribute *battr = of->kn->priv; + struct kobject *kobj = sysfs_file_kobj(of->kn); + loff_t size = file_inode(of->file)->i_size; + + if (size) { + if (size <= pos) + return -EFBIG; + count = min_t(ssize_t, count, size - pos); + } + if (!count) + return 0; + + if (!battr->write) + return -EIO; + + return battr->write(of->file, kobj, battr, buf, pos, count); +} + +static int sysfs_kf_bin_mmap(struct kernfs_open_file *of, + struct vm_area_struct *vma) +{ + const struct bin_attribute *battr = of->kn->priv; + struct kobject *kobj = sysfs_file_kobj(of->kn); + + return battr->mmap(of->file, kobj, battr, vma); +} + +static loff_t sysfs_kf_bin_llseek(struct kernfs_open_file *of, loff_t offset, + int whence) +{ + const struct bin_attribute *battr = of->kn->priv; + struct kobject *kobj = sysfs_file_kobj(of->kn); + + if (battr->llseek) + return battr->llseek(of->file, kobj, battr, offset, whence); + else + return generic_file_llseek(of->file, offset, whence); +} + +static int sysfs_kf_bin_open(struct kernfs_open_file *of) +{ + const struct bin_attribute *battr = of->kn->priv; + + if (battr->f_mapping) + of->file->f_mapping = battr->f_mapping(); + + return 0; +} + +void sysfs_notify(struct kobject *kobj, const char *dir, const char *attr) +{ + struct kernfs_node *kn = kobj->sd, *tmp; + + if (kn && dir) + kn = kernfs_find_and_get(kn, dir); + else + kernfs_get(kn); + + if (kn && attr) { + tmp = kernfs_find_and_get(kn, attr); + kernfs_put(kn); + kn = tmp; + } + + if (kn) { + kernfs_notify(kn); + kernfs_put(kn); + } +} +EXPORT_SYMBOL_GPL(sysfs_notify); + +static const struct kernfs_ops sysfs_file_kfops_empty = { +}; + +static const struct kernfs_ops sysfs_file_kfops_ro = { + .seq_show = sysfs_kf_seq_show, +}; + +static const struct kernfs_ops sysfs_file_kfops_wo = { + .write = sysfs_kf_write, +}; + +static const struct kernfs_ops sysfs_file_kfops_rw = { + .seq_show = sysfs_kf_seq_show, + .write = sysfs_kf_write, +}; + +static const struct kernfs_ops sysfs_prealloc_kfops_ro = { + .read = sysfs_kf_read, + .prealloc = true, +}; + +static const struct kernfs_ops sysfs_prealloc_kfops_wo = { + .write = sysfs_kf_write, + .prealloc = true, +}; + +static const struct kernfs_ops sysfs_prealloc_kfops_rw = { + .read = sysfs_kf_read, + .write = sysfs_kf_write, + .prealloc = true, +}; + +static const struct kernfs_ops sysfs_bin_kfops_ro = { + .read = sysfs_kf_bin_read, +}; + +static const struct kernfs_ops sysfs_bin_kfops_wo = { + .write = sysfs_kf_bin_write, +}; + +static const struct kernfs_ops sysfs_bin_kfops_rw = { + .read = sysfs_kf_bin_read, + .write = sysfs_kf_bin_write, +}; + +static const struct kernfs_ops sysfs_bin_kfops_mmap = { + .read = sysfs_kf_bin_read, + .write = sysfs_kf_bin_write, + .mmap = sysfs_kf_bin_mmap, + .open = sysfs_kf_bin_open, + .llseek = sysfs_kf_bin_llseek, +}; + +int sysfs_add_file_mode_ns(struct kernfs_node *parent, + const struct attribute *attr, umode_t mode, kuid_t uid, + kgid_t gid, const struct ns_common *ns) +{ + struct kobject *kobj = parent->priv; + const struct sysfs_ops *sysfs_ops = kobj->ktype->sysfs_ops; + struct lock_class_key *key = NULL; + const struct kernfs_ops *ops = NULL; + struct kernfs_node *kn; + + /* every kobject with an attribute needs a ktype assigned */ + if (WARN(!sysfs_ops, KERN_ERR + "missing sysfs attribute operations for kobject: %s\n", + kobject_name(kobj))) + return -EINVAL; + + if (mode & SYSFS_PREALLOC) { + if (sysfs_ops->show && sysfs_ops->store) + ops = &sysfs_prealloc_kfops_rw; + else if (sysfs_ops->show) + ops = &sysfs_prealloc_kfops_ro; + else if (sysfs_ops->store) + ops = &sysfs_prealloc_kfops_wo; + } else { + if (sysfs_ops->show && sysfs_ops->store) + ops = &sysfs_file_kfops_rw; + else if (sysfs_ops->show) + ops = &sysfs_file_kfops_ro; + else if (sysfs_ops->store) + ops = &sysfs_file_kfops_wo; + } + + if (!ops) + ops = &sysfs_file_kfops_empty; + +#ifdef CONFIG_DEBUG_LOCK_ALLOC + if (!attr->ignore_lockdep) + key = attr->key ?: (struct lock_class_key *)&attr->skey; +#endif + + kn = __kernfs_create_file(parent, attr->name, mode & 0777, uid, gid, + PAGE_SIZE, ops, (void *)attr, ns, key); + if (IS_ERR(kn)) { + if (PTR_ERR(kn) == -EEXIST) + sysfs_warn_dup(parent, attr->name); + return PTR_ERR(kn); + } + return 0; +} + +int sysfs_add_bin_file_mode_ns(struct kernfs_node *parent, + const struct bin_attribute *battr, umode_t mode, size_t size, + kuid_t uid, kgid_t gid, const struct ns_common *ns) +{ + const struct attribute *attr = &battr->attr; + struct lock_class_key *key = NULL; + const struct kernfs_ops *ops; + struct kernfs_node *kn; + + if (battr->mmap) + ops = &sysfs_bin_kfops_mmap; + else if (battr->read && battr->write) + ops = &sysfs_bin_kfops_rw; + else if (battr->read) + ops = &sysfs_bin_kfops_ro; + else if (battr->write) + ops = &sysfs_bin_kfops_wo; + else + ops = &sysfs_file_kfops_empty; + +#ifdef CONFIG_DEBUG_LOCK_ALLOC + if (!attr->ignore_lockdep) + key = attr->key ?: (struct lock_class_key *)&attr->skey; +#endif + + kn = __kernfs_create_file(parent, attr->name, mode & 0777, uid, gid, + size, ops, (void *)attr, ns, key); + if (IS_ERR(kn)) { + if (PTR_ERR(kn) == -EEXIST) + sysfs_warn_dup(parent, attr->name); + return PTR_ERR(kn); + } + return 0; +} + +/** + * sysfs_create_file_ns - create an attribute file for an object with custom ns + * @kobj: object we're creating for + * @attr: attribute descriptor + * @ns: namespace the new file should belong to + */ +int sysfs_create_file_ns(struct kobject *kobj, const struct attribute *attr, + const struct ns_common *ns) +{ + kuid_t uid; + kgid_t gid; + + if (WARN_ON(!kobj || !kobj->sd || !attr)) + return -EINVAL; + + kobject_get_ownership(kobj, &uid, &gid); + return sysfs_add_file_mode_ns(kobj->sd, attr, attr->mode, uid, gid, ns); +} +EXPORT_SYMBOL_GPL(sysfs_create_file_ns); + +int sysfs_create_files(struct kobject *kobj, const struct attribute * const *ptr) +{ + int err = 0; + int i; + + for (i = 0; ptr[i] && !err; i++) + err = sysfs_create_file(kobj, ptr[i]); + if (err) + while (--i >= 0) + sysfs_remove_file(kobj, ptr[i]); + return err; +} +EXPORT_SYMBOL_GPL(sysfs_create_files); + +/** + * sysfs_add_file_to_group - add an attribute file to a pre-existing group. + * @kobj: object we're acting for. + * @attr: attribute descriptor. + * @group: group name. + */ +int sysfs_add_file_to_group(struct kobject *kobj, + const struct attribute *attr, const char *group) +{ + struct kernfs_node *parent; + kuid_t uid; + kgid_t gid; + int error; + + if (group) { + parent = kernfs_find_and_get(kobj->sd, group); + } else { + parent = kobj->sd; + kernfs_get(parent); + } + + if (!parent) + return -ENOENT; + + kobject_get_ownership(kobj, &uid, &gid); + error = sysfs_add_file_mode_ns(parent, attr, attr->mode, uid, gid, + NULL); + kernfs_put(parent); + + return error; +} +EXPORT_SYMBOL_GPL(sysfs_add_file_to_group); + +/** + * sysfs_chmod_file - update the modified mode value on an object attribute. + * @kobj: object we're acting for. + * @attr: attribute descriptor. + * @mode: file permissions. + * + */ +int sysfs_chmod_file(struct kobject *kobj, const struct attribute *attr, + umode_t mode) +{ + struct kernfs_node *kn; + struct iattr newattrs; + int rc; + + kn = kernfs_find_and_get(kobj->sd, attr->name); + if (!kn) + return -ENOENT; + + newattrs.ia_mode = (mode & S_IALLUGO) | (kn->mode & ~S_IALLUGO); + newattrs.ia_valid = ATTR_MODE; + + rc = kernfs_setattr(kn, &newattrs); + + kernfs_put(kn); + return rc; +} +EXPORT_SYMBOL_GPL(sysfs_chmod_file); + +/** + * sysfs_break_active_protection - break "active" protection + * @kobj: The kernel object @attr is associated with. + * @attr: The attribute to break the "active" protection for. + * + * With sysfs, just like kernfs, deletion of an attribute is postponed until + * all active .show() and .store() callbacks have finished unless this function + * is called. Hence this function is useful in methods that implement self + * deletion. + */ +struct kernfs_node *sysfs_break_active_protection(struct kobject *kobj, + const struct attribute *attr) +{ + struct kernfs_node *kn; + + kobject_get(kobj); + kn = kernfs_find_and_get(kobj->sd, attr->name); + if (kn) + kernfs_break_active_protection(kn); + else + kobject_put(kobj); + return kn; +} +EXPORT_SYMBOL_GPL(sysfs_break_active_protection); + +/** + * sysfs_unbreak_active_protection - restore "active" protection + * @kn: Pointer returned by sysfs_break_active_protection(). + * + * Undo the effects of sysfs_break_active_protection(). Since this function + * calls kernfs_put() on the kernfs node that corresponds to the 'attr' + * argument passed to sysfs_break_active_protection() that attribute may have + * been removed between the sysfs_break_active_protection() and + * sysfs_unbreak_active_protection() calls, it is not safe to access @kn after + * this function has returned. + */ +void sysfs_unbreak_active_protection(struct kernfs_node *kn) +{ + struct kobject *kobj = sysfs_file_kobj(kn); + + kernfs_unbreak_active_protection(kn); + kernfs_put(kn); + kobject_put(kobj); +} +EXPORT_SYMBOL_GPL(sysfs_unbreak_active_protection); + +/** + * sysfs_remove_file_ns - remove an object attribute with a custom ns tag + * @kobj: object we're acting for + * @attr: attribute descriptor + * @ns: namespace tag of the file to remove + * + * Hash the attribute name and namespace tag and kill the victim. + */ +void sysfs_remove_file_ns(struct kobject *kobj, const struct attribute *attr, + const struct ns_common *ns) +{ + struct kernfs_node *parent = kobj->sd; + + kernfs_remove_by_name_ns(parent, attr->name, ns); +} +EXPORT_SYMBOL_GPL(sysfs_remove_file_ns); + +/** + * sysfs_remove_file_self - remove an object attribute from its own method + * @kobj: object we're acting for + * @attr: attribute descriptor + * + * See kernfs_remove_self() for details. + */ +bool sysfs_remove_file_self(struct kobject *kobj, const struct attribute *attr) +{ + struct kernfs_node *parent = kobj->sd; + struct kernfs_node *kn; + bool ret; + + kn = kernfs_find_and_get(parent, attr->name); + if (WARN_ON_ONCE(!kn)) + return false; + + ret = kernfs_remove_self(kn); + + kernfs_put(kn); + return ret; +} +EXPORT_SYMBOL_GPL(sysfs_remove_file_self); + +void sysfs_remove_files(struct kobject *kobj, const struct attribute * const *ptr) +{ + int i; + + for (i = 0; ptr[i]; i++) + sysfs_remove_file(kobj, ptr[i]); +} +EXPORT_SYMBOL_GPL(sysfs_remove_files); + +/** + * sysfs_remove_file_from_group - remove an attribute file from a group. + * @kobj: object we're acting for. + * @attr: attribute descriptor. + * @group: group name. + */ +void sysfs_remove_file_from_group(struct kobject *kobj, + const struct attribute *attr, const char *group) +{ + struct kernfs_node *parent; + + if (group) { + parent = kernfs_find_and_get(kobj->sd, group); + } else { + parent = kobj->sd; + kernfs_get(parent); + } + + if (parent) { + kernfs_remove_by_name(parent, attr->name); + kernfs_put(parent); + } +} +EXPORT_SYMBOL_GPL(sysfs_remove_file_from_group); + +/** + * sysfs_create_bin_file - create binary file for object. + * @kobj: object. + * @attr: attribute descriptor. + */ +int sysfs_create_bin_file(struct kobject *kobj, + const struct bin_attribute *attr) +{ + kuid_t uid; + kgid_t gid; + + if (WARN_ON(!kobj || !kobj->sd || !attr)) + return -EINVAL; + + kobject_get_ownership(kobj, &uid, &gid); + return sysfs_add_bin_file_mode_ns(kobj->sd, attr, attr->attr.mode, + attr->size, uid, gid, NULL); +} +EXPORT_SYMBOL_GPL(sysfs_create_bin_file); + +/** + * sysfs_remove_bin_file - remove binary file for object. + * @kobj: object. + * @attr: attribute descriptor. + */ +void sysfs_remove_bin_file(struct kobject *kobj, + const struct bin_attribute *attr) +{ + kernfs_remove_by_name(kobj->sd, attr->attr.name); +} +EXPORT_SYMBOL_GPL(sysfs_remove_bin_file); + +static int internal_change_owner(struct kernfs_node *kn, kuid_t kuid, + kgid_t kgid) +{ + struct iattr newattrs = { + .ia_valid = ATTR_UID | ATTR_GID, + .ia_uid = kuid, + .ia_gid = kgid, + }; + return kernfs_setattr(kn, &newattrs); +} + +/** + * sysfs_link_change_owner - change owner of a sysfs file. + * @kobj: object of the kernfs_node the symlink is located in. + * @targ: object of the kernfs_node the symlink points to. + * @name: name of the link. + * @kuid: new owner's kuid + * @kgid: new owner's kgid + * + * This function looks up the sysfs symlink entry @name under @kobj and changes + * the ownership to @kuid/@kgid. The symlink is looked up in the namespace of + * @targ. + * + * Returns 0 on success or error code on failure. + */ +int sysfs_link_change_owner(struct kobject *kobj, struct kobject *targ, + const char *name, kuid_t kuid, kgid_t kgid) +{ + struct kernfs_node *kn = NULL; + int error; + + if (!name || !kobj->state_in_sysfs || !targ->state_in_sysfs) + return -EINVAL; + + error = -ENOENT; + kn = kernfs_find_and_get_ns(kobj->sd, name, targ->sd->ns); + if (!kn) + goto out; + + error = -EINVAL; + if (kernfs_type(kn) != KERNFS_LINK) + goto out; + if (kn->symlink.target_kn->priv != targ) + goto out; + + error = internal_change_owner(kn, kuid, kgid); + +out: + kernfs_put(kn); + return error; +} + +/** + * sysfs_file_change_owner - change owner of a sysfs file. + * @kobj: object. + * @name: name of the file to change. + * @kuid: new owner's kuid + * @kgid: new owner's kgid + * + * This function looks up the sysfs entry @name under @kobj and changes the + * ownership to @kuid/@kgid. + * + * Returns 0 on success or error code on failure. + */ +int sysfs_file_change_owner(struct kobject *kobj, const char *name, kuid_t kuid, + kgid_t kgid) +{ + struct kernfs_node *kn; + int error; + + if (!name) + return -EINVAL; + + if (!kobj->state_in_sysfs) + return -EINVAL; + + kn = kernfs_find_and_get(kobj->sd, name); + if (!kn) + return -ENOENT; + + error = internal_change_owner(kn, kuid, kgid); + + kernfs_put(kn); + + return error; +} + +/** + * sysfs_change_owner - change owner of the given object. + * @kobj: object. + * @kuid: new owner's kuid + * @kgid: new owner's kgid + * + * Change the owner of the default directory, files, groups, and attributes of + * @kobj to @kuid/@kgid. Note that sysfs_change_owner mirrors how the sysfs + * entries for a kobject are added by driver core. In summary, + * sysfs_change_owner() takes care of the default directory entry for @kobj, + * the default attributes associated with the ktype of @kobj and the default + * attributes associated with the ktype of @kobj. + * Additional properties not added by driver core have to be changed by the + * driver or subsystem which created them. This is similar to how + * driver/subsystem specific entries are removed. + * + * Returns 0 on success or error code on failure. + */ +int sysfs_change_owner(struct kobject *kobj, kuid_t kuid, kgid_t kgid) +{ + int error; + const struct kobj_type *ktype; + + if (!kobj->state_in_sysfs) + return -EINVAL; + + /* Change the owner of the kobject itself. */ + error = internal_change_owner(kobj->sd, kuid, kgid); + if (error) + return error; + + ktype = get_ktype(kobj); + if (ktype) { + /* + * Change owner of the default groups associated with the + * ktype of @kobj. + */ + error = sysfs_groups_change_owner(kobj, ktype->default_groups, + kuid, kgid); + if (error) + return error; + } + + return 0; +} + +/** + * sysfs_emit - scnprintf equivalent, aware of PAGE_SIZE buffer. + * @buf: start of PAGE_SIZE buffer. + * @fmt: format + * @...: optional arguments to @format + * + * + * Returns number of characters written to @buf. + */ +int sysfs_emit(char *buf, const char *fmt, ...) +{ + va_list args; + int len; + + if (WARN(!buf || offset_in_page(buf), + "invalid sysfs_emit: buf:%p\n", buf)) + return 0; + + va_start(args, fmt); + len = vscnprintf(buf, PAGE_SIZE, fmt, args); + va_end(args); + + return len; +} +EXPORT_SYMBOL_GPL(sysfs_emit); + +/** + * sysfs_emit_at - scnprintf equivalent, aware of PAGE_SIZE buffer. + * @buf: start of PAGE_SIZE buffer. + * @at: offset in @buf to start write in bytes + * @at must be >= 0 && < PAGE_SIZE + * @fmt: format + * @...: optional arguments to @fmt + * + * + * Returns number of characters written starting at &@buf[@at]. + */ +int sysfs_emit_at(char *buf, int at, const char *fmt, ...) +{ + va_list args; + int len; + + if (WARN(!buf || offset_in_page(buf) || at < 0 || at >= PAGE_SIZE, + "invalid sysfs_emit_at: buf:%p at:%d\n", buf, at)) + return 0; + + va_start(args, fmt); + len = vscnprintf(buf + at, PAGE_SIZE - at, fmt, args); + va_end(args); + + return len; +} +EXPORT_SYMBOL_GPL(sysfs_emit_at); + +/** + * sysfs_bin_attr_simple_read - read callback to simply copy from memory. + * @file: attribute file which is being read. + * @kobj: object to which the attribute belongs. + * @attr: attribute descriptor. + * @buf: destination buffer. + * @off: offset in bytes from which to read. + * @count: maximum number of bytes to read. + * + * Simple ->read() callback for bin_attributes backed by a buffer in memory. + * The @private and @size members in struct bin_attribute must be set to the + * buffer's location and size before the bin_attribute is created in sysfs. + * + * Bounds check for @off and @count is done in sysfs_kf_bin_read(). + * Negative value check for @off is done in vfs_setpos() and default_llseek(). + * + * Returns number of bytes written to @buf. + */ +ssize_t sysfs_bin_attr_simple_read(struct file *file, struct kobject *kobj, + const struct bin_attribute *attr, char *buf, + loff_t off, size_t count) +{ + memcpy(buf, attr->private + off, count); + return count; +} +EXPORT_SYMBOL_GPL(sysfs_bin_attr_simple_read); diff --git a/fs/sysfs/group.c b/fs/sysfs/group.c new file mode 100644 index 000000000..4e1e4f18a --- /dev/null +++ b/fs/sysfs/group.c @@ -0,0 +1,638 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/sysfs/group.c - Operations for adding/removing multiple files at once. + * + * Copyright (c) 2003 Patrick Mochel + * Copyright (c) 2003 Open Source Development Lab + * Copyright (c) 2013 Greg Kroah-Hartman + * Copyright (c) 2013 The Linux Foundation + */ + +#include +#include +#include +#include +#include +#include +#include "sysfs.h" + + +static void remove_files(struct kernfs_node *parent, + const struct attribute_group *grp) +{ + struct attribute *const *attr; + const struct bin_attribute *const *bin_attr; + + if (grp->attrs) + for (attr = grp->attrs; *attr; attr++) + kernfs_remove_by_name(parent, (*attr)->name); + if (grp->bin_attrs) + for (bin_attr = grp->bin_attrs; *bin_attr; bin_attr++) + kernfs_remove_by_name(parent, (*bin_attr)->attr.name); +} + +static umode_t __first_visible(const struct attribute_group *grp, struct kobject *kobj) +{ + if (grp->attrs && grp->attrs[0] && grp->is_visible) + return grp->is_visible(kobj, grp->attrs[0], 0); + + if (grp->attrs && grp->attrs[0] && grp->is_visible_const) + return grp->is_visible_const(kobj, grp->attrs[0], 0); + + if (grp->bin_attrs && grp->bin_attrs[0] && grp->is_bin_visible) + return grp->is_bin_visible(kobj, grp->bin_attrs[0], 0); + + return 0; +} + +static int create_files(struct kernfs_node *parent, struct kobject *kobj, + kuid_t uid, kgid_t gid, + const struct attribute_group *grp, int update) +{ + struct attribute *const *attr; + const struct bin_attribute *const *bin_attr; + int error = 0, i; + + if (grp->attrs) { + for (i = 0, attr = grp->attrs; *attr && !error; i++, attr++) { + umode_t mode = (*attr)->mode; + + /* + * In update mode, we're changing the permissions or + * visibility. Do this by first removing then + * re-adding (if required) the file. + */ + if (update) + kernfs_remove_by_name(parent, (*attr)->name); + if (grp->is_visible || grp->is_visible_const) { + if (grp->is_visible) + mode = grp->is_visible(kobj, *attr, i); + else + mode = grp->is_visible_const(kobj, *attr, i); + mode &= ~SYSFS_GROUP_INVISIBLE; + if (!mode) + continue; + } + + WARN(mode & ~(SYSFS_PREALLOC | 0664), + "Attribute %s: Invalid permissions 0%o\n", + (*attr)->name, mode); + + mode &= SYSFS_PREALLOC | 0664; + error = sysfs_add_file_mode_ns(parent, *attr, mode, uid, + gid, NULL); + if (unlikely(error)) + break; + } + if (error) { + remove_files(parent, grp); + goto exit; + } + } + + if (grp->bin_attrs) { + for (i = 0, bin_attr = grp->bin_attrs; *bin_attr; i++, bin_attr++) { + umode_t mode = (*bin_attr)->attr.mode; + size_t size = (*bin_attr)->size; + + if (update) + kernfs_remove_by_name(parent, + (*bin_attr)->attr.name); + if (grp->is_bin_visible) { + mode = grp->is_bin_visible(kobj, *bin_attr, i); + mode &= ~SYSFS_GROUP_INVISIBLE; + if (!mode) + continue; + } + if (grp->bin_size) + size = grp->bin_size(kobj, *bin_attr, i); + + WARN(mode & ~(SYSFS_PREALLOC | 0664), + "Attribute %s: Invalid permissions 0%o\n", + (*bin_attr)->attr.name, mode); + + mode &= SYSFS_PREALLOC | 0664; + error = sysfs_add_bin_file_mode_ns(parent, *bin_attr, + mode, size, uid, gid, + NULL); + if (error) + break; + } + if (error) + remove_files(parent, grp); + } +exit: + return error; +} + + +static int internal_create_group(struct kobject *kobj, int update, + const struct attribute_group *grp) +{ + struct kernfs_node *kn; + kuid_t uid; + kgid_t gid; + int error; + + if (WARN_ON(!kobj || (!update && !kobj->sd))) + return -EINVAL; + + /* Updates may happen before the object has been instantiated */ + if (unlikely(update && !kobj->sd)) + return -EINVAL; + + if (!grp->attrs && !grp->bin_attrs) { + pr_debug("sysfs: (bin_)attrs not set by subsystem for group: %s/%s, skipping\n", + kobj->name, grp->name ?: ""); + return 0; + } + + kobject_get_ownership(kobj, &uid, &gid); + if (grp->name) { + umode_t mode = __first_visible(grp, kobj); + + if (mode & SYSFS_GROUP_INVISIBLE) + mode = 0; + else + mode = S_IRWXU | S_IRUGO | S_IXUGO; + + if (update) { + kn = kernfs_find_and_get(kobj->sd, grp->name); + if (!kn) { + pr_debug("attr grp %s/%s not created yet\n", + kobj->name, grp->name); + /* may have been invisible prior to this update */ + update = 0; + } else if (!mode) { + sysfs_remove_group(kobj, grp); + kernfs_put(kn); + return 0; + } + } + + if (!update) { + if (!mode) + return 0; + kn = kernfs_create_dir_ns(kobj->sd, grp->name, mode, + uid, gid, kobj, NULL); + if (IS_ERR(kn)) { + if (PTR_ERR(kn) == -EEXIST) + sysfs_warn_dup(kobj->sd, grp->name); + return PTR_ERR(kn); + } + } + } else { + kn = kobj->sd; + } + + kernfs_get(kn); + error = create_files(kn, kobj, uid, gid, grp, update); + if (error) { + if (grp->name && !update) + kernfs_remove(kn); + } + kernfs_put(kn); + + if (grp->name && update) + kernfs_put(kn); + + return error; +} + +/** + * sysfs_create_group - given a directory kobject, create an attribute group + * @kobj: The kobject to create the group on + * @grp: The attribute group to create + * + * This function creates a group for the first time. It will explicitly + * warn and error if any of the attribute files being created already exist. + * + * Returns 0 on success or error code on failure. + */ +int sysfs_create_group(struct kobject *kobj, + const struct attribute_group *grp) +{ + return internal_create_group(kobj, 0, grp); +} +EXPORT_SYMBOL_GPL(sysfs_create_group); + +static int internal_create_groups(struct kobject *kobj, int update, + const struct attribute_group *const *groups) +{ + int error = 0; + int i; + + if (!groups) + return 0; + + for (i = 0; groups[i]; i++) { + error = internal_create_group(kobj, update, groups[i]); + if (error) { + while (--i >= 0) + sysfs_remove_group(kobj, groups[i]); + break; + } + } + return error; +} + +/** + * sysfs_create_groups - given a directory kobject, create a bunch of attribute groups + * @kobj: The kobject to create the group on + * @groups: The attribute groups to create, NULL terminated + * + * This function creates a bunch of attribute groups. If an error occurs when + * creating a group, all previously created groups will be removed, unwinding + * everything back to the original state when this function was called. + * It will explicitly warn and error if any of the attribute files being + * created already exist. + * + * Returns 0 on success or error code from sysfs_create_group on failure. + */ +int sysfs_create_groups(struct kobject *kobj, + const struct attribute_group *const *groups) +{ + return internal_create_groups(kobj, 0, groups); +} +EXPORT_SYMBOL_GPL(sysfs_create_groups); + +/** + * sysfs_update_groups - given a directory kobject, create a bunch of attribute groups + * @kobj: The kobject to update the group on + * @groups: The attribute groups to update, NULL terminated + * + * This function update a bunch of attribute groups. If an error occurs when + * updating a group, all previously updated groups will be removed together + * with already existing (not updated) attributes. + * + * Returns 0 on success or error code from sysfs_update_group on failure. + */ +int sysfs_update_groups(struct kobject *kobj, + const struct attribute_group *const *groups) +{ + return internal_create_groups(kobj, 1, groups); +} +EXPORT_SYMBOL_GPL(sysfs_update_groups); + +/** + * sysfs_update_group - given a directory kobject, update an attribute group + * @kobj: The kobject to update the group on + * @grp: The attribute group to update + * + * This function updates an attribute group. Unlike + * sysfs_create_group(), it will explicitly not warn or error if any + * of the attribute files being created already exist. Furthermore, + * if the visibility of the files has changed through the is_visible() + * callback, it will update the permissions and add or remove the + * relevant files. Changing a group's name (subdirectory name under + * kobj's directory in sysfs) is not allowed. + * + * The primary use for this function is to call it after making a change + * that affects group visibility. + * + * Returns 0 on success or error code on failure. + */ +int sysfs_update_group(struct kobject *kobj, + const struct attribute_group *grp) +{ + return internal_create_group(kobj, 1, grp); +} +EXPORT_SYMBOL_GPL(sysfs_update_group); + +/** + * sysfs_remove_group: remove a group from a kobject + * @kobj: kobject to remove the group from + * @grp: group to remove + * + * This function removes a group of attributes from a kobject. The attributes + * previously have to have been created for this group, otherwise it will fail. + */ +void sysfs_remove_group(struct kobject *kobj, + const struct attribute_group *grp) +{ + struct kernfs_node *parent = kobj->sd; + struct kernfs_node *kn; + + if (grp->name) { + kn = kernfs_find_and_get(parent, grp->name); + if (!kn) { + pr_debug("sysfs group '%s' not found for kobject '%s'\n", + grp->name, kobject_name(kobj)); + return; + } + } else { + kn = parent; + kernfs_get(kn); + } + + remove_files(kn, grp); + if (grp->name) + kernfs_remove(kn); + + kernfs_put(kn); +} +EXPORT_SYMBOL_GPL(sysfs_remove_group); + +/** + * sysfs_remove_groups - remove a list of groups + * + * @kobj: The kobject for the groups to be removed from + * @groups: NULL terminated list of groups to be removed + * + * If groups is not NULL, remove the specified groups from the kobject. + */ +void sysfs_remove_groups(struct kobject *kobj, + const struct attribute_group *const *groups) +{ + int i; + + if (!groups) + return; + for (i = 0; groups[i]; i++) + sysfs_remove_group(kobj, groups[i]); +} +EXPORT_SYMBOL_GPL(sysfs_remove_groups); + +/** + * sysfs_merge_group - merge files into a pre-existing named attribute group. + * @kobj: The kobject containing the group. + * @grp: The files to create and the attribute group they belong to. + * + * This function returns an error if the group doesn't exist, the .name field is + * NULL or any of the files already exist in that group, in which case none of + * the new files are created. + */ +int sysfs_merge_group(struct kobject *kobj, + const struct attribute_group *grp) +{ + struct kernfs_node *parent; + kuid_t uid; + kgid_t gid; + int error = 0; + struct attribute *const *attr; + int i; + + parent = kernfs_find_and_get(kobj->sd, grp->name); + if (!parent) + return -ENOENT; + + kobject_get_ownership(kobj, &uid, &gid); + + for ((i = 0, attr = grp->attrs); *attr && !error; (++i, ++attr)) + error = sysfs_add_file_mode_ns(parent, *attr, (*attr)->mode, + uid, gid, NULL); + if (error) { + while (--i >= 0) + kernfs_remove_by_name(parent, (*--attr)->name); + } + kernfs_put(parent); + + return error; +} +EXPORT_SYMBOL_GPL(sysfs_merge_group); + +/** + * sysfs_unmerge_group - remove files from a pre-existing named attribute group. + * @kobj: The kobject containing the group. + * @grp: The files to remove and the attribute group they belong to. + */ +void sysfs_unmerge_group(struct kobject *kobj, + const struct attribute_group *grp) +{ + struct kernfs_node *parent; + struct attribute *const *attr; + + parent = kernfs_find_and_get(kobj->sd, grp->name); + if (parent) { + for (attr = grp->attrs; *attr; ++attr) + kernfs_remove_by_name(parent, (*attr)->name); + kernfs_put(parent); + } +} +EXPORT_SYMBOL_GPL(sysfs_unmerge_group); + +/** + * sysfs_add_link_to_group - add a symlink to an attribute group. + * @kobj: The kobject containing the group. + * @group_name: The name of the group. + * @target: The target kobject of the symlink to create. + * @link_name: The name of the symlink to create. + */ +int sysfs_add_link_to_group(struct kobject *kobj, const char *group_name, + struct kobject *target, const char *link_name) +{ + struct kernfs_node *parent; + int error = 0; + + parent = kernfs_find_and_get(kobj->sd, group_name); + if (!parent) + return -ENOENT; + + error = sysfs_create_link_sd(parent, target, link_name); + kernfs_put(parent); + + return error; +} +EXPORT_SYMBOL_GPL(sysfs_add_link_to_group); + +/** + * sysfs_remove_link_from_group - remove a symlink from an attribute group. + * @kobj: The kobject containing the group. + * @group_name: The name of the group. + * @link_name: The name of the symlink to remove. + */ +void sysfs_remove_link_from_group(struct kobject *kobj, const char *group_name, + const char *link_name) +{ + struct kernfs_node *parent; + + parent = kernfs_find_and_get(kobj->sd, group_name); + if (parent) { + kernfs_remove_by_name(parent, link_name); + kernfs_put(parent); + } +} +EXPORT_SYMBOL_GPL(sysfs_remove_link_from_group); + +/** + * compat_only_sysfs_link_entry_to_kobj - add a symlink to a kobject pointing + * to a group or an attribute + * @kobj: The kobject containing the group. + * @target_kobj: The target kobject. + * @target_name: The name of the target group or attribute. + * @symlink_name: The name of the symlink file (target_name will be + * considered if symlink_name is NULL). + */ +int compat_only_sysfs_link_entry_to_kobj(struct kobject *kobj, + struct kobject *target_kobj, + const char *target_name, + const char *symlink_name) +{ + struct kernfs_node *target; + struct kernfs_node *entry; + struct kernfs_node *link; + + /* + * We don't own @target_kobj and it may be removed at any time. + * Synchronize using sysfs_symlink_target_lock. See sysfs_remove_dir() + * for details. + */ + spin_lock(&sysfs_symlink_target_lock); + target = target_kobj->sd; + if (target) + kernfs_get(target); + spin_unlock(&sysfs_symlink_target_lock); + if (!target) + return -ENOENT; + + entry = kernfs_find_and_get(target, target_name); + if (!entry) { + kernfs_put(target); + return -ENOENT; + } + + if (!symlink_name) + symlink_name = target_name; + + link = kernfs_create_link(kobj->sd, symlink_name, entry); + if (PTR_ERR(link) == -EEXIST) + sysfs_warn_dup(kobj->sd, symlink_name); + + kernfs_put(entry); + kernfs_put(target); + return PTR_ERR_OR_ZERO(link); +} +EXPORT_SYMBOL_GPL(compat_only_sysfs_link_entry_to_kobj); + +static int sysfs_group_attrs_change_owner(struct kobject *kobj, + struct kernfs_node *grp_kn, + const struct attribute_group *grp, + struct iattr *newattrs) +{ + struct kernfs_node *kn; + int error, i; + umode_t mode; + + if (grp->attrs) { + struct attribute *const *attr; + + for (i = 0, attr = grp->attrs; *attr; i++, attr++) { + if (grp->is_visible || grp->is_visible_const) { + if (grp->is_visible) + mode = grp->is_visible(kobj, *attr, i); + else + mode = grp->is_visible_const(kobj, *attr, i); + if (mode & SYSFS_GROUP_INVISIBLE) + break; + if (!mode) + continue; + } + kn = kernfs_find_and_get(grp_kn, (*attr)->name); + if (!kn) + return -ENOENT; + + error = kernfs_setattr(kn, newattrs); + kernfs_put(kn); + if (error) + return error; + } + } + + if (grp->bin_attrs) { + const struct bin_attribute *const *bin_attr; + + for (i = 0, bin_attr = grp->bin_attrs; *bin_attr; i++, bin_attr++) { + if (grp->is_bin_visible) { + mode = grp->is_bin_visible(kobj, *bin_attr, i); + if (mode & SYSFS_GROUP_INVISIBLE) + break; + if (!mode) + continue; + } + kn = kernfs_find_and_get(grp_kn, (*bin_attr)->attr.name); + if (!kn) + return -ENOENT; + + error = kernfs_setattr(kn, newattrs); + kernfs_put(kn); + if (error) + return error; + } + } + + return 0; +} + +/** + * sysfs_group_change_owner - change owner of an attribute group. + * @kobj: The kobject containing the group. + * @grp: The attribute group. + * @kuid: new owner's kuid + * @kgid: new owner's kgid + * + * Returns 0 on success or error code on failure. + */ +int sysfs_group_change_owner(struct kobject *kobj, + const struct attribute_group *grp, kuid_t kuid, + kgid_t kgid) +{ + struct kernfs_node *grp_kn; + int error; + struct iattr newattrs = { + .ia_valid = ATTR_UID | ATTR_GID, + .ia_uid = kuid, + .ia_gid = kgid, + }; + + if (!kobj->state_in_sysfs) + return -EINVAL; + + if (grp->name) { + grp_kn = kernfs_find_and_get(kobj->sd, grp->name); + } else { + kernfs_get(kobj->sd); + grp_kn = kobj->sd; + } + if (!grp_kn) + return -ENOENT; + + error = kernfs_setattr(grp_kn, &newattrs); + if (!error) + error = sysfs_group_attrs_change_owner(kobj, grp_kn, grp, &newattrs); + + kernfs_put(grp_kn); + + return error; +} +EXPORT_SYMBOL_GPL(sysfs_group_change_owner); + +/** + * sysfs_groups_change_owner - change owner of a set of attribute groups. + * @kobj: The kobject containing the groups. + * @groups: The attribute groups. + * @kuid: new owner's kuid + * @kgid: new owner's kgid + * + * Returns 0 on success or error code on failure. + */ +int sysfs_groups_change_owner(struct kobject *kobj, + const struct attribute_group *const *groups, + kuid_t kuid, kgid_t kgid) +{ + int error = 0, i; + + if (!kobj->state_in_sysfs) + return -EINVAL; + + if (!groups) + return 0; + + for (i = 0; groups[i]; i++) { + error = sysfs_group_change_owner(kobj, groups[i], kuid, kgid); + if (error) + break; + } + + return error; +} +EXPORT_SYMBOL_GPL(sysfs_groups_change_owner); diff --git a/fs/sysfs/mount.c b/fs/sysfs/mount.c new file mode 100644 index 000000000..88c10823f --- /dev/null +++ b/fs/sysfs/mount.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/sysfs/symlink.c - operations for initializing and mounting sysfs + * + * Copyright (c) 2001-3 Patrick Mochel + * Copyright (c) 2007 SUSE Linux Products GmbH + * Copyright (c) 2007 Tejun Heo + * + * Please see Documentation/filesystems/sysfs.rst for more information. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sysfs.h" + +static struct kernfs_root *sysfs_root; +struct kernfs_node *sysfs_root_kn; + +static void sysfs_fs_context_free(struct fs_context *fc) +{ + struct kernfs_fs_context *kfc = fc->fs_private; + + if (kfc->ns_tag) + kobj_ns_drop(KOBJ_NS_TYPE_NET, kfc->ns_tag); + kernfs_free_fs_context(fc); + kfree(kfc); +} + +static const struct fs_context_operations sysfs_fs_context_ops = { + .free = sysfs_fs_context_free, + .get_tree = kernfs_get_tree, +}; + +static int sysfs_init_fs_context(struct fs_context *fc) +{ + struct kernfs_fs_context *kfc; + struct ns_common *ns; + + if (!(fc->sb_flags & SB_KERNMOUNT)) { + if (!kobj_ns_current_may_mount(KOBJ_NS_TYPE_NET)) + return -EPERM; + } + + kfc = kzalloc_obj(struct kernfs_fs_context); + if (!kfc) + return -ENOMEM; + + kfc->ns_tag = ns = kobj_ns_grab_current(KOBJ_NS_TYPE_NET); + kfc->root = sysfs_root; + kfc->magic = SYSFS_MAGIC; + fc->fs_private = kfc; + fc->ops = &sysfs_fs_context_ops; + if (ns) { + struct net *netns = to_net_ns(ns); + + put_user_ns(fc->user_ns); + fc->user_ns = get_user_ns(netns->user_ns); + } + fc->global = true; + return 0; +} + +static void sysfs_kill_sb(struct super_block *sb) +{ + struct ns_common *ns = (struct ns_common *)kernfs_super_ns(sb); + + kernfs_kill_sb(sb); + kobj_ns_drop(KOBJ_NS_TYPE_NET, ns); +} + +static struct file_system_type sysfs_fs_type = { + .name = "sysfs", + .init_fs_context = sysfs_init_fs_context, + .kill_sb = sysfs_kill_sb, + .fs_flags = FS_USERNS_MOUNT | FS_USERNS_MOUNT_RESTRICTED, +}; + +int __init sysfs_init(void) +{ + int err; + + sysfs_root = kernfs_create_root(NULL, KERNFS_ROOT_EXTRA_OPEN_PERM_CHECK, + NULL); + if (IS_ERR(sysfs_root)) + return PTR_ERR(sysfs_root); + + sysfs_root_kn = kernfs_root_to_node(sysfs_root); + + err = register_filesystem(&sysfs_fs_type); + if (err) { + kernfs_destroy_root(sysfs_root); + return err; + } + + return 0; +} diff --git a/fs/sysfs/symlink.c b/fs/sysfs/symlink.c new file mode 100644 index 000000000..5f9c05fb1 --- /dev/null +++ b/fs/sysfs/symlink.c @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/sysfs/symlink.c - sysfs symlink implementation + * + * Copyright (c) 2001-3 Patrick Mochel + * Copyright (c) 2007 SUSE Linux Products GmbH + * Copyright (c) 2007 Tejun Heo + * + * Please see Documentation/filesystems/sysfs.rst for more information. + */ + +#include +#include +#include +#include +#include + +#include "sysfs.h" + +static int sysfs_do_create_link_sd(struct kernfs_node *parent, + struct kobject *target_kobj, + const char *name, int warn) +{ + struct kernfs_node *kn, *target = NULL; + + if (WARN_ON(!name || !parent)) + return -EINVAL; + + /* + * We don't own @target_kobj and it may be removed at any time. + * Synchronize using sysfs_symlink_target_lock. See + * sysfs_remove_dir() for details. + */ + spin_lock(&sysfs_symlink_target_lock); + if (target_kobj->sd) { + target = target_kobj->sd; + kernfs_get(target); + } + spin_unlock(&sysfs_symlink_target_lock); + + if (!target) + return -ENOENT; + + kn = kernfs_create_link(parent, name, target); + kernfs_put(target); + + if (!IS_ERR(kn)) + return 0; + + if (warn && PTR_ERR(kn) == -EEXIST) + sysfs_warn_dup(parent, name); + return PTR_ERR(kn); +} + +/** + * sysfs_create_link_sd - create symlink to a given object. + * @kn: directory we're creating the link in. + * @target: object we're pointing to. + * @name: name of the symlink. + */ +int sysfs_create_link_sd(struct kernfs_node *kn, struct kobject *target, + const char *name) +{ + return sysfs_do_create_link_sd(kn, target, name, 1); +} + +static int sysfs_do_create_link(struct kobject *kobj, struct kobject *target, + const char *name, int warn) +{ + struct kernfs_node *parent = NULL; + + if (!kobj) + parent = sysfs_root_kn; + else + parent = kobj->sd; + + if (!parent) + return -EFAULT; + + return sysfs_do_create_link_sd(parent, target, name, warn); +} + +/** + * sysfs_create_link - create symlink between two objects. + * @kobj: object whose directory we're creating the link in. + * @target: object we're pointing to. + * @name: name of the symlink. + */ +int sysfs_create_link(struct kobject *kobj, struct kobject *target, + const char *name) +{ + return sysfs_do_create_link(kobj, target, name, 1); +} +EXPORT_SYMBOL_GPL(sysfs_create_link); + +/** + * sysfs_create_link_nowarn - create symlink between two objects. + * @kobj: object whose directory we're creating the link in. + * @target: object we're pointing to. + * @name: name of the symlink. + * + * This function does the same as sysfs_create_link(), but it + * doesn't warn if the link already exists. + */ +int sysfs_create_link_nowarn(struct kobject *kobj, struct kobject *target, + const char *name) +{ + return sysfs_do_create_link(kobj, target, name, 0); +} +EXPORT_SYMBOL_GPL(sysfs_create_link_nowarn); + +/** + * sysfs_delete_link - remove symlink in object's directory. + * @kobj: object we're acting for. + * @targ: object we're pointing to. + * @name: name of the symlink to remove. + * + * Unlike sysfs_remove_link sysfs_delete_link has enough information + * to successfully delete symlinks in tagged directories. + */ +void sysfs_delete_link(struct kobject *kobj, struct kobject *targ, + const char *name) +{ + const struct ns_common *ns = NULL; + + /* + * We don't own @target and it may be removed at any time. + * Synchronize using sysfs_symlink_target_lock. See + * sysfs_remove_dir() for details. + */ + spin_lock(&sysfs_symlink_target_lock); + if (targ->sd && kernfs_ns_enabled(kobj->sd)) + ns = targ->sd->ns; + spin_unlock(&sysfs_symlink_target_lock); + kernfs_remove_by_name_ns(kobj->sd, name, ns); +} + +/** + * sysfs_remove_link - remove symlink in object's directory. + * @kobj: object we're acting for. + * @name: name of the symlink to remove. + */ +void sysfs_remove_link(struct kobject *kobj, const char *name) +{ + struct kernfs_node *parent = NULL; + + if (!kobj) + parent = sysfs_root_kn; + else + parent = kobj->sd; + + kernfs_remove_by_name(parent, name); +} +EXPORT_SYMBOL_GPL(sysfs_remove_link); + +/** + * sysfs_rename_link_ns - rename symlink in object's directory. + * @kobj: object we're acting for. + * @targ: object we're pointing to. + * @old: previous name of the symlink. + * @new: new name of the symlink. + * @new_ns: new namespace of the symlink. + * + * A helper function for the common rename symlink idiom. + */ +int sysfs_rename_link_ns(struct kobject *kobj, struct kobject *targ, + const char *old, const char *new, + const struct ns_common *new_ns) +{ + struct kernfs_node *parent, *kn = NULL; + const struct ns_common *old_ns = NULL; + int result; + + if (!kobj) + parent = sysfs_root_kn; + else + parent = kobj->sd; + + if (targ->sd) + old_ns = targ->sd->ns; + + result = -ENOENT; + kn = kernfs_find_and_get_ns(parent, old, old_ns); + if (!kn) + goto out; + + result = -EINVAL; + if (kernfs_type(kn) != KERNFS_LINK) + goto out; + if (kn->symlink.target_kn->priv != targ) + goto out; + + result = kernfs_rename_ns(kn, parent, new, new_ns); + +out: + kernfs_put(kn); + return result; +} +EXPORT_SYMBOL_GPL(sysfs_rename_link_ns); diff --git a/fs/sysfs/sysfs.h b/fs/sysfs/sysfs.h new file mode 100644 index 000000000..f4583dcaf --- /dev/null +++ b/fs/sysfs/sysfs.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * fs/sysfs/sysfs.h - sysfs internal header file + * + * Copyright (c) 2001-3 Patrick Mochel + * Copyright (c) 2007 SUSE Linux Products GmbH + * Copyright (c) 2007 Tejun Heo + */ + +#ifndef __SYSFS_INTERNAL_H +#define __SYSFS_INTERNAL_H + +#include + +/* + * mount.c + */ +extern struct kernfs_node *sysfs_root_kn; + +/* + * dir.c + */ +extern spinlock_t sysfs_symlink_target_lock; + +void sysfs_warn_dup(struct kernfs_node *parent, const char *name); + +/* + * file.c + */ +int sysfs_add_file_mode_ns(struct kernfs_node *parent, + const struct attribute *attr, umode_t amode, kuid_t uid, + kgid_t gid, const struct ns_common *ns); +int sysfs_add_bin_file_mode_ns(struct kernfs_node *parent, + const struct bin_attribute *battr, umode_t mode, size_t size, + kuid_t uid, kgid_t gid, const struct ns_common *ns); + +/* + * symlink.c + */ +int sysfs_create_link_sd(struct kernfs_node *kn, struct kobject *target, + const char *name); + +#endif /* __SYSFS_INTERNAL_H */ diff --git a/fs/tests/binfmt_elf_kunit.c b/fs/tests/binfmt_elf_kunit.c new file mode 100644 index 000000000..11d734fec --- /dev/null +++ b/fs/tests/binfmt_elf_kunit.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include + +static void total_mapping_size_test(struct kunit *test) +{ + struct elf_phdr empty[] = { + { .p_type = PT_LOAD, .p_vaddr = 0, .p_memsz = 0, }, + { .p_type = PT_INTERP, .p_vaddr = 10, .p_memsz = 999999, }, + }; + /* + * readelf -lW /bin/mount | grep '^ .*0x0' | awk '{print "\t\t{ .p_type = PT_" \ + * $1 ", .p_vaddr = " $3 ", .p_memsz = " $6 ", },"}' + */ + struct elf_phdr mount[] = { + { .p_type = PT_PHDR, .p_vaddr = 0x00000040, .p_memsz = 0x0002d8, }, + { .p_type = PT_INTERP, .p_vaddr = 0x00000318, .p_memsz = 0x00001c, }, + { .p_type = PT_LOAD, .p_vaddr = 0x00000000, .p_memsz = 0x0033a8, }, + { .p_type = PT_LOAD, .p_vaddr = 0x00004000, .p_memsz = 0x005c91, }, + { .p_type = PT_LOAD, .p_vaddr = 0x0000a000, .p_memsz = 0x0022f8, }, + { .p_type = PT_LOAD, .p_vaddr = 0x0000d330, .p_memsz = 0x000d40, }, + { .p_type = PT_DYNAMIC, .p_vaddr = 0x0000d928, .p_memsz = 0x000200, }, + { .p_type = PT_NOTE, .p_vaddr = 0x00000338, .p_memsz = 0x000030, }, + { .p_type = PT_NOTE, .p_vaddr = 0x00000368, .p_memsz = 0x000044, }, + { .p_type = PT_GNU_PROPERTY, .p_vaddr = 0x00000338, .p_memsz = 0x000030, }, + { .p_type = PT_GNU_EH_FRAME, .p_vaddr = 0x0000b490, .p_memsz = 0x0001ec, }, + { .p_type = PT_GNU_STACK, .p_vaddr = 0x00000000, .p_memsz = 0x000000, }, + { .p_type = PT_GNU_RELRO, .p_vaddr = 0x0000d330, .p_memsz = 0x000cd0, }, + }; + size_t mount_size = 0xE070; + /* https://lore.kernel.org/linux-fsdevel/YfF18Dy85mCntXrx@fractal.localdomain */ + struct elf_phdr unordered[] = { + { .p_type = PT_LOAD, .p_vaddr = 0x00000000, .p_memsz = 0x0033a8, }, + { .p_type = PT_LOAD, .p_vaddr = 0x0000d330, .p_memsz = 0x000d40, }, + { .p_type = PT_LOAD, .p_vaddr = 0x00004000, .p_memsz = 0x005c91, }, + { .p_type = PT_LOAD, .p_vaddr = 0x0000a000, .p_memsz = 0x0022f8, }, + }; + + /* No headers, no size. */ + KUNIT_EXPECT_EQ(test, total_mapping_size(NULL, 0), 0); + KUNIT_EXPECT_EQ(test, total_mapping_size(empty, 0), 0); + /* Empty headers, no size. */ + KUNIT_EXPECT_EQ(test, total_mapping_size(empty, 1), 0); + /* No PT_LOAD headers, no size. */ + KUNIT_EXPECT_EQ(test, total_mapping_size(&empty[1], 1), 0); + /* Empty PT_LOAD and non-PT_LOAD headers, no size. */ + KUNIT_EXPECT_EQ(test, total_mapping_size(empty, 2), 0); + + /* Normal set of PT_LOADS, and expected size. */ + KUNIT_EXPECT_EQ(test, total_mapping_size(mount, ARRAY_SIZE(mount)), mount_size); + /* Unordered PT_LOADs result in same size. */ + KUNIT_EXPECT_EQ(test, total_mapping_size(unordered, ARRAY_SIZE(unordered)), mount_size); +} + +static struct kunit_case binfmt_elf_test_cases[] = { + KUNIT_CASE(total_mapping_size_test), + {}, +}; + +static struct kunit_suite binfmt_elf_test_suite = { + .name = KBUILD_MODNAME, + .test_cases = binfmt_elf_test_cases, +}; + +kunit_test_suite(binfmt_elf_test_suite); diff --git a/fs/tests/exec_kunit.c b/fs/tests/exec_kunit.c new file mode 100644 index 000000000..1c32cac09 --- /dev/null +++ b/fs/tests/exec_kunit.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include + +struct bprm_stack_limits_result { + struct linux_binprm bprm; + int expected_rc; + unsigned long expected_argmin; +}; + +static const struct bprm_stack_limits_result bprm_stack_limits_results[] = { + /* Negative argc/envc counts produce -E2BIG */ + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ULONG_MAX, + .argc = INT_MIN, .envc = INT_MIN }, .expected_rc = -E2BIG }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ULONG_MAX, + .argc = 5, .envc = -1 }, .expected_rc = -E2BIG }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ULONG_MAX, + .argc = -1, .envc = 10 }, .expected_rc = -E2BIG }, + /* The max value of argc or envc is MAX_ARG_STRINGS. */ + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ULONG_MAX, + .argc = INT_MAX, .envc = INT_MAX }, .expected_rc = -E2BIG }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ULONG_MAX, + .argc = MAX_ARG_STRINGS, .envc = MAX_ARG_STRINGS }, .expected_rc = -E2BIG }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ULONG_MAX, + .argc = 0, .envc = MAX_ARG_STRINGS }, .expected_rc = -E2BIG }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ULONG_MAX, + .argc = MAX_ARG_STRINGS, .envc = 0 }, .expected_rc = -E2BIG }, + /* + * On 32-bit system these argc and envc counts, while likely impossible + * to represent within the associated TASK_SIZE, could overflow the + * limit calculation, and bypass the ptr_size <= limit check. + */ + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ULONG_MAX, + .argc = 0x20000001, .envc = 0x20000001 }, .expected_rc = -E2BIG }, +#ifdef CONFIG_MMU + /* Make sure a pathological bprm->p doesn't cause an overflow. */ + { { .p = sizeof(void *), .rlim_stack.rlim_cur = ULONG_MAX, + .argc = 10, .envc = 10 }, .expected_rc = -E2BIG }, +#endif + /* + * 0 rlim_stack will get raised to ARG_MAX. With 1 string pointer, + * we should see p - ARG_MAX + sizeof(void *). + */ + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 0, + .argc = 1, .envc = 0 }, .expected_argmin = ULONG_MAX - ARG_MAX + sizeof(void *)}, + /* Validate that argc is always raised to a minimum of 1. */ + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 0, + .argc = 0, .envc = 0 }, .expected_argmin = ULONG_MAX - ARG_MAX + sizeof(void *)}, + /* + * 0 rlim_stack will get raised to ARG_MAX. With pointers filling ARG_MAX, + * we should see -E2BIG. (Note argc is always raised to at least 1.) + */ + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 0, + .argc = ARG_MAX / sizeof(void *), .envc = 0 }, .expected_rc = -E2BIG }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 0, + .argc = 0, .envc = ARG_MAX / sizeof(void *) - 1 }, .expected_rc = -E2BIG }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 0, + .argc = ARG_MAX / sizeof(void *) + 1, .envc = 0 }, .expected_rc = -E2BIG }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 0, + .argc = 0, .envc = ARG_MAX / sizeof(void *) }, .expected_rc = -E2BIG }, + /* And with one less, we see space for exactly 1 pointer. */ + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 0, + .argc = (ARG_MAX / sizeof(void *)) - 1, .envc = 0 }, + .expected_argmin = ULONG_MAX - sizeof(void *) }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 0, + .argc = 0, .envc = (ARG_MAX / sizeof(void *)) - 2, }, + .expected_argmin = ULONG_MAX - sizeof(void *) }, + /* If we raise rlim_stack / 4 to exactly ARG_MAX, nothing changes. */ + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ARG_MAX * 4, + .argc = ARG_MAX / sizeof(void *), .envc = 0 }, .expected_rc = -E2BIG }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ARG_MAX * 4, + .argc = 0, .envc = ARG_MAX / sizeof(void *) - 1 }, .expected_rc = -E2BIG }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ARG_MAX * 4, + .argc = ARG_MAX / sizeof(void *) + 1, .envc = 0 }, .expected_rc = -E2BIG }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ARG_MAX * 4, + .argc = 0, .envc = ARG_MAX / sizeof(void *) }, .expected_rc = -E2BIG }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ARG_MAX * 4, + .argc = (ARG_MAX / sizeof(void *)) - 1, .envc = 0 }, + .expected_argmin = ULONG_MAX - sizeof(void *) }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = ARG_MAX * 4, + .argc = 0, .envc = (ARG_MAX / sizeof(void *)) - 2, }, + .expected_argmin = ULONG_MAX - sizeof(void *) }, + /* But raising it another pointer * 4 will provide space for 1 more pointer. */ + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = (ARG_MAX + sizeof(void *)) * 4, + .argc = ARG_MAX / sizeof(void *), .envc = 0 }, + .expected_argmin = ULONG_MAX - sizeof(void *) }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = (ARG_MAX + sizeof(void *)) * 4, + .argc = 0, .envc = ARG_MAX / sizeof(void *) - 1 }, + .expected_argmin = ULONG_MAX - sizeof(void *) }, + /* Raising rlim_stack / 4 to _STK_LIM / 4 * 3 will see more space. */ + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * (_STK_LIM / 4 * 3), + .argc = 0, .envc = 0 }, + .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, + /* But raising it any further will see no increase. */ + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * (_STK_LIM / 4 * 3 + sizeof(void *)), + .argc = 0, .envc = 0 }, + .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, + { { .p = ULONG_MAX, .rlim_stack.rlim_cur = 4 * _STK_LIM, + .argc = 0, .envc = 0 }, + .expected_argmin = ULONG_MAX - (_STK_LIM / 4 * 3) + sizeof(void *) }, +}; + +static void exec_test_bprm_stack_limits(struct kunit *test) +{ + /* Double-check the constants. */ + KUNIT_EXPECT_EQ(test, _STK_LIM, SZ_8M); + KUNIT_EXPECT_EQ(test, ARG_MAX, 32 * SZ_4K); + KUNIT_EXPECT_EQ(test, MAX_ARG_STRINGS, 0x7FFFFFFF); + + for (int i = 0; i < ARRAY_SIZE(bprm_stack_limits_results); i++) { + const struct bprm_stack_limits_result *result = &bprm_stack_limits_results[i]; + struct linux_binprm bprm = result->bprm; + int rc; + + rc = bprm_stack_limits(&bprm); + KUNIT_EXPECT_EQ_MSG(test, rc, result->expected_rc, "on loop %d", i); +#ifdef CONFIG_MMU + KUNIT_EXPECT_EQ_MSG(test, bprm.argmin, result->expected_argmin, "on loop %d", i); +#endif + } +} + +static struct kunit_case exec_test_cases[] = { + KUNIT_CASE(exec_test_bprm_stack_limits), + {}, +}; + +static struct kunit_suite exec_test_suite = { + .name = "exec", + .test_cases = exec_test_cases, +}; + +kunit_test_suite(exec_test_suite); diff --git a/fs/timerfd.c b/fs/timerfd.c new file mode 100644 index 000000000..fe845af0b --- /dev/null +++ b/fs/timerfd.c @@ -0,0 +1,621 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs/timerfd.c + * + * Copyright (C) 2007 Davide Libenzi + * + * + * Thanks to Thomas Gleixner for code reviews and useful comments. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct timerfd_ctx { + union { + struct hrtimer tmr; + struct alarm alarm; + } t; + ktime_t tintv; + ktime_t moffs; + wait_queue_head_t wqh; + u64 ticks; + int clockid; + short unsigned expired; + short unsigned settime_flags; /* to show in fdinfo */ + struct rcu_head rcu; + struct list_head clist; + spinlock_t cancel_lock; + bool might_cancel; +}; + +static LIST_HEAD(cancel_list); +static DEFINE_SPINLOCK(cancel_lock); + +static inline bool isalarm(struct timerfd_ctx *ctx) +{ + return ctx->clockid == CLOCK_REALTIME_ALARM || + ctx->clockid == CLOCK_BOOTTIME_ALARM; +} + +static void __timerfd_triggered(struct timerfd_ctx *ctx) +{ + lockdep_assert_held(&ctx->wqh.lock); + + ctx->expired = 1; + ctx->ticks++; + wake_up_locked_poll(&ctx->wqh, EPOLLIN); +} + +/* + * This gets called when the timer event triggers. We set the "expired" + * flag, but we do not re-arm the timer (in case it's necessary, + * tintv != 0) until the timer is accessed. + */ +static void timerfd_triggered(struct timerfd_ctx *ctx) +{ + guard(spinlock_irqsave)(&ctx->wqh.lock); + __timerfd_triggered(ctx); +} + +static enum hrtimer_restart timerfd_tmrproc(struct hrtimer *htmr) +{ + struct timerfd_ctx *ctx = container_of(htmr, struct timerfd_ctx, + t.tmr); + timerfd_triggered(ctx); + return HRTIMER_NORESTART; +} + +static void timerfd_alarmproc(struct alarm *alarm, ktime_t now) +{ + struct timerfd_ctx *ctx = container_of(alarm, struct timerfd_ctx, + t.alarm); + timerfd_triggered(ctx); +} + +/* + * Called when the clock was set to cancel the timers in the cancel + * list. This will wake up processes waiting on these timers. The + * wake-up requires ctx->ticks to be non zero, therefore we increment + * it before calling wake_up_locked(). + */ +void timerfd_clock_was_set(void) +{ + ktime_t moffs = ktime_mono_to_real(0); + struct timerfd_ctx *ctx; + unsigned long flags; + + rcu_read_lock(); + list_for_each_entry_rcu(ctx, &cancel_list, clist) { + if (!ctx->might_cancel) + continue; + spin_lock_irqsave(&ctx->wqh.lock, flags); + if (ctx->moffs != moffs) { + ctx->moffs = KTIME_MAX; + ctx->ticks++; + wake_up_locked_poll(&ctx->wqh, EPOLLIN); + } + spin_unlock_irqrestore(&ctx->wqh.lock, flags); + } + rcu_read_unlock(); +} + +static void timerfd_resume_work(struct work_struct *work) +{ + timerfd_clock_was_set(); +} + +static DECLARE_WORK(timerfd_work, timerfd_resume_work); + +/* + * Invoked from timekeeping_resume(). Defer the actual update to work so + * timerfd_clock_was_set() runs in task context. + */ +void timerfd_resume(void) +{ + schedule_work(&timerfd_work); +} + +static void __timerfd_remove_cancel(struct timerfd_ctx *ctx) +{ + if (ctx->might_cancel) { + ctx->might_cancel = false; + spin_lock(&cancel_lock); + list_del_rcu(&ctx->clist); + spin_unlock(&cancel_lock); + } +} + +static void timerfd_remove_cancel(struct timerfd_ctx *ctx) +{ + spin_lock(&ctx->cancel_lock); + __timerfd_remove_cancel(ctx); + spin_unlock(&ctx->cancel_lock); +} + +static bool timerfd_canceled(struct timerfd_ctx *ctx) +{ + if (!ctx->might_cancel || ctx->moffs != KTIME_MAX) + return false; + ctx->moffs = ktime_mono_to_real(0); + return true; +} + +static void timerfd_setup_cancel(struct timerfd_ctx *ctx, int flags) +{ + spin_lock(&ctx->cancel_lock); + if ((ctx->clockid == CLOCK_REALTIME || + ctx->clockid == CLOCK_REALTIME_ALARM) && + (flags & TFD_TIMER_ABSTIME) && (flags & TFD_TIMER_CANCEL_ON_SET)) { + if (!ctx->might_cancel) { + ctx->might_cancel = true; + spin_lock(&cancel_lock); + list_add_rcu(&ctx->clist, &cancel_list); + spin_unlock(&cancel_lock); + } + } else { + __timerfd_remove_cancel(ctx); + } + spin_unlock(&ctx->cancel_lock); +} + +static ktime_t timerfd_get_remaining(struct timerfd_ctx *ctx) +{ + ktime_t remaining; + + if (isalarm(ctx)) + remaining = alarm_expires_remaining(&ctx->t.alarm); + else + remaining = hrtimer_expires_remaining_adjusted(&ctx->t.tmr); + + return remaining < 0 ? 0: remaining; +} + +static void timerfd_alarm_start(struct timerfd_ctx *ctx, ktime_t exp, bool relative) +{ + /* Start the timer. If it's expired already, handle the callback. */ + if (!alarm_start_timer(&ctx->t.alarm, exp, relative)) + __timerfd_triggered(ctx); +} + +static u64 timerfd_alarm_restart(struct timerfd_ctx *ctx) +{ + /* -1 to account for ctx->ticks++ in __timerfd_triggered() */ + u64 ticks = alarm_forward_now(&ctx->t.alarm, ctx->tintv) - 1; + + timerfd_alarm_start(ctx, alarm_get_expires(&ctx->t.alarm), false); + return ticks; +} + +static void timerfd_hrtimer_start(struct timerfd_ctx *ctx, ktime_t exp, + const enum hrtimer_mode mode) +{ + /* Start the timer. If it's expired already, handle the callback. */ + if (!hrtimer_start_range_ns_user(&ctx->t.tmr, exp, 0, mode)) + __timerfd_triggered(ctx); +} + +static u64 timerfd_hrtimer_restart(struct timerfd_ctx *ctx) +{ + /* -1 to account for ctx->ticks++ in __timerfd_triggered() */ + u64 ticks = hrtimer_forward_now(&ctx->t.tmr, ctx->tintv) - 1; + + timerfd_hrtimer_start(ctx, hrtimer_get_expires(&ctx->t.tmr), HRTIMER_MODE_ABS); + return ticks; +} + +static u64 timerfd_restart(struct timerfd_ctx *ctx) +{ + if (isalarm(ctx)) + return timerfd_alarm_restart(ctx); + return timerfd_hrtimer_restart(ctx); +} + +static int timerfd_setup(struct timerfd_ctx *ctx, int flags, + const struct itimerspec64 *ktmr) +{ + int clockid = ctx->clockid; + enum hrtimer_mode htmode; + ktime_t texp; + + htmode = (flags & TFD_TIMER_ABSTIME) ? HRTIMER_MODE_ABS: HRTIMER_MODE_REL; + + texp = timespec64_to_ktime(ktmr->it_value); + ctx->expired = 0; + ctx->ticks = 0; + ctx->tintv = timespec64_to_ktime(ktmr->it_interval); + + if (isalarm(ctx)) { + alarm_init(&ctx->t.alarm, + ctx->clockid == CLOCK_REALTIME_ALARM ? + ALARM_REALTIME : ALARM_BOOTTIME, + timerfd_alarmproc); + } else { + hrtimer_setup(&ctx->t.tmr, timerfd_tmrproc, clockid, htmode); + } + + if (texp != 0) { + if (flags & TFD_TIMER_ABSTIME) + texp = timens_ktime_to_host(clockid, texp); + if (isalarm(ctx)) + timerfd_alarm_start(ctx, texp, !(flags & TFD_TIMER_ABSTIME)); + else + timerfd_hrtimer_start(ctx, texp, htmode); + + if (timerfd_canceled(ctx)) + return -ECANCELED; + } + + ctx->settime_flags = flags & TFD_SETTIME_FLAGS; + return 0; +} + +static int timerfd_release(struct inode *inode, struct file *file) +{ + struct timerfd_ctx *ctx = file->private_data; + + timerfd_remove_cancel(ctx); + + if (isalarm(ctx)) + alarm_cancel(&ctx->t.alarm); + else + hrtimer_cancel(&ctx->t.tmr); + kfree_rcu(ctx, rcu); + return 0; +} + +static __poll_t timerfd_poll(struct file *file, poll_table *wait) +{ + struct timerfd_ctx *ctx = file->private_data; + __poll_t events = 0; + unsigned long flags; + + poll_wait(file, &ctx->wqh, wait); + + spin_lock_irqsave(&ctx->wqh.lock, flags); + if (ctx->ticks) + events |= EPOLLIN; + spin_unlock_irqrestore(&ctx->wqh.lock, flags); + + return events; +} + +static ssize_t timerfd_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct file *file = iocb->ki_filp; + struct timerfd_ctx *ctx = file->private_data; + ssize_t res; + u64 ticks = 0; + + if (iov_iter_count(to) < sizeof(ticks)) + return -EINVAL; + + spin_lock_irq(&ctx->wqh.lock); + if (file->f_flags & O_NONBLOCK || iocb->ki_flags & IOCB_NOWAIT) + res = -EAGAIN; + else + res = wait_event_interruptible_locked_irq(ctx->wqh, ctx->ticks); + + /* + * If clock has changed, we do not care about the + * ticks and we do not rearm the timer. Userspace must + * reevaluate anyway. + */ + if (timerfd_canceled(ctx)) { + ctx->ticks = 0; + ctx->expired = 0; + res = -ECANCELED; + } + + if (ctx->ticks) { + unsigned int expired = ctx->expired; + + ticks = ctx->ticks; + ctx->expired = 0; + ctx->ticks = 0; + + /* + * If tintv != 0, this is a periodic timer that needs to be + * re-armed. We avoid doing it in the timer callback to avoid + * DoS attacks specifying a very short timer period. + */ + if (expired && ctx->tintv) + ticks += timerfd_restart(ctx); + } + spin_unlock_irq(&ctx->wqh.lock); + if (ticks) { + res = copy_to_iter(&ticks, sizeof(ticks), to); + if (!res) + res = -EFAULT; + } + return res; +} + +#ifdef CONFIG_PROC_FS +static void timerfd_show(struct seq_file *m, struct file *file) +{ + struct timerfd_ctx *ctx = file->private_data; + struct timespec64 value, interval; + + spin_lock_irq(&ctx->wqh.lock); + value = ktime_to_timespec64(timerfd_get_remaining(ctx)); + interval = ktime_to_timespec64(ctx->tintv); + spin_unlock_irq(&ctx->wqh.lock); + + seq_printf(m, + "clockid: %d\n" + "ticks: %llu\n" + "settime flags: 0%o\n" + "it_value: (%llu, %llu)\n" + "it_interval: (%llu, %llu)\n", + ctx->clockid, + (unsigned long long)ctx->ticks, + ctx->settime_flags, + (unsigned long long)value.tv_sec, + (unsigned long long)value.tv_nsec, + (unsigned long long)interval.tv_sec, + (unsigned long long)interval.tv_nsec); +} +#else +#define timerfd_show NULL +#endif + +#ifdef CONFIG_CHECKPOINT_RESTORE +static long timerfd_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + struct timerfd_ctx *ctx = file->private_data; + int ret = 0; + + switch (cmd) { + case TFD_IOC_SET_TICKS: { + u64 ticks; + + if (copy_from_user(&ticks, (u64 __user *)arg, sizeof(ticks))) + return -EFAULT; + if (!ticks) + return -EINVAL; + + spin_lock_irq(&ctx->wqh.lock); + if (!timerfd_canceled(ctx)) { + ctx->ticks = ticks; + wake_up_locked_poll(&ctx->wqh, EPOLLIN); + } else + ret = -ECANCELED; + spin_unlock_irq(&ctx->wqh.lock); + break; + } + default: + ret = -ENOTTY; + break; + } + + return ret; +} +#else +#define timerfd_ioctl NULL +#endif + +static const struct file_operations timerfd_fops = { + .release = timerfd_release, + .poll = timerfd_poll, + .read_iter = timerfd_read_iter, + .llseek = noop_llseek, + .show_fdinfo = timerfd_show, + .unlocked_ioctl = timerfd_ioctl, +}; + +SYSCALL_DEFINE2(timerfd_create, int, clockid, int, flags) +{ + struct timerfd_ctx *ctx __free(kfree) = NULL; + int ret; + + /* Check the TFD_* constants for consistency. */ + BUILD_BUG_ON(TFD_CLOEXEC != O_CLOEXEC); + BUILD_BUG_ON(TFD_NONBLOCK != O_NONBLOCK); + + if ((flags & ~TFD_CREATE_FLAGS) || + (clockid != CLOCK_MONOTONIC && + clockid != CLOCK_REALTIME && + clockid != CLOCK_REALTIME_ALARM && + clockid != CLOCK_BOOTTIME && + clockid != CLOCK_BOOTTIME_ALARM)) + return -EINVAL; + + if ((clockid == CLOCK_REALTIME_ALARM || + clockid == CLOCK_BOOTTIME_ALARM) && + !capable(CAP_WAKE_ALARM)) + return -EPERM; + + ctx = kzalloc_obj(*ctx); + if (!ctx) + return -ENOMEM; + + init_waitqueue_head(&ctx->wqh); + spin_lock_init(&ctx->cancel_lock); + ctx->clockid = clockid; + + if (isalarm(ctx)) + alarm_init(&ctx->t.alarm, + ctx->clockid == CLOCK_REALTIME_ALARM ? + ALARM_REALTIME : ALARM_BOOTTIME, + timerfd_alarmproc); + else + hrtimer_setup(&ctx->t.tmr, timerfd_tmrproc, clockid, HRTIMER_MODE_ABS); + + ctx->moffs = ktime_mono_to_real(0); + + ret = FD_ADD(flags & TFD_SHARED_FCNTL_FLAGS, + anon_inode_getfile_fmode("[timerfd]", &timerfd_fops, ctx, + O_RDWR | (flags & TFD_SHARED_FCNTL_FLAGS), + FMODE_NOWAIT)); + if (ret >= 0) + retain_and_null_ptr(ctx); + return ret; +} + +static int do_timerfd_settime(int ufd, int flags, + const struct itimerspec64 *new, + struct itimerspec64 *old) +{ + struct timerfd_ctx *ctx; + int ret; + + if ((flags & ~TFD_SETTIME_FLAGS) || + !itimerspec64_valid(new)) + return -EINVAL; + + CLASS(fd, f)(ufd); + if (fd_empty(f)) + return -EBADF; + + if (fd_file(f)->f_op != &timerfd_fops) + return -EINVAL; + + ctx = fd_file(f)->private_data; + + if (isalarm(ctx) && !capable(CAP_WAKE_ALARM)) + return -EPERM; + + timerfd_setup_cancel(ctx, flags); + + /* + * We need to stop the existing timer before reprogramming + * it to the new values. + */ + for (;;) { + spin_lock_irq(&ctx->wqh.lock); + + if (isalarm(ctx)) { + if (alarm_try_to_cancel(&ctx->t.alarm) >= 0) + break; + } else { + if (hrtimer_try_to_cancel(&ctx->t.tmr) >= 0) + break; + } + spin_unlock_irq(&ctx->wqh.lock); + + if (isalarm(ctx)) + hrtimer_cancel_wait_running(&ctx->t.alarm.timer); + else + hrtimer_cancel_wait_running(&ctx->t.tmr); + } + + /* + * If the timer is expired and it's periodic, we need to advance it + * because the caller may want to know the previous expiration time. + * We do not update "ticks" and "expired" since the timer will be + * re-programmed again in the following timerfd_setup() call. + */ + if (ctx->expired && ctx->tintv) { + if (isalarm(ctx)) + alarm_forward_now(&ctx->t.alarm, ctx->tintv); + else + hrtimer_forward_now(&ctx->t.tmr, ctx->tintv); + } + + old->it_value = ktime_to_timespec64(timerfd_get_remaining(ctx)); + old->it_interval = ktime_to_timespec64(ctx->tintv); + + /* + * Re-program the timer to the new value ... + */ + ret = timerfd_setup(ctx, flags, new); + + spin_unlock_irq(&ctx->wqh.lock); + return ret; +} + +static int do_timerfd_gettime(int ufd, struct itimerspec64 *t) +{ + struct timerfd_ctx *ctx; + CLASS(fd, f)(ufd); + + if (fd_empty(f)) + return -EBADF; + if (fd_file(f)->f_op != &timerfd_fops) + return -EINVAL; + ctx = fd_file(f)->private_data; + + spin_lock_irq(&ctx->wqh.lock); + if (ctx->expired && ctx->tintv) { + ctx->expired = 0; + ctx->ticks += timerfd_restart(ctx); + } + t->it_value = ktime_to_timespec64(timerfd_get_remaining(ctx)); + t->it_interval = ktime_to_timespec64(ctx->tintv); + spin_unlock_irq(&ctx->wqh.lock); + return 0; +} + +SYSCALL_DEFINE4(timerfd_settime, int, ufd, int, flags, + const struct __kernel_itimerspec __user *, utmr, + struct __kernel_itimerspec __user *, otmr) +{ + struct itimerspec64 new, old; + int ret; + + if (get_itimerspec64(&new, utmr)) + return -EFAULT; + ret = do_timerfd_settime(ufd, flags, &new, &old); + if (ret) + return ret; + if (otmr && put_itimerspec64(&old, otmr)) + return -EFAULT; + + return ret; +} + +SYSCALL_DEFINE2(timerfd_gettime, int, ufd, struct __kernel_itimerspec __user *, otmr) +{ + struct itimerspec64 kotmr; + int ret = do_timerfd_gettime(ufd, &kotmr); + if (ret) + return ret; + return put_itimerspec64(&kotmr, otmr) ? -EFAULT : 0; +} + +#ifdef CONFIG_COMPAT_32BIT_TIME +SYSCALL_DEFINE4(timerfd_settime32, int, ufd, int, flags, + const struct old_itimerspec32 __user *, utmr, + struct old_itimerspec32 __user *, otmr) +{ + struct itimerspec64 new, old; + int ret; + + if (get_old_itimerspec32(&new, utmr)) + return -EFAULT; + ret = do_timerfd_settime(ufd, flags, &new, &old); + if (ret) + return ret; + if (otmr && put_old_itimerspec32(&old, otmr)) + return -EFAULT; + return ret; +} + +SYSCALL_DEFINE2(timerfd_gettime32, int, ufd, + struct old_itimerspec32 __user *, otmr) +{ + struct itimerspec64 kotmr; + int ret = do_timerfd_gettime(ufd, &kotmr); + if (ret) + return ret; + return put_old_itimerspec32(&kotmr, otmr) ? -EFAULT : 0; +} +#endif diff --git a/fs/tracefs/Makefile b/fs/tracefs/Makefile new file mode 100644 index 000000000..73c56da8e --- /dev/null +++ b/fs/tracefs/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0-only +tracefs-objs := inode.o +tracefs-objs += event_inode.o + +obj-$(CONFIG_TRACING) += tracefs.o + diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c new file mode 100644 index 000000000..604ba3e84 --- /dev/null +++ b/fs/tracefs/event_inode.c @@ -0,0 +1,928 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * event_inode.c - part of tracefs, a pseudo file system for activating tracing + * + * Copyright (C) 2020-23 VMware Inc, author: Steven Rostedt + * Copyright (C) 2020-23 VMware Inc, author: Ajay Kaher + * Copyright (C) 2023 Google, author: Steven Rostedt + * + * eventfs is used to dynamically create inodes and dentries based on the + * meta data provided by the tracing system. + * + * eventfs stores the meta-data of files/dirs and holds off on creating + * inodes/dentries of the files. When accessed, the eventfs will create the + * inodes/dentries in a just-in-time (JIT) manner. The eventfs will clean up + * and delete the inodes/dentries when they are no longer referenced. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +/* + * eventfs_mutex protects the eventfs_inode (ei) dentry. Any access + * to the ei->dentry must be done under this mutex and after checking + * if ei->is_freed is not set. When ei->is_freed is set, the dentry + * is on its way to being freed after the last dput() is made on it. + */ +static DEFINE_MUTEX(eventfs_mutex); + +/* Choose something "unique" ;-) */ +#define EVENTFS_FILE_INODE_INO 0x12c4e37 + +struct eventfs_root_inode { + struct eventfs_inode ei; + struct dentry *events_dir; +}; + +static struct eventfs_root_inode *get_root_inode(struct eventfs_inode *ei) +{ + WARN_ON_ONCE(!ei->is_events); + return container_of(ei, struct eventfs_root_inode, ei); +} + +/* Just try to make something consistent and unique */ +static int eventfs_dir_ino(struct eventfs_inode *ei) +{ + if (!ei->ino) { + ei->ino = get_next_ino(); + /* Must not have the file inode number */ + if (ei->ino == EVENTFS_FILE_INODE_INO) + ei->ino = get_next_ino(); + } + + return ei->ino; +} + +/* + * The eventfs_inode (ei) itself is protected by SRCU. It is released from + * its parent's list and will have is_freed set (under eventfs_mutex). + * After the SRCU grace period is over and the last dput() is called + * the ei is freed. + */ +DEFINE_STATIC_SRCU(eventfs_srcu); + +/* Mode is unsigned short, use the upper bits for flags */ +enum { + EVENTFS_SAVE_MODE = BIT(16), + EVENTFS_SAVE_UID = BIT(17), + EVENTFS_SAVE_GID = BIT(18), +}; + +#define EVENTFS_MODE_MASK (EVENTFS_SAVE_MODE - 1) + +static void free_ei_rcu(struct rcu_head *rcu) +{ + struct eventfs_inode *ei = container_of(rcu, struct eventfs_inode, rcu); + struct eventfs_root_inode *rei; + + kfree(ei->entry_attrs); + kfree_const(ei->name); + if (ei->is_events) { + rei = get_root_inode(ei); + kfree(rei); + } else { + kfree(ei); + } +} + +/* + * eventfs_inode reference count management. + * + * NOTE! We count only references from dentries, in the + * form 'dentry->d_fsdata'. There are also references from + * directory inodes ('ti->private'), but the dentry reference + * count is always a superset of the inode reference count. + */ +static void release_ei(struct kref *ref) +{ + struct eventfs_inode *ei = container_of(ref, struct eventfs_inode, kref); + const struct eventfs_entry *entry; + + WARN_ON_ONCE(!ei->is_freed); + + for (int i = 0; i < ei->nr_entries; i++) { + entry = &ei->entries[i]; + if (entry->release) + entry->release(entry->name, ei->data); + } + + call_srcu(&eventfs_srcu, &ei->rcu, free_ei_rcu); +} + +static inline void put_ei(struct eventfs_inode *ei) +{ + if (ei) + kref_put(&ei->kref, release_ei); +} + +static inline void free_ei(struct eventfs_inode *ei) +{ + if (ei) { + /* The ei should have no children if it is being freed. */ + WARN_ON_ONCE(!list_empty(&ei->children)); + ei->is_freed = 1; + /* + * The SRCU iteration has a smp_rmb() to make sure it + * sees a child (that may have already been freed) + * before it reads is_free. If is_free is set, it must + * not use the child it acquired from ei->children, as + * the list may be used for SRCU. + */ + smp_wmb(); + put_ei(ei); + } +} + +/* + * Called when creation of an ei fails, do not call release() functions. + */ +static inline void cleanup_ei(struct eventfs_inode *ei) +{ + if (ei) { + /* Set nr_entries to 0 to prevent release() function being called */ + ei->nr_entries = 0; + free_ei(ei); + } +} + +static inline struct eventfs_inode *get_ei(struct eventfs_inode *ei) +{ + if (ei) + kref_get(&ei->kref); + return ei; +} + +static struct dentry *eventfs_root_lookup(struct inode *dir, + struct dentry *dentry, + unsigned int flags); +static int eventfs_iterate(struct file *file, struct dir_context *ctx); + +static void update_attr(struct eventfs_attr *attr, struct iattr *iattr) +{ + unsigned int ia_valid = iattr->ia_valid; + + if (ia_valid & ATTR_MODE) { + attr->mode = (attr->mode & ~EVENTFS_MODE_MASK) | + (iattr->ia_mode & EVENTFS_MODE_MASK) | + EVENTFS_SAVE_MODE; + } + if (ia_valid & ATTR_UID) { + attr->mode |= EVENTFS_SAVE_UID; + attr->uid = iattr->ia_uid; + } + if (ia_valid & ATTR_GID) { + attr->mode |= EVENTFS_SAVE_GID; + attr->gid = iattr->ia_gid; + } +} + +static int eventfs_set_attr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *iattr) +{ + const struct eventfs_entry *entry; + struct eventfs_inode *ei; + const char *name; + int ret; + + guard(mutex)(&eventfs_mutex); + ei = dentry->d_fsdata; + /* Do not allow changes if the event is about to be removed. */ + if (ei->is_freed) + return -ENODEV; + + /* Preallocate the children mode array if necessary */ + if (!(dentry->d_inode->i_mode & S_IFDIR)) { + if (!ei->entry_attrs) { + ei->entry_attrs = kzalloc_objs(*ei->entry_attrs, + ei->nr_entries, GFP_NOFS); + if (!ei->entry_attrs) + return -ENOMEM; + } + } + + ret = simple_setattr(idmap, dentry, iattr); + if (ret < 0) + return ret; + + /* + * If this is a dir, then update the ei cache, only the file + * mode is saved in the ei->m_children, and the ownership is + * determined by the parent directory. + */ + if (dentry->d_inode->i_mode & S_IFDIR) { + /* Just use the inode permissions for the events directory */ + if (!ei->is_events) + update_attr(&ei->attr, iattr); + + } else { + name = dentry->d_name.name; + + for (int i = 0; i < ei->nr_entries; i++) { + entry = &ei->entries[i]; + if (strcmp(name, entry->name) == 0) { + update_attr(&ei->entry_attrs[i], iattr); + break; + } + } + } + return ret; +} + +static const struct inode_operations eventfs_dir_inode_operations = { + .lookup = eventfs_root_lookup, + .setattr = eventfs_set_attr, +}; + +static const struct inode_operations eventfs_file_inode_operations = { + .setattr = eventfs_set_attr, +}; + +static const struct file_operations eventfs_file_operations = { + .read = generic_read_dir, + .iterate_shared = eventfs_iterate, + .llseek = generic_file_llseek, +}; + +static void eventfs_set_attrs(struct eventfs_inode *ei, bool update_uid, kuid_t uid, + bool update_gid, kgid_t gid, int level) +{ + struct eventfs_inode *ei_child; + + lockdep_assert_held(&eventfs_mutex); + + /* Update events// */ + if (WARN_ON_ONCE(level > 3)) + return; + + if (update_uid) { + ei->attr.mode &= ~EVENTFS_SAVE_UID; + ei->attr.uid = uid; + } + + if (update_gid) { + ei->attr.mode &= ~EVENTFS_SAVE_GID; + ei->attr.gid = gid; + } + + list_for_each_entry(ei_child, &ei->children, list) { + eventfs_set_attrs(ei_child, update_uid, uid, update_gid, gid, level + 1); + } + + if (!ei->entry_attrs) + return; + + for (int i = 0; i < ei->nr_entries; i++) { + if (update_uid) { + ei->entry_attrs[i].mode &= ~EVENTFS_SAVE_UID; + ei->entry_attrs[i].uid = uid; + } + if (update_gid) { + ei->entry_attrs[i].mode &= ~EVENTFS_SAVE_GID; + ei->entry_attrs[i].gid = gid; + } + } + +} + +/* + * On a remount of tracefs, if UID or GID options are set, then + * the mount point inode permissions should be used. + * Reset the saved permission flags appropriately. + */ +void eventfs_remount(struct tracefs_inode *ti, bool update_uid, bool update_gid) +{ + struct eventfs_inode *ei = ti->private; + + /* Only the events directory does the updates */ + if (!ei || !ei->is_events || ei->is_freed) + return; + + eventfs_set_attrs(ei, update_uid, ti->vfs_inode.i_uid, + update_gid, ti->vfs_inode.i_gid, 0); +} + +static void update_inode_attr(struct inode *inode, umode_t mode, + struct eventfs_attr *attr, struct eventfs_root_inode *rei) +{ + if (attr && attr->mode & EVENTFS_SAVE_MODE) + inode->i_mode = attr->mode & EVENTFS_MODE_MASK; + else + inode->i_mode = mode; + + if (attr && attr->mode & EVENTFS_SAVE_UID) + inode->i_uid = attr->uid; + else + inode->i_uid = rei->ei.attr.uid; + + if (attr && attr->mode & EVENTFS_SAVE_GID) + inode->i_gid = attr->gid; + else + inode->i_gid = rei->ei.attr.gid; +} + +static struct inode *eventfs_get_inode(struct dentry *dentry, struct eventfs_attr *attr, + umode_t mode, struct eventfs_inode *ei) +{ + struct eventfs_root_inode *rei; + struct eventfs_inode *pei; + struct tracefs_inode *ti; + struct inode *inode; + + inode = tracefs_get_inode(dentry->d_sb); + if (!inode) + return NULL; + + ti = get_tracefs(inode); + ti->private = ei; + ti->flags |= TRACEFS_EVENT_INODE; + + /* Find the top dentry that holds the "events" directory */ + do { + dentry = dentry->d_parent; + /* Directories always have d_fsdata */ + pei = dentry->d_fsdata; + } while (!pei->is_events); + + rei = get_root_inode(pei); + + update_inode_attr(inode, mode, attr, rei); + + return inode; +} + +/** + * lookup_file - look up a file in the tracefs filesystem + * @parent_ei: Pointer to the eventfs_inode that represents parent of the file + * @dentry: the dentry to look up + * @mode: the permission that the file should have. + * @attr: saved attributes changed by user + * @data: something that the caller will want to get to later on. + * @fop: struct file_operations that should be used for this file. + * + * This function creates a dentry that represents a file in the eventsfs_inode + * directory. The inode.i_private pointer will point to @data in the open() + * call. + */ +static struct dentry *lookup_file(struct eventfs_inode *parent_ei, + struct dentry *dentry, + umode_t mode, + struct eventfs_attr *attr, + void *data, + const struct file_operations *fop) +{ + struct inode *inode; + + if (!(mode & S_IFMT)) + mode |= S_IFREG; + + if (WARN_ON_ONCE(!S_ISREG(mode))) + return ERR_PTR(-EIO); + + /* Only directories have ti->private set to an ei, not files */ + inode = eventfs_get_inode(dentry, attr, mode, NULL); + if (unlikely(!inode)) + return ERR_PTR(-ENOMEM); + + inode->i_op = &eventfs_file_inode_operations; + inode->i_fop = fop; + inode->i_private = data; + + /* All files will have the same inode number */ + inode->i_ino = EVENTFS_FILE_INODE_INO; + + // Files have their parent's ei as their fsdata + dentry->d_fsdata = get_ei(parent_ei); + + return d_splice_alias(inode, dentry); +}; + +/** + * lookup_dir_entry - look up a dir in the tracefs filesystem + * @dentry: the directory to look up + * @pei: Pointer to the parent eventfs_inode if available + * @ei: the eventfs_inode that represents the directory to create + * + * This function will look up a dentry for a directory represented by + * a eventfs_inode. + */ +static struct dentry *lookup_dir_entry(struct dentry *dentry, + struct eventfs_inode *pei, struct eventfs_inode *ei) +{ + struct inode *inode; + umode_t mode = S_IFDIR | S_IRWXU | S_IRUGO | S_IXUGO; + + inode = eventfs_get_inode(dentry, &ei->attr, mode, ei); + if (unlikely(!inode)) + return ERR_PTR(-ENOMEM); + + inode->i_op = &eventfs_dir_inode_operations; + inode->i_fop = &eventfs_file_operations; + + /* All directories will have the same inode number */ + inode->i_ino = eventfs_dir_ino(ei); + + dentry->d_fsdata = get_ei(ei); + + return d_splice_alias(inode, dentry); +} + +static inline struct eventfs_inode *init_ei(struct eventfs_inode *ei, const char *name) +{ + ei->name = kstrdup_const(name, GFP_KERNEL); + if (!ei->name) + return NULL; + kref_init(&ei->kref); + return ei; +} + +static inline struct eventfs_inode *alloc_ei(const char *name) +{ + struct eventfs_inode *ei = kzalloc_obj(*ei); + struct eventfs_inode *result; + + if (!ei) + return NULL; + + result = init_ei(ei, name); + if (!result) + kfree(ei); + + return result; +} + +static inline struct eventfs_inode *alloc_root_ei(const char *name) +{ + struct eventfs_root_inode *rei = kzalloc_obj(*rei); + struct eventfs_inode *ei; + + if (!rei) + return NULL; + + rei->ei.is_events = 1; + ei = init_ei(&rei->ei, name); + if (!ei) + kfree(rei); + + return ei; +} + +/** + * eventfs_d_release - dentry is going away + * @dentry: dentry which has the reference to remove. + * + * Remove the association between a dentry from an eventfs_inode. + */ +void eventfs_d_release(struct dentry *dentry) +{ + put_ei(dentry->d_fsdata); +} + +/** + * lookup_file_dentry - create a dentry for a file of an eventfs_inode + * @dentry: The parent dentry under which the new file's dentry will be created + * @ei: the eventfs_inode that the file will be created under + * @idx: the index into the entry_attrs[] of the @ei + * @mode: The mode of the file. + * @data: The data to use to set the inode of the file with on open() + * @fops: The fops of the file to be created. + * + * This function creates a dentry for a file associated with an + * eventfs_inode @ei. It uses the entry attributes specified by @idx, + * if available. The file will have the specified @mode and its inode will be + * set up with @data upon open. The file operations will be set to @fops. + * + * Return: Returns a pointer to the newly created file's dentry or an error + * pointer. + */ +static struct dentry * +lookup_file_dentry(struct dentry *dentry, + struct eventfs_inode *ei, int idx, + umode_t mode, void *data, + const struct file_operations *fops) +{ + struct eventfs_attr *attr = NULL; + + if (ei->entry_attrs) + attr = &ei->entry_attrs[idx]; + + return lookup_file(ei, dentry, mode, attr, data, fops); +} + +/** + * eventfs_root_lookup - lookup routine to create file/dir + * @dir: in which a lookup is being done + * @dentry: file/dir dentry + * @flags: Just passed to simple_lookup() + * + * Used to create dynamic file/dir with-in @dir, search with-in @ei + * list, if @dentry found go ahead and create the file/dir + */ + +static struct dentry *eventfs_root_lookup(struct inode *dir, + struct dentry *dentry, + unsigned int flags) +{ + struct eventfs_inode *ei_child; + struct tracefs_inode *ti; + struct eventfs_inode *ei; + const char *name = dentry->d_name.name; + + ti = get_tracefs(dir); + if (WARN_ON_ONCE(!(ti->flags & TRACEFS_EVENT_INODE))) + return ERR_PTR(-EIO); + + guard(mutex)(&eventfs_mutex); + + ei = ti->private; + if (!ei || ei->is_freed) + return NULL; + + list_for_each_entry(ei_child, &ei->children, list) { + if (strcmp(ei_child->name, name) != 0) + continue; + /* A child is freed and removed from the list at the same time */ + if (WARN_ON_ONCE(ei_child->is_freed)) + return NULL; + return lookup_dir_entry(dentry, ei, ei_child); + } + + for (int i = 0; i < ei->nr_entries; i++) { + void *data; + umode_t mode; + const struct file_operations *fops; + const struct eventfs_entry *entry = &ei->entries[i]; + + if (strcmp(name, entry->name) != 0) + continue; + + data = ei->data; + if (entry->callback(name, &mode, &data, &fops) <= 0) + return NULL; + + return lookup_file_dentry(dentry, ei, i, mode, data, fops); + + } + return NULL; +} + +/* + * Walk the children of a eventfs_inode to fill in getdents(). + */ +static int eventfs_iterate(struct file *file, struct dir_context *ctx) +{ + const struct file_operations *fops; + struct inode *f_inode = file_inode(file); + const struct eventfs_entry *entry; + struct eventfs_inode *ei_child; + struct tracefs_inode *ti; + struct eventfs_inode *ei; + const char *name; + umode_t mode; + int ino; + int i, r, c; + + if (!dir_emit_dots(file, ctx)) + return 0; + + ti = get_tracefs(f_inode); + if (!(ti->flags & TRACEFS_EVENT_INODE)) + return -EINVAL; + + /* Logic should prevent ctx->pos from going out of range */ + if (WARN_ON_ONCE(ctx->pos < 2 || ctx->pos > 0x7fffffffULL)) + return -EINVAL; + + c = ctx->pos - 2; + + guard(srcu)(&eventfs_srcu); + + scoped_guard(mutex, &eventfs_mutex) { + ei = READ_ONCE(ti->private); + if (!ei || ei->is_freed) + return -EINVAL; + } + + /* + * Need to create the dentries and inodes to have a consistent + * inode number. + */ + + /* Start at 'c' to jump over already read entries */ + for (i = c; i < ei->nr_entries; i++, ctx->pos++) { + void *cdata = ei->data; + + entry = &ei->entries[i]; + name = entry->name; + + /* If ei->is_freed then just bail here, nothing more to do */ + scoped_guard(mutex, &eventfs_mutex) { + if (ei->is_freed) + return -EINVAL; + r = entry->callback(name, &mode, &cdata, &fops); + } + if (r <= 0) + continue; + + ino = EVENTFS_FILE_INODE_INO; + + if (!dir_emit(ctx, name, strlen(name), ino, DT_REG)) + return -EINVAL; + } + + /* Subtract the skipped entries above */ + c -= min((unsigned int)c, (unsigned int)ei->nr_entries); + + list_for_each_entry_srcu(ei_child, &ei->children, list, + srcu_read_lock_held(&eventfs_srcu)) { + + /* + * If the ei is being freed, then the ei->children may be + * being used as the rcu list, which means the next element + * may be garbage. The ei->is_free is set before switching + * the ei->children over to ei->rcu. The read memory barrier + * here makes sure the ei_child is read before is_free is + * updated. + * + * Matches the smp_wmb() in free_ei() + */ + smp_rmb(); + if (ei->is_freed) + return -EINVAL; + + if (c > 0) { + c--; + continue; + } + + ctx->pos++; + + if (ei_child->is_freed) + continue; + + name = ei_child->name; + + ino = eventfs_dir_ino(ei_child); + + if (!dir_emit(ctx, name, strlen(name), ino, DT_DIR)) { + /* Incremented ctx->pos without adding something, reset it */ + ctx->pos--; + return -EINVAL; + } + } + return 1; +} + +/** + * eventfs_create_dir - Create the eventfs_inode for this directory + * @name: The name of the directory to create. + * @parent: The eventfs_inode of the parent directory. + * @entries: A list of entries that represent the files under this directory + * @size: The number of @entries + * @data: The default data to pass to the files (an entry may override it). + * + * This function creates the descriptor to represent a directory in the + * eventfs. This descriptor is an eventfs_inode, and it is returned to be + * used to create other children underneath. + * + * The @entries is an array of eventfs_entry structures which has: + * const char *name + * eventfs_callback callback; + * + * The name is the name of the file, and the callback is a pointer to a function + * that will be called when the file is reference (either by lookup or by + * reading a directory). The callback is of the prototype: + * + * int callback(const char *name, umode_t *mode, void **data, + * const struct file_operations **fops); + * + * When a file needs to be created, this callback will be called with + * name = the name of the file being created (so that the same callback + * may be used for multiple files). + * mode = a place to set the file's mode + * data = A pointer to @data, and the callback may replace it, which will + * cause the file created to pass the new data to the open() call. + * fops = the fops to use for the created file. + * + * NB. @callback is called while holding internal locks of the eventfs + * system. The callback must not call any code that might also call into + * the tracefs or eventfs system or it will risk creating a deadlock. + */ +struct eventfs_inode *eventfs_create_dir(const char *name, struct eventfs_inode *parent, + const struct eventfs_entry *entries, + int size, void *data) +{ + struct eventfs_inode *ei; + + if (!parent) + return ERR_PTR(-EINVAL); + + ei = alloc_ei(name); + if (!ei) + return ERR_PTR(-ENOMEM); + + ei->entries = entries; + ei->nr_entries = size; + ei->data = data; + INIT_LIST_HEAD(&ei->children); + INIT_LIST_HEAD(&ei->list); + + scoped_guard(mutex, &eventfs_mutex) { + if (!parent->is_freed) + list_add_tail_rcu(&ei->list, &parent->children); + } + /* Was the parent freed? */ + if (list_empty(&ei->list)) { + cleanup_ei(ei); + ei = ERR_PTR(-EBUSY); + } + return ei; +} + +/** + * eventfs_create_events_dir - create the top level events directory + * @name: The name of the top level directory to create. + * @parent: Parent dentry for this file in the tracefs directory. + * @entries: A list of entries that represent the files under this directory + * @size: The number of @entries + * @data: The default data to pass to the files (an entry may override it). + * + * This function creates the top of the trace event directory. + * + * See eventfs_create_dir() for use of @entries. + */ +struct eventfs_inode *eventfs_create_events_dir(const char *name, struct dentry *parent, + const struct eventfs_entry *entries, + int size, void *data) +{ + struct dentry *dentry; + struct eventfs_root_inode *rei; + struct eventfs_inode *ei; + struct tracefs_inode *ti; + struct inode *inode; + kuid_t uid; + kgid_t gid; + + if (security_locked_down(LOCKDOWN_TRACEFS)) + return NULL; + + dentry = tracefs_start_creating(name, parent); + if (IS_ERR(dentry)) + return ERR_CAST(dentry); + + ei = alloc_root_ei(name); + if (!ei) + goto fail; + + inode = tracefs_get_inode(dentry->d_sb); + if (unlikely(!inode)) + goto fail; + + // Note: we have a ref to the dentry from tracefs_start_creating() + rei = get_root_inode(ei); + rei->events_dir = dentry; + + ei->entries = entries; + ei->nr_entries = size; + ei->data = data; + + /* Save the ownership of this directory */ + uid = d_inode(dentry->d_parent)->i_uid; + gid = d_inode(dentry->d_parent)->i_gid; + + /* + * The ei->attr will be used as the default values for the + * files beneath this directory. + */ + ei->attr.uid = uid; + ei->attr.gid = gid; + + INIT_LIST_HEAD(&ei->children); + INIT_LIST_HEAD(&ei->list); + + ti = get_tracefs(inode); + ti->flags |= TRACEFS_EVENT_INODE; + ti->private = ei; + + inode->i_mode = S_IFDIR | S_IRWXU | S_IRUGO | S_IXUGO; + inode->i_uid = uid; + inode->i_gid = gid; + inode->i_op = &eventfs_dir_inode_operations; + inode->i_fop = &eventfs_file_operations; + + dentry->d_fsdata = get_ei(ei); + + /* + * Keep all eventfs directories with i_nlink == 1. + * Due to the dynamic nature of the dentry creations and not + * wanting to add a pointer to the parent eventfs_inode in the + * eventfs_inode structure, keeping the i_nlink in sync with the + * number of directories would cause too much complexity for + * something not worth much. Keeping directory links at 1 + * tells userspace not to trust the link number. + */ + d_make_persistent(dentry, inode); + /* The dentry of the "events" parent does keep track though */ + inc_nlink(dentry->d_parent->d_inode); + fsnotify_mkdir(dentry->d_parent->d_inode, dentry); + tracefs_end_creating(dentry); + + return ei; + + fail: + cleanup_ei(ei); + tracefs_failed_creating(dentry); + return ERR_PTR(-ENOMEM); +} + +/** + * eventfs_remove_rec - remove eventfs dir or file from list + * @ei: eventfs_inode to be removed. + * @level: prevent recursion from going more than 3 levels deep. + * + * This function recursively removes eventfs_inodes which + * contains info of files and/or directories. + */ +static void eventfs_remove_rec(struct eventfs_inode *ei, int level) +{ + struct eventfs_inode *ei_child, *tmp; + + /* + * Check recursion depth. It should never be greater than 3: + * 0 - events/ + * 1 - events/group/ + * 2 - events/group/event/ + * 3 - events/group/event/file + */ + if (WARN_ON_ONCE(level > 3)) + return; + + /* search for nested folders or files */ + list_for_each_entry_safe(ei_child, tmp, &ei->children, list) + eventfs_remove_rec(ei_child, level + 1); + + list_del_rcu(&ei->list); + free_ei(ei); +} + +/** + * eventfs_remove_dir - remove eventfs dir or file from list + * @ei: eventfs_inode to be removed. + * + * This function acquire the eventfs_mutex lock and call eventfs_remove_rec() + */ +void eventfs_remove_dir(struct eventfs_inode *ei) +{ + if (!ei) + return; + + guard(mutex)(&eventfs_mutex); + eventfs_remove_rec(ei, 0); +} + +/** + * eventfs_remove_events_dir - remove the top level eventfs directory + * @ei: the event_inode returned by eventfs_create_events_dir(). + * + * This function removes the events main directory + */ +void eventfs_remove_events_dir(struct eventfs_inode *ei) +{ + struct eventfs_root_inode *rei; + struct dentry *dentry; + + rei = get_root_inode(ei); + dentry = rei->events_dir; + if (!dentry) + return; + + rei->events_dir = NULL; + eventfs_remove_dir(ei); + + /* + * Matches the dget() done by tracefs_start_creating() + * in eventfs_create_events_dir() when it the dentry was + * created. In other words, it's a normal dentry that + * sticks around while the other ei->dentry are created + * and destroyed dynamically. + */ + d_invalidate(dentry); + d_make_discardable(dentry); +} + +int eventfs_remount_lock(void) +{ + mutex_lock(&eventfs_mutex); + return srcu_read_lock(&eventfs_srcu); +} + +void eventfs_remount_unlock(int srcu_idx) +{ + srcu_read_unlock(&eventfs_srcu, srcu_idx); + mutex_unlock(&eventfs_mutex); +} diff --git a/fs/tracefs/inode.c b/fs/tracefs/inode.c new file mode 100644 index 000000000..f3d6188a3 --- /dev/null +++ b/fs/tracefs/inode.c @@ -0,0 +1,809 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * inode.c - part of tracefs, a pseudo file system for activating tracing + * + * Based on debugfs by: Greg Kroah-Hartman + * + * Copyright (C) 2014 Red Hat Inc, author: Steven Rostedt + * + * tracefs is the file system that is used by the tracing infrastructure. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +#define TRACEFS_DEFAULT_MODE 0700 +static struct kmem_cache *tracefs_inode_cachep __ro_after_init; + +static struct vfsmount *tracefs_mount; +static int tracefs_mount_count; +static bool tracefs_registered; + +/* + * Keep track of all tracefs_inodes in order to update their + * flags if necessary on a remount. + */ +static DEFINE_SPINLOCK(tracefs_inode_lock); +static LIST_HEAD(tracefs_inodes); + +static struct inode *tracefs_alloc_inode(struct super_block *sb) +{ + struct tracefs_inode *ti; + unsigned long flags; + + ti = alloc_inode_sb(sb, tracefs_inode_cachep, GFP_KERNEL); + if (!ti) + return NULL; + + spin_lock_irqsave(&tracefs_inode_lock, flags); + list_add_rcu(&ti->list, &tracefs_inodes); + spin_unlock_irqrestore(&tracefs_inode_lock, flags); + + return &ti->vfs_inode; +} + +static void tracefs_free_inode(struct inode *inode) +{ + struct tracefs_inode *ti = get_tracefs(inode); + + kmem_cache_free(tracefs_inode_cachep, ti); +} + +static void tracefs_destroy_inode(struct inode *inode) +{ + struct tracefs_inode *ti = get_tracefs(inode); + unsigned long flags; + + spin_lock_irqsave(&tracefs_inode_lock, flags); + list_del_rcu(&ti->list); + spin_unlock_irqrestore(&tracefs_inode_lock, flags); +} + +static ssize_t default_read_file(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + return 0; +} + +static ssize_t default_write_file(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + return count; +} + +static const struct file_operations tracefs_file_operations = { + .read = default_read_file, + .write = default_write_file, + .open = simple_open, + .llseek = noop_llseek, +}; + +static struct tracefs_dir_ops { + int (*mkdir)(const char *name); + int (*rmdir)(const char *name); +} tracefs_ops __ro_after_init; + +static struct dentry *tracefs_syscall_mkdir(struct mnt_idmap *idmap, + struct inode *inode, struct dentry *dentry, + umode_t mode) +{ + struct tracefs_inode *ti; + struct name_snapshot name; + int ret; + + /* + * This is a new directory that does not take the default of + * the rootfs. It becomes the default permissions for all the + * files and directories underneath it. + */ + ti = get_tracefs(inode); + ti->flags |= TRACEFS_INSTANCE_INODE; + ti->private = inode; + + /* + * The mkdir call can call the generic functions that create + * the files within the tracefs system. It is up to the individual + * mkdir routine to handle races. + */ + take_dentry_name_snapshot(&name, dentry); + inode_unlock(inode); + ret = tracefs_ops.mkdir(name.name.name); + inode_lock(inode); + release_dentry_name_snapshot(&name); + + return ERR_PTR(ret); +} + +static int tracefs_syscall_rmdir(struct inode *inode, struct dentry *dentry) +{ + struct name_snapshot name; + int ret; + + /* + * The rmdir call can call the generic functions that create + * the files within the tracefs system. It is up to the individual + * rmdir routine to handle races. + * This time we need to unlock not only the parent (inode) but + * also the directory that is being deleted. + */ + take_dentry_name_snapshot(&name, dentry); + inode_unlock(inode); + inode_unlock(d_inode(dentry)); + + ret = tracefs_ops.rmdir(name.name.name); + + inode_lock_nested(inode, I_MUTEX_PARENT); + inode_lock(d_inode(dentry)); + release_dentry_name_snapshot(&name); + + return ret; +} + +static void set_tracefs_inode_owner(struct inode *inode) +{ + struct tracefs_inode *ti = get_tracefs(inode); + struct inode *root_inode = ti->private; + kuid_t uid; + kgid_t gid; + + uid = root_inode->i_uid; + gid = root_inode->i_gid; + + /* + * If the root is not the mount point, then check the root's + * permissions. If it was never set, then default to the + * mount point. + */ + if (root_inode != d_inode(root_inode->i_sb->s_root)) { + struct tracefs_inode *rti; + + rti = get_tracefs(root_inode); + root_inode = d_inode(root_inode->i_sb->s_root); + + if (!(rti->flags & TRACEFS_UID_PERM_SET)) + uid = root_inode->i_uid; + + if (!(rti->flags & TRACEFS_GID_PERM_SET)) + gid = root_inode->i_gid; + } + + /* + * If this inode has never been referenced, then update + * the permissions to the superblock. + */ + if (!(ti->flags & TRACEFS_UID_PERM_SET)) + inode->i_uid = uid; + + if (!(ti->flags & TRACEFS_GID_PERM_SET)) + inode->i_gid = gid; +} + +static int tracefs_permission(struct mnt_idmap *idmap, + struct inode *inode, int mask) +{ + set_tracefs_inode_owner(inode); + return generic_permission(idmap, inode, mask); +} + +static int tracefs_getattr(struct mnt_idmap *idmap, + const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int flags) +{ + struct inode *inode = d_backing_inode(path->dentry); + + set_tracefs_inode_owner(inode); + generic_fillattr(idmap, request_mask, inode, stat); + return 0; +} + +static int tracefs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +{ + unsigned int ia_valid = attr->ia_valid; + struct inode *inode = d_inode(dentry); + struct tracefs_inode *ti = get_tracefs(inode); + + if (ia_valid & ATTR_UID) + ti->flags |= TRACEFS_UID_PERM_SET; + + if (ia_valid & ATTR_GID) + ti->flags |= TRACEFS_GID_PERM_SET; + + return simple_setattr(idmap, dentry, attr); +} + +static const struct inode_operations tracefs_instance_dir_inode_operations = { + .lookup = simple_lookup, + .mkdir = tracefs_syscall_mkdir, + .rmdir = tracefs_syscall_rmdir, + .permission = tracefs_permission, + .getattr = tracefs_getattr, + .setattr = tracefs_setattr, +}; + +static const struct inode_operations tracefs_dir_inode_operations = { + .lookup = simple_lookup, + .permission = tracefs_permission, + .getattr = tracefs_getattr, + .setattr = tracefs_setattr, +}; + +static const struct inode_operations tracefs_file_inode_operations = { + .permission = tracefs_permission, + .getattr = tracefs_getattr, + .setattr = tracefs_setattr, +}; + +struct inode *tracefs_get_inode(struct super_block *sb) +{ + struct inode *inode = new_inode(sb); + if (inode) { + inode->i_ino = get_next_ino(); + simple_inode_init_ts(inode); + } + return inode; +} + +struct tracefs_fs_info { + kuid_t uid; + kgid_t gid; + umode_t mode; + /* Opt_* bitfield. */ + unsigned int opts; +}; + +enum { + Opt_uid, + Opt_gid, + Opt_mode, +}; + +static const struct fs_parameter_spec tracefs_param_specs[] = { + fsparam_gid ("gid", Opt_gid), + fsparam_u32oct ("mode", Opt_mode), + fsparam_uid ("uid", Opt_uid), + {} +}; + +static int tracefs_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct tracefs_fs_info *opts = fc->s_fs_info; + struct fs_parse_result result; + int opt; + + opt = fs_parse(fc, tracefs_param_specs, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_uid: + opts->uid = result.uid; + break; + case Opt_gid: + opts->gid = result.gid; + break; + case Opt_mode: + opts->mode = result.uint_32 & S_IALLUGO; + break; + /* + * We might like to report bad mount options here; + * but traditionally tracefs has ignored all mount options + */ + } + + opts->opts |= BIT(opt); + + return 0; +} + +static int tracefs_apply_options(struct super_block *sb, bool remount) +{ + struct tracefs_fs_info *fsi = sb->s_fs_info; + struct inode *inode = d_inode(sb->s_root); + struct tracefs_inode *ti; + bool update_uid, update_gid; + int srcu_idx; + umode_t tmp_mode; + + /* + * On remount, only reset mode/uid/gid if they were provided as mount + * options. + */ + + if (!remount || fsi->opts & BIT(Opt_mode)) { + tmp_mode = READ_ONCE(inode->i_mode) & ~S_IALLUGO; + tmp_mode |= fsi->mode; + WRITE_ONCE(inode->i_mode, tmp_mode); + } + + if (!remount || fsi->opts & BIT(Opt_uid)) + inode->i_uid = fsi->uid; + + if (!remount || fsi->opts & BIT(Opt_gid)) + inode->i_gid = fsi->gid; + + if (remount && (fsi->opts & BIT(Opt_uid) || fsi->opts & BIT(Opt_gid))) { + + update_uid = fsi->opts & BIT(Opt_uid); + update_gid = fsi->opts & BIT(Opt_gid); + + srcu_idx = eventfs_remount_lock(); + rcu_read_lock(); + list_for_each_entry_rcu(ti, &tracefs_inodes, list) { + if (update_uid) { + ti->flags &= ~TRACEFS_UID_PERM_SET; + ti->vfs_inode.i_uid = fsi->uid; + } + + if (update_gid) { + ti->flags &= ~TRACEFS_GID_PERM_SET; + ti->vfs_inode.i_gid = fsi->gid; + } + + /* + * Note, the above ti->vfs_inode updates are + * used in eventfs_remount() so they must come + * before calling it. + */ + if (ti->flags & TRACEFS_EVENT_INODE) + eventfs_remount(ti, update_uid, update_gid); + } + rcu_read_unlock(); + eventfs_remount_unlock(srcu_idx); + } + + return 0; +} + +static int tracefs_reconfigure(struct fs_context *fc) +{ + struct super_block *sb = fc->root->d_sb; + struct tracefs_fs_info *sb_opts = sb->s_fs_info; + struct tracefs_fs_info *new_opts = fc->s_fs_info; + + if (!new_opts) + return 0; + + sync_filesystem(sb); + /* structure copy of new mount options to sb */ + *sb_opts = *new_opts; + + return tracefs_apply_options(sb, true); +} + +static int tracefs_show_options(struct seq_file *m, struct dentry *root) +{ + struct tracefs_fs_info *fsi = root->d_sb->s_fs_info; + + if (!uid_eq(fsi->uid, GLOBAL_ROOT_UID)) + seq_printf(m, ",uid=%u", + from_kuid_munged(&init_user_ns, fsi->uid)); + if (!gid_eq(fsi->gid, GLOBAL_ROOT_GID)) + seq_printf(m, ",gid=%u", + from_kgid_munged(&init_user_ns, fsi->gid)); + if (fsi->mode != TRACEFS_DEFAULT_MODE) + seq_printf(m, ",mode=%o", fsi->mode); + + return 0; +} + +static int tracefs_drop_inode(struct inode *inode) +{ + struct tracefs_inode *ti = get_tracefs(inode); + + /* + * This inode is being freed and cannot be used for + * eventfs. Clear the flag so that it doesn't call into + * eventfs during the remount flag updates. The eventfs_inode + * gets freed after an SRCU cycle, so the content will still + * be safe if the iteration is going on now. + */ + ti->flags &= ~TRACEFS_EVENT_INODE; + + return 1; +} + +static const struct super_operations tracefs_super_operations = { + .alloc_inode = tracefs_alloc_inode, + .free_inode = tracefs_free_inode, + .destroy_inode = tracefs_destroy_inode, + .drop_inode = tracefs_drop_inode, + .statfs = simple_statfs, + .show_options = tracefs_show_options, +}; + +/* + * It would be cleaner if eventfs had its own dentry ops. + * + * Note that d_revalidate is called potentially under RCU, + * so it can't take the eventfs mutex etc. It's fine - if + * we open a file just as it's marked dead, things will + * still work just fine, and just see the old stale case. + */ +static void tracefs_d_release(struct dentry *dentry) +{ + if (dentry->d_fsdata) + eventfs_d_release(dentry); +} + +static int tracefs_d_revalidate(struct inode *inode, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + struct eventfs_inode *ei = dentry->d_fsdata; + + return !(ei && ei->is_freed); +} + +static int tracefs_d_delete(const struct dentry *dentry) +{ + /* + * We want to keep eventfs dentries around but not tracefs + * ones. eventfs dentries have content in d_fsdata. + * Use d_fsdata to determine if it's a eventfs dentry or not. + */ + return dentry->d_fsdata == NULL; +} + +static const struct dentry_operations tracefs_dentry_operations = { + .d_revalidate = tracefs_d_revalidate, + .d_release = tracefs_d_release, + .d_delete = tracefs_d_delete, +}; + +static int tracefs_fill_super(struct super_block *sb, struct fs_context *fc) +{ + static const struct tree_descr trace_files[] = {{""}}; + int err; + + err = simple_fill_super(sb, TRACEFS_MAGIC, trace_files); + if (err) + return err; + + sb->s_op = &tracefs_super_operations; + tracefs_apply_options(sb, false); + set_default_d_op(sb, &tracefs_dentry_operations); + + return 0; +} + +static int tracefs_get_tree(struct fs_context *fc) +{ + int err = get_tree_single(fc, tracefs_fill_super); + + if (err) + return err; + + return tracefs_reconfigure(fc); +} + +static void tracefs_free_fc(struct fs_context *fc) +{ + kfree(fc->s_fs_info); +} + +static const struct fs_context_operations tracefs_context_ops = { + .free = tracefs_free_fc, + .parse_param = tracefs_parse_param, + .get_tree = tracefs_get_tree, + .reconfigure = tracefs_reconfigure, +}; + +static int tracefs_init_fs_context(struct fs_context *fc) +{ + struct tracefs_fs_info *fsi; + + fsi = kzalloc_obj(struct tracefs_fs_info); + if (!fsi) + return -ENOMEM; + + fsi->mode = TRACEFS_DEFAULT_MODE; + + fc->s_fs_info = fsi; + fc->ops = &tracefs_context_ops; + return 0; +} + +static struct file_system_type trace_fs_type = { + .owner = THIS_MODULE, + .name = "tracefs", + .init_fs_context = tracefs_init_fs_context, + .parameters = tracefs_param_specs, + .kill_sb = kill_anon_super, +}; +MODULE_ALIAS_FS("tracefs"); + +struct dentry *tracefs_start_creating(const char *name, struct dentry *parent) +{ + struct dentry *dentry; + int error; + + pr_debug("tracefs: creating file '%s'\n",name); + + error = simple_pin_fs(&trace_fs_type, &tracefs_mount, + &tracefs_mount_count); + if (error) + return ERR_PTR(error); + + /* If the parent is not specified, we create it in the root. + * We need the root dentry to do this, which is in the super + * block. A pointer to that is in the struct vfsmount that we + * have around. + */ + if (!parent) + parent = tracefs_mount->mnt_root; + + dentry = simple_start_creating(parent, name); + if (IS_ERR(dentry)) + simple_release_fs(&tracefs_mount, &tracefs_mount_count); + + return dentry; +} + +struct dentry *tracefs_failed_creating(struct dentry *dentry) +{ + simple_done_creating(dentry); + simple_release_fs(&tracefs_mount, &tracefs_mount_count); + return NULL; +} + +struct dentry *tracefs_end_creating(struct dentry *dentry) +{ + simple_done_creating(dentry); + return dentry; // borrowed +} + +/* Find the inode that this will use for default */ +static struct inode *instance_inode(struct dentry *parent, struct inode *inode) +{ + struct tracefs_inode *ti; + + /* If parent is NULL then use root inode */ + if (!parent) + return d_inode(inode->i_sb->s_root); + + /* Find the inode that is flagged as an instance or the root inode */ + while (!IS_ROOT(parent)) { + ti = get_tracefs(d_inode(parent)); + if (ti->flags & TRACEFS_INSTANCE_INODE) + break; + parent = parent->d_parent; + } + + return d_inode(parent); +} + +/** + * tracefs_create_file - create a file in the tracefs filesystem + * @name: a pointer to a string containing the name of the file to create. + * @mode: the permission that the file should have. + * @parent: a pointer to the parent dentry for this file. This should be a + * directory dentry if set. If this parameter is NULL, then the + * file will be created in the root of the tracefs filesystem. + * @data: a pointer to something that the caller will want to get to later + * on. The inode.i_private pointer will point to this value on + * the open() call. + * @fops: a pointer to a struct file_operations that should be used for + * this file. + * + * This is the basic "create a file" function for tracefs. It allows for a + * wide range of flexibility in creating a file, or a directory (if you want + * to create a directory, the tracefs_create_dir() function is + * recommended to be used instead.) + * + * This function will return a pointer to a dentry if it succeeds. This + * pointer must be passed to the tracefs_remove() function when the file is + * to be removed (no automatic cleanup happens if your module is unloaded, + * you are responsible here.) If an error occurs, %NULL will be returned. + * + * If tracefs is not enabled in the kernel, the value -%ENODEV will be + * returned. + */ +struct dentry *tracefs_create_file(const char *name, umode_t mode, + struct dentry *parent, void *data, + const struct file_operations *fops) +{ + struct tracefs_inode *ti; + struct dentry *dentry; + struct inode *inode; + + if (security_locked_down(LOCKDOWN_TRACEFS)) + return NULL; + + if (!(mode & S_IFMT)) + mode |= S_IFREG; + BUG_ON(!S_ISREG(mode)); + dentry = tracefs_start_creating(name, parent); + + if (IS_ERR(dentry)) + return NULL; + + inode = tracefs_get_inode(dentry->d_sb); + if (unlikely(!inode)) + return tracefs_failed_creating(dentry); + + ti = get_tracefs(inode); + ti->private = instance_inode(parent, inode); + + inode->i_mode = mode; + inode->i_op = &tracefs_file_inode_operations; + inode->i_fop = fops ? fops : &tracefs_file_operations; + inode->i_private = data; + inode->i_uid = d_inode(dentry->d_parent)->i_uid; + inode->i_gid = d_inode(dentry->d_parent)->i_gid; + d_make_persistent(dentry, inode); + fsnotify_create(d_inode(dentry->d_parent), dentry); + return tracefs_end_creating(dentry); +} +EXPORT_SYMBOL_GPL(tracefs_create_file); + +static struct dentry *__create_dir(const char *name, struct dentry *parent, + const struct inode_operations *ops) +{ + struct tracefs_inode *ti; + struct dentry *dentry = tracefs_start_creating(name, parent); + struct inode *inode; + + if (IS_ERR(dentry)) + return NULL; + + inode = tracefs_get_inode(dentry->d_sb); + if (unlikely(!inode)) + return tracefs_failed_creating(dentry); + + /* Do not set bits for OTH */ + inode->i_mode = S_IFDIR | S_IRWXU | S_IRUSR| S_IRGRP | S_IXUSR | S_IXGRP; + inode->i_op = ops; + inode->i_fop = &simple_dir_operations; + inode->i_uid = d_inode(dentry->d_parent)->i_uid; + inode->i_gid = d_inode(dentry->d_parent)->i_gid; + + ti = get_tracefs(inode); + ti->private = instance_inode(parent, inode); + + /* directory inodes start off with i_nlink == 2 (for "." entry) */ + inc_nlink(inode); + d_make_persistent(dentry, inode); + inc_nlink(d_inode(dentry->d_parent)); + fsnotify_mkdir(d_inode(dentry->d_parent), dentry); + return tracefs_end_creating(dentry); +} + +/** + * tracefs_create_dir - create a directory in the tracefs filesystem + * @name: a pointer to a string containing the name of the directory to + * create. + * @parent: a pointer to the parent dentry for this file. This should be a + * directory dentry if set. If this parameter is NULL, then the + * directory will be created in the root of the tracefs filesystem. + * + * This function creates a directory in tracefs with the given name. + * + * This function will return a pointer to a dentry if it succeeds. This + * pointer must be passed to the tracefs_remove() function when the file is + * to be removed. If an error occurs, %NULL will be returned. + * + * If tracing is not enabled in the kernel, the value -%ENODEV will be + * returned. + */ +struct dentry *tracefs_create_dir(const char *name, struct dentry *parent) +{ + if (security_locked_down(LOCKDOWN_TRACEFS)) + return NULL; + + return __create_dir(name, parent, &tracefs_dir_inode_operations); +} + +/** + * tracefs_create_instance_dir - create the tracing instances directory + * @name: The name of the instances directory to create + * @parent: The parent directory that the instances directory will exist + * @mkdir: The function to call when a mkdir is performed. + * @rmdir: The function to call when a rmdir is performed. + * + * Only one instances directory is allowed. + * + * The instances directory is special as it allows for mkdir and rmdir + * to be done by userspace. When a mkdir or rmdir is performed, the inode + * locks are released and the methods passed in (@mkdir and @rmdir) are + * called without locks and with the name of the directory being created + * within the instances directory. + * + * Returns the dentry of the instances directory. + */ +__init struct dentry *tracefs_create_instance_dir(const char *name, + struct dentry *parent, + int (*mkdir)(const char *name), + int (*rmdir)(const char *name)) +{ + struct dentry *dentry; + + /* Only allow one instance of the instances directory. */ + if (WARN_ON(tracefs_ops.mkdir || tracefs_ops.rmdir)) + return NULL; + + dentry = __create_dir(name, parent, &tracefs_instance_dir_inode_operations); + if (!dentry) + return NULL; + + tracefs_ops.mkdir = mkdir; + tracefs_ops.rmdir = rmdir; + + return dentry; +} + +static void remove_one(struct dentry *victim) +{ + simple_release_fs(&tracefs_mount, &tracefs_mount_count); +} + +/** + * tracefs_remove - recursively removes a directory + * @dentry: a pointer to a the dentry of the directory to be removed. + * + * This function recursively removes a directory tree in tracefs that + * was previously created with a call to another tracefs function + * (like tracefs_create_file() or variants thereof.) + */ +void tracefs_remove(struct dentry *dentry) +{ + if (IS_ERR_OR_NULL(dentry)) + return; + + simple_pin_fs(&trace_fs_type, &tracefs_mount, &tracefs_mount_count); + simple_recursive_removal(dentry, remove_one); + simple_release_fs(&tracefs_mount, &tracefs_mount_count); +} + +/** + * tracefs_initialized - Tells whether tracefs has been registered + */ +bool tracefs_initialized(void) +{ + return tracefs_registered; +} + +static void init_once(void *foo) +{ + struct tracefs_inode *ti = (struct tracefs_inode *) foo; + + /* inode_init_once() calls memset() on the vfs_inode portion */ + inode_init_once(&ti->vfs_inode); + + /* Zero out the rest */ + memset_after(ti, 0, vfs_inode); +} + +static int __init tracefs_init(void) +{ + int retval; + + tracefs_inode_cachep = kmem_cache_create("tracefs_inode_cache", + sizeof(struct tracefs_inode), + 0, (SLAB_RECLAIM_ACCOUNT| + SLAB_ACCOUNT), + init_once); + if (!tracefs_inode_cachep) + return -ENOMEM; + + retval = sysfs_create_mount_point(kernel_kobj, "tracing"); + if (retval) + return -EINVAL; + + retval = register_filesystem(&trace_fs_type); + if (!retval) + tracefs_registered = true; + + return retval; +} +core_initcall(tracefs_init); diff --git a/fs/tracefs/internal.h b/fs/tracefs/internal.h new file mode 100644 index 000000000..c61481d04 --- /dev/null +++ b/fs/tracefs/internal.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _TRACEFS_INTERNAL_H +#define _TRACEFS_INTERNAL_H + +enum { + TRACEFS_EVENT_INODE = BIT(1), + TRACEFS_GID_PERM_SET = BIT(2), + TRACEFS_UID_PERM_SET = BIT(3), + TRACEFS_INSTANCE_INODE = BIT(4), +}; + +struct tracefs_inode { + struct inode vfs_inode; + /* The below gets initialized with memset_after(ti, 0, vfs_inode) */ + struct list_head list; + unsigned long flags; + void *private; +}; + +/* + * struct eventfs_attr - cache the mode and ownership of a eventfs entry + * @mode: saved mode plus flags of what is saved + * @uid: saved uid if changed + * @gid: saved gid if changed + */ +struct eventfs_attr { + int mode; + kuid_t uid; + kgid_t gid; +}; + +/* + * struct eventfs_inode - hold the properties of the eventfs directories. + * @list: link list into the parent directory + * @rcu: Union with @list for freeing + * @children: link list into the child eventfs_inode + * @entries: the array of entries representing the files in the directory + * @name: the name of the directory to create + * @entry_attrs: Saved mode and ownership of the @d_children + * @data: The private data to pass to the callbacks + * @attr: Saved mode and ownership of eventfs_inode itself + * @is_freed: Flag set if the eventfs is on its way to be freed + * Note if is_freed is set, then dentry is corrupted. + * @is_events: Flag set for only the top level "events" directory + * @nr_entries: The number of items in @entries + * @ino: The saved inode number + */ +struct eventfs_inode { + struct list_head list; + union { + struct list_head children; + struct rcu_head rcu; + }; + const struct eventfs_entry *entries; + const char *name; + struct eventfs_attr *entry_attrs; + void *data; + struct eventfs_attr attr; + struct kref kref; + unsigned int is_freed:1; + unsigned int is_events:1; + unsigned int nr_entries:30; + unsigned int ino; +}; + +static inline struct tracefs_inode *get_tracefs(const struct inode *inode) +{ + return container_of(inode, struct tracefs_inode, vfs_inode); +} + +struct dentry *tracefs_start_creating(const char *name, struct dentry *parent); +struct dentry *tracefs_end_creating(struct dentry *dentry); +struct dentry *tracefs_failed_creating(struct dentry *dentry); +struct inode *tracefs_get_inode(struct super_block *sb); + +void eventfs_remount(struct tracefs_inode *ti, bool update_uid, bool update_gid); +void eventfs_d_release(struct dentry *dentry); + +int eventfs_remount_lock(void); +void eventfs_remount_unlock(int srcu_idx); + +#endif /* _TRACEFS_INTERNAL_H */ diff --git a/fs/ubifs/Kconfig b/fs/ubifs/Kconfig new file mode 100644 index 000000000..45d3d207f --- /dev/null +++ b/fs/ubifs/Kconfig @@ -0,0 +1,101 @@ +# SPDX-License-Identifier: GPL-2.0-only +config UBIFS_FS + tristate "UBIFS file system support" + select CRC16 + select CRC32 + select CRYPTO if UBIFS_FS_ADVANCED_COMPR + select CRYPTO if UBIFS_FS_LZO + select CRYPTO if UBIFS_FS_ZLIB + select CRYPTO if UBIFS_FS_ZSTD + select CRYPTO_LZO if UBIFS_FS_LZO + select CRYPTO_DEFLATE if UBIFS_FS_ZLIB + select CRYPTO_ZSTD if UBIFS_FS_ZSTD + select CRYPTO_HASH_INFO + select UBIFS_FS_XATTR if FS_ENCRYPTION + select FS_ENCRYPTION_ALGS if FS_ENCRYPTION + depends on MTD_UBI + help + UBIFS is a file system for flash devices which works on top of UBI. + +if UBIFS_FS + +config UBIFS_FS_ADVANCED_COMPR + bool "Advanced compression options" + help + This option allows to explicitly choose which compressions, if any, + are enabled in UBIFS. Removing compressors means inability to read + existing file systems. + + If unsure, say 'N'. + +config UBIFS_FS_LZO + bool "LZO compression support" if UBIFS_FS_ADVANCED_COMPR + default y + help + LZO compressor is generally faster than zlib but compresses worse. + Say 'Y' if unsure. + +config UBIFS_FS_ZLIB + bool "ZLIB compression support" if UBIFS_FS_ADVANCED_COMPR + default y + help + Zlib compresses better than LZO but it is slower. Say 'Y' if unsure. + +config UBIFS_FS_ZSTD + bool "ZSTD compression support" if UBIFS_FS_ADVANCED_COMPR + depends on UBIFS_FS + default y + help + ZSTD compresses is a big win in speed over Zlib and + in compression ratio over LZO. Say 'Y' if unsure. + +config UBIFS_ATIME_SUPPORT + bool "Access time support" + default n + help + Originally UBIFS did not support atime, because it looked like a bad idea due + increased flash wear. This option adds atime support and it is disabled by default + to preserve the old behavior. If you enable this option, UBIFS starts updating atime, + which means that file-system read operations will cause writes (inode atime + updates). This may affect file-system performance and increase flash device wear, + so be careful. How often atime is updated depends on the selected strategy: + strictatime is the "heavy", relatime is "lighter", etc. + + If unsure, say 'N' + +config UBIFS_FS_XATTR + bool "UBIFS XATTR support" + default y + help + Saying Y here includes support for extended attributes (xattrs). + Xattrs are name:value pairs associated with inodes by + the kernel or by users (see the attr(5) manual page). + + If unsure, say Y. + +config UBIFS_FS_SECURITY + bool "UBIFS Security Labels" + depends on UBIFS_FS_XATTR + default y + help + Security labels provide an access control facility to support Linux + Security Models (LSMs) accepted by AppArmor, SELinux, Smack and TOMOYO + Linux. This option enables an extended attribute handler for file + security labels in the ubifs filesystem, so that it requires enabling + the extended attribute support in advance. + + If you are not using a security module, say N. + +config UBIFS_FS_AUTHENTICATION + bool "UBIFS authentication support" + select KEYS + select CRYPTO_HMAC + select SYSTEM_DATA_VERIFICATION + help + Enable authentication support for UBIFS. This feature offers protection + against offline changes for both data and metadata of the filesystem. + If you say yes here you should also select a hashing algorithm such as + sha256, these are not selected automatically since there are many + different options. + +endif # UBIFS_FS diff --git a/fs/ubifs/Makefile b/fs/ubifs/Makefile new file mode 100644 index 000000000..314c80b24 --- /dev/null +++ b/fs/ubifs/Makefile @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_UBIFS_FS) += ubifs.o + +ubifs-y += shrinker.o journal.o file.o dir.o super.o sb.o io.o +ubifs-y += tnc.o master.o scan.o replay.o log.o commit.o gc.o orphan.o +ubifs-y += budget.o find.o tnc_commit.o compress.o lpt.o lprops.o +ubifs-y += recovery.o ioctl.o lpt_commit.o tnc_misc.o debug.o +ubifs-y += misc.o sysfs.o +ubifs-$(CONFIG_FS_ENCRYPTION) += crypto.o +ubifs-$(CONFIG_UBIFS_FS_XATTR) += xattr.o +ubifs-$(CONFIG_UBIFS_FS_AUTHENTICATION) += auth.o diff --git a/fs/ubifs/auth.c b/fs/ubifs/auth.c new file mode 100644 index 000000000..a4a0158f7 --- /dev/null +++ b/fs/ubifs/auth.c @@ -0,0 +1,530 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * This file is part of UBIFS. + * + * Copyright (C) 2018 Pengutronix, Sascha Hauer + */ + +/* + * This file implements various helper functions for UBIFS authentication support + */ + +#include +#include +#include +#include +#include + +#include "ubifs.h" + +/** + * __ubifs_node_calc_hash - calculate the hash of a UBIFS node + * @c: UBIFS file-system description object + * @node: the node to calculate a hash for + * @hash: the returned hash + * + * Returns 0 for success or a negative error code otherwise. + */ +int __ubifs_node_calc_hash(const struct ubifs_info *c, const void *node, + u8 *hash) +{ + const struct ubifs_ch *ch = node; + + return crypto_shash_tfm_digest(c->hash_tfm, node, le32_to_cpu(ch->len), + hash); +} + +/** + * ubifs_hash_calc_hmac - calculate a HMAC from a hash + * @c: UBIFS file-system description object + * @hash: the node to calculate a HMAC for + * @hmac: the returned HMAC + * + * Returns 0 for success or a negative error code otherwise. + */ +static int ubifs_hash_calc_hmac(const struct ubifs_info *c, const u8 *hash, + u8 *hmac) +{ + return crypto_shash_tfm_digest(c->hmac_tfm, hash, c->hash_len, hmac); +} + +/** + * ubifs_prepare_auth_node - Prepare an authentication node + * @c: UBIFS file-system description object + * @node: the node to calculate a hash for + * @inhash: input hash of previous nodes + * + * This function prepares an authentication node for writing onto flash. + * It creates a HMAC from the given input hash and writes it to the node. + * + * Returns 0 for success or a negative error code otherwise. + */ +int ubifs_prepare_auth_node(struct ubifs_info *c, void *node, + struct shash_desc *inhash) +{ + struct ubifs_auth_node *auth = node; + u8 hash[UBIFS_HASH_ARR_SZ]; + int err; + + { + SHASH_DESC_ON_STACK(hash_desc, c->hash_tfm); + + hash_desc->tfm = c->hash_tfm; + ubifs_shash_copy_state(c, inhash, hash_desc); + + err = crypto_shash_final(hash_desc, hash); + if (err) + return err; + } + + err = ubifs_hash_calc_hmac(c, hash, auth->hmac); + if (err) + return err; + + auth->ch.node_type = UBIFS_AUTH_NODE; + ubifs_prepare_node(c, auth, ubifs_auth_node_sz(c), 0); + return 0; +} + +static struct shash_desc *ubifs_get_desc(const struct ubifs_info *c, + struct crypto_shash *tfm) +{ + struct shash_desc *desc; + int err; + + if (!ubifs_authenticated(c)) + return NULL; + + desc = kmalloc(sizeof(*desc) + crypto_shash_descsize(tfm), GFP_KERNEL); + if (!desc) + return ERR_PTR(-ENOMEM); + + desc->tfm = tfm; + + err = crypto_shash_init(desc); + if (err) { + kfree(desc); + return ERR_PTR(err); + } + + return desc; +} + +/** + * __ubifs_hash_get_desc - get a descriptor suitable for hashing a node + * @c: UBIFS file-system description object + * + * This function returns a descriptor suitable for hashing a node. Free after use + * with kfree. + */ +struct shash_desc *__ubifs_hash_get_desc(const struct ubifs_info *c) +{ + return ubifs_get_desc(c, c->hash_tfm); +} + +/** + * ubifs_bad_hash - Report hash mismatches + * @c: UBIFS file-system description object + * @node: the node + * @hash: the expected hash + * @lnum: the LEB @node was read from + * @offs: offset in LEB @node was read from + * + * This function reports a hash mismatch when a node has a different hash than + * expected. + */ +void ubifs_bad_hash(const struct ubifs_info *c, const void *node, const u8 *hash, + int lnum, int offs) +{ + int len = min(c->hash_len, 20); + int cropped = len != c->hash_len; + const char *cont = cropped ? "..." : ""; + + u8 calc[UBIFS_HASH_ARR_SZ]; + + __ubifs_node_calc_hash(c, node, calc); + + ubifs_err(c, "hash mismatch on node at LEB %d:%d", lnum, offs); + ubifs_err(c, "hash expected: %*ph%s", len, hash, cont); + ubifs_err(c, "hash calculated: %*ph%s", len, calc, cont); +} + +/** + * __ubifs_node_check_hash - check the hash of a node against given hash + * @c: UBIFS file-system description object + * @node: the node + * @expected: the expected hash + * + * This function calculates a hash over a node and compares it to the given hash. + * Returns 0 if both hashes are equal or authentication is disabled, otherwise a + * negative error code is returned. + */ +int __ubifs_node_check_hash(const struct ubifs_info *c, const void *node, + const u8 *expected) +{ + u8 calc[UBIFS_HASH_ARR_SZ]; + int err; + + err = __ubifs_node_calc_hash(c, node, calc); + if (err) + return err; + + if (ubifs_check_hash(c, expected, calc)) + return -EPERM; + + return 0; +} + +/** + * ubifs_sb_verify_signature - verify the signature of a superblock + * @c: UBIFS file-system description object + * @sup: The superblock node + * + * To support offline signed images the superblock can be signed with a + * PKCS#7 signature. The signature is placed directly behind the superblock + * node in an ubifs_sig_node. + * + * Returns 0 when the signature can be successfully verified or a negative + * error code if not. + */ +int ubifs_sb_verify_signature(struct ubifs_info *c, + const struct ubifs_sb_node *sup) +{ + int err; + struct ubifs_scan_leb *sleb; + struct ubifs_scan_node *snod; + const struct ubifs_sig_node *signode; + + sleb = ubifs_scan(c, UBIFS_SB_LNUM, UBIFS_SB_NODE_SZ, c->sbuf, 0); + if (IS_ERR(sleb)) { + err = PTR_ERR(sleb); + return err; + } + + if (sleb->nodes_cnt == 0) { + ubifs_err(c, "Unable to find signature node"); + err = -EINVAL; + goto out_destroy; + } + + snod = list_first_entry(&sleb->nodes, struct ubifs_scan_node, list); + + if (snod->type != UBIFS_SIG_NODE) { + ubifs_err(c, "Signature node is of wrong type"); + err = -EINVAL; + goto out_destroy; + } + + signode = snod->node; + + if (le32_to_cpu(signode->len) > snod->len + sizeof(struct ubifs_sig_node)) { + ubifs_err(c, "invalid signature len %d", le32_to_cpu(signode->len)); + err = -EINVAL; + goto out_destroy; + } + + if (le32_to_cpu(signode->type) != UBIFS_SIGNATURE_TYPE_PKCS7) { + ubifs_err(c, "Signature type %d is not supported\n", + le32_to_cpu(signode->type)); + err = -EINVAL; + goto out_destroy; + } + + err = verify_pkcs7_signature(sup, sizeof(struct ubifs_sb_node), + signode->sig, le32_to_cpu(signode->len), + NULL, VERIFYING_UNSPECIFIED_SIGNATURE, + NULL, NULL); + + if (err) + ubifs_err(c, "Failed to verify signature"); + else + ubifs_msg(c, "Successfully verified super block signature"); + +out_destroy: + ubifs_scan_destroy(sleb); + + return err; +} + +/** + * ubifs_init_authentication - initialize UBIFS authentication support + * @c: UBIFS file-system description object + * + * This function returns 0 for success or a negative error code otherwise. + */ +int ubifs_init_authentication(struct ubifs_info *c) +{ + struct key *keyring_key; + const struct user_key_payload *ukp; + int err; + char hmac_name[CRYPTO_MAX_ALG_NAME]; + + if (!c->auth_hash_name) { + ubifs_err(c, "authentication hash name needed with authentication"); + return -EINVAL; + } + + c->auth_hash_algo = match_string(hash_algo_name, HASH_ALGO__LAST, + c->auth_hash_name); + if ((int)c->auth_hash_algo < 0) { + ubifs_err(c, "Unknown hash algo %s specified", + c->auth_hash_name); + return -EINVAL; + } + + snprintf(hmac_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)", + c->auth_hash_name); + + keyring_key = request_key(&key_type_logon, c->auth_key_name, NULL); + + if (IS_ERR(keyring_key)) { + ubifs_err(c, "Failed to request key: %ld", + PTR_ERR(keyring_key)); + return PTR_ERR(keyring_key); + } + + down_read(&keyring_key->sem); + + if (keyring_key->type != &key_type_logon) { + ubifs_err(c, "key type must be logon"); + err = -ENOKEY; + goto out; + } + + ukp = user_key_payload_locked(keyring_key); + if (!ukp) { + /* key was revoked before we acquired its semaphore */ + err = -EKEYREVOKED; + goto out; + } + + c->hash_tfm = crypto_alloc_shash(c->auth_hash_name, 0, 0); + if (IS_ERR(c->hash_tfm)) { + err = PTR_ERR(c->hash_tfm); + ubifs_err(c, "Can not allocate %s: %d", + c->auth_hash_name, err); + goto out; + } + + c->hash_len = crypto_shash_digestsize(c->hash_tfm); + if (c->hash_len > UBIFS_HASH_ARR_SZ) { + ubifs_err(c, "hash %s is bigger than maximum allowed hash size (%d > %d)", + c->auth_hash_name, c->hash_len, UBIFS_HASH_ARR_SZ); + err = -EINVAL; + goto out_free_hash; + } + + c->hmac_tfm = crypto_alloc_shash(hmac_name, 0, 0); + if (IS_ERR(c->hmac_tfm)) { + err = PTR_ERR(c->hmac_tfm); + ubifs_err(c, "Can not allocate %s: %d", hmac_name, err); + goto out_free_hash; + } + + c->hmac_desc_len = crypto_shash_digestsize(c->hmac_tfm); + if (c->hmac_desc_len > UBIFS_HMAC_ARR_SZ) { + ubifs_err(c, "hmac %s is bigger than maximum allowed hmac size (%d > %d)", + hmac_name, c->hmac_desc_len, UBIFS_HMAC_ARR_SZ); + err = -EINVAL; + goto out_free_hmac; + } + + err = crypto_shash_setkey(c->hmac_tfm, ukp->data, ukp->datalen); + if (err) + goto out_free_hmac; + + c->authenticated = true; + + c->log_hash = ubifs_hash_get_desc(c); + if (IS_ERR(c->log_hash)) { + err = PTR_ERR(c->log_hash); + goto out_free_hmac; + } + + err = 0; + +out_free_hmac: + if (err) + crypto_free_shash(c->hmac_tfm); +out_free_hash: + if (err) + crypto_free_shash(c->hash_tfm); +out: + up_read(&keyring_key->sem); + key_put(keyring_key); + + return err; +} + +/** + * __ubifs_exit_authentication - release resource + * @c: UBIFS file-system description object + * + * This function releases the authentication related resources. + */ +void __ubifs_exit_authentication(struct ubifs_info *c) +{ + if (!ubifs_authenticated(c)) + return; + + crypto_free_shash(c->hmac_tfm); + crypto_free_shash(c->hash_tfm); + kfree(c->log_hash); +} + +/** + * ubifs_node_calc_hmac - calculate the HMAC of a UBIFS node + * @c: UBIFS file-system description object + * @node: the node to insert a HMAC into. + * @len: the length of the node + * @ofs_hmac: the offset in the node where the HMAC is inserted + * @hmac: returned HMAC + * + * This function calculates a HMAC of a UBIFS node. The HMAC is expected to be + * embedded into the node, so this area is not covered by the HMAC. Also not + * covered is the UBIFS_NODE_MAGIC and the CRC of the node. + */ +static int ubifs_node_calc_hmac(const struct ubifs_info *c, const void *node, + int len, int ofs_hmac, void *hmac) +{ + SHASH_DESC_ON_STACK(shash, c->hmac_tfm); + int hmac_len = c->hmac_desc_len; + int err; + + ubifs_assert(c, ofs_hmac > 8); + ubifs_assert(c, ofs_hmac + hmac_len < len); + + shash->tfm = c->hmac_tfm; + + err = crypto_shash_init(shash); + if (err) + return err; + + /* behind common node header CRC up to HMAC begin */ + err = crypto_shash_update(shash, node + 8, ofs_hmac - 8); + if (err < 0) + return err; + + /* behind HMAC, if any */ + if (len - ofs_hmac - hmac_len > 0) { + err = crypto_shash_update(shash, node + ofs_hmac + hmac_len, + len - ofs_hmac - hmac_len); + if (err < 0) + return err; + } + + return crypto_shash_final(shash, hmac); +} + +/** + * __ubifs_node_insert_hmac - insert a HMAC into a UBIFS node + * @c: UBIFS file-system description object + * @node: the node to insert a HMAC into. + * @len: the length of the node + * @ofs_hmac: the offset in the node where the HMAC is inserted + * + * This function inserts a HMAC at offset @ofs_hmac into the node given in + * @node. + * + * This function returns 0 for success or a negative error code otherwise. + */ +int __ubifs_node_insert_hmac(const struct ubifs_info *c, void *node, int len, + int ofs_hmac) +{ + return ubifs_node_calc_hmac(c, node, len, ofs_hmac, node + ofs_hmac); +} + +/** + * __ubifs_node_verify_hmac - verify the HMAC of UBIFS node + * @c: UBIFS file-system description object + * @node: the node to insert a HMAC into. + * @len: the length of the node + * @ofs_hmac: the offset in the node where the HMAC is inserted + * + * This function verifies the HMAC at offset @ofs_hmac of the node given in + * @node. Returns 0 if successful or a negative error code otherwise. + */ +int __ubifs_node_verify_hmac(const struct ubifs_info *c, const void *node, + int len, int ofs_hmac) +{ + int hmac_len = c->hmac_desc_len; + u8 *hmac; + int err; + + hmac = kmalloc(hmac_len, GFP_NOFS); + if (!hmac) + return -ENOMEM; + + err = ubifs_node_calc_hmac(c, node, len, ofs_hmac, hmac); + if (err) { + kfree(hmac); + return err; + } + + err = crypto_memneq(hmac, node + ofs_hmac, hmac_len); + + kfree(hmac); + + if (!err) + return 0; + + return -EPERM; +} + +int __ubifs_shash_copy_state(const struct ubifs_info *c, struct shash_desc *src, + struct shash_desc *target) +{ + u8 *state; + int err; + + state = kmalloc(crypto_shash_descsize(src->tfm), GFP_NOFS); + if (!state) + return -ENOMEM; + + err = crypto_shash_export(src, state); + if (err) + goto out; + + err = crypto_shash_import(target, state); + +out: + kfree(state); + + return err; +} + +/** + * ubifs_hmac_wkm - Create a HMAC of the well known message + * @c: UBIFS file-system description object + * @hmac: The HMAC of the well known message + * + * This function creates a HMAC of a well known message. This is used + * to check if the provided key is suitable to authenticate a UBIFS + * image. This is only a convenience to the user to provide a better + * error message when the wrong key is provided. + * + * This function returns 0 for success or a negative error code otherwise. + */ +int ubifs_hmac_wkm(struct ubifs_info *c, u8 *hmac) +{ + const char well_known_message[] = "UBIFS"; + + if (!ubifs_authenticated(c)) + return 0; + + return crypto_shash_tfm_digest(c->hmac_tfm, well_known_message, + sizeof(well_known_message) - 1, hmac); +} + +/* + * ubifs_hmac_zero - test if a HMAC is zero + * @c: UBIFS file-system description object + * @hmac: the HMAC to test + * + * This function tests if a HMAC is zero and returns true if it is + * and false otherwise. + */ +bool ubifs_hmac_zero(struct ubifs_info *c, const u8 *hmac) +{ + return !memchr_inv(hmac, 0, c->hmac_desc_len); +} diff --git a/fs/ubifs/budget.c b/fs/ubifs/budget.c new file mode 100644 index 000000000..d76eb7b39 --- /dev/null +++ b/fs/ubifs/budget.c @@ -0,0 +1,714 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * + * Authors: Adrian Hunter + * Artem Bityutskiy (Битюцкий Артём) + */ + +/* + * This file implements the budgeting sub-system which is responsible for UBIFS + * space management. + * + * Factors such as compression, wasted space at the ends of LEBs, space in other + * journal heads, the effect of updates on the index, and so on, make it + * impossible to accurately predict the amount of space needed. Consequently + * approximations are used. + */ + +#include "ubifs.h" +#include +#include + +/* + * When pessimistic budget calculations say that there is no enough space, + * UBIFS starts writing back dirty inodes and pages, doing garbage collection, + * or committing. The below constant defines maximum number of times UBIFS + * repeats the operations. + */ +#define MAX_MKSPC_RETRIES 3 + +/* + * The below constant defines amount of dirty pages which should be written + * back at when trying to shrink the liability. + */ +#define NR_TO_WRITE 16 + +/** + * shrink_liability - write-back some dirty pages/inodes. + * @c: UBIFS file-system description object + * @nr_to_write: how many dirty pages to write-back + * + * This function shrinks UBIFS liability by means of writing back some amount + * of dirty inodes and their pages. + * + * Note, this function synchronizes even VFS inodes which are locked + * (@i_mutex) by the caller of the budgeting function, because write-back does + * not touch @i_mutex. + */ +static void shrink_liability(struct ubifs_info *c, int nr_to_write) +{ + down_read(&c->vfs_sb->s_umount); + writeback_inodes_sb_nr(c->vfs_sb, nr_to_write, WB_REASON_FS_FREE_SPACE); + up_read(&c->vfs_sb->s_umount); +} + +/** + * run_gc - run garbage collector. + * @c: UBIFS file-system description object + * + * This function runs garbage collector to make some more free space. Returns + * zero if a free LEB has been produced, %-EAGAIN if commit is required, and a + * negative error code in case of failure. + */ +static int run_gc(struct ubifs_info *c) +{ + int lnum; + + /* Make some free space by garbage-collecting dirty space */ + down_read(&c->commit_sem); + lnum = ubifs_garbage_collect(c, 1); + up_read(&c->commit_sem); + if (lnum < 0) + return lnum; + + /* GC freed one LEB, return it to lprops */ + dbg_budg("GC freed LEB %d", lnum); + return ubifs_return_leb(c, lnum); +} + +/** + * get_liability - calculate current liability. + * @c: UBIFS file-system description object + * + * This function calculates and returns current UBIFS liability, i.e. the + * amount of bytes UBIFS has "promised" to write to the media. + */ +static long long get_liability(struct ubifs_info *c) +{ + long long liab; + + spin_lock(&c->space_lock); + liab = c->bi.idx_growth + c->bi.data_growth + c->bi.dd_growth; + spin_unlock(&c->space_lock); + return liab; +} + +/** + * make_free_space - make more free space on the file-system. + * @c: UBIFS file-system description object + * + * This function is called when an operation cannot be budgeted because there + * is supposedly no free space. But in most cases there is some free space: + * o budgeting is pessimistic, so it always budgets more than it is actually + * needed, so shrinking the liability is one way to make free space - the + * cached data will take less space then it was budgeted for; + * o GC may turn some dark space into free space (budgeting treats dark space + * as not available); + * o commit may free some LEB, i.e., turn freeable LEBs into free LEBs. + * + * So this function tries to do the above. Returns %-EAGAIN if some free space + * was presumably made and the caller has to re-try budgeting the operation. + * Returns %-ENOSPC if it couldn't do more free space, and other negative error + * codes on failures. + */ +static int make_free_space(struct ubifs_info *c) +{ + int err, retries = 0; + long long liab1, liab2; + + do { + liab1 = get_liability(c); + /* + * We probably have some dirty pages or inodes (liability), try + * to write them back. + */ + dbg_budg("liability %lld, run write-back", liab1); + shrink_liability(c, NR_TO_WRITE); + + liab2 = get_liability(c); + if (liab2 < liab1) + return -EAGAIN; + + dbg_budg("new liability %lld (not shrunk)", liab2); + + /* Liability did not shrink again, try GC */ + dbg_budg("Run GC"); + err = run_gc(c); + if (!err) + return -EAGAIN; + + if (err != -EAGAIN && err != -ENOSPC) + /* Some real error happened */ + return err; + + dbg_budg("Run commit (retries %d)", retries); + err = ubifs_run_commit(c); + if (err) + return err; + } while (retries++ < MAX_MKSPC_RETRIES); + + return -ENOSPC; +} + +/** + * ubifs_calc_min_idx_lebs - calculate amount of LEBs for the index. + * @c: UBIFS file-system description object + * + * This function calculates and returns the number of LEBs which should be kept + * for index usage. + */ +int ubifs_calc_min_idx_lebs(struct ubifs_info *c) +{ + int idx_lebs; + long long idx_size; + + idx_size = c->bi.old_idx_sz + c->bi.idx_growth + c->bi.uncommitted_idx; + /* And make sure we have thrice the index size of space reserved */ + idx_size += idx_size << 1; + /* + * We do not maintain 'old_idx_size' as 'old_idx_lebs'/'old_idx_bytes' + * pair, nor similarly the two variables for the new index size, so we + * have to do this costly 64-bit division on fast-path. + */ + idx_lebs = div_u64(idx_size + c->idx_leb_size - 1, c->idx_leb_size); + /* + * The index head is not available for the in-the-gaps method, so add an + * extra LEB to compensate. + */ + idx_lebs += 1; + if (idx_lebs < MIN_INDEX_LEBS) + idx_lebs = MIN_INDEX_LEBS; + return idx_lebs; +} + +/** + * ubifs_calc_available - calculate available FS space. + * @c: UBIFS file-system description object + * @min_idx_lebs: minimum number of LEBs reserved for the index + * + * This function calculates and returns amount of FS space available for use. + */ +long long ubifs_calc_available(const struct ubifs_info *c, int min_idx_lebs) +{ + int subtract_lebs; + long long available; + + available = c->main_bytes - c->lst.total_used; + + /* + * Now 'available' contains theoretically available flash space + * assuming there is no index, so we have to subtract the space which + * is reserved for the index. + */ + subtract_lebs = min_idx_lebs; + + /* Take into account that GC reserves one LEB for its own needs */ + subtract_lebs += 1; + + /* + * Since different write types go to different heads, we should + * reserve one leb for each head. + */ + subtract_lebs += c->jhead_cnt; + + /* We also reserve one LEB for deletions, which bypass budgeting */ + subtract_lebs += 1; + + available -= (long long)subtract_lebs * c->leb_size; + + /* Subtract the dead space which is not available for use */ + available -= c->lst.total_dead; + + /* + * Subtract dark space, which might or might not be usable - it depends + * on the data which we have on the media and which will be written. If + * this is a lot of uncompressed or not-compressible data, the dark + * space cannot be used. + */ + available -= c->lst.total_dark; + + /* + * However, there is more dark space. The index may be bigger than + * @min_idx_lebs. Those extra LEBs are assumed to be available, but + * their dark space is not included in total_dark, so it is subtracted + * here. + */ + if (c->lst.idx_lebs > min_idx_lebs) { + subtract_lebs = c->lst.idx_lebs - min_idx_lebs; + available -= subtract_lebs * c->dark_wm; + } + + /* The calculations are rough and may end up with a negative number */ + return available > 0 ? available : 0; +} + +/** + * can_use_rp - check whether the user is allowed to use reserved pool. + * @c: UBIFS file-system description object + * + * UBIFS has so-called "reserved pool" which is flash space reserved + * for the superuser and for uses whose UID/GID is recorded in UBIFS superblock. + * This function checks whether current user is allowed to use reserved pool. + * Returns %1 current user is allowed to use reserved pool and %0 otherwise. + */ +static int can_use_rp(struct ubifs_info *c) +{ + if (uid_eq(current_fsuid(), c->rp_uid) || capable(CAP_SYS_RESOURCE) || + (!gid_eq(c->rp_gid, GLOBAL_ROOT_GID) && in_group_p(c->rp_gid))) + return 1; + return 0; +} + +/** + * do_budget_space - reserve flash space for index and data growth. + * @c: UBIFS file-system description object + * + * This function makes sure UBIFS has enough free LEBs for index growth and + * data. + * + * When budgeting index space, UBIFS reserves thrice as many LEBs as the index + * would take if it was consolidated and written to the flash. This guarantees + * that the "in-the-gaps" commit method always succeeds and UBIFS will always + * be able to commit dirty index. So this function basically adds amount of + * budgeted index space to the size of the current index, multiplies this by 3, + * and makes sure this does not exceed the amount of free LEBs. + * + * Notes about @c->bi.min_idx_lebs and @c->lst.idx_lebs variables: + * o @c->lst.idx_lebs is the number of LEBs the index currently uses. It might + * be large, because UBIFS does not do any index consolidation as long as + * there is free space. IOW, the index may take a lot of LEBs, but the LEBs + * will contain a lot of dirt. + * o @c->bi.min_idx_lebs is the number of LEBS the index presumably takes. IOW, + * the index may be consolidated to take up to @c->bi.min_idx_lebs LEBs. + * + * This function returns zero in case of success, and %-ENOSPC in case of + * failure. + */ +static int do_budget_space(struct ubifs_info *c) +{ + long long outstanding, available; + int lebs, rsvd_idx_lebs, min_idx_lebs; + + /* First budget index space */ + min_idx_lebs = ubifs_calc_min_idx_lebs(c); + + /* Now 'min_idx_lebs' contains number of LEBs to reserve */ + if (min_idx_lebs > c->lst.idx_lebs) + rsvd_idx_lebs = min_idx_lebs - c->lst.idx_lebs; + else + rsvd_idx_lebs = 0; + + /* + * The number of LEBs that are available to be used by the index is: + * + * @c->lst.empty_lebs + @c->freeable_cnt + @c->idx_gc_cnt - + * @c->lst.taken_empty_lebs + * + * @c->lst.empty_lebs are available because they are empty. + * @c->freeable_cnt are available because they contain only free and + * dirty space, @c->idx_gc_cnt are available because they are index + * LEBs that have been garbage collected and are awaiting the commit + * before they can be used. And the in-the-gaps method will grab these + * if it needs them. @c->lst.taken_empty_lebs are empty LEBs that have + * already been allocated for some purpose. + * + * Note, @c->idx_gc_cnt is included to both @c->lst.empty_lebs (because + * these LEBs are empty) and to @c->lst.taken_empty_lebs (because they + * are taken until after the commit). + * + * Note, @c->lst.taken_empty_lebs may temporarily be higher by one + * because of the way we serialize LEB allocations and budgeting. See a + * comment in 'ubifs_find_free_space()'. + */ + lebs = c->lst.empty_lebs + c->freeable_cnt + c->idx_gc_cnt - + c->lst.taken_empty_lebs; + if (unlikely(rsvd_idx_lebs > lebs)) { + dbg_budg("out of indexing space: min_idx_lebs %d (old %d), rsvd_idx_lebs %d", + min_idx_lebs, c->bi.min_idx_lebs, rsvd_idx_lebs); + return -ENOSPC; + } + + available = ubifs_calc_available(c, min_idx_lebs); + outstanding = c->bi.data_growth + c->bi.dd_growth; + + if (unlikely(available < outstanding)) { + dbg_budg("out of data space: available %lld, outstanding %lld", + available, outstanding); + return -ENOSPC; + } + + if (available - outstanding <= c->rp_size && !can_use_rp(c)) + return -ENOSPC; + + c->bi.min_idx_lebs = min_idx_lebs; + return 0; +} + +/** + * calc_idx_growth - calculate approximate index growth from budgeting request. + * @c: UBIFS file-system description object + * @req: budgeting request + * + * For now we assume each new node adds one znode. But this is rather poor + * approximation, though. + */ +static int calc_idx_growth(const struct ubifs_info *c, + const struct ubifs_budget_req *req) +{ + int znodes; + + znodes = req->new_ino + (req->new_page << UBIFS_BLOCKS_PER_PAGE_SHIFT) + + req->new_dent; + return znodes * c->max_idx_node_sz; +} + +/** + * calc_data_growth - calculate approximate amount of new data from budgeting + * request. + * @c: UBIFS file-system description object + * @req: budgeting request + */ +static int calc_data_growth(const struct ubifs_info *c, + const struct ubifs_budget_req *req) +{ + int data_growth; + + data_growth = req->new_ino ? c->bi.inode_budget : 0; + if (req->new_page) + data_growth += c->bi.page_budget; + if (req->new_dent) + data_growth += c->bi.dent_budget; + data_growth += req->new_ino_d; + return data_growth; +} + +/** + * calc_dd_growth - calculate approximate amount of data which makes other data + * dirty from budgeting request. + * @c: UBIFS file-system description object + * @req: budgeting request + */ +static int calc_dd_growth(const struct ubifs_info *c, + const struct ubifs_budget_req *req) +{ + int dd_growth; + + dd_growth = req->dirtied_page ? c->bi.page_budget : 0; + + if (req->dirtied_ino) + dd_growth += c->bi.inode_budget * req->dirtied_ino; + if (req->mod_dent) + dd_growth += c->bi.dent_budget; + dd_growth += req->dirtied_ino_d; + return dd_growth; +} + +/** + * ubifs_budget_space - ensure there is enough space to complete an operation. + * @c: UBIFS file-system description object + * @req: budget request + * + * This function allocates budget for an operation. It uses pessimistic + * approximation of how much flash space the operation needs. The goal of this + * function is to make sure UBIFS always has flash space to flush all dirty + * pages, dirty inodes, and dirty znodes (liability). This function may force + * commit, garbage-collection or write-back. Returns zero in case of success, + * %-ENOSPC if there is no free space and other negative error codes in case of + * failures. + */ +int ubifs_budget_space(struct ubifs_info *c, struct ubifs_budget_req *req) +{ + int err, idx_growth, data_growth, dd_growth, retried = 0; + + ubifs_assert(c, req->new_page <= 1); + ubifs_assert(c, req->dirtied_page <= 1); + ubifs_assert(c, req->new_dent <= 1); + ubifs_assert(c, req->mod_dent <= 1); + ubifs_assert(c, req->new_ino <= 1); + ubifs_assert(c, req->new_ino_d <= UBIFS_MAX_INO_DATA); + ubifs_assert(c, req->dirtied_ino <= 4); + ubifs_assert(c, req->dirtied_ino_d <= UBIFS_MAX_INO_DATA * 4); + ubifs_assert(c, !(req->new_ino_d & 7)); + ubifs_assert(c, !(req->dirtied_ino_d & 7)); + + data_growth = calc_data_growth(c, req); + dd_growth = calc_dd_growth(c, req); + if (!data_growth && !dd_growth) + return 0; + idx_growth = calc_idx_growth(c, req); + +again: + spin_lock(&c->space_lock); + ubifs_assert(c, c->bi.idx_growth >= 0); + ubifs_assert(c, c->bi.data_growth >= 0); + ubifs_assert(c, c->bi.dd_growth >= 0); + + if (unlikely(c->bi.nospace) && (c->bi.nospace_rp || !can_use_rp(c))) { + dbg_budg("no space"); + spin_unlock(&c->space_lock); + return -ENOSPC; + } + + c->bi.idx_growth += idx_growth; + c->bi.data_growth += data_growth; + c->bi.dd_growth += dd_growth; + + err = do_budget_space(c); + if (likely(!err)) { + req->idx_growth = idx_growth; + req->data_growth = data_growth; + req->dd_growth = dd_growth; + spin_unlock(&c->space_lock); + return 0; + } + + /* Restore the old values */ + c->bi.idx_growth -= idx_growth; + c->bi.data_growth -= data_growth; + c->bi.dd_growth -= dd_growth; + spin_unlock(&c->space_lock); + + if (req->fast) { + dbg_budg("no space for fast budgeting"); + return err; + } + + err = make_free_space(c); + cond_resched(); + if (err == -EAGAIN) { + dbg_budg("try again"); + goto again; + } else if (err == -ENOSPC) { + if (!retried) { + retried = 1; + dbg_budg("-ENOSPC, but anyway try once again"); + goto again; + } + dbg_budg("FS is full, -ENOSPC"); + c->bi.nospace = 1; + if (can_use_rp(c) || c->rp_size == 0) + c->bi.nospace_rp = 1; + smp_wmb(); + } else + ubifs_err(c, "cannot budget space, error %d", err); + return err; +} + +/** + * ubifs_release_budget - release budgeted free space. + * @c: UBIFS file-system description object + * @req: budget request + * + * This function releases the space budgeted by 'ubifs_budget_space()'. Note, + * since the index changes (which were budgeted for in @req->idx_growth) will + * only be written to the media on commit, this function moves the index budget + * from @c->bi.idx_growth to @c->bi.uncommitted_idx. The latter will be zeroed + * by the commit operation. + */ +void ubifs_release_budget(struct ubifs_info *c, struct ubifs_budget_req *req) +{ + ubifs_assert(c, req->new_page <= 1); + ubifs_assert(c, req->dirtied_page <= 1); + ubifs_assert(c, req->new_dent <= 1); + ubifs_assert(c, req->mod_dent <= 1); + ubifs_assert(c, req->new_ino <= 1); + ubifs_assert(c, req->new_ino_d <= UBIFS_MAX_INO_DATA); + ubifs_assert(c, req->dirtied_ino <= 4); + ubifs_assert(c, req->dirtied_ino_d <= UBIFS_MAX_INO_DATA * 4); + ubifs_assert(c, !(req->new_ino_d & 7)); + ubifs_assert(c, !(req->dirtied_ino_d & 7)); + if (!req->recalculate) { + ubifs_assert(c, req->idx_growth >= 0); + ubifs_assert(c, req->data_growth >= 0); + ubifs_assert(c, req->dd_growth >= 0); + } + + if (req->recalculate) { + req->data_growth = calc_data_growth(c, req); + req->dd_growth = calc_dd_growth(c, req); + req->idx_growth = calc_idx_growth(c, req); + } + + if (!req->data_growth && !req->dd_growth) + return; + + c->bi.nospace = c->bi.nospace_rp = 0; + smp_wmb(); + + spin_lock(&c->space_lock); + c->bi.idx_growth -= req->idx_growth; + c->bi.uncommitted_idx += req->idx_growth; + c->bi.data_growth -= req->data_growth; + c->bi.dd_growth -= req->dd_growth; + c->bi.min_idx_lebs = ubifs_calc_min_idx_lebs(c); + + ubifs_assert(c, c->bi.idx_growth >= 0); + ubifs_assert(c, c->bi.data_growth >= 0); + ubifs_assert(c, c->bi.dd_growth >= 0); + ubifs_assert(c, c->bi.min_idx_lebs < c->main_lebs); + ubifs_assert(c, !(c->bi.idx_growth & 7)); + ubifs_assert(c, !(c->bi.data_growth & 7)); + ubifs_assert(c, !(c->bi.dd_growth & 7)); + spin_unlock(&c->space_lock); +} + +/** + * ubifs_convert_page_budget - convert budget of a new page. + * @c: UBIFS file-system description object + * + * This function converts budget which was allocated for a new page of data to + * the budget of changing an existing page of data. The latter is smaller than + * the former, so this function only does simple re-calculation and does not + * involve any write-back. + */ +void ubifs_convert_page_budget(struct ubifs_info *c) +{ + spin_lock(&c->space_lock); + /* Release the index growth reservation */ + c->bi.idx_growth -= c->max_idx_node_sz << UBIFS_BLOCKS_PER_PAGE_SHIFT; + /* Release the data growth reservation */ + c->bi.data_growth -= c->bi.page_budget; + /* Increase the dirty data growth reservation instead */ + c->bi.dd_growth += c->bi.page_budget; + /* And re-calculate the indexing space reservation */ + c->bi.min_idx_lebs = ubifs_calc_min_idx_lebs(c); + spin_unlock(&c->space_lock); +} + +/** + * ubifs_release_dirty_inode_budget - release dirty inode budget. + * @c: UBIFS file-system description object + * @ui: UBIFS inode to release the budget for + * + * This function releases budget corresponding to a dirty inode. It is usually + * called when after the inode has been written to the media and marked as + * clean. It also causes the "no space" flags to be cleared. + */ +void ubifs_release_dirty_inode_budget(struct ubifs_info *c, + struct ubifs_inode *ui) +{ + struct ubifs_budget_req req; + + memset(&req, 0, sizeof(struct ubifs_budget_req)); + /* The "no space" flags will be cleared because dd_growth is > 0 */ + req.dd_growth = c->bi.inode_budget + ALIGN(ui->data_len, 8); + ubifs_release_budget(c, &req); +} + +/** + * ubifs_reported_space - calculate reported free space. + * @c: the UBIFS file-system description object + * @free: amount of free space + * + * This function calculates amount of free space which will be reported to + * user-space. User-space application tend to expect that if the file-system + * (e.g., via the 'statfs()' call) reports that it has N bytes available, they + * are able to write a file of size N. UBIFS attaches node headers to each data + * node and it has to write indexing nodes as well. This introduces additional + * overhead, and UBIFS has to report slightly less free space to meet the above + * expectations. + * + * This function assumes free space is made up of uncompressed data nodes and + * full index nodes (one per data node, tripled because we always allow enough + * space to write the index thrice). + * + * Note, the calculation is pessimistic, which means that most of the time + * UBIFS reports less space than it actually has. + */ +long long ubifs_reported_space(const struct ubifs_info *c, long long free) +{ + int divisor, factor, f; + + /* + * Reported space size is @free * X, where X is UBIFS block size + * divided by UBIFS block size + all overhead one data block + * introduces. The overhead is the node header + indexing overhead. + * + * Indexing overhead calculations are based on the following formula: + * I = N/(f - 1) + 1, where I - number of indexing nodes, N - number + * of data nodes, f - fanout. Because effective UBIFS fanout is twice + * as less than maximum fanout, we assume that each data node + * introduces 3 * @c->max_idx_node_sz / (@c->fanout/2 - 1) bytes. + * Note, the multiplier 3 is because UBIFS reserves thrice as more space + * for the index. + */ + f = c->fanout > 3 ? c->fanout >> 1 : 2; + factor = UBIFS_BLOCK_SIZE; + divisor = UBIFS_MAX_DATA_NODE_SZ; + divisor += (c->max_idx_node_sz * 3) / (f - 1); + free *= factor; + return div_u64(free, divisor); +} + +/** + * ubifs_get_free_space_nolock - return amount of free space. + * @c: UBIFS file-system description object + * + * This function calculates amount of free space to report to user-space. + * + * Because UBIFS may introduce substantial overhead (the index, node headers, + * alignment, wastage at the end of LEBs, etc), it cannot report real amount of + * free flash space it has (well, because not all dirty space is reclaimable, + * UBIFS does not actually know the real amount). If UBIFS did so, it would + * bread user expectations about what free space is. Users seem to accustomed + * to assume that if the file-system reports N bytes of free space, they would + * be able to fit a file of N bytes to the FS. This almost works for + * traditional file-systems, because they have way less overhead than UBIFS. + * So, to keep users happy, UBIFS tries to take the overhead into account. + */ +long long ubifs_get_free_space_nolock(struct ubifs_info *c) +{ + int rsvd_idx_lebs, lebs; + long long available, outstanding, free; + + ubifs_assert(c, c->bi.min_idx_lebs == ubifs_calc_min_idx_lebs(c)); + outstanding = c->bi.data_growth + c->bi.dd_growth; + available = ubifs_calc_available(c, c->bi.min_idx_lebs); + + /* + * When reporting free space to user-space, UBIFS guarantees that it is + * possible to write a file of free space size. This means that for + * empty LEBs we may use more precise calculations than + * 'ubifs_calc_available()' is using. Namely, we know that in empty + * LEBs we would waste only @c->leb_overhead bytes, not @c->dark_wm. + * Thus, amend the available space. + * + * Note, the calculations below are similar to what we have in + * 'do_budget_space()', so refer there for comments. + */ + if (c->bi.min_idx_lebs > c->lst.idx_lebs) + rsvd_idx_lebs = c->bi.min_idx_lebs - c->lst.idx_lebs; + else + rsvd_idx_lebs = 0; + lebs = c->lst.empty_lebs + c->freeable_cnt + c->idx_gc_cnt - + c->lst.taken_empty_lebs; + lebs -= rsvd_idx_lebs; + available += lebs * (c->dark_wm - c->leb_overhead); + + if (available > outstanding) + free = ubifs_reported_space(c, available - outstanding); + else + free = 0; + return free; +} + +/** + * ubifs_get_free_space - return amount of free space. + * @c: UBIFS file-system description object + * + * This function calculates and returns amount of free space to report to + * user-space. + */ +long long ubifs_get_free_space(struct ubifs_info *c) +{ + long long free; + + spin_lock(&c->space_lock); + free = ubifs_get_free_space_nolock(c); + spin_unlock(&c->space_lock); + + return free; +} diff --git a/fs/ubifs/commit.c b/fs/ubifs/commit.c new file mode 100644 index 000000000..5b3a84009 --- /dev/null +++ b/fs/ubifs/commit.c @@ -0,0 +1,733 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * + * Authors: Adrian Hunter + * Artem Bityutskiy (Битюцкий Артём) + */ + +/* + * This file implements functions that manage the running of the commit process. + * Each affected module has its own functions to accomplish their part in the + * commit and those functions are called here. + * + * The commit is the process whereby all updates to the index and LEB properties + * are written out together and the journal becomes empty. This keeps the + * file system consistent - at all times the state can be recreated by reading + * the index and LEB properties and then replaying the journal. + * + * The commit is split into two parts named "commit start" and "commit end". + * During commit start, the commit process has exclusive access to the journal + * by holding the commit semaphore down for writing. As few I/O operations as + * possible are performed during commit start, instead the nodes that are to be + * written are merely identified. During commit end, the commit semaphore is no + * longer held and the journal is again in operation, allowing users to continue + * to use the file system while the bulk of the commit I/O is performed. The + * purpose of this two-step approach is to prevent the commit from causing any + * latency blips. Note that in any case, the commit does not prevent lookups + * (as permitted by the TNC mutex), or access to VFS data structures e.g. page + * cache. + */ + +#include +#include +#include +#include "ubifs.h" + +/* + * nothing_to_commit - check if there is nothing to commit. + * @c: UBIFS file-system description object + * + * This is a helper function which checks if there is anything to commit. It is + * used as an optimization to avoid starting the commit if it is not really + * necessary. Indeed, the commit operation always assumes flash I/O (e.g., + * writing the commit start node to the log), and it is better to avoid doing + * this unnecessarily. E.g., 'ubifs_sync_fs()' runs the commit, but if there is + * nothing to commit, it is more optimal to avoid any flash I/O. + * + * This function has to be called with @c->commit_sem locked for writing - + * this function does not take LPT/TNC locks because the @c->commit_sem + * guarantees that we have exclusive access to the TNC and LPT data structures. + * + * This function returns %1 if there is nothing to commit and %0 otherwise. + */ +static int nothing_to_commit(struct ubifs_info *c) +{ + /* + * During mounting or remounting from R/O mode to R/W mode we may + * commit for various recovery-related reasons. + */ + if (c->mounting || c->remounting_rw) + return 0; + + /* + * If the root TNC node is dirty, we definitely have something to + * commit. + */ + if (c->zroot.znode && ubifs_zn_dirty(c->zroot.znode)) + return 0; + + /* + * Increasing @c->dirty_pn_cnt/@c->dirty_nn_cnt and marking + * nnodes/pnodes as dirty in run_gc() could race with following + * checking, which leads inconsistent states between @c->nroot + * and @c->dirty_pn_cnt/@c->dirty_nn_cnt, holding @c->lp_mutex + * to avoid that. + */ + mutex_lock(&c->lp_mutex); + /* + * Even though the TNC is clean, the LPT tree may have dirty nodes. For + * example, this may happen if the budgeting subsystem invoked GC to + * make some free space, and the GC found an LEB with only dirty and + * free space. In this case GC would just change the lprops of this + * LEB (by turning all space into free space) and unmap it. + */ + if (c->nroot && test_bit(DIRTY_CNODE, &c->nroot->flags)) { + mutex_unlock(&c->lp_mutex); + return 0; + } + + ubifs_assert(c, atomic_long_read(&c->dirty_zn_cnt) == 0); + ubifs_assert(c, c->dirty_pn_cnt == 0); + ubifs_assert(c, c->dirty_nn_cnt == 0); + mutex_unlock(&c->lp_mutex); + + return 1; +} + +/** + * do_commit - commit the journal. + * @c: UBIFS file-system description object + * + * This function implements UBIFS commit. It has to be called with commit lock + * locked. Returns zero in case of success and a negative error code in case of + * failure. + */ +static int do_commit(struct ubifs_info *c) +{ + int err, new_ltail_lnum, old_ltail_lnum, i; + struct ubifs_zbranch zroot; + struct ubifs_lp_stats lst; + + dbg_cmt("start"); + ubifs_assert(c, !c->ro_media && !c->ro_mount); + + if (c->ro_error) { + err = -EROFS; + goto out_up; + } + + if (nothing_to_commit(c)) { + up_write(&c->commit_sem); + err = 0; + goto out_cancel; + } + + /* Sync all write buffers (necessary for recovery) */ + for (i = 0; i < c->jhead_cnt; i++) { + err = ubifs_wbuf_sync(&c->jheads[i].wbuf); + if (err) + goto out_up; + } + + c->cmt_no += 1; + err = ubifs_gc_start_commit(c); + if (err) + goto out_up; + err = dbg_check_lprops(c); + if (err) + goto out_up; + err = ubifs_log_start_commit(c, &new_ltail_lnum); + if (err) + goto out_up; + err = ubifs_tnc_start_commit(c, &zroot); + if (err) + goto out_up; + err = ubifs_lpt_start_commit(c); + if (err) + goto out_up; + err = ubifs_orphan_start_commit(c); + if (err) + goto out_up; + + ubifs_get_lp_stats(c, &lst); + + up_write(&c->commit_sem); + + err = ubifs_tnc_end_commit(c); + if (err) + goto out; + err = ubifs_lpt_end_commit(c); + if (err) + goto out; + err = ubifs_orphan_end_commit(c); + if (err) + goto out; + err = dbg_check_old_index(c, &zroot); + if (err) + goto out; + + c->mst_node->cmt_no = cpu_to_le64(c->cmt_no); + c->mst_node->log_lnum = cpu_to_le32(new_ltail_lnum); + c->mst_node->root_lnum = cpu_to_le32(zroot.lnum); + c->mst_node->root_offs = cpu_to_le32(zroot.offs); + c->mst_node->root_len = cpu_to_le32(zroot.len); + c->mst_node->ihead_lnum = cpu_to_le32(c->ihead_lnum); + c->mst_node->ihead_offs = cpu_to_le32(c->ihead_offs); + c->mst_node->index_size = cpu_to_le64(c->bi.old_idx_sz); + c->mst_node->lpt_lnum = cpu_to_le32(c->lpt_lnum); + c->mst_node->lpt_offs = cpu_to_le32(c->lpt_offs); + c->mst_node->nhead_lnum = cpu_to_le32(c->nhead_lnum); + c->mst_node->nhead_offs = cpu_to_le32(c->nhead_offs); + c->mst_node->ltab_lnum = cpu_to_le32(c->ltab_lnum); + c->mst_node->ltab_offs = cpu_to_le32(c->ltab_offs); + c->mst_node->lsave_lnum = cpu_to_le32(c->lsave_lnum); + c->mst_node->lsave_offs = cpu_to_le32(c->lsave_offs); + c->mst_node->lscan_lnum = cpu_to_le32(c->lscan_lnum); + c->mst_node->empty_lebs = cpu_to_le32(lst.empty_lebs); + c->mst_node->idx_lebs = cpu_to_le32(lst.idx_lebs); + c->mst_node->total_free = cpu_to_le64(lst.total_free); + c->mst_node->total_dirty = cpu_to_le64(lst.total_dirty); + c->mst_node->total_used = cpu_to_le64(lst.total_used); + c->mst_node->total_dead = cpu_to_le64(lst.total_dead); + c->mst_node->total_dark = cpu_to_le64(lst.total_dark); + if (c->no_orphs) + c->mst_node->flags |= cpu_to_le32(UBIFS_MST_NO_ORPHS); + else + c->mst_node->flags &= ~cpu_to_le32(UBIFS_MST_NO_ORPHS); + + old_ltail_lnum = c->ltail_lnum; + err = ubifs_log_end_commit(c, new_ltail_lnum); + if (err) + goto out; + + err = ubifs_log_post_commit(c, old_ltail_lnum); + if (err) + goto out; + err = ubifs_gc_end_commit(c); + if (err) + goto out; + err = ubifs_lpt_post_commit(c); + if (err) + goto out; + +out_cancel: + spin_lock(&c->cs_lock); + c->cmt_state = COMMIT_RESTING; + wake_up(&c->cmt_wq); + dbg_cmt("commit end"); + spin_unlock(&c->cs_lock); + return 0; + +out_up: + up_write(&c->commit_sem); +out: + ubifs_err(c, "commit failed, error %d", err); + spin_lock(&c->cs_lock); + c->cmt_state = COMMIT_BROKEN; + wake_up(&c->cmt_wq); + spin_unlock(&c->cs_lock); + ubifs_ro_mode(c, err); + return err; +} + +/** + * run_bg_commit - run background commit if it is needed. + * @c: UBIFS file-system description object + * + * This function runs background commit if it is needed. Returns zero in case + * of success and a negative error code in case of failure. + */ +static int run_bg_commit(struct ubifs_info *c) +{ + spin_lock(&c->cs_lock); + /* + * Run background commit only if background commit was requested or if + * commit is required. + */ + if (c->cmt_state != COMMIT_BACKGROUND && + c->cmt_state != COMMIT_REQUIRED) + goto out; + spin_unlock(&c->cs_lock); + + down_write(&c->commit_sem); + spin_lock(&c->cs_lock); + if (c->cmt_state == COMMIT_REQUIRED) + c->cmt_state = COMMIT_RUNNING_REQUIRED; + else if (c->cmt_state == COMMIT_BACKGROUND) + c->cmt_state = COMMIT_RUNNING_BACKGROUND; + else + goto out_cmt_unlock; + spin_unlock(&c->cs_lock); + + return do_commit(c); + +out_cmt_unlock: + up_write(&c->commit_sem); +out: + spin_unlock(&c->cs_lock); + return 0; +} + +/** + * ubifs_bg_thread - UBIFS background thread function. + * @info: points to the file-system description object + * + * This function implements various file-system background activities: + * o when a write-buffer timer expires it synchronizes the appropriate + * write-buffer; + * o when the journal is about to be full, it starts in-advance commit. + * + * Note, other stuff like background garbage collection may be added here in + * future. + */ +int ubifs_bg_thread(void *info) +{ + int err; + struct ubifs_info *c = info; + + ubifs_msg(c, "background thread \"%s\" started, PID %d", + c->bgt_name, current->pid); + set_freezable(); + + while (1) { + if (kthread_should_stop()) + break; + + if (try_to_freeze()) + continue; + + set_current_state(TASK_INTERRUPTIBLE); + /* Check if there is something to do */ + if (!c->need_bgt) { + /* + * Nothing prevents us from going sleep now and + * be never woken up and block the task which + * could wait in 'kthread_stop()' forever. + */ + if (kthread_should_stop()) + break; + schedule(); + continue; + } else + __set_current_state(TASK_RUNNING); + + c->need_bgt = 0; + err = ubifs_bg_wbufs_sync(c); + if (err) + ubifs_ro_mode(c, err); + + run_bg_commit(c); + cond_resched(); + } + + ubifs_msg(c, "background thread \"%s\" stops", c->bgt_name); + return 0; +} + +/** + * ubifs_commit_required - set commit state to "required". + * @c: UBIFS file-system description object + * + * This function is called if a commit is required but cannot be done from the + * calling function, so it is just flagged instead. + */ +void ubifs_commit_required(struct ubifs_info *c) +{ + spin_lock(&c->cs_lock); + switch (c->cmt_state) { + case COMMIT_RESTING: + case COMMIT_BACKGROUND: + dbg_cmt("old: %s, new: %s", dbg_cstate(c->cmt_state), + dbg_cstate(COMMIT_REQUIRED)); + c->cmt_state = COMMIT_REQUIRED; + break; + case COMMIT_RUNNING_BACKGROUND: + dbg_cmt("old: %s, new: %s", dbg_cstate(c->cmt_state), + dbg_cstate(COMMIT_RUNNING_REQUIRED)); + c->cmt_state = COMMIT_RUNNING_REQUIRED; + break; + case COMMIT_REQUIRED: + case COMMIT_RUNNING_REQUIRED: + case COMMIT_BROKEN: + break; + } + spin_unlock(&c->cs_lock); +} + +/** + * ubifs_request_bg_commit - notify the background thread to do a commit. + * @c: UBIFS file-system description object + * + * This function is called if the journal is full enough to make a commit + * worthwhile, so background thread is kicked to start it. + */ +void ubifs_request_bg_commit(struct ubifs_info *c) +{ + spin_lock(&c->cs_lock); + if (c->cmt_state == COMMIT_RESTING) { + dbg_cmt("old: %s, new: %s", dbg_cstate(c->cmt_state), + dbg_cstate(COMMIT_BACKGROUND)); + c->cmt_state = COMMIT_BACKGROUND; + spin_unlock(&c->cs_lock); + ubifs_wake_up_bgt(c); + } else + spin_unlock(&c->cs_lock); +} + +/** + * wait_for_commit - wait for commit. + * @c: UBIFS file-system description object + * + * This function sleeps until the commit operation is no longer running. + */ +static int wait_for_commit(struct ubifs_info *c) +{ + dbg_cmt("pid %d goes sleep", current->pid); + + /* + * The following sleeps if the condition is false, and will be woken + * when the commit ends. It is possible, although very unlikely, that we + * will wake up and see the subsequent commit running, rather than the + * one we were waiting for, and go back to sleep. However, we will be + * woken again, so there is no danger of sleeping forever. + */ + wait_event(c->cmt_wq, c->cmt_state != COMMIT_RUNNING_BACKGROUND && + c->cmt_state != COMMIT_RUNNING_REQUIRED); + dbg_cmt("commit finished, pid %d woke up", current->pid); + return 0; +} + +/** + * ubifs_run_commit - run or wait for commit. + * @c: UBIFS file-system description object + * + * This function runs commit and returns zero in case of success and a negative + * error code in case of failure. + */ +int ubifs_run_commit(struct ubifs_info *c) +{ + int err = 0; + + spin_lock(&c->cs_lock); + if (c->cmt_state == COMMIT_BROKEN) { + err = -EROFS; + goto out; + } + + if (c->cmt_state == COMMIT_RUNNING_BACKGROUND) + /* + * We set the commit state to 'running required' to indicate + * that we want it to complete as quickly as possible. + */ + c->cmt_state = COMMIT_RUNNING_REQUIRED; + + if (c->cmt_state == COMMIT_RUNNING_REQUIRED) { + spin_unlock(&c->cs_lock); + return wait_for_commit(c); + } + spin_unlock(&c->cs_lock); + + /* Ok, the commit is indeed needed */ + + down_write(&c->commit_sem); + spin_lock(&c->cs_lock); + /* + * Since we unlocked 'c->cs_lock', the state may have changed, so + * re-check it. + */ + if (c->cmt_state == COMMIT_BROKEN) { + err = -EROFS; + goto out_cmt_unlock; + } + + if (c->cmt_state == COMMIT_RUNNING_BACKGROUND) + c->cmt_state = COMMIT_RUNNING_REQUIRED; + + if (c->cmt_state == COMMIT_RUNNING_REQUIRED) { + up_write(&c->commit_sem); + spin_unlock(&c->cs_lock); + return wait_for_commit(c); + } + c->cmt_state = COMMIT_RUNNING_REQUIRED; + spin_unlock(&c->cs_lock); + + err = do_commit(c); + return err; + +out_cmt_unlock: + up_write(&c->commit_sem); +out: + spin_unlock(&c->cs_lock); + return err; +} + +/** + * ubifs_gc_should_commit - determine if it is time for GC to run commit. + * @c: UBIFS file-system description object + * + * This function is called by garbage collection to determine if commit should + * be run. If commit state is @COMMIT_BACKGROUND, which means that the journal + * is full enough to start commit, this function returns true. It is not + * absolutely necessary to commit yet, but it feels like this should be better + * then to keep doing GC. This function returns %1 if GC has to initiate commit + * and %0 if not. + */ +int ubifs_gc_should_commit(struct ubifs_info *c) +{ + int ret = 0; + + spin_lock(&c->cs_lock); + if (c->cmt_state == COMMIT_BACKGROUND) { + dbg_cmt("commit required now"); + c->cmt_state = COMMIT_REQUIRED; + } else + dbg_cmt("commit not requested"); + if (c->cmt_state == COMMIT_REQUIRED) + ret = 1; + spin_unlock(&c->cs_lock); + return ret; +} + +/* + * Everything below is related to debugging. + */ + +/** + * struct idx_node - hold index nodes during index tree traversal. + * @list: list + * @iip: index in parent (slot number of this indexing node in the parent + * indexing node) + * @upper_key: all keys in this indexing node have to be less or equivalent to + * this key + * @idx: index node (8-byte aligned because all node structures must be 8-byte + * aligned) + */ +struct idx_node { + struct list_head list; + int iip; + union ubifs_key upper_key; + struct ubifs_idx_node idx __aligned(8); +}; + +/** + * dbg_old_index_check_init - get information for the next old index check. + * @c: UBIFS file-system description object + * @zroot: root of the index + * + * This function records information about the index that will be needed for the + * next old index check i.e. 'dbg_check_old_index()'. + * + * This function returns %0 on success and a negative error code on failure. + */ +int dbg_old_index_check_init(struct ubifs_info *c, struct ubifs_zbranch *zroot) +{ + struct ubifs_idx_node *idx; + int lnum, offs, len, err = 0; + struct ubifs_debug_info *d = c->dbg; + + d->old_zroot = *zroot; + lnum = d->old_zroot.lnum; + offs = d->old_zroot.offs; + len = d->old_zroot.len; + + idx = kmalloc(c->max_idx_node_sz, GFP_NOFS); + if (!idx) + return -ENOMEM; + + err = ubifs_read_node(c, idx, UBIFS_IDX_NODE, len, lnum, offs); + if (err) + goto out; + + d->old_zroot_level = le16_to_cpu(idx->level); + d->old_zroot_sqnum = le64_to_cpu(idx->ch.sqnum); +out: + kfree(idx); + return err; +} + +/** + * dbg_check_old_index - check the old copy of the index. + * @c: UBIFS file-system description object + * @zroot: root of the new index + * + * In order to be able to recover from an unclean unmount, a complete copy of + * the index must exist on flash. This is the "old" index. The commit process + * must write the "new" index to flash without overwriting or destroying any + * part of the old index. This function is run at commit end in order to check + * that the old index does indeed exist completely intact. + * + * This function returns %0 on success and a negative error code on failure. + */ +int dbg_check_old_index(struct ubifs_info *c, struct ubifs_zbranch *zroot) +{ + int lnum, offs, len, err = 0, last_level, child_cnt; + int first = 1, iip; + struct ubifs_debug_info *d = c->dbg; + union ubifs_key lower_key, upper_key, l_key, u_key; + unsigned long long last_sqnum; + struct ubifs_idx_node *idx; + struct list_head list; + struct idx_node *i; + size_t sz; + + if (!dbg_is_chk_index(c)) + return 0; + + INIT_LIST_HEAD(&list); + + sz = sizeof(struct idx_node) + ubifs_idx_node_sz(c, c->fanout) - + UBIFS_IDX_NODE_SZ; + + /* Start at the old zroot */ + lnum = d->old_zroot.lnum; + offs = d->old_zroot.offs; + len = d->old_zroot.len; + iip = 0; + + /* + * Traverse the index tree preorder depth-first i.e. do a node and then + * its subtrees from left to right. + */ + while (1) { + struct ubifs_branch *br; + + /* Get the next index node */ + i = kmalloc(sz, GFP_NOFS); + if (!i) { + err = -ENOMEM; + goto out_free; + } + i->iip = iip; + /* Keep the index nodes on our path in a linked list */ + list_add_tail(&i->list, &list); + /* Read the index node */ + idx = &i->idx; + err = ubifs_read_node(c, idx, UBIFS_IDX_NODE, len, lnum, offs); + if (err) + goto out_free; + /* Validate index node */ + child_cnt = le16_to_cpu(idx->child_cnt); + if (child_cnt < 1 || child_cnt > c->fanout) { + err = 1; + goto out_dump; + } + if (first) { + first = 0; + /* Check root level and sqnum */ + if (le16_to_cpu(idx->level) != d->old_zroot_level) { + err = 2; + goto out_dump; + } + if (le64_to_cpu(idx->ch.sqnum) != d->old_zroot_sqnum) { + err = 3; + goto out_dump; + } + /* Set last values as though root had a parent */ + last_level = le16_to_cpu(idx->level) + 1; + last_sqnum = le64_to_cpu(idx->ch.sqnum) + 1; + key_read(c, ubifs_idx_key(c, idx), &lower_key); + highest_ino_key(c, &upper_key, INUM_WATERMARK); + } + key_copy(c, &upper_key, &i->upper_key); + if (le16_to_cpu(idx->level) != last_level - 1) { + err = 3; + goto out_dump; + } + /* + * The index is always written bottom up hence a child's sqnum + * is always less than the parents. + */ + if (le64_to_cpu(idx->ch.sqnum) >= last_sqnum) { + err = 4; + goto out_dump; + } + /* Check key range */ + key_read(c, ubifs_idx_key(c, idx), &l_key); + br = ubifs_idx_branch(c, idx, child_cnt - 1); + key_read(c, &br->key, &u_key); + if (keys_cmp(c, &lower_key, &l_key) > 0) { + err = 5; + goto out_dump; + } + if (keys_cmp(c, &upper_key, &u_key) < 0) { + err = 6; + goto out_dump; + } + if (keys_cmp(c, &upper_key, &u_key) == 0) + if (!is_hash_key(c, &u_key)) { + err = 7; + goto out_dump; + } + /* Go to next index node */ + if (le16_to_cpu(idx->level) == 0) { + /* At the bottom, so go up until can go right */ + while (1) { + /* Drop the bottom of the list */ + list_del(&i->list); + kfree(i); + /* No more list means we are done */ + if (list_empty(&list)) + goto out; + /* Look at the new bottom */ + i = list_entry(list.prev, struct idx_node, + list); + idx = &i->idx; + /* Can we go right */ + if (iip + 1 < le16_to_cpu(idx->child_cnt)) { + iip = iip + 1; + break; + } else + /* Nope, so go up again */ + iip = i->iip; + } + } else + /* Go down left */ + iip = 0; + /* + * We have the parent in 'idx' and now we set up for reading the + * child pointed to by slot 'iip'. + */ + last_level = le16_to_cpu(idx->level); + last_sqnum = le64_to_cpu(idx->ch.sqnum); + br = ubifs_idx_branch(c, idx, iip); + lnum = le32_to_cpu(br->lnum); + offs = le32_to_cpu(br->offs); + len = le32_to_cpu(br->len); + key_read(c, &br->key, &lower_key); + if (iip + 1 < le16_to_cpu(idx->child_cnt)) { + br = ubifs_idx_branch(c, idx, iip + 1); + key_read(c, &br->key, &upper_key); + } else + key_copy(c, &i->upper_key, &upper_key); + } +out: + err = dbg_old_index_check_init(c, zroot); + if (err) + goto out_free; + + return 0; + +out_dump: + ubifs_err(c, "dumping index node (iip=%d)", i->iip); + ubifs_dump_node(c, idx, ubifs_idx_node_sz(c, c->fanout)); + list_del(&i->list); + kfree(i); + if (!list_empty(&list)) { + i = list_entry(list.prev, struct idx_node, list); + ubifs_err(c, "dumping parent index node"); + ubifs_dump_node(c, &i->idx, ubifs_idx_node_sz(c, c->fanout)); + } +out_free: + while (!list_empty(&list)) { + i = list_entry(list.next, struct idx_node, list); + list_del(&i->list); + kfree(i); + } + ubifs_err(c, "failed, error %d", err); + if (err > 0) + err = -EINVAL; + return err; +} diff --git a/fs/ubifs/compress.c b/fs/ubifs/compress.c new file mode 100644 index 000000000..059a02691 --- /dev/null +++ b/fs/ubifs/compress.c @@ -0,0 +1,369 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * Copyright (C) 2006, 2007 University of Szeged, Hungary + * + * Authors: Adrian Hunter + * Artem Bityutskiy (Битюцкий Артём) + * Zoltan Sogor + */ + +/* + * This file provides a single place to access to compression and + * decompression. + */ + +#include +#include +#include "ubifs.h" + +union ubifs_in_ptr { + const void *buf; + struct folio *folio; +}; + +/* Fake description object for the "none" compressor */ +static struct ubifs_compressor none_compr = { + .compr_type = UBIFS_COMPR_NONE, + .name = "none", + .capi_name = "", +}; + +#ifdef CONFIG_UBIFS_FS_LZO +static struct ubifs_compressor lzo_compr = { + .compr_type = UBIFS_COMPR_LZO, + .name = "lzo", + .capi_name = "lzo", +}; +#else +static struct ubifs_compressor lzo_compr = { + .compr_type = UBIFS_COMPR_LZO, + .name = "lzo", +}; +#endif + +#ifdef CONFIG_UBIFS_FS_ZLIB +static struct ubifs_compressor zlib_compr = { + .compr_type = UBIFS_COMPR_ZLIB, + .name = "zlib", + .capi_name = "deflate", +}; +#else +static struct ubifs_compressor zlib_compr = { + .compr_type = UBIFS_COMPR_ZLIB, + .name = "zlib", +}; +#endif + +#ifdef CONFIG_UBIFS_FS_ZSTD +static struct ubifs_compressor zstd_compr = { + .compr_type = UBIFS_COMPR_ZSTD, + .name = "zstd", + .capi_name = "zstd", +}; +#else +static struct ubifs_compressor zstd_compr = { + .compr_type = UBIFS_COMPR_ZSTD, + .name = "zstd", +}; +#endif + +/* All UBIFS compressors */ +struct ubifs_compressor *ubifs_compressors[UBIFS_COMPR_TYPES_CNT]; + +static void ubifs_compress_common(int *compr_type, union ubifs_in_ptr in_ptr, + size_t in_offset, int in_len, bool in_folio, + void *out_buf, int *out_len) +{ + struct ubifs_compressor *compr = ubifs_compressors[*compr_type]; + int dlen = *out_len; + int err; + + if (*compr_type == UBIFS_COMPR_NONE) + goto no_compr; + + /* If the input data is small, do not even try to compress it */ + if (in_len < UBIFS_MIN_COMPR_LEN) + goto no_compr; + + dlen = min(dlen, in_len - UBIFS_MIN_COMPRESS_DIFF); + + do { + ACOMP_REQUEST_ON_STACK(req, compr->cc); + DECLARE_CRYPTO_WAIT(wait); + + acomp_request_set_callback(req, 0, NULL, NULL); + if (in_folio) + acomp_request_set_src_folio(req, in_ptr.folio, + in_offset, in_len); + else + acomp_request_set_src_dma(req, in_ptr.buf, in_len); + acomp_request_set_dst_dma(req, out_buf, dlen); + err = crypto_acomp_compress(req); + dlen = req->dlen; + if (err != -EAGAIN) + break; + + req = ACOMP_REQUEST_CLONE(req, GFP_NOFS | __GFP_NOWARN); + acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + err = crypto_acomp_compress(req); + err = crypto_wait_req(err, &wait); + dlen = req->dlen; + acomp_request_free(req); + } while (0); + + *out_len = dlen; + if (err) + goto no_compr; + + return; + +no_compr: + if (in_folio) + memcpy_from_folio(out_buf, in_ptr.folio, in_offset, in_len); + else + memcpy(out_buf, in_ptr.buf, in_len); + *out_len = in_len; + *compr_type = UBIFS_COMPR_NONE; +} + +/** + * ubifs_compress - compress data. + * @c: UBIFS file-system description object + * @in_buf: data to compress + * @in_len: length of the data to compress + * @out_buf: output buffer where compressed data should be stored + * @out_len: output buffer length is returned here + * @compr_type: type of compression to use on enter, actually used compression + * type on exit + * + * This function compresses input buffer @in_buf of length @in_len and stores + * the result in the output buffer @out_buf and the resulting length in + * @out_len. If the input buffer does not compress, it is just copied to the + * @out_buf. The same happens if @compr_type is %UBIFS_COMPR_NONE or if + * compression error occurred. + * + * Note, if the input buffer was not compressed, it is copied to the output + * buffer and %UBIFS_COMPR_NONE is returned in @compr_type. + */ +void ubifs_compress(const struct ubifs_info *c, const void *in_buf, + int in_len, void *out_buf, int *out_len, int *compr_type) +{ + union ubifs_in_ptr in_ptr = { .buf = in_buf }; + + ubifs_compress_common(compr_type, in_ptr, 0, in_len, false, + out_buf, out_len); +} + +/** + * ubifs_compress_folio - compress folio. + * @c: UBIFS file-system description object + * @in_folio: data to compress + * @in_offset: offset into @in_folio + * @in_len: length of the data to compress + * @out_buf: output buffer where compressed data should be stored + * @out_len: output buffer length is returned here + * @compr_type: type of compression to use on enter, actually used compression + * type on exit + * + * This function compresses input folio @in_folio of length @in_len and + * stores the result in the output buffer @out_buf and the resulting length + * in @out_len. If the input buffer does not compress, it is just copied + * to the @out_buf. The same happens if @compr_type is %UBIFS_COMPR_NONE + * or if compression error occurred. + * + * Note, if the input buffer was not compressed, it is copied to the output + * buffer and %UBIFS_COMPR_NONE is returned in @compr_type. + */ +void ubifs_compress_folio(const struct ubifs_info *c, struct folio *in_folio, + size_t in_offset, int in_len, void *out_buf, + int *out_len, int *compr_type) +{ + union ubifs_in_ptr in_ptr = { .folio = in_folio }; + + ubifs_compress_common(compr_type, in_ptr, in_offset, in_len, true, + out_buf, out_len); +} + +static int ubifs_decompress_common(const struct ubifs_info *c, + const void *in_buf, int in_len, + void *out_ptr, size_t out_offset, + int *out_len, bool out_folio, + int compr_type) +{ + struct ubifs_compressor *compr; + int dlen = *out_len; + int err; + + if (unlikely(compr_type < 0 || compr_type >= UBIFS_COMPR_TYPES_CNT)) { + ubifs_err(c, "invalid compression type %d", compr_type); + return -EINVAL; + } + + compr = ubifs_compressors[compr_type]; + + if (unlikely(!compr->capi_name)) { + ubifs_err(c, "%s compression is not compiled in", compr->name); + return -EINVAL; + } + + if (compr_type == UBIFS_COMPR_NONE) { + if (out_folio) + memcpy_to_folio(out_ptr, out_offset, in_buf, in_len); + else + memcpy(out_ptr, in_buf, in_len); + *out_len = in_len; + return 0; + } + + do { + ACOMP_REQUEST_ON_STACK(req, compr->cc); + DECLARE_CRYPTO_WAIT(wait); + + acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + acomp_request_set_src_dma(req, in_buf, in_len); + if (out_folio) + acomp_request_set_dst_folio(req, out_ptr, out_offset, + dlen); + else + acomp_request_set_dst_dma(req, out_ptr, dlen); + err = crypto_acomp_decompress(req); + dlen = req->dlen; + if (err != -EAGAIN) + break; + + req = ACOMP_REQUEST_CLONE(req, GFP_NOFS | __GFP_NOWARN); + err = crypto_acomp_decompress(req); + err = crypto_wait_req(err, &wait); + dlen = req->dlen; + acomp_request_free(req); + } while (0); + + *out_len = dlen; + if (err) + ubifs_err(c, "cannot decompress %d bytes, compressor %s, error %d", + in_len, compr->name, err); + + return err; +} + +/** + * ubifs_decompress - decompress data. + * @c: UBIFS file-system description object + * @in_buf: data to decompress + * @in_len: length of the data to decompress + * @out_buf: output buffer where decompressed data should + * @out_len: output length is returned here + * @compr_type: type of compression + * + * This function decompresses data from buffer @in_buf into buffer @out_buf. + * The length of the uncompressed data is returned in @out_len. This functions + * returns %0 on success or a negative error code on failure. + */ +int ubifs_decompress(const struct ubifs_info *c, const void *in_buf, + int in_len, void *out_buf, int *out_len, int compr_type) +{ + return ubifs_decompress_common(c, in_buf, in_len, out_buf, 0, out_len, + false, compr_type); +} + +/** + * ubifs_decompress_folio - decompress folio. + * @c: UBIFS file-system description object + * @in_buf: data to decompress + * @in_len: length of the data to decompress + * @out_folio: output folio where decompressed data should + * @out_offset: offset into @out_folio + * @out_len: output length is returned here + * @compr_type: type of compression + * + * This function decompresses data from buffer @in_buf into folio + * @out_folio. The length of the uncompressed data is returned in + * @out_len. This functions returns %0 on success or a negative error + * code on failure. + */ +int ubifs_decompress_folio(const struct ubifs_info *c, const void *in_buf, + int in_len, struct folio *out_folio, + size_t out_offset, int *out_len, int compr_type) +{ + return ubifs_decompress_common(c, in_buf, in_len, out_folio, + out_offset, out_len, true, compr_type); +} + +/** + * compr_init - initialize a compressor. + * @compr: compressor description object + * + * This function initializes the requested compressor and returns zero in case + * of success or a negative error code in case of failure. + */ +static int __init compr_init(struct ubifs_compressor *compr) +{ + if (compr->capi_name) { + compr->cc = crypto_alloc_acomp(compr->capi_name, 0, 0); + if (IS_ERR(compr->cc)) { + pr_err("UBIFS error (pid %d): cannot initialize compressor %s, error %ld", + current->pid, compr->name, PTR_ERR(compr->cc)); + return PTR_ERR(compr->cc); + } + } + + ubifs_compressors[compr->compr_type] = compr; + return 0; +} + +/** + * compr_exit - de-initialize a compressor. + * @compr: compressor description object + */ +static void compr_exit(struct ubifs_compressor *compr) +{ + if (compr->capi_name) + crypto_free_acomp(compr->cc); +} + +/** + * ubifs_compressors_init - initialize UBIFS compressors. + * + * This function initializes the compressor which were compiled in. Returns + * zero in case of success and a negative error code in case of failure. + */ +int __init ubifs_compressors_init(void) +{ + int err; + + err = compr_init(&lzo_compr); + if (err) + return err; + + err = compr_init(&zstd_compr); + if (err) + goto out_lzo; + + err = compr_init(&zlib_compr); + if (err) + goto out_zstd; + + ubifs_compressors[UBIFS_COMPR_NONE] = &none_compr; + return 0; + +out_zstd: + compr_exit(&zstd_compr); +out_lzo: + compr_exit(&lzo_compr); + return err; +} + +/** + * ubifs_compressors_exit - de-initialize UBIFS compressors. + */ +void ubifs_compressors_exit(void) +{ + compr_exit(&lzo_compr); + compr_exit(&zlib_compr); + compr_exit(&zstd_compr); +} diff --git a/fs/ubifs/crypto.c b/fs/ubifs/crypto.c new file mode 100644 index 000000000..0b14d004a --- /dev/null +++ b/fs/ubifs/crypto.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "ubifs.h" + +static int ubifs_crypt_get_context(struct inode *inode, void *ctx, size_t len) +{ + return ubifs_xattr_get(inode, UBIFS_XATTR_NAME_ENCRYPTION_CONTEXT, + ctx, len); +} + +static int ubifs_crypt_set_context(struct inode *inode, const void *ctx, + size_t len, void *fs_data) +{ + /* + * Creating an encryption context is done unlocked since we + * operate on a new inode which is not visible to other users + * at this point. So, no need to check whether inode is locked. + */ + return ubifs_xattr_set(inode, UBIFS_XATTR_NAME_ENCRYPTION_CONTEXT, + ctx, len, 0, false); +} + +static bool ubifs_crypt_empty_dir(struct inode *inode) +{ + return ubifs_check_dir_empty(inode) == 0; +} + +/** + * ubifs_encrypt - Encrypt data. + * @inode: inode which refers to the data node + * @dn: data node to encrypt + * @in_len: length of data to be compressed + * @out_len: allocated memory size for the data area of @dn + * @block: logical block number of the block + * + * This function encrypt a possibly-compressed data in the data node. + * The encrypted data length will store in @out_len. + */ +int ubifs_encrypt(const struct inode *inode, struct ubifs_data_node *dn, + unsigned int in_len, unsigned int *out_len, int block) +{ + struct ubifs_info *c = inode->i_sb->s_fs_info; + void *p = &dn->data; + unsigned int pad_len = round_up(in_len, UBIFS_CIPHER_BLOCK_SIZE); + int err; + + ubifs_assert(c, pad_len <= *out_len); + dn->compr_size = cpu_to_le16(in_len); + + /* pad to full block cipher length */ + if (pad_len != in_len) + memset(p + in_len, 0, pad_len - in_len); + + err = fscrypt_encrypt_block_inplace(inode, virt_to_page(p), pad_len, + offset_in_page(p), block); + if (err) { + ubifs_err(c, "fscrypt_encrypt_block_inplace() failed: %d", err); + return err; + } + *out_len = pad_len; + + return 0; +} + +int ubifs_decrypt(const struct inode *inode, struct ubifs_data_node *dn, + unsigned int *out_len, int block) +{ + struct ubifs_info *c = inode->i_sb->s_fs_info; + int err; + unsigned int clen = le16_to_cpu(dn->compr_size); + unsigned int dlen = *out_len; + + if (clen <= 0 || clen > UBIFS_BLOCK_SIZE || clen > dlen) { + ubifs_err(c, "bad compr_size: %i", clen); + return -EINVAL; + } + + ubifs_assert(c, dlen <= UBIFS_BLOCK_SIZE); + err = fscrypt_decrypt_block_inplace(inode, virt_to_page(&dn->data), + dlen, offset_in_page(&dn->data), + block); + if (err) { + ubifs_err(c, "fscrypt_decrypt_block_inplace() failed: %d", err); + return err; + } + *out_len = clen; + + return 0; +} + +const struct fscrypt_operations ubifs_crypt_operations = { + .inode_info_offs = (int)offsetof(struct ubifs_inode, i_crypt_info) - + (int)offsetof(struct ubifs_inode, vfs_inode), + .legacy_key_prefix = "ubifs:", + .get_context = ubifs_crypt_get_context, + .set_context = ubifs_crypt_set_context, + .empty_dir = ubifs_crypt_empty_dir, +}; diff --git a/fs/ubifs/debug.c b/fs/ubifs/debug.c new file mode 100644 index 000000000..5794de5a9 --- /dev/null +++ b/fs/ubifs/debug.c @@ -0,0 +1,3052 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation + * + * Authors: Artem Bityutskiy (Битюцкий Артём) + * Adrian Hunter + */ + +/* + * This file implements most of the debugging stuff which is compiled in only + * when it is enabled. But some debugging check functions are implemented in + * corresponding subsystem, just because they are closely related and utilize + * various local functions of those subsystems. + */ + +#include +#include +#include +#include +#include +#include +#include "ubifs.h" + +static DEFINE_SPINLOCK(dbg_lock); + +static const char *get_key_fmt(int fmt) +{ + switch (fmt) { + case UBIFS_SIMPLE_KEY_FMT: + return "simple"; + default: + return "unknown/invalid format"; + } +} + +static const char *get_key_hash(int hash) +{ + switch (hash) { + case UBIFS_KEY_HASH_R5: + return "R5"; + case UBIFS_KEY_HASH_TEST: + return "test"; + default: + return "unknown/invalid name hash"; + } +} + +static const char *get_key_type(int type) +{ + switch (type) { + case UBIFS_INO_KEY: + return "inode"; + case UBIFS_DENT_KEY: + return "direntry"; + case UBIFS_XENT_KEY: + return "xentry"; + case UBIFS_DATA_KEY: + return "data"; + case UBIFS_TRUN_KEY: + return "truncate"; + default: + return "unknown/invalid key"; + } +} + +static const char *get_dent_type(int type) +{ + switch (type) { + case UBIFS_ITYPE_REG: + return "file"; + case UBIFS_ITYPE_DIR: + return "dir"; + case UBIFS_ITYPE_LNK: + return "symlink"; + case UBIFS_ITYPE_BLK: + return "blkdev"; + case UBIFS_ITYPE_CHR: + return "char dev"; + case UBIFS_ITYPE_FIFO: + return "fifo"; + case UBIFS_ITYPE_SOCK: + return "socket"; + default: + return "unknown/invalid type"; + } +} + +const char *dbg_snprintf_key(const struct ubifs_info *c, + const union ubifs_key *key, char *buffer, int len) +{ + char *p = buffer; + int type = key_type(c, key); + + if (c->key_fmt == UBIFS_SIMPLE_KEY_FMT) { + switch (type) { + case UBIFS_INO_KEY: + len -= snprintf(p, len, "(%lu, %s)", + (unsigned long)key_inum(c, key), + get_key_type(type)); + break; + case UBIFS_DENT_KEY: + case UBIFS_XENT_KEY: + len -= snprintf(p, len, "(%lu, %s, %#08x)", + (unsigned long)key_inum(c, key), + get_key_type(type), key_hash(c, key)); + break; + case UBIFS_DATA_KEY: + len -= snprintf(p, len, "(%lu, %s, %u)", + (unsigned long)key_inum(c, key), + get_key_type(type), key_block(c, key)); + break; + case UBIFS_TRUN_KEY: + len -= snprintf(p, len, "(%lu, %s)", + (unsigned long)key_inum(c, key), + get_key_type(type)); + break; + default: + len -= snprintf(p, len, "(bad key type: %#08x, %#08x)", + key->u32[0], key->u32[1]); + } + } else + len -= snprintf(p, len, "bad key format %d", c->key_fmt); + ubifs_assert(c, len > 0); + return p; +} + +const char *dbg_ntype(int type) +{ + switch (type) { + case UBIFS_PAD_NODE: + return "padding node"; + case UBIFS_SB_NODE: + return "superblock node"; + case UBIFS_MST_NODE: + return "master node"; + case UBIFS_REF_NODE: + return "reference node"; + case UBIFS_INO_NODE: + return "inode node"; + case UBIFS_DENT_NODE: + return "direntry node"; + case UBIFS_XENT_NODE: + return "xentry node"; + case UBIFS_DATA_NODE: + return "data node"; + case UBIFS_TRUN_NODE: + return "truncate node"; + case UBIFS_IDX_NODE: + return "indexing node"; + case UBIFS_CS_NODE: + return "commit start node"; + case UBIFS_ORPH_NODE: + return "orphan node"; + case UBIFS_AUTH_NODE: + return "auth node"; + default: + return "unknown node"; + } +} + +static const char *dbg_gtype(int type) +{ + switch (type) { + case UBIFS_NO_NODE_GROUP: + return "no node group"; + case UBIFS_IN_NODE_GROUP: + return "in node group"; + case UBIFS_LAST_OF_NODE_GROUP: + return "last of node group"; + default: + return "unknown"; + } +} + +const char *dbg_cstate(int cmt_state) +{ + switch (cmt_state) { + case COMMIT_RESTING: + return "commit resting"; + case COMMIT_BACKGROUND: + return "background commit requested"; + case COMMIT_REQUIRED: + return "commit required"; + case COMMIT_RUNNING_BACKGROUND: + return "BACKGROUND commit running"; + case COMMIT_RUNNING_REQUIRED: + return "commit running and required"; + case COMMIT_BROKEN: + return "broken commit"; + default: + return "unknown commit state"; + } +} + +const char *dbg_jhead(int jhead) +{ + switch (jhead) { + case GCHD: + return "0 (GC)"; + case BASEHD: + return "1 (base)"; + case DATAHD: + return "2 (data)"; + default: + return "unknown journal head"; + } +} + +static void dump_ch(const struct ubifs_ch *ch) +{ + pr_err("\tmagic %#x\n", le32_to_cpu(ch->magic)); + pr_err("\tcrc %#x\n", le32_to_cpu(ch->crc)); + pr_err("\tnode_type %d (%s)\n", ch->node_type, + dbg_ntype(ch->node_type)); + pr_err("\tgroup_type %d (%s)\n", ch->group_type, + dbg_gtype(ch->group_type)); + pr_err("\tsqnum %llu\n", + (unsigned long long)le64_to_cpu(ch->sqnum)); + pr_err("\tlen %u\n", le32_to_cpu(ch->len)); +} + +void ubifs_dump_inode(struct ubifs_info *c, const struct inode *inode) +{ + const struct ubifs_inode *ui = ubifs_inode(inode); + struct fscrypt_name nm = {0}; + union ubifs_key key; + struct ubifs_dent_node *dent, *pdent = NULL; + int count = 2; + + pr_err("Dump in-memory inode:"); + pr_err("\tinode %llu\n", inode->i_ino); + pr_err("\tsize %llu\n", + (unsigned long long)i_size_read(inode)); + pr_err("\tnlink %u\n", inode->i_nlink); + pr_err("\tuid %u\n", (unsigned int)i_uid_read(inode)); + pr_err("\tgid %u\n", (unsigned int)i_gid_read(inode)); + pr_err("\tatime %u.%u\n", + (unsigned int) inode_get_atime_sec(inode), + (unsigned int) inode_get_atime_nsec(inode)); + pr_err("\tmtime %u.%u\n", + (unsigned int) inode_get_mtime_sec(inode), + (unsigned int) inode_get_mtime_nsec(inode)); + pr_err("\tctime %u.%u\n", + (unsigned int) inode_get_ctime_sec(inode), + (unsigned int) inode_get_ctime_nsec(inode)); + pr_err("\tcreat_sqnum %llu\n", ui->creat_sqnum); + pr_err("\txattr_size %u\n", ui->xattr_size); + pr_err("\txattr_cnt %u\n", ui->xattr_cnt); + pr_err("\txattr_names %u\n", ui->xattr_names); + pr_err("\tdirty %u\n", ui->dirty); + pr_err("\txattr %u\n", ui->xattr); + pr_err("\tbulk_read %u\n", ui->bulk_read); + pr_err("\tsynced_i_size %llu\n", + (unsigned long long)ui->synced_i_size); + pr_err("\tui_size %llu\n", + (unsigned long long)ui->ui_size); + pr_err("\tflags %d\n", ui->flags); + pr_err("\tcompr_type %d\n", ui->compr_type); + pr_err("\tlast_page_read %lu\n", ui->last_page_read); + pr_err("\tread_in_a_row %lu\n", ui->read_in_a_row); + pr_err("\tdata_len %d\n", ui->data_len); + + if (!S_ISDIR(inode->i_mode)) + return; + + pr_err("List of directory entries:\n"); + ubifs_assert(c, !mutex_is_locked(&c->tnc_mutex)); + + lowest_dent_key(c, &key, inode->i_ino); + while (1) { + dent = ubifs_tnc_next_ent(c, &key, &nm); + if (IS_ERR(dent)) { + if (PTR_ERR(dent) != -ENOENT) + pr_err("error %ld\n", PTR_ERR(dent)); + break; + } + + pr_err("\t%d: inode %llu, type %s, len %d\n", + count++, (unsigned long long) le64_to_cpu(dent->inum), + get_dent_type(dent->type), + le16_to_cpu(dent->nlen)); + + fname_name(&nm) = dent->name; + fname_len(&nm) = le16_to_cpu(dent->nlen); + kfree(pdent); + pdent = dent; + key_read(c, &dent->key, &key); + } + kfree(pdent); +} + +void ubifs_dump_node(const struct ubifs_info *c, const void *node, int node_len) +{ + int i, n, type, safe_len, max_node_len, min_node_len; + union ubifs_key key; + const struct ubifs_ch *ch = node; + char key_buf[DBG_KEY_BUF_LEN]; + + /* If the magic is incorrect, just hexdump the first bytes */ + if (le32_to_cpu(ch->magic) != UBIFS_NODE_MAGIC) { + pr_err("Not a node, first %zu bytes:", UBIFS_CH_SZ); + print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 32, 1, + (void *)node, UBIFS_CH_SZ, 1); + return; + } + + /* Skip dumping unknown type node */ + type = ch->node_type; + if (type < 0 || type >= UBIFS_NODE_TYPES_CNT) { + pr_err("node type %d was not recognized\n", type); + return; + } + + spin_lock(&dbg_lock); + dump_ch(node); + + if (c->ranges[type].max_len == 0) { + max_node_len = min_node_len = c->ranges[type].len; + } else { + max_node_len = c->ranges[type].max_len; + min_node_len = c->ranges[type].min_len; + } + safe_len = le32_to_cpu(ch->len); + safe_len = safe_len > 0 ? safe_len : 0; + safe_len = min3(safe_len, max_node_len, node_len); + if (safe_len < min_node_len) { + pr_err("node len(%d) is too short for %s, left %d bytes:\n", + safe_len, dbg_ntype(type), + safe_len > UBIFS_CH_SZ ? + safe_len - (int)UBIFS_CH_SZ : 0); + if (safe_len > UBIFS_CH_SZ) + print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 32, 1, + (void *)node + UBIFS_CH_SZ, + safe_len - UBIFS_CH_SZ, 0); + goto out_unlock; + } + if (safe_len != le32_to_cpu(ch->len)) + pr_err("\ttruncated node length %d\n", safe_len); + + switch (type) { + case UBIFS_PAD_NODE: + { + const struct ubifs_pad_node *pad = node; + + pr_err("\tpad_len %u\n", le32_to_cpu(pad->pad_len)); + break; + } + case UBIFS_SB_NODE: + { + const struct ubifs_sb_node *sup = node; + unsigned int sup_flags = le32_to_cpu(sup->flags); + + pr_err("\tkey_hash %d (%s)\n", + (int)sup->key_hash, get_key_hash(sup->key_hash)); + pr_err("\tkey_fmt %d (%s)\n", + (int)sup->key_fmt, get_key_fmt(sup->key_fmt)); + pr_err("\tflags %#x\n", sup_flags); + pr_err("\tbig_lpt %u\n", + !!(sup_flags & UBIFS_FLG_BIGLPT)); + pr_err("\tspace_fixup %u\n", + !!(sup_flags & UBIFS_FLG_SPACE_FIXUP)); + pr_err("\tmin_io_size %u\n", le32_to_cpu(sup->min_io_size)); + pr_err("\tleb_size %u\n", le32_to_cpu(sup->leb_size)); + pr_err("\tleb_cnt %u\n", le32_to_cpu(sup->leb_cnt)); + pr_err("\tmax_leb_cnt %u\n", le32_to_cpu(sup->max_leb_cnt)); + pr_err("\tmax_bud_bytes %llu\n", + (unsigned long long)le64_to_cpu(sup->max_bud_bytes)); + pr_err("\tlog_lebs %u\n", le32_to_cpu(sup->log_lebs)); + pr_err("\tlpt_lebs %u\n", le32_to_cpu(sup->lpt_lebs)); + pr_err("\torph_lebs %u\n", le32_to_cpu(sup->orph_lebs)); + pr_err("\tjhead_cnt %u\n", le32_to_cpu(sup->jhead_cnt)); + pr_err("\tfanout %u\n", le32_to_cpu(sup->fanout)); + pr_err("\tlsave_cnt %u\n", le32_to_cpu(sup->lsave_cnt)); + pr_err("\tdefault_compr %u\n", + (int)le16_to_cpu(sup->default_compr)); + pr_err("\trp_size %llu\n", + (unsigned long long)le64_to_cpu(sup->rp_size)); + pr_err("\trp_uid %u\n", le32_to_cpu(sup->rp_uid)); + pr_err("\trp_gid %u\n", le32_to_cpu(sup->rp_gid)); + pr_err("\tfmt_version %u\n", le32_to_cpu(sup->fmt_version)); + pr_err("\ttime_gran %u\n", le32_to_cpu(sup->time_gran)); + pr_err("\tUUID %pUB\n", sup->uuid); + break; + } + case UBIFS_MST_NODE: + { + const struct ubifs_mst_node *mst = node; + + pr_err("\thighest_inum %llu\n", + (unsigned long long)le64_to_cpu(mst->highest_inum)); + pr_err("\tcommit number %llu\n", + (unsigned long long)le64_to_cpu(mst->cmt_no)); + pr_err("\tflags %#x\n", le32_to_cpu(mst->flags)); + pr_err("\tlog_lnum %u\n", le32_to_cpu(mst->log_lnum)); + pr_err("\troot_lnum %u\n", le32_to_cpu(mst->root_lnum)); + pr_err("\troot_offs %u\n", le32_to_cpu(mst->root_offs)); + pr_err("\troot_len %u\n", le32_to_cpu(mst->root_len)); + pr_err("\tgc_lnum %u\n", le32_to_cpu(mst->gc_lnum)); + pr_err("\tihead_lnum %u\n", le32_to_cpu(mst->ihead_lnum)); + pr_err("\tihead_offs %u\n", le32_to_cpu(mst->ihead_offs)); + pr_err("\tindex_size %llu\n", + (unsigned long long)le64_to_cpu(mst->index_size)); + pr_err("\tlpt_lnum %u\n", le32_to_cpu(mst->lpt_lnum)); + pr_err("\tlpt_offs %u\n", le32_to_cpu(mst->lpt_offs)); + pr_err("\tnhead_lnum %u\n", le32_to_cpu(mst->nhead_lnum)); + pr_err("\tnhead_offs %u\n", le32_to_cpu(mst->nhead_offs)); + pr_err("\tltab_lnum %u\n", le32_to_cpu(mst->ltab_lnum)); + pr_err("\tltab_offs %u\n", le32_to_cpu(mst->ltab_offs)); + pr_err("\tlsave_lnum %u\n", le32_to_cpu(mst->lsave_lnum)); + pr_err("\tlsave_offs %u\n", le32_to_cpu(mst->lsave_offs)); + pr_err("\tlscan_lnum %u\n", le32_to_cpu(mst->lscan_lnum)); + pr_err("\tleb_cnt %u\n", le32_to_cpu(mst->leb_cnt)); + pr_err("\tempty_lebs %u\n", le32_to_cpu(mst->empty_lebs)); + pr_err("\tidx_lebs %u\n", le32_to_cpu(mst->idx_lebs)); + pr_err("\ttotal_free %llu\n", + (unsigned long long)le64_to_cpu(mst->total_free)); + pr_err("\ttotal_dirty %llu\n", + (unsigned long long)le64_to_cpu(mst->total_dirty)); + pr_err("\ttotal_used %llu\n", + (unsigned long long)le64_to_cpu(mst->total_used)); + pr_err("\ttotal_dead %llu\n", + (unsigned long long)le64_to_cpu(mst->total_dead)); + pr_err("\ttotal_dark %llu\n", + (unsigned long long)le64_to_cpu(mst->total_dark)); + break; + } + case UBIFS_REF_NODE: + { + const struct ubifs_ref_node *ref = node; + + pr_err("\tlnum %u\n", le32_to_cpu(ref->lnum)); + pr_err("\toffs %u\n", le32_to_cpu(ref->offs)); + pr_err("\tjhead %u\n", le32_to_cpu(ref->jhead)); + break; + } + case UBIFS_INO_NODE: + { + const struct ubifs_ino_node *ino = node; + + key_read(c, &ino->key, &key); + pr_err("\tkey %s\n", + dbg_snprintf_key(c, &key, key_buf, DBG_KEY_BUF_LEN)); + pr_err("\tcreat_sqnum %llu\n", + (unsigned long long)le64_to_cpu(ino->creat_sqnum)); + pr_err("\tsize %llu\n", + (unsigned long long)le64_to_cpu(ino->size)); + pr_err("\tnlink %u\n", le32_to_cpu(ino->nlink)); + pr_err("\tatime %lld.%u\n", + (long long)le64_to_cpu(ino->atime_sec), + le32_to_cpu(ino->atime_nsec)); + pr_err("\tmtime %lld.%u\n", + (long long)le64_to_cpu(ino->mtime_sec), + le32_to_cpu(ino->mtime_nsec)); + pr_err("\tctime %lld.%u\n", + (long long)le64_to_cpu(ino->ctime_sec), + le32_to_cpu(ino->ctime_nsec)); + pr_err("\tuid %u\n", le32_to_cpu(ino->uid)); + pr_err("\tgid %u\n", le32_to_cpu(ino->gid)); + pr_err("\tmode %u\n", le32_to_cpu(ino->mode)); + pr_err("\tflags %#x\n", le32_to_cpu(ino->flags)); + pr_err("\txattr_cnt %u\n", le32_to_cpu(ino->xattr_cnt)); + pr_err("\txattr_size %u\n", le32_to_cpu(ino->xattr_size)); + pr_err("\txattr_names %u\n", le32_to_cpu(ino->xattr_names)); + pr_err("\tcompr_type %#x\n", + (int)le16_to_cpu(ino->compr_type)); + pr_err("\tdata len %u\n", le32_to_cpu(ino->data_len)); + break; + } + case UBIFS_DENT_NODE: + case UBIFS_XENT_NODE: + { + const struct ubifs_dent_node *dent = node; + int nlen = le16_to_cpu(dent->nlen); + + key_read(c, &dent->key, &key); + pr_err("\tkey %s\n", + dbg_snprintf_key(c, &key, key_buf, DBG_KEY_BUF_LEN)); + pr_err("\tinum %llu\n", + (unsigned long long)le64_to_cpu(dent->inum)); + pr_err("\ttype %d\n", (int)dent->type); + pr_err("\tnlen %d\n", nlen); + pr_err("\tname "); + + if (nlen > UBIFS_MAX_NLEN || + nlen > safe_len - UBIFS_DENT_NODE_SZ) + pr_err("(bad name length, not printing, bad or corrupted node)"); + else { + for (i = 0; i < nlen && dent->name[i]; i++) + pr_cont("%c", isprint(dent->name[i]) ? + dent->name[i] : '?'); + } + pr_cont("\n"); + + break; + } + case UBIFS_DATA_NODE: + { + const struct ubifs_data_node *dn = node; + + key_read(c, &dn->key, &key); + pr_err("\tkey %s\n", + dbg_snprintf_key(c, &key, key_buf, DBG_KEY_BUF_LEN)); + pr_err("\tsize %u\n", le32_to_cpu(dn->size)); + pr_err("\tcompr_typ %d\n", + (int)le16_to_cpu(dn->compr_type)); + pr_err("\tdata size %u\n", + le32_to_cpu(ch->len) - (unsigned int)UBIFS_DATA_NODE_SZ); + pr_err("\tdata (length = %d):\n", + safe_len - (int)UBIFS_DATA_NODE_SZ); + print_hex_dump(KERN_ERR, "\t", DUMP_PREFIX_OFFSET, 32, 1, + (void *)&dn->data, + safe_len - (int)UBIFS_DATA_NODE_SZ, 0); + break; + } + case UBIFS_TRUN_NODE: + { + const struct ubifs_trun_node *trun = node; + + pr_err("\tinum %u\n", le32_to_cpu(trun->inum)); + pr_err("\told_size %llu\n", + (unsigned long long)le64_to_cpu(trun->old_size)); + pr_err("\tnew_size %llu\n", + (unsigned long long)le64_to_cpu(trun->new_size)); + break; + } + case UBIFS_IDX_NODE: + { + const struct ubifs_idx_node *idx = node; + int max_child_cnt = (safe_len - UBIFS_IDX_NODE_SZ) / + (ubifs_idx_node_sz(c, 1) - + UBIFS_IDX_NODE_SZ); + + n = min_t(int, le16_to_cpu(idx->child_cnt), max_child_cnt); + pr_err("\tchild_cnt %d\n", (int)le16_to_cpu(idx->child_cnt)); + pr_err("\tlevel %d\n", (int)le16_to_cpu(idx->level)); + pr_err("\tBranches:\n"); + + for (i = 0; i < n && i < c->fanout; i++) { + const struct ubifs_branch *br; + + br = ubifs_idx_branch(c, idx, i); + key_read(c, &br->key, &key); + pr_err("\t%d: LEB %d:%d len %d key %s\n", + i, le32_to_cpu(br->lnum), le32_to_cpu(br->offs), + le32_to_cpu(br->len), + dbg_snprintf_key(c, &key, key_buf, + DBG_KEY_BUF_LEN)); + } + break; + } + case UBIFS_CS_NODE: + break; + case UBIFS_ORPH_NODE: + { + const struct ubifs_orph_node *orph = node; + + pr_err("\tcommit number %llu\n", + (unsigned long long) + le64_to_cpu(orph->cmt_no) & LLONG_MAX); + pr_err("\tlast node flag %llu\n", + (unsigned long long)(le64_to_cpu(orph->cmt_no)) >> 63); + n = (safe_len - UBIFS_ORPH_NODE_SZ) >> 3; + pr_err("\t%d orphan inode numbers:\n", n); + for (i = 0; i < n; i++) + pr_err("\t ino %llu\n", + (unsigned long long)le64_to_cpu(orph->inos[i])); + break; + } + case UBIFS_AUTH_NODE: + { + break; + } + default: + pr_err("node type %d was not recognized\n", type); + } + +out_unlock: + spin_unlock(&dbg_lock); +} + +void ubifs_dump_budget_req(const struct ubifs_budget_req *req) +{ + spin_lock(&dbg_lock); + pr_err("Budgeting request: new_ino %d, dirtied_ino %d\n", + req->new_ino, req->dirtied_ino); + pr_err("\tnew_ino_d %d, dirtied_ino_d %d\n", + req->new_ino_d, req->dirtied_ino_d); + pr_err("\tnew_page %d, dirtied_page %d\n", + req->new_page, req->dirtied_page); + pr_err("\tnew_dent %d, mod_dent %d\n", + req->new_dent, req->mod_dent); + pr_err("\tidx_growth %d\n", req->idx_growth); + pr_err("\tdata_growth %d dd_growth %d\n", + req->data_growth, req->dd_growth); + spin_unlock(&dbg_lock); +} + +void ubifs_dump_lstats(const struct ubifs_lp_stats *lst) +{ + spin_lock(&dbg_lock); + pr_err("(pid %d) Lprops statistics: empty_lebs %d, idx_lebs %d\n", + current->pid, lst->empty_lebs, lst->idx_lebs); + pr_err("\ttaken_empty_lebs %d, total_free %lld, total_dirty %lld\n", + lst->taken_empty_lebs, lst->total_free, lst->total_dirty); + pr_err("\ttotal_used %lld, total_dark %lld, total_dead %lld\n", + lst->total_used, lst->total_dark, lst->total_dead); + spin_unlock(&dbg_lock); +} + +void ubifs_dump_budg(struct ubifs_info *c, const struct ubifs_budg_info *bi) +{ + int i; + struct rb_node *rb; + struct ubifs_bud *bud; + struct ubifs_gced_idx_leb *idx_gc; + long long available, outstanding, free; + + spin_lock(&c->space_lock); + spin_lock(&dbg_lock); + pr_err("(pid %d) Budgeting info: data budget sum %lld, total budget sum %lld\n", + current->pid, bi->data_growth + bi->dd_growth, + bi->data_growth + bi->dd_growth + bi->idx_growth); + pr_err("\tbudg_data_growth %lld, budg_dd_growth %lld, budg_idx_growth %lld\n", + bi->data_growth, bi->dd_growth, bi->idx_growth); + pr_err("\tmin_idx_lebs %d, old_idx_sz %llu, uncommitted_idx %lld\n", + bi->min_idx_lebs, bi->old_idx_sz, bi->uncommitted_idx); + pr_err("\tpage_budget %d, inode_budget %d, dent_budget %d\n", + bi->page_budget, bi->inode_budget, bi->dent_budget); + pr_err("\tnospace %u, nospace_rp %u\n", bi->nospace, bi->nospace_rp); + pr_err("\tdark_wm %d, dead_wm %d, max_idx_node_sz %d\n", + c->dark_wm, c->dead_wm, c->max_idx_node_sz); + + if (bi != &c->bi) + /* + * If we are dumping saved budgeting data, do not print + * additional information which is about the current state, not + * the old one which corresponded to the saved budgeting data. + */ + goto out_unlock; + + pr_err("\tfreeable_cnt %d, calc_idx_sz %lld, idx_gc_cnt %d\n", + c->freeable_cnt, c->calc_idx_sz, c->idx_gc_cnt); + pr_err("\tdirty_pg_cnt %ld, dirty_zn_cnt %ld, clean_zn_cnt %ld\n", + atomic_long_read(&c->dirty_pg_cnt), + atomic_long_read(&c->dirty_zn_cnt), + atomic_long_read(&c->clean_zn_cnt)); + pr_err("\tgc_lnum %d, ihead_lnum %d\n", c->gc_lnum, c->ihead_lnum); + + /* If we are in R/O mode, journal heads do not exist */ + if (c->jheads) + for (i = 0; i < c->jhead_cnt; i++) + pr_err("\tjhead %s\t LEB %d\n", + dbg_jhead(c->jheads[i].wbuf.jhead), + c->jheads[i].wbuf.lnum); + for (rb = rb_first(&c->buds); rb; rb = rb_next(rb)) { + bud = rb_entry(rb, struct ubifs_bud, rb); + pr_err("\tbud LEB %d\n", bud->lnum); + } + list_for_each_entry(bud, &c->old_buds, list) + pr_err("\told bud LEB %d\n", bud->lnum); + list_for_each_entry(idx_gc, &c->idx_gc, list) + pr_err("\tGC'ed idx LEB %d unmap %d\n", + idx_gc->lnum, idx_gc->unmap); + pr_err("\tcommit state %d\n", c->cmt_state); + + /* Print budgeting predictions */ + available = ubifs_calc_available(c, c->bi.min_idx_lebs); + outstanding = c->bi.data_growth + c->bi.dd_growth; + free = ubifs_get_free_space_nolock(c); + pr_err("Budgeting predictions:\n"); + pr_err("\tavailable: %lld, outstanding %lld, free %lld\n", + available, outstanding, free); +out_unlock: + spin_unlock(&dbg_lock); + spin_unlock(&c->space_lock); +} + +void ubifs_dump_lprop(const struct ubifs_info *c, const struct ubifs_lprops *lp) +{ + int i, spc, dark = 0, dead = 0; + struct rb_node *rb; + struct ubifs_bud *bud; + + spc = lp->free + lp->dirty; + if (spc < c->dead_wm) + dead = spc; + else + dark = ubifs_calc_dark(c, spc); + + if (lp->flags & LPROPS_INDEX) + pr_err("LEB %-7d free %-8d dirty %-8d used %-8d free + dirty %-8d flags %#x (", + lp->lnum, lp->free, lp->dirty, c->leb_size - spc, spc, + lp->flags); + else + pr_err("LEB %-7d free %-8d dirty %-8d used %-8d free + dirty %-8d dark %-4d dead %-4d nodes fit %-3d flags %#-4x (", + lp->lnum, lp->free, lp->dirty, c->leb_size - spc, spc, + dark, dead, (int)(spc / UBIFS_MAX_NODE_SZ), lp->flags); + + if (lp->flags & LPROPS_TAKEN) { + if (lp->flags & LPROPS_INDEX) + pr_cont("index, taken"); + else + pr_cont("taken"); + } else { + const char *s; + + if (lp->flags & LPROPS_INDEX) { + switch (lp->flags & LPROPS_CAT_MASK) { + case LPROPS_DIRTY_IDX: + s = "dirty index"; + break; + case LPROPS_FRDI_IDX: + s = "freeable index"; + break; + default: + s = "index"; + } + } else { + switch (lp->flags & LPROPS_CAT_MASK) { + case LPROPS_UNCAT: + s = "not categorized"; + break; + case LPROPS_DIRTY: + s = "dirty"; + break; + case LPROPS_FREE: + s = "free"; + break; + case LPROPS_EMPTY: + s = "empty"; + break; + case LPROPS_FREEABLE: + s = "freeable"; + break; + default: + s = NULL; + break; + } + } + pr_cont("%s", s); + } + + for (rb = rb_first((struct rb_root *)&c->buds); rb; rb = rb_next(rb)) { + bud = rb_entry(rb, struct ubifs_bud, rb); + if (bud->lnum == lp->lnum) { + int head = 0; + for (i = 0; i < c->jhead_cnt; i++) { + /* + * Note, if we are in R/O mode or in the middle + * of mounting/re-mounting, the write-buffers do + * not exist. + */ + if (c->jheads && + lp->lnum == c->jheads[i].wbuf.lnum) { + pr_cont(", jhead %s", dbg_jhead(i)); + head = 1; + } + } + if (!head) + pr_cont(", bud of jhead %s", + dbg_jhead(bud->jhead)); + } + } + if (lp->lnum == c->gc_lnum) + pr_cont(", GC LEB"); + pr_cont(")\n"); +} + +void ubifs_dump_lprops(struct ubifs_info *c) +{ + int lnum, err; + struct ubifs_lprops lp; + struct ubifs_lp_stats lst; + + pr_err("(pid %d) start dumping LEB properties\n", current->pid); + ubifs_get_lp_stats(c, &lst); + ubifs_dump_lstats(&lst); + + for (lnum = c->main_first; lnum < c->leb_cnt; lnum++) { + err = ubifs_read_one_lp(c, lnum, &lp); + if (err) { + ubifs_err(c, "cannot read lprops for LEB %d", lnum); + continue; + } + + ubifs_dump_lprop(c, &lp); + } + pr_err("(pid %d) finish dumping LEB properties\n", current->pid); +} + +void ubifs_dump_lpt_info(struct ubifs_info *c) +{ + int i; + + spin_lock(&dbg_lock); + pr_err("(pid %d) dumping LPT information\n", current->pid); + pr_err("\tlpt_sz: %lld\n", c->lpt_sz); + pr_err("\tpnode_sz: %d\n", c->pnode_sz); + pr_err("\tnnode_sz: %d\n", c->nnode_sz); + pr_err("\tltab_sz: %d\n", c->ltab_sz); + pr_err("\tlsave_sz: %d\n", c->lsave_sz); + pr_err("\tbig_lpt: %u\n", c->big_lpt); + pr_err("\tlpt_hght: %d\n", c->lpt_hght); + pr_err("\tpnode_cnt: %d\n", c->pnode_cnt); + pr_err("\tnnode_cnt: %d\n", c->nnode_cnt); + pr_err("\tdirty_pn_cnt: %d\n", c->dirty_pn_cnt); + pr_err("\tdirty_nn_cnt: %d\n", c->dirty_nn_cnt); + pr_err("\tlsave_cnt: %d\n", c->lsave_cnt); + pr_err("\tspace_bits: %d\n", c->space_bits); + pr_err("\tlpt_lnum_bits: %d\n", c->lpt_lnum_bits); + pr_err("\tlpt_offs_bits: %d\n", c->lpt_offs_bits); + pr_err("\tlpt_spc_bits: %d\n", c->lpt_spc_bits); + pr_err("\tpcnt_bits: %d\n", c->pcnt_bits); + pr_err("\tlnum_bits: %d\n", c->lnum_bits); + pr_err("\tLPT root is at %d:%d\n", c->lpt_lnum, c->lpt_offs); + pr_err("\tLPT head is at %d:%d\n", + c->nhead_lnum, c->nhead_offs); + pr_err("\tLPT ltab is at %d:%d\n", c->ltab_lnum, c->ltab_offs); + if (c->big_lpt) + pr_err("\tLPT lsave is at %d:%d\n", + c->lsave_lnum, c->lsave_offs); + for (i = 0; i < c->lpt_lebs; i++) + pr_err("\tLPT LEB %d free %d dirty %d tgc %d cmt %d\n", + i + c->lpt_first, c->ltab[i].free, c->ltab[i].dirty, + c->ltab[i].tgc, c->ltab[i].cmt); + spin_unlock(&dbg_lock); +} + +void ubifs_dump_leb(const struct ubifs_info *c, int lnum) +{ + struct ubifs_scan_leb *sleb; + struct ubifs_scan_node *snod; + void *buf; + + pr_err("(pid %d) start dumping LEB %d\n", current->pid, lnum); + + buf = __vmalloc(c->leb_size, GFP_NOFS); + if (!buf) { + ubifs_err(c, "cannot allocate memory for dumping LEB %d", lnum); + return; + } + + sleb = ubifs_scan(c, lnum, 0, buf, 0); + if (IS_ERR(sleb)) { + ubifs_err(c, "scan error %d", (int)PTR_ERR(sleb)); + goto out; + } + + pr_err("LEB %d has %d nodes ending at %d\n", lnum, + sleb->nodes_cnt, sleb->endpt); + + list_for_each_entry(snod, &sleb->nodes, list) { + cond_resched(); + pr_err("Dumping node at LEB %d:%d len %d\n", lnum, + snod->offs, snod->len); + ubifs_dump_node(c, snod->node, c->leb_size - snod->offs); + } + + pr_err("(pid %d) finish dumping LEB %d\n", current->pid, lnum); + ubifs_scan_destroy(sleb); + +out: + vfree(buf); +} + +void ubifs_dump_znode(const struct ubifs_info *c, + const struct ubifs_znode *znode) +{ + int n; + const struct ubifs_zbranch *zbr; + char key_buf[DBG_KEY_BUF_LEN]; + + spin_lock(&dbg_lock); + if (znode->parent) + zbr = &znode->parent->zbranch[znode->iip]; + else + zbr = &c->zroot; + + pr_err("znode %p, LEB %d:%d len %d parent %p iip %d level %d child_cnt %d flags %lx\n", + znode, zbr->lnum, zbr->offs, zbr->len, znode->parent, znode->iip, + znode->level, znode->child_cnt, znode->flags); + + if (znode->child_cnt <= 0 || znode->child_cnt > c->fanout) { + spin_unlock(&dbg_lock); + return; + } + + pr_err("zbranches:\n"); + for (n = 0; n < znode->child_cnt; n++) { + zbr = &znode->zbranch[n]; + if (znode->level > 0) + pr_err("\t%d: znode %p LEB %d:%d len %d key %s\n", + n, zbr->znode, zbr->lnum, zbr->offs, zbr->len, + dbg_snprintf_key(c, &zbr->key, key_buf, + DBG_KEY_BUF_LEN)); + else + pr_err("\t%d: LNC %p LEB %d:%d len %d key %s\n", + n, zbr->znode, zbr->lnum, zbr->offs, zbr->len, + dbg_snprintf_key(c, &zbr->key, key_buf, + DBG_KEY_BUF_LEN)); + } + spin_unlock(&dbg_lock); +} + +void ubifs_dump_heap(struct ubifs_info *c, struct ubifs_lpt_heap *heap, int cat) +{ + int i; + + pr_err("(pid %d) start dumping heap cat %d (%d elements)\n", + current->pid, cat, heap->cnt); + for (i = 0; i < heap->cnt; i++) { + struct ubifs_lprops *lprops = heap->arr[i]; + + pr_err("\t%d. LEB %d hpos %d free %d dirty %d flags %d\n", + i, lprops->lnum, lprops->hpos, lprops->free, + lprops->dirty, lprops->flags); + } + pr_err("(pid %d) finish dumping heap\n", current->pid); +} + +void ubifs_dump_pnode(struct ubifs_info *c, struct ubifs_pnode *pnode, + struct ubifs_nnode *parent, int iip) +{ + int i; + + pr_err("(pid %d) dumping pnode:\n", current->pid); + pr_err("\taddress %zx parent %zx cnext %zx\n", + (size_t)pnode, (size_t)parent, (size_t)pnode->cnext); + pr_err("\tflags %lu iip %d level %d num %d\n", + pnode->flags, iip, pnode->level, pnode->num); + for (i = 0; i < UBIFS_LPT_FANOUT; i++) { + struct ubifs_lprops *lp = &pnode->lprops[i]; + + pr_err("\t%d: free %d dirty %d flags %d lnum %d\n", + i, lp->free, lp->dirty, lp->flags, lp->lnum); + } +} + +void ubifs_dump_tnc(struct ubifs_info *c) +{ + struct ubifs_znode *znode; + int level; + + pr_err("\n"); + pr_err("(pid %d) start dumping TNC tree\n", current->pid); + if (c->zroot.znode) { + znode = ubifs_tnc_levelorder_next(c, c->zroot.znode, NULL); + level = znode->level; + pr_err("== Level %d ==\n", level); + while (znode) { + if (level != znode->level) { + level = znode->level; + pr_err("== Level %d ==\n", level); + } + ubifs_dump_znode(c, znode); + znode = ubifs_tnc_levelorder_next(c, c->zroot.znode, znode); + } + } else { + pr_err("empty TNC tree in memory\n"); + } + pr_err("(pid %d) finish dumping TNC tree\n", current->pid); +} + +static int dump_znode(struct ubifs_info *c, struct ubifs_znode *znode, + void *priv) +{ + ubifs_dump_znode(c, znode); + return 0; +} + +/** + * ubifs_dump_index - dump the on-flash index. + * @c: UBIFS file-system description object + * + * This function dumps whole UBIFS indexing B-tree, unlike 'ubifs_dump_tnc()' + * which dumps only in-memory znodes and does not read znodes which from flash. + */ +void ubifs_dump_index(struct ubifs_info *c) +{ + dbg_walk_index(c, NULL, dump_znode, NULL); +} + +/** + * dbg_save_space_info - save information about flash space. + * @c: UBIFS file-system description object + * + * This function saves information about UBIFS free space, dirty space, etc, in + * order to check it later. + */ +void dbg_save_space_info(struct ubifs_info *c) +{ + struct ubifs_debug_info *d = c->dbg; + int freeable_cnt; + + spin_lock(&c->space_lock); + memcpy(&d->saved_lst, &c->lst, sizeof(struct ubifs_lp_stats)); + memcpy(&d->saved_bi, &c->bi, sizeof(struct ubifs_budg_info)); + d->saved_idx_gc_cnt = c->idx_gc_cnt; + + /* + * We use a dirty hack here and zero out @c->freeable_cnt, because it + * affects the free space calculations, and UBIFS might not know about + * all freeable eraseblocks. Indeed, we know about freeable eraseblocks + * only when we read their lprops, and we do this only lazily, upon the + * need. So at any given point of time @c->freeable_cnt might be not + * exactly accurate. + * + * Just one example about the issue we hit when we did not zero + * @c->freeable_cnt. + * 1. The file-system is mounted R/O, c->freeable_cnt is %0. We save the + * amount of free space in @d->saved_free + * 2. We re-mount R/W, which makes UBIFS to read the "lsave" + * information from flash, where we cache LEBs from various + * categories ('ubifs_remount_fs()' -> 'ubifs_lpt_init()' + * -> 'lpt_init_wr()' -> 'read_lsave()' -> 'ubifs_lpt_lookup()' + * -> 'ubifs_get_pnode()' -> 'update_cats()' + * -> 'ubifs_add_to_cat()'). + * 3. Lsave contains a freeable eraseblock, and @c->freeable_cnt + * becomes %1. + * 4. We calculate the amount of free space when the re-mount is + * finished in 'dbg_check_space_info()' and it does not match + * @d->saved_free. + */ + freeable_cnt = c->freeable_cnt; + c->freeable_cnt = 0; + d->saved_free = ubifs_get_free_space_nolock(c); + c->freeable_cnt = freeable_cnt; + spin_unlock(&c->space_lock); +} + +/** + * dbg_check_space_info - check flash space information. + * @c: UBIFS file-system description object + * + * This function compares current flash space information with the information + * which was saved when the 'dbg_save_space_info()' function was called. + * Returns zero if the information has not changed, and %-EINVAL if it has + * changed. + */ +int dbg_check_space_info(struct ubifs_info *c) +{ + struct ubifs_debug_info *d = c->dbg; + struct ubifs_lp_stats lst; + long long free; + int freeable_cnt; + + spin_lock(&c->space_lock); + freeable_cnt = c->freeable_cnt; + c->freeable_cnt = 0; + free = ubifs_get_free_space_nolock(c); + c->freeable_cnt = freeable_cnt; + spin_unlock(&c->space_lock); + + if (free != d->saved_free) { + ubifs_err(c, "free space changed from %lld to %lld", + d->saved_free, free); + goto out; + } + + return 0; + +out: + ubifs_msg(c, "saved lprops statistics dump"); + ubifs_dump_lstats(&d->saved_lst); + ubifs_msg(c, "saved budgeting info dump"); + ubifs_dump_budg(c, &d->saved_bi); + ubifs_msg(c, "saved idx_gc_cnt %d", d->saved_idx_gc_cnt); + ubifs_msg(c, "current lprops statistics dump"); + ubifs_get_lp_stats(c, &lst); + ubifs_dump_lstats(&lst); + ubifs_msg(c, "current budgeting info dump"); + ubifs_dump_budg(c, &c->bi); + dump_stack(); + return -EINVAL; +} + +/** + * dbg_check_synced_i_size - check synchronized inode size. + * @c: UBIFS file-system description object + * @inode: inode to check + * + * If inode is clean, synchronized inode size has to be equivalent to current + * inode size. This function has to be called only for locked inodes (@i_mutex + * has to be locked). Returns %0 if synchronized inode size if correct, and + * %-EINVAL if not. + */ +int dbg_check_synced_i_size(const struct ubifs_info *c, struct inode *inode) +{ + int err = 0; + struct ubifs_inode *ui = ubifs_inode(inode); + + if (!dbg_is_chk_gen(c)) + return 0; + if (!S_ISREG(inode->i_mode)) + return 0; + + mutex_lock(&ui->ui_mutex); + spin_lock(&ui->ui_lock); + if (ui->ui_size != ui->synced_i_size && !ui->dirty) { + ubifs_err(c, "ui_size is %lld, synced_i_size is %lld, but inode is clean", + ui->ui_size, ui->synced_i_size); + ubifs_err(c, "i_ino %llu, i_mode %#x, i_size %lld", inode->i_ino, + inode->i_mode, i_size_read(inode)); + dump_stack(); + err = -EINVAL; + } + spin_unlock(&ui->ui_lock); + mutex_unlock(&ui->ui_mutex); + return err; +} + +/* + * dbg_check_dir - check directory inode size and link count. + * @c: UBIFS file-system description object + * @dir: the directory to calculate size for + * @size: the result is returned here + * + * This function makes sure that directory size and link count are correct. + * Returns zero in case of success and a negative error code in case of + * failure. + * + * Note, it is good idea to make sure the @dir->i_mutex is locked before + * calling this function. + */ +int dbg_check_dir(struct ubifs_info *c, const struct inode *dir) +{ + unsigned int nlink = 2; + union ubifs_key key; + struct ubifs_dent_node *dent, *pdent = NULL; + struct fscrypt_name nm = {0}; + loff_t size = UBIFS_INO_NODE_SZ; + + if (!dbg_is_chk_gen(c)) + return 0; + + if (!S_ISDIR(dir->i_mode)) + return 0; + + lowest_dent_key(c, &key, dir->i_ino); + while (1) { + int err; + + dent = ubifs_tnc_next_ent(c, &key, &nm); + if (IS_ERR(dent)) { + err = PTR_ERR(dent); + if (err == -ENOENT) + break; + kfree(pdent); + return err; + } + + fname_name(&nm) = dent->name; + fname_len(&nm) = le16_to_cpu(dent->nlen); + size += CALC_DENT_SIZE(fname_len(&nm)); + if (dent->type == UBIFS_ITYPE_DIR) + nlink += 1; + kfree(pdent); + pdent = dent; + key_read(c, &dent->key, &key); + } + kfree(pdent); + + if (i_size_read(dir) != size) { + ubifs_err(c, "directory inode %llu has size %llu, but calculated size is %llu", + dir->i_ino, (unsigned long long)i_size_read(dir), + (unsigned long long)size); + ubifs_dump_inode(c, dir); + dump_stack(); + return -EINVAL; + } + if (dir->i_nlink != nlink) { + ubifs_err(c, "directory inode %llu has nlink %u, but calculated nlink is %u", + dir->i_ino, dir->i_nlink, nlink); + ubifs_dump_inode(c, dir); + dump_stack(); + return -EINVAL; + } + + return 0; +} + +/** + * dbg_check_key_order - make sure that colliding keys are properly ordered. + * @c: UBIFS file-system description object + * @zbr1: first zbranch + * @zbr2: following zbranch + * + * In UBIFS indexing B-tree colliding keys has to be sorted in binary order of + * names of the direntries/xentries which are referred by the keys. This + * function reads direntries/xentries referred by @zbr1 and @zbr2 and makes + * sure the name of direntry/xentry referred by @zbr1 is less than + * direntry/xentry referred by @zbr2. Returns zero if this is true, %1 if not, + * and a negative error code in case of failure. + */ +static int dbg_check_key_order(struct ubifs_info *c, struct ubifs_zbranch *zbr1, + struct ubifs_zbranch *zbr2) +{ + int err, nlen1, nlen2, cmp; + struct ubifs_dent_node *dent1, *dent2; + union ubifs_key key; + char key_buf[DBG_KEY_BUF_LEN]; + + ubifs_assert(c, !keys_cmp(c, &zbr1->key, &zbr2->key)); + dent1 = kmalloc(UBIFS_MAX_DENT_NODE_SZ, GFP_NOFS); + if (!dent1) + return -ENOMEM; + dent2 = kmalloc(UBIFS_MAX_DENT_NODE_SZ, GFP_NOFS); + if (!dent2) { + err = -ENOMEM; + goto out_free; + } + + err = ubifs_tnc_read_node(c, zbr1, dent1); + if (err) + goto out_free; + err = ubifs_validate_entry(c, dent1); + if (err) + goto out_free; + + err = ubifs_tnc_read_node(c, zbr2, dent2); + if (err) + goto out_free; + err = ubifs_validate_entry(c, dent2); + if (err) + goto out_free; + + /* Make sure node keys are the same as in zbranch */ + err = 1; + key_read(c, &dent1->key, &key); + if (keys_cmp(c, &zbr1->key, &key)) { + ubifs_err(c, "1st entry at %d:%d has key %s", zbr1->lnum, + zbr1->offs, dbg_snprintf_key(c, &key, key_buf, + DBG_KEY_BUF_LEN)); + ubifs_err(c, "but it should have key %s according to tnc", + dbg_snprintf_key(c, &zbr1->key, key_buf, + DBG_KEY_BUF_LEN)); + ubifs_dump_node(c, dent1, UBIFS_MAX_DENT_NODE_SZ); + goto out_free; + } + + key_read(c, &dent2->key, &key); + if (keys_cmp(c, &zbr2->key, &key)) { + ubifs_err(c, "2nd entry at %d:%d has key %s", zbr1->lnum, + zbr1->offs, dbg_snprintf_key(c, &key, key_buf, + DBG_KEY_BUF_LEN)); + ubifs_err(c, "but it should have key %s according to tnc", + dbg_snprintf_key(c, &zbr2->key, key_buf, + DBG_KEY_BUF_LEN)); + ubifs_dump_node(c, dent2, UBIFS_MAX_DENT_NODE_SZ); + goto out_free; + } + + nlen1 = le16_to_cpu(dent1->nlen); + nlen2 = le16_to_cpu(dent2->nlen); + + cmp = memcmp(dent1->name, dent2->name, min_t(int, nlen1, nlen2)); + if (cmp < 0 || (cmp == 0 && nlen1 < nlen2)) { + err = 0; + goto out_free; + } + if (cmp == 0 && nlen1 == nlen2) + ubifs_err(c, "2 xent/dent nodes with the same name"); + else + ubifs_err(c, "bad order of colliding key %s", + dbg_snprintf_key(c, &key, key_buf, DBG_KEY_BUF_LEN)); + + ubifs_msg(c, "first node at %d:%d\n", zbr1->lnum, zbr1->offs); + ubifs_dump_node(c, dent1, UBIFS_MAX_DENT_NODE_SZ); + ubifs_msg(c, "second node at %d:%d\n", zbr2->lnum, zbr2->offs); + ubifs_dump_node(c, dent2, UBIFS_MAX_DENT_NODE_SZ); + +out_free: + kfree(dent2); + kfree(dent1); + return err; +} + +/** + * dbg_check_znode - check if znode is all right. + * @c: UBIFS file-system description object + * @zbr: zbranch which points to this znode + * + * This function makes sure that znode referred to by @zbr is all right. + * Returns zero if it is, and %-EINVAL if it is not. + */ +static int dbg_check_znode(struct ubifs_info *c, struct ubifs_zbranch *zbr) +{ + struct ubifs_znode *znode = zbr->znode; + struct ubifs_znode *zp = znode->parent; + int n, err, cmp; + + if (znode->child_cnt <= 0 || znode->child_cnt > c->fanout) { + err = 1; + goto out; + } + if (znode->level < 0) { + err = 2; + goto out; + } + if (znode->iip < 0 || znode->iip >= c->fanout) { + err = 3; + goto out; + } + + if (zbr->len == 0) + /* Only dirty zbranch may have no on-flash nodes */ + if (!ubifs_zn_dirty(znode)) { + err = 4; + goto out; + } + + if (ubifs_zn_dirty(znode)) { + /* + * If znode is dirty, its parent has to be dirty as well. The + * order of the operation is important, so we have to have + * memory barriers. + */ + smp_mb(); + if (zp && !ubifs_zn_dirty(zp)) { + /* + * The dirty flag is atomic and is cleared outside the + * TNC mutex, so znode's dirty flag may now have + * been cleared. The child is always cleared before the + * parent, so we just need to check again. + */ + smp_mb(); + if (ubifs_zn_dirty(znode)) { + err = 5; + goto out; + } + } + } + + if (zp) { + const union ubifs_key *min, *max; + + if (znode->level != zp->level - 1) { + err = 6; + goto out; + } + + /* Make sure the 'parent' pointer in our znode is correct */ + err = ubifs_search_zbranch(c, zp, &zbr->key, &n); + if (!err) { + /* This zbranch does not exist in the parent */ + err = 7; + goto out; + } + + if (znode->iip >= zp->child_cnt) { + err = 8; + goto out; + } + + if (znode->iip != n) { + /* This may happen only in case of collisions */ + if (keys_cmp(c, &zp->zbranch[n].key, + &zp->zbranch[znode->iip].key)) { + err = 9; + goto out; + } + n = znode->iip; + } + + /* + * Make sure that the first key in our znode is greater than or + * equal to the key in the pointing zbranch. + */ + min = &zbr->key; + cmp = keys_cmp(c, min, &znode->zbranch[0].key); + if (cmp == 1) { + err = 10; + goto out; + } + + if (n + 1 < zp->child_cnt) { + max = &zp->zbranch[n + 1].key; + + /* + * Make sure the last key in our znode is less or + * equivalent than the key in the zbranch which goes + * after our pointing zbranch. + */ + cmp = keys_cmp(c, max, + &znode->zbranch[znode->child_cnt - 1].key); + if (cmp == -1) { + err = 11; + goto out; + } + } + } else { + /* This may only be root znode */ + if (zbr != &c->zroot) { + err = 12; + goto out; + } + } + + /* + * Make sure that next key is greater or equivalent then the previous + * one. + */ + for (n = 1; n < znode->child_cnt; n++) { + cmp = keys_cmp(c, &znode->zbranch[n - 1].key, + &znode->zbranch[n].key); + if (cmp > 0) { + err = 13; + goto out; + } + if (cmp == 0) { + /* This can only be keys with colliding hash */ + if (!is_hash_key(c, &znode->zbranch[n].key)) { + err = 14; + goto out; + } + + if (znode->level != 0 || c->replaying) + continue; + + /* + * Colliding keys should follow binary order of + * corresponding xentry/dentry names. + */ + err = dbg_check_key_order(c, &znode->zbranch[n - 1], + &znode->zbranch[n]); + if (err < 0) + return err; + if (err) { + err = 15; + goto out; + } + } + } + + for (n = 0; n < znode->child_cnt; n++) { + if (!znode->zbranch[n].znode && + (znode->zbranch[n].lnum == 0 || + znode->zbranch[n].len == 0)) { + err = 16; + goto out; + } + + if (znode->zbranch[n].lnum != 0 && + znode->zbranch[n].len == 0) { + err = 17; + goto out; + } + + if (znode->zbranch[n].lnum == 0 && + znode->zbranch[n].len != 0) { + err = 18; + goto out; + } + + if (znode->zbranch[n].lnum == 0 && + znode->zbranch[n].offs != 0) { + err = 19; + goto out; + } + + if (znode->level != 0 && znode->zbranch[n].znode) + if (znode->zbranch[n].znode->parent != znode) { + err = 20; + goto out; + } + } + + return 0; + +out: + ubifs_err(c, "failed, error %d", err); + ubifs_msg(c, "dump of the znode"); + ubifs_dump_znode(c, znode); + if (zp) { + ubifs_msg(c, "dump of the parent znode"); + ubifs_dump_znode(c, zp); + } + dump_stack(); + return -EINVAL; +} + +/** + * dbg_check_tnc - check TNC tree. + * @c: UBIFS file-system description object + * @extra: do extra checks that are possible at start commit + * + * This function traverses whole TNC tree and checks every znode. Returns zero + * if everything is all right and %-EINVAL if something is wrong with TNC. + */ +int dbg_check_tnc(struct ubifs_info *c, int extra) +{ + struct ubifs_znode *znode; + long clean_cnt = 0, dirty_cnt = 0; + int err, last; + + if (!dbg_is_chk_index(c)) + return 0; + + ubifs_assert(c, mutex_is_locked(&c->tnc_mutex)); + if (!c->zroot.znode) + return 0; + + znode = ubifs_tnc_postorder_first(c->zroot.znode); + while (1) { + struct ubifs_znode *prev; + struct ubifs_zbranch *zbr; + + if (!znode->parent) + zbr = &c->zroot; + else + zbr = &znode->parent->zbranch[znode->iip]; + + err = dbg_check_znode(c, zbr); + if (err) + return err; + + if (extra) { + if (ubifs_zn_dirty(znode)) + dirty_cnt += 1; + else + clean_cnt += 1; + } + + prev = znode; + znode = ubifs_tnc_postorder_next(c, znode); + if (!znode) + break; + + /* + * If the last key of this znode is equivalent to the first key + * of the next znode (collision), then check order of the keys. + */ + last = prev->child_cnt - 1; + if (prev->level == 0 && znode->level == 0 && !c->replaying && + !keys_cmp(c, &prev->zbranch[last].key, + &znode->zbranch[0].key)) { + err = dbg_check_key_order(c, &prev->zbranch[last], + &znode->zbranch[0]); + if (err < 0) + return err; + if (err) { + ubifs_msg(c, "first znode"); + ubifs_dump_znode(c, prev); + ubifs_msg(c, "second znode"); + ubifs_dump_znode(c, znode); + return -EINVAL; + } + } + } + + if (extra) { + if (clean_cnt != atomic_long_read(&c->clean_zn_cnt)) { + ubifs_err(c, "incorrect clean_zn_cnt %ld, calculated %ld", + atomic_long_read(&c->clean_zn_cnt), + clean_cnt); + return -EINVAL; + } + if (dirty_cnt != atomic_long_read(&c->dirty_zn_cnt)) { + ubifs_err(c, "incorrect dirty_zn_cnt %ld, calculated %ld", + atomic_long_read(&c->dirty_zn_cnt), + dirty_cnt); + return -EINVAL; + } + } + + return 0; +} + +/** + * dbg_walk_index - walk the on-flash index. + * @c: UBIFS file-system description object + * @leaf_cb: called for each leaf node + * @znode_cb: called for each indexing node + * @priv: private data which is passed to callbacks + * + * This function walks the UBIFS index and calls the @leaf_cb for each leaf + * node and @znode_cb for each indexing node. Returns zero in case of success + * and a negative error code in case of failure. + * + * It would be better if this function removed every znode it pulled to into + * the TNC, so that the behavior more closely matched the non-debugging + * behavior. + */ +int dbg_walk_index(struct ubifs_info *c, dbg_leaf_callback leaf_cb, + dbg_znode_callback znode_cb, void *priv) +{ + int err; + struct ubifs_zbranch *zbr; + struct ubifs_znode *znode, *child; + + mutex_lock(&c->tnc_mutex); + /* If the root indexing node is not in TNC - pull it */ + if (!c->zroot.znode) { + c->zroot.znode = ubifs_load_znode(c, &c->zroot, NULL, 0); + if (IS_ERR(c->zroot.znode)) { + err = PTR_ERR(c->zroot.znode); + c->zroot.znode = NULL; + goto out_unlock; + } + } + + /* + * We are going to traverse the indexing tree in the postorder manner. + * Go down and find the leftmost indexing node where we are going to + * start from. + */ + znode = c->zroot.znode; + while (znode->level > 0) { + zbr = &znode->zbranch[0]; + child = zbr->znode; + if (!child) { + child = ubifs_load_znode(c, zbr, znode, 0); + if (IS_ERR(child)) { + err = PTR_ERR(child); + goto out_unlock; + } + } + + znode = child; + } + + /* Iterate over all indexing nodes */ + while (1) { + int idx; + + cond_resched(); + + if (znode_cb) { + err = znode_cb(c, znode, priv); + if (err) { + ubifs_err(c, "znode checking function returned error %d", + err); + ubifs_dump_znode(c, znode); + goto out_dump; + } + } + if (leaf_cb && znode->level == 0) { + for (idx = 0; idx < znode->child_cnt; idx++) { + zbr = &znode->zbranch[idx]; + err = leaf_cb(c, zbr, priv); + if (err) { + ubifs_err(c, "leaf checking function returned error %d, for leaf at LEB %d:%d", + err, zbr->lnum, zbr->offs); + goto out_dump; + } + } + } + + if (!znode->parent) + break; + + idx = znode->iip + 1; + znode = znode->parent; + if (idx < znode->child_cnt) { + /* Switch to the next index in the parent */ + zbr = &znode->zbranch[idx]; + child = zbr->znode; + if (!child) { + child = ubifs_load_znode(c, zbr, znode, idx); + if (IS_ERR(child)) { + err = PTR_ERR(child); + goto out_unlock; + } + zbr->znode = child; + } + znode = child; + } else + /* + * This is the last child, switch to the parent and + * continue. + */ + continue; + + /* Go to the lowest leftmost znode in the new sub-tree */ + while (znode->level > 0) { + zbr = &znode->zbranch[0]; + child = zbr->znode; + if (!child) { + child = ubifs_load_znode(c, zbr, znode, 0); + if (IS_ERR(child)) { + err = PTR_ERR(child); + goto out_unlock; + } + zbr->znode = child; + } + znode = child; + } + } + + mutex_unlock(&c->tnc_mutex); + return 0; + +out_dump: + if (znode->parent) + zbr = &znode->parent->zbranch[znode->iip]; + else + zbr = &c->zroot; + ubifs_msg(c, "dump of znode at LEB %d:%d", zbr->lnum, zbr->offs); + ubifs_dump_znode(c, znode); +out_unlock: + mutex_unlock(&c->tnc_mutex); + return err; +} + +/** + * add_size - add znode size to partially calculated index size. + * @c: UBIFS file-system description object + * @znode: znode to add size for + * @priv: partially calculated index size + * + * This is a helper function for 'dbg_check_idx_size()' which is called for + * every indexing node and adds its size to the 'long long' variable pointed to + * by @priv. + */ +static int add_size(struct ubifs_info *c, struct ubifs_znode *znode, void *priv) +{ + long long *idx_size = priv; + int add; + + add = ubifs_idx_node_sz(c, znode->child_cnt); + add = ALIGN(add, 8); + *idx_size += add; + return 0; +} + +/** + * dbg_check_idx_size - check index size. + * @c: UBIFS file-system description object + * @idx_size: size to check + * + * This function walks the UBIFS index, calculates its size and checks that the + * size is equivalent to @idx_size. Returns zero in case of success and a + * negative error code in case of failure. + */ +int dbg_check_idx_size(struct ubifs_info *c, long long idx_size) +{ + int err; + long long calc = 0; + + if (!dbg_is_chk_index(c)) + return 0; + + err = dbg_walk_index(c, NULL, add_size, &calc); + if (err) { + ubifs_err(c, "error %d while walking the index", err); + goto out_err; + } + + if (calc != idx_size) { + ubifs_err(c, "index size check failed: calculated size is %lld, should be %lld", + calc, idx_size); + dump_stack(); + err = -EINVAL; + goto out_err; + } + + return 0; + +out_err: + ubifs_destroy_tnc_tree(c); + return err; +} + +/** + * struct fsck_inode - information about an inode used when checking the file-system. + * @rb: link in the RB-tree of inodes + * @inum: inode number + * @mode: inode type, permissions, etc + * @nlink: inode link count + * @xattr_cnt: count of extended attributes + * @references: how many directory/xattr entries refer this inode (calculated + * while walking the index) + * @calc_cnt: for directory inode count of child directories + * @size: inode size (read from on-flash inode) + * @xattr_sz: summary size of all extended attributes (read from on-flash + * inode) + * @calc_sz: for directories calculated directory size + * @calc_xcnt: count of extended attributes + * @calc_xsz: calculated summary size of all extended attributes + * @xattr_nms: sum of lengths of all extended attribute names belonging to this + * inode (read from on-flash inode) + * @calc_xnms: calculated sum of lengths of all extended attribute names + */ +struct fsck_inode { + struct rb_node rb; + ino_t inum; + umode_t mode; + unsigned int nlink; + unsigned int xattr_cnt; + int references; + int calc_cnt; + long long size; + unsigned int xattr_sz; + long long calc_sz; + long long calc_xcnt; + long long calc_xsz; + unsigned int xattr_nms; + long long calc_xnms; +}; + +/** + * struct fsck_data - private FS checking information. + * @inodes: RB-tree of all inodes (contains @struct fsck_inode objects) + */ +struct fsck_data { + struct rb_root inodes; +}; + +/** + * add_inode - add inode information to RB-tree of inodes. + * @c: UBIFS file-system description object + * @fsckd: FS checking information + * @ino: raw UBIFS inode to add + * + * This is a helper function for 'check_leaf()' which adds information about + * inode @ino to the RB-tree of inodes. Returns inode information pointer in + * case of success and a negative error code in case of failure. + */ +static struct fsck_inode *add_inode(struct ubifs_info *c, + struct fsck_data *fsckd, + struct ubifs_ino_node *ino) +{ + struct rb_node **p, *parent = NULL; + struct fsck_inode *fscki; + ino_t inum = key_inum_flash(c, &ino->key); + struct inode *inode; + struct ubifs_inode *ui; + + p = &fsckd->inodes.rb_node; + while (*p) { + parent = *p; + fscki = rb_entry(parent, struct fsck_inode, rb); + if (inum < fscki->inum) + p = &(*p)->rb_left; + else if (inum > fscki->inum) + p = &(*p)->rb_right; + else + return fscki; + } + + if (inum > c->highest_inum) { + ubifs_err(c, "too high inode number, max. is %lu", + (unsigned long)c->highest_inum); + return ERR_PTR(-EINVAL); + } + + fscki = kzalloc_obj(struct fsck_inode, GFP_NOFS); + if (!fscki) + return ERR_PTR(-ENOMEM); + + inode = ilookup(c->vfs_sb, inum); + + fscki->inum = inum; + /* + * If the inode is present in the VFS inode cache, use it instead of + * the on-flash inode which might be out-of-date. E.g., the size might + * be out-of-date. If we do not do this, the following may happen, for + * example: + * 1. A power cut happens + * 2. We mount the file-system R/O, the replay process fixes up the + * inode size in the VFS cache, but on on-flash. + * 3. 'check_leaf()' fails because it hits a data node beyond inode + * size. + */ + if (!inode) { + fscki->nlink = le32_to_cpu(ino->nlink); + fscki->size = le64_to_cpu(ino->size); + fscki->xattr_cnt = le32_to_cpu(ino->xattr_cnt); + fscki->xattr_sz = le32_to_cpu(ino->xattr_size); + fscki->xattr_nms = le32_to_cpu(ino->xattr_names); + fscki->mode = le32_to_cpu(ino->mode); + } else { + ui = ubifs_inode(inode); + fscki->nlink = inode->i_nlink; + fscki->size = inode->i_size; + fscki->xattr_cnt = ui->xattr_cnt; + fscki->xattr_sz = ui->xattr_size; + fscki->xattr_nms = ui->xattr_names; + fscki->mode = inode->i_mode; + iput(inode); + } + + if (S_ISDIR(fscki->mode)) { + fscki->calc_sz = UBIFS_INO_NODE_SZ; + fscki->calc_cnt = 2; + } + + rb_link_node(&fscki->rb, parent, p); + rb_insert_color(&fscki->rb, &fsckd->inodes); + + return fscki; +} + +/** + * search_inode - search inode in the RB-tree of inodes. + * @fsckd: FS checking information + * @inum: inode number to search + * + * This is a helper function for 'check_leaf()' which searches inode @inum in + * the RB-tree of inodes and returns an inode information pointer or %NULL if + * the inode was not found. + */ +static struct fsck_inode *search_inode(struct fsck_data *fsckd, ino_t inum) +{ + struct rb_node *p; + struct fsck_inode *fscki; + + p = fsckd->inodes.rb_node; + while (p) { + fscki = rb_entry(p, struct fsck_inode, rb); + if (inum < fscki->inum) + p = p->rb_left; + else if (inum > fscki->inum) + p = p->rb_right; + else + return fscki; + } + return NULL; +} + +/** + * read_add_inode - read inode node and add it to RB-tree of inodes. + * @c: UBIFS file-system description object + * @fsckd: FS checking information + * @inum: inode number to read + * + * This is a helper function for 'check_leaf()' which finds inode node @inum in + * the index, reads it, and adds it to the RB-tree of inodes. Returns inode + * information pointer in case of success and a negative error code in case of + * failure. + */ +static struct fsck_inode *read_add_inode(struct ubifs_info *c, + struct fsck_data *fsckd, ino_t inum) +{ + int n, err; + union ubifs_key key; + struct ubifs_znode *znode; + struct ubifs_zbranch *zbr; + struct ubifs_ino_node *ino; + struct fsck_inode *fscki; + + fscki = search_inode(fsckd, inum); + if (fscki) + return fscki; + + ino_key_init(c, &key, inum); + err = ubifs_lookup_level0(c, &key, &znode, &n); + if (!err) { + ubifs_err(c, "inode %lu not found in index", (unsigned long)inum); + return ERR_PTR(-ENOENT); + } else if (err < 0) { + ubifs_err(c, "error %d while looking up inode %lu", + err, (unsigned long)inum); + return ERR_PTR(err); + } + + zbr = &znode->zbranch[n]; + if (zbr->len < UBIFS_INO_NODE_SZ) { + ubifs_err(c, "bad node %lu node length %d", + (unsigned long)inum, zbr->len); + return ERR_PTR(-EINVAL); + } + + ino = kmalloc(zbr->len, GFP_NOFS); + if (!ino) + return ERR_PTR(-ENOMEM); + + err = ubifs_tnc_read_node(c, zbr, ino); + if (err) { + ubifs_err(c, "cannot read inode node at LEB %d:%d, error %d", + zbr->lnum, zbr->offs, err); + kfree(ino); + return ERR_PTR(err); + } + + fscki = add_inode(c, fsckd, ino); + kfree(ino); + if (IS_ERR(fscki)) { + ubifs_err(c, "error %ld while adding inode %lu node", + PTR_ERR(fscki), (unsigned long)inum); + return fscki; + } + + return fscki; +} + +/** + * check_leaf - check leaf node. + * @c: UBIFS file-system description object + * @zbr: zbranch of the leaf node to check + * @priv: FS checking information + * + * This is a helper function for 'dbg_check_filesystem()' which is called for + * every single leaf node while walking the indexing tree. It checks that the + * leaf node referred from the indexing tree exists, has correct CRC, and does + * some other basic validation. This function is also responsible for building + * an RB-tree of inodes - it adds all inodes into the RB-tree. It also + * calculates reference count, size, etc for each inode in order to later + * compare them to the information stored inside the inodes and detect possible + * inconsistencies. Returns zero in case of success and a negative error code + * in case of failure. + */ +static int check_leaf(struct ubifs_info *c, struct ubifs_zbranch *zbr, + void *priv) +{ + ino_t inum; + void *node; + struct ubifs_ch *ch; + int err, type = key_type(c, &zbr->key); + struct fsck_inode *fscki; + + if (zbr->len < UBIFS_CH_SZ) { + ubifs_err(c, "bad leaf length %d (LEB %d:%d)", + zbr->len, zbr->lnum, zbr->offs); + return -EINVAL; + } + + node = kmalloc(zbr->len, GFP_NOFS); + if (!node) + return -ENOMEM; + + err = ubifs_tnc_read_node(c, zbr, node); + if (err) { + ubifs_err(c, "cannot read leaf node at LEB %d:%d, error %d", + zbr->lnum, zbr->offs, err); + goto out_free; + } + + /* If this is an inode node, add it to RB-tree of inodes */ + if (type == UBIFS_INO_KEY) { + fscki = add_inode(c, priv, node); + if (IS_ERR(fscki)) { + err = PTR_ERR(fscki); + ubifs_err(c, "error %d while adding inode node", err); + goto out_dump; + } + goto out; + } + + if (type != UBIFS_DENT_KEY && type != UBIFS_XENT_KEY && + type != UBIFS_DATA_KEY) { + ubifs_err(c, "unexpected node type %d at LEB %d:%d", + type, zbr->lnum, zbr->offs); + err = -EINVAL; + goto out_free; + } + + ch = node; + if (le64_to_cpu(ch->sqnum) > c->max_sqnum) { + ubifs_err(c, "too high sequence number, max. is %llu", + c->max_sqnum); + err = -EINVAL; + goto out_dump; + } + + if (type == UBIFS_DATA_KEY) { + long long blk_offs; + struct ubifs_data_node *dn = node; + + ubifs_assert(c, zbr->len >= UBIFS_DATA_NODE_SZ); + + /* + * Search the inode node this data node belongs to and insert + * it to the RB-tree of inodes. + */ + inum = key_inum_flash(c, &dn->key); + fscki = read_add_inode(c, priv, inum); + if (IS_ERR(fscki)) { + err = PTR_ERR(fscki); + ubifs_err(c, "error %d while processing data node and trying to find inode node %lu", + err, (unsigned long)inum); + goto out_dump; + } + + /* Make sure the data node is within inode size */ + blk_offs = key_block_flash(c, &dn->key); + blk_offs <<= UBIFS_BLOCK_SHIFT; + blk_offs += le32_to_cpu(dn->size); + if (blk_offs > fscki->size) { + ubifs_err(c, "data node at LEB %d:%d is not within inode size %lld", + zbr->lnum, zbr->offs, fscki->size); + err = -EINVAL; + goto out_dump; + } + } else { + int nlen; + struct ubifs_dent_node *dent = node; + struct fsck_inode *fscki1; + + ubifs_assert(c, zbr->len >= UBIFS_DENT_NODE_SZ); + + err = ubifs_validate_entry(c, dent); + if (err) + goto out_dump; + + /* + * Search the inode node this entry refers to and the parent + * inode node and insert them to the RB-tree of inodes. + */ + inum = le64_to_cpu(dent->inum); + fscki = read_add_inode(c, priv, inum); + if (IS_ERR(fscki)) { + err = PTR_ERR(fscki); + ubifs_err(c, "error %d while processing entry node and trying to find inode node %lu", + err, (unsigned long)inum); + goto out_dump; + } + + /* Count how many direntries or xentries refers this inode */ + fscki->references += 1; + + inum = key_inum_flash(c, &dent->key); + fscki1 = read_add_inode(c, priv, inum); + if (IS_ERR(fscki1)) { + err = PTR_ERR(fscki1); + ubifs_err(c, "error %d while processing entry node and trying to find parent inode node %lu", + err, (unsigned long)inum); + goto out_dump; + } + + nlen = le16_to_cpu(dent->nlen); + if (type == UBIFS_XENT_KEY) { + fscki1->calc_xcnt += 1; + fscki1->calc_xsz += CALC_DENT_SIZE(nlen); + fscki1->calc_xsz += CALC_XATTR_BYTES(fscki->size); + fscki1->calc_xnms += nlen; + } else { + fscki1->calc_sz += CALC_DENT_SIZE(nlen); + if (dent->type == UBIFS_ITYPE_DIR) + fscki1->calc_cnt += 1; + } + } + +out: + kfree(node); + return 0; + +out_dump: + ubifs_msg(c, "dump of node at LEB %d:%d", zbr->lnum, zbr->offs); + ubifs_dump_node(c, node, zbr->len); +out_free: + kfree(node); + return err; +} + +/** + * free_inodes - free RB-tree of inodes. + * @fsckd: FS checking information + */ +static void free_inodes(struct fsck_data *fsckd) +{ + struct fsck_inode *fscki, *n; + + rbtree_postorder_for_each_entry_safe(fscki, n, &fsckd->inodes, rb) + kfree(fscki); +} + +/** + * check_inodes - checks all inodes. + * @c: UBIFS file-system description object + * @fsckd: FS checking information + * + * This is a helper function for 'dbg_check_filesystem()' which walks the + * RB-tree of inodes after the index scan has been finished, and checks that + * inode nlink, size, etc are correct. Returns zero if inodes are fine, + * %-EINVAL if not, and a negative error code in case of failure. + */ +static int check_inodes(struct ubifs_info *c, struct fsck_data *fsckd) +{ + int n, err; + union ubifs_key key; + struct ubifs_znode *znode; + struct ubifs_zbranch *zbr; + struct ubifs_ino_node *ino; + struct fsck_inode *fscki; + struct rb_node *this = rb_first(&fsckd->inodes); + + while (this) { + fscki = rb_entry(this, struct fsck_inode, rb); + this = rb_next(this); + + if (S_ISDIR(fscki->mode)) { + /* + * Directories have to have exactly one reference (they + * cannot have hardlinks), although root inode is an + * exception. + */ + if (fscki->inum != UBIFS_ROOT_INO && + fscki->references != 1) { + ubifs_err(c, "directory inode %lu has %d direntries which refer it, but should be 1", + (unsigned long)fscki->inum, + fscki->references); + goto out_dump; + } + if (fscki->inum == UBIFS_ROOT_INO && + fscki->references != 0) { + ubifs_err(c, "root inode %lu has non-zero (%d) direntries which refer it", + (unsigned long)fscki->inum, + fscki->references); + goto out_dump; + } + if (fscki->calc_sz != fscki->size) { + ubifs_err(c, "directory inode %lu size is %lld, but calculated size is %lld", + (unsigned long)fscki->inum, + fscki->size, fscki->calc_sz); + goto out_dump; + } + if (fscki->calc_cnt != fscki->nlink) { + ubifs_err(c, "directory inode %lu nlink is %d, but calculated nlink is %d", + (unsigned long)fscki->inum, + fscki->nlink, fscki->calc_cnt); + goto out_dump; + } + } else { + if (fscki->references != fscki->nlink) { + ubifs_err(c, "inode %lu nlink is %d, but calculated nlink is %d", + (unsigned long)fscki->inum, + fscki->nlink, fscki->references); + goto out_dump; + } + } + if (fscki->xattr_sz != fscki->calc_xsz) { + ubifs_err(c, "inode %lu has xattr size %u, but calculated size is %lld", + (unsigned long)fscki->inum, fscki->xattr_sz, + fscki->calc_xsz); + goto out_dump; + } + if (fscki->xattr_cnt != fscki->calc_xcnt) { + ubifs_err(c, "inode %lu has %u xattrs, but calculated count is %lld", + (unsigned long)fscki->inum, + fscki->xattr_cnt, fscki->calc_xcnt); + goto out_dump; + } + if (fscki->xattr_nms != fscki->calc_xnms) { + ubifs_err(c, "inode %lu has xattr names' size %u, but calculated names' size is %lld", + (unsigned long)fscki->inum, fscki->xattr_nms, + fscki->calc_xnms); + goto out_dump; + } + } + + return 0; + +out_dump: + /* Read the bad inode and dump it */ + ino_key_init(c, &key, fscki->inum); + err = ubifs_lookup_level0(c, &key, &znode, &n); + if (!err) { + ubifs_err(c, "inode %lu not found in index", + (unsigned long)fscki->inum); + return -ENOENT; + } else if (err < 0) { + ubifs_err(c, "error %d while looking up inode %lu", + err, (unsigned long)fscki->inum); + return err; + } + + zbr = &znode->zbranch[n]; + ino = kmalloc(zbr->len, GFP_NOFS); + if (!ino) + return -ENOMEM; + + err = ubifs_tnc_read_node(c, zbr, ino); + if (err) { + ubifs_err(c, "cannot read inode node at LEB %d:%d, error %d", + zbr->lnum, zbr->offs, err); + kfree(ino); + return err; + } + + ubifs_msg(c, "dump of the inode %lu sitting in LEB %d:%d", + (unsigned long)fscki->inum, zbr->lnum, zbr->offs); + ubifs_dump_node(c, ino, zbr->len); + kfree(ino); + return -EINVAL; +} + +/** + * dbg_check_filesystem - check the file-system. + * @c: UBIFS file-system description object + * + * This function checks the file system, namely: + * o makes sure that all leaf nodes exist and their CRCs are correct; + * o makes sure inode nlink, size, xattr size/count are correct (for all + * inodes). + * + * The function reads whole indexing tree and all nodes, so it is pretty + * heavy-weight. Returns zero if the file-system is consistent, %-EINVAL if + * not, and a negative error code in case of failure. + */ +int dbg_check_filesystem(struct ubifs_info *c) +{ + int err; + struct fsck_data fsckd; + + if (!dbg_is_chk_fs(c)) + return 0; + + fsckd.inodes = RB_ROOT; + err = dbg_walk_index(c, check_leaf, NULL, &fsckd); + if (err) + goto out_free; + + err = check_inodes(c, &fsckd); + if (err) + goto out_free; + + free_inodes(&fsckd); + return 0; + +out_free: + ubifs_err(c, "file-system check failed with error %d", err); + dump_stack(); + free_inodes(&fsckd); + return err; +} + +/** + * dbg_check_data_nodes_order - check that list of data nodes is sorted. + * @c: UBIFS file-system description object + * @head: the list of nodes ('struct ubifs_scan_node' objects) + * + * This function returns zero if the list of data nodes is sorted correctly, + * and %-EINVAL if not. + */ +int dbg_check_data_nodes_order(struct ubifs_info *c, struct list_head *head) +{ + struct list_head *cur; + struct ubifs_scan_node *sa, *sb; + + if (!dbg_is_chk_gen(c)) + return 0; + + for (cur = head->next; cur->next != head; cur = cur->next) { + ino_t inuma, inumb; + uint32_t blka, blkb; + + cond_resched(); + sa = container_of(cur, struct ubifs_scan_node, list); + sb = container_of(cur->next, struct ubifs_scan_node, list); + + if (sa->type != UBIFS_DATA_NODE) { + ubifs_err(c, "bad node type %d", sa->type); + ubifs_dump_node(c, sa->node, c->leb_size - sa->offs); + return -EINVAL; + } + if (sb->type != UBIFS_DATA_NODE) { + ubifs_err(c, "bad node type %d", sb->type); + ubifs_dump_node(c, sb->node, c->leb_size - sb->offs); + return -EINVAL; + } + + inuma = key_inum(c, &sa->key); + inumb = key_inum(c, &sb->key); + + if (inuma < inumb) + continue; + if (inuma > inumb) { + ubifs_err(c, "larger inum %lu goes before inum %lu", + (unsigned long)inuma, (unsigned long)inumb); + goto error_dump; + } + + blka = key_block(c, &sa->key); + blkb = key_block(c, &sb->key); + + if (blka > blkb) { + ubifs_err(c, "larger block %u goes before %u", blka, blkb); + goto error_dump; + } + if (blka == blkb) { + ubifs_err(c, "two data nodes for the same block"); + goto error_dump; + } + } + + return 0; + +error_dump: + ubifs_dump_node(c, sa->node, c->leb_size - sa->offs); + ubifs_dump_node(c, sb->node, c->leb_size - sb->offs); + return -EINVAL; +} + +/** + * dbg_check_nondata_nodes_order - check that list of data nodes is sorted. + * @c: UBIFS file-system description object + * @head: the list of nodes ('struct ubifs_scan_node' objects) + * + * This function returns zero if the list of non-data nodes is sorted correctly, + * and %-EINVAL if not. + */ +int dbg_check_nondata_nodes_order(struct ubifs_info *c, struct list_head *head) +{ + struct list_head *cur; + struct ubifs_scan_node *sa, *sb; + + if (!dbg_is_chk_gen(c)) + return 0; + + for (cur = head->next; cur->next != head; cur = cur->next) { + ino_t inuma, inumb; + uint32_t hasha, hashb; + + cond_resched(); + sa = container_of(cur, struct ubifs_scan_node, list); + sb = container_of(cur->next, struct ubifs_scan_node, list); + + if (sa->type != UBIFS_INO_NODE && sa->type != UBIFS_DENT_NODE && + sa->type != UBIFS_XENT_NODE) { + ubifs_err(c, "bad node type %d", sa->type); + ubifs_dump_node(c, sa->node, c->leb_size - sa->offs); + return -EINVAL; + } + if (sb->type != UBIFS_INO_NODE && sb->type != UBIFS_DENT_NODE && + sb->type != UBIFS_XENT_NODE) { + ubifs_err(c, "bad node type %d", sb->type); + ubifs_dump_node(c, sb->node, c->leb_size - sb->offs); + return -EINVAL; + } + + if (sa->type != UBIFS_INO_NODE && sb->type == UBIFS_INO_NODE) { + ubifs_err(c, "non-inode node goes before inode node"); + goto error_dump; + } + + if (sa->type == UBIFS_INO_NODE && sb->type != UBIFS_INO_NODE) + continue; + + if (sa->type == UBIFS_INO_NODE && sb->type == UBIFS_INO_NODE) { + /* Inode nodes are sorted in descending size order */ + if (sa->len < sb->len) { + ubifs_err(c, "smaller inode node goes first"); + goto error_dump; + } + continue; + } + + /* + * This is either a dentry or xentry, which should be sorted in + * ascending (parent ino, hash) order. + */ + inuma = key_inum(c, &sa->key); + inumb = key_inum(c, &sb->key); + + if (inuma < inumb) + continue; + if (inuma > inumb) { + ubifs_err(c, "larger inum %lu goes before inum %lu", + (unsigned long)inuma, (unsigned long)inumb); + goto error_dump; + } + + hasha = key_block(c, &sa->key); + hashb = key_block(c, &sb->key); + + if (hasha > hashb) { + ubifs_err(c, "larger hash %u goes before %u", + hasha, hashb); + goto error_dump; + } + } + + return 0; + +error_dump: + ubifs_msg(c, "dumping first node"); + ubifs_dump_node(c, sa->node, c->leb_size - sa->offs); + ubifs_msg(c, "dumping second node"); + ubifs_dump_node(c, sb->node, c->leb_size - sb->offs); + return -EINVAL; +} + +static inline int chance(unsigned int n, unsigned int out_of) +{ + return !!(get_random_u32_below(out_of) + 1 <= n); + +} + +static int power_cut_emulated(struct ubifs_info *c, int lnum, int write) +{ + struct ubifs_debug_info *d = c->dbg; + + ubifs_assert(c, dbg_is_tst_rcvry(c)); + + if (!d->pc_cnt) { + /* First call - decide delay to the power cut */ + if (chance(1, 2)) { + unsigned long delay; + + if (chance(1, 2)) { + d->pc_delay = 1; + /* Fail within 1 minute */ + delay = get_random_u32_below(60000); + d->pc_timeout = jiffies; + d->pc_timeout += msecs_to_jiffies(delay); + ubifs_warn(c, "failing after %lums", delay); + } else { + d->pc_delay = 2; + delay = get_random_u32_below(10000); + /* Fail within 10000 operations */ + d->pc_cnt_max = delay; + ubifs_warn(c, "failing after %lu calls", delay); + } + } + + d->pc_cnt += 1; + } + + /* Determine if failure delay has expired */ + if (d->pc_delay == 1 && time_before(jiffies, d->pc_timeout)) + return 0; + if (d->pc_delay == 2 && d->pc_cnt++ < d->pc_cnt_max) + return 0; + + if (lnum == UBIFS_SB_LNUM) { + if (write && chance(1, 2)) + return 0; + if (chance(19, 20)) + return 0; + ubifs_warn(c, "failing in super block LEB %d", lnum); + } else if (lnum == UBIFS_MST_LNUM || lnum == UBIFS_MST_LNUM + 1) { + if (chance(19, 20)) + return 0; + ubifs_warn(c, "failing in master LEB %d", lnum); + } else if (lnum >= UBIFS_LOG_LNUM && lnum <= c->log_last) { + if (write && chance(99, 100)) + return 0; + if (chance(399, 400)) + return 0; + ubifs_warn(c, "failing in log LEB %d", lnum); + } else if (lnum >= c->lpt_first && lnum <= c->lpt_last) { + if (write && chance(7, 8)) + return 0; + if (chance(19, 20)) + return 0; + ubifs_warn(c, "failing in LPT LEB %d", lnum); + } else if (lnum >= c->orph_first && lnum <= c->orph_last) { + if (write && chance(1, 2)) + return 0; + if (chance(9, 10)) + return 0; + ubifs_warn(c, "failing in orphan LEB %d", lnum); + } else if (lnum == c->ihead_lnum) { + if (chance(99, 100)) + return 0; + ubifs_warn(c, "failing in index head LEB %d", lnum); + } else if (c->jheads && lnum == c->jheads[GCHD].wbuf.lnum) { + if (chance(9, 10)) + return 0; + ubifs_warn(c, "failing in GC head LEB %d", lnum); + } else if (write && !RB_EMPTY_ROOT(&c->buds) && + !ubifs_search_bud(c, lnum)) { + if (chance(19, 20)) + return 0; + ubifs_warn(c, "failing in non-bud LEB %d", lnum); + } else if (c->cmt_state == COMMIT_RUNNING_BACKGROUND || + c->cmt_state == COMMIT_RUNNING_REQUIRED) { + if (chance(999, 1000)) + return 0; + ubifs_warn(c, "failing in bud LEB %d commit running", lnum); + } else { + if (chance(9999, 10000)) + return 0; + ubifs_warn(c, "failing in bud LEB %d commit not running", lnum); + } + + d->pc_happened = 1; + ubifs_warn(c, "========== Power cut emulated =========="); + dump_stack(); + return 1; +} + +static int corrupt_data(const struct ubifs_info *c, const void *buf, + unsigned int len) +{ + unsigned int from, to, ffs = chance(1, 2); + unsigned char *p = (void *)buf; + + from = get_random_u32_below(len); + /* Corruption span max to end of write unit */ + to = min(len, ALIGN(from + 1, c->max_write_size)); + + ubifs_warn(c, "filled bytes %u-%u with %s", from, to - 1, + ffs ? "0xFFs" : "random data"); + + if (ffs) + memset(p + from, 0xFF, to - from); + else + get_random_bytes(p + from, to - from); + + return to; +} + +int dbg_leb_write(struct ubifs_info *c, int lnum, const void *buf, + int offs, int len) +{ + int err, failing; + + if (dbg_is_power_cut(c)) + return -EROFS; + + failing = power_cut_emulated(c, lnum, 1); + if (failing) { + len = corrupt_data(c, buf, len); + ubifs_warn(c, "actually write %d bytes to LEB %d:%d (the buffer was corrupted)", + len, lnum, offs); + } + err = ubi_leb_write(c->ubi, lnum, buf, offs, len); + if (err) + return err; + if (failing) + return -EROFS; + return 0; +} + +int dbg_leb_change(struct ubifs_info *c, int lnum, const void *buf, + int len) +{ + int err; + + if (dbg_is_power_cut(c)) + return -EROFS; + if (power_cut_emulated(c, lnum, 1)) + return -EROFS; + err = ubi_leb_change(c->ubi, lnum, buf, len); + if (err) + return err; + if (power_cut_emulated(c, lnum, 1)) + return -EROFS; + return 0; +} + +int dbg_leb_unmap(struct ubifs_info *c, int lnum) +{ + int err; + + if (dbg_is_power_cut(c)) + return -EROFS; + if (power_cut_emulated(c, lnum, 0)) + return -EROFS; + err = ubi_leb_unmap(c->ubi, lnum); + if (err) + return err; + if (power_cut_emulated(c, lnum, 0)) + return -EROFS; + return 0; +} + +int dbg_leb_map(struct ubifs_info *c, int lnum) +{ + int err; + + if (dbg_is_power_cut(c)) + return -EROFS; + if (power_cut_emulated(c, lnum, 0)) + return -EROFS; + err = ubi_leb_map(c->ubi, lnum); + if (err) + return err; + if (power_cut_emulated(c, lnum, 0)) + return -EROFS; + return 0; +} + +/* + * Root directory for UBIFS stuff in debugfs. Contains sub-directories which + * contain the stuff specific to particular file-system mounts. + */ +static struct dentry *dfs_rootdir; + +static int dfs_file_open(struct inode *inode, struct file *file) +{ + file->private_data = inode->i_private; + return nonseekable_open(inode, file); +} + +/** + * provide_user_output - provide output to the user reading a debugfs file. + * @val: boolean value for the answer + * @u: the buffer to store the answer at + * @count: size of the buffer + * @ppos: position in the @u output buffer + * + * This is a simple helper function which stores @val boolean value in the user + * buffer when the user reads one of UBIFS debugfs files. Returns amount of + * bytes written to @u in case of success and a negative error code in case of + * failure. + */ +static int provide_user_output(int val, char __user *u, size_t count, + loff_t *ppos) +{ + char buf[3]; + + if (val) + buf[0] = '1'; + else + buf[0] = '0'; + buf[1] = '\n'; + buf[2] = 0x00; + + return simple_read_from_buffer(u, count, ppos, buf, 2); +} + +static ssize_t dfs_file_read(struct file *file, char __user *u, size_t count, + loff_t *ppos) +{ + struct dentry *dent = file->f_path.dentry; + struct ubifs_info *c = file->private_data; + struct ubifs_debug_info *d = c->dbg; + int val; + + if (dent == d->dfs_chk_gen) + val = d->chk_gen; + else if (dent == d->dfs_chk_index) + val = d->chk_index; + else if (dent == d->dfs_chk_orph) + val = d->chk_orph; + else if (dent == d->dfs_chk_lprops) + val = d->chk_lprops; + else if (dent == d->dfs_chk_fs) + val = d->chk_fs; + else if (dent == d->dfs_tst_rcvry) + val = d->tst_rcvry; + else if (dent == d->dfs_ro_error) + val = c->ro_error; + else + return -EINVAL; + + return provide_user_output(val, u, count, ppos); +} + +/** + * interpret_user_input - interpret user debugfs file input. + * @u: user-provided buffer with the input + * @count: buffer size + * + * This is a helper function which interpret user input to a boolean UBIFS + * debugfs file. Returns %0 or %1 in case of success and a negative error code + * in case of failure. + */ +static int interpret_user_input(const char __user *u, size_t count) +{ + size_t buf_size; + char buf[8]; + + buf_size = min_t(size_t, count, (sizeof(buf) - 1)); + if (copy_from_user(buf, u, buf_size)) + return -EFAULT; + + if (buf[0] == '1') + return 1; + else if (buf[0] == '0') + return 0; + + return -EINVAL; +} + +static ssize_t dfs_file_write(struct file *file, const char __user *u, + size_t count, loff_t *ppos) +{ + struct ubifs_info *c = file->private_data; + struct ubifs_debug_info *d = c->dbg; + struct dentry *dent = file->f_path.dentry; + int val; + + if (file->f_path.dentry == d->dfs_dump_lprops) { + ubifs_dump_lprops(c); + return count; + } + if (file->f_path.dentry == d->dfs_dump_budg) { + ubifs_dump_budg(c, &c->bi); + return count; + } + if (file->f_path.dentry == d->dfs_dump_tnc) { + mutex_lock(&c->tnc_mutex); + ubifs_dump_tnc(c); + mutex_unlock(&c->tnc_mutex); + return count; + } + + val = interpret_user_input(u, count); + if (val < 0) + return val; + + if (dent == d->dfs_chk_gen) + d->chk_gen = val; + else if (dent == d->dfs_chk_index) + d->chk_index = val; + else if (dent == d->dfs_chk_orph) + d->chk_orph = val; + else if (dent == d->dfs_chk_lprops) + d->chk_lprops = val; + else if (dent == d->dfs_chk_fs) + d->chk_fs = val; + else if (dent == d->dfs_tst_rcvry) + d->tst_rcvry = val; + else if (dent == d->dfs_ro_error) + c->ro_error = !!val; + else + return -EINVAL; + + return count; +} + +static const struct file_operations dfs_fops = { + .open = dfs_file_open, + .read = dfs_file_read, + .write = dfs_file_write, + .owner = THIS_MODULE, +}; + +/** + * dbg_debugfs_init_fs - initialize debugfs for UBIFS instance. + * @c: UBIFS file-system description object + * + * This function creates all debugfs files for this instance of UBIFS. + * + * Note, the only reason we have not merged this function with the + * 'ubifs_debugging_init()' function is because it is better to initialize + * debugfs interfaces at the very end of the mount process, and remove them at + * the very beginning of the mount process. + */ +void dbg_debugfs_init_fs(struct ubifs_info *c) +{ + int n; + const char *fname; + struct ubifs_debug_info *d = c->dbg; + + n = snprintf(d->dfs_dir_name, UBIFS_DFS_DIR_LEN, UBIFS_DFS_DIR_NAME, + c->vi.ubi_num, c->vi.vol_id); + if (n >= UBIFS_DFS_DIR_LEN) { + /* The array size is too small */ + return; + } + + fname = d->dfs_dir_name; + d->dfs_dir = debugfs_create_dir(fname, dfs_rootdir); + + fname = "dump_lprops"; + d->dfs_dump_lprops = debugfs_create_file(fname, S_IWUSR, d->dfs_dir, c, + &dfs_fops); + + fname = "dump_budg"; + d->dfs_dump_budg = debugfs_create_file(fname, S_IWUSR, d->dfs_dir, c, + &dfs_fops); + + fname = "dump_tnc"; + d->dfs_dump_tnc = debugfs_create_file(fname, S_IWUSR, d->dfs_dir, c, + &dfs_fops); + + fname = "chk_general"; + d->dfs_chk_gen = debugfs_create_file(fname, S_IRUSR | S_IWUSR, + d->dfs_dir, c, &dfs_fops); + + fname = "chk_index"; + d->dfs_chk_index = debugfs_create_file(fname, S_IRUSR | S_IWUSR, + d->dfs_dir, c, &dfs_fops); + + fname = "chk_orphans"; + d->dfs_chk_orph = debugfs_create_file(fname, S_IRUSR | S_IWUSR, + d->dfs_dir, c, &dfs_fops); + + fname = "chk_lprops"; + d->dfs_chk_lprops = debugfs_create_file(fname, S_IRUSR | S_IWUSR, + d->dfs_dir, c, &dfs_fops); + + fname = "chk_fs"; + d->dfs_chk_fs = debugfs_create_file(fname, S_IRUSR | S_IWUSR, + d->dfs_dir, c, &dfs_fops); + + fname = "tst_recovery"; + d->dfs_tst_rcvry = debugfs_create_file(fname, S_IRUSR | S_IWUSR, + d->dfs_dir, c, &dfs_fops); + + fname = "ro_error"; + d->dfs_ro_error = debugfs_create_file(fname, S_IRUSR | S_IWUSR, + d->dfs_dir, c, &dfs_fops); +} + +/** + * dbg_debugfs_exit_fs - remove all debugfs files. + * @c: UBIFS file-system description object + */ +void dbg_debugfs_exit_fs(struct ubifs_info *c) +{ + debugfs_remove_recursive(c->dbg->dfs_dir); +} + +struct ubifs_global_debug_info ubifs_dbg; + +static struct dentry *dfs_chk_gen; +static struct dentry *dfs_chk_index; +static struct dentry *dfs_chk_orph; +static struct dentry *dfs_chk_lprops; +static struct dentry *dfs_chk_fs; +static struct dentry *dfs_tst_rcvry; + +static ssize_t dfs_global_file_read(struct file *file, char __user *u, + size_t count, loff_t *ppos) +{ + struct dentry *dent = file->f_path.dentry; + int val; + + if (dent == dfs_chk_gen) + val = ubifs_dbg.chk_gen; + else if (dent == dfs_chk_index) + val = ubifs_dbg.chk_index; + else if (dent == dfs_chk_orph) + val = ubifs_dbg.chk_orph; + else if (dent == dfs_chk_lprops) + val = ubifs_dbg.chk_lprops; + else if (dent == dfs_chk_fs) + val = ubifs_dbg.chk_fs; + else if (dent == dfs_tst_rcvry) + val = ubifs_dbg.tst_rcvry; + else + return -EINVAL; + + return provide_user_output(val, u, count, ppos); +} + +static ssize_t dfs_global_file_write(struct file *file, const char __user *u, + size_t count, loff_t *ppos) +{ + struct dentry *dent = file->f_path.dentry; + int val; + + val = interpret_user_input(u, count); + if (val < 0) + return val; + + if (dent == dfs_chk_gen) + ubifs_dbg.chk_gen = val; + else if (dent == dfs_chk_index) + ubifs_dbg.chk_index = val; + else if (dent == dfs_chk_orph) + ubifs_dbg.chk_orph = val; + else if (dent == dfs_chk_lprops) + ubifs_dbg.chk_lprops = val; + else if (dent == dfs_chk_fs) + ubifs_dbg.chk_fs = val; + else if (dent == dfs_tst_rcvry) + ubifs_dbg.tst_rcvry = val; + else + return -EINVAL; + + return count; +} + +static const struct file_operations dfs_global_fops = { + .read = dfs_global_file_read, + .write = dfs_global_file_write, + .owner = THIS_MODULE, +}; + +/** + * dbg_debugfs_init - initialize debugfs file-system. + * + * UBIFS uses debugfs file-system to expose various debugging knobs to + * user-space. This function creates "ubifs" directory in the debugfs + * file-system. + */ +void dbg_debugfs_init(void) +{ + const char *fname; + + fname = "ubifs"; + dfs_rootdir = debugfs_create_dir(fname, NULL); + + fname = "chk_general"; + dfs_chk_gen = debugfs_create_file(fname, S_IRUSR | S_IWUSR, dfs_rootdir, + NULL, &dfs_global_fops); + + fname = "chk_index"; + dfs_chk_index = debugfs_create_file(fname, S_IRUSR | S_IWUSR, + dfs_rootdir, NULL, &dfs_global_fops); + + fname = "chk_orphans"; + dfs_chk_orph = debugfs_create_file(fname, S_IRUSR | S_IWUSR, + dfs_rootdir, NULL, &dfs_global_fops); + + fname = "chk_lprops"; + dfs_chk_lprops = debugfs_create_file(fname, S_IRUSR | S_IWUSR, + dfs_rootdir, NULL, &dfs_global_fops); + + fname = "chk_fs"; + dfs_chk_fs = debugfs_create_file(fname, S_IRUSR | S_IWUSR, dfs_rootdir, + NULL, &dfs_global_fops); + + fname = "tst_recovery"; + dfs_tst_rcvry = debugfs_create_file(fname, S_IRUSR | S_IWUSR, + dfs_rootdir, NULL, &dfs_global_fops); +} + +/** + * dbg_debugfs_exit - remove the "ubifs" directory from debugfs file-system. + */ +void dbg_debugfs_exit(void) +{ + debugfs_remove_recursive(dfs_rootdir); +} + +void ubifs_assert_failed(struct ubifs_info *c, const char *expr, + const char *file, int line) +{ + ubifs_err(c, "UBIFS assert failed: %s, in %s:%u", expr, file, line); + + switch (c->assert_action) { + case ASSACT_PANIC: + BUG(); + break; + + case ASSACT_RO: + ubifs_ro_mode(c, -EINVAL); + break; + + case ASSACT_REPORT: + default: + dump_stack(); + break; + + } +} + +/** + * ubifs_debugging_init - initialize UBIFS debugging. + * @c: UBIFS file-system description object + * + * This function initializes debugging-related data for the file system. + * Returns zero in case of success and a negative error code in case of + * failure. + */ +int ubifs_debugging_init(struct ubifs_info *c) +{ + c->dbg = kzalloc_obj(struct ubifs_debug_info); + if (!c->dbg) + return -ENOMEM; + + return 0; +} + +/** + * ubifs_debugging_exit - free debugging data. + * @c: UBIFS file-system description object + */ +void ubifs_debugging_exit(struct ubifs_info *c) +{ + kfree(c->dbg); +} diff --git a/fs/ubifs/debug.h b/fs/ubifs/debug.h new file mode 100644 index 000000000..d425861e6 --- /dev/null +++ b/fs/ubifs/debug.h @@ -0,0 +1,305 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * + * Authors: Artem Bityutskiy (Битюцкий Артём) + * Adrian Hunter + */ + +#ifndef __UBIFS_DEBUG_H__ +#define __UBIFS_DEBUG_H__ + +/* Checking helper functions */ +typedef int (*dbg_leaf_callback)(struct ubifs_info *c, + struct ubifs_zbranch *zbr, void *priv); +typedef int (*dbg_znode_callback)(struct ubifs_info *c, + struct ubifs_znode *znode, void *priv); + +/* + * The UBIFS debugfs directory name pattern and maximum name length (3 for "ubi" + * + 1 for "_" and 2 for UBI device numbers and 3 for volume number and 1 for + * the trailing zero byte. + */ +#define UBIFS_DFS_DIR_NAME "ubi%d_%d" +#define UBIFS_DFS_DIR_LEN (3 + 1 + 2 + 3 + 1) + +/** + * ubifs_debug_info - per-FS debugging information. + * @old_zroot: old index root - used by 'dbg_check_old_index()' + * @old_zroot_level: old index root level - used by 'dbg_check_old_index()' + * @old_zroot_sqnum: old index root sqnum - used by 'dbg_check_old_index()' + * + * @pc_happened: non-zero if an emulated power cut happened + * @pc_delay: 0=>don't delay, 1=>delay a time, 2=>delay a number of calls + * @pc_timeout: time in jiffies when delay of failure mode expires + * @pc_cnt: current number of calls to failure mode I/O functions + * @pc_cnt_max: number of calls by which to delay failure mode + * + * @chk_lpt_sz: used by LPT tree size checker + * @chk_lpt_sz2: used by LPT tree size checker + * @chk_lpt_wastage: used by LPT tree size checker + * @chk_lpt_lebs: used by LPT tree size checker + * @new_nhead_offs: used by LPT tree size checker + * @new_ihead_lnum: used by debugging to check @c->ihead_lnum + * @new_ihead_offs: used by debugging to check @c->ihead_offs + * + * @saved_lst: saved lprops statistics (used by 'dbg_save_space_info()') + * @saved_bi: saved budgeting information + * @saved_free: saved amount of free space + * @saved_idx_gc_cnt: saved value of @c->idx_gc_cnt + * + * @chk_gen: if general extra checks are enabled + * @chk_index: if index xtra checks are enabled + * @chk_orph: if orphans extra checks are enabled + * @chk_lprops: if lprops extra checks are enabled + * @chk_fs: if UBIFS contents extra checks are enabled + * @tst_rcvry: if UBIFS recovery testing mode enabled + * + * @dfs_dir_name: name of debugfs directory containing this file-system's files + * @dfs_dir: direntry object of the file-system debugfs directory + * @dfs_dump_lprops: "dump lprops" debugfs knob + * @dfs_dump_budg: "dump budgeting information" debugfs knob + * @dfs_dump_tnc: "dump TNC" debugfs knob + * @dfs_chk_gen: debugfs knob to enable UBIFS general extra checks + * @dfs_chk_index: debugfs knob to enable UBIFS index extra checks + * @dfs_chk_orph: debugfs knob to enable UBIFS orphans extra checks + * @dfs_chk_lprops: debugfs knob to enable UBIFS LEP properties extra checks + * @dfs_chk_fs: debugfs knob to enable UBIFS contents extra checks + * @dfs_tst_rcvry: debugfs knob to enable UBIFS recovery testing + * @dfs_ro_error: debugfs knob to switch UBIFS to R/O mode (different to + * re-mounting to R/O mode because it does not flush any buffers + * and UBIFS just starts returning -EROFS on all write + * operations) + */ +struct ubifs_debug_info { + struct ubifs_zbranch old_zroot; + int old_zroot_level; + unsigned long long old_zroot_sqnum; + + int pc_happened; + int pc_delay; + unsigned long pc_timeout; + unsigned int pc_cnt; + unsigned int pc_cnt_max; + + long long chk_lpt_sz; + long long chk_lpt_sz2; + long long chk_lpt_wastage; + int chk_lpt_lebs; + int new_nhead_offs; + int new_ihead_lnum; + int new_ihead_offs; + + struct ubifs_lp_stats saved_lst; + struct ubifs_budg_info saved_bi; + long long saved_free; + int saved_idx_gc_cnt; + + unsigned int chk_gen:1; + unsigned int chk_index:1; + unsigned int chk_orph:1; + unsigned int chk_lprops:1; + unsigned int chk_fs:1; + unsigned int tst_rcvry:1; + + char dfs_dir_name[UBIFS_DFS_DIR_LEN]; + struct dentry *dfs_dir; + struct dentry *dfs_dump_lprops; + struct dentry *dfs_dump_budg; + struct dentry *dfs_dump_tnc; + struct dentry *dfs_chk_gen; + struct dentry *dfs_chk_index; + struct dentry *dfs_chk_orph; + struct dentry *dfs_chk_lprops; + struct dentry *dfs_chk_fs; + struct dentry *dfs_tst_rcvry; + struct dentry *dfs_ro_error; +}; + +/** + * ubifs_global_debug_info - global (not per-FS) UBIFS debugging information. + * + * @chk_gen: if general extra checks are enabled + * @chk_index: if index xtra checks are enabled + * @chk_orph: if orphans extra checks are enabled + * @chk_lprops: if lprops extra checks are enabled + * @chk_fs: if UBIFS contents extra checks are enabled + * @tst_rcvry: if UBIFS recovery testing mode enabled + */ +struct ubifs_global_debug_info { + unsigned int chk_gen:1; + unsigned int chk_index:1; + unsigned int chk_orph:1; + unsigned int chk_lprops:1; + unsigned int chk_fs:1; + unsigned int tst_rcvry:1; +}; + +void ubifs_assert_failed(struct ubifs_info *c, const char *expr, + const char *file, int line); + +#define ubifs_assert(c, expr) do { \ + if (unlikely(!(expr))) { \ + ubifs_assert_failed((struct ubifs_info *)c, #expr, __FILE__, \ + __LINE__); \ + } \ +} while (0) + +#define ubifs_assert_cmt_locked(c) do { \ + if (unlikely(down_write_trylock(&(c)->commit_sem))) { \ + up_write(&(c)->commit_sem); \ + ubifs_err(c, "commit lock is not locked!\n"); \ + ubifs_assert(c, 0); \ + } \ +} while (0) + +#define ubifs_dbg_msg(type, fmt, ...) \ + pr_debug("UBIFS DBG " type " (pid %d): " fmt "\n", current->pid, \ + ##__VA_ARGS__) + +#define DBG_KEY_BUF_LEN 48 +#define ubifs_dbg_msg_key(type, key, fmt, ...) do { \ + char __tmp_key_buf[DBG_KEY_BUF_LEN]; \ + pr_debug("UBIFS DBG " type " (pid %d): " fmt "%s\n", current->pid, \ + ##__VA_ARGS__, \ + dbg_snprintf_key(c, key, __tmp_key_buf, DBG_KEY_BUF_LEN)); \ +} while (0) + +/* General messages */ +#define dbg_gen(fmt, ...) ubifs_dbg_msg("gen", fmt, ##__VA_ARGS__) +/* Additional journal messages */ +#define dbg_jnl(fmt, ...) ubifs_dbg_msg("jnl", fmt, ##__VA_ARGS__) +#define dbg_jnlk(key, fmt, ...) \ + ubifs_dbg_msg_key("jnl", key, fmt, ##__VA_ARGS__) +/* Additional TNC messages */ +#define dbg_tnc(fmt, ...) ubifs_dbg_msg("tnc", fmt, ##__VA_ARGS__) +#define dbg_tnck(key, fmt, ...) \ + ubifs_dbg_msg_key("tnc", key, fmt, ##__VA_ARGS__) +/* Additional lprops messages */ +#define dbg_lp(fmt, ...) ubifs_dbg_msg("lp", fmt, ##__VA_ARGS__) +/* Additional LEB find messages */ +#define dbg_find(fmt, ...) ubifs_dbg_msg("find", fmt, ##__VA_ARGS__) +/* Additional mount messages */ +#define dbg_mnt(fmt, ...) ubifs_dbg_msg("mnt", fmt, ##__VA_ARGS__) +#define dbg_mntk(key, fmt, ...) \ + ubifs_dbg_msg_key("mnt", key, fmt, ##__VA_ARGS__) +/* Additional I/O messages */ +#define dbg_io(fmt, ...) ubifs_dbg_msg("io", fmt, ##__VA_ARGS__) +/* Additional commit messages */ +#define dbg_cmt(fmt, ...) ubifs_dbg_msg("cmt", fmt, ##__VA_ARGS__) +/* Additional budgeting messages */ +#define dbg_budg(fmt, ...) ubifs_dbg_msg("budg", fmt, ##__VA_ARGS__) +/* Additional log messages */ +#define dbg_log(fmt, ...) ubifs_dbg_msg("log", fmt, ##__VA_ARGS__) +/* Additional gc messages */ +#define dbg_gc(fmt, ...) ubifs_dbg_msg("gc", fmt, ##__VA_ARGS__) +/* Additional scan messages */ +#define dbg_scan(fmt, ...) ubifs_dbg_msg("scan", fmt, ##__VA_ARGS__) +/* Additional recovery messages */ +#define dbg_rcvry(fmt, ...) ubifs_dbg_msg("rcvry", fmt, ##__VA_ARGS__) + +extern struct ubifs_global_debug_info ubifs_dbg; + +static inline int dbg_is_chk_gen(const struct ubifs_info *c) +{ + return !!(ubifs_dbg.chk_gen || c->dbg->chk_gen); +} +static inline int dbg_is_chk_index(const struct ubifs_info *c) +{ + return !!(ubifs_dbg.chk_index || c->dbg->chk_index); +} +static inline int dbg_is_chk_orph(const struct ubifs_info *c) +{ + return !!(ubifs_dbg.chk_orph || c->dbg->chk_orph); +} +static inline int dbg_is_chk_lprops(const struct ubifs_info *c) +{ + return !!(ubifs_dbg.chk_lprops || c->dbg->chk_lprops); +} +static inline int dbg_is_chk_fs(const struct ubifs_info *c) +{ + return !!(ubifs_dbg.chk_fs || c->dbg->chk_fs); +} +static inline int dbg_is_tst_rcvry(const struct ubifs_info *c) +{ + return !!(ubifs_dbg.tst_rcvry || c->dbg->tst_rcvry); +} +static inline int dbg_is_power_cut(const struct ubifs_info *c) +{ + return !!c->dbg->pc_happened; +} + +int ubifs_debugging_init(struct ubifs_info *c); +void ubifs_debugging_exit(struct ubifs_info *c); + +/* Dump functions */ +const char *dbg_ntype(int type); +const char *dbg_cstate(int cmt_state); +const char *dbg_jhead(int jhead); +const char *dbg_get_key_dump(const struct ubifs_info *c, + const union ubifs_key *key); +const char *dbg_snprintf_key(const struct ubifs_info *c, + const union ubifs_key *key, char *buffer, int len); +void ubifs_dump_inode(struct ubifs_info *c, const struct inode *inode); +void ubifs_dump_node(const struct ubifs_info *c, const void *node, + int node_len); +void ubifs_dump_budget_req(const struct ubifs_budget_req *req); +void ubifs_dump_lstats(const struct ubifs_lp_stats *lst); +void ubifs_dump_budg(struct ubifs_info *c, const struct ubifs_budg_info *bi); +void ubifs_dump_lprop(const struct ubifs_info *c, + const struct ubifs_lprops *lp); +void ubifs_dump_lprops(struct ubifs_info *c); +void ubifs_dump_lpt_info(struct ubifs_info *c); +void ubifs_dump_leb(const struct ubifs_info *c, int lnum); +void ubifs_dump_znode(const struct ubifs_info *c, + const struct ubifs_znode *znode); +void ubifs_dump_heap(struct ubifs_info *c, struct ubifs_lpt_heap *heap, + int cat); +void ubifs_dump_pnode(struct ubifs_info *c, struct ubifs_pnode *pnode, + struct ubifs_nnode *parent, int iip); +void ubifs_dump_tnc(struct ubifs_info *c); +void ubifs_dump_index(struct ubifs_info *c); +void ubifs_dump_lpt_lebs(const struct ubifs_info *c); + +int dbg_walk_index(struct ubifs_info *c, dbg_leaf_callback leaf_cb, + dbg_znode_callback znode_cb, void *priv); + +/* Checking functions */ +void dbg_save_space_info(struct ubifs_info *c); +int dbg_check_space_info(struct ubifs_info *c); +int dbg_check_lprops(struct ubifs_info *c); +int dbg_old_index_check_init(struct ubifs_info *c, struct ubifs_zbranch *zroot); +int dbg_check_old_index(struct ubifs_info *c, struct ubifs_zbranch *zroot); +int dbg_check_cats(struct ubifs_info *c); +int dbg_check_ltab(struct ubifs_info *c); +int dbg_chk_lpt_free_spc(struct ubifs_info *c); +int dbg_chk_lpt_sz(struct ubifs_info *c, int action, int len); +int dbg_check_synced_i_size(const struct ubifs_info *c, struct inode *inode); +int dbg_check_dir(struct ubifs_info *c, const struct inode *dir); +int dbg_check_tnc(struct ubifs_info *c, int extra); +int dbg_check_idx_size(struct ubifs_info *c, long long idx_size); +int dbg_check_filesystem(struct ubifs_info *c); +void dbg_check_heap(struct ubifs_info *c, struct ubifs_lpt_heap *heap, int cat, + int add_pos); +int dbg_check_lpt_nodes(struct ubifs_info *c, struct ubifs_cnode *cnode, + int row, int col); +int dbg_check_inode_size(struct ubifs_info *c, const struct inode *inode, + loff_t size); +int dbg_check_data_nodes_order(struct ubifs_info *c, struct list_head *head); +int dbg_check_nondata_nodes_order(struct ubifs_info *c, struct list_head *head); + +int dbg_leb_write(struct ubifs_info *c, int lnum, const void *buf, int offs, + int len); +int dbg_leb_change(struct ubifs_info *c, int lnum, const void *buf, int len); +int dbg_leb_unmap(struct ubifs_info *c, int lnum); +int dbg_leb_map(struct ubifs_info *c, int lnum); + +/* Debugfs-related stuff */ +void dbg_debugfs_init(void); +void dbg_debugfs_exit(void); +void dbg_debugfs_init_fs(struct ubifs_info *c); +void dbg_debugfs_exit_fs(struct ubifs_info *c); + +#endif /* !__UBIFS_DEBUG_H__ */ diff --git a/fs/ubifs/dir.c b/fs/ubifs/dir.c new file mode 100644 index 000000000..23ec92416 --- /dev/null +++ b/fs/ubifs/dir.c @@ -0,0 +1,1772 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * Copyright (C) 2006, 2007 University of Szeged, Hungary + * + * Authors: Artem Bityutskiy (Битюцкий Артём) + * Adrian Hunter + * Zoltan Sogor + */ + +/* + * This file implements directory operations. + * + * All FS operations in this file allocate budget before writing anything to the + * media. If they fail to allocate it, the error is returned. The only + * exceptions are 'ubifs_unlink()' and 'ubifs_rmdir()' which keep working even + * if they unable to allocate the budget, because deletion %-ENOSPC failure is + * not what users are usually ready to get. UBIFS budgeting subsystem has some + * space reserved for these purposes. + * + * All operations in this file write all inodes which they change straight + * away, instead of marking them dirty. For example, 'ubifs_link()' changes + * @i_size of the parent inode and writes the parent inode together with the + * target inode. This was done to simplify file-system recovery which would + * otherwise be very difficult to do. The only exception is rename which marks + * the re-named inode dirty (because its @i_ctime is updated) but does not + * write it, but just marks it as dirty. + */ + +#include "ubifs.h" + +/** + * inherit_flags - inherit flags of the parent inode. + * @dir: parent inode + * @mode: new inode mode flags + * + * This is a helper function for 'ubifs_new_inode()' which inherits flag of the + * parent directory inode @dir. UBIFS inodes inherit the following flags: + * o %UBIFS_COMPR_FL, which is useful to switch compression on/of on + * sub-directory basis; + * o %UBIFS_SYNC_FL - useful for the same reasons; + * o %UBIFS_DIRSYNC_FL - similar, but relevant only to directories. + * + * This function returns the inherited flags. + */ +static int inherit_flags(const struct inode *dir, umode_t mode) +{ + int flags; + const struct ubifs_inode *ui = ubifs_inode(dir); + + if (!S_ISDIR(dir->i_mode)) + /* + * The parent is not a directory, which means that an extended + * attribute inode is being created. No flags. + */ + return 0; + + flags = ui->flags & (UBIFS_COMPR_FL | UBIFS_SYNC_FL | UBIFS_DIRSYNC_FL); + if (!S_ISDIR(mode)) + /* The "DIRSYNC" flag only applies to directories */ + flags &= ~UBIFS_DIRSYNC_FL; + return flags; +} + +/** + * ubifs_new_inode - allocate new UBIFS inode object. + * @c: UBIFS file-system description object + * @dir: parent directory inode + * @mode: inode mode flags + * @is_xattr: whether the inode is xattr inode + * + * This function finds an unused inode number, allocates new inode and + * initializes it. Non-xattr new inode may be written with xattrs(selinux/ + * encryption) before writing dentry, which could cause inconsistent problem + * when powercut happens between two operations. To deal with it, non-xattr + * new inode is initialized with zero-nlink and added into orphan list, caller + * should make sure that inode is relinked later, and make sure that orphan + * removing and journal writing into an committing atomic operation. Returns + * new inode in case of success and an error code in case of failure. + */ +struct inode *ubifs_new_inode(struct ubifs_info *c, struct inode *dir, + umode_t mode, bool is_xattr) +{ + int err; + struct inode *inode; + struct ubifs_inode *ui; + bool encrypted = false; + + inode = new_inode(c->vfs_sb); + ui = ubifs_inode(inode); + if (!inode) + return ERR_PTR(-ENOMEM); + + /* + * Set 'S_NOCMTIME' to prevent VFS form updating [mc]time of inodes and + * marking them dirty in file write path (see 'file_update_time()'). + * UBIFS has to fully control "clean <-> dirty" transitions of inodes + * to make budgeting work. + */ + inode->i_flags |= S_NOCMTIME; + + inode_init_owner(&nop_mnt_idmap, inode, dir, mode); + simple_inode_init_ts(inode); + inode->i_mapping->nrpages = 0; + + if (!is_xattr) { + err = fscrypt_prepare_new_inode(dir, inode, &encrypted); + if (err) { + ubifs_err(c, "fscrypt_prepare_new_inode failed: %i", err); + goto out_iput; + } + } + + switch (mode & S_IFMT) { + case S_IFREG: + inode->i_mapping->a_ops = &ubifs_file_address_operations; + inode->i_op = &ubifs_file_inode_operations; + inode->i_fop = &ubifs_file_operations; + break; + case S_IFDIR: + inode->i_op = &ubifs_dir_inode_operations; + inode->i_fop = &ubifs_dir_operations; + inode->i_size = ui->ui_size = UBIFS_INO_NODE_SZ; + break; + case S_IFLNK: + inode->i_op = &ubifs_symlink_inode_operations; + break; + case S_IFSOCK: + case S_IFIFO: + case S_IFBLK: + case S_IFCHR: + inode->i_op = &ubifs_file_inode_operations; + break; + default: + BUG(); + } + + ui->flags = inherit_flags(dir, mode); + ubifs_set_inode_flags(inode); + if (S_ISREG(mode)) + ui->compr_type = c->default_compr; + else + ui->compr_type = UBIFS_COMPR_NONE; + ui->synced_i_size = 0; + + spin_lock(&c->cnt_lock); + /* Inode number overflow is currently not supported */ + if (c->highest_inum >= INUM_WARN_WATERMARK) { + if (c->highest_inum >= INUM_WATERMARK) { + spin_unlock(&c->cnt_lock); + ubifs_err(c, "out of inode numbers"); + err = -EINVAL; + goto out_iput; + } + ubifs_warn(c, "running out of inode numbers (current %lu, max %u)", + (unsigned long)c->highest_inum, INUM_WATERMARK); + } + + inode->i_ino = ++c->highest_inum; + /* + * The creation sequence number remains with this inode for its + * lifetime. All nodes for this inode have a greater sequence number, + * and so it is possible to distinguish obsolete nodes belonging to a + * previous incarnation of the same inode number - for example, for the + * purpose of rebuilding the index. + */ + ui->creat_sqnum = ++c->max_sqnum; + spin_unlock(&c->cnt_lock); + + if (!is_xattr) { + set_nlink(inode, 0); + err = ubifs_add_orphan(c, inode->i_ino); + if (err) { + ubifs_err(c, "ubifs_add_orphan failed: %i", err); + goto out_iput; + } + down_read(&c->commit_sem); + ui->del_cmtno = c->cmt_no; + up_read(&c->commit_sem); + } + + if (encrypted) { + err = fscrypt_set_context(inode, NULL); + if (err) { + if (!is_xattr) { + set_nlink(inode, 1); + ubifs_delete_orphan(c, inode->i_ino); + } + ubifs_err(c, "fscrypt_set_context failed: %i", err); + goto out_iput; + } + } + + return inode; + +out_iput: + make_bad_inode(inode); + iput(inode); + return ERR_PTR(err); +} + +static int dbg_check_name(const struct ubifs_info *c, + const struct ubifs_dent_node *dent, + const struct fscrypt_name *nm) +{ + if (!dbg_is_chk_gen(c)) + return 0; + if (le16_to_cpu(dent->nlen) != fname_len(nm)) + return -EINVAL; + if (memcmp(dent->name, fname_name(nm), fname_len(nm))) + return -EINVAL; + return 0; +} + +static struct dentry *ubifs_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + int err; + union ubifs_key key; + struct inode *inode = NULL; + struct ubifs_dent_node *dent = NULL; + struct ubifs_info *c = dir->i_sb->s_fs_info; + struct fscrypt_name nm; + + dbg_gen("'%pd' in dir ino %llu", dentry, dir->i_ino); + + err = fscrypt_prepare_lookup(dir, dentry, &nm); + if (err == -ENOENT) + return d_splice_alias(NULL, dentry); + if (err) + return ERR_PTR(err); + + if (fname_len(&nm) > UBIFS_MAX_NLEN) { + inode = ERR_PTR(-ENAMETOOLONG); + goto done; + } + + dent = kmalloc(UBIFS_MAX_DENT_NODE_SZ, GFP_NOFS); + if (!dent) { + inode = ERR_PTR(-ENOMEM); + goto done; + } + + if (fname_name(&nm) == NULL) { + if (nm.hash & ~UBIFS_S_KEY_HASH_MASK) + goto done; /* ENOENT */ + dent_key_init_hash(c, &key, dir->i_ino, nm.hash); + err = ubifs_tnc_lookup_dh(c, &key, dent, nm.minor_hash); + } else { + dent_key_init(c, &key, dir->i_ino, &nm); + err = ubifs_tnc_lookup_nm(c, &key, dent, &nm); + } + + if (err) { + if (err == -ENOENT) + dbg_gen("not found"); + else + inode = ERR_PTR(err); + goto done; + } + + if (dbg_check_name(c, dent, &nm)) { + inode = ERR_PTR(-EINVAL); + goto done; + } + + inode = ubifs_iget(dir->i_sb, le64_to_cpu(dent->inum)); + if (IS_ERR(inode)) { + /* + * This should not happen. Probably the file-system needs + * checking. + */ + err = PTR_ERR(inode); + ubifs_err(c, "dead directory entry '%pd', error %d", + dentry, err); + ubifs_ro_mode(c, err); + goto done; + } + + if (IS_ENCRYPTED(dir) && + (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode)) && + !fscrypt_has_permitted_context(dir, inode)) { + ubifs_warn(c, "Inconsistent encryption contexts: %llu/%llu", + dir->i_ino, inode->i_ino); + iput(inode); + inode = ERR_PTR(-EPERM); + } + +done: + kfree(dent); + fscrypt_free_filename(&nm); + return d_splice_alias(inode, dentry); +} + +static int ubifs_prepare_create(struct inode *dir, struct dentry *dentry, + struct fscrypt_name *nm) +{ + if (fscrypt_is_nokey_name(dentry)) + return -ENOKEY; + + return fscrypt_setup_filename(dir, &dentry->d_name, 0, nm); +} + +static int ubifs_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct inode *inode; + struct ubifs_info *c = dir->i_sb->s_fs_info; + struct ubifs_budget_req req = { .new_ino = 1, .new_dent = 1, + .dirtied_ino = 1 }; + struct ubifs_inode *dir_ui = ubifs_inode(dir); + struct fscrypt_name nm; + int err, sz_change; + + /* + * Budget request settings: new inode, new direntry, changing the + * parent directory inode. + */ + + dbg_gen("dent '%pd', mode %#hx in dir ino %llu", + dentry, mode, dir->i_ino); + + err = ubifs_budget_space(c, &req); + if (err) + return err; + + err = ubifs_prepare_create(dir, dentry, &nm); + if (err) + goto out_budg; + + sz_change = CALC_DENT_SIZE(fname_len(&nm)); + + inode = ubifs_new_inode(c, dir, mode, false); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + goto out_fname; + } + + err = ubifs_init_security(dir, inode, &dentry->d_name); + if (err) + goto out_inode; + + set_nlink(inode, 1); + mutex_lock(&dir_ui->ui_mutex); + dir->i_size += sz_change; + dir_ui->ui_size = dir->i_size; + inode_set_mtime_to_ts(dir, + inode_set_ctime_to_ts(dir, inode_get_ctime(inode))); + err = ubifs_jnl_update(c, dir, &nm, inode, 0, 0, 1); + if (err) + goto out_cancel; + mutex_unlock(&dir_ui->ui_mutex); + + ubifs_release_budget(c, &req); + fscrypt_free_filename(&nm); + insert_inode_hash(inode); + d_instantiate(dentry, inode); + return 0; + +out_cancel: + dir->i_size -= sz_change; + dir_ui->ui_size = dir->i_size; + mutex_unlock(&dir_ui->ui_mutex); + set_nlink(inode, 0); +out_inode: + iput(inode); +out_fname: + fscrypt_free_filename(&nm); +out_budg: + ubifs_release_budget(c, &req); + ubifs_err(c, "cannot create regular file, error %d", err); + return err; +} + +static struct inode *create_whiteout(struct inode *dir, struct dentry *dentry) +{ + int err; + umode_t mode = S_IFCHR | WHITEOUT_MODE; + struct inode *inode; + struct ubifs_info *c = dir->i_sb->s_fs_info; + + /* + * Create an inode('nlink = 1') for whiteout without updating journal, + * let ubifs_jnl_rename() store it on flash to complete rename whiteout + * atomically. + */ + + dbg_gen("dent '%pd', mode %#hx in dir ino %llu", + dentry, mode, dir->i_ino); + + inode = ubifs_new_inode(c, dir, mode, false); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + goto out_free; + } + + init_special_inode(inode, inode->i_mode, WHITEOUT_DEV); + ubifs_assert(c, inode->i_op == &ubifs_file_inode_operations); + + err = ubifs_init_security(dir, inode, &dentry->d_name); + if (err) + goto out_inode; + + /* The dir size is updated by do_rename. */ + insert_inode_hash(inode); + + return inode; + +out_inode: + iput(inode); +out_free: + ubifs_err(c, "cannot create whiteout file, error %d", err); + return ERR_PTR(err); +} + +/** + * lock_2_inodes - a wrapper for locking two UBIFS inodes. + * @inode1: first inode + * @inode2: second inode + * + * We do not implement any tricks to guarantee strict lock ordering, because + * VFS has already done it for us on the @i_mutex. So this is just a simple + * wrapper function. + */ +static void lock_2_inodes(struct inode *inode1, struct inode *inode2) +{ + mutex_lock_nested(&ubifs_inode(inode1)->ui_mutex, WB_MUTEX_1); + mutex_lock_nested(&ubifs_inode(inode2)->ui_mutex, WB_MUTEX_2); +} + +/** + * unlock_2_inodes - a wrapper for unlocking two UBIFS inodes. + * @inode1: first inode + * @inode2: second inode + */ +static void unlock_2_inodes(struct inode *inode1, struct inode *inode2) +{ + mutex_unlock(&ubifs_inode(inode2)->ui_mutex); + mutex_unlock(&ubifs_inode(inode1)->ui_mutex); +} + +static int ubifs_tmpfile(struct mnt_idmap *idmap, struct inode *dir, + struct file *file, umode_t mode) +{ + struct dentry *dentry = file->f_path.dentry; + struct inode *inode; + struct ubifs_info *c = dir->i_sb->s_fs_info; + struct ubifs_budget_req req = { .new_ino = 1, .new_dent = 1, + .dirtied_ino = 1}; + struct ubifs_budget_req ino_req = { .dirtied_ino = 1 }; + struct ubifs_inode *ui; + int err, instantiated = 0; + struct fscrypt_name nm; + + /* + * Budget request settings: new inode, new direntry, changing the + * parent directory inode. + * Allocate budget separately for new dirtied inode, the budget will + * be released via writeback. + */ + + dbg_gen("dent '%pd', mode %#hx in dir ino %llu", + dentry, mode, dir->i_ino); + + err = fscrypt_setup_filename(dir, &dentry->d_name, 0, &nm); + if (err) + return err; + + err = ubifs_budget_space(c, &req); + if (err) { + fscrypt_free_filename(&nm); + return err; + } + + err = ubifs_budget_space(c, &ino_req); + if (err) { + ubifs_release_budget(c, &req); + fscrypt_free_filename(&nm); + return err; + } + + inode = ubifs_new_inode(c, dir, mode, false); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + goto out_budg; + } + ui = ubifs_inode(inode); + + err = ubifs_init_security(dir, inode, &dentry->d_name); + if (err) + goto out_inode; + + set_nlink(inode, 1); + mutex_lock(&ui->ui_mutex); + insert_inode_hash(inode); + d_tmpfile(file, inode); + ubifs_assert(c, ui->dirty); + + instantiated = 1; + mutex_unlock(&ui->ui_mutex); + + lock_2_inodes(dir, inode); + err = ubifs_jnl_update(c, dir, &nm, inode, 1, 0, 1); + if (err) + goto out_cancel; + unlock_2_inodes(dir, inode); + + ubifs_release_budget(c, &req); + fscrypt_free_filename(&nm); + + return finish_open_simple(file, 0); + +out_cancel: + unlock_2_inodes(dir, inode); +out_inode: + if (!instantiated) + iput(inode); +out_budg: + ubifs_release_budget(c, &req); + if (!instantiated) + ubifs_release_budget(c, &ino_req); + fscrypt_free_filename(&nm); + ubifs_err(c, "cannot create temporary file, error %d", err); + return err; +} + +/** + * vfs_dent_type - get VFS directory entry type. + * @type: UBIFS directory entry type + * + * This function converts UBIFS directory entry type into VFS directory entry + * type. + */ +static unsigned int vfs_dent_type(uint8_t type) +{ + switch (type) { + case UBIFS_ITYPE_REG: + return DT_REG; + case UBIFS_ITYPE_DIR: + return DT_DIR; + case UBIFS_ITYPE_LNK: + return DT_LNK; + case UBIFS_ITYPE_BLK: + return DT_BLK; + case UBIFS_ITYPE_CHR: + return DT_CHR; + case UBIFS_ITYPE_FIFO: + return DT_FIFO; + case UBIFS_ITYPE_SOCK: + return DT_SOCK; + default: + BUG(); + } + return 0; +} + +struct ubifs_dir_data { + struct ubifs_dent_node *dent; + u64 cookie; +}; + +/* + * The classical Unix view for directory is that it is a linear array of + * (name, inode number) entries. Linux/VFS assumes this model as well. + * Particularly, 'readdir()' call wants us to return a directory entry offset + * which later may be used to continue 'readdir()'ing the directory or to + * 'seek()' to that specific direntry. Obviously UBIFS does not really fit this + * model because directory entries are identified by keys, which may collide. + * + * UBIFS uses directory entry hash value for directory offsets, so + * 'seekdir()'/'telldir()' may not always work because of possible key + * collisions. But UBIFS guarantees that consecutive 'readdir()' calls work + * properly by means of saving full directory entry name in the private field + * of the file description object. + * + * This means that UBIFS cannot support NFS which requires full + * 'seekdir()'/'telldir()' support. + */ +static int ubifs_readdir(struct file *file, struct dir_context *ctx) +{ + int fstr_real_len = 0, err = 0; + struct fscrypt_name nm; + struct fscrypt_str fstr = {0}; + union ubifs_key key; + struct ubifs_dent_node *dent; + struct inode *dir = file_inode(file); + struct ubifs_info *c = dir->i_sb->s_fs_info; + bool encrypted = IS_ENCRYPTED(dir); + struct ubifs_dir_data *data = file->private_data; + + dbg_gen("dir ino %llu, f_pos %#llx", dir->i_ino, ctx->pos); + + if (ctx->pos > UBIFS_S_KEY_HASH_MASK || ctx->pos == 2) + /* + * The directory was seek'ed to a senseless position or there + * are no more entries. + */ + return 0; + + if (encrypted) { + err = fscrypt_prepare_readdir(dir); + if (err) + return err; + + err = fscrypt_fname_alloc_buffer(UBIFS_MAX_NLEN, &fstr); + if (err) + return err; + + fstr_real_len = fstr.len; + } + + if (data->cookie == 0) { + /* + * The file was seek'ed, which means that @data->dent + * is now invalid. This may also be just the first + * 'ubifs_readdir()' invocation, in which case + * @data->dent is NULL, and the below code is + * basically a no-op. + */ + kfree(data->dent); + data->dent = NULL; + } + + /* + * 'ubifs_dir_llseek()' sets @data->cookie to zero, and we use this + * for detecting whether the file was seek'ed. + */ + data->cookie = 1; + + /* File positions 0 and 1 correspond to "." and ".." */ + if (ctx->pos < 2) { + ubifs_assert(c, !data->dent); + if (!dir_emit_dots(file, ctx)) { + if (encrypted) + fscrypt_fname_free_buffer(&fstr); + return 0; + } + + /* Find the first entry in TNC and save it */ + lowest_dent_key(c, &key, dir->i_ino); + fname_len(&nm) = 0; + dent = ubifs_tnc_next_ent(c, &key, &nm); + if (IS_ERR(dent)) { + err = PTR_ERR(dent); + goto out; + } + + ctx->pos = key_hash_flash(c, &dent->key); + data->dent = dent; + } + + dent = data->dent; + if (!dent) { + /* + * The directory was seek'ed to and is now readdir'ed. + * Find the entry corresponding to @ctx->pos or the closest one. + */ + dent_key_init_hash(c, &key, dir->i_ino, ctx->pos); + fname_len(&nm) = 0; + dent = ubifs_tnc_next_ent(c, &key, &nm); + if (IS_ERR(dent)) { + err = PTR_ERR(dent); + goto out; + } + ctx->pos = key_hash_flash(c, &dent->key); + data->dent = dent; + } + + while (1) { + dbg_gen("ino %llu, new f_pos %#x", + (unsigned long long)le64_to_cpu(dent->inum), + key_hash_flash(c, &dent->key)); + ubifs_assert(c, le64_to_cpu(dent->ch.sqnum) > + ubifs_inode(dir)->creat_sqnum); + + fname_len(&nm) = le16_to_cpu(dent->nlen); + fname_name(&nm) = dent->name; + + if (encrypted) { + fstr.len = fstr_real_len; + + err = fscrypt_fname_disk_to_usr(dir, key_hash_flash(c, + &dent->key), + le32_to_cpu(dent->cookie), + &nm.disk_name, &fstr); + if (err) + goto out; + } else { + fstr.len = fname_len(&nm); + fstr.name = fname_name(&nm); + } + + if (!dir_emit(ctx, fstr.name, fstr.len, + le64_to_cpu(dent->inum), + vfs_dent_type(dent->type))) { + if (encrypted) + fscrypt_fname_free_buffer(&fstr); + return 0; + } + + /* Switch to the next entry */ + key_read(c, &dent->key, &key); + dent = ubifs_tnc_next_ent(c, &key, &nm); + if (IS_ERR(dent)) { + err = PTR_ERR(dent); + goto out; + } + + kfree(data->dent); + ctx->pos = key_hash_flash(c, &dent->key); + data->dent = dent; + cond_resched(); + } + +out: + kfree(data->dent); + data->dent = NULL; + + if (encrypted) + fscrypt_fname_free_buffer(&fstr); + + if (err != -ENOENT) + ubifs_err(c, "cannot find next direntry, error %d", err); + else + /* + * -ENOENT is a non-fatal error in this context, the TNC uses + * it to indicate that the cursor moved past the current directory + * and readdir() has to stop. + */ + err = 0; + + + /* 2 is a special value indicating that there are no more direntries */ + ctx->pos = 2; + return err; +} + +/* Free saved readdir() state when the directory is closed */ +static int ubifs_dir_release(struct inode *dir, struct file *file) +{ + struct ubifs_dir_data *data = file->private_data; + + kfree(data->dent); + kfree(data); + file->private_data = NULL; + return 0; +} + +static int ubifs_link(struct dentry *old_dentry, struct inode *dir, + struct dentry *dentry) +{ + struct ubifs_info *c = dir->i_sb->s_fs_info; + struct inode *inode = d_inode(old_dentry); + struct ubifs_inode *ui = ubifs_inode(inode); + struct ubifs_inode *dir_ui = ubifs_inode(dir); + int err, sz_change; + struct ubifs_budget_req req = { .new_dent = 1, .dirtied_ino = 2, + .dirtied_ino_d = ALIGN(ui->data_len, 8) }; + struct fscrypt_name nm; + + /* + * Budget request settings: new direntry, changing the target inode, + * changing the parent inode. + */ + + dbg_gen("dent '%pd' to ino %llu (nlink %d) in dir ino %llu", + dentry, inode->i_ino, + inode->i_nlink, dir->i_ino); + ubifs_assert(c, inode_is_locked(dir)); + ubifs_assert(c, inode_is_locked(inode)); + + err = fscrypt_prepare_link(old_dentry, dir, dentry); + if (err) + return err; + + err = fscrypt_setup_filename(dir, &dentry->d_name, 0, &nm); + if (err) + return err; + + sz_change = CALC_DENT_SIZE(fname_len(&nm)); + + err = dbg_check_synced_i_size(c, inode); + if (err) + goto out_fname; + + err = ubifs_budget_space(c, &req); + if (err) + goto out_fname; + + lock_2_inodes(dir, inode); + + inc_nlink(inode); + ihold(inode); + inode_set_ctime_current(inode); + dir->i_size += sz_change; + dir_ui->ui_size = dir->i_size; + inode_set_mtime_to_ts(dir, + inode_set_ctime_to_ts(dir, inode_get_ctime(inode))); + err = ubifs_jnl_update(c, dir, &nm, inode, 0, 0, inode->i_nlink == 1); + if (err) + goto out_cancel; + unlock_2_inodes(dir, inode); + + ubifs_release_budget(c, &req); + d_instantiate(dentry, inode); + fscrypt_free_filename(&nm); + return 0; + +out_cancel: + dir->i_size -= sz_change; + dir_ui->ui_size = dir->i_size; + drop_nlink(inode); + unlock_2_inodes(dir, inode); + ubifs_release_budget(c, &req); + iput(inode); +out_fname: + fscrypt_free_filename(&nm); + return err; +} + +static int ubifs_unlink(struct inode *dir, struct dentry *dentry) +{ + struct ubifs_info *c = dir->i_sb->s_fs_info; + struct inode *inode = d_inode(dentry); + struct ubifs_inode *dir_ui = ubifs_inode(dir); + int err, sz_change, budgeted = 1; + struct ubifs_budget_req req = { .mod_dent = 1, .dirtied_ino = 2 }; + unsigned int saved_nlink = inode->i_nlink; + struct fscrypt_name nm; + + /* + * Budget request settings: deletion direntry, deletion inode (+1 for + * @dirtied_ino), changing the parent directory inode. If budgeting + * fails, go ahead anyway because we have extra space reserved for + * deletions. + */ + + dbg_gen("dent '%pd' from ino %llu (nlink %d) in dir ino %llu", + dentry, inode->i_ino, + inode->i_nlink, dir->i_ino); + + err = fscrypt_setup_filename(dir, &dentry->d_name, 1, &nm); + if (err) + return err; + + err = ubifs_purge_xattrs(inode); + if (err) + return err; + + sz_change = CALC_DENT_SIZE(fname_len(&nm)); + + ubifs_assert(c, inode_is_locked(dir)); + ubifs_assert(c, inode_is_locked(inode)); + err = dbg_check_synced_i_size(c, inode); + if (err) + goto out_fname; + + err = ubifs_budget_space(c, &req); + if (err) { + if (err != -ENOSPC) + goto out_fname; + budgeted = 0; + } + + lock_2_inodes(dir, inode); + inode_set_ctime_current(inode); + drop_nlink(inode); + dir->i_size -= sz_change; + dir_ui->ui_size = dir->i_size; + inode_set_mtime_to_ts(dir, + inode_set_ctime_to_ts(dir, inode_get_ctime(inode))); + err = ubifs_jnl_update(c, dir, &nm, inode, 1, 0, 0); + if (err) + goto out_cancel; + unlock_2_inodes(dir, inode); + + if (budgeted) + ubifs_release_budget(c, &req); + else { + /* We've deleted something - clean the "no space" flags */ + c->bi.nospace = c->bi.nospace_rp = 0; + smp_wmb(); + } + fscrypt_free_filename(&nm); + return 0; + +out_cancel: + dir->i_size += sz_change; + dir_ui->ui_size = dir->i_size; + set_nlink(inode, saved_nlink); + unlock_2_inodes(dir, inode); + if (budgeted) + ubifs_release_budget(c, &req); +out_fname: + fscrypt_free_filename(&nm); + return err; +} + +/** + * ubifs_check_dir_empty - check if a directory is empty or not. + * @dir: VFS inode object of the directory to check + * + * This function checks if directory @dir is empty. Returns zero if the + * directory is empty, %-ENOTEMPTY if it is not, and other negative error codes + * in case of errors. + */ +int ubifs_check_dir_empty(struct inode *dir) +{ + struct ubifs_info *c = dir->i_sb->s_fs_info; + struct fscrypt_name nm = { 0 }; + struct ubifs_dent_node *dent; + union ubifs_key key; + int err; + + lowest_dent_key(c, &key, dir->i_ino); + dent = ubifs_tnc_next_ent(c, &key, &nm); + if (IS_ERR(dent)) { + err = PTR_ERR(dent); + if (err == -ENOENT) + err = 0; + } else { + kfree(dent); + err = -ENOTEMPTY; + } + return err; +} + +static int ubifs_rmdir(struct inode *dir, struct dentry *dentry) +{ + struct ubifs_info *c = dir->i_sb->s_fs_info; + struct inode *inode = d_inode(dentry); + int err, sz_change, budgeted = 1; + struct ubifs_inode *dir_ui = ubifs_inode(dir); + struct ubifs_budget_req req = { .mod_dent = 1, .dirtied_ino = 2 }; + struct fscrypt_name nm; + + /* + * Budget request settings: deletion direntry, deletion inode and + * changing the parent inode. If budgeting fails, go ahead anyway + * because we have extra space reserved for deletions. + */ + + dbg_gen("directory '%pd', ino %llu in dir ino %llu", dentry, + inode->i_ino, dir->i_ino); + ubifs_assert(c, inode_is_locked(dir)); + ubifs_assert(c, inode_is_locked(inode)); + err = ubifs_check_dir_empty(d_inode(dentry)); + if (err) + return err; + + err = fscrypt_setup_filename(dir, &dentry->d_name, 1, &nm); + if (err) + return err; + + err = ubifs_purge_xattrs(inode); + if (err) + return err; + + sz_change = CALC_DENT_SIZE(fname_len(&nm)); + + err = ubifs_budget_space(c, &req); + if (err) { + if (err != -ENOSPC) + goto out_fname; + budgeted = 0; + } + + lock_2_inodes(dir, inode); + inode_set_ctime_current(inode); + clear_nlink(inode); + drop_nlink(dir); + dir->i_size -= sz_change; + dir_ui->ui_size = dir->i_size; + inode_set_mtime_to_ts(dir, + inode_set_ctime_to_ts(dir, inode_get_ctime(inode))); + err = ubifs_jnl_update(c, dir, &nm, inode, 1, 0, 0); + if (err) + goto out_cancel; + unlock_2_inodes(dir, inode); + + if (budgeted) + ubifs_release_budget(c, &req); + else { + /* We've deleted something - clean the "no space" flags */ + c->bi.nospace = c->bi.nospace_rp = 0; + smp_wmb(); + } + fscrypt_free_filename(&nm); + return 0; + +out_cancel: + dir->i_size += sz_change; + dir_ui->ui_size = dir->i_size; + inc_nlink(dir); + set_nlink(inode, 2); + unlock_2_inodes(dir, inode); + if (budgeted) + ubifs_release_budget(c, &req); +out_fname: + fscrypt_free_filename(&nm); + return err; +} + +static struct dentry *ubifs_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct inode *inode; + struct ubifs_inode *dir_ui = ubifs_inode(dir); + struct ubifs_info *c = dir->i_sb->s_fs_info; + int err, sz_change; + struct ubifs_budget_req req = { .new_ino = 1, .new_dent = 1, + .dirtied_ino = 1}; + struct fscrypt_name nm; + + /* + * Budget request settings: new inode, new direntry and changing parent + * directory inode. + */ + + dbg_gen("dent '%pd', mode %#hx in dir ino %llu", + dentry, mode, dir->i_ino); + + err = ubifs_budget_space(c, &req); + if (err) + return ERR_PTR(err); + + err = ubifs_prepare_create(dir, dentry, &nm); + if (err) + goto out_budg; + + sz_change = CALC_DENT_SIZE(fname_len(&nm)); + + inode = ubifs_new_inode(c, dir, mode, false); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + goto out_fname; + } + + err = ubifs_init_security(dir, inode, &dentry->d_name); + if (err) + goto out_inode; + + set_nlink(inode, 1); + mutex_lock(&dir_ui->ui_mutex); + insert_inode_hash(inode); + inc_nlink(inode); + inc_nlink(dir); + dir->i_size += sz_change; + dir_ui->ui_size = dir->i_size; + inode_set_mtime_to_ts(dir, + inode_set_ctime_to_ts(dir, inode_get_ctime(inode))); + err = ubifs_jnl_update(c, dir, &nm, inode, 0, 0, 1); + if (err) { + ubifs_err(c, "cannot create directory, error %d", err); + goto out_cancel; + } + mutex_unlock(&dir_ui->ui_mutex); + + ubifs_release_budget(c, &req); + d_instantiate(dentry, inode); + fscrypt_free_filename(&nm); + return NULL; + +out_cancel: + dir->i_size -= sz_change; + dir_ui->ui_size = dir->i_size; + drop_nlink(dir); + mutex_unlock(&dir_ui->ui_mutex); + set_nlink(inode, 0); +out_inode: + iput(inode); +out_fname: + fscrypt_free_filename(&nm); +out_budg: + ubifs_release_budget(c, &req); + return ERR_PTR(err); +} + +static int ubifs_mknod(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) +{ + struct inode *inode; + struct ubifs_inode *ui; + struct ubifs_inode *dir_ui = ubifs_inode(dir); + struct ubifs_info *c = dir->i_sb->s_fs_info; + union ubifs_dev_desc *dev = NULL; + int sz_change; + int err, devlen = 0; + struct ubifs_budget_req req = { .new_ino = 1, .new_dent = 1, + .dirtied_ino = 1 }; + struct fscrypt_name nm; + + /* + * Budget request settings: new inode, new direntry and changing parent + * directory inode. + */ + + dbg_gen("dent '%pd' in dir ino %llu", dentry, dir->i_ino); + + if (S_ISBLK(mode) || S_ISCHR(mode)) { + dev = kmalloc_obj(union ubifs_dev_desc, GFP_NOFS); + if (!dev) + return -ENOMEM; + devlen = ubifs_encode_dev(dev, rdev); + } + + req.new_ino_d = ALIGN(devlen, 8); + err = ubifs_budget_space(c, &req); + if (err) { + kfree(dev); + return err; + } + + err = ubifs_prepare_create(dir, dentry, &nm); + if (err) { + kfree(dev); + goto out_budg; + } + + sz_change = CALC_DENT_SIZE(fname_len(&nm)); + + inode = ubifs_new_inode(c, dir, mode, false); + if (IS_ERR(inode)) { + kfree(dev); + err = PTR_ERR(inode); + goto out_fname; + } + + err = ubifs_init_security(dir, inode, &dentry->d_name); + if (err) { + kfree(dev); + goto out_inode; + } + + init_special_inode(inode, inode->i_mode, rdev); + inode->i_size = ubifs_inode(inode)->ui_size = devlen; + ui = ubifs_inode(inode); + ui->data = dev; + ui->data_len = devlen; + set_nlink(inode, 1); + + mutex_lock(&dir_ui->ui_mutex); + dir->i_size += sz_change; + dir_ui->ui_size = dir->i_size; + inode_set_mtime_to_ts(dir, + inode_set_ctime_to_ts(dir, inode_get_ctime(inode))); + err = ubifs_jnl_update(c, dir, &nm, inode, 0, 0, 1); + if (err) + goto out_cancel; + mutex_unlock(&dir_ui->ui_mutex); + + ubifs_release_budget(c, &req); + insert_inode_hash(inode); + d_instantiate(dentry, inode); + fscrypt_free_filename(&nm); + return 0; + +out_cancel: + dir->i_size -= sz_change; + dir_ui->ui_size = dir->i_size; + mutex_unlock(&dir_ui->ui_mutex); + set_nlink(inode, 0); +out_inode: + iput(inode); +out_fname: + fscrypt_free_filename(&nm); +out_budg: + ubifs_release_budget(c, &req); + return err; +} + +static int ubifs_symlink(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const char *symname) +{ + struct inode *inode; + struct ubifs_inode *ui; + struct ubifs_inode *dir_ui = ubifs_inode(dir); + struct ubifs_info *c = dir->i_sb->s_fs_info; + int err, sz_change, len = strlen(symname); + struct fscrypt_str disk_link; + struct ubifs_budget_req req = { .new_ino = 1, .new_dent = 1, + .dirtied_ino = 1 }; + struct fscrypt_name nm; + + dbg_gen("dent '%pd', target '%s' in dir ino %llu", dentry, + symname, dir->i_ino); + + err = fscrypt_prepare_symlink(dir, symname, len, UBIFS_MAX_INO_DATA, + &disk_link); + if (err) + return err; + + /* + * Budget request settings: new inode, new direntry and changing parent + * directory inode. + */ + req.new_ino_d = ALIGN(disk_link.len - 1, 8); + err = ubifs_budget_space(c, &req); + if (err) + return err; + + err = ubifs_prepare_create(dir, dentry, &nm); + if (err) + goto out_budg; + + sz_change = CALC_DENT_SIZE(fname_len(&nm)); + + inode = ubifs_new_inode(c, dir, S_IFLNK | S_IRWXUGO, false); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + goto out_fname; + } + + err = ubifs_init_security(dir, inode, &dentry->d_name); + if (err) + goto out_inode; + + ui = ubifs_inode(inode); + ui->data = kmalloc(disk_link.len, GFP_NOFS); + if (!ui->data) { + err = -ENOMEM; + goto out_inode; + } + + if (IS_ENCRYPTED(inode)) { + disk_link.name = ui->data; /* encrypt directly into ui->data */ + err = fscrypt_encrypt_symlink(inode, symname, len, &disk_link); + if (err) + goto out_inode; + } else { + memcpy(ui->data, disk_link.name, disk_link.len); + inode->i_link = ui->data; + } + + /* + * The terminating zero byte is not written to the flash media and it + * is put just to make later in-memory string processing simpler. Thus, + * data length is @disk_link.len - 1, not @disk_link.len. + */ + ui->data_len = disk_link.len - 1; + inode->i_size = ubifs_inode(inode)->ui_size = disk_link.len - 1; + set_nlink(inode, 1); + + mutex_lock(&dir_ui->ui_mutex); + dir->i_size += sz_change; + dir_ui->ui_size = dir->i_size; + inode_set_mtime_to_ts(dir, + inode_set_ctime_to_ts(dir, inode_get_ctime(inode))); + err = ubifs_jnl_update(c, dir, &nm, inode, 0, 0, 1); + if (err) + goto out_cancel; + mutex_unlock(&dir_ui->ui_mutex); + + insert_inode_hash(inode); + d_instantiate(dentry, inode); + err = 0; + goto out_fname; + +out_cancel: + dir->i_size -= sz_change; + dir_ui->ui_size = dir->i_size; + mutex_unlock(&dir_ui->ui_mutex); + set_nlink(inode, 0); +out_inode: + /* Free inode->i_link before inode is marked as bad. */ + fscrypt_free_inode(inode); + iput(inode); +out_fname: + fscrypt_free_filename(&nm); +out_budg: + ubifs_release_budget(c, &req); + return err; +} + +/** + * lock_4_inodes - a wrapper for locking three UBIFS inodes. + * @inode1: first inode + * @inode2: second inode + * @inode3: third inode + * @inode4: fourth inode + * + * This function is used for 'ubifs_rename()' and @inode1 may be the same as + * @inode2 whereas @inode3 and @inode4 may be %NULL. + * + * We do not implement any tricks to guarantee strict lock ordering, because + * VFS has already done it for us on the @i_mutex. So this is just a simple + * wrapper function. + */ +static void lock_4_inodes(struct inode *inode1, struct inode *inode2, + struct inode *inode3, struct inode *inode4) +{ + mutex_lock_nested(&ubifs_inode(inode1)->ui_mutex, WB_MUTEX_1); + if (inode2 != inode1) + mutex_lock_nested(&ubifs_inode(inode2)->ui_mutex, WB_MUTEX_2); + if (inode3) + mutex_lock_nested(&ubifs_inode(inode3)->ui_mutex, WB_MUTEX_3); + if (inode4) + mutex_lock_nested(&ubifs_inode(inode4)->ui_mutex, WB_MUTEX_4); +} + +/** + * unlock_4_inodes - a wrapper for unlocking three UBIFS inodes for rename. + * @inode1: first inode + * @inode2: second inode + * @inode3: third inode + * @inode4: fourth inode + */ +static void unlock_4_inodes(struct inode *inode1, struct inode *inode2, + struct inode *inode3, struct inode *inode4) +{ + if (inode4) + mutex_unlock(&ubifs_inode(inode4)->ui_mutex); + if (inode3) + mutex_unlock(&ubifs_inode(inode3)->ui_mutex); + if (inode1 != inode2) + mutex_unlock(&ubifs_inode(inode2)->ui_mutex); + mutex_unlock(&ubifs_inode(inode1)->ui_mutex); +} + +static int do_rename(struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry, + unsigned int flags) +{ + struct ubifs_info *c = old_dir->i_sb->s_fs_info; + struct inode *old_inode = d_inode(old_dentry); + struct inode *new_inode = d_inode(new_dentry); + struct inode *whiteout = NULL; + struct ubifs_inode *old_inode_ui = ubifs_inode(old_inode); + struct ubifs_inode *whiteout_ui = NULL; + int err, release, sync = 0, move = (new_dir != old_dir); + int is_dir = S_ISDIR(old_inode->i_mode); + int unlink = !!new_inode, new_sz, old_sz; + struct ubifs_budget_req req = { .new_dent = 1, .mod_dent = 1, + .dirtied_ino = 3 }; + struct ubifs_budget_req ino_req = { .dirtied_ino = 1, + .dirtied_ino_d = ALIGN(old_inode_ui->data_len, 8) }; + struct ubifs_budget_req wht_req; + unsigned int saved_nlink; + struct fscrypt_name old_nm, new_nm; + + /* + * Budget request settings: + * req: deletion direntry, new direntry, removing the old inode, + * and changing old and new parent directory inodes. + * + * wht_req: new whiteout inode for RENAME_WHITEOUT. + * + * ino_req: marks the target inode as dirty and does not write it. + */ + + dbg_gen("dent '%pd' ino %llu in dir ino %llu to dent '%pd' in dir ino %llu flags 0x%x", + old_dentry, old_inode->i_ino, old_dir->i_ino, + new_dentry, new_dir->i_ino, flags); + + if (unlink) { + ubifs_assert(c, inode_is_locked(new_inode)); + + /* Budget for old inode's data when its nlink > 1. */ + req.dirtied_ino_d = ALIGN(ubifs_inode(new_inode)->data_len, 8); + err = ubifs_purge_xattrs(new_inode); + if (err) + return err; + } + + if (unlink && is_dir) { + err = ubifs_check_dir_empty(new_inode); + if (err) + return err; + } + + err = fscrypt_setup_filename(old_dir, &old_dentry->d_name, 0, &old_nm); + if (err) + return err; + + err = fscrypt_setup_filename(new_dir, &new_dentry->d_name, 0, &new_nm); + if (err) { + fscrypt_free_filename(&old_nm); + return err; + } + + new_sz = CALC_DENT_SIZE(fname_len(&new_nm)); + old_sz = CALC_DENT_SIZE(fname_len(&old_nm)); + + err = ubifs_budget_space(c, &req); + if (err) { + fscrypt_free_filename(&old_nm); + fscrypt_free_filename(&new_nm); + return err; + } + err = ubifs_budget_space(c, &ino_req); + if (err) { + fscrypt_free_filename(&old_nm); + fscrypt_free_filename(&new_nm); + ubifs_release_budget(c, &req); + return err; + } + + if (flags & RENAME_WHITEOUT) { + union ubifs_dev_desc *dev = NULL; + + dev = kmalloc_obj(union ubifs_dev_desc, GFP_NOFS); + if (!dev) { + err = -ENOMEM; + goto out_release; + } + + /* + * The whiteout inode without dentry is pinned in memory, + * umount won't happen during rename process because we + * got parent dentry. + */ + whiteout = create_whiteout(old_dir, old_dentry); + if (IS_ERR(whiteout)) { + err = PTR_ERR(whiteout); + kfree(dev); + goto out_release; + } + + whiteout_ui = ubifs_inode(whiteout); + whiteout_ui->data = dev; + whiteout_ui->data_len = ubifs_encode_dev(dev, MKDEV(0, 0)); + ubifs_assert(c, !whiteout_ui->dirty); + + memset(&wht_req, 0, sizeof(struct ubifs_budget_req)); + wht_req.new_ino = 1; + wht_req.new_ino_d = ALIGN(whiteout_ui->data_len, 8); + /* + * To avoid deadlock between space budget (holds ui_mutex and + * waits wb work) and writeback work(waits ui_mutex), do space + * budget before ubifs inodes locked. + */ + err = ubifs_budget_space(c, &wht_req); + if (err) { + iput(whiteout); + goto out_release; + } + set_nlink(whiteout, 1); + + /* Add the old_dentry size to the old_dir size. */ + old_sz -= CALC_DENT_SIZE(fname_len(&old_nm)); + } + + lock_4_inodes(old_dir, new_dir, new_inode, whiteout); + + /* + * Like most other Unix systems, set the @i_ctime for inodes on a + * rename. + */ + simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry); + + /* We must adjust parent link count when renaming directories */ + if (is_dir) { + if (move) { + /* + * @old_dir loses a link because we are moving + * @old_inode to a different directory. + */ + drop_nlink(old_dir); + /* + * @new_dir only gains a link if we are not also + * overwriting an existing directory. + */ + if (!unlink) + inc_nlink(new_dir); + } else { + /* + * @old_inode is not moving to a different directory, + * but @old_dir still loses a link if we are + * overwriting an existing directory. + */ + if (unlink) + drop_nlink(old_dir); + } + } + + old_dir->i_size -= old_sz; + ubifs_inode(old_dir)->ui_size = old_dir->i_size; + + /* + * And finally, if we unlinked a direntry which happened to have the + * same name as the moved direntry, we have to decrement @i_nlink of + * the unlinked inode. + */ + if (unlink) { + /* + * Directories cannot have hard-links, so if this is a + * directory, just clear @i_nlink. + */ + saved_nlink = new_inode->i_nlink; + if (is_dir) + clear_nlink(new_inode); + else + drop_nlink(new_inode); + } else { + new_dir->i_size += new_sz; + ubifs_inode(new_dir)->ui_size = new_dir->i_size; + } + + /* + * Do not ask 'ubifs_jnl_rename()' to flush write-buffer if @old_inode + * is dirty, because this will be done later on at the end of + * 'ubifs_rename()'. + */ + if (IS_SYNC(old_inode)) { + sync = IS_DIRSYNC(old_dir) || IS_DIRSYNC(new_dir); + if (unlink && IS_SYNC(new_inode)) + sync = 1; + /* + * S_SYNC flag of whiteout inherits from the old_dir, and we + * have already checked the old dir inode. So there is no need + * to check whiteout. + */ + } + + err = ubifs_jnl_rename(c, old_dir, old_inode, &old_nm, new_dir, + new_inode, &new_nm, whiteout, sync, !!whiteout); + if (err) + goto out_cancel; + + unlock_4_inodes(old_dir, new_dir, new_inode, whiteout); + ubifs_release_budget(c, &req); + + if (whiteout) { + ubifs_release_budget(c, &wht_req); + iput(whiteout); + } + + mutex_lock(&old_inode_ui->ui_mutex); + release = old_inode_ui->dirty; + mark_inode_dirty_sync(old_inode); + mutex_unlock(&old_inode_ui->ui_mutex); + + if (release) + ubifs_release_budget(c, &ino_req); + if (IS_SYNC(old_inode)) + /* + * Rename finished here. Although old inode cannot be updated + * on flash, old ctime is not a big problem, don't return err + * code to userspace. + */ + old_inode->i_sb->s_op->write_inode(old_inode, NULL); + + fscrypt_free_filename(&old_nm); + fscrypt_free_filename(&new_nm); + return 0; + +out_cancel: + if (unlink) { + set_nlink(new_inode, saved_nlink); + } else { + new_dir->i_size -= new_sz; + ubifs_inode(new_dir)->ui_size = new_dir->i_size; + } + old_dir->i_size += old_sz; + ubifs_inode(old_dir)->ui_size = old_dir->i_size; + if (is_dir) { + if (move) { + inc_nlink(old_dir); + if (!unlink) + drop_nlink(new_dir); + } else { + if (unlink) + inc_nlink(old_dir); + } + } + unlock_4_inodes(old_dir, new_dir, new_inode, whiteout); + if (whiteout) { + ubifs_release_budget(c, &wht_req); + set_nlink(whiteout, 0); + iput(whiteout); + } +out_release: + ubifs_release_budget(c, &ino_req); + ubifs_release_budget(c, &req); + fscrypt_free_filename(&old_nm); + fscrypt_free_filename(&new_nm); + return err; +} + +static int ubifs_xrename(struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry) +{ + struct ubifs_info *c = old_dir->i_sb->s_fs_info; + struct ubifs_budget_req req = { .new_dent = 1, .mod_dent = 1, + .dirtied_ino = 2 }; + int sync = IS_DIRSYNC(old_dir) || IS_DIRSYNC(new_dir); + struct inode *fst_inode = d_inode(old_dentry); + struct inode *snd_inode = d_inode(new_dentry); + int err; + struct fscrypt_name fst_nm, snd_nm; + + ubifs_assert(c, fst_inode && snd_inode); + + /* + * Budget request settings: changing two direntries, changing the two + * parent directory inodes. + */ + + dbg_gen("dent '%pd' ino %llu in dir ino %llu exchange dent '%pd' ino %llu in dir ino %llu", + old_dentry, fst_inode->i_ino, old_dir->i_ino, + new_dentry, snd_inode->i_ino, new_dir->i_ino); + + err = fscrypt_setup_filename(old_dir, &old_dentry->d_name, 0, &fst_nm); + if (err) + return err; + + err = fscrypt_setup_filename(new_dir, &new_dentry->d_name, 0, &snd_nm); + if (err) { + fscrypt_free_filename(&fst_nm); + return err; + } + + err = ubifs_budget_space(c, &req); + if (err) + goto out; + + lock_4_inodes(old_dir, new_dir, NULL, NULL); + + simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry); + + if (old_dir != new_dir) { + if (S_ISDIR(fst_inode->i_mode) && !S_ISDIR(snd_inode->i_mode)) { + inc_nlink(new_dir); + drop_nlink(old_dir); + } + else if (!S_ISDIR(fst_inode->i_mode) && S_ISDIR(snd_inode->i_mode)) { + drop_nlink(new_dir); + inc_nlink(old_dir); + } + } + + err = ubifs_jnl_xrename(c, old_dir, fst_inode, &fst_nm, new_dir, + snd_inode, &snd_nm, sync); + + unlock_4_inodes(old_dir, new_dir, NULL, NULL); + ubifs_release_budget(c, &req); + +out: + fscrypt_free_filename(&fst_nm); + fscrypt_free_filename(&snd_nm); + return err; +} + +static int ubifs_rename(struct mnt_idmap *idmap, + struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry, + unsigned int flags) +{ + int err; + struct ubifs_info *c = old_dir->i_sb->s_fs_info; + + if (flags & ~(RENAME_NOREPLACE | RENAME_WHITEOUT | RENAME_EXCHANGE)) + return -EINVAL; + + ubifs_assert(c, inode_is_locked(old_dir)); + ubifs_assert(c, inode_is_locked(new_dir)); + + err = fscrypt_prepare_rename(old_dir, old_dentry, new_dir, new_dentry, + flags); + if (err) + return err; + + if (flags & RENAME_EXCHANGE) + return ubifs_xrename(old_dir, old_dentry, new_dir, new_dentry); + + return do_rename(old_dir, old_dentry, new_dir, new_dentry, flags); +} + +int ubifs_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags) +{ + loff_t size; + struct inode *inode = d_inode(path->dentry); + struct ubifs_inode *ui = ubifs_inode(inode); + + mutex_lock(&ui->ui_mutex); + + if (ui->flags & UBIFS_APPEND_FL) + stat->attributes |= STATX_ATTR_APPEND; + if (ui->flags & UBIFS_COMPR_FL) + stat->attributes |= STATX_ATTR_COMPRESSED; + if (ui->flags & UBIFS_CRYPT_FL) + stat->attributes |= STATX_ATTR_ENCRYPTED; + if (ui->flags & UBIFS_IMMUTABLE_FL) + stat->attributes |= STATX_ATTR_IMMUTABLE; + + stat->attributes_mask |= (STATX_ATTR_APPEND | + STATX_ATTR_COMPRESSED | + STATX_ATTR_ENCRYPTED | + STATX_ATTR_IMMUTABLE); + + generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat); + stat->blksize = UBIFS_BLOCK_SIZE; + stat->size = ui->ui_size; + + /* + * Unfortunately, the 'stat()' system call was designed for block + * device based file systems, and it is not appropriate for UBIFS, + * because UBIFS does not have notion of "block". For example, it is + * difficult to tell how many block a directory takes - it actually + * takes less than 300 bytes, but we have to round it to block size, + * which introduces large mistake. This makes utilities like 'du' to + * report completely senseless numbers. This is the reason why UBIFS + * goes the same way as JFFS2 - it reports zero blocks for everything + * but regular files, which makes more sense than reporting completely + * wrong sizes. + */ + if (S_ISREG(inode->i_mode)) { + size = ui->xattr_size; + size += stat->size; + size = ALIGN(size, UBIFS_BLOCK_SIZE); + /* + * Note, user-space expects 512-byte blocks count irrespectively + * of what was reported in @stat->size. + */ + stat->blocks = size >> 9; + } else + stat->blocks = 0; + mutex_unlock(&ui->ui_mutex); + return 0; +} + +static int ubifs_dir_open(struct inode *inode, struct file *file) +{ + struct ubifs_dir_data *data; + + data = kzalloc_obj(struct ubifs_dir_data); + if (!data) + return -ENOMEM; + file->private_data = data; + return 0; +} + +static loff_t ubifs_dir_llseek(struct file *file, loff_t offset, int whence) +{ + struct ubifs_dir_data *data = file->private_data; + + return generic_llseek_cookie(file, offset, whence, &data->cookie); +} + +const struct inode_operations ubifs_dir_inode_operations = { + .lookup = ubifs_lookup, + .create = ubifs_create, + .link = ubifs_link, + .symlink = ubifs_symlink, + .unlink = ubifs_unlink, + .mkdir = ubifs_mkdir, + .rmdir = ubifs_rmdir, + .mknod = ubifs_mknod, + .rename = ubifs_rename, + .setattr = ubifs_setattr, + .getattr = ubifs_getattr, + .listxattr = ubifs_listxattr, + .update_time = ubifs_update_time, + .tmpfile = ubifs_tmpfile, + .fileattr_get = ubifs_fileattr_get, + .fileattr_set = ubifs_fileattr_set, +}; + +const struct file_operations ubifs_dir_operations = { + .open = ubifs_dir_open, + .llseek = ubifs_dir_llseek, + .release = ubifs_dir_release, + .read = generic_read_dir, + .iterate_shared = ubifs_readdir, + .fsync = ubifs_fsync, + .unlocked_ioctl = ubifs_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = ubifs_compat_ioctl, +#endif +}; diff --git a/fs/ubifs/file.c b/fs/ubifs/file.c new file mode 100644 index 000000000..e73c28b12 --- /dev/null +++ b/fs/ubifs/file.c @@ -0,0 +1,1666 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * + * Authors: Artem Bityutskiy (Битюцкий Артём) + * Adrian Hunter + */ + +/* + * This file implements VFS file and inode operations for regular files, device + * nodes and symlinks as well as address space operations. + * + * UBIFS uses 2 page flags: @PG_private and @PG_checked. @PG_private is set if + * the page is dirty and is used for optimization purposes - dirty pages are + * not budgeted so the flag shows that 'ubifs_write_end()' should not release + * the budget for this page. The @PG_checked flag is set if full budgeting is + * required for the page e.g., when it corresponds to a file hole or it is + * beyond the file size. The budgeting is done in 'ubifs_write_begin()', because + * it is OK to fail in this function, and the budget is released in + * 'ubifs_write_end()'. So the @PG_private and @PG_checked flags carry + * information about how the page was budgeted, to make it possible to release + * the budget properly. + * + * A thing to keep in mind: inode @i_mutex is locked in most VFS operations we + * implement. However, this is not true for 'ubifs_writepage()', which may be + * called with @i_mutex unlocked. For example, when flusher thread is doing + * background write-back, it calls 'ubifs_writepage()' with unlocked @i_mutex. + * At "normal" work-paths the @i_mutex is locked in 'ubifs_writepage()', e.g. + * in the "sys_write -> alloc_pages -> direct reclaim path". So, in + * 'ubifs_writepage()' we are only guaranteed that the page is locked. + * + * Similarly, @i_mutex is not always locked in 'ubifs_read_folio()', e.g., the + * read-ahead path does not lock it ("sys_read -> generic_file_aio_read -> + * ondemand_readahead -> read_folio"). In case of readahead, @I_SYNC flag is not + * set as well. However, UBIFS disables readahead. + */ + +#include "ubifs.h" +#include +#include +#include + +static int read_block(struct inode *inode, struct folio *folio, size_t offset, + unsigned int block, struct ubifs_data_node *dn) +{ + struct ubifs_info *c = inode->i_sb->s_fs_info; + int err, len, out_len; + union ubifs_key key; + unsigned int dlen; + + data_key_init(c, &key, inode->i_ino, block); + err = ubifs_tnc_lookup(c, &key, dn); + if (err) { + if (err == -ENOENT) + /* Not found, so it must be a hole */ + folio_zero_range(folio, offset, UBIFS_BLOCK_SIZE); + return err; + } + + ubifs_assert(c, le64_to_cpu(dn->ch.sqnum) > + ubifs_inode(inode)->creat_sqnum); + len = le32_to_cpu(dn->size); + if (len <= 0 || len > UBIFS_BLOCK_SIZE) + goto dump; + + dlen = le32_to_cpu(dn->ch.len) - UBIFS_DATA_NODE_SZ; + + if (IS_ENCRYPTED(inode)) { + err = ubifs_decrypt(inode, dn, &dlen, block); + if (err) + goto dump; + } + + out_len = UBIFS_BLOCK_SIZE; + err = ubifs_decompress_folio(c, &dn->data, dlen, folio, offset, + &out_len, le16_to_cpu(dn->compr_type)); + if (err || len != out_len) + goto dump; + + /* + * Data length can be less than a full block, even for blocks that are + * not the last in the file (e.g., as a result of making a hole and + * appending data). Ensure that the remainder is zeroed out. + */ + if (len < UBIFS_BLOCK_SIZE) + folio_zero_range(folio, offset + len, UBIFS_BLOCK_SIZE - len); + + return 0; + +dump: + ubifs_err(c, "bad data node (block %u, inode %llu)", + block, inode->i_ino); + ubifs_dump_node(c, dn, UBIFS_MAX_DATA_NODE_SZ); + return -EINVAL; +} + +static int do_readpage(struct folio *folio) +{ + int err = 0, i; + unsigned int block, beyond; + struct ubifs_data_node *dn = NULL; + struct inode *inode = folio->mapping->host; + struct ubifs_info *c = inode->i_sb->s_fs_info; + loff_t i_size = i_size_read(inode); + size_t offset = 0; + + dbg_gen("ino %llu, pg %lu, i_size %lld, flags %#lx", + inode->i_ino, folio->index, i_size, folio->flags.f); + ubifs_assert(c, !folio_test_checked(folio)); + ubifs_assert(c, !folio->private); + + block = folio->index << UBIFS_BLOCKS_PER_PAGE_SHIFT; + beyond = (i_size + UBIFS_BLOCK_SIZE - 1) >> UBIFS_BLOCK_SHIFT; + if (block >= beyond) { + /* Reading beyond inode */ + folio_set_checked(folio); + folio_zero_range(folio, 0, folio_size(folio)); + goto out; + } + + dn = kmalloc(UBIFS_MAX_DATA_NODE_SZ, GFP_NOFS); + if (!dn) { + err = -ENOMEM; + goto out; + } + + i = 0; + while (1) { + int ret; + + if (block >= beyond) { + /* Reading beyond inode */ + err = -ENOENT; + folio_zero_range(folio, offset, UBIFS_BLOCK_SIZE); + } else { + ret = read_block(inode, folio, offset, block, dn); + if (ret) { + err = ret; + if (err != -ENOENT) + break; + } else if (block + 1 == beyond) { + int dlen = le32_to_cpu(dn->size); + int ilen = i_size & (UBIFS_BLOCK_SIZE - 1); + + if (ilen && ilen < dlen) + folio_zero_range(folio, offset + ilen, dlen - ilen); + } + } + if (++i >= (UBIFS_BLOCKS_PER_PAGE << folio_order(folio))) + break; + block += 1; + offset += UBIFS_BLOCK_SIZE; + } + + if (err) { + struct ubifs_info *c = inode->i_sb->s_fs_info; + if (err == -ENOENT) { + /* Not found, so it must be a hole */ + folio_set_checked(folio); + dbg_gen("hole"); + err = 0; + } else { + ubifs_err(c, "cannot read page %lu of inode %llu, error %d", + folio->index, inode->i_ino, err); + } + } + +out: + kfree(dn); + if (!err) + folio_mark_uptodate(folio); + return err; +} + +/** + * release_new_page_budget - release budget of a new page. + * @c: UBIFS file-system description object + * + * This is a helper function which releases budget corresponding to the budget + * of one new page of data. + */ +static void release_new_page_budget(struct ubifs_info *c) +{ + struct ubifs_budget_req req = { .recalculate = 1, .new_page = 1 }; + + ubifs_release_budget(c, &req); +} + +/** + * release_existing_page_budget - release budget of an existing page. + * @c: UBIFS file-system description object + * + * This is a helper function which releases budget corresponding to the budget + * of changing one page of data which already exists on the flash media. + */ +static void release_existing_page_budget(struct ubifs_info *c) +{ + struct ubifs_budget_req req = { .dd_growth = c->bi.page_budget}; + + ubifs_release_budget(c, &req); +} + +static int write_begin_slow(struct address_space *mapping, + loff_t pos, unsigned len, struct folio **foliop) +{ + struct inode *inode = mapping->host; + struct ubifs_info *c = inode->i_sb->s_fs_info; + pgoff_t index = pos >> PAGE_SHIFT; + struct ubifs_budget_req req = { .new_page = 1 }; + int err, appending = !!(pos + len > inode->i_size); + struct folio *folio; + + dbg_gen("ino %llu, pos %llu, len %u, i_size %lld", + inode->i_ino, pos, len, inode->i_size); + + /* + * At the slow path we have to budget before locking the folio, because + * budgeting may force write-back, which would wait on locked folios and + * deadlock if we had the folio locked. At this point we do not know + * anything about the folio, so assume that this is a new folio which is + * written to a hole. This corresponds to largest budget. Later the + * budget will be amended if this is not true. + */ + if (appending) + /* We are appending data, budget for inode change */ + req.dirtied_ino = 1; + + err = ubifs_budget_space(c, &req); + if (unlikely(err)) + return err; + + folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN, + mapping_gfp_mask(mapping)); + if (IS_ERR(folio)) { + ubifs_release_budget(c, &req); + return PTR_ERR(folio); + } + + if (!folio_test_uptodate(folio)) { + if (pos == folio_pos(folio) && len >= folio_size(folio)) + folio_set_checked(folio); + else { + err = do_readpage(folio); + if (err) { + folio_unlock(folio); + folio_put(folio); + ubifs_release_budget(c, &req); + return err; + } + } + } + + if (folio->private) + /* + * The folio is dirty, which means it was budgeted twice: + * o first time the budget was allocated by the task which + * made the folio dirty and set the private field; + * o and then we budgeted for it for the second time at the + * very beginning of this function. + * + * So what we have to do is to release the folio budget we + * allocated. + */ + release_new_page_budget(c); + else if (!folio_test_checked(folio)) + /* + * We are changing a folio which already exists on the media. + * This means that changing the folio does not make the amount + * of indexing information larger, and this part of the budget + * which we have already acquired may be released. + */ + ubifs_convert_page_budget(c); + + if (appending) { + struct ubifs_inode *ui = ubifs_inode(inode); + + /* + * 'ubifs_write_end()' is optimized from the fast-path part of + * 'ubifs_write_begin()' and expects the @ui_mutex to be locked + * if data is appended. + */ + mutex_lock(&ui->ui_mutex); + if (ui->dirty) + /* + * The inode is dirty already, so we may free the + * budget we allocated. + */ + ubifs_release_dirty_inode_budget(c, ui); + } + + *foliop = folio; + return 0; +} + +/** + * allocate_budget - allocate budget for 'ubifs_write_begin()'. + * @c: UBIFS file-system description object + * @folio: folio to allocate budget for + * @ui: UBIFS inode object the page belongs to + * @appending: non-zero if the page is appended + * + * This is a helper function for 'ubifs_write_begin()' which allocates budget + * for the operation. The budget is allocated differently depending on whether + * this is appending, whether the page is dirty or not, and so on. This + * function leaves the @ui->ui_mutex locked in case of appending. + * + * Returns: %0 in case of success and %-ENOSPC in case of failure. + */ +static int allocate_budget(struct ubifs_info *c, struct folio *folio, + struct ubifs_inode *ui, int appending) +{ + struct ubifs_budget_req req = { .fast = 1 }; + + if (folio->private) { + if (!appending) + /* + * The folio is dirty and we are not appending, which + * means no budget is needed at all. + */ + return 0; + + mutex_lock(&ui->ui_mutex); + if (ui->dirty) + /* + * The page is dirty and we are appending, so the inode + * has to be marked as dirty. However, it is already + * dirty, so we do not need any budget. We may return, + * but @ui->ui_mutex hast to be left locked because we + * should prevent write-back from flushing the inode + * and freeing the budget. The lock will be released in + * 'ubifs_write_end()'. + */ + return 0; + + /* + * The page is dirty, we are appending, the inode is clean, so + * we need to budget the inode change. + */ + req.dirtied_ino = 1; + } else { + if (folio_test_checked(folio)) + /* + * The page corresponds to a hole and does not + * exist on the media. So changing it makes + * the amount of indexing information + * larger, and we have to budget for a new + * page. + */ + req.new_page = 1; + else + /* + * Not a hole, the change will not add any new + * indexing information, budget for page + * change. + */ + req.dirtied_page = 1; + + if (appending) { + mutex_lock(&ui->ui_mutex); + if (!ui->dirty) + /* + * The inode is clean but we will have to mark + * it as dirty because we are appending. This + * needs a budget. + */ + req.dirtied_ino = 1; + } + } + + return ubifs_budget_space(c, &req); +} + +/* + * This function is called when a page of data is going to be written. Since + * the page of data will not necessarily go to the flash straight away, UBIFS + * has to reserve space on the media for it, which is done by means of + * budgeting. + * + * This is the hot-path of the file-system and we are trying to optimize it as + * much as possible. For this reasons it is split on 2 parts - slow and fast. + * + * There many budgeting cases: + * o a new page is appended - we have to budget for a new page and for + * changing the inode; however, if the inode is already dirty, there is + * no need to budget for it; + * o an existing clean page is changed - we have budget for it; if the page + * does not exist on the media (a hole), we have to budget for a new + * page; otherwise, we may budget for changing an existing page; the + * difference between these cases is that changing an existing page does + * not introduce anything new to the FS indexing information, so it does + * not grow, and smaller budget is acquired in this case; + * o an existing dirty page is changed - no need to budget at all, because + * the page budget has been acquired by earlier, when the page has been + * marked dirty. + * + * UBIFS budgeting sub-system may force write-back if it thinks there is no + * space to reserve. This imposes some locking restrictions and makes it + * impossible to take into account the above cases, and makes it impossible to + * optimize budgeting. + * + * The solution for this is that the fast path of 'ubifs_write_begin()' assumes + * there is a plenty of flash space and the budget will be acquired quickly, + * without forcing write-back. The slow path does not make this assumption. + */ +static int ubifs_write_begin(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, + struct folio **foliop, void **fsdata) +{ + struct inode *inode = mapping->host; + struct ubifs_info *c = inode->i_sb->s_fs_info; + struct ubifs_inode *ui = ubifs_inode(inode); + pgoff_t index = pos >> PAGE_SHIFT; + int err, appending = !!(pos + len > inode->i_size); + int skipped_read = 0; + struct folio *folio; + + ubifs_assert(c, ubifs_inode(inode)->ui_size == inode->i_size); + ubifs_assert(c, !c->ro_media && !c->ro_mount); + + if (unlikely(c->ro_error)) + return -EROFS; + + /* Try out the fast-path part first */ + folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN, + mapping_gfp_mask(mapping)); + if (IS_ERR(folio)) + return PTR_ERR(folio); + + if (!folio_test_uptodate(folio)) { + /* The page is not loaded from the flash */ + if (pos == folio_pos(folio) && len >= folio_size(folio)) { + /* + * We change whole page so no need to load it. But we + * do not know whether this page exists on the media or + * not, so we assume the latter because it requires + * larger budget. The assumption is that it is better + * to budget a bit more than to read the page from the + * media. Thus, we are setting the @PG_checked flag + * here. + */ + folio_set_checked(folio); + skipped_read = 1; + } else { + err = do_readpage(folio); + if (err) { + folio_unlock(folio); + folio_put(folio); + return err; + } + } + } + + err = allocate_budget(c, folio, ui, appending); + if (unlikely(err)) { + ubifs_assert(c, err == -ENOSPC); + /* + * If we skipped reading the page because we were going to + * write all of it, then it is not up to date. + */ + if (skipped_read) + folio_clear_checked(folio); + /* + * Budgeting failed which means it would have to force + * write-back but didn't, because we set the @fast flag in the + * request. Write-back cannot be done now, while we have the + * page locked, because it would deadlock. Unlock and free + * everything and fall-back to slow-path. + */ + if (appending) { + ubifs_assert(c, mutex_is_locked(&ui->ui_mutex)); + mutex_unlock(&ui->ui_mutex); + } + folio_unlock(folio); + folio_put(folio); + + return write_begin_slow(mapping, pos, len, foliop); + } + + /* + * Whee, we acquired budgeting quickly - without involving + * garbage-collection, committing or forcing write-back. We return + * with @ui->ui_mutex locked if we are appending pages, and unlocked + * otherwise. This is an optimization (slightly hacky though). + */ + *foliop = folio; + return 0; +} + +/** + * cancel_budget - cancel budget. + * @c: UBIFS file-system description object + * @folio: folio to cancel budget for + * @ui: UBIFS inode object the page belongs to + * @appending: non-zero if the page is appended + * + * This is a helper function for a page write operation. It unlocks the + * @ui->ui_mutex in case of appending. + */ +static void cancel_budget(struct ubifs_info *c, struct folio *folio, + struct ubifs_inode *ui, int appending) +{ + if (appending) { + if (!ui->dirty) + ubifs_release_dirty_inode_budget(c, ui); + mutex_unlock(&ui->ui_mutex); + } + if (!folio->private) { + if (folio_test_checked(folio)) + release_new_page_budget(c); + else + release_existing_page_budget(c); + } +} + +static int ubifs_write_end(const struct kiocb *iocb, + struct address_space *mapping, loff_t pos, + unsigned len, unsigned copied, + struct folio *folio, void *fsdata) +{ + struct inode *inode = mapping->host; + struct ubifs_inode *ui = ubifs_inode(inode); + struct ubifs_info *c = inode->i_sb->s_fs_info; + loff_t end_pos = pos + len; + int appending = !!(end_pos > inode->i_size); + + dbg_gen("ino %llu, pos %llu, pg %lu, len %u, copied %d, i_size %lld", + inode->i_ino, pos, folio->index, len, copied, inode->i_size); + + if (unlikely(copied < len && !folio_test_uptodate(folio))) { + /* + * VFS copied less data to the folio than it intended and + * declared in its '->write_begin()' call via the @len + * argument. If the folio was not up-to-date, + * the 'ubifs_write_begin()' function did + * not load it from the media (for optimization reasons). This + * means that part of the folio contains garbage. So read the + * folio now. + */ + dbg_gen("copied %d instead of %d, read page and repeat", + copied, len); + cancel_budget(c, folio, ui, appending); + folio_clear_checked(folio); + + /* + * Return 0 to force VFS to repeat the whole operation, or the + * error code if 'do_readpage()' fails. + */ + copied = do_readpage(folio); + goto out; + } + + if (len == folio_size(folio)) + folio_mark_uptodate(folio); + + if (!folio->private) { + folio_attach_private(folio, (void *)1); + atomic_long_inc(&c->dirty_pg_cnt); + filemap_dirty_folio(mapping, folio); + } + + if (appending) { + i_size_write(inode, end_pos); + ui->ui_size = end_pos; + /* + * We do not set @I_DIRTY_PAGES (which means that + * the inode has dirty pages), this was done in + * filemap_dirty_folio(). + */ + __mark_inode_dirty(inode, I_DIRTY_DATASYNC); + ubifs_assert(c, mutex_is_locked(&ui->ui_mutex)); + mutex_unlock(&ui->ui_mutex); + } + +out: + folio_unlock(folio); + folio_put(folio); + return copied; +} + +/** + * populate_page - copy data nodes into a page for bulk-read. + * @c: UBIFS file-system description object + * @folio: folio + * @bu: bulk-read information + * @n: next zbranch slot + * + * Returns: %0 on success and a negative error code on failure. + */ +static int populate_page(struct ubifs_info *c, struct folio *folio, + struct bu_info *bu, int *n) +{ + int i = 0, nn = *n, offs = bu->zbranch[0].offs, hole = 0, read = 0; + struct inode *inode = folio->mapping->host; + loff_t i_size = i_size_read(inode); + unsigned int page_block; + size_t offset = 0; + pgoff_t end_index; + + dbg_gen("ino %llu, pg %lu, i_size %lld, flags %#lx", + inode->i_ino, folio->index, i_size, folio->flags.f); + + end_index = (i_size - 1) >> PAGE_SHIFT; + if (!i_size || folio->index > end_index) { + hole = 1; + folio_zero_range(folio, 0, folio_size(folio)); + goto out_hole; + } + + page_block = folio->index << UBIFS_BLOCKS_PER_PAGE_SHIFT; + while (1) { + int err, len, out_len, dlen; + + if (nn >= bu->cnt) { + hole = 1; + folio_zero_range(folio, offset, UBIFS_BLOCK_SIZE); + } else if (key_block(c, &bu->zbranch[nn].key) == page_block) { + struct ubifs_data_node *dn; + + dn = bu->buf + (bu->zbranch[nn].offs - offs); + + ubifs_assert(c, le64_to_cpu(dn->ch.sqnum) > + ubifs_inode(inode)->creat_sqnum); + + len = le32_to_cpu(dn->size); + if (len <= 0 || len > UBIFS_BLOCK_SIZE) + goto out_err; + + dlen = le32_to_cpu(dn->ch.len) - UBIFS_DATA_NODE_SZ; + out_len = UBIFS_BLOCK_SIZE; + + if (IS_ENCRYPTED(inode)) { + err = ubifs_decrypt(inode, dn, &dlen, page_block); + if (err) + goto out_err; + } + + err = ubifs_decompress_folio( + c, &dn->data, dlen, folio, offset, &out_len, + le16_to_cpu(dn->compr_type)); + if (err || len != out_len) + goto out_err; + + if (len < UBIFS_BLOCK_SIZE) + folio_zero_range(folio, offset + len, + UBIFS_BLOCK_SIZE - len); + + nn += 1; + read = (i << UBIFS_BLOCK_SHIFT) + len; + } else if (key_block(c, &bu->zbranch[nn].key) < page_block) { + nn += 1; + continue; + } else { + hole = 1; + folio_zero_range(folio, offset, UBIFS_BLOCK_SIZE); + } + if (++i >= UBIFS_BLOCKS_PER_PAGE) + break; + offset += UBIFS_BLOCK_SIZE; + page_block += 1; + } + + if (end_index == folio->index) { + int len = i_size & (PAGE_SIZE - 1); + + if (len && len < read) + folio_zero_range(folio, len, read - len); + } + +out_hole: + if (hole) { + folio_set_checked(folio); + dbg_gen("hole"); + } + + folio_mark_uptodate(folio); + *n = nn; + return 0; + +out_err: + ubifs_err(c, "bad data node (block %u, inode %llu)", + page_block, inode->i_ino); + return -EINVAL; +} + +/** + * ubifs_do_bulk_read - do bulk-read. + * @c: UBIFS file-system description object + * @bu: bulk-read information + * @folio1: first folio to read + * + * Returns: %1 if the bulk-read is done, otherwise %0 is returned. + */ +static int ubifs_do_bulk_read(struct ubifs_info *c, struct bu_info *bu, + struct folio *folio1) +{ + pgoff_t offset = folio1->index, end_index; + struct address_space *mapping = folio1->mapping; + struct inode *inode = mapping->host; + struct ubifs_inode *ui = ubifs_inode(inode); + int err, page_idx, page_cnt, ret = 0, n = 0; + int allocate = bu->buf ? 0 : 1; + loff_t isize; + gfp_t ra_gfp_mask = readahead_gfp_mask(mapping) & ~__GFP_FS; + + err = ubifs_tnc_get_bu_keys(c, bu); + if (err) + goto out_warn; + + if (bu->eof) { + /* Turn off bulk-read at the end of the file */ + ui->read_in_a_row = 1; + ui->bulk_read = 0; + } + + page_cnt = bu->blk_cnt >> UBIFS_BLOCKS_PER_PAGE_SHIFT; + if (!page_cnt) { + /* + * This happens when there are multiple blocks per page and the + * blocks for the first page we are looking for, are not + * together. If all the pages were like this, bulk-read would + * reduce performance, so we turn it off for a while. + */ + goto out_bu_off; + } + + if (bu->cnt) { + if (allocate) { + /* + * Allocate bulk-read buffer depending on how many data + * nodes we are going to read. + */ + bu->buf_len = bu->zbranch[bu->cnt - 1].offs + + bu->zbranch[bu->cnt - 1].len - + bu->zbranch[0].offs; + ubifs_assert(c, bu->buf_len > 0); + ubifs_assert(c, bu->buf_len <= c->leb_size); + bu->buf = kmalloc(bu->buf_len, GFP_NOFS | __GFP_NOWARN); + if (!bu->buf) + goto out_bu_off; + } + + err = ubifs_tnc_bulk_read(c, bu); + if (err) + goto out_warn; + } + + err = populate_page(c, folio1, bu, &n); + if (err) + goto out_warn; + + folio_unlock(folio1); + ret = 1; + + isize = i_size_read(inode); + if (isize == 0) + goto out_free; + end_index = ((isize - 1) >> PAGE_SHIFT); + + for (page_idx = 1; page_idx < page_cnt; page_idx++) { + pgoff_t page_offset = offset + page_idx; + struct folio *folio; + + if (page_offset > end_index) + break; + folio = __filemap_get_folio(mapping, page_offset, + FGP_LOCK|FGP_ACCESSED|FGP_CREAT|FGP_NOWAIT, + ra_gfp_mask); + if (IS_ERR(folio)) + break; + if (!folio_test_uptodate(folio)) + err = populate_page(c, folio, bu, &n); + folio_unlock(folio); + folio_put(folio); + if (err) + break; + } + + ui->last_page_read = offset + page_idx - 1; + +out_free: + if (allocate) + kfree(bu->buf); + return ret; + +out_warn: + ubifs_warn(c, "ignoring error %d and skipping bulk-read", err); + goto out_free; + +out_bu_off: + ui->read_in_a_row = ui->bulk_read = 0; + goto out_free; +} + +/** + * ubifs_bulk_read - determine whether to bulk-read and, if so, do it. + * @folio: folio from which to start bulk-read. + * + * Some flash media are capable of reading sequentially at faster rates. UBIFS + * bulk-read facility is designed to take advantage of that, by reading in one + * go consecutive data nodes that are also located consecutively in the same + * LEB. + * + * Returns: %1 if a bulk-read is done and %0 otherwise. + */ +static int ubifs_bulk_read(struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + struct ubifs_info *c = inode->i_sb->s_fs_info; + struct ubifs_inode *ui = ubifs_inode(inode); + pgoff_t index = folio->index, last_page_read = ui->last_page_read; + struct bu_info *bu; + int err = 0, allocated = 0; + + ui->last_page_read = index; + if (!c->bulk_read) + return 0; + + /* + * Bulk-read is protected by @ui->ui_mutex, but it is an optimization, + * so don't bother if we cannot lock the mutex. + */ + if (!mutex_trylock(&ui->ui_mutex)) + return 0; + + if (index != last_page_read + 1) { + /* Turn off bulk-read if we stop reading sequentially */ + ui->read_in_a_row = 1; + if (ui->bulk_read) + ui->bulk_read = 0; + goto out_unlock; + } + + if (!ui->bulk_read) { + ui->read_in_a_row += 1; + if (ui->read_in_a_row < 3) + goto out_unlock; + /* Three reads in a row, so switch on bulk-read */ + ui->bulk_read = 1; + } + + /* + * If possible, try to use pre-allocated bulk-read information, which + * is protected by @c->bu_mutex. + */ + if (mutex_trylock(&c->bu_mutex)) + bu = &c->bu; + else { + bu = kmalloc_obj(struct bu_info, GFP_NOFS | __GFP_NOWARN); + if (!bu) + goto out_unlock; + + bu->buf = NULL; + allocated = 1; + } + + bu->buf_len = c->max_bu_buf_len; + data_key_init(c, &bu->key, inode->i_ino, + folio->index << UBIFS_BLOCKS_PER_PAGE_SHIFT); + err = ubifs_do_bulk_read(c, bu, folio); + + if (!allocated) + mutex_unlock(&c->bu_mutex); + else + kfree(bu); + +out_unlock: + mutex_unlock(&ui->ui_mutex); + return err; +} + +static int ubifs_read_folio(struct file *file, struct folio *folio) +{ + if (ubifs_bulk_read(folio)) + return 0; + do_readpage(folio); + folio_unlock(folio); + return 0; +} + +static int do_writepage(struct folio *folio, size_t len) +{ + int err = 0, blen; + unsigned int block; + size_t offset = 0; + union ubifs_key key; + struct inode *inode = folio->mapping->host; + struct ubifs_info *c = inode->i_sb->s_fs_info; + +#ifdef UBIFS_DEBUG + struct ubifs_inode *ui = ubifs_inode(inode); + spin_lock(&ui->ui_lock); + ubifs_assert(c, folio->index <= ui->synced_i_size >> PAGE_SHIFT); + spin_unlock(&ui->ui_lock); +#endif + + folio_start_writeback(folio); + + block = folio->index << UBIFS_BLOCKS_PER_PAGE_SHIFT; + for (;;) { + blen = min_t(size_t, len, UBIFS_BLOCK_SIZE); + data_key_init(c, &key, inode->i_ino, block); + err = ubifs_jnl_write_data(c, inode, &key, folio, offset, blen); + if (err) + break; + len -= blen; + if (!len) + break; + block += 1; + offset += blen; + } + if (err) { + mapping_set_error(folio->mapping, err); + ubifs_err(c, "cannot write folio %lu of inode %llu, error %d", + folio->index, inode->i_ino, err); + ubifs_ro_mode(c, err); + } + + ubifs_assert(c, folio->private != NULL); + if (folio_test_checked(folio)) + release_new_page_budget(c); + else + release_existing_page_budget(c); + + atomic_long_dec(&c->dirty_pg_cnt); + folio_detach_private(folio); + folio_clear_checked(folio); + + folio_unlock(folio); + folio_end_writeback(folio); + return err; +} + +/* + * When writing-back dirty inodes, VFS first writes-back pages belonging to the + * inode, then the inode itself. For UBIFS this may cause a problem. Consider a + * situation when a we have an inode with size 0, then a megabyte of data is + * appended to the inode, then write-back starts and flushes some amount of the + * dirty pages, the journal becomes full, commit happens and finishes, and then + * an unclean reboot happens. When the file system is mounted next time, the + * inode size would still be 0, but there would be many pages which are beyond + * the inode size, they would be indexed and consume flash space. Because the + * journal has been committed, the replay would not be able to detect this + * situation and correct the inode size. This means UBIFS would have to scan + * whole index and correct all inode sizes, which is long an unacceptable. + * + * To prevent situations like this, UBIFS writes pages back only if they are + * within the last synchronized inode size, i.e. the size which has been + * written to the flash media last time. Otherwise, UBIFS forces inode + * write-back, thus making sure the on-flash inode contains current inode size, + * and then keeps writing pages back. + * + * Some locking issues explanation. 'ubifs_writepage()' first is called with + * the page locked, and it locks @ui_mutex. However, write-back does take inode + * @i_mutex, which means other VFS operations may be run on this inode at the + * same time. And the problematic one is truncation to smaller size, from where + * we have to call 'truncate_setsize()', which first changes @inode->i_size, + * then drops the truncated pages. And while dropping the pages, it takes the + * page lock. This means that 'do_truncation()' cannot call 'truncate_setsize()' + * with @ui_mutex locked, because it would deadlock with 'ubifs_writepage()'. + * This means that @inode->i_size is changed while @ui_mutex is unlocked. + * + * XXX(truncate): with the new truncate sequence this is not true anymore, + * and the calls to truncate_setsize can be move around freely. They should + * be moved to the very end of the truncate sequence. + * + * But in 'ubifs_writepage()' we have to guarantee that we do not write beyond + * inode size. How do we do this if @inode->i_size may became smaller while we + * are in the middle of 'ubifs_writepage()'? The UBIFS solution is the + * @ui->ui_isize "shadow" field which UBIFS uses instead of @inode->i_size + * internally and updates it under @ui_mutex. + * + * Q: why we do not worry that if we race with truncation, we may end up with a + * situation when the inode is truncated while we are in the middle of + * 'do_writepage()', so we do write beyond inode size? + * A: If we are in the middle of 'do_writepage()', truncation would be locked + * on the page lock and it would not write the truncated inode node to the + * journal before we have finished. + */ +static int ubifs_writepage(struct folio *folio, struct writeback_control *wbc) +{ + struct inode *inode = folio->mapping->host; + struct ubifs_info *c = inode->i_sb->s_fs_info; + struct ubifs_inode *ui = ubifs_inode(inode); + loff_t i_size = i_size_read(inode), synced_i_size; + int err, len = folio_size(folio); + + dbg_gen("ino %llu, pg %lu, pg flags %#lx", + inode->i_ino, folio->index, folio->flags.f); + ubifs_assert(c, folio->private != NULL); + + /* Is the folio fully outside @i_size? (truncate in progress) */ + if (folio_pos(folio) >= i_size) { + err = 0; + goto out_unlock; + } + + spin_lock(&ui->ui_lock); + synced_i_size = ui->synced_i_size; + spin_unlock(&ui->ui_lock); + + /* Is the folio fully inside i_size? */ + if (folio_pos(folio) + len <= i_size) { + if (folio_pos(folio) + len > synced_i_size) { + err = inode->i_sb->s_op->write_inode(inode, NULL); + if (err) + goto out_redirty; + /* + * The inode has been written, but the write-buffer has + * not been synchronized, so in case of an unclean + * reboot we may end up with some pages beyond inode + * size, but they would be in the journal (because + * commit flushes write buffers) and recovery would deal + * with this. + */ + } + return do_writepage(folio, len); + } + + /* + * The folio straddles @i_size. It must be zeroed out on each and every + * writepage invocation because it may be mmapped. "A file is mapped + * in multiples of the page size. For a file that is not a multiple of + * the page size, the remaining memory is zeroed when mapped, and + * writes to that region are not written out to the file." + */ + len = i_size - folio_pos(folio); + folio_zero_segment(folio, len, folio_size(folio)); + + if (i_size > synced_i_size) { + err = inode->i_sb->s_op->write_inode(inode, NULL); + if (err) + goto out_redirty; + } + + return do_writepage(folio, len); +out_redirty: + /* + * folio_redirty_for_writepage() won't call ubifs_dirty_inode() because + * it passes I_DIRTY_PAGES flag while calling __mark_inode_dirty(), so + * there is no need to do space budget for dirty inode. + */ + folio_redirty_for_writepage(wbc, folio); +out_unlock: + folio_unlock(folio); + return err; +} + +static int ubifs_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct folio *folio = NULL; + int error; + + while ((folio = writeback_iter(mapping, wbc, folio, &error))) + error = ubifs_writepage(folio, wbc); + return error; +} + +/** + * do_attr_changes - change inode attributes. + * @inode: inode to change attributes for + * @attr: describes attributes to change + */ +static void do_attr_changes(struct inode *inode, const struct iattr *attr) +{ + if (attr->ia_valid & ATTR_UID) + inode->i_uid = attr->ia_uid; + if (attr->ia_valid & ATTR_GID) + inode->i_gid = attr->ia_gid; + if (attr->ia_valid & ATTR_ATIME) + inode_set_atime_to_ts(inode, attr->ia_atime); + if (attr->ia_valid & ATTR_MTIME) + inode_set_mtime_to_ts(inode, attr->ia_mtime); + if (attr->ia_valid & ATTR_CTIME) + inode_set_ctime_to_ts(inode, attr->ia_ctime); + if (attr->ia_valid & ATTR_MODE) { + umode_t mode = attr->ia_mode; + + if (!in_group_p(inode->i_gid) && !capable(CAP_FSETID)) + mode &= ~S_ISGID; + inode->i_mode = mode; + } +} + +/** + * do_truncation - truncate an inode. + * @c: UBIFS file-system description object + * @inode: inode to truncate + * @attr: inode attribute changes description + * + * This function implements VFS '->setattr()' call when the inode is truncated + * to a smaller size. + * + * Returns: %0 in case of success and a negative error code + * in case of failure. + */ +static int do_truncation(struct ubifs_info *c, struct inode *inode, + const struct iattr *attr) +{ + int err; + struct ubifs_budget_req req; + loff_t old_size = inode->i_size, new_size = attr->ia_size; + int offset = new_size & (UBIFS_BLOCK_SIZE - 1), budgeted = 1; + struct ubifs_inode *ui = ubifs_inode(inode); + + dbg_gen("ino %llu, size %lld -> %lld", inode->i_ino, old_size, new_size); + memset(&req, 0, sizeof(struct ubifs_budget_req)); + + /* + * If this is truncation to a smaller size, and we do not truncate on a + * block boundary, budget for changing one data block, because the last + * block will be re-written. + */ + if (new_size & (UBIFS_BLOCK_SIZE - 1)) + req.dirtied_page = 1; + + req.dirtied_ino = 1; + /* A funny way to budget for truncation node */ + req.dirtied_ino_d = UBIFS_TRUN_NODE_SZ; + err = ubifs_budget_space(c, &req); + if (err) { + /* + * Treat truncations to zero as deletion and always allow them, + * just like we do for '->unlink()'. + */ + if (new_size || err != -ENOSPC) + return err; + budgeted = 0; + } + + truncate_setsize(inode, new_size); + + if (offset) { + pgoff_t index = new_size >> PAGE_SHIFT; + struct folio *folio; + + folio = filemap_lock_folio(inode->i_mapping, index); + if (!IS_ERR(folio)) { + if (folio_test_dirty(folio)) { + /* + * 'ubifs_jnl_truncate()' will try to truncate + * the last data node, but it contains + * out-of-date data because the page is dirty. + * Write the page now, so that + * 'ubifs_jnl_truncate()' will see an already + * truncated (and up to date) data node. + */ + ubifs_assert(c, folio->private != NULL); + + folio_clear_dirty_for_io(folio); + if (UBIFS_BLOCKS_PER_PAGE_SHIFT) + offset = offset_in_folio(folio, + new_size); + err = do_writepage(folio, offset); + folio_put(folio); + if (err) + goto out_budg; + /* + * We could now tell 'ubifs_jnl_truncate()' not + * to read the last block. + */ + } else { + /* + * We could 'kmap()' the page and pass the data + * to 'ubifs_jnl_truncate()' to save it from + * having to read it. + */ + folio_unlock(folio); + folio_put(folio); + } + } + } + + mutex_lock(&ui->ui_mutex); + ui->ui_size = inode->i_size; + /* Truncation changes inode [mc]time */ + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + /* Other attributes may be changed at the same time as well */ + do_attr_changes(inode, attr); + err = ubifs_jnl_truncate(c, inode, old_size, new_size); + mutex_unlock(&ui->ui_mutex); + +out_budg: + if (budgeted) + ubifs_release_budget(c, &req); + else { + c->bi.nospace = c->bi.nospace_rp = 0; + smp_wmb(); + } + return err; +} + +/** + * do_setattr - change inode attributes. + * @c: UBIFS file-system description object + * @inode: inode to change attributes for + * @attr: inode attribute changes description + * + * This function implements VFS '->setattr()' call for all cases except + * truncations to smaller size. + * + * Returns: %0 in case of success and a negative + * error code in case of failure. + */ +static int do_setattr(struct ubifs_info *c, struct inode *inode, + const struct iattr *attr) +{ + int err, release; + loff_t new_size = attr->ia_size; + struct ubifs_inode *ui = ubifs_inode(inode); + struct ubifs_budget_req req = { .dirtied_ino = 1, + .dirtied_ino_d = ALIGN(ui->data_len, 8) }; + + err = ubifs_budget_space(c, &req); + if (err) + return err; + + if (attr->ia_valid & ATTR_SIZE) { + dbg_gen("size %lld -> %lld", inode->i_size, new_size); + truncate_setsize(inode, new_size); + } + + mutex_lock(&ui->ui_mutex); + if (attr->ia_valid & ATTR_SIZE) { + /* Truncation changes inode [mc]time */ + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + /* 'truncate_setsize()' changed @i_size, update @ui_size */ + ui->ui_size = inode->i_size; + } + + do_attr_changes(inode, attr); + + release = ui->dirty; + if (attr->ia_valid & ATTR_SIZE) + /* + * Inode length changed, so we have to make sure + * @I_DIRTY_DATASYNC is set. + */ + __mark_inode_dirty(inode, I_DIRTY_DATASYNC); + else + mark_inode_dirty_sync(inode); + mutex_unlock(&ui->ui_mutex); + + if (release) + ubifs_release_budget(c, &req); + if (IS_SYNC(inode)) + err = inode->i_sb->s_op->write_inode(inode, NULL); + return err; +} + +int ubifs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +{ + int err; + struct inode *inode = d_inode(dentry); + struct ubifs_info *c = inode->i_sb->s_fs_info; + + dbg_gen("ino %llu, mode %#x, ia_valid %#x", + inode->i_ino, inode->i_mode, attr->ia_valid); + err = setattr_prepare(&nop_mnt_idmap, dentry, attr); + if (err) + return err; + + err = dbg_check_synced_i_size(c, inode); + if (err) + return err; + + err = fscrypt_prepare_setattr(dentry, attr); + if (err) + return err; + + if ((attr->ia_valid & ATTR_SIZE) && attr->ia_size < inode->i_size) + /* Truncation to a smaller size */ + err = do_truncation(c, inode, attr); + else + err = do_setattr(c, inode, attr); + + return err; +} + +static void ubifs_invalidate_folio(struct folio *folio, size_t offset, + size_t length) +{ + struct inode *inode = folio->mapping->host; + struct ubifs_info *c = inode->i_sb->s_fs_info; + + ubifs_assert(c, folio_test_private(folio)); + if (offset || length < folio_size(folio)) + /* Partial folio remains dirty */ + return; + + if (folio_test_checked(folio)) + release_new_page_budget(c); + else + release_existing_page_budget(c); + + atomic_long_dec(&c->dirty_pg_cnt); + folio_detach_private(folio); + folio_clear_checked(folio); +} + +int ubifs_fsync(struct file *file, loff_t start, loff_t end, int datasync) +{ + struct inode *inode = file->f_mapping->host; + struct ubifs_info *c = inode->i_sb->s_fs_info; + int err; + + dbg_gen("syncing inode %llu", inode->i_ino); + + if (c->ro_mount) + /* + * For some really strange reasons VFS does not filter out + * 'fsync()' for R/O mounted file-systems as per 2.6.39. + */ + return 0; + + err = file_write_and_wait_range(file, start, end); + if (err) + return err; + inode_lock(inode); + + /* Synchronize the inode unless this is a 'datasync()' call. */ + if (!datasync || (inode_state_read_once(inode) & I_DIRTY_DATASYNC)) { + err = inode->i_sb->s_op->write_inode(inode, NULL); + if (err) + goto out; + } + + /* + * Nodes related to this inode may still sit in a write-buffer. Flush + * them. + */ + err = ubifs_sync_wbufs_by_inode(c, inode); +out: + inode_unlock(inode); + return err; +} + +/** + * mctime_update_needed - check if mtime or ctime update is needed. + * @inode: the inode to do the check for + * @now: current time + * + * This helper function checks if the inode mtime/ctime should be updated or + * not. If current values of the time-stamps are within the UBIFS inode time + * granularity, they are not updated. This is an optimization. + * + * Returns: %1 if time update is needed, %0 if not + */ +static inline int mctime_update_needed(const struct inode *inode, + const struct timespec64 *now) +{ + struct timespec64 ctime = inode_get_ctime(inode); + struct timespec64 mtime = inode_get_mtime(inode); + + if (!timespec64_equal(&mtime, now) || !timespec64_equal(&ctime, now)) + return 1; + return 0; +} + +int ubifs_update_time(struct inode *inode, enum fs_update_time type, + unsigned int flags) +{ + struct ubifs_inode *ui = ubifs_inode(inode); + struct ubifs_info *c = inode->i_sb->s_fs_info; + struct ubifs_budget_req req = { .dirtied_ino = 1, + .dirtied_ino_d = ALIGN(ui->data_len, 8) }; + int err, release; + + /* ubifs sets S_NOCMTIME on all inodes, this should not happen. */ + if (WARN_ON_ONCE(type != FS_UPD_ATIME)) + return -EIO; + + if (!IS_ENABLED(CONFIG_UBIFS_ATIME_SUPPORT)) + return generic_update_time(inode, type, flags); + + if (flags & IOCB_NOWAIT) + return -EAGAIN; + + err = ubifs_budget_space(c, &req); + if (err) + return err; + + mutex_lock(&ui->ui_mutex); + inode_update_time(inode, type, flags); + release = ui->dirty; + __mark_inode_dirty(inode, I_DIRTY_SYNC); + mutex_unlock(&ui->ui_mutex); + if (release) + ubifs_release_budget(c, &req); + return 0; +} + +/** + * update_mctime - update mtime and ctime of an inode. + * @inode: inode to update + * + * This function updates mtime and ctime of the inode if it is not equivalent to + * current time. + * + * Returns: %0 in case of success and a negative error code in + * case of failure. + */ +static int update_mctime(struct inode *inode) +{ + struct timespec64 now = current_time(inode); + struct ubifs_inode *ui = ubifs_inode(inode); + struct ubifs_info *c = inode->i_sb->s_fs_info; + + if (mctime_update_needed(inode, &now)) { + int err, release; + struct ubifs_budget_req req = { .dirtied_ino = 1, + .dirtied_ino_d = ALIGN(ui->data_len, 8) }; + + err = ubifs_budget_space(c, &req); + if (err) + return err; + + mutex_lock(&ui->ui_mutex); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + release = ui->dirty; + mark_inode_dirty_sync(inode); + mutex_unlock(&ui->ui_mutex); + if (release) + ubifs_release_budget(c, &req); + } + + return 0; +} + +static ssize_t ubifs_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + int err = update_mctime(file_inode(iocb->ki_filp)); + if (err) + return err; + + return generic_file_write_iter(iocb, from); +} + +static bool ubifs_dirty_folio(struct address_space *mapping, + struct folio *folio) +{ + bool ret; + struct ubifs_info *c = mapping->host->i_sb->s_fs_info; + + ret = filemap_dirty_folio(mapping, folio); + /* + * An attempt to dirty a page without budgeting for it - should not + * happen. + */ + ubifs_assert(c, ret == false); + return ret; +} + +static bool ubifs_release_folio(struct folio *folio, gfp_t unused_gfp_flags) +{ + struct inode *inode = folio->mapping->host; + struct ubifs_info *c = inode->i_sb->s_fs_info; + + if (folio_test_writeback(folio)) + return false; + + /* + * Page is private but not dirty, weird? There is one condition + * making it happened. ubifs_writepage skipped the page because + * page index beyonds isize (for example. truncated by other + * process named A), then the page is invalidated by fadvise64 + * syscall before being truncated by process A. + */ + ubifs_assert(c, folio_test_private(folio)); + if (folio_test_checked(folio)) + release_new_page_budget(c); + else + release_existing_page_budget(c); + + atomic_long_dec(&c->dirty_pg_cnt); + folio_detach_private(folio); + folio_clear_checked(folio); + return true; +} + +/* + * mmap()d file has taken write protection fault and is being made writable. + * UBIFS must ensure page is budgeted for. + */ +static vm_fault_t ubifs_vm_page_mkwrite(struct vm_fault *vmf) +{ + struct folio *folio = page_folio(vmf->page); + struct inode *inode = file_inode(vmf->vma->vm_file); + struct ubifs_info *c = inode->i_sb->s_fs_info; + struct timespec64 now = current_time(inode); + struct ubifs_budget_req req = { .new_page = 1 }; + int err, update_time; + + dbg_gen("ino %llu, pg %lu, i_size %lld", inode->i_ino, folio->index, + i_size_read(inode)); + ubifs_assert(c, !c->ro_media && !c->ro_mount); + + if (unlikely(c->ro_error)) + return VM_FAULT_SIGBUS; /* -EROFS */ + + /* + * We have not locked @folio so far so we may budget for changing the + * folio. Note, we cannot do this after we locked the folio, because + * budgeting may cause write-back which would cause deadlock. + * + * At the moment we do not know whether the folio is dirty or not, so we + * assume that it is not and budget for a new folio. We could look at + * the @PG_private flag and figure this out, but we may race with write + * back and the folio state may change by the time we lock it, so this + * would need additional care. We do not bother with this at the + * moment, although it might be good idea to do. Instead, we allocate + * budget for a new folio and amend it later on if the folio was in fact + * dirty. + * + * The budgeting-related logic of this function is similar to what we + * do in 'ubifs_write_begin()' and 'ubifs_write_end()'. Glance there + * for more comments. + */ + update_time = mctime_update_needed(inode, &now); + if (update_time) + /* + * We have to change inode time stamp which requires extra + * budgeting. + */ + req.dirtied_ino = 1; + + err = ubifs_budget_space(c, &req); + if (unlikely(err)) { + if (err == -ENOSPC) + ubifs_warn(c, "out of space for mmapped file (inode number %llu)", + inode->i_ino); + return VM_FAULT_SIGBUS; + } + + folio_lock(folio); + if (unlikely(folio->mapping != inode->i_mapping || + folio_pos(folio) >= i_size_read(inode))) { + /* Folio got truncated out from underneath us */ + goto sigbus; + } + + if (folio->private) + release_new_page_budget(c); + else { + if (!folio_test_checked(folio)) + ubifs_convert_page_budget(c); + folio_attach_private(folio, (void *)1); + atomic_long_inc(&c->dirty_pg_cnt); + filemap_dirty_folio(folio->mapping, folio); + } + + if (update_time) { + int release; + struct ubifs_inode *ui = ubifs_inode(inode); + + mutex_lock(&ui->ui_mutex); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + release = ui->dirty; + mark_inode_dirty_sync(inode); + mutex_unlock(&ui->ui_mutex); + if (release) + ubifs_release_dirty_inode_budget(c, ui); + } + + folio_wait_stable(folio); + return VM_FAULT_LOCKED; + +sigbus: + folio_unlock(folio); + ubifs_release_budget(c, &req); + return VM_FAULT_SIGBUS; +} + +static const struct vm_operations_struct ubifs_file_vm_ops = { + .fault = filemap_fault, + .map_pages = filemap_map_pages, + .page_mkwrite = ubifs_vm_page_mkwrite, +}; + +static int ubifs_file_mmap_prepare(struct vm_area_desc *desc) +{ + int err; + + err = generic_file_mmap_prepare(desc); + if (err) + return err; + desc->vm_ops = &ubifs_file_vm_ops; + + if (IS_ENABLED(CONFIG_UBIFS_ATIME_SUPPORT)) + file_accessed(desc->file); + + return 0; +} + +static const char *ubifs_get_link(struct dentry *dentry, + struct inode *inode, + struct delayed_call *done) +{ + struct ubifs_inode *ui = ubifs_inode(inode); + + if (!IS_ENCRYPTED(inode)) + return ui->data; + + if (!dentry) + return ERR_PTR(-ECHILD); + + return fscrypt_get_symlink(inode, ui->data, ui->data_len, done); +} + +static int ubifs_symlink_getattr(struct mnt_idmap *idmap, + const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int query_flags) +{ + ubifs_getattr(idmap, path, stat, request_mask, query_flags); + + if (IS_ENCRYPTED(d_inode(path->dentry))) + return fscrypt_symlink_getattr(path, stat); + return 0; +} + +const struct address_space_operations ubifs_file_address_operations = { + .read_folio = ubifs_read_folio, + .writepages = ubifs_writepages, + .write_begin = ubifs_write_begin, + .write_end = ubifs_write_end, + .invalidate_folio = ubifs_invalidate_folio, + .dirty_folio = ubifs_dirty_folio, + .migrate_folio = filemap_migrate_folio, + .release_folio = ubifs_release_folio, +}; + +const struct inode_operations ubifs_file_inode_operations = { + .setattr = ubifs_setattr, + .getattr = ubifs_getattr, + .listxattr = ubifs_listxattr, + .update_time = ubifs_update_time, + .fileattr_get = ubifs_fileattr_get, + .fileattr_set = ubifs_fileattr_set, +}; + +const struct inode_operations ubifs_symlink_inode_operations = { + .get_link = ubifs_get_link, + .setattr = ubifs_setattr, + .getattr = ubifs_symlink_getattr, + .listxattr = ubifs_listxattr, + .update_time = ubifs_update_time, +}; + +const struct file_operations ubifs_file_operations = { + .llseek = generic_file_llseek, + .read_iter = generic_file_read_iter, + .write_iter = ubifs_write_iter, + .mmap_prepare = ubifs_file_mmap_prepare, + .fsync = ubifs_fsync, + .unlocked_ioctl = ubifs_ioctl, + .splice_read = filemap_splice_read, + .splice_write = iter_file_splice_write, + .open = fscrypt_file_open, +#ifdef CONFIG_COMPAT + .compat_ioctl = ubifs_compat_ioctl, +#endif +}; diff --git a/fs/ubifs/find.c b/fs/ubifs/find.c new file mode 100644 index 000000000..643718906 --- /dev/null +++ b/fs/ubifs/find.c @@ -0,0 +1,963 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * + * Authors: Artem Bityutskiy (Битюцкий Артём) + * Adrian Hunter + */ + +/* + * This file contains functions for finding LEBs for various purposes e.g. + * garbage collection. In general, lprops category heaps and lists are used + * for fast access, falling back on scanning the LPT as a last resort. + */ + +#include +#include "ubifs.h" + +/** + * struct scan_data - data provided to scan callback functions + * @min_space: minimum number of bytes for which to scan + * @pick_free: whether it is OK to scan for empty LEBs + * @lnum: LEB number found is returned here + * @exclude_index: whether to exclude index LEBs + */ +struct scan_data { + int min_space; + int pick_free; + int lnum; + int exclude_index; +}; + +/** + * valuable - determine whether LEB properties are valuable. + * @c: the UBIFS file-system description object + * @lprops: LEB properties + * + * This function return %1 if the LEB properties should be added to the LEB + * properties tree in memory. Otherwise %0 is returned. + */ +static int valuable(struct ubifs_info *c, const struct ubifs_lprops *lprops) +{ + int n, cat = lprops->flags & LPROPS_CAT_MASK; + struct ubifs_lpt_heap *heap; + + switch (cat) { + case LPROPS_DIRTY: + case LPROPS_DIRTY_IDX: + case LPROPS_FREE: + heap = &c->lpt_heap[cat - 1]; + if (heap->cnt < heap->max_cnt) + return 1; + if (lprops->free + lprops->dirty >= c->dark_wm) + return 1; + return 0; + case LPROPS_EMPTY: + n = c->lst.empty_lebs + c->freeable_cnt - + c->lst.taken_empty_lebs; + if (n < c->lsave_cnt) + return 1; + return 0; + case LPROPS_FREEABLE: + return 1; + case LPROPS_FRDI_IDX: + return 1; + } + return 0; +} + +/** + * scan_for_dirty_cb - dirty space scan callback. + * @c: the UBIFS file-system description object + * @lprops: LEB properties to scan + * @in_tree: whether the LEB properties are in main memory + * @arg: information passed to and from the caller of the scan + * + * This function returns a code that indicates whether the scan should continue + * (%LPT_SCAN_CONTINUE), whether the LEB properties should be added to the tree + * in main memory (%LPT_SCAN_ADD), or whether the scan should stop + * (%LPT_SCAN_STOP). + */ +static int scan_for_dirty_cb(struct ubifs_info *c, + const struct ubifs_lprops *lprops, int in_tree, + void *arg) +{ + struct scan_data *data = arg; + int ret = LPT_SCAN_CONTINUE; + + /* Exclude LEBs that are currently in use */ + if (lprops->flags & LPROPS_TAKEN) + return LPT_SCAN_CONTINUE; + /* Determine whether to add these LEB properties to the tree */ + if (!in_tree && valuable(c, lprops)) + ret |= LPT_SCAN_ADD; + /* Exclude LEBs with too little space */ + if (lprops->free + lprops->dirty < data->min_space) + return ret; + /* If specified, exclude index LEBs */ + if (data->exclude_index && lprops->flags & LPROPS_INDEX) + return ret; + /* If specified, exclude empty or freeable LEBs */ + if (lprops->free + lprops->dirty == c->leb_size) { + if (!data->pick_free) + return ret; + /* Exclude LEBs with too little dirty space (unless it is empty) */ + } else if (lprops->dirty < c->dead_wm) + return ret; + /* Finally we found space */ + data->lnum = lprops->lnum; + return LPT_SCAN_ADD | LPT_SCAN_STOP; +} + +/** + * scan_for_dirty - find a data LEB with free space. + * @c: the UBIFS file-system description object + * @min_space: minimum amount free plus dirty space the returned LEB has to + * have + * @pick_free: if it is OK to return a free or freeable LEB + * @exclude_index: whether to exclude index LEBs + * + * This function returns a pointer to the LEB properties found or a negative + * error code. + */ +static const struct ubifs_lprops *scan_for_dirty(struct ubifs_info *c, + int min_space, int pick_free, + int exclude_index) +{ + const struct ubifs_lprops *lprops; + struct ubifs_lpt_heap *heap; + struct scan_data data; + int err, i; + + /* There may be an LEB with enough dirty space on the free heap */ + heap = &c->lpt_heap[LPROPS_FREE - 1]; + for (i = 0; i < heap->cnt; i++) { + lprops = heap->arr[i]; + if (lprops->free + lprops->dirty < min_space) + continue; + if (lprops->dirty < c->dead_wm) + continue; + return lprops; + } + /* + * A LEB may have fallen off of the bottom of the dirty heap, and ended + * up as uncategorized even though it has enough dirty space for us now, + * so check the uncategorized list. N.B. neither empty nor freeable LEBs + * can end up as uncategorized because they are kept on lists not + * finite-sized heaps. + */ + list_for_each_entry(lprops, &c->uncat_list, list) { + if (lprops->flags & LPROPS_TAKEN) + continue; + if (lprops->free + lprops->dirty < min_space) + continue; + if (exclude_index && (lprops->flags & LPROPS_INDEX)) + continue; + if (lprops->dirty < c->dead_wm) + continue; + return lprops; + } + /* We have looked everywhere in main memory, now scan the flash */ + if (c->pnodes_have >= c->pnode_cnt) + /* All pnodes are in memory, so skip scan */ + return ERR_PTR(-ENOSPC); + data.min_space = min_space; + data.pick_free = pick_free; + data.lnum = -1; + data.exclude_index = exclude_index; + err = ubifs_lpt_scan_nolock(c, -1, c->lscan_lnum, scan_for_dirty_cb, + &data); + if (err) + return ERR_PTR(err); + ubifs_assert(c, data.lnum >= c->main_first && data.lnum < c->leb_cnt); + c->lscan_lnum = data.lnum; + lprops = ubifs_lpt_lookup_dirty(c, data.lnum); + if (IS_ERR(lprops)) + return lprops; + ubifs_assert(c, lprops->lnum == data.lnum); + ubifs_assert(c, lprops->free + lprops->dirty >= min_space); + ubifs_assert(c, lprops->dirty >= c->dead_wm || + (pick_free && + lprops->free + lprops->dirty == c->leb_size)); + ubifs_assert(c, !(lprops->flags & LPROPS_TAKEN)); + ubifs_assert(c, !exclude_index || !(lprops->flags & LPROPS_INDEX)); + return lprops; +} + +/** + * ubifs_find_dirty_leb - find a dirty LEB for the Garbage Collector. + * @c: the UBIFS file-system description object + * @ret_lp: LEB properties are returned here on exit + * @min_space: minimum amount free plus dirty space the returned LEB has to + * have + * @pick_free: controls whether it is OK to pick empty or index LEBs + * + * This function tries to find a dirty logical eraseblock which has at least + * @min_space free and dirty space. It prefers to take an LEB from the dirty or + * dirty index heap, and it falls-back to LPT scanning if the heaps are empty + * or do not have an LEB which satisfies the @min_space criteria. + * + * Note, LEBs which have less than dead watermark of free + dirty space are + * never picked by this function. + * + * The additional @pick_free argument controls if this function has to return a + * free or freeable LEB if one is present. For example, GC must to set it to %1, + * when called from the journal space reservation function, because the + * appearance of free space may coincide with the loss of enough dirty space + * for GC to succeed anyway. + * + * In contrast, if the Garbage Collector is called from budgeting, it should + * just make free space, not return LEBs which are already free or freeable. + * + * In addition @pick_free is set to %2 by the recovery process in order to + * recover gc_lnum in which case an index LEB must not be returned. + * + * This function returns zero and the LEB properties of found dirty LEB in case + * of success, %-ENOSPC if no dirty LEB was found and a negative error code in + * case of other failures. The returned LEB is marked as "taken". + */ +int ubifs_find_dirty_leb(struct ubifs_info *c, struct ubifs_lprops *ret_lp, + int min_space, int pick_free) +{ + int err = 0, sum, exclude_index = pick_free == 2 ? 1 : 0; + const struct ubifs_lprops *lp = NULL, *idx_lp = NULL; + struct ubifs_lpt_heap *heap, *idx_heap; + + ubifs_get_lprops(c); + + if (pick_free) { + int lebs, rsvd_idx_lebs = 0; + + spin_lock(&c->space_lock); + lebs = c->lst.empty_lebs + c->idx_gc_cnt; + lebs += c->freeable_cnt - c->lst.taken_empty_lebs; + + /* + * Note, the index may consume more LEBs than have been reserved + * for it. It is OK because it might be consolidated by GC. + * But if the index takes fewer LEBs than it is reserved for it, + * this function must avoid picking those reserved LEBs. + */ + if (c->bi.min_idx_lebs >= c->lst.idx_lebs) { + rsvd_idx_lebs = c->bi.min_idx_lebs - c->lst.idx_lebs; + exclude_index = 1; + } + spin_unlock(&c->space_lock); + + /* Check if there are enough free LEBs for the index */ + if (rsvd_idx_lebs < lebs) { + /* OK, try to find an empty LEB */ + lp = ubifs_fast_find_empty(c); + if (lp) + goto found; + + /* Or a freeable LEB */ + lp = ubifs_fast_find_freeable(c); + if (lp) + goto found; + } else + /* + * We cannot pick free/freeable LEBs in the below code. + */ + pick_free = 0; + } else { + spin_lock(&c->space_lock); + exclude_index = (c->bi.min_idx_lebs >= c->lst.idx_lebs); + spin_unlock(&c->space_lock); + } + + /* Look on the dirty and dirty index heaps */ + heap = &c->lpt_heap[LPROPS_DIRTY - 1]; + idx_heap = &c->lpt_heap[LPROPS_DIRTY_IDX - 1]; + + if (idx_heap->cnt && !exclude_index) { + idx_lp = idx_heap->arr[0]; + sum = idx_lp->free + idx_lp->dirty; + /* + * Since we reserve thrice as much space for the index than it + * actually takes, it does not make sense to pick indexing LEBs + * with less than, say, half LEB of dirty space. May be half is + * not the optimal boundary - this should be tested and + * checked. This boundary should determine how much we use + * in-the-gaps to consolidate the index comparing to how much + * we use garbage collector to consolidate it. The "half" + * criteria just feels to be fine. + */ + if (sum < min_space || sum < c->half_leb_size) + idx_lp = NULL; + } + + if (heap->cnt) { + lp = heap->arr[0]; + if (lp->dirty + lp->free < min_space) + lp = NULL; + } + + /* Pick the LEB with most space */ + if (idx_lp && lp) { + if (idx_lp->free + idx_lp->dirty >= lp->free + lp->dirty) + lp = idx_lp; + } else if (idx_lp && !lp) + lp = idx_lp; + + if (lp) { + ubifs_assert(c, lp->free + lp->dirty >= c->dead_wm); + goto found; + } + + /* Did not find a dirty LEB on the dirty heaps, have to scan */ + dbg_find("scanning LPT for a dirty LEB"); + lp = scan_for_dirty(c, min_space, pick_free, exclude_index); + if (IS_ERR(lp)) { + err = PTR_ERR(lp); + goto out; + } + ubifs_assert(c, lp->dirty >= c->dead_wm || + (pick_free && lp->free + lp->dirty == c->leb_size)); + +found: + dbg_find("found LEB %d, free %d, dirty %d, flags %#x", + lp->lnum, lp->free, lp->dirty, lp->flags); + + lp = ubifs_change_lp(c, lp, LPROPS_NC, LPROPS_NC, + lp->flags | LPROPS_TAKEN, 0); + if (IS_ERR(lp)) { + err = PTR_ERR(lp); + goto out; + } + + memcpy(ret_lp, lp, sizeof(struct ubifs_lprops)); + +out: + ubifs_release_lprops(c); + return err; +} + +/** + * scan_for_free_cb - free space scan callback. + * @c: the UBIFS file-system description object + * @lprops: LEB properties to scan + * @in_tree: whether the LEB properties are in main memory + * @arg: information passed to and from the caller of the scan + * + * This function returns a code that indicates whether the scan should continue + * (%LPT_SCAN_CONTINUE), whether the LEB properties should be added to the tree + * in main memory (%LPT_SCAN_ADD), or whether the scan should stop + * (%LPT_SCAN_STOP). + */ +static int scan_for_free_cb(struct ubifs_info *c, + const struct ubifs_lprops *lprops, int in_tree, + void *arg) +{ + struct scan_data *data = arg; + int ret = LPT_SCAN_CONTINUE; + + /* Exclude LEBs that are currently in use */ + if (lprops->flags & LPROPS_TAKEN) + return LPT_SCAN_CONTINUE; + /* Determine whether to add these LEB properties to the tree */ + if (!in_tree && valuable(c, lprops)) + ret |= LPT_SCAN_ADD; + /* Exclude index LEBs */ + if (lprops->flags & LPROPS_INDEX) + return ret; + /* Exclude LEBs with too little space */ + if (lprops->free < data->min_space) + return ret; + /* If specified, exclude empty LEBs */ + if (!data->pick_free && lprops->free == c->leb_size) + return ret; + /* + * LEBs that have only free and dirty space must not be allocated + * because they may have been unmapped already or they may have data + * that is obsolete only because of nodes that are still sitting in a + * wbuf. + */ + if (lprops->free + lprops->dirty == c->leb_size && lprops->dirty > 0) + return ret; + /* Finally we found space */ + data->lnum = lprops->lnum; + return LPT_SCAN_ADD | LPT_SCAN_STOP; +} + +/** + * do_find_free_space - find a data LEB with free space. + * @c: the UBIFS file-system description object + * @min_space: minimum amount of free space required + * @pick_free: whether it is OK to scan for empty LEBs + * @squeeze: whether to try to find space in a non-empty LEB first + * + * This function returns a pointer to the LEB properties found or a negative + * error code. + */ +static +const struct ubifs_lprops *do_find_free_space(struct ubifs_info *c, + int min_space, int pick_free, + int squeeze) +{ + const struct ubifs_lprops *lprops; + struct ubifs_lpt_heap *heap; + struct scan_data data; + int err, i; + + if (squeeze) { + lprops = ubifs_fast_find_free(c); + if (lprops && lprops->free >= min_space) + return lprops; + } + if (pick_free) { + lprops = ubifs_fast_find_empty(c); + if (lprops) + return lprops; + } + if (!squeeze) { + lprops = ubifs_fast_find_free(c); + if (lprops && lprops->free >= min_space) + return lprops; + } + /* There may be an LEB with enough free space on the dirty heap */ + heap = &c->lpt_heap[LPROPS_DIRTY - 1]; + for (i = 0; i < heap->cnt; i++) { + lprops = heap->arr[i]; + if (lprops->free >= min_space) + return lprops; + } + /* + * A LEB may have fallen off of the bottom of the free heap, and ended + * up as uncategorized even though it has enough free space for us now, + * so check the uncategorized list. N.B. neither empty nor freeable LEBs + * can end up as uncategorized because they are kept on lists not + * finite-sized heaps. + */ + list_for_each_entry(lprops, &c->uncat_list, list) { + if (lprops->flags & LPROPS_TAKEN) + continue; + if (lprops->flags & LPROPS_INDEX) + continue; + if (lprops->free >= min_space) + return lprops; + } + /* We have looked everywhere in main memory, now scan the flash */ + if (c->pnodes_have >= c->pnode_cnt) + /* All pnodes are in memory, so skip scan */ + return ERR_PTR(-ENOSPC); + data.min_space = min_space; + data.pick_free = pick_free; + data.lnum = -1; + err = ubifs_lpt_scan_nolock(c, -1, c->lscan_lnum, + scan_for_free_cb, + &data); + if (err) + return ERR_PTR(err); + ubifs_assert(c, data.lnum >= c->main_first && data.lnum < c->leb_cnt); + c->lscan_lnum = data.lnum; + lprops = ubifs_lpt_lookup_dirty(c, data.lnum); + if (IS_ERR(lprops)) + return lprops; + ubifs_assert(c, lprops->lnum == data.lnum); + ubifs_assert(c, lprops->free >= min_space); + ubifs_assert(c, !(lprops->flags & LPROPS_TAKEN)); + ubifs_assert(c, !(lprops->flags & LPROPS_INDEX)); + return lprops; +} + +/** + * ubifs_find_free_space - find a data LEB with free space. + * @c: the UBIFS file-system description object + * @min_space: minimum amount of required free space + * @offs: contains offset of where free space starts on exit + * @squeeze: whether to try to find space in a non-empty LEB first + * + * This function looks for an LEB with at least @min_space bytes of free space. + * It tries to find an empty LEB if possible. If no empty LEBs are available, + * this function searches for a non-empty data LEB. The returned LEB is marked + * as "taken". + * + * This function returns found LEB number in case of success, %-ENOSPC if it + * failed to find a LEB with @min_space bytes of free space and other a negative + * error codes in case of failure. + */ +int ubifs_find_free_space(struct ubifs_info *c, int min_space, int *offs, + int squeeze) +{ + const struct ubifs_lprops *lprops; + int lebs, rsvd_idx_lebs, pick_free = 0, err, lnum, flags; + + dbg_find("min_space %d", min_space); + ubifs_get_lprops(c); + + /* Check if there are enough empty LEBs for commit */ + spin_lock(&c->space_lock); + if (c->bi.min_idx_lebs > c->lst.idx_lebs) + rsvd_idx_lebs = c->bi.min_idx_lebs - c->lst.idx_lebs; + else + rsvd_idx_lebs = 0; + lebs = c->lst.empty_lebs + c->freeable_cnt + c->idx_gc_cnt - + c->lst.taken_empty_lebs; + if (rsvd_idx_lebs < lebs) + /* + * OK to allocate an empty LEB, but we still don't want to go + * looking for one if there aren't any. + */ + if (c->lst.empty_lebs - c->lst.taken_empty_lebs > 0) { + pick_free = 1; + /* + * Because we release the space lock, we must account + * for this allocation here. After the LEB properties + * flags have been updated, we subtract one. Note, the + * result of this is that lprops also decreases + * @taken_empty_lebs in 'ubifs_change_lp()', so it is + * off by one for a short period of time which may + * introduce a small disturbance to budgeting + * calculations, but this is harmless because at the + * worst case this would make the budgeting subsystem + * be more pessimistic than needed. + * + * Fundamentally, this is about serialization of the + * budgeting and lprops subsystems. We could make the + * @space_lock a mutex and avoid dropping it before + * calling 'ubifs_change_lp()', but mutex is more + * heavy-weight, and we want budgeting to be as fast as + * possible. + */ + c->lst.taken_empty_lebs += 1; + } + spin_unlock(&c->space_lock); + + lprops = do_find_free_space(c, min_space, pick_free, squeeze); + if (IS_ERR(lprops)) { + err = PTR_ERR(lprops); + goto out; + } + + lnum = lprops->lnum; + flags = lprops->flags | LPROPS_TAKEN; + + lprops = ubifs_change_lp(c, lprops, LPROPS_NC, LPROPS_NC, flags, 0); + if (IS_ERR(lprops)) { + err = PTR_ERR(lprops); + goto out; + } + + if (pick_free) { + spin_lock(&c->space_lock); + c->lst.taken_empty_lebs -= 1; + spin_unlock(&c->space_lock); + } + + *offs = c->leb_size - lprops->free; + ubifs_release_lprops(c); + + if (*offs == 0) { + /* + * Ensure that empty LEBs have been unmapped. They may not have + * been, for example, because of an unclean unmount. Also + * LEBs that were freeable LEBs (free + dirty == leb_size) will + * not have been unmapped. + */ + err = ubifs_leb_unmap(c, lnum); + if (err) + return err; + } + + dbg_find("found LEB %d, free %d", lnum, c->leb_size - *offs); + ubifs_assert(c, *offs <= c->leb_size - min_space); + return lnum; + +out: + if (pick_free) { + spin_lock(&c->space_lock); + c->lst.taken_empty_lebs -= 1; + spin_unlock(&c->space_lock); + } + ubifs_release_lprops(c); + return err; +} + +/** + * scan_for_idx_cb - callback used by the scan for a free LEB for the index. + * @c: the UBIFS file-system description object + * @lprops: LEB properties to scan + * @in_tree: whether the LEB properties are in main memory + * @arg: information passed to and from the caller of the scan + * + * This function returns a code that indicates whether the scan should continue + * (%LPT_SCAN_CONTINUE), whether the LEB properties should be added to the tree + * in main memory (%LPT_SCAN_ADD), or whether the scan should stop + * (%LPT_SCAN_STOP). + */ +static int scan_for_idx_cb(struct ubifs_info *c, + const struct ubifs_lprops *lprops, int in_tree, + void *arg) +{ + struct scan_data *data = arg; + int ret = LPT_SCAN_CONTINUE; + + /* Exclude LEBs that are currently in use */ + if (lprops->flags & LPROPS_TAKEN) + return LPT_SCAN_CONTINUE; + /* Determine whether to add these LEB properties to the tree */ + if (!in_tree && valuable(c, lprops)) + ret |= LPT_SCAN_ADD; + /* Exclude index LEBS */ + if (lprops->flags & LPROPS_INDEX) + return ret; + /* Exclude LEBs that cannot be made empty */ + if (lprops->free + lprops->dirty != c->leb_size) + return ret; + /* + * We are allocating for the index so it is safe to allocate LEBs with + * only free and dirty space, because write buffers are sync'd at commit + * start. + */ + data->lnum = lprops->lnum; + return LPT_SCAN_ADD | LPT_SCAN_STOP; +} + +/** + * scan_for_leb_for_idx - scan for a free LEB for the index. + * @c: the UBIFS file-system description object + */ +static const struct ubifs_lprops *scan_for_leb_for_idx(struct ubifs_info *c) +{ + const struct ubifs_lprops *lprops; + struct scan_data data; + int err; + + data.lnum = -1; + err = ubifs_lpt_scan_nolock(c, -1, c->lscan_lnum, scan_for_idx_cb, + &data); + if (err) + return ERR_PTR(err); + ubifs_assert(c, data.lnum >= c->main_first && data.lnum < c->leb_cnt); + c->lscan_lnum = data.lnum; + lprops = ubifs_lpt_lookup_dirty(c, data.lnum); + if (IS_ERR(lprops)) + return lprops; + ubifs_assert(c, lprops->lnum == data.lnum); + ubifs_assert(c, lprops->free + lprops->dirty == c->leb_size); + ubifs_assert(c, !(lprops->flags & LPROPS_TAKEN)); + ubifs_assert(c, !(lprops->flags & LPROPS_INDEX)); + return lprops; +} + +/** + * ubifs_find_free_leb_for_idx - find a free LEB for the index. + * @c: the UBIFS file-system description object + * + * This function looks for a free LEB and returns that LEB number. The returned + * LEB is marked as "taken", "index". + * + * Only empty LEBs are allocated. This is for two reasons. First, the commit + * calculates the number of LEBs to allocate based on the assumption that they + * will be empty. Secondly, free space at the end of an index LEB is not + * guaranteed to be empty because it may have been used by the in-the-gaps + * method prior to an unclean unmount. + * + * If no LEB is found %-ENOSPC is returned. For other failures another negative + * error code is returned. + */ +int ubifs_find_free_leb_for_idx(struct ubifs_info *c) +{ + const struct ubifs_lprops *lprops; + int lnum = -1, err, flags; + + ubifs_get_lprops(c); + + lprops = ubifs_fast_find_empty(c); + if (!lprops) { + lprops = ubifs_fast_find_freeable(c); + if (!lprops) { + /* + * The first condition means the following: go scan the + * LPT if there are uncategorized lprops, which means + * there may be freeable LEBs there (UBIFS does not + * store the information about freeable LEBs in the + * master node). + */ + if (c->in_a_category_cnt != c->main_lebs || + c->lst.empty_lebs - c->lst.taken_empty_lebs > 0) { + ubifs_assert(c, c->freeable_cnt == 0); + lprops = scan_for_leb_for_idx(c); + if (IS_ERR(lprops)) { + err = PTR_ERR(lprops); + goto out; + } + } + } + } + + if (!lprops) { + err = -ENOSPC; + goto out; + } + + lnum = lprops->lnum; + + dbg_find("found LEB %d, free %d, dirty %d, flags %#x", + lnum, lprops->free, lprops->dirty, lprops->flags); + + flags = lprops->flags | LPROPS_TAKEN | LPROPS_INDEX; + lprops = ubifs_change_lp(c, lprops, c->leb_size, 0, flags, 0); + if (IS_ERR(lprops)) { + err = PTR_ERR(lprops); + goto out; + } + + ubifs_release_lprops(c); + + /* + * Ensure that empty LEBs have been unmapped. They may not have been, + * for example, because of an unclean unmount. Also LEBs that were + * freeable LEBs (free + dirty == leb_size) will not have been unmapped. + */ + err = ubifs_leb_unmap(c, lnum); + if (err) { + ubifs_change_one_lp(c, lnum, LPROPS_NC, LPROPS_NC, 0, + LPROPS_TAKEN | LPROPS_INDEX, 0); + return err; + } + + return lnum; + +out: + ubifs_release_lprops(c); + return err; +} + +static int cmp_dirty_idx(const void *a, const void *b) +{ + const struct ubifs_lprops *lpa = *(const struct ubifs_lprops **)a; + const struct ubifs_lprops *lpb = *(const struct ubifs_lprops **)b; + + return lpa->dirty + lpa->free - lpb->dirty - lpb->free; +} + +/** + * ubifs_save_dirty_idx_lnums - save an array of the most dirty index LEB nos. + * @c: the UBIFS file-system description object + * + * This function is called each commit to create an array of LEB numbers of + * dirty index LEBs sorted in order of dirty and free space. This is used by + * the in-the-gaps method of TNC commit. + */ +int ubifs_save_dirty_idx_lnums(struct ubifs_info *c) +{ + int i; + + ubifs_get_lprops(c); + /* Copy the LPROPS_DIRTY_IDX heap */ + c->dirty_idx.cnt = c->lpt_heap[LPROPS_DIRTY_IDX - 1].cnt; + memcpy(c->dirty_idx.arr, c->lpt_heap[LPROPS_DIRTY_IDX - 1].arr, + sizeof(void *) * c->dirty_idx.cnt); + /* Sort it so that the dirtiest is now at the end */ + sort(c->dirty_idx.arr, c->dirty_idx.cnt, sizeof(void *), + cmp_dirty_idx, NULL); + dbg_find("found %d dirty index LEBs", c->dirty_idx.cnt); + if (c->dirty_idx.cnt) + dbg_find("dirtiest index LEB is %d with dirty %d and free %d", + c->dirty_idx.arr[c->dirty_idx.cnt - 1]->lnum, + c->dirty_idx.arr[c->dirty_idx.cnt - 1]->dirty, + c->dirty_idx.arr[c->dirty_idx.cnt - 1]->free); + /* Replace the lprops pointers with LEB numbers */ + for (i = 0; i < c->dirty_idx.cnt; i++) + c->dirty_idx.arr[i] = (void *)(size_t)c->dirty_idx.arr[i]->lnum; + ubifs_release_lprops(c); + return 0; +} + +/** + * scan_dirty_idx_cb - callback used by the scan for a dirty index LEB. + * @c: the UBIFS file-system description object + * @lprops: LEB properties to scan + * @in_tree: whether the LEB properties are in main memory + * @arg: information passed to and from the caller of the scan + * + * This function returns a code that indicates whether the scan should continue + * (%LPT_SCAN_CONTINUE), whether the LEB properties should be added to the tree + * in main memory (%LPT_SCAN_ADD), or whether the scan should stop + * (%LPT_SCAN_STOP). + */ +static int scan_dirty_idx_cb(struct ubifs_info *c, + const struct ubifs_lprops *lprops, int in_tree, + void *arg) +{ + struct scan_data *data = arg; + int ret = LPT_SCAN_CONTINUE; + + /* Exclude LEBs that are currently in use */ + if (lprops->flags & LPROPS_TAKEN) + return LPT_SCAN_CONTINUE; + /* Determine whether to add these LEB properties to the tree */ + if (!in_tree && valuable(c, lprops)) + ret |= LPT_SCAN_ADD; + /* Exclude non-index LEBs */ + if (!(lprops->flags & LPROPS_INDEX)) + return ret; + /* Exclude LEBs with too little space */ + if (lprops->free + lprops->dirty < c->min_idx_node_sz) + return ret; + /* Finally we found space */ + data->lnum = lprops->lnum; + return LPT_SCAN_ADD | LPT_SCAN_STOP; +} + +/** + * find_dirty_idx_leb - find a dirty index LEB. + * @c: the UBIFS file-system description object + * + * This function returns LEB number upon success and a negative error code upon + * failure. In particular, -ENOSPC is returned if a dirty index LEB is not + * found. + * + * Note that this function scans the entire LPT but it is called very rarely. + */ +static int find_dirty_idx_leb(struct ubifs_info *c) +{ + const struct ubifs_lprops *lprops; + struct ubifs_lpt_heap *heap; + struct scan_data data; + int err, i, ret; + + /* Check all structures in memory first */ + data.lnum = -1; + heap = &c->lpt_heap[LPROPS_DIRTY_IDX - 1]; + for (i = 0; i < heap->cnt; i++) { + lprops = heap->arr[i]; + ret = scan_dirty_idx_cb(c, lprops, 1, &data); + if (ret & LPT_SCAN_STOP) + goto found; + } + list_for_each_entry(lprops, &c->frdi_idx_list, list) { + ret = scan_dirty_idx_cb(c, lprops, 1, &data); + if (ret & LPT_SCAN_STOP) + goto found; + } + list_for_each_entry(lprops, &c->uncat_list, list) { + ret = scan_dirty_idx_cb(c, lprops, 1, &data); + if (ret & LPT_SCAN_STOP) + goto found; + } + if (c->pnodes_have >= c->pnode_cnt) + /* All pnodes are in memory, so skip scan */ + return -ENOSPC; + err = ubifs_lpt_scan_nolock(c, -1, c->lscan_lnum, scan_dirty_idx_cb, + &data); + if (err) + return err; +found: + ubifs_assert(c, data.lnum >= c->main_first && data.lnum < c->leb_cnt); + c->lscan_lnum = data.lnum; + lprops = ubifs_lpt_lookup_dirty(c, data.lnum); + if (IS_ERR(lprops)) + return PTR_ERR(lprops); + ubifs_assert(c, lprops->lnum == data.lnum); + ubifs_assert(c, lprops->free + lprops->dirty >= c->min_idx_node_sz); + ubifs_assert(c, !(lprops->flags & LPROPS_TAKEN)); + ubifs_assert(c, (lprops->flags & LPROPS_INDEX)); + + dbg_find("found dirty LEB %d, free %d, dirty %d, flags %#x", + lprops->lnum, lprops->free, lprops->dirty, lprops->flags); + + lprops = ubifs_change_lp(c, lprops, LPROPS_NC, LPROPS_NC, + lprops->flags | LPROPS_TAKEN, 0); + if (IS_ERR(lprops)) + return PTR_ERR(lprops); + + return lprops->lnum; +} + +/** + * get_idx_gc_leb - try to get a LEB number from trivial GC. + * @c: the UBIFS file-system description object + */ +static int get_idx_gc_leb(struct ubifs_info *c) +{ + const struct ubifs_lprops *lp; + int err, lnum; + + err = ubifs_get_idx_gc_leb(c); + if (err < 0) + return err; + lnum = err; + /* + * The LEB was due to be unmapped after the commit but + * it is needed now for this commit. + */ + lp = ubifs_lpt_lookup_dirty(c, lnum); + if (IS_ERR(lp)) + return PTR_ERR(lp); + lp = ubifs_change_lp(c, lp, LPROPS_NC, LPROPS_NC, + lp->flags | LPROPS_INDEX, -1); + if (IS_ERR(lp)) + return PTR_ERR(lp); + dbg_find("LEB %d, dirty %d and free %d flags %#x", + lp->lnum, lp->dirty, lp->free, lp->flags); + return lnum; +} + +/** + * find_dirtiest_idx_leb - find dirtiest index LEB from dirtiest array. + * @c: the UBIFS file-system description object + */ +static int find_dirtiest_idx_leb(struct ubifs_info *c) +{ + const struct ubifs_lprops *lp; + int lnum; + + while (1) { + if (!c->dirty_idx.cnt) + return -ENOSPC; + /* The lprops pointers were replaced by LEB numbers */ + lnum = (size_t)c->dirty_idx.arr[--c->dirty_idx.cnt]; + lp = ubifs_lpt_lookup(c, lnum); + if (IS_ERR(lp)) + return PTR_ERR(lp); + if ((lp->flags & LPROPS_TAKEN) || !(lp->flags & LPROPS_INDEX)) + continue; + lp = ubifs_change_lp(c, lp, LPROPS_NC, LPROPS_NC, + lp->flags | LPROPS_TAKEN, 0); + if (IS_ERR(lp)) + return PTR_ERR(lp); + break; + } + dbg_find("LEB %d, dirty %d and free %d flags %#x", lp->lnum, lp->dirty, + lp->free, lp->flags); + ubifs_assert(c, lp->flags & LPROPS_TAKEN); + ubifs_assert(c, lp->flags & LPROPS_INDEX); + return lnum; +} + +/** + * ubifs_find_dirty_idx_leb - try to find dirtiest index LEB as at last commit. + * @c: the UBIFS file-system description object + * + * This function attempts to find an untaken index LEB with the most free and + * dirty space that can be used without overwriting index nodes that were in the + * last index committed. + */ +int ubifs_find_dirty_idx_leb(struct ubifs_info *c) +{ + int err; + + ubifs_get_lprops(c); + + /* + * We made an array of the dirtiest index LEB numbers as at the start of + * last commit. Try that array first. + */ + err = find_dirtiest_idx_leb(c); + + /* Next try scanning the entire LPT */ + if (err == -ENOSPC) + err = find_dirty_idx_leb(c); + + /* Finally take any index LEBs awaiting trivial GC */ + if (err == -ENOSPC) + err = get_idx_gc_leb(c); + + ubifs_release_lprops(c); + return err; +} diff --git a/fs/ubifs/gc.c b/fs/ubifs/gc.c new file mode 100644 index 000000000..933c79b5c --- /dev/null +++ b/fs/ubifs/gc.c @@ -0,0 +1,1015 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * + * Authors: Adrian Hunter + * Artem Bityutskiy (Битюцкий Артём) + */ + +/* + * This file implements garbage collection. The procedure for garbage collection + * is different depending on whether a LEB as an index LEB (contains index + * nodes) or not. For non-index LEBs, garbage collection finds a LEB which + * contains a lot of dirty space (obsolete nodes), and copies the non-obsolete + * nodes to the journal, at which point the garbage-collected LEB is free to be + * reused. For index LEBs, garbage collection marks the non-obsolete index nodes + * dirty in the TNC, and after the next commit, the garbage-collected LEB is + * to be reused. Garbage collection will cause the number of dirty index nodes + * to grow, however sufficient space is reserved for the index to ensure the + * commit will never run out of space. + * + * Notes about dead watermark. At current UBIFS implementation we assume that + * LEBs which have less than @c->dead_wm bytes of free + dirty space are full + * and not worth garbage-collecting. The dead watermark is one min. I/O unit + * size, or min. UBIFS node size, depending on what is greater. Indeed, UBIFS + * Garbage Collector has to synchronize the GC head's write buffer before + * returning, so this is about wasting one min. I/O unit. However, UBIFS GC can + * actually reclaim even very small pieces of dirty space by garbage collecting + * enough dirty LEBs, but we do not bother doing this at this implementation. + * + * Notes about dark watermark. The results of GC work depends on how big are + * the UBIFS nodes GC deals with. Large nodes make GC waste more space. Indeed, + * if GC move data from LEB A to LEB B and nodes in LEB A are large, GC would + * have to waste large pieces of free space at the end of LEB B, because nodes + * from LEB A would not fit. And the worst situation is when all nodes are of + * maximum size. So dark watermark is the amount of free + dirty space in LEB + * which are guaranteed to be reclaimable. If LEB has less space, the GC might + * be unable to reclaim it. So, LEBs with free + dirty greater than dark + * watermark are "good" LEBs from GC's point of view. The other LEBs are not so + * good, and GC takes extra care when moving them. + */ + +#include +#include +#include +#include "ubifs.h" + +/* + * GC may need to move more than one LEB to make progress. The below constants + * define "soft" and "hard" limits on the number of LEBs the garbage collector + * may move. + */ +#define SOFT_LEBS_LIMIT 4 +#define HARD_LEBS_LIMIT 32 + +/** + * switch_gc_head - switch the garbage collection journal head. + * @c: UBIFS file-system description object + * + * This function switch the GC head to the next LEB which is reserved in + * @c->gc_lnum. Returns %0 in case of success, %-EAGAIN if commit is required, + * and other negative error code in case of failures. + */ +static int switch_gc_head(struct ubifs_info *c) +{ + int err, gc_lnum = c->gc_lnum; + struct ubifs_wbuf *wbuf = &c->jheads[GCHD].wbuf; + + ubifs_assert(c, gc_lnum != -1); + dbg_gc("switch GC head from LEB %d:%d to LEB %d (waste %d bytes)", + wbuf->lnum, wbuf->offs + wbuf->used, gc_lnum, + c->leb_size - wbuf->offs - wbuf->used); + + err = ubifs_wbuf_sync_nolock(wbuf); + if (err) + return err; + + /* + * The GC write-buffer was synchronized, we may safely unmap + * 'c->gc_lnum'. + */ + err = ubifs_leb_unmap(c, gc_lnum); + if (err) + return err; + + err = ubifs_add_bud_to_log(c, GCHD, gc_lnum, 0); + if (err) + return err; + + c->gc_lnum = -1; + err = ubifs_wbuf_seek_nolock(wbuf, gc_lnum, 0); + return err; +} + +/** + * data_nodes_cmp - compare 2 data nodes. + * @priv: UBIFS file-system description object + * @a: first data node + * @b: second data node + * + * This function compares data nodes @a and @b. Returns %1 if @a has greater + * inode or block number, and %-1 otherwise. + */ +static int data_nodes_cmp(void *priv, const struct list_head *a, + const struct list_head *b) +{ + ino_t inuma, inumb; + struct ubifs_info *c = priv; + struct ubifs_scan_node *sa, *sb; + + if (a == b) + return 0; + + sa = list_entry(a, struct ubifs_scan_node, list); + sb = list_entry(b, struct ubifs_scan_node, list); + + ubifs_assert(c, key_type(c, &sa->key) == UBIFS_DATA_KEY); + ubifs_assert(c, key_type(c, &sb->key) == UBIFS_DATA_KEY); + ubifs_assert(c, sa->type == UBIFS_DATA_NODE); + ubifs_assert(c, sb->type == UBIFS_DATA_NODE); + + inuma = key_inum(c, &sa->key); + inumb = key_inum(c, &sb->key); + + if (inuma == inumb) { + unsigned int blka = key_block(c, &sa->key); + unsigned int blkb = key_block(c, &sb->key); + + if (blka <= blkb) + return -1; + } else if (inuma <= inumb) + return -1; + + return 1; +} + +/* + * nondata_nodes_cmp - compare 2 non-data nodes. + * @priv: UBIFS file-system description object + * @a: first node + * @a: second node + * + * This function compares nodes @a and @b. It makes sure that inode nodes go + * first and sorted by length in descending order. Directory entry nodes go + * after inode nodes and are sorted in ascending hash valuer order. + */ +static int nondata_nodes_cmp(void *priv, const struct list_head *a, + const struct list_head *b) +{ + ino_t inuma, inumb; + struct ubifs_info *c = priv; + struct ubifs_scan_node *sa, *sb; + + if (a == b) + return 0; + + sa = list_entry(a, struct ubifs_scan_node, list); + sb = list_entry(b, struct ubifs_scan_node, list); + + ubifs_assert(c, key_type(c, &sa->key) != UBIFS_DATA_KEY && + key_type(c, &sb->key) != UBIFS_DATA_KEY); + ubifs_assert(c, sa->type != UBIFS_DATA_NODE && + sb->type != UBIFS_DATA_NODE); + + /* Inodes go before directory entries */ + if (sa->type == UBIFS_INO_NODE) { + if (sb->type == UBIFS_INO_NODE) + return sb->len - sa->len; + return -1; + } + if (sb->type == UBIFS_INO_NODE) + return 1; + + ubifs_assert(c, key_type(c, &sa->key) == UBIFS_DENT_KEY || + key_type(c, &sa->key) == UBIFS_XENT_KEY); + ubifs_assert(c, key_type(c, &sb->key) == UBIFS_DENT_KEY || + key_type(c, &sb->key) == UBIFS_XENT_KEY); + ubifs_assert(c, sa->type == UBIFS_DENT_NODE || + sa->type == UBIFS_XENT_NODE); + ubifs_assert(c, sb->type == UBIFS_DENT_NODE || + sb->type == UBIFS_XENT_NODE); + + inuma = key_inum(c, &sa->key); + inumb = key_inum(c, &sb->key); + + if (inuma == inumb) { + uint32_t hasha = key_hash(c, &sa->key); + uint32_t hashb = key_hash(c, &sb->key); + + if (hasha <= hashb) + return -1; + } else if (inuma <= inumb) + return -1; + + return 1; +} + +/** + * sort_nodes - sort nodes for GC. + * @c: UBIFS file-system description object + * @sleb: describes nodes to sort and contains the result on exit + * @nondata: contains non-data nodes on exit + * @min: minimum node size is returned here + * + * This function sorts the list of inodes to garbage collect. First of all, it + * kills obsolete nodes and separates data and non-data nodes to the + * @sleb->nodes and @nondata lists correspondingly. + * + * Data nodes are then sorted in block number order - this is important for + * bulk-read; data nodes with lower inode number go before data nodes with + * higher inode number, and data nodes with lower block number go before data + * nodes with higher block number; + * + * Non-data nodes are sorted as follows. + * o First go inode nodes - they are sorted in descending length order. + * o Then go directory entry nodes - they are sorted in hash order, which + * should supposedly optimize 'readdir()'. Direntry nodes with lower parent + * inode number go before direntry nodes with higher parent inode number, + * and direntry nodes with lower name hash values go before direntry nodes + * with higher name hash values. + * + * This function returns zero in case of success and a negative error code in + * case of failure. + */ +static int sort_nodes(struct ubifs_info *c, struct ubifs_scan_leb *sleb, + struct list_head *nondata, int *min) +{ + int err; + struct ubifs_scan_node *snod, *tmp; + + *min = INT_MAX; + + /* Separate data nodes and non-data nodes */ + list_for_each_entry_safe(snod, tmp, &sleb->nodes, list) { + ubifs_assert(c, snod->type == UBIFS_INO_NODE || + snod->type == UBIFS_DATA_NODE || + snod->type == UBIFS_DENT_NODE || + snod->type == UBIFS_XENT_NODE || + snod->type == UBIFS_TRUN_NODE || + snod->type == UBIFS_AUTH_NODE); + + if (snod->type != UBIFS_INO_NODE && + snod->type != UBIFS_DATA_NODE && + snod->type != UBIFS_DENT_NODE && + snod->type != UBIFS_XENT_NODE) { + /* Probably truncation node, zap it */ + list_del(&snod->list); + kfree(snod); + continue; + } + + ubifs_assert(c, key_type(c, &snod->key) == UBIFS_DATA_KEY || + key_type(c, &snod->key) == UBIFS_INO_KEY || + key_type(c, &snod->key) == UBIFS_DENT_KEY || + key_type(c, &snod->key) == UBIFS_XENT_KEY); + + err = ubifs_tnc_has_node(c, &snod->key, 0, sleb->lnum, + snod->offs, 0); + if (err < 0) + return err; + + if (!err) { + /* The node is obsolete, remove it from the list */ + list_del(&snod->list); + kfree(snod); + continue; + } + + if (snod->len < *min) + *min = snod->len; + + if (key_type(c, &snod->key) != UBIFS_DATA_KEY) + list_move_tail(&snod->list, nondata); + } + + /* Sort data and non-data nodes */ + list_sort(c, &sleb->nodes, &data_nodes_cmp); + list_sort(c, nondata, &nondata_nodes_cmp); + + err = dbg_check_data_nodes_order(c, &sleb->nodes); + if (err) + return err; + err = dbg_check_nondata_nodes_order(c, nondata); + if (err) + return err; + return 0; +} + +/** + * move_node - move a node. + * @c: UBIFS file-system description object + * @sleb: describes the LEB to move nodes from + * @snod: the mode to move + * @wbuf: write-buffer to move node to + * + * This function moves node @snod to @wbuf, changes TNC correspondingly, and + * destroys @snod. Returns zero in case of success and a negative error code in + * case of failure. + */ +static int move_node(struct ubifs_info *c, struct ubifs_scan_leb *sleb, + struct ubifs_scan_node *snod, struct ubifs_wbuf *wbuf) +{ + int err, new_lnum = wbuf->lnum, new_offs = wbuf->offs + wbuf->used; + + cond_resched(); + err = ubifs_wbuf_write_nolock(wbuf, snod->node, snod->len); + if (err) + return err; + + err = ubifs_tnc_replace(c, &snod->key, sleb->lnum, + snod->offs, new_lnum, new_offs, + snod->len); + list_del(&snod->list); + kfree(snod); + return err; +} + +/** + * move_nodes - move nodes. + * @c: UBIFS file-system description object + * @sleb: describes the LEB to move nodes from + * + * This function moves valid nodes from data LEB described by @sleb to the GC + * journal head. This function returns zero in case of success, %-EAGAIN if + * commit is required, and other negative error codes in case of other + * failures. + */ +static int move_nodes(struct ubifs_info *c, struct ubifs_scan_leb *sleb) +{ + int err, min; + LIST_HEAD(nondata); + struct ubifs_wbuf *wbuf = &c->jheads[GCHD].wbuf; + + if (wbuf->lnum == -1) { + /* + * The GC journal head is not set, because it is the first GC + * invocation since mount. + */ + err = switch_gc_head(c); + if (err) + return err; + } + + err = sort_nodes(c, sleb, &nondata, &min); + if (err) + goto out; + + /* Write nodes to their new location. Use the first-fit strategy */ + while (1) { + int avail, moved = 0; + struct ubifs_scan_node *snod, *tmp; + + /* Move data nodes */ + list_for_each_entry_safe(snod, tmp, &sleb->nodes, list) { + avail = c->leb_size - wbuf->offs - wbuf->used - + ubifs_auth_node_sz(c); + if (snod->len > avail) + /* + * Do not skip data nodes in order to optimize + * bulk-read. + */ + break; + + err = ubifs_shash_update(c, c->jheads[GCHD].log_hash, + snod->node, snod->len); + if (err) + goto out; + + err = move_node(c, sleb, snod, wbuf); + if (err) + goto out; + moved = 1; + } + + /* Move non-data nodes */ + list_for_each_entry_safe(snod, tmp, &nondata, list) { + avail = c->leb_size - wbuf->offs - wbuf->used - + ubifs_auth_node_sz(c); + if (avail < min) + break; + + if (snod->len > avail) { + /* + * Keep going only if this is an inode with + * some data. Otherwise stop and switch the GC + * head. IOW, we assume that data-less inode + * nodes and direntry nodes are roughly of the + * same size. + */ + if (key_type(c, &snod->key) == UBIFS_DENT_KEY || + snod->len == UBIFS_INO_NODE_SZ) + break; + continue; + } + + err = ubifs_shash_update(c, c->jheads[GCHD].log_hash, + snod->node, snod->len); + if (err) + goto out; + + err = move_node(c, sleb, snod, wbuf); + if (err) + goto out; + moved = 1; + } + + if (ubifs_authenticated(c) && moved) { + struct ubifs_auth_node *auth; + + auth = kmalloc(ubifs_auth_node_sz(c), GFP_NOFS); + if (!auth) { + err = -ENOMEM; + goto out; + } + + err = ubifs_prepare_auth_node(c, auth, + c->jheads[GCHD].log_hash); + if (err) { + kfree(auth); + goto out; + } + + err = ubifs_wbuf_write_nolock(wbuf, auth, + ubifs_auth_node_sz(c)); + if (err) { + kfree(auth); + goto out; + } + + ubifs_add_dirt(c, wbuf->lnum, ubifs_auth_node_sz(c)); + } + + if (list_empty(&sleb->nodes) && list_empty(&nondata)) + break; + + /* + * Waste the rest of the space in the LEB and switch to the + * next LEB. + */ + err = switch_gc_head(c); + if (err) + goto out; + } + + return 0; + +out: + list_splice_tail(&nondata, &sleb->nodes); + return err; +} + +/** + * gc_sync_wbufs - sync write-buffers for GC. + * @c: UBIFS file-system description object + * + * We must guarantee that obsoleting nodes are on flash. Unfortunately they may + * be in a write-buffer instead. That is, a node could be written to a + * write-buffer, obsoleting another node in a LEB that is GC'd. If that LEB is + * erased before the write-buffer is sync'd and then there is an unclean + * unmount, then an existing node is lost. To avoid this, we sync all + * write-buffers. + * + * This function returns %0 on success or a negative error code on failure. + */ +static int gc_sync_wbufs(struct ubifs_info *c) +{ + int err, i; + + for (i = 0; i < c->jhead_cnt; i++) { + if (i == GCHD) + continue; + err = ubifs_wbuf_sync(&c->jheads[i].wbuf); + if (err) + return err; + } + return 0; +} + +/** + * ubifs_garbage_collect_leb - garbage-collect a logical eraseblock. + * @c: UBIFS file-system description object + * @lp: describes the LEB to garbage collect + * + * This function garbage-collects an LEB and returns one of the @LEB_FREED, + * @LEB_RETAINED, etc positive codes in case of success, %-EAGAIN if commit is + * required, and other negative error codes in case of failures. + */ +int ubifs_garbage_collect_leb(struct ubifs_info *c, struct ubifs_lprops *lp) +{ + struct ubifs_scan_leb *sleb; + struct ubifs_scan_node *snod; + struct ubifs_wbuf *wbuf = &c->jheads[GCHD].wbuf; + int err = 0, lnum = lp->lnum; + + ubifs_assert(c, c->gc_lnum != -1 || wbuf->offs + wbuf->used == 0 || + c->need_recovery); + ubifs_assert(c, c->gc_lnum != lnum); + ubifs_assert(c, wbuf->lnum != lnum); + + if (lp->free + lp->dirty == c->leb_size) { + /* Special case - a free LEB */ + dbg_gc("LEB %d is free, return it", lp->lnum); + ubifs_assert(c, !(lp->flags & LPROPS_INDEX)); + + if (lp->free != c->leb_size) { + /* + * Write buffers must be sync'd before unmapping + * freeable LEBs, because one of them may contain data + * which obsoletes something in 'lp->lnum'. + */ + err = gc_sync_wbufs(c); + if (err) + return err; + err = ubifs_change_one_lp(c, lp->lnum, c->leb_size, + 0, 0, 0, 0); + if (err) + return err; + } + err = ubifs_leb_unmap(c, lp->lnum); + if (err) + return err; + + if (c->gc_lnum == -1) { + c->gc_lnum = lnum; + return LEB_RETAINED; + } + + return LEB_FREED; + } + + /* + * We scan the entire LEB even though we only really need to scan up to + * (c->leb_size - lp->free). + */ + sleb = ubifs_scan(c, lnum, 0, c->sbuf, 0); + if (IS_ERR(sleb)) + return PTR_ERR(sleb); + + ubifs_assert(c, !list_empty(&sleb->nodes)); + snod = list_entry(sleb->nodes.next, struct ubifs_scan_node, list); + + if (snod->type == UBIFS_IDX_NODE) { + struct ubifs_gced_idx_leb *idx_gc; + + dbg_gc("indexing LEB %d (free %d, dirty %d)", + lnum, lp->free, lp->dirty); + list_for_each_entry(snod, &sleb->nodes, list) { + struct ubifs_idx_node *idx = snod->node; + int level = le16_to_cpu(idx->level); + + ubifs_assert(c, snod->type == UBIFS_IDX_NODE); + key_read(c, ubifs_idx_key(c, idx), &snod->key); + err = ubifs_dirty_idx_node(c, &snod->key, level, lnum, + snod->offs); + if (err) + goto out; + } + + idx_gc = kmalloc_obj(struct ubifs_gced_idx_leb, GFP_NOFS); + if (!idx_gc) { + err = -ENOMEM; + goto out; + } + + idx_gc->lnum = lnum; + idx_gc->unmap = 0; + list_add(&idx_gc->list, &c->idx_gc); + + /* + * Don't release the LEB until after the next commit, because + * it may contain data which is needed for recovery. So + * although we freed this LEB, it will become usable only after + * the commit. + */ + err = ubifs_change_one_lp(c, lnum, c->leb_size, 0, 0, + LPROPS_INDEX, 1); + if (err) + goto out; + err = LEB_FREED_IDX; + } else { + dbg_gc("data LEB %d (free %d, dirty %d)", + lnum, lp->free, lp->dirty); + + err = move_nodes(c, sleb); + if (err) + goto out_inc_seq; + + err = gc_sync_wbufs(c); + if (err) + goto out_inc_seq; + + err = ubifs_change_one_lp(c, lnum, c->leb_size, 0, 0, 0, 0); + if (err) + goto out_inc_seq; + + /* Allow for races with TNC */ + c->gced_lnum = lnum; + smp_wmb(); + c->gc_seq += 1; + smp_wmb(); + + if (c->gc_lnum == -1) { + c->gc_lnum = lnum; + err = LEB_RETAINED; + } else { + err = ubifs_wbuf_sync_nolock(wbuf); + if (err) + goto out; + + err = ubifs_leb_unmap(c, lnum); + if (err) + goto out; + + err = LEB_FREED; + } + } + +out: + ubifs_scan_destroy(sleb); + return err; + +out_inc_seq: + /* We may have moved at least some nodes so allow for races with TNC */ + c->gced_lnum = lnum; + smp_wmb(); + c->gc_seq += 1; + smp_wmb(); + goto out; +} + +/** + * ubifs_garbage_collect - UBIFS garbage collector. + * @c: UBIFS file-system description object + * @anyway: do GC even if there are free LEBs + * + * This function does out-of-place garbage collection. The return codes are: + * o positive LEB number if the LEB has been freed and may be used; + * o %-EAGAIN if the caller has to run commit; + * o %-ENOSPC if GC failed to make any progress; + * o other negative error codes in case of other errors. + * + * Garbage collector writes data to the journal when GC'ing data LEBs, and just + * marking indexing nodes dirty when GC'ing indexing LEBs. Thus, at some point + * commit may be required. But commit cannot be run from inside GC, because the + * caller might be holding the commit lock, so %-EAGAIN is returned instead; + * And this error code means that the caller has to run commit, and re-run GC + * if there is still no free space. + * + * There are many reasons why this function may return %-EAGAIN: + * o the log is full and there is no space to write an LEB reference for + * @c->gc_lnum; + * o the journal is too large and exceeds size limitations; + * o GC moved indexing LEBs, but they can be used only after the commit; + * o the shrinker fails to find clean znodes to free and requests the commit; + * o etc. + * + * Note, if the file-system is close to be full, this function may return + * %-EAGAIN infinitely, so the caller has to limit amount of re-invocations of + * the function. E.g., this happens if the limits on the journal size are too + * tough and GC writes too much to the journal before an LEB is freed. This + * might also mean that the journal is too large, and the TNC becomes to big, + * so that the shrinker is constantly called, finds not clean znodes to free, + * and requests commit. Well, this may also happen if the journal is all right, + * but another kernel process consumes too much memory. Anyway, infinite + * %-EAGAIN may happen, but in some extreme/misconfiguration cases. + */ +int ubifs_garbage_collect(struct ubifs_info *c, int anyway) +{ + int i, err, ret, min_space = c->dead_wm; + struct ubifs_lprops lp; + struct ubifs_wbuf *wbuf = &c->jheads[GCHD].wbuf; + + ubifs_assert_cmt_locked(c); + ubifs_assert(c, !c->ro_media && !c->ro_mount); + + if (ubifs_gc_should_commit(c)) + return -EAGAIN; + + mutex_lock_nested(&wbuf->io_mutex, wbuf->jhead); + + if (c->ro_error) { + ret = -EROFS; + goto out_unlock; + } + + /* We expect the write-buffer to be empty on entry */ + ubifs_assert(c, !wbuf->used); + + for (i = 0; ; i++) { + int space_before, space_after; + + /* Maybe continue after find and break before find */ + lp.lnum = -1; + + cond_resched(); + + /* Give the commit an opportunity to run */ + if (ubifs_gc_should_commit(c)) { + ret = -EAGAIN; + break; + } + + if (i > SOFT_LEBS_LIMIT && !list_empty(&c->idx_gc)) { + /* + * We've done enough iterations. Indexing LEBs were + * moved and will be available after the commit. + */ + dbg_gc("soft limit, some index LEBs GC'ed, -EAGAIN"); + ubifs_commit_required(c); + ret = -EAGAIN; + break; + } + + if (i > HARD_LEBS_LIMIT) { + /* + * We've moved too many LEBs and have not made + * progress, give up. + */ + dbg_gc("hard limit, -ENOSPC"); + ret = -ENOSPC; + break; + } + + /* + * Empty and freeable LEBs can turn up while we waited for + * the wbuf lock, or while we have been running GC. In that + * case, we should just return one of those instead of + * continuing to GC dirty LEBs. Hence we request + * 'ubifs_find_dirty_leb()' to return an empty LEB if it can. + */ + ret = ubifs_find_dirty_leb(c, &lp, min_space, anyway ? 0 : 1); + if (ret) { + if (ret == -ENOSPC) + dbg_gc("no more dirty LEBs"); + break; + } + + dbg_gc("found LEB %d: free %d, dirty %d, sum %d (min. space %d)", + lp.lnum, lp.free, lp.dirty, lp.free + lp.dirty, + min_space); + + space_before = c->leb_size - wbuf->offs - wbuf->used; + if (wbuf->lnum == -1) + space_before = 0; + + ret = ubifs_garbage_collect_leb(c, &lp); + if (ret < 0) { + if (ret == -EAGAIN) { + /* + * This is not error, so we have to return the + * LEB to lprops. But if 'ubifs_return_leb()' + * fails, its failure code is propagated to the + * caller instead of the original '-EAGAIN'. + */ + err = ubifs_return_leb(c, lp.lnum); + if (err) { + ret = err; + /* + * An LEB may always be "taken", + * so setting ubifs to read-only, + * and then executing sync wbuf will + * return -EROFS and enter the "out" + * error branch. + */ + ubifs_ro_mode(c, ret); + } + /* Maybe double return LEB if goto out */ + lp.lnum = -1; + break; + } + goto out; + } + + if (ret == LEB_FREED) { + /* An LEB has been freed and is ready for use */ + dbg_gc("LEB %d freed, return", lp.lnum); + ret = lp.lnum; + break; + } + + if (ret == LEB_FREED_IDX) { + /* + * This was an indexing LEB and it cannot be + * immediately used. And instead of requesting the + * commit straight away, we try to garbage collect some + * more. + */ + dbg_gc("indexing LEB %d freed, continue", lp.lnum); + continue; + } + + ubifs_assert(c, ret == LEB_RETAINED); + space_after = c->leb_size - wbuf->offs - wbuf->used; + dbg_gc("LEB %d retained, freed %d bytes", lp.lnum, + space_after - space_before); + + if (space_after > space_before) { + /* GC makes progress, keep working */ + min_space >>= 1; + if (min_space < c->dead_wm) + min_space = c->dead_wm; + continue; + } + + dbg_gc("did not make progress"); + + /* + * GC moved an LEB bud have not done any progress. This means + * that the previous GC head LEB contained too few free space + * and the LEB which was GC'ed contained only large nodes which + * did not fit that space. + * + * We can do 2 things: + * 1. pick another LEB in a hope it'll contain a small node + * which will fit the space we have at the end of current GC + * head LEB, but there is no guarantee, so we try this out + * unless we have already been working for too long; + * 2. request an LEB with more dirty space, which will force + * 'ubifs_find_dirty_leb()' to start scanning the lprops + * table, instead of just picking one from the heap + * (previously it already picked the dirtiest LEB). + */ + if (i < SOFT_LEBS_LIMIT) { + dbg_gc("try again"); + continue; + } + + min_space <<= 1; + if (min_space > c->dark_wm) + min_space = c->dark_wm; + dbg_gc("set min. space to %d", min_space); + } + + if (ret == -ENOSPC && !list_empty(&c->idx_gc)) { + dbg_gc("no space, some index LEBs GC'ed, -EAGAIN"); + ubifs_commit_required(c); + ret = -EAGAIN; + } + + err = ubifs_wbuf_sync_nolock(wbuf); + if (!err) + err = ubifs_leb_unmap(c, c->gc_lnum); + if (err) { + ret = err; + goto out; + } +out_unlock: + mutex_unlock(&wbuf->io_mutex); + return ret; + +out: + ubifs_assert(c, ret < 0); + ubifs_assert(c, ret != -ENOSPC && ret != -EAGAIN); + ubifs_wbuf_sync_nolock(wbuf); + ubifs_ro_mode(c, ret); + mutex_unlock(&wbuf->io_mutex); + if (lp.lnum != -1) + ubifs_return_leb(c, lp.lnum); + return ret; +} + +/** + * ubifs_gc_start_commit - garbage collection at start of commit. + * @c: UBIFS file-system description object + * + * If a LEB has only dirty and free space, then we may safely unmap it and make + * it free. Note, we cannot do this with indexing LEBs because dirty space may + * correspond index nodes that are required for recovery. In that case, the + * LEB cannot be unmapped until after the next commit. + * + * This function returns %0 upon success and a negative error code upon failure. + */ +int ubifs_gc_start_commit(struct ubifs_info *c) +{ + struct ubifs_gced_idx_leb *idx_gc; + const struct ubifs_lprops *lp; + int err = 0, flags; + + ubifs_get_lprops(c); + + /* + * Unmap (non-index) freeable LEBs. Note that recovery requires that all + * wbufs are sync'd before this, which is done in 'do_commit()'. + */ + while (1) { + lp = ubifs_fast_find_freeable(c); + if (!lp) + break; + ubifs_assert(c, !(lp->flags & LPROPS_TAKEN)); + ubifs_assert(c, !(lp->flags & LPROPS_INDEX)); + err = ubifs_leb_unmap(c, lp->lnum); + if (err) + goto out; + lp = ubifs_change_lp(c, lp, c->leb_size, 0, lp->flags, 0); + if (IS_ERR(lp)) { + err = PTR_ERR(lp); + goto out; + } + ubifs_assert(c, !(lp->flags & LPROPS_TAKEN)); + ubifs_assert(c, !(lp->flags & LPROPS_INDEX)); + } + + /* Mark GC'd index LEBs OK to unmap after this commit finishes */ + list_for_each_entry(idx_gc, &c->idx_gc, list) + idx_gc->unmap = 1; + + /* Record index freeable LEBs for unmapping after commit */ + while (1) { + lp = ubifs_fast_find_frdi_idx(c); + if (IS_ERR(lp)) { + err = PTR_ERR(lp); + goto out; + } + if (!lp) + break; + idx_gc = kmalloc_obj(struct ubifs_gced_idx_leb, GFP_NOFS); + if (!idx_gc) { + err = -ENOMEM; + goto out; + } + ubifs_assert(c, !(lp->flags & LPROPS_TAKEN)); + ubifs_assert(c, lp->flags & LPROPS_INDEX); + /* Don't release the LEB until after the next commit */ + flags = (lp->flags | LPROPS_TAKEN) ^ LPROPS_INDEX; + lp = ubifs_change_lp(c, lp, c->leb_size, 0, flags, 1); + if (IS_ERR(lp)) { + err = PTR_ERR(lp); + kfree(idx_gc); + goto out; + } + ubifs_assert(c, lp->flags & LPROPS_TAKEN); + ubifs_assert(c, !(lp->flags & LPROPS_INDEX)); + idx_gc->lnum = lp->lnum; + idx_gc->unmap = 1; + list_add(&idx_gc->list, &c->idx_gc); + } +out: + ubifs_release_lprops(c); + return err; +} + +/** + * ubifs_gc_end_commit - garbage collection at end of commit. + * @c: UBIFS file-system description object + * + * This function completes out-of-place garbage collection of index LEBs. + */ +int ubifs_gc_end_commit(struct ubifs_info *c) +{ + struct ubifs_gced_idx_leb *idx_gc, *tmp; + struct ubifs_wbuf *wbuf; + int err = 0; + + wbuf = &c->jheads[GCHD].wbuf; + mutex_lock_nested(&wbuf->io_mutex, wbuf->jhead); + list_for_each_entry_safe(idx_gc, tmp, &c->idx_gc, list) + if (idx_gc->unmap) { + dbg_gc("LEB %d", idx_gc->lnum); + err = ubifs_leb_unmap(c, idx_gc->lnum); + if (err) + goto out; + err = ubifs_change_one_lp(c, idx_gc->lnum, LPROPS_NC, + LPROPS_NC, 0, LPROPS_TAKEN, -1); + if (err) + goto out; + list_del(&idx_gc->list); + kfree(idx_gc); + } +out: + mutex_unlock(&wbuf->io_mutex); + return err; +} + +/** + * ubifs_destroy_idx_gc - destroy idx_gc list. + * @c: UBIFS file-system description object + * + * This function destroys the @c->idx_gc list. It is called when unmounting + * so locks are not needed. Returns zero in case of success and a negative + * error code in case of failure. + */ +void ubifs_destroy_idx_gc(struct ubifs_info *c) +{ + while (!list_empty(&c->idx_gc)) { + struct ubifs_gced_idx_leb *idx_gc; + + idx_gc = list_entry(c->idx_gc.next, struct ubifs_gced_idx_leb, + list); + c->idx_gc_cnt -= 1; + list_del(&idx_gc->list); + kfree(idx_gc); + } +} + +/** + * ubifs_get_idx_gc_leb - get a LEB from GC'd index LEB list. + * @c: UBIFS file-system description object + * + * Called during start commit so locks are not needed. + */ +int ubifs_get_idx_gc_leb(struct ubifs_info *c) +{ + struct ubifs_gced_idx_leb *idx_gc; + int lnum; + + if (list_empty(&c->idx_gc)) + return -ENOSPC; + idx_gc = list_entry(c->idx_gc.next, struct ubifs_gced_idx_leb, list); + lnum = idx_gc->lnum; + /* c->idx_gc_cnt is updated by the caller when lprops are updated */ + list_del(&idx_gc->list); + kfree(idx_gc); + return lnum; +} diff --git a/fs/ubifs/io.c b/fs/ubifs/io.c new file mode 100644 index 000000000..6c6d68242 --- /dev/null +++ b/fs/ubifs/io.c @@ -0,0 +1,1262 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * Copyright (C) 2006, 2007 University of Szeged, Hungary + * + * Authors: Artem Bityutskiy (Битюцкий Артём) + * Adrian Hunter + * Zoltan Sogor + */ + +/* + * This file implements UBIFS I/O subsystem which provides various I/O-related + * helper functions (reading/writing/checking/validating nodes) and implements + * write-buffering support. Write buffers help to save space which otherwise + * would have been wasted for padding to the nearest minimal I/O unit boundary. + * Instead, data first goes to the write-buffer and is flushed when the + * buffer is full or when it is not used for some time (by timer). This is + * similar to the mechanism is used by JFFS2. + * + * UBIFS distinguishes between minimum write size (@c->min_io_size) and maximum + * write size (@c->max_write_size). The latter is the maximum amount of bytes + * the underlying flash is able to program at a time, and writing in + * @c->max_write_size units should presumably be faster. Obviously, + * @c->min_io_size <= @c->max_write_size. Write-buffers are of + * @c->max_write_size bytes in size for maximum performance. However, when a + * write-buffer is flushed, only the portion of it (aligned to @c->min_io_size + * boundary) which contains data is written, not the whole write-buffer, + * because this is more space-efficient. + * + * This optimization adds few complications to the code. Indeed, on the one + * hand, we want to write in optimal @c->max_write_size bytes chunks, which + * also means aligning writes at the @c->max_write_size bytes offsets. On the + * other hand, we do not want to waste space when synchronizing the write + * buffer, so during synchronization we writes in smaller chunks. And this makes + * the next write offset to be not aligned to @c->max_write_size bytes. So the + * have to make sure that the write-buffer offset (@wbuf->offs) becomes aligned + * to @c->max_write_size bytes again. We do this by temporarily shrinking + * write-buffer size (@wbuf->size). + * + * Write-buffers are defined by 'struct ubifs_wbuf' objects and protected by + * mutexes defined inside these objects. Since sometimes upper-level code + * has to lock the write-buffer (e.g. journal space reservation code), many + * functions related to write-buffers have "nolock" suffix which means that the + * caller has to lock the write-buffer before calling this function. + * + * UBIFS stores nodes at 64 bit-aligned addresses. If the node length is not + * aligned, UBIFS starts the next node from the aligned address, and the padded + * bytes may contain any rubbish. In other words, UBIFS does not put padding + * bytes in those small gaps. Common headers of nodes store real node lengths, + * not aligned lengths. Indexing nodes also store real lengths in branches. + * + * UBIFS uses padding when it pads to the next min. I/O unit. In this case it + * uses padding nodes or padding bytes, if the padding node does not fit. + * + * All UBIFS nodes are protected by CRC checksums and UBIFS checks CRC when + * they are read from the flash media. + */ + +#include +#include +#include "ubifs.h" + +/** + * ubifs_ro_mode - switch UBIFS to read read-only mode. + * @c: UBIFS file-system description object + * @err: error code which is the reason of switching to R/O mode + */ +void ubifs_ro_mode(struct ubifs_info *c, int err) +{ + if (!c->ro_error) { + c->ro_error = 1; + c->no_chk_data_crc = 0; + c->vfs_sb->s_flags |= SB_RDONLY; + ubifs_warn(c, "switched to read-only mode, error %d", err); + dump_stack(); + } +} + +/* + * Below are simple wrappers over UBI I/O functions which include some + * additional checks and UBIFS debugging stuff. See corresponding UBI function + * for more information. + */ + +int ubifs_leb_read(const struct ubifs_info *c, int lnum, void *buf, int offs, + int len, int even_ebadmsg) +{ + int err; + + err = ubi_read(c->ubi, lnum, buf, offs, len); + /* + * In case of %-EBADMSG print the error message only if the + * @even_ebadmsg is true. + */ + if (err && (err != -EBADMSG || even_ebadmsg)) { + ubifs_err(c, "reading %d bytes from LEB %d:%d failed, error %d", + len, lnum, offs, err); + dump_stack(); + } + return err; +} + +int ubifs_leb_write(struct ubifs_info *c, int lnum, const void *buf, int offs, + int len) +{ + int err; + + ubifs_assert(c, !c->ro_media && !c->ro_mount); + if (c->ro_error) + return -EROFS; + if (!dbg_is_tst_rcvry(c)) + err = ubi_leb_write(c->ubi, lnum, buf, offs, len); + else + err = dbg_leb_write(c, lnum, buf, offs, len); + if (err) { + ubifs_err(c, "writing %d bytes to LEB %d:%d failed, error %d", + len, lnum, offs, err); + ubifs_ro_mode(c, err); + dump_stack(); + } + return err; +} + +int ubifs_leb_change(struct ubifs_info *c, int lnum, const void *buf, int len) +{ + int err; + + ubifs_assert(c, !c->ro_media && !c->ro_mount); + if (c->ro_error) + return -EROFS; + if (!dbg_is_tst_rcvry(c)) + err = ubi_leb_change(c->ubi, lnum, buf, len); + else + err = dbg_leb_change(c, lnum, buf, len); + if (err) { + ubifs_err(c, "changing %d bytes in LEB %d failed, error %d", + len, lnum, err); + ubifs_ro_mode(c, err); + dump_stack(); + } + return err; +} + +int ubifs_leb_unmap(struct ubifs_info *c, int lnum) +{ + int err; + + ubifs_assert(c, !c->ro_media && !c->ro_mount); + if (c->ro_error) + return -EROFS; + if (!dbg_is_tst_rcvry(c)) + err = ubi_leb_unmap(c->ubi, lnum); + else + err = dbg_leb_unmap(c, lnum); + if (err) { + ubifs_err(c, "unmap LEB %d failed, error %d", lnum, err); + ubifs_ro_mode(c, err); + dump_stack(); + } + return err; +} + +int ubifs_leb_map(struct ubifs_info *c, int lnum) +{ + int err; + + ubifs_assert(c, !c->ro_media && !c->ro_mount); + if (c->ro_error) + return -EROFS; + if (!dbg_is_tst_rcvry(c)) + err = ubi_leb_map(c->ubi, lnum); + else + err = dbg_leb_map(c, lnum); + if (err) { + ubifs_err(c, "mapping LEB %d failed, error %d", lnum, err); + ubifs_ro_mode(c, err); + dump_stack(); + } + return err; +} + +int ubifs_is_mapped(const struct ubifs_info *c, int lnum) +{ + int err; + + err = ubi_is_mapped(c->ubi, lnum); + if (err < 0) { + ubifs_err(c, "ubi_is_mapped failed for LEB %d, error %d", + lnum, err); + dump_stack(); + } + return err; +} + +static void record_magic_error(struct ubifs_stats_info *stats) +{ + if (stats) + stats->magic_errors++; +} + +static void record_node_error(struct ubifs_stats_info *stats) +{ + if (stats) + stats->node_errors++; +} + +static void record_crc_error(struct ubifs_stats_info *stats) +{ + if (stats) + stats->crc_errors++; +} + +/** + * ubifs_check_node - check node. + * @c: UBIFS file-system description object + * @buf: node to check + * @len: node length + * @lnum: logical eraseblock number + * @offs: offset within the logical eraseblock + * @quiet: print no messages + * @must_chk_crc: indicates whether to always check the CRC + * + * This function checks node magic number and CRC checksum. This function also + * validates node length to prevent UBIFS from becoming crazy when an attacker + * feeds it a file-system image with incorrect nodes. For example, too large + * node length in the common header could cause UBIFS to read memory outside of + * allocated buffer when checking the CRC checksum. + * + * This function may skip data nodes CRC checking if @c->no_chk_data_crc is + * true, which is controlled by corresponding UBIFS mount option. However, if + * @must_chk_crc is true, then @c->no_chk_data_crc is ignored and CRC is + * checked. Similarly, if @c->mounting or @c->remounting_rw is true (we are + * mounting or re-mounting to R/W mode), @c->no_chk_data_crc is ignored and CRC + * is checked. This is because during mounting or re-mounting from R/O mode to + * R/W mode we may read journal nodes (when replying the journal or doing the + * recovery) and the journal nodes may potentially be corrupted, so checking is + * required. + * + * This function returns zero in case of success and %-EUCLEAN in case of bad + * CRC or magic. + */ +int ubifs_check_node(const struct ubifs_info *c, const void *buf, int len, + int lnum, int offs, int quiet, int must_chk_crc) +{ + int err = -EINVAL, type, node_len; + uint32_t crc, node_crc, magic; + const struct ubifs_ch *ch = buf; + + ubifs_assert(c, lnum >= 0 && lnum < c->leb_cnt && offs >= 0); + ubifs_assert(c, !(offs & 7) && offs < c->leb_size); + + magic = le32_to_cpu(ch->magic); + if (magic != UBIFS_NODE_MAGIC) { + if (!quiet) + ubifs_err(c, "bad magic %#08x, expected %#08x", + magic, UBIFS_NODE_MAGIC); + record_magic_error(c->stats); + err = -EUCLEAN; + goto out; + } + + type = ch->node_type; + if (type < 0 || type >= UBIFS_NODE_TYPES_CNT) { + if (!quiet) + ubifs_err(c, "bad node type %d", type); + record_node_error(c->stats); + goto out; + } + + node_len = le32_to_cpu(ch->len); + if (node_len + offs > c->leb_size) + goto out_len; + + if (c->ranges[type].max_len == 0) { + if (node_len != c->ranges[type].len) + goto out_len; + } else if (node_len < c->ranges[type].min_len || + node_len > c->ranges[type].max_len) + goto out_len; + + if (!must_chk_crc && type == UBIFS_DATA_NODE && !c->mounting && + !c->remounting_rw && c->no_chk_data_crc) + return 0; + + crc = crc32(UBIFS_CRC32_INIT, buf + 8, node_len - 8); + node_crc = le32_to_cpu(ch->crc); + if (crc != node_crc) { + if (!quiet) + ubifs_err(c, "bad CRC: calculated %#08x, read %#08x", + crc, node_crc); + record_crc_error(c->stats); + err = -EUCLEAN; + goto out; + } + + return 0; + +out_len: + if (!quiet) + ubifs_err(c, "bad node length %d", node_len); +out: + if (!quiet) { + ubifs_err(c, "bad node at LEB %d:%d", lnum, offs); + ubifs_dump_node(c, buf, len); + dump_stack(); + } + return err; +} + +/** + * ubifs_pad - pad flash space. + * @c: UBIFS file-system description object + * @buf: buffer to put padding to + * @pad: how many bytes to pad + * + * The flash media obliges us to write only in chunks of %c->min_io_size and + * when we have to write less data we add padding node to the write-buffer and + * pad it to the next minimal I/O unit's boundary. Padding nodes help when the + * media is being scanned. If the amount of wasted space is not enough to fit a + * padding node which takes %UBIFS_PAD_NODE_SZ bytes, we write padding bytes + * pattern (%UBIFS_PADDING_BYTE). + * + * Padding nodes are also used to fill gaps when the "commit-in-gaps" method is + * used. + */ +void ubifs_pad(const struct ubifs_info *c, void *buf, int pad) +{ + ubifs_assert(c, pad >= 0); + + if (pad >= UBIFS_PAD_NODE_SZ) { + struct ubifs_ch *ch = buf; + struct ubifs_pad_node *pad_node = buf; + + ch->magic = cpu_to_le32(UBIFS_NODE_MAGIC); + ch->node_type = UBIFS_PAD_NODE; + ch->group_type = UBIFS_NO_NODE_GROUP; + ch->padding[0] = ch->padding[1] = 0; + ch->sqnum = 0; + ch->len = cpu_to_le32(UBIFS_PAD_NODE_SZ); + pad -= UBIFS_PAD_NODE_SZ; + pad_node->pad_len = cpu_to_le32(pad); + ubifs_crc_node(buf, UBIFS_PAD_NODE_SZ); + memset(buf + UBIFS_PAD_NODE_SZ, 0, pad); + } else if (pad > 0) + /* Too little space, padding node won't fit */ + memset(buf, UBIFS_PADDING_BYTE, pad); +} + +/** + * next_sqnum - get next sequence number. + * @c: UBIFS file-system description object + */ +static unsigned long long next_sqnum(struct ubifs_info *c) +{ + unsigned long long sqnum; + + spin_lock(&c->cnt_lock); + sqnum = ++c->max_sqnum; + spin_unlock(&c->cnt_lock); + + if (unlikely(sqnum >= SQNUM_WARN_WATERMARK)) { + if (sqnum >= SQNUM_WATERMARK) { + ubifs_err(c, "sequence number overflow %llu, end of life", + sqnum); + ubifs_ro_mode(c, -EINVAL); + } + ubifs_warn(c, "running out of sequence numbers, end of life soon"); + } + + return sqnum; +} + +void ubifs_init_node(struct ubifs_info *c, void *node, int len, int pad) +{ + struct ubifs_ch *ch = node; + unsigned long long sqnum = next_sqnum(c); + + ubifs_assert(c, len >= UBIFS_CH_SZ); + + ch->magic = cpu_to_le32(UBIFS_NODE_MAGIC); + ch->len = cpu_to_le32(len); + ch->group_type = UBIFS_NO_NODE_GROUP; + ch->sqnum = cpu_to_le64(sqnum); + ch->padding[0] = ch->padding[1] = 0; + + if (pad) { + len = ALIGN(len, 8); + pad = ALIGN(len, c->min_io_size) - len; + ubifs_pad(c, node + len, pad); + } +} + +void ubifs_crc_node(void *node, int len) +{ + struct ubifs_ch *ch = node; + uint32_t crc; + + crc = crc32(UBIFS_CRC32_INIT, node + 8, len - 8); + ch->crc = cpu_to_le32(crc); +} + +/** + * ubifs_prepare_node_hmac - prepare node to be written to flash. + * @c: UBIFS file-system description object + * @node: the node to pad + * @len: node length + * @hmac_offs: offset of the HMAC in the node + * @pad: if the buffer has to be padded + * + * This function prepares node at @node to be written to the media - it + * calculates node CRC, fills the common header, and adds proper padding up to + * the next minimum I/O unit if @pad is not zero. if @hmac_offs is positive then + * a HMAC is inserted into the node at the given offset. + * + * This function returns 0 for success or a negative error code otherwise. + */ +int ubifs_prepare_node_hmac(struct ubifs_info *c, void *node, int len, + int hmac_offs, int pad) +{ + int err; + + ubifs_init_node(c, node, len, pad); + + if (hmac_offs > 0) { + err = ubifs_node_insert_hmac(c, node, len, hmac_offs); + if (err) + return err; + } + + ubifs_crc_node(node, len); + + return 0; +} + +/** + * ubifs_prepare_node - prepare node to be written to flash. + * @c: UBIFS file-system description object + * @node: the node to pad + * @len: node length + * @pad: if the buffer has to be padded + * + * This function prepares node at @node to be written to the media - it + * calculates node CRC, fills the common header, and adds proper padding up to + * the next minimum I/O unit if @pad is not zero. + */ +void ubifs_prepare_node(struct ubifs_info *c, void *node, int len, int pad) +{ + /* + * Deliberately ignore return value since this function can only fail + * when a hmac offset is given. + */ + ubifs_prepare_node_hmac(c, node, len, 0, pad); +} + +/** + * ubifs_prep_grp_node - prepare node of a group to be written to flash. + * @c: UBIFS file-system description object + * @node: the node to pad + * @len: node length + * @last: indicates the last node of the group + * + * This function prepares node at @node to be written to the media - it + * calculates node CRC and fills the common header. + */ +void ubifs_prep_grp_node(struct ubifs_info *c, void *node, int len, int last) +{ + struct ubifs_ch *ch = node; + unsigned long long sqnum = next_sqnum(c); + + ubifs_assert(c, len >= UBIFS_CH_SZ); + + ch->magic = cpu_to_le32(UBIFS_NODE_MAGIC); + ch->len = cpu_to_le32(len); + if (last) + ch->group_type = UBIFS_LAST_OF_NODE_GROUP; + else + ch->group_type = UBIFS_IN_NODE_GROUP; + ch->sqnum = cpu_to_le64(sqnum); + ch->padding[0] = ch->padding[1] = 0; + ubifs_crc_node(node, len); +} + +/** + * wbuf_timer_callback_nolock - write-buffer timer callback function. + * @timer: timer data (write-buffer descriptor) + * + * This function is called when the write-buffer timer expires. + */ +static enum hrtimer_restart wbuf_timer_callback_nolock(struct hrtimer *timer) +{ + struct ubifs_wbuf *wbuf = container_of(timer, struct ubifs_wbuf, timer); + + dbg_io("jhead %s", dbg_jhead(wbuf->jhead)); + wbuf->need_sync = 1; + wbuf->c->need_wbuf_sync = 1; + ubifs_wake_up_bgt(wbuf->c); + return HRTIMER_NORESTART; +} + +/** + * new_wbuf_timer_nolock - start new write-buffer timer. + * @c: UBIFS file-system description object + * @wbuf: write-buffer descriptor + */ +static void new_wbuf_timer_nolock(struct ubifs_info *c, struct ubifs_wbuf *wbuf) +{ + ktime_t softlimit = ms_to_ktime(dirty_writeback_interval * 10); + unsigned long long delta = dirty_writeback_interval; + + /* centi to milli, milli to nano, then 10% */ + delta *= 10ULL * NSEC_PER_MSEC / 10ULL; + + ubifs_assert(c, !hrtimer_active(&wbuf->timer)); + ubifs_assert(c, delta <= ULONG_MAX); + + if (wbuf->no_timer) + return; + dbg_io("set timer for jhead %s, %llu-%llu millisecs", + dbg_jhead(wbuf->jhead), + div_u64(ktime_to_ns(softlimit), USEC_PER_SEC), + div_u64(ktime_to_ns(softlimit) + delta, USEC_PER_SEC)); + hrtimer_start_range_ns(&wbuf->timer, softlimit, delta, + HRTIMER_MODE_REL); +} + +/** + * cancel_wbuf_timer_nolock - cancel write-buffer timer. + * @wbuf: write-buffer descriptor + */ +static void cancel_wbuf_timer_nolock(struct ubifs_wbuf *wbuf) +{ + if (wbuf->no_timer) + return; + wbuf->need_sync = 0; + hrtimer_cancel(&wbuf->timer); +} + +/** + * ubifs_wbuf_sync_nolock - synchronize write-buffer. + * @wbuf: write-buffer to synchronize + * + * This function synchronizes write-buffer @buf and returns zero in case of + * success or a negative error code in case of failure. + * + * Note, although write-buffers are of @c->max_write_size, this function does + * not necessarily writes all @c->max_write_size bytes to the flash. Instead, + * if the write-buffer is only partially filled with data, only the used part + * of the write-buffer (aligned on @c->min_io_size boundary) is synchronized. + * This way we waste less space. + */ +int ubifs_wbuf_sync_nolock(struct ubifs_wbuf *wbuf) +{ + struct ubifs_info *c = wbuf->c; + int err, dirt, sync_len; + + cancel_wbuf_timer_nolock(wbuf); + if (!wbuf->used || wbuf->lnum == -1) + /* Write-buffer is empty or not seeked */ + return 0; + + dbg_io("LEB %d:%d, %d bytes, jhead %s", + wbuf->lnum, wbuf->offs, wbuf->used, dbg_jhead(wbuf->jhead)); + ubifs_assert(c, !(wbuf->avail & 7)); + ubifs_assert(c, wbuf->offs + wbuf->size <= c->leb_size); + ubifs_assert(c, wbuf->size >= c->min_io_size); + ubifs_assert(c, wbuf->size <= c->max_write_size); + ubifs_assert(c, wbuf->size % c->min_io_size == 0); + ubifs_assert(c, !c->ro_media && !c->ro_mount); + if (c->leb_size - wbuf->offs >= c->max_write_size) + ubifs_assert(c, !((wbuf->offs + wbuf->size) % c->max_write_size)); + + if (c->ro_error) + return -EROFS; + + /* + * Do not write whole write buffer but write only the minimum necessary + * amount of min. I/O units. + */ + sync_len = ALIGN(wbuf->used, c->min_io_size); + dirt = sync_len - wbuf->used; + if (dirt) + ubifs_pad(c, wbuf->buf + wbuf->used, dirt); + err = ubifs_leb_write(c, wbuf->lnum, wbuf->buf, wbuf->offs, sync_len); + if (err) + return err; + + spin_lock(&wbuf->lock); + wbuf->offs += sync_len; + /* + * Now @wbuf->offs is not necessarily aligned to @c->max_write_size. + * But our goal is to optimize writes and make sure we write in + * @c->max_write_size chunks and to @c->max_write_size-aligned offset. + * Thus, if @wbuf->offs is not aligned to @c->max_write_size now, make + * sure that @wbuf->offs + @wbuf->size is aligned to + * @c->max_write_size. This way we make sure that after next + * write-buffer flush we are again at the optimal offset (aligned to + * @c->max_write_size). + */ + if (c->leb_size - wbuf->offs < c->max_write_size) + wbuf->size = c->leb_size - wbuf->offs; + else if (wbuf->offs & (c->max_write_size - 1)) + wbuf->size = ALIGN(wbuf->offs, c->max_write_size) - wbuf->offs; + else + wbuf->size = c->max_write_size; + wbuf->avail = wbuf->size; + wbuf->used = 0; + wbuf->next_ino = 0; + spin_unlock(&wbuf->lock); + + if (wbuf->sync_callback) + err = wbuf->sync_callback(c, wbuf->lnum, + c->leb_size - wbuf->offs, dirt); + return err; +} + +/** + * ubifs_wbuf_seek_nolock - seek write-buffer. + * @wbuf: write-buffer + * @lnum: logical eraseblock number to seek to + * @offs: logical eraseblock offset to seek to + * + * This function targets the write-buffer to logical eraseblock @lnum:@offs. + * The write-buffer has to be empty. Returns zero in case of success and a + * negative error code in case of failure. + */ +int ubifs_wbuf_seek_nolock(struct ubifs_wbuf *wbuf, int lnum, int offs) +{ + const struct ubifs_info *c = wbuf->c; + + dbg_io("LEB %d:%d, jhead %s", lnum, offs, dbg_jhead(wbuf->jhead)); + ubifs_assert(c, lnum >= 0 && lnum < c->leb_cnt); + ubifs_assert(c, offs >= 0 && offs <= c->leb_size); + ubifs_assert(c, offs % c->min_io_size == 0 && !(offs & 7)); + ubifs_assert(c, lnum != wbuf->lnum); + ubifs_assert(c, wbuf->used == 0); + + spin_lock(&wbuf->lock); + wbuf->lnum = lnum; + wbuf->offs = offs; + if (c->leb_size - wbuf->offs < c->max_write_size) + wbuf->size = c->leb_size - wbuf->offs; + else if (wbuf->offs & (c->max_write_size - 1)) + wbuf->size = ALIGN(wbuf->offs, c->max_write_size) - wbuf->offs; + else + wbuf->size = c->max_write_size; + wbuf->avail = wbuf->size; + wbuf->used = 0; + spin_unlock(&wbuf->lock); + + return 0; +} + +/** + * ubifs_bg_wbufs_sync - synchronize write-buffers. + * @c: UBIFS file-system description object + * + * This function is called by background thread to synchronize write-buffers. + * Returns zero in case of success and a negative error code in case of + * failure. + */ +int ubifs_bg_wbufs_sync(struct ubifs_info *c) +{ + int err, i; + + ubifs_assert(c, !c->ro_media && !c->ro_mount); + if (!c->need_wbuf_sync) + return 0; + c->need_wbuf_sync = 0; + + if (c->ro_error) { + err = -EROFS; + goto out_timers; + } + + dbg_io("synchronize"); + for (i = 0; i < c->jhead_cnt; i++) { + struct ubifs_wbuf *wbuf = &c->jheads[i].wbuf; + + cond_resched(); + + /* + * If the mutex is locked then wbuf is being changed, so + * synchronization is not necessary. + */ + if (mutex_is_locked(&wbuf->io_mutex)) + continue; + + mutex_lock_nested(&wbuf->io_mutex, wbuf->jhead); + if (!wbuf->need_sync) { + mutex_unlock(&wbuf->io_mutex); + continue; + } + + err = ubifs_wbuf_sync_nolock(wbuf); + mutex_unlock(&wbuf->io_mutex); + if (err) { + ubifs_err(c, "cannot sync write-buffer, error %d", err); + ubifs_ro_mode(c, err); + goto out_timers; + } + } + + return 0; + +out_timers: + /* Cancel all timers to prevent repeated errors */ + for (i = 0; i < c->jhead_cnt; i++) { + struct ubifs_wbuf *wbuf = &c->jheads[i].wbuf; + + mutex_lock_nested(&wbuf->io_mutex, wbuf->jhead); + cancel_wbuf_timer_nolock(wbuf); + mutex_unlock(&wbuf->io_mutex); + } + return err; +} + +/** + * ubifs_wbuf_write_nolock - write data to flash via write-buffer. + * @wbuf: write-buffer + * @buf: node to write + * @len: node length + * + * This function writes data to flash via write-buffer @wbuf. This means that + * the last piece of the node won't reach the flash media immediately if it + * does not take whole max. write unit (@c->max_write_size). Instead, the node + * will sit in RAM until the write-buffer is synchronized (e.g., by timer, or + * because more data are appended to the write-buffer). + * + * This function returns zero in case of success and a negative error code in + * case of failure. If the node cannot be written because there is no more + * space in this logical eraseblock, %-ENOSPC is returned. + */ +int ubifs_wbuf_write_nolock(struct ubifs_wbuf *wbuf, void *buf, int len) +{ + struct ubifs_info *c = wbuf->c; + int err, n, written = 0, aligned_len = ALIGN(len, 8); + + dbg_io("%d bytes (%s) to jhead %s wbuf at LEB %d:%d", len, + dbg_ntype(((struct ubifs_ch *)buf)->node_type), + dbg_jhead(wbuf->jhead), wbuf->lnum, wbuf->offs + wbuf->used); + ubifs_assert(c, len > 0 && wbuf->lnum >= 0 && wbuf->lnum < c->leb_cnt); + ubifs_assert(c, wbuf->offs >= 0 && wbuf->offs % c->min_io_size == 0); + ubifs_assert(c, !(wbuf->offs & 7) && wbuf->offs <= c->leb_size); + ubifs_assert(c, wbuf->avail > 0 && wbuf->avail <= wbuf->size); + ubifs_assert(c, wbuf->size >= c->min_io_size); + ubifs_assert(c, wbuf->size <= c->max_write_size); + ubifs_assert(c, wbuf->size % c->min_io_size == 0); + ubifs_assert(c, mutex_is_locked(&wbuf->io_mutex)); + ubifs_assert(c, !c->ro_media && !c->ro_mount); + ubifs_assert(c, !c->space_fixup); + if (c->leb_size - wbuf->offs >= c->max_write_size) + ubifs_assert(c, !((wbuf->offs + wbuf->size) % c->max_write_size)); + + if (c->leb_size - wbuf->offs - wbuf->used < aligned_len) { + err = -ENOSPC; + goto out; + } + + cancel_wbuf_timer_nolock(wbuf); + + if (c->ro_error) + return -EROFS; + + if (aligned_len <= wbuf->avail) { + /* + * The node is not very large and fits entirely within + * write-buffer. + */ + memcpy(wbuf->buf + wbuf->used, buf, len); + if (aligned_len > len) { + ubifs_assert(c, aligned_len - len < 8); + ubifs_pad(c, wbuf->buf + wbuf->used + len, aligned_len - len); + } + + if (aligned_len == wbuf->avail) { + dbg_io("flush jhead %s wbuf to LEB %d:%d", + dbg_jhead(wbuf->jhead), wbuf->lnum, wbuf->offs); + err = ubifs_leb_write(c, wbuf->lnum, wbuf->buf, + wbuf->offs, wbuf->size); + if (err) + goto out; + + spin_lock(&wbuf->lock); + wbuf->offs += wbuf->size; + if (c->leb_size - wbuf->offs >= c->max_write_size) + wbuf->size = c->max_write_size; + else + wbuf->size = c->leb_size - wbuf->offs; + wbuf->avail = wbuf->size; + wbuf->used = 0; + wbuf->next_ino = 0; + spin_unlock(&wbuf->lock); + } else { + spin_lock(&wbuf->lock); + wbuf->avail -= aligned_len; + wbuf->used += aligned_len; + spin_unlock(&wbuf->lock); + } + + goto exit; + } + + if (wbuf->used) { + /* + * The node is large enough and does not fit entirely within + * current available space. We have to fill and flush + * write-buffer and switch to the next max. write unit. + */ + dbg_io("flush jhead %s wbuf to LEB %d:%d", + dbg_jhead(wbuf->jhead), wbuf->lnum, wbuf->offs); + memcpy(wbuf->buf + wbuf->used, buf, wbuf->avail); + err = ubifs_leb_write(c, wbuf->lnum, wbuf->buf, wbuf->offs, + wbuf->size); + if (err) + goto out; + + wbuf->offs += wbuf->size; + len -= wbuf->avail; + aligned_len -= wbuf->avail; + written += wbuf->avail; + } else if (wbuf->offs & (c->max_write_size - 1)) { + /* + * The write-buffer offset is not aligned to + * @c->max_write_size and @wbuf->size is less than + * @c->max_write_size. Write @wbuf->size bytes to make sure the + * following writes are done in optimal @c->max_write_size + * chunks. + */ + dbg_io("write %d bytes to LEB %d:%d", + wbuf->size, wbuf->lnum, wbuf->offs); + err = ubifs_leb_write(c, wbuf->lnum, buf, wbuf->offs, + wbuf->size); + if (err) + goto out; + + wbuf->offs += wbuf->size; + len -= wbuf->size; + aligned_len -= wbuf->size; + written += wbuf->size; + } + + /* + * The remaining data may take more whole max. write units, so write the + * remains multiple to max. write unit size directly to the flash media. + * We align node length to 8-byte boundary because we anyway flash wbuf + * if the remaining space is less than 8 bytes. + */ + n = aligned_len >> c->max_write_shift; + if (n) { + int m = n - 1; + + dbg_io("write %d bytes to LEB %d:%d", n, wbuf->lnum, + wbuf->offs); + + if (m) { + /* '(n-1)<max_write_shift < len' is always true. */ + m <<= c->max_write_shift; + err = ubifs_leb_write(c, wbuf->lnum, buf + written, + wbuf->offs, m); + if (err) + goto out; + wbuf->offs += m; + aligned_len -= m; + len -= m; + written += m; + } + + /* + * The non-written len of buf may be less than 'n' because + * parameter 'len' is not 8 bytes aligned, so here we read + * min(len, n) bytes from buf. + */ + n = 1 << c->max_write_shift; + memcpy(wbuf->buf, buf + written, min(len, n)); + if (n > len) { + ubifs_assert(c, n - len < 8); + ubifs_pad(c, wbuf->buf + len, n - len); + } + + err = ubifs_leb_write(c, wbuf->lnum, wbuf->buf, wbuf->offs, n); + if (err) + goto out; + wbuf->offs += n; + aligned_len -= n; + len -= min(len, n); + written += n; + } + + spin_lock(&wbuf->lock); + if (aligned_len) { + /* + * And now we have what's left and what does not take whole + * max. write unit, so write it to the write-buffer and we are + * done. + */ + memcpy(wbuf->buf, buf + written, len); + if (aligned_len > len) { + ubifs_assert(c, aligned_len - len < 8); + ubifs_pad(c, wbuf->buf + len, aligned_len - len); + } + } + + if (c->leb_size - wbuf->offs >= c->max_write_size) + wbuf->size = c->max_write_size; + else + wbuf->size = c->leb_size - wbuf->offs; + wbuf->avail = wbuf->size - aligned_len; + wbuf->used = aligned_len; + wbuf->next_ino = 0; + spin_unlock(&wbuf->lock); + +exit: + if (wbuf->sync_callback) { + int free = c->leb_size - wbuf->offs - wbuf->used; + + err = wbuf->sync_callback(c, wbuf->lnum, free, 0); + if (err) + goto out; + } + + if (wbuf->used) + new_wbuf_timer_nolock(c, wbuf); + + return 0; + +out: + ubifs_err(c, "cannot write %d bytes to LEB %d:%d, error %d", + len, wbuf->lnum, wbuf->offs, err); + ubifs_dump_node(c, buf, written + len); + dump_stack(); + ubifs_dump_leb(c, wbuf->lnum); + return err; +} + +/** + * ubifs_write_node_hmac - write node to the media. + * @c: UBIFS file-system description object + * @buf: the node to write + * @len: node length + * @lnum: logical eraseblock number + * @offs: offset within the logical eraseblock + * @hmac_offs: offset of the HMAC within the node + * + * This function automatically fills node magic number, assigns sequence + * number, and calculates node CRC checksum. The length of the @buf buffer has + * to be aligned to the minimal I/O unit size. This function automatically + * appends padding node and padding bytes if needed. Returns zero in case of + * success and a negative error code in case of failure. + */ +int ubifs_write_node_hmac(struct ubifs_info *c, void *buf, int len, int lnum, + int offs, int hmac_offs) +{ + int err, buf_len = ALIGN(len, c->min_io_size); + + dbg_io("LEB %d:%d, %s, length %d (aligned %d)", + lnum, offs, dbg_ntype(((struct ubifs_ch *)buf)->node_type), len, + buf_len); + ubifs_assert(c, lnum >= 0 && lnum < c->leb_cnt && offs >= 0); + ubifs_assert(c, offs % c->min_io_size == 0 && offs < c->leb_size); + ubifs_assert(c, !c->ro_media && !c->ro_mount); + ubifs_assert(c, !c->space_fixup); + + if (c->ro_error) + return -EROFS; + + err = ubifs_prepare_node_hmac(c, buf, len, hmac_offs, 1); + if (err) + return err; + + err = ubifs_leb_write(c, lnum, buf, offs, buf_len); + if (err) + ubifs_dump_node(c, buf, len); + + return err; +} + +/** + * ubifs_write_node - write node to the media. + * @c: UBIFS file-system description object + * @buf: the node to write + * @len: node length + * @lnum: logical eraseblock number + * @offs: offset within the logical eraseblock + * + * This function automatically fills node magic number, assigns sequence + * number, and calculates node CRC checksum. The length of the @buf buffer has + * to be aligned to the minimal I/O unit size. This function automatically + * appends padding node and padding bytes if needed. Returns zero in case of + * success and a negative error code in case of failure. + */ +int ubifs_write_node(struct ubifs_info *c, void *buf, int len, int lnum, + int offs) +{ + return ubifs_write_node_hmac(c, buf, len, lnum, offs, -1); +} + +/** + * ubifs_read_node_wbuf - read node from the media or write-buffer. + * @wbuf: wbuf to check for un-written data + * @buf: buffer to read to + * @type: node type + * @len: node length + * @lnum: logical eraseblock number + * @offs: offset within the logical eraseblock + * + * This function reads a node of known type and length, checks it and stores + * in @buf. If the node partially or fully sits in the write-buffer, this + * function takes data from the buffer, otherwise it reads the flash media. + * Returns zero in case of success, %-EUCLEAN if CRC mismatched and a negative + * error code in case of failure. + */ +int ubifs_read_node_wbuf(struct ubifs_wbuf *wbuf, void *buf, int type, int len, + int lnum, int offs) +{ + const struct ubifs_info *c = wbuf->c; + int err, rlen, overlap; + struct ubifs_ch *ch = buf; + + dbg_io("LEB %d:%d, %s, length %d, jhead %s", lnum, offs, + dbg_ntype(type), len, dbg_jhead(wbuf->jhead)); + ubifs_assert(c, wbuf && lnum >= 0 && lnum < c->leb_cnt && offs >= 0); + ubifs_assert(c, !(offs & 7) && offs < c->leb_size); + ubifs_assert(c, type >= 0 && type < UBIFS_NODE_TYPES_CNT); + + spin_lock(&wbuf->lock); + overlap = (lnum == wbuf->lnum && offs + len > wbuf->offs); + if (!overlap) { + /* We may safely unlock the write-buffer and read the data */ + spin_unlock(&wbuf->lock); + return ubifs_read_node(c, buf, type, len, lnum, offs); + } + + /* Don't read under wbuf */ + rlen = wbuf->offs - offs; + if (rlen < 0) + rlen = 0; + + /* Copy the rest from the write-buffer */ + memcpy(buf + rlen, wbuf->buf + offs + rlen - wbuf->offs, len - rlen); + spin_unlock(&wbuf->lock); + + if (rlen > 0) { + /* Read everything that goes before write-buffer */ + err = ubifs_leb_read(c, lnum, buf, offs, rlen, 0); + if (err && err != -EBADMSG) + return err; + } + + if (type != ch->node_type) { + ubifs_err(c, "bad node type (%d but expected %d)", + ch->node_type, type); + goto out; + } + + err = ubifs_check_node(c, buf, len, lnum, offs, 0, 0); + if (err) { + ubifs_err(c, "expected node type %d", type); + return err; + } + + rlen = le32_to_cpu(ch->len); + if (rlen != len) { + ubifs_err(c, "bad node length %d, expected %d", rlen, len); + goto out; + } + + return 0; + +out: + ubifs_err(c, "bad node at LEB %d:%d", lnum, offs); + ubifs_dump_node(c, buf, len); + dump_stack(); + return -EINVAL; +} + +/** + * ubifs_read_node - read node. + * @c: UBIFS file-system description object + * @buf: buffer to read to + * @type: node type + * @len: node length (not aligned) + * @lnum: logical eraseblock number + * @offs: offset within the logical eraseblock + * + * This function reads a node of known type and length, checks it and + * stores in @buf. Returns zero in case of success, %-EUCLEAN if CRC mismatched + * and a negative error code in case of failure. + */ +int ubifs_read_node(const struct ubifs_info *c, void *buf, int type, int len, + int lnum, int offs) +{ + int err, l; + struct ubifs_ch *ch = buf; + + dbg_io("LEB %d:%d, %s, length %d", lnum, offs, dbg_ntype(type), len); + ubifs_assert(c, lnum >= 0 && lnum < c->leb_cnt && offs >= 0); + ubifs_assert(c, len >= UBIFS_CH_SZ && offs + len <= c->leb_size); + ubifs_assert(c, !(offs & 7) && offs < c->leb_size); + ubifs_assert(c, type >= 0 && type < UBIFS_NODE_TYPES_CNT); + + err = ubifs_leb_read(c, lnum, buf, offs, len, 0); + if (err && err != -EBADMSG) + return err; + + if (type != ch->node_type) { + ubifs_errc(c, "bad node type (%d but expected %d)", + ch->node_type, type); + goto out; + } + + err = ubifs_check_node(c, buf, len, lnum, offs, 0, 0); + if (err) { + ubifs_errc(c, "expected node type %d", type); + return err; + } + + l = le32_to_cpu(ch->len); + if (l != len) { + ubifs_errc(c, "bad node length %d, expected %d", l, len); + goto out; + } + + return 0; + +out: + ubifs_errc(c, "bad node at LEB %d:%d, LEB mapping status %d", lnum, + offs, ubi_is_mapped(c->ubi, lnum)); + if (!c->probing) { + ubifs_dump_node(c, buf, len); + dump_stack(); + } + return -EINVAL; +} + +/** + * ubifs_wbuf_init - initialize write-buffer. + * @c: UBIFS file-system description object + * @wbuf: write-buffer to initialize + * + * This function initializes write-buffer. Returns zero in case of success + * %-ENOMEM in case of failure. + */ +int ubifs_wbuf_init(struct ubifs_info *c, struct ubifs_wbuf *wbuf) +{ + size_t size; + + wbuf->buf = kmalloc(c->max_write_size, GFP_KERNEL); + if (!wbuf->buf) + return -ENOMEM; + + size = (c->max_write_size / UBIFS_CH_SZ + 1) * sizeof(ino_t); + wbuf->inodes = kmalloc(size, GFP_KERNEL); + if (!wbuf->inodes) { + kfree(wbuf->buf); + wbuf->buf = NULL; + return -ENOMEM; + } + + wbuf->used = 0; + wbuf->lnum = wbuf->offs = -1; + /* + * If the LEB starts at the max. write size aligned address, then + * write-buffer size has to be set to @c->max_write_size. Otherwise, + * set it to something smaller so that it ends at the closest max. + * write size boundary. + */ + size = c->max_write_size - (c->leb_start % c->max_write_size); + wbuf->avail = wbuf->size = size; + wbuf->sync_callback = NULL; + mutex_init(&wbuf->io_mutex); + spin_lock_init(&wbuf->lock); + wbuf->c = c; + wbuf->next_ino = 0; + + hrtimer_setup(&wbuf->timer, wbuf_timer_callback_nolock, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + return 0; +} + +/** + * ubifs_wbuf_add_ino_nolock - add an inode number into the wbuf inode array. + * @wbuf: the write-buffer where to add + * @inum: the inode number + * + * This function adds an inode number to the inode array of the write-buffer. + */ +void ubifs_wbuf_add_ino_nolock(struct ubifs_wbuf *wbuf, ino_t inum) +{ + if (!wbuf->buf) + /* NOR flash or something similar */ + return; + + spin_lock(&wbuf->lock); + if (wbuf->used) + wbuf->inodes[wbuf->next_ino++] = inum; + spin_unlock(&wbuf->lock); +} + +/** + * wbuf_has_ino - returns if the wbuf contains data from the inode. + * @wbuf: the write-buffer + * @inum: the inode number + * + * This function returns with %1 if the write-buffer contains some data from the + * given inode otherwise it returns with %0. + */ +static int wbuf_has_ino(struct ubifs_wbuf *wbuf, ino_t inum) +{ + int i, ret = 0; + + spin_lock(&wbuf->lock); + for (i = 0; i < wbuf->next_ino; i++) + if (inum == wbuf->inodes[i]) { + ret = 1; + break; + } + spin_unlock(&wbuf->lock); + + return ret; +} + +/** + * ubifs_sync_wbufs_by_inode - synchronize write-buffers for an inode. + * @c: UBIFS file-system description object + * @inode: inode to synchronize + * + * This function synchronizes write-buffers which contain nodes belonging to + * @inode. Returns zero in case of success and a negative error code in case of + * failure. + */ +int ubifs_sync_wbufs_by_inode(struct ubifs_info *c, struct inode *inode) +{ + int i, err = 0; + + for (i = 0; i < c->jhead_cnt; i++) { + struct ubifs_wbuf *wbuf = &c->jheads[i].wbuf; + + if (i == GCHD) + /* + * GC head is special, do not look at it. Even if the + * head contains something related to this inode, it is + * a _copy_ of corresponding on-flash node which sits + * somewhere else. + */ + continue; + + if (!wbuf_has_ino(wbuf, inode->i_ino)) + continue; + + mutex_lock_nested(&wbuf->io_mutex, wbuf->jhead); + if (wbuf_has_ino(wbuf, inode->i_ino)) + err = ubifs_wbuf_sync_nolock(wbuf); + mutex_unlock(&wbuf->io_mutex); + + if (err) { + ubifs_ro_mode(c, err); + return err; + } + } + return 0; +} diff --git a/fs/ubifs/ioctl.c b/fs/ubifs/ioctl.c new file mode 100644 index 000000000..79536b2e3 --- /dev/null +++ b/fs/ubifs/ioctl.c @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * Copyright (C) 2006, 2007 University of Szeged, Hungary + * + * Authors: Zoltan Sogor + * Artem Bityutskiy (Битюцкий Артём) + * Adrian Hunter + */ + +/* This file implements EXT2-compatible extended attribute ioctl() calls */ + +#include +#include +#include +#include "ubifs.h" + +/* Need to be kept consistent with checked flags in ioctl2ubifs() */ +#define UBIFS_SETTABLE_IOCTL_FLAGS \ + (FS_COMPR_FL | FS_SYNC_FL | FS_APPEND_FL | \ + FS_IMMUTABLE_FL | FS_DIRSYNC_FL) + +/* Need to be kept consistent with checked flags in ubifs2ioctl() */ +#define UBIFS_GETTABLE_IOCTL_FLAGS \ + (UBIFS_SETTABLE_IOCTL_FLAGS | FS_ENCRYPT_FL) + +/** + * ubifs_set_inode_flags - set VFS inode flags. + * @inode: VFS inode to set flags for + * + * This function propagates flags from UBIFS inode object to VFS inode object. + */ +void ubifs_set_inode_flags(struct inode *inode) +{ + unsigned int flags = ubifs_inode(inode)->flags; + + inode->i_flags &= ~(S_SYNC | S_APPEND | S_IMMUTABLE | S_DIRSYNC | + S_ENCRYPTED); + if (flags & UBIFS_SYNC_FL) + inode->i_flags |= S_SYNC; + if (flags & UBIFS_APPEND_FL) + inode->i_flags |= S_APPEND; + if (flags & UBIFS_IMMUTABLE_FL) + inode->i_flags |= S_IMMUTABLE; + if (flags & UBIFS_DIRSYNC_FL) + inode->i_flags |= S_DIRSYNC; + if (flags & UBIFS_CRYPT_FL) + inode->i_flags |= S_ENCRYPTED; +} + +/* + * ioctl2ubifs - convert ioctl inode flags to UBIFS inode flags. + * @ioctl_flags: flags to convert + * + * This function converts ioctl flags (@FS_COMPR_FL, etc) to UBIFS inode flags + * (@UBIFS_COMPR_FL, etc). + */ +static int ioctl2ubifs(int ioctl_flags) +{ + int ubifs_flags = 0; + + if (ioctl_flags & FS_COMPR_FL) + ubifs_flags |= UBIFS_COMPR_FL; + if (ioctl_flags & FS_SYNC_FL) + ubifs_flags |= UBIFS_SYNC_FL; + if (ioctl_flags & FS_APPEND_FL) + ubifs_flags |= UBIFS_APPEND_FL; + if (ioctl_flags & FS_IMMUTABLE_FL) + ubifs_flags |= UBIFS_IMMUTABLE_FL; + if (ioctl_flags & FS_DIRSYNC_FL) + ubifs_flags |= UBIFS_DIRSYNC_FL; + + return ubifs_flags; +} + +/* + * ubifs2ioctl - convert UBIFS inode flags to ioctl inode flags. + * @ubifs_flags: flags to convert + * + * This function converts UBIFS inode flags (@UBIFS_COMPR_FL, etc) to ioctl + * flags (@FS_COMPR_FL, etc). + */ +static int ubifs2ioctl(int ubifs_flags) +{ + int ioctl_flags = 0; + + if (ubifs_flags & UBIFS_COMPR_FL) + ioctl_flags |= FS_COMPR_FL; + if (ubifs_flags & UBIFS_SYNC_FL) + ioctl_flags |= FS_SYNC_FL; + if (ubifs_flags & UBIFS_APPEND_FL) + ioctl_flags |= FS_APPEND_FL; + if (ubifs_flags & UBIFS_IMMUTABLE_FL) + ioctl_flags |= FS_IMMUTABLE_FL; + if (ubifs_flags & UBIFS_DIRSYNC_FL) + ioctl_flags |= FS_DIRSYNC_FL; + if (ubifs_flags & UBIFS_CRYPT_FL) + ioctl_flags |= FS_ENCRYPT_FL; + + return ioctl_flags; +} + +static int setflags(struct inode *inode, int flags) +{ + int err, release; + struct ubifs_inode *ui = ubifs_inode(inode); + struct ubifs_info *c = inode->i_sb->s_fs_info; + struct ubifs_budget_req req = { .dirtied_ino = 1, + .dirtied_ino_d = ALIGN(ui->data_len, 8) }; + + err = ubifs_budget_space(c, &req); + if (err) + return err; + + mutex_lock(&ui->ui_mutex); + ui->flags &= ~ioctl2ubifs(UBIFS_SETTABLE_IOCTL_FLAGS); + ui->flags |= ioctl2ubifs(flags); + ubifs_set_inode_flags(inode); + inode_set_ctime_current(inode); + release = ui->dirty; + mark_inode_dirty_sync(inode); + mutex_unlock(&ui->ui_mutex); + + if (release) + ubifs_release_budget(c, &req); + if (IS_SYNC(inode)) + err = write_inode_now(inode, 1); + return err; +} + +int ubifs_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + int flags = ubifs2ioctl(ubifs_inode(inode)->flags); + + if (d_is_special(dentry)) + return -ENOTTY; + + dbg_gen("get flags: %#x, i_flags %#x", flags, inode->i_flags); + fileattr_fill_flags(fa, flags); + + return 0; +} + +int ubifs_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa) +{ + struct inode *inode = d_inode(dentry); + int flags = fa->flags; + + if (d_is_special(dentry)) + return -ENOTTY; + + if (fileattr_has_fsx(fa)) + return -EOPNOTSUPP; + + if (flags & ~UBIFS_GETTABLE_IOCTL_FLAGS) + return -EOPNOTSUPP; + + flags &= UBIFS_SETTABLE_IOCTL_FLAGS; + + if (!S_ISDIR(inode->i_mode)) + flags &= ~FS_DIRSYNC_FL; + + dbg_gen("set flags: %#x, i_flags %#x", flags, inode->i_flags); + return setflags(inode, flags); +} + +long ubifs_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + int err; + struct inode *inode = file_inode(file); + + switch (cmd) { + case FS_IOC_SET_ENCRYPTION_POLICY: { + struct ubifs_info *c = inode->i_sb->s_fs_info; + + err = ubifs_enable_encryption(c); + if (err) + return err; + + return fscrypt_ioctl_set_policy(file, (const void __user *)arg); + } + case FS_IOC_GET_ENCRYPTION_POLICY: + return fscrypt_ioctl_get_policy(file, (void __user *)arg); + + case FS_IOC_GET_ENCRYPTION_POLICY_EX: + return fscrypt_ioctl_get_policy_ex(file, (void __user *)arg); + + case FS_IOC_ADD_ENCRYPTION_KEY: + return fscrypt_ioctl_add_key(file, (void __user *)arg); + + case FS_IOC_REMOVE_ENCRYPTION_KEY: + return fscrypt_ioctl_remove_key(file, (void __user *)arg); + + case FS_IOC_REMOVE_ENCRYPTION_KEY_ALL_USERS: + return fscrypt_ioctl_remove_key_all_users(file, + (void __user *)arg); + case FS_IOC_GET_ENCRYPTION_KEY_STATUS: + return fscrypt_ioctl_get_key_status(file, (void __user *)arg); + + case FS_IOC_GET_ENCRYPTION_NONCE: + return fscrypt_ioctl_get_nonce(file, (void __user *)arg); + + default: + return -ENOTTY; + } +} + +#ifdef CONFIG_COMPAT +long ubifs_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + switch (cmd) { + case FS_IOC_SET_ENCRYPTION_POLICY: + case FS_IOC_GET_ENCRYPTION_POLICY: + case FS_IOC_GET_ENCRYPTION_POLICY_EX: + case FS_IOC_ADD_ENCRYPTION_KEY: + case FS_IOC_REMOVE_ENCRYPTION_KEY: + case FS_IOC_REMOVE_ENCRYPTION_KEY_ALL_USERS: + case FS_IOC_GET_ENCRYPTION_KEY_STATUS: + case FS_IOC_GET_ENCRYPTION_NONCE: + break; + default: + return -ENOIOCTLCMD; + } + return ubifs_ioctl(file, cmd, (unsigned long)compat_ptr(arg)); +} +#endif diff --git a/fs/ubifs/journal.c b/fs/ubifs/journal.c new file mode 100644 index 000000000..40a95a2fa --- /dev/null +++ b/fs/ubifs/journal.c @@ -0,0 +1,2059 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * + * Authors: Artem Bityutskiy (Битюцкий Артём) + * Adrian Hunter + */ + +/* + * This file implements UBIFS journal. + * + * The journal consists of 2 parts - the log and bud LEBs. The log has fixed + * length and position, while a bud logical eraseblock is any LEB in the main + * area. Buds contain file system data - data nodes, inode nodes, etc. The log + * contains only references to buds and some other stuff like commit + * start node. The idea is that when we commit the journal, we do + * not copy the data, the buds just become indexed. Since after the commit the + * nodes in bud eraseblocks become leaf nodes of the file system index tree, we + * use term "bud". Analogy is obvious, bud eraseblocks contain nodes which will + * become leafs in the future. + * + * The journal is multi-headed because we want to write data to the journal as + * optimally as possible. It is nice to have nodes belonging to the same inode + * in one LEB, so we may write data owned by different inodes to different + * journal heads, although at present only one data head is used. + * + * For recovery reasons, the base head contains all inode nodes, all directory + * entry nodes and all truncate nodes. This means that the other heads contain + * only data nodes. + * + * Bud LEBs may be half-indexed. For example, if the bud was not full at the + * time of commit, the bud is retained to continue to be used in the journal, + * even though the "front" of the LEB is now indexed. In that case, the log + * reference contains the offset where the bud starts for the purposes of the + * journal. + * + * The journal size has to be limited, because the larger is the journal, the + * longer it takes to mount UBIFS (scanning the journal) and the more memory it + * takes (indexing in the TNC). + * + * All the journal write operations like 'ubifs_jnl_update()' here, which write + * multiple UBIFS nodes to the journal at one go, are atomic with respect to + * unclean reboots. Should the unclean reboot happen, the recovery code drops + * all the nodes. + */ + +#include "ubifs.h" + +/** + * zero_ino_node_unused - zero out unused fields of an on-flash inode node. + * @ino: the inode to zero out + */ +static inline void zero_ino_node_unused(struct ubifs_ino_node *ino) +{ + memset(ino->padding1, 0, 4); + memset(ino->padding2, 0, 26); +} + +/** + * zero_dent_node_unused - zero out unused fields of an on-flash directory + * entry node. + * @dent: the directory entry to zero out + */ +static inline void zero_dent_node_unused(struct ubifs_dent_node *dent) +{ + dent->padding1 = 0; +} + +/** + * zero_trun_node_unused - zero out unused fields of an on-flash truncation + * node. + * @trun: the truncation node to zero out + */ +static inline void zero_trun_node_unused(struct ubifs_trun_node *trun) +{ + memset(trun->padding, 0, 12); +} + +static void ubifs_add_auth_dirt(struct ubifs_info *c, int lnum) +{ + if (ubifs_authenticated(c)) + ubifs_add_dirt(c, lnum, ubifs_auth_node_sz(c)); +} + +/** + * reserve_space - reserve space in the journal. + * @c: UBIFS file-system description object + * @jhead: journal head number + * @len: node length + * + * This function reserves space in journal head @head. If the reservation + * succeeded, the journal head stays locked and later has to be unlocked using + * 'release_head()'. Returns zero in case of success, %-EAGAIN if commit has to + * be done, and other negative error codes in case of other failures. + */ +static int reserve_space(struct ubifs_info *c, int jhead, int len) +{ + int err = 0, err1, retries = 0, avail, lnum, offs, squeeze; + struct ubifs_wbuf *wbuf = &c->jheads[jhead].wbuf; + + /* + * Typically, the base head has smaller nodes written to it, so it is + * better to try to allocate space at the ends of eraseblocks. This is + * what the squeeze parameter does. + */ + ubifs_assert(c, !c->ro_media && !c->ro_mount); + squeeze = (jhead == BASEHD); +again: + mutex_lock_nested(&wbuf->io_mutex, wbuf->jhead); + + if (c->ro_error) { + err = -EROFS; + goto out_unlock; + } + + avail = c->leb_size - wbuf->offs - wbuf->used; + if (wbuf->lnum != -1 && avail >= len) + return 0; + + /* + * Write buffer wasn't seek'ed or there is no enough space - look for an + * LEB with some empty space. + */ + lnum = ubifs_find_free_space(c, len, &offs, squeeze); + if (lnum >= 0) + goto out; + + err = lnum; + if (err != -ENOSPC) + goto out_unlock; + + /* + * No free space, we have to run garbage collector to make + * some. But the write-buffer mutex has to be unlocked because + * GC also takes it. + */ + dbg_jnl("no free space in jhead %s, run GC", dbg_jhead(jhead)); + mutex_unlock(&wbuf->io_mutex); + + lnum = ubifs_garbage_collect(c, 0); + if (lnum < 0) { + err = lnum; + if (err != -ENOSPC) + return err; + + /* + * GC could not make a free LEB. But someone else may + * have allocated new bud for this journal head, + * because we dropped @wbuf->io_mutex, so try once + * again. + */ + dbg_jnl("GC couldn't make a free LEB for jhead %s", + dbg_jhead(jhead)); + if (retries++ < 2) { + dbg_jnl("retry (%d)", retries); + goto again; + } + + dbg_jnl("return -ENOSPC"); + return err; + } + + mutex_lock_nested(&wbuf->io_mutex, wbuf->jhead); + dbg_jnl("got LEB %d for jhead %s", lnum, dbg_jhead(jhead)); + avail = c->leb_size - wbuf->offs - wbuf->used; + + if (wbuf->lnum != -1 && avail >= len) { + /* + * Someone else has switched the journal head and we have + * enough space now. This happens when more than one process is + * trying to write to the same journal head at the same time. + */ + dbg_jnl("return LEB %d back, already have LEB %d:%d", + lnum, wbuf->lnum, wbuf->offs + wbuf->used); + err = ubifs_return_leb(c, lnum); + if (err) + goto out_unlock; + return 0; + } + + offs = 0; + +out: + /* + * Make sure we synchronize the write-buffer before we add the new bud + * to the log. Otherwise we may have a power cut after the log + * reference node for the last bud (@lnum) is written but before the + * write-buffer data are written to the next-to-last bud + * (@wbuf->lnum). And the effect would be that the recovery would see + * that there is corruption in the next-to-last bud. + */ + err = ubifs_wbuf_sync_nolock(wbuf); + if (err) + goto out_return; + err = ubifs_add_bud_to_log(c, jhead, lnum, offs); + if (err) + goto out_return; + err = ubifs_wbuf_seek_nolock(wbuf, lnum, offs); + if (err) + goto out_unlock; + + return 0; + +out_unlock: + mutex_unlock(&wbuf->io_mutex); + return err; + +out_return: + /* An error occurred and the LEB has to be returned to lprops */ + ubifs_assert(c, err < 0); + err1 = ubifs_return_leb(c, lnum); + if (err1 && err == -EAGAIN) + /* + * Return original error code only if it is not %-EAGAIN, + * which is not really an error. Otherwise, return the error + * code of 'ubifs_return_leb()'. + */ + err = err1; + mutex_unlock(&wbuf->io_mutex); + return err; +} + +static int ubifs_hash_nodes(struct ubifs_info *c, void *node, + int len, struct shash_desc *hash) +{ + int auth_node_size = ubifs_auth_node_sz(c); + int err; + + while (1) { + const struct ubifs_ch *ch = node; + int nodelen = le32_to_cpu(ch->len); + + ubifs_assert(c, len >= auth_node_size); + + if (len == auth_node_size) + break; + + ubifs_assert(c, len > nodelen); + ubifs_assert(c, ch->magic == cpu_to_le32(UBIFS_NODE_MAGIC)); + + err = ubifs_shash_update(c, hash, (void *)node, nodelen); + if (err) + return err; + + node += ALIGN(nodelen, 8); + len -= ALIGN(nodelen, 8); + } + + return ubifs_prepare_auth_node(c, node, hash); +} + +/** + * write_head - write data to a journal head. + * @c: UBIFS file-system description object + * @jhead: journal head + * @buf: buffer to write + * @len: length to write + * @lnum: LEB number written is returned here + * @offs: offset written is returned here + * @sync: non-zero if the write-buffer has to by synchronized + * + * This function writes data to the reserved space of journal head @jhead. + * Returns zero in case of success and a negative error code in case of + * failure. + */ +static int write_head(struct ubifs_info *c, int jhead, void *buf, int len, + int *lnum, int *offs, int sync) +{ + int err; + struct ubifs_wbuf *wbuf = &c->jheads[jhead].wbuf; + + ubifs_assert(c, jhead != GCHD); + + *lnum = c->jheads[jhead].wbuf.lnum; + *offs = c->jheads[jhead].wbuf.offs + c->jheads[jhead].wbuf.used; + dbg_jnl("jhead %s, LEB %d:%d, len %d", + dbg_jhead(jhead), *lnum, *offs, len); + + if (ubifs_authenticated(c)) { + err = ubifs_hash_nodes(c, buf, len, c->jheads[jhead].log_hash); + if (err) + return err; + } + + err = ubifs_wbuf_write_nolock(wbuf, buf, len); + if (err) + return err; + if (sync) + err = ubifs_wbuf_sync_nolock(wbuf); + return err; +} + +/** + * __queue_and_wait - queue a task and wait until the task is waked up. + * @c: UBIFS file-system description object + * + * This function adds current task in queue and waits until the task is waked + * up. This function should be called with @c->reserve_space_wq locked. + */ +static void __queue_and_wait(struct ubifs_info *c) +{ + DEFINE_WAIT(wait); + + __add_wait_queue_entry_tail_exclusive(&c->reserve_space_wq, &wait); + set_current_state(TASK_UNINTERRUPTIBLE); + spin_unlock(&c->reserve_space_wq.lock); + + schedule(); + finish_wait(&c->reserve_space_wq, &wait); +} + +/** + * wait_for_reservation - try queuing current task to wait until waked up. + * @c: UBIFS file-system description object + * + * This function queues current task to wait until waked up, if queuing is + * started(@c->need_wait_space is not %0). Returns %true if current task is + * added in queue, otherwise %false is returned. + */ +static bool wait_for_reservation(struct ubifs_info *c) +{ + if (likely(atomic_read(&c->need_wait_space) == 0)) + /* Quick path to check whether queuing is started. */ + return false; + + spin_lock(&c->reserve_space_wq.lock); + if (atomic_read(&c->need_wait_space) == 0) { + /* Queuing is not started, don't queue current task. */ + spin_unlock(&c->reserve_space_wq.lock); + return false; + } + + __queue_and_wait(c); + return true; +} + +/** + * wake_up_reservation - wake up first task in queue or stop queuing. + * @c: UBIFS file-system description object + * + * This function wakes up the first task in queue if it exists, or stops + * queuing if no tasks in queue. + */ +static void wake_up_reservation(struct ubifs_info *c) +{ + spin_lock(&c->reserve_space_wq.lock); + if (waitqueue_active(&c->reserve_space_wq)) + wake_up_locked(&c->reserve_space_wq); + else + /* + * Compared with wait_for_reservation(), set @c->need_wait_space + * under the protection of wait queue lock, which can avoid that + * @c->need_wait_space is set to 0 after new task queued. + */ + atomic_set(&c->need_wait_space, 0); + spin_unlock(&c->reserve_space_wq.lock); +} + +/** + * add_or_start_queue - add current task in queue or start queuing. + * @c: UBIFS file-system description object + * + * This function starts queuing if queuing is not started, otherwise adds + * current task in queue. + */ +static void add_or_start_queue(struct ubifs_info *c) +{ + spin_lock(&c->reserve_space_wq.lock); + if (atomic_cmpxchg(&c->need_wait_space, 0, 1) == 0) { + /* Starts queuing, task can go on directly. */ + spin_unlock(&c->reserve_space_wq.lock); + return; + } + + /* + * There are at least two tasks have retried more than 32 times + * at certain point, first task has started queuing, just queue + * the left tasks. + */ + __queue_and_wait(c); +} + +/** + * make_reservation - reserve journal space. + * @c: UBIFS file-system description object + * @jhead: journal head + * @len: how many bytes to reserve + * + * This function makes space reservation in journal head @jhead. The function + * takes the commit lock and locks the journal head, and the caller has to + * unlock the head and finish the reservation with 'finish_reservation()'. + * Returns zero in case of success and a negative error code in case of + * failure. + * + * Note, the journal head may be unlocked as soon as the data is written, while + * the commit lock has to be released after the data has been added to the + * TNC. + */ +static int make_reservation(struct ubifs_info *c, int jhead, int len) +{ + int err, cmt_retries = 0, nospc_retries = 0; + bool blocked = wait_for_reservation(c); + +again: + down_read(&c->commit_sem); + err = reserve_space(c, jhead, len); + if (!err) { + /* c->commit_sem will get released via finish_reservation(). */ + goto out_wake_up; + } + up_read(&c->commit_sem); + + if (err == -ENOSPC) { + /* + * GC could not make any progress. We should try to commit + * because it could make some dirty space and GC would make + * progress, so make the error -EAGAIN so that the below + * will commit and re-try. + */ + nospc_retries++; + dbg_jnl("no space, retry"); + err = -EAGAIN; + } + + if (err != -EAGAIN) + goto out; + + /* + * -EAGAIN means that the journal is full or too large, or the above + * code wants to do one commit. Do this and re-try. + */ + if (cmt_retries > 128) { + /* + * This should not happen unless: + * 1. The journal size limitations are too tough. + * 2. The budgeting is incorrect. We always have to be able to + * write to the media, because all operations are budgeted. + * Deletions are not budgeted, though, but we reserve an + * extra LEB for them. + */ + ubifs_err(c, "stuck in space allocation, nospc_retries %d", + nospc_retries); + err = -ENOSPC; + goto out; + } else if (cmt_retries > 32) { + /* + * It's almost impossible to happen, unless there are many tasks + * making reservation concurrently and someone task has retried + * gc + commit for many times, generated available space during + * this period are grabbed by other tasks. + * But if it happens, start queuing up all tasks that will make + * space reservation, then there is only one task making space + * reservation at any time, and it can always make success under + * the premise of correct budgeting. + */ + ubifs_warn(c, "too many space allocation cmt_retries (%d) " + "nospc_retries (%d), start queuing tasks", + cmt_retries, nospc_retries); + + if (!blocked) { + blocked = true; + add_or_start_queue(c); + } + } + + dbg_jnl("-EAGAIN, commit and retry (retried %d times)", + cmt_retries); + cmt_retries += 1; + + err = ubifs_run_commit(c); + if (err) + goto out_wake_up; + goto again; + +out: + ubifs_err(c, "cannot reserve %d bytes in jhead %d, error %d", + len, jhead, err); + if (err == -ENOSPC) { + /* This are some budgeting problems, print useful information */ + down_write(&c->commit_sem); + dump_stack(); + ubifs_dump_budg(c, &c->bi); + ubifs_dump_lprops(c); + cmt_retries = dbg_check_lprops(c); + up_write(&c->commit_sem); + } +out_wake_up: + if (blocked) { + /* + * Only tasks that have ever started queuing or ever been queued + * can wake up other queued tasks, which can make sure that + * there is only one task waked up to make space reservation. + * For example: + * task A task B task C + * make_reservation make_reservation + * reserve_space // 0 + * wake_up_reservation + * atomic_cmpxchg // 0, start queuing + * reserve_space + * wait_for_reservation + * __queue_and_wait + * add_wait_queue + * if (blocked) // false + * // So that task C won't be waked up to race with task B + */ + wake_up_reservation(c); + } + return err; +} + +/** + * release_head - release a journal head. + * @c: UBIFS file-system description object + * @jhead: journal head + * + * This function releases journal head @jhead which was locked by + * the 'make_reservation()' function. It has to be called after each successful + * 'make_reservation()' invocation. + */ +static inline void release_head(struct ubifs_info *c, int jhead) +{ + mutex_unlock(&c->jheads[jhead].wbuf.io_mutex); +} + +/** + * finish_reservation - finish a reservation. + * @c: UBIFS file-system description object + * + * This function finishes journal space reservation. It must be called after + * 'make_reservation()'. + */ +static void finish_reservation(struct ubifs_info *c) +{ + up_read(&c->commit_sem); +} + +/** + * get_dent_type - translate VFS inode mode to UBIFS directory entry type. + * @mode: inode mode + */ +static int get_dent_type(int mode) +{ + switch (mode & S_IFMT) { + case S_IFREG: + return UBIFS_ITYPE_REG; + case S_IFDIR: + return UBIFS_ITYPE_DIR; + case S_IFLNK: + return UBIFS_ITYPE_LNK; + case S_IFBLK: + return UBIFS_ITYPE_BLK; + case S_IFCHR: + return UBIFS_ITYPE_CHR; + case S_IFIFO: + return UBIFS_ITYPE_FIFO; + case S_IFSOCK: + return UBIFS_ITYPE_SOCK; + default: + BUG(); + } + return 0; +} + +/** + * pack_inode - pack an inode node. + * @c: UBIFS file-system description object + * @ino: buffer in which to pack inode node + * @inode: inode to pack + * @last: indicates the last node of the group + */ +static void pack_inode(struct ubifs_info *c, struct ubifs_ino_node *ino, + const struct inode *inode, int last) +{ + int data_len = 0, last_reference = !inode->i_nlink; + struct ubifs_inode *ui = ubifs_inode(inode); + + ino->ch.node_type = UBIFS_INO_NODE; + ino_key_init_flash(c, &ino->key, inode->i_ino); + ino->creat_sqnum = cpu_to_le64(ui->creat_sqnum); + ino->atime_sec = cpu_to_le64(inode_get_atime_sec(inode)); + ino->atime_nsec = cpu_to_le32(inode_get_atime_nsec(inode)); + ino->ctime_sec = cpu_to_le64(inode_get_ctime_sec(inode)); + ino->ctime_nsec = cpu_to_le32(inode_get_ctime_nsec(inode)); + ino->mtime_sec = cpu_to_le64(inode_get_mtime_sec(inode)); + ino->mtime_nsec = cpu_to_le32(inode_get_mtime_nsec(inode)); + ino->uid = cpu_to_le32(i_uid_read(inode)); + ino->gid = cpu_to_le32(i_gid_read(inode)); + ino->mode = cpu_to_le32(inode->i_mode); + ino->flags = cpu_to_le32(ui->flags); + ino->size = cpu_to_le64(ui->ui_size); + ino->nlink = cpu_to_le32(inode->i_nlink); + ino->compr_type = cpu_to_le16(ui->compr_type); + ino->data_len = cpu_to_le32(ui->data_len); + ino->xattr_cnt = cpu_to_le32(ui->xattr_cnt); + ino->xattr_size = cpu_to_le32(ui->xattr_size); + ino->xattr_names = cpu_to_le32(ui->xattr_names); + zero_ino_node_unused(ino); + + /* + * Drop the attached data if this is a deletion inode, the data is not + * needed anymore. + */ + if (!last_reference) { + memcpy(ino->data, ui->data, ui->data_len); + data_len = ui->data_len; + } + + ubifs_prep_grp_node(c, ino, UBIFS_INO_NODE_SZ + data_len, last); +} + +/** + * mark_inode_clean - mark UBIFS inode as clean. + * @c: UBIFS file-system description object + * @ui: UBIFS inode to mark as clean + * + * This helper function marks UBIFS inode @ui as clean by cleaning the + * @ui->dirty flag and releasing its budget. Note, VFS may still treat the + * inode as dirty and try to write it back, but 'ubifs_write_inode()' would + * just do nothing. + */ +static void mark_inode_clean(struct ubifs_info *c, struct ubifs_inode *ui) +{ + if (ui->dirty) + ubifs_release_dirty_inode_budget(c, ui); + ui->dirty = 0; +} + +static void set_dent_cookie(struct ubifs_info *c, struct ubifs_dent_node *dent) +{ + if (c->double_hash) + dent->cookie = (__force __le32) get_random_u32(); + else + dent->cookie = 0; +} + +/** + * ubifs_jnl_update - update inode. + * @c: UBIFS file-system description object + * @dir: parent inode or host inode in case of extended attributes + * @nm: directory entry name + * @inode: inode to update + * @deletion: indicates a directory entry deletion i.e unlink or rmdir + * @xent: non-zero if the directory entry is an extended attribute entry + * @in_orphan: indicates whether the @inode is in orphan list + * + * This function updates an inode by writing a directory entry (or extended + * attribute entry), the inode itself, and the parent directory inode (or the + * host inode) to the journal. + * + * The function writes the host inode @dir last, which is important in case of + * extended attributes. Indeed, then we guarantee that if the host inode gets + * synchronized (with 'fsync()'), and the write-buffer it sits in gets flushed, + * the extended attribute inode gets flushed too. And this is exactly what the + * user expects - synchronizing the host inode synchronizes its extended + * attributes. Similarly, this guarantees that if @dir is synchronized, its + * directory entry corresponding to @nm gets synchronized too. + * + * If the inode (@inode) or the parent directory (@dir) are synchronous, this + * function synchronizes the write-buffer. + * + * This function marks the @dir and @inode inodes as clean and returns zero on + * success. In case of failure, a negative error code is returned. + */ +int ubifs_jnl_update(struct ubifs_info *c, const struct inode *dir, + const struct fscrypt_name *nm, const struct inode *inode, + int deletion, int xent, int in_orphan) +{ + int err, dlen, ilen, len, lnum, ino_offs, dent_offs, orphan_added = 0; + int aligned_dlen, aligned_ilen, sync = IS_DIRSYNC(dir); + int last_reference = !!(deletion && inode->i_nlink == 0); + struct ubifs_inode *ui = ubifs_inode(inode); + struct ubifs_inode *host_ui = ubifs_inode(dir); + struct ubifs_dent_node *dent; + struct ubifs_ino_node *ino; + union ubifs_key dent_key, ino_key; + u8 hash_dent[UBIFS_HASH_ARR_SZ]; + u8 hash_ino[UBIFS_HASH_ARR_SZ]; + u8 hash_ino_host[UBIFS_HASH_ARR_SZ]; + + ubifs_assert(c, mutex_is_locked(&host_ui->ui_mutex)); + + dlen = UBIFS_DENT_NODE_SZ + fname_len(nm) + 1; + ilen = UBIFS_INO_NODE_SZ; + + /* + * If the last reference to the inode is being deleted, then there is + * no need to attach and write inode data, it is being deleted anyway. + * And if the inode is being deleted, no need to synchronize + * write-buffer even if the inode is synchronous. + */ + if (!last_reference) { + ilen += ui->data_len; + sync |= IS_SYNC(inode); + } + + aligned_dlen = ALIGN(dlen, 8); + aligned_ilen = ALIGN(ilen, 8); + + len = aligned_dlen + aligned_ilen + UBIFS_INO_NODE_SZ; + /* Make sure to also account for extended attributes */ + if (ubifs_authenticated(c)) + len += ALIGN(host_ui->data_len, 8) + ubifs_auth_node_sz(c); + else + len += host_ui->data_len; + + dent = kzalloc(len, GFP_NOFS); + if (!dent) + return -ENOMEM; + + /* Make reservation before allocating sequence numbers */ + err = make_reservation(c, BASEHD, len); + if (err) + goto out_free; + + if (!xent) { + dent->ch.node_type = UBIFS_DENT_NODE; + if (fname_name(nm) == NULL) + dent_key_init_hash(c, &dent_key, dir->i_ino, nm->hash); + else + dent_key_init(c, &dent_key, dir->i_ino, nm); + } else { + dent->ch.node_type = UBIFS_XENT_NODE; + xent_key_init(c, &dent_key, dir->i_ino, nm); + } + + key_write(c, &dent_key, dent->key); + dent->inum = deletion ? 0 : cpu_to_le64(inode->i_ino); + dent->type = get_dent_type(inode->i_mode); + dent->nlen = cpu_to_le16(fname_len(nm)); + memcpy(dent->name, fname_name(nm), fname_len(nm)); + dent->name[fname_len(nm)] = '\0'; + set_dent_cookie(c, dent); + + zero_dent_node_unused(dent); + ubifs_prep_grp_node(c, dent, dlen, 0); + err = ubifs_node_calc_hash(c, dent, hash_dent); + if (err) + goto out_release; + + ino = (void *)dent + aligned_dlen; + pack_inode(c, ino, inode, 0); + err = ubifs_node_calc_hash(c, ino, hash_ino); + if (err) + goto out_release; + + ino = (void *)ino + aligned_ilen; + pack_inode(c, ino, dir, 1); + err = ubifs_node_calc_hash(c, ino, hash_ino_host); + if (err) + goto out_release; + + if (last_reference && !in_orphan) { + err = ubifs_add_orphan(c, inode->i_ino); + if (err) { + release_head(c, BASEHD); + goto out_finish; + } + ui->del_cmtno = c->cmt_no; + orphan_added = 1; + } + + err = write_head(c, BASEHD, dent, len, &lnum, &dent_offs, sync); + if (err) + goto out_release; + if (!sync) { + struct ubifs_wbuf *wbuf = &c->jheads[BASEHD].wbuf; + + ubifs_wbuf_add_ino_nolock(wbuf, inode->i_ino); + ubifs_wbuf_add_ino_nolock(wbuf, dir->i_ino); + } + release_head(c, BASEHD); + kfree(dent); + ubifs_add_auth_dirt(c, lnum); + + if (deletion) { + if (fname_name(nm) == NULL) + err = ubifs_tnc_remove_dh(c, &dent_key, nm->minor_hash); + else + err = ubifs_tnc_remove_nm(c, &dent_key, nm); + if (err) + goto out_ro; + err = ubifs_add_dirt(c, lnum, dlen); + } else + err = ubifs_tnc_add_nm(c, &dent_key, lnum, dent_offs, dlen, + hash_dent, nm); + if (err) + goto out_ro; + + /* + * Note, we do not remove the inode from TNC even if the last reference + * to it has just been deleted, because the inode may still be opened. + * Instead, the inode has been added to orphan lists and the orphan + * subsystem will take further care about it. + */ + ino_key_init(c, &ino_key, inode->i_ino); + ino_offs = dent_offs + aligned_dlen; + err = ubifs_tnc_add(c, &ino_key, lnum, ino_offs, ilen, hash_ino); + if (err) + goto out_ro; + + ino_key_init(c, &ino_key, dir->i_ino); + ino_offs += aligned_ilen; + err = ubifs_tnc_add(c, &ino_key, lnum, ino_offs, + UBIFS_INO_NODE_SZ + host_ui->data_len, hash_ino_host); + if (err) + goto out_ro; + + if (in_orphan && inode->i_nlink) + ubifs_delete_orphan(c, inode->i_ino); + + finish_reservation(c); + spin_lock(&ui->ui_lock); + ui->synced_i_size = ui->ui_size; + spin_unlock(&ui->ui_lock); + if (xent) { + spin_lock(&host_ui->ui_lock); + host_ui->synced_i_size = host_ui->ui_size; + spin_unlock(&host_ui->ui_lock); + } + mark_inode_clean(c, ui); + mark_inode_clean(c, host_ui); + return 0; + +out_finish: + finish_reservation(c); +out_free: + kfree(dent); + return err; + +out_release: + release_head(c, BASEHD); + kfree(dent); +out_ro: + ubifs_ro_mode(c, err); + if (orphan_added) + ubifs_delete_orphan(c, inode->i_ino); + finish_reservation(c); + return err; +} + +/** + * ubifs_jnl_write_data - write a data node to the journal. + * @c: UBIFS file-system description object + * @inode: inode the data node belongs to + * @key: node key + * @folio: buffer to write + * @offset: offset to write at + * @len: data length (must not exceed %UBIFS_BLOCK_SIZE) + * + * This function writes a data node to the journal. Returns %0 if the data node + * was successfully written, and a negative error code in case of failure. + */ +int ubifs_jnl_write_data(struct ubifs_info *c, const struct inode *inode, + const union ubifs_key *key, struct folio *folio, + size_t offset, int len) +{ + struct ubifs_data_node *data; + int err, lnum, offs, compr_type, out_len, compr_len, auth_len; + int dlen = COMPRESSED_DATA_NODE_BUF_SZ, allocated = 1; + int write_len; + struct ubifs_inode *ui = ubifs_inode(inode); + bool encrypted = IS_ENCRYPTED(inode); + u8 hash[UBIFS_HASH_ARR_SZ]; + + dbg_jnlk(key, "ino %lu, blk %u, len %d, key ", + (unsigned long)key_inum(c, key), key_block(c, key), len); + ubifs_assert(c, len <= UBIFS_BLOCK_SIZE); + + if (encrypted) + dlen += UBIFS_CIPHER_BLOCK_SIZE; + + auth_len = ubifs_auth_node_sz(c); + + data = kmalloc(dlen + auth_len, GFP_NOFS | __GFP_NOWARN); + if (!data) { + /* + * Fall-back to the write reserve buffer. Note, we might be + * currently on the memory reclaim path, when the kernel is + * trying to free some memory by writing out dirty pages. The + * write reserve buffer helps us to guarantee that we are + * always able to write the data. + */ + allocated = 0; + mutex_lock(&c->write_reserve_mutex); + data = c->write_reserve_buf; + } + + data->ch.node_type = UBIFS_DATA_NODE; + key_write(c, key, &data->key); + data->size = cpu_to_le32(len); + + if (!(ui->flags & UBIFS_COMPR_FL)) + /* Compression is disabled for this inode */ + compr_type = UBIFS_COMPR_NONE; + else + compr_type = ui->compr_type; + + out_len = compr_len = dlen - UBIFS_DATA_NODE_SZ; + ubifs_compress_folio(c, folio, offset, len, &data->data, &compr_len, + &compr_type); + ubifs_assert(c, compr_len <= UBIFS_BLOCK_SIZE); + + if (encrypted) { + err = ubifs_encrypt(inode, data, compr_len, &out_len, key_block(c, key)); + if (err) + goto out_free; + + } else { + data->compr_size = 0; + out_len = compr_len; + } + + dlen = UBIFS_DATA_NODE_SZ + out_len; + if (ubifs_authenticated(c)) + write_len = ALIGN(dlen, 8) + auth_len; + else + write_len = dlen; + + data->compr_type = cpu_to_le16(compr_type); + + /* Make reservation before allocating sequence numbers */ + err = make_reservation(c, DATAHD, write_len); + if (err) + goto out_free; + + ubifs_prepare_node(c, data, dlen, 0); + err = write_head(c, DATAHD, data, write_len, &lnum, &offs, 0); + if (err) + goto out_release; + + err = ubifs_node_calc_hash(c, data, hash); + if (err) + goto out_release; + + ubifs_wbuf_add_ino_nolock(&c->jheads[DATAHD].wbuf, key_inum(c, key)); + release_head(c, DATAHD); + + ubifs_add_auth_dirt(c, lnum); + + err = ubifs_tnc_add(c, key, lnum, offs, dlen, hash); + if (err) + goto out_ro; + + finish_reservation(c); + if (!allocated) + mutex_unlock(&c->write_reserve_mutex); + else + kfree(data); + return 0; + +out_release: + release_head(c, DATAHD); +out_ro: + ubifs_ro_mode(c, err); + finish_reservation(c); +out_free: + if (!allocated) + mutex_unlock(&c->write_reserve_mutex); + else + kfree(data); + return err; +} + +/** + * ubifs_jnl_write_inode - flush inode to the journal. + * @c: UBIFS file-system description object + * @inode: inode to flush + * + * This function writes inode @inode to the journal. If the inode is + * synchronous, it also synchronizes the write-buffer. Returns zero in case of + * success and a negative error code in case of failure. + */ +int ubifs_jnl_write_inode(struct ubifs_info *c, const struct inode *inode) +{ + int err, lnum, offs; + struct ubifs_ino_node *ino, *ino_start; + struct ubifs_inode *ui = ubifs_inode(inode); + int sync = 0, write_len = 0, ilen = UBIFS_INO_NODE_SZ; + int last_reference = !inode->i_nlink; + int kill_xattrs = ui->xattr_cnt && last_reference; + u8 hash[UBIFS_HASH_ARR_SZ]; + + dbg_jnl("ino %llu, nlink %u", inode->i_ino, inode->i_nlink); + + if (kill_xattrs && ui->xattr_cnt > ubifs_xattr_max_cnt(c)) { + ubifs_err(c, "Cannot delete inode, it has too many xattrs!"); + err = -EPERM; + ubifs_ro_mode(c, err); + return err; + } + + /* + * If the inode is being deleted, do not write the attached data. No + * need to synchronize the write-buffer either. + */ + if (!last_reference) { + ilen += ui->data_len; + sync = IS_SYNC(inode); + } else if (kill_xattrs) { + write_len += UBIFS_INO_NODE_SZ * ui->xattr_cnt; + } + + if (ubifs_authenticated(c)) + write_len += ALIGN(ilen, 8) + ubifs_auth_node_sz(c); + else + write_len += ilen; + + ino_start = ino = kmalloc(write_len, GFP_NOFS); + if (!ino) + return -ENOMEM; + + /* Make reservation before allocating sequence numbers */ + err = make_reservation(c, BASEHD, write_len); + if (err) + goto out_free; + + if (kill_xattrs) { + union ubifs_key key; + struct fscrypt_name nm = {0}; + struct inode *xino; + struct ubifs_dent_node *xent, *pxent = NULL; + + lowest_xent_key(c, &key, inode->i_ino); + while (1) { + xent = ubifs_tnc_next_ent(c, &key, &nm); + if (IS_ERR(xent)) { + err = PTR_ERR(xent); + if (err == -ENOENT) + break; + + kfree(pxent); + goto out_release; + } + + fname_name(&nm) = xent->name; + fname_len(&nm) = le16_to_cpu(xent->nlen); + + xino = ubifs_iget(c->vfs_sb, le64_to_cpu(xent->inum)); + if (IS_ERR(xino)) { + err = PTR_ERR(xino); + ubifs_err(c, "dead directory entry '%s', error %d", + xent->name, err); + ubifs_ro_mode(c, err); + kfree(pxent); + kfree(xent); + goto out_release; + } + ubifs_assert(c, ubifs_inode(xino)->xattr); + + clear_nlink(xino); + pack_inode(c, ino, xino, 0); + ino = (void *)ino + UBIFS_INO_NODE_SZ; + iput(xino); + + kfree(pxent); + pxent = xent; + key_read(c, &xent->key, &key); + } + kfree(pxent); + } + + pack_inode(c, ino, inode, 1); + err = ubifs_node_calc_hash(c, ino, hash); + if (err) + goto out_release; + + err = write_head(c, BASEHD, ino_start, write_len, &lnum, &offs, sync); + if (err) + goto out_release; + if (!sync) + ubifs_wbuf_add_ino_nolock(&c->jheads[BASEHD].wbuf, + inode->i_ino); + release_head(c, BASEHD); + + if (last_reference) { + err = ubifs_tnc_remove_ino(c, inode->i_ino); + if (err) + goto out_ro; + ubifs_delete_orphan(c, inode->i_ino); + err = ubifs_add_dirt(c, lnum, write_len); + } else { + union ubifs_key key; + + ubifs_add_auth_dirt(c, lnum); + + ino_key_init(c, &key, inode->i_ino); + err = ubifs_tnc_add(c, &key, lnum, offs, ilen, hash); + } + if (err) + goto out_ro; + + finish_reservation(c); + spin_lock(&ui->ui_lock); + ui->synced_i_size = ui->ui_size; + spin_unlock(&ui->ui_lock); + kfree(ino_start); + return 0; + +out_release: + release_head(c, BASEHD); +out_ro: + ubifs_ro_mode(c, err); + finish_reservation(c); +out_free: + kfree(ino_start); + return err; +} + +/** + * ubifs_jnl_delete_inode - delete an inode. + * @c: UBIFS file-system description object + * @inode: inode to delete + * + * This function deletes inode @inode which includes removing it from orphans, + * deleting it from TNC and, in some cases, writing a deletion inode to the + * journal. + * + * When regular file inodes are unlinked or a directory inode is removed, the + * 'ubifs_jnl_update()' function writes a corresponding deletion inode and + * direntry to the media, and adds the inode to orphans. After this, when the + * last reference to this inode has been dropped, this function is called. In + * general, it has to write one more deletion inode to the media, because if + * a commit happened between 'ubifs_jnl_update()' and + * 'ubifs_jnl_delete_inode()', the deletion inode is not in the journal + * anymore, and in fact it might not be on the flash anymore, because it might + * have been garbage-collected already. And for optimization reasons UBIFS does + * not read the orphan area if it has been unmounted cleanly, so it would have + * no indication in the journal that there is a deleted inode which has to be + * removed from TNC. + * + * However, if there was no commit between 'ubifs_jnl_update()' and + * 'ubifs_jnl_delete_inode()', then there is no need to write the deletion + * inode to the media for the second time. And this is quite a typical case. + * + * This function returns zero in case of success and a negative error code in + * case of failure. + */ +int ubifs_jnl_delete_inode(struct ubifs_info *c, const struct inode *inode) +{ + int err; + struct ubifs_inode *ui = ubifs_inode(inode); + + ubifs_assert(c, inode->i_nlink == 0); + + if (ui->xattr_cnt || ui->del_cmtno != c->cmt_no) + /* A commit happened for sure or inode hosts xattrs */ + return ubifs_jnl_write_inode(c, inode); + + down_read(&c->commit_sem); + /* + * Check commit number again, because the first test has been done + * without @c->commit_sem, so a commit might have happened. + */ + if (ui->del_cmtno != c->cmt_no) { + up_read(&c->commit_sem); + return ubifs_jnl_write_inode(c, inode); + } + + err = ubifs_tnc_remove_ino(c, inode->i_ino); + if (err) + ubifs_ro_mode(c, err); + else + ubifs_delete_orphan(c, inode->i_ino); + up_read(&c->commit_sem); + return err; +} + +/** + * ubifs_jnl_xrename - cross rename two directory entries. + * @c: UBIFS file-system description object + * @fst_dir: parent inode of 1st directory entry to exchange + * @fst_inode: 1st inode to exchange + * @fst_nm: name of 1st inode to exchange + * @snd_dir: parent inode of 2nd directory entry to exchange + * @snd_inode: 2nd inode to exchange + * @snd_nm: name of 2nd inode to exchange + * @sync: non-zero if the write-buffer has to be synchronized + * + * This function implements the cross rename operation which may involve + * writing 2 inodes and 2 directory entries. It marks the written inodes as clean + * and returns zero on success. In case of failure, a negative error code is + * returned. + */ +int ubifs_jnl_xrename(struct ubifs_info *c, const struct inode *fst_dir, + const struct inode *fst_inode, + const struct fscrypt_name *fst_nm, + const struct inode *snd_dir, + const struct inode *snd_inode, + const struct fscrypt_name *snd_nm, int sync) +{ + union ubifs_key key; + struct ubifs_dent_node *dent1, *dent2; + int err, dlen1, dlen2, lnum, offs, len, plen = UBIFS_INO_NODE_SZ; + int aligned_dlen1, aligned_dlen2; + int twoparents = (fst_dir != snd_dir); + void *p; + u8 hash_dent1[UBIFS_HASH_ARR_SZ]; + u8 hash_dent2[UBIFS_HASH_ARR_SZ]; + u8 hash_p1[UBIFS_HASH_ARR_SZ]; + u8 hash_p2[UBIFS_HASH_ARR_SZ]; + + ubifs_assert(c, ubifs_inode(fst_dir)->data_len == 0); + ubifs_assert(c, ubifs_inode(snd_dir)->data_len == 0); + ubifs_assert(c, mutex_is_locked(&ubifs_inode(fst_dir)->ui_mutex)); + ubifs_assert(c, mutex_is_locked(&ubifs_inode(snd_dir)->ui_mutex)); + + dlen1 = UBIFS_DENT_NODE_SZ + fname_len(snd_nm) + 1; + dlen2 = UBIFS_DENT_NODE_SZ + fname_len(fst_nm) + 1; + aligned_dlen1 = ALIGN(dlen1, 8); + aligned_dlen2 = ALIGN(dlen2, 8); + + len = aligned_dlen1 + aligned_dlen2 + ALIGN(plen, 8); + if (twoparents) + len += plen; + + len += ubifs_auth_node_sz(c); + + dent1 = kzalloc(len, GFP_NOFS); + if (!dent1) + return -ENOMEM; + + /* Make reservation before allocating sequence numbers */ + err = make_reservation(c, BASEHD, len); + if (err) + goto out_free; + + /* Make new dent for 1st entry */ + dent1->ch.node_type = UBIFS_DENT_NODE; + dent_key_init_flash(c, &dent1->key, snd_dir->i_ino, snd_nm); + dent1->inum = cpu_to_le64(fst_inode->i_ino); + dent1->type = get_dent_type(fst_inode->i_mode); + dent1->nlen = cpu_to_le16(fname_len(snd_nm)); + memcpy(dent1->name, fname_name(snd_nm), fname_len(snd_nm)); + dent1->name[fname_len(snd_nm)] = '\0'; + set_dent_cookie(c, dent1); + zero_dent_node_unused(dent1); + ubifs_prep_grp_node(c, dent1, dlen1, 0); + err = ubifs_node_calc_hash(c, dent1, hash_dent1); + if (err) + goto out_release; + + /* Make new dent for 2nd entry */ + dent2 = (void *)dent1 + aligned_dlen1; + dent2->ch.node_type = UBIFS_DENT_NODE; + dent_key_init_flash(c, &dent2->key, fst_dir->i_ino, fst_nm); + dent2->inum = cpu_to_le64(snd_inode->i_ino); + dent2->type = get_dent_type(snd_inode->i_mode); + dent2->nlen = cpu_to_le16(fname_len(fst_nm)); + memcpy(dent2->name, fname_name(fst_nm), fname_len(fst_nm)); + dent2->name[fname_len(fst_nm)] = '\0'; + set_dent_cookie(c, dent2); + zero_dent_node_unused(dent2); + ubifs_prep_grp_node(c, dent2, dlen2, 0); + err = ubifs_node_calc_hash(c, dent2, hash_dent2); + if (err) + goto out_release; + + p = (void *)dent2 + aligned_dlen2; + if (!twoparents) { + pack_inode(c, p, fst_dir, 1); + err = ubifs_node_calc_hash(c, p, hash_p1); + if (err) + goto out_release; + } else { + pack_inode(c, p, fst_dir, 0); + err = ubifs_node_calc_hash(c, p, hash_p1); + if (err) + goto out_release; + p += ALIGN(plen, 8); + pack_inode(c, p, snd_dir, 1); + err = ubifs_node_calc_hash(c, p, hash_p2); + if (err) + goto out_release; + } + + err = write_head(c, BASEHD, dent1, len, &lnum, &offs, sync); + if (err) + goto out_release; + if (!sync) { + struct ubifs_wbuf *wbuf = &c->jheads[BASEHD].wbuf; + + ubifs_wbuf_add_ino_nolock(wbuf, fst_dir->i_ino); + ubifs_wbuf_add_ino_nolock(wbuf, snd_dir->i_ino); + } + release_head(c, BASEHD); + + ubifs_add_auth_dirt(c, lnum); + + dent_key_init(c, &key, snd_dir->i_ino, snd_nm); + err = ubifs_tnc_add_nm(c, &key, lnum, offs, dlen1, hash_dent1, snd_nm); + if (err) + goto out_ro; + + offs += aligned_dlen1; + dent_key_init(c, &key, fst_dir->i_ino, fst_nm); + err = ubifs_tnc_add_nm(c, &key, lnum, offs, dlen2, hash_dent2, fst_nm); + if (err) + goto out_ro; + + offs += aligned_dlen2; + + ino_key_init(c, &key, fst_dir->i_ino); + err = ubifs_tnc_add(c, &key, lnum, offs, plen, hash_p1); + if (err) + goto out_ro; + + if (twoparents) { + offs += ALIGN(plen, 8); + ino_key_init(c, &key, snd_dir->i_ino); + err = ubifs_tnc_add(c, &key, lnum, offs, plen, hash_p2); + if (err) + goto out_ro; + } + + finish_reservation(c); + + mark_inode_clean(c, ubifs_inode(fst_dir)); + if (twoparents) + mark_inode_clean(c, ubifs_inode(snd_dir)); + kfree(dent1); + return 0; + +out_release: + release_head(c, BASEHD); +out_ro: + ubifs_ro_mode(c, err); + finish_reservation(c); +out_free: + kfree(dent1); + return err; +} + +/** + * ubifs_jnl_rename - rename a directory entry. + * @c: UBIFS file-system description object + * @old_dir: parent inode of directory entry to rename + * @old_inode: directory entry's inode to rename + * @old_nm: name of the old directory entry to rename + * @new_dir: parent inode of directory entry to rename + * @new_inode: new directory entry's inode (or directory entry's inode to + * replace) + * @new_nm: new name of the new directory entry + * @whiteout: whiteout inode + * @sync: non-zero if the write-buffer has to be synchronized + * @delete_orphan: indicates an orphan entry deletion for @whiteout + * + * This function implements the re-name operation which may involve writing up + * to 4 inodes(new inode, whiteout inode, old and new parent directory inodes) + * and 2 directory entries. It marks the written inodes as clean and returns + * zero on success. In case of failure, a negative error code is returned. + */ +int ubifs_jnl_rename(struct ubifs_info *c, const struct inode *old_dir, + const struct inode *old_inode, + const struct fscrypt_name *old_nm, + const struct inode *new_dir, + const struct inode *new_inode, + const struct fscrypt_name *new_nm, + const struct inode *whiteout, int sync, int delete_orphan) +{ + void *p; + union ubifs_key key; + struct ubifs_dent_node *dent, *dent2; + int err, dlen1, dlen2, ilen, wlen, lnum, offs, len, orphan_added = 0; + int aligned_dlen1, aligned_dlen2, plen = UBIFS_INO_NODE_SZ; + int last_reference = !!(new_inode && new_inode->i_nlink == 0); + int move = (old_dir != new_dir); + struct ubifs_inode *new_ui, *whiteout_ui; + u8 hash_old_dir[UBIFS_HASH_ARR_SZ]; + u8 hash_new_dir[UBIFS_HASH_ARR_SZ]; + u8 hash_new_inode[UBIFS_HASH_ARR_SZ]; + u8 hash_whiteout_inode[UBIFS_HASH_ARR_SZ]; + u8 hash_dent1[UBIFS_HASH_ARR_SZ]; + u8 hash_dent2[UBIFS_HASH_ARR_SZ]; + + ubifs_assert(c, ubifs_inode(old_dir)->data_len == 0); + ubifs_assert(c, ubifs_inode(new_dir)->data_len == 0); + ubifs_assert(c, mutex_is_locked(&ubifs_inode(old_dir)->ui_mutex)); + ubifs_assert(c, mutex_is_locked(&ubifs_inode(new_dir)->ui_mutex)); + + dlen1 = UBIFS_DENT_NODE_SZ + fname_len(new_nm) + 1; + dlen2 = UBIFS_DENT_NODE_SZ + fname_len(old_nm) + 1; + if (new_inode) { + new_ui = ubifs_inode(new_inode); + ubifs_assert(c, mutex_is_locked(&new_ui->ui_mutex)); + ilen = UBIFS_INO_NODE_SZ; + if (!last_reference) + ilen += new_ui->data_len; + } else + ilen = 0; + + if (whiteout) { + whiteout_ui = ubifs_inode(whiteout); + ubifs_assert(c, mutex_is_locked(&whiteout_ui->ui_mutex)); + ubifs_assert(c, whiteout->i_nlink == 1); + ubifs_assert(c, !whiteout_ui->dirty); + wlen = UBIFS_INO_NODE_SZ; + wlen += whiteout_ui->data_len; + } else + wlen = 0; + + aligned_dlen1 = ALIGN(dlen1, 8); + aligned_dlen2 = ALIGN(dlen2, 8); + len = aligned_dlen1 + aligned_dlen2 + ALIGN(ilen, 8) + + ALIGN(wlen, 8) + ALIGN(plen, 8); + if (move) + len += plen; + + len += ubifs_auth_node_sz(c); + + dent = kzalloc(len, GFP_NOFS); + if (!dent) + return -ENOMEM; + + /* Make reservation before allocating sequence numbers */ + err = make_reservation(c, BASEHD, len); + if (err) + goto out_free; + + /* Make new dent */ + dent->ch.node_type = UBIFS_DENT_NODE; + dent_key_init_flash(c, &dent->key, new_dir->i_ino, new_nm); + dent->inum = cpu_to_le64(old_inode->i_ino); + dent->type = get_dent_type(old_inode->i_mode); + dent->nlen = cpu_to_le16(fname_len(new_nm)); + memcpy(dent->name, fname_name(new_nm), fname_len(new_nm)); + dent->name[fname_len(new_nm)] = '\0'; + set_dent_cookie(c, dent); + zero_dent_node_unused(dent); + ubifs_prep_grp_node(c, dent, dlen1, 0); + err = ubifs_node_calc_hash(c, dent, hash_dent1); + if (err) + goto out_release; + + dent2 = (void *)dent + aligned_dlen1; + dent2->ch.node_type = UBIFS_DENT_NODE; + dent_key_init_flash(c, &dent2->key, old_dir->i_ino, old_nm); + + if (whiteout) { + dent2->inum = cpu_to_le64(whiteout->i_ino); + dent2->type = get_dent_type(whiteout->i_mode); + } else { + /* Make deletion dent */ + dent2->inum = 0; + dent2->type = DT_UNKNOWN; + } + dent2->nlen = cpu_to_le16(fname_len(old_nm)); + memcpy(dent2->name, fname_name(old_nm), fname_len(old_nm)); + dent2->name[fname_len(old_nm)] = '\0'; + set_dent_cookie(c, dent2); + zero_dent_node_unused(dent2); + ubifs_prep_grp_node(c, dent2, dlen2, 0); + err = ubifs_node_calc_hash(c, dent2, hash_dent2); + if (err) + goto out_release; + + p = (void *)dent2 + aligned_dlen2; + if (new_inode) { + pack_inode(c, p, new_inode, 0); + err = ubifs_node_calc_hash(c, p, hash_new_inode); + if (err) + goto out_release; + + p += ALIGN(ilen, 8); + } + + if (whiteout) { + pack_inode(c, p, whiteout, 0); + err = ubifs_node_calc_hash(c, p, hash_whiteout_inode); + if (err) + goto out_release; + + p += ALIGN(wlen, 8); + } + + if (!move) { + pack_inode(c, p, old_dir, 1); + err = ubifs_node_calc_hash(c, p, hash_old_dir); + if (err) + goto out_release; + } else { + pack_inode(c, p, old_dir, 0); + err = ubifs_node_calc_hash(c, p, hash_old_dir); + if (err) + goto out_release; + + p += ALIGN(plen, 8); + pack_inode(c, p, new_dir, 1); + err = ubifs_node_calc_hash(c, p, hash_new_dir); + if (err) + goto out_release; + } + + if (last_reference) { + err = ubifs_add_orphan(c, new_inode->i_ino); + if (err) { + release_head(c, BASEHD); + goto out_finish; + } + new_ui->del_cmtno = c->cmt_no; + orphan_added = 1; + } + + err = write_head(c, BASEHD, dent, len, &lnum, &offs, sync); + if (err) + goto out_release; + if (!sync) { + struct ubifs_wbuf *wbuf = &c->jheads[BASEHD].wbuf; + + ubifs_wbuf_add_ino_nolock(wbuf, new_dir->i_ino); + ubifs_wbuf_add_ino_nolock(wbuf, old_dir->i_ino); + if (new_inode) + ubifs_wbuf_add_ino_nolock(&c->jheads[BASEHD].wbuf, + new_inode->i_ino); + if (whiteout) + ubifs_wbuf_add_ino_nolock(&c->jheads[BASEHD].wbuf, + whiteout->i_ino); + } + release_head(c, BASEHD); + + ubifs_add_auth_dirt(c, lnum); + + dent_key_init(c, &key, new_dir->i_ino, new_nm); + err = ubifs_tnc_add_nm(c, &key, lnum, offs, dlen1, hash_dent1, new_nm); + if (err) + goto out_ro; + + offs += aligned_dlen1; + if (whiteout) { + dent_key_init(c, &key, old_dir->i_ino, old_nm); + err = ubifs_tnc_add_nm(c, &key, lnum, offs, dlen2, hash_dent2, old_nm); + if (err) + goto out_ro; + } else { + err = ubifs_add_dirt(c, lnum, dlen2); + if (err) + goto out_ro; + + dent_key_init(c, &key, old_dir->i_ino, old_nm); + err = ubifs_tnc_remove_nm(c, &key, old_nm); + if (err) + goto out_ro; + } + + offs += aligned_dlen2; + if (new_inode) { + ino_key_init(c, &key, new_inode->i_ino); + err = ubifs_tnc_add(c, &key, lnum, offs, ilen, hash_new_inode); + if (err) + goto out_ro; + offs += ALIGN(ilen, 8); + } + + if (whiteout) { + ino_key_init(c, &key, whiteout->i_ino); + err = ubifs_tnc_add(c, &key, lnum, offs, wlen, + hash_whiteout_inode); + if (err) + goto out_ro; + offs += ALIGN(wlen, 8); + } + + ino_key_init(c, &key, old_dir->i_ino); + err = ubifs_tnc_add(c, &key, lnum, offs, plen, hash_old_dir); + if (err) + goto out_ro; + + if (move) { + offs += ALIGN(plen, 8); + ino_key_init(c, &key, new_dir->i_ino); + err = ubifs_tnc_add(c, &key, lnum, offs, plen, hash_new_dir); + if (err) + goto out_ro; + } + + if (delete_orphan) + ubifs_delete_orphan(c, whiteout->i_ino); + + finish_reservation(c); + if (new_inode) { + mark_inode_clean(c, new_ui); + spin_lock(&new_ui->ui_lock); + new_ui->synced_i_size = new_ui->ui_size; + spin_unlock(&new_ui->ui_lock); + } + /* + * No need to mark whiteout inode clean. + * Whiteout doesn't have non-zero size, no need to update + * synced_i_size for whiteout_ui. + */ + mark_inode_clean(c, ubifs_inode(old_dir)); + if (move) + mark_inode_clean(c, ubifs_inode(new_dir)); + kfree(dent); + return 0; + +out_release: + release_head(c, BASEHD); +out_ro: + ubifs_ro_mode(c, err); + if (orphan_added) + ubifs_delete_orphan(c, new_inode->i_ino); +out_finish: + finish_reservation(c); +out_free: + kfree(dent); + return err; +} + +/** + * truncate_data_node - re-compress/encrypt a truncated data node. + * @c: UBIFS file-system description object + * @inode: inode which refers to the data node + * @block: data block number + * @dn: data node to re-compress + * @new_len: new length + * @dn_size: size of the data node @dn in memory + * + * This function is used when an inode is truncated and the last data node of + * the inode has to be re-compressed/encrypted and re-written. + */ +static int truncate_data_node(const struct ubifs_info *c, const struct inode *inode, + unsigned int block, struct ubifs_data_node *dn, + int *new_len, int dn_size) +{ + void *buf; + int err, dlen, compr_type, out_len, data_size; + + out_len = le32_to_cpu(dn->size); + buf = kmalloc(out_len, GFP_NOFS); + if (!buf) + return -ENOMEM; + + dlen = le32_to_cpu(dn->ch.len) - UBIFS_DATA_NODE_SZ; + data_size = dn_size - UBIFS_DATA_NODE_SZ; + compr_type = le16_to_cpu(dn->compr_type); + + if (IS_ENCRYPTED(inode)) { + err = ubifs_decrypt(inode, dn, &dlen, block); + if (err) + goto out; + } + + if (compr_type == UBIFS_COMPR_NONE) { + out_len = *new_len; + } else { + err = ubifs_decompress(c, &dn->data, dlen, buf, &out_len, compr_type); + if (err) + goto out; + + ubifs_compress(c, buf, *new_len, &dn->data, &out_len, &compr_type); + } + + if (IS_ENCRYPTED(inode)) { + err = ubifs_encrypt(inode, dn, out_len, &data_size, block); + if (err) + goto out; + + out_len = data_size; + } else { + dn->compr_size = 0; + } + + ubifs_assert(c, out_len <= UBIFS_BLOCK_SIZE); + dn->compr_type = cpu_to_le16(compr_type); + dn->size = cpu_to_le32(*new_len); + *new_len = UBIFS_DATA_NODE_SZ + out_len; + err = 0; +out: + kfree(buf); + return err; +} + +/** + * ubifs_jnl_truncate - update the journal for a truncation. + * @c: UBIFS file-system description object + * @inode: inode to truncate + * @old_size: old size + * @new_size: new size + * + * When the size of a file decreases due to truncation, a truncation node is + * written, the journal tree is updated, and the last data block is re-written + * if it has been affected. The inode is also updated in order to synchronize + * the new inode size. + * + * This function marks the inode as clean and returns zero on success. In case + * of failure, a negative error code is returned. + */ +int ubifs_jnl_truncate(struct ubifs_info *c, const struct inode *inode, + loff_t old_size, loff_t new_size) +{ + union ubifs_key key, to_key; + struct ubifs_ino_node *ino; + struct ubifs_trun_node *trun; + struct ubifs_data_node *dn; + int err, dlen, len, lnum, offs, bit, sz, sync = IS_SYNC(inode); + int dn_size; + struct ubifs_inode *ui = ubifs_inode(inode); + ino_t inum = inode->i_ino; + unsigned int blk; + u8 hash_ino[UBIFS_HASH_ARR_SZ]; + u8 hash_dn[UBIFS_HASH_ARR_SZ]; + + dbg_jnl("ino %lu, size %lld -> %lld", + (unsigned long)inum, old_size, new_size); + ubifs_assert(c, !ui->data_len); + ubifs_assert(c, S_ISREG(inode->i_mode)); + ubifs_assert(c, mutex_is_locked(&ui->ui_mutex)); + + dn_size = COMPRESSED_DATA_NODE_BUF_SZ; + + if (IS_ENCRYPTED(inode)) + dn_size += UBIFS_CIPHER_BLOCK_SIZE; + + sz = UBIFS_TRUN_NODE_SZ + UBIFS_INO_NODE_SZ + + dn_size + ubifs_auth_node_sz(c); + + ino = kmalloc(sz, GFP_NOFS); + if (!ino) + return -ENOMEM; + + trun = (void *)ino + UBIFS_INO_NODE_SZ; + trun->ch.node_type = UBIFS_TRUN_NODE; + trun->inum = cpu_to_le32(inum); + trun->old_size = cpu_to_le64(old_size); + trun->new_size = cpu_to_le64(new_size); + zero_trun_node_unused(trun); + + dlen = new_size & (UBIFS_BLOCK_SIZE - 1); + if (dlen) { + /* Get last data block so it can be truncated */ + dn = (void *)trun + UBIFS_TRUN_NODE_SZ; + blk = new_size >> UBIFS_BLOCK_SHIFT; + data_key_init(c, &key, inum, blk); + dbg_jnlk(&key, "last block key "); + err = ubifs_tnc_lookup(c, &key, dn); + if (err == -ENOENT) + dlen = 0; /* Not found (so it is a hole) */ + else if (err) + goto out_free; + else { + int dn_len = le32_to_cpu(dn->size); + + if (dn_len <= 0 || dn_len > UBIFS_BLOCK_SIZE) { + ubifs_err(c, "bad data node (block %u, inode %llu)", + blk, inode->i_ino); + ubifs_dump_node(c, dn, dn_size); + err = -EUCLEAN; + goto out_free; + } + + if (dn_len <= dlen) + dlen = 0; /* Nothing to do */ + else { + err = truncate_data_node(c, inode, blk, dn, + &dlen, dn_size); + if (err) + goto out_free; + } + } + } + + /* Must make reservation before allocating sequence numbers */ + len = UBIFS_TRUN_NODE_SZ + UBIFS_INO_NODE_SZ; + + if (ubifs_authenticated(c)) + len += ALIGN(dlen, 8) + ubifs_auth_node_sz(c); + else + len += dlen; + + err = make_reservation(c, BASEHD, len); + if (err) + goto out_free; + + pack_inode(c, ino, inode, 0); + err = ubifs_node_calc_hash(c, ino, hash_ino); + if (err) + goto out_release; + + ubifs_prep_grp_node(c, trun, UBIFS_TRUN_NODE_SZ, dlen ? 0 : 1); + if (dlen) { + ubifs_prep_grp_node(c, dn, dlen, 1); + err = ubifs_node_calc_hash(c, dn, hash_dn); + if (err) + goto out_release; + } + + err = write_head(c, BASEHD, ino, len, &lnum, &offs, sync); + if (err) + goto out_release; + if (!sync) + ubifs_wbuf_add_ino_nolock(&c->jheads[BASEHD].wbuf, inum); + release_head(c, BASEHD); + + ubifs_add_auth_dirt(c, lnum); + + if (dlen) { + sz = offs + UBIFS_INO_NODE_SZ + UBIFS_TRUN_NODE_SZ; + err = ubifs_tnc_add(c, &key, lnum, sz, dlen, hash_dn); + if (err) + goto out_ro; + } + + ino_key_init(c, &key, inum); + err = ubifs_tnc_add(c, &key, lnum, offs, UBIFS_INO_NODE_SZ, hash_ino); + if (err) + goto out_ro; + + err = ubifs_add_dirt(c, lnum, UBIFS_TRUN_NODE_SZ); + if (err) + goto out_ro; + + bit = new_size & (UBIFS_BLOCK_SIZE - 1); + blk = (new_size >> UBIFS_BLOCK_SHIFT) + (bit ? 1 : 0); + data_key_init(c, &key, inum, blk); + + bit = old_size & (UBIFS_BLOCK_SIZE - 1); + blk = (old_size >> UBIFS_BLOCK_SHIFT) - (bit ? 0 : 1); + data_key_init(c, &to_key, inum, blk); + + err = ubifs_tnc_remove_range(c, &key, &to_key); + if (err) + goto out_ro; + + finish_reservation(c); + spin_lock(&ui->ui_lock); + ui->synced_i_size = ui->ui_size; + spin_unlock(&ui->ui_lock); + mark_inode_clean(c, ui); + kfree(ino); + return 0; + +out_release: + release_head(c, BASEHD); +out_ro: + ubifs_ro_mode(c, err); + finish_reservation(c); +out_free: + kfree(ino); + return err; +} + + +/** + * ubifs_jnl_delete_xattr - delete an extended attribute. + * @c: UBIFS file-system description object + * @host: host inode + * @inode: extended attribute inode + * @nm: extended attribute entry name + * + * This function delete an extended attribute which is very similar to + * un-linking regular files - it writes a deletion xentry, a deletion inode and + * updates the target inode. Returns zero in case of success and a negative + * error code in case of failure. + */ +int ubifs_jnl_delete_xattr(struct ubifs_info *c, const struct inode *host, + const struct inode *inode, + const struct fscrypt_name *nm) +{ + int err, xlen, hlen, len, lnum, xent_offs, aligned_xlen, write_len; + struct ubifs_dent_node *xent; + struct ubifs_ino_node *ino; + union ubifs_key xent_key, key1, key2; + int sync = IS_DIRSYNC(host); + struct ubifs_inode *host_ui = ubifs_inode(host); + u8 hash[UBIFS_HASH_ARR_SZ]; + + ubifs_assert(c, inode->i_nlink == 0); + ubifs_assert(c, mutex_is_locked(&host_ui->ui_mutex)); + + /* + * Since we are deleting the inode, we do not bother to attach any data + * to it and assume its length is %UBIFS_INO_NODE_SZ. + */ + xlen = UBIFS_DENT_NODE_SZ + fname_len(nm) + 1; + aligned_xlen = ALIGN(xlen, 8); + hlen = host_ui->data_len + UBIFS_INO_NODE_SZ; + len = aligned_xlen + UBIFS_INO_NODE_SZ + ALIGN(hlen, 8); + + write_len = len + ubifs_auth_node_sz(c); + + xent = kzalloc(write_len, GFP_NOFS); + if (!xent) + return -ENOMEM; + + /* Make reservation before allocating sequence numbers */ + err = make_reservation(c, BASEHD, write_len); + if (err) { + kfree(xent); + return err; + } + + xent->ch.node_type = UBIFS_XENT_NODE; + xent_key_init(c, &xent_key, host->i_ino, nm); + key_write(c, &xent_key, xent->key); + xent->inum = 0; + xent->type = get_dent_type(inode->i_mode); + xent->nlen = cpu_to_le16(fname_len(nm)); + memcpy(xent->name, fname_name(nm), fname_len(nm)); + xent->name[fname_len(nm)] = '\0'; + zero_dent_node_unused(xent); + ubifs_prep_grp_node(c, xent, xlen, 0); + + ino = (void *)xent + aligned_xlen; + pack_inode(c, ino, inode, 0); + ino = (void *)ino + UBIFS_INO_NODE_SZ; + pack_inode(c, ino, host, 1); + err = ubifs_node_calc_hash(c, ino, hash); + if (err) + goto out_release; + + err = write_head(c, BASEHD, xent, write_len, &lnum, &xent_offs, sync); + if (!sync && !err) + ubifs_wbuf_add_ino_nolock(&c->jheads[BASEHD].wbuf, host->i_ino); + release_head(c, BASEHD); + + ubifs_add_auth_dirt(c, lnum); + kfree(xent); + if (err) + goto out_ro; + + /* Remove the extended attribute entry from TNC */ + err = ubifs_tnc_remove_nm(c, &xent_key, nm); + if (err) + goto out_ro; + err = ubifs_add_dirt(c, lnum, xlen); + if (err) + goto out_ro; + + /* + * Remove all nodes belonging to the extended attribute inode from TNC. + * Well, there actually must be only one node - the inode itself. + */ + lowest_ino_key(c, &key1, inode->i_ino); + highest_ino_key(c, &key2, inode->i_ino); + err = ubifs_tnc_remove_range(c, &key1, &key2); + if (err) + goto out_ro; + err = ubifs_add_dirt(c, lnum, UBIFS_INO_NODE_SZ); + if (err) + goto out_ro; + + /* And update TNC with the new host inode position */ + ino_key_init(c, &key1, host->i_ino); + err = ubifs_tnc_add(c, &key1, lnum, xent_offs + len - hlen, hlen, hash); + if (err) + goto out_ro; + + finish_reservation(c); + spin_lock(&host_ui->ui_lock); + host_ui->synced_i_size = host_ui->ui_size; + spin_unlock(&host_ui->ui_lock); + mark_inode_clean(c, host_ui); + return 0; + +out_release: + kfree(xent); + release_head(c, BASEHD); +out_ro: + ubifs_ro_mode(c, err); + finish_reservation(c); + return err; +} + +/** + * ubifs_jnl_change_xattr - change an extended attribute. + * @c: UBIFS file-system description object + * @inode: extended attribute inode + * @host: host inode + * + * This function writes the updated version of an extended attribute inode and + * the host inode to the journal (to the base head). The host inode is written + * after the extended attribute inode in order to guarantee that the extended + * attribute will be flushed when the inode is synchronized by 'fsync()' and + * consequently, the write-buffer is synchronized. This function returns zero + * in case of success and a negative error code in case of failure. + */ +int ubifs_jnl_change_xattr(struct ubifs_info *c, const struct inode *inode, + const struct inode *host) +{ + int err, len1, len2, aligned_len, aligned_len1, lnum, offs; + struct ubifs_inode *host_ui = ubifs_inode(host); + struct ubifs_ino_node *ino; + union ubifs_key key; + int sync = IS_DIRSYNC(host); + u8 hash_host[UBIFS_HASH_ARR_SZ]; + u8 hash[UBIFS_HASH_ARR_SZ]; + + dbg_jnl("ino %llu, ino %llu", host->i_ino, inode->i_ino); + ubifs_assert(c, inode->i_nlink > 0); + ubifs_assert(c, mutex_is_locked(&host_ui->ui_mutex)); + + len1 = UBIFS_INO_NODE_SZ + host_ui->data_len; + len2 = UBIFS_INO_NODE_SZ + ubifs_inode(inode)->data_len; + aligned_len1 = ALIGN(len1, 8); + aligned_len = aligned_len1 + ALIGN(len2, 8); + + aligned_len += ubifs_auth_node_sz(c); + + ino = kzalloc(aligned_len, GFP_NOFS); + if (!ino) + return -ENOMEM; + + /* Make reservation before allocating sequence numbers */ + err = make_reservation(c, BASEHD, aligned_len); + if (err) + goto out_free; + + pack_inode(c, ino, host, 0); + err = ubifs_node_calc_hash(c, ino, hash_host); + if (err) + goto out_release; + pack_inode(c, (void *)ino + aligned_len1, inode, 1); + err = ubifs_node_calc_hash(c, (void *)ino + aligned_len1, hash); + if (err) + goto out_release; + + err = write_head(c, BASEHD, ino, aligned_len, &lnum, &offs, 0); + if (!sync && !err) { + struct ubifs_wbuf *wbuf = &c->jheads[BASEHD].wbuf; + + ubifs_wbuf_add_ino_nolock(wbuf, host->i_ino); + ubifs_wbuf_add_ino_nolock(wbuf, inode->i_ino); + } + release_head(c, BASEHD); + if (err) + goto out_ro; + + ubifs_add_auth_dirt(c, lnum); + + ino_key_init(c, &key, host->i_ino); + err = ubifs_tnc_add(c, &key, lnum, offs, len1, hash_host); + if (err) + goto out_ro; + + ino_key_init(c, &key, inode->i_ino); + err = ubifs_tnc_add(c, &key, lnum, offs + aligned_len1, len2, hash); + if (err) + goto out_ro; + + finish_reservation(c); + spin_lock(&host_ui->ui_lock); + host_ui->synced_i_size = host_ui->ui_size; + spin_unlock(&host_ui->ui_lock); + mark_inode_clean(c, host_ui); + kfree(ino); + return 0; + +out_release: + release_head(c, BASEHD); +out_ro: + ubifs_ro_mode(c, err); + finish_reservation(c); +out_free: + kfree(ino); + return err; +} + diff --git a/fs/ubifs/key.h b/fs/ubifs/key.h new file mode 100644 index 000000000..8142d9d6f --- /dev/null +++ b/fs/ubifs/key.h @@ -0,0 +1,543 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * + * Authors: Artem Bityutskiy (Битюцкий Артём) + * Adrian Hunter + */ + +/* + * This header contains various key-related definitions and helper function. + * UBIFS allows several key schemes, so we access key fields only via these + * helpers. At the moment only one key scheme is supported. + * + * Simple key scheme + * ~~~~~~~~~~~~~~~~~ + * + * Keys are 64-bits long. First 32-bits are inode number (parent inode number + * in case of direntry key). Next 3 bits are node type. The last 29 bits are + * 4KiB offset in case of inode node, and direntry hash in case of a direntry + * node. We use "r5" hash borrowed from reiserfs. + */ + +/* + * Lot's of the key helpers require a struct ubifs_info *c as the first parameter. + * But we are not using it at all currently. That's designed for future extensions of + * different c->key_format. But right now, there is only one key type, UBIFS_SIMPLE_KEY_FMT. + */ + +#ifndef __UBIFS_KEY_H__ +#define __UBIFS_KEY_H__ + +/** + * key_mask_hash - mask a valid hash value. + * @val: value to be masked + * + * We use hash values as offset in directories, so values %0 and %1 are + * reserved for "." and "..". %2 is reserved for "end of readdir" marker. This + * function makes sure the reserved values are not used. + */ +static inline uint32_t key_mask_hash(uint32_t hash) +{ + hash &= UBIFS_S_KEY_HASH_MASK; + if (unlikely(hash <= 2)) + hash += 3; + return hash; +} + +/** + * key_r5_hash - R5 hash function (borrowed from reiserfs). + * @s: direntry name + * @len: name length + */ +static inline uint32_t key_r5_hash(const char *s, int len) +{ + uint32_t a = 0; + const signed char *str = (const signed char *)s; + + while (len--) { + a += *str << 4; + a += *str >> 4; + a *= 11; + str++; + } + + return key_mask_hash(a); +} + +/** + * key_test_hash - testing hash function. + * @str: direntry name + * @len: name length + */ +static inline uint32_t key_test_hash(const char *str, int len) +{ + uint32_t a = 0; + + len = min_t(uint32_t, len, 4); + memcpy(&a, str, len); + return key_mask_hash(a); +} + +/** + * ino_key_init - initialize inode key. + * @c: UBIFS file-system description object + * @key: key to initialize + * @inum: inode number + */ +static inline void ino_key_init(const struct ubifs_info *c, + union ubifs_key *key, ino_t inum) +{ + key->u32[0] = inum; + key->u32[1] = UBIFS_INO_KEY << UBIFS_S_KEY_BLOCK_BITS; +} + +/** + * ino_key_init_flash - initialize on-flash inode key. + * @c: UBIFS file-system description object + * @k: key to initialize + * @inum: inode number + */ +static inline void ino_key_init_flash(const struct ubifs_info *c, void *k, + ino_t inum) +{ + union ubifs_key *key = k; + + key->j32[0] = cpu_to_le32(inum); + key->j32[1] = cpu_to_le32(UBIFS_INO_KEY << UBIFS_S_KEY_BLOCK_BITS); + memset(k + 8, 0, UBIFS_MAX_KEY_LEN - 8); +} + +/** + * lowest_ino_key - get the lowest possible inode key. + * @c: UBIFS file-system description object + * @key: key to initialize + * @inum: inode number + */ +static inline void lowest_ino_key(const struct ubifs_info *c, + union ubifs_key *key, ino_t inum) +{ + key->u32[0] = inum; + key->u32[1] = 0; +} + +/** + * highest_ino_key - get the highest possible inode key. + * @c: UBIFS file-system description object + * @key: key to initialize + * @inum: inode number + */ +static inline void highest_ino_key(const struct ubifs_info *c, + union ubifs_key *key, ino_t inum) +{ + key->u32[0] = inum; + key->u32[1] = 0xffffffff; +} + +/** + * dent_key_init - initialize directory entry key. + * @c: UBIFS file-system description object + * @key: key to initialize + * @inum: parent inode number + * @nm: direntry name and length. Not a string when encrypted! + */ +static inline void dent_key_init(const struct ubifs_info *c, + union ubifs_key *key, ino_t inum, + const struct fscrypt_name *nm) +{ + uint32_t hash = c->key_hash(fname_name(nm), fname_len(nm)); + + ubifs_assert(c, !(hash & ~UBIFS_S_KEY_HASH_MASK)); + key->u32[0] = inum; + key->u32[1] = hash | (UBIFS_DENT_KEY << UBIFS_S_KEY_HASH_BITS); +} + +/** + * dent_key_init_hash - initialize directory entry key without re-calculating + * hash function. + * @c: UBIFS file-system description object + * @key: key to initialize + * @inum: parent inode number + * @hash: direntry name hash + */ +static inline void dent_key_init_hash(const struct ubifs_info *c, + union ubifs_key *key, ino_t inum, + uint32_t hash) +{ + ubifs_assert(c, !(hash & ~UBIFS_S_KEY_HASH_MASK)); + key->u32[0] = inum; + key->u32[1] = hash | (UBIFS_DENT_KEY << UBIFS_S_KEY_HASH_BITS); +} + +/** + * dent_key_init_flash - initialize on-flash directory entry key. + * @c: UBIFS file-system description object + * @k: key to initialize + * @inum: parent inode number + * @nm: direntry name and length + */ +static inline void dent_key_init_flash(const struct ubifs_info *c, void *k, + ino_t inum, + const struct fscrypt_name *nm) +{ + union ubifs_key *key = k; + uint32_t hash = c->key_hash(fname_name(nm), fname_len(nm)); + + ubifs_assert(c, !(hash & ~UBIFS_S_KEY_HASH_MASK)); + key->j32[0] = cpu_to_le32(inum); + key->j32[1] = cpu_to_le32(hash | + (UBIFS_DENT_KEY << UBIFS_S_KEY_HASH_BITS)); + memset(k + 8, 0, UBIFS_MAX_KEY_LEN - 8); +} + +/** + * lowest_dent_key - get the lowest possible directory entry key. + * @c: UBIFS file-system description object + * @key: where to store the lowest key + * @inum: parent inode number + */ +static inline void lowest_dent_key(const struct ubifs_info *c, + union ubifs_key *key, ino_t inum) +{ + key->u32[0] = inum; + key->u32[1] = UBIFS_DENT_KEY << UBIFS_S_KEY_HASH_BITS; +} + +/** + * xent_key_init - initialize extended attribute entry key. + * @c: UBIFS file-system description object + * @key: key to initialize + * @inum: host inode number + * @nm: extended attribute entry name and length + */ +static inline void xent_key_init(const struct ubifs_info *c, + union ubifs_key *key, ino_t inum, + const struct fscrypt_name *nm) +{ + uint32_t hash = c->key_hash(fname_name(nm), fname_len(nm)); + + ubifs_assert(c, !(hash & ~UBIFS_S_KEY_HASH_MASK)); + key->u32[0] = inum; + key->u32[1] = hash | (UBIFS_XENT_KEY << UBIFS_S_KEY_HASH_BITS); +} + +/** + * xent_key_init_flash - initialize on-flash extended attribute entry key. + * @c: UBIFS file-system description object + * @k: key to initialize + * @inum: host inode number + * @nm: extended attribute entry name and length + */ +static inline void xent_key_init_flash(const struct ubifs_info *c, void *k, + ino_t inum, const struct fscrypt_name *nm) +{ + union ubifs_key *key = k; + uint32_t hash = c->key_hash(fname_name(nm), fname_len(nm)); + + ubifs_assert(c, !(hash & ~UBIFS_S_KEY_HASH_MASK)); + key->j32[0] = cpu_to_le32(inum); + key->j32[1] = cpu_to_le32(hash | + (UBIFS_XENT_KEY << UBIFS_S_KEY_HASH_BITS)); + memset(k + 8, 0, UBIFS_MAX_KEY_LEN - 8); +} + +/** + * lowest_xent_key - get the lowest possible extended attribute entry key. + * @c: UBIFS file-system description object + * @key: where to store the lowest key + * @inum: host inode number + */ +static inline void lowest_xent_key(const struct ubifs_info *c, + union ubifs_key *key, ino_t inum) +{ + key->u32[0] = inum; + key->u32[1] = UBIFS_XENT_KEY << UBIFS_S_KEY_HASH_BITS; +} + +/** + * data_key_init - initialize data key. + * @c: UBIFS file-system description object + * @key: key to initialize + * @inum: inode number + * @block: block number + */ +static inline void data_key_init(const struct ubifs_info *c, + union ubifs_key *key, ino_t inum, + unsigned int block) +{ + ubifs_assert(c, !(block & ~UBIFS_S_KEY_BLOCK_MASK)); + key->u32[0] = inum; + key->u32[1] = block | (UBIFS_DATA_KEY << UBIFS_S_KEY_BLOCK_BITS); +} + +/** + * highest_data_key - get the highest possible data key for an inode. + * @c: UBIFS file-system description object + * @key: key to initialize + * @inum: inode number + */ +static inline void highest_data_key(const struct ubifs_info *c, + union ubifs_key *key, ino_t inum) +{ + data_key_init(c, key, inum, UBIFS_S_KEY_BLOCK_MASK); +} + +/** + * trun_key_init - initialize truncation node key. + * @c: UBIFS file-system description object + * @key: key to initialize + * @inum: inode number + * + * Note, UBIFS does not have truncation keys on the media and this function is + * only used for purposes of replay. + */ +static inline void trun_key_init(const struct ubifs_info *c, + union ubifs_key *key, ino_t inum) +{ + key->u32[0] = inum; + key->u32[1] = UBIFS_TRUN_KEY << UBIFS_S_KEY_BLOCK_BITS; +} + +/** + * invalid_key_init - initialize invalid node key. + * @c: UBIFS file-system description object + * @key: key to initialize + * + * This is a helper function which marks a @key object as invalid. + */ +static inline void invalid_key_init(const struct ubifs_info *c, + union ubifs_key *key) +{ + key->u32[0] = 0xDEADBEAF; + key->u32[1] = UBIFS_INVALID_KEY; +} + +/** + * key_type - get key type. + * @c: UBIFS file-system description object + * @key: key to get type of + */ +static inline int key_type(const struct ubifs_info *c, + const union ubifs_key *key) +{ + return key->u32[1] >> UBIFS_S_KEY_BLOCK_BITS; +} + +/** + * key_type_flash - get type of a on-flash formatted key. + * @c: UBIFS file-system description object + * @k: key to get type of + */ +static inline int key_type_flash(const struct ubifs_info *c, const void *k) +{ + const union ubifs_key *key = k; + + return le32_to_cpu(key->j32[1]) >> UBIFS_S_KEY_BLOCK_BITS; +} + +/** + * key_inum - fetch inode number from key. + * @c: UBIFS file-system description object + * @k: key to fetch inode number from + */ +static inline ino_t key_inum(const struct ubifs_info *c, const void *k) +{ + const union ubifs_key *key = k; + + return key->u32[0]; +} + +/** + * key_inum_flash - fetch inode number from an on-flash formatted key. + * @c: UBIFS file-system description object + * @k: key to fetch inode number from + */ +static inline ino_t key_inum_flash(const struct ubifs_info *c, const void *k) +{ + const union ubifs_key *key = k; + + return le32_to_cpu(key->j32[0]); +} + +/** + * key_hash - get directory entry hash. + * @c: UBIFS file-system description object + * @key: the key to get hash from + */ +static inline uint32_t key_hash(const struct ubifs_info *c, + const union ubifs_key *key) +{ + return key->u32[1] & UBIFS_S_KEY_HASH_MASK; +} + +/** + * key_hash_flash - get directory entry hash from an on-flash formatted key. + * @c: UBIFS file-system description object + * @k: the key to get hash from + */ +static inline uint32_t key_hash_flash(const struct ubifs_info *c, const void *k) +{ + const union ubifs_key *key = k; + + return le32_to_cpu(key->j32[1]) & UBIFS_S_KEY_HASH_MASK; +} + +/** + * key_block - get data block number. + * @c: UBIFS file-system description object + * @key: the key to get the block number from + */ +static inline unsigned int key_block(const struct ubifs_info *c, + const union ubifs_key *key) +{ + return key->u32[1] & UBIFS_S_KEY_BLOCK_MASK; +} + +/** + * key_block_flash - get data block number from an on-flash formatted key. + * @c: UBIFS file-system description object + * @k: the key to get the block number from + */ +static inline unsigned int key_block_flash(const struct ubifs_info *c, + const void *k) +{ + const union ubifs_key *key = k; + + return le32_to_cpu(key->j32[1]) & UBIFS_S_KEY_BLOCK_MASK; +} + +/** + * key_read - transform a key to in-memory format. + * @c: UBIFS file-system description object + * @from: the key to transform + * @to: the key to store the result + */ +static inline void key_read(const struct ubifs_info *c, const void *from, + union ubifs_key *to) +{ + const union ubifs_key *f = from; + + to->u32[0] = le32_to_cpu(f->j32[0]); + to->u32[1] = le32_to_cpu(f->j32[1]); +} + +/** + * key_write - transform a key from in-memory format. + * @c: UBIFS file-system description object + * @from: the key to transform + * @to: the key to store the result + */ +static inline void key_write(const struct ubifs_info *c, + const union ubifs_key *from, void *to) +{ + union ubifs_key *t = to; + + t->j32[0] = cpu_to_le32(from->u32[0]); + t->j32[1] = cpu_to_le32(from->u32[1]); + memset(to + 8, 0, UBIFS_MAX_KEY_LEN - 8); +} + +/** + * key_write_idx - transform a key from in-memory format for the index. + * @c: UBIFS file-system description object + * @from: the key to transform + * @to: the key to store the result + */ +static inline void key_write_idx(const struct ubifs_info *c, + const union ubifs_key *from, void *to) +{ + union ubifs_key *t = to; + + t->j32[0] = cpu_to_le32(from->u32[0]); + t->j32[1] = cpu_to_le32(from->u32[1]); +} + +/** + * key_copy - copy a key. + * @c: UBIFS file-system description object + * @from: the key to copy from + * @to: the key to copy to + */ +static inline void key_copy(const struct ubifs_info *c, + const union ubifs_key *from, union ubifs_key *to) +{ + to->u64[0] = from->u64[0]; +} + +/** + * keys_cmp - compare keys. + * @c: UBIFS file-system description object + * @key1: the first key to compare + * @key2: the second key to compare + * + * This function compares 2 keys and returns %-1 if @key1 is less than + * @key2, %0 if the keys are equivalent and %1 if @key1 is greater than @key2. + */ +static inline int keys_cmp(const struct ubifs_info *c, + const union ubifs_key *key1, + const union ubifs_key *key2) +{ + if (key1->u32[0] < key2->u32[0]) + return -1; + if (key1->u32[0] > key2->u32[0]) + return 1; + if (key1->u32[1] < key2->u32[1]) + return -1; + if (key1->u32[1] > key2->u32[1]) + return 1; + + return 0; +} + +/** + * keys_eq - determine if keys are equivalent. + * @c: UBIFS file-system description object + * @key1: the first key to compare + * @key2: the second key to compare + * + * This function compares 2 keys and returns %1 if @key1 is equal to @key2 and + * %0 if not. + */ +static inline int keys_eq(const struct ubifs_info *c, + const union ubifs_key *key1, + const union ubifs_key *key2) +{ + if (key1->u32[0] != key2->u32[0]) + return 0; + if (key1->u32[1] != key2->u32[1]) + return 0; + return 1; +} + +/** + * is_hash_key - is a key vulnerable to hash collisions. + * @c: UBIFS file-system description object + * @key: key + * + * This function returns %1 if @key is a hashed key or %0 otherwise. + */ +static inline int is_hash_key(const struct ubifs_info *c, + const union ubifs_key *key) +{ + int type = key_type(c, key); + + return type == UBIFS_DENT_KEY || type == UBIFS_XENT_KEY; +} + +/** + * key_max_inode_size - get maximum file size allowed by current key format. + * @c: UBIFS file-system description object + */ +static inline unsigned long long key_max_inode_size(const struct ubifs_info *c) +{ + switch (c->key_fmt) { + case UBIFS_SIMPLE_KEY_FMT: + return (1ULL << UBIFS_S_KEY_BLOCK_BITS) * UBIFS_BLOCK_SIZE; + default: + return 0; + } +} + +#endif /* !__UBIFS_KEY_H__ */ diff --git a/fs/ubifs/log.c b/fs/ubifs/log.c new file mode 100644 index 000000000..c6edcbf35 --- /dev/null +++ b/fs/ubifs/log.c @@ -0,0 +1,762 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * + * Authors: Artem Bityutskiy (Битюцкий Артём) + * Adrian Hunter + */ + +/* + * This file is a part of UBIFS journal implementation and contains various + * functions which manipulate the log. The log is a fixed area on the flash + * which does not contain any data but refers to buds. The log is a part of the + * journal. + */ + +#include "ubifs.h" + +static int dbg_check_bud_bytes(struct ubifs_info *c); + +/** + * ubifs_search_bud - search bud LEB. + * @c: UBIFS file-system description object + * @lnum: logical eraseblock number to search + * + * This function searches bud LEB @lnum. Returns bud description object in case + * of success and %NULL if there is no bud with this LEB number. + */ +struct ubifs_bud *ubifs_search_bud(struct ubifs_info *c, int lnum) +{ + struct rb_node *p; + struct ubifs_bud *bud; + + spin_lock(&c->buds_lock); + p = c->buds.rb_node; + while (p) { + bud = rb_entry(p, struct ubifs_bud, rb); + if (lnum < bud->lnum) + p = p->rb_left; + else if (lnum > bud->lnum) + p = p->rb_right; + else { + spin_unlock(&c->buds_lock); + return bud; + } + } + spin_unlock(&c->buds_lock); + return NULL; +} + +/** + * ubifs_get_wbuf - get the wbuf associated with a LEB, if there is one. + * @c: UBIFS file-system description object + * @lnum: logical eraseblock number to search + * + * This functions returns the wbuf for @lnum or %NULL if there is not one. + */ +struct ubifs_wbuf *ubifs_get_wbuf(struct ubifs_info *c, int lnum) +{ + struct rb_node *p; + struct ubifs_bud *bud; + int jhead; + + if (!c->jheads) + return NULL; + + spin_lock(&c->buds_lock); + p = c->buds.rb_node; + while (p) { + bud = rb_entry(p, struct ubifs_bud, rb); + if (lnum < bud->lnum) + p = p->rb_left; + else if (lnum > bud->lnum) + p = p->rb_right; + else { + jhead = bud->jhead; + spin_unlock(&c->buds_lock); + return &c->jheads[jhead].wbuf; + } + } + spin_unlock(&c->buds_lock); + return NULL; +} + +/** + * empty_log_bytes - calculate amount of empty space in the log. + * @c: UBIFS file-system description object + */ +static inline long long empty_log_bytes(const struct ubifs_info *c) +{ + long long h, t; + + h = (long long)c->lhead_lnum * c->leb_size + c->lhead_offs; + t = (long long)c->ltail_lnum * c->leb_size; + + if (h > t) + return c->log_bytes - h + t; + else if (h != t) + return t - h; + else if (c->lhead_lnum != c->ltail_lnum) + return 0; + else + return c->log_bytes; +} + +/** + * ubifs_add_bud - add bud LEB to the tree of buds and its journal head list. + * @c: UBIFS file-system description object + * @bud: the bud to add + */ +void ubifs_add_bud(struct ubifs_info *c, struct ubifs_bud *bud) +{ + struct rb_node **p, *parent = NULL; + struct ubifs_bud *b; + struct ubifs_jhead *jhead; + + spin_lock(&c->buds_lock); + p = &c->buds.rb_node; + while (*p) { + parent = *p; + b = rb_entry(parent, struct ubifs_bud, rb); + ubifs_assert(c, bud->lnum != b->lnum); + if (bud->lnum < b->lnum) + p = &(*p)->rb_left; + else + p = &(*p)->rb_right; + } + + rb_link_node(&bud->rb, parent, p); + rb_insert_color(&bud->rb, &c->buds); + if (c->jheads) { + jhead = &c->jheads[bud->jhead]; + list_add_tail(&bud->list, &jhead->buds_list); + } else + ubifs_assert(c, c->replaying && c->ro_mount); + + /* + * Note, although this is a new bud, we anyway account this space now, + * before any data has been written to it, because this is about to + * guarantee fixed mount time, and this bud will anyway be read and + * scanned. + */ + c->bud_bytes += c->leb_size - bud->start; + + dbg_log("LEB %d:%d, jhead %s, bud_bytes %lld", bud->lnum, + bud->start, dbg_jhead(bud->jhead), c->bud_bytes); + spin_unlock(&c->buds_lock); +} + +/** + * ubifs_add_bud_to_log - add a new bud to the log. + * @c: UBIFS file-system description object + * @jhead: journal head the bud belongs to + * @lnum: LEB number of the bud + * @offs: starting offset of the bud + * + * This function writes a reference node for the new bud LEB @lnum to the log, + * and adds it to the buds trees. It also makes sure that log size does not + * exceed the 'c->max_bud_bytes' limit. Returns zero in case of success, + * %-EAGAIN if commit is required, and a negative error code in case of + * failure. + */ +int ubifs_add_bud_to_log(struct ubifs_info *c, int jhead, int lnum, int offs) +{ + int err; + struct ubifs_bud *bud; + struct ubifs_ref_node *ref; + + bud = kmalloc_obj(struct ubifs_bud, GFP_NOFS); + if (!bud) + return -ENOMEM; + ref = kzalloc(c->ref_node_alsz, GFP_NOFS); + if (!ref) { + kfree(bud); + return -ENOMEM; + } + + mutex_lock(&c->log_mutex); + ubifs_assert(c, !c->ro_media && !c->ro_mount); + if (c->ro_error) { + err = -EROFS; + goto out_unlock; + } + + /* Make sure we have enough space in the log */ + if (empty_log_bytes(c) - c->ref_node_alsz < c->min_log_bytes) { + dbg_log("not enough log space - %lld, required %d", + empty_log_bytes(c), c->min_log_bytes); + ubifs_commit_required(c); + err = -EAGAIN; + goto out_unlock; + } + + /* + * Make sure the amount of space in buds will not exceed the + * 'c->max_bud_bytes' limit, because we want to guarantee mount time + * limits. + * + * It is not necessary to hold @c->buds_lock when reading @c->bud_bytes + * because we are holding @c->log_mutex. All @c->bud_bytes take place + * when both @c->log_mutex and @c->bud_bytes are locked. + */ + if (c->bud_bytes + c->leb_size - offs > c->max_bud_bytes) { + dbg_log("bud bytes %lld (%lld max), require commit", + c->bud_bytes, c->max_bud_bytes); + ubifs_commit_required(c); + err = -EAGAIN; + goto out_unlock; + } + + /* + * If the journal is full enough - start background commit. Note, it is + * OK to read 'c->cmt_state' without spinlock because integer reads + * are atomic in the kernel. + */ + if (c->bud_bytes >= c->bg_bud_bytes && + c->cmt_state == COMMIT_RESTING) { + dbg_log("bud bytes %lld (%lld max), initiate BG commit", + c->bud_bytes, c->max_bud_bytes); + ubifs_request_bg_commit(c); + } + + bud->lnum = lnum; + bud->start = offs; + bud->jhead = jhead; + bud->log_hash = NULL; + + ref->ch.node_type = UBIFS_REF_NODE; + ref->lnum = cpu_to_le32(bud->lnum); + ref->offs = cpu_to_le32(bud->start); + ref->jhead = cpu_to_le32(jhead); + + if (c->lhead_offs > c->leb_size - c->ref_node_alsz) { + c->lhead_lnum = ubifs_next_log_lnum(c, c->lhead_lnum); + ubifs_assert(c, c->lhead_lnum != c->ltail_lnum); + c->lhead_offs = 0; + } + + if (c->lhead_offs == 0) { + /* Must ensure next log LEB has been unmapped */ + err = ubifs_leb_unmap(c, c->lhead_lnum); + if (err) + goto out_unlock; + } + + if (bud->start == 0) { + /* + * Before writing the LEB reference which refers an empty LEB + * to the log, we have to make sure it is mapped, because + * otherwise we'd risk to refer an LEB with garbage in case of + * an unclean reboot, because the target LEB might have been + * unmapped, but not yet physically erased. + */ + err = ubifs_leb_map(c, bud->lnum); + if (err) + goto out_unlock; + } + + dbg_log("write ref LEB %d:%d", + c->lhead_lnum, c->lhead_offs); + err = ubifs_write_node(c, ref, UBIFS_REF_NODE_SZ, c->lhead_lnum, + c->lhead_offs); + if (err) + goto out_unlock; + + err = ubifs_shash_update(c, c->log_hash, ref, UBIFS_REF_NODE_SZ); + if (err) + goto out_unlock; + + err = ubifs_shash_copy_state(c, c->log_hash, c->jheads[jhead].log_hash); + if (err) + goto out_unlock; + + c->lhead_offs += c->ref_node_alsz; + + ubifs_add_bud(c, bud); + + mutex_unlock(&c->log_mutex); + kfree(ref); + return 0; + +out_unlock: + mutex_unlock(&c->log_mutex); + kfree(ref); + kfree(bud); + return err; +} + +/** + * remove_buds - remove used buds. + * @c: UBIFS file-system description object + * + * This function removes use buds from the buds tree. It does not remove the + * buds which are pointed to by journal heads. + */ +static void remove_buds(struct ubifs_info *c) +{ + struct rb_node *p; + + ubifs_assert(c, list_empty(&c->old_buds)); + c->cmt_bud_bytes = 0; + spin_lock(&c->buds_lock); + p = rb_first(&c->buds); + while (p) { + struct rb_node *p1 = p; + struct ubifs_bud *bud; + struct ubifs_wbuf *wbuf; + + p = rb_next(p); + bud = rb_entry(p1, struct ubifs_bud, rb); + wbuf = &c->jheads[bud->jhead].wbuf; + + if (wbuf->lnum == bud->lnum) { + /* + * Do not remove buds which are pointed to by journal + * heads (non-closed buds). + */ + c->cmt_bud_bytes += wbuf->offs - bud->start; + dbg_log("preserve %d:%d, jhead %s, bud bytes %d, cmt_bud_bytes %lld", + bud->lnum, bud->start, dbg_jhead(bud->jhead), + wbuf->offs - bud->start, c->cmt_bud_bytes); + bud->start = wbuf->offs; + } else { + c->cmt_bud_bytes += c->leb_size - bud->start; + dbg_log("remove %d:%d, jhead %s, bud bytes %d, cmt_bud_bytes %lld", + bud->lnum, bud->start, dbg_jhead(bud->jhead), + c->leb_size - bud->start, c->cmt_bud_bytes); + rb_erase(p1, &c->buds); + /* + * If the commit does not finish, the recovery will need + * to replay the journal, in which case the old buds + * must be unchanged. Do not release them until post + * commit i.e. do not allow them to be garbage + * collected. + */ + list_move(&bud->list, &c->old_buds); + } + } + spin_unlock(&c->buds_lock); +} + +/** + * ubifs_log_start_commit - start commit. + * @c: UBIFS file-system description object + * @ltail_lnum: return new log tail LEB number + * + * The commit operation starts with writing "commit start" node to the log and + * reference nodes for all journal heads which will define new journal after + * the commit has been finished. The commit start and reference nodes are + * written in one go to the nearest empty log LEB (hence, when commit is + * finished UBIFS may safely unmap all the previous log LEBs). This function + * returns zero in case of success and a negative error code in case of + * failure. + */ +int ubifs_log_start_commit(struct ubifs_info *c, int *ltail_lnum) +{ + void *buf; + struct ubifs_cs_node *cs; + struct ubifs_ref_node *ref; + int err, i, max_len, len; + + err = dbg_check_bud_bytes(c); + if (err) + return err; + + max_len = UBIFS_CS_NODE_SZ + c->jhead_cnt * UBIFS_REF_NODE_SZ; + max_len = ALIGN(max_len, c->min_io_size); + buf = cs = kmalloc(max_len, GFP_NOFS); + if (!buf) + return -ENOMEM; + + cs->ch.node_type = UBIFS_CS_NODE; + cs->cmt_no = cpu_to_le64(c->cmt_no); + ubifs_prepare_node(c, cs, UBIFS_CS_NODE_SZ, 0); + + err = ubifs_shash_init(c, c->log_hash); + if (err) + goto out; + + err = ubifs_shash_update(c, c->log_hash, cs, UBIFS_CS_NODE_SZ); + if (err < 0) + goto out; + + /* + * Note, we do not lock 'c->log_mutex' because this is the commit start + * phase and we are exclusively using the log. And we do not lock + * write-buffer because nobody can write to the file-system at this + * phase. + */ + + len = UBIFS_CS_NODE_SZ; + for (i = 0; i < c->jhead_cnt; i++) { + int lnum = c->jheads[i].wbuf.lnum; + int offs = c->jheads[i].wbuf.offs; + + if (lnum == -1 || offs == c->leb_size) + continue; + + dbg_log("add ref to LEB %d:%d for jhead %s", + lnum, offs, dbg_jhead(i)); + ref = buf + len; + ref->ch.node_type = UBIFS_REF_NODE; + ref->lnum = cpu_to_le32(lnum); + ref->offs = cpu_to_le32(offs); + ref->jhead = cpu_to_le32(i); + + ubifs_prepare_node(c, ref, UBIFS_REF_NODE_SZ, 0); + len += UBIFS_REF_NODE_SZ; + + err = ubifs_shash_update(c, c->log_hash, ref, + UBIFS_REF_NODE_SZ); + if (err) + goto out; + ubifs_shash_copy_state(c, c->log_hash, c->jheads[i].log_hash); + } + + ubifs_pad(c, buf + len, ALIGN(len, c->min_io_size) - len); + + /* Switch to the next log LEB */ + if (c->lhead_offs) { + c->lhead_lnum = ubifs_next_log_lnum(c, c->lhead_lnum); + ubifs_assert(c, c->lhead_lnum != c->ltail_lnum); + c->lhead_offs = 0; + } + + /* Must ensure next LEB has been unmapped */ + err = ubifs_leb_unmap(c, c->lhead_lnum); + if (err) + goto out; + + len = ALIGN(len, c->min_io_size); + dbg_log("writing commit start at LEB %d:0, len %d", c->lhead_lnum, len); + err = ubifs_leb_write(c, c->lhead_lnum, cs, 0, len); + if (err) + goto out; + + *ltail_lnum = c->lhead_lnum; + + c->lhead_offs += len; + ubifs_assert(c, c->lhead_offs < c->leb_size); + + remove_buds(c); + + /* + * We have started the commit and now users may use the rest of the log + * for new writes. + */ + c->min_log_bytes = 0; + +out: + kfree(buf); + return err; +} + +/** + * ubifs_log_end_commit - end commit. + * @c: UBIFS file-system description object + * @ltail_lnum: new log tail LEB number + * + * This function is called on when the commit operation was finished. It + * moves log tail to new position and updates the master node so that it stores + * the new log tail LEB number. Returns zero in case of success and a negative + * error code in case of failure. + */ +int ubifs_log_end_commit(struct ubifs_info *c, int ltail_lnum) +{ + int err; + + /* + * At this phase we have to lock 'c->log_mutex' because UBIFS allows FS + * writes during commit. Its only short "commit" start phase when + * writers are blocked. + */ + mutex_lock(&c->log_mutex); + + dbg_log("old tail was LEB %d:0, new tail is LEB %d:0", + c->ltail_lnum, ltail_lnum); + + c->ltail_lnum = ltail_lnum; + /* + * The commit is finished and from now on it must be guaranteed that + * there is always enough space for the next commit. + */ + c->min_log_bytes = c->leb_size; + + spin_lock(&c->buds_lock); + c->bud_bytes -= c->cmt_bud_bytes; + spin_unlock(&c->buds_lock); + + err = dbg_check_bud_bytes(c); + if (err) + goto out; + + err = ubifs_write_master(c); + +out: + mutex_unlock(&c->log_mutex); + return err; +} + +/** + * ubifs_log_post_commit - things to do after commit is completed. + * @c: UBIFS file-system description object + * @old_ltail_lnum: old log tail LEB number + * + * Release buds only after commit is completed, because they must be unchanged + * if recovery is needed. + * + * Unmap log LEBs only after commit is completed, because they may be needed for + * recovery. + * + * This function returns %0 on success and a negative error code on failure. + */ +int ubifs_log_post_commit(struct ubifs_info *c, int old_ltail_lnum) +{ + int lnum, err = 0; + + while (!list_empty(&c->old_buds)) { + struct ubifs_bud *bud; + + bud = list_entry(c->old_buds.next, struct ubifs_bud, list); + err = ubifs_return_leb(c, bud->lnum); + if (err) + return err; + list_del(&bud->list); + kfree(bud->log_hash); + kfree(bud); + } + mutex_lock(&c->log_mutex); + for (lnum = old_ltail_lnum; lnum != c->ltail_lnum; + lnum = ubifs_next_log_lnum(c, lnum)) { + dbg_log("unmap log LEB %d", lnum); + err = ubifs_leb_unmap(c, lnum); + if (err) + goto out; + } +out: + mutex_unlock(&c->log_mutex); + return err; +} + +/** + * struct done_ref - references that have been done. + * @rb: rb-tree node + * @lnum: LEB number + */ +struct done_ref { + struct rb_node rb; + int lnum; +}; + +/** + * done_already - determine if a reference has been done already. + * @done_tree: rb-tree to store references that have been done + * @lnum: LEB number of reference + * + * This function returns %1 if the reference has been done, %0 if not, otherwise + * a negative error code is returned. + */ +static int done_already(struct rb_root *done_tree, int lnum) +{ + struct rb_node **p = &done_tree->rb_node, *parent = NULL; + struct done_ref *dr; + + while (*p) { + parent = *p; + dr = rb_entry(parent, struct done_ref, rb); + if (lnum < dr->lnum) + p = &(*p)->rb_left; + else if (lnum > dr->lnum) + p = &(*p)->rb_right; + else + return 1; + } + + dr = kzalloc_obj(struct done_ref, GFP_NOFS); + if (!dr) + return -ENOMEM; + + dr->lnum = lnum; + + rb_link_node(&dr->rb, parent, p); + rb_insert_color(&dr->rb, done_tree); + + return 0; +} + +/** + * destroy_done_tree - destroy the done tree. + * @done_tree: done tree to destroy + */ +static void destroy_done_tree(struct rb_root *done_tree) +{ + struct done_ref *dr, *n; + + rbtree_postorder_for_each_entry_safe(dr, n, done_tree, rb) + kfree(dr); +} + +/** + * add_node - add a node to the consolidated log. + * @c: UBIFS file-system description object + * @buf: buffer to which to add + * @lnum: LEB number to which to write is passed and returned here + * @offs: offset to where to write is passed and returned here + * @node: node to add + * + * This function returns %0 on success and a negative error code on failure. + */ +static int add_node(struct ubifs_info *c, void *buf, int *lnum, int *offs, + void *node) +{ + struct ubifs_ch *ch = node; + int len = le32_to_cpu(ch->len), remains = c->leb_size - *offs; + + if (len > remains) { + int sz = ALIGN(*offs, c->min_io_size), err; + + ubifs_pad(c, buf + *offs, sz - *offs); + err = ubifs_leb_change(c, *lnum, buf, sz); + if (err) + return err; + *lnum = ubifs_next_log_lnum(c, *lnum); + *offs = 0; + } + memcpy(buf + *offs, node, len); + *offs += ALIGN(len, 8); + return 0; +} + +/** + * ubifs_consolidate_log - consolidate the log. + * @c: UBIFS file-system description object + * + * Repeated failed commits could cause the log to be full, but at least 1 LEB is + * needed for commit. This function rewrites the reference nodes in the log + * omitting duplicates, and failed CS nodes, and leaving no gaps. + * + * This function returns %0 on success and a negative error code on failure. + */ +int ubifs_consolidate_log(struct ubifs_info *c) +{ + struct ubifs_scan_leb *sleb; + struct ubifs_scan_node *snod; + struct rb_root done_tree = RB_ROOT; + int lnum, err, first = 1, write_lnum, offs = 0; + void *buf; + + dbg_rcvry("log tail LEB %d, log head LEB %d", c->ltail_lnum, + c->lhead_lnum); + buf = vmalloc(c->leb_size); + if (!buf) + return -ENOMEM; + lnum = c->ltail_lnum; + write_lnum = lnum; + while (1) { + sleb = ubifs_scan(c, lnum, 0, c->sbuf, 0); + if (IS_ERR(sleb)) { + err = PTR_ERR(sleb); + goto out_free; + } + list_for_each_entry(snod, &sleb->nodes, list) { + switch (snod->type) { + case UBIFS_REF_NODE: { + struct ubifs_ref_node *ref = snod->node; + int ref_lnum = le32_to_cpu(ref->lnum); + + err = done_already(&done_tree, ref_lnum); + if (err < 0) + goto out_scan; + if (err != 1) { + err = add_node(c, buf, &write_lnum, + &offs, snod->node); + if (err) + goto out_scan; + } + break; + } + case UBIFS_CS_NODE: + if (!first) + break; + err = add_node(c, buf, &write_lnum, &offs, + snod->node); + if (err) + goto out_scan; + first = 0; + break; + } + } + ubifs_scan_destroy(sleb); + if (lnum == c->lhead_lnum) + break; + lnum = ubifs_next_log_lnum(c, lnum); + } + if (offs) { + int sz = ALIGN(offs, c->min_io_size); + + ubifs_pad(c, buf + offs, sz - offs); + err = ubifs_leb_change(c, write_lnum, buf, sz); + if (err) + goto out_free; + offs = ALIGN(offs, c->min_io_size); + } + destroy_done_tree(&done_tree); + vfree(buf); + if (write_lnum == c->lhead_lnum) { + ubifs_err(c, "log is too full"); + return -EINVAL; + } + /* Unmap remaining LEBs */ + lnum = write_lnum; + do { + lnum = ubifs_next_log_lnum(c, lnum); + err = ubifs_leb_unmap(c, lnum); + if (err) + return err; + } while (lnum != c->lhead_lnum); + c->lhead_lnum = write_lnum; + c->lhead_offs = offs; + dbg_rcvry("new log head at %d:%d", c->lhead_lnum, c->lhead_offs); + return 0; + +out_scan: + ubifs_scan_destroy(sleb); +out_free: + destroy_done_tree(&done_tree); + vfree(buf); + return err; +} + +/** + * dbg_check_bud_bytes - make sure bud bytes calculation are all right. + * @c: UBIFS file-system description object + * + * This function makes sure the amount of flash space used by closed buds + * ('c->bud_bytes' is correct). Returns zero in case of success and %-EINVAL in + * case of failure. + */ +static int dbg_check_bud_bytes(struct ubifs_info *c) +{ + int i, err = 0; + struct ubifs_bud *bud; + long long bud_bytes = 0; + + if (!dbg_is_chk_gen(c)) + return 0; + + spin_lock(&c->buds_lock); + for (i = 0; i < c->jhead_cnt; i++) + list_for_each_entry(bud, &c->jheads[i].buds_list, list) + bud_bytes += c->leb_size - bud->start; + + if (c->bud_bytes != bud_bytes) { + ubifs_err(c, "bad bud_bytes %lld, calculated %lld", + c->bud_bytes, bud_bytes); + err = -EINVAL; + } + spin_unlock(&c->buds_lock); + + return err; +} diff --git a/fs/ubifs/lprops.c b/fs/ubifs/lprops.c new file mode 100644 index 000000000..8788740ec --- /dev/null +++ b/fs/ubifs/lprops.c @@ -0,0 +1,1307 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * + * Authors: Adrian Hunter + * Artem Bityutskiy (Битюцкий Артём) + */ + +/* + * This file implements the functions that access LEB properties and their + * categories. LEBs are categorized based on the needs of UBIFS, and the + * categories are stored as either heaps or lists to provide a fast way of + * finding a LEB in a particular category. For example, UBIFS may need to find + * an empty LEB for the journal, or a very dirty LEB for garbage collection. + */ + +#include "ubifs.h" + +/** + * get_heap_comp_val - get the LEB properties value for heap comparisons. + * @lprops: LEB properties + * @cat: LEB category + */ +static int get_heap_comp_val(struct ubifs_lprops *lprops, int cat) +{ + switch (cat) { + case LPROPS_FREE: + return lprops->free; + case LPROPS_DIRTY_IDX: + return lprops->free + lprops->dirty; + default: + return lprops->dirty; + } +} + +/** + * move_up_lpt_heap - move a new heap entry up as far as possible. + * @c: UBIFS file-system description object + * @heap: LEB category heap + * @lprops: LEB properties to move + * @cat: LEB category + * + * New entries to a heap are added at the bottom and then moved up until the + * parent's value is greater. In the case of LPT's category heaps, the value + * is either the amount of free space or the amount of dirty space, depending + * on the category. + */ +static void move_up_lpt_heap(struct ubifs_info *c, struct ubifs_lpt_heap *heap, + struct ubifs_lprops *lprops, int cat) +{ + int val1, val2, hpos; + + hpos = lprops->hpos; + if (!hpos) + return; /* Already top of the heap */ + val1 = get_heap_comp_val(lprops, cat); + /* Compare to parent and, if greater, move up the heap */ + do { + int ppos = (hpos - 1) / 2; + + val2 = get_heap_comp_val(heap->arr[ppos], cat); + if (val2 >= val1) + return; + /* Greater than parent so move up */ + heap->arr[ppos]->hpos = hpos; + heap->arr[hpos] = heap->arr[ppos]; + heap->arr[ppos] = lprops; + lprops->hpos = ppos; + hpos = ppos; + } while (hpos); +} + +/** + * adjust_lpt_heap - move a changed heap entry up or down the heap. + * @c: UBIFS file-system description object + * @heap: LEB category heap + * @lprops: LEB properties to move + * @hpos: heap position of @lprops + * @cat: LEB category + * + * Changed entries in a heap are moved up or down until the parent's value is + * greater. In the case of LPT's category heaps, the value is either the amount + * of free space or the amount of dirty space, depending on the category. + */ +static void adjust_lpt_heap(struct ubifs_info *c, struct ubifs_lpt_heap *heap, + struct ubifs_lprops *lprops, int hpos, int cat) +{ + int val1, val2, val3, cpos; + + val1 = get_heap_comp_val(lprops, cat); + /* Compare to parent and, if greater than parent, move up the heap */ + if (hpos) { + int ppos = (hpos - 1) / 2; + + val2 = get_heap_comp_val(heap->arr[ppos], cat); + if (val1 > val2) { + /* Greater than parent so move up */ + while (1) { + heap->arr[ppos]->hpos = hpos; + heap->arr[hpos] = heap->arr[ppos]; + heap->arr[ppos] = lprops; + lprops->hpos = ppos; + hpos = ppos; + if (!hpos) + return; + ppos = (hpos - 1) / 2; + val2 = get_heap_comp_val(heap->arr[ppos], cat); + if (val1 <= val2) + return; + /* Still greater than parent so keep going */ + } + } + } + + /* Not greater than parent, so compare to children */ + while (1) { + /* Compare to left child */ + cpos = hpos * 2 + 1; + if (cpos >= heap->cnt) + return; + val2 = get_heap_comp_val(heap->arr[cpos], cat); + if (val1 < val2) { + /* Less than left child, so promote biggest child */ + if (cpos + 1 < heap->cnt) { + val3 = get_heap_comp_val(heap->arr[cpos + 1], + cat); + if (val3 > val2) + cpos += 1; /* Right child is bigger */ + } + heap->arr[cpos]->hpos = hpos; + heap->arr[hpos] = heap->arr[cpos]; + heap->arr[cpos] = lprops; + lprops->hpos = cpos; + hpos = cpos; + continue; + } + /* Compare to right child */ + cpos += 1; + if (cpos >= heap->cnt) + return; + val3 = get_heap_comp_val(heap->arr[cpos], cat); + if (val1 < val3) { + /* Less than right child, so promote right child */ + heap->arr[cpos]->hpos = hpos; + heap->arr[hpos] = heap->arr[cpos]; + heap->arr[cpos] = lprops; + lprops->hpos = cpos; + hpos = cpos; + continue; + } + return; + } +} + +/** + * add_to_lpt_heap - add LEB properties to a LEB category heap. + * @c: UBIFS file-system description object + * @lprops: LEB properties to add + * @cat: LEB category + * + * This function returns %1 if @lprops is added to the heap for LEB category + * @cat, otherwise %0 is returned because the heap is full. + */ +static int add_to_lpt_heap(struct ubifs_info *c, struct ubifs_lprops *lprops, + int cat) +{ + struct ubifs_lpt_heap *heap = &c->lpt_heap[cat - 1]; + + if (heap->cnt >= heap->max_cnt) { + const int b = LPT_HEAP_SZ / 2 - 1; + int cpos, val1, val2; + + /* Compare to some other LEB on the bottom of heap */ + /* Pick a position kind of randomly */ + cpos = (((size_t)lprops >> 4) & b) + b; + ubifs_assert(c, cpos >= b); + ubifs_assert(c, cpos < LPT_HEAP_SZ); + ubifs_assert(c, cpos < heap->cnt); + + val1 = get_heap_comp_val(lprops, cat); + val2 = get_heap_comp_val(heap->arr[cpos], cat); + if (val1 > val2) { + struct ubifs_lprops *lp; + + lp = heap->arr[cpos]; + lp->flags &= ~LPROPS_CAT_MASK; + lp->flags |= LPROPS_UNCAT; + list_add(&lp->list, &c->uncat_list); + lprops->hpos = cpos; + heap->arr[cpos] = lprops; + move_up_lpt_heap(c, heap, lprops, cat); + dbg_check_heap(c, heap, cat, lprops->hpos); + return 1; /* Added to heap */ + } + dbg_check_heap(c, heap, cat, -1); + return 0; /* Not added to heap */ + } else { + lprops->hpos = heap->cnt++; + heap->arr[lprops->hpos] = lprops; + move_up_lpt_heap(c, heap, lprops, cat); + dbg_check_heap(c, heap, cat, lprops->hpos); + return 1; /* Added to heap */ + } +} + +/** + * remove_from_lpt_heap - remove LEB properties from a LEB category heap. + * @c: UBIFS file-system description object + * @lprops: LEB properties to remove + * @cat: LEB category + */ +static void remove_from_lpt_heap(struct ubifs_info *c, + struct ubifs_lprops *lprops, int cat) +{ + struct ubifs_lpt_heap *heap; + int hpos = lprops->hpos; + + heap = &c->lpt_heap[cat - 1]; + ubifs_assert(c, hpos >= 0 && hpos < heap->cnt); + ubifs_assert(c, heap->arr[hpos] == lprops); + heap->cnt -= 1; + if (hpos < heap->cnt) { + heap->arr[hpos] = heap->arr[heap->cnt]; + heap->arr[hpos]->hpos = hpos; + adjust_lpt_heap(c, heap, heap->arr[hpos], hpos, cat); + } + dbg_check_heap(c, heap, cat, -1); +} + +/** + * lpt_heap_replace - replace lprops in a category heap. + * @c: UBIFS file-system description object + * @new_lprops: LEB properties with which to replace + * @cat: LEB category + * + * During commit it is sometimes necessary to copy a pnode (see dirty_cow_pnode) + * and the lprops that the pnode contains. When that happens, references in + * the category heaps to those lprops must be updated to point to the new + * lprops. This function does that. + */ +static void lpt_heap_replace(struct ubifs_info *c, + struct ubifs_lprops *new_lprops, int cat) +{ + struct ubifs_lpt_heap *heap; + int hpos = new_lprops->hpos; + + heap = &c->lpt_heap[cat - 1]; + heap->arr[hpos] = new_lprops; +} + +/** + * ubifs_add_to_cat - add LEB properties to a category list or heap. + * @c: UBIFS file-system description object + * @lprops: LEB properties to add + * @cat: LEB category to which to add + * + * LEB properties are categorized to enable fast find operations. + */ +void ubifs_add_to_cat(struct ubifs_info *c, struct ubifs_lprops *lprops, + int cat) +{ + switch (cat) { + case LPROPS_DIRTY: + case LPROPS_DIRTY_IDX: + case LPROPS_FREE: + if (add_to_lpt_heap(c, lprops, cat)) + break; + /* No more room on heap so make it un-categorized */ + cat = LPROPS_UNCAT; + fallthrough; + case LPROPS_UNCAT: + list_add(&lprops->list, &c->uncat_list); + break; + case LPROPS_EMPTY: + list_add(&lprops->list, &c->empty_list); + break; + case LPROPS_FREEABLE: + list_add(&lprops->list, &c->freeable_list); + c->freeable_cnt += 1; + break; + case LPROPS_FRDI_IDX: + list_add(&lprops->list, &c->frdi_idx_list); + break; + default: + ubifs_assert(c, 0); + } + + lprops->flags &= ~LPROPS_CAT_MASK; + lprops->flags |= cat; + c->in_a_category_cnt += 1; + ubifs_assert(c, c->in_a_category_cnt <= c->main_lebs); +} + +/** + * ubifs_remove_from_cat - remove LEB properties from a category list or heap. + * @c: UBIFS file-system description object + * @lprops: LEB properties to remove + * @cat: LEB category from which to remove + * + * LEB properties are categorized to enable fast find operations. + */ +static void ubifs_remove_from_cat(struct ubifs_info *c, + struct ubifs_lprops *lprops, int cat) +{ + switch (cat) { + case LPROPS_DIRTY: + case LPROPS_DIRTY_IDX: + case LPROPS_FREE: + remove_from_lpt_heap(c, lprops, cat); + break; + case LPROPS_FREEABLE: + c->freeable_cnt -= 1; + ubifs_assert(c, c->freeable_cnt >= 0); + fallthrough; + case LPROPS_UNCAT: + case LPROPS_EMPTY: + case LPROPS_FRDI_IDX: + ubifs_assert(c, !list_empty(&lprops->list)); + list_del(&lprops->list); + break; + default: + ubifs_assert(c, 0); + } + + c->in_a_category_cnt -= 1; + ubifs_assert(c, c->in_a_category_cnt >= 0); +} + +/** + * ubifs_replace_cat - replace lprops in a category list or heap. + * @c: UBIFS file-system description object + * @old_lprops: LEB properties to replace + * @new_lprops: LEB properties with which to replace + * + * During commit it is sometimes necessary to copy a pnode (see dirty_cow_pnode) + * and the lprops that the pnode contains. When that happens, references in + * category lists and heaps must be replaced. This function does that. + */ +void ubifs_replace_cat(struct ubifs_info *c, struct ubifs_lprops *old_lprops, + struct ubifs_lprops *new_lprops) +{ + int cat; + + cat = new_lprops->flags & LPROPS_CAT_MASK; + switch (cat) { + case LPROPS_DIRTY: + case LPROPS_DIRTY_IDX: + case LPROPS_FREE: + lpt_heap_replace(c, new_lprops, cat); + break; + case LPROPS_UNCAT: + case LPROPS_EMPTY: + case LPROPS_FREEABLE: + case LPROPS_FRDI_IDX: + list_replace(&old_lprops->list, &new_lprops->list); + break; + default: + ubifs_assert(c, 0); + } +} + +/** + * ubifs_ensure_cat - ensure LEB properties are categorized. + * @c: UBIFS file-system description object + * @lprops: LEB properties + * + * A LEB may have fallen off of the bottom of a heap, and ended up as + * un-categorized even though it has enough space for us now. If that is the + * case this function will put the LEB back onto a heap. + */ +void ubifs_ensure_cat(struct ubifs_info *c, struct ubifs_lprops *lprops) +{ + int cat = lprops->flags & LPROPS_CAT_MASK; + + if (cat != LPROPS_UNCAT) + return; + cat = ubifs_categorize_lprops(c, lprops); + if (cat == LPROPS_UNCAT) + return; + ubifs_remove_from_cat(c, lprops, LPROPS_UNCAT); + ubifs_add_to_cat(c, lprops, cat); +} + +/** + * ubifs_categorize_lprops - categorize LEB properties. + * @c: UBIFS file-system description object + * @lprops: LEB properties to categorize + * + * LEB properties are categorized to enable fast find operations. This function + * returns the LEB category to which the LEB properties belong. Note however + * that if the LEB category is stored as a heap and the heap is full, the + * LEB properties may have their category changed to %LPROPS_UNCAT. + */ +int ubifs_categorize_lprops(const struct ubifs_info *c, + const struct ubifs_lprops *lprops) +{ + if (lprops->flags & LPROPS_TAKEN) + return LPROPS_UNCAT; + + if (lprops->free == c->leb_size) { + ubifs_assert(c, !(lprops->flags & LPROPS_INDEX)); + return LPROPS_EMPTY; + } + + if (lprops->free + lprops->dirty == c->leb_size) { + if (lprops->flags & LPROPS_INDEX) + return LPROPS_FRDI_IDX; + else + return LPROPS_FREEABLE; + } + + if (lprops->flags & LPROPS_INDEX) { + if (lprops->dirty + lprops->free >= c->min_idx_node_sz) + return LPROPS_DIRTY_IDX; + } else { + if (lprops->dirty >= c->dead_wm && + lprops->dirty > lprops->free) + return LPROPS_DIRTY; + if (lprops->free > 0) + return LPROPS_FREE; + } + + return LPROPS_UNCAT; +} + +/** + * change_category - change LEB properties category. + * @c: UBIFS file-system description object + * @lprops: LEB properties to re-categorize + * + * LEB properties are categorized to enable fast find operations. When the LEB + * properties change they must be re-categorized. + */ +static void change_category(struct ubifs_info *c, struct ubifs_lprops *lprops) +{ + int old_cat = lprops->flags & LPROPS_CAT_MASK; + int new_cat = ubifs_categorize_lprops(c, lprops); + + if (old_cat == new_cat) { + struct ubifs_lpt_heap *heap; + + /* lprops on a heap now must be moved up or down */ + if (new_cat < 1 || new_cat > LPROPS_HEAP_CNT) + return; /* Not on a heap */ + heap = &c->lpt_heap[new_cat - 1]; + adjust_lpt_heap(c, heap, lprops, lprops->hpos, new_cat); + } else { + ubifs_remove_from_cat(c, lprops, old_cat); + ubifs_add_to_cat(c, lprops, new_cat); + } +} + +/** + * ubifs_calc_dark - calculate LEB dark space size. + * @c: the UBIFS file-system description object + * @spc: amount of free and dirty space in the LEB + * + * This function calculates and returns amount of dark space in an LEB which + * has @spc bytes of free and dirty space. + * + * UBIFS is trying to account the space which might not be usable, and this + * space is called "dark space". For example, if an LEB has only %512 free + * bytes, it is dark space, because it cannot fit a large data node. + */ +int ubifs_calc_dark(const struct ubifs_info *c, int spc) +{ + ubifs_assert(c, !(spc & 7)); + + if (spc < c->dark_wm) + return spc; + + /* + * If we have slightly more space then the dark space watermark, we can + * anyway safely assume it we'll be able to write a node of the + * smallest size there. + */ + if (spc - c->dark_wm < MIN_WRITE_SZ) + return spc - MIN_WRITE_SZ; + + return c->dark_wm; +} + +/** + * is_lprops_dirty - determine if LEB properties are dirty. + * @c: the UBIFS file-system description object + * @lprops: LEB properties to test + */ +static int is_lprops_dirty(struct ubifs_info *c, struct ubifs_lprops *lprops) +{ + struct ubifs_pnode *pnode; + int pos; + + pos = (lprops->lnum - c->main_first) & (UBIFS_LPT_FANOUT - 1); + pnode = (struct ubifs_pnode *)container_of(lprops - pos, + struct ubifs_pnode, + lprops[0]); + return !test_bit(COW_CNODE, &pnode->flags) && + test_bit(DIRTY_CNODE, &pnode->flags); +} + +/** + * ubifs_change_lp - change LEB properties. + * @c: the UBIFS file-system description object + * @lp: LEB properties to change + * @free: new free space amount + * @dirty: new dirty space amount + * @flags: new flags + * @idx_gc_cnt: change to the count of @idx_gc list + * + * This function changes LEB properties (@free, @dirty or @flag). However, the + * property which has the %LPROPS_NC value is not changed. Returns a pointer to + * the updated LEB properties on success and a negative error code on failure. + * + * Note, the LEB properties may have had to be copied (due to COW) and + * consequently the pointer returned may not be the same as the pointer + * passed. + */ +const struct ubifs_lprops *ubifs_change_lp(struct ubifs_info *c, + const struct ubifs_lprops *lp, + int free, int dirty, int flags, + int idx_gc_cnt) +{ + /* + * This is the only function that is allowed to change lprops, so we + * discard the "const" qualifier. + */ + struct ubifs_lprops *lprops = (struct ubifs_lprops *)lp; + + dbg_lp("LEB %d, free %d, dirty %d, flags %d", + lprops->lnum, free, dirty, flags); + + ubifs_assert(c, mutex_is_locked(&c->lp_mutex)); + ubifs_assert(c, c->lst.empty_lebs >= 0 && + c->lst.empty_lebs <= c->main_lebs); + ubifs_assert(c, c->freeable_cnt >= 0); + ubifs_assert(c, c->freeable_cnt <= c->main_lebs); + ubifs_assert(c, c->lst.taken_empty_lebs >= 0); + ubifs_assert(c, c->lst.taken_empty_lebs <= c->lst.empty_lebs); + ubifs_assert(c, !(c->lst.total_free & 7) && !(c->lst.total_dirty & 7)); + ubifs_assert(c, !(c->lst.total_dead & 7) && !(c->lst.total_dark & 7)); + ubifs_assert(c, !(c->lst.total_used & 7)); + ubifs_assert(c, free == LPROPS_NC || free >= 0); + ubifs_assert(c, dirty == LPROPS_NC || dirty >= 0); + + if (!is_lprops_dirty(c, lprops)) { + lprops = ubifs_lpt_lookup_dirty(c, lprops->lnum); + if (IS_ERR(lprops)) + return lprops; + } else + ubifs_assert(c, lprops == ubifs_lpt_lookup_dirty(c, lprops->lnum)); + + ubifs_assert(c, !(lprops->free & 7) && !(lprops->dirty & 7)); + + spin_lock(&c->space_lock); + if ((lprops->flags & LPROPS_TAKEN) && lprops->free == c->leb_size) + c->lst.taken_empty_lebs -= 1; + + if (!(lprops->flags & LPROPS_INDEX)) { + int old_spc; + + old_spc = lprops->free + lprops->dirty; + if (old_spc < c->dead_wm) + c->lst.total_dead -= old_spc; + else + c->lst.total_dark -= ubifs_calc_dark(c, old_spc); + + c->lst.total_used -= c->leb_size - old_spc; + } + + if (free != LPROPS_NC) { + free = ALIGN(free, 8); + c->lst.total_free += free - lprops->free; + + /* Increase or decrease empty LEBs counter if needed */ + if (free == c->leb_size) { + if (lprops->free != c->leb_size) + c->lst.empty_lebs += 1; + } else if (lprops->free == c->leb_size) + c->lst.empty_lebs -= 1; + lprops->free = free; + } + + if (dirty != LPROPS_NC) { + dirty = ALIGN(dirty, 8); + c->lst.total_dirty += dirty - lprops->dirty; + lprops->dirty = dirty; + } + + if (flags != LPROPS_NC) { + /* Take care about indexing LEBs counter if needed */ + if ((lprops->flags & LPROPS_INDEX)) { + if (!(flags & LPROPS_INDEX)) + c->lst.idx_lebs -= 1; + } else if (flags & LPROPS_INDEX) + c->lst.idx_lebs += 1; + lprops->flags = flags; + } + + if (!(lprops->flags & LPROPS_INDEX)) { + int new_spc; + + new_spc = lprops->free + lprops->dirty; + if (new_spc < c->dead_wm) + c->lst.total_dead += new_spc; + else + c->lst.total_dark += ubifs_calc_dark(c, new_spc); + + c->lst.total_used += c->leb_size - new_spc; + } + + if ((lprops->flags & LPROPS_TAKEN) && lprops->free == c->leb_size) + c->lst.taken_empty_lebs += 1; + + change_category(c, lprops); + c->idx_gc_cnt += idx_gc_cnt; + spin_unlock(&c->space_lock); + return lprops; +} + +/** + * ubifs_get_lp_stats - get lprops statistics. + * @c: UBIFS file-system description object + * @lst: return statistics + */ +void ubifs_get_lp_stats(struct ubifs_info *c, struct ubifs_lp_stats *lst) +{ + spin_lock(&c->space_lock); + memcpy(lst, &c->lst, sizeof(struct ubifs_lp_stats)); + spin_unlock(&c->space_lock); +} + +/** + * ubifs_change_one_lp - change LEB properties. + * @c: the UBIFS file-system description object + * @lnum: LEB to change properties for + * @free: amount of free space + * @dirty: amount of dirty space + * @flags_set: flags to set + * @flags_clean: flags to clean + * @idx_gc_cnt: change to the count of idx_gc list + * + * This function changes properties of LEB @lnum. It is a helper wrapper over + * 'ubifs_change_lp()' which hides lprops get/release. The arguments are the + * same as in case of 'ubifs_change_lp()'. Returns zero in case of success and + * a negative error code in case of failure. + */ +int ubifs_change_one_lp(struct ubifs_info *c, int lnum, int free, int dirty, + int flags_set, int flags_clean, int idx_gc_cnt) +{ + int err = 0, flags; + const struct ubifs_lprops *lp; + + ubifs_get_lprops(c); + + lp = ubifs_lpt_lookup_dirty(c, lnum); + if (IS_ERR(lp)) { + err = PTR_ERR(lp); + goto out; + } + + flags = (lp->flags | flags_set) & ~flags_clean; + lp = ubifs_change_lp(c, lp, free, dirty, flags, idx_gc_cnt); + if (IS_ERR(lp)) + err = PTR_ERR(lp); + +out: + ubifs_release_lprops(c); + if (err) + ubifs_err(c, "cannot change properties of LEB %d, error %d", + lnum, err); + return err; +} + +/** + * ubifs_update_one_lp - update LEB properties. + * @c: the UBIFS file-system description object + * @lnum: LEB to change properties for + * @free: amount of free space + * @dirty: amount of dirty space to add + * @flags_set: flags to set + * @flags_clean: flags to clean + * + * This function is the same as 'ubifs_change_one_lp()' but @dirty is added to + * current dirty space, not substitutes it. + */ +int ubifs_update_one_lp(struct ubifs_info *c, int lnum, int free, int dirty, + int flags_set, int flags_clean) +{ + int err = 0, flags; + const struct ubifs_lprops *lp; + + ubifs_get_lprops(c); + + lp = ubifs_lpt_lookup_dirty(c, lnum); + if (IS_ERR(lp)) { + err = PTR_ERR(lp); + goto out; + } + + flags = (lp->flags | flags_set) & ~flags_clean; + lp = ubifs_change_lp(c, lp, free, lp->dirty + dirty, flags, 0); + if (IS_ERR(lp)) + err = PTR_ERR(lp); + +out: + ubifs_release_lprops(c); + if (err) + ubifs_err(c, "cannot update properties of LEB %d, error %d", + lnum, err); + return err; +} + +/** + * ubifs_read_one_lp - read LEB properties. + * @c: the UBIFS file-system description object + * @lnum: LEB to read properties for + * @lp: where to store read properties + * + * This helper function reads properties of a LEB @lnum and stores them in @lp. + * Returns zero in case of success and a negative error code in case of + * failure. + */ +int ubifs_read_one_lp(struct ubifs_info *c, int lnum, struct ubifs_lprops *lp) +{ + int err = 0; + const struct ubifs_lprops *lpp; + + ubifs_get_lprops(c); + + lpp = ubifs_lpt_lookup(c, lnum); + if (IS_ERR(lpp)) { + err = PTR_ERR(lpp); + ubifs_err(c, "cannot read properties of LEB %d, error %d", + lnum, err); + goto out; + } + + memcpy(lp, lpp, sizeof(struct ubifs_lprops)); + +out: + ubifs_release_lprops(c); + return err; +} + +/** + * ubifs_fast_find_free - try to find a LEB with free space quickly. + * @c: the UBIFS file-system description object + * + * This function returns LEB properties for a LEB with free space or %NULL if + * the function is unable to find a LEB quickly. + */ +const struct ubifs_lprops *ubifs_fast_find_free(struct ubifs_info *c) +{ + struct ubifs_lprops *lprops; + struct ubifs_lpt_heap *heap; + + ubifs_assert(c, mutex_is_locked(&c->lp_mutex)); + + heap = &c->lpt_heap[LPROPS_FREE - 1]; + if (heap->cnt == 0) + return NULL; + + lprops = heap->arr[0]; + ubifs_assert(c, !(lprops->flags & LPROPS_TAKEN)); + ubifs_assert(c, !(lprops->flags & LPROPS_INDEX)); + return lprops; +} + +/** + * ubifs_fast_find_empty - try to find an empty LEB quickly. + * @c: the UBIFS file-system description object + * + * This function returns LEB properties for an empty LEB or %NULL if the + * function is unable to find an empty LEB quickly. + */ +const struct ubifs_lprops *ubifs_fast_find_empty(struct ubifs_info *c) +{ + struct ubifs_lprops *lprops; + + ubifs_assert(c, mutex_is_locked(&c->lp_mutex)); + + if (list_empty(&c->empty_list)) + return NULL; + + lprops = list_entry(c->empty_list.next, struct ubifs_lprops, list); + ubifs_assert(c, !(lprops->flags & LPROPS_TAKEN)); + ubifs_assert(c, !(lprops->flags & LPROPS_INDEX)); + ubifs_assert(c, lprops->free == c->leb_size); + return lprops; +} + +/** + * ubifs_fast_find_freeable - try to find a freeable LEB quickly. + * @c: the UBIFS file-system description object + * + * This function returns LEB properties for a freeable LEB or %NULL if the + * function is unable to find a freeable LEB quickly. + */ +const struct ubifs_lprops *ubifs_fast_find_freeable(struct ubifs_info *c) +{ + struct ubifs_lprops *lprops; + + ubifs_assert(c, mutex_is_locked(&c->lp_mutex)); + + if (list_empty(&c->freeable_list)) + return NULL; + + lprops = list_entry(c->freeable_list.next, struct ubifs_lprops, list); + ubifs_assert(c, !(lprops->flags & LPROPS_TAKEN)); + ubifs_assert(c, !(lprops->flags & LPROPS_INDEX)); + ubifs_assert(c, lprops->free + lprops->dirty == c->leb_size); + ubifs_assert(c, c->freeable_cnt > 0); + return lprops; +} + +/** + * ubifs_fast_find_frdi_idx - try to find a freeable index LEB quickly. + * @c: the UBIFS file-system description object + * + * This function returns LEB properties for a freeable index LEB or %NULL if the + * function is unable to find a freeable index LEB quickly. + */ +const struct ubifs_lprops *ubifs_fast_find_frdi_idx(struct ubifs_info *c) +{ + struct ubifs_lprops *lprops; + + ubifs_assert(c, mutex_is_locked(&c->lp_mutex)); + + if (list_empty(&c->frdi_idx_list)) + return NULL; + + lprops = list_entry(c->frdi_idx_list.next, struct ubifs_lprops, list); + ubifs_assert(c, !(lprops->flags & LPROPS_TAKEN)); + ubifs_assert(c, (lprops->flags & LPROPS_INDEX)); + ubifs_assert(c, lprops->free + lprops->dirty == c->leb_size); + return lprops; +} + +/* + * Everything below is related to debugging. + */ + +/** + * dbg_check_cats - check category heaps and lists. + * @c: UBIFS file-system description object + * + * This function returns %0 on success and a negative error code on failure. + */ +int dbg_check_cats(struct ubifs_info *c) +{ + struct ubifs_lprops *lprops; + struct list_head *pos; + int i, cat; + + if (!dbg_is_chk_gen(c) && !dbg_is_chk_lprops(c)) + return 0; + + list_for_each_entry(lprops, &c->empty_list, list) { + if (lprops->free != c->leb_size) { + ubifs_err(c, "non-empty LEB %d on empty list (free %d dirty %d flags %d)", + lprops->lnum, lprops->free, lprops->dirty, + lprops->flags); + return -EINVAL; + } + if (lprops->flags & LPROPS_TAKEN) { + ubifs_err(c, "taken LEB %d on empty list (free %d dirty %d flags %d)", + lprops->lnum, lprops->free, lprops->dirty, + lprops->flags); + return -EINVAL; + } + } + + i = 0; + list_for_each_entry(lprops, &c->freeable_list, list) { + if (lprops->free + lprops->dirty != c->leb_size) { + ubifs_err(c, "non-freeable LEB %d on freeable list (free %d dirty %d flags %d)", + lprops->lnum, lprops->free, lprops->dirty, + lprops->flags); + return -EINVAL; + } + if (lprops->flags & LPROPS_TAKEN) { + ubifs_err(c, "taken LEB %d on freeable list (free %d dirty %d flags %d)", + lprops->lnum, lprops->free, lprops->dirty, + lprops->flags); + return -EINVAL; + } + i += 1; + } + if (i != c->freeable_cnt) { + ubifs_err(c, "freeable list count %d expected %d", i, + c->freeable_cnt); + return -EINVAL; + } + + i = 0; + list_for_each(pos, &c->idx_gc) + i += 1; + if (i != c->idx_gc_cnt) { + ubifs_err(c, "idx_gc list count %d expected %d", i, + c->idx_gc_cnt); + return -EINVAL; + } + + list_for_each_entry(lprops, &c->frdi_idx_list, list) { + if (lprops->free + lprops->dirty != c->leb_size) { + ubifs_err(c, "non-freeable LEB %d on frdi_idx list (free %d dirty %d flags %d)", + lprops->lnum, lprops->free, lprops->dirty, + lprops->flags); + return -EINVAL; + } + if (lprops->flags & LPROPS_TAKEN) { + ubifs_err(c, "taken LEB %d on frdi_idx list (free %d dirty %d flags %d)", + lprops->lnum, lprops->free, lprops->dirty, + lprops->flags); + return -EINVAL; + } + if (!(lprops->flags & LPROPS_INDEX)) { + ubifs_err(c, "non-index LEB %d on frdi_idx list (free %d dirty %d flags %d)", + lprops->lnum, lprops->free, lprops->dirty, + lprops->flags); + return -EINVAL; + } + } + + for (cat = 1; cat <= LPROPS_HEAP_CNT; cat++) { + struct ubifs_lpt_heap *heap = &c->lpt_heap[cat - 1]; + + for (i = 0; i < heap->cnt; i++) { + lprops = heap->arr[i]; + if (!lprops) { + ubifs_err(c, "null ptr in LPT heap cat %d", cat); + return -EINVAL; + } + if (lprops->hpos != i) { + ubifs_err(c, "bad ptr in LPT heap cat %d", cat); + return -EINVAL; + } + if (lprops->flags & LPROPS_TAKEN) { + ubifs_err(c, "taken LEB in LPT heap cat %d", cat); + return -EINVAL; + } + } + } + + return 0; +} + +void dbg_check_heap(struct ubifs_info *c, struct ubifs_lpt_heap *heap, int cat, + int add_pos) +{ + int i = 0, j, err = 0; + + if (!dbg_is_chk_gen(c) && !dbg_is_chk_lprops(c)) + return; + + for (i = 0; i < heap->cnt; i++) { + struct ubifs_lprops *lprops = heap->arr[i]; + struct ubifs_lprops *lp; + + if (i != add_pos) + if ((lprops->flags & LPROPS_CAT_MASK) != cat) { + err = 1; + goto out; + } + if (lprops->hpos != i) { + err = 2; + goto out; + } + lp = ubifs_lpt_lookup(c, lprops->lnum); + if (IS_ERR(lp)) { + err = 3; + goto out; + } + if (lprops != lp) { + ubifs_err(c, "lprops %zx lp %zx lprops->lnum %d lp->lnum %d", + (size_t)lprops, (size_t)lp, lprops->lnum, + lp->lnum); + err = 4; + goto out; + } + for (j = 0; j < i; j++) { + lp = heap->arr[j]; + if (lp == lprops) { + err = 5; + goto out; + } + if (lp->lnum == lprops->lnum) { + err = 6; + goto out; + } + } + } +out: + if (err) { + ubifs_err(c, "failed cat %d hpos %d err %d", cat, i, err); + dump_stack(); + ubifs_dump_heap(c, heap, cat); + } +} + +/** + * scan_check_cb - scan callback. + * @c: the UBIFS file-system description object + * @lp: LEB properties to scan + * @in_tree: whether the LEB properties are in main memory + * @arg: lprops statistics to update + * + * This function returns a code that indicates whether the scan should continue + * (%LPT_SCAN_CONTINUE), whether the LEB properties should be added to the tree + * in main memory (%LPT_SCAN_ADD), or whether the scan should stop + * (%LPT_SCAN_STOP). + */ +static int scan_check_cb(struct ubifs_info *c, + const struct ubifs_lprops *lp, int in_tree, + void *arg) +{ + struct ubifs_lp_stats *lst = arg; + struct ubifs_scan_leb *sleb; + struct ubifs_scan_node *snod; + int cat, lnum = lp->lnum, is_idx = 0, used = 0, free, dirty, ret; + void *buf = NULL; + + cat = lp->flags & LPROPS_CAT_MASK; + if (cat != LPROPS_UNCAT) { + cat = ubifs_categorize_lprops(c, lp); + if (cat != (lp->flags & LPROPS_CAT_MASK)) { + ubifs_err(c, "bad LEB category %d expected %d", + (lp->flags & LPROPS_CAT_MASK), cat); + return -EINVAL; + } + } + + /* Check lp is on its category list (if it has one) */ + if (in_tree) { + struct list_head *list = NULL; + + switch (cat) { + case LPROPS_EMPTY: + list = &c->empty_list; + break; + case LPROPS_FREEABLE: + list = &c->freeable_list; + break; + case LPROPS_FRDI_IDX: + list = &c->frdi_idx_list; + break; + case LPROPS_UNCAT: + list = &c->uncat_list; + break; + } + if (list) { + struct ubifs_lprops *lprops; + int found = 0; + + list_for_each_entry(lprops, list, list) { + if (lprops == lp) { + found = 1; + break; + } + } + if (!found) { + ubifs_err(c, "bad LPT list (category %d)", cat); + return -EINVAL; + } + } + } + + /* Check lp is on its category heap (if it has one) */ + if (in_tree && cat > 0 && cat <= LPROPS_HEAP_CNT) { + struct ubifs_lpt_heap *heap = &c->lpt_heap[cat - 1]; + + if ((lp->hpos != -1 && heap->arr[lp->hpos]->lnum != lnum) || + lp != heap->arr[lp->hpos]) { + ubifs_err(c, "bad LPT heap (category %d)", cat); + return -EINVAL; + } + } + + /* + * After an unclean unmount, empty and freeable LEBs + * may contain garbage - do not scan them. + */ + if (lp->free == c->leb_size) { + lst->empty_lebs += 1; + lst->total_free += c->leb_size; + lst->total_dark += ubifs_calc_dark(c, c->leb_size); + return LPT_SCAN_CONTINUE; + } + if (lp->free + lp->dirty == c->leb_size && + !(lp->flags & LPROPS_INDEX)) { + lst->total_free += lp->free; + lst->total_dirty += lp->dirty; + lst->total_dark += ubifs_calc_dark(c, c->leb_size); + return LPT_SCAN_CONTINUE; + } + + buf = __vmalloc(c->leb_size, GFP_NOFS); + if (!buf) + return -ENOMEM; + + sleb = ubifs_scan(c, lnum, 0, buf, 0); + if (IS_ERR(sleb)) { + ret = PTR_ERR(sleb); + if (ret == -EUCLEAN) { + ubifs_dump_lprops(c); + ubifs_dump_budg(c, &c->bi); + } + goto out; + } + + is_idx = -1; + list_for_each_entry(snod, &sleb->nodes, list) { + int found, level = 0; + + cond_resched(); + + if (is_idx == -1) + is_idx = (snod->type == UBIFS_IDX_NODE) ? 1 : 0; + + if (is_idx && snod->type != UBIFS_IDX_NODE) { + ubifs_err(c, "indexing node in data LEB %d:%d", + lnum, snod->offs); + goto out_destroy; + } + + if (snod->type == UBIFS_IDX_NODE) { + struct ubifs_idx_node *idx = snod->node; + + key_read(c, ubifs_idx_key(c, idx), &snod->key); + level = le16_to_cpu(idx->level); + } + + found = ubifs_tnc_has_node(c, &snod->key, level, lnum, + snod->offs, is_idx); + if (found) { + if (found < 0) + goto out_destroy; + used += ALIGN(snod->len, 8); + } + } + + free = c->leb_size - sleb->endpt; + dirty = sleb->endpt - used; + + if (free > c->leb_size || free < 0 || dirty > c->leb_size || + dirty < 0) { + ubifs_err(c, "bad calculated accounting for LEB %d: free %d, dirty %d", + lnum, free, dirty); + goto out_destroy; + } + + if (lp->free + lp->dirty == c->leb_size && + free + dirty == c->leb_size) + if ((is_idx && !(lp->flags & LPROPS_INDEX)) || + (!is_idx && free == c->leb_size) || + lp->free == c->leb_size) { + /* + * Empty or freeable LEBs could contain index + * nodes from an uncompleted commit due to an + * unclean unmount. Or they could be empty for + * the same reason. Or it may simply not have been + * unmapped. + */ + free = lp->free; + dirty = lp->dirty; + is_idx = 0; + } + + if (is_idx && lp->free + lp->dirty == free + dirty && + lnum != c->ihead_lnum) { + /* + * After an unclean unmount, an index LEB could have a different + * amount of free space than the value recorded by lprops. That + * is because the in-the-gaps method may use free space or + * create free space (as a side-effect of using ubi_leb_change + * and not writing the whole LEB). The incorrect free space + * value is not a problem because the index is only ever + * allocated empty LEBs, so there will never be an attempt to + * write to the free space at the end of an index LEB - except + * by the in-the-gaps method for which it is not a problem. + */ + free = lp->free; + dirty = lp->dirty; + } + + if (lp->free != free || lp->dirty != dirty) + goto out_print; + + if (is_idx && !(lp->flags & LPROPS_INDEX)) { + if (free == c->leb_size) + /* Free but not unmapped LEB, it's fine */ + is_idx = 0; + else { + ubifs_err(c, "indexing node without indexing flag"); + goto out_print; + } + } + + if (!is_idx && (lp->flags & LPROPS_INDEX)) { + ubifs_err(c, "data node with indexing flag"); + goto out_print; + } + + if (free == c->leb_size) + lst->empty_lebs += 1; + + if (is_idx) + lst->idx_lebs += 1; + + if (!(lp->flags & LPROPS_INDEX)) + lst->total_used += c->leb_size - free - dirty; + lst->total_free += free; + lst->total_dirty += dirty; + + if (!(lp->flags & LPROPS_INDEX)) { + int spc = free + dirty; + + if (spc < c->dead_wm) + lst->total_dead += spc; + else + lst->total_dark += ubifs_calc_dark(c, spc); + } + + ubifs_scan_destroy(sleb); + vfree(buf); + return LPT_SCAN_CONTINUE; + +out_print: + ubifs_err(c, "bad accounting of LEB %d: free %d, dirty %d flags %#x, should be free %d, dirty %d", + lnum, lp->free, lp->dirty, lp->flags, free, dirty); + ubifs_dump_leb(c, lnum); +out_destroy: + ubifs_scan_destroy(sleb); + ret = -EINVAL; +out: + vfree(buf); + return ret; +} + +/** + * dbg_check_lprops - check all LEB properties. + * @c: UBIFS file-system description object + * + * This function checks all LEB properties and makes sure they are all correct. + * It returns zero if everything is fine, %-EINVAL if there is an inconsistency + * and other negative error codes in case of other errors. This function is + * called while the file system is locked (because of commit start), so no + * additional locking is required. Note that locking the LPT mutex would cause + * a circular lock dependency with the TNC mutex. + */ +int dbg_check_lprops(struct ubifs_info *c) +{ + int i, err; + struct ubifs_lp_stats lst; + + if (!dbg_is_chk_lprops(c)) + return 0; + + /* + * As we are going to scan the media, the write buffers have to be + * synchronized. + */ + for (i = 0; i < c->jhead_cnt; i++) { + err = ubifs_wbuf_sync(&c->jheads[i].wbuf); + if (err) + return err; + } + + memset(&lst, 0, sizeof(struct ubifs_lp_stats)); + err = ubifs_lpt_scan_nolock(c, c->main_first, c->leb_cnt - 1, + scan_check_cb, &lst); + if (err && err != -ENOSPC) + goto out; + + if (lst.empty_lebs != c->lst.empty_lebs || + lst.idx_lebs != c->lst.idx_lebs || + lst.total_free != c->lst.total_free || + lst.total_dirty != c->lst.total_dirty || + lst.total_used != c->lst.total_used) { + ubifs_err(c, "bad overall accounting"); + ubifs_err(c, "calculated: empty_lebs %d, idx_lebs %d, total_free %lld, total_dirty %lld, total_used %lld", + lst.empty_lebs, lst.idx_lebs, lst.total_free, + lst.total_dirty, lst.total_used); + ubifs_err(c, "read from lprops: empty_lebs %d, idx_lebs %d, total_free %lld, total_dirty %lld, total_used %lld", + c->lst.empty_lebs, c->lst.idx_lebs, c->lst.total_free, + c->lst.total_dirty, c->lst.total_used); + err = -EINVAL; + goto out; + } + + if (lst.total_dead != c->lst.total_dead || + lst.total_dark != c->lst.total_dark) { + ubifs_err(c, "bad dead/dark space accounting"); + ubifs_err(c, "calculated: total_dead %lld, total_dark %lld", + lst.total_dead, lst.total_dark); + ubifs_err(c, "read from lprops: total_dead %lld, total_dark %lld", + c->lst.total_dead, c->lst.total_dark); + err = -EINVAL; + goto out; + } + + err = dbg_check_cats(c); +out: + return err; +} diff --git a/fs/ubifs/lpt.c b/fs/ubifs/lpt.c new file mode 100644 index 000000000..b3986705c --- /dev/null +++ b/fs/ubifs/lpt.c @@ -0,0 +1,2435 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * + * Authors: Adrian Hunter + * Artem Bityutskiy (Битюцкий Артём) + */ + +/* + * This file implements the LEB properties tree (LPT) area. The LPT area + * contains the LEB properties tree, a table of LPT area eraseblocks (ltab), and + * (for the "big" model) a table of saved LEB numbers (lsave). The LPT area sits + * between the log and the orphan area. + * + * The LPT area is like a miniature self-contained file system. It is required + * that it never runs out of space, is fast to access and update, and scales + * logarithmically. The LEB properties tree is implemented as a wandering tree + * much like the TNC, and the LPT area has its own garbage collection. + * + * The LPT has two slightly different forms called the "small model" and the + * "big model". The small model is used when the entire LEB properties table + * can be written into a single eraseblock. In that case, garbage collection + * consists of just writing the whole table, which therefore makes all other + * eraseblocks reusable. In the case of the big model, dirty eraseblocks are + * selected for garbage collection, which consists of marking the clean nodes in + * that LEB as dirty, and then only the dirty nodes are written out. Also, in + * the case of the big model, a table of LEB numbers is saved so that the entire + * LPT does not to be scanned looking for empty eraseblocks when UBIFS is first + * mounted. + */ + +#include "ubifs.h" +#include +#include +#include + +/** + * do_calc_lpt_geom - calculate sizes for the LPT area. + * @c: the UBIFS file-system description object + * + * Calculate the sizes of LPT bit fields, nodes, and tree, based on the + * properties of the flash and whether LPT is "big" (c->big_lpt). + */ +static void do_calc_lpt_geom(struct ubifs_info *c) +{ + int i, n, bits, per_leb_wastage, max_pnode_cnt; + long long sz, tot_wastage; + + n = c->main_lebs + c->max_leb_cnt - c->leb_cnt; + max_pnode_cnt = DIV_ROUND_UP(n, UBIFS_LPT_FANOUT); + + c->lpt_hght = 1; + n = UBIFS_LPT_FANOUT; + while (n < max_pnode_cnt) { + c->lpt_hght += 1; + n <<= UBIFS_LPT_FANOUT_SHIFT; + } + + c->pnode_cnt = DIV_ROUND_UP(c->main_lebs, UBIFS_LPT_FANOUT); + + n = DIV_ROUND_UP(c->pnode_cnt, UBIFS_LPT_FANOUT); + c->nnode_cnt = n; + for (i = 1; i < c->lpt_hght; i++) { + n = DIV_ROUND_UP(n, UBIFS_LPT_FANOUT); + c->nnode_cnt += n; + } + + c->space_bits = fls(c->leb_size) - 3; + c->lpt_lnum_bits = fls(c->lpt_lebs); + c->lpt_offs_bits = fls(c->leb_size - 1); + c->lpt_spc_bits = fls(c->leb_size); + + n = DIV_ROUND_UP(c->max_leb_cnt, UBIFS_LPT_FANOUT); + c->pcnt_bits = fls(n - 1); + + c->lnum_bits = fls(c->max_leb_cnt - 1); + + bits = UBIFS_LPT_CRC_BITS + UBIFS_LPT_TYPE_BITS + + (c->big_lpt ? c->pcnt_bits : 0) + + (c->space_bits * 2 + 1) * UBIFS_LPT_FANOUT; + c->pnode_sz = (bits + 7) / 8; + + bits = UBIFS_LPT_CRC_BITS + UBIFS_LPT_TYPE_BITS + + (c->big_lpt ? c->pcnt_bits : 0) + + (c->lpt_lnum_bits + c->lpt_offs_bits) * UBIFS_LPT_FANOUT; + c->nnode_sz = (bits + 7) / 8; + + bits = UBIFS_LPT_CRC_BITS + UBIFS_LPT_TYPE_BITS + + c->lpt_lebs * c->lpt_spc_bits * 2; + c->ltab_sz = (bits + 7) / 8; + + bits = UBIFS_LPT_CRC_BITS + UBIFS_LPT_TYPE_BITS + + c->lnum_bits * c->lsave_cnt; + c->lsave_sz = (bits + 7) / 8; + + /* Calculate the minimum LPT size */ + c->lpt_sz = (long long)c->pnode_cnt * c->pnode_sz; + c->lpt_sz += (long long)c->nnode_cnt * c->nnode_sz; + c->lpt_sz += c->ltab_sz; + if (c->big_lpt) + c->lpt_sz += c->lsave_sz; + + /* Add wastage */ + sz = c->lpt_sz; + per_leb_wastage = max_t(int, c->pnode_sz, c->nnode_sz); + sz += per_leb_wastage; + tot_wastage = per_leb_wastage; + while (sz > c->leb_size) { + sz += per_leb_wastage; + sz -= c->leb_size; + tot_wastage += per_leb_wastage; + } + tot_wastage += ALIGN(sz, c->min_io_size) - sz; + c->lpt_sz += tot_wastage; +} + +/** + * ubifs_calc_lpt_geom - calculate and check sizes for the LPT area. + * @c: the UBIFS file-system description object + * + * This function returns %0 on success and a negative error code on failure. + */ +int ubifs_calc_lpt_geom(struct ubifs_info *c) +{ + int lebs_needed; + long long sz; + + do_calc_lpt_geom(c); + + /* Verify that lpt_lebs is big enough */ + sz = c->lpt_sz * 2; /* Must have at least 2 times the size */ + lebs_needed = div_u64(sz + c->leb_size - 1, c->leb_size); + if (lebs_needed > c->lpt_lebs) { + ubifs_err(c, "too few LPT LEBs"); + return -EINVAL; + } + + /* Verify that ltab fits in a single LEB (since ltab is a single node */ + if (c->ltab_sz > c->leb_size) { + ubifs_err(c, "LPT ltab too big"); + return -EINVAL; + } + + c->check_lpt_free = c->big_lpt; + return 0; +} + +/** + * calc_dflt_lpt_geom - calculate default LPT geometry. + * @c: the UBIFS file-system description object + * @main_lebs: number of main area LEBs is passed and returned here + * @big_lpt: whether the LPT area is "big" is returned here + * + * The size of the LPT area depends on parameters that themselves are dependent + * on the size of the LPT area. This function, successively recalculates the LPT + * area geometry until the parameters and resultant geometry are consistent. + * + * This function returns %0 on success and a negative error code on failure. + */ +static int calc_dflt_lpt_geom(struct ubifs_info *c, int *main_lebs, + int *big_lpt) +{ + int i, lebs_needed; + long long sz; + + /* Start by assuming the minimum number of LPT LEBs */ + c->lpt_lebs = UBIFS_MIN_LPT_LEBS; + c->main_lebs = *main_lebs - c->lpt_lebs; + if (c->main_lebs <= 0) + return -EINVAL; + + /* And assume we will use the small LPT model */ + c->big_lpt = 0; + + /* + * Calculate the geometry based on assumptions above and then see if it + * makes sense + */ + do_calc_lpt_geom(c); + + /* Small LPT model must have lpt_sz < leb_size */ + if (c->lpt_sz > c->leb_size) { + /* Nope, so try again using big LPT model */ + c->big_lpt = 1; + do_calc_lpt_geom(c); + } + + /* Now check there are enough LPT LEBs */ + for (i = 0; i < 64 ; i++) { + sz = c->lpt_sz * 4; /* Allow 4 times the size */ + lebs_needed = div_u64(sz + c->leb_size - 1, c->leb_size); + if (lebs_needed > c->lpt_lebs) { + /* Not enough LPT LEBs so try again with more */ + c->lpt_lebs = lebs_needed; + c->main_lebs = *main_lebs - c->lpt_lebs; + if (c->main_lebs <= 0) + return -EINVAL; + do_calc_lpt_geom(c); + continue; + } + if (c->ltab_sz > c->leb_size) { + ubifs_err(c, "LPT ltab too big"); + return -EINVAL; + } + *main_lebs = c->main_lebs; + *big_lpt = c->big_lpt; + return 0; + } + return -EINVAL; +} + +/** + * pack_bits - pack bit fields end-to-end. + * @c: UBIFS file-system description object + * @addr: address at which to pack (passed and next address returned) + * @pos: bit position at which to pack (passed and next position returned) + * @val: value to pack + * @nrbits: number of bits of value to pack (1-32) + */ +static void pack_bits(const struct ubifs_info *c, uint8_t **addr, int *pos, uint32_t val, int nrbits) +{ + uint8_t *p = *addr; + int b = *pos; + + ubifs_assert(c, nrbits > 0); + ubifs_assert(c, nrbits <= 32); + ubifs_assert(c, *pos >= 0); + ubifs_assert(c, *pos < 8); + ubifs_assert(c, (val >> nrbits) == 0 || nrbits == 32); + if (b) { + *p |= ((uint8_t)val) << b; + nrbits += b; + if (nrbits > 8) { + *++p = (uint8_t)(val >>= (8 - b)); + if (nrbits > 16) { + *++p = (uint8_t)(val >>= 8); + if (nrbits > 24) { + *++p = (uint8_t)(val >>= 8); + if (nrbits > 32) + *++p = (uint8_t)(val >>= 8); + } + } + } + } else { + *p = (uint8_t)val; + if (nrbits > 8) { + *++p = (uint8_t)(val >>= 8); + if (nrbits > 16) { + *++p = (uint8_t)(val >>= 8); + if (nrbits > 24) + *++p = (uint8_t)(val >>= 8); + } + } + } + b = nrbits & 7; + if (b == 0) + p++; + *addr = p; + *pos = b; +} + +/** + * ubifs_unpack_bits - unpack bit fields. + * @c: UBIFS file-system description object + * @addr: address at which to unpack (passed and next address returned) + * @pos: bit position at which to unpack (passed and next position returned) + * @nrbits: number of bits of value to unpack (1-32) + * + * This functions returns the value unpacked. + */ +uint32_t ubifs_unpack_bits(const struct ubifs_info *c, uint8_t **addr, int *pos, int nrbits) +{ + const int k = 32 - nrbits; + uint8_t *p = *addr; + int b = *pos; + uint32_t val; + const int bytes = (nrbits + b + 7) >> 3; + + ubifs_assert(c, nrbits > 0); + ubifs_assert(c, nrbits <= 32); + ubifs_assert(c, *pos >= 0); + ubifs_assert(c, *pos < 8); + if (b) { + switch (bytes) { + case 2: + val = p[1]; + break; + case 3: + val = p[1] | ((uint32_t)p[2] << 8); + break; + case 4: + val = p[1] | ((uint32_t)p[2] << 8) | + ((uint32_t)p[3] << 16); + break; + case 5: + val = p[1] | ((uint32_t)p[2] << 8) | + ((uint32_t)p[3] << 16) | + ((uint32_t)p[4] << 24); + } + val <<= (8 - b); + val |= *p >> b; + nrbits += b; + } else { + switch (bytes) { + case 1: + val = p[0]; + break; + case 2: + val = p[0] | ((uint32_t)p[1] << 8); + break; + case 3: + val = p[0] | ((uint32_t)p[1] << 8) | + ((uint32_t)p[2] << 16); + break; + case 4: + val = p[0] | ((uint32_t)p[1] << 8) | + ((uint32_t)p[2] << 16) | + ((uint32_t)p[3] << 24); + break; + } + } + val <<= k; + val >>= k; + b = nrbits & 7; + p += nrbits >> 3; + *addr = p; + *pos = b; + ubifs_assert(c, (val >> nrbits) == 0 || nrbits - b == 32); + return val; +} + +/** + * ubifs_pack_pnode - pack all the bit fields of a pnode. + * @c: UBIFS file-system description object + * @buf: buffer into which to pack + * @pnode: pnode to pack + */ +void ubifs_pack_pnode(struct ubifs_info *c, void *buf, + struct ubifs_pnode *pnode) +{ + uint8_t *addr = buf + UBIFS_LPT_CRC_BYTES; + int i, pos = 0; + uint16_t crc; + + pack_bits(c, &addr, &pos, UBIFS_LPT_PNODE, UBIFS_LPT_TYPE_BITS); + if (c->big_lpt) + pack_bits(c, &addr, &pos, pnode->num, c->pcnt_bits); + for (i = 0; i < UBIFS_LPT_FANOUT; i++) { + pack_bits(c, &addr, &pos, pnode->lprops[i].free >> 3, + c->space_bits); + pack_bits(c, &addr, &pos, pnode->lprops[i].dirty >> 3, + c->space_bits); + if (pnode->lprops[i].flags & LPROPS_INDEX) + pack_bits(c, &addr, &pos, 1, 1); + else + pack_bits(c, &addr, &pos, 0, 1); + } + crc = crc16(-1, buf + UBIFS_LPT_CRC_BYTES, + c->pnode_sz - UBIFS_LPT_CRC_BYTES); + addr = buf; + pos = 0; + pack_bits(c, &addr, &pos, crc, UBIFS_LPT_CRC_BITS); +} + +/** + * ubifs_pack_nnode - pack all the bit fields of a nnode. + * @c: UBIFS file-system description object + * @buf: buffer into which to pack + * @nnode: nnode to pack + */ +void ubifs_pack_nnode(struct ubifs_info *c, void *buf, + struct ubifs_nnode *nnode) +{ + uint8_t *addr = buf + UBIFS_LPT_CRC_BYTES; + int i, pos = 0; + uint16_t crc; + + pack_bits(c, &addr, &pos, UBIFS_LPT_NNODE, UBIFS_LPT_TYPE_BITS); + if (c->big_lpt) + pack_bits(c, &addr, &pos, nnode->num, c->pcnt_bits); + for (i = 0; i < UBIFS_LPT_FANOUT; i++) { + int lnum = nnode->nbranch[i].lnum; + + if (lnum == 0) + lnum = c->lpt_last + 1; + pack_bits(c, &addr, &pos, lnum - c->lpt_first, c->lpt_lnum_bits); + pack_bits(c, &addr, &pos, nnode->nbranch[i].offs, + c->lpt_offs_bits); + } + crc = crc16(-1, buf + UBIFS_LPT_CRC_BYTES, + c->nnode_sz - UBIFS_LPT_CRC_BYTES); + addr = buf; + pos = 0; + pack_bits(c, &addr, &pos, crc, UBIFS_LPT_CRC_BITS); +} + +/** + * ubifs_pack_ltab - pack the LPT's own lprops table. + * @c: UBIFS file-system description object + * @buf: buffer into which to pack + * @ltab: LPT's own lprops table to pack + */ +void ubifs_pack_ltab(struct ubifs_info *c, void *buf, + struct ubifs_lpt_lprops *ltab) +{ + uint8_t *addr = buf + UBIFS_LPT_CRC_BYTES; + int i, pos = 0; + uint16_t crc; + + pack_bits(c, &addr, &pos, UBIFS_LPT_LTAB, UBIFS_LPT_TYPE_BITS); + for (i = 0; i < c->lpt_lebs; i++) { + pack_bits(c, &addr, &pos, ltab[i].free, c->lpt_spc_bits); + pack_bits(c, &addr, &pos, ltab[i].dirty, c->lpt_spc_bits); + } + crc = crc16(-1, buf + UBIFS_LPT_CRC_BYTES, + c->ltab_sz - UBIFS_LPT_CRC_BYTES); + addr = buf; + pos = 0; + pack_bits(c, &addr, &pos, crc, UBIFS_LPT_CRC_BITS); +} + +/** + * ubifs_pack_lsave - pack the LPT's save table. + * @c: UBIFS file-system description object + * @buf: buffer into which to pack + * @lsave: LPT's save table to pack + */ +void ubifs_pack_lsave(struct ubifs_info *c, void *buf, int *lsave) +{ + uint8_t *addr = buf + UBIFS_LPT_CRC_BYTES; + int i, pos = 0; + uint16_t crc; + + pack_bits(c, &addr, &pos, UBIFS_LPT_LSAVE, UBIFS_LPT_TYPE_BITS); + for (i = 0; i < c->lsave_cnt; i++) + pack_bits(c, &addr, &pos, lsave[i], c->lnum_bits); + crc = crc16(-1, buf + UBIFS_LPT_CRC_BYTES, + c->lsave_sz - UBIFS_LPT_CRC_BYTES); + addr = buf; + pos = 0; + pack_bits(c, &addr, &pos, crc, UBIFS_LPT_CRC_BITS); +} + +/** + * ubifs_add_lpt_dirt - add dirty space to LPT LEB properties. + * @c: UBIFS file-system description object + * @lnum: LEB number to which to add dirty space + * @dirty: amount of dirty space to add + */ +void ubifs_add_lpt_dirt(struct ubifs_info *c, int lnum, int dirty) +{ + if (!dirty || !lnum) + return; + dbg_lp("LEB %d add %d to %d", + lnum, dirty, c->ltab[lnum - c->lpt_first].dirty); + ubifs_assert(c, lnum >= c->lpt_first && lnum <= c->lpt_last); + c->ltab[lnum - c->lpt_first].dirty += dirty; +} + +/** + * set_ltab - set LPT LEB properties. + * @c: UBIFS file-system description object + * @lnum: LEB number + * @free: amount of free space + * @dirty: amount of dirty space + */ +static void set_ltab(struct ubifs_info *c, int lnum, int free, int dirty) +{ + dbg_lp("LEB %d free %d dirty %d to %d %d", + lnum, c->ltab[lnum - c->lpt_first].free, + c->ltab[lnum - c->lpt_first].dirty, free, dirty); + ubifs_assert(c, lnum >= c->lpt_first && lnum <= c->lpt_last); + c->ltab[lnum - c->lpt_first].free = free; + c->ltab[lnum - c->lpt_first].dirty = dirty; +} + +/** + * ubifs_add_nnode_dirt - add dirty space to LPT LEB properties. + * @c: UBIFS file-system description object + * @nnode: nnode for which to add dirt + */ +void ubifs_add_nnode_dirt(struct ubifs_info *c, struct ubifs_nnode *nnode) +{ + struct ubifs_nnode *np = nnode->parent; + + if (np) + ubifs_add_lpt_dirt(c, np->nbranch[nnode->iip].lnum, + c->nnode_sz); + else { + ubifs_add_lpt_dirt(c, c->lpt_lnum, c->nnode_sz); + if (!(c->lpt_drty_flgs & LTAB_DIRTY)) { + c->lpt_drty_flgs |= LTAB_DIRTY; + ubifs_add_lpt_dirt(c, c->ltab_lnum, c->ltab_sz); + } + } +} + +/** + * add_pnode_dirt - add dirty space to LPT LEB properties. + * @c: UBIFS file-system description object + * @pnode: pnode for which to add dirt + */ +static void add_pnode_dirt(struct ubifs_info *c, struct ubifs_pnode *pnode) +{ + ubifs_add_lpt_dirt(c, pnode->parent->nbranch[pnode->iip].lnum, + c->pnode_sz); +} + +/** + * calc_nnode_num - calculate nnode number. + * @row: the row in the tree (root is zero) + * @col: the column in the row (leftmost is zero) + * + * The nnode number is a number that uniquely identifies a nnode and can be used + * easily to traverse the tree from the root to that nnode. + * + * This function calculates and returns the nnode number for the nnode at @row + * and @col. + */ +static int calc_nnode_num(int row, int col) +{ + int num, bits; + + num = 1; + while (row--) { + bits = (col & (UBIFS_LPT_FANOUT - 1)); + col >>= UBIFS_LPT_FANOUT_SHIFT; + num <<= UBIFS_LPT_FANOUT_SHIFT; + num |= bits; + } + return num; +} + +/** + * calc_nnode_num_from_parent - calculate nnode number. + * @c: UBIFS file-system description object + * @parent: parent nnode + * @iip: index in parent + * + * The nnode number is a number that uniquely identifies a nnode and can be used + * easily to traverse the tree from the root to that nnode. + * + * This function calculates and returns the nnode number based on the parent's + * nnode number and the index in parent. + */ +static int calc_nnode_num_from_parent(const struct ubifs_info *c, + struct ubifs_nnode *parent, int iip) +{ + int num, shft; + + if (!parent) + return 1; + shft = (c->lpt_hght - parent->level) * UBIFS_LPT_FANOUT_SHIFT; + num = parent->num ^ (1 << shft); + num |= (UBIFS_LPT_FANOUT + iip) << shft; + return num; +} + +/** + * calc_pnode_num_from_parent - calculate pnode number. + * @c: UBIFS file-system description object + * @parent: parent nnode + * @iip: index in parent + * + * The pnode number is a number that uniquely identifies a pnode and can be used + * easily to traverse the tree from the root to that pnode. + * + * This function calculates and returns the pnode number based on the parent's + * nnode number and the index in parent. + */ +static int calc_pnode_num_from_parent(const struct ubifs_info *c, + struct ubifs_nnode *parent, int iip) +{ + int i, n = c->lpt_hght - 1, pnum = parent->num, num = 0; + + for (i = 0; i < n; i++) { + num <<= UBIFS_LPT_FANOUT_SHIFT; + num |= pnum & (UBIFS_LPT_FANOUT - 1); + pnum >>= UBIFS_LPT_FANOUT_SHIFT; + } + num <<= UBIFS_LPT_FANOUT_SHIFT; + num |= iip; + return num; +} + +/** + * ubifs_create_dflt_lpt - create default LPT. + * @c: UBIFS file-system description object + * @main_lebs: number of main area LEBs is passed and returned here + * @lpt_first: LEB number of first LPT LEB + * @lpt_lebs: number of LEBs for LPT is passed and returned here + * @big_lpt: use big LPT model is passed and returned here + * @hash: hash of the LPT is returned here + * + * This function returns %0 on success and a negative error code on failure. + */ +int ubifs_create_dflt_lpt(struct ubifs_info *c, int *main_lebs, int lpt_first, + int *lpt_lebs, int *big_lpt, u8 *hash) +{ + int lnum, err = 0, node_sz, iopos, i, j, cnt, len, alen, row; + int blnum, boffs, bsz, bcnt; + struct ubifs_pnode *pnode = NULL; + struct ubifs_nnode *nnode = NULL; + void *buf = NULL, *p; + struct ubifs_lpt_lprops *ltab = NULL; + int *lsave = NULL; + struct shash_desc *desc; + + err = calc_dflt_lpt_geom(c, main_lebs, big_lpt); + if (err) + return err; + *lpt_lebs = c->lpt_lebs; + + /* Needed by 'ubifs_pack_nnode()' and 'set_ltab()' */ + c->lpt_first = lpt_first; + /* Needed by 'set_ltab()' */ + c->lpt_last = lpt_first + c->lpt_lebs - 1; + /* Needed by 'ubifs_pack_lsave()' */ + c->main_first = c->leb_cnt - *main_lebs; + + desc = ubifs_hash_get_desc(c); + if (IS_ERR(desc)) + return PTR_ERR(desc); + + lsave = kmalloc_objs(int, c->lsave_cnt); + pnode = kzalloc_obj(struct ubifs_pnode); + nnode = kzalloc_obj(struct ubifs_nnode); + buf = vmalloc(c->leb_size); + ltab = vmalloc_array(c->lpt_lebs, + sizeof(struct ubifs_lpt_lprops)); + if (!pnode || !nnode || !buf || !ltab || !lsave) { + err = -ENOMEM; + goto out; + } + + ubifs_assert(c, !c->ltab); + c->ltab = ltab; /* Needed by set_ltab */ + + /* Initialize LPT's own lprops */ + for (i = 0; i < c->lpt_lebs; i++) { + ltab[i].free = c->leb_size; + ltab[i].dirty = 0; + ltab[i].tgc = 0; + ltab[i].cmt = 0; + } + + lnum = lpt_first; + p = buf; + /* Number of leaf nodes (pnodes) */ + cnt = c->pnode_cnt; + + /* + * The first pnode contains the LEB properties for the LEBs that contain + * the root inode node and the root index node of the index tree. + */ + node_sz = ALIGN(ubifs_idx_node_sz(c, 1), 8); + iopos = ALIGN(node_sz, c->min_io_size); + pnode->lprops[0].free = c->leb_size - iopos; + pnode->lprops[0].dirty = iopos - node_sz; + pnode->lprops[0].flags = LPROPS_INDEX; + + node_sz = UBIFS_INO_NODE_SZ; + iopos = ALIGN(node_sz, c->min_io_size); + pnode->lprops[1].free = c->leb_size - iopos; + pnode->lprops[1].dirty = iopos - node_sz; + + for (i = 2; i < UBIFS_LPT_FANOUT; i++) + pnode->lprops[i].free = c->leb_size; + + /* Add first pnode */ + ubifs_pack_pnode(c, p, pnode); + err = ubifs_shash_update(c, desc, p, c->pnode_sz); + if (err) + goto out; + + p += c->pnode_sz; + len = c->pnode_sz; + pnode->num += 1; + + /* Reset pnode values for remaining pnodes */ + pnode->lprops[0].free = c->leb_size; + pnode->lprops[0].dirty = 0; + pnode->lprops[0].flags = 0; + + pnode->lprops[1].free = c->leb_size; + pnode->lprops[1].dirty = 0; + + /* + * To calculate the internal node branches, we keep information about + * the level below. + */ + blnum = lnum; /* LEB number of level below */ + boffs = 0; /* Offset of level below */ + bcnt = cnt; /* Number of nodes in level below */ + bsz = c->pnode_sz; /* Size of nodes in level below */ + + /* Add all remaining pnodes */ + for (i = 1; i < cnt; i++) { + if (len + c->pnode_sz > c->leb_size) { + alen = ALIGN(len, c->min_io_size); + set_ltab(c, lnum, c->leb_size - alen, alen - len); + memset(p, 0xff, alen - len); + err = ubifs_leb_change(c, lnum++, buf, alen); + if (err) + goto out; + p = buf; + len = 0; + } + ubifs_pack_pnode(c, p, pnode); + err = ubifs_shash_update(c, desc, p, c->pnode_sz); + if (err) + goto out; + + p += c->pnode_sz; + len += c->pnode_sz; + /* + * pnodes are simply numbered left to right starting at zero, + * which means the pnode number can be used easily to traverse + * down the tree to the corresponding pnode. + */ + pnode->num += 1; + } + + row = 0; + for (i = UBIFS_LPT_FANOUT; cnt > i; i <<= UBIFS_LPT_FANOUT_SHIFT) + row += 1; + /* Add all nnodes, one level at a time */ + while (1) { + /* Number of internal nodes (nnodes) at next level */ + cnt = DIV_ROUND_UP(cnt, UBIFS_LPT_FANOUT); + for (i = 0; i < cnt; i++) { + if (len + c->nnode_sz > c->leb_size) { + alen = ALIGN(len, c->min_io_size); + set_ltab(c, lnum, c->leb_size - alen, + alen - len); + memset(p, 0xff, alen - len); + err = ubifs_leb_change(c, lnum++, buf, alen); + if (err) + goto out; + p = buf; + len = 0; + } + /* Only 1 nnode at this level, so it is the root */ + if (cnt == 1) { + c->lpt_lnum = lnum; + c->lpt_offs = len; + } + /* Set branches to the level below */ + for (j = 0; j < UBIFS_LPT_FANOUT; j++) { + if (bcnt) { + if (boffs + bsz > c->leb_size) { + blnum += 1; + boffs = 0; + } + nnode->nbranch[j].lnum = blnum; + nnode->nbranch[j].offs = boffs; + boffs += bsz; + bcnt--; + } else { + nnode->nbranch[j].lnum = 0; + nnode->nbranch[j].offs = 0; + } + } + nnode->num = calc_nnode_num(row, i); + ubifs_pack_nnode(c, p, nnode); + p += c->nnode_sz; + len += c->nnode_sz; + } + /* Only 1 nnode at this level, so it is the root */ + if (cnt == 1) + break; + /* Update the information about the level below */ + bcnt = cnt; + bsz = c->nnode_sz; + row -= 1; + } + + if (*big_lpt) { + /* Need to add LPT's save table */ + if (len + c->lsave_sz > c->leb_size) { + alen = ALIGN(len, c->min_io_size); + set_ltab(c, lnum, c->leb_size - alen, alen - len); + memset(p, 0xff, alen - len); + err = ubifs_leb_change(c, lnum++, buf, alen); + if (err) + goto out; + p = buf; + len = 0; + } + + c->lsave_lnum = lnum; + c->lsave_offs = len; + + for (i = 0; i < c->lsave_cnt && i < *main_lebs; i++) + lsave[i] = c->main_first + i; + for (; i < c->lsave_cnt; i++) + lsave[i] = c->main_first; + + ubifs_pack_lsave(c, p, lsave); + p += c->lsave_sz; + len += c->lsave_sz; + } + + /* Need to add LPT's own LEB properties table */ + if (len + c->ltab_sz > c->leb_size) { + alen = ALIGN(len, c->min_io_size); + set_ltab(c, lnum, c->leb_size - alen, alen - len); + memset(p, 0xff, alen - len); + err = ubifs_leb_change(c, lnum++, buf, alen); + if (err) + goto out; + p = buf; + len = 0; + } + + c->ltab_lnum = lnum; + c->ltab_offs = len; + + /* Update ltab before packing it */ + len += c->ltab_sz; + alen = ALIGN(len, c->min_io_size); + set_ltab(c, lnum, c->leb_size - alen, alen - len); + + ubifs_pack_ltab(c, p, ltab); + p += c->ltab_sz; + + /* Write remaining buffer */ + memset(p, 0xff, alen - len); + err = ubifs_leb_change(c, lnum, buf, alen); + if (err) + goto out; + + err = ubifs_shash_final(c, desc, hash); + if (err) + goto out; + + c->nhead_lnum = lnum; + c->nhead_offs = ALIGN(len, c->min_io_size); + + dbg_lp("space_bits %d", c->space_bits); + dbg_lp("lpt_lnum_bits %d", c->lpt_lnum_bits); + dbg_lp("lpt_offs_bits %d", c->lpt_offs_bits); + dbg_lp("lpt_spc_bits %d", c->lpt_spc_bits); + dbg_lp("pcnt_bits %d", c->pcnt_bits); + dbg_lp("lnum_bits %d", c->lnum_bits); + dbg_lp("pnode_sz %d", c->pnode_sz); + dbg_lp("nnode_sz %d", c->nnode_sz); + dbg_lp("ltab_sz %d", c->ltab_sz); + dbg_lp("lsave_sz %d", c->lsave_sz); + dbg_lp("lsave_cnt %d", c->lsave_cnt); + dbg_lp("lpt_hght %d", c->lpt_hght); + dbg_lp("big_lpt %u", c->big_lpt); + dbg_lp("LPT root is at %d:%d", c->lpt_lnum, c->lpt_offs); + dbg_lp("LPT head is at %d:%d", c->nhead_lnum, c->nhead_offs); + dbg_lp("LPT ltab is at %d:%d", c->ltab_lnum, c->ltab_offs); + if (c->big_lpt) + dbg_lp("LPT lsave is at %d:%d", c->lsave_lnum, c->lsave_offs); +out: + c->ltab = NULL; + kfree(desc); + kfree(lsave); + vfree(ltab); + vfree(buf); + kfree(nnode); + kfree(pnode); + return err; +} + +/** + * update_cats - add LEB properties of a pnode to LEB category lists and heaps. + * @c: UBIFS file-system description object + * @pnode: pnode + * + * When a pnode is loaded into memory, the LEB properties it contains are added, + * by this function, to the LEB category lists and heaps. + */ +static void update_cats(struct ubifs_info *c, struct ubifs_pnode *pnode) +{ + int i; + + for (i = 0; i < UBIFS_LPT_FANOUT; i++) { + int cat = pnode->lprops[i].flags & LPROPS_CAT_MASK; + int lnum = pnode->lprops[i].lnum; + + if (!lnum) + return; + ubifs_add_to_cat(c, &pnode->lprops[i], cat); + } +} + +/** + * replace_cats - add LEB properties of a pnode to LEB category lists and heaps. + * @c: UBIFS file-system description object + * @old_pnode: pnode copied + * @new_pnode: pnode copy + * + * During commit it is sometimes necessary to copy a pnode + * (see dirty_cow_pnode). When that happens, references in + * category lists and heaps must be replaced. This function does that. + */ +static void replace_cats(struct ubifs_info *c, struct ubifs_pnode *old_pnode, + struct ubifs_pnode *new_pnode) +{ + int i; + + for (i = 0; i < UBIFS_LPT_FANOUT; i++) { + if (!new_pnode->lprops[i].lnum) + return; + ubifs_replace_cat(c, &old_pnode->lprops[i], + &new_pnode->lprops[i]); + } +} + +/** + * check_lpt_crc - check LPT node crc is correct. + * @c: UBIFS file-system description object + * @buf: buffer containing node + * @len: length of node + * + * This function returns %0 on success and a negative error code on failure. + */ +static int check_lpt_crc(const struct ubifs_info *c, void *buf, int len) +{ + int pos = 0; + uint8_t *addr = buf; + uint16_t crc, calc_crc; + + crc = ubifs_unpack_bits(c, &addr, &pos, UBIFS_LPT_CRC_BITS); + calc_crc = crc16(-1, buf + UBIFS_LPT_CRC_BYTES, + len - UBIFS_LPT_CRC_BYTES); + if (crc != calc_crc) { + ubifs_err(c, "invalid crc in LPT node: crc %hx calc %hx", + crc, calc_crc); + dump_stack(); + return -EINVAL; + } + return 0; +} + +/** + * check_lpt_type - check LPT node type is correct. + * @c: UBIFS file-system description object + * @addr: address of type bit field is passed and returned updated here + * @pos: position of type bit field is passed and returned updated here + * @type: expected type + * + * This function returns %0 on success and a negative error code on failure. + */ +static int check_lpt_type(const struct ubifs_info *c, uint8_t **addr, + int *pos, int type) +{ + int node_type; + + node_type = ubifs_unpack_bits(c, addr, pos, UBIFS_LPT_TYPE_BITS); + if (node_type != type) { + ubifs_err(c, "invalid type (%d) in LPT node type %d", + node_type, type); + dump_stack(); + return -EINVAL; + } + return 0; +} + +/** + * unpack_pnode - unpack a pnode. + * @c: UBIFS file-system description object + * @buf: buffer containing packed pnode to unpack + * @pnode: pnode structure to fill + * + * This function returns %0 on success and a negative error code on failure. + */ +static int unpack_pnode(const struct ubifs_info *c, void *buf, + struct ubifs_pnode *pnode) +{ + uint8_t *addr = buf + UBIFS_LPT_CRC_BYTES; + int i, pos = 0, err; + + err = check_lpt_type(c, &addr, &pos, UBIFS_LPT_PNODE); + if (err) + return err; + if (c->big_lpt) + pnode->num = ubifs_unpack_bits(c, &addr, &pos, c->pcnt_bits); + for (i = 0; i < UBIFS_LPT_FANOUT; i++) { + struct ubifs_lprops * const lprops = &pnode->lprops[i]; + + lprops->free = ubifs_unpack_bits(c, &addr, &pos, c->space_bits); + lprops->free <<= 3; + lprops->dirty = ubifs_unpack_bits(c, &addr, &pos, c->space_bits); + lprops->dirty <<= 3; + + if (ubifs_unpack_bits(c, &addr, &pos, 1)) + lprops->flags = LPROPS_INDEX; + else + lprops->flags = 0; + lprops->flags |= ubifs_categorize_lprops(c, lprops); + } + err = check_lpt_crc(c, buf, c->pnode_sz); + return err; +} + +/** + * ubifs_unpack_nnode - unpack a nnode. + * @c: UBIFS file-system description object + * @buf: buffer containing packed nnode to unpack + * @nnode: nnode structure to fill + * + * This function returns %0 on success and a negative error code on failure. + */ +int ubifs_unpack_nnode(const struct ubifs_info *c, void *buf, + struct ubifs_nnode *nnode) +{ + uint8_t *addr = buf + UBIFS_LPT_CRC_BYTES; + int i, pos = 0, err; + + err = check_lpt_type(c, &addr, &pos, UBIFS_LPT_NNODE); + if (err) + return err; + if (c->big_lpt) + nnode->num = ubifs_unpack_bits(c, &addr, &pos, c->pcnt_bits); + for (i = 0; i < UBIFS_LPT_FANOUT; i++) { + int lnum; + + lnum = ubifs_unpack_bits(c, &addr, &pos, c->lpt_lnum_bits) + + c->lpt_first; + if (lnum == c->lpt_last + 1) + lnum = 0; + nnode->nbranch[i].lnum = lnum; + nnode->nbranch[i].offs = ubifs_unpack_bits(c, &addr, &pos, + c->lpt_offs_bits); + } + err = check_lpt_crc(c, buf, c->nnode_sz); + return err; +} + +/** + * unpack_ltab - unpack the LPT's own lprops table. + * @c: UBIFS file-system description object + * @buf: buffer from which to unpack + * + * This function returns %0 on success and a negative error code on failure. + */ +static int unpack_ltab(const struct ubifs_info *c, void *buf) +{ + uint8_t *addr = buf + UBIFS_LPT_CRC_BYTES; + int i, pos = 0, err; + + err = check_lpt_type(c, &addr, &pos, UBIFS_LPT_LTAB); + if (err) + return err; + for (i = 0; i < c->lpt_lebs; i++) { + int free = ubifs_unpack_bits(c, &addr, &pos, c->lpt_spc_bits); + int dirty = ubifs_unpack_bits(c, &addr, &pos, c->lpt_spc_bits); + + if (free < 0 || free > c->leb_size || dirty < 0 || + dirty > c->leb_size || free + dirty > c->leb_size) + return -EINVAL; + + c->ltab[i].free = free; + c->ltab[i].dirty = dirty; + c->ltab[i].tgc = 0; + c->ltab[i].cmt = 0; + } + err = check_lpt_crc(c, buf, c->ltab_sz); + return err; +} + +/** + * unpack_lsave - unpack the LPT's save table. + * @c: UBIFS file-system description object + * @buf: buffer from which to unpack + * + * This function returns %0 on success and a negative error code on failure. + */ +static int unpack_lsave(const struct ubifs_info *c, void *buf) +{ + uint8_t *addr = buf + UBIFS_LPT_CRC_BYTES; + int i, pos = 0, err; + + err = check_lpt_type(c, &addr, &pos, UBIFS_LPT_LSAVE); + if (err) + return err; + for (i = 0; i < c->lsave_cnt; i++) { + int lnum = ubifs_unpack_bits(c, &addr, &pos, c->lnum_bits); + + if (lnum < c->main_first || lnum >= c->leb_cnt) + return -EINVAL; + c->lsave[i] = lnum; + } + err = check_lpt_crc(c, buf, c->lsave_sz); + return err; +} + +/** + * validate_nnode - validate a nnode. + * @c: UBIFS file-system description object + * @nnode: nnode to validate + * @parent: parent nnode (or NULL for the root nnode) + * @iip: index in parent + * + * This function returns %0 on success and a negative error code on failure. + */ +static int validate_nnode(const struct ubifs_info *c, struct ubifs_nnode *nnode, + struct ubifs_nnode *parent, int iip) +{ + int i, lvl, max_offs; + + if (c->big_lpt) { + int num = calc_nnode_num_from_parent(c, parent, iip); + + if (nnode->num != num) + return -EINVAL; + } + lvl = parent ? parent->level - 1 : c->lpt_hght; + if (lvl < 1) + return -EINVAL; + if (lvl == 1) + max_offs = c->leb_size - c->pnode_sz; + else + max_offs = c->leb_size - c->nnode_sz; + for (i = 0; i < UBIFS_LPT_FANOUT; i++) { + int lnum = nnode->nbranch[i].lnum; + int offs = nnode->nbranch[i].offs; + + if (lnum == 0) { + if (offs != 0) + return -EINVAL; + continue; + } + if (lnum < c->lpt_first || lnum > c->lpt_last) + return -EINVAL; + if (offs < 0 || offs > max_offs) + return -EINVAL; + } + return 0; +} + +/** + * validate_pnode - validate a pnode. + * @c: UBIFS file-system description object + * @pnode: pnode to validate + * @parent: parent nnode + * @iip: index in parent + * + * This function returns %0 on success and a negative error code on failure. + */ +static int validate_pnode(const struct ubifs_info *c, struct ubifs_pnode *pnode, + struct ubifs_nnode *parent, int iip) +{ + int i; + + if (c->big_lpt) { + int num = calc_pnode_num_from_parent(c, parent, iip); + + if (pnode->num != num) + return -EINVAL; + } + for (i = 0; i < UBIFS_LPT_FANOUT; i++) { + int free = pnode->lprops[i].free; + int dirty = pnode->lprops[i].dirty; + + if (free < 0 || free > c->leb_size || free % c->min_io_size || + (free & 7)) + return -EINVAL; + if (dirty < 0 || dirty > c->leb_size || (dirty & 7)) + return -EINVAL; + if (dirty + free > c->leb_size) + return -EINVAL; + } + return 0; +} + +/** + * set_pnode_lnum - set LEB numbers on a pnode. + * @c: UBIFS file-system description object + * @pnode: pnode to update + * + * This function calculates the LEB numbers for the LEB properties it contains + * based on the pnode number. + */ +static void set_pnode_lnum(const struct ubifs_info *c, + struct ubifs_pnode *pnode) +{ + int i, lnum; + + lnum = (pnode->num << UBIFS_LPT_FANOUT_SHIFT) + c->main_first; + for (i = 0; i < UBIFS_LPT_FANOUT; i++) { + if (lnum >= c->leb_cnt) + return; + pnode->lprops[i].lnum = lnum++; + } +} + +/** + * ubifs_read_nnode - read a nnode from flash and link it to the tree in memory. + * @c: UBIFS file-system description object + * @parent: parent nnode (or NULL for the root) + * @iip: index in parent + * + * This function returns %0 on success and a negative error code on failure. + */ +int ubifs_read_nnode(struct ubifs_info *c, struct ubifs_nnode *parent, int iip) +{ + struct ubifs_nbranch *branch = NULL; + struct ubifs_nnode *nnode = NULL; + void *buf = c->lpt_nod_buf; + int err, lnum, offs; + + if (parent) { + branch = &parent->nbranch[iip]; + lnum = branch->lnum; + offs = branch->offs; + } else { + lnum = c->lpt_lnum; + offs = c->lpt_offs; + } + nnode = kzalloc_obj(struct ubifs_nnode, GFP_NOFS); + if (!nnode) { + err = -ENOMEM; + goto out; + } + if (lnum == 0) { + /* + * This nnode was not written which just means that the LEB + * properties in the subtree below it describe empty LEBs. We + * make the nnode as though we had read it, which in fact means + * doing almost nothing. + */ + if (c->big_lpt) + nnode->num = calc_nnode_num_from_parent(c, parent, iip); + } else { + err = ubifs_leb_read(c, lnum, buf, offs, c->nnode_sz, 1); + if (err) + goto out; + err = ubifs_unpack_nnode(c, buf, nnode); + if (err) + goto out; + } + err = validate_nnode(c, nnode, parent, iip); + if (err) + goto out; + if (!c->big_lpt) + nnode->num = calc_nnode_num_from_parent(c, parent, iip); + if (parent) { + branch->nnode = nnode; + nnode->level = parent->level - 1; + } else { + c->nroot = nnode; + nnode->level = c->lpt_hght; + } + nnode->parent = parent; + nnode->iip = iip; + return 0; + +out: + ubifs_err(c, "error %d reading nnode at %d:%d", err, lnum, offs); + dump_stack(); + kfree(nnode); + return err; +} + +/** + * read_pnode - read a pnode from flash and link it to the tree in memory. + * @c: UBIFS file-system description object + * @parent: parent nnode + * @iip: index in parent + * + * This function returns %0 on success and a negative error code on failure. + */ +static int read_pnode(struct ubifs_info *c, struct ubifs_nnode *parent, int iip) +{ + struct ubifs_nbranch *branch; + struct ubifs_pnode *pnode = NULL; + void *buf = c->lpt_nod_buf; + int err, lnum, offs; + + branch = &parent->nbranch[iip]; + lnum = branch->lnum; + offs = branch->offs; + pnode = kzalloc_obj(struct ubifs_pnode, GFP_NOFS); + if (!pnode) + return -ENOMEM; + + if (lnum == 0) { + /* + * This pnode was not written which just means that the LEB + * properties in it describe empty LEBs. We make the pnode as + * though we had read it. + */ + int i; + + if (c->big_lpt) + pnode->num = calc_pnode_num_from_parent(c, parent, iip); + for (i = 0; i < UBIFS_LPT_FANOUT; i++) { + struct ubifs_lprops * const lprops = &pnode->lprops[i]; + + lprops->free = c->leb_size; + lprops->flags = ubifs_categorize_lprops(c, lprops); + } + } else { + err = ubifs_leb_read(c, lnum, buf, offs, c->pnode_sz, 1); + if (err) + goto out; + err = unpack_pnode(c, buf, pnode); + if (err) + goto out; + } + err = validate_pnode(c, pnode, parent, iip); + if (err) + goto out; + if (!c->big_lpt) + pnode->num = calc_pnode_num_from_parent(c, parent, iip); + branch->pnode = pnode; + pnode->parent = parent; + pnode->iip = iip; + set_pnode_lnum(c, pnode); + c->pnodes_have += 1; + return 0; + +out: + ubifs_err(c, "error %d reading pnode at %d:%d", err, lnum, offs); + ubifs_dump_pnode(c, pnode, parent, iip); + dump_stack(); + ubifs_err(c, "calc num: %d", calc_pnode_num_from_parent(c, parent, iip)); + kfree(pnode); + return err; +} + +/** + * read_ltab - read LPT's own lprops table. + * @c: UBIFS file-system description object + * + * This function returns %0 on success and a negative error code on failure. + */ +static int read_ltab(struct ubifs_info *c) +{ + int err; + void *buf; + + buf = vmalloc(c->ltab_sz); + if (!buf) + return -ENOMEM; + err = ubifs_leb_read(c, c->ltab_lnum, buf, c->ltab_offs, c->ltab_sz, 1); + if (err) + goto out; + err = unpack_ltab(c, buf); +out: + vfree(buf); + return err; +} + +/** + * read_lsave - read LPT's save table. + * @c: UBIFS file-system description object + * + * This function returns %0 on success and a negative error code on failure. + */ +static int read_lsave(struct ubifs_info *c) +{ + int err, i; + void *buf; + + buf = vmalloc(c->lsave_sz); + if (!buf) + return -ENOMEM; + err = ubifs_leb_read(c, c->lsave_lnum, buf, c->lsave_offs, + c->lsave_sz, 1); + if (err) + goto out; + err = unpack_lsave(c, buf); + if (err) + goto out; + for (i = 0; i < c->lsave_cnt; i++) { + int lnum = c->lsave[i]; + struct ubifs_lprops *lprops; + + /* + * Due to automatic resizing, the values in the lsave table + * could be beyond the volume size - just ignore them. + */ + if (lnum >= c->leb_cnt) + continue; + lprops = ubifs_lpt_lookup(c, lnum); + if (IS_ERR(lprops)) { + err = PTR_ERR(lprops); + goto out; + } + } +out: + vfree(buf); + return err; +} + +/** + * ubifs_get_nnode - get a nnode. + * @c: UBIFS file-system description object + * @parent: parent nnode (or NULL for the root) + * @iip: index in parent + * + * This function returns a pointer to the nnode on success or a negative error + * code on failure. + */ +struct ubifs_nnode *ubifs_get_nnode(struct ubifs_info *c, + struct ubifs_nnode *parent, int iip) +{ + struct ubifs_nbranch *branch; + struct ubifs_nnode *nnode; + int err; + + branch = &parent->nbranch[iip]; + nnode = branch->nnode; + if (nnode) + return nnode; + err = ubifs_read_nnode(c, parent, iip); + if (err) + return ERR_PTR(err); + return branch->nnode; +} + +/** + * ubifs_get_pnode - get a pnode. + * @c: UBIFS file-system description object + * @parent: parent nnode + * @iip: index in parent + * + * This function returns a pointer to the pnode on success or a negative error + * code on failure. + */ +struct ubifs_pnode *ubifs_get_pnode(struct ubifs_info *c, + struct ubifs_nnode *parent, int iip) +{ + struct ubifs_nbranch *branch; + struct ubifs_pnode *pnode; + int err; + + branch = &parent->nbranch[iip]; + pnode = branch->pnode; + if (pnode) + return pnode; + err = read_pnode(c, parent, iip); + if (err) + return ERR_PTR(err); + update_cats(c, branch->pnode); + return branch->pnode; +} + +/** + * ubifs_pnode_lookup - lookup a pnode in the LPT. + * @c: UBIFS file-system description object + * @i: pnode number (0 to (main_lebs - 1) / UBIFS_LPT_FANOUT) + * + * This function returns a pointer to the pnode on success or a negative + * error code on failure. + */ +struct ubifs_pnode *ubifs_pnode_lookup(struct ubifs_info *c, int i) +{ + int err, h, iip, shft; + struct ubifs_nnode *nnode; + + if (!c->nroot) { + err = ubifs_read_nnode(c, NULL, 0); + if (err) + return ERR_PTR(err); + } + i <<= UBIFS_LPT_FANOUT_SHIFT; + nnode = c->nroot; + shft = c->lpt_hght * UBIFS_LPT_FANOUT_SHIFT; + for (h = 1; h < c->lpt_hght; h++) { + iip = ((i >> shft) & (UBIFS_LPT_FANOUT - 1)); + shft -= UBIFS_LPT_FANOUT_SHIFT; + nnode = ubifs_get_nnode(c, nnode, iip); + if (IS_ERR(nnode)) + return ERR_CAST(nnode); + } + iip = ((i >> shft) & (UBIFS_LPT_FANOUT - 1)); + return ubifs_get_pnode(c, nnode, iip); +} + +/** + * ubifs_lpt_lookup - lookup LEB properties in the LPT. + * @c: UBIFS file-system description object + * @lnum: LEB number to lookup + * + * This function returns a pointer to the LEB properties on success or a + * negative error code on failure. + */ +struct ubifs_lprops *ubifs_lpt_lookup(struct ubifs_info *c, int lnum) +{ + int i, iip; + struct ubifs_pnode *pnode; + + i = lnum - c->main_first; + pnode = ubifs_pnode_lookup(c, i >> UBIFS_LPT_FANOUT_SHIFT); + if (IS_ERR(pnode)) + return ERR_CAST(pnode); + iip = (i & (UBIFS_LPT_FANOUT - 1)); + dbg_lp("LEB %d, free %d, dirty %d, flags %d", lnum, + pnode->lprops[iip].free, pnode->lprops[iip].dirty, + pnode->lprops[iip].flags); + return &pnode->lprops[iip]; +} + +/** + * dirty_cow_nnode - ensure a nnode is not being committed. + * @c: UBIFS file-system description object + * @nnode: nnode to check + * + * Returns dirtied nnode on success or negative error code on failure. + */ +static struct ubifs_nnode *dirty_cow_nnode(struct ubifs_info *c, + struct ubifs_nnode *nnode) +{ + struct ubifs_nnode *n; + int i; + + if (!test_bit(COW_CNODE, &nnode->flags)) { + /* nnode is not being committed */ + if (!test_and_set_bit(DIRTY_CNODE, &nnode->flags)) { + c->dirty_nn_cnt += 1; + ubifs_add_nnode_dirt(c, nnode); + } + return nnode; + } + + /* nnode is being committed, so copy it */ + n = kmemdup(nnode, sizeof(struct ubifs_nnode), GFP_NOFS); + if (unlikely(!n)) + return ERR_PTR(-ENOMEM); + + n->cnext = NULL; + __set_bit(DIRTY_CNODE, &n->flags); + __clear_bit(COW_CNODE, &n->flags); + + /* The children now have new parent */ + for (i = 0; i < UBIFS_LPT_FANOUT; i++) { + struct ubifs_nbranch *branch = &n->nbranch[i]; + + if (branch->cnode) + branch->cnode->parent = n; + } + + ubifs_assert(c, !test_bit(OBSOLETE_CNODE, &nnode->flags)); + __set_bit(OBSOLETE_CNODE, &nnode->flags); + + c->dirty_nn_cnt += 1; + ubifs_add_nnode_dirt(c, nnode); + if (nnode->parent) + nnode->parent->nbranch[n->iip].nnode = n; + else + c->nroot = n; + return n; +} + +/** + * dirty_cow_pnode - ensure a pnode is not being committed. + * @c: UBIFS file-system description object + * @pnode: pnode to check + * + * Returns dirtied pnode on success or negative error code on failure. + */ +static struct ubifs_pnode *dirty_cow_pnode(struct ubifs_info *c, + struct ubifs_pnode *pnode) +{ + struct ubifs_pnode *p; + + if (!test_bit(COW_CNODE, &pnode->flags)) { + /* pnode is not being committed */ + if (!test_and_set_bit(DIRTY_CNODE, &pnode->flags)) { + c->dirty_pn_cnt += 1; + add_pnode_dirt(c, pnode); + } + return pnode; + } + + /* pnode is being committed, so copy it */ + p = kmemdup(pnode, sizeof(struct ubifs_pnode), GFP_NOFS); + if (unlikely(!p)) + return ERR_PTR(-ENOMEM); + + p->cnext = NULL; + __set_bit(DIRTY_CNODE, &p->flags); + __clear_bit(COW_CNODE, &p->flags); + replace_cats(c, pnode, p); + + ubifs_assert(c, !test_bit(OBSOLETE_CNODE, &pnode->flags)); + __set_bit(OBSOLETE_CNODE, &pnode->flags); + + c->dirty_pn_cnt += 1; + add_pnode_dirt(c, pnode); + pnode->parent->nbranch[p->iip].pnode = p; + return p; +} + +/** + * ubifs_lpt_lookup_dirty - lookup LEB properties in the LPT. + * @c: UBIFS file-system description object + * @lnum: LEB number to lookup + * + * This function returns a pointer to the LEB properties on success or a + * negative error code on failure. + */ +struct ubifs_lprops *ubifs_lpt_lookup_dirty(struct ubifs_info *c, int lnum) +{ + int err, i, h, iip, shft; + struct ubifs_nnode *nnode; + struct ubifs_pnode *pnode; + + if (!c->nroot) { + err = ubifs_read_nnode(c, NULL, 0); + if (err) + return ERR_PTR(err); + } + nnode = c->nroot; + nnode = dirty_cow_nnode(c, nnode); + if (IS_ERR(nnode)) + return ERR_CAST(nnode); + i = lnum - c->main_first; + shft = c->lpt_hght * UBIFS_LPT_FANOUT_SHIFT; + for (h = 1; h < c->lpt_hght; h++) { + iip = ((i >> shft) & (UBIFS_LPT_FANOUT - 1)); + shft -= UBIFS_LPT_FANOUT_SHIFT; + nnode = ubifs_get_nnode(c, nnode, iip); + if (IS_ERR(nnode)) + return ERR_CAST(nnode); + nnode = dirty_cow_nnode(c, nnode); + if (IS_ERR(nnode)) + return ERR_CAST(nnode); + } + iip = ((i >> shft) & (UBIFS_LPT_FANOUT - 1)); + pnode = ubifs_get_pnode(c, nnode, iip); + if (IS_ERR(pnode)) + return ERR_CAST(pnode); + pnode = dirty_cow_pnode(c, pnode); + if (IS_ERR(pnode)) + return ERR_CAST(pnode); + iip = (i & (UBIFS_LPT_FANOUT - 1)); + dbg_lp("LEB %d, free %d, dirty %d, flags %d", lnum, + pnode->lprops[iip].free, pnode->lprops[iip].dirty, + pnode->lprops[iip].flags); + ubifs_assert(c, test_bit(DIRTY_CNODE, &pnode->flags)); + return &pnode->lprops[iip]; +} + +/** + * ubifs_lpt_calc_hash - Calculate hash of the LPT pnodes + * @c: UBIFS file-system description object + * @hash: the returned hash of the LPT pnodes + * + * This function iterates over the LPT pnodes and creates a hash over them. + * Returns 0 for success or a negative error code otherwise. + */ +int ubifs_lpt_calc_hash(struct ubifs_info *c, u8 *hash) +{ + struct ubifs_nnode *nnode, *nn; + struct ubifs_cnode *cnode; + struct shash_desc *desc; + int iip = 0, i; + int bufsiz = max_t(int, c->nnode_sz, c->pnode_sz); + void *buf; + int err; + + if (!ubifs_authenticated(c)) + return 0; + + if (!c->nroot) { + err = ubifs_read_nnode(c, NULL, 0); + if (err) + return err; + } + + desc = ubifs_hash_get_desc(c); + if (IS_ERR(desc)) + return PTR_ERR(desc); + + buf = kmalloc(bufsiz, GFP_NOFS); + if (!buf) { + err = -ENOMEM; + goto out; + } + + cnode = (struct ubifs_cnode *)c->nroot; + + while (cnode) { + nnode = cnode->parent; + nn = (struct ubifs_nnode *)cnode; + if (cnode->level > 1) { + while (iip < UBIFS_LPT_FANOUT) { + if (nn->nbranch[iip].lnum == 0) { + /* Go right */ + iip++; + continue; + } + + nnode = ubifs_get_nnode(c, nn, iip); + if (IS_ERR(nnode)) { + err = PTR_ERR(nnode); + goto out; + } + + /* Go down */ + iip = 0; + cnode = (struct ubifs_cnode *)nnode; + break; + } + if (iip < UBIFS_LPT_FANOUT) + continue; + } else { + struct ubifs_pnode *pnode; + + for (i = 0; i < UBIFS_LPT_FANOUT; i++) { + if (nn->nbranch[i].lnum == 0) + continue; + pnode = ubifs_get_pnode(c, nn, i); + if (IS_ERR(pnode)) { + err = PTR_ERR(pnode); + goto out; + } + + ubifs_pack_pnode(c, buf, pnode); + err = ubifs_shash_update(c, desc, buf, + c->pnode_sz); + if (err) + goto out; + } + } + /* Go up and to the right */ + iip = cnode->iip + 1; + cnode = (struct ubifs_cnode *)nnode; + } + + err = ubifs_shash_final(c, desc, hash); +out: + kfree(desc); + kfree(buf); + + return err; +} + +/** + * lpt_check_hash - check the hash of the LPT. + * @c: UBIFS file-system description object + * + * This function calculates a hash over all pnodes in the LPT and compares it with + * the hash stored in the master node. Returns %0 on success and a negative error + * code on failure. + */ +static int lpt_check_hash(struct ubifs_info *c) +{ + int err; + u8 hash[UBIFS_HASH_ARR_SZ]; + + if (!ubifs_authenticated(c)) + return 0; + + err = ubifs_lpt_calc_hash(c, hash); + if (err) + return err; + + if (ubifs_check_hash(c, c->mst_node->hash_lpt, hash)) { + err = -EPERM; + ubifs_err(c, "Failed to authenticate LPT"); + } else { + err = 0; + } + + return err; +} + +/** + * lpt_init_rd - initialize the LPT for reading. + * @c: UBIFS file-system description object + * + * This function returns %0 on success and a negative error code on failure. + */ +static int lpt_init_rd(struct ubifs_info *c) +{ + int err, i; + + c->ltab = vmalloc_array(c->lpt_lebs, + sizeof(struct ubifs_lpt_lprops)); + if (!c->ltab) + return -ENOMEM; + + i = max_t(int, c->nnode_sz, c->pnode_sz); + c->lpt_nod_buf = kmalloc(i, GFP_KERNEL); + if (!c->lpt_nod_buf) + return -ENOMEM; + + for (i = 0; i < LPROPS_HEAP_CNT; i++) { + c->lpt_heap[i].arr = kmalloc_array(LPT_HEAP_SZ, + sizeof(void *), + GFP_KERNEL); + if (!c->lpt_heap[i].arr) + return -ENOMEM; + c->lpt_heap[i].cnt = 0; + c->lpt_heap[i].max_cnt = LPT_HEAP_SZ; + } + + c->dirty_idx.arr = kmalloc_array(LPT_HEAP_SZ, sizeof(void *), + GFP_KERNEL); + if (!c->dirty_idx.arr) + return -ENOMEM; + c->dirty_idx.cnt = 0; + c->dirty_idx.max_cnt = LPT_HEAP_SZ; + + err = read_ltab(c); + if (err) + return err; + + err = lpt_check_hash(c); + if (err) + return err; + + dbg_lp("space_bits %d", c->space_bits); + dbg_lp("lpt_lnum_bits %d", c->lpt_lnum_bits); + dbg_lp("lpt_offs_bits %d", c->lpt_offs_bits); + dbg_lp("lpt_spc_bits %d", c->lpt_spc_bits); + dbg_lp("pcnt_bits %d", c->pcnt_bits); + dbg_lp("lnum_bits %d", c->lnum_bits); + dbg_lp("pnode_sz %d", c->pnode_sz); + dbg_lp("nnode_sz %d", c->nnode_sz); + dbg_lp("ltab_sz %d", c->ltab_sz); + dbg_lp("lsave_sz %d", c->lsave_sz); + dbg_lp("lsave_cnt %d", c->lsave_cnt); + dbg_lp("lpt_hght %d", c->lpt_hght); + dbg_lp("big_lpt %u", c->big_lpt); + dbg_lp("LPT root is at %d:%d", c->lpt_lnum, c->lpt_offs); + dbg_lp("LPT head is at %d:%d", c->nhead_lnum, c->nhead_offs); + dbg_lp("LPT ltab is at %d:%d", c->ltab_lnum, c->ltab_offs); + if (c->big_lpt) + dbg_lp("LPT lsave is at %d:%d", c->lsave_lnum, c->lsave_offs); + + return 0; +} + +/** + * lpt_init_wr - initialize the LPT for writing. + * @c: UBIFS file-system description object + * + * 'lpt_init_rd()' must have been called already. + * + * This function returns %0 on success and a negative error code on failure. + */ +static int lpt_init_wr(struct ubifs_info *c) +{ + int err, i; + + c->ltab_cmt = vmalloc_array(c->lpt_lebs, + sizeof(struct ubifs_lpt_lprops)); + if (!c->ltab_cmt) + return -ENOMEM; + + c->lpt_buf = vmalloc(c->leb_size); + if (!c->lpt_buf) + return -ENOMEM; + + if (c->big_lpt) { + c->lsave = kmalloc_objs(int, c->lsave_cnt, GFP_NOFS); + if (!c->lsave) + return -ENOMEM; + err = read_lsave(c); + if (err) + return err; + } + + for (i = 0; i < c->lpt_lebs; i++) + if (c->ltab[i].free == c->leb_size) { + err = ubifs_leb_unmap(c, i + c->lpt_first); + if (err) + return err; + } + + return 0; +} + +/** + * ubifs_lpt_init - initialize the LPT. + * @c: UBIFS file-system description object + * @rd: whether to initialize lpt for reading + * @wr: whether to initialize lpt for writing + * + * For mounting 'rw', @rd and @wr are both true. For mounting 'ro', @rd is true + * and @wr is false. For mounting from 'ro' to 'rw', @rd is false and @wr is + * true. + * + * This function returns %0 on success and a negative error code on failure. + */ +int ubifs_lpt_init(struct ubifs_info *c, int rd, int wr) +{ + int err; + + if (rd) { + err = lpt_init_rd(c); + if (err) + goto out_err; + } + + if (wr) { + err = lpt_init_wr(c); + if (err) + goto out_err; + } + + return 0; + +out_err: + if (wr) + ubifs_lpt_free(c, 1); + if (rd) + ubifs_lpt_free(c, 0); + return err; +} + +/** + * struct lpt_scan_node - somewhere to put nodes while we scan LPT. + * @nnode: where to keep a nnode + * @pnode: where to keep a pnode + * @cnode: where to keep a cnode + * @in_tree: is the node in the tree in memory + * @ptr: union of node pointers + * @ptr.nnode: pointer to the nnode (if it is an nnode) which may be here or in + * the tree + * @ptr.pnode: ditto for pnode + * @ptr.cnode: ditto for cnode + */ +struct lpt_scan_node { + union { + struct ubifs_nnode nnode; + struct ubifs_pnode pnode; + struct ubifs_cnode cnode; + }; + int in_tree; + union { + struct ubifs_nnode *nnode; + struct ubifs_pnode *pnode; + struct ubifs_cnode *cnode; + } ptr; +}; + +/** + * scan_get_nnode - for the scan, get a nnode from either the tree or flash. + * @c: the UBIFS file-system description object + * @path: where to put the nnode + * @parent: parent of the nnode + * @iip: index in parent of the nnode + * + * This function returns a pointer to the nnode on success or a negative error + * code on failure. + */ +static struct ubifs_nnode *scan_get_nnode(struct ubifs_info *c, + struct lpt_scan_node *path, + struct ubifs_nnode *parent, int iip) +{ + struct ubifs_nbranch *branch; + struct ubifs_nnode *nnode; + void *buf = c->lpt_nod_buf; + int err; + + branch = &parent->nbranch[iip]; + nnode = branch->nnode; + if (nnode) { + path->in_tree = 1; + path->ptr.nnode = nnode; + return nnode; + } + nnode = &path->nnode; + path->in_tree = 0; + path->ptr.nnode = nnode; + memset(nnode, 0, sizeof(struct ubifs_nnode)); + if (branch->lnum == 0) { + /* + * This nnode was not written which just means that the LEB + * properties in the subtree below it describe empty LEBs. We + * make the nnode as though we had read it, which in fact means + * doing almost nothing. + */ + if (c->big_lpt) + nnode->num = calc_nnode_num_from_parent(c, parent, iip); + } else { + err = ubifs_leb_read(c, branch->lnum, buf, branch->offs, + c->nnode_sz, 1); + if (err) + return ERR_PTR(err); + err = ubifs_unpack_nnode(c, buf, nnode); + if (err) + return ERR_PTR(err); + } + err = validate_nnode(c, nnode, parent, iip); + if (err) + return ERR_PTR(err); + if (!c->big_lpt) + nnode->num = calc_nnode_num_from_parent(c, parent, iip); + nnode->level = parent->level - 1; + nnode->parent = parent; + nnode->iip = iip; + return nnode; +} + +/** + * scan_get_pnode - for the scan, get a pnode from either the tree or flash. + * @c: the UBIFS file-system description object + * @path: where to put the pnode + * @parent: parent of the pnode + * @iip: index in parent of the pnode + * + * This function returns a pointer to the pnode on success or a negative error + * code on failure. + */ +static struct ubifs_pnode *scan_get_pnode(struct ubifs_info *c, + struct lpt_scan_node *path, + struct ubifs_nnode *parent, int iip) +{ + struct ubifs_nbranch *branch; + struct ubifs_pnode *pnode; + void *buf = c->lpt_nod_buf; + int err; + + branch = &parent->nbranch[iip]; + pnode = branch->pnode; + if (pnode) { + path->in_tree = 1; + path->ptr.pnode = pnode; + return pnode; + } + pnode = &path->pnode; + path->in_tree = 0; + path->ptr.pnode = pnode; + memset(pnode, 0, sizeof(struct ubifs_pnode)); + if (branch->lnum == 0) { + /* + * This pnode was not written which just means that the LEB + * properties in it describe empty LEBs. We make the pnode as + * though we had read it. + */ + int i; + + if (c->big_lpt) + pnode->num = calc_pnode_num_from_parent(c, parent, iip); + for (i = 0; i < UBIFS_LPT_FANOUT; i++) { + struct ubifs_lprops * const lprops = &pnode->lprops[i]; + + lprops->free = c->leb_size; + lprops->flags = ubifs_categorize_lprops(c, lprops); + } + } else { + ubifs_assert(c, branch->lnum >= c->lpt_first && + branch->lnum <= c->lpt_last); + ubifs_assert(c, branch->offs >= 0 && branch->offs < c->leb_size); + err = ubifs_leb_read(c, branch->lnum, buf, branch->offs, + c->pnode_sz, 1); + if (err) + return ERR_PTR(err); + err = unpack_pnode(c, buf, pnode); + if (err) + return ERR_PTR(err); + } + err = validate_pnode(c, pnode, parent, iip); + if (err) + return ERR_PTR(err); + if (!c->big_lpt) + pnode->num = calc_pnode_num_from_parent(c, parent, iip); + pnode->parent = parent; + pnode->iip = iip; + set_pnode_lnum(c, pnode); + return pnode; +} + +/** + * ubifs_lpt_scan_nolock - scan the LPT. + * @c: the UBIFS file-system description object + * @start_lnum: LEB number from which to start scanning + * @end_lnum: LEB number at which to stop scanning + * @scan_cb: callback function called for each lprops + * @data: data to be passed to the callback function + * + * This function returns %0 on success and a negative error code on failure. + */ +int ubifs_lpt_scan_nolock(struct ubifs_info *c, int start_lnum, int end_lnum, + ubifs_lpt_scan_callback scan_cb, void *data) +{ + int err = 0, i, h, iip, shft; + struct ubifs_nnode *nnode; + struct ubifs_pnode *pnode; + struct lpt_scan_node *path; + + if (start_lnum == -1) { + start_lnum = end_lnum + 1; + if (start_lnum >= c->leb_cnt) + start_lnum = c->main_first; + } + + ubifs_assert(c, start_lnum >= c->main_first && start_lnum < c->leb_cnt); + ubifs_assert(c, end_lnum >= c->main_first && end_lnum < c->leb_cnt); + + if (!c->nroot) { + err = ubifs_read_nnode(c, NULL, 0); + if (err) + return err; + } + + path = kmalloc_objs(struct lpt_scan_node, c->lpt_hght + 1, GFP_NOFS); + if (!path) + return -ENOMEM; + + path[0].ptr.nnode = c->nroot; + path[0].in_tree = 1; +again: + /* Descend to the pnode containing start_lnum */ + nnode = c->nroot; + i = start_lnum - c->main_first; + shft = c->lpt_hght * UBIFS_LPT_FANOUT_SHIFT; + for (h = 1; h < c->lpt_hght; h++) { + iip = ((i >> shft) & (UBIFS_LPT_FANOUT - 1)); + shft -= UBIFS_LPT_FANOUT_SHIFT; + nnode = scan_get_nnode(c, path + h, nnode, iip); + if (IS_ERR(nnode)) { + err = PTR_ERR(nnode); + goto out; + } + } + iip = ((i >> shft) & (UBIFS_LPT_FANOUT - 1)); + pnode = scan_get_pnode(c, path + h, nnode, iip); + if (IS_ERR(pnode)) { + err = PTR_ERR(pnode); + goto out; + } + iip = (i & (UBIFS_LPT_FANOUT - 1)); + + /* Loop for each lprops */ + while (1) { + struct ubifs_lprops *lprops = &pnode->lprops[iip]; + int ret, lnum = lprops->lnum; + + ret = scan_cb(c, lprops, path[h].in_tree, data); + if (ret < 0) { + err = ret; + goto out; + } + if (ret & LPT_SCAN_ADD) { + /* Add all the nodes in path to the tree in memory */ + for (h = 1; h < c->lpt_hght; h++) { + const size_t sz = sizeof(struct ubifs_nnode); + struct ubifs_nnode *parent; + + if (path[h].in_tree) + continue; + nnode = kmemdup(&path[h].nnode, sz, GFP_NOFS); + if (!nnode) { + err = -ENOMEM; + goto out; + } + parent = nnode->parent; + parent->nbranch[nnode->iip].nnode = nnode; + path[h].ptr.nnode = nnode; + path[h].in_tree = 1; + path[h + 1].cnode.parent = nnode; + } + if (path[h].in_tree) + ubifs_ensure_cat(c, lprops); + else { + const size_t sz = sizeof(struct ubifs_pnode); + struct ubifs_nnode *parent; + + pnode = kmemdup(&path[h].pnode, sz, GFP_NOFS); + if (!pnode) { + err = -ENOMEM; + goto out; + } + parent = pnode->parent; + parent->nbranch[pnode->iip].pnode = pnode; + path[h].ptr.pnode = pnode; + path[h].in_tree = 1; + update_cats(c, pnode); + c->pnodes_have += 1; + } + err = dbg_check_lpt_nodes(c, (struct ubifs_cnode *) + c->nroot, 0, 0); + if (err) + goto out; + err = dbg_check_cats(c); + if (err) + goto out; + } + if (ret & LPT_SCAN_STOP) { + err = 0; + break; + } + /* Get the next lprops */ + if (lnum == end_lnum) { + /* + * We got to the end without finding what we were + * looking for + */ + err = -ENOSPC; + goto out; + } + if (lnum + 1 >= c->leb_cnt) { + /* Wrap-around to the beginning */ + start_lnum = c->main_first; + goto again; + } + if (iip + 1 < UBIFS_LPT_FANOUT) { + /* Next lprops is in the same pnode */ + iip += 1; + continue; + } + /* We need to get the next pnode. Go up until we can go right */ + iip = pnode->iip; + while (1) { + h -= 1; + ubifs_assert(c, h >= 0); + nnode = path[h].ptr.nnode; + if (iip + 1 < UBIFS_LPT_FANOUT) + break; + iip = nnode->iip; + } + /* Go right */ + iip += 1; + /* Descend to the pnode */ + h += 1; + for (; h < c->lpt_hght; h++) { + nnode = scan_get_nnode(c, path + h, nnode, iip); + if (IS_ERR(nnode)) { + err = PTR_ERR(nnode); + goto out; + } + iip = 0; + } + pnode = scan_get_pnode(c, path + h, nnode, iip); + if (IS_ERR(pnode)) { + err = PTR_ERR(pnode); + goto out; + } + iip = 0; + } +out: + kfree(path); + return err; +} + +/** + * dbg_chk_pnode - check a pnode. + * @c: the UBIFS file-system description object + * @pnode: pnode to check + * @col: pnode column + * + * This function returns %0 on success and a negative error code on failure. + */ +static int dbg_chk_pnode(struct ubifs_info *c, struct ubifs_pnode *pnode, + int col) +{ + int i; + + if (pnode->num != col) { + ubifs_err(c, "pnode num %d expected %d parent num %d iip %d", + pnode->num, col, pnode->parent->num, pnode->iip); + return -EINVAL; + } + for (i = 0; i < UBIFS_LPT_FANOUT; i++) { + struct ubifs_lprops *lp, *lprops = &pnode->lprops[i]; + int lnum = (pnode->num << UBIFS_LPT_FANOUT_SHIFT) + i + + c->main_first; + int found, cat = lprops->flags & LPROPS_CAT_MASK; + struct ubifs_lpt_heap *heap; + struct list_head *list = NULL; + + if (lnum >= c->leb_cnt) + continue; + if (lprops->lnum != lnum) { + ubifs_err(c, "bad LEB number %d expected %d", + lprops->lnum, lnum); + return -EINVAL; + } + if (lprops->flags & LPROPS_TAKEN) { + if (cat != LPROPS_UNCAT) { + ubifs_err(c, "LEB %d taken but not uncat %d", + lprops->lnum, cat); + return -EINVAL; + } + continue; + } + if (lprops->flags & LPROPS_INDEX) { + switch (cat) { + case LPROPS_UNCAT: + case LPROPS_DIRTY_IDX: + case LPROPS_FRDI_IDX: + break; + default: + ubifs_err(c, "LEB %d index but cat %d", + lprops->lnum, cat); + return -EINVAL; + } + } else { + switch (cat) { + case LPROPS_UNCAT: + case LPROPS_DIRTY: + case LPROPS_FREE: + case LPROPS_EMPTY: + case LPROPS_FREEABLE: + break; + default: + ubifs_err(c, "LEB %d not index but cat %d", + lprops->lnum, cat); + return -EINVAL; + } + } + switch (cat) { + case LPROPS_UNCAT: + list = &c->uncat_list; + break; + case LPROPS_EMPTY: + list = &c->empty_list; + break; + case LPROPS_FREEABLE: + list = &c->freeable_list; + break; + case LPROPS_FRDI_IDX: + list = &c->frdi_idx_list; + break; + } + found = 0; + switch (cat) { + case LPROPS_DIRTY: + case LPROPS_DIRTY_IDX: + case LPROPS_FREE: + heap = &c->lpt_heap[cat - 1]; + if (lprops->hpos < heap->cnt && + heap->arr[lprops->hpos] == lprops) + found = 1; + break; + case LPROPS_UNCAT: + case LPROPS_EMPTY: + case LPROPS_FREEABLE: + case LPROPS_FRDI_IDX: + list_for_each_entry(lp, list, list) + if (lprops == lp) { + found = 1; + break; + } + break; + } + if (!found) { + ubifs_err(c, "LEB %d cat %d not found in cat heap/list", + lprops->lnum, cat); + return -EINVAL; + } + switch (cat) { + case LPROPS_EMPTY: + if (lprops->free != c->leb_size) { + ubifs_err(c, "LEB %d cat %d free %d dirty %d", + lprops->lnum, cat, lprops->free, + lprops->dirty); + return -EINVAL; + } + break; + case LPROPS_FREEABLE: + case LPROPS_FRDI_IDX: + if (lprops->free + lprops->dirty != c->leb_size) { + ubifs_err(c, "LEB %d cat %d free %d dirty %d", + lprops->lnum, cat, lprops->free, + lprops->dirty); + return -EINVAL; + } + break; + } + } + return 0; +} + +/** + * dbg_check_lpt_nodes - check nnodes and pnodes. + * @c: the UBIFS file-system description object + * @cnode: next cnode (nnode or pnode) to check + * @row: row of cnode (root is zero) + * @col: column of cnode (leftmost is zero) + * + * This function returns %0 on success and a negative error code on failure. + */ +int dbg_check_lpt_nodes(struct ubifs_info *c, struct ubifs_cnode *cnode, + int row, int col) +{ + struct ubifs_nnode *nnode, *nn; + struct ubifs_cnode *cn; + int num, iip = 0, err; + + if (!dbg_is_chk_lprops(c)) + return 0; + + while (cnode) { + ubifs_assert(c, row >= 0); + nnode = cnode->parent; + if (cnode->level) { + /* cnode is a nnode */ + num = calc_nnode_num(row, col); + if (cnode->num != num) { + ubifs_err(c, "nnode num %d expected %d parent num %d iip %d", + cnode->num, num, + (nnode ? nnode->num : 0), cnode->iip); + return -EINVAL; + } + nn = (struct ubifs_nnode *)cnode; + while (iip < UBIFS_LPT_FANOUT) { + cn = nn->nbranch[iip].cnode; + if (cn) { + /* Go down */ + row += 1; + col <<= UBIFS_LPT_FANOUT_SHIFT; + col += iip; + iip = 0; + cnode = cn; + break; + } + /* Go right */ + iip += 1; + } + if (iip < UBIFS_LPT_FANOUT) + continue; + } else { + struct ubifs_pnode *pnode; + + /* cnode is a pnode */ + pnode = (struct ubifs_pnode *)cnode; + err = dbg_chk_pnode(c, pnode, col); + if (err) + return err; + } + /* Go up and to the right */ + row -= 1; + col >>= UBIFS_LPT_FANOUT_SHIFT; + iip = cnode->iip + 1; + cnode = (struct ubifs_cnode *)nnode; + } + return 0; +} diff --git a/fs/ubifs/lpt_commit.c b/fs/ubifs/lpt_commit.c new file mode 100644 index 000000000..f2cb21458 --- /dev/null +++ b/fs/ubifs/lpt_commit.c @@ -0,0 +1,1995 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * + * Authors: Adrian Hunter + * Artem Bityutskiy (Битюцкий Артём) + */ + +/* + * This file implements commit-related functionality of the LEB properties + * subsystem. + */ + +#include +#include +#include +#include "ubifs.h" + +static int dbg_populate_lsave(struct ubifs_info *c); + +/** + * first_dirty_cnode - find first dirty cnode. + * @c: UBIFS file-system description object + * @nnode: nnode at which to start + * + * This function returns the first dirty cnode or %NULL if there is not one. + */ +static struct ubifs_cnode *first_dirty_cnode(const struct ubifs_info *c, struct ubifs_nnode *nnode) +{ + ubifs_assert(c, nnode); + while (1) { + int i, cont = 0; + + for (i = 0; i < UBIFS_LPT_FANOUT; i++) { + struct ubifs_cnode *cnode; + + cnode = nnode->nbranch[i].cnode; + if (cnode && + test_bit(DIRTY_CNODE, &cnode->flags)) { + if (cnode->level == 0) + return cnode; + nnode = (struct ubifs_nnode *)cnode; + cont = 1; + break; + } + } + if (!cont) + return (struct ubifs_cnode *)nnode; + } +} + +/** + * next_dirty_cnode - find next dirty cnode. + * @c: UBIFS file-system description object + * @cnode: cnode from which to begin searching + * + * This function returns the next dirty cnode or %NULL if there is not one. + */ +static struct ubifs_cnode *next_dirty_cnode(const struct ubifs_info *c, struct ubifs_cnode *cnode) +{ + struct ubifs_nnode *nnode; + int i; + + ubifs_assert(c, cnode); + nnode = cnode->parent; + if (!nnode) + return NULL; + for (i = cnode->iip + 1; i < UBIFS_LPT_FANOUT; i++) { + cnode = nnode->nbranch[i].cnode; + if (cnode && test_bit(DIRTY_CNODE, &cnode->flags)) { + if (cnode->level == 0) + return cnode; /* cnode is a pnode */ + /* cnode is a nnode */ + return first_dirty_cnode(c, (struct ubifs_nnode *)cnode); + } + } + return (struct ubifs_cnode *)nnode; +} + +/** + * get_cnodes_to_commit - create list of dirty cnodes to commit. + * @c: UBIFS file-system description object + * + * This function returns the number of cnodes to commit. + */ +static int get_cnodes_to_commit(struct ubifs_info *c) +{ + struct ubifs_cnode *cnode, *cnext; + int cnt = 0; + + if (!c->nroot) + return 0; + + if (!test_bit(DIRTY_CNODE, &c->nroot->flags)) + return 0; + + c->lpt_cnext = first_dirty_cnode(c, c->nroot); + cnode = c->lpt_cnext; + if (!cnode) + return 0; + cnt += 1; + while (1) { + ubifs_assert(c, !test_bit(COW_CNODE, &cnode->flags)); + __set_bit(COW_CNODE, &cnode->flags); + cnext = next_dirty_cnode(c, cnode); + if (!cnext) { + cnode->cnext = c->lpt_cnext; + break; + } + cnode->cnext = cnext; + cnode = cnext; + cnt += 1; + } + dbg_cmt("committing %d cnodes", cnt); + dbg_lp("committing %d cnodes", cnt); + ubifs_assert(c, cnt == c->dirty_nn_cnt + c->dirty_pn_cnt); + return cnt; +} + +/** + * upd_ltab - update LPT LEB properties. + * @c: UBIFS file-system description object + * @lnum: LEB number + * @free: amount of free space + * @dirty: amount of dirty space to add + */ +static void upd_ltab(struct ubifs_info *c, int lnum, int free, int dirty) +{ + dbg_lp("LEB %d free %d dirty %d to %d +%d", + lnum, c->ltab[lnum - c->lpt_first].free, + c->ltab[lnum - c->lpt_first].dirty, free, dirty); + ubifs_assert(c, lnum >= c->lpt_first && lnum <= c->lpt_last); + c->ltab[lnum - c->lpt_first].free = free; + c->ltab[lnum - c->lpt_first].dirty += dirty; +} + +/** + * alloc_lpt_leb - allocate an LPT LEB that is empty. + * @c: UBIFS file-system description object + * @lnum: LEB number is passed and returned here + * + * This function finds the next empty LEB in the ltab starting from @lnum. If a + * an empty LEB is found it is returned in @lnum and the function returns %0. + * Otherwise the function returns -ENOSPC. Note however, that LPT is designed + * never to run out of space. + */ +static int alloc_lpt_leb(struct ubifs_info *c, int *lnum) +{ + int i, n; + + n = *lnum - c->lpt_first + 1; + for (i = n; i < c->lpt_lebs; i++) { + if (c->ltab[i].tgc || c->ltab[i].cmt) + continue; + if (c->ltab[i].free == c->leb_size) { + c->ltab[i].cmt = 1; + *lnum = i + c->lpt_first; + return 0; + } + } + + for (i = 0; i < n; i++) { + if (c->ltab[i].tgc || c->ltab[i].cmt) + continue; + if (c->ltab[i].free == c->leb_size) { + c->ltab[i].cmt = 1; + *lnum = i + c->lpt_first; + return 0; + } + } + return -ENOSPC; +} + +/** + * layout_cnodes - layout cnodes for commit. + * @c: UBIFS file-system description object + * + * This function returns %0 on success and a negative error code on failure. + */ +static int layout_cnodes(struct ubifs_info *c) +{ + int lnum, offs, len, alen, done_lsave, done_ltab, err; + struct ubifs_cnode *cnode; + + err = dbg_chk_lpt_sz(c, 0, 0); + if (err) + return err; + cnode = c->lpt_cnext; + if (!cnode) + return 0; + lnum = c->nhead_lnum; + offs = c->nhead_offs; + /* Try to place lsave and ltab nicely */ + done_lsave = !c->big_lpt; + done_ltab = 0; + if (!done_lsave && offs + c->lsave_sz <= c->leb_size) { + done_lsave = 1; + c->lsave_lnum = lnum; + c->lsave_offs = offs; + offs += c->lsave_sz; + dbg_chk_lpt_sz(c, 1, c->lsave_sz); + } + + if (offs + c->ltab_sz <= c->leb_size) { + done_ltab = 1; + c->ltab_lnum = lnum; + c->ltab_offs = offs; + offs += c->ltab_sz; + dbg_chk_lpt_sz(c, 1, c->ltab_sz); + } + + do { + if (cnode->level) { + len = c->nnode_sz; + c->dirty_nn_cnt -= 1; + } else { + len = c->pnode_sz; + c->dirty_pn_cnt -= 1; + } + while (offs + len > c->leb_size) { + alen = ALIGN(offs, c->min_io_size); + upd_ltab(c, lnum, c->leb_size - alen, alen - offs); + dbg_chk_lpt_sz(c, 2, c->leb_size - offs); + err = alloc_lpt_leb(c, &lnum); + if (err) + goto no_space; + offs = 0; + ubifs_assert(c, lnum >= c->lpt_first && + lnum <= c->lpt_last); + /* Try to place lsave and ltab nicely */ + if (!done_lsave) { + done_lsave = 1; + c->lsave_lnum = lnum; + c->lsave_offs = offs; + offs += c->lsave_sz; + dbg_chk_lpt_sz(c, 1, c->lsave_sz); + continue; + } + if (!done_ltab) { + done_ltab = 1; + c->ltab_lnum = lnum; + c->ltab_offs = offs; + offs += c->ltab_sz; + dbg_chk_lpt_sz(c, 1, c->ltab_sz); + continue; + } + break; + } + if (cnode->parent) { + cnode->parent->nbranch[cnode->iip].lnum = lnum; + cnode->parent->nbranch[cnode->iip].offs = offs; + } else { + c->lpt_lnum = lnum; + c->lpt_offs = offs; + } + offs += len; + dbg_chk_lpt_sz(c, 1, len); + cnode = cnode->cnext; + } while (cnode && cnode != c->lpt_cnext); + + /* Make sure to place LPT's save table */ + if (!done_lsave) { + if (offs + c->lsave_sz > c->leb_size) { + alen = ALIGN(offs, c->min_io_size); + upd_ltab(c, lnum, c->leb_size - alen, alen - offs); + dbg_chk_lpt_sz(c, 2, c->leb_size - offs); + err = alloc_lpt_leb(c, &lnum); + if (err) + goto no_space; + offs = 0; + ubifs_assert(c, lnum >= c->lpt_first && + lnum <= c->lpt_last); + } + done_lsave = 1; + c->lsave_lnum = lnum; + c->lsave_offs = offs; + offs += c->lsave_sz; + dbg_chk_lpt_sz(c, 1, c->lsave_sz); + } + + /* Make sure to place LPT's own lprops table */ + if (!done_ltab) { + if (offs + c->ltab_sz > c->leb_size) { + alen = ALIGN(offs, c->min_io_size); + upd_ltab(c, lnum, c->leb_size - alen, alen - offs); + dbg_chk_lpt_sz(c, 2, c->leb_size - offs); + err = alloc_lpt_leb(c, &lnum); + if (err) + goto no_space; + offs = 0; + ubifs_assert(c, lnum >= c->lpt_first && + lnum <= c->lpt_last); + } + c->ltab_lnum = lnum; + c->ltab_offs = offs; + offs += c->ltab_sz; + dbg_chk_lpt_sz(c, 1, c->ltab_sz); + } + + alen = ALIGN(offs, c->min_io_size); + upd_ltab(c, lnum, c->leb_size - alen, alen - offs); + dbg_chk_lpt_sz(c, 4, alen - offs); + err = dbg_chk_lpt_sz(c, 3, alen); + if (err) + return err; + return 0; + +no_space: + ubifs_err(c, "LPT out of space at LEB %d:%d needing %d, done_ltab %d, done_lsave %d", + lnum, offs, len, done_ltab, done_lsave); + ubifs_dump_lpt_info(c); + ubifs_dump_lpt_lebs(c); + dump_stack(); + return err; +} + +/** + * realloc_lpt_leb - allocate an LPT LEB that is empty. + * @c: UBIFS file-system description object + * @lnum: LEB number is passed and returned here + * + * This function duplicates exactly the results of the function alloc_lpt_leb. + * It is used during end commit to reallocate the same LEB numbers that were + * allocated by alloc_lpt_leb during start commit. + * + * This function finds the next LEB that was allocated by the alloc_lpt_leb + * function starting from @lnum. If a LEB is found it is returned in @lnum and + * the function returns %0. Otherwise the function returns -ENOSPC. + * Note however, that LPT is designed never to run out of space. + */ +static int realloc_lpt_leb(struct ubifs_info *c, int *lnum) +{ + int i, n; + + n = *lnum - c->lpt_first + 1; + for (i = n; i < c->lpt_lebs; i++) + if (c->ltab[i].cmt) { + c->ltab[i].cmt = 0; + *lnum = i + c->lpt_first; + return 0; + } + + for (i = 0; i < n; i++) + if (c->ltab[i].cmt) { + c->ltab[i].cmt = 0; + *lnum = i + c->lpt_first; + return 0; + } + return -ENOSPC; +} + +/** + * write_cnodes - write cnodes for commit. + * @c: UBIFS file-system description object + * + * This function returns %0 on success and a negative error code on failure. + */ +static int write_cnodes(struct ubifs_info *c) +{ + int lnum, offs, len, from, err, wlen, alen, done_ltab, done_lsave; + struct ubifs_cnode *cnode; + void *buf = c->lpt_buf; + + cnode = c->lpt_cnext; + if (!cnode) + return 0; + lnum = c->nhead_lnum; + offs = c->nhead_offs; + from = offs; + /* Ensure empty LEB is unmapped */ + if (offs == 0) { + err = ubifs_leb_unmap(c, lnum); + if (err) + return err; + } + /* Try to place lsave and ltab nicely */ + done_lsave = !c->big_lpt; + done_ltab = 0; + if (!done_lsave && offs + c->lsave_sz <= c->leb_size) { + done_lsave = 1; + ubifs_pack_lsave(c, buf + offs, c->lsave); + offs += c->lsave_sz; + dbg_chk_lpt_sz(c, 1, c->lsave_sz); + } + + if (offs + c->ltab_sz <= c->leb_size) { + done_ltab = 1; + ubifs_pack_ltab(c, buf + offs, c->ltab_cmt); + offs += c->ltab_sz; + dbg_chk_lpt_sz(c, 1, c->ltab_sz); + } + + /* Loop for each cnode */ + do { + if (cnode->level) + len = c->nnode_sz; + else + len = c->pnode_sz; + while (offs + len > c->leb_size) { + wlen = offs - from; + if (wlen) { + alen = ALIGN(wlen, c->min_io_size); + memset(buf + offs, 0xff, alen - wlen); + err = ubifs_leb_write(c, lnum, buf + from, from, + alen); + if (err) + return err; + } + dbg_chk_lpt_sz(c, 2, c->leb_size - offs); + err = realloc_lpt_leb(c, &lnum); + if (err) + goto no_space; + offs = from = 0; + ubifs_assert(c, lnum >= c->lpt_first && + lnum <= c->lpt_last); + err = ubifs_leb_unmap(c, lnum); + if (err) + return err; + /* Try to place lsave and ltab nicely */ + if (!done_lsave) { + done_lsave = 1; + ubifs_pack_lsave(c, buf + offs, c->lsave); + offs += c->lsave_sz; + dbg_chk_lpt_sz(c, 1, c->lsave_sz); + continue; + } + if (!done_ltab) { + done_ltab = 1; + ubifs_pack_ltab(c, buf + offs, c->ltab_cmt); + offs += c->ltab_sz; + dbg_chk_lpt_sz(c, 1, c->ltab_sz); + continue; + } + break; + } + if (cnode->level) + ubifs_pack_nnode(c, buf + offs, + (struct ubifs_nnode *)cnode); + else + ubifs_pack_pnode(c, buf + offs, + (struct ubifs_pnode *)cnode); + /* + * The reason for the barriers is the same as in case of TNC. + * See comment in 'write_index()'. 'dirty_cow_nnode()' and + * 'dirty_cow_pnode()' are the functions for which this is + * important. + */ + clear_bit(DIRTY_CNODE, &cnode->flags); + smp_mb__before_atomic(); + clear_bit(COW_CNODE, &cnode->flags); + smp_mb__after_atomic(); + offs += len; + dbg_chk_lpt_sz(c, 1, len); + cnode = cnode->cnext; + } while (cnode && cnode != c->lpt_cnext); + + /* Make sure to place LPT's save table */ + if (!done_lsave) { + if (offs + c->lsave_sz > c->leb_size) { + wlen = offs - from; + alen = ALIGN(wlen, c->min_io_size); + memset(buf + offs, 0xff, alen - wlen); + err = ubifs_leb_write(c, lnum, buf + from, from, alen); + if (err) + return err; + dbg_chk_lpt_sz(c, 2, c->leb_size - offs); + err = realloc_lpt_leb(c, &lnum); + if (err) + goto no_space; + offs = from = 0; + ubifs_assert(c, lnum >= c->lpt_first && + lnum <= c->lpt_last); + err = ubifs_leb_unmap(c, lnum); + if (err) + return err; + } + done_lsave = 1; + ubifs_pack_lsave(c, buf + offs, c->lsave); + offs += c->lsave_sz; + dbg_chk_lpt_sz(c, 1, c->lsave_sz); + } + + /* Make sure to place LPT's own lprops table */ + if (!done_ltab) { + if (offs + c->ltab_sz > c->leb_size) { + wlen = offs - from; + alen = ALIGN(wlen, c->min_io_size); + memset(buf + offs, 0xff, alen - wlen); + err = ubifs_leb_write(c, lnum, buf + from, from, alen); + if (err) + return err; + dbg_chk_lpt_sz(c, 2, c->leb_size - offs); + err = realloc_lpt_leb(c, &lnum); + if (err) + goto no_space; + offs = from = 0; + ubifs_assert(c, lnum >= c->lpt_first && + lnum <= c->lpt_last); + err = ubifs_leb_unmap(c, lnum); + if (err) + return err; + } + ubifs_pack_ltab(c, buf + offs, c->ltab_cmt); + offs += c->ltab_sz; + dbg_chk_lpt_sz(c, 1, c->ltab_sz); + } + + /* Write remaining data in buffer */ + wlen = offs - from; + alen = ALIGN(wlen, c->min_io_size); + memset(buf + offs, 0xff, alen - wlen); + err = ubifs_leb_write(c, lnum, buf + from, from, alen); + if (err) + return err; + + dbg_chk_lpt_sz(c, 4, alen - wlen); + err = dbg_chk_lpt_sz(c, 3, ALIGN(offs, c->min_io_size)); + if (err) + return err; + + c->nhead_lnum = lnum; + c->nhead_offs = ALIGN(offs, c->min_io_size); + + dbg_lp("LPT root is at %d:%d", c->lpt_lnum, c->lpt_offs); + dbg_lp("LPT head is at %d:%d", c->nhead_lnum, c->nhead_offs); + dbg_lp("LPT ltab is at %d:%d", c->ltab_lnum, c->ltab_offs); + if (c->big_lpt) + dbg_lp("LPT lsave is at %d:%d", c->lsave_lnum, c->lsave_offs); + + return 0; + +no_space: + ubifs_err(c, "LPT out of space mismatch at LEB %d:%d needing %d, done_ltab %d, done_lsave %d", + lnum, offs, len, done_ltab, done_lsave); + ubifs_dump_lpt_info(c); + ubifs_dump_lpt_lebs(c); + dump_stack(); + return err; +} + +/** + * next_pnode_to_dirty - find next pnode to dirty. + * @c: UBIFS file-system description object + * @pnode: pnode + * + * This function returns the next pnode to dirty or %NULL if there are no more + * pnodes. Note that pnodes that have never been written (lnum == 0) are + * skipped. + */ +static struct ubifs_pnode *next_pnode_to_dirty(struct ubifs_info *c, + struct ubifs_pnode *pnode) +{ + struct ubifs_nnode *nnode; + int iip; + + /* Try to go right */ + nnode = pnode->parent; + for (iip = pnode->iip + 1; iip < UBIFS_LPT_FANOUT; iip++) { + if (nnode->nbranch[iip].lnum) + return ubifs_get_pnode(c, nnode, iip); + } + + /* Go up while can't go right */ + do { + iip = nnode->iip + 1; + nnode = nnode->parent; + if (!nnode) + return NULL; + for (; iip < UBIFS_LPT_FANOUT; iip++) { + if (nnode->nbranch[iip].lnum) + break; + } + } while (iip >= UBIFS_LPT_FANOUT); + + /* Go right */ + nnode = ubifs_get_nnode(c, nnode, iip); + if (IS_ERR(nnode)) + return ERR_CAST(nnode); + + /* Go down to level 1 */ + while (nnode->level > 1) { + for (iip = 0; iip < UBIFS_LPT_FANOUT; iip++) { + if (nnode->nbranch[iip].lnum) + break; + } + if (iip >= UBIFS_LPT_FANOUT) { + /* + * Should not happen, but we need to keep going + * if it does. + */ + iip = 0; + } + nnode = ubifs_get_nnode(c, nnode, iip); + if (IS_ERR(nnode)) + return ERR_CAST(nnode); + } + + for (iip = 0; iip < UBIFS_LPT_FANOUT; iip++) + if (nnode->nbranch[iip].lnum) + break; + if (iip >= UBIFS_LPT_FANOUT) + /* Should not happen, but we need to keep going if it does */ + iip = 0; + return ubifs_get_pnode(c, nnode, iip); +} + +/** + * add_pnode_dirt - add dirty space to LPT LEB properties. + * @c: UBIFS file-system description object + * @pnode: pnode for which to add dirt + */ +static void add_pnode_dirt(struct ubifs_info *c, struct ubifs_pnode *pnode) +{ + ubifs_add_lpt_dirt(c, pnode->parent->nbranch[pnode->iip].lnum, + c->pnode_sz); +} + +/** + * do_make_pnode_dirty - mark a pnode dirty. + * @c: UBIFS file-system description object + * @pnode: pnode to mark dirty + */ +static void do_make_pnode_dirty(struct ubifs_info *c, struct ubifs_pnode *pnode) +{ + /* Assumes cnext list is empty i.e. not called during commit */ + if (!test_and_set_bit(DIRTY_CNODE, &pnode->flags)) { + struct ubifs_nnode *nnode; + + c->dirty_pn_cnt += 1; + add_pnode_dirt(c, pnode); + /* Mark parent and ancestors dirty too */ + nnode = pnode->parent; + while (nnode) { + if (!test_and_set_bit(DIRTY_CNODE, &nnode->flags)) { + c->dirty_nn_cnt += 1; + ubifs_add_nnode_dirt(c, nnode); + nnode = nnode->parent; + } else + break; + } + } +} + +/** + * make_tree_dirty - mark the entire LEB properties tree dirty. + * @c: UBIFS file-system description object + * + * This function is used by the "small" LPT model to cause the entire LEB + * properties tree to be written. The "small" LPT model does not use LPT + * garbage collection because it is more efficient to write the entire tree + * (because it is small). + * + * This function returns %0 on success and a negative error code on failure. + */ +static int make_tree_dirty(struct ubifs_info *c) +{ + struct ubifs_pnode *pnode; + + pnode = ubifs_pnode_lookup(c, 0); + if (IS_ERR(pnode)) + return PTR_ERR(pnode); + + while (pnode) { + do_make_pnode_dirty(c, pnode); + pnode = next_pnode_to_dirty(c, pnode); + if (IS_ERR(pnode)) + return PTR_ERR(pnode); + } + return 0; +} + +/** + * need_write_all - determine if the LPT area is running out of free space. + * @c: UBIFS file-system description object + * + * This function returns %1 if the LPT area is running out of free space and %0 + * if it is not. + */ +static int need_write_all(struct ubifs_info *c) +{ + long long free = 0; + int i; + + for (i = 0; i < c->lpt_lebs; i++) { + if (i + c->lpt_first == c->nhead_lnum) + free += c->leb_size - c->nhead_offs; + else if (c->ltab[i].free == c->leb_size) + free += c->leb_size; + else if (c->ltab[i].free + c->ltab[i].dirty == c->leb_size) + free += c->leb_size; + } + /* Less than twice the size left */ + if (free <= c->lpt_sz * 2) + return 1; + return 0; +} + +/** + * lpt_tgc_start - start trivial garbage collection of LPT LEBs. + * @c: UBIFS file-system description object + * + * LPT trivial garbage collection is where a LPT LEB contains only dirty and + * free space and so may be reused as soon as the next commit is completed. + * This function is called during start commit to mark LPT LEBs for trivial GC. + */ +static void lpt_tgc_start(struct ubifs_info *c) +{ + int i; + + for (i = 0; i < c->lpt_lebs; i++) { + if (i + c->lpt_first == c->nhead_lnum) + continue; + if (c->ltab[i].dirty > 0 && + c->ltab[i].free + c->ltab[i].dirty == c->leb_size) { + c->ltab[i].tgc = 1; + c->ltab[i].free = c->leb_size; + c->ltab[i].dirty = 0; + dbg_lp("LEB %d", i + c->lpt_first); + } + } +} + +/** + * lpt_tgc_end - end trivial garbage collection of LPT LEBs. + * @c: UBIFS file-system description object + * + * LPT trivial garbage collection is where a LPT LEB contains only dirty and + * free space and so may be reused as soon as the next commit is completed. + * This function is called after the commit is completed (master node has been + * written) and un-maps LPT LEBs that were marked for trivial GC. + */ +static int lpt_tgc_end(struct ubifs_info *c) +{ + int i, err; + + for (i = 0; i < c->lpt_lebs; i++) + if (c->ltab[i].tgc) { + err = ubifs_leb_unmap(c, i + c->lpt_first); + if (err) + return err; + c->ltab[i].tgc = 0; + dbg_lp("LEB %d", i + c->lpt_first); + } + return 0; +} + +/** + * populate_lsave - fill the lsave array with important LEB numbers. + * @c: the UBIFS file-system description object + * + * This function is only called for the "big" model. It records a small number + * of LEB numbers of important LEBs. Important LEBs are ones that are (from + * most important to least important): empty, freeable, freeable index, dirty + * index, dirty or free. Upon mount, we read this list of LEB numbers and bring + * their pnodes into memory. That will stop us from having to scan the LPT + * straight away. For the "small" model we assume that scanning the LPT is no + * big deal. + */ +static void populate_lsave(struct ubifs_info *c) +{ + struct ubifs_lprops *lprops; + struct ubifs_lpt_heap *heap; + int i, cnt = 0; + + ubifs_assert(c, c->big_lpt); + if (!(c->lpt_drty_flgs & LSAVE_DIRTY)) { + c->lpt_drty_flgs |= LSAVE_DIRTY; + ubifs_add_lpt_dirt(c, c->lsave_lnum, c->lsave_sz); + } + + if (dbg_populate_lsave(c)) + return; + + list_for_each_entry(lprops, &c->empty_list, list) { + c->lsave[cnt++] = lprops->lnum; + if (cnt >= c->lsave_cnt) + return; + } + list_for_each_entry(lprops, &c->freeable_list, list) { + c->lsave[cnt++] = lprops->lnum; + if (cnt >= c->lsave_cnt) + return; + } + list_for_each_entry(lprops, &c->frdi_idx_list, list) { + c->lsave[cnt++] = lprops->lnum; + if (cnt >= c->lsave_cnt) + return; + } + heap = &c->lpt_heap[LPROPS_DIRTY_IDX - 1]; + for (i = 0; i < heap->cnt; i++) { + c->lsave[cnt++] = heap->arr[i]->lnum; + if (cnt >= c->lsave_cnt) + return; + } + heap = &c->lpt_heap[LPROPS_DIRTY - 1]; + for (i = 0; i < heap->cnt; i++) { + c->lsave[cnt++] = heap->arr[i]->lnum; + if (cnt >= c->lsave_cnt) + return; + } + heap = &c->lpt_heap[LPROPS_FREE - 1]; + for (i = 0; i < heap->cnt; i++) { + c->lsave[cnt++] = heap->arr[i]->lnum; + if (cnt >= c->lsave_cnt) + return; + } + /* Fill it up completely */ + while (cnt < c->lsave_cnt) + c->lsave[cnt++] = c->main_first; +} + +/** + * nnode_lookup - lookup a nnode in the LPT. + * @c: UBIFS file-system description object + * @i: nnode number + * + * This function returns a pointer to the nnode on success or a negative + * error code on failure. + */ +static struct ubifs_nnode *nnode_lookup(struct ubifs_info *c, int i) +{ + int err, iip; + struct ubifs_nnode *nnode; + + if (!c->nroot) { + err = ubifs_read_nnode(c, NULL, 0); + if (err) + return ERR_PTR(err); + } + nnode = c->nroot; + while (1) { + iip = i & (UBIFS_LPT_FANOUT - 1); + i >>= UBIFS_LPT_FANOUT_SHIFT; + if (!i) + break; + nnode = ubifs_get_nnode(c, nnode, iip); + if (IS_ERR(nnode)) + return nnode; + } + return nnode; +} + +/** + * make_nnode_dirty - find a nnode and, if found, make it dirty. + * @c: UBIFS file-system description object + * @node_num: nnode number of nnode to make dirty + * @lnum: LEB number where nnode was written + * @offs: offset where nnode was written + * + * This function is used by LPT garbage collection. LPT garbage collection is + * used only for the "big" LPT model (c->big_lpt == 1). Garbage collection + * simply involves marking all the nodes in the LEB being garbage-collected as + * dirty. The dirty nodes are written next commit, after which the LEB is free + * to be reused. + * + * This function returns %0 on success and a negative error code on failure. + */ +static int make_nnode_dirty(struct ubifs_info *c, int node_num, int lnum, + int offs) +{ + struct ubifs_nnode *nnode; + + nnode = nnode_lookup(c, node_num); + if (IS_ERR(nnode)) + return PTR_ERR(nnode); + if (nnode->parent) { + struct ubifs_nbranch *branch; + + branch = &nnode->parent->nbranch[nnode->iip]; + if (branch->lnum != lnum || branch->offs != offs) + return 0; /* nnode is obsolete */ + } else if (c->lpt_lnum != lnum || c->lpt_offs != offs) + return 0; /* nnode is obsolete */ + /* Assumes cnext list is empty i.e. not called during commit */ + if (!test_and_set_bit(DIRTY_CNODE, &nnode->flags)) { + c->dirty_nn_cnt += 1; + ubifs_add_nnode_dirt(c, nnode); + /* Mark parent and ancestors dirty too */ + nnode = nnode->parent; + while (nnode) { + if (!test_and_set_bit(DIRTY_CNODE, &nnode->flags)) { + c->dirty_nn_cnt += 1; + ubifs_add_nnode_dirt(c, nnode); + nnode = nnode->parent; + } else + break; + } + } + return 0; +} + +/** + * make_pnode_dirty - find a pnode and, if found, make it dirty. + * @c: UBIFS file-system description object + * @node_num: pnode number of pnode to make dirty + * @lnum: LEB number where pnode was written + * @offs: offset where pnode was written + * + * This function is used by LPT garbage collection. LPT garbage collection is + * used only for the "big" LPT model (c->big_lpt == 1). Garbage collection + * simply involves marking all the nodes in the LEB being garbage-collected as + * dirty. The dirty nodes are written next commit, after which the LEB is free + * to be reused. + * + * This function returns %0 on success and a negative error code on failure. + */ +static int make_pnode_dirty(struct ubifs_info *c, int node_num, int lnum, + int offs) +{ + struct ubifs_pnode *pnode; + struct ubifs_nbranch *branch; + + pnode = ubifs_pnode_lookup(c, node_num); + if (IS_ERR(pnode)) + return PTR_ERR(pnode); + branch = &pnode->parent->nbranch[pnode->iip]; + if (branch->lnum != lnum || branch->offs != offs) + return 0; + do_make_pnode_dirty(c, pnode); + return 0; +} + +/** + * make_ltab_dirty - make ltab node dirty. + * @c: UBIFS file-system description object + * @lnum: LEB number where ltab was written + * @offs: offset where ltab was written + * + * This function is used by LPT garbage collection. LPT garbage collection is + * used only for the "big" LPT model (c->big_lpt == 1). Garbage collection + * simply involves marking all the nodes in the LEB being garbage-collected as + * dirty. The dirty nodes are written next commit, after which the LEB is free + * to be reused. + * + * This function returns %0 on success and a negative error code on failure. + */ +static int make_ltab_dirty(struct ubifs_info *c, int lnum, int offs) +{ + if (lnum != c->ltab_lnum || offs != c->ltab_offs) + return 0; /* This ltab node is obsolete */ + if (!(c->lpt_drty_flgs & LTAB_DIRTY)) { + c->lpt_drty_flgs |= LTAB_DIRTY; + ubifs_add_lpt_dirt(c, c->ltab_lnum, c->ltab_sz); + } + return 0; +} + +/** + * make_lsave_dirty - make lsave node dirty. + * @c: UBIFS file-system description object + * @lnum: LEB number where lsave was written + * @offs: offset where lsave was written + * + * This function is used by LPT garbage collection. LPT garbage collection is + * used only for the "big" LPT model (c->big_lpt == 1). Garbage collection + * simply involves marking all the nodes in the LEB being garbage-collected as + * dirty. The dirty nodes are written next commit, after which the LEB is free + * to be reused. + * + * This function returns %0 on success and a negative error code on failure. + */ +static int make_lsave_dirty(struct ubifs_info *c, int lnum, int offs) +{ + if (lnum != c->lsave_lnum || offs != c->lsave_offs) + return 0; /* This lsave node is obsolete */ + if (!(c->lpt_drty_flgs & LSAVE_DIRTY)) { + c->lpt_drty_flgs |= LSAVE_DIRTY; + ubifs_add_lpt_dirt(c, c->lsave_lnum, c->lsave_sz); + } + return 0; +} + +/** + * make_node_dirty - make node dirty. + * @c: UBIFS file-system description object + * @node_type: LPT node type + * @node_num: node number + * @lnum: LEB number where node was written + * @offs: offset where node was written + * + * This function is used by LPT garbage collection. LPT garbage collection is + * used only for the "big" LPT model (c->big_lpt == 1). Garbage collection + * simply involves marking all the nodes in the LEB being garbage-collected as + * dirty. The dirty nodes are written next commit, after which the LEB is free + * to be reused. + * + * This function returns %0 on success and a negative error code on failure. + */ +static int make_node_dirty(struct ubifs_info *c, int node_type, int node_num, + int lnum, int offs) +{ + switch (node_type) { + case UBIFS_LPT_NNODE: + return make_nnode_dirty(c, node_num, lnum, offs); + case UBIFS_LPT_PNODE: + return make_pnode_dirty(c, node_num, lnum, offs); + case UBIFS_LPT_LTAB: + return make_ltab_dirty(c, lnum, offs); + case UBIFS_LPT_LSAVE: + return make_lsave_dirty(c, lnum, offs); + } + return -EINVAL; +} + +/** + * get_lpt_node_len - return the length of a node based on its type. + * @c: UBIFS file-system description object + * @node_type: LPT node type + */ +static int get_lpt_node_len(const struct ubifs_info *c, int node_type) +{ + switch (node_type) { + case UBIFS_LPT_NNODE: + return c->nnode_sz; + case UBIFS_LPT_PNODE: + return c->pnode_sz; + case UBIFS_LPT_LTAB: + return c->ltab_sz; + case UBIFS_LPT_LSAVE: + return c->lsave_sz; + } + return 0; +} + +/** + * get_pad_len - return the length of padding in a buffer. + * @c: UBIFS file-system description object + * @buf: buffer + * @len: length of buffer + */ +static int get_pad_len(const struct ubifs_info *c, uint8_t *buf, int len) +{ + int offs, pad_len; + + if (c->min_io_size == 1) + return 0; + offs = c->leb_size - len; + pad_len = ALIGN(offs, c->min_io_size) - offs; + return pad_len; +} + +/** + * get_lpt_node_type - return type (and node number) of a node in a buffer. + * @c: UBIFS file-system description object + * @buf: buffer + * @node_num: node number is returned here + */ +static int get_lpt_node_type(const struct ubifs_info *c, uint8_t *buf, + int *node_num) +{ + uint8_t *addr = buf + UBIFS_LPT_CRC_BYTES; + int pos = 0, node_type; + + node_type = ubifs_unpack_bits(c, &addr, &pos, UBIFS_LPT_TYPE_BITS); + *node_num = ubifs_unpack_bits(c, &addr, &pos, c->pcnt_bits); + return node_type; +} + +/** + * is_a_node - determine if a buffer contains a node. + * @c: UBIFS file-system description object + * @buf: buffer + * @len: length of buffer + * + * This function returns %1 if the buffer contains a node or %0 if it does not. + */ +static int is_a_node(const struct ubifs_info *c, uint8_t *buf, int len) +{ + uint8_t *addr = buf + UBIFS_LPT_CRC_BYTES; + int pos = 0, node_type, node_len; + uint16_t crc, calc_crc; + + if (len < UBIFS_LPT_CRC_BYTES + (UBIFS_LPT_TYPE_BITS + 7) / 8) + return 0; + node_type = ubifs_unpack_bits(c, &addr, &pos, UBIFS_LPT_TYPE_BITS); + if (node_type == UBIFS_LPT_NOT_A_NODE) + return 0; + node_len = get_lpt_node_len(c, node_type); + if (!node_len || node_len > len) + return 0; + pos = 0; + addr = buf; + crc = ubifs_unpack_bits(c, &addr, &pos, UBIFS_LPT_CRC_BITS); + calc_crc = crc16(-1, buf + UBIFS_LPT_CRC_BYTES, + node_len - UBIFS_LPT_CRC_BYTES); + if (crc != calc_crc) + return 0; + return 1; +} + +/** + * lpt_gc_lnum - garbage collect a LPT LEB. + * @c: UBIFS file-system description object + * @lnum: LEB number to garbage collect + * + * LPT garbage collection is used only for the "big" LPT model + * (c->big_lpt == 1). Garbage collection simply involves marking all the nodes + * in the LEB being garbage-collected as dirty. The dirty nodes are written + * next commit, after which the LEB is free to be reused. + * + * This function returns %0 on success and a negative error code on failure. + */ +static int lpt_gc_lnum(struct ubifs_info *c, int lnum) +{ + int err, len = c->leb_size, node_type, node_num, node_len, offs; + void *buf = c->lpt_buf; + + dbg_lp("LEB %d", lnum); + + err = ubifs_leb_read(c, lnum, buf, 0, c->leb_size, 1); + if (err) + return err; + + while (1) { + if (!is_a_node(c, buf, len)) { + int pad_len; + + pad_len = get_pad_len(c, buf, len); + if (pad_len) { + buf += pad_len; + len -= pad_len; + continue; + } + return 0; + } + node_type = get_lpt_node_type(c, buf, &node_num); + node_len = get_lpt_node_len(c, node_type); + offs = c->leb_size - len; + ubifs_assert(c, node_len != 0); + mutex_lock(&c->lp_mutex); + err = make_node_dirty(c, node_type, node_num, lnum, offs); + mutex_unlock(&c->lp_mutex); + if (err) + return err; + buf += node_len; + len -= node_len; + } + return 0; +} + +/** + * lpt_gc - LPT garbage collection. + * @c: UBIFS file-system description object + * + * Select a LPT LEB for LPT garbage collection and call 'lpt_gc_lnum()'. + * Returns %0 on success and a negative error code on failure. + */ +static int lpt_gc(struct ubifs_info *c) +{ + int i, lnum = -1, dirty = 0; + + mutex_lock(&c->lp_mutex); + for (i = 0; i < c->lpt_lebs; i++) { + ubifs_assert(c, !c->ltab[i].tgc); + if (i + c->lpt_first == c->nhead_lnum || + c->ltab[i].free + c->ltab[i].dirty == c->leb_size) + continue; + if (c->ltab[i].dirty > dirty) { + dirty = c->ltab[i].dirty; + lnum = i + c->lpt_first; + } + } + mutex_unlock(&c->lp_mutex); + if (lnum == -1) + return -ENOSPC; + return lpt_gc_lnum(c, lnum); +} + +/** + * ubifs_lpt_start_commit - UBIFS commit starts. + * @c: the UBIFS file-system description object + * + * This function has to be called when UBIFS starts the commit operation. + * This function "freezes" all currently dirty LEB properties and does not + * change them anymore. Further changes are saved and tracked separately + * because they are not part of this commit. This function returns zero in case + * of success and a negative error code in case of failure. + */ +int ubifs_lpt_start_commit(struct ubifs_info *c) +{ + int err, cnt; + + dbg_lp(""); + + mutex_lock(&c->lp_mutex); + err = dbg_chk_lpt_free_spc(c); + if (err) + goto out; + err = dbg_check_ltab(c); + if (err) + goto out; + + if (c->check_lpt_free) { + /* + * We ensure there is enough free space in + * ubifs_lpt_post_commit() by marking nodes dirty. That + * information is lost when we unmount, so we also need + * to check free space once after mounting also. + */ + c->check_lpt_free = 0; + while (need_write_all(c)) { + mutex_unlock(&c->lp_mutex); + err = lpt_gc(c); + if (err) + return err; + mutex_lock(&c->lp_mutex); + } + } + + lpt_tgc_start(c); + + if (!c->dirty_pn_cnt) { + dbg_cmt("no cnodes to commit"); + err = 0; + goto out; + } + + if (!c->big_lpt && need_write_all(c)) { + /* If needed, write everything */ + err = make_tree_dirty(c); + if (err) + goto out; + lpt_tgc_start(c); + } + + if (c->big_lpt) + populate_lsave(c); + + cnt = get_cnodes_to_commit(c); + ubifs_assert(c, cnt != 0); + + err = layout_cnodes(c); + if (err) + goto out; + + err = ubifs_lpt_calc_hash(c, c->mst_node->hash_lpt); + if (err) + goto out; + + /* Copy the LPT's own lprops for end commit to write */ + memcpy(c->ltab_cmt, c->ltab, + sizeof(struct ubifs_lpt_lprops) * c->lpt_lebs); + c->lpt_drty_flgs &= ~(LTAB_DIRTY | LSAVE_DIRTY); + +out: + mutex_unlock(&c->lp_mutex); + return err; +} + +/** + * free_obsolete_cnodes - free obsolete cnodes for commit end. + * @c: UBIFS file-system description object + */ +static void free_obsolete_cnodes(struct ubifs_info *c) +{ + struct ubifs_cnode *cnode, *cnext; + + cnext = c->lpt_cnext; + if (!cnext) + return; + do { + cnode = cnext; + cnext = cnode->cnext; + if (test_bit(OBSOLETE_CNODE, &cnode->flags)) + kfree(cnode); + else + cnode->cnext = NULL; + } while (cnext != c->lpt_cnext); + c->lpt_cnext = NULL; +} + +/** + * ubifs_lpt_end_commit - finish the commit operation. + * @c: the UBIFS file-system description object + * + * This function has to be called when the commit operation finishes. It + * flushes the changes which were "frozen" by 'ubifs_lprops_start_commit()' to + * the media. Returns zero in case of success and a negative error code in case + * of failure. + */ +int ubifs_lpt_end_commit(struct ubifs_info *c) +{ + int err; + + dbg_lp(""); + + if (!c->lpt_cnext) + return 0; + + err = write_cnodes(c); + if (err) + return err; + + mutex_lock(&c->lp_mutex); + free_obsolete_cnodes(c); + mutex_unlock(&c->lp_mutex); + + return 0; +} + +/** + * ubifs_lpt_post_commit - post commit LPT trivial GC and LPT GC. + * @c: UBIFS file-system description object + * + * LPT trivial GC is completed after a commit. Also LPT GC is done after a + * commit for the "big" LPT model. + */ +int ubifs_lpt_post_commit(struct ubifs_info *c) +{ + int err; + + mutex_lock(&c->lp_mutex); + err = lpt_tgc_end(c); + if (err) + goto out; + if (c->big_lpt) + while (need_write_all(c)) { + mutex_unlock(&c->lp_mutex); + err = lpt_gc(c); + if (err) + return err; + mutex_lock(&c->lp_mutex); + } +out: + mutex_unlock(&c->lp_mutex); + return err; +} + +/** + * first_nnode - find the first nnode in memory. + * @c: UBIFS file-system description object + * @hght: height of tree where nnode found is returned here + * + * This function returns a pointer to the nnode found or %NULL if no nnode is + * found. This function is a helper to 'ubifs_lpt_free()'. + */ +static struct ubifs_nnode *first_nnode(struct ubifs_info *c, int *hght) +{ + struct ubifs_nnode *nnode; + int h, i, found; + + nnode = c->nroot; + *hght = 0; + if (!nnode) + return NULL; + for (h = 1; h < c->lpt_hght; h++) { + found = 0; + for (i = 0; i < UBIFS_LPT_FANOUT; i++) { + if (nnode->nbranch[i].nnode) { + found = 1; + nnode = nnode->nbranch[i].nnode; + *hght = h; + break; + } + } + if (!found) + break; + } + return nnode; +} + +/** + * next_nnode - find the next nnode in memory. + * @c: UBIFS file-system description object + * @nnode: nnode from which to start. + * @hght: height of tree where nnode is, is passed and returned here + * + * This function returns a pointer to the nnode found or %NULL if no nnode is + * found. This function is a helper to 'ubifs_lpt_free()'. + */ +static struct ubifs_nnode *next_nnode(struct ubifs_info *c, + struct ubifs_nnode *nnode, int *hght) +{ + struct ubifs_nnode *parent; + int iip, h, i, found; + + parent = nnode->parent; + if (!parent) + return NULL; + if (nnode->iip == UBIFS_LPT_FANOUT - 1) { + *hght -= 1; + return parent; + } + for (iip = nnode->iip + 1; iip < UBIFS_LPT_FANOUT; iip++) { + nnode = parent->nbranch[iip].nnode; + if (nnode) + break; + } + if (!nnode) { + *hght -= 1; + return parent; + } + for (h = *hght + 1; h < c->lpt_hght; h++) { + found = 0; + for (i = 0; i < UBIFS_LPT_FANOUT; i++) { + if (nnode->nbranch[i].nnode) { + found = 1; + nnode = nnode->nbranch[i].nnode; + *hght = h; + break; + } + } + if (!found) + break; + } + return nnode; +} + +/** + * ubifs_lpt_free - free resources owned by the LPT. + * @c: UBIFS file-system description object + * @wr_only: free only resources used for writing + */ +void ubifs_lpt_free(struct ubifs_info *c, int wr_only) +{ + struct ubifs_nnode *nnode; + int i, hght; + + /* Free write-only things first */ + + free_obsolete_cnodes(c); /* Leftover from a failed commit */ + + vfree(c->ltab_cmt); + c->ltab_cmt = NULL; + vfree(c->lpt_buf); + c->lpt_buf = NULL; + kfree(c->lsave); + c->lsave = NULL; + + if (wr_only) + return; + + /* Now free the rest */ + + nnode = first_nnode(c, &hght); + while (nnode) { + for (i = 0; i < UBIFS_LPT_FANOUT; i++) + kfree(nnode->nbranch[i].nnode); + nnode = next_nnode(c, nnode, &hght); + } + for (i = 0; i < LPROPS_HEAP_CNT; i++) + kfree(c->lpt_heap[i].arr); + kfree(c->dirty_idx.arr); + kfree(c->nroot); + vfree(c->ltab); + kfree(c->lpt_nod_buf); +} + +/* + * Everything below is related to debugging. + */ + +/** + * dbg_is_all_ff - determine if a buffer contains only 0xFF bytes. + * @buf: buffer + * @len: buffer length + */ +static int dbg_is_all_ff(uint8_t *buf, int len) +{ + int i; + + for (i = 0; i < len; i++) + if (buf[i] != 0xff) + return 0; + return 1; +} + +/** + * dbg_is_nnode_dirty - determine if a nnode is dirty. + * @c: the UBIFS file-system description object + * @lnum: LEB number where nnode was written + * @offs: offset where nnode was written + */ +static int dbg_is_nnode_dirty(struct ubifs_info *c, int lnum, int offs) +{ + struct ubifs_nnode *nnode; + int hght; + + /* Entire tree is in memory so first_nnode / next_nnode are OK */ + nnode = first_nnode(c, &hght); + for (; nnode; nnode = next_nnode(c, nnode, &hght)) { + struct ubifs_nbranch *branch; + + cond_resched(); + if (nnode->parent) { + branch = &nnode->parent->nbranch[nnode->iip]; + if (branch->lnum != lnum || branch->offs != offs) + continue; + if (test_bit(DIRTY_CNODE, &nnode->flags)) + return 1; + return 0; + } else { + if (c->lpt_lnum != lnum || c->lpt_offs != offs) + continue; + if (test_bit(DIRTY_CNODE, &nnode->flags)) + return 1; + return 0; + } + } + return 1; +} + +/** + * dbg_is_pnode_dirty - determine if a pnode is dirty. + * @c: the UBIFS file-system description object + * @lnum: LEB number where pnode was written + * @offs: offset where pnode was written + */ +static int dbg_is_pnode_dirty(struct ubifs_info *c, int lnum, int offs) +{ + int i, cnt; + + cnt = DIV_ROUND_UP(c->main_lebs, UBIFS_LPT_FANOUT); + for (i = 0; i < cnt; i++) { + struct ubifs_pnode *pnode; + struct ubifs_nbranch *branch; + + cond_resched(); + pnode = ubifs_pnode_lookup(c, i); + if (IS_ERR(pnode)) + return PTR_ERR(pnode); + branch = &pnode->parent->nbranch[pnode->iip]; + if (branch->lnum != lnum || branch->offs != offs) + continue; + if (test_bit(DIRTY_CNODE, &pnode->flags)) + return 1; + return 0; + } + return 1; +} + +/** + * dbg_is_ltab_dirty - determine if a ltab node is dirty. + * @c: the UBIFS file-system description object + * @lnum: LEB number where ltab node was written + * @offs: offset where ltab node was written + */ +static int dbg_is_ltab_dirty(struct ubifs_info *c, int lnum, int offs) +{ + if (lnum != c->ltab_lnum || offs != c->ltab_offs) + return 1; + return (c->lpt_drty_flgs & LTAB_DIRTY) != 0; +} + +/** + * dbg_is_lsave_dirty - determine if a lsave node is dirty. + * @c: the UBIFS file-system description object + * @lnum: LEB number where lsave node was written + * @offs: offset where lsave node was written + */ +static int dbg_is_lsave_dirty(struct ubifs_info *c, int lnum, int offs) +{ + if (lnum != c->lsave_lnum || offs != c->lsave_offs) + return 1; + return (c->lpt_drty_flgs & LSAVE_DIRTY) != 0; +} + +/** + * dbg_is_node_dirty - determine if a node is dirty. + * @c: the UBIFS file-system description object + * @node_type: node type + * @lnum: LEB number where node was written + * @offs: offset where node was written + */ +static int dbg_is_node_dirty(struct ubifs_info *c, int node_type, int lnum, + int offs) +{ + switch (node_type) { + case UBIFS_LPT_NNODE: + return dbg_is_nnode_dirty(c, lnum, offs); + case UBIFS_LPT_PNODE: + return dbg_is_pnode_dirty(c, lnum, offs); + case UBIFS_LPT_LTAB: + return dbg_is_ltab_dirty(c, lnum, offs); + case UBIFS_LPT_LSAVE: + return dbg_is_lsave_dirty(c, lnum, offs); + } + return 1; +} + +/** + * dbg_check_ltab_lnum - check the ltab for a LPT LEB number. + * @c: the UBIFS file-system description object + * @lnum: LEB number where node was written + * + * This function returns %0 on success and a negative error code on failure. + */ +static int dbg_check_ltab_lnum(struct ubifs_info *c, int lnum) +{ + int err, len = c->leb_size, dirty = 0, node_type, node_num, node_len; + int ret; + void *buf, *p; + + if (!dbg_is_chk_lprops(c)) + return 0; + + buf = p = __vmalloc(c->leb_size, GFP_NOFS); + if (!buf) { + ubifs_err(c, "cannot allocate memory for ltab checking"); + return 0; + } + + dbg_lp("LEB %d", lnum); + + err = ubifs_leb_read(c, lnum, buf, 0, c->leb_size, 1); + if (err) + goto out; + + while (1) { + if (!is_a_node(c, p, len)) { + int i, pad_len; + + pad_len = get_pad_len(c, p, len); + if (pad_len) { + p += pad_len; + len -= pad_len; + dirty += pad_len; + continue; + } + if (!dbg_is_all_ff(p, len)) { + ubifs_err(c, "invalid empty space in LEB %d at %d", + lnum, c->leb_size - len); + err = -EINVAL; + } + i = lnum - c->lpt_first; + if (len != c->ltab[i].free) { + ubifs_err(c, "invalid free space in LEB %d (free %d, expected %d)", + lnum, len, c->ltab[i].free); + err = -EINVAL; + } + if (dirty != c->ltab[i].dirty) { + ubifs_err(c, "invalid dirty space in LEB %d (dirty %d, expected %d)", + lnum, dirty, c->ltab[i].dirty); + err = -EINVAL; + } + goto out; + } + node_type = get_lpt_node_type(c, p, &node_num); + node_len = get_lpt_node_len(c, node_type); + ret = dbg_is_node_dirty(c, node_type, lnum, c->leb_size - len); + if (ret == 1) + dirty += node_len; + p += node_len; + len -= node_len; + } + +out: + vfree(buf); + return err; +} + +/** + * dbg_check_ltab - check the free and dirty space in the ltab. + * @c: the UBIFS file-system description object + * + * This function returns %0 on success and a negative error code on failure. + */ +int dbg_check_ltab(struct ubifs_info *c) +{ + int lnum, err, i, cnt; + + if (!dbg_is_chk_lprops(c)) + return 0; + + /* Bring the entire tree into memory */ + cnt = DIV_ROUND_UP(c->main_lebs, UBIFS_LPT_FANOUT); + for (i = 0; i < cnt; i++) { + struct ubifs_pnode *pnode; + + pnode = ubifs_pnode_lookup(c, i); + if (IS_ERR(pnode)) + return PTR_ERR(pnode); + cond_resched(); + } + + /* Check nodes */ + err = dbg_check_lpt_nodes(c, (struct ubifs_cnode *)c->nroot, 0, 0); + if (err) + return err; + + /* Check each LEB */ + for (lnum = c->lpt_first; lnum <= c->lpt_last; lnum++) { + err = dbg_check_ltab_lnum(c, lnum); + if (err) { + ubifs_err(c, "failed at LEB %d", lnum); + return err; + } + } + + dbg_lp("succeeded"); + return 0; +} + +/** + * dbg_chk_lpt_free_spc - check LPT free space is enough to write entire LPT. + * @c: the UBIFS file-system description object + * + * This function returns %0 on success and a negative error code on failure. + */ +int dbg_chk_lpt_free_spc(struct ubifs_info *c) +{ + long long free = 0; + int i; + + if (!dbg_is_chk_lprops(c)) + return 0; + + for (i = 0; i < c->lpt_lebs; i++) { + if (c->ltab[i].tgc || c->ltab[i].cmt) + continue; + if (i + c->lpt_first == c->nhead_lnum) + free += c->leb_size - c->nhead_offs; + else if (c->ltab[i].free == c->leb_size) + free += c->leb_size; + } + if (free < c->lpt_sz) { + ubifs_err(c, "LPT space error: free %lld lpt_sz %lld", + free, c->lpt_sz); + ubifs_dump_lpt_info(c); + ubifs_dump_lpt_lebs(c); + dump_stack(); + return -EINVAL; + } + return 0; +} + +/** + * dbg_chk_lpt_sz - check LPT does not write more than LPT size. + * @c: the UBIFS file-system description object + * @action: what to do + * @len: length written + * + * This function returns %0 on success and a negative error code on failure. + * The @action argument may be one of: + * o %0 - LPT debugging checking starts, initialize debugging variables; + * o %1 - wrote an LPT node, increase LPT size by @len bytes; + * o %2 - switched to a different LEB and wasted @len bytes; + * o %3 - check that we've written the right number of bytes. + * o %4 - wasted @len bytes; + */ +int dbg_chk_lpt_sz(struct ubifs_info *c, int action, int len) +{ + struct ubifs_debug_info *d = c->dbg; + long long chk_lpt_sz, lpt_sz; + int err = 0; + + if (!dbg_is_chk_lprops(c)) + return 0; + + switch (action) { + case 0: + d->chk_lpt_sz = 0; + d->chk_lpt_sz2 = 0; + d->chk_lpt_lebs = 0; + d->chk_lpt_wastage = 0; + if (c->dirty_pn_cnt > c->pnode_cnt) { + ubifs_err(c, "dirty pnodes %d exceed max %d", + c->dirty_pn_cnt, c->pnode_cnt); + err = -EINVAL; + } + if (c->dirty_nn_cnt > c->nnode_cnt) { + ubifs_err(c, "dirty nnodes %d exceed max %d", + c->dirty_nn_cnt, c->nnode_cnt); + err = -EINVAL; + } + return err; + case 1: + d->chk_lpt_sz += len; + return 0; + case 2: + d->chk_lpt_sz += len; + d->chk_lpt_wastage += len; + d->chk_lpt_lebs += 1; + return 0; + case 3: + chk_lpt_sz = c->leb_size; + chk_lpt_sz *= d->chk_lpt_lebs; + chk_lpt_sz += len - c->nhead_offs; + if (d->chk_lpt_sz != chk_lpt_sz) { + ubifs_err(c, "LPT wrote %lld but space used was %lld", + d->chk_lpt_sz, chk_lpt_sz); + err = -EINVAL; + } + if (d->chk_lpt_sz > c->lpt_sz) { + ubifs_err(c, "LPT wrote %lld but lpt_sz is %lld", + d->chk_lpt_sz, c->lpt_sz); + err = -EINVAL; + } + if (d->chk_lpt_sz2 && d->chk_lpt_sz != d->chk_lpt_sz2) { + ubifs_err(c, "LPT layout size %lld but wrote %lld", + d->chk_lpt_sz, d->chk_lpt_sz2); + err = -EINVAL; + } + if (d->chk_lpt_sz2 && d->new_nhead_offs != len) { + ubifs_err(c, "LPT new nhead offs: expected %d was %d", + d->new_nhead_offs, len); + err = -EINVAL; + } + lpt_sz = (long long)c->pnode_cnt * c->pnode_sz; + lpt_sz += (long long)c->nnode_cnt * c->nnode_sz; + lpt_sz += c->ltab_sz; + if (c->big_lpt) + lpt_sz += c->lsave_sz; + if (d->chk_lpt_sz - d->chk_lpt_wastage > lpt_sz) { + ubifs_err(c, "LPT chk_lpt_sz %lld + waste %lld exceeds %lld", + d->chk_lpt_sz, d->chk_lpt_wastage, lpt_sz); + err = -EINVAL; + } + if (err) { + ubifs_dump_lpt_info(c); + ubifs_dump_lpt_lebs(c); + dump_stack(); + } + d->chk_lpt_sz2 = d->chk_lpt_sz; + d->chk_lpt_sz = 0; + d->chk_lpt_wastage = 0; + d->chk_lpt_lebs = 0; + d->new_nhead_offs = len; + return err; + case 4: + d->chk_lpt_sz += len; + d->chk_lpt_wastage += len; + return 0; + default: + return -EINVAL; + } +} + +/** + * dump_lpt_leb - dump an LPT LEB. + * @c: UBIFS file-system description object + * @lnum: LEB number to dump + * + * This function dumps an LEB from LPT area. Nodes in this area are very + * different to nodes in the main area (e.g., they do not have common headers, + * they do not have 8-byte alignments, etc), so we have a separate function to + * dump LPT area LEBs. Note, LPT has to be locked by the caller. + */ +static void dump_lpt_leb(const struct ubifs_info *c, int lnum) +{ + int err, len = c->leb_size, node_type, node_num, node_len, offs; + void *buf, *p; + + pr_err("(pid %d) start dumping LEB %d\n", current->pid, lnum); + buf = p = __vmalloc(c->leb_size, GFP_NOFS); + if (!buf) { + ubifs_err(c, "cannot allocate memory to dump LPT"); + return; + } + + err = ubifs_leb_read(c, lnum, buf, 0, c->leb_size, 1); + if (err) + goto out; + + while (1) { + offs = c->leb_size - len; + if (!is_a_node(c, p, len)) { + int pad_len; + + pad_len = get_pad_len(c, p, len); + if (pad_len) { + pr_err("LEB %d:%d, pad %d bytes\n", + lnum, offs, pad_len); + p += pad_len; + len -= pad_len; + continue; + } + if (len) + pr_err("LEB %d:%d, free %d bytes\n", + lnum, offs, len); + break; + } + + node_type = get_lpt_node_type(c, p, &node_num); + switch (node_type) { + case UBIFS_LPT_PNODE: + { + node_len = c->pnode_sz; + if (c->big_lpt) + pr_err("LEB %d:%d, pnode num %d\n", + lnum, offs, node_num); + else + pr_err("LEB %d:%d, pnode\n", lnum, offs); + break; + } + case UBIFS_LPT_NNODE: + { + int i; + struct ubifs_nnode nnode; + + node_len = c->nnode_sz; + if (c->big_lpt) + pr_err("LEB %d:%d, nnode num %d, ", + lnum, offs, node_num); + else + pr_err("LEB %d:%d, nnode, ", + lnum, offs); + err = ubifs_unpack_nnode(c, p, &nnode); + if (err) { + pr_err("failed to unpack_node, error %d\n", + err); + break; + } + for (i = 0; i < UBIFS_LPT_FANOUT; i++) { + pr_cont("%d:%d", nnode.nbranch[i].lnum, + nnode.nbranch[i].offs); + if (i != UBIFS_LPT_FANOUT - 1) + pr_cont(", "); + } + pr_cont("\n"); + break; + } + case UBIFS_LPT_LTAB: + node_len = c->ltab_sz; + pr_err("LEB %d:%d, ltab\n", lnum, offs); + break; + case UBIFS_LPT_LSAVE: + node_len = c->lsave_sz; + pr_err("LEB %d:%d, lsave len\n", lnum, offs); + break; + default: + ubifs_err(c, "LPT node type %d not recognized", node_type); + goto out; + } + + p += node_len; + len -= node_len; + } + + pr_err("(pid %d) finish dumping LEB %d\n", current->pid, lnum); +out: + vfree(buf); +} + +/** + * ubifs_dump_lpt_lebs - dump LPT lebs. + * @c: UBIFS file-system description object + * + * This function dumps all LPT LEBs. The caller has to make sure the LPT is + * locked. + */ +void ubifs_dump_lpt_lebs(const struct ubifs_info *c) +{ + int i; + + pr_err("(pid %d) start dumping all LPT LEBs\n", current->pid); + for (i = 0; i < c->lpt_lebs; i++) + dump_lpt_leb(c, i + c->lpt_first); + pr_err("(pid %d) finish dumping all LPT LEBs\n", current->pid); +} + +/** + * dbg_populate_lsave - debugging version of 'populate_lsave()' + * @c: UBIFS file-system description object + * + * This is a debugging version for 'populate_lsave()' which populates lsave + * with random LEBs instead of useful LEBs, which is good for test coverage. + * Returns zero if lsave has not been populated (this debugging feature is + * disabled) an non-zero if lsave has been populated. + */ +static int dbg_populate_lsave(struct ubifs_info *c) +{ + struct ubifs_lprops *lprops; + struct ubifs_lpt_heap *heap; + int i; + + if (!dbg_is_chk_gen(c)) + return 0; + if (get_random_u32_below(4)) + return 0; + + for (i = 0; i < c->lsave_cnt; i++) + c->lsave[i] = c->main_first; + + list_for_each_entry(lprops, &c->empty_list, list) + c->lsave[get_random_u32_below(c->lsave_cnt)] = lprops->lnum; + list_for_each_entry(lprops, &c->freeable_list, list) + c->lsave[get_random_u32_below(c->lsave_cnt)] = lprops->lnum; + list_for_each_entry(lprops, &c->frdi_idx_list, list) + c->lsave[get_random_u32_below(c->lsave_cnt)] = lprops->lnum; + + heap = &c->lpt_heap[LPROPS_DIRTY_IDX - 1]; + for (i = 0; i < heap->cnt; i++) + c->lsave[get_random_u32_below(c->lsave_cnt)] = heap->arr[i]->lnum; + heap = &c->lpt_heap[LPROPS_DIRTY - 1]; + for (i = 0; i < heap->cnt; i++) + c->lsave[get_random_u32_below(c->lsave_cnt)] = heap->arr[i]->lnum; + heap = &c->lpt_heap[LPROPS_FREE - 1]; + for (i = 0; i < heap->cnt; i++) + c->lsave[get_random_u32_below(c->lsave_cnt)] = heap->arr[i]->lnum; + + return 1; +} diff --git a/fs/ubifs/master.c b/fs/ubifs/master.c new file mode 100644 index 000000000..a148760fa --- /dev/null +++ b/fs/ubifs/master.c @@ -0,0 +1,476 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * + * Authors: Artem Bityutskiy (Битюцкий Артём) + * Adrian Hunter + */ + +/* This file implements reading and writing the master node */ + +#include "ubifs.h" + +/** + * ubifs_compare_master_node - compare two UBIFS master nodes + * @c: UBIFS file-system description object + * @m1: the first node + * @m2: the second node + * + * This function compares two UBIFS master nodes. Returns 0 if they are equal + * and nonzero if not. + */ +int ubifs_compare_master_node(struct ubifs_info *c, void *m1, void *m2) +{ + int ret; + int behind; + int hmac_offs = offsetof(struct ubifs_mst_node, hmac); + + /* + * Do not compare the common node header since the sequence number and + * hence the CRC are different. + */ + ret = memcmp(m1 + UBIFS_CH_SZ, m2 + UBIFS_CH_SZ, + hmac_offs - UBIFS_CH_SZ); + if (ret) + return ret; + + /* + * Do not compare the embedded HMAC as well which also must be different + * due to the different common node header. + */ + behind = hmac_offs + UBIFS_MAX_HMAC_LEN; + + if (UBIFS_MST_NODE_SZ > behind) + return memcmp(m1 + behind, m2 + behind, UBIFS_MST_NODE_SZ - behind); + + return 0; +} + +/* mst_node_check_hash - Check hash of a master node + * @c: UBIFS file-system description object + * @mst: The master node + * @expected: The expected hash of the master node + * + * This checks the hash of a master node against a given expected hash. + * Note that we have two master nodes on a UBIFS image which have different + * sequence numbers and consequently different CRCs. To be able to match + * both master nodes we exclude the common node header containing the sequence + * number and CRC from the hash. + * + * Returns 0 if the hashes are equal, a negative error code otherwise. + */ +static int mst_node_check_hash(const struct ubifs_info *c, + const struct ubifs_mst_node *mst, + const u8 *expected) +{ + u8 calc[UBIFS_MAX_HASH_LEN]; + const void *node = mst; + int ret; + + ret = crypto_shash_tfm_digest(c->hash_tfm, node + sizeof(struct ubifs_ch), + UBIFS_MST_NODE_SZ - sizeof(struct ubifs_ch), + calc); + if (ret) + return ret; + + if (ubifs_check_hash(c, expected, calc)) + return -EPERM; + + return 0; +} + +/** + * scan_for_master - search the valid master node. + * @c: UBIFS file-system description object + * + * This function scans the master node LEBs and search for the latest master + * node. Returns zero in case of success, %-EUCLEAN if there master area is + * corrupted and requires recovery, and a negative error code in case of + * failure. + */ +static int scan_for_master(struct ubifs_info *c) +{ + struct ubifs_scan_leb *sleb; + struct ubifs_scan_node *snod; + int lnum, offs = 0, nodes_cnt, err; + + lnum = UBIFS_MST_LNUM; + + sleb = ubifs_scan(c, lnum, 0, c->sbuf, 1); + if (IS_ERR(sleb)) + return PTR_ERR(sleb); + nodes_cnt = sleb->nodes_cnt; + if (nodes_cnt > 0) { + snod = list_entry(sleb->nodes.prev, struct ubifs_scan_node, + list); + if (snod->type != UBIFS_MST_NODE) + goto out_dump; + memcpy(c->mst_node, snod->node, snod->len); + offs = snod->offs; + } + ubifs_scan_destroy(sleb); + + lnum += 1; + + sleb = ubifs_scan(c, lnum, 0, c->sbuf, 1); + if (IS_ERR(sleb)) + return PTR_ERR(sleb); + if (sleb->nodes_cnt != nodes_cnt) + goto out; + if (!sleb->nodes_cnt) + goto out; + snod = list_entry(sleb->nodes.prev, struct ubifs_scan_node, list); + if (snod->type != UBIFS_MST_NODE) + goto out_dump; + if (snod->offs != offs) + goto out; + if (ubifs_compare_master_node(c, c->mst_node, snod->node)) + goto out; + + c->mst_offs = offs; + ubifs_scan_destroy(sleb); + + if (!ubifs_authenticated(c)) + return 0; + + if (ubifs_hmac_zero(c, c->mst_node->hmac)) { + err = mst_node_check_hash(c, c->mst_node, + c->sup_node->hash_mst); + if (err) + ubifs_err(c, "Failed to verify master node hash"); + } else { + err = ubifs_node_verify_hmac(c, c->mst_node, + sizeof(struct ubifs_mst_node), + offsetof(struct ubifs_mst_node, hmac)); + if (err) + ubifs_err(c, "Failed to verify master node HMAC"); + } + + if (err) + return -EPERM; + + return 0; + +out: + ubifs_scan_destroy(sleb); + return -EUCLEAN; + +out_dump: + ubifs_err(c, "unexpected node type %d master LEB %d:%d", + snod->type, lnum, snod->offs); + ubifs_scan_destroy(sleb); + return -EINVAL; +} + +/** + * validate_master - validate master node. + * @c: UBIFS file-system description object + * + * This function validates data which was read from master node. Returns zero + * if the data is all right and %-EINVAL if not. + */ +static int validate_master(const struct ubifs_info *c) +{ + long long main_sz; + int err; + + if (c->max_sqnum >= SQNUM_WATERMARK) { + err = 1; + goto out; + } + + if (c->cmt_no >= c->max_sqnum) { + err = 2; + goto out; + } + + if (c->highest_inum >= INUM_WATERMARK) { + err = 3; + goto out; + } + + if (c->lhead_lnum < UBIFS_LOG_LNUM || + c->lhead_lnum >= UBIFS_LOG_LNUM + c->log_lebs || + c->lhead_offs < 0 || c->lhead_offs >= c->leb_size || + c->lhead_offs & (c->min_io_size - 1)) { + err = 4; + goto out; + } + + if (c->zroot.lnum >= c->leb_cnt || c->zroot.lnum < c->main_first || + c->zroot.offs >= c->leb_size || c->zroot.offs & 7) { + err = 5; + goto out; + } + + if (c->zroot.len < c->ranges[UBIFS_IDX_NODE].min_len || + c->zroot.len > c->ranges[UBIFS_IDX_NODE].max_len) { + err = 6; + goto out; + } + + if (c->gc_lnum >= c->leb_cnt || c->gc_lnum < c->main_first) { + err = 7; + goto out; + } + + if (c->ihead_lnum >= c->leb_cnt || c->ihead_lnum < c->main_first || + c->ihead_offs % c->min_io_size || c->ihead_offs < 0 || + c->ihead_offs > c->leb_size || c->ihead_offs & 7) { + err = 8; + goto out; + } + + main_sz = (long long)c->main_lebs * c->leb_size; + if (c->bi.old_idx_sz & 7 || c->bi.old_idx_sz >= main_sz) { + err = 9; + goto out; + } + + if (c->lpt_lnum < c->lpt_first || c->lpt_lnum > c->lpt_last || + c->lpt_offs < 0 || c->lpt_offs + c->nnode_sz > c->leb_size) { + err = 10; + goto out; + } + + if (c->nhead_lnum < c->lpt_first || c->nhead_lnum > c->lpt_last || + c->nhead_offs < 0 || c->nhead_offs % c->min_io_size || + c->nhead_offs > c->leb_size) { + err = 11; + goto out; + } + + if (c->ltab_lnum < c->lpt_first || c->ltab_lnum > c->lpt_last || + c->ltab_offs < 0 || + c->ltab_offs + c->ltab_sz > c->leb_size) { + err = 12; + goto out; + } + + if (c->big_lpt && (c->lsave_lnum < c->lpt_first || + c->lsave_lnum > c->lpt_last || c->lsave_offs < 0 || + c->lsave_offs + c->lsave_sz > c->leb_size)) { + err = 13; + goto out; + } + + if (c->lscan_lnum < c->main_first || c->lscan_lnum >= c->leb_cnt) { + err = 14; + goto out; + } + + if (c->lst.empty_lebs < 0 || c->lst.empty_lebs > c->main_lebs - 2) { + err = 15; + goto out; + } + + if (c->lst.idx_lebs < 0 || c->lst.idx_lebs > c->main_lebs - 1) { + err = 16; + goto out; + } + + if (c->lst.total_free < 0 || c->lst.total_free > main_sz || + c->lst.total_free & 7) { + err = 17; + goto out; + } + + if (c->lst.total_dirty < 0 || (c->lst.total_dirty & 7)) { + err = 18; + goto out; + } + + if (c->lst.total_used < 0 || (c->lst.total_used & 7)) { + err = 19; + goto out; + } + + if (c->lst.total_free + c->lst.total_dirty + + c->lst.total_used > main_sz) { + err = 20; + goto out; + } + + if (c->lst.total_dead + c->lst.total_dark + + c->lst.total_used + c->bi.old_idx_sz > main_sz) { + err = 21; + goto out; + } + + if (c->lst.total_dead < 0 || + c->lst.total_dead > c->lst.total_free + c->lst.total_dirty || + c->lst.total_dead & 7) { + err = 22; + goto out; + } + + if (c->lst.total_dark < 0 || + c->lst.total_dark > c->lst.total_free + c->lst.total_dirty || + c->lst.total_dark & 7) { + err = 23; + goto out; + } + + return 0; + +out: + ubifs_err(c, "bad master node at offset %d error %d", c->mst_offs, err); + ubifs_dump_node(c, c->mst_node, c->mst_node_alsz); + return -EINVAL; +} + +/** + * ubifs_read_master - read master node. + * @c: UBIFS file-system description object + * + * This function finds and reads the master node during file-system mount. If + * the flash is empty, it creates default master node as well. Returns zero in + * case of success and a negative error code in case of failure. + */ +int ubifs_read_master(struct ubifs_info *c) +{ + int err, old_leb_cnt; + + c->mst_node = kzalloc(c->mst_node_alsz, GFP_KERNEL); + if (!c->mst_node) + return -ENOMEM; + + err = scan_for_master(c); + if (err) { + if (err == -EUCLEAN) + err = ubifs_recover_master_node(c); + if (err) + /* + * Note, we do not free 'c->mst_node' here because the + * unmount routine will take care of this. + */ + return err; + } + + /* Make sure that the recovery flag is clear */ + c->mst_node->flags &= cpu_to_le32(~UBIFS_MST_RCVRY); + + c->max_sqnum = le64_to_cpu(c->mst_node->ch.sqnum); + c->highest_inum = le64_to_cpu(c->mst_node->highest_inum); + c->cmt_no = le64_to_cpu(c->mst_node->cmt_no); + c->zroot.lnum = le32_to_cpu(c->mst_node->root_lnum); + c->zroot.offs = le32_to_cpu(c->mst_node->root_offs); + c->zroot.len = le32_to_cpu(c->mst_node->root_len); + c->lhead_lnum = le32_to_cpu(c->mst_node->log_lnum); + c->gc_lnum = le32_to_cpu(c->mst_node->gc_lnum); + c->ihead_lnum = le32_to_cpu(c->mst_node->ihead_lnum); + c->ihead_offs = le32_to_cpu(c->mst_node->ihead_offs); + c->bi.old_idx_sz = le64_to_cpu(c->mst_node->index_size); + c->lpt_lnum = le32_to_cpu(c->mst_node->lpt_lnum); + c->lpt_offs = le32_to_cpu(c->mst_node->lpt_offs); + c->nhead_lnum = le32_to_cpu(c->mst_node->nhead_lnum); + c->nhead_offs = le32_to_cpu(c->mst_node->nhead_offs); + c->ltab_lnum = le32_to_cpu(c->mst_node->ltab_lnum); + c->ltab_offs = le32_to_cpu(c->mst_node->ltab_offs); + c->lsave_lnum = le32_to_cpu(c->mst_node->lsave_lnum); + c->lsave_offs = le32_to_cpu(c->mst_node->lsave_offs); + c->lscan_lnum = le32_to_cpu(c->mst_node->lscan_lnum); + c->lst.empty_lebs = le32_to_cpu(c->mst_node->empty_lebs); + c->lst.idx_lebs = le32_to_cpu(c->mst_node->idx_lebs); + old_leb_cnt = le32_to_cpu(c->mst_node->leb_cnt); + c->lst.total_free = le64_to_cpu(c->mst_node->total_free); + c->lst.total_dirty = le64_to_cpu(c->mst_node->total_dirty); + c->lst.total_used = le64_to_cpu(c->mst_node->total_used); + c->lst.total_dead = le64_to_cpu(c->mst_node->total_dead); + c->lst.total_dark = le64_to_cpu(c->mst_node->total_dark); + + ubifs_copy_hash(c, c->mst_node->hash_root_idx, c->zroot.hash); + + c->calc_idx_sz = c->bi.old_idx_sz; + + if (c->mst_node->flags & cpu_to_le32(UBIFS_MST_NO_ORPHS)) + c->no_orphs = 1; + + if (old_leb_cnt != c->leb_cnt) { + /* The file system has been resized */ + int growth = c->leb_cnt - old_leb_cnt; + + if (c->leb_cnt < old_leb_cnt || + c->leb_cnt < UBIFS_MIN_LEB_CNT) { + ubifs_err(c, "bad leb_cnt on master node"); + ubifs_dump_node(c, c->mst_node, c->mst_node_alsz); + return -EINVAL; + } + + dbg_mnt("Auto resizing (master) from %d LEBs to %d LEBs", + old_leb_cnt, c->leb_cnt); + c->lst.empty_lebs += growth; + c->lst.total_free += growth * (long long)c->leb_size; + c->lst.total_dark += growth * (long long)c->dark_wm; + + /* + * Reflect changes back onto the master node. N.B. the master + * node gets written immediately whenever mounting (or + * remounting) in read-write mode, so we do not need to write it + * here. + */ + c->mst_node->leb_cnt = cpu_to_le32(c->leb_cnt); + c->mst_node->empty_lebs = cpu_to_le32(c->lst.empty_lebs); + c->mst_node->total_free = cpu_to_le64(c->lst.total_free); + c->mst_node->total_dark = cpu_to_le64(c->lst.total_dark); + } + + err = validate_master(c); + if (err) + return err; + + err = dbg_old_index_check_init(c, &c->zroot); + + return err; +} + +/** + * ubifs_write_master - write master node. + * @c: UBIFS file-system description object + * + * This function writes the master node. Returns zero in case of success and a + * negative error code in case of failure. The master node is written twice to + * enable recovery. + */ +int ubifs_write_master(struct ubifs_info *c) +{ + int err, lnum, offs, len; + + ubifs_assert(c, !c->ro_media && !c->ro_mount); + if (c->ro_error) + return -EROFS; + + lnum = UBIFS_MST_LNUM; + offs = c->mst_offs + c->mst_node_alsz; + len = UBIFS_MST_NODE_SZ; + + if (offs + UBIFS_MST_NODE_SZ > c->leb_size) { + err = ubifs_leb_unmap(c, lnum); + if (err) + return err; + offs = 0; + } + + c->mst_offs = offs; + c->mst_node->highest_inum = cpu_to_le64(c->highest_inum); + + ubifs_copy_hash(c, c->zroot.hash, c->mst_node->hash_root_idx); + err = ubifs_write_node_hmac(c, c->mst_node, len, lnum, offs, + offsetof(struct ubifs_mst_node, hmac)); + if (err) + return err; + + lnum += 1; + + if (offs == 0) { + err = ubifs_leb_unmap(c, lnum); + if (err) + return err; + } + err = ubifs_write_node_hmac(c, c->mst_node, len, lnum, offs, + offsetof(struct ubifs_mst_node, hmac)); + + return err; +} diff --git a/fs/ubifs/misc.c b/fs/ubifs/misc.c new file mode 100644 index 000000000..cd23de0f2 --- /dev/null +++ b/fs/ubifs/misc.c @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include "ubifs.h" + +/* Normal UBIFS messages */ +void ubifs_msg(const struct ubifs_info *c, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + pr_notice("UBIFS (ubi%d:%d): %pV\n", + c->vi.ubi_num, c->vi.vol_id, &vaf); + + va_end(args); +} \ + +/* UBIFS error messages */ +void ubifs_err(const struct ubifs_info *c, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + pr_err("UBIFS error (ubi%d:%d pid %d): %ps: %pV\n", + c->vi.ubi_num, c->vi.vol_id, current->pid, + __builtin_return_address(0), + &vaf); + + va_end(args); +} \ + +/* UBIFS warning messages */ +void ubifs_warn(const struct ubifs_info *c, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + pr_warn("UBIFS warning (ubi%d:%d pid %d): %ps: %pV\n", + c->vi.ubi_num, c->vi.vol_id, current->pid, + __builtin_return_address(0), + &vaf); + + va_end(args); +} + +static char *assert_names[] = { + [ASSACT_REPORT] = "report", + [ASSACT_RO] = "read-only", + [ASSACT_PANIC] = "panic", +}; + +const char *ubifs_assert_action_name(struct ubifs_info *c) +{ + return assert_names[c->assert_action]; +} diff --git a/fs/ubifs/misc.h b/fs/ubifs/misc.h new file mode 100644 index 000000000..615878e88 --- /dev/null +++ b/fs/ubifs/misc.h @@ -0,0 +1,289 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation + * + * Authors: Artem Bityutskiy (Битюцкий Артём) + * Adrian Hunter + */ + +/* + * This file contains miscellaneous helper functions. + */ + +#ifndef __UBIFS_MISC_H__ +#define __UBIFS_MISC_H__ + +/** + * ubifs_zn_dirty - check if znode is dirty. + * @znode: znode to check + * + * This helper function returns %1 if @znode is dirty and %0 otherwise. + */ +static inline int ubifs_zn_dirty(const struct ubifs_znode *znode) +{ + return !!test_bit(DIRTY_ZNODE, &znode->flags); +} + +/** + * ubifs_zn_obsolete - check if znode is obsolete. + * @znode: znode to check + * + * This helper function returns %1 if @znode is obsolete and %0 otherwise. + */ +static inline int ubifs_zn_obsolete(const struct ubifs_znode *znode) +{ + return !!test_bit(OBSOLETE_ZNODE, &znode->flags); +} + +/** + * ubifs_zn_cow - check if znode has to be copied on write. + * @znode: znode to check + * + * This helper function returns %1 if @znode is has COW flag set and %0 + * otherwise. + */ +static inline int ubifs_zn_cow(const struct ubifs_znode *znode) +{ + return !!test_bit(COW_ZNODE, &znode->flags); +} + +/** + * ubifs_wake_up_bgt - wake up background thread. + * @c: UBIFS file-system description object + */ +static inline void ubifs_wake_up_bgt(struct ubifs_info *c) +{ + if (c->bgt && !c->need_bgt) { + c->need_bgt = 1; + wake_up_process(c->bgt); + } +} + +/** + * ubifs_tnc_find_child - find next child in znode. + * @znode: znode to search at + * @start: the zbranch index to start at + * + * This helper function looks for znode child starting at index @start. Returns + * the child or %NULL if no children were found. + */ +static inline struct ubifs_znode * +ubifs_tnc_find_child(struct ubifs_znode *znode, int start) +{ + while (start < znode->child_cnt) { + if (znode->zbranch[start].znode) + return znode->zbranch[start].znode; + start += 1; + } + + return NULL; +} + +/** + * ubifs_inode - get UBIFS inode information by VFS 'struct inode' object. + * @inode: the VFS 'struct inode' pointer + */ +static inline struct ubifs_inode *ubifs_inode(const struct inode *inode) +{ + return container_of(inode, struct ubifs_inode, vfs_inode); +} + +/** + * ubifs_compr_present - check if compressor was compiled in. + * @compr_type: compressor type to check + * @c: the UBIFS file-system description object + * + * This function returns %1 of compressor of type @compr_type is present, and + * %0 if not. + */ +static inline int ubifs_compr_present(struct ubifs_info *c, int compr_type) +{ + ubifs_assert(c, compr_type >= 0 && compr_type < UBIFS_COMPR_TYPES_CNT); + return !!ubifs_compressors[compr_type]->capi_name; +} + +/** + * ubifs_compr_name - get compressor name string by its type. + * @compr_type: compressor type + * @c: the UBIFS file-system description object + * + * This function returns compressor type string. + */ +static inline const char *ubifs_compr_name(struct ubifs_info *c, int compr_type) +{ + ubifs_assert(c, compr_type >= 0 && compr_type < UBIFS_COMPR_TYPES_CNT); + return ubifs_compressors[compr_type]->name; +} + +/** + * ubifs_wbuf_sync - synchronize write-buffer. + * @wbuf: write-buffer to synchronize + * + * This is the same as 'ubifs_wbuf_sync_nolock()' but it does not assume + * that the write-buffer is already locked. + */ +static inline int ubifs_wbuf_sync(struct ubifs_wbuf *wbuf) +{ + int err; + + mutex_lock_nested(&wbuf->io_mutex, wbuf->jhead); + err = ubifs_wbuf_sync_nolock(wbuf); + mutex_unlock(&wbuf->io_mutex); + return err; +} + +/** + * ubifs_encode_dev - encode device node IDs. + * @dev: UBIFS device node information + * @rdev: device IDs to encode + * + * This is a helper function which encodes major/minor numbers of a device node + * into UBIFS device node description. We use standard Linux "new" and "huge" + * encodings. + */ +static inline int ubifs_encode_dev(union ubifs_dev_desc *dev, dev_t rdev) +{ + dev->new = cpu_to_le32(new_encode_dev(rdev)); + return sizeof(dev->new); +} + +/** + * ubifs_add_dirt - add dirty space to LEB properties. + * @c: the UBIFS file-system description object + * @lnum: LEB to add dirty space for + * @dirty: dirty space to add + * + * This is a helper function which increased amount of dirty LEB space. Returns + * zero in case of success and a negative error code in case of failure. + */ +static inline int ubifs_add_dirt(struct ubifs_info *c, int lnum, int dirty) +{ + return ubifs_update_one_lp(c, lnum, LPROPS_NC, dirty, 0, 0); +} + +/** + * ubifs_return_leb - return LEB to lprops. + * @c: the UBIFS file-system description object + * @lnum: LEB to return + * + * This helper function cleans the "taken" flag of a logical eraseblock in the + * lprops. Returns zero in case of success and a negative error code in case of + * failure. + */ +static inline int ubifs_return_leb(struct ubifs_info *c, int lnum) +{ + return ubifs_change_one_lp(c, lnum, LPROPS_NC, LPROPS_NC, 0, + LPROPS_TAKEN, 0); +} + +/** + * ubifs_idx_node_sz - return index node size. + * @c: the UBIFS file-system description object + * @child_cnt: number of children of this index node + */ +static inline int ubifs_idx_node_sz(const struct ubifs_info *c, int child_cnt) +{ + return UBIFS_IDX_NODE_SZ + (UBIFS_BRANCH_SZ + c->key_len + c->hash_len) + * child_cnt; +} + +/** + * ubifs_idx_branch - return pointer to an index branch. + * @c: the UBIFS file-system description object + * @idx: index node + * @bnum: branch number + */ +static inline +struct ubifs_branch *ubifs_idx_branch(const struct ubifs_info *c, + const struct ubifs_idx_node *idx, + int bnum) +{ + return (struct ubifs_branch *)((void *)idx->branches + + (UBIFS_BRANCH_SZ + c->key_len + c->hash_len) * bnum); +} + +/** + * ubifs_idx_key - return pointer to an index key. + * @c: the UBIFS file-system description object + * @idx: index node + */ +static inline void *ubifs_idx_key(const struct ubifs_info *c, + const struct ubifs_idx_node *idx) +{ + return (void *)((struct ubifs_branch *)idx->branches)->key; +} + +/** + * ubifs_tnc_lookup - look up a file-system node. + * @c: UBIFS file-system description object + * @key: node key to lookup + * @node: the node is returned here + * + * This function look up and reads node with key @key. The caller has to make + * sure the @node buffer is large enough to fit the node. Returns zero in case + * of success, %-ENOENT if the node was not found, and a negative error code in + * case of failure. + */ +static inline int ubifs_tnc_lookup(struct ubifs_info *c, + const union ubifs_key *key, void *node) +{ + return ubifs_tnc_locate(c, key, node, NULL, NULL); +} + +/** + * ubifs_get_lprops - get reference to LEB properties. + * @c: the UBIFS file-system description object + * + * This function locks lprops. Lprops have to be unlocked by + * 'ubifs_release_lprops()'. + */ +static inline void ubifs_get_lprops(struct ubifs_info *c) +{ + mutex_lock(&c->lp_mutex); +} + +/** + * ubifs_release_lprops - release lprops lock. + * @c: the UBIFS file-system description object + * + * This function has to be called after each 'ubifs_get_lprops()' call to + * unlock lprops. + */ +static inline void ubifs_release_lprops(struct ubifs_info *c) +{ + ubifs_assert(c, mutex_is_locked(&c->lp_mutex)); + ubifs_assert(c, c->lst.empty_lebs >= 0 && + c->lst.empty_lebs <= c->main_lebs); + mutex_unlock(&c->lp_mutex); +} + +/** + * ubifs_next_log_lnum - switch to the next log LEB. + * @c: UBIFS file-system description object + * @lnum: current log LEB + * + * This helper function returns the log LEB number which goes next after LEB + * 'lnum'. + */ +static inline int ubifs_next_log_lnum(const struct ubifs_info *c, int lnum) +{ + lnum += 1; + if (lnum > c->log_last) + lnum = UBIFS_LOG_LNUM; + + return lnum; +} + +static inline int ubifs_xattr_max_cnt(struct ubifs_info *c) +{ + int max_xattrs = (c->leb_size / 2) / UBIFS_INO_NODE_SZ; + + ubifs_assert(c, max_xattrs < c->max_orphans); + return max_xattrs; +} + +const char *ubifs_assert_action_name(struct ubifs_info *c); + +#endif /* __UBIFS_MISC_H__ */ diff --git a/fs/ubifs/orphan.c b/fs/ubifs/orphan.c new file mode 100644 index 000000000..85e467646 --- /dev/null +++ b/fs/ubifs/orphan.c @@ -0,0 +1,947 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * + * Author: Adrian Hunter + */ + +#include "ubifs.h" + +/* + * An orphan is an inode number whose inode node has been committed to the index + * with a link count of zero. That happens when an open file is deleted + * (unlinked) and then a commit is run. In the normal course of events the inode + * would be deleted when the file is closed. However in the case of an unclean + * unmount, orphans need to be accounted for. After an unclean unmount, the + * orphans' inodes must be deleted which means either scanning the entire index + * looking for them, or keeping a list on flash somewhere. This unit implements + * the latter approach. + * + * The orphan area is a fixed number of LEBs situated between the LPT area and + * the main area. The number of orphan area LEBs is specified when the file + * system is created. The minimum number is 1. The size of the orphan area + * should be so that it can hold the maximum number of orphans that are expected + * to ever exist at one time. + * + * The number of orphans that can fit in a LEB is: + * + * (c->leb_size - UBIFS_ORPH_NODE_SZ) / sizeof(__le64) + * + * For example: a 15872 byte LEB can fit 1980 orphans so 1 LEB may be enough. + * + * Orphans are accumulated in a rb-tree. When an inode's link count drops to + * zero, the inode number is added to the rb-tree. It is removed from the tree + * when the inode is deleted. Any new orphans that are in the orphan tree when + * the commit is run, are written to the orphan area in 1 or more orphan nodes. + * If the orphan area is full, it is consolidated to make space. There is + * always enough space because validation prevents the user from creating more + * than the maximum number of orphans allowed. + */ + +static int dbg_check_orphans(struct ubifs_info *c); + +/** + * ubifs_add_orphan - add an orphan. + * @c: UBIFS file-system description object + * @inum: orphan inode number + * + * Add an orphan. This function is called when an inodes link count drops to + * zero. + */ +int ubifs_add_orphan(struct ubifs_info *c, ino_t inum) +{ + struct ubifs_orphan *orphan, *o; + struct rb_node **p, *parent = NULL; + + orphan = kzalloc_obj(struct ubifs_orphan, GFP_NOFS); + if (!orphan) + return -ENOMEM; + orphan->inum = inum; + orphan->new = 1; + + spin_lock(&c->orphan_lock); + if (c->tot_orphans >= c->max_orphans) { + spin_unlock(&c->orphan_lock); + kfree(orphan); + return -ENFILE; + } + p = &c->orph_tree.rb_node; + while (*p) { + parent = *p; + o = rb_entry(parent, struct ubifs_orphan, rb); + if (inum < o->inum) + p = &(*p)->rb_left; + else if (inum > o->inum) + p = &(*p)->rb_right; + else { + ubifs_err(c, "ino %lu orphaned twice", (unsigned long)inum); + spin_unlock(&c->orphan_lock); + kfree(orphan); + return -EINVAL; + } + } + c->tot_orphans += 1; + c->new_orphans += 1; + rb_link_node(&orphan->rb, parent, p); + rb_insert_color(&orphan->rb, &c->orph_tree); + list_add_tail(&orphan->list, &c->orph_list); + list_add_tail(&orphan->new_list, &c->orph_new); + + spin_unlock(&c->orphan_lock); + dbg_gen("ino %lu", (unsigned long)inum); + return 0; +} + +static struct ubifs_orphan *lookup_orphan(struct ubifs_info *c, ino_t inum) +{ + struct ubifs_orphan *o; + struct rb_node *p; + + p = c->orph_tree.rb_node; + while (p) { + o = rb_entry(p, struct ubifs_orphan, rb); + if (inum < o->inum) + p = p->rb_left; + else if (inum > o->inum) + p = p->rb_right; + else { + return o; + } + } + return NULL; +} + +static void __orphan_drop(struct ubifs_info *c, struct ubifs_orphan *o) +{ + rb_erase(&o->rb, &c->orph_tree); + list_del(&o->list); + c->tot_orphans -= 1; + + if (o->new) { + list_del(&o->new_list); + c->new_orphans -= 1; + } + + kfree(o); +} + +static void orphan_delete(struct ubifs_info *c, struct ubifs_orphan *orph) +{ + if (orph->del) { + dbg_gen("deleted twice ino %lu", (unsigned long)orph->inum); + return; + } + + if (orph->cmt) { + orph->del = 1; + rb_erase(&orph->rb, &c->orph_tree); + orph->dnext = c->orph_dnext; + c->orph_dnext = orph; + dbg_gen("delete later ino %lu", (unsigned long)orph->inum); + return; + } + + __orphan_drop(c, orph); +} + +/** + * ubifs_delete_orphan - delete an orphan. + * @c: UBIFS file-system description object + * @inum: orphan inode number + * + * Delete an orphan. This function is called when an inode is deleted. + */ +void ubifs_delete_orphan(struct ubifs_info *c, ino_t inum) +{ + struct ubifs_orphan *orph; + + spin_lock(&c->orphan_lock); + + orph = lookup_orphan(c, inum); + if (!orph) { + spin_unlock(&c->orphan_lock); + ubifs_err(c, "missing orphan ino %lu", (unsigned long)inum); + dump_stack(); + + return; + } + + orphan_delete(c, orph); + + spin_unlock(&c->orphan_lock); +} + +/** + * ubifs_orphan_start_commit - start commit of orphans. + * @c: UBIFS file-system description object + * + * Start commit of orphans. + */ +int ubifs_orphan_start_commit(struct ubifs_info *c) +{ + struct ubifs_orphan *orphan, **last; + + spin_lock(&c->orphan_lock); + last = &c->orph_cnext; + list_for_each_entry(orphan, &c->orph_new, new_list) { + ubifs_assert(c, orphan->new); + ubifs_assert(c, !orphan->cmt); + orphan->new = 0; + orphan->cmt = 1; + *last = orphan; + last = &orphan->cnext; + } + *last = NULL; + c->cmt_orphans = c->new_orphans; + c->new_orphans = 0; + dbg_cmt("%d orphans to commit", c->cmt_orphans); + INIT_LIST_HEAD(&c->orph_new); + if (c->tot_orphans == 0) + c->no_orphs = 1; + else + c->no_orphs = 0; + spin_unlock(&c->orphan_lock); + return 0; +} + +/** + * avail_orphs - calculate available space. + * @c: UBIFS file-system description object + * + * This function returns the number of orphans that can be written in the + * available space. + */ +static int avail_orphs(struct ubifs_info *c) +{ + int avail_lebs, avail, gap; + + avail_lebs = c->orph_lebs - (c->ohead_lnum - c->orph_first) - 1; + avail = avail_lebs * + ((c->leb_size - UBIFS_ORPH_NODE_SZ) / sizeof(__le64)); + gap = c->leb_size - c->ohead_offs; + if (gap >= UBIFS_ORPH_NODE_SZ + sizeof(__le64)) + avail += (gap - UBIFS_ORPH_NODE_SZ) / sizeof(__le64); + return avail; +} + +/** + * tot_avail_orphs - calculate total space. + * @c: UBIFS file-system description object + * + * This function returns the number of orphans that can be written in half + * the total space. That leaves half the space for adding new orphans. + */ +static int tot_avail_orphs(struct ubifs_info *c) +{ + int avail_lebs, avail; + + avail_lebs = c->orph_lebs; + avail = avail_lebs * + ((c->leb_size - UBIFS_ORPH_NODE_SZ) / sizeof(__le64)); + return avail / 2; +} + +/** + * do_write_orph_node - write a node to the orphan head. + * @c: UBIFS file-system description object + * @len: length of node + * @atomic: write atomically + * + * This function writes a node to the orphan head from the orphan buffer. If + * %atomic is not zero, then the write is done atomically. On success, %0 is + * returned, otherwise a negative error code is returned. + */ +static int do_write_orph_node(struct ubifs_info *c, int len, int atomic) +{ + int err = 0; + + if (atomic) { + ubifs_assert(c, c->ohead_offs == 0); + ubifs_prepare_node(c, c->orph_buf, len, 1); + len = ALIGN(len, c->min_io_size); + err = ubifs_leb_change(c, c->ohead_lnum, c->orph_buf, len); + } else { + if (c->ohead_offs == 0) { + /* Ensure LEB has been unmapped */ + err = ubifs_leb_unmap(c, c->ohead_lnum); + if (err) + return err; + } + err = ubifs_write_node(c, c->orph_buf, len, c->ohead_lnum, + c->ohead_offs); + } + return err; +} + +/** + * write_orph_node - write an orphan node. + * @c: UBIFS file-system description object + * @atomic: write atomically + * + * This function builds an orphan node from the cnext list and writes it to the + * orphan head. On success, %0 is returned, otherwise a negative error code + * is returned. + */ +static int write_orph_node(struct ubifs_info *c, int atomic) +{ + struct ubifs_orphan *orphan, *cnext; + struct ubifs_orph_node *orph; + int gap, err, len, cnt, i; + + ubifs_assert(c, c->cmt_orphans > 0); + gap = c->leb_size - c->ohead_offs; + if (gap < UBIFS_ORPH_NODE_SZ + sizeof(__le64)) { + c->ohead_lnum += 1; + c->ohead_offs = 0; + gap = c->leb_size; + if (c->ohead_lnum > c->orph_last) { + /* + * We limit the number of orphans so that this should + * never happen. + */ + ubifs_err(c, "out of space in orphan area"); + return -EINVAL; + } + } + cnt = (gap - UBIFS_ORPH_NODE_SZ) / sizeof(__le64); + if (cnt > c->cmt_orphans) + cnt = c->cmt_orphans; + len = UBIFS_ORPH_NODE_SZ + cnt * sizeof(__le64); + ubifs_assert(c, c->orph_buf); + orph = c->orph_buf; + orph->ch.node_type = UBIFS_ORPH_NODE; + spin_lock(&c->orphan_lock); + cnext = c->orph_cnext; + for (i = 0; i < cnt; i++) { + orphan = cnext; + ubifs_assert(c, orphan->cmt); + orph->inos[i] = cpu_to_le64(orphan->inum); + orphan->cmt = 0; + cnext = orphan->cnext; + orphan->cnext = NULL; + } + c->orph_cnext = cnext; + c->cmt_orphans -= cnt; + spin_unlock(&c->orphan_lock); + if (c->cmt_orphans) + orph->cmt_no = cpu_to_le64(c->cmt_no); + else + /* Mark the last node of the commit */ + orph->cmt_no = cpu_to_le64((c->cmt_no) | (1ULL << 63)); + ubifs_assert(c, c->ohead_offs + len <= c->leb_size); + ubifs_assert(c, c->ohead_lnum >= c->orph_first); + ubifs_assert(c, c->ohead_lnum <= c->orph_last); + err = do_write_orph_node(c, len, atomic); + c->ohead_offs += ALIGN(len, c->min_io_size); + c->ohead_offs = ALIGN(c->ohead_offs, 8); + return err; +} + +/** + * write_orph_nodes - write orphan nodes until there are no more to commit. + * @c: UBIFS file-system description object + * @atomic: write atomically + * + * This function writes orphan nodes for all the orphans to commit. On success, + * %0 is returned, otherwise a negative error code is returned. + */ +static int write_orph_nodes(struct ubifs_info *c, int atomic) +{ + int err; + + while (c->cmt_orphans > 0) { + err = write_orph_node(c, atomic); + if (err) + return err; + } + if (atomic) { + int lnum; + + /* Unmap any unused LEBs after consolidation */ + for (lnum = c->ohead_lnum + 1; lnum <= c->orph_last; lnum++) { + err = ubifs_leb_unmap(c, lnum); + if (err) + return err; + } + } + return 0; +} + +/** + * consolidate - consolidate the orphan area. + * @c: UBIFS file-system description object + * + * This function enables consolidation by putting all the orphans into the list + * to commit. The list is in the order that the orphans were added, and the + * LEBs are written atomically in order, so at no time can orphans be lost by + * an unclean unmount. + * + * This function returns %0 on success and a negative error code on failure. + */ +static int consolidate(struct ubifs_info *c) +{ + int tot_avail = tot_avail_orphs(c), err = 0; + + spin_lock(&c->orphan_lock); + dbg_cmt("there is space for %d orphans and there are %d", + tot_avail, c->tot_orphans); + if (c->tot_orphans - c->new_orphans <= tot_avail) { + struct ubifs_orphan *orphan, **last; + int cnt = 0; + + /* Change the cnext list to include all non-new orphans */ + last = &c->orph_cnext; + list_for_each_entry(orphan, &c->orph_list, list) { + if (orphan->new) + continue; + orphan->cmt = 1; + *last = orphan; + last = &orphan->cnext; + cnt += 1; + } + *last = NULL; + ubifs_assert(c, cnt == c->tot_orphans - c->new_orphans); + c->cmt_orphans = cnt; + c->ohead_lnum = c->orph_first; + c->ohead_offs = 0; + } else { + /* + * We limit the number of orphans so that this should + * never happen. + */ + ubifs_err(c, "out of space in orphan area"); + err = -EINVAL; + } + spin_unlock(&c->orphan_lock); + return err; +} + +/** + * commit_orphans - commit orphans. + * @c: UBIFS file-system description object + * + * This function commits orphans to flash. On success, %0 is returned, + * otherwise a negative error code is returned. + */ +static int commit_orphans(struct ubifs_info *c) +{ + int avail, atomic = 0, err; + + ubifs_assert(c, c->cmt_orphans > 0); + avail = avail_orphs(c); + if (avail < c->cmt_orphans) { + /* Not enough space to write new orphans, so consolidate */ + err = consolidate(c); + if (err) + return err; + atomic = 1; + } + err = write_orph_nodes(c, atomic); + return err; +} + +/** + * erase_deleted - erase the orphans marked for deletion. + * @c: UBIFS file-system description object + * + * During commit, the orphans being committed cannot be deleted, so they are + * marked for deletion and deleted by this function. Also, the recovery + * adds killed orphans to the deletion list, and therefore they are deleted + * here too. + */ +static void erase_deleted(struct ubifs_info *c) +{ + struct ubifs_orphan *orphan, *dnext; + + spin_lock(&c->orphan_lock); + dnext = c->orph_dnext; + while (dnext) { + orphan = dnext; + dnext = orphan->dnext; + ubifs_assert(c, !orphan->new); + ubifs_assert(c, orphan->del); + list_del(&orphan->list); + c->tot_orphans -= 1; + dbg_gen("deleting orphan ino %lu", (unsigned long)orphan->inum); + kfree(orphan); + } + c->orph_dnext = NULL; + spin_unlock(&c->orphan_lock); +} + +/** + * ubifs_orphan_end_commit - end commit of orphans. + * @c: UBIFS file-system description object + * + * End commit of orphans. + */ +int ubifs_orphan_end_commit(struct ubifs_info *c) +{ + int err; + + if (c->cmt_orphans != 0) { + err = commit_orphans(c); + if (err) + return err; + } + erase_deleted(c); + err = dbg_check_orphans(c); + return err; +} + +/** + * ubifs_clear_orphans - erase all LEBs used for orphans. + * @c: UBIFS file-system description object + * + * If recovery is not required, then the orphans from the previous session + * are not needed. This function locates the LEBs used to record + * orphans, and un-maps them. + */ +int ubifs_clear_orphans(struct ubifs_info *c) +{ + int lnum, err; + + for (lnum = c->orph_first; lnum <= c->orph_last; lnum++) { + err = ubifs_leb_unmap(c, lnum); + if (err) + return err; + } + c->ohead_lnum = c->orph_first; + c->ohead_offs = 0; + return 0; +} + +/** + * do_kill_orphans - remove orphan inodes from the index. + * @c: UBIFS file-system description object + * @sleb: scanned LEB + * @last_cmt_no: cmt_no of last orphan node read is passed and returned here + * @outofdate: whether the LEB is out of date is returned here + * @last_flagged: whether the end orphan node is encountered + * + * This function is a helper to the 'kill_orphans()' function. It goes through + * every orphan node in a LEB and for every inode number recorded, removes + * all keys for that inode from the TNC. + */ +static int do_kill_orphans(struct ubifs_info *c, struct ubifs_scan_leb *sleb, + unsigned long long *last_cmt_no, int *outofdate, + int *last_flagged) +{ + struct ubifs_scan_node *snod; + struct ubifs_orph_node *orph; + struct ubifs_ino_node *ino = NULL; + unsigned long long cmt_no; + ino_t inum; + int i, n, err, first = 1; + + ino = kmalloc(UBIFS_MAX_INO_NODE_SZ, GFP_NOFS); + if (!ino) + return -ENOMEM; + + list_for_each_entry(snod, &sleb->nodes, list) { + if (snod->type != UBIFS_ORPH_NODE) { + ubifs_err(c, "invalid node type %d in orphan area at %d:%d", + snod->type, sleb->lnum, snod->offs); + ubifs_dump_node(c, snod->node, + c->leb_size - snod->offs); + err = -EINVAL; + goto out_free; + } + + orph = snod->node; + + /* Check commit number */ + cmt_no = le64_to_cpu(orph->cmt_no) & LLONG_MAX; + /* + * The commit number on the master node may be less, because + * of a failed commit. If there are several failed commits in a + * row, the commit number written on orphan nodes will continue + * to increase (because the commit number is adjusted here) even + * though the commit number on the master node stays the same + * because the master node has not been re-written. + */ + if (cmt_no > c->cmt_no) + c->cmt_no = cmt_no; + if (cmt_no < *last_cmt_no && *last_flagged) { + /* + * The last orphan node had a higher commit number and + * was flagged as the last written for that commit + * number. That makes this orphan node, out of date. + */ + if (!first) { + ubifs_err(c, "out of order commit number %llu in orphan node at %d:%d", + cmt_no, sleb->lnum, snod->offs); + ubifs_dump_node(c, snod->node, + c->leb_size - snod->offs); + err = -EINVAL; + goto out_free; + } + dbg_rcvry("out of date LEB %d", sleb->lnum); + *outofdate = 1; + err = 0; + goto out_free; + } + + if (first) + first = 0; + + n = (le32_to_cpu(orph->ch.len) - UBIFS_ORPH_NODE_SZ) >> 3; + for (i = 0; i < n; i++) { + union ubifs_key key; + + inum = le64_to_cpu(orph->inos[i]); + + ino_key_init(c, &key, inum); + err = ubifs_tnc_lookup(c, &key, ino); + if (err && err != -ENOENT) + goto out_free; + + /* + * Check whether an inode can really get deleted. + * linkat() with O_TMPFILE allows rebirth of an inode. + */ + if (err == 0 && ino->nlink == 0) { + dbg_rcvry("deleting orphaned inode %lu", + (unsigned long)inum); + + err = ubifs_tnc_remove_ino(c, inum); + if (err) + goto out_ro; + } + } + + *last_cmt_no = cmt_no; + if (le64_to_cpu(orph->cmt_no) & (1ULL << 63)) { + dbg_rcvry("last orph node for commit %llu at %d:%d", + cmt_no, sleb->lnum, snod->offs); + *last_flagged = 1; + } else + *last_flagged = 0; + } + + err = 0; +out_free: + kfree(ino); + return err; + +out_ro: + ubifs_ro_mode(c, err); + kfree(ino); + return err; +} + +/** + * kill_orphans - remove all orphan inodes from the index. + * @c: UBIFS file-system description object + * + * If recovery is required, then orphan inodes recorded during the previous + * session (which ended with an unclean unmount) must be deleted from the index. + * This is done by updating the TNC, but since the index is not updated until + * the next commit, the LEBs where the orphan information is recorded are not + * erased until the next commit. + */ +static int kill_orphans(struct ubifs_info *c) +{ + unsigned long long last_cmt_no = 0; + int lnum, err = 0, outofdate = 0, last_flagged = 0; + + c->ohead_lnum = c->orph_first; + c->ohead_offs = 0; + /* Check no-orphans flag and skip this if no orphans */ + if (c->no_orphs) { + dbg_rcvry("no orphans"); + return 0; + } + /* + * Orph nodes always start at c->orph_first and are written to each + * successive LEB in turn. Generally unused LEBs will have been unmapped + * but may contain out of date orphan nodes if the unmap didn't go + * through. In addition, the last orphan node written for each commit is + * marked (top bit of orph->cmt_no is set to 1). It is possible that + * there are orphan nodes from the next commit (i.e. the commit did not + * complete successfully). In that case, no orphans will have been lost + * due to the way that orphans are written, and any orphans added will + * be valid orphans anyway and so can be deleted. + */ + for (lnum = c->orph_first; lnum <= c->orph_last; lnum++) { + struct ubifs_scan_leb *sleb; + + dbg_rcvry("LEB %d", lnum); + sleb = ubifs_scan(c, lnum, 0, c->sbuf, 1); + if (IS_ERR(sleb)) { + if (PTR_ERR(sleb) == -EUCLEAN) + sleb = ubifs_recover_leb(c, lnum, 0, + c->sbuf, -1); + if (IS_ERR(sleb)) { + err = PTR_ERR(sleb); + break; + } + } + err = do_kill_orphans(c, sleb, &last_cmt_no, &outofdate, + &last_flagged); + if (err || outofdate) { + ubifs_scan_destroy(sleb); + break; + } + if (sleb->endpt) { + c->ohead_lnum = lnum; + c->ohead_offs = sleb->endpt; + } + ubifs_scan_destroy(sleb); + } + return err; +} + +/** + * ubifs_mount_orphans - delete orphan inodes and erase LEBs that recorded them. + * @c: UBIFS file-system description object + * @unclean: indicates recovery from unclean unmount + * @read_only: indicates read only mount + * + * This function is called when mounting to erase orphans from the previous + * session. If UBIFS was not unmounted cleanly, then the inodes recorded as + * orphans are deleted. + */ +int ubifs_mount_orphans(struct ubifs_info *c, int unclean, int read_only) +{ + int err = 0; + + c->max_orphans = tot_avail_orphs(c); + + if (!read_only) { + c->orph_buf = vmalloc(c->leb_size); + if (!c->orph_buf) + return -ENOMEM; + } + + if (unclean) + err = kill_orphans(c); + else if (!read_only) + err = ubifs_clear_orphans(c); + + return err; +} + +/* + * Everything below is related to debugging. + */ + +struct check_orphan { + struct rb_node rb; + ino_t inum; +}; + +struct check_info { + unsigned long last_ino; + unsigned long tot_inos; + unsigned long missing; + unsigned long long leaf_cnt; + struct ubifs_ino_node *node; + struct rb_root root; +}; + +static bool dbg_find_orphan(struct ubifs_info *c, ino_t inum) +{ + bool found = false; + + spin_lock(&c->orphan_lock); + found = !!lookup_orphan(c, inum); + spin_unlock(&c->orphan_lock); + + return found; +} + +static int dbg_ins_check_orphan(struct rb_root *root, ino_t inum) +{ + struct check_orphan *orphan, *o; + struct rb_node **p, *parent = NULL; + + orphan = kzalloc_obj(struct check_orphan, GFP_NOFS); + if (!orphan) + return -ENOMEM; + orphan->inum = inum; + + p = &root->rb_node; + while (*p) { + parent = *p; + o = rb_entry(parent, struct check_orphan, rb); + if (inum < o->inum) + p = &(*p)->rb_left; + else if (inum > o->inum) + p = &(*p)->rb_right; + else { + kfree(orphan); + return 0; + } + } + rb_link_node(&orphan->rb, parent, p); + rb_insert_color(&orphan->rb, root); + return 0; +} + +static int dbg_find_check_orphan(struct rb_root *root, ino_t inum) +{ + struct check_orphan *o; + struct rb_node *p; + + p = root->rb_node; + while (p) { + o = rb_entry(p, struct check_orphan, rb); + if (inum < o->inum) + p = p->rb_left; + else if (inum > o->inum) + p = p->rb_right; + else + return 1; + } + return 0; +} + +static void dbg_free_check_tree(struct rb_root *root) +{ + struct check_orphan *o, *n; + + rbtree_postorder_for_each_entry_safe(o, n, root, rb) + kfree(o); +} + +static int dbg_orphan_check(struct ubifs_info *c, struct ubifs_zbranch *zbr, + void *priv) +{ + struct check_info *ci = priv; + ino_t inum; + int err; + + inum = key_inum(c, &zbr->key); + if (inum != ci->last_ino) { + /* + * Lowest node type is the inode node or xattr entry(when + * selinux/encryption is enabled), so it comes first + */ + if (key_type(c, &zbr->key) != UBIFS_INO_KEY && + key_type(c, &zbr->key) != UBIFS_XENT_KEY) + ubifs_err(c, "found orphan node ino %lu, type %d", + (unsigned long)inum, key_type(c, &zbr->key)); + ci->last_ino = inum; + ci->tot_inos += 1; + err = ubifs_tnc_read_node(c, zbr, ci->node); + if (err) { + ubifs_err(c, "node read failed, error %d", err); + return err; + } + if (ci->node->nlink == 0) + /* Must be recorded as an orphan */ + if (!dbg_find_check_orphan(&ci->root, inum) && + !dbg_find_orphan(c, inum)) { + ubifs_err(c, "missing orphan, ino %lu", + (unsigned long)inum); + ci->missing += 1; + } + } + ci->leaf_cnt += 1; + return 0; +} + +static int dbg_read_orphans(struct check_info *ci, struct ubifs_scan_leb *sleb) +{ + struct ubifs_scan_node *snod; + struct ubifs_orph_node *orph; + ino_t inum; + int i, n, err; + + list_for_each_entry(snod, &sleb->nodes, list) { + cond_resched(); + if (snod->type != UBIFS_ORPH_NODE) + continue; + orph = snod->node; + n = (le32_to_cpu(orph->ch.len) - UBIFS_ORPH_NODE_SZ) >> 3; + for (i = 0; i < n; i++) { + inum = le64_to_cpu(orph->inos[i]); + err = dbg_ins_check_orphan(&ci->root, inum); + if (err) + return err; + } + } + return 0; +} + +static int dbg_scan_orphans(struct ubifs_info *c, struct check_info *ci) +{ + int lnum, err = 0; + void *buf; + + /* Check no-orphans flag and skip this if no orphans */ + if (c->no_orphs) + return 0; + + buf = __vmalloc(c->leb_size, GFP_NOFS); + if (!buf) { + ubifs_err(c, "cannot allocate memory to check orphans"); + return 0; + } + + for (lnum = c->orph_first; lnum <= c->orph_last; lnum++) { + struct ubifs_scan_leb *sleb; + + sleb = ubifs_scan(c, lnum, 0, buf, 0); + if (IS_ERR(sleb)) { + err = PTR_ERR(sleb); + break; + } + + err = dbg_read_orphans(ci, sleb); + ubifs_scan_destroy(sleb); + if (err) + break; + } + + vfree(buf); + return err; +} + +static int dbg_check_orphans(struct ubifs_info *c) +{ + struct check_info ci; + int err; + + if (!dbg_is_chk_orph(c)) + return 0; + + ci.last_ino = 0; + ci.tot_inos = 0; + ci.missing = 0; + ci.leaf_cnt = 0; + ci.root = RB_ROOT; + ci.node = kmalloc(UBIFS_MAX_INO_NODE_SZ, GFP_NOFS); + if (!ci.node) { + ubifs_err(c, "out of memory"); + return -ENOMEM; + } + + err = dbg_scan_orphans(c, &ci); + if (err) + goto out; + + err = dbg_walk_index(c, &dbg_orphan_check, NULL, &ci); + if (err) { + ubifs_err(c, "cannot scan TNC, error %d", err); + goto out; + } + + if (ci.missing) { + ubifs_err(c, "%lu missing orphan(s)", ci.missing); + err = -EINVAL; + goto out; + } + + dbg_cmt("last inode number is %lu", ci.last_ino); + dbg_cmt("total number of inodes is %lu", ci.tot_inos); + dbg_cmt("total number of leaf nodes is %llu", ci.leaf_cnt); + +out: + dbg_free_check_tree(&ci.root); + kfree(ci.node); + return err; +} diff --git a/fs/ubifs/recovery.c b/fs/ubifs/recovery.c new file mode 100644 index 000000000..5c9dbf0a0 --- /dev/null +++ b/fs/ubifs/recovery.c @@ -0,0 +1,1586 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation + * + * Authors: Adrian Hunter + * Artem Bityutskiy (Битюцкий Артём) + */ + +/* + * This file implements functions needed to recover from unclean un-mounts. + * When UBIFS is mounted, it checks a flag on the master node to determine if + * an un-mount was completed successfully. If not, the process of mounting + * incorporates additional checking and fixing of on-flash data structures. + * UBIFS always cleans away all remnants of an unclean un-mount, so that + * errors do not accumulate. However UBIFS defers recovery if it is mounted + * read-only, and the flash is not modified in that case. + * + * The general UBIFS approach to the recovery is that it recovers from + * corruptions which could be caused by power cuts, but it refuses to recover + * from corruption caused by other reasons. And UBIFS tries to distinguish + * between these 2 reasons of corruptions and silently recover in the former + * case and loudly complain in the latter case. + * + * UBIFS writes only to erased LEBs, so it writes only to the flash space + * containing only 0xFFs. UBIFS also always writes strictly from the beginning + * of the LEB to the end. And UBIFS assumes that the underlying flash media + * writes in @c->max_write_size bytes at a time. + * + * Hence, if UBIFS finds a corrupted node at offset X, it expects only the min. + * I/O unit corresponding to offset X to contain corrupted data, all the + * following min. I/O units have to contain empty space (all 0xFFs). If this is + * not true, the corruption cannot be the result of a power cut, and UBIFS + * refuses to mount. + */ + +#include +#include +#include "ubifs.h" + +/** + * is_empty - determine whether a buffer is empty (contains all 0xff). + * @buf: buffer to clean + * @len: length of buffer + * + * This function returns %1 if the buffer is empty (contains all 0xff) otherwise + * %0 is returned. + */ +static int is_empty(void *buf, int len) +{ + uint8_t *p = buf; + int i; + + for (i = 0; i < len; i++) + if (*p++ != 0xff) + return 0; + return 1; +} + +/** + * first_non_ff - find offset of the first non-0xff byte. + * @buf: buffer to search in + * @len: length of buffer + * + * This function returns offset of the first non-0xff byte in @buf or %-1 if + * the buffer contains only 0xff bytes. + */ +static int first_non_ff(void *buf, int len) +{ + uint8_t *p = buf; + int i; + + for (i = 0; i < len; i++) + if (*p++ != 0xff) + return i; + return -1; +} + +/** + * get_master_node - get the last valid master node allowing for corruption. + * @c: UBIFS file-system description object + * @lnum: LEB number + * @pbuf: buffer containing the LEB read, is returned here + * @mst: master node, if found, is returned here + * @cor: corruption, if found, is returned here + * + * This function allocates a buffer, reads the LEB into it, and finds and + * returns the last valid master node allowing for one area of corruption. + * The corrupt area, if there is one, must be consistent with the assumption + * that it is the result of an unclean unmount while the master node was being + * written. Under those circumstances, it is valid to use the previously written + * master node. + * + * This function returns %0 on success and a negative error code on failure. + */ +static int get_master_node(const struct ubifs_info *c, int lnum, void **pbuf, + struct ubifs_mst_node **mst, void **cor) +{ + const int sz = c->mst_node_alsz; + int err, offs, len; + void *sbuf, *buf; + + sbuf = vmalloc(c->leb_size); + if (!sbuf) + return -ENOMEM; + + err = ubifs_leb_read(c, lnum, sbuf, 0, c->leb_size, 0); + if (err && err != -EBADMSG) + goto out_free; + + /* Find the first position that is definitely not a node */ + offs = 0; + buf = sbuf; + len = c->leb_size; + while (offs + UBIFS_MST_NODE_SZ <= c->leb_size) { + struct ubifs_ch *ch = buf; + + if (le32_to_cpu(ch->magic) != UBIFS_NODE_MAGIC) + break; + offs += sz; + buf += sz; + len -= sz; + } + /* See if there was a valid master node before that */ + if (offs) { + int ret; + + offs -= sz; + buf -= sz; + len += sz; + ret = ubifs_scan_a_node(c, buf, len, lnum, offs, 1); + if (ret != SCANNED_A_NODE && offs) { + /* Could have been corruption so check one place back */ + offs -= sz; + buf -= sz; + len += sz; + ret = ubifs_scan_a_node(c, buf, len, lnum, offs, 1); + if (ret != SCANNED_A_NODE) + /* + * We accept only one area of corruption because + * we are assuming that it was caused while + * trying to write a master node. + */ + goto out_err; + } + if (ret == SCANNED_A_NODE) { + struct ubifs_ch *ch = buf; + + if (ch->node_type != UBIFS_MST_NODE) + goto out_err; + dbg_rcvry("found a master node at %d:%d", lnum, offs); + *mst = buf; + offs += sz; + buf += sz; + len -= sz; + } + } + /* Check for corruption */ + if (offs < c->leb_size) { + if (!is_empty(buf, min_t(int, len, sz))) { + *cor = buf; + dbg_rcvry("found corruption at %d:%d", lnum, offs); + } + offs += sz; + buf += sz; + len -= sz; + } + /* Check remaining empty space */ + if (offs < c->leb_size) + if (!is_empty(buf, len)) + goto out_err; + *pbuf = sbuf; + return 0; + +out_err: + err = -EINVAL; +out_free: + vfree(sbuf); + *mst = NULL; + *cor = NULL; + return err; +} + +/** + * write_rcvrd_mst_node - write recovered master node. + * @c: UBIFS file-system description object + * @mst: master node + * + * This function returns %0 on success and a negative error code on failure. + */ +static int write_rcvrd_mst_node(struct ubifs_info *c, + struct ubifs_mst_node *mst) +{ + int err = 0, lnum = UBIFS_MST_LNUM, sz = c->mst_node_alsz; + __le32 save_flags; + + dbg_rcvry("recovery"); + + save_flags = mst->flags; + mst->flags |= cpu_to_le32(UBIFS_MST_RCVRY); + + err = ubifs_prepare_node_hmac(c, mst, UBIFS_MST_NODE_SZ, + offsetof(struct ubifs_mst_node, hmac), 1); + if (err) + goto out; + err = ubifs_leb_change(c, lnum, mst, sz); + if (err) + goto out; + err = ubifs_leb_change(c, lnum + 1, mst, sz); + if (err) + goto out; +out: + mst->flags = save_flags; + return err; +} + +/** + * ubifs_recover_master_node - recover the master node. + * @c: UBIFS file-system description object + * + * This function recovers the master node from corruption that may occur due to + * an unclean unmount. + * + * This function returns %0 on success and a negative error code on failure. + */ +int ubifs_recover_master_node(struct ubifs_info *c) +{ + void *buf1 = NULL, *buf2 = NULL, *cor1 = NULL, *cor2 = NULL; + struct ubifs_mst_node *mst1 = NULL, *mst2 = NULL, *mst; + const int sz = c->mst_node_alsz; + int err, offs1, offs2; + + dbg_rcvry("recovery"); + + err = get_master_node(c, UBIFS_MST_LNUM, &buf1, &mst1, &cor1); + if (err) + goto out_free; + + err = get_master_node(c, UBIFS_MST_LNUM + 1, &buf2, &mst2, &cor2); + if (err) + goto out_free; + + if (mst1) { + offs1 = (void *)mst1 - buf1; + if ((le32_to_cpu(mst1->flags) & UBIFS_MST_RCVRY) && + (offs1 == 0 && !cor1)) { + /* + * mst1 was written by recovery at offset 0 with no + * corruption. + */ + dbg_rcvry("recovery recovery"); + mst = mst1; + } else if (mst2) { + offs2 = (void *)mst2 - buf2; + if (offs1 == offs2) { + /* Same offset, so must be the same */ + if (ubifs_compare_master_node(c, mst1, mst2)) + goto out_err; + mst = mst1; + } else if (offs2 + sz == offs1) { + /* 1st LEB was written, 2nd was not */ + if (cor1) + goto out_err; + mst = mst1; + } else if (offs1 == 0 && + c->leb_size - offs2 - sz < sz) { + /* 1st LEB was unmapped and written, 2nd not */ + if (cor1) + goto out_err; + mst = mst1; + } else + goto out_err; + } else { + /* + * 2nd LEB was unmapped and about to be written, so + * there must be only one master node in the first LEB + * and no corruption. + */ + if (offs1 != 0 || cor1) + goto out_err; + mst = mst1; + } + } else { + if (!mst2) + goto out_err; + /* + * 1st LEB was unmapped and about to be written, so there must + * be no room left in 2nd LEB. + */ + offs2 = (void *)mst2 - buf2; + if (offs2 + sz + sz <= c->leb_size) + goto out_err; + mst = mst2; + } + + ubifs_msg(c, "recovered master node from LEB %d", + (mst == mst1 ? UBIFS_MST_LNUM : UBIFS_MST_LNUM + 1)); + + memcpy(c->mst_node, mst, UBIFS_MST_NODE_SZ); + + if (c->ro_mount) { + /* Read-only mode. Keep a copy for switching to rw mode */ + c->rcvrd_mst_node = kmalloc(sz, GFP_KERNEL); + if (!c->rcvrd_mst_node) { + err = -ENOMEM; + goto out_free; + } + memcpy(c->rcvrd_mst_node, c->mst_node, UBIFS_MST_NODE_SZ); + + /* + * We had to recover the master node, which means there was an + * unclean reboot. However, it is possible that the master node + * is clean at this point, i.e., %UBIFS_MST_DIRTY is not set. + * E.g., consider the following chain of events: + * + * 1. UBIFS was cleanly unmounted, so the master node is clean + * 2. UBIFS is being mounted R/W and starts changing the master + * node in the first (%UBIFS_MST_LNUM). A power cut happens, + * so this LEB ends up with some amount of garbage at the + * end. + * 3. UBIFS is being mounted R/O. We reach this place and + * recover the master node from the second LEB + * (%UBIFS_MST_LNUM + 1). But we cannot update the media + * because we are being mounted R/O. We have to defer the + * operation. + * 4. However, this master node (@c->mst_node) is marked as + * clean (since the step 1). And if we just return, the + * mount code will be confused and won't recover the master + * node when it is re-mounter R/W later. + * + * Thus, to force the recovery by marking the master node as + * dirty. + */ + c->mst_node->flags |= cpu_to_le32(UBIFS_MST_DIRTY); + } else { + /* Write the recovered master node */ + c->max_sqnum = le64_to_cpu(mst->ch.sqnum) - 1; + err = write_rcvrd_mst_node(c, c->mst_node); + if (err) + goto out_free; + } + + vfree(buf2); + vfree(buf1); + + return 0; + +out_err: + err = -EINVAL; +out_free: + ubifs_err(c, "failed to recover master node"); + if (mst1) { + ubifs_err(c, "dumping first master node"); + ubifs_dump_node(c, mst1, c->leb_size - ((void *)mst1 - buf1)); + } + if (mst2) { + ubifs_err(c, "dumping second master node"); + ubifs_dump_node(c, mst2, c->leb_size - ((void *)mst2 - buf2)); + } + vfree(buf2); + vfree(buf1); + return err; +} + +/** + * ubifs_write_rcvrd_mst_node - write the recovered master node. + * @c: UBIFS file-system description object + * + * This function writes the master node that was recovered during mounting in + * read-only mode and must now be written because we are remounting rw. + * + * This function returns %0 on success and a negative error code on failure. + */ +int ubifs_write_rcvrd_mst_node(struct ubifs_info *c) +{ + int err; + + if (!c->rcvrd_mst_node) + return 0; + c->rcvrd_mst_node->flags |= cpu_to_le32(UBIFS_MST_DIRTY); + c->mst_node->flags |= cpu_to_le32(UBIFS_MST_DIRTY); + err = write_rcvrd_mst_node(c, c->rcvrd_mst_node); + if (err) + return err; + kfree(c->rcvrd_mst_node); + c->rcvrd_mst_node = NULL; + return 0; +} + +/** + * is_last_write - determine if an offset was in the last write to a LEB. + * @c: UBIFS file-system description object + * @buf: buffer to check + * @offs: offset to check + * + * This function returns %1 if @offs was in the last write to the LEB whose data + * is in @buf, otherwise %0 is returned. The determination is made by checking + * for subsequent empty space starting from the next @c->max_write_size + * boundary. + */ +static int is_last_write(const struct ubifs_info *c, void *buf, int offs) +{ + int empty_offs, check_len; + uint8_t *p; + + /* + * Round up to the next @c->max_write_size boundary i.e. @offs is in + * the last wbuf written. After that should be empty space. + */ + empty_offs = ALIGN(offs + 1, c->max_write_size); + check_len = c->leb_size - empty_offs; + p = buf + empty_offs - offs; + return is_empty(p, check_len); +} + +/** + * clean_buf - clean the data from an LEB sitting in a buffer. + * @c: UBIFS file-system description object + * @buf: buffer to clean + * @lnum: LEB number to clean + * @offs: offset from which to clean + * @len: length of buffer + * + * This function pads up to the next min_io_size boundary (if there is one) and + * sets empty space to all 0xff. @buf, @offs and @len are updated to the next + * @c->min_io_size boundary. + */ +static void clean_buf(const struct ubifs_info *c, void **buf, int lnum, + int *offs, int *len) +{ + int empty_offs, pad_len; + + dbg_rcvry("cleaning corruption at %d:%d", lnum, *offs); + + ubifs_assert(c, !(*offs & 7)); + empty_offs = ALIGN(*offs, c->min_io_size); + pad_len = empty_offs - *offs; + ubifs_pad(c, *buf, pad_len); + *offs += pad_len; + *buf += pad_len; + *len -= pad_len; + memset(*buf, 0xff, c->leb_size - empty_offs); +} + +/** + * no_more_nodes - determine if there are no more nodes in a buffer. + * @c: UBIFS file-system description object + * @buf: buffer to check + * @len: length of buffer + * @lnum: LEB number of the LEB from which @buf was read + * @offs: offset from which @buf was read + * + * This function ensures that the corrupted node at @offs is the last thing + * written to a LEB. This function returns %1 if more data is not found and + * %0 if more data is found. + */ +static int no_more_nodes(const struct ubifs_info *c, void *buf, int len, + int lnum, int offs) +{ + struct ubifs_ch *ch = buf; + int skip, dlen = le32_to_cpu(ch->len); + + /* Check for empty space after the corrupt node's common header */ + skip = ALIGN(offs + UBIFS_CH_SZ, c->max_write_size) - offs; + if (is_empty(buf + skip, len - skip)) + return 1; + /* + * The area after the common header size is not empty, so the common + * header must be intact. Check it. + */ + if (ubifs_check_node(c, buf, len, lnum, offs, 1, 0) != -EUCLEAN) { + dbg_rcvry("unexpected bad common header at %d:%d", lnum, offs); + return 0; + } + /* Now we know the corrupt node's length we can skip over it */ + skip = ALIGN(offs + dlen, c->max_write_size) - offs; + /* After which there should be empty space */ + if (is_empty(buf + skip, len - skip)) + return 1; + dbg_rcvry("unexpected data at %d:%d", lnum, offs + skip); + return 0; +} + +/** + * fix_unclean_leb - fix an unclean LEB. + * @c: UBIFS file-system description object + * @sleb: scanned LEB information + * @start: offset where scan started + */ +static int fix_unclean_leb(struct ubifs_info *c, struct ubifs_scan_leb *sleb, + int start) +{ + int lnum = sleb->lnum, endpt = start; + + /* Get the end offset of the last node we are keeping */ + if (!list_empty(&sleb->nodes)) { + struct ubifs_scan_node *snod; + + snod = list_entry(sleb->nodes.prev, + struct ubifs_scan_node, list); + endpt = snod->offs + snod->len; + } + + if (c->ro_mount && !c->remounting_rw) { + /* Add to recovery list */ + struct ubifs_unclean_leb *ucleb; + + dbg_rcvry("need to fix LEB %d start %d endpt %d", + lnum, start, sleb->endpt); + ucleb = kzalloc_obj(struct ubifs_unclean_leb, GFP_NOFS); + if (!ucleb) + return -ENOMEM; + ucleb->lnum = lnum; + ucleb->endpt = endpt; + list_add_tail(&ucleb->list, &c->unclean_leb_list); + } else { + /* Write the fixed LEB back to flash */ + int err; + + dbg_rcvry("fixing LEB %d start %d endpt %d", + lnum, start, sleb->endpt); + if (endpt == 0) { + err = ubifs_leb_unmap(c, lnum); + if (err) + return err; + } else { + int len = ALIGN(endpt, c->min_io_size); + + if (start) { + err = ubifs_leb_read(c, lnum, sleb->buf, 0, + start, 1); + if (err) + return err; + } + /* Pad to min_io_size */ + if (len > endpt) { + int pad_len = len - ALIGN(endpt, 8); + + if (pad_len > 0) { + void *buf = sleb->buf + len - pad_len; + + ubifs_pad(c, buf, pad_len); + } + } + err = ubifs_leb_change(c, lnum, sleb->buf, len); + if (err) + return err; + } + } + return 0; +} + +/** + * drop_last_group - drop the last group of nodes. + * @sleb: scanned LEB information + * @offs: offset of dropped nodes is returned here + * + * This is a helper function for 'ubifs_recover_leb()' which drops the last + * group of nodes of the scanned LEB. + */ +static void drop_last_group(struct ubifs_scan_leb *sleb, int *offs) +{ + while (!list_empty(&sleb->nodes)) { + struct ubifs_scan_node *snod; + struct ubifs_ch *ch; + + snod = list_entry(sleb->nodes.prev, struct ubifs_scan_node, + list); + ch = snod->node; + if (ch->group_type != UBIFS_IN_NODE_GROUP) + break; + + dbg_rcvry("dropping grouped node at %d:%d", + sleb->lnum, snod->offs); + *offs = snod->offs; + list_del(&snod->list); + kfree(snod); + sleb->nodes_cnt -= 1; + } +} + +/** + * drop_last_node - drop the last node. + * @sleb: scanned LEB information + * @offs: offset of dropped nodes is returned here + * + * This is a helper function for 'ubifs_recover_leb()' which drops the last + * node of the scanned LEB. + */ +static void drop_last_node(struct ubifs_scan_leb *sleb, int *offs) +{ + struct ubifs_scan_node *snod; + + if (!list_empty(&sleb->nodes)) { + snod = list_entry(sleb->nodes.prev, struct ubifs_scan_node, + list); + + dbg_rcvry("dropping last node at %d:%d", + sleb->lnum, snod->offs); + *offs = snod->offs; + list_del(&snod->list); + kfree(snod); + sleb->nodes_cnt -= 1; + } +} + +/** + * ubifs_recover_leb - scan and recover a LEB. + * @c: UBIFS file-system description object + * @lnum: LEB number + * @offs: offset + * @sbuf: LEB-sized buffer to use + * @jhead: journal head number this LEB belongs to (%-1 if the LEB does not + * belong to any journal head) + * + * This function does a scan of a LEB, but caters for errors that might have + * been caused by the unclean unmount from which we are attempting to recover. + * Returns the scanned information on success and a negative error code on + * failure. + */ +struct ubifs_scan_leb *ubifs_recover_leb(struct ubifs_info *c, int lnum, + int offs, void *sbuf, int jhead) +{ + int ret = 0, err, len = c->leb_size - offs, start = offs, min_io_unit; + int grouped = jhead == -1 ? 0 : c->jheads[jhead].grouped; + struct ubifs_scan_leb *sleb; + void *buf = sbuf + offs; + + dbg_rcvry("%d:%d, jhead %d, grouped %d", lnum, offs, jhead, grouped); + + sleb = ubifs_start_scan(c, lnum, offs, sbuf); + if (IS_ERR(sleb)) + return sleb; + + ubifs_assert(c, len >= 8); + while (len >= 8) { + dbg_scan("look at LEB %d:%d (%d bytes left)", + lnum, offs, len); + + cond_resched(); + + /* + * Scan quietly until there is an error from which we cannot + * recover + */ + ret = ubifs_scan_a_node(c, buf, len, lnum, offs, 1); + if (ret == SCANNED_A_NODE) { + /* A valid node, and not a padding node */ + struct ubifs_ch *ch = buf; + int node_len; + + err = ubifs_add_snod(c, sleb, buf, offs); + if (err) + goto error; + node_len = ALIGN(le32_to_cpu(ch->len), 8); + offs += node_len; + buf += node_len; + len -= node_len; + } else if (ret > 0) { + /* Padding bytes or a valid padding node */ + offs += ret; + buf += ret; + len -= ret; + } else if (ret == SCANNED_EMPTY_SPACE || + ret == SCANNED_GARBAGE || + ret == SCANNED_A_BAD_PAD_NODE || + ret == SCANNED_A_CORRUPT_NODE) { + dbg_rcvry("found corruption (%d) at %d:%d", + ret, lnum, offs); + break; + } else { + ubifs_err(c, "unexpected return value %d", ret); + err = -EINVAL; + goto error; + } + } + + if (ret == SCANNED_GARBAGE || ret == SCANNED_A_BAD_PAD_NODE) { + if (!is_last_write(c, buf, offs)) + goto corrupted_rescan; + } else if (ret == SCANNED_A_CORRUPT_NODE) { + if (!no_more_nodes(c, buf, len, lnum, offs)) + goto corrupted_rescan; + } else if (!is_empty(buf, len)) { + if (!is_last_write(c, buf, offs)) { + int corruption = first_non_ff(buf, len); + + /* + * See header comment for this file for more + * explanations about the reasons we have this check. + */ + ubifs_err(c, "corrupt empty space LEB %d:%d, corruption starts at %d", + lnum, offs, corruption); + /* Make sure we dump interesting non-0xFF data */ + offs += corruption; + buf += corruption; + goto corrupted; + } + } + + min_io_unit = round_down(offs, c->min_io_size); + if (grouped) + /* + * If nodes are grouped, always drop the incomplete group at + * the end. + */ + drop_last_group(sleb, &offs); + + if (jhead == GCHD) { + /* + * If this LEB belongs to the GC head then while we are in the + * middle of the same min. I/O unit keep dropping nodes. So + * basically, what we want is to make sure that the last min. + * I/O unit where we saw the corruption is dropped completely + * with all the uncorrupted nodes which may possibly sit there. + * + * In other words, let's name the min. I/O unit where the + * corruption starts B, and the previous min. I/O unit A. The + * below code tries to deal with a situation when half of B + * contains valid nodes or the end of a valid node, and the + * second half of B contains corrupted data or garbage. This + * means that UBIFS had been writing to B just before the power + * cut happened. I do not know how realistic is this scenario + * that half of the min. I/O unit had been written successfully + * and the other half not, but this is possible in our 'failure + * mode emulation' infrastructure at least. + * + * So what is the problem, why we need to drop those nodes? Why + * can't we just clean-up the second half of B by putting a + * padding node there? We can, and this works fine with one + * exception which was reproduced with power cut emulation + * testing and happens extremely rarely. + * + * Imagine the file-system is full, we run GC which starts + * moving valid nodes from LEB X to LEB Y (obviously, LEB Y is + * the current GC head LEB). The @c->gc_lnum is -1, which means + * that GC will retain LEB X and will try to continue. Imagine + * that LEB X is currently the dirtiest LEB, and the amount of + * used space in LEB Y is exactly the same as amount of free + * space in LEB X. + * + * And a power cut happens when nodes are moved from LEB X to + * LEB Y. We are here trying to recover LEB Y which is the GC + * head LEB. We find the min. I/O unit B as described above. + * Then we clean-up LEB Y by padding min. I/O unit. And later + * 'ubifs_rcvry_gc_commit()' function fails, because it cannot + * find a dirty LEB which could be GC'd into LEB Y! Even LEB X + * does not match because the amount of valid nodes there does + * not fit the free space in LEB Y any more! And this is + * because of the padding node which we added to LEB Y. The + * user-visible effect of this which I once observed and + * analysed is that we cannot mount the file-system with + * -ENOSPC error. + * + * So obviously, to make sure that situation does not happen we + * should free min. I/O unit B in LEB Y completely and the last + * used min. I/O unit in LEB Y should be A. This is basically + * what the below code tries to do. + */ + while (offs > min_io_unit) + drop_last_node(sleb, &offs); + } + + buf = sbuf + offs; + len = c->leb_size - offs; + + clean_buf(c, &buf, lnum, &offs, &len); + ubifs_end_scan(c, sleb, lnum, offs); + + err = fix_unclean_leb(c, sleb, start); + if (err) + goto error; + + return sleb; + +corrupted_rescan: + /* Re-scan the corrupted data with verbose messages */ + ubifs_err(c, "corruption %d", ret); + ubifs_scan_a_node(c, buf, len, lnum, offs, 0); +corrupted: + ubifs_scanned_corruption(c, lnum, offs, buf); + err = -EUCLEAN; +error: + ubifs_err(c, "LEB %d scanning failed", lnum); + ubifs_scan_destroy(sleb); + return ERR_PTR(err); +} + +/** + * get_cs_sqnum - get commit start sequence number. + * @c: UBIFS file-system description object + * @lnum: LEB number of commit start node + * @offs: offset of commit start node + * @cs_sqnum: commit start sequence number is returned here + * + * This function returns %0 on success and a negative error code on failure. + */ +static int get_cs_sqnum(struct ubifs_info *c, int lnum, int offs, + unsigned long long *cs_sqnum) +{ + struct ubifs_cs_node *cs_node = NULL; + int err, ret; + + dbg_rcvry("at %d:%d", lnum, offs); + cs_node = kmalloc(UBIFS_CS_NODE_SZ, GFP_KERNEL); + if (!cs_node) + return -ENOMEM; + if (c->leb_size - offs < UBIFS_CS_NODE_SZ) + goto out_err; + err = ubifs_leb_read(c, lnum, (void *)cs_node, offs, + UBIFS_CS_NODE_SZ, 0); + if (err && err != -EBADMSG) + goto out_free; + ret = ubifs_scan_a_node(c, cs_node, UBIFS_CS_NODE_SZ, lnum, offs, 0); + if (ret != SCANNED_A_NODE) { + ubifs_err(c, "Not a valid node"); + goto out_err; + } + if (cs_node->ch.node_type != UBIFS_CS_NODE) { + ubifs_err(c, "Not a CS node, type is %d", cs_node->ch.node_type); + goto out_err; + } + if (le64_to_cpu(cs_node->cmt_no) != c->cmt_no) { + ubifs_err(c, "CS node cmt_no %llu != current cmt_no %llu", + (unsigned long long)le64_to_cpu(cs_node->cmt_no), + c->cmt_no); + goto out_err; + } + *cs_sqnum = le64_to_cpu(cs_node->ch.sqnum); + dbg_rcvry("commit start sqnum %llu", *cs_sqnum); + kfree(cs_node); + return 0; + +out_err: + err = -EINVAL; +out_free: + ubifs_err(c, "failed to get CS sqnum"); + kfree(cs_node); + return err; +} + +/** + * ubifs_recover_log_leb - scan and recover a log LEB. + * @c: UBIFS file-system description object + * @lnum: LEB number + * @offs: offset + * @sbuf: LEB-sized buffer to use + * + * This function does a scan of a LEB, but caters for errors that might have + * been caused by unclean reboots from which we are attempting to recover + * (assume that only the last log LEB can be corrupted by an unclean reboot). + * + * This function returns %0 on success and a negative error code on failure. + */ +struct ubifs_scan_leb *ubifs_recover_log_leb(struct ubifs_info *c, int lnum, + int offs, void *sbuf) +{ + struct ubifs_scan_leb *sleb; + int next_lnum; + + dbg_rcvry("LEB %d", lnum); + next_lnum = lnum + 1; + if (next_lnum >= UBIFS_LOG_LNUM + c->log_lebs) + next_lnum = UBIFS_LOG_LNUM; + if (next_lnum != c->ltail_lnum) { + /* + * We can only recover at the end of the log, so check that the + * next log LEB is empty or out of date. + */ + sleb = ubifs_scan(c, next_lnum, 0, sbuf, 0); + if (IS_ERR(sleb)) + return sleb; + if (sleb->nodes_cnt) { + struct ubifs_scan_node *snod; + unsigned long long cs_sqnum = c->cs_sqnum; + + snod = list_entry(sleb->nodes.next, + struct ubifs_scan_node, list); + if (cs_sqnum == 0) { + int err; + + err = get_cs_sqnum(c, lnum, offs, &cs_sqnum); + if (err) { + ubifs_scan_destroy(sleb); + return ERR_PTR(err); + } + } + if (snod->sqnum > cs_sqnum) { + ubifs_err(c, "unrecoverable log corruption in LEB %d", + lnum); + ubifs_scan_destroy(sleb); + return ERR_PTR(-EUCLEAN); + } + } + ubifs_scan_destroy(sleb); + } + return ubifs_recover_leb(c, lnum, offs, sbuf, -1); +} + +/** + * recover_head - recover a head. + * @c: UBIFS file-system description object + * @lnum: LEB number of head to recover + * @offs: offset of head to recover + * @sbuf: LEB-sized buffer to use + * + * This function ensures that there is no data on the flash at a head location. + * + * This function returns %0 on success and a negative error code on failure. + */ +static int recover_head(struct ubifs_info *c, int lnum, int offs, void *sbuf) +{ + int len = c->max_write_size, err; + + if (offs + len > c->leb_size) + len = c->leb_size - offs; + + if (!len) + return 0; + + /* Read at the head location and check it is empty flash */ + err = ubifs_leb_read(c, lnum, sbuf, offs, len, 1); + if (err || !is_empty(sbuf, len)) { + dbg_rcvry("cleaning head at %d:%d", lnum, offs); + if (offs == 0) + return ubifs_leb_unmap(c, lnum); + err = ubifs_leb_read(c, lnum, sbuf, 0, offs, 1); + if (err) + return err; + return ubifs_leb_change(c, lnum, sbuf, offs); + } + + return 0; +} + +/** + * ubifs_recover_inl_heads - recover index and LPT heads. + * @c: UBIFS file-system description object + * @sbuf: LEB-sized buffer to use + * + * This function ensures that there is no data on the flash at the index and + * LPT head locations. + * + * This deals with the recovery of a half-completed journal commit. UBIFS is + * careful never to overwrite the last version of the index or the LPT. Because + * the index and LPT are wandering trees, data from a half-completed commit will + * not be referenced anywhere in UBIFS. The data will be either in LEBs that are + * assumed to be empty and will be unmapped anyway before use, or in the index + * and LPT heads. + * + * This function returns %0 on success and a negative error code on failure. + */ +int ubifs_recover_inl_heads(struct ubifs_info *c, void *sbuf) +{ + int err; + + ubifs_assert(c, !c->ro_mount || c->remounting_rw); + + dbg_rcvry("checking index head at %d:%d", c->ihead_lnum, c->ihead_offs); + err = recover_head(c, c->ihead_lnum, c->ihead_offs, sbuf); + if (err) + return err; + + dbg_rcvry("checking LPT head at %d:%d", c->nhead_lnum, c->nhead_offs); + + return recover_head(c, c->nhead_lnum, c->nhead_offs, sbuf); +} + +/** + * clean_an_unclean_leb - read and write a LEB to remove corruption. + * @c: UBIFS file-system description object + * @ucleb: unclean LEB information + * @sbuf: LEB-sized buffer to use + * + * This function reads a LEB up to a point pre-determined by the mount recovery, + * checks the nodes, and writes the result back to the flash, thereby cleaning + * off any following corruption, or non-fatal ECC errors. + * + * This function returns %0 on success and a negative error code on failure. + */ +static int clean_an_unclean_leb(struct ubifs_info *c, + struct ubifs_unclean_leb *ucleb, void *sbuf) +{ + int err, lnum = ucleb->lnum, offs = 0, len = ucleb->endpt, quiet = 1; + void *buf = sbuf; + + dbg_rcvry("LEB %d len %d", lnum, len); + + if (len == 0) { + /* Nothing to read, just unmap it */ + return ubifs_leb_unmap(c, lnum); + } + + err = ubifs_leb_read(c, lnum, buf, offs, len, 0); + if (err && err != -EBADMSG) + return err; + + while (len >= 8) { + int ret; + + cond_resched(); + + /* Scan quietly until there is an error */ + ret = ubifs_scan_a_node(c, buf, len, lnum, offs, quiet); + + if (ret == SCANNED_A_NODE) { + /* A valid node, and not a padding node */ + struct ubifs_ch *ch = buf; + int node_len; + + node_len = ALIGN(le32_to_cpu(ch->len), 8); + offs += node_len; + buf += node_len; + len -= node_len; + continue; + } + + if (ret > 0) { + /* Padding bytes or a valid padding node */ + offs += ret; + buf += ret; + len -= ret; + continue; + } + + if (ret == SCANNED_EMPTY_SPACE) { + ubifs_err(c, "unexpected empty space at %d:%d", + lnum, offs); + return -EUCLEAN; + } + + if (quiet) { + /* Redo the last scan but noisily */ + quiet = 0; + continue; + } + + ubifs_scanned_corruption(c, lnum, offs, buf); + return -EUCLEAN; + } + + /* Pad to min_io_size */ + len = ALIGN(ucleb->endpt, c->min_io_size); + if (len > ucleb->endpt) { + int pad_len = len - ALIGN(ucleb->endpt, 8); + + if (pad_len > 0) { + buf = c->sbuf + len - pad_len; + ubifs_pad(c, buf, pad_len); + } + } + + /* Write back the LEB atomically */ + err = ubifs_leb_change(c, lnum, sbuf, len); + if (err) + return err; + + dbg_rcvry("cleaned LEB %d", lnum); + + return 0; +} + +/** + * ubifs_clean_lebs - clean LEBs recovered during read-only mount. + * @c: UBIFS file-system description object + * @sbuf: LEB-sized buffer to use + * + * This function cleans a LEB identified during recovery that needs to be + * written but was not because UBIFS was mounted read-only. This happens when + * remounting to read-write mode. + * + * This function returns %0 on success and a negative error code on failure. + */ +int ubifs_clean_lebs(struct ubifs_info *c, void *sbuf) +{ + dbg_rcvry("recovery"); + while (!list_empty(&c->unclean_leb_list)) { + struct ubifs_unclean_leb *ucleb; + int err; + + ucleb = list_entry(c->unclean_leb_list.next, + struct ubifs_unclean_leb, list); + err = clean_an_unclean_leb(c, ucleb, sbuf); + if (err) + return err; + list_del(&ucleb->list); + kfree(ucleb); + } + return 0; +} + +/** + * grab_empty_leb - grab an empty LEB to use as GC LEB and run commit. + * @c: UBIFS file-system description object + * + * This is a helper function for 'ubifs_rcvry_gc_commit()' which grabs an empty + * LEB to be used as GC LEB (@c->gc_lnum), and then runs the commit. Returns + * zero in case of success and a negative error code in case of failure. + */ +static int grab_empty_leb(struct ubifs_info *c) +{ + int lnum, err; + + /* + * Note, it is very important to first search for an empty LEB and then + * run the commit, not vice-versa. The reason is that there might be + * only one empty LEB at the moment, the one which has been the + * @c->gc_lnum just before the power cut happened. During the regular + * UBIFS operation (not now) @c->gc_lnum is marked as "taken", so no + * one but GC can grab it. But at this moment this single empty LEB is + * not marked as taken, so if we run commit - what happens? Right, the + * commit will grab it and write the index there. Remember that the + * index always expands as long as there is free space, and it only + * starts consolidating when we run out of space. + * + * IOW, if we run commit now, we might not be able to find a free LEB + * after this. + */ + lnum = ubifs_find_free_leb_for_idx(c); + if (lnum < 0) { + ubifs_err(c, "could not find an empty LEB"); + ubifs_dump_lprops(c); + ubifs_dump_budg(c, &c->bi); + return lnum; + } + + /* Reset the index flag */ + err = ubifs_change_one_lp(c, lnum, LPROPS_NC, LPROPS_NC, 0, + LPROPS_INDEX, 0); + if (err) + return err; + + c->gc_lnum = lnum; + dbg_rcvry("found empty LEB %d, run commit", lnum); + + return ubifs_run_commit(c); +} + +/** + * ubifs_rcvry_gc_commit - recover the GC LEB number and run the commit. + * @c: UBIFS file-system description object + * + * Out-of-place garbage collection requires always one empty LEB with which to + * start garbage collection. The LEB number is recorded in c->gc_lnum and is + * written to the master node on unmounting. In the case of an unclean unmount + * the value of gc_lnum recorded in the master node is out of date and cannot + * be used. Instead, recovery must allocate an empty LEB for this purpose. + * However, there may not be enough empty space, in which case it must be + * possible to GC the dirtiest LEB into the GC head LEB. + * + * This function also runs the commit which causes the TNC updates from + * size-recovery and orphans to be written to the flash. That is important to + * ensure correct replay order for subsequent mounts. + * + * This function returns %0 on success and a negative error code on failure. + */ +int ubifs_rcvry_gc_commit(struct ubifs_info *c) +{ + struct ubifs_wbuf *wbuf = &c->jheads[GCHD].wbuf; + struct ubifs_lprops lp; + int err; + + dbg_rcvry("GC head LEB %d, offs %d", wbuf->lnum, wbuf->offs); + + c->gc_lnum = -1; + if (wbuf->lnum == -1 || wbuf->offs == c->leb_size) + return grab_empty_leb(c); + + err = ubifs_find_dirty_leb(c, &lp, wbuf->offs, 2); + if (err) { + if (err != -ENOSPC) + return err; + + dbg_rcvry("could not find a dirty LEB"); + return grab_empty_leb(c); + } + + ubifs_assert(c, !(lp.flags & LPROPS_INDEX)); + ubifs_assert(c, lp.free + lp.dirty >= wbuf->offs); + + /* + * We run the commit before garbage collection otherwise subsequent + * mounts will see the GC and orphan deletion in a different order. + */ + dbg_rcvry("committing"); + err = ubifs_run_commit(c); + if (err) + return err; + + dbg_rcvry("GC'ing LEB %d", lp.lnum); + mutex_lock_nested(&wbuf->io_mutex, wbuf->jhead); + err = ubifs_garbage_collect_leb(c, &lp); + if (err >= 0) { + int err2 = ubifs_wbuf_sync_nolock(wbuf); + + if (err2) + err = err2; + } + mutex_unlock(&wbuf->io_mutex); + if (err < 0) { + ubifs_err(c, "GC failed, error %d", err); + if (err == -EAGAIN) + err = -EINVAL; + return err; + } + + ubifs_assert(c, err == LEB_RETAINED); + if (err != LEB_RETAINED) + return -EINVAL; + + err = ubifs_leb_unmap(c, c->gc_lnum); + if (err) + return err; + + dbg_rcvry("allocated LEB %d for GC", lp.lnum); + return 0; +} + +/** + * struct size_entry - inode size information for recovery. + * @rb: link in the RB-tree of sizes + * @inum: inode number + * @i_size: size on inode + * @d_size: maximum size based on data nodes + * @exists: indicates whether the inode exists + * @inode: inode if pinned in memory awaiting rw mode to fix it + */ +struct size_entry { + struct rb_node rb; + ino_t inum; + loff_t i_size; + loff_t d_size; + int exists; + struct inode *inode; +}; + +/** + * add_ino - add an entry to the size tree. + * @c: UBIFS file-system description object + * @inum: inode number + * @i_size: size on inode + * @d_size: maximum size based on data nodes + * @exists: indicates whether the inode exists + */ +static int add_ino(struct ubifs_info *c, ino_t inum, loff_t i_size, + loff_t d_size, int exists) +{ + struct rb_node **p = &c->size_tree.rb_node, *parent = NULL; + struct size_entry *e; + + while (*p) { + parent = *p; + e = rb_entry(parent, struct size_entry, rb); + if (inum < e->inum) + p = &(*p)->rb_left; + else + p = &(*p)->rb_right; + } + + e = kzalloc_obj(struct size_entry); + if (!e) + return -ENOMEM; + + e->inum = inum; + e->i_size = i_size; + e->d_size = d_size; + e->exists = exists; + + rb_link_node(&e->rb, parent, p); + rb_insert_color(&e->rb, &c->size_tree); + + return 0; +} + +/** + * find_ino - find an entry on the size tree. + * @c: UBIFS file-system description object + * @inum: inode number + */ +static struct size_entry *find_ino(struct ubifs_info *c, ino_t inum) +{ + struct rb_node *p = c->size_tree.rb_node; + struct size_entry *e; + + while (p) { + e = rb_entry(p, struct size_entry, rb); + if (inum < e->inum) + p = p->rb_left; + else if (inum > e->inum) + p = p->rb_right; + else + return e; + } + return NULL; +} + +/** + * remove_ino - remove an entry from the size tree. + * @c: UBIFS file-system description object + * @inum: inode number + */ +static void remove_ino(struct ubifs_info *c, ino_t inum) +{ + struct size_entry *e = find_ino(c, inum); + + if (!e) + return; + rb_erase(&e->rb, &c->size_tree); + kfree(e); +} + +/** + * ubifs_destroy_size_tree - free resources related to the size tree. + * @c: UBIFS file-system description object + */ +void ubifs_destroy_size_tree(struct ubifs_info *c) +{ + struct size_entry *e, *n; + + rbtree_postorder_for_each_entry_safe(e, n, &c->size_tree, rb) { + iput(e->inode); + kfree(e); + } + + c->size_tree = RB_ROOT; +} + +/** + * ubifs_recover_size_accum - accumulate inode sizes for recovery. + * @c: UBIFS file-system description object + * @key: node key + * @deletion: node is for a deletion + * @new_size: inode size + * + * This function has two purposes: + * 1) to ensure there are no data nodes that fall outside the inode size + * 2) to ensure there are no data nodes for inodes that do not exist + * To accomplish those purposes, a rb-tree is constructed containing an entry + * for each inode number in the journal that has not been deleted, and recording + * the size from the inode node, the maximum size of any data node (also altered + * by truncations) and a flag indicating a inode number for which no inode node + * was present in the journal. + * + * Note that there is still the possibility that there are data nodes that have + * been committed that are beyond the inode size, however the only way to find + * them would be to scan the entire index. Alternatively, some provision could + * be made to record the size of inodes at the start of commit, which would seem + * very cumbersome for a scenario that is quite unlikely and the only negative + * consequence of which is wasted space. + * + * This functions returns %0 on success and a negative error code on failure. + */ +int ubifs_recover_size_accum(struct ubifs_info *c, union ubifs_key *key, + int deletion, loff_t new_size) +{ + ino_t inum = key_inum(c, key); + struct size_entry *e; + int err; + + switch (key_type(c, key)) { + case UBIFS_INO_KEY: + if (deletion) + remove_ino(c, inum); + else { + e = find_ino(c, inum); + if (e) { + e->i_size = new_size; + e->exists = 1; + } else { + err = add_ino(c, inum, new_size, 0, 1); + if (err) + return err; + } + } + break; + case UBIFS_DATA_KEY: + e = find_ino(c, inum); + if (e) { + if (new_size > e->d_size) + e->d_size = new_size; + } else { + err = add_ino(c, inum, 0, new_size, 0); + if (err) + return err; + } + break; + case UBIFS_TRUN_KEY: + e = find_ino(c, inum); + if (e) + e->d_size = new_size; + break; + } + return 0; +} + +/** + * fix_size_in_place - fix inode size in place on flash. + * @c: UBIFS file-system description object + * @e: inode size information for recovery + */ +static int fix_size_in_place(struct ubifs_info *c, struct size_entry *e) +{ + struct ubifs_ino_node *ino = c->sbuf; + unsigned char *p; + union ubifs_key key; + int err, lnum, offs, len; + loff_t i_size; + + /* Locate the inode node LEB number and offset */ + ino_key_init(c, &key, e->inum); + err = ubifs_tnc_locate(c, &key, ino, &lnum, &offs); + if (err) + goto out; + /* + * If the size recorded on the inode node is greater than the size that + * was calculated from nodes in the journal then don't change the inode. + */ + i_size = le64_to_cpu(ino->size); + if (i_size >= e->d_size) + return 0; + /* Read the LEB */ + err = ubifs_leb_read(c, lnum, c->sbuf, 0, c->leb_size, 1); + if (err) + goto out; + /* Change the size field and recalculate the CRC */ + ino = c->sbuf + offs; + ino->size = cpu_to_le64(e->d_size); + len = le32_to_cpu(ino->ch.len); + ubifs_crc_node((void *)ino, len); + /* Work out where data in the LEB ends and free space begins */ + p = c->sbuf; + len = c->leb_size - 1; + while (p[len] == 0xff) + len -= 1; + len = ALIGN(len + 1, c->min_io_size); + /* Atomically write the fixed LEB back again */ + err = ubifs_leb_change(c, lnum, c->sbuf, len); + if (err) + goto out; + dbg_rcvry("inode %lu at %d:%d size %lld -> %lld", + (unsigned long)e->inum, lnum, offs, i_size, e->d_size); + return 0; + +out: + ubifs_warn(c, "inode %lu failed to fix size %lld -> %lld error %d", + (unsigned long)e->inum, e->i_size, e->d_size, err); + return err; +} + +/** + * inode_fix_size - fix inode size + * @c: UBIFS file-system description object + * @e: inode size information for recovery + */ +static int inode_fix_size(struct ubifs_info *c, struct size_entry *e) +{ + struct inode *inode; + struct ubifs_inode *ui; + int err; + + if (c->ro_mount) + ubifs_assert(c, !e->inode); + + if (e->inode) { + /* Remounting rw, pick up inode we stored earlier */ + inode = e->inode; + } else { + inode = ubifs_iget(c->vfs_sb, e->inum); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + if (inode->i_size >= e->d_size) { + /* + * The original inode in the index already has a size + * big enough, nothing to do + */ + iput(inode); + return 0; + } + + dbg_rcvry("ino %lu size %lld -> %lld", + (unsigned long)e->inum, + inode->i_size, e->d_size); + + ui = ubifs_inode(inode); + + inode->i_size = e->d_size; + ui->ui_size = e->d_size; + ui->synced_i_size = e->d_size; + + e->inode = inode; + } + + /* + * In readonly mode just keep the inode pinned in memory until we go + * readwrite. In readwrite mode write the inode to the journal with the + * fixed size. + */ + if (c->ro_mount) + return 0; + + err = ubifs_jnl_write_inode(c, inode); + + iput(inode); + + if (err) + return err; + + rb_erase(&e->rb, &c->size_tree); + kfree(e); + + return 0; +} + +/** + * ubifs_recover_size - recover inode size. + * @c: UBIFS file-system description object + * @in_place: If true, do a in-place size fixup + * + * This function attempts to fix inode size discrepancies identified by the + * 'ubifs_recover_size_accum()' function. + * + * This functions returns %0 on success and a negative error code on failure. + */ +int ubifs_recover_size(struct ubifs_info *c, bool in_place) +{ + struct rb_node *this = rb_first(&c->size_tree); + + while (this) { + struct size_entry *e; + int err; + + e = rb_entry(this, struct size_entry, rb); + + this = rb_next(this); + + if (!e->exists) { + union ubifs_key key; + + ino_key_init(c, &key, e->inum); + err = ubifs_tnc_lookup(c, &key, c->sbuf); + if (err && err != -ENOENT) + return err; + if (err == -ENOENT) { + /* Remove data nodes that have no inode */ + dbg_rcvry("removing ino %lu", + (unsigned long)e->inum); + err = ubifs_tnc_remove_ino(c, e->inum); + if (err) + return err; + } else { + struct ubifs_ino_node *ino = c->sbuf; + + e->exists = 1; + e->i_size = le64_to_cpu(ino->size); + } + } + + if (e->exists && e->i_size < e->d_size) { + ubifs_assert(c, !(c->ro_mount && in_place)); + + /* + * We found data that is outside the found inode size, + * fixup the inode size + */ + + if (in_place) { + err = fix_size_in_place(c, e); + if (err) + return err; + iput(e->inode); + } else { + err = inode_fix_size(c, e); + if (err) + return err; + continue; + } + } + + rb_erase(&e->rb, &c->size_tree); + kfree(e); + } + + return 0; +} diff --git a/fs/ubifs/replay.c b/fs/ubifs/replay.c new file mode 100644 index 000000000..263045e05 --- /dev/null +++ b/fs/ubifs/replay.c @@ -0,0 +1,1252 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * + * Authors: Adrian Hunter + * Artem Bityutskiy (Битюцкий Артём) + */ + +/* + * This file contains journal replay code. It runs when the file-system is being + * mounted and requires no locking. + * + * The larger is the journal, the longer it takes to scan it, so the longer it + * takes to mount UBIFS. This is why the journal has limited size which may be + * changed depending on the system requirements. But a larger journal gives + * faster I/O speed because it writes the index less frequently. So this is a + * trade-off. Also, the journal is indexed by the in-memory index (TNC), so the + * larger is the journal, the more memory its index may consume. + */ + +#include "ubifs.h" +#include +#include + +/** + * struct replay_entry - replay list entry. + * @lnum: logical eraseblock number of the node + * @offs: node offset + * @len: node length + * @hash: node hash + * @deletion: non-zero if this entry corresponds to a node deletion + * @sqnum: node sequence number + * @list: links the replay list + * @key: node key + * @nm: directory entry name + * @old_size: truncation old size + * @new_size: truncation new size + * + * The replay process first scans all buds and builds the replay list, then + * sorts the replay list in nodes sequence number order, and then inserts all + * the replay entries to the TNC. + */ +struct replay_entry { + int lnum; + int offs; + int len; + u8 hash[UBIFS_HASH_ARR_SZ]; + unsigned int deletion:1; + unsigned long long sqnum; + struct list_head list; + union ubifs_key key; + union { + struct fscrypt_name nm; + struct { + loff_t old_size; + loff_t new_size; + }; + }; +}; + +/** + * struct bud_entry - entry in the list of buds to replay. + * @list: next bud in the list + * @bud: bud description object + * @sqnum: reference node sequence number + * @free: free bytes in the bud + * @dirty: dirty bytes in the bud + */ +struct bud_entry { + struct list_head list; + struct ubifs_bud *bud; + unsigned long long sqnum; + int free; + int dirty; +}; + +/** + * set_bud_lprops - set free and dirty space used by a bud. + * @c: UBIFS file-system description object + * @b: bud entry which describes the bud + * + * This function makes sure the LEB properties of bud @b are set correctly + * after the replay. Returns zero in case of success and a negative error code + * in case of failure. + */ +static int set_bud_lprops(struct ubifs_info *c, struct bud_entry *b) +{ + const struct ubifs_lprops *lp; + int err = 0, dirty; + + ubifs_get_lprops(c); + + lp = ubifs_lpt_lookup_dirty(c, b->bud->lnum); + if (IS_ERR(lp)) { + err = PTR_ERR(lp); + goto out; + } + + dirty = lp->dirty; + if (b->bud->start == 0 && (lp->free != c->leb_size || lp->dirty != 0)) { + /* + * The LEB was added to the journal with a starting offset of + * zero which means the LEB must have been empty. The LEB + * property values should be @lp->free == @c->leb_size and + * @lp->dirty == 0, but that is not the case. The reason is that + * the LEB had been garbage collected before it became the bud, + * and there was no commit in between. The garbage collector + * resets the free and dirty space without recording it + * anywhere except lprops, so if there was no commit then + * lprops does not have that information. + * + * We do not need to adjust free space because the scan has told + * us the exact value which is recorded in the replay entry as + * @b->free. + * + * However we do need to subtract from the dirty space the + * amount of space that the garbage collector reclaimed, which + * is the whole LEB minus the amount of space that was free. + */ + dbg_mnt("bud LEB %d was GC'd (%d free, %d dirty)", b->bud->lnum, + lp->free, lp->dirty); + dbg_gc("bud LEB %d was GC'd (%d free, %d dirty)", b->bud->lnum, + lp->free, lp->dirty); + dirty -= c->leb_size - lp->free; + /* + * If the replay order was perfect the dirty space would now be + * zero. The order is not perfect because the journal heads + * race with each other. This is not a problem but is does mean + * that the dirty space may temporarily exceed c->leb_size + * during the replay. + */ + if (dirty != 0) + dbg_mnt("LEB %d lp: %d free %d dirty replay: %d free %d dirty", + b->bud->lnum, lp->free, lp->dirty, b->free, + b->dirty); + } + lp = ubifs_change_lp(c, lp, b->free, dirty + b->dirty, + lp->flags | LPROPS_TAKEN, 0); + if (IS_ERR(lp)) { + err = PTR_ERR(lp); + goto out; + } + + /* Make sure the journal head points to the latest bud */ + err = ubifs_wbuf_seek_nolock(&c->jheads[b->bud->jhead].wbuf, + b->bud->lnum, c->leb_size - b->free); + +out: + ubifs_release_lprops(c); + return err; +} + +/** + * set_buds_lprops - set free and dirty space for all replayed buds. + * @c: UBIFS file-system description object + * + * This function sets LEB properties for all replayed buds. Returns zero in + * case of success and a negative error code in case of failure. + */ +static int set_buds_lprops(struct ubifs_info *c) +{ + struct bud_entry *b; + int err; + + list_for_each_entry(b, &c->replay_buds, list) { + err = set_bud_lprops(c, b); + if (err) + return err; + } + + return 0; +} + +/** + * trun_remove_range - apply a replay entry for a truncation to the TNC. + * @c: UBIFS file-system description object + * @r: replay entry of truncation + */ +static int trun_remove_range(struct ubifs_info *c, struct replay_entry *r) +{ + unsigned min_blk, max_blk; + union ubifs_key min_key, max_key; + ino_t ino; + + min_blk = r->new_size / UBIFS_BLOCK_SIZE; + if (r->new_size & (UBIFS_BLOCK_SIZE - 1)) + min_blk += 1; + + max_blk = r->old_size / UBIFS_BLOCK_SIZE; + if ((r->old_size & (UBIFS_BLOCK_SIZE - 1)) == 0) + max_blk -= 1; + + ino = key_inum(c, &r->key); + + data_key_init(c, &min_key, ino, min_blk); + data_key_init(c, &max_key, ino, max_blk); + + return ubifs_tnc_remove_range(c, &min_key, &max_key); +} + +/** + * inode_still_linked - check whether inode in question will be re-linked. + * @c: UBIFS file-system description object + * @rino: replay entry to test + * + * O_TMPFILE files can be re-linked, this means link count goes from 0 to 1. + * This case needs special care, otherwise all references to the inode will + * be removed upon the first replay entry of an inode with link count 0 + * is found. + */ +static bool inode_still_linked(struct ubifs_info *c, struct replay_entry *rino) +{ + struct replay_entry *r; + + ubifs_assert(c, rino->deletion); + ubifs_assert(c, key_type(c, &rino->key) == UBIFS_INO_KEY); + + /* + * Find the most recent entry for the inode behind @rino and check + * whether it is a deletion. + */ + list_for_each_entry_reverse(r, &c->replay_list, list) { + ubifs_assert(c, r->sqnum >= rino->sqnum); + if (key_inum(c, &r->key) == key_inum(c, &rino->key) && + key_type(c, &r->key) == UBIFS_INO_KEY) + return r->deletion == 0; + + } + + ubifs_assert(c, 0); + return false; +} + +/** + * apply_replay_entry - apply a replay entry to the TNC. + * @c: UBIFS file-system description object + * @r: replay entry to apply + * + * Apply a replay entry to the TNC. + */ +static int apply_replay_entry(struct ubifs_info *c, struct replay_entry *r) +{ + int err; + + dbg_mntk(&r->key, "LEB %d:%d len %d deletion %d sqnum %llu key ", + r->lnum, r->offs, r->len, r->deletion, r->sqnum); + + if (is_hash_key(c, &r->key)) { + if (r->deletion) + err = ubifs_tnc_remove_nm(c, &r->key, &r->nm); + else + err = ubifs_tnc_add_nm(c, &r->key, r->lnum, r->offs, + r->len, r->hash, &r->nm); + } else { + if (r->deletion) + switch (key_type(c, &r->key)) { + case UBIFS_INO_KEY: + { + ino_t inum = key_inum(c, &r->key); + + if (inode_still_linked(c, r)) { + err = 0; + break; + } + + err = ubifs_tnc_remove_ino(c, inum); + break; + } + case UBIFS_TRUN_KEY: + err = trun_remove_range(c, r); + break; + default: + err = ubifs_tnc_remove(c, &r->key); + break; + } + else + err = ubifs_tnc_add(c, &r->key, r->lnum, r->offs, + r->len, r->hash); + if (err) + return err; + + if (c->need_recovery) + err = ubifs_recover_size_accum(c, &r->key, r->deletion, + r->new_size); + } + + return err; +} + +/** + * replay_entries_cmp - compare 2 replay entries. + * @priv: UBIFS file-system description object + * @a: first replay entry + * @b: second replay entry + * + * This is a comparios function for 'list_sort()' which compares 2 replay + * entries @a and @b by comparing their sequence number. Returns %1 if @a has + * greater sequence number and %-1 otherwise. + */ +static int replay_entries_cmp(void *priv, const struct list_head *a, + const struct list_head *b) +{ + struct ubifs_info *c = priv; + struct replay_entry *ra, *rb; + + if (a == b) + return 0; + + ra = list_entry(a, struct replay_entry, list); + rb = list_entry(b, struct replay_entry, list); + ubifs_assert(c, ra->sqnum != rb->sqnum); + if (ra->sqnum > rb->sqnum) + return 1; + return -1; +} + +/** + * apply_replay_list - apply the replay list to the TNC. + * @c: UBIFS file-system description object + * + * Apply all entries in the replay list to the TNC. Returns zero in case of + * success and a negative error code in case of failure. + */ +static int apply_replay_list(struct ubifs_info *c) +{ + struct replay_entry *r; + int err; + + list_sort(c, &c->replay_list, &replay_entries_cmp); + + list_for_each_entry(r, &c->replay_list, list) { + cond_resched(); + + err = apply_replay_entry(c, r); + if (err) + return err; + } + + return 0; +} + +/** + * destroy_replay_list - destroy the replay. + * @c: UBIFS file-system description object + * + * Destroy the replay list. + */ +static void destroy_replay_list(struct ubifs_info *c) +{ + struct replay_entry *r, *tmp; + + list_for_each_entry_safe(r, tmp, &c->replay_list, list) { + if (is_hash_key(c, &r->key)) + kfree(fname_name(&r->nm)); + list_del(&r->list); + kfree(r); + } +} + +/** + * insert_node - insert a node to the replay list + * @c: UBIFS file-system description object + * @lnum: node logical eraseblock number + * @offs: node offset + * @len: node length + * @hash: node hash + * @key: node key + * @sqnum: sequence number + * @deletion: non-zero if this is a deletion + * @used: number of bytes in use in a LEB + * @old_size: truncation old size + * @new_size: truncation new size + * + * This function inserts a scanned non-direntry node to the replay list. The + * replay list contains @struct replay_entry elements, and we sort this list in + * sequence number order before applying it. The replay list is applied at the + * very end of the replay process. Since the list is sorted in sequence number + * order, the older modifications are applied first. This function returns zero + * in case of success and a negative error code in case of failure. + */ +static int insert_node(struct ubifs_info *c, int lnum, int offs, int len, + const u8 *hash, union ubifs_key *key, + unsigned long long sqnum, int deletion, int *used, + loff_t old_size, loff_t new_size) +{ + struct replay_entry *r; + + dbg_mntk(key, "add LEB %d:%d, key ", lnum, offs); + + if (key_inum(c, key) >= c->highest_inum) + c->highest_inum = key_inum(c, key); + + r = kzalloc_obj(struct replay_entry); + if (!r) + return -ENOMEM; + + if (!deletion) + *used += ALIGN(len, 8); + r->lnum = lnum; + r->offs = offs; + r->len = len; + ubifs_copy_hash(c, hash, r->hash); + r->deletion = !!deletion; + r->sqnum = sqnum; + key_copy(c, key, &r->key); + r->old_size = old_size; + r->new_size = new_size; + + list_add_tail(&r->list, &c->replay_list); + return 0; +} + +/** + * insert_dent - insert a directory entry node into the replay list. + * @c: UBIFS file-system description object + * @lnum: node logical eraseblock number + * @offs: node offset + * @len: node length + * @hash: node hash + * @key: node key + * @name: directory entry name + * @nlen: directory entry name length + * @sqnum: sequence number + * @deletion: non-zero if this is a deletion + * @used: number of bytes in use in a LEB + * + * This function inserts a scanned directory entry node or an extended + * attribute entry to the replay list. Returns zero in case of success and a + * negative error code in case of failure. + */ +static int insert_dent(struct ubifs_info *c, int lnum, int offs, int len, + const u8 *hash, union ubifs_key *key, + const char *name, int nlen, unsigned long long sqnum, + int deletion, int *used) +{ + struct replay_entry *r; + char *nbuf; + + dbg_mntk(key, "add LEB %d:%d, key ", lnum, offs); + if (key_inum(c, key) >= c->highest_inum) + c->highest_inum = key_inum(c, key); + + r = kzalloc_obj(struct replay_entry); + if (!r) + return -ENOMEM; + + nbuf = kmalloc(nlen + 1, GFP_KERNEL); + if (!nbuf) { + kfree(r); + return -ENOMEM; + } + + if (!deletion) + *used += ALIGN(len, 8); + r->lnum = lnum; + r->offs = offs; + r->len = len; + ubifs_copy_hash(c, hash, r->hash); + r->deletion = !!deletion; + r->sqnum = sqnum; + key_copy(c, key, &r->key); + fname_len(&r->nm) = nlen; + memcpy(nbuf, name, nlen); + nbuf[nlen] = '\0'; + fname_name(&r->nm) = nbuf; + + list_add_tail(&r->list, &c->replay_list); + return 0; +} + +/** + * ubifs_validate_entry - validate directory or extended attribute entry node. + * @c: UBIFS file-system description object + * @dent: the node to validate + * + * This function validates directory or extended attribute entry node @dent. + * Returns zero if the node is all right and a %-EINVAL if not. + */ +int ubifs_validate_entry(struct ubifs_info *c, + const struct ubifs_dent_node *dent) +{ + int key_type = key_type_flash(c, dent->key); + int nlen = le16_to_cpu(dent->nlen); + + if (le32_to_cpu(dent->ch.len) != nlen + UBIFS_DENT_NODE_SZ + 1 || + dent->type >= UBIFS_ITYPES_CNT || + nlen > UBIFS_MAX_NLEN || dent->name[nlen] != 0 || + (key_type == UBIFS_XENT_KEY && strnlen(dent->name, nlen) != nlen) || + le64_to_cpu(dent->inum) > MAX_INUM) { + ubifs_err(c, "bad %s node", key_type == UBIFS_DENT_KEY ? + "directory entry" : "extended attribute entry"); + return -EINVAL; + } + + if (key_type != UBIFS_DENT_KEY && key_type != UBIFS_XENT_KEY) { + ubifs_err(c, "bad key type %d", key_type); + return -EINVAL; + } + + return 0; +} + +/** + * is_last_bud - check if the bud is the last in the journal head. + * @c: UBIFS file-system description object + * @bud: bud description object + * + * This function checks if bud @bud is the last bud in its journal head. This + * information is then used by 'replay_bud()' to decide whether the bud can + * have corruptions or not. Indeed, only last buds can be corrupted by power + * cuts. Returns %1 if this is the last bud, and %0 if not. + */ +static int is_last_bud(struct ubifs_info *c, struct ubifs_bud *bud) +{ + struct ubifs_jhead *jh = &c->jheads[bud->jhead]; + struct ubifs_bud *next; + uint32_t data; + int err; + + if (list_is_last(&bud->list, &jh->buds_list)) + return 1; + + /* + * The following is a quirk to make sure we work correctly with UBIFS + * images used with older UBIFS. + * + * Normally, the last bud will be the last in the journal head's list + * of bud. However, there is one exception if the UBIFS image belongs + * to older UBIFS. This is fairly unlikely: one would need to use old + * UBIFS, then have a power cut exactly at the right point, and then + * try to mount this image with new UBIFS. + * + * The exception is: it is possible to have 2 buds A and B, A goes + * before B, and B is the last, bud B is contains no data, and bud A is + * corrupted at the end. The reason is that in older versions when the + * journal code switched the next bud (from A to B), it first added a + * log reference node for the new bud (B), and only after this it + * synchronized the write-buffer of current bud (A). But later this was + * changed and UBIFS started to always synchronize the write-buffer of + * the bud (A) before writing the log reference for the new bud (B). + * + * But because older UBIFS always synchronized A's write-buffer before + * writing to B, we can recognize this exceptional situation but + * checking the contents of bud B - if it is empty, then A can be + * treated as the last and we can recover it. + * + * TODO: remove this piece of code in a couple of years (today it is + * 16.05.2011). + */ + next = list_entry(bud->list.next, struct ubifs_bud, list); + if (!list_is_last(&next->list, &jh->buds_list)) + return 0; + + err = ubifs_leb_read(c, next->lnum, (char *)&data, next->start, 4, 1); + if (err) + return 0; + + return data == 0xFFFFFFFF; +} + +/* authenticate_sleb_hash is split out for stack usage */ +static int noinline_for_stack +authenticate_sleb_hash(struct ubifs_info *c, + struct shash_desc *log_hash, u8 *hash) +{ + SHASH_DESC_ON_STACK(hash_desc, c->hash_tfm); + + hash_desc->tfm = c->hash_tfm; + + ubifs_shash_copy_state(c, log_hash, hash_desc); + return crypto_shash_final(hash_desc, hash); +} + +/** + * authenticate_sleb - authenticate one scan LEB + * @c: UBIFS file-system description object + * @sleb: the scan LEB to authenticate + * @log_hash: + * @is_last: if true, this is the last LEB + * + * This function iterates over the buds of a single LEB authenticating all buds + * with the authentication nodes on this LEB. Authentication nodes are written + * after some buds and contain a HMAC covering the authentication node itself + * and the buds between the last authentication node and the current + * authentication node. It can happen that the last buds cannot be authenticated + * because a powercut happened when some nodes were written but not the + * corresponding authentication node. This function returns the number of nodes + * that could be authenticated or a negative error code. + */ +static int authenticate_sleb(struct ubifs_info *c, struct ubifs_scan_leb *sleb, + struct shash_desc *log_hash, int is_last) +{ + int n_not_auth = 0; + struct ubifs_scan_node *snod; + int n_nodes = 0; + int err; + u8 hash[UBIFS_HASH_ARR_SZ]; + u8 hmac[UBIFS_HMAC_ARR_SZ]; + + if (!ubifs_authenticated(c)) + return sleb->nodes_cnt; + + list_for_each_entry(snod, &sleb->nodes, list) { + + n_nodes++; + + if (snod->type == UBIFS_AUTH_NODE) { + struct ubifs_auth_node *auth = snod->node; + + err = authenticate_sleb_hash(c, log_hash, hash); + if (err) + goto out; + + err = crypto_shash_tfm_digest(c->hmac_tfm, hash, + c->hash_len, hmac); + if (err) + goto out; + + err = ubifs_check_hmac(c, auth->hmac, hmac); + if (err) { + err = -EPERM; + goto out; + } + n_not_auth = 0; + } else { + err = crypto_shash_update(log_hash, snod->node, + snod->len); + if (err) + goto out; + n_not_auth++; + } + } + + /* + * A powercut can happen when some nodes were written, but not yet + * the corresponding authentication node. This may only happen on + * the last bud though. + */ + if (n_not_auth) { + if (is_last) { + dbg_mnt("%d unauthenticated nodes found on LEB %d, Ignoring them", + n_not_auth, sleb->lnum); + err = 0; + } else { + dbg_mnt("%d unauthenticated nodes found on non-last LEB %d", + n_not_auth, sleb->lnum); + err = -EPERM; + } + } else { + err = 0; + } +out: + return err ? err : n_nodes - n_not_auth; +} + +/** + * replay_bud - replay a bud logical eraseblock. + * @c: UBIFS file-system description object + * @b: bud entry which describes the bud + * + * This function replays bud @bud, recovers it if needed, and adds all nodes + * from this bud to the replay list. Returns zero in case of success and a + * negative error code in case of failure. + */ +static int replay_bud(struct ubifs_info *c, struct bud_entry *b) +{ + int is_last = is_last_bud(c, b->bud); + int err = 0, used = 0, lnum = b->bud->lnum, offs = b->bud->start; + int n_nodes, n = 0; + struct ubifs_scan_leb *sleb; + struct ubifs_scan_node *snod; + + dbg_mnt("replay bud LEB %d, head %d, offs %d, is_last %d", + lnum, b->bud->jhead, offs, is_last); + + if (c->need_recovery && is_last) + /* + * Recover only last LEBs in the journal heads, because power + * cuts may cause corruptions only in these LEBs, because only + * these LEBs could possibly be written to at the power cut + * time. + */ + sleb = ubifs_recover_leb(c, lnum, offs, c->sbuf, b->bud->jhead); + else + sleb = ubifs_scan(c, lnum, offs, c->sbuf, 0); + if (IS_ERR(sleb)) + return PTR_ERR(sleb); + + n_nodes = authenticate_sleb(c, sleb, b->bud->log_hash, is_last); + if (n_nodes < 0) { + err = n_nodes; + goto out; + } + + ubifs_shash_copy_state(c, b->bud->log_hash, + c->jheads[b->bud->jhead].log_hash); + + /* + * The bud does not have to start from offset zero - the beginning of + * the 'lnum' LEB may contain previously committed data. One of the + * things we have to do in replay is to correctly update lprops with + * newer information about this LEB. + * + * At this point lprops thinks that this LEB has 'c->leb_size - offs' + * bytes of free space because it only contain information about + * committed data. + * + * But we know that real amount of free space is 'c->leb_size - + * sleb->endpt', and the space in the 'lnum' LEB between 'offs' and + * 'sleb->endpt' is used by bud data. We have to correctly calculate + * how much of these data are dirty and update lprops with this + * information. + * + * The dirt in that LEB region is comprised of padding nodes, deletion + * nodes, truncation nodes and nodes which are obsoleted by subsequent + * nodes in this LEB. So instead of calculating clean space, we + * calculate used space ('used' variable). + */ + + list_for_each_entry(snod, &sleb->nodes, list) { + u8 hash[UBIFS_HASH_ARR_SZ]; + int deletion = 0; + + cond_resched(); + + if (snod->sqnum >= SQNUM_WATERMARK) { + ubifs_err(c, "file system's life ended"); + goto out_dump; + } + + ubifs_node_calc_hash(c, snod->node, hash); + + if (snod->sqnum > c->max_sqnum) + c->max_sqnum = snod->sqnum; + + switch (snod->type) { + case UBIFS_INO_NODE: + { + struct ubifs_ino_node *ino = snod->node; + loff_t new_size = le64_to_cpu(ino->size); + + if (le32_to_cpu(ino->nlink) == 0) + deletion = 1; + err = insert_node(c, lnum, snod->offs, snod->len, hash, + &snod->key, snod->sqnum, deletion, + &used, 0, new_size); + break; + } + case UBIFS_DATA_NODE: + { + struct ubifs_data_node *dn = snod->node; + loff_t new_size = le32_to_cpu(dn->size) + + key_block(c, &snod->key) * + UBIFS_BLOCK_SIZE; + + err = insert_node(c, lnum, snod->offs, snod->len, hash, + &snod->key, snod->sqnum, deletion, + &used, 0, new_size); + break; + } + case UBIFS_DENT_NODE: + case UBIFS_XENT_NODE: + { + struct ubifs_dent_node *dent = snod->node; + + err = ubifs_validate_entry(c, dent); + if (err) + goto out_dump; + + err = insert_dent(c, lnum, snod->offs, snod->len, hash, + &snod->key, dent->name, + le16_to_cpu(dent->nlen), snod->sqnum, + !le64_to_cpu(dent->inum), &used); + break; + } + case UBIFS_TRUN_NODE: + { + struct ubifs_trun_node *trun = snod->node; + loff_t old_size = le64_to_cpu(trun->old_size); + loff_t new_size = le64_to_cpu(trun->new_size); + union ubifs_key key; + + /* Validate truncation node */ + if (old_size < 0 || old_size > c->max_inode_sz || + new_size < 0 || new_size > c->max_inode_sz || + old_size <= new_size) { + ubifs_err(c, "bad truncation node"); + goto out_dump; + } + + /* + * Create a fake truncation key just to use the same + * functions which expect nodes to have keys. + */ + trun_key_init(c, &key, le32_to_cpu(trun->inum)); + err = insert_node(c, lnum, snod->offs, snod->len, hash, + &key, snod->sqnum, 1, &used, + old_size, new_size); + break; + } + case UBIFS_AUTH_NODE: + break; + default: + ubifs_err(c, "unexpected node type %d in bud LEB %d:%d", + snod->type, lnum, snod->offs); + err = -EINVAL; + goto out_dump; + } + if (err) + goto out; + + n++; + if (n == n_nodes) + break; + } + + ubifs_assert(c, ubifs_search_bud(c, lnum)); + ubifs_assert(c, sleb->endpt - offs >= used); + ubifs_assert(c, sleb->endpt % c->min_io_size == 0); + + b->dirty = sleb->endpt - offs - used; + b->free = c->leb_size - sleb->endpt; + dbg_mnt("bud LEB %d replied: dirty %d, free %d", + lnum, b->dirty, b->free); + +out: + ubifs_scan_destroy(sleb); + return err; + +out_dump: + ubifs_err(c, "bad node is at LEB %d:%d", lnum, snod->offs); + ubifs_dump_node(c, snod->node, c->leb_size - snod->offs); + ubifs_scan_destroy(sleb); + return -EINVAL; +} + +/** + * replay_buds - replay all buds. + * @c: UBIFS file-system description object + * + * This function returns zero in case of success and a negative error code in + * case of failure. + */ +static int replay_buds(struct ubifs_info *c) +{ + struct bud_entry *b; + int err; + unsigned long long prev_sqnum = 0; + + list_for_each_entry(b, &c->replay_buds, list) { + err = replay_bud(c, b); + if (err) + return err; + + ubifs_assert(c, b->sqnum > prev_sqnum); + prev_sqnum = b->sqnum; + } + + return 0; +} + +/** + * destroy_bud_list - destroy the list of buds to replay. + * @c: UBIFS file-system description object + */ +static void destroy_bud_list(struct ubifs_info *c) +{ + struct bud_entry *b; + + while (!list_empty(&c->replay_buds)) { + b = list_entry(c->replay_buds.next, struct bud_entry, list); + list_del(&b->list); + kfree(b); + } +} + +/** + * add_replay_bud - add a bud to the list of buds to replay. + * @c: UBIFS file-system description object + * @lnum: bud logical eraseblock number to replay + * @offs: bud start offset + * @jhead: journal head to which this bud belongs + * @sqnum: reference node sequence number + * + * This function returns zero in case of success and a negative error code in + * case of failure. + */ +static int add_replay_bud(struct ubifs_info *c, int lnum, int offs, int jhead, + unsigned long long sqnum) +{ + struct ubifs_bud *bud; + struct bud_entry *b; + int err; + + dbg_mnt("add replay bud LEB %d:%d, head %d", lnum, offs, jhead); + + bud = kmalloc_obj(struct ubifs_bud); + if (!bud) + return -ENOMEM; + + b = kmalloc_obj(struct bud_entry); + if (!b) { + err = -ENOMEM; + goto out; + } + + bud->lnum = lnum; + bud->start = offs; + bud->jhead = jhead; + bud->log_hash = ubifs_hash_get_desc(c); + if (IS_ERR(bud->log_hash)) { + err = PTR_ERR(bud->log_hash); + goto out; + } + + ubifs_shash_copy_state(c, c->log_hash, bud->log_hash); + + ubifs_add_bud(c, bud); + + b->bud = bud; + b->sqnum = sqnum; + list_add_tail(&b->list, &c->replay_buds); + + return 0; +out: + kfree(bud); + kfree(b); + + return err; +} + +/** + * validate_ref - validate a reference node. + * @c: UBIFS file-system description object + * @ref: the reference node to validate + * + * This function returns %1 if a bud reference already exists for the LEB. %0 is + * returned if the reference node is new, otherwise %-EINVAL is returned if + * validation failed. + */ +static int validate_ref(struct ubifs_info *c, const struct ubifs_ref_node *ref) +{ + struct ubifs_bud *bud; + int lnum = le32_to_cpu(ref->lnum); + unsigned int offs = le32_to_cpu(ref->offs); + unsigned int jhead = le32_to_cpu(ref->jhead); + + /* + * ref->offs may point to the end of LEB when the journal head points + * to the end of LEB and we write reference node for it during commit. + * So this is why we require 'offs > c->leb_size'. + */ + if (jhead >= c->jhead_cnt || lnum >= c->leb_cnt || + lnum < c->main_first || offs > c->leb_size || + offs & (c->min_io_size - 1)) + return -EINVAL; + + /* Make sure we have not already looked at this bud */ + bud = ubifs_search_bud(c, lnum); + if (bud) { + if (bud->jhead == jhead && bud->start <= offs) + return 1; + ubifs_err(c, "bud at LEB %d:%d was already referred", lnum, offs); + return -EINVAL; + } + + return 0; +} + +/** + * replay_log_leb - replay a log logical eraseblock. + * @c: UBIFS file-system description object + * @lnum: log logical eraseblock to replay + * @offs: offset to start replaying from + * @sbuf: scan buffer + * + * This function replays a log LEB and returns zero in case of success, %1 if + * this is the last LEB in the log, and a negative error code in case of + * failure. + */ +static int replay_log_leb(struct ubifs_info *c, int lnum, int offs, void *sbuf) +{ + int err; + struct ubifs_scan_leb *sleb; + struct ubifs_scan_node *snod; + const struct ubifs_cs_node *node; + + dbg_mnt("replay log LEB %d:%d", lnum, offs); + sleb = ubifs_scan(c, lnum, offs, sbuf, c->need_recovery); + if (IS_ERR(sleb)) { + if (PTR_ERR(sleb) != -EUCLEAN || !c->need_recovery) + return PTR_ERR(sleb); + /* + * Note, the below function will recover this log LEB only if + * it is the last, because unclean reboots can possibly corrupt + * only the tail of the log. + */ + sleb = ubifs_recover_log_leb(c, lnum, offs, sbuf); + if (IS_ERR(sleb)) + return PTR_ERR(sleb); + } + + if (sleb->nodes_cnt == 0) { + err = 1; + goto out; + } + + node = sleb->buf; + snod = list_entry(sleb->nodes.next, struct ubifs_scan_node, list); + if (c->cs_sqnum == 0) { + /* + * This is the first log LEB we are looking at, make sure that + * the first node is a commit start node. Also record its + * sequence number so that UBIFS can determine where the log + * ends, because all nodes which were have higher sequence + * numbers. + */ + if (snod->type != UBIFS_CS_NODE) { + ubifs_err(c, "first log node at LEB %d:%d is not CS node", + lnum, offs); + goto out_dump; + } + if (le64_to_cpu(node->cmt_no) != c->cmt_no) { + ubifs_err(c, "first CS node at LEB %d:%d has wrong commit number %llu expected %llu", + lnum, offs, + (unsigned long long)le64_to_cpu(node->cmt_no), + c->cmt_no); + goto out_dump; + } + + c->cs_sqnum = le64_to_cpu(node->ch.sqnum); + dbg_mnt("commit start sqnum %llu", c->cs_sqnum); + + err = ubifs_shash_init(c, c->log_hash); + if (err) + goto out; + + err = ubifs_shash_update(c, c->log_hash, node, UBIFS_CS_NODE_SZ); + if (err < 0) + goto out; + } + + if (snod->sqnum < c->cs_sqnum) { + /* + * This means that we reached end of log and now + * look to the older log data, which was already + * committed but the eraseblock was not erased (UBIFS + * only un-maps it). So this basically means we have to + * exit with "end of log" code. + */ + err = 1; + goto out; + } + + /* Make sure the first node sits at offset zero of the LEB */ + if (snod->offs != 0) { + ubifs_err(c, "first node is not at zero offset"); + goto out_dump; + } + + list_for_each_entry(snod, &sleb->nodes, list) { + cond_resched(); + + if (snod->sqnum >= SQNUM_WATERMARK) { + ubifs_err(c, "file system's life ended"); + goto out_dump; + } + + if (snod->sqnum < c->cs_sqnum) { + ubifs_err(c, "bad sqnum %llu, commit sqnum %llu", + snod->sqnum, c->cs_sqnum); + goto out_dump; + } + + if (snod->sqnum > c->max_sqnum) + c->max_sqnum = snod->sqnum; + + switch (snod->type) { + case UBIFS_REF_NODE: { + const struct ubifs_ref_node *ref = snod->node; + + err = validate_ref(c, ref); + if (err == 1) + break; /* Already have this bud */ + if (err) + goto out_dump; + + err = ubifs_shash_update(c, c->log_hash, ref, + UBIFS_REF_NODE_SZ); + if (err) + goto out; + + err = add_replay_bud(c, le32_to_cpu(ref->lnum), + le32_to_cpu(ref->offs), + le32_to_cpu(ref->jhead), + snod->sqnum); + if (err) + goto out; + + break; + } + case UBIFS_CS_NODE: + /* Make sure it sits at the beginning of LEB */ + if (snod->offs != 0) { + ubifs_err(c, "unexpected node in log"); + goto out_dump; + } + break; + default: + ubifs_err(c, "unexpected node in log"); + goto out_dump; + } + } + + if (sleb->endpt || c->lhead_offs >= c->leb_size) { + c->lhead_lnum = lnum; + c->lhead_offs = sleb->endpt; + } + + err = !sleb->endpt; +out: + ubifs_scan_destroy(sleb); + return err; + +out_dump: + ubifs_err(c, "log error detected while replaying the log at LEB %d:%d", + lnum, offs + snod->offs); + ubifs_dump_node(c, snod->node, c->leb_size - snod->offs); + ubifs_scan_destroy(sleb); + return -EINVAL; +} + +/** + * take_ihead - update the status of the index head in lprops to 'taken'. + * @c: UBIFS file-system description object + * + * This function returns the amount of free space in the index head LEB or a + * negative error code. + */ +static int take_ihead(struct ubifs_info *c) +{ + const struct ubifs_lprops *lp; + int err, free; + + ubifs_get_lprops(c); + + lp = ubifs_lpt_lookup_dirty(c, c->ihead_lnum); + if (IS_ERR(lp)) { + err = PTR_ERR(lp); + goto out; + } + + free = lp->free; + + lp = ubifs_change_lp(c, lp, LPROPS_NC, LPROPS_NC, + lp->flags | LPROPS_TAKEN, 0); + if (IS_ERR(lp)) { + err = PTR_ERR(lp); + goto out; + } + + err = free; +out: + ubifs_release_lprops(c); + return err; +} + +/** + * ubifs_replay_journal - replay journal. + * @c: UBIFS file-system description object + * + * This function scans the journal, replays and cleans it up. It makes sure all + * memory data structures related to uncommitted journal are built (dirty TNC + * tree, tree of buds, modified lprops, etc). + */ +int ubifs_replay_journal(struct ubifs_info *c) +{ + int err, lnum, free; + + BUILD_BUG_ON(UBIFS_TRUN_KEY > 5); + + /* Update the status of the index head in lprops to 'taken' */ + free = take_ihead(c); + if (free < 0) + return free; /* Error code */ + + if (c->ihead_offs != c->leb_size - free) { + ubifs_err(c, "bad index head LEB %d:%d", c->ihead_lnum, + c->ihead_offs); + return -EINVAL; + } + + dbg_mnt("start replaying the journal"); + c->replaying = 1; + lnum = c->ltail_lnum = c->lhead_lnum; + + do { + err = replay_log_leb(c, lnum, 0, c->sbuf); + if (err == 1) { + if (lnum != c->lhead_lnum) + /* We hit the end of the log */ + break; + + /* + * The head of the log must always start with the + * "commit start" node on a properly formatted UBIFS. + * But we found no nodes at all, which means that + * something went wrong and we cannot proceed mounting + * the file-system. + */ + ubifs_err(c, "no UBIFS nodes found at the log head LEB %d:%d, possibly corrupted", + lnum, 0); + err = -EINVAL; + } + if (err) + goto out; + lnum = ubifs_next_log_lnum(c, lnum); + } while (lnum != c->ltail_lnum); + + err = replay_buds(c); + if (err) + goto out; + + err = apply_replay_list(c); + if (err) + goto out; + + err = set_buds_lprops(c); + if (err) + goto out; + + /* + * UBIFS budgeting calculations use @c->bi.uncommitted_idx variable + * to roughly estimate index growth. Things like @c->bi.min_idx_lebs + * depend on it. This means we have to initialize it to make sure + * budgeting works properly. + */ + c->bi.uncommitted_idx = atomic_long_read(&c->dirty_zn_cnt); + c->bi.uncommitted_idx *= c->max_idx_node_sz; + + ubifs_assert(c, c->bud_bytes <= c->max_bud_bytes || c->need_recovery); + dbg_mnt("finished, log head LEB %d:%d, max_sqnum %llu, highest_inum %lu", + c->lhead_lnum, c->lhead_offs, c->max_sqnum, + (unsigned long)c->highest_inum); +out: + destroy_replay_list(c); + destroy_bud_list(c); + c->replaying = 0; + return err; +} diff --git a/fs/ubifs/sb.c b/fs/ubifs/sb.c new file mode 100644 index 000000000..e7693b94e --- /dev/null +++ b/fs/ubifs/sb.c @@ -0,0 +1,956 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * + * Authors: Artem Bityutskiy (Битюцкий Артём) + * Adrian Hunter + */ + +/* + * This file implements UBIFS superblock. The superblock is stored at the first + * LEB of the volume and is never changed by UBIFS. Only user-space tools may + * change it. The superblock node mostly contains geometry information. + */ + +#include "ubifs.h" +#include +#include +#include + +/* + * Default journal size in logical eraseblocks as a percent of total + * flash size. + */ +#define DEFAULT_JNL_PERCENT 5 + +/* Default maximum journal size in bytes */ +#define DEFAULT_MAX_JNL (32*1024*1024) + +/* Default indexing tree fanout */ +#define DEFAULT_FANOUT 8 + +/* Default number of data journal heads */ +#define DEFAULT_JHEADS_CNT 1 + +/* Default positions of different LEBs in the main area */ +#define DEFAULT_IDX_LEB 0 +#define DEFAULT_DATA_LEB 1 +#define DEFAULT_GC_LEB 2 + +/* Default number of LEB numbers in LPT's save table */ +#define DEFAULT_LSAVE_CNT 256 + +/* Default reserved pool size as a percent of maximum free space */ +#define DEFAULT_RP_PERCENT 5 + +/* The default maximum size of reserved pool in bytes */ +#define DEFAULT_MAX_RP_SIZE (5*1024*1024) + +/* Default time granularity in nanoseconds */ +#define DEFAULT_TIME_GRAN 1000000000 + +static int get_default_compressor(struct ubifs_info *c) +{ + if (ubifs_compr_present(c, UBIFS_COMPR_ZSTD)) + return UBIFS_COMPR_ZSTD; + + if (ubifs_compr_present(c, UBIFS_COMPR_LZO)) + return UBIFS_COMPR_LZO; + + if (ubifs_compr_present(c, UBIFS_COMPR_ZLIB)) + return UBIFS_COMPR_ZLIB; + + return UBIFS_COMPR_NONE; +} + +/** + * create_default_filesystem - format empty UBI volume. + * @c: UBIFS file-system description object + * + * This function creates default empty file-system. Returns zero in case of + * success and a negative error code in case of failure. + */ +static int create_default_filesystem(struct ubifs_info *c) +{ + struct ubifs_sb_node *sup; + struct ubifs_mst_node *mst; + struct ubifs_idx_node *idx; + struct ubifs_branch *br; + struct ubifs_ino_node *ino; + struct ubifs_cs_node *cs; + union ubifs_key key; + int err, tmp, jnl_lebs, log_lebs, max_buds, main_lebs, main_first; + int lpt_lebs, lpt_first, orph_lebs, big_lpt, ino_waste, sup_flags = 0; + int min_leb_cnt = UBIFS_MIN_LEB_CNT; + int idx_node_size; + long long tmp64, main_bytes; + __le64 tmp_le64; + struct timespec64 ts; + u8 hash[UBIFS_HASH_ARR_SZ]; + u8 hash_lpt[UBIFS_HASH_ARR_SZ]; + + /* Some functions called from here depend on the @c->key_len filed */ + c->key_len = UBIFS_SK_LEN; + + /* + * First of all, we have to calculate default file-system geometry - + * log size, journal size, etc. + */ + if (c->leb_cnt < 0x7FFFFFFF / DEFAULT_JNL_PERCENT) + /* We can first multiply then divide and have no overflow */ + jnl_lebs = c->leb_cnt * DEFAULT_JNL_PERCENT / 100; + else + jnl_lebs = (c->leb_cnt / 100) * DEFAULT_JNL_PERCENT; + + if (jnl_lebs < UBIFS_MIN_JNL_LEBS) + jnl_lebs = UBIFS_MIN_JNL_LEBS; + if (jnl_lebs * c->leb_size > DEFAULT_MAX_JNL) + jnl_lebs = DEFAULT_MAX_JNL / c->leb_size; + + /* + * The log should be large enough to fit reference nodes for all bud + * LEBs. Because buds do not have to start from the beginning of LEBs + * (half of the LEB may contain committed data), the log should + * generally be larger, make it twice as large. + */ + tmp = 2 * (c->ref_node_alsz * jnl_lebs) + c->leb_size - 1; + log_lebs = tmp / c->leb_size; + /* Plus one LEB reserved for commit */ + log_lebs += 1; + if (c->leb_cnt - min_leb_cnt > 8) { + /* And some extra space to allow writes while committing */ + log_lebs += 1; + min_leb_cnt += 1; + } + + max_buds = jnl_lebs - log_lebs; + if (max_buds < UBIFS_MIN_BUD_LEBS) + max_buds = UBIFS_MIN_BUD_LEBS; + + /* + * Orphan nodes are stored in a separate area. One node can store a lot + * of orphan inode numbers, but when new orphan comes we just add a new + * orphan node. At some point the nodes are consolidated into one + * orphan node. + */ + orph_lebs = UBIFS_MIN_ORPH_LEBS; + if (c->leb_cnt - min_leb_cnt > 1) + /* + * For debugging purposes it is better to have at least 2 + * orphan LEBs, because the orphan subsystem would need to do + * consolidations and would be stressed more. + */ + orph_lebs += 1; + + main_lebs = c->leb_cnt - UBIFS_SB_LEBS - UBIFS_MST_LEBS - log_lebs; + main_lebs -= orph_lebs; + + lpt_first = UBIFS_LOG_LNUM + log_lebs; + c->lsave_cnt = DEFAULT_LSAVE_CNT; + c->max_leb_cnt = c->leb_cnt; + err = ubifs_create_dflt_lpt(c, &main_lebs, lpt_first, &lpt_lebs, + &big_lpt, hash_lpt); + if (err) + return err; + + dbg_gen("LEB Properties Tree created (LEBs %d-%d)", lpt_first, + lpt_first + lpt_lebs - 1); + + main_first = c->leb_cnt - main_lebs; + + sup = kzalloc(ALIGN(UBIFS_SB_NODE_SZ, c->min_io_size), GFP_KERNEL); + mst = kzalloc(c->mst_node_alsz, GFP_KERNEL); + idx_node_size = ubifs_idx_node_sz(c, 1); + idx = kzalloc(ALIGN(idx_node_size, c->min_io_size), GFP_KERNEL); + ino = kzalloc(ALIGN(UBIFS_INO_NODE_SZ, c->min_io_size), GFP_KERNEL); + cs = kzalloc(ALIGN(UBIFS_CS_NODE_SZ, c->min_io_size), GFP_KERNEL); + + if (!sup || !mst || !idx || !ino || !cs) { + err = -ENOMEM; + goto out; + } + + /* Create default superblock */ + + tmp64 = (long long)max_buds * c->leb_size; + if (big_lpt) + sup_flags |= UBIFS_FLG_BIGLPT; + if (ubifs_default_version > 4) + sup_flags |= UBIFS_FLG_DOUBLE_HASH; + + if (ubifs_authenticated(c)) { + sup_flags |= UBIFS_FLG_AUTHENTICATION; + sup->hash_algo = cpu_to_le16(c->auth_hash_algo); + err = ubifs_hmac_wkm(c, sup->hmac_wkm); + if (err) + goto out; + } else { + sup->hash_algo = cpu_to_le16(0xffff); + } + + sup->ch.node_type = UBIFS_SB_NODE; + sup->key_hash = UBIFS_KEY_HASH_R5; + sup->flags = cpu_to_le32(sup_flags); + sup->min_io_size = cpu_to_le32(c->min_io_size); + sup->leb_size = cpu_to_le32(c->leb_size); + sup->leb_cnt = cpu_to_le32(c->leb_cnt); + sup->max_leb_cnt = cpu_to_le32(c->max_leb_cnt); + sup->max_bud_bytes = cpu_to_le64(tmp64); + sup->log_lebs = cpu_to_le32(log_lebs); + sup->lpt_lebs = cpu_to_le32(lpt_lebs); + sup->orph_lebs = cpu_to_le32(orph_lebs); + sup->jhead_cnt = cpu_to_le32(DEFAULT_JHEADS_CNT); + sup->fanout = cpu_to_le32(DEFAULT_FANOUT); + sup->lsave_cnt = cpu_to_le32(c->lsave_cnt); + sup->fmt_version = cpu_to_le32(ubifs_default_version); + sup->time_gran = cpu_to_le32(DEFAULT_TIME_GRAN); + if (c->mount_opts.override_compr) + sup->default_compr = cpu_to_le16(c->mount_opts.compr_type); + else + sup->default_compr = cpu_to_le16(get_default_compressor(c)); + + generate_random_uuid(sup->uuid); + + main_bytes = (long long)main_lebs * c->leb_size; + tmp64 = div_u64(main_bytes * DEFAULT_RP_PERCENT, 100); + if (tmp64 > DEFAULT_MAX_RP_SIZE) + tmp64 = DEFAULT_MAX_RP_SIZE; + sup->rp_size = cpu_to_le64(tmp64); + sup->ro_compat_version = cpu_to_le32(UBIFS_RO_COMPAT_VERSION); + + dbg_gen("default superblock created at LEB 0:0"); + + /* Create default master node */ + + mst->ch.node_type = UBIFS_MST_NODE; + mst->log_lnum = cpu_to_le32(UBIFS_LOG_LNUM); + mst->highest_inum = cpu_to_le64(UBIFS_FIRST_INO); + mst->cmt_no = 0; + mst->root_lnum = cpu_to_le32(main_first + DEFAULT_IDX_LEB); + mst->root_offs = 0; + tmp = ubifs_idx_node_sz(c, 1); + mst->root_len = cpu_to_le32(tmp); + mst->gc_lnum = cpu_to_le32(main_first + DEFAULT_GC_LEB); + mst->ihead_lnum = cpu_to_le32(main_first + DEFAULT_IDX_LEB); + mst->ihead_offs = cpu_to_le32(ALIGN(tmp, c->min_io_size)); + mst->index_size = cpu_to_le64(ALIGN(tmp, 8)); + mst->lpt_lnum = cpu_to_le32(c->lpt_lnum); + mst->lpt_offs = cpu_to_le32(c->lpt_offs); + mst->nhead_lnum = cpu_to_le32(c->nhead_lnum); + mst->nhead_offs = cpu_to_le32(c->nhead_offs); + mst->ltab_lnum = cpu_to_le32(c->ltab_lnum); + mst->ltab_offs = cpu_to_le32(c->ltab_offs); + mst->lsave_lnum = cpu_to_le32(c->lsave_lnum); + mst->lsave_offs = cpu_to_le32(c->lsave_offs); + mst->lscan_lnum = cpu_to_le32(main_first); + mst->empty_lebs = cpu_to_le32(main_lebs - 2); + mst->idx_lebs = cpu_to_le32(1); + mst->leb_cnt = cpu_to_le32(c->leb_cnt); + ubifs_copy_hash(c, hash_lpt, mst->hash_lpt); + + /* Calculate lprops statistics */ + tmp64 = main_bytes; + tmp64 -= ALIGN(ubifs_idx_node_sz(c, 1), c->min_io_size); + tmp64 -= ALIGN(UBIFS_INO_NODE_SZ, c->min_io_size); + mst->total_free = cpu_to_le64(tmp64); + + tmp64 = ALIGN(ubifs_idx_node_sz(c, 1), c->min_io_size); + ino_waste = ALIGN(UBIFS_INO_NODE_SZ, c->min_io_size) - + UBIFS_INO_NODE_SZ; + tmp64 += ino_waste; + tmp64 -= ALIGN(ubifs_idx_node_sz(c, 1), 8); + mst->total_dirty = cpu_to_le64(tmp64); + + /* The indexing LEB does not contribute to dark space */ + tmp64 = ((long long)(c->main_lebs - 1) * c->dark_wm); + mst->total_dark = cpu_to_le64(tmp64); + + mst->total_used = cpu_to_le64(UBIFS_INO_NODE_SZ); + + dbg_gen("default master node created at LEB %d:0", UBIFS_MST_LNUM); + + /* Create the root indexing node */ + + c->key_fmt = UBIFS_SIMPLE_KEY_FMT; + c->key_hash = key_r5_hash; + + idx->ch.node_type = UBIFS_IDX_NODE; + idx->child_cnt = cpu_to_le16(1); + ino_key_init(c, &key, UBIFS_ROOT_INO); + br = ubifs_idx_branch(c, idx, 0); + key_write_idx(c, &key, &br->key); + br->lnum = cpu_to_le32(main_first + DEFAULT_DATA_LEB); + br->len = cpu_to_le32(UBIFS_INO_NODE_SZ); + + dbg_gen("default root indexing node created LEB %d:0", + main_first + DEFAULT_IDX_LEB); + + /* Create default root inode */ + + ino_key_init_flash(c, &ino->key, UBIFS_ROOT_INO); + ino->ch.node_type = UBIFS_INO_NODE; + ino->creat_sqnum = cpu_to_le64(++c->max_sqnum); + ino->nlink = cpu_to_le32(2); + + ktime_get_coarse_real_ts64(&ts); + tmp_le64 = cpu_to_le64(ts.tv_sec); + ino->atime_sec = tmp_le64; + ino->ctime_sec = tmp_le64; + ino->mtime_sec = tmp_le64; + ino->atime_nsec = 0; + ino->ctime_nsec = 0; + ino->mtime_nsec = 0; + ino->mode = cpu_to_le32(S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO); + ino->size = cpu_to_le64(UBIFS_INO_NODE_SZ); + + /* Set compression enabled by default */ + ino->flags = cpu_to_le32(UBIFS_COMPR_FL); + + dbg_gen("root inode created at LEB %d:0", + main_first + DEFAULT_DATA_LEB); + + /* + * The first node in the log has to be the commit start node. This is + * always the case during normal file-system operation. Write a fake + * commit start node to the log. + */ + + cs->ch.node_type = UBIFS_CS_NODE; + + err = ubifs_write_node_hmac(c, sup, UBIFS_SB_NODE_SZ, 0, 0, + offsetof(struct ubifs_sb_node, hmac)); + if (err) + goto out; + + err = ubifs_write_node(c, ino, UBIFS_INO_NODE_SZ, + main_first + DEFAULT_DATA_LEB, 0); + if (err) + goto out; + + ubifs_node_calc_hash(c, ino, hash); + ubifs_copy_hash(c, hash, ubifs_branch_hash(c, br)); + + err = ubifs_write_node(c, idx, idx_node_size, main_first + DEFAULT_IDX_LEB, 0); + if (err) + goto out; + + ubifs_node_calc_hash(c, idx, hash); + ubifs_copy_hash(c, hash, mst->hash_root_idx); + + err = ubifs_write_node_hmac(c, mst, UBIFS_MST_NODE_SZ, UBIFS_MST_LNUM, 0, + offsetof(struct ubifs_mst_node, hmac)); + if (err) + goto out; + + err = ubifs_write_node_hmac(c, mst, UBIFS_MST_NODE_SZ, UBIFS_MST_LNUM + 1, + 0, offsetof(struct ubifs_mst_node, hmac)); + if (err) + goto out; + + err = ubifs_write_node(c, cs, UBIFS_CS_NODE_SZ, UBIFS_LOG_LNUM, 0); + if (err) + goto out; + + ubifs_msg(c, "default file-system created"); + + err = 0; +out: + kfree(sup); + kfree(mst); + kfree(idx); + kfree(ino); + kfree(cs); + + return err; +} + +/** + * validate_sb - validate superblock node. + * @c: UBIFS file-system description object + * @sup: superblock node + * + * This function validates superblock node @sup. Since most of data was read + * from the superblock and stored in @c, the function validates fields in @c + * instead. Returns zero in case of success and %-EINVAL in case of validation + * failure. + */ +static int validate_sb(struct ubifs_info *c, struct ubifs_sb_node *sup) +{ + long long max_bytes; + int err = 1, min_leb_cnt; + + if (!c->key_hash) { + err = 2; + goto failed; + } + + if (sup->key_fmt != UBIFS_SIMPLE_KEY_FMT) { + err = 3; + goto failed; + } + + if (le32_to_cpu(sup->min_io_size) != c->min_io_size) { + ubifs_err(c, "min. I/O unit mismatch: %d in superblock, %d real", + le32_to_cpu(sup->min_io_size), c->min_io_size); + goto failed; + } + + if (le32_to_cpu(sup->leb_size) != c->leb_size) { + ubifs_err(c, "LEB size mismatch: %d in superblock, %d real", + le32_to_cpu(sup->leb_size), c->leb_size); + goto failed; + } + + if (c->log_lebs < UBIFS_MIN_LOG_LEBS || + c->lpt_lebs < UBIFS_MIN_LPT_LEBS || + c->orph_lebs < UBIFS_MIN_ORPH_LEBS || + c->main_lebs < UBIFS_MIN_MAIN_LEBS) { + err = 4; + goto failed; + } + + /* + * Calculate minimum allowed amount of main area LEBs. This is very + * similar to %UBIFS_MIN_LEB_CNT, but we take into account real what we + * have just read from the superblock. + */ + min_leb_cnt = UBIFS_SB_LEBS + UBIFS_MST_LEBS + c->log_lebs; + min_leb_cnt += c->lpt_lebs + c->orph_lebs + c->jhead_cnt + 6; + + if (c->leb_cnt < min_leb_cnt || c->leb_cnt > c->vi.size) { + ubifs_err(c, "bad LEB count: %d in superblock, %d on UBI volume, %d minimum required", + c->leb_cnt, c->vi.size, min_leb_cnt); + goto failed; + } + + if (c->max_leb_cnt < c->leb_cnt) { + ubifs_err(c, "max. LEB count %d less than LEB count %d", + c->max_leb_cnt, c->leb_cnt); + goto failed; + } + + if (c->main_lebs < UBIFS_MIN_MAIN_LEBS) { + ubifs_err(c, "too few main LEBs count %d, must be at least %d", + c->main_lebs, UBIFS_MIN_MAIN_LEBS); + goto failed; + } + + max_bytes = (long long)c->leb_size * UBIFS_MIN_BUD_LEBS; + if (c->max_bud_bytes < max_bytes) { + ubifs_err(c, "too small journal (%lld bytes), must be at least %lld bytes", + c->max_bud_bytes, max_bytes); + goto failed; + } + + max_bytes = (long long)c->leb_size * c->main_lebs; + if (c->max_bud_bytes > max_bytes) { + ubifs_err(c, "too large journal size (%lld bytes), only %lld bytes available in the main area", + c->max_bud_bytes, max_bytes); + goto failed; + } + + if (c->jhead_cnt < NONDATA_JHEADS_CNT + 1 || + c->jhead_cnt > NONDATA_JHEADS_CNT + UBIFS_MAX_JHEADS) { + err = 9; + goto failed; + } + + if (c->fanout < UBIFS_MIN_FANOUT || + ubifs_idx_node_sz(c, c->fanout) > c->leb_size) { + err = 10; + goto failed; + } + + if (c->lsave_cnt < 0 || (c->lsave_cnt > DEFAULT_LSAVE_CNT && + c->lsave_cnt > c->max_leb_cnt - UBIFS_SB_LEBS - UBIFS_MST_LEBS - + c->log_lebs - c->lpt_lebs - c->orph_lebs)) { + err = 11; + goto failed; + } + + if (UBIFS_SB_LEBS + UBIFS_MST_LEBS + c->log_lebs + c->lpt_lebs + + c->orph_lebs + c->main_lebs != c->leb_cnt) { + err = 12; + goto failed; + } + + if (c->default_compr >= UBIFS_COMPR_TYPES_CNT) { + err = 13; + goto failed; + } + + if (c->rp_size < 0 || max_bytes < c->rp_size) { + err = 14; + goto failed; + } + + if (le32_to_cpu(sup->time_gran) > 1000000000 || + le32_to_cpu(sup->time_gran) < 1) { + err = 15; + goto failed; + } + + if (!c->double_hash && c->fmt_version >= 5) { + err = 16; + goto failed; + } + + if (c->encrypted && c->fmt_version < 5) { + err = 17; + goto failed; + } + + return 0; + +failed: + ubifs_err(c, "bad superblock, error %d", err); + ubifs_dump_node(c, sup, ALIGN(UBIFS_SB_NODE_SZ, c->min_io_size)); + return -EINVAL; +} + +/** + * ubifs_read_sb_node - read superblock node. + * @c: UBIFS file-system description object + * + * This function returns a pointer to the superblock node or a negative error + * code. Note, the user of this function is responsible of kfree()'ing the + * returned superblock buffer. + */ +static struct ubifs_sb_node *ubifs_read_sb_node(struct ubifs_info *c) +{ + struct ubifs_sb_node *sup; + int err; + + sup = kmalloc(ALIGN(UBIFS_SB_NODE_SZ, c->min_io_size), GFP_NOFS); + if (!sup) + return ERR_PTR(-ENOMEM); + + err = ubifs_read_node(c, sup, UBIFS_SB_NODE, UBIFS_SB_NODE_SZ, + UBIFS_SB_LNUM, 0); + if (err) { + kfree(sup); + return ERR_PTR(err); + } + + return sup; +} + +static int authenticate_sb_node(struct ubifs_info *c, + const struct ubifs_sb_node *sup) +{ + unsigned int sup_flags = le32_to_cpu(sup->flags); + u8 hmac_wkm[UBIFS_HMAC_ARR_SZ]; + int authenticated = !!(sup_flags & UBIFS_FLG_AUTHENTICATION); + int hash_algo; + int err; + + if (c->authenticated && !authenticated) { + ubifs_err(c, "authenticated FS forced, but found FS without authentication"); + return -EINVAL; + } + + if (!c->authenticated && authenticated) { + ubifs_err(c, "authenticated FS found, but no key given"); + return -EINVAL; + } + + ubifs_msg(c, "Mounting in %sauthenticated mode", + c->authenticated ? "" : "un"); + + if (!c->authenticated) + return 0; + + if (!IS_ENABLED(CONFIG_UBIFS_FS_AUTHENTICATION)) + return -EOPNOTSUPP; + + hash_algo = le16_to_cpu(sup->hash_algo); + if (hash_algo >= HASH_ALGO__LAST) { + ubifs_err(c, "superblock uses unknown hash algo %d", + hash_algo); + return -EINVAL; + } + + if (strcmp(hash_algo_name[hash_algo], c->auth_hash_name)) { + ubifs_err(c, "This filesystem uses %s for hashing," + " but %s is specified", hash_algo_name[hash_algo], + c->auth_hash_name); + return -EINVAL; + } + + /* + * The super block node can either be authenticated by a HMAC or + * by a signature in a ubifs_sig_node directly following the + * super block node to support offline image creation. + */ + if (ubifs_hmac_zero(c, sup->hmac)) { + err = ubifs_sb_verify_signature(c, sup); + } else { + err = ubifs_hmac_wkm(c, hmac_wkm); + if (err) + return err; + if (ubifs_check_hmac(c, hmac_wkm, sup->hmac_wkm)) { + ubifs_err(c, "provided key does not fit"); + return -ENOKEY; + } + err = ubifs_node_verify_hmac(c, sup, sizeof(*sup), + offsetof(struct ubifs_sb_node, + hmac)); + } + + if (err) + ubifs_err(c, "Failed to authenticate superblock: %d", err); + + return err; +} + +/** + * ubifs_write_sb_node - write superblock node. + * @c: UBIFS file-system description object + * @sup: superblock node read with 'ubifs_read_sb_node()' + * + * This function returns %0 on success and a negative error code on failure. + */ +int ubifs_write_sb_node(struct ubifs_info *c, struct ubifs_sb_node *sup) +{ + int len = ALIGN(UBIFS_SB_NODE_SZ, c->min_io_size); + int err; + + err = ubifs_prepare_node_hmac(c, sup, UBIFS_SB_NODE_SZ, + offsetof(struct ubifs_sb_node, hmac), 1); + if (err) + return err; + + return ubifs_leb_change(c, UBIFS_SB_LNUM, sup, len); +} + +/** + * ubifs_read_superblock - read superblock. + * @c: UBIFS file-system description object + * + * This function finds, reads and checks the superblock. If an empty UBI volume + * is being mounted, this function creates default superblock. Returns zero in + * case of success, and a negative error code in case of failure. + */ +int ubifs_read_superblock(struct ubifs_info *c) +{ + int err, sup_flags; + struct ubifs_sb_node *sup; + + if (c->empty) { + err = create_default_filesystem(c); + if (err) + return err; + } + + sup = ubifs_read_sb_node(c); + if (IS_ERR(sup)) + return PTR_ERR(sup); + + c->sup_node = sup; + + c->fmt_version = le32_to_cpu(sup->fmt_version); + c->ro_compat_version = le32_to_cpu(sup->ro_compat_version); + + /* + * The software supports all previous versions but not future versions, + * due to the unavailability of time-travelling equipment. + */ + if (c->fmt_version > UBIFS_FORMAT_VERSION) { + ubifs_assert(c, !c->ro_media || c->ro_mount); + if (!c->ro_mount || + c->ro_compat_version > UBIFS_RO_COMPAT_VERSION) { + ubifs_err(c, "on-flash format version is w%d/r%d, but software only supports up to version w%d/r%d", + c->fmt_version, c->ro_compat_version, + UBIFS_FORMAT_VERSION, + UBIFS_RO_COMPAT_VERSION); + if (c->ro_compat_version <= UBIFS_RO_COMPAT_VERSION) { + ubifs_msg(c, "only R/O mounting is possible"); + err = -EROFS; + } else + err = -EINVAL; + goto out; + } + + /* + * The FS is mounted R/O, and the media format is + * R/O-compatible with the UBIFS implementation, so we can + * mount. + */ + c->rw_incompat = 1; + } + + if (c->fmt_version < 3) { + ubifs_err(c, "on-flash format version %d is not supported", + c->fmt_version); + err = -EINVAL; + goto out; + } + + switch (sup->key_hash) { + case UBIFS_KEY_HASH_R5: + c->key_hash = key_r5_hash; + c->key_hash_type = UBIFS_KEY_HASH_R5; + break; + + case UBIFS_KEY_HASH_TEST: + c->key_hash = key_test_hash; + c->key_hash_type = UBIFS_KEY_HASH_TEST; + break; + } + + c->key_fmt = sup->key_fmt; + + switch (c->key_fmt) { + case UBIFS_SIMPLE_KEY_FMT: + c->key_len = UBIFS_SK_LEN; + break; + default: + ubifs_err(c, "unsupported key format"); + err = -EINVAL; + goto out; + } + + c->leb_cnt = le32_to_cpu(sup->leb_cnt); + c->max_leb_cnt = le32_to_cpu(sup->max_leb_cnt); + c->max_bud_bytes = le64_to_cpu(sup->max_bud_bytes); + c->log_lebs = le32_to_cpu(sup->log_lebs); + c->lpt_lebs = le32_to_cpu(sup->lpt_lebs); + c->orph_lebs = le32_to_cpu(sup->orph_lebs); + c->jhead_cnt = le32_to_cpu(sup->jhead_cnt) + NONDATA_JHEADS_CNT; + c->fanout = le32_to_cpu(sup->fanout); + c->lsave_cnt = le32_to_cpu(sup->lsave_cnt); + c->rp_size = le64_to_cpu(sup->rp_size); + c->rp_uid = make_kuid(&init_user_ns, le32_to_cpu(sup->rp_uid)); + c->rp_gid = make_kgid(&init_user_ns, le32_to_cpu(sup->rp_gid)); + sup_flags = le32_to_cpu(sup->flags); + if (!c->mount_opts.override_compr) + c->default_compr = le16_to_cpu(sup->default_compr); + + c->vfs_sb->s_time_gran = le32_to_cpu(sup->time_gran); + memcpy(&c->uuid, &sup->uuid, 16); + c->big_lpt = !!(sup_flags & UBIFS_FLG_BIGLPT); + c->space_fixup = !!(sup_flags & UBIFS_FLG_SPACE_FIXUP); + c->double_hash = !!(sup_flags & UBIFS_FLG_DOUBLE_HASH); + c->encrypted = !!(sup_flags & UBIFS_FLG_ENCRYPTION); + + err = authenticate_sb_node(c, sup); + if (err) + goto out; + + if ((sup_flags & ~UBIFS_FLG_MASK) != 0) { + ubifs_err(c, "Unknown feature flags found: %#x", + sup_flags & ~UBIFS_FLG_MASK); + err = -EINVAL; + goto out; + } + + if (!IS_ENABLED(CONFIG_FS_ENCRYPTION) && c->encrypted) { + ubifs_err(c, "file system contains encrypted files but UBIFS" + " was built without crypto support."); + err = -EINVAL; + goto out; + } + + /* Automatically increase file system size to the maximum size */ + if (c->leb_cnt < c->vi.size && c->leb_cnt < c->max_leb_cnt) { + int old_leb_cnt = c->leb_cnt; + + c->leb_cnt = min_t(int, c->max_leb_cnt, c->vi.size); + sup->leb_cnt = cpu_to_le32(c->leb_cnt); + + c->superblock_need_write = 1; + + dbg_mnt("Auto resizing from %d LEBs to %d LEBs", + old_leb_cnt, c->leb_cnt); + } + + c->log_bytes = (long long)c->log_lebs * c->leb_size; + c->log_last = UBIFS_LOG_LNUM + c->log_lebs - 1; + c->lpt_first = UBIFS_LOG_LNUM + c->log_lebs; + c->lpt_last = c->lpt_first + c->lpt_lebs - 1; + c->orph_first = c->lpt_last + 1; + c->orph_last = c->orph_first + c->orph_lebs - 1; + c->main_lebs = c->leb_cnt - UBIFS_SB_LEBS - UBIFS_MST_LEBS; + c->main_lebs -= c->log_lebs + c->lpt_lebs + c->orph_lebs; + c->main_first = c->leb_cnt - c->main_lebs; + + err = validate_sb(c, sup); +out: + return err; +} + +/** + * fixup_leb - fixup/unmap an LEB containing free space. + * @c: UBIFS file-system description object + * @lnum: the LEB number to fix up + * @len: number of used bytes in LEB (starting at offset 0) + * + * This function reads the contents of the given LEB number @lnum, then fixes + * it up, so that empty min. I/O units in the end of LEB are actually erased on + * flash (rather than being just all-0xff real data). If the LEB is completely + * empty, it is simply unmapped. + */ +static int fixup_leb(struct ubifs_info *c, int lnum, int len) +{ + int err; + + ubifs_assert(c, len >= 0); + ubifs_assert(c, len % c->min_io_size == 0); + ubifs_assert(c, len < c->leb_size); + + if (len == 0) { + dbg_mnt("unmap empty LEB %d", lnum); + return ubifs_leb_unmap(c, lnum); + } + + dbg_mnt("fixup LEB %d, data len %d", lnum, len); + err = ubifs_leb_read(c, lnum, c->sbuf, 0, len, 1); + if (err) + return err; + + return ubifs_leb_change(c, lnum, c->sbuf, len); +} + +/** + * fixup_free_space - find & remap all LEBs containing free space. + * @c: UBIFS file-system description object + * + * This function walks through all LEBs in the filesystem and fiexes up those + * containing free/empty space. + */ +static int fixup_free_space(struct ubifs_info *c) +{ + int lnum, err = 0; + struct ubifs_lprops *lprops; + + ubifs_get_lprops(c); + + /* Fixup LEBs in the master area */ + for (lnum = UBIFS_MST_LNUM; lnum < UBIFS_LOG_LNUM; lnum++) { + err = fixup_leb(c, lnum, c->mst_offs + c->mst_node_alsz); + if (err) + goto out; + } + + /* Unmap unused log LEBs */ + lnum = ubifs_next_log_lnum(c, c->lhead_lnum); + while (lnum != c->ltail_lnum) { + err = fixup_leb(c, lnum, 0); + if (err) + goto out; + lnum = ubifs_next_log_lnum(c, lnum); + } + + /* + * Fixup the log head which contains the only a CS node at the + * beginning. + */ + err = fixup_leb(c, c->lhead_lnum, + ALIGN(UBIFS_CS_NODE_SZ, c->min_io_size)); + if (err) + goto out; + + /* Fixup LEBs in the LPT area */ + for (lnum = c->lpt_first; lnum <= c->lpt_last; lnum++) { + int free = c->ltab[lnum - c->lpt_first].free; + + if (free > 0) { + err = fixup_leb(c, lnum, c->leb_size - free); + if (err) + goto out; + } + } + + /* Unmap LEBs in the orphans area */ + for (lnum = c->orph_first; lnum <= c->orph_last; lnum++) { + err = fixup_leb(c, lnum, 0); + if (err) + goto out; + } + + /* Fixup LEBs in the main area */ + for (lnum = c->main_first; lnum < c->leb_cnt; lnum++) { + lprops = ubifs_lpt_lookup(c, lnum); + if (IS_ERR(lprops)) { + err = PTR_ERR(lprops); + goto out; + } + + if (lprops->free > 0) { + err = fixup_leb(c, lnum, c->leb_size - lprops->free); + if (err) + goto out; + } + } + +out: + ubifs_release_lprops(c); + return err; +} + +/** + * ubifs_fixup_free_space - find & fix all LEBs with free space. + * @c: UBIFS file-system description object + * + * This function fixes up LEBs containing free space on first mount, if the + * appropriate flag was set when the FS was created. Each LEB with one or more + * empty min. I/O unit (i.e. free-space-count > 0) is re-written, to make sure + * the free space is actually erased. E.g., this is necessary for some NAND + * chips, since the free space may have been programmed like real "0xff" data + * (generating a non-0xff ECC), causing future writes to the not-really-erased + * NAND pages to behave badly. After the space is fixed up, the superblock flag + * is cleared, so that this is skipped for all future mounts. + */ +int ubifs_fixup_free_space(struct ubifs_info *c) +{ + int err; + struct ubifs_sb_node *sup = c->sup_node; + + ubifs_assert(c, c->space_fixup); + ubifs_assert(c, !c->ro_mount); + + ubifs_msg(c, "start fixing up free space"); + + err = fixup_free_space(c); + if (err) + return err; + + /* Free-space fixup is no longer required */ + c->space_fixup = 0; + sup->flags &= cpu_to_le32(~UBIFS_FLG_SPACE_FIXUP); + + c->superblock_need_write = 1; + + ubifs_msg(c, "free space fixup complete"); + return err; +} + +int ubifs_enable_encryption(struct ubifs_info *c) +{ + int err; + struct ubifs_sb_node *sup = c->sup_node; + + if (!IS_ENABLED(CONFIG_FS_ENCRYPTION)) + return -EOPNOTSUPP; + + if (c->encrypted) + return 0; + + if (c->ro_mount || c->ro_media) + return -EROFS; + + if (c->fmt_version < 5) { + ubifs_err(c, "on-flash format version 5 is needed for encryption"); + return -EINVAL; + } + + sup->flags |= cpu_to_le32(UBIFS_FLG_ENCRYPTION); + + err = ubifs_write_sb_node(c, sup); + if (!err) + c->encrypted = 1; + + return err; +} diff --git a/fs/ubifs/scan.c b/fs/ubifs/scan.c new file mode 100644 index 000000000..cbff2d800 --- /dev/null +++ b/fs/ubifs/scan.c @@ -0,0 +1,366 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation + * + * Authors: Adrian Hunter + * Artem Bityutskiy (Битюцкий Артём) + */ + +/* + * This file implements the scan which is a general-purpose function for + * determining what nodes are in an eraseblock. The scan is used to replay the + * journal, to do garbage collection. for the TNC in-the-gaps method, and by + * debugging functions. + */ + +#include "ubifs.h" + +/** + * scan_padding_bytes - scan for padding bytes. + * @buf: buffer to scan + * @len: length of buffer + * + * This function returns the number of padding bytes on success and + * %SCANNED_GARBAGE on failure. + */ +static int scan_padding_bytes(void *buf, int len) +{ + int pad_len = 0, max_pad_len = min_t(int, UBIFS_PAD_NODE_SZ, len); + uint8_t *p = buf; + + dbg_scan("not a node"); + + while (pad_len < max_pad_len && *p++ == UBIFS_PADDING_BYTE) + pad_len += 1; + + if (!pad_len || (pad_len & 7)) + return SCANNED_GARBAGE; + + dbg_scan("%d padding bytes", pad_len); + + return pad_len; +} + +/** + * ubifs_scan_a_node - scan for a node or padding. + * @c: UBIFS file-system description object + * @buf: buffer to scan + * @len: length of buffer + * @lnum: logical eraseblock number + * @offs: offset within the logical eraseblock + * @quiet: print no messages + * + * This function returns a scanning code to indicate what was scanned. + */ +int ubifs_scan_a_node(const struct ubifs_info *c, void *buf, int len, int lnum, + int offs, int quiet) +{ + struct ubifs_ch *ch = buf; + uint32_t magic; + + magic = le32_to_cpu(ch->magic); + + if (magic == 0xFFFFFFFF) { + dbg_scan("hit empty space at LEB %d:%d", lnum, offs); + return SCANNED_EMPTY_SPACE; + } + + if (magic != UBIFS_NODE_MAGIC) + return scan_padding_bytes(buf, len); + + if (len < UBIFS_CH_SZ) + return SCANNED_GARBAGE; + + dbg_scan("scanning %s at LEB %d:%d", + dbg_ntype(ch->node_type), lnum, offs); + + if (ubifs_check_node(c, buf, len, lnum, offs, quiet, 1)) + return SCANNED_A_CORRUPT_NODE; + + if (ch->node_type == UBIFS_PAD_NODE) { + struct ubifs_pad_node *pad = buf; + int pad_len = le32_to_cpu(pad->pad_len); + int node_len = le32_to_cpu(ch->len); + + /* Validate the padding node */ + if (pad_len < 0 || + offs + node_len + pad_len > c->leb_size) { + if (!quiet) { + ubifs_err(c, "bad pad node at LEB %d:%d", + lnum, offs); + ubifs_dump_node(c, pad, len); + } + return SCANNED_A_BAD_PAD_NODE; + } + + /* Make the node pads to 8-byte boundary */ + if ((node_len + pad_len) & 7) { + if (!quiet) + ubifs_err(c, "bad padding length %d - %d", + offs, offs + node_len + pad_len); + return SCANNED_A_BAD_PAD_NODE; + } + + dbg_scan("%d bytes padded at LEB %d:%d, offset now %d", pad_len, + lnum, offs, ALIGN(offs + node_len + pad_len, 8)); + + return node_len + pad_len; + } + + return SCANNED_A_NODE; +} + +/** + * ubifs_start_scan - create LEB scanning information at start of scan. + * @c: UBIFS file-system description object + * @lnum: logical eraseblock number + * @offs: offset to start at (usually zero) + * @sbuf: scan buffer (must be c->leb_size) + * + * This function returns the scanned information on success and a negative error + * code on failure. + */ +struct ubifs_scan_leb *ubifs_start_scan(const struct ubifs_info *c, int lnum, + int offs, void *sbuf) +{ + struct ubifs_scan_leb *sleb; + int err; + + dbg_scan("scan LEB %d:%d", lnum, offs); + + sleb = kzalloc_obj(struct ubifs_scan_leb, GFP_NOFS); + if (!sleb) + return ERR_PTR(-ENOMEM); + + sleb->lnum = lnum; + INIT_LIST_HEAD(&sleb->nodes); + sleb->buf = sbuf; + + err = ubifs_leb_read(c, lnum, sbuf + offs, offs, c->leb_size - offs, 0); + if (err && err != -EBADMSG) { + ubifs_err(c, "cannot read %d bytes from LEB %d:%d, error %d", + c->leb_size - offs, lnum, offs, err); + kfree(sleb); + return ERR_PTR(err); + } + + /* + * Note, we ignore integrity errors (EBASMSG) because all the nodes are + * protected by CRC checksums. + */ + return sleb; +} + +/** + * ubifs_end_scan - update LEB scanning information at end of scan. + * @c: UBIFS file-system description object + * @sleb: scanning information + * @lnum: logical eraseblock number + * @offs: offset to start at (usually zero) + */ +void ubifs_end_scan(const struct ubifs_info *c, struct ubifs_scan_leb *sleb, + int lnum, int offs) +{ + dbg_scan("stop scanning LEB %d at offset %d", lnum, offs); + ubifs_assert(c, offs % c->min_io_size == 0); + + sleb->endpt = ALIGN(offs, c->min_io_size); +} + +/** + * ubifs_add_snod - add a scanned node to LEB scanning information. + * @c: UBIFS file-system description object + * @sleb: scanning information + * @buf: buffer containing node + * @offs: offset of node on flash + * + * This function returns %0 on success and a negative error code on failure. + */ +int ubifs_add_snod(const struct ubifs_info *c, struct ubifs_scan_leb *sleb, + void *buf, int offs) +{ + struct ubifs_ch *ch = buf; + struct ubifs_ino_node *ino = buf; + struct ubifs_scan_node *snod; + + snod = kmalloc_obj(struct ubifs_scan_node, GFP_NOFS); + if (!snod) + return -ENOMEM; + + snod->sqnum = le64_to_cpu(ch->sqnum); + snod->type = ch->node_type; + snod->offs = offs; + snod->len = le32_to_cpu(ch->len); + snod->node = buf; + + switch (ch->node_type) { + case UBIFS_INO_NODE: + case UBIFS_DENT_NODE: + case UBIFS_XENT_NODE: + case UBIFS_DATA_NODE: + /* + * The key is in the same place in all keyed + * nodes. + */ + key_read(c, &ino->key, &snod->key); + break; + default: + invalid_key_init(c, &snod->key); + break; + } + list_add_tail(&snod->list, &sleb->nodes); + sleb->nodes_cnt += 1; + return 0; +} + +/** + * ubifs_scanned_corruption - print information after UBIFS scanned corruption. + * @c: UBIFS file-system description object + * @lnum: LEB number of corruption + * @offs: offset of corruption + * @buf: buffer containing corruption + */ +void ubifs_scanned_corruption(const struct ubifs_info *c, int lnum, int offs, + void *buf) +{ + int len; + + ubifs_err(c, "corruption at LEB %d:%d", lnum, offs); + len = c->leb_size - offs; + if (len > 8192) + len = 8192; + ubifs_err(c, "first %d bytes from LEB %d:%d", len, lnum, offs); + print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_OFFSET, 32, 4, buf, len, 1); +} + +/** + * ubifs_scan - scan a logical eraseblock. + * @c: UBIFS file-system description object + * @lnum: logical eraseblock number + * @offs: offset to start at (usually zero) + * @sbuf: scan buffer (must be of @c->leb_size bytes in size) + * @quiet: print no messages + * + * This function scans LEB number @lnum and returns complete information about + * its contents. Returns the scanned information in case of success and, + * %-EUCLEAN if the LEB neads recovery, and other negative error codes in case + * of failure. + * + * If @quiet is non-zero, this function does not print large and scary + * error messages and flash dumps in case of errors. + */ +struct ubifs_scan_leb *ubifs_scan(const struct ubifs_info *c, int lnum, + int offs, void *sbuf, int quiet) +{ + void *buf = sbuf + offs; + int err, len = c->leb_size - offs; + struct ubifs_scan_leb *sleb; + + sleb = ubifs_start_scan(c, lnum, offs, sbuf); + if (IS_ERR(sleb)) + return sleb; + + while (len >= 8) { + struct ubifs_ch *ch = buf; + int node_len, ret; + + dbg_scan("look at LEB %d:%d (%d bytes left)", + lnum, offs, len); + + cond_resched(); + + ret = ubifs_scan_a_node(c, buf, len, lnum, offs, quiet); + if (ret > 0) { + /* Padding bytes or a valid padding node */ + offs += ret; + buf += ret; + len -= ret; + continue; + } + + if (ret == SCANNED_EMPTY_SPACE) + /* Empty space is checked later */ + break; + + switch (ret) { + case SCANNED_GARBAGE: + ubifs_err(c, "garbage"); + goto corrupted; + case SCANNED_A_NODE: + break; + case SCANNED_A_CORRUPT_NODE: + case SCANNED_A_BAD_PAD_NODE: + ubifs_err(c, "bad node"); + goto corrupted; + default: + ubifs_err(c, "unknown"); + err = -EINVAL; + goto error; + } + + err = ubifs_add_snod(c, sleb, buf, offs); + if (err) + goto error; + + node_len = ALIGN(le32_to_cpu(ch->len), 8); + offs += node_len; + buf += node_len; + len -= node_len; + } + + if (offs % c->min_io_size) { + if (!quiet) + ubifs_err(c, "empty space starts at non-aligned offset %d", + offs); + goto corrupted; + } + + ubifs_end_scan(c, sleb, lnum, offs); + + for (; len > 4; offs += 4, buf = buf + 4, len -= 4) + if (*(uint32_t *)buf != 0xffffffff) + break; + for (; len; offs++, buf++, len--) + if (*(uint8_t *)buf != 0xff) { + if (!quiet) + ubifs_err(c, "corrupt empty space at LEB %d:%d", + lnum, offs); + goto corrupted; + } + + return sleb; + +corrupted: + if (!quiet) { + ubifs_scanned_corruption(c, lnum, offs, buf); + ubifs_err(c, "LEB %d scanning failed", lnum); + } + err = -EUCLEAN; + ubifs_scan_destroy(sleb); + return ERR_PTR(err); + +error: + ubifs_err(c, "LEB %d scanning failed, error %d", lnum, err); + ubifs_scan_destroy(sleb); + return ERR_PTR(err); +} + +/** + * ubifs_scan_destroy - destroy LEB scanning information. + * @sleb: scanning information to free + */ +void ubifs_scan_destroy(struct ubifs_scan_leb *sleb) +{ + struct ubifs_scan_node *node; + struct list_head *head; + + head = &sleb->nodes; + while (!list_empty(head)) { + node = list_entry(head->next, struct ubifs_scan_node, list); + list_del(&node->list); + kfree(node); + } + kfree(sleb); +} diff --git a/fs/ubifs/shrinker.c b/fs/ubifs/shrinker.c new file mode 100644 index 000000000..d00a6f20a --- /dev/null +++ b/fs/ubifs/shrinker.c @@ -0,0 +1,319 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * + * Authors: Artem Bityutskiy (Битюцкий Артём) + * Adrian Hunter + */ + +/* + * This file implements UBIFS shrinker which evicts clean znodes from the TNC + * tree when Linux VM needs more RAM. + * + * We do not implement any LRU lists to find oldest znodes to free because it + * would add additional overhead to the file system fast paths. So the shrinker + * just walks the TNC tree when searching for znodes to free. + * + * If the root of a TNC sub-tree is clean and old enough, then the children are + * also clean and old enough. So the shrinker walks the TNC in level order and + * dumps entire sub-trees. + * + * The age of znodes is just the time-stamp when they were last looked at. + * The current shrinker first tries to evict old znodes, then young ones. + * + * Since the shrinker is global, it has to protect against races with FS + * un-mounts, which is done by the 'ubifs_infos_lock' and 'c->umount_mutex'. + */ + +#include "ubifs.h" + +/* List of all UBIFS file-system instances */ +LIST_HEAD(ubifs_infos); + +/* + * We number each shrinker run and record the number on the ubifs_info structure + * so that we can easily work out which ubifs_info structures have already been + * done by the current run. + */ +static unsigned int shrinker_run_no; + +/* Protects 'ubifs_infos' list */ +DEFINE_SPINLOCK(ubifs_infos_lock); + +/* Global clean znode counter (for all mounted UBIFS instances) */ +atomic_long_t ubifs_clean_zn_cnt; + +/** + * shrink_tnc - shrink TNC tree. + * @c: UBIFS file-system description object + * @nr: number of znodes to free + * @age: the age of znodes to free + * @contention: if any contention, this is set to %1 + * + * This function traverses TNC tree and frees clean znodes. It does not free + * clean znodes which younger then @age. Returns number of freed znodes. + */ +static int shrink_tnc(struct ubifs_info *c, int nr, int age, int *contention) +{ + int total_freed = 0; + struct ubifs_znode *znode, *zprev; + time64_t time = ktime_get_seconds(); + + ubifs_assert(c, mutex_is_locked(&c->umount_mutex)); + ubifs_assert(c, mutex_is_locked(&c->tnc_mutex)); + + if (!c->zroot.znode || atomic_long_read(&c->clean_zn_cnt) == 0) + return 0; + + /* + * Traverse the TNC tree in levelorder manner, so that it is possible + * to destroy large sub-trees. Indeed, if a znode is old, then all its + * children are older or of the same age. + * + * Note, we are holding 'c->tnc_mutex', so we do not have to lock the + * 'c->space_lock' when _reading_ 'c->clean_zn_cnt', because it is + * changed only when the 'c->tnc_mutex' is held. + */ + zprev = NULL; + znode = ubifs_tnc_levelorder_next(c, c->zroot.znode, NULL); + while (znode && total_freed < nr && + atomic_long_read(&c->clean_zn_cnt) > 0) { + int freed; + + /* + * If the znode is clean, but it is in the 'c->cnext' list, this + * means that this znode has just been written to flash as a + * part of commit and was marked clean. They will be removed + * from the list at end commit. We cannot change the list, + * because it is not protected by any mutex (design decision to + * make commit really independent and parallel to main I/O). So + * we just skip these znodes. + * + * Note, the 'clean_zn_cnt' counters are not updated until + * after the commit, so the UBIFS shrinker does not report + * the znodes which are in the 'c->cnext' list as freeable. + * + * Also note, if the root of a sub-tree is not in 'c->cnext', + * then the whole sub-tree is not in 'c->cnext' as well, so it + * is safe to dump whole sub-tree. + */ + + if (znode->cnext) { + /* + * Very soon these znodes will be removed from the list + * and become freeable. + */ + *contention = 1; + } else if (!ubifs_zn_dirty(znode) && + abs(time - znode->time) >= age) { + if (znode->parent) + znode->parent->zbranch[znode->iip].znode = NULL; + else + c->zroot.znode = NULL; + + freed = ubifs_destroy_tnc_subtree(c, znode); + atomic_long_sub(freed, &ubifs_clean_zn_cnt); + atomic_long_sub(freed, &c->clean_zn_cnt); + total_freed += freed; + znode = zprev; + } + + if (unlikely(!c->zroot.znode)) + break; + + zprev = znode; + znode = ubifs_tnc_levelorder_next(c, c->zroot.znode, znode); + cond_resched(); + } + + return total_freed; +} + +/** + * shrink_tnc_trees - shrink UBIFS TNC trees. + * @nr: number of znodes to free + * @age: the age of znodes to free + * @contention: if any contention, this is set to %1 + * + * This function walks the list of mounted UBIFS file-systems and frees clean + * znodes which are older than @age, until at least @nr znodes are freed. + * Returns the number of freed znodes. + */ +static int shrink_tnc_trees(int nr, int age, int *contention) +{ + struct ubifs_info *c; + struct list_head *p; + unsigned int run_no; + int freed = 0; + + spin_lock(&ubifs_infos_lock); + do { + run_no = ++shrinker_run_no; + } while (run_no == 0); + /* Iterate over all mounted UBIFS file-systems and try to shrink them */ + p = ubifs_infos.next; + while (p != &ubifs_infos) { + c = list_entry(p, struct ubifs_info, infos_list); + /* + * We move the ones we do to the end of the list, so we stop + * when we see one we have already done. + */ + if (c->shrinker_run_no == run_no) + break; + if (!mutex_trylock(&c->umount_mutex)) { + /* Some un-mount is in progress, try next FS */ + *contention = 1; + p = p->next; + continue; + } + /* + * We're holding 'c->umount_mutex', so the file-system won't go + * away. + */ + if (!mutex_trylock(&c->tnc_mutex)) { + mutex_unlock(&c->umount_mutex); + *contention = 1; + p = p->next; + continue; + } + spin_unlock(&ubifs_infos_lock); + /* + * OK, now we have TNC locked, the file-system cannot go away - + * it is safe to reap the cache. + */ + c->shrinker_run_no = run_no; + freed += shrink_tnc(c, nr, age, contention); + mutex_unlock(&c->tnc_mutex); + spin_lock(&ubifs_infos_lock); + /* Get the next list element before we move this one */ + p = p->next; + /* + * Move this one to the end of the list to provide some + * fairness. + */ + list_move_tail(&c->infos_list, &ubifs_infos); + mutex_unlock(&c->umount_mutex); + if (freed >= nr) + break; + } + spin_unlock(&ubifs_infos_lock); + return freed; +} + +/** + * kick_a_thread - kick a background thread to start commit. + * + * This function kicks a background thread to start background commit. Returns + * %-1 if a thread was kicked or there is another reason to assume the memory + * will soon be freed or become freeable. If there are no dirty znodes, returns + * %0. + */ +static int kick_a_thread(void) +{ + int i; + struct ubifs_info *c; + + /* + * Iterate over all mounted UBIFS file-systems and find out if there is + * already an ongoing commit operation there. If no, then iterate for + * the second time and initiate background commit. + */ + spin_lock(&ubifs_infos_lock); + for (i = 0; i < 2; i++) { + list_for_each_entry(c, &ubifs_infos, infos_list) { + long dirty_zn_cnt; + + if (!mutex_trylock(&c->umount_mutex)) { + /* + * Some un-mount is in progress, it will + * certainly free memory, so just return. + */ + spin_unlock(&ubifs_infos_lock); + return -1; + } + + dirty_zn_cnt = atomic_long_read(&c->dirty_zn_cnt); + + if (!dirty_zn_cnt || c->cmt_state == COMMIT_BROKEN || + c->ro_mount || c->ro_error) { + mutex_unlock(&c->umount_mutex); + continue; + } + + if (c->cmt_state != COMMIT_RESTING) { + spin_unlock(&ubifs_infos_lock); + mutex_unlock(&c->umount_mutex); + return -1; + } + + if (i == 1) { + list_move_tail(&c->infos_list, &ubifs_infos); + spin_unlock(&ubifs_infos_lock); + + ubifs_request_bg_commit(c); + mutex_unlock(&c->umount_mutex); + return -1; + } + mutex_unlock(&c->umount_mutex); + } + } + spin_unlock(&ubifs_infos_lock); + + return 0; +} + +unsigned long ubifs_shrink_count(struct shrinker *shrink, + struct shrink_control *sc) +{ + long clean_zn_cnt = atomic_long_read(&ubifs_clean_zn_cnt); + + /* + * Due to the way UBIFS updates the clean znode counter it may + * temporarily be negative. + */ + return clean_zn_cnt >= 0 ? clean_zn_cnt : 1; +} + +unsigned long ubifs_shrink_scan(struct shrinker *shrink, + struct shrink_control *sc) +{ + unsigned long nr = sc->nr_to_scan; + int contention = 0; + unsigned long freed; + long clean_zn_cnt = atomic_long_read(&ubifs_clean_zn_cnt); + + if (!clean_zn_cnt) { + /* + * No clean znodes, nothing to reap. All we can do in this case + * is to kick background threads to start commit, which will + * probably make clean znodes which, in turn, will be freeable. + * And we return -1 which means will make VM call us again + * later. + */ + dbg_tnc("no clean znodes, kick a thread"); + return kick_a_thread(); + } + + freed = shrink_tnc_trees(nr, OLD_ZNODE_AGE, &contention); + if (freed >= nr) + goto out; + + dbg_tnc("not enough old znodes, try to free young ones"); + freed += shrink_tnc_trees(nr - freed, YOUNG_ZNODE_AGE, &contention); + if (freed >= nr) + goto out; + + dbg_tnc("not enough young znodes, free all"); + freed += shrink_tnc_trees(nr - freed, 0, &contention); + + if (!freed && contention) { + dbg_tnc("freed nothing, but contention"); + return SHRINK_STOP; + } + +out: + dbg_tnc("%lu znodes were freed, requested %lu", freed, nr); + return freed; +} diff --git a/fs/ubifs/super.c b/fs/ubifs/super.c new file mode 100644 index 000000000..389727868 --- /dev/null +++ b/fs/ubifs/super.c @@ -0,0 +1,2515 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * + * Authors: Artem Bityutskiy (Битюцкий Артём) + * Adrian Hunter + */ + +/* + * This file implements UBIFS initialization and VFS superblock operations. Some + * initialization stuff which is rather large and complex is placed at + * corresponding subsystems, but most of it is here. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ubifs.h" + +static int ubifs_default_version_set(const char *val, const struct kernel_param *kp) +{ + int n = 0, ret; + + ret = kstrtoint(val, 10, &n); + if (ret != 0 || n < 4 || n > UBIFS_FORMAT_VERSION) + return -EINVAL; + return param_set_int(val, kp); +} + +static const struct kernel_param_ops ubifs_default_version_ops = { + .set = ubifs_default_version_set, + .get = param_get_int, +}; + +int ubifs_default_version = UBIFS_FORMAT_VERSION; +module_param_cb(default_version, &ubifs_default_version_ops, &ubifs_default_version, 0600); + +/* + * Maximum amount of memory we may 'kmalloc()' without worrying that we are + * allocating too much. + */ +#define UBIFS_KMALLOC_OK (128*1024) + +/* Slab cache for UBIFS inodes */ +static struct kmem_cache *ubifs_inode_slab; + +/* UBIFS TNC shrinker description */ +static struct shrinker *ubifs_shrinker_info; + +/** + * validate_inode - validate inode. + * @c: UBIFS file-system description object + * @inode: the inode to validate + * + * This is a helper function for 'ubifs_iget()' which validates various fields + * of a newly built inode to make sure they contain sane values and prevent + * possible vulnerabilities. Returns zero if the inode is all right and + * a non-zero error code if not. + */ +static int validate_inode(struct ubifs_info *c, const struct inode *inode) +{ + int err; + const struct ubifs_inode *ui = ubifs_inode(inode); + + if (inode->i_size > c->max_inode_sz) { + ubifs_err(c, "inode is too large (%lld)", + (long long)inode->i_size); + return 1; + } + + if (ui->compr_type >= UBIFS_COMPR_TYPES_CNT) { + ubifs_err(c, "unknown compression type %d", ui->compr_type); + return 2; + } + + if (ui->xattr_names + ui->xattr_cnt > XATTR_LIST_MAX) + return 3; + + if (ui->data_len < 0 || ui->data_len > UBIFS_MAX_INO_DATA) + return 4; + + if (ui->xattr && !S_ISREG(inode->i_mode)) + return 5; + + if (!ubifs_compr_present(c, ui->compr_type)) { + ubifs_warn(c, "inode %llu uses '%s' compression, but it was not compiled in", + inode->i_ino, ubifs_compr_name(c, ui->compr_type)); + } + + err = dbg_check_dir(c, inode); + return err; +} + +struct inode *ubifs_iget(struct super_block *sb, unsigned long inum) +{ + int err; + union ubifs_key key; + struct ubifs_ino_node *ino; + struct ubifs_info *c = sb->s_fs_info; + struct inode *inode; + struct ubifs_inode *ui; + + dbg_gen("inode %lu", inum); + + inode = iget_locked(sb, inum); + if (!inode) + return ERR_PTR(-ENOMEM); + if (!(inode_state_read_once(inode) & I_NEW)) + return inode; + ui = ubifs_inode(inode); + + ino = kmalloc(UBIFS_MAX_INO_NODE_SZ, GFP_NOFS); + if (!ino) { + err = -ENOMEM; + goto out; + } + + ino_key_init(c, &key, inode->i_ino); + + err = ubifs_tnc_lookup(c, &key, ino); + if (err) + goto out_ino; + + inode->i_flags |= S_NOCMTIME; + + if (!IS_ENABLED(CONFIG_UBIFS_ATIME_SUPPORT)) + inode->i_flags |= S_NOATIME; + + set_nlink(inode, le32_to_cpu(ino->nlink)); + i_uid_write(inode, le32_to_cpu(ino->uid)); + i_gid_write(inode, le32_to_cpu(ino->gid)); + inode_set_atime(inode, (int64_t)le64_to_cpu(ino->atime_sec), + le32_to_cpu(ino->atime_nsec)); + inode_set_mtime(inode, (int64_t)le64_to_cpu(ino->mtime_sec), + le32_to_cpu(ino->mtime_nsec)); + inode_set_ctime(inode, (int64_t)le64_to_cpu(ino->ctime_sec), + le32_to_cpu(ino->ctime_nsec)); + inode->i_mode = le32_to_cpu(ino->mode); + inode->i_size = le64_to_cpu(ino->size); + + ui->data_len = le32_to_cpu(ino->data_len); + ui->flags = le32_to_cpu(ino->flags); + ui->compr_type = le16_to_cpu(ino->compr_type); + ui->creat_sqnum = le64_to_cpu(ino->creat_sqnum); + ui->xattr_cnt = le32_to_cpu(ino->xattr_cnt); + ui->xattr_size = le32_to_cpu(ino->xattr_size); + ui->xattr_names = le32_to_cpu(ino->xattr_names); + ui->synced_i_size = ui->ui_size = inode->i_size; + + ui->xattr = (ui->flags & UBIFS_XATTR_FL) ? 1 : 0; + + err = validate_inode(c, inode); + if (err) + goto out_invalid; + + switch (inode->i_mode & S_IFMT) { + case S_IFREG: + inode->i_mapping->a_ops = &ubifs_file_address_operations; + inode->i_op = &ubifs_file_inode_operations; + inode->i_fop = &ubifs_file_operations; + if (ui->xattr) { + ui->data = kmalloc(ui->data_len + 1, GFP_NOFS); + if (!ui->data) { + err = -ENOMEM; + goto out_ino; + } + memcpy(ui->data, ino->data, ui->data_len); + ((char *)ui->data)[ui->data_len] = '\0'; + } else if (ui->data_len != 0) { + err = 10; + goto out_invalid; + } + break; + case S_IFDIR: + inode->i_op = &ubifs_dir_inode_operations; + inode->i_fop = &ubifs_dir_operations; + if (ui->data_len != 0) { + err = 11; + goto out_invalid; + } + break; + case S_IFLNK: + inode->i_op = &ubifs_symlink_inode_operations; + if (ui->data_len <= 0 || ui->data_len > UBIFS_MAX_INO_DATA) { + err = 12; + goto out_invalid; + } + ui->data = kmalloc(ui->data_len + 1, GFP_NOFS); + if (!ui->data) { + err = -ENOMEM; + goto out_ino; + } + memcpy(ui->data, ino->data, ui->data_len); + ((char *)ui->data)[ui->data_len] = '\0'; + break; + case S_IFBLK: + case S_IFCHR: + { + dev_t rdev; + union ubifs_dev_desc *dev; + + ui->data = kmalloc_obj(union ubifs_dev_desc, GFP_NOFS); + if (!ui->data) { + err = -ENOMEM; + goto out_ino; + } + + dev = (union ubifs_dev_desc *)ino->data; + if (ui->data_len == sizeof(dev->new)) + rdev = new_decode_dev(le32_to_cpu(dev->new)); + else if (ui->data_len == sizeof(dev->huge)) + rdev = huge_decode_dev(le64_to_cpu(dev->huge)); + else { + err = 13; + goto out_invalid; + } + memcpy(ui->data, ino->data, ui->data_len); + inode->i_op = &ubifs_file_inode_operations; + init_special_inode(inode, inode->i_mode, rdev); + break; + } + case S_IFSOCK: + case S_IFIFO: + inode->i_op = &ubifs_file_inode_operations; + init_special_inode(inode, inode->i_mode, 0); + if (ui->data_len != 0) { + err = 14; + goto out_invalid; + } + break; + default: + err = 15; + goto out_invalid; + } + + kfree(ino); + ubifs_set_inode_flags(inode); + unlock_new_inode(inode); + return inode; + +out_invalid: + ubifs_err(c, "inode %llu validation failed, error %d", inode->i_ino, err); + ubifs_dump_node(c, ino, UBIFS_MAX_INO_NODE_SZ); + ubifs_dump_inode(c, inode); + err = -EINVAL; +out_ino: + kfree(ino); +out: + ubifs_err(c, "failed to read inode %llu, error %d", inode->i_ino, err); + iget_failed(inode); + return ERR_PTR(err); +} + +static struct inode *ubifs_alloc_inode(struct super_block *sb) +{ + struct ubifs_inode *ui; + + ui = alloc_inode_sb(sb, ubifs_inode_slab, GFP_NOFS); + if (!ui) + return NULL; + + memset((void *)ui + sizeof(struct inode), 0, + sizeof(struct ubifs_inode) - sizeof(struct inode)); + mutex_init(&ui->ui_mutex); + init_rwsem(&ui->xattr_sem); + spin_lock_init(&ui->ui_lock); + return &ui->vfs_inode; +}; + +static void ubifs_free_inode(struct inode *inode) +{ + struct ubifs_inode *ui = ubifs_inode(inode); + + kfree(ui->data); + fscrypt_free_inode(inode); + + kmem_cache_free(ubifs_inode_slab, ui); +} + +/* + * Note, Linux write-back code calls this without 'i_mutex'. + */ +static int ubifs_write_inode(struct inode *inode, struct writeback_control *wbc) +{ + int err = 0; + struct ubifs_info *c = inode->i_sb->s_fs_info; + struct ubifs_inode *ui = ubifs_inode(inode); + + ubifs_assert(c, !ui->xattr); + if (is_bad_inode(inode)) + return 0; + + mutex_lock(&ui->ui_mutex); + /* + * Due to races between write-back forced by budgeting + * (see 'sync_some_inodes()') and background write-back, the inode may + * have already been synchronized, do not do this again. This might + * also happen if it was synchronized in an VFS operation, e.g. + * 'ubifs_link()'. + */ + if (!ui->dirty) { + mutex_unlock(&ui->ui_mutex); + return 0; + } + + /* + * As an optimization, do not write orphan inodes to the media just + * because this is not needed. + */ + dbg_gen("inode %llu, mode %#x, nlink %u", + inode->i_ino, (int)inode->i_mode, inode->i_nlink); + if (inode->i_nlink) { + err = ubifs_jnl_write_inode(c, inode); + if (err) + ubifs_err(c, "can't write inode %llu, error %d", + inode->i_ino, err); + else + err = dbg_check_inode_size(c, inode, ui->ui_size); + } + + ui->dirty = 0; + mutex_unlock(&ui->ui_mutex); + ubifs_release_dirty_inode_budget(c, ui); + return err; +} + +static int ubifs_drop_inode(struct inode *inode) +{ + int drop = inode_generic_drop(inode); + + if (!drop) + drop = fscrypt_drop_inode(inode); + + return drop; +} + +static void ubifs_evict_inode(struct inode *inode) +{ + int err; + struct ubifs_info *c = inode->i_sb->s_fs_info; + struct ubifs_inode *ui = ubifs_inode(inode); + + if (ui->xattr) + /* + * Extended attribute inode deletions are fully handled in + * 'ubifs_removexattr()'. These inodes are special and have + * limited usage, so there is nothing to do here. + */ + goto out; + + dbg_gen("inode %llu, mode %#x", inode->i_ino, (int)inode->i_mode); + ubifs_assert(c, !icount_read_once(inode)); + + truncate_inode_pages_final(&inode->i_data); + + if (inode->i_nlink) + goto done; + + if (is_bad_inode(inode)) + goto out; + + ui->ui_size = inode->i_size = 0; + err = ubifs_jnl_delete_inode(c, inode); + if (err) + /* + * Worst case we have a lost orphan inode wasting space, so a + * simple error message is OK here. + */ + ubifs_err(c, "can't delete inode %llu, error %d", + inode->i_ino, err); + +out: + if (ui->dirty) + ubifs_release_dirty_inode_budget(c, ui); + else { + /* We've deleted something - clean the "no space" flags */ + c->bi.nospace = c->bi.nospace_rp = 0; + smp_wmb(); + } +done: + clear_inode(inode); + fscrypt_put_encryption_info(inode); +} + +static void ubifs_dirty_inode(struct inode *inode, int flags) +{ + struct ubifs_info *c = inode->i_sb->s_fs_info; + struct ubifs_inode *ui = ubifs_inode(inode); + + ubifs_assert(c, mutex_is_locked(&ui->ui_mutex)); + if (!ui->dirty) { + ui->dirty = 1; + dbg_gen("inode %llu", inode->i_ino); + } +} + +static int ubifs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct ubifs_info *c = dentry->d_sb->s_fs_info; + unsigned long long free; + __le32 *uuid = (__le32 *)c->uuid; + + free = ubifs_get_free_space(c); + dbg_gen("free space %lld bytes (%lld blocks)", + free, free >> UBIFS_BLOCK_SHIFT); + + buf->f_type = UBIFS_SUPER_MAGIC; + buf->f_bsize = UBIFS_BLOCK_SIZE; + buf->f_blocks = c->block_cnt; + buf->f_bfree = free >> UBIFS_BLOCK_SHIFT; + if (free > c->report_rp_size) + buf->f_bavail = (free - c->report_rp_size) >> UBIFS_BLOCK_SHIFT; + else + buf->f_bavail = 0; + buf->f_files = 0; + buf->f_ffree = 0; + buf->f_namelen = UBIFS_MAX_NLEN; + buf->f_fsid.val[0] = le32_to_cpu(uuid[0]) ^ le32_to_cpu(uuid[2]); + buf->f_fsid.val[1] = le32_to_cpu(uuid[1]) ^ le32_to_cpu(uuid[3]); + ubifs_assert(c, buf->f_bfree <= c->block_cnt); + return 0; +} + +static int ubifs_show_options(struct seq_file *s, struct dentry *root) +{ + struct ubifs_info *c = root->d_sb->s_fs_info; + + if (c->mount_opts.unmount_mode == 2) + seq_puts(s, ",fast_unmount"); + else if (c->mount_opts.unmount_mode == 1) + seq_puts(s, ",norm_unmount"); + + if (c->mount_opts.bulk_read == 2) + seq_puts(s, ",bulk_read"); + else if (c->mount_opts.bulk_read == 1) + seq_puts(s, ",no_bulk_read"); + + if (c->mount_opts.chk_data_crc == 2) + seq_puts(s, ",chk_data_crc"); + else if (c->mount_opts.chk_data_crc == 1) + seq_puts(s, ",no_chk_data_crc"); + + if (c->mount_opts.override_compr) { + seq_printf(s, ",compr=%s", + ubifs_compr_name(c, c->mount_opts.compr_type)); + } + + seq_printf(s, ",assert=%s", ubifs_assert_action_name(c)); + seq_printf(s, ",ubi=%d,vol=%d", c->vi.ubi_num, c->vi.vol_id); + + return 0; +} + +static int ubifs_sync_fs(struct super_block *sb, int wait) +{ + int i, err; + struct ubifs_info *c = sb->s_fs_info; + + /* + * Zero @wait is just an advisory thing to help the file system shove + * lots of data into the queues, and there will be the second + * '->sync_fs()' call, with non-zero @wait. + */ + if (!wait) + return 0; + + /* + * Synchronize write buffers, because 'ubifs_run_commit()' does not + * do this if it waits for an already running commit. + */ + for (i = 0; i < c->jhead_cnt; i++) { + err = ubifs_wbuf_sync(&c->jheads[i].wbuf); + if (err) + return err; + } + + /* + * Strictly speaking, it is not necessary to commit the journal here, + * synchronizing write-buffers would be enough. But committing makes + * UBIFS free space predictions much more accurate, so we want to let + * the user be able to get more accurate results of 'statfs()' after + * they synchronize the file system. + */ + err = ubifs_run_commit(c); + if (err) + return err; + + return ubi_sync(c->vi.ubi_num); +} + +/** + * init_constants_early - initialize UBIFS constants. + * @c: UBIFS file-system description object + * + * This function initialize UBIFS constants which do not need the superblock to + * be read. It also checks that the UBI volume satisfies basic UBIFS + * requirements. Returns zero in case of success and a negative error code in + * case of failure. + */ +static int init_constants_early(struct ubifs_info *c) +{ + if (c->vi.corrupted) { + ubifs_warn(c, "UBI volume is corrupted - read-only mode"); + c->ro_media = 1; + } + + if (c->di.ro_mode) { + ubifs_msg(c, "read-only UBI device"); + c->ro_media = 1; + } + + if (c->vi.vol_type == UBI_STATIC_VOLUME) { + ubifs_msg(c, "static UBI volume - read-only mode"); + c->ro_media = 1; + } + + c->leb_cnt = c->vi.size; + c->leb_size = c->vi.usable_leb_size; + c->leb_start = c->di.leb_start; + c->half_leb_size = c->leb_size / 2; + c->min_io_size = c->di.min_io_size; + c->min_io_shift = fls(c->min_io_size) - 1; + c->max_write_size = c->di.max_write_size; + c->max_write_shift = fls(c->max_write_size) - 1; + + if (c->leb_size < UBIFS_MIN_LEB_SZ) { + ubifs_errc(c, "too small LEBs (%d bytes), min. is %d bytes", + c->leb_size, UBIFS_MIN_LEB_SZ); + return -EINVAL; + } + + if (c->leb_cnt < UBIFS_MIN_LEB_CNT) { + ubifs_errc(c, "too few LEBs (%d), min. is %d", + c->leb_cnt, UBIFS_MIN_LEB_CNT); + return -EINVAL; + } + + if (!is_power_of_2(c->min_io_size)) { + ubifs_errc(c, "bad min. I/O size %d", c->min_io_size); + return -EINVAL; + } + + /* + * Maximum write size has to be greater or equivalent to min. I/O + * size, and be multiple of min. I/O size. + */ + if (c->max_write_size < c->min_io_size || + c->max_write_size % c->min_io_size || + !is_power_of_2(c->max_write_size)) { + ubifs_errc(c, "bad write buffer size %d for %d min. I/O unit", + c->max_write_size, c->min_io_size); + return -EINVAL; + } + + /* + * UBIFS aligns all node to 8-byte boundary, so to make function in + * io.c simpler, assume minimum I/O unit size to be 8 bytes if it is + * less than 8. + */ + if (c->min_io_size < 8) { + c->min_io_size = 8; + c->min_io_shift = 3; + if (c->max_write_size < c->min_io_size) { + c->max_write_size = c->min_io_size; + c->max_write_shift = c->min_io_shift; + } + } + + c->ref_node_alsz = ALIGN(UBIFS_REF_NODE_SZ, c->min_io_size); + c->mst_node_alsz = ALIGN(UBIFS_MST_NODE_SZ, c->min_io_size); + + /* + * Initialize node length ranges which are mostly needed for node + * length validation. + */ + c->ranges[UBIFS_PAD_NODE].len = UBIFS_PAD_NODE_SZ; + c->ranges[UBIFS_SB_NODE].len = UBIFS_SB_NODE_SZ; + c->ranges[UBIFS_MST_NODE].len = UBIFS_MST_NODE_SZ; + c->ranges[UBIFS_REF_NODE].len = UBIFS_REF_NODE_SZ; + c->ranges[UBIFS_TRUN_NODE].len = UBIFS_TRUN_NODE_SZ; + c->ranges[UBIFS_CS_NODE].len = UBIFS_CS_NODE_SZ; + c->ranges[UBIFS_AUTH_NODE].min_len = UBIFS_AUTH_NODE_SZ; + c->ranges[UBIFS_AUTH_NODE].max_len = UBIFS_AUTH_NODE_SZ + + UBIFS_MAX_HMAC_LEN; + c->ranges[UBIFS_SIG_NODE].min_len = UBIFS_SIG_NODE_SZ; + c->ranges[UBIFS_SIG_NODE].max_len = c->leb_size - UBIFS_SB_NODE_SZ; + + c->ranges[UBIFS_INO_NODE].min_len = UBIFS_INO_NODE_SZ; + c->ranges[UBIFS_INO_NODE].max_len = UBIFS_MAX_INO_NODE_SZ; + c->ranges[UBIFS_ORPH_NODE].min_len = + UBIFS_ORPH_NODE_SZ + sizeof(__le64); + c->ranges[UBIFS_ORPH_NODE].max_len = c->leb_size; + c->ranges[UBIFS_DENT_NODE].min_len = UBIFS_DENT_NODE_SZ; + c->ranges[UBIFS_DENT_NODE].max_len = UBIFS_MAX_DENT_NODE_SZ; + c->ranges[UBIFS_XENT_NODE].min_len = UBIFS_XENT_NODE_SZ; + c->ranges[UBIFS_XENT_NODE].max_len = UBIFS_MAX_XENT_NODE_SZ; + c->ranges[UBIFS_DATA_NODE].min_len = UBIFS_DATA_NODE_SZ; + c->ranges[UBIFS_DATA_NODE].max_len = UBIFS_MAX_DATA_NODE_SZ; + /* + * Minimum indexing node size is amended later when superblock is + * read and the key length is known. + */ + c->ranges[UBIFS_IDX_NODE].min_len = UBIFS_IDX_NODE_SZ + UBIFS_BRANCH_SZ; + /* + * Maximum indexing node size is amended later when superblock is + * read and the fanout is known. + */ + c->ranges[UBIFS_IDX_NODE].max_len = INT_MAX; + + /* + * Initialize dead and dark LEB space watermarks. See gc.c for comments + * about these values. + */ + c->dead_wm = ALIGN(MIN_WRITE_SZ, c->min_io_size); + c->dark_wm = ALIGN(UBIFS_MAX_NODE_SZ, c->min_io_size); + + /* + * Calculate how many bytes would be wasted at the end of LEB if it was + * fully filled with data nodes of maximum size. This is used in + * calculations when reporting free space. + */ + c->leb_overhead = c->leb_size % UBIFS_MAX_DATA_NODE_SZ; + + /* Buffer size for bulk-reads */ + c->max_bu_buf_len = UBIFS_MAX_BULK_READ * UBIFS_MAX_DATA_NODE_SZ; + if (c->max_bu_buf_len > c->leb_size) + c->max_bu_buf_len = c->leb_size; + + /* Log is ready, preserve one LEB for commits. */ + c->min_log_bytes = c->leb_size; + + return 0; +} + +/** + * bud_wbuf_callback - bud LEB write-buffer synchronization call-back. + * @c: UBIFS file-system description object + * @lnum: LEB the write-buffer was synchronized to + * @free: how many free bytes left in this LEB + * @pad: how many bytes were padded + * + * This is a callback function which is called by the I/O unit when the + * write-buffer is synchronized. We need this to correctly maintain space + * accounting in bud logical eraseblocks. This function returns zero in case of + * success and a negative error code in case of failure. + * + * This function actually belongs to the journal, but we keep it here because + * we want to keep it static. + */ +static int bud_wbuf_callback(struct ubifs_info *c, int lnum, int free, int pad) +{ + return ubifs_update_one_lp(c, lnum, free, pad, 0, 0); +} + +/* + * init_constants_sb - initialize UBIFS constants. + * @c: UBIFS file-system description object + * + * This is a helper function which initializes various UBIFS constants after + * the superblock has been read. It also checks various UBIFS parameters and + * makes sure they are all right. Returns zero in case of success and a + * negative error code in case of failure. + */ +static int init_constants_sb(struct ubifs_info *c) +{ + int tmp, err; + long long tmp64; + + c->main_bytes = (long long)c->main_lebs * c->leb_size; + c->max_znode_sz = sizeof(struct ubifs_znode) + + c->fanout * sizeof(struct ubifs_zbranch); + + tmp = ubifs_idx_node_sz(c, 1); + c->ranges[UBIFS_IDX_NODE].min_len = tmp; + c->min_idx_node_sz = ALIGN(tmp, 8); + + tmp = ubifs_idx_node_sz(c, c->fanout); + c->ranges[UBIFS_IDX_NODE].max_len = tmp; + c->max_idx_node_sz = ALIGN(tmp, 8); + + /* Make sure LEB size is large enough to fit full commit */ + tmp = UBIFS_CS_NODE_SZ + UBIFS_REF_NODE_SZ * c->jhead_cnt; + tmp = ALIGN(tmp, c->min_io_size); + if (tmp > c->leb_size) { + ubifs_err(c, "too small LEB size %d, at least %d needed", + c->leb_size, tmp); + return -EINVAL; + } + + /* + * Make sure that the log is large enough to fit reference nodes for + * all buds plus one reserved LEB. + */ + tmp64 = c->max_bud_bytes + c->leb_size - 1; + c->max_bud_cnt = div_u64(tmp64, c->leb_size); + tmp = (c->ref_node_alsz * c->max_bud_cnt + c->leb_size - 1); + tmp /= c->leb_size; + tmp += 1; + if (c->log_lebs < tmp) { + ubifs_err(c, "too small log %d LEBs, required min. %d LEBs", + c->log_lebs, tmp); + return -EINVAL; + } + + /* + * When budgeting we assume worst-case scenarios when the pages are not + * be compressed and direntries are of the maximum size. + * + * Note, data, which may be stored in inodes is budgeted separately, so + * it is not included into 'c->bi.inode_budget'. + */ + c->bi.page_budget = UBIFS_MAX_DATA_NODE_SZ * UBIFS_BLOCKS_PER_PAGE; + c->bi.inode_budget = UBIFS_INO_NODE_SZ; + c->bi.dent_budget = UBIFS_MAX_DENT_NODE_SZ; + + /* + * When the amount of flash space used by buds becomes + * 'c->max_bud_bytes', UBIFS just blocks all writers and starts commit. + * The writers are unblocked when the commit is finished. To avoid + * writers to be blocked UBIFS initiates background commit in advance, + * when number of bud bytes becomes above the limit defined below. + */ + c->bg_bud_bytes = (c->max_bud_bytes * 13) >> 4; + + /* + * Ensure minimum journal size. All the bytes in the journal heads are + * considered to be used, when calculating the current journal usage. + * Consequently, if the journal is too small, UBIFS will treat it as + * always full. + */ + tmp64 = (long long)(c->jhead_cnt + 1) * c->leb_size + 1; + if (c->bg_bud_bytes < tmp64) + c->bg_bud_bytes = tmp64; + if (c->max_bud_bytes < tmp64 + c->leb_size) + c->max_bud_bytes = tmp64 + c->leb_size; + + err = ubifs_calc_lpt_geom(c); + if (err) + return err; + + /* Initialize effective LEB size used in budgeting calculations */ + c->idx_leb_size = c->leb_size - c->max_idx_node_sz; + return 0; +} + +/* + * init_constants_master - initialize UBIFS constants. + * @c: UBIFS file-system description object + * + * This is a helper function which initializes various UBIFS constants after + * the master node has been read. It also checks various UBIFS parameters and + * makes sure they are all right. + */ +static void init_constants_master(struct ubifs_info *c) +{ + long long tmp64; + + c->bi.min_idx_lebs = ubifs_calc_min_idx_lebs(c); + c->report_rp_size = ubifs_reported_space(c, c->rp_size); + + /* + * Calculate total amount of FS blocks. This number is not used + * internally because it does not make much sense for UBIFS, but it is + * necessary to report something for the 'statfs()' call. + * + * Subtract the LEB reserved for GC, the LEB which is reserved for + * deletions, minimum LEBs for the index, the LEBs which are reserved + * for each journal head. + */ + tmp64 = c->main_lebs - 1 - 1 - MIN_INDEX_LEBS - c->jhead_cnt; + tmp64 *= (long long)c->leb_size - c->leb_overhead; + tmp64 = ubifs_reported_space(c, tmp64); + c->block_cnt = tmp64 >> UBIFS_BLOCK_SHIFT; +} + +/** + * take_gc_lnum - reserve GC LEB. + * @c: UBIFS file-system description object + * + * This function ensures that the LEB reserved for garbage collection is marked + * as "taken" in lprops. We also have to set free space to LEB size and dirty + * space to zero, because lprops may contain out-of-date information if the + * file-system was un-mounted before it has been committed. This function + * returns zero in case of success and a negative error code in case of + * failure. + */ +static int take_gc_lnum(struct ubifs_info *c) +{ + int err; + + if (c->gc_lnum == -1) { + ubifs_err(c, "no LEB for GC"); + return -EINVAL; + } + + /* And we have to tell lprops that this LEB is taken */ + err = ubifs_change_one_lp(c, c->gc_lnum, c->leb_size, 0, + LPROPS_TAKEN, 0, 0); + return err; +} + +/** + * alloc_wbufs - allocate write-buffers. + * @c: UBIFS file-system description object + * + * This helper function allocates and initializes UBIFS write-buffers. Returns + * zero in case of success and %-ENOMEM in case of failure. + */ +static int alloc_wbufs(struct ubifs_info *c) +{ + int i, err; + + c->jheads = kzalloc_objs(struct ubifs_jhead, c->jhead_cnt); + if (!c->jheads) + return -ENOMEM; + + /* Initialize journal heads */ + for (i = 0; i < c->jhead_cnt; i++) { + INIT_LIST_HEAD(&c->jheads[i].buds_list); + err = ubifs_wbuf_init(c, &c->jheads[i].wbuf); + if (err) + goto out_wbuf; + + c->jheads[i].wbuf.sync_callback = &bud_wbuf_callback; + c->jheads[i].wbuf.jhead = i; + c->jheads[i].grouped = 1; + c->jheads[i].log_hash = ubifs_hash_get_desc(c); + if (IS_ERR(c->jheads[i].log_hash)) { + err = PTR_ERR(c->jheads[i].log_hash); + goto out_log_hash; + } + } + + /* + * Garbage Collector head does not need to be synchronized by timer. + * Also GC head nodes are not grouped. + */ + c->jheads[GCHD].wbuf.no_timer = 1; + c->jheads[GCHD].grouped = 0; + + return 0; + +out_log_hash: + kfree(c->jheads[i].wbuf.buf); + kfree(c->jheads[i].wbuf.inodes); + +out_wbuf: + while (i--) { + kfree(c->jheads[i].wbuf.buf); + kfree(c->jheads[i].wbuf.inodes); + kfree(c->jheads[i].log_hash); + } + kfree(c->jheads); + c->jheads = NULL; + + return err; +} + +/** + * free_wbufs - free write-buffers. + * @c: UBIFS file-system description object + */ +static void free_wbufs(struct ubifs_info *c) +{ + int i; + + if (c->jheads) { + for (i = 0; i < c->jhead_cnt; i++) { + kfree(c->jheads[i].wbuf.buf); + kfree(c->jheads[i].wbuf.inodes); + kfree(c->jheads[i].log_hash); + } + kfree(c->jheads); + c->jheads = NULL; + } +} + +/** + * free_orphans - free orphans. + * @c: UBIFS file-system description object + */ +static void free_orphans(struct ubifs_info *c) +{ + struct ubifs_orphan *orph; + + while (c->orph_dnext) { + orph = c->orph_dnext; + c->orph_dnext = orph->dnext; + list_del(&orph->list); + kfree(orph); + } + + while (!list_empty(&c->orph_list)) { + orph = list_entry(c->orph_list.next, struct ubifs_orphan, list); + list_del(&orph->list); + kfree(orph); + ubifs_err(c, "orphan list not empty at unmount"); + } + + vfree(c->orph_buf); + c->orph_buf = NULL; +} + +/** + * free_buds - free per-bud objects. + * @c: UBIFS file-system description object + */ +static void free_buds(struct ubifs_info *c) +{ + struct ubifs_bud *bud, *n; + + rbtree_postorder_for_each_entry_safe(bud, n, &c->buds, rb) { + kfree(bud->log_hash); + kfree(bud); + } +} + +/** + * check_volume_empty - check if the UBI volume is empty. + * @c: UBIFS file-system description object + * + * This function checks if the UBIFS volume is empty by looking if its LEBs are + * mapped or not. The result of checking is stored in the @c->empty variable. + * Returns zero in case of success and a negative error code in case of + * failure. + */ +static int check_volume_empty(struct ubifs_info *c) +{ + int lnum, err; + + c->empty = 1; + for (lnum = 0; lnum < c->leb_cnt; lnum++) { + err = ubifs_is_mapped(c, lnum); + if (unlikely(err < 0)) + return err; + if (err == 1) { + c->empty = 0; + break; + } + + cond_resched(); + } + + return 0; +} + +/* + * UBIFS mount options. + * + * Opt_fast_unmount: do not run a journal commit before un-mounting + * Opt_norm_unmount: run a journal commit before un-mounting + * Opt_bulk_read: enable bulk-reads + * Opt_no_bulk_read: disable bulk-reads + * Opt_chk_data_crc: check CRCs when reading data nodes + * Opt_no_chk_data_crc: do not check CRCs when reading data nodes + * Opt_override_compr: override default compressor + * Opt_assert: set ubifs_assert() action + * Opt_auth_key: The key name used for authentication + * Opt_auth_hash_name: The hash type used for authentication + * Opt_err: just end of array marker + */ +enum { + Opt_fast_unmount, + Opt_norm_unmount, + Opt_bulk_read, + Opt_no_bulk_read, + Opt_chk_data_crc, + Opt_no_chk_data_crc, + Opt_override_compr, + Opt_assert, + Opt_auth_key, + Opt_auth_hash_name, + Opt_ignore, +}; + +static const struct constant_table ubifs_param_compr[] = { + { "none", UBIFS_COMPR_NONE }, + { "lzo", UBIFS_COMPR_LZO }, + { "zlib", UBIFS_COMPR_ZLIB }, + { "zstd", UBIFS_COMPR_ZSTD }, + {} +}; + +static const struct constant_table ubifs_param_assert[] = { + { "report", ASSACT_REPORT }, + { "read-only", ASSACT_RO }, + { "panic", ASSACT_PANIC }, + {} +}; + +static const struct fs_parameter_spec ubifs_fs_param_spec[] = { + fsparam_flag ("fast_unmount", Opt_fast_unmount), + fsparam_flag ("norm_unmount", Opt_norm_unmount), + fsparam_flag ("bulk_read", Opt_bulk_read), + fsparam_flag ("no_bulk_read", Opt_no_bulk_read), + fsparam_flag ("chk_data_crc", Opt_chk_data_crc), + fsparam_flag ("no_chk_data_crc", Opt_no_chk_data_crc), + fsparam_enum ("compr", Opt_override_compr, ubifs_param_compr), + fsparam_enum ("assert", Opt_assert, ubifs_param_assert), + fsparam_string ("auth_key", Opt_auth_key), + fsparam_string ("auth_hash_name", Opt_auth_hash_name), + fsparam_string ("ubi", Opt_ignore), + fsparam_string ("vol", Opt_ignore), + {} +}; + +struct ubifs_fs_context { + struct ubifs_mount_opts mount_opts; + char *auth_key_name; + char *auth_hash_name; + unsigned int no_chk_data_crc:1; + unsigned int bulk_read:1; + unsigned int default_compr:2; + unsigned int assert_action:2; +}; + +/** + * ubifs_parse_param - parse a parameter. + * @fc: the filesystem context + * @param: the parameter to parse + * + * This function parses UBIFS mount options and returns zero in case success + * and a negative error code in case of failure. + */ +static int ubifs_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct ubifs_fs_context *ctx = fc->fs_private; + struct fs_parse_result result; + bool is_remount = (fc->purpose & FS_CONTEXT_FOR_RECONFIGURE); + int opt; + + opt = fs_parse(fc, ubifs_fs_param_spec, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + /* + * %Opt_fast_unmount and %Opt_norm_unmount options are ignored. + * We accept them in order to be backward-compatible. But this + * should be removed at some point. + */ + case Opt_fast_unmount: + ctx->mount_opts.unmount_mode = 2; + break; + case Opt_norm_unmount: + ctx->mount_opts.unmount_mode = 1; + break; + case Opt_bulk_read: + ctx->mount_opts.bulk_read = 2; + ctx->bulk_read = 1; + break; + case Opt_no_bulk_read: + ctx->mount_opts.bulk_read = 1; + ctx->bulk_read = 0; + break; + case Opt_chk_data_crc: + ctx->mount_opts.chk_data_crc = 2; + ctx->no_chk_data_crc = 0; + break; + case Opt_no_chk_data_crc: + ctx->mount_opts.chk_data_crc = 1; + ctx->no_chk_data_crc = 1; + break; + case Opt_override_compr: + ctx->mount_opts.compr_type = result.uint_32; + ctx->mount_opts.override_compr = 1; + ctx->default_compr = ctx->mount_opts.compr_type; + break; + case Opt_assert: + ctx->assert_action = result.uint_32; + break; + case Opt_auth_key: + if (!is_remount) { + kfree(ctx->auth_key_name); + ctx->auth_key_name = param->string; + param->string = NULL; + } + break; + case Opt_auth_hash_name: + if (!is_remount) { + kfree(ctx->auth_hash_name); + ctx->auth_hash_name = param->string; + param->string = NULL; + } + break; + case Opt_ignore: + break; + } + + return 0; +} + +/* + * ubifs_release_options - release mount parameters which have been dumped. + * @c: UBIFS file-system description object + */ +static void ubifs_release_options(struct ubifs_info *c) +{ + kfree(c->auth_key_name); + c->auth_key_name = NULL; + kfree(c->auth_hash_name); + c->auth_hash_name = NULL; +} + +/** + * destroy_journal - destroy journal data structures. + * @c: UBIFS file-system description object + * + * This function destroys journal data structures including those that may have + * been created by recovery functions. + */ +static void destroy_journal(struct ubifs_info *c) +{ + while (!list_empty(&c->unclean_leb_list)) { + struct ubifs_unclean_leb *ucleb; + + ucleb = list_entry(c->unclean_leb_list.next, + struct ubifs_unclean_leb, list); + list_del(&ucleb->list); + kfree(ucleb); + } + while (!list_empty(&c->old_buds)) { + struct ubifs_bud *bud; + + bud = list_entry(c->old_buds.next, struct ubifs_bud, list); + list_del(&bud->list); + kfree(bud->log_hash); + kfree(bud); + } + ubifs_destroy_idx_gc(c); + ubifs_destroy_size_tree(c); + ubifs_tnc_close(c); + free_buds(c); +} + +/** + * bu_init - initialize bulk-read information. + * @c: UBIFS file-system description object + */ +static void bu_init(struct ubifs_info *c) +{ + ubifs_assert(c, c->bulk_read == 1); + + if (c->bu.buf) + return; /* Already initialized */ + +again: + c->bu.buf = kmalloc(c->max_bu_buf_len, GFP_KERNEL | __GFP_NOWARN); + if (!c->bu.buf) { + if (c->max_bu_buf_len > UBIFS_KMALLOC_OK) { + c->max_bu_buf_len = UBIFS_KMALLOC_OK; + goto again; + } + + /* Just disable bulk-read */ + ubifs_warn(c, "cannot allocate %d bytes of memory for bulk-read, disabling it", + c->max_bu_buf_len); + c->mount_opts.bulk_read = 1; + c->bulk_read = 0; + return; + } +} + +/** + * check_free_space - check if there is enough free space to mount. + * @c: UBIFS file-system description object + * + * This function makes sure UBIFS has enough free space to be mounted in + * read/write mode. UBIFS must always have some free space to allow deletions. + */ +static int check_free_space(struct ubifs_info *c) +{ + ubifs_assert(c, c->dark_wm > 0); + if (c->lst.total_free + c->lst.total_dirty < c->dark_wm) { + ubifs_err(c, "insufficient free space to mount in R/W mode"); + ubifs_dump_budg(c, &c->bi); + ubifs_dump_lprops(c); + return -ENOSPC; + } + return 0; +} + +/** + * mount_ubifs - mount UBIFS file-system. + * @c: UBIFS file-system description object + * + * This function mounts UBIFS file system. Returns zero in case of success and + * a negative error code in case of failure. + */ +static int mount_ubifs(struct ubifs_info *c) +{ + int err; + long long x, y; + size_t sz; + + c->ro_mount = !!sb_rdonly(c->vfs_sb); + /* Suppress error messages while probing if SB_SILENT is set */ + c->probing = !!(c->vfs_sb->s_flags & SB_SILENT); + + err = init_constants_early(c); + if (err) + return err; + + err = ubifs_debugging_init(c); + if (err) + return err; + + err = ubifs_sysfs_register(c); + if (err) + goto out_debugging; + + err = check_volume_empty(c); + if (err) + goto out_free; + + if (c->empty && (c->ro_mount || c->ro_media)) { + /* + * This UBI volume is empty, and read-only, or the file system + * is mounted read-only - we cannot format it. + */ + ubifs_err(c, "can't format empty UBI volume: read-only %s", + c->ro_media ? "UBI volume" : "mount"); + err = -EROFS; + goto out_free; + } + + if (c->ro_media && !c->ro_mount) { + ubifs_err(c, "cannot mount read-write - read-only media"); + err = -EROFS; + goto out_free; + } + + /* + * The requirement for the buffer is that it should fit indexing B-tree + * height amount of integers. We assume the height if the TNC tree will + * never exceed 64. + */ + err = -ENOMEM; + c->bottom_up_buf = kmalloc_objs(int, BOTTOM_UP_HEIGHT); + if (!c->bottom_up_buf) + goto out_free; + + c->sbuf = vmalloc(c->leb_size); + if (!c->sbuf) + goto out_free; + + if (!c->ro_mount) { + c->ileb_buf = vmalloc(c->leb_size); + if (!c->ileb_buf) + goto out_free; + } + + if (c->bulk_read == 1) + bu_init(c); + + if (!c->ro_mount) { + c->write_reserve_buf = kmalloc(COMPRESSED_DATA_NODE_BUF_SZ + \ + UBIFS_CIPHER_BLOCK_SIZE, + GFP_KERNEL); + if (!c->write_reserve_buf) + goto out_free; + } + + c->mounting = 1; + + if (c->auth_key_name) { + if (IS_ENABLED(CONFIG_UBIFS_FS_AUTHENTICATION)) { + err = ubifs_init_authentication(c); + if (err) + goto out_free; + } else { + ubifs_err(c, "auth_key_name, but UBIFS is built without" + " authentication support"); + err = -EINVAL; + goto out_free; + } + } + + err = ubifs_read_superblock(c); + if (err) + goto out_auth; + + c->probing = 0; + + /* + * Make sure the compressor which is set as default in the superblock + * or overridden by mount options is actually compiled in. + */ + if (!ubifs_compr_present(c, c->default_compr)) { + ubifs_err(c, "'compressor \"%s\" is not compiled in", + ubifs_compr_name(c, c->default_compr)); + err = -ENOTSUPP; + goto out_auth; + } + + err = init_constants_sb(c); + if (err) + goto out_auth; + + sz = ALIGN(c->max_idx_node_sz, c->min_io_size) * 2; + c->cbuf = kmalloc(sz, GFP_NOFS); + if (!c->cbuf) { + err = -ENOMEM; + goto out_auth; + } + + err = alloc_wbufs(c); + if (err) + goto out_cbuf; + + sprintf(c->bgt_name, BGT_NAME_PATTERN, c->vi.ubi_num, c->vi.vol_id); + if (!c->ro_mount) { + /* Create background thread */ + c->bgt = kthread_run(ubifs_bg_thread, c, "%s", c->bgt_name); + if (IS_ERR(c->bgt)) { + err = PTR_ERR(c->bgt); + c->bgt = NULL; + ubifs_err(c, "cannot spawn \"%s\", error %d", + c->bgt_name, err); + goto out_wbufs; + } + } + + err = ubifs_read_master(c); + if (err) + goto out_master; + + init_constants_master(c); + + if ((c->mst_node->flags & cpu_to_le32(UBIFS_MST_DIRTY)) != 0) { + ubifs_msg(c, "recovery needed"); + c->need_recovery = 1; + } + + if (c->need_recovery && !c->ro_mount) { + err = ubifs_recover_inl_heads(c, c->sbuf); + if (err) + goto out_master; + } + + err = ubifs_lpt_init(c, 1, !c->ro_mount); + if (err) + goto out_master; + + if (!c->ro_mount && c->space_fixup) { + err = ubifs_fixup_free_space(c); + if (err) + goto out_lpt; + } + + if (!c->ro_mount && !c->need_recovery) { + /* + * Set the "dirty" flag so that if we reboot uncleanly we + * will notice this immediately on the next mount. + */ + c->mst_node->flags |= cpu_to_le32(UBIFS_MST_DIRTY); + err = ubifs_write_master(c); + if (err) + goto out_lpt; + } + + /* + * Handle offline signed images: Now that the master node is + * written and its validation no longer depends on the hash + * in the superblock, we can update the offline signed + * superblock with a HMAC version, + */ + if (ubifs_authenticated(c) && ubifs_hmac_zero(c, c->sup_node->hmac)) { + err = ubifs_hmac_wkm(c, c->sup_node->hmac_wkm); + if (err) + goto out_lpt; + c->superblock_need_write = 1; + } + + if (!c->ro_mount && c->superblock_need_write) { + err = ubifs_write_sb_node(c, c->sup_node); + if (err) + goto out_lpt; + c->superblock_need_write = 0; + } + + err = dbg_check_idx_size(c, c->bi.old_idx_sz); + if (err) + goto out_lpt; + + err = ubifs_replay_journal(c); + if (err) + goto out_journal; + + /* Calculate 'min_idx_lebs' after journal replay */ + c->bi.min_idx_lebs = ubifs_calc_min_idx_lebs(c); + + err = ubifs_mount_orphans(c, c->need_recovery, c->ro_mount); + if (err) + goto out_orphans; + + if (!c->ro_mount) { + int lnum; + + err = check_free_space(c); + if (err) + goto out_orphans; + + /* Check for enough log space */ + lnum = c->lhead_lnum + 1; + if (lnum >= UBIFS_LOG_LNUM + c->log_lebs) + lnum = UBIFS_LOG_LNUM; + if (lnum == c->ltail_lnum) { + err = ubifs_consolidate_log(c); + if (err) + goto out_orphans; + } + + if (c->need_recovery) { + if (!ubifs_authenticated(c)) { + err = ubifs_recover_size(c, true); + if (err) + goto out_orphans; + } + + err = ubifs_rcvry_gc_commit(c); + if (err) + goto out_orphans; + + if (ubifs_authenticated(c)) { + err = ubifs_recover_size(c, false); + if (err) + goto out_orphans; + } + } else { + err = take_gc_lnum(c); + if (err) + goto out_orphans; + + /* + * GC LEB may contain garbage if there was an unclean + * reboot, and it should be un-mapped. + */ + err = ubifs_leb_unmap(c, c->gc_lnum); + if (err) + goto out_orphans; + } + + err = dbg_check_lprops(c); + if (err) + goto out_orphans; + } else if (c->need_recovery) { + err = ubifs_recover_size(c, false); + if (err) + goto out_orphans; + } else { + /* + * Even if we mount read-only, we have to set space in GC LEB + * to proper value because this affects UBIFS free space + * reporting. We do not want to have a situation when + * re-mounting from R/O to R/W changes amount of free space. + */ + err = take_gc_lnum(c); + if (err) + goto out_orphans; + } + + spin_lock(&ubifs_infos_lock); + list_add_tail(&c->infos_list, &ubifs_infos); + spin_unlock(&ubifs_infos_lock); + + if (c->need_recovery) { + if (c->ro_mount) + ubifs_msg(c, "recovery deferred"); + else { + c->need_recovery = 0; + ubifs_msg(c, "recovery completed"); + /* + * GC LEB has to be empty and taken at this point. But + * the journal head LEBs may also be accounted as + * "empty taken" if they are empty. + */ + ubifs_assert(c, c->lst.taken_empty_lebs > 0); + } + } else + ubifs_assert(c, c->lst.taken_empty_lebs > 0); + + err = dbg_check_filesystem(c); + if (err) + goto out_infos; + + dbg_debugfs_init_fs(c); + + c->mounting = 0; + + ubifs_msg(c, "UBIFS: mounted UBI device %d, volume %d, name \"%s\"%s", + c->vi.ubi_num, c->vi.vol_id, c->vi.name, + c->ro_mount ? ", R/O mode" : ""); + x = (long long)c->main_lebs * c->leb_size; + y = (long long)c->log_lebs * c->leb_size + c->max_bud_bytes; + ubifs_msg(c, "LEB size: %d bytes (%d KiB), min./max. I/O unit sizes: %d bytes/%d bytes", + c->leb_size, c->leb_size >> 10, c->min_io_size, + c->max_write_size); + ubifs_msg(c, "FS size: %lld bytes (%lld MiB, %d LEBs), max %d LEBs, journal size %lld bytes (%lld MiB, %d LEBs)", + x, x >> 20, c->main_lebs, c->max_leb_cnt, + y, y >> 20, c->log_lebs + c->max_bud_cnt); + ubifs_msg(c, "reserved for root: %llu bytes (%llu KiB)", + c->report_rp_size, c->report_rp_size >> 10); + ubifs_msg(c, "media format: w%d/r%d (latest is w%d/r%d), UUID %pUB%s", + c->fmt_version, c->ro_compat_version, + UBIFS_FORMAT_VERSION, UBIFS_RO_COMPAT_VERSION, c->uuid, + c->big_lpt ? ", big LPT model" : ", small LPT model"); + + dbg_gen("default compressor: %s", ubifs_compr_name(c, c->default_compr)); + dbg_gen("data journal heads: %d", + c->jhead_cnt - NONDATA_JHEADS_CNT); + dbg_gen("log LEBs: %d (%d - %d)", + c->log_lebs, UBIFS_LOG_LNUM, c->log_last); + dbg_gen("LPT area LEBs: %d (%d - %d)", + c->lpt_lebs, c->lpt_first, c->lpt_last); + dbg_gen("orphan area LEBs: %d (%d - %d)", + c->orph_lebs, c->orph_first, c->orph_last); + dbg_gen("main area LEBs: %d (%d - %d)", + c->main_lebs, c->main_first, c->leb_cnt - 1); + dbg_gen("index LEBs: %d", c->lst.idx_lebs); + dbg_gen("total index bytes: %llu (%llu KiB, %llu MiB)", + c->bi.old_idx_sz, c->bi.old_idx_sz >> 10, + c->bi.old_idx_sz >> 20); + dbg_gen("key hash type: %d", c->key_hash_type); + dbg_gen("tree fanout: %d", c->fanout); + dbg_gen("reserved GC LEB: %d", c->gc_lnum); + dbg_gen("max. znode size %d", c->max_znode_sz); + dbg_gen("max. index node size %d", c->max_idx_node_sz); + dbg_gen("node sizes: data %zu, inode %zu, dentry %zu", + UBIFS_DATA_NODE_SZ, UBIFS_INO_NODE_SZ, UBIFS_DENT_NODE_SZ); + dbg_gen("node sizes: trun %zu, sb %zu, master %zu", + UBIFS_TRUN_NODE_SZ, UBIFS_SB_NODE_SZ, UBIFS_MST_NODE_SZ); + dbg_gen("node sizes: ref %zu, cmt. start %zu, orph %zu", + UBIFS_REF_NODE_SZ, UBIFS_CS_NODE_SZ, UBIFS_ORPH_NODE_SZ); + dbg_gen("max. node sizes: data %zu, inode %zu dentry %zu, idx %d", + UBIFS_MAX_DATA_NODE_SZ, UBIFS_MAX_INO_NODE_SZ, + UBIFS_MAX_DENT_NODE_SZ, ubifs_idx_node_sz(c, c->fanout)); + dbg_gen("dead watermark: %d", c->dead_wm); + dbg_gen("dark watermark: %d", c->dark_wm); + dbg_gen("LEB overhead: %d", c->leb_overhead); + x = (long long)c->main_lebs * c->dark_wm; + dbg_gen("max. dark space: %lld (%lld KiB, %lld MiB)", + x, x >> 10, x >> 20); + dbg_gen("maximum bud bytes: %lld (%lld KiB, %lld MiB)", + c->max_bud_bytes, c->max_bud_bytes >> 10, + c->max_bud_bytes >> 20); + dbg_gen("BG commit bud bytes: %lld (%lld KiB, %lld MiB)", + c->bg_bud_bytes, c->bg_bud_bytes >> 10, + c->bg_bud_bytes >> 20); + dbg_gen("current bud bytes %lld (%lld KiB, %lld MiB)", + c->bud_bytes, c->bud_bytes >> 10, c->bud_bytes >> 20); + dbg_gen("max. seq. number: %llu", c->max_sqnum); + dbg_gen("commit number: %llu", c->cmt_no); + dbg_gen("max. xattrs per inode: %d", ubifs_xattr_max_cnt(c)); + dbg_gen("max orphans: %d", c->max_orphans); + + return 0; + +out_infos: + spin_lock(&ubifs_infos_lock); + list_del(&c->infos_list); + spin_unlock(&ubifs_infos_lock); +out_orphans: + free_orphans(c); +out_journal: + destroy_journal(c); +out_lpt: + ubifs_lpt_free(c, 0); +out_master: + kfree(c->mst_node); + kfree(c->rcvrd_mst_node); + if (c->bgt) + kthread_stop(c->bgt); +out_wbufs: + free_wbufs(c); +out_cbuf: + kfree(c->cbuf); +out_auth: + ubifs_exit_authentication(c); +out_free: + kfree(c->write_reserve_buf); + kfree(c->bu.buf); + vfree(c->ileb_buf); + vfree(c->sbuf); + kfree(c->bottom_up_buf); + kfree(c->sup_node); + ubifs_sysfs_unregister(c); +out_debugging: + ubifs_debugging_exit(c); + return err; +} + +/** + * ubifs_umount - un-mount UBIFS file-system. + * @c: UBIFS file-system description object + * + * Note, this function is called to free allocated resourced when un-mounting, + * as well as free resources when an error occurred while we were half way + * through mounting (error path cleanup function). So it has to make sure the + * resource was actually allocated before freeing it. + */ +static void ubifs_umount(struct ubifs_info *c) +{ + dbg_gen("un-mounting UBI device %d, volume %d", c->vi.ubi_num, + c->vi.vol_id); + + dbg_debugfs_exit_fs(c); + spin_lock(&ubifs_infos_lock); + list_del(&c->infos_list); + spin_unlock(&ubifs_infos_lock); + + if (c->bgt) + kthread_stop(c->bgt); + + destroy_journal(c); + free_wbufs(c); + free_orphans(c); + ubifs_lpt_free(c, 0); + ubifs_exit_authentication(c); + + ubifs_release_options(c); + kfree(c->cbuf); + kfree(c->rcvrd_mst_node); + kfree(c->mst_node); + kfree(c->write_reserve_buf); + kfree(c->bu.buf); + vfree(c->ileb_buf); + vfree(c->sbuf); + kfree(c->bottom_up_buf); + kfree(c->sup_node); + ubifs_debugging_exit(c); + ubifs_sysfs_unregister(c); +} + +/** + * ubifs_remount_rw - re-mount in read-write mode. + * @c: UBIFS file-system description object + * + * UBIFS avoids allocating many unnecessary resources when mounted in read-only + * mode. This function allocates the needed resources and re-mounts UBIFS in + * read-write mode. + */ +static int ubifs_remount_rw(struct ubifs_info *c) +{ + int err, lnum; + + if (c->rw_incompat) { + ubifs_err(c, "the file-system is not R/W-compatible"); + ubifs_msg(c, "on-flash format version is w%d/r%d, but software only supports up to version w%d/r%d", + c->fmt_version, c->ro_compat_version, + UBIFS_FORMAT_VERSION, UBIFS_RO_COMPAT_VERSION); + return -EROFS; + } + + mutex_lock(&c->umount_mutex); + dbg_save_space_info(c); + c->remounting_rw = 1; + c->ro_mount = 0; + + if (c->space_fixup) { + err = ubifs_fixup_free_space(c); + if (err) + goto out; + } + + err = check_free_space(c); + if (err) + goto out; + + if (c->need_recovery) { + ubifs_msg(c, "completing deferred recovery"); + err = ubifs_write_rcvrd_mst_node(c); + if (err) + goto out; + if (!ubifs_authenticated(c)) { + err = ubifs_recover_size(c, true); + if (err) + goto out; + } + err = ubifs_clean_lebs(c, c->sbuf); + if (err) + goto out; + err = ubifs_recover_inl_heads(c, c->sbuf); + if (err) + goto out; + } else { + /* A readonly mount is not allowed to have orphans */ + ubifs_assert(c, c->tot_orphans == 0); + err = ubifs_clear_orphans(c); + if (err) + goto out; + } + + if (!(c->mst_node->flags & cpu_to_le32(UBIFS_MST_DIRTY))) { + c->mst_node->flags |= cpu_to_le32(UBIFS_MST_DIRTY); + err = ubifs_write_master(c); + if (err) + goto out; + } + + if (c->superblock_need_write) { + struct ubifs_sb_node *sup = c->sup_node; + + err = ubifs_write_sb_node(c, sup); + if (err) + goto out; + + c->superblock_need_write = 0; + } + + c->ileb_buf = vmalloc(c->leb_size); + if (!c->ileb_buf) { + err = -ENOMEM; + goto out; + } + + c->write_reserve_buf = kmalloc(COMPRESSED_DATA_NODE_BUF_SZ + \ + UBIFS_CIPHER_BLOCK_SIZE, GFP_KERNEL); + if (!c->write_reserve_buf) { + err = -ENOMEM; + goto out; + } + + err = ubifs_lpt_init(c, 0, 1); + if (err) + goto out; + + /* Create background thread */ + c->bgt = kthread_run(ubifs_bg_thread, c, "%s", c->bgt_name); + if (IS_ERR(c->bgt)) { + err = PTR_ERR(c->bgt); + c->bgt = NULL; + ubifs_err(c, "cannot spawn \"%s\", error %d", + c->bgt_name, err); + goto out; + } + + c->orph_buf = vmalloc(c->leb_size); + if (!c->orph_buf) { + err = -ENOMEM; + goto out; + } + + /* Check for enough log space */ + lnum = c->lhead_lnum + 1; + if (lnum >= UBIFS_LOG_LNUM + c->log_lebs) + lnum = UBIFS_LOG_LNUM; + if (lnum == c->ltail_lnum) { + err = ubifs_consolidate_log(c); + if (err) + goto out; + } + + if (c->need_recovery) { + err = ubifs_rcvry_gc_commit(c); + if (err) + goto out; + + if (ubifs_authenticated(c)) { + err = ubifs_recover_size(c, false); + if (err) + goto out; + } + } else { + err = ubifs_leb_unmap(c, c->gc_lnum); + } + if (err) + goto out; + + dbg_gen("re-mounted read-write"); + c->remounting_rw = 0; + + if (c->need_recovery) { + c->need_recovery = 0; + ubifs_msg(c, "deferred recovery completed"); + } else { + /* + * Do not run the debugging space check if the were doing + * recovery, because when we saved the information we had the + * file-system in a state where the TNC and lprops has been + * modified in memory, but all the I/O operations (including a + * commit) were deferred. So the file-system was in + * "non-committed" state. Now the file-system is in committed + * state, and of course the amount of free space will change + * because, for example, the old index size was imprecise. + */ + err = dbg_check_space_info(c); + } + + mutex_unlock(&c->umount_mutex); + return err; + +out: + c->ro_mount = 1; + vfree(c->orph_buf); + c->orph_buf = NULL; + if (c->bgt) { + kthread_stop(c->bgt); + c->bgt = NULL; + } + kfree(c->write_reserve_buf); + c->write_reserve_buf = NULL; + vfree(c->ileb_buf); + c->ileb_buf = NULL; + ubifs_lpt_free(c, 1); + c->remounting_rw = 0; + mutex_unlock(&c->umount_mutex); + return err; +} + +/** + * ubifs_remount_ro - re-mount in read-only mode. + * @c: UBIFS file-system description object + * + * We assume VFS has stopped writing. Possibly the background thread could be + * running a commit, however kthread_stop will wait in that case. + */ +static void ubifs_remount_ro(struct ubifs_info *c) +{ + int i, err; + + ubifs_assert(c, !c->need_recovery); + ubifs_assert(c, !c->ro_mount); + + mutex_lock(&c->umount_mutex); + if (c->bgt) { + kthread_stop(c->bgt); + c->bgt = NULL; + } + + dbg_save_space_info(c); + + for (i = 0; i < c->jhead_cnt; i++) { + err = ubifs_wbuf_sync(&c->jheads[i].wbuf); + if (err) + ubifs_ro_mode(c, err); + } + + c->mst_node->flags &= ~cpu_to_le32(UBIFS_MST_DIRTY); + c->mst_node->flags |= cpu_to_le32(UBIFS_MST_NO_ORPHS); + c->mst_node->gc_lnum = cpu_to_le32(c->gc_lnum); + err = ubifs_write_master(c); + if (err) + ubifs_ro_mode(c, err); + + vfree(c->orph_buf); + c->orph_buf = NULL; + kfree(c->write_reserve_buf); + c->write_reserve_buf = NULL; + vfree(c->ileb_buf); + c->ileb_buf = NULL; + ubifs_lpt_free(c, 1); + c->ro_mount = 1; + err = dbg_check_space_info(c); + if (err) + ubifs_ro_mode(c, err); + mutex_unlock(&c->umount_mutex); +} + +static void ubifs_put_super(struct super_block *sb) +{ + int i; + struct ubifs_info *c = sb->s_fs_info; + + ubifs_msg(c, "un-mount UBI device %d", c->vi.ubi_num); + + /* + * The following asserts are only valid if there has not been a failure + * of the media. For example, there will be dirty inodes if we failed + * to write them back because of I/O errors. + */ + if (!c->ro_error) { + ubifs_assert(c, c->bi.idx_growth == 0); + ubifs_assert(c, c->bi.dd_growth == 0); + ubifs_assert(c, c->bi.data_growth == 0); + } + + /* + * The 'c->umount_lock' prevents races between UBIFS memory shrinker + * and file system un-mount. Namely, it prevents the shrinker from + * picking this superblock for shrinking - it will be just skipped if + * the mutex is locked. + */ + mutex_lock(&c->umount_mutex); + if (!c->ro_mount) { + /* + * First of all kill the background thread to make sure it does + * not interfere with un-mounting and freeing resources. + */ + if (c->bgt) { + kthread_stop(c->bgt); + c->bgt = NULL; + } + + /* + * On fatal errors c->ro_error is set to 1, in which case we do + * not write the master node. + */ + if (!c->ro_error) { + int err; + + /* Synchronize write-buffers */ + for (i = 0; i < c->jhead_cnt; i++) { + err = ubifs_wbuf_sync(&c->jheads[i].wbuf); + if (err) + ubifs_ro_mode(c, err); + } + + /* + * We are being cleanly unmounted which means the + * orphans were killed - indicate this in the master + * node. Also save the reserved GC LEB number. + */ + c->mst_node->flags &= ~cpu_to_le32(UBIFS_MST_DIRTY); + c->mst_node->flags |= cpu_to_le32(UBIFS_MST_NO_ORPHS); + c->mst_node->gc_lnum = cpu_to_le32(c->gc_lnum); + err = ubifs_write_master(c); + if (err) + /* + * Recovery will attempt to fix the master area + * next mount, so we just print a message and + * continue to unmount normally. + */ + ubifs_err(c, "failed to write master node, error %d", + err); + } else { + for (i = 0; i < c->jhead_cnt; i++) + /* Make sure write-buffer timers are canceled */ + hrtimer_cancel(&c->jheads[i].wbuf.timer); + } + } + + ubifs_umount(c); + ubi_close_volume(c->ubi); + mutex_unlock(&c->umount_mutex); +} + +static int ubifs_reconfigure(struct fs_context *fc) +{ + struct ubifs_fs_context *ctx = fc->fs_private; + struct super_block *sb = fc->root->d_sb; + int err; + struct ubifs_info *c = sb->s_fs_info; + + sync_filesystem(sb); + dbg_gen("old flags %#lx, new flags %#x", sb->s_flags, fc->sb_flags); + + /* + * Apply the mount option changes. + * auth_key_name and auth_hash_name are ignored on remount. + */ + c->mount_opts = ctx->mount_opts; + c->bulk_read = ctx->bulk_read; + c->no_chk_data_crc = ctx->no_chk_data_crc; + c->default_compr = ctx->default_compr; + c->assert_action = ctx->assert_action; + + if (c->ro_mount && !(fc->sb_flags & SB_RDONLY)) { + if (c->ro_error) { + ubifs_msg(c, "cannot re-mount R/W due to prior errors"); + return -EROFS; + } + if (c->ro_media) { + ubifs_msg(c, "cannot re-mount R/W - UBI volume is R/O"); + return -EROFS; + } + err = ubifs_remount_rw(c); + if (err) + return err; + } else if (!c->ro_mount && (fc->sb_flags & SB_RDONLY)) { + if (c->ro_error) { + ubifs_msg(c, "cannot re-mount R/O due to prior errors"); + return -EROFS; + } + ubifs_remount_ro(c); + } + + if (c->bulk_read == 1) + bu_init(c); + else { + dbg_gen("disable bulk-read"); + mutex_lock(&c->bu_mutex); + kfree(c->bu.buf); + c->bu.buf = NULL; + mutex_unlock(&c->bu_mutex); + } + + if (!c->need_recovery) + ubifs_assert(c, c->lst.taken_empty_lebs > 0); + + return 0; +} + +const struct super_operations ubifs_super_operations = { + .alloc_inode = ubifs_alloc_inode, + .free_inode = ubifs_free_inode, + .put_super = ubifs_put_super, + .write_inode = ubifs_write_inode, + .drop_inode = ubifs_drop_inode, + .evict_inode = ubifs_evict_inode, + .statfs = ubifs_statfs, + .dirty_inode = ubifs_dirty_inode, + .show_options = ubifs_show_options, + .sync_fs = ubifs_sync_fs, +}; + +/** + * open_ubi - parse UBI device name string and open the UBI device. + * @fc: The filesystem context + * @mode: UBI volume open mode + * + * The primary method of mounting UBIFS is by specifying the UBI volume + * character device node path. However, UBIFS may also be mounted without any + * character device node using one of the following methods: + * + * o ubiX_Y - mount UBI device number X, volume Y; + * o ubiY - mount UBI device number 0, volume Y; + * o ubiX:NAME - mount UBI device X, volume with name NAME; + * o ubi:NAME - mount UBI device 0, volume with name NAME. + * + * Alternative '!' separator may be used instead of ':' (because some shells + * like busybox may interpret ':' as an NFS host name separator). This function + * returns UBI volume description object in case of success and a negative + * error code in case of failure. + */ +static struct ubi_volume_desc *open_ubi(struct fs_context *fc, int mode) +{ + struct ubi_volume_desc *ubi; + const char *name = fc->source; + int dev, vol; + char *endptr; + + /* First, try to open using the device node path method */ + ubi = ubi_open_volume_path(name, mode); + if (!IS_ERR(ubi)) + return ubi; + + /* Try the "nodev" method */ + if (name[0] != 'u' || name[1] != 'b' || name[2] != 'i') + goto invalid_source; + + /* ubi:NAME method */ + if ((name[3] == ':' || name[3] == '!') && name[4] != '\0') + return ubi_open_volume_nm(0, name + 4, mode); + + if (!isdigit(name[3])) + goto invalid_source; + + dev = simple_strtoul(name + 3, &endptr, 0); + + /* ubiY method */ + if (*endptr == '\0') + return ubi_open_volume(0, dev, mode); + + /* ubiX_Y method */ + if (*endptr == '_' && isdigit(endptr[1])) { + vol = simple_strtoul(endptr + 1, &endptr, 0); + if (*endptr != '\0') + goto invalid_source; + return ubi_open_volume(dev, vol, mode); + } + + /* ubiX:NAME method */ + if ((*endptr == ':' || *endptr == '!') && endptr[1] != '\0') + return ubi_open_volume_nm(dev, ++endptr, mode); + +invalid_source: + return ERR_PTR(invalf(fc, "Invalid source name")); +} + +static struct ubifs_info *alloc_ubifs_info(struct ubi_volume_desc *ubi) +{ + struct ubifs_info *c; + + c = kzalloc_obj(struct ubifs_info); + if (c) { + spin_lock_init(&c->cnt_lock); + spin_lock_init(&c->cs_lock); + spin_lock_init(&c->buds_lock); + spin_lock_init(&c->space_lock); + spin_lock_init(&c->orphan_lock); + init_rwsem(&c->commit_sem); + mutex_init(&c->lp_mutex); + mutex_init(&c->tnc_mutex); + mutex_init(&c->log_mutex); + mutex_init(&c->umount_mutex); + mutex_init(&c->bu_mutex); + mutex_init(&c->write_reserve_mutex); + init_waitqueue_head(&c->cmt_wq); + init_waitqueue_head(&c->reserve_space_wq); + atomic_set(&c->need_wait_space, 0); + c->buds = RB_ROOT; + c->old_idx = RB_ROOT; + c->size_tree = RB_ROOT; + c->orph_tree = RB_ROOT; + INIT_LIST_HEAD(&c->infos_list); + INIT_LIST_HEAD(&c->idx_gc); + INIT_LIST_HEAD(&c->replay_list); + INIT_LIST_HEAD(&c->replay_buds); + INIT_LIST_HEAD(&c->uncat_list); + INIT_LIST_HEAD(&c->empty_list); + INIT_LIST_HEAD(&c->freeable_list); + INIT_LIST_HEAD(&c->frdi_idx_list); + INIT_LIST_HEAD(&c->unclean_leb_list); + INIT_LIST_HEAD(&c->old_buds); + INIT_LIST_HEAD(&c->orph_list); + INIT_LIST_HEAD(&c->orph_new); + c->no_chk_data_crc = 1; + c->assert_action = ASSACT_RO; + + c->highest_inum = UBIFS_FIRST_INO; + c->lhead_lnum = c->ltail_lnum = UBIFS_LOG_LNUM; + + ubi_get_volume_info(ubi, &c->vi); + ubi_get_device_info(c->vi.ubi_num, &c->di); + } + return c; +} + +static int ubifs_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct ubifs_info *c = sb->s_fs_info; + struct ubifs_fs_context *ctx = fc->fs_private; + struct inode *root; + int err; + + c->vfs_sb = sb; + /* Re-open the UBI device in read-write mode */ + c->ubi = ubi_open_volume(c->vi.ubi_num, c->vi.vol_id, UBI_READWRITE); + if (IS_ERR(c->ubi)) { + err = PTR_ERR(c->ubi); + goto out; + } + + /* Copy in parsed mount options */ + c->mount_opts = ctx->mount_opts; + c->auth_key_name = ctx->auth_key_name; + c->auth_hash_name = ctx->auth_hash_name; + c->no_chk_data_crc = ctx->no_chk_data_crc; + c->bulk_read = ctx->bulk_read; + c->default_compr = ctx->default_compr; + c->assert_action = ctx->assert_action; + + /* ubifs_info owns auth strings now */ + ctx->auth_key_name = NULL; + ctx->auth_hash_name = NULL; + + /* + * UBIFS provides 'backing_dev_info' in order to disable read-ahead. For + * UBIFS, I/O is not deferred, it is done immediately in read_folio, + * which means the user would have to wait not just for their own I/O + * but the read-ahead I/O as well i.e. completely pointless. + * + * Read-ahead will be disabled because @sb->s_bdi->ra_pages is 0. Also + * @sb->s_bdi->capabilities are initialized to 0 so there won't be any + * writeback happening. + */ + err = super_setup_bdi_name(sb, "ubifs_%d_%d", c->vi.ubi_num, + c->vi.vol_id); + if (err) + goto out_close; + sb->s_bdi->ra_pages = 0; + sb->s_bdi->io_pages = 0; + + sb->s_fs_info = c; + sb->s_magic = UBIFS_SUPER_MAGIC; + sb->s_blocksize = UBIFS_BLOCK_SIZE; + sb->s_blocksize_bits = UBIFS_BLOCK_SHIFT; + sb->s_maxbytes = c->max_inode_sz = key_max_inode_size(c); + if (c->max_inode_sz > MAX_LFS_FILESIZE) + sb->s_maxbytes = c->max_inode_sz = MAX_LFS_FILESIZE; + sb->s_op = &ubifs_super_operations; + sb->s_xattr = ubifs_xattr_handlers; + fscrypt_set_ops(sb, &ubifs_crypt_operations); + + mutex_lock(&c->umount_mutex); + err = mount_ubifs(c); + if (err) { + ubifs_assert(c, err < 0); + goto out_unlock; + } + + /* Read the root inode */ + root = ubifs_iget(sb, UBIFS_ROOT_INO); + if (IS_ERR(root)) { + err = PTR_ERR(root); + goto out_umount; + } + + generic_set_sb_d_ops(sb); + sb->s_root = d_make_root(root); + if (!sb->s_root) { + err = -ENOMEM; + goto out_umount; + } + + super_set_uuid(sb, c->uuid, sizeof(c->uuid)); + super_set_sysfs_name_generic(sb, UBIFS_DFS_DIR_NAME, + c->vi.ubi_num, c->vi.vol_id); + + mutex_unlock(&c->umount_mutex); + return 0; + +out_umount: + ubifs_umount(c); +out_unlock: + mutex_unlock(&c->umount_mutex); +out_close: + ubifs_release_options(c); + ubi_close_volume(c->ubi); +out: + return err; +} + +static int sb_test(struct super_block *sb, struct fs_context *fc) +{ + struct ubifs_info *c1 = fc->s_fs_info; + struct ubifs_info *c = sb->s_fs_info; + + return c->vi.cdev == c1->vi.cdev; +} + +static int ubifs_get_tree(struct fs_context *fc) +{ + struct ubi_volume_desc *ubi; + struct ubifs_info *c; + struct super_block *sb; + int err; + + if (!fc->source || !*fc->source) + return invalf(fc, "No source specified"); + + dbg_gen("name %s, flags %#x", fc->source, fc->sb_flags); + + /* + * Get UBI device number and volume ID. Mount it read-only so far + * because this might be a new mount point, and UBI allows only one + * read-write user at a time. + */ + ubi = open_ubi(fc, UBI_READONLY); + if (IS_ERR(ubi)) { + err = PTR_ERR(ubi); + if (!(fc->sb_flags & SB_SILENT)) + pr_err("UBIFS error (pid: %d): cannot open \"%s\", error %d", + current->pid, fc->source, err); + return err; + } + + c = alloc_ubifs_info(ubi); + if (!c) { + err = -ENOMEM; + goto out_close; + } + fc->s_fs_info = c; + + dbg_gen("opened ubi%d_%d", c->vi.ubi_num, c->vi.vol_id); + + sb = sget_fc(fc, sb_test, set_anon_super_fc); + if (IS_ERR(sb)) { + err = PTR_ERR(sb); + kfree(c); + goto out_close; + } + + if (sb->s_root) { + struct ubifs_info *c1 = sb->s_fs_info; + kfree(c); + /* A new mount point for already mounted UBIFS */ + dbg_gen("this ubi volume is already mounted"); + if (!!(fc->sb_flags & SB_RDONLY) != c1->ro_mount) { + err = -EBUSY; + goto out_deact; + } + } else { + err = ubifs_fill_super(sb, fc); + if (err) + goto out_deact; + /* We do not support atime */ + sb->s_flags |= SB_ACTIVE; + if (IS_ENABLED(CONFIG_UBIFS_ATIME_SUPPORT)) + ubifs_msg(c, "full atime support is enabled."); + else + sb->s_flags |= SB_NOATIME; + } + + /* 'fill_super()' opens ubi again so we must close it here */ + ubi_close_volume(ubi); + + fc->root = dget(sb->s_root); + return 0; + +out_deact: + deactivate_locked_super(sb); +out_close: + ubi_close_volume(ubi); + return err; +} + +static void kill_ubifs_super(struct super_block *s) +{ + struct ubifs_info *c = s->s_fs_info; + kill_anon_super(s); + kfree(c); +} + +static void ubifs_free_fc(struct fs_context *fc) +{ + struct ubifs_fs_context *ctx = fc->fs_private; + + if (ctx) { + kfree(ctx->auth_key_name); + kfree(ctx->auth_hash_name); + kfree(ctx); + } +} + +static const struct fs_context_operations ubifs_context_ops = { + .free = ubifs_free_fc, + .parse_param = ubifs_parse_param, + .get_tree = ubifs_get_tree, + .reconfigure = ubifs_reconfigure, +}; + +static int ubifs_init_fs_context(struct fs_context *fc) +{ + struct ubifs_fs_context *ctx; + + ctx = kzalloc_obj(struct ubifs_fs_context); + if (!ctx) + return -ENOMEM; + + if (fc->purpose != FS_CONTEXT_FOR_RECONFIGURE) { + /* Iniitialize for first mount */ + ctx->no_chk_data_crc = 1; + ctx->assert_action = ASSACT_RO; + } else { + struct ubifs_info *c = fc->root->d_sb->s_fs_info; + + /* + * Preserve existing options across remounts. + * auth_key_name and auth_hash_name are not remountable. + */ + ctx->mount_opts = c->mount_opts; + ctx->bulk_read = c->bulk_read; + ctx->no_chk_data_crc = c->no_chk_data_crc; + ctx->default_compr = c->default_compr; + ctx->assert_action = c->assert_action; + } + + fc->ops = &ubifs_context_ops; + fc->fs_private = ctx; + + return 0; +} + +static struct file_system_type ubifs_fs_type = { + .name = "ubifs", + .owner = THIS_MODULE, + .init_fs_context = ubifs_init_fs_context, + .parameters = ubifs_fs_param_spec, + .kill_sb = kill_ubifs_super, +}; +MODULE_ALIAS_FS("ubifs"); + +/* + * Inode slab cache constructor. + */ +static void inode_slab_ctor(void *obj) +{ + struct ubifs_inode *ui = obj; + inode_init_once(&ui->vfs_inode); +} + +static int __init ubifs_init(void) +{ + int err = -ENOMEM; + + BUILD_BUG_ON(sizeof(struct ubifs_ch) != 24); + + /* Make sure node sizes are 8-byte aligned */ + BUILD_BUG_ON(UBIFS_CH_SZ & 7); + BUILD_BUG_ON(UBIFS_INO_NODE_SZ & 7); + BUILD_BUG_ON(UBIFS_DENT_NODE_SZ & 7); + BUILD_BUG_ON(UBIFS_XENT_NODE_SZ & 7); + BUILD_BUG_ON(UBIFS_DATA_NODE_SZ & 7); + BUILD_BUG_ON(UBIFS_TRUN_NODE_SZ & 7); + BUILD_BUG_ON(UBIFS_SB_NODE_SZ & 7); + BUILD_BUG_ON(UBIFS_MST_NODE_SZ & 7); + BUILD_BUG_ON(UBIFS_REF_NODE_SZ & 7); + BUILD_BUG_ON(UBIFS_CS_NODE_SZ & 7); + BUILD_BUG_ON(UBIFS_ORPH_NODE_SZ & 7); + + BUILD_BUG_ON(UBIFS_MAX_DENT_NODE_SZ & 7); + BUILD_BUG_ON(UBIFS_MAX_XENT_NODE_SZ & 7); + BUILD_BUG_ON(UBIFS_MAX_DATA_NODE_SZ & 7); + BUILD_BUG_ON(UBIFS_MAX_INO_NODE_SZ & 7); + BUILD_BUG_ON(UBIFS_MAX_NODE_SZ & 7); + BUILD_BUG_ON(MIN_WRITE_SZ & 7); + + /* Check min. node size */ + BUILD_BUG_ON(UBIFS_INO_NODE_SZ < MIN_WRITE_SZ); + BUILD_BUG_ON(UBIFS_DENT_NODE_SZ < MIN_WRITE_SZ); + BUILD_BUG_ON(UBIFS_XENT_NODE_SZ < MIN_WRITE_SZ); + BUILD_BUG_ON(UBIFS_TRUN_NODE_SZ < MIN_WRITE_SZ); + + BUILD_BUG_ON(UBIFS_MAX_DENT_NODE_SZ > UBIFS_MAX_NODE_SZ); + BUILD_BUG_ON(UBIFS_MAX_XENT_NODE_SZ > UBIFS_MAX_NODE_SZ); + BUILD_BUG_ON(UBIFS_MAX_DATA_NODE_SZ > UBIFS_MAX_NODE_SZ); + BUILD_BUG_ON(UBIFS_MAX_INO_NODE_SZ > UBIFS_MAX_NODE_SZ); + + /* Defined node sizes */ + BUILD_BUG_ON(UBIFS_SB_NODE_SZ != 4096); + BUILD_BUG_ON(UBIFS_MST_NODE_SZ != 512); + BUILD_BUG_ON(UBIFS_INO_NODE_SZ != 160); + BUILD_BUG_ON(UBIFS_REF_NODE_SZ != 64); + + /* + * We use 2 bit wide bit-fields to store compression type, which should + * be amended if more compressors are added. The bit-fields are: + * @compr_type in 'struct ubifs_inode', @default_compr in + * 'struct ubifs_info' and @compr_type in 'struct ubifs_mount_opts'. + */ + BUILD_BUG_ON(UBIFS_COMPR_TYPES_CNT > 4); + + /* + * We require that PAGE_SIZE is greater-than-or-equal-to + * UBIFS_BLOCK_SIZE. It is assumed that both are powers of 2. + */ + if (PAGE_SIZE < UBIFS_BLOCK_SIZE) { + pr_err("UBIFS error (pid %d): VFS page cache size is %u bytes, but UBIFS requires at least 4096 bytes", + current->pid, (unsigned int)PAGE_SIZE); + return -EINVAL; + } + + ubifs_inode_slab = kmem_cache_create("ubifs_inode_slab", + sizeof(struct ubifs_inode), 0, + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, + &inode_slab_ctor); + if (!ubifs_inode_slab) + return -ENOMEM; + + ubifs_shrinker_info = shrinker_alloc(0, "ubifs-slab"); + if (!ubifs_shrinker_info) + goto out_slab; + + ubifs_shrinker_info->count_objects = ubifs_shrink_count; + ubifs_shrinker_info->scan_objects = ubifs_shrink_scan; + + shrinker_register(ubifs_shrinker_info); + + err = ubifs_compressors_init(); + if (err) + goto out_shrinker; + + dbg_debugfs_init(); + + err = ubifs_sysfs_init(); + if (err) + goto out_dbg; + + err = register_filesystem(&ubifs_fs_type); + if (err) { + pr_err("UBIFS error (pid %d): cannot register file system, error %d", + current->pid, err); + goto out_sysfs; + } + return 0; + +out_sysfs: + ubifs_sysfs_exit(); +out_dbg: + dbg_debugfs_exit(); + ubifs_compressors_exit(); +out_shrinker: + shrinker_free(ubifs_shrinker_info); +out_slab: + kmem_cache_destroy(ubifs_inode_slab); + return err; +} +/* late_initcall to let compressors initialize first */ +late_initcall(ubifs_init); + +static void __exit ubifs_exit(void) +{ + WARN_ON(!list_empty(&ubifs_infos)); + WARN_ON(atomic_long_read(&ubifs_clean_zn_cnt) != 0); + + dbg_debugfs_exit(); + ubifs_sysfs_exit(); + ubifs_compressors_exit(); + shrinker_free(ubifs_shrinker_info); + + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(ubifs_inode_slab); + unregister_filesystem(&ubifs_fs_type); +} +module_exit(ubifs_exit); + +MODULE_LICENSE("GPL"); +MODULE_VERSION(__stringify(UBIFS_VERSION)); +MODULE_AUTHOR("Artem Bityutskiy, Adrian Hunter"); +MODULE_DESCRIPTION("UBIFS - UBI File System"); diff --git a/fs/ubifs/sysfs.c b/fs/ubifs/sysfs.c new file mode 100644 index 000000000..dbf967dcf --- /dev/null +++ b/fs/ubifs/sysfs.c @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2021 Cisco Systems + * + * Author: Stefan Schaeckeler + */ + + +#include +#include "ubifs.h" + +enum attr_id_t { + attr_errors_magic, + attr_errors_node, + attr_errors_crc, +}; + +struct ubifs_attr { + struct attribute attr; + enum attr_id_t attr_id; +}; + +#define UBIFS_ATTR(_name, _mode, _id) \ +static struct ubifs_attr ubifs_attr_##_name = { \ + .attr = {.name = __stringify(_name), .mode = _mode }, \ + .attr_id = attr_##_id, \ +} + +#define UBIFS_ATTR_FUNC(_name, _mode) UBIFS_ATTR(_name, _mode, _name) + +UBIFS_ATTR_FUNC(errors_magic, 0444); +UBIFS_ATTR_FUNC(errors_crc, 0444); +UBIFS_ATTR_FUNC(errors_node, 0444); + +#define ATTR_LIST(name) (&ubifs_attr_##name.attr) + +static struct attribute *ubifs_attrs[] = { + ATTR_LIST(errors_magic), + ATTR_LIST(errors_node), + ATTR_LIST(errors_crc), + NULL, +}; +ATTRIBUTE_GROUPS(ubifs); + +static ssize_t ubifs_attr_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct ubifs_info *sbi = container_of(kobj, struct ubifs_info, + kobj); + + struct ubifs_attr *a = container_of(attr, struct ubifs_attr, attr); + + switch (a->attr_id) { + case attr_errors_magic: + return sysfs_emit(buf, "%u\n", sbi->stats->magic_errors); + case attr_errors_node: + return sysfs_emit(buf, "%u\n", sbi->stats->node_errors); + case attr_errors_crc: + return sysfs_emit(buf, "%u\n", sbi->stats->crc_errors); + } + return 0; +}; + +static void ubifs_sb_release(struct kobject *kobj) +{ + struct ubifs_info *c = container_of(kobj, struct ubifs_info, kobj); + + complete(&c->kobj_unregister); +} + +static const struct sysfs_ops ubifs_attr_ops = { + .show = ubifs_attr_show, +}; + +static const struct kobj_type ubifs_sb_ktype = { + .default_groups = ubifs_groups, + .sysfs_ops = &ubifs_attr_ops, + .release = ubifs_sb_release, +}; + +static const struct kobj_type ubifs_ktype = { + .sysfs_ops = &ubifs_attr_ops, +}; + +static struct kset ubifs_kset = { + .kobj = {.ktype = &ubifs_ktype}, +}; + +int ubifs_sysfs_register(struct ubifs_info *c) +{ + int ret, n; + char dfs_dir_name[UBIFS_DFS_DIR_LEN]; + + c->stats = kzalloc_obj(struct ubifs_stats_info); + if (!c->stats) { + ret = -ENOMEM; + goto out_last; + } + n = snprintf(dfs_dir_name, UBIFS_DFS_DIR_LEN, UBIFS_DFS_DIR_NAME, + c->vi.ubi_num, c->vi.vol_id); + + if (n >= UBIFS_DFS_DIR_LEN) { + /* The array size is too small */ + ret = -EINVAL; + goto out_free; + } + + c->kobj.kset = &ubifs_kset; + init_completion(&c->kobj_unregister); + + ret = kobject_init_and_add(&c->kobj, &ubifs_sb_ktype, NULL, + "%s", dfs_dir_name); + if (ret) + goto out_put; + + return 0; + +out_put: + kobject_put(&c->kobj); + wait_for_completion(&c->kobj_unregister); +out_free: + kfree(c->stats); +out_last: + ubifs_err(c, "cannot create sysfs entry for ubifs%d_%d, error %d\n", + c->vi.ubi_num, c->vi.vol_id, ret); + return ret; +} + +void ubifs_sysfs_unregister(struct ubifs_info *c) +{ + kobject_del(&c->kobj); + kobject_put(&c->kobj); + wait_for_completion(&c->kobj_unregister); + + kfree(c->stats); +} + +int __init ubifs_sysfs_init(void) +{ + int ret; + + kobject_set_name(&ubifs_kset.kobj, "ubifs"); + ubifs_kset.kobj.parent = fs_kobj; + ret = kset_register(&ubifs_kset); + if (ret) + kset_put(&ubifs_kset); + + return ret; +} + +void ubifs_sysfs_exit(void) +{ + kset_unregister(&ubifs_kset); +} diff --git a/fs/ubifs/tnc.c b/fs/ubifs/tnc.c new file mode 100644 index 000000000..c9d8935f6 --- /dev/null +++ b/fs/ubifs/tnc.c @@ -0,0 +1,3575 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * + * Authors: Adrian Hunter + * Artem Bityutskiy (Битюцкий Артём) + */ + +/* + * This file implements TNC (Tree Node Cache) which caches indexing nodes of + * the UBIFS B-tree. + * + * At the moment the locking rules of the TNC tree are quite simple and + * straightforward. We just have a mutex and lock it when we traverse the + * tree. If a znode is not in memory, we read it from flash while still having + * the mutex locked. + */ + +#include +#include +#include "ubifs.h" + +static int try_read_node(const struct ubifs_info *c, void *buf, int type, + struct ubifs_zbranch *zbr); +static int fallible_read_node(struct ubifs_info *c, const union ubifs_key *key, + struct ubifs_zbranch *zbr, void *node); + +/* + * Returned codes of 'matches_name()' and 'fallible_matches_name()' functions. + * @NAME_LESS: name corresponding to the first argument is less than second + * @NAME_MATCHES: names match + * @NAME_GREATER: name corresponding to the second argument is greater than + * first + * @NOT_ON_MEDIA: node referred by zbranch does not exist on the media + * + * These constants were introduce to improve readability. + */ +enum { + NAME_LESS = 0, + NAME_MATCHES = 1, + NAME_GREATER = 2, + NOT_ON_MEDIA = 3, +}; + +static void do_insert_old_idx(struct ubifs_info *c, + struct ubifs_old_idx *old_idx) +{ + struct ubifs_old_idx *o; + struct rb_node **p, *parent = NULL; + + p = &c->old_idx.rb_node; + while (*p) { + parent = *p; + o = rb_entry(parent, struct ubifs_old_idx, rb); + if (old_idx->lnum < o->lnum) + p = &(*p)->rb_left; + else if (old_idx->lnum > o->lnum) + p = &(*p)->rb_right; + else if (old_idx->offs < o->offs) + p = &(*p)->rb_left; + else if (old_idx->offs > o->offs) + p = &(*p)->rb_right; + else { + ubifs_err(c, "old idx added twice!"); + kfree(old_idx); + return; + } + } + rb_link_node(&old_idx->rb, parent, p); + rb_insert_color(&old_idx->rb, &c->old_idx); +} + +/** + * insert_old_idx - record an index node obsoleted since the last commit start. + * @c: UBIFS file-system description object + * @lnum: LEB number of obsoleted index node + * @offs: offset of obsoleted index node + * + * Returns %0 on success, and a negative error code on failure. + * + * For recovery, there must always be a complete intact version of the index on + * flash at all times. That is called the "old index". It is the index as at the + * time of the last successful commit. Many of the index nodes in the old index + * may be dirty, but they must not be erased until the next successful commit + * (at which point that index becomes the old index). + * + * That means that the garbage collection and the in-the-gaps method of + * committing must be able to determine if an index node is in the old index. + * Most of the old index nodes can be found by looking up the TNC using the + * 'lookup_znode()' function. However, some of the old index nodes may have + * been deleted from the current index or may have been changed so much that + * they cannot be easily found. In those cases, an entry is added to an RB-tree. + * That is what this function does. The RB-tree is ordered by LEB number and + * offset because they uniquely identify the old index node. + */ +static int insert_old_idx(struct ubifs_info *c, int lnum, int offs) +{ + struct ubifs_old_idx *old_idx; + + old_idx = kmalloc_obj(struct ubifs_old_idx, GFP_NOFS); + if (unlikely(!old_idx)) + return -ENOMEM; + old_idx->lnum = lnum; + old_idx->offs = offs; + do_insert_old_idx(c, old_idx); + + return 0; +} + +/** + * insert_old_idx_znode - record a znode obsoleted since last commit start. + * @c: UBIFS file-system description object + * @znode: znode of obsoleted index node + * + * Returns %0 on success, and a negative error code on failure. + */ +int insert_old_idx_znode(struct ubifs_info *c, struct ubifs_znode *znode) +{ + if (znode->parent) { + struct ubifs_zbranch *zbr; + + zbr = &znode->parent->zbranch[znode->iip]; + if (zbr->len) + return insert_old_idx(c, zbr->lnum, zbr->offs); + } else + if (c->zroot.len) + return insert_old_idx(c, c->zroot.lnum, + c->zroot.offs); + return 0; +} + +/** + * ins_clr_old_idx_znode - record a znode obsoleted since last commit start. + * @c: UBIFS file-system description object + * @znode: znode of obsoleted index node + * + * Returns %0 on success, and a negative error code on failure. + */ +static int ins_clr_old_idx_znode(struct ubifs_info *c, + struct ubifs_znode *znode) +{ + int err; + + if (znode->parent) { + struct ubifs_zbranch *zbr; + + zbr = &znode->parent->zbranch[znode->iip]; + if (zbr->len) { + err = insert_old_idx(c, zbr->lnum, zbr->offs); + if (err) + return err; + zbr->lnum = 0; + zbr->offs = 0; + zbr->len = 0; + } + } else + if (c->zroot.len) { + err = insert_old_idx(c, c->zroot.lnum, c->zroot.offs); + if (err) + return err; + c->zroot.lnum = 0; + c->zroot.offs = 0; + c->zroot.len = 0; + } + return 0; +} + +/** + * destroy_old_idx - destroy the old_idx RB-tree. + * @c: UBIFS file-system description object + * + * During start commit, the old_idx RB-tree is used to avoid overwriting index + * nodes that were in the index last commit but have since been deleted. This + * is necessary for recovery i.e. the old index must be kept intact until the + * new index is successfully written. The old-idx RB-tree is used for the + * in-the-gaps method of writing index nodes and is destroyed every commit. + */ +void destroy_old_idx(struct ubifs_info *c) +{ + struct ubifs_old_idx *old_idx, *n; + + rbtree_postorder_for_each_entry_safe(old_idx, n, &c->old_idx, rb) + kfree(old_idx); + + c->old_idx = RB_ROOT; +} + +/** + * copy_znode - copy a dirty znode. + * @c: UBIFS file-system description object + * @znode: znode to copy + * + * A dirty znode being committed may not be changed, so it is copied. + */ +static struct ubifs_znode *copy_znode(struct ubifs_info *c, + struct ubifs_znode *znode) +{ + struct ubifs_znode *zn; + + zn = kmemdup(znode, c->max_znode_sz, GFP_NOFS); + if (unlikely(!zn)) + return ERR_PTR(-ENOMEM); + + zn->cnext = NULL; + __set_bit(DIRTY_ZNODE, &zn->flags); + __clear_bit(COW_ZNODE, &zn->flags); + + return zn; +} + +/** + * add_idx_dirt - add dirt due to a dirty znode. + * @c: UBIFS file-system description object + * @lnum: LEB number of index node + * @dirt: size of index node + * + * This function updates lprops dirty space and the new size of the index. + */ +static int add_idx_dirt(struct ubifs_info *c, int lnum, int dirt) +{ + c->calc_idx_sz -= ALIGN(dirt, 8); + return ubifs_add_dirt(c, lnum, dirt); +} + +/** + * replace_znode - replace old znode with new znode. + * @c: UBIFS file-system description object + * @new_zn: new znode + * @old_zn: old znode + * @zbr: the branch of parent znode + * + * Replace old znode with new znode in TNC. + */ +static void replace_znode(struct ubifs_info *c, struct ubifs_znode *new_zn, + struct ubifs_znode *old_zn, struct ubifs_zbranch *zbr) +{ + ubifs_assert(c, !ubifs_zn_obsolete(old_zn)); + __set_bit(OBSOLETE_ZNODE, &old_zn->flags); + + if (old_zn->level != 0) { + int i; + const int n = new_zn->child_cnt; + + /* The children now have new parent */ + for (i = 0; i < n; i++) { + struct ubifs_zbranch *child = &new_zn->zbranch[i]; + + if (child->znode) + child->znode->parent = new_zn; + } + } + + zbr->znode = new_zn; + zbr->lnum = 0; + zbr->offs = 0; + zbr->len = 0; + + atomic_long_inc(&c->dirty_zn_cnt); +} + +/** + * dirty_cow_znode - ensure a znode is not being committed. + * @c: UBIFS file-system description object + * @zbr: branch of znode to check + * + * Returns dirtied znode on success or negative error code on failure. + */ +static struct ubifs_znode *dirty_cow_znode(struct ubifs_info *c, + struct ubifs_zbranch *zbr) +{ + struct ubifs_znode *znode = zbr->znode; + struct ubifs_znode *zn; + int err; + + if (!ubifs_zn_cow(znode)) { + /* znode is not being committed */ + if (!test_and_set_bit(DIRTY_ZNODE, &znode->flags)) { + atomic_long_inc(&c->dirty_zn_cnt); + atomic_long_dec(&c->clean_zn_cnt); + atomic_long_dec(&ubifs_clean_zn_cnt); + err = add_idx_dirt(c, zbr->lnum, zbr->len); + if (unlikely(err)) + return ERR_PTR(err); + } + return znode; + } + + zn = copy_znode(c, znode); + if (IS_ERR(zn)) + return zn; + + if (zbr->len) { + struct ubifs_old_idx *old_idx; + + old_idx = kmalloc_obj(struct ubifs_old_idx, GFP_NOFS); + if (unlikely(!old_idx)) { + err = -ENOMEM; + goto out; + } + old_idx->lnum = zbr->lnum; + old_idx->offs = zbr->offs; + + err = add_idx_dirt(c, zbr->lnum, zbr->len); + if (err) { + kfree(old_idx); + goto out; + } + + do_insert_old_idx(c, old_idx); + } + + replace_znode(c, zn, znode, zbr); + + return zn; + +out: + kfree(zn); + return ERR_PTR(err); +} + +/** + * lnc_add - add a leaf node to the leaf node cache. + * @c: UBIFS file-system description object + * @zbr: zbranch of leaf node + * @node: leaf node + * + * Leaf nodes are non-index nodes directory entry nodes or data nodes. The + * purpose of the leaf node cache is to save re-reading the same leaf node over + * and over again. Most things are cached by VFS, however the file system must + * cache directory entries for readdir and for resolving hash collisions. The + * present implementation of the leaf node cache is extremely simple, and + * allows for error returns that are not used but that may be needed if a more + * complex implementation is created. + * + * Note, this function does not add the @node object to LNC directly, but + * allocates a copy of the object and adds the copy to LNC. The reason for this + * is that @node has been allocated outside of the TNC subsystem and will be + * used with @c->tnc_mutex unlock upon return from the TNC subsystem. But LNC + * may be changed at any time, e.g. freed by the shrinker. + */ +static int lnc_add(struct ubifs_info *c, struct ubifs_zbranch *zbr, + const void *node) +{ + int err; + void *lnc_node; + const struct ubifs_dent_node *dent = node; + + ubifs_assert(c, !zbr->leaf); + ubifs_assert(c, zbr->len != 0); + ubifs_assert(c, is_hash_key(c, &zbr->key)); + + err = ubifs_validate_entry(c, dent); + if (err) { + dump_stack(); + ubifs_dump_node(c, dent, zbr->len); + return err; + } + + lnc_node = kmemdup(node, zbr->len, GFP_NOFS); + if (!lnc_node) + /* We don't have to have the cache, so no error */ + return 0; + + zbr->leaf = lnc_node; + return 0; +} + + /** + * lnc_add_directly - add a leaf node to the leaf-node-cache. + * @c: UBIFS file-system description object + * @zbr: zbranch of leaf node + * @node: leaf node + * + * This function is similar to 'lnc_add()', but it does not create a copy of + * @node but inserts @node to TNC directly. + */ +static int lnc_add_directly(struct ubifs_info *c, struct ubifs_zbranch *zbr, + void *node) +{ + int err; + + ubifs_assert(c, !zbr->leaf); + ubifs_assert(c, zbr->len != 0); + + err = ubifs_validate_entry(c, node); + if (err) { + dump_stack(); + ubifs_dump_node(c, node, zbr->len); + return err; + } + + zbr->leaf = node; + return 0; +} + +/** + * lnc_free - remove a leaf node from the leaf node cache. + * @zbr: zbranch of leaf node + */ +static void lnc_free(struct ubifs_zbranch *zbr) +{ + if (!zbr->leaf) + return; + kfree(zbr->leaf); + zbr->leaf = NULL; +} + +/** + * tnc_read_hashed_node - read a "hashed" leaf node. + * @c: UBIFS file-system description object + * @zbr: key and position of the node + * @node: node is returned here + * + * This function reads a "hashed" node defined by @zbr from the leaf node cache + * (in it is there) or from the hash media, in which case the node is also + * added to LNC. Returns zero in case of success or a negative error + * code in case of failure. + */ +static int tnc_read_hashed_node(struct ubifs_info *c, struct ubifs_zbranch *zbr, + void *node) +{ + int err; + + ubifs_assert(c, is_hash_key(c, &zbr->key)); + + if (zbr->leaf) { + /* Read from the leaf node cache */ + ubifs_assert(c, zbr->len != 0); + memcpy(node, zbr->leaf, zbr->len); + return 0; + } + + if (c->replaying) { + err = fallible_read_node(c, &zbr->key, zbr, node); + /* + * When the node was not found, return -ENOENT, 0 otherwise. + * Negative return codes stay as-is. + */ + if (err == 0) + err = -ENOENT; + else if (err == 1) + err = 0; + } else { + err = ubifs_tnc_read_node(c, zbr, node); + } + if (err) + return err; + + /* Add the node to the leaf node cache */ + err = lnc_add(c, zbr, node); + return err; +} + +/** + * try_read_node - read a node if it is a node. + * @c: UBIFS file-system description object + * @buf: buffer to read to + * @type: node type + * @zbr: the zbranch describing the node to read + * + * This function tries to read a node of known type and length, checks it and + * stores it in @buf. This function returns %1 if a node is present and %0 if + * a node is not present. A negative error code is returned for I/O errors. + * This function performs that same function as ubifs_read_node except that + * it does not require that there is actually a node present and instead + * the return code indicates if a node was read. + * + * Note, this function does not check CRC of data nodes if @c->no_chk_data_crc + * is true (it is controlled by corresponding mount option). However, if + * @c->mounting or @c->remounting_rw is true (we are mounting or re-mounting to + * R/W mode), @c->no_chk_data_crc is ignored and CRC is checked. This is + * because during mounting or re-mounting from R/O mode to R/W mode we may read + * journal nodes (when replying the journal or doing the recovery) and the + * journal nodes may potentially be corrupted, so checking is required. + */ +static int try_read_node(const struct ubifs_info *c, void *buf, int type, + struct ubifs_zbranch *zbr) +{ + int len = zbr->len; + int lnum = zbr->lnum; + int offs = zbr->offs; + int err, node_len; + struct ubifs_ch *ch = buf; + uint32_t crc, node_crc; + + dbg_io("LEB %d:%d, %s, length %d", lnum, offs, dbg_ntype(type), len); + + err = ubifs_leb_read(c, lnum, buf, offs, len, 1); + if (err) { + ubifs_err(c, "cannot read node type %d from LEB %d:%d, error %d", + type, lnum, offs, err); + return err; + } + + if (le32_to_cpu(ch->magic) != UBIFS_NODE_MAGIC) + return 0; + + if (ch->node_type != type) + return 0; + + node_len = le32_to_cpu(ch->len); + if (node_len != len) + return 0; + + if (type != UBIFS_DATA_NODE || !c->no_chk_data_crc || c->mounting || + c->remounting_rw) { + crc = crc32(UBIFS_CRC32_INIT, buf + 8, node_len - 8); + node_crc = le32_to_cpu(ch->crc); + if (crc != node_crc) + return 0; + } + + err = ubifs_node_check_hash(c, buf, zbr->hash); + if (err) { + ubifs_bad_hash(c, buf, zbr->hash, lnum, offs); + return 0; + } + + return 1; +} + +/** + * fallible_read_node - try to read a leaf node. + * @c: UBIFS file-system description object + * @key: key of node to read + * @zbr: position of node + * @node: node returned + * + * This function tries to read a node and returns %1 if the node is read, %0 + * if the node is not present, and a negative error code in the case of error. + */ +static int fallible_read_node(struct ubifs_info *c, const union ubifs_key *key, + struct ubifs_zbranch *zbr, void *node) +{ + int ret; + + dbg_tnck(key, "LEB %d:%d, key ", zbr->lnum, zbr->offs); + + ret = try_read_node(c, node, key_type(c, key), zbr); + if (ret == 1) { + union ubifs_key node_key; + struct ubifs_dent_node *dent = node; + + /* All nodes have key in the same place */ + key_read(c, &dent->key, &node_key); + if (keys_cmp(c, key, &node_key) != 0) + ret = 0; + } + if (ret == 0 && c->replaying) + dbg_mntk(key, "dangling branch LEB %d:%d len %d, key ", + zbr->lnum, zbr->offs, zbr->len); + return ret; +} + +/** + * matches_name - determine if a direntry or xattr entry matches a given name. + * @c: UBIFS file-system description object + * @zbr: zbranch of dent + * @nm: name to match + * + * This function checks if xentry/direntry referred by zbranch @zbr matches name + * @nm. Returns %NAME_MATCHES if it does, %NAME_LESS if the name referred by + * @zbr is less than @nm, and %NAME_GREATER if it is greater than @nm. In case + * of failure, a negative error code is returned. + */ +static int matches_name(struct ubifs_info *c, struct ubifs_zbranch *zbr, + const struct fscrypt_name *nm) +{ + struct ubifs_dent_node *dent; + int nlen, err; + + /* If possible, match against the dent in the leaf node cache */ + if (!zbr->leaf) { + dent = kmalloc(zbr->len, GFP_NOFS); + if (!dent) + return -ENOMEM; + + err = ubifs_tnc_read_node(c, zbr, dent); + if (err) + goto out_free; + + /* Add the node to the leaf node cache */ + err = lnc_add_directly(c, zbr, dent); + if (err) + goto out_free; + } else + dent = zbr->leaf; + + nlen = le16_to_cpu(dent->nlen); + err = memcmp(dent->name, fname_name(nm), min_t(int, nlen, fname_len(nm))); + if (err == 0) { + if (nlen == fname_len(nm)) + return NAME_MATCHES; + else if (nlen < fname_len(nm)) + return NAME_LESS; + else + return NAME_GREATER; + } else if (err < 0) + return NAME_LESS; + else + return NAME_GREATER; + +out_free: + kfree(dent); + return err; +} + +/** + * get_znode - get a TNC znode that may not be loaded yet. + * @c: UBIFS file-system description object + * @znode: parent znode + * @n: znode branch slot number + * + * This function returns the znode or a negative error code. + */ +static struct ubifs_znode *get_znode(struct ubifs_info *c, + struct ubifs_znode *znode, int n) +{ + struct ubifs_zbranch *zbr; + + zbr = &znode->zbranch[n]; + if (zbr->znode) + znode = zbr->znode; + else + znode = ubifs_load_znode(c, zbr, znode, n); + return znode; +} + +/** + * tnc_next - find next TNC entry. + * @c: UBIFS file-system description object + * @zn: znode is passed and returned here + * @n: znode branch slot number is passed and returned here + * + * This function returns %0 if the next TNC entry is found, %-ENOENT if there is + * no next entry, or a negative error code otherwise. + */ +static int tnc_next(struct ubifs_info *c, struct ubifs_znode **zn, int *n) +{ + struct ubifs_znode *znode = *zn; + int nn = *n; + + nn += 1; + if (nn < znode->child_cnt) { + *n = nn; + return 0; + } + while (1) { + struct ubifs_znode *zp; + + zp = znode->parent; + if (!zp) + return -ENOENT; + nn = znode->iip + 1; + znode = zp; + if (nn < znode->child_cnt) { + znode = get_znode(c, znode, nn); + if (IS_ERR(znode)) + return PTR_ERR(znode); + while (znode->level != 0) { + znode = get_znode(c, znode, 0); + if (IS_ERR(znode)) + return PTR_ERR(znode); + } + nn = 0; + break; + } + } + *zn = znode; + *n = nn; + return 0; +} + +/** + * tnc_prev - find previous TNC entry. + * @c: UBIFS file-system description object + * @zn: znode is returned here + * @n: znode branch slot number is passed and returned here + * + * This function returns %0 if the previous TNC entry is found, %-ENOENT if + * there is no next entry, or a negative error code otherwise. + */ +static int tnc_prev(struct ubifs_info *c, struct ubifs_znode **zn, int *n) +{ + struct ubifs_znode *znode = *zn; + int nn = *n; + + if (nn > 0) { + *n = nn - 1; + return 0; + } + while (1) { + struct ubifs_znode *zp; + + zp = znode->parent; + if (!zp) + return -ENOENT; + nn = znode->iip - 1; + znode = zp; + if (nn >= 0) { + znode = get_znode(c, znode, nn); + if (IS_ERR(znode)) + return PTR_ERR(znode); + while (znode->level != 0) { + nn = znode->child_cnt - 1; + znode = get_znode(c, znode, nn); + if (IS_ERR(znode)) + return PTR_ERR(znode); + } + nn = znode->child_cnt - 1; + break; + } + } + *zn = znode; + *n = nn; + return 0; +} + +/** + * resolve_collision - resolve a collision. + * @c: UBIFS file-system description object + * @key: key of a directory or extended attribute entry + * @zn: znode is returned here + * @n: zbranch number is passed and returned here + * @nm: name of the entry + * + * This function is called for "hashed" keys to make sure that the found key + * really corresponds to the looked up node (directory or extended attribute + * entry). It returns %1 and sets @zn and @n if the collision is resolved. + * %0 is returned if @nm is not found and @zn and @n are set to the previous + * entry, i.e. to the entry after which @nm could follow if it were in TNC. + * This means that @n may be set to %-1 if the leftmost key in @zn is the + * previous one. A negative error code is returned on failures. + */ +static int resolve_collision(struct ubifs_info *c, const union ubifs_key *key, + struct ubifs_znode **zn, int *n, + const struct fscrypt_name *nm) +{ + int err; + + err = matches_name(c, &(*zn)->zbranch[*n], nm); + if (unlikely(err < 0)) + return err; + if (err == NAME_MATCHES) + return 1; + + if (err == NAME_GREATER) { + /* Look left */ + while (1) { + err = tnc_prev(c, zn, n); + if (err == -ENOENT) { + ubifs_assert(c, *n == 0); + *n = -1; + return 0; + } + if (err < 0) + return err; + if (keys_cmp(c, &(*zn)->zbranch[*n].key, key)) { + /* + * We have found the branch after which we would + * like to insert, but inserting in this znode + * may still be wrong. Consider the following 3 + * znodes, in the case where we are resolving a + * collision with Key2. + * + * znode zp + * ---------------------- + * level 1 | Key0 | Key1 | + * ----------------------- + * | | + * znode za | | znode zb + * ------------ ------------ + * level 0 | Key0 | | Key2 | + * ------------ ------------ + * + * The lookup finds Key2 in znode zb. Lets say + * there is no match and the name is greater so + * we look left. When we find Key0, we end up + * here. If we return now, we will insert into + * znode za at slot n = 1. But that is invalid + * according to the parent's keys. Key2 must + * be inserted into znode zb. + * + * Note, this problem is not relevant for the + * case when we go right, because + * 'tnc_insert()' would correct the parent key. + */ + if (*n == (*zn)->child_cnt - 1) { + err = tnc_next(c, zn, n); + if (err) { + /* Should be impossible */ + ubifs_assert(c, 0); + if (err == -ENOENT) + err = -EINVAL; + return err; + } + ubifs_assert(c, *n == 0); + *n = -1; + } + return 0; + } + err = matches_name(c, &(*zn)->zbranch[*n], nm); + if (err < 0) + return err; + if (err == NAME_LESS) + return 0; + if (err == NAME_MATCHES) + return 1; + ubifs_assert(c, err == NAME_GREATER); + } + } else { + int nn = *n; + struct ubifs_znode *znode = *zn; + + /* Look right */ + while (1) { + err = tnc_next(c, &znode, &nn); + if (err == -ENOENT) + return 0; + if (err < 0) + return err; + if (keys_cmp(c, &znode->zbranch[nn].key, key)) + return 0; + err = matches_name(c, &znode->zbranch[nn], nm); + if (err < 0) + return err; + if (err == NAME_GREATER) + return 0; + *zn = znode; + *n = nn; + if (err == NAME_MATCHES) + return 1; + ubifs_assert(c, err == NAME_LESS); + } + } +} + +/** + * fallible_matches_name - determine if a dent matches a given name. + * @c: UBIFS file-system description object + * @zbr: zbranch of dent + * @nm: name to match + * + * This is a "fallible" version of 'matches_name()' function which does not + * panic if the direntry/xentry referred by @zbr does not exist on the media. + * + * This function checks if xentry/direntry referred by zbranch @zbr matches name + * @nm. Returns %NAME_MATCHES it does, %NAME_LESS if the name referred by @zbr + * is less than @nm, %NAME_GREATER if it is greater than @nm, and @NOT_ON_MEDIA + * if xentry/direntry referred by @zbr does not exist on the media. A negative + * error code is returned in case of failure. + */ +static int fallible_matches_name(struct ubifs_info *c, + struct ubifs_zbranch *zbr, + const struct fscrypt_name *nm) +{ + struct ubifs_dent_node *dent; + int nlen, err; + + /* If possible, match against the dent in the leaf node cache */ + if (!zbr->leaf) { + dent = kmalloc(zbr->len, GFP_NOFS); + if (!dent) + return -ENOMEM; + + err = fallible_read_node(c, &zbr->key, zbr, dent); + if (err < 0) + goto out_free; + if (err == 0) { + /* The node was not present */ + err = NOT_ON_MEDIA; + goto out_free; + } + ubifs_assert(c, err == 1); + + err = lnc_add_directly(c, zbr, dent); + if (err) + goto out_free; + } else + dent = zbr->leaf; + + nlen = le16_to_cpu(dent->nlen); + err = memcmp(dent->name, fname_name(nm), min_t(int, nlen, fname_len(nm))); + if (err == 0) { + if (nlen == fname_len(nm)) + return NAME_MATCHES; + else if (nlen < fname_len(nm)) + return NAME_LESS; + else + return NAME_GREATER; + } else if (err < 0) + return NAME_LESS; + else + return NAME_GREATER; + +out_free: + kfree(dent); + return err; +} + +/** + * fallible_resolve_collision - resolve a collision even if nodes are missing. + * @c: UBIFS file-system description object + * @key: key + * @zn: znode is returned here + * @n: branch number is passed and returned here + * @nm: name of directory entry + * @adding: indicates caller is adding a key to the TNC + * + * This is a "fallible" version of the 'resolve_collision()' function which + * does not panic if one of the nodes referred to by TNC does not exist on the + * media. This may happen when replaying the journal if a deleted node was + * Garbage-collected and the commit was not done. A branch that refers to a node + * that is not present is called a dangling branch. The following are the return + * codes for this function: + * o if @nm was found, %1 is returned and @zn and @n are set to the found + * branch; + * o if we are @adding and @nm was not found, %0 is returned; + * o if we are not @adding and @nm was not found, but a dangling branch was + * found, then %1 is returned and @zn and @n are set to the dangling branch; + * o a negative error code is returned in case of failure. + */ +static int fallible_resolve_collision(struct ubifs_info *c, + const union ubifs_key *key, + struct ubifs_znode **zn, int *n, + const struct fscrypt_name *nm, + int adding) +{ + struct ubifs_znode *o_znode = NULL, *znode = *zn; + int o_n, err, cmp, unsure = 0, nn = *n; + + cmp = fallible_matches_name(c, &znode->zbranch[nn], nm); + if (unlikely(cmp < 0)) + return cmp; + if (cmp == NAME_MATCHES) + return 1; + if (cmp == NOT_ON_MEDIA) { + o_znode = znode; + o_n = nn; + /* + * We are unlucky and hit a dangling branch straight away. + * Now we do not really know where to go to find the needed + * branch - to the left or to the right. Well, let's try left. + */ + unsure = 1; + } else if (!adding) + unsure = 1; /* Remove a dangling branch wherever it is */ + + if (cmp == NAME_GREATER || unsure) { + /* Look left */ + while (1) { + err = tnc_prev(c, zn, n); + if (err == -ENOENT) { + ubifs_assert(c, *n == 0); + *n = -1; + break; + } + if (err < 0) + return err; + if (keys_cmp(c, &(*zn)->zbranch[*n].key, key)) { + /* See comments in 'resolve_collision()' */ + if (*n == (*zn)->child_cnt - 1) { + err = tnc_next(c, zn, n); + if (err) { + /* Should be impossible */ + ubifs_assert(c, 0); + if (err == -ENOENT) + err = -EINVAL; + return err; + } + ubifs_assert(c, *n == 0); + *n = -1; + } + break; + } + err = fallible_matches_name(c, &(*zn)->zbranch[*n], nm); + if (err < 0) + return err; + if (err == NAME_MATCHES) + return 1; + if (err == NOT_ON_MEDIA) { + o_znode = *zn; + o_n = *n; + continue; + } + if (!adding) + continue; + if (err == NAME_LESS) + break; + else + unsure = 0; + } + } + + if (cmp == NAME_LESS || unsure) { + /* Look right */ + *zn = znode; + *n = nn; + while (1) { + err = tnc_next(c, &znode, &nn); + if (err == -ENOENT) + break; + if (err < 0) + return err; + if (keys_cmp(c, &znode->zbranch[nn].key, key)) + break; + err = fallible_matches_name(c, &znode->zbranch[nn], nm); + if (err < 0) + return err; + if (err == NAME_GREATER) + break; + *zn = znode; + *n = nn; + if (err == NAME_MATCHES) + return 1; + if (err == NOT_ON_MEDIA) { + o_znode = znode; + o_n = nn; + } + } + } + + /* Never match a dangling branch when adding */ + if (adding || !o_znode) + return 0; + + dbg_mntk(key, "dangling match LEB %d:%d len %d key ", + o_znode->zbranch[o_n].lnum, o_znode->zbranch[o_n].offs, + o_znode->zbranch[o_n].len); + *zn = o_znode; + *n = o_n; + return 1; +} + +/** + * matches_position - determine if a zbranch matches a given position. + * @zbr: zbranch of dent + * @lnum: LEB number of dent to match + * @offs: offset of dent to match + * + * This function returns %1 if @lnum:@offs matches, and %0 otherwise. + */ +static int matches_position(struct ubifs_zbranch *zbr, int lnum, int offs) +{ + if (zbr->lnum == lnum && zbr->offs == offs) + return 1; + else + return 0; +} + +/** + * resolve_collision_directly - resolve a collision directly. + * @c: UBIFS file-system description object + * @key: key of directory entry + * @zn: znode is passed and returned here + * @n: zbranch number is passed and returned here + * @lnum: LEB number of dent node to match + * @offs: offset of dent node to match + * + * This function is used for "hashed" keys to make sure the found directory or + * extended attribute entry node is what was looked for. It is used when the + * flash address of the right node is known (@lnum:@offs) which makes it much + * easier to resolve collisions (no need to read entries and match full + * names). This function returns %1 and sets @zn and @n if the collision is + * resolved, %0 if @lnum:@offs is not found and @zn and @n are set to the + * previous directory entry. Otherwise a negative error code is returned. + */ +static int resolve_collision_directly(struct ubifs_info *c, + const union ubifs_key *key, + struct ubifs_znode **zn, int *n, + int lnum, int offs) +{ + struct ubifs_znode *znode; + int nn, err; + + znode = *zn; + nn = *n; + if (matches_position(&znode->zbranch[nn], lnum, offs)) + return 1; + + /* Look left */ + while (1) { + err = tnc_prev(c, &znode, &nn); + if (err == -ENOENT) + break; + if (err < 0) + return err; + if (keys_cmp(c, &znode->zbranch[nn].key, key)) + break; + if (matches_position(&znode->zbranch[nn], lnum, offs)) { + *zn = znode; + *n = nn; + return 1; + } + } + + /* Look right */ + znode = *zn; + nn = *n; + while (1) { + err = tnc_next(c, &znode, &nn); + if (err == -ENOENT) + return 0; + if (err < 0) + return err; + if (keys_cmp(c, &znode->zbranch[nn].key, key)) + return 0; + *zn = znode; + *n = nn; + if (matches_position(&znode->zbranch[nn], lnum, offs)) + return 1; + } +} + +/** + * dirty_cow_bottom_up - dirty a znode and its ancestors. + * @c: UBIFS file-system description object + * @znode: znode to dirty + * + * If we do not have a unique key that resides in a znode, then we cannot + * dirty that znode from the top down (i.e. by using lookup_level0_dirty) + * This function records the path back to the last dirty ancestor, and then + * dirties the znodes on that path. + */ +static struct ubifs_znode *dirty_cow_bottom_up(struct ubifs_info *c, + struct ubifs_znode *znode) +{ + struct ubifs_znode *zp; + int *path = c->bottom_up_buf, p = 0; + + ubifs_assert(c, c->zroot.znode); + ubifs_assert(c, znode); + if (c->zroot.znode->level > BOTTOM_UP_HEIGHT) { + kfree(c->bottom_up_buf); + c->bottom_up_buf = kmalloc_objs(int, c->zroot.znode->level, + GFP_NOFS); + if (!c->bottom_up_buf) + return ERR_PTR(-ENOMEM); + path = c->bottom_up_buf; + } + if (c->zroot.znode->level) { + /* Go up until parent is dirty */ + while (1) { + int n; + + zp = znode->parent; + if (!zp) + break; + n = znode->iip; + ubifs_assert(c, p < c->zroot.znode->level); + path[p++] = n; + if (!zp->cnext && ubifs_zn_dirty(znode)) + break; + znode = zp; + } + } + + /* Come back down, dirtying as we go */ + while (1) { + struct ubifs_zbranch *zbr; + + zp = znode->parent; + if (zp) { + ubifs_assert(c, path[p - 1] >= 0); + ubifs_assert(c, path[p - 1] < zp->child_cnt); + zbr = &zp->zbranch[path[--p]]; + znode = dirty_cow_znode(c, zbr); + } else { + ubifs_assert(c, znode == c->zroot.znode); + znode = dirty_cow_znode(c, &c->zroot); + } + if (IS_ERR(znode) || !p) + break; + ubifs_assert(c, path[p - 1] >= 0); + ubifs_assert(c, path[p - 1] < znode->child_cnt); + znode = znode->zbranch[path[p - 1]].znode; + } + + return znode; +} + +/** + * ubifs_lookup_level0 - search for zero-level znode. + * @c: UBIFS file-system description object + * @key: key to lookup + * @zn: znode is returned here + * @n: znode branch slot number is returned here + * + * This function looks up the TNC tree and search for zero-level znode which + * refers key @key. The found zero-level znode is returned in @zn. There are 3 + * cases: + * o exact match, i.e. the found zero-level znode contains key @key, then %1 + * is returned and slot number of the matched branch is stored in @n; + * o not exact match, which means that zero-level znode does not contain + * @key, then %0 is returned and slot number of the closest branch or %-1 + * is stored in @n; In this case calling tnc_next() is mandatory. + * o @key is so small that it is even less than the lowest key of the + * leftmost zero-level node, then %0 is returned and %0 is stored in @n. + * + * Note, when the TNC tree is traversed, some znodes may be absent, then this + * function reads corresponding indexing nodes and inserts them to TNC. In + * case of failure, a negative error code is returned. + */ +int ubifs_lookup_level0(struct ubifs_info *c, const union ubifs_key *key, + struct ubifs_znode **zn, int *n) +{ + int err, exact; + struct ubifs_znode *znode; + time64_t time = ktime_get_seconds(); + + dbg_tnck(key, "search key "); + ubifs_assert(c, key_type(c, key) < UBIFS_INVALID_KEY); + + znode = c->zroot.znode; + if (unlikely(!znode)) { + znode = ubifs_load_znode(c, &c->zroot, NULL, 0); + if (IS_ERR(znode)) + return PTR_ERR(znode); + } + + znode->time = time; + + while (1) { + struct ubifs_zbranch *zbr; + + exact = ubifs_search_zbranch(c, znode, key, n); + + if (znode->level == 0) + break; + + if (*n < 0) + *n = 0; + zbr = &znode->zbranch[*n]; + + if (zbr->znode) { + znode->time = time; + znode = zbr->znode; + continue; + } + + /* znode is not in TNC cache, load it from the media */ + znode = ubifs_load_znode(c, zbr, znode, *n); + if (IS_ERR(znode)) + return PTR_ERR(znode); + } + + *zn = znode; + if (exact || !is_hash_key(c, key) || *n != -1) { + dbg_tnc("found %d, lvl %d, n %d", exact, znode->level, *n); + return exact; + } + + /* + * Here is a tricky place. We have not found the key and this is a + * "hashed" key, which may collide. The rest of the code deals with + * situations like this: + * + * | 3 | 5 | + * / \ + * | 3 | 5 | | 6 | 7 | (x) + * + * Or more a complex example: + * + * | 1 | 5 | + * / \ + * | 1 | 3 | | 5 | 8 | + * \ / + * | 5 | 5 | | 6 | 7 | (x) + * + * In the examples, if we are looking for key "5", we may reach nodes + * marked with "(x)". In this case what we have do is to look at the + * left and see if there is "5" key there. If there is, we have to + * return it. + * + * Note, this whole situation is possible because we allow to have + * elements which are equivalent to the next key in the parent in the + * children of current znode. For example, this happens if we split a + * znode like this: | 3 | 5 | 5 | 6 | 7 |, which results in something + * like this: + * | 3 | 5 | + * / \ + * | 3 | 5 | | 5 | 6 | 7 | + * ^ + * And this becomes what is at the first "picture" after key "5" marked + * with "^" is removed. What could be done is we could prohibit + * splitting in the middle of the colliding sequence. Also, when + * removing the leftmost key, we would have to correct the key of the + * parent node, which would introduce additional complications. Namely, + * if we changed the leftmost key of the parent znode, the garbage + * collector would be unable to find it (GC is doing this when GC'ing + * indexing LEBs). Although we already have an additional RB-tree where + * we save such changed znodes (see 'ins_clr_old_idx_znode()') until + * after the commit. But anyway, this does not look easy to implement + * so we did not try this. + */ + err = tnc_prev(c, &znode, n); + if (err == -ENOENT) { + dbg_tnc("found 0, lvl %d, n -1", znode->level); + *n = -1; + return 0; + } + if (unlikely(err < 0)) + return err; + if (keys_cmp(c, key, &znode->zbranch[*n].key)) { + dbg_tnc("found 0, lvl %d, n -1", znode->level); + *n = -1; + return 0; + } + + dbg_tnc("found 1, lvl %d, n %d", znode->level, *n); + *zn = znode; + return 1; +} + +/** + * lookup_level0_dirty - search for zero-level znode dirtying. + * @c: UBIFS file-system description object + * @key: key to lookup + * @zn: znode is returned here + * @n: znode branch slot number is returned here + * + * This function looks up the TNC tree and search for zero-level znode which + * refers key @key. The found zero-level znode is returned in @zn. There are 3 + * cases: + * o exact match, i.e. the found zero-level znode contains key @key, then %1 + * is returned and slot number of the matched branch is stored in @n; + * o not exact match, which means that zero-level znode does not contain @key + * then %0 is returned and slot number of the closed branch is stored in + * @n; + * o @key is so small that it is even less than the lowest key of the + * leftmost zero-level node, then %0 is returned and %-1 is stored in @n. + * + * Additionally all znodes in the path from the root to the located zero-level + * znode are marked as dirty. + * + * Note, when the TNC tree is traversed, some znodes may be absent, then this + * function reads corresponding indexing nodes and inserts them to TNC. In + * case of failure, a negative error code is returned. + */ +static int lookup_level0_dirty(struct ubifs_info *c, const union ubifs_key *key, + struct ubifs_znode **zn, int *n) +{ + int err, exact; + struct ubifs_znode *znode; + time64_t time = ktime_get_seconds(); + + dbg_tnck(key, "search and dirty key "); + + znode = c->zroot.znode; + if (unlikely(!znode)) { + znode = ubifs_load_znode(c, &c->zroot, NULL, 0); + if (IS_ERR(znode)) + return PTR_ERR(znode); + } + + znode = dirty_cow_znode(c, &c->zroot); + if (IS_ERR(znode)) + return PTR_ERR(znode); + + znode->time = time; + + while (1) { + struct ubifs_zbranch *zbr; + + exact = ubifs_search_zbranch(c, znode, key, n); + + if (znode->level == 0) + break; + + if (*n < 0) + *n = 0; + zbr = &znode->zbranch[*n]; + + if (zbr->znode) { + znode->time = time; + znode = dirty_cow_znode(c, zbr); + if (IS_ERR(znode)) + return PTR_ERR(znode); + continue; + } + + /* znode is not in TNC cache, load it from the media */ + znode = ubifs_load_znode(c, zbr, znode, *n); + if (IS_ERR(znode)) + return PTR_ERR(znode); + znode = dirty_cow_znode(c, zbr); + if (IS_ERR(znode)) + return PTR_ERR(znode); + } + + *zn = znode; + if (exact || !is_hash_key(c, key) || *n != -1) { + dbg_tnc("found %d, lvl %d, n %d", exact, znode->level, *n); + return exact; + } + + /* + * See huge comment at 'lookup_level0_dirty()' what is the rest of the + * code. + */ + err = tnc_prev(c, &znode, n); + if (err == -ENOENT) { + *n = -1; + dbg_tnc("found 0, lvl %d, n -1", znode->level); + return 0; + } + if (unlikely(err < 0)) + return err; + if (keys_cmp(c, key, &znode->zbranch[*n].key)) { + *n = -1; + dbg_tnc("found 0, lvl %d, n -1", znode->level); + return 0; + } + + if (znode->cnext || !ubifs_zn_dirty(znode)) { + znode = dirty_cow_bottom_up(c, znode); + if (IS_ERR(znode)) + return PTR_ERR(znode); + } + + dbg_tnc("found 1, lvl %d, n %d", znode->level, *n); + *zn = znode; + return 1; +} + +/** + * maybe_leb_gced - determine if a LEB may have been garbage collected. + * @c: UBIFS file-system description object + * @lnum: LEB number + * @gc_seq1: garbage collection sequence number + * + * This function determines if @lnum may have been garbage collected since + * sequence number @gc_seq1. If it may have been then %1 is returned, otherwise + * %0 is returned. + */ +static int maybe_leb_gced(struct ubifs_info *c, int lnum, int gc_seq1) +{ + int gc_seq2, gced_lnum; + + gced_lnum = c->gced_lnum; + smp_rmb(); + gc_seq2 = c->gc_seq; + /* Same seq means no GC */ + if (gc_seq1 == gc_seq2) + return 0; + /* Different by more than 1 means we don't know */ + if (gc_seq1 + 1 != gc_seq2) + return 1; + /* + * We have seen the sequence number has increased by 1. Now we need to + * be sure we read the right LEB number, so read it again. + */ + smp_rmb(); + if (gced_lnum != c->gced_lnum) + return 1; + /* Finally we can check lnum */ + if (gced_lnum == lnum) + return 1; + return 0; +} + +/** + * ubifs_tnc_locate - look up a file-system node and return it and its location. + * @c: UBIFS file-system description object + * @key: node key to lookup + * @node: the node is returned here + * @lnum: LEB number is returned here + * @offs: offset is returned here + * + * This function looks up and reads node with key @key. The caller has to make + * sure the @node buffer is large enough to fit the node. Returns zero in case + * of success, %-ENOENT if the node was not found, and a negative error code in + * case of failure. The node location can be returned in @lnum and @offs. + */ +int ubifs_tnc_locate(struct ubifs_info *c, const union ubifs_key *key, + void *node, int *lnum, int *offs) +{ + int found, n, err, safely = 0, gc_seq1; + struct ubifs_znode *znode; + struct ubifs_zbranch zbr, *zt; + +again: + mutex_lock(&c->tnc_mutex); + found = ubifs_lookup_level0(c, key, &znode, &n); + if (!found) { + err = -ENOENT; + goto out; + } else if (found < 0) { + err = found; + goto out; + } + zt = &znode->zbranch[n]; + if (lnum) { + *lnum = zt->lnum; + *offs = zt->offs; + } + if (is_hash_key(c, key)) { + /* + * In this case the leaf node cache gets used, so we pass the + * address of the zbranch and keep the mutex locked + */ + err = tnc_read_hashed_node(c, zt, node); + goto out; + } + if (safely) { + err = ubifs_tnc_read_node(c, zt, node); + goto out; + } + /* Drop the TNC mutex prematurely and race with garbage collection */ + zbr = znode->zbranch[n]; + gc_seq1 = c->gc_seq; + mutex_unlock(&c->tnc_mutex); + + if (ubifs_get_wbuf(c, zbr.lnum)) { + /* We do not GC journal heads */ + err = ubifs_tnc_read_node(c, &zbr, node); + return err; + } + + err = fallible_read_node(c, key, &zbr, node); + if (err <= 0 || maybe_leb_gced(c, zbr.lnum, gc_seq1)) { + /* + * The node may have been GC'ed out from under us so try again + * while keeping the TNC mutex locked. + */ + safely = 1; + goto again; + } + return 0; + +out: + mutex_unlock(&c->tnc_mutex); + return err; +} + +/** + * ubifs_tnc_get_bu_keys - lookup keys for bulk-read. + * @c: UBIFS file-system description object + * @bu: bulk-read parameters and results + * + * Lookup consecutive data node keys for the same inode that reside + * consecutively in the same LEB. This function returns zero in case of success + * and a negative error code in case of failure. + * + * Note, if the bulk-read buffer length (@bu->buf_len) is known, this function + * makes sure bulk-read nodes fit the buffer. Otherwise, this function prepares + * maximum possible amount of nodes for bulk-read. + */ +int ubifs_tnc_get_bu_keys(struct ubifs_info *c, struct bu_info *bu) +{ + int n, err = 0, lnum = -1, offs; + int len; + unsigned int block = key_block(c, &bu->key); + struct ubifs_znode *znode; + + bu->cnt = 0; + bu->blk_cnt = 0; + bu->eof = 0; + + mutex_lock(&c->tnc_mutex); + /* Find first key */ + err = ubifs_lookup_level0(c, &bu->key, &znode, &n); + if (err < 0) + goto out; + if (err) { + /* Key found */ + len = znode->zbranch[n].len; + /* The buffer must be big enough for at least 1 node */ + if (len > bu->buf_len) { + err = -EINVAL; + goto out; + } + /* Add this key */ + bu->zbranch[bu->cnt++] = znode->zbranch[n]; + bu->blk_cnt += 1; + lnum = znode->zbranch[n].lnum; + offs = ALIGN(znode->zbranch[n].offs + len, 8); + } + while (1) { + struct ubifs_zbranch *zbr; + union ubifs_key *key; + unsigned int next_block; + + /* Find next key */ + err = tnc_next(c, &znode, &n); + if (err) + goto out; + zbr = &znode->zbranch[n]; + key = &zbr->key; + /* See if there is another data key for this file */ + if (key_inum(c, key) != key_inum(c, &bu->key) || + key_type(c, key) != UBIFS_DATA_KEY) { + err = -ENOENT; + goto out; + } + if (lnum < 0) { + /* First key found */ + lnum = zbr->lnum; + offs = ALIGN(zbr->offs + zbr->len, 8); + len = zbr->len; + if (len > bu->buf_len) { + err = -EINVAL; + goto out; + } + } else { + /* + * The data nodes must be in consecutive positions in + * the same LEB. + */ + if (zbr->lnum != lnum || zbr->offs != offs) + goto out; + offs += ALIGN(zbr->len, 8); + len = ALIGN(len, 8) + zbr->len; + /* Must not exceed buffer length */ + if (len > bu->buf_len) + goto out; + } + /* Allow for holes */ + next_block = key_block(c, key); + bu->blk_cnt += (next_block - block - 1); + if (bu->blk_cnt >= UBIFS_MAX_BULK_READ) + goto out; + block = next_block; + /* Add this key */ + bu->zbranch[bu->cnt++] = *zbr; + bu->blk_cnt += 1; + /* See if we have room for more */ + if (bu->cnt >= UBIFS_MAX_BULK_READ) + goto out; + if (bu->blk_cnt >= UBIFS_MAX_BULK_READ) + goto out; + } +out: + if (err == -ENOENT) { + bu->eof = 1; + err = 0; + } + bu->gc_seq = c->gc_seq; + mutex_unlock(&c->tnc_mutex); + if (err) + return err; + /* + * An enormous hole could cause bulk-read to encompass too many + * page cache pages, so limit the number here. + */ + if (bu->blk_cnt > UBIFS_MAX_BULK_READ) + bu->blk_cnt = UBIFS_MAX_BULK_READ; + /* + * Ensure that bulk-read covers a whole number of page cache + * pages. + */ + if (UBIFS_BLOCKS_PER_PAGE == 1 || + !(bu->blk_cnt & (UBIFS_BLOCKS_PER_PAGE - 1))) + return 0; + if (bu->eof) { + /* At the end of file we can round up */ + bu->blk_cnt += UBIFS_BLOCKS_PER_PAGE - 1; + return 0; + } + /* Exclude data nodes that do not make up a whole page cache page */ + block = key_block(c, &bu->key) + bu->blk_cnt; + block &= ~(UBIFS_BLOCKS_PER_PAGE - 1); + while (bu->cnt) { + if (key_block(c, &bu->zbranch[bu->cnt - 1].key) < block) + break; + bu->cnt -= 1; + } + return 0; +} + +/** + * read_wbuf - bulk-read from a LEB with a wbuf. + * @wbuf: wbuf that may overlap the read + * @buf: buffer into which to read + * @len: read length + * @lnum: LEB number from which to read + * @offs: offset from which to read + * + * This functions returns %0 on success or a negative error code on failure. + */ +static int read_wbuf(struct ubifs_wbuf *wbuf, void *buf, int len, int lnum, + int offs) +{ + const struct ubifs_info *c = wbuf->c; + int rlen, overlap; + + dbg_io("LEB %d:%d, length %d", lnum, offs, len); + ubifs_assert(c, wbuf && lnum >= 0 && lnum < c->leb_cnt && offs >= 0); + ubifs_assert(c, !(offs & 7) && offs < c->leb_size); + ubifs_assert(c, offs + len <= c->leb_size); + + spin_lock(&wbuf->lock); + overlap = (lnum == wbuf->lnum && offs + len > wbuf->offs); + if (!overlap) { + /* We may safely unlock the write-buffer and read the data */ + spin_unlock(&wbuf->lock); + return ubifs_leb_read(c, lnum, buf, offs, len, 0); + } + + /* Don't read under wbuf */ + rlen = wbuf->offs - offs; + if (rlen < 0) + rlen = 0; + + /* Copy the rest from the write-buffer */ + memcpy(buf + rlen, wbuf->buf + offs + rlen - wbuf->offs, len - rlen); + spin_unlock(&wbuf->lock); + + if (rlen > 0) + /* Read everything that goes before write-buffer */ + return ubifs_leb_read(c, lnum, buf, offs, rlen, 0); + + return 0; +} + +/** + * validate_data_node - validate data nodes for bulk-read. + * @c: UBIFS file-system description object + * @buf: buffer containing data node to validate + * @zbr: zbranch of data node to validate + * + * This functions returns %0 on success or a negative error code on failure. + */ +static int validate_data_node(struct ubifs_info *c, void *buf, + struct ubifs_zbranch *zbr) +{ + union ubifs_key key1; + struct ubifs_ch *ch = buf; + int err, len; + + if (ch->node_type != UBIFS_DATA_NODE) { + ubifs_err(c, "bad node type (%d but expected %d)", + ch->node_type, UBIFS_DATA_NODE); + goto out_err; + } + + err = ubifs_check_node(c, buf, zbr->len, zbr->lnum, zbr->offs, 0, 0); + if (err) { + ubifs_err(c, "expected node type %d", UBIFS_DATA_NODE); + goto out; + } + + err = ubifs_node_check_hash(c, buf, zbr->hash); + if (err) { + ubifs_bad_hash(c, buf, zbr->hash, zbr->lnum, zbr->offs); + return err; + } + + len = le32_to_cpu(ch->len); + if (len != zbr->len) { + ubifs_err(c, "bad node length %d, expected %d", len, zbr->len); + goto out_err; + } + + /* Make sure the key of the read node is correct */ + key_read(c, buf + UBIFS_KEY_OFFSET, &key1); + if (!keys_eq(c, &zbr->key, &key1)) { + ubifs_err(c, "bad key in node at LEB %d:%d", + zbr->lnum, zbr->offs); + dbg_tnck(&zbr->key, "looked for key "); + dbg_tnck(&key1, "found node's key "); + goto out_err; + } + + return 0; + +out_err: + err = -EINVAL; +out: + ubifs_err(c, "bad node at LEB %d:%d", zbr->lnum, zbr->offs); + ubifs_dump_node(c, buf, zbr->len); + dump_stack(); + return err; +} + +/** + * ubifs_tnc_bulk_read - read a number of data nodes in one go. + * @c: UBIFS file-system description object + * @bu: bulk-read parameters and results + * + * This functions reads and validates the data nodes that were identified by the + * 'ubifs_tnc_get_bu_keys()' function. This functions returns %0 on success, + * -EAGAIN to indicate a race with GC, or another negative error code on + * failure. + */ +int ubifs_tnc_bulk_read(struct ubifs_info *c, struct bu_info *bu) +{ + int lnum = bu->zbranch[0].lnum, offs = bu->zbranch[0].offs, len, err, i; + struct ubifs_wbuf *wbuf; + void *buf; + + len = bu->zbranch[bu->cnt - 1].offs; + len += bu->zbranch[bu->cnt - 1].len - offs; + if (len > bu->buf_len) { + ubifs_err(c, "buffer too small %d vs %d", bu->buf_len, len); + return -EINVAL; + } + + /* Do the read */ + wbuf = ubifs_get_wbuf(c, lnum); + if (wbuf) + err = read_wbuf(wbuf, bu->buf, len, lnum, offs); + else + err = ubifs_leb_read(c, lnum, bu->buf, offs, len, 0); + + /* Check for a race with GC */ + if (maybe_leb_gced(c, lnum, bu->gc_seq)) + return -EAGAIN; + + if (err && err != -EBADMSG) { + ubifs_err(c, "failed to read from LEB %d:%d, error %d", + lnum, offs, err); + dump_stack(); + dbg_tnck(&bu->key, "key "); + return err; + } + + /* Validate the nodes read */ + buf = bu->buf; + for (i = 0; i < bu->cnt; i++) { + err = validate_data_node(c, buf, &bu->zbranch[i]); + if (err) + return err; + buf = buf + ALIGN(bu->zbranch[i].len, 8); + } + + return 0; +} + +/** + * do_lookup_nm- look up a "hashed" node. + * @c: UBIFS file-system description object + * @key: node key to lookup + * @node: the node is returned here + * @nm: node name + * + * This function looks up and reads a node which contains name hash in the key. + * Since the hash may have collisions, there may be many nodes with the same + * key, so we have to sequentially look to all of them until the needed one is + * found. This function returns zero in case of success, %-ENOENT if the node + * was not found, and a negative error code in case of failure. + */ +static int do_lookup_nm(struct ubifs_info *c, const union ubifs_key *key, + void *node, const struct fscrypt_name *nm) +{ + int found, n, err; + struct ubifs_znode *znode; + + dbg_tnck(key, "key "); + mutex_lock(&c->tnc_mutex); + found = ubifs_lookup_level0(c, key, &znode, &n); + if (!found) { + err = -ENOENT; + goto out_unlock; + } else if (found < 0) { + err = found; + goto out_unlock; + } + + ubifs_assert(c, n >= 0); + + err = resolve_collision(c, key, &znode, &n, nm); + dbg_tnc("rc returned %d, znode %p, n %d", err, znode, n); + if (unlikely(err < 0)) + goto out_unlock; + if (err == 0) { + err = -ENOENT; + goto out_unlock; + } + + err = tnc_read_hashed_node(c, &znode->zbranch[n], node); + +out_unlock: + mutex_unlock(&c->tnc_mutex); + return err; +} + +/** + * ubifs_tnc_lookup_nm - look up a "hashed" node. + * @c: UBIFS file-system description object + * @key: node key to lookup + * @node: the node is returned here + * @nm: node name + * + * This function looks up and reads a node which contains name hash in the key. + * Since the hash may have collisions, there may be many nodes with the same + * key, so we have to sequentially look to all of them until the needed one is + * found. This function returns zero in case of success, %-ENOENT if the node + * was not found, and a negative error code in case of failure. + */ +int ubifs_tnc_lookup_nm(struct ubifs_info *c, const union ubifs_key *key, + void *node, const struct fscrypt_name *nm) +{ + int err, len; + const struct ubifs_dent_node *dent = node; + + /* + * We assume that in most of the cases there are no name collisions and + * 'ubifs_tnc_lookup()' returns us the right direntry. + */ + err = ubifs_tnc_lookup(c, key, node); + if (err) + return err; + + len = le16_to_cpu(dent->nlen); + if (fname_len(nm) == len && !memcmp(dent->name, fname_name(nm), len)) + return 0; + + /* + * Unluckily, there are hash collisions and we have to iterate over + * them look at each direntry with colliding name hash sequentially. + */ + + return do_lookup_nm(c, key, node, nm); +} + +static int search_dh_cookie(struct ubifs_info *c, const union ubifs_key *key, + struct ubifs_dent_node *dent, uint32_t cookie, + struct ubifs_znode **zn, int *n, int exact) +{ + int err; + struct ubifs_znode *znode = *zn; + struct ubifs_zbranch *zbr; + union ubifs_key *dkey; + + if (!exact) { + err = tnc_next(c, &znode, n); + if (err) + return err; + } + + for (;;) { + zbr = &znode->zbranch[*n]; + dkey = &zbr->key; + + if (key_inum(c, dkey) != key_inum(c, key) || + key_type(c, dkey) != key_type(c, key)) { + return -ENOENT; + } + + err = tnc_read_hashed_node(c, zbr, dent); + if (err) + return err; + + if (key_hash(c, key) == key_hash(c, dkey) && + le32_to_cpu(dent->cookie) == cookie) { + *zn = znode; + return 0; + } + + err = tnc_next(c, &znode, n); + if (err) + return err; + } +} + +static int do_lookup_dh(struct ubifs_info *c, const union ubifs_key *key, + struct ubifs_dent_node *dent, uint32_t cookie) +{ + int n, err; + struct ubifs_znode *znode; + union ubifs_key start_key; + + ubifs_assert(c, is_hash_key(c, key)); + + lowest_dent_key(c, &start_key, key_inum(c, key)); + + mutex_lock(&c->tnc_mutex); + err = ubifs_lookup_level0(c, &start_key, &znode, &n); + if (unlikely(err < 0)) + goto out_unlock; + + err = search_dh_cookie(c, key, dent, cookie, &znode, &n, err); + +out_unlock: + mutex_unlock(&c->tnc_mutex); + return err; +} + +/** + * ubifs_tnc_lookup_dh - look up a "double hashed" node. + * @c: UBIFS file-system description object + * @key: node key to lookup + * @node: the node is returned here + * @cookie: node cookie for collision resolution + * + * This function looks up and reads a node which contains name hash in the key. + * Since the hash may have collisions, there may be many nodes with the same + * key, so we have to sequentially look to all of them until the needed one + * with the same cookie value is found. + * This function returns zero in case of success, %-ENOENT if the node + * was not found, and a negative error code in case of failure. + */ +int ubifs_tnc_lookup_dh(struct ubifs_info *c, const union ubifs_key *key, + void *node, uint32_t cookie) +{ + int err; + const struct ubifs_dent_node *dent = node; + + if (!c->double_hash) + return -EOPNOTSUPP; + + /* + * We assume that in most of the cases there are no name collisions and + * 'ubifs_tnc_lookup()' returns us the right direntry. + */ + err = ubifs_tnc_lookup(c, key, node); + if (err) + return err; + + if (le32_to_cpu(dent->cookie) == cookie) + return 0; + + /* + * Unluckily, there are hash collisions and we have to iterate over + * them look at each direntry with colliding name hash sequentially. + */ + return do_lookup_dh(c, key, node, cookie); +} + +/** + * correct_parent_keys - correct parent znodes' keys. + * @c: UBIFS file-system description object + * @znode: znode to correct parent znodes for + * + * This is a helper function for 'tnc_insert()'. When the key of the leftmost + * zbranch changes, keys of parent znodes have to be corrected. This helper + * function is called in such situations and corrects the keys if needed. + */ +static void correct_parent_keys(const struct ubifs_info *c, + struct ubifs_znode *znode) +{ + union ubifs_key *key, *key1; + + ubifs_assert(c, znode->parent); + ubifs_assert(c, znode->iip == 0); + + key = &znode->zbranch[0].key; + key1 = &znode->parent->zbranch[0].key; + + while (keys_cmp(c, key, key1) < 0) { + key_copy(c, key, key1); + znode = znode->parent; + znode->alt = 1; + if (!znode->parent || znode->iip) + break; + key1 = &znode->parent->zbranch[0].key; + } +} + +/** + * insert_zbranch - insert a zbranch into a znode. + * @c: UBIFS file-system description object + * @znode: znode into which to insert + * @zbr: zbranch to insert + * @n: slot number to insert to + * + * This is a helper function for 'tnc_insert()'. UBIFS does not allow "gaps" in + * znode's array of zbranches and keeps zbranches consolidated, so when a new + * zbranch has to be inserted to the @znode->zbranches[]' array at the @n-th + * slot, zbranches starting from @n have to be moved right. + */ +static void insert_zbranch(struct ubifs_info *c, struct ubifs_znode *znode, + const struct ubifs_zbranch *zbr, int n) +{ + int i; + + ubifs_assert(c, ubifs_zn_dirty(znode)); + + if (znode->level) { + for (i = znode->child_cnt; i > n; i--) { + znode->zbranch[i] = znode->zbranch[i - 1]; + if (znode->zbranch[i].znode) + znode->zbranch[i].znode->iip = i; + } + if (zbr->znode) + zbr->znode->iip = n; + } else + for (i = znode->child_cnt; i > n; i--) + znode->zbranch[i] = znode->zbranch[i - 1]; + + znode->zbranch[n] = *zbr; + znode->child_cnt += 1; + + /* + * After inserting at slot zero, the lower bound of the key range of + * this znode may have changed. If this znode is subsequently split + * then the upper bound of the key range may change, and furthermore + * it could change to be lower than the original lower bound. If that + * happens, then it will no longer be possible to find this znode in the + * TNC using the key from the index node on flash. That is bad because + * if it is not found, we will assume it is obsolete and may overwrite + * it. Then if there is an unclean unmount, we will start using the + * old index which will be broken. + * + * So we first mark znodes that have insertions at slot zero, and then + * if they are split we add their lnum/offs to the old_idx tree. + */ + if (n == 0) + znode->alt = 1; +} + +/** + * tnc_insert - insert a node into TNC. + * @c: UBIFS file-system description object + * @znode: znode to insert into + * @zbr: branch to insert + * @n: slot number to insert new zbranch to + * + * This function inserts a new node described by @zbr into znode @znode. If + * znode does not have a free slot for new zbranch, it is split. Parent znodes + * are splat as well if needed. Returns zero in case of success or a negative + * error code in case of failure. + */ +static int tnc_insert(struct ubifs_info *c, struct ubifs_znode *znode, + struct ubifs_zbranch *zbr, int n) +{ + struct ubifs_znode *zn, *zi, *zp; + int i, keep, move, appending = 0; + union ubifs_key *key = &zbr->key, *key1; + + ubifs_assert(c, n >= 0 && n <= c->fanout); + + /* Implement naive insert for now */ +again: + zp = znode->parent; + if (znode->child_cnt < c->fanout) { + ubifs_assert(c, n != c->fanout); + dbg_tnck(key, "inserted at %d level %d, key ", n, znode->level); + + insert_zbranch(c, znode, zbr, n); + + /* Ensure parent's key is correct */ + if (n == 0 && zp && znode->iip == 0) + correct_parent_keys(c, znode); + + return 0; + } + + /* + * Unfortunately, @znode does not have more empty slots and we have to + * split it. + */ + dbg_tnck(key, "splitting level %d, key ", znode->level); + + if (znode->alt) + /* + * We can no longer be sure of finding this znode by key, so we + * record it in the old_idx tree. + */ + ins_clr_old_idx_znode(c, znode); + + zn = kzalloc(c->max_znode_sz, GFP_NOFS); + if (!zn) + return -ENOMEM; + zn->parent = zp; + zn->level = znode->level; + + /* Decide where to split */ + if (znode->level == 0 && key_type(c, key) == UBIFS_DATA_KEY) { + /* Try not to split consecutive data keys */ + if (n == c->fanout) { + key1 = &znode->zbranch[n - 1].key; + if (key_inum(c, key1) == key_inum(c, key) && + key_type(c, key1) == UBIFS_DATA_KEY) + appending = 1; + } else + goto check_split; + } else if (appending && n != c->fanout) { + /* Try not to split consecutive data keys */ + appending = 0; +check_split: + if (n >= (c->fanout + 1) / 2) { + key1 = &znode->zbranch[0].key; + if (key_inum(c, key1) == key_inum(c, key) && + key_type(c, key1) == UBIFS_DATA_KEY) { + key1 = &znode->zbranch[n].key; + if (key_inum(c, key1) != key_inum(c, key) || + key_type(c, key1) != UBIFS_DATA_KEY) { + keep = n; + move = c->fanout - keep; + zi = znode; + goto do_split; + } + } + } + } + + if (appending) { + keep = c->fanout; + move = 0; + } else { + keep = (c->fanout + 1) / 2; + move = c->fanout - keep; + } + + /* + * Although we don't at present, we could look at the neighbors and see + * if we can move some zbranches there. + */ + + if (n < keep) { + /* Insert into existing znode */ + zi = znode; + move += 1; + keep -= 1; + } else { + /* Insert into new znode */ + zi = zn; + n -= keep; + /* Re-parent */ + if (zn->level != 0) + zbr->znode->parent = zn; + } + +do_split: + + __set_bit(DIRTY_ZNODE, &zn->flags); + atomic_long_inc(&c->dirty_zn_cnt); + + zn->child_cnt = move; + znode->child_cnt = keep; + + dbg_tnc("moving %d, keeping %d", move, keep); + + /* Move zbranch */ + for (i = 0; i < move; i++) { + zn->zbranch[i] = znode->zbranch[keep + i]; + /* Re-parent */ + if (zn->level != 0) + if (zn->zbranch[i].znode) { + zn->zbranch[i].znode->parent = zn; + zn->zbranch[i].znode->iip = i; + } + } + + /* Insert new key and branch */ + dbg_tnck(key, "inserting at %d level %d, key ", n, zn->level); + + insert_zbranch(c, zi, zbr, n); + + /* Insert new znode (produced by spitting) into the parent */ + if (zp) { + if (n == 0 && zi == znode && znode->iip == 0) + correct_parent_keys(c, znode); + + /* Locate insertion point */ + n = znode->iip + 1; + + /* Tail recursion */ + zbr->key = zn->zbranch[0].key; + zbr->znode = zn; + zbr->lnum = 0; + zbr->offs = 0; + zbr->len = 0; + znode = zp; + + goto again; + } + + /* We have to split root znode */ + dbg_tnc("creating new zroot at level %d", znode->level + 1); + + zi = kzalloc(c->max_znode_sz, GFP_NOFS); + if (!zi) + return -ENOMEM; + + zi->child_cnt = 2; + zi->level = znode->level + 1; + + __set_bit(DIRTY_ZNODE, &zi->flags); + atomic_long_inc(&c->dirty_zn_cnt); + + zi->zbranch[0].key = znode->zbranch[0].key; + zi->zbranch[0].znode = znode; + zi->zbranch[0].lnum = c->zroot.lnum; + zi->zbranch[0].offs = c->zroot.offs; + zi->zbranch[0].len = c->zroot.len; + zi->zbranch[1].key = zn->zbranch[0].key; + zi->zbranch[1].znode = zn; + + c->zroot.lnum = 0; + c->zroot.offs = 0; + c->zroot.len = 0; + c->zroot.znode = zi; + + zn->parent = zi; + zn->iip = 1; + znode->parent = zi; + znode->iip = 0; + + return 0; +} + +/** + * ubifs_tnc_add - add a node to TNC. + * @c: UBIFS file-system description object + * @key: key to add + * @lnum: LEB number of node + * @offs: node offset + * @len: node length + * @hash: The hash over the node + * + * This function adds a node with key @key to TNC. The node may be new or it may + * obsolete some existing one. Returns %0 on success or negative error code on + * failure. + */ +int ubifs_tnc_add(struct ubifs_info *c, const union ubifs_key *key, int lnum, + int offs, int len, const u8 *hash) +{ + int found, n, err = 0; + struct ubifs_znode *znode; + + mutex_lock(&c->tnc_mutex); + dbg_tnck(key, "%d:%d, len %d, key ", lnum, offs, len); + found = lookup_level0_dirty(c, key, &znode, &n); + if (!found) { + struct ubifs_zbranch zbr; + + zbr.znode = NULL; + zbr.lnum = lnum; + zbr.offs = offs; + zbr.len = len; + ubifs_copy_hash(c, hash, zbr.hash); + key_copy(c, key, &zbr.key); + err = tnc_insert(c, znode, &zbr, n + 1); + } else if (found == 1) { + struct ubifs_zbranch *zbr = &znode->zbranch[n]; + + lnc_free(zbr); + err = ubifs_add_dirt(c, zbr->lnum, zbr->len); + zbr->lnum = lnum; + zbr->offs = offs; + zbr->len = len; + ubifs_copy_hash(c, hash, zbr->hash); + } else + err = found; + if (!err) + err = dbg_check_tnc(c, 0); + mutex_unlock(&c->tnc_mutex); + + return err; +} + +/** + * ubifs_tnc_replace - replace a node in the TNC only if the old node is found. + * @c: UBIFS file-system description object + * @key: key to add + * @old_lnum: LEB number of old node + * @old_offs: old node offset + * @lnum: LEB number of node + * @offs: node offset + * @len: node length + * + * This function replaces a node with key @key in the TNC only if the old node + * is found. This function is called by garbage collection when node are moved. + * Returns %0 on success or negative error code on failure. + */ +int ubifs_tnc_replace(struct ubifs_info *c, const union ubifs_key *key, + int old_lnum, int old_offs, int lnum, int offs, int len) +{ + int found, n, err = 0; + struct ubifs_znode *znode; + + mutex_lock(&c->tnc_mutex); + dbg_tnck(key, "old LEB %d:%d, new LEB %d:%d, len %d, key ", old_lnum, + old_offs, lnum, offs, len); + found = lookup_level0_dirty(c, key, &znode, &n); + if (found < 0) { + err = found; + goto out_unlock; + } + + if (found == 1) { + struct ubifs_zbranch *zbr = &znode->zbranch[n]; + + found = 0; + if (zbr->lnum == old_lnum && zbr->offs == old_offs) { + lnc_free(zbr); + err = ubifs_add_dirt(c, zbr->lnum, zbr->len); + if (err) + goto out_unlock; + zbr->lnum = lnum; + zbr->offs = offs; + zbr->len = len; + found = 1; + } else if (is_hash_key(c, key)) { + found = resolve_collision_directly(c, key, &znode, &n, + old_lnum, old_offs); + dbg_tnc("rc returned %d, znode %p, n %d, LEB %d:%d", + found, znode, n, old_lnum, old_offs); + if (found < 0) { + err = found; + goto out_unlock; + } + + if (found) { + /* Ensure the znode is dirtied */ + if (znode->cnext || !ubifs_zn_dirty(znode)) { + znode = dirty_cow_bottom_up(c, znode); + if (IS_ERR(znode)) { + err = PTR_ERR(znode); + goto out_unlock; + } + } + zbr = &znode->zbranch[n]; + lnc_free(zbr); + err = ubifs_add_dirt(c, zbr->lnum, + zbr->len); + if (err) + goto out_unlock; + zbr->lnum = lnum; + zbr->offs = offs; + zbr->len = len; + } + } + } + + if (!found) + err = ubifs_add_dirt(c, lnum, len); + + if (!err) + err = dbg_check_tnc(c, 0); + +out_unlock: + mutex_unlock(&c->tnc_mutex); + return err; +} + +/** + * ubifs_tnc_add_nm - add a "hashed" node to TNC. + * @c: UBIFS file-system description object + * @key: key to add + * @lnum: LEB number of node + * @offs: node offset + * @len: node length + * @hash: The hash over the node + * @nm: node name + * + * This is the same as 'ubifs_tnc_add()' but it should be used with keys which + * may have collisions, like directory entry keys. + */ +int ubifs_tnc_add_nm(struct ubifs_info *c, const union ubifs_key *key, + int lnum, int offs, int len, const u8 *hash, + const struct fscrypt_name *nm) +{ + int found, n, err = 0; + struct ubifs_znode *znode; + + mutex_lock(&c->tnc_mutex); + dbg_tnck(key, "LEB %d:%d, key ", lnum, offs); + found = lookup_level0_dirty(c, key, &znode, &n); + if (found < 0) { + err = found; + goto out_unlock; + } + + if (found == 1) { + if (c->replaying) + found = fallible_resolve_collision(c, key, &znode, &n, + nm, 1); + else + found = resolve_collision(c, key, &znode, &n, nm); + dbg_tnc("rc returned %d, znode %p, n %d", found, znode, n); + if (found < 0) { + err = found; + goto out_unlock; + } + + /* Ensure the znode is dirtied */ + if (znode->cnext || !ubifs_zn_dirty(znode)) { + znode = dirty_cow_bottom_up(c, znode); + if (IS_ERR(znode)) { + err = PTR_ERR(znode); + goto out_unlock; + } + } + + if (found == 1) { + struct ubifs_zbranch *zbr = &znode->zbranch[n]; + + lnc_free(zbr); + err = ubifs_add_dirt(c, zbr->lnum, zbr->len); + zbr->lnum = lnum; + zbr->offs = offs; + zbr->len = len; + ubifs_copy_hash(c, hash, zbr->hash); + goto out_unlock; + } + } + + if (!found) { + struct ubifs_zbranch zbr; + + zbr.znode = NULL; + zbr.lnum = lnum; + zbr.offs = offs; + zbr.len = len; + ubifs_copy_hash(c, hash, zbr.hash); + key_copy(c, key, &zbr.key); + err = tnc_insert(c, znode, &zbr, n + 1); + if (err) + goto out_unlock; + if (c->replaying) { + /* + * We did not find it in the index so there may be a + * dangling branch still in the index. So we remove it + * by passing 'ubifs_tnc_remove_nm()' the same key but + * an unmatchable name. + */ + struct fscrypt_name noname = { .disk_name = { .name = "", .len = 1 } }; + + err = dbg_check_tnc(c, 0); + mutex_unlock(&c->tnc_mutex); + if (err) + return err; + return ubifs_tnc_remove_nm(c, key, &noname); + } + } + +out_unlock: + if (!err) + err = dbg_check_tnc(c, 0); + mutex_unlock(&c->tnc_mutex); + return err; +} + +/** + * tnc_delete - delete a znode form TNC. + * @c: UBIFS file-system description object + * @znode: znode to delete from + * @n: zbranch slot number to delete + * + * This function deletes a leaf node from @n-th slot of @znode. Returns zero in + * case of success and a negative error code in case of failure. + */ +static int tnc_delete(struct ubifs_info *c, struct ubifs_znode *znode, int n) +{ + struct ubifs_zbranch *zbr; + struct ubifs_znode *zp; + int i, err; + + /* Delete without merge for now */ + ubifs_assert(c, znode->level == 0); + ubifs_assert(c, n >= 0 && n < c->fanout); + dbg_tnck(&znode->zbranch[n].key, "deleting key "); + + zbr = &znode->zbranch[n]; + lnc_free(zbr); + + err = ubifs_add_dirt(c, zbr->lnum, zbr->len); + if (err) { + ubifs_dump_znode(c, znode); + return err; + } + + /* We do not "gap" zbranch slots */ + for (i = n; i < znode->child_cnt - 1; i++) + znode->zbranch[i] = znode->zbranch[i + 1]; + znode->child_cnt -= 1; + + if (znode->child_cnt > 0) + return 0; + + /* + * This was the last zbranch, we have to delete this znode from the + * parent. + */ + + do { + ubifs_assert(c, !ubifs_zn_obsolete(znode)); + ubifs_assert(c, ubifs_zn_dirty(znode)); + + zp = znode->parent; + n = znode->iip; + + atomic_long_dec(&c->dirty_zn_cnt); + + err = insert_old_idx_znode(c, znode); + if (err) + return err; + + if (znode->cnext) { + __set_bit(OBSOLETE_ZNODE, &znode->flags); + atomic_long_inc(&c->clean_zn_cnt); + atomic_long_inc(&ubifs_clean_zn_cnt); + } else + kfree(znode); + znode = zp; + } while (znode->child_cnt == 1); /* while removing last child */ + + /* Remove from znode, entry n - 1 */ + znode->child_cnt -= 1; + ubifs_assert(c, znode->level != 0); + for (i = n; i < znode->child_cnt; i++) { + znode->zbranch[i] = znode->zbranch[i + 1]; + if (znode->zbranch[i].znode) + znode->zbranch[i].znode->iip = i; + } + + /* + * If this is the root and it has only 1 child then + * collapse the tree. + */ + if (!znode->parent) { + while (znode->child_cnt == 1 && znode->level != 0) { + zp = znode; + zbr = &znode->zbranch[0]; + znode = get_znode(c, znode, 0); + if (IS_ERR(znode)) + return PTR_ERR(znode); + znode = dirty_cow_znode(c, zbr); + if (IS_ERR(znode)) + return PTR_ERR(znode); + znode->parent = NULL; + znode->iip = 0; + if (c->zroot.len) { + err = insert_old_idx(c, c->zroot.lnum, + c->zroot.offs); + if (err) + return err; + } + c->zroot.lnum = zbr->lnum; + c->zroot.offs = zbr->offs; + c->zroot.len = zbr->len; + c->zroot.znode = znode; + ubifs_assert(c, !ubifs_zn_obsolete(zp)); + ubifs_assert(c, ubifs_zn_dirty(zp)); + atomic_long_dec(&c->dirty_zn_cnt); + + if (zp->cnext) { + __set_bit(OBSOLETE_ZNODE, &zp->flags); + atomic_long_inc(&c->clean_zn_cnt); + atomic_long_inc(&ubifs_clean_zn_cnt); + } else + kfree(zp); + } + } + + return 0; +} + +/** + * ubifs_tnc_remove - remove an index entry of a node. + * @c: UBIFS file-system description object + * @key: key of node + * + * Returns %0 on success or negative error code on failure. + */ +int ubifs_tnc_remove(struct ubifs_info *c, const union ubifs_key *key) +{ + int found, n, err = 0; + struct ubifs_znode *znode; + + mutex_lock(&c->tnc_mutex); + dbg_tnck(key, "key "); + found = lookup_level0_dirty(c, key, &znode, &n); + if (found < 0) { + err = found; + goto out_unlock; + } + if (found == 1) + err = tnc_delete(c, znode, n); + if (!err) + err = dbg_check_tnc(c, 0); + +out_unlock: + mutex_unlock(&c->tnc_mutex); + return err; +} + +/** + * ubifs_tnc_remove_nm - remove an index entry for a "hashed" node. + * @c: UBIFS file-system description object + * @key: key of node + * @nm: directory entry name + * + * Returns %0 on success or negative error code on failure. + */ +int ubifs_tnc_remove_nm(struct ubifs_info *c, const union ubifs_key *key, + const struct fscrypt_name *nm) +{ + int n, err; + struct ubifs_znode *znode; + + mutex_lock(&c->tnc_mutex); + dbg_tnck(key, "key "); + err = lookup_level0_dirty(c, key, &znode, &n); + if (err < 0) + goto out_unlock; + + if (err) { + if (c->replaying) + err = fallible_resolve_collision(c, key, &znode, &n, + nm, 0); + else + err = resolve_collision(c, key, &znode, &n, nm); + dbg_tnc("rc returned %d, znode %p, n %d", err, znode, n); + if (err < 0) + goto out_unlock; + if (err) { + /* Ensure the znode is dirtied */ + if (znode->cnext || !ubifs_zn_dirty(znode)) { + znode = dirty_cow_bottom_up(c, znode); + if (IS_ERR(znode)) { + err = PTR_ERR(znode); + goto out_unlock; + } + } + err = tnc_delete(c, znode, n); + } + } + +out_unlock: + if (!err) + err = dbg_check_tnc(c, 0); + mutex_unlock(&c->tnc_mutex); + return err; +} + +/** + * ubifs_tnc_remove_dh - remove an index entry for a "double hashed" node. + * @c: UBIFS file-system description object + * @key: key of node + * @cookie: node cookie for collision resolution + * + * Returns %0 on success or negative error code on failure. + */ +int ubifs_tnc_remove_dh(struct ubifs_info *c, const union ubifs_key *key, + uint32_t cookie) +{ + int n, err; + struct ubifs_znode *znode; + struct ubifs_dent_node *dent; + struct ubifs_zbranch *zbr; + + if (!c->double_hash) + return -EOPNOTSUPP; + + mutex_lock(&c->tnc_mutex); + err = lookup_level0_dirty(c, key, &znode, &n); + if (err <= 0) + goto out_unlock; + + zbr = &znode->zbranch[n]; + dent = kmalloc(UBIFS_MAX_DENT_NODE_SZ, GFP_NOFS); + if (!dent) { + err = -ENOMEM; + goto out_unlock; + } + + err = tnc_read_hashed_node(c, zbr, dent); + if (err) + goto out_free; + + /* If the cookie does not match, we're facing a hash collision. */ + if (le32_to_cpu(dent->cookie) != cookie) { + union ubifs_key start_key; + + lowest_dent_key(c, &start_key, key_inum(c, key)); + + err = ubifs_lookup_level0(c, &start_key, &znode, &n); + if (unlikely(err < 0)) + goto out_free; + + err = search_dh_cookie(c, key, dent, cookie, &znode, &n, err); + if (err) + goto out_free; + } + + if (znode->cnext || !ubifs_zn_dirty(znode)) { + znode = dirty_cow_bottom_up(c, znode); + if (IS_ERR(znode)) { + err = PTR_ERR(znode); + goto out_free; + } + } + err = tnc_delete(c, znode, n); + +out_free: + kfree(dent); +out_unlock: + if (!err) + err = dbg_check_tnc(c, 0); + mutex_unlock(&c->tnc_mutex); + return err; +} + +/** + * key_in_range - determine if a key falls within a range of keys. + * @c: UBIFS file-system description object + * @key: key to check + * @from_key: lowest key in range + * @to_key: highest key in range + * + * This function returns %1 if the key is in range and %0 otherwise. + */ +static int key_in_range(struct ubifs_info *c, union ubifs_key *key, + union ubifs_key *from_key, union ubifs_key *to_key) +{ + if (keys_cmp(c, key, from_key) < 0) + return 0; + if (keys_cmp(c, key, to_key) > 0) + return 0; + return 1; +} + +/** + * ubifs_tnc_remove_range - remove index entries in range. + * @c: UBIFS file-system description object + * @from_key: lowest key to remove + * @to_key: highest key to remove + * + * This function removes index entries starting at @from_key and ending at + * @to_key. This function returns zero in case of success and a negative error + * code in case of failure. + */ +int ubifs_tnc_remove_range(struct ubifs_info *c, union ubifs_key *from_key, + union ubifs_key *to_key) +{ + int i, n, k, err = 0; + struct ubifs_znode *znode; + union ubifs_key *key; + + mutex_lock(&c->tnc_mutex); + while (1) { + /* Find first level 0 znode that contains keys to remove */ + err = ubifs_lookup_level0(c, from_key, &znode, &n); + if (err < 0) + goto out_unlock; + + if (err) + key = from_key; + else { + err = tnc_next(c, &znode, &n); + if (err == -ENOENT) { + err = 0; + goto out_unlock; + } + if (err < 0) + goto out_unlock; + key = &znode->zbranch[n].key; + if (!key_in_range(c, key, from_key, to_key)) { + err = 0; + goto out_unlock; + } + } + + /* Ensure the znode is dirtied */ + if (znode->cnext || !ubifs_zn_dirty(znode)) { + znode = dirty_cow_bottom_up(c, znode); + if (IS_ERR(znode)) { + err = PTR_ERR(znode); + goto out_unlock; + } + } + + /* Remove all keys in range except the first */ + for (i = n + 1, k = 0; i < znode->child_cnt; i++, k++) { + key = &znode->zbranch[i].key; + if (!key_in_range(c, key, from_key, to_key)) + break; + lnc_free(&znode->zbranch[i]); + err = ubifs_add_dirt(c, znode->zbranch[i].lnum, + znode->zbranch[i].len); + if (err) { + ubifs_dump_znode(c, znode); + goto out_unlock; + } + dbg_tnck(key, "removing key "); + } + if (k) { + for (i = n + 1 + k; i < znode->child_cnt; i++) + znode->zbranch[i - k] = znode->zbranch[i]; + znode->child_cnt -= k; + } + + /* Now delete the first */ + err = tnc_delete(c, znode, n); + if (err) + goto out_unlock; + } + +out_unlock: + if (!err) + err = dbg_check_tnc(c, 0); + mutex_unlock(&c->tnc_mutex); + return err; +} + +/** + * ubifs_tnc_remove_ino - remove an inode from TNC. + * @c: UBIFS file-system description object + * @inum: inode number to remove + * + * This function remove inode @inum and all the extended attributes associated + * with the anode from TNC and returns zero in case of success or a negative + * error code in case of failure. + */ +int ubifs_tnc_remove_ino(struct ubifs_info *c, ino_t inum) +{ + union ubifs_key key1, key2; + struct ubifs_dent_node *xent, *pxent = NULL; + struct fscrypt_name nm = {0}; + + dbg_tnc("ino %lu", (unsigned long)inum); + + /* + * Walk all extended attribute entries and remove them together with + * corresponding extended attribute inodes. + */ + lowest_xent_key(c, &key1, inum); + while (1) { + ino_t xattr_inum; + int err; + + xent = ubifs_tnc_next_ent(c, &key1, &nm); + if (IS_ERR(xent)) { + err = PTR_ERR(xent); + if (err == -ENOENT) + break; + kfree(pxent); + return err; + } + + xattr_inum = le64_to_cpu(xent->inum); + dbg_tnc("xent '%s', ino %lu", xent->name, + (unsigned long)xattr_inum); + + fname_name(&nm) = xent->name; + fname_len(&nm) = le16_to_cpu(xent->nlen); + err = ubifs_tnc_remove_nm(c, &key1, &nm); + if (err) { + kfree(pxent); + kfree(xent); + return err; + } + + lowest_ino_key(c, &key1, xattr_inum); + highest_ino_key(c, &key2, xattr_inum); + err = ubifs_tnc_remove_range(c, &key1, &key2); + if (err) { + kfree(pxent); + kfree(xent); + return err; + } + + kfree(pxent); + pxent = xent; + key_read(c, &xent->key, &key1); + } + + kfree(pxent); + lowest_ino_key(c, &key1, inum); + highest_ino_key(c, &key2, inum); + + return ubifs_tnc_remove_range(c, &key1, &key2); +} + +/** + * ubifs_tnc_next_ent - walk directory or extended attribute entries. + * @c: UBIFS file-system description object + * @key: key of last entry + * @nm: name of last entry found or %NULL + * + * This function finds and reads the next directory or extended attribute entry + * after the given key (@key) if there is one. @nm is used to resolve + * collisions. + * + * If the name of the current entry is not known and only the key is known, + * @nm->name has to be %NULL. In this case the semantics of this function is a + * little bit different and it returns the entry corresponding to this key, not + * the next one. If the key was not found, the closest "right" entry is + * returned. + * + * If the fist entry has to be found, @key has to contain the lowest possible + * key value for this inode and @name has to be %NULL. + * + * This function returns the found directory or extended attribute entry node + * in case of success, %-ENOENT is returned if no entry was found, and a + * negative error code is returned in case of failure. + */ +struct ubifs_dent_node *ubifs_tnc_next_ent(struct ubifs_info *c, + union ubifs_key *key, + const struct fscrypt_name *nm) +{ + int n, err, type = key_type(c, key); + struct ubifs_znode *znode; + struct ubifs_dent_node *dent; + struct ubifs_zbranch *zbr; + union ubifs_key *dkey; + + dbg_tnck(key, "key "); + ubifs_assert(c, is_hash_key(c, key)); + + mutex_lock(&c->tnc_mutex); + err = ubifs_lookup_level0(c, key, &znode, &n); + if (unlikely(err < 0)) + goto out_unlock; + + if (fname_len(nm) > 0) { + if (err) { + /* Handle collisions */ + if (c->replaying) + err = fallible_resolve_collision(c, key, &znode, &n, + nm, 0); + else + err = resolve_collision(c, key, &znode, &n, nm); + dbg_tnc("rc returned %d, znode %p, n %d", + err, znode, n); + if (unlikely(err < 0)) + goto out_unlock; + } + + /* Now find next entry */ + err = tnc_next(c, &znode, &n); + if (unlikely(err)) + goto out_unlock; + } else { + /* + * The full name of the entry was not given, in which case the + * behavior of this function is a little different and it + * returns current entry, not the next one. + */ + if (!err) { + /* + * However, the given key does not exist in the TNC + * tree and @znode/@n variables contain the closest + * "preceding" element. Switch to the next one. + */ + err = tnc_next(c, &znode, &n); + if (err) + goto out_unlock; + } + } + + zbr = &znode->zbranch[n]; + dent = kmalloc(zbr->len, GFP_NOFS); + if (unlikely(!dent)) { + err = -ENOMEM; + goto out_unlock; + } + + /* + * The above 'tnc_next()' call could lead us to the next inode, check + * this. + */ + dkey = &zbr->key; + if (key_inum(c, dkey) != key_inum(c, key) || + key_type(c, dkey) != type) { + err = -ENOENT; + goto out_free; + } + + err = tnc_read_hashed_node(c, zbr, dent); + if (unlikely(err)) + goto out_free; + + mutex_unlock(&c->tnc_mutex); + return dent; + +out_free: + kfree(dent); +out_unlock: + mutex_unlock(&c->tnc_mutex); + return ERR_PTR(err); +} + +/** + * tnc_destroy_cnext - destroy left-over obsolete znodes from a failed commit. + * @c: UBIFS file-system description object + * + * Destroy left-over obsolete znodes from a failed commit. + */ +static void tnc_destroy_cnext(struct ubifs_info *c) +{ + struct ubifs_znode *cnext; + + if (!c->cnext) + return; + ubifs_assert(c, c->cmt_state == COMMIT_BROKEN); + cnext = c->cnext; + do { + struct ubifs_znode *znode = cnext; + + cnext = cnext->cnext; + if (ubifs_zn_obsolete(znode)) + kfree(znode); + else if (!ubifs_zn_cow(znode)) { + /* + * Don't forget to update clean znode count after + * committing failed, because ubifs will check this + * count while closing tnc. Non-obsolete znode could + * be re-dirtied during committing process, so dirty + * flag is untrustable. The flag 'COW_ZNODE' is set + * for each dirty znode before committing, and it is + * cleared as long as the znode become clean, so we + * can statistic clean znode count according to this + * flag. + */ + atomic_long_inc(&c->clean_zn_cnt); + atomic_long_inc(&ubifs_clean_zn_cnt); + } + } while (cnext && cnext != c->cnext); +} + +/** + * ubifs_tnc_close - close TNC subsystem and free all related resources. + * @c: UBIFS file-system description object + */ +void ubifs_tnc_close(struct ubifs_info *c) +{ + tnc_destroy_cnext(c); + ubifs_destroy_tnc_tree(c); + kfree(c->gap_lebs); + kfree(c->ilebs); + destroy_old_idx(c); +} + +/** + * left_znode - get the znode to the left. + * @c: UBIFS file-system description object + * @znode: znode + * + * This function returns a pointer to the znode to the left of @znode or NULL if + * there is not one. A negative error code is returned on failure. + */ +static struct ubifs_znode *left_znode(struct ubifs_info *c, + struct ubifs_znode *znode) +{ + int level = znode->level; + + while (1) { + int n = znode->iip - 1; + + /* Go up until we can go left */ + znode = znode->parent; + if (!znode) + return NULL; + if (n >= 0) { + /* Now go down the rightmost branch to 'level' */ + znode = get_znode(c, znode, n); + if (IS_ERR(znode)) + return znode; + while (znode->level != level) { + n = znode->child_cnt - 1; + znode = get_znode(c, znode, n); + if (IS_ERR(znode)) + return znode; + } + break; + } + } + return znode; +} + +/** + * right_znode - get the znode to the right. + * @c: UBIFS file-system description object + * @znode: znode + * + * This function returns a pointer to the znode to the right of @znode or NULL + * if there is not one. A negative error code is returned on failure. + */ +static struct ubifs_znode *right_znode(struct ubifs_info *c, + struct ubifs_znode *znode) +{ + int level = znode->level; + + while (1) { + int n = znode->iip + 1; + + /* Go up until we can go right */ + znode = znode->parent; + if (!znode) + return NULL; + if (n < znode->child_cnt) { + /* Now go down the leftmost branch to 'level' */ + znode = get_znode(c, znode, n); + if (IS_ERR(znode)) + return znode; + while (znode->level != level) { + znode = get_znode(c, znode, 0); + if (IS_ERR(znode)) + return znode; + } + break; + } + } + return znode; +} + +/** + * lookup_znode - find a particular indexing node from TNC. + * @c: UBIFS file-system description object + * @key: index node key to lookup + * @level: index node level + * @lnum: index node LEB number + * @offs: index node offset + * + * This function searches an indexing node by its first key @key and its + * address @lnum:@offs. It looks up the indexing tree by pulling all indexing + * nodes it traverses to TNC. This function is called for indexing nodes which + * were found on the media by scanning, for example when garbage-collecting or + * when doing in-the-gaps commit. This means that the indexing node which is + * looked for does not have to have exactly the same leftmost key @key, because + * the leftmost key may have been changed, in which case TNC will contain a + * dirty znode which still refers the same @lnum:@offs. This function is clever + * enough to recognize such indexing nodes. + * + * Note, if a znode was deleted or changed too much, then this function will + * not find it. For situations like this UBIFS has the old index RB-tree + * (indexed by @lnum:@offs). + * + * This function returns a pointer to the znode found or %NULL if it is not + * found. A negative error code is returned on failure. + */ +static struct ubifs_znode *lookup_znode(struct ubifs_info *c, + union ubifs_key *key, int level, + int lnum, int offs) +{ + struct ubifs_znode *znode, *zn; + int n, nn; + + ubifs_assert(c, key_type(c, key) < UBIFS_INVALID_KEY); + + /* + * The arguments have probably been read off flash, so don't assume + * they are valid. + */ + if (level < 0) + return ERR_PTR(-EINVAL); + + /* Get the root znode */ + znode = c->zroot.znode; + if (!znode) { + znode = ubifs_load_znode(c, &c->zroot, NULL, 0); + if (IS_ERR(znode)) + return znode; + } + /* Check if it is the one we are looking for */ + if (c->zroot.lnum == lnum && c->zroot.offs == offs) + return znode; + /* Descend to the parent level i.e. (level + 1) */ + if (level >= znode->level) + return NULL; + while (1) { + ubifs_search_zbranch(c, znode, key, &n); + if (n < 0) { + /* + * We reached a znode where the leftmost key is greater + * than the key we are searching for. This is the same + * situation as the one described in a huge comment at + * the end of the 'ubifs_lookup_level0()' function. And + * for exactly the same reasons we have to try to look + * left before giving up. + */ + znode = left_znode(c, znode); + if (!znode) + return NULL; + if (IS_ERR(znode)) + return znode; + ubifs_search_zbranch(c, znode, key, &n); + ubifs_assert(c, n >= 0); + } + if (znode->level == level + 1) + break; + znode = get_znode(c, znode, n); + if (IS_ERR(znode)) + return znode; + } + /* Check if the child is the one we are looking for */ + if (znode->zbranch[n].lnum == lnum && znode->zbranch[n].offs == offs) + return get_znode(c, znode, n); + /* If the key is unique, there is nowhere else to look */ + if (!is_hash_key(c, key)) + return NULL; + /* + * The key is not unique and so may be also in the znodes to either + * side. + */ + zn = znode; + nn = n; + /* Look left */ + while (1) { + /* Move one branch to the left */ + if (n) + n -= 1; + else { + znode = left_znode(c, znode); + if (!znode) + break; + if (IS_ERR(znode)) + return znode; + n = znode->child_cnt - 1; + } + /* Check it */ + if (znode->zbranch[n].lnum == lnum && + znode->zbranch[n].offs == offs) + return get_znode(c, znode, n); + /* Stop if the key is less than the one we are looking for */ + if (keys_cmp(c, &znode->zbranch[n].key, key) < 0) + break; + } + /* Back to the middle */ + znode = zn; + n = nn; + /* Look right */ + while (1) { + /* Move one branch to the right */ + if (++n >= znode->child_cnt) { + znode = right_znode(c, znode); + if (!znode) + break; + if (IS_ERR(znode)) + return znode; + n = 0; + } + /* Check it */ + if (znode->zbranch[n].lnum == lnum && + znode->zbranch[n].offs == offs) + return get_znode(c, znode, n); + /* Stop if the key is greater than the one we are looking for */ + if (keys_cmp(c, &znode->zbranch[n].key, key) > 0) + break; + } + return NULL; +} + +/** + * is_idx_node_in_tnc - determine if an index node is in the TNC. + * @c: UBIFS file-system description object + * @key: key of index node + * @level: index node level + * @lnum: LEB number of index node + * @offs: offset of index node + * + * This function returns %0 if the index node is not referred to in the TNC, %1 + * if the index node is referred to in the TNC and the corresponding znode is + * dirty, %2 if an index node is referred to in the TNC and the corresponding + * znode is clean, and a negative error code in case of failure. + * + * Note, the @key argument has to be the key of the first child. Also note, + * this function relies on the fact that 0:0 is never a valid LEB number and + * offset for a main-area node. + */ +int is_idx_node_in_tnc(struct ubifs_info *c, union ubifs_key *key, int level, + int lnum, int offs) +{ + struct ubifs_znode *znode; + + znode = lookup_znode(c, key, level, lnum, offs); + if (!znode) + return 0; + if (IS_ERR(znode)) + return PTR_ERR(znode); + + return ubifs_zn_dirty(znode) ? 1 : 2; +} + +/** + * is_leaf_node_in_tnc - determine if a non-indexing not is in the TNC. + * @c: UBIFS file-system description object + * @key: node key + * @lnum: node LEB number + * @offs: node offset + * + * This function returns %1 if the node is referred to in the TNC, %0 if it is + * not, and a negative error code in case of failure. + * + * Note, this function relies on the fact that 0:0 is never a valid LEB number + * and offset for a main-area node. + */ +static int is_leaf_node_in_tnc(struct ubifs_info *c, union ubifs_key *key, + int lnum, int offs) +{ + struct ubifs_zbranch *zbr; + struct ubifs_znode *znode, *zn; + int n, found, err, nn; + const int unique = !is_hash_key(c, key); + + found = ubifs_lookup_level0(c, key, &znode, &n); + if (found < 0) + return found; /* Error code */ + if (!found) + return 0; + zbr = &znode->zbranch[n]; + if (lnum == zbr->lnum && offs == zbr->offs) + return 1; /* Found it */ + if (unique) + return 0; + /* + * Because the key is not unique, we have to look left + * and right as well + */ + zn = znode; + nn = n; + /* Look left */ + while (1) { + err = tnc_prev(c, &znode, &n); + if (err == -ENOENT) + break; + if (err) + return err; + if (keys_cmp(c, key, &znode->zbranch[n].key)) + break; + zbr = &znode->zbranch[n]; + if (lnum == zbr->lnum && offs == zbr->offs) + return 1; /* Found it */ + } + /* Look right */ + znode = zn; + n = nn; + while (1) { + err = tnc_next(c, &znode, &n); + if (err) { + if (err == -ENOENT) + return 0; + return err; + } + if (keys_cmp(c, key, &znode->zbranch[n].key)) + break; + zbr = &znode->zbranch[n]; + if (lnum == zbr->lnum && offs == zbr->offs) + return 1; /* Found it */ + } + return 0; +} + +/** + * ubifs_tnc_has_node - determine whether a node is in the TNC. + * @c: UBIFS file-system description object + * @key: node key + * @level: index node level (if it is an index node) + * @lnum: node LEB number + * @offs: node offset + * @is_idx: non-zero if the node is an index node + * + * This function returns %1 if the node is in the TNC, %0 if it is not, and a + * negative error code in case of failure. For index nodes, @key has to be the + * key of the first child. An index node is considered to be in the TNC only if + * the corresponding znode is clean or has not been loaded. + */ +int ubifs_tnc_has_node(struct ubifs_info *c, union ubifs_key *key, int level, + int lnum, int offs, int is_idx) +{ + int err; + + mutex_lock(&c->tnc_mutex); + if (is_idx) { + err = is_idx_node_in_tnc(c, key, level, lnum, offs); + if (err < 0) + goto out_unlock; + if (err == 1) + /* The index node was found but it was dirty */ + err = 0; + else if (err == 2) + /* The index node was found and it was clean */ + err = 1; + else + BUG_ON(err != 0); + } else + err = is_leaf_node_in_tnc(c, key, lnum, offs); + +out_unlock: + mutex_unlock(&c->tnc_mutex); + return err; +} + +/** + * ubifs_dirty_idx_node - dirty an index node. + * @c: UBIFS file-system description object + * @key: index node key + * @level: index node level + * @lnum: index node LEB number + * @offs: index node offset + * + * This function loads and dirties an index node so that it can be garbage + * collected. The @key argument has to be the key of the first child. This + * function relies on the fact that 0:0 is never a valid LEB number and offset + * for a main-area node. Returns %0 on success and a negative error code on + * failure. + */ +int ubifs_dirty_idx_node(struct ubifs_info *c, union ubifs_key *key, int level, + int lnum, int offs) +{ + struct ubifs_znode *znode; + int err = 0; + + mutex_lock(&c->tnc_mutex); + znode = lookup_znode(c, key, level, lnum, offs); + if (!znode) + goto out_unlock; + if (IS_ERR(znode)) { + err = PTR_ERR(znode); + goto out_unlock; + } + znode = dirty_cow_bottom_up(c, znode); + if (IS_ERR(znode)) { + err = PTR_ERR(znode); + goto out_unlock; + } + +out_unlock: + mutex_unlock(&c->tnc_mutex); + return err; +} + +/** + * dbg_check_inode_size - check if inode size is correct. + * @c: UBIFS file-system description object + * @inode: inode to check + * @size: inode size + * + * This function makes sure that the inode size (@size) is correct and it does + * not have any pages beyond @size. Returns zero if the inode is OK, %-EINVAL + * if it has a data page beyond @size, and other negative error code in case of + * other errors. + */ +int dbg_check_inode_size(struct ubifs_info *c, const struct inode *inode, + loff_t size) +{ + int err, n; + union ubifs_key from_key, to_key, *key; + struct ubifs_znode *znode; + unsigned int block; + + if (!S_ISREG(inode->i_mode)) + return 0; + if (!dbg_is_chk_gen(c)) + return 0; + + block = (size + UBIFS_BLOCK_SIZE - 1) >> UBIFS_BLOCK_SHIFT; + data_key_init(c, &from_key, inode->i_ino, block); + highest_data_key(c, &to_key, inode->i_ino); + + mutex_lock(&c->tnc_mutex); + err = ubifs_lookup_level0(c, &from_key, &znode, &n); + if (err < 0) + goto out_unlock; + + if (err) { + key = &from_key; + goto out_dump; + } + + err = tnc_next(c, &znode, &n); + if (err == -ENOENT) { + err = 0; + goto out_unlock; + } + if (err < 0) + goto out_unlock; + + ubifs_assert(c, err == 0); + key = &znode->zbranch[n].key; + if (!key_in_range(c, key, &from_key, &to_key)) + goto out_unlock; + +out_dump: + block = key_block(c, key); + ubifs_err(c, "inode %llu has size %lld, but there are data at offset %lld", + inode->i_ino, size, + ((loff_t)block) << UBIFS_BLOCK_SHIFT); + mutex_unlock(&c->tnc_mutex); + ubifs_dump_inode(c, inode); + dump_stack(); + return -EINVAL; + +out_unlock: + mutex_unlock(&c->tnc_mutex); + return err; +} diff --git a/fs/ubifs/tnc_commit.c b/fs/ubifs/tnc_commit.c new file mode 100644 index 000000000..7bf267206 --- /dev/null +++ b/fs/ubifs/tnc_commit.c @@ -0,0 +1,1112 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * + * Authors: Adrian Hunter + * Artem Bityutskiy (Битюцкий Артём) + */ + +/* This file implements TNC functions for committing */ + +#include +#include "ubifs.h" + +/** + * make_idx_node - make an index node for fill-the-gaps method of TNC commit. + * @c: UBIFS file-system description object + * @idx: buffer in which to place new index node + * @znode: znode from which to make new index node + * @lnum: LEB number where new index node will be written + * @offs: offset where new index node will be written + * @len: length of new index node + */ +static int make_idx_node(struct ubifs_info *c, struct ubifs_idx_node *idx, + struct ubifs_znode *znode, int lnum, int offs, int len) +{ + struct ubifs_znode *zp; + u8 hash[UBIFS_HASH_ARR_SZ]; + int i, err; + + /* Make index node */ + idx->ch.node_type = UBIFS_IDX_NODE; + idx->child_cnt = cpu_to_le16(znode->child_cnt); + idx->level = cpu_to_le16(znode->level); + for (i = 0; i < znode->child_cnt; i++) { + struct ubifs_branch *br = ubifs_idx_branch(c, idx, i); + struct ubifs_zbranch *zbr = &znode->zbranch[i]; + + key_write_idx(c, &zbr->key, &br->key); + br->lnum = cpu_to_le32(zbr->lnum); + br->offs = cpu_to_le32(zbr->offs); + br->len = cpu_to_le32(zbr->len); + ubifs_copy_hash(c, zbr->hash, ubifs_branch_hash(c, br)); + if (!zbr->lnum || !zbr->len) { + ubifs_err(c, "bad ref in znode"); + ubifs_dump_znode(c, znode); + if (zbr->znode) + ubifs_dump_znode(c, zbr->znode); + + return -EINVAL; + } + } + ubifs_prepare_node(c, idx, len, 0); + ubifs_node_calc_hash(c, idx, hash); + + znode->lnum = lnum; + znode->offs = offs; + znode->len = len; + + err = insert_old_idx_znode(c, znode); + + /* Update the parent */ + zp = znode->parent; + if (zp) { + struct ubifs_zbranch *zbr; + + zbr = &zp->zbranch[znode->iip]; + zbr->lnum = lnum; + zbr->offs = offs; + zbr->len = len; + ubifs_copy_hash(c, hash, zbr->hash); + } else { + c->zroot.lnum = lnum; + c->zroot.offs = offs; + c->zroot.len = len; + ubifs_copy_hash(c, hash, c->zroot.hash); + } + c->calc_idx_sz += ALIGN(len, 8); + + atomic_long_dec(&c->dirty_zn_cnt); + + ubifs_assert(c, ubifs_zn_dirty(znode)); + ubifs_assert(c, ubifs_zn_cow(znode)); + + /* + * Note, unlike 'write_index()' we do not add memory barriers here + * because this function is called with @c->tnc_mutex locked. + */ + __clear_bit(DIRTY_ZNODE, &znode->flags); + __clear_bit(COW_ZNODE, &znode->flags); + + return err; +} + +/** + * fill_gap - make index nodes in gaps in dirty index LEBs. + * @c: UBIFS file-system description object + * @lnum: LEB number that gap appears in + * @gap_start: offset of start of gap + * @gap_end: offset of end of gap + * @dirt: adds dirty space to this + * + * This function returns the number of index nodes written into the gap. + */ +static int fill_gap(struct ubifs_info *c, int lnum, int gap_start, int gap_end, + int *dirt) +{ + int len, gap_remains, gap_pos, written, pad_len; + + ubifs_assert(c, (gap_start & 7) == 0); + ubifs_assert(c, (gap_end & 7) == 0); + ubifs_assert(c, gap_end >= gap_start); + + gap_remains = gap_end - gap_start; + if (!gap_remains) + return 0; + gap_pos = gap_start; + written = 0; + while (c->enext) { + len = ubifs_idx_node_sz(c, c->enext->child_cnt); + if (len < gap_remains) { + struct ubifs_znode *znode = c->enext; + const int alen = ALIGN(len, 8); + int err; + + ubifs_assert(c, alen <= gap_remains); + err = make_idx_node(c, c->ileb_buf + gap_pos, znode, + lnum, gap_pos, len); + if (err) + return err; + gap_remains -= alen; + gap_pos += alen; + c->enext = znode->cnext; + if (c->enext == c->cnext) + c->enext = NULL; + written += 1; + } else + break; + } + if (gap_end == c->leb_size) { + c->ileb_len = ALIGN(gap_pos, c->min_io_size); + /* Pad to end of min_io_size */ + pad_len = c->ileb_len - gap_pos; + } else + /* Pad to end of gap */ + pad_len = gap_remains; + dbg_gc("LEB %d:%d to %d len %d nodes written %d wasted bytes %d", + lnum, gap_start, gap_end, gap_end - gap_start, written, pad_len); + ubifs_pad(c, c->ileb_buf + gap_pos, pad_len); + *dirt += pad_len; + return written; +} + +/** + * find_old_idx - find an index node obsoleted since the last commit start. + * @c: UBIFS file-system description object + * @lnum: LEB number of obsoleted index node + * @offs: offset of obsoleted index node + * + * Returns %1 if found and %0 otherwise. + */ +static int find_old_idx(struct ubifs_info *c, int lnum, int offs) +{ + struct ubifs_old_idx *o; + struct rb_node *p; + + p = c->old_idx.rb_node; + while (p) { + o = rb_entry(p, struct ubifs_old_idx, rb); + if (lnum < o->lnum) + p = p->rb_left; + else if (lnum > o->lnum) + p = p->rb_right; + else if (offs < o->offs) + p = p->rb_left; + else if (offs > o->offs) + p = p->rb_right; + else + return 1; + } + return 0; +} + +/** + * is_idx_node_in_use - determine if an index node can be overwritten. + * @c: UBIFS file-system description object + * @key: key of index node + * @level: index node level + * @lnum: LEB number of index node + * @offs: offset of index node + * + * If @key / @lnum / @offs identify an index node that was not part of the old + * index, then this function returns %0 (obsolete). Else if the index node was + * part of the old index but is now dirty %1 is returned, else if it is clean %2 + * is returned. A negative error code is returned on failure. + */ +static int is_idx_node_in_use(struct ubifs_info *c, union ubifs_key *key, + int level, int lnum, int offs) +{ + int ret; + + ret = is_idx_node_in_tnc(c, key, level, lnum, offs); + if (ret < 0) + return ret; /* Error code */ + if (ret == 0) + if (find_old_idx(c, lnum, offs)) + return 1; + return ret; +} + +/** + * layout_leb_in_gaps - layout index nodes using in-the-gaps method. + * @c: UBIFS file-system description object + * @p: return LEB number in @c->gap_lebs[p] + * + * This function lays out new index nodes for dirty znodes using in-the-gaps + * method of TNC commit. + * This function merely puts the next znode into the next gap, making no attempt + * to try to maximise the number of znodes that fit. + * This function returns the number of index nodes written into the gaps, or a + * negative error code on failure. + */ +static int layout_leb_in_gaps(struct ubifs_info *c, int p) +{ + struct ubifs_scan_leb *sleb; + struct ubifs_scan_node *snod; + int lnum, dirt = 0, gap_start, gap_end, err, written, tot_written; + + tot_written = 0; + /* Get an index LEB with lots of obsolete index nodes */ + lnum = ubifs_find_dirty_idx_leb(c); + if (lnum < 0) + /* + * There also may be dirt in the index head that could be + * filled, however we do not check there at present. + */ + return lnum; /* Error code */ + c->gap_lebs[p] = lnum; + dbg_gc("LEB %d", lnum); + /* + * Scan the index LEB. We use the generic scan for this even though + * it is more comprehensive and less efficient than is needed for this + * purpose. + */ + sleb = ubifs_scan(c, lnum, 0, c->ileb_buf, 0); + c->ileb_len = 0; + if (IS_ERR(sleb)) + return PTR_ERR(sleb); + gap_start = 0; + list_for_each_entry(snod, &sleb->nodes, list) { + struct ubifs_idx_node *idx; + int in_use, level; + + ubifs_assert(c, snod->type == UBIFS_IDX_NODE); + idx = snod->node; + key_read(c, ubifs_idx_key(c, idx), &snod->key); + level = le16_to_cpu(idx->level); + /* Determine if the index node is in use (not obsolete) */ + in_use = is_idx_node_in_use(c, &snod->key, level, lnum, + snod->offs); + if (in_use < 0) { + ubifs_scan_destroy(sleb); + return in_use; /* Error code */ + } + if (in_use) { + if (in_use == 1) + dirt += ALIGN(snod->len, 8); + /* + * The obsolete index nodes form gaps that can be + * overwritten. This gap has ended because we have + * found an index node that is still in use + * i.e. not obsolete + */ + gap_end = snod->offs; + /* Try to fill gap */ + written = fill_gap(c, lnum, gap_start, gap_end, &dirt); + if (written < 0) { + ubifs_scan_destroy(sleb); + return written; /* Error code */ + } + tot_written += written; + gap_start = ALIGN(snod->offs + snod->len, 8); + } + } + ubifs_scan_destroy(sleb); + c->ileb_len = c->leb_size; + gap_end = c->leb_size; + /* Try to fill gap */ + written = fill_gap(c, lnum, gap_start, gap_end, &dirt); + if (written < 0) + return written; /* Error code */ + tot_written += written; + if (tot_written == 0) { + struct ubifs_lprops lp; + + dbg_gc("LEB %d wrote %d index nodes", lnum, tot_written); + err = ubifs_read_one_lp(c, lnum, &lp); + if (err) + return err; + if (lp.free == c->leb_size) { + /* + * We must have snatched this LEB from the idx_gc list + * so we need to correct the free and dirty space. + */ + err = ubifs_change_one_lp(c, lnum, + c->leb_size - c->ileb_len, + dirt, 0, 0, 0); + if (err) + return err; + } + return 0; + } + err = ubifs_change_one_lp(c, lnum, c->leb_size - c->ileb_len, dirt, + 0, 0, 0); + if (err) + return err; + err = ubifs_leb_change(c, lnum, c->ileb_buf, c->ileb_len); + if (err) + return err; + dbg_gc("LEB %d wrote %d index nodes", lnum, tot_written); + return tot_written; +} + +/** + * get_leb_cnt - calculate the number of empty LEBs needed to commit. + * @c: UBIFS file-system description object + * @cnt: number of znodes to commit + * + * This function returns the number of empty LEBs needed to commit @cnt znodes + * to the current index head. The number is not exact and may be more than + * needed. + */ +static int get_leb_cnt(struct ubifs_info *c, int cnt) +{ + int d; + + /* Assume maximum index node size (i.e. overestimate space needed) */ + cnt -= (c->leb_size - c->ihead_offs) / c->max_idx_node_sz; + if (cnt < 0) + cnt = 0; + d = c->leb_size / c->max_idx_node_sz; + return DIV_ROUND_UP(cnt, d); +} + +/** + * layout_in_gaps - in-the-gaps method of committing TNC. + * @c: UBIFS file-system description object + * @cnt: number of dirty znodes to commit. + * + * This function lays out new index nodes for dirty znodes using in-the-gaps + * method of TNC commit. + * + * This function returns %0 on success and a negative error code on failure. + */ +static int layout_in_gaps(struct ubifs_info *c, int cnt) +{ + int err, leb_needed_cnt, written, p = 0, old_idx_lebs, *gap_lebs; + + dbg_gc("%d znodes to write", cnt); + + c->gap_lebs = kmalloc_objs(int, c->lst.idx_lebs + 1, GFP_NOFS); + if (!c->gap_lebs) + return -ENOMEM; + + old_idx_lebs = c->lst.idx_lebs; + do { + ubifs_assert(c, p < c->lst.idx_lebs); + written = layout_leb_in_gaps(c, p); + if (written < 0) { + err = written; + if (err != -ENOSPC) { + kfree(c->gap_lebs); + c->gap_lebs = NULL; + return err; + } + if (!dbg_is_chk_index(c)) { + /* + * Do not print scary warnings if the debugging + * option which forces in-the-gaps is enabled. + */ + ubifs_warn(c, "out of space"); + ubifs_dump_budg(c, &c->bi); + ubifs_dump_lprops(c); + } + /* Try to commit anyway */ + break; + } + p++; + cnt -= written; + leb_needed_cnt = get_leb_cnt(c, cnt); + dbg_gc("%d znodes remaining, need %d LEBs, have %d", cnt, + leb_needed_cnt, c->ileb_cnt); + /* + * Dynamically change the size of @c->gap_lebs to prevent + * oob, because @c->lst.idx_lebs could be increased by + * function @get_idx_gc_leb (called by layout_leb_in_gaps-> + * ubifs_find_dirty_idx_leb) during loop. Only enlarge + * @c->gap_lebs when needed. + * + */ + if (leb_needed_cnt > c->ileb_cnt && p >= old_idx_lebs && + old_idx_lebs < c->lst.idx_lebs) { + old_idx_lebs = c->lst.idx_lebs; + gap_lebs = krealloc(c->gap_lebs, sizeof(int) * + (old_idx_lebs + 1), GFP_NOFS); + if (!gap_lebs) { + kfree(c->gap_lebs); + c->gap_lebs = NULL; + return -ENOMEM; + } + c->gap_lebs = gap_lebs; + } + } while (leb_needed_cnt > c->ileb_cnt); + + c->gap_lebs[p] = -1; + return 0; +} + +/** + * layout_in_empty_space - layout index nodes in empty space. + * @c: UBIFS file-system description object + * + * This function lays out new index nodes for dirty znodes using empty LEBs. + * + * This function returns %0 on success and a negative error code on failure. + */ +static int layout_in_empty_space(struct ubifs_info *c) +{ + struct ubifs_znode *znode, *cnext, *zp; + int lnum, offs, len, next_len, buf_len, buf_offs, used, avail; + int wlen, blen, err; + + cnext = c->enext; + if (!cnext) + return 0; + + lnum = c->ihead_lnum; + buf_offs = c->ihead_offs; + + buf_len = ubifs_idx_node_sz(c, c->fanout); + buf_len = ALIGN(buf_len, c->min_io_size); + used = 0; + avail = buf_len; + + /* Ensure there is enough room for first write */ + next_len = ubifs_idx_node_sz(c, cnext->child_cnt); + if (buf_offs + next_len > c->leb_size) + lnum = -1; + + while (1) { + znode = cnext; + + len = ubifs_idx_node_sz(c, znode->child_cnt); + + /* Determine the index node position */ + if (lnum == -1) { + if (c->ileb_nxt >= c->ileb_cnt) { + ubifs_err(c, "out of space"); + return -ENOSPC; + } + lnum = c->ilebs[c->ileb_nxt++]; + buf_offs = 0; + used = 0; + avail = buf_len; + } + + offs = buf_offs + used; + + znode->lnum = lnum; + znode->offs = offs; + znode->len = len; + + /* Update the parent */ + zp = znode->parent; + if (zp) { + struct ubifs_zbranch *zbr; + int i; + + i = znode->iip; + zbr = &zp->zbranch[i]; + zbr->lnum = lnum; + zbr->offs = offs; + zbr->len = len; + } else { + c->zroot.lnum = lnum; + c->zroot.offs = offs; + c->zroot.len = len; + } + c->calc_idx_sz += ALIGN(len, 8); + + /* + * Once lprops is updated, we can decrease the dirty znode count + * but it is easier to just do it here. + */ + atomic_long_dec(&c->dirty_zn_cnt); + + /* + * Calculate the next index node length to see if there is + * enough room for it + */ + cnext = znode->cnext; + if (cnext == c->cnext) + next_len = 0; + else + next_len = ubifs_idx_node_sz(c, cnext->child_cnt); + + /* Update buffer positions */ + wlen = used + len; + used += ALIGN(len, 8); + avail -= ALIGN(len, 8); + + if (next_len != 0 && + buf_offs + used + next_len <= c->leb_size && + avail > 0) + continue; + + if (avail <= 0 && next_len && + buf_offs + used + next_len <= c->leb_size) + blen = buf_len; + else + blen = ALIGN(wlen, c->min_io_size); + + /* The buffer is full or there are no more znodes to do */ + buf_offs += blen; + if (next_len) { + if (buf_offs + next_len > c->leb_size) { + err = ubifs_update_one_lp(c, lnum, + c->leb_size - buf_offs, blen - used, + 0, 0); + if (err) + return err; + lnum = -1; + } + used -= blen; + if (used < 0) + used = 0; + avail = buf_len - used; + continue; + } + err = ubifs_update_one_lp(c, lnum, c->leb_size - buf_offs, + blen - used, 0, 0); + if (err) + return err; + break; + } + + c->dbg->new_ihead_lnum = lnum; + c->dbg->new_ihead_offs = buf_offs; + + return 0; +} + +/** + * layout_commit - determine positions of index nodes to commit. + * @c: UBIFS file-system description object + * @no_space: indicates that insufficient empty LEBs were allocated + * @cnt: number of znodes to commit + * + * Calculate and update the positions of index nodes to commit. If there were + * an insufficient number of empty LEBs allocated, then index nodes are placed + * into the gaps created by obsolete index nodes in non-empty index LEBs. For + * this purpose, an obsolete index node is one that was not in the index as at + * the end of the last commit. To write "in-the-gaps" requires that those index + * LEBs are updated atomically in-place. + */ +static int layout_commit(struct ubifs_info *c, int no_space, int cnt) +{ + int err; + + if (no_space) { + err = layout_in_gaps(c, cnt); + if (err) + return err; + } + err = layout_in_empty_space(c); + return err; +} + +/** + * find_first_dirty - find first dirty znode. + * @znode: znode to begin searching from + */ +static struct ubifs_znode *find_first_dirty(struct ubifs_znode *znode) +{ + int i, cont; + + if (!znode) + return NULL; + + while (1) { + if (znode->level == 0) { + if (ubifs_zn_dirty(znode)) + return znode; + return NULL; + } + cont = 0; + for (i = 0; i < znode->child_cnt; i++) { + struct ubifs_zbranch *zbr = &znode->zbranch[i]; + + if (zbr->znode && ubifs_zn_dirty(zbr->znode)) { + znode = zbr->znode; + cont = 1; + break; + } + } + if (!cont) { + if (ubifs_zn_dirty(znode)) + return znode; + return NULL; + } + } +} + +/** + * find_next_dirty - find next dirty znode. + * @znode: znode to begin searching from + */ +static struct ubifs_znode *find_next_dirty(struct ubifs_znode *znode) +{ + int n = znode->iip + 1; + + znode = znode->parent; + if (!znode) + return NULL; + for (; n < znode->child_cnt; n++) { + struct ubifs_zbranch *zbr = &znode->zbranch[n]; + + if (zbr->znode && ubifs_zn_dirty(zbr->znode)) + return find_first_dirty(zbr->znode); + } + return znode; +} + +/** + * get_znodes_to_commit - create list of dirty znodes to commit. + * @c: UBIFS file-system description object + * + * This function returns the number of znodes to commit. + */ +static int get_znodes_to_commit(struct ubifs_info *c) +{ + struct ubifs_znode *znode, *cnext; + int cnt = 0; + + c->cnext = find_first_dirty(c->zroot.znode); + znode = c->enext = c->cnext; + if (!znode) { + dbg_cmt("no znodes to commit"); + return 0; + } + cnt += 1; + while (1) { + ubifs_assert(c, !ubifs_zn_cow(znode)); + __set_bit(COW_ZNODE, &znode->flags); + znode->alt = 0; + cnext = find_next_dirty(znode); + if (!cnext) { + ubifs_assert(c, !znode->parent); + znode->cparent = NULL; + znode->cnext = c->cnext; + break; + } + znode->cparent = znode->parent; + znode->ciip = znode->iip; + znode->cnext = cnext; + znode = cnext; + cnt += 1; + } + dbg_cmt("committing %d znodes", cnt); + ubifs_assert(c, cnt == atomic_long_read(&c->dirty_zn_cnt)); + return cnt; +} + +/** + * alloc_idx_lebs - allocate empty LEBs to be used to commit. + * @c: UBIFS file-system description object + * @cnt: number of znodes to commit + * + * This function returns %-ENOSPC if it cannot allocate a sufficient number of + * empty LEBs. %0 is returned on success, otherwise a negative error code + * is returned. + */ +static int alloc_idx_lebs(struct ubifs_info *c, int cnt) +{ + int i, leb_cnt, lnum; + + c->ileb_cnt = 0; + c->ileb_nxt = 0; + leb_cnt = get_leb_cnt(c, cnt); + dbg_cmt("need about %d empty LEBS for TNC commit", leb_cnt); + if (!leb_cnt) + return 0; + c->ilebs = kmalloc_objs(int, leb_cnt, GFP_NOFS); + if (!c->ilebs) + return -ENOMEM; + for (i = 0; i < leb_cnt; i++) { + lnum = ubifs_find_free_leb_for_idx(c); + if (lnum < 0) + return lnum; + c->ilebs[c->ileb_cnt++] = lnum; + dbg_cmt("LEB %d", lnum); + } + if (dbg_is_chk_index(c) && !get_random_u32_below(8)) + return -ENOSPC; + return 0; +} + +/** + * free_unused_idx_lebs - free unused LEBs that were allocated for the commit. + * @c: UBIFS file-system description object + * + * It is possible that we allocate more empty LEBs for the commit than we need. + * This functions frees the surplus. + * + * This function returns %0 on success and a negative error code on failure. + */ +static int free_unused_idx_lebs(struct ubifs_info *c) +{ + int i, err = 0, lnum, er; + + for (i = c->ileb_nxt; i < c->ileb_cnt; i++) { + lnum = c->ilebs[i]; + dbg_cmt("LEB %d", lnum); + er = ubifs_change_one_lp(c, lnum, LPROPS_NC, LPROPS_NC, 0, + LPROPS_INDEX | LPROPS_TAKEN, 0); + if (!err) + err = er; + } + return err; +} + +/** + * free_idx_lebs - free unused LEBs after commit end. + * @c: UBIFS file-system description object + * + * This function returns %0 on success and a negative error code on failure. + */ +static int free_idx_lebs(struct ubifs_info *c) +{ + int err; + + err = free_unused_idx_lebs(c); + kfree(c->ilebs); + c->ilebs = NULL; + return err; +} + +/** + * ubifs_tnc_start_commit - start TNC commit. + * @c: UBIFS file-system description object + * @zroot: new index root position is returned here + * + * This function prepares the list of indexing nodes to commit and lays out + * their positions on flash. If there is not enough free space it uses the + * in-gap commit method. Returns zero in case of success and a negative error + * code in case of failure. + */ +int ubifs_tnc_start_commit(struct ubifs_info *c, struct ubifs_zbranch *zroot) +{ + int err = 0, cnt; + + mutex_lock(&c->tnc_mutex); + err = dbg_check_tnc(c, 1); + if (err) + goto out; + cnt = get_znodes_to_commit(c); + if (cnt != 0) { + int no_space = 0; + + err = alloc_idx_lebs(c, cnt); + if (err == -ENOSPC) + no_space = 1; + else if (err) + goto out_free; + err = layout_commit(c, no_space, cnt); + if (err) + goto out_free; + ubifs_assert(c, atomic_long_read(&c->dirty_zn_cnt) == 0); + err = free_unused_idx_lebs(c); + if (err) + goto out; + } + destroy_old_idx(c); + memcpy(zroot, &c->zroot, sizeof(struct ubifs_zbranch)); + + err = ubifs_save_dirty_idx_lnums(c); + if (err) + goto out; + + spin_lock(&c->space_lock); + /* + * Although we have not finished committing yet, update size of the + * committed index ('c->bi.old_idx_sz') and zero out the index growth + * budget. It is OK to do this now, because we've reserved all the + * space which is needed to commit the index, and it is save for the + * budgeting subsystem to assume the index is already committed, + * even though it is not. + */ + ubifs_assert(c, c->bi.min_idx_lebs == ubifs_calc_min_idx_lebs(c)); + c->bi.old_idx_sz = c->calc_idx_sz; + c->bi.uncommitted_idx = 0; + c->bi.min_idx_lebs = ubifs_calc_min_idx_lebs(c); + spin_unlock(&c->space_lock); + mutex_unlock(&c->tnc_mutex); + + dbg_cmt("number of index LEBs %d", c->lst.idx_lebs); + dbg_cmt("size of index %llu", c->calc_idx_sz); + return err; + +out_free: + free_idx_lebs(c); +out: + mutex_unlock(&c->tnc_mutex); + return err; +} + +/** + * write_index - write index nodes. + * @c: UBIFS file-system description object + * + * This function writes the index nodes whose positions were laid out in the + * layout_in_empty_space function. + */ +static int write_index(struct ubifs_info *c) +{ + struct ubifs_idx_node *idx; + struct ubifs_znode *znode, *cnext; + int i, lnum, offs, len, next_len, buf_len, buf_offs, used; + int avail, wlen, err, lnum_pos = 0, blen, nxt_offs; + + cnext = c->enext; + if (!cnext) + return 0; + + /* + * Always write index nodes to the index head so that index nodes and + * other types of nodes are never mixed in the same erase block. + */ + lnum = c->ihead_lnum; + buf_offs = c->ihead_offs; + + /* Allocate commit buffer */ + buf_len = ALIGN(c->max_idx_node_sz, c->min_io_size); + used = 0; + avail = buf_len; + + /* Ensure there is enough room for first write */ + next_len = ubifs_idx_node_sz(c, cnext->child_cnt); + if (buf_offs + next_len > c->leb_size) { + err = ubifs_update_one_lp(c, lnum, LPROPS_NC, 0, 0, + LPROPS_TAKEN); + if (err) + return err; + lnum = -1; + } + + while (1) { + u8 hash[UBIFS_HASH_ARR_SZ]; + + cond_resched(); + + znode = cnext; + idx = c->cbuf + used; + + /* Make index node */ + idx->ch.node_type = UBIFS_IDX_NODE; + idx->child_cnt = cpu_to_le16(znode->child_cnt); + idx->level = cpu_to_le16(znode->level); + for (i = 0; i < znode->child_cnt; i++) { + struct ubifs_branch *br = ubifs_idx_branch(c, idx, i); + struct ubifs_zbranch *zbr = &znode->zbranch[i]; + + key_write_idx(c, &zbr->key, &br->key); + br->lnum = cpu_to_le32(zbr->lnum); + br->offs = cpu_to_le32(zbr->offs); + br->len = cpu_to_le32(zbr->len); + ubifs_copy_hash(c, zbr->hash, ubifs_branch_hash(c, br)); + if (!zbr->lnum || !zbr->len) { + ubifs_err(c, "bad ref in znode"); + ubifs_dump_znode(c, znode); + if (zbr->znode) + ubifs_dump_znode(c, zbr->znode); + + return -EINVAL; + } + } + len = ubifs_idx_node_sz(c, znode->child_cnt); + ubifs_prepare_node(c, idx, len, 0); + ubifs_node_calc_hash(c, idx, hash); + + mutex_lock(&c->tnc_mutex); + + if (znode->cparent) + ubifs_copy_hash(c, hash, + znode->cparent->zbranch[znode->ciip].hash); + + if (znode->parent) { + if (!ubifs_zn_obsolete(znode)) + ubifs_copy_hash(c, hash, + znode->parent->zbranch[znode->iip].hash); + } else { + ubifs_copy_hash(c, hash, c->zroot.hash); + } + + mutex_unlock(&c->tnc_mutex); + + /* Determine the index node position */ + if (lnum == -1) { + lnum = c->ilebs[lnum_pos++]; + buf_offs = 0; + used = 0; + avail = buf_len; + } + offs = buf_offs + used; + + if (lnum != znode->lnum || offs != znode->offs || + len != znode->len) { + ubifs_err(c, "inconsistent znode posn"); + return -EINVAL; + } + + /* Grab some stuff from znode while we still can */ + cnext = znode->cnext; + + ubifs_assert(c, ubifs_zn_dirty(znode)); + ubifs_assert(c, ubifs_zn_cow(znode)); + + /* + * It is important that other threads should see %DIRTY_ZNODE + * flag cleared before %COW_ZNODE. Specifically, it matters in + * the 'dirty_cow_znode()' function. This is the reason for the + * first barrier. Also, we want the bit changes to be seen to + * other threads ASAP, to avoid unnecessary copying, which is + * the reason for the second barrier. + */ + clear_bit(DIRTY_ZNODE, &znode->flags); + smp_mb__before_atomic(); + clear_bit(COW_ZNODE, &znode->flags); + smp_mb__after_atomic(); + + /* + * We have marked the znode as clean but have not updated the + * @c->clean_zn_cnt counter. If this znode becomes dirty again + * before 'free_obsolete_znodes()' is called, then + * @c->clean_zn_cnt will be decremented before it gets + * incremented (resulting in 2 decrements for the same znode). + * This means that @c->clean_zn_cnt may become negative for a + * while. + * + * Q: why we cannot increment @c->clean_zn_cnt? + * A: because we do not have the @c->tnc_mutex locked, and the + * following code would be racy and buggy: + * + * if (!ubifs_zn_obsolete(znode)) { + * atomic_long_inc(&c->clean_zn_cnt); + * atomic_long_inc(&ubifs_clean_zn_cnt); + * } + * + * Thus, we just delay the @c->clean_zn_cnt update until we + * have the mutex locked. + */ + + /* Do not access znode from this point on */ + + /* Update buffer positions */ + wlen = used + len; + used += ALIGN(len, 8); + avail -= ALIGN(len, 8); + + /* + * Calculate the next index node length to see if there is + * enough room for it + */ + if (cnext == c->cnext) + next_len = 0; + else + next_len = ubifs_idx_node_sz(c, cnext->child_cnt); + + nxt_offs = buf_offs + used + next_len; + if (next_len && nxt_offs <= c->leb_size) { + if (avail > 0) + continue; + else + blen = buf_len; + } else { + wlen = ALIGN(wlen, 8); + blen = ALIGN(wlen, c->min_io_size); + ubifs_pad(c, c->cbuf + wlen, blen - wlen); + } + + /* The buffer is full or there are no more znodes to do */ + err = ubifs_leb_write(c, lnum, c->cbuf, buf_offs, blen); + if (err) + return err; + buf_offs += blen; + if (next_len) { + if (nxt_offs > c->leb_size) { + err = ubifs_update_one_lp(c, lnum, LPROPS_NC, 0, + 0, LPROPS_TAKEN); + if (err) + return err; + lnum = -1; + } + used -= blen; + if (used < 0) + used = 0; + avail = buf_len - used; + memmove(c->cbuf, c->cbuf + blen, used); + continue; + } + break; + } + + if (lnum != c->dbg->new_ihead_lnum || + buf_offs != c->dbg->new_ihead_offs) { + ubifs_err(c, "inconsistent ihead"); + return -EINVAL; + } + + c->ihead_lnum = lnum; + c->ihead_offs = buf_offs; + + return 0; +} + +/** + * free_obsolete_znodes - free obsolete znodes. + * @c: UBIFS file-system description object + * + * At the end of commit end, obsolete znodes are freed. + */ +static void free_obsolete_znodes(struct ubifs_info *c) +{ + struct ubifs_znode *znode, *cnext; + + cnext = c->cnext; + do { + znode = cnext; + cnext = znode->cnext; + if (ubifs_zn_obsolete(znode)) + kfree(znode); + else { + znode->cnext = NULL; + atomic_long_inc(&c->clean_zn_cnt); + atomic_long_inc(&ubifs_clean_zn_cnt); + } + } while (cnext != c->cnext); +} + +/** + * return_gap_lebs - return LEBs used by the in-gap commit method. + * @c: UBIFS file-system description object + * + * This function clears the "taken" flag for the LEBs which were used by the + * "commit in-the-gaps" method. + */ +static int return_gap_lebs(struct ubifs_info *c) +{ + int *p, err; + + if (!c->gap_lebs) + return 0; + + dbg_cmt(""); + for (p = c->gap_lebs; *p != -1; p++) { + err = ubifs_change_one_lp(c, *p, LPROPS_NC, LPROPS_NC, 0, + LPROPS_TAKEN, 0); + if (err) + return err; + } + + kfree(c->gap_lebs); + c->gap_lebs = NULL; + return 0; +} + +/** + * ubifs_tnc_end_commit - update the TNC for commit end. + * @c: UBIFS file-system description object + * + * Write the dirty znodes. + */ +int ubifs_tnc_end_commit(struct ubifs_info *c) +{ + int err; + + if (!c->cnext) + return 0; + + err = return_gap_lebs(c); + if (err) + return err; + + err = write_index(c); + if (err) + return err; + + mutex_lock(&c->tnc_mutex); + + dbg_cmt("TNC height is %d", c->zroot.znode->level + 1); + + free_obsolete_znodes(c); + + c->cnext = NULL; + kfree(c->ilebs); + c->ilebs = NULL; + + mutex_unlock(&c->tnc_mutex); + + return 0; +} diff --git a/fs/ubifs/tnc_misc.c b/fs/ubifs/tnc_misc.c new file mode 100644 index 000000000..10b222dc6 --- /dev/null +++ b/fs/ubifs/tnc_misc.c @@ -0,0 +1,517 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * + * Authors: Adrian Hunter + * Artem Bityutskiy (Битюцкий Артём) + */ + +/* + * This file contains miscelanious TNC-related functions shared betweend + * different files. This file does not form any logically separate TNC + * sub-system. The file was created because there is a lot of TNC code and + * putting it all in one file would make that file too big and unreadable. + */ + +#include "ubifs.h" + +/** + * ubifs_tnc_levelorder_next - next TNC tree element in levelorder traversal. + * @c: UBIFS file-system description object + * @zr: root of the subtree to traverse + * @znode: previous znode + * + * This function implements levelorder TNC traversal. The LNC is ignored. + * Returns the next element or %NULL if @znode is already the last one. + */ +struct ubifs_znode *ubifs_tnc_levelorder_next(const struct ubifs_info *c, + struct ubifs_znode *zr, + struct ubifs_znode *znode) +{ + int level, iip, level_search = 0; + struct ubifs_znode *zn; + + ubifs_assert(c, zr); + + if (unlikely(!znode)) + return zr; + + if (unlikely(znode == zr)) { + if (znode->level == 0) + return NULL; + return ubifs_tnc_find_child(zr, 0); + } + + level = znode->level; + + iip = znode->iip; + while (1) { + ubifs_assert(c, znode->level <= zr->level); + + /* + * First walk up until there is a znode with next branch to + * look at. + */ + while (znode->parent != zr && iip >= znode->parent->child_cnt) { + znode = znode->parent; + iip = znode->iip; + } + + if (unlikely(znode->parent == zr && + iip >= znode->parent->child_cnt)) { + /* This level is done, switch to the lower one */ + level -= 1; + if (level_search || level < 0) + /* + * We were already looking for znode at lower + * level ('level_search'). As we are here + * again, it just does not exist. Or all levels + * were finished ('level < 0'). + */ + return NULL; + + level_search = 1; + iip = -1; + znode = ubifs_tnc_find_child(zr, 0); + ubifs_assert(c, znode); + } + + /* Switch to the next index */ + zn = ubifs_tnc_find_child(znode->parent, iip + 1); + if (!zn) { + /* No more children to look at, we have walk up */ + iip = znode->parent->child_cnt; + continue; + } + + /* Walk back down to the level we came from ('level') */ + while (zn->level != level) { + znode = zn; + zn = ubifs_tnc_find_child(zn, 0); + if (!zn) { + /* + * This path is not too deep so it does not + * reach 'level'. Try next path. + */ + iip = znode->iip; + break; + } + } + + if (zn) { + ubifs_assert(c, zn->level >= 0); + return zn; + } + } +} + +/** + * ubifs_search_zbranch - search znode branch. + * @c: UBIFS file-system description object + * @znode: znode to search in + * @key: key to search for + * @n: znode branch slot number is returned here + * + * This is a helper function which search branch with key @key in @znode using + * binary search. The result of the search may be: + * o exact match, then %1 is returned, and the slot number of the branch is + * stored in @n; + * o no exact match, then %0 is returned and the slot number of the left + * closest branch is returned in @n; the slot if all keys in this znode are + * greater than @key, then %-1 is returned in @n. + */ +int ubifs_search_zbranch(const struct ubifs_info *c, + const struct ubifs_znode *znode, + const union ubifs_key *key, int *n) +{ + int beg = 0, end = znode->child_cnt, mid; + int cmp; + const struct ubifs_zbranch *zbr = &znode->zbranch[0]; + + ubifs_assert(c, end > beg); + + while (end > beg) { + mid = (beg + end) >> 1; + cmp = keys_cmp(c, key, &zbr[mid].key); + if (cmp > 0) + beg = mid + 1; + else if (cmp < 0) + end = mid; + else { + *n = mid; + return 1; + } + } + + *n = end - 1; + + /* The insert point is after *n */ + ubifs_assert(c, *n >= -1 && *n < znode->child_cnt); + if (*n == -1) + ubifs_assert(c, keys_cmp(c, key, &zbr[0].key) < 0); + else + ubifs_assert(c, keys_cmp(c, key, &zbr[*n].key) > 0); + if (*n + 1 < znode->child_cnt) + ubifs_assert(c, keys_cmp(c, key, &zbr[*n + 1].key) < 0); + + return 0; +} + +/** + * ubifs_tnc_postorder_first - find first znode to do postorder tree traversal. + * @znode: znode to start at (root of the sub-tree to traverse) + * + * Find the lowest leftmost znode in a subtree of the TNC tree. The LNC is + * ignored. + */ +struct ubifs_znode *ubifs_tnc_postorder_first(struct ubifs_znode *znode) +{ + if (unlikely(!znode)) + return NULL; + + while (znode->level > 0) { + struct ubifs_znode *child; + + child = ubifs_tnc_find_child(znode, 0); + if (!child) + return znode; + znode = child; + } + + return znode; +} + +/** + * ubifs_tnc_postorder_next - next TNC tree element in postorder traversal. + * @c: UBIFS file-system description object + * @znode: previous znode + * + * This function implements postorder TNC traversal. The LNC is ignored. + * Returns the next element or %NULL if @znode is already the last one. + */ +struct ubifs_znode *ubifs_tnc_postorder_next(const struct ubifs_info *c, + struct ubifs_znode *znode) +{ + struct ubifs_znode *zn; + + ubifs_assert(c, znode); + if (unlikely(!znode->parent)) + return NULL; + + /* Switch to the next index in the parent */ + zn = ubifs_tnc_find_child(znode->parent, znode->iip + 1); + if (!zn) + /* This is in fact the last child, return parent */ + return znode->parent; + + /* Go to the first znode in this new subtree */ + return ubifs_tnc_postorder_first(zn); +} + +/** + * ubifs_destroy_tnc_subtree - destroy all znodes connected to a subtree. + * @c: UBIFS file-system description object + * @znode: znode defining subtree to destroy + * + * This function destroys subtree of the TNC tree. Returns number of clean + * znodes in the subtree. + */ +long ubifs_destroy_tnc_subtree(const struct ubifs_info *c, + struct ubifs_znode *znode) +{ + struct ubifs_znode *zn = ubifs_tnc_postorder_first(znode); + long clean_freed = 0; + int n; + + ubifs_assert(c, zn); + while (1) { + for (n = 0; n < zn->child_cnt; n++) { + if (!zn->zbranch[n].znode) + continue; + + if (zn->level > 0 && + !ubifs_zn_dirty(zn->zbranch[n].znode)) + clean_freed += 1; + + cond_resched(); + kfree(zn->zbranch[n].znode); + } + + if (zn == znode) { + if (!ubifs_zn_dirty(zn)) + clean_freed += 1; + kfree(zn); + return clean_freed; + } + + zn = ubifs_tnc_postorder_next(c, zn); + } +} + +/** + * ubifs_destroy_tnc_tree - destroy all znodes connected to the TNC tree. + * @c: UBIFS file-system description object + * + * This function destroys the whole TNC tree and updates clean global znode + * count. + */ +void ubifs_destroy_tnc_tree(struct ubifs_info *c) +{ + long n, freed; + + if (!c->zroot.znode) + return; + + n = atomic_long_read(&c->clean_zn_cnt); + freed = ubifs_destroy_tnc_subtree(c, c->zroot.znode); + ubifs_assert(c, freed == n); + atomic_long_sub(n, &ubifs_clean_zn_cnt); + + c->zroot.znode = NULL; +} + +/** + * read_znode - read an indexing node from flash and fill znode. + * @c: UBIFS file-system description object + * @zzbr: the zbranch describing the node to read + * @znode: znode to read to + * + * This function reads an indexing node from the flash media and fills znode + * with the read data. Returns zero in case of success and a negative error + * code in case of failure. The read indexing node is validated and if anything + * is wrong with it, this function prints complaint messages and returns + * %-EINVAL. + */ +static int read_znode(struct ubifs_info *c, struct ubifs_zbranch *zzbr, + struct ubifs_znode *znode) +{ + int lnum = zzbr->lnum; + int offs = zzbr->offs; + int len = zzbr->len; + int i, err, type, cmp; + struct ubifs_idx_node *idx; + + idx = kmalloc(c->max_idx_node_sz, GFP_NOFS); + if (!idx) + return -ENOMEM; + + err = ubifs_read_node(c, idx, UBIFS_IDX_NODE, len, lnum, offs); + if (err < 0) { + kfree(idx); + return err; + } + + err = ubifs_node_check_hash(c, idx, zzbr->hash); + if (err) { + ubifs_bad_hash(c, idx, zzbr->hash, lnum, offs); + kfree(idx); + return err; + } + + znode->child_cnt = le16_to_cpu(idx->child_cnt); + znode->level = le16_to_cpu(idx->level); + + dbg_tnc("LEB %d:%d, level %d, %d branch", + lnum, offs, znode->level, znode->child_cnt); + + if (znode->child_cnt > c->fanout || znode->level > UBIFS_MAX_LEVELS) { + ubifs_err(c, "current fanout %d, branch count %d", + c->fanout, znode->child_cnt); + ubifs_err(c, "max levels %d, znode level %d", + UBIFS_MAX_LEVELS, znode->level); + goto out_dump; + } + + for (i = 0; i < znode->child_cnt; i++) { + struct ubifs_branch *br = ubifs_idx_branch(c, idx, i); + struct ubifs_zbranch *zbr = &znode->zbranch[i]; + + key_read(c, &br->key, &zbr->key); + zbr->lnum = le32_to_cpu(br->lnum); + zbr->offs = le32_to_cpu(br->offs); + zbr->len = le32_to_cpu(br->len); + ubifs_copy_hash(c, ubifs_branch_hash(c, br), zbr->hash); + zbr->znode = NULL; + + /* Validate branch */ + + if (zbr->lnum < c->main_first || + zbr->lnum >= c->leb_cnt || zbr->offs < 0 || + zbr->offs + zbr->len > c->leb_size || zbr->offs & 7) { + ubifs_err(c, "bad branch %d", i); + goto out_dump; + } + + switch (key_type(c, &zbr->key)) { + case UBIFS_INO_KEY: + case UBIFS_DATA_KEY: + case UBIFS_DENT_KEY: + case UBIFS_XENT_KEY: + break; + default: + ubifs_err(c, "bad key type at slot %d: %d", + i, key_type(c, &zbr->key)); + goto out_dump; + } + + if (znode->level) + continue; + + type = key_type(c, &zbr->key); + if (c->ranges[type].max_len == 0) { + if (zbr->len != c->ranges[type].len) { + ubifs_err(c, "bad target node (type %d) length (%d)", + type, zbr->len); + ubifs_err(c, "have to be %d", c->ranges[type].len); + goto out_dump; + } + } else if (zbr->len < c->ranges[type].min_len || + zbr->len > c->ranges[type].max_len) { + ubifs_err(c, "bad target node (type %d) length (%d)", + type, zbr->len); + ubifs_err(c, "have to be in range of %d-%d", + c->ranges[type].min_len, + c->ranges[type].max_len); + goto out_dump; + } + } + + /* + * Ensure that the next key is greater or equivalent to the + * previous one. + */ + for (i = 0; i < znode->child_cnt - 1; i++) { + const union ubifs_key *key1, *key2; + + key1 = &znode->zbranch[i].key; + key2 = &znode->zbranch[i + 1].key; + + cmp = keys_cmp(c, key1, key2); + if (cmp > 0) { + ubifs_err(c, "bad key order (keys %d and %d)", i, i + 1); + goto out_dump; + } else if (cmp == 0 && !is_hash_key(c, key1)) { + /* These can only be keys with colliding hash */ + ubifs_err(c, "keys %d and %d are not hashed but equivalent", + i, i + 1); + goto out_dump; + } + } + + kfree(idx); + return 0; + +out_dump: + ubifs_err(c, "bad indexing node at LEB %d:%d", lnum, offs); + ubifs_dump_node(c, idx, c->max_idx_node_sz); + kfree(idx); + return -EINVAL; +} + +/** + * ubifs_load_znode - load znode to TNC cache. + * @c: UBIFS file-system description object + * @zbr: znode branch + * @parent: znode's parent + * @iip: index in parent + * + * This function loads znode pointed to by @zbr into the TNC cache and + * returns pointer to it in case of success and a negative error code in case + * of failure. + */ +struct ubifs_znode *ubifs_load_znode(struct ubifs_info *c, + struct ubifs_zbranch *zbr, + struct ubifs_znode *parent, int iip) +{ + int err; + struct ubifs_znode *znode; + + ubifs_assert(c, !zbr->znode); + /* + * A slab cache is not presently used for znodes because the znode size + * depends on the fanout which is stored in the superblock. + */ + znode = kzalloc(c->max_znode_sz, GFP_NOFS); + if (!znode) + return ERR_PTR(-ENOMEM); + + err = read_znode(c, zbr, znode); + if (err) + goto out; + + atomic_long_inc(&c->clean_zn_cnt); + + /* + * Increment the global clean znode counter as well. It is OK that + * global and per-FS clean znode counters may be inconsistent for some + * short time (because we might be preempted at this point), the global + * one is only used in shrinker. + */ + atomic_long_inc(&ubifs_clean_zn_cnt); + + zbr->znode = znode; + znode->parent = parent; + znode->time = ktime_get_seconds(); + znode->iip = iip; + + return znode; + +out: + kfree(znode); + return ERR_PTR(err); +} + +/** + * ubifs_tnc_read_node - read a leaf node from the flash media. + * @c: UBIFS file-system description object + * @zbr: key and position of the node + * @node: node is returned here + * + * This function reads a node defined by @zbr from the flash media. Returns + * zero in case of success or a negative error code in case of failure. + */ +int ubifs_tnc_read_node(struct ubifs_info *c, struct ubifs_zbranch *zbr, + void *node) +{ + union ubifs_key key1, *key = &zbr->key; + int err, type = key_type(c, key); + struct ubifs_wbuf *wbuf; + + /* + * 'zbr' has to point to on-flash node. The node may sit in a bud and + * may even be in a write buffer, so we have to take care about this. + */ + wbuf = ubifs_get_wbuf(c, zbr->lnum); + if (wbuf) + err = ubifs_read_node_wbuf(wbuf, node, type, zbr->len, + zbr->lnum, zbr->offs); + else + err = ubifs_read_node(c, node, type, zbr->len, zbr->lnum, + zbr->offs); + + if (err) { + dbg_tnck(key, "key "); + return err; + } + + /* Make sure the key of the read node is correct */ + key_read(c, node + UBIFS_KEY_OFFSET, &key1); + if (!keys_eq(c, key, &key1)) { + ubifs_err(c, "bad key in node at LEB %d:%d", + zbr->lnum, zbr->offs); + dbg_tnck(key, "looked for key "); + dbg_tnck(&key1, "but found node's key "); + ubifs_dump_node(c, node, zbr->len); + return -EINVAL; + } + + err = ubifs_node_check_hash(c, node, zbr->hash); + if (err) { + ubifs_bad_hash(c, node, zbr->hash, zbr->lnum, zbr->offs); + return err; + } + + return 0; +} diff --git a/fs/ubifs/ubifs-media.h b/fs/ubifs/ubifs-media.h new file mode 100644 index 000000000..3c9792cbb --- /dev/null +++ b/fs/ubifs/ubifs-media.h @@ -0,0 +1,855 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * + * Authors: Artem Bityutskiy (Битюцкий Артём) + * Adrian Hunter + */ + +/* + * This file describes UBIFS on-flash format and contains definitions of all the + * relevant data structures and constants. + * + * All UBIFS on-flash objects are stored in the form of nodes. All nodes start + * with the UBIFS node magic number and have the same common header. Nodes + * always sit at 8-byte aligned positions on the media and node header sizes are + * also 8-byte aligned (except for the indexing node and the padding node). + */ + +#ifndef __UBIFS_MEDIA_H__ +#define __UBIFS_MEDIA_H__ + +/* UBIFS node magic number (must not have the padding byte first or last) */ +#define UBIFS_NODE_MAGIC 0x06101831 + +/* + * UBIFS on-flash format version. This version is increased when the on-flash + * format is changing. If this happens, UBIFS is will support older versions as + * well. But older UBIFS code will not support newer formats. Format changes + * will be rare and only when absolutely necessary, e.g. to fix a bug or to add + * a new feature. + * + * UBIFS went into mainline kernel with format version 4. The older formats + * were development formats. + */ +#define UBIFS_FORMAT_VERSION 5 + +/* + * Read-only compatibility version. If the UBIFS format is changed, older UBIFS + * implementations will not be able to mount newer formats in read-write mode. + * However, depending on the change, it may be possible to mount newer formats + * in R/O mode. This is indicated by the R/O compatibility version which is + * stored in the super-block. + * + * This is needed to support boot-loaders which only need R/O mounting. With + * this flag it is possible to do UBIFS format changes without a need to update + * boot-loaders. + */ +#define UBIFS_RO_COMPAT_VERSION 0 + +/* Minimum logical eraseblock size in bytes */ +#define UBIFS_MIN_LEB_SZ (15*1024) + +/* Initial CRC32 value used when calculating CRC checksums */ +#define UBIFS_CRC32_INIT 0xFFFFFFFFU + +/* + * UBIFS does not try to compress data if its length is less than the below + * constant. + */ +#define UBIFS_MIN_COMPR_LEN 128 + +/* + * If compressed data length is less than %UBIFS_MIN_COMPRESS_DIFF bytes + * shorter than uncompressed data length, UBIFS prefers to leave this data + * node uncompress, because it'll be read faster. + */ +#define UBIFS_MIN_COMPRESS_DIFF 64 + +/* Root inode number */ +#define UBIFS_ROOT_INO 1 + +/* Lowest inode number used for regular inodes (not UBIFS-only internal ones) */ +#define UBIFS_FIRST_INO 64 + +/* + * Maximum file name and extended attribute length (must be a multiple of 8, + * minus 1). + */ +#define UBIFS_MAX_NLEN 255 + +/* Maximum number of data journal heads */ +#define UBIFS_MAX_JHEADS 1 + +/* + * Size of UBIFS data block. Note, UBIFS is not a block oriented file-system, + * which means that it does not treat the underlying media as consisting of + * blocks like in case of hard drives. Do not be confused. UBIFS block is just + * the maximum amount of data which one data node can have or which can be + * attached to an inode node. + */ +#define UBIFS_BLOCK_SIZE 4096 +#define UBIFS_BLOCK_SHIFT 12 + +/* UBIFS padding byte pattern (must not be first or last byte of node magic) */ +#define UBIFS_PADDING_BYTE 0xCE + +/* Maximum possible key length */ +#define UBIFS_MAX_KEY_LEN 16 + +/* Key length ("simple" format) */ +#define UBIFS_SK_LEN 8 + +/* Minimum index tree fanout */ +#define UBIFS_MIN_FANOUT 3 + +/* Maximum number of levels in UBIFS indexing B-tree */ +#define UBIFS_MAX_LEVELS 512 + +/* Maximum amount of data attached to an inode in bytes */ +#define UBIFS_MAX_INO_DATA UBIFS_BLOCK_SIZE + +/* LEB Properties Tree fanout (must be power of 2) and fanout shift */ +#define UBIFS_LPT_FANOUT 4 +#define UBIFS_LPT_FANOUT_SHIFT 2 + +/* LEB Properties Tree bit field sizes */ +#define UBIFS_LPT_CRC_BITS 16 +#define UBIFS_LPT_CRC_BYTES 2 +#define UBIFS_LPT_TYPE_BITS 4 + +/* The key is always at the same position in all keyed nodes */ +#define UBIFS_KEY_OFFSET offsetof(struct ubifs_ino_node, key) + +/* Garbage collector journal head number */ +#define UBIFS_GC_HEAD 0 +/* Base journal head number */ +#define UBIFS_BASE_HEAD 1 +/* Data journal head number */ +#define UBIFS_DATA_HEAD 2 + +/* + * LEB Properties Tree node types. + * + * UBIFS_LPT_PNODE: LPT leaf node (contains LEB properties) + * UBIFS_LPT_NNODE: LPT internal node + * UBIFS_LPT_LTAB: LPT's own lprops table + * UBIFS_LPT_LSAVE: LPT's save table (big model only) + * UBIFS_LPT_NODE_CNT: count of LPT node types + * UBIFS_LPT_NOT_A_NODE: all ones (15 for 4 bits) is never a valid node type + */ +enum { + UBIFS_LPT_PNODE, + UBIFS_LPT_NNODE, + UBIFS_LPT_LTAB, + UBIFS_LPT_LSAVE, + UBIFS_LPT_NODE_CNT, + UBIFS_LPT_NOT_A_NODE = (1 << UBIFS_LPT_TYPE_BITS) - 1, +}; + +/* + * UBIFS inode types. + * + * UBIFS_ITYPE_REG: regular file + * UBIFS_ITYPE_DIR: directory + * UBIFS_ITYPE_LNK: soft link + * UBIFS_ITYPE_BLK: block device node + * UBIFS_ITYPE_CHR: character device node + * UBIFS_ITYPE_FIFO: fifo + * UBIFS_ITYPE_SOCK: socket + * UBIFS_ITYPES_CNT: count of supported file types + */ +enum { + UBIFS_ITYPE_REG, + UBIFS_ITYPE_DIR, + UBIFS_ITYPE_LNK, + UBIFS_ITYPE_BLK, + UBIFS_ITYPE_CHR, + UBIFS_ITYPE_FIFO, + UBIFS_ITYPE_SOCK, + UBIFS_ITYPES_CNT, +}; + +/* + * Supported key hash functions. + * + * UBIFS_KEY_HASH_R5: R5 hash + * UBIFS_KEY_HASH_TEST: test hash which just returns first 4 bytes of the name + */ +enum { + UBIFS_KEY_HASH_R5, + UBIFS_KEY_HASH_TEST, +}; + +/* + * Supported key formats. + * + * UBIFS_SIMPLE_KEY_FMT: simple key format + */ +enum { + UBIFS_SIMPLE_KEY_FMT, +}; + +/* + * The simple key format uses 29 bits for storing UBIFS block number and hash + * value. + */ +#define UBIFS_S_KEY_BLOCK_BITS 29 +#define UBIFS_S_KEY_BLOCK_MASK 0x1FFFFFFF +#define UBIFS_S_KEY_HASH_BITS UBIFS_S_KEY_BLOCK_BITS +#define UBIFS_S_KEY_HASH_MASK UBIFS_S_KEY_BLOCK_MASK + +/* + * Key types. + * + * UBIFS_INO_KEY: inode node key + * UBIFS_DATA_KEY: data node key + * UBIFS_DENT_KEY: directory entry node key + * UBIFS_XENT_KEY: extended attribute entry key + * UBIFS_KEY_TYPES_CNT: number of supported key types + */ +enum { + UBIFS_INO_KEY, + UBIFS_DATA_KEY, + UBIFS_DENT_KEY, + UBIFS_XENT_KEY, + UBIFS_KEY_TYPES_CNT, +}; + +/* Count of LEBs reserved for the superblock area */ +#define UBIFS_SB_LEBS 1 +/* Count of LEBs reserved for the master area */ +#define UBIFS_MST_LEBS 2 + +/* First LEB of the superblock area */ +#define UBIFS_SB_LNUM 0 +/* First LEB of the master area */ +#define UBIFS_MST_LNUM (UBIFS_SB_LNUM + UBIFS_SB_LEBS) +/* First LEB of the log area */ +#define UBIFS_LOG_LNUM (UBIFS_MST_LNUM + UBIFS_MST_LEBS) + +/* + * The below constants define the absolute minimum values for various UBIFS + * media areas. Many of them actually depend of flash geometry and the FS + * configuration (number of journal heads, orphan LEBs, etc). This means that + * the smallest volume size which can be used for UBIFS cannot be pre-defined + * by these constants. The file-system that meets the below limitation will not + * necessarily mount. UBIFS does run-time calculations and validates the FS + * size. + */ + +/* Minimum number of logical eraseblocks in the log */ +#define UBIFS_MIN_LOG_LEBS 2 +/* Minimum number of bud logical eraseblocks (one for each head) */ +#define UBIFS_MIN_BUD_LEBS 3 +/* Minimum number of journal logical eraseblocks */ +#define UBIFS_MIN_JNL_LEBS (UBIFS_MIN_LOG_LEBS + UBIFS_MIN_BUD_LEBS) +/* Minimum number of LPT area logical eraseblocks */ +#define UBIFS_MIN_LPT_LEBS 2 +/* Minimum number of orphan area logical eraseblocks */ +#define UBIFS_MIN_ORPH_LEBS 1 +/* + * Minimum number of main area logical eraseblocks (buds, 3 for the index, 1 + * for GC, 1 for deletions, and at least 1 for committed data). + */ +#define UBIFS_MIN_MAIN_LEBS (UBIFS_MIN_BUD_LEBS + 6) + +/* Minimum number of logical eraseblocks */ +#define UBIFS_MIN_LEB_CNT (UBIFS_SB_LEBS + UBIFS_MST_LEBS + \ + UBIFS_MIN_LOG_LEBS + UBIFS_MIN_LPT_LEBS + \ + UBIFS_MIN_ORPH_LEBS + UBIFS_MIN_MAIN_LEBS) + +/* Node sizes (N.B. these are guaranteed to be multiples of 8) */ +#define UBIFS_CH_SZ sizeof(struct ubifs_ch) +#define UBIFS_INO_NODE_SZ sizeof(struct ubifs_ino_node) +#define UBIFS_DATA_NODE_SZ sizeof(struct ubifs_data_node) +#define UBIFS_DENT_NODE_SZ sizeof(struct ubifs_dent_node) +#define UBIFS_TRUN_NODE_SZ sizeof(struct ubifs_trun_node) +#define UBIFS_PAD_NODE_SZ sizeof(struct ubifs_pad_node) +#define UBIFS_SB_NODE_SZ sizeof(struct ubifs_sb_node) +#define UBIFS_MST_NODE_SZ sizeof(struct ubifs_mst_node) +#define UBIFS_REF_NODE_SZ sizeof(struct ubifs_ref_node) +#define UBIFS_IDX_NODE_SZ sizeof(struct ubifs_idx_node) +#define UBIFS_CS_NODE_SZ sizeof(struct ubifs_cs_node) +#define UBIFS_ORPH_NODE_SZ sizeof(struct ubifs_orph_node) +#define UBIFS_AUTH_NODE_SZ sizeof(struct ubifs_auth_node) +#define UBIFS_SIG_NODE_SZ sizeof(struct ubifs_sig_node) + +/* Extended attribute entry nodes are identical to directory entry nodes */ +#define UBIFS_XENT_NODE_SZ UBIFS_DENT_NODE_SZ +/* Only this does not have to be multiple of 8 bytes */ +#define UBIFS_BRANCH_SZ sizeof(struct ubifs_branch) + +/* Maximum node sizes (N.B. these are guaranteed to be multiples of 8) */ +#define UBIFS_MAX_DATA_NODE_SZ (UBIFS_DATA_NODE_SZ + UBIFS_BLOCK_SIZE) +#define UBIFS_MAX_INO_NODE_SZ (UBIFS_INO_NODE_SZ + UBIFS_MAX_INO_DATA) +#define UBIFS_MAX_DENT_NODE_SZ (UBIFS_DENT_NODE_SZ + UBIFS_MAX_NLEN + 1) +#define UBIFS_MAX_XENT_NODE_SZ UBIFS_MAX_DENT_NODE_SZ + +/* The largest UBIFS node */ +#define UBIFS_MAX_NODE_SZ UBIFS_MAX_INO_NODE_SZ + +/* The maxmimum size of a hash, enough for sha512 */ +#define UBIFS_MAX_HASH_LEN 64 + +/* The maxmimum size of a hmac, enough for hmac(sha512) */ +#define UBIFS_MAX_HMAC_LEN 64 + +/* + * xattr name of UBIFS encryption context, we don't use a prefix + * nor a long name to not waste space on the flash. + */ +#define UBIFS_XATTR_NAME_ENCRYPTION_CONTEXT "c" + +/* Type field in ubifs_sig_node */ +#define UBIFS_SIGNATURE_TYPE_PKCS7 1 + +/* + * On-flash inode flags. + * + * UBIFS_COMPR_FL: use compression for this inode + * UBIFS_SYNC_FL: I/O on this inode has to be synchronous + * UBIFS_IMMUTABLE_FL: inode is immutable + * UBIFS_APPEND_FL: writes to the inode may only append data + * UBIFS_DIRSYNC_FL: I/O on this directory inode has to be synchronous + * UBIFS_XATTR_FL: this inode is the inode for an extended attribute value + * UBIFS_CRYPT_FL: use encryption for this inode + * + * Note, these are on-flash flags which correspond to ioctl flags + * (@FS_COMPR_FL, etc). They have the same values now, but generally, do not + * have to be the same. + */ +enum { + UBIFS_COMPR_FL = 0x01, + UBIFS_SYNC_FL = 0x02, + UBIFS_IMMUTABLE_FL = 0x04, + UBIFS_APPEND_FL = 0x08, + UBIFS_DIRSYNC_FL = 0x10, + UBIFS_XATTR_FL = 0x20, + UBIFS_CRYPT_FL = 0x40, +}; + +/* Inode flag bits used by UBIFS */ +#define UBIFS_FL_MASK 0x0000001F + +/* + * UBIFS compression algorithms. + * + * UBIFS_COMPR_NONE: no compression + * UBIFS_COMPR_LZO: LZO compression + * UBIFS_COMPR_ZLIB: ZLIB compression + * UBIFS_COMPR_ZSTD: ZSTD compression + * UBIFS_COMPR_TYPES_CNT: count of supported compression types + */ +enum { + UBIFS_COMPR_NONE, + UBIFS_COMPR_LZO, + UBIFS_COMPR_ZLIB, + UBIFS_COMPR_ZSTD, + UBIFS_COMPR_TYPES_CNT, +}; + +/* + * UBIFS node types. + * + * UBIFS_INO_NODE: inode node + * UBIFS_DATA_NODE: data node + * UBIFS_DENT_NODE: directory entry node + * UBIFS_XENT_NODE: extended attribute node + * UBIFS_TRUN_NODE: truncation node + * UBIFS_PAD_NODE: padding node + * UBIFS_SB_NODE: superblock node + * UBIFS_MST_NODE: master node + * UBIFS_REF_NODE: LEB reference node + * UBIFS_IDX_NODE: index node + * UBIFS_CS_NODE: commit start node + * UBIFS_ORPH_NODE: orphan node + * UBIFS_AUTH_NODE: authentication node + * UBIFS_SIG_NODE: signature node + * UBIFS_NODE_TYPES_CNT: count of supported node types + * + * Note, we index arrays by these numbers, so keep them low and contiguous. + * Node type constants for inodes, direntries and so on have to be the same as + * corresponding key type constants. + */ +enum { + UBIFS_INO_NODE, + UBIFS_DATA_NODE, + UBIFS_DENT_NODE, + UBIFS_XENT_NODE, + UBIFS_TRUN_NODE, + UBIFS_PAD_NODE, + UBIFS_SB_NODE, + UBIFS_MST_NODE, + UBIFS_REF_NODE, + UBIFS_IDX_NODE, + UBIFS_CS_NODE, + UBIFS_ORPH_NODE, + UBIFS_AUTH_NODE, + UBIFS_SIG_NODE, + UBIFS_NODE_TYPES_CNT, +}; + +/* + * Master node flags. + * + * UBIFS_MST_DIRTY: rebooted uncleanly - master node is dirty + * UBIFS_MST_NO_ORPHS: no orphan inodes present + * UBIFS_MST_RCVRY: written by recovery + */ +enum { + UBIFS_MST_DIRTY = 1, + UBIFS_MST_NO_ORPHS = 2, + UBIFS_MST_RCVRY = 4, +}; + +/* + * Node group type (used by recovery to recover whole group or none). + * + * UBIFS_NO_NODE_GROUP: this node is not part of a group + * UBIFS_IN_NODE_GROUP: this node is a part of a group + * UBIFS_LAST_OF_NODE_GROUP: this node is the last in a group + */ +enum { + UBIFS_NO_NODE_GROUP = 0, + UBIFS_IN_NODE_GROUP, + UBIFS_LAST_OF_NODE_GROUP, +}; + +/* + * Superblock flags. + * + * UBIFS_FLG_BIGLPT: if "big" LPT model is used if set + * UBIFS_FLG_SPACE_FIXUP: first-mount "fixup" of free space within LEBs needed + * UBIFS_FLG_DOUBLE_HASH: store a 32bit cookie in directory entry nodes to + * support 64bit cookies for lookups by hash + * UBIFS_FLG_ENCRYPTION: this filesystem contains encrypted files + * UBIFS_FLG_AUTHENTICATION: this filesystem contains hashes for authentication + */ +enum { + UBIFS_FLG_BIGLPT = 0x02, + UBIFS_FLG_SPACE_FIXUP = 0x04, + UBIFS_FLG_DOUBLE_HASH = 0x08, + UBIFS_FLG_ENCRYPTION = 0x10, + UBIFS_FLG_AUTHENTICATION = 0x20, +}; + +#define UBIFS_FLG_MASK (UBIFS_FLG_BIGLPT | UBIFS_FLG_SPACE_FIXUP | \ + UBIFS_FLG_DOUBLE_HASH | UBIFS_FLG_ENCRYPTION | \ + UBIFS_FLG_AUTHENTICATION) + +/** + * struct ubifs_ch - common header node. + * @magic: UBIFS node magic number (%UBIFS_NODE_MAGIC) + * @crc: CRC-32 checksum of the node header + * @sqnum: sequence number + * @len: full node length + * @node_type: node type + * @group_type: node group type + * @padding: reserved for future, zeroes + * + * Every UBIFS node starts with this common part. If the node has a key, the + * key always goes next. + */ +struct ubifs_ch { + __le32 magic; + __le32 crc; + __le64 sqnum; + __le32 len; + __u8 node_type; + __u8 group_type; + __u8 padding[2]; +} __packed; + +/** + * union ubifs_dev_desc - device node descriptor. + * @new: new type device descriptor + * @huge: huge type device descriptor + * + * This data structure describes major/minor numbers of a device node. In an + * inode is a device node then its data contains an object of this type. UBIFS + * uses standard Linux "new" and "huge" device node encodings. + */ +union ubifs_dev_desc { + __le32 new; + __le64 huge; +} __packed; + +/** + * struct ubifs_ino_node - inode node. + * @ch: common header + * @key: node key + * @creat_sqnum: sequence number at time of creation + * @size: inode size in bytes (amount of uncompressed data) + * @atime_sec: access time seconds + * @ctime_sec: creation time seconds + * @mtime_sec: modification time seconds + * @atime_nsec: access time nanoseconds + * @ctime_nsec: creation time nanoseconds + * @mtime_nsec: modification time nanoseconds + * @nlink: number of hard links + * @uid: owner ID + * @gid: group ID + * @mode: access flags + * @flags: per-inode flags (%UBIFS_COMPR_FL, %UBIFS_SYNC_FL, etc) + * @data_len: inode data length + * @xattr_cnt: count of extended attributes this inode has + * @xattr_size: summarized size of all extended attributes in bytes + * @padding1: reserved for future, zeroes + * @xattr_names: sum of lengths of all extended attribute names belonging to + * this inode + * @compr_type: compression type used for this inode + * @padding2: reserved for future, zeroes + * @data: data attached to the inode + * + * Note, even though inode compression type is defined by @compr_type, some + * nodes of this inode may be compressed with different compressor - this + * happens if compression type is changed while the inode already has data + * nodes. But @compr_type will be use for further writes to the inode. + * + * Note, do not forget to amend 'zero_ino_node_unused()' function when changing + * the padding fields. + */ +struct ubifs_ino_node { + struct ubifs_ch ch; + __u8 key[UBIFS_MAX_KEY_LEN]; + __le64 creat_sqnum; + __le64 size; + __le64 atime_sec; + __le64 ctime_sec; + __le64 mtime_sec; + __le32 atime_nsec; + __le32 ctime_nsec; + __le32 mtime_nsec; + __le32 nlink; + __le32 uid; + __le32 gid; + __le32 mode; + __le32 flags; + __le32 data_len; + __le32 xattr_cnt; + __le32 xattr_size; + __u8 padding1[4]; /* Watch 'zero_ino_node_unused()' if changing! */ + __le32 xattr_names; + __le16 compr_type; + __u8 padding2[26]; /* Watch 'zero_ino_node_unused()' if changing! */ + __u8 data[]; +} __packed; + +/** + * struct ubifs_dent_node - directory entry node. + * @ch: common header + * @key: node key + * @inum: target inode number + * @padding1: reserved for future, zeroes + * @type: type of the target inode (%UBIFS_ITYPE_REG, %UBIFS_ITYPE_DIR, etc) + * @nlen: name length + * @cookie: A 32bits random number, used to construct a 64bits + * identifier. + * @name: zero-terminated name + * + * Note, do not forget to amend 'zero_dent_node_unused()' function when + * changing the padding fields. + */ +struct ubifs_dent_node { + struct ubifs_ch ch; + __u8 key[UBIFS_MAX_KEY_LEN]; + __le64 inum; + __u8 padding1; + __u8 type; + __le16 nlen; + __le32 cookie; + __u8 name[]; +} __packed; + +/** + * struct ubifs_data_node - data node. + * @ch: common header + * @key: node key + * @size: uncompressed data size in bytes + * @compr_type: compression type (%UBIFS_COMPR_NONE, %UBIFS_COMPR_LZO, etc) + * @compr_size: compressed data size in bytes, only valid when data is encrypted + * @data: data + * + */ +struct ubifs_data_node { + struct ubifs_ch ch; + __u8 key[UBIFS_MAX_KEY_LEN]; + __le32 size; + __le16 compr_type; + __le16 compr_size; + __u8 data[]; +} __packed; + +/** + * struct ubifs_trun_node - truncation node. + * @ch: common header + * @inum: truncated inode number + * @padding: reserved for future, zeroes + * @old_size: size before truncation + * @new_size: size after truncation + * + * This node exists only in the journal and never goes to the main area. Note, + * do not forget to amend 'zero_trun_node_unused()' function when changing the + * padding fields. + */ +struct ubifs_trun_node { + struct ubifs_ch ch; + __le32 inum; + __u8 padding[12]; /* Watch 'zero_trun_node_unused()' if changing! */ + __le64 old_size; + __le64 new_size; +} __packed; + +/** + * struct ubifs_pad_node - padding node. + * @ch: common header + * @pad_len: how many bytes after this node are unused (because padded) + * @padding: reserved for future, zeroes + */ +struct ubifs_pad_node { + struct ubifs_ch ch; + __le32 pad_len; +} __packed; + +/** + * struct ubifs_sb_node - superblock node. + * @ch: common header + * @padding: reserved for future, zeroes + * @key_hash: type of hash function used in keys + * @key_fmt: format of the key + * @flags: file-system flags (%UBIFS_FLG_BIGLPT, etc) + * @min_io_size: minimal input/output unit size + * @leb_size: logical eraseblock size in bytes + * @leb_cnt: count of LEBs used by file-system + * @max_leb_cnt: maximum count of LEBs used by file-system + * @max_bud_bytes: maximum amount of data stored in buds + * @log_lebs: log size in logical eraseblocks + * @lpt_lebs: number of LEBs used for lprops table + * @orph_lebs: number of LEBs used for recording orphans + * @jhead_cnt: count of journal heads + * @fanout: tree fanout (max. number of links per indexing node) + * @lsave_cnt: number of LEB numbers in LPT's save table + * @fmt_version: UBIFS on-flash format version + * @default_compr: default compression algorithm (%UBIFS_COMPR_LZO, etc) + * @padding1: reserved for future, zeroes + * @rp_uid: reserve pool UID + * @rp_gid: reserve pool GID + * @rp_size: size of the reserved pool in bytes + * @padding2: reserved for future, zeroes + * @time_gran: time granularity in nanoseconds + * @uuid: UUID generated when the file system image was created + * @ro_compat_version: UBIFS R/O compatibility version + * @hmac: HMAC to authenticate the superblock node + * @hmac_wkm: HMAC of a well known message (the string "UBIFS") as a convenience + * to the user to check if the correct key is passed. + * @hash_algo: The hash algo used for this filesystem (one of enum hash_algo) + * @hash_mst: hash of the master node, only valid for signed images in which the + * master node does not contain a hmac + */ +struct ubifs_sb_node { + struct ubifs_ch ch; + __u8 padding[2]; + __u8 key_hash; + __u8 key_fmt; + __le32 flags; + __le32 min_io_size; + __le32 leb_size; + __le32 leb_cnt; + __le32 max_leb_cnt; + __le64 max_bud_bytes; + __le32 log_lebs; + __le32 lpt_lebs; + __le32 orph_lebs; + __le32 jhead_cnt; + __le32 fanout; + __le32 lsave_cnt; + __le32 fmt_version; + __le16 default_compr; + __u8 padding1[2]; + __le32 rp_uid; + __le32 rp_gid; + __le64 rp_size; + __le32 time_gran; + __u8 uuid[16]; + __le32 ro_compat_version; + __u8 hmac[UBIFS_MAX_HMAC_LEN]; + __u8 hmac_wkm[UBIFS_MAX_HMAC_LEN]; + __le16 hash_algo; + __u8 hash_mst[UBIFS_MAX_HASH_LEN]; + __u8 padding2[3774]; +} __packed; + +/** + * struct ubifs_mst_node - master node. + * @ch: common header + * @highest_inum: highest inode number in the committed index + * @cmt_no: commit number + * @flags: various flags (%UBIFS_MST_DIRTY, etc) + * @log_lnum: start of the log + * @root_lnum: LEB number of the root indexing node + * @root_offs: offset within @root_lnum + * @root_len: root indexing node length + * @gc_lnum: LEB reserved for garbage collection (%-1 value means the LEB was + * not reserved and should be reserved on mount) + * @ihead_lnum: LEB number of index head + * @ihead_offs: offset of index head + * @index_size: size of index on flash + * @total_free: total free space in bytes + * @total_dirty: total dirty space in bytes + * @total_used: total used space in bytes (includes only data LEBs) + * @total_dead: total dead space in bytes (includes only data LEBs) + * @total_dark: total dark space in bytes (includes only data LEBs) + * @lpt_lnum: LEB number of LPT root nnode + * @lpt_offs: offset of LPT root nnode + * @nhead_lnum: LEB number of LPT head + * @nhead_offs: offset of LPT head + * @ltab_lnum: LEB number of LPT's own lprops table + * @ltab_offs: offset of LPT's own lprops table + * @lsave_lnum: LEB number of LPT's save table (big model only) + * @lsave_offs: offset of LPT's save table (big model only) + * @lscan_lnum: LEB number of last LPT scan + * @empty_lebs: number of empty logical eraseblocks + * @idx_lebs: number of indexing logical eraseblocks + * @leb_cnt: count of LEBs used by file-system + * @hash_root_idx: the hash of the root index node + * @hash_lpt: the hash of the LPT + * @hmac: HMAC to authenticate the master node + * @padding: reserved for future, zeroes + */ +struct ubifs_mst_node { + struct ubifs_ch ch; + __le64 highest_inum; + __le64 cmt_no; + __le32 flags; + __le32 log_lnum; + __le32 root_lnum; + __le32 root_offs; + __le32 root_len; + __le32 gc_lnum; + __le32 ihead_lnum; + __le32 ihead_offs; + __le64 index_size; + __le64 total_free; + __le64 total_dirty; + __le64 total_used; + __le64 total_dead; + __le64 total_dark; + __le32 lpt_lnum; + __le32 lpt_offs; + __le32 nhead_lnum; + __le32 nhead_offs; + __le32 ltab_lnum; + __le32 ltab_offs; + __le32 lsave_lnum; + __le32 lsave_offs; + __le32 lscan_lnum; + __le32 empty_lebs; + __le32 idx_lebs; + __le32 leb_cnt; + __u8 hash_root_idx[UBIFS_MAX_HASH_LEN]; + __u8 hash_lpt[UBIFS_MAX_HASH_LEN]; + __u8 hmac[UBIFS_MAX_HMAC_LEN]; + __u8 padding[152]; +} __packed; + +/** + * struct ubifs_ref_node - logical eraseblock reference node. + * @ch: common header + * @lnum: the referred logical eraseblock number + * @offs: start offset in the referred LEB + * @jhead: journal head number + * @padding: reserved for future, zeroes + */ +struct ubifs_ref_node { + struct ubifs_ch ch; + __le32 lnum; + __le32 offs; + __le32 jhead; + __u8 padding[28]; +} __packed; + +/** + * struct ubifs_auth_node - node for authenticating other nodes + * @ch: common header + * @hmac: The HMAC + */ +struct ubifs_auth_node { + struct ubifs_ch ch; + __u8 hmac[]; +} __packed; + +/** + * struct ubifs_sig_node - node for signing other nodes + * @ch: common header + * @type: type of the signature, currently only UBIFS_SIGNATURE_TYPE_PKCS7 + * supported + * @len: The length of the signature data + * @padding: reserved for future, zeroes + * @sig: The signature data + */ +struct ubifs_sig_node { + struct ubifs_ch ch; + __le32 type; + __le32 len; + __u8 padding[32]; + __u8 sig[]; +} __packed; + +/** + * struct ubifs_branch - key/reference/length branch + * @lnum: LEB number of the target node + * @offs: offset within @lnum + * @len: target node length + * @key: key + * + * In an authenticated UBIFS we have the hash of the referenced node after @key. + * This can't be added to the struct type definition because @key is a + * dynamically sized element already. + */ +struct ubifs_branch { + __le32 lnum; + __le32 offs; + __le32 len; + __u8 key[]; +} __packed; + +/** + * struct ubifs_idx_node - indexing node. + * @ch: common header + * @child_cnt: number of child index nodes + * @level: tree level + * @branches: LEB number / offset / length / key branches + */ +struct ubifs_idx_node { + struct ubifs_ch ch; + __le16 child_cnt; + __le16 level; + __u8 branches[]; +} __packed; + +/** + * struct ubifs_cs_node - commit start node. + * @ch: common header + * @cmt_no: commit number + */ +struct ubifs_cs_node { + struct ubifs_ch ch; + __le64 cmt_no; +} __packed; + +/** + * struct ubifs_orph_node - orphan node. + * @ch: common header + * @cmt_no: commit number (also top bit is set on the last node of the commit) + * @inos: inode numbers of orphans + */ +struct ubifs_orph_node { + struct ubifs_ch ch; + __le64 cmt_no; + __le64 inos[]; +} __packed; + +#endif /* __UBIFS_MEDIA_H__ */ diff --git a/fs/ubifs/ubifs.h b/fs/ubifs/ubifs.h new file mode 100644 index 000000000..b62a154c7 --- /dev/null +++ b/fs/ubifs/ubifs.h @@ -0,0 +1,2159 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation + * + * Authors: Artem Bityutskiy (Битюцкий Артём) + * Adrian Hunter + */ + +#ifndef __UBIFS_H__ +#define __UBIFS_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ubifs-media.h" + +/* Version of this UBIFS implementation */ +#define UBIFS_VERSION 1 + +/* UBIFS file system VFS magic number */ +#define UBIFS_SUPER_MAGIC 0x24051905 + +/* Number of UBIFS blocks per VFS page */ +#define UBIFS_BLOCKS_PER_PAGE (PAGE_SIZE / UBIFS_BLOCK_SIZE) +#define UBIFS_BLOCKS_PER_PAGE_SHIFT (PAGE_SHIFT - UBIFS_BLOCK_SHIFT) + +/* "File system end of life" sequence number watermark */ +#define SQNUM_WARN_WATERMARK 0xFFFFFFFF00000000ULL +#define SQNUM_WATERMARK 0xFFFFFFFFFF000000ULL + +/* + * Minimum amount of LEBs reserved for the index. At present the index needs at + * least 2 LEBs: one for the index head and one for in-the-gaps method (which + * currently does not cater for the index head and so excludes it from + * consideration). + */ +#define MIN_INDEX_LEBS 2 + +/* Minimum amount of data UBIFS writes to the flash */ +#define MIN_WRITE_SZ (UBIFS_DATA_NODE_SZ + 8) + +/* + * Currently we do not support inode number overlapping and re-using, so this + * watermark defines dangerous inode number level. This should be fixed later, + * although it is difficult to exceed current limit. Another option is to use + * 64-bit inode numbers, but this means more overhead. + */ +#define INUM_WARN_WATERMARK 0xFFF00000 +#define INUM_WATERMARK 0xFFFFFF00 + +/* Maximum number of entries in each LPT (LEB category) heap */ +#define LPT_HEAP_SZ 256 + +/* + * Background thread name pattern. The numbers are UBI device and volume + * numbers. + */ +#define BGT_NAME_PATTERN "ubifs_bgt%d_%d" + +/* Maximum possible inode number (only 32-bit inodes are supported now) */ +#define MAX_INUM 0xFFFFFFFF + +/* Number of non-data journal heads */ +#define NONDATA_JHEADS_CNT 2 + +/* Shorter names for journal head numbers for internal usage */ +#define GCHD UBIFS_GC_HEAD +#define BASEHD UBIFS_BASE_HEAD +#define DATAHD UBIFS_DATA_HEAD + +/* 'No change' value for 'ubifs_change_lp()' */ +#define LPROPS_NC 0x80000001 + +/* + * There is no notion of truncation key because truncation nodes do not exist + * in TNC. However, when replaying, it is handy to introduce fake "truncation" + * keys for truncation nodes because the code becomes simpler. So we define + * %UBIFS_TRUN_KEY type. + * + * But otherwise, out of the journal reply scope, the truncation keys are + * invalid. + */ +#define UBIFS_TRUN_KEY UBIFS_KEY_TYPES_CNT +#define UBIFS_INVALID_KEY UBIFS_KEY_TYPES_CNT + +/* + * How much a directory entry/extended attribute entry adds to the parent/host + * inode. + */ +#define CALC_DENT_SIZE(name_len) ALIGN(UBIFS_DENT_NODE_SZ + (name_len) + 1, 8) + +/* How much an extended attribute adds to the host inode */ +#define CALC_XATTR_BYTES(data_len) ALIGN(UBIFS_INO_NODE_SZ + (data_len) + 1, 8) + +/* + * Znodes which were not touched for 'OLD_ZNODE_AGE' seconds are considered + * "old", and znode which were touched last 'YOUNG_ZNODE_AGE' seconds ago are + * considered "young". This is used by shrinker when selecting znode to trim + * off. + */ +#define OLD_ZNODE_AGE 20 +#define YOUNG_ZNODE_AGE 5 + +#ifdef CONFIG_FS_ENCRYPTION +#define UBIFS_CIPHER_BLOCK_SIZE FSCRYPT_CONTENTS_ALIGNMENT +#else +#define UBIFS_CIPHER_BLOCK_SIZE 0 +#endif + +/* + * How much memory is needed for a buffer where we compress a data node. + */ +#define COMPRESSED_DATA_NODE_BUF_SZ \ + (UBIFS_DATA_NODE_SZ + UBIFS_BLOCK_SIZE) + +/* Maximum expected tree height for use by bottom_up_buf */ +#define BOTTOM_UP_HEIGHT 64 + +/* Maximum number of data nodes to bulk-read */ +#define UBIFS_MAX_BULK_READ 32 + +#ifdef CONFIG_UBIFS_FS_AUTHENTICATION +#define UBIFS_HASH_ARR_SZ UBIFS_MAX_HASH_LEN +#define UBIFS_HMAC_ARR_SZ UBIFS_MAX_HMAC_LEN +#else +#define UBIFS_HASH_ARR_SZ 0 +#define UBIFS_HMAC_ARR_SZ 0 +#endif + +/* + * Lockdep classes for UBIFS inode @ui_mutex. + */ +enum { + WB_MUTEX_1 = 0, + WB_MUTEX_2 = 1, + WB_MUTEX_3 = 2, + WB_MUTEX_4 = 3, +}; + +/* + * Znode flags (actually, bit numbers which store the flags). + * + * DIRTY_ZNODE: znode is dirty + * COW_ZNODE: znode is being committed and a new instance of this znode has to + * be created before changing this znode + * OBSOLETE_ZNODE: znode is obsolete, which means it was deleted, but it is + * still in the commit list and the ongoing commit operation + * will commit it, and delete this znode after it is done + */ +enum { + DIRTY_ZNODE = 0, + COW_ZNODE = 1, + OBSOLETE_ZNODE = 2, +}; + +/* + * Commit states. + * + * COMMIT_RESTING: commit is not wanted + * COMMIT_BACKGROUND: background commit has been requested + * COMMIT_REQUIRED: commit is required + * COMMIT_RUNNING_BACKGROUND: background commit is running + * COMMIT_RUNNING_REQUIRED: commit is running and it is required + * COMMIT_BROKEN: commit failed + */ +enum { + COMMIT_RESTING = 0, + COMMIT_BACKGROUND, + COMMIT_REQUIRED, + COMMIT_RUNNING_BACKGROUND, + COMMIT_RUNNING_REQUIRED, + COMMIT_BROKEN, +}; + +/* + * 'ubifs_scan_a_node()' return values. + * + * SCANNED_GARBAGE: scanned garbage + * SCANNED_EMPTY_SPACE: scanned empty space + * SCANNED_A_NODE: scanned a valid node + * SCANNED_A_CORRUPT_NODE: scanned a corrupted node + * SCANNED_A_BAD_PAD_NODE: scanned a padding node with invalid pad length + * + * Greater than zero means: 'scanned that number of padding bytes' + */ +enum { + SCANNED_GARBAGE = 0, + SCANNED_EMPTY_SPACE = -1, + SCANNED_A_NODE = -2, + SCANNED_A_CORRUPT_NODE = -3, + SCANNED_A_BAD_PAD_NODE = -4, +}; + +/* + * LPT cnode flag bits. + * + * DIRTY_CNODE: cnode is dirty + * OBSOLETE_CNODE: cnode is being committed and has been copied (or deleted), + * so it can (and must) be freed when the commit is finished + * COW_CNODE: cnode is being committed and must be copied before writing + */ +enum { + DIRTY_CNODE = 0, + OBSOLETE_CNODE = 1, + COW_CNODE = 2, +}; + +/* + * Dirty flag bits (lpt_drty_flgs) for LPT special nodes. + * + * LTAB_DIRTY: ltab node is dirty + * LSAVE_DIRTY: lsave node is dirty + */ +enum { + LTAB_DIRTY = 1, + LSAVE_DIRTY = 2, +}; + +/* + * Return codes used by the garbage collector. + * @LEB_FREED: the logical eraseblock was freed and is ready to use + * @LEB_FREED_IDX: indexing LEB was freed and can be used only after the commit + * @LEB_RETAINED: the logical eraseblock was freed and retained for GC purposes + */ +enum { + LEB_FREED, + LEB_FREED_IDX, + LEB_RETAINED, +}; + +/* + * Action taken upon a failed ubifs_assert(). + * @ASSACT_REPORT: just report the failed assertion + * @ASSACT_RO: switch to read-only mode + * @ASSACT_PANIC: call BUG() and possible panic the kernel + */ +enum { + ASSACT_REPORT = 0, + ASSACT_RO, + ASSACT_PANIC, +}; + +struct folio; + +/** + * struct ubifs_old_idx - index node obsoleted since last commit start. + * @rb: rb-tree node + * @lnum: LEB number of obsoleted index node + * @offs: offset of obsoleted index node + */ +struct ubifs_old_idx { + struct rb_node rb; + int lnum; + int offs; +}; + +/* The below union makes it easier to deal with keys */ +union ubifs_key { + uint8_t u8[UBIFS_SK_LEN]; + uint32_t u32[UBIFS_SK_LEN/4]; + uint64_t u64[UBIFS_SK_LEN/8]; + __le32 j32[UBIFS_SK_LEN/4]; +}; + +/** + * struct ubifs_scan_node - UBIFS scanned node information. + * @list: list of scanned nodes + * @key: key of node scanned (if it has one) + * @sqnum: sequence number + * @type: type of node scanned + * @offs: offset with LEB of node scanned + * @len: length of node scanned + * @node: raw node + */ +struct ubifs_scan_node { + struct list_head list; + union ubifs_key key; + unsigned long long sqnum; + int type; + int offs; + int len; + void *node; +}; + +/** + * struct ubifs_scan_leb - UBIFS scanned LEB information. + * @lnum: logical eraseblock number + * @nodes_cnt: number of nodes scanned + * @nodes: list of struct ubifs_scan_node + * @endpt: end point (and therefore the start of empty space) + * @buf: buffer containing entire LEB scanned + */ +struct ubifs_scan_leb { + int lnum; + int nodes_cnt; + struct list_head nodes; + int endpt; + void *buf; +}; + +/** + * struct ubifs_gced_idx_leb - garbage-collected indexing LEB. + * @list: list + * @lnum: LEB number + * @unmap: OK to unmap this LEB + * + * This data structure is used to temporary store garbage-collected indexing + * LEBs - they are not released immediately, but only after the next commit. + * This is needed to guarantee recoverability. + */ +struct ubifs_gced_idx_leb { + struct list_head list; + int lnum; + int unmap; +}; + +/** + * struct ubifs_inode - UBIFS in-memory inode description. + * @vfs_inode: VFS inode description object + * @creat_sqnum: sequence number at time of creation + * @del_cmtno: commit number corresponding to the time the inode was deleted, + * protected by @c->commit_sem; + * @xattr_size: summarized size of all extended attributes in bytes + * @xattr_cnt: count of extended attributes this inode has + * @xattr_names: sum of lengths of all extended attribute names belonging to + * this inode + * @dirty: non-zero if the inode is dirty + * @xattr: non-zero if this is an extended attribute inode + * @bulk_read: non-zero if bulk-read should be used + * @ui_mutex: serializes inode write-back with the rest of VFS operations, + * serializes "clean <-> dirty" state changes, serializes bulk-read, + * protects @dirty, @bulk_read, @ui_size, and @xattr_size + * @xattr_sem: serilizes write operations (remove|set|create) on xattr + * @ui_lock: protects @synced_i_size + * @synced_i_size: synchronized size of inode, i.e. the value of inode size + * currently stored on the flash; used only for regular file + * inodes + * @ui_size: inode size used by UBIFS when writing to flash + * @flags: inode flags (@UBIFS_COMPR_FL, etc) + * @compr_type: default compression type used for this inode + * @last_page_read: page number of last page read (for bulk read) + * @read_in_a_row: number of consecutive pages read in a row (for bulk read) + * @data_len: length of the data attached to the inode + * @data: inode's data + * @i_crypt_info: inode's fscrypt information + * + * @ui_mutex exists for two main reasons. At first it prevents inodes from + * being written back while UBIFS changing them, being in the middle of an VFS + * operation. This way UBIFS makes sure the inode fields are consistent. For + * example, in 'ubifs_rename()' we change 4 inodes simultaneously, and + * write-back must not write any of them before we have finished. + * + * The second reason is budgeting - UBIFS has to budget all operations. If an + * operation is going to mark an inode dirty, it has to allocate budget for + * this. It cannot just mark it dirty because there is no guarantee there will + * be enough flash space to write the inode back later. This means UBIFS has + * to have full control over inode "clean <-> dirty" transitions (and pages + * actually). But unfortunately, VFS marks inodes dirty in many places, and it + * does not ask the file-system if it is allowed to do so (there is a notifier, + * but it is not enough), i.e., there is no mechanism to synchronize with this. + * So UBIFS has its own inode dirty flag and its own mutex to serialize + * "clean <-> dirty" transitions. + * + * The @synced_i_size field is used to make sure we never write pages which are + * beyond last synchronized inode size. See 'ubifs_writepage()' for more + * information. + * + * The @ui_size is a "shadow" variable for @inode->i_size and UBIFS uses + * @ui_size instead of @inode->i_size. The reason for this is that UBIFS cannot + * make sure @inode->i_size is always changed under @ui_mutex, because it + * cannot call 'truncate_setsize()' with @ui_mutex locked, because it would + * deadlock with 'ubifs_writepage()' (see file.c). All the other inode fields + * are changed under @ui_mutex, so they do not need "shadow" fields. Note, one + * could consider to rework locking and base it on "shadow" fields. + */ +struct ubifs_inode { + struct inode vfs_inode; + unsigned long long creat_sqnum; + unsigned long long del_cmtno; + unsigned int xattr_size; + unsigned int xattr_cnt; + unsigned int xattr_names; + unsigned int dirty:1; + unsigned int xattr:1; + unsigned int bulk_read:1; + unsigned int compr_type:2; + struct mutex ui_mutex; + struct rw_semaphore xattr_sem; + spinlock_t ui_lock; + loff_t synced_i_size; + loff_t ui_size; + int flags; + pgoff_t last_page_read; + pgoff_t read_in_a_row; + int data_len; + void *data; +#ifdef CONFIG_FS_ENCRYPTION + struct fscrypt_inode_info *i_crypt_info; +#endif +}; + +/** + * struct ubifs_unclean_leb - records a LEB recovered under read-only mode. + * @list: list + * @lnum: LEB number of recovered LEB + * @endpt: offset where recovery ended + * + * This structure records a LEB identified during recovery that needs to be + * cleaned but was not because UBIFS was mounted read-only. The information + * is used to clean the LEB when remounting to read-write mode. + */ +struct ubifs_unclean_leb { + struct list_head list; + int lnum; + int endpt; +}; + +/* + * LEB properties flags. + * + * LPROPS_UNCAT: not categorized + * LPROPS_DIRTY: dirty > free, dirty >= @c->dead_wm, not index + * LPROPS_DIRTY_IDX: dirty + free > @c->min_idx_node_sze and index + * LPROPS_FREE: free > 0, dirty < @c->dead_wm, not empty, not index + * LPROPS_HEAP_CNT: number of heaps used for storing categorized LEBs + * LPROPS_EMPTY: LEB is empty, not taken + * LPROPS_FREEABLE: free + dirty == leb_size, not index, not taken + * LPROPS_FRDI_IDX: free + dirty == leb_size and index, may be taken + * LPROPS_CAT_MASK: mask for the LEB categories above + * LPROPS_TAKEN: LEB was taken (this flag is not saved on the media) + * LPROPS_INDEX: LEB contains indexing nodes (this flag also exists on flash) + */ +enum { + LPROPS_UNCAT = 0, + LPROPS_DIRTY = 1, + LPROPS_DIRTY_IDX = 2, + LPROPS_FREE = 3, + LPROPS_HEAP_CNT = 3, + LPROPS_EMPTY = 4, + LPROPS_FREEABLE = 5, + LPROPS_FRDI_IDX = 6, + LPROPS_CAT_MASK = 15, + LPROPS_TAKEN = 16, + LPROPS_INDEX = 32, +}; + +/** + * struct ubifs_lprops - logical eraseblock properties. + * @free: amount of free space in bytes + * @dirty: amount of dirty space in bytes + * @flags: LEB properties flags (see above) + * @lnum: LEB number + * @list: list of same-category lprops (for LPROPS_EMPTY and LPROPS_FREEABLE) + * @hpos: heap position in heap of same-category lprops (other categories) + */ +struct ubifs_lprops { + int free; + int dirty; + int flags; + int lnum; + union { + struct list_head list; + int hpos; + }; +}; + +/** + * struct ubifs_lpt_lprops - LPT logical eraseblock properties. + * @free: amount of free space in bytes + * @dirty: amount of dirty space in bytes + * @tgc: trivial GC flag (1 => unmap after commit end) + * @cmt: commit flag (1 => reserved for commit) + */ +struct ubifs_lpt_lprops { + int free; + int dirty; + unsigned tgc:1; + unsigned cmt:1; +}; + +/** + * struct ubifs_lp_stats - statistics of eraseblocks in the main area. + * @empty_lebs: number of empty LEBs + * @taken_empty_lebs: number of taken LEBs + * @idx_lebs: number of indexing LEBs + * @total_free: total free space in bytes (includes all LEBs) + * @total_dirty: total dirty space in bytes (includes all LEBs) + * @total_used: total used space in bytes (does not include index LEBs) + * @total_dead: total dead space in bytes (does not include index LEBs) + * @total_dark: total dark space in bytes (does not include index LEBs) + * + * The @taken_empty_lebs field counts the LEBs that are in the transient state + * of having been "taken" for use but not yet written to. @taken_empty_lebs is + * needed to account correctly for @gc_lnum, otherwise @empty_lebs could be + * used by itself (in which case 'unused_lebs' would be a better name). In the + * case of @gc_lnum, it is "taken" at mount time or whenever a LEB is retained + * by GC, but unlike other empty LEBs that are "taken", it may not be written + * straight away (i.e. before the next commit start or unmount), so either + * @gc_lnum must be specially accounted for, or the current approach followed + * i.e. count it under @taken_empty_lebs. + * + * @empty_lebs includes @taken_empty_lebs. + * + * @total_used, @total_dead and @total_dark fields do not account indexing + * LEBs. + */ +struct ubifs_lp_stats { + int empty_lebs; + int taken_empty_lebs; + int idx_lebs; + long long total_free; + long long total_dirty; + long long total_used; + long long total_dead; + long long total_dark; +}; + +struct ubifs_nnode; + +/** + * struct ubifs_cnode - LEB Properties Tree common node. + * @parent: parent nnode + * @cnext: next cnode to commit + * @flags: flags (%DIRTY_LPT_NODE or %OBSOLETE_LPT_NODE) + * @iip: index in parent + * @level: level in the tree (zero for pnodes, greater than zero for nnodes) + * @num: node number + */ +struct ubifs_cnode { + struct ubifs_nnode *parent; + struct ubifs_cnode *cnext; + unsigned long flags; + int iip; + int level; + int num; +}; + +/** + * struct ubifs_pnode - LEB Properties Tree leaf node. + * @parent: parent nnode + * @cnext: next cnode to commit + * @flags: flags (%DIRTY_LPT_NODE or %OBSOLETE_LPT_NODE) + * @iip: index in parent + * @level: level in the tree (always zero for pnodes) + * @num: node number + * @lprops: LEB properties array + */ +struct ubifs_pnode { + struct ubifs_nnode *parent; + struct ubifs_cnode *cnext; + unsigned long flags; + int iip; + int level; + int num; + struct ubifs_lprops lprops[UBIFS_LPT_FANOUT]; +}; + +/** + * struct ubifs_nbranch - LEB Properties Tree internal node branch. + * @lnum: LEB number of child + * @offs: offset of child + * @nnode: nnode child + * @pnode: pnode child + * @cnode: cnode child + */ +struct ubifs_nbranch { + int lnum; + int offs; + union { + struct ubifs_nnode *nnode; + struct ubifs_pnode *pnode; + struct ubifs_cnode *cnode; + }; +}; + +/** + * struct ubifs_nnode - LEB Properties Tree internal node. + * @parent: parent nnode + * @cnext: next cnode to commit + * @flags: flags (%DIRTY_LPT_NODE or %OBSOLETE_LPT_NODE) + * @iip: index in parent + * @level: level in the tree (always greater than zero for nnodes) + * @num: node number + * @nbranch: branches to child nodes + */ +struct ubifs_nnode { + struct ubifs_nnode *parent; + struct ubifs_cnode *cnext; + unsigned long flags; + int iip; + int level; + int num; + struct ubifs_nbranch nbranch[UBIFS_LPT_FANOUT]; +}; + +/** + * struct ubifs_lpt_heap - heap of categorized lprops. + * @arr: heap array + * @cnt: number in heap + * @max_cnt: maximum number allowed in heap + * + * There are %LPROPS_HEAP_CNT heaps. + */ +struct ubifs_lpt_heap { + struct ubifs_lprops **arr; + int cnt; + int max_cnt; +}; + +/* + * Return codes for LPT scan callback function. + * + * LPT_SCAN_CONTINUE: continue scanning + * LPT_SCAN_ADD: add the LEB properties scanned to the tree in memory + * LPT_SCAN_STOP: stop scanning + */ +enum { + LPT_SCAN_CONTINUE = 0, + LPT_SCAN_ADD = 1, + LPT_SCAN_STOP = 2, +}; + +struct ubifs_info; + +/* Callback used by the 'ubifs_lpt_scan_nolock()' function */ +typedef int (*ubifs_lpt_scan_callback)(struct ubifs_info *c, + const struct ubifs_lprops *lprops, + int in_tree, void *data); + +/** + * struct ubifs_wbuf - UBIFS write-buffer. + * @c: UBIFS file-system description object + * @buf: write-buffer (of min. flash I/O unit size) + * @lnum: logical eraseblock number the write-buffer points to + * @offs: write-buffer offset in this logical eraseblock + * @avail: number of bytes available in the write-buffer + * @used: number of used bytes in the write-buffer + * @size: write-buffer size (in [@c->min_io_size, @c->max_write_size] range) + * @jhead: journal head the mutex belongs to (note, needed only to shut lockdep + * up by 'mutex_lock_nested()). + * @sync_callback: write-buffer synchronization callback + * @io_mutex: serializes write-buffer I/O + * @lock: serializes @buf, @lnum, @offs, @avail, @used, @next_ino and @inodes + * fields + * @timer: write-buffer timer + * @no_timer: non-zero if this write-buffer does not have a timer + * @need_sync: non-zero if the timer expired and the wbuf needs sync'ing + * @next_ino: points to the next position of the following inode number + * @inodes: stores the inode numbers of the nodes which are in wbuf + * + * The write-buffer synchronization callback is called when the write-buffer is + * synchronized in order to notify how much space was wasted due to + * write-buffer padding and how much free space is left in the LEB. + * + * Note: the fields @buf, @lnum, @offs, @avail and @used can be read under + * spin-lock or mutex because they are written under both mutex and spin-lock. + * @buf is appended to under mutex but overwritten under both mutex and + * spin-lock. Thus the data between @buf and @buf + @used can be read under + * spinlock. + */ +struct ubifs_wbuf { + struct ubifs_info *c; + void *buf; + int lnum; + int offs; + int avail; + int used; + int size; + int jhead; + int (*sync_callback)(struct ubifs_info *c, int lnum, int free, int pad); + struct mutex io_mutex; + spinlock_t lock; + struct hrtimer timer; + unsigned int no_timer:1; + unsigned int need_sync:1; + int next_ino; + ino_t *inodes; +}; + +/** + * struct ubifs_bud - bud logical eraseblock. + * @lnum: logical eraseblock number + * @start: where the (uncommitted) bud data starts + * @jhead: journal head number this bud belongs to + * @list: link in the list buds belonging to the same journal head + * @rb: link in the tree of all buds + * @log_hash: the log hash from the commit start node up to this bud + */ +struct ubifs_bud { + int lnum; + int start; + int jhead; + struct list_head list; + struct rb_node rb; + struct shash_desc *log_hash; +}; + +/** + * struct ubifs_jhead - journal head. + * @wbuf: head's write-buffer + * @buds_list: list of bud LEBs belonging to this journal head + * @grouped: non-zero if UBIFS groups nodes when writing to this journal head + * @log_hash: the log hash from the commit start node up to this journal head + * + * Note, the @buds list is protected by the @c->buds_lock. + */ +struct ubifs_jhead { + struct ubifs_wbuf wbuf; + struct list_head buds_list; + unsigned int grouped:1; + struct shash_desc *log_hash; +}; + +/** + * struct ubifs_zbranch - key/coordinate/length branch stored in znodes. + * @key: key + * @znode: znode address in memory + * @lnum: LEB number of the target node (indexing node or data node) + * @offs: target node offset within @lnum + * @len: target node length + * @hash: the hash of the target node + */ +struct ubifs_zbranch { + union ubifs_key key; + union { + struct ubifs_znode *znode; + void *leaf; + }; + int lnum; + int offs; + int len; + u8 hash[UBIFS_HASH_ARR_SZ]; +}; + +/** + * struct ubifs_znode - in-memory representation of an indexing node. + * @parent: parent znode or NULL if it is the root + * @cnext: next znode to commit + * @cparent: parent node for this commit + * @ciip: index in cparent's zbranch array + * @flags: znode flags (%DIRTY_ZNODE, %COW_ZNODE or %OBSOLETE_ZNODE) + * @time: last access time (seconds) + * @level: level of the entry in the TNC tree + * @child_cnt: count of child znodes + * @iip: index in parent's zbranch array + * @alt: lower bound of key range has altered i.e. child inserted at slot 0 + * @lnum: LEB number of the corresponding indexing node + * @offs: offset of the corresponding indexing node + * @len: length of the corresponding indexing node + * @zbranch: array of znode branches (@c->fanout elements) + * + * Note! The @lnum, @offs, and @len fields are not really needed - we have them + * only for internal consistency check. They could be removed to save some RAM. + */ +struct ubifs_znode { + struct ubifs_znode *parent; + struct ubifs_znode *cnext; + struct ubifs_znode *cparent; + int ciip; + unsigned long flags; + time64_t time; + int level; + int child_cnt; + int iip; + int alt; + int lnum; + int offs; + int len; + struct ubifs_zbranch zbranch[]; +}; + +/** + * struct bu_info - bulk-read information. + * @key: first data node key + * @zbranch: zbranches of data nodes to bulk read + * @buf: buffer to read into + * @buf_len: buffer length + * @gc_seq: GC sequence number to detect races with GC + * @cnt: number of data nodes for bulk read + * @blk_cnt: number of data blocks including holes + * @oef: end of file reached + */ +struct bu_info { + union ubifs_key key; + struct ubifs_zbranch zbranch[UBIFS_MAX_BULK_READ]; + void *buf; + int buf_len; + int gc_seq; + int cnt; + int blk_cnt; + int eof; +}; + +/** + * struct ubifs_node_range - node length range description data structure. + * @len: fixed node length + * @min_len: minimum possible node length + * @max_len: maximum possible node length + * + * If @max_len is %0, the node has fixed length @len. + */ +struct ubifs_node_range { + union { + int len; + int min_len; + }; + int max_len; +}; + +/** + * struct ubifs_compressor - UBIFS compressor description structure. + * @compr_type: compressor type (%UBIFS_COMPR_LZO, etc) + * @cc: cryptoapi compressor handle + * @name: compressor name + * @capi_name: cryptoapi compressor name + */ +struct ubifs_compressor { + int compr_type; + struct crypto_acomp *cc; + const char *name; + const char *capi_name; +}; + +/** + * struct ubifs_budget_req - budget requirements of an operation. + * + * @fast: non-zero if the budgeting should try to acquire budget quickly and + * should not try to call write-back + * @recalculate: non-zero if @idx_growth, @data_growth, and @dd_growth fields + * have to be re-calculated + * @new_page: non-zero if the operation adds a new page + * @dirtied_page: non-zero if the operation makes a page dirty + * @new_dent: non-zero if the operation adds a new directory entry + * @mod_dent: non-zero if the operation removes or modifies an existing + * directory entry + * @new_ino: non-zero if the operation adds a new inode + * @new_ino_d: how much data newly created inode contains + * @dirtied_ino: how many inodes the operation makes dirty + * @dirtied_ino_d: how much data dirtied inode contains + * @idx_growth: how much the index will supposedly grow + * @data_growth: how much new data the operation will supposedly add + * @dd_growth: how much data that makes other data dirty the operation will + * supposedly add + * + * @idx_growth, @data_growth and @dd_growth are not used in budget request. The + * budgeting subsystem caches index and data growth values there to avoid + * re-calculating them when the budget is released. However, if @idx_growth is + * %-1, it is calculated by the release function using other fields. + * + * An inode may contain 4KiB of data at max., thus the widths of @new_ino_d + * is 13 bits, and @dirtied_ino_d - 15, because up to 4 inodes may be made + * dirty by the re-name operation. + * + * Note, UBIFS aligns node lengths to 8-bytes boundary, so the requester has to + * make sure the amount of inode data which contribute to @new_ino_d and + * @dirtied_ino_d fields are aligned. + */ +struct ubifs_budget_req { + unsigned int fast:1; + unsigned int recalculate:1; +#ifndef UBIFS_DEBUG + unsigned int new_page:1; + unsigned int dirtied_page:1; + unsigned int new_dent:1; + unsigned int mod_dent:1; + unsigned int new_ino:1; + unsigned int new_ino_d:13; + unsigned int dirtied_ino:4; + unsigned int dirtied_ino_d:15; +#else + /* Not bit-fields to check for overflows */ + unsigned int new_page; + unsigned int dirtied_page; + unsigned int new_dent; + unsigned int mod_dent; + unsigned int new_ino; + unsigned int new_ino_d; + unsigned int dirtied_ino; + unsigned int dirtied_ino_d; +#endif + int idx_growth; + int data_growth; + int dd_growth; +}; + +/** + * struct ubifs_orphan - stores the inode number of an orphan. + * @rb: rb-tree node of rb-tree of orphans sorted by inode number + * @list: list head of list of orphans in order added + * @new_list: list head of list of orphans added since the last commit + * @cnext: next orphan to commit + * @dnext: next orphan to delete + * @inum: inode number + * @new: %1 => added since the last commit, otherwise %0 + * @cmt: %1 => commit pending, otherwise %0 + * @del: %1 => delete pending, otherwise %0 + */ +struct ubifs_orphan { + struct rb_node rb; + struct list_head list; + struct list_head new_list; + struct ubifs_orphan *cnext; + struct ubifs_orphan *dnext; + ino_t inum; + unsigned new:1; + unsigned cmt:1; + unsigned del:1; +}; + +/** + * struct ubifs_mount_opts - UBIFS-specific mount options information. + * @unmount_mode: selected unmount mode (%0 default, %1 normal, %2 fast) + * @bulk_read: enable/disable bulk-reads (%0 default, %1 disable, %2 enable) + * @chk_data_crc: enable/disable CRC data checking when reading data nodes + * (%0 default, %1 disable, %2 enable) + * @override_compr: override default compressor (%0 - do not override and use + * superblock compressor, %1 - override and use compressor + * specified in @compr_type) + * @compr_type: compressor type to override the superblock compressor with + * (%UBIFS_COMPR_NONE, etc) + */ +struct ubifs_mount_opts { + unsigned int unmount_mode:2; + unsigned int bulk_read:2; + unsigned int chk_data_crc:2; + unsigned int override_compr:1; + unsigned int compr_type:2; +}; + +/** + * struct ubifs_budg_info - UBIFS budgeting information. + * @idx_growth: amount of bytes budgeted for index growth + * @data_growth: amount of bytes budgeted for cached data + * @dd_growth: amount of bytes budgeted for cached data that will make + * other data dirty + * @uncommitted_idx: amount of bytes were budgeted for growth of the index, but + * which still have to be taken into account because the index + * has not been committed so far + * @old_idx_sz: size of index on flash + * @min_idx_lebs: minimum number of LEBs required for the index + * @nospace: non-zero if the file-system does not have flash space (used as + * optimization) + * @nospace_rp: the same as @nospace, but additionally means that even reserved + * pool is full + * @page_budget: budget for a page (constant, never changed after mount) + * @inode_budget: budget for an inode (constant, never changed after mount) + * @dent_budget: budget for a directory entry (constant, never changed after + * mount) + */ +struct ubifs_budg_info { + long long idx_growth; + long long data_growth; + long long dd_growth; + long long uncommitted_idx; + unsigned long long old_idx_sz; + int min_idx_lebs; + unsigned int nospace:1; + unsigned int nospace_rp:1; + int page_budget; + int inode_budget; + int dent_budget; +}; + +/** + * ubifs_stats_info - per-FS statistics information. + * @magic_errors: number of bad magic numbers (will be reset with a new mount). + * @node_errors: number of bad nodes (will be reset with a new mount). + * @crc_errors: number of bad crcs (will be reset with a new mount). + */ +struct ubifs_stats_info { + unsigned int magic_errors; + unsigned int node_errors; + unsigned int crc_errors; +}; + +struct ubifs_debug_info; + +/** + * struct ubifs_info - UBIFS file-system description data structure + * (per-superblock). + * @vfs_sb: VFS @struct super_block object + * @sup_node: The super block node as read from the device + * + * @highest_inum: highest used inode number + * @max_sqnum: current global sequence number + * @cmt_no: commit number of the last successfully completed commit, protected + * by @commit_sem + * @cnt_lock: protects @highest_inum and @max_sqnum counters + * @fmt_version: UBIFS on-flash format version + * @ro_compat_version: R/O compatibility version + * @uuid: UUID from super block + * + * @lhead_lnum: log head logical eraseblock number + * @lhead_offs: log head offset + * @ltail_lnum: log tail logical eraseblock number (offset is always 0) + * @log_mutex: protects the log, @lhead_lnum, @lhead_offs, @ltail_lnum, and + * @bud_bytes + * @min_log_bytes: minimum required number of bytes in the log + * @cmt_bud_bytes: used during commit to temporarily amount of bytes in + * committed buds + * + * @buds: tree of all buds indexed by bud LEB number + * @bud_bytes: how many bytes of flash is used by buds + * @buds_lock: protects the @buds tree, @bud_bytes, and per-journal head bud + * lists + * @jhead_cnt: count of journal heads + * @jheads: journal heads (head zero is base head) + * @max_bud_bytes: maximum number of bytes allowed in buds + * @bg_bud_bytes: number of bud bytes when background commit is initiated + * @old_buds: buds to be released after commit ends + * @max_bud_cnt: maximum number of buds + * @need_wait_space: Non %0 means space reservation tasks need to wait in queue + * @reserve_space_wq: wait queue to sleep on if @need_wait_space is not %0 + * + * @commit_sem: synchronizes committer with other processes + * @cmt_state: commit state + * @cs_lock: commit state lock + * @cmt_wq: wait queue to sleep on if the log is full and a commit is running + * + * @big_lpt: flag that LPT is too big to write whole during commit + * @space_fixup: flag indicating that free space in LEBs needs to be cleaned up + * @double_hash: flag indicating that we can do lookups by hash + * @encrypted: flag indicating that this file system contains encrypted files + * @no_chk_data_crc: do not check CRCs when reading data nodes (except during + * recovery) + * @bulk_read: enable bulk-reads + * @default_compr: default compression algorithm (%UBIFS_COMPR_LZO, etc) + * @rw_incompat: the media is not R/W compatible + * @assert_action: action to take when a ubifs_assert() fails + * @authenticated: flag indigating the FS is mounted in authenticated mode + * + * @tnc_mutex: protects the Tree Node Cache (TNC), @zroot, @cnext, @enext, and + * @calc_idx_sz + * @zroot: zbranch which points to the root index node and znode + * @cnext: next znode to commit + * @enext: next znode to commit to empty space + * @gap_lebs: array of LEBs used by the in-gaps commit method + * @cbuf: commit buffer + * @ileb_buf: buffer for commit in-the-gaps method + * @ileb_len: length of data in ileb_buf + * @ihead_lnum: LEB number of index head + * @ihead_offs: offset of index head + * @ilebs: pre-allocated index LEBs + * @ileb_cnt: number of pre-allocated index LEBs + * @ileb_nxt: next pre-allocated index LEBs + * @old_idx: tree of index nodes obsoleted since the last commit start + * @bottom_up_buf: a buffer which is used by 'dirty_cow_bottom_up()' in tnc.c + * + * @mst_node: master node + * @mst_offs: offset of valid master node + * + * @max_bu_buf_len: maximum bulk-read buffer length + * @bu_mutex: protects the pre-allocated bulk-read buffer and @c->bu + * @bu: pre-allocated bulk-read information + * + * @write_reserve_mutex: protects @write_reserve_buf + * @write_reserve_buf: on the write path we allocate memory, which might + * sometimes be unavailable, in which case we use this + * write reserve buffer + * + * @log_lebs: number of logical eraseblocks in the log + * @log_bytes: log size in bytes + * @log_last: last LEB of the log + * @lpt_lebs: number of LEBs used for lprops table + * @lpt_first: first LEB of the lprops table area + * @lpt_last: last LEB of the lprops table area + * @orph_lebs: number of LEBs used for the orphan area + * @orph_first: first LEB of the orphan area + * @orph_last: last LEB of the orphan area + * @main_lebs: count of LEBs in the main area + * @main_first: first LEB of the main area + * @main_bytes: main area size in bytes + * + * @key_hash_type: type of the key hash + * @key_hash: direntry key hash function + * @key_fmt: key format + * @key_len: key length + * @hash_len: The length of the index node hashes + * @fanout: fanout of the index tree (number of links per indexing node) + * + * @min_io_size: minimal input/output unit size + * @min_io_shift: number of bits in @min_io_size minus one + * @max_write_size: maximum amount of bytes the underlying flash can write at a + * time (MTD write buffer size) + * @max_write_shift: number of bits in @max_write_size minus one + * @leb_size: logical eraseblock size in bytes + * @leb_start: starting offset of logical eraseblocks within physical + * eraseblocks + * @half_leb_size: half LEB size + * @idx_leb_size: how many bytes of an LEB are effectively available when it is + * used to store indexing nodes (@leb_size - @max_idx_node_sz) + * @leb_cnt: count of logical eraseblocks + * @max_leb_cnt: maximum count of logical eraseblocks + * @ro_media: the underlying UBI volume is read-only + * @ro_mount: the file-system was mounted as read-only + * @ro_error: UBIFS switched to R/O mode because an error happened + * + * @dirty_pg_cnt: number of dirty pages (not used) + * @dirty_zn_cnt: number of dirty znodes + * @clean_zn_cnt: number of clean znodes + * + * @space_lock: protects @bi and @lst + * @lst: lprops statistics + * @bi: budgeting information + * @calc_idx_sz: temporary variable which is used to calculate new index size + * (contains accurate new index size at end of TNC commit start) + * + * @ref_node_alsz: size of the LEB reference node aligned to the min. flash + * I/O unit + * @mst_node_alsz: master node aligned size + * @min_idx_node_sz: minimum indexing node aligned on 8-bytes boundary + * @max_idx_node_sz: maximum indexing node aligned on 8-bytes boundary + * @max_inode_sz: maximum possible inode size in bytes + * @max_znode_sz: size of znode in bytes + * + * @leb_overhead: how many bytes are wasted in an LEB when it is filled with + * data nodes of maximum size - used in free space reporting + * @dead_wm: LEB dead space watermark + * @dark_wm: LEB dark space watermark + * @block_cnt: count of 4KiB blocks on the FS + * + * @ranges: UBIFS node length ranges + * @ubi: UBI volume descriptor + * @di: UBI device information + * @vi: UBI volume information + * + * @orph_tree: rb-tree of orphan inode numbers + * @orph_list: list of orphan inode numbers in order added + * @orph_new: list of orphan inode numbers added since last commit + * @orph_cnext: next orphan to commit + * @orph_dnext: next orphan to delete + * @orphan_lock: lock for orph_tree and orph_new + * @orph_buf: buffer for orphan nodes + * @new_orphans: number of orphans since last commit + * @cmt_orphans: number of orphans being committed + * @tot_orphans: number of orphans in the rb_tree + * @max_orphans: maximum number of orphans allowed + * @ohead_lnum: orphan head LEB number + * @ohead_offs: orphan head offset + * @no_orphs: non-zero if there are no orphans + * + * @bgt: UBIFS background thread + * @bgt_name: background thread name + * @need_bgt: if background thread should run + * @need_wbuf_sync: if write-buffers have to be synchronized + * + * @gc_lnum: LEB number used for garbage collection + * @sbuf: a buffer of LEB size used by GC and replay for scanning + * @idx_gc: list of index LEBs that have been garbage collected + * @idx_gc_cnt: number of elements on the idx_gc list + * @gc_seq: incremented for every non-index LEB garbage collected + * @gced_lnum: last non-index LEB that was garbage collected + * + * @infos_list: links all 'ubifs_info' objects + * @umount_mutex: serializes shrinker and un-mount + * @shrinker_run_no: shrinker run number + * + * @space_bits: number of bits needed to record free or dirty space + * @lpt_lnum_bits: number of bits needed to record a LEB number in the LPT + * @lpt_offs_bits: number of bits needed to record an offset in the LPT + * @lpt_spc_bits: number of bits needed to space in the LPT + * @pcnt_bits: number of bits needed to record pnode or nnode number + * @lnum_bits: number of bits needed to record LEB number + * @nnode_sz: size of on-flash nnode + * @pnode_sz: size of on-flash pnode + * @ltab_sz: size of on-flash LPT lprops table + * @lsave_sz: size of on-flash LPT save table + * @pnode_cnt: number of pnodes + * @nnode_cnt: number of nnodes + * @lpt_hght: height of the LPT + * @pnodes_have: number of pnodes in memory + * + * @lp_mutex: protects lprops table and all the other lprops-related fields + * @lpt_lnum: LEB number of the root nnode of the LPT + * @lpt_offs: offset of the root nnode of the LPT + * @nhead_lnum: LEB number of LPT head + * @nhead_offs: offset of LPT head + * @lpt_drty_flgs: dirty flags for LPT special nodes e.g. ltab + * @dirty_nn_cnt: number of dirty nnodes + * @dirty_pn_cnt: number of dirty pnodes + * @check_lpt_free: flag that indicates LPT GC may be needed + * @lpt_sz: LPT size + * @lpt_nod_buf: buffer for an on-flash nnode or pnode + * @lpt_buf: buffer of LEB size used by LPT + * @nroot: address in memory of the root nnode of the LPT + * @lpt_cnext: next LPT node to commit + * @lpt_heap: array of heaps of categorized lprops + * @dirty_idx: a (reverse sorted) copy of the LPROPS_DIRTY_IDX heap as at + * previous commit start + * @uncat_list: list of un-categorized LEBs + * @empty_list: list of empty LEBs + * @freeable_list: list of freeable non-index LEBs (free + dirty == @leb_size) + * @frdi_idx_list: list of freeable index LEBs (free + dirty == @leb_size) + * @freeable_cnt: number of freeable LEBs in @freeable_list + * @in_a_category_cnt: count of lprops which are in a certain category, which + * basically meants that they were loaded from the flash + * + * @ltab_lnum: LEB number of LPT's own lprops table + * @ltab_offs: offset of LPT's own lprops table + * @ltab: LPT's own lprops table + * @ltab_cmt: LPT's own lprops table (commit copy) + * @lsave_cnt: number of LEB numbers in LPT's save table + * @lsave_lnum: LEB number of LPT's save table + * @lsave_offs: offset of LPT's save table + * @lsave: LPT's save table + * @lscan_lnum: LEB number of last LPT scan + * + * @rp_size: size of the reserved pool in bytes + * @report_rp_size: size of the reserved pool reported to user-space + * @rp_uid: reserved pool user ID + * @rp_gid: reserved pool group ID + * + * @hash_tfm: the hash transformation used for hashing nodes + * @hmac_tfm: the HMAC transformation for this filesystem + * @hmac_desc_len: length of the HMAC used for authentication + * @auth_key_name: the authentication key name + * @auth_hash_name: the name of the hash algorithm used for authentication + * @auth_hash_algo: the authentication hash used for this fs + * @log_hash: the log hash from the commit start node up to the latest reference + * node. + * + * @empty: %1 if the UBI device is empty + * @need_recovery: %1 if the file-system needs recovery + * @replaying: %1 during journal replay + * @mounting: %1 while mounting + * @probing: %1 while attempting to mount if SB_SILENT mount flag is set + * @remounting_rw: %1 while re-mounting from R/O mode to R/W mode + * @replay_list: temporary list used during journal replay + * @replay_buds: list of buds to replay + * @cs_sqnum: sequence number of first node in the log (commit start node) + * @unclean_leb_list: LEBs to recover when re-mounting R/O mounted FS to R/W + * mode + * @rcvrd_mst_node: recovered master node to write when re-mounting R/O mounted + * FS to R/W mode + * @size_tree: inode size information for recovery + * @mount_opts: UBIFS-specific mount options + * + * @dbg: debugging-related information + * @stats: statistics exported over sysfs + * + * @kobj: kobject for /sys/fs/ubifs/ + * @kobj_unregister: completion to unregister sysfs kobject + */ +struct ubifs_info { + struct super_block *vfs_sb; + struct ubifs_sb_node *sup_node; + + ino_t highest_inum; + unsigned long long max_sqnum; + unsigned long long cmt_no; + spinlock_t cnt_lock; + int fmt_version; + int ro_compat_version; + unsigned char uuid[16]; + + int lhead_lnum; + int lhead_offs; + int ltail_lnum; + struct mutex log_mutex; + int min_log_bytes; + long long cmt_bud_bytes; + + struct rb_root buds; + long long bud_bytes; + spinlock_t buds_lock; + int jhead_cnt; + struct ubifs_jhead *jheads; + long long max_bud_bytes; + long long bg_bud_bytes; + struct list_head old_buds; + int max_bud_cnt; + atomic_t need_wait_space; + wait_queue_head_t reserve_space_wq; + + struct rw_semaphore commit_sem; + int cmt_state; + spinlock_t cs_lock; + wait_queue_head_t cmt_wq; + + struct kobject kobj; + struct completion kobj_unregister; + + unsigned int big_lpt:1; + unsigned int space_fixup:1; + unsigned int double_hash:1; + unsigned int encrypted:1; + unsigned int no_chk_data_crc:1; + unsigned int bulk_read:1; + unsigned int default_compr:2; + unsigned int rw_incompat:1; + unsigned int assert_action:2; + unsigned int authenticated:1; + unsigned int superblock_need_write:1; + + struct mutex tnc_mutex; + struct ubifs_zbranch zroot; + struct ubifs_znode *cnext; + struct ubifs_znode *enext; + int *gap_lebs; + void *cbuf; + void *ileb_buf; + int ileb_len; + int ihead_lnum; + int ihead_offs; + int *ilebs; + int ileb_cnt; + int ileb_nxt; + struct rb_root old_idx; + int *bottom_up_buf; + + struct ubifs_mst_node *mst_node; + int mst_offs; + + int max_bu_buf_len; + struct mutex bu_mutex; + struct bu_info bu; + + struct mutex write_reserve_mutex; + void *write_reserve_buf; + + int log_lebs; + long long log_bytes; + int log_last; + int lpt_lebs; + int lpt_first; + int lpt_last; + int orph_lebs; + int orph_first; + int orph_last; + int main_lebs; + int main_first; + long long main_bytes; + + uint8_t key_hash_type; + uint32_t (*key_hash)(const char *str, int len); + int key_fmt; + int key_len; + int hash_len; + int fanout; + + int min_io_size; + int min_io_shift; + int max_write_size; + int max_write_shift; + int leb_size; + int leb_start; + int half_leb_size; + int idx_leb_size; + int leb_cnt; + int max_leb_cnt; + unsigned int ro_media:1; + unsigned int ro_mount:1; + unsigned int ro_error:1; + + atomic_long_t dirty_pg_cnt; + atomic_long_t dirty_zn_cnt; + atomic_long_t clean_zn_cnt; + + spinlock_t space_lock; + struct ubifs_lp_stats lst; + struct ubifs_budg_info bi; + unsigned long long calc_idx_sz; + + int ref_node_alsz; + int mst_node_alsz; + int min_idx_node_sz; + int max_idx_node_sz; + long long max_inode_sz; + int max_znode_sz; + + int leb_overhead; + int dead_wm; + int dark_wm; + int block_cnt; + + struct ubifs_node_range ranges[UBIFS_NODE_TYPES_CNT]; + struct ubi_volume_desc *ubi; + struct ubi_device_info di; + struct ubi_volume_info vi; + + struct rb_root orph_tree; + struct list_head orph_list; + struct list_head orph_new; + struct ubifs_orphan *orph_cnext; + struct ubifs_orphan *orph_dnext; + spinlock_t orphan_lock; + void *orph_buf; + int new_orphans; + int cmt_orphans; + int tot_orphans; + int max_orphans; + int ohead_lnum; + int ohead_offs; + int no_orphs; + + struct task_struct *bgt; + char bgt_name[sizeof(BGT_NAME_PATTERN) + 9]; + int need_bgt; + int need_wbuf_sync; + + int gc_lnum; + void *sbuf; + struct list_head idx_gc; + int idx_gc_cnt; + int gc_seq; + int gced_lnum; + + struct list_head infos_list; + struct mutex umount_mutex; + unsigned int shrinker_run_no; + + int space_bits; + int lpt_lnum_bits; + int lpt_offs_bits; + int lpt_spc_bits; + int pcnt_bits; + int lnum_bits; + int nnode_sz; + int pnode_sz; + int ltab_sz; + int lsave_sz; + int pnode_cnt; + int nnode_cnt; + int lpt_hght; + int pnodes_have; + + struct mutex lp_mutex; + int lpt_lnum; + int lpt_offs; + int nhead_lnum; + int nhead_offs; + int lpt_drty_flgs; + int dirty_nn_cnt; + int dirty_pn_cnt; + int check_lpt_free; + long long lpt_sz; + void *lpt_nod_buf; + void *lpt_buf; + struct ubifs_nnode *nroot; + struct ubifs_cnode *lpt_cnext; + struct ubifs_lpt_heap lpt_heap[LPROPS_HEAP_CNT]; + struct ubifs_lpt_heap dirty_idx; + struct list_head uncat_list; + struct list_head empty_list; + struct list_head freeable_list; + struct list_head frdi_idx_list; + int freeable_cnt; + int in_a_category_cnt; + + int ltab_lnum; + int ltab_offs; + struct ubifs_lpt_lprops *ltab; + struct ubifs_lpt_lprops *ltab_cmt; + int lsave_cnt; + int lsave_lnum; + int lsave_offs; + int *lsave; + int lscan_lnum; + + long long rp_size; + long long report_rp_size; + kuid_t rp_uid; + kgid_t rp_gid; + + struct crypto_shash *hash_tfm; + struct crypto_shash *hmac_tfm; + int hmac_desc_len; + char *auth_key_name; + char *auth_hash_name; + enum hash_algo auth_hash_algo; + + struct shash_desc *log_hash; + + /* The below fields are used only during mounting and re-mounting */ + unsigned int empty:1; + unsigned int need_recovery:1; + unsigned int replaying:1; + unsigned int mounting:1; + unsigned int remounting_rw:1; + unsigned int probing:1; + struct list_head replay_list; + struct list_head replay_buds; + unsigned long long cs_sqnum; + struct list_head unclean_leb_list; + struct ubifs_mst_node *rcvrd_mst_node; + struct rb_root size_tree; + struct ubifs_mount_opts mount_opts; + + struct ubifs_debug_info *dbg; + struct ubifs_stats_info *stats; +}; + +extern struct list_head ubifs_infos; +extern spinlock_t ubifs_infos_lock; +extern atomic_long_t ubifs_clean_zn_cnt; +extern const struct super_operations ubifs_super_operations; +extern const struct address_space_operations ubifs_file_address_operations; +extern const struct file_operations ubifs_file_operations; +extern const struct inode_operations ubifs_file_inode_operations; +extern const struct file_operations ubifs_dir_operations; +extern const struct inode_operations ubifs_dir_inode_operations; +extern const struct inode_operations ubifs_symlink_inode_operations; +extern struct ubifs_compressor *ubifs_compressors[UBIFS_COMPR_TYPES_CNT]; +extern int ubifs_default_version; + +/* auth.c */ +static inline int ubifs_authenticated(const struct ubifs_info *c) +{ + return (IS_ENABLED(CONFIG_UBIFS_FS_AUTHENTICATION)) && c->authenticated; +} + +struct shash_desc *__ubifs_hash_get_desc(const struct ubifs_info *c); +static inline struct shash_desc *ubifs_hash_get_desc(const struct ubifs_info *c) +{ + return ubifs_authenticated(c) ? __ubifs_hash_get_desc(c) : NULL; +} + +static inline int ubifs_shash_init(const struct ubifs_info *c, + struct shash_desc *desc) +{ + if (ubifs_authenticated(c)) + return crypto_shash_init(desc); + else + return 0; +} + +static inline int ubifs_shash_update(const struct ubifs_info *c, + struct shash_desc *desc, const void *buf, + unsigned int len) +{ + int err = 0; + + if (ubifs_authenticated(c)) { + err = crypto_shash_update(desc, buf, len); + if (err < 0) + return err; + } + + return 0; +} + +static inline int ubifs_shash_final(const struct ubifs_info *c, + struct shash_desc *desc, u8 *out) +{ + return ubifs_authenticated(c) ? crypto_shash_final(desc, out) : 0; +} + +int __ubifs_node_calc_hash(const struct ubifs_info *c, const void *buf, + u8 *hash); +static inline int ubifs_node_calc_hash(const struct ubifs_info *c, + const void *buf, u8 *hash) +{ + if (ubifs_authenticated(c)) + return __ubifs_node_calc_hash(c, buf, hash); + else + return 0; +} + +int ubifs_prepare_auth_node(struct ubifs_info *c, void *node, + struct shash_desc *inhash); + +/** + * ubifs_check_hash - compare two hashes + * @c: UBIFS file-system description object + * @expected: first hash + * @got: second hash + * + * Compare two hashes @expected and @got. Returns 0 when they are equal, a + * negative error code otherwise. + */ +static inline int ubifs_check_hash(const struct ubifs_info *c, + const u8 *expected, const u8 *got) +{ + return crypto_memneq(expected, got, c->hash_len); +} + +/** + * ubifs_check_hmac - compare two HMACs + * @c: UBIFS file-system description object + * @expected: first HMAC + * @got: second HMAC + * + * Compare two hashes @expected and @got. Returns 0 when they are equal, a + * negative error code otherwise. + */ +static inline int ubifs_check_hmac(const struct ubifs_info *c, + const u8 *expected, const u8 *got) +{ + return crypto_memneq(expected, got, c->hmac_desc_len); +} + +#ifdef CONFIG_UBIFS_FS_AUTHENTICATION +void ubifs_bad_hash(const struct ubifs_info *c, const void *node, + const u8 *hash, int lnum, int offs); +#else +static inline void ubifs_bad_hash(const struct ubifs_info *c, const void *node, + const u8 *hash, int lnum, int offs) {}; +#endif + +int __ubifs_node_check_hash(const struct ubifs_info *c, const void *buf, + const u8 *expected); +static inline int ubifs_node_check_hash(const struct ubifs_info *c, + const void *buf, const u8 *expected) +{ + if (ubifs_authenticated(c)) + return __ubifs_node_check_hash(c, buf, expected); + else + return 0; +} + +int ubifs_init_authentication(struct ubifs_info *c); +void __ubifs_exit_authentication(struct ubifs_info *c); +static inline void ubifs_exit_authentication(struct ubifs_info *c) +{ + if (ubifs_authenticated(c)) + __ubifs_exit_authentication(c); +} + +/** + * ubifs_branch_hash - returns a pointer to the hash of a branch + * @c: UBIFS file-system description object + * @br: branch to get the hash from + * + * This returns a pointer to the hash of a branch. Since the key already is a + * dynamically sized object we cannot use a struct member here. + */ +static inline u8 *ubifs_branch_hash(struct ubifs_info *c, + struct ubifs_branch *br) +{ + return (void *)br + sizeof(*br) + c->key_len; +} + +/** + * ubifs_copy_hash - copy a hash + * @c: UBIFS file-system description object + * @from: source hash + * @to: destination hash + * + * With authentication this copies a hash, otherwise does nothing. + */ +static inline void ubifs_copy_hash(const struct ubifs_info *c, const u8 *from, + u8 *to) +{ + if (ubifs_authenticated(c)) + memcpy(to, from, c->hash_len); +} + +int __ubifs_node_insert_hmac(const struct ubifs_info *c, void *buf, + int len, int ofs_hmac); +static inline int ubifs_node_insert_hmac(const struct ubifs_info *c, void *buf, + int len, int ofs_hmac) +{ + if (ubifs_authenticated(c)) + return __ubifs_node_insert_hmac(c, buf, len, ofs_hmac); + else + return 0; +} + +int __ubifs_node_verify_hmac(const struct ubifs_info *c, const void *buf, + int len, int ofs_hmac); +static inline int ubifs_node_verify_hmac(const struct ubifs_info *c, + const void *buf, int len, int ofs_hmac) +{ + if (ubifs_authenticated(c)) + return __ubifs_node_verify_hmac(c, buf, len, ofs_hmac); + else + return 0; +} + +/** + * ubifs_auth_node_sz - returns the size of an authentication node + * @c: UBIFS file-system description object + * + * This function returns the size of an authentication node which can + * be 0 for unauthenticated filesystems or the real size of an auth node + * authentication is enabled. + */ +static inline int ubifs_auth_node_sz(const struct ubifs_info *c) +{ + if (ubifs_authenticated(c)) + return sizeof(struct ubifs_auth_node) + c->hmac_desc_len; + else + return 0; +} +int ubifs_sb_verify_signature(struct ubifs_info *c, + const struct ubifs_sb_node *sup); +bool ubifs_hmac_zero(struct ubifs_info *c, const u8 *hmac); + +int ubifs_hmac_wkm(struct ubifs_info *c, u8 *hmac); + +int __ubifs_shash_copy_state(const struct ubifs_info *c, struct shash_desc *src, + struct shash_desc *target); +static inline int ubifs_shash_copy_state(const struct ubifs_info *c, + struct shash_desc *src, + struct shash_desc *target) +{ + if (ubifs_authenticated(c)) + return __ubifs_shash_copy_state(c, src, target); + else + return 0; +} + +/* io.c */ +void ubifs_ro_mode(struct ubifs_info *c, int err); +int ubifs_leb_read(const struct ubifs_info *c, int lnum, void *buf, int offs, + int len, int even_ebadmsg); +int ubifs_leb_write(struct ubifs_info *c, int lnum, const void *buf, int offs, + int len); +int ubifs_leb_change(struct ubifs_info *c, int lnum, const void *buf, int len); +int ubifs_leb_unmap(struct ubifs_info *c, int lnum); +int ubifs_leb_map(struct ubifs_info *c, int lnum); +int ubifs_is_mapped(const struct ubifs_info *c, int lnum); +int ubifs_wbuf_write_nolock(struct ubifs_wbuf *wbuf, void *buf, int len); +int ubifs_wbuf_seek_nolock(struct ubifs_wbuf *wbuf, int lnum, int offs); +int ubifs_wbuf_init(struct ubifs_info *c, struct ubifs_wbuf *wbuf); +int ubifs_read_node(const struct ubifs_info *c, void *buf, int type, int len, + int lnum, int offs); +int ubifs_read_node_wbuf(struct ubifs_wbuf *wbuf, void *buf, int type, int len, + int lnum, int offs); +int ubifs_write_node(struct ubifs_info *c, void *node, int len, int lnum, + int offs); +int ubifs_write_node_hmac(struct ubifs_info *c, void *buf, int len, int lnum, + int offs, int hmac_offs); +int ubifs_check_node(const struct ubifs_info *c, const void *buf, int len, + int lnum, int offs, int quiet, int must_chk_crc); +void ubifs_init_node(struct ubifs_info *c, void *buf, int len, int pad); +void ubifs_crc_node(void *buf, int len); +void ubifs_prepare_node(struct ubifs_info *c, void *buf, int len, int pad); +int ubifs_prepare_node_hmac(struct ubifs_info *c, void *node, int len, + int hmac_offs, int pad); +void ubifs_prep_grp_node(struct ubifs_info *c, void *node, int len, int last); +int ubifs_io_init(struct ubifs_info *c); +void ubifs_pad(const struct ubifs_info *c, void *buf, int pad); +int ubifs_wbuf_sync_nolock(struct ubifs_wbuf *wbuf); +int ubifs_bg_wbufs_sync(struct ubifs_info *c); +void ubifs_wbuf_add_ino_nolock(struct ubifs_wbuf *wbuf, ino_t inum); +int ubifs_sync_wbufs_by_inode(struct ubifs_info *c, struct inode *inode); + +/* scan.c */ +struct ubifs_scan_leb *ubifs_scan(const struct ubifs_info *c, int lnum, + int offs, void *sbuf, int quiet); +void ubifs_scan_destroy(struct ubifs_scan_leb *sleb); +int ubifs_scan_a_node(const struct ubifs_info *c, void *buf, int len, int lnum, + int offs, int quiet); +struct ubifs_scan_leb *ubifs_start_scan(const struct ubifs_info *c, int lnum, + int offs, void *sbuf); +void ubifs_end_scan(const struct ubifs_info *c, struct ubifs_scan_leb *sleb, + int lnum, int offs); +int ubifs_add_snod(const struct ubifs_info *c, struct ubifs_scan_leb *sleb, + void *buf, int offs); +void ubifs_scanned_corruption(const struct ubifs_info *c, int lnum, int offs, + void *buf); + +/* log.c */ +void ubifs_add_bud(struct ubifs_info *c, struct ubifs_bud *bud); +void ubifs_create_buds_lists(struct ubifs_info *c); +int ubifs_add_bud_to_log(struct ubifs_info *c, int jhead, int lnum, int offs); +struct ubifs_bud *ubifs_search_bud(struct ubifs_info *c, int lnum); +struct ubifs_wbuf *ubifs_get_wbuf(struct ubifs_info *c, int lnum); +int ubifs_log_start_commit(struct ubifs_info *c, int *ltail_lnum); +int ubifs_log_end_commit(struct ubifs_info *c, int new_ltail_lnum); +int ubifs_log_post_commit(struct ubifs_info *c, int old_ltail_lnum); +int ubifs_consolidate_log(struct ubifs_info *c); + +/* journal.c */ +int ubifs_jnl_update(struct ubifs_info *c, const struct inode *dir, + const struct fscrypt_name *nm, const struct inode *inode, + int deletion, int xent, int in_orphan); +int ubifs_jnl_write_data(struct ubifs_info *c, const struct inode *inode, + const union ubifs_key *key, struct folio *folio, + size_t offset, int len); +int ubifs_jnl_write_inode(struct ubifs_info *c, const struct inode *inode); +int ubifs_jnl_delete_inode(struct ubifs_info *c, const struct inode *inode); +int ubifs_jnl_xrename(struct ubifs_info *c, const struct inode *fst_dir, + const struct inode *fst_inode, + const struct fscrypt_name *fst_nm, + const struct inode *snd_dir, + const struct inode *snd_inode, + const struct fscrypt_name *snd_nm, int sync); +int ubifs_jnl_rename(struct ubifs_info *c, const struct inode *old_dir, + const struct inode *old_inode, + const struct fscrypt_name *old_nm, + const struct inode *new_dir, + const struct inode *new_inode, + const struct fscrypt_name *new_nm, + const struct inode *whiteout, int sync, int delete_orphan); +int ubifs_jnl_truncate(struct ubifs_info *c, const struct inode *inode, + loff_t old_size, loff_t new_size); +int ubifs_jnl_delete_xattr(struct ubifs_info *c, const struct inode *host, + const struct inode *inode, const struct fscrypt_name *nm); +int ubifs_jnl_change_xattr(struct ubifs_info *c, const struct inode *inode1, + const struct inode *inode2); + +/* budget.c */ +int ubifs_budget_space(struct ubifs_info *c, struct ubifs_budget_req *req); +void ubifs_release_budget(struct ubifs_info *c, struct ubifs_budget_req *req); +void ubifs_release_dirty_inode_budget(struct ubifs_info *c, + struct ubifs_inode *ui); +int ubifs_budget_inode_op(struct ubifs_info *c, struct inode *inode, + struct ubifs_budget_req *req); +void ubifs_release_ino_dirty(struct ubifs_info *c, struct inode *inode, + struct ubifs_budget_req *req); +void ubifs_cancel_ino_op(struct ubifs_info *c, struct inode *inode, + struct ubifs_budget_req *req); +long long ubifs_get_free_space(struct ubifs_info *c); +long long ubifs_get_free_space_nolock(struct ubifs_info *c); +int ubifs_calc_min_idx_lebs(struct ubifs_info *c); +void ubifs_convert_page_budget(struct ubifs_info *c); +long long ubifs_reported_space(const struct ubifs_info *c, long long free); +long long ubifs_calc_available(const struct ubifs_info *c, int min_idx_lebs); + +/* find.c */ +int ubifs_find_free_space(struct ubifs_info *c, int min_space, int *offs, + int squeeze); +int ubifs_find_free_leb_for_idx(struct ubifs_info *c); +int ubifs_find_dirty_leb(struct ubifs_info *c, struct ubifs_lprops *ret_lp, + int min_space, int pick_free); +int ubifs_find_dirty_idx_leb(struct ubifs_info *c); +int ubifs_save_dirty_idx_lnums(struct ubifs_info *c); + +/* tnc.c */ +int ubifs_lookup_level0(struct ubifs_info *c, const union ubifs_key *key, + struct ubifs_znode **zn, int *n); +int ubifs_tnc_lookup_nm(struct ubifs_info *c, const union ubifs_key *key, + void *node, const struct fscrypt_name *nm); +int ubifs_tnc_lookup_dh(struct ubifs_info *c, const union ubifs_key *key, + void *node, uint32_t secondary_hash); +int ubifs_tnc_locate(struct ubifs_info *c, const union ubifs_key *key, + void *node, int *lnum, int *offs); +int ubifs_tnc_add(struct ubifs_info *c, const union ubifs_key *key, int lnum, + int offs, int len, const u8 *hash); +int ubifs_tnc_replace(struct ubifs_info *c, const union ubifs_key *key, + int old_lnum, int old_offs, int lnum, int offs, int len); +int ubifs_tnc_add_nm(struct ubifs_info *c, const union ubifs_key *key, + int lnum, int offs, int len, const u8 *hash, + const struct fscrypt_name *nm); +int ubifs_tnc_remove(struct ubifs_info *c, const union ubifs_key *key); +int ubifs_tnc_remove_nm(struct ubifs_info *c, const union ubifs_key *key, + const struct fscrypt_name *nm); +int ubifs_tnc_remove_dh(struct ubifs_info *c, const union ubifs_key *key, + uint32_t cookie); +int ubifs_tnc_remove_range(struct ubifs_info *c, union ubifs_key *from_key, + union ubifs_key *to_key); +int ubifs_tnc_remove_ino(struct ubifs_info *c, ino_t inum); +struct ubifs_dent_node *ubifs_tnc_next_ent(struct ubifs_info *c, + union ubifs_key *key, + const struct fscrypt_name *nm); +void ubifs_tnc_close(struct ubifs_info *c); +int ubifs_tnc_has_node(struct ubifs_info *c, union ubifs_key *key, int level, + int lnum, int offs, int is_idx); +int ubifs_dirty_idx_node(struct ubifs_info *c, union ubifs_key *key, int level, + int lnum, int offs); +/* Shared by tnc.c for tnc_commit.c */ +void destroy_old_idx(struct ubifs_info *c); +int is_idx_node_in_tnc(struct ubifs_info *c, union ubifs_key *key, int level, + int lnum, int offs); +int insert_old_idx_znode(struct ubifs_info *c, struct ubifs_znode *znode); +int ubifs_tnc_get_bu_keys(struct ubifs_info *c, struct bu_info *bu); +int ubifs_tnc_bulk_read(struct ubifs_info *c, struct bu_info *bu); + +/* tnc_misc.c */ +struct ubifs_znode *ubifs_tnc_levelorder_next(const struct ubifs_info *c, + struct ubifs_znode *zr, + struct ubifs_znode *znode); +int ubifs_search_zbranch(const struct ubifs_info *c, + const struct ubifs_znode *znode, + const union ubifs_key *key, int *n); +struct ubifs_znode *ubifs_tnc_postorder_first(struct ubifs_znode *znode); +struct ubifs_znode *ubifs_tnc_postorder_next(const struct ubifs_info *c, + struct ubifs_znode *znode); +long ubifs_destroy_tnc_subtree(const struct ubifs_info *c, + struct ubifs_znode *zr); +void ubifs_destroy_tnc_tree(struct ubifs_info *c); +struct ubifs_znode *ubifs_load_znode(struct ubifs_info *c, + struct ubifs_zbranch *zbr, + struct ubifs_znode *parent, int iip); +int ubifs_tnc_read_node(struct ubifs_info *c, struct ubifs_zbranch *zbr, + void *node); + +/* tnc_commit.c */ +int ubifs_tnc_start_commit(struct ubifs_info *c, struct ubifs_zbranch *zroot); +int ubifs_tnc_end_commit(struct ubifs_info *c); + +/* shrinker.c */ +unsigned long ubifs_shrink_scan(struct shrinker *shrink, + struct shrink_control *sc); +unsigned long ubifs_shrink_count(struct shrinker *shrink, + struct shrink_control *sc); + +/* commit.c */ +int ubifs_bg_thread(void *info); +void ubifs_commit_required(struct ubifs_info *c); +void ubifs_request_bg_commit(struct ubifs_info *c); +int ubifs_run_commit(struct ubifs_info *c); +void ubifs_recovery_commit(struct ubifs_info *c); +int ubifs_gc_should_commit(struct ubifs_info *c); +void ubifs_wait_for_commit(struct ubifs_info *c); + +/* master.c */ +int ubifs_compare_master_node(struct ubifs_info *c, void *m1, void *m2); +int ubifs_read_master(struct ubifs_info *c); +int ubifs_write_master(struct ubifs_info *c); + +/* sb.c */ +int ubifs_read_superblock(struct ubifs_info *c); +int ubifs_write_sb_node(struct ubifs_info *c, struct ubifs_sb_node *sup); +int ubifs_fixup_free_space(struct ubifs_info *c); +int ubifs_enable_encryption(struct ubifs_info *c); + +/* replay.c */ +int ubifs_validate_entry(struct ubifs_info *c, + const struct ubifs_dent_node *dent); +int ubifs_replay_journal(struct ubifs_info *c); + +/* gc.c */ +int ubifs_garbage_collect(struct ubifs_info *c, int anyway); +int ubifs_gc_start_commit(struct ubifs_info *c); +int ubifs_gc_end_commit(struct ubifs_info *c); +void ubifs_destroy_idx_gc(struct ubifs_info *c); +int ubifs_get_idx_gc_leb(struct ubifs_info *c); +int ubifs_garbage_collect_leb(struct ubifs_info *c, struct ubifs_lprops *lp); + +/* orphan.c */ +int ubifs_add_orphan(struct ubifs_info *c, ino_t inum); +void ubifs_delete_orphan(struct ubifs_info *c, ino_t inum); +int ubifs_orphan_start_commit(struct ubifs_info *c); +int ubifs_orphan_end_commit(struct ubifs_info *c); +int ubifs_mount_orphans(struct ubifs_info *c, int unclean, int read_only); +int ubifs_clear_orphans(struct ubifs_info *c); + +/* lpt.c */ +int ubifs_calc_lpt_geom(struct ubifs_info *c); +int ubifs_create_dflt_lpt(struct ubifs_info *c, int *main_lebs, int lpt_first, + int *lpt_lebs, int *big_lpt, u8 *hash); +int ubifs_lpt_init(struct ubifs_info *c, int rd, int wr); +struct ubifs_lprops *ubifs_lpt_lookup(struct ubifs_info *c, int lnum); +struct ubifs_lprops *ubifs_lpt_lookup_dirty(struct ubifs_info *c, int lnum); +int ubifs_lpt_scan_nolock(struct ubifs_info *c, int start_lnum, int end_lnum, + ubifs_lpt_scan_callback scan_cb, void *data); + +/* Shared by lpt.c for lpt_commit.c */ +void ubifs_pack_lsave(struct ubifs_info *c, void *buf, int *lsave); +void ubifs_pack_ltab(struct ubifs_info *c, void *buf, + struct ubifs_lpt_lprops *ltab); +void ubifs_pack_pnode(struct ubifs_info *c, void *buf, + struct ubifs_pnode *pnode); +void ubifs_pack_nnode(struct ubifs_info *c, void *buf, + struct ubifs_nnode *nnode); +struct ubifs_pnode *ubifs_get_pnode(struct ubifs_info *c, + struct ubifs_nnode *parent, int iip); +struct ubifs_nnode *ubifs_get_nnode(struct ubifs_info *c, + struct ubifs_nnode *parent, int iip); +struct ubifs_pnode *ubifs_pnode_lookup(struct ubifs_info *c, int i); +int ubifs_read_nnode(struct ubifs_info *c, struct ubifs_nnode *parent, int iip); +void ubifs_add_lpt_dirt(struct ubifs_info *c, int lnum, int dirty); +void ubifs_add_nnode_dirt(struct ubifs_info *c, struct ubifs_nnode *nnode); +uint32_t ubifs_unpack_bits(const struct ubifs_info *c, uint8_t **addr, int *pos, int nrbits); +struct ubifs_nnode *ubifs_first_nnode(struct ubifs_info *c, int *hght); +/* Needed only in debugging code in lpt_commit.c */ +int ubifs_unpack_nnode(const struct ubifs_info *c, void *buf, + struct ubifs_nnode *nnode); +int ubifs_lpt_calc_hash(struct ubifs_info *c, u8 *hash); + +/* lpt_commit.c */ +int ubifs_lpt_start_commit(struct ubifs_info *c); +int ubifs_lpt_end_commit(struct ubifs_info *c); +int ubifs_lpt_post_commit(struct ubifs_info *c); +void ubifs_lpt_free(struct ubifs_info *c, int wr_only); + +/* lprops.c */ +const struct ubifs_lprops *ubifs_change_lp(struct ubifs_info *c, + const struct ubifs_lprops *lp, + int free, int dirty, int flags, + int idx_gc_cnt); +void ubifs_get_lp_stats(struct ubifs_info *c, struct ubifs_lp_stats *lst); +void ubifs_add_to_cat(struct ubifs_info *c, struct ubifs_lprops *lprops, + int cat); +void ubifs_replace_cat(struct ubifs_info *c, struct ubifs_lprops *old_lprops, + struct ubifs_lprops *new_lprops); +void ubifs_ensure_cat(struct ubifs_info *c, struct ubifs_lprops *lprops); +int ubifs_categorize_lprops(const struct ubifs_info *c, + const struct ubifs_lprops *lprops); +int ubifs_change_one_lp(struct ubifs_info *c, int lnum, int free, int dirty, + int flags_set, int flags_clean, int idx_gc_cnt); +int ubifs_update_one_lp(struct ubifs_info *c, int lnum, int free, int dirty, + int flags_set, int flags_clean); +int ubifs_read_one_lp(struct ubifs_info *c, int lnum, struct ubifs_lprops *lp); +const struct ubifs_lprops *ubifs_fast_find_free(struct ubifs_info *c); +const struct ubifs_lprops *ubifs_fast_find_empty(struct ubifs_info *c); +const struct ubifs_lprops *ubifs_fast_find_freeable(struct ubifs_info *c); +const struct ubifs_lprops *ubifs_fast_find_frdi_idx(struct ubifs_info *c); +int ubifs_calc_dark(const struct ubifs_info *c, int spc); + +/* file.c */ +int ubifs_fsync(struct file *file, loff_t start, loff_t end, int datasync); +int ubifs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr); +int ubifs_update_time(struct inode *inode, enum fs_update_time type, + unsigned int flags); + +/* dir.c */ +struct inode *ubifs_new_inode(struct ubifs_info *c, struct inode *dir, + umode_t mode, bool is_xattr); +int ubifs_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *stat, u32 request_mask, unsigned int flags); +int ubifs_check_dir_empty(struct inode *dir); + +/* xattr.c */ +int ubifs_xattr_set(struct inode *host, const char *name, const void *value, + size_t size, int flags, bool check_lock); +ssize_t ubifs_xattr_get(struct inode *host, const char *name, void *buf, + size_t size); + +#ifdef CONFIG_UBIFS_FS_XATTR +extern const struct xattr_handler * const ubifs_xattr_handlers[]; +ssize_t ubifs_listxattr(struct dentry *dentry, char *buffer, size_t size); +int ubifs_purge_xattrs(struct inode *host); +#else +#define ubifs_listxattr NULL +#define ubifs_xattr_handlers NULL +static inline int ubifs_purge_xattrs(struct inode *host) +{ + return 0; +} +#endif + +#ifdef CONFIG_UBIFS_FS_SECURITY +extern int ubifs_init_security(struct inode *dentry, struct inode *inode, + const struct qstr *qstr); +#else +static inline int ubifs_init_security(struct inode *dentry, + struct inode *inode, const struct qstr *qstr) +{ + return 0; +} +#endif + + +/* super.c */ +struct inode *ubifs_iget(struct super_block *sb, unsigned long inum); + +/* recovery.c */ +int ubifs_recover_master_node(struct ubifs_info *c); +int ubifs_write_rcvrd_mst_node(struct ubifs_info *c); +struct ubifs_scan_leb *ubifs_recover_leb(struct ubifs_info *c, int lnum, + int offs, void *sbuf, int jhead); +struct ubifs_scan_leb *ubifs_recover_log_leb(struct ubifs_info *c, int lnum, + int offs, void *sbuf); +int ubifs_recover_inl_heads(struct ubifs_info *c, void *sbuf); +int ubifs_clean_lebs(struct ubifs_info *c, void *sbuf); +int ubifs_rcvry_gc_commit(struct ubifs_info *c); +int ubifs_recover_size_accum(struct ubifs_info *c, union ubifs_key *key, + int deletion, loff_t new_size); +int ubifs_recover_size(struct ubifs_info *c, bool in_place); +void ubifs_destroy_size_tree(struct ubifs_info *c); + +/* ioctl.c */ +int ubifs_fileattr_get(struct dentry *dentry, struct file_kattr *fa); +int ubifs_fileattr_set(struct mnt_idmap *idmap, + struct dentry *dentry, struct file_kattr *fa); +long ubifs_ioctl(struct file *file, unsigned int cmd, unsigned long arg); +void ubifs_set_inode_flags(struct inode *inode); +#ifdef CONFIG_COMPAT +long ubifs_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg); +#endif + +/* compressor.c */ +int __init ubifs_compressors_init(void); +void ubifs_compressors_exit(void); +void ubifs_compress(const struct ubifs_info *c, const void *in_buf, int in_len, + void *out_buf, int *out_len, int *compr_type); +void ubifs_compress_folio(const struct ubifs_info *c, struct folio *folio, + size_t offset, int in_len, void *out_buf, + int *out_len, int *compr_type); +int ubifs_decompress(const struct ubifs_info *c, const void *buf, int len, + void *out, int *out_len, int compr_type); +int ubifs_decompress_folio(const struct ubifs_info *c, const void *buf, + int len, struct folio *folio, size_t offset, + int *out_len, int compr_type); + +/* sysfs.c */ +int ubifs_sysfs_init(void); +void ubifs_sysfs_exit(void); +int ubifs_sysfs_register(struct ubifs_info *c); +void ubifs_sysfs_unregister(struct ubifs_info *c); + +#include "debug.h" +#include "misc.h" +#include "key.h" + +#ifndef CONFIG_FS_ENCRYPTION +static inline int ubifs_encrypt(const struct inode *inode, + struct ubifs_data_node *dn, + unsigned int in_len, unsigned int *out_len, + int block) +{ + struct ubifs_info *c = inode->i_sb->s_fs_info; + ubifs_assert(c, 0); + return -EOPNOTSUPP; +} +static inline int ubifs_decrypt(const struct inode *inode, + struct ubifs_data_node *dn, + unsigned int *out_len, int block) +{ + struct ubifs_info *c = inode->i_sb->s_fs_info; + ubifs_assert(c, 0); + return -EOPNOTSUPP; +} +#else +/* crypto.c */ +int ubifs_encrypt(const struct inode *inode, struct ubifs_data_node *dn, + unsigned int in_len, unsigned int *out_len, int block); +int ubifs_decrypt(const struct inode *inode, struct ubifs_data_node *dn, + unsigned int *out_len, int block); +#endif + +extern const struct fscrypt_operations ubifs_crypt_operations; + +/* Normal UBIFS messages */ +__printf(2, 3) +void ubifs_msg(const struct ubifs_info *c, const char *fmt, ...); +__printf(2, 3) +void ubifs_err(const struct ubifs_info *c, const char *fmt, ...); +__printf(2, 3) +void ubifs_warn(const struct ubifs_info *c, const char *fmt, ...); +/* + * A conditional variant of 'ubifs_err()' which doesn't output anything + * if probing (ie. SB_SILENT set). + */ +#define ubifs_errc(c, fmt, ...) \ +do { \ + if (!(c)->probing) \ + ubifs_err(c, fmt, ##__VA_ARGS__); \ +} while (0) + +#endif /* !__UBIFS_H__ */ diff --git a/fs/ubifs/xattr.c b/fs/ubifs/xattr.c new file mode 100644 index 000000000..b5a9ab9d8 --- /dev/null +++ b/fs/ubifs/xattr.c @@ -0,0 +1,706 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This file is part of UBIFS. + * + * Copyright (C) 2006-2008 Nokia Corporation. + * + * Authors: Artem Bityutskiy (Битюцкий Артём) + * Adrian Hunter + */ + +/* + * This file implements UBIFS extended attributes support. + * + * Extended attributes are implemented as regular inodes with attached data, + * which limits extended attribute size to UBIFS block size (4KiB). Names of + * extended attributes are described by extended attribute entries (xentries), + * which are almost identical to directory entries, but have different key type. + * + * In other words, the situation with extended attributes is very similar to + * directories. Indeed, any inode (but of course not xattr inodes) may have a + * number of associated xentries, just like directory inodes have associated + * directory entries. Extended attribute entries store the name of the extended + * attribute, the host inode number, and the extended attribute inode number. + * Similarly, direntries store the name, the parent and the target inode + * numbers. Thus, most of the common UBIFS mechanisms may be re-used for + * extended attributes. + * + * The number of extended attributes is not limited, but there is Linux + * limitation on the maximum possible size of the list of all extended + * attributes associated with an inode (%XATTR_LIST_MAX), so UBIFS makes sure + * the sum of all extended attribute names of the inode does not exceed that + * limit. + * + * Extended attributes are synchronous, which means they are written to the + * flash media synchronously and there is no write-back for extended attribute + * inodes. The extended attribute values are not stored in compressed form on + * the media. + * + * Since extended attributes are represented by regular inodes, they are cached + * in the VFS inode cache. The xentries are cached in the LNC cache (see + * tnc.c). + * + * ACL support is not implemented. + */ + +#include "ubifs.h" +#include +#include +#include + +static const struct inode_operations empty_iops; +static const struct file_operations empty_fops; + +/** + * create_xattr - create an extended attribute. + * @c: UBIFS file-system description object + * @host: host inode + * @nm: extended attribute name + * @value: extended attribute value + * @size: size of extended attribute value + * + * This is a helper function which creates an extended attribute of name @nm + * and value @value for inode @host. The host inode is also updated on flash + * because the ctime and extended attribute accounting data changes. This + * function returns zero in case of success and a negative error code in case + * of failure. + */ +static int create_xattr(struct ubifs_info *c, struct inode *host, + const struct fscrypt_name *nm, const void *value, int size) +{ + int err, names_len; + struct inode *inode; + struct ubifs_inode *ui, *host_ui = ubifs_inode(host); + struct ubifs_budget_req req = { .new_ino = 1, .new_dent = 1, + .new_ino_d = ALIGN(size, 8), .dirtied_ino = 1, + .dirtied_ino_d = ALIGN(host_ui->data_len, 8) }; + + if (host_ui->xattr_cnt >= ubifs_xattr_max_cnt(c)) { + ubifs_err(c, "inode %llu already has too many xattrs (%d), cannot create more", + host->i_ino, host_ui->xattr_cnt); + return -ENOSPC; + } + /* + * Linux limits the maximum size of the extended attribute names list + * to %XATTR_LIST_MAX. This means we should not allow creating more + * extended attributes if the name list becomes larger. This limitation + * is artificial for UBIFS, though. + */ + names_len = host_ui->xattr_names + host_ui->xattr_cnt + fname_len(nm) + 1; + if (names_len > XATTR_LIST_MAX) { + ubifs_err(c, "cannot add one more xattr name to inode %llu, total names length would become %d, max. is %d", + host->i_ino, names_len, XATTR_LIST_MAX); + return -ENOSPC; + } + + err = ubifs_budget_space(c, &req); + if (err) + return err; + + inode = ubifs_new_inode(c, host, S_IFREG | S_IRWXUGO, true); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + goto out_budg; + } + + /* Re-define all operations to be "nothing" */ + inode->i_mapping->a_ops = &empty_aops; + inode->i_op = &empty_iops; + inode->i_fop = &empty_fops; + + inode->i_flags |= S_SYNC | S_NOATIME | S_NOCMTIME; + ui = ubifs_inode(inode); + ui->xattr = 1; + ui->flags |= UBIFS_XATTR_FL; + ui->data = kmemdup(value, size, GFP_NOFS); + if (!ui->data) { + err = -ENOMEM; + goto out_free; + } + inode->i_size = ui->ui_size = size; + ui->data_len = size; + + mutex_lock(&host_ui->ui_mutex); + inode_set_ctime_current(host); + host_ui->xattr_cnt += 1; + host_ui->xattr_size += CALC_DENT_SIZE(fname_len(nm)); + host_ui->xattr_size += CALC_XATTR_BYTES(size); + host_ui->xattr_names += fname_len(nm); + + /* + * We handle UBIFS_XATTR_NAME_ENCRYPTION_CONTEXT here because we + * have to set the UBIFS_CRYPT_FL flag on the host inode. + * To avoid multiple updates of the same inode in the same operation, + * let's do it here. + */ + if (strcmp(fname_name(nm), UBIFS_XATTR_NAME_ENCRYPTION_CONTEXT) == 0) + host_ui->flags |= UBIFS_CRYPT_FL; + + err = ubifs_jnl_update(c, host, nm, inode, 0, 1, 0); + if (err) + goto out_cancel; + ubifs_set_inode_flags(host); + mutex_unlock(&host_ui->ui_mutex); + + ubifs_release_budget(c, &req); + insert_inode_hash(inode); + iput(inode); + return 0; + +out_cancel: + host_ui->xattr_cnt -= 1; + host_ui->xattr_size -= CALC_DENT_SIZE(fname_len(nm)); + host_ui->xattr_size -= CALC_XATTR_BYTES(size); + host_ui->xattr_names -= fname_len(nm); + host_ui->flags &= ~UBIFS_CRYPT_FL; + mutex_unlock(&host_ui->ui_mutex); +out_free: + make_bad_inode(inode); + iput(inode); +out_budg: + ubifs_release_budget(c, &req); + return err; +} + +/** + * change_xattr - change an extended attribute. + * @c: UBIFS file-system description object + * @host: host inode + * @inode: extended attribute inode + * @value: extended attribute value + * @size: size of extended attribute value + * + * This helper function changes the value of extended attribute @inode with new + * data from @value. Returns zero in case of success and a negative error code + * in case of failure. + */ +static int change_xattr(struct ubifs_info *c, struct inode *host, + struct inode *inode, const void *value, int size) +{ + int err; + struct ubifs_inode *host_ui = ubifs_inode(host); + struct ubifs_inode *ui = ubifs_inode(inode); + void *buf = NULL; + int old_size; + struct ubifs_budget_req req = { .dirtied_ino = 2, + .dirtied_ino_d = ALIGN(size, 8) + ALIGN(host_ui->data_len, 8) }; + + ubifs_assert(c, ui->data_len == inode->i_size); + err = ubifs_budget_space(c, &req); + if (err) + return err; + + buf = kmemdup(value, size, GFP_NOFS); + if (!buf) { + err = -ENOMEM; + goto out_free; + } + kfree(ui->data); + ui->data = buf; + inode->i_size = ui->ui_size = size; + old_size = ui->data_len; + ui->data_len = size; + + mutex_lock(&host_ui->ui_mutex); + inode_set_ctime_current(host); + host_ui->xattr_size -= CALC_XATTR_BYTES(old_size); + host_ui->xattr_size += CALC_XATTR_BYTES(size); + + /* + * It is important to write the host inode after the xattr inode + * because if the host inode gets synchronized (via 'fsync()'), then + * the extended attribute inode gets synchronized, because it goes + * before the host inode in the write-buffer. + */ + err = ubifs_jnl_change_xattr(c, inode, host); + if (err) + goto out_cancel; + mutex_unlock(&host_ui->ui_mutex); + + ubifs_release_budget(c, &req); + return 0; + +out_cancel: + host_ui->xattr_size -= CALC_XATTR_BYTES(size); + host_ui->xattr_size += CALC_XATTR_BYTES(old_size); + mutex_unlock(&host_ui->ui_mutex); + make_bad_inode(inode); +out_free: + ubifs_release_budget(c, &req); + return err; +} + +static struct inode *iget_xattr(struct ubifs_info *c, ino_t inum) +{ + struct inode *inode; + + inode = ubifs_iget(c->vfs_sb, inum); + if (IS_ERR(inode)) { + ubifs_err(c, "dead extended attribute entry, error %d", + (int)PTR_ERR(inode)); + return inode; + } + if (ubifs_inode(inode)->xattr) + return inode; + ubifs_err(c, "corrupt extended attribute entry"); + iput(inode); + return ERR_PTR(-EINVAL); +} + +int ubifs_xattr_set(struct inode *host, const char *name, const void *value, + size_t size, int flags, bool check_lock) +{ + struct inode *inode; + struct ubifs_info *c = host->i_sb->s_fs_info; + struct fscrypt_name nm = { .disk_name = FSTR_INIT((char *)name, strlen(name))}; + struct ubifs_dent_node *xent; + union ubifs_key key; + int err; + + if (check_lock) + ubifs_assert(c, inode_is_locked(host)); + + if (size > UBIFS_MAX_INO_DATA) + return -ERANGE; + + if (fname_len(&nm) > UBIFS_MAX_NLEN) + return -ENAMETOOLONG; + + xent = kmalloc(UBIFS_MAX_XENT_NODE_SZ, GFP_NOFS); + if (!xent) + return -ENOMEM; + + down_write(&ubifs_inode(host)->xattr_sem); + /* + * The extended attribute entries are stored in LNC, so multiple + * look-ups do not involve reading the flash. + */ + xent_key_init(c, &key, host->i_ino, &nm); + err = ubifs_tnc_lookup_nm(c, &key, xent, &nm); + if (err) { + if (err != -ENOENT) + goto out_free; + + if (flags & XATTR_REPLACE) + /* We are asked not to create the xattr */ + err = -ENODATA; + else + err = create_xattr(c, host, &nm, value, size); + goto out_free; + } + + if (flags & XATTR_CREATE) { + /* We are asked not to replace the xattr */ + err = -EEXIST; + goto out_free; + } + + inode = iget_xattr(c, le64_to_cpu(xent->inum)); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + goto out_free; + } + + err = change_xattr(c, host, inode, value, size); + iput(inode); + +out_free: + up_write(&ubifs_inode(host)->xattr_sem); + kfree(xent); + return err; +} + +ssize_t ubifs_xattr_get(struct inode *host, const char *name, void *buf, + size_t size) +{ + struct inode *inode; + struct ubifs_info *c = host->i_sb->s_fs_info; + struct fscrypt_name nm = { .disk_name = FSTR_INIT((char *)name, strlen(name))}; + struct ubifs_inode *ui; + struct ubifs_dent_node *xent; + union ubifs_key key; + int err; + + if (fname_len(&nm) > UBIFS_MAX_NLEN) + return -ENAMETOOLONG; + + xent = kmalloc(UBIFS_MAX_XENT_NODE_SZ, GFP_NOFS); + if (!xent) + return -ENOMEM; + + down_read(&ubifs_inode(host)->xattr_sem); + xent_key_init(c, &key, host->i_ino, &nm); + err = ubifs_tnc_lookup_nm(c, &key, xent, &nm); + if (err) { + if (err == -ENOENT) + err = -ENODATA; + goto out_cleanup; + } + + inode = iget_xattr(c, le64_to_cpu(xent->inum)); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + goto out_cleanup; + } + + ui = ubifs_inode(inode); + ubifs_assert(c, inode->i_size == ui->data_len); + ubifs_assert(c, ubifs_inode(host)->xattr_size > ui->data_len); + + if (buf) { + /* If @buf is %NULL we are supposed to return the length */ + if (ui->data_len > size) { + err = -ERANGE; + goto out_iput; + } + + memcpy(buf, ui->data, ui->data_len); + } + err = ui->data_len; + +out_iput: + iput(inode); +out_cleanup: + up_read(&ubifs_inode(host)->xattr_sem); + kfree(xent); + return err; +} + +static bool xattr_visible(const char *name) +{ + /* File encryption related xattrs are for internal use only */ + if (strcmp(name, UBIFS_XATTR_NAME_ENCRYPTION_CONTEXT) == 0) + return false; + + /* Show trusted namespace only for "power" users */ + if (strncmp(name, XATTR_TRUSTED_PREFIX, + XATTR_TRUSTED_PREFIX_LEN) == 0 && !capable(CAP_SYS_ADMIN)) + return false; + + return true; +} + +ssize_t ubifs_listxattr(struct dentry *dentry, char *buffer, size_t size) +{ + union ubifs_key key; + struct inode *host = d_inode(dentry); + struct ubifs_info *c = host->i_sb->s_fs_info; + struct ubifs_inode *host_ui = ubifs_inode(host); + struct ubifs_dent_node *xent, *pxent = NULL; + int err, len, written = 0; + struct fscrypt_name nm = {0}; + + dbg_gen("ino %llu ('%pd'), buffer size %zd", host->i_ino, + dentry, size); + + down_read(&host_ui->xattr_sem); + len = host_ui->xattr_names + host_ui->xattr_cnt; + if (!buffer) { + /* + * We should return the minimum buffer size which will fit a + * null-terminated list of all the extended attribute names. + */ + err = len; + goto out_err; + } + + if (len > size) { + err = -ERANGE; + goto out_err; + } + + lowest_xent_key(c, &key, host->i_ino); + while (1) { + xent = ubifs_tnc_next_ent(c, &key, &nm); + if (IS_ERR(xent)) { + err = PTR_ERR(xent); + break; + } + + fname_name(&nm) = xent->name; + fname_len(&nm) = le16_to_cpu(xent->nlen); + + if (xattr_visible(xent->name)) { + memcpy(buffer + written, fname_name(&nm), fname_len(&nm) + 1); + written += fname_len(&nm) + 1; + } + + kfree(pxent); + pxent = xent; + key_read(c, &xent->key, &key); + } + kfree(pxent); + up_read(&host_ui->xattr_sem); + + if (err != -ENOENT) { + ubifs_err(c, "cannot find next direntry, error %d", err); + return err; + } + + ubifs_assert(c, written <= size); + return written; + +out_err: + up_read(&host_ui->xattr_sem); + return err; +} + +static int remove_xattr(struct ubifs_info *c, struct inode *host, + struct inode *inode, const struct fscrypt_name *nm) +{ + int err; + struct ubifs_inode *host_ui = ubifs_inode(host); + struct ubifs_inode *ui = ubifs_inode(inode); + struct ubifs_budget_req req = { .dirtied_ino = 2, .mod_dent = 1, + .dirtied_ino_d = ALIGN(host_ui->data_len, 8) }; + + ubifs_assert(c, ui->data_len == inode->i_size); + + err = ubifs_budget_space(c, &req); + if (err) + return err; + + mutex_lock(&host_ui->ui_mutex); + inode_set_ctime_current(host); + host_ui->xattr_cnt -= 1; + host_ui->xattr_size -= CALC_DENT_SIZE(fname_len(nm)); + host_ui->xattr_size -= CALC_XATTR_BYTES(ui->data_len); + host_ui->xattr_names -= fname_len(nm); + + err = ubifs_jnl_delete_xattr(c, host, inode, nm); + if (err) + goto out_cancel; + mutex_unlock(&host_ui->ui_mutex); + + ubifs_release_budget(c, &req); + return 0; + +out_cancel: + host_ui->xattr_cnt += 1; + host_ui->xattr_size += CALC_DENT_SIZE(fname_len(nm)); + host_ui->xattr_size += CALC_XATTR_BYTES(ui->data_len); + host_ui->xattr_names += fname_len(nm); + mutex_unlock(&host_ui->ui_mutex); + ubifs_release_budget(c, &req); + make_bad_inode(inode); + return err; +} + +int ubifs_purge_xattrs(struct inode *host) +{ + union ubifs_key key; + struct ubifs_info *c = host->i_sb->s_fs_info; + struct ubifs_dent_node *xent, *pxent = NULL; + struct inode *xino; + struct fscrypt_name nm = {0}; + int err; + + if (ubifs_inode(host)->xattr_cnt <= ubifs_xattr_max_cnt(c)) + return 0; + + ubifs_warn(c, "inode %llu has too many xattrs, doing a non-atomic deletion", + host->i_ino); + + down_write(&ubifs_inode(host)->xattr_sem); + lowest_xent_key(c, &key, host->i_ino); + while (1) { + xent = ubifs_tnc_next_ent(c, &key, &nm); + if (IS_ERR(xent)) { + err = PTR_ERR(xent); + break; + } + + fname_name(&nm) = xent->name; + fname_len(&nm) = le16_to_cpu(xent->nlen); + + xino = ubifs_iget(c->vfs_sb, le64_to_cpu(xent->inum)); + if (IS_ERR(xino)) { + err = PTR_ERR(xino); + ubifs_err(c, "dead directory entry '%s', error %d", + xent->name, err); + ubifs_ro_mode(c, err); + goto out_err; + } + + ubifs_assert(c, ubifs_inode(xino)->xattr); + + clear_nlink(xino); + err = remove_xattr(c, host, xino, &nm); + iput(xino); + if (err) { + ubifs_err(c, "cannot remove xattr, error %d", err); + goto out_err; + } + + kfree(pxent); + pxent = xent; + key_read(c, &xent->key, &key); + } + kfree(pxent); + up_write(&ubifs_inode(host)->xattr_sem); + + if (err != -ENOENT) { + ubifs_err(c, "cannot find next direntry, error %d", err); + return err; + } + + return 0; + +out_err: + kfree(pxent); + kfree(xent); + up_write(&ubifs_inode(host)->xattr_sem); + return err; +} + +static int ubifs_xattr_remove(struct inode *host, const char *name) +{ + struct inode *inode; + struct ubifs_info *c = host->i_sb->s_fs_info; + struct fscrypt_name nm = { .disk_name = FSTR_INIT((char *)name, strlen(name))}; + struct ubifs_dent_node *xent; + union ubifs_key key; + int err; + + ubifs_assert(c, inode_is_locked(host)); + + if (fname_len(&nm) > UBIFS_MAX_NLEN) + return -ENAMETOOLONG; + + xent = kmalloc(UBIFS_MAX_XENT_NODE_SZ, GFP_NOFS); + if (!xent) + return -ENOMEM; + + down_write(&ubifs_inode(host)->xattr_sem); + xent_key_init(c, &key, host->i_ino, &nm); + err = ubifs_tnc_lookup_nm(c, &key, xent, &nm); + if (err) { + if (err == -ENOENT) + err = -ENODATA; + goto out_free; + } + + inode = iget_xattr(c, le64_to_cpu(xent->inum)); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + goto out_free; + } + + ubifs_assert(c, inode->i_nlink == 1); + clear_nlink(inode); + err = remove_xattr(c, host, inode, &nm); + if (err) + set_nlink(inode, 1); + + /* If @i_nlink is 0, 'iput()' will delete the inode */ + iput(inode); + +out_free: + up_write(&ubifs_inode(host)->xattr_sem); + kfree(xent); + return err; +} + +#ifdef CONFIG_UBIFS_FS_SECURITY +static int init_xattrs(struct inode *inode, const struct xattr *xattr_array, + void *fs_info) +{ + const struct xattr *xattr; + char *name; + int err = 0; + + for (xattr = xattr_array; xattr->name != NULL; xattr++) { + name = kmalloc(XATTR_SECURITY_PREFIX_LEN + + strlen(xattr->name) + 1, GFP_NOFS); + if (!name) { + err = -ENOMEM; + break; + } + strcpy(name, XATTR_SECURITY_PREFIX); + strcpy(name + XATTR_SECURITY_PREFIX_LEN, xattr->name); + /* + * creating a new inode without holding the inode rwsem, + * no need to check whether inode is locked. + */ + err = ubifs_xattr_set(inode, name, xattr->value, + xattr->value_len, 0, false); + kfree(name); + if (err < 0) + break; + } + + return err; +} + +int ubifs_init_security(struct inode *dentry, struct inode *inode, + const struct qstr *qstr) +{ + int err; + + err = security_inode_init_security(inode, dentry, qstr, + &init_xattrs, NULL); + if (err) { + struct ubifs_info *c = dentry->i_sb->s_fs_info; + ubifs_err(c, "cannot initialize security for inode %llu, error %d", + inode->i_ino, err); + } + return err; +} +#endif + +static int xattr_get(const struct xattr_handler *handler, + struct dentry *dentry, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + dbg_gen("xattr '%s', ino %llu ('%pd'), buf size %zd", name, + inode->i_ino, dentry, size); + + name = xattr_full_name(handler, name); + return ubifs_xattr_get(inode, name, buffer, size); +} + +static int xattr_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, + struct dentry *dentry, struct inode *inode, + const char *name, const void *value, + size_t size, int flags) +{ + dbg_gen("xattr '%s', host ino %llu ('%pd'), size %zd", + name, inode->i_ino, dentry, size); + + name = xattr_full_name(handler, name); + + if (value) + return ubifs_xattr_set(inode, name, value, size, flags, true); + else + return ubifs_xattr_remove(inode, name); +} + +static const struct xattr_handler ubifs_user_xattr_handler = { + .prefix = XATTR_USER_PREFIX, + .get = xattr_get, + .set = xattr_set, +}; + +static const struct xattr_handler ubifs_trusted_xattr_handler = { + .prefix = XATTR_TRUSTED_PREFIX, + .get = xattr_get, + .set = xattr_set, +}; + +#ifdef CONFIG_UBIFS_FS_SECURITY +static const struct xattr_handler ubifs_security_xattr_handler = { + .prefix = XATTR_SECURITY_PREFIX, + .get = xattr_get, + .set = xattr_set, +}; +#endif + +const struct xattr_handler * const ubifs_xattr_handlers[] = { + &ubifs_user_xattr_handler, + &ubifs_trusted_xattr_handler, +#ifdef CONFIG_UBIFS_FS_SECURITY + &ubifs_security_xattr_handler, +#endif + NULL +}; diff --git a/fs/udf/Kconfig b/fs/udf/Kconfig new file mode 100644 index 000000000..8f7ce30d4 --- /dev/null +++ b/fs/udf/Kconfig @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0-only +config UDF_FS + tristate "UDF file system support" + select BUFFER_HEAD + select CRC_ITU_T + select NLS + select LEGACY_DIRECT_IO + help + This is a file system used on some CD-ROMs and DVDs. Since the + file system is supported by multiple operating systems and is more + compatible with standard unix file systems, it is also suitable for + removable USB disks. Say Y if you intend to mount DVD discs or CDRW's + written in packet mode, or if you want to use UDF for removable USB + disks. Please read . + + To compile this file system support as a module, choose M here: the + module will be called udf. + + If unsure, say N. diff --git a/fs/udf/Makefile b/fs/udf/Makefile new file mode 100644 index 000000000..63981cd33 --- /dev/null +++ b/fs/udf/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the linux udf-filesystem routines. +# + +obj-$(CONFIG_UDF_FS) += udf.o + +udf-objs := balloc.o dir.o file.o ialloc.o inode.o lowlevel.o namei.o \ + partition.o super.o truncate.o symlink.o \ + directory.o misc.o udftime.o unicode.o diff --git a/fs/udf/balloc.c b/fs/udf/balloc.c new file mode 100644 index 000000000..30cec5600 --- /dev/null +++ b/fs/udf/balloc.c @@ -0,0 +1,744 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * balloc.c + * + * PURPOSE + * Block allocation handling routines for the OSTA-UDF(tm) filesystem. + * + * COPYRIGHT + * (C) 1999-2001 Ben Fennema + * (C) 1999 Stelias Computing Inc + * + * HISTORY + * + * 02/24/99 blf Created. + * + */ + +#include "udfdecl.h" + +#include +#include + +#include "udf_i.h" +#include "udf_sb.h" + +#define udf_clear_bit __test_and_clear_bit_le +#define udf_set_bit __test_and_set_bit_le +#define udf_test_bit test_bit_le +#define udf_find_next_one_bit find_next_bit_le + +static int read_block_bitmap(struct super_block *sb, + struct udf_bitmap *bitmap, unsigned int block, + unsigned long bitmap_nr) +{ + struct buffer_head *bh = NULL; + int i; + int max_bits, off, count; + struct kernel_lb_addr loc; + + loc.logicalBlockNum = bitmap->s_extPosition; + loc.partitionReferenceNum = UDF_SB(sb)->s_partition; + + bh = sb_bread(sb, udf_get_lb_pblock(sb, &loc, block)); + bitmap->s_block_bitmap[bitmap_nr] = bh; + if (!bh) + return -EIO; + + /* Check consistency of Space Bitmap buffer. */ + max_bits = sb->s_blocksize * 8; + if (!bitmap_nr) { + off = sizeof(struct spaceBitmapDesc) << 3; + count = min(max_bits - off, bitmap->s_nr_groups); + } else { + /* + * Rough check if bitmap number is too big to have any bitmap + * blocks reserved. + */ + if (bitmap_nr > + (bitmap->s_nr_groups >> (sb->s_blocksize_bits + 3)) + 2) + return 0; + off = 0; + count = bitmap->s_nr_groups - bitmap_nr * max_bits + + (sizeof(struct spaceBitmapDesc) << 3); + count = min(count, max_bits); + } + + for (i = 0; i < count; i++) + if (udf_test_bit(i + off, bh->b_data)) { + bitmap->s_block_bitmap[bitmap_nr] = + ERR_PTR(-EFSCORRUPTED); + brelse(bh); + return -EFSCORRUPTED; + } + return 0; +} + +static int load_block_bitmap(struct super_block *sb, + struct udf_bitmap *bitmap, + unsigned int block_group) +{ + int retval = 0; + int nr_groups = bitmap->s_nr_groups; + + if (block_group >= nr_groups) { + udf_debug("block_group (%u) >= nr_groups (%d)\n", + block_group, nr_groups); + return -EFSCORRUPTED; + } + + if (bitmap->s_block_bitmap[block_group]) { + /* + * The bitmap failed verification in the past. No point in + * trying again. + */ + if (IS_ERR(bitmap->s_block_bitmap[block_group])) + return PTR_ERR(bitmap->s_block_bitmap[block_group]); + return block_group; + } + + retval = read_block_bitmap(sb, bitmap, block_group, block_group); + if (retval < 0) + return retval; + + return block_group; +} + +static void udf_add_free_space(struct super_block *sb, u16 partition, u32 cnt) +{ + struct udf_sb_info *sbi = UDF_SB(sb); + struct logicalVolIntegrityDesc *lvid; + + if (!sbi->s_lvid_bh) + return; + + lvid = (struct logicalVolIntegrityDesc *)sbi->s_lvid_bh->b_data; + le32_add_cpu(&lvid->freeSpaceTable[partition], cnt); + udf_updated_lvid(sb); +} + +static void udf_bitmap_free_blocks(struct super_block *sb, + struct udf_bitmap *bitmap, + struct kernel_lb_addr *bloc, + uint32_t offset, + uint32_t count) +{ + struct udf_sb_info *sbi = UDF_SB(sb); + struct buffer_head *bh = NULL; + unsigned long block; + unsigned long block_group; + unsigned long bit; + unsigned long i; + int bitmap_nr; + unsigned long overflow; + + mutex_lock(&sbi->s_alloc_mutex); + /* We make sure this cannot overflow when mounting the filesystem */ + block = bloc->logicalBlockNum + offset + + (sizeof(struct spaceBitmapDesc) << 3); + do { + overflow = 0; + block_group = block >> (sb->s_blocksize_bits + 3); + bit = block % (sb->s_blocksize << 3); + + /* + * Check to see if we are freeing blocks across a group boundary. + */ + if (bit + count > (sb->s_blocksize << 3)) { + overflow = bit + count - (sb->s_blocksize << 3); + count -= overflow; + } + bitmap_nr = load_block_bitmap(sb, bitmap, block_group); + if (bitmap_nr < 0) + goto error_return; + + bh = bitmap->s_block_bitmap[bitmap_nr]; + for (i = 0; i < count; i++) { + if (udf_set_bit(bit + i, bh->b_data)) { + udf_debug("bit %lu already set\n", bit + i); + udf_debug("byte=%2x\n", + ((__u8 *)bh->b_data)[(bit + i) >> 3]); + } + } + udf_add_free_space(sb, sbi->s_partition, count); + mark_buffer_dirty(bh); + if (overflow) { + block += count; + count = overflow; + } + } while (overflow); + +error_return: + mutex_unlock(&sbi->s_alloc_mutex); +} + +static int udf_bitmap_prealloc_blocks(struct super_block *sb, + struct udf_bitmap *bitmap, + uint16_t partition, uint32_t first_block, + uint32_t block_count) +{ + struct udf_sb_info *sbi = UDF_SB(sb); + int alloc_count = 0; + int bit, block, block_group; + int bitmap_nr; + struct buffer_head *bh; + __u32 part_len; + + mutex_lock(&sbi->s_alloc_mutex); + part_len = sbi->s_partmaps[partition].s_partition_len; + if (first_block >= part_len) + goto out; + + if (first_block + block_count > part_len) + block_count = part_len - first_block; + + do { + block = first_block + (sizeof(struct spaceBitmapDesc) << 3); + block_group = block >> (sb->s_blocksize_bits + 3); + + bitmap_nr = load_block_bitmap(sb, bitmap, block_group); + if (bitmap_nr < 0) + goto out; + bh = bitmap->s_block_bitmap[bitmap_nr]; + + bit = block % (sb->s_blocksize << 3); + + while (bit < (sb->s_blocksize << 3) && block_count > 0) { + if (!udf_clear_bit(bit, bh->b_data)) + goto out; + block_count--; + alloc_count++; + bit++; + block++; + } + mark_buffer_dirty(bh); + } while (block_count > 0); + +out: + udf_add_free_space(sb, partition, -alloc_count); + mutex_unlock(&sbi->s_alloc_mutex); + return alloc_count; +} + +static udf_pblk_t udf_bitmap_new_block(struct super_block *sb, + struct udf_bitmap *bitmap, uint16_t partition, + uint32_t goal, int *err) +{ + struct udf_sb_info *sbi = UDF_SB(sb); + int newbit, bit = 0; + udf_pblk_t block; + int block_group, group_start; + int end_goal, nr_groups, bitmap_nr, i; + struct buffer_head *bh = NULL; + char *ptr; + udf_pblk_t newblock = 0; + + *err = -ENOSPC; + mutex_lock(&sbi->s_alloc_mutex); + +repeat: + if (goal >= sbi->s_partmaps[partition].s_partition_len) + goal = 0; + + nr_groups = bitmap->s_nr_groups; + block = goal + (sizeof(struct spaceBitmapDesc) << 3); + block_group = block >> (sb->s_blocksize_bits + 3); + group_start = block_group ? 0 : sizeof(struct spaceBitmapDesc); + + bitmap_nr = load_block_bitmap(sb, bitmap, block_group); + if (bitmap_nr < 0) + goto error_return; + bh = bitmap->s_block_bitmap[bitmap_nr]; + ptr = memscan((char *)bh->b_data + group_start, 0xFF, + sb->s_blocksize - group_start); + + if ((ptr - ((char *)bh->b_data)) < sb->s_blocksize) { + bit = block % (sb->s_blocksize << 3); + if (udf_test_bit(bit, bh->b_data)) + goto got_block; + + end_goal = (bit + 63) & ~63; + bit = udf_find_next_one_bit(bh->b_data, end_goal, bit); + if (bit < end_goal) + goto got_block; + + ptr = memscan((char *)bh->b_data + (bit >> 3), 0xFF, + sb->s_blocksize - ((bit + 7) >> 3)); + newbit = (ptr - ((char *)bh->b_data)) << 3; + if (newbit < sb->s_blocksize << 3) { + bit = newbit; + goto search_back; + } + + newbit = udf_find_next_one_bit(bh->b_data, + sb->s_blocksize << 3, bit); + if (newbit < sb->s_blocksize << 3) { + bit = newbit; + goto got_block; + } + } + + for (i = 0; i < (nr_groups * 2); i++) { + block_group++; + if (block_group >= nr_groups) + block_group = 0; + group_start = block_group ? 0 : sizeof(struct spaceBitmapDesc); + + bitmap_nr = load_block_bitmap(sb, bitmap, block_group); + if (bitmap_nr < 0) + goto error_return; + bh = bitmap->s_block_bitmap[bitmap_nr]; + if (i < nr_groups) { + ptr = memscan((char *)bh->b_data + group_start, 0xFF, + sb->s_blocksize - group_start); + if ((ptr - ((char *)bh->b_data)) < sb->s_blocksize) { + bit = (ptr - ((char *)bh->b_data)) << 3; + break; + } + } else { + bit = udf_find_next_one_bit(bh->b_data, + sb->s_blocksize << 3, + group_start << 3); + if (bit < sb->s_blocksize << 3) + break; + } + } + if (i >= (nr_groups * 2)) { + mutex_unlock(&sbi->s_alloc_mutex); + return newblock; + } + if (bit < sb->s_blocksize << 3) + goto search_back; + else + bit = udf_find_next_one_bit(bh->b_data, sb->s_blocksize << 3, + group_start << 3); + if (bit >= sb->s_blocksize << 3) { + mutex_unlock(&sbi->s_alloc_mutex); + return 0; + } + +search_back: + i = 0; + while (i < 7 && bit > (group_start << 3) && + udf_test_bit(bit - 1, bh->b_data)) { + ++i; + --bit; + } + +got_block: + newblock = bit + (block_group << (sb->s_blocksize_bits + 3)) - + (sizeof(struct spaceBitmapDesc) << 3); + + if (newblock >= sbi->s_partmaps[partition].s_partition_len) { + /* + * Ran off the end of the bitmap, and bits following are + * non-compliant (not all zero) + */ + udf_err(sb, "bitmap for partition %d corrupted (block %u marked" + " as free, partition length is %u)\n", partition, + newblock, sbi->s_partmaps[partition].s_partition_len); + goto error_return; + } + + if (!udf_clear_bit(bit, bh->b_data)) { + udf_debug("bit already cleared for block %d\n", bit); + goto repeat; + } + + mark_buffer_dirty(bh); + + udf_add_free_space(sb, partition, -1); + mutex_unlock(&sbi->s_alloc_mutex); + *err = 0; + return newblock; + +error_return: + *err = -EIO; + mutex_unlock(&sbi->s_alloc_mutex); + return 0; +} + +static void udf_table_free_blocks(struct super_block *sb, + struct inode *table, + struct kernel_lb_addr *bloc, + uint32_t offset, + uint32_t count) +{ + struct udf_sb_info *sbi = UDF_SB(sb); + uint32_t start, end; + uint32_t elen; + struct kernel_lb_addr eloc; + struct extent_position oepos, epos; + int8_t etype; + struct udf_inode_info *iinfo; + int ret = 0; + + mutex_lock(&sbi->s_alloc_mutex); + iinfo = UDF_I(table); + udf_add_free_space(sb, sbi->s_partition, count); + + start = bloc->logicalBlockNum + offset; + end = bloc->logicalBlockNum + offset + count - 1; + + epos.offset = oepos.offset = sizeof(struct unallocSpaceEntry); + elen = 0; + epos.block = oepos.block = iinfo->i_location; + epos.bh = oepos.bh = NULL; + + while (count) { + ret = udf_next_aext(table, &epos, &eloc, &elen, &etype, 1); + if (ret < 0) + goto error_return; + if (ret == 0) + break; + if (((eloc.logicalBlockNum + + (elen >> sb->s_blocksize_bits)) == start)) { + if ((0x3FFFFFFF - elen) < + (count << sb->s_blocksize_bits)) { + uint32_t tmp = ((0x3FFFFFFF - elen) >> + sb->s_blocksize_bits); + count -= tmp; + start += tmp; + elen = (etype << 30) | + (0x40000000 - sb->s_blocksize); + } else { + elen = (etype << 30) | + (elen + + (count << sb->s_blocksize_bits)); + start += count; + count = 0; + } + udf_write_aext(table, &oepos, &eloc, elen, 1); + } else if (eloc.logicalBlockNum == (end + 1)) { + if ((0x3FFFFFFF - elen) < + (count << sb->s_blocksize_bits)) { + uint32_t tmp = ((0x3FFFFFFF - elen) >> + sb->s_blocksize_bits); + count -= tmp; + end -= tmp; + eloc.logicalBlockNum -= tmp; + elen = (etype << 30) | + (0x40000000 - sb->s_blocksize); + } else { + eloc.logicalBlockNum = start; + elen = (etype << 30) | + (elen + + (count << sb->s_blocksize_bits)); + end -= count; + count = 0; + } + udf_write_aext(table, &oepos, &eloc, elen, 1); + } + + if (epos.bh != oepos.bh) { + oepos.block = epos.block; + brelse(oepos.bh); + get_bh(epos.bh); + oepos.bh = epos.bh; + oepos.offset = 0; + } else { + oepos.offset = epos.offset; + } + } + + if (count) { + /* + * NOTE: we CANNOT use udf_add_aext here, as it can try to + * allocate a new block, and since we hold the super block + * lock already very bad things would happen :) + * + * We copy the behavior of udf_add_aext, but instead of + * trying to allocate a new block close to the existing one, + * we just steal a block from the extent we are trying to add. + * + * It would be nice if the blocks were close together, but it + * isn't required. + */ + + int adsize; + + eloc.logicalBlockNum = start; + elen = EXT_RECORDED_ALLOCATED | + (count << sb->s_blocksize_bits); + + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_SHORT) + adsize = sizeof(struct short_ad); + else if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_LONG) + adsize = sizeof(struct long_ad); + else + goto error_return; + + if (epos.offset + (2 * adsize) > sb->s_blocksize) { + /* Steal a block from the extent being free'd */ + udf_setup_indirect_aext(table, eloc.logicalBlockNum, + &epos); + + eloc.logicalBlockNum++; + elen -= sb->s_blocksize; + } + + /* It's possible that stealing the block emptied the extent */ + if (elen) + __udf_add_aext(table, &epos, &eloc, elen, 1); + } + +error_return: + brelse(epos.bh); + brelse(oepos.bh); + + mutex_unlock(&sbi->s_alloc_mutex); + return; +} + +static int udf_table_prealloc_blocks(struct super_block *sb, + struct inode *table, uint16_t partition, + uint32_t first_block, uint32_t block_count) +{ + struct udf_sb_info *sbi = UDF_SB(sb); + int alloc_count = 0; + uint32_t elen, adsize; + struct kernel_lb_addr eloc; + struct extent_position epos; + int8_t etype = -1; + struct udf_inode_info *iinfo; + int ret = 0; + /* AED block freed by udf_delete_aext(), released after unlock */ + struct kernel_lb_addr freed = { .partitionReferenceNum = 0xFFFF }; + + if (first_block >= sbi->s_partmaps[partition].s_partition_len) + return 0; + + iinfo = UDF_I(table); + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_SHORT) + adsize = sizeof(struct short_ad); + else if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_LONG) + adsize = sizeof(struct long_ad); + else + return 0; + + mutex_lock(&sbi->s_alloc_mutex); + epos.offset = sizeof(struct unallocSpaceEntry); + epos.block = iinfo->i_location; + epos.bh = NULL; + eloc.logicalBlockNum = 0xFFFFFFFF; + + while (first_block != eloc.logicalBlockNum) { + ret = udf_next_aext(table, &epos, &eloc, &elen, &etype, 1); + if (ret < 0) + goto err_out; + if (ret == 0) + break; + udf_debug("eloc=%u, elen=%u, first_block=%u\n", + eloc.logicalBlockNum, elen, first_block); + } + + if (first_block == eloc.logicalBlockNum) { + epos.offset -= adsize; + + alloc_count = (elen >> sb->s_blocksize_bits); + if (alloc_count > block_count) { + alloc_count = block_count; + eloc.logicalBlockNum += alloc_count; + elen -= (alloc_count << sb->s_blocksize_bits); + udf_write_aext(table, &epos, &eloc, + (etype << 30) | elen, 1); + } else + udf_delete_aext(table, epos, &freed); + } else { + alloc_count = 0; + } + +err_out: + brelse(epos.bh); + + if (alloc_count) + udf_add_free_space(sb, partition, -alloc_count); + mutex_unlock(&sbi->s_alloc_mutex); + if (freed.partitionReferenceNum != 0xFFFF) + udf_free_blocks(sb, table, &freed, 0, 1); + return alloc_count; +} + +static udf_pblk_t udf_table_new_block(struct super_block *sb, + struct inode *table, uint16_t partition, + uint32_t goal, int *err) +{ + struct udf_sb_info *sbi = UDF_SB(sb); + /* AED block freed by udf_delete_aext(), released after unlock */ + struct kernel_lb_addr freed = { .partitionReferenceNum = 0xFFFF }; + uint32_t spread = 0xFFFFFFFF, nspread = 0xFFFFFFFF; + udf_pblk_t newblock = 0; + uint32_t adsize; + uint32_t elen, goal_elen = 0; + struct kernel_lb_addr eloc, goal_eloc; + struct extent_position epos, goal_epos; + int8_t etype; + struct udf_inode_info *iinfo = UDF_I(table); + int ret = 0; + + *err = -ENOSPC; + + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_SHORT) + adsize = sizeof(struct short_ad); + else if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_LONG) + adsize = sizeof(struct long_ad); + else + return newblock; + + mutex_lock(&sbi->s_alloc_mutex); + if (goal >= sbi->s_partmaps[partition].s_partition_len) + goal = 0; + + /* We search for the closest matching block to goal. If we find + a exact hit, we stop. Otherwise we keep going till we run out + of extents. We store the buffer_head, bloc, and extoffset + of the current closest match and use that when we are done. + */ + epos.offset = sizeof(struct unallocSpaceEntry); + epos.block = iinfo->i_location; + epos.bh = goal_epos.bh = NULL; + + while (spread) { + ret = udf_next_aext(table, &epos, &eloc, &elen, &etype, 1); + if (ret <= 0) + break; + if (goal >= eloc.logicalBlockNum) { + if (goal < eloc.logicalBlockNum + + (elen >> sb->s_blocksize_bits)) + nspread = 0; + else + nspread = goal - eloc.logicalBlockNum - + (elen >> sb->s_blocksize_bits); + } else { + nspread = eloc.logicalBlockNum - goal; + } + + if (nspread < spread) { + spread = nspread; + if (goal_epos.bh != epos.bh) { + brelse(goal_epos.bh); + goal_epos.bh = epos.bh; + get_bh(goal_epos.bh); + } + goal_epos.block = epos.block; + goal_epos.offset = epos.offset - adsize; + goal_eloc = eloc; + goal_elen = (etype << 30) | elen; + } + } + + brelse(epos.bh); + + if (ret < 0 || spread == 0xFFFFFFFF) { + brelse(goal_epos.bh); + mutex_unlock(&sbi->s_alloc_mutex); + if (ret < 0) + *err = ret; + return 0; + } + + /* Only allocate blocks from the beginning of the extent. + That way, we only delete (empty) extents, never have to insert an + extent because of splitting */ + /* This works, but very poorly.... */ + + newblock = goal_eloc.logicalBlockNum; + goal_eloc.logicalBlockNum++; + goal_elen -= sb->s_blocksize; + + if (goal_elen) + udf_write_aext(table, &goal_epos, &goal_eloc, goal_elen, 1); + else + udf_delete_aext(table, goal_epos, &freed); + brelse(goal_epos.bh); + + udf_add_free_space(sb, partition, -1); + + mutex_unlock(&sbi->s_alloc_mutex); + if (freed.partitionReferenceNum != 0xFFFF) + udf_free_blocks(sb, table, &freed, 0, 1); + *err = 0; + return newblock; +} + +void udf_free_blocks(struct super_block *sb, struct inode *inode, + struct kernel_lb_addr *bloc, uint32_t offset, + uint32_t count) +{ + uint16_t partition = bloc->partitionReferenceNum; + struct udf_part_map *map = &UDF_SB(sb)->s_partmaps[partition]; + uint32_t blk; + + if (check_add_overflow(bloc->logicalBlockNum, offset, &blk) || + check_add_overflow(blk, count, &blk) || + blk > map->s_partition_len) { + udf_debug("Invalid request to free blocks: (%d, %u), off %u, " + "len %u, partition len %u\n", + partition, bloc->logicalBlockNum, offset, count, + map->s_partition_len); + return; + } + + if (map->s_partition_flags & UDF_PART_FLAG_UNALLOC_BITMAP) { + udf_bitmap_free_blocks(sb, map->s_uspace.s_bitmap, + bloc, offset, count); + } else if (map->s_partition_flags & UDF_PART_FLAG_UNALLOC_TABLE) { + udf_table_free_blocks(sb, map->s_uspace.s_table, + bloc, offset, count); + } + + if (inode) { + inode_sub_bytes(inode, + ((sector_t)count) << sb->s_blocksize_bits); + } +} + +inline int udf_prealloc_blocks(struct super_block *sb, + struct inode *inode, + uint16_t partition, uint32_t first_block, + uint32_t block_count) +{ + struct udf_part_map *map = &UDF_SB(sb)->s_partmaps[partition]; + int allocated; + + if (map->s_partition_flags & UDF_PART_FLAG_UNALLOC_BITMAP) + allocated = udf_bitmap_prealloc_blocks(sb, + map->s_uspace.s_bitmap, + partition, first_block, + block_count); + else if (map->s_partition_flags & UDF_PART_FLAG_UNALLOC_TABLE) + allocated = udf_table_prealloc_blocks(sb, + map->s_uspace.s_table, + partition, first_block, + block_count); + else + return 0; + + if (inode && allocated > 0) + inode_add_bytes(inode, allocated << sb->s_blocksize_bits); + return allocated; +} + +inline udf_pblk_t udf_new_block(struct super_block *sb, + struct inode *inode, + uint16_t partition, uint32_t goal, int *err) +{ + struct udf_part_map *map = &UDF_SB(sb)->s_partmaps[partition]; + udf_pblk_t block; + + if (map->s_partition_flags & UDF_PART_FLAG_UNALLOC_BITMAP) + block = udf_bitmap_new_block(sb, + map->s_uspace.s_bitmap, + partition, goal, err); + else if (map->s_partition_flags & UDF_PART_FLAG_UNALLOC_TABLE) + block = udf_table_new_block(sb, + map->s_uspace.s_table, + partition, goal, err); + else { + *err = -EIO; + return 0; + } + if (inode && block) + inode_add_bytes(inode, sb->s_blocksize); + return block; +} diff --git a/fs/udf/dir.c b/fs/udf/dir.c new file mode 100644 index 000000000..425e3c162 --- /dev/null +++ b/fs/udf/dir.c @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * dir.c + * + * PURPOSE + * Directory handling routines for the OSTA-UDF(tm) filesystem. + * + * COPYRIGHT + * (C) 1998-2004 Ben Fennema + * + * HISTORY + * + * 10/05/98 dgb Split directory operations into its own file + * Implemented directory reads via do_udf_readdir + * 10/06/98 Made directory operations work! + * 11/17/98 Rewrote directory to support ICBTAG_FLAG_AD_LONG + * 11/25/98 blf Rewrote directory handling (readdir+lookup) to support reading + * across blocks. + * 12/12/98 Split out the lookup code to namei.c. bulk of directory + * code now in directory.c:udf_fileident_read. + */ + +#include "udfdecl.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "udf_i.h" +#include "udf_sb.h" + +static int udf_readdir(struct file *file, struct dir_context *ctx) +{ + struct inode *dir = file_inode(file); + loff_t nf_pos, emit_pos = 0; + int flen; + unsigned char *fname = NULL; + int ret = 0; + struct super_block *sb = dir->i_sb; + bool pos_valid = false; + struct udf_fileident_iter iter; + + if (ctx->pos == 0) { + if (!dir_emit_dot(file, ctx)) + return 0; + ctx->pos = 1; + } + nf_pos = (ctx->pos - 1) << 2; + if (nf_pos >= dir->i_size) + goto out; + + /* + * Something changed since last readdir (either lseek was called or dir + * changed)? We need to verify the position correctly points at the + * beginning of some dir entry so that the directory parsing code does + * not get confused. Since UDF does not have any reliable way of + * identifying beginning of dir entry (names are under user control), + * we need to scan the directory from the beginning. + */ + if (!inode_eq_iversion(dir, *(u64 *)file->private_data)) { + emit_pos = nf_pos; + nf_pos = 0; + } else { + pos_valid = true; + } + + fname = kmalloc(UDF_NAME_LEN, GFP_KERNEL); + if (!fname) { + ret = -ENOMEM; + goto out; + } + + for (ret = udf_fiiter_init(&iter, dir, nf_pos); + !ret && iter.pos < dir->i_size; + ret = udf_fiiter_advance(&iter)) { + struct kernel_lb_addr tloc; + udf_pblk_t iblock; + + /* Still not at offset where user asked us to read from? */ + if (iter.pos < emit_pos) + continue; + + /* Update file position only if we got past the current one */ + pos_valid = true; + ctx->pos = (iter.pos >> 2) + 1; + + if (iter.fi.fileCharacteristics & FID_FILE_CHAR_DELETED) { + if (!UDF_QUERY_FLAG(sb, UDF_FLAG_UNDELETE)) + continue; + } + + if (iter.fi.fileCharacteristics & FID_FILE_CHAR_HIDDEN) { + if (!UDF_QUERY_FLAG(sb, UDF_FLAG_UNHIDE)) + continue; + } + + if (iter.fi.fileCharacteristics & FID_FILE_CHAR_PARENT) { + if (!dir_emit_dotdot(file, ctx)) + goto out_iter; + continue; + } + + flen = udf_get_filename(sb, iter.name, + iter.fi.lengthFileIdent, fname, UDF_NAME_LEN); + if (flen < 0) + continue; + + tloc = lelb_to_cpu(iter.fi.icb.extLocation); + iblock = udf_get_lb_pblock(sb, &tloc, 0); + if (!dir_emit(ctx, fname, flen, iblock, DT_UNKNOWN)) + goto out_iter; + } + + if (!ret) { + ctx->pos = (iter.pos >> 2) + 1; + pos_valid = true; + } +out_iter: + udf_fiiter_release(&iter); +out: + if (pos_valid) + *(u64 *)file->private_data = inode_query_iversion(dir); + kfree(fname); + + return ret; +} + +static int udf_dir_open(struct inode *inode, struct file *file) +{ + file->private_data = kzalloc(sizeof(u64), GFP_KERNEL); + if (!file->private_data) + return -ENOMEM; + return 0; +} + +static int udf_dir_release(struct inode *inode, struct file *file) +{ + kfree(file->private_data); + return 0; +} + +static loff_t udf_dir_llseek(struct file *file, loff_t offset, int whence) +{ + return generic_llseek_cookie(file, offset, whence, + (u64 *)file->private_data); +} + +/* readdir and lookup functions */ +const struct file_operations udf_dir_operations = { + .open = udf_dir_open, + .release = udf_dir_release, + .llseek = udf_dir_llseek, + .read = generic_read_dir, + .iterate_shared = udf_readdir, + .unlocked_ioctl = udf_ioctl, + .fsync = simple_fsync, + .setlease = generic_setlease, +}; diff --git a/fs/udf/directory.c b/fs/udf/directory.c new file mode 100644 index 000000000..6b4ac7464 --- /dev/null +++ b/fs/udf/directory.c @@ -0,0 +1,543 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * directory.c + * + * PURPOSE + * Directory related functions + * + */ + +#include "udfdecl.h" +#include "udf_i.h" + +#include +#include +#include +#include +#include + +static int udf_verify_fi(struct udf_fileident_iter *iter) +{ + unsigned int len; + + if (iter->fi.descTag.tagIdent != cpu_to_le16(TAG_IDENT_FID)) { + udf_err(iter->dir->i_sb, + "directory (ino %llu) has entry at pos %llu with incorrect tag %x\n", + iter->dir->i_ino, (unsigned long long)iter->pos, + le16_to_cpu(iter->fi.descTag.tagIdent)); + return -EFSCORRUPTED; + } + len = udf_dir_entry_len(&iter->fi); + if (le16_to_cpu(iter->fi.lengthOfImpUse) & 3) { + udf_err(iter->dir->i_sb, + "directory (ino %llu) has entry at pos %llu with unaligned length of impUse field\n", + iter->dir->i_ino, (unsigned long long)iter->pos); + return -EFSCORRUPTED; + } + /* + * This is in fact allowed by the spec due to long impUse field but + * we don't support it. If there is real media with this large impUse + * field, support can be added. + */ + if (len > 1 << iter->dir->i_blkbits) { + udf_err(iter->dir->i_sb, + "directory (ino %llu) has too big (%u) entry at pos %llu\n", + iter->dir->i_ino, len, (unsigned long long)iter->pos); + return -EFSCORRUPTED; + } + if (iter->pos + len > iter->dir->i_size) { + udf_err(iter->dir->i_sb, + "directory (ino %llu) has entry past directory size at pos %llu\n", + iter->dir->i_ino, (unsigned long long)iter->pos); + return -EFSCORRUPTED; + } + if (udf_dir_entry_len(&iter->fi) != + sizeof(struct tag) + le16_to_cpu(iter->fi.descTag.descCRCLength)) { + udf_err(iter->dir->i_sb, + "directory (ino %llu) has entry where CRC length (%u) does not match entry length (%u)\n", + iter->dir->i_ino, + (unsigned)le16_to_cpu(iter->fi.descTag.descCRCLength), + (unsigned)(udf_dir_entry_len(&iter->fi) - + sizeof(struct tag))); + return -EFSCORRUPTED; + } + return 0; +} + +static int udf_copy_fi(struct udf_fileident_iter *iter) +{ + struct udf_inode_info *iinfo = UDF_I(iter->dir); + u32 blksize = 1 << iter->dir->i_blkbits; + u32 off, len, nameoff; + int err; + + /* Skip copying when we are at EOF */ + if (iter->pos >= iter->dir->i_size) { + iter->name = NULL; + return 0; + } + if (iter->dir->i_size < iter->pos + sizeof(struct fileIdentDesc)) { + udf_err(iter->dir->i_sb, + "directory (ino %llu) has entry straddling EOF\n", + iter->dir->i_ino); + return -EFSCORRUPTED; + } + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { + memcpy(&iter->fi, iinfo->i_data + iinfo->i_lenEAttr + iter->pos, + sizeof(struct fileIdentDesc)); + err = udf_verify_fi(iter); + if (err < 0) + return err; + iter->name = iinfo->i_data + iinfo->i_lenEAttr + iter->pos + + sizeof(struct fileIdentDesc) + + le16_to_cpu(iter->fi.lengthOfImpUse); + return 0; + } + + off = iter->pos & (blksize - 1); + len = min_t(u32, sizeof(struct fileIdentDesc), blksize - off); + memcpy(&iter->fi, iter->bh[0]->b_data + off, len); + if (len < sizeof(struct fileIdentDesc)) + memcpy((char *)(&iter->fi) + len, iter->bh[1]->b_data, + sizeof(struct fileIdentDesc) - len); + err = udf_verify_fi(iter); + if (err < 0) + return err; + + /* Handle directory entry name */ + nameoff = off + sizeof(struct fileIdentDesc) + + le16_to_cpu(iter->fi.lengthOfImpUse); + if (off + udf_dir_entry_len(&iter->fi) <= blksize) { + iter->name = iter->bh[0]->b_data + nameoff; + } else if (nameoff >= blksize) { + iter->name = iter->bh[1]->b_data + (nameoff - blksize); + } else { + iter->name = iter->namebuf; + len = blksize - nameoff; + memcpy(iter->name, iter->bh[0]->b_data + nameoff, len); + memcpy(iter->name + len, iter->bh[1]->b_data, + iter->fi.lengthFileIdent - len); + } + return 0; +} + +/* Readahead 8k once we are at 8k boundary */ +static void udf_readahead_dir(struct udf_fileident_iter *iter) +{ + unsigned int ralen = 16 >> (iter->dir->i_blkbits - 9); + struct buffer_head *tmp, *bha[16]; + int i, num; + udf_pblk_t blk; + + if (iter->loffset & (ralen - 1)) + return; + + if (iter->loffset + ralen > (iter->elen >> iter->dir->i_blkbits)) + ralen = (iter->elen >> iter->dir->i_blkbits) - iter->loffset; + num = 0; + for (i = 0; i < ralen; i++) { + blk = udf_get_lb_pblock(iter->dir->i_sb, &iter->eloc, + iter->loffset + i); + tmp = sb_getblk(iter->dir->i_sb, blk); + if (tmp && !buffer_uptodate(tmp) && !buffer_locked(tmp)) + bha[num++] = tmp; + else + brelse(tmp); + } + if (num) { + bh_readahead_batch(num, bha, REQ_RAHEAD); + for (i = 0; i < num; i++) + brelse(bha[i]); + } +} + +static struct buffer_head *udf_fiiter_bread_blk(struct udf_fileident_iter *iter) +{ + udf_pblk_t blk; + + udf_readahead_dir(iter); + blk = udf_get_lb_pblock(iter->dir->i_sb, &iter->eloc, iter->loffset); + return sb_bread(iter->dir->i_sb, blk); +} + +/* + * Updates loffset to point to next directory block; eloc, elen & epos are + * updated if we need to traverse to the next extent as well. + */ +static int udf_fiiter_advance_blk(struct udf_fileident_iter *iter) +{ + int8_t etype = -1; + int err = 0; + + iter->loffset++; + if (iter->loffset < DIV_ROUND_UP(iter->elen, 1<dir->i_blkbits)) + return 0; + + iter->loffset = 0; + err = udf_next_aext(iter->dir, &iter->epos, &iter->eloc, + &iter->elen, &etype, 1); + if (err < 0) + return err; + else if (err == 0 || etype != (EXT_RECORDED_ALLOCATED >> 30)) { + if (iter->pos == iter->dir->i_size) { + iter->elen = 0; + return 0; + } + udf_err(iter->dir->i_sb, + "extent after position %llu not allocated in directory (ino %llu)\n", + (unsigned long long)iter->pos, iter->dir->i_ino); + return -EFSCORRUPTED; + } + return 0; +} + +static int udf_fiiter_load_bhs(struct udf_fileident_iter *iter) +{ + int blksize = 1 << iter->dir->i_blkbits; + int off = iter->pos & (blksize - 1); + int err; + struct fileIdentDesc *fi; + + /* Is there any further extent we can map from? */ + if (!iter->bh[0] && iter->elen) { + iter->bh[0] = udf_fiiter_bread_blk(iter); + if (!iter->bh[0]) { + err = -ENOMEM; + goto out_brelse; + } + if (!buffer_uptodate(iter->bh[0])) { + err = -EIO; + goto out_brelse; + } + } + /* There's no next block so we are done */ + if (iter->pos >= iter->dir->i_size) + return 0; + /* Need to fetch next block as well? */ + if (off + sizeof(struct fileIdentDesc) > blksize) + goto fetch_next; + fi = (struct fileIdentDesc *)(iter->bh[0]->b_data + off); + /* Need to fetch next block to get name? */ + if (off + udf_dir_entry_len(fi) > blksize) { +fetch_next: + err = udf_fiiter_advance_blk(iter); + if (err) + goto out_brelse; + iter->bh[1] = udf_fiiter_bread_blk(iter); + if (!iter->bh[1]) { + err = -ENOMEM; + goto out_brelse; + } + if (!buffer_uptodate(iter->bh[1])) { + err = -EIO; + goto out_brelse; + } + } + return 0; +out_brelse: + brelse(iter->bh[0]); + brelse(iter->bh[1]); + iter->bh[0] = iter->bh[1] = NULL; + return err; +} + +int udf_fiiter_init(struct udf_fileident_iter *iter, struct inode *dir, + loff_t pos) +{ + struct udf_inode_info *iinfo = UDF_I(dir); + int err = 0; + int8_t etype; + + iter->dir = dir; + iter->bh[0] = iter->bh[1] = NULL; + iter->pos = pos; + iter->elen = 0; + iter->epos.bh = NULL; + iter->name = NULL; + /* + * When directory is verified, we don't expect directory iteration to + * fail and it can be difficult to undo without corrupting filesystem. + * So just do not allow memory allocation failures here. + */ + iter->namebuf = kmalloc(UDF_NAME_LEN_CS0, GFP_KERNEL | __GFP_NOFAIL); + + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { + err = udf_copy_fi(iter); + goto out; + } + + err = inode_bmap(dir, iter->pos >> dir->i_blkbits, &iter->epos, + &iter->eloc, &iter->elen, &iter->loffset, &etype); + if (err <= 0 || etype != (EXT_RECORDED_ALLOCATED >> 30)) { + if (pos == dir->i_size) + return 0; + udf_err(dir->i_sb, + "position %llu not allocated in directory (ino %llu)\n", + (unsigned long long)pos, dir->i_ino); + err = -EFSCORRUPTED; + goto out; + } + err = udf_fiiter_load_bhs(iter); + if (err < 0) + goto out; + err = udf_copy_fi(iter); +out: + if (err < 0) + udf_fiiter_release(iter); + return err; +} + +int udf_fiiter_advance(struct udf_fileident_iter *iter) +{ + unsigned int oldoff, len; + int blksize = 1 << iter->dir->i_blkbits; + int err; + + oldoff = iter->pos & (blksize - 1); + len = udf_dir_entry_len(&iter->fi); + iter->pos += len; + if (UDF_I(iter->dir)->i_alloc_type != ICBTAG_FLAG_AD_IN_ICB) { + if (oldoff + len >= blksize) { + brelse(iter->bh[0]); + iter->bh[0] = NULL; + /* Next block already loaded? */ + if (iter->bh[1]) { + iter->bh[0] = iter->bh[1]; + iter->bh[1] = NULL; + } else { + err = udf_fiiter_advance_blk(iter); + if (err < 0) + return err; + } + } + err = udf_fiiter_load_bhs(iter); + if (err < 0) + return err; + } + return udf_copy_fi(iter); +} + +void udf_fiiter_release(struct udf_fileident_iter *iter) +{ + iter->dir = NULL; + brelse(iter->bh[0]); + brelse(iter->bh[1]); + iter->bh[0] = iter->bh[1] = NULL; + kfree(iter->namebuf); + iter->namebuf = NULL; +} + +static void udf_copy_to_bufs(void *buf1, int len1, void *buf2, int len2, + int off, void *src, int len) +{ + int copy; + + if (off >= len1) { + off -= len1; + } else { + copy = min(off + len, len1) - off; + memcpy(buf1 + off, src, copy); + src += copy; + len -= copy; + off = 0; + } + if (len > 0) { + if (WARN_ON_ONCE(off + len > len2 || !buf2)) + return; + memcpy(buf2 + off, src, len); + } +} + +static uint16_t udf_crc_fi_bufs(void *buf1, int len1, void *buf2, int len2, + int off, int len) +{ + int copy; + uint16_t crc = 0; + + if (off >= len1) { + off -= len1; + } else { + copy = min(off + len, len1) - off; + crc = crc_itu_t(crc, buf1 + off, copy); + len -= copy; + off = 0; + } + if (len > 0) { + if (WARN_ON_ONCE(off + len > len2 || !buf2)) + return 0; + crc = crc_itu_t(crc, buf2 + off, len); + } + return crc; +} + +static void udf_copy_fi_to_bufs(char *buf1, int len1, char *buf2, int len2, + int off, struct fileIdentDesc *fi, + uint8_t *impuse, uint8_t *name) +{ + uint16_t crc; + int fioff = off; + int crcoff = off + sizeof(struct tag); + unsigned int crclen = udf_dir_entry_len(fi) - sizeof(struct tag); + char zeros[UDF_NAME_PAD] = {}; + int endoff = off + udf_dir_entry_len(fi); + + udf_copy_to_bufs(buf1, len1, buf2, len2, off, fi, + sizeof(struct fileIdentDesc)); + off += sizeof(struct fileIdentDesc); + if (impuse) + udf_copy_to_bufs(buf1, len1, buf2, len2, off, impuse, + le16_to_cpu(fi->lengthOfImpUse)); + off += le16_to_cpu(fi->lengthOfImpUse); + if (name) { + udf_copy_to_bufs(buf1, len1, buf2, len2, off, name, + fi->lengthFileIdent); + off += fi->lengthFileIdent; + udf_copy_to_bufs(buf1, len1, buf2, len2, off, zeros, + endoff - off); + } + + crc = udf_crc_fi_bufs(buf1, len1, buf2, len2, crcoff, crclen); + fi->descTag.descCRC = cpu_to_le16(crc); + fi->descTag.descCRCLength = cpu_to_le16(crclen); + fi->descTag.tagChecksum = udf_tag_checksum(&fi->descTag); + + udf_copy_to_bufs(buf1, len1, buf2, len2, fioff, fi, sizeof(struct tag)); +} + +void udf_fiiter_write_fi(struct udf_fileident_iter *iter, uint8_t *impuse) +{ + struct udf_inode_info *iinfo = UDF_I(iter->dir); + void *buf1, *buf2 = NULL; + int len1, len2 = 0, off; + int blksize = 1 << iter->dir->i_blkbits; + + off = iter->pos & (blksize - 1); + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { + buf1 = iinfo->i_data + iinfo->i_lenEAttr; + len1 = iter->dir->i_size; + } else { + buf1 = iter->bh[0]->b_data; + len1 = blksize; + if (iter->bh[1]) { + buf2 = iter->bh[1]->b_data; + len2 = blksize; + } + } + + udf_copy_fi_to_bufs(buf1, len1, buf2, len2, off, &iter->fi, impuse, + iter->name == iter->namebuf ? iter->name : NULL); + + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { + mark_inode_dirty(iter->dir); + } else { + mmb_mark_buffer_dirty(iter->bh[0], &iinfo->i_metadata_bhs); + if (iter->bh[1]) + mmb_mark_buffer_dirty(iter->bh[1], + &iinfo->i_metadata_bhs); + } + inode_inc_iversion(iter->dir); +} + +void udf_fiiter_update_elen(struct udf_fileident_iter *iter, uint32_t new_elen) +{ + struct udf_inode_info *iinfo = UDF_I(iter->dir); + int diff = new_elen - iter->elen; + + /* Skip update when we already went past the last extent */ + if (!iter->elen) + return; + iter->elen = new_elen; + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_SHORT) + iter->epos.offset -= sizeof(struct short_ad); + else if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_LONG) + iter->epos.offset -= sizeof(struct long_ad); + udf_write_aext(iter->dir, &iter->epos, &iter->eloc, iter->elen, 1); + iinfo->i_lenExtents += diff; + mark_inode_dirty(iter->dir); +} + +/* Append new block to directory. @iter is expected to point at EOF */ +int udf_fiiter_append_blk(struct udf_fileident_iter *iter) +{ + struct udf_inode_info *iinfo = UDF_I(iter->dir); + int blksize = 1 << iter->dir->i_blkbits; + struct buffer_head *bh; + sector_t block; + uint32_t old_elen = iter->elen; + int err; + int8_t etype; + + if (WARN_ON_ONCE(iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB)) + return -EINVAL; + + /* Round up last extent in the file */ + udf_fiiter_update_elen(iter, ALIGN(iter->elen, blksize)); + + /* Allocate new block and refresh mapping information */ + block = iinfo->i_lenExtents >> iter->dir->i_blkbits; + bh = udf_bread(iter->dir, block, 1, &err); + if (!bh) { + udf_fiiter_update_elen(iter, old_elen); + return err; + } + err = inode_bmap(iter->dir, block, &iter->epos, &iter->eloc, &iter->elen, + &iter->loffset, &etype); + if (err <= 0 || etype != (EXT_RECORDED_ALLOCATED >> 30)) { + udf_err(iter->dir->i_sb, + "block %llu not allocated in directory (ino %llu)\n", + (unsigned long long)block, iter->dir->i_ino); + return -EFSCORRUPTED; + } + if (!(iter->pos & (blksize - 1))) { + brelse(iter->bh[0]); + iter->bh[0] = bh; + } else { + iter->bh[1] = bh; + } + return 0; +} + +struct short_ad *udf_get_fileshortad(uint8_t *ptr, int maxoffset, uint32_t *offset, + int inc) +{ + struct short_ad *sa; + + if ((!ptr) || (!offset)) { + pr_err("%s: invalidparms\n", __func__); + return NULL; + } + + if ((*offset + sizeof(struct short_ad)) > maxoffset) + return NULL; + else { + sa = (struct short_ad *)ptr; + if (sa->extLength == 0) + return NULL; + } + + if (inc) + *offset += sizeof(struct short_ad); + return sa; +} + +struct long_ad *udf_get_filelongad(uint8_t *ptr, int maxoffset, uint32_t *offset, int inc) +{ + struct long_ad *la; + + if ((!ptr) || (!offset)) { + pr_err("%s: invalidparms\n", __func__); + return NULL; + } + + if ((*offset + sizeof(struct long_ad)) > maxoffset) + return NULL; + else { + la = (struct long_ad *)ptr; + if (la->extLength == 0) + return NULL; + } + + if (inc) + *offset += sizeof(struct long_ad); + return la; +} diff --git a/fs/udf/ecma_167.h b/fs/udf/ecma_167.h new file mode 100644 index 000000000..415b050b9 --- /dev/null +++ b/fs/udf/ecma_167.h @@ -0,0 +1,816 @@ +/* + * ecma_167.h + * + * This file is based on ECMA-167 3rd edition (June 1997) + * https://www.ecma.ch + * + * Copyright (c) 2001-2002 Ben Fennema + * Copyright (c) 2017-2019 Pali Rohár + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions, and the following disclaimer, + * without modification. + * 2. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * Alternatively, this software may be distributed under the terms of the + * GNU Public License ("GPL"). + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/** + * @file + * ECMA-167r3 defines and structure definitions + */ + +#include + +#ifndef _ECMA_167_H +#define _ECMA_167_H 1 + +/* Character sets and coding - d-characters (ECMA 167r3 1/7.2) */ +typedef uint8_t dchars; + +/* Character set specification (ECMA 167r3 1/7.2.1) */ +struct charspec { + uint8_t charSetType; + uint8_t charSetInfo[63]; +} __packed; + +/* Character Set Type (ECMA 167r3 1/7.2.1.1) */ +#define CHARSPEC_TYPE_CS0 0x00 /* (1/7.2.2) */ +#define CHARSPEC_TYPE_CS1 0x01 /* (1/7.2.3) */ +#define CHARSPEC_TYPE_CS2 0x02 /* (1/7.2.4) */ +#define CHARSPEC_TYPE_CS3 0x03 /* (1/7.2.5) */ +#define CHARSPEC_TYPE_CS4 0x04 /* (1/7.2.6) */ +#define CHARSPEC_TYPE_CS5 0x05 /* (1/7.2.7) */ +#define CHARSPEC_TYPE_CS6 0x06 /* (1/7.2.8) */ +#define CHARSPEC_TYPE_CS7 0x07 /* (1/7.2.9) */ +#define CHARSPEC_TYPE_CS8 0x08 /* (1/7.2.10) */ + +/* Fixed-length character fields - d-string (EMCA 167r3 1/7.2.12) */ +typedef uint8_t dstring; + +/* Timestamp (ECMA 167r3 1/7.3) */ +struct timestamp { + __le16 typeAndTimezone; + __le16 year; + uint8_t month; + uint8_t day; + uint8_t hour; + uint8_t minute; + uint8_t second; + uint8_t centiseconds; + uint8_t hundredsOfMicroseconds; + uint8_t microseconds; +} __packed; + +/* Type and Time Zone (ECMA 167r3 1/7.3.1) */ +#define TIMESTAMP_TYPE_MASK 0xF000 +#define TIMESTAMP_TYPE_CUT 0x0000 +#define TIMESTAMP_TYPE_LOCAL 0x1000 +#define TIMESTAMP_TYPE_AGREEMENT 0x2000 +#define TIMESTAMP_TIMEZONE_MASK 0x0FFF + +/* Entity identifier (ECMA 167r3 1/7.4) */ +struct regid { + uint8_t flags; + uint8_t ident[23]; + uint8_t identSuffix[8]; +} __packed; + +/* Flags (ECMA 167r3 1/7.4.1) */ +#define ENTITYID_FLAGS_DIRTY 0x01 +#define ENTITYID_FLAGS_PROTECTED 0x02 + +/* Volume Structure Descriptor (ECMA 167r3 2/9.1) */ +#define VSD_STD_ID_LEN 5 +struct volStructDesc { + uint8_t structType; + uint8_t stdIdent[VSD_STD_ID_LEN]; + uint8_t structVersion; + uint8_t structData[2041]; +} __packed; + +/* Standard Identifier (EMCA 167r2 2/9.1.2) */ +#define VSD_STD_ID_NSR02 "NSR02" /* (3/9.1) */ + +/* Standard Identifier (ECMA 167r3 2/9.1.2) */ +#define VSD_STD_ID_BEA01 "BEA01" /* (2/9.2) */ +#define VSD_STD_ID_BOOT2 "BOOT2" /* (2/9.4) */ +#define VSD_STD_ID_CD001 "CD001" /* (ECMA-119) */ +#define VSD_STD_ID_CDW02 "CDW02" /* (ECMA-168) */ +#define VSD_STD_ID_NSR03 "NSR03" /* (3/9.1) */ +#define VSD_STD_ID_TEA01 "TEA01" /* (2/9.3) */ + +/* Beginning Extended Area Descriptor (ECMA 167r3 2/9.2) */ +struct beginningExtendedAreaDesc { + uint8_t structType; + uint8_t stdIdent[VSD_STD_ID_LEN]; + uint8_t structVersion; + uint8_t structData[2041]; +} __packed; + +/* Terminating Extended Area Descriptor (ECMA 167r3 2/9.3) */ +struct terminatingExtendedAreaDesc { + uint8_t structType; + uint8_t stdIdent[VSD_STD_ID_LEN]; + uint8_t structVersion; + uint8_t structData[2041]; +} __packed; + +/* Boot Descriptor (ECMA 167r3 2/9.4) */ +struct bootDesc { + uint8_t structType; + uint8_t stdIdent[VSD_STD_ID_LEN]; + uint8_t structVersion; + uint8_t reserved1; + struct regid archType; + struct regid bootIdent; + __le32 bootExtLocation; + __le32 bootExtLength; + __le64 loadAddress; + __le64 startAddress; + struct timestamp descCreationDateAndTime; + __le16 flags; + uint8_t reserved2[32]; + uint8_t bootUse[1906]; +} __packed; + +/* Flags (ECMA 167r3 2/9.4.12) */ +#define BOOT_FLAGS_ERASE 0x01 + +/* Extent Descriptor (ECMA 167r3 3/7.1) */ +struct extent_ad { + __le32 extLength; + __le32 extLocation; +} __packed; + +struct kernel_extent_ad { + uint32_t extLength; + uint32_t extLocation; +}; + +/* Descriptor Tag (ECMA 167r3 3/7.2) */ +struct tag { + __le16 tagIdent; + __le16 descVersion; + uint8_t tagChecksum; + uint8_t reserved; + __le16 tagSerialNum; + __le16 descCRC; + __le16 descCRCLength; + __le32 tagLocation; +} __packed; + +/* Tag Identifier (ECMA 167r3 3/7.2.1) */ +#define TAG_IDENT_PVD 0x0001 +#define TAG_IDENT_AVDP 0x0002 +#define TAG_IDENT_VDP 0x0003 +#define TAG_IDENT_IUVD 0x0004 +#define TAG_IDENT_PD 0x0005 +#define TAG_IDENT_LVD 0x0006 +#define TAG_IDENT_USD 0x0007 +#define TAG_IDENT_TD 0x0008 +#define TAG_IDENT_LVID 0x0009 + +/* NSR Descriptor (ECMA 167r3 3/9.1) */ +struct NSRDesc { + uint8_t structType; + uint8_t stdIdent[VSD_STD_ID_LEN]; + uint8_t structVersion; + uint8_t reserved; + uint8_t structData[2040]; +} __packed; + +/* Generic Descriptor */ +struct genericDesc { + struct tag descTag; + __le32 volDescSeqNum; + uint8_t reserved[492]; +} __packed; + +/* Primary Volume Descriptor (ECMA 167r3 3/10.1) */ +struct primaryVolDesc { + struct tag descTag; + __le32 volDescSeqNum; + __le32 primaryVolDescNum; + dstring volIdent[32]; + __le16 volSeqNum; + __le16 maxVolSeqNum; + __le16 interchangeLvl; + __le16 maxInterchangeLvl; + __le32 charSetList; + __le32 maxCharSetList; + dstring volSetIdent[128]; + struct charspec descCharSet; + struct charspec explanatoryCharSet; + struct extent_ad volAbstract; + struct extent_ad volCopyright; + struct regid appIdent; + struct timestamp recordingDateAndTime; + struct regid impIdent; + uint8_t impUse[64]; + __le32 predecessorVolDescSeqLocation; + __le16 flags; + uint8_t reserved[22]; +} __packed; + +/* Flags (ECMA 167r3 3/10.1.21) */ +#define PVD_FLAGS_VSID_COMMON 0x0001 + +/* Anchor Volume Descriptor Pointer (ECMA 167r3 3/10.2) */ +struct anchorVolDescPtr { + struct tag descTag; + struct extent_ad mainVolDescSeqExt; + struct extent_ad reserveVolDescSeqExt; + uint8_t reserved[480]; +} __packed; + +/* Volume Descriptor Pointer (ECMA 167r3 3/10.3) */ +struct volDescPtr { + struct tag descTag; + __le32 volDescSeqNum; + struct extent_ad nextVolDescSeqExt; + uint8_t reserved[484]; +} __packed; + +/* Implementation Use Volume Descriptor (ECMA 167r3 3/10.4) */ +struct impUseVolDesc { + struct tag descTag; + __le32 volDescSeqNum; + struct regid impIdent; + uint8_t impUse[460]; +} __packed; + +/* Partition Descriptor (ECMA 167r3 3/10.5) */ +struct partitionDesc { + struct tag descTag; + __le32 volDescSeqNum; + __le16 partitionFlags; + __le16 partitionNumber; + struct regid partitionContents; + uint8_t partitionContentsUse[128]; + __le32 accessType; + __le32 partitionStartingLocation; + __le32 partitionLength; + struct regid impIdent; + uint8_t impUse[128]; + uint8_t reserved[156]; +} __packed; + +/* Partition Flags (ECMA 167r3 3/10.5.3) */ +#define PD_PARTITION_FLAGS_ALLOC 0x0001 + +/* Partition Contents (ECMA 167r2 3/10.5.3) */ +#define PD_PARTITION_CONTENTS_NSR02 "+NSR02" + +/* Partition Contents (ECMA 167r3 3/10.5.5) */ +#define PD_PARTITION_CONTENTS_FDC01 "+FDC01" +#define PD_PARTITION_CONTENTS_CD001 "+CD001" +#define PD_PARTITION_CONTENTS_CDW02 "+CDW02" +#define PD_PARTITION_CONTENTS_NSR03 "+NSR03" + +/* Access Type (ECMA 167r3 3/10.5.7) */ +#define PD_ACCESS_TYPE_NONE 0x00000000 +#define PD_ACCESS_TYPE_READ_ONLY 0x00000001 +#define PD_ACCESS_TYPE_WRITE_ONCE 0x00000002 +#define PD_ACCESS_TYPE_REWRITABLE 0x00000003 +#define PD_ACCESS_TYPE_OVERWRITABLE 0x00000004 + +/* Logical Volume Descriptor (ECMA 167r3 3/10.6) */ +struct logicalVolDesc { + struct tag descTag; + __le32 volDescSeqNum; + struct charspec descCharSet; + dstring logicalVolIdent[128]; + __le32 logicalBlockSize; + struct regid domainIdent; + uint8_t logicalVolContentsUse[16]; + __le32 mapTableLength; + __le32 numPartitionMaps; + struct regid impIdent; + uint8_t impUse[128]; + struct extent_ad integritySeqExt; + uint8_t partitionMaps[]; +} __packed; + +/* Generic Partition Map (ECMA 167r3 3/10.7.1) */ +struct genericPartitionMap { + uint8_t partitionMapType; + uint8_t partitionMapLength; + uint8_t partitionMapping[]; +} __packed; + +/* Partition Map Type (ECMA 167r3 3/10.7.1.1) */ +#define GP_PARTITION_MAP_TYPE_UNDEF 0x00 +#define GP_PARTITION_MAP_TYPE_1 0x01 +#define GP_PARTITION_MAP_TYPE_2 0x02 + +/* Type 1 Partition Map (ECMA 167r3 3/10.7.2) */ +struct genericPartitionMap1 { + uint8_t partitionMapType; + uint8_t partitionMapLength; + __le16 volSeqNum; + __le16 partitionNum; +} __packed; + +/* Type 2 Partition Map (ECMA 167r3 3/10.7.3) */ +struct genericPartitionMap2 { + uint8_t partitionMapType; + uint8_t partitionMapLength; + uint8_t partitionIdent[62]; +} __packed; + +/* Unallocated Space Descriptor (ECMA 167r3 3/10.8) */ +struct unallocSpaceDesc { + struct tag descTag; + __le32 volDescSeqNum; + __le32 numAllocDescs; + struct extent_ad allocDescs[]; +} __packed; + +/* Terminating Descriptor (ECMA 167r3 3/10.9) */ +struct terminatingDesc { + struct tag descTag; + uint8_t reserved[496]; +} __packed; + +/* Logical Volume Integrity Descriptor (ECMA 167r3 3/10.10) */ +struct logicalVolIntegrityDesc { + struct tag descTag; + struct timestamp recordingDateAndTime; + __le32 integrityType; + struct extent_ad nextIntegrityExt; + uint8_t logicalVolContentsUse[32]; + __le32 numOfPartitions; + __le32 lengthOfImpUse; + __le32 freeSpaceTable[]; + /* __le32 sizeTable[]; */ + /* uint8_t impUse[]; */ +} __packed; + +/* Integrity Type (ECMA 167r3 3/10.10.3) */ +#define LVID_INTEGRITY_TYPE_OPEN 0x00000000 +#define LVID_INTEGRITY_TYPE_CLOSE 0x00000001 + +/* Recorded Address (ECMA 167r3 4/7.1) */ +struct lb_addr { + __le32 logicalBlockNum; + __le16 partitionReferenceNum; +} __packed; + +/* ... and its in-core analog */ +struct kernel_lb_addr { + uint32_t logicalBlockNum; + uint16_t partitionReferenceNum; +}; + +/* Short Allocation Descriptor (ECMA 167r3 4/14.14.1) */ +struct short_ad { + __le32 extLength; + __le32 extPosition; +} __packed; + +/* Long Allocation Descriptor (ECMA 167r3 4/14.14.2) */ +struct long_ad { + __le32 extLength; + struct lb_addr extLocation; + uint8_t impUse[6]; +} __packed; + +struct kernel_long_ad { + uint32_t extLength; + struct kernel_lb_addr extLocation; + uint8_t impUse[6]; +}; + +/* Extended Allocation Descriptor (ECMA 167r3 4/14.14.3) */ +struct ext_ad { + __le32 extLength; + __le32 recordedLength; + __le32 informationLength; + struct lb_addr extLocation; +} __packed; + +struct kernel_ext_ad { + uint32_t extLength; + uint32_t recordedLength; + uint32_t informationLength; + struct kernel_lb_addr extLocation; +}; + +/* Descriptor Tag (ECMA 167r3 4/7.2 - See 3/7.2) */ + +/* Tag Identifier (ECMA 167r3 4/7.2.1) */ +#define TAG_IDENT_FSD 0x0100 +#define TAG_IDENT_FID 0x0101 +#define TAG_IDENT_AED 0x0102 +#define TAG_IDENT_IE 0x0103 +#define TAG_IDENT_TE 0x0104 +#define TAG_IDENT_FE 0x0105 +#define TAG_IDENT_EAHD 0x0106 +#define TAG_IDENT_USE 0x0107 +#define TAG_IDENT_SBD 0x0108 +#define TAG_IDENT_PIE 0x0109 +#define TAG_IDENT_EFE 0x010A + +/* File Set Descriptor (ECMA 167r3 4/14.1) */ +struct fileSetDesc { + struct tag descTag; + struct timestamp recordingDateAndTime; + __le16 interchangeLvl; + __le16 maxInterchangeLvl; + __le32 charSetList; + __le32 maxCharSetList; + __le32 fileSetNum; + __le32 fileSetDescNum; + struct charspec logicalVolIdentCharSet; + dstring logicalVolIdent[128]; + struct charspec fileSetCharSet; + dstring fileSetIdent[32]; + dstring copyrightFileIdent[32]; + dstring abstractFileIdent[32]; + struct long_ad rootDirectoryICB; + struct regid domainIdent; + struct long_ad nextExt; + struct long_ad streamDirectoryICB; + uint8_t reserved[32]; +} __packed; + +/* Partition Header Descriptor (ECMA 167r3 4/14.3) */ +struct partitionHeaderDesc { + struct short_ad unallocSpaceTable; + struct short_ad unallocSpaceBitmap; + struct short_ad partitionIntegrityTable; + struct short_ad freedSpaceTable; + struct short_ad freedSpaceBitmap; + uint8_t reserved[88]; +} __packed; + +/* File Identifier Descriptor (ECMA 167r3 4/14.4) */ +struct fileIdentDesc { + struct tag descTag; + __le16 fileVersionNum; + uint8_t fileCharacteristics; + uint8_t lengthFileIdent; + struct long_ad icb; + __le16 lengthOfImpUse; + /* uint8_t impUse[]; */ + /* uint8_t fileIdent[]; */ + /* uint8_t padding[]; */ +} __packed; + +/* File Characteristics (ECMA 167r3 4/14.4.3) */ +#define FID_FILE_CHAR_HIDDEN 0x01 +#define FID_FILE_CHAR_DIRECTORY 0x02 +#define FID_FILE_CHAR_DELETED 0x04 +#define FID_FILE_CHAR_PARENT 0x08 +#define FID_FILE_CHAR_METADATA 0x10 + +/* Allocation Ext Descriptor (ECMA 167r3 4/14.5) */ +struct allocExtDesc { + struct tag descTag; + __le32 previousAllocExtLocation; + __le32 lengthAllocDescs; +} __packed; + +/* ICB Tag (ECMA 167r3 4/14.6) */ +struct icbtag { + __le32 priorRecordedNumDirectEntries; + __le16 strategyType; + __le16 strategyParameter; + __le16 numEntries; + uint8_t reserved; + uint8_t fileType; + struct lb_addr parentICBLocation; + __le16 flags; +} __packed; + +/* Strategy Type (ECMA 167r3 4/14.6.2) */ +#define ICBTAG_STRATEGY_TYPE_UNDEF 0x0000 +#define ICBTAG_STRATEGY_TYPE_1 0x0001 +#define ICBTAG_STRATEGY_TYPE_2 0x0002 +#define ICBTAG_STRATEGY_TYPE_3 0x0003 +#define ICBTAG_STRATEGY_TYPE_4 0x0004 + +/* File Type (ECMA 167r3 4/14.6.6) */ +#define ICBTAG_FILE_TYPE_UNDEF 0x00 +#define ICBTAG_FILE_TYPE_USE 0x01 +#define ICBTAG_FILE_TYPE_PIE 0x02 +#define ICBTAG_FILE_TYPE_IE 0x03 +#define ICBTAG_FILE_TYPE_DIRECTORY 0x04 +#define ICBTAG_FILE_TYPE_REGULAR 0x05 +#define ICBTAG_FILE_TYPE_BLOCK 0x06 +#define ICBTAG_FILE_TYPE_CHAR 0x07 +#define ICBTAG_FILE_TYPE_EA 0x08 +#define ICBTAG_FILE_TYPE_FIFO 0x09 +#define ICBTAG_FILE_TYPE_SOCKET 0x0A +#define ICBTAG_FILE_TYPE_TE 0x0B +#define ICBTAG_FILE_TYPE_SYMLINK 0x0C +#define ICBTAG_FILE_TYPE_STREAMDIR 0x0D + +/* Flags (ECMA 167r3 4/14.6.8) */ +#define ICBTAG_FLAG_AD_MASK 0x0007 +#define ICBTAG_FLAG_AD_SHORT 0x0000 +#define ICBTAG_FLAG_AD_LONG 0x0001 +#define ICBTAG_FLAG_AD_EXTENDED 0x0002 +#define ICBTAG_FLAG_AD_IN_ICB 0x0003 +#define ICBTAG_FLAG_SORTED 0x0008 +#define ICBTAG_FLAG_NONRELOCATABLE 0x0010 +#define ICBTAG_FLAG_ARCHIVE 0x0020 +#define ICBTAG_FLAG_SETUID 0x0040 +#define ICBTAG_FLAG_SETGID 0x0080 +#define ICBTAG_FLAG_STICKY 0x0100 +#define ICBTAG_FLAG_CONTIGUOUS 0x0200 +#define ICBTAG_FLAG_SYSTEM 0x0400 +#define ICBTAG_FLAG_TRANSFORMED 0x0800 +#define ICBTAG_FLAG_MULTIVERSIONS 0x1000 +#define ICBTAG_FLAG_STREAM 0x2000 + +/* Indirect Entry (ECMA 167r3 4/14.7) */ +struct indirectEntry { + struct tag descTag; + struct icbtag icbTag; + struct long_ad indirectICB; +} __packed; + +/* Terminal Entry (ECMA 167r3 4/14.8) */ +struct terminalEntry { + struct tag descTag; + struct icbtag icbTag; +} __packed; + +/* File Entry (ECMA 167r3 4/14.9) */ +struct fileEntry { + struct tag descTag; + struct icbtag icbTag; + __le32 uid; + __le32 gid; + __le32 permissions; + __le16 fileLinkCount; + uint8_t recordFormat; + uint8_t recordDisplayAttr; + __le32 recordLength; + __le64 informationLength; + __le64 logicalBlocksRecorded; + struct timestamp accessTime; + struct timestamp modificationTime; + struct timestamp attrTime; + __le32 checkpoint; + struct long_ad extendedAttrICB; + struct regid impIdent; + __le64 uniqueID; + __le32 lengthExtendedAttr; + __le32 lengthAllocDescs; + uint8_t extendedAttr[]; + /* uint8_t allocDescs[]; */ +} __packed; + +/* Permissions (ECMA 167r3 4/14.9.5) */ +#define FE_PERM_O_EXEC 0x00000001U +#define FE_PERM_O_WRITE 0x00000002U +#define FE_PERM_O_READ 0x00000004U +#define FE_PERM_O_CHATTR 0x00000008U +#define FE_PERM_O_DELETE 0x00000010U +#define FE_PERM_G_EXEC 0x00000020U +#define FE_PERM_G_WRITE 0x00000040U +#define FE_PERM_G_READ 0x00000080U +#define FE_PERM_G_CHATTR 0x00000100U +#define FE_PERM_G_DELETE 0x00000200U +#define FE_PERM_U_EXEC 0x00000400U +#define FE_PERM_U_WRITE 0x00000800U +#define FE_PERM_U_READ 0x00001000U +#define FE_PERM_U_CHATTR 0x00002000U +#define FE_PERM_U_DELETE 0x00004000U + +/* Record Format (ECMA 167r3 4/14.9.7) */ +#define FE_RECORD_FMT_UNDEF 0x00 +#define FE_RECORD_FMT_FIXED_PAD 0x01 +#define FE_RECORD_FMT_FIXED 0x02 +#define FE_RECORD_FMT_VARIABLE8 0x03 +#define FE_RECORD_FMT_VARIABLE16 0x04 +#define FE_RECORD_FMT_VARIABLE16_MSB 0x05 +#define FE_RECORD_FMT_VARIABLE32 0x06 +#define FE_RECORD_FMT_PRINT 0x07 +#define FE_RECORD_FMT_LF 0x08 +#define FE_RECORD_FMT_CR 0x09 +#define FE_RECORD_FMT_CRLF 0x0A +#define FE_RECORD_FMT_LFCR 0x0B + +/* Record Display Attributes (ECMA 167r3 4/14.9.8) */ +#define FE_RECORD_DISPLAY_ATTR_UNDEF 0x00 +#define FE_RECORD_DISPLAY_ATTR_1 0x01 +#define FE_RECORD_DISPLAY_ATTR_2 0x02 +#define FE_RECORD_DISPLAY_ATTR_3 0x03 + +/* Extended Attribute Header Descriptor (ECMA 167r3 4/14.10.1) */ +struct extendedAttrHeaderDesc { + struct tag descTag; + __le32 impAttrLocation; + __le32 appAttrLocation; +} __packed; + +/* Generic Format (ECMA 167r3 4/14.10.2) */ +struct genericFormat { + __le32 attrType; + uint8_t attrSubtype; + uint8_t reserved[3]; + __le32 attrLength; + uint8_t attrData[]; +} __packed; + +/* Character Set Information (ECMA 167r3 4/14.10.3) */ +struct charSetInfo { + __le32 attrType; + uint8_t attrSubtype; + uint8_t reserved[3]; + __le32 attrLength; + __le32 escapeSeqLength; + uint8_t charSetType; + uint8_t escapeSeq[]; +} __packed; + +/* Alternate Permissions (ECMA 167r3 4/14.10.4) */ +struct altPerms { + __le32 attrType; + uint8_t attrSubtype; + uint8_t reserved[3]; + __le32 attrLength; + __le16 ownerIdent; + __le16 groupIdent; + __le16 permission; +} __packed; + +/* File Times Extended Attribute (ECMA 167r3 4/14.10.5) */ +struct fileTimesExtAttr { + __le32 attrType; + uint8_t attrSubtype; + uint8_t reserved[3]; + __le32 attrLength; + __le32 dataLength; + __le32 fileTimeExistence; + uint8_t fileTimes; +} __packed; + +/* FileTimeExistence (ECMA 167r3 4/14.10.5.6) */ +#define FTE_CREATION 0x00000001 +#define FTE_DELETION 0x00000004 +#define FTE_EFFECTIVE 0x00000008 +#define FTE_BACKUP 0x00000002 + +/* Information Times Extended Attribute (ECMA 167r3 4/14.10.6) */ +struct infoTimesExtAttr { + __le32 attrType; + uint8_t attrSubtype; + uint8_t reserved[3]; + __le32 attrLength; + __le32 dataLength; + __le32 infoTimeExistence; + uint8_t infoTimes[]; +} __packed; + +/* Device Specification (ECMA 167r3 4/14.10.7) */ +struct deviceSpec { + __le32 attrType; + uint8_t attrSubtype; + uint8_t reserved[3]; + __le32 attrLength; + __le32 impUseLength; + __le32 majorDeviceIdent; + __le32 minorDeviceIdent; + uint8_t impUse[]; +} __packed; + +/* Implementation Use Extended Attr (ECMA 167r3 4/14.10.8) */ +struct impUseExtAttr { + __le32 attrType; + uint8_t attrSubtype; + uint8_t reserved[3]; + __le32 attrLength; + __le32 impUseLength; + struct regid impIdent; + uint8_t impUse[]; +} __packed; + +/* Application Use Extended Attribute (ECMA 167r3 4/14.10.9) */ +struct appUseExtAttr { + __le32 attrType; + uint8_t attrSubtype; + uint8_t reserved[3]; + __le32 attrLength; + __le32 appUseLength; + struct regid appIdent; + uint8_t appUse[]; +} __packed; + +#define EXTATTR_CHAR_SET 1 +#define EXTATTR_ALT_PERMS 3 +#define EXTATTR_FILE_TIMES 5 +#define EXTATTR_INFO_TIMES 6 +#define EXTATTR_DEV_SPEC 12 +#define EXTATTR_IMP_USE 2048 +#define EXTATTR_APP_USE 65536 +#define EXTATTR_SUBTYPE 1 + +/* Unallocated Space Entry (ECMA 167r3 4/14.11) */ +struct unallocSpaceEntry { + struct tag descTag; + struct icbtag icbTag; + __le32 lengthAllocDescs; + uint8_t allocDescs[]; +} __packed; + +/* Space Bitmap Descriptor (ECMA 167r3 4/14.12) */ +struct spaceBitmapDesc { + struct tag descTag; + __le32 numOfBits; + __le32 numOfBytes; + uint8_t bitmap[]; +} __packed; + +/* Partition Integrity Entry (ECMA 167r3 4/14.13) */ +struct partitionIntegrityEntry { + struct tag descTag; + struct icbtag icbTag; + struct timestamp recordingDateAndTime; + uint8_t integrityType; + uint8_t reserved[175]; + struct regid impIdent; + uint8_t impUse[256]; +} __packed; + +/* Short Allocation Descriptor (ECMA 167r3 4/14.14.1) */ + +/* Extent Length (ECMA 167r3 4/14.14.1.1) */ +#define EXT_LENGTH_MASK 0x3FFFFFFF +#define EXT_TYPE_MASK 0xC0000000 +#define EXT_RECORDED_ALLOCATED 0x00000000 +#define EXT_NOT_RECORDED_ALLOCATED 0x40000000 +#define EXT_NOT_RECORDED_NOT_ALLOCATED 0x80000000 +#define EXT_NEXT_EXTENT_ALLOCDESCS 0xC0000000 + +/* Long Allocation Descriptor (ECMA 167r3 4/14.14.2) */ + +/* Extended Allocation Descriptor (ECMA 167r3 4/14.14.3) */ + +/* Logical Volume Header Descriptor (ECMA 167r3 4/14.15) */ +struct logicalVolHeaderDesc { + __le64 uniqueID; + uint8_t reserved[24]; +} __packed; + +/* Path Component (ECMA 167r3 4/14.16.1) */ +struct pathComponent { + uint8_t componentType; + uint8_t lengthComponentIdent; + __le16 componentFileVersionNum; + dchars componentIdent[]; +} __packed; + +/* File Entry (ECMA 167r3 4/14.17) */ +struct extendedFileEntry { + struct tag descTag; + struct icbtag icbTag; + __le32 uid; + __le32 gid; + __le32 permissions; + __le16 fileLinkCount; + uint8_t recordFormat; + uint8_t recordDisplayAttr; + __le32 recordLength; + __le64 informationLength; + __le64 objectSize; + __le64 logicalBlocksRecorded; + struct timestamp accessTime; + struct timestamp modificationTime; + struct timestamp createTime; + struct timestamp attrTime; + __le32 checkpoint; + __le32 reserved; + struct long_ad extendedAttrICB; + struct long_ad streamDirectoryICB; + struct regid impIdent; + __le64 uniqueID; + __le32 lengthExtendedAttr; + __le32 lengthAllocDescs; + uint8_t extendedAttr[]; + /* uint8_t allocDescs[]; */ +} __packed; + +#endif /* _ECMA_167_H */ diff --git a/fs/udf/file.c b/fs/udf/file.c new file mode 100644 index 000000000..57d11606a --- /dev/null +++ b/fs/udf/file.c @@ -0,0 +1,256 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * file.c + * + * PURPOSE + * File handling routines for the OSTA-UDF(tm) filesystem. + * + * COPYRIGHT + * (C) 1998-1999 Dave Boynton + * (C) 1998-2004 Ben Fennema + * (C) 1999-2000 Stelias Computing Inc + * + * HISTORY + * + * 10/02/98 dgb Attempt to integrate into udf.o + * 10/07/98 Switched to using generic_readpage, etc., like isofs + * And it works! + * 12/06/98 blf Added udf_file_read. uses generic_file_read for all cases but + * ICBTAG_FLAG_AD_IN_ICB. + * 04/06/99 64 bit file handling on 32 bit systems taken from ext2 file.c + * 05/12/99 Preliminary file write support + */ + +#include "udfdecl.h" +#include +#include +#include +#include /* memset */ +#include +#include +#include +#include +#include + +#include "udf_i.h" +#include "udf_sb.h" + +static vm_fault_t udf_page_mkwrite(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + struct inode *inode = file_inode(vma->vm_file); + struct address_space *mapping = inode->i_mapping; + struct folio *folio = page_folio(vmf->page); + loff_t size; + unsigned int end; + vm_fault_t ret = VM_FAULT_LOCKED; + int err; + + sb_start_pagefault(inode->i_sb); + file_update_time(vma->vm_file); + filemap_invalidate_lock_shared(mapping); + folio_lock(folio); + size = i_size_read(inode); + if (folio->mapping != inode->i_mapping || folio_pos(folio) >= size) { + folio_unlock(folio); + ret = VM_FAULT_NOPAGE; + goto out_unlock; + } + /* Space is already allocated for in-ICB file */ + if (UDF_I(inode)->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) + goto out_dirty; + if (folio->index == size >> PAGE_SHIFT) + end = size & ~PAGE_MASK; + else + end = PAGE_SIZE; + err = __block_write_begin(folio, 0, end, udf_get_block); + if (err) { + folio_unlock(folio); + ret = vmf_fs_error(err); + goto out_unlock; + } + + block_commit_write(folio, 0, end); +out_dirty: + folio_mark_dirty(folio); + folio_wait_stable(folio); +out_unlock: + filemap_invalidate_unlock_shared(mapping); + sb_end_pagefault(inode->i_sb); + return ret; +} + +static const struct vm_operations_struct udf_file_vm_ops = { + .fault = filemap_fault, + .map_pages = filemap_map_pages, + .page_mkwrite = udf_page_mkwrite, +}; + +static ssize_t udf_file_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + ssize_t retval; + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct udf_inode_info *iinfo = UDF_I(inode); + + inode_lock(inode); + + retval = generic_write_checks(iocb, from); + if (retval <= 0) + goto out; + + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB && + inode->i_sb->s_blocksize < (udf_file_entry_alloc_offset(inode) + + iocb->ki_pos + iov_iter_count(from))) { + filemap_invalidate_lock(inode->i_mapping); + retval = udf_expand_file_adinicb(inode); + filemap_invalidate_unlock(inode->i_mapping); + if (retval) + goto out; + } + + retval = __generic_file_write_iter(iocb, from); +out: + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB && retval > 0) { + down_write(&iinfo->i_data_sem); + iinfo->i_lenAlloc = inode->i_size; + up_write(&iinfo->i_data_sem); + } + inode_unlock(inode); + + if (retval > 0) { + mark_inode_dirty(inode); + retval = generic_write_sync(iocb, retval); + } + + return retval; +} + +long udf_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct inode *inode = file_inode(filp); + long old_block, new_block; + int result; + + if (file_permission(filp, MAY_READ) != 0) { + udf_debug("no permission to access inode %llu\n", inode->i_ino); + return -EPERM; + } + + if (!arg && ((cmd == UDF_GETVOLIDENT) || (cmd == UDF_GETEASIZE) || + (cmd == UDF_RELOCATE_BLOCKS) || (cmd == UDF_GETEABLOCK))) { + udf_debug("invalid argument to udf_ioctl\n"); + return -EINVAL; + } + + switch (cmd) { + case UDF_GETVOLIDENT: + if (copy_to_user((char __user *)arg, + UDF_SB(inode->i_sb)->s_volume_ident, 32)) + return -EFAULT; + return 0; + case UDF_RELOCATE_BLOCKS: + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (get_user(old_block, (long __user *)arg)) + return -EFAULT; + result = udf_relocate_blocks(inode->i_sb, + old_block, &new_block); + if (result == 0) + result = put_user(new_block, (long __user *)arg); + return result; + case UDF_GETEASIZE: + return put_user(UDF_I(inode)->i_lenEAttr, (int __user *)arg); + case UDF_GETEABLOCK: + return copy_to_user((char __user *)arg, + UDF_I(inode)->i_data, + UDF_I(inode)->i_lenEAttr) ? -EFAULT : 0; + default: + return -ENOIOCTLCMD; + } + + return 0; +} + +static int udf_release_file(struct inode *inode, struct file *filp) +{ + if (filp->f_mode & FMODE_WRITE && + atomic_read(&inode->i_writecount) == 1) { + /* + * Grab i_mutex to avoid races with writes changing i_size + * while we are running. + */ + inode_lock(inode); + down_write(&UDF_I(inode)->i_data_sem); + udf_discard_prealloc(inode); + udf_truncate_tail_extent(inode); + up_write(&UDF_I(inode)->i_data_sem); + inode_unlock(inode); + } + return 0; +} + +static int udf_file_mmap(struct file *file, struct vm_area_struct *vma) +{ + file_accessed(file); + vma->vm_ops = &udf_file_vm_ops; + + return 0; +} + +const struct file_operations udf_file_operations = { + .read_iter = generic_file_read_iter, + .unlocked_ioctl = udf_ioctl, + .open = generic_file_open, + .mmap = udf_file_mmap, + .write_iter = udf_file_write_iter, + .release = udf_release_file, + .fsync = simple_fsync, + .splice_read = filemap_splice_read, + .splice_write = iter_file_splice_write, + .llseek = generic_file_llseek, + .setlease = generic_setlease, +}; + +static int udf_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +{ + struct inode *inode = d_inode(dentry); + struct super_block *sb = inode->i_sb; + int error; + + error = setattr_prepare(&nop_mnt_idmap, dentry, attr); + if (error) + return error; + + if ((attr->ia_valid & ATTR_UID) && + UDF_QUERY_FLAG(sb, UDF_FLAG_UID_SET) && + !uid_eq(attr->ia_uid, UDF_SB(sb)->s_uid)) + return -EPERM; + if ((attr->ia_valid & ATTR_GID) && + UDF_QUERY_FLAG(sb, UDF_FLAG_GID_SET) && + !gid_eq(attr->ia_gid, UDF_SB(sb)->s_gid)) + return -EPERM; + + if ((attr->ia_valid & ATTR_SIZE) && + attr->ia_size != i_size_read(inode)) { + filemap_invalidate_lock(inode->i_mapping); + error = udf_setsize(inode, attr->ia_size); + filemap_invalidate_unlock(inode->i_mapping); + if (error) + return error; + } + + if (attr->ia_valid & ATTR_MODE) + udf_update_extra_perms(inode, attr->ia_mode); + + setattr_copy(&nop_mnt_idmap, inode, attr); + mark_inode_dirty(inode); + if (IS_SYNC(inode)) + sync_inode_metadata(inode, 1); + return 0; +} + +const struct inode_operations udf_file_inode_operations = { + .setattr = udf_setattr, +}; diff --git a/fs/udf/ialloc.c b/fs/udf/ialloc.c new file mode 100644 index 000000000..5f1f969f4 --- /dev/null +++ b/fs/udf/ialloc.c @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * ialloc.c + * + * PURPOSE + * Inode allocation handling routines for the OSTA-UDF(tm) filesystem. + * + * COPYRIGHT + * (C) 1998-2001 Ben Fennema + * + * HISTORY + * + * 02/24/99 blf Created. + * + */ + +#include "udfdecl.h" +#include +#include +#include + +#include "udf_i.h" +#include "udf_sb.h" + +void udf_free_inode(struct inode *inode) +{ + udf_free_blocks(inode->i_sb, NULL, &UDF_I(inode)->i_location, 0, 1); +} + +struct inode *udf_new_inode(struct inode *dir, umode_t mode) +{ + struct super_block *sb = dir->i_sb; + struct udf_sb_info *sbi = UDF_SB(sb); + struct inode *inode; + udf_pblk_t block; + uint32_t start = UDF_I(dir)->i_location.logicalBlockNum; + struct udf_inode_info *iinfo; + struct udf_inode_info *dinfo = UDF_I(dir); + int err; + + inode = new_inode(sb); + + if (!inode) + return ERR_PTR(-ENOMEM); + + iinfo = UDF_I(inode); + if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_USE_EXTENDED_FE)) { + iinfo->i_efe = 1; + if (UDF_VERS_USE_EXTENDED_FE > sbi->s_udfrev) + sbi->s_udfrev = UDF_VERS_USE_EXTENDED_FE; + iinfo->i_data = kzalloc(inode->i_sb->s_blocksize - + sizeof(struct extendedFileEntry), + GFP_KERNEL); + } else { + iinfo->i_efe = 0; + iinfo->i_data = kzalloc(inode->i_sb->s_blocksize - + sizeof(struct fileEntry), + GFP_KERNEL); + } + if (!iinfo->i_data) { + make_bad_inode(inode); + iput(inode); + return ERR_PTR(-ENOMEM); + } + + err = -ENOSPC; + block = udf_new_block(dir->i_sb, NULL, + dinfo->i_location.partitionReferenceNum, + start, &err); + if (err) { + make_bad_inode(inode); + iput(inode); + return ERR_PTR(err); + } + + iinfo->i_unique = lvid_get_unique_id(sb); + inode->i_generation = iinfo->i_unique; + + inode_init_owner(&nop_mnt_idmap, inode, dir, mode); + if (UDF_QUERY_FLAG(sb, UDF_FLAG_UID_SET)) + inode->i_uid = sbi->s_uid; + if (UDF_QUERY_FLAG(sb, UDF_FLAG_GID_SET)) + inode->i_gid = sbi->s_gid; + + iinfo->i_location.logicalBlockNum = block; + iinfo->i_location.partitionReferenceNum = + dinfo->i_location.partitionReferenceNum; + inode->i_ino = udf_get_lb_pblock(sb, &iinfo->i_location, 0); + inode->i_blocks = 0; + iinfo->i_lenEAttr = 0; + iinfo->i_lenAlloc = 0; + iinfo->i_use = 0; + iinfo->i_checkpoint = 1; + iinfo->i_extraPerms = FE_PERM_U_CHATTR; + udf_update_extra_perms(inode, mode); + + if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_USE_AD_IN_ICB)) + iinfo->i_alloc_type = ICBTAG_FLAG_AD_IN_ICB; + else if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_USE_SHORT_AD)) + iinfo->i_alloc_type = ICBTAG_FLAG_AD_SHORT; + else + iinfo->i_alloc_type = ICBTAG_FLAG_AD_LONG; + simple_inode_init_ts(inode); + iinfo->i_crtime = inode_get_mtime(inode); + if (unlikely(insert_inode_locked(inode) < 0)) { + make_bad_inode(inode); + iput(inode); + return ERR_PTR(-EIO); + } + mark_inode_dirty(inode); + + return inode; +} diff --git a/fs/udf/inode.c b/fs/udf/inode.c new file mode 100644 index 000000000..e45e546a7 --- /dev/null +++ b/fs/udf/inode.c @@ -0,0 +1,2493 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * inode.c + * + * PURPOSE + * Inode handling routines for the OSTA-UDF(tm) filesystem. + * + * COPYRIGHT + * (C) 1998 Dave Boynton + * (C) 1998-2004 Ben Fennema + * (C) 1999-2000 Stelias Computing Inc + * + * HISTORY + * + * 10/04/98 dgb Added rudimentary directory functions + * 10/07/98 Fully working udf_block_map! It works! + * 11/25/98 bmap altered to better support extents + * 12/06/98 blf partition support in udf_iget, udf_block_map + * and udf_read_inode + * 12/12/98 rewrote udf_block_map to handle next extents and descs across + * block boundaries (which is not actually allowed) + * 12/20/98 added support for strategy 4096 + * 03/07/99 rewrote udf_block_map (again) + * New funcs, inode_bmap, udf_next_aext + * 04/19/99 Support for writing device EA's for major/minor # + */ + +#include "udfdecl.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "udf_i.h" +#include "udf_sb.h" + +#define EXTENT_MERGE_SIZE 5 + +#define FE_MAPPED_PERMS (FE_PERM_U_READ | FE_PERM_U_WRITE | FE_PERM_U_EXEC | \ + FE_PERM_G_READ | FE_PERM_G_WRITE | FE_PERM_G_EXEC | \ + FE_PERM_O_READ | FE_PERM_O_WRITE | FE_PERM_O_EXEC) + +#define FE_DELETE_PERMS (FE_PERM_U_DELETE | FE_PERM_G_DELETE | \ + FE_PERM_O_DELETE) + +struct udf_map_rq; + +static umode_t udf_convert_permissions(struct fileEntry *); +static int udf_alloc_i_data(struct inode *inode, size_t size); +static int inode_getblk(struct inode *inode, struct udf_map_rq *map); +static int udf_insert_aext(struct inode *, struct extent_position, + struct kernel_lb_addr, uint32_t); +static void udf_split_extents(struct inode *, int *, int, udf_pblk_t, + struct kernel_long_ad *, int *); +static void udf_prealloc_extents(struct inode *, int, int, + struct kernel_long_ad *, int *); +static void udf_merge_extents(struct inode *, struct kernel_long_ad *, int *); +static int udf_update_extents(struct inode *, struct kernel_long_ad *, int, + int, struct extent_position *); +static int udf_get_block_wb(struct inode *inode, sector_t block, + struct buffer_head *bh_result, int create); + +static void __udf_clear_extent_cache(struct inode *inode) +{ + struct udf_inode_info *iinfo = UDF_I(inode); + + if (iinfo->cached_extent.lstart != -1) { + brelse(iinfo->cached_extent.epos.bh); + iinfo->cached_extent.lstart = -1; + } +} + +/* Invalidate extent cache */ +static void udf_clear_extent_cache(struct inode *inode) +{ + struct udf_inode_info *iinfo = UDF_I(inode); + + spin_lock(&iinfo->i_extent_cache_lock); + __udf_clear_extent_cache(inode); + spin_unlock(&iinfo->i_extent_cache_lock); +} + +/* Return contents of extent cache */ +static int udf_read_extent_cache(struct inode *inode, loff_t bcount, + loff_t *lbcount, struct extent_position *pos) +{ + struct udf_inode_info *iinfo = UDF_I(inode); + int ret = 0; + + spin_lock(&iinfo->i_extent_cache_lock); + if ((iinfo->cached_extent.lstart <= bcount) && + (iinfo->cached_extent.lstart != -1)) { + /* Cache hit */ + *lbcount = iinfo->cached_extent.lstart; + memcpy(pos, &iinfo->cached_extent.epos, + sizeof(struct extent_position)); + if (pos->bh) + get_bh(pos->bh); + ret = 1; + } + spin_unlock(&iinfo->i_extent_cache_lock); + return ret; +} + +/* Add extent to extent cache */ +static void udf_update_extent_cache(struct inode *inode, loff_t estart, + struct extent_position *pos) +{ + struct udf_inode_info *iinfo = UDF_I(inode); + + spin_lock(&iinfo->i_extent_cache_lock); + /* Invalidate previously cached extent */ + __udf_clear_extent_cache(inode); + if (pos->bh) + get_bh(pos->bh); + memcpy(&iinfo->cached_extent.epos, pos, sizeof(*pos)); + iinfo->cached_extent.lstart = estart; + switch (iinfo->i_alloc_type) { + case ICBTAG_FLAG_AD_SHORT: + iinfo->cached_extent.epos.offset -= sizeof(struct short_ad); + break; + case ICBTAG_FLAG_AD_LONG: + iinfo->cached_extent.epos.offset -= sizeof(struct long_ad); + break; + } + spin_unlock(&iinfo->i_extent_cache_lock); +} + +void udf_evict_inode(struct inode *inode) +{ + struct udf_inode_info *iinfo = UDF_I(inode); + int want_delete = 0; + + if (!is_bad_inode(inode)) { + if (!inode->i_nlink) { + want_delete = 1; + udf_setsize(inode, 0); + sync_inode_metadata(inode, IS_SYNC(inode)); + } + if (iinfo->i_alloc_type != ICBTAG_FLAG_AD_IN_ICB && + inode->i_size != iinfo->i_lenExtents) { + udf_warn(inode->i_sb, + "Inode %llu (mode %o) has inode size %llu different from extent length %llu. Filesystem need not be standards compliant.\n", + inode->i_ino, inode->i_mode, + (unsigned long long)inode->i_size, + (unsigned long long)iinfo->i_lenExtents); + } + } + truncate_inode_pages_final(&inode->i_data); + if (!want_delete) + mmb_sync(&iinfo->i_metadata_bhs); + mmb_invalidate(&iinfo->i_metadata_bhs); + clear_inode(inode); + kfree(iinfo->i_data); + iinfo->i_data = NULL; + udf_clear_extent_cache(inode); + if (want_delete) { + udf_free_inode(inode); + } +} + +static void udf_write_failed(struct address_space *mapping, loff_t to) +{ + struct inode *inode = mapping->host; + struct udf_inode_info *iinfo = UDF_I(inode); + loff_t isize = inode->i_size; + + if (to > isize) { + truncate_pagecache(inode, isize); + if (iinfo->i_alloc_type != ICBTAG_FLAG_AD_IN_ICB) { + down_write(&iinfo->i_data_sem); + udf_clear_extent_cache(inode); + udf_truncate_extents(inode); + up_write(&iinfo->i_data_sem); + } + } +} + +static int udf_handle_page_wb(struct folio *folio, + struct writeback_control *wbc) +{ + struct inode *inode = folio->mapping->host; + struct udf_inode_info *iinfo = UDF_I(inode); + + /* + * Inodes in the normal format are handled by the generic code. This + * check is race-free as the folio lock protects us from inode type + * conversion. + */ + if (iinfo->i_alloc_type != ICBTAG_FLAG_AD_IN_ICB) + return 1; + + memcpy_from_file_folio(iinfo->i_data + iinfo->i_lenEAttr, folio, + 0, i_size_read(inode)); + folio_unlock(folio); + mark_inode_dirty(inode); + return 0; +} + +static int udf_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + return __mpage_writepages(mapping, wbc, udf_get_block_wb, + udf_handle_page_wb); +} + +static void udf_adinicb_read_folio(struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + struct udf_inode_info *iinfo = UDF_I(inode); + loff_t isize = i_size_read(inode); + + folio_fill_tail(folio, 0, iinfo->i_data + iinfo->i_lenEAttr, isize); + folio_mark_uptodate(folio); +} + +static int udf_read_folio(struct file *file, struct folio *folio) +{ + struct udf_inode_info *iinfo = UDF_I(file_inode(file)); + + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { + udf_adinicb_read_folio(folio); + folio_unlock(folio); + return 0; + } + return mpage_read_folio(folio, udf_get_block); +} + +static void udf_readahead(struct readahead_control *rac) +{ + struct udf_inode_info *iinfo = UDF_I(rac->mapping->host); + + /* + * No readahead needed for in-ICB files and udf_get_block() would get + * confused for such file anyway. + */ + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) + return; + + mpage_readahead(rac, udf_get_block); +} + +static int udf_write_begin(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, + struct folio **foliop, void **fsdata) +{ + struct file *file = iocb->ki_filp; + struct udf_inode_info *iinfo = UDF_I(file_inode(file)); + struct folio *folio; + int ret; + + if (iinfo->i_alloc_type != ICBTAG_FLAG_AD_IN_ICB) { + ret = block_write_begin(mapping, pos, len, foliop, + udf_get_block); + if (unlikely(ret)) + udf_write_failed(mapping, pos + len); + return ret; + } + if (WARN_ON_ONCE(pos >= PAGE_SIZE)) + return -EIO; + folio = __filemap_get_folio(mapping, 0, FGP_WRITEBEGIN, + mapping_gfp_mask(mapping)); + if (IS_ERR(folio)) + return PTR_ERR(folio); + *foliop = folio; + if (!folio_test_uptodate(folio)) + udf_adinicb_read_folio(folio); + return 0; +} + +static int udf_write_end(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct folio *folio, void *fsdata) +{ + struct inode *inode = file_inode(iocb->ki_filp); + loff_t last_pos; + + if (UDF_I(inode)->i_alloc_type != ICBTAG_FLAG_AD_IN_ICB) + return generic_write_end(iocb, mapping, pos, len, copied, folio, + fsdata); + last_pos = pos + copied; + if (last_pos > inode->i_size) + i_size_write(inode, last_pos); + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + + return copied; +} + +static ssize_t udf_direct_IO(struct kiocb *iocb, struct iov_iter *iter) +{ + struct file *file = iocb->ki_filp; + struct address_space *mapping = file->f_mapping; + struct inode *inode = mapping->host; + size_t count = iov_iter_count(iter); + ssize_t ret; + + /* Fallback to buffered IO for in-ICB files */ + if (UDF_I(inode)->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) + return 0; + ret = blockdev_direct_IO(iocb, inode, iter, udf_get_block); + if (unlikely(ret < 0 && iov_iter_rw(iter) == WRITE)) + udf_write_failed(mapping, iocb->ki_pos + count); + return ret; +} + +static sector_t udf_bmap(struct address_space *mapping, sector_t block) +{ + struct udf_inode_info *iinfo = UDF_I(mapping->host); + + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) + return -EINVAL; + return generic_block_bmap(mapping, block, udf_get_block); +} + +const struct address_space_operations udf_aops = { + .dirty_folio = block_dirty_folio, + .invalidate_folio = block_invalidate_folio, + .read_folio = udf_read_folio, + .readahead = udf_readahead, + .writepages = udf_writepages, + .write_begin = udf_write_begin, + .write_end = udf_write_end, + .direct_IO = udf_direct_IO, + .bmap = udf_bmap, + .migrate_folio = buffer_migrate_folio, +}; + +#define UDF_MAP_CREATE 0x01 /* Mapping can allocate new blocks */ +#define UDF_MAP_NOPREALLOC 0x02 /* Do not preallocate blocks */ + +#define UDF_BLK_MAPPED 0x01 /* Block was successfully mapped */ +#define UDF_BLK_NEW 0x02 /* Block was freshly allocated */ + +struct udf_map_rq { + sector_t lblk; + udf_pblk_t pblk; + int iflags; /* UDF_MAP_ flags determining behavior */ + int oflags; /* UDF_BLK_ flags reporting results */ +}; + +static int udf_map_block(struct inode *inode, struct udf_map_rq *map) +{ + int ret; + struct udf_inode_info *iinfo = UDF_I(inode); + + if (WARN_ON_ONCE(iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB)) + return -EFSCORRUPTED; + + map->oflags = 0; + if (!(map->iflags & UDF_MAP_CREATE)) { + struct kernel_lb_addr eloc; + uint32_t elen; + sector_t offset; + struct extent_position epos = {}; + int8_t etype; + + down_read(&iinfo->i_data_sem); + ret = inode_bmap(inode, map->lblk, &epos, &eloc, &elen, &offset, + &etype); + if (ret < 0) + goto out_read; + if (ret > 0 && etype == (EXT_RECORDED_ALLOCATED >> 30)) { + map->pblk = udf_get_lb_pblock(inode->i_sb, &eloc, + offset); + map->oflags |= UDF_BLK_MAPPED; + ret = 0; + } +out_read: + up_read(&iinfo->i_data_sem); + brelse(epos.bh); + + return ret; + } + + down_write(&iinfo->i_data_sem); + /* + * Block beyond EOF and prealloc extents? Just discard preallocation + * as it is not useful and complicates things. + */ + if (((loff_t)map->lblk) << inode->i_blkbits >= iinfo->i_lenExtents) + udf_discard_prealloc(inode); + udf_clear_extent_cache(inode); + ret = inode_getblk(inode, map); + up_write(&iinfo->i_data_sem); + return ret; +} + +/* + * Expand file stored in ICB to a normal one-block-file + * + * This function requires i_mutex held + */ +int udf_expand_file_adinicb(struct inode *inode) +{ + struct folio *folio; + struct udf_inode_info *iinfo = UDF_I(inode); + struct udf_map_rq map = { + .lblk = 0, + .iflags = UDF_MAP_CREATE, + }; + int err; + + WARN_ON_ONCE(!inode_is_locked(inode)); + if (!iinfo->i_lenAlloc) { + down_write(&iinfo->i_data_sem); + if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_USE_SHORT_AD)) + iinfo->i_alloc_type = ICBTAG_FLAG_AD_SHORT; + else + iinfo->i_alloc_type = ICBTAG_FLAG_AD_LONG; + up_write(&iinfo->i_data_sem); + mark_inode_dirty(inode); + return 0; + } + + folio = __filemap_get_folio(inode->i_mapping, 0, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, GFP_KERNEL); + if (IS_ERR(folio)) + return PTR_ERR(folio); + + if (!folio_test_uptodate(folio)) + udf_adinicb_read_folio(folio); + down_write(&iinfo->i_data_sem); + memset(iinfo->i_data + iinfo->i_lenEAttr, 0x00, + iinfo->i_lenAlloc); + iinfo->i_lenAlloc = 0; + if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_USE_SHORT_AD)) + iinfo->i_alloc_type = ICBTAG_FLAG_AD_SHORT; + else + iinfo->i_alloc_type = ICBTAG_FLAG_AD_LONG; + up_write(&iinfo->i_data_sem); + + /* Allocate the block underlying the data */ + err = udf_map_block(inode, &map); + if (err < 0) + goto restore; + + folio_mark_dirty(folio); + folio_unlock(folio); + err = filemap_fdatawrite(inode->i_mapping); + if (err) { + /* Restore everything back so that we don't lose data... */ + folio_lock(folio); +restore: + down_write(&iinfo->i_data_sem); + memcpy_from_folio(iinfo->i_data + iinfo->i_lenEAttr, + folio, 0, inode->i_size); + iinfo->i_alloc_type = ICBTAG_FLAG_AD_IN_ICB; + iinfo->i_lenAlloc = inode->i_size; + up_write(&iinfo->i_data_sem); + folio_unlock(folio); + } + folio_put(folio); + mark_inode_dirty(inode); + + return err; +} + +static int __udf_get_block(struct inode *inode, sector_t block, + struct buffer_head *bh_result, int flags) +{ + int err; + struct udf_map_rq map = { + .lblk = block, + .iflags = flags, + }; + + err = udf_map_block(inode, &map); + if (err < 0) + return err; + if (map.oflags & UDF_BLK_MAPPED) { + map_bh(bh_result, inode->i_sb, map.pblk); + if (map.oflags & UDF_BLK_NEW) + set_buffer_new(bh_result); + } + return 0; +} + +int udf_get_block(struct inode *inode, sector_t block, + struct buffer_head *bh_result, int create) +{ + int flags = create ? UDF_MAP_CREATE : 0; + + /* + * We preallocate blocks only for regular files. It also makes sense + * for directories but there's a problem when to drop the + * preallocation. We might use some delayed work for that but I feel + * it's overengineering for a filesystem like UDF. + */ + if (!S_ISREG(inode->i_mode)) + flags |= UDF_MAP_NOPREALLOC; + return __udf_get_block(inode, block, bh_result, flags); +} + +/* + * We shouldn't be allocating blocks on page writeback since we allocate them + * on page fault. We can spot dirty buffers without allocated blocks though + * when truncate expands file. These however don't have valid data so we can + * safely ignore them. So never allocate blocks from page writeback. + */ +static int udf_get_block_wb(struct inode *inode, sector_t block, + struct buffer_head *bh_result, int create) +{ + return __udf_get_block(inode, block, bh_result, 0); +} + +/* Extend the file with new blocks totaling 'new_block_bytes', + * return the number of extents added + */ +static int udf_do_extend_file(struct inode *inode, + struct extent_position *last_pos, + struct kernel_long_ad *last_ext, + loff_t new_block_bytes) +{ + uint32_t add; + int count = 0, fake = !(last_ext->extLength & UDF_EXTENT_LENGTH_MASK); + struct super_block *sb = inode->i_sb; + struct udf_inode_info *iinfo; + int err; + + /* The previous extent is fake and we should not extend by anything + * - there's nothing to do... */ + if (!new_block_bytes && fake) + return 0; + + iinfo = UDF_I(inode); + /* Round the last extent up to a multiple of block size */ + if (last_ext->extLength & (sb->s_blocksize - 1)) { + last_ext->extLength = + (last_ext->extLength & UDF_EXTENT_FLAG_MASK) | + (((last_ext->extLength & UDF_EXTENT_LENGTH_MASK) + + sb->s_blocksize - 1) & ~(sb->s_blocksize - 1)); + iinfo->i_lenExtents = + (iinfo->i_lenExtents + sb->s_blocksize - 1) & + ~((u64)sb->s_blocksize - 1); + } + + add = 0; + /* Can we merge with the previous extent? */ + if ((last_ext->extLength & UDF_EXTENT_FLAG_MASK) == + EXT_NOT_RECORDED_NOT_ALLOCATED) { + add = (1 << 30) - sb->s_blocksize - + (last_ext->extLength & UDF_EXTENT_LENGTH_MASK); + if (add > new_block_bytes) + add = new_block_bytes; + new_block_bytes -= add; + last_ext->extLength += add; + } + + if (fake) { + err = udf_add_aext(inode, last_pos, &last_ext->extLocation, + last_ext->extLength, 1); + if (err < 0) + goto out_err; + count++; + } else { + struct kernel_lb_addr tmploc; + uint32_t tmplen; + int8_t tmptype; + + udf_write_aext(inode, last_pos, &last_ext->extLocation, + last_ext->extLength, 1); + + /* + * We've rewritten the last extent. If we are going to add + * more extents, we may need to enter possible following + * empty indirect extent. + */ + if (new_block_bytes) { + err = udf_next_aext(inode, last_pos, &tmploc, &tmplen, + &tmptype, 0); + if (err < 0) + goto out_err; + } + } + iinfo->i_lenExtents += add; + + /* Managed to do everything necessary? */ + if (!new_block_bytes) + goto out; + + /* All further extents will be NOT_RECORDED_NOT_ALLOCATED */ + last_ext->extLocation.logicalBlockNum = 0; + last_ext->extLocation.partitionReferenceNum = 0; + add = (1 << 30) - sb->s_blocksize; + last_ext->extLength = EXT_NOT_RECORDED_NOT_ALLOCATED | add; + + /* Create enough extents to cover the whole hole */ + while (new_block_bytes > add) { + new_block_bytes -= add; + err = udf_add_aext(inode, last_pos, &last_ext->extLocation, + last_ext->extLength, 1); + if (err) + goto out_err; + iinfo->i_lenExtents += add; + count++; + } + if (new_block_bytes) { + last_ext->extLength = EXT_NOT_RECORDED_NOT_ALLOCATED | + new_block_bytes; + err = udf_add_aext(inode, last_pos, &last_ext->extLocation, + last_ext->extLength, 1); + if (err) + goto out_err; + iinfo->i_lenExtents += new_block_bytes; + count++; + } + +out: + /* last_pos should point to the last written extent... */ + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_SHORT) + last_pos->offset -= sizeof(struct short_ad); + else if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_LONG) + last_pos->offset -= sizeof(struct long_ad); + else + return -EIO; + + return count; +out_err: + /* Remove extents we've created so far */ + udf_clear_extent_cache(inode); + udf_truncate_extents(inode); + return err; +} + +/* Extend the final block of the file to final_block_len bytes */ +static void udf_do_extend_final_block(struct inode *inode, + struct extent_position *last_pos, + struct kernel_long_ad *last_ext, + uint32_t new_elen) +{ + uint32_t added_bytes; + + /* + * Extent already large enough? It may be already rounded up to block + * size... + */ + if (new_elen <= (last_ext->extLength & UDF_EXTENT_LENGTH_MASK)) + return; + added_bytes = new_elen - (last_ext->extLength & UDF_EXTENT_LENGTH_MASK); + last_ext->extLength += added_bytes; + UDF_I(inode)->i_lenExtents += added_bytes; + + udf_write_aext(inode, last_pos, &last_ext->extLocation, + last_ext->extLength, 1); +} + +static int udf_extend_file(struct inode *inode, loff_t newsize) +{ + + struct extent_position epos; + struct kernel_lb_addr eloc; + uint32_t elen; + int8_t etype; + struct super_block *sb = inode->i_sb; + sector_t first_block = newsize >> sb->s_blocksize_bits, offset; + loff_t new_elen; + int adsize; + struct udf_inode_info *iinfo = UDF_I(inode); + struct kernel_long_ad extent; + int err = 0; + bool within_last_ext; + + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_SHORT) + adsize = sizeof(struct short_ad); + else if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_LONG) + adsize = sizeof(struct long_ad); + else + BUG(); + + down_write(&iinfo->i_data_sem); + /* + * When creating hole in file, just don't bother with preserving + * preallocation. It likely won't be very useful anyway. + */ + udf_discard_prealloc(inode); + + err = inode_bmap(inode, first_block, &epos, &eloc, &elen, &offset, &etype); + if (err < 0) + goto out; + within_last_ext = (err == 1); + /* We don't expect extents past EOF... */ + WARN_ON_ONCE(within_last_ext && + elen > ((loff_t)offset + 1) << inode->i_blkbits); + + if ((!epos.bh && epos.offset == udf_file_entry_alloc_offset(inode)) || + (epos.bh && epos.offset == sizeof(struct allocExtDesc))) { + /* File has no extents at all or has empty last + * indirect extent! Create a fake extent... */ + extent.extLocation.logicalBlockNum = 0; + extent.extLocation.partitionReferenceNum = 0; + extent.extLength = EXT_NOT_RECORDED_NOT_ALLOCATED; + } else { + epos.offset -= adsize; + err = udf_next_aext(inode, &epos, &extent.extLocation, + &extent.extLength, &etype, 0); + if (err <= 0) + goto out; + extent.extLength |= etype << 30; + } + + new_elen = ((loff_t)offset << inode->i_blkbits) | + (newsize & (sb->s_blocksize - 1)); + + /* File has extent covering the new size (could happen when extending + * inside a block)? + */ + if (within_last_ext) { + /* Extending file within the last file block */ + udf_do_extend_final_block(inode, &epos, &extent, new_elen); + } else { + err = udf_do_extend_file(inode, &epos, &extent, new_elen); + } + + if (err < 0) + goto out; + err = 0; +out: + brelse(epos.bh); + up_write(&iinfo->i_data_sem); + return err; +} + +static int inode_getblk(struct inode *inode, struct udf_map_rq *map) +{ + struct kernel_long_ad laarr[EXTENT_MERGE_SIZE]; + struct extent_position prev_epos, cur_epos, next_epos; + int count = 0, startnum = 0, endnum = 0; + uint32_t elen = 0, tmpelen; + struct kernel_lb_addr eloc, tmpeloc; + int c = 1; + loff_t lbcount = 0, b_off = 0; + udf_pblk_t newblocknum; + sector_t offset = 0; + int8_t etype, tmpetype; + struct udf_inode_info *iinfo = UDF_I(inode); + udf_pblk_t goal = 0, pgoal = 0; + int lastblock = 0; + bool isBeyondEOF = false; + int ret = 0; + + prev_epos.offset = udf_file_entry_alloc_offset(inode); + prev_epos.block = iinfo->i_location; + prev_epos.bh = NULL; + cur_epos = next_epos = prev_epos; + b_off = (loff_t)map->lblk << inode->i_sb->s_blocksize_bits; + + /* find the extent which contains the block we are looking for. + alternate between laarr[0] and laarr[1] for locations of the + current extent, and the previous extent */ + do { + if (prev_epos.bh != cur_epos.bh) { + brelse(prev_epos.bh); + get_bh(cur_epos.bh); + prev_epos.bh = cur_epos.bh; + } + if (cur_epos.bh != next_epos.bh) { + brelse(cur_epos.bh); + get_bh(next_epos.bh); + cur_epos.bh = next_epos.bh; + } + + lbcount += elen; + + prev_epos.block = cur_epos.block; + cur_epos.block = next_epos.block; + + prev_epos.offset = cur_epos.offset; + cur_epos.offset = next_epos.offset; + + ret = udf_next_aext(inode, &next_epos, &eloc, &elen, &etype, 1); + if (ret < 0) { + goto out_free; + } else if (ret == 0) { + isBeyondEOF = true; + break; + } + + c = !c; + + laarr[c].extLength = (etype << 30) | elen; + laarr[c].extLocation = eloc; + + if (etype != (EXT_NOT_RECORDED_NOT_ALLOCATED >> 30)) + pgoal = eloc.logicalBlockNum + + ((elen + inode->i_sb->s_blocksize - 1) >> + inode->i_sb->s_blocksize_bits); + + count++; + } while (lbcount + elen <= b_off); + + b_off -= lbcount; + offset = b_off >> inode->i_sb->s_blocksize_bits; + /* + * Move prev_epos and cur_epos into indirect extent if we are at + * the pointer to it + */ + ret = udf_next_aext(inode, &prev_epos, &tmpeloc, &tmpelen, &tmpetype, 0); + if (ret < 0) + goto out_free; + ret = udf_next_aext(inode, &cur_epos, &tmpeloc, &tmpelen, &tmpetype, 0); + if (ret < 0) + goto out_free; + + /* if the extent is allocated and recorded, return the block + if the extent is not a multiple of the blocksize, round up */ + + if (!isBeyondEOF && etype == (EXT_RECORDED_ALLOCATED >> 30)) { + if (elen & (inode->i_sb->s_blocksize - 1)) { + elen = EXT_RECORDED_ALLOCATED | + ((elen + inode->i_sb->s_blocksize - 1) & + ~(inode->i_sb->s_blocksize - 1)); + iinfo->i_lenExtents = + ALIGN(iinfo->i_lenExtents, + inode->i_sb->s_blocksize); + udf_write_aext(inode, &cur_epos, &eloc, elen, 1); + } + map->oflags = UDF_BLK_MAPPED; + map->pblk = udf_get_lb_pblock(inode->i_sb, &eloc, offset); + ret = 0; + goto out_free; + } + + /* Are we beyond EOF and preallocated extent? */ + if (isBeyondEOF) { + loff_t hole_len; + + if (count) { + if (c) + laarr[0] = laarr[1]; + startnum = 1; + } else { + /* Create a fake extent when there's not one */ + memset(&laarr[0].extLocation, 0x00, + sizeof(struct kernel_lb_addr)); + laarr[0].extLength = EXT_NOT_RECORDED_NOT_ALLOCATED; + /* Will udf_do_extend_file() create real extent from + a fake one? */ + startnum = (offset > 0); + } + /* Create extents for the hole between EOF and offset */ + hole_len = (loff_t)offset << inode->i_blkbits; + ret = udf_do_extend_file(inode, &prev_epos, laarr, hole_len); + if (ret < 0) + goto out_free; + c = 0; + offset = 0; + count += ret; + /* + * Is there any real extent? - otherwise we overwrite the fake + * one... + */ + if (count) + c = !c; + laarr[c].extLength = EXT_NOT_RECORDED_NOT_ALLOCATED | + inode->i_sb->s_blocksize; + memset(&laarr[c].extLocation, 0x00, + sizeof(struct kernel_lb_addr)); + count++; + endnum = c + 1; + lastblock = 1; + } else { + endnum = startnum = ((count > 2) ? 2 : count); + + /* if the current extent is in position 0, + swap it with the previous */ + if (!c && count != 1) { + laarr[2] = laarr[0]; + laarr[0] = laarr[1]; + laarr[1] = laarr[2]; + c = 1; + } + + /* if the current block is located in an extent, + read the next extent */ + ret = udf_next_aext(inode, &next_epos, &eloc, &elen, &etype, 0); + if (ret > 0) { + laarr[c + 1].extLength = (etype << 30) | elen; + laarr[c + 1].extLocation = eloc; + count++; + startnum++; + endnum++; + } else if (ret == 0) + lastblock = 1; + else + goto out_free; + } + + /* if the current extent is not recorded but allocated, get the + * block in the extent corresponding to the requested block */ + if ((laarr[c].extLength >> 30) == (EXT_NOT_RECORDED_ALLOCATED >> 30)) + newblocknum = laarr[c].extLocation.logicalBlockNum + offset; + else { /* otherwise, allocate a new block */ + if (iinfo->i_next_alloc_block == map->lblk) + goal = iinfo->i_next_alloc_goal; + if (!goal) + goal = pgoal; + if (!goal) + goal = iinfo->i_location.logicalBlockNum + 1; + + newblocknum = udf_new_block(inode->i_sb, inode, + iinfo->i_location.partitionReferenceNum, + goal, &ret); + if (!newblocknum) + goto out_free; + if (isBeyondEOF) + iinfo->i_lenExtents += inode->i_sb->s_blocksize; + } + + /* if the extent the requsted block is located in contains multiple + * blocks, split the extent into at most three extents. blocks prior + * to requested block, requested block, and blocks after requested + * block */ + udf_split_extents(inode, &c, offset, newblocknum, laarr, &endnum); + + if (!(map->iflags & UDF_MAP_NOPREALLOC)) + udf_prealloc_extents(inode, c, lastblock, laarr, &endnum); + + /* merge any continuous blocks in laarr */ + udf_merge_extents(inode, laarr, &endnum); + + /* write back the new extents, inserting new extents if the new number + * of extents is greater than the old number, and deleting extents if + * the new number of extents is less than the old number */ + ret = udf_update_extents(inode, laarr, startnum, endnum, &prev_epos); + if (ret < 0) + goto out_free; + + map->pblk = udf_get_pblock(inode->i_sb, newblocknum, + iinfo->i_location.partitionReferenceNum, 0); + if (!map->pblk) { + ret = -EFSCORRUPTED; + goto out_free; + } + map->oflags = UDF_BLK_NEW | UDF_BLK_MAPPED; + iinfo->i_next_alloc_block = map->lblk + 1; + iinfo->i_next_alloc_goal = newblocknum + 1; + inode_set_ctime_current(inode); + + mark_inode_dirty(inode); + ret = 0; +out_free: + brelse(prev_epos.bh); + brelse(cur_epos.bh); + brelse(next_epos.bh); + return ret; +} + +static void udf_split_extents(struct inode *inode, int *c, int offset, + udf_pblk_t newblocknum, + struct kernel_long_ad *laarr, int *endnum) +{ + unsigned long blocksize = inode->i_sb->s_blocksize; + unsigned char blocksize_bits = inode->i_sb->s_blocksize_bits; + + if ((laarr[*c].extLength >> 30) == (EXT_NOT_RECORDED_ALLOCATED >> 30) || + (laarr[*c].extLength >> 30) == + (EXT_NOT_RECORDED_NOT_ALLOCATED >> 30)) { + int curr = *c; + int blen = ((laarr[curr].extLength & UDF_EXTENT_LENGTH_MASK) + + blocksize - 1) >> blocksize_bits; + int8_t etype = (laarr[curr].extLength >> 30); + + if (blen == 1) + ; + else if (!offset || blen == offset + 1) { + laarr[curr + 2] = laarr[curr + 1]; + laarr[curr + 1] = laarr[curr]; + } else { + laarr[curr + 3] = laarr[curr + 1]; + laarr[curr + 2] = laarr[curr + 1] = laarr[curr]; + } + + if (offset) { + if (etype == (EXT_NOT_RECORDED_ALLOCATED >> 30)) { + udf_free_blocks(inode->i_sb, inode, + &laarr[curr].extLocation, + 0, offset); + laarr[curr].extLength = + EXT_NOT_RECORDED_NOT_ALLOCATED | + (offset << blocksize_bits); + laarr[curr].extLocation.logicalBlockNum = 0; + laarr[curr].extLocation. + partitionReferenceNum = 0; + } else + laarr[curr].extLength = (etype << 30) | + (offset << blocksize_bits); + curr++; + (*c)++; + (*endnum)++; + } + + laarr[curr].extLocation.logicalBlockNum = newblocknum; + if (etype == (EXT_NOT_RECORDED_NOT_ALLOCATED >> 30)) + laarr[curr].extLocation.partitionReferenceNum = + UDF_I(inode)->i_location.partitionReferenceNum; + laarr[curr].extLength = EXT_RECORDED_ALLOCATED | + blocksize; + curr++; + + if (blen != offset + 1) { + if (etype == (EXT_NOT_RECORDED_ALLOCATED >> 30)) + laarr[curr].extLocation.logicalBlockNum += + offset + 1; + laarr[curr].extLength = (etype << 30) | + ((blen - (offset + 1)) << blocksize_bits); + curr++; + (*endnum)++; + } + } +} + +static void udf_prealloc_extents(struct inode *inode, int c, int lastblock, + struct kernel_long_ad *laarr, + int *endnum) +{ + int start, length = 0, currlength = 0, i; + + if (*endnum >= (c + 1)) { + if (!lastblock) + return; + else + start = c; + } else { + if ((laarr[c + 1].extLength >> 30) == + (EXT_NOT_RECORDED_ALLOCATED >> 30)) { + start = c + 1; + length = currlength = + (((laarr[c + 1].extLength & + UDF_EXTENT_LENGTH_MASK) + + inode->i_sb->s_blocksize - 1) >> + inode->i_sb->s_blocksize_bits); + } else + start = c; + } + + for (i = start + 1; i <= *endnum; i++) { + if (i == *endnum) { + if (lastblock) + length += UDF_DEFAULT_PREALLOC_BLOCKS; + } else if ((laarr[i].extLength >> 30) == + (EXT_NOT_RECORDED_NOT_ALLOCATED >> 30)) { + length += (((laarr[i].extLength & + UDF_EXTENT_LENGTH_MASK) + + inode->i_sb->s_blocksize - 1) >> + inode->i_sb->s_blocksize_bits); + } else + break; + } + + if (length) { + int next = laarr[start].extLocation.logicalBlockNum + + (((laarr[start].extLength & UDF_EXTENT_LENGTH_MASK) + + inode->i_sb->s_blocksize - 1) >> + inode->i_sb->s_blocksize_bits); + int numalloc = udf_prealloc_blocks(inode->i_sb, inode, + laarr[start].extLocation.partitionReferenceNum, + next, (UDF_DEFAULT_PREALLOC_BLOCKS > length ? + length : UDF_DEFAULT_PREALLOC_BLOCKS) - + currlength); + if (numalloc) { + if (start == (c + 1)) + laarr[start].extLength += + (numalloc << + inode->i_sb->s_blocksize_bits); + else { + memmove(&laarr[c + 2], &laarr[c + 1], + sizeof(struct long_ad) * (*endnum - (c + 1))); + (*endnum)++; + laarr[c + 1].extLocation.logicalBlockNum = next; + laarr[c + 1].extLocation.partitionReferenceNum = + laarr[c].extLocation. + partitionReferenceNum; + laarr[c + 1].extLength = + EXT_NOT_RECORDED_ALLOCATED | + (numalloc << + inode->i_sb->s_blocksize_bits); + start = c + 1; + } + + for (i = start + 1; numalloc && i < *endnum; i++) { + int elen = ((laarr[i].extLength & + UDF_EXTENT_LENGTH_MASK) + + inode->i_sb->s_blocksize - 1) >> + inode->i_sb->s_blocksize_bits; + + if (elen > numalloc) { + laarr[i].extLength -= + (numalloc << + inode->i_sb->s_blocksize_bits); + numalloc = 0; + } else { + numalloc -= elen; + if (*endnum > (i + 1)) + memmove(&laarr[i], + &laarr[i + 1], + sizeof(struct long_ad) * + (*endnum - (i + 1))); + i--; + (*endnum)--; + } + } + UDF_I(inode)->i_lenExtents += + numalloc << inode->i_sb->s_blocksize_bits; + } + } +} + +static void udf_merge_extents(struct inode *inode, struct kernel_long_ad *laarr, + int *endnum) +{ + int i; + unsigned long blocksize = inode->i_sb->s_blocksize; + unsigned char blocksize_bits = inode->i_sb->s_blocksize_bits; + + for (i = 0; i < (*endnum - 1); i++) { + struct kernel_long_ad *li /*l[i]*/ = &laarr[i]; + struct kernel_long_ad *lip1 /*l[i plus 1]*/ = &laarr[i + 1]; + + if (((li->extLength >> 30) == (lip1->extLength >> 30)) && + (((li->extLength >> 30) == + (EXT_NOT_RECORDED_NOT_ALLOCATED >> 30)) || + ((lip1->extLocation.logicalBlockNum - + li->extLocation.logicalBlockNum) == + (((li->extLength & UDF_EXTENT_LENGTH_MASK) + + blocksize - 1) >> blocksize_bits)))) { + + if (((li->extLength & UDF_EXTENT_LENGTH_MASK) + + (lip1->extLength & UDF_EXTENT_LENGTH_MASK) + + blocksize - 1) <= UDF_EXTENT_LENGTH_MASK) { + li->extLength = lip1->extLength + + (((li->extLength & + UDF_EXTENT_LENGTH_MASK) + + blocksize - 1) & ~(blocksize - 1)); + if (*endnum > (i + 2)) + memmove(&laarr[i + 1], &laarr[i + 2], + sizeof(struct long_ad) * + (*endnum - (i + 2))); + i--; + (*endnum)--; + } + } else if (((li->extLength >> 30) == + (EXT_NOT_RECORDED_ALLOCATED >> 30)) && + ((lip1->extLength >> 30) == + (EXT_NOT_RECORDED_NOT_ALLOCATED >> 30))) { + udf_free_blocks(inode->i_sb, inode, &li->extLocation, 0, + ((li->extLength & + UDF_EXTENT_LENGTH_MASK) + + blocksize - 1) >> blocksize_bits); + li->extLocation.logicalBlockNum = 0; + li->extLocation.partitionReferenceNum = 0; + + if (((li->extLength & UDF_EXTENT_LENGTH_MASK) + + (lip1->extLength & UDF_EXTENT_LENGTH_MASK) + + blocksize - 1) & ~UDF_EXTENT_LENGTH_MASK) { + lip1->extLength = (lip1->extLength - + (li->extLength & + UDF_EXTENT_LENGTH_MASK) + + UDF_EXTENT_LENGTH_MASK) & + ~(blocksize - 1); + li->extLength = (li->extLength & + UDF_EXTENT_FLAG_MASK) + + (UDF_EXTENT_LENGTH_MASK + 1) - + blocksize; + } else { + li->extLength = lip1->extLength + + (((li->extLength & + UDF_EXTENT_LENGTH_MASK) + + blocksize - 1) & ~(blocksize - 1)); + if (*endnum > (i + 2)) + memmove(&laarr[i + 1], &laarr[i + 2], + sizeof(struct long_ad) * + (*endnum - (i + 2))); + i--; + (*endnum)--; + } + } else if ((li->extLength >> 30) == + (EXT_NOT_RECORDED_ALLOCATED >> 30)) { + udf_free_blocks(inode->i_sb, inode, + &li->extLocation, 0, + ((li->extLength & + UDF_EXTENT_LENGTH_MASK) + + blocksize - 1) >> blocksize_bits); + li->extLocation.logicalBlockNum = 0; + li->extLocation.partitionReferenceNum = 0; + li->extLength = (li->extLength & + UDF_EXTENT_LENGTH_MASK) | + EXT_NOT_RECORDED_NOT_ALLOCATED; + } + } +} + +static int udf_update_extents(struct inode *inode, struct kernel_long_ad *laarr, + int startnum, int endnum, + struct extent_position *epos) +{ + int start = 0, i; + struct kernel_lb_addr tmploc; + uint32_t tmplen; + int8_t tmpetype; + int err; + + if (startnum > endnum) { + for (i = 0; i < (startnum - endnum); i++) + udf_delete_aext(inode, *epos, NULL); + } else if (startnum < endnum) { + for (i = 0; i < (endnum - startnum); i++) { + err = udf_insert_aext(inode, *epos, + laarr[i].extLocation, + laarr[i].extLength); + /* + * If we fail here, we are likely corrupting the extent + * list and leaking blocks. At least stop early to + * limit the damage. + */ + if (err < 0) + return err; + err = udf_next_aext(inode, epos, &laarr[i].extLocation, + &laarr[i].extLength, &tmpetype, 1); + if (err < 0) + return err; + start++; + } + } + + for (i = start; i < endnum; i++) { + err = udf_next_aext(inode, epos, &tmploc, &tmplen, &tmpetype, 0); + if (err < 0) + return err; + + udf_write_aext(inode, epos, &laarr[i].extLocation, + laarr[i].extLength, 1); + } + return 0; +} + +struct buffer_head *udf_bread(struct inode *inode, udf_pblk_t block, + int create, int *err) +{ + struct buffer_head *bh = NULL; + struct udf_map_rq map = { + .lblk = block, + .iflags = UDF_MAP_NOPREALLOC | (create ? UDF_MAP_CREATE : 0), + }; + + *err = udf_map_block(inode, &map); + if (*err || !(map.oflags & UDF_BLK_MAPPED)) + return NULL; + + bh = sb_getblk(inode->i_sb, map.pblk); + if (!bh) { + *err = -ENOMEM; + return NULL; + } + if (map.oflags & UDF_BLK_NEW) { + lock_buffer(bh); + memset(bh->b_data, 0x00, inode->i_sb->s_blocksize); + set_buffer_uptodate(bh); + unlock_buffer(bh); + mmb_mark_buffer_dirty(bh, &UDF_I(inode)->i_metadata_bhs); + return bh; + } + + if (bh_read(bh, 0) >= 0) + return bh; + + brelse(bh); + *err = -EIO; + return NULL; +} + +int udf_setsize(struct inode *inode, loff_t newsize) +{ + int err = 0; + struct udf_inode_info *iinfo; + unsigned int bsize = i_blocksize(inode); + + if (!(S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) || + S_ISLNK(inode->i_mode))) + return -EINVAL; + + iinfo = UDF_I(inode); + if (newsize > inode->i_size) { + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { + if (bsize >= + (udf_file_entry_alloc_offset(inode) + newsize)) { + down_write(&iinfo->i_data_sem); + iinfo->i_lenAlloc = newsize; + up_write(&iinfo->i_data_sem); + goto set_size; + } + err = udf_expand_file_adinicb(inode); + if (err) + return err; + } + err = udf_extend_file(inode, newsize); + if (err) + return err; +set_size: + truncate_setsize(inode, newsize); + } else { + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { + down_write(&iinfo->i_data_sem); + udf_clear_extent_cache(inode); + memset(iinfo->i_data + iinfo->i_lenEAttr + newsize, + 0x00, bsize - newsize - + udf_file_entry_alloc_offset(inode)); + iinfo->i_lenAlloc = newsize; + truncate_setsize(inode, newsize); + up_write(&iinfo->i_data_sem); + goto update_time; + } + err = block_truncate_page(inode->i_mapping, newsize, + udf_get_block); + if (err) + return err; + truncate_setsize(inode, newsize); + down_write(&iinfo->i_data_sem); + udf_clear_extent_cache(inode); + err = udf_truncate_extents(inode); + up_write(&iinfo->i_data_sem); + if (err) + return err; + } +update_time: + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + mark_inode_dirty(inode); + return err; +} + +/* + * Maximum length of linked list formed by ICB hierarchy. The chosen number is + * arbitrary - just that we hopefully don't limit any real use of rewritten + * inode on write-once media but avoid looping for too long on corrupted media. + */ +#define UDF_MAX_ICB_NESTING 1024 + +static int udf_read_inode(struct inode *inode, bool hidden_inode) +{ + struct buffer_head *bh = NULL; + struct fileEntry *fe; + struct extendedFileEntry *efe; + uint16_t ident; + struct udf_inode_info *iinfo = UDF_I(inode); + struct udf_sb_info *sbi = UDF_SB(inode->i_sb); + struct kernel_lb_addr *iloc = &iinfo->i_location; + unsigned int link_count; + unsigned int indirections = 0; + int bs = inode->i_sb->s_blocksize; + int ret = -EIO; + uint32_t uid, gid; + struct timespec64 ts; + +reread: + if (iloc->partitionReferenceNum >= sbi->s_partitions) { + udf_debug("partition reference: %u > logical volume partitions: %u\n", + iloc->partitionReferenceNum, sbi->s_partitions); + return -EIO; + } + + if (iloc->logicalBlockNum >= + sbi->s_partmaps[iloc->partitionReferenceNum].s_partition_len) { + udf_debug("block=%u, partition=%u out of range\n", + iloc->logicalBlockNum, iloc->partitionReferenceNum); + return -EIO; + } + + /* + * Set defaults, but the inode is still incomplete! + * Note: get_new_inode() sets the following on a new inode: + * i_sb = sb + * i_no = ino + * i_flags = sb->s_flags + * i_state = 0 + * clean_inode(): zero fills and sets + * i_count = 1 + * i_nlink = 1 + * i_op = NULL; + */ + bh = udf_read_ptagged(inode->i_sb, iloc, 0, &ident); + if (!bh) { + udf_err(inode->i_sb, "(ino %llu) failed !bh\n", inode->i_ino); + return -EIO; + } + + if (ident != TAG_IDENT_FE && ident != TAG_IDENT_EFE && + ident != TAG_IDENT_USE) { + udf_err(inode->i_sb, "(ino %llu) failed ident=%u\n", + inode->i_ino, ident); + goto out; + } + + fe = (struct fileEntry *)bh->b_data; + efe = (struct extendedFileEntry *)bh->b_data; + + if (fe->icbTag.strategyType == cpu_to_le16(4096)) { + struct buffer_head *ibh; + + ibh = udf_read_ptagged(inode->i_sb, iloc, 1, &ident); + if (ident == TAG_IDENT_IE && ibh) { + struct kernel_lb_addr loc; + struct indirectEntry *ie; + + ie = (struct indirectEntry *)ibh->b_data; + loc = lelb_to_cpu(ie->indirectICB.extLocation); + + if (ie->indirectICB.extLength) { + brelse(ibh); + memcpy(&iinfo->i_location, &loc, + sizeof(struct kernel_lb_addr)); + if (++indirections > UDF_MAX_ICB_NESTING) { + udf_err(inode->i_sb, + "too many ICBs in ICB hierarchy" + " (max %d supported)\n", + UDF_MAX_ICB_NESTING); + goto out; + } + brelse(bh); + goto reread; + } + } + brelse(ibh); + } else if (fe->icbTag.strategyType != cpu_to_le16(4)) { + udf_err(inode->i_sb, "unsupported strategy type: %u\n", + le16_to_cpu(fe->icbTag.strategyType)); + goto out; + } + if (fe->icbTag.strategyType == cpu_to_le16(4)) + iinfo->i_strat4096 = 0; + else /* if (fe->icbTag.strategyType == cpu_to_le16(4096)) */ + iinfo->i_strat4096 = 1; + + iinfo->i_alloc_type = le16_to_cpu(fe->icbTag.flags) & + ICBTAG_FLAG_AD_MASK; + if (iinfo->i_alloc_type != ICBTAG_FLAG_AD_SHORT && + iinfo->i_alloc_type != ICBTAG_FLAG_AD_LONG && + iinfo->i_alloc_type != ICBTAG_FLAG_AD_IN_ICB) { + ret = -EIO; + goto out; + } + iinfo->i_hidden = hidden_inode; + iinfo->i_unique = 0; + iinfo->i_lenEAttr = 0; + iinfo->i_lenExtents = 0; + iinfo->i_lenAlloc = 0; + iinfo->i_next_alloc_block = 0; + iinfo->i_next_alloc_goal = 0; + if (fe->descTag.tagIdent == cpu_to_le16(TAG_IDENT_EFE)) { + iinfo->i_efe = 1; + iinfo->i_use = 0; + ret = udf_alloc_i_data(inode, bs - + sizeof(struct extendedFileEntry)); + if (ret) + goto out; + memcpy(iinfo->i_data, + bh->b_data + sizeof(struct extendedFileEntry), + bs - sizeof(struct extendedFileEntry)); + } else if (fe->descTag.tagIdent == cpu_to_le16(TAG_IDENT_FE)) { + iinfo->i_efe = 0; + iinfo->i_use = 0; + ret = udf_alloc_i_data(inode, bs - sizeof(struct fileEntry)); + if (ret) + goto out; + memcpy(iinfo->i_data, + bh->b_data + sizeof(struct fileEntry), + bs - sizeof(struct fileEntry)); + } else if (fe->descTag.tagIdent == cpu_to_le16(TAG_IDENT_USE)) { + iinfo->i_efe = 0; + iinfo->i_use = 1; + iinfo->i_lenAlloc = le32_to_cpu( + ((struct unallocSpaceEntry *)bh->b_data)-> + lengthAllocDescs); + if (iinfo->i_lenAlloc > bs - sizeof(struct unallocSpaceEntry)) { + ret = -EFSCORRUPTED; + goto out; + } + ret = udf_alloc_i_data(inode, bs - + sizeof(struct unallocSpaceEntry)); + if (ret) + goto out; + memcpy(iinfo->i_data, + bh->b_data + sizeof(struct unallocSpaceEntry), + bs - sizeof(struct unallocSpaceEntry)); + brelse(bh); + return 0; + } + + ret = -EIO; + read_lock(&sbi->s_cred_lock); + uid = le32_to_cpu(fe->uid); + if (uid == UDF_INVALID_ID || + UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_UID_SET)) + inode->i_uid = sbi->s_uid; + else + i_uid_write(inode, uid); + + gid = le32_to_cpu(fe->gid); + if (gid == UDF_INVALID_ID || + UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_GID_SET)) + inode->i_gid = sbi->s_gid; + else + i_gid_write(inode, gid); + + if (fe->icbTag.fileType != ICBTAG_FILE_TYPE_DIRECTORY && + sbi->s_fmode != UDF_INVALID_MODE) + inode->i_mode = sbi->s_fmode; + else if (fe->icbTag.fileType == ICBTAG_FILE_TYPE_DIRECTORY && + sbi->s_dmode != UDF_INVALID_MODE) + inode->i_mode = sbi->s_dmode; + else + inode->i_mode = udf_convert_permissions(fe); + inode->i_mode &= ~sbi->s_umask; + iinfo->i_extraPerms = le32_to_cpu(fe->permissions) & ~FE_MAPPED_PERMS; + + read_unlock(&sbi->s_cred_lock); + + link_count = le16_to_cpu(fe->fileLinkCount); + if (!link_count) { + if (!hidden_inode) { + ret = -ESTALE; + goto out; + } + link_count = 1; + } + set_nlink(inode, link_count); + + inode->i_size = le64_to_cpu(fe->informationLength); + iinfo->i_lenExtents = inode->i_size; + + if (iinfo->i_efe == 0) { + inode->i_blocks = le64_to_cpu(fe->logicalBlocksRecorded) << + (inode->i_sb->s_blocksize_bits - 9); + + udf_disk_stamp_to_time(&ts, fe->accessTime); + inode_set_atime_to_ts(inode, ts); + udf_disk_stamp_to_time(&ts, fe->modificationTime); + inode_set_mtime_to_ts(inode, ts); + udf_disk_stamp_to_time(&ts, fe->attrTime); + inode_set_ctime_to_ts(inode, ts); + + iinfo->i_unique = le64_to_cpu(fe->uniqueID); + iinfo->i_lenEAttr = le32_to_cpu(fe->lengthExtendedAttr); + iinfo->i_lenAlloc = le32_to_cpu(fe->lengthAllocDescs); + iinfo->i_checkpoint = le32_to_cpu(fe->checkpoint); + iinfo->i_streamdir = 0; + iinfo->i_lenStreams = 0; + } else { + inode->i_blocks = le64_to_cpu(efe->logicalBlocksRecorded) << + (inode->i_sb->s_blocksize_bits - 9); + + udf_disk_stamp_to_time(&ts, efe->accessTime); + inode_set_atime_to_ts(inode, ts); + udf_disk_stamp_to_time(&ts, efe->modificationTime); + inode_set_mtime_to_ts(inode, ts); + udf_disk_stamp_to_time(&ts, efe->attrTime); + inode_set_ctime_to_ts(inode, ts); + udf_disk_stamp_to_time(&iinfo->i_crtime, efe->createTime); + + iinfo->i_unique = le64_to_cpu(efe->uniqueID); + iinfo->i_lenEAttr = le32_to_cpu(efe->lengthExtendedAttr); + iinfo->i_lenAlloc = le32_to_cpu(efe->lengthAllocDescs); + iinfo->i_checkpoint = le32_to_cpu(efe->checkpoint); + + /* Named streams */ + iinfo->i_streamdir = (efe->streamDirectoryICB.extLength != 0); + iinfo->i_locStreamdir = + lelb_to_cpu(efe->streamDirectoryICB.extLocation); + iinfo->i_lenStreams = le64_to_cpu(efe->objectSize); + if (iinfo->i_lenStreams >= inode->i_size) + iinfo->i_lenStreams -= inode->i_size; + else + iinfo->i_lenStreams = 0; + } + inode->i_generation = iinfo->i_unique; + + /* + * Sanity check length of allocation descriptors and extended attrs to + * avoid integer overflows + */ + if (iinfo->i_lenEAttr > bs || iinfo->i_lenAlloc > bs) + goto out; + /* Now do exact checks */ + if (udf_file_entry_alloc_offset(inode) + iinfo->i_lenAlloc > bs) + goto out; + /* Sanity checks for files in ICB so that we don't get confused later */ + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { + /* + * For file in ICB data is stored in allocation descriptor + * so sizes should match + */ + if (iinfo->i_lenAlloc != inode->i_size) + goto out; + /* File in ICB has to fit in there... */ + if (inode->i_size > bs - udf_file_entry_alloc_offset(inode)) + goto out; + } + + switch (fe->icbTag.fileType) { + case ICBTAG_FILE_TYPE_DIRECTORY: + inode->i_op = &udf_dir_inode_operations; + inode->i_fop = &udf_dir_operations; + inode->i_mode |= S_IFDIR; + inc_nlink(inode); + break; + case ICBTAG_FILE_TYPE_REALTIME: + case ICBTAG_FILE_TYPE_REGULAR: + case ICBTAG_FILE_TYPE_UNDEF: + case ICBTAG_FILE_TYPE_VAT20: + inode->i_data.a_ops = &udf_aops; + inode->i_op = &udf_file_inode_operations; + inode->i_fop = &udf_file_operations; + inode->i_mode |= S_IFREG; + break; + case ICBTAG_FILE_TYPE_BLOCK: + inode->i_mode |= S_IFBLK; + break; + case ICBTAG_FILE_TYPE_CHAR: + inode->i_mode |= S_IFCHR; + break; + case ICBTAG_FILE_TYPE_FIFO: + init_special_inode(inode, inode->i_mode | S_IFIFO, 0); + break; + case ICBTAG_FILE_TYPE_SOCKET: + init_special_inode(inode, inode->i_mode | S_IFSOCK, 0); + break; + case ICBTAG_FILE_TYPE_SYMLINK: + inode->i_data.a_ops = &udf_symlink_aops; + inode->i_op = &udf_symlink_inode_operations; + inode_nohighmem(inode); + inode->i_mode = S_IFLNK | 0777; + break; + case ICBTAG_FILE_TYPE_MAIN: + udf_debug("METADATA FILE-----\n"); + break; + case ICBTAG_FILE_TYPE_MIRROR: + udf_debug("METADATA MIRROR FILE-----\n"); + break; + case ICBTAG_FILE_TYPE_BITMAP: + udf_debug("METADATA BITMAP FILE-----\n"); + break; + default: + udf_err(inode->i_sb, "(ino %llu) failed unknown file type=%u\n", + inode->i_ino, fe->icbTag.fileType); + goto out; + } + if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) { + struct deviceSpec *dsea = + (struct deviceSpec *)udf_get_extendedattr(inode, 12, 1); + if (dsea) { + init_special_inode(inode, inode->i_mode, + MKDEV(le32_to_cpu(dsea->majorDeviceIdent), + le32_to_cpu(dsea->minorDeviceIdent))); + /* Developer ID ??? */ + } else + goto out; + } + ret = 0; +out: + brelse(bh); + return ret; +} + +static int udf_alloc_i_data(struct inode *inode, size_t size) +{ + struct udf_inode_info *iinfo = UDF_I(inode); + iinfo->i_data = kmalloc(size, GFP_KERNEL); + if (!iinfo->i_data) + return -ENOMEM; + return 0; +} + +static umode_t udf_convert_permissions(struct fileEntry *fe) +{ + umode_t mode; + uint32_t permissions; + uint32_t flags; + + permissions = le32_to_cpu(fe->permissions); + flags = le16_to_cpu(fe->icbTag.flags); + + mode = ((permissions) & 0007) | + ((permissions >> 2) & 0070) | + ((permissions >> 4) & 0700) | + ((flags & ICBTAG_FLAG_SETUID) ? S_ISUID : 0) | + ((flags & ICBTAG_FLAG_SETGID) ? S_ISGID : 0) | + ((flags & ICBTAG_FLAG_STICKY) ? S_ISVTX : 0); + + return mode; +} + +void udf_update_extra_perms(struct inode *inode, umode_t mode) +{ + struct udf_inode_info *iinfo = UDF_I(inode); + + /* + * UDF 2.01 sec. 3.3.3.3 Note 2: + * In Unix, delete permission tracks write + */ + iinfo->i_extraPerms &= ~FE_DELETE_PERMS; + if (mode & 0200) + iinfo->i_extraPerms |= FE_PERM_U_DELETE; + if (mode & 0020) + iinfo->i_extraPerms |= FE_PERM_G_DELETE; + if (mode & 0002) + iinfo->i_extraPerms |= FE_PERM_O_DELETE; +} + +int udf_sync_inode_metadata(struct inode *inode, struct writeback_control *wbc) +{ + struct buffer_head *bh; + int err = 0; + + bh = sb_getblk(inode->i_sb, + udf_get_lb_pblock(inode->i_sb, + &UDF_I(inode)->i_location, 0)); + if (!bh) + return -EIO; + + sync_dirty_buffer(bh); + if (buffer_write_io_error(bh)) { + udf_warn(inode->i_sb, "IO error syncing udf inode [%08llx]\n", + inode->i_ino); + err = -EIO; + goto out; + } + err = mmb_sync(&UDF_I(inode)->i_metadata_bhs); +out: + brelse(bh); + return err; +} + +static void udf_adjust_time(struct udf_inode_info *iinfo, struct timespec64 time) +{ + if (iinfo->i_crtime.tv_sec > time.tv_sec || + (iinfo->i_crtime.tv_sec == time.tv_sec && + iinfo->i_crtime.tv_nsec > time.tv_nsec)) + iinfo->i_crtime = time; +} + +int udf_write_inode(struct inode *inode, struct writeback_control *wbc) +{ + struct buffer_head *bh = NULL; + struct fileEntry *fe; + struct extendedFileEntry *efe; + uint64_t lb_recorded; + uint32_t udfperms; + uint16_t icbflags; + uint16_t crclen; + struct udf_sb_info *sbi = UDF_SB(inode->i_sb); + unsigned char blocksize_bits = inode->i_sb->s_blocksize_bits; + struct udf_inode_info *iinfo = UDF_I(inode); + + bh = sb_getblk(inode->i_sb, + udf_get_lb_pblock(inode->i_sb, &iinfo->i_location, 0)); + if (!bh) { + udf_debug("getblk failure\n"); + return -EIO; + } + + lock_buffer(bh); + memset(bh->b_data, 0, inode->i_sb->s_blocksize); + fe = (struct fileEntry *)bh->b_data; + efe = (struct extendedFileEntry *)bh->b_data; + + if (iinfo->i_use) { + struct unallocSpaceEntry *use = + (struct unallocSpaceEntry *)bh->b_data; + + use->lengthAllocDescs = cpu_to_le32(iinfo->i_lenAlloc); + memcpy(bh->b_data + sizeof(struct unallocSpaceEntry), + iinfo->i_data, inode->i_sb->s_blocksize - + sizeof(struct unallocSpaceEntry)); + use->descTag.tagIdent = cpu_to_le16(TAG_IDENT_USE); + crclen = sizeof(struct unallocSpaceEntry); + + goto finish; + } + + if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_UID_FORGET)) + fe->uid = cpu_to_le32(UDF_INVALID_ID); + else + fe->uid = cpu_to_le32(i_uid_read(inode)); + + if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_GID_FORGET)) + fe->gid = cpu_to_le32(UDF_INVALID_ID); + else + fe->gid = cpu_to_le32(i_gid_read(inode)); + + udfperms = ((inode->i_mode & 0007)) | + ((inode->i_mode & 0070) << 2) | + ((inode->i_mode & 0700) << 4); + + udfperms |= iinfo->i_extraPerms; + fe->permissions = cpu_to_le32(udfperms); + + if (S_ISDIR(inode->i_mode) && inode->i_nlink > 0) + fe->fileLinkCount = cpu_to_le16(inode->i_nlink - 1); + else { + if (iinfo->i_hidden) + fe->fileLinkCount = cpu_to_le16(0); + else + fe->fileLinkCount = cpu_to_le16(inode->i_nlink); + } + + fe->informationLength = cpu_to_le64(inode->i_size); + + if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) { + struct regid *eid; + struct deviceSpec *dsea = + (struct deviceSpec *)udf_get_extendedattr(inode, 12, 1); + if (!dsea) { + dsea = (struct deviceSpec *) + udf_add_extendedattr(inode, + sizeof(struct deviceSpec) + + sizeof(struct regid), 12, 0x3); + dsea->attrType = cpu_to_le32(12); + dsea->attrSubtype = 1; + dsea->attrLength = cpu_to_le32( + sizeof(struct deviceSpec) + + sizeof(struct regid)); + dsea->impUseLength = cpu_to_le32(sizeof(struct regid)); + } + eid = (struct regid *)dsea->impUse; + memset(eid, 0, sizeof(*eid)); + strcpy(eid->ident, UDF_ID_DEVELOPER); + eid->identSuffix[0] = UDF_OS_CLASS_UNIX; + eid->identSuffix[1] = UDF_OS_ID_LINUX; + dsea->majorDeviceIdent = cpu_to_le32(imajor(inode)); + dsea->minorDeviceIdent = cpu_to_le32(iminor(inode)); + } + + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) + lb_recorded = 0; /* No extents => no blocks! */ + else + lb_recorded = + (inode->i_blocks + (1 << (blocksize_bits - 9)) - 1) >> + (blocksize_bits - 9); + + if (iinfo->i_efe == 0) { + memcpy(bh->b_data + sizeof(struct fileEntry), + iinfo->i_data, + inode->i_sb->s_blocksize - sizeof(struct fileEntry)); + fe->logicalBlocksRecorded = cpu_to_le64(lb_recorded); + + udf_time_to_disk_stamp(&fe->accessTime, inode_get_atime(inode)); + udf_time_to_disk_stamp(&fe->modificationTime, inode_get_mtime(inode)); + udf_time_to_disk_stamp(&fe->attrTime, inode_get_ctime(inode)); + memset(&(fe->impIdent), 0, sizeof(struct regid)); + strcpy(fe->impIdent.ident, UDF_ID_DEVELOPER); + fe->impIdent.identSuffix[0] = UDF_OS_CLASS_UNIX; + fe->impIdent.identSuffix[1] = UDF_OS_ID_LINUX; + fe->uniqueID = cpu_to_le64(iinfo->i_unique); + fe->lengthExtendedAttr = cpu_to_le32(iinfo->i_lenEAttr); + fe->lengthAllocDescs = cpu_to_le32(iinfo->i_lenAlloc); + fe->checkpoint = cpu_to_le32(iinfo->i_checkpoint); + fe->descTag.tagIdent = cpu_to_le16(TAG_IDENT_FE); + crclen = sizeof(struct fileEntry); + } else { + memcpy(bh->b_data + sizeof(struct extendedFileEntry), + iinfo->i_data, + inode->i_sb->s_blocksize - + sizeof(struct extendedFileEntry)); + efe->objectSize = + cpu_to_le64(inode->i_size + iinfo->i_lenStreams); + efe->logicalBlocksRecorded = cpu_to_le64(lb_recorded); + + if (iinfo->i_streamdir) { + struct long_ad *icb_lad = &efe->streamDirectoryICB; + + icb_lad->extLocation = + cpu_to_lelb(iinfo->i_locStreamdir); + icb_lad->extLength = + cpu_to_le32(inode->i_sb->s_blocksize); + } + + udf_adjust_time(iinfo, inode_get_atime(inode)); + udf_adjust_time(iinfo, inode_get_mtime(inode)); + udf_adjust_time(iinfo, inode_get_ctime(inode)); + + udf_time_to_disk_stamp(&efe->accessTime, + inode_get_atime(inode)); + udf_time_to_disk_stamp(&efe->modificationTime, + inode_get_mtime(inode)); + udf_time_to_disk_stamp(&efe->createTime, iinfo->i_crtime); + udf_time_to_disk_stamp(&efe->attrTime, inode_get_ctime(inode)); + + memset(&(efe->impIdent), 0, sizeof(efe->impIdent)); + strcpy(efe->impIdent.ident, UDF_ID_DEVELOPER); + efe->impIdent.identSuffix[0] = UDF_OS_CLASS_UNIX; + efe->impIdent.identSuffix[1] = UDF_OS_ID_LINUX; + efe->uniqueID = cpu_to_le64(iinfo->i_unique); + efe->lengthExtendedAttr = cpu_to_le32(iinfo->i_lenEAttr); + efe->lengthAllocDescs = cpu_to_le32(iinfo->i_lenAlloc); + efe->checkpoint = cpu_to_le32(iinfo->i_checkpoint); + efe->descTag.tagIdent = cpu_to_le16(TAG_IDENT_EFE); + crclen = sizeof(struct extendedFileEntry); + } + +finish: + if (iinfo->i_strat4096) { + fe->icbTag.strategyType = cpu_to_le16(4096); + fe->icbTag.strategyParameter = cpu_to_le16(1); + fe->icbTag.numEntries = cpu_to_le16(2); + } else { + fe->icbTag.strategyType = cpu_to_le16(4); + fe->icbTag.numEntries = cpu_to_le16(1); + } + + if (iinfo->i_use) + fe->icbTag.fileType = ICBTAG_FILE_TYPE_USE; + else if (S_ISDIR(inode->i_mode)) + fe->icbTag.fileType = ICBTAG_FILE_TYPE_DIRECTORY; + else if (S_ISREG(inode->i_mode)) + fe->icbTag.fileType = ICBTAG_FILE_TYPE_REGULAR; + else if (S_ISLNK(inode->i_mode)) + fe->icbTag.fileType = ICBTAG_FILE_TYPE_SYMLINK; + else if (S_ISBLK(inode->i_mode)) + fe->icbTag.fileType = ICBTAG_FILE_TYPE_BLOCK; + else if (S_ISCHR(inode->i_mode)) + fe->icbTag.fileType = ICBTAG_FILE_TYPE_CHAR; + else if (S_ISFIFO(inode->i_mode)) + fe->icbTag.fileType = ICBTAG_FILE_TYPE_FIFO; + else if (S_ISSOCK(inode->i_mode)) + fe->icbTag.fileType = ICBTAG_FILE_TYPE_SOCKET; + + icbflags = iinfo->i_alloc_type | + ((inode->i_mode & S_ISUID) ? ICBTAG_FLAG_SETUID : 0) | + ((inode->i_mode & S_ISGID) ? ICBTAG_FLAG_SETGID : 0) | + ((inode->i_mode & S_ISVTX) ? ICBTAG_FLAG_STICKY : 0) | + (le16_to_cpu(fe->icbTag.flags) & + ~(ICBTAG_FLAG_AD_MASK | ICBTAG_FLAG_SETUID | + ICBTAG_FLAG_SETGID | ICBTAG_FLAG_STICKY)); + + fe->icbTag.flags = cpu_to_le16(icbflags); + if (sbi->s_udfrev >= 0x0200) + fe->descTag.descVersion = cpu_to_le16(3); + else + fe->descTag.descVersion = cpu_to_le16(2); + fe->descTag.tagSerialNum = cpu_to_le16(sbi->s_serial_number); + fe->descTag.tagLocation = cpu_to_le32( + iinfo->i_location.logicalBlockNum); + crclen += iinfo->i_lenEAttr + iinfo->i_lenAlloc - sizeof(struct tag); + fe->descTag.descCRCLength = cpu_to_le16(crclen); + fe->descTag.descCRC = cpu_to_le16(crc_itu_t(0, (char *)fe + sizeof(struct tag), + crclen)); + fe->descTag.tagChecksum = udf_tag_checksum(&fe->descTag); + + set_buffer_uptodate(bh); + unlock_buffer(bh); + + /* write the data blocks */ + mark_buffer_dirty(bh); + brelse(bh); + set_inode_metadata_writeback(inode); + + return 0; +} + +struct inode *__udf_iget(struct super_block *sb, struct kernel_lb_addr *ino, + bool hidden_inode) +{ + unsigned long block = udf_get_lb_pblock(sb, ino, 0); + struct inode *inode = iget_locked(sb, block); + int err; + + if (!inode) + return ERR_PTR(-ENOMEM); + + if (!(inode_state_read_once(inode) & I_NEW)) { + if (UDF_I(inode)->i_hidden != hidden_inode) { + iput(inode); + return ERR_PTR(-EFSCORRUPTED); + } + return inode; + } + + memcpy(&UDF_I(inode)->i_location, ino, sizeof(struct kernel_lb_addr)); + err = udf_read_inode(inode, hidden_inode); + if (err < 0) { + iget_failed(inode); + return ERR_PTR(err); + } + unlock_new_inode(inode); + + return inode; +} + +int udf_setup_indirect_aext(struct inode *inode, udf_pblk_t block, + struct extent_position *epos) +{ + struct super_block *sb = inode->i_sb; + struct buffer_head *bh; + struct allocExtDesc *aed; + struct extent_position nepos; + struct kernel_lb_addr neloc; + int ver, adsize; + int err = 0; + + if (UDF_I(inode)->i_alloc_type == ICBTAG_FLAG_AD_SHORT) + adsize = sizeof(struct short_ad); + else if (UDF_I(inode)->i_alloc_type == ICBTAG_FLAG_AD_LONG) + adsize = sizeof(struct long_ad); + else + return -EIO; + + neloc.logicalBlockNum = block; + neloc.partitionReferenceNum = epos->block.partitionReferenceNum; + + bh = sb_getblk(sb, udf_get_lb_pblock(sb, &neloc, 0)); + if (!bh) + return -EIO; + lock_buffer(bh); + memset(bh->b_data, 0x00, sb->s_blocksize); + set_buffer_uptodate(bh); + unlock_buffer(bh); + mmb_mark_buffer_dirty(bh, &UDF_I(inode)->i_metadata_bhs); + + aed = (struct allocExtDesc *)(bh->b_data); + if (!UDF_QUERY_FLAG(sb, UDF_FLAG_STRICT)) { + aed->previousAllocExtLocation = + cpu_to_le32(epos->block.logicalBlockNum); + } + aed->lengthAllocDescs = cpu_to_le32(0); + if (UDF_SB(sb)->s_udfrev >= 0x0200) + ver = 3; + else + ver = 2; + udf_new_tag(bh->b_data, TAG_IDENT_AED, ver, 1, block, + sizeof(struct tag)); + + nepos.block = neloc; + nepos.offset = sizeof(struct allocExtDesc); + nepos.bh = bh; + + /* + * Do we have to copy current last extent to make space for indirect + * one? + */ + if (epos->offset + adsize > sb->s_blocksize) { + struct kernel_lb_addr cp_loc; + uint32_t cp_len; + int8_t cp_type; + + epos->offset -= adsize; + err = udf_current_aext(inode, epos, &cp_loc, &cp_len, &cp_type, 0); + if (err <= 0) + goto err_out; + cp_len |= ((uint32_t)cp_type) << 30; + + __udf_add_aext(inode, &nepos, &cp_loc, cp_len, 1); + udf_write_aext(inode, epos, &nepos.block, + sb->s_blocksize | EXT_NEXT_EXTENT_ALLOCDESCS, 0); + } else { + __udf_add_aext(inode, epos, &nepos.block, + sb->s_blocksize | EXT_NEXT_EXTENT_ALLOCDESCS, 0); + } + + brelse(epos->bh); + *epos = nepos; + + return 0; +err_out: + brelse(bh); + return err; +} + +/* + * Append extent at the given position - should be the first free one in inode + * / indirect extent. This function assumes there is enough space in the inode + * or indirect extent. Use udf_add_aext() if you didn't check for this before. + */ +int __udf_add_aext(struct inode *inode, struct extent_position *epos, + struct kernel_lb_addr *eloc, uint32_t elen, int inc) +{ + struct udf_inode_info *iinfo = UDF_I(inode); + struct allocExtDesc *aed; + int adsize; + + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_SHORT) + adsize = sizeof(struct short_ad); + else if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_LONG) + adsize = sizeof(struct long_ad); + else + return -EIO; + + if (!epos->bh) { + WARN_ON(iinfo->i_lenAlloc != + epos->offset - udf_file_entry_alloc_offset(inode)); + } else { + aed = (struct allocExtDesc *)epos->bh->b_data; + WARN_ON(le32_to_cpu(aed->lengthAllocDescs) != + epos->offset - sizeof(struct allocExtDesc)); + WARN_ON(epos->offset + adsize > inode->i_sb->s_blocksize); + } + + udf_write_aext(inode, epos, eloc, elen, inc); + + if (!epos->bh) { + iinfo->i_lenAlloc += adsize; + mark_inode_dirty(inode); + } else { + aed = (struct allocExtDesc *)epos->bh->b_data; + le32_add_cpu(&aed->lengthAllocDescs, adsize); + if (!UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_STRICT) || + UDF_SB(inode->i_sb)->s_udfrev >= 0x0201) + udf_update_tag(epos->bh->b_data, + epos->offset + (inc ? 0 : adsize)); + else + udf_update_tag(epos->bh->b_data, + sizeof(struct allocExtDesc)); + mmb_mark_buffer_dirty(epos->bh, &iinfo->i_metadata_bhs); + } + + return 0; +} + +/* + * Append extent at given position - should be the first free one in inode + * / indirect extent. Takes care of allocating and linking indirect blocks. + */ +int udf_add_aext(struct inode *inode, struct extent_position *epos, + struct kernel_lb_addr *eloc, uint32_t elen, int inc) +{ + int adsize; + struct super_block *sb = inode->i_sb; + + if (UDF_I(inode)->i_alloc_type == ICBTAG_FLAG_AD_SHORT) + adsize = sizeof(struct short_ad); + else if (UDF_I(inode)->i_alloc_type == ICBTAG_FLAG_AD_LONG) + adsize = sizeof(struct long_ad); + else + return -EIO; + + if (epos->offset + (2 * adsize) > sb->s_blocksize) { + int err; + udf_pblk_t new_block; + + new_block = udf_new_block(sb, NULL, + epos->block.partitionReferenceNum, + epos->block.logicalBlockNum, &err); + if (!new_block) + return -ENOSPC; + + err = udf_setup_indirect_aext(inode, new_block, epos); + if (err) + return err; + } + + return __udf_add_aext(inode, epos, eloc, elen, inc); +} + +void udf_write_aext(struct inode *inode, struct extent_position *epos, + struct kernel_lb_addr *eloc, uint32_t elen, int inc) +{ + int adsize; + uint8_t *ptr; + struct short_ad *sad; + struct long_ad *lad; + struct udf_inode_info *iinfo = UDF_I(inode); + + if (!epos->bh) + ptr = iinfo->i_data + epos->offset - + udf_file_entry_alloc_offset(inode) + + iinfo->i_lenEAttr; + else + ptr = epos->bh->b_data + epos->offset; + + switch (iinfo->i_alloc_type) { + case ICBTAG_FLAG_AD_SHORT: + sad = (struct short_ad *)ptr; + sad->extLength = cpu_to_le32(elen); + sad->extPosition = cpu_to_le32(eloc->logicalBlockNum); + adsize = sizeof(struct short_ad); + break; + case ICBTAG_FLAG_AD_LONG: + lad = (struct long_ad *)ptr; + lad->extLength = cpu_to_le32(elen); + lad->extLocation = cpu_to_lelb(*eloc); + memset(lad->impUse, 0x00, sizeof(lad->impUse)); + adsize = sizeof(struct long_ad); + break; + default: + return; + } + + if (epos->bh) { + if (!UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_STRICT) || + UDF_SB(inode->i_sb)->s_udfrev >= 0x0201) { + struct allocExtDesc *aed = + (struct allocExtDesc *)epos->bh->b_data; + udf_update_tag(epos->bh->b_data, + le32_to_cpu(aed->lengthAllocDescs) + + sizeof(struct allocExtDesc)); + } + mmb_mark_buffer_dirty(epos->bh, &iinfo->i_metadata_bhs); + } else { + mark_inode_dirty(inode); + } + + if (inc) + epos->offset += adsize; +} + +/* + * Only 1 indirect extent in a row really makes sense but allow upto 16 in case + * someone does some weird stuff. + */ +#define UDF_MAX_INDIR_EXTS 16 + +/* + * Returns 1 on success, -errno on error, 0 on hit EOF. + */ +int udf_next_aext(struct inode *inode, struct extent_position *epos, + struct kernel_lb_addr *eloc, uint32_t *elen, int8_t *etype, + int inc) +{ + unsigned int indirections = 0; + int ret = 0; + udf_pblk_t block; + + while (1) { + ret = udf_current_aext(inode, epos, eloc, elen, + etype, inc); + if (ret <= 0) + return ret; + if (*etype != (EXT_NEXT_EXTENT_ALLOCDESCS >> 30)) + return ret; + + if (++indirections > UDF_MAX_INDIR_EXTS) { + udf_err(inode->i_sb, + "too many indirect extents in inode %llu\n", + inode->i_ino); + return -EFSCORRUPTED; + } + + epos->block = *eloc; + epos->offset = sizeof(struct allocExtDesc); + brelse(epos->bh); + block = udf_get_lb_pblock(inode->i_sb, &epos->block, 0); + epos->bh = sb_bread(inode->i_sb, block); + if (!epos->bh) { + udf_debug("reading block %u failed!\n", block); + return -EIO; + } + } +} + +/* + * Returns 1 on success, -errno on error, 0 on hit EOF. + */ +int udf_current_aext(struct inode *inode, struct extent_position *epos, + struct kernel_lb_addr *eloc, uint32_t *elen, int8_t *etype, + int inc) +{ + int alen; + uint8_t *ptr; + struct short_ad *sad; + struct long_ad *lad; + struct udf_inode_info *iinfo = UDF_I(inode); + + if (!epos->bh) { + if (!epos->offset) + epos->offset = udf_file_entry_alloc_offset(inode); + ptr = iinfo->i_data + epos->offset - + udf_file_entry_alloc_offset(inode) + + iinfo->i_lenEAttr; + alen = udf_file_entry_alloc_offset(inode) + + iinfo->i_lenAlloc; + } else { + struct allocExtDesc *header = + (struct allocExtDesc *)epos->bh->b_data; + + if (!epos->offset) + epos->offset = sizeof(struct allocExtDesc); + ptr = epos->bh->b_data + epos->offset; + if (check_add_overflow(sizeof(struct allocExtDesc), + le32_to_cpu(header->lengthAllocDescs), &alen)) + return -1; + + if (alen > epos->bh->b_size) + return -1; + } + + switch (iinfo->i_alloc_type) { + case ICBTAG_FLAG_AD_SHORT: + sad = udf_get_fileshortad(ptr, alen, &epos->offset, inc); + if (!sad) + return 0; + *etype = le32_to_cpu(sad->extLength) >> 30; + eloc->logicalBlockNum = le32_to_cpu(sad->extPosition); + eloc->partitionReferenceNum = + iinfo->i_location.partitionReferenceNum; + *elen = le32_to_cpu(sad->extLength) & UDF_EXTENT_LENGTH_MASK; + break; + case ICBTAG_FLAG_AD_LONG: + lad = udf_get_filelongad(ptr, alen, &epos->offset, inc); + if (!lad) + return 0; + *etype = le32_to_cpu(lad->extLength) >> 30; + *eloc = lelb_to_cpu(lad->extLocation); + *elen = le32_to_cpu(lad->extLength) & UDF_EXTENT_LENGTH_MASK; + break; + default: + udf_debug("alloc_type = %u unsupported\n", iinfo->i_alloc_type); + return -EINVAL; + } + + if (eloc->partitionReferenceNum >= UDF_SB(inode->i_sb)->s_partitions) { + udf_debug("invalid partition reference %u (partitions %u)\n", + eloc->partitionReferenceNum, + UDF_SB(inode->i_sb)->s_partitions); + return -EFSCORRUPTED; + } + + return 1; +} + +static int udf_insert_aext(struct inode *inode, struct extent_position epos, + struct kernel_lb_addr neloc, uint32_t nelen) +{ + struct kernel_lb_addr oeloc; + uint32_t oelen; + int8_t etype; + int ret; + + if (epos.bh) + get_bh(epos.bh); + + while (1) { + ret = udf_next_aext(inode, &epos, &oeloc, &oelen, &etype, 0); + if (ret <= 0) + break; + udf_write_aext(inode, &epos, &neloc, nelen, 1); + neloc = oeloc; + nelen = (etype << 30) | oelen; + } + if (ret == 0) + ret = udf_add_aext(inode, &epos, &neloc, nelen, 1); + brelse(epos.bh); + + return ret; +} + +int8_t udf_delete_aext(struct inode *inode, struct extent_position epos, + struct kernel_lb_addr *freed) +{ + struct extent_position oepos; + int adsize; + int8_t etype; + struct allocExtDesc *aed; + struct udf_inode_info *iinfo; + struct kernel_lb_addr eloc; + uint32_t elen; + int ret; + + if (epos.bh) { + get_bh(epos.bh); + get_bh(epos.bh); + } + + iinfo = UDF_I(inode); + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_SHORT) + adsize = sizeof(struct short_ad); + else if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_LONG) + adsize = sizeof(struct long_ad); + else + adsize = 0; + + oepos = epos; + if (udf_next_aext(inode, &epos, &eloc, &elen, &etype, 1) <= 0) + return -1; + + while (1) { + ret = udf_next_aext(inode, &epos, &eloc, &elen, &etype, 1); + if (ret < 0) { + brelse(epos.bh); + brelse(oepos.bh); + return -1; + } + if (ret == 0) + break; + udf_write_aext(inode, &oepos, &eloc, (etype << 30) | elen, 1); + if (oepos.bh != epos.bh) { + oepos.block = epos.block; + brelse(oepos.bh); + get_bh(epos.bh); + oepos.bh = epos.bh; + oepos.offset = epos.offset - adsize; + } + } + memset(&eloc, 0x00, sizeof(struct kernel_lb_addr)); + elen = 0; + + if (epos.bh != oepos.bh) { + /* + * The block that held the now-empty allocation extent must be + * returned to free space. When the caller already holds + * s_alloc_mutex (the space-table allocator in balloc.c), + * freeing it inline would recurse through udf_free_blocks() + * into udf_table_free_blocks() and deadlock re-acquiring + * s_alloc_mutex. In that case report the block to the caller, + * which frees it after dropping the lock. + */ + if (freed) + *freed = epos.block; + else + udf_free_blocks(inode->i_sb, inode, &epos.block, 0, 1); + udf_write_aext(inode, &oepos, &eloc, elen, 1); + udf_write_aext(inode, &oepos, &eloc, elen, 1); + if (!oepos.bh) { + iinfo->i_lenAlloc -= (adsize * 2); + mark_inode_dirty(inode); + } else { + aed = (struct allocExtDesc *)oepos.bh->b_data; + le32_add_cpu(&aed->lengthAllocDescs, -(2 * adsize)); + if (!UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_STRICT) || + UDF_SB(inode->i_sb)->s_udfrev >= 0x0201) + udf_update_tag(oepos.bh->b_data, + oepos.offset - (2 * adsize)); + else + udf_update_tag(oepos.bh->b_data, + sizeof(struct allocExtDesc)); + mmb_mark_buffer_dirty(oepos.bh, &iinfo->i_metadata_bhs); + } + } else { + udf_write_aext(inode, &oepos, &eloc, elen, 1); + if (!oepos.bh) { + iinfo->i_lenAlloc -= adsize; + mark_inode_dirty(inode); + } else { + aed = (struct allocExtDesc *)oepos.bh->b_data; + le32_add_cpu(&aed->lengthAllocDescs, -adsize); + if (!UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_STRICT) || + UDF_SB(inode->i_sb)->s_udfrev >= 0x0201) + udf_update_tag(oepos.bh->b_data, + epos.offset - adsize); + else + udf_update_tag(oepos.bh->b_data, + sizeof(struct allocExtDesc)); + mmb_mark_buffer_dirty(oepos.bh, &iinfo->i_metadata_bhs); + } + } + + brelse(epos.bh); + brelse(oepos.bh); + + return (elen >> 30); +} + +/* + * Returns 1 on success, -errno on error, 0 on hit EOF. + */ +int inode_bmap(struct inode *inode, sector_t block, struct extent_position *pos, + struct kernel_lb_addr *eloc, uint32_t *elen, sector_t *offset, + int8_t *etype) +{ + unsigned char blocksize_bits = inode->i_sb->s_blocksize_bits; + loff_t lbcount = 0, bcount = (loff_t) block << blocksize_bits; + struct udf_inode_info *iinfo; + int err = 0; + + iinfo = UDF_I(inode); + if (!udf_read_extent_cache(inode, bcount, &lbcount, pos)) { + pos->offset = 0; + pos->block = iinfo->i_location; + pos->bh = NULL; + } + *elen = 0; + do { + err = udf_next_aext(inode, pos, eloc, elen, etype, 1); + if (err <= 0) { + if (err == 0) { + *offset = (bcount - lbcount) >> blocksize_bits; + iinfo->i_lenExtents = lbcount; + } + return err; + } + lbcount += *elen; + } while (lbcount <= bcount); + /* update extent cache */ + udf_update_extent_cache(inode, lbcount - *elen, pos); + *offset = (bcount + *elen - lbcount) >> blocksize_bits; + + return 1; +} diff --git a/fs/udf/lowlevel.c b/fs/udf/lowlevel.c new file mode 100644 index 000000000..9d847a7a0 --- /dev/null +++ b/fs/udf/lowlevel.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * lowlevel.c + * + * PURPOSE + * Low Level Device Routines for the UDF filesystem + * + * COPYRIGHT + * (C) 1999-2001 Ben Fennema + * + * HISTORY + * + * 03/26/99 blf Created. + */ + +#include "udfdecl.h" + +#include +#include +#include + +#include "udf_sb.h" + +unsigned int udf_get_last_session(struct super_block *sb) +{ + struct cdrom_device_info *cdi = disk_to_cdi(sb->s_bdev->bd_disk); + struct cdrom_multisession ms_info; + + if (!cdi) { + udf_debug("CDROMMULTISESSION not supported.\n"); + return 0; + } + + ms_info.addr_format = CDROM_LBA; + if (cdrom_multisession(cdi, &ms_info) == 0) { + udf_debug("XA disk: %s, vol_desc_start=%d\n", + ms_info.xa_flag ? "yes" : "no", ms_info.addr.lba); + if (ms_info.xa_flag) /* necessary for a valid ms_info.addr */ + return ms_info.addr.lba; + } + return 0; +} + +udf_pblk_t udf_get_last_block(struct super_block *sb) +{ + struct cdrom_device_info *cdi = disk_to_cdi(sb->s_bdev->bd_disk); + unsigned long lblock = 0; + + /* + * The cdrom layer call failed or returned obviously bogus value? + * Try using the device size... + */ + if (!cdi || cdrom_get_last_written(cdi, &lblock) || lblock == 0) { + if (sb_bdev_nr_blocks(sb) > ~(udf_pblk_t)0) + return 0; + lblock = sb_bdev_nr_blocks(sb); + } + + if (lblock) + return lblock - 1; + return 0; +} diff --git a/fs/udf/misc.c b/fs/udf/misc.c new file mode 100644 index 000000000..6928e378f --- /dev/null +++ b/fs/udf/misc.c @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * misc.c + * + * PURPOSE + * Miscellaneous routines for the OSTA-UDF(tm) filesystem. + * + * COPYRIGHT + * (C) 1998 Dave Boynton + * (C) 1998-2004 Ben Fennema + * (C) 1999-2000 Stelias Computing Inc + * + * HISTORY + * + * 04/19/99 blf partial support for reading/writing specific EA's + */ + +#include "udfdecl.h" + +#include +#include +#include + +#include "udf_i.h" +#include "udf_sb.h" + +struct genericFormat *udf_add_extendedattr(struct inode *inode, uint32_t size, + uint32_t type, uint8_t loc) +{ + uint8_t *ea = NULL, *ad = NULL; + int offset; + uint16_t crclen; + struct udf_inode_info *iinfo = UDF_I(inode); + + ea = iinfo->i_data; + if (iinfo->i_lenEAttr) { + ad = iinfo->i_data + iinfo->i_lenEAttr; + } else { + ad = ea; + size += sizeof(struct extendedAttrHeaderDesc); + } + + offset = inode->i_sb->s_blocksize - udf_file_entry_alloc_offset(inode) - + iinfo->i_lenAlloc; + + /* TODO - Check for FreeEASpace */ + + if (loc & 0x01 && offset >= size) { + struct extendedAttrHeaderDesc *eahd; + eahd = (struct extendedAttrHeaderDesc *)ea; + + if (iinfo->i_lenAlloc) + memmove(&ad[size], ad, iinfo->i_lenAlloc); + + if (iinfo->i_lenEAttr) { + /* check checksum/crc */ + if (eahd->descTag.tagIdent != + cpu_to_le16(TAG_IDENT_EAHD) || + le32_to_cpu(eahd->descTag.tagLocation) != + iinfo->i_location.logicalBlockNum) + return NULL; + } else { + struct udf_sb_info *sbi = UDF_SB(inode->i_sb); + + size -= sizeof(struct extendedAttrHeaderDesc); + iinfo->i_lenEAttr += + sizeof(struct extendedAttrHeaderDesc); + eahd->descTag.tagIdent = cpu_to_le16(TAG_IDENT_EAHD); + if (sbi->s_udfrev >= 0x0200) + eahd->descTag.descVersion = cpu_to_le16(3); + else + eahd->descTag.descVersion = cpu_to_le16(2); + eahd->descTag.tagSerialNum = + cpu_to_le16(sbi->s_serial_number); + eahd->descTag.tagLocation = cpu_to_le32( + iinfo->i_location.logicalBlockNum); + eahd->impAttrLocation = cpu_to_le32(0xFFFFFFFF); + eahd->appAttrLocation = cpu_to_le32(0xFFFFFFFF); + } + + offset = iinfo->i_lenEAttr; + if (type < 2048) { + if (le32_to_cpu(eahd->appAttrLocation) < + iinfo->i_lenEAttr) { + uint32_t aal = + le32_to_cpu(eahd->appAttrLocation); + memmove(&ea[offset - aal + size], + &ea[aal], offset - aal); + offset -= aal; + eahd->appAttrLocation = + cpu_to_le32(aal + size); + } + if (le32_to_cpu(eahd->impAttrLocation) < + iinfo->i_lenEAttr) { + uint32_t ial = + le32_to_cpu(eahd->impAttrLocation); + memmove(&ea[offset - ial + size], + &ea[ial], offset - ial); + offset -= ial; + eahd->impAttrLocation = + cpu_to_le32(ial + size); + } + } else if (type < 65536) { + if (le32_to_cpu(eahd->appAttrLocation) < + iinfo->i_lenEAttr) { + uint32_t aal = + le32_to_cpu(eahd->appAttrLocation); + memmove(&ea[offset - aal + size], + &ea[aal], offset - aal); + offset -= aal; + eahd->appAttrLocation = + cpu_to_le32(aal + size); + } + } + /* rewrite CRC + checksum of eahd */ + crclen = sizeof(struct extendedAttrHeaderDesc) - sizeof(struct tag); + eahd->descTag.descCRCLength = cpu_to_le16(crclen); + eahd->descTag.descCRC = cpu_to_le16(crc_itu_t(0, (char *)eahd + + sizeof(struct tag), crclen)); + eahd->descTag.tagChecksum = udf_tag_checksum(&eahd->descTag); + iinfo->i_lenEAttr += size; + return (struct genericFormat *)&ea[offset]; + } + + return NULL; +} + +struct genericFormat *udf_get_extendedattr(struct inode *inode, uint32_t type, + uint8_t subtype) +{ + struct genericFormat *gaf; + uint8_t *ea = NULL; + uint32_t offset; + struct udf_inode_info *iinfo = UDF_I(inode); + + ea = iinfo->i_data; + + if (iinfo->i_lenEAttr) { + struct extendedAttrHeaderDesc *eahd; + eahd = (struct extendedAttrHeaderDesc *)ea; + + /* check checksum/crc */ + if (eahd->descTag.tagIdent != + cpu_to_le16(TAG_IDENT_EAHD) || + le32_to_cpu(eahd->descTag.tagLocation) != + iinfo->i_location.logicalBlockNum) + return NULL; + + if (type < 2048) + offset = sizeof(struct extendedAttrHeaderDesc); + else if (type < 65536) + offset = le32_to_cpu(eahd->impAttrLocation); + else + offset = le32_to_cpu(eahd->appAttrLocation); + + while (offset + sizeof(*gaf) < iinfo->i_lenEAttr) { + uint32_t attrLength; + + gaf = (struct genericFormat *)&ea[offset]; + attrLength = le32_to_cpu(gaf->attrLength); + + /* Detect undersized elements and buffer overflows */ + if ((attrLength < sizeof(*gaf)) || + (attrLength > (iinfo->i_lenEAttr - offset))) + break; + + if (le32_to_cpu(gaf->attrType) == type && + gaf->attrSubtype == subtype) + return gaf; + else + offset += attrLength; + } + } + + return NULL; +} + +/* + * udf_read_tagged + * + * PURPOSE + * Read the first block of a tagged descriptor. + * + * HISTORY + * July 1, 1997 - Andrew E. Mileski + * Written, tested, and released. + */ +struct buffer_head *udf_read_tagged(struct super_block *sb, uint32_t block, + uint32_t location, uint16_t *ident) +{ + struct tag *tag_p; + struct buffer_head *bh = NULL; + u8 checksum; + + /* Read the block */ + if (block == 0xFFFFFFFF) + return NULL; + + bh = sb_bread(sb, block); + if (!bh) { + udf_err(sb, "read failed, block=%u, location=%u\n", + block, location); + return NULL; + } + + tag_p = (struct tag *)(bh->b_data); + + *ident = le16_to_cpu(tag_p->tagIdent); + + if (location != le32_to_cpu(tag_p->tagLocation)) { + udf_debug("location mismatch block %u, tag %u != %u\n", + block, le32_to_cpu(tag_p->tagLocation), location); + goto error_out; + } + + /* Verify the tag checksum */ + checksum = udf_tag_checksum(tag_p); + if (checksum != tag_p->tagChecksum) { + udf_err(sb, "tag checksum failed, block %u: 0x%02x != 0x%02x\n", + block, checksum, tag_p->tagChecksum); + goto error_out; + } + + /* Verify the tag version */ + if (tag_p->descVersion != cpu_to_le16(0x0002U) && + tag_p->descVersion != cpu_to_le16(0x0003U)) { + udf_err(sb, "tag version 0x%04x != 0x0002 || 0x0003, block %u\n", + le16_to_cpu(tag_p->descVersion), block); + goto error_out; + } + + /* Verify the descriptor CRC */ + if (le16_to_cpu(tag_p->descCRCLength) + sizeof(struct tag) > sb->s_blocksize) { + udf_err(sb, "block %u: CRC length %u exceeds block size\n", + block, le16_to_cpu(tag_p->descCRCLength)); + goto error_out; + } + if (le16_to_cpu(tag_p->descCRC) == crc_itu_t(0, + bh->b_data + sizeof(struct tag), + le16_to_cpu(tag_p->descCRCLength))) + return bh; + + udf_debug("Crc failure block %u: crc = %u, crclen = %u\n", block, + le16_to_cpu(tag_p->descCRC), + le16_to_cpu(tag_p->descCRCLength)); +error_out: + brelse(bh); + return NULL; +} + +struct buffer_head *udf_read_ptagged(struct super_block *sb, + struct kernel_lb_addr *loc, + uint32_t offset, uint16_t *ident) +{ + return udf_read_tagged(sb, udf_get_lb_pblock(sb, loc, offset), + loc->logicalBlockNum + offset, ident); +} + +void udf_update_tag(char *data, int length) +{ + struct tag *tptr = (struct tag *)data; + length -= sizeof(struct tag); + + tptr->descCRCLength = cpu_to_le16(length); + tptr->descCRC = cpu_to_le16(crc_itu_t(0, data + sizeof(struct tag), length)); + tptr->tagChecksum = udf_tag_checksum(tptr); +} + +void udf_new_tag(char *data, uint16_t ident, uint16_t version, uint16_t snum, + uint32_t loc, int length) +{ + struct tag *tptr = (struct tag *)data; + tptr->tagIdent = cpu_to_le16(ident); + tptr->descVersion = cpu_to_le16(version); + tptr->tagSerialNum = cpu_to_le16(snum); + tptr->tagLocation = cpu_to_le32(loc); + udf_update_tag(data, length); +} + +u8 udf_tag_checksum(const struct tag *t) +{ + u8 *data = (u8 *)t; + u8 checksum = 0; + int i; + for (i = 0; i < sizeof(struct tag); ++i) + if (i != 4) /* position of checksum */ + checksum += data[i]; + return checksum; +} diff --git a/fs/udf/namei.c b/fs/udf/namei.c new file mode 100644 index 000000000..b90841ac0 --- /dev/null +++ b/fs/udf/namei.c @@ -0,0 +1,1028 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * namei.c + * + * PURPOSE + * Inode name handling routines for the OSTA-UDF(tm) filesystem. + * + * COPYRIGHT + * (C) 1998-2004 Ben Fennema + * (C) 1999-2000 Stelias Computing Inc + * + * HISTORY + * + * 12/12/98 blf Created. Split out the lookup code from dir.c + * 04/19/99 blf link, mknod, symlink support + */ + +#include "udfdecl.h" + +#include "udf_i.h" +#include "udf_sb.h" +#include +#include +#include +#include +#include +#include +#include +#include + +static inline int udf_match(int len1, const unsigned char *name1, int len2, + const unsigned char *name2) +{ + if (len1 != len2) + return 0; + + return !memcmp(name1, name2, len1); +} + +/** + * udf_fiiter_find_entry - find entry in given directory. + * + * @dir: directory inode to search in + * @child: qstr of the name + * @iter: iter to use for searching + * + * This function searches in the directory @dir for a file name @child. When + * found, @iter points to the position in the directory with given entry. + * + * Returns 0 on success, < 0 on error (including -ENOENT). + */ +static int udf_fiiter_find_entry(struct inode *dir, const struct qstr *child, + struct udf_fileident_iter *iter) +{ + int flen; + unsigned char *fname = NULL; + struct super_block *sb = dir->i_sb; + int isdotdot = child->len == 2 && + child->name[0] == '.' && child->name[1] == '.'; + int ret; + + fname = kmalloc(UDF_NAME_LEN, GFP_KERNEL); + if (!fname) + return -ENOMEM; + + for (ret = udf_fiiter_init(iter, dir, 0); + !ret && iter->pos < dir->i_size; + ret = udf_fiiter_advance(iter)) { + if (iter->fi.fileCharacteristics & FID_FILE_CHAR_DELETED) { + if (!UDF_QUERY_FLAG(sb, UDF_FLAG_UNDELETE)) + continue; + } + + if (iter->fi.fileCharacteristics & FID_FILE_CHAR_HIDDEN) { + if (!UDF_QUERY_FLAG(sb, UDF_FLAG_UNHIDE)) + continue; + } + + if ((iter->fi.fileCharacteristics & FID_FILE_CHAR_PARENT) && + isdotdot) + goto out_ok; + + if (!iter->fi.lengthFileIdent) + continue; + + flen = udf_get_filename(sb, iter->name, + iter->fi.lengthFileIdent, fname, UDF_NAME_LEN); + if (flen < 0) { + ret = flen; + goto out_err; + } + + if (udf_match(flen, fname, child->len, child->name)) + goto out_ok; + } + if (!ret) + ret = -ENOENT; + +out_err: + udf_fiiter_release(iter); +out_ok: + kfree(fname); + + return ret; +} + +static struct dentry *udf_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + struct inode *inode = NULL; + struct udf_fileident_iter iter; + int err; + + if (dentry->d_name.len > UDF_NAME_LEN) + return ERR_PTR(-ENAMETOOLONG); + + err = udf_fiiter_find_entry(dir, &dentry->d_name, &iter); + if (err < 0 && err != -ENOENT) + return ERR_PTR(err); + + if (err == 0) { + struct kernel_lb_addr loc; + + loc = lelb_to_cpu(iter.fi.icb.extLocation); + udf_fiiter_release(&iter); + + inode = udf_iget(dir->i_sb, &loc); + } + + return d_splice_alias(inode, dentry); +} + +static int udf_expand_dir_adinicb(struct inode *inode, udf_pblk_t *block) +{ + udf_pblk_t newblock; + struct buffer_head *dbh = NULL; + struct kernel_lb_addr eloc; + struct extent_position epos; + uint8_t alloctype; + struct udf_inode_info *iinfo = UDF_I(inode); + struct udf_fileident_iter iter; + uint8_t *impuse; + int ret; + + if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_USE_SHORT_AD)) + alloctype = ICBTAG_FLAG_AD_SHORT; + else + alloctype = ICBTAG_FLAG_AD_LONG; + + if (!inode->i_size) { + iinfo->i_alloc_type = alloctype; + mark_inode_dirty(inode); + return 0; + } + + /* alloc block, and copy data to it */ + *block = udf_new_block(inode->i_sb, inode, + iinfo->i_location.partitionReferenceNum, + iinfo->i_location.logicalBlockNum, &ret); + if (!(*block)) + return ret; + newblock = udf_get_pblock(inode->i_sb, *block, + iinfo->i_location.partitionReferenceNum, + 0); + if (newblock == 0xffffffff) + return -EFSCORRUPTED; + dbh = sb_getblk(inode->i_sb, newblock); + if (!dbh) + return -ENOMEM; + lock_buffer(dbh); + memcpy(dbh->b_data, iinfo->i_data, inode->i_size); + memset(dbh->b_data + inode->i_size, 0, + inode->i_sb->s_blocksize - inode->i_size); + set_buffer_uptodate(dbh); + unlock_buffer(dbh); + + /* Drop inline data, add block instead */ + iinfo->i_alloc_type = alloctype; + memset(iinfo->i_data + iinfo->i_lenEAttr, 0, iinfo->i_lenAlloc); + iinfo->i_lenAlloc = 0; + eloc.logicalBlockNum = *block; + eloc.partitionReferenceNum = + iinfo->i_location.partitionReferenceNum; + iinfo->i_lenExtents = inode->i_size; + epos.bh = NULL; + epos.block = iinfo->i_location; + epos.offset = udf_file_entry_alloc_offset(inode); + ret = udf_add_aext(inode, &epos, &eloc, inode->i_size, 0); + brelse(epos.bh); + if (ret < 0) { + brelse(dbh); + udf_free_blocks(inode->i_sb, inode, &eloc, 0, 1); + return ret; + } + mark_inode_dirty(inode); + + /* Now fixup tags in moved directory entries */ + for (ret = udf_fiiter_init(&iter, inode, 0); + !ret && iter.pos < inode->i_size; + ret = udf_fiiter_advance(&iter)) { + iter.fi.descTag.tagLocation = cpu_to_le32(*block); + if (iter.fi.lengthOfImpUse != cpu_to_le16(0)) + impuse = dbh->b_data + iter.pos + + sizeof(struct fileIdentDesc); + else + impuse = NULL; + udf_fiiter_write_fi(&iter, impuse); + } + brelse(dbh); + /* + * We don't expect the iteration to fail as the directory has been + * already verified to be correct + */ + WARN_ON_ONCE(ret); + udf_fiiter_release(&iter); + + return 0; +} + +static int udf_fiiter_add_entry(struct inode *dir, struct dentry *dentry, + struct udf_fileident_iter *iter) +{ + struct udf_inode_info *dinfo = UDF_I(dir); + int nfidlen, namelen = 0; + int ret; + int off, blksize = 1 << dir->i_blkbits; + udf_pblk_t block; + char name[UDF_NAME_LEN_CS0]; + + if (dentry) { + namelen = udf_put_filename(dir->i_sb, dentry->d_name.name, + dentry->d_name.len, + name, UDF_NAME_LEN_CS0); + if (!namelen) + return -ENAMETOOLONG; + } + nfidlen = ALIGN(sizeof(struct fileIdentDesc) + namelen, UDF_NAME_PAD); + + for (ret = udf_fiiter_init(iter, dir, 0); + !ret && iter->pos < dir->i_size; + ret = udf_fiiter_advance(iter)) { + if (iter->fi.fileCharacteristics & FID_FILE_CHAR_DELETED) { + if (udf_dir_entry_len(&iter->fi) == nfidlen) { + iter->fi.descTag.tagSerialNum = cpu_to_le16(1); + iter->fi.fileVersionNum = cpu_to_le16(1); + iter->fi.fileCharacteristics = 0; + iter->fi.lengthFileIdent = namelen; + iter->fi.lengthOfImpUse = cpu_to_le16(0); + memcpy(iter->namebuf, name, namelen); + iter->name = iter->namebuf; + return 0; + } + } + } + if (ret) { + udf_fiiter_release(iter); + return ret; + } + if (dinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB && + blksize - udf_ext0_offset(dir) - iter->pos < nfidlen) { + udf_fiiter_release(iter); + ret = udf_expand_dir_adinicb(dir, &block); + if (ret) + return ret; + ret = udf_fiiter_init(iter, dir, dir->i_size); + if (ret < 0) + return ret; + } + + /* Get blocknumber to use for entry tag */ + if (dinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { + block = dinfo->i_location.logicalBlockNum; + } else { + block = iter->eloc.logicalBlockNum + + ((iter->elen - 1) >> dir->i_blkbits); + } + off = iter->pos & (blksize - 1); + if (!off) + off = blksize; + /* Entry fits into current block? */ + if (blksize - udf_ext0_offset(dir) - off >= nfidlen) + goto store_fi; + + ret = udf_fiiter_append_blk(iter); + if (ret) { + udf_fiiter_release(iter); + return ret; + } + + /* Entry will be completely in the new block? Update tag location... */ + if (!(iter->pos & (blksize - 1))) + block = iter->eloc.logicalBlockNum + + ((iter->elen - 1) >> dir->i_blkbits); +store_fi: + memset(&iter->fi, 0, sizeof(struct fileIdentDesc)); + if (UDF_SB(dir->i_sb)->s_udfrev >= 0x0200) + udf_new_tag((char *)(&iter->fi), TAG_IDENT_FID, 3, 1, block, + sizeof(struct tag)); + else + udf_new_tag((char *)(&iter->fi), TAG_IDENT_FID, 2, 1, block, + sizeof(struct tag)); + iter->fi.fileVersionNum = cpu_to_le16(1); + iter->fi.lengthFileIdent = namelen; + iter->fi.lengthOfImpUse = cpu_to_le16(0); + memcpy(iter->namebuf, name, namelen); + iter->name = iter->namebuf; + + dir->i_size += nfidlen; + if (dinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { + dinfo->i_lenAlloc += nfidlen; + } else { + /* Truncate last extent to proper size */ + udf_fiiter_update_elen(iter, iter->elen - + (dinfo->i_lenExtents - dir->i_size)); + } + mark_inode_dirty(dir); + + return 0; +} + +static void udf_fiiter_delete_entry(struct udf_fileident_iter *iter) +{ + iter->fi.fileCharacteristics |= FID_FILE_CHAR_DELETED; + + if (UDF_QUERY_FLAG(iter->dir->i_sb, UDF_FLAG_STRICT)) + memset(&iter->fi.icb, 0x00, sizeof(struct long_ad)); + + udf_fiiter_write_fi(iter, NULL); +} + +static void udf_add_fid_counter(struct super_block *sb, bool dir, int val) +{ + struct logicalVolIntegrityDescImpUse *lvidiu = udf_sb_lvidiu(sb); + + if (!lvidiu) + return; + mutex_lock(&UDF_SB(sb)->s_alloc_mutex); + if (dir) + le32_add_cpu(&lvidiu->numDirs, val); + else + le32_add_cpu(&lvidiu->numFiles, val); + udf_updated_lvid(sb); + mutex_unlock(&UDF_SB(sb)->s_alloc_mutex); +} + +static int udf_add_nondir(struct dentry *dentry, struct inode *inode) +{ + struct udf_inode_info *iinfo = UDF_I(inode); + struct inode *dir = d_inode(dentry->d_parent); + struct udf_fileident_iter iter; + int err; + + err = udf_fiiter_add_entry(dir, dentry, &iter); + if (err) { + inode_dec_link_count(inode); + discard_new_inode(inode); + return err; + } + iter.fi.icb.extLength = cpu_to_le32(inode->i_sb->s_blocksize); + iter.fi.icb.extLocation = cpu_to_lelb(iinfo->i_location); + *(__le32 *)((struct allocDescImpUse *)iter.fi.icb.impUse)->impUse = + cpu_to_le32(iinfo->i_unique & 0x00000000FFFFFFFFUL); + udf_fiiter_write_fi(&iter, NULL); + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + mark_inode_dirty(dir); + udf_fiiter_release(&iter); + udf_add_fid_counter(dir->i_sb, false, 1); + d_instantiate_new(dentry, inode); + + return 0; +} + +static int udf_create(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct inode *inode = udf_new_inode(dir, mode); + + if (IS_ERR(inode)) + return PTR_ERR(inode); + + inode->i_data.a_ops = &udf_aops; + inode->i_op = &udf_file_inode_operations; + inode->i_fop = &udf_file_operations; + mark_inode_dirty(inode); + + return udf_add_nondir(dentry, inode); +} + +static int udf_tmpfile(struct mnt_idmap *idmap, struct inode *dir, + struct file *file, umode_t mode) +{ + struct inode *inode = udf_new_inode(dir, mode); + + if (IS_ERR(inode)) + return PTR_ERR(inode); + + inode->i_data.a_ops = &udf_aops; + inode->i_op = &udf_file_inode_operations; + inode->i_fop = &udf_file_operations; + mark_inode_dirty(inode); + d_tmpfile(file, inode); + unlock_new_inode(inode); + return finish_open_simple(file, 0); +} + +static int udf_mknod(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) +{ + struct inode *inode; + + if (!old_valid_dev(rdev)) + return -EINVAL; + + inode = udf_new_inode(dir, mode); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + init_special_inode(inode, mode, rdev); + return udf_add_nondir(dentry, inode); +} + +static struct dentry *udf_mkdir(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +{ + struct inode *inode; + struct udf_fileident_iter iter; + int err; + struct udf_inode_info *dinfo = UDF_I(dir); + struct udf_inode_info *iinfo; + + inode = udf_new_inode(dir, mode); + if (IS_ERR(inode)) + return ERR_CAST(inode); + + iinfo = UDF_I(inode); + inode->i_op = &udf_dir_inode_operations; + inode->i_fop = &udf_dir_operations; + err = udf_fiiter_add_entry(inode, NULL, &iter); + if (err) { + clear_nlink(inode); + discard_new_inode(inode); + return ERR_PTR(err); + } + set_nlink(inode, 2); + iter.fi.icb.extLength = cpu_to_le32(inode->i_sb->s_blocksize); + iter.fi.icb.extLocation = cpu_to_lelb(dinfo->i_location); + *(__le32 *)((struct allocDescImpUse *)iter.fi.icb.impUse)->impUse = + cpu_to_le32(dinfo->i_unique & 0x00000000FFFFFFFFUL); + iter.fi.fileCharacteristics = + FID_FILE_CHAR_DIRECTORY | FID_FILE_CHAR_PARENT; + udf_fiiter_write_fi(&iter, NULL); + udf_fiiter_release(&iter); + mark_inode_dirty(inode); + + err = udf_fiiter_add_entry(dir, dentry, &iter); + if (err) { + clear_nlink(inode); + discard_new_inode(inode); + return ERR_PTR(err); + } + iter.fi.icb.extLength = cpu_to_le32(inode->i_sb->s_blocksize); + iter.fi.icb.extLocation = cpu_to_lelb(iinfo->i_location); + *(__le32 *)((struct allocDescImpUse *)iter.fi.icb.impUse)->impUse = + cpu_to_le32(iinfo->i_unique & 0x00000000FFFFFFFFUL); + iter.fi.fileCharacteristics |= FID_FILE_CHAR_DIRECTORY; + udf_fiiter_write_fi(&iter, NULL); + udf_fiiter_release(&iter); + udf_add_fid_counter(dir->i_sb, true, 1); + inc_nlink(dir); + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + mark_inode_dirty(dir); + d_instantiate_new(dentry, inode); + + return NULL; +} + +static int empty_dir(struct inode *dir) +{ + struct udf_fileident_iter iter; + int ret; + + for (ret = udf_fiiter_init(&iter, dir, 0); + !ret && iter.pos < dir->i_size; + ret = udf_fiiter_advance(&iter)) { + if (iter.fi.lengthFileIdent && + !(iter.fi.fileCharacteristics & FID_FILE_CHAR_DELETED)) { + udf_fiiter_release(&iter); + return 0; + } + } + udf_fiiter_release(&iter); + + return 1; +} + +static int udf_rmdir(struct inode *dir, struct dentry *dentry) +{ + int ret; + struct inode *inode = d_inode(dentry); + struct udf_fileident_iter iter; + struct kernel_lb_addr tloc; + + ret = udf_fiiter_find_entry(dir, &dentry->d_name, &iter); + if (ret) + goto out; + + ret = -EFSCORRUPTED; + tloc = lelb_to_cpu(iter.fi.icb.extLocation); + if (udf_get_lb_pblock(dir->i_sb, &tloc, 0) != inode->i_ino) + goto end_rmdir; + ret = -ENOTEMPTY; + if (!empty_dir(inode)) + goto end_rmdir; + udf_fiiter_delete_entry(&iter); + if (inode->i_nlink != 2) + udf_warn(inode->i_sb, "empty directory has nlink != 2 (%u)\n", + inode->i_nlink); + clear_nlink(inode); + inode->i_size = 0; + if (dir->i_nlink >= 3) + inode_dec_link_count(dir); + else + udf_warn(inode->i_sb, "parent dir link count too low (%u)\n", + dir->i_nlink); + udf_add_fid_counter(dir->i_sb, true, -1); + inode_set_mtime_to_ts(dir, + inode_set_ctime_to_ts(dir, inode_set_ctime_current(inode))); + mark_inode_dirty(dir); + ret = 0; +end_rmdir: + udf_fiiter_release(&iter); +out: + return ret; +} + +static int udf_unlink(struct inode *dir, struct dentry *dentry) +{ + int ret; + struct inode *inode = d_inode(dentry); + struct udf_fileident_iter iter; + struct kernel_lb_addr tloc; + + ret = udf_fiiter_find_entry(dir, &dentry->d_name, &iter); + if (ret) + goto out; + + ret = -EFSCORRUPTED; + tloc = lelb_to_cpu(iter.fi.icb.extLocation); + if (udf_get_lb_pblock(dir->i_sb, &tloc, 0) != inode->i_ino) + goto end_unlink; + + if (!inode->i_nlink) { + udf_debug("Deleting nonexistent file (%llu), %u\n", + inode->i_ino, inode->i_nlink); + set_nlink(inode, 1); + } + udf_fiiter_delete_entry(&iter); + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + mark_inode_dirty(dir); + inode_dec_link_count(inode); + udf_add_fid_counter(dir->i_sb, false, -1); + inode_set_ctime_to_ts(inode, inode_get_ctime(dir)); + ret = 0; +end_unlink: + udf_fiiter_release(&iter); +out: + return ret; +} + +static int udf_symlink(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const char *symname) +{ + struct inode *inode; + struct pathComponent *pc; + const char *compstart; + struct extent_position epos = {}; + int eoffset, elen = 0; + uint8_t *ea; + int err; + udf_pblk_t block; + unsigned char *name = NULL; + int namelen; + struct udf_inode_info *iinfo; + struct super_block *sb = dir->i_sb; + + name = kmalloc(UDF_NAME_LEN_CS0, GFP_KERNEL); + if (!name) { + err = -ENOMEM; + goto out; + } + + inode = udf_new_inode(dir, S_IFLNK | 0777); + if (IS_ERR(inode)) { + err = PTR_ERR(inode); + goto out; + } + + iinfo = UDF_I(inode); + down_write(&iinfo->i_data_sem); + inode->i_data.a_ops = &udf_symlink_aops; + inode->i_op = &udf_symlink_inode_operations; + inode_nohighmem(inode); + + if (iinfo->i_alloc_type != ICBTAG_FLAG_AD_IN_ICB) { + struct kernel_lb_addr eloc; + uint32_t bsize; + + block = udf_new_block(sb, inode, + iinfo->i_location.partitionReferenceNum, + iinfo->i_location.logicalBlockNum, &err); + if (!block) + goto out_no_entry; + epos.block = iinfo->i_location; + epos.offset = udf_file_entry_alloc_offset(inode); + epos.bh = NULL; + eloc.logicalBlockNum = block; + eloc.partitionReferenceNum = + iinfo->i_location.partitionReferenceNum; + bsize = sb->s_blocksize; + iinfo->i_lenExtents = bsize; + err = udf_add_aext(inode, &epos, &eloc, bsize, 0); + brelse(epos.bh); + if (err < 0) { + udf_free_blocks(sb, inode, &eloc, 0, 1); + goto out_no_entry; + } + + block = udf_get_pblock(sb, block, + iinfo->i_location.partitionReferenceNum, + 0); + epos.bh = sb_getblk(sb, block); + if (unlikely(!epos.bh)) { + err = -ENOMEM; + udf_free_blocks(sb, inode, &eloc, 0, 1); + goto out_no_entry; + } + lock_buffer(epos.bh); + memset(epos.bh->b_data, 0x00, bsize); + set_buffer_uptodate(epos.bh); + unlock_buffer(epos.bh); + mmb_mark_buffer_dirty(epos.bh, &iinfo->i_metadata_bhs); + ea = epos.bh->b_data + udf_ext0_offset(inode); + } else + ea = iinfo->i_data + iinfo->i_lenEAttr; + + eoffset = sb->s_blocksize - udf_ext0_offset(inode); + pc = (struct pathComponent *)ea; + + if (*symname == '/') { + do { + symname++; + } while (*symname == '/'); + + pc->componentType = 1; + pc->lengthComponentIdent = 0; + pc->componentFileVersionNum = 0; + elen += sizeof(struct pathComponent); + } + + err = -ENAMETOOLONG; + + while (*symname) { + if (elen + sizeof(struct pathComponent) > eoffset) + goto out_no_entry; + + pc = (struct pathComponent *)(ea + elen); + + compstart = symname; + + do { + symname++; + } while (*symname && *symname != '/'); + + pc->componentType = 5; + pc->lengthComponentIdent = 0; + pc->componentFileVersionNum = 0; + if (compstart[0] == '.') { + if ((symname - compstart) == 1) + pc->componentType = 4; + else if ((symname - compstart) == 2 && + compstart[1] == '.') + pc->componentType = 3; + } + + if (pc->componentType == 5) { + namelen = udf_put_filename(sb, compstart, + symname - compstart, + name, UDF_NAME_LEN_CS0); + if (!namelen) + goto out_no_entry; + + if (elen + sizeof(struct pathComponent) + namelen > + eoffset) + goto out_no_entry; + else + pc->lengthComponentIdent = namelen; + + memcpy(pc->componentIdent, name, namelen); + } + + elen += sizeof(struct pathComponent) + pc->lengthComponentIdent; + + if (*symname) { + do { + symname++; + } while (*symname == '/'); + } + } + + brelse(epos.bh); + inode->i_size = elen; + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) + iinfo->i_lenAlloc = inode->i_size; + else + udf_truncate_tail_extent(inode); + mark_inode_dirty(inode); + up_write(&iinfo->i_data_sem); + + err = udf_add_nondir(dentry, inode); +out: + kfree(name); + return err; + +out_no_entry: + up_write(&iinfo->i_data_sem); + inode_dec_link_count(inode); + discard_new_inode(inode); + goto out; +} + +static int udf_link(struct dentry *old_dentry, struct inode *dir, + struct dentry *dentry) +{ + struct inode *inode = d_inode(old_dentry); + struct udf_fileident_iter iter; + int err; + + err = udf_fiiter_add_entry(dir, dentry, &iter); + if (err) + return err; + iter.fi.icb.extLength = cpu_to_le32(inode->i_sb->s_blocksize); + iter.fi.icb.extLocation = cpu_to_lelb(UDF_I(inode)->i_location); + if (UDF_SB(inode->i_sb)->s_lvid_bh) { + *(__le32 *)((struct allocDescImpUse *)iter.fi.icb.impUse)->impUse = + cpu_to_le32(lvid_get_unique_id(inode->i_sb)); + } + udf_fiiter_write_fi(&iter, NULL); + udf_fiiter_release(&iter); + + inc_nlink(inode); + udf_add_fid_counter(dir->i_sb, false, 1); + inode_set_ctime_current(inode); + mark_inode_dirty(inode); + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + mark_inode_dirty(dir); + ihold(inode); + d_instantiate(dentry, inode); + + return 0; +} + +/* Anybody can rename anything with this: the permission checks are left to the + * higher-level routines. + */ +static int udf_rename(struct mnt_idmap *idmap, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) +{ + struct inode *old_inode = d_inode(old_dentry); + struct inode *new_inode = d_inode(new_dentry); + struct udf_fileident_iter oiter, niter, diriter; + bool has_diriter = false, is_dir = false; + int retval; + struct kernel_lb_addr tloc; + + if (flags & ~RENAME_NOREPLACE) + return -EINVAL; + + retval = udf_fiiter_find_entry(old_dir, &old_dentry->d_name, &oiter); + if (retval) + return retval; + + tloc = lelb_to_cpu(oiter.fi.icb.extLocation); + if (udf_get_lb_pblock(old_dir->i_sb, &tloc, 0) != old_inode->i_ino) { + retval = -ENOENT; + goto out_oiter; + } + + if (S_ISDIR(old_inode->i_mode)) { + if (new_inode) { + retval = -ENOTEMPTY; + if (!empty_dir(new_inode)) + goto out_oiter; + retval = -EFSCORRUPTED; + if (new_inode->i_nlink != 2) + goto out_oiter; + } + retval = -EFSCORRUPTED; + if (old_dir->i_nlink < 3) + goto out_oiter; + is_dir = true; + } else if (new_inode) { + retval = -EFSCORRUPTED; + if (new_inode->i_nlink < 1) + goto out_oiter; + } + if (is_dir && old_dir != new_dir) { + retval = udf_fiiter_find_entry(old_inode, &dotdot_name, + &diriter); + if (retval == -ENOENT) { + udf_err(old_inode->i_sb, + "directory (ino %llu) has no '..' entry\n", + old_inode->i_ino); + retval = -EFSCORRUPTED; + } + if (retval) + goto out_oiter; + has_diriter = true; + tloc = lelb_to_cpu(diriter.fi.icb.extLocation); + if (udf_get_lb_pblock(old_inode->i_sb, &tloc, 0) != + old_dir->i_ino) { + retval = -EFSCORRUPTED; + udf_err(old_inode->i_sb, + "directory (ino %llu) has parent entry pointing to another inode (%llu != %u)\n", + old_inode->i_ino, old_dir->i_ino, + udf_get_lb_pblock(old_inode->i_sb, &tloc, 0)); + goto out_oiter; + } + } + + retval = udf_fiiter_find_entry(new_dir, &new_dentry->d_name, &niter); + if (retval && retval != -ENOENT) + goto out_oiter; + /* Entry found but not passed by VFS? */ + if (!retval && !new_inode) { + retval = -EFSCORRUPTED; + udf_fiiter_release(&niter); + goto out_oiter; + } + /* Entry not found? Need to add one... */ + if (retval) { + udf_fiiter_release(&niter); + retval = udf_fiiter_add_entry(new_dir, new_dentry, &niter); + if (retval) + goto out_oiter; + } + + /* + * Like most other Unix systems, set the ctime for inodes on a + * rename. + */ + inode_set_ctime_current(old_inode); + mark_inode_dirty(old_inode); + + /* + * ok, that's it + */ + niter.fi.fileVersionNum = oiter.fi.fileVersionNum; + niter.fi.fileCharacteristics = oiter.fi.fileCharacteristics; + memcpy(&(niter.fi.icb), &(oiter.fi.icb), sizeof(oiter.fi.icb)); + udf_fiiter_write_fi(&niter, NULL); + udf_fiiter_release(&niter); + + /* + * The old entry may have moved due to new entry allocation. Find it + * again. + */ + udf_fiiter_release(&oiter); + retval = udf_fiiter_find_entry(old_dir, &old_dentry->d_name, &oiter); + if (retval) { + udf_err(old_dir->i_sb, + "failed to find renamed entry again in directory (ino %llu)\n", + old_dir->i_ino); + } else { + udf_fiiter_delete_entry(&oiter); + udf_fiiter_release(&oiter); + } + + if (new_inode) { + inode_set_ctime_current(new_inode); + inode_dec_link_count(new_inode); + udf_add_fid_counter(old_dir->i_sb, S_ISDIR(new_inode->i_mode), + -1); + } + inode_set_mtime_to_ts(old_dir, inode_set_ctime_current(old_dir)); + inode_set_mtime_to_ts(new_dir, inode_set_ctime_current(new_dir)); + mark_inode_dirty(old_dir); + mark_inode_dirty(new_dir); + + if (has_diriter) { + diriter.fi.icb.extLocation = + cpu_to_lelb(UDF_I(new_dir)->i_location); + udf_fiiter_write_fi(&diriter, NULL); + udf_fiiter_release(&diriter); + } + + if (is_dir) { + inode_dec_link_count(old_dir); + if (new_inode) + inode_dec_link_count(new_inode); + else { + inc_nlink(new_dir); + mark_inode_dirty(new_dir); + } + } + return 0; +out_oiter: + if (has_diriter) + udf_fiiter_release(&diriter); + udf_fiiter_release(&oiter); + + return retval; +} + +static struct dentry *udf_get_parent(struct dentry *child) +{ + struct kernel_lb_addr tloc; + struct udf_fileident_iter iter; + int err; + + err = udf_fiiter_find_entry(d_inode(child), &dotdot_name, &iter); + if (err) + return ERR_PTR(err); + + tloc = lelb_to_cpu(iter.fi.icb.extLocation); + udf_fiiter_release(&iter); + return d_obtain_alias(udf_iget(child->d_sb, &tloc)); +} + + +static struct dentry *udf_nfs_get_inode(struct super_block *sb, u32 block, + u16 partref, __u32 generation) +{ + struct inode *inode; + struct kernel_lb_addr loc; + + if (block == 0) + return ERR_PTR(-ESTALE); + + loc.logicalBlockNum = block; + loc.partitionReferenceNum = partref; + inode = udf_iget(sb, &loc); + + if (IS_ERR(inode)) + return ERR_CAST(inode); + + if (generation && inode->i_generation != generation) { + iput(inode); + return ERR_PTR(-ESTALE); + } + return d_obtain_alias(inode); +} + +static struct dentry *udf_fh_to_dentry(struct super_block *sb, + struct fid *fid, int fh_len, int fh_type) +{ + if (fh_len < 3 || + (fh_type != FILEID_UDF_WITH_PARENT && + fh_type != FILEID_UDF_WITHOUT_PARENT)) + return NULL; + + return udf_nfs_get_inode(sb, fid->udf.block, fid->udf.partref, + fid->udf.generation); +} + +static struct dentry *udf_fh_to_parent(struct super_block *sb, + struct fid *fid, int fh_len, int fh_type) +{ + if (fh_len < 5 || fh_type != FILEID_UDF_WITH_PARENT) + return NULL; + + return udf_nfs_get_inode(sb, fid->udf.parent_block, + fid->udf.parent_partref, + fid->udf.parent_generation); +} +static int udf_encode_fh(struct inode *inode, __u32 *fh, int *lenp, + struct inode *parent) +{ + int len = *lenp; + struct kernel_lb_addr location = UDF_I(inode)->i_location; + struct fid *fid = (struct fid *)fh; + int type = FILEID_UDF_WITHOUT_PARENT; + + if (parent && (len < 5)) { + *lenp = 5; + return FILEID_INVALID; + } else if (len < 3) { + *lenp = 3; + return FILEID_INVALID; + } + + *lenp = 3; + fid->udf.block = location.logicalBlockNum; + fid->udf.partref = location.partitionReferenceNum; + fid->udf.parent_partref = 0; + fid->udf.generation = inode->i_generation; + + if (parent) { + location = UDF_I(parent)->i_location; + fid->udf.parent_block = location.logicalBlockNum; + fid->udf.parent_partref = location.partitionReferenceNum; + fid->udf.parent_generation = inode->i_generation; + *lenp = 5; + type = FILEID_UDF_WITH_PARENT; + } + + return type; +} + +const struct export_operations udf_export_ops = { + .encode_fh = udf_encode_fh, + .fh_to_dentry = udf_fh_to_dentry, + .fh_to_parent = udf_fh_to_parent, + .get_parent = udf_get_parent, +}; + +const struct inode_operations udf_dir_inode_operations = { + .lookup = udf_lookup, + .create = udf_create, + .link = udf_link, + .unlink = udf_unlink, + .symlink = udf_symlink, + .mkdir = udf_mkdir, + .rmdir = udf_rmdir, + .mknod = udf_mknod, + .rename = udf_rename, + .tmpfile = udf_tmpfile, +}; diff --git a/fs/udf/osta_udf.h b/fs/udf/osta_udf.h new file mode 100644 index 000000000..157de0ec0 --- /dev/null +++ b/fs/udf/osta_udf.h @@ -0,0 +1,305 @@ +/* + * osta_udf.h + * + * This file is based on OSTA UDF(tm) 2.60 (March 1, 2005) + * http://www.osta.org + * + * Copyright (c) 2001-2004 Ben Fennema + * Copyright (c) 2017-2019 Pali Rohár + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions, and the following disclaimer, + * without modification. + * 2. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * Alternatively, this software may be distributed under the terms of the + * GNU Public License ("GPL"). + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/** + * @file + * OSTA-UDF defines and structure definitions + */ + +#include "ecma_167.h" + +#ifndef _OSTA_UDF_H +#define _OSTA_UDF_H 1 + +/* OSTA CS0 Charspec (UDF 2.60 2.1.2) */ +#define UDF_CHAR_SET_TYPE 0 +#define UDF_CHAR_SET_INFO "OSTA Compressed Unicode" + +/* Entity Identifier (UDF 2.60 2.1.5) */ +/* Identifiers (UDF 2.60 2.1.5.2) */ +/* Implementation Use Extended Attribute (UDF 2.60 3.3.4.5) */ +/* Virtual Allocation Table (UDF 1.50 2.2.10) */ +/* Logical Volume Extended Information (UDF 1.50 Errata, DCN 5003, 3.3.4.5.1.3) */ +/* OS2EA (UDF 1.50 3.3.4.5.3.1) */ +/* MacUniqueIDTable (UDF 1.50 3.3.4.5.4.3) */ +/* MacResourceFork (UDF 1.50 3.3.4.5.4.4) */ +#define UDF_ID_DEVELOPER "*Linux UDFFS" +#define UDF_ID_COMPLIANT "*OSTA UDF Compliant" +#define UDF_ID_LV_INFO "*UDF LV Info" +#define UDF_ID_FREE_EA "*UDF FreeEASpace" +#define UDF_ID_FREE_APP_EA "*UDF FreeAppEASpace" +#define UDF_ID_DVD_CGMS "*UDF DVD CGMS Info" +#define UDF_ID_VAT_LVEXTENSION "*UDF VAT LVExtension" +#define UDF_ID_OS2_EA "*UDF OS/2 EA" +#define UDF_ID_OS2_EA_LENGTH "*UDF OS/2 EALength" +#define UDF_ID_MAC_VOLUME "*UDF Mac VolumeInfo" +#define UDF_ID_MAC_FINDER "*UDF Mac FinderInfo" +#define UDF_ID_MAC_UNIQUE "*UDF Mac UniqueIDTable" +#define UDF_ID_MAC_RESOURCE "*UDF Mac ResourceFork" +#define UDF_ID_OS400_DIRINFO "*UDF OS/400 DirInfo" +#define UDF_ID_VIRTUAL "*UDF Virtual Partition" +#define UDF_ID_SPARABLE "*UDF Sparable Partition" +#define UDF_ID_ALLOC "*UDF Virtual Alloc Tbl" +#define UDF_ID_SPARING "*UDF Sparing Table" +#define UDF_ID_METADATA "*UDF Metadata Partition" + +/* Identifier Suffix (UDF 2.60 2.1.5.3) */ +#define DOMAIN_FLAGS_HARD_WRITE_PROTECT 0x01 +#define DOMAIN_FLAGS_SOFT_WRITE_PROTECT 0x02 + +struct domainIdentSuffix { + __le16 UDFRevision; + uint8_t domainFlags; + uint8_t reserved[5]; +} __packed; + +struct UDFIdentSuffix { + __le16 UDFRevision; + uint8_t OSClass; + uint8_t OSIdentifier; + uint8_t reserved[4]; +} __packed; + +struct impIdentSuffix { + uint8_t OSClass; + uint8_t OSIdentifier; + uint8_t impUse[6]; +} __packed; + +struct appIdentSuffix { + uint8_t impUse[8]; +} __packed; + +/* Logical Volume Integrity Descriptor (UDF 2.60 2.2.6) */ +/* Implementation Use (UDF 2.60 2.2.6.4) */ +struct logicalVolIntegrityDescImpUse { + struct regid impIdent; + __le32 numFiles; + __le32 numDirs; + __le16 minUDFReadRev; + __le16 minUDFWriteRev; + __le16 maxUDFWriteRev; + uint8_t impUse[]; +} __packed; + +/* Implementation Use Volume Descriptor (UDF 2.60 2.2.7) */ +/* Implementation Use (UDF 2.60 2.2.7.2) */ +struct impUseVolDescImpUse { + struct charspec LVICharset; + dstring logicalVolIdent[128]; + dstring LVInfo1[36]; + dstring LVInfo2[36]; + dstring LVInfo3[36]; + struct regid impIdent; + uint8_t impUse[128]; +} __packed; + +struct udfPartitionMap2 { + uint8_t partitionMapType; + uint8_t partitionMapLength; + uint8_t reserved1[2]; + struct regid partIdent; + __le16 volSeqNum; + __le16 partitionNum; +} __packed; + +/* Virtual Partition Map (UDF 2.60 2.2.8) */ +struct virtualPartitionMap { + uint8_t partitionMapType; + uint8_t partitionMapLength; + uint8_t reserved1[2]; + struct regid partIdent; + __le16 volSeqNum; + __le16 partitionNum; + uint8_t reserved2[24]; +} __packed; + +/* Sparable Partition Map (UDF 2.60 2.2.9) */ +struct sparablePartitionMap { + uint8_t partitionMapType; + uint8_t partitionMapLength; + uint8_t reserved1[2]; + struct regid partIdent; + __le16 volSeqNum; + __le16 partitionNum; + __le16 packetLength; + uint8_t numSparingTables; + uint8_t reserved2[1]; + __le32 sizeSparingTable; + __le32 locSparingTable[4]; +} __packed; + +/* Metadata Partition Map (UDF 2.60 2.2.10) */ +struct metadataPartitionMap { + uint8_t partitionMapType; + uint8_t partitionMapLength; + uint8_t reserved1[2]; + struct regid partIdent; + __le16 volSeqNum; + __le16 partitionNum; + __le32 metadataFileLoc; + __le32 metadataMirrorFileLoc; + __le32 metadataBitmapFileLoc; + __le32 allocUnitSize; + __le16 alignUnitSize; + uint8_t flags; + uint8_t reserved2[5]; +} __packed; + +/* Virtual Allocation Table (UDF 2.60 2.2.11) */ +struct virtualAllocationTable20 { + __le16 lengthHeader; + __le16 lengthImpUse; + dstring logicalVolIdent[128]; + __le32 previousVATICBLoc; + __le32 numFiles; + __le32 numDirs; + __le16 minUDFReadRev; + __le16 minUDFWriteRev; + __le16 maxUDFWriteRev; + __le16 reserved; + uint8_t impUse[]; + /* __le32 vatEntry[]; */ +} __packed; + +#define ICBTAG_FILE_TYPE_VAT20 0xF8U + +/* Sparing Table (UDF 2.60 2.2.12) */ +struct sparingEntry { + __le32 origLocation; + __le32 mappedLocation; +} __packed; + +struct sparingTable { + struct tag descTag; + struct regid sparingIdent; + __le16 reallocationTableLen; + __le16 reserved; + __le32 sequenceNum; + struct sparingEntry mapEntry[]; +} __packed; + +/* Metadata File (and Metadata Mirror File) (UDF 2.60 2.2.13.1) */ +#define ICBTAG_FILE_TYPE_MAIN 0xFA +#define ICBTAG_FILE_TYPE_MIRROR 0xFB +#define ICBTAG_FILE_TYPE_BITMAP 0xFC + +/* struct long_ad ICB - ADImpUse (UDF 2.60 2.2.4.3) */ +struct allocDescImpUse { + __le16 flags; + uint8_t impUse[4]; +} __packed; + +#define AD_IU_EXT_ERASED 0x0001 + +/* Real-Time Files (UDF 2.60 6.11) */ +#define ICBTAG_FILE_TYPE_REALTIME 0xF9U + +/* Implementation Use Extended Attribute (UDF 2.60 3.3.4.5) */ +/* FreeEASpace (UDF 2.60 3.3.4.5.1.1) */ +struct freeEaSpace { + __le16 headerChecksum; + uint8_t freeEASpace[]; +} __packed; + +/* DVD Copyright Management Information (UDF 2.60 3.3.4.5.1.2) */ +struct DVDCopyrightImpUse { + __le16 headerChecksum; + uint8_t CGMSInfo; + uint8_t dataType; + uint8_t protectionSystemInfo[4]; +} __packed; + +/* Logical Volume Extended Information (UDF 1.50 Errata, DCN 5003, 3.3.4.5.1.3) */ +struct LVExtensionEA { + __le16 headerChecksum; + __le64 verificationID; + __le32 numFiles; + __le32 numDirs; + dstring logicalVolIdent[128]; +} __packed; + +/* Application Use Extended Attribute (UDF 2.60 3.3.4.6) */ +/* FreeAppEASpace (UDF 2.60 3.3.4.6.1) */ +struct freeAppEASpace { + __le16 headerChecksum; + uint8_t freeEASpace[]; +} __packed; + +/* UDF Defined System Stream (UDF 2.60 3.3.7) */ +#define UDF_ID_UNIQUE_ID "*UDF Unique ID Mapping Data" +#define UDF_ID_NON_ALLOC "*UDF Non-Allocatable Space" +#define UDF_ID_POWER_CAL "*UDF Power Cal Table" +#define UDF_ID_BACKUP "*UDF Backup" + +/* UDF Defined Non-System Streams (UDF 2.60 3.3.8) */ +#define UDF_ID_MAC_RESOURCE_FORK_STREAM "*UDF Macintosh Resource Fork" +/* #define UDF_ID_OS2_EA "*UDF OS/2 EA" */ +#define UDF_ID_NT_ACL "*UDF NT ACL" +#define UDF_ID_UNIX_ACL "*UDF UNIX ACL" + +/* Operating System Identifiers (UDF 2.60 6.3) */ +#define UDF_OS_CLASS_UNDEF 0x00U +#define UDF_OS_CLASS_DOS 0x01U +#define UDF_OS_CLASS_OS2 0x02U +#define UDF_OS_CLASS_MAC 0x03U +#define UDF_OS_CLASS_UNIX 0x04U +#define UDF_OS_CLASS_WIN9X 0x05U +#define UDF_OS_CLASS_WINNT 0x06U +#define UDF_OS_CLASS_OS400 0x07U +#define UDF_OS_CLASS_BEOS 0x08U +#define UDF_OS_CLASS_WINCE 0x09U + +#define UDF_OS_ID_UNDEF 0x00U +#define UDF_OS_ID_DOS 0x00U +#define UDF_OS_ID_OS2 0x00U +#define UDF_OS_ID_MAC 0x00U +#define UDF_OS_ID_MAX_OSX 0x01U +#define UDF_OS_ID_UNIX 0x00U +#define UDF_OS_ID_AIX 0x01U +#define UDF_OS_ID_SOLARIS 0x02U +#define UDF_OS_ID_HPUX 0x03U +#define UDF_OS_ID_IRIX 0x04U +#define UDF_OS_ID_LINUX 0x05U +#define UDF_OS_ID_MKLINUX 0x06U +#define UDF_OS_ID_FREEBSD 0x07U +#define UDF_OS_ID_NETBSD 0x08U +#define UDF_OS_ID_WIN9X 0x00U +#define UDF_OS_ID_WINNT 0x00U +#define UDF_OS_ID_OS400 0x00U +#define UDF_OS_ID_BEOS 0x00U +#define UDF_OS_ID_WINCE 0x00U + +#endif /* _OSTA_UDF_H */ diff --git a/fs/udf/partition.c b/fs/udf/partition.c new file mode 100644 index 000000000..ad8dcedca --- /dev/null +++ b/fs/udf/partition.c @@ -0,0 +1,340 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * partition.c + * + * PURPOSE + * Partition handling routines for the OSTA-UDF(tm) filesystem. + * + * COPYRIGHT + * (C) 1998-2001 Ben Fennema + * + * HISTORY + * + * 12/06/98 blf Created file. + * + */ + +#include "udfdecl.h" +#include "udf_sb.h" +#include "udf_i.h" + +#include +#include +#include + +uint32_t udf_get_pblock(struct super_block *sb, uint32_t block, + uint16_t partition, uint32_t offset) +{ + struct udf_sb_info *sbi = UDF_SB(sb); + struct udf_part_map *map; + if (partition >= sbi->s_partitions) { + udf_debug("block=%u, partition=%u, offset=%u: invalid partition\n", + block, partition, offset); + return 0xFFFFFFFF; + } + map = &sbi->s_partmaps[partition]; + if (map->s_partition_func) + return map->s_partition_func(sb, block, partition, offset); + else + return map->s_partition_root + block + offset; +} + +uint32_t udf_get_pblock_virt15(struct super_block *sb, uint32_t block, + uint16_t partition, uint32_t offset) +{ + struct buffer_head *bh = NULL; + uint32_t newblock; + uint32_t index; + uint32_t loc; + struct udf_sb_info *sbi = UDF_SB(sb); + struct udf_part_map *map; + struct udf_virtual_data *vdata; + struct udf_inode_info *iinfo = UDF_I(sbi->s_vat_inode); + int err; + + map = &sbi->s_partmaps[partition]; + vdata = &map->s_type_specific.s_virtual; + + if (block >= vdata->s_num_entries) { + udf_debug("Trying to access block beyond end of VAT (%u max %u)\n", + block, vdata->s_num_entries); + return 0xFFFFFFFF; + } + + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { + loc = le32_to_cpu(((__le32 *)(iinfo->i_data + + vdata->s_start_offset))[block]); + goto translate; + } + index = (sb->s_blocksize - vdata->s_start_offset) / sizeof(uint32_t); + if (block >= index) { + block -= index; + newblock = 1 + (block / (sb->s_blocksize / sizeof(uint32_t))); + index = block % (sb->s_blocksize / sizeof(uint32_t)); + } else { + newblock = 0; + index = vdata->s_start_offset / sizeof(uint32_t) + block; + } + + bh = udf_bread(sbi->s_vat_inode, newblock, 0, &err); + if (!bh) { + udf_debug("get_pblock(UDF_VIRTUAL_MAP:%p,%u,%u)\n", + sb, block, partition); + return 0xFFFFFFFF; + } + + loc = le32_to_cpu(((__le32 *)bh->b_data)[index]); + + brelse(bh); + +translate: + if (iinfo->i_location.partitionReferenceNum == partition) { + udf_debug("recursive call to udf_get_pblock!\n"); + return 0xFFFFFFFF; + } + + return udf_get_pblock(sb, loc, + iinfo->i_location.partitionReferenceNum, + offset); +} + +inline uint32_t udf_get_pblock_virt20(struct super_block *sb, uint32_t block, + uint16_t partition, uint32_t offset) +{ + return udf_get_pblock_virt15(sb, block, partition, offset); +} + +uint32_t udf_get_pblock_spar15(struct super_block *sb, uint32_t block, + uint16_t partition, uint32_t offset) +{ + int i; + struct sparingTable *st = NULL; + struct udf_sb_info *sbi = UDF_SB(sb); + struct udf_part_map *map; + uint32_t packet; + struct udf_sparing_data *sdata; + + map = &sbi->s_partmaps[partition]; + sdata = &map->s_type_specific.s_sparing; + packet = (block + offset) & ~(sdata->s_packet_len - 1); + + for (i = 0; i < 4; i++) { + if (sdata->s_spar_map[i] != NULL) { + st = (struct sparingTable *) + sdata->s_spar_map[i]->b_data; + break; + } + } + + if (st) { + for (i = 0; i < le16_to_cpu(st->reallocationTableLen); i++) { + struct sparingEntry *entry = &st->mapEntry[i]; + u32 origLoc = le32_to_cpu(entry->origLocation); + if (origLoc >= 0xFFFFFFF0) + break; + else if (origLoc == packet) + return le32_to_cpu(entry->mappedLocation) + + ((block + offset) & + (sdata->s_packet_len - 1)); + else if (origLoc > packet) + break; + } + } + + return map->s_partition_root + block + offset; +} + +int udf_relocate_blocks(struct super_block *sb, long old_block, long *new_block) +{ + struct udf_sparing_data *sdata; + struct sparingTable *st = NULL; + struct sparingEntry mapEntry; + uint32_t packet; + int i, j, k, l; + struct udf_sb_info *sbi = UDF_SB(sb); + u16 reallocationTableLen; + struct buffer_head *bh; + int ret = 0; + + mutex_lock(&sbi->s_alloc_mutex); + for (i = 0; i < sbi->s_partitions; i++) { + struct udf_part_map *map = &sbi->s_partmaps[i]; + if (old_block > map->s_partition_root && + old_block < map->s_partition_root + map->s_partition_len) { + sdata = &map->s_type_specific.s_sparing; + packet = (old_block - map->s_partition_root) & + ~(sdata->s_packet_len - 1); + + for (j = 0; j < 4; j++) + if (sdata->s_spar_map[j] != NULL) { + st = (struct sparingTable *) + sdata->s_spar_map[j]->b_data; + break; + } + + if (!st) { + ret = 1; + goto out; + } + + reallocationTableLen = + le16_to_cpu(st->reallocationTableLen); + for (k = 0; k < reallocationTableLen; k++) { + struct sparingEntry *entry = &st->mapEntry[k]; + u32 origLoc = le32_to_cpu(entry->origLocation); + + if (origLoc == 0xFFFFFFFF) { + for (; j < 4; j++) { + int len; + bh = sdata->s_spar_map[j]; + if (!bh) + continue; + + st = (struct sparingTable *) + bh->b_data; + entry->origLocation = + cpu_to_le32(packet); + len = + sizeof(struct sparingTable) + + reallocationTableLen * + sizeof(struct sparingEntry); + udf_update_tag((char *)st, len); + mark_buffer_dirty(bh); + } + *new_block = le32_to_cpu( + entry->mappedLocation) + + ((old_block - + map->s_partition_root) & + (sdata->s_packet_len - 1)); + ret = 0; + goto out; + } else if (origLoc == packet) { + *new_block = le32_to_cpu( + entry->mappedLocation) + + ((old_block - + map->s_partition_root) & + (sdata->s_packet_len - 1)); + ret = 0; + goto out; + } else if (origLoc > packet) + break; + } + + for (l = k; l < reallocationTableLen; l++) { + struct sparingEntry *entry = &st->mapEntry[l]; + u32 origLoc = le32_to_cpu(entry->origLocation); + + if (origLoc != 0xFFFFFFFF) + continue; + + for (; j < 4; j++) { + bh = sdata->s_spar_map[j]; + if (!bh) + continue; + + st = (struct sparingTable *)bh->b_data; + mapEntry = st->mapEntry[l]; + mapEntry.origLocation = + cpu_to_le32(packet); + memmove(&st->mapEntry[k + 1], + &st->mapEntry[k], + (l - k) * + sizeof(struct sparingEntry)); + st->mapEntry[k] = mapEntry; + udf_update_tag((char *)st, + sizeof(struct sparingTable) + + reallocationTableLen * + sizeof(struct sparingEntry)); + mark_buffer_dirty(bh); + } + *new_block = + le32_to_cpu( + st->mapEntry[k].mappedLocation) + + ((old_block - map->s_partition_root) & + (sdata->s_packet_len - 1)); + ret = 0; + goto out; + } + + ret = 1; + goto out; + } /* if old_block */ + } + + if (i == sbi->s_partitions) { + /* outside of partitions */ + /* for now, fail =) */ + ret = 1; + } + +out: + mutex_unlock(&sbi->s_alloc_mutex); + return ret; +} + +static uint32_t udf_try_read_meta(struct inode *inode, uint32_t block, + uint16_t partition, uint32_t offset) +{ + struct super_block *sb = inode->i_sb; + struct udf_part_map *map; + struct kernel_lb_addr eloc; + uint32_t elen; + sector_t ext_offset; + struct extent_position epos = {}; + uint32_t phyblock; + int8_t etype; + int err = 0; + + err = inode_bmap(inode, block, &epos, &eloc, &elen, &ext_offset, &etype); + if (err <= 0 || etype != (EXT_RECORDED_ALLOCATED >> 30)) + phyblock = 0xFFFFFFFF; + else { + map = &UDF_SB(sb)->s_partmaps[partition]; + /* map to sparable/physical partition desc */ + phyblock = udf_get_pblock(sb, eloc.logicalBlockNum, + map->s_type_specific.s_metadata.s_phys_partition_ref, + ext_offset + offset); + } + + brelse(epos.bh); + return phyblock; +} + +uint32_t udf_get_pblock_meta25(struct super_block *sb, uint32_t block, + uint16_t partition, uint32_t offset) +{ + struct udf_sb_info *sbi = UDF_SB(sb); + struct udf_part_map *map; + struct udf_meta_data *mdata; + uint32_t retblk; + struct inode *inode; + + udf_debug("READING from METADATA\n"); + + map = &sbi->s_partmaps[partition]; + mdata = &map->s_type_specific.s_metadata; + inode = mdata->s_metadata_fe ? : mdata->s_mirror_fe; + + if (!inode) + return 0xFFFFFFFF; + + retblk = udf_try_read_meta(inode, block, partition, offset); + if (retblk == 0xFFFFFFFF && mdata->s_metadata_fe) { + udf_warn(sb, "error reading from METADATA, trying to read from MIRROR\n"); + if (!(mdata->s_flags & MF_MIRROR_FE_LOADED)) { + mdata->s_mirror_fe = udf_find_metadata_inode_efe(sb, + mdata->s_mirror_file_loc, + mdata->s_phys_partition_ref); + if (IS_ERR(mdata->s_mirror_fe)) + mdata->s_mirror_fe = NULL; + mdata->s_flags |= MF_MIRROR_FE_LOADED; + } + + inode = mdata->s_mirror_fe; + if (!inode) + return 0xFFFFFFFF; + retblk = udf_try_read_meta(inode, block, partition, offset); + } + + return retblk; +} diff --git a/fs/udf/super.c b/fs/udf/super.c new file mode 100644 index 000000000..2ba5973ef --- /dev/null +++ b/fs/udf/super.c @@ -0,0 +1,2584 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * super.c + * + * PURPOSE + * Super block routines for the OSTA-UDF(tm) filesystem. + * + * DESCRIPTION + * OSTA-UDF(tm) = Optical Storage Technology Association + * Universal Disk Format. + * + * This code is based on version 2.00 of the UDF specification, + * and revision 3 of the ECMA 167 standard [equivalent to ISO 13346]. + * http://www.osta.org/ + * https://www.ecma.ch/ + * https://www.iso.org/ + * + * COPYRIGHT + * (C) 1998 Dave Boynton + * (C) 1998-2004 Ben Fennema + * (C) 2000 Stelias Computing Inc + * + * HISTORY + * + * 09/24/98 dgb changed to allow compiling outside of kernel, and + * added some debugging. + * 10/01/98 dgb updated to allow (some) possibility of compiling w/2.0.34 + * 10/16/98 attempting some multi-session support + * 10/17/98 added freespace count for "df" + * 11/11/98 gr added novrs option + * 11/26/98 dgb added fileset,anchor mount options + * 12/06/98 blf really hosed things royally. vat/sparing support. sequenced + * vol descs. rewrote option handling based on isofs + * 12/20/98 find the free space bitmap (if it exists) + */ + +#include "udfdecl.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "udf_sb.h" +#include "udf_i.h" + +#include +#include + +enum { + VDS_POS_PRIMARY_VOL_DESC, + VDS_POS_UNALLOC_SPACE_DESC, + VDS_POS_LOGICAL_VOL_DESC, + VDS_POS_IMP_USE_VOL_DESC, + VDS_POS_LENGTH +}; + +#define VSD_FIRST_SECTOR_OFFSET 32768 +#define VSD_MAX_SECTOR_OFFSET 0x800000 + +/* + * Maximum number of Terminating Descriptor / Logical Volume Integrity + * Descriptor redirections. The chosen numbers are arbitrary - just that we + * hopefully don't limit any real use of rewritten inode on write-once media + * but avoid looping for too long on corrupted media. + */ +#define UDF_MAX_TD_NESTING 64 +#define UDF_MAX_LVID_NESTING 1000 + +enum { UDF_MAX_LINKS = 0xffff }; +/* + * We limit filesize to 4TB. This is arbitrary as the on-disk format supports + * more but because the file space is described by a linked list of extents, + * each of which can have at most 1GB, the creation and handling of extents + * gets unusably slow beyond certain point... + */ +#define UDF_MAX_FILESIZE (1ULL << 42) + +/* These are the "meat" - everything else is stuffing */ +static int udf_fill_super(struct super_block *sb, struct fs_context *fc); +static void udf_put_super(struct super_block *); +static int udf_sync_fs(struct super_block *, int); +static void udf_load_logicalvolint(struct super_block *, struct kernel_extent_ad); +static void udf_open_lvid(struct super_block *); +static void udf_close_lvid(struct super_block *); +static unsigned int udf_count_free(struct super_block *); +static int udf_statfs(struct dentry *, struct kstatfs *); +static int udf_show_options(struct seq_file *, struct dentry *); +static int udf_init_fs_context(struct fs_context *fc); +static int udf_parse_param(struct fs_context *fc, struct fs_parameter *param); +static int udf_reconfigure(struct fs_context *fc); +static void udf_free_fc(struct fs_context *fc); +static const struct fs_parameter_spec udf_param_spec[]; + +struct logicalVolIntegrityDescImpUse *udf_sb_lvidiu(struct super_block *sb) +{ + struct logicalVolIntegrityDesc *lvid; + unsigned int partnum; + unsigned int offset; + + if (!UDF_SB(sb)->s_lvid_bh) + return NULL; + lvid = (struct logicalVolIntegrityDesc *)UDF_SB(sb)->s_lvid_bh->b_data; + partnum = le32_to_cpu(lvid->numOfPartitions); + /* The offset is to skip freeSpaceTable and sizeTable arrays */ + offset = partnum * 2 * sizeof(uint32_t); + return (struct logicalVolIntegrityDescImpUse *) + (((uint8_t *)(lvid + 1)) + offset); +} + +/* UDF filesystem type */ +static int udf_get_tree(struct fs_context *fc) +{ + return get_tree_bdev(fc, udf_fill_super); +} + +static const struct fs_context_operations udf_context_ops = { + .parse_param = udf_parse_param, + .get_tree = udf_get_tree, + .reconfigure = udf_reconfigure, + .free = udf_free_fc, +}; + +static struct file_system_type udf_fstype = { + .owner = THIS_MODULE, + .name = "udf", + .kill_sb = kill_block_super, + .fs_flags = FS_REQUIRES_DEV, + .init_fs_context = udf_init_fs_context, + .parameters = udf_param_spec, +}; +MODULE_ALIAS_FS("udf"); + +static struct kmem_cache *udf_inode_cachep; + +static struct inode *udf_alloc_inode(struct super_block *sb) +{ + struct udf_inode_info *ei; + ei = alloc_inode_sb(sb, udf_inode_cachep, GFP_KERNEL); + if (!ei) + return NULL; + + ei->i_unique = 0; + ei->i_lenExtents = 0; + ei->i_lenStreams = 0; + ei->i_next_alloc_block = 0; + ei->i_next_alloc_goal = 0; + ei->i_strat4096 = 0; + ei->i_streamdir = 0; + ei->i_hidden = 0; + init_rwsem(&ei->i_data_sem); + ei->cached_extent.lstart = -1; + spin_lock_init(&ei->i_extent_cache_lock); + inode_set_iversion(&ei->vfs_inode, 1); + mmb_init(&ei->i_metadata_bhs, &ei->vfs_inode.i_data); + + return &ei->vfs_inode; +} + +static void udf_free_in_core_inode(struct inode *inode) +{ + kmem_cache_free(udf_inode_cachep, UDF_I(inode)); +} + +static void init_once(void *foo) +{ + struct udf_inode_info *ei = foo; + + ei->i_data = NULL; + inode_init_once(&ei->vfs_inode); +} + +static int __init init_inodecache(void) +{ + udf_inode_cachep = kmem_cache_create("udf_inode_cache", + sizeof(struct udf_inode_info), + 0, (SLAB_RECLAIM_ACCOUNT | + SLAB_ACCOUNT), + init_once); + if (!udf_inode_cachep) + return -ENOMEM; + return 0; +} + +static void destroy_inodecache(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(udf_inode_cachep); +} + +/* Superblock operations */ +static const struct super_operations udf_sb_ops = { + .alloc_inode = udf_alloc_inode, + .free_inode = udf_free_in_core_inode, + .write_inode = udf_write_inode, + .sync_inode_metadata = udf_sync_inode_metadata, + .evict_inode = udf_evict_inode, + .put_super = udf_put_super, + .sync_fs = udf_sync_fs, + .statfs = udf_statfs, + .show_options = udf_show_options, +}; + +struct udf_options { + unsigned int blocksize; + unsigned int session; + unsigned int lastblock; + unsigned int anchor; + unsigned int flags; + umode_t umask; + kgid_t gid; + kuid_t uid; + umode_t fmode; + umode_t dmode; + struct nls_table *nls_map; +}; + +/* + * UDF has historically preserved prior mount options across + * a remount, so copy those here if remounting, otherwise set + * initial mount defaults. + */ +static void udf_init_options(struct fs_context *fc, struct udf_options *uopt) +{ + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE) { + struct super_block *sb = fc->root->d_sb; + struct udf_sb_info *sbi = UDF_SB(sb); + + uopt->flags = sbi->s_flags; + uopt->uid = sbi->s_uid; + uopt->gid = sbi->s_gid; + uopt->umask = sbi->s_umask; + uopt->fmode = sbi->s_fmode; + uopt->dmode = sbi->s_dmode; + uopt->nls_map = NULL; + } else { + uopt->flags = (1 << UDF_FLAG_USE_AD_IN_ICB) | + (1 << UDF_FLAG_STRICT); + /* + * By default we'll use overflow[ug]id when UDF + * inode [ug]id == -1 + */ + uopt->uid = make_kuid(current_user_ns(), overflowuid); + uopt->gid = make_kgid(current_user_ns(), overflowgid); + uopt->umask = 0; + uopt->fmode = UDF_INVALID_MODE; + uopt->dmode = UDF_INVALID_MODE; + uopt->nls_map = NULL; + uopt->session = 0xFFFFFFFF; + } +} + +static int udf_init_fs_context(struct fs_context *fc) +{ + struct udf_options *uopt; + + uopt = kzalloc_obj(*uopt); + if (!uopt) + return -ENOMEM; + + udf_init_options(fc, uopt); + + fc->fs_private = uopt; + fc->ops = &udf_context_ops; + + return 0; +} + +static void udf_free_fc(struct fs_context *fc) +{ + struct udf_options *uopt = fc->fs_private; + + unload_nls(uopt->nls_map); + kfree(fc->fs_private); +} + +static int __init init_udf_fs(void) +{ + int err; + + err = init_inodecache(); + if (err) + goto out1; + err = register_filesystem(&udf_fstype); + if (err) + goto out; + + return 0; + +out: + destroy_inodecache(); + +out1: + return err; +} + +static void __exit exit_udf_fs(void) +{ + unregister_filesystem(&udf_fstype); + destroy_inodecache(); +} + +static int udf_sb_alloc_partition_maps(struct super_block *sb, u32 count) +{ + struct udf_sb_info *sbi = UDF_SB(sb); + + sbi->s_partmaps = kzalloc_objs(*sbi->s_partmaps, count); + if (!sbi->s_partmaps) { + sbi->s_partitions = 0; + return -ENOMEM; + } + + sbi->s_partitions = count; + return 0; +} + +static void udf_sb_free_bitmap(struct udf_bitmap *bitmap) +{ + int i; + int nr_groups = bitmap->s_nr_groups; + + for (i = 0; i < nr_groups; i++) + if (!IS_ERR_OR_NULL(bitmap->s_block_bitmap[i])) + brelse(bitmap->s_block_bitmap[i]); + + kvfree(bitmap); +} + +static void udf_free_partition(struct udf_part_map *map) +{ + int i; + struct udf_meta_data *mdata; + + if (map->s_partition_flags & UDF_PART_FLAG_UNALLOC_TABLE) + iput(map->s_uspace.s_table); + if (map->s_partition_flags & UDF_PART_FLAG_UNALLOC_BITMAP) + udf_sb_free_bitmap(map->s_uspace.s_bitmap); + if (map->s_partition_type == UDF_SPARABLE_MAP15) + for (i = 0; i < 4; i++) + brelse(map->s_type_specific.s_sparing.s_spar_map[i]); + else if (map->s_partition_type == UDF_METADATA_MAP25) { + mdata = &map->s_type_specific.s_metadata; + iput(mdata->s_metadata_fe); + mdata->s_metadata_fe = NULL; + + iput(mdata->s_mirror_fe); + mdata->s_mirror_fe = NULL; + + iput(mdata->s_bitmap_fe); + mdata->s_bitmap_fe = NULL; + } +} + +static void udf_sb_free_partitions(struct super_block *sb) +{ + struct udf_sb_info *sbi = UDF_SB(sb); + int i; + + if (!sbi->s_partmaps) + return; + for (i = 0; i < sbi->s_partitions; i++) + udf_free_partition(&sbi->s_partmaps[i]); + kfree(sbi->s_partmaps); + sbi->s_partmaps = NULL; +} + +static int udf_show_options(struct seq_file *seq, struct dentry *root) +{ + struct super_block *sb = root->d_sb; + struct udf_sb_info *sbi = UDF_SB(sb); + + if (!UDF_QUERY_FLAG(sb, UDF_FLAG_STRICT)) + seq_puts(seq, ",nostrict"); + if (UDF_QUERY_FLAG(sb, UDF_FLAG_BLOCKSIZE_SET)) + seq_printf(seq, ",bs=%lu", sb->s_blocksize); + if (UDF_QUERY_FLAG(sb, UDF_FLAG_UNHIDE)) + seq_puts(seq, ",unhide"); + if (UDF_QUERY_FLAG(sb, UDF_FLAG_UNDELETE)) + seq_puts(seq, ",undelete"); + if (!UDF_QUERY_FLAG(sb, UDF_FLAG_USE_AD_IN_ICB)) + seq_puts(seq, ",noadinicb"); + if (UDF_QUERY_FLAG(sb, UDF_FLAG_USE_SHORT_AD)) + seq_puts(seq, ",shortad"); + if (UDF_QUERY_FLAG(sb, UDF_FLAG_UID_FORGET)) + seq_puts(seq, ",uid=forget"); + if (UDF_QUERY_FLAG(sb, UDF_FLAG_GID_FORGET)) + seq_puts(seq, ",gid=forget"); + if (UDF_QUERY_FLAG(sb, UDF_FLAG_UID_SET)) + seq_printf(seq, ",uid=%u", from_kuid(&init_user_ns, sbi->s_uid)); + if (UDF_QUERY_FLAG(sb, UDF_FLAG_GID_SET)) + seq_printf(seq, ",gid=%u", from_kgid(&init_user_ns, sbi->s_gid)); + if (sbi->s_umask != 0) + seq_printf(seq, ",umask=%ho", sbi->s_umask); + if (sbi->s_fmode != UDF_INVALID_MODE) + seq_printf(seq, ",mode=%ho", sbi->s_fmode); + if (sbi->s_dmode != UDF_INVALID_MODE) + seq_printf(seq, ",dmode=%ho", sbi->s_dmode); + if (UDF_QUERY_FLAG(sb, UDF_FLAG_SESSION_SET)) + seq_printf(seq, ",session=%d", sbi->s_session); + if (UDF_QUERY_FLAG(sb, UDF_FLAG_LASTBLOCK_SET)) + seq_printf(seq, ",lastblock=%u", sbi->s_last_block); + if (sbi->s_anchor != 0) + seq_printf(seq, ",anchor=%u", sbi->s_anchor); + if (sbi->s_nls_map) + seq_printf(seq, ",iocharset=%s", sbi->s_nls_map->charset); + else + seq_puts(seq, ",iocharset=utf8"); + + return 0; +} + +/* + * udf_parse_param + * + * PURPOSE + * Parse mount options. + * + * DESCRIPTION + * The following mount options are supported: + * + * gid= Set the default group. + * umask= Set the default umask. + * mode= Set the default file permissions. + * dmode= Set the default directory permissions. + * uid= Set the default user. + * bs= Set the block size. + * unhide Show otherwise hidden files. + * undelete Show deleted files in lists. + * adinicb Embed data in the inode (default) + * noadinicb Don't embed data in the inode + * shortad Use short ad's + * longad Use long ad's (default) + * nostrict Unset strict conformance + * iocharset= Set the NLS character set + * + * The remaining are for debugging and disaster recovery: + * + * novrs Skip volume sequence recognition + * + * The following expect a offset from 0. + * + * session= Set the CDROM session (default= last session) + * anchor= Override standard anchor location. (default= 256) + * volume= Override the VolumeDesc location. (unused) + * partition= Override the PartitionDesc location. (unused) + * lastblock= Set the last block of the filesystem/ + * + * The following expect a offset from the partition root. + * + * fileset= Override the fileset block location. (unused) + * rootdir= Override the root directory location. (unused) + * WARNING: overriding the rootdir to a non-directory may + * yield highly unpredictable results. + * + * PRE-CONDITIONS + * fc fs_context with pointer to mount options variable. + * param Pointer to fs_parameter being parsed. + * + * POST-CONDITIONS + * 0 Mount options parsed okay. + * errno Error parsing mount options. + * + * HISTORY + * July 1, 1997 - Andrew E. Mileski + * Written, tested, and released. + */ + +enum { + Opt_novrs, Opt_nostrict, Opt_bs, Opt_unhide, Opt_undelete, + Opt_noadinicb, Opt_adinicb, Opt_shortad, Opt_longad, + Opt_gid, Opt_uid, Opt_umask, Opt_session, Opt_lastblock, + Opt_anchor, Opt_volume, Opt_partition, Opt_fileset, + Opt_rootdir, Opt_utf8, Opt_iocharset, Opt_err, Opt_fmode, Opt_dmode +}; + +static const struct fs_parameter_spec udf_param_spec[] = { + fsparam_flag ("novrs", Opt_novrs), + fsparam_flag ("nostrict", Opt_nostrict), + fsparam_u32 ("bs", Opt_bs), + fsparam_flag ("unhide", Opt_unhide), + fsparam_flag ("undelete", Opt_undelete), + fsparam_flag_no ("adinicb", Opt_adinicb), + fsparam_flag ("shortad", Opt_shortad), + fsparam_flag ("longad", Opt_longad), + fsparam_string ("gid", Opt_gid), + fsparam_string ("uid", Opt_uid), + fsparam_u32 ("umask", Opt_umask), + fsparam_u32 ("session", Opt_session), + fsparam_u32 ("lastblock", Opt_lastblock), + fsparam_u32 ("anchor", Opt_anchor), + fsparam_u32 ("volume", Opt_volume), + fsparam_u32 ("partition", Opt_partition), + fsparam_u32 ("fileset", Opt_fileset), + fsparam_u32 ("rootdir", Opt_rootdir), + fsparam_flag ("utf8", Opt_utf8), + fsparam_string ("iocharset", Opt_iocharset), + fsparam_u32 ("mode", Opt_fmode), + fsparam_u32 ("dmode", Opt_dmode), + {} + }; + +static int udf_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + unsigned int uv; + unsigned int n; + struct udf_options *uopt = fc->fs_private; + struct fs_parse_result result; + int token; + bool remount = (fc->purpose & FS_CONTEXT_FOR_RECONFIGURE); + + token = fs_parse(fc, udf_param_spec, param, &result); + if (token < 0) + return token; + + switch (token) { + case Opt_novrs: + uopt->flags |= (1 << UDF_FLAG_NOVRS); + break; + case Opt_bs: + n = result.uint_32; + if (n != 512 && n != 1024 && n != 2048 && n != 4096) + return -EINVAL; + uopt->blocksize = n; + uopt->flags |= (1 << UDF_FLAG_BLOCKSIZE_SET); + break; + case Opt_unhide: + uopt->flags |= (1 << UDF_FLAG_UNHIDE); + break; + case Opt_undelete: + uopt->flags |= (1 << UDF_FLAG_UNDELETE); + break; + case Opt_adinicb: + if (result.negated) + uopt->flags &= ~(1 << UDF_FLAG_USE_AD_IN_ICB); + else + uopt->flags |= (1 << UDF_FLAG_USE_AD_IN_ICB); + break; + case Opt_shortad: + uopt->flags |= (1 << UDF_FLAG_USE_SHORT_AD); + break; + case Opt_longad: + uopt->flags &= ~(1 << UDF_FLAG_USE_SHORT_AD); + break; + case Opt_gid: + if (kstrtoint(param->string, 10, &uv) == 0) { + kgid_t gid = make_kgid(current_user_ns(), uv); + if (!gid_valid(gid)) + return -EINVAL; + uopt->gid = gid; + uopt->flags |= (1 << UDF_FLAG_GID_SET); + } else if (!strcmp(param->string, "forget")) { + uopt->flags |= (1 << UDF_FLAG_GID_FORGET); + } else if (!strcmp(param->string, "ignore")) { + /* this option is superseded by gid= */ + ; + } else { + return -EINVAL; + } + break; + case Opt_uid: + if (kstrtoint(param->string, 10, &uv) == 0) { + kuid_t uid = make_kuid(current_user_ns(), uv); + if (!uid_valid(uid)) + return -EINVAL; + uopt->uid = uid; + uopt->flags |= (1 << UDF_FLAG_UID_SET); + } else if (!strcmp(param->string, "forget")) { + uopt->flags |= (1 << UDF_FLAG_UID_FORGET); + } else if (!strcmp(param->string, "ignore")) { + /* this option is superseded by uid= */ + ; + } else { + return -EINVAL; + } + break; + case Opt_umask: + uopt->umask = result.uint_32; + break; + case Opt_nostrict: + uopt->flags &= ~(1 << UDF_FLAG_STRICT); + break; + case Opt_session: + uopt->session = result.uint_32; + if (!remount) + uopt->flags |= (1 << UDF_FLAG_SESSION_SET); + break; + case Opt_lastblock: + uopt->lastblock = result.uint_32; + if (!remount) + uopt->flags |= (1 << UDF_FLAG_LASTBLOCK_SET); + break; + case Opt_anchor: + uopt->anchor = result.uint_32; + break; + case Opt_volume: + case Opt_partition: + case Opt_fileset: + case Opt_rootdir: + /* Ignored (never implemented properly) */ + break; + case Opt_utf8: + if (!remount) { + unload_nls(uopt->nls_map); + uopt->nls_map = NULL; + } + break; + case Opt_iocharset: + if (!remount) { + unload_nls(uopt->nls_map); + uopt->nls_map = NULL; + } + /* When nls_map is not loaded then UTF-8 is used */ + if (!remount && strcmp(param->string, "utf8") != 0) { + uopt->nls_map = load_nls(param->string); + if (!uopt->nls_map) { + errorf(fc, "iocharset %s not found", + param->string); + return -EINVAL; + } + } + break; + case Opt_fmode: + uopt->fmode = result.uint_32 & 0777; + break; + case Opt_dmode: + uopt->dmode = result.uint_32 & 0777; + break; + default: + return -EINVAL; + } + return 0; +} + +static int udf_reconfigure(struct fs_context *fc) +{ + struct udf_options *uopt = fc->fs_private; + struct super_block *sb = fc->root->d_sb; + struct udf_sb_info *sbi = UDF_SB(sb); + int readonly = fc->sb_flags & SB_RDONLY; + int error = 0; + + if (!readonly && UDF_QUERY_FLAG(sb, UDF_FLAG_RW_INCOMPAT)) + return -EACCES; + + sync_filesystem(sb); + + write_lock(&sbi->s_cred_lock); + sbi->s_flags = uopt->flags; + sbi->s_uid = uopt->uid; + sbi->s_gid = uopt->gid; + sbi->s_umask = uopt->umask; + sbi->s_fmode = uopt->fmode; + sbi->s_dmode = uopt->dmode; + write_unlock(&sbi->s_cred_lock); + + if (readonly == sb_rdonly(sb)) + goto out_unlock; + + if (readonly) + udf_close_lvid(sb); + else + udf_open_lvid(sb); + +out_unlock: + return error; +} + +/* + * Check VSD descriptor. Returns -1 in case we are at the end of volume + * recognition area, 0 if the descriptor is valid but non-interesting, 1 if + * we found one of NSR descriptors we are looking for. + */ +static int identify_vsd(const struct volStructDesc *vsd) +{ + int ret = 0; + + if (!memcmp(vsd->stdIdent, VSD_STD_ID_CD001, VSD_STD_ID_LEN)) { + switch (vsd->structType) { + case 0: + udf_debug("ISO9660 Boot Record found\n"); + break; + case 1: + udf_debug("ISO9660 Primary Volume Descriptor found\n"); + break; + case 2: + udf_debug("ISO9660 Supplementary Volume Descriptor found\n"); + break; + case 3: + udf_debug("ISO9660 Volume Partition Descriptor found\n"); + break; + case 255: + udf_debug("ISO9660 Volume Descriptor Set Terminator found\n"); + break; + default: + udf_debug("ISO9660 VRS (%u) found\n", vsd->structType); + break; + } + } else if (!memcmp(vsd->stdIdent, VSD_STD_ID_BEA01, VSD_STD_ID_LEN)) + ; /* ret = 0 */ + else if (!memcmp(vsd->stdIdent, VSD_STD_ID_NSR02, VSD_STD_ID_LEN)) + ret = 1; + else if (!memcmp(vsd->stdIdent, VSD_STD_ID_NSR03, VSD_STD_ID_LEN)) + ret = 1; + else if (!memcmp(vsd->stdIdent, VSD_STD_ID_BOOT2, VSD_STD_ID_LEN)) + ; /* ret = 0 */ + else if (!memcmp(vsd->stdIdent, VSD_STD_ID_CDW02, VSD_STD_ID_LEN)) + ; /* ret = 0 */ + else { + /* TEA01 or invalid id : end of volume recognition area */ + ret = -1; + } + + return ret; +} + +/* + * Check Volume Structure Descriptors (ECMA 167 2/9.1) + * We also check any "CD-ROM Volume Descriptor Set" (ECMA 167 2/8.3.1) + * @return 1 if NSR02 or NSR03 found, + * -1 if first sector read error, 0 otherwise + */ +static int udf_check_vsd(struct super_block *sb) +{ + struct volStructDesc *vsd = NULL; + loff_t sector = VSD_FIRST_SECTOR_OFFSET; + int sectorsize; + struct buffer_head *bh = NULL; + int nsr = 0; + struct udf_sb_info *sbi; + loff_t session_offset; + + sbi = UDF_SB(sb); + if (sb->s_blocksize < sizeof(struct volStructDesc)) + sectorsize = sizeof(struct volStructDesc); + else + sectorsize = sb->s_blocksize; + + session_offset = (loff_t)sbi->s_session << sb->s_blocksize_bits; + sector += session_offset; + + udf_debug("Starting at sector %u (%lu byte sectors)\n", + (unsigned int)(sector >> sb->s_blocksize_bits), + sb->s_blocksize); + /* Process the sequence (if applicable). The hard limit on the sector + * offset is arbitrary, hopefully large enough so that all valid UDF + * filesystems will be recognised. There is no mention of an upper + * bound to the size of the volume recognition area in the standard. + * The limit will prevent the code to read all the sectors of a + * specially crafted image (like a bluray disc full of CD001 sectors), + * potentially causing minutes or even hours of uninterruptible I/O + * activity. This actually happened with uninitialised SSD partitions + * (all 0xFF) before the check for the limit and all valid IDs were + * added */ + for (; !nsr && sector < VSD_MAX_SECTOR_OFFSET; sector += sectorsize) { + /* Read a block */ + bh = sb_bread(sb, sector >> sb->s_blocksize_bits); + if (!bh) + break; + + vsd = (struct volStructDesc *)(bh->b_data + + (sector & (sb->s_blocksize - 1))); + nsr = identify_vsd(vsd); + /* Found NSR or end? */ + if (nsr) { + brelse(bh); + break; + } + /* + * Special handling for improperly formatted VRS (e.g., Win10) + * where components are separated by 2048 bytes even though + * sectors are 4K + */ + if (sb->s_blocksize == 4096) { + nsr = identify_vsd(vsd + 1); + /* Ignore unknown IDs... */ + if (nsr < 0) + nsr = 0; + } + brelse(bh); + } + + if (nsr > 0) + return 1; + else if (!bh && sector - session_offset == VSD_FIRST_SECTOR_OFFSET) + return -1; + else + return 0; +} + +static int udf_verify_domain_identifier(struct super_block *sb, + struct regid *ident, char *dname) +{ + struct domainIdentSuffix *suffix; + + if (memcmp(ident->ident, UDF_ID_COMPLIANT, strlen(UDF_ID_COMPLIANT))) { + udf_warn(sb, "Not OSTA UDF compliant %s descriptor.\n", dname); + goto force_ro; + } + if (ident->flags & ENTITYID_FLAGS_DIRTY) { + udf_warn(sb, "Possibly not OSTA UDF compliant %s descriptor.\n", + dname); + goto force_ro; + } + suffix = (struct domainIdentSuffix *)ident->identSuffix; + if ((suffix->domainFlags & DOMAIN_FLAGS_HARD_WRITE_PROTECT) || + (suffix->domainFlags & DOMAIN_FLAGS_SOFT_WRITE_PROTECT)) { + if (!sb_rdonly(sb)) { + udf_warn(sb, "Descriptor for %s marked write protected." + " Forcing read only mount.\n", dname); + } + goto force_ro; + } + return 0; + +force_ro: + if (!sb_rdonly(sb)) + return -EACCES; + UDF_SET_FLAG(sb, UDF_FLAG_RW_INCOMPAT); + return 0; +} + +static int udf_load_fileset(struct super_block *sb, struct fileSetDesc *fset, + struct kernel_lb_addr *root) +{ + int ret; + + ret = udf_verify_domain_identifier(sb, &fset->domainIdent, "file set"); + if (ret < 0) + return ret; + + *root = lelb_to_cpu(fset->rootDirectoryICB.extLocation); + UDF_SB(sb)->s_serial_number = le16_to_cpu(fset->descTag.tagSerialNum); + + udf_debug("Rootdir at block=%u, partition=%u\n", + root->logicalBlockNum, root->partitionReferenceNum); + return 0; +} + +static int udf_find_fileset(struct super_block *sb, + struct kernel_lb_addr *fileset, + struct kernel_lb_addr *root) +{ + struct buffer_head *bh; + uint16_t ident; + int ret; + + if (fileset->logicalBlockNum == 0xFFFFFFFF && + fileset->partitionReferenceNum == 0xFFFF) + return -EINVAL; + + bh = udf_read_ptagged(sb, fileset, 0, &ident); + if (!bh) + return -EIO; + if (ident != TAG_IDENT_FSD) { + brelse(bh); + return -EINVAL; + } + + udf_debug("Fileset at block=%u, partition=%u\n", + fileset->logicalBlockNum, fileset->partitionReferenceNum); + + UDF_SB(sb)->s_partition = fileset->partitionReferenceNum; + ret = udf_load_fileset(sb, (struct fileSetDesc *)bh->b_data, root); + brelse(bh); + return ret; +} + +/* + * Load primary Volume Descriptor Sequence + * + * Return <0 on error, 0 on success. -EAGAIN is special meaning next sequence + * should be tried. + */ +static int udf_load_pvoldesc(struct super_block *sb, sector_t block) +{ + struct primaryVolDesc *pvoldesc; + uint8_t *outstr; + struct buffer_head *bh; + uint16_t ident; + int ret; + struct timestamp *ts; + + outstr = kzalloc(128, GFP_KERNEL); + if (!outstr) + return -ENOMEM; + + bh = udf_read_tagged(sb, block, block, &ident); + if (!bh) { + ret = -EAGAIN; + goto out2; + } + + if (ident != TAG_IDENT_PVD) { + ret = -EIO; + goto out_bh; + } + + pvoldesc = (struct primaryVolDesc *)bh->b_data; + + udf_disk_stamp_to_time(&UDF_SB(sb)->s_record_time, + pvoldesc->recordingDateAndTime); + ts = &pvoldesc->recordingDateAndTime; + udf_debug("recording time %04u/%02u/%02u %02u:%02u (%x)\n", + le16_to_cpu(ts->year), ts->month, ts->day, ts->hour, + ts->minute, le16_to_cpu(ts->typeAndTimezone)); + + ret = udf_dstrCS0toChar(sb, outstr, 31, pvoldesc->volIdent, 32); + if (ret < 0) { + strscpy_pad(UDF_SB(sb)->s_volume_ident, "InvalidName"); + pr_warn("incorrect volume identification, setting to " + "'InvalidName'\n"); + } else { + strscpy_pad(UDF_SB(sb)->s_volume_ident, outstr); + } + udf_debug("volIdent[] = '%s'\n", UDF_SB(sb)->s_volume_ident); + + ret = udf_dstrCS0toChar(sb, outstr, 127, pvoldesc->volSetIdent, 128); + if (ret < 0) { + ret = 0; + goto out_bh; + } + outstr[ret] = 0; + udf_debug("volSetIdent[] = '%s'\n", outstr); + + ret = 0; +out_bh: + brelse(bh); +out2: + kfree(outstr); + return ret; +} + +struct inode *udf_find_metadata_inode_efe(struct super_block *sb, + u32 meta_file_loc, u32 partition_ref) +{ + struct kernel_lb_addr addr; + struct inode *metadata_fe; + + addr.logicalBlockNum = meta_file_loc; + addr.partitionReferenceNum = partition_ref; + + metadata_fe = udf_iget_special(sb, &addr); + + if (IS_ERR(metadata_fe)) { + udf_warn(sb, "metadata inode efe not found\n"); + return metadata_fe; + } + if (UDF_I(metadata_fe)->i_alloc_type != ICBTAG_FLAG_AD_SHORT) { + udf_warn(sb, "metadata inode efe does not have short allocation descriptors!\n"); + iput(metadata_fe); + return ERR_PTR(-EIO); + } + + return metadata_fe; +} + +static int udf_load_metadata_files(struct super_block *sb, int partition, + int type1_index) +{ + struct udf_sb_info *sbi = UDF_SB(sb); + struct udf_part_map *map; + struct udf_meta_data *mdata; + struct kernel_lb_addr addr; + struct inode *fe; + + map = &sbi->s_partmaps[partition]; + mdata = &map->s_type_specific.s_metadata; + mdata->s_phys_partition_ref = type1_index; + + /* metadata address */ + udf_debug("Metadata file location: block = %u part = %u\n", + mdata->s_meta_file_loc, mdata->s_phys_partition_ref); + + fe = udf_find_metadata_inode_efe(sb, mdata->s_meta_file_loc, + mdata->s_phys_partition_ref); + if (IS_ERR(fe)) { + /* mirror file entry */ + udf_debug("Mirror metadata file location: block = %u part = %u\n", + mdata->s_mirror_file_loc, mdata->s_phys_partition_ref); + + fe = udf_find_metadata_inode_efe(sb, mdata->s_mirror_file_loc, + mdata->s_phys_partition_ref); + + if (IS_ERR(fe)) { + udf_err(sb, "Both metadata and mirror metadata inode efe can not found\n"); + return PTR_ERR(fe); + } + mdata->s_mirror_fe = fe; + } else + mdata->s_metadata_fe = fe; + + + /* + * bitmap file entry + * Note: + * Load only if bitmap file location differs from 0xFFFFFFFF (DCN-5102) + */ + if (mdata->s_bitmap_file_loc != 0xFFFFFFFF) { + addr.logicalBlockNum = mdata->s_bitmap_file_loc; + addr.partitionReferenceNum = mdata->s_phys_partition_ref; + + udf_debug("Bitmap file location: block = %u part = %u\n", + addr.logicalBlockNum, addr.partitionReferenceNum); + + fe = udf_iget_special(sb, &addr); + if (IS_ERR(fe)) { + if (sb_rdonly(sb)) + udf_warn(sb, "bitmap inode efe not found but it's ok since the disc is mounted read-only\n"); + else { + udf_err(sb, "bitmap inode efe not found and attempted read-write mount\n"); + return PTR_ERR(fe); + } + } else + mdata->s_bitmap_fe = fe; + } + + udf_debug("udf_load_metadata_files Ok\n"); + return 0; +} + +int udf_compute_nr_groups(struct super_block *sb, u32 partition) +{ + struct udf_part_map *map = &UDF_SB(sb)->s_partmaps[partition]; + return DIV_ROUND_UP(map->s_partition_len + + (sizeof(struct spaceBitmapDesc) << 3), + sb->s_blocksize * 8); +} + +static struct udf_bitmap *udf_sb_alloc_bitmap(struct super_block *sb, u32 index) +{ + struct udf_bitmap *bitmap; + int nr_groups = udf_compute_nr_groups(sb, index); + + bitmap = kvzalloc_flex(*bitmap, s_block_bitmap, nr_groups); + if (!bitmap) + return NULL; + + bitmap->s_nr_groups = nr_groups; + return bitmap; +} + +static int check_partition_desc(struct super_block *sb, + struct partitionDesc *p, + struct udf_part_map *map) +{ + bool umap, utable, fmap, ftable; + struct partitionHeaderDesc *phd; + + switch (le32_to_cpu(p->accessType)) { + case PD_ACCESS_TYPE_READ_ONLY: + case PD_ACCESS_TYPE_WRITE_ONCE: + case PD_ACCESS_TYPE_NONE: + goto force_ro; + } + + /* No Partition Header Descriptor? */ + if (strcmp(p->partitionContents.ident, PD_PARTITION_CONTENTS_NSR02) && + strcmp(p->partitionContents.ident, PD_PARTITION_CONTENTS_NSR03)) + goto force_ro; + + phd = (struct partitionHeaderDesc *)p->partitionContentsUse; + utable = phd->unallocSpaceTable.extLength; + umap = phd->unallocSpaceBitmap.extLength; + ftable = phd->freedSpaceTable.extLength; + fmap = phd->freedSpaceBitmap.extLength; + + /* No allocation info? */ + if (!utable && !umap && !ftable && !fmap) + goto force_ro; + + /* We don't support blocks that require erasing before overwrite */ + if (ftable || fmap) + goto force_ro; + /* UDF 2.60: 2.3.3 - no mixing of tables & bitmaps, no VAT. */ + if (utable && umap) + goto force_ro; + + if (map->s_partition_type == UDF_VIRTUAL_MAP15 || + map->s_partition_type == UDF_VIRTUAL_MAP20 || + map->s_partition_type == UDF_METADATA_MAP25) + goto force_ro; + + return 0; +force_ro: + if (!sb_rdonly(sb)) + return -EACCES; + UDF_SET_FLAG(sb, UDF_FLAG_RW_INCOMPAT); + return 0; +} + +static int udf_fill_partdesc_info(struct super_block *sb, + struct partitionDesc *p, int p_index) +{ + struct udf_part_map *map; + struct udf_sb_info *sbi = UDF_SB(sb); + struct partitionHeaderDesc *phd; + u32 sum; + int err; + + map = &sbi->s_partmaps[p_index]; + + map->s_partition_len = le32_to_cpu(p->partitionLength); /* blocks */ + map->s_partition_root = le32_to_cpu(p->partitionStartingLocation); + if (check_add_overflow(map->s_partition_root, map->s_partition_len, + &sum)) { + udf_err(sb, "Partition %d has invalid location %u + %u\n", + p_index, map->s_partition_root, map->s_partition_len); + return -EFSCORRUPTED; + } + + if (p->accessType == cpu_to_le32(PD_ACCESS_TYPE_READ_ONLY)) + map->s_partition_flags |= UDF_PART_FLAG_READ_ONLY; + if (p->accessType == cpu_to_le32(PD_ACCESS_TYPE_WRITE_ONCE)) + map->s_partition_flags |= UDF_PART_FLAG_WRITE_ONCE; + if (p->accessType == cpu_to_le32(PD_ACCESS_TYPE_REWRITABLE)) + map->s_partition_flags |= UDF_PART_FLAG_REWRITABLE; + if (p->accessType == cpu_to_le32(PD_ACCESS_TYPE_OVERWRITABLE)) + map->s_partition_flags |= UDF_PART_FLAG_OVERWRITABLE; + + udf_debug("Partition (%d type %x) starts at physical %u, block length %u\n", + p_index, map->s_partition_type, + map->s_partition_root, map->s_partition_len); + + err = check_partition_desc(sb, p, map); + if (err) + return err; + + /* + * Skip loading allocation info it we cannot ever write to the fs. + * This is a correctness thing as we may have decided to force ro mount + * to avoid allocation info we don't support. + */ + if (UDF_QUERY_FLAG(sb, UDF_FLAG_RW_INCOMPAT)) + return 0; + + phd = (struct partitionHeaderDesc *)p->partitionContentsUse; + if (phd->unallocSpaceTable.extLength) { + struct kernel_lb_addr loc = { + .logicalBlockNum = le32_to_cpu( + phd->unallocSpaceTable.extPosition), + .partitionReferenceNum = p_index, + }; + struct inode *inode; + + inode = udf_iget_special(sb, &loc); + if (IS_ERR(inode)) { + udf_debug("cannot load unallocSpaceTable (part %d)\n", + p_index); + return PTR_ERR(inode); + } + map->s_uspace.s_table = inode; + map->s_partition_flags |= UDF_PART_FLAG_UNALLOC_TABLE; + udf_debug("unallocSpaceTable (part %d) @ %llu\n", + p_index, map->s_uspace.s_table->i_ino); + } + + if (phd->unallocSpaceBitmap.extLength) { + struct udf_bitmap *bitmap = udf_sb_alloc_bitmap(sb, p_index); + if (!bitmap) + return -ENOMEM; + map->s_uspace.s_bitmap = bitmap; + bitmap->s_extPosition = le32_to_cpu( + phd->unallocSpaceBitmap.extPosition); + map->s_partition_flags |= UDF_PART_FLAG_UNALLOC_BITMAP; + /* Check whether math over bitmap won't overflow. */ + if (check_add_overflow(map->s_partition_len, + sizeof(struct spaceBitmapDesc) << 3, + &sum)) { + udf_err(sb, "Partition %d is too long (%u)\n", p_index, + map->s_partition_len); + return -EFSCORRUPTED; + } + udf_debug("unallocSpaceBitmap (part %d) @ %u\n", + p_index, bitmap->s_extPosition); + } + + return 0; +} + +static void udf_find_vat_block(struct super_block *sb, int p_index, + int type1_index, sector_t start_block) +{ + struct udf_sb_info *sbi = UDF_SB(sb); + struct udf_part_map *map = &sbi->s_partmaps[p_index]; + sector_t vat_block; + struct kernel_lb_addr ino; + struct inode *inode; + + /* + * VAT file entry is in the last recorded block. Some broken disks have + * it a few blocks before so try a bit harder... + */ + ino.partitionReferenceNum = type1_index; + for (vat_block = start_block; + vat_block >= map->s_partition_root && + vat_block >= start_block - 3; vat_block--) { + ino.logicalBlockNum = vat_block - map->s_partition_root; + inode = udf_iget_special(sb, &ino); + if (!IS_ERR(inode)) { + sbi->s_vat_inode = inode; + break; + } + } +} + +static int udf_load_vat(struct super_block *sb, int p_index, int type1_index) +{ + struct udf_sb_info *sbi = UDF_SB(sb); + struct udf_part_map *map = &sbi->s_partmaps[p_index]; + struct buffer_head *bh = NULL; + struct udf_inode_info *vati; + struct virtualAllocationTable20 *vat20; + sector_t blocks = sb_bdev_nr_blocks(sb); + + udf_find_vat_block(sb, p_index, type1_index, sbi->s_last_block); + if (!sbi->s_vat_inode && + sbi->s_last_block != blocks - 1) { + pr_notice("Failed to read VAT inode from the last recorded block (%lu), retrying with the last block of the device (%lu).\n", + (unsigned long)sbi->s_last_block, + (unsigned long)blocks - 1); + udf_find_vat_block(sb, p_index, type1_index, blocks - 1); + } + if (!sbi->s_vat_inode) + return -EIO; + + if (map->s_partition_type == UDF_VIRTUAL_MAP15) { + map->s_type_specific.s_virtual.s_start_offset = 0; + if (sbi->s_vat_inode->i_size < 36) { + udf_err(sb, "Too short VAT inode size %lld\n", + sbi->s_vat_inode->i_size); + return -EFSCORRUPTED; + } + map->s_type_specific.s_virtual.s_num_entries = + (sbi->s_vat_inode->i_size - 36) >> 2; + } else if (map->s_partition_type == UDF_VIRTUAL_MAP20) { + vati = UDF_I(sbi->s_vat_inode); + if (vati->i_alloc_type != ICBTAG_FLAG_AD_IN_ICB) { + int err = 0; + + bh = udf_bread(sbi->s_vat_inode, 0, 0, &err); + if (!bh) { + if (!err) + err = -EFSCORRUPTED; + return err; + } + vat20 = (struct virtualAllocationTable20 *)bh->b_data; + } else { + vat20 = (struct virtualAllocationTable20 *) + vati->i_data; + } + + map->s_type_specific.s_virtual.s_start_offset = + le16_to_cpu(vat20->lengthHeader); + if (map->s_type_specific.s_virtual.s_start_offset + > sbi->s_vat_inode->i_size) { + udf_err(sb, "Corrupted VAT header length %u (VAT inode size %lld)\n", + map->s_type_specific.s_virtual.s_start_offset, + sbi->s_vat_inode->i_size); + brelse(bh); + return -EFSCORRUPTED; + } + map->s_type_specific.s_virtual.s_num_entries = + (sbi->s_vat_inode->i_size - + map->s_type_specific.s_virtual. + s_start_offset) >> 2; + brelse(bh); + } + return 0; +} + +/* + * Load partition descriptor block + * + * Returns <0 on error, 0 on success, -EAGAIN is special - try next descriptor + * sequence. + */ +static int udf_load_partdesc(struct super_block *sb, sector_t block) +{ + struct buffer_head *bh; + struct partitionDesc *p; + struct udf_part_map *map; + struct udf_sb_info *sbi = UDF_SB(sb); + int i, type1_idx; + uint16_t partitionNumber; + uint16_t ident; + int ret; + + bh = udf_read_tagged(sb, block, block, &ident); + if (!bh) + return -EAGAIN; + if (ident != TAG_IDENT_PD) { + ret = 0; + goto out_bh; + } + + p = (struct partitionDesc *)bh->b_data; + partitionNumber = le16_to_cpu(p->partitionNumber); + + /* First scan for TYPE1 and SPARABLE partitions */ + for (i = 0; i < sbi->s_partitions; i++) { + map = &sbi->s_partmaps[i]; + udf_debug("Searching map: (%u == %u)\n", + map->s_partition_num, partitionNumber); + if (map->s_partition_num == partitionNumber && + (map->s_partition_type == UDF_TYPE1_MAP15 || + map->s_partition_type == UDF_SPARABLE_MAP15)) + break; + } + + if (i >= sbi->s_partitions) { + udf_debug("Partition (%u) not found in partition map\n", + partitionNumber); + ret = 0; + goto out_bh; + } + + ret = udf_fill_partdesc_info(sb, p, i); + if (ret < 0) + goto out_bh; + + /* + * Now rescan for VIRTUAL or METADATA partitions when SPARABLE and + * PHYSICAL partitions are already set up + */ + type1_idx = i; + map = NULL; /* supress 'maybe used uninitialized' warning */ + for (i = 0; i < sbi->s_partitions; i++) { + map = &sbi->s_partmaps[i]; + + if (map->s_partition_num == partitionNumber && + (map->s_partition_type == UDF_VIRTUAL_MAP15 || + map->s_partition_type == UDF_VIRTUAL_MAP20 || + map->s_partition_type == UDF_METADATA_MAP25)) + break; + } + + if (i >= sbi->s_partitions) { + ret = 0; + goto out_bh; + } + + ret = udf_fill_partdesc_info(sb, p, i); + if (ret < 0) + goto out_bh; + + if (map->s_partition_type == UDF_METADATA_MAP25) { + ret = udf_load_metadata_files(sb, i, type1_idx); + if (ret < 0) { + udf_err(sb, "error loading MetaData partition map %d\n", + i); + goto out_bh; + } + } else { + /* + * If we have a partition with virtual map, we don't handle + * writing to it (we overwrite blocks instead of relocating + * them). + */ + if (!sb_rdonly(sb)) { + ret = -EACCES; + goto out_bh; + } + UDF_SET_FLAG(sb, UDF_FLAG_RW_INCOMPAT); + ret = udf_load_vat(sb, i, type1_idx); + if (ret < 0) + goto out_bh; + } + ret = 0; +out_bh: + /* In case loading failed, we handle cleanup in udf_fill_super */ + brelse(bh); + return ret; +} + +static int udf_load_sparable_map(struct super_block *sb, + struct udf_part_map *map, + struct sparablePartitionMap *spm) +{ + uint32_t loc; + uint16_t ident; + struct sparingTable *st; + struct udf_sparing_data *sdata = &map->s_type_specific.s_sparing; + int i; + struct buffer_head *bh; + + map->s_partition_type = UDF_SPARABLE_MAP15; + sdata->s_packet_len = le16_to_cpu(spm->packetLength); + if (!is_power_of_2(sdata->s_packet_len)) { + udf_err(sb, "error loading logical volume descriptor: " + "Invalid packet length %u\n", + (unsigned)sdata->s_packet_len); + return -EIO; + } + if (spm->numSparingTables > 4) { + udf_err(sb, "error loading logical volume descriptor: " + "Too many sparing tables (%d)\n", + (int)spm->numSparingTables); + return -EIO; + } + if (le32_to_cpu(spm->sizeSparingTable) > sb->s_blocksize) { + udf_err(sb, "error loading logical volume descriptor: " + "Too big sparing table size (%u)\n", + le32_to_cpu(spm->sizeSparingTable)); + return -EIO; + } + + for (i = 0; i < spm->numSparingTables; i++) { + loc = le32_to_cpu(spm->locSparingTable[i]); + bh = udf_read_tagged(sb, loc, loc, &ident); + if (!bh) + continue; + + st = (struct sparingTable *)bh->b_data; + if (ident != 0 || + strncmp(st->sparingIdent.ident, UDF_ID_SPARING, + strlen(UDF_ID_SPARING)) || + struct_size(st, mapEntry, + le16_to_cpu(st->reallocationTableLen)) > + sb->s_blocksize) { + brelse(bh); + continue; + } + + sdata->s_spar_map[i] = bh; + } + map->s_partition_func = udf_get_pblock_spar15; + return 0; +} + +static int udf_load_logicalvol(struct super_block *sb, sector_t block, + struct kernel_lb_addr *fileset) +{ + struct logicalVolDesc *lvd; + int i, offset; + uint8_t type; + struct udf_sb_info *sbi = UDF_SB(sb); + struct genericPartitionMap *gpm; + uint16_t ident; + struct buffer_head *bh; + unsigned int table_len, part_map_count; + int ret; + + bh = udf_read_tagged(sb, block, block, &ident); + if (!bh) + return -EAGAIN; + BUG_ON(ident != TAG_IDENT_LVD); + lvd = (struct logicalVolDesc *)bh->b_data; + table_len = le32_to_cpu(lvd->mapTableLength); + if (table_len > sb->s_blocksize - sizeof(*lvd)) { + udf_err(sb, "error loading logical volume descriptor: " + "Partition table too long (%u > %lu)\n", table_len, + sb->s_blocksize - sizeof(*lvd)); + ret = -EIO; + goto out_bh; + } + + ret = udf_verify_domain_identifier(sb, &lvd->domainIdent, + "logical volume"); + if (ret) + goto out_bh; + + part_map_count = le32_to_cpu(lvd->numPartitionMaps); + if (part_map_count > table_len / sizeof(struct genericPartitionMap1)) { + udf_err(sb, "error loading logical volume descriptor: " + "Too many partition maps (%u > %u)\n", part_map_count, + table_len / (unsigned)sizeof(struct genericPartitionMap1)); + ret = -EIO; + goto out_bh; + } + ret = udf_sb_alloc_partition_maps(sb, part_map_count); + if (ret) + goto out_bh; + + for (i = 0, offset = 0; + i < sbi->s_partitions && offset < table_len; + i++, offset += gpm->partitionMapLength) { + struct udf_part_map *map = &sbi->s_partmaps[i]; + gpm = (struct genericPartitionMap *) + &(lvd->partitionMaps[offset]); + type = gpm->partitionMapType; + if (type == 1) { + struct genericPartitionMap1 *gpm1 = + (struct genericPartitionMap1 *)gpm; + map->s_partition_type = UDF_TYPE1_MAP15; + map->s_volumeseqnum = le16_to_cpu(gpm1->volSeqNum); + map->s_partition_num = le16_to_cpu(gpm1->partitionNum); + map->s_partition_func = NULL; + } else if (type == 2) { + struct udfPartitionMap2 *upm2 = + (struct udfPartitionMap2 *)gpm; + if (!strncmp(upm2->partIdent.ident, UDF_ID_VIRTUAL, + strlen(UDF_ID_VIRTUAL))) { + u16 suf = + le16_to_cpu(((__le16 *)upm2->partIdent. + identSuffix)[0]); + if (suf < 0x0200) { + map->s_partition_type = + UDF_VIRTUAL_MAP15; + map->s_partition_func = + udf_get_pblock_virt15; + } else { + map->s_partition_type = + UDF_VIRTUAL_MAP20; + map->s_partition_func = + udf_get_pblock_virt20; + } + } else if (!strncmp(upm2->partIdent.ident, + UDF_ID_SPARABLE, + strlen(UDF_ID_SPARABLE))) { + ret = udf_load_sparable_map(sb, map, + (struct sparablePartitionMap *)gpm); + if (ret < 0) + goto out_bh; + } else if (!strncmp(upm2->partIdent.ident, + UDF_ID_METADATA, + strlen(UDF_ID_METADATA))) { + struct udf_meta_data *mdata = + &map->s_type_specific.s_metadata; + struct metadataPartitionMap *mdm = + (struct metadataPartitionMap *) + &(lvd->partitionMaps[offset]); + udf_debug("Parsing Logical vol part %d type %u id=%s\n", + i, type, UDF_ID_METADATA); + + map->s_partition_type = UDF_METADATA_MAP25; + map->s_partition_func = udf_get_pblock_meta25; + + mdata->s_meta_file_loc = + le32_to_cpu(mdm->metadataFileLoc); + mdata->s_mirror_file_loc = + le32_to_cpu(mdm->metadataMirrorFileLoc); + mdata->s_bitmap_file_loc = + le32_to_cpu(mdm->metadataBitmapFileLoc); + mdata->s_alloc_unit_size = + le32_to_cpu(mdm->allocUnitSize); + mdata->s_align_unit_size = + le16_to_cpu(mdm->alignUnitSize); + if (mdm->flags & 0x01) + mdata->s_flags |= MF_DUPLICATE_MD; + + udf_debug("Metadata Ident suffix=0x%x\n", + le16_to_cpu(*(__le16 *) + mdm->partIdent.identSuffix)); + udf_debug("Metadata part num=%u\n", + le16_to_cpu(mdm->partitionNum)); + udf_debug("Metadata part alloc unit size=%u\n", + le32_to_cpu(mdm->allocUnitSize)); + udf_debug("Metadata file loc=%u\n", + le32_to_cpu(mdm->metadataFileLoc)); + udf_debug("Mirror file loc=%u\n", + le32_to_cpu(mdm->metadataMirrorFileLoc)); + udf_debug("Bitmap file loc=%u\n", + le32_to_cpu(mdm->metadataBitmapFileLoc)); + udf_debug("Flags: %d %u\n", + mdata->s_flags, mdm->flags); + } else { + udf_debug("Unknown ident: %s\n", + upm2->partIdent.ident); + continue; + } + map->s_volumeseqnum = le16_to_cpu(upm2->volSeqNum); + map->s_partition_num = le16_to_cpu(upm2->partitionNum); + } + udf_debug("Partition (%d:%u) type %u on volume %u\n", + i, map->s_partition_num, type, map->s_volumeseqnum); + } + + if (fileset) { + struct long_ad *la = (struct long_ad *)&(lvd->logicalVolContentsUse[0]); + + *fileset = lelb_to_cpu(la->extLocation); + udf_debug("FileSet found in LogicalVolDesc at block=%u, partition=%u\n", + fileset->logicalBlockNum, + fileset->partitionReferenceNum); + } + if (lvd->integritySeqExt.extLength) + udf_load_logicalvolint(sb, leea_to_cpu(lvd->integritySeqExt)); + ret = 0; + + if (!sbi->s_lvid_bh) { + /* We can't generate unique IDs without a valid LVID */ + if (sb_rdonly(sb)) { + UDF_SET_FLAG(sb, UDF_FLAG_RW_INCOMPAT); + } else { + udf_warn(sb, "Damaged or missing LVID, forcing " + "readonly mount\n"); + ret = -EACCES; + } + } +out_bh: + brelse(bh); + return ret; +} + +static bool udf_lvid_valid(struct super_block *sb, + struct logicalVolIntegrityDesc *lvid) +{ + u32 parts, impuselen; + + parts = le32_to_cpu(lvid->numOfPartitions); + impuselen = le32_to_cpu(lvid->lengthOfImpUse); + if (parts >= sb->s_blocksize || impuselen >= sb->s_blocksize || + sizeof(struct logicalVolIntegrityDesc) + impuselen + + 2 * parts * sizeof(u32) > sb->s_blocksize) + return false; + return true; +} + +/* + * Find the prevailing Logical Volume Integrity Descriptor. + */ +static void udf_load_logicalvolint(struct super_block *sb, struct kernel_extent_ad loc) +{ + struct buffer_head *bh, *final_bh; + uint16_t ident; + struct udf_sb_info *sbi = UDF_SB(sb); + struct logicalVolIntegrityDesc *lvid; + int indirections = 0; + + while (++indirections <= UDF_MAX_LVID_NESTING) { + final_bh = NULL; + while (loc.extLength > 0 && + (bh = udf_read_tagged(sb, loc.extLocation, + loc.extLocation, &ident))) { + if (ident != TAG_IDENT_LVID) { + brelse(bh); + break; + } + + brelse(final_bh); + final_bh = bh; + + loc.extLength -= sb->s_blocksize; + loc.extLocation++; + } + + if (!final_bh) + return; + + lvid = (struct logicalVolIntegrityDesc *)final_bh->b_data; + if (udf_lvid_valid(sb, lvid)) { + brelse(sbi->s_lvid_bh); + sbi->s_lvid_bh = final_bh; + } else { + udf_warn(sb, "Corrupted LVID (parts=%u, impuselen=%u), " + "ignoring.\n", + le32_to_cpu(lvid->numOfPartitions), + le32_to_cpu(lvid->lengthOfImpUse)); + } + + if (lvid->nextIntegrityExt.extLength == 0) + return; + + loc = leea_to_cpu(lvid->nextIntegrityExt); + } + + udf_warn(sb, "Too many LVID indirections (max %u), ignoring.\n", + UDF_MAX_LVID_NESTING); + brelse(sbi->s_lvid_bh); + sbi->s_lvid_bh = NULL; +} + +/* + * Step for reallocation of table of partition descriptor sequence numbers. + * Must be power of 2. + */ +#define PART_DESC_ALLOC_STEP 32 + +struct part_desc_seq_scan_data { + struct udf_vds_record rec; + u32 partnum; +}; + +struct desc_seq_scan_data { + struct udf_vds_record vds[VDS_POS_LENGTH]; + unsigned int size_part_descs; + unsigned int num_part_descs; + struct part_desc_seq_scan_data *part_descs_loc; +}; + +static struct udf_vds_record *handle_partition_descriptor( + struct buffer_head *bh, + struct desc_seq_scan_data *data) +{ + struct partitionDesc *desc = (struct partitionDesc *)bh->b_data; + int partnum; + int i; + + partnum = le16_to_cpu(desc->partitionNumber); + for (i = 0; i < data->num_part_descs; i++) + if (partnum == data->part_descs_loc[i].partnum) + return &(data->part_descs_loc[i].rec); + if (data->num_part_descs >= data->size_part_descs) { + struct part_desc_seq_scan_data *new_loc; + unsigned int new_size; + + new_size = data->num_part_descs + PART_DESC_ALLOC_STEP; + new_loc = kzalloc_objs(*new_loc, new_size); + if (!new_loc) + return ERR_PTR(-ENOMEM); + memcpy(new_loc, data->part_descs_loc, + data->size_part_descs * sizeof(*new_loc)); + kfree(data->part_descs_loc); + data->part_descs_loc = new_loc; + data->size_part_descs = new_size; + } + data->part_descs_loc[data->num_part_descs].partnum = partnum; + return &(data->part_descs_loc[data->num_part_descs++].rec); +} + + +static struct udf_vds_record *get_volume_descriptor_record(uint16_t ident, + struct buffer_head *bh, struct desc_seq_scan_data *data) +{ + switch (ident) { + case TAG_IDENT_PVD: /* ISO 13346 3/10.1 */ + return &(data->vds[VDS_POS_PRIMARY_VOL_DESC]); + case TAG_IDENT_IUVD: /* ISO 13346 3/10.4 */ + return &(data->vds[VDS_POS_IMP_USE_VOL_DESC]); + case TAG_IDENT_LVD: /* ISO 13346 3/10.6 */ + return &(data->vds[VDS_POS_LOGICAL_VOL_DESC]); + case TAG_IDENT_USD: /* ISO 13346 3/10.8 */ + return &(data->vds[VDS_POS_UNALLOC_SPACE_DESC]); + case TAG_IDENT_PD: /* ISO 13346 3/10.5 */ + return handle_partition_descriptor(bh, data); + } + return NULL; +} + +/* + * Process a main/reserve volume descriptor sequence. + * @block First block of first extent of the sequence. + * @lastblock Lastblock of first extent of the sequence. + * @fileset There we store extent containing root fileset + * + * Returns <0 on error, 0 on success. -EAGAIN is special - try next descriptor + * sequence + */ +static noinline int udf_process_sequence( + struct super_block *sb, + sector_t block, sector_t lastblock, + struct kernel_lb_addr *fileset) +{ + struct buffer_head *bh = NULL; + struct udf_vds_record *curr; + struct generic_desc *gd; + struct volDescPtr *vdp; + bool done = false; + uint32_t vdsn; + uint16_t ident; + int ret; + unsigned int indirections = 0; + struct desc_seq_scan_data data; + unsigned int i; + + memset(data.vds, 0, sizeof(struct udf_vds_record) * VDS_POS_LENGTH); + data.size_part_descs = PART_DESC_ALLOC_STEP; + data.num_part_descs = 0; + data.part_descs_loc = kzalloc_objs(*data.part_descs_loc, + data.size_part_descs); + if (!data.part_descs_loc) + return -ENOMEM; + + /* + * Read the main descriptor sequence and find which descriptors + * are in it. + */ + for (; (!done && block <= lastblock); block++) { + bh = udf_read_tagged(sb, block, block, &ident); + if (!bh) + break; + + /* Process each descriptor (ISO 13346 3/8.3-8.4) */ + gd = (struct generic_desc *)bh->b_data; + vdsn = le32_to_cpu(gd->volDescSeqNum); + switch (ident) { + case TAG_IDENT_VDP: /* ISO 13346 3/10.3 */ + if (++indirections > UDF_MAX_TD_NESTING) { + udf_err(sb, "too many Volume Descriptor " + "Pointers (max %u supported)\n", + UDF_MAX_TD_NESTING); + brelse(bh); + ret = -EIO; + goto out; + } + + vdp = (struct volDescPtr *)bh->b_data; + block = le32_to_cpu(vdp->nextVolDescSeqExt.extLocation); + lastblock = le32_to_cpu( + vdp->nextVolDescSeqExt.extLength) >> + sb->s_blocksize_bits; + lastblock += block - 1; + /* For loop is going to increment 'block' again */ + block--; + break; + case TAG_IDENT_PVD: /* ISO 13346 3/10.1 */ + case TAG_IDENT_IUVD: /* ISO 13346 3/10.4 */ + case TAG_IDENT_LVD: /* ISO 13346 3/10.6 */ + case TAG_IDENT_USD: /* ISO 13346 3/10.8 */ + case TAG_IDENT_PD: /* ISO 13346 3/10.5 */ + curr = get_volume_descriptor_record(ident, bh, &data); + if (IS_ERR(curr)) { + brelse(bh); + ret = PTR_ERR(curr); + goto out; + } + /* Descriptor we don't care about? */ + if (!curr) + break; + if (vdsn >= curr->volDescSeqNum) { + curr->volDescSeqNum = vdsn; + curr->block = block; + } + break; + case TAG_IDENT_TD: /* ISO 13346 3/10.9 */ + done = true; + break; + } + brelse(bh); + } + /* + * Now read interesting descriptors again and process them + * in a suitable order + */ + if (!data.vds[VDS_POS_PRIMARY_VOL_DESC].block) { + udf_err(sb, "Primary Volume Descriptor not found!\n"); + ret = -EAGAIN; + goto out; + } + ret = udf_load_pvoldesc(sb, data.vds[VDS_POS_PRIMARY_VOL_DESC].block); + if (ret < 0) + goto out; + + if (data.vds[VDS_POS_LOGICAL_VOL_DESC].block) { + ret = udf_load_logicalvol(sb, + data.vds[VDS_POS_LOGICAL_VOL_DESC].block, + fileset); + if (ret < 0) + goto out; + } + + /* Now handle prevailing Partition Descriptors */ + for (i = 0; i < data.num_part_descs; i++) { + ret = udf_load_partdesc(sb, data.part_descs_loc[i].rec.block); + if (ret < 0) + goto out; + } + ret = 0; +out: + kfree(data.part_descs_loc); + return ret; +} + +/* + * Load Volume Descriptor Sequence described by anchor in bh + * + * Returns <0 on error, 0 on success + */ +static int udf_load_sequence(struct super_block *sb, struct buffer_head *bh, + struct kernel_lb_addr *fileset) +{ + struct anchorVolDescPtr *anchor; + sector_t main_s, main_e, reserve_s, reserve_e; + int ret; + + anchor = (struct anchorVolDescPtr *)bh->b_data; + + /* Locate the main sequence */ + main_s = le32_to_cpu(anchor->mainVolDescSeqExt.extLocation); + main_e = le32_to_cpu(anchor->mainVolDescSeqExt.extLength); + main_e = main_e >> sb->s_blocksize_bits; + main_e += main_s - 1; + + /* Locate the reserve sequence */ + reserve_s = le32_to_cpu(anchor->reserveVolDescSeqExt.extLocation); + reserve_e = le32_to_cpu(anchor->reserveVolDescSeqExt.extLength); + reserve_e = reserve_e >> sb->s_blocksize_bits; + reserve_e += reserve_s - 1; + + /* Process the main & reserve sequences */ + /* responsible for finding the PartitionDesc(s) */ + ret = udf_process_sequence(sb, main_s, main_e, fileset); + if (ret != -EAGAIN) + return ret; + udf_sb_free_partitions(sb); + ret = udf_process_sequence(sb, reserve_s, reserve_e, fileset); + if (ret < 0) { + udf_sb_free_partitions(sb); + /* No sequence was OK, return -EIO */ + if (ret == -EAGAIN) + ret = -EIO; + } + return ret; +} + +/* + * Check whether there is an anchor block in the given block and + * load Volume Descriptor Sequence if so. + * + * Returns <0 on error, 0 on success, -EAGAIN is special - try next anchor + * block + */ +static int udf_check_anchor_block(struct super_block *sb, sector_t block, + struct kernel_lb_addr *fileset) +{ + struct buffer_head *bh; + uint16_t ident; + int ret; + + bh = udf_read_tagged(sb, block, block, &ident); + if (!bh) + return -EAGAIN; + if (ident != TAG_IDENT_AVDP) { + brelse(bh); + return -EAGAIN; + } + ret = udf_load_sequence(sb, bh, fileset); + brelse(bh); + return ret; +} + +/* + * Search for an anchor volume descriptor pointer. + * + * Returns < 0 on error, 0 on success. -EAGAIN is special - try next set + * of anchors. + */ +static int udf_scan_anchors(struct super_block *sb, udf_pblk_t *lastblock, + struct kernel_lb_addr *fileset) +{ + udf_pblk_t last[6]; + int i; + struct udf_sb_info *sbi = UDF_SB(sb); + int last_count = 0; + int ret; + + /* First try user provided anchor */ + if (sbi->s_anchor) { + ret = udf_check_anchor_block(sb, sbi->s_anchor, fileset); + if (ret != -EAGAIN) + return ret; + } + /* + * according to spec, anchor is in either: + * block 256 + * lastblock-256 + * lastblock + * however, if the disc isn't closed, it could be 512. + */ + ret = udf_check_anchor_block(sb, sbi->s_session + 256, fileset); + if (ret != -EAGAIN) + return ret; + /* + * The trouble is which block is the last one. Drives often misreport + * this so we try various possibilities. + */ + last[last_count++] = *lastblock; + if (*lastblock >= 1) + last[last_count++] = *lastblock - 1; + last[last_count++] = *lastblock + 1; + if (*lastblock >= 2) + last[last_count++] = *lastblock - 2; + if (*lastblock >= 150) + last[last_count++] = *lastblock - 150; + if (*lastblock >= 152) + last[last_count++] = *lastblock - 152; + + for (i = 0; i < last_count; i++) { + if (last[i] >= sb_bdev_nr_blocks(sb)) + continue; + ret = udf_check_anchor_block(sb, last[i], fileset); + if (ret != -EAGAIN) { + if (!ret) + *lastblock = last[i]; + return ret; + } + if (last[i] < 256) + continue; + ret = udf_check_anchor_block(sb, last[i] - 256, fileset); + if (ret != -EAGAIN) { + if (!ret) + *lastblock = last[i]; + return ret; + } + } + + /* Finally try block 512 in case media is open */ + return udf_check_anchor_block(sb, sbi->s_session + 512, fileset); +} + +/* + * Check Volume Structure Descriptor, find Anchor block and load Volume + * Descriptor Sequence. + * + * Returns < 0 on error, 0 on success. -EAGAIN is special meaning anchor + * block was not found. + */ +static int udf_load_vrs(struct super_block *sb, struct udf_options *uopt, + int silent, struct kernel_lb_addr *fileset) +{ + struct udf_sb_info *sbi = UDF_SB(sb); + int nsr = 0; + int ret; + + if (!sb_set_blocksize(sb, uopt->blocksize)) { + if (!silent) + udf_warn(sb, "Bad block size\n"); + return -EINVAL; + } + sbi->s_last_block = uopt->lastblock; + if (!UDF_QUERY_FLAG(sb, UDF_FLAG_NOVRS)) { + /* Check that it is NSR02 compliant */ + nsr = udf_check_vsd(sb); + if (!nsr) { + if (!silent) + udf_warn(sb, "No VRS found\n"); + return -EINVAL; + } + if (nsr == -1) + udf_debug("Failed to read sector at offset %d. " + "Assuming open disc. Skipping validity " + "check\n", VSD_FIRST_SECTOR_OFFSET); + if (!sbi->s_last_block) + sbi->s_last_block = udf_get_last_block(sb); + } else { + udf_debug("Validity check skipped because of novrs option\n"); + } + + /* Look for anchor block and load Volume Descriptor Sequence */ + sbi->s_anchor = uopt->anchor; + ret = udf_scan_anchors(sb, &sbi->s_last_block, fileset); + if (ret < 0) { + if (!silent && ret == -EAGAIN) + udf_warn(sb, "No anchor found\n"); + return ret; + } + return 0; +} + +static void udf_mark_buffer_dirty(struct buffer_head *bh) +{ + /* + * We set buffer uptodate unconditionally here to avoid spurious + * warnings from mark_buffer_dirty() when previous EIO has marked + * the buffer as !uptodate + */ + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); +} + +static void udf_finalize_lvid(struct logicalVolIntegrityDesc *lvid) +{ + struct timespec64 ts; + + ktime_get_real_ts64(&ts); + udf_time_to_disk_stamp(&lvid->recordingDateAndTime, ts); + lvid->descTag.descCRC = cpu_to_le16( + crc_itu_t(0, (char *)lvid + sizeof(struct tag), + le16_to_cpu(lvid->descTag.descCRCLength))); + lvid->descTag.tagChecksum = udf_tag_checksum(&lvid->descTag); +} + +static void udf_open_lvid(struct super_block *sb) +{ + struct udf_sb_info *sbi = UDF_SB(sb); + struct buffer_head *bh = sbi->s_lvid_bh; + struct logicalVolIntegrityDesc *lvid; + struct logicalVolIntegrityDescImpUse *lvidiu; + + if (!bh) + return; + lvid = (struct logicalVolIntegrityDesc *)bh->b_data; + lvidiu = udf_sb_lvidiu(sb); + if (!lvidiu) + return; + + mutex_lock(&sbi->s_alloc_mutex); + lvidiu->impIdent.identSuffix[0] = UDF_OS_CLASS_UNIX; + lvidiu->impIdent.identSuffix[1] = UDF_OS_ID_LINUX; + if (le32_to_cpu(lvid->integrityType) == LVID_INTEGRITY_TYPE_CLOSE) + lvid->integrityType = cpu_to_le32(LVID_INTEGRITY_TYPE_OPEN); + else + UDF_SET_FLAG(sb, UDF_FLAG_INCONSISTENT); + + udf_finalize_lvid(lvid); + udf_mark_buffer_dirty(bh); + sbi->s_lvid_dirty = 0; + mutex_unlock(&sbi->s_alloc_mutex); + /* Make opening of filesystem visible on the media immediately */ + sync_dirty_buffer(bh); +} + +static void udf_close_lvid(struct super_block *sb) +{ + struct udf_sb_info *sbi = UDF_SB(sb); + struct buffer_head *bh = sbi->s_lvid_bh; + struct logicalVolIntegrityDesc *lvid; + struct logicalVolIntegrityDescImpUse *lvidiu; + + if (!bh) + return; + lvid = (struct logicalVolIntegrityDesc *)bh->b_data; + lvidiu = udf_sb_lvidiu(sb); + if (!lvidiu) + return; + + mutex_lock(&sbi->s_alloc_mutex); + lvidiu->impIdent.identSuffix[0] = UDF_OS_CLASS_UNIX; + lvidiu->impIdent.identSuffix[1] = UDF_OS_ID_LINUX; + if (UDF_MAX_WRITE_VERSION > le16_to_cpu(lvidiu->maxUDFWriteRev)) + lvidiu->maxUDFWriteRev = cpu_to_le16(UDF_MAX_WRITE_VERSION); + if (sbi->s_udfrev > le16_to_cpu(lvidiu->minUDFReadRev)) + lvidiu->minUDFReadRev = cpu_to_le16(sbi->s_udfrev); + if (sbi->s_udfrev > le16_to_cpu(lvidiu->minUDFWriteRev)) + lvidiu->minUDFWriteRev = cpu_to_le16(sbi->s_udfrev); + if (!UDF_QUERY_FLAG(sb, UDF_FLAG_INCONSISTENT)) + lvid->integrityType = cpu_to_le32(LVID_INTEGRITY_TYPE_CLOSE); + + udf_finalize_lvid(lvid); + udf_mark_buffer_dirty(bh); + sbi->s_lvid_dirty = 0; + mutex_unlock(&sbi->s_alloc_mutex); + /* Make closing of filesystem visible on the media immediately */ + sync_dirty_buffer(bh); +} + +u64 lvid_get_unique_id(struct super_block *sb) +{ + struct buffer_head *bh; + struct udf_sb_info *sbi = UDF_SB(sb); + struct logicalVolIntegrityDesc *lvid; + struct logicalVolHeaderDesc *lvhd; + u64 uniqueID; + u64 ret; + + bh = sbi->s_lvid_bh; + if (!bh) + return 0; + + lvid = (struct logicalVolIntegrityDesc *)bh->b_data; + lvhd = (struct logicalVolHeaderDesc *)lvid->logicalVolContentsUse; + + mutex_lock(&sbi->s_alloc_mutex); + ret = uniqueID = le64_to_cpu(lvhd->uniqueID); + if (!(++uniqueID & 0xFFFFFFFF)) + uniqueID += 16; + lvhd->uniqueID = cpu_to_le64(uniqueID); + udf_updated_lvid(sb); + mutex_unlock(&sbi->s_alloc_mutex); + + return ret; +} + +static int udf_fill_super(struct super_block *sb, struct fs_context *fc) +{ + int ret = -EINVAL; + struct inode *inode = NULL; + struct udf_options *uopt = fc->fs_private; + struct kernel_lb_addr rootdir, fileset; + struct udf_sb_info *sbi; + bool lvid_open = false; + int silent = fc->sb_flags & SB_SILENT; + + sbi = kzalloc_obj(*sbi); + if (!sbi) + return -ENOMEM; + + sb->s_fs_info = sbi; + + mutex_init(&sbi->s_alloc_mutex); + + fileset.logicalBlockNum = 0xFFFFFFFF; + fileset.partitionReferenceNum = 0xFFFF; + + sbi->s_flags = uopt->flags; + sbi->s_uid = uopt->uid; + sbi->s_gid = uopt->gid; + sbi->s_umask = uopt->umask; + sbi->s_fmode = uopt->fmode; + sbi->s_dmode = uopt->dmode; + sbi->s_nls_map = uopt->nls_map; + uopt->nls_map = NULL; + rwlock_init(&sbi->s_cred_lock); + + if (uopt->session == 0xFFFFFFFF) + sbi->s_session = udf_get_last_session(sb); + else + sbi->s_session = uopt->session; + + udf_debug("Multi-session=%d\n", sbi->s_session); + + /* Fill in the rest of the superblock */ + sb->s_op = &udf_sb_ops; + sb->s_export_op = &udf_export_ops; + + sb->s_magic = UDF_SUPER_MAGIC; + sb->s_time_gran = 1000; + + if (uopt->flags & (1 << UDF_FLAG_BLOCKSIZE_SET)) { + ret = udf_load_vrs(sb, uopt, silent, &fileset); + } else { + uopt->blocksize = bdev_logical_block_size(sb->s_bdev); + while (uopt->blocksize <= 4096) { + ret = udf_load_vrs(sb, uopt, silent, &fileset); + if (ret < 0) { + if (!silent && ret != -EACCES) { + pr_notice("Scanning with blocksize %u failed\n", + uopt->blocksize); + } + brelse(sbi->s_lvid_bh); + sbi->s_lvid_bh = NULL; + /* + * EACCES is special - we want to propagate to + * upper layers that we cannot handle RW mount. + */ + if (ret == -EACCES) + break; + } else + break; + + uopt->blocksize <<= 1; + } + } + if (ret < 0) { + if (ret == -EAGAIN) { + udf_warn(sb, "No partition found (1)\n"); + ret = -EINVAL; + } + goto error_out; + } + + udf_debug("Lastblock=%u\n", sbi->s_last_block); + + if (sbi->s_lvid_bh) { + struct logicalVolIntegrityDescImpUse *lvidiu = + udf_sb_lvidiu(sb); + uint16_t minUDFReadRev; + uint16_t minUDFWriteRev; + + if (!lvidiu) { + ret = -EINVAL; + goto error_out; + } + minUDFReadRev = le16_to_cpu(lvidiu->minUDFReadRev); + minUDFWriteRev = le16_to_cpu(lvidiu->minUDFWriteRev); + if (minUDFReadRev > UDF_MAX_READ_VERSION) { + udf_err(sb, "minUDFReadRev=%x (max is %x)\n", + minUDFReadRev, + UDF_MAX_READ_VERSION); + ret = -EINVAL; + goto error_out; + } else if (minUDFWriteRev > UDF_MAX_WRITE_VERSION) { + if (!sb_rdonly(sb)) { + ret = -EACCES; + goto error_out; + } + UDF_SET_FLAG(sb, UDF_FLAG_RW_INCOMPAT); + } + + sbi->s_udfrev = minUDFWriteRev; + + if (minUDFReadRev >= UDF_VERS_USE_EXTENDED_FE) + UDF_SET_FLAG(sb, UDF_FLAG_USE_EXTENDED_FE); + if (minUDFReadRev >= UDF_VERS_USE_STREAMS) + UDF_SET_FLAG(sb, UDF_FLAG_USE_STREAMS); + } + + if (!sbi->s_partitions) { + udf_warn(sb, "No partition found (2)\n"); + ret = -EINVAL; + goto error_out; + } + + if (sbi->s_partmaps[sbi->s_partition].s_partition_flags & + UDF_PART_FLAG_READ_ONLY) { + if (!sb_rdonly(sb)) { + ret = -EACCES; + goto error_out; + } + UDF_SET_FLAG(sb, UDF_FLAG_RW_INCOMPAT); + } + + ret = udf_find_fileset(sb, &fileset, &rootdir); + if (ret < 0) { + udf_warn(sb, "No fileset found\n"); + goto error_out; + } + + if (!silent) { + struct timestamp ts; + udf_time_to_disk_stamp(&ts, sbi->s_record_time); + udf_info("Mounting volume '%s', timestamp %04u/%02u/%02u %02u:%02u (%x)\n", + sbi->s_volume_ident, + le16_to_cpu(ts.year), ts.month, ts.day, + ts.hour, ts.minute, le16_to_cpu(ts.typeAndTimezone)); + } + if (!sb_rdonly(sb)) { + udf_open_lvid(sb); + lvid_open = true; + } + + /* Assign the root inode */ + /* assign inodes by physical block number */ + /* perhaps it's not extensible enough, but for now ... */ + inode = udf_iget(sb, &rootdir); + if (IS_ERR(inode)) { + udf_err(sb, "Error in udf_iget, block=%u, partition=%u\n", + rootdir.logicalBlockNum, rootdir.partitionReferenceNum); + ret = PTR_ERR(inode); + goto error_out; + } + + /* Allocate a dentry for the root inode */ + sb->s_root = d_make_root(inode); + if (!sb->s_root) { + udf_err(sb, "Couldn't allocate root dentry\n"); + ret = -ENOMEM; + goto error_out; + } + sb->s_maxbytes = UDF_MAX_FILESIZE; + sb->s_max_links = UDF_MAX_LINKS; + return 0; + +error_out: + iput(sbi->s_vat_inode); + unload_nls(sbi->s_nls_map); + if (lvid_open) + udf_close_lvid(sb); + brelse(sbi->s_lvid_bh); + udf_sb_free_partitions(sb); + kfree(sbi); + sb->s_fs_info = NULL; + + return ret; +} + +void _udf_err(struct super_block *sb, const char *function, + const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + pr_err("error (device %s): %s: %pV", sb->s_id, function, &vaf); + + va_end(args); +} + +void _udf_warn(struct super_block *sb, const char *function, + const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + pr_warn("warning (device %s): %s: %pV", sb->s_id, function, &vaf); + + va_end(args); +} + +static void udf_put_super(struct super_block *sb) +{ + struct udf_sb_info *sbi; + + sbi = UDF_SB(sb); + + iput(sbi->s_vat_inode); + unload_nls(sbi->s_nls_map); + if (!sb_rdonly(sb)) + udf_close_lvid(sb); + brelse(sbi->s_lvid_bh); + udf_sb_free_partitions(sb); + mutex_destroy(&sbi->s_alloc_mutex); + kfree(sb->s_fs_info); + sb->s_fs_info = NULL; +} + +static int udf_sync_fs(struct super_block *sb, int wait) +{ + struct udf_sb_info *sbi = UDF_SB(sb); + + mutex_lock(&sbi->s_alloc_mutex); + if (sbi->s_lvid_dirty) { + struct buffer_head *bh = sbi->s_lvid_bh; + struct logicalVolIntegrityDesc *lvid; + + lvid = (struct logicalVolIntegrityDesc *)bh->b_data; + udf_finalize_lvid(lvid); + + /* + * Blockdevice will be synced later so we don't have to submit + * the buffer for IO + */ + udf_mark_buffer_dirty(bh); + sbi->s_lvid_dirty = 0; + } + mutex_unlock(&sbi->s_alloc_mutex); + + return 0; +} + +static int udf_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + struct udf_sb_info *sbi = UDF_SB(sb); + struct logicalVolIntegrityDescImpUse *lvidiu; + u64 id = huge_encode_dev(sb->s_bdev->bd_dev); + + lvidiu = udf_sb_lvidiu(sb); + buf->f_type = UDF_SUPER_MAGIC; + buf->f_bsize = sb->s_blocksize; + buf->f_blocks = sbi->s_partmaps[sbi->s_partition].s_partition_len; + buf->f_bfree = udf_count_free(sb); + buf->f_bavail = buf->f_bfree; + /* + * Let's pretend each free block is also a free 'inode' since UDF does + * not have separate preallocated table of inodes. + */ + buf->f_files = (lvidiu != NULL ? (le32_to_cpu(lvidiu->numFiles) + + le32_to_cpu(lvidiu->numDirs)) : 0) + + buf->f_bfree; + buf->f_ffree = buf->f_bfree; + buf->f_namelen = UDF_NAME_LEN; + buf->f_fsid = u64_to_fsid(id); + + return 0; +} + +static unsigned int udf_count_free_bitmap(struct super_block *sb, + struct udf_bitmap *bitmap) +{ + struct buffer_head *bh = NULL; + unsigned int accum = 0; + int index; + udf_pblk_t block = 0, newblock; + struct kernel_lb_addr loc; + uint32_t bytes; + uint8_t *ptr; + uint16_t ident; + struct spaceBitmapDesc *bm; + + loc.logicalBlockNum = bitmap->s_extPosition; + loc.partitionReferenceNum = UDF_SB(sb)->s_partition; + bh = udf_read_ptagged(sb, &loc, 0, &ident); + + if (!bh) { + udf_err(sb, "udf_count_free failed\n"); + goto out; + } else if (ident != TAG_IDENT_SBD) { + brelse(bh); + udf_err(sb, "udf_count_free failed\n"); + goto out; + } + + bm = (struct spaceBitmapDesc *)bh->b_data; + bytes = le32_to_cpu(bm->numOfBytes); + index = sizeof(struct spaceBitmapDesc); /* offset in first block only */ + ptr = (uint8_t *)bh->b_data; + + while (bytes > 0) { + u32 cur_bytes = min_t(u32, bytes, sb->s_blocksize - index); + accum += bitmap_weight((const unsigned long *)(ptr + index), + cur_bytes * 8); + bytes -= cur_bytes; + if (bytes) { + brelse(bh); + newblock = udf_get_lb_pblock(sb, &loc, ++block); + bh = sb_bread(sb, newblock); + if (!bh) { + udf_debug("read failed\n"); + goto out; + } + index = 0; + ptr = (uint8_t *)bh->b_data; + } + } + brelse(bh); +out: + return accum; +} + +static unsigned int udf_count_free_table(struct super_block *sb, + struct inode *table) +{ + unsigned int accum = 0; + uint32_t elen; + struct kernel_lb_addr eloc; + struct extent_position epos; + int8_t etype; + + mutex_lock(&UDF_SB(sb)->s_alloc_mutex); + epos.block = UDF_I(table)->i_location; + epos.offset = sizeof(struct unallocSpaceEntry); + epos.bh = NULL; + + while (udf_next_aext(table, &epos, &eloc, &elen, &etype, 1) > 0) + accum += (elen >> table->i_sb->s_blocksize_bits); + + brelse(epos.bh); + mutex_unlock(&UDF_SB(sb)->s_alloc_mutex); + + return accum; +} + +static unsigned int udf_count_free(struct super_block *sb) +{ + unsigned int accum = 0; + struct udf_sb_info *sbi = UDF_SB(sb); + struct udf_part_map *map; + unsigned int part = sbi->s_partition; + int ptype = sbi->s_partmaps[part].s_partition_type; + + if (ptype == UDF_METADATA_MAP25) { + part = sbi->s_partmaps[part].s_type_specific.s_metadata. + s_phys_partition_ref; + } else if (ptype == UDF_VIRTUAL_MAP15 || ptype == UDF_VIRTUAL_MAP20) { + /* + * Filesystems with VAT are append-only and we cannot write to + * them. Let's just report 0 here. + */ + return 0; + } + + if (sbi->s_lvid_bh) { + struct logicalVolIntegrityDesc *lvid = + (struct logicalVolIntegrityDesc *) + sbi->s_lvid_bh->b_data; + if (le32_to_cpu(lvid->numOfPartitions) > part) { + accum = le32_to_cpu( + lvid->freeSpaceTable[part]); + if (accum == 0xFFFFFFFF) + accum = 0; + } + } + + if (accum) + return accum; + + map = &sbi->s_partmaps[part]; + if (map->s_partition_flags & UDF_PART_FLAG_UNALLOC_BITMAP) { + accum += udf_count_free_bitmap(sb, + map->s_uspace.s_bitmap); + } + if (accum) + return accum; + + if (map->s_partition_flags & UDF_PART_FLAG_UNALLOC_TABLE) { + accum += udf_count_free_table(sb, + map->s_uspace.s_table); + } + return accum; +} + +MODULE_AUTHOR("Ben Fennema"); +MODULE_DESCRIPTION("Universal Disk Format Filesystem"); +MODULE_LICENSE("GPL"); +module_init(init_udf_fs) +module_exit(exit_udf_fs) diff --git a/fs/udf/symlink.c b/fs/udf/symlink.c new file mode 100644 index 000000000..a05d1888a --- /dev/null +++ b/fs/udf/symlink.c @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * symlink.c + * + * PURPOSE + * Symlink handling routines for the OSTA-UDF(tm) filesystem. + * + * COPYRIGHT + * (C) 1998-2001 Ben Fennema + * (C) 1999 Stelias Computing Inc + * + * HISTORY + * + * 04/16/99 blf Created. + * + */ + +#include "udfdecl.h" +#include +#include +#include +#include +#include +#include +#include +#include "udf_i.h" + +static int udf_pc_to_char(struct super_block *sb, unsigned char *from, + int fromlen, unsigned char *to, int tolen) +{ + struct pathComponent *pc; + int elen = 0; + int comp_len; + unsigned char *p = to; + + /* Reserve one byte for terminating \0 */ + tolen--; + while (elen < fromlen) { + if (fromlen - elen < sizeof(struct pathComponent)) + return -EIO; + pc = (struct pathComponent *)(from + elen); + elen += sizeof(struct pathComponent); + switch (pc->componentType) { + case 1: + /* + * Symlink points to some place which should be agreed + * upon between originator and receiver of the media. Ignore. + */ + if (pc->lengthComponentIdent > 0) { + elen += pc->lengthComponentIdent; + break; + } + fallthrough; + case 2: + if (tolen == 0) + return -ENAMETOOLONG; + p = to; + *p++ = '/'; + tolen--; + break; + case 3: + if (tolen < 3) + return -ENAMETOOLONG; + memcpy(p, "../", 3); + p += 3; + tolen -= 3; + break; + case 4: + if (tolen < 2) + return -ENAMETOOLONG; + memcpy(p, "./", 2); + p += 2; + tolen -= 2; + /* that would be . - just ignore */ + break; + case 5: + elen += pc->lengthComponentIdent; + if (elen > fromlen) + return -EIO; + comp_len = udf_get_filename(sb, pc->componentIdent, + pc->lengthComponentIdent, + p, tolen); + if (comp_len < 0) + return comp_len; + + p += comp_len; + tolen -= comp_len; + if (tolen == 0) + return -ENAMETOOLONG; + *p++ = '/'; + tolen--; + break; + } + } + if (p > to + 1) + p[-1] = '\0'; + else + p[0] = '\0'; + return 0; +} + +static int udf_symlink_filler(struct file *file, struct folio *folio) +{ + struct inode *inode = folio->mapping->host; + struct buffer_head *bh = NULL; + unsigned char *symlink; + int err = 0; + unsigned char *p = folio_address(folio); + struct udf_inode_info *iinfo = UDF_I(inode); + + /* We don't support symlinks longer than one block */ + if (inode->i_size > inode->i_sb->s_blocksize) { + err = -ENAMETOOLONG; + goto out; + } + + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) { + symlink = iinfo->i_data + iinfo->i_lenEAttr; + } else { + bh = udf_bread(inode, 0, 0, &err); + if (!bh) { + if (!err) + err = -EFSCORRUPTED; + goto out; + } + symlink = bh->b_data; + } + + err = udf_pc_to_char(inode->i_sb, symlink, inode->i_size, p, PAGE_SIZE); + brelse(bh); +out: + folio_end_read(folio, err == 0); + return err; +} + +static int udf_symlink_getattr(struct mnt_idmap *idmap, + const struct path *path, struct kstat *stat, + u32 request_mask, unsigned int flags) +{ + struct dentry *dentry = path->dentry; + struct inode *inode = d_backing_inode(dentry); + struct folio *folio; + + generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat); + folio = read_mapping_folio(inode->i_mapping, 0, NULL); + if (IS_ERR(folio)) + return PTR_ERR(folio); + /* + * UDF uses non-trivial encoding of symlinks so i_size does not match + * number of characters reported by readlink(2) which apparently some + * applications expect. Also POSIX says that "The value returned in the + * st_size field shall be the length of the contents of the symbolic + * link, and shall not count a trailing null if one is present." So + * let's report the length of string returned by readlink(2) for + * st_size. + */ + stat->size = strlen(folio_address(folio)); + folio_put(folio); + + return 0; +} + +/* + * symlinks can't do much... + */ +const struct address_space_operations udf_symlink_aops = { + .read_folio = udf_symlink_filler, +}; + +const struct inode_operations udf_symlink_inode_operations = { + .get_link = page_get_link, + .getattr = udf_symlink_getattr, +}; diff --git a/fs/udf/truncate.c b/fs/udf/truncate.c new file mode 100644 index 000000000..0990f94b8 --- /dev/null +++ b/fs/udf/truncate.c @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * truncate.c + * + * PURPOSE + * Truncate handling routines for the OSTA-UDF(tm) filesystem. + * + * COPYRIGHT + * (C) 1999-2004 Ben Fennema + * (C) 1999 Stelias Computing Inc + * + * HISTORY + * + * 02/24/99 blf Created. + * + */ + +#include "udfdecl.h" +#include +#include + +#include "udf_i.h" +#include "udf_sb.h" + +static void extent_trunc(struct inode *inode, struct extent_position *epos, + struct kernel_lb_addr *eloc, int8_t etype, uint32_t elen, + uint32_t nelen) +{ + struct kernel_lb_addr neloc = {}; + int last_block = (elen + inode->i_sb->s_blocksize - 1) >> + inode->i_sb->s_blocksize_bits; + int first_block = (nelen + inode->i_sb->s_blocksize - 1) >> + inode->i_sb->s_blocksize_bits; + + if (nelen) { + if (etype == (EXT_NOT_RECORDED_ALLOCATED >> 30)) { + udf_free_blocks(inode->i_sb, inode, eloc, 0, + last_block); + etype = (EXT_NOT_RECORDED_NOT_ALLOCATED >> 30); + } else + neloc = *eloc; + nelen = (etype << 30) | nelen; + } + + if (elen != nelen) { + udf_write_aext(inode, epos, &neloc, nelen, 0); + if (last_block > first_block) { + if (etype == (EXT_RECORDED_ALLOCATED >> 30)) + mark_inode_dirty(inode); + + if (etype != (EXT_NOT_RECORDED_NOT_ALLOCATED >> 30)) + udf_free_blocks(inode->i_sb, inode, eloc, + first_block, + last_block - first_block); + } + } +} + +/* + * Truncate the last extent to match i_size. This function assumes + * that preallocation extent is already truncated. + */ +void udf_truncate_tail_extent(struct inode *inode) +{ + struct extent_position epos = {}; + struct kernel_lb_addr eloc; + uint32_t elen, nelen; + uint64_t lbcount = 0; + int8_t etype = -1, netype; + int adsize; + struct udf_inode_info *iinfo = UDF_I(inode); + int ret; + + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB || + inode->i_size == iinfo->i_lenExtents) + return; + /* Are we going to delete the file anyway? */ + if (inode->i_nlink == 0) + return; + + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_SHORT) + adsize = sizeof(struct short_ad); + else if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_LONG) + adsize = sizeof(struct long_ad); + else + BUG(); + + /* Find the last extent in the file */ + while (1) { + ret = udf_next_aext(inode, &epos, &eloc, &elen, &netype, 1); + if (ret <= 0) + break; + etype = netype; + lbcount += elen; + if (lbcount > inode->i_size) { + if (lbcount - inode->i_size >= inode->i_sb->s_blocksize) + udf_warn(inode->i_sb, + "Too long extent after EOF in inode %u: i_size: %lld lbcount: %lld extent %u+%u\n", + (unsigned)inode->i_ino, + (long long)inode->i_size, + (long long)lbcount, + (unsigned)eloc.logicalBlockNum, + (unsigned)elen); + nelen = elen - (lbcount - inode->i_size); + epos.offset -= adsize; + extent_trunc(inode, &epos, &eloc, etype, elen, nelen); + epos.offset += adsize; + if (udf_next_aext(inode, &epos, &eloc, &elen, + &netype, 1) > 0) + udf_err(inode->i_sb, + "Extent after EOF in inode %u\n", + (unsigned)inode->i_ino); + break; + } + } + /* This inode entry is in-memory only and thus we don't have to mark + * the inode dirty */ + if (ret >= 0) + iinfo->i_lenExtents = inode->i_size; + brelse(epos.bh); +} + +void udf_discard_prealloc(struct inode *inode) +{ + struct extent_position epos = {}; + struct extent_position prev_epos = {}; + struct kernel_lb_addr eloc; + uint32_t elen; + uint64_t lbcount = 0; + int8_t etype = -1; + struct udf_inode_info *iinfo = UDF_I(inode); + int bsize = i_blocksize(inode); + int8_t tmpetype = -1; + int ret; + + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB || + ALIGN(inode->i_size, bsize) == ALIGN(iinfo->i_lenExtents, bsize)) + return; + + epos.block = iinfo->i_location; + + /* Find the last extent in the file */ + while (1) { + ret = udf_next_aext(inode, &epos, &eloc, &elen, &tmpetype, 0); + if (ret < 0) + goto out; + if (ret == 0) + break; + brelse(prev_epos.bh); + prev_epos = epos; + if (prev_epos.bh) + get_bh(prev_epos.bh); + + ret = udf_next_aext(inode, &epos, &eloc, &elen, &etype, 1); + if (ret < 0) + goto out; + lbcount += elen; + } + + if (etype == (EXT_NOT_RECORDED_ALLOCATED >> 30)) { + lbcount -= elen; + udf_delete_aext(inode, prev_epos, NULL); + udf_free_blocks(inode->i_sb, inode, &eloc, 0, + DIV_ROUND_UP(elen, bsize)); + } + /* This inode entry is in-memory only and thus we don't have to mark + * the inode dirty */ + iinfo->i_lenExtents = lbcount; +out: + brelse(epos.bh); + brelse(prev_epos.bh); +} + +static void udf_update_alloc_ext_desc(struct inode *inode, + struct extent_position *epos, + u32 lenalloc) +{ + struct super_block *sb = inode->i_sb; + struct udf_sb_info *sbi = UDF_SB(sb); + + struct allocExtDesc *aed = (struct allocExtDesc *) (epos->bh->b_data); + int len = sizeof(struct allocExtDesc); + + aed->lengthAllocDescs = cpu_to_le32(lenalloc); + if (!UDF_QUERY_FLAG(sb, UDF_FLAG_STRICT) || sbi->s_udfrev >= 0x0201) + len += lenalloc; + + udf_update_tag(epos->bh->b_data, len); + mmb_mark_buffer_dirty(epos->bh, &UDF_I(inode)->i_metadata_bhs); +} + +/* + * Truncate extents of inode to inode->i_size. This function can be used only + * for making file shorter. For making file longer, udf_extend_file() has to + * be used. + */ +int udf_truncate_extents(struct inode *inode) +{ + struct extent_position epos; + struct kernel_lb_addr eloc, neloc = {}; + uint32_t elen, nelen = 0, indirect_ext_len = 0, lenalloc; + int8_t etype; + struct super_block *sb = inode->i_sb; + sector_t first_block = inode->i_size >> sb->s_blocksize_bits, offset; + loff_t byte_offset; + int adsize; + struct udf_inode_info *iinfo = UDF_I(inode); + int ret = 0; + + if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_SHORT) + adsize = sizeof(struct short_ad); + else if (iinfo->i_alloc_type == ICBTAG_FLAG_AD_LONG) + adsize = sizeof(struct long_ad); + else + BUG(); + + ret = inode_bmap(inode, first_block, &epos, &eloc, &elen, &offset, &etype); + if (ret < 0) + return ret; + byte_offset = (offset << sb->s_blocksize_bits) + + (inode->i_size & (sb->s_blocksize - 1)); + if (ret == 0) { + /* We should extend the file? */ + WARN_ON(byte_offset); + return 0; + } + epos.offset -= adsize; + extent_trunc(inode, &epos, &eloc, etype, elen, byte_offset); + epos.offset += adsize; + if (byte_offset) + lenalloc = epos.offset; + else + lenalloc = epos.offset - adsize; + + if (!epos.bh) + lenalloc -= udf_file_entry_alloc_offset(inode); + else + lenalloc -= sizeof(struct allocExtDesc); + + while ((ret = udf_current_aext(inode, &epos, &eloc, + &elen, &etype, 0)) > 0) { + if (etype == (EXT_NEXT_EXTENT_ALLOCDESCS >> 30)) { + udf_write_aext(inode, &epos, &neloc, nelen, 0); + if (indirect_ext_len) { + /* We managed to free all extents in the + * indirect extent - free it too */ + BUG_ON(!epos.bh); + udf_free_blocks(sb, NULL, &epos.block, + 0, indirect_ext_len); + } else if (!epos.bh) { + iinfo->i_lenAlloc = lenalloc; + mark_inode_dirty(inode); + } else + udf_update_alloc_ext_desc(inode, + &epos, lenalloc); + brelse(epos.bh); + epos.offset = sizeof(struct allocExtDesc); + epos.block = eloc; + epos.bh = sb_bread(sb, + udf_get_lb_pblock(sb, &eloc, 0)); + /* Error reading indirect block? */ + if (!epos.bh) + return -EIO; + if (elen) + indirect_ext_len = + (elen + sb->s_blocksize - 1) >> + sb->s_blocksize_bits; + else + indirect_ext_len = 1; + } else { + extent_trunc(inode, &epos, &eloc, etype, elen, 0); + epos.offset += adsize; + } + } + + if (ret < 0) { + brelse(epos.bh); + return ret; + } + + if (indirect_ext_len) { + BUG_ON(!epos.bh); + udf_free_blocks(sb, NULL, &epos.block, 0, indirect_ext_len); + } else if (!epos.bh) { + iinfo->i_lenAlloc = lenalloc; + mark_inode_dirty(inode); + } else + udf_update_alloc_ext_desc(inode, &epos, lenalloc); + iinfo->i_lenExtents = inode->i_size; + + brelse(epos.bh); + return 0; +} diff --git a/fs/udf/udf_i.h b/fs/udf/udf_i.h new file mode 100644 index 000000000..fdaa88c49 --- /dev/null +++ b/fs/udf/udf_i.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _UDF_I_H +#define _UDF_I_H + +struct extent_position { + struct buffer_head *bh; + uint32_t offset; + struct kernel_lb_addr block; +}; + +struct udf_ext_cache { + /* Extent position */ + struct extent_position epos; + /* Start logical offset in bytes */ + loff_t lstart; +}; + +/* + * The i_data_sem and i_mutex serve for protection of allocation information + * of a regular files and symlinks. This includes all extents belonging to + * the file/symlink, a fact whether data are in-inode or in external data + * blocks, preallocation, goal block information... When extents are read, + * i_mutex or i_data_sem must be held (for reading is enough in case of + * i_data_sem). When extents are changed, i_data_sem must be held for writing + * and also i_mutex must be held. + * + * For directories i_mutex is used for all the necessary protection. + */ + +struct udf_inode_info { + struct timespec64 i_crtime; + /* Physical address of inode */ + struct kernel_lb_addr i_location; + __u64 i_unique; + __u32 i_lenEAttr; + __u32 i_lenAlloc; + __u64 i_lenExtents; + __u32 i_next_alloc_block; + __u32 i_next_alloc_goal; + __u32 i_checkpoint; + __u32 i_extraPerms; + unsigned i_alloc_type : 3; + unsigned i_efe : 1; /* extendedFileEntry */ + unsigned i_use : 1; /* unallocSpaceEntry */ + unsigned i_strat4096 : 1; + unsigned i_streamdir : 1; + unsigned i_hidden : 1; /* hidden system inode */ + unsigned reserved : 24; + __u8 *i_data; + struct kernel_lb_addr i_locStreamdir; + __u64 i_lenStreams; + struct rw_semaphore i_data_sem; + struct mapping_metadata_bhs i_metadata_bhs; + struct udf_ext_cache cached_extent; + /* Spinlock for protecting extent cache */ + spinlock_t i_extent_cache_lock; + struct inode vfs_inode; +}; + +static inline struct udf_inode_info *UDF_I(struct inode *inode) +{ + return container_of(inode, struct udf_inode_info, vfs_inode); +} + +#endif /* _UDF_I_H) */ diff --git a/fs/udf/udf_sb.h b/fs/udf/udf_sb.h new file mode 100644 index 000000000..8399accc7 --- /dev/null +++ b/fs/udf/udf_sb.h @@ -0,0 +1,183 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_UDF_SB_H +#define __LINUX_UDF_SB_H + +#include +#include +#include + +/* + * Even UDF 2.6 media should have version <= 0x250 but apparently there are + * some broken filesystems with version set to 0x260. Accommodate those. + */ +#define UDF_MAX_READ_VERSION 0x0260 +#define UDF_MAX_WRITE_VERSION 0x0201 + +#define UDF_FLAG_USE_EXTENDED_FE 0 +#define UDF_VERS_USE_EXTENDED_FE 0x0200 +#define UDF_FLAG_USE_STREAMS 1 +#define UDF_VERS_USE_STREAMS 0x0200 +#define UDF_FLAG_USE_SHORT_AD 2 +#define UDF_FLAG_USE_AD_IN_ICB 3 +#define UDF_FLAG_USE_FILE_CTIME_EA 4 +#define UDF_FLAG_STRICT 5 +#define UDF_FLAG_UNDELETE 6 +#define UDF_FLAG_UNHIDE 7 +#define UDF_FLAG_NOVRS 8 +#define UDF_FLAG_UID_FORGET 11 /* save -1 for uid to disk */ +#define UDF_FLAG_GID_FORGET 12 +#define UDF_FLAG_UID_SET 13 +#define UDF_FLAG_GID_SET 14 +#define UDF_FLAG_SESSION_SET 15 +#define UDF_FLAG_LASTBLOCK_SET 16 +#define UDF_FLAG_BLOCKSIZE_SET 17 +#define UDF_FLAG_INCONSISTENT 18 +#define UDF_FLAG_RW_INCOMPAT 19 /* Set when we find RW incompatible + * feature */ + +#define UDF_PART_FLAG_UNALLOC_BITMAP 0x0001 +#define UDF_PART_FLAG_UNALLOC_TABLE 0x0002 +#define UDF_PART_FLAG_READ_ONLY 0x0010 +#define UDF_PART_FLAG_WRITE_ONCE 0x0020 +#define UDF_PART_FLAG_REWRITABLE 0x0040 +#define UDF_PART_FLAG_OVERWRITABLE 0x0080 + +#define UDF_MAX_BLOCK_LOADED 8 + +#define UDF_TYPE1_MAP15 0x1511U +#define UDF_VIRTUAL_MAP15 0x1512U +#define UDF_VIRTUAL_MAP20 0x2012U +#define UDF_SPARABLE_MAP15 0x1522U +#define UDF_METADATA_MAP25 0x2511U + +#define UDF_INVALID_MODE ((umode_t)-1) + +#define MF_DUPLICATE_MD 0x01 +#define MF_MIRROR_FE_LOADED 0x02 + +struct udf_meta_data { + __u32 s_meta_file_loc; + __u32 s_mirror_file_loc; + __u32 s_bitmap_file_loc; + __u32 s_alloc_unit_size; + __u16 s_align_unit_size; + /* + * Partition Reference Number of the associated physical / sparable + * partition + */ + __u16 s_phys_partition_ref; + int s_flags; + struct inode *s_metadata_fe; + struct inode *s_mirror_fe; + struct inode *s_bitmap_fe; +}; + +struct udf_sparing_data { + __u16 s_packet_len; + struct buffer_head *s_spar_map[4]; +}; + +struct udf_virtual_data { + __u32 s_num_entries; + __u16 s_start_offset; +}; + +struct udf_bitmap { + __u32 s_extPosition; + int s_nr_groups; + struct buffer_head *s_block_bitmap[] __counted_by(s_nr_groups); +}; + +struct udf_part_map { + union { + struct udf_bitmap *s_bitmap; + struct inode *s_table; + } s_uspace; + __u32 s_partition_root; + __u32 s_partition_len; + __u16 s_partition_type; + __u16 s_partition_num; + union { + struct udf_sparing_data s_sparing; + struct udf_virtual_data s_virtual; + struct udf_meta_data s_metadata; + } s_type_specific; + __u32 (*s_partition_func)(struct super_block *, __u32, __u16, __u32); + __u16 s_volumeseqnum; + __u16 s_partition_flags; +}; + +#pragma pack() + +struct udf_sb_info { + struct udf_part_map *s_partmaps; + __u8 s_volume_ident[32]; + + /* Overall info */ + __u16 s_partitions; + __u16 s_partition; + + /* Sector headers */ + __s32 s_session; + __u32 s_anchor; + __u32 s_last_block; + + struct buffer_head *s_lvid_bh; + + /* Default permissions */ + umode_t s_umask; + kgid_t s_gid; + kuid_t s_uid; + umode_t s_fmode; + umode_t s_dmode; + /* Lock protecting consistency of above permission settings */ + rwlock_t s_cred_lock; + + /* Root Info */ + struct timespec64 s_record_time; + + /* Fileset Info */ + __u16 s_serial_number; + + /* highest UDF revision we have recorded to this media */ + __u16 s_udfrev; + + /* Miscellaneous flags */ + unsigned long s_flags; + + /* Encoding info */ + struct nls_table *s_nls_map; + + /* VAT inode */ + struct inode *s_vat_inode; + + struct mutex s_alloc_mutex; + /* Protected by s_alloc_mutex */ + unsigned int s_lvid_dirty; +}; + +static inline struct udf_sb_info *UDF_SB(struct super_block *sb) +{ + return sb->s_fs_info; +} + +struct logicalVolIntegrityDescImpUse *udf_sb_lvidiu(struct super_block *sb); + +int udf_compute_nr_groups(struct super_block *sb, u32 partition); + +static inline int UDF_QUERY_FLAG(struct super_block *sb, int flag) +{ + return test_bit(flag, &UDF_SB(sb)->s_flags); +} + +static inline void UDF_SET_FLAG(struct super_block *sb, int flag) +{ + set_bit(flag, &UDF_SB(sb)->s_flags); +} + +static inline void UDF_CLEAR_FLAG(struct super_block *sb, int flag) +{ + clear_bit(flag, &UDF_SB(sb)->s_flags); +} + +#endif /* __LINUX_UDF_SB_H */ diff --git a/fs/udf/udfdecl.h b/fs/udf/udfdecl.h new file mode 100644 index 000000000..7d5a19814 --- /dev/null +++ b/fs/udf/udfdecl.h @@ -0,0 +1,262 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __UDF_DECL_H +#define __UDF_DECL_H + +#define pr_fmt(fmt) "UDF-fs: " fmt + +#include "ecma_167.h" +#include "osta_udf.h" + +#include +#include +#include +#include + +#include "udf_sb.h" +#include "udfend.h" +#include "udf_i.h" + +#define UDF_DEFAULT_PREALLOC_BLOCKS 8 + +extern __printf(3, 4) void _udf_err(struct super_block *sb, + const char *function, const char *fmt, ...); +#define udf_err(sb, fmt, ...) \ + _udf_err(sb, __func__, fmt, ##__VA_ARGS__) + +extern __printf(3, 4) void _udf_warn(struct super_block *sb, + const char *function, const char *fmt, ...); +#define udf_warn(sb, fmt, ...) \ + _udf_warn(sb, __func__, fmt, ##__VA_ARGS__) + +#define udf_info(fmt, ...) \ + pr_info("INFO " fmt, ##__VA_ARGS__) + +#define udf_debug(fmt, ...) \ + pr_debug("%s:%d:%s: " fmt, __FILE__, __LINE__, __func__, ##__VA_ARGS__) + +#define UDF_EXTENT_LENGTH_MASK 0x3FFFFFFF +#define UDF_EXTENT_FLAG_MASK 0xC0000000 + +#define UDF_INVALID_ID ((uint32_t)-1) + +#define UDF_NAME_PAD 4 +#define UDF_NAME_LEN 254 +#define UDF_NAME_LEN_CS0 255 + +static inline size_t udf_file_entry_alloc_offset(struct inode *inode) +{ + struct udf_inode_info *iinfo = UDF_I(inode); + if (iinfo->i_use) + return sizeof(struct unallocSpaceEntry); + else if (iinfo->i_efe) + return sizeof(struct extendedFileEntry) + iinfo->i_lenEAttr; + else + return sizeof(struct fileEntry) + iinfo->i_lenEAttr; +} + +static inline size_t udf_ext0_offset(struct inode *inode) +{ + if (UDF_I(inode)->i_alloc_type == ICBTAG_FLAG_AD_IN_ICB) + return udf_file_entry_alloc_offset(inode); + else + return 0; +} + +/* computes tag checksum */ +u8 udf_tag_checksum(const struct tag *t); + +typedef uint32_t udf_pblk_t; + +struct dentry; +struct inode; +struct task_struct; +struct buffer_head; +struct super_block; + +extern const struct export_operations udf_export_ops; +extern const struct inode_operations udf_dir_inode_operations; +extern const struct file_operations udf_dir_operations; +extern const struct inode_operations udf_file_inode_operations; +extern const struct file_operations udf_file_operations; +extern const struct inode_operations udf_symlink_inode_operations; +extern const struct address_space_operations udf_aops; +extern const struct address_space_operations udf_symlink_aops; + +struct udf_fileident_iter { + struct inode *dir; /* Directory we are working with */ + loff_t pos; /* Logical position in a dir */ + struct buffer_head *bh[2]; /* Buffer containing 'pos' and possibly + * next buffer if entry straddles + * blocks */ + struct kernel_lb_addr eloc; /* Start of extent containing 'pos' */ + uint32_t elen; /* Length of extent containing 'pos' */ + sector_t loffset; /* Block offset of 'pos' within above + * extent */ + struct extent_position epos; /* Position after the above extent */ + struct fileIdentDesc fi; /* Copied directory entry */ + uint8_t *name; /* Pointer to entry name */ + uint8_t *namebuf; /* Storage for entry name in case + * the name is split between two blocks + */ +}; + +struct udf_vds_record { + uint32_t block; + uint32_t volDescSeqNum; +}; + +struct generic_desc { + struct tag descTag; + __le32 volDescSeqNum; +}; + + +/* super.c */ + +static inline void udf_updated_lvid(struct super_block *sb) +{ + struct buffer_head *bh = UDF_SB(sb)->s_lvid_bh; + + BUG_ON(!bh); + WARN_ON_ONCE(((struct logicalVolIntegrityDesc *) + bh->b_data)->integrityType != + cpu_to_le32(LVID_INTEGRITY_TYPE_OPEN)); + UDF_SB(sb)->s_lvid_dirty = 1; +} +extern u64 lvid_get_unique_id(struct super_block *sb); +struct inode *udf_find_metadata_inode_efe(struct super_block *sb, + u32 meta_file_loc, u32 partition_num); + +/* namei.c */ +static inline unsigned int udf_dir_entry_len(struct fileIdentDesc *cfi) +{ + return ALIGN(sizeof(struct fileIdentDesc) + + le16_to_cpu(cfi->lengthOfImpUse) + cfi->lengthFileIdent, + UDF_NAME_PAD); +} + +/* file.c */ +extern long udf_ioctl(struct file *, unsigned int, unsigned long); + +/* inode.c */ +extern struct inode *__udf_iget(struct super_block *, struct kernel_lb_addr *, + bool hidden_inode); +static inline struct inode *udf_iget_special(struct super_block *sb, + struct kernel_lb_addr *ino) +{ + return __udf_iget(sb, ino, true); +} +static inline struct inode *udf_iget(struct super_block *sb, + struct kernel_lb_addr *ino) +{ + return __udf_iget(sb, ino, false); +} +extern int udf_expand_file_adinicb(struct inode *); +extern struct buffer_head *udf_bread(struct inode *inode, udf_pblk_t block, + int create, int *err); +extern int udf_setsize(struct inode *, loff_t); +extern void udf_evict_inode(struct inode *); +extern int udf_write_inode(struct inode *, struct writeback_control *wbc); +int udf_sync_inode_metadata(struct inode *, struct writeback_control *wbc); +extern int inode_bmap(struct inode *inode, sector_t block, + struct extent_position *pos, struct kernel_lb_addr *eloc, + uint32_t *elen, sector_t *offset, int8_t *etype); +int udf_get_block(struct inode *, sector_t, struct buffer_head *, int); +extern int udf_setup_indirect_aext(struct inode *inode, udf_pblk_t block, + struct extent_position *epos); +extern int __udf_add_aext(struct inode *inode, struct extent_position *epos, + struct kernel_lb_addr *eloc, uint32_t elen, int inc); +extern int udf_add_aext(struct inode *, struct extent_position *, + struct kernel_lb_addr *, uint32_t, int); +extern void udf_write_aext(struct inode *, struct extent_position *, + struct kernel_lb_addr *, uint32_t, int); +extern int8_t udf_delete_aext(struct inode *, struct extent_position, + struct kernel_lb_addr *); +extern int udf_next_aext(struct inode *inode, struct extent_position *epos, + struct kernel_lb_addr *eloc, uint32_t *elen, + int8_t *etype, int inc); +extern int udf_current_aext(struct inode *inode, struct extent_position *epos, + struct kernel_lb_addr *eloc, uint32_t *elen, + int8_t *etype, int inc); +extern void udf_update_extra_perms(struct inode *inode, umode_t mode); + +/* misc.c */ +extern struct genericFormat *udf_add_extendedattr(struct inode *, uint32_t, + uint32_t, uint8_t); +extern struct genericFormat *udf_get_extendedattr(struct inode *, uint32_t, + uint8_t); +extern struct buffer_head *udf_read_tagged(struct super_block *, uint32_t, + uint32_t, uint16_t *); +extern struct buffer_head *udf_read_ptagged(struct super_block *, + struct kernel_lb_addr *, uint32_t, + uint16_t *); +extern void udf_update_tag(char *, int); +extern void udf_new_tag(char *, uint16_t, uint16_t, uint16_t, uint32_t, int); + +/* lowlevel.c */ +extern unsigned int udf_get_last_session(struct super_block *); +udf_pblk_t udf_get_last_block(struct super_block *); + +/* partition.c */ +extern uint32_t udf_get_pblock(struct super_block *, uint32_t, uint16_t, + uint32_t); +extern uint32_t udf_get_pblock_virt15(struct super_block *, uint32_t, uint16_t, + uint32_t); +extern uint32_t udf_get_pblock_virt20(struct super_block *, uint32_t, uint16_t, + uint32_t); +extern uint32_t udf_get_pblock_spar15(struct super_block *, uint32_t, uint16_t, + uint32_t); +extern uint32_t udf_get_pblock_meta25(struct super_block *, uint32_t, uint16_t, + uint32_t); +extern int udf_relocate_blocks(struct super_block *, long, long *); + +static inline uint32_t +udf_get_lb_pblock(struct super_block *sb, struct kernel_lb_addr *loc, + uint32_t offset) +{ + return udf_get_pblock(sb, loc->logicalBlockNum, + loc->partitionReferenceNum, offset); +} + +/* unicode.c */ +extern int udf_get_filename(struct super_block *, const uint8_t *, int, + uint8_t *, int); +extern int udf_put_filename(struct super_block *, const uint8_t *, int, + uint8_t *, int); +extern int udf_dstrCS0toChar(struct super_block *, uint8_t *, int, + const uint8_t *, int); + +/* ialloc.c */ +extern void udf_free_inode(struct inode *); +extern struct inode *udf_new_inode(struct inode *, umode_t); + +/* truncate.c */ +extern void udf_truncate_tail_extent(struct inode *); +extern void udf_discard_prealloc(struct inode *); +extern int udf_truncate_extents(struct inode *); + +/* balloc.c */ +extern void udf_free_blocks(struct super_block *, struct inode *, + struct kernel_lb_addr *, uint32_t, uint32_t); +extern int udf_prealloc_blocks(struct super_block *, struct inode *, uint16_t, + uint32_t, uint32_t); +extern udf_pblk_t udf_new_block(struct super_block *sb, struct inode *inode, + uint16_t partition, uint32_t goal, int *err); + +/* directory.c */ +int udf_fiiter_init(struct udf_fileident_iter *iter, struct inode *dir, + loff_t pos); +int udf_fiiter_advance(struct udf_fileident_iter *iter); +void udf_fiiter_release(struct udf_fileident_iter *iter); +void udf_fiiter_write_fi(struct udf_fileident_iter *iter, uint8_t *impuse); +void udf_fiiter_update_elen(struct udf_fileident_iter *iter, uint32_t new_elen); +int udf_fiiter_append_blk(struct udf_fileident_iter *iter); +extern struct long_ad *udf_get_filelongad(uint8_t *, int, uint32_t *, int); +extern struct short_ad *udf_get_fileshortad(uint8_t *, int, uint32_t *, int); + +/* udftime.c */ +extern void udf_disk_stamp_to_time(struct timespec64 *dest, + struct timestamp src); +extern void udf_time_to_disk_stamp(struct timestamp *dest, struct timespec64 src); + +#endif /* __UDF_DECL_H */ diff --git a/fs/udf/udfend.h b/fs/udf/udfend.h new file mode 100644 index 000000000..a4363ac2c --- /dev/null +++ b/fs/udf/udfend.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __UDF_ENDIAN_H +#define __UDF_ENDIAN_H + +#include +#include + +static inline struct kernel_lb_addr lelb_to_cpu(struct lb_addr in) +{ + struct kernel_lb_addr out; + + out.logicalBlockNum = le32_to_cpu(in.logicalBlockNum); + out.partitionReferenceNum = le16_to_cpu(in.partitionReferenceNum); + + return out; +} + +static inline struct lb_addr cpu_to_lelb(struct kernel_lb_addr in) +{ + struct lb_addr out; + + out.logicalBlockNum = cpu_to_le32(in.logicalBlockNum); + out.partitionReferenceNum = cpu_to_le16(in.partitionReferenceNum); + + return out; +} + +static inline struct short_ad lesa_to_cpu(struct short_ad in) +{ + struct short_ad out; + + out.extLength = le32_to_cpu(in.extLength); + out.extPosition = le32_to_cpu(in.extPosition); + + return out; +} + +static inline struct short_ad cpu_to_lesa(struct short_ad in) +{ + struct short_ad out; + + out.extLength = cpu_to_le32(in.extLength); + out.extPosition = cpu_to_le32(in.extPosition); + + return out; +} + +static inline struct kernel_long_ad lela_to_cpu(struct long_ad in) +{ + struct kernel_long_ad out; + + out.extLength = le32_to_cpu(in.extLength); + out.extLocation = lelb_to_cpu(in.extLocation); + + return out; +} + +static inline struct long_ad cpu_to_lela(struct kernel_long_ad in) +{ + struct long_ad out; + + out.extLength = cpu_to_le32(in.extLength); + out.extLocation = cpu_to_lelb(in.extLocation); + + return out; +} + +static inline struct kernel_extent_ad leea_to_cpu(struct extent_ad in) +{ + struct kernel_extent_ad out; + + out.extLength = le32_to_cpu(in.extLength); + out.extLocation = le32_to_cpu(in.extLocation); + + return out; +} + +#endif /* __UDF_ENDIAN_H */ diff --git a/fs/udf/udftime.c b/fs/udf/udftime.c new file mode 100644 index 000000000..78ecc6336 --- /dev/null +++ b/fs/udf/udftime.c @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: LGPL-2.0+ +/* Copyright (C) 1993, 1994, 1995, 1996, 1997 Free Software Foundation, Inc. + This file is part of the GNU C Library. + Contributed by Paul Eggert (eggert@twinsun.com). */ + +/* + * dgb 10/02/98: ripped this from glibc source to help convert timestamps + * to unix time + * 10/04/98: added new table-based lookup after seeing how ugly + * the gnu code is + * blf 09/27/99: ripped out all the old code and inserted new table from + * John Brockmeyer (without leap second corrections) + * rewrote udf_stamp_to_time and fixed timezone accounting in + * udf_time_to_stamp. + */ + +/* + * We don't take into account leap seconds. This may be correct or incorrect. + * For more NIST information (especially dealing with leap seconds), see: + * http://www.boulder.nist.gov/timefreq/pubs/bulletin/leapsecond.htm + */ + +#include "udfdecl.h" + +#include +#include +#include + +void +udf_disk_stamp_to_time(struct timespec64 *dest, struct timestamp src) +{ + u16 typeAndTimezone = le16_to_cpu(src.typeAndTimezone); + u16 year = le16_to_cpu(src.year); + uint8_t type = typeAndTimezone >> 12; + int16_t offset; + + if (type == 1) { + offset = typeAndTimezone << 4; + /* sign extent offset */ + offset = (offset >> 4); + if (offset == -2047) /* unspecified offset */ + offset = 0; + } else + offset = 0; + + dest->tv_sec = mktime64(year, src.month, src.day, src.hour, src.minute, + src.second); + dest->tv_sec -= offset * 60; + + /* + * Sanitize nanosecond field since reportedly some filesystems are + * recorded with bogus sub-second values. + */ + if (src.centiseconds < 100 && src.hundredsOfMicroseconds < 100 && + src.microseconds < 100) { + dest->tv_nsec = 1000 * (src.centiseconds * 10000 + + src.hundredsOfMicroseconds * 100 + src.microseconds); + } else { + dest->tv_nsec = 0; + } +} + +void +udf_time_to_disk_stamp(struct timestamp *dest, struct timespec64 ts) +{ + time64_t seconds; + int16_t offset; + struct tm tm; + + offset = -sys_tz.tz_minuteswest; + + dest->typeAndTimezone = cpu_to_le16(0x1000 | (offset & 0x0FFF)); + + seconds = ts.tv_sec + offset * 60; + time64_to_tm(seconds, 0, &tm); + dest->year = cpu_to_le16(tm.tm_year + 1900); + dest->month = tm.tm_mon + 1; + dest->day = tm.tm_mday; + dest->hour = tm.tm_hour; + dest->minute = tm.tm_min; + dest->second = tm.tm_sec; + dest->centiseconds = ts.tv_nsec / 10000000; + dest->hundredsOfMicroseconds = (ts.tv_nsec / 1000 - + dest->centiseconds * 10000) / 100; + dest->microseconds = (ts.tv_nsec / 1000 - dest->centiseconds * 10000 - + dest->hundredsOfMicroseconds * 100); +} + +/* EOF */ diff --git a/fs/udf/unicode.c b/fs/udf/unicode.c new file mode 100644 index 000000000..87580ff82 --- /dev/null +++ b/fs/udf/unicode.c @@ -0,0 +1,399 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * unicode.c + * + * PURPOSE + * Routines for converting between UTF-8 and OSTA Compressed Unicode. + * Also handles filename mangling + * + * DESCRIPTION + * OSTA Compressed Unicode is explained in the OSTA UDF specification. + * http://www.osta.org/ + * UTF-8 is explained in the IETF RFC XXXX. + * ftp://ftp.internic.net/rfc/rfcxxxx.txt + * + */ + +#include "udfdecl.h" + +#include +#include +#include /* for memset */ +#include +#include +#include + +#include "udf_sb.h" + +#define PLANE_SIZE 0x10000 +#define UNICODE_MAX 0x10ffff +#define SURROGATE_MASK 0xfffff800 +#define SURROGATE_PAIR 0x0000d800 +#define SURROGATE_LOW 0x00000400 +#define SURROGATE_CHAR_BITS 10 +#define SURROGATE_CHAR_MASK ((1 << SURROGATE_CHAR_BITS) - 1) + +#define ILLEGAL_CHAR_MARK '_' +#define EXT_MARK '.' +#define CRC_MARK '#' +#define EXT_SIZE 5 +/* Number of chars we need to store generated CRC to make filename unique */ +#define CRC_LEN 5 + +static unicode_t get_utf16_char(const uint8_t *str_i, int str_i_max_len, + int str_i_idx, int u_ch, unicode_t *ret) +{ + unicode_t c; + int start_idx = str_i_idx; + + /* Expand OSTA compressed Unicode to Unicode */ + c = str_i[str_i_idx++]; + if (u_ch > 1) + c = (c << 8) | str_i[str_i_idx++]; + if ((c & SURROGATE_MASK) == SURROGATE_PAIR) { + unicode_t next; + + /* Trailing surrogate char */ + if (str_i_idx >= str_i_max_len) { + c = UNICODE_MAX + 1; + goto out; + } + + /* Low surrogate must follow the high one... */ + if (c & SURROGATE_LOW) { + c = UNICODE_MAX + 1; + goto out; + } + + WARN_ON_ONCE(u_ch != 2); + next = str_i[str_i_idx++] << 8; + next |= str_i[str_i_idx++]; + if ((next & SURROGATE_MASK) != SURROGATE_PAIR || + !(next & SURROGATE_LOW)) { + c = UNICODE_MAX + 1; + goto out; + } + + c = PLANE_SIZE + + ((c & SURROGATE_CHAR_MASK) << SURROGATE_CHAR_BITS) + + (next & SURROGATE_CHAR_MASK); + } +out: + *ret = c; + return str_i_idx - start_idx; +} + + +static int udf_name_conv_char(uint8_t *str_o, int str_o_max_len, + int *str_o_idx, + const uint8_t *str_i, int str_i_max_len, + int *str_i_idx, + int u_ch, int *needsCRC, + int (*conv_f)(wchar_t, unsigned char *, int), + int translate) +{ + unicode_t c; + int illChar = 0; + int len, gotch = 0; + + while (!gotch && *str_i_idx < str_i_max_len) { + if (*str_o_idx >= str_o_max_len) { + *needsCRC = 1; + return gotch; + } + + len = get_utf16_char(str_i, str_i_max_len, *str_i_idx, u_ch, + &c); + /* These chars cannot be converted. Replace them. */ + if (c == 0 || c > UNICODE_MAX || (conv_f && c > MAX_WCHAR_T) || + (translate && c == '/')) { + illChar = 1; + if (!translate) + gotch = 1; + } else if (illChar) + break; + else + gotch = 1; + *str_i_idx += len; + } + if (illChar) { + *needsCRC = 1; + c = ILLEGAL_CHAR_MARK; + gotch = 1; + } + if (gotch) { + if (conv_f) { + len = conv_f(c, &str_o[*str_o_idx], + str_o_max_len - *str_o_idx); + } else { + len = utf32_to_utf8(c, &str_o[*str_o_idx], + str_o_max_len - *str_o_idx); + if (len < 0) + len = -ENAMETOOLONG; + } + /* Valid character? */ + if (len >= 0) + *str_o_idx += len; + else if (len == -ENAMETOOLONG) { + *needsCRC = 1; + gotch = 0; + } else { + str_o[(*str_o_idx)++] = ILLEGAL_CHAR_MARK; + *needsCRC = 1; + } + } + return gotch; +} + +static int udf_name_from_CS0(struct super_block *sb, + uint8_t *str_o, int str_max_len, + const uint8_t *ocu, int ocu_len, + int translate) +{ + uint32_t c; + uint8_t cmp_id; + int idx, len; + int u_ch; + int needsCRC = 0; + int ext_i_len, ext_max_len; + int str_o_len = 0; /* Length of resulting output */ + int ext_o_len = 0; /* Extension output length */ + int ext_crc_len = 0; /* Extension output length if used with CRC */ + int i_ext = -1; /* Extension position in input buffer */ + int o_crc = 0; /* Rightmost possible output pos for CRC+ext */ + unsigned short valueCRC; + uint8_t ext[EXT_SIZE * NLS_MAX_CHARSET_SIZE + 1]; + uint8_t crc[CRC_LEN]; + int (*conv_f)(wchar_t, unsigned char *, int); + + if (str_max_len <= 0) + return 0; + + if (ocu_len == 0) { + memset(str_o, 0, str_max_len); + return 0; + } + + if (UDF_SB(sb)->s_nls_map) + conv_f = UDF_SB(sb)->s_nls_map->uni2char; + else + conv_f = NULL; + + cmp_id = ocu[0]; + if (cmp_id != 8 && cmp_id != 16) { + memset(str_o, 0, str_max_len); + pr_err("unknown compression code (%u)\n", cmp_id); + return -EINVAL; + } + u_ch = cmp_id >> 3; + + ocu++; + ocu_len--; + + if (ocu_len % u_ch) { + pr_err("incorrect filename length (%d)\n", ocu_len + 1); + return -EINVAL; + } + + if (translate) { + /* Look for extension */ + for (idx = ocu_len - u_ch, ext_i_len = 0; + (idx >= 0) && (ext_i_len < EXT_SIZE); + idx -= u_ch, ext_i_len++) { + c = ocu[idx]; + if (u_ch > 1) + c = (c << 8) | ocu[idx + 1]; + + if (c == EXT_MARK) { + if (ext_i_len) + i_ext = idx; + break; + } + } + if (i_ext >= 0) { + /* Convert extension */ + ext_max_len = min_t(int, sizeof(ext), str_max_len); + ext[ext_o_len++] = EXT_MARK; + idx = i_ext + u_ch; + while (udf_name_conv_char(ext, ext_max_len, &ext_o_len, + ocu, ocu_len, &idx, + u_ch, &needsCRC, + conv_f, translate)) { + if ((ext_o_len + CRC_LEN) < str_max_len) + ext_crc_len = ext_o_len; + } + } + } + + idx = 0; + while (1) { + if (translate && (idx == i_ext)) { + if (str_o_len > (str_max_len - ext_o_len)) + needsCRC = 1; + break; + } + + if (!udf_name_conv_char(str_o, str_max_len, &str_o_len, + ocu, ocu_len, &idx, + u_ch, &needsCRC, conv_f, translate)) + break; + + if (translate && + (str_o_len <= (str_max_len - ext_o_len - CRC_LEN))) + o_crc = str_o_len; + } + + if (translate) { + if (str_o_len > 0 && str_o_len <= 2 && str_o[0] == '.' && + (str_o_len == 1 || str_o[1] == '.')) + needsCRC = 1; + if (needsCRC) { + str_o_len = o_crc; + valueCRC = crc_itu_t(0, ocu, ocu_len); + crc[0] = CRC_MARK; + crc[1] = hex_asc_upper_hi(valueCRC >> 8); + crc[2] = hex_asc_upper_lo(valueCRC >> 8); + crc[3] = hex_asc_upper_hi(valueCRC); + crc[4] = hex_asc_upper_lo(valueCRC); + len = min_t(int, CRC_LEN, str_max_len - str_o_len); + memcpy(&str_o[str_o_len], crc, len); + str_o_len += len; + ext_o_len = ext_crc_len; + } + if (ext_o_len > 0) { + memcpy(&str_o[str_o_len], ext, ext_o_len); + str_o_len += ext_o_len; + } + } + + return str_o_len; +} + +static int udf_name_to_CS0(struct super_block *sb, + uint8_t *ocu, int ocu_max_len, + const uint8_t *str_i, int str_len) +{ + int i, len; + unsigned int max_val; + int u_len, u_ch; + unicode_t uni_char; + int (*conv_f)(const unsigned char *, int, wchar_t *); + + if (ocu_max_len <= 0) + return 0; + + if (UDF_SB(sb)->s_nls_map) + conv_f = UDF_SB(sb)->s_nls_map->char2uni; + else + conv_f = NULL; + + memset(ocu, 0, ocu_max_len); + ocu[0] = 8; + max_val = 0xff; + u_ch = 1; + +try_again: + u_len = 1; + for (i = 0; i < str_len; i += len) { + /* Name didn't fit? */ + if (u_len + u_ch > ocu_max_len) + return 0; + if (conv_f) { + wchar_t wchar; + + len = conv_f(&str_i[i], str_len - i, &wchar); + if (len > 0) + uni_char = wchar; + } else { + len = utf8_to_utf32(&str_i[i], str_len - i, + &uni_char); + } + /* Invalid character, deal with it */ + if (len <= 0 || uni_char > UNICODE_MAX) { + len = 1; + uni_char = '?'; + } + + if (uni_char > max_val) { + unicode_t c; + + if (max_val == 0xff) { + max_val = 0xffff; + ocu[0] = 0x10; + u_ch = 2; + goto try_again; + } + /* + * Use UTF-16 encoding for chars outside we + * cannot encode directly. + */ + if (u_len + 2 * u_ch > ocu_max_len) + return 0; + + uni_char -= PLANE_SIZE; + c = SURROGATE_PAIR | + ((uni_char >> SURROGATE_CHAR_BITS) & + SURROGATE_CHAR_MASK); + ocu[u_len++] = (uint8_t)(c >> 8); + ocu[u_len++] = (uint8_t)(c & 0xff); + uni_char = SURROGATE_PAIR | SURROGATE_LOW | + (uni_char & SURROGATE_CHAR_MASK); + } + + if (max_val == 0xffff) + ocu[u_len++] = (uint8_t)(uni_char >> 8); + ocu[u_len++] = (uint8_t)(uni_char & 0xff); + } + + return u_len; +} + +/* + * Convert CS0 dstring to output charset. Warning: This function may truncate + * input string if it is too long as it is used for informational strings only + * and it is better to truncate the string than to refuse mounting a media. + */ +int udf_dstrCS0toChar(struct super_block *sb, uint8_t *utf_o, int o_len, + const uint8_t *ocu_i, int i_len) +{ + int s_len = 0; + + if (i_len > 0) { + s_len = ocu_i[i_len - 1]; + if (s_len >= i_len) { + pr_warn("incorrect dstring lengths (%d/%d)," + " truncating\n", s_len, i_len); + s_len = i_len - 1; + /* 2-byte encoding? Need to round properly... */ + if (ocu_i[0] == 16) + s_len -= (s_len - 1) & 2; + } + } + + return udf_name_from_CS0(sb, utf_o, o_len, ocu_i, s_len, 0); +} + +int udf_get_filename(struct super_block *sb, const uint8_t *sname, int slen, + uint8_t *dname, int dlen) +{ + int ret; + + if (!slen) + return -EIO; + + if (dlen <= 0) + return 0; + + ret = udf_name_from_CS0(sb, dname, dlen, sname, slen, 1); + /* Zero length filename isn't valid... */ + if (ret == 0) + ret = -EINVAL; + return ret; +} + +int udf_put_filename(struct super_block *sb, const uint8_t *sname, int slen, + uint8_t *dname, int dlen) +{ + return udf_name_to_CS0(sb, dname, dlen, sname, slen); +} + diff --git a/fs/ufs/Kconfig b/fs/ufs/Kconfig new file mode 100644 index 000000000..9301e7ecd --- /dev/null +++ b/fs/ufs/Kconfig @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: GPL-2.0-only +config UFS_FS + tristate "UFS file system support (read only)" + depends on BLOCK + select BUFFER_HEAD + help + BSD and derivate versions of Unix (such as SunOS, FreeBSD, NetBSD, + OpenBSD and NeXTstep) use a file system called UFS. Some System V + Unixes can create and mount hard disk partitions and diskettes using + this file system as well. Saying Y here will allow you to read from + these partitions; if you also want to write to them, say Y to the + experimental "UFS file system write support", below. Please read the + file for more information. + + The recently released UFS2 variant (used in FreeBSD 5.x) is + READ-ONLY supported. + + Note that this option is generally not needed for floppies, since a + good portable way to transport files and directories between unixes + (and even other operating systems) is given by the tar program ("man + tar" or preferably "info tar"). + + When accessing NeXTstep files, you may need to convert them from the + NeXT character set to the Latin1 character set; use the program + recode ("info recode") for this purpose. + + To compile the UFS file system support as a module, choose M here: the + module will be called ufs. + + If you haven't heard about all of this before, it's safe to say N. + +config UFS_FS_WRITE + bool "UFS file system write support (DANGEROUS)" + depends on UFS_FS + help + Say Y here if you want to try writing to UFS partitions. This is + experimental, so you should back up your UFS partitions beforehand. + +config UFS_DEBUG + bool "UFS debugging" + depends on UFS_FS + help + If you are experiencing any problems with the UFS filesystem, say + Y here. This will result in _many_ additional debugging messages to be + written to the system log. diff --git a/fs/ufs/Makefile b/fs/ufs/Makefile new file mode 100644 index 000000000..042344c85 --- /dev/null +++ b/fs/ufs/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the Linux ufs filesystem routines. +# + +obj-$(CONFIG_UFS_FS) += ufs.o + +ufs-objs := balloc.o cylinder.o dir.o file.o ialloc.o inode.o \ + namei.o super.o util.o +ccflags-$(CONFIG_UFS_DEBUG) += -DDEBUG diff --git a/fs/ufs/balloc.c b/fs/ufs/balloc.c new file mode 100644 index 000000000..628edfde3 --- /dev/null +++ b/fs/ufs/balloc.c @@ -0,0 +1,936 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ufs/balloc.c + * + * Copyright (C) 1998 + * Daniel Pirkl + * Charles University, Faculty of Mathematics and Physics + * + * UFS2 write support Evgeniy Dushistov , 2007 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ufs_fs.h" +#include "ufs.h" +#include "swab.h" +#include "util.h" + +#define INVBLOCK ((u64)-1L) + +static u64 ufs_add_fragments(struct inode *, u64, unsigned, unsigned); +static u64 ufs_alloc_fragments(struct inode *, unsigned, u64, unsigned, int *); +static u64 ufs_alloccg_block(struct inode *, struct ufs_cg_private_info *, u64, int *); +static u64 ufs_bitmap_search (struct super_block *, struct ufs_cg_private_info *, u64, unsigned); +static unsigned char ufs_fragtable_8fpb[], ufs_fragtable_other[]; +static void ufs_clusteracct(struct super_block *, struct ufs_cg_private_info *, unsigned, int); + +static void adjust_free_blocks(struct super_block *sb, + struct ufs_cylinder_group *ucg, + struct ufs_cg_private_info *ucpi, + unsigned fragment, int delta) +{ + struct ufs_sb_private_info *uspi = UFS_SB(sb)->s_uspi; + + if ((UFS_SB(sb)->s_flags & UFS_CG_MASK) == UFS_CG_44BSD) + ufs_clusteracct(sb, ucpi, fragment, delta); + + fs32_add(sb, &ucg->cg_cs.cs_nbfree, delta); + uspi->cs_total.cs_nbfree += delta; + fs32_add(sb, &UFS_SB(sb)->fs_cs(ucpi->c_cgx).cs_nbfree, delta); + + if (uspi->fs_magic != UFS2_MAGIC) { + unsigned cylno = ufs_cbtocylno(fragment); + + fs16_add(sb, &ubh_cg_blks(ucpi, cylno, + ufs_cbtorpos(fragment)), delta); + fs32_add(sb, &ubh_cg_blktot(ucpi, cylno), delta); + } +} + +/* + * Free 'count' fragments from fragment number 'fragment' + */ +void ufs_free_fragments(struct inode *inode, u64 fragment, unsigned count) +{ + struct super_block * sb; + struct ufs_sb_private_info * uspi; + struct ufs_cg_private_info * ucpi; + struct ufs_cylinder_group * ucg; + unsigned cgno, bit, end_bit, bbase, blkmap, i; + + sb = inode->i_sb; + uspi = UFS_SB(sb)->s_uspi; + + UFSD("ENTER, fragment %llu, count %u\n", + (unsigned long long)fragment, count); + + if (ufs_fragnum(fragment) + count > uspi->s_fpb) + ufs_error (sb, "ufs_free_fragments", "internal error"); + + mutex_lock(&UFS_SB(sb)->s_lock); + + cgno = ufs_dtog(uspi, fragment); + bit = ufs_dtogd(uspi, fragment); + if (cgno >= uspi->s_ncg) { + ufs_panic (sb, "ufs_free_fragments", "freeing blocks are outside device"); + goto failed; + } + + ucpi = ufs_load_cylinder (sb, cgno); + if (!ucpi) + goto failed; + ucg = ubh_get_ucg (UCPI_UBH(ucpi)); + if (!ufs_cg_chkmagic(sb, ucg)) { + ufs_panic (sb, "ufs_free_fragments", "internal error, bad magic number on cg %u", cgno); + goto failed; + } + + end_bit = bit + count; + bbase = ufs_blknum (bit); + blkmap = ubh_blkmap (UCPI_UBH(ucpi), ucpi->c_freeoff, bbase); + ufs_fragacct (sb, blkmap, ucg->cg_frsum, -1); + for (i = bit; i < end_bit; i++) { + if (ubh_isclr (UCPI_UBH(ucpi), ucpi->c_freeoff, i)) + ubh_setbit (UCPI_UBH(ucpi), ucpi->c_freeoff, i); + else + ufs_error (sb, "ufs_free_fragments", + "bit already cleared for fragment %u", i); + } + + inode_sub_bytes(inode, count << uspi->s_fshift); + fs32_add(sb, &ucg->cg_cs.cs_nffree, count); + uspi->cs_total.cs_nffree += count; + fs32_add(sb, &UFS_SB(sb)->fs_cs(cgno).cs_nffree, count); + blkmap = ubh_blkmap (UCPI_UBH(ucpi), ucpi->c_freeoff, bbase); + ufs_fragacct(sb, blkmap, ucg->cg_frsum, 1); + + /* + * Trying to reassemble free fragments into block + */ + if (ubh_isblockset(uspi, ucpi, bbase)) { + fs32_sub(sb, &ucg->cg_cs.cs_nffree, uspi->s_fpb); + uspi->cs_total.cs_nffree -= uspi->s_fpb; + fs32_sub(sb, &UFS_SB(sb)->fs_cs(cgno).cs_nffree, uspi->s_fpb); + adjust_free_blocks(sb, ucg, ucpi, bbase, 1); + } + + ubh_mark_buffer_dirty (USPI_UBH(uspi)); + ubh_mark_buffer_dirty (UCPI_UBH(ucpi)); + if (sb->s_flags & SB_SYNCHRONOUS) + ubh_sync_block(UCPI_UBH(ucpi)); + ufs_mark_sb_dirty(sb); + + mutex_unlock(&UFS_SB(sb)->s_lock); + UFSD("EXIT\n"); + return; + +failed: + mutex_unlock(&UFS_SB(sb)->s_lock); + UFSD("EXIT (FAILED)\n"); + return; +} + +/* + * Free 'count' fragments from fragment number 'fragment' (free whole blocks) + */ +void ufs_free_blocks(struct inode *inode, u64 fragment, unsigned count) +{ + struct super_block * sb; + struct ufs_sb_private_info * uspi; + struct ufs_cg_private_info * ucpi; + struct ufs_cylinder_group * ucg; + unsigned overflow, cgno, bit, end_bit, i; + + sb = inode->i_sb; + uspi = UFS_SB(sb)->s_uspi; + + UFSD("ENTER, fragment %llu, count %u\n", + (unsigned long long)fragment, count); + + if ((fragment & uspi->s_fpbmask) || (count & uspi->s_fpbmask)) { + ufs_error (sb, "ufs_free_blocks", "internal error, " + "fragment %llu, count %u\n", + (unsigned long long)fragment, count); + goto failed; + } + + mutex_lock(&UFS_SB(sb)->s_lock); + +do_more: + overflow = 0; + cgno = ufs_dtog(uspi, fragment); + bit = ufs_dtogd(uspi, fragment); + if (cgno >= uspi->s_ncg) { + ufs_panic (sb, "ufs_free_blocks", "freeing blocks are outside device"); + goto failed_unlock; + } + end_bit = bit + count; + if (end_bit > uspi->s_fpg) { + overflow = bit + count - uspi->s_fpg; + count -= overflow; + end_bit -= overflow; + } + + ucpi = ufs_load_cylinder (sb, cgno); + if (!ucpi) + goto failed_unlock; + ucg = ubh_get_ucg (UCPI_UBH(ucpi)); + if (!ufs_cg_chkmagic(sb, ucg)) { + ufs_panic (sb, "ufs_free_blocks", "internal error, bad magic number on cg %u", cgno); + goto failed_unlock; + } + + for (i = bit; i < end_bit; i += uspi->s_fpb) { + if (ubh_isblockset(uspi, ucpi, i)) { + ufs_error(sb, "ufs_free_blocks", "freeing free fragment"); + } + ubh_setblock(uspi, ucpi, i); + inode_sub_bytes(inode, uspi->s_fpb << uspi->s_fshift); + adjust_free_blocks(sb, ucg, ucpi, i, 1); + } + + ubh_mark_buffer_dirty (USPI_UBH(uspi)); + ubh_mark_buffer_dirty (UCPI_UBH(ucpi)); + if (sb->s_flags & SB_SYNCHRONOUS) + ubh_sync_block(UCPI_UBH(ucpi)); + + if (overflow) { + fragment += count; + count = overflow; + goto do_more; + } + + ufs_mark_sb_dirty(sb); + mutex_unlock(&UFS_SB(sb)->s_lock); + UFSD("EXIT\n"); + return; + +failed_unlock: + mutex_unlock(&UFS_SB(sb)->s_lock); +failed: + UFSD("EXIT (FAILED)\n"); + return; +} + +/* + * Modify inode page cache in such way: + * have - blocks with b_blocknr equal to oldb...oldb+count-1 + * get - blocks with b_blocknr equal to newb...newb+count-1 + * also we suppose that oldb...oldb+count-1 blocks + * situated at the end of file. + * + * We can come here from ufs_writepage or ufs_prepare_write, + * locked_folio is argument of these functions, so we already lock it. + */ +static void ufs_change_blocknr(struct inode *inode, sector_t beg, + unsigned int count, sector_t oldb, + sector_t newb, struct folio *locked_folio) +{ + struct folio *folio; + const unsigned blks_per_page = + 1 << (PAGE_SHIFT - inode->i_blkbits); + const unsigned mask = blks_per_page - 1; + struct address_space * const mapping = inode->i_mapping; + pgoff_t index, cur_index, last_index; + unsigned pos, j, lblock; + sector_t end, i; + struct buffer_head *head, *bh; + + UFSD("ENTER, ino %llu, count %u, oldb %llu, newb %llu\n", + inode->i_ino, count, + (unsigned long long)oldb, (unsigned long long)newb); + + BUG_ON(!folio_test_locked(locked_folio)); + + cur_index = locked_folio->index; + end = count + beg; + last_index = end >> (PAGE_SHIFT - inode->i_blkbits); + for (i = beg; i < end; i = (i | mask) + 1) { + index = i >> (PAGE_SHIFT - inode->i_blkbits); + + if (likely(cur_index != index)) { + folio = ufs_get_locked_folio(mapping, index); + if (!folio) /* it was truncated */ + continue; + if (IS_ERR(folio)) {/* or EIO */ + ufs_error(inode->i_sb, __func__, + "read of page %llu failed\n", + (unsigned long long)index); + continue; + } + } else + folio = locked_folio; + + head = folio_buffers(folio); + bh = head; + pos = i & mask; + for (j = 0; j < pos; ++j) + bh = bh->b_this_page; + + if (unlikely(index == last_index)) + lblock = end & mask; + else + lblock = blks_per_page; + + do { + if (j >= lblock) + break; + pos = (i - beg) + j; + + if (!buffer_mapped(bh)) + map_bh(bh, inode->i_sb, oldb + pos); + if (bh_read(bh, 0) < 0) { + ufs_error(inode->i_sb, __func__, + "read of block failed\n"); + break; + } + + UFSD(" change from %llu to %llu, pos %u\n", + (unsigned long long)(pos + oldb), + (unsigned long long)(pos + newb), pos); + + bh->b_blocknr = newb + pos; + clean_bdev_bh_alias(bh); + mark_buffer_dirty(bh); + ++j; + bh = bh->b_this_page; + } while (bh != head); + + if (likely(cur_index != index)) + ufs_put_locked_folio(folio); + } + UFSD("EXIT\n"); +} + +static void ufs_clear_frags(struct inode *inode, sector_t beg, unsigned int n, + int sync) +{ + struct buffer_head *bh; + sector_t end = beg + n; + + for (; beg < end; ++beg) { + bh = sb_getblk(inode->i_sb, beg); + lock_buffer(bh); + memset(bh->b_data, 0, inode->i_sb->s_blocksize); + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + unlock_buffer(bh); + if (IS_SYNC(inode) || sync) + sync_dirty_buffer(bh); + brelse(bh); + } +} + +u64 ufs_new_fragments(struct inode *inode, void *p, u64 fragment, + u64 goal, unsigned count, int *err, + struct folio *locked_folio) +{ + struct super_block * sb; + struct ufs_sb_private_info * uspi; + struct ufs_super_block_first * usb1; + unsigned cgno, oldcount, newcount; + u64 tmp, request, result; + + UFSD("ENTER, ino %llu, fragment %llu, goal %llu, count %u\n", + inode->i_ino, (unsigned long long)fragment, + (unsigned long long)goal, count); + + sb = inode->i_sb; + uspi = UFS_SB(sb)->s_uspi; + usb1 = ubh_get_usb_first(uspi); + *err = -ENOSPC; + + mutex_lock(&UFS_SB(sb)->s_lock); + tmp = ufs_data_ptr_to_cpu(sb, p); + + if (count + ufs_fragnum(fragment) > uspi->s_fpb) { + ufs_warning(sb, "ufs_new_fragments", "internal warning" + " fragment %llu, count %u", + (unsigned long long)fragment, count); + count = uspi->s_fpb - ufs_fragnum(fragment); + } + oldcount = ufs_fragnum (fragment); + newcount = oldcount + count; + + /* + * Somebody else has just allocated our fragments + */ + if (oldcount) { + if (!tmp) { + ufs_error(sb, "ufs_new_fragments", "internal error, " + "fragment %llu, tmp %llu\n", + (unsigned long long)fragment, + (unsigned long long)tmp); + mutex_unlock(&UFS_SB(sb)->s_lock); + return INVBLOCK; + } + if (fragment < UFS_I(inode)->i_lastfrag) { + UFSD("EXIT (ALREADY ALLOCATED)\n"); + mutex_unlock(&UFS_SB(sb)->s_lock); + return 0; + } + } + else { + if (tmp) { + UFSD("EXIT (ALREADY ALLOCATED)\n"); + mutex_unlock(&UFS_SB(sb)->s_lock); + return 0; + } + } + + /* + * There is not enough space for user on the device + */ + if (unlikely(ufs_freefrags(uspi) <= uspi->s_root_blocks)) { + if (!capable(CAP_SYS_RESOURCE)) { + mutex_unlock(&UFS_SB(sb)->s_lock); + UFSD("EXIT (FAILED)\n"); + return 0; + } + } + + if (goal >= uspi->s_size) + goal = 0; + if (goal == 0) + cgno = ufs_inotocg (inode->i_ino); + else + cgno = ufs_dtog(uspi, goal); + + /* + * allocate new fragment + */ + if (oldcount == 0) { + result = ufs_alloc_fragments (inode, cgno, goal, count, err); + if (result) { + ufs_clear_frags(inode, result + oldcount, + newcount - oldcount, locked_folio != NULL); + *err = 0; + write_seqlock(&UFS_I(inode)->meta_lock); + ufs_cpu_to_data_ptr(sb, p, result); + UFS_I(inode)->i_lastfrag = + max(UFS_I(inode)->i_lastfrag, fragment + count); + write_sequnlock(&UFS_I(inode)->meta_lock); + } + mutex_unlock(&UFS_SB(sb)->s_lock); + UFSD("EXIT, result %llu\n", (unsigned long long)result); + return result; + } + + /* + * resize block + */ + result = ufs_add_fragments(inode, tmp, oldcount, newcount); + if (result) { + *err = 0; + read_seqlock_excl(&UFS_I(inode)->meta_lock); + UFS_I(inode)->i_lastfrag = max(UFS_I(inode)->i_lastfrag, + fragment + count); + read_sequnlock_excl(&UFS_I(inode)->meta_lock); + ufs_clear_frags(inode, result + oldcount, newcount - oldcount, + locked_folio != NULL); + mutex_unlock(&UFS_SB(sb)->s_lock); + UFSD("EXIT, result %llu\n", (unsigned long long)result); + return result; + } + + /* + * allocate new block and move data + */ + if (fs32_to_cpu(sb, usb1->fs_optim) == UFS_OPTSPACE) { + request = newcount; + if (uspi->cs_total.cs_nffree < uspi->s_space_to_time) + usb1->fs_optim = cpu_to_fs32(sb, UFS_OPTTIME); + } else { + request = uspi->s_fpb; + if (uspi->cs_total.cs_nffree > uspi->s_time_to_space) + usb1->fs_optim = cpu_to_fs32(sb, UFS_OPTSPACE); + } + result = ufs_alloc_fragments (inode, cgno, goal, request, err); + if (result) { + ufs_clear_frags(inode, result + oldcount, newcount - oldcount, + locked_folio != NULL); + mutex_unlock(&UFS_SB(sb)->s_lock); + ufs_change_blocknr(inode, fragment - oldcount, oldcount, + uspi->s_sbbase + tmp, + uspi->s_sbbase + result, locked_folio); + *err = 0; + write_seqlock(&UFS_I(inode)->meta_lock); + ufs_cpu_to_data_ptr(sb, p, result); + UFS_I(inode)->i_lastfrag = max(UFS_I(inode)->i_lastfrag, + fragment + count); + write_sequnlock(&UFS_I(inode)->meta_lock); + if (newcount < request) + ufs_free_fragments (inode, result + newcount, request - newcount); + ufs_free_fragments (inode, tmp, oldcount); + UFSD("EXIT, result %llu\n", (unsigned long long)result); + return result; + } + + mutex_unlock(&UFS_SB(sb)->s_lock); + UFSD("EXIT (FAILED)\n"); + return 0; +} + +static bool try_add_frags(struct inode *inode, unsigned frags) +{ + unsigned size = frags * i_blocksize(inode); + spin_lock(&inode->i_lock); + __inode_add_bytes(inode, size); + if (unlikely((u32)inode->i_blocks != inode->i_blocks)) { + __inode_sub_bytes(inode, size); + spin_unlock(&inode->i_lock); + return false; + } + spin_unlock(&inode->i_lock); + return true; +} + +static u64 ufs_add_fragments(struct inode *inode, u64 fragment, + unsigned oldcount, unsigned newcount) +{ + struct super_block * sb; + struct ufs_sb_private_info * uspi; + struct ufs_cg_private_info * ucpi; + struct ufs_cylinder_group * ucg; + unsigned cgno, fragno, fragoff, count, fragsize, i; + + UFSD("ENTER, fragment %llu, oldcount %u, newcount %u\n", + (unsigned long long)fragment, oldcount, newcount); + + sb = inode->i_sb; + uspi = UFS_SB(sb)->s_uspi; + count = newcount - oldcount; + + cgno = ufs_dtog(uspi, fragment); + if (fs32_to_cpu(sb, UFS_SB(sb)->fs_cs(cgno).cs_nffree) < count) + return 0; + if ((ufs_fragnum (fragment) + newcount) > uspi->s_fpb) + return 0; + ucpi = ufs_load_cylinder (sb, cgno); + if (!ucpi) + return 0; + ucg = ubh_get_ucg (UCPI_UBH(ucpi)); + if (!ufs_cg_chkmagic(sb, ucg)) { + ufs_panic (sb, "ufs_add_fragments", + "internal error, bad magic number on cg %u", cgno); + return 0; + } + + fragno = ufs_dtogd(uspi, fragment); + fragoff = ufs_fragnum (fragno); + for (i = oldcount; i < newcount; i++) + if (ubh_isclr (UCPI_UBH(ucpi), ucpi->c_freeoff, fragno + i)) + return 0; + + if (!try_add_frags(inode, count)) + return 0; + /* + * Block can be extended + */ + ucg->cg_time = ufs_get_seconds(sb); + for (i = newcount; i < (uspi->s_fpb - fragoff); i++) + if (ubh_isclr (UCPI_UBH(ucpi), ucpi->c_freeoff, fragno + i)) + break; + fragsize = i - oldcount; + if (!fs32_to_cpu(sb, ucg->cg_frsum[fragsize])) + ufs_panic (sb, "ufs_add_fragments", + "internal error or corrupted bitmap on cg %u", cgno); + fs32_sub(sb, &ucg->cg_frsum[fragsize], 1); + if (fragsize != count) + fs32_add(sb, &ucg->cg_frsum[fragsize - count], 1); + for (i = oldcount; i < newcount; i++) + ubh_clrbit (UCPI_UBH(ucpi), ucpi->c_freeoff, fragno + i); + + fs32_sub(sb, &ucg->cg_cs.cs_nffree, count); + fs32_sub(sb, &UFS_SB(sb)->fs_cs(cgno).cs_nffree, count); + uspi->cs_total.cs_nffree -= count; + + ubh_mark_buffer_dirty (USPI_UBH(uspi)); + ubh_mark_buffer_dirty (UCPI_UBH(ucpi)); + if (sb->s_flags & SB_SYNCHRONOUS) + ubh_sync_block(UCPI_UBH(ucpi)); + ufs_mark_sb_dirty(sb); + + UFSD("EXIT, fragment %llu\n", (unsigned long long)fragment); + + return fragment; +} + +#define UFS_TEST_FREE_SPACE_CG \ + ucg = (struct ufs_cylinder_group *) UFS_SB(sb)->s_ucg[cgno]->b_data; \ + if (fs32_to_cpu(sb, ucg->cg_cs.cs_nbfree)) \ + goto cg_found; \ + for (k = count; k < uspi->s_fpb; k++) \ + if (fs32_to_cpu(sb, ucg->cg_frsum[k])) \ + goto cg_found; + +static u64 ufs_alloc_fragments(struct inode *inode, unsigned cgno, + u64 goal, unsigned count, int *err) +{ + struct super_block * sb; + struct ufs_sb_private_info * uspi; + struct ufs_cg_private_info * ucpi; + struct ufs_cylinder_group * ucg; + unsigned oldcg, i, j, k, allocsize; + u64 result; + + UFSD("ENTER, ino %llu, cgno %u, goal %llu, count %u\n", + inode->i_ino, cgno, (unsigned long long)goal, count); + + sb = inode->i_sb; + uspi = UFS_SB(sb)->s_uspi; + oldcg = cgno; + + /* + * 1. searching on preferred cylinder group + */ + UFS_TEST_FREE_SPACE_CG + + /* + * 2. quadratic rehash + */ + for (j = 1; j < uspi->s_ncg; j *= 2) { + cgno += j; + if (cgno >= uspi->s_ncg) + cgno -= uspi->s_ncg; + UFS_TEST_FREE_SPACE_CG + } + + /* + * 3. brute force search + * We start at i = 2 ( 0 is checked at 1.step, 1 at 2.step ) + */ + cgno = (oldcg + 1) % uspi->s_ncg; + for (j = 2; j < uspi->s_ncg; j++) { + cgno++; + if (cgno >= uspi->s_ncg) + cgno = 0; + UFS_TEST_FREE_SPACE_CG + } + + UFSD("EXIT (FAILED)\n"); + return 0; + +cg_found: + ucpi = ufs_load_cylinder (sb, cgno); + if (!ucpi) + return 0; + ucg = ubh_get_ucg (UCPI_UBH(ucpi)); + if (!ufs_cg_chkmagic(sb, ucg)) + ufs_panic (sb, "ufs_alloc_fragments", + "internal error, bad magic number on cg %u", cgno); + ucg->cg_time = ufs_get_seconds(sb); + + if (count == uspi->s_fpb) { + result = ufs_alloccg_block (inode, ucpi, goal, err); + if (result == INVBLOCK) + return 0; + goto succed; + } + + for (allocsize = count; allocsize < uspi->s_fpb; allocsize++) + if (fs32_to_cpu(sb, ucg->cg_frsum[allocsize]) != 0) + break; + + if (allocsize == uspi->s_fpb) { + result = ufs_alloccg_block (inode, ucpi, goal, err); + if (result == INVBLOCK) + return 0; + goal = ufs_dtogd(uspi, result); + for (i = count; i < uspi->s_fpb; i++) + ubh_setbit (UCPI_UBH(ucpi), ucpi->c_freeoff, goal + i); + i = uspi->s_fpb - count; + + inode_sub_bytes(inode, i << uspi->s_fshift); + fs32_add(sb, &ucg->cg_cs.cs_nffree, i); + uspi->cs_total.cs_nffree += i; + fs32_add(sb, &UFS_SB(sb)->fs_cs(cgno).cs_nffree, i); + fs32_add(sb, &ucg->cg_frsum[i], 1); + goto succed; + } + + result = ufs_bitmap_search (sb, ucpi, goal, allocsize); + if (result == INVBLOCK) + return 0; + if (!try_add_frags(inode, count)) + return 0; + for (i = 0; i < count; i++) + ubh_clrbit (UCPI_UBH(ucpi), ucpi->c_freeoff, result + i); + + fs32_sub(sb, &ucg->cg_cs.cs_nffree, count); + uspi->cs_total.cs_nffree -= count; + fs32_sub(sb, &UFS_SB(sb)->fs_cs(cgno).cs_nffree, count); + fs32_sub(sb, &ucg->cg_frsum[allocsize], 1); + + if (count != allocsize) + fs32_add(sb, &ucg->cg_frsum[allocsize - count], 1); + +succed: + ubh_mark_buffer_dirty (USPI_UBH(uspi)); + ubh_mark_buffer_dirty (UCPI_UBH(ucpi)); + if (sb->s_flags & SB_SYNCHRONOUS) + ubh_sync_block(UCPI_UBH(ucpi)); + ufs_mark_sb_dirty(sb); + + result += cgno * uspi->s_fpg; + UFSD("EXIT3, result %llu\n", (unsigned long long)result); + return result; +} + +static u64 ufs_alloccg_block(struct inode *inode, + struct ufs_cg_private_info *ucpi, + u64 goal, int *err) +{ + struct super_block * sb; + struct ufs_sb_private_info * uspi; + struct ufs_cylinder_group * ucg; + u64 result; + + UFSD("ENTER, goal %llu\n", (unsigned long long)goal); + + sb = inode->i_sb; + uspi = UFS_SB(sb)->s_uspi; + ucg = ubh_get_ucg(UCPI_UBH(ucpi)); + + if (goal == 0) { + goal = ucpi->c_rotor; + goto norot; + } + goal = ufs_blknum (goal); + goal = ufs_dtogd(uspi, goal); + + /* + * If the requested block is available, use it. + */ + if (ubh_isblockset(uspi, ucpi, goal)) { + result = goal; + goto gotit; + } + +norot: + result = ufs_bitmap_search (sb, ucpi, goal, uspi->s_fpb); + if (result == INVBLOCK) + return INVBLOCK; + ucpi->c_rotor = result; +gotit: + if (!try_add_frags(inode, uspi->s_fpb)) + return 0; + ubh_clrblock(uspi, ucpi, result); + adjust_free_blocks(sb, ucg, ucpi, result, -1); + + UFSD("EXIT, result %llu\n", (unsigned long long)result); + + return result; +} + +static unsigned ubh_scanc(struct ufs_sb_private_info *uspi, + struct ufs_buffer_head *ubh, + unsigned begin, unsigned size, + unsigned char *table, unsigned char mask) +{ + unsigned rest, offset; + unsigned char *cp; + + + offset = begin & ~uspi->s_fmask; + begin >>= uspi->s_fshift; + for (;;) { + if ((offset + size) < uspi->s_fsize) + rest = size; + else + rest = uspi->s_fsize - offset; + size -= rest; + cp = ubh->bh[begin]->b_data + offset; + while ((table[*cp++] & mask) == 0 && --rest) + ; + if (rest || !size) + break; + begin++; + offset = 0; + } + return (size + rest); +} + +/* + * Find a block of the specified size in the specified cylinder group. + * @sp: pointer to super block + * @ucpi: pointer to cylinder group info + * @goal: near which block we want find new one + * @count: specified size + */ +static u64 ufs_bitmap_search(struct super_block *sb, + struct ufs_cg_private_info *ucpi, + u64 goal, unsigned count) +{ + /* + * Bit patterns for identifying fragments in the block map + * used as ((map & mask_arr) == want_arr) + */ + static const int mask_arr[9] = { + 0x3, 0x7, 0xf, 0x1f, 0x3f, 0x7f, 0xff, 0x1ff, 0x3ff + }; + static const int want_arr[9] = { + 0x0, 0x2, 0x6, 0xe, 0x1e, 0x3e, 0x7e, 0xfe, 0x1fe + }; + struct ufs_sb_private_info *uspi = UFS_SB(sb)->s_uspi; + unsigned start, length, loc; + unsigned pos, want, blockmap, mask, end; + u64 result; + + UFSD("ENTER, cg %u, goal %llu, count %u\n", ucpi->c_cgx, + (unsigned long long)goal, count); + + if (goal) + start = ufs_dtogd(uspi, goal) >> 3; + else + start = ucpi->c_frotor >> 3; + + length = ((uspi->s_fpg + 7) >> 3) - start; + loc = ubh_scanc(uspi, UCPI_UBH(ucpi), ucpi->c_freeoff + start, length, + (uspi->s_fpb == 8) ? ufs_fragtable_8fpb : ufs_fragtable_other, + 1 << (count - 1 + (uspi->s_fpb & 7))); + if (loc == 0) { + length = start + 1; + loc = ubh_scanc(uspi, UCPI_UBH(ucpi), ucpi->c_freeoff, length, + (uspi->s_fpb == 8) ? ufs_fragtable_8fpb : + ufs_fragtable_other, + 1 << (count - 1 + (uspi->s_fpb & 7))); + if (loc == 0) { + ufs_error(sb, "ufs_bitmap_search", + "bitmap corrupted on cg %u, start %u," + " length %u, count %u, freeoff %u\n", + ucpi->c_cgx, start, length, count, + ucpi->c_freeoff); + return INVBLOCK; + } + start = 0; + } + result = (start + length - loc) << 3; + ucpi->c_frotor = result; + + /* + * found the byte in the map + */ + + for (end = result + 8; result < end; result += uspi->s_fpb) { + blockmap = ubh_blkmap(UCPI_UBH(ucpi), ucpi->c_freeoff, result); + blockmap <<= 1; + mask = mask_arr[count]; + want = want_arr[count]; + for (pos = 0; pos <= uspi->s_fpb - count; pos++) { + if ((blockmap & mask) == want) { + UFSD("EXIT, result %llu\n", + (unsigned long long)result); + return result + pos; + } + mask <<= 1; + want <<= 1; + } + } + + ufs_error(sb, "ufs_bitmap_search", "block not in map on cg %u\n", + ucpi->c_cgx); + UFSD("EXIT (FAILED)\n"); + return INVBLOCK; +} + +static void ufs_clusteracct(struct super_block * sb, + struct ufs_cg_private_info * ucpi, unsigned frag, int cnt) +{ + struct ufs_sb_private_info * uspi = UFS_SB(sb)->s_uspi; + int i, start, end, forw, back; + unsigned blkno = ufs_fragstoblks(frag); + + if (uspi->s_contigsumsize <= 0) + return; + + if (cnt > 0) + ubh_setbit(UCPI_UBH(ucpi), ucpi->c_clusteroff, blkno); + else + ubh_clrbit(UCPI_UBH(ucpi), ucpi->c_clusteroff, blkno); + + /* + * Find the size of the cluster going forward. + */ + start = blkno + 1; + end = start + uspi->s_contigsumsize; + if ( end >= ucpi->c_nclusterblks) + end = ucpi->c_nclusterblks; + i = ubh_find_next_zero_bit (UCPI_UBH(ucpi), ucpi->c_clusteroff, end, start); + if (i > end) + i = end; + forw = i - start; + + /* + * Find the size of the cluster going backward. + */ + start = blkno - 1; + end = start - uspi->s_contigsumsize; + if (end < 0 ) + end = -1; + i = ubh_find_last_zero_bit (UCPI_UBH(ucpi), ucpi->c_clusteroff, start, end); + if ( i < end) + i = end; + back = start - i; + + /* + * Account for old cluster and the possibly new forward and + * back clusters. + */ + i = back + forw + 1; + if (i > uspi->s_contigsumsize) + i = uspi->s_contigsumsize; + fs32_add(sb, (__fs32*)ubh_get_addr(UCPI_UBH(ucpi), ucpi->c_clustersumoff + (i << 2)), cnt); + if (back > 0) + fs32_sub(sb, (__fs32*)ubh_get_addr(UCPI_UBH(ucpi), ucpi->c_clustersumoff + (back << 2)), cnt); + if (forw > 0) + fs32_sub(sb, (__fs32*)ubh_get_addr(UCPI_UBH(ucpi), ucpi->c_clustersumoff + (forw << 2)), cnt); +} + + +static unsigned char ufs_fragtable_8fpb[] = { + 0x00, 0x01, 0x01, 0x02, 0x01, 0x01, 0x02, 0x04, 0x01, 0x01, 0x01, 0x03, 0x02, 0x03, 0x04, 0x08, + 0x01, 0x01, 0x01, 0x03, 0x01, 0x01, 0x03, 0x05, 0x02, 0x03, 0x03, 0x02, 0x04, 0x05, 0x08, 0x10, + 0x01, 0x01, 0x01, 0x03, 0x01, 0x01, 0x03, 0x05, 0x01, 0x01, 0x01, 0x03, 0x03, 0x03, 0x05, 0x09, + 0x02, 0x03, 0x03, 0x02, 0x03, 0x03, 0x02, 0x06, 0x04, 0x05, 0x05, 0x06, 0x08, 0x09, 0x10, 0x20, + 0x01, 0x01, 0x01, 0x03, 0x01, 0x01, 0x03, 0x05, 0x01, 0x01, 0x01, 0x03, 0x03, 0x03, 0x05, 0x09, + 0x01, 0x01, 0x01, 0x03, 0x01, 0x01, 0x03, 0x05, 0x03, 0x03, 0x03, 0x03, 0x05, 0x05, 0x09, 0x11, + 0x02, 0x03, 0x03, 0x02, 0x03, 0x03, 0x02, 0x06, 0x03, 0x03, 0x03, 0x03, 0x02, 0x03, 0x06, 0x0A, + 0x04, 0x05, 0x05, 0x06, 0x05, 0x05, 0x06, 0x04, 0x08, 0x09, 0x09, 0x0A, 0x10, 0x11, 0x20, 0x40, + 0x01, 0x01, 0x01, 0x03, 0x01, 0x01, 0x03, 0x05, 0x01, 0x01, 0x01, 0x03, 0x03, 0x03, 0x05, 0x09, + 0x01, 0x01, 0x01, 0x03, 0x01, 0x01, 0x03, 0x05, 0x03, 0x03, 0x03, 0x03, 0x05, 0x05, 0x09, 0x11, + 0x01, 0x01, 0x01, 0x03, 0x01, 0x01, 0x03, 0x05, 0x01, 0x01, 0x01, 0x03, 0x03, 0x03, 0x05, 0x09, + 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x07, 0x05, 0x05, 0x05, 0x07, 0x09, 0x09, 0x11, 0x21, + 0x02, 0x03, 0x03, 0x02, 0x03, 0x03, 0x02, 0x06, 0x03, 0x03, 0x03, 0x03, 0x02, 0x03, 0x06, 0x0A, + 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x07, 0x02, 0x03, 0x03, 0x02, 0x06, 0x07, 0x0A, 0x12, + 0x04, 0x05, 0x05, 0x06, 0x05, 0x05, 0x06, 0x04, 0x05, 0x05, 0x05, 0x07, 0x06, 0x07, 0x04, 0x0C, + 0x08, 0x09, 0x09, 0x0A, 0x09, 0x09, 0x0A, 0x0C, 0x10, 0x11, 0x11, 0x12, 0x20, 0x21, 0x40, 0x80, +}; + +static unsigned char ufs_fragtable_other[] = { + 0x00, 0x16, 0x16, 0x2A, 0x16, 0x16, 0x26, 0x4E, 0x16, 0x16, 0x16, 0x3E, 0x2A, 0x3E, 0x4E, 0x8A, + 0x16, 0x16, 0x16, 0x3E, 0x16, 0x16, 0x36, 0x5E, 0x16, 0x16, 0x16, 0x3E, 0x3E, 0x3E, 0x5E, 0x9E, + 0x16, 0x16, 0x16, 0x3E, 0x16, 0x16, 0x36, 0x5E, 0x16, 0x16, 0x16, 0x3E, 0x3E, 0x3E, 0x5E, 0x9E, + 0x2A, 0x3E, 0x3E, 0x2A, 0x3E, 0x3E, 0x2E, 0x6E, 0x3E, 0x3E, 0x3E, 0x3E, 0x2A, 0x3E, 0x6E, 0xAA, + 0x16, 0x16, 0x16, 0x3E, 0x16, 0x16, 0x36, 0x5E, 0x16, 0x16, 0x16, 0x3E, 0x3E, 0x3E, 0x5E, 0x9E, + 0x16, 0x16, 0x16, 0x3E, 0x16, 0x16, 0x36, 0x5E, 0x16, 0x16, 0x16, 0x3E, 0x3E, 0x3E, 0x5E, 0x9E, + 0x26, 0x36, 0x36, 0x2E, 0x36, 0x36, 0x26, 0x6E, 0x36, 0x36, 0x36, 0x3E, 0x2E, 0x3E, 0x6E, 0xAE, + 0x4E, 0x5E, 0x5E, 0x6E, 0x5E, 0x5E, 0x6E, 0x4E, 0x5E, 0x5E, 0x5E, 0x7E, 0x6E, 0x7E, 0x4E, 0xCE, + 0x16, 0x16, 0x16, 0x3E, 0x16, 0x16, 0x36, 0x5E, 0x16, 0x16, 0x16, 0x3E, 0x3E, 0x3E, 0x5E, 0x9E, + 0x16, 0x16, 0x16, 0x3E, 0x16, 0x16, 0x36, 0x5E, 0x16, 0x16, 0x16, 0x3E, 0x3E, 0x3E, 0x5E, 0x9E, + 0x16, 0x16, 0x16, 0x3E, 0x16, 0x16, 0x36, 0x5E, 0x16, 0x16, 0x16, 0x3E, 0x3E, 0x3E, 0x5E, 0x9E, + 0x3E, 0x3E, 0x3E, 0x3E, 0x3E, 0x3E, 0x3E, 0x7E, 0x3E, 0x3E, 0x3E, 0x3E, 0x3E, 0x3E, 0x7E, 0xBE, + 0x2A, 0x3E, 0x3E, 0x2A, 0x3E, 0x3E, 0x2E, 0x6E, 0x3E, 0x3E, 0x3E, 0x3E, 0x2A, 0x3E, 0x6E, 0xAA, + 0x3E, 0x3E, 0x3E, 0x3E, 0x3E, 0x3E, 0x3E, 0x7E, 0x3E, 0x3E, 0x3E, 0x3E, 0x3E, 0x3E, 0x7E, 0xBE, + 0x4E, 0x5E, 0x5E, 0x6E, 0x5E, 0x5E, 0x6E, 0x4E, 0x5E, 0x5E, 0x5E, 0x7E, 0x6E, 0x7E, 0x4E, 0xCE, + 0x8A, 0x9E, 0x9E, 0xAA, 0x9E, 0x9E, 0xAE, 0xCE, 0x9E, 0x9E, 0x9E, 0xBE, 0xAA, 0xBE, 0xCE, 0x8A, +}; diff --git a/fs/ufs/cylinder.c b/fs/ufs/cylinder.c new file mode 100644 index 000000000..a2813270c --- /dev/null +++ b/fs/ufs/cylinder.c @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ufs/cylinder.c + * + * Copyright (C) 1998 + * Daniel Pirkl + * Charles University, Faculty of Mathematics and Physics + * + * ext2 - inode (block) bitmap caching inspired + */ + +#include +#include +#include +#include +#include + +#include + +#include "ufs_fs.h" +#include "ufs.h" +#include "swab.h" +#include "util.h" + +/* + * Read cylinder group into cache. The memory space for ufs_cg_private_info + * structure is already allocated during ufs_read_super. + */ +static bool ufs_read_cylinder(struct super_block *sb, + unsigned cgno, unsigned bitmap_nr) +{ + struct ufs_sb_info * sbi = UFS_SB(sb); + struct ufs_sb_private_info * uspi; + struct ufs_cg_private_info * ucpi; + struct ufs_cylinder_group * ucg; + unsigned i, j; + + UFSD("ENTER, cgno %u, bitmap_nr %u\n", cgno, bitmap_nr); + uspi = sbi->s_uspi; + ucpi = sbi->s_ucpi[bitmap_nr]; + ucg = (struct ufs_cylinder_group *)sbi->s_ucg[cgno]->b_data; + + UCPI_UBH(ucpi)->fragment = ufs_cgcmin(cgno); + UCPI_UBH(ucpi)->count = uspi->s_cgsize >> sb->s_blocksize_bits; + /* + * We have already the first fragment of cylinder group block in buffer + */ + UCPI_UBH(ucpi)->bh[0] = sbi->s_ucg[cgno]; + for (i = 1; i < UCPI_UBH(ucpi)->count; i++) { + UCPI_UBH(ucpi)->bh[i] = sb_bread(sb, UCPI_UBH(ucpi)->fragment + i); + if (!UCPI_UBH(ucpi)->bh[i]) + goto failed; + } + sbi->s_cgno[bitmap_nr] = cgno; + + ucpi->c_cgx = fs32_to_cpu(sb, ucg->cg_cgx); + ucpi->c_ncyl = fs16_to_cpu(sb, ucg->cg_ncyl); + ucpi->c_niblk = fs16_to_cpu(sb, ucg->cg_niblk); + ucpi->c_ndblk = fs32_to_cpu(sb, ucg->cg_ndblk); + ucpi->c_rotor = fs32_to_cpu(sb, ucg->cg_rotor); + ucpi->c_frotor = fs32_to_cpu(sb, ucg->cg_frotor); + ucpi->c_irotor = fs32_to_cpu(sb, ucg->cg_irotor); + ucpi->c_btotoff = fs32_to_cpu(sb, ucg->cg_btotoff); + ucpi->c_boff = fs32_to_cpu(sb, ucg->cg_boff); + ucpi->c_iusedoff = fs32_to_cpu(sb, ucg->cg_iusedoff); + ucpi->c_freeoff = fs32_to_cpu(sb, ucg->cg_freeoff); + ucpi->c_nextfreeoff = fs32_to_cpu(sb, ucg->cg_nextfreeoff); + ucpi->c_clustersumoff = fs32_to_cpu(sb, ucg->cg_u.cg_44.cg_clustersumoff); + ucpi->c_clusteroff = fs32_to_cpu(sb, ucg->cg_u.cg_44.cg_clusteroff); + ucpi->c_nclusterblks = fs32_to_cpu(sb, ucg->cg_u.cg_44.cg_nclusterblks); + UFSD("EXIT\n"); + return true; + +failed: + for (j = 1; j < i; j++) + brelse(UCPI_UBH(ucpi)->bh[j]); + sbi->s_cgno[bitmap_nr] = UFS_CGNO_EMPTY; + ufs_error (sb, "ufs_read_cylinder", "can't read cylinder group block %u", cgno); + return false; +} + +/* + * Remove cylinder group from cache, doesn't release memory + * allocated for cylinder group (this is done at ufs_put_super only). + */ +void ufs_put_cylinder (struct super_block * sb, unsigned bitmap_nr) +{ + struct ufs_sb_info * sbi = UFS_SB(sb); + struct ufs_sb_private_info * uspi; + struct ufs_cg_private_info * ucpi; + struct ufs_cylinder_group * ucg; + unsigned i; + + UFSD("ENTER, bitmap_nr %u\n", bitmap_nr); + + uspi = sbi->s_uspi; + if (sbi->s_cgno[bitmap_nr] == UFS_CGNO_EMPTY) { + UFSD("EXIT\n"); + return; + } + ucpi = sbi->s_ucpi[bitmap_nr]; + ucg = ubh_get_ucg(UCPI_UBH(ucpi)); + + if (uspi->s_ncg > UFS_MAX_GROUP_LOADED && bitmap_nr >= sbi->s_cg_loaded) { + ufs_panic (sb, "ufs_put_cylinder", "internal error"); + return; + } + /* + * rotor is not so important data, so we put it to disk + * at the end of working with cylinder + */ + ucg->cg_rotor = cpu_to_fs32(sb, ucpi->c_rotor); + ucg->cg_frotor = cpu_to_fs32(sb, ucpi->c_frotor); + ucg->cg_irotor = cpu_to_fs32(sb, ucpi->c_irotor); + ubh_mark_buffer_dirty (UCPI_UBH(ucpi)); + for (i = 1; i < UCPI_UBH(ucpi)->count; i++) { + brelse (UCPI_UBH(ucpi)->bh[i]); + } + + sbi->s_cgno[bitmap_nr] = UFS_CGNO_EMPTY; + UFSD("EXIT\n"); +} + +/* + * Find cylinder group in cache and return it as pointer. + * If cylinder group is not in cache, we will load it from disk. + * + * The cache is managed by LRU algorithm. + */ +struct ufs_cg_private_info * ufs_load_cylinder ( + struct super_block * sb, unsigned cgno) +{ + struct ufs_sb_info * sbi = UFS_SB(sb); + struct ufs_sb_private_info * uspi; + struct ufs_cg_private_info * ucpi; + unsigned cg, i, j; + + UFSD("ENTER, cgno %u\n", cgno); + + uspi = sbi->s_uspi; + if (cgno >= uspi->s_ncg) { + ufs_panic (sb, "ufs_load_cylinder", "internal error, high number of cg"); + return NULL; + } + /* + * Cylinder group number cg it in cache and it was last used + */ + if (sbi->s_cgno[0] == cgno) { + UFSD("EXIT\n"); + return sbi->s_ucpi[0]; + } + /* + * Number of cylinder groups is not higher than UFS_MAX_GROUP_LOADED + */ + if (uspi->s_ncg <= UFS_MAX_GROUP_LOADED) { + if (sbi->s_cgno[cgno] != UFS_CGNO_EMPTY) { + if (sbi->s_cgno[cgno] != cgno) { + ufs_panic (sb, "ufs_load_cylinder", "internal error, wrong number of cg in cache"); + UFSD("EXIT (FAILED)\n"); + return NULL; + } + } else { + if (unlikely(!ufs_read_cylinder (sb, cgno, cgno))) { + UFSD("EXIT (FAILED)\n"); + return NULL; + } + } + UFSD("EXIT\n"); + return sbi->s_ucpi[cgno]; + } + /* + * Cylinder group number cg is in cache but it was not last used, + * we will move to the first position + */ + for (i = 0; i < sbi->s_cg_loaded && sbi->s_cgno[i] != cgno; i++); + if (i < sbi->s_cg_loaded && sbi->s_cgno[i] == cgno) { + cg = sbi->s_cgno[i]; + ucpi = sbi->s_ucpi[i]; + for (j = i; j > 0; j--) { + sbi->s_cgno[j] = sbi->s_cgno[j-1]; + sbi->s_ucpi[j] = sbi->s_ucpi[j-1]; + } + sbi->s_cgno[0] = cg; + sbi->s_ucpi[0] = ucpi; + /* + * Cylinder group number cg is not in cache, we will read it from disk + * and put it to the first position + */ + } else { + if (sbi->s_cg_loaded < UFS_MAX_GROUP_LOADED) + sbi->s_cg_loaded++; + else + ufs_put_cylinder (sb, UFS_MAX_GROUP_LOADED-1); + ucpi = sbi->s_ucpi[sbi->s_cg_loaded - 1]; + for (j = sbi->s_cg_loaded - 1; j > 0; j--) { + sbi->s_cgno[j] = sbi->s_cgno[j-1]; + sbi->s_ucpi[j] = sbi->s_ucpi[j-1]; + } + sbi->s_ucpi[0] = ucpi; + if (unlikely(!ufs_read_cylinder (sb, cgno, 0))) { + UFSD("EXIT (FAILED)\n"); + return NULL; + } + } + UFSD("EXIT\n"); + return sbi->s_ucpi[0]; +} diff --git a/fs/ufs/dir.c b/fs/ufs/dir.c new file mode 100644 index 000000000..e62fe5667 --- /dev/null +++ b/fs/ufs/dir.c @@ -0,0 +1,658 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ufs/ufs_dir.c + * + * Copyright (C) 1996 + * Adrian Rodriguez (adrian@franklins-tower.rutgers.edu) + * Laboratory for Computer Science Research Computing Facility + * Rutgers, The State University of New Jersey + * + * swab support by Francois-Rene Rideau 19970406 + * + * 4.4BSD (FreeBSD) support added on February 1st 1998 by + * Niels Kristian Bech Jensen partially based + * on code by Martin von Loewis . + * + * Migration to usage of "page cache" on May 2006 by + * Evgeniy Dushistov based on ext2 code base. + */ + +#include +#include +#include +#include +#include + +#include "ufs_fs.h" +#include "ufs.h" +#include "swab.h" +#include "util.h" + +/* + * NOTE! unlike strncmp, ufs_match returns 1 for success, 0 for failure. + * + * len <= UFS_MAXNAMLEN and de != NULL are guaranteed by caller. + */ +static inline int ufs_match(struct super_block *sb, int len, + const unsigned char *name, struct ufs_dir_entry *de) +{ + if (len != ufs_get_de_namlen(sb, de)) + return 0; + if (!de->d_ino) + return 0; + return !memcmp(name, de->d_name, len); +} + +static void ufs_commit_chunk(struct folio *folio, loff_t pos, unsigned len) +{ + struct address_space *mapping = folio->mapping; + struct inode *dir = mapping->host; + + inode_inc_iversion(dir); + block_write_end(pos, len, len, folio); + if (pos+len > dir->i_size) { + i_size_write(dir, pos+len); + mark_inode_dirty(dir); + } + folio_unlock(folio); +} + +static int ufs_handle_dirsync(struct inode *dir) +{ + int err; + + err = filemap_write_and_wait(dir->i_mapping); + if (!err) + err = sync_inode_metadata(dir, 1); + return err; +} + +ino_t ufs_inode_by_name(struct inode *dir, const struct qstr *qstr) +{ + ino_t res = 0; + struct ufs_dir_entry *de; + struct folio *folio; + + de = ufs_find_entry(dir, qstr, &folio); + if (de) { + res = fs32_to_cpu(dir->i_sb, de->d_ino); + folio_release_kmap(folio, de); + } + return res; +} + + +int ufs_set_link(struct inode *dir, struct ufs_dir_entry *de, + struct folio *folio, struct inode *inode, + bool update_times) +{ + loff_t pos = folio_pos(folio) + offset_in_folio(folio, de); + unsigned len = fs16_to_cpu(dir->i_sb, de->d_reclen); + int err; + + folio_lock(folio); + err = ufs_prepare_chunk(folio, pos, len); + if (unlikely(err)) { + folio_unlock(folio); + return err; + } + + de->d_ino = cpu_to_fs32(dir->i_sb, inode->i_ino); + ufs_set_de_type(dir->i_sb, de, inode->i_mode); + + ufs_commit_chunk(folio, pos, len); + if (update_times) + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + mark_inode_dirty(dir); + return ufs_handle_dirsync(dir); +} + +static bool ufs_check_folio(struct folio *folio, char *kaddr) +{ + struct inode *dir = folio->mapping->host; + struct super_block *sb = dir->i_sb; + unsigned offs, rec_len; + unsigned limit = folio_size(folio); + const unsigned chunk_mask = UFS_SB(sb)->s_uspi->s_dirblksize - 1; + struct ufs_dir_entry *p; + char *error; + + if (dir->i_size < folio_pos(folio) + limit) { + limit = offset_in_folio(folio, dir->i_size); + if (limit & chunk_mask) + goto Ebadsize; + if (!limit) + goto out; + } + for (offs = 0; offs <= limit - UFS_DIR_REC_LEN(1); offs += rec_len) { + p = (struct ufs_dir_entry *)(kaddr + offs); + rec_len = fs16_to_cpu(sb, p->d_reclen); + + if (rec_len < UFS_DIR_REC_LEN(1)) + goto Eshort; + if (rec_len & 3) + goto Ealign; + if (rec_len < UFS_DIR_REC_LEN(ufs_get_de_namlen(sb, p))) + goto Enamelen; + if (((offs + rec_len - 1) ^ offs) & ~chunk_mask) + goto Espan; + if (fs32_to_cpu(sb, p->d_ino) > (UFS_SB(sb)->s_uspi->s_ipg * + UFS_SB(sb)->s_uspi->s_ncg)) + goto Einumber; + } + if (offs != limit) + goto Eend; +out: + folio_set_checked(folio); + return true; + + /* Too bad, we had an error */ + +Ebadsize: + ufs_error(sb, __func__, + "size of directory #%llu is not a multiple of chunk size", + dir->i_ino + ); + goto fail; +Eshort: + error = "rec_len is smaller than minimal"; + goto bad_entry; +Ealign: + error = "unaligned directory entry"; + goto bad_entry; +Enamelen: + error = "rec_len is too small for name_len"; + goto bad_entry; +Espan: + error = "directory entry across blocks"; + goto bad_entry; +Einumber: + error = "inode out of bounds"; +bad_entry: + ufs_error(sb, __func__, "bad entry in directory #%llu: %s - " + "offset=%llu, rec_len=%d, name_len=%d", + dir->i_ino, error, folio_pos(folio) + offs, + rec_len, ufs_get_de_namlen(sb, p)); + goto fail; +Eend: + p = (struct ufs_dir_entry *)(kaddr + offs); + ufs_error(sb, __func__, + "entry in directory #%llu spans the page boundary" + "offset=%llu", + dir->i_ino, folio_pos(folio) + offs); +fail: + return false; +} + +static void *ufs_get_folio(struct inode *dir, unsigned long n, + struct folio **foliop) +{ + struct address_space *mapping = dir->i_mapping; + struct folio *folio = read_mapping_folio(mapping, n, NULL); + void *kaddr; + + if (IS_ERR(folio)) + return ERR_CAST(folio); + kaddr = kmap_local_folio(folio, 0); + if (unlikely(!folio_test_checked(folio))) { + if (!ufs_check_folio(folio, kaddr)) + goto fail; + } + *foliop = folio; + return kaddr; + +fail: + folio_release_kmap(folio, kaddr); + return ERR_PTR(-EIO); +} + +/* + * Return the offset into page `page_nr' of the last valid + * byte in that page, plus one. + */ +static unsigned +ufs_last_byte(struct inode *inode, unsigned long page_nr) +{ + unsigned last_byte = inode->i_size; + + last_byte -= page_nr << PAGE_SHIFT; + if (last_byte > PAGE_SIZE) + last_byte = PAGE_SIZE; + return last_byte; +} + +static inline struct ufs_dir_entry * +ufs_next_entry(struct super_block *sb, struct ufs_dir_entry *p) +{ + return (struct ufs_dir_entry *)((char *)p + + fs16_to_cpu(sb, p->d_reclen)); +} + +struct ufs_dir_entry *ufs_dotdot(struct inode *dir, struct folio **foliop) +{ + struct ufs_dir_entry *de = ufs_get_folio(dir, 0, foliop); + + if (!IS_ERR(de)) + return ufs_next_entry(dir->i_sb, de); + + return NULL; +} + +/* + * ufs_find_entry() + * + * finds an entry in the specified directory with the wanted name. It + * returns the page in which the entry was found, and the entry itself + * (as a parameter - res_dir). Page is returned mapped and unlocked. + * Entry is guaranteed to be valid. + */ +struct ufs_dir_entry *ufs_find_entry(struct inode *dir, const struct qstr *qstr, + struct folio **foliop) +{ + struct super_block *sb = dir->i_sb; + const unsigned char *name = qstr->name; + int namelen = qstr->len; + unsigned reclen = UFS_DIR_REC_LEN(namelen); + unsigned long start, n; + unsigned long npages = dir_pages(dir); + struct ufs_inode_info *ui = UFS_I(dir); + struct ufs_dir_entry *de; + + UFSD("ENTER, dir_ino %llu, name %s, namlen %u\n", dir->i_ino, name, namelen); + + if (npages == 0 || namelen > UFS_MAXNAMLEN) + goto out; + + start = ui->i_dir_start_lookup; + + if (start >= npages) + start = 0; + n = start; + do { + char *kaddr = ufs_get_folio(dir, n, foliop); + + if (!IS_ERR(kaddr)) { + de = (struct ufs_dir_entry *)kaddr; + kaddr += ufs_last_byte(dir, n) - reclen; + while ((char *) de <= kaddr) { + if (ufs_match(sb, namelen, name, de)) + goto found; + de = ufs_next_entry(sb, de); + } + folio_release_kmap(*foliop, kaddr); + } + if (++n >= npages) + n = 0; + } while (n != start); +out: + return NULL; + +found: + ui->i_dir_start_lookup = n; + return de; +} + +/* + * Parent is locked. + */ +int ufs_add_link(struct dentry *dentry, struct inode *inode) +{ + struct inode *dir = d_inode(dentry->d_parent); + const unsigned char *name = dentry->d_name.name; + int namelen = dentry->d_name.len; + struct super_block *sb = dir->i_sb; + unsigned reclen = UFS_DIR_REC_LEN(namelen); + const unsigned int chunk_size = UFS_SB(sb)->s_uspi->s_dirblksize; + unsigned short rec_len, name_len; + struct folio *folio = NULL; + struct ufs_dir_entry *de; + unsigned long npages = dir_pages(dir); + unsigned long n; + loff_t pos; + int err; + + UFSD("ENTER, name %s, namelen %u\n", name, namelen); + + /* + * We take care of directory expansion in the same loop. + * This code plays outside i_size, so it locks the folio + * to protect that region. + */ + for (n = 0; n <= npages; n++) { + char *kaddr = ufs_get_folio(dir, n, &folio); + char *dir_end; + + if (IS_ERR(kaddr)) + return PTR_ERR(kaddr); + folio_lock(folio); + dir_end = kaddr + ufs_last_byte(dir, n); + de = (struct ufs_dir_entry *)kaddr; + kaddr += folio_size(folio) - reclen; + while ((char *)de <= kaddr) { + if ((char *)de == dir_end) { + /* We hit i_size */ + name_len = 0; + rec_len = chunk_size; + de->d_reclen = cpu_to_fs16(sb, chunk_size); + de->d_ino = 0; + goto got_it; + } + if (de->d_reclen == 0) { + ufs_error(dir->i_sb, __func__, + "zero-length directory entry"); + err = -EIO; + goto out_unlock; + } + err = -EEXIST; + if (ufs_match(sb, namelen, name, de)) + goto out_unlock; + name_len = UFS_DIR_REC_LEN(ufs_get_de_namlen(sb, de)); + rec_len = fs16_to_cpu(sb, de->d_reclen); + if (!de->d_ino && rec_len >= reclen) + goto got_it; + if (rec_len >= name_len + reclen) + goto got_it; + de = (struct ufs_dir_entry *) ((char *) de + rec_len); + } + folio_unlock(folio); + folio_release_kmap(folio, kaddr); + } + BUG(); + return -EINVAL; + +got_it: + pos = folio_pos(folio) + offset_in_folio(folio, de); + err = ufs_prepare_chunk(folio, pos, rec_len); + if (err) + goto out_unlock; + if (de->d_ino) { + struct ufs_dir_entry *de1 = + (struct ufs_dir_entry *) ((char *) de + name_len); + de1->d_reclen = cpu_to_fs16(sb, rec_len - name_len); + de->d_reclen = cpu_to_fs16(sb, name_len); + + de = de1; + } + + ufs_set_de_namlen(sb, de, namelen); + memcpy(de->d_name, name, namelen + 1); + de->d_ino = cpu_to_fs32(sb, inode->i_ino); + ufs_set_de_type(sb, de, inode->i_mode); + + ufs_commit_chunk(folio, pos, rec_len); + inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); + + mark_inode_dirty(dir); + err = ufs_handle_dirsync(dir); + /* OFFSET_CACHE */ +out_put: + folio_release_kmap(folio, de); + return err; +out_unlock: + folio_unlock(folio); + goto out_put; +} + +static inline unsigned +ufs_validate_entry(struct super_block *sb, char *base, + unsigned offset, unsigned mask) +{ + struct ufs_dir_entry *de = (struct ufs_dir_entry*)(base + offset); + struct ufs_dir_entry *p = (struct ufs_dir_entry*)(base + (offset&mask)); + while ((char*)p < (char*)de) + p = ufs_next_entry(sb, p); + return (char *)p - base; +} + + +/* + * This is blatantly stolen from ext2fs + */ +static int +ufs_readdir(struct file *file, struct dir_context *ctx) +{ + loff_t pos = ctx->pos; + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + unsigned int offset = pos & ~PAGE_MASK; + unsigned long n = pos >> PAGE_SHIFT; + unsigned long npages = dir_pages(inode); + unsigned chunk_mask = ~(UFS_SB(sb)->s_uspi->s_dirblksize - 1); + bool need_revalidate = !inode_eq_iversion(inode, *(u64 *)file->private_data); + unsigned flags = UFS_SB(sb)->s_flags; + + UFSD("BEGIN\n"); + + if (pos > inode->i_size - UFS_DIR_REC_LEN(1)) + return 0; + + for ( ; n < npages; n++, offset = 0) { + struct ufs_dir_entry *de; + struct folio *folio; + char *kaddr = ufs_get_folio(inode, n, &folio); + char *limit; + + if (IS_ERR(kaddr)) { + ufs_error(sb, __func__, + "bad page in #%llu", + inode->i_ino); + ctx->pos += PAGE_SIZE - offset; + return PTR_ERR(kaddr); + } + if (unlikely(need_revalidate)) { + if (offset) { + offset = ufs_validate_entry(sb, kaddr, offset, chunk_mask); + ctx->pos = (n<private_data = inode_query_iversion(inode); + need_revalidate = false; + } + de = (struct ufs_dir_entry *)(kaddr+offset); + limit = kaddr + ufs_last_byte(inode, n) - UFS_DIR_REC_LEN(1); + for ( ;(char*)de <= limit; de = ufs_next_entry(sb, de)) { + if (de->d_ino) { + unsigned char d_type = DT_UNKNOWN; + + UFSD("filldir(%s,%u)\n", de->d_name, + fs32_to_cpu(sb, de->d_ino)); + UFSD("namlen %u\n", ufs_get_de_namlen(sb, de)); + + if ((flags & UFS_DE_MASK) == UFS_DE_44BSD) + d_type = de->d_u.d_44.d_type; + + if (!dir_emit(ctx, de->d_name, + ufs_get_de_namlen(sb, de), + fs32_to_cpu(sb, de->d_ino), + d_type)) { + folio_release_kmap(folio, de); + return 0; + } + } + ctx->pos += fs16_to_cpu(sb, de->d_reclen); + } + folio_release_kmap(folio, kaddr); + } + return 0; +} + + +/* + * ufs_delete_entry deletes a directory entry by merging it with the + * previous entry. + */ +int ufs_delete_entry(struct inode *inode, struct ufs_dir_entry *dir, + struct folio *folio) +{ + struct super_block *sb = inode->i_sb; + size_t from, to; + char *kaddr; + loff_t pos; + struct ufs_dir_entry *de, *pde = NULL; + int err; + + UFSD("ENTER\n"); + + from = offset_in_folio(folio, dir); + to = from + fs16_to_cpu(sb, dir->d_reclen); + kaddr = (char *)dir - from; + from &= ~(UFS_SB(sb)->s_uspi->s_dirblksize - 1); + de = (struct ufs_dir_entry *) (kaddr + from); + + UFSD("ino %u, reclen %u, namlen %u, name %s\n", + fs32_to_cpu(sb, de->d_ino), + fs16_to_cpu(sb, de->d_reclen), + ufs_get_de_namlen(sb, de), de->d_name); + + while ((char*)de < (char*)dir) { + if (de->d_reclen == 0) { + ufs_error(inode->i_sb, __func__, + "zero-length directory entry"); + return -EIO; + } + pde = de; + de = ufs_next_entry(sb, de); + } + if (pde) + from = offset_in_folio(folio, pde); + pos = folio_pos(folio) + from; + folio_lock(folio); + err = ufs_prepare_chunk(folio, pos, to - from); + if (unlikely(err)) { + folio_unlock(folio); + return err; + } + if (pde) + pde->d_reclen = cpu_to_fs16(sb, to - from); + dir->d_ino = 0; + ufs_commit_chunk(folio, pos, to - from); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + mark_inode_dirty(inode); + return ufs_handle_dirsync(inode); +} + +int ufs_make_empty(struct inode * inode, struct inode *dir) +{ + struct super_block * sb = dir->i_sb; + struct address_space *mapping = inode->i_mapping; + struct folio *folio = filemap_grab_folio(mapping, 0); + const unsigned int chunk_size = UFS_SB(sb)->s_uspi->s_dirblksize; + struct ufs_dir_entry * de; + int err; + char *kaddr; + + if (IS_ERR(folio)) + return PTR_ERR(folio); + + err = ufs_prepare_chunk(folio, 0, chunk_size); + if (err) { + folio_unlock(folio); + goto fail; + } + + kaddr = kmap_local_folio(folio, 0); + memset(kaddr, 0, folio_size(folio)); + + de = (struct ufs_dir_entry *)kaddr; + + de->d_ino = cpu_to_fs32(sb, inode->i_ino); + ufs_set_de_type(sb, de, inode->i_mode); + ufs_set_de_namlen(sb, de, 1); + de->d_reclen = cpu_to_fs16(sb, UFS_DIR_REC_LEN(1)); + strcpy (de->d_name, "."); + de = (struct ufs_dir_entry *) + ((char *)de + fs16_to_cpu(sb, de->d_reclen)); + de->d_ino = cpu_to_fs32(sb, dir->i_ino); + ufs_set_de_type(sb, de, dir->i_mode); + de->d_reclen = cpu_to_fs16(sb, chunk_size - UFS_DIR_REC_LEN(1)); + ufs_set_de_namlen(sb, de, 2); + strcpy (de->d_name, ".."); + kunmap_local(kaddr); + + ufs_commit_chunk(folio, 0, chunk_size); + err = ufs_handle_dirsync(inode); +fail: + folio_put(folio); + return err; +} + +/* + * routine to check that the specified directory is empty (for rmdir) + */ +int ufs_empty_dir(struct inode * inode) +{ + struct super_block *sb = inode->i_sb; + struct folio *folio; + char *kaddr; + unsigned long i, npages = dir_pages(inode); + + for (i = 0; i < npages; i++) { + struct ufs_dir_entry *de; + + kaddr = ufs_get_folio(inode, i, &folio); + if (IS_ERR(kaddr)) + continue; + + de = (struct ufs_dir_entry *)kaddr; + kaddr += ufs_last_byte(inode, i) - UFS_DIR_REC_LEN(1); + + while ((char *)de <= kaddr) { + if (de->d_reclen == 0) { + ufs_error(inode->i_sb, __func__, + "zero-length directory entry: " + "kaddr=%p, de=%p\n", kaddr, de); + goto not_empty; + } + if (de->d_ino) { + u16 namelen=ufs_get_de_namlen(sb, de); + /* check for . and .. */ + if (de->d_name[0] != '.') + goto not_empty; + if (namelen > 2) + goto not_empty; + if (namelen < 2) { + if (inode->i_ino != + fs32_to_cpu(sb, de->d_ino)) + goto not_empty; + } else if (de->d_name[1] != '.') + goto not_empty; + } + de = ufs_next_entry(sb, de); + } + folio_release_kmap(folio, kaddr); + } + return 1; + +not_empty: + folio_release_kmap(folio, kaddr); + return 0; +} + +static int ufs_dir_open(struct inode *inode, struct file *file) +{ + file->private_data = kzalloc(sizeof(u64), GFP_KERNEL); + if (!file->private_data) + return -ENOMEM; + return 0; +} + +static int ufs_dir_release(struct inode *inode, struct file *file) +{ + kfree(file->private_data); + return 0; +} + +static loff_t ufs_dir_llseek(struct file *file, loff_t offset, int whence) +{ + return generic_llseek_cookie(file, offset, whence, + (u64 *)file->private_data); +} + +const struct file_operations ufs_dir_operations = { + .open = ufs_dir_open, + .release = ufs_dir_release, + .read = generic_read_dir, + .iterate_shared = ufs_readdir, + .fsync = simple_fsync, + .llseek = ufs_dir_llseek, + .setlease = generic_setlease, +}; diff --git a/fs/ufs/file.c b/fs/ufs/file.c new file mode 100644 index 000000000..85c509ced --- /dev/null +++ b/fs/ufs/file.c @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ufs/file.c + * + * Copyright (C) 1998 + * Daniel Pirkl + * Charles University, Faculty of Mathematics and Physics + * + * from + * + * linux/fs/ext2/file.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * from + * + * linux/fs/minix/file.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * ext2 fs regular file handling primitives + */ + +#include +#include + +#include "ufs_fs.h" +#include "ufs.h" + +/* + * We have mostly NULL's here: the current defaults are ok for + * the ufs filesystem. + */ + +const struct file_operations ufs_file_operations = { + .llseek = generic_file_llseek, + .read_iter = generic_file_read_iter, + .write_iter = generic_file_write_iter, + .mmap_prepare = generic_file_mmap_prepare, + .open = generic_file_open, + .fsync = simple_fsync, + .splice_read = filemap_splice_read, + .splice_write = iter_file_splice_write, + .setlease = generic_setlease, +}; diff --git a/fs/ufs/ialloc.c b/fs/ufs/ialloc.c new file mode 100644 index 000000000..8e51f4630 --- /dev/null +++ b/fs/ufs/ialloc.c @@ -0,0 +1,355 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ufs/ialloc.c + * + * Copyright (c) 1998 + * Daniel Pirkl + * Charles University, Faculty of Mathematics and Physics + * + * from + * + * linux/fs/ext2/ialloc.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * BSD ufs-inspired inode and directory allocation by + * Stephen Tweedie (sct@dcs.ed.ac.uk), 1993 + * Big-endian to little-endian byte-swapping/bitmaps by + * David S. Miller (davem@caip.rutgers.edu), 1995 + * + * UFS2 write support added by + * Evgeniy Dushistov , 2007 + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ufs_fs.h" +#include "ufs.h" +#include "swab.h" +#include "util.h" + +/* + * NOTE! When we get the inode, we're the only people + * that have access to it, and as such there are no + * race conditions we have to worry about. The inode + * is not on the hash-lists, and it cannot be reached + * through the filesystem because the directory entry + * has been deleted earlier. + * + * HOWEVER: we must make sure that we get no aliases, + * which means that we have to call "clear_inode()" + * _before_ we mark the inode not in use in the inode + * bitmaps. Otherwise a newly created file might use + * the same inode number (not actually the same pointer + * though), and then we'd have two inodes sharing the + * same inode number and space on the harddisk. + */ +void ufs_free_inode (struct inode * inode) +{ + struct super_block * sb; + struct ufs_sb_private_info * uspi; + struct ufs_cg_private_info * ucpi; + struct ufs_cylinder_group * ucg; + int is_directory; + unsigned ino, cg, bit; + + UFSD("ENTER, ino %llu\n", inode->i_ino); + + sb = inode->i_sb; + uspi = UFS_SB(sb)->s_uspi; + + ino = inode->i_ino; + + mutex_lock(&UFS_SB(sb)->s_lock); + + if (!((ino > 1) && (ino < (uspi->s_ncg * uspi->s_ipg )))) { + ufs_warning(sb, "ufs_free_inode", "reserved inode or nonexistent inode %u\n", ino); + mutex_unlock(&UFS_SB(sb)->s_lock); + return; + } + + cg = ufs_inotocg (ino); + bit = ufs_inotocgoff (ino); + ucpi = ufs_load_cylinder (sb, cg); + if (!ucpi) { + mutex_unlock(&UFS_SB(sb)->s_lock); + return; + } + ucg = ubh_get_ucg(UCPI_UBH(ucpi)); + if (!ufs_cg_chkmagic(sb, ucg)) + ufs_panic (sb, "ufs_free_fragments", "internal error, bad cg magic number"); + + ucg->cg_time = ufs_get_seconds(sb); + + is_directory = S_ISDIR(inode->i_mode); + + if (ubh_isclr (UCPI_UBH(ucpi), ucpi->c_iusedoff, bit)) + ufs_error(sb, "ufs_free_inode", "bit already cleared for inode %u", ino); + else { + ubh_clrbit (UCPI_UBH(ucpi), ucpi->c_iusedoff, bit); + if (ino < ucpi->c_irotor) + ucpi->c_irotor = ino; + fs32_add(sb, &ucg->cg_cs.cs_nifree, 1); + uspi->cs_total.cs_nifree++; + fs32_add(sb, &UFS_SB(sb)->fs_cs(cg).cs_nifree, 1); + + if (is_directory) { + fs32_sub(sb, &ucg->cg_cs.cs_ndir, 1); + uspi->cs_total.cs_ndir--; + fs32_sub(sb, &UFS_SB(sb)->fs_cs(cg).cs_ndir, 1); + } + } + + ubh_mark_buffer_dirty (USPI_UBH(uspi)); + ubh_mark_buffer_dirty (UCPI_UBH(ucpi)); + if (sb->s_flags & SB_SYNCHRONOUS) + ubh_sync_block(UCPI_UBH(ucpi)); + + ufs_mark_sb_dirty(sb); + mutex_unlock(&UFS_SB(sb)->s_lock); + UFSD("EXIT\n"); +} + +/* + * Nullify new chunk of inodes, + * BSD people also set ui_gen field of inode + * during nullification, but we not care about + * that because of linux ufs do not support NFS + */ +static void ufs2_init_inodes_chunk(struct super_block *sb, + struct ufs_cg_private_info *ucpi, + struct ufs_cylinder_group *ucg) +{ + struct buffer_head *bh; + struct ufs_sb_private_info *uspi = UFS_SB(sb)->s_uspi; + sector_t beg = uspi->s_sbbase + + ufs_inotofsba(ucpi->c_cgx * uspi->s_ipg + + fs32_to_cpu(sb, ucg->cg_u.cg_u2.cg_initediblk)); + sector_t end = beg + uspi->s_fpb; + + UFSD("ENTER cgno %d\n", ucpi->c_cgx); + + for (; beg < end; ++beg) { + bh = sb_getblk(sb, beg); + lock_buffer(bh); + memset(bh->b_data, 0, sb->s_blocksize); + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + unlock_buffer(bh); + if (sb->s_flags & SB_SYNCHRONOUS) + sync_dirty_buffer(bh); + brelse(bh); + } + + fs32_add(sb, &ucg->cg_u.cg_u2.cg_initediblk, uspi->s_inopb); + ubh_mark_buffer_dirty(UCPI_UBH(ucpi)); + if (sb->s_flags & SB_SYNCHRONOUS) + ubh_sync_block(UCPI_UBH(ucpi)); + + UFSD("EXIT\n"); +} + +/* + * There are two policies for allocating an inode. If the new inode is + * a directory, then a forward search is made for a block group with both + * free space and a low directory-to-inode ratio; if that fails, then of + * the groups with above-average free space, that group with the fewest + * directories already is chosen. + * + * For other inodes, search forward from the parent directory's block + * group to find a free inode. + */ +struct inode *ufs_new_inode(struct inode *dir, umode_t mode) +{ + struct super_block * sb; + struct ufs_sb_info * sbi; + struct ufs_sb_private_info * uspi; + struct ufs_cg_private_info * ucpi; + struct ufs_cylinder_group * ucg; + struct inode * inode; + struct timespec64 ts; + unsigned cg, bit, i, j, start; + struct ufs_inode_info *ufsi; + int err = -ENOSPC; + + UFSD("ENTER\n"); + + /* Cannot create files in a deleted directory */ + if (!dir || !dir->i_nlink) + return ERR_PTR(-EPERM); + sb = dir->i_sb; + inode = new_inode(sb); + if (!inode) + return ERR_PTR(-ENOMEM); + ufsi = UFS_I(inode); + sbi = UFS_SB(sb); + uspi = sbi->s_uspi; + + mutex_lock(&sbi->s_lock); + + /* + * Try to place the inode in its parent directory + */ + i = ufs_inotocg(dir->i_ino); + if (sbi->fs_cs(i).cs_nifree) { + cg = i; + goto cg_found; + } + + /* + * Use a quadratic hash to find a group with a free inode + */ + for ( j = 1; j < uspi->s_ncg; j <<= 1 ) { + i += j; + if (i >= uspi->s_ncg) + i -= uspi->s_ncg; + if (sbi->fs_cs(i).cs_nifree) { + cg = i; + goto cg_found; + } + } + + /* + * That failed: try linear search for a free inode + */ + i = ufs_inotocg(dir->i_ino) + 1; + for (j = 2; j < uspi->s_ncg; j++) { + i++; + if (i >= uspi->s_ncg) + i = 0; + if (sbi->fs_cs(i).cs_nifree) { + cg = i; + goto cg_found; + } + } + + goto failed; + +cg_found: + ucpi = ufs_load_cylinder (sb, cg); + if (!ucpi) { + err = -EIO; + goto failed; + } + ucg = ubh_get_ucg(UCPI_UBH(ucpi)); + if (!ufs_cg_chkmagic(sb, ucg)) + ufs_panic (sb, "ufs_new_inode", "internal error, bad cg magic number"); + + start = ucpi->c_irotor; + bit = ubh_find_next_zero_bit (UCPI_UBH(ucpi), ucpi->c_iusedoff, uspi->s_ipg, start); + if (!(bit < uspi->s_ipg)) { + bit = ubh_find_first_zero_bit (UCPI_UBH(ucpi), ucpi->c_iusedoff, start); + if (!(bit < start)) { + ufs_error (sb, "ufs_new_inode", + "cylinder group %u corrupted - error in inode bitmap\n", cg); + err = -EIO; + goto failed; + } + } + UFSD("start = %u, bit = %u, ipg = %u\n", start, bit, uspi->s_ipg); + if (ubh_isclr (UCPI_UBH(ucpi), ucpi->c_iusedoff, bit)) + ubh_setbit (UCPI_UBH(ucpi), ucpi->c_iusedoff, bit); + else { + ufs_panic (sb, "ufs_new_inode", "internal error"); + err = -EIO; + goto failed; + } + + if (uspi->fs_magic == UFS2_MAGIC) { + u32 initediblk = fs32_to_cpu(sb, ucg->cg_u.cg_u2.cg_initediblk); + + if (bit + uspi->s_inopb > initediblk && + initediblk < fs32_to_cpu(sb, ucg->cg_u.cg_u2.cg_niblk)) + ufs2_init_inodes_chunk(sb, ucpi, ucg); + } + + fs32_sub(sb, &ucg->cg_cs.cs_nifree, 1); + uspi->cs_total.cs_nifree--; + fs32_sub(sb, &sbi->fs_cs(cg).cs_nifree, 1); + + if (S_ISDIR(mode)) { + fs32_add(sb, &ucg->cg_cs.cs_ndir, 1); + uspi->cs_total.cs_ndir++; + fs32_add(sb, &sbi->fs_cs(cg).cs_ndir, 1); + } + ubh_mark_buffer_dirty (USPI_UBH(uspi)); + ubh_mark_buffer_dirty (UCPI_UBH(ucpi)); + if (sb->s_flags & SB_SYNCHRONOUS) + ubh_sync_block(UCPI_UBH(ucpi)); + ufs_mark_sb_dirty(sb); + + inode->i_ino = cg * uspi->s_ipg + bit; + inode_init_owner(&nop_mnt_idmap, inode, dir, mode); + inode->i_blocks = 0; + inode->i_generation = 0; + simple_inode_init_ts(inode); + ufsi->i_flags = UFS_I(dir)->i_flags; + ufsi->i_lastfrag = 0; + ufsi->i_shadow = 0; + ufsi->i_osync = 0; + ufsi->i_oeftflag = 0; + ufsi->i_dir_start_lookup = 0; + memset(&ufsi->i_u1, 0, sizeof(ufsi->i_u1)); + if (insert_inode_locked(inode) < 0) { + err = -EIO; + goto failed; + } + mark_inode_dirty(inode); + + if (uspi->fs_magic == UFS2_MAGIC) { + struct buffer_head *bh; + struct ufs2_inode *ufs2_inode; + + /* + * setup birth date, we do it here because of there is no sense + * to hold it in struct ufs_inode_info, and lose 64 bit + */ + bh = sb_bread(sb, uspi->s_sbbase + ufs_inotofsba(inode->i_ino)); + if (!bh) { + ufs_warning(sb, "ufs_read_inode", + "unable to read inode %llu\n", + inode->i_ino); + err = -EIO; + goto fail_remove_inode; + } + lock_buffer(bh); + ufs2_inode = (struct ufs2_inode *)bh->b_data; + ufs2_inode += ufs_inotofsbo(inode->i_ino); + ktime_get_real_ts64(&ts); + ufs2_inode->ui_birthtime = cpu_to_fs64(sb, ts.tv_sec); + ufs2_inode->ui_birthnsec = cpu_to_fs32(sb, ts.tv_nsec); + mark_buffer_dirty(bh); + unlock_buffer(bh); + if (sb->s_flags & SB_SYNCHRONOUS) + sync_dirty_buffer(bh); + brelse(bh); + } + mutex_unlock(&sbi->s_lock); + + UFSD("allocating inode %llu\n", inode->i_ino); + UFSD("EXIT\n"); + return inode; + +fail_remove_inode: + mutex_unlock(&sbi->s_lock); + clear_nlink(inode); + discard_new_inode(inode); + UFSD("EXIT (FAILED): err %d\n", err); + return ERR_PTR(err); +failed: + mutex_unlock(&sbi->s_lock); + make_bad_inode(inode); + iput (inode); + UFSD("EXIT (FAILED): err %d\n", err); + return ERR_PTR(err); +} diff --git a/fs/ufs/inode.c b/fs/ufs/inode.c new file mode 100644 index 000000000..440d014cc --- /dev/null +++ b/fs/ufs/inode.c @@ -0,0 +1,1222 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ufs/inode.c + * + * Copyright (C) 1998 + * Daniel Pirkl + * Charles University, Faculty of Mathematics and Physics + * + * from + * + * linux/fs/ext2/inode.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * from + * + * linux/fs/minix/inode.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * Goal-directed block allocation by Stephen Tweedie (sct@dcs.ed.ac.uk), 1993 + * Big-endian to little-endian byte-swapping/bitmaps by + * David S. Miller (davem@caip.rutgers.edu), 1995 + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ufs_fs.h" +#include "ufs.h" +#include "swab.h" +#include "util.h" + +static int ufs_block_to_path(struct inode *inode, sector_t i_block, unsigned offsets[4]) +{ + struct ufs_sb_private_info *uspi = UFS_SB(inode->i_sb)->s_uspi; + int ptrs = uspi->s_apb; + int ptrs_bits = uspi->s_apbshift; + const long direct_blocks = UFS_NDADDR, + indirect_blocks = ptrs, + double_blocks = (1 << (ptrs_bits * 2)); + int n = 0; + + + UFSD("ptrs=uspi->s_apb = %d,double_blocks=%ld \n",ptrs,double_blocks); + if (i_block < direct_blocks) { + offsets[n++] = i_block; + } else if ((i_block -= direct_blocks) < indirect_blocks) { + offsets[n++] = UFS_IND_BLOCK; + offsets[n++] = i_block; + } else if ((i_block -= indirect_blocks) < double_blocks) { + offsets[n++] = UFS_DIND_BLOCK; + offsets[n++] = i_block >> ptrs_bits; + offsets[n++] = i_block & (ptrs - 1); + } else if (((i_block -= double_blocks) >> (ptrs_bits * 2)) < ptrs) { + offsets[n++] = UFS_TIND_BLOCK; + offsets[n++] = i_block >> (ptrs_bits * 2); + offsets[n++] = (i_block >> ptrs_bits) & (ptrs - 1); + offsets[n++] = i_block & (ptrs - 1); + } else { + ufs_warning(inode->i_sb, "ufs_block_to_path", "block > big"); + } + return n; +} + +typedef struct { + void *p; + union { + __fs32 key32; + __fs64 key64; + }; + struct buffer_head *bh; +} Indirect; + +static inline int grow_chain32(struct ufs_inode_info *ufsi, + struct buffer_head *bh, __fs32 *v, + Indirect *from, Indirect *to) +{ + Indirect *p; + unsigned seq; + to->bh = bh; + do { + seq = read_seqbegin(&ufsi->meta_lock); + to->key32 = *(__fs32 *)(to->p = v); + for (p = from; p <= to && p->key32 == *(__fs32 *)p->p; p++) + ; + } while (read_seqretry(&ufsi->meta_lock, seq)); + return (p > to); +} + +static inline int grow_chain64(struct ufs_inode_info *ufsi, + struct buffer_head *bh, __fs64 *v, + Indirect *from, Indirect *to) +{ + Indirect *p; + unsigned seq; + to->bh = bh; + do { + seq = read_seqbegin(&ufsi->meta_lock); + to->key64 = *(__fs64 *)(to->p = v); + for (p = from; p <= to && p->key64 == *(__fs64 *)p->p; p++) + ; + } while (read_seqretry(&ufsi->meta_lock, seq)); + return (p > to); +} + +/* + * Returns the location of the fragment from + * the beginning of the filesystem. + */ + +static u64 ufs_frag_map(struct inode *inode, unsigned offsets[4], int depth) +{ + struct ufs_inode_info *ufsi = UFS_I(inode); + struct super_block *sb = inode->i_sb; + struct ufs_sb_private_info *uspi = UFS_SB(sb)->s_uspi; + u64 mask = (u64) uspi->s_apbmask>>uspi->s_fpbshift; + int shift = uspi->s_apbshift-uspi->s_fpbshift; + Indirect chain[4], *q = chain; + unsigned *p; + unsigned flags = UFS_SB(sb)->s_flags; + u64 res = 0; + + UFSD(": uspi->s_fpbshift = %d ,uspi->s_apbmask = %x, mask=%llx\n", + uspi->s_fpbshift, uspi->s_apbmask, + (unsigned long long)mask); + + if (depth == 0) + goto no_block; + +again: + p = offsets; + + if ((flags & UFS_TYPE_MASK) == UFS_TYPE_UFS2) + goto ufs2; + + if (!grow_chain32(ufsi, NULL, &ufsi->i_u1.i_data[*p++], chain, q)) + goto changed; + if (!q->key32) + goto no_block; + while (--depth) { + __fs32 *ptr; + struct buffer_head *bh; + unsigned n = *p++; + + bh = sb_bread(sb, uspi->s_sbbase + + fs32_to_cpu(sb, q->key32) + (n>>shift)); + if (!bh) + goto no_block; + ptr = (__fs32 *)bh->b_data + (n & mask); + if (!grow_chain32(ufsi, bh, ptr, chain, ++q)) + goto changed; + if (!q->key32) + goto no_block; + } + res = fs32_to_cpu(sb, q->key32); + goto found; + +ufs2: + if (!grow_chain64(ufsi, NULL, &ufsi->i_u1.u2_i_data[*p++], chain, q)) + goto changed; + if (!q->key64) + goto no_block; + + while (--depth) { + __fs64 *ptr; + struct buffer_head *bh; + unsigned n = *p++; + + bh = sb_bread(sb, uspi->s_sbbase + + fs64_to_cpu(sb, q->key64) + (n>>shift)); + if (!bh) + goto no_block; + ptr = (__fs64 *)bh->b_data + (n & mask); + if (!grow_chain64(ufsi, bh, ptr, chain, ++q)) + goto changed; + if (!q->key64) + goto no_block; + } + res = fs64_to_cpu(sb, q->key64); +found: + res += uspi->s_sbbase; +no_block: + while (q > chain) { + brelse(q->bh); + q--; + } + return res; + +changed: + while (q > chain) { + brelse(q->bh); + q--; + } + goto again; +} + +/* + * Unpacking tails: we have a file with partial final block and + * we had been asked to extend it. If the fragment being written + * is within the same block, we need to extend the tail just to cover + * that fragment. Otherwise the tail is extended to full block. + * + * Note that we might need to create a _new_ tail, but that will + * be handled elsewhere; this is strictly for resizing old + * ones. + */ +static bool +ufs_extend_tail(struct inode *inode, u64 writes_to, + int *err, struct folio *locked_folio) +{ + struct ufs_inode_info *ufsi = UFS_I(inode); + struct super_block *sb = inode->i_sb; + struct ufs_sb_private_info *uspi = UFS_SB(sb)->s_uspi; + unsigned lastfrag = ufsi->i_lastfrag; /* it's a short file, so unsigned is enough */ + unsigned block = ufs_fragstoblks(lastfrag); + unsigned new_size; + void *p; + u64 tmp; + + if (writes_to < (lastfrag | uspi->s_fpbmask)) + new_size = (writes_to & uspi->s_fpbmask) + 1; + else + new_size = uspi->s_fpb; + + p = ufs_get_direct_data_ptr(uspi, ufsi, block); + tmp = ufs_new_fragments(inode, p, lastfrag, ufs_data_ptr_to_cpu(sb, p), + new_size - (lastfrag & uspi->s_fpbmask), err, + locked_folio); + return tmp != 0; +} + +/** + * ufs_inode_getfrag() - allocate new fragment(s) + * @inode: pointer to inode + * @index: number of block pointer within the inode's array. + * @new_fragment: number of new allocated fragment(s) + * @err: we set it if something wrong + * @new: we set it if we allocate new block + * @locked_folio: for ufs_new_fragments() + */ +static u64 ufs_inode_getfrag(struct inode *inode, unsigned index, + sector_t new_fragment, int *err, + int *new, struct folio *locked_folio) +{ + struct ufs_inode_info *ufsi = UFS_I(inode); + struct super_block *sb = inode->i_sb; + struct ufs_sb_private_info *uspi = UFS_SB(sb)->s_uspi; + u64 tmp, goal, lastfrag; + unsigned nfrags = uspi->s_fpb; + void *p; + + p = ufs_get_direct_data_ptr(uspi, ufsi, index); + tmp = ufs_data_ptr_to_cpu(sb, p); + if (tmp) + goto out; + + lastfrag = ufsi->i_lastfrag; + + /* will that be a new tail? */ + if (new_fragment < UFS_NDIR_FRAGMENT && new_fragment >= lastfrag) + nfrags = (new_fragment & uspi->s_fpbmask) + 1; + + goal = 0; + if (index) { + goal = ufs_data_ptr_to_cpu(sb, + ufs_get_direct_data_ptr(uspi, ufsi, index - 1)); + if (goal) + goal += uspi->s_fpb; + } + tmp = ufs_new_fragments(inode, p, ufs_blknum(new_fragment), + goal, nfrags, err, locked_folio); + + if (!tmp) { + *err = -ENOSPC; + return 0; + } + + if (new) + *new = 1; + inode_set_ctime_current(inode); + if (IS_SYNC(inode)) + ufs_sync_inode (inode); + mark_inode_dirty(inode); +out: + return tmp + uspi->s_sbbase; +} + +/** + * ufs_inode_getblock() - allocate new block + * @inode: pointer to inode + * @ind_block: block number of the indirect block + * @index: number of pointer within the indirect block + * @new_fragment: number of new allocated fragment + * (block will hold this fragment and also uspi->s_fpb-1) + * @err: see ufs_inode_getfrag() + * @new: see ufs_inode_getfrag() + * @locked_folio: see ufs_inode_getfrag() + */ +static u64 ufs_inode_getblock(struct inode *inode, u64 ind_block, + unsigned index, sector_t new_fragment, int *err, + int *new, struct folio *locked_folio) +{ + struct super_block *sb = inode->i_sb; + struct ufs_sb_private_info *uspi = UFS_SB(sb)->s_uspi; + int shift = uspi->s_apbshift - uspi->s_fpbshift; + u64 tmp = 0, goal; + struct buffer_head *bh; + void *p; + + if (!ind_block) + return 0; + + bh = sb_bread(sb, ind_block + (index >> shift)); + if (unlikely(!bh)) { + *err = -EIO; + return 0; + } + + index &= uspi->s_apbmask >> uspi->s_fpbshift; + if (uspi->fs_magic == UFS2_MAGIC) + p = (__fs64 *)bh->b_data + index; + else + p = (__fs32 *)bh->b_data + index; + + tmp = ufs_data_ptr_to_cpu(sb, p); + if (tmp) + goto out; + + if (index && (uspi->fs_magic == UFS2_MAGIC ? + (tmp = fs64_to_cpu(sb, ((__fs64 *)bh->b_data)[index-1])) : + (tmp = fs32_to_cpu(sb, ((__fs32 *)bh->b_data)[index-1])))) + goal = tmp + uspi->s_fpb; + else + goal = bh->b_blocknr + uspi->s_fpb; + tmp = ufs_new_fragments(inode, p, ufs_blknum(new_fragment), goal, + uspi->s_fpb, err, locked_folio); + if (!tmp) + goto out; + + if (new) + *new = 1; + + mark_buffer_dirty(bh); + if (IS_SYNC(inode)) + sync_dirty_buffer(bh); + inode_set_ctime_current(inode); + mark_inode_dirty(inode); +out: + brelse (bh); + UFSD("EXIT\n"); + if (tmp) + tmp += uspi->s_sbbase; + return tmp; +} + +/** + * ufs_getfrag_block() - `get_block_t' function, interface between UFS and + * read_folio, writepages and so on + */ + +static int ufs_getfrag_block(struct inode *inode, sector_t fragment, struct buffer_head *bh_result, int create) +{ + struct super_block *sb = inode->i_sb; + struct ufs_sb_private_info *uspi = UFS_SB(sb)->s_uspi; + int err = 0, new = 0; + unsigned offsets[4]; + int depth = ufs_block_to_path(inode, fragment >> uspi->s_fpbshift, offsets); + u64 phys64 = 0; + unsigned frag = fragment & uspi->s_fpbmask; + + phys64 = ufs_frag_map(inode, offsets, depth); + if (!create) + goto done; + + if (phys64) { + if (fragment >= UFS_NDIR_FRAGMENT) + goto done; + read_seqlock_excl(&UFS_I(inode)->meta_lock); + if (fragment < UFS_I(inode)->i_lastfrag) { + read_sequnlock_excl(&UFS_I(inode)->meta_lock); + goto done; + } + read_sequnlock_excl(&UFS_I(inode)->meta_lock); + } + /* This code entered only while writing ....? */ + + mutex_lock(&UFS_I(inode)->truncate_mutex); + + UFSD("ENTER, ino %llu, fragment %llu\n", inode->i_ino, (unsigned long long)fragment); + if (unlikely(!depth)) { + ufs_warning(sb, "ufs_get_block", "block > big"); + err = -EIO; + goto out; + } + + if (UFS_I(inode)->i_lastfrag < UFS_NDIR_FRAGMENT) { + unsigned lastfrag = UFS_I(inode)->i_lastfrag; + unsigned tailfrags = lastfrag & uspi->s_fpbmask; + if (tailfrags && fragment >= lastfrag) { + if (!ufs_extend_tail(inode, fragment, + &err, bh_result->b_folio)) + goto out; + } + } + + if (depth == 1) { + phys64 = ufs_inode_getfrag(inode, offsets[0], fragment, + &err, &new, bh_result->b_folio); + } else { + int i; + phys64 = ufs_inode_getfrag(inode, offsets[0], fragment, + &err, NULL, NULL); + for (i = 1; i < depth - 1; i++) + phys64 = ufs_inode_getblock(inode, phys64, offsets[i], + fragment, &err, NULL, NULL); + phys64 = ufs_inode_getblock(inode, phys64, offsets[depth - 1], + fragment, &err, &new, bh_result->b_folio); + } +out: + if (phys64) { + phys64 += frag; + map_bh(bh_result, sb, phys64); + if (new) + set_buffer_new(bh_result); + } + mutex_unlock(&UFS_I(inode)->truncate_mutex); + return err; + +done: + if (phys64) + map_bh(bh_result, sb, phys64 + frag); + return 0; +} + +static int ufs_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + return mpage_writepages(mapping, wbc, ufs_getfrag_block); +} + +static int ufs_read_folio(struct file *file, struct folio *folio) +{ + return block_read_full_folio(folio, ufs_getfrag_block); +} + +int ufs_prepare_chunk(struct folio *folio, loff_t pos, unsigned len) +{ + return __block_write_begin(folio, pos, len, ufs_getfrag_block); +} + +static void ufs_truncate_blocks(struct inode *); + +static void ufs_write_failed(struct address_space *mapping, loff_t to) +{ + struct inode *inode = mapping->host; + + if (to > inode->i_size) { + truncate_pagecache(inode, inode->i_size); + ufs_truncate_blocks(inode); + } +} + +static int ufs_write_begin(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, + struct folio **foliop, void **fsdata) +{ + int ret; + + ret = block_write_begin(mapping, pos, len, foliop, ufs_getfrag_block); + if (unlikely(ret)) + ufs_write_failed(mapping, pos + len); + + return ret; +} + +static int ufs_write_end(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned len, unsigned copied, + struct folio *folio, void *fsdata) +{ + int ret; + + ret = generic_write_end(iocb, mapping, pos, len, copied, folio, fsdata); + if (ret < len) + ufs_write_failed(mapping, pos + len); + return ret; +} + +static sector_t ufs_bmap(struct address_space *mapping, sector_t block) +{ + return generic_block_bmap(mapping,block,ufs_getfrag_block); +} + +const struct address_space_operations ufs_aops = { + .dirty_folio = block_dirty_folio, + .invalidate_folio = block_invalidate_folio, + .read_folio = ufs_read_folio, + .writepages = ufs_writepages, + .write_begin = ufs_write_begin, + .write_end = ufs_write_end, + .migrate_folio = buffer_migrate_folio, + .bmap = ufs_bmap +}; + +static void ufs_set_inode_ops(struct inode *inode) +{ + if (S_ISREG(inode->i_mode)) { + inode->i_op = &ufs_file_inode_operations; + inode->i_fop = &ufs_file_operations; + inode->i_mapping->a_ops = &ufs_aops; + } else if (S_ISDIR(inode->i_mode)) { + inode->i_op = &ufs_dir_inode_operations; + inode->i_fop = &ufs_dir_operations; + inode->i_mapping->a_ops = &ufs_aops; + } else if (S_ISLNK(inode->i_mode)) { + if (!inode->i_blocks) { + inode->i_link = (char *)UFS_I(inode)->i_u1.i_symlink; + inode->i_op = &simple_symlink_inode_operations; + } else { + inode->i_mapping->a_ops = &ufs_aops; + inode->i_op = &page_symlink_inode_operations; + inode_nohighmem(inode); + } + } else + init_special_inode(inode, inode->i_mode, + ufs_get_inode_dev(inode->i_sb, UFS_I(inode))); +} + +static int ufs1_read_inode(struct inode *inode, struct ufs_inode *ufs_inode) +{ + struct ufs_inode_info *ufsi = UFS_I(inode); + struct super_block *sb = inode->i_sb; + umode_t mode; + + /* + * Copy data to the in-core inode. + */ + inode->i_mode = mode = fs16_to_cpu(sb, ufs_inode->ui_mode); + set_nlink(inode, fs16_to_cpu(sb, ufs_inode->ui_nlink)); + if (inode->i_nlink == 0) + return -ESTALE; + + /* + * Linux now has 32-bit uid and gid, so we can support EFT. + */ + i_uid_write(inode, ufs_get_inode_uid(sb, ufs_inode)); + i_gid_write(inode, ufs_get_inode_gid(sb, ufs_inode)); + + inode->i_size = fs64_to_cpu(sb, ufs_inode->ui_size); + inode_set_atime(inode, + (signed)fs32_to_cpu(sb, ufs_inode->ui_atime.tv_sec), + 0); + inode_set_ctime(inode, + (signed)fs32_to_cpu(sb, ufs_inode->ui_ctime.tv_sec), + 0); + inode_set_mtime(inode, + (signed)fs32_to_cpu(sb, ufs_inode->ui_mtime.tv_sec), + 0); + inode->i_blocks = fs32_to_cpu(sb, ufs_inode->ui_blocks); + inode->i_generation = fs32_to_cpu(sb, ufs_inode->ui_gen); + ufsi->i_flags = fs32_to_cpu(sb, ufs_inode->ui_flags); + ufsi->i_shadow = fs32_to_cpu(sb, ufs_inode->ui_u3.ui_sun.ui_shadow); + ufsi->i_oeftflag = fs32_to_cpu(sb, ufs_inode->ui_u3.ui_sun.ui_oeftflag); + + + if (S_ISCHR(mode) || S_ISBLK(mode) || inode->i_blocks) { + memcpy(ufsi->i_u1.i_data, &ufs_inode->ui_u2.ui_addr, + sizeof(ufs_inode->ui_u2.ui_addr)); + } else { + memcpy(ufsi->i_u1.i_symlink, ufs_inode->ui_u2.ui_symlink, + sizeof(ufs_inode->ui_u2.ui_symlink) - 1); + ufsi->i_u1.i_symlink[sizeof(ufs_inode->ui_u2.ui_symlink) - 1] = 0; + } + return 0; +} + +static int ufs2_read_inode(struct inode *inode, struct ufs2_inode *ufs2_inode) +{ + struct ufs_inode_info *ufsi = UFS_I(inode); + struct super_block *sb = inode->i_sb; + umode_t mode; + + UFSD("Reading ufs2 inode, ino %llu\n", inode->i_ino); + /* + * Copy data to the in-core inode. + */ + inode->i_mode = mode = fs16_to_cpu(sb, ufs2_inode->ui_mode); + set_nlink(inode, fs16_to_cpu(sb, ufs2_inode->ui_nlink)); + if (inode->i_nlink == 0) + return -ESTALE; + + /* + * Linux now has 32-bit uid and gid, so we can support EFT. + */ + i_uid_write(inode, fs32_to_cpu(sb, ufs2_inode->ui_uid)); + i_gid_write(inode, fs32_to_cpu(sb, ufs2_inode->ui_gid)); + + inode->i_size = fs64_to_cpu(sb, ufs2_inode->ui_size); + inode_set_atime(inode, fs64_to_cpu(sb, ufs2_inode->ui_atime), + fs32_to_cpu(sb, ufs2_inode->ui_atimensec)); + inode_set_ctime(inode, fs64_to_cpu(sb, ufs2_inode->ui_ctime), + fs32_to_cpu(sb, ufs2_inode->ui_ctimensec)); + inode_set_mtime(inode, fs64_to_cpu(sb, ufs2_inode->ui_mtime), + fs32_to_cpu(sb, ufs2_inode->ui_mtimensec)); + inode->i_blocks = fs64_to_cpu(sb, ufs2_inode->ui_blocks); + inode->i_generation = fs32_to_cpu(sb, ufs2_inode->ui_gen); + ufsi->i_flags = fs32_to_cpu(sb, ufs2_inode->ui_flags); + /* + ufsi->i_shadow = fs32_to_cpu(sb, ufs_inode->ui_u3.ui_sun.ui_shadow); + ufsi->i_oeftflag = fs32_to_cpu(sb, ufs_inode->ui_u3.ui_sun.ui_oeftflag); + */ + + if (S_ISCHR(mode) || S_ISBLK(mode) || inode->i_blocks) { + memcpy(ufsi->i_u1.u2_i_data, &ufs2_inode->ui_u2.ui_addr, + sizeof(ufs2_inode->ui_u2.ui_addr)); + } else { + memcpy(ufsi->i_u1.i_symlink, ufs2_inode->ui_u2.ui_symlink, + sizeof(ufs2_inode->ui_u2.ui_symlink) - 1); + ufsi->i_u1.i_symlink[sizeof(ufs2_inode->ui_u2.ui_symlink) - 1] = 0; + } + return 0; +} + +struct inode *ufs_iget(struct super_block *sb, unsigned long ino) +{ + struct ufs_inode_info *ufsi; + struct ufs_sb_private_info *uspi = UFS_SB(sb)->s_uspi; + struct buffer_head * bh; + struct inode *inode; + int err = -EIO; + + UFSD("ENTER, ino %lu\n", ino); + + if (ino < UFS_ROOTINO || ino > (uspi->s_ncg * uspi->s_ipg)) { + ufs_warning(sb, "ufs_read_inode", "bad inode number (%lu)\n", + ino); + return ERR_PTR(-EIO); + } + + inode = iget_locked(sb, ino); + if (!inode) + return ERR_PTR(-ENOMEM); + if (!(inode_state_read_once(inode) & I_NEW)) + return inode; + + ufsi = UFS_I(inode); + + bh = sb_bread(sb, uspi->s_sbbase + ufs_inotofsba(inode->i_ino)); + if (!bh) { + ufs_warning(sb, "ufs_read_inode", "unable to read inode %llu\n", + inode->i_ino); + goto bad_inode; + } + if ((UFS_SB(sb)->s_flags & UFS_TYPE_MASK) == UFS_TYPE_UFS2) { + struct ufs2_inode *ufs2_inode = (struct ufs2_inode *)bh->b_data; + + err = ufs2_read_inode(inode, + ufs2_inode + ufs_inotofsbo(inode->i_ino)); + } else { + struct ufs_inode *ufs_inode = (struct ufs_inode *)bh->b_data; + + err = ufs1_read_inode(inode, + ufs_inode + ufs_inotofsbo(inode->i_ino)); + } + brelse(bh); + if (err) + goto bad_inode; + + inode_inc_iversion(inode); + ufsi->i_lastfrag = + (inode->i_size + uspi->s_fsize - 1) >> uspi->s_fshift; + ufsi->i_dir_start_lookup = 0; + ufsi->i_osync = 0; + + ufs_set_inode_ops(inode); + + UFSD("EXIT\n"); + unlock_new_inode(inode); + return inode; + +bad_inode: + iget_failed(inode); + return ERR_PTR(err); +} + +static void ufs1_update_inode(struct inode *inode, struct ufs_inode *ufs_inode) +{ + struct super_block *sb = inode->i_sb; + struct ufs_inode_info *ufsi = UFS_I(inode); + + ufs_inode->ui_mode = cpu_to_fs16(sb, inode->i_mode); + ufs_inode->ui_nlink = cpu_to_fs16(sb, inode->i_nlink); + + ufs_set_inode_uid(sb, ufs_inode, i_uid_read(inode)); + ufs_set_inode_gid(sb, ufs_inode, i_gid_read(inode)); + + ufs_inode->ui_size = cpu_to_fs64(sb, inode->i_size); + ufs_inode->ui_atime.tv_sec = cpu_to_fs32(sb, + inode_get_atime_sec(inode)); + ufs_inode->ui_atime.tv_usec = 0; + ufs_inode->ui_ctime.tv_sec = cpu_to_fs32(sb, + inode_get_ctime_sec(inode)); + ufs_inode->ui_ctime.tv_usec = 0; + ufs_inode->ui_mtime.tv_sec = cpu_to_fs32(sb, + inode_get_mtime_sec(inode)); + ufs_inode->ui_mtime.tv_usec = 0; + ufs_inode->ui_blocks = cpu_to_fs32(sb, inode->i_blocks); + ufs_inode->ui_flags = cpu_to_fs32(sb, ufsi->i_flags); + ufs_inode->ui_gen = cpu_to_fs32(sb, inode->i_generation); + + if ((UFS_SB(sb)->s_flags & UFS_UID_MASK) == UFS_UID_EFT) { + ufs_inode->ui_u3.ui_sun.ui_shadow = cpu_to_fs32(sb, ufsi->i_shadow); + ufs_inode->ui_u3.ui_sun.ui_oeftflag = cpu_to_fs32(sb, ufsi->i_oeftflag); + } + + if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) { + /* ufs_inode->ui_u2.ui_addr.ui_db[0] = cpu_to_fs32(sb, inode->i_rdev); */ + ufs_inode->ui_u2.ui_addr.ui_db[0] = ufsi->i_u1.i_data[0]; + } else if (inode->i_blocks) { + memcpy(&ufs_inode->ui_u2.ui_addr, ufsi->i_u1.i_data, + sizeof(ufs_inode->ui_u2.ui_addr)); + } + else { + memcpy(&ufs_inode->ui_u2.ui_symlink, ufsi->i_u1.i_symlink, + sizeof(ufs_inode->ui_u2.ui_symlink)); + } + + if (!inode->i_nlink) + memset (ufs_inode, 0, sizeof(struct ufs_inode)); +} + +static void ufs2_update_inode(struct inode *inode, struct ufs2_inode *ufs_inode) +{ + struct super_block *sb = inode->i_sb; + struct ufs_inode_info *ufsi = UFS_I(inode); + + UFSD("ENTER\n"); + ufs_inode->ui_mode = cpu_to_fs16(sb, inode->i_mode); + ufs_inode->ui_nlink = cpu_to_fs16(sb, inode->i_nlink); + + ufs_inode->ui_uid = cpu_to_fs32(sb, i_uid_read(inode)); + ufs_inode->ui_gid = cpu_to_fs32(sb, i_gid_read(inode)); + + ufs_inode->ui_size = cpu_to_fs64(sb, inode->i_size); + ufs_inode->ui_atime = cpu_to_fs64(sb, inode_get_atime_sec(inode)); + ufs_inode->ui_atimensec = cpu_to_fs32(sb, + inode_get_atime_nsec(inode)); + ufs_inode->ui_ctime = cpu_to_fs64(sb, inode_get_ctime_sec(inode)); + ufs_inode->ui_ctimensec = cpu_to_fs32(sb, + inode_get_ctime_nsec(inode)); + ufs_inode->ui_mtime = cpu_to_fs64(sb, inode_get_mtime_sec(inode)); + ufs_inode->ui_mtimensec = cpu_to_fs32(sb, + inode_get_mtime_nsec(inode)); + + ufs_inode->ui_blocks = cpu_to_fs64(sb, inode->i_blocks); + ufs_inode->ui_flags = cpu_to_fs32(sb, ufsi->i_flags); + ufs_inode->ui_gen = cpu_to_fs32(sb, inode->i_generation); + + if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) { + /* ufs_inode->ui_u2.ui_addr.ui_db[0] = cpu_to_fs32(sb, inode->i_rdev); */ + ufs_inode->ui_u2.ui_addr.ui_db[0] = ufsi->i_u1.u2_i_data[0]; + } else if (inode->i_blocks) { + memcpy(&ufs_inode->ui_u2.ui_addr, ufsi->i_u1.u2_i_data, + sizeof(ufs_inode->ui_u2.ui_addr)); + } else { + memcpy(&ufs_inode->ui_u2.ui_symlink, ufsi->i_u1.i_symlink, + sizeof(ufs_inode->ui_u2.ui_symlink)); + } + + if (!inode->i_nlink) + memset (ufs_inode, 0, sizeof(struct ufs2_inode)); + UFSD("EXIT\n"); +} + +static int ufs_update_inode(struct inode * inode, int do_sync) +{ + struct super_block *sb = inode->i_sb; + struct ufs_sb_private_info *uspi = UFS_SB(sb)->s_uspi; + struct buffer_head * bh; + + UFSD("ENTER, ino %llu\n", inode->i_ino); + + if (inode->i_ino < UFS_ROOTINO || + inode->i_ino > (uspi->s_ncg * uspi->s_ipg)) { + ufs_warning (sb, "ufs_read_inode", "bad inode number (%llu)\n", inode->i_ino); + return -1; + } + + bh = sb_bread(sb, ufs_inotofsba(inode->i_ino)); + if (!bh) { + ufs_warning (sb, "ufs_read_inode", "unable to read inode %llu\n", inode->i_ino); + return -1; + } + if (uspi->fs_magic == UFS2_MAGIC) { + struct ufs2_inode *ufs2_inode = (struct ufs2_inode *)bh->b_data; + + ufs2_update_inode(inode, + ufs2_inode + ufs_inotofsbo(inode->i_ino)); + } else { + struct ufs_inode *ufs_inode = (struct ufs_inode *) bh->b_data; + + ufs1_update_inode(inode, ufs_inode + ufs_inotofsbo(inode->i_ino)); + } + + mark_buffer_dirty(bh); + if (do_sync) + sync_dirty_buffer(bh); + brelse (bh); + + UFSD("EXIT\n"); + return 0; +} + +int ufs_write_inode(struct inode *inode, struct writeback_control *wbc) +{ + return ufs_update_inode(inode, wbc->sync_mode == WB_SYNC_ALL); +} + +int ufs_sync_inode (struct inode *inode) +{ + return ufs_update_inode (inode, 1); +} + +void ufs_evict_inode(struct inode * inode) +{ + int want_delete = 0; + + if (!inode->i_nlink && !is_bad_inode(inode)) + want_delete = 1; + + truncate_inode_pages_final(&inode->i_data); + if (want_delete) { + inode->i_size = 0; + if (inode->i_blocks && + (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) || + S_ISLNK(inode->i_mode))) + ufs_truncate_blocks(inode); + ufs_update_inode(inode, inode_needs_sync(inode)); + } + + clear_inode(inode); + + if (want_delete) + ufs_free_inode(inode); +} + +struct to_free { + struct inode *inode; + u64 to; + unsigned count; +}; + +static inline void free_data(struct to_free *ctx, u64 from, unsigned count) +{ + if (ctx->count && ctx->to != from) { + ufs_free_blocks(ctx->inode, ctx->to - ctx->count, ctx->count); + ctx->count = 0; + } + ctx->count += count; + ctx->to = from + count; +} + +#define DIRECT_FRAGMENT ((inode->i_size + uspi->s_fsize - 1) >> uspi->s_fshift) + +/* + * used only for truncation down to direct blocks. + */ +static void ufs_trunc_direct(struct inode *inode) +{ + struct ufs_inode_info *ufsi = UFS_I(inode); + struct super_block *sb = inode->i_sb; + struct ufs_sb_private_info *uspi = UFS_SB(sb)->s_uspi; + unsigned int new_frags, old_frags; + unsigned int old_slot, new_slot; + unsigned int old_tail, new_tail; + struct to_free ctx = {.inode = inode}; + + UFSD("ENTER: ino %llu\n", inode->i_ino); + + new_frags = DIRECT_FRAGMENT; + // new_frags = first fragment past the new EOF + old_frags = min_t(u64, UFS_NDIR_FRAGMENT, ufsi->i_lastfrag); + // old_frags = first fragment past the old EOF or covered by indirects + + if (new_frags >= old_frags) // expanding - nothing to free + goto done; + + old_tail = ufs_fragnum(old_frags); + old_slot = ufs_fragstoblks(old_frags); + new_tail = ufs_fragnum(new_frags); + new_slot = ufs_fragstoblks(new_frags); + + if (old_slot == new_slot) { // old_tail > 0 + void *p = ufs_get_direct_data_ptr(uspi, ufsi, old_slot); + u64 tmp = ufs_data_ptr_to_cpu(sb, p); + if (!tmp) + ufs_panic(sb, __func__, "internal error"); + if (!new_tail) { + write_seqlock(&ufsi->meta_lock); + ufs_data_ptr_clear(uspi, p); + write_sequnlock(&ufsi->meta_lock); + } + ufs_free_fragments(inode, tmp + new_tail, old_tail - new_tail); + } else { + unsigned int slot = new_slot; + + if (new_tail) { + void *p = ufs_get_direct_data_ptr(uspi, ufsi, slot++); + u64 tmp = ufs_data_ptr_to_cpu(sb, p); + if (!tmp) + ufs_panic(sb, __func__, "internal error"); + + ufs_free_fragments(inode, tmp + new_tail, + uspi->s_fpb - new_tail); + } + while (slot < old_slot) { + void *p = ufs_get_direct_data_ptr(uspi, ufsi, slot++); + u64 tmp = ufs_data_ptr_to_cpu(sb, p); + if (!tmp) + continue; + write_seqlock(&ufsi->meta_lock); + ufs_data_ptr_clear(uspi, p); + write_sequnlock(&ufsi->meta_lock); + + free_data(&ctx, tmp, uspi->s_fpb); + } + + free_data(&ctx, 0, 0); + + if (old_tail) { + void *p = ufs_get_direct_data_ptr(uspi, ufsi, slot); + u64 tmp = ufs_data_ptr_to_cpu(sb, p); + if (!tmp) + ufs_panic(sb, __func__, "internal error"); + write_seqlock(&ufsi->meta_lock); + ufs_data_ptr_clear(uspi, p); + write_sequnlock(&ufsi->meta_lock); + + ufs_free_fragments(inode, tmp, old_tail); + } + } +done: + UFSD("EXIT: ino %llu\n", inode->i_ino); +} + +static void free_full_branch(struct inode *inode, u64 ind_block, int depth) +{ + struct super_block *sb = inode->i_sb; + struct ufs_sb_private_info *uspi = UFS_SB(sb)->s_uspi; + struct ufs_buffer_head *ubh = ubh_bread(sb, ind_block, uspi->s_bsize); + unsigned i; + + if (!ubh) + return; + + if (--depth) { + for (i = 0; i < uspi->s_apb; i++) { + void *p = ubh_get_data_ptr(uspi, ubh, i); + u64 block = ufs_data_ptr_to_cpu(sb, p); + if (block) + free_full_branch(inode, block, depth); + } + } else { + struct to_free ctx = {.inode = inode}; + + for (i = 0; i < uspi->s_apb; i++) { + void *p = ubh_get_data_ptr(uspi, ubh, i); + u64 block = ufs_data_ptr_to_cpu(sb, p); + if (block) + free_data(&ctx, block, uspi->s_fpb); + } + free_data(&ctx, 0, 0); + } + + ubh_bforget(ubh); + ufs_free_blocks(inode, ind_block, uspi->s_fpb); +} + +static void free_branch_tail(struct inode *inode, unsigned from, struct ufs_buffer_head *ubh, int depth) +{ + struct super_block *sb = inode->i_sb; + struct ufs_sb_private_info *uspi = UFS_SB(sb)->s_uspi; + unsigned i; + + if (--depth) { + for (i = from; i < uspi->s_apb ; i++) { + void *p = ubh_get_data_ptr(uspi, ubh, i); + u64 block = ufs_data_ptr_to_cpu(sb, p); + if (block) { + write_seqlock(&UFS_I(inode)->meta_lock); + ufs_data_ptr_clear(uspi, p); + write_sequnlock(&UFS_I(inode)->meta_lock); + ubh_mark_buffer_dirty(ubh); + free_full_branch(inode, block, depth); + } + } + } else { + struct to_free ctx = {.inode = inode}; + + for (i = from; i < uspi->s_apb; i++) { + void *p = ubh_get_data_ptr(uspi, ubh, i); + u64 block = ufs_data_ptr_to_cpu(sb, p); + if (block) { + write_seqlock(&UFS_I(inode)->meta_lock); + ufs_data_ptr_clear(uspi, p); + write_sequnlock(&UFS_I(inode)->meta_lock); + ubh_mark_buffer_dirty(ubh); + free_data(&ctx, block, uspi->s_fpb); + } + } + free_data(&ctx, 0, 0); + } + if (IS_SYNC(inode) && ubh_buffer_dirty(ubh)) + ubh_sync_block(ubh); + ubh_brelse(ubh); +} + +static int ufs_alloc_lastblock(struct inode *inode, loff_t size) +{ + int err = 0; + struct super_block *sb = inode->i_sb; + struct address_space *mapping = inode->i_mapping; + struct ufs_sb_private_info *uspi = UFS_SB(sb)->s_uspi; + unsigned i, end; + sector_t lastfrag; + struct folio *folio; + struct buffer_head *bh; + u64 phys64; + + lastfrag = (size + uspi->s_fsize - 1) >> uspi->s_fshift; + + if (!lastfrag) + goto out; + + lastfrag--; + + folio = ufs_get_locked_folio(mapping, lastfrag >> + (PAGE_SHIFT - inode->i_blkbits)); + if (IS_ERR(folio)) { + err = -EIO; + goto out; + } + + end = lastfrag & ((1 << (PAGE_SHIFT - inode->i_blkbits)) - 1); + bh = folio_buffers(folio); + for (i = 0; i < end; ++i) + bh = bh->b_this_page; + + err = ufs_getfrag_block(inode, lastfrag, bh, 1); + + if (unlikely(err)) + goto out_unlock; + + if (buffer_new(bh)) { + clear_buffer_new(bh); + clean_bdev_bh_alias(bh); + /* + * we do not zeroize fragment, because of + * if it maped to hole, it already contains zeroes + */ + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + folio_mark_dirty(folio); + } + + if (lastfrag >= UFS_IND_FRAGMENT) { + end = uspi->s_fpb - ufs_fragnum(lastfrag) - 1; + phys64 = bh->b_blocknr + 1; + for (i = 0; i < end; ++i) { + bh = sb_getblk(sb, i + phys64); + lock_buffer(bh); + memset(bh->b_data, 0, sb->s_blocksize); + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); + unlock_buffer(bh); + sync_dirty_buffer(bh); + brelse(bh); + } + } +out_unlock: + ufs_put_locked_folio(folio); +out: + return err; +} + +static void ufs_truncate_blocks(struct inode *inode) +{ + struct ufs_inode_info *ufsi = UFS_I(inode); + struct super_block *sb = inode->i_sb; + struct ufs_sb_private_info *uspi = UFS_SB(sb)->s_uspi; + unsigned offsets[4]; + int depth; + int depth2; + unsigned i; + struct ufs_buffer_head *ubh[3]; + void *p; + u64 block; + + if (inode->i_size) { + sector_t last = (inode->i_size - 1) >> uspi->s_bshift; + depth = ufs_block_to_path(inode, last, offsets); + if (!depth) + return; + } else { + depth = 1; + } + + for (depth2 = depth - 1; depth2; depth2--) + if (offsets[depth2] != uspi->s_apb - 1) + break; + + mutex_lock(&ufsi->truncate_mutex); + if (depth == 1) { + ufs_trunc_direct(inode); + offsets[0] = UFS_IND_BLOCK; + } else { + /* get the blocks that should be partially emptied */ + p = ufs_get_direct_data_ptr(uspi, ufsi, offsets[0]++); + for (i = 0; i < depth2; i++) { + block = ufs_data_ptr_to_cpu(sb, p); + if (!block) + break; + ubh[i] = ubh_bread(sb, block, uspi->s_bsize); + if (!ubh[i]) { + write_seqlock(&ufsi->meta_lock); + ufs_data_ptr_clear(uspi, p); + write_sequnlock(&ufsi->meta_lock); + break; + } + p = ubh_get_data_ptr(uspi, ubh[i], offsets[i + 1]++); + } + while (i--) + free_branch_tail(inode, offsets[i + 1], ubh[i], depth - i - 1); + } + for (i = offsets[0]; i <= UFS_TIND_BLOCK; i++) { + p = ufs_get_direct_data_ptr(uspi, ufsi, i); + block = ufs_data_ptr_to_cpu(sb, p); + if (block) { + write_seqlock(&ufsi->meta_lock); + ufs_data_ptr_clear(uspi, p); + write_sequnlock(&ufsi->meta_lock); + free_full_branch(inode, block, i - UFS_IND_BLOCK + 1); + } + } + read_seqlock_excl(&ufsi->meta_lock); + ufsi->i_lastfrag = DIRECT_FRAGMENT; + read_sequnlock_excl(&ufsi->meta_lock); + mark_inode_dirty(inode); + mutex_unlock(&ufsi->truncate_mutex); +} + +static int ufs_truncate(struct inode *inode, loff_t size) +{ + int err = 0; + + UFSD("ENTER: ino %llu, i_size: %llu, old_i_size: %llu\n", + inode->i_ino, (unsigned long long)size, + (unsigned long long)i_size_read(inode)); + + if (!(S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) || + S_ISLNK(inode->i_mode))) + return -EINVAL; + if (IS_APPEND(inode) || IS_IMMUTABLE(inode)) + return -EPERM; + + err = ufs_alloc_lastblock(inode, size); + + if (err) + goto out; + + block_truncate_page(inode->i_mapping, size, ufs_getfrag_block); + + truncate_setsize(inode, size); + + ufs_truncate_blocks(inode); + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + mark_inode_dirty(inode); +out: + UFSD("EXIT: err %d\n", err); + return err; +} + +int ufs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr) +{ + struct inode *inode = d_inode(dentry); + unsigned int ia_valid = attr->ia_valid; + int error; + + error = setattr_prepare(&nop_mnt_idmap, dentry, attr); + if (error) + return error; + + if (ia_valid & ATTR_SIZE && attr->ia_size != inode->i_size) { + error = ufs_truncate(inode, attr->ia_size); + if (error) + return error; + } + + setattr_copy(&nop_mnt_idmap, inode, attr); + mark_inode_dirty(inode); + return 0; +} + +const struct inode_operations ufs_file_inode_operations = { + .setattr = ufs_setattr, +}; diff --git a/fs/ufs/namei.c b/fs/ufs/namei.c new file mode 100644 index 000000000..6703f3bcf --- /dev/null +++ b/fs/ufs/namei.c @@ -0,0 +1,329 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ufs/namei.c + * + * Migration to usage of "page cache" on May 2006 by + * Evgeniy Dushistov based on ext2 code base. + * + * Copyright (C) 1998 + * Daniel Pirkl + * Charles University, Faculty of Mathematics and Physics + * + * from + * + * linux/fs/ext2/namei.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * from + * + * linux/fs/minix/namei.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * Big-endian to little-endian byte-swapping/bitmaps by + * David S. Miller (davem@caip.rutgers.edu), 1995 + */ + +#include +#include + +#include "ufs_fs.h" +#include "ufs.h" +#include "util.h" + +static inline int ufs_add_nondir(struct dentry *dentry, struct inode *inode) +{ + int err = ufs_add_link(dentry, inode); + if (!err) { + d_instantiate_new(dentry, inode); + return 0; + } + inode_dec_link_count(inode); + discard_new_inode(inode); + return err; +} + +static struct dentry *ufs_lookup(struct inode * dir, struct dentry *dentry, unsigned int flags) +{ + struct inode * inode = NULL; + ino_t ino; + + if (dentry->d_name.len > UFS_MAXNAMLEN) + return ERR_PTR(-ENAMETOOLONG); + + ino = ufs_inode_by_name(dir, &dentry->d_name); + if (ino) + inode = ufs_iget(dir->i_sb, ino); + return d_splice_alias(inode, dentry); +} + +/* + * By the time this is called, we already have created + * the directory cache entry for the new file, but it + * is so far negative - it has no inode. + * + * If the create succeeds, we fill in the inode information + * with d_instantiate(). + */ +static int ufs_create (struct mnt_idmap * idmap, + struct inode * dir, struct dentry * dentry, umode_t mode) +{ + struct inode *inode; + + inode = ufs_new_inode(dir, mode); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + inode->i_op = &ufs_file_inode_operations; + inode->i_fop = &ufs_file_operations; + inode->i_mapping->a_ops = &ufs_aops; + mark_inode_dirty(inode); + return ufs_add_nondir(dentry, inode); +} + +static int ufs_mknod(struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t rdev) +{ + struct inode *inode; + int err; + + if (!old_valid_dev(rdev)) + return -EINVAL; + + inode = ufs_new_inode(dir, mode); + err = PTR_ERR(inode); + if (!IS_ERR(inode)) { + init_special_inode(inode, mode, rdev); + ufs_set_inode_dev(inode->i_sb, UFS_I(inode), rdev); + mark_inode_dirty(inode); + err = ufs_add_nondir(dentry, inode); + } + return err; +} + +static int ufs_symlink (struct mnt_idmap * idmap, struct inode * dir, + struct dentry * dentry, const char * symname) +{ + struct super_block * sb = dir->i_sb; + int err; + unsigned l = strlen(symname)+1; + struct inode * inode; + + if (l > sb->s_blocksize) + return -ENAMETOOLONG; + + inode = ufs_new_inode(dir, S_IFLNK | S_IRWXUGO); + err = PTR_ERR(inode); + if (IS_ERR(inode)) + return err; + + if (l > UFS_SB(sb)->s_uspi->s_maxsymlinklen) { + /* slow symlink */ + inode->i_op = &page_symlink_inode_operations; + inode_nohighmem(inode); + inode->i_mapping->a_ops = &ufs_aops; + err = page_symlink(inode, symname, l); + if (err) + goto out_fail; + } else { + /* fast symlink */ + inode->i_op = &simple_symlink_inode_operations; + inode->i_link = (char *)UFS_I(inode)->i_u1.i_symlink; + memcpy(inode->i_link, symname, l); + inode->i_size = l-1; + } + mark_inode_dirty(inode); + + return ufs_add_nondir(dentry, inode); + +out_fail: + inode_dec_link_count(inode); + discard_new_inode(inode); + return err; +} + +static int ufs_link (struct dentry * old_dentry, struct inode * dir, + struct dentry *dentry) +{ + struct inode *inode = d_inode(old_dentry); + int error; + + inode_set_ctime_current(inode); + inode_inc_link_count(inode); + ihold(inode); + + error = ufs_add_link(dentry, inode); + if (error) { + inode_dec_link_count(inode); + iput(inode); + } else + d_instantiate(dentry, inode); + return error; +} + +static struct dentry *ufs_mkdir(struct mnt_idmap * idmap, struct inode * dir, + struct dentry * dentry, umode_t mode) +{ + struct inode * inode; + int err; + + inode_inc_link_count(dir); + + inode = ufs_new_inode(dir, mode); + err = PTR_ERR(inode); + if (IS_ERR(inode)) + goto out_dir; + + inode->i_op = &ufs_dir_inode_operations; + inode->i_fop = &ufs_dir_operations; + inode->i_mapping->a_ops = &ufs_aops; + + inode_inc_link_count(inode); + + err = ufs_make_empty(inode, dir); + if (err) + goto out_fail; + + err = ufs_add_link(dentry, inode); + if (err) + goto out_fail; + + d_instantiate_new(dentry, inode); + return NULL; + +out_fail: + inode_dec_link_count(inode); + inode_dec_link_count(inode); + discard_new_inode(inode); +out_dir: + inode_dec_link_count(dir); + return ERR_PTR(err); +} + +static int ufs_unlink(struct inode *dir, struct dentry *dentry) +{ + struct inode * inode = d_inode(dentry); + struct ufs_dir_entry *de; + struct folio *folio; + int err; + + de = ufs_find_entry(dir, &dentry->d_name, &folio); + if (!de) + return -ENOENT; + + err = ufs_delete_entry(dir, de, folio); + if (!err) { + inode_set_ctime_to_ts(inode, inode_get_ctime(dir)); + inode_dec_link_count(inode); + } + folio_release_kmap(folio, de); + return err; +} + +static int ufs_rmdir (struct inode * dir, struct dentry *dentry) +{ + struct inode * inode = d_inode(dentry); + int err= -ENOTEMPTY; + + if (ufs_empty_dir (inode)) { + err = ufs_unlink(dir, dentry); + if (!err) { + inode->i_size = 0; + inode_dec_link_count(inode); + inode_dec_link_count(dir); + } + } + return err; +} + +static int ufs_rename(struct mnt_idmap *idmap, struct inode *old_dir, + struct dentry *old_dentry, struct inode *new_dir, + struct dentry *new_dentry, unsigned int flags) +{ + struct inode *old_inode = d_inode(old_dentry); + struct inode *new_inode = d_inode(new_dentry); + struct folio *dir_folio = NULL; + struct ufs_dir_entry * dir_de = NULL; + struct folio *old_folio; + struct ufs_dir_entry *old_de; + int err; + + if (flags & ~RENAME_NOREPLACE) + return -EINVAL; + + old_de = ufs_find_entry(old_dir, &old_dentry->d_name, &old_folio); + if (!old_de) + return -ENOENT; + + if (S_ISDIR(old_inode->i_mode)) { + err = -EIO; + dir_de = ufs_dotdot(old_inode, &dir_folio); + if (!dir_de) + goto out_old; + } + + if (new_inode) { + struct folio *new_folio; + struct ufs_dir_entry *new_de; + + err = -ENOTEMPTY; + if (dir_de && !ufs_empty_dir(new_inode)) + goto out_dir; + + err = -ENOENT; + new_de = ufs_find_entry(new_dir, &new_dentry->d_name, &new_folio); + if (!new_de) + goto out_dir; + err = ufs_set_link(new_dir, new_de, new_folio, old_inode, 1); + folio_release_kmap(new_folio, new_de); + if (err) + goto out_dir; + inode_set_ctime_current(new_inode); + if (dir_de) + drop_nlink(new_inode); + inode_dec_link_count(new_inode); + } else { + err = ufs_add_link(new_dentry, old_inode); + if (err) + goto out_dir; + if (dir_de) + inode_inc_link_count(new_dir); + } + + /* + * Like most other Unix systems, set the ctime for inodes on a + * rename. + */ + inode_set_ctime_current(old_inode); + mark_inode_dirty(old_inode); + + err = ufs_delete_entry(old_dir, old_de, old_folio); + if (!err && dir_de) { + if (old_dir != new_dir) + err = ufs_set_link(old_inode, dir_de, dir_folio, + new_dir, 0); + inode_dec_link_count(old_dir); + } +out_dir: + if (dir_de) + folio_release_kmap(dir_folio, dir_de); +out_old: + folio_release_kmap(old_folio, old_de); + return err; +} + +const struct inode_operations ufs_dir_inode_operations = { + .create = ufs_create, + .lookup = ufs_lookup, + .link = ufs_link, + .unlink = ufs_unlink, + .symlink = ufs_symlink, + .mkdir = ufs_mkdir, + .rmdir = ufs_rmdir, + .mknod = ufs_mknod, + .rename = ufs_rename, +}; diff --git a/fs/ufs/super.c b/fs/ufs/super.c new file mode 100644 index 000000000..6dcf6d048 --- /dev/null +++ b/fs/ufs/super.c @@ -0,0 +1,1495 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * linux/fs/ufs/super.c + * + * Copyright (C) 1998 + * Daniel Pirkl + * Charles University, Faculty of Mathematics and Physics + */ + +/* Derived from + * + * linux/fs/ext2/super.c + * + * Copyright (C) 1992, 1993, 1994, 1995 + * Remy Card (card@masi.ibp.fr) + * Laboratoire MASI - Institut Blaise Pascal + * Universite Pierre et Marie Curie (Paris VI) + * + * from + * + * linux/fs/minix/inode.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * Big-endian to little-endian byte-swapping/bitmaps by + * David S. Miller (davem@caip.rutgers.edu), 1995 + */ + +/* + * Inspired by + * + * linux/fs/ufs/super.c + * + * Copyright (C) 1996 + * Adrian Rodriguez (adrian@franklins-tower.rutgers.edu) + * Laboratory for Computer Science Research Computing Facility + * Rutgers, The State University of New Jersey + * + * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) + * + * Kernel module support added on 96/04/26 by + * Stefan Reinauer + * + * Module usage counts added on 96/04/29 by + * Gertjan van Wingerde + * + * Clean swab support on 19970406 by + * Francois-Rene Rideau + * + * 4.4BSD (FreeBSD) support added on February 1st 1998 by + * Niels Kristian Bech Jensen partially based + * on code by Martin von Loewis . + * + * NeXTstep support added on February 5th 1998 by + * Niels Kristian Bech Jensen . + * + * write support Daniel Pirkl 1998 + * + * HP/UX hfs filesystem support added by + * Martin K. Petersen , August 1999 + * + * UFS2 (of FreeBSD 5.x) support added by + * Niraj Kumar , Jan 2004 + * + * UFS2 write support added by + * Evgeniy Dushistov , 2007 + */ + +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ufs_fs.h" +#include "ufs.h" +#include "swab.h" +#include "util.h" + +static struct inode *ufs_nfs_get_inode(struct super_block *sb, u64 ino, u32 generation) +{ + struct ufs_sb_private_info *uspi = UFS_SB(sb)->s_uspi; + struct inode *inode; + + if (ino < UFS_ROOTINO || ino > (u64)uspi->s_ncg * uspi->s_ipg) + return ERR_PTR(-ESTALE); + + inode = ufs_iget(sb, ino); + if (IS_ERR(inode)) + return ERR_CAST(inode); + if (generation && inode->i_generation != generation) { + iput(inode); + return ERR_PTR(-ESTALE); + } + return inode; +} + +static struct dentry *ufs_fh_to_dentry(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + return generic_fh_to_dentry(sb, fid, fh_len, fh_type, ufs_nfs_get_inode); +} + +static struct dentry *ufs_fh_to_parent(struct super_block *sb, struct fid *fid, + int fh_len, int fh_type) +{ + return generic_fh_to_parent(sb, fid, fh_len, fh_type, ufs_nfs_get_inode); +} + +static struct dentry *ufs_get_parent(struct dentry *child) +{ + ino_t ino; + + ino = ufs_inode_by_name(d_inode(child), &dotdot_name); + if (!ino) + return ERR_PTR(-ENOENT); + return d_obtain_alias(ufs_iget(child->d_sb, ino)); +} + +static const struct export_operations ufs_export_ops = { + .encode_fh = generic_encode_ino32_fh, + .fh_to_dentry = ufs_fh_to_dentry, + .fh_to_parent = ufs_fh_to_parent, + .get_parent = ufs_get_parent, +}; + +#ifdef CONFIG_UFS_DEBUG +/* + * Print contents of ufs_super_block, useful for debugging + */ +static void ufs_print_super_stuff(struct super_block *sb, + struct ufs_super_block_first *usb1, + struct ufs_super_block_second *usb2, + struct ufs_super_block_third *usb3) +{ + u32 magic = fs32_to_cpu(sb, usb3->fs_magic); + + pr_debug("ufs_print_super_stuff\n"); + pr_debug(" magic: 0x%x\n", magic); + if (fs32_to_cpu(sb, usb3->fs_magic) == UFS2_MAGIC) { + pr_debug(" fs_size: %llu\n", (unsigned long long) + fs64_to_cpu(sb, usb3->fs_un1.fs_u2.fs_size)); + pr_debug(" fs_dsize: %llu\n", (unsigned long long) + fs64_to_cpu(sb, usb3->fs_un1.fs_u2.fs_dsize)); + pr_debug(" bsize: %u\n", + fs32_to_cpu(sb, usb1->fs_bsize)); + pr_debug(" fsize: %u\n", + fs32_to_cpu(sb, usb1->fs_fsize)); + pr_debug(" fs_volname: %s\n", usb2->fs_un.fs_u2.fs_volname); + pr_debug(" fs_sblockloc: %llu\n", (unsigned long long) + fs64_to_cpu(sb, usb2->fs_un.fs_u2.fs_sblockloc)); + pr_debug(" cs_ndir(No of dirs): %llu\n", (unsigned long long) + fs64_to_cpu(sb, usb2->fs_un.fs_u2.cs_ndir)); + pr_debug(" cs_nbfree(No of free blocks): %llu\n", + (unsigned long long) + fs64_to_cpu(sb, usb2->fs_un.fs_u2.cs_nbfree)); + pr_info(" cs_nifree(Num of free inodes): %llu\n", + (unsigned long long) + fs64_to_cpu(sb, usb3->fs_un1.fs_u2.cs_nifree)); + pr_info(" cs_nffree(Num of free frags): %llu\n", + (unsigned long long) + fs64_to_cpu(sb, usb3->fs_un1.fs_u2.cs_nffree)); + pr_info(" fs_maxsymlinklen: %u\n", + fs32_to_cpu(sb, usb3->fs_un2.fs_44.fs_maxsymlinklen)); + } else { + pr_debug(" sblkno: %u\n", fs32_to_cpu(sb, usb1->fs_sblkno)); + pr_debug(" cblkno: %u\n", fs32_to_cpu(sb, usb1->fs_cblkno)); + pr_debug(" iblkno: %u\n", fs32_to_cpu(sb, usb1->fs_iblkno)); + pr_debug(" dblkno: %u\n", fs32_to_cpu(sb, usb1->fs_dblkno)); + pr_debug(" cgoffset: %u\n", + fs32_to_cpu(sb, usb1->fs_cgoffset)); + pr_debug(" ~cgmask: 0x%x\n", + ~fs32_to_cpu(sb, usb1->fs_cgmask)); + pr_debug(" size: %u\n", fs32_to_cpu(sb, usb1->fs_size)); + pr_debug(" dsize: %u\n", fs32_to_cpu(sb, usb1->fs_dsize)); + pr_debug(" ncg: %u\n", fs32_to_cpu(sb, usb1->fs_ncg)); + pr_debug(" bsize: %u\n", fs32_to_cpu(sb, usb1->fs_bsize)); + pr_debug(" fsize: %u\n", fs32_to_cpu(sb, usb1->fs_fsize)); + pr_debug(" frag: %u\n", fs32_to_cpu(sb, usb1->fs_frag)); + pr_debug(" fragshift: %u\n", + fs32_to_cpu(sb, usb1->fs_fragshift)); + pr_debug(" ~fmask: %u\n", ~fs32_to_cpu(sb, usb1->fs_fmask)); + pr_debug(" fshift: %u\n", fs32_to_cpu(sb, usb1->fs_fshift)); + pr_debug(" sbsize: %u\n", fs32_to_cpu(sb, usb1->fs_sbsize)); + pr_debug(" spc: %u\n", fs32_to_cpu(sb, usb1->fs_spc)); + pr_debug(" cpg: %u\n", fs32_to_cpu(sb, usb1->fs_cpg)); + pr_debug(" ipg: %u\n", fs32_to_cpu(sb, usb1->fs_ipg)); + pr_debug(" fpg: %u\n", fs32_to_cpu(sb, usb1->fs_fpg)); + pr_debug(" csaddr: %u\n", fs32_to_cpu(sb, usb1->fs_csaddr)); + pr_debug(" cssize: %u\n", fs32_to_cpu(sb, usb1->fs_cssize)); + pr_debug(" cgsize: %u\n", fs32_to_cpu(sb, usb1->fs_cgsize)); + pr_debug(" fstodb: %u\n", + fs32_to_cpu(sb, usb1->fs_fsbtodb)); + pr_debug(" nrpos: %u\n", fs32_to_cpu(sb, usb3->fs_nrpos)); + pr_debug(" ndir %u\n", + fs32_to_cpu(sb, usb1->fs_cstotal.cs_ndir)); + pr_debug(" nifree %u\n", + fs32_to_cpu(sb, usb1->fs_cstotal.cs_nifree)); + pr_debug(" nbfree %u\n", + fs32_to_cpu(sb, usb1->fs_cstotal.cs_nbfree)); + pr_debug(" nffree %u\n", + fs32_to_cpu(sb, usb1->fs_cstotal.cs_nffree)); + } + pr_debug("\n"); +} + +/* + * Print contents of ufs_cylinder_group, useful for debugging + */ +static void ufs_print_cylinder_stuff(struct super_block *sb, + struct ufs_cylinder_group *cg) +{ + pr_debug("\nufs_print_cylinder_stuff\n"); + pr_debug("size of ucg: %zu\n", sizeof(struct ufs_cylinder_group)); + pr_debug(" magic: %x\n", fs32_to_cpu(sb, cg->cg_magic)); + pr_debug(" time: %u\n", fs32_to_cpu(sb, cg->cg_time)); + pr_debug(" cgx: %u\n", fs32_to_cpu(sb, cg->cg_cgx)); + pr_debug(" ncyl: %u\n", fs16_to_cpu(sb, cg->cg_ncyl)); + pr_debug(" niblk: %u\n", fs16_to_cpu(sb, cg->cg_niblk)); + pr_debug(" ndblk: %u\n", fs32_to_cpu(sb, cg->cg_ndblk)); + pr_debug(" cs_ndir: %u\n", fs32_to_cpu(sb, cg->cg_cs.cs_ndir)); + pr_debug(" cs_nbfree: %u\n", fs32_to_cpu(sb, cg->cg_cs.cs_nbfree)); + pr_debug(" cs_nifree: %u\n", fs32_to_cpu(sb, cg->cg_cs.cs_nifree)); + pr_debug(" cs_nffree: %u\n", fs32_to_cpu(sb, cg->cg_cs.cs_nffree)); + pr_debug(" rotor: %u\n", fs32_to_cpu(sb, cg->cg_rotor)); + pr_debug(" frotor: %u\n", fs32_to_cpu(sb, cg->cg_frotor)); + pr_debug(" irotor: %u\n", fs32_to_cpu(sb, cg->cg_irotor)); + pr_debug(" frsum: %u, %u, %u, %u, %u, %u, %u, %u\n", + fs32_to_cpu(sb, cg->cg_frsum[0]), fs32_to_cpu(sb, cg->cg_frsum[1]), + fs32_to_cpu(sb, cg->cg_frsum[2]), fs32_to_cpu(sb, cg->cg_frsum[3]), + fs32_to_cpu(sb, cg->cg_frsum[4]), fs32_to_cpu(sb, cg->cg_frsum[5]), + fs32_to_cpu(sb, cg->cg_frsum[6]), fs32_to_cpu(sb, cg->cg_frsum[7])); + pr_debug(" btotoff: %u\n", fs32_to_cpu(sb, cg->cg_btotoff)); + pr_debug(" boff: %u\n", fs32_to_cpu(sb, cg->cg_boff)); + pr_debug(" iuseoff: %u\n", fs32_to_cpu(sb, cg->cg_iusedoff)); + pr_debug(" freeoff: %u\n", fs32_to_cpu(sb, cg->cg_freeoff)); + pr_debug(" nextfreeoff: %u\n", fs32_to_cpu(sb, cg->cg_nextfreeoff)); + pr_debug(" clustersumoff %u\n", + fs32_to_cpu(sb, cg->cg_u.cg_44.cg_clustersumoff)); + pr_debug(" clusteroff %u\n", + fs32_to_cpu(sb, cg->cg_u.cg_44.cg_clusteroff)); + pr_debug(" nclusterblks %u\n", + fs32_to_cpu(sb, cg->cg_u.cg_44.cg_nclusterblks)); + pr_debug("\n"); +} +#else +# define ufs_print_super_stuff(sb, usb1, usb2, usb3) /**/ +# define ufs_print_cylinder_stuff(sb, cg) /**/ +#endif /* CONFIG_UFS_DEBUG */ + +static const struct super_operations ufs_super_ops; + +void ufs_error (struct super_block * sb, const char * function, + const char * fmt, ...) +{ + struct ufs_sb_private_info * uspi; + struct ufs_super_block_first * usb1; + struct va_format vaf; + va_list args; + + uspi = UFS_SB(sb)->s_uspi; + usb1 = ubh_get_usb_first(uspi); + + if (!sb_rdonly(sb)) { + usb1->fs_clean = UFS_FSBAD; + ubh_mark_buffer_dirty(USPI_UBH(uspi)); + ufs_mark_sb_dirty(sb); + sb->s_flags |= SB_RDONLY; + } + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + switch (UFS_SB(sb)->s_on_err) { + case UFS_MOUNT_ONERROR_PANIC: + panic("panic (device %s): %s: %pV\n", + sb->s_id, function, &vaf); + + case UFS_MOUNT_ONERROR_LOCK: + case UFS_MOUNT_ONERROR_UMOUNT: + case UFS_MOUNT_ONERROR_REPAIR: + pr_crit("error (device %s): %s: %pV\n", + sb->s_id, function, &vaf); + } + va_end(args); +} + +void ufs_panic (struct super_block * sb, const char * function, + const char * fmt, ...) +{ + struct ufs_sb_private_info * uspi; + struct ufs_super_block_first * usb1; + struct va_format vaf; + va_list args; + + uspi = UFS_SB(sb)->s_uspi; + usb1 = ubh_get_usb_first(uspi); + + if (!sb_rdonly(sb)) { + usb1->fs_clean = UFS_FSBAD; + ubh_mark_buffer_dirty(USPI_UBH(uspi)); + ufs_mark_sb_dirty(sb); + } + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + sb->s_flags |= SB_RDONLY; + pr_crit("panic (device %s): %s: %pV\n", + sb->s_id, function, &vaf); + va_end(args); +} + +void ufs_warning (struct super_block * sb, const char * function, + const char * fmt, ...) +{ + struct va_format vaf; + va_list args; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + pr_warn("(device %s): %s: %pV\n", + sb->s_id, function, &vaf); + va_end(args); +} + +enum { Opt_type, Opt_onerror }; + +static const struct constant_table ufs_param_ufstype[] = { + {"old", UFS_MOUNT_UFSTYPE_OLD}, + {"sunx86", UFS_MOUNT_UFSTYPE_SUNx86}, + {"sun", UFS_MOUNT_UFSTYPE_SUN}, + {"sunos", UFS_MOUNT_UFSTYPE_SUNOS}, + {"44bsd", UFS_MOUNT_UFSTYPE_44BSD}, + {"ufs2", UFS_MOUNT_UFSTYPE_UFS2}, + {"5xbsd", UFS_MOUNT_UFSTYPE_UFS2}, + {"hp", UFS_MOUNT_UFSTYPE_HP}, + {"nextstep-cd", UFS_MOUNT_UFSTYPE_NEXTSTEP_CD}, + {"nextstep", UFS_MOUNT_UFSTYPE_NEXTSTEP}, + {"openstep", UFS_MOUNT_UFSTYPE_OPENSTEP}, + {} +}; + +static const struct constant_table ufs_param_onerror[] = { + {"panic", UFS_MOUNT_ONERROR_PANIC}, + {"lock", UFS_MOUNT_ONERROR_LOCK}, + {"umount", UFS_MOUNT_ONERROR_UMOUNT}, + {"repair", UFS_MOUNT_ONERROR_REPAIR}, + {} +}; + +static const struct fs_parameter_spec ufs_param_spec[] = { + fsparam_enum ("ufstype", Opt_type, ufs_param_ufstype), + fsparam_enum ("onerror", Opt_onerror, ufs_param_onerror), + {} +}; + +struct ufs_fs_context { + unsigned int flavour, on_err; +}; + +static int ufs_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct ufs_fs_context *ctx = fc->fs_private; + struct fs_parse_result result; + int opt; + + UFSD("ENTER\n"); + + opt = fs_parse(fc, ufs_param_spec, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_type: + if (ctx->flavour == result.uint_32) /* no-op */ + return 0; + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE) { + pr_err("ufstype can't be changed during remount\n"); + return -EINVAL; + } + if (ctx->flavour) { + pr_err("conflicting ufstype options\n"); + return -EINVAL; + } + ctx->flavour = result.uint_32; + break; + case Opt_onerror: + ctx->on_err = result.uint_32; + break; + default: + return -EINVAL; + } + return 0; +} + +/* + * Different types of UFS hold fs_cstotal in different + * places, and use different data structure for it. + * To make things simpler we just copy fs_cstotal to ufs_sb_private_info + */ +static void ufs_setup_cstotal(struct super_block *sb) +{ + struct ufs_sb_info *sbi = UFS_SB(sb); + struct ufs_sb_private_info *uspi = sbi->s_uspi; + struct ufs_super_block_first *usb1; + struct ufs_super_block_second *usb2; + struct ufs_super_block_third *usb3; + unsigned mtype = sbi->s_flavour; + + UFSD("ENTER, mtype=%u\n", mtype); + usb1 = ubh_get_usb_first(uspi); + usb2 = ubh_get_usb_second(uspi); + usb3 = ubh_get_usb_third(uspi); + + if ((mtype == UFS_MOUNT_UFSTYPE_44BSD && + (usb2->fs_un.fs_u2.fs_maxbsize == usb1->fs_bsize)) || + mtype == UFS_MOUNT_UFSTYPE_UFS2) { + /*we have statistic in different place, then usual*/ + uspi->cs_total.cs_ndir = fs64_to_cpu(sb, usb2->fs_un.fs_u2.cs_ndir); + uspi->cs_total.cs_nbfree = fs64_to_cpu(sb, usb2->fs_un.fs_u2.cs_nbfree); + uspi->cs_total.cs_nifree = fs64_to_cpu(sb, usb3->fs_un1.fs_u2.cs_nifree); + uspi->cs_total.cs_nffree = fs64_to_cpu(sb, usb3->fs_un1.fs_u2.cs_nffree); + } else { + uspi->cs_total.cs_ndir = fs32_to_cpu(sb, usb1->fs_cstotal.cs_ndir); + uspi->cs_total.cs_nbfree = fs32_to_cpu(sb, usb1->fs_cstotal.cs_nbfree); + uspi->cs_total.cs_nifree = fs32_to_cpu(sb, usb1->fs_cstotal.cs_nifree); + uspi->cs_total.cs_nffree = fs32_to_cpu(sb, usb1->fs_cstotal.cs_nffree); + } + UFSD("EXIT\n"); +} + +/* + * Read on-disk structures associated with cylinder groups + */ +static int ufs_read_cylinder_structures(struct super_block *sb) +{ + struct ufs_sb_info *sbi = UFS_SB(sb); + struct ufs_sb_private_info *uspi = sbi->s_uspi; + unsigned char * base, * space; + unsigned size, blks, i; + + UFSD("ENTER\n"); + + /* + * Read cs structures from (usually) first data block + * on the device. + */ + size = uspi->s_cssize; + blks = (size + uspi->s_fsize - 1) >> uspi->s_fshift; + base = space = kmalloc(size, GFP_NOFS); + if (!base) + goto failed; + sbi->s_csp = (struct ufs_csum *)space; + for (i = 0; i < blks; i++) { + struct buffer_head *bh = sb_bread(sb, uspi->s_csaddr + i); + if (!bh) + goto failed; + memcpy(space, bh->b_data, uspi->s_fsize); + space += uspi->s_fsize; + brelse (bh); + } + + /* + * Read cylinder group (we read only first fragment from block + * at this time) and prepare internal data structures for cg caching. + */ + sbi->s_ucg = kmalloc_objs(struct buffer_head *, uspi->s_ncg, GFP_NOFS); + if (!sbi->s_ucg) + goto failed; + for (i = 0; i < uspi->s_ncg; i++) + sbi->s_ucg[i] = NULL; + for (i = 0; i < UFS_MAX_GROUP_LOADED; i++) { + sbi->s_ucpi[i] = NULL; + sbi->s_cgno[i] = UFS_CGNO_EMPTY; + } + for (i = 0; i < uspi->s_ncg; i++) { + UFSD("read cg %u\n", i); + if (!(sbi->s_ucg[i] = sb_bread(sb, ufs_cgcmin(i)))) + goto failed; + if (!ufs_cg_chkmagic (sb, (struct ufs_cylinder_group *) sbi->s_ucg[i]->b_data)) + goto failed; + + ufs_print_cylinder_stuff(sb, (struct ufs_cylinder_group *) sbi->s_ucg[i]->b_data); + } + for (i = 0; i < UFS_MAX_GROUP_LOADED; i++) { + if (!(sbi->s_ucpi[i] = kmalloc_obj(struct ufs_cg_private_info, GFP_NOFS))) + goto failed; + sbi->s_cgno[i] = UFS_CGNO_EMPTY; + } + sbi->s_cg_loaded = 0; + UFSD("EXIT\n"); + return 1; + +failed: + kfree (base); + if (sbi->s_ucg) { + for (i = 0; i < uspi->s_ncg; i++) + if (sbi->s_ucg[i]) + brelse (sbi->s_ucg[i]); + kfree (sbi->s_ucg); + for (i = 0; i < UFS_MAX_GROUP_LOADED; i++) + kfree (sbi->s_ucpi[i]); + } + UFSD("EXIT (FAILED)\n"); + return 0; +} + +/* + * Sync our internal copy of fs_cstotal with disk + */ +static void ufs_put_cstotal(struct super_block *sb) +{ + unsigned mtype = UFS_SB(sb)->s_flavour; + struct ufs_sb_private_info *uspi = UFS_SB(sb)->s_uspi; + struct ufs_super_block_first *usb1; + struct ufs_super_block_second *usb2; + struct ufs_super_block_third *usb3; + + UFSD("ENTER\n"); + usb1 = ubh_get_usb_first(uspi); + usb2 = ubh_get_usb_second(uspi); + usb3 = ubh_get_usb_third(uspi); + + if (mtype == UFS_MOUNT_UFSTYPE_UFS2) { + /*we have statistic in different place, then usual*/ + usb2->fs_un.fs_u2.cs_ndir = + cpu_to_fs64(sb, uspi->cs_total.cs_ndir); + usb2->fs_un.fs_u2.cs_nbfree = + cpu_to_fs64(sb, uspi->cs_total.cs_nbfree); + usb3->fs_un1.fs_u2.cs_nifree = + cpu_to_fs64(sb, uspi->cs_total.cs_nifree); + usb3->fs_un1.fs_u2.cs_nffree = + cpu_to_fs64(sb, uspi->cs_total.cs_nffree); + goto out; + } + + if (mtype == UFS_MOUNT_UFSTYPE_44BSD && + (usb2->fs_un.fs_u2.fs_maxbsize == usb1->fs_bsize)) { + /* store stats in both old and new places */ + usb2->fs_un.fs_u2.cs_ndir = + cpu_to_fs64(sb, uspi->cs_total.cs_ndir); + usb2->fs_un.fs_u2.cs_nbfree = + cpu_to_fs64(sb, uspi->cs_total.cs_nbfree); + usb3->fs_un1.fs_u2.cs_nifree = + cpu_to_fs64(sb, uspi->cs_total.cs_nifree); + usb3->fs_un1.fs_u2.cs_nffree = + cpu_to_fs64(sb, uspi->cs_total.cs_nffree); + } + usb1->fs_cstotal.cs_ndir = cpu_to_fs32(sb, uspi->cs_total.cs_ndir); + usb1->fs_cstotal.cs_nbfree = cpu_to_fs32(sb, uspi->cs_total.cs_nbfree); + usb1->fs_cstotal.cs_nifree = cpu_to_fs32(sb, uspi->cs_total.cs_nifree); + usb1->fs_cstotal.cs_nffree = cpu_to_fs32(sb, uspi->cs_total.cs_nffree); +out: + ubh_mark_buffer_dirty(USPI_UBH(uspi)); + ufs_print_super_stuff(sb, usb1, usb2, usb3); + UFSD("EXIT\n"); +} + +/** + * ufs_put_super_internal() - put on-disk intrenal structures + * @sb: pointer to super_block structure + * Put on-disk structures associated with cylinder groups + * and write them back to disk, also update cs_total on disk + */ +static void ufs_put_super_internal(struct super_block *sb) +{ + struct ufs_sb_info *sbi = UFS_SB(sb); + struct ufs_sb_private_info *uspi = sbi->s_uspi; + unsigned char * base, * space; + unsigned blks, size, i; + + + UFSD("ENTER\n"); + + ufs_put_cstotal(sb); + size = uspi->s_cssize; + blks = (size + uspi->s_fsize - 1) >> uspi->s_fshift; + base = space = (char*) sbi->s_csp; + for (i = 0; i < blks; i++, space += uspi->s_fsize) { + struct buffer_head *bh = sb_bread(sb, uspi->s_csaddr + i); + + if (unlikely(!bh)) { // better than an oops... + ufs_panic(sb, __func__, + "can't write part of cylinder group summary"); + continue; + } + memcpy(bh->b_data, space, uspi->s_fsize); + mark_buffer_dirty(bh); + brelse(bh); + } + for (i = 0; i < sbi->s_cg_loaded; i++) { + ufs_put_cylinder (sb, i); + kfree (sbi->s_ucpi[i]); + } + for (; i < UFS_MAX_GROUP_LOADED; i++) + kfree (sbi->s_ucpi[i]); + for (i = 0; i < uspi->s_ncg; i++) + brelse (sbi->s_ucg[i]); + kfree (sbi->s_ucg); + kfree (base); + + UFSD("EXIT\n"); +} + +static int ufs_sync_fs(struct super_block *sb, int wait) +{ + struct ufs_sb_private_info * uspi; + struct ufs_super_block_first * usb1; + struct ufs_super_block_third * usb3; + unsigned flags; + + mutex_lock(&UFS_SB(sb)->s_lock); + + UFSD("ENTER\n"); + + flags = UFS_SB(sb)->s_flags; + uspi = UFS_SB(sb)->s_uspi; + usb1 = ubh_get_usb_first(uspi); + usb3 = ubh_get_usb_third(uspi); + + usb1->fs_time = ufs_get_seconds(sb); + if ((flags & UFS_ST_MASK) == UFS_ST_SUN || + (flags & UFS_ST_MASK) == UFS_ST_SUNOS || + (flags & UFS_ST_MASK) == UFS_ST_SUNx86) + ufs_set_fs_state(sb, usb1, usb3, + UFS_FSOK - fs32_to_cpu(sb, usb1->fs_time)); + ufs_put_cstotal(sb); + + UFSD("EXIT\n"); + mutex_unlock(&UFS_SB(sb)->s_lock); + + return 0; +} + +static void delayed_sync_fs(struct work_struct *work) +{ + struct ufs_sb_info *sbi; + + sbi = container_of(work, struct ufs_sb_info, sync_work.work); + + spin_lock(&sbi->work_lock); + sbi->work_queued = 0; + spin_unlock(&sbi->work_lock); + + ufs_sync_fs(sbi->sb, 1); +} + +void ufs_mark_sb_dirty(struct super_block *sb) +{ + struct ufs_sb_info *sbi = UFS_SB(sb); + unsigned long delay; + + spin_lock(&sbi->work_lock); + if (!sbi->work_queued) { + delay = msecs_to_jiffies(dirty_writeback_interval * 10); + queue_delayed_work(system_dfl_long_wq, &sbi->sync_work, delay); + sbi->work_queued = 1; + } + spin_unlock(&sbi->work_lock); +} + +static void ufs_put_super(struct super_block *sb) +{ + struct ufs_sb_info * sbi = UFS_SB(sb); + + UFSD("ENTER\n"); + + if (!sb_rdonly(sb)) + ufs_put_super_internal(sb); + cancel_delayed_work_sync(&sbi->sync_work); + + ubh_brelse_uspi (sbi->s_uspi); + kfree (sbi->s_uspi); + kfree (sbi); + sb->s_fs_info = NULL; + UFSD("EXIT\n"); + return; +} + +static u64 ufs_max_bytes(struct super_block *sb) +{ + struct ufs_sb_private_info *uspi = UFS_SB(sb)->s_uspi; + int bits = uspi->s_apbshift; + u64 res; + + if (bits > 21) + res = ~0ULL; + else + res = UFS_NDADDR + (1LL << bits) + (1LL << (2*bits)) + + (1LL << (3*bits)); + + if (res >= (MAX_LFS_FILESIZE >> uspi->s_bshift)) + return MAX_LFS_FILESIZE; + return res << uspi->s_bshift; +} + +static int ufs_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct ufs_fs_context *ctx = fc->fs_private; + int silent = fc->sb_flags & SB_SILENT; + struct ufs_sb_info * sbi; + struct ufs_sb_private_info * uspi; + struct ufs_super_block_first * usb1; + struct ufs_super_block_second * usb2; + struct ufs_super_block_third * usb3; + struct ufs_buffer_head * ubh; + struct inode *inode; + unsigned block_size, super_block_size; + unsigned flags; + unsigned super_block_offset; + unsigned maxsymlen; + int ret = -EINVAL; + + uspi = NULL; + ubh = NULL; + flags = 0; + + UFSD("ENTER\n"); + +#ifndef CONFIG_UFS_FS_WRITE + if (!sb_rdonly(sb)) { + pr_err("ufs was compiled with read-only support, can't be mounted as read-write\n"); + return -EROFS; + } +#endif + + sbi = kzalloc_obj(struct ufs_sb_info); + if (!sbi) + goto failed_nomem; + sb->s_fs_info = sbi; + sbi->sb = sb; + + UFSD("flag %u\n", (int)(sb_rdonly(sb))); + + mutex_init(&sbi->s_lock); + spin_lock_init(&sbi->work_lock); + INIT_DELAYED_WORK(&sbi->sync_work, delayed_sync_fs); + + sbi->s_flavour = ctx->flavour; + sbi->s_on_err = ctx->on_err; + + if (!sbi->s_flavour) { + if (!silent) + pr_err("You didn't specify the type of your ufs filesystem\n\n" + "mount -t ufs -o ufstype=" + "sun|sunx86|44bsd|ufs2|5xbsd|old|hp|nextstep|nextstep-cd|openstep ...\n\n" + ">>>WARNING<<< Wrong ufstype may corrupt your filesystem, " + "default is ufstype=old\n"); + sbi->s_flavour = UFS_MOUNT_UFSTYPE_OLD; + } + + uspi = kzalloc_obj(struct ufs_sb_private_info); + sbi->s_uspi = uspi; + if (!uspi) + goto failed; + uspi->s_dirblksize = UFS_SECTOR_SIZE; + super_block_offset=UFS_SBLOCK; + + sb->s_maxbytes = MAX_LFS_FILESIZE; + + sb->s_time_gran = NSEC_PER_SEC; + sb->s_time_min = S32_MIN; + sb->s_time_max = S32_MAX; + + switch (sbi->s_flavour) { + case UFS_MOUNT_UFSTYPE_44BSD: + UFSD("ufstype=44bsd\n"); + uspi->s_fsize = block_size = 512; + uspi->s_fmask = ~(512 - 1); + uspi->s_fshift = 9; + uspi->s_sbsize = super_block_size = 1536; + uspi->s_sbbase = 0; + flags |= UFS_DE_44BSD | UFS_UID_44BSD | UFS_ST_44BSD | UFS_CG_44BSD; + break; + case UFS_MOUNT_UFSTYPE_UFS2: + UFSD("ufstype=ufs2\n"); + super_block_offset=SBLOCK_UFS2; + uspi->s_fsize = block_size = 512; + uspi->s_fmask = ~(512 - 1); + uspi->s_fshift = 9; + uspi->s_sbsize = super_block_size = 1536; + uspi->s_sbbase = 0; + sb->s_time_gran = 1; + sb->s_time_min = S64_MIN; + sb->s_time_max = S64_MAX; + flags |= UFS_TYPE_UFS2 | UFS_DE_44BSD | UFS_UID_44BSD | UFS_ST_44BSD | UFS_CG_44BSD; + break; + + case UFS_MOUNT_UFSTYPE_SUN: + UFSD("ufstype=sun\n"); + uspi->s_fsize = block_size = 1024; + uspi->s_fmask = ~(1024 - 1); + uspi->s_fshift = 10; + uspi->s_sbsize = super_block_size = 2048; + uspi->s_sbbase = 0; + uspi->s_maxsymlinklen = 0; /* Not supported on disk */ + flags |= UFS_DE_OLD | UFS_UID_EFT | UFS_ST_SUN | UFS_CG_SUN; + break; + + case UFS_MOUNT_UFSTYPE_SUNOS: + UFSD("ufstype=sunos\n"); + uspi->s_fsize = block_size = 1024; + uspi->s_fmask = ~(1024 - 1); + uspi->s_fshift = 10; + uspi->s_sbsize = 2048; + super_block_size = 2048; + uspi->s_sbbase = 0; + uspi->s_maxsymlinklen = 0; /* Not supported on disk */ + flags |= UFS_DE_OLD | UFS_UID_OLD | UFS_ST_SUNOS | UFS_CG_SUN; + break; + + case UFS_MOUNT_UFSTYPE_SUNx86: + UFSD("ufstype=sunx86\n"); + uspi->s_fsize = block_size = 1024; + uspi->s_fmask = ~(1024 - 1); + uspi->s_fshift = 10; + uspi->s_sbsize = super_block_size = 2048; + uspi->s_sbbase = 0; + uspi->s_maxsymlinklen = 0; /* Not supported on disk */ + flags |= UFS_DE_OLD | UFS_UID_EFT | UFS_ST_SUNx86 | UFS_CG_SUN; + break; + + case UFS_MOUNT_UFSTYPE_OLD: + UFSD("ufstype=old\n"); + uspi->s_fsize = block_size = 1024; + uspi->s_fmask = ~(1024 - 1); + uspi->s_fshift = 10; + uspi->s_sbsize = super_block_size = 2048; + uspi->s_sbbase = 0; + flags |= UFS_DE_OLD | UFS_UID_OLD | UFS_ST_OLD | UFS_CG_OLD; + if (!sb_rdonly(sb)) { + if (!silent) + pr_info("ufstype=old is supported read-only\n"); + sb->s_flags |= SB_RDONLY; + } + break; + + case UFS_MOUNT_UFSTYPE_NEXTSTEP: + UFSD("ufstype=nextstep\n"); + uspi->s_fsize = block_size = 1024; + uspi->s_fmask = ~(1024 - 1); + uspi->s_fshift = 10; + uspi->s_sbsize = super_block_size = 2048; + uspi->s_sbbase = 0; + uspi->s_dirblksize = 1024; + flags |= UFS_DE_OLD | UFS_UID_OLD | UFS_ST_OLD | UFS_CG_OLD; + if (!sb_rdonly(sb)) { + if (!silent) + pr_info("ufstype=nextstep is supported read-only\n"); + sb->s_flags |= SB_RDONLY; + } + break; + + case UFS_MOUNT_UFSTYPE_NEXTSTEP_CD: + UFSD("ufstype=nextstep-cd\n"); + uspi->s_fsize = block_size = 2048; + uspi->s_fmask = ~(2048 - 1); + uspi->s_fshift = 11; + uspi->s_sbsize = super_block_size = 2048; + uspi->s_sbbase = 0; + uspi->s_dirblksize = 1024; + flags |= UFS_DE_OLD | UFS_UID_OLD | UFS_ST_OLD | UFS_CG_OLD; + if (!sb_rdonly(sb)) { + if (!silent) + pr_info("ufstype=nextstep-cd is supported read-only\n"); + sb->s_flags |= SB_RDONLY; + } + break; + + case UFS_MOUNT_UFSTYPE_OPENSTEP: + UFSD("ufstype=openstep\n"); + uspi->s_fsize = block_size = 1024; + uspi->s_fmask = ~(1024 - 1); + uspi->s_fshift = 10; + uspi->s_sbsize = super_block_size = 2048; + uspi->s_sbbase = 0; + uspi->s_dirblksize = 1024; + flags |= UFS_DE_44BSD | UFS_UID_44BSD | UFS_ST_44BSD | UFS_CG_44BSD; + if (!sb_rdonly(sb)) { + if (!silent) + pr_info("ufstype=openstep is supported read-only\n"); + sb->s_flags |= SB_RDONLY; + } + break; + + case UFS_MOUNT_UFSTYPE_HP: + UFSD("ufstype=hp\n"); + uspi->s_fsize = block_size = 1024; + uspi->s_fmask = ~(1024 - 1); + uspi->s_fshift = 10; + uspi->s_sbsize = super_block_size = 2048; + uspi->s_sbbase = 0; + flags |= UFS_DE_OLD | UFS_UID_OLD | UFS_ST_OLD | UFS_CG_OLD; + if (!sb_rdonly(sb)) { + if (!silent) + pr_info("ufstype=hp is supported read-only\n"); + sb->s_flags |= SB_RDONLY; + } + break; + default: + if (!silent) + pr_err("unknown ufstype\n"); + goto failed; + } + +again: + if (!sb_set_blocksize(sb, block_size)) { + pr_err("failed to set blocksize\n"); + goto failed; + } + + /* + * read ufs super block from device + */ + + ubh = ubh_bread_uspi(uspi, sb, uspi->s_sbbase + super_block_offset/block_size, super_block_size); + + if (!ubh) + goto failed; + + usb1 = ubh_get_usb_first(uspi); + usb2 = ubh_get_usb_second(uspi); + usb3 = ubh_get_usb_third(uspi); + + /* Sort out mod used on SunOS 4.1.3 for fs_state */ + uspi->s_postblformat = fs32_to_cpu(sb, usb3->fs_postblformat); + if (((flags & UFS_ST_MASK) == UFS_ST_SUNOS) && + (uspi->s_postblformat != UFS_42POSTBLFMT)) { + flags &= ~UFS_ST_MASK; + flags |= UFS_ST_SUN; + } + + if ((flags & UFS_ST_MASK) == UFS_ST_44BSD && + uspi->s_postblformat == UFS_42POSTBLFMT) { + if (!silent) + pr_err("this is not a 44bsd filesystem"); + goto failed; + } + + /* + * Check ufs magic number + */ + sbi->s_bytesex = BYTESEX_LE; + switch ((uspi->fs_magic = fs32_to_cpu(sb, usb3->fs_magic))) { + case UFS_MAGIC: + case UFS_MAGIC_BW: + case UFS2_MAGIC: + case UFS_MAGIC_LFN: + case UFS_MAGIC_FEA: + case UFS_MAGIC_4GB: + goto magic_found; + } + sbi->s_bytesex = BYTESEX_BE; + switch ((uspi->fs_magic = fs32_to_cpu(sb, usb3->fs_magic))) { + case UFS_MAGIC: + case UFS_MAGIC_BW: + case UFS2_MAGIC: + case UFS_MAGIC_LFN: + case UFS_MAGIC_FEA: + case UFS_MAGIC_4GB: + goto magic_found; + } + + if ((sbi->s_flavour == UFS_MOUNT_UFSTYPE_NEXTSTEP + || sbi->s_flavour == UFS_MOUNT_UFSTYPE_NEXTSTEP_CD + || sbi->s_flavour == UFS_MOUNT_UFSTYPE_OPENSTEP) + && uspi->s_sbbase < 256) { + ubh_brelse_uspi(uspi); + ubh = NULL; + uspi->s_sbbase += 8; + goto again; + } + if (!silent) + pr_err("%s(): bad magic number\n", __func__); + goto failed; + +magic_found: + /* + * Check block and fragment sizes + */ + uspi->s_bsize = fs32_to_cpu(sb, usb1->fs_bsize); + uspi->s_fsize = fs32_to_cpu(sb, usb1->fs_fsize); + uspi->s_sbsize = fs32_to_cpu(sb, usb1->fs_sbsize); + uspi->s_fmask = fs32_to_cpu(sb, usb1->fs_fmask); + uspi->s_fshift = fs32_to_cpu(sb, usb1->fs_fshift); + + if (!is_power_of_2(uspi->s_fsize)) { + pr_err("%s(): fragment size %u is not a power of 2\n", + __func__, uspi->s_fsize); + goto failed; + } + if (uspi->s_fsize < 512) { + pr_err("%s(): fragment size %u is too small\n", + __func__, uspi->s_fsize); + goto failed; + } + if (uspi->s_fsize > 4096) { + pr_err("%s(): fragment size %u is too large\n", + __func__, uspi->s_fsize); + goto failed; + } + if (!is_power_of_2(uspi->s_bsize)) { + pr_err("%s(): block size %u is not a power of 2\n", + __func__, uspi->s_bsize); + goto failed; + } + if (uspi->s_bsize < 4096) { + pr_err("%s(): block size %u is too small\n", + __func__, uspi->s_bsize); + goto failed; + } + if (uspi->s_bsize / uspi->s_fsize > 8) { + pr_err("%s(): too many fragments per block (%u)\n", + __func__, uspi->s_bsize / uspi->s_fsize); + goto failed; + } + if (uspi->s_fsize != block_size || uspi->s_sbsize != super_block_size) { + ubh_brelse_uspi(uspi); + ubh = NULL; + block_size = uspi->s_fsize; + super_block_size = uspi->s_sbsize; + UFSD("another value of block_size or super_block_size %u, %u\n", block_size, super_block_size); + goto again; + } + + sbi->s_flags = flags;/*after that line some functions use s_flags*/ + ufs_print_super_stuff(sb, usb1, usb2, usb3); + + /* + * Check, if file system was correctly unmounted. + * If not, make it read only. + */ + if (((flags & UFS_ST_MASK) == UFS_ST_44BSD) || + ((flags & UFS_ST_MASK) == UFS_ST_OLD) || + (((flags & UFS_ST_MASK) == UFS_ST_SUN || + (flags & UFS_ST_MASK) == UFS_ST_SUNOS || + (flags & UFS_ST_MASK) == UFS_ST_SUNx86) && + (ufs_get_fs_state(sb, usb1, usb3) == (UFS_FSOK - fs32_to_cpu(sb, usb1->fs_time))))) { + switch(usb1->fs_clean) { + case UFS_FSCLEAN: + UFSD("fs is clean\n"); + break; + case UFS_FSSTABLE: + UFSD("fs is stable\n"); + break; + case UFS_FSLOG: + UFSD("fs is logging fs\n"); + break; + case UFS_FSOSF1: + UFSD("fs is DEC OSF/1\n"); + break; + case UFS_FSACTIVE: + pr_err("%s(): fs is active\n", __func__); + sb->s_flags |= SB_RDONLY; + break; + case UFS_FSBAD: + pr_err("%s(): fs is bad\n", __func__); + sb->s_flags |= SB_RDONLY; + break; + default: + pr_err("%s(): can't grok fs_clean 0x%x\n", + __func__, usb1->fs_clean); + sb->s_flags |= SB_RDONLY; + break; + } + } else { + pr_err("%s(): fs needs fsck\n", __func__); + sb->s_flags |= SB_RDONLY; + } + + /* + * Read ufs_super_block into internal data structures + */ + sb->s_op = &ufs_super_ops; + sb->s_export_op = &ufs_export_ops; + + sb->s_magic = fs32_to_cpu(sb, usb3->fs_magic); + + uspi->s_sblkno = fs32_to_cpu(sb, usb1->fs_sblkno); + uspi->s_cblkno = fs32_to_cpu(sb, usb1->fs_cblkno); + uspi->s_iblkno = fs32_to_cpu(sb, usb1->fs_iblkno); + uspi->s_dblkno = fs32_to_cpu(sb, usb1->fs_dblkno); + uspi->s_cgoffset = fs32_to_cpu(sb, usb1->fs_cgoffset); + uspi->s_cgmask = fs32_to_cpu(sb, usb1->fs_cgmask); + + if ((flags & UFS_TYPE_MASK) == UFS_TYPE_UFS2) { + uspi->s_size = fs64_to_cpu(sb, usb3->fs_un1.fs_u2.fs_size); + uspi->s_dsize = fs64_to_cpu(sb, usb3->fs_un1.fs_u2.fs_dsize); + } else { + uspi->s_size = fs32_to_cpu(sb, usb1->fs_size); + uspi->s_dsize = fs32_to_cpu(sb, usb1->fs_dsize); + } + + uspi->s_ncg = fs32_to_cpu(sb, usb1->fs_ncg); + /* s_bsize already set */ + /* s_fsize already set */ + uspi->s_fpb = fs32_to_cpu(sb, usb1->fs_frag); + uspi->s_minfree = fs32_to_cpu(sb, usb1->fs_minfree); + uspi->s_bmask = fs32_to_cpu(sb, usb1->fs_bmask); + uspi->s_fmask = fs32_to_cpu(sb, usb1->fs_fmask); + uspi->s_bshift = fs32_to_cpu(sb, usb1->fs_bshift); + uspi->s_fshift = fs32_to_cpu(sb, usb1->fs_fshift); + UFSD("uspi->s_bshift = %d,uspi->s_fshift = %d", uspi->s_bshift, + uspi->s_fshift); + uspi->s_fpbshift = fs32_to_cpu(sb, usb1->fs_fragshift); + uspi->s_fsbtodb = fs32_to_cpu(sb, usb1->fs_fsbtodb); + /* s_sbsize already set */ + uspi->s_csmask = fs32_to_cpu(sb, usb1->fs_csmask); + uspi->s_csshift = fs32_to_cpu(sb, usb1->fs_csshift); + uspi->s_nindir = fs32_to_cpu(sb, usb1->fs_nindir); + uspi->s_inopb = fs32_to_cpu(sb, usb1->fs_inopb); + uspi->s_nspf = fs32_to_cpu(sb, usb1->fs_nspf); + uspi->s_npsect = ufs_get_fs_npsect(sb, usb1, usb3); + uspi->s_interleave = fs32_to_cpu(sb, usb1->fs_interleave); + uspi->s_trackskew = fs32_to_cpu(sb, usb1->fs_trackskew); + + if (uspi->fs_magic == UFS2_MAGIC) + uspi->s_csaddr = fs64_to_cpu(sb, usb3->fs_un1.fs_u2.fs_csaddr); + else + uspi->s_csaddr = fs32_to_cpu(sb, usb1->fs_csaddr); + + uspi->s_cssize = fs32_to_cpu(sb, usb1->fs_cssize); + uspi->s_cgsize = fs32_to_cpu(sb, usb1->fs_cgsize); + uspi->s_ntrak = fs32_to_cpu(sb, usb1->fs_ntrak); + uspi->s_nsect = fs32_to_cpu(sb, usb1->fs_nsect); + uspi->s_spc = fs32_to_cpu(sb, usb1->fs_spc); + uspi->s_ipg = fs32_to_cpu(sb, usb1->fs_ipg); + uspi->s_fpg = fs32_to_cpu(sb, usb1->fs_fpg); + uspi->s_cpc = fs32_to_cpu(sb, usb2->fs_un.fs_u1.fs_cpc); + uspi->s_contigsumsize = fs32_to_cpu(sb, usb3->fs_un2.fs_44.fs_contigsumsize); + uspi->s_qbmask = ufs_get_fs_qbmask(sb, usb3); + uspi->s_qfmask = ufs_get_fs_qfmask(sb, usb3); + uspi->s_nrpos = fs32_to_cpu(sb, usb3->fs_nrpos); + uspi->s_postbloff = fs32_to_cpu(sb, usb3->fs_postbloff); + uspi->s_rotbloff = fs32_to_cpu(sb, usb3->fs_rotbloff); + + uspi->s_root_blocks = mul_u64_u32_div(uspi->s_dsize, + uspi->s_minfree, 100); + if (uspi->s_minfree <= 5) { + uspi->s_time_to_space = ~0ULL; + uspi->s_space_to_time = 0; + usb1->fs_optim = cpu_to_fs32(sb, UFS_OPTSPACE); + } else { + uspi->s_time_to_space = (uspi->s_root_blocks / 2) + 1; + uspi->s_space_to_time = mul_u64_u32_div(uspi->s_dsize, + uspi->s_minfree - 2, 100) - 1; + } + + /* + * Compute another frequently used values + */ + uspi->s_fpbmask = uspi->s_fpb - 1; + if ((flags & UFS_TYPE_MASK) == UFS_TYPE_UFS2) + uspi->s_apbshift = uspi->s_bshift - 3; + else + uspi->s_apbshift = uspi->s_bshift - 2; + + uspi->s_apb = 1 << uspi->s_apbshift; + uspi->s_apbmask = uspi->s_apb - 1; + uspi->s_nspfshift = uspi->s_fshift - UFS_SECTOR_BITS; + uspi->s_nspb = uspi->s_nspf << uspi->s_fpbshift; + uspi->s_inopf = uspi->s_inopb >> uspi->s_fpbshift; + uspi->s_bpf = uspi->s_fsize << 3; + uspi->s_bpfshift = uspi->s_fshift + 3; + uspi->s_bpfmask = uspi->s_bpf - 1; + if (sbi->s_flavour == UFS_MOUNT_UFSTYPE_44BSD || + sbi->s_flavour == UFS_MOUNT_UFSTYPE_UFS2) + uspi->s_maxsymlinklen = + fs32_to_cpu(sb, usb3->fs_un2.fs_44.fs_maxsymlinklen); + + if (uspi->fs_magic == UFS2_MAGIC) + maxsymlen = 2 * 4 * (UFS_NDADDR + UFS_NINDIR); + else + maxsymlen = 4 * (UFS_NDADDR + UFS_NINDIR); + if (uspi->s_maxsymlinklen > maxsymlen) { + ufs_warning(sb, __func__, "ufs_read_super: excessive maximum " + "fast symlink size (%u)\n", uspi->s_maxsymlinklen); + uspi->s_maxsymlinklen = maxsymlen; + } + sb->s_maxbytes = ufs_max_bytes(sb); + sb->s_max_links = UFS_LINK_MAX; + + inode = ufs_iget(sb, UFS_ROOTINO); + if (IS_ERR(inode)) { + ret = PTR_ERR(inode); + goto failed; + } + sb->s_root = d_make_root(inode); + if (!sb->s_root) { + ret = -ENOMEM; + goto failed; + } + + ufs_setup_cstotal(sb); + /* + * Read cylinder group structures + */ + if (!sb_rdonly(sb)) + if (!ufs_read_cylinder_structures(sb)) + goto failed; + + UFSD("EXIT\n"); + return 0; + +failed: + if (ubh) + ubh_brelse_uspi (uspi); + kfree (uspi); + kfree(sbi); + sb->s_fs_info = NULL; + UFSD("EXIT (FAILED)\n"); + return ret; + +failed_nomem: + UFSD("EXIT (NOMEM)\n"); + return -ENOMEM; +} + +static int ufs_reconfigure(struct fs_context *fc) +{ + struct ufs_sb_private_info * uspi; + struct ufs_super_block_first * usb1; + struct ufs_super_block_third * usb3; + struct ufs_fs_context *ctx = fc->fs_private; + struct super_block *sb = fc->root->d_sb; + unsigned int ufstype; + unsigned int flags; + + sync_filesystem(sb); + mutex_lock(&UFS_SB(sb)->s_lock); + uspi = UFS_SB(sb)->s_uspi; + flags = UFS_SB(sb)->s_flags; + usb1 = ubh_get_usb_first(uspi); + usb3 = ubh_get_usb_third(uspi); + + ufstype = UFS_SB(sb)->s_flavour; + + if ((bool)(fc->sb_flags & SB_RDONLY) == sb_rdonly(sb)) { + UFS_SB(sb)->s_on_err = ctx->on_err; + mutex_unlock(&UFS_SB(sb)->s_lock); + return 0; + } + + /* + * fs was mouted as rw, remounting ro + */ + if (fc->sb_flags & SB_RDONLY) { + ufs_put_super_internal(sb); + usb1->fs_time = ufs_get_seconds(sb); + if ((flags & UFS_ST_MASK) == UFS_ST_SUN + || (flags & UFS_ST_MASK) == UFS_ST_SUNOS + || (flags & UFS_ST_MASK) == UFS_ST_SUNx86) + ufs_set_fs_state(sb, usb1, usb3, + UFS_FSOK - fs32_to_cpu(sb, usb1->fs_time)); + ubh_mark_buffer_dirty (USPI_UBH(uspi)); + sb->s_flags |= SB_RDONLY; + } else { + /* + * fs was mounted as ro, remounting rw + */ +#ifndef CONFIG_UFS_FS_WRITE + pr_err("ufs was compiled with read-only support, can't be mounted as read-write\n"); + mutex_unlock(&UFS_SB(sb)->s_lock); + return -EINVAL; +#else + if (ufstype != UFS_MOUNT_UFSTYPE_SUN && + ufstype != UFS_MOUNT_UFSTYPE_SUNOS && + ufstype != UFS_MOUNT_UFSTYPE_44BSD && + ufstype != UFS_MOUNT_UFSTYPE_SUNx86 && + ufstype != UFS_MOUNT_UFSTYPE_UFS2) { + pr_err("this ufstype is read-only supported\n"); + mutex_unlock(&UFS_SB(sb)->s_lock); + return -EINVAL; + } + if (!ufs_read_cylinder_structures(sb)) { + pr_err("failed during remounting\n"); + mutex_unlock(&UFS_SB(sb)->s_lock); + return -EPERM; + } + sb->s_flags &= ~SB_RDONLY; +#endif + } + UFS_SB(sb)->s_on_err = ctx->on_err; + mutex_unlock(&UFS_SB(sb)->s_lock); + return 0; +} + +static int ufs_show_options(struct seq_file *seq, struct dentry *root) +{ + struct ufs_sb_info *sbi = UFS_SB(root->d_sb); + unsigned mval = sbi->s_flavour; + const struct constant_table *tp; + + tp = ufs_param_ufstype; + while (tp->value && tp->value != mval) + ++tp; + seq_printf(seq, ",ufstype=%s", tp->name); + + tp = ufs_param_onerror; + mval = sbi->s_on_err; + while (tp->value && tp->value != mval) + ++tp; + seq_printf(seq, ",onerror=%s", tp->name); + + return 0; +} + +static int ufs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + struct ufs_sb_private_info *uspi= UFS_SB(sb)->s_uspi; + unsigned flags = UFS_SB(sb)->s_flags; + u64 id = huge_encode_dev(sb->s_bdev->bd_dev); + + mutex_lock(&UFS_SB(sb)->s_lock); + + if ((flags & UFS_TYPE_MASK) == UFS_TYPE_UFS2) + buf->f_type = UFS2_MAGIC; + else + buf->f_type = UFS_MAGIC; + + buf->f_blocks = uspi->s_dsize; + buf->f_bfree = ufs_freefrags(uspi); + buf->f_ffree = uspi->cs_total.cs_nifree; + buf->f_bsize = sb->s_blocksize; + buf->f_bavail = (buf->f_bfree > uspi->s_root_blocks) + ? (buf->f_bfree - uspi->s_root_blocks) : 0; + buf->f_files = uspi->s_ncg * uspi->s_ipg; + buf->f_namelen = UFS_MAXNAMLEN; + buf->f_fsid = u64_to_fsid(id); + + mutex_unlock(&UFS_SB(sb)->s_lock); + + return 0; +} + +static struct kmem_cache * ufs_inode_cachep; + +static struct inode *ufs_alloc_inode(struct super_block *sb) +{ + struct ufs_inode_info *ei; + + ei = alloc_inode_sb(sb, ufs_inode_cachep, GFP_NOFS); + if (!ei) + return NULL; + + inode_set_iversion(&ei->vfs_inode, 1); + seqlock_init(&ei->meta_lock); + mutex_init(&ei->truncate_mutex); + return &ei->vfs_inode; +} + +static void ufs_free_in_core_inode(struct inode *inode) +{ + kmem_cache_free(ufs_inode_cachep, UFS_I(inode)); +} + +static void init_once(void *foo) +{ + struct ufs_inode_info *ei = (struct ufs_inode_info *) foo; + + inode_init_once(&ei->vfs_inode); +} + +static int __init init_inodecache(void) +{ + ufs_inode_cachep = kmem_cache_create_usercopy("ufs_inode_cache", + sizeof(struct ufs_inode_info), 0, + (SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT), + offsetof(struct ufs_inode_info, i_u1.i_symlink), + sizeof_field(struct ufs_inode_info, + i_u1.i_symlink), + init_once); + if (ufs_inode_cachep == NULL) + return -ENOMEM; + return 0; +} + +static void destroy_inodecache(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy cache. + */ + rcu_barrier(); + kmem_cache_destroy(ufs_inode_cachep); +} + +static const struct super_operations ufs_super_ops = { + .alloc_inode = ufs_alloc_inode, + .free_inode = ufs_free_in_core_inode, + .write_inode = ufs_write_inode, + .evict_inode = ufs_evict_inode, + .put_super = ufs_put_super, + .sync_fs = ufs_sync_fs, + .statfs = ufs_statfs, + .show_options = ufs_show_options, +}; + +static int ufs_get_tree(struct fs_context *fc) +{ + return get_tree_bdev(fc, ufs_fill_super); +} + +static void ufs_free_fc(struct fs_context *fc) +{ + kfree(fc->fs_private); +} + +static const struct fs_context_operations ufs_context_ops = { + .parse_param = ufs_parse_param, + .get_tree = ufs_get_tree, + .reconfigure = ufs_reconfigure, + .free = ufs_free_fc, +}; + +static int ufs_init_fs_context(struct fs_context *fc) +{ + struct ufs_fs_context *ctx; + + ctx = kzalloc_obj(*ctx); + if (!ctx) + return -ENOMEM; + + if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE) { + struct super_block *sb = fc->root->d_sb; + struct ufs_sb_info *sbi = UFS_SB(sb); + + ctx->flavour = sbi->s_flavour; + ctx->on_err = sbi->s_on_err; + } else { + ctx->flavour = 0; + ctx->on_err = UFS_MOUNT_ONERROR_LOCK; + } + + fc->fs_private = ctx; + fc->ops = &ufs_context_ops; + + return 0; +} + +static struct file_system_type ufs_fs_type = { + .owner = THIS_MODULE, + .name = "ufs", + .kill_sb = kill_block_super, + .init_fs_context = ufs_init_fs_context, + .parameters = ufs_param_spec, + .fs_flags = FS_REQUIRES_DEV, +}; +MODULE_ALIAS_FS("ufs"); + +static int __init init_ufs_fs(void) +{ + int err = init_inodecache(); + if (err) + goto out1; + err = register_filesystem(&ufs_fs_type); + if (err) + goto out; + return 0; +out: + destroy_inodecache(); +out1: + return err; +} + +static void __exit exit_ufs_fs(void) +{ + unregister_filesystem(&ufs_fs_type); + destroy_inodecache(); +} + +module_init(init_ufs_fs) +module_exit(exit_ufs_fs) +MODULE_DESCRIPTION("UFS Filesystem"); +MODULE_LICENSE("GPL"); diff --git a/fs/ufs/swab.h b/fs/ufs/swab.h new file mode 100644 index 000000000..a0e1d8c82 --- /dev/null +++ b/fs/ufs/swab.h @@ -0,0 +1,116 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * linux/fs/ufs/swab.h + * + * Copyright (C) 1997, 1998 Francois-Rene Rideau + * Copyright (C) 1998 Jakub Jelinek + * Copyright (C) 2001 Christoph Hellwig + */ + +#ifndef _UFS_SWAB_H +#define _UFS_SWAB_H + +/* + * Notes: + * HERE WE ASSUME EITHER BIG OR LITTLE ENDIAN UFSes + * in case there are ufs implementations that have strange bytesexes, + * you'll need to modify code here as well as in ufs_super.c and ufs_fs.h + * to support them. + */ + +enum { + BYTESEX_LE, + BYTESEX_BE +}; + +static inline u64 +fs64_to_cpu(struct super_block *sbp, __fs64 n) +{ + if (UFS_SB(sbp)->s_bytesex == BYTESEX_LE) + return le64_to_cpu((__force __le64)n); + else + return be64_to_cpu((__force __be64)n); +} + +static inline __fs64 +cpu_to_fs64(struct super_block *sbp, u64 n) +{ + if (UFS_SB(sbp)->s_bytesex == BYTESEX_LE) + return (__force __fs64)cpu_to_le64(n); + else + return (__force __fs64)cpu_to_be64(n); +} + +static inline u32 +fs32_to_cpu(struct super_block *sbp, __fs32 n) +{ + if (UFS_SB(sbp)->s_bytesex == BYTESEX_LE) + return le32_to_cpu((__force __le32)n); + else + return be32_to_cpu((__force __be32)n); +} + +static inline __fs32 +cpu_to_fs32(struct super_block *sbp, u32 n) +{ + if (UFS_SB(sbp)->s_bytesex == BYTESEX_LE) + return (__force __fs32)cpu_to_le32(n); + else + return (__force __fs32)cpu_to_be32(n); +} + +static inline void +fs32_add(struct super_block *sbp, __fs32 *n, int d) +{ + if (UFS_SB(sbp)->s_bytesex == BYTESEX_LE) + le32_add_cpu((__le32 *)n, d); + else + be32_add_cpu((__be32 *)n, d); +} + +static inline void +fs32_sub(struct super_block *sbp, __fs32 *n, int d) +{ + if (UFS_SB(sbp)->s_bytesex == BYTESEX_LE) + le32_add_cpu((__le32 *)n, -d); + else + be32_add_cpu((__be32 *)n, -d); +} + +static inline u16 +fs16_to_cpu(struct super_block *sbp, __fs16 n) +{ + if (UFS_SB(sbp)->s_bytesex == BYTESEX_LE) + return le16_to_cpu((__force __le16)n); + else + return be16_to_cpu((__force __be16)n); +} + +static inline __fs16 +cpu_to_fs16(struct super_block *sbp, u16 n) +{ + if (UFS_SB(sbp)->s_bytesex == BYTESEX_LE) + return (__force __fs16)cpu_to_le16(n); + else + return (__force __fs16)cpu_to_be16(n); +} + +static inline void +fs16_add(struct super_block *sbp, __fs16 *n, int d) +{ + if (UFS_SB(sbp)->s_bytesex == BYTESEX_LE) + le16_add_cpu((__le16 *)n, d); + else + be16_add_cpu((__be16 *)n, d); +} + +static inline void +fs16_sub(struct super_block *sbp, __fs16 *n, int d) +{ + if (UFS_SB(sbp)->s_bytesex == BYTESEX_LE) + le16_add_cpu((__le16 *)n, -d); + else + be16_add_cpu((__be16 *)n, -d); +} + +#endif /* _UFS_SWAB_H */ diff --git a/fs/ufs/ufs.h b/fs/ufs/ufs.h new file mode 100644 index 000000000..788e02505 --- /dev/null +++ b/fs/ufs/ufs.h @@ -0,0 +1,164 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _UFS_UFS_H +#define _UFS_UFS_H 1 + +#ifdef pr_fmt +#undef pr_fmt +#endif + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#define UFS_MAX_GROUP_LOADED 8 +#define UFS_CGNO_EMPTY ((unsigned)-1) + +struct ufs_sb_private_info; +struct ufs_cg_private_info; +struct ufs_csum; + +struct ufs_sb_info { + struct ufs_sb_private_info * s_uspi; + struct ufs_csum * s_csp; + unsigned s_bytesex; + unsigned s_flags; + struct buffer_head ** s_ucg; + struct ufs_cg_private_info * s_ucpi[UFS_MAX_GROUP_LOADED]; + unsigned s_cgno[UFS_MAX_GROUP_LOADED]; + unsigned short s_cg_loaded; + unsigned s_flavour; + unsigned s_on_err; + struct super_block *sb; + int work_queued; /* non-zero if the delayed work is queued */ + struct delayed_work sync_work; /* FS sync delayed work */ + spinlock_t work_lock; /* protects sync_work and work_queued */ + struct mutex s_lock; +}; + +struct ufs_inode_info { + union { + __fs32 i_data[15]; + __u8 i_symlink[2 * 4 * 15]; + __fs64 u2_i_data[15]; + } i_u1; + __u32 i_flags; + __u32 i_shadow; + __u32 i_unused1; + __u32 i_unused2; + __u32 i_oeftflag; + __u16 i_osync; + __u64 i_lastfrag; + seqlock_t meta_lock; + struct mutex truncate_mutex; + __u32 i_dir_start_lookup; + struct inode vfs_inode; +}; + +/* mount options */ +#define UFS_MOUNT_ONERROR_PANIC 0x00000001 +#define UFS_MOUNT_ONERROR_LOCK 0x00000002 +#define UFS_MOUNT_ONERROR_UMOUNT 0x00000004 +#define UFS_MOUNT_ONERROR_REPAIR 0x00000008 + +#define UFS_MOUNT_UFSTYPE_OLD 0x00000010 +#define UFS_MOUNT_UFSTYPE_44BSD 0x00000020 +#define UFS_MOUNT_UFSTYPE_SUN 0x00000040 +#define UFS_MOUNT_UFSTYPE_NEXTSTEP 0x00000080 +#define UFS_MOUNT_UFSTYPE_NEXTSTEP_CD 0x00000100 +#define UFS_MOUNT_UFSTYPE_OPENSTEP 0x00000200 +#define UFS_MOUNT_UFSTYPE_SUNx86 0x00000400 +#define UFS_MOUNT_UFSTYPE_HP 0x00000800 +#define UFS_MOUNT_UFSTYPE_UFS2 0x00001000 +#define UFS_MOUNT_UFSTYPE_SUNOS 0x00002000 + +/* + * Debug code + */ +#ifdef CONFIG_UFS_DEBUG +# define UFSD(f, a...) { \ + pr_debug("UFSD (%s, %d): %s:", \ + __FILE__, __LINE__, __func__); \ + pr_debug(f, ## a); \ + } +#else +# define UFSD(f, a...) /**/ +#endif + +/* balloc.c */ +void ufs_free_fragments (struct inode *, u64 fragment, unsigned count); +void ufs_free_blocks (struct inode *, u64 fragment, unsigned count); +u64 ufs_new_fragments(struct inode *, void *, u64 fragment, u64 goal, + unsigned count, int *err, struct folio *); + +/* cylinder.c */ +extern struct ufs_cg_private_info * ufs_load_cylinder (struct super_block *, unsigned); +extern void ufs_put_cylinder (struct super_block *, unsigned); + +/* dir.c */ +extern const struct inode_operations ufs_dir_inode_operations; + +int ufs_add_link(struct dentry *, struct inode *); +ino_t ufs_inode_by_name(struct inode *, const struct qstr *); +int ufs_make_empty(struct inode *, struct inode *); +struct ufs_dir_entry *ufs_find_entry(struct inode *, const struct qstr *, + struct folio **); +int ufs_delete_entry(struct inode *, struct ufs_dir_entry *, struct folio *); +int ufs_empty_dir(struct inode *); +struct ufs_dir_entry *ufs_dotdot(struct inode *, struct folio **); +int ufs_set_link(struct inode *dir, struct ufs_dir_entry *de, + struct folio *folio, struct inode *inode, bool update_times); + +/* file.c */ +extern const struct inode_operations ufs_file_inode_operations; +extern const struct file_operations ufs_file_operations; +extern const struct address_space_operations ufs_aops; + +/* ialloc.c */ +extern void ufs_free_inode (struct inode *inode); +extern struct inode * ufs_new_inode (struct inode *, umode_t); + +/* inode.c */ +extern struct inode *ufs_iget(struct super_block *, unsigned long); +extern int ufs_write_inode (struct inode *, struct writeback_control *); +extern int ufs_sync_inode (struct inode *); +extern void ufs_evict_inode (struct inode *); +extern int ufs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *attr); + +/* namei.c */ +extern const struct file_operations ufs_dir_operations; + +/* super.c */ +extern __printf(3, 4) +void ufs_warning(struct super_block *, const char *, const char *, ...); +extern __printf(3, 4) +void ufs_error(struct super_block *, const char *, const char *, ...); +extern __printf(3, 4) +void ufs_panic(struct super_block *, const char *, const char *, ...); +void ufs_mark_sb_dirty(struct super_block *sb); + +static inline struct ufs_sb_info *UFS_SB(struct super_block *sb) +{ + return sb->s_fs_info; +} + +static inline struct ufs_inode_info *UFS_I(struct inode *inode) +{ + return container_of(inode, struct ufs_inode_info, vfs_inode); +} + +/* + * Give cylinder group number for a file system block. + * Give cylinder group block number for a file system block. + */ +/* #define ufs_dtog(d) ((d) / uspi->s_fpg) */ +static inline u64 ufs_dtog(struct ufs_sb_private_info * uspi, u64 b) +{ + do_div(b, uspi->s_fpg); + return b; +} +/* #define ufs_dtogd(d) ((d) % uspi->s_fpg) */ +static inline u32 ufs_dtogd(struct ufs_sb_private_info * uspi, u64 b) +{ + return do_div(b, uspi->s_fpg); +} + +#endif /* _UFS_UFS_H */ diff --git a/fs/ufs/ufs_fs.h b/fs/ufs/ufs_fs.h new file mode 100644 index 000000000..b8dc354ae --- /dev/null +++ b/fs/ufs/ufs_fs.h @@ -0,0 +1,958 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * linux/include/linux/ufs_fs.h + * + * Copyright (C) 1996 + * Adrian Rodriguez (adrian@franklins-tower.rutgers.edu) + * Laboratory for Computer Science Research Computing Facility + * Rutgers, The State University of New Jersey + * + * Clean swab support by Fare + * just hope no one is using NNUUXXI on __?64 structure elements + * 64-bit clean thanks to Maciej W. Rozycki + * + * 4.4BSD (FreeBSD) support added on February 1st 1998 by + * Niels Kristian Bech Jensen partially based + * on code by Martin von Loewis . + * + * NeXTstep support added on February 5th 1998 by + * Niels Kristian Bech Jensen . + * + * Write support by Daniel Pirkl + * + * HP/UX hfs filesystem support added by + * Martin K. Petersen , August 1999 + * + * UFS2 (of FreeBSD 5.x) support added by + * Niraj Kumar , Jan 2004 + * + */ + +#ifndef __LINUX_UFS_FS_H +#define __LINUX_UFS_FS_H + +#include +#include +#include +#include +#include + +#include +typedef __u64 __bitwise __fs64; +typedef __u32 __bitwise __fs32; +typedef __u16 __bitwise __fs16; + +#define UFS_BBLOCK 0 +#define UFS_BBSIZE 8192 +#define UFS_SBLOCK 8192 +#define UFS_SBSIZE 8192 + +#define UFS_SECTOR_SIZE 512 +#define UFS_SECTOR_BITS 9 +#define UFS_MAGIC 0x00011954 +#define UFS_MAGIC_BW 0x0f242697 +#define UFS2_MAGIC 0x19540119 +#define UFS_CIGAM 0x54190100 /* byteswapped MAGIC */ + +/* Copied from FreeBSD */ +/* + * Each disk drive contains some number of filesystems. + * A filesystem consists of a number of cylinder groups. + * Each cylinder group has inodes and data. + * + * A filesystem is described by its super-block, which in turn + * describes the cylinder groups. The super-block is critical + * data and is replicated in each cylinder group to protect against + * catastrophic loss. This is done at `newfs' time and the critical + * super-block data does not change, so the copies need not be + * referenced further unless disaster strikes. + * + * For filesystem fs, the offsets of the various blocks of interest + * are given in the super block as: + * [fs->fs_sblkno] Super-block + * [fs->fs_cblkno] Cylinder group block + * [fs->fs_iblkno] Inode blocks + * [fs->fs_dblkno] Data blocks + * The beginning of cylinder group cg in fs, is given by + * the ``cgbase(fs, cg)'' macro. + * + * Depending on the architecture and the media, the superblock may + * reside in any one of four places. For tiny media where every block + * counts, it is placed at the very front of the partition. Historically, + * UFS1 placed it 8K from the front to leave room for the disk label and + * a small bootstrap. For UFS2 it got moved to 64K from the front to leave + * room for the disk label and a bigger bootstrap, and for really piggy + * systems we check at 256K from the front if the first three fail. In + * all cases the size of the superblock will be SBLOCKSIZE. All values are + * given in byte-offset form, so they do not imply a sector size. The + * SBLOCKSEARCH specifies the order in which the locations should be searched. + */ +#define SBLOCK_FLOPPY 0 +#define SBLOCK_UFS1 8192 +#define SBLOCK_UFS2 65536 +#define SBLOCK_PIGGY 262144 +#define SBLOCKSIZE 8192 +#define SBLOCKSEARCH \ + { SBLOCK_UFS2, SBLOCK_UFS1, SBLOCK_FLOPPY, SBLOCK_PIGGY, -1 } + + +/* HP specific MAGIC values */ + +#define UFS_MAGIC_LFN 0x00095014 /* fs supports filenames > 14 chars */ +#define UFS_CIGAM_LFN 0x14500900 /* srahc 41 < semanelif stroppus sf */ + +#define UFS_MAGIC_SEC 0x00612195 /* B1 security fs */ +#define UFS_CIGAM_SEC 0x95216100 + +#define UFS_MAGIC_FEA 0x00195612 /* fs_featurebits supported */ +#define UFS_CIGAM_FEA 0x12561900 + +#define UFS_MAGIC_4GB 0x05231994 /* fs > 4 GB && fs_featurebits */ +#define UFS_CIGAM_4GB 0x94192305 + +/* Seems somebody at HP goofed here. B1 and lfs are both 0x2 !?! */ +#define UFS_FSF_LFN 0x00000001 /* long file names */ +#define UFS_FSF_B1 0x00000002 /* B1 security */ +#define UFS_FSF_LFS 0x00000002 /* large files */ +#define UFS_FSF_LUID 0x00000004 /* large UIDs */ + +/* End of HP stuff */ + + +#define UFS_BSIZE 8192 +#define UFS_MINBSIZE 4096 +#define UFS_FSIZE 1024 +#define UFS_MAXFRAG (UFS_BSIZE / UFS_FSIZE) + +#define UFS_NDADDR 12 +#define UFS_NINDIR 3 + +#define UFS_IND_BLOCK (UFS_NDADDR + 0) +#define UFS_DIND_BLOCK (UFS_NDADDR + 1) +#define UFS_TIND_BLOCK (UFS_NDADDR + 2) + +#define UFS_NDIR_FRAGMENT (UFS_NDADDR << uspi->s_fpbshift) +#define UFS_IND_FRAGMENT (UFS_IND_BLOCK << uspi->s_fpbshift) +#define UFS_DIND_FRAGMENT (UFS_DIND_BLOCK << uspi->s_fpbshift) +#define UFS_TIND_FRAGMENT (UFS_TIND_BLOCK << uspi->s_fpbshift) + +#define UFS_ROOTINO 2 +#define UFS_FIRST_INO (UFS_ROOTINO + 1) + +#define UFS_USEEFT ((__u16)65535) + +/* fs_clean values */ +#define UFS_FSOK 0x7c269d38 +#define UFS_FSACTIVE ((__s8)0x00) +#define UFS_FSCLEAN ((__s8)0x01) +#define UFS_FSSTABLE ((__s8)0x02) +#define UFS_FSOSF1 ((__s8)0x03) /* is this correct for DEC OSF/1? */ +#define UFS_FSBAD ((__s8)0xff) + +/* Solaris-specific fs_clean values */ +#define UFS_FSSUSPEND ((__s8)0xfe) /* temporarily suspended */ +#define UFS_FSLOG ((__s8)0xfd) /* logging fs */ +#define UFS_FSFIX ((__s8)0xfc) /* being repaired while mounted */ + +/* From here to next blank line, s_flags for ufs_sb_info */ +/* directory entry encoding */ +#define UFS_DE_MASK 0x00000010 /* mask for the following */ +#define UFS_DE_OLD 0x00000000 +#define UFS_DE_44BSD 0x00000010 +/* uid encoding */ +#define UFS_UID_MASK 0x00000060 /* mask for the following */ +#define UFS_UID_OLD 0x00000000 +#define UFS_UID_44BSD 0x00000020 +#define UFS_UID_EFT 0x00000040 +/* superblock state encoding */ +#define UFS_ST_MASK 0x00000700 /* mask for the following */ +#define UFS_ST_OLD 0x00000000 +#define UFS_ST_44BSD 0x00000100 +#define UFS_ST_SUN 0x00000200 /* Solaris */ +#define UFS_ST_SUNOS 0x00000300 +#define UFS_ST_SUNx86 0x00000400 /* Solaris x86 */ +/*cylinder group encoding */ +#define UFS_CG_MASK 0x00003000 /* mask for the following */ +#define UFS_CG_OLD 0x00000000 +#define UFS_CG_44BSD 0x00002000 +#define UFS_CG_SUN 0x00001000 +/* filesystem type encoding */ +#define UFS_TYPE_MASK 0x00010000 /* mask for the following */ +#define UFS_TYPE_UFS1 0x00000000 +#define UFS_TYPE_UFS2 0x00010000 + + +/* fs_inodefmt options */ +#define UFS_42INODEFMT -1 +#define UFS_44INODEFMT 2 + +/* + * MINFREE gives the minimum acceptable percentage of file system + * blocks which may be free. If the freelist drops below this level + * only the superuser may continue to allocate blocks. This may + * be set to 0 if no reserve of free blocks is deemed necessary, + * however throughput drops by fifty percent if the file system + * is run at between 95% and 100% full; thus the minimum default + * value of fs_minfree is 5%. However, to get good clustering + * performance, 10% is a better choice. hence we use 10% as our + * default value. With 10% free space, fragmentation is not a + * problem, so we choose to optimize for time. + */ +#define UFS_MINFREE 5 +#define UFS_DEFAULTOPT UFS_OPTTIME + +/* + * Turn file system block numbers into disk block addresses. + * This maps file system blocks to device size blocks. + */ +#define ufs_fsbtodb(uspi, b) ((b) << (uspi)->s_fsbtodb) +#define ufs_dbtofsb(uspi, b) ((b) >> (uspi)->s_fsbtodb) + +/* + * Cylinder group macros to locate things in cylinder groups. + * They calc file system addresses of cylinder group data structures. + */ +#define ufs_cgbase(c) (uspi->s_fpg * (c)) +#define ufs_cgstart(c) ((uspi)->fs_magic == UFS2_MAGIC ? ufs_cgbase(c) : \ + (ufs_cgbase(c) + uspi->s_cgoffset * ((c) & ~uspi->s_cgmask))) +#define ufs_cgsblock(c) (ufs_cgstart(c) + uspi->s_sblkno) /* super blk */ +#define ufs_cgcmin(c) (ufs_cgstart(c) + uspi->s_cblkno) /* cg block */ +#define ufs_cgimin(c) (ufs_cgstart(c) + uspi->s_iblkno) /* inode blk */ +#define ufs_cgdmin(c) (ufs_cgstart(c) + uspi->s_dblkno) /* 1st data */ + +/* + * Macros for handling inode numbers: + * inode number to file system block offset. + * inode number to cylinder group number. + * inode number to file system block address. + */ +#define ufs_inotocg(x) ((unsigned int)(x) / uspi->s_ipg) +#define ufs_inotocgoff(x) ((unsigned int)(x) % uspi->s_ipg) +#define ufs_inotofsba(x) (((u64)ufs_cgimin(ufs_inotocg(x))) + ufs_inotocgoff(x) / uspi->s_inopf) +#define ufs_inotofsbo(x) ((unsigned int)(x) % uspi->s_inopf) + +/* + * Compute the cylinder and rotational position of a cyl block addr. + */ +#define ufs_cbtocylno(bno) \ + ((bno) * uspi->s_nspf / uspi->s_spc) +#define ufs_cbtorpos(bno) \ + ((UFS_SB(sb)->s_flags & UFS_CG_SUN) ? \ + (((((bno) * uspi->s_nspf % uspi->s_spc) % \ + uspi->s_nsect) * \ + uspi->s_nrpos) / uspi->s_nsect) \ + : \ + ((((bno) * uspi->s_nspf % uspi->s_spc / uspi->s_nsect \ + * uspi->s_trackskew + (bno) * uspi->s_nspf % uspi->s_spc \ + % uspi->s_nsect * uspi->s_interleave) % uspi->s_nsect \ + * uspi->s_nrpos) / uspi->s_npsect)) + +/* + * The following macros optimize certain frequently calculated + * quantities by using shifts and masks in place of divisions + * modulos and multiplications. + */ +#define ufs_blkoff(loc) ((loc) & uspi->s_qbmask) +#define ufs_fragoff(loc) ((loc) & uspi->s_qfmask) +#define ufs_lblktosize(blk) ((blk) << uspi->s_bshift) +#define ufs_lblkno(loc) ((loc) >> uspi->s_bshift) +#define ufs_numfrags(loc) ((loc) >> uspi->s_fshift) +#define ufs_blkroundup(size) (((size) + uspi->s_qbmask) & uspi->s_bmask) +#define ufs_fragroundup(size) (((size) + uspi->s_qfmask) & uspi->s_fmask) +#define ufs_fragstoblks(frags) ((frags) >> uspi->s_fpbshift) +#define ufs_blkstofrags(blks) ((blks) << uspi->s_fpbshift) +#define ufs_fragnum(fsb) ((fsb) & uspi->s_fpbmask) +#define ufs_blknum(fsb) ((fsb) & ~uspi->s_fpbmask) + +#define UFS_MAXNAMLEN 255 +#define UFS_MAXMNTLEN 512 +#define UFS2_MAXMNTLEN 468 +#define UFS2_MAXVOLLEN 32 +#define UFS_MAXCSBUFS 31 +#define UFS_LINK_MAX 32000 +/* +#define UFS2_NOCSPTRS ((128 / sizeof(void *)) - 4) +*/ +#define UFS2_NOCSPTRS 28 + +/* + * UFS_DIR_PAD defines the directory entries boundaries + * (must be a multiple of 4) + */ +#define UFS_DIR_PAD 4 +#define UFS_DIR_ROUND (UFS_DIR_PAD - 1) +#define UFS_DIR_REC_LEN(name_len) (((name_len) + 1 + 8 + UFS_DIR_ROUND) & ~UFS_DIR_ROUND) + +struct ufs_timeval { + __fs32 tv_sec; + __fs32 tv_usec; +}; + +struct ufs_dir_entry { + __fs32 d_ino; /* inode number of this entry */ + __fs16 d_reclen; /* length of this entry */ + union { + __fs16 d_namlen; /* actual length of d_name */ + struct { + __u8 d_type; /* file type */ + __u8 d_namlen; /* length of string in d_name */ + } d_44; + } d_u; + __u8 d_name[UFS_MAXNAMLEN + 1]; /* file name */ +}; + +struct ufs_csum { + __fs32 cs_ndir; /* number of directories */ + __fs32 cs_nbfree; /* number of free blocks */ + __fs32 cs_nifree; /* number of free inodes */ + __fs32 cs_nffree; /* number of free frags */ +}; +struct ufs2_csum_total { + __fs64 cs_ndir; /* number of directories */ + __fs64 cs_nbfree; /* number of free blocks */ + __fs64 cs_nifree; /* number of free inodes */ + __fs64 cs_nffree; /* number of free frags */ + __fs64 cs_numclusters; /* number of free clusters */ + __fs64 cs_spare[3]; /* future expansion */ +}; + +struct ufs_csum_core { + __u64 cs_ndir; /* number of directories */ + __u64 cs_nbfree; /* number of free blocks */ + __u64 cs_nifree; /* number of free inodes */ + __u64 cs_nffree; /* number of free frags */ + __u64 cs_numclusters; /* number of free clusters */ +}; + +/* + * File system flags + */ +#define UFS_UNCLEAN 0x01 /* file system not clean at mount (unused) */ +#define UFS_DOSOFTDEP 0x02 /* file system using soft dependencies */ +#define UFS_NEEDSFSCK 0x04 /* needs sync fsck (FreeBSD compat, unused) */ +#define UFS_INDEXDIRS 0x08 /* kernel supports indexed directories */ +#define UFS_ACLS 0x10 /* file system has ACLs enabled */ +#define UFS_MULTILABEL 0x20 /* file system is MAC multi-label */ +#define UFS_FLAGS_UPDATED 0x80 /* flags have been moved to new location */ + +#if 0 +/* + * This is the actual superblock, as it is laid out on the disk. + * Do NOT use this structure, because of sizeof(ufs_super_block) > 512 and + * it may occupy several blocks, use + * struct ufs_super_block_(first,second,third) instead. + */ +struct ufs_super_block { + union { + struct { + __fs32 fs_link; /* UNUSED */ + } fs_42; + struct { + __fs32 fs_state; /* file system state flag */ + } fs_sun; + } fs_u0; + __fs32 fs_rlink; /* UNUSED */ + __fs32 fs_sblkno; /* addr of super-block in filesys */ + __fs32 fs_cblkno; /* offset of cyl-block in filesys */ + __fs32 fs_iblkno; /* offset of inode-blocks in filesys */ + __fs32 fs_dblkno; /* offset of first data after cg */ + __fs32 fs_cgoffset; /* cylinder group offset in cylinder */ + __fs32 fs_cgmask; /* used to calc mod fs_ntrak */ + __fs32 fs_time; /* last time written -- time_t */ + __fs32 fs_size; /* number of blocks in fs */ + __fs32 fs_dsize; /* number of data blocks in fs */ + __fs32 fs_ncg; /* number of cylinder groups */ + __fs32 fs_bsize; /* size of basic blocks in fs */ + __fs32 fs_fsize; /* size of frag blocks in fs */ + __fs32 fs_frag; /* number of frags in a block in fs */ +/* these are configuration parameters */ + __fs32 fs_minfree; /* minimum percentage of free blocks */ + __fs32 fs_rotdelay; /* num of ms for optimal next block */ + __fs32 fs_rps; /* disk revolutions per second */ +/* these fields can be computed from the others */ + __fs32 fs_bmask; /* ``blkoff'' calc of blk offsets */ + __fs32 fs_fmask; /* ``fragoff'' calc of frag offsets */ + __fs32 fs_bshift; /* ``lblkno'' calc of logical blkno */ + __fs32 fs_fshift; /* ``numfrags'' calc number of frags */ +/* these are configuration parameters */ + __fs32 fs_maxcontig; /* max number of contiguous blks */ + __fs32 fs_maxbpg; /* max number of blks per cyl group */ +/* these fields can be computed from the others */ + __fs32 fs_fragshift; /* block to frag shift */ + __fs32 fs_fsbtodb; /* fsbtodb and dbtofsb shift constant */ + __fs32 fs_sbsize; /* actual size of super block */ + __fs32 fs_csmask; /* csum block offset */ + __fs32 fs_csshift; /* csum block number */ + __fs32 fs_nindir; /* value of NINDIR */ + __fs32 fs_inopb; /* value of INOPB */ + __fs32 fs_nspf; /* value of NSPF */ +/* yet another configuration parameter */ + __fs32 fs_optim; /* optimization preference, see below */ +/* these fields are derived from the hardware */ + union { + struct { + __fs32 fs_npsect; /* # sectors/track including spares */ + } fs_sun; + struct { + __fs32 fs_state; /* file system state time stamp */ + } fs_sunx86; + } fs_u1; + __fs32 fs_interleave; /* hardware sector interleave */ + __fs32 fs_trackskew; /* sector 0 skew, per track */ +/* a unique id for this filesystem (currently unused and unmaintained) */ +/* In 4.3 Tahoe this space is used by fs_headswitch and fs_trkseek */ +/* Neither of those fields is used in the Tahoe code right now but */ +/* there could be problems if they are. */ + __fs32 fs_id[2]; /* file system id */ +/* sizes determined by number of cylinder groups and their sizes */ + __fs32 fs_csaddr; /* blk addr of cyl grp summary area */ + __fs32 fs_cssize; /* size of cyl grp summary area */ + __fs32 fs_cgsize; /* cylinder group size */ +/* these fields are derived from the hardware */ + __fs32 fs_ntrak; /* tracks per cylinder */ + __fs32 fs_nsect; /* sectors per track */ + __fs32 fs_spc; /* sectors per cylinder */ +/* this comes from the disk driver partitioning */ + __fs32 fs_ncyl; /* cylinders in file system */ +/* these fields can be computed from the others */ + __fs32 fs_cpg; /* cylinders per group */ + __fs32 fs_ipg; /* inodes per cylinder group */ + __fs32 fs_fpg; /* blocks per group * fs_frag */ +/* this data must be re-computed after crashes */ + struct ufs_csum fs_cstotal; /* cylinder summary information */ +/* these fields are cleared at mount time */ + __s8 fs_fmod; /* super block modified flag */ + __s8 fs_clean; /* file system is clean flag */ + __s8 fs_ronly; /* mounted read-only flag */ + __s8 fs_flags; + union { + struct { + __s8 fs_fsmnt[UFS_MAXMNTLEN];/* name mounted on */ + __fs32 fs_cgrotor; /* last cg searched */ + __fs32 fs_csp[UFS_MAXCSBUFS];/*list of fs_cs info buffers */ + __fs32 fs_maxcluster; + __fs32 fs_cpc; /* cyl per cycle in postbl */ + __fs16 fs_opostbl[16][8]; /* old rotation block list head */ + } fs_u1; + struct { + __s8 fs_fsmnt[UFS2_MAXMNTLEN]; /* name mounted on */ + __u8 fs_volname[UFS2_MAXVOLLEN]; /* volume name */ + __fs64 fs_swuid; /* system-wide uid */ + __fs32 fs_pad; /* due to alignment of fs_swuid */ + __fs32 fs_cgrotor; /* last cg searched */ + __fs32 fs_ocsp[UFS2_NOCSPTRS]; /*list of fs_cs info buffers */ + __fs32 fs_contigdirs;/*# of contiguously allocated dirs */ + __fs32 fs_csp; /* cg summary info buffer for fs_cs */ + __fs32 fs_maxcluster; + __fs32 fs_active;/* used by snapshots to track fs */ + __fs32 fs_old_cpc; /* cyl per cycle in postbl */ + __fs32 fs_maxbsize;/*maximum blocking factor permitted */ + __fs64 fs_sparecon64[17];/*old rotation block list head */ + __fs64 fs_sblockloc; /* byte offset of standard superblock */ + struct ufs2_csum_total fs_cstotal;/*cylinder summary information*/ + struct ufs_timeval fs_time; /* last time written */ + __fs64 fs_size; /* number of blocks in fs */ + __fs64 fs_dsize; /* number of data blocks in fs */ + __fs64 fs_csaddr; /* blk addr of cyl grp summary area */ + __fs64 fs_pendingblocks;/* blocks in process of being freed */ + __fs32 fs_pendinginodes;/*inodes in process of being freed */ + } fs_u2; + } fs_u11; + union { + struct { + __fs32 fs_sparecon[53];/* reserved for future constants */ + __fs32 fs_reclaim; + __fs32 fs_sparecon2[1]; + __fs32 fs_state; /* file system state time stamp */ + __fs32 fs_qbmask[2]; /* ~usb_bmask */ + __fs32 fs_qfmask[2]; /* ~usb_fmask */ + } fs_sun; + struct { + __fs32 fs_sparecon[53];/* reserved for future constants */ + __fs32 fs_reclaim; + __fs32 fs_sparecon2[1]; + __fs32 fs_npsect; /* # sectors/track including spares */ + __fs32 fs_qbmask[2]; /* ~usb_bmask */ + __fs32 fs_qfmask[2]; /* ~usb_fmask */ + } fs_sunx86; + struct { + __fs32 fs_sparecon[50];/* reserved for future constants */ + __fs32 fs_contigsumsize;/* size of cluster summary array */ + __fs32 fs_maxsymlinklen;/* max length of an internal symlink */ + __fs32 fs_inodefmt; /* format of on-disk inodes */ + __fs32 fs_maxfilesize[2]; /* max representable file size */ + __fs32 fs_qbmask[2]; /* ~usb_bmask */ + __fs32 fs_qfmask[2]; /* ~usb_fmask */ + __fs32 fs_state; /* file system state time stamp */ + } fs_44; + } fs_u2; + __fs32 fs_postblformat; /* format of positional layout tables */ + __fs32 fs_nrpos; /* number of rotational positions */ + __fs32 fs_postbloff; /* (__s16) rotation block list head */ + __fs32 fs_rotbloff; /* (__u8) blocks for each rotation */ + __fs32 fs_magic; /* magic number */ + __u8 fs_space[1]; /* list of blocks for each rotation */ +}; +#endif/*struct ufs_super_block*/ + +/* + * Preference for optimization. + */ +#define UFS_OPTTIME 0 /* minimize allocation time */ +#define UFS_OPTSPACE 1 /* minimize disk fragmentation */ + +/* + * Rotational layout table format types + */ +#define UFS_42POSTBLFMT -1 /* 4.2BSD rotational table format */ +#define UFS_DYNAMICPOSTBLFMT 1 /* dynamic rotational table format */ + +/* + * Convert cylinder group to base address of its global summary info. + */ +#define fs_cs(indx) s_csp[(indx)] + +/* + * Cylinder group block for a file system. + * + * Writable fields in the cylinder group are protected by the associated + * super block lock fs->fs_lock. + */ +#define CG_MAGIC 0x090255 +#define ufs_cg_chkmagic(sb, ucg) \ + (fs32_to_cpu((sb), (ucg)->cg_magic) == CG_MAGIC) +/* + * Macros for access to old cylinder group array structures + */ +#define ufs_ocg_blktot(sb, ucg) fs32_to_cpu((sb), ((struct ufs_old_cylinder_group *)(ucg))->cg_btot) +#define ufs_ocg_blks(sb, ucg, cylno) fs32_to_cpu((sb), ((struct ufs_old_cylinder_group *)(ucg))->cg_b[cylno]) +#define ufs_ocg_inosused(sb, ucg) fs32_to_cpu((sb), ((struct ufs_old_cylinder_group *)(ucg))->cg_iused) +#define ufs_ocg_blksfree(sb, ucg) fs32_to_cpu((sb), ((struct ufs_old_cylinder_group *)(ucg))->cg_free) +#define ufs_ocg_chkmagic(sb, ucg) \ + (fs32_to_cpu((sb), ((struct ufs_old_cylinder_group *)(ucg))->cg_magic) == CG_MAGIC) + +/* + * size of this structure is 172 B + */ +struct ufs_cylinder_group { + __fs32 cg_link; /* linked list of cyl groups */ + __fs32 cg_magic; /* magic number */ + __fs32 cg_time; /* time last written */ + __fs32 cg_cgx; /* we are the cgx'th cylinder group */ + __fs16 cg_ncyl; /* number of cyl's this cg */ + __fs16 cg_niblk; /* number of inode blocks this cg */ + __fs32 cg_ndblk; /* number of data blocks this cg */ + struct ufs_csum cg_cs; /* cylinder summary information */ + __fs32 cg_rotor; /* position of last used block */ + __fs32 cg_frotor; /* position of last used frag */ + __fs32 cg_irotor; /* position of last used inode */ + __fs32 cg_frsum[UFS_MAXFRAG]; /* counts of available frags */ + __fs32 cg_btotoff; /* (__u32) block totals per cylinder */ + __fs32 cg_boff; /* (short) free block positions */ + __fs32 cg_iusedoff; /* (char) used inode map */ + __fs32 cg_freeoff; /* (u_char) free block map */ + __fs32 cg_nextfreeoff; /* (u_char) next available space */ + union { + struct { + __fs32 cg_clustersumoff; /* (u_int32) counts of avail clusters */ + __fs32 cg_clusteroff; /* (u_int8) free cluster map */ + __fs32 cg_nclusterblks; /* number of clusters this cg */ + __fs32 cg_sparecon[13]; /* reserved for future use */ + } cg_44; + struct { + __fs32 cg_clustersumoff;/* (u_int32) counts of avail clusters */ + __fs32 cg_clusteroff; /* (u_int8) free cluster map */ + __fs32 cg_nclusterblks;/* number of clusters this cg */ + __fs32 cg_niblk; /* number of inode blocks this cg */ + __fs32 cg_initediblk; /* last initialized inode */ + __fs32 cg_sparecon32[3];/* reserved for future use */ + __fs64 cg_time; /* time last written */ + __fs64 cg_sparecon[3]; /* reserved for future use */ + } cg_u2; + __fs32 cg_sparecon[16]; /* reserved for future use */ + } cg_u; + __u8 cg_space[1]; /* space for cylinder group maps */ +/* actually longer */ +}; + +/* Historic Cylinder group info */ +struct ufs_old_cylinder_group { + __fs32 cg_link; /* linked list of cyl groups */ + __fs32 cg_rlink; /* for incore cyl groups */ + __fs32 cg_time; /* time last written */ + __fs32 cg_cgx; /* we are the cgx'th cylinder group */ + __fs16 cg_ncyl; /* number of cyl's this cg */ + __fs16 cg_niblk; /* number of inode blocks this cg */ + __fs32 cg_ndblk; /* number of data blocks this cg */ + struct ufs_csum cg_cs; /* cylinder summary information */ + __fs32 cg_rotor; /* position of last used block */ + __fs32 cg_frotor; /* position of last used frag */ + __fs32 cg_irotor; /* position of last used inode */ + __fs32 cg_frsum[8]; /* counts of available frags */ + __fs32 cg_btot[32]; /* block totals per cylinder */ + __fs16 cg_b[32][8]; /* positions of free blocks */ + __u8 cg_iused[256]; /* used inode map */ + __fs32 cg_magic; /* magic number */ + __u8 cg_free[1]; /* free block map */ +/* actually longer */ +}; + +/* + * structure of an on-disk inode + */ +struct ufs_inode { + __fs16 ui_mode; /* 0x0 */ + __fs16 ui_nlink; /* 0x2 */ + union { + struct { + __fs16 ui_suid; /* 0x4 */ + __fs16 ui_sgid; /* 0x6 */ + } oldids; + __fs32 ui_inumber; /* 0x4 lsf: inode number */ + __fs32 ui_author; /* 0x4 GNU HURD: author */ + } ui_u1; + __fs64 ui_size; /* 0x8 */ + struct ufs_timeval ui_atime; /* 0x10 access */ + struct ufs_timeval ui_mtime; /* 0x18 modification */ + struct ufs_timeval ui_ctime; /* 0x20 creation */ + union { + struct { + __fs32 ui_db[UFS_NDADDR];/* 0x28 data blocks */ + __fs32 ui_ib[UFS_NINDIR];/* 0x58 indirect blocks */ + } ui_addr; + __u8 ui_symlink[4*(UFS_NDADDR+UFS_NINDIR)];/* 0x28 fast symlink */ + } ui_u2; + __fs32 ui_flags; /* 0x64 immutable, append-only... */ + __fs32 ui_blocks; /* 0x68 blocks in use */ + __fs32 ui_gen; /* 0x6c like ext2 i_version, for NFS support */ + union { + struct { + __fs32 ui_shadow; /* 0x70 shadow inode with security data */ + __fs32 ui_uid; /* 0x74 long EFT version of uid */ + __fs32 ui_gid; /* 0x78 long EFT version of gid */ + __fs32 ui_oeftflag; /* 0x7c reserved */ + } ui_sun; + struct { + __fs32 ui_uid; /* 0x70 File owner */ + __fs32 ui_gid; /* 0x74 File group */ + __fs32 ui_spare[2]; /* 0x78 reserved */ + } ui_44; + struct { + __fs32 ui_uid; /* 0x70 */ + __fs32 ui_gid; /* 0x74 */ + __fs16 ui_modeh; /* 0x78 mode high bits */ + __fs16 ui_spare; /* 0x7A unused */ + __fs32 ui_trans; /* 0x7c filesystem translator */ + } ui_hurd; + } ui_u3; +}; + +#define UFS_NXADDR 2 /* External addresses in inode. */ +struct ufs2_inode { + __fs16 ui_mode; /* 0: IFMT, permissions; see below. */ + __fs16 ui_nlink; /* 2: File link count. */ + __fs32 ui_uid; /* 4: File owner. */ + __fs32 ui_gid; /* 8: File group. */ + __fs32 ui_blksize; /* 12: Inode blocksize. */ + __fs64 ui_size; /* 16: File byte count. */ + __fs64 ui_blocks; /* 24: Bytes actually held. */ + __fs64 ui_atime; /* 32: Last access time. */ + __fs64 ui_mtime; /* 40: Last modified time. */ + __fs64 ui_ctime; /* 48: Last inode change time. */ + __fs64 ui_birthtime; /* 56: Inode creation time. */ + __fs32 ui_mtimensec; /* 64: Last modified time. */ + __fs32 ui_atimensec; /* 68: Last access time. */ + __fs32 ui_ctimensec; /* 72: Last inode change time. */ + __fs32 ui_birthnsec; /* 76: Inode creation time. */ + __fs32 ui_gen; /* 80: Generation number. */ + __fs32 ui_kernflags; /* 84: Kernel flags. */ + __fs32 ui_flags; /* 88: Status flags (chflags). */ + __fs32 ui_extsize; /* 92: External attributes block. */ + __fs64 ui_extb[UFS_NXADDR];/* 96: External attributes block. */ + union { + struct { + __fs64 ui_db[UFS_NDADDR]; /* 112: Direct disk blocks. */ + __fs64 ui_ib[UFS_NINDIR];/* 208: Indirect disk blocks.*/ + } ui_addr; + __u8 ui_symlink[2*4*(UFS_NDADDR+UFS_NINDIR)];/* 0x28 fast symlink */ + } ui_u2; + __fs64 ui_spare[3]; /* 232: Reserved; currently unused */ +}; + + +/* FreeBSD has these in sys/stat.h */ +/* ui_flags that can be set by a file owner */ +#define UFS_UF_SETTABLE 0x0000ffff +#define UFS_UF_NODUMP 0x00000001 /* do not dump */ +#define UFS_UF_IMMUTABLE 0x00000002 /* immutable (can't "change") */ +#define UFS_UF_APPEND 0x00000004 /* append-only */ +#define UFS_UF_OPAQUE 0x00000008 /* directory is opaque (unionfs) */ +#define UFS_UF_NOUNLINK 0x00000010 /* can't be removed or renamed */ +/* ui_flags that only root can set */ +#define UFS_SF_SETTABLE 0xffff0000 +#define UFS_SF_ARCHIVED 0x00010000 /* archived */ +#define UFS_SF_IMMUTABLE 0x00020000 /* immutable (can't "change") */ +#define UFS_SF_APPEND 0x00040000 /* append-only */ +#define UFS_SF_NOUNLINK 0x00100000 /* can't be removed or renamed */ + +/* + * This structure is used for reading disk structures larger + * than the size of fragment. + */ +struct ufs_buffer_head { + __u64 fragment; /* first fragment */ + __u64 count; /* number of fragments */ + struct buffer_head * bh[UFS_MAXFRAG]; /* buffers */ +}; + +struct ufs_cg_private_info { + struct ufs_buffer_head c_ubh; + __u32 c_cgx; /* number of cylidner group */ + __u16 c_ncyl; /* number of cyl's this cg */ + __u16 c_niblk; /* number of inode blocks this cg */ + __u32 c_ndblk; /* number of data blocks this cg */ + __u32 c_rotor; /* position of last used block */ + __u32 c_frotor; /* position of last used frag */ + __u32 c_irotor; /* position of last used inode */ + __u32 c_btotoff; /* (__u32) block totals per cylinder */ + __u32 c_boff; /* (short) free block positions */ + __u32 c_iusedoff; /* (char) used inode map */ + __u32 c_freeoff; /* (u_char) free block map */ + __u32 c_nextfreeoff; /* (u_char) next available space */ + __u32 c_clustersumoff;/* (u_int32) counts of avail clusters */ + __u32 c_clusteroff; /* (u_int8) free cluster map */ + __u32 c_nclusterblks; /* number of clusters this cg */ +}; + + +struct ufs_sb_private_info { + struct ufs_buffer_head s_ubh; /* buffer containing super block */ + struct ufs_csum_core cs_total; + __u32 s_sblkno; /* offset of super-blocks in filesys */ + __u32 s_cblkno; /* offset of cg-block in filesys */ + __u32 s_iblkno; /* offset of inode-blocks in filesys */ + __u32 s_dblkno; /* offset of first data after cg */ + __u32 s_cgoffset; /* cylinder group offset in cylinder */ + __u32 s_cgmask; /* used to calc mod fs_ntrak */ + __u64 s_size; /* number of blocks (fragments) in fs */ + __u64 s_dsize; /* number of data blocks in fs */ + __u32 s_ncg; /* number of cylinder groups */ + __u32 s_bsize; /* size of basic blocks */ + __u32 s_fsize; /* size of fragments */ + __u32 s_fpb; /* fragments per block */ + __u32 s_minfree; /* minimum percentage of free blocks */ + __u32 s_bmask; /* `blkoff'' calc of blk offsets */ + __u32 s_fmask; /* s_fsize mask */ + __u32 s_bshift; /* `lblkno'' calc of logical blkno */ + __u32 s_fshift; /* s_fsize shift */ + __u32 s_fpbshift; /* fragments per block shift */ + __u32 s_fsbtodb; /* fsbtodb and dbtofsb shift constant */ + __u32 s_sbsize; /* actual size of super block */ + __u32 s_csmask; /* csum block offset */ + __u32 s_csshift; /* csum block number */ + __u32 s_nindir; /* value of NINDIR */ + __u32 s_inopb; /* value of INOPB */ + __u32 s_nspf; /* value of NSPF */ + __u32 s_npsect; /* # sectors/track including spares */ + __u32 s_interleave; /* hardware sector interleave */ + __u32 s_trackskew; /* sector 0 skew, per track */ + __u64 s_csaddr; /* blk addr of cyl grp summary area */ + __u32 s_cssize; /* size of cyl grp summary area */ + __u32 s_cgsize; /* cylinder group size */ + __u32 s_ntrak; /* tracks per cylinder */ + __u32 s_nsect; /* sectors per track */ + __u32 s_spc; /* sectors per cylinder */ + __u32 s_ipg; /* inodes per cylinder group */ + __u32 s_fpg; /* fragments per group */ + __u32 s_cpc; /* cyl per cycle in postbl */ + __s32 s_contigsumsize;/* size of cluster summary array, 44bsd */ + __s64 s_qbmask; /* ~usb_bmask */ + __s64 s_qfmask; /* ~usb_fmask */ + __s32 s_postblformat; /* format of positional layout tables */ + __s32 s_nrpos; /* number of rotational positions */ + __s32 s_postbloff; /* (__s16) rotation block list head */ + __s32 s_rotbloff; /* (__u8) blocks for each rotation */ + + __u32 s_fpbmask; /* fragments per block mask */ + __u32 s_apb; /* address per block */ + __u32 s_apbmask; /* address per block mask */ + __u32 s_apbshift; /* address per block shift */ + __u32 s_nspfshift; /* number of sector per fragment shift */ + __u32 s_nspb; /* number of sector per block */ + __u32 s_inopf; /* inodes per fragment */ + __u32 s_sbbase; /* offset of NeXTstep superblock */ + __u32 s_bpf; /* bits per fragment */ + __u32 s_bpfshift; /* bits per fragment shift*/ + __u32 s_bpfmask; /* bits per fragment mask */ + + __u32 s_maxsymlinklen;/* upper limit on fast symlinks' size */ + __s32 fs_magic; /* filesystem magic */ + unsigned int s_dirblksize; + __u64 s_root_blocks; + __u64 s_time_to_space; + __u64 s_space_to_time; +}; + +/* + * Sizes of this structures are: + * ufs_super_block_first 512 + * ufs_super_block_second 512 + * ufs_super_block_third 356 + */ +struct ufs_super_block_first { + union { + struct { + __fs32 fs_link; /* UNUSED */ + } fs_42; + struct { + __fs32 fs_state; /* file system state flag */ + } fs_sun; + } fs_u0; + __fs32 fs_rlink; + __fs32 fs_sblkno; + __fs32 fs_cblkno; + __fs32 fs_iblkno; + __fs32 fs_dblkno; + __fs32 fs_cgoffset; + __fs32 fs_cgmask; + __fs32 fs_time; + __fs32 fs_size; + __fs32 fs_dsize; + __fs32 fs_ncg; + __fs32 fs_bsize; + __fs32 fs_fsize; + __fs32 fs_frag; + __fs32 fs_minfree; + __fs32 fs_rotdelay; + __fs32 fs_rps; + __fs32 fs_bmask; + __fs32 fs_fmask; + __fs32 fs_bshift; + __fs32 fs_fshift; + __fs32 fs_maxcontig; + __fs32 fs_maxbpg; + __fs32 fs_fragshift; + __fs32 fs_fsbtodb; + __fs32 fs_sbsize; + __fs32 fs_csmask; + __fs32 fs_csshift; + __fs32 fs_nindir; + __fs32 fs_inopb; + __fs32 fs_nspf; + __fs32 fs_optim; + union { + struct { + __fs32 fs_npsect; + } fs_sun; + struct { + __fs32 fs_state; + } fs_sunx86; + } fs_u1; + __fs32 fs_interleave; + __fs32 fs_trackskew; + __fs32 fs_id[2]; + __fs32 fs_csaddr; + __fs32 fs_cssize; + __fs32 fs_cgsize; + __fs32 fs_ntrak; + __fs32 fs_nsect; + __fs32 fs_spc; + __fs32 fs_ncyl; + __fs32 fs_cpg; + __fs32 fs_ipg; + __fs32 fs_fpg; + struct ufs_csum fs_cstotal; + __s8 fs_fmod; + __s8 fs_clean; + __s8 fs_ronly; + __s8 fs_flags; + __s8 fs_fsmnt[UFS_MAXMNTLEN - 212]; + +}; + +struct ufs_super_block_second { + union { + struct { + __s8 fs_fsmnt[212]; + __fs32 fs_cgrotor; + __fs32 fs_csp[UFS_MAXCSBUFS]; + __fs32 fs_maxcluster; + __fs32 fs_cpc; + __fs16 fs_opostbl[82]; + } fs_u1; + struct { + __s8 fs_fsmnt[UFS2_MAXMNTLEN - UFS_MAXMNTLEN + 212]; + __u8 fs_volname[UFS2_MAXVOLLEN]; + __fs64 fs_swuid; + __fs32 fs_pad; + __fs32 fs_cgrotor; + __fs32 fs_ocsp[UFS2_NOCSPTRS]; + __fs32 fs_contigdirs; + __fs32 fs_csp; + __fs32 fs_maxcluster; + __fs32 fs_active; + __fs32 fs_old_cpc; + __fs32 fs_maxbsize; + __fs64 fs_sparecon64[17]; + __fs64 fs_sblockloc; + __fs64 cs_ndir; + __fs64 cs_nbfree; + } fs_u2; + } fs_un; +}; + +struct ufs_super_block_third { + union { + struct { + __fs16 fs_opostbl[46]; + } fs_u1; + struct { + __fs64 cs_nifree; /* number of free inodes */ + __fs64 cs_nffree; /* number of free frags */ + __fs64 cs_numclusters; /* number of free clusters */ + __fs64 cs_spare[3]; /* future expansion */ + struct ufs_timeval fs_time; /* last time written */ + __fs64 fs_size; /* number of blocks in fs */ + __fs64 fs_dsize; /* number of data blocks in fs */ + __fs64 fs_csaddr; /* blk addr of cyl grp summary area */ + __fs64 fs_pendingblocks;/* blocks in process of being freed */ + __fs32 fs_pendinginodes;/*inodes in process of being freed */ + } __attribute__ ((packed)) fs_u2; + } fs_un1; + union { + struct { + __fs32 fs_sparecon[53];/* reserved for future constants */ + __fs32 fs_reclaim; + __fs32 fs_sparecon2[1]; + __fs32 fs_state; /* file system state time stamp */ + __fs32 fs_qbmask[2]; /* ~usb_bmask */ + __fs32 fs_qfmask[2]; /* ~usb_fmask */ + } fs_sun; + struct { + __fs32 fs_sparecon[53];/* reserved for future constants */ + __fs32 fs_reclaim; + __fs32 fs_sparecon2[1]; + __fs32 fs_npsect; /* # sectors/track including spares */ + __fs32 fs_qbmask[2]; /* ~usb_bmask */ + __fs32 fs_qfmask[2]; /* ~usb_fmask */ + } fs_sunx86; + struct { + __fs32 fs_sparecon[50];/* reserved for future constants */ + __fs32 fs_contigsumsize;/* size of cluster summary array */ + __fs32 fs_maxsymlinklen;/* max length of an internal symlink */ + __fs32 fs_inodefmt; /* format of on-disk inodes */ + __fs32 fs_maxfilesize[2]; /* max representable file size */ + __fs32 fs_qbmask[2]; /* ~usb_bmask */ + __fs32 fs_qfmask[2]; /* ~usb_fmask */ + __fs32 fs_state; /* file system state time stamp */ + } fs_44; + } fs_un2; + __fs32 fs_postblformat; + __fs32 fs_nrpos; + __fs32 fs_postbloff; + __fs32 fs_rotbloff; + __fs32 fs_magic; + __u8 fs_space[1]; +}; + +#endif /* __LINUX_UFS_FS_H */ diff --git a/fs/ufs/util.c b/fs/ufs/util.c new file mode 100644 index 000000000..dff6f7461 --- /dev/null +++ b/fs/ufs/util.c @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/ufs/util.c + * + * Copyright (C) 1998 + * Daniel Pirkl + * Charles University, Faculty of Mathematics and Physics + */ + +#include +#include +#include + +#include "ufs_fs.h" +#include "ufs.h" +#include "swab.h" +#include "util.h" + +struct ufs_buffer_head * _ubh_bread_ (struct ufs_sb_private_info * uspi, + struct super_block *sb, u64 fragment, u64 size) +{ + struct ufs_buffer_head * ubh; + unsigned i, j ; + u64 count = 0; + if (size & ~uspi->s_fmask) + return NULL; + count = size >> uspi->s_fshift; + if (count > UFS_MAXFRAG) + return NULL; + ubh = kmalloc_obj(struct ufs_buffer_head, GFP_NOFS); + if (!ubh) + return NULL; + ubh->fragment = fragment; + ubh->count = count; + for (i = 0; i < count; i++) + if (!(ubh->bh[i] = sb_bread(sb, fragment + i))) + goto failed; + for (; i < UFS_MAXFRAG; i++) + ubh->bh[i] = NULL; + return ubh; +failed: + for (j = 0; j < i; j++) + brelse (ubh->bh[j]); + kfree(ubh); + return NULL; +} + +struct ufs_buffer_head * ubh_bread_uspi (struct ufs_sb_private_info * uspi, + struct super_block *sb, u64 fragment, u64 size) +{ + unsigned i, j; + u64 count = 0; + if (size & ~uspi->s_fmask) + return NULL; + count = size >> uspi->s_fshift; + if (count <= 0 || count > UFS_MAXFRAG) + return NULL; + USPI_UBH(uspi)->fragment = fragment; + USPI_UBH(uspi)->count = count; + for (i = 0; i < count; i++) + if (!(USPI_UBH(uspi)->bh[i] = sb_bread(sb, fragment + i))) + goto failed; + for (; i < UFS_MAXFRAG; i++) + USPI_UBH(uspi)->bh[i] = NULL; + return USPI_UBH(uspi); +failed: + for (j = 0; j < i; j++) + brelse (USPI_UBH(uspi)->bh[j]); + return NULL; +} + +void ubh_brelse (struct ufs_buffer_head * ubh) +{ + unsigned i; + if (!ubh) + return; + for (i = 0; i < ubh->count; i++) + brelse (ubh->bh[i]); + kfree (ubh); +} + +void ubh_brelse_uspi (struct ufs_sb_private_info * uspi) +{ + unsigned i; + if (!USPI_UBH(uspi)) + return; + for ( i = 0; i < USPI_UBH(uspi)->count; i++ ) { + brelse (USPI_UBH(uspi)->bh[i]); + USPI_UBH(uspi)->bh[i] = NULL; + } +} + +void ubh_mark_buffer_dirty (struct ufs_buffer_head * ubh) +{ + unsigned i; + if (!ubh) + return; + for ( i = 0; i < ubh->count; i++ ) + mark_buffer_dirty (ubh->bh[i]); +} + +void ubh_sync_block(struct ufs_buffer_head *ubh) +{ + if (ubh) { + unsigned i; + + for (i = 0; i < ubh->count; i++) + write_dirty_buffer(ubh->bh[i], 0); + + for (i = 0; i < ubh->count; i++) + wait_on_buffer(ubh->bh[i]); + } +} + +void ubh_bforget (struct ufs_buffer_head * ubh) +{ + unsigned i; + if (!ubh) + return; + for ( i = 0; i < ubh->count; i++ ) if ( ubh->bh[i] ) + bforget (ubh->bh[i]); +} + +int ubh_buffer_dirty (struct ufs_buffer_head * ubh) +{ + unsigned i; + unsigned result = 0; + if (!ubh) + return 0; + for ( i = 0; i < ubh->count; i++ ) + result |= buffer_dirty(ubh->bh[i]); + return result; +} + +dev_t +ufs_get_inode_dev(struct super_block *sb, struct ufs_inode_info *ufsi) +{ + __u32 fs32; + dev_t dev; + + if ((UFS_SB(sb)->s_flags & UFS_ST_MASK) == UFS_ST_SUNx86) + fs32 = fs32_to_cpu(sb, ufsi->i_u1.i_data[1]); + else + fs32 = fs32_to_cpu(sb, ufsi->i_u1.i_data[0]); + switch (UFS_SB(sb)->s_flags & UFS_ST_MASK) { + case UFS_ST_SUNx86: + case UFS_ST_SUN: + if ((fs32 & 0xffff0000) == 0 || + (fs32 & 0xffff0000) == 0xffff0000) + dev = old_decode_dev(fs32 & 0x7fff); + else + dev = MKDEV(sysv_major(fs32), sysv_minor(fs32)); + break; + + default: + dev = old_decode_dev(fs32); + break; + } + return dev; +} + +void +ufs_set_inode_dev(struct super_block *sb, struct ufs_inode_info *ufsi, dev_t dev) +{ + __u32 fs32; + + switch (UFS_SB(sb)->s_flags & UFS_ST_MASK) { + case UFS_ST_SUNx86: + case UFS_ST_SUN: + fs32 = sysv_encode_dev(dev); + if ((fs32 & 0xffff8000) == 0) { + fs32 = old_encode_dev(dev); + } + break; + + default: + fs32 = old_encode_dev(dev); + break; + } + if ((UFS_SB(sb)->s_flags & UFS_ST_MASK) == UFS_ST_SUNx86) + ufsi->i_u1.i_data[1] = cpu_to_fs32(sb, fs32); + else + ufsi->i_u1.i_data[0] = cpu_to_fs32(sb, fs32); +} + +/** + * ufs_get_locked_folio() - locate, pin and lock a pagecache folio, if not exist + * read it from disk. + * @mapping: the address_space to search + * @index: the page index + * + * Locates the desired pagecache folio, if not exist we'll read it, + * locks it, increments its reference + * count and returns its address. + * + */ +struct folio *ufs_get_locked_folio(struct address_space *mapping, + pgoff_t index) +{ + struct inode *inode = mapping->host; + struct folio *folio = filemap_lock_folio(mapping, index); + if (IS_ERR(folio)) { + folio = read_mapping_folio(mapping, index, NULL); + + if (IS_ERR(folio)) { + printk(KERN_ERR "ufs_change_blocknr: read_mapping_folio error: ino %llu, index: %lu\n", + mapping->host->i_ino, index); + return folio; + } + + folio_lock(folio); + + if (unlikely(folio->mapping == NULL)) { + /* Truncate got there first */ + folio_unlock(folio); + folio_put(folio); + return NULL; + } + } + if (!folio_buffers(folio)) + create_empty_buffers(folio, 1 << inode->i_blkbits, 0); + return folio; +} diff --git a/fs/ufs/util.h b/fs/ufs/util.h new file mode 100644 index 000000000..391bb4f11 --- /dev/null +++ b/fs/ufs/util.h @@ -0,0 +1,597 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * linux/fs/ufs/util.h + * + * Copyright (C) 1998 + * Daniel Pirkl + * Charles University, Faculty of Mathematics and Physics + */ + +#include +#include +#include "swab.h" + +/* + * functions used for retyping + */ +static inline struct ufs_buffer_head *UCPI_UBH(struct ufs_cg_private_info *cpi) +{ + return &cpi->c_ubh; +} +static inline struct ufs_buffer_head *USPI_UBH(struct ufs_sb_private_info *spi) +{ + return &spi->s_ubh; +} + + + +/* + * macros used for accessing structures + */ +static inline s32 +ufs_get_fs_state(struct super_block *sb, struct ufs_super_block_first *usb1, + struct ufs_super_block_third *usb3) +{ + switch (UFS_SB(sb)->s_flags & UFS_ST_MASK) { + case UFS_ST_SUNOS: + if (fs32_to_cpu(sb, usb3->fs_postblformat) == UFS_42POSTBLFMT) + return fs32_to_cpu(sb, usb1->fs_u0.fs_sun.fs_state); + fallthrough; /* to UFS_ST_SUN */ + case UFS_ST_SUN: + return fs32_to_cpu(sb, usb3->fs_un2.fs_sun.fs_state); + case UFS_ST_SUNx86: + return fs32_to_cpu(sb, usb1->fs_u1.fs_sunx86.fs_state); + case UFS_ST_44BSD: + default: + return fs32_to_cpu(sb, usb3->fs_un2.fs_44.fs_state); + } +} + +static inline void +ufs_set_fs_state(struct super_block *sb, struct ufs_super_block_first *usb1, + struct ufs_super_block_third *usb3, s32 value) +{ + switch (UFS_SB(sb)->s_flags & UFS_ST_MASK) { + case UFS_ST_SUNOS: + if (fs32_to_cpu(sb, usb3->fs_postblformat) == UFS_42POSTBLFMT) { + usb1->fs_u0.fs_sun.fs_state = cpu_to_fs32(sb, value); + break; + } + fallthrough; /* to UFS_ST_SUN */ + case UFS_ST_SUN: + usb3->fs_un2.fs_sun.fs_state = cpu_to_fs32(sb, value); + break; + case UFS_ST_SUNx86: + usb1->fs_u1.fs_sunx86.fs_state = cpu_to_fs32(sb, value); + break; + case UFS_ST_44BSD: + usb3->fs_un2.fs_44.fs_state = cpu_to_fs32(sb, value); + break; + } +} + +static inline u32 +ufs_get_fs_npsect(struct super_block *sb, struct ufs_super_block_first *usb1, + struct ufs_super_block_third *usb3) +{ + if ((UFS_SB(sb)->s_flags & UFS_ST_MASK) == UFS_ST_SUNx86) + return fs32_to_cpu(sb, usb3->fs_un2.fs_sunx86.fs_npsect); + else + return fs32_to_cpu(sb, usb1->fs_u1.fs_sun.fs_npsect); +} + +static inline u64 +ufs_get_fs_qbmask(struct super_block *sb, struct ufs_super_block_third *usb3) +{ + __fs64 tmp; + + switch (UFS_SB(sb)->s_flags & UFS_ST_MASK) { + case UFS_ST_SUNOS: + case UFS_ST_SUN: + ((__fs32 *)&tmp)[0] = usb3->fs_un2.fs_sun.fs_qbmask[0]; + ((__fs32 *)&tmp)[1] = usb3->fs_un2.fs_sun.fs_qbmask[1]; + break; + case UFS_ST_SUNx86: + ((__fs32 *)&tmp)[0] = usb3->fs_un2.fs_sunx86.fs_qbmask[0]; + ((__fs32 *)&tmp)[1] = usb3->fs_un2.fs_sunx86.fs_qbmask[1]; + break; + case UFS_ST_44BSD: + ((__fs32 *)&tmp)[0] = usb3->fs_un2.fs_44.fs_qbmask[0]; + ((__fs32 *)&tmp)[1] = usb3->fs_un2.fs_44.fs_qbmask[1]; + break; + } + + return fs64_to_cpu(sb, tmp); +} + +static inline u64 +ufs_get_fs_qfmask(struct super_block *sb, struct ufs_super_block_third *usb3) +{ + __fs64 tmp; + + switch (UFS_SB(sb)->s_flags & UFS_ST_MASK) { + case UFS_ST_SUNOS: + case UFS_ST_SUN: + ((__fs32 *)&tmp)[0] = usb3->fs_un2.fs_sun.fs_qfmask[0]; + ((__fs32 *)&tmp)[1] = usb3->fs_un2.fs_sun.fs_qfmask[1]; + break; + case UFS_ST_SUNx86: + ((__fs32 *)&tmp)[0] = usb3->fs_un2.fs_sunx86.fs_qfmask[0]; + ((__fs32 *)&tmp)[1] = usb3->fs_un2.fs_sunx86.fs_qfmask[1]; + break; + case UFS_ST_44BSD: + ((__fs32 *)&tmp)[0] = usb3->fs_un2.fs_44.fs_qfmask[0]; + ((__fs32 *)&tmp)[1] = usb3->fs_un2.fs_44.fs_qfmask[1]; + break; + } + + return fs64_to_cpu(sb, tmp); +} + +static inline u16 +ufs_get_de_namlen(struct super_block *sb, struct ufs_dir_entry *de) +{ + if ((UFS_SB(sb)->s_flags & UFS_DE_MASK) == UFS_DE_OLD) + return fs16_to_cpu(sb, de->d_u.d_namlen); + else + return de->d_u.d_44.d_namlen; /* XXX this seems wrong */ +} + +static inline void +ufs_set_de_namlen(struct super_block *sb, struct ufs_dir_entry *de, u16 value) +{ + if ((UFS_SB(sb)->s_flags & UFS_DE_MASK) == UFS_DE_OLD) + de->d_u.d_namlen = cpu_to_fs16(sb, value); + else + de->d_u.d_44.d_namlen = value; /* XXX this seems wrong */ +} + +static inline void +ufs_set_de_type(struct super_block *sb, struct ufs_dir_entry *de, int mode) +{ + if ((UFS_SB(sb)->s_flags & UFS_DE_MASK) != UFS_DE_44BSD) + return; + + /* + * TODO turn this into a table lookup + */ + switch (mode & S_IFMT) { + case S_IFSOCK: + de->d_u.d_44.d_type = DT_SOCK; + break; + case S_IFLNK: + de->d_u.d_44.d_type = DT_LNK; + break; + case S_IFREG: + de->d_u.d_44.d_type = DT_REG; + break; + case S_IFBLK: + de->d_u.d_44.d_type = DT_BLK; + break; + case S_IFDIR: + de->d_u.d_44.d_type = DT_DIR; + break; + case S_IFCHR: + de->d_u.d_44.d_type = DT_CHR; + break; + case S_IFIFO: + de->d_u.d_44.d_type = DT_FIFO; + break; + default: + de->d_u.d_44.d_type = DT_UNKNOWN; + } +} + +static inline u32 +ufs_get_inode_uid(struct super_block *sb, struct ufs_inode *inode) +{ + switch (UFS_SB(sb)->s_flags & UFS_UID_MASK) { + case UFS_UID_44BSD: + return fs32_to_cpu(sb, inode->ui_u3.ui_44.ui_uid); + case UFS_UID_EFT: + if (inode->ui_u1.oldids.ui_suid == 0xFFFF) + return fs32_to_cpu(sb, inode->ui_u3.ui_sun.ui_uid); + fallthrough; + default: + return fs16_to_cpu(sb, inode->ui_u1.oldids.ui_suid); + } +} + +static inline void +ufs_set_inode_uid(struct super_block *sb, struct ufs_inode *inode, u32 value) +{ + switch (UFS_SB(sb)->s_flags & UFS_UID_MASK) { + case UFS_UID_44BSD: + inode->ui_u3.ui_44.ui_uid = cpu_to_fs32(sb, value); + inode->ui_u1.oldids.ui_suid = cpu_to_fs16(sb, value); + break; + case UFS_UID_EFT: + inode->ui_u3.ui_sun.ui_uid = cpu_to_fs32(sb, value); + if (value > 0xFFFF) + value = 0xFFFF; + fallthrough; + default: + inode->ui_u1.oldids.ui_suid = cpu_to_fs16(sb, value); + break; + } +} + +static inline u32 +ufs_get_inode_gid(struct super_block *sb, struct ufs_inode *inode) +{ + switch (UFS_SB(sb)->s_flags & UFS_UID_MASK) { + case UFS_UID_44BSD: + return fs32_to_cpu(sb, inode->ui_u3.ui_44.ui_gid); + case UFS_UID_EFT: + if (inode->ui_u1.oldids.ui_sgid == 0xFFFF) + return fs32_to_cpu(sb, inode->ui_u3.ui_sun.ui_gid); + fallthrough; + default: + return fs16_to_cpu(sb, inode->ui_u1.oldids.ui_sgid); + } +} + +static inline void +ufs_set_inode_gid(struct super_block *sb, struct ufs_inode *inode, u32 value) +{ + switch (UFS_SB(sb)->s_flags & UFS_UID_MASK) { + case UFS_UID_44BSD: + inode->ui_u3.ui_44.ui_gid = cpu_to_fs32(sb, value); + inode->ui_u1.oldids.ui_sgid = cpu_to_fs16(sb, value); + break; + case UFS_UID_EFT: + inode->ui_u3.ui_sun.ui_gid = cpu_to_fs32(sb, value); + if (value > 0xFFFF) + value = 0xFFFF; + fallthrough; + default: + inode->ui_u1.oldids.ui_sgid = cpu_to_fs16(sb, value); + break; + } +} + +dev_t ufs_get_inode_dev(struct super_block *, struct ufs_inode_info *); +void ufs_set_inode_dev(struct super_block *, struct ufs_inode_info *, dev_t); +int ufs_prepare_chunk(struct folio *folio, loff_t pos, unsigned len); + +/* + * These functions manipulate ufs buffers + */ +#define ubh_bread(sb,fragment,size) _ubh_bread_(uspi,sb,fragment,size) +extern struct ufs_buffer_head * _ubh_bread_(struct ufs_sb_private_info *, struct super_block *, u64 , u64); +extern struct ufs_buffer_head * ubh_bread_uspi(struct ufs_sb_private_info *, struct super_block *, u64, u64); +extern void ubh_brelse (struct ufs_buffer_head *); +extern void ubh_brelse_uspi (struct ufs_sb_private_info *); +extern void ubh_mark_buffer_dirty (struct ufs_buffer_head *); +extern void ubh_sync_block(struct ufs_buffer_head *); +extern void ubh_bforget (struct ufs_buffer_head *); +extern int ubh_buffer_dirty (struct ufs_buffer_head *); + +/* This functions works with cache pages*/ +struct folio *ufs_get_locked_folio(struct address_space *mapping, pgoff_t index); +static inline void ufs_put_locked_folio(struct folio *folio) +{ + folio_unlock(folio); + folio_put(folio); +} + +/* + * macros and inline function to get important structures from ufs_sb_private_info + */ + +static inline void *get_usb_offset(struct ufs_sb_private_info *uspi, + unsigned int offset) +{ + unsigned int index; + + index = offset >> uspi->s_fshift; + offset &= ~uspi->s_fmask; + return uspi->s_ubh.bh[index]->b_data + offset; +} + +#define ubh_get_usb_first(uspi) \ + ((struct ufs_super_block_first *)get_usb_offset((uspi), 0)) + +#define ubh_get_usb_second(uspi) \ + ((struct ufs_super_block_second *)get_usb_offset((uspi), UFS_SECTOR_SIZE)) + +#define ubh_get_usb_third(uspi) \ + ((struct ufs_super_block_third *)get_usb_offset((uspi), 2*UFS_SECTOR_SIZE)) + + +#define ubh_get_ucg(ubh) \ + ((struct ufs_cylinder_group *)((ubh)->bh[0]->b_data)) + + +/* + * Extract byte from ufs_buffer_head + * Extract the bits for a block from a map inside ufs_buffer_head + */ +#define ubh_get_addr8(ubh,begin) \ + ((u8*)(ubh)->bh[(begin) >> uspi->s_fshift]->b_data + \ + ((begin) & ~uspi->s_fmask)) + +#define ubh_get_addr16(ubh,begin) \ + (((__fs16*)((ubh)->bh[(begin) >> (uspi->s_fshift-1)]->b_data)) + \ + ((begin) & ((uspi->fsize>>1) - 1))) + +#define ubh_get_addr32(ubh,begin) \ + (((__fs32*)((ubh)->bh[(begin) >> (uspi->s_fshift-2)]->b_data)) + \ + ((begin) & ((uspi->s_fsize>>2) - 1))) + +#define ubh_get_addr64(ubh,begin) \ + (((__fs64*)((ubh)->bh[(begin) >> (uspi->s_fshift-3)]->b_data)) + \ + ((begin) & ((uspi->s_fsize>>3) - 1))) + +#define ubh_get_addr ubh_get_addr8 + +static inline void *ubh_get_data_ptr(struct ufs_sb_private_info *uspi, + struct ufs_buffer_head *ubh, + u64 blk) +{ + if (uspi->fs_magic == UFS2_MAGIC) + return ubh_get_addr64(ubh, blk); + else + return ubh_get_addr32(ubh, blk); +} + +#define ubh_blkmap(ubh,begin,bit) \ + ((*ubh_get_addr(ubh, (begin) + ((bit) >> 3)) >> ((bit) & 7)) & (0xff >> (UFS_MAXFRAG - uspi->s_fpb))) + +static inline u64 +ufs_freefrags(struct ufs_sb_private_info *uspi) +{ + return ufs_blkstofrags(uspi->cs_total.cs_nbfree) + + uspi->cs_total.cs_nffree; +} + +/* + * Macros to access cylinder group array structures + */ +#define ubh_cg_blktot(ucpi,cylno) \ + (*((__fs32*)ubh_get_addr(UCPI_UBH(ucpi), (ucpi)->c_btotoff + ((cylno) << 2)))) + +#define ubh_cg_blks(ucpi,cylno,rpos) \ + (*((__fs16*)ubh_get_addr(UCPI_UBH(ucpi), \ + (ucpi)->c_boff + (((cylno) * uspi->s_nrpos + (rpos)) << 1 )))) + +/* + * Bitmap operations + * These functions work like classical bitmap operations. + * The difference is that we don't have the whole bitmap + * in one contiguous chunk of memory, but in several buffers. + * The parameters of each function are super_block, ufs_buffer_head and + * position of the beginning of the bitmap. + */ +#define ubh_setbit(ubh,begin,bit) \ + (*ubh_get_addr(ubh, (begin) + ((bit) >> 3)) |= (1 << ((bit) & 7))) + +#define ubh_clrbit(ubh,begin,bit) \ + (*ubh_get_addr (ubh, (begin) + ((bit) >> 3)) &= ~(1 << ((bit) & 7))) + +#define ubh_isset(ubh,begin,bit) \ + (*ubh_get_addr (ubh, (begin) + ((bit) >> 3)) & (1 << ((bit) & 7))) + +#define ubh_isclr(ubh,begin,bit) (!ubh_isset(ubh,begin,bit)) + +#define ubh_find_first_zero_bit(ubh,begin,size) _ubh_find_next_zero_bit_(uspi,ubh,begin,size,0) + +#define ubh_find_next_zero_bit(ubh,begin,size,offset) _ubh_find_next_zero_bit_(uspi,ubh,begin,size,offset) +static inline unsigned _ubh_find_next_zero_bit_( + struct ufs_sb_private_info * uspi, struct ufs_buffer_head * ubh, + unsigned begin, unsigned size, unsigned offset) +{ + unsigned base, count, pos; + + size -= offset; + begin <<= 3; + offset += begin; + base = offset >> uspi->s_bpfshift; + offset &= uspi->s_bpfmask; + for (;;) { + count = min_t(unsigned int, size + offset, uspi->s_bpf); + size -= count - offset; + pos = find_next_zero_bit_le(ubh->bh[base]->b_data, count, offset); + if (pos < count || !size) + break; + base++; + offset = 0; + } + return (base << uspi->s_bpfshift) + pos - begin; +} + +static inline unsigned find_last_zero_bit (unsigned char * bitmap, + unsigned size, unsigned offset) +{ + unsigned bit, i; + unsigned char * mapp; + unsigned char map; + + mapp = bitmap + (size >> 3); + map = *mapp--; + bit = 1 << (size & 7); + for (i = size; i > offset; i--) { + if ((map & bit) == 0) + break; + if ((i & 7) != 0) { + bit >>= 1; + } else { + map = *mapp--; + bit = 1 << 7; + } + } + return i; +} + +#define ubh_find_last_zero_bit(ubh,begin,size,offset) _ubh_find_last_zero_bit_(uspi,ubh,begin,size,offset) +static inline unsigned _ubh_find_last_zero_bit_( + struct ufs_sb_private_info * uspi, struct ufs_buffer_head * ubh, + unsigned begin, unsigned start, unsigned end) +{ + unsigned base, count, pos, size; + + size = start - end; + begin <<= 3; + start += begin; + base = start >> uspi->s_bpfshift; + start &= uspi->s_bpfmask; + for (;;) { + count = min_t(unsigned int, + size + (uspi->s_bpf - start), uspi->s_bpf) + - (uspi->s_bpf - start); + size -= count; + pos = find_last_zero_bit (ubh->bh[base]->b_data, + start, start - count); + if (pos > start - count || !size) + break; + base--; + start = uspi->s_bpf; + } + return (base << uspi->s_bpfshift) + pos - begin; +} + +static inline int ubh_isblockset(struct ufs_sb_private_info *uspi, + struct ufs_cg_private_info *ucpi, unsigned int frag) +{ + struct ufs_buffer_head *ubh = UCPI_UBH(ucpi); + u8 *p = ubh_get_addr(ubh, ucpi->c_freeoff + (frag >> 3)); + u8 mask; + + switch (uspi->s_fpb) { + case 8: + return *p == 0xff; + case 4: + mask = 0x0f << (frag & 4); + return (*p & mask) == mask; + case 2: + mask = 0x03 << (frag & 6); + return (*p & mask) == mask; + case 1: + mask = 0x01 << (frag & 7); + return (*p & mask) == mask; + } + return 0; +} + +static inline void ubh_clrblock(struct ufs_sb_private_info *uspi, + struct ufs_cg_private_info *ucpi, unsigned int frag) +{ + struct ufs_buffer_head *ubh = UCPI_UBH(ucpi); + u8 *p = ubh_get_addr(ubh, ucpi->c_freeoff + (frag >> 3)); + + switch (uspi->s_fpb) { + case 8: + *p = 0x00; + return; + case 4: + *p &= ~(0x0f << (frag & 4)); + return; + case 2: + *p &= ~(0x03 << (frag & 6)); + return; + case 1: + *p &= ~(0x01 << (frag & 7)); + return; + } +} + +static inline void ubh_setblock(struct ufs_sb_private_info * uspi, + struct ufs_cg_private_info *ucpi, unsigned int frag) +{ + struct ufs_buffer_head *ubh = UCPI_UBH(ucpi); + u8 *p = ubh_get_addr(ubh, ucpi->c_freeoff + (frag >> 3)); + + switch (uspi->s_fpb) { + case 8: + *p = 0xff; + return; + case 4: + *p |= 0x0f << (frag & 4); + return; + case 2: + *p |= 0x03 << (frag & 6); + return; + case 1: + *p |= 0x01 << (frag & 7); + return; + } +} + +static inline void ufs_fragacct (struct super_block * sb, unsigned blockmap, + __fs32 * fraglist, int cnt) +{ + struct ufs_sb_private_info * uspi; + unsigned fragsize, pos; + + uspi = UFS_SB(sb)->s_uspi; + + fragsize = 0; + for (pos = 0; pos < uspi->s_fpb; pos++) { + if (blockmap & (1 << pos)) { + fragsize++; + } + else if (fragsize > 0) { + fs32_add(sb, &fraglist[fragsize], cnt); + fragsize = 0; + } + } + if (fragsize > 0 && fragsize < uspi->s_fpb) + fs32_add(sb, &fraglist[fragsize], cnt); +} + +static inline void *ufs_get_direct_data_ptr(struct ufs_sb_private_info *uspi, + struct ufs_inode_info *ufsi, + unsigned blk) +{ + BUG_ON(blk > UFS_TIND_BLOCK); + return uspi->fs_magic == UFS2_MAGIC ? + (void *)&ufsi->i_u1.u2_i_data[blk] : + (void *)&ufsi->i_u1.i_data[blk]; +} + +static inline u64 ufs_data_ptr_to_cpu(struct super_block *sb, void *p) +{ + return UFS_SB(sb)->s_uspi->fs_magic == UFS2_MAGIC ? + fs64_to_cpu(sb, *(__fs64 *)p) : + fs32_to_cpu(sb, *(__fs32 *)p); +} + +static inline void ufs_cpu_to_data_ptr(struct super_block *sb, void *p, u64 val) +{ + if (UFS_SB(sb)->s_uspi->fs_magic == UFS2_MAGIC) + *(__fs64 *)p = cpu_to_fs64(sb, val); + else + *(__fs32 *)p = cpu_to_fs32(sb, val); +} + +static inline void ufs_data_ptr_clear(struct ufs_sb_private_info *uspi, + void *p) +{ + if (uspi->fs_magic == UFS2_MAGIC) + *(__fs64 *)p = 0; + else + *(__fs32 *)p = 0; +} + +static inline int ufs_is_data_ptr_zero(struct ufs_sb_private_info *uspi, + void *p) +{ + if (uspi->fs_magic == UFS2_MAGIC) + return *(__fs64 *)p == 0; + else + return *(__fs32 *)p == 0; +} + +static inline __fs32 ufs_get_seconds(struct super_block *sbp) +{ + time64_t now = ktime_get_real_seconds(); + + /* Signed 32-bit interpretation wraps around in 2038, which + * happens in ufs1 inode stamps but not ufs2 using 64-bits + * stamps. For superblock and blockgroup, let's assume + * unsigned 32-bit stamps, which are good until y2106. + * Wrap around rather than clamp here to make the dirty + * file system detection work in the superblock stamp. + */ + return cpu_to_fs32(sbp, lower_32_bits(now)); +} diff --git a/fs/unicode/.gitignore b/fs/unicode/.gitignore new file mode 100644 index 000000000..51cdf3fb4 --- /dev/null +++ b/fs/unicode/.gitignore @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +/mkutf8data +/utf8data.c diff --git a/fs/unicode/Kconfig b/fs/unicode/Kconfig new file mode 100644 index 000000000..4ad2c3655 --- /dev/null +++ b/fs/unicode/Kconfig @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# UTF-8 normalization +# +config UNICODE + tristate "UTF-8 normalization and casefolding support" + help + Say Y here to enable UTF-8 NFD normalization and NFD+CF casefolding + support. If you say M here the large table of case foldings will + be a separate loadable module that gets requested only when a file + system actually use it. + +config UNICODE_NORMALIZATION_KUNIT_TEST + tristate "Test UTF-8 normalization support" + depends on UNICODE && KUNIT + default KUNIT_ALL_TESTS diff --git a/fs/unicode/Makefile b/fs/unicode/Makefile new file mode 100644 index 000000000..d95be7fb9 --- /dev/null +++ b/fs/unicode/Makefile @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: GPL-2.0 + +ifneq ($(CONFIG_UNICODE),) +obj-y += unicode.o +endif +obj-$(CONFIG_UNICODE) += utf8data.o +obj-$(CONFIG_UNICODE_NORMALIZATION_KUNIT_TEST) += tests/utf8_kunit.o + +unicode-y := utf8-norm.o utf8-core.o + +$(obj)/utf8-data.o: $(obj)/utf8data.c + +# In the normal build, the checked-in utf8data.c is just shipped. +# +# To generate utf8data.c from UCD, put *.txt files in this directory +# and pass REGENERATE_UTF8DATA=1 from the command line. +ifdef REGENERATE_UTF8DATA + +quiet_cmd_utf8data = GEN $@ + cmd_utf8data = $< \ + -a $(src)/DerivedAge.txt \ + -c $(src)/DerivedCombiningClass.txt \ + -p $(src)/DerivedCoreProperties.txt \ + -d $(src)/UnicodeData.txt \ + -f $(src)/CaseFolding.txt \ + -n $(src)/NormalizationCorrections.txt \ + -t $(src)/NormalizationTest.txt \ + -o $@ + +$(obj)/utf8data.c: $(obj)/mkutf8data $(filter %.txt, $(cmd_utf8data)) FORCE + $(call if_changed,utf8data) + +else + +$(obj)/utf8data.c: $(src)/utf8data.c_shipped FORCE + $(call if_changed,copy) + +endif + +targets += utf8data.c +hostprogs += mkutf8data diff --git a/fs/unicode/README.utf8data b/fs/unicode/README.utf8data new file mode 100644 index 000000000..f75567e28 --- /dev/null +++ b/fs/unicode/README.utf8data @@ -0,0 +1,57 @@ +The utf8data.c file in this directory is generated from the Unicode +Character Database for version 12.1.0 of the Unicode standard. + +The full set of files can be found here: + + http://www.unicode.org/Public/12.1.0/ucd/ + +Individual source links: + + https://www.unicode.org/Public/12.1.0/ucd/CaseFolding.txt + https://www.unicode.org/Public/12.1.0/ucd/DerivedAge.txt + https://www.unicode.org/Public/12.1.0/ucd/extracted/DerivedCombiningClass.txt + https://www.unicode.org/Public/12.1.0/ucd/DerivedCoreProperties.txt + https://www.unicode.org/Public/12.1.0/ucd/NormalizationCorrections.txt + https://www.unicode.org/Public/12.1.0/ucd/NormalizationTest.txt + https://www.unicode.org/Public/12.1.0/ucd/UnicodeData.txt + +md5sums (verify by running "md5sum -c README.utf8data"): + + 900e76da1d822a160fd6b8c0b1d70094 CaseFolding.txt + 131256380bff4fea8ad4a851616f2f10 DerivedAge.txt + e731a4089b30002144e107e3d6f8d1fa DerivedCombiningClass.txt + a47c9fbd7ff92a9b261ba9831e68778a DerivedCoreProperties.txt + fcab6dad15e440879d92f315978f93d3 NormalizationCorrections.txt + f9ff1c55a60decf436100f791b44aa98 NormalizationTest.txt + 755f6af699f8c8d2d958da411f78f6c6 UnicodeData.txt + +sha1sums (verify by running "sha1sum -c README.utf8data"): + + dc9245f6803c4ac99555c361f5052e0b13eb779b CaseFolding.txt + 3281104f237184cdb5d869e86eb8573678ada7da DerivedAge.txt + 2f5f995ccb96e0fa84b15151b35d5e2681535175 DerivedCombiningClass.txt + 5b8698a3fcd5018e1987f296b02e2c17e696415e DerivedCoreProperties.txt + cd83935fbc012345d8792d2c704f69497e753835 NormalizationCorrections.txt + ea419aae505b337b0d99a83fa83fe58ddff7c19f NormalizationTest.txt + dc973c0fc93d6f09d9ab9f70d1c9f89c447f0526 UnicodeData.txt + + +To update to the newer version of the Unicode standard, the latest +released version of the UCD can be found here: + + http://www.unicode.org/Public/UCD/latest/ + +Then, build under fs/unicode/ with REGENERATE_UTF8DATA=1: + + make REGENERATE_UTF8DATA=1 fs/unicode/ + +After sanity checking the newly generated utf8data.c file (the +version generated from the 12.1.0 UCD should be 4,109 lines long, and +have a total size of 324k) and/or comparing it with the older version +of utf8data.c_shipped, rename it to utf8data.c_shipped. + +If you are a kernel developer updating to a newer version of the +Unicode Character Database, please update this README.utf8data file +with the version of the UCD that was used, the md5sum and sha1sums of +the *.txt files, before checking in the new versions of the utf8data.c +and README.utf8data files. diff --git a/fs/unicode/mkutf8data.c b/fs/unicode/mkutf8data.c new file mode 100644 index 000000000..401f5d3ae --- /dev/null +++ b/fs/unicode/mkutf8data.c @@ -0,0 +1,3434 @@ +/* + * Copyright (c) 2014 SGI. + * All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it would be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/* Generator for a compact trie for unicode normalization */ + +#include +#include +#include +#include +#include +#include +#include +#include + +/* Default names of the in- and output files. */ + +#define AGE_NAME "DerivedAge.txt" +#define CCC_NAME "DerivedCombiningClass.txt" +#define PROP_NAME "DerivedCoreProperties.txt" +#define DATA_NAME "UnicodeData.txt" +#define FOLD_NAME "CaseFolding.txt" +#define NORM_NAME "NormalizationCorrections.txt" +#define TEST_NAME "NormalizationTest.txt" +#define UTF8_NAME "utf8data.c" + +const char *age_name = AGE_NAME; +const char *ccc_name = CCC_NAME; +const char *prop_name = PROP_NAME; +const char *data_name = DATA_NAME; +const char *fold_name = FOLD_NAME; +const char *norm_name = NORM_NAME; +const char *test_name = TEST_NAME; +const char *utf8_name = UTF8_NAME; + +int verbose = 0; + +/* An arbitrary line size limit on input lines. */ + +#define LINESIZE 1024 +char line[LINESIZE]; +char buf0[LINESIZE]; +char buf1[LINESIZE]; +char buf2[LINESIZE]; +char buf3[LINESIZE]; + +const char *argv0; + +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) + +/* ------------------------------------------------------------------ */ + +/* + * Unicode version numbers consist of three parts: major, minor, and a + * revision. These numbers are packed into an unsigned int to obtain + * a single version number. + * + * To save space in the generated trie, the unicode version is not + * stored directly, instead we calculate a generation number from the + * unicode versions seen in the DerivedAge file, and use that as an + * index into a table of unicode versions. + */ +#define UNICODE_MAJ_SHIFT (16) +#define UNICODE_MIN_SHIFT (8) + +#define UNICODE_MAJ_MAX ((unsigned short)-1) +#define UNICODE_MIN_MAX ((unsigned char)-1) +#define UNICODE_REV_MAX ((unsigned char)-1) + +#define UNICODE_AGE(MAJ,MIN,REV) \ + (((unsigned int)(MAJ) << UNICODE_MAJ_SHIFT) | \ + ((unsigned int)(MIN) << UNICODE_MIN_SHIFT) | \ + ((unsigned int)(REV))) + +unsigned int *ages; +int ages_count; + +unsigned int unicode_maxage; + +static int age_valid(unsigned int major, unsigned int minor, + unsigned int revision) +{ + if (major > UNICODE_MAJ_MAX) + return 0; + if (minor > UNICODE_MIN_MAX) + return 0; + if (revision > UNICODE_REV_MAX) + return 0; + return 1; +} + +/* ------------------------------------------------------------------ */ + +/* + * utf8trie_t + * + * A compact binary tree, used to decode UTF-8 characters. + * + * Internal nodes are one byte for the node itself, and up to three + * bytes for an offset into the tree. The first byte contains the + * following information: + * NEXTBYTE - flag - advance to next byte if set + * BITNUM - 3 bit field - the bit number to tested + * OFFLEN - 2 bit field - number of bytes in the offset + * if offlen == 0 (non-branching node) + * RIGHTPATH - 1 bit field - set if the following node is for the + * right-hand path (tested bit is set) + * TRIENODE - 1 bit field - set if the following node is an internal + * node, otherwise it is a leaf node + * if offlen != 0 (branching node) + * LEFTNODE - 1 bit field - set if the left-hand node is internal + * RIGHTNODE - 1 bit field - set if the right-hand node is internal + * + * Due to the way utf8 works, there cannot be branching nodes with + * NEXTBYTE set, and moreover those nodes always have a righthand + * descendant. + */ +typedef unsigned char utf8trie_t; +#define BITNUM 0x07 +#define NEXTBYTE 0x08 +#define OFFLEN 0x30 +#define OFFLEN_SHIFT 4 +#define RIGHTPATH 0x40 +#define TRIENODE 0x80 +#define RIGHTNODE 0x40 +#define LEFTNODE 0x80 + +/* + * utf8leaf_t + * + * The leaves of the trie are embedded in the trie, and so the same + * underlying datatype, unsigned char. + * + * leaf[0]: The unicode version, stored as a generation number that is + * an index into utf8agetab[]. With this we can filter code + * points based on the unicode version in which they were + * defined. The CCC of a non-defined code point is 0. + * leaf[1]: Canonical Combining Class. During normalization, we need + * to do a stable sort into ascending order of all characters + * with a non-zero CCC that occur between two characters with + * a CCC of 0, or at the begin or end of a string. + * The unicode standard guarantees that all CCC values are + * between 0 and 254 inclusive, which leaves 255 available as + * a special value. + * Code points with CCC 0 are known as stoppers. + * leaf[2]: Decomposition. If leaf[1] == 255, then leaf[2] is the + * start of a NUL-terminated string that is the decomposition + * of the character. + * The CCC of a decomposable character is the same as the CCC + * of the first character of its decomposition. + * Some characters decompose as the empty string: these are + * characters with the Default_Ignorable_Code_Point property. + * These do affect normalization, as they all have CCC 0. + * + * The decompositions in the trie have been fully expanded. + * + * Casefolding, if applicable, is also done using decompositions. + */ +typedef unsigned char utf8leaf_t; + +#define LEAF_GEN(LEAF) ((LEAF)[0]) +#define LEAF_CCC(LEAF) ((LEAF)[1]) +#define LEAF_STR(LEAF) ((const char*)((LEAF) + 2)) + +#define MAXGEN (255) + +#define MINCCC (0) +#define MAXCCC (254) +#define STOPPER (0) +#define DECOMPOSE (255) +#define HANGUL ((char)(255)) + +#define UTF8HANGULLEAF (12) + +struct tree; +static utf8leaf_t *utf8nlookup(struct tree *, unsigned char *, + const char *, size_t); +static utf8leaf_t *utf8lookup(struct tree *, unsigned char *, const char *); + +unsigned char *utf8data; +size_t utf8data_size; + +utf8trie_t *nfdi; +utf8trie_t *nfdicf; + +/* ------------------------------------------------------------------ */ + +/* + * UTF8 valid ranges. + * + * The UTF-8 encoding spreads the bits of a 32bit word over several + * bytes. This table gives the ranges that can be held and how they'd + * be represented. + * + * 0x00000000 0x0000007F: 0xxxxxxx + * 0x00000000 0x000007FF: 110xxxxx 10xxxxxx + * 0x00000000 0x0000FFFF: 1110xxxx 10xxxxxx 10xxxxxx + * 0x00000000 0x001FFFFF: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + * 0x00000000 0x03FFFFFF: 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx + * 0x00000000 0x7FFFFFFF: 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx + * + * There is an additional requirement on UTF-8, in that only the + * shortest representation of a 32bit value is to be used. A decoder + * must not decode sequences that do not satisfy this requirement. + * Thus the allowed ranges have a lower bound. + * + * 0x00000000 0x0000007F: 0xxxxxxx + * 0x00000080 0x000007FF: 110xxxxx 10xxxxxx + * 0x00000800 0x0000FFFF: 1110xxxx 10xxxxxx 10xxxxxx + * 0x00010000 0x001FFFFF: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + * 0x00200000 0x03FFFFFF: 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx + * 0x04000000 0x7FFFFFFF: 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx + * + * Actual unicode characters are limited to the range 0x0 - 0x10FFFF, + * 17 planes of 65536 values. This limits the sequences actually seen + * even more, to just the following. + * + * 0 - 0x7f: 0 0x7f + * 0x80 - 0x7ff: 0xc2 0x80 0xdf 0xbf + * 0x800 - 0xffff: 0xe0 0xa0 0x80 0xef 0xbf 0xbf + * 0x10000 - 0x10ffff: 0xf0 0x90 0x80 0x80 0xf4 0x8f 0xbf 0xbf + * + * Even within those ranges not all values are allowed: the surrogates + * 0xd800 - 0xdfff should never be seen. + * + * Note that the longest sequence seen with valid usage is 4 bytes, + * the same a single UTF-32 character. This makes the UTF-8 + * representation of Unicode strictly smaller than UTF-32. + * + * The shortest sequence requirement was introduced by: + * Corrigendum #1: UTF-8 Shortest Form + * It can be found here: + * http://www.unicode.org/versions/corrigendum1.html + * + */ + +#define UTF8_2_BITS 0xC0 +#define UTF8_3_BITS 0xE0 +#define UTF8_4_BITS 0xF0 +#define UTF8_N_BITS 0x80 +#define UTF8_2_MASK 0xE0 +#define UTF8_3_MASK 0xF0 +#define UTF8_4_MASK 0xF8 +#define UTF8_N_MASK 0xC0 +#define UTF8_V_MASK 0x3F +#define UTF8_V_SHIFT 6 + +static int utf8encode(char *str, unsigned int val) +{ + int len; + + if (val < 0x80) { + str[0] = val; + len = 1; + } else if (val < 0x800) { + str[1] = val & UTF8_V_MASK; + str[1] |= UTF8_N_BITS; + val >>= UTF8_V_SHIFT; + str[0] = val; + str[0] |= UTF8_2_BITS; + len = 2; + } else if (val < 0x10000) { + str[2] = val & UTF8_V_MASK; + str[2] |= UTF8_N_BITS; + val >>= UTF8_V_SHIFT; + str[1] = val & UTF8_V_MASK; + str[1] |= UTF8_N_BITS; + val >>= UTF8_V_SHIFT; + str[0] = val; + str[0] |= UTF8_3_BITS; + len = 3; + } else if (val < 0x110000) { + str[3] = val & UTF8_V_MASK; + str[3] |= UTF8_N_BITS; + val >>= UTF8_V_SHIFT; + str[2] = val & UTF8_V_MASK; + str[2] |= UTF8_N_BITS; + val >>= UTF8_V_SHIFT; + str[1] = val & UTF8_V_MASK; + str[1] |= UTF8_N_BITS; + val >>= UTF8_V_SHIFT; + str[0] = val; + str[0] |= UTF8_4_BITS; + len = 4; + } else { + printf("%#x: illegal val\n", val); + len = 0; + } + return len; +} + +static unsigned int utf8decode(const char *str) +{ + const unsigned char *s = (const unsigned char*)str; + unsigned int unichar = 0; + + if (*s < 0x80) { + unichar = *s; + } else if (*s < UTF8_3_BITS) { + unichar = *s++ & 0x1F; + unichar <<= UTF8_V_SHIFT; + unichar |= *s & 0x3F; + } else if (*s < UTF8_4_BITS) { + unichar = *s++ & 0x0F; + unichar <<= UTF8_V_SHIFT; + unichar |= *s++ & 0x3F; + unichar <<= UTF8_V_SHIFT; + unichar |= *s & 0x3F; + } else { + unichar = *s++ & 0x0F; + unichar <<= UTF8_V_SHIFT; + unichar |= *s++ & 0x3F; + unichar <<= UTF8_V_SHIFT; + unichar |= *s++ & 0x3F; + unichar <<= UTF8_V_SHIFT; + unichar |= *s & 0x3F; + } + return unichar; +} + +static int utf32valid(unsigned int unichar) +{ + return unichar < 0x110000; +} + +#define HANGUL_SYLLABLE(U) ((U) >= 0xAC00 && (U) <= 0xD7A3) + +#define NODE 1 +#define LEAF 0 + +struct tree { + void *root; + int childnode; + const char *type; + unsigned int maxage; + struct tree *next; + int (*leaf_equal)(void *, void *); + void (*leaf_print)(void *, int); + int (*leaf_mark)(void *); + int (*leaf_size)(void *); + int *(*leaf_index)(struct tree *, void *); + unsigned char *(*leaf_emit)(void *, unsigned char *); + int leafindex[0x110000]; + int index; +}; + +struct node { + int index; + int offset; + int mark; + int size; + struct node *parent; + void *left; + void *right; + unsigned char bitnum; + unsigned char nextbyte; + unsigned char leftnode; + unsigned char rightnode; + unsigned int keybits; + unsigned int keymask; +}; + +/* + * Example lookup function for a tree. + */ +static void *lookup(struct tree *tree, const char *key) +{ + struct node *node; + void *leaf = NULL; + + node = tree->root; + while (!leaf && node) { + if (node->nextbyte) + key++; + if (*key & (1 << (node->bitnum & 7))) { + /* Right leg */ + if (node->rightnode == NODE) { + node = node->right; + } else if (node->rightnode == LEAF) { + leaf = node->right; + } else { + node = NULL; + } + } else { + /* Left leg */ + if (node->leftnode == NODE) { + node = node->left; + } else if (node->leftnode == LEAF) { + leaf = node->left; + } else { + node = NULL; + } + } + } + + return leaf; +} + +/* + * A simple non-recursive tree walker: keep track of visits to the + * left and right branches in the leftmask and rightmask. + */ +static void tree_walk(struct tree *tree) +{ + struct node *node; + unsigned int leftmask; + unsigned int rightmask; + unsigned int bitmask; + int indent = 1; + int nodes, singletons, leaves; + + nodes = singletons = leaves = 0; + + printf("%s_%x root %p\n", tree->type, tree->maxage, tree->root); + if (tree->childnode == LEAF) { + assert(tree->root); + tree->leaf_print(tree->root, indent); + leaves = 1; + } else { + assert(tree->childnode == NODE); + node = tree->root; + leftmask = rightmask = 0; + while (node) { + printf("%*snode @ %p bitnum %d nextbyte %d" + " left %p right %p mask %x bits %x\n", + indent, "", node, + node->bitnum, node->nextbyte, + node->left, node->right, + node->keymask, node->keybits); + nodes += 1; + if (!(node->left && node->right)) + singletons += 1; + + while (node) { + bitmask = 1 << node->bitnum; + if ((leftmask & bitmask) == 0) { + leftmask |= bitmask; + if (node->leftnode == LEAF) { + assert(node->left); + tree->leaf_print(node->left, + indent+1); + leaves += 1; + } else if (node->left) { + assert(node->leftnode == NODE); + indent += 1; + node = node->left; + break; + } + } + if ((rightmask & bitmask) == 0) { + rightmask |= bitmask; + if (node->rightnode == LEAF) { + assert(node->right); + tree->leaf_print(node->right, + indent+1); + leaves += 1; + } else if (node->right) { + assert(node->rightnode == NODE); + indent += 1; + node = node->right; + break; + } + } + leftmask &= ~bitmask; + rightmask &= ~bitmask; + node = node->parent; + indent -= 1; + } + } + } + printf("nodes %d leaves %d singletons %d\n", + nodes, leaves, singletons); +} + +/* + * Allocate an initialize a new internal node. + */ +static struct node *alloc_node(struct node *parent) +{ + struct node *node; + int bitnum; + + node = malloc(sizeof(*node)); + node->left = node->right = NULL; + node->parent = parent; + node->leftnode = NODE; + node->rightnode = NODE; + node->keybits = 0; + node->keymask = 0; + node->mark = 0; + node->index = 0; + node->offset = -1; + node->size = 4; + + if (node->parent) { + bitnum = parent->bitnum; + if ((bitnum & 7) == 0) { + node->bitnum = bitnum + 7 + 8; + node->nextbyte = 1; + } else { + node->bitnum = bitnum - 1; + node->nextbyte = 0; + } + } else { + node->bitnum = 7; + node->nextbyte = 0; + } + + return node; +} + +/* + * Insert a new leaf into the tree, and collapse any subtrees that are + * fully populated and end in identical leaves. A nextbyte tagged + * internal node will not be removed to preserve the tree's integrity. + * Note that due to the structure of utf8, no nextbyte tagged node + * will be a candidate for removal. + */ +static int insert(struct tree *tree, char *key, int keylen, void *leaf) +{ + struct node *node; + struct node *parent; + void **cursor; + int keybits; + + assert(keylen >= 1 && keylen <= 4); + + node = NULL; + cursor = &tree->root; + keybits = 8 * keylen; + + /* Insert, creating path along the way. */ + while (keybits) { + if (!*cursor) + *cursor = alloc_node(node); + node = *cursor; + if (node->nextbyte) + key++; + if (*key & (1 << (node->bitnum & 7))) + cursor = &node->right; + else + cursor = &node->left; + keybits--; + } + *cursor = leaf; + + /* Merge subtrees if possible. */ + while (node) { + if (*key & (1 << (node->bitnum & 7))) + node->rightnode = LEAF; + else + node->leftnode = LEAF; + if (node->nextbyte) + break; + if (node->leftnode == NODE || node->rightnode == NODE) + break; + assert(node->left); + assert(node->right); + /* Compare */ + if (! tree->leaf_equal(node->left, node->right)) + break; + /* Keep left, drop right leaf. */ + leaf = node->left; + /* Check in parent */ + parent = node->parent; + if (!parent) { + /* root of tree! */ + tree->root = leaf; + tree->childnode = LEAF; + } else if (parent->left == node) { + parent->left = leaf; + parent->leftnode = LEAF; + if (parent->right) { + parent->keymask = 0; + parent->keybits = 0; + } else { + parent->keymask |= (1 << node->bitnum); + } + } else if (parent->right == node) { + parent->right = leaf; + parent->rightnode = LEAF; + if (parent->left) { + parent->keymask = 0; + parent->keybits = 0; + } else { + parent->keymask |= (1 << node->bitnum); + parent->keybits |= (1 << node->bitnum); + } + } else { + /* internal tree error */ + assert(0); + } + free(node); + node = parent; + } + + /* Propagate keymasks up along singleton chains. */ + while (node) { + parent = node->parent; + if (!parent) + break; + /* Nix the mask for parents with two children. */ + if (node->keymask == 0) { + parent->keymask = 0; + parent->keybits = 0; + } else if (parent->left && parent->right) { + parent->keymask = 0; + parent->keybits = 0; + } else { + assert((parent->keymask & node->keymask) == 0); + parent->keymask |= node->keymask; + parent->keymask |= (1 << parent->bitnum); + parent->keybits |= node->keybits; + if (parent->right) + parent->keybits |= (1 << parent->bitnum); + } + node = parent; + } + + return 0; +} + +/* + * Prune internal nodes. + * + * Fully populated subtrees that end at the same leaf have already + * been collapsed. There are still internal nodes that have for both + * their left and right branches a sequence of singletons that make + * identical choices and end in identical leaves. The keymask and + * keybits collected in the nodes describe the choices made in these + * singleton chains. When they are identical for the left and right + * branch of a node, and the two leaves comare identical, the node in + * question can be removed. + * + * Note that nodes with the nextbyte tag set will not be removed by + * this to ensure tree integrity. Note as well that the structure of + * utf8 ensures that these nodes would not have been candidates for + * removal in any case. + */ +static void prune(struct tree *tree) +{ + struct node *node; + struct node *left; + struct node *right; + struct node *parent; + void *leftleaf; + void *rightleaf; + unsigned int leftmask; + unsigned int rightmask; + unsigned int bitmask; + int count; + + if (verbose > 0) + printf("Pruning %s_%x\n", tree->type, tree->maxage); + + count = 0; + if (tree->childnode == LEAF) + return; + if (!tree->root) + return; + + leftmask = rightmask = 0; + node = tree->root; + while (node) { + if (node->nextbyte) + goto advance; + if (node->leftnode == LEAF) + goto advance; + if (node->rightnode == LEAF) + goto advance; + if (!node->left) + goto advance; + if (!node->right) + goto advance; + left = node->left; + right = node->right; + if (left->keymask == 0) + goto advance; + if (right->keymask == 0) + goto advance; + if (left->keymask != right->keymask) + goto advance; + if (left->keybits != right->keybits) + goto advance; + leftleaf = NULL; + while (!leftleaf) { + assert(left->left || left->right); + if (left->leftnode == LEAF) + leftleaf = left->left; + else if (left->rightnode == LEAF) + leftleaf = left->right; + else if (left->left) + left = left->left; + else if (left->right) + left = left->right; + else + assert(0); + } + rightleaf = NULL; + while (!rightleaf) { + assert(right->left || right->right); + if (right->leftnode == LEAF) + rightleaf = right->left; + else if (right->rightnode == LEAF) + rightleaf = right->right; + else if (right->left) + right = right->left; + else if (right->right) + right = right->right; + else + assert(0); + } + if (! tree->leaf_equal(leftleaf, rightleaf)) + goto advance; + /* + * This node has identical singleton-only subtrees. + * Remove it. + */ + parent = node->parent; + left = node->left; + right = node->right; + if (parent->left == node) + parent->left = left; + else if (parent->right == node) + parent->right = left; + else + assert(0); + left->parent = parent; + left->keymask |= (1 << node->bitnum); + node->left = NULL; + while (node) { + bitmask = 1 << node->bitnum; + leftmask &= ~bitmask; + rightmask &= ~bitmask; + if (node->leftnode == NODE && node->left) { + left = node->left; + free(node); + count++; + node = left; + } else if (node->rightnode == NODE && node->right) { + right = node->right; + free(node); + count++; + node = right; + } else { + node = NULL; + } + } + /* Propagate keymasks up along singleton chains. */ + node = parent; + /* Force re-check */ + bitmask = 1 << node->bitnum; + leftmask &= ~bitmask; + rightmask &= ~bitmask; + for (;;) { + if (node->left && node->right) + break; + if (node->left) { + left = node->left; + node->keymask |= left->keymask; + node->keybits |= left->keybits; + } + if (node->right) { + right = node->right; + node->keymask |= right->keymask; + node->keybits |= right->keybits; + } + node->keymask |= (1 << node->bitnum); + node = node->parent; + /* Force re-check */ + bitmask = 1 << node->bitnum; + leftmask &= ~bitmask; + rightmask &= ~bitmask; + } + advance: + bitmask = 1 << node->bitnum; + if ((leftmask & bitmask) == 0 && + node->leftnode == NODE && + node->left) { + leftmask |= bitmask; + node = node->left; + } else if ((rightmask & bitmask) == 0 && + node->rightnode == NODE && + node->right) { + rightmask |= bitmask; + node = node->right; + } else { + leftmask &= ~bitmask; + rightmask &= ~bitmask; + node = node->parent; + } + } + if (verbose > 0) + printf("Pruned %d nodes\n", count); +} + +/* + * Mark the nodes in the tree that lead to leaves that must be + * emitted. + */ +static void mark_nodes(struct tree *tree) +{ + struct node *node; + struct node *n; + unsigned int leftmask; + unsigned int rightmask; + unsigned int bitmask; + int marked; + + marked = 0; + if (verbose > 0) + printf("Marking %s_%x\n", tree->type, tree->maxage); + if (tree->childnode == LEAF) + goto done; + + assert(tree->childnode == NODE); + node = tree->root; + leftmask = rightmask = 0; + while (node) { + bitmask = 1 << node->bitnum; + if ((leftmask & bitmask) == 0) { + leftmask |= bitmask; + if (node->leftnode == LEAF) { + assert(node->left); + if (tree->leaf_mark(node->left)) { + n = node; + while (n && !n->mark) { + marked++; + n->mark = 1; + n = n->parent; + } + } + } else if (node->left) { + assert(node->leftnode == NODE); + node = node->left; + continue; + } + } + if ((rightmask & bitmask) == 0) { + rightmask |= bitmask; + if (node->rightnode == LEAF) { + assert(node->right); + if (tree->leaf_mark(node->right)) { + n = node; + while (n && !n->mark) { + marked++; + n->mark = 1; + n = n->parent; + } + } + } else if (node->right) { + assert(node->rightnode == NODE); + node = node->right; + continue; + } + } + leftmask &= ~bitmask; + rightmask &= ~bitmask; + node = node->parent; + } + + /* second pass: left siblings and singletons */ + + assert(tree->childnode == NODE); + node = tree->root; + leftmask = rightmask = 0; + while (node) { + bitmask = 1 << node->bitnum; + if ((leftmask & bitmask) == 0) { + leftmask |= bitmask; + if (node->leftnode == LEAF) { + assert(node->left); + if (tree->leaf_mark(node->left)) { + n = node; + while (n && !n->mark) { + marked++; + n->mark = 1; + n = n->parent; + } + } + } else if (node->left) { + assert(node->leftnode == NODE); + node = node->left; + if (!node->mark && node->parent->mark) { + marked++; + node->mark = 1; + } + continue; + } + } + if ((rightmask & bitmask) == 0) { + rightmask |= bitmask; + if (node->rightnode == LEAF) { + assert(node->right); + if (tree->leaf_mark(node->right)) { + n = node; + while (n && !n->mark) { + marked++; + n->mark = 1; + n = n->parent; + } + } + } else if (node->right) { + assert(node->rightnode == NODE); + node = node->right; + if (!node->mark && node->parent->mark && + !node->parent->left) { + marked++; + node->mark = 1; + } + continue; + } + } + leftmask &= ~bitmask; + rightmask &= ~bitmask; + node = node->parent; + } +done: + if (verbose > 0) + printf("Marked %d nodes\n", marked); +} + +/* + * Compute the index of each node and leaf, which is the offset in the + * emitted trie. These values must be pre-computed because relative + * offsets between nodes are used to navigate the tree. + */ +static int index_nodes(struct tree *tree, int index) +{ + struct node *node; + unsigned int leftmask; + unsigned int rightmask; + unsigned int bitmask; + int count; + int indent; + + /* Align to a cache line (or half a cache line?). */ + while (index % 64) + index++; + tree->index = index; + indent = 1; + count = 0; + + if (verbose > 0) + printf("Indexing %s_%x: %d\n", tree->type, tree->maxage, index); + if (tree->childnode == LEAF) { + index += tree->leaf_size(tree->root); + goto done; + } + + assert(tree->childnode == NODE); + node = tree->root; + leftmask = rightmask = 0; + while (node) { + if (!node->mark) + goto skip; + count++; + if (node->index != index) + node->index = index; + index += node->size; +skip: + while (node) { + bitmask = 1 << node->bitnum; + if (node->mark && (leftmask & bitmask) == 0) { + leftmask |= bitmask; + if (node->leftnode == LEAF) { + assert(node->left); + *tree->leaf_index(tree, node->left) = + index; + index += tree->leaf_size(node->left); + count++; + } else if (node->left) { + assert(node->leftnode == NODE); + indent += 1; + node = node->left; + break; + } + } + if (node->mark && (rightmask & bitmask) == 0) { + rightmask |= bitmask; + if (node->rightnode == LEAF) { + assert(node->right); + *tree->leaf_index(tree, node->right) = index; + index += tree->leaf_size(node->right); + count++; + } else if (node->right) { + assert(node->rightnode == NODE); + indent += 1; + node = node->right; + break; + } + } + leftmask &= ~bitmask; + rightmask &= ~bitmask; + node = node->parent; + indent -= 1; + } + } +done: + /* Round up to a multiple of 16 */ + while (index % 16) + index++; + if (verbose > 0) + printf("Final index %d\n", index); + return index; +} + +/* + * Mark the nodes in a subtree, helper for size_nodes(). + */ +static int mark_subtree(struct node *node) +{ + int changed; + + if (!node || node->mark) + return 0; + node->mark = 1; + node->index = node->parent->index; + changed = 1; + if (node->leftnode == NODE) + changed += mark_subtree(node->left); + if (node->rightnode == NODE) + changed += mark_subtree(node->right); + return changed; +} + +/* + * Compute the size of nodes and leaves. We start by assuming that + * each node needs to store a three-byte offset. The indexes of the + * nodes are calculated based on that, and then this function is + * called to see if the sizes of some nodes can be reduced. This is + * repeated until no more changes are seen. + */ +static int size_nodes(struct tree *tree) +{ + struct tree *next; + struct node *node; + struct node *right; + struct node *n; + unsigned int leftmask; + unsigned int rightmask; + unsigned int bitmask; + unsigned int pathbits; + unsigned int pathmask; + unsigned int nbit; + int changed; + int offset; + int size; + int indent; + + indent = 1; + changed = 0; + size = 0; + + if (verbose > 0) + printf("Sizing %s_%x\n", tree->type, tree->maxage); + if (tree->childnode == LEAF) + goto done; + + assert(tree->childnode == NODE); + pathbits = 0; + pathmask = 0; + node = tree->root; + leftmask = rightmask = 0; + while (node) { + if (!node->mark) + goto skip; + offset = 0; + if (!node->left || !node->right) { + size = 1; + } else { + if (node->rightnode == NODE) { + /* + * If the right node is not marked, + * look for a corresponding node in + * the next tree. Such a node need + * not exist. + */ + right = node->right; + next = tree->next; + while (!right->mark) { + assert(next); + n = next->root; + while (n->bitnum != node->bitnum) { + nbit = 1 << n->bitnum; + if (!(pathmask & nbit)) + break; + if (pathbits & nbit) { + if (n->rightnode == LEAF) + break; + n = n->right; + } else { + if (n->leftnode == LEAF) + break; + n = n->left; + } + } + if (n->bitnum != node->bitnum) + break; + n = n->right; + right = n; + next = next->next; + } + /* Make sure the right node is marked. */ + if (!right->mark) + changed += mark_subtree(right); + offset = right->index - node->index; + } else { + offset = *tree->leaf_index(tree, node->right); + offset -= node->index; + } + assert(offset >= 0); + assert(offset <= 0xffffff); + if (offset <= 0xff) { + size = 2; + } else if (offset <= 0xffff) { + size = 3; + } else { /* offset <= 0xffffff */ + size = 4; + } + } + if (node->size != size || node->offset != offset) { + node->size = size; + node->offset = offset; + changed++; + } +skip: + while (node) { + bitmask = 1 << node->bitnum; + pathmask |= bitmask; + if (node->mark && (leftmask & bitmask) == 0) { + leftmask |= bitmask; + if (node->leftnode == LEAF) { + assert(node->left); + } else if (node->left) { + assert(node->leftnode == NODE); + indent += 1; + node = node->left; + break; + } + } + if (node->mark && (rightmask & bitmask) == 0) { + rightmask |= bitmask; + pathbits |= bitmask; + if (node->rightnode == LEAF) { + assert(node->right); + } else if (node->right) { + assert(node->rightnode == NODE); + indent += 1; + node = node->right; + break; + } + } + leftmask &= ~bitmask; + rightmask &= ~bitmask; + pathmask &= ~bitmask; + pathbits &= ~bitmask; + node = node->parent; + indent -= 1; + } + } +done: + if (verbose > 0) + printf("Found %d changes\n", changed); + return changed; +} + +/* + * Emit a trie for the given tree into the data array. + */ +static void emit(struct tree *tree, unsigned char *data) +{ + struct node *node; + unsigned int leftmask; + unsigned int rightmask; + unsigned int bitmask; + int offlen; + int offset; + int index; + int indent; + int size; + int bytes; + int leaves; + int nodes[4]; + unsigned char byte; + + nodes[0] = nodes[1] = nodes[2] = nodes[3] = 0; + leaves = 0; + bytes = 0; + index = tree->index; + data += index; + indent = 1; + if (verbose > 0) + printf("Emitting %s_%x\n", tree->type, tree->maxage); + if (tree->childnode == LEAF) { + assert(tree->root); + tree->leaf_emit(tree->root, data); + size = tree->leaf_size(tree->root); + index += size; + leaves++; + goto done; + } + + assert(tree->childnode == NODE); + node = tree->root; + leftmask = rightmask = 0; + while (node) { + if (!node->mark) + goto skip; + assert(node->offset != -1); + assert(node->index == index); + + byte = 0; + if (node->nextbyte) + byte |= NEXTBYTE; + byte |= (node->bitnum & BITNUM); + if (node->left && node->right) { + if (node->leftnode == NODE) + byte |= LEFTNODE; + if (node->rightnode == NODE) + byte |= RIGHTNODE; + if (node->offset <= 0xff) + offlen = 1; + else if (node->offset <= 0xffff) + offlen = 2; + else + offlen = 3; + nodes[offlen]++; + offset = node->offset; + byte |= offlen << OFFLEN_SHIFT; + *data++ = byte; + index++; + while (offlen--) { + *data++ = offset & 0xff; + index++; + offset >>= 8; + } + } else if (node->left) { + if (node->leftnode == NODE) + byte |= TRIENODE; + nodes[0]++; + *data++ = byte; + index++; + } else if (node->right) { + byte |= RIGHTNODE; + if (node->rightnode == NODE) + byte |= TRIENODE; + nodes[0]++; + *data++ = byte; + index++; + } else { + assert(0); + } +skip: + while (node) { + bitmask = 1 << node->bitnum; + if (node->mark && (leftmask & bitmask) == 0) { + leftmask |= bitmask; + if (node->leftnode == LEAF) { + assert(node->left); + data = tree->leaf_emit(node->left, + data); + size = tree->leaf_size(node->left); + index += size; + bytes += size; + leaves++; + } else if (node->left) { + assert(node->leftnode == NODE); + indent += 1; + node = node->left; + break; + } + } + if (node->mark && (rightmask & bitmask) == 0) { + rightmask |= bitmask; + if (node->rightnode == LEAF) { + assert(node->right); + data = tree->leaf_emit(node->right, + data); + size = tree->leaf_size(node->right); + index += size; + bytes += size; + leaves++; + } else if (node->right) { + assert(node->rightnode == NODE); + indent += 1; + node = node->right; + break; + } + } + leftmask &= ~bitmask; + rightmask &= ~bitmask; + node = node->parent; + indent -= 1; + } + } +done: + if (verbose > 0) { + printf("Emitted %d (%d) leaves", + leaves, bytes); + printf(" %d (%d+%d+%d+%d) nodes", + nodes[0] + nodes[1] + nodes[2] + nodes[3], + nodes[0], nodes[1], nodes[2], nodes[3]); + printf(" %d total\n", index - tree->index); + } +} + +/* ------------------------------------------------------------------ */ + +/* + * Unicode data. + * + * We need to keep track of the Canonical Combining Class, the Age, + * and decompositions for a code point. + * + * For the Age, we store the index into the ages table. Effectively + * this is a generation number that the table maps to a unicode + * version. + * + * The correction field is used to indicate that this entry is in the + * corrections array, which contains decompositions that were + * corrected in later revisions. The value of the correction field is + * the Unicode version in which the mapping was corrected. + */ +struct unicode_data { + unsigned int code; + int ccc; + int gen; + int correction; + unsigned int *utf32nfdi; + unsigned int *utf32nfdicf; + char *utf8nfdi; + char *utf8nfdicf; +}; + +struct unicode_data unicode_data[0x110000]; +struct unicode_data *corrections; +int corrections_count; + +struct tree *nfdi_tree; +struct tree *nfdicf_tree; + +struct tree *trees; +int trees_count; + +/* + * Check the corrections array to see if this entry was corrected at + * some point. + */ +static struct unicode_data *corrections_lookup(struct unicode_data *u) +{ + int i; + + for (i = 0; i != corrections_count; i++) + if (u->code == corrections[i].code) + return &corrections[i]; + return u; +} + +static int nfdi_equal(void *l, void *r) +{ + struct unicode_data *left = l; + struct unicode_data *right = r; + + if (left->gen != right->gen) + return 0; + if (left->ccc != right->ccc) + return 0; + if (left->utf8nfdi && right->utf8nfdi && + strcmp(left->utf8nfdi, right->utf8nfdi) == 0) + return 1; + if (left->utf8nfdi || right->utf8nfdi) + return 0; + return 1; +} + +static int nfdicf_equal(void *l, void *r) +{ + struct unicode_data *left = l; + struct unicode_data *right = r; + + if (left->gen != right->gen) + return 0; + if (left->ccc != right->ccc) + return 0; + if (left->utf8nfdicf && right->utf8nfdicf && + strcmp(left->utf8nfdicf, right->utf8nfdicf) == 0) + return 1; + if (left->utf8nfdicf && right->utf8nfdicf) + return 0; + if (left->utf8nfdicf || right->utf8nfdicf) + return 0; + if (left->utf8nfdi && right->utf8nfdi && + strcmp(left->utf8nfdi, right->utf8nfdi) == 0) + return 1; + if (left->utf8nfdi || right->utf8nfdi) + return 0; + return 1; +} + +static void nfdi_print(void *l, int indent) +{ + struct unicode_data *leaf = l; + + printf("%*sleaf @ %p code %X ccc %d gen %d", indent, "", leaf, + leaf->code, leaf->ccc, leaf->gen); + + if (leaf->utf8nfdi && leaf->utf8nfdi[0] == HANGUL) + printf(" nfdi \"%s\"", "HANGUL SYLLABLE"); + else if (leaf->utf8nfdi) + printf(" nfdi \"%s\"", (const char*)leaf->utf8nfdi); + + printf("\n"); +} + +static void nfdicf_print(void *l, int indent) +{ + struct unicode_data *leaf = l; + + printf("%*sleaf @ %p code %X ccc %d gen %d", indent, "", leaf, + leaf->code, leaf->ccc, leaf->gen); + + if (leaf->utf8nfdicf) + printf(" nfdicf \"%s\"", (const char*)leaf->utf8nfdicf); + else if (leaf->utf8nfdi && leaf->utf8nfdi[0] == HANGUL) + printf(" nfdi \"%s\"", "HANGUL SYLLABLE"); + else if (leaf->utf8nfdi) + printf(" nfdi \"%s\"", (const char*)leaf->utf8nfdi); + printf("\n"); +} + +static int nfdi_mark(void *l) +{ + return 1; +} + +static int nfdicf_mark(void *l) +{ + struct unicode_data *leaf = l; + + if (leaf->utf8nfdicf) + return 1; + return 0; +} + +static int correction_mark(void *l) +{ + struct unicode_data *leaf = l; + + return leaf->correction; +} + +static int nfdi_size(void *l) +{ + struct unicode_data *leaf = l; + int size = 2; + + if (HANGUL_SYLLABLE(leaf->code)) + size += 1; + else if (leaf->utf8nfdi) + size += strlen(leaf->utf8nfdi) + 1; + return size; +} + +static int nfdicf_size(void *l) +{ + struct unicode_data *leaf = l; + int size = 2; + + if (HANGUL_SYLLABLE(leaf->code)) + size += 1; + else if (leaf->utf8nfdicf) + size += strlen(leaf->utf8nfdicf) + 1; + else if (leaf->utf8nfdi) + size += strlen(leaf->utf8nfdi) + 1; + return size; +} + +static int *nfdi_index(struct tree *tree, void *l) +{ + struct unicode_data *leaf = l; + + return &tree->leafindex[leaf->code]; +} + +static int *nfdicf_index(struct tree *tree, void *l) +{ + struct unicode_data *leaf = l; + + return &tree->leafindex[leaf->code]; +} + +static unsigned char *nfdi_emit(void *l, unsigned char *data) +{ + struct unicode_data *leaf = l; + unsigned char *s; + + *data++ = leaf->gen; + + if (HANGUL_SYLLABLE(leaf->code)) { + *data++ = DECOMPOSE; + *data++ = HANGUL; + } else if (leaf->utf8nfdi) { + *data++ = DECOMPOSE; + s = (unsigned char*)leaf->utf8nfdi; + while ((*data++ = *s++) != 0) + ; + } else { + *data++ = leaf->ccc; + } + return data; +} + +static unsigned char *nfdicf_emit(void *l, unsigned char *data) +{ + struct unicode_data *leaf = l; + unsigned char *s; + + *data++ = leaf->gen; + + if (HANGUL_SYLLABLE(leaf->code)) { + *data++ = DECOMPOSE; + *data++ = HANGUL; + } else if (leaf->utf8nfdicf) { + *data++ = DECOMPOSE; + s = (unsigned char*)leaf->utf8nfdicf; + while ((*data++ = *s++) != 0) + ; + } else if (leaf->utf8nfdi) { + *data++ = DECOMPOSE; + s = (unsigned char*)leaf->utf8nfdi; + while ((*data++ = *s++) != 0) + ; + } else { + *data++ = leaf->ccc; + } + return data; +} + +static void utf8_create(struct unicode_data *data) +{ + char utf[18*4+1]; + char *u; + unsigned int *um; + int i; + + if (data->utf8nfdi) { + assert(data->utf8nfdi[0] == HANGUL); + return; + } + + u = utf; + um = data->utf32nfdi; + if (um) { + for (i = 0; um[i]; i++) + u += utf8encode(u, um[i]); + *u = '\0'; + data->utf8nfdi = strdup(utf); + } + u = utf; + um = data->utf32nfdicf; + if (um) { + for (i = 0; um[i]; i++) + u += utf8encode(u, um[i]); + *u = '\0'; + if (!data->utf8nfdi || strcmp(data->utf8nfdi, utf)) + data->utf8nfdicf = strdup(utf); + } +} + +static void utf8_init(void) +{ + unsigned int unichar; + int i; + + for (unichar = 0; unichar != 0x110000; unichar++) + utf8_create(&unicode_data[unichar]); + + for (i = 0; i != corrections_count; i++) + utf8_create(&corrections[i]); +} + +static void trees_init(void) +{ + struct unicode_data *data; + unsigned int maxage; + unsigned int nextage; + int count; + int i; + int j; + + /* Count the number of different ages. */ + count = 0; + nextage = (unsigned int)-1; + do { + maxage = nextage; + nextage = 0; + for (i = 0; i <= corrections_count; i++) { + data = &corrections[i]; + if (nextage < data->correction && + data->correction < maxage) + nextage = data->correction; + } + count++; + } while (nextage); + + /* Two trees per age: nfdi and nfdicf */ + trees_count = count * 2; + trees = calloc(trees_count, sizeof(struct tree)); + + /* Assign ages to the trees. */ + count = trees_count; + nextage = (unsigned int)-1; + do { + maxage = nextage; + trees[--count].maxage = maxage; + trees[--count].maxage = maxage; + nextage = 0; + for (i = 0; i <= corrections_count; i++) { + data = &corrections[i]; + if (nextage < data->correction && + data->correction < maxage) + nextage = data->correction; + } + } while (nextage); + + /* The ages assigned above are off by one. */ + for (i = 0; i != trees_count; i++) { + j = 0; + while (ages[j] < trees[i].maxage) + j++; + trees[i].maxage = ages[j-1]; + } + + /* Set up the forwarding between trees. */ + trees[trees_count-2].next = &trees[trees_count-1]; + trees[trees_count-1].leaf_mark = nfdi_mark; + trees[trees_count-2].leaf_mark = nfdicf_mark; + for (i = 0; i != trees_count-2; i += 2) { + trees[i].next = &trees[trees_count-2]; + trees[i].leaf_mark = correction_mark; + trees[i+1].next = &trees[trees_count-1]; + trees[i+1].leaf_mark = correction_mark; + } + + /* Assign the callouts. */ + for (i = 0; i != trees_count; i += 2) { + trees[i].type = "nfdicf"; + trees[i].leaf_equal = nfdicf_equal; + trees[i].leaf_print = nfdicf_print; + trees[i].leaf_size = nfdicf_size; + trees[i].leaf_index = nfdicf_index; + trees[i].leaf_emit = nfdicf_emit; + + trees[i+1].type = "nfdi"; + trees[i+1].leaf_equal = nfdi_equal; + trees[i+1].leaf_print = nfdi_print; + trees[i+1].leaf_size = nfdi_size; + trees[i+1].leaf_index = nfdi_index; + trees[i+1].leaf_emit = nfdi_emit; + } + + /* Finish init. */ + for (i = 0; i != trees_count; i++) + trees[i].childnode = NODE; +} + +static void trees_populate(void) +{ + struct unicode_data *data; + unsigned int unichar; + char keyval[4]; + int keylen; + int i; + + for (i = 0; i != trees_count; i++) { + if (verbose > 0) { + printf("Populating %s_%x\n", + trees[i].type, trees[i].maxage); + } + for (unichar = 0; unichar != 0x110000; unichar++) { + if (unicode_data[unichar].gen < 0) + continue; + keylen = utf8encode(keyval, unichar); + data = corrections_lookup(&unicode_data[unichar]); + if (data->correction <= trees[i].maxage) + data = &unicode_data[unichar]; + insert(&trees[i], keyval, keylen, data); + } + } +} + +static void trees_reduce(void) +{ + int i; + int size; + int changed; + + for (i = 0; i != trees_count; i++) + prune(&trees[i]); + for (i = 0; i != trees_count; i++) + mark_nodes(&trees[i]); + do { + size = 0; + for (i = 0; i != trees_count; i++) + size = index_nodes(&trees[i], size); + changed = 0; + for (i = 0; i != trees_count; i++) + changed += size_nodes(&trees[i]); + } while (changed); + + utf8data = calloc(size, 1); + utf8data_size = size; + for (i = 0; i != trees_count; i++) + emit(&trees[i], utf8data); + + if (verbose > 0) { + for (i = 0; i != trees_count; i++) { + printf("%s_%x idx %d\n", + trees[i].type, trees[i].maxage, trees[i].index); + } + } + + nfdi = utf8data + trees[trees_count-1].index; + nfdicf = utf8data + trees[trees_count-2].index; + + nfdi_tree = &trees[trees_count-1]; + nfdicf_tree = &trees[trees_count-2]; +} + +static void verify(struct tree *tree) +{ + struct unicode_data *data; + utf8leaf_t *leaf; + unsigned int unichar; + char key[4]; + unsigned char hangul[UTF8HANGULLEAF]; + int report; + int nocf; + + if (verbose > 0) + printf("Verifying %s_%x\n", tree->type, tree->maxage); + nocf = strcmp(tree->type, "nfdicf"); + + for (unichar = 0; unichar != 0x110000; unichar++) { + report = 0; + data = corrections_lookup(&unicode_data[unichar]); + if (data->correction <= tree->maxage) + data = &unicode_data[unichar]; + utf8encode(key,unichar); + leaf = utf8lookup(tree, hangul, key); + + if (!leaf) { + if (data->gen != -1) + report++; + if (unichar < 0xd800 || unichar > 0xdfff) + report++; + } else { + if (unichar >= 0xd800 && unichar <= 0xdfff) + report++; + if (data->gen == -1) + report++; + if (data->gen != LEAF_GEN(leaf)) + report++; + if (LEAF_CCC(leaf) == DECOMPOSE) { + if (HANGUL_SYLLABLE(data->code)) { + if (data->utf8nfdi[0] != HANGUL) + report++; + } else if (nocf) { + if (!data->utf8nfdi) { + report++; + } else if (strcmp(data->utf8nfdi, + LEAF_STR(leaf))) { + report++; + } + } else { + if (!data->utf8nfdicf && + !data->utf8nfdi) { + report++; + } else if (data->utf8nfdicf) { + if (strcmp(data->utf8nfdicf, + LEAF_STR(leaf))) + report++; + } else if (strcmp(data->utf8nfdi, + LEAF_STR(leaf))) { + report++; + } + } + } else if (data->ccc != LEAF_CCC(leaf)) { + report++; + } + } + if (report) { + printf("%X code %X gen %d ccc %d" + " nfdi -> \"%s\"", + unichar, data->code, data->gen, + data->ccc, + data->utf8nfdi); + if (leaf) { + printf(" gen %d ccc %d" + " nfdi -> \"%s\"", + LEAF_GEN(leaf), + LEAF_CCC(leaf), + LEAF_CCC(leaf) == DECOMPOSE ? + LEAF_STR(leaf) : ""); + } + printf("\n"); + } + } +} + +static void trees_verify(void) +{ + int i; + + for (i = 0; i != trees_count; i++) + verify(&trees[i]); +} + +/* ------------------------------------------------------------------ */ + +static void help(void) +{ + printf("Usage: %s [options]\n", argv0); + printf("\n"); + printf("This program creates an a data trie used for parsing and\n"); + printf("normalization of UTF-8 strings. The trie is derived from\n"); + printf("a set of input files from the Unicode character database\n"); + printf("found at: http://www.unicode.org/Public/UCD/latest/ucd/\n"); + printf("\n"); + printf("The generated tree supports two normalization forms:\n"); + printf("\n"); + printf("\tnfdi:\n"); + printf("\t- Apply unicode normalization form NFD.\n"); + printf("\t- Remove any Default_Ignorable_Code_Point.\n"); + printf("\n"); + printf("\tnfdicf:\n"); + printf("\t- Apply unicode normalization form NFD.\n"); + printf("\t- Remove any Default_Ignorable_Code_Point.\n"); + printf("\t- Apply a full casefold (C + F).\n"); + printf("\n"); + printf("These forms were chosen as being most useful when dealing\n"); + printf("with file names: NFD catches most cases where characters\n"); + printf("should be considered equivalent. The ignorables are mostly\n"); + printf("invisible, making names hard to type.\n"); + printf("\n"); + printf("The options to specify the files to be used are listed\n"); + printf("below with their default values, which are the names used\n"); + printf("by version 11.0.0 of the Unicode Character Database.\n"); + printf("\n"); + printf("The input files:\n"); + printf("\t-a %s\n", AGE_NAME); + printf("\t-c %s\n", CCC_NAME); + printf("\t-p %s\n", PROP_NAME); + printf("\t-d %s\n", DATA_NAME); + printf("\t-f %s\n", FOLD_NAME); + printf("\t-n %s\n", NORM_NAME); + printf("\n"); + printf("Additionally, the generated tables are tested using:\n"); + printf("\t-t %s\n", TEST_NAME); + printf("\n"); + printf("Finally, the output file:\n"); + printf("\t-o %s\n", UTF8_NAME); + printf("\n"); +} + +static void usage(void) +{ + help(); + exit(1); +} + +static void open_fail(const char *name, int error) +{ + printf("Error %d opening %s: %s\n", error, name, strerror(error)); + exit(1); +} + +static void file_fail(const char *filename) +{ + printf("Error parsing %s\n", filename); + exit(1); +} + +static void line_fail(const char *filename, const char *line) +{ + printf("Error parsing %s:%s\n", filename, line); + exit(1); +} + +/* ------------------------------------------------------------------ */ + +static void print_utf32(unsigned int *utf32str) +{ + int i; + + for (i = 0; utf32str[i]; i++) + printf(" %X", utf32str[i]); +} + +static void print_utf32nfdi(unsigned int unichar) +{ + printf(" %X ->", unichar); + print_utf32(unicode_data[unichar].utf32nfdi); + printf("\n"); +} + +static void print_utf32nfdicf(unsigned int unichar) +{ + printf(" %X ->", unichar); + print_utf32(unicode_data[unichar].utf32nfdicf); + printf("\n"); +} + +/* ------------------------------------------------------------------ */ + +static void age_init(void) +{ + FILE *file; + unsigned int first; + unsigned int last; + unsigned int unichar; + unsigned int major; + unsigned int minor; + unsigned int revision; + int gen; + int count; + int ret; + + if (verbose > 0) + printf("Parsing %s\n", age_name); + + file = fopen(age_name, "r"); + if (!file) + open_fail(age_name, errno); + count = 0; + + gen = 0; + while (fgets(line, LINESIZE, file)) { + ret = sscanf(line, "# Age=V%d_%d_%d", + &major, &minor, &revision); + if (ret == 3) { + ages_count++; + if (verbose > 1) + printf(" Age V%d_%d_%d\n", + major, minor, revision); + if (!age_valid(major, minor, revision)) + line_fail(age_name, line); + continue; + } + ret = sscanf(line, "# Age=V%d_%d", &major, &minor); + if (ret == 2) { + ages_count++; + if (verbose > 1) + printf(" Age V%d_%d\n", major, minor); + if (!age_valid(major, minor, 0)) + line_fail(age_name, line); + continue; + } + } + + /* We must have found something above. */ + if (verbose > 1) + printf("%d age entries\n", ages_count); + if (ages_count == 0 || ages_count > MAXGEN) + file_fail(age_name); + + /* There is a 0 entry. */ + ages_count++; + ages = calloc(ages_count + 1, sizeof(*ages)); + /* And a guard entry. */ + ages[ages_count] = (unsigned int)-1; + + rewind(file); + count = 0; + gen = 0; + while (fgets(line, LINESIZE, file)) { + ret = sscanf(line, "# Age=V%d_%d_%d", + &major, &minor, &revision); + if (ret == 3) { + ages[++gen] = + UNICODE_AGE(major, minor, revision); + if (verbose > 1) + printf(" Age V%d_%d_%d = gen %d\n", + major, minor, revision, gen); + if (!age_valid(major, minor, revision)) + line_fail(age_name, line); + continue; + } + ret = sscanf(line, "# Age=V%d_%d", &major, &minor); + if (ret == 2) { + ages[++gen] = UNICODE_AGE(major, minor, 0); + if (verbose > 1) + printf(" Age V%d_%d = %d\n", + major, minor, gen); + if (!age_valid(major, minor, 0)) + line_fail(age_name, line); + continue; + } + ret = sscanf(line, "%X..%X ; %d.%d #", + &first, &last, &major, &minor); + if (ret == 4) { + for (unichar = first; unichar <= last; unichar++) + unicode_data[unichar].gen = gen; + count += 1 + last - first; + if (verbose > 1) + printf(" %X..%X gen %d\n", first, last, gen); + if (!utf32valid(first) || !utf32valid(last)) + line_fail(age_name, line); + continue; + } + ret = sscanf(line, "%X ; %d.%d #", &unichar, &major, &minor); + if (ret == 3) { + unicode_data[unichar].gen = gen; + count++; + if (verbose > 1) + printf(" %X gen %d\n", unichar, gen); + if (!utf32valid(unichar)) + line_fail(age_name, line); + continue; + } + } + unicode_maxage = ages[gen]; + fclose(file); + + /* Nix surrogate block */ + if (verbose > 1) + printf(" Removing surrogate block D800..DFFF\n"); + for (unichar = 0xd800; unichar <= 0xdfff; unichar++) + unicode_data[unichar].gen = -1; + + if (verbose > 0) + printf("Found %d entries\n", count); + if (count == 0) + file_fail(age_name); +} + +static void ccc_init(void) +{ + FILE *file; + unsigned int first; + unsigned int last; + unsigned int unichar; + unsigned int value; + int count; + int ret; + + if (verbose > 0) + printf("Parsing %s\n", ccc_name); + + file = fopen(ccc_name, "r"); + if (!file) + open_fail(ccc_name, errno); + + count = 0; + while (fgets(line, LINESIZE, file)) { + ret = sscanf(line, "%X..%X ; %d #", &first, &last, &value); + if (ret == 3) { + for (unichar = first; unichar <= last; unichar++) { + unicode_data[unichar].ccc = value; + count++; + } + if (verbose > 1) + printf(" %X..%X ccc %d\n", first, last, value); + if (!utf32valid(first) || !utf32valid(last)) + line_fail(ccc_name, line); + continue; + } + ret = sscanf(line, "%X ; %d #", &unichar, &value); + if (ret == 2) { + unicode_data[unichar].ccc = value; + count++; + if (verbose > 1) + printf(" %X ccc %d\n", unichar, value); + if (!utf32valid(unichar)) + line_fail(ccc_name, line); + continue; + } + } + fclose(file); + + if (verbose > 0) + printf("Found %d entries\n", count); + if (count == 0) + file_fail(ccc_name); +} + +static int ignore_compatibility_form(char *type) +{ + int i; + char *ignored_types[] = {"font", "noBreak", "initial", "medial", + "final", "isolated", "circle", "super", + "sub", "vertical", "wide", "narrow", + "small", "square", "fraction", "compat"}; + + for (i = 0 ; i < ARRAY_SIZE(ignored_types); i++) + if (strcmp(type, ignored_types[i]) == 0) + return 1; + return 0; +} + +static void nfdi_init(void) +{ + FILE *file; + unsigned int unichar; + unsigned int mapping[19]; /* Magic - guaranteed not to be exceeded. */ + char *s; + char *type; + unsigned int *um; + int count; + int i; + int ret; + + if (verbose > 0) + printf("Parsing %s\n", data_name); + file = fopen(data_name, "r"); + if (!file) + open_fail(data_name, errno); + + count = 0; + while (fgets(line, LINESIZE, file)) { + ret = sscanf(line, "%X;%*[^;];%*[^;];%*[^;];%*[^;];%[^;];", + &unichar, buf0); + if (ret != 2) + continue; + if (!utf32valid(unichar)) + line_fail(data_name, line); + + s = buf0; + /* skip over */ + if (*s == '<') { + type = ++s; + while (*++s != '>'); + *s++ = '\0'; + if(ignore_compatibility_form(type)) + continue; + } + /* decode the decomposition into UTF-32 */ + i = 0; + while (*s) { + mapping[i] = strtoul(s, &s, 16); + if (!utf32valid(mapping[i])) + line_fail(data_name, line); + i++; + } + mapping[i++] = 0; + + um = malloc(i * sizeof(unsigned int)); + memcpy(um, mapping, i * sizeof(unsigned int)); + unicode_data[unichar].utf32nfdi = um; + + if (verbose > 1) + print_utf32nfdi(unichar); + count++; + } + fclose(file); + if (verbose > 0) + printf("Found %d entries\n", count); + if (count == 0) + file_fail(data_name); +} + +static void nfdicf_init(void) +{ + FILE *file; + unsigned int unichar; + unsigned int mapping[19]; /* Magic - guaranteed not to be exceeded. */ + char status; + char *s; + unsigned int *um; + int i; + int count; + int ret; + + if (verbose > 0) + printf("Parsing %s\n", fold_name); + file = fopen(fold_name, "r"); + if (!file) + open_fail(fold_name, errno); + + count = 0; + while (fgets(line, LINESIZE, file)) { + ret = sscanf(line, "%X; %c; %[^;];", &unichar, &status, buf0); + if (ret != 3) + continue; + if (!utf32valid(unichar)) + line_fail(fold_name, line); + /* Use the C+F casefold. */ + if (status != 'C' && status != 'F') + continue; + s = buf0; + if (*s == '<') + while (*s++ != ' ') + ; + i = 0; + while (*s) { + mapping[i] = strtoul(s, &s, 16); + if (!utf32valid(mapping[i])) + line_fail(fold_name, line); + i++; + } + mapping[i++] = 0; + + um = malloc(i * sizeof(unsigned int)); + memcpy(um, mapping, i * sizeof(unsigned int)); + unicode_data[unichar].utf32nfdicf = um; + + if (verbose > 1) + print_utf32nfdicf(unichar); + count++; + } + fclose(file); + if (verbose > 0) + printf("Found %d entries\n", count); + if (count == 0) + file_fail(fold_name); +} + +static void ignore_init(void) +{ + FILE *file; + unsigned int unichar; + unsigned int first; + unsigned int last; + unsigned int *um; + int count; + int ret; + + if (verbose > 0) + printf("Parsing %s\n", prop_name); + file = fopen(prop_name, "r"); + if (!file) + open_fail(prop_name, errno); + assert(file); + count = 0; + while (fgets(line, LINESIZE, file)) { + ret = sscanf(line, "%X..%X ; %s # ", &first, &last, buf0); + if (ret == 3) { + if (strcmp(buf0, "Default_Ignorable_Code_Point")) + continue; + if (!utf32valid(first) || !utf32valid(last)) + line_fail(prop_name, line); + for (unichar = first; unichar <= last; unichar++) { + free(unicode_data[unichar].utf32nfdi); + um = malloc(sizeof(unsigned int)); + *um = 0; + unicode_data[unichar].utf32nfdi = um; + free(unicode_data[unichar].utf32nfdicf); + um = malloc(sizeof(unsigned int)); + *um = 0; + unicode_data[unichar].utf32nfdicf = um; + count++; + } + if (verbose > 1) + printf(" %X..%X Default_Ignorable_Code_Point\n", + first, last); + continue; + } + ret = sscanf(line, "%X ; %s # ", &unichar, buf0); + if (ret == 2) { + if (strcmp(buf0, "Default_Ignorable_Code_Point")) + continue; + if (!utf32valid(unichar)) + line_fail(prop_name, line); + free(unicode_data[unichar].utf32nfdi); + um = malloc(sizeof(unsigned int)); + *um = 0; + unicode_data[unichar].utf32nfdi = um; + free(unicode_data[unichar].utf32nfdicf); + um = malloc(sizeof(unsigned int)); + *um = 0; + unicode_data[unichar].utf32nfdicf = um; + if (verbose > 1) + printf(" %X Default_Ignorable_Code_Point\n", + unichar); + count++; + continue; + } + } + fclose(file); + + if (verbose > 0) + printf("Found %d entries\n", count); + if (count == 0) + file_fail(prop_name); +} + +static void corrections_init(void) +{ + FILE *file; + unsigned int unichar; + unsigned int major; + unsigned int minor; + unsigned int revision; + unsigned int age; + unsigned int *um; + unsigned int mapping[19]; /* Magic - guaranteed not to be exceeded. */ + char *s; + int i; + int count; + int ret; + + if (verbose > 0) + printf("Parsing %s\n", norm_name); + file = fopen(norm_name, "r"); + if (!file) + open_fail(norm_name, errno); + + count = 0; + while (fgets(line, LINESIZE, file)) { + ret = sscanf(line, "%X;%[^;];%[^;];%d.%d.%d #", + &unichar, buf0, buf1, + &major, &minor, &revision); + if (ret != 6) + continue; + if (!utf32valid(unichar) || !age_valid(major, minor, revision)) + line_fail(norm_name, line); + count++; + } + corrections = calloc(count, sizeof(struct unicode_data)); + corrections_count = count; + rewind(file); + + count = 0; + while (fgets(line, LINESIZE, file)) { + ret = sscanf(line, "%X;%[^;];%[^;];%d.%d.%d #", + &unichar, buf0, buf1, + &major, &minor, &revision); + if (ret != 6) + continue; + if (!utf32valid(unichar) || !age_valid(major, minor, revision)) + line_fail(norm_name, line); + corrections[count] = unicode_data[unichar]; + assert(corrections[count].code == unichar); + age = UNICODE_AGE(major, minor, revision); + corrections[count].correction = age; + + i = 0; + s = buf0; + while (*s) { + mapping[i] = strtoul(s, &s, 16); + if (!utf32valid(mapping[i])) + line_fail(norm_name, line); + i++; + } + mapping[i++] = 0; + + um = malloc(i * sizeof(unsigned int)); + memcpy(um, mapping, i * sizeof(unsigned int)); + corrections[count].utf32nfdi = um; + + if (verbose > 1) + printf(" %X -> %s -> %s V%d_%d_%d\n", + unichar, buf0, buf1, major, minor, revision); + count++; + } + fclose(file); + + if (verbose > 0) + printf("Found %d entries\n", count); + if (count == 0) + file_fail(norm_name); +} + +/* ------------------------------------------------------------------ */ + +/* + * Hangul decomposition (algorithm from Section 3.12 of Unicode 6.3.0) + * + * AC00;;Lo;0;L;;;;;N;;;;; + * D7A3;;Lo;0;L;;;;;N;;;;; + * + * SBase = 0xAC00 + * LBase = 0x1100 + * VBase = 0x1161 + * TBase = 0x11A7 + * LCount = 19 + * VCount = 21 + * TCount = 28 + * NCount = 588 (VCount * TCount) + * SCount = 11172 (LCount * NCount) + * + * Decomposition: + * SIndex = s - SBase + * + * LV (Canonical/Full) + * LIndex = SIndex / NCount + * VIndex = (Sindex % NCount) / TCount + * LPart = LBase + LIndex + * VPart = VBase + VIndex + * + * LVT (Canonical) + * LVIndex = (SIndex / TCount) * TCount + * TIndex = (Sindex % TCount) + * LVPart = SBase + LVIndex + * TPart = TBase + TIndex + * + * LVT (Full) + * LIndex = SIndex / NCount + * VIndex = (Sindex % NCount) / TCount + * TIndex = (Sindex % TCount) + * LPart = LBase + LIndex + * VPart = VBase + VIndex + * if (TIndex == 0) { + * d = + * } else { + * TPart = TBase + TIndex + * d = + * } + * + */ + +static void hangul_decompose(void) +{ + unsigned int sb = 0xAC00; + unsigned int lb = 0x1100; + unsigned int vb = 0x1161; + unsigned int tb = 0x11a7; + /* unsigned int lc = 19; */ + unsigned int vc = 21; + unsigned int tc = 28; + unsigned int nc = (vc * tc); + /* unsigned int sc = (lc * nc); */ + unsigned int unichar; + unsigned int mapping[4]; + unsigned int *um; + int count; + int i; + + if (verbose > 0) + printf("Decomposing hangul\n"); + /* Hangul */ + count = 0; + for (unichar = 0xAC00; unichar <= 0xD7A3; unichar++) { + unsigned int si = unichar - sb; + unsigned int li = si / nc; + unsigned int vi = (si % nc) / tc; + unsigned int ti = si % tc; + + i = 0; + mapping[i++] = lb + li; + mapping[i++] = vb + vi; + if (ti) + mapping[i++] = tb + ti; + mapping[i++] = 0; + + assert(!unicode_data[unichar].utf32nfdi); + um = malloc(i * sizeof(unsigned int)); + memcpy(um, mapping, i * sizeof(unsigned int)); + unicode_data[unichar].utf32nfdi = um; + + assert(!unicode_data[unichar].utf32nfdicf); + um = malloc(i * sizeof(unsigned int)); + memcpy(um, mapping, i * sizeof(unsigned int)); + unicode_data[unichar].utf32nfdicf = um; + + /* + * Add a cookie as a reminder that the hangul syllable + * decompositions must not be stored in the generated + * trie. + */ + unicode_data[unichar].utf8nfdi = malloc(2); + unicode_data[unichar].utf8nfdi[0] = HANGUL; + unicode_data[unichar].utf8nfdi[1] = '\0'; + + if (verbose > 1) + print_utf32nfdi(unichar); + + count++; + } + if (verbose > 0) + printf("Created %d entries\n", count); +} + +static void nfdi_decompose(void) +{ + unsigned int unichar; + unsigned int mapping[19]; /* Magic - guaranteed not to be exceeded. */ + unsigned int *um; + unsigned int *dc; + int count; + int i; + int j; + int ret; + + if (verbose > 0) + printf("Decomposing nfdi\n"); + + count = 0; + for (unichar = 0; unichar != 0x110000; unichar++) { + if (!unicode_data[unichar].utf32nfdi) + continue; + for (;;) { + ret = 1; + i = 0; + um = unicode_data[unichar].utf32nfdi; + while (*um) { + dc = unicode_data[*um].utf32nfdi; + if (dc) { + for (j = 0; dc[j]; j++) + mapping[i++] = dc[j]; + ret = 0; + } else { + mapping[i++] = *um; + } + um++; + } + mapping[i++] = 0; + if (ret) + break; + free(unicode_data[unichar].utf32nfdi); + um = malloc(i * sizeof(unsigned int)); + memcpy(um, mapping, i * sizeof(unsigned int)); + unicode_data[unichar].utf32nfdi = um; + } + /* Add this decomposition to nfdicf if there is no entry. */ + if (!unicode_data[unichar].utf32nfdicf) { + um = malloc(i * sizeof(unsigned int)); + memcpy(um, mapping, i * sizeof(unsigned int)); + unicode_data[unichar].utf32nfdicf = um; + } + if (verbose > 1) + print_utf32nfdi(unichar); + count++; + } + if (verbose > 0) + printf("Processed %d entries\n", count); +} + +static void nfdicf_decompose(void) +{ + unsigned int unichar; + unsigned int mapping[19]; /* Magic - guaranteed not to be exceeded. */ + unsigned int *um; + unsigned int *dc; + int count; + int i; + int j; + int ret; + + if (verbose > 0) + printf("Decomposing nfdicf\n"); + count = 0; + for (unichar = 0; unichar != 0x110000; unichar++) { + if (!unicode_data[unichar].utf32nfdicf) + continue; + for (;;) { + ret = 1; + i = 0; + um = unicode_data[unichar].utf32nfdicf; + while (*um) { + dc = unicode_data[*um].utf32nfdicf; + if (dc) { + for (j = 0; dc[j]; j++) + mapping[i++] = dc[j]; + ret = 0; + } else { + mapping[i++] = *um; + } + um++; + } + mapping[i++] = 0; + if (ret) + break; + free(unicode_data[unichar].utf32nfdicf); + um = malloc(i * sizeof(unsigned int)); + memcpy(um, mapping, i * sizeof(unsigned int)); + unicode_data[unichar].utf32nfdicf = um; + } + if (verbose > 1) + print_utf32nfdicf(unichar); + count++; + } + if (verbose > 0) + printf("Processed %d entries\n", count); +} + +/* ------------------------------------------------------------------ */ + +int utf8agemax(struct tree *, const char *); +int utf8nagemax(struct tree *, const char *, size_t); +int utf8agemin(struct tree *, const char *); +int utf8nagemin(struct tree *, const char *, size_t); +ssize_t utf8len(struct tree *, const char *); +ssize_t utf8nlen(struct tree *, const char *, size_t); +struct utf8cursor; +int utf8cursor(struct utf8cursor *, struct tree *, const char *); +int utf8ncursor(struct utf8cursor *, struct tree *, const char *, size_t); +int utf8byte(struct utf8cursor *); + +/* + * Hangul decomposition (algorithm from Section 3.12 of Unicode 6.3.0) + * + * AC00;;Lo;0;L;;;;;N;;;;; + * D7A3;;Lo;0;L;;;;;N;;;;; + * + * SBase = 0xAC00 + * LBase = 0x1100 + * VBase = 0x1161 + * TBase = 0x11A7 + * LCount = 19 + * VCount = 21 + * TCount = 28 + * NCount = 588 (VCount * TCount) + * SCount = 11172 (LCount * NCount) + * + * Decomposition: + * SIndex = s - SBase + * + * LV (Canonical/Full) + * LIndex = SIndex / NCount + * VIndex = (Sindex % NCount) / TCount + * LPart = LBase + LIndex + * VPart = VBase + VIndex + * + * LVT (Canonical) + * LVIndex = (SIndex / TCount) * TCount + * TIndex = (Sindex % TCount) + * LVPart = SBase + LVIndex + * TPart = TBase + TIndex + * + * LVT (Full) + * LIndex = SIndex / NCount + * VIndex = (Sindex % NCount) / TCount + * TIndex = (Sindex % TCount) + * LPart = LBase + LIndex + * VPart = VBase + VIndex + * if (TIndex == 0) { + * d = + * } else { + * TPart = TBase + TIndex + * d = + * } + */ + +/* Constants */ +#define SB (0xAC00) +#define LB (0x1100) +#define VB (0x1161) +#define TB (0x11A7) +#define LC (19) +#define VC (21) +#define TC (28) +#define NC (VC * TC) +#define SC (LC * NC) + +/* Algorithmic decomposition of hangul syllable. */ +static utf8leaf_t *utf8hangul(const char *str, unsigned char *hangul) +{ + unsigned int si; + unsigned int li; + unsigned int vi; + unsigned int ti; + unsigned char *h; + + /* Calculate the SI, LI, VI, and TI values. */ + si = utf8decode(str) - SB; + li = si / NC; + vi = (si % NC) / TC; + ti = si % TC; + + /* Fill in base of leaf. */ + h = hangul; + LEAF_GEN(h) = 2; + LEAF_CCC(h) = DECOMPOSE; + h += 2; + + /* Add LPart, a 3-byte UTF-8 sequence. */ + h += utf8encode((char *)h, li + LB); + + /* Add VPart, a 3-byte UTF-8 sequence. */ + h += utf8encode((char *)h, vi + VB); + + /* Add TPart if required, also a 3-byte UTF-8 sequence. */ + if (ti) + h += utf8encode((char *)h, ti + TB); + + /* Terminate string. */ + h[0] = '\0'; + + return hangul; +} + +/* + * Use trie to scan s, touching at most len bytes. + * Returns the leaf if one exists, NULL otherwise. + * + * A non-NULL return guarantees that the UTF-8 sequence starting at s + * is well-formed and corresponds to a known unicode code point. The + * shorthand for this will be "is valid UTF-8 unicode". + */ +static utf8leaf_t *utf8nlookup(struct tree *tree, unsigned char *hangul, + const char *s, size_t len) +{ + utf8trie_t *trie; + int offlen; + int offset; + int mask; + int node; + + if (!tree) + return NULL; + if (len == 0) + return NULL; + node = 1; + trie = utf8data + tree->index; + while (node) { + offlen = (*trie & OFFLEN) >> OFFLEN_SHIFT; + if (*trie & NEXTBYTE) { + if (--len == 0) + return NULL; + s++; + } + mask = 1 << (*trie & BITNUM); + if (*s & mask) { + /* Right leg */ + if (offlen) { + /* Right node at offset of trie */ + node = (*trie & RIGHTNODE); + offset = trie[offlen]; + while (--offlen) { + offset <<= 8; + offset |= trie[offlen]; + } + trie += offset; + } else if (*trie & RIGHTPATH) { + /* Right node after this node */ + node = (*trie & TRIENODE); + trie++; + } else { + /* No right node. */ + return NULL; + } + } else { + /* Left leg */ + if (offlen) { + /* Left node after this node. */ + node = (*trie & LEFTNODE); + trie += offlen + 1; + } else if (*trie & RIGHTPATH) { + /* No left node. */ + return NULL; + } else { + /* Left node after this node */ + node = (*trie & TRIENODE); + trie++; + } + } + } + /* + * Hangul decomposition is done algorithmically. These are the + * codepoints >= 0xAC00 and <= 0xD7A3. Their UTF-8 encoding is + * always 3 bytes long, so s has been advanced twice, and the + * start of the sequence is at s-2. + */ + if (LEAF_CCC(trie) == DECOMPOSE && LEAF_STR(trie)[0] == HANGUL) + trie = utf8hangul(s - 2, hangul); + return trie; +} + +/* + * Use trie to scan s. + * Returns the leaf if one exists, NULL otherwise. + * + * Forwards to trie_nlookup(). + */ +static utf8leaf_t *utf8lookup(struct tree *tree, unsigned char *hangul, + const char *s) +{ + return utf8nlookup(tree, hangul, s, (size_t)-1); +} + +/* + * Return the number of bytes used by the current UTF-8 sequence. + * Assumes the input points to the first byte of a valid UTF-8 + * sequence. + */ +static inline int utf8clen(const char *s) +{ + unsigned char c = *s; + return 1 + (c >= 0xC0) + (c >= 0xE0) + (c >= 0xF0); +} + +/* + * Maximum age of any character in s. + * Return -1 if s is not valid UTF-8 unicode. + * Return 0 if only non-assigned code points are used. + */ +int utf8agemax(struct tree *tree, const char *s) +{ + utf8leaf_t *leaf; + int age = 0; + int leaf_age; + unsigned char hangul[UTF8HANGULLEAF]; + + if (!tree) + return -1; + + while (*s) { + leaf = utf8lookup(tree, hangul, s); + if (!leaf) + return -1; + leaf_age = ages[LEAF_GEN(leaf)]; + if (leaf_age <= tree->maxage && leaf_age > age) + age = leaf_age; + s += utf8clen(s); + } + return age; +} + +/* + * Minimum age of any character in s. + * Return -1 if s is not valid UTF-8 unicode. + * Return 0 if non-assigned code points are used. + */ +int utf8agemin(struct tree *tree, const char *s) +{ + utf8leaf_t *leaf; + int age; + int leaf_age; + unsigned char hangul[UTF8HANGULLEAF]; + + if (!tree) + return -1; + age = tree->maxage; + while (*s) { + leaf = utf8lookup(tree, hangul, s); + if (!leaf) + return -1; + leaf_age = ages[LEAF_GEN(leaf)]; + if (leaf_age <= tree->maxage && leaf_age < age) + age = leaf_age; + s += utf8clen(s); + } + return age; +} + +/* + * Maximum age of any character in s, touch at most len bytes. + * Return -1 if s is not valid UTF-8 unicode. + */ +int utf8nagemax(struct tree *tree, const char *s, size_t len) +{ + utf8leaf_t *leaf; + int age = 0; + int leaf_age; + unsigned char hangul[UTF8HANGULLEAF]; + + if (!tree) + return -1; + + while (len && *s) { + leaf = utf8nlookup(tree, hangul, s, len); + if (!leaf) + return -1; + leaf_age = ages[LEAF_GEN(leaf)]; + if (leaf_age <= tree->maxage && leaf_age > age) + age = leaf_age; + len -= utf8clen(s); + s += utf8clen(s); + } + return age; +} + +/* + * Maximum age of any character in s, touch at most len bytes. + * Return -1 if s is not valid UTF-8 unicode. + */ +int utf8nagemin(struct tree *tree, const char *s, size_t len) +{ + utf8leaf_t *leaf; + int leaf_age; + int age; + unsigned char hangul[UTF8HANGULLEAF]; + + if (!tree) + return -1; + age = tree->maxage; + while (len && *s) { + leaf = utf8nlookup(tree, hangul, s, len); + if (!leaf) + return -1; + leaf_age = ages[LEAF_GEN(leaf)]; + if (leaf_age <= tree->maxage && leaf_age < age) + age = leaf_age; + len -= utf8clen(s); + s += utf8clen(s); + } + return age; +} + +/* + * Length of the normalization of s. + * Return -1 if s is not valid UTF-8 unicode. + * + * A string of Default_Ignorable_Code_Point has length 0. + */ +ssize_t utf8len(struct tree *tree, const char *s) +{ + utf8leaf_t *leaf; + size_t ret = 0; + unsigned char hangul[UTF8HANGULLEAF]; + + if (!tree) + return -1; + while (*s) { + leaf = utf8lookup(tree, hangul, s); + if (!leaf) + return -1; + if (ages[LEAF_GEN(leaf)] > tree->maxage) + ret += utf8clen(s); + else if (LEAF_CCC(leaf) == DECOMPOSE) + ret += strlen(LEAF_STR(leaf)); + else + ret += utf8clen(s); + s += utf8clen(s); + } + return ret; +} + +/* + * Length of the normalization of s, touch at most len bytes. + * Return -1 if s is not valid UTF-8 unicode. + */ +ssize_t utf8nlen(struct tree *tree, const char *s, size_t len) +{ + utf8leaf_t *leaf; + size_t ret = 0; + unsigned char hangul[UTF8HANGULLEAF]; + + if (!tree) + return -1; + while (len && *s) { + leaf = utf8nlookup(tree, hangul, s, len); + if (!leaf) + return -1; + if (ages[LEAF_GEN(leaf)] > tree->maxage) + ret += utf8clen(s); + else if (LEAF_CCC(leaf) == DECOMPOSE) + ret += strlen(LEAF_STR(leaf)); + else + ret += utf8clen(s); + len -= utf8clen(s); + s += utf8clen(s); + } + return ret; +} + +/* + * Cursor structure used by the normalizer. + */ +struct utf8cursor { + struct tree *tree; + const char *s; + const char *p; + const char *ss; + const char *sp; + unsigned int len; + unsigned int slen; + short int ccc; + short int nccc; + unsigned int unichar; + unsigned char hangul[UTF8HANGULLEAF]; +}; + +/* + * Set up an utf8cursor for use by utf8byte(). + * + * s : string. + * len : length of s. + * u8c : pointer to cursor. + * trie : utf8trie_t to use for normalization. + * + * Returns -1 on error, 0 on success. + */ +int utf8ncursor(struct utf8cursor *u8c, struct tree *tree, const char *s, + size_t len) +{ + if (!tree) + return -1; + if (!s) + return -1; + u8c->tree = tree; + u8c->s = s; + u8c->p = NULL; + u8c->ss = NULL; + u8c->sp = NULL; + u8c->len = len; + u8c->slen = 0; + u8c->ccc = STOPPER; + u8c->nccc = STOPPER; + u8c->unichar = 0; + /* Check we didn't clobber the maximum length. */ + if (u8c->len != len) + return -1; + /* The first byte of s may not be an utf8 continuation. */ + if (len > 0 && (*s & 0xC0) == 0x80) + return -1; + return 0; +} + +/* + * Set up an utf8cursor for use by utf8byte(). + * + * s : NUL-terminated string. + * u8c : pointer to cursor. + * trie : utf8trie_t to use for normalization. + * + * Returns -1 on error, 0 on success. + */ +int utf8cursor(struct utf8cursor *u8c, struct tree *tree, const char *s) +{ + return utf8ncursor(u8c, tree, s, (unsigned int)-1); +} + +/* + * Get one byte from the normalized form of the string described by u8c. + * + * Returns the byte cast to an unsigned char on succes, and -1 on failure. + * + * The cursor keeps track of the location in the string in u8c->s. + * When a character is decomposed, the current location is stored in + * u8c->p, and u8c->s is set to the start of the decomposition. Note + * that bytes from a decomposition do not count against u8c->len. + * + * Characters are emitted if they match the current CCC in u8c->ccc. + * Hitting end-of-string while u8c->ccc == STOPPER means we're done, + * and the function returns 0 in that case. + * + * Sorting by CCC is done by repeatedly scanning the string. The + * values of u8c->s and u8c->p are stored in u8c->ss and u8c->sp at + * the start of the scan. The first pass finds the lowest CCC to be + * emitted and stores it in u8c->nccc, the second pass emits the + * characters with this CCC and finds the next lowest CCC. This limits + * the number of passes to 1 + the number of different CCCs in the + * sequence being scanned. + * + * Therefore: + * u8c->p != NULL -> a decomposition is being scanned. + * u8c->ss != NULL -> this is a repeating scan. + * u8c->ccc == -1 -> this is the first scan of a repeating scan. + */ +int utf8byte(struct utf8cursor *u8c) +{ + utf8leaf_t *leaf; + int ccc; + + for (;;) { + /* Check for the end of a decomposed character. */ + if (u8c->p && *u8c->s == '\0') { + u8c->s = u8c->p; + u8c->p = NULL; + } + + /* Check for end-of-string. */ + if (!u8c->p && (u8c->len == 0 || *u8c->s == '\0')) { + /* There is no next byte. */ + if (u8c->ccc == STOPPER) + return 0; + /* End-of-string during a scan counts as a stopper. */ + ccc = STOPPER; + goto ccc_mismatch; + } else if ((*u8c->s & 0xC0) == 0x80) { + /* This is a continuation of the current character. */ + if (!u8c->p) + u8c->len--; + return (unsigned char)*u8c->s++; + } + + /* Look up the data for the current character. */ + if (u8c->p) { + leaf = utf8lookup(u8c->tree, u8c->hangul, u8c->s); + } else { + leaf = utf8nlookup(u8c->tree, u8c->hangul, + u8c->s, u8c->len); + } + + /* No leaf found implies that the input is a binary blob. */ + if (!leaf) + return -1; + + /* Characters that are too new have CCC 0. */ + if (ages[LEAF_GEN(leaf)] > u8c->tree->maxage) { + ccc = STOPPER; + } else if ((ccc = LEAF_CCC(leaf)) == DECOMPOSE) { + u8c->len -= utf8clen(u8c->s); + u8c->p = u8c->s + utf8clen(u8c->s); + u8c->s = LEAF_STR(leaf); + /* Empty decomposition implies CCC 0. */ + if (*u8c->s == '\0') { + if (u8c->ccc == STOPPER) + continue; + ccc = STOPPER; + goto ccc_mismatch; + } + leaf = utf8lookup(u8c->tree, u8c->hangul, u8c->s); + ccc = LEAF_CCC(leaf); + } + u8c->unichar = utf8decode(u8c->s); + + /* + * If this is not a stopper, then see if it updates + * the next canonical class to be emitted. + */ + if (ccc != STOPPER && u8c->ccc < ccc && ccc < u8c->nccc) + u8c->nccc = ccc; + + /* + * Return the current byte if this is the current + * combining class. + */ + if (ccc == u8c->ccc) { + if (!u8c->p) + u8c->len--; + return (unsigned char)*u8c->s++; + } + + /* Current combining class mismatch. */ + ccc_mismatch: + if (u8c->nccc == STOPPER) { + /* + * Scan forward for the first canonical class + * to be emitted. Save the position from + * which to restart. + */ + assert(u8c->ccc == STOPPER); + u8c->ccc = MINCCC - 1; + u8c->nccc = ccc; + u8c->sp = u8c->p; + u8c->ss = u8c->s; + u8c->slen = u8c->len; + if (!u8c->p) + u8c->len -= utf8clen(u8c->s); + u8c->s += utf8clen(u8c->s); + } else if (ccc != STOPPER) { + /* Not a stopper, and not the ccc we're emitting. */ + if (!u8c->p) + u8c->len -= utf8clen(u8c->s); + u8c->s += utf8clen(u8c->s); + } else if (u8c->nccc != MAXCCC + 1) { + /* At a stopper, restart for next ccc. */ + u8c->ccc = u8c->nccc; + u8c->nccc = MAXCCC + 1; + u8c->s = u8c->ss; + u8c->p = u8c->sp; + u8c->len = u8c->slen; + } else { + /* All done, proceed from here. */ + u8c->ccc = STOPPER; + u8c->nccc = STOPPER; + u8c->sp = NULL; + u8c->ss = NULL; + u8c->slen = 0; + } + } +} + +/* ------------------------------------------------------------------ */ + +static int normalize_line(struct tree *tree) +{ + char *s; + char *t; + int c; + struct utf8cursor u8c; + + /* First test: null-terminated string. */ + s = buf2; + t = buf3; + if (utf8cursor(&u8c, tree, s)) + return -1; + while ((c = utf8byte(&u8c)) > 0) + if (c != (unsigned char)*t++) + return -1; + if (c < 0) + return -1; + if (*t != 0) + return -1; + + /* Second test: length-limited string. */ + s = buf2; + /* Replace NUL with a value that will cause an error if seen. */ + s[strlen(s) + 1] = -1; + t = buf3; + if (utf8cursor(&u8c, tree, s)) + return -1; + while ((c = utf8byte(&u8c)) > 0) + if (c != (unsigned char)*t++) + return -1; + if (c < 0) + return -1; + if (*t != 0) + return -1; + + return 0; +} + +static void normalization_test(void) +{ + FILE *file; + unsigned int unichar; + struct unicode_data *data; + char *s; + char *t; + int ret; + int ignorables; + int tests = 0; + int failures = 0; + + if (verbose > 0) + printf("Parsing %s\n", test_name); + /* Step one, read data from file. */ + file = fopen(test_name, "r"); + if (!file) + open_fail(test_name, errno); + + while (fgets(line, LINESIZE, file)) { + ret = sscanf(line, "%[^;];%*[^;];%[^;];%*[^;];%*[^;];", + buf0, buf1); + if (ret != 2 || *line == '#') + continue; + s = buf0; + t = buf2; + while (*s) { + unichar = strtoul(s, &s, 16); + t += utf8encode(t, unichar); + } + *t = '\0'; + + ignorables = 0; + s = buf1; + t = buf3; + while (*s) { + unichar = strtoul(s, &s, 16); + data = &unicode_data[unichar]; + if (data->utf8nfdi && !*data->utf8nfdi) + ignorables = 1; + else + t += utf8encode(t, unichar); + } + *t = '\0'; + + tests++; + if (normalize_line(nfdi_tree) < 0) { + printf("Line %s -> %s", buf0, buf1); + if (ignorables) + printf(" (ignorables removed)"); + printf(" failure\n"); + failures++; + } + } + fclose(file); + if (verbose > 0) + printf("Ran %d tests with %d failures\n", tests, failures); + if (failures) + file_fail(test_name); +} + +/* ------------------------------------------------------------------ */ + +static void write_file(void) +{ + FILE *file; + int i; + int j; + int t; + int gen; + + if (verbose > 0) + printf("Writing %s\n", utf8_name); + file = fopen(utf8_name, "w"); + if (!file) + open_fail(utf8_name, errno); + + fprintf(file, "/* This file is generated code, do not edit. */\n"); + fprintf(file, "\n"); + fprintf(file, "#include \n"); + fprintf(file, "#include \n"); + fprintf(file, "#include \"utf8n.h\"\n"); + fprintf(file, "\n"); + fprintf(file, "static const unsigned int utf8agetab[] = {\n"); + for (i = 0; i != ages_count; i++) + fprintf(file, "\t%#x%s\n", ages[i], + ages[i] == unicode_maxage ? "" : ","); + fprintf(file, "};\n"); + fprintf(file, "\n"); + fprintf(file, "static const struct utf8data utf8nfdicfdata[] = {\n"); + t = 0; + for (gen = 0; gen < ages_count; gen++) { + fprintf(file, "\t{ %#x, %d }%s\n", + ages[gen], trees[t].index, + ages[gen] == unicode_maxage ? "" : ","); + if (trees[t].maxage == ages[gen]) + t += 2; + } + fprintf(file, "};\n"); + fprintf(file, "\n"); + fprintf(file, "static const struct utf8data utf8nfdidata[] = {\n"); + t = 1; + for (gen = 0; gen < ages_count; gen++) { + fprintf(file, "\t{ %#x, %d }%s\n", + ages[gen], trees[t].index, + ages[gen] == unicode_maxage ? "" : ","); + if (trees[t].maxage == ages[gen]) + t += 2; + } + fprintf(file, "};\n"); + fprintf(file, "\n"); + fprintf(file, "static const unsigned char utf8data[%zd] = {\n", + utf8data_size); + t = 0; + for (i = 0; i != utf8data_size; i += 16) { + if (i == trees[t].index) { + fprintf(file, "\t/* %s_%x */\n", + trees[t].type, trees[t].maxage); + if (t < trees_count-1) + t++; + } + fprintf(file, "\t"); + for (j = i; j != i + 16; j++) + fprintf(file, "0x%.2x%s", utf8data[j], + (j < utf8data_size -1 ? "," : "")); + fprintf(file, "\n"); + } + fprintf(file, "};\n"); + fprintf(file, "\n"); + fprintf(file, "const struct utf8data_table utf8_data_table = {\n"); + fprintf(file, "\t.utf8agetab = utf8agetab,\n"); + fprintf(file, "\t.utf8agetab_size = ARRAY_SIZE(utf8agetab),\n"); + fprintf(file, "\n"); + fprintf(file, "\t.utf8nfdicfdata = utf8nfdicfdata,\n"); + fprintf(file, "\t.utf8nfdicfdata_size = ARRAY_SIZE(utf8nfdicfdata),\n"); + fprintf(file, "\n"); + fprintf(file, "\t.utf8nfdidata = utf8nfdidata,\n"); + fprintf(file, "\t.utf8nfdidata_size = ARRAY_SIZE(utf8nfdidata),\n"); + fprintf(file, "\n"); + fprintf(file, "\t.utf8data = utf8data,\n"); + fprintf(file, "};\n"); + fprintf(file, "EXPORT_SYMBOL_GPL(utf8_data_table);"); + fprintf(file, "\n"); + fprintf(file, "MODULE_DESCRIPTION(\"UTF8 data table\");\n"); + fprintf(file, "MODULE_LICENSE(\"GPL v2\");\n"); + fclose(file); +} + +/* ------------------------------------------------------------------ */ + +int main(int argc, char *argv[]) +{ + unsigned int unichar; + int opt; + + argv0 = argv[0]; + + while ((opt = getopt(argc, argv, "a:c:d:f:hn:o:p:t:v")) != -1) { + switch (opt) { + case 'a': + age_name = optarg; + break; + case 'c': + ccc_name = optarg; + break; + case 'd': + data_name = optarg; + break; + case 'f': + fold_name = optarg; + break; + case 'n': + norm_name = optarg; + break; + case 'o': + utf8_name = optarg; + break; + case 'p': + prop_name = optarg; + break; + case 't': + test_name = optarg; + break; + case 'v': + verbose++; + break; + case 'h': + help(); + exit(0); + default: + usage(); + } + } + + if (verbose > 1) + help(); + for (unichar = 0; unichar != 0x110000; unichar++) + unicode_data[unichar].code = unichar; + age_init(); + ccc_init(); + nfdi_init(); + nfdicf_init(); + ignore_init(); + corrections_init(); + hangul_decompose(); + nfdi_decompose(); + nfdicf_decompose(); + utf8_init(); + trees_init(); + trees_populate(); + trees_reduce(); + trees_verify(); + /* Prevent "unused function" warning. */ + (void)lookup(nfdi_tree, " "); + if (verbose > 2) + tree_walk(nfdi_tree); + if (verbose > 2) + tree_walk(nfdicf_tree); + normalization_test(); + write_file(); + + return 0; +} diff --git a/fs/unicode/tests/.kunitconfig b/fs/unicode/tests/.kunitconfig new file mode 100644 index 000000000..62dd5c171 --- /dev/null +++ b/fs/unicode/tests/.kunitconfig @@ -0,0 +1,3 @@ +CONFIG_KUNIT=y +CONFIG_UNICODE=y +CONFIG_UNICODE_NORMALIZATION_KUNIT_TEST=y diff --git a/fs/unicode/tests/utf8_kunit.c b/fs/unicode/tests/utf8_kunit.c new file mode 100644 index 000000000..5063e8138 --- /dev/null +++ b/fs/unicode/tests/utf8_kunit.c @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KUnit tests for utf-8 support. + * + * Copyright 2017 Collabora Ltd. + */ + +#include +#include + +#include "../utf8n.h" + +static const struct { + /* UTF-8 strings in this vector _must_ be NULL-terminated. */ + unsigned char str[10]; + unsigned char dec[10]; +} nfdi_test_data[] = { + /* Trivial sequence */ + { + /* "ABba" decomposes to itself */ + .str = "aBba", + .dec = "aBba", + }, + /* Simple equivalent sequences */ + { + /* 'VULGAR FRACTION ONE QUARTER' cannot decompose to + 'NUMBER 1' + 'FRACTION SLASH' + 'NUMBER 4' on + canonical decomposition */ + .str = {0xc2, 0xbc, 0x00}, + .dec = {0xc2, 0xbc, 0x00}, + }, + { + /* 'LATIN SMALL LETTER A WITH DIAERESIS' decomposes to + 'LETTER A' + 'COMBINING DIAERESIS' */ + .str = {0xc3, 0xa4, 0x00}, + .dec = {0x61, 0xcc, 0x88, 0x00}, + }, + { + /* 'LATIN SMALL LETTER LJ' can't decompose to + 'LETTER L' + 'LETTER J' on canonical decomposition */ + .str = {0xC7, 0x89, 0x00}, + .dec = {0xC7, 0x89, 0x00}, + }, + { + /* GREEK ANO TELEIA decomposes to MIDDLE DOT */ + .str = {0xCE, 0x87, 0x00}, + .dec = {0xC2, 0xB7, 0x00} + }, + /* Canonical ordering */ + { + /* A + 'COMBINING ACUTE ACCENT' + 'COMBINING OGONEK' decomposes + to A + 'COMBINING OGONEK' + 'COMBINING ACUTE ACCENT' */ + .str = {0x41, 0xcc, 0x81, 0xcc, 0xa8, 0x0}, + .dec = {0x41, 0xcc, 0xa8, 0xcc, 0x81, 0x0}, + }, + { + /* 'LATIN SMALL LETTER A WITH DIAERESIS' + 'COMBINING OGONEK' + decomposes to + 'LETTER A' + 'COMBINING OGONEK' + 'COMBINING DIAERESIS' */ + .str = {0xc3, 0xa4, 0xCC, 0xA8, 0x00}, + + .dec = {0x61, 0xCC, 0xA8, 0xcc, 0x88, 0x00}, + }, + +}; + +static const struct { + /* UTF-8 strings in this vector _must_ be NULL-terminated. */ + unsigned char str[30]; + unsigned char ncf[30]; +} nfdicf_test_data[] = { + /* Trivial sequences */ + { + /* "ABba" folds to lowercase */ + .str = {0x41, 0x42, 0x62, 0x61, 0x00}, + .ncf = {0x61, 0x62, 0x62, 0x61, 0x00}, + }, + { + /* All ASCII folds to lower-case */ + .str = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0.1", + .ncf = "abcdefghijklmnopqrstuvwxyz0.1", + }, + { + /* LATIN SMALL LETTER SHARP S folds to + LATIN SMALL LETTER S + LATIN SMALL LETTER S */ + .str = {0xc3, 0x9f, 0x00}, + .ncf = {0x73, 0x73, 0x00}, + }, + { + /* LATIN CAPITAL LETTER A WITH RING ABOVE folds to + LATIN SMALL LETTER A + COMBINING RING ABOVE */ + .str = {0xC3, 0x85, 0x00}, + .ncf = {0x61, 0xcc, 0x8a, 0x00}, + }, + /* Introduced by UTF-8.0.0. */ + /* Cherokee letters are interesting test-cases because they fold + to upper-case. Before 8.0.0, Cherokee lowercase were + undefined, thus, the folding from LC is not stable between + 7.0.0 -> 8.0.0, but it is from UC. */ + { + /* CHEROKEE SMALL LETTER A folds to CHEROKEE LETTER A */ + .str = {0xea, 0xad, 0xb0, 0x00}, + .ncf = {0xe1, 0x8e, 0xa0, 0x00}, + }, + { + /* CHEROKEE SMALL LETTER YE folds to CHEROKEE LETTER YE */ + .str = {0xe1, 0x8f, 0xb8, 0x00}, + .ncf = {0xe1, 0x8f, 0xb0, 0x00}, + }, + { + /* OLD HUNGARIAN CAPITAL LETTER AMB folds to + OLD HUNGARIAN SMALL LETTER AMB */ + .str = {0xf0, 0x90, 0xb2, 0x83, 0x00}, + .ncf = {0xf0, 0x90, 0xb3, 0x83, 0x00}, + }, + /* Introduced by UTF-9.0.0. */ + { + /* OSAGE CAPITAL LETTER CHA folds to + OSAGE SMALL LETTER CHA */ + .str = {0xf0, 0x90, 0x92, 0xb5, 0x00}, + .ncf = {0xf0, 0x90, 0x93, 0x9d, 0x00}, + }, + { + /* LATIN CAPITAL LETTER SMALL CAPITAL I folds to + LATIN LETTER SMALL CAPITAL I */ + .str = {0xea, 0x9e, 0xae, 0x00}, + .ncf = {0xc9, 0xaa, 0x00}, + }, + /* Introduced by UTF-11.0.0. */ + { + /* GEORGIAN SMALL LETTER AN folds to GEORGIAN MTAVRULI + CAPITAL LETTER AN */ + .str = {0xe1, 0xb2, 0x90, 0x00}, + .ncf = {0xe1, 0x83, 0x90, 0x00}, + } +}; + +static ssize_t utf8len(const struct unicode_map *um, enum utf8_normalization n, + const char *s) +{ + return utf8nlen(um, n, s, (size_t)-1); +} + +static int utf8cursor(struct utf8cursor *u8c, const struct unicode_map *um, + enum utf8_normalization n, const char *s) +{ + return utf8ncursor(u8c, um, n, s, (unsigned int)-1); +} + +static void check_utf8_nfdi(struct kunit *test) +{ + int i; + struct utf8cursor u8c; + struct unicode_map *um = test->priv; + + for (i = 0; i < ARRAY_SIZE(nfdi_test_data); i++) { + int len = strlen(nfdi_test_data[i].str); + int nlen = strlen(nfdi_test_data[i].dec); + int j = 0; + unsigned char c; + int ret; + + KUNIT_EXPECT_EQ(test, utf8len(um, UTF8_NFDI, nfdi_test_data[i].str), nlen); + KUNIT_EXPECT_EQ(test, utf8nlen(um, UTF8_NFDI, nfdi_test_data[i].str, len), + nlen); + + + ret = utf8cursor(&u8c, um, UTF8_NFDI, nfdi_test_data[i].str); + KUNIT_EXPECT_TRUE_MSG(test, ret >= 0, "Can't create cursor\n"); + + while ((c = utf8byte(&u8c)) > 0) { + KUNIT_EXPECT_EQ_MSG(test, c, nfdi_test_data[i].dec[j], + "Unexpected byte 0x%x should be 0x%x\n", + c, nfdi_test_data[i].dec[j]); + j++; + } + + KUNIT_EXPECT_EQ(test, j, nlen); + } +} + +static void check_utf8_nfdicf(struct kunit *test) +{ + int i; + struct utf8cursor u8c; + struct unicode_map *um = test->priv; + + for (i = 0; i < ARRAY_SIZE(nfdicf_test_data); i++) { + int len = strlen(nfdicf_test_data[i].str); + int nlen = strlen(nfdicf_test_data[i].ncf); + int j = 0; + int ret; + unsigned char c; + + KUNIT_EXPECT_EQ(test, utf8len(um, UTF8_NFDICF, nfdicf_test_data[i].str), + nlen); + KUNIT_EXPECT_EQ(test, utf8nlen(um, UTF8_NFDICF, nfdicf_test_data[i].str, len), + nlen); + + ret = utf8cursor(&u8c, um, UTF8_NFDICF, nfdicf_test_data[i].str); + KUNIT_EXPECT_TRUE_MSG(test, ret >= 0, "Can't create cursor\n"); + + while ((c = utf8byte(&u8c)) > 0) { + KUNIT_EXPECT_EQ_MSG(test, c, nfdicf_test_data[i].ncf[j], + "Unexpected byte 0x%x should be 0x%x\n", + c, nfdicf_test_data[i].ncf[j]); + j++; + } + + KUNIT_EXPECT_EQ(test, j, nlen); + } +} + +static void check_utf8_comparisons(struct kunit *test) +{ + int i; + struct unicode_map *um = test->priv; + + for (i = 0; i < ARRAY_SIZE(nfdi_test_data); i++) { + const struct qstr s1 = {.name = nfdi_test_data[i].str, + .len = sizeof(nfdi_test_data[i].str)}; + const struct qstr s2 = {.name = nfdi_test_data[i].dec, + .len = sizeof(nfdi_test_data[i].dec)}; + + /* strncmp returns 0 when strings are equal */ + KUNIT_EXPECT_TRUE_MSG(test, utf8_strncmp(um, &s1, &s2) == 0, + "%s %s comparison mismatch\n", s1.name, s2.name); + } + + for (i = 0; i < ARRAY_SIZE(nfdicf_test_data); i++) { + const struct qstr s1 = {.name = nfdicf_test_data[i].str, + .len = sizeof(nfdicf_test_data[i].str)}; + const struct qstr s2 = {.name = nfdicf_test_data[i].ncf, + .len = sizeof(nfdicf_test_data[i].ncf)}; + + /* strncasecmp returns 0 when strings are equal */ + KUNIT_EXPECT_TRUE_MSG(test, utf8_strncasecmp(um, &s1, &s2) == 0, + "%s %s comparison mismatch\n", s1.name, s2.name); + } +} + +static void check_supported_versions(struct kunit *test) +{ + struct unicode_map *um = test->priv; + /* Unicode 7.0.0 should be supported. */ + KUNIT_EXPECT_TRUE(test, utf8version_is_supported(um, UNICODE_AGE(7, 0, 0))); + + /* Unicode 9.0.0 should be supported. */ + KUNIT_EXPECT_TRUE(test, utf8version_is_supported(um, UNICODE_AGE(9, 0, 0))); + + /* Unicode 1x.0.0 (the latest version) should be supported. */ + KUNIT_EXPECT_TRUE(test, utf8version_is_supported(um, UTF8_LATEST)); + + /* Next versions don't exist. */ + KUNIT_EXPECT_FALSE(test, utf8version_is_supported(um, UNICODE_AGE(13, 0, 0))); + KUNIT_EXPECT_FALSE(test, utf8version_is_supported(um, UNICODE_AGE(0, 0, 0))); + KUNIT_EXPECT_FALSE(test, utf8version_is_supported(um, UNICODE_AGE(-1, -1, -1))); +} + +static struct kunit_case unicode_normalization_test_cases[] = { + KUNIT_CASE(check_supported_versions), + KUNIT_CASE(check_utf8_comparisons), + KUNIT_CASE(check_utf8_nfdicf), + KUNIT_CASE(check_utf8_nfdi), + {} +}; + +static int init_test_ucd(struct kunit *test) +{ + struct unicode_map *um = utf8_load(UTF8_LATEST); + + test->priv = um; + + KUNIT_EXPECT_EQ_MSG(test, IS_ERR(um), 0, + "%s: Unable to load utf8 table.\n", __func__); + + return 0; +} + +static void exit_test_ucd(struct kunit *test) +{ + utf8_unload(test->priv); +} + +static struct kunit_suite unicode_normalization_test_suite = { + .name = "unicode_normalization", + .test_cases = unicode_normalization_test_cases, + .init = init_test_ucd, + .exit = exit_test_ucd, +}; + +kunit_test_suite(unicode_normalization_test_suite); + + +MODULE_AUTHOR("Gabriel Krisman Bertazi "); +MODULE_DESCRIPTION("KUnit tests for utf-8 support."); +MODULE_LICENSE("GPL"); diff --git a/fs/unicode/utf8-core.c b/fs/unicode/utf8-core.c new file mode 100644 index 000000000..f313532f5 --- /dev/null +++ b/fs/unicode/utf8-core.c @@ -0,0 +1,222 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include +#include +#include +#include +#include +#include + +#include "utf8n.h" + +int utf8_validate(const struct unicode_map *um, const struct qstr *str) +{ + if (utf8nlen(um, UTF8_NFDI, str->name, str->len) < 0) + return -1; + return 0; +} +EXPORT_SYMBOL(utf8_validate); + +int utf8_strncmp(const struct unicode_map *um, + const struct qstr *s1, const struct qstr *s2) +{ + struct utf8cursor cur1, cur2; + int c1, c2; + + if (utf8ncursor(&cur1, um, UTF8_NFDI, s1->name, s1->len) < 0) + return -EINVAL; + + if (utf8ncursor(&cur2, um, UTF8_NFDI, s2->name, s2->len) < 0) + return -EINVAL; + + do { + c1 = utf8byte(&cur1); + c2 = utf8byte(&cur2); + + if (c1 < 0 || c2 < 0) + return -EINVAL; + if (c1 != c2) + return 1; + } while (c1); + + return 0; +} +EXPORT_SYMBOL(utf8_strncmp); + +int utf8_strncasecmp(const struct unicode_map *um, + const struct qstr *s1, const struct qstr *s2) +{ + struct utf8cursor cur1, cur2; + int c1, c2; + + if (utf8ncursor(&cur1, um, UTF8_NFDICF, s1->name, s1->len) < 0) + return -EINVAL; + + if (utf8ncursor(&cur2, um, UTF8_NFDICF, s2->name, s2->len) < 0) + return -EINVAL; + + do { + c1 = utf8byte(&cur1); + c2 = utf8byte(&cur2); + + if (c1 < 0 || c2 < 0) + return -EINVAL; + if (c1 != c2) + return 1; + } while (c1); + + return 0; +} +EXPORT_SYMBOL(utf8_strncasecmp); + +/* String cf is expected to be a valid UTF-8 casefolded + * string. + */ +int utf8_strncasecmp_folded(const struct unicode_map *um, + const struct qstr *cf, + const struct qstr *s1) +{ + struct utf8cursor cur1; + int c1, c2; + int i = 0; + + if (utf8ncursor(&cur1, um, UTF8_NFDICF, s1->name, s1->len) < 0) + return -EINVAL; + + do { + c1 = utf8byte(&cur1); + c2 = cf->name[i++]; + if (c1 < 0) + return -EINVAL; + if (c1 != c2) + return 1; + } while (c1); + + return 0; +} +EXPORT_SYMBOL(utf8_strncasecmp_folded); + +int utf8_casefold(const struct unicode_map *um, const struct qstr *str, + unsigned char *dest, size_t dlen) +{ + struct utf8cursor cur; + size_t nlen = 0; + + if (utf8ncursor(&cur, um, UTF8_NFDICF, str->name, str->len) < 0) + return -EINVAL; + + for (nlen = 0; nlen < dlen; nlen++) { + int c = utf8byte(&cur); + + dest[nlen] = c; + if (!c) + return nlen; + if (c == -1) + break; + } + return -EINVAL; +} +EXPORT_SYMBOL(utf8_casefold); + +int utf8_casefold_hash(const struct unicode_map *um, const void *salt, + struct qstr *str) +{ + struct utf8cursor cur; + int c; + unsigned long hash = init_name_hash(salt); + + if (utf8ncursor(&cur, um, UTF8_NFDICF, str->name, str->len) < 0) + return -EINVAL; + + while ((c = utf8byte(&cur))) { + if (c < 0) + return -EINVAL; + hash = partial_name_hash((unsigned char)c, hash); + } + str->hash = end_name_hash(hash); + return 0; +} +EXPORT_SYMBOL(utf8_casefold_hash); + +static const struct utf8data *find_table_version(const struct utf8data *table, + size_t nr_entries, unsigned int version) +{ + size_t i = nr_entries - 1; + + while (version < table[i].maxage) + i--; + if (version > table[i].maxage) + return NULL; + return &table[i]; +} + +struct unicode_map *utf8_load(unsigned int version) +{ + struct unicode_map *um; + + um = kzalloc_obj(struct unicode_map); + if (!um) + return ERR_PTR(-ENOMEM); + um->version = version; + + um->tables = symbol_request(utf8_data_table); + if (!um->tables) + goto out_free_um; + + if (!utf8version_is_supported(um, version)) + goto out_symbol_put; + um->ntab[UTF8_NFDI] = find_table_version(um->tables->utf8nfdidata, + um->tables->utf8nfdidata_size, um->version); + if (!um->ntab[UTF8_NFDI]) + goto out_symbol_put; + um->ntab[UTF8_NFDICF] = find_table_version(um->tables->utf8nfdicfdata, + um->tables->utf8nfdicfdata_size, um->version); + if (!um->ntab[UTF8_NFDICF]) + goto out_symbol_put; + return um; + +out_symbol_put: + symbol_put(utf8_data_table); +out_free_um: + kfree(um); + return ERR_PTR(-EINVAL); +} +EXPORT_SYMBOL(utf8_load); + +void utf8_unload(struct unicode_map *um) +{ + if (um) { + symbol_put(utf8_data_table); + kfree(um); + } +} +EXPORT_SYMBOL(utf8_unload); + +/** + * utf8_parse_version - Parse a UTF-8 version number from a string + * + * @version: input string + * + * Returns the parsed version on success, negative code on error + */ +int utf8_parse_version(char *version) +{ + substring_t args[3]; + unsigned int maj, min, rev; + static const struct match_token token[] = { + {1, "%u.%u.%u"}, + {0, NULL} + }; + + if (match_token(version, token, args) != 1) + return -EINVAL; + + if (match_uint(&args[0], &maj) || match_uint(&args[1], &min) || + match_uint(&args[2], &rev)) + return -EINVAL; + + if (maj > U8_MAX || min > U8_MAX || rev > U8_MAX) + return -EINVAL; + return UNICODE_AGE(maj, min, rev); +} +EXPORT_SYMBOL(utf8_parse_version); diff --git a/fs/unicode/utf8-norm.c b/fs/unicode/utf8-norm.c new file mode 100644 index 000000000..7b998c99c --- /dev/null +++ b/fs/unicode/utf8-norm.c @@ -0,0 +1,594 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2014 SGI. + * All rights reserved. + */ + +#include "utf8n.h" + +int utf8version_is_supported(const struct unicode_map *um, unsigned int version) +{ + int i = um->tables->utf8agetab_size - 1; + + while (i >= 0 && um->tables->utf8agetab[i] != 0) { + if (version == um->tables->utf8agetab[i]) + return 1; + i--; + } + return 0; +} + +/* + * UTF-8 valid ranges. + * + * The UTF-8 encoding spreads the bits of a 32bit word over several + * bytes. This table gives the ranges that can be held and how they'd + * be represented. + * + * 0x00000000 0x0000007F: 0xxxxxxx + * 0x00000000 0x000007FF: 110xxxxx 10xxxxxx + * 0x00000000 0x0000FFFF: 1110xxxx 10xxxxxx 10xxxxxx + * 0x00000000 0x001FFFFF: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + * 0x00000000 0x03FFFFFF: 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx + * 0x00000000 0x7FFFFFFF: 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx + * + * There is an additional requirement on UTF-8, in that only the + * shortest representation of a 32bit value is to be used. A decoder + * must not decode sequences that do not satisfy this requirement. + * Thus the allowed ranges have a lower bound. + * + * 0x00000000 0x0000007F: 0xxxxxxx + * 0x00000080 0x000007FF: 110xxxxx 10xxxxxx + * 0x00000800 0x0000FFFF: 1110xxxx 10xxxxxx 10xxxxxx + * 0x00010000 0x001FFFFF: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + * 0x00200000 0x03FFFFFF: 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx + * 0x04000000 0x7FFFFFFF: 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx + * + * Actual unicode characters are limited to the range 0x0 - 0x10FFFF, + * 17 planes of 65536 values. This limits the sequences actually seen + * even more, to just the following. + * + * 0 - 0x7F: 0 - 0x7F + * 0x80 - 0x7FF: 0xC2 0x80 - 0xDF 0xBF + * 0x800 - 0xFFFF: 0xE0 0xA0 0x80 - 0xEF 0xBF 0xBF + * 0x10000 - 0x10FFFF: 0xF0 0x90 0x80 0x80 - 0xF4 0x8F 0xBF 0xBF + * + * Within those ranges the surrogates 0xD800 - 0xDFFF are not allowed. + * + * Note that the longest sequence seen with valid usage is 4 bytes, + * the same a single UTF-32 character. This makes the UTF-8 + * representation of Unicode strictly smaller than UTF-32. + * + * The shortest sequence requirement was introduced by: + * Corrigendum #1: UTF-8 Shortest Form + * It can be found here: + * http://www.unicode.org/versions/corrigendum1.html + * + */ + +/* + * Return the number of bytes used by the current UTF-8 sequence. + * Assumes the input points to the first byte of a valid UTF-8 + * sequence. + */ +static inline int utf8clen(const char *s) +{ + unsigned char c = *s; + + return 1 + (c >= 0xC0) + (c >= 0xE0) + (c >= 0xF0); +} + +/* + * Decode a 3-byte UTF-8 sequence. + */ +static unsigned int +utf8decode3(const char *str) +{ + unsigned int uc; + + uc = *str++ & 0x0F; + uc <<= 6; + uc |= *str++ & 0x3F; + uc <<= 6; + uc |= *str++ & 0x3F; + + return uc; +} + +/* + * Encode a 3-byte UTF-8 sequence. + */ +static int +utf8encode3(char *str, unsigned int val) +{ + str[2] = (val & 0x3F) | 0x80; + val >>= 6; + str[1] = (val & 0x3F) | 0x80; + val >>= 6; + str[0] = val | 0xE0; + + return 3; +} + +/* + * utf8trie_t + * + * A compact binary tree, used to decode UTF-8 characters. + * + * Internal nodes are one byte for the node itself, and up to three + * bytes for an offset into the tree. The first byte contains the + * following information: + * NEXTBYTE - flag - advance to next byte if set + * BITNUM - 3 bit field - the bit number to tested + * OFFLEN - 2 bit field - number of bytes in the offset + * if offlen == 0 (non-branching node) + * RIGHTPATH - 1 bit field - set if the following node is for the + * right-hand path (tested bit is set) + * TRIENODE - 1 bit field - set if the following node is an internal + * node, otherwise it is a leaf node + * if offlen != 0 (branching node) + * LEFTNODE - 1 bit field - set if the left-hand node is internal + * RIGHTNODE - 1 bit field - set if the right-hand node is internal + * + * Due to the way utf8 works, there cannot be branching nodes with + * NEXTBYTE set, and moreover those nodes always have a righthand + * descendant. + */ +typedef const unsigned char utf8trie_t; +#define BITNUM 0x07 +#define NEXTBYTE 0x08 +#define OFFLEN 0x30 +#define OFFLEN_SHIFT 4 +#define RIGHTPATH 0x40 +#define TRIENODE 0x80 +#define RIGHTNODE 0x40 +#define LEFTNODE 0x80 + +/* + * utf8leaf_t + * + * The leaves of the trie are embedded in the trie, and so the same + * underlying datatype: unsigned char. + * + * leaf[0]: The unicode version, stored as a generation number that is + * an index into ->utf8agetab[]. With this we can filter code + * points based on the unicode version in which they were + * defined. The CCC of a non-defined code point is 0. + * leaf[1]: Canonical Combining Class. During normalization, we need + * to do a stable sort into ascending order of all characters + * with a non-zero CCC that occur between two characters with + * a CCC of 0, or at the begin or end of a string. + * The unicode standard guarantees that all CCC values are + * between 0 and 254 inclusive, which leaves 255 available as + * a special value. + * Code points with CCC 0 are known as stoppers. + * leaf[2]: Decomposition. If leaf[1] == 255, then leaf[2] is the + * start of a NUL-terminated string that is the decomposition + * of the character. + * The CCC of a decomposable character is the same as the CCC + * of the first character of its decomposition. + * Some characters decompose as the empty string: these are + * characters with the Default_Ignorable_Code_Point property. + * These do affect normalization, as they all have CCC 0. + * + * The decompositions in the trie have been fully expanded, with the + * exception of Hangul syllables, which are decomposed algorithmically. + * + * Casefolding, if applicable, is also done using decompositions. + * + * The trie is constructed in such a way that leaves exist for all + * UTF-8 sequences that match the criteria from the "UTF-8 valid + * ranges" comment above, and only for those sequences. Therefore a + * lookup in the trie can be used to validate the UTF-8 input. + */ +typedef const unsigned char utf8leaf_t; + +#define LEAF_GEN(LEAF) ((LEAF)[0]) +#define LEAF_CCC(LEAF) ((LEAF)[1]) +#define LEAF_STR(LEAF) ((const char *)((LEAF) + 2)) + +#define MINCCC (0) +#define MAXCCC (254) +#define STOPPER (0) +#define DECOMPOSE (255) + +/* Marker for hangul syllable decomposition. */ +#define HANGUL ((char)(255)) +/* Size of the synthesized leaf used for Hangul syllable decomposition. */ +#define UTF8HANGULLEAF (12) + +/* + * Hangul decomposition (algorithm from Section 3.12 of Unicode 6.3.0) + * + * AC00;;Lo;0;L;;;;;N;;;;; + * D7A3;;Lo;0;L;;;;;N;;;;; + * + * SBase = 0xAC00 + * LBase = 0x1100 + * VBase = 0x1161 + * TBase = 0x11A7 + * LCount = 19 + * VCount = 21 + * TCount = 28 + * NCount = 588 (VCount * TCount) + * SCount = 11172 (LCount * NCount) + * + * Decomposition: + * SIndex = s - SBase + * + * LV (Canonical/Full) + * LIndex = SIndex / NCount + * VIndex = (Sindex % NCount) / TCount + * LPart = LBase + LIndex + * VPart = VBase + VIndex + * + * LVT (Canonical) + * LVIndex = (SIndex / TCount) * TCount + * TIndex = (Sindex % TCount) + * LVPart = SBase + LVIndex + * TPart = TBase + TIndex + * + * LVT (Full) + * LIndex = SIndex / NCount + * VIndex = (Sindex % NCount) / TCount + * TIndex = (Sindex % TCount) + * LPart = LBase + LIndex + * VPart = VBase + VIndex + * if (TIndex == 0) { + * d = + * } else { + * TPart = TBase + TIndex + * d = + * } + */ + +/* Constants */ +#define SB (0xAC00) +#define LB (0x1100) +#define VB (0x1161) +#define TB (0x11A7) +#define LC (19) +#define VC (21) +#define TC (28) +#define NC (VC * TC) +#define SC (LC * NC) + +/* Algorithmic decomposition of hangul syllable. */ +static utf8leaf_t * +utf8hangul(const char *str, unsigned char *hangul) +{ + unsigned int si; + unsigned int li; + unsigned int vi; + unsigned int ti; + unsigned char *h; + + /* Calculate the SI, LI, VI, and TI values. */ + si = utf8decode3(str) - SB; + li = si / NC; + vi = (si % NC) / TC; + ti = si % TC; + + /* Fill in base of leaf. */ + h = hangul; + LEAF_GEN(h) = 2; + LEAF_CCC(h) = DECOMPOSE; + h += 2; + + /* Add LPart, a 3-byte UTF-8 sequence. */ + h += utf8encode3((char *)h, li + LB); + + /* Add VPart, a 3-byte UTF-8 sequence. */ + h += utf8encode3((char *)h, vi + VB); + + /* Add TPart if required, also a 3-byte UTF-8 sequence. */ + if (ti) + h += utf8encode3((char *)h, ti + TB); + + /* Terminate string. */ + h[0] = '\0'; + + return hangul; +} + +/* + * Use trie to scan s, touching at most len bytes. + * Returns the leaf if one exists, NULL otherwise. + * + * A non-NULL return guarantees that the UTF-8 sequence starting at s + * is well-formed and corresponds to a known unicode code point. The + * shorthand for this will be "is valid UTF-8 unicode". + */ +static utf8leaf_t *utf8nlookup(const struct unicode_map *um, + enum utf8_normalization n, unsigned char *hangul, const char *s, + size_t len) +{ + utf8trie_t *trie = um->tables->utf8data + um->ntab[n]->offset; + int offlen; + int offset; + int mask; + int node; + + if (len == 0) + return NULL; + + node = 1; + while (node) { + offlen = (*trie & OFFLEN) >> OFFLEN_SHIFT; + if (*trie & NEXTBYTE) { + if (--len == 0) + return NULL; + s++; + } + mask = 1 << (*trie & BITNUM); + if (*s & mask) { + /* Right leg */ + if (offlen) { + /* Right node at offset of trie */ + node = (*trie & RIGHTNODE); + offset = trie[offlen]; + while (--offlen) { + offset <<= 8; + offset |= trie[offlen]; + } + trie += offset; + } else if (*trie & RIGHTPATH) { + /* Right node after this node */ + node = (*trie & TRIENODE); + trie++; + } else { + /* No right node. */ + return NULL; + } + } else { + /* Left leg */ + if (offlen) { + /* Left node after this node. */ + node = (*trie & LEFTNODE); + trie += offlen + 1; + } else if (*trie & RIGHTPATH) { + /* No left node. */ + return NULL; + } else { + /* Left node after this node */ + node = (*trie & TRIENODE); + trie++; + } + } + } + /* + * Hangul decomposition is done algorithmically. These are the + * codepoints >= 0xAC00 and <= 0xD7A3. Their UTF-8 encoding is + * always 3 bytes long, so s has been advanced twice, and the + * start of the sequence is at s-2. + */ + if (LEAF_CCC(trie) == DECOMPOSE && LEAF_STR(trie)[0] == HANGUL) + trie = utf8hangul(s - 2, hangul); + return trie; +} + +/* + * Use trie to scan s. + * Returns the leaf if one exists, NULL otherwise. + * + * Forwards to utf8nlookup(). + */ +static utf8leaf_t *utf8lookup(const struct unicode_map *um, + enum utf8_normalization n, unsigned char *hangul, const char *s) +{ + return utf8nlookup(um, n, hangul, s, (size_t)-1); +} + +/* + * Length of the normalization of s, touch at most len bytes. + * Return -1 if s is not valid UTF-8 unicode. + */ +ssize_t utf8nlen(const struct unicode_map *um, enum utf8_normalization n, + const char *s, size_t len) +{ + utf8leaf_t *leaf; + size_t ret = 0; + unsigned char hangul[UTF8HANGULLEAF]; + + while (len && *s) { + leaf = utf8nlookup(um, n, hangul, s, len); + if (!leaf) + return -1; + if (um->tables->utf8agetab[LEAF_GEN(leaf)] > + um->ntab[n]->maxage) + ret += utf8clen(s); + else if (LEAF_CCC(leaf) == DECOMPOSE) + ret += strlen(LEAF_STR(leaf)); + else + ret += utf8clen(s); + len -= utf8clen(s); + s += utf8clen(s); + } + return ret; +} + +/* + * Set up an utf8cursor for use by utf8byte(). + * + * u8c : pointer to cursor. + * data : const struct utf8data to use for normalization. + * s : string. + * len : length of s. + * + * Returns -1 on error, 0 on success. + */ +int utf8ncursor(struct utf8cursor *u8c, const struct unicode_map *um, + enum utf8_normalization n, const char *s, size_t len) +{ + if (!s) + return -1; + u8c->um = um; + u8c->n = n; + u8c->s = s; + u8c->p = NULL; + u8c->ss = NULL; + u8c->sp = NULL; + u8c->len = len; + u8c->slen = 0; + u8c->ccc = STOPPER; + u8c->nccc = STOPPER; + /* Check we didn't clobber the maximum length. */ + if (u8c->len != len) + return -1; + /* The first byte of s may not be an utf8 continuation. */ + if (len > 0 && (*s & 0xC0) == 0x80) + return -1; + return 0; +} + +/* + * Get one byte from the normalized form of the string described by u8c. + * + * Returns the byte cast to an unsigned char on succes, and -1 on failure. + * + * The cursor keeps track of the location in the string in u8c->s. + * When a character is decomposed, the current location is stored in + * u8c->p, and u8c->s is set to the start of the decomposition. Note + * that bytes from a decomposition do not count against u8c->len. + * + * Characters are emitted if they match the current CCC in u8c->ccc. + * Hitting end-of-string while u8c->ccc == STOPPER means we're done, + * and the function returns 0 in that case. + * + * Sorting by CCC is done by repeatedly scanning the string. The + * values of u8c->s and u8c->p are stored in u8c->ss and u8c->sp at + * the start of the scan. The first pass finds the lowest CCC to be + * emitted and stores it in u8c->nccc, the second pass emits the + * characters with this CCC and finds the next lowest CCC. This limits + * the number of passes to 1 + the number of different CCCs in the + * sequence being scanned. + * + * Therefore: + * u8c->p != NULL -> a decomposition is being scanned. + * u8c->ss != NULL -> this is a repeating scan. + * u8c->ccc == -1 -> this is the first scan of a repeating scan. + */ +int utf8byte(struct utf8cursor *u8c) +{ + utf8leaf_t *leaf; + int ccc; + + for (;;) { + /* Check for the end of a decomposed character. */ + if (u8c->p && *u8c->s == '\0') { + u8c->s = u8c->p; + u8c->p = NULL; + } + + /* Check for end-of-string. */ + if (!u8c->p && (u8c->len == 0 || *u8c->s == '\0')) { + /* There is no next byte. */ + if (u8c->ccc == STOPPER) + return 0; + /* End-of-string during a scan counts as a stopper. */ + ccc = STOPPER; + goto ccc_mismatch; + } else if ((*u8c->s & 0xC0) == 0x80) { + /* This is a continuation of the current character. */ + if (!u8c->p) + u8c->len--; + return (unsigned char)*u8c->s++; + } + + /* Look up the data for the current character. */ + if (u8c->p) { + leaf = utf8lookup(u8c->um, u8c->n, u8c->hangul, u8c->s); + } else { + leaf = utf8nlookup(u8c->um, u8c->n, u8c->hangul, + u8c->s, u8c->len); + } + + /* No leaf found implies that the input is a binary blob. */ + if (!leaf) + return -1; + + ccc = LEAF_CCC(leaf); + /* Characters that are too new have CCC 0. */ + if (u8c->um->tables->utf8agetab[LEAF_GEN(leaf)] > + u8c->um->ntab[u8c->n]->maxage) { + ccc = STOPPER; + } else if (ccc == DECOMPOSE) { + u8c->len -= utf8clen(u8c->s); + u8c->p = u8c->s + utf8clen(u8c->s); + u8c->s = LEAF_STR(leaf); + /* Empty decomposition implies CCC 0. */ + if (*u8c->s == '\0') { + if (u8c->ccc == STOPPER) + continue; + ccc = STOPPER; + goto ccc_mismatch; + } + + leaf = utf8lookup(u8c->um, u8c->n, u8c->hangul, u8c->s); + if (!leaf) + return -1; + ccc = LEAF_CCC(leaf); + } + + /* + * If this is not a stopper, then see if it updates + * the next canonical class to be emitted. + */ + if (ccc != STOPPER && u8c->ccc < ccc && ccc < u8c->nccc) + u8c->nccc = ccc; + + /* + * Return the current byte if this is the current + * combining class. + */ + if (ccc == u8c->ccc) { + if (!u8c->p) + u8c->len--; + return (unsigned char)*u8c->s++; + } + + /* Current combining class mismatch. */ +ccc_mismatch: + if (u8c->nccc == STOPPER) { + /* + * Scan forward for the first canonical class + * to be emitted. Save the position from + * which to restart. + */ + u8c->ccc = MINCCC - 1; + u8c->nccc = ccc; + u8c->sp = u8c->p; + u8c->ss = u8c->s; + u8c->slen = u8c->len; + if (!u8c->p) + u8c->len -= utf8clen(u8c->s); + u8c->s += utf8clen(u8c->s); + } else if (ccc != STOPPER) { + /* Not a stopper, and not the ccc we're emitting. */ + if (!u8c->p) + u8c->len -= utf8clen(u8c->s); + u8c->s += utf8clen(u8c->s); + } else if (u8c->nccc != MAXCCC + 1) { + /* At a stopper, restart for next ccc. */ + u8c->ccc = u8c->nccc; + u8c->nccc = MAXCCC + 1; + u8c->s = u8c->ss; + u8c->p = u8c->sp; + u8c->len = u8c->slen; + } else { + /* All done, proceed from here. */ + u8c->ccc = STOPPER; + u8c->nccc = STOPPER; + u8c->sp = NULL; + u8c->ss = NULL; + u8c->slen = 0; + } + } +} + +#if IS_MODULE(CONFIG_UNICODE_NORMALIZATION_KUNIT_TEST) +EXPORT_SYMBOL_GPL(utf8version_is_supported); +EXPORT_SYMBOL_GPL(utf8nlen); +EXPORT_SYMBOL_GPL(utf8ncursor); +EXPORT_SYMBOL_GPL(utf8byte); +#endif diff --git a/fs/unicode/utf8data.c_shipped b/fs/unicode/utf8data.c_shipped new file mode 100644 index 000000000..73a93d49b --- /dev/null +++ b/fs/unicode/utf8data.c_shipped @@ -0,0 +1,4124 @@ +/* This file is generated code, do not edit. */ + +#include +#include +#include "utf8n.h" + +static const unsigned int utf8agetab[] = { + 0, + 0x10100, + 0x20000, + 0x20100, + 0x30000, + 0x30100, + 0x30200, + 0x40000, + 0x40100, + 0x50000, + 0x50100, + 0x50200, + 0x60000, + 0x60100, + 0x60200, + 0x60300, + 0x70000, + 0x80000, + 0x90000, + 0xa0000, + 0xb0000, + 0xc0000, + 0xc0100 +}; + +static const struct utf8data utf8nfdicfdata[] = { + { 0, 0 }, + { 0x10100, 0 }, + { 0x20000, 0 }, + { 0x20100, 0 }, + { 0x30000, 0 }, + { 0x30100, 0 }, + { 0x30200, 1792 }, + { 0x40000, 3200 }, + { 0x40100, 3200 }, + { 0x50000, 3200 }, + { 0x50100, 3200 }, + { 0x50200, 3200 }, + { 0x60000, 3200 }, + { 0x60100, 3200 }, + { 0x60200, 3200 }, + { 0x60300, 3200 }, + { 0x70000, 3200 }, + { 0x80000, 3200 }, + { 0x90000, 3200 }, + { 0xa0000, 3200 }, + { 0xb0000, 3200 }, + { 0xc0000, 3200 }, + { 0xc0100, 3200 } +}; + +static const struct utf8data utf8nfdidata[] = { + { 0, 896 }, + { 0x10100, 896 }, + { 0x20000, 896 }, + { 0x20100, 896 }, + { 0x30000, 896 }, + { 0x30100, 896 }, + { 0x30200, 2496 }, + { 0x40000, 20736 }, + { 0x40100, 20736 }, + { 0x50000, 20736 }, + { 0x50100, 20736 }, + { 0x50200, 20736 }, + { 0x60000, 20736 }, + { 0x60100, 20736 }, + { 0x60200, 20736 }, + { 0x60300, 20736 }, + { 0x70000, 20736 }, + { 0x80000, 20736 }, + { 0x90000, 20736 }, + { 0xa0000, 20736 }, + { 0xb0000, 20736 }, + { 0xc0000, 20736 }, + { 0xc0100, 20736 } +}; + +static const unsigned char utf8data[64256] = { + /* nfdicf_30100 */ + 0xd7,0x07,0x66,0x84,0x0c,0x01,0x00,0xc6,0xd5,0x16,0xe4,0x99,0x1a,0xe3,0x63,0x15, + 0xe2,0x4c,0x0e,0xc1,0xe0,0x4e,0x0d,0xcf,0x86,0x65,0x2d,0x0d,0x01,0x00,0xd4,0xb8, + 0xd3,0x27,0xe2,0x89,0xa3,0xe1,0xce,0x35,0xe0,0x2c,0x22,0xcf,0x86,0xc5,0xe4,0x15, + 0x6d,0xe3,0x60,0x68,0xe2,0xf6,0x65,0xe1,0x29,0x65,0xe0,0xee,0x64,0xcf,0x86,0xe5, + 0xb3,0x64,0x64,0x96,0x64,0x0b,0x00,0xd2,0x0e,0xe1,0xb5,0x3c,0xe0,0xba,0xa3,0xcf, + 0x86,0xcf,0x06,0x01,0x00,0xd1,0x0c,0xe0,0x1e,0xa9,0xcf,0x86,0xcf,0x06,0x02,0xff, + 0xff,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x01,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x01, + 0x00,0xe4,0xe1,0x45,0xe3,0x3b,0x45,0xd2,0x06,0xcf,0x06,0x01,0x00,0xe1,0x87,0xad, + 0xd0,0x21,0xcf,0x86,0xe5,0x81,0xaa,0xe4,0x00,0xaa,0xe3,0xbf,0xa9,0xe2,0x9e,0xa9, + 0xe1,0x8d,0xa9,0x10,0x08,0x01,0xff,0xe8,0xb1,0x88,0x00,0x01,0xff,0xe6,0x9b,0xb4, + 0x00,0xcf,0x86,0xe5,0x63,0xac,0xd4,0x19,0xe3,0xa2,0xab,0xe2,0x81,0xab,0xe1,0x70, + 0xab,0x10,0x08,0x01,0xff,0xe9,0xb9,0xbf,0x00,0x01,0xff,0xe8,0xab,0x96,0x00,0xe3, + 0x09,0xac,0xe2,0xe8,0xab,0xe1,0xd7,0xab,0x10,0x08,0x01,0xff,0xe7,0xb8,0xb7,0x00, + 0x01,0xff,0xe9,0x9b,0xbb,0x00,0x83,0xe2,0x19,0xfa,0xe1,0xf2,0xf6,0xe0,0x6f,0xf5, + 0xcf,0x86,0xd5,0x31,0xc4,0xe3,0x54,0x4e,0xe2,0xf5,0x4c,0xe1,0xa4,0xcc,0xe0,0x9c, + 0x4b,0xcf,0x86,0xe5,0x8e,0x49,0xe4,0xaf,0x46,0xe3,0x11,0xbd,0xe2,0x68,0xbc,0xe1, + 0x43,0xbc,0xe0,0x1c,0xbc,0xcf,0x86,0xe5,0xe9,0xbb,0x94,0x07,0x63,0xd4,0xbb,0x07, + 0x00,0x07,0x00,0xe4,0xdb,0xf4,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x05,0x00,0xd2,0x0b, + 0xe1,0xea,0xe1,0xcf,0x86,0xcf,0x06,0x05,0x00,0xd1,0x0e,0xe0,0xd9,0xe2,0xcf,0x86, + 0xe5,0x9e,0xe2,0xcf,0x06,0x11,0x00,0xd0,0x0b,0xcf,0x86,0xe5,0xd9,0xe2,0xcf,0x06, + 0x13,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xe4,0x74,0xf4,0xe3,0x5d,0xf3, + 0xd2,0xa0,0xe1,0x13,0xe7,0xd0,0x21,0xcf,0x86,0xe5,0x14,0xe4,0xe4,0x90,0xe3,0xe3, + 0x4e,0xe3,0xe2,0x2d,0xe3,0xe1,0x1b,0xe3,0x10,0x08,0x05,0xff,0xe4,0xb8,0xbd,0x00, + 0x05,0xff,0xe4,0xb8,0xb8,0x00,0xcf,0x86,0xd5,0x1c,0xe4,0x70,0xe5,0xe3,0x2f,0xe5, + 0xe2,0x0e,0xe5,0xe1,0xfd,0xe4,0x10,0x08,0x05,0xff,0xe5,0x92,0xa2,0x00,0x05,0xff, + 0xe5,0x93,0xb6,0x00,0xd4,0x34,0xd3,0x18,0xe2,0xf7,0xe5,0xe1,0xe6,0xe5,0x10,0x09, + 0x05,0xff,0xf0,0xa1,0x9a,0xa8,0x00,0x05,0xff,0xf0,0xa1,0x9b,0xaa,0x00,0xe2,0x17, + 0xe6,0x91,0x11,0x10,0x09,0x05,0xff,0xf0,0xa1,0x8d,0xaa,0x00,0x05,0xff,0xe5,0xac, + 0x88,0x00,0x05,0xff,0xe5,0xac,0xbe,0x00,0xe3,0x5d,0xe6,0xd2,0x14,0xe1,0x2c,0xe6, + 0x10,0x08,0x05,0xff,0xe5,0xaf,0xb3,0x00,0x05,0xff,0xf0,0xa1,0xac,0x98,0x00,0xe1, + 0x38,0xe6,0x10,0x08,0x05,0xff,0xe5,0xbc,0xb3,0x00,0x05,0xff,0xe5,0xb0,0xa2,0x00, + 0xd1,0xd5,0xd0,0x6a,0xcf,0x86,0xe5,0x8d,0xeb,0xd4,0x19,0xe3,0xc6,0xea,0xe2,0xa4, + 0xea,0xe1,0x93,0xea,0x10,0x08,0x05,0xff,0xe6,0xb4,0xbe,0x00,0x05,0xff,0xe6,0xb5, + 0xb7,0x00,0xd3,0x18,0xe2,0x10,0xeb,0xe1,0xff,0xea,0x10,0x09,0x05,0xff,0xf0,0xa3, + 0xbd,0x9e,0x00,0x05,0xff,0xf0,0xa3,0xbe,0x8e,0x00,0xd2,0x13,0xe1,0x28,0xeb,0x10, + 0x08,0x05,0xff,0xe7,0x81,0xbd,0x00,0x05,0xff,0xe7,0x81,0xb7,0x00,0xd1,0x11,0x10, + 0x08,0x05,0xff,0xe7,0x85,0x85,0x00,0x05,0xff,0xf0,0xa4,0x89,0xa3,0x00,0x10,0x08, + 0x05,0xff,0xe7,0x86,0x9c,0x00,0x05,0xff,0xe4,0x8e,0xab,0x00,0xcf,0x86,0xe5,0x2a, + 0xed,0xd4,0x1a,0xe3,0x62,0xec,0xe2,0x48,0xec,0xe1,0x35,0xec,0x10,0x08,0x05,0xff, + 0xe7,0x9b,0xb4,0x00,0x05,0xff,0xf0,0xa5,0x83,0xb3,0x00,0xd3,0x16,0xe2,0xaa,0xec, + 0xe1,0x98,0xec,0x10,0x08,0x05,0xff,0xe7,0xa3,0x8c,0x00,0x05,0xff,0xe4,0x83,0xa3, + 0x00,0xd2,0x13,0xe1,0xc6,0xec,0x10,0x08,0x05,0xff,0xe4,0x84,0xaf,0x00,0x05,0xff, + 0xe7,0xa9,0x80,0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa5,0xa5,0xbc,0x00,0x05, + 0xff,0xf0,0xa5,0xaa,0xa7,0x00,0x10,0x09,0x05,0xff,0xf0,0xa5,0xaa,0xa7,0x00,0x05, + 0xff,0xe7,0xaa,0xae,0x00,0xe0,0xdc,0xef,0xcf,0x86,0xd5,0x1d,0xe4,0x51,0xee,0xe3, + 0x0d,0xee,0xe2,0xeb,0xed,0xe1,0xda,0xed,0x10,0x09,0x05,0xff,0xf0,0xa3,0x8d,0x9f, + 0x00,0x05,0xff,0xe4,0x8f,0x95,0x00,0xd4,0x19,0xe3,0xf8,0xee,0xe2,0xd4,0xee,0xe1, + 0xc3,0xee,0x10,0x08,0x05,0xff,0xe8,0x8d,0x93,0x00,0x05,0xff,0xe8,0x8f,0x8a,0x00, + 0xd3,0x18,0xe2,0x43,0xef,0xe1,0x32,0xef,0x10,0x09,0x05,0xff,0xf0,0xa6,0xbe,0xb1, + 0x00,0x05,0xff,0xf0,0xa7,0x83,0x92,0x00,0xd2,0x13,0xe1,0x5b,0xef,0x10,0x08,0x05, + 0xff,0xe8,0x9a,0x88,0x00,0x05,0xff,0xe8,0x9c,0x8e,0x00,0xd1,0x10,0x10,0x08,0x05, + 0xff,0xe8,0x9c,0xa8,0x00,0x05,0xff,0xe8,0x9d,0xab,0x00,0x10,0x08,0x05,0xff,0xe8, + 0x9e,0x86,0x00,0x05,0xff,0xe4,0xb5,0x97,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + /* nfdi_30100 */ + 0x57,0x04,0x01,0x00,0xc6,0xd5,0x16,0xe4,0xc2,0x59,0xe3,0xfb,0x54,0xe2,0x74,0x4f, + 0xc1,0xe0,0xa0,0x4d,0xcf,0x86,0x65,0x84,0x4d,0x01,0x00,0xd4,0xb8,0xd3,0x27,0xe2, + 0x0c,0xa0,0xe1,0xdf,0x8d,0xe0,0x39,0x71,0xcf,0x86,0xc5,0xe4,0x98,0x69,0xe3,0xe3, + 0x64,0xe2,0x79,0x62,0xe1,0xac,0x61,0xe0,0x71,0x61,0xcf,0x86,0xe5,0x36,0x61,0x64, + 0x19,0x61,0x0b,0x00,0xd2,0x0e,0xe1,0xc2,0xa0,0xe0,0x3d,0xa0,0xcf,0x86,0xcf,0x06, + 0x01,0x00,0xd1,0x0c,0xe0,0xa1,0xa5,0xcf,0x86,0xcf,0x06,0x02,0xff,0xff,0xd0,0x08, + 0xcf,0x86,0xcf,0x06,0x01,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x01,0x00,0xe4,0x9e, + 0xb6,0xe3,0x18,0xae,0xd2,0x06,0xcf,0x06,0x01,0x00,0xe1,0x0a,0xaa,0xd0,0x21,0xcf, + 0x86,0xe5,0x04,0xa7,0xe4,0x83,0xa6,0xe3,0x42,0xa6,0xe2,0x21,0xa6,0xe1,0x10,0xa6, + 0x10,0x08,0x01,0xff,0xe8,0xb1,0x88,0x00,0x01,0xff,0xe6,0x9b,0xb4,0x00,0xcf,0x86, + 0xe5,0xe6,0xa8,0xd4,0x19,0xe3,0x25,0xa8,0xe2,0x04,0xa8,0xe1,0xf3,0xa7,0x10,0x08, + 0x01,0xff,0xe9,0xb9,0xbf,0x00,0x01,0xff,0xe8,0xab,0x96,0x00,0xe3,0x8c,0xa8,0xe2, + 0x6b,0xa8,0xe1,0x5a,0xa8,0x10,0x08,0x01,0xff,0xe7,0xb8,0xb7,0x00,0x01,0xff,0xe9, + 0x9b,0xbb,0x00,0x83,0xe2,0x9c,0xf6,0xe1,0x75,0xf3,0xe0,0xf2,0xf1,0xcf,0x86,0xd5, + 0x31,0xc4,0xe3,0x6d,0xcc,0xe2,0x46,0xca,0xe1,0x27,0xc9,0xe0,0xb7,0xbf,0xcf,0x86, + 0xe5,0xaa,0xbb,0xe4,0xa3,0xba,0xe3,0x94,0xb9,0xe2,0xeb,0xb8,0xe1,0xc6,0xb8,0xe0, + 0x9f,0xb8,0xcf,0x86,0xe5,0x6c,0xb8,0x94,0x07,0x63,0x57,0xb8,0x07,0x00,0x07,0x00, + 0xe4,0x5e,0xf1,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x05,0x00,0xd2,0x0b,0xe1,0x6d,0xde, + 0xcf,0x86,0xcf,0x06,0x05,0x00,0xd1,0x0e,0xe0,0x5c,0xdf,0xcf,0x86,0xe5,0x21,0xdf, + 0xcf,0x06,0x11,0x00,0xd0,0x0b,0xcf,0x86,0xe5,0x5c,0xdf,0xcf,0x06,0x13,0x00,0xcf, + 0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xe4,0xf7,0xf0,0xe3,0xe0,0xef,0xd2,0xa0,0xe1, + 0x96,0xe3,0xd0,0x21,0xcf,0x86,0xe5,0x97,0xe0,0xe4,0x13,0xe0,0xe3,0xd1,0xdf,0xe2, + 0xb0,0xdf,0xe1,0x9e,0xdf,0x10,0x08,0x05,0xff,0xe4,0xb8,0xbd,0x00,0x05,0xff,0xe4, + 0xb8,0xb8,0x00,0xcf,0x86,0xd5,0x1c,0xe4,0xf3,0xe1,0xe3,0xb2,0xe1,0xe2,0x91,0xe1, + 0xe1,0x80,0xe1,0x10,0x08,0x05,0xff,0xe5,0x92,0xa2,0x00,0x05,0xff,0xe5,0x93,0xb6, + 0x00,0xd4,0x34,0xd3,0x18,0xe2,0x7a,0xe2,0xe1,0x69,0xe2,0x10,0x09,0x05,0xff,0xf0, + 0xa1,0x9a,0xa8,0x00,0x05,0xff,0xf0,0xa1,0x9b,0xaa,0x00,0xe2,0x9a,0xe2,0x91,0x11, + 0x10,0x09,0x05,0xff,0xf0,0xa1,0x8d,0xaa,0x00,0x05,0xff,0xe5,0xac,0x88,0x00,0x05, + 0xff,0xe5,0xac,0xbe,0x00,0xe3,0xe0,0xe2,0xd2,0x14,0xe1,0xaf,0xe2,0x10,0x08,0x05, + 0xff,0xe5,0xaf,0xb3,0x00,0x05,0xff,0xf0,0xa1,0xac,0x98,0x00,0xe1,0xbb,0xe2,0x10, + 0x08,0x05,0xff,0xe5,0xbc,0xb3,0x00,0x05,0xff,0xe5,0xb0,0xa2,0x00,0xd1,0xd5,0xd0, + 0x6a,0xcf,0x86,0xe5,0x10,0xe8,0xd4,0x19,0xe3,0x49,0xe7,0xe2,0x27,0xe7,0xe1,0x16, + 0xe7,0x10,0x08,0x05,0xff,0xe6,0xb4,0xbe,0x00,0x05,0xff,0xe6,0xb5,0xb7,0x00,0xd3, + 0x18,0xe2,0x93,0xe7,0xe1,0x82,0xe7,0x10,0x09,0x05,0xff,0xf0,0xa3,0xbd,0x9e,0x00, + 0x05,0xff,0xf0,0xa3,0xbe,0x8e,0x00,0xd2,0x13,0xe1,0xab,0xe7,0x10,0x08,0x05,0xff, + 0xe7,0x81,0xbd,0x00,0x05,0xff,0xe7,0x81,0xb7,0x00,0xd1,0x11,0x10,0x08,0x05,0xff, + 0xe7,0x85,0x85,0x00,0x05,0xff,0xf0,0xa4,0x89,0xa3,0x00,0x10,0x08,0x05,0xff,0xe7, + 0x86,0x9c,0x00,0x05,0xff,0xe4,0x8e,0xab,0x00,0xcf,0x86,0xe5,0xad,0xe9,0xd4,0x1a, + 0xe3,0xe5,0xe8,0xe2,0xcb,0xe8,0xe1,0xb8,0xe8,0x10,0x08,0x05,0xff,0xe7,0x9b,0xb4, + 0x00,0x05,0xff,0xf0,0xa5,0x83,0xb3,0x00,0xd3,0x16,0xe2,0x2d,0xe9,0xe1,0x1b,0xe9, + 0x10,0x08,0x05,0xff,0xe7,0xa3,0x8c,0x00,0x05,0xff,0xe4,0x83,0xa3,0x00,0xd2,0x13, + 0xe1,0x49,0xe9,0x10,0x08,0x05,0xff,0xe4,0x84,0xaf,0x00,0x05,0xff,0xe7,0xa9,0x80, + 0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa5,0xa5,0xbc,0x00,0x05,0xff,0xf0,0xa5, + 0xaa,0xa7,0x00,0x10,0x09,0x05,0xff,0xf0,0xa5,0xaa,0xa7,0x00,0x05,0xff,0xe7,0xaa, + 0xae,0x00,0xe0,0x5f,0xec,0xcf,0x86,0xd5,0x1d,0xe4,0xd4,0xea,0xe3,0x90,0xea,0xe2, + 0x6e,0xea,0xe1,0x5d,0xea,0x10,0x09,0x05,0xff,0xf0,0xa3,0x8d,0x9f,0x00,0x05,0xff, + 0xe4,0x8f,0x95,0x00,0xd4,0x19,0xe3,0x7b,0xeb,0xe2,0x57,0xeb,0xe1,0x46,0xeb,0x10, + 0x08,0x05,0xff,0xe8,0x8d,0x93,0x00,0x05,0xff,0xe8,0x8f,0x8a,0x00,0xd3,0x18,0xe2, + 0xc6,0xeb,0xe1,0xb5,0xeb,0x10,0x09,0x05,0xff,0xf0,0xa6,0xbe,0xb1,0x00,0x05,0xff, + 0xf0,0xa7,0x83,0x92,0x00,0xd2,0x13,0xe1,0xde,0xeb,0x10,0x08,0x05,0xff,0xe8,0x9a, + 0x88,0x00,0x05,0xff,0xe8,0x9c,0x8e,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe8,0x9c, + 0xa8,0x00,0x05,0xff,0xe8,0x9d,0xab,0x00,0x10,0x08,0x05,0xff,0xe8,0x9e,0x86,0x00, + 0x05,0xff,0xe4,0xb5,0x97,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + /* nfdicf_30200 */ + 0xd7,0x07,0x66,0x84,0x05,0x01,0x00,0xc6,0xd5,0x16,0xe4,0x99,0x13,0xe3,0x63,0x0e, + 0xe2,0x4c,0x07,0xc1,0xe0,0x4e,0x06,0xcf,0x86,0x65,0x2d,0x06,0x01,0x00,0xd4,0x2a, + 0xe3,0xd0,0x35,0xe2,0x88,0x9c,0xe1,0xcd,0x2e,0xe0,0x2b,0x1b,0xcf,0x86,0xc5,0xe4, + 0x14,0x66,0xe3,0x5f,0x61,0xe2,0xf5,0x5e,0xe1,0x28,0x5e,0xe0,0xed,0x5d,0xcf,0x86, + 0xe5,0xb2,0x5d,0x64,0x95,0x5d,0x0b,0x00,0x83,0xe2,0xa7,0xf3,0xe1,0x80,0xf0,0xe0, + 0xfd,0xee,0xcf,0x86,0xd5,0x31,0xc4,0xe3,0xe2,0x47,0xe2,0x83,0x46,0xe1,0x32,0xc6, + 0xe0,0x2a,0x45,0xcf,0x86,0xe5,0x1c,0x43,0xe4,0x3d,0x40,0xe3,0x9f,0xb6,0xe2,0xf6, + 0xb5,0xe1,0xd1,0xb5,0xe0,0xaa,0xb5,0xcf,0x86,0xe5,0x77,0xb5,0x94,0x07,0x63,0x62, + 0xb5,0x07,0x00,0x07,0x00,0xe4,0x69,0xee,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x05,0x00, + 0xd2,0x0b,0xe1,0x78,0xdb,0xcf,0x86,0xcf,0x06,0x05,0x00,0xd1,0x0e,0xe0,0x67,0xdc, + 0xcf,0x86,0xe5,0x2c,0xdc,0xcf,0x06,0x11,0x00,0xd0,0x0b,0xcf,0x86,0xe5,0x67,0xdc, + 0xcf,0x06,0x13,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xe4,0x02,0xee,0xe3, + 0xeb,0xec,0xd2,0xa0,0xe1,0xa1,0xe0,0xd0,0x21,0xcf,0x86,0xe5,0xa2,0xdd,0xe4,0x1e, + 0xdd,0xe3,0xdc,0xdc,0xe2,0xbb,0xdc,0xe1,0xa9,0xdc,0x10,0x08,0x05,0xff,0xe4,0xb8, + 0xbd,0x00,0x05,0xff,0xe4,0xb8,0xb8,0x00,0xcf,0x86,0xd5,0x1c,0xe4,0xfe,0xde,0xe3, + 0xbd,0xde,0xe2,0x9c,0xde,0xe1,0x8b,0xde,0x10,0x08,0x05,0xff,0xe5,0x92,0xa2,0x00, + 0x05,0xff,0xe5,0x93,0xb6,0x00,0xd4,0x34,0xd3,0x18,0xe2,0x85,0xdf,0xe1,0x74,0xdf, + 0x10,0x09,0x05,0xff,0xf0,0xa1,0x9a,0xa8,0x00,0x05,0xff,0xf0,0xa1,0x9b,0xaa,0x00, + 0xe2,0xa5,0xdf,0x91,0x11,0x10,0x09,0x05,0xff,0xf0,0xa1,0x8d,0xaa,0x00,0x05,0xff, + 0xe5,0xac,0x88,0x00,0x05,0xff,0xe5,0xac,0xbe,0x00,0xe3,0xeb,0xdf,0xd2,0x14,0xe1, + 0xba,0xdf,0x10,0x08,0x05,0xff,0xe5,0xaf,0xb3,0x00,0x05,0xff,0xf0,0xa1,0xac,0x98, + 0x00,0xe1,0xc6,0xdf,0x10,0x08,0x05,0xff,0xe5,0xbc,0xb3,0x00,0x05,0xff,0xe5,0xb0, + 0xa2,0x00,0xd1,0xd5,0xd0,0x6a,0xcf,0x86,0xe5,0x1b,0xe5,0xd4,0x19,0xe3,0x54,0xe4, + 0xe2,0x32,0xe4,0xe1,0x21,0xe4,0x10,0x08,0x05,0xff,0xe6,0xb4,0xbe,0x00,0x05,0xff, + 0xe6,0xb5,0xb7,0x00,0xd3,0x18,0xe2,0x9e,0xe4,0xe1,0x8d,0xe4,0x10,0x09,0x05,0xff, + 0xf0,0xa3,0xbd,0x9e,0x00,0x05,0xff,0xf0,0xa3,0xbe,0x8e,0x00,0xd2,0x13,0xe1,0xb6, + 0xe4,0x10,0x08,0x05,0xff,0xe7,0x81,0xbd,0x00,0x05,0xff,0xe7,0x81,0xb7,0x00,0xd1, + 0x11,0x10,0x08,0x05,0xff,0xe7,0x85,0x85,0x00,0x05,0xff,0xf0,0xa4,0x89,0xa3,0x00, + 0x10,0x08,0x05,0xff,0xe7,0x86,0x9c,0x00,0x05,0xff,0xe4,0x8e,0xab,0x00,0xcf,0x86, + 0xe5,0xb8,0xe6,0xd4,0x1a,0xe3,0xf0,0xe5,0xe2,0xd6,0xe5,0xe1,0xc3,0xe5,0x10,0x08, + 0x05,0xff,0xe7,0x9b,0xb4,0x00,0x05,0xff,0xf0,0xa5,0x83,0xb3,0x00,0xd3,0x16,0xe2, + 0x38,0xe6,0xe1,0x26,0xe6,0x10,0x08,0x05,0xff,0xe7,0xa3,0x8c,0x00,0x05,0xff,0xe4, + 0x83,0xa3,0x00,0xd2,0x13,0xe1,0x54,0xe6,0x10,0x08,0x05,0xff,0xe4,0x84,0xaf,0x00, + 0x05,0xff,0xe7,0xa9,0x80,0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa5,0xa5,0xbc, + 0x00,0x05,0xff,0xf0,0xa5,0xaa,0xa7,0x00,0x10,0x09,0x05,0xff,0xf0,0xa5,0xaa,0xa7, + 0x00,0x05,0xff,0xe7,0xaa,0xae,0x00,0xe0,0x6a,0xe9,0xcf,0x86,0xd5,0x1d,0xe4,0xdf, + 0xe7,0xe3,0x9b,0xe7,0xe2,0x79,0xe7,0xe1,0x68,0xe7,0x10,0x09,0x05,0xff,0xf0,0xa3, + 0x8d,0x9f,0x00,0x05,0xff,0xe4,0x8f,0x95,0x00,0xd4,0x19,0xe3,0x86,0xe8,0xe2,0x62, + 0xe8,0xe1,0x51,0xe8,0x10,0x08,0x05,0xff,0xe8,0x8d,0x93,0x00,0x05,0xff,0xe8,0x8f, + 0x8a,0x00,0xd3,0x18,0xe2,0xd1,0xe8,0xe1,0xc0,0xe8,0x10,0x09,0x05,0xff,0xf0,0xa6, + 0xbe,0xb1,0x00,0x05,0xff,0xf0,0xa7,0x83,0x92,0x00,0xd2,0x13,0xe1,0xe9,0xe8,0x10, + 0x08,0x05,0xff,0xe8,0x9a,0x88,0x00,0x05,0xff,0xe8,0x9c,0x8e,0x00,0xd1,0x10,0x10, + 0x08,0x05,0xff,0xe8,0x9c,0xa8,0x00,0x05,0xff,0xe8,0x9d,0xab,0x00,0x10,0x08,0x05, + 0xff,0xe8,0x9e,0x86,0x00,0x05,0xff,0xe4,0xb5,0x97,0x00,0x00,0x00,0x00,0x00,0x00, + /* nfdi_30200 */ + 0x57,0x04,0x01,0x00,0xc6,0xd5,0x16,0xe4,0x82,0x53,0xe3,0xbb,0x4e,0xe2,0x34,0x49, + 0xc1,0xe0,0x60,0x47,0xcf,0x86,0x65,0x44,0x47,0x01,0x00,0xd4,0x2a,0xe3,0x1c,0x9a, + 0xe2,0xcb,0x99,0xe1,0x9e,0x87,0xe0,0xf8,0x6a,0xcf,0x86,0xc5,0xe4,0x57,0x63,0xe3, + 0xa2,0x5e,0xe2,0x38,0x5c,0xe1,0x6b,0x5b,0xe0,0x30,0x5b,0xcf,0x86,0xe5,0xf5,0x5a, + 0x64,0xd8,0x5a,0x0b,0x00,0x83,0xe2,0xea,0xf0,0xe1,0xc3,0xed,0xe0,0x40,0xec,0xcf, + 0x86,0xd5,0x31,0xc4,0xe3,0xbb,0xc6,0xe2,0x94,0xc4,0xe1,0x75,0xc3,0xe0,0x05,0xba, + 0xcf,0x86,0xe5,0xf8,0xb5,0xe4,0xf1,0xb4,0xe3,0xe2,0xb3,0xe2,0x39,0xb3,0xe1,0x14, + 0xb3,0xe0,0xed,0xb2,0xcf,0x86,0xe5,0xba,0xb2,0x94,0x07,0x63,0xa5,0xb2,0x07,0x00, + 0x07,0x00,0xe4,0xac,0xeb,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x05,0x00,0xd2,0x0b,0xe1, + 0xbb,0xd8,0xcf,0x86,0xcf,0x06,0x05,0x00,0xd1,0x0e,0xe0,0xaa,0xd9,0xcf,0x86,0xe5, + 0x6f,0xd9,0xcf,0x06,0x11,0x00,0xd0,0x0b,0xcf,0x86,0xe5,0xaa,0xd9,0xcf,0x06,0x13, + 0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xe4,0x45,0xeb,0xe3,0x2e,0xea,0xd2, + 0xa0,0xe1,0xe4,0xdd,0xd0,0x21,0xcf,0x86,0xe5,0xe5,0xda,0xe4,0x61,0xda,0xe3,0x1f, + 0xda,0xe2,0xfe,0xd9,0xe1,0xec,0xd9,0x10,0x08,0x05,0xff,0xe4,0xb8,0xbd,0x00,0x05, + 0xff,0xe4,0xb8,0xb8,0x00,0xcf,0x86,0xd5,0x1c,0xe4,0x41,0xdc,0xe3,0x00,0xdc,0xe2, + 0xdf,0xdb,0xe1,0xce,0xdb,0x10,0x08,0x05,0xff,0xe5,0x92,0xa2,0x00,0x05,0xff,0xe5, + 0x93,0xb6,0x00,0xd4,0x34,0xd3,0x18,0xe2,0xc8,0xdc,0xe1,0xb7,0xdc,0x10,0x09,0x05, + 0xff,0xf0,0xa1,0x9a,0xa8,0x00,0x05,0xff,0xf0,0xa1,0x9b,0xaa,0x00,0xe2,0xe8,0xdc, + 0x91,0x11,0x10,0x09,0x05,0xff,0xf0,0xa1,0x8d,0xaa,0x00,0x05,0xff,0xe5,0xac,0x88, + 0x00,0x05,0xff,0xe5,0xac,0xbe,0x00,0xe3,0x2e,0xdd,0xd2,0x14,0xe1,0xfd,0xdc,0x10, + 0x08,0x05,0xff,0xe5,0xaf,0xb3,0x00,0x05,0xff,0xf0,0xa1,0xac,0x98,0x00,0xe1,0x09, + 0xdd,0x10,0x08,0x05,0xff,0xe5,0xbc,0xb3,0x00,0x05,0xff,0xe5,0xb0,0xa2,0x00,0xd1, + 0xd5,0xd0,0x6a,0xcf,0x86,0xe5,0x5e,0xe2,0xd4,0x19,0xe3,0x97,0xe1,0xe2,0x75,0xe1, + 0xe1,0x64,0xe1,0x10,0x08,0x05,0xff,0xe6,0xb4,0xbe,0x00,0x05,0xff,0xe6,0xb5,0xb7, + 0x00,0xd3,0x18,0xe2,0xe1,0xe1,0xe1,0xd0,0xe1,0x10,0x09,0x05,0xff,0xf0,0xa3,0xbd, + 0x9e,0x00,0x05,0xff,0xf0,0xa3,0xbe,0x8e,0x00,0xd2,0x13,0xe1,0xf9,0xe1,0x10,0x08, + 0x05,0xff,0xe7,0x81,0xbd,0x00,0x05,0xff,0xe7,0x81,0xb7,0x00,0xd1,0x11,0x10,0x08, + 0x05,0xff,0xe7,0x85,0x85,0x00,0x05,0xff,0xf0,0xa4,0x89,0xa3,0x00,0x10,0x08,0x05, + 0xff,0xe7,0x86,0x9c,0x00,0x05,0xff,0xe4,0x8e,0xab,0x00,0xcf,0x86,0xe5,0xfb,0xe3, + 0xd4,0x1a,0xe3,0x33,0xe3,0xe2,0x19,0xe3,0xe1,0x06,0xe3,0x10,0x08,0x05,0xff,0xe7, + 0x9b,0xb4,0x00,0x05,0xff,0xf0,0xa5,0x83,0xb3,0x00,0xd3,0x16,0xe2,0x7b,0xe3,0xe1, + 0x69,0xe3,0x10,0x08,0x05,0xff,0xe7,0xa3,0x8c,0x00,0x05,0xff,0xe4,0x83,0xa3,0x00, + 0xd2,0x13,0xe1,0x97,0xe3,0x10,0x08,0x05,0xff,0xe4,0x84,0xaf,0x00,0x05,0xff,0xe7, + 0xa9,0x80,0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa5,0xa5,0xbc,0x00,0x05,0xff, + 0xf0,0xa5,0xaa,0xa7,0x00,0x10,0x09,0x05,0xff,0xf0,0xa5,0xaa,0xa7,0x00,0x05,0xff, + 0xe7,0xaa,0xae,0x00,0xe0,0xad,0xe6,0xcf,0x86,0xd5,0x1d,0xe4,0x22,0xe5,0xe3,0xde, + 0xe4,0xe2,0xbc,0xe4,0xe1,0xab,0xe4,0x10,0x09,0x05,0xff,0xf0,0xa3,0x8d,0x9f,0x00, + 0x05,0xff,0xe4,0x8f,0x95,0x00,0xd4,0x19,0xe3,0xc9,0xe5,0xe2,0xa5,0xe5,0xe1,0x94, + 0xe5,0x10,0x08,0x05,0xff,0xe8,0x8d,0x93,0x00,0x05,0xff,0xe8,0x8f,0x8a,0x00,0xd3, + 0x18,0xe2,0x14,0xe6,0xe1,0x03,0xe6,0x10,0x09,0x05,0xff,0xf0,0xa6,0xbe,0xb1,0x00, + 0x05,0xff,0xf0,0xa7,0x83,0x92,0x00,0xd2,0x13,0xe1,0x2c,0xe6,0x10,0x08,0x05,0xff, + 0xe8,0x9a,0x88,0x00,0x05,0xff,0xe8,0x9c,0x8e,0x00,0xd1,0x10,0x10,0x08,0x05,0xff, + 0xe8,0x9c,0xa8,0x00,0x05,0xff,0xe8,0x9d,0xab,0x00,0x10,0x08,0x05,0xff,0xe8,0x9e, + 0x86,0x00,0x05,0xff,0xe4,0xb5,0x97,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + /* nfdicf_c0100 */ + 0xd7,0xb0,0x56,0x04,0x01,0x00,0x95,0xa8,0xd4,0x5e,0xd3,0x2e,0xd2,0x16,0xd1,0x0a, + 0x10,0x04,0x01,0x00,0x01,0xff,0x61,0x00,0x10,0x06,0x01,0xff,0x62,0x00,0x01,0xff, + 0x63,0x00,0xd1,0x0c,0x10,0x06,0x01,0xff,0x64,0x00,0x01,0xff,0x65,0x00,0x10,0x06, + 0x01,0xff,0x66,0x00,0x01,0xff,0x67,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x06,0x01,0xff, + 0x68,0x00,0x01,0xff,0x69,0x00,0x10,0x06,0x01,0xff,0x6a,0x00,0x01,0xff,0x6b,0x00, + 0xd1,0x0c,0x10,0x06,0x01,0xff,0x6c,0x00,0x01,0xff,0x6d,0x00,0x10,0x06,0x01,0xff, + 0x6e,0x00,0x01,0xff,0x6f,0x00,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x06,0x01,0xff, + 0x70,0x00,0x01,0xff,0x71,0x00,0x10,0x06,0x01,0xff,0x72,0x00,0x01,0xff,0x73,0x00, + 0xd1,0x0c,0x10,0x06,0x01,0xff,0x74,0x00,0x01,0xff,0x75,0x00,0x10,0x06,0x01,0xff, + 0x76,0x00,0x01,0xff,0x77,0x00,0x92,0x16,0xd1,0x0c,0x10,0x06,0x01,0xff,0x78,0x00, + 0x01,0xff,0x79,0x00,0x10,0x06,0x01,0xff,0x7a,0x00,0x01,0x00,0x01,0x00,0x01,0x00, + 0xc6,0xe5,0xf9,0x14,0xe4,0x6f,0x0d,0xe3,0x39,0x08,0xe2,0x22,0x01,0xc1,0xd0,0x24, + 0xcf,0x86,0x55,0x04,0x01,0x00,0xd4,0x07,0x63,0xd8,0x43,0x01,0x00,0x93,0x13,0x52, + 0x04,0x01,0x00,0x91,0x0b,0x10,0x04,0x01,0x00,0x01,0xff,0xce,0xbc,0x00,0x01,0x00, + 0x01,0x00,0xcf,0x86,0xe5,0xb3,0x44,0xd4,0x7f,0xd3,0x3f,0xd2,0x20,0xd1,0x10,0x10, + 0x08,0x01,0xff,0x61,0xcc,0x80,0x00,0x01,0xff,0x61,0xcc,0x81,0x00,0x10,0x08,0x01, + 0xff,0x61,0xcc,0x82,0x00,0x01,0xff,0x61,0xcc,0x83,0x00,0xd1,0x10,0x10,0x08,0x01, + 0xff,0x61,0xcc,0x88,0x00,0x01,0xff,0x61,0xcc,0x8a,0x00,0x10,0x07,0x01,0xff,0xc3, + 0xa6,0x00,0x01,0xff,0x63,0xcc,0xa7,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff, + 0x65,0xcc,0x80,0x00,0x01,0xff,0x65,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x65,0xcc, + 0x82,0x00,0x01,0xff,0x65,0xcc,0x88,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x69,0xcc, + 0x80,0x00,0x01,0xff,0x69,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x69,0xcc,0x82,0x00, + 0x01,0xff,0x69,0xcc,0x88,0x00,0xd3,0x3b,0xd2,0x1f,0xd1,0x0f,0x10,0x07,0x01,0xff, + 0xc3,0xb0,0x00,0x01,0xff,0x6e,0xcc,0x83,0x00,0x10,0x08,0x01,0xff,0x6f,0xcc,0x80, + 0x00,0x01,0xff,0x6f,0xcc,0x81,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x6f,0xcc,0x82, + 0x00,0x01,0xff,0x6f,0xcc,0x83,0x00,0x10,0x08,0x01,0xff,0x6f,0xcc,0x88,0x00,0x01, + 0x00,0xd2,0x1f,0xd1,0x0f,0x10,0x07,0x01,0xff,0xc3,0xb8,0x00,0x01,0xff,0x75,0xcc, + 0x80,0x00,0x10,0x08,0x01,0xff,0x75,0xcc,0x81,0x00,0x01,0xff,0x75,0xcc,0x82,0x00, + 0xd1,0x10,0x10,0x08,0x01,0xff,0x75,0xcc,0x88,0x00,0x01,0xff,0x79,0xcc,0x81,0x00, + 0x10,0x07,0x01,0xff,0xc3,0xbe,0x00,0x01,0xff,0x73,0x73,0x00,0xe1,0xd4,0x03,0xe0, + 0xeb,0x01,0xcf,0x86,0xd5,0xfb,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x01,0xff,0x61,0xcc,0x84,0x00,0x01,0xff,0x61,0xcc,0x84,0x00,0x10,0x08,0x01,0xff, + 0x61,0xcc,0x86,0x00,0x01,0xff,0x61,0xcc,0x86,0x00,0xd1,0x10,0x10,0x08,0x01,0xff, + 0x61,0xcc,0xa8,0x00,0x01,0xff,0x61,0xcc,0xa8,0x00,0x10,0x08,0x01,0xff,0x63,0xcc, + 0x81,0x00,0x01,0xff,0x63,0xcc,0x81,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff, + 0x63,0xcc,0x82,0x00,0x01,0xff,0x63,0xcc,0x82,0x00,0x10,0x08,0x01,0xff,0x63,0xcc, + 0x87,0x00,0x01,0xff,0x63,0xcc,0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x63,0xcc, + 0x8c,0x00,0x01,0xff,0x63,0xcc,0x8c,0x00,0x10,0x08,0x01,0xff,0x64,0xcc,0x8c,0x00, + 0x01,0xff,0x64,0xcc,0x8c,0x00,0xd3,0x3b,0xd2,0x1b,0xd1,0x0b,0x10,0x07,0x01,0xff, + 0xc4,0x91,0x00,0x01,0x00,0x10,0x08,0x01,0xff,0x65,0xcc,0x84,0x00,0x01,0xff,0x65, + 0xcc,0x84,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x65,0xcc,0x86,0x00,0x01,0xff,0x65, + 0xcc,0x86,0x00,0x10,0x08,0x01,0xff,0x65,0xcc,0x87,0x00,0x01,0xff,0x65,0xcc,0x87, + 0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x65,0xcc,0xa8,0x00,0x01,0xff,0x65, + 0xcc,0xa8,0x00,0x10,0x08,0x01,0xff,0x65,0xcc,0x8c,0x00,0x01,0xff,0x65,0xcc,0x8c, + 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x67,0xcc,0x82,0x00,0x01,0xff,0x67,0xcc,0x82, + 0x00,0x10,0x08,0x01,0xff,0x67,0xcc,0x86,0x00,0x01,0xff,0x67,0xcc,0x86,0x00,0xd4, + 0x7b,0xd3,0x3b,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x67,0xcc,0x87,0x00,0x01, + 0xff,0x67,0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x67,0xcc,0xa7,0x00,0x01,0xff,0x67, + 0xcc,0xa7,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x68,0xcc,0x82,0x00,0x01,0xff,0x68, + 0xcc,0x82,0x00,0x10,0x07,0x01,0xff,0xc4,0xa7,0x00,0x01,0x00,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x01,0xff,0x69,0xcc,0x83,0x00,0x01,0xff,0x69,0xcc,0x83,0x00,0x10,0x08, + 0x01,0xff,0x69,0xcc,0x84,0x00,0x01,0xff,0x69,0xcc,0x84,0x00,0xd1,0x10,0x10,0x08, + 0x01,0xff,0x69,0xcc,0x86,0x00,0x01,0xff,0x69,0xcc,0x86,0x00,0x10,0x08,0x01,0xff, + 0x69,0xcc,0xa8,0x00,0x01,0xff,0x69,0xcc,0xa8,0x00,0xd3,0x37,0xd2,0x17,0xd1,0x0c, + 0x10,0x08,0x01,0xff,0x69,0xcc,0x87,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xc4,0xb3, + 0x00,0x01,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x6a,0xcc,0x82,0x00,0x01,0xff,0x6a, + 0xcc,0x82,0x00,0x10,0x08,0x01,0xff,0x6b,0xcc,0xa7,0x00,0x01,0xff,0x6b,0xcc,0xa7, + 0x00,0xd2,0x1c,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0x6c,0xcc,0x81,0x00,0x10, + 0x08,0x01,0xff,0x6c,0xcc,0x81,0x00,0x01,0xff,0x6c,0xcc,0xa7,0x00,0xd1,0x10,0x10, + 0x08,0x01,0xff,0x6c,0xcc,0xa7,0x00,0x01,0xff,0x6c,0xcc,0x8c,0x00,0x10,0x08,0x01, + 0xff,0x6c,0xcc,0x8c,0x00,0x01,0xff,0xc5,0x80,0x00,0xcf,0x86,0xd5,0xed,0xd4,0x72, + 0xd3,0x37,0xd2,0x17,0xd1,0x0b,0x10,0x04,0x01,0x00,0x01,0xff,0xc5,0x82,0x00,0x10, + 0x04,0x01,0x00,0x01,0xff,0x6e,0xcc,0x81,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x6e, + 0xcc,0x81,0x00,0x01,0xff,0x6e,0xcc,0xa7,0x00,0x10,0x08,0x01,0xff,0x6e,0xcc,0xa7, + 0x00,0x01,0xff,0x6e,0xcc,0x8c,0x00,0xd2,0x1b,0xd1,0x10,0x10,0x08,0x01,0xff,0x6e, + 0xcc,0x8c,0x00,0x01,0xff,0xca,0xbc,0x6e,0x00,0x10,0x07,0x01,0xff,0xc5,0x8b,0x00, + 0x01,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x6f,0xcc,0x84,0x00,0x01,0xff,0x6f,0xcc, + 0x84,0x00,0x10,0x08,0x01,0xff,0x6f,0xcc,0x86,0x00,0x01,0xff,0x6f,0xcc,0x86,0x00, + 0xd3,0x3b,0xd2,0x1b,0xd1,0x10,0x10,0x08,0x01,0xff,0x6f,0xcc,0x8b,0x00,0x01,0xff, + 0x6f,0xcc,0x8b,0x00,0x10,0x07,0x01,0xff,0xc5,0x93,0x00,0x01,0x00,0xd1,0x10,0x10, + 0x08,0x01,0xff,0x72,0xcc,0x81,0x00,0x01,0xff,0x72,0xcc,0x81,0x00,0x10,0x08,0x01, + 0xff,0x72,0xcc,0xa7,0x00,0x01,0xff,0x72,0xcc,0xa7,0x00,0xd2,0x20,0xd1,0x10,0x10, + 0x08,0x01,0xff,0x72,0xcc,0x8c,0x00,0x01,0xff,0x72,0xcc,0x8c,0x00,0x10,0x08,0x01, + 0xff,0x73,0xcc,0x81,0x00,0x01,0xff,0x73,0xcc,0x81,0x00,0xd1,0x10,0x10,0x08,0x01, + 0xff,0x73,0xcc,0x82,0x00,0x01,0xff,0x73,0xcc,0x82,0x00,0x10,0x08,0x01,0xff,0x73, + 0xcc,0xa7,0x00,0x01,0xff,0x73,0xcc,0xa7,0x00,0xd4,0x7b,0xd3,0x3b,0xd2,0x20,0xd1, + 0x10,0x10,0x08,0x01,0xff,0x73,0xcc,0x8c,0x00,0x01,0xff,0x73,0xcc,0x8c,0x00,0x10, + 0x08,0x01,0xff,0x74,0xcc,0xa7,0x00,0x01,0xff,0x74,0xcc,0xa7,0x00,0xd1,0x10,0x10, + 0x08,0x01,0xff,0x74,0xcc,0x8c,0x00,0x01,0xff,0x74,0xcc,0x8c,0x00,0x10,0x07,0x01, + 0xff,0xc5,0xa7,0x00,0x01,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x75,0xcc, + 0x83,0x00,0x01,0xff,0x75,0xcc,0x83,0x00,0x10,0x08,0x01,0xff,0x75,0xcc,0x84,0x00, + 0x01,0xff,0x75,0xcc,0x84,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x75,0xcc,0x86,0x00, + 0x01,0xff,0x75,0xcc,0x86,0x00,0x10,0x08,0x01,0xff,0x75,0xcc,0x8a,0x00,0x01,0xff, + 0x75,0xcc,0x8a,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x75,0xcc, + 0x8b,0x00,0x01,0xff,0x75,0xcc,0x8b,0x00,0x10,0x08,0x01,0xff,0x75,0xcc,0xa8,0x00, + 0x01,0xff,0x75,0xcc,0xa8,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x77,0xcc,0x82,0x00, + 0x01,0xff,0x77,0xcc,0x82,0x00,0x10,0x08,0x01,0xff,0x79,0xcc,0x82,0x00,0x01,0xff, + 0x79,0xcc,0x82,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x79,0xcc,0x88,0x00, + 0x01,0xff,0x7a,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x7a,0xcc,0x81,0x00,0x01,0xff, + 0x7a,0xcc,0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x7a,0xcc,0x87,0x00,0x01,0xff, + 0x7a,0xcc,0x8c,0x00,0x10,0x08,0x01,0xff,0x7a,0xcc,0x8c,0x00,0x01,0xff,0x73,0x00, + 0xe0,0x65,0x01,0xcf,0x86,0xd5,0xb4,0xd4,0x5a,0xd3,0x2f,0xd2,0x16,0xd1,0x0b,0x10, + 0x04,0x01,0x00,0x01,0xff,0xc9,0x93,0x00,0x10,0x07,0x01,0xff,0xc6,0x83,0x00,0x01, + 0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xc6,0x85,0x00,0x01,0x00,0x10,0x07,0x01,0xff, + 0xc9,0x94,0x00,0x01,0xff,0xc6,0x88,0x00,0xd2,0x19,0xd1,0x0b,0x10,0x04,0x01,0x00, + 0x01,0xff,0xc9,0x96,0x00,0x10,0x07,0x01,0xff,0xc9,0x97,0x00,0x01,0xff,0xc6,0x8c, + 0x00,0x51,0x04,0x01,0x00,0x10,0x07,0x01,0xff,0xc7,0x9d,0x00,0x01,0xff,0xc9,0x99, + 0x00,0xd3,0x32,0xd2,0x19,0xd1,0x0e,0x10,0x07,0x01,0xff,0xc9,0x9b,0x00,0x01,0xff, + 0xc6,0x92,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0xc9,0xa0,0x00,0xd1,0x0b,0x10,0x07, + 0x01,0xff,0xc9,0xa3,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xc9,0xa9,0x00,0x01,0xff, + 0xc9,0xa8,0x00,0xd2,0x0f,0x91,0x0b,0x10,0x07,0x01,0xff,0xc6,0x99,0x00,0x01,0x00, + 0x01,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xc9,0xaf,0x00,0x01,0xff,0xc9,0xb2,0x00, + 0x10,0x04,0x01,0x00,0x01,0xff,0xc9,0xb5,0x00,0xd4,0x5d,0xd3,0x34,0xd2,0x1b,0xd1, + 0x10,0x10,0x08,0x01,0xff,0x6f,0xcc,0x9b,0x00,0x01,0xff,0x6f,0xcc,0x9b,0x00,0x10, + 0x07,0x01,0xff,0xc6,0xa3,0x00,0x01,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xc6,0xa5, + 0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xca,0x80,0x00,0x01,0xff,0xc6,0xa8,0x00,0xd2, + 0x0f,0x91,0x0b,0x10,0x04,0x01,0x00,0x01,0xff,0xca,0x83,0x00,0x01,0x00,0xd1,0x0b, + 0x10,0x07,0x01,0xff,0xc6,0xad,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xca,0x88,0x00, + 0x01,0xff,0x75,0xcc,0x9b,0x00,0xd3,0x33,0xd2,0x1d,0xd1,0x0f,0x10,0x08,0x01,0xff, + 0x75,0xcc,0x9b,0x00,0x01,0xff,0xca,0x8a,0x00,0x10,0x07,0x01,0xff,0xca,0x8b,0x00, + 0x01,0xff,0xc6,0xb4,0x00,0xd1,0x0b,0x10,0x04,0x01,0x00,0x01,0xff,0xc6,0xb6,0x00, + 0x10,0x04,0x01,0x00,0x01,0xff,0xca,0x92,0x00,0xd2,0x0f,0x91,0x0b,0x10,0x07,0x01, + 0xff,0xc6,0xb9,0x00,0x01,0x00,0x01,0x00,0x91,0x0b,0x10,0x07,0x01,0xff,0xc6,0xbd, + 0x00,0x01,0x00,0x01,0x00,0xcf,0x86,0xd5,0xd4,0xd4,0x44,0xd3,0x16,0x52,0x04,0x01, + 0x00,0x51,0x07,0x01,0xff,0xc7,0x86,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0xc7,0x89, + 0x00,0xd2,0x12,0x91,0x0b,0x10,0x07,0x01,0xff,0xc7,0x89,0x00,0x01,0x00,0x01,0xff, + 0xc7,0x8c,0x00,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0x61,0xcc,0x8c,0x00,0x10, + 0x08,0x01,0xff,0x61,0xcc,0x8c,0x00,0x01,0xff,0x69,0xcc,0x8c,0x00,0xd3,0x46,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x69,0xcc,0x8c,0x00,0x01,0xff,0x6f,0xcc,0x8c, + 0x00,0x10,0x08,0x01,0xff,0x6f,0xcc,0x8c,0x00,0x01,0xff,0x75,0xcc,0x8c,0x00,0xd1, + 0x12,0x10,0x08,0x01,0xff,0x75,0xcc,0x8c,0x00,0x01,0xff,0x75,0xcc,0x88,0xcc,0x84, + 0x00,0x10,0x0a,0x01,0xff,0x75,0xcc,0x88,0xcc,0x84,0x00,0x01,0xff,0x75,0xcc,0x88, + 0xcc,0x81,0x00,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x75,0xcc,0x88,0xcc,0x81, + 0x00,0x01,0xff,0x75,0xcc,0x88,0xcc,0x8c,0x00,0x10,0x0a,0x01,0xff,0x75,0xcc,0x88, + 0xcc,0x8c,0x00,0x01,0xff,0x75,0xcc,0x88,0xcc,0x80,0x00,0xd1,0x0e,0x10,0x0a,0x01, + 0xff,0x75,0xcc,0x88,0xcc,0x80,0x00,0x01,0x00,0x10,0x0a,0x01,0xff,0x61,0xcc,0x88, + 0xcc,0x84,0x00,0x01,0xff,0x61,0xcc,0x88,0xcc,0x84,0x00,0xd4,0x87,0xd3,0x41,0xd2, + 0x26,0xd1,0x14,0x10,0x0a,0x01,0xff,0x61,0xcc,0x87,0xcc,0x84,0x00,0x01,0xff,0x61, + 0xcc,0x87,0xcc,0x84,0x00,0x10,0x09,0x01,0xff,0xc3,0xa6,0xcc,0x84,0x00,0x01,0xff, + 0xc3,0xa6,0xcc,0x84,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xc7,0xa5,0x00,0x01,0x00, + 0x10,0x08,0x01,0xff,0x67,0xcc,0x8c,0x00,0x01,0xff,0x67,0xcc,0x8c,0x00,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x01,0xff,0x6b,0xcc,0x8c,0x00,0x01,0xff,0x6b,0xcc,0x8c,0x00, + 0x10,0x08,0x01,0xff,0x6f,0xcc,0xa8,0x00,0x01,0xff,0x6f,0xcc,0xa8,0x00,0xd1,0x14, + 0x10,0x0a,0x01,0xff,0x6f,0xcc,0xa8,0xcc,0x84,0x00,0x01,0xff,0x6f,0xcc,0xa8,0xcc, + 0x84,0x00,0x10,0x09,0x01,0xff,0xca,0x92,0xcc,0x8c,0x00,0x01,0xff,0xca,0x92,0xcc, + 0x8c,0x00,0xd3,0x38,0xd2,0x1a,0xd1,0x0f,0x10,0x08,0x01,0xff,0x6a,0xcc,0x8c,0x00, + 0x01,0xff,0xc7,0xb3,0x00,0x10,0x07,0x01,0xff,0xc7,0xb3,0x00,0x01,0x00,0xd1,0x10, + 0x10,0x08,0x01,0xff,0x67,0xcc,0x81,0x00,0x01,0xff,0x67,0xcc,0x81,0x00,0x10,0x07, + 0x04,0xff,0xc6,0x95,0x00,0x04,0xff,0xc6,0xbf,0x00,0xd2,0x24,0xd1,0x10,0x10,0x08, + 0x04,0xff,0x6e,0xcc,0x80,0x00,0x04,0xff,0x6e,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff, + 0x61,0xcc,0x8a,0xcc,0x81,0x00,0x01,0xff,0x61,0xcc,0x8a,0xcc,0x81,0x00,0xd1,0x12, + 0x10,0x09,0x01,0xff,0xc3,0xa6,0xcc,0x81,0x00,0x01,0xff,0xc3,0xa6,0xcc,0x81,0x00, + 0x10,0x09,0x01,0xff,0xc3,0xb8,0xcc,0x81,0x00,0x01,0xff,0xc3,0xb8,0xcc,0x81,0x00, + 0xe2,0x31,0x02,0xe1,0xc3,0x44,0xe0,0xc8,0x01,0xcf,0x86,0xd5,0xfb,0xd4,0x80,0xd3, + 0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x61,0xcc,0x8f,0x00,0x01,0xff,0x61, + 0xcc,0x8f,0x00,0x10,0x08,0x01,0xff,0x61,0xcc,0x91,0x00,0x01,0xff,0x61,0xcc,0x91, + 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x65,0xcc,0x8f,0x00,0x01,0xff,0x65,0xcc,0x8f, + 0x00,0x10,0x08,0x01,0xff,0x65,0xcc,0x91,0x00,0x01,0xff,0x65,0xcc,0x91,0x00,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x69,0xcc,0x8f,0x00,0x01,0xff,0x69,0xcc,0x8f, + 0x00,0x10,0x08,0x01,0xff,0x69,0xcc,0x91,0x00,0x01,0xff,0x69,0xcc,0x91,0x00,0xd1, + 0x10,0x10,0x08,0x01,0xff,0x6f,0xcc,0x8f,0x00,0x01,0xff,0x6f,0xcc,0x8f,0x00,0x10, + 0x08,0x01,0xff,0x6f,0xcc,0x91,0x00,0x01,0xff,0x6f,0xcc,0x91,0x00,0xd3,0x40,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x72,0xcc,0x8f,0x00,0x01,0xff,0x72,0xcc,0x8f, + 0x00,0x10,0x08,0x01,0xff,0x72,0xcc,0x91,0x00,0x01,0xff,0x72,0xcc,0x91,0x00,0xd1, + 0x10,0x10,0x08,0x01,0xff,0x75,0xcc,0x8f,0x00,0x01,0xff,0x75,0xcc,0x8f,0x00,0x10, + 0x08,0x01,0xff,0x75,0xcc,0x91,0x00,0x01,0xff,0x75,0xcc,0x91,0x00,0xd2,0x20,0xd1, + 0x10,0x10,0x08,0x04,0xff,0x73,0xcc,0xa6,0x00,0x04,0xff,0x73,0xcc,0xa6,0x00,0x10, + 0x08,0x04,0xff,0x74,0xcc,0xa6,0x00,0x04,0xff,0x74,0xcc,0xa6,0x00,0xd1,0x0b,0x10, + 0x07,0x04,0xff,0xc8,0x9d,0x00,0x04,0x00,0x10,0x08,0x04,0xff,0x68,0xcc,0x8c,0x00, + 0x04,0xff,0x68,0xcc,0x8c,0x00,0xd4,0x79,0xd3,0x31,0xd2,0x16,0xd1,0x0b,0x10,0x07, + 0x06,0xff,0xc6,0x9e,0x00,0x07,0x00,0x10,0x07,0x04,0xff,0xc8,0xa3,0x00,0x04,0x00, + 0xd1,0x0b,0x10,0x07,0x04,0xff,0xc8,0xa5,0x00,0x04,0x00,0x10,0x08,0x04,0xff,0x61, + 0xcc,0x87,0x00,0x04,0xff,0x61,0xcc,0x87,0x00,0xd2,0x24,0xd1,0x10,0x10,0x08,0x04, + 0xff,0x65,0xcc,0xa7,0x00,0x04,0xff,0x65,0xcc,0xa7,0x00,0x10,0x0a,0x04,0xff,0x6f, + 0xcc,0x88,0xcc,0x84,0x00,0x04,0xff,0x6f,0xcc,0x88,0xcc,0x84,0x00,0xd1,0x14,0x10, + 0x0a,0x04,0xff,0x6f,0xcc,0x83,0xcc,0x84,0x00,0x04,0xff,0x6f,0xcc,0x83,0xcc,0x84, + 0x00,0x10,0x08,0x04,0xff,0x6f,0xcc,0x87,0x00,0x04,0xff,0x6f,0xcc,0x87,0x00,0xd3, + 0x27,0xe2,0x21,0x43,0xd1,0x14,0x10,0x0a,0x04,0xff,0x6f,0xcc,0x87,0xcc,0x84,0x00, + 0x04,0xff,0x6f,0xcc,0x87,0xcc,0x84,0x00,0x10,0x08,0x04,0xff,0x79,0xcc,0x84,0x00, + 0x04,0xff,0x79,0xcc,0x84,0x00,0xd2,0x13,0x51,0x04,0x08,0x00,0x10,0x08,0x08,0xff, + 0xe2,0xb1,0xa5,0x00,0x08,0xff,0xc8,0xbc,0x00,0xd1,0x0b,0x10,0x04,0x08,0x00,0x08, + 0xff,0xc6,0x9a,0x00,0x10,0x08,0x08,0xff,0xe2,0xb1,0xa6,0x00,0x08,0x00,0xcf,0x86, + 0x95,0x5f,0x94,0x5b,0xd3,0x2f,0xd2,0x16,0xd1,0x0b,0x10,0x04,0x08,0x00,0x08,0xff, + 0xc9,0x82,0x00,0x10,0x04,0x09,0x00,0x09,0xff,0xc6,0x80,0x00,0xd1,0x0e,0x10,0x07, + 0x09,0xff,0xca,0x89,0x00,0x09,0xff,0xca,0x8c,0x00,0x10,0x07,0x09,0xff,0xc9,0x87, + 0x00,0x09,0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x09,0xff,0xc9,0x89,0x00,0x09,0x00, + 0x10,0x07,0x09,0xff,0xc9,0x8b,0x00,0x09,0x00,0xd1,0x0b,0x10,0x07,0x09,0xff,0xc9, + 0x8d,0x00,0x09,0x00,0x10,0x07,0x09,0xff,0xc9,0x8f,0x00,0x09,0x00,0x01,0x00,0x01, + 0x00,0xd1,0x8b,0xd0,0x0c,0xcf,0x86,0xe5,0x10,0x43,0x64,0xef,0x42,0x01,0xe6,0xcf, + 0x86,0xd5,0x2a,0xe4,0x99,0x43,0xe3,0x7f,0x43,0xd2,0x11,0xe1,0x5e,0x43,0x10,0x07, + 0x01,0xff,0xcc,0x80,0x00,0x01,0xff,0xcc,0x81,0x00,0xe1,0x65,0x43,0x10,0x09,0x01, + 0xff,0xcc,0x88,0xcc,0x81,0x00,0x01,0xff,0xce,0xb9,0x00,0xd4,0x0f,0x93,0x0b,0x92, + 0x07,0x61,0xab,0x43,0x01,0xea,0x06,0xe6,0x06,0xe6,0xd3,0x2c,0xd2,0x16,0xd1,0x0b, + 0x10,0x07,0x0a,0xff,0xcd,0xb1,0x00,0x0a,0x00,0x10,0x07,0x0a,0xff,0xcd,0xb3,0x00, + 0x0a,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xca,0xb9,0x00,0x01,0x00,0x10,0x07,0x0a, + 0xff,0xcd,0xb7,0x00,0x0a,0x00,0xd2,0x07,0x61,0x97,0x43,0x00,0x00,0x51,0x04,0x09, + 0x00,0x10,0x06,0x01,0xff,0x3b,0x00,0x10,0xff,0xcf,0xb3,0x00,0xe0,0x31,0x01,0xcf, + 0x86,0xd5,0xd3,0xd4,0x5f,0xd3,0x21,0x52,0x04,0x00,0x00,0xd1,0x0d,0x10,0x04,0x01, + 0x00,0x01,0xff,0xc2,0xa8,0xcc,0x81,0x00,0x10,0x09,0x01,0xff,0xce,0xb1,0xcc,0x81, + 0x00,0x01,0xff,0xc2,0xb7,0x00,0xd2,0x1f,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb5, + 0xcc,0x81,0x00,0x01,0xff,0xce,0xb7,0xcc,0x81,0x00,0x10,0x09,0x01,0xff,0xce,0xb9, + 0xcc,0x81,0x00,0x00,0x00,0xd1,0x0d,0x10,0x09,0x01,0xff,0xce,0xbf,0xcc,0x81,0x00, + 0x00,0x00,0x10,0x09,0x01,0xff,0xcf,0x85,0xcc,0x81,0x00,0x01,0xff,0xcf,0x89,0xcc, + 0x81,0x00,0xd3,0x3c,0xd2,0x20,0xd1,0x12,0x10,0x0b,0x01,0xff,0xce,0xb9,0xcc,0x88, + 0xcc,0x81,0x00,0x01,0xff,0xce,0xb1,0x00,0x10,0x07,0x01,0xff,0xce,0xb2,0x00,0x01, + 0xff,0xce,0xb3,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xce,0xb4,0x00,0x01,0xff,0xce, + 0xb5,0x00,0x10,0x07,0x01,0xff,0xce,0xb6,0x00,0x01,0xff,0xce,0xb7,0x00,0xd2,0x1c, + 0xd1,0x0e,0x10,0x07,0x01,0xff,0xce,0xb8,0x00,0x01,0xff,0xce,0xb9,0x00,0x10,0x07, + 0x01,0xff,0xce,0xba,0x00,0x01,0xff,0xce,0xbb,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff, + 0xce,0xbc,0x00,0x01,0xff,0xce,0xbd,0x00,0x10,0x07,0x01,0xff,0xce,0xbe,0x00,0x01, + 0xff,0xce,0xbf,0x00,0xe4,0x85,0x43,0xd3,0x35,0xd2,0x19,0xd1,0x0e,0x10,0x07,0x01, + 0xff,0xcf,0x80,0x00,0x01,0xff,0xcf,0x81,0x00,0x10,0x04,0x00,0x00,0x01,0xff,0xcf, + 0x83,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xcf,0x84,0x00,0x01,0xff,0xcf,0x85,0x00, + 0x10,0x07,0x01,0xff,0xcf,0x86,0x00,0x01,0xff,0xcf,0x87,0x00,0xe2,0x2b,0x43,0xd1, + 0x0e,0x10,0x07,0x01,0xff,0xcf,0x88,0x00,0x01,0xff,0xcf,0x89,0x00,0x10,0x09,0x01, + 0xff,0xce,0xb9,0xcc,0x88,0x00,0x01,0xff,0xcf,0x85,0xcc,0x88,0x00,0xcf,0x86,0xd5, + 0x94,0xd4,0x3c,0xd3,0x13,0x92,0x0f,0x51,0x04,0x01,0x00,0x10,0x07,0x01,0xff,0xcf, + 0x83,0x00,0x01,0x00,0x01,0x00,0xd2,0x07,0x61,0x3a,0x43,0x01,0x00,0xd1,0x12,0x10, + 0x09,0x01,0xff,0xce,0xbf,0xcc,0x81,0x00,0x01,0xff,0xcf,0x85,0xcc,0x81,0x00,0x10, + 0x09,0x01,0xff,0xcf,0x89,0xcc,0x81,0x00,0x0a,0xff,0xcf,0x97,0x00,0xd3,0x2c,0xd2, + 0x11,0xe1,0x46,0x43,0x10,0x07,0x01,0xff,0xce,0xb2,0x00,0x01,0xff,0xce,0xb8,0x00, + 0xd1,0x10,0x10,0x09,0x01,0xff,0xcf,0x92,0xcc,0x88,0x00,0x01,0xff,0xcf,0x86,0x00, + 0x10,0x07,0x01,0xff,0xcf,0x80,0x00,0x04,0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x06, + 0xff,0xcf,0x99,0x00,0x06,0x00,0x10,0x07,0x01,0xff,0xcf,0x9b,0x00,0x04,0x00,0xd1, + 0x0b,0x10,0x07,0x01,0xff,0xcf,0x9d,0x00,0x04,0x00,0x10,0x07,0x01,0xff,0xcf,0x9f, + 0x00,0x04,0x00,0xd4,0x58,0xd3,0x2c,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xcf, + 0xa1,0x00,0x04,0x00,0x10,0x07,0x01,0xff,0xcf,0xa3,0x00,0x01,0x00,0xd1,0x0b,0x10, + 0x07,0x01,0xff,0xcf,0xa5,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xcf,0xa7,0x00,0x01, + 0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xcf,0xa9,0x00,0x01,0x00,0x10,0x07, + 0x01,0xff,0xcf,0xab,0x00,0x01,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xcf,0xad,0x00, + 0x01,0x00,0x10,0x07,0x01,0xff,0xcf,0xaf,0x00,0x01,0x00,0xd3,0x2b,0xd2,0x12,0x91, + 0x0e,0x10,0x07,0x01,0xff,0xce,0xba,0x00,0x01,0xff,0xcf,0x81,0x00,0x01,0x00,0xd1, + 0x0e,0x10,0x07,0x05,0xff,0xce,0xb8,0x00,0x05,0xff,0xce,0xb5,0x00,0x10,0x04,0x06, + 0x00,0x07,0xff,0xcf,0xb8,0x00,0xd2,0x16,0xd1,0x0b,0x10,0x04,0x07,0x00,0x07,0xff, + 0xcf,0xb2,0x00,0x10,0x07,0x07,0xff,0xcf,0xbb,0x00,0x07,0x00,0xd1,0x0b,0x10,0x04, + 0x08,0x00,0x08,0xff,0xcd,0xbb,0x00,0x10,0x07,0x08,0xff,0xcd,0xbc,0x00,0x08,0xff, + 0xcd,0xbd,0x00,0xe3,0xed,0x46,0xe2,0x3d,0x05,0xe1,0x27,0x02,0xe0,0x66,0x01,0xcf, + 0x86,0xd5,0xf0,0xd4,0x7e,0xd3,0x40,0xd2,0x22,0xd1,0x12,0x10,0x09,0x04,0xff,0xd0, + 0xb5,0xcc,0x80,0x00,0x01,0xff,0xd0,0xb5,0xcc,0x88,0x00,0x10,0x07,0x01,0xff,0xd1, + 0x92,0x00,0x01,0xff,0xd0,0xb3,0xcc,0x81,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd1, + 0x94,0x00,0x01,0xff,0xd1,0x95,0x00,0x10,0x07,0x01,0xff,0xd1,0x96,0x00,0x01,0xff, + 0xd1,0x96,0xcc,0x88,0x00,0xd2,0x1c,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd1,0x98,0x00, + 0x01,0xff,0xd1,0x99,0x00,0x10,0x07,0x01,0xff,0xd1,0x9a,0x00,0x01,0xff,0xd1,0x9b, + 0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xd0,0xba,0xcc,0x81,0x00,0x04,0xff,0xd0,0xb8, + 0xcc,0x80,0x00,0x10,0x09,0x01,0xff,0xd1,0x83,0xcc,0x86,0x00,0x01,0xff,0xd1,0x9f, + 0x00,0xd3,0x38,0xd2,0x1c,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd0,0xb0,0x00,0x01,0xff, + 0xd0,0xb1,0x00,0x10,0x07,0x01,0xff,0xd0,0xb2,0x00,0x01,0xff,0xd0,0xb3,0x00,0xd1, + 0x0e,0x10,0x07,0x01,0xff,0xd0,0xb4,0x00,0x01,0xff,0xd0,0xb5,0x00,0x10,0x07,0x01, + 0xff,0xd0,0xb6,0x00,0x01,0xff,0xd0,0xb7,0x00,0xd2,0x1e,0xd1,0x10,0x10,0x07,0x01, + 0xff,0xd0,0xb8,0x00,0x01,0xff,0xd0,0xb8,0xcc,0x86,0x00,0x10,0x07,0x01,0xff,0xd0, + 0xba,0x00,0x01,0xff,0xd0,0xbb,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd0,0xbc,0x00, + 0x01,0xff,0xd0,0xbd,0x00,0x10,0x07,0x01,0xff,0xd0,0xbe,0x00,0x01,0xff,0xd0,0xbf, + 0x00,0xe4,0x25,0x42,0xd3,0x38,0xd2,0x1c,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd1,0x80, + 0x00,0x01,0xff,0xd1,0x81,0x00,0x10,0x07,0x01,0xff,0xd1,0x82,0x00,0x01,0xff,0xd1, + 0x83,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd1,0x84,0x00,0x01,0xff,0xd1,0x85,0x00, + 0x10,0x07,0x01,0xff,0xd1,0x86,0x00,0x01,0xff,0xd1,0x87,0x00,0xd2,0x1c,0xd1,0x0e, + 0x10,0x07,0x01,0xff,0xd1,0x88,0x00,0x01,0xff,0xd1,0x89,0x00,0x10,0x07,0x01,0xff, + 0xd1,0x8a,0x00,0x01,0xff,0xd1,0x8b,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd1,0x8c, + 0x00,0x01,0xff,0xd1,0x8d,0x00,0x10,0x07,0x01,0xff,0xd1,0x8e,0x00,0x01,0xff,0xd1, + 0x8f,0x00,0xcf,0x86,0xd5,0x07,0x64,0xcf,0x41,0x01,0x00,0xd4,0x58,0xd3,0x2c,0xd2, + 0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd1,0xa1,0x00,0x01,0x00,0x10,0x07,0x01,0xff, + 0xd1,0xa3,0x00,0x01,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd1,0xa5,0x00,0x01,0x00, + 0x10,0x07,0x01,0xff,0xd1,0xa7,0x00,0x01,0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01, + 0xff,0xd1,0xa9,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd1,0xab,0x00,0x01,0x00,0xd1, + 0x0b,0x10,0x07,0x01,0xff,0xd1,0xad,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd1,0xaf, + 0x00,0x01,0x00,0xd3,0x33,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd1,0xb1,0x00, + 0x01,0x00,0x10,0x07,0x01,0xff,0xd1,0xb3,0x00,0x01,0x00,0xd1,0x0b,0x10,0x07,0x01, + 0xff,0xd1,0xb5,0x00,0x01,0x00,0x10,0x09,0x01,0xff,0xd1,0xb5,0xcc,0x8f,0x00,0x01, + 0xff,0xd1,0xb5,0xcc,0x8f,0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd1,0xb9, + 0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd1,0xbb,0x00,0x01,0x00,0xd1,0x0b,0x10,0x07, + 0x01,0xff,0xd1,0xbd,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd1,0xbf,0x00,0x01,0x00, + 0xe0,0x41,0x01,0xcf,0x86,0xd5,0x8e,0xd4,0x36,0xd3,0x11,0xe2,0x91,0x41,0xe1,0x88, + 0x41,0x10,0x07,0x01,0xff,0xd2,0x81,0x00,0x01,0x00,0xd2,0x0f,0x51,0x04,0x04,0x00, + 0x10,0x07,0x06,0xff,0xd2,0x8b,0x00,0x06,0x00,0xd1,0x0b,0x10,0x07,0x04,0xff,0xd2, + 0x8d,0x00,0x04,0x00,0x10,0x07,0x04,0xff,0xd2,0x8f,0x00,0x04,0x00,0xd3,0x2c,0xd2, + 0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd2,0x91,0x00,0x01,0x00,0x10,0x07,0x01,0xff, + 0xd2,0x93,0x00,0x01,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd2,0x95,0x00,0x01,0x00, + 0x10,0x07,0x01,0xff,0xd2,0x97,0x00,0x01,0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01, + 0xff,0xd2,0x99,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd2,0x9b,0x00,0x01,0x00,0xd1, + 0x0b,0x10,0x07,0x01,0xff,0xd2,0x9d,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd2,0x9f, + 0x00,0x01,0x00,0xd4,0x58,0xd3,0x2c,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd2, + 0xa1,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd2,0xa3,0x00,0x01,0x00,0xd1,0x0b,0x10, + 0x07,0x01,0xff,0xd2,0xa5,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd2,0xa7,0x00,0x01, + 0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd2,0xa9,0x00,0x01,0x00,0x10,0x07, + 0x01,0xff,0xd2,0xab,0x00,0x01,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd2,0xad,0x00, + 0x01,0x00,0x10,0x07,0x01,0xff,0xd2,0xaf,0x00,0x01,0x00,0xd3,0x2c,0xd2,0x16,0xd1, + 0x0b,0x10,0x07,0x01,0xff,0xd2,0xb1,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd2,0xb3, + 0x00,0x01,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd2,0xb5,0x00,0x01,0x00,0x10,0x07, + 0x01,0xff,0xd2,0xb7,0x00,0x01,0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd2, + 0xb9,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd2,0xbb,0x00,0x01,0x00,0xd1,0x0b,0x10, + 0x07,0x01,0xff,0xd2,0xbd,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xd2,0xbf,0x00,0x01, + 0x00,0xcf,0x86,0xd5,0xdc,0xd4,0x5a,0xd3,0x36,0xd2,0x20,0xd1,0x10,0x10,0x07,0x01, + 0xff,0xd3,0x8f,0x00,0x01,0xff,0xd0,0xb6,0xcc,0x86,0x00,0x10,0x09,0x01,0xff,0xd0, + 0xb6,0xcc,0x86,0x00,0x01,0xff,0xd3,0x84,0x00,0xd1,0x0b,0x10,0x04,0x01,0x00,0x06, + 0xff,0xd3,0x86,0x00,0x10,0x04,0x06,0x00,0x01,0xff,0xd3,0x88,0x00,0xd2,0x16,0xd1, + 0x0b,0x10,0x04,0x01,0x00,0x06,0xff,0xd3,0x8a,0x00,0x10,0x04,0x06,0x00,0x01,0xff, + 0xd3,0x8c,0x00,0xe1,0x69,0x40,0x10,0x04,0x01,0x00,0x06,0xff,0xd3,0x8e,0x00,0xd3, + 0x41,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xd0,0xb0,0xcc,0x86,0x00,0x01,0xff, + 0xd0,0xb0,0xcc,0x86,0x00,0x10,0x09,0x01,0xff,0xd0,0xb0,0xcc,0x88,0x00,0x01,0xff, + 0xd0,0xb0,0xcc,0x88,0x00,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd3,0x95,0x00,0x01,0x00, + 0x10,0x09,0x01,0xff,0xd0,0xb5,0xcc,0x86,0x00,0x01,0xff,0xd0,0xb5,0xcc,0x86,0x00, + 0xd2,0x1d,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd3,0x99,0x00,0x01,0x00,0x10,0x09,0x01, + 0xff,0xd3,0x99,0xcc,0x88,0x00,0x01,0xff,0xd3,0x99,0xcc,0x88,0x00,0xd1,0x12,0x10, + 0x09,0x01,0xff,0xd0,0xb6,0xcc,0x88,0x00,0x01,0xff,0xd0,0xb6,0xcc,0x88,0x00,0x10, + 0x09,0x01,0xff,0xd0,0xb7,0xcc,0x88,0x00,0x01,0xff,0xd0,0xb7,0xcc,0x88,0x00,0xd4, + 0x82,0xd3,0x41,0xd2,0x1d,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd3,0xa1,0x00,0x01,0x00, + 0x10,0x09,0x01,0xff,0xd0,0xb8,0xcc,0x84,0x00,0x01,0xff,0xd0,0xb8,0xcc,0x84,0x00, + 0xd1,0x12,0x10,0x09,0x01,0xff,0xd0,0xb8,0xcc,0x88,0x00,0x01,0xff,0xd0,0xb8,0xcc, + 0x88,0x00,0x10,0x09,0x01,0xff,0xd0,0xbe,0xcc,0x88,0x00,0x01,0xff,0xd0,0xbe,0xcc, + 0x88,0x00,0xd2,0x1d,0xd1,0x0b,0x10,0x07,0x01,0xff,0xd3,0xa9,0x00,0x01,0x00,0x10, + 0x09,0x01,0xff,0xd3,0xa9,0xcc,0x88,0x00,0x01,0xff,0xd3,0xa9,0xcc,0x88,0x00,0xd1, + 0x12,0x10,0x09,0x04,0xff,0xd1,0x8d,0xcc,0x88,0x00,0x04,0xff,0xd1,0x8d,0xcc,0x88, + 0x00,0x10,0x09,0x01,0xff,0xd1,0x83,0xcc,0x84,0x00,0x01,0xff,0xd1,0x83,0xcc,0x84, + 0x00,0xd3,0x41,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xd1,0x83,0xcc,0x88,0x00, + 0x01,0xff,0xd1,0x83,0xcc,0x88,0x00,0x10,0x09,0x01,0xff,0xd1,0x83,0xcc,0x8b,0x00, + 0x01,0xff,0xd1,0x83,0xcc,0x8b,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xd1,0x87,0xcc, + 0x88,0x00,0x01,0xff,0xd1,0x87,0xcc,0x88,0x00,0x10,0x07,0x08,0xff,0xd3,0xb7,0x00, + 0x08,0x00,0xd2,0x1d,0xd1,0x12,0x10,0x09,0x01,0xff,0xd1,0x8b,0xcc,0x88,0x00,0x01, + 0xff,0xd1,0x8b,0xcc,0x88,0x00,0x10,0x07,0x09,0xff,0xd3,0xbb,0x00,0x09,0x00,0xd1, + 0x0b,0x10,0x07,0x09,0xff,0xd3,0xbd,0x00,0x09,0x00,0x10,0x07,0x09,0xff,0xd3,0xbf, + 0x00,0x09,0x00,0xe1,0x26,0x02,0xe0,0x78,0x01,0xcf,0x86,0xd5,0xb0,0xd4,0x58,0xd3, + 0x2c,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x06,0xff,0xd4,0x81,0x00,0x06,0x00,0x10,0x07, + 0x06,0xff,0xd4,0x83,0x00,0x06,0x00,0xd1,0x0b,0x10,0x07,0x06,0xff,0xd4,0x85,0x00, + 0x06,0x00,0x10,0x07,0x06,0xff,0xd4,0x87,0x00,0x06,0x00,0xd2,0x16,0xd1,0x0b,0x10, + 0x07,0x06,0xff,0xd4,0x89,0x00,0x06,0x00,0x10,0x07,0x06,0xff,0xd4,0x8b,0x00,0x06, + 0x00,0xd1,0x0b,0x10,0x07,0x06,0xff,0xd4,0x8d,0x00,0x06,0x00,0x10,0x07,0x06,0xff, + 0xd4,0x8f,0x00,0x06,0x00,0xd3,0x2c,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x09,0xff,0xd4, + 0x91,0x00,0x09,0x00,0x10,0x07,0x09,0xff,0xd4,0x93,0x00,0x09,0x00,0xd1,0x0b,0x10, + 0x07,0x0a,0xff,0xd4,0x95,0x00,0x0a,0x00,0x10,0x07,0x0a,0xff,0xd4,0x97,0x00,0x0a, + 0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x0a,0xff,0xd4,0x99,0x00,0x0a,0x00,0x10,0x07, + 0x0a,0xff,0xd4,0x9b,0x00,0x0a,0x00,0xd1,0x0b,0x10,0x07,0x0a,0xff,0xd4,0x9d,0x00, + 0x0a,0x00,0x10,0x07,0x0a,0xff,0xd4,0x9f,0x00,0x0a,0x00,0xd4,0x58,0xd3,0x2c,0xd2, + 0x16,0xd1,0x0b,0x10,0x07,0x0a,0xff,0xd4,0xa1,0x00,0x0a,0x00,0x10,0x07,0x0a,0xff, + 0xd4,0xa3,0x00,0x0a,0x00,0xd1,0x0b,0x10,0x07,0x0b,0xff,0xd4,0xa5,0x00,0x0b,0x00, + 0x10,0x07,0x0c,0xff,0xd4,0xa7,0x00,0x0c,0x00,0xd2,0x16,0xd1,0x0b,0x10,0x07,0x10, + 0xff,0xd4,0xa9,0x00,0x10,0x00,0x10,0x07,0x10,0xff,0xd4,0xab,0x00,0x10,0x00,0xd1, + 0x0b,0x10,0x07,0x10,0xff,0xd4,0xad,0x00,0x10,0x00,0x10,0x07,0x10,0xff,0xd4,0xaf, + 0x00,0x10,0x00,0xd3,0x35,0xd2,0x19,0xd1,0x0b,0x10,0x04,0x00,0x00,0x01,0xff,0xd5, + 0xa1,0x00,0x10,0x07,0x01,0xff,0xd5,0xa2,0x00,0x01,0xff,0xd5,0xa3,0x00,0xd1,0x0e, + 0x10,0x07,0x01,0xff,0xd5,0xa4,0x00,0x01,0xff,0xd5,0xa5,0x00,0x10,0x07,0x01,0xff, + 0xd5,0xa6,0x00,0x01,0xff,0xd5,0xa7,0x00,0xd2,0x1c,0xd1,0x0e,0x10,0x07,0x01,0xff, + 0xd5,0xa8,0x00,0x01,0xff,0xd5,0xa9,0x00,0x10,0x07,0x01,0xff,0xd5,0xaa,0x00,0x01, + 0xff,0xd5,0xab,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd5,0xac,0x00,0x01,0xff,0xd5, + 0xad,0x00,0x10,0x07,0x01,0xff,0xd5,0xae,0x00,0x01,0xff,0xd5,0xaf,0x00,0xcf,0x86, + 0xe5,0x08,0x3f,0xd4,0x70,0xd3,0x38,0xd2,0x1c,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd5, + 0xb0,0x00,0x01,0xff,0xd5,0xb1,0x00,0x10,0x07,0x01,0xff,0xd5,0xb2,0x00,0x01,0xff, + 0xd5,0xb3,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd5,0xb4,0x00,0x01,0xff,0xd5,0xb5, + 0x00,0x10,0x07,0x01,0xff,0xd5,0xb6,0x00,0x01,0xff,0xd5,0xb7,0x00,0xd2,0x1c,0xd1, + 0x0e,0x10,0x07,0x01,0xff,0xd5,0xb8,0x00,0x01,0xff,0xd5,0xb9,0x00,0x10,0x07,0x01, + 0xff,0xd5,0xba,0x00,0x01,0xff,0xd5,0xbb,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd5, + 0xbc,0x00,0x01,0xff,0xd5,0xbd,0x00,0x10,0x07,0x01,0xff,0xd5,0xbe,0x00,0x01,0xff, + 0xd5,0xbf,0x00,0xe3,0x87,0x3e,0xd2,0x1c,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd6,0x80, + 0x00,0x01,0xff,0xd6,0x81,0x00,0x10,0x07,0x01,0xff,0xd6,0x82,0x00,0x01,0xff,0xd6, + 0x83,0x00,0xd1,0x0e,0x10,0x07,0x01,0xff,0xd6,0x84,0x00,0x01,0xff,0xd6,0x85,0x00, + 0x10,0x07,0x01,0xff,0xd6,0x86,0x00,0x00,0x00,0xe0,0x2f,0x3f,0xcf,0x86,0xe5,0xc0, + 0x3e,0xe4,0x97,0x3e,0xe3,0x76,0x3e,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10, + 0x04,0x01,0x00,0x01,0xff,0xd5,0xa5,0xd6,0x82,0x00,0xe4,0x3e,0x25,0xe3,0xc3,0x1a, + 0xe2,0x7b,0x81,0xe1,0xc0,0x13,0xd0,0x1e,0xcf,0x86,0xc5,0xe4,0x08,0x4b,0xe3,0x53, + 0x46,0xe2,0xe9,0x43,0xe1,0x1c,0x43,0xe0,0xe1,0x42,0xcf,0x86,0xe5,0xa6,0x42,0x64, + 0x89,0x42,0x0b,0x00,0xcf,0x86,0xe5,0xfa,0x01,0xe4,0x03,0x56,0xe3,0x76,0x01,0xe2, + 0x8e,0x53,0xd1,0x0c,0xe0,0xef,0x52,0xcf,0x86,0x65,0x8d,0x52,0x04,0x00,0xe0,0x0d, + 0x01,0xcf,0x86,0xd5,0x0a,0xe4,0x10,0x53,0x63,0xff,0x52,0x0a,0x00,0xd4,0x80,0xd3, + 0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0x80,0x00,0x01,0xff,0xe2, + 0xb4,0x81,0x00,0x10,0x08,0x01,0xff,0xe2,0xb4,0x82,0x00,0x01,0xff,0xe2,0xb4,0x83, + 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0x84,0x00,0x01,0xff,0xe2,0xb4,0x85, + 0x00,0x10,0x08,0x01,0xff,0xe2,0xb4,0x86,0x00,0x01,0xff,0xe2,0xb4,0x87,0x00,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0x88,0x00,0x01,0xff,0xe2,0xb4,0x89, + 0x00,0x10,0x08,0x01,0xff,0xe2,0xb4,0x8a,0x00,0x01,0xff,0xe2,0xb4,0x8b,0x00,0xd1, + 0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0x8c,0x00,0x01,0xff,0xe2,0xb4,0x8d,0x00,0x10, + 0x08,0x01,0xff,0xe2,0xb4,0x8e,0x00,0x01,0xff,0xe2,0xb4,0x8f,0x00,0xd3,0x40,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0x90,0x00,0x01,0xff,0xe2,0xb4,0x91, + 0x00,0x10,0x08,0x01,0xff,0xe2,0xb4,0x92,0x00,0x01,0xff,0xe2,0xb4,0x93,0x00,0xd1, + 0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0x94,0x00,0x01,0xff,0xe2,0xb4,0x95,0x00,0x10, + 0x08,0x01,0xff,0xe2,0xb4,0x96,0x00,0x01,0xff,0xe2,0xb4,0x97,0x00,0xd2,0x20,0xd1, + 0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0x98,0x00,0x01,0xff,0xe2,0xb4,0x99,0x00,0x10, + 0x08,0x01,0xff,0xe2,0xb4,0x9a,0x00,0x01,0xff,0xe2,0xb4,0x9b,0x00,0xd1,0x10,0x10, + 0x08,0x01,0xff,0xe2,0xb4,0x9c,0x00,0x01,0xff,0xe2,0xb4,0x9d,0x00,0x10,0x08,0x01, + 0xff,0xe2,0xb4,0x9e,0x00,0x01,0xff,0xe2,0xb4,0x9f,0x00,0xcf,0x86,0xe5,0x42,0x52, + 0x94,0x50,0xd3,0x3c,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0xa0,0x00, + 0x01,0xff,0xe2,0xb4,0xa1,0x00,0x10,0x08,0x01,0xff,0xe2,0xb4,0xa2,0x00,0x01,0xff, + 0xe2,0xb4,0xa3,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0xb4,0xa4,0x00,0x01,0xff, + 0xe2,0xb4,0xa5,0x00,0x10,0x04,0x00,0x00,0x0d,0xff,0xe2,0xb4,0xa7,0x00,0x52,0x04, + 0x00,0x00,0x91,0x0c,0x10,0x04,0x00,0x00,0x0d,0xff,0xe2,0xb4,0xad,0x00,0x00,0x00, + 0x01,0x00,0xd2,0x1b,0xe1,0xfc,0x52,0xe0,0xad,0x52,0xcf,0x86,0x95,0x0f,0x94,0x0b, + 0x93,0x07,0x62,0x92,0x52,0x04,0x00,0x04,0x00,0x04,0x00,0x04,0x00,0xd1,0x13,0xe0, + 0xd3,0x53,0xcf,0x86,0x95,0x0a,0xe4,0xa8,0x53,0x63,0x97,0x53,0x04,0x00,0x04,0x00, + 0xd0,0x0d,0xcf,0x86,0x95,0x07,0x64,0x22,0x54,0x08,0x00,0x04,0x00,0xcf,0x86,0x55, + 0x04,0x04,0x00,0x54,0x04,0x04,0x00,0xd3,0x07,0x62,0x2f,0x54,0x04,0x00,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8f,0xb0,0x00,0x11,0xff,0xe1,0x8f,0xb1,0x00, + 0x10,0x08,0x11,0xff,0xe1,0x8f,0xb2,0x00,0x11,0xff,0xe1,0x8f,0xb3,0x00,0x91,0x10, + 0x10,0x08,0x11,0xff,0xe1,0x8f,0xb4,0x00,0x11,0xff,0xe1,0x8f,0xb5,0x00,0x00,0x00, + 0xd4,0x1c,0xe3,0xe0,0x56,0xe2,0x17,0x56,0xe1,0xda,0x55,0xe0,0xbb,0x55,0xcf,0x86, + 0x95,0x0a,0xe4,0xa4,0x55,0x63,0x88,0x55,0x04,0x00,0x04,0x00,0xe3,0xd2,0x01,0xe2, + 0x2b,0x5a,0xd1,0x0c,0xe0,0x4c,0x59,0xcf,0x86,0x65,0x25,0x59,0x0a,0x00,0xe0,0x9c, + 0x59,0xcf,0x86,0xd5,0xc5,0xd4,0x45,0xd3,0x31,0xd2,0x1c,0xd1,0x0e,0x10,0x07,0x12, + 0xff,0xd0,0xb2,0x00,0x12,0xff,0xd0,0xb4,0x00,0x10,0x07,0x12,0xff,0xd0,0xbe,0x00, + 0x12,0xff,0xd1,0x81,0x00,0x51,0x07,0x12,0xff,0xd1,0x82,0x00,0x10,0x07,0x12,0xff, + 0xd1,0x8a,0x00,0x12,0xff,0xd1,0xa3,0x00,0x92,0x10,0x91,0x0c,0x10,0x08,0x12,0xff, + 0xea,0x99,0x8b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x14,0xff,0xe1,0x83,0x90,0x00,0x14,0xff,0xe1,0x83,0x91,0x00,0x10,0x08, + 0x14,0xff,0xe1,0x83,0x92,0x00,0x14,0xff,0xe1,0x83,0x93,0x00,0xd1,0x10,0x10,0x08, + 0x14,0xff,0xe1,0x83,0x94,0x00,0x14,0xff,0xe1,0x83,0x95,0x00,0x10,0x08,0x14,0xff, + 0xe1,0x83,0x96,0x00,0x14,0xff,0xe1,0x83,0x97,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x14,0xff,0xe1,0x83,0x98,0x00,0x14,0xff,0xe1,0x83,0x99,0x00,0x10,0x08,0x14,0xff, + 0xe1,0x83,0x9a,0x00,0x14,0xff,0xe1,0x83,0x9b,0x00,0xd1,0x10,0x10,0x08,0x14,0xff, + 0xe1,0x83,0x9c,0x00,0x14,0xff,0xe1,0x83,0x9d,0x00,0x10,0x08,0x14,0xff,0xe1,0x83, + 0x9e,0x00,0x14,0xff,0xe1,0x83,0x9f,0x00,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x14,0xff,0xe1,0x83,0xa0,0x00,0x14,0xff,0xe1,0x83,0xa1,0x00,0x10,0x08, + 0x14,0xff,0xe1,0x83,0xa2,0x00,0x14,0xff,0xe1,0x83,0xa3,0x00,0xd1,0x10,0x10,0x08, + 0x14,0xff,0xe1,0x83,0xa4,0x00,0x14,0xff,0xe1,0x83,0xa5,0x00,0x10,0x08,0x14,0xff, + 0xe1,0x83,0xa6,0x00,0x14,0xff,0xe1,0x83,0xa7,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x14,0xff,0xe1,0x83,0xa8,0x00,0x14,0xff,0xe1,0x83,0xa9,0x00,0x10,0x08,0x14,0xff, + 0xe1,0x83,0xaa,0x00,0x14,0xff,0xe1,0x83,0xab,0x00,0xd1,0x10,0x10,0x08,0x14,0xff, + 0xe1,0x83,0xac,0x00,0x14,0xff,0xe1,0x83,0xad,0x00,0x10,0x08,0x14,0xff,0xe1,0x83, + 0xae,0x00,0x14,0xff,0xe1,0x83,0xaf,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x14,0xff,0xe1,0x83,0xb0,0x00,0x14,0xff,0xe1,0x83,0xb1,0x00,0x10,0x08,0x14,0xff, + 0xe1,0x83,0xb2,0x00,0x14,0xff,0xe1,0x83,0xb3,0x00,0xd1,0x10,0x10,0x08,0x14,0xff, + 0xe1,0x83,0xb4,0x00,0x14,0xff,0xe1,0x83,0xb5,0x00,0x10,0x08,0x14,0xff,0xe1,0x83, + 0xb6,0x00,0x14,0xff,0xe1,0x83,0xb7,0x00,0xd2,0x1c,0xd1,0x10,0x10,0x08,0x14,0xff, + 0xe1,0x83,0xb8,0x00,0x14,0xff,0xe1,0x83,0xb9,0x00,0x10,0x08,0x14,0xff,0xe1,0x83, + 0xba,0x00,0x00,0x00,0xd1,0x0c,0x10,0x04,0x00,0x00,0x14,0xff,0xe1,0x83,0xbd,0x00, + 0x10,0x08,0x14,0xff,0xe1,0x83,0xbe,0x00,0x14,0xff,0xe1,0x83,0xbf,0x00,0xe2,0x9d, + 0x08,0xe1,0x48,0x04,0xe0,0x1c,0x02,0xcf,0x86,0xe5,0x11,0x01,0xd4,0x84,0xd3,0x40, + 0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x61,0xcc,0xa5,0x00,0x01,0xff,0x61,0xcc, + 0xa5,0x00,0x10,0x08,0x01,0xff,0x62,0xcc,0x87,0x00,0x01,0xff,0x62,0xcc,0x87,0x00, + 0xd1,0x10,0x10,0x08,0x01,0xff,0x62,0xcc,0xa3,0x00,0x01,0xff,0x62,0xcc,0xa3,0x00, + 0x10,0x08,0x01,0xff,0x62,0xcc,0xb1,0x00,0x01,0xff,0x62,0xcc,0xb1,0x00,0xd2,0x24, + 0xd1,0x14,0x10,0x0a,0x01,0xff,0x63,0xcc,0xa7,0xcc,0x81,0x00,0x01,0xff,0x63,0xcc, + 0xa7,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x64,0xcc,0x87,0x00,0x01,0xff,0x64,0xcc, + 0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x64,0xcc,0xa3,0x00,0x01,0xff,0x64,0xcc, + 0xa3,0x00,0x10,0x08,0x01,0xff,0x64,0xcc,0xb1,0x00,0x01,0xff,0x64,0xcc,0xb1,0x00, + 0xd3,0x48,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x64,0xcc,0xa7,0x00,0x01,0xff, + 0x64,0xcc,0xa7,0x00,0x10,0x08,0x01,0xff,0x64,0xcc,0xad,0x00,0x01,0xff,0x64,0xcc, + 0xad,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x65,0xcc,0x84,0xcc,0x80,0x00,0x01,0xff, + 0x65,0xcc,0x84,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0x65,0xcc,0x84,0xcc,0x81,0x00, + 0x01,0xff,0x65,0xcc,0x84,0xcc,0x81,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff, + 0x65,0xcc,0xad,0x00,0x01,0xff,0x65,0xcc,0xad,0x00,0x10,0x08,0x01,0xff,0x65,0xcc, + 0xb0,0x00,0x01,0xff,0x65,0xcc,0xb0,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x65,0xcc, + 0xa7,0xcc,0x86,0x00,0x01,0xff,0x65,0xcc,0xa7,0xcc,0x86,0x00,0x10,0x08,0x01,0xff, + 0x66,0xcc,0x87,0x00,0x01,0xff,0x66,0xcc,0x87,0x00,0xd4,0x84,0xd3,0x40,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x01,0xff,0x67,0xcc,0x84,0x00,0x01,0xff,0x67,0xcc,0x84,0x00, + 0x10,0x08,0x01,0xff,0x68,0xcc,0x87,0x00,0x01,0xff,0x68,0xcc,0x87,0x00,0xd1,0x10, + 0x10,0x08,0x01,0xff,0x68,0xcc,0xa3,0x00,0x01,0xff,0x68,0xcc,0xa3,0x00,0x10,0x08, + 0x01,0xff,0x68,0xcc,0x88,0x00,0x01,0xff,0x68,0xcc,0x88,0x00,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x01,0xff,0x68,0xcc,0xa7,0x00,0x01,0xff,0x68,0xcc,0xa7,0x00,0x10,0x08, + 0x01,0xff,0x68,0xcc,0xae,0x00,0x01,0xff,0x68,0xcc,0xae,0x00,0xd1,0x10,0x10,0x08, + 0x01,0xff,0x69,0xcc,0xb0,0x00,0x01,0xff,0x69,0xcc,0xb0,0x00,0x10,0x0a,0x01,0xff, + 0x69,0xcc,0x88,0xcc,0x81,0x00,0x01,0xff,0x69,0xcc,0x88,0xcc,0x81,0x00,0xd3,0x40, + 0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x6b,0xcc,0x81,0x00,0x01,0xff,0x6b,0xcc, + 0x81,0x00,0x10,0x08,0x01,0xff,0x6b,0xcc,0xa3,0x00,0x01,0xff,0x6b,0xcc,0xa3,0x00, + 0xd1,0x10,0x10,0x08,0x01,0xff,0x6b,0xcc,0xb1,0x00,0x01,0xff,0x6b,0xcc,0xb1,0x00, + 0x10,0x08,0x01,0xff,0x6c,0xcc,0xa3,0x00,0x01,0xff,0x6c,0xcc,0xa3,0x00,0xd2,0x24, + 0xd1,0x14,0x10,0x0a,0x01,0xff,0x6c,0xcc,0xa3,0xcc,0x84,0x00,0x01,0xff,0x6c,0xcc, + 0xa3,0xcc,0x84,0x00,0x10,0x08,0x01,0xff,0x6c,0xcc,0xb1,0x00,0x01,0xff,0x6c,0xcc, + 0xb1,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x6c,0xcc,0xad,0x00,0x01,0xff,0x6c,0xcc, + 0xad,0x00,0x10,0x08,0x01,0xff,0x6d,0xcc,0x81,0x00,0x01,0xff,0x6d,0xcc,0x81,0x00, + 0xcf,0x86,0xe5,0x15,0x01,0xd4,0x88,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01, + 0xff,0x6d,0xcc,0x87,0x00,0x01,0xff,0x6d,0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x6d, + 0xcc,0xa3,0x00,0x01,0xff,0x6d,0xcc,0xa3,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x6e, + 0xcc,0x87,0x00,0x01,0xff,0x6e,0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x6e,0xcc,0xa3, + 0x00,0x01,0xff,0x6e,0xcc,0xa3,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x6e, + 0xcc,0xb1,0x00,0x01,0xff,0x6e,0xcc,0xb1,0x00,0x10,0x08,0x01,0xff,0x6e,0xcc,0xad, + 0x00,0x01,0xff,0x6e,0xcc,0xad,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x83, + 0xcc,0x81,0x00,0x01,0xff,0x6f,0xcc,0x83,0xcc,0x81,0x00,0x10,0x0a,0x01,0xff,0x6f, + 0xcc,0x83,0xcc,0x88,0x00,0x01,0xff,0x6f,0xcc,0x83,0xcc,0x88,0x00,0xd3,0x48,0xd2, + 0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x84,0xcc,0x80,0x00,0x01,0xff,0x6f, + 0xcc,0x84,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x84,0xcc,0x81,0x00,0x01, + 0xff,0x6f,0xcc,0x84,0xcc,0x81,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x70,0xcc,0x81, + 0x00,0x01,0xff,0x70,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x70,0xcc,0x87,0x00,0x01, + 0xff,0x70,0xcc,0x87,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x72,0xcc,0x87, + 0x00,0x01,0xff,0x72,0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x72,0xcc,0xa3,0x00,0x01, + 0xff,0x72,0xcc,0xa3,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x72,0xcc,0xa3,0xcc,0x84, + 0x00,0x01,0xff,0x72,0xcc,0xa3,0xcc,0x84,0x00,0x10,0x08,0x01,0xff,0x72,0xcc,0xb1, + 0x00,0x01,0xff,0x72,0xcc,0xb1,0x00,0xd4,0x8c,0xd3,0x48,0xd2,0x20,0xd1,0x10,0x10, + 0x08,0x01,0xff,0x73,0xcc,0x87,0x00,0x01,0xff,0x73,0xcc,0x87,0x00,0x10,0x08,0x01, + 0xff,0x73,0xcc,0xa3,0x00,0x01,0xff,0x73,0xcc,0xa3,0x00,0xd1,0x14,0x10,0x0a,0x01, + 0xff,0x73,0xcc,0x81,0xcc,0x87,0x00,0x01,0xff,0x73,0xcc,0x81,0xcc,0x87,0x00,0x10, + 0x0a,0x01,0xff,0x73,0xcc,0x8c,0xcc,0x87,0x00,0x01,0xff,0x73,0xcc,0x8c,0xcc,0x87, + 0x00,0xd2,0x24,0xd1,0x14,0x10,0x0a,0x01,0xff,0x73,0xcc,0xa3,0xcc,0x87,0x00,0x01, + 0xff,0x73,0xcc,0xa3,0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x74,0xcc,0x87,0x00,0x01, + 0xff,0x74,0xcc,0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x74,0xcc,0xa3,0x00,0x01, + 0xff,0x74,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x74,0xcc,0xb1,0x00,0x01,0xff,0x74, + 0xcc,0xb1,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x74,0xcc,0xad, + 0x00,0x01,0xff,0x74,0xcc,0xad,0x00,0x10,0x08,0x01,0xff,0x75,0xcc,0xa4,0x00,0x01, + 0xff,0x75,0xcc,0xa4,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x75,0xcc,0xb0,0x00,0x01, + 0xff,0x75,0xcc,0xb0,0x00,0x10,0x08,0x01,0xff,0x75,0xcc,0xad,0x00,0x01,0xff,0x75, + 0xcc,0xad,0x00,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x75,0xcc,0x83,0xcc,0x81, + 0x00,0x01,0xff,0x75,0xcc,0x83,0xcc,0x81,0x00,0x10,0x0a,0x01,0xff,0x75,0xcc,0x84, + 0xcc,0x88,0x00,0x01,0xff,0x75,0xcc,0x84,0xcc,0x88,0x00,0xd1,0x10,0x10,0x08,0x01, + 0xff,0x76,0xcc,0x83,0x00,0x01,0xff,0x76,0xcc,0x83,0x00,0x10,0x08,0x01,0xff,0x76, + 0xcc,0xa3,0x00,0x01,0xff,0x76,0xcc,0xa3,0x00,0xe0,0x11,0x02,0xcf,0x86,0xd5,0xe2, + 0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x77,0xcc,0x80,0x00, + 0x01,0xff,0x77,0xcc,0x80,0x00,0x10,0x08,0x01,0xff,0x77,0xcc,0x81,0x00,0x01,0xff, + 0x77,0xcc,0x81,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x77,0xcc,0x88,0x00,0x01,0xff, + 0x77,0xcc,0x88,0x00,0x10,0x08,0x01,0xff,0x77,0xcc,0x87,0x00,0x01,0xff,0x77,0xcc, + 0x87,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x77,0xcc,0xa3,0x00,0x01,0xff, + 0x77,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x78,0xcc,0x87,0x00,0x01,0xff,0x78,0xcc, + 0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x78,0xcc,0x88,0x00,0x01,0xff,0x78,0xcc, + 0x88,0x00,0x10,0x08,0x01,0xff,0x79,0xcc,0x87,0x00,0x01,0xff,0x79,0xcc,0x87,0x00, + 0xd3,0x33,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x7a,0xcc,0x82,0x00,0x01,0xff, + 0x7a,0xcc,0x82,0x00,0x10,0x08,0x01,0xff,0x7a,0xcc,0xa3,0x00,0x01,0xff,0x7a,0xcc, + 0xa3,0x00,0xe1,0x12,0x59,0x10,0x08,0x01,0xff,0x7a,0xcc,0xb1,0x00,0x01,0xff,0x7a, + 0xcc,0xb1,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x77,0xcc,0x8a,0x00,0x01, + 0xff,0x79,0xcc,0x8a,0x00,0x10,0x08,0x01,0xff,0x61,0xca,0xbe,0x00,0x02,0xff,0x73, + 0xcc,0x87,0x00,0x51,0x04,0x0a,0x00,0x10,0x07,0x0a,0xff,0x73,0x73,0x00,0x0a,0x00, + 0xd4,0x98,0xd3,0x48,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x61,0xcc,0xa3,0x00, + 0x01,0xff,0x61,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x61,0xcc,0x89,0x00,0x01,0xff, + 0x61,0xcc,0x89,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x61,0xcc,0x82,0xcc,0x81,0x00, + 0x01,0xff,0x61,0xcc,0x82,0xcc,0x81,0x00,0x10,0x0a,0x01,0xff,0x61,0xcc,0x82,0xcc, + 0x80,0x00,0x01,0xff,0x61,0xcc,0x82,0xcc,0x80,0x00,0xd2,0x28,0xd1,0x14,0x10,0x0a, + 0x01,0xff,0x61,0xcc,0x82,0xcc,0x89,0x00,0x01,0xff,0x61,0xcc,0x82,0xcc,0x89,0x00, + 0x10,0x0a,0x01,0xff,0x61,0xcc,0x82,0xcc,0x83,0x00,0x01,0xff,0x61,0xcc,0x82,0xcc, + 0x83,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x61,0xcc,0xa3,0xcc,0x82,0x00,0x01,0xff, + 0x61,0xcc,0xa3,0xcc,0x82,0x00,0x10,0x0a,0x01,0xff,0x61,0xcc,0x86,0xcc,0x81,0x00, + 0x01,0xff,0x61,0xcc,0x86,0xcc,0x81,0x00,0xd3,0x50,0xd2,0x28,0xd1,0x14,0x10,0x0a, + 0x01,0xff,0x61,0xcc,0x86,0xcc,0x80,0x00,0x01,0xff,0x61,0xcc,0x86,0xcc,0x80,0x00, + 0x10,0x0a,0x01,0xff,0x61,0xcc,0x86,0xcc,0x89,0x00,0x01,0xff,0x61,0xcc,0x86,0xcc, + 0x89,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x61,0xcc,0x86,0xcc,0x83,0x00,0x01,0xff, + 0x61,0xcc,0x86,0xcc,0x83,0x00,0x10,0x0a,0x01,0xff,0x61,0xcc,0xa3,0xcc,0x86,0x00, + 0x01,0xff,0x61,0xcc,0xa3,0xcc,0x86,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff, + 0x65,0xcc,0xa3,0x00,0x01,0xff,0x65,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x65,0xcc, + 0x89,0x00,0x01,0xff,0x65,0xcc,0x89,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x65,0xcc, + 0x83,0x00,0x01,0xff,0x65,0xcc,0x83,0x00,0x10,0x0a,0x01,0xff,0x65,0xcc,0x82,0xcc, + 0x81,0x00,0x01,0xff,0x65,0xcc,0x82,0xcc,0x81,0x00,0xcf,0x86,0xe5,0x31,0x01,0xd4, + 0x90,0xd3,0x50,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x65,0xcc,0x82,0xcc,0x80, + 0x00,0x01,0xff,0x65,0xcc,0x82,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0x65,0xcc,0x82, + 0xcc,0x89,0x00,0x01,0xff,0x65,0xcc,0x82,0xcc,0x89,0x00,0xd1,0x14,0x10,0x0a,0x01, + 0xff,0x65,0xcc,0x82,0xcc,0x83,0x00,0x01,0xff,0x65,0xcc,0x82,0xcc,0x83,0x00,0x10, + 0x0a,0x01,0xff,0x65,0xcc,0xa3,0xcc,0x82,0x00,0x01,0xff,0x65,0xcc,0xa3,0xcc,0x82, + 0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x69,0xcc,0x89,0x00,0x01,0xff,0x69, + 0xcc,0x89,0x00,0x10,0x08,0x01,0xff,0x69,0xcc,0xa3,0x00,0x01,0xff,0x69,0xcc,0xa3, + 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x6f,0xcc,0xa3,0x00,0x01,0xff,0x6f,0xcc,0xa3, + 0x00,0x10,0x08,0x01,0xff,0x6f,0xcc,0x89,0x00,0x01,0xff,0x6f,0xcc,0x89,0x00,0xd3, + 0x50,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x82,0xcc,0x81,0x00,0x01, + 0xff,0x6f,0xcc,0x82,0xcc,0x81,0x00,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x82,0xcc,0x80, + 0x00,0x01,0xff,0x6f,0xcc,0x82,0xcc,0x80,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x6f, + 0xcc,0x82,0xcc,0x89,0x00,0x01,0xff,0x6f,0xcc,0x82,0xcc,0x89,0x00,0x10,0x0a,0x01, + 0xff,0x6f,0xcc,0x82,0xcc,0x83,0x00,0x01,0xff,0x6f,0xcc,0x82,0xcc,0x83,0x00,0xd2, + 0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x6f,0xcc,0xa3,0xcc,0x82,0x00,0x01,0xff,0x6f, + 0xcc,0xa3,0xcc,0x82,0x00,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0x81,0x00,0x01, + 0xff,0x6f,0xcc,0x9b,0xcc,0x81,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x9b, + 0xcc,0x80,0x00,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0x6f, + 0xcc,0x9b,0xcc,0x89,0x00,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0x89,0x00,0xd4,0x98,0xd3, + 0x48,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0x83,0x00,0x01, + 0xff,0x6f,0xcc,0x9b,0xcc,0x83,0x00,0x10,0x0a,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0xa3, + 0x00,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0xa3,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x75, + 0xcc,0xa3,0x00,0x01,0xff,0x75,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x75,0xcc,0x89, + 0x00,0x01,0xff,0x75,0xcc,0x89,0x00,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x75, + 0xcc,0x9b,0xcc,0x81,0x00,0x01,0xff,0x75,0xcc,0x9b,0xcc,0x81,0x00,0x10,0x0a,0x01, + 0xff,0x75,0xcc,0x9b,0xcc,0x80,0x00,0x01,0xff,0x75,0xcc,0x9b,0xcc,0x80,0x00,0xd1, + 0x14,0x10,0x0a,0x01,0xff,0x75,0xcc,0x9b,0xcc,0x89,0x00,0x01,0xff,0x75,0xcc,0x9b, + 0xcc,0x89,0x00,0x10,0x0a,0x01,0xff,0x75,0xcc,0x9b,0xcc,0x83,0x00,0x01,0xff,0x75, + 0xcc,0x9b,0xcc,0x83,0x00,0xd3,0x44,0xd2,0x24,0xd1,0x14,0x10,0x0a,0x01,0xff,0x75, + 0xcc,0x9b,0xcc,0xa3,0x00,0x01,0xff,0x75,0xcc,0x9b,0xcc,0xa3,0x00,0x10,0x08,0x01, + 0xff,0x79,0xcc,0x80,0x00,0x01,0xff,0x79,0xcc,0x80,0x00,0xd1,0x10,0x10,0x08,0x01, + 0xff,0x79,0xcc,0xa3,0x00,0x01,0xff,0x79,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x79, + 0xcc,0x89,0x00,0x01,0xff,0x79,0xcc,0x89,0x00,0xd2,0x1c,0xd1,0x10,0x10,0x08,0x01, + 0xff,0x79,0xcc,0x83,0x00,0x01,0xff,0x79,0xcc,0x83,0x00,0x10,0x08,0x0a,0xff,0xe1, + 0xbb,0xbb,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xe1,0xbb,0xbd,0x00,0x0a, + 0x00,0x10,0x08,0x0a,0xff,0xe1,0xbb,0xbf,0x00,0x0a,0x00,0xe1,0xbf,0x02,0xe0,0xa1, + 0x01,0xcf,0x86,0xd5,0xc6,0xd4,0x6c,0xd3,0x18,0xe2,0x0e,0x59,0xe1,0xf7,0x58,0x10, + 0x09,0x01,0xff,0xce,0xb1,0xcc,0x93,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0x00,0xd2, + 0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb1,0xcc,0x93,0x00,0x01,0xff,0xce,0xb1, + 0xcc,0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0xb1,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff, + 0xce,0xb1,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xb1,0xcc, + 0x93,0xcc,0x81,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01, + 0xff,0xce,0xb1,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xcd,0x82, + 0x00,0xd3,0x18,0xe2,0x4a,0x59,0xe1,0x33,0x59,0x10,0x09,0x01,0xff,0xce,0xb5,0xcc, + 0x93,0x00,0x01,0xff,0xce,0xb5,0xcc,0x94,0x00,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01, + 0xff,0xce,0xb5,0xcc,0x93,0x00,0x01,0xff,0xce,0xb5,0xcc,0x94,0x00,0x10,0x0b,0x01, + 0xff,0xce,0xb5,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0xb5,0xcc,0x94,0xcc,0x80, + 0x00,0x91,0x16,0x10,0x0b,0x01,0xff,0xce,0xb5,0xcc,0x93,0xcc,0x81,0x00,0x01,0xff, + 0xce,0xb5,0xcc,0x94,0xcc,0x81,0x00,0x00,0x00,0xd4,0x6c,0xd3,0x18,0xe2,0x74,0x59, + 0xe1,0x5d,0x59,0x10,0x09,0x01,0xff,0xce,0xb7,0xcc,0x93,0x00,0x01,0xff,0xce,0xb7, + 0xcc,0x94,0x00,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb7,0xcc,0x93,0x00, + 0x01,0xff,0xce,0xb7,0xcc,0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcc, + 0x80,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01, + 0xff,0xce,0xb7,0xcc,0x93,0xcc,0x81,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcc,0x81, + 0x00,0x10,0x0b,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0xb7, + 0xcc,0x94,0xcd,0x82,0x00,0xd3,0x18,0xe2,0xb0,0x59,0xe1,0x99,0x59,0x10,0x09,0x01, + 0xff,0xce,0xb9,0xcc,0x93,0x00,0x01,0xff,0xce,0xb9,0xcc,0x94,0x00,0xd2,0x28,0xd1, + 0x12,0x10,0x09,0x01,0xff,0xce,0xb9,0xcc,0x93,0x00,0x01,0xff,0xce,0xb9,0xcc,0x94, + 0x00,0x10,0x0b,0x01,0xff,0xce,0xb9,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0xb9, + 0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xb9,0xcc,0x93,0xcc, + 0x81,0x00,0x01,0xff,0xce,0xb9,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01,0xff,0xce, + 0xb9,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0xb9,0xcc,0x94,0xcd,0x82,0x00,0xcf, + 0x86,0xd5,0xac,0xd4,0x5a,0xd3,0x18,0xe2,0xed,0x59,0xe1,0xd6,0x59,0x10,0x09,0x01, + 0xff,0xce,0xbf,0xcc,0x93,0x00,0x01,0xff,0xce,0xbf,0xcc,0x94,0x00,0xd2,0x28,0xd1, + 0x12,0x10,0x09,0x01,0xff,0xce,0xbf,0xcc,0x93,0x00,0x01,0xff,0xce,0xbf,0xcc,0x94, + 0x00,0x10,0x0b,0x01,0xff,0xce,0xbf,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0xbf, + 0xcc,0x94,0xcc,0x80,0x00,0x91,0x16,0x10,0x0b,0x01,0xff,0xce,0xbf,0xcc,0x93,0xcc, + 0x81,0x00,0x01,0xff,0xce,0xbf,0xcc,0x94,0xcc,0x81,0x00,0x00,0x00,0xd3,0x18,0xe2, + 0x17,0x5a,0xe1,0x00,0x5a,0x10,0x09,0x01,0xff,0xcf,0x85,0xcc,0x93,0x00,0x01,0xff, + 0xcf,0x85,0xcc,0x94,0x00,0xd2,0x1c,0xd1,0x0d,0x10,0x04,0x00,0x00,0x01,0xff,0xcf, + 0x85,0xcc,0x94,0x00,0x10,0x04,0x00,0x00,0x01,0xff,0xcf,0x85,0xcc,0x94,0xcc,0x80, + 0x00,0xd1,0x0f,0x10,0x04,0x00,0x00,0x01,0xff,0xcf,0x85,0xcc,0x94,0xcc,0x81,0x00, + 0x10,0x04,0x00,0x00,0x01,0xff,0xcf,0x85,0xcc,0x94,0xcd,0x82,0x00,0xe4,0xd3,0x5a, + 0xd3,0x18,0xe2,0x52,0x5a,0xe1,0x3b,0x5a,0x10,0x09,0x01,0xff,0xcf,0x89,0xcc,0x93, + 0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0x00,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01,0xff, + 0xcf,0x89,0xcc,0x93,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0x00,0x10,0x0b,0x01,0xff, + 0xcf,0x89,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcc,0x80,0x00, + 0xd1,0x16,0x10,0x0b,0x01,0xff,0xcf,0x89,0xcc,0x93,0xcc,0x81,0x00,0x01,0xff,0xcf, + 0x89,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01,0xff,0xcf,0x89,0xcc,0x93,0xcd,0x82, + 0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcd,0x82,0x00,0xe0,0xd9,0x02,0xcf,0x86,0xe5, + 0x91,0x01,0xd4,0xc8,0xd3,0x64,0xd2,0x30,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xb1, + 0xcc,0x93,0xce,0xb9,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xce,0xb9,0x00,0x10,0x0d, + 0x01,0xff,0xce,0xb1,0xcc,0x93,0xcc,0x80,0xce,0xb9,0x00,0x01,0xff,0xce,0xb1,0xcc, + 0x94,0xcc,0x80,0xce,0xb9,0x00,0xd1,0x1a,0x10,0x0d,0x01,0xff,0xce,0xb1,0xcc,0x93, + 0xcc,0x81,0xce,0xb9,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xcc,0x81,0xce,0xb9,0x00, + 0x10,0x0d,0x01,0xff,0xce,0xb1,0xcc,0x93,0xcd,0x82,0xce,0xb9,0x00,0x01,0xff,0xce, + 0xb1,0xcc,0x94,0xcd,0x82,0xce,0xb9,0x00,0xd2,0x30,0xd1,0x16,0x10,0x0b,0x01,0xff, + 0xce,0xb1,0xcc,0x93,0xce,0xb9,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xce,0xb9,0x00, + 0x10,0x0d,0x01,0xff,0xce,0xb1,0xcc,0x93,0xcc,0x80,0xce,0xb9,0x00,0x01,0xff,0xce, + 0xb1,0xcc,0x94,0xcc,0x80,0xce,0xb9,0x00,0xd1,0x1a,0x10,0x0d,0x01,0xff,0xce,0xb1, + 0xcc,0x93,0xcc,0x81,0xce,0xb9,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xcc,0x81,0xce, + 0xb9,0x00,0x10,0x0d,0x01,0xff,0xce,0xb1,0xcc,0x93,0xcd,0x82,0xce,0xb9,0x00,0x01, + 0xff,0xce,0xb1,0xcc,0x94,0xcd,0x82,0xce,0xb9,0x00,0xd3,0x64,0xd2,0x30,0xd1,0x16, + 0x10,0x0b,0x01,0xff,0xce,0xb7,0xcc,0x93,0xce,0xb9,0x00,0x01,0xff,0xce,0xb7,0xcc, + 0x94,0xce,0xb9,0x00,0x10,0x0d,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcc,0x80,0xce,0xb9, + 0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcc,0x80,0xce,0xb9,0x00,0xd1,0x1a,0x10,0x0d, + 0x01,0xff,0xce,0xb7,0xcc,0x93,0xcc,0x81,0xce,0xb9,0x00,0x01,0xff,0xce,0xb7,0xcc, + 0x94,0xcc,0x81,0xce,0xb9,0x00,0x10,0x0d,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcd,0x82, + 0xce,0xb9,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcd,0x82,0xce,0xb9,0x00,0xd2,0x30, + 0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xb7,0xcc,0x93,0xce,0xb9,0x00,0x01,0xff,0xce, + 0xb7,0xcc,0x94,0xce,0xb9,0x00,0x10,0x0d,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcc,0x80, + 0xce,0xb9,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcc,0x80,0xce,0xb9,0x00,0xd1,0x1a, + 0x10,0x0d,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcc,0x81,0xce,0xb9,0x00,0x01,0xff,0xce, + 0xb7,0xcc,0x94,0xcc,0x81,0xce,0xb9,0x00,0x10,0x0d,0x01,0xff,0xce,0xb7,0xcc,0x93, + 0xcd,0x82,0xce,0xb9,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcd,0x82,0xce,0xb9,0x00, + 0xd4,0xc8,0xd3,0x64,0xd2,0x30,0xd1,0x16,0x10,0x0b,0x01,0xff,0xcf,0x89,0xcc,0x93, + 0xce,0xb9,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xce,0xb9,0x00,0x10,0x0d,0x01,0xff, + 0xcf,0x89,0xcc,0x93,0xcc,0x80,0xce,0xb9,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcc, + 0x80,0xce,0xb9,0x00,0xd1,0x1a,0x10,0x0d,0x01,0xff,0xcf,0x89,0xcc,0x93,0xcc,0x81, + 0xce,0xb9,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcc,0x81,0xce,0xb9,0x00,0x10,0x0d, + 0x01,0xff,0xcf,0x89,0xcc,0x93,0xcd,0x82,0xce,0xb9,0x00,0x01,0xff,0xcf,0x89,0xcc, + 0x94,0xcd,0x82,0xce,0xb9,0x00,0xd2,0x30,0xd1,0x16,0x10,0x0b,0x01,0xff,0xcf,0x89, + 0xcc,0x93,0xce,0xb9,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xce,0xb9,0x00,0x10,0x0d, + 0x01,0xff,0xcf,0x89,0xcc,0x93,0xcc,0x80,0xce,0xb9,0x00,0x01,0xff,0xcf,0x89,0xcc, + 0x94,0xcc,0x80,0xce,0xb9,0x00,0xd1,0x1a,0x10,0x0d,0x01,0xff,0xcf,0x89,0xcc,0x93, + 0xcc,0x81,0xce,0xb9,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcc,0x81,0xce,0xb9,0x00, + 0x10,0x0d,0x01,0xff,0xcf,0x89,0xcc,0x93,0xcd,0x82,0xce,0xb9,0x00,0x01,0xff,0xcf, + 0x89,0xcc,0x94,0xcd,0x82,0xce,0xb9,0x00,0xd3,0x49,0xd2,0x26,0xd1,0x12,0x10,0x09, + 0x01,0xff,0xce,0xb1,0xcc,0x86,0x00,0x01,0xff,0xce,0xb1,0xcc,0x84,0x00,0x10,0x0b, + 0x01,0xff,0xce,0xb1,0xcc,0x80,0xce,0xb9,0x00,0x01,0xff,0xce,0xb1,0xce,0xb9,0x00, + 0xd1,0x0f,0x10,0x0b,0x01,0xff,0xce,0xb1,0xcc,0x81,0xce,0xb9,0x00,0x00,0x00,0x10, + 0x09,0x01,0xff,0xce,0xb1,0xcd,0x82,0x00,0x01,0xff,0xce,0xb1,0xcd,0x82,0xce,0xb9, + 0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb1,0xcc,0x86,0x00,0x01,0xff, + 0xce,0xb1,0xcc,0x84,0x00,0x10,0x09,0x01,0xff,0xce,0xb1,0xcc,0x80,0x00,0x01,0xff, + 0xce,0xb1,0xcc,0x81,0x00,0xe1,0xf3,0x5a,0x10,0x09,0x01,0xff,0xce,0xb1,0xce,0xb9, + 0x00,0x01,0x00,0xcf,0x86,0xd5,0xbd,0xd4,0x7e,0xd3,0x44,0xd2,0x21,0xd1,0x0d,0x10, + 0x04,0x01,0x00,0x01,0xff,0xc2,0xa8,0xcd,0x82,0x00,0x10,0x0b,0x01,0xff,0xce,0xb7, + 0xcc,0x80,0xce,0xb9,0x00,0x01,0xff,0xce,0xb7,0xce,0xb9,0x00,0xd1,0x0f,0x10,0x0b, + 0x01,0xff,0xce,0xb7,0xcc,0x81,0xce,0xb9,0x00,0x00,0x00,0x10,0x09,0x01,0xff,0xce, + 0xb7,0xcd,0x82,0x00,0x01,0xff,0xce,0xb7,0xcd,0x82,0xce,0xb9,0x00,0xd2,0x24,0xd1, + 0x12,0x10,0x09,0x01,0xff,0xce,0xb5,0xcc,0x80,0x00,0x01,0xff,0xce,0xb5,0xcc,0x81, + 0x00,0x10,0x09,0x01,0xff,0xce,0xb7,0xcc,0x80,0x00,0x01,0xff,0xce,0xb7,0xcc,0x81, + 0x00,0xe1,0x02,0x5b,0x10,0x09,0x01,0xff,0xce,0xb7,0xce,0xb9,0x00,0x01,0xff,0xe1, + 0xbe,0xbf,0xcc,0x80,0x00,0xd3,0x18,0xe2,0x28,0x5b,0xe1,0x11,0x5b,0x10,0x09,0x01, + 0xff,0xce,0xb9,0xcc,0x86,0x00,0x01,0xff,0xce,0xb9,0xcc,0x84,0x00,0xe2,0x4c,0x5b, + 0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb9,0xcc,0x86,0x00,0x01,0xff,0xce,0xb9,0xcc, + 0x84,0x00,0x10,0x09,0x01,0xff,0xce,0xb9,0xcc,0x80,0x00,0x01,0xff,0xce,0xb9,0xcc, + 0x81,0x00,0xd4,0x51,0xd3,0x18,0xe2,0x6f,0x5b,0xe1,0x58,0x5b,0x10,0x09,0x01,0xff, + 0xcf,0x85,0xcc,0x86,0x00,0x01,0xff,0xcf,0x85,0xcc,0x84,0x00,0xd2,0x24,0xd1,0x12, + 0x10,0x09,0x01,0xff,0xcf,0x85,0xcc,0x86,0x00,0x01,0xff,0xcf,0x85,0xcc,0x84,0x00, + 0x10,0x09,0x01,0xff,0xcf,0x85,0xcc,0x80,0x00,0x01,0xff,0xcf,0x85,0xcc,0x81,0x00, + 0xe1,0x8f,0x5b,0x10,0x09,0x01,0xff,0xcf,0x81,0xcc,0x94,0x00,0x01,0xff,0xc2,0xa8, + 0xcc,0x80,0x00,0xd3,0x3b,0xd2,0x18,0x51,0x04,0x00,0x00,0x10,0x0b,0x01,0xff,0xcf, + 0x89,0xcc,0x80,0xce,0xb9,0x00,0x01,0xff,0xcf,0x89,0xce,0xb9,0x00,0xd1,0x0f,0x10, + 0x0b,0x01,0xff,0xcf,0x89,0xcc,0x81,0xce,0xb9,0x00,0x00,0x00,0x10,0x09,0x01,0xff, + 0xcf,0x89,0xcd,0x82,0x00,0x01,0xff,0xcf,0x89,0xcd,0x82,0xce,0xb9,0x00,0xd2,0x24, + 0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xbf,0xcc,0x80,0x00,0x01,0xff,0xce,0xbf,0xcc, + 0x81,0x00,0x10,0x09,0x01,0xff,0xcf,0x89,0xcc,0x80,0x00,0x01,0xff,0xcf,0x89,0xcc, + 0x81,0x00,0xe1,0x99,0x5b,0x10,0x09,0x01,0xff,0xcf,0x89,0xce,0xb9,0x00,0x01,0xff, + 0xc2,0xb4,0x00,0xe0,0x0c,0x68,0xcf,0x86,0xe5,0x23,0x02,0xe4,0x25,0x01,0xe3,0x85, + 0x5e,0xd2,0x2a,0xe1,0x5f,0x5c,0xe0,0xdd,0x5b,0xcf,0x86,0xe5,0xbb,0x5b,0x94,0x1b, + 0xe3,0xa4,0x5b,0x92,0x14,0x91,0x10,0x10,0x08,0x01,0xff,0xe2,0x80,0x82,0x00,0x01, + 0xff,0xe2,0x80,0x83,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd1,0xd6,0xd0,0x46,0xcf, + 0x86,0x55,0x04,0x01,0x00,0xd4,0x29,0xd3,0x13,0x52,0x04,0x01,0x00,0x51,0x04,0x01, + 0x00,0x10,0x07,0x01,0xff,0xcf,0x89,0x00,0x01,0x00,0x92,0x12,0x51,0x04,0x01,0x00, + 0x10,0x06,0x01,0xff,0x6b,0x00,0x01,0xff,0x61,0xcc,0x8a,0x00,0x01,0x00,0xe3,0x25, + 0x5d,0x92,0x10,0x51,0x04,0x01,0x00,0x10,0x08,0x01,0xff,0xe2,0x85,0x8e,0x00,0x01, + 0x00,0x01,0x00,0xcf,0x86,0xd5,0x0a,0xe4,0x42,0x5d,0x63,0x2d,0x5d,0x06,0x00,0x94, + 0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0x85,0xb0,0x00,0x01, + 0xff,0xe2,0x85,0xb1,0x00,0x10,0x08,0x01,0xff,0xe2,0x85,0xb2,0x00,0x01,0xff,0xe2, + 0x85,0xb3,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0x85,0xb4,0x00,0x01,0xff,0xe2, + 0x85,0xb5,0x00,0x10,0x08,0x01,0xff,0xe2,0x85,0xb6,0x00,0x01,0xff,0xe2,0x85,0xb7, + 0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0x85,0xb8,0x00,0x01,0xff,0xe2, + 0x85,0xb9,0x00,0x10,0x08,0x01,0xff,0xe2,0x85,0xba,0x00,0x01,0xff,0xe2,0x85,0xbb, + 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0x85,0xbc,0x00,0x01,0xff,0xe2,0x85,0xbd, + 0x00,0x10,0x08,0x01,0xff,0xe2,0x85,0xbe,0x00,0x01,0xff,0xe2,0x85,0xbf,0x00,0x01, + 0x00,0xe0,0x34,0x5d,0xcf,0x86,0xe5,0x13,0x5d,0xe4,0xf2,0x5c,0xe3,0xe1,0x5c,0xe2, + 0xd4,0x5c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x04,0xff,0xe2,0x86,0x84,0x00, + 0xe3,0x23,0x61,0xe2,0xf0,0x60,0xd1,0x0c,0xe0,0x9d,0x60,0xcf,0x86,0x65,0x7e,0x60, + 0x01,0x00,0xd0,0x62,0xcf,0x86,0x55,0x04,0x01,0x00,0x54,0x04,0x01,0x00,0xd3,0x18, + 0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x08,0x01,0xff,0xe2,0x93,0x90,0x00, + 0x01,0xff,0xe2,0x93,0x91,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0x93, + 0x92,0x00,0x01,0xff,0xe2,0x93,0x93,0x00,0x10,0x08,0x01,0xff,0xe2,0x93,0x94,0x00, + 0x01,0xff,0xe2,0x93,0x95,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe2,0x93,0x96,0x00, + 0x01,0xff,0xe2,0x93,0x97,0x00,0x10,0x08,0x01,0xff,0xe2,0x93,0x98,0x00,0x01,0xff, + 0xe2,0x93,0x99,0x00,0xcf,0x86,0xe5,0x57,0x60,0x94,0x80,0xd3,0x40,0xd2,0x20,0xd1, + 0x10,0x10,0x08,0x01,0xff,0xe2,0x93,0x9a,0x00,0x01,0xff,0xe2,0x93,0x9b,0x00,0x10, + 0x08,0x01,0xff,0xe2,0x93,0x9c,0x00,0x01,0xff,0xe2,0x93,0x9d,0x00,0xd1,0x10,0x10, + 0x08,0x01,0xff,0xe2,0x93,0x9e,0x00,0x01,0xff,0xe2,0x93,0x9f,0x00,0x10,0x08,0x01, + 0xff,0xe2,0x93,0xa0,0x00,0x01,0xff,0xe2,0x93,0xa1,0x00,0xd2,0x20,0xd1,0x10,0x10, + 0x08,0x01,0xff,0xe2,0x93,0xa2,0x00,0x01,0xff,0xe2,0x93,0xa3,0x00,0x10,0x08,0x01, + 0xff,0xe2,0x93,0xa4,0x00,0x01,0xff,0xe2,0x93,0xa5,0x00,0xd1,0x10,0x10,0x08,0x01, + 0xff,0xe2,0x93,0xa6,0x00,0x01,0xff,0xe2,0x93,0xa7,0x00,0x10,0x08,0x01,0xff,0xe2, + 0x93,0xa8,0x00,0x01,0xff,0xe2,0x93,0xa9,0x00,0x01,0x00,0xd4,0x0c,0xe3,0x33,0x62, + 0xe2,0x2c,0x62,0xcf,0x06,0x04,0x00,0xe3,0x0c,0x65,0xe2,0xff,0x63,0xe1,0x2e,0x02, + 0xe0,0x84,0x01,0xcf,0x86,0xe5,0x01,0x01,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x08,0xff,0xe2,0xb0,0xb0,0x00,0x08,0xff,0xe2,0xb0,0xb1,0x00,0x10,0x08, + 0x08,0xff,0xe2,0xb0,0xb2,0x00,0x08,0xff,0xe2,0xb0,0xb3,0x00,0xd1,0x10,0x10,0x08, + 0x08,0xff,0xe2,0xb0,0xb4,0x00,0x08,0xff,0xe2,0xb0,0xb5,0x00,0x10,0x08,0x08,0xff, + 0xe2,0xb0,0xb6,0x00,0x08,0xff,0xe2,0xb0,0xb7,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x08,0xff,0xe2,0xb0,0xb8,0x00,0x08,0xff,0xe2,0xb0,0xb9,0x00,0x10,0x08,0x08,0xff, + 0xe2,0xb0,0xba,0x00,0x08,0xff,0xe2,0xb0,0xbb,0x00,0xd1,0x10,0x10,0x08,0x08,0xff, + 0xe2,0xb0,0xbc,0x00,0x08,0xff,0xe2,0xb0,0xbd,0x00,0x10,0x08,0x08,0xff,0xe2,0xb0, + 0xbe,0x00,0x08,0xff,0xe2,0xb0,0xbf,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x08,0xff,0xe2,0xb1,0x80,0x00,0x08,0xff,0xe2,0xb1,0x81,0x00,0x10,0x08,0x08,0xff, + 0xe2,0xb1,0x82,0x00,0x08,0xff,0xe2,0xb1,0x83,0x00,0xd1,0x10,0x10,0x08,0x08,0xff, + 0xe2,0xb1,0x84,0x00,0x08,0xff,0xe2,0xb1,0x85,0x00,0x10,0x08,0x08,0xff,0xe2,0xb1, + 0x86,0x00,0x08,0xff,0xe2,0xb1,0x87,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x08,0xff, + 0xe2,0xb1,0x88,0x00,0x08,0xff,0xe2,0xb1,0x89,0x00,0x10,0x08,0x08,0xff,0xe2,0xb1, + 0x8a,0x00,0x08,0xff,0xe2,0xb1,0x8b,0x00,0xd1,0x10,0x10,0x08,0x08,0xff,0xe2,0xb1, + 0x8c,0x00,0x08,0xff,0xe2,0xb1,0x8d,0x00,0x10,0x08,0x08,0xff,0xe2,0xb1,0x8e,0x00, + 0x08,0xff,0xe2,0xb1,0x8f,0x00,0x94,0x7c,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x08,0xff,0xe2,0xb1,0x90,0x00,0x08,0xff,0xe2,0xb1,0x91,0x00,0x10,0x08,0x08,0xff, + 0xe2,0xb1,0x92,0x00,0x08,0xff,0xe2,0xb1,0x93,0x00,0xd1,0x10,0x10,0x08,0x08,0xff, + 0xe2,0xb1,0x94,0x00,0x08,0xff,0xe2,0xb1,0x95,0x00,0x10,0x08,0x08,0xff,0xe2,0xb1, + 0x96,0x00,0x08,0xff,0xe2,0xb1,0x97,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x08,0xff, + 0xe2,0xb1,0x98,0x00,0x08,0xff,0xe2,0xb1,0x99,0x00,0x10,0x08,0x08,0xff,0xe2,0xb1, + 0x9a,0x00,0x08,0xff,0xe2,0xb1,0x9b,0x00,0xd1,0x10,0x10,0x08,0x08,0xff,0xe2,0xb1, + 0x9c,0x00,0x08,0xff,0xe2,0xb1,0x9d,0x00,0x10,0x08,0x08,0xff,0xe2,0xb1,0x9e,0x00, + 0x00,0x00,0x08,0x00,0xcf,0x86,0xd5,0x07,0x64,0xef,0x61,0x08,0x00,0xd4,0x63,0xd3, + 0x32,0xd2,0x1b,0xd1,0x0c,0x10,0x08,0x09,0xff,0xe2,0xb1,0xa1,0x00,0x09,0x00,0x10, + 0x07,0x09,0xff,0xc9,0xab,0x00,0x09,0xff,0xe1,0xb5,0xbd,0x00,0xd1,0x0b,0x10,0x07, + 0x09,0xff,0xc9,0xbd,0x00,0x09,0x00,0x10,0x04,0x09,0x00,0x09,0xff,0xe2,0xb1,0xa8, + 0x00,0xd2,0x18,0xd1,0x0c,0x10,0x04,0x09,0x00,0x09,0xff,0xe2,0xb1,0xaa,0x00,0x10, + 0x04,0x09,0x00,0x09,0xff,0xe2,0xb1,0xac,0x00,0xd1,0x0b,0x10,0x04,0x09,0x00,0x0a, + 0xff,0xc9,0x91,0x00,0x10,0x07,0x0a,0xff,0xc9,0xb1,0x00,0x0a,0xff,0xc9,0x90,0x00, + 0xd3,0x27,0xd2,0x17,0xd1,0x0b,0x10,0x07,0x0b,0xff,0xc9,0x92,0x00,0x0a,0x00,0x10, + 0x08,0x0a,0xff,0xe2,0xb1,0xb3,0x00,0x0a,0x00,0x91,0x0c,0x10,0x04,0x09,0x00,0x09, + 0xff,0xe2,0xb1,0xb6,0x00,0x09,0x00,0x52,0x04,0x0a,0x00,0x51,0x04,0x0a,0x00,0x10, + 0x07,0x0b,0xff,0xc8,0xbf,0x00,0x0b,0xff,0xc9,0x80,0x00,0xe0,0x83,0x01,0xcf,0x86, + 0xd5,0xc0,0xd4,0x60,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2, + 0x81,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0x83,0x00,0x08,0x00,0xd1,0x0c, + 0x10,0x08,0x08,0xff,0xe2,0xb2,0x85,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2, + 0x87,0x00,0x08,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2,0x89,0x00, + 0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0x8b,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08, + 0x08,0xff,0xe2,0xb2,0x8d,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0x8f,0x00, + 0x08,0x00,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2,0x91,0x00, + 0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0x93,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08, + 0x08,0xff,0xe2,0xb2,0x95,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0x97,0x00, + 0x08,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2,0x99,0x00,0x08,0x00, + 0x10,0x08,0x08,0xff,0xe2,0xb2,0x9b,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08,0x08,0xff, + 0xe2,0xb2,0x9d,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0x9f,0x00,0x08,0x00, + 0xd4,0x60,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2,0xa1,0x00, + 0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0xa3,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08, + 0x08,0xff,0xe2,0xb2,0xa5,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0xa7,0x00, + 0x08,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2,0xa9,0x00,0x08,0x00, + 0x10,0x08,0x08,0xff,0xe2,0xb2,0xab,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08,0x08,0xff, + 0xe2,0xb2,0xad,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0xaf,0x00,0x08,0x00, + 0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2,0xb1,0x00,0x08,0x00, + 0x10,0x08,0x08,0xff,0xe2,0xb2,0xb3,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08,0x08,0xff, + 0xe2,0xb2,0xb5,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0xb7,0x00,0x08,0x00, + 0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2,0xb9,0x00,0x08,0x00,0x10,0x08, + 0x08,0xff,0xe2,0xb2,0xbb,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb2, + 0xbd,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb2,0xbf,0x00,0x08,0x00,0xcf,0x86, + 0xd5,0xc0,0xd4,0x60,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb3, + 0x81,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb3,0x83,0x00,0x08,0x00,0xd1,0x0c, + 0x10,0x08,0x08,0xff,0xe2,0xb3,0x85,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb3, + 0x87,0x00,0x08,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb3,0x89,0x00, + 0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb3,0x8b,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08, + 0x08,0xff,0xe2,0xb3,0x8d,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb3,0x8f,0x00, + 0x08,0x00,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb3,0x91,0x00, + 0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb3,0x93,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08, + 0x08,0xff,0xe2,0xb3,0x95,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb3,0x97,0x00, + 0x08,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb3,0x99,0x00,0x08,0x00, + 0x10,0x08,0x08,0xff,0xe2,0xb3,0x9b,0x00,0x08,0x00,0xd1,0x0c,0x10,0x08,0x08,0xff, + 0xe2,0xb3,0x9d,0x00,0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb3,0x9f,0x00,0x08,0x00, + 0xd4,0x3b,0xd3,0x1c,0x92,0x18,0xd1,0x0c,0x10,0x08,0x08,0xff,0xe2,0xb3,0xa1,0x00, + 0x08,0x00,0x10,0x08,0x08,0xff,0xe2,0xb3,0xa3,0x00,0x08,0x00,0x08,0x00,0xd2,0x10, + 0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x0b,0xff,0xe2,0xb3,0xac,0x00,0xe1,0x3b, + 0x5f,0x10,0x04,0x0b,0x00,0x0b,0xff,0xe2,0xb3,0xae,0x00,0xe3,0x40,0x5f,0x92,0x10, + 0x51,0x04,0x0b,0xe6,0x10,0x08,0x0d,0xff,0xe2,0xb3,0xb3,0x00,0x0d,0x00,0x00,0x00, + 0xe2,0x98,0x08,0xd1,0x0b,0xe0,0x11,0x67,0xcf,0x86,0xcf,0x06,0x01,0x00,0xe0,0x65, + 0x6c,0xcf,0x86,0xe5,0xa7,0x05,0xd4,0x06,0xcf,0x06,0x04,0x00,0xd3,0x0c,0xe2,0xf8, + 0x67,0xe1,0x8f,0x67,0xcf,0x06,0x04,0x00,0xe2,0xdb,0x01,0xe1,0x26,0x01,0xd0,0x09, + 0xcf,0x86,0x65,0xf4,0x67,0x0a,0x00,0xcf,0x86,0xd5,0xc0,0xd4,0x60,0xd3,0x30,0xd2, + 0x18,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x99,0x81,0x00,0x0a,0x00,0x10,0x08,0x0a, + 0xff,0xea,0x99,0x83,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x99,0x85, + 0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x99,0x87,0x00,0x0a,0x00,0xd2,0x18,0xd1, + 0x0c,0x10,0x08,0x0a,0xff,0xea,0x99,0x89,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea, + 0x99,0x8b,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x99,0x8d,0x00,0x0a, + 0x00,0x10,0x08,0x0a,0xff,0xea,0x99,0x8f,0x00,0x0a,0x00,0xd3,0x30,0xd2,0x18,0xd1, + 0x0c,0x10,0x08,0x0a,0xff,0xea,0x99,0x91,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea, + 0x99,0x93,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x99,0x95,0x00,0x0a, + 0x00,0x10,0x08,0x0a,0xff,0xea,0x99,0x97,0x00,0x0a,0x00,0xd2,0x18,0xd1,0x0c,0x10, + 0x08,0x0a,0xff,0xea,0x99,0x99,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x99,0x9b, + 0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x99,0x9d,0x00,0x0a,0x00,0x10, + 0x08,0x0a,0xff,0xea,0x99,0x9f,0x00,0x0a,0x00,0xe4,0x5d,0x67,0xd3,0x30,0xd2,0x18, + 0xd1,0x0c,0x10,0x08,0x0c,0xff,0xea,0x99,0xa1,0x00,0x0c,0x00,0x10,0x08,0x0a,0xff, + 0xea,0x99,0xa3,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x99,0xa5,0x00, + 0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x99,0xa7,0x00,0x0a,0x00,0xd2,0x18,0xd1,0x0c, + 0x10,0x08,0x0a,0xff,0xea,0x99,0xa9,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x99, + 0xab,0x00,0x0a,0x00,0xe1,0x0c,0x67,0x10,0x08,0x0a,0xff,0xea,0x99,0xad,0x00,0x0a, + 0x00,0xe0,0x35,0x67,0xcf,0x86,0x95,0xab,0xd4,0x60,0xd3,0x30,0xd2,0x18,0xd1,0x0c, + 0x10,0x08,0x0a,0xff,0xea,0x9a,0x81,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9a, + 0x83,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9a,0x85,0x00,0x0a,0x00, + 0x10,0x08,0x0a,0xff,0xea,0x9a,0x87,0x00,0x0a,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08, + 0x0a,0xff,0xea,0x9a,0x89,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9a,0x8b,0x00, + 0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9a,0x8d,0x00,0x0a,0x00,0x10,0x08, + 0x0a,0xff,0xea,0x9a,0x8f,0x00,0x0a,0x00,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08, + 0x0a,0xff,0xea,0x9a,0x91,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9a,0x93,0x00, + 0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9a,0x95,0x00,0x0a,0x00,0x10,0x08, + 0x0a,0xff,0xea,0x9a,0x97,0x00,0x0a,0x00,0xe2,0x92,0x66,0xd1,0x0c,0x10,0x08,0x10, + 0xff,0xea,0x9a,0x99,0x00,0x10,0x00,0x10,0x08,0x10,0xff,0xea,0x9a,0x9b,0x00,0x10, + 0x00,0x0b,0x00,0xe1,0x10,0x02,0xd0,0xb9,0xcf,0x86,0xd5,0x07,0x64,0x9e,0x66,0x08, + 0x00,0xd4,0x58,0xd3,0x28,0xd2,0x10,0x51,0x04,0x09,0x00,0x10,0x08,0x0a,0xff,0xea, + 0x9c,0xa3,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9c,0xa5,0x00,0x0a, + 0x00,0x10,0x08,0x0a,0xff,0xea,0x9c,0xa7,0x00,0x0a,0x00,0xd2,0x18,0xd1,0x0c,0x10, + 0x08,0x0a,0xff,0xea,0x9c,0xa9,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9c,0xab, + 0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9c,0xad,0x00,0x0a,0x00,0x10, + 0x08,0x0a,0xff,0xea,0x9c,0xaf,0x00,0x0a,0x00,0xd3,0x28,0xd2,0x10,0x51,0x04,0x0a, + 0x00,0x10,0x08,0x0a,0xff,0xea,0x9c,0xb3,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a, + 0xff,0xea,0x9c,0xb5,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9c,0xb7,0x00,0x0a, + 0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9c,0xb9,0x00,0x0a,0x00,0x10, + 0x08,0x0a,0xff,0xea,0x9c,0xbb,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea, + 0x9c,0xbd,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9c,0xbf,0x00,0x0a,0x00,0xcf, + 0x86,0xd5,0xc0,0xd4,0x60,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea, + 0x9d,0x81,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0x83,0x00,0x0a,0x00,0xd1, + 0x0c,0x10,0x08,0x0a,0xff,0xea,0x9d,0x85,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea, + 0x9d,0x87,0x00,0x0a,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9d,0x89, + 0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0x8b,0x00,0x0a,0x00,0xd1,0x0c,0x10, + 0x08,0x0a,0xff,0xea,0x9d,0x8d,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0x8f, + 0x00,0x0a,0x00,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9d,0x91, + 0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0x93,0x00,0x0a,0x00,0xd1,0x0c,0x10, + 0x08,0x0a,0xff,0xea,0x9d,0x95,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0x97, + 0x00,0x0a,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9d,0x99,0x00,0x0a, + 0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0x9b,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a, + 0xff,0xea,0x9d,0x9d,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0x9f,0x00,0x0a, + 0x00,0xd4,0x60,0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9d,0xa1, + 0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0xa3,0x00,0x0a,0x00,0xd1,0x0c,0x10, + 0x08,0x0a,0xff,0xea,0x9d,0xa5,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0xa7, + 0x00,0x0a,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9d,0xa9,0x00,0x0a, + 0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0xab,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a, + 0xff,0xea,0x9d,0xad,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0xaf,0x00,0x0a, + 0x00,0x53,0x04,0x0a,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x04,0x0a,0x00,0x0a,0xff,0xea, + 0x9d,0xba,0x00,0x10,0x04,0x0a,0x00,0x0a,0xff,0xea,0x9d,0xbc,0x00,0xd1,0x0c,0x10, + 0x04,0x0a,0x00,0x0a,0xff,0xe1,0xb5,0xb9,0x00,0x10,0x08,0x0a,0xff,0xea,0x9d,0xbf, + 0x00,0x0a,0x00,0xe0,0x71,0x01,0xcf,0x86,0xd5,0xa6,0xd4,0x4e,0xd3,0x30,0xd2,0x18, + 0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9e,0x81,0x00,0x0a,0x00,0x10,0x08,0x0a,0xff, + 0xea,0x9e,0x83,0x00,0x0a,0x00,0xd1,0x0c,0x10,0x08,0x0a,0xff,0xea,0x9e,0x85,0x00, + 0x0a,0x00,0x10,0x08,0x0a,0xff,0xea,0x9e,0x87,0x00,0x0a,0x00,0xd2,0x10,0x51,0x04, + 0x0a,0x00,0x10,0x04,0x0a,0x00,0x0a,0xff,0xea,0x9e,0x8c,0x00,0xe1,0x9a,0x64,0x10, + 0x04,0x0a,0x00,0x0c,0xff,0xc9,0xa5,0x00,0xd3,0x28,0xd2,0x18,0xd1,0x0c,0x10,0x08, + 0x0c,0xff,0xea,0x9e,0x91,0x00,0x0c,0x00,0x10,0x08,0x0d,0xff,0xea,0x9e,0x93,0x00, + 0x0d,0x00,0x51,0x04,0x10,0x00,0x10,0x08,0x10,0xff,0xea,0x9e,0x97,0x00,0x10,0x00, + 0xd2,0x18,0xd1,0x0c,0x10,0x08,0x10,0xff,0xea,0x9e,0x99,0x00,0x10,0x00,0x10,0x08, + 0x10,0xff,0xea,0x9e,0x9b,0x00,0x10,0x00,0xd1,0x0c,0x10,0x08,0x10,0xff,0xea,0x9e, + 0x9d,0x00,0x10,0x00,0x10,0x08,0x10,0xff,0xea,0x9e,0x9f,0x00,0x10,0x00,0xd4,0x63, + 0xd3,0x30,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x0c,0xff,0xea,0x9e,0xa1,0x00,0x0c,0x00, + 0x10,0x08,0x0c,0xff,0xea,0x9e,0xa3,0x00,0x0c,0x00,0xd1,0x0c,0x10,0x08,0x0c,0xff, + 0xea,0x9e,0xa5,0x00,0x0c,0x00,0x10,0x08,0x0c,0xff,0xea,0x9e,0xa7,0x00,0x0c,0x00, + 0xd2,0x1a,0xd1,0x0c,0x10,0x08,0x0c,0xff,0xea,0x9e,0xa9,0x00,0x0c,0x00,0x10,0x07, + 0x0d,0xff,0xc9,0xa6,0x00,0x10,0xff,0xc9,0x9c,0x00,0xd1,0x0e,0x10,0x07,0x10,0xff, + 0xc9,0xa1,0x00,0x10,0xff,0xc9,0xac,0x00,0x10,0x07,0x12,0xff,0xc9,0xaa,0x00,0x14, + 0x00,0xd3,0x35,0xd2,0x1d,0xd1,0x0e,0x10,0x07,0x10,0xff,0xca,0x9e,0x00,0x10,0xff, + 0xca,0x87,0x00,0x10,0x07,0x11,0xff,0xca,0x9d,0x00,0x11,0xff,0xea,0xad,0x93,0x00, + 0xd1,0x0c,0x10,0x08,0x11,0xff,0xea,0x9e,0xb5,0x00,0x11,0x00,0x10,0x08,0x11,0xff, + 0xea,0x9e,0xb7,0x00,0x11,0x00,0xd2,0x18,0xd1,0x0c,0x10,0x08,0x14,0xff,0xea,0x9e, + 0xb9,0x00,0x14,0x00,0x10,0x08,0x15,0xff,0xea,0x9e,0xbb,0x00,0x15,0x00,0xd1,0x0c, + 0x10,0x08,0x15,0xff,0xea,0x9e,0xbd,0x00,0x15,0x00,0x10,0x08,0x15,0xff,0xea,0x9e, + 0xbf,0x00,0x15,0x00,0xcf,0x86,0xe5,0xd4,0x63,0x94,0x2f,0x93,0x2b,0xd2,0x10,0x51, + 0x04,0x00,0x00,0x10,0x08,0x15,0xff,0xea,0x9f,0x83,0x00,0x15,0x00,0xd1,0x0f,0x10, + 0x08,0x15,0xff,0xea,0x9e,0x94,0x00,0x15,0xff,0xca,0x82,0x00,0x10,0x08,0x15,0xff, + 0xe1,0xb6,0x8e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xe4,0xb4,0x66,0xd3,0x1d,0xe2, + 0x5b,0x64,0xe1,0x0a,0x64,0xe0,0xf7,0x63,0xcf,0x86,0xe5,0xd8,0x63,0x94,0x0b,0x93, + 0x07,0x62,0xc3,0x63,0x08,0x00,0x08,0x00,0x08,0x00,0xd2,0x0f,0xe1,0x5a,0x65,0xe0, + 0x27,0x65,0xcf,0x86,0x65,0x0c,0x65,0x0a,0x00,0xd1,0xab,0xd0,0x1a,0xcf,0x86,0xe5, + 0x17,0x66,0xe4,0xfa,0x65,0xe3,0xe1,0x65,0xe2,0xd4,0x65,0x91,0x08,0x10,0x04,0x00, + 0x00,0x0c,0x00,0x0c,0x00,0xcf,0x86,0x55,0x04,0x10,0x00,0xd4,0x0b,0x93,0x07,0x62, + 0x27,0x66,0x11,0x00,0x00,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x11,0xff, + 0xe1,0x8e,0xa0,0x00,0x11,0xff,0xe1,0x8e,0xa1,0x00,0x10,0x08,0x11,0xff,0xe1,0x8e, + 0xa2,0x00,0x11,0xff,0xe1,0x8e,0xa3,0x00,0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8e, + 0xa4,0x00,0x11,0xff,0xe1,0x8e,0xa5,0x00,0x10,0x08,0x11,0xff,0xe1,0x8e,0xa6,0x00, + 0x11,0xff,0xe1,0x8e,0xa7,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8e, + 0xa8,0x00,0x11,0xff,0xe1,0x8e,0xa9,0x00,0x10,0x08,0x11,0xff,0xe1,0x8e,0xaa,0x00, + 0x11,0xff,0xe1,0x8e,0xab,0x00,0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8e,0xac,0x00, + 0x11,0xff,0xe1,0x8e,0xad,0x00,0x10,0x08,0x11,0xff,0xe1,0x8e,0xae,0x00,0x11,0xff, + 0xe1,0x8e,0xaf,0x00,0xe0,0xb2,0x65,0xcf,0x86,0xe5,0x01,0x01,0xd4,0x80,0xd3,0x40, + 0xd2,0x20,0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8e,0xb0,0x00,0x11,0xff,0xe1,0x8e, + 0xb1,0x00,0x10,0x08,0x11,0xff,0xe1,0x8e,0xb2,0x00,0x11,0xff,0xe1,0x8e,0xb3,0x00, + 0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8e,0xb4,0x00,0x11,0xff,0xe1,0x8e,0xb5,0x00, + 0x10,0x08,0x11,0xff,0xe1,0x8e,0xb6,0x00,0x11,0xff,0xe1,0x8e,0xb7,0x00,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8e,0xb8,0x00,0x11,0xff,0xe1,0x8e,0xb9,0x00, + 0x10,0x08,0x11,0xff,0xe1,0x8e,0xba,0x00,0x11,0xff,0xe1,0x8e,0xbb,0x00,0xd1,0x10, + 0x10,0x08,0x11,0xff,0xe1,0x8e,0xbc,0x00,0x11,0xff,0xe1,0x8e,0xbd,0x00,0x10,0x08, + 0x11,0xff,0xe1,0x8e,0xbe,0x00,0x11,0xff,0xe1,0x8e,0xbf,0x00,0xd3,0x40,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8f,0x80,0x00,0x11,0xff,0xe1,0x8f,0x81,0x00, + 0x10,0x08,0x11,0xff,0xe1,0x8f,0x82,0x00,0x11,0xff,0xe1,0x8f,0x83,0x00,0xd1,0x10, + 0x10,0x08,0x11,0xff,0xe1,0x8f,0x84,0x00,0x11,0xff,0xe1,0x8f,0x85,0x00,0x10,0x08, + 0x11,0xff,0xe1,0x8f,0x86,0x00,0x11,0xff,0xe1,0x8f,0x87,0x00,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x11,0xff,0xe1,0x8f,0x88,0x00,0x11,0xff,0xe1,0x8f,0x89,0x00,0x10,0x08, + 0x11,0xff,0xe1,0x8f,0x8a,0x00,0x11,0xff,0xe1,0x8f,0x8b,0x00,0xd1,0x10,0x10,0x08, + 0x11,0xff,0xe1,0x8f,0x8c,0x00,0x11,0xff,0xe1,0x8f,0x8d,0x00,0x10,0x08,0x11,0xff, + 0xe1,0x8f,0x8e,0x00,0x11,0xff,0xe1,0x8f,0x8f,0x00,0xd4,0x80,0xd3,0x40,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x11,0xff,0xe1,0x8f,0x90,0x00,0x11,0xff,0xe1,0x8f,0x91,0x00, + 0x10,0x08,0x11,0xff,0xe1,0x8f,0x92,0x00,0x11,0xff,0xe1,0x8f,0x93,0x00,0xd1,0x10, + 0x10,0x08,0x11,0xff,0xe1,0x8f,0x94,0x00,0x11,0xff,0xe1,0x8f,0x95,0x00,0x10,0x08, + 0x11,0xff,0xe1,0x8f,0x96,0x00,0x11,0xff,0xe1,0x8f,0x97,0x00,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x11,0xff,0xe1,0x8f,0x98,0x00,0x11,0xff,0xe1,0x8f,0x99,0x00,0x10,0x08, + 0x11,0xff,0xe1,0x8f,0x9a,0x00,0x11,0xff,0xe1,0x8f,0x9b,0x00,0xd1,0x10,0x10,0x08, + 0x11,0xff,0xe1,0x8f,0x9c,0x00,0x11,0xff,0xe1,0x8f,0x9d,0x00,0x10,0x08,0x11,0xff, + 0xe1,0x8f,0x9e,0x00,0x11,0xff,0xe1,0x8f,0x9f,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x11,0xff,0xe1,0x8f,0xa0,0x00,0x11,0xff,0xe1,0x8f,0xa1,0x00,0x10,0x08, + 0x11,0xff,0xe1,0x8f,0xa2,0x00,0x11,0xff,0xe1,0x8f,0xa3,0x00,0xd1,0x10,0x10,0x08, + 0x11,0xff,0xe1,0x8f,0xa4,0x00,0x11,0xff,0xe1,0x8f,0xa5,0x00,0x10,0x08,0x11,0xff, + 0xe1,0x8f,0xa6,0x00,0x11,0xff,0xe1,0x8f,0xa7,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x11,0xff,0xe1,0x8f,0xa8,0x00,0x11,0xff,0xe1,0x8f,0xa9,0x00,0x10,0x08,0x11,0xff, + 0xe1,0x8f,0xaa,0x00,0x11,0xff,0xe1,0x8f,0xab,0x00,0xd1,0x10,0x10,0x08,0x11,0xff, + 0xe1,0x8f,0xac,0x00,0x11,0xff,0xe1,0x8f,0xad,0x00,0x10,0x08,0x11,0xff,0xe1,0x8f, + 0xae,0x00,0x11,0xff,0xe1,0x8f,0xaf,0x00,0xd1,0x0c,0xe0,0xeb,0x63,0xcf,0x86,0xcf, + 0x06,0x02,0xff,0xff,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x01,0x00,0xcf,0x86,0xd5,0x06, + 0xcf,0x06,0x01,0x00,0xd4,0xae,0xd3,0x09,0xe2,0x54,0x64,0xcf,0x06,0x01,0x00,0xd2, + 0x27,0xe1,0x1f,0x70,0xe0,0x26,0x6e,0xcf,0x86,0xe5,0x3f,0x6d,0xe4,0xce,0x6c,0xe3, + 0x99,0x6c,0xe2,0x78,0x6c,0xe1,0x67,0x6c,0x10,0x08,0x01,0xff,0xe5,0x88,0x87,0x00, + 0x01,0xff,0xe5,0xba,0xa6,0x00,0xe1,0x74,0x74,0xe0,0xe8,0x73,0xcf,0x86,0xe5,0x22, + 0x73,0xd4,0x3b,0x93,0x37,0xd2,0x1d,0xd1,0x0e,0x10,0x07,0x01,0xff,0x66,0x66,0x00, + 0x01,0xff,0x66,0x69,0x00,0x10,0x07,0x01,0xff,0x66,0x6c,0x00,0x01,0xff,0x66,0x66, + 0x69,0x00,0xd1,0x0f,0x10,0x08,0x01,0xff,0x66,0x66,0x6c,0x00,0x01,0xff,0x73,0x74, + 0x00,0x10,0x07,0x01,0xff,0x73,0x74,0x00,0x00,0x00,0x00,0x00,0xe3,0xc8,0x72,0xd2, + 0x11,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0xff,0xd5,0xb4,0xd5,0xb6,0x00, + 0xd1,0x12,0x10,0x09,0x01,0xff,0xd5,0xb4,0xd5,0xa5,0x00,0x01,0xff,0xd5,0xb4,0xd5, + 0xab,0x00,0x10,0x09,0x01,0xff,0xd5,0xbe,0xd5,0xb6,0x00,0x01,0xff,0xd5,0xb4,0xd5, + 0xad,0x00,0xd3,0x09,0xe2,0x40,0x74,0xcf,0x06,0x01,0x00,0xd2,0x13,0xe1,0x30,0x75, + 0xe0,0xc1,0x74,0xcf,0x86,0xe5,0x9e,0x74,0x64,0x8d,0x74,0x06,0xff,0x00,0xe1,0x96, + 0x75,0xe0,0x63,0x75,0xcf,0x86,0xd5,0x18,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08, + 0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4,0x7c, + 0xd3,0x3c,0xd2,0x1c,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0xef,0xbd,0x81,0x00, + 0x10,0x08,0x01,0xff,0xef,0xbd,0x82,0x00,0x01,0xff,0xef,0xbd,0x83,0x00,0xd1,0x10, + 0x10,0x08,0x01,0xff,0xef,0xbd,0x84,0x00,0x01,0xff,0xef,0xbd,0x85,0x00,0x10,0x08, + 0x01,0xff,0xef,0xbd,0x86,0x00,0x01,0xff,0xef,0xbd,0x87,0x00,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x01,0xff,0xef,0xbd,0x88,0x00,0x01,0xff,0xef,0xbd,0x89,0x00,0x10,0x08, + 0x01,0xff,0xef,0xbd,0x8a,0x00,0x01,0xff,0xef,0xbd,0x8b,0x00,0xd1,0x10,0x10,0x08, + 0x01,0xff,0xef,0xbd,0x8c,0x00,0x01,0xff,0xef,0xbd,0x8d,0x00,0x10,0x08,0x01,0xff, + 0xef,0xbd,0x8e,0x00,0x01,0xff,0xef,0xbd,0x8f,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x01,0xff,0xef,0xbd,0x90,0x00,0x01,0xff,0xef,0xbd,0x91,0x00,0x10,0x08, + 0x01,0xff,0xef,0xbd,0x92,0x00,0x01,0xff,0xef,0xbd,0x93,0x00,0xd1,0x10,0x10,0x08, + 0x01,0xff,0xef,0xbd,0x94,0x00,0x01,0xff,0xef,0xbd,0x95,0x00,0x10,0x08,0x01,0xff, + 0xef,0xbd,0x96,0x00,0x01,0xff,0xef,0xbd,0x97,0x00,0x92,0x1c,0xd1,0x10,0x10,0x08, + 0x01,0xff,0xef,0xbd,0x98,0x00,0x01,0xff,0xef,0xbd,0x99,0x00,0x10,0x08,0x01,0xff, + 0xef,0xbd,0x9a,0x00,0x01,0x00,0x01,0x00,0x83,0xe2,0x87,0xb3,0xe1,0x60,0xb0,0xe0, + 0xdd,0xae,0xcf,0x86,0xe5,0x81,0x9b,0xc4,0xe3,0xc1,0x07,0xe2,0x62,0x06,0xe1,0x11, + 0x86,0xe0,0x09,0x05,0xcf,0x86,0xe5,0xfb,0x02,0xd4,0x1c,0xe3,0x7f,0x76,0xe2,0xd6, + 0x75,0xe1,0xb1,0x75,0xe0,0x8a,0x75,0xcf,0x86,0xe5,0x57,0x75,0x94,0x07,0x63,0x42, + 0x75,0x07,0x00,0x07,0x00,0xe3,0x2b,0x78,0xe2,0xf0,0x77,0xe1,0x77,0x01,0xe0,0x88, + 0x77,0xcf,0x86,0xe5,0x21,0x01,0xd4,0x90,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10,0x09, + 0x05,0xff,0xf0,0x90,0x90,0xa8,0x00,0x05,0xff,0xf0,0x90,0x90,0xa9,0x00,0x10,0x09, + 0x05,0xff,0xf0,0x90,0x90,0xaa,0x00,0x05,0xff,0xf0,0x90,0x90,0xab,0x00,0xd1,0x12, + 0x10,0x09,0x05,0xff,0xf0,0x90,0x90,0xac,0x00,0x05,0xff,0xf0,0x90,0x90,0xad,0x00, + 0x10,0x09,0x05,0xff,0xf0,0x90,0x90,0xae,0x00,0x05,0xff,0xf0,0x90,0x90,0xaf,0x00, + 0xd2,0x24,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0x90,0x90,0xb0,0x00,0x05,0xff,0xf0, + 0x90,0x90,0xb1,0x00,0x10,0x09,0x05,0xff,0xf0,0x90,0x90,0xb2,0x00,0x05,0xff,0xf0, + 0x90,0x90,0xb3,0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0x90,0x90,0xb4,0x00,0x05, + 0xff,0xf0,0x90,0x90,0xb5,0x00,0x10,0x09,0x05,0xff,0xf0,0x90,0x90,0xb6,0x00,0x05, + 0xff,0xf0,0x90,0x90,0xb7,0x00,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10,0x09,0x05,0xff, + 0xf0,0x90,0x90,0xb8,0x00,0x05,0xff,0xf0,0x90,0x90,0xb9,0x00,0x10,0x09,0x05,0xff, + 0xf0,0x90,0x90,0xba,0x00,0x05,0xff,0xf0,0x90,0x90,0xbb,0x00,0xd1,0x12,0x10,0x09, + 0x05,0xff,0xf0,0x90,0x90,0xbc,0x00,0x05,0xff,0xf0,0x90,0x90,0xbd,0x00,0x10,0x09, + 0x05,0xff,0xf0,0x90,0x90,0xbe,0x00,0x05,0xff,0xf0,0x90,0x90,0xbf,0x00,0xd2,0x24, + 0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0x90,0x91,0x80,0x00,0x05,0xff,0xf0,0x90,0x91, + 0x81,0x00,0x10,0x09,0x05,0xff,0xf0,0x90,0x91,0x82,0x00,0x05,0xff,0xf0,0x90,0x91, + 0x83,0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0x90,0x91,0x84,0x00,0x05,0xff,0xf0, + 0x90,0x91,0x85,0x00,0x10,0x09,0x05,0xff,0xf0,0x90,0x91,0x86,0x00,0x05,0xff,0xf0, + 0x90,0x91,0x87,0x00,0x94,0x4c,0x93,0x48,0xd2,0x24,0xd1,0x12,0x10,0x09,0x05,0xff, + 0xf0,0x90,0x91,0x88,0x00,0x05,0xff,0xf0,0x90,0x91,0x89,0x00,0x10,0x09,0x05,0xff, + 0xf0,0x90,0x91,0x8a,0x00,0x05,0xff,0xf0,0x90,0x91,0x8b,0x00,0xd1,0x12,0x10,0x09, + 0x05,0xff,0xf0,0x90,0x91,0x8c,0x00,0x05,0xff,0xf0,0x90,0x91,0x8d,0x00,0x10,0x09, + 0x07,0xff,0xf0,0x90,0x91,0x8e,0x00,0x07,0xff,0xf0,0x90,0x91,0x8f,0x00,0x05,0x00, + 0x05,0x00,0xd0,0xa0,0xcf,0x86,0xd5,0x07,0x64,0x30,0x76,0x07,0x00,0xd4,0x07,0x63, + 0x3d,0x76,0x07,0x00,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x90, + 0x93,0x98,0x00,0x12,0xff,0xf0,0x90,0x93,0x99,0x00,0x10,0x09,0x12,0xff,0xf0,0x90, + 0x93,0x9a,0x00,0x12,0xff,0xf0,0x90,0x93,0x9b,0x00,0xd1,0x12,0x10,0x09,0x12,0xff, + 0xf0,0x90,0x93,0x9c,0x00,0x12,0xff,0xf0,0x90,0x93,0x9d,0x00,0x10,0x09,0x12,0xff, + 0xf0,0x90,0x93,0x9e,0x00,0x12,0xff,0xf0,0x90,0x93,0x9f,0x00,0xd2,0x24,0xd1,0x12, + 0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xa0,0x00,0x12,0xff,0xf0,0x90,0x93,0xa1,0x00, + 0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xa2,0x00,0x12,0xff,0xf0,0x90,0x93,0xa3,0x00, + 0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xa4,0x00,0x12,0xff,0xf0,0x90,0x93, + 0xa5,0x00,0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xa6,0x00,0x12,0xff,0xf0,0x90,0x93, + 0xa7,0x00,0xcf,0x86,0xe5,0xc6,0x75,0xd4,0x90,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10, + 0x09,0x12,0xff,0xf0,0x90,0x93,0xa8,0x00,0x12,0xff,0xf0,0x90,0x93,0xa9,0x00,0x10, + 0x09,0x12,0xff,0xf0,0x90,0x93,0xaa,0x00,0x12,0xff,0xf0,0x90,0x93,0xab,0x00,0xd1, + 0x12,0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xac,0x00,0x12,0xff,0xf0,0x90,0x93,0xad, + 0x00,0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xae,0x00,0x12,0xff,0xf0,0x90,0x93,0xaf, + 0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xb0,0x00,0x12,0xff, + 0xf0,0x90,0x93,0xb1,0x00,0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xb2,0x00,0x12,0xff, + 0xf0,0x90,0x93,0xb3,0x00,0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xb4,0x00, + 0x12,0xff,0xf0,0x90,0x93,0xb5,0x00,0x10,0x09,0x12,0xff,0xf0,0x90,0x93,0xb6,0x00, + 0x12,0xff,0xf0,0x90,0x93,0xb7,0x00,0x93,0x28,0x92,0x24,0xd1,0x12,0x10,0x09,0x12, + 0xff,0xf0,0x90,0x93,0xb8,0x00,0x12,0xff,0xf0,0x90,0x93,0xb9,0x00,0x10,0x09,0x12, + 0xff,0xf0,0x90,0x93,0xba,0x00,0x12,0xff,0xf0,0x90,0x93,0xbb,0x00,0x00,0x00,0x12, + 0x00,0xd4,0x1f,0xe3,0xdf,0x76,0xe2,0x6a,0x76,0xe1,0x09,0x76,0xe0,0xea,0x75,0xcf, + 0x86,0xe5,0xb7,0x75,0x94,0x0a,0xe3,0xa2,0x75,0x62,0x99,0x75,0x07,0x00,0x07,0x00, + 0xe3,0xde,0x78,0xe2,0xaf,0x78,0xd1,0x09,0xe0,0x4c,0x78,0xcf,0x06,0x0b,0x00,0xe0, + 0x7f,0x78,0xcf,0x86,0xe5,0x21,0x01,0xd4,0x90,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10, + 0x09,0x11,0xff,0xf0,0x90,0xb3,0x80,0x00,0x11,0xff,0xf0,0x90,0xb3,0x81,0x00,0x10, + 0x09,0x11,0xff,0xf0,0x90,0xb3,0x82,0x00,0x11,0xff,0xf0,0x90,0xb3,0x83,0x00,0xd1, + 0x12,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x84,0x00,0x11,0xff,0xf0,0x90,0xb3,0x85, + 0x00,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x86,0x00,0x11,0xff,0xf0,0x90,0xb3,0x87, + 0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x88,0x00,0x11,0xff, + 0xf0,0x90,0xb3,0x89,0x00,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x8a,0x00,0x11,0xff, + 0xf0,0x90,0xb3,0x8b,0x00,0xd1,0x12,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x8c,0x00, + 0x11,0xff,0xf0,0x90,0xb3,0x8d,0x00,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x8e,0x00, + 0x11,0xff,0xf0,0x90,0xb3,0x8f,0x00,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10,0x09,0x11, + 0xff,0xf0,0x90,0xb3,0x90,0x00,0x11,0xff,0xf0,0x90,0xb3,0x91,0x00,0x10,0x09,0x11, + 0xff,0xf0,0x90,0xb3,0x92,0x00,0x11,0xff,0xf0,0x90,0xb3,0x93,0x00,0xd1,0x12,0x10, + 0x09,0x11,0xff,0xf0,0x90,0xb3,0x94,0x00,0x11,0xff,0xf0,0x90,0xb3,0x95,0x00,0x10, + 0x09,0x11,0xff,0xf0,0x90,0xb3,0x96,0x00,0x11,0xff,0xf0,0x90,0xb3,0x97,0x00,0xd2, + 0x24,0xd1,0x12,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x98,0x00,0x11,0xff,0xf0,0x90, + 0xb3,0x99,0x00,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x9a,0x00,0x11,0xff,0xf0,0x90, + 0xb3,0x9b,0x00,0xd1,0x12,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x9c,0x00,0x11,0xff, + 0xf0,0x90,0xb3,0x9d,0x00,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0x9e,0x00,0x11,0xff, + 0xf0,0x90,0xb3,0x9f,0x00,0xd4,0x90,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10,0x09,0x11, + 0xff,0xf0,0x90,0xb3,0xa0,0x00,0x11,0xff,0xf0,0x90,0xb3,0xa1,0x00,0x10,0x09,0x11, + 0xff,0xf0,0x90,0xb3,0xa2,0x00,0x11,0xff,0xf0,0x90,0xb3,0xa3,0x00,0xd1,0x12,0x10, + 0x09,0x11,0xff,0xf0,0x90,0xb3,0xa4,0x00,0x11,0xff,0xf0,0x90,0xb3,0xa5,0x00,0x10, + 0x09,0x11,0xff,0xf0,0x90,0xb3,0xa6,0x00,0x11,0xff,0xf0,0x90,0xb3,0xa7,0x00,0xd2, + 0x24,0xd1,0x12,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0xa8,0x00,0x11,0xff,0xf0,0x90, + 0xb3,0xa9,0x00,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0xaa,0x00,0x11,0xff,0xf0,0x90, + 0xb3,0xab,0x00,0xd1,0x12,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0xac,0x00,0x11,0xff, + 0xf0,0x90,0xb3,0xad,0x00,0x10,0x09,0x11,0xff,0xf0,0x90,0xb3,0xae,0x00,0x11,0xff, + 0xf0,0x90,0xb3,0xaf,0x00,0x93,0x23,0x92,0x1f,0xd1,0x12,0x10,0x09,0x11,0xff,0xf0, + 0x90,0xb3,0xb0,0x00,0x11,0xff,0xf0,0x90,0xb3,0xb1,0x00,0x10,0x09,0x11,0xff,0xf0, + 0x90,0xb3,0xb2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0xd5,0x15,0xe4,0x91, + 0x7b,0xe3,0x9b,0x79,0xe2,0x94,0x78,0xe1,0xe4,0x77,0xe0,0x9d,0x77,0xcf,0x06,0x0c, + 0x00,0xe4,0xeb,0x7e,0xe3,0x44,0x7e,0xe2,0xed,0x7d,0xd1,0x0c,0xe0,0xb2,0x7d,0xcf, + 0x86,0x65,0x93,0x7d,0x14,0x00,0xe0,0xb6,0x7d,0xcf,0x86,0x55,0x04,0x00,0x00,0xd4, + 0x90,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10,0x09,0x10,0xff,0xf0,0x91,0xa3,0x80,0x00, + 0x10,0xff,0xf0,0x91,0xa3,0x81,0x00,0x10,0x09,0x10,0xff,0xf0,0x91,0xa3,0x82,0x00, + 0x10,0xff,0xf0,0x91,0xa3,0x83,0x00,0xd1,0x12,0x10,0x09,0x10,0xff,0xf0,0x91,0xa3, + 0x84,0x00,0x10,0xff,0xf0,0x91,0xa3,0x85,0x00,0x10,0x09,0x10,0xff,0xf0,0x91,0xa3, + 0x86,0x00,0x10,0xff,0xf0,0x91,0xa3,0x87,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x10, + 0xff,0xf0,0x91,0xa3,0x88,0x00,0x10,0xff,0xf0,0x91,0xa3,0x89,0x00,0x10,0x09,0x10, + 0xff,0xf0,0x91,0xa3,0x8a,0x00,0x10,0xff,0xf0,0x91,0xa3,0x8b,0x00,0xd1,0x12,0x10, + 0x09,0x10,0xff,0xf0,0x91,0xa3,0x8c,0x00,0x10,0xff,0xf0,0x91,0xa3,0x8d,0x00,0x10, + 0x09,0x10,0xff,0xf0,0x91,0xa3,0x8e,0x00,0x10,0xff,0xf0,0x91,0xa3,0x8f,0x00,0xd3, + 0x48,0xd2,0x24,0xd1,0x12,0x10,0x09,0x10,0xff,0xf0,0x91,0xa3,0x90,0x00,0x10,0xff, + 0xf0,0x91,0xa3,0x91,0x00,0x10,0x09,0x10,0xff,0xf0,0x91,0xa3,0x92,0x00,0x10,0xff, + 0xf0,0x91,0xa3,0x93,0x00,0xd1,0x12,0x10,0x09,0x10,0xff,0xf0,0x91,0xa3,0x94,0x00, + 0x10,0xff,0xf0,0x91,0xa3,0x95,0x00,0x10,0x09,0x10,0xff,0xf0,0x91,0xa3,0x96,0x00, + 0x10,0xff,0xf0,0x91,0xa3,0x97,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x10,0xff,0xf0, + 0x91,0xa3,0x98,0x00,0x10,0xff,0xf0,0x91,0xa3,0x99,0x00,0x10,0x09,0x10,0xff,0xf0, + 0x91,0xa3,0x9a,0x00,0x10,0xff,0xf0,0x91,0xa3,0x9b,0x00,0xd1,0x12,0x10,0x09,0x10, + 0xff,0xf0,0x91,0xa3,0x9c,0x00,0x10,0xff,0xf0,0x91,0xa3,0x9d,0x00,0x10,0x09,0x10, + 0xff,0xf0,0x91,0xa3,0x9e,0x00,0x10,0xff,0xf0,0x91,0xa3,0x9f,0x00,0xd1,0x11,0xe0, + 0x12,0x81,0xcf,0x86,0xe5,0x09,0x81,0xe4,0xd2,0x80,0xcf,0x06,0x00,0x00,0xe0,0xdb, + 0x82,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x09,0xe3,0x10,0x81,0xcf,0x06, + 0x0c,0x00,0xd3,0x06,0xcf,0x06,0x00,0x00,0xe2,0x3b,0x82,0xe1,0x16,0x82,0xd0,0x06, + 0xcf,0x06,0x00,0x00,0xcf,0x86,0xa5,0x21,0x01,0xd4,0x90,0xd3,0x48,0xd2,0x24,0xd1, + 0x12,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xa0,0x00,0x14,0xff,0xf0,0x96,0xb9,0xa1, + 0x00,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xa2,0x00,0x14,0xff,0xf0,0x96,0xb9,0xa3, + 0x00,0xd1,0x12,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xa4,0x00,0x14,0xff,0xf0,0x96, + 0xb9,0xa5,0x00,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xa6,0x00,0x14,0xff,0xf0,0x96, + 0xb9,0xa7,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xa8,0x00, + 0x14,0xff,0xf0,0x96,0xb9,0xa9,0x00,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xaa,0x00, + 0x14,0xff,0xf0,0x96,0xb9,0xab,0x00,0xd1,0x12,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9, + 0xac,0x00,0x14,0xff,0xf0,0x96,0xb9,0xad,0x00,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9, + 0xae,0x00,0x14,0xff,0xf0,0x96,0xb9,0xaf,0x00,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10, + 0x09,0x14,0xff,0xf0,0x96,0xb9,0xb0,0x00,0x14,0xff,0xf0,0x96,0xb9,0xb1,0x00,0x10, + 0x09,0x14,0xff,0xf0,0x96,0xb9,0xb2,0x00,0x14,0xff,0xf0,0x96,0xb9,0xb3,0x00,0xd1, + 0x12,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xb4,0x00,0x14,0xff,0xf0,0x96,0xb9,0xb5, + 0x00,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xb6,0x00,0x14,0xff,0xf0,0x96,0xb9,0xb7, + 0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xb8,0x00,0x14,0xff, + 0xf0,0x96,0xb9,0xb9,0x00,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xba,0x00,0x14,0xff, + 0xf0,0x96,0xb9,0xbb,0x00,0xd1,0x12,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xbc,0x00, + 0x14,0xff,0xf0,0x96,0xb9,0xbd,0x00,0x10,0x09,0x14,0xff,0xf0,0x96,0xb9,0xbe,0x00, + 0x14,0xff,0xf0,0x96,0xb9,0xbf,0x00,0x14,0x00,0xd2,0x14,0xe1,0x25,0x82,0xe0,0x1c, + 0x82,0xcf,0x86,0xe5,0xdd,0x81,0xe4,0x9a,0x81,0xcf,0x06,0x12,0x00,0xd1,0x0b,0xe0, + 0x51,0x83,0xcf,0x86,0xcf,0x06,0x00,0x00,0xe0,0x95,0x8b,0xcf,0x86,0xd5,0x22,0xe4, + 0xd0,0x88,0xe3,0x93,0x88,0xe2,0x38,0x88,0xe1,0x31,0x88,0xe0,0x2a,0x88,0xcf,0x86, + 0xe5,0xfb,0x87,0xe4,0xe2,0x87,0x93,0x07,0x62,0xd1,0x87,0x12,0xe6,0x12,0xe6,0xe4, + 0x36,0x89,0xe3,0x2f,0x89,0xd2,0x09,0xe1,0xb8,0x88,0xcf,0x06,0x10,0x00,0xe1,0x1f, + 0x89,0xe0,0xec,0x88,0xcf,0x86,0xe5,0x21,0x01,0xd4,0x90,0xd3,0x48,0xd2,0x24,0xd1, + 0x12,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xa2,0x00,0x12,0xff,0xf0,0x9e,0xa4,0xa3, + 0x00,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xa4,0x00,0x12,0xff,0xf0,0x9e,0xa4,0xa5, + 0x00,0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xa6,0x00,0x12,0xff,0xf0,0x9e, + 0xa4,0xa7,0x00,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xa8,0x00,0x12,0xff,0xf0,0x9e, + 0xa4,0xa9,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xaa,0x00, + 0x12,0xff,0xf0,0x9e,0xa4,0xab,0x00,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xac,0x00, + 0x12,0xff,0xf0,0x9e,0xa4,0xad,0x00,0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4, + 0xae,0x00,0x12,0xff,0xf0,0x9e,0xa4,0xaf,0x00,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4, + 0xb0,0x00,0x12,0xff,0xf0,0x9e,0xa4,0xb1,0x00,0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10, + 0x09,0x12,0xff,0xf0,0x9e,0xa4,0xb2,0x00,0x12,0xff,0xf0,0x9e,0xa4,0xb3,0x00,0x10, + 0x09,0x12,0xff,0xf0,0x9e,0xa4,0xb4,0x00,0x12,0xff,0xf0,0x9e,0xa4,0xb5,0x00,0xd1, + 0x12,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xb6,0x00,0x12,0xff,0xf0,0x9e,0xa4,0xb7, + 0x00,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xb8,0x00,0x12,0xff,0xf0,0x9e,0xa4,0xb9, + 0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xba,0x00,0x12,0xff, + 0xf0,0x9e,0xa4,0xbb,0x00,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xbc,0x00,0x12,0xff, + 0xf0,0x9e,0xa4,0xbd,0x00,0xd1,0x12,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa4,0xbe,0x00, + 0x12,0xff,0xf0,0x9e,0xa4,0xbf,0x00,0x10,0x09,0x12,0xff,0xf0,0x9e,0xa5,0x80,0x00, + 0x12,0xff,0xf0,0x9e,0xa5,0x81,0x00,0x94,0x1e,0x93,0x1a,0x92,0x16,0x91,0x12,0x10, + 0x09,0x12,0xff,0xf0,0x9e,0xa5,0x82,0x00,0x12,0xff,0xf0,0x9e,0xa5,0x83,0x00,0x12, + 0x00,0x12,0x00,0x12,0x00,0x12,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + /* nfdi_c0100 */ + 0x57,0x04,0x01,0x00,0xc6,0xe5,0xac,0x13,0xe4,0x41,0x0c,0xe3,0x7a,0x07,0xe2,0xf3, + 0x01,0xc1,0xd0,0x1f,0xcf,0x86,0x55,0x04,0x01,0x00,0x94,0x15,0x53,0x04,0x01,0x00, + 0x52,0x04,0x01,0x00,0x91,0x09,0x10,0x04,0x01,0x00,0x01,0xff,0x00,0x01,0x00,0x01, + 0x00,0xcf,0x86,0xd5,0xe4,0xd4,0x7c,0xd3,0x3c,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01, + 0xff,0x41,0xcc,0x80,0x00,0x01,0xff,0x41,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x41, + 0xcc,0x82,0x00,0x01,0xff,0x41,0xcc,0x83,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x41, + 0xcc,0x88,0x00,0x01,0xff,0x41,0xcc,0x8a,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0x43, + 0xcc,0xa7,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x45,0xcc,0x80,0x00,0x01, + 0xff,0x45,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x45,0xcc,0x82,0x00,0x01,0xff,0x45, + 0xcc,0x88,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x49,0xcc,0x80,0x00,0x01,0xff,0x49, + 0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x49,0xcc,0x82,0x00,0x01,0xff,0x49,0xcc,0x88, + 0x00,0xd3,0x38,0xd2,0x1c,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0x4e,0xcc,0x83, + 0x00,0x10,0x08,0x01,0xff,0x4f,0xcc,0x80,0x00,0x01,0xff,0x4f,0xcc,0x81,0x00,0xd1, + 0x10,0x10,0x08,0x01,0xff,0x4f,0xcc,0x82,0x00,0x01,0xff,0x4f,0xcc,0x83,0x00,0x10, + 0x08,0x01,0xff,0x4f,0xcc,0x88,0x00,0x01,0x00,0xd2,0x1c,0xd1,0x0c,0x10,0x04,0x01, + 0x00,0x01,0xff,0x55,0xcc,0x80,0x00,0x10,0x08,0x01,0xff,0x55,0xcc,0x81,0x00,0x01, + 0xff,0x55,0xcc,0x82,0x00,0x91,0x10,0x10,0x08,0x01,0xff,0x55,0xcc,0x88,0x00,0x01, + 0xff,0x59,0xcc,0x81,0x00,0x01,0x00,0xd4,0x7c,0xd3,0x3c,0xd2,0x20,0xd1,0x10,0x10, + 0x08,0x01,0xff,0x61,0xcc,0x80,0x00,0x01,0xff,0x61,0xcc,0x81,0x00,0x10,0x08,0x01, + 0xff,0x61,0xcc,0x82,0x00,0x01,0xff,0x61,0xcc,0x83,0x00,0xd1,0x10,0x10,0x08,0x01, + 0xff,0x61,0xcc,0x88,0x00,0x01,0xff,0x61,0xcc,0x8a,0x00,0x10,0x04,0x01,0x00,0x01, + 0xff,0x63,0xcc,0xa7,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x65,0xcc,0x80, + 0x00,0x01,0xff,0x65,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x65,0xcc,0x82,0x00,0x01, + 0xff,0x65,0xcc,0x88,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x69,0xcc,0x80,0x00,0x01, + 0xff,0x69,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x69,0xcc,0x82,0x00,0x01,0xff,0x69, + 0xcc,0x88,0x00,0xd3,0x38,0xd2,0x1c,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0x6e, + 0xcc,0x83,0x00,0x10,0x08,0x01,0xff,0x6f,0xcc,0x80,0x00,0x01,0xff,0x6f,0xcc,0x81, + 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x6f,0xcc,0x82,0x00,0x01,0xff,0x6f,0xcc,0x83, + 0x00,0x10,0x08,0x01,0xff,0x6f,0xcc,0x88,0x00,0x01,0x00,0xd2,0x1c,0xd1,0x0c,0x10, + 0x04,0x01,0x00,0x01,0xff,0x75,0xcc,0x80,0x00,0x10,0x08,0x01,0xff,0x75,0xcc,0x81, + 0x00,0x01,0xff,0x75,0xcc,0x82,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x75,0xcc,0x88, + 0x00,0x01,0xff,0x79,0xcc,0x81,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0x79,0xcc,0x88, + 0x00,0xe1,0x9a,0x03,0xe0,0xd3,0x01,0xcf,0x86,0xd5,0xf4,0xd4,0x80,0xd3,0x40,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x41,0xcc,0x84,0x00,0x01,0xff,0x61,0xcc,0x84, + 0x00,0x10,0x08,0x01,0xff,0x41,0xcc,0x86,0x00,0x01,0xff,0x61,0xcc,0x86,0x00,0xd1, + 0x10,0x10,0x08,0x01,0xff,0x41,0xcc,0xa8,0x00,0x01,0xff,0x61,0xcc,0xa8,0x00,0x10, + 0x08,0x01,0xff,0x43,0xcc,0x81,0x00,0x01,0xff,0x63,0xcc,0x81,0x00,0xd2,0x20,0xd1, + 0x10,0x10,0x08,0x01,0xff,0x43,0xcc,0x82,0x00,0x01,0xff,0x63,0xcc,0x82,0x00,0x10, + 0x08,0x01,0xff,0x43,0xcc,0x87,0x00,0x01,0xff,0x63,0xcc,0x87,0x00,0xd1,0x10,0x10, + 0x08,0x01,0xff,0x43,0xcc,0x8c,0x00,0x01,0xff,0x63,0xcc,0x8c,0x00,0x10,0x08,0x01, + 0xff,0x44,0xcc,0x8c,0x00,0x01,0xff,0x64,0xcc,0x8c,0x00,0xd3,0x34,0xd2,0x14,0x51, + 0x04,0x01,0x00,0x10,0x08,0x01,0xff,0x45,0xcc,0x84,0x00,0x01,0xff,0x65,0xcc,0x84, + 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x45,0xcc,0x86,0x00,0x01,0xff,0x65,0xcc,0x86, + 0x00,0x10,0x08,0x01,0xff,0x45,0xcc,0x87,0x00,0x01,0xff,0x65,0xcc,0x87,0x00,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x45,0xcc,0xa8,0x00,0x01,0xff,0x65,0xcc,0xa8, + 0x00,0x10,0x08,0x01,0xff,0x45,0xcc,0x8c,0x00,0x01,0xff,0x65,0xcc,0x8c,0x00,0xd1, + 0x10,0x10,0x08,0x01,0xff,0x47,0xcc,0x82,0x00,0x01,0xff,0x67,0xcc,0x82,0x00,0x10, + 0x08,0x01,0xff,0x47,0xcc,0x86,0x00,0x01,0xff,0x67,0xcc,0x86,0x00,0xd4,0x74,0xd3, + 0x34,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x47,0xcc,0x87,0x00,0x01,0xff,0x67, + 0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x47,0xcc,0xa7,0x00,0x01,0xff,0x67,0xcc,0xa7, + 0x00,0x91,0x10,0x10,0x08,0x01,0xff,0x48,0xcc,0x82,0x00,0x01,0xff,0x68,0xcc,0x82, + 0x00,0x01,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x49,0xcc,0x83,0x00,0x01, + 0xff,0x69,0xcc,0x83,0x00,0x10,0x08,0x01,0xff,0x49,0xcc,0x84,0x00,0x01,0xff,0x69, + 0xcc,0x84,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x49,0xcc,0x86,0x00,0x01,0xff,0x69, + 0xcc,0x86,0x00,0x10,0x08,0x01,0xff,0x49,0xcc,0xa8,0x00,0x01,0xff,0x69,0xcc,0xa8, + 0x00,0xd3,0x30,0xd2,0x10,0x91,0x0c,0x10,0x08,0x01,0xff,0x49,0xcc,0x87,0x00,0x01, + 0x00,0x01,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x4a,0xcc,0x82,0x00,0x01,0xff,0x6a, + 0xcc,0x82,0x00,0x10,0x08,0x01,0xff,0x4b,0xcc,0xa7,0x00,0x01,0xff,0x6b,0xcc,0xa7, + 0x00,0xd2,0x1c,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0x4c,0xcc,0x81,0x00,0x10, + 0x08,0x01,0xff,0x6c,0xcc,0x81,0x00,0x01,0xff,0x4c,0xcc,0xa7,0x00,0xd1,0x10,0x10, + 0x08,0x01,0xff,0x6c,0xcc,0xa7,0x00,0x01,0xff,0x4c,0xcc,0x8c,0x00,0x10,0x08,0x01, + 0xff,0x6c,0xcc,0x8c,0x00,0x01,0x00,0xcf,0x86,0xd5,0xd4,0xd4,0x60,0xd3,0x30,0xd2, + 0x10,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0x4e,0xcc,0x81,0x00,0xd1, + 0x10,0x10,0x08,0x01,0xff,0x6e,0xcc,0x81,0x00,0x01,0xff,0x4e,0xcc,0xa7,0x00,0x10, + 0x08,0x01,0xff,0x6e,0xcc,0xa7,0x00,0x01,0xff,0x4e,0xcc,0x8c,0x00,0xd2,0x10,0x91, + 0x0c,0x10,0x08,0x01,0xff,0x6e,0xcc,0x8c,0x00,0x01,0x00,0x01,0x00,0xd1,0x10,0x10, + 0x08,0x01,0xff,0x4f,0xcc,0x84,0x00,0x01,0xff,0x6f,0xcc,0x84,0x00,0x10,0x08,0x01, + 0xff,0x4f,0xcc,0x86,0x00,0x01,0xff,0x6f,0xcc,0x86,0x00,0xd3,0x34,0xd2,0x14,0x91, + 0x10,0x10,0x08,0x01,0xff,0x4f,0xcc,0x8b,0x00,0x01,0xff,0x6f,0xcc,0x8b,0x00,0x01, + 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x52,0xcc,0x81,0x00,0x01,0xff,0x72,0xcc,0x81, + 0x00,0x10,0x08,0x01,0xff,0x52,0xcc,0xa7,0x00,0x01,0xff,0x72,0xcc,0xa7,0x00,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x52,0xcc,0x8c,0x00,0x01,0xff,0x72,0xcc,0x8c, + 0x00,0x10,0x08,0x01,0xff,0x53,0xcc,0x81,0x00,0x01,0xff,0x73,0xcc,0x81,0x00,0xd1, + 0x10,0x10,0x08,0x01,0xff,0x53,0xcc,0x82,0x00,0x01,0xff,0x73,0xcc,0x82,0x00,0x10, + 0x08,0x01,0xff,0x53,0xcc,0xa7,0x00,0x01,0xff,0x73,0xcc,0xa7,0x00,0xd4,0x74,0xd3, + 0x34,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x53,0xcc,0x8c,0x00,0x01,0xff,0x73, + 0xcc,0x8c,0x00,0x10,0x08,0x01,0xff,0x54,0xcc,0xa7,0x00,0x01,0xff,0x74,0xcc,0xa7, + 0x00,0x91,0x10,0x10,0x08,0x01,0xff,0x54,0xcc,0x8c,0x00,0x01,0xff,0x74,0xcc,0x8c, + 0x00,0x01,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x55,0xcc,0x83,0x00,0x01, + 0xff,0x75,0xcc,0x83,0x00,0x10,0x08,0x01,0xff,0x55,0xcc,0x84,0x00,0x01,0xff,0x75, + 0xcc,0x84,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x55,0xcc,0x86,0x00,0x01,0xff,0x75, + 0xcc,0x86,0x00,0x10,0x08,0x01,0xff,0x55,0xcc,0x8a,0x00,0x01,0xff,0x75,0xcc,0x8a, + 0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x55,0xcc,0x8b,0x00,0x01, + 0xff,0x75,0xcc,0x8b,0x00,0x10,0x08,0x01,0xff,0x55,0xcc,0xa8,0x00,0x01,0xff,0x75, + 0xcc,0xa8,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x57,0xcc,0x82,0x00,0x01,0xff,0x77, + 0xcc,0x82,0x00,0x10,0x08,0x01,0xff,0x59,0xcc,0x82,0x00,0x01,0xff,0x79,0xcc,0x82, + 0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x59,0xcc,0x88,0x00,0x01,0xff,0x5a, + 0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x7a,0xcc,0x81,0x00,0x01,0xff,0x5a,0xcc,0x87, + 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x7a,0xcc,0x87,0x00,0x01,0xff,0x5a,0xcc,0x8c, + 0x00,0x10,0x08,0x01,0xff,0x7a,0xcc,0x8c,0x00,0x01,0x00,0xd0,0x4a,0xcf,0x86,0x55, + 0x04,0x01,0x00,0xd4,0x2c,0xd3,0x18,0x92,0x14,0x91,0x10,0x10,0x08,0x01,0xff,0x4f, + 0xcc,0x9b,0x00,0x01,0xff,0x6f,0xcc,0x9b,0x00,0x01,0x00,0x01,0x00,0x52,0x04,0x01, + 0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0x55,0xcc,0x9b,0x00,0x93, + 0x14,0x92,0x10,0x91,0x0c,0x10,0x08,0x01,0xff,0x75,0xcc,0x9b,0x00,0x01,0x00,0x01, + 0x00,0x01,0x00,0x01,0x00,0xcf,0x86,0xd5,0xb4,0xd4,0x24,0x53,0x04,0x01,0x00,0x52, + 0x04,0x01,0x00,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0x41,0xcc,0x8c,0x00,0x10, + 0x08,0x01,0xff,0x61,0xcc,0x8c,0x00,0x01,0xff,0x49,0xcc,0x8c,0x00,0xd3,0x46,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x69,0xcc,0x8c,0x00,0x01,0xff,0x4f,0xcc,0x8c, + 0x00,0x10,0x08,0x01,0xff,0x6f,0xcc,0x8c,0x00,0x01,0xff,0x55,0xcc,0x8c,0x00,0xd1, + 0x12,0x10,0x08,0x01,0xff,0x75,0xcc,0x8c,0x00,0x01,0xff,0x55,0xcc,0x88,0xcc,0x84, + 0x00,0x10,0x0a,0x01,0xff,0x75,0xcc,0x88,0xcc,0x84,0x00,0x01,0xff,0x55,0xcc,0x88, + 0xcc,0x81,0x00,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x75,0xcc,0x88,0xcc,0x81, + 0x00,0x01,0xff,0x55,0xcc,0x88,0xcc,0x8c,0x00,0x10,0x0a,0x01,0xff,0x75,0xcc,0x88, + 0xcc,0x8c,0x00,0x01,0xff,0x55,0xcc,0x88,0xcc,0x80,0x00,0xd1,0x0e,0x10,0x0a,0x01, + 0xff,0x75,0xcc,0x88,0xcc,0x80,0x00,0x01,0x00,0x10,0x0a,0x01,0xff,0x41,0xcc,0x88, + 0xcc,0x84,0x00,0x01,0xff,0x61,0xcc,0x88,0xcc,0x84,0x00,0xd4,0x80,0xd3,0x3a,0xd2, + 0x26,0xd1,0x14,0x10,0x0a,0x01,0xff,0x41,0xcc,0x87,0xcc,0x84,0x00,0x01,0xff,0x61, + 0xcc,0x87,0xcc,0x84,0x00,0x10,0x09,0x01,0xff,0xc3,0x86,0xcc,0x84,0x00,0x01,0xff, + 0xc3,0xa6,0xcc,0x84,0x00,0x51,0x04,0x01,0x00,0x10,0x08,0x01,0xff,0x47,0xcc,0x8c, + 0x00,0x01,0xff,0x67,0xcc,0x8c,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x4b, + 0xcc,0x8c,0x00,0x01,0xff,0x6b,0xcc,0x8c,0x00,0x10,0x08,0x01,0xff,0x4f,0xcc,0xa8, + 0x00,0x01,0xff,0x6f,0xcc,0xa8,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x4f,0xcc,0xa8, + 0xcc,0x84,0x00,0x01,0xff,0x6f,0xcc,0xa8,0xcc,0x84,0x00,0x10,0x09,0x01,0xff,0xc6, + 0xb7,0xcc,0x8c,0x00,0x01,0xff,0xca,0x92,0xcc,0x8c,0x00,0xd3,0x24,0xd2,0x10,0x91, + 0x0c,0x10,0x08,0x01,0xff,0x6a,0xcc,0x8c,0x00,0x01,0x00,0x01,0x00,0x91,0x10,0x10, + 0x08,0x01,0xff,0x47,0xcc,0x81,0x00,0x01,0xff,0x67,0xcc,0x81,0x00,0x04,0x00,0xd2, + 0x24,0xd1,0x10,0x10,0x08,0x04,0xff,0x4e,0xcc,0x80,0x00,0x04,0xff,0x6e,0xcc,0x80, + 0x00,0x10,0x0a,0x01,0xff,0x41,0xcc,0x8a,0xcc,0x81,0x00,0x01,0xff,0x61,0xcc,0x8a, + 0xcc,0x81,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xc3,0x86,0xcc,0x81,0x00,0x01,0xff, + 0xc3,0xa6,0xcc,0x81,0x00,0x10,0x09,0x01,0xff,0xc3,0x98,0xcc,0x81,0x00,0x01,0xff, + 0xc3,0xb8,0xcc,0x81,0x00,0xe2,0x07,0x02,0xe1,0xae,0x01,0xe0,0x93,0x01,0xcf,0x86, + 0xd5,0xf4,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x41,0xcc, + 0x8f,0x00,0x01,0xff,0x61,0xcc,0x8f,0x00,0x10,0x08,0x01,0xff,0x41,0xcc,0x91,0x00, + 0x01,0xff,0x61,0xcc,0x91,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x45,0xcc,0x8f,0x00, + 0x01,0xff,0x65,0xcc,0x8f,0x00,0x10,0x08,0x01,0xff,0x45,0xcc,0x91,0x00,0x01,0xff, + 0x65,0xcc,0x91,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x49,0xcc,0x8f,0x00, + 0x01,0xff,0x69,0xcc,0x8f,0x00,0x10,0x08,0x01,0xff,0x49,0xcc,0x91,0x00,0x01,0xff, + 0x69,0xcc,0x91,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x4f,0xcc,0x8f,0x00,0x01,0xff, + 0x6f,0xcc,0x8f,0x00,0x10,0x08,0x01,0xff,0x4f,0xcc,0x91,0x00,0x01,0xff,0x6f,0xcc, + 0x91,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x52,0xcc,0x8f,0x00, + 0x01,0xff,0x72,0xcc,0x8f,0x00,0x10,0x08,0x01,0xff,0x52,0xcc,0x91,0x00,0x01,0xff, + 0x72,0xcc,0x91,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x55,0xcc,0x8f,0x00,0x01,0xff, + 0x75,0xcc,0x8f,0x00,0x10,0x08,0x01,0xff,0x55,0xcc,0x91,0x00,0x01,0xff,0x75,0xcc, + 0x91,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x04,0xff,0x53,0xcc,0xa6,0x00,0x04,0xff, + 0x73,0xcc,0xa6,0x00,0x10,0x08,0x04,0xff,0x54,0xcc,0xa6,0x00,0x04,0xff,0x74,0xcc, + 0xa6,0x00,0x51,0x04,0x04,0x00,0x10,0x08,0x04,0xff,0x48,0xcc,0x8c,0x00,0x04,0xff, + 0x68,0xcc,0x8c,0x00,0xd4,0x68,0xd3,0x20,0xd2,0x0c,0x91,0x08,0x10,0x04,0x06,0x00, + 0x07,0x00,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x08,0x04,0xff,0x41,0xcc,0x87,0x00, + 0x04,0xff,0x61,0xcc,0x87,0x00,0xd2,0x24,0xd1,0x10,0x10,0x08,0x04,0xff,0x45,0xcc, + 0xa7,0x00,0x04,0xff,0x65,0xcc,0xa7,0x00,0x10,0x0a,0x04,0xff,0x4f,0xcc,0x88,0xcc, + 0x84,0x00,0x04,0xff,0x6f,0xcc,0x88,0xcc,0x84,0x00,0xd1,0x14,0x10,0x0a,0x04,0xff, + 0x4f,0xcc,0x83,0xcc,0x84,0x00,0x04,0xff,0x6f,0xcc,0x83,0xcc,0x84,0x00,0x10,0x08, + 0x04,0xff,0x4f,0xcc,0x87,0x00,0x04,0xff,0x6f,0xcc,0x87,0x00,0x93,0x30,0xd2,0x24, + 0xd1,0x14,0x10,0x0a,0x04,0xff,0x4f,0xcc,0x87,0xcc,0x84,0x00,0x04,0xff,0x6f,0xcc, + 0x87,0xcc,0x84,0x00,0x10,0x08,0x04,0xff,0x59,0xcc,0x84,0x00,0x04,0xff,0x79,0xcc, + 0x84,0x00,0x51,0x04,0x07,0x00,0x10,0x04,0x07,0x00,0x08,0x00,0x08,0x00,0xcf,0x86, + 0x95,0x14,0x94,0x10,0x93,0x0c,0x92,0x08,0x11,0x04,0x08,0x00,0x09,0x00,0x09,0x00, + 0x09,0x00,0x01,0x00,0x01,0x00,0xd0,0x22,0xcf,0x86,0x55,0x04,0x01,0x00,0x94,0x18, + 0x53,0x04,0x01,0x00,0xd2,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x04,0x00,0x04,0x00, + 0x11,0x04,0x04,0x00,0x07,0x00,0x01,0x00,0xcf,0x86,0xd5,0x18,0x54,0x04,0x01,0x00, + 0x53,0x04,0x01,0x00,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00, + 0x04,0x00,0x94,0x18,0x53,0x04,0x01,0x00,0xd2,0x08,0x11,0x04,0x01,0x00,0x04,0x00, + 0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x07,0x00,0x07,0x00,0xe1,0x35,0x01,0xd0, + 0x72,0xcf,0x86,0xd5,0x24,0x54,0x04,0x01,0xe6,0xd3,0x10,0x52,0x04,0x01,0xe6,0x91, + 0x08,0x10,0x04,0x01,0xe6,0x01,0xe8,0x01,0xdc,0x92,0x0c,0x51,0x04,0x01,0xdc,0x10, + 0x04,0x01,0xe8,0x01,0xd8,0x01,0xdc,0xd4,0x2c,0xd3,0x1c,0xd2,0x10,0xd1,0x08,0x10, + 0x04,0x01,0xdc,0x01,0xca,0x10,0x04,0x01,0xca,0x01,0xdc,0x51,0x04,0x01,0xdc,0x10, + 0x04,0x01,0xdc,0x01,0xca,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0xca,0x01,0xdc,0x01, + 0xdc,0x01,0xdc,0xd3,0x08,0x12,0x04,0x01,0xdc,0x01,0x01,0xd2,0x0c,0x91,0x08,0x10, + 0x04,0x01,0x01,0x01,0xdc,0x01,0xdc,0x91,0x08,0x10,0x04,0x01,0xdc,0x01,0xe6,0x01, + 0xe6,0xcf,0x86,0xd5,0x7f,0xd4,0x47,0xd3,0x2e,0xd2,0x19,0xd1,0x0e,0x10,0x07,0x01, + 0xff,0xcc,0x80,0x00,0x01,0xff,0xcc,0x81,0x00,0x10,0x04,0x01,0xe6,0x01,0xff,0xcc, + 0x93,0x00,0xd1,0x0d,0x10,0x09,0x01,0xff,0xcc,0x88,0xcc,0x81,0x00,0x01,0xf0,0x10, + 0x04,0x04,0xe6,0x04,0xdc,0xd2,0x08,0x11,0x04,0x04,0xdc,0x04,0xe6,0xd1,0x08,0x10, + 0x04,0x04,0xe6,0x04,0xdc,0x10,0x04,0x04,0xdc,0x06,0xff,0x00,0xd3,0x18,0xd2,0x0c, + 0x51,0x04,0x07,0xe6,0x10,0x04,0x07,0xe6,0x07,0xdc,0x51,0x04,0x07,0xdc,0x10,0x04, + 0x07,0xdc,0x07,0xe6,0xd2,0x10,0xd1,0x08,0x10,0x04,0x08,0xe8,0x08,0xdc,0x10,0x04, + 0x08,0xdc,0x08,0xe6,0xd1,0x08,0x10,0x04,0x08,0xe9,0x07,0xea,0x10,0x04,0x07,0xea, + 0x07,0xe9,0xd4,0x14,0x93,0x10,0x92,0x0c,0x51,0x04,0x01,0xea,0x10,0x04,0x04,0xe9, + 0x06,0xe6,0x06,0xe6,0x06,0xe6,0xd3,0x13,0x52,0x04,0x0a,0x00,0x91,0x0b,0x10,0x07, + 0x01,0xff,0xca,0xb9,0x00,0x01,0x00,0x0a,0x00,0xd2,0x0c,0x51,0x04,0x00,0x00,0x10, + 0x04,0x01,0x00,0x09,0x00,0x51,0x04,0x09,0x00,0x10,0x06,0x01,0xff,0x3b,0x00,0x10, + 0x00,0xd0,0xe1,0xcf,0x86,0xd5,0x7a,0xd4,0x5f,0xd3,0x21,0x52,0x04,0x00,0x00,0xd1, + 0x0d,0x10,0x04,0x01,0x00,0x01,0xff,0xc2,0xa8,0xcc,0x81,0x00,0x10,0x09,0x01,0xff, + 0xce,0x91,0xcc,0x81,0x00,0x01,0xff,0xc2,0xb7,0x00,0xd2,0x1f,0xd1,0x12,0x10,0x09, + 0x01,0xff,0xce,0x95,0xcc,0x81,0x00,0x01,0xff,0xce,0x97,0xcc,0x81,0x00,0x10,0x09, + 0x01,0xff,0xce,0x99,0xcc,0x81,0x00,0x00,0x00,0xd1,0x0d,0x10,0x09,0x01,0xff,0xce, + 0x9f,0xcc,0x81,0x00,0x00,0x00,0x10,0x09,0x01,0xff,0xce,0xa5,0xcc,0x81,0x00,0x01, + 0xff,0xce,0xa9,0xcc,0x81,0x00,0x93,0x17,0x92,0x13,0x91,0x0f,0x10,0x0b,0x01,0xff, + 0xce,0xb9,0xcc,0x88,0xcc,0x81,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4, + 0x4a,0xd3,0x10,0x92,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0x01, + 0x00,0xd2,0x16,0x51,0x04,0x01,0x00,0x10,0x09,0x01,0xff,0xce,0x99,0xcc,0x88,0x00, + 0x01,0xff,0xce,0xa5,0xcc,0x88,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb1,0xcc, + 0x81,0x00,0x01,0xff,0xce,0xb5,0xcc,0x81,0x00,0x10,0x09,0x01,0xff,0xce,0xb7,0xcc, + 0x81,0x00,0x01,0xff,0xce,0xb9,0xcc,0x81,0x00,0x93,0x17,0x92,0x13,0x91,0x0f,0x10, + 0x0b,0x01,0xff,0xcf,0x85,0xcc,0x88,0xcc,0x81,0x00,0x01,0x00,0x01,0x00,0x01,0x00, + 0x01,0x00,0xcf,0x86,0xd5,0x7b,0xd4,0x39,0x53,0x04,0x01,0x00,0xd2,0x16,0x51,0x04, + 0x01,0x00,0x10,0x09,0x01,0xff,0xce,0xb9,0xcc,0x88,0x00,0x01,0xff,0xcf,0x85,0xcc, + 0x88,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xbf,0xcc,0x81,0x00,0x01,0xff,0xcf, + 0x85,0xcc,0x81,0x00,0x10,0x09,0x01,0xff,0xcf,0x89,0xcc,0x81,0x00,0x0a,0x00,0xd3, + 0x26,0xd2,0x11,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0xcf,0x92,0xcc, + 0x81,0x00,0xd1,0x0d,0x10,0x09,0x01,0xff,0xcf,0x92,0xcc,0x88,0x00,0x01,0x00,0x10, + 0x04,0x01,0x00,0x04,0x00,0xd2,0x0c,0x51,0x04,0x06,0x00,0x10,0x04,0x01,0x00,0x04, + 0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x04,0x00,0x10,0x04,0x01,0x00,0x04,0x00,0xd4, + 0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x04,0x00,0x01,0x00,0x01, + 0x00,0x01,0x00,0xd3,0x10,0x52,0x04,0x01,0x00,0x51,0x04,0x05,0x00,0x10,0x04,0x06, + 0x00,0x07,0x00,0x12,0x04,0x07,0x00,0x08,0x00,0xe3,0x47,0x04,0xe2,0xbe,0x02,0xe1, + 0x07,0x01,0xd0,0x8b,0xcf,0x86,0xd5,0x6c,0xd4,0x53,0xd3,0x30,0xd2,0x1f,0xd1,0x12, + 0x10,0x09,0x04,0xff,0xd0,0x95,0xcc,0x80,0x00,0x01,0xff,0xd0,0x95,0xcc,0x88,0x00, + 0x10,0x04,0x01,0x00,0x01,0xff,0xd0,0x93,0xcc,0x81,0x00,0x51,0x04,0x01,0x00,0x10, + 0x04,0x01,0x00,0x01,0xff,0xd0,0x86,0xcc,0x88,0x00,0x52,0x04,0x01,0x00,0xd1,0x12, + 0x10,0x09,0x01,0xff,0xd0,0x9a,0xcc,0x81,0x00,0x04,0xff,0xd0,0x98,0xcc,0x80,0x00, + 0x10,0x09,0x01,0xff,0xd0,0xa3,0xcc,0x86,0x00,0x01,0x00,0x53,0x04,0x01,0x00,0x92, + 0x11,0x91,0x0d,0x10,0x04,0x01,0x00,0x01,0xff,0xd0,0x98,0xcc,0x86,0x00,0x01,0x00, + 0x01,0x00,0x54,0x04,0x01,0x00,0x53,0x04,0x01,0x00,0x92,0x11,0x91,0x0d,0x10,0x04, + 0x01,0x00,0x01,0xff,0xd0,0xb8,0xcc,0x86,0x00,0x01,0x00,0x01,0x00,0xcf,0x86,0xd5, + 0x57,0x54,0x04,0x01,0x00,0xd3,0x30,0xd2,0x1f,0xd1,0x12,0x10,0x09,0x04,0xff,0xd0, + 0xb5,0xcc,0x80,0x00,0x01,0xff,0xd0,0xb5,0xcc,0x88,0x00,0x10,0x04,0x01,0x00,0x01, + 0xff,0xd0,0xb3,0xcc,0x81,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xff, + 0xd1,0x96,0xcc,0x88,0x00,0x52,0x04,0x01,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xd0, + 0xba,0xcc,0x81,0x00,0x04,0xff,0xd0,0xb8,0xcc,0x80,0x00,0x10,0x09,0x01,0xff,0xd1, + 0x83,0xcc,0x86,0x00,0x01,0x00,0x54,0x04,0x01,0x00,0x93,0x1a,0x52,0x04,0x01,0x00, + 0x51,0x04,0x01,0x00,0x10,0x09,0x01,0xff,0xd1,0xb4,0xcc,0x8f,0x00,0x01,0xff,0xd1, + 0xb5,0xcc,0x8f,0x00,0x01,0x00,0xd0,0x2e,0xcf,0x86,0x95,0x28,0x94,0x24,0xd3,0x18, + 0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xe6,0x51,0x04,0x01,0xe6, + 0x10,0x04,0x01,0xe6,0x0a,0xe6,0x92,0x08,0x11,0x04,0x04,0x00,0x06,0x00,0x04,0x00, + 0x01,0x00,0x01,0x00,0xcf,0x86,0xd5,0xbe,0xd4,0x4a,0xd3,0x2a,0xd2,0x1a,0xd1,0x0d, + 0x10,0x04,0x01,0x00,0x01,0xff,0xd0,0x96,0xcc,0x86,0x00,0x10,0x09,0x01,0xff,0xd0, + 0xb6,0xcc,0x86,0x00,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x06,0x00,0x10,0x04, + 0x06,0x00,0x01,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x01,0x00,0x06,0x00,0x10,0x04, + 0x06,0x00,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x06,0x00,0x10,0x04,0x06,0x00, + 0x09,0x00,0xd3,0x3a,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xd0,0x90,0xcc,0x86, + 0x00,0x01,0xff,0xd0,0xb0,0xcc,0x86,0x00,0x10,0x09,0x01,0xff,0xd0,0x90,0xcc,0x88, + 0x00,0x01,0xff,0xd0,0xb0,0xcc,0x88,0x00,0x51,0x04,0x01,0x00,0x10,0x09,0x01,0xff, + 0xd0,0x95,0xcc,0x86,0x00,0x01,0xff,0xd0,0xb5,0xcc,0x86,0x00,0xd2,0x16,0x51,0x04, + 0x01,0x00,0x10,0x09,0x01,0xff,0xd3,0x98,0xcc,0x88,0x00,0x01,0xff,0xd3,0x99,0xcc, + 0x88,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xd0,0x96,0xcc,0x88,0x00,0x01,0xff,0xd0, + 0xb6,0xcc,0x88,0x00,0x10,0x09,0x01,0xff,0xd0,0x97,0xcc,0x88,0x00,0x01,0xff,0xd0, + 0xb7,0xcc,0x88,0x00,0xd4,0x74,0xd3,0x3a,0xd2,0x16,0x51,0x04,0x01,0x00,0x10,0x09, + 0x01,0xff,0xd0,0x98,0xcc,0x84,0x00,0x01,0xff,0xd0,0xb8,0xcc,0x84,0x00,0xd1,0x12, + 0x10,0x09,0x01,0xff,0xd0,0x98,0xcc,0x88,0x00,0x01,0xff,0xd0,0xb8,0xcc,0x88,0x00, + 0x10,0x09,0x01,0xff,0xd0,0x9e,0xcc,0x88,0x00,0x01,0xff,0xd0,0xbe,0xcc,0x88,0x00, + 0xd2,0x16,0x51,0x04,0x01,0x00,0x10,0x09,0x01,0xff,0xd3,0xa8,0xcc,0x88,0x00,0x01, + 0xff,0xd3,0xa9,0xcc,0x88,0x00,0xd1,0x12,0x10,0x09,0x04,0xff,0xd0,0xad,0xcc,0x88, + 0x00,0x04,0xff,0xd1,0x8d,0xcc,0x88,0x00,0x10,0x09,0x01,0xff,0xd0,0xa3,0xcc,0x84, + 0x00,0x01,0xff,0xd1,0x83,0xcc,0x84,0x00,0xd3,0x3a,0xd2,0x24,0xd1,0x12,0x10,0x09, + 0x01,0xff,0xd0,0xa3,0xcc,0x88,0x00,0x01,0xff,0xd1,0x83,0xcc,0x88,0x00,0x10,0x09, + 0x01,0xff,0xd0,0xa3,0xcc,0x8b,0x00,0x01,0xff,0xd1,0x83,0xcc,0x8b,0x00,0x91,0x12, + 0x10,0x09,0x01,0xff,0xd0,0xa7,0xcc,0x88,0x00,0x01,0xff,0xd1,0x87,0xcc,0x88,0x00, + 0x08,0x00,0x92,0x16,0x91,0x12,0x10,0x09,0x01,0xff,0xd0,0xab,0xcc,0x88,0x00,0x01, + 0xff,0xd1,0x8b,0xcc,0x88,0x00,0x09,0x00,0x09,0x00,0xd1,0x74,0xd0,0x36,0xcf,0x86, + 0xd5,0x10,0x54,0x04,0x06,0x00,0x93,0x08,0x12,0x04,0x09,0x00,0x0a,0x00,0x0a,0x00, + 0xd4,0x10,0x93,0x0c,0x52,0x04,0x0a,0x00,0x11,0x04,0x0b,0x00,0x0c,0x00,0x10,0x00, + 0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00, + 0x01,0x00,0xcf,0x86,0xd5,0x24,0x54,0x04,0x01,0x00,0xd3,0x10,0x52,0x04,0x01,0x00, + 0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x92,0x0c,0x91,0x08,0x10,0x04, + 0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08, + 0x10,0x04,0x14,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd0,0xba, + 0xcf,0x86,0xd5,0x4c,0xd4,0x24,0x53,0x04,0x01,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04, + 0x14,0x00,0x01,0x00,0x10,0x04,0x04,0x00,0x00,0x00,0xd1,0x08,0x10,0x04,0x00,0x00, + 0x10,0x00,0x10,0x04,0x10,0x00,0x0d,0x00,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04, + 0x00,0x00,0x02,0xdc,0x02,0xe6,0x51,0x04,0x02,0xe6,0x10,0x04,0x02,0xdc,0x02,0xe6, + 0x92,0x0c,0x51,0x04,0x02,0xe6,0x10,0x04,0x02,0xde,0x02,0xdc,0x02,0xe6,0xd4,0x2c, + 0xd3,0x10,0x92,0x0c,0x51,0x04,0x02,0xe6,0x10,0x04,0x08,0xdc,0x02,0xdc,0x02,0xdc, + 0xd2,0x0c,0x51,0x04,0x02,0xe6,0x10,0x04,0x02,0xdc,0x02,0xe6,0xd1,0x08,0x10,0x04, + 0x02,0xe6,0x02,0xde,0x10,0x04,0x02,0xe4,0x02,0xe6,0xd3,0x20,0xd2,0x10,0xd1,0x08, + 0x10,0x04,0x01,0x0a,0x01,0x0b,0x10,0x04,0x01,0x0c,0x01,0x0d,0xd1,0x08,0x10,0x04, + 0x01,0x0e,0x01,0x0f,0x10,0x04,0x01,0x10,0x01,0x11,0xd2,0x10,0xd1,0x08,0x10,0x04, + 0x01,0x12,0x01,0x13,0x10,0x04,0x09,0x13,0x01,0x14,0xd1,0x08,0x10,0x04,0x01,0x15, + 0x01,0x16,0x10,0x04,0x01,0x00,0x01,0x17,0xcf,0x86,0xd5,0x28,0x94,0x24,0x93,0x20, + 0xd2,0x10,0xd1,0x08,0x10,0x04,0x01,0x00,0x01,0x18,0x10,0x04,0x01,0x19,0x01,0x00, + 0xd1,0x08,0x10,0x04,0x02,0xe6,0x08,0xdc,0x10,0x04,0x08,0x00,0x08,0x12,0x00,0x00, + 0x01,0x00,0xd4,0x1c,0x53,0x04,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04, + 0x01,0x00,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x14,0x00,0x93,0x10, + 0x52,0x04,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0xe2,0xfb,0x01,0xe1,0x2b,0x01,0xd0,0xa8,0xcf,0x86,0xd5,0x55,0xd4,0x28,0xd3,0x10, + 0x52,0x04,0x07,0x00,0x91,0x08,0x10,0x04,0x0d,0x00,0x10,0x00,0x0a,0x00,0xd2,0x0c, + 0x51,0x04,0x0a,0x00,0x10,0x04,0x0a,0x00,0x08,0x00,0x91,0x08,0x10,0x04,0x01,0x00, + 0x07,0x00,0x07,0x00,0xd3,0x0c,0x52,0x04,0x07,0xe6,0x11,0x04,0x07,0xe6,0x0a,0xe6, + 0xd2,0x10,0xd1,0x08,0x10,0x04,0x0a,0x1e,0x0a,0x1f,0x10,0x04,0x0a,0x20,0x01,0x00, + 0xd1,0x09,0x10,0x05,0x0f,0xff,0x00,0x00,0x00,0x10,0x04,0x08,0x00,0x01,0x00,0xd4, + 0x3d,0x93,0x39,0xd2,0x1a,0xd1,0x08,0x10,0x04,0x0c,0x00,0x01,0x00,0x10,0x09,0x01, + 0xff,0xd8,0xa7,0xd9,0x93,0x00,0x01,0xff,0xd8,0xa7,0xd9,0x94,0x00,0xd1,0x12,0x10, + 0x09,0x01,0xff,0xd9,0x88,0xd9,0x94,0x00,0x01,0xff,0xd8,0xa7,0xd9,0x95,0x00,0x10, + 0x09,0x01,0xff,0xd9,0x8a,0xd9,0x94,0x00,0x01,0x00,0x01,0x00,0x53,0x04,0x01,0x00, + 0x92,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x0a,0x00,0x0a,0x00,0xcf,0x86, + 0xd5,0x5c,0xd4,0x20,0x53,0x04,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04, + 0x01,0x00,0x01,0x1b,0xd1,0x08,0x10,0x04,0x01,0x1c,0x01,0x1d,0x10,0x04,0x01,0x1e, + 0x01,0x1f,0xd3,0x20,0xd2,0x10,0xd1,0x08,0x10,0x04,0x01,0x20,0x01,0x21,0x10,0x04, + 0x01,0x22,0x04,0xe6,0xd1,0x08,0x10,0x04,0x04,0xe6,0x04,0xdc,0x10,0x04,0x07,0xdc, + 0x07,0xe6,0xd2,0x0c,0x91,0x08,0x10,0x04,0x07,0xe6,0x08,0xe6,0x08,0xe6,0xd1,0x08, + 0x10,0x04,0x08,0xdc,0x08,0xe6,0x10,0x04,0x08,0xe6,0x0c,0xdc,0xd4,0x10,0x53,0x04, + 0x01,0x00,0x52,0x04,0x01,0x00,0x11,0x04,0x01,0x00,0x06,0x00,0x93,0x10,0x92,0x0c, + 0x91,0x08,0x10,0x04,0x01,0x23,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd0,0x22, + 0xcf,0x86,0x55,0x04,0x01,0x00,0x54,0x04,0x01,0x00,0x53,0x04,0x01,0x00,0xd2,0x08, + 0x11,0x04,0x04,0x00,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x04,0x00, + 0xcf,0x86,0xd5,0x5b,0xd4,0x2e,0xd3,0x1e,0x92,0x1a,0xd1,0x0d,0x10,0x09,0x01,0xff, + 0xdb,0x95,0xd9,0x94,0x00,0x01,0x00,0x10,0x09,0x01,0xff,0xdb,0x81,0xd9,0x94,0x00, + 0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00, + 0x04,0x00,0xd3,0x19,0xd2,0x11,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xff, + 0xdb,0x92,0xd9,0x94,0x00,0x11,0x04,0x01,0x00,0x01,0xe6,0x52,0x04,0x01,0xe6,0xd1, + 0x08,0x10,0x04,0x01,0xe6,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xe6,0xd4,0x38,0xd3, + 0x1c,0xd2,0x0c,0x51,0x04,0x01,0xe6,0x10,0x04,0x01,0xe6,0x01,0xdc,0xd1,0x08,0x10, + 0x04,0x01,0xe6,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xe6,0xd2,0x10,0xd1,0x08,0x10, + 0x04,0x01,0xe6,0x01,0x00,0x10,0x04,0x01,0xdc,0x01,0xe6,0x91,0x08,0x10,0x04,0x01, + 0xe6,0x01,0xdc,0x07,0x00,0x53,0x04,0x01,0x00,0xd2,0x08,0x11,0x04,0x01,0x00,0x04, + 0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x07,0x00,0xd1,0xc8,0xd0,0x76,0xcf, + 0x86,0xd5,0x28,0xd4,0x14,0x53,0x04,0x04,0x00,0x52,0x04,0x04,0x00,0x51,0x04,0x04, + 0x00,0x10,0x04,0x00,0x00,0x04,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x04, + 0x00,0x04,0x24,0x04,0x00,0x04,0x00,0x04,0x00,0xd4,0x14,0x53,0x04,0x04,0x00,0x52, + 0x04,0x04,0x00,0x91,0x08,0x10,0x04,0x04,0x00,0x07,0x00,0x07,0x00,0xd3,0x1c,0xd2, + 0x0c,0x91,0x08,0x10,0x04,0x04,0xe6,0x04,0xdc,0x04,0xe6,0xd1,0x08,0x10,0x04,0x04, + 0xdc,0x04,0xe6,0x10,0x04,0x04,0xe6,0x04,0xdc,0xd2,0x0c,0x51,0x04,0x04,0xdc,0x10, + 0x04,0x04,0xe6,0x04,0xdc,0xd1,0x08,0x10,0x04,0x04,0xdc,0x04,0xe6,0x10,0x04,0x04, + 0xdc,0x04,0xe6,0xcf,0x86,0xd5,0x3c,0x94,0x38,0xd3,0x1c,0xd2,0x0c,0x51,0x04,0x04, + 0xe6,0x10,0x04,0x04,0xdc,0x04,0xe6,0xd1,0x08,0x10,0x04,0x04,0xdc,0x04,0xe6,0x10, + 0x04,0x04,0xdc,0x04,0xe6,0xd2,0x10,0xd1,0x08,0x10,0x04,0x04,0xdc,0x04,0xe6,0x10, + 0x04,0x04,0xe6,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x07,0x00,0x07,0x00,0x08, + 0x00,0x94,0x10,0x53,0x04,0x08,0x00,0x52,0x04,0x08,0x00,0x11,0x04,0x08,0x00,0x0a, + 0x00,0x0a,0x00,0xd0,0x1e,0xcf,0x86,0x55,0x04,0x04,0x00,0x54,0x04,0x04,0x00,0x93, + 0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x04,0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0xcf,0x86,0x55,0x04,0x09,0x00,0xd4,0x14,0x53,0x04,0x09,0x00,0x92,0x0c,0x51, + 0x04,0x09,0x00,0x10,0x04,0x09,0x00,0x09,0xe6,0x09,0xe6,0xd3,0x10,0x92,0x0c,0x51, + 0x04,0x09,0xe6,0x10,0x04,0x09,0xdc,0x09,0xe6,0x09,0x00,0xd2,0x0c,0x51,0x04,0x09, + 0x00,0x10,0x04,0x09,0x00,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x14,0xdc,0x14, + 0x00,0xe4,0xf8,0x57,0xe3,0x45,0x3f,0xe2,0xf4,0x3e,0xe1,0xc7,0x2c,0xe0,0x21,0x10, + 0xcf,0x86,0xc5,0xe4,0x80,0x08,0xe3,0xcb,0x03,0xe2,0x61,0x01,0xd1,0x94,0xd0,0x5a, + 0xcf,0x86,0xd5,0x20,0x54,0x04,0x0b,0x00,0xd3,0x0c,0x52,0x04,0x0b,0x00,0x11,0x04, + 0x0b,0x00,0x0b,0xe6,0x92,0x0c,0x51,0x04,0x0b,0xe6,0x10,0x04,0x0b,0x00,0x0b,0xe6, + 0x0b,0xe6,0xd4,0x24,0xd3,0x10,0x52,0x04,0x0b,0xe6,0x91,0x08,0x10,0x04,0x0b,0x00, + 0x0b,0xe6,0x0b,0xe6,0xd2,0x0c,0x91,0x08,0x10,0x04,0x0b,0x00,0x0b,0xe6,0x0b,0xe6, + 0x11,0x04,0x0b,0xe6,0x00,0x00,0x53,0x04,0x0b,0x00,0x52,0x04,0x0b,0x00,0x51,0x04, + 0x0b,0x00,0x10,0x04,0x0b,0x00,0x00,0x00,0xcf,0x86,0xd5,0x20,0x54,0x04,0x0c,0x00, + 0x53,0x04,0x0c,0x00,0xd2,0x0c,0x91,0x08,0x10,0x04,0x0c,0x00,0x0c,0xdc,0x0c,0xdc, + 0x51,0x04,0x00,0x00,0x10,0x04,0x0c,0x00,0x00,0x00,0x94,0x14,0x53,0x04,0x13,0x00, + 0x92,0x0c,0x51,0x04,0x13,0x00,0x10,0x04,0x13,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0xd0,0x4a,0xcf,0x86,0x55,0x04,0x00,0x00,0xd4,0x20,0xd3,0x10,0x92,0x0c,0x91,0x08, + 0x10,0x04,0x0d,0x00,0x10,0x00,0x0d,0x00,0x0d,0x00,0x52,0x04,0x0d,0x00,0x91,0x08, + 0x10,0x04,0x0d,0x00,0x10,0x00,0x10,0x00,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x10,0x00, + 0x10,0x04,0x10,0x00,0x11,0x00,0x91,0x08,0x10,0x04,0x11,0x00,0x00,0x00,0x12,0x00, + 0x52,0x04,0x12,0x00,0x11,0x04,0x12,0x00,0x00,0x00,0xcf,0x86,0xd5,0x18,0x54,0x04, + 0x00,0x00,0x93,0x10,0x92,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x14,0xdc, + 0x12,0xe6,0x12,0xe6,0xd4,0x30,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x12,0xe6,0x10,0x04, + 0x12,0x00,0x11,0xdc,0x51,0x04,0x0d,0xe6,0x10,0x04,0x0d,0xdc,0x0d,0xe6,0xd2,0x0c, + 0x91,0x08,0x10,0x04,0x0d,0xe6,0x0d,0xdc,0x0d,0xe6,0x91,0x08,0x10,0x04,0x0d,0xe6, + 0x0d,0xdc,0x0d,0xdc,0xd3,0x1c,0xd2,0x10,0xd1,0x08,0x10,0x04,0x0d,0x1b,0x0d,0x1c, + 0x10,0x04,0x0d,0x1d,0x0d,0xe6,0x51,0x04,0x0d,0xe6,0x10,0x04,0x0d,0xdc,0x0d,0xe6, + 0xd2,0x10,0xd1,0x08,0x10,0x04,0x0d,0xe6,0x0d,0xdc,0x10,0x04,0x0d,0xdc,0x0d,0xe6, + 0x51,0x04,0x0d,0xe6,0x10,0x04,0x0d,0xe6,0x10,0xe6,0xe1,0x3a,0x01,0xd0,0x77,0xcf, + 0x86,0xd5,0x20,0x94,0x1c,0x93,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x0b,0x00,0x01, + 0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x07,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01, + 0x00,0xd4,0x1b,0x53,0x04,0x01,0x00,0x92,0x13,0x91,0x0f,0x10,0x04,0x01,0x00,0x01, + 0xff,0xe0,0xa4,0xa8,0xe0,0xa4,0xbc,0x00,0x01,0x00,0x01,0x00,0xd3,0x26,0xd2,0x13, + 0x91,0x0f,0x10,0x04,0x01,0x00,0x01,0xff,0xe0,0xa4,0xb0,0xe0,0xa4,0xbc,0x00,0x01, + 0x00,0x91,0x0f,0x10,0x0b,0x01,0xff,0xe0,0xa4,0xb3,0xe0,0xa4,0xbc,0x00,0x01,0x00, + 0x01,0x00,0xd2,0x08,0x11,0x04,0x01,0x00,0x0c,0x00,0x91,0x08,0x10,0x04,0x01,0x07, + 0x01,0x00,0x01,0x00,0xcf,0x86,0xd5,0x8c,0xd4,0x18,0x53,0x04,0x01,0x00,0x52,0x04, + 0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x01,0x09,0x10,0x04,0x0b,0x00,0x0c,0x00, + 0xd3,0x1c,0xd2,0x10,0xd1,0x08,0x10,0x04,0x01,0x00,0x01,0xe6,0x10,0x04,0x01,0xdc, + 0x01,0xe6,0x91,0x08,0x10,0x04,0x01,0xe6,0x0b,0x00,0x0c,0x00,0xd2,0x2c,0xd1,0x16, + 0x10,0x0b,0x01,0xff,0xe0,0xa4,0x95,0xe0,0xa4,0xbc,0x00,0x01,0xff,0xe0,0xa4,0x96, + 0xe0,0xa4,0xbc,0x00,0x10,0x0b,0x01,0xff,0xe0,0xa4,0x97,0xe0,0xa4,0xbc,0x00,0x01, + 0xff,0xe0,0xa4,0x9c,0xe0,0xa4,0xbc,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xe0,0xa4, + 0xa1,0xe0,0xa4,0xbc,0x00,0x01,0xff,0xe0,0xa4,0xa2,0xe0,0xa4,0xbc,0x00,0x10,0x0b, + 0x01,0xff,0xe0,0xa4,0xab,0xe0,0xa4,0xbc,0x00,0x01,0xff,0xe0,0xa4,0xaf,0xe0,0xa4, + 0xbc,0x00,0x54,0x04,0x01,0x00,0xd3,0x14,0x92,0x10,0xd1,0x08,0x10,0x04,0x01,0x00, + 0x0a,0x00,0x10,0x04,0x0a,0x00,0x0c,0x00,0x0c,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04, + 0x10,0x00,0x0b,0x00,0x10,0x04,0x0b,0x00,0x09,0x00,0x91,0x08,0x10,0x04,0x09,0x00, + 0x08,0x00,0x09,0x00,0xd0,0x86,0xcf,0x86,0xd5,0x44,0xd4,0x2c,0xd3,0x18,0xd2,0x0c, + 0x91,0x08,0x10,0x04,0x10,0x00,0x01,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x00,0x00, + 0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00, + 0x10,0x04,0x00,0x00,0x01,0x00,0x93,0x14,0x92,0x10,0xd1,0x08,0x10,0x04,0x01,0x00, + 0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4,0x14,0x53,0x04, + 0x01,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00, + 0xd3,0x18,0xd2,0x10,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x01,0x00, + 0x00,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0xd2,0x08,0x11,0x04,0x01,0x00,0x00,0x00, + 0x91,0x08,0x10,0x04,0x01,0x07,0x07,0x00,0x01,0x00,0xcf,0x86,0xd5,0x7b,0xd4,0x42, + 0xd3,0x14,0x52,0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04, + 0x00,0x00,0x01,0x00,0xd2,0x17,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04, + 0x00,0x00,0x01,0xff,0xe0,0xa7,0x87,0xe0,0xa6,0xbe,0x00,0xd1,0x0f,0x10,0x0b,0x01, + 0xff,0xe0,0xa7,0x87,0xe0,0xa7,0x97,0x00,0x01,0x09,0x10,0x04,0x08,0x00,0x00,0x00, + 0xd3,0x10,0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00, + 0x52,0x04,0x00,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xe0,0xa6,0xa1,0xe0,0xa6,0xbc, + 0x00,0x01,0xff,0xe0,0xa6,0xa2,0xe0,0xa6,0xbc,0x00,0x10,0x04,0x00,0x00,0x01,0xff, + 0xe0,0xa6,0xaf,0xe0,0xa6,0xbc,0x00,0xd4,0x10,0x93,0x0c,0x52,0x04,0x01,0x00,0x11, + 0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x53,0x04,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01, + 0x00,0x10,0x04,0x01,0x00,0x0b,0x00,0x51,0x04,0x13,0x00,0x10,0x04,0x14,0xe6,0x00, + 0x00,0xe2,0x48,0x02,0xe1,0x4f,0x01,0xd0,0xa4,0xcf,0x86,0xd5,0x4c,0xd4,0x34,0xd3, + 0x1c,0xd2,0x10,0xd1,0x08,0x10,0x04,0x00,0x00,0x07,0x00,0x10,0x04,0x01,0x00,0x07, + 0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01, + 0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x01, + 0x00,0x93,0x14,0x92,0x10,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x00, + 0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4,0x14,0x53,0x04,0x01,0x00,0x92,0x0c,0x91, + 0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0xd3,0x2e,0xd2,0x17,0xd1, + 0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0xe0,0xa8,0xb2, + 0xe0,0xa8,0xbc,0x00,0xd1,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x10,0x0b,0x01,0xff, + 0xe0,0xa8,0xb8,0xe0,0xa8,0xbc,0x00,0x00,0x00,0xd2,0x08,0x11,0x04,0x01,0x00,0x00, + 0x00,0x91,0x08,0x10,0x04,0x01,0x07,0x00,0x00,0x01,0x00,0xcf,0x86,0xd5,0x80,0xd4, + 0x34,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x51, + 0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x01, + 0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x01, + 0x09,0x00,0x00,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x0a,0x00,0x00, + 0x00,0x00,0x00,0xd2,0x25,0xd1,0x0f,0x10,0x04,0x00,0x00,0x01,0xff,0xe0,0xa8,0x96, + 0xe0,0xa8,0xbc,0x00,0x10,0x0b,0x01,0xff,0xe0,0xa8,0x97,0xe0,0xa8,0xbc,0x00,0x01, + 0xff,0xe0,0xa8,0x9c,0xe0,0xa8,0xbc,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00, + 0x10,0x0b,0x01,0xff,0xe0,0xa8,0xab,0xe0,0xa8,0xbc,0x00,0x00,0x00,0xd4,0x10,0x93, + 0x0c,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x93,0x14,0x52, + 0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x0a,0x00,0x10,0x04,0x14,0x00,0x00, + 0x00,0x00,0x00,0xd0,0x82,0xcf,0x86,0xd5,0x40,0xd4,0x2c,0xd3,0x18,0xd2,0x0c,0x91, + 0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x01, + 0x00,0x01,0x00,0x52,0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x07,0x00,0x01,0x00,0x10, + 0x04,0x00,0x00,0x01,0x00,0x93,0x10,0x92,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x00, + 0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4,0x14,0x53,0x04,0x01,0x00,0x92,0x0c,0x91, + 0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0xd3,0x18,0xd2,0x0c,0x91, + 0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x01, + 0x00,0x01,0x00,0xd2,0x08,0x11,0x04,0x01,0x00,0x00,0x00,0x91,0x08,0x10,0x04,0x01, + 0x07,0x01,0x00,0x01,0x00,0xcf,0x86,0xd5,0x3c,0xd4,0x28,0xd3,0x10,0x52,0x04,0x01, + 0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01, + 0x00,0x10,0x04,0x00,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x01,0x09,0x00, + 0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0xd4,0x18,0x93,0x14,0xd2,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x07, + 0x00,0x07,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0xd3,0x10,0x92,0x0c,0x91, + 0x08,0x10,0x04,0x0d,0x00,0x07,0x00,0x00,0x00,0x00,0x00,0x92,0x0c,0x91,0x08,0x10, + 0x04,0x00,0x00,0x11,0x00,0x13,0x00,0x13,0x00,0xe1,0x24,0x01,0xd0,0x86,0xcf,0x86, + 0xd5,0x44,0xd4,0x2c,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00, + 0x01,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00, + 0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0x93,0x14, + 0x92,0x10,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00, + 0x01,0x00,0x01,0x00,0xd4,0x14,0x53,0x04,0x01,0x00,0x92,0x0c,0x91,0x08,0x10,0x04, + 0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04, + 0x01,0x00,0x00,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x07,0x00,0x01,0x00, + 0xd2,0x08,0x11,0x04,0x01,0x00,0x00,0x00,0x91,0x08,0x10,0x04,0x01,0x07,0x01,0x00, + 0x01,0x00,0xcf,0x86,0xd5,0x73,0xd4,0x45,0xd3,0x14,0x52,0x04,0x01,0x00,0xd1,0x08, + 0x10,0x04,0x0a,0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0xd2,0x1e,0xd1,0x0f, + 0x10,0x0b,0x01,0xff,0xe0,0xad,0x87,0xe0,0xad,0x96,0x00,0x00,0x00,0x10,0x04,0x00, + 0x00,0x01,0xff,0xe0,0xad,0x87,0xe0,0xac,0xbe,0x00,0x91,0x0f,0x10,0x0b,0x01,0xff, + 0xe0,0xad,0x87,0xe0,0xad,0x97,0x00,0x01,0x09,0x00,0x00,0xd3,0x0c,0x52,0x04,0x00, + 0x00,0x11,0x04,0x00,0x00,0x01,0x00,0x52,0x04,0x00,0x00,0xd1,0x16,0x10,0x0b,0x01, + 0xff,0xe0,0xac,0xa1,0xe0,0xac,0xbc,0x00,0x01,0xff,0xe0,0xac,0xa2,0xe0,0xac,0xbc, + 0x00,0x10,0x04,0x00,0x00,0x01,0x00,0xd4,0x14,0x93,0x10,0xd2,0x08,0x11,0x04,0x01, + 0x00,0x0a,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x93,0x10,0x92,0x0c,0x91, + 0x08,0x10,0x04,0x01,0x00,0x07,0x00,0x0c,0x00,0x0c,0x00,0x00,0x00,0xd0,0xb1,0xcf, + 0x86,0xd5,0x63,0xd4,0x28,0xd3,0x14,0xd2,0x08,0x11,0x04,0x00,0x00,0x01,0x00,0x91, + 0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10, + 0x04,0x01,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0xd3,0x1f,0xd2,0x0c,0x91, + 0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x91,0x0f,0x10,0x0b,0x01,0xff,0xe0, + 0xae,0x92,0xe0,0xaf,0x97,0x00,0x01,0x00,0x00,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04, + 0x00,0x00,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x91,0x08,0x10,0x04,0x01,0x00, + 0x00,0x00,0x01,0x00,0xd4,0x2c,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x00,0x00,0x10,0x04, + 0x00,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x00,0x00,0xd2,0x0c, + 0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x01,0x00, + 0xd3,0x10,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x08,0x00,0x01,0x00, + 0xd2,0x08,0x11,0x04,0x01,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0xcf,0x86, + 0xd5,0x61,0xd4,0x45,0xd3,0x14,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00, + 0x00,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0xd2,0x1e,0xd1,0x08,0x10,0x04,0x01,0x00, + 0x00,0x00,0x10,0x0b,0x01,0xff,0xe0,0xaf,0x86,0xe0,0xae,0xbe,0x00,0x01,0xff,0xe0, + 0xaf,0x87,0xe0,0xae,0xbe,0x00,0x91,0x0f,0x10,0x0b,0x01,0xff,0xe0,0xaf,0x86,0xe0, + 0xaf,0x97,0x00,0x01,0x09,0x00,0x00,0x93,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x0a, + 0x00,0x00,0x00,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0x00, + 0x00,0xd4,0x14,0x93,0x10,0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x08, + 0x00,0x01,0x00,0x01,0x00,0xd3,0x10,0x92,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01, + 0x00,0x07,0x00,0x07,0x00,0x92,0x0c,0x51,0x04,0x07,0x00,0x10,0x04,0x07,0x00,0x00, + 0x00,0x00,0x00,0xe3,0x1c,0x04,0xe2,0x1a,0x02,0xd1,0xf3,0xd0,0x76,0xcf,0x86,0xd5, + 0x3c,0xd4,0x28,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x10,0x00,0x01,0x00,0x01, + 0x00,0x91,0x08,0x10,0x04,0x14,0x00,0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00,0x91, + 0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10, + 0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4,0x14,0x53,0x04,0x01, + 0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0xd3, + 0x10,0x52,0x04,0x01,0x00,0x91,0x08,0x10,0x04,0x10,0x00,0x01,0x00,0x01,0x00,0xd2, + 0x08,0x11,0x04,0x01,0x00,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x0a,0x00,0x01, + 0x00,0xcf,0x86,0xd5,0x53,0xd4,0x2f,0xd3,0x10,0x52,0x04,0x01,0x00,0x91,0x08,0x10, + 0x04,0x01,0x00,0x00,0x00,0x01,0x00,0xd2,0x13,0x91,0x0f,0x10,0x0b,0x01,0xff,0xe0, + 0xb1,0x86,0xe0,0xb1,0x96,0x00,0x00,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00, + 0x01,0x09,0x00,0x00,0xd3,0x14,0x52,0x04,0x00,0x00,0xd1,0x08,0x10,0x04,0x00,0x00, + 0x01,0x54,0x10,0x04,0x01,0x5b,0x00,0x00,0x92,0x0c,0x51,0x04,0x0a,0x00,0x10,0x04, + 0x11,0x00,0x00,0x00,0x00,0x00,0xd4,0x14,0x93,0x10,0xd2,0x08,0x11,0x04,0x01,0x00, + 0x0a,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x93,0x10,0x52,0x04,0x00,0x00, + 0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x15,0x00,0x0a,0x00,0xd0,0x76,0xcf,0x86, + 0xd5,0x3c,0xd4,0x28,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x12,0x00,0x10,0x00, + 0x01,0x00,0x91,0x08,0x10,0x04,0x14,0x00,0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00, + 0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x93,0x10,0x92,0x0c,0x91,0x08, + 0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4,0x14,0x53,0x04, + 0x01,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00, + 0xd3,0x10,0x52,0x04,0x01,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00, + 0xd2,0x08,0x11,0x04,0x01,0x00,0x00,0x00,0x91,0x08,0x10,0x04,0x07,0x07,0x07,0x00, + 0x01,0x00,0xcf,0x86,0xd5,0x82,0xd4,0x5e,0xd3,0x2a,0xd2,0x13,0x91,0x0f,0x10,0x0b, + 0x01,0xff,0xe0,0xb2,0xbf,0xe0,0xb3,0x95,0x00,0x01,0x00,0x01,0x00,0xd1,0x08,0x10, + 0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0xe0,0xb3,0x86,0xe0,0xb3, + 0x95,0x00,0xd2,0x28,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe0,0xb3,0x86,0xe0,0xb3,0x96, + 0x00,0x00,0x00,0x10,0x0b,0x01,0xff,0xe0,0xb3,0x86,0xe0,0xb3,0x82,0x00,0x01,0xff, + 0xe0,0xb3,0x86,0xe0,0xb3,0x82,0xe0,0xb3,0x95,0x00,0x91,0x08,0x10,0x04,0x01,0x00, + 0x01,0x09,0x00,0x00,0xd3,0x14,0x52,0x04,0x00,0x00,0xd1,0x08,0x10,0x04,0x00,0x00, + 0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00, + 0x10,0x04,0x01,0x00,0x00,0x00,0xd4,0x14,0x93,0x10,0xd2,0x08,0x11,0x04,0x01,0x00, + 0x09,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x93,0x14,0x92,0x10,0xd1,0x08, + 0x10,0x04,0x00,0x00,0x09,0x00,0x10,0x04,0x09,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0xe1,0x06,0x01,0xd0,0x6e,0xcf,0x86,0xd5,0x3c,0xd4,0x28,0xd3,0x18,0xd2,0x0c,0x91, + 0x08,0x10,0x04,0x13,0x00,0x10,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x01, + 0x00,0x01,0x00,0x52,0x04,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01, + 0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01, + 0x00,0x01,0x00,0xd4,0x14,0x53,0x04,0x01,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x01, + 0x00,0x0c,0x00,0x01,0x00,0x01,0x00,0x53,0x04,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01, + 0x00,0x10,0x04,0x0c,0x00,0x13,0x09,0x91,0x08,0x10,0x04,0x13,0x09,0x0a,0x00,0x01, + 0x00,0xcf,0x86,0xd5,0x65,0xd4,0x45,0xd3,0x10,0x52,0x04,0x01,0x00,0x91,0x08,0x10, + 0x04,0x0a,0x00,0x00,0x00,0x01,0x00,0xd2,0x1e,0xd1,0x08,0x10,0x04,0x01,0x00,0x00, + 0x00,0x10,0x0b,0x01,0xff,0xe0,0xb5,0x86,0xe0,0xb4,0xbe,0x00,0x01,0xff,0xe0,0xb5, + 0x87,0xe0,0xb4,0xbe,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe0,0xb5,0x86,0xe0,0xb5, + 0x97,0x00,0x01,0x09,0x10,0x04,0x0c,0x00,0x12,0x00,0xd3,0x10,0x52,0x04,0x00,0x00, + 0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x00,0x01,0x00,0x52,0x04,0x12,0x00,0x51,0x04, + 0x12,0x00,0x10,0x04,0x12,0x00,0x11,0x00,0xd4,0x14,0x93,0x10,0xd2,0x08,0x11,0x04, + 0x01,0x00,0x0a,0x00,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0xd3,0x0c,0x52,0x04, + 0x0a,0x00,0x11,0x04,0x0a,0x00,0x12,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x12,0x00, + 0x0a,0x00,0x0a,0x00,0x0a,0x00,0xd0,0x5a,0xcf,0x86,0xd5,0x34,0xd4,0x18,0x93,0x14, + 0xd2,0x08,0x11,0x04,0x00,0x00,0x04,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x04,0x00, + 0x04,0x00,0x04,0x00,0xd3,0x10,0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04, + 0x04,0x00,0x00,0x00,0x92,0x08,0x11,0x04,0x00,0x00,0x04,0x00,0x04,0x00,0x54,0x04, + 0x04,0x00,0xd3,0x10,0x92,0x0c,0x51,0x04,0x04,0x00,0x10,0x04,0x00,0x00,0x04,0x00, + 0x04,0x00,0x52,0x04,0x04,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x04,0x00,0x00,0x00, + 0xcf,0x86,0xd5,0x77,0xd4,0x28,0xd3,0x10,0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00, + 0x10,0x04,0x04,0x00,0x00,0x00,0xd2,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x04,0x09, + 0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x04,0x00,0xd3,0x14,0x52,0x04, + 0x04,0x00,0xd1,0x08,0x10,0x04,0x04,0x00,0x00,0x00,0x10,0x04,0x04,0x00,0x00,0x00, + 0xd2,0x13,0x51,0x04,0x04,0x00,0x10,0x0b,0x04,0xff,0xe0,0xb7,0x99,0xe0,0xb7,0x8a, + 0x00,0x04,0x00,0xd1,0x19,0x10,0x0b,0x04,0xff,0xe0,0xb7,0x99,0xe0,0xb7,0x8f,0x00, + 0x04,0xff,0xe0,0xb7,0x99,0xe0,0xb7,0x8f,0xe0,0xb7,0x8a,0x00,0x10,0x0b,0x04,0xff, + 0xe0,0xb7,0x99,0xe0,0xb7,0x9f,0x00,0x04,0x00,0xd4,0x10,0x93,0x0c,0x52,0x04,0x00, + 0x00,0x11,0x04,0x00,0x00,0x10,0x00,0x10,0x00,0x93,0x14,0xd2,0x08,0x11,0x04,0x00, + 0x00,0x04,0x00,0x91,0x08,0x10,0x04,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xe2, + 0x31,0x01,0xd1,0x58,0xd0,0x3a,0xcf,0x86,0xd5,0x18,0x94,0x14,0x93,0x10,0x92,0x0c, + 0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00, + 0x54,0x04,0x01,0x00,0x53,0x04,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01,0x67,0x10,0x04, + 0x01,0x09,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x01,0x00,0xcf,0x86, + 0x95,0x18,0xd4,0x0c,0x53,0x04,0x01,0x00,0x12,0x04,0x01,0x6b,0x01,0x00,0x53,0x04, + 0x01,0x00,0x12,0x04,0x01,0x00,0x00,0x00,0x00,0x00,0xd0,0x9e,0xcf,0x86,0xd5,0x54, + 0xd4,0x3c,0xd3,0x20,0xd2,0x10,0xd1,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x10,0x04, + 0x01,0x00,0x00,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x15,0x00, + 0x01,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x01,0x00,0x15,0x00,0x10,0x04,0x01,0x00, + 0x00,0x00,0x91,0x08,0x10,0x04,0x15,0x00,0x01,0x00,0x15,0x00,0xd3,0x08,0x12,0x04, + 0x15,0x00,0x01,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x15,0x00,0x01,0x00,0x01,0x00, + 0x01,0x00,0xd4,0x30,0xd3,0x1c,0xd2,0x0c,0x91,0x08,0x10,0x04,0x15,0x00,0x01,0x00, + 0x01,0x00,0xd1,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x10,0x04,0x00,0x00,0x01,0x00, + 0xd2,0x08,0x11,0x04,0x15,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x15,0x00,0x01,0x00, + 0x01,0x00,0x53,0x04,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01,0x76,0x10,0x04,0x15,0x09, + 0x01,0x00,0x11,0x04,0x01,0x00,0x00,0x00,0xcf,0x86,0x95,0x34,0xd4,0x20,0xd3,0x14, + 0x52,0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x01,0x00, + 0x00,0x00,0x52,0x04,0x01,0x7a,0x11,0x04,0x01,0x00,0x00,0x00,0x53,0x04,0x01,0x00, + 0xd2,0x08,0x11,0x04,0x01,0x00,0x00,0x00,0x11,0x04,0x01,0x00,0x0d,0x00,0x00,0x00, + 0xe1,0x2b,0x01,0xd0,0x3e,0xcf,0x86,0xd5,0x14,0x54,0x04,0x02,0x00,0x53,0x04,0x02, + 0x00,0x92,0x08,0x11,0x04,0x02,0xdc,0x02,0x00,0x02,0x00,0x54,0x04,0x02,0x00,0xd3, + 0x14,0x52,0x04,0x02,0x00,0xd1,0x08,0x10,0x04,0x02,0x00,0x02,0xdc,0x10,0x04,0x02, + 0x00,0x02,0xdc,0x92,0x0c,0x91,0x08,0x10,0x04,0x02,0x00,0x02,0xd8,0x02,0x00,0x02, + 0x00,0xcf,0x86,0xd5,0x73,0xd4,0x36,0xd3,0x17,0x92,0x13,0x51,0x04,0x02,0x00,0x10, + 0x04,0x02,0x00,0x02,0xff,0xe0,0xbd,0x82,0xe0,0xbe,0xb7,0x00,0x02,0x00,0xd2,0x0c, + 0x91,0x08,0x10,0x04,0x00,0x00,0x02,0x00,0x02,0x00,0x91,0x0f,0x10,0x04,0x02,0x00, + 0x02,0xff,0xe0,0xbd,0x8c,0xe0,0xbe,0xb7,0x00,0x02,0x00,0xd3,0x26,0xd2,0x13,0x51, + 0x04,0x02,0x00,0x10,0x0b,0x02,0xff,0xe0,0xbd,0x91,0xe0,0xbe,0xb7,0x00,0x02,0x00, + 0x51,0x04,0x02,0x00,0x10,0x04,0x02,0x00,0x02,0xff,0xe0,0xbd,0x96,0xe0,0xbe,0xb7, + 0x00,0x52,0x04,0x02,0x00,0x91,0x0f,0x10,0x0b,0x02,0xff,0xe0,0xbd,0x9b,0xe0,0xbe, + 0xb7,0x00,0x02,0x00,0x02,0x00,0xd4,0x27,0x53,0x04,0x02,0x00,0xd2,0x17,0xd1,0x0f, + 0x10,0x04,0x02,0x00,0x02,0xff,0xe0,0xbd,0x80,0xe0,0xbe,0xb5,0x00,0x10,0x04,0x04, + 0x00,0x0a,0x00,0x91,0x08,0x10,0x04,0x0a,0x00,0x00,0x00,0x00,0x00,0xd3,0x35,0xd2, + 0x17,0xd1,0x08,0x10,0x04,0x00,0x00,0x02,0x81,0x10,0x04,0x02,0x82,0x02,0xff,0xe0, + 0xbd,0xb1,0xe0,0xbd,0xb2,0x00,0xd1,0x0f,0x10,0x04,0x02,0x84,0x02,0xff,0xe0,0xbd, + 0xb1,0xe0,0xbd,0xb4,0x00,0x10,0x0b,0x02,0xff,0xe0,0xbe,0xb2,0xe0,0xbe,0x80,0x00, + 0x02,0x00,0xd2,0x13,0x91,0x0f,0x10,0x0b,0x02,0xff,0xe0,0xbe,0xb3,0xe0,0xbe,0x80, + 0x00,0x02,0x00,0x02,0x82,0x11,0x04,0x02,0x82,0x02,0x00,0xd0,0xd3,0xcf,0x86,0xd5, + 0x65,0xd4,0x27,0xd3,0x1f,0xd2,0x13,0x91,0x0f,0x10,0x04,0x02,0x82,0x02,0xff,0xe0, + 0xbd,0xb1,0xe0,0xbe,0x80,0x00,0x02,0xe6,0x91,0x08,0x10,0x04,0x02,0x09,0x02,0x00, + 0x02,0xe6,0x12,0x04,0x02,0x00,0x0c,0x00,0xd3,0x1f,0xd2,0x13,0x51,0x04,0x02,0x00, + 0x10,0x04,0x02,0x00,0x02,0xff,0xe0,0xbe,0x92,0xe0,0xbe,0xb7,0x00,0x51,0x04,0x02, + 0x00,0x10,0x04,0x04,0x00,0x02,0x00,0xd2,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x02, + 0x00,0x02,0x00,0x91,0x0f,0x10,0x04,0x02,0x00,0x02,0xff,0xe0,0xbe,0x9c,0xe0,0xbe, + 0xb7,0x00,0x02,0x00,0xd4,0x3d,0xd3,0x26,0xd2,0x13,0x51,0x04,0x02,0x00,0x10,0x0b, + 0x02,0xff,0xe0,0xbe,0xa1,0xe0,0xbe,0xb7,0x00,0x02,0x00,0x51,0x04,0x02,0x00,0x10, + 0x04,0x02,0x00,0x02,0xff,0xe0,0xbe,0xa6,0xe0,0xbe,0xb7,0x00,0x52,0x04,0x02,0x00, + 0x91,0x0f,0x10,0x0b,0x02,0xff,0xe0,0xbe,0xab,0xe0,0xbe,0xb7,0x00,0x02,0x00,0x04, + 0x00,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x04,0x00,0x02,0x00,0x02,0x00,0x02, + 0x00,0xd2,0x13,0x91,0x0f,0x10,0x04,0x04,0x00,0x02,0xff,0xe0,0xbe,0x90,0xe0,0xbe, + 0xb5,0x00,0x04,0x00,0x91,0x08,0x10,0x04,0x04,0x00,0x00,0x00,0x04,0x00,0xcf,0x86, + 0x95,0x4c,0xd4,0x24,0xd3,0x10,0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04, + 0x04,0xdc,0x04,0x00,0x52,0x04,0x04,0x00,0xd1,0x08,0x10,0x04,0x04,0x00,0x00,0x00, + 0x10,0x04,0x0a,0x00,0x04,0x00,0xd3,0x14,0xd2,0x08,0x11,0x04,0x08,0x00,0x0a,0x00, + 0x91,0x08,0x10,0x04,0x0a,0x00,0x0b,0x00,0x0b,0x00,0x92,0x10,0xd1,0x08,0x10,0x04, + 0x0b,0x00,0x0c,0x00,0x10,0x04,0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86, + 0xe5,0xf7,0x04,0xe4,0x79,0x03,0xe3,0x7b,0x01,0xe2,0x04,0x01,0xd1,0x7f,0xd0,0x65, + 0xcf,0x86,0x55,0x04,0x04,0x00,0xd4,0x33,0xd3,0x1f,0xd2,0x0c,0x51,0x04,0x04,0x00, + 0x10,0x04,0x0a,0x00,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x0b,0x04,0xff,0xe1,0x80, + 0xa5,0xe1,0x80,0xae,0x00,0x04,0x00,0x92,0x10,0xd1,0x08,0x10,0x04,0x0a,0x00,0x04, + 0x00,0x10,0x04,0x04,0x00,0x0a,0x00,0x04,0x00,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x04, + 0x00,0x10,0x04,0x04,0x00,0x0a,0x00,0x51,0x04,0x0a,0x00,0x10,0x04,0x04,0x00,0x04, + 0x07,0x92,0x10,0xd1,0x08,0x10,0x04,0x04,0x00,0x04,0x09,0x10,0x04,0x0a,0x09,0x0a, + 0x00,0x0a,0x00,0xcf,0x86,0x95,0x14,0x54,0x04,0x04,0x00,0x53,0x04,0x04,0x00,0x92, + 0x08,0x11,0x04,0x04,0x00,0x0a,0x00,0x0a,0x00,0x0a,0x00,0xd0,0x2e,0xcf,0x86,0x95, + 0x28,0xd4,0x14,0x53,0x04,0x0a,0x00,0x52,0x04,0x0a,0x00,0x91,0x08,0x10,0x04,0x0a, + 0x00,0x0a,0xdc,0x0a,0x00,0x53,0x04,0x0a,0x00,0xd2,0x08,0x11,0x04,0x0a,0x00,0x0b, + 0x00,0x11,0x04,0x0b,0x00,0x0a,0x00,0x01,0x00,0xcf,0x86,0xd5,0x24,0x94,0x20,0xd3, + 0x10,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x00,0x00,0x0d,0x00,0x52, + 0x04,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x00,0x00,0x01,0x00,0x54, + 0x04,0x01,0x00,0xd3,0x10,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01, + 0x00,0x06,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x06,0x00,0x08,0x00,0x10,0x04,0x08, + 0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x08,0x00,0x0d,0x00,0x0d,0x00,0xd1,0x3e,0xd0, + 0x06,0xcf,0x06,0x01,0x00,0xcf,0x86,0xd5,0x1d,0x54,0x04,0x01,0x00,0x53,0x04,0x01, + 0x00,0xd2,0x08,0x11,0x04,0x01,0x00,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b, + 0x00,0x01,0xff,0x00,0x94,0x15,0x93,0x11,0x92,0x0d,0x91,0x09,0x10,0x05,0x01,0xff, + 0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd0,0x1e,0xcf,0x86,0x55, + 0x04,0x01,0x00,0x94,0x14,0x93,0x10,0x92,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01, + 0x00,0x0b,0x00,0x0b,0x00,0x01,0x00,0x01,0x00,0xcf,0x86,0x55,0x04,0x01,0x00,0x54, + 0x04,0x01,0x00,0x53,0x04,0x01,0x00,0x92,0x08,0x11,0x04,0x01,0x00,0x0b,0x00,0x0b, + 0x00,0xe2,0x21,0x01,0xd1,0x6c,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x94,0x14,0x93,0x10, + 0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00,0x04,0x00, + 0x04,0x00,0x04,0x00,0xcf,0x86,0x95,0x48,0xd4,0x24,0xd3,0x10,0x52,0x04,0x04,0x00, + 0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00,0xd2,0x0c,0x91,0x08,0x10,0x04, + 0x04,0x00,0x00,0x00,0x04,0x00,0x11,0x04,0x04,0x00,0x00,0x00,0xd3,0x10,0x52,0x04, + 0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x00,0x00,0xd2,0x0c,0x91,0x08, + 0x10,0x04,0x04,0x00,0x00,0x00,0x04,0x00,0x11,0x04,0x04,0x00,0x00,0x00,0x04,0x00, + 0xd0,0x62,0xcf,0x86,0xd5,0x28,0x94,0x24,0xd3,0x10,0x52,0x04,0x04,0x00,0x51,0x04, + 0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00,0xd2,0x0c,0x91,0x08,0x10,0x04,0x04,0x00, + 0x00,0x00,0x04,0x00,0x11,0x04,0x04,0x00,0x00,0x00,0x04,0x00,0xd4,0x14,0x53,0x04, + 0x04,0x00,0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00, + 0xd3,0x14,0xd2,0x0c,0x91,0x08,0x10,0x04,0x04,0x00,0x00,0x00,0x04,0x00,0x11,0x04, + 0x04,0x00,0x00,0x00,0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00, + 0x00,0x00,0xcf,0x86,0xd5,0x38,0xd4,0x24,0xd3,0x14,0xd2,0x0c,0x91,0x08,0x10,0x04, + 0x04,0x00,0x00,0x00,0x04,0x00,0x11,0x04,0x04,0x00,0x00,0x00,0x52,0x04,0x04,0x00, + 0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00,0x93,0x10,0x52,0x04,0x04,0x00, + 0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x00,0x00,0x04,0x00,0x94,0x14,0x53,0x04, + 0x04,0x00,0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00, + 0x04,0x00,0xd1,0x9c,0xd0,0x3e,0xcf,0x86,0x95,0x38,0xd4,0x14,0x53,0x04,0x04,0x00, + 0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00,0xd3,0x14, + 0xd2,0x0c,0x91,0x08,0x10,0x04,0x04,0x00,0x00,0x00,0x04,0x00,0x11,0x04,0x04,0x00, + 0x00,0x00,0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00, + 0x04,0x00,0xcf,0x86,0xd5,0x34,0xd4,0x14,0x93,0x10,0x52,0x04,0x04,0x00,0x51,0x04, + 0x04,0x00,0x10,0x04,0x04,0x00,0x08,0x00,0x04,0x00,0x53,0x04,0x04,0x00,0xd2,0x0c, + 0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x00,0x00,0xd1,0x08,0x10,0x04,0x00,0x00, + 0x0c,0xe6,0x10,0x04,0x0c,0xe6,0x08,0xe6,0xd4,0x14,0x93,0x10,0x92,0x0c,0x91,0x08, + 0x10,0x04,0x08,0x00,0x04,0x00,0x04,0x00,0x04,0x00,0x04,0x00,0x53,0x04,0x04,0x00, + 0x52,0x04,0x04,0x00,0x91,0x08,0x10,0x04,0x04,0x00,0x00,0x00,0x00,0x00,0xd0,0x1a, + 0xcf,0x86,0x95,0x14,0x54,0x04,0x08,0x00,0x53,0x04,0x08,0x00,0x92,0x08,0x11,0x04, + 0x08,0x00,0x00,0x00,0x00,0x00,0x04,0x00,0xcf,0x86,0x55,0x04,0x04,0x00,0x54,0x04, + 0x04,0x00,0xd3,0x10,0x52,0x04,0x04,0x00,0x91,0x08,0x10,0x04,0x04,0x00,0x11,0x00, + 0x00,0x00,0x52,0x04,0x11,0x00,0x11,0x04,0x11,0x00,0x00,0x00,0xd3,0x30,0xd2,0x2a, + 0xd1,0x24,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08, + 0x10,0x04,0x0b,0x00,0x04,0x00,0x04,0x00,0x04,0x00,0x04,0x00,0x04,0x00,0x04,0x00, + 0xcf,0x06,0x04,0x00,0xcf,0x06,0x04,0x00,0xcf,0x06,0x04,0x00,0xd2,0x6c,0xd1,0x24, + 0xd0,0x06,0xcf,0x06,0x04,0x00,0xcf,0x86,0x55,0x04,0x04,0x00,0x54,0x04,0x04,0x00, + 0x93,0x10,0x52,0x04,0x04,0x00,0x51,0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x0b,0x00, + 0x0b,0x00,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x54,0x04,0x04,0x00,0x53,0x04,0x04,0x00, + 0x52,0x04,0x04,0x00,0x91,0x08,0x10,0x04,0x04,0x00,0x00,0x00,0x00,0x00,0x04,0x00, + 0xcf,0x86,0x55,0x04,0x04,0x00,0x54,0x04,0x04,0x00,0xd3,0x10,0x92,0x0c,0x91,0x08, + 0x10,0x04,0x04,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x92,0x0c,0x91,0x08,0x10,0x04, + 0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd1,0x80,0xd0,0x46,0xcf,0x86,0xd5,0x28, + 0xd4,0x14,0x53,0x04,0x06,0x00,0x52,0x04,0x06,0x00,0x91,0x08,0x10,0x04,0x06,0x00, + 0x00,0x00,0x06,0x00,0x93,0x10,0x52,0x04,0x06,0x00,0x91,0x08,0x10,0x04,0x06,0x09, + 0x00,0x00,0x00,0x00,0x00,0x00,0x54,0x04,0x06,0x00,0x93,0x14,0x52,0x04,0x06,0x00, + 0xd1,0x08,0x10,0x04,0x06,0x09,0x06,0x00,0x10,0x04,0x06,0x00,0x00,0x00,0x00,0x00, + 0xcf,0x86,0xd5,0x10,0x54,0x04,0x06,0x00,0x93,0x08,0x12,0x04,0x06,0x00,0x00,0x00, + 0x00,0x00,0xd4,0x14,0x53,0x04,0x06,0x00,0x52,0x04,0x06,0x00,0x91,0x08,0x10,0x04, + 0x06,0x00,0x00,0x00,0x06,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x06,0x00, + 0x00,0x00,0x06,0x00,0x00,0x00,0x00,0x00,0xd0,0x1b,0xcf,0x86,0x55,0x04,0x04,0x00, + 0x54,0x04,0x04,0x00,0x93,0x0d,0x52,0x04,0x04,0x00,0x11,0x05,0x04,0xff,0x00,0x04, + 0x00,0x04,0x00,0xcf,0x86,0xd5,0x24,0x54,0x04,0x04,0x00,0xd3,0x10,0x92,0x0c,0x51, + 0x04,0x04,0x00,0x10,0x04,0x04,0x09,0x04,0x00,0x04,0x00,0x52,0x04,0x04,0x00,0x91, + 0x08,0x10,0x04,0x04,0x00,0x07,0xe6,0x00,0x00,0xd4,0x10,0x53,0x04,0x04,0x00,0x92, + 0x08,0x11,0x04,0x04,0x00,0x00,0x00,0x00,0x00,0x53,0x04,0x07,0x00,0x92,0x08,0x11, + 0x04,0x07,0x00,0x00,0x00,0x00,0x00,0xe4,0xb7,0x03,0xe3,0x58,0x01,0xd2,0x8f,0xd1, + 0x53,0xd0,0x35,0xcf,0x86,0x95,0x2f,0xd4,0x1f,0x53,0x04,0x04,0x00,0xd2,0x0d,0x51, + 0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x04,0xff,0x00,0x51,0x05,0x04,0xff,0x00,0x10, + 0x05,0x04,0xff,0x00,0x00,0x00,0x53,0x04,0x04,0x00,0x92,0x08,0x11,0x04,0x04,0x00, + 0x00,0x00,0x00,0x00,0x04,0x00,0xcf,0x86,0x55,0x04,0x04,0x00,0x54,0x04,0x04,0x00, + 0x53,0x04,0x04,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x14,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0xd0,0x22,0xcf,0x86,0x55,0x04,0x04,0x00,0x94,0x18,0x53,0x04,0x04,0x00, + 0x92,0x10,0xd1,0x08,0x10,0x04,0x04,0x00,0x04,0xe4,0x10,0x04,0x0a,0x00,0x00,0x00, + 0x00,0x00,0x0b,0x00,0xcf,0x86,0x55,0x04,0x0b,0x00,0x54,0x04,0x0b,0x00,0x93,0x0c, + 0x52,0x04,0x0b,0x00,0x11,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0xd1,0x80,0xd0,0x42, + 0xcf,0x86,0xd5,0x1c,0x54,0x04,0x07,0x00,0x53,0x04,0x07,0x00,0x52,0x04,0x07,0x00, + 0xd1,0x08,0x10,0x04,0x07,0x00,0x10,0x00,0x10,0x04,0x10,0x00,0x00,0x00,0xd4,0x0c, + 0x53,0x04,0x07,0x00,0x12,0x04,0x07,0x00,0x00,0x00,0x53,0x04,0x07,0x00,0x92,0x10, + 0xd1,0x08,0x10,0x04,0x07,0x00,0x07,0xde,0x10,0x04,0x07,0xe6,0x07,0xdc,0x00,0x00, + 0xcf,0x86,0xd5,0x18,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x07,0x00, + 0x00,0x00,0x00,0x00,0x07,0x00,0x07,0x00,0x07,0x00,0xd4,0x10,0x53,0x04,0x07,0x00, + 0x52,0x04,0x07,0x00,0x11,0x04,0x07,0x00,0x00,0x00,0x93,0x10,0x52,0x04,0x07,0x00, + 0x91,0x08,0x10,0x04,0x07,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd0,0x1a,0xcf,0x86, + 0x55,0x04,0x08,0x00,0x94,0x10,0x53,0x04,0x08,0x00,0x92,0x08,0x11,0x04,0x08,0x00, + 0x0b,0x00,0x00,0x00,0x08,0x00,0xcf,0x86,0x95,0x28,0xd4,0x10,0x53,0x04,0x08,0x00, + 0x92,0x08,0x11,0x04,0x08,0x00,0x00,0x00,0x00,0x00,0x53,0x04,0x08,0x00,0xd2,0x0c, + 0x51,0x04,0x08,0x00,0x10,0x04,0x0b,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x08,0x00, + 0x07,0x00,0xd2,0xe4,0xd1,0x80,0xd0,0x2e,0xcf,0x86,0x95,0x28,0x54,0x04,0x08,0x00, + 0xd3,0x10,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x08,0xe6, + 0xd2,0x0c,0x91,0x08,0x10,0x04,0x08,0xdc,0x08,0x00,0x08,0x00,0x11,0x04,0x00,0x00, + 0x08,0x00,0x0b,0x00,0xcf,0x86,0xd5,0x18,0x54,0x04,0x0b,0x00,0x53,0x04,0x0b,0x00, + 0x52,0x04,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b,0x00,0x00,0x00,0xd4,0x14, + 0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0b,0x09,0x0b,0x00,0x0b,0x00,0x0b,0x00, + 0x0b,0x00,0xd3,0x10,0x52,0x04,0x0b,0x00,0x91,0x08,0x10,0x04,0x0b,0x00,0x0b,0xe6, + 0x0b,0xe6,0x52,0x04,0x0b,0xe6,0xd1,0x08,0x10,0x04,0x0b,0xe6,0x00,0x00,0x10,0x04, + 0x00,0x00,0x0b,0xdc,0xd0,0x5e,0xcf,0x86,0xd5,0x20,0xd4,0x10,0x53,0x04,0x0b,0x00, + 0x92,0x08,0x11,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0x53,0x04,0x0b,0x00,0x92,0x08, + 0x11,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0xd4,0x10,0x53,0x04,0x0b,0x00,0x52,0x04, + 0x0b,0x00,0x11,0x04,0x0b,0x00,0x00,0x00,0xd3,0x10,0x52,0x04,0x10,0xe6,0x91,0x08, + 0x10,0x04,0x10,0xe6,0x10,0xdc,0x10,0xdc,0xd2,0x0c,0x51,0x04,0x10,0xdc,0x10,0x04, + 0x10,0xdc,0x10,0xe6,0xd1,0x08,0x10,0x04,0x10,0xe6,0x10,0xdc,0x10,0x04,0x10,0x00, + 0x00,0x00,0xcf,0x06,0x00,0x00,0xe1,0x1e,0x01,0xd0,0xaa,0xcf,0x86,0xd5,0x6e,0xd4, + 0x53,0xd3,0x17,0x52,0x04,0x09,0x00,0x51,0x04,0x09,0x00,0x10,0x0b,0x09,0xff,0xe1, + 0xac,0x85,0xe1,0xac,0xb5,0x00,0x09,0x00,0xd2,0x1e,0xd1,0x0f,0x10,0x0b,0x09,0xff, + 0xe1,0xac,0x87,0xe1,0xac,0xb5,0x00,0x09,0x00,0x10,0x0b,0x09,0xff,0xe1,0xac,0x89, + 0xe1,0xac,0xb5,0x00,0x09,0x00,0xd1,0x0f,0x10,0x0b,0x09,0xff,0xe1,0xac,0x8b,0xe1, + 0xac,0xb5,0x00,0x09,0x00,0x10,0x0b,0x09,0xff,0xe1,0xac,0x8d,0xe1,0xac,0xb5,0x00, + 0x09,0x00,0x93,0x17,0x92,0x13,0x51,0x04,0x09,0x00,0x10,0x0b,0x09,0xff,0xe1,0xac, + 0x91,0xe1,0xac,0xb5,0x00,0x09,0x00,0x09,0x00,0x09,0x00,0x54,0x04,0x09,0x00,0xd3, + 0x10,0x52,0x04,0x09,0x00,0x91,0x08,0x10,0x04,0x09,0x07,0x09,0x00,0x09,0x00,0xd2, + 0x13,0x51,0x04,0x09,0x00,0x10,0x04,0x09,0x00,0x09,0xff,0xe1,0xac,0xba,0xe1,0xac, + 0xb5,0x00,0x91,0x0f,0x10,0x04,0x09,0x00,0x09,0xff,0xe1,0xac,0xbc,0xe1,0xac,0xb5, + 0x00,0x09,0x00,0xcf,0x86,0xd5,0x3d,0x94,0x39,0xd3,0x31,0xd2,0x25,0xd1,0x16,0x10, + 0x0b,0x09,0xff,0xe1,0xac,0xbe,0xe1,0xac,0xb5,0x00,0x09,0xff,0xe1,0xac,0xbf,0xe1, + 0xac,0xb5,0x00,0x10,0x04,0x09,0x00,0x09,0xff,0xe1,0xad,0x82,0xe1,0xac,0xb5,0x00, + 0x91,0x08,0x10,0x04,0x09,0x09,0x09,0x00,0x09,0x00,0x12,0x04,0x09,0x00,0x00,0x00, + 0x09,0x00,0xd4,0x1c,0x53,0x04,0x09,0x00,0xd2,0x0c,0x51,0x04,0x09,0x00,0x10,0x04, + 0x09,0x00,0x09,0xe6,0x91,0x08,0x10,0x04,0x09,0xdc,0x09,0xe6,0x09,0xe6,0xd3,0x08, + 0x12,0x04,0x09,0xe6,0x09,0x00,0x52,0x04,0x09,0x00,0x91,0x08,0x10,0x04,0x09,0x00, + 0x00,0x00,0x00,0x00,0xd0,0x2e,0xcf,0x86,0x55,0x04,0x0a,0x00,0xd4,0x18,0x53,0x04, + 0x0a,0x00,0xd2,0x0c,0x51,0x04,0x0a,0x00,0x10,0x04,0x0a,0x09,0x0d,0x09,0x11,0x04, + 0x0d,0x00,0x0a,0x00,0x53,0x04,0x0a,0x00,0x92,0x08,0x11,0x04,0x0a,0x00,0x0d,0x00, + 0x0d,0x00,0xcf,0x86,0x55,0x04,0x0c,0x00,0xd4,0x14,0x93,0x10,0x52,0x04,0x0c,0x00, + 0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x07,0x0c,0x00,0x0c,0x00,0xd3,0x0c,0x92,0x08, + 0x11,0x04,0x0c,0x00,0x0c,0x09,0x00,0x00,0x12,0x04,0x00,0x00,0x0c,0x00,0xe3,0xb2, + 0x01,0xe2,0x09,0x01,0xd1,0x4c,0xd0,0x2a,0xcf,0x86,0x55,0x04,0x0a,0x00,0x54,0x04, + 0x0a,0x00,0xd3,0x10,0x52,0x04,0x0a,0x00,0x51,0x04,0x0a,0x00,0x10,0x04,0x0a,0x00, + 0x0a,0x07,0x92,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x0a,0x00,0x0a,0x00, + 0xcf,0x86,0x95,0x1c,0x94,0x18,0x53,0x04,0x0a,0x00,0xd2,0x08,0x11,0x04,0x0a,0x00, + 0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x0a,0x00,0x0a,0x00,0x0a,0x00,0x0a,0x00, + 0xd0,0x3a,0xcf,0x86,0xd5,0x18,0x94,0x14,0x53,0x04,0x12,0x00,0x92,0x0c,0x91,0x08, + 0x10,0x04,0x12,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x14,0x00,0x54,0x04,0x14,0x00, + 0x53,0x04,0x14,0x00,0xd2,0x0c,0x51,0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x00,0x00, + 0x91,0x08,0x10,0x04,0x00,0x00,0x14,0x00,0x14,0x00,0xcf,0x86,0xd5,0x2c,0xd4,0x08, + 0x13,0x04,0x0d,0x00,0x00,0x00,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x0b,0xe6,0x10,0x04, + 0x0b,0xe6,0x0b,0x00,0x91,0x08,0x10,0x04,0x0b,0x01,0x0b,0xdc,0x0b,0xdc,0x92,0x08, + 0x11,0x04,0x0b,0xdc,0x0b,0xe6,0x0b,0xdc,0xd4,0x28,0xd3,0x10,0x92,0x0c,0x91,0x08, + 0x10,0x04,0x0b,0xe6,0x0b,0x00,0x0b,0x01,0x0b,0x01,0xd2,0x0c,0x91,0x08,0x10,0x04, + 0x0b,0x01,0x0b,0x00,0x0b,0x00,0x91,0x08,0x10,0x04,0x0b,0x00,0x0b,0xdc,0x0b,0x00, + 0xd3,0x1c,0xd2,0x0c,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b,0x00,0x0d,0x00,0xd1,0x08, + 0x10,0x04,0x0d,0xe6,0x0d,0x00,0x10,0x04,0x0d,0x00,0x13,0x00,0x92,0x0c,0x51,0x04, + 0x10,0xe6,0x10,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0xd1,0x1c,0xd0,0x06,0xcf,0x06, + 0x07,0x00,0xcf,0x86,0x55,0x04,0x07,0x00,0x94,0x0c,0x53,0x04,0x07,0x00,0x12,0x04, + 0x07,0x00,0x08,0x00,0x08,0x00,0xd0,0x06,0xcf,0x06,0x08,0x00,0xcf,0x86,0xd5,0x40, + 0xd4,0x2c,0xd3,0x10,0x92,0x0c,0x51,0x04,0x08,0xe6,0x10,0x04,0x08,0xdc,0x08,0xe6, + 0x09,0xe6,0xd2,0x0c,0x51,0x04,0x09,0xe6,0x10,0x04,0x09,0xdc,0x0a,0xe6,0xd1,0x08, + 0x10,0x04,0x0a,0xe6,0x0a,0xea,0x10,0x04,0x0a,0xd6,0x0a,0xdc,0x93,0x10,0x92,0x0c, + 0x91,0x08,0x10,0x04,0x0a,0xca,0x0a,0xe6,0x0a,0xe6,0x0a,0xe6,0x0a,0xe6,0xd4,0x14, + 0x93,0x10,0x52,0x04,0x0a,0xe6,0x51,0x04,0x0a,0xe6,0x10,0x04,0x0a,0xe6,0x10,0xe6, + 0x10,0xe6,0xd3,0x10,0x52,0x04,0x10,0xe6,0x51,0x04,0x10,0xe6,0x10,0x04,0x13,0xe8, + 0x13,0xe4,0xd2,0x10,0xd1,0x08,0x10,0x04,0x13,0xe4,0x13,0xdc,0x10,0x04,0x00,0x00, + 0x12,0xe6,0xd1,0x08,0x10,0x04,0x0c,0xe9,0x0b,0xdc,0x10,0x04,0x09,0xe6,0x09,0xdc, + 0xe2,0x80,0x08,0xe1,0x48,0x04,0xe0,0x1c,0x02,0xcf,0x86,0xe5,0x11,0x01,0xd4,0x84, + 0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x41,0xcc,0xa5,0x00,0x01,0xff, + 0x61,0xcc,0xa5,0x00,0x10,0x08,0x01,0xff,0x42,0xcc,0x87,0x00,0x01,0xff,0x62,0xcc, + 0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x42,0xcc,0xa3,0x00,0x01,0xff,0x62,0xcc, + 0xa3,0x00,0x10,0x08,0x01,0xff,0x42,0xcc,0xb1,0x00,0x01,0xff,0x62,0xcc,0xb1,0x00, + 0xd2,0x24,0xd1,0x14,0x10,0x0a,0x01,0xff,0x43,0xcc,0xa7,0xcc,0x81,0x00,0x01,0xff, + 0x63,0xcc,0xa7,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x44,0xcc,0x87,0x00,0x01,0xff, + 0x64,0xcc,0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x44,0xcc,0xa3,0x00,0x01,0xff, + 0x64,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x44,0xcc,0xb1,0x00,0x01,0xff,0x64,0xcc, + 0xb1,0x00,0xd3,0x48,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x44,0xcc,0xa7,0x00, + 0x01,0xff,0x64,0xcc,0xa7,0x00,0x10,0x08,0x01,0xff,0x44,0xcc,0xad,0x00,0x01,0xff, + 0x64,0xcc,0xad,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x45,0xcc,0x84,0xcc,0x80,0x00, + 0x01,0xff,0x65,0xcc,0x84,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0x45,0xcc,0x84,0xcc, + 0x81,0x00,0x01,0xff,0x65,0xcc,0x84,0xcc,0x81,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x01,0xff,0x45,0xcc,0xad,0x00,0x01,0xff,0x65,0xcc,0xad,0x00,0x10,0x08,0x01,0xff, + 0x45,0xcc,0xb0,0x00,0x01,0xff,0x65,0xcc,0xb0,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff, + 0x45,0xcc,0xa7,0xcc,0x86,0x00,0x01,0xff,0x65,0xcc,0xa7,0xcc,0x86,0x00,0x10,0x08, + 0x01,0xff,0x46,0xcc,0x87,0x00,0x01,0xff,0x66,0xcc,0x87,0x00,0xd4,0x84,0xd3,0x40, + 0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x47,0xcc,0x84,0x00,0x01,0xff,0x67,0xcc, + 0x84,0x00,0x10,0x08,0x01,0xff,0x48,0xcc,0x87,0x00,0x01,0xff,0x68,0xcc,0x87,0x00, + 0xd1,0x10,0x10,0x08,0x01,0xff,0x48,0xcc,0xa3,0x00,0x01,0xff,0x68,0xcc,0xa3,0x00, + 0x10,0x08,0x01,0xff,0x48,0xcc,0x88,0x00,0x01,0xff,0x68,0xcc,0x88,0x00,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x01,0xff,0x48,0xcc,0xa7,0x00,0x01,0xff,0x68,0xcc,0xa7,0x00, + 0x10,0x08,0x01,0xff,0x48,0xcc,0xae,0x00,0x01,0xff,0x68,0xcc,0xae,0x00,0xd1,0x10, + 0x10,0x08,0x01,0xff,0x49,0xcc,0xb0,0x00,0x01,0xff,0x69,0xcc,0xb0,0x00,0x10,0x0a, + 0x01,0xff,0x49,0xcc,0x88,0xcc,0x81,0x00,0x01,0xff,0x69,0xcc,0x88,0xcc,0x81,0x00, + 0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x4b,0xcc,0x81,0x00,0x01,0xff, + 0x6b,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x4b,0xcc,0xa3,0x00,0x01,0xff,0x6b,0xcc, + 0xa3,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x4b,0xcc,0xb1,0x00,0x01,0xff,0x6b,0xcc, + 0xb1,0x00,0x10,0x08,0x01,0xff,0x4c,0xcc,0xa3,0x00,0x01,0xff,0x6c,0xcc,0xa3,0x00, + 0xd2,0x24,0xd1,0x14,0x10,0x0a,0x01,0xff,0x4c,0xcc,0xa3,0xcc,0x84,0x00,0x01,0xff, + 0x6c,0xcc,0xa3,0xcc,0x84,0x00,0x10,0x08,0x01,0xff,0x4c,0xcc,0xb1,0x00,0x01,0xff, + 0x6c,0xcc,0xb1,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x4c,0xcc,0xad,0x00,0x01,0xff, + 0x6c,0xcc,0xad,0x00,0x10,0x08,0x01,0xff,0x4d,0xcc,0x81,0x00,0x01,0xff,0x6d,0xcc, + 0x81,0x00,0xcf,0x86,0xe5,0x15,0x01,0xd4,0x88,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10, + 0x08,0x01,0xff,0x4d,0xcc,0x87,0x00,0x01,0xff,0x6d,0xcc,0x87,0x00,0x10,0x08,0x01, + 0xff,0x4d,0xcc,0xa3,0x00,0x01,0xff,0x6d,0xcc,0xa3,0x00,0xd1,0x10,0x10,0x08,0x01, + 0xff,0x4e,0xcc,0x87,0x00,0x01,0xff,0x6e,0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x4e, + 0xcc,0xa3,0x00,0x01,0xff,0x6e,0xcc,0xa3,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01, + 0xff,0x4e,0xcc,0xb1,0x00,0x01,0xff,0x6e,0xcc,0xb1,0x00,0x10,0x08,0x01,0xff,0x4e, + 0xcc,0xad,0x00,0x01,0xff,0x6e,0xcc,0xad,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x4f, + 0xcc,0x83,0xcc,0x81,0x00,0x01,0xff,0x6f,0xcc,0x83,0xcc,0x81,0x00,0x10,0x0a,0x01, + 0xff,0x4f,0xcc,0x83,0xcc,0x88,0x00,0x01,0xff,0x6f,0xcc,0x83,0xcc,0x88,0x00,0xd3, + 0x48,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x4f,0xcc,0x84,0xcc,0x80,0x00,0x01, + 0xff,0x6f,0xcc,0x84,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0x4f,0xcc,0x84,0xcc,0x81, + 0x00,0x01,0xff,0x6f,0xcc,0x84,0xcc,0x81,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x50, + 0xcc,0x81,0x00,0x01,0xff,0x70,0xcc,0x81,0x00,0x10,0x08,0x01,0xff,0x50,0xcc,0x87, + 0x00,0x01,0xff,0x70,0xcc,0x87,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x52, + 0xcc,0x87,0x00,0x01,0xff,0x72,0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x52,0xcc,0xa3, + 0x00,0x01,0xff,0x72,0xcc,0xa3,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x52,0xcc,0xa3, + 0xcc,0x84,0x00,0x01,0xff,0x72,0xcc,0xa3,0xcc,0x84,0x00,0x10,0x08,0x01,0xff,0x52, + 0xcc,0xb1,0x00,0x01,0xff,0x72,0xcc,0xb1,0x00,0xd4,0x8c,0xd3,0x48,0xd2,0x20,0xd1, + 0x10,0x10,0x08,0x01,0xff,0x53,0xcc,0x87,0x00,0x01,0xff,0x73,0xcc,0x87,0x00,0x10, + 0x08,0x01,0xff,0x53,0xcc,0xa3,0x00,0x01,0xff,0x73,0xcc,0xa3,0x00,0xd1,0x14,0x10, + 0x0a,0x01,0xff,0x53,0xcc,0x81,0xcc,0x87,0x00,0x01,0xff,0x73,0xcc,0x81,0xcc,0x87, + 0x00,0x10,0x0a,0x01,0xff,0x53,0xcc,0x8c,0xcc,0x87,0x00,0x01,0xff,0x73,0xcc,0x8c, + 0xcc,0x87,0x00,0xd2,0x24,0xd1,0x14,0x10,0x0a,0x01,0xff,0x53,0xcc,0xa3,0xcc,0x87, + 0x00,0x01,0xff,0x73,0xcc,0xa3,0xcc,0x87,0x00,0x10,0x08,0x01,0xff,0x54,0xcc,0x87, + 0x00,0x01,0xff,0x74,0xcc,0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x54,0xcc,0xa3, + 0x00,0x01,0xff,0x74,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x54,0xcc,0xb1,0x00,0x01, + 0xff,0x74,0xcc,0xb1,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x54, + 0xcc,0xad,0x00,0x01,0xff,0x74,0xcc,0xad,0x00,0x10,0x08,0x01,0xff,0x55,0xcc,0xa4, + 0x00,0x01,0xff,0x75,0xcc,0xa4,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x55,0xcc,0xb0, + 0x00,0x01,0xff,0x75,0xcc,0xb0,0x00,0x10,0x08,0x01,0xff,0x55,0xcc,0xad,0x00,0x01, + 0xff,0x75,0xcc,0xad,0x00,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x55,0xcc,0x83, + 0xcc,0x81,0x00,0x01,0xff,0x75,0xcc,0x83,0xcc,0x81,0x00,0x10,0x0a,0x01,0xff,0x55, + 0xcc,0x84,0xcc,0x88,0x00,0x01,0xff,0x75,0xcc,0x84,0xcc,0x88,0x00,0xd1,0x10,0x10, + 0x08,0x01,0xff,0x56,0xcc,0x83,0x00,0x01,0xff,0x76,0xcc,0x83,0x00,0x10,0x08,0x01, + 0xff,0x56,0xcc,0xa3,0x00,0x01,0xff,0x76,0xcc,0xa3,0x00,0xe0,0x10,0x02,0xcf,0x86, + 0xd5,0xe1,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x57,0xcc, + 0x80,0x00,0x01,0xff,0x77,0xcc,0x80,0x00,0x10,0x08,0x01,0xff,0x57,0xcc,0x81,0x00, + 0x01,0xff,0x77,0xcc,0x81,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x57,0xcc,0x88,0x00, + 0x01,0xff,0x77,0xcc,0x88,0x00,0x10,0x08,0x01,0xff,0x57,0xcc,0x87,0x00,0x01,0xff, + 0x77,0xcc,0x87,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x57,0xcc,0xa3,0x00, + 0x01,0xff,0x77,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x58,0xcc,0x87,0x00,0x01,0xff, + 0x78,0xcc,0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x58,0xcc,0x88,0x00,0x01,0xff, + 0x78,0xcc,0x88,0x00,0x10,0x08,0x01,0xff,0x59,0xcc,0x87,0x00,0x01,0xff,0x79,0xcc, + 0x87,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x5a,0xcc,0x82,0x00, + 0x01,0xff,0x7a,0xcc,0x82,0x00,0x10,0x08,0x01,0xff,0x5a,0xcc,0xa3,0x00,0x01,0xff, + 0x7a,0xcc,0xa3,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x5a,0xcc,0xb1,0x00,0x01,0xff, + 0x7a,0xcc,0xb1,0x00,0x10,0x08,0x01,0xff,0x68,0xcc,0xb1,0x00,0x01,0xff,0x74,0xcc, + 0x88,0x00,0x92,0x1d,0xd1,0x10,0x10,0x08,0x01,0xff,0x77,0xcc,0x8a,0x00,0x01,0xff, + 0x79,0xcc,0x8a,0x00,0x10,0x04,0x01,0x00,0x02,0xff,0xc5,0xbf,0xcc,0x87,0x00,0x0a, + 0x00,0xd4,0x98,0xd3,0x48,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x41,0xcc,0xa3, + 0x00,0x01,0xff,0x61,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x41,0xcc,0x89,0x00,0x01, + 0xff,0x61,0xcc,0x89,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x41,0xcc,0x82,0xcc,0x81, + 0x00,0x01,0xff,0x61,0xcc,0x82,0xcc,0x81,0x00,0x10,0x0a,0x01,0xff,0x41,0xcc,0x82, + 0xcc,0x80,0x00,0x01,0xff,0x61,0xcc,0x82,0xcc,0x80,0x00,0xd2,0x28,0xd1,0x14,0x10, + 0x0a,0x01,0xff,0x41,0xcc,0x82,0xcc,0x89,0x00,0x01,0xff,0x61,0xcc,0x82,0xcc,0x89, + 0x00,0x10,0x0a,0x01,0xff,0x41,0xcc,0x82,0xcc,0x83,0x00,0x01,0xff,0x61,0xcc,0x82, + 0xcc,0x83,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x41,0xcc,0xa3,0xcc,0x82,0x00,0x01, + 0xff,0x61,0xcc,0xa3,0xcc,0x82,0x00,0x10,0x0a,0x01,0xff,0x41,0xcc,0x86,0xcc,0x81, + 0x00,0x01,0xff,0x61,0xcc,0x86,0xcc,0x81,0x00,0xd3,0x50,0xd2,0x28,0xd1,0x14,0x10, + 0x0a,0x01,0xff,0x41,0xcc,0x86,0xcc,0x80,0x00,0x01,0xff,0x61,0xcc,0x86,0xcc,0x80, + 0x00,0x10,0x0a,0x01,0xff,0x41,0xcc,0x86,0xcc,0x89,0x00,0x01,0xff,0x61,0xcc,0x86, + 0xcc,0x89,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x41,0xcc,0x86,0xcc,0x83,0x00,0x01, + 0xff,0x61,0xcc,0x86,0xcc,0x83,0x00,0x10,0x0a,0x01,0xff,0x41,0xcc,0xa3,0xcc,0x86, + 0x00,0x01,0xff,0x61,0xcc,0xa3,0xcc,0x86,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01, + 0xff,0x45,0xcc,0xa3,0x00,0x01,0xff,0x65,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x45, + 0xcc,0x89,0x00,0x01,0xff,0x65,0xcc,0x89,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x45, + 0xcc,0x83,0x00,0x01,0xff,0x65,0xcc,0x83,0x00,0x10,0x0a,0x01,0xff,0x45,0xcc,0x82, + 0xcc,0x81,0x00,0x01,0xff,0x65,0xcc,0x82,0xcc,0x81,0x00,0xcf,0x86,0xe5,0x31,0x01, + 0xd4,0x90,0xd3,0x50,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x45,0xcc,0x82,0xcc, + 0x80,0x00,0x01,0xff,0x65,0xcc,0x82,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0x45,0xcc, + 0x82,0xcc,0x89,0x00,0x01,0xff,0x65,0xcc,0x82,0xcc,0x89,0x00,0xd1,0x14,0x10,0x0a, + 0x01,0xff,0x45,0xcc,0x82,0xcc,0x83,0x00,0x01,0xff,0x65,0xcc,0x82,0xcc,0x83,0x00, + 0x10,0x0a,0x01,0xff,0x45,0xcc,0xa3,0xcc,0x82,0x00,0x01,0xff,0x65,0xcc,0xa3,0xcc, + 0x82,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0x49,0xcc,0x89,0x00,0x01,0xff, + 0x69,0xcc,0x89,0x00,0x10,0x08,0x01,0xff,0x49,0xcc,0xa3,0x00,0x01,0xff,0x69,0xcc, + 0xa3,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0x4f,0xcc,0xa3,0x00,0x01,0xff,0x6f,0xcc, + 0xa3,0x00,0x10,0x08,0x01,0xff,0x4f,0xcc,0x89,0x00,0x01,0xff,0x6f,0xcc,0x89,0x00, + 0xd3,0x50,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x4f,0xcc,0x82,0xcc,0x81,0x00, + 0x01,0xff,0x6f,0xcc,0x82,0xcc,0x81,0x00,0x10,0x0a,0x01,0xff,0x4f,0xcc,0x82,0xcc, + 0x80,0x00,0x01,0xff,0x6f,0xcc,0x82,0xcc,0x80,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff, + 0x4f,0xcc,0x82,0xcc,0x89,0x00,0x01,0xff,0x6f,0xcc,0x82,0xcc,0x89,0x00,0x10,0x0a, + 0x01,0xff,0x4f,0xcc,0x82,0xcc,0x83,0x00,0x01,0xff,0x6f,0xcc,0x82,0xcc,0x83,0x00, + 0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x4f,0xcc,0xa3,0xcc,0x82,0x00,0x01,0xff, + 0x6f,0xcc,0xa3,0xcc,0x82,0x00,0x10,0x0a,0x01,0xff,0x4f,0xcc,0x9b,0xcc,0x81,0x00, + 0x01,0xff,0x6f,0xcc,0x9b,0xcc,0x81,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0x4f,0xcc, + 0x9b,0xcc,0x80,0x00,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff, + 0x4f,0xcc,0x9b,0xcc,0x89,0x00,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0x89,0x00,0xd4,0x98, + 0xd3,0x48,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0x4f,0xcc,0x9b,0xcc,0x83,0x00, + 0x01,0xff,0x6f,0xcc,0x9b,0xcc,0x83,0x00,0x10,0x0a,0x01,0xff,0x4f,0xcc,0x9b,0xcc, + 0xa3,0x00,0x01,0xff,0x6f,0xcc,0x9b,0xcc,0xa3,0x00,0xd1,0x10,0x10,0x08,0x01,0xff, + 0x55,0xcc,0xa3,0x00,0x01,0xff,0x75,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff,0x55,0xcc, + 0x89,0x00,0x01,0xff,0x75,0xcc,0x89,0x00,0xd2,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff, + 0x55,0xcc,0x9b,0xcc,0x81,0x00,0x01,0xff,0x75,0xcc,0x9b,0xcc,0x81,0x00,0x10,0x0a, + 0x01,0xff,0x55,0xcc,0x9b,0xcc,0x80,0x00,0x01,0xff,0x75,0xcc,0x9b,0xcc,0x80,0x00, + 0xd1,0x14,0x10,0x0a,0x01,0xff,0x55,0xcc,0x9b,0xcc,0x89,0x00,0x01,0xff,0x75,0xcc, + 0x9b,0xcc,0x89,0x00,0x10,0x0a,0x01,0xff,0x55,0xcc,0x9b,0xcc,0x83,0x00,0x01,0xff, + 0x75,0xcc,0x9b,0xcc,0x83,0x00,0xd3,0x44,0xd2,0x24,0xd1,0x14,0x10,0x0a,0x01,0xff, + 0x55,0xcc,0x9b,0xcc,0xa3,0x00,0x01,0xff,0x75,0xcc,0x9b,0xcc,0xa3,0x00,0x10,0x08, + 0x01,0xff,0x59,0xcc,0x80,0x00,0x01,0xff,0x79,0xcc,0x80,0x00,0xd1,0x10,0x10,0x08, + 0x01,0xff,0x59,0xcc,0xa3,0x00,0x01,0xff,0x79,0xcc,0xa3,0x00,0x10,0x08,0x01,0xff, + 0x59,0xcc,0x89,0x00,0x01,0xff,0x79,0xcc,0x89,0x00,0x92,0x14,0x91,0x10,0x10,0x08, + 0x01,0xff,0x59,0xcc,0x83,0x00,0x01,0xff,0x79,0xcc,0x83,0x00,0x0a,0x00,0x0a,0x00, + 0xe1,0xc0,0x04,0xe0,0x80,0x02,0xcf,0x86,0xe5,0x2d,0x01,0xd4,0xa8,0xd3,0x54,0xd2, + 0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb1,0xcc,0x93,0x00,0x01,0xff,0xce,0xb1, + 0xcc,0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0xb1,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff, + 0xce,0xb1,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xb1,0xcc, + 0x93,0xcc,0x81,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01, + 0xff,0xce,0xb1,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xcd,0x82, + 0x00,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0x91,0xcc,0x93,0x00,0x01,0xff, + 0xce,0x91,0xcc,0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0x91,0xcc,0x93,0xcc,0x80,0x00, + 0x01,0xff,0xce,0x91,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce, + 0x91,0xcc,0x93,0xcc,0x81,0x00,0x01,0xff,0xce,0x91,0xcc,0x94,0xcc,0x81,0x00,0x10, + 0x0b,0x01,0xff,0xce,0x91,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0x91,0xcc,0x94, + 0xcd,0x82,0x00,0xd3,0x42,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb5,0xcc, + 0x93,0x00,0x01,0xff,0xce,0xb5,0xcc,0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0xb5,0xcc, + 0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0xb5,0xcc,0x94,0xcc,0x80,0x00,0x91,0x16,0x10, + 0x0b,0x01,0xff,0xce,0xb5,0xcc,0x93,0xcc,0x81,0x00,0x01,0xff,0xce,0xb5,0xcc,0x94, + 0xcc,0x81,0x00,0x00,0x00,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0x95,0xcc, + 0x93,0x00,0x01,0xff,0xce,0x95,0xcc,0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0x95,0xcc, + 0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0x95,0xcc,0x94,0xcc,0x80,0x00,0x91,0x16,0x10, + 0x0b,0x01,0xff,0xce,0x95,0xcc,0x93,0xcc,0x81,0x00,0x01,0xff,0xce,0x95,0xcc,0x94, + 0xcc,0x81,0x00,0x00,0x00,0xd4,0xa8,0xd3,0x54,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01, + 0xff,0xce,0xb7,0xcc,0x93,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0x00,0x10,0x0b,0x01, + 0xff,0xce,0xb7,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcc,0x80, + 0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcc,0x81,0x00,0x01,0xff, + 0xce,0xb7,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcd, + 0x82,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcd,0x82,0x00,0xd2,0x28,0xd1,0x12,0x10, + 0x09,0x01,0xff,0xce,0x97,0xcc,0x93,0x00,0x01,0xff,0xce,0x97,0xcc,0x94,0x00,0x10, + 0x0b,0x01,0xff,0xce,0x97,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0x97,0xcc,0x94, + 0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0x97,0xcc,0x93,0xcc,0x81,0x00, + 0x01,0xff,0xce,0x97,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01,0xff,0xce,0x97,0xcc, + 0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0x97,0xcc,0x94,0xcd,0x82,0x00,0xd3,0x54,0xd2, + 0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb9,0xcc,0x93,0x00,0x01,0xff,0xce,0xb9, + 0xcc,0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0xb9,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff, + 0xce,0xb9,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xb9,0xcc, + 0x93,0xcc,0x81,0x00,0x01,0xff,0xce,0xb9,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01, + 0xff,0xce,0xb9,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0xb9,0xcc,0x94,0xcd,0x82, + 0x00,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0x99,0xcc,0x93,0x00,0x01,0xff, + 0xce,0x99,0xcc,0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0x99,0xcc,0x93,0xcc,0x80,0x00, + 0x01,0xff,0xce,0x99,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce, + 0x99,0xcc,0x93,0xcc,0x81,0x00,0x01,0xff,0xce,0x99,0xcc,0x94,0xcc,0x81,0x00,0x10, + 0x0b,0x01,0xff,0xce,0x99,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0x99,0xcc,0x94, + 0xcd,0x82,0x00,0xcf,0x86,0xe5,0x13,0x01,0xd4,0x84,0xd3,0x42,0xd2,0x28,0xd1,0x12, + 0x10,0x09,0x01,0xff,0xce,0xbf,0xcc,0x93,0x00,0x01,0xff,0xce,0xbf,0xcc,0x94,0x00, + 0x10,0x0b,0x01,0xff,0xce,0xbf,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0xbf,0xcc, + 0x94,0xcc,0x80,0x00,0x91,0x16,0x10,0x0b,0x01,0xff,0xce,0xbf,0xcc,0x93,0xcc,0x81, + 0x00,0x01,0xff,0xce,0xbf,0xcc,0x94,0xcc,0x81,0x00,0x00,0x00,0xd2,0x28,0xd1,0x12, + 0x10,0x09,0x01,0xff,0xce,0x9f,0xcc,0x93,0x00,0x01,0xff,0xce,0x9f,0xcc,0x94,0x00, + 0x10,0x0b,0x01,0xff,0xce,0x9f,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xce,0x9f,0xcc, + 0x94,0xcc,0x80,0x00,0x91,0x16,0x10,0x0b,0x01,0xff,0xce,0x9f,0xcc,0x93,0xcc,0x81, + 0x00,0x01,0xff,0xce,0x9f,0xcc,0x94,0xcc,0x81,0x00,0x00,0x00,0xd3,0x54,0xd2,0x28, + 0xd1,0x12,0x10,0x09,0x01,0xff,0xcf,0x85,0xcc,0x93,0x00,0x01,0xff,0xcf,0x85,0xcc, + 0x94,0x00,0x10,0x0b,0x01,0xff,0xcf,0x85,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xcf, + 0x85,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xcf,0x85,0xcc,0x93, + 0xcc,0x81,0x00,0x01,0xff,0xcf,0x85,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01,0xff, + 0xcf,0x85,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xcf,0x85,0xcc,0x94,0xcd,0x82,0x00, + 0xd2,0x1c,0xd1,0x0d,0x10,0x04,0x00,0x00,0x01,0xff,0xce,0xa5,0xcc,0x94,0x00,0x10, + 0x04,0x00,0x00,0x01,0xff,0xce,0xa5,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x0f,0x10,0x04, + 0x00,0x00,0x01,0xff,0xce,0xa5,0xcc,0x94,0xcc,0x81,0x00,0x10,0x04,0x00,0x00,0x01, + 0xff,0xce,0xa5,0xcc,0x94,0xcd,0x82,0x00,0xd4,0xa8,0xd3,0x54,0xd2,0x28,0xd1,0x12, + 0x10,0x09,0x01,0xff,0xcf,0x89,0xcc,0x93,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0x00, + 0x10,0x0b,0x01,0xff,0xcf,0x89,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xcf,0x89,0xcc, + 0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xcf,0x89,0xcc,0x93,0xcc,0x81, + 0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01,0xff,0xcf,0x89, + 0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcd,0x82,0x00,0xd2,0x28, + 0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xa9,0xcc,0x93,0x00,0x01,0xff,0xce,0xa9,0xcc, + 0x94,0x00,0x10,0x0b,0x01,0xff,0xce,0xa9,0xcc,0x93,0xcc,0x80,0x00,0x01,0xff,0xce, + 0xa9,0xcc,0x94,0xcc,0x80,0x00,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xa9,0xcc,0x93, + 0xcc,0x81,0x00,0x01,0xff,0xce,0xa9,0xcc,0x94,0xcc,0x81,0x00,0x10,0x0b,0x01,0xff, + 0xce,0xa9,0xcc,0x93,0xcd,0x82,0x00,0x01,0xff,0xce,0xa9,0xcc,0x94,0xcd,0x82,0x00, + 0xd3,0x48,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb1,0xcc,0x80,0x00,0x01, + 0xff,0xce,0xb1,0xcc,0x81,0x00,0x10,0x09,0x01,0xff,0xce,0xb5,0xcc,0x80,0x00,0x01, + 0xff,0xce,0xb5,0xcc,0x81,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb7,0xcc,0x80, + 0x00,0x01,0xff,0xce,0xb7,0xcc,0x81,0x00,0x10,0x09,0x01,0xff,0xce,0xb9,0xcc,0x80, + 0x00,0x01,0xff,0xce,0xb9,0xcc,0x81,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff, + 0xce,0xbf,0xcc,0x80,0x00,0x01,0xff,0xce,0xbf,0xcc,0x81,0x00,0x10,0x09,0x01,0xff, + 0xcf,0x85,0xcc,0x80,0x00,0x01,0xff,0xcf,0x85,0xcc,0x81,0x00,0x91,0x12,0x10,0x09, + 0x01,0xff,0xcf,0x89,0xcc,0x80,0x00,0x01,0xff,0xcf,0x89,0xcc,0x81,0x00,0x00,0x00, + 0xe0,0xe1,0x02,0xcf,0x86,0xe5,0x91,0x01,0xd4,0xc8,0xd3,0x64,0xd2,0x30,0xd1,0x16, + 0x10,0x0b,0x01,0xff,0xce,0xb1,0xcc,0x93,0xcd,0x85,0x00,0x01,0xff,0xce,0xb1,0xcc, + 0x94,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff,0xce,0xb1,0xcc,0x93,0xcc,0x80,0xcd,0x85, + 0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xcc,0x80,0xcd,0x85,0x00,0xd1,0x1a,0x10,0x0d, + 0x01,0xff,0xce,0xb1,0xcc,0x93,0xcc,0x81,0xcd,0x85,0x00,0x01,0xff,0xce,0xb1,0xcc, + 0x94,0xcc,0x81,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff,0xce,0xb1,0xcc,0x93,0xcd,0x82, + 0xcd,0x85,0x00,0x01,0xff,0xce,0xb1,0xcc,0x94,0xcd,0x82,0xcd,0x85,0x00,0xd2,0x30, + 0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0x91,0xcc,0x93,0xcd,0x85,0x00,0x01,0xff,0xce, + 0x91,0xcc,0x94,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff,0xce,0x91,0xcc,0x93,0xcc,0x80, + 0xcd,0x85,0x00,0x01,0xff,0xce,0x91,0xcc,0x94,0xcc,0x80,0xcd,0x85,0x00,0xd1,0x1a, + 0x10,0x0d,0x01,0xff,0xce,0x91,0xcc,0x93,0xcc,0x81,0xcd,0x85,0x00,0x01,0xff,0xce, + 0x91,0xcc,0x94,0xcc,0x81,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff,0xce,0x91,0xcc,0x93, + 0xcd,0x82,0xcd,0x85,0x00,0x01,0xff,0xce,0x91,0xcc,0x94,0xcd,0x82,0xcd,0x85,0x00, + 0xd3,0x64,0xd2,0x30,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcd,0x85, + 0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff,0xce,0xb7, + 0xcc,0x93,0xcc,0x80,0xcd,0x85,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcc,0x80,0xcd, + 0x85,0x00,0xd1,0x1a,0x10,0x0d,0x01,0xff,0xce,0xb7,0xcc,0x93,0xcc,0x81,0xcd,0x85, + 0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcc,0x81,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff, + 0xce,0xb7,0xcc,0x93,0xcd,0x82,0xcd,0x85,0x00,0x01,0xff,0xce,0xb7,0xcc,0x94,0xcd, + 0x82,0xcd,0x85,0x00,0xd2,0x30,0xd1,0x16,0x10,0x0b,0x01,0xff,0xce,0x97,0xcc,0x93, + 0xcd,0x85,0x00,0x01,0xff,0xce,0x97,0xcc,0x94,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff, + 0xce,0x97,0xcc,0x93,0xcc,0x80,0xcd,0x85,0x00,0x01,0xff,0xce,0x97,0xcc,0x94,0xcc, + 0x80,0xcd,0x85,0x00,0xd1,0x1a,0x10,0x0d,0x01,0xff,0xce,0x97,0xcc,0x93,0xcc,0x81, + 0xcd,0x85,0x00,0x01,0xff,0xce,0x97,0xcc,0x94,0xcc,0x81,0xcd,0x85,0x00,0x10,0x0d, + 0x01,0xff,0xce,0x97,0xcc,0x93,0xcd,0x82,0xcd,0x85,0x00,0x01,0xff,0xce,0x97,0xcc, + 0x94,0xcd,0x82,0xcd,0x85,0x00,0xd4,0xc8,0xd3,0x64,0xd2,0x30,0xd1,0x16,0x10,0x0b, + 0x01,0xff,0xcf,0x89,0xcc,0x93,0xcd,0x85,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcd, + 0x85,0x00,0x10,0x0d,0x01,0xff,0xcf,0x89,0xcc,0x93,0xcc,0x80,0xcd,0x85,0x00,0x01, + 0xff,0xcf,0x89,0xcc,0x94,0xcc,0x80,0xcd,0x85,0x00,0xd1,0x1a,0x10,0x0d,0x01,0xff, + 0xcf,0x89,0xcc,0x93,0xcc,0x81,0xcd,0x85,0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcc, + 0x81,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff,0xcf,0x89,0xcc,0x93,0xcd,0x82,0xcd,0x85, + 0x00,0x01,0xff,0xcf,0x89,0xcc,0x94,0xcd,0x82,0xcd,0x85,0x00,0xd2,0x30,0xd1,0x16, + 0x10,0x0b,0x01,0xff,0xce,0xa9,0xcc,0x93,0xcd,0x85,0x00,0x01,0xff,0xce,0xa9,0xcc, + 0x94,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff,0xce,0xa9,0xcc,0x93,0xcc,0x80,0xcd,0x85, + 0x00,0x01,0xff,0xce,0xa9,0xcc,0x94,0xcc,0x80,0xcd,0x85,0x00,0xd1,0x1a,0x10,0x0d, + 0x01,0xff,0xce,0xa9,0xcc,0x93,0xcc,0x81,0xcd,0x85,0x00,0x01,0xff,0xce,0xa9,0xcc, + 0x94,0xcc,0x81,0xcd,0x85,0x00,0x10,0x0d,0x01,0xff,0xce,0xa9,0xcc,0x93,0xcd,0x82, + 0xcd,0x85,0x00,0x01,0xff,0xce,0xa9,0xcc,0x94,0xcd,0x82,0xcd,0x85,0x00,0xd3,0x49, + 0xd2,0x26,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0xb1,0xcc,0x86,0x00,0x01,0xff,0xce, + 0xb1,0xcc,0x84,0x00,0x10,0x0b,0x01,0xff,0xce,0xb1,0xcc,0x80,0xcd,0x85,0x00,0x01, + 0xff,0xce,0xb1,0xcd,0x85,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xce,0xb1,0xcc,0x81, + 0xcd,0x85,0x00,0x00,0x00,0x10,0x09,0x01,0xff,0xce,0xb1,0xcd,0x82,0x00,0x01,0xff, + 0xce,0xb1,0xcd,0x82,0xcd,0x85,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xce, + 0x91,0xcc,0x86,0x00,0x01,0xff,0xce,0x91,0xcc,0x84,0x00,0x10,0x09,0x01,0xff,0xce, + 0x91,0xcc,0x80,0x00,0x01,0xff,0xce,0x91,0xcc,0x81,0x00,0xd1,0x0d,0x10,0x09,0x01, + 0xff,0xce,0x91,0xcd,0x85,0x00,0x01,0x00,0x10,0x07,0x01,0xff,0xce,0xb9,0x00,0x01, + 0x00,0xcf,0x86,0xe5,0x16,0x01,0xd4,0x8f,0xd3,0x44,0xd2,0x21,0xd1,0x0d,0x10,0x04, + 0x01,0x00,0x01,0xff,0xc2,0xa8,0xcd,0x82,0x00,0x10,0x0b,0x01,0xff,0xce,0xb7,0xcc, + 0x80,0xcd,0x85,0x00,0x01,0xff,0xce,0xb7,0xcd,0x85,0x00,0xd1,0x0f,0x10,0x0b,0x01, + 0xff,0xce,0xb7,0xcc,0x81,0xcd,0x85,0x00,0x00,0x00,0x10,0x09,0x01,0xff,0xce,0xb7, + 0xcd,0x82,0x00,0x01,0xff,0xce,0xb7,0xcd,0x82,0xcd,0x85,0x00,0xd2,0x24,0xd1,0x12, + 0x10,0x09,0x01,0xff,0xce,0x95,0xcc,0x80,0x00,0x01,0xff,0xce,0x95,0xcc,0x81,0x00, + 0x10,0x09,0x01,0xff,0xce,0x97,0xcc,0x80,0x00,0x01,0xff,0xce,0x97,0xcc,0x81,0x00, + 0xd1,0x13,0x10,0x09,0x01,0xff,0xce,0x97,0xcd,0x85,0x00,0x01,0xff,0xe1,0xbe,0xbf, + 0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0xe1,0xbe,0xbf,0xcc,0x81,0x00,0x01,0xff,0xe1, + 0xbe,0xbf,0xcd,0x82,0x00,0xd3,0x40,0xd2,0x28,0xd1,0x12,0x10,0x09,0x01,0xff,0xce, + 0xb9,0xcc,0x86,0x00,0x01,0xff,0xce,0xb9,0xcc,0x84,0x00,0x10,0x0b,0x01,0xff,0xce, + 0xb9,0xcc,0x88,0xcc,0x80,0x00,0x01,0xff,0xce,0xb9,0xcc,0x88,0xcc,0x81,0x00,0x51, + 0x04,0x00,0x00,0x10,0x09,0x01,0xff,0xce,0xb9,0xcd,0x82,0x00,0x01,0xff,0xce,0xb9, + 0xcc,0x88,0xcd,0x82,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xce,0x99,0xcc, + 0x86,0x00,0x01,0xff,0xce,0x99,0xcc,0x84,0x00,0x10,0x09,0x01,0xff,0xce,0x99,0xcc, + 0x80,0x00,0x01,0xff,0xce,0x99,0xcc,0x81,0x00,0xd1,0x0e,0x10,0x04,0x00,0x00,0x01, + 0xff,0xe1,0xbf,0xbe,0xcc,0x80,0x00,0x10,0x0a,0x01,0xff,0xe1,0xbf,0xbe,0xcc,0x81, + 0x00,0x01,0xff,0xe1,0xbf,0xbe,0xcd,0x82,0x00,0xd4,0x93,0xd3,0x4e,0xd2,0x28,0xd1, + 0x12,0x10,0x09,0x01,0xff,0xcf,0x85,0xcc,0x86,0x00,0x01,0xff,0xcf,0x85,0xcc,0x84, + 0x00,0x10,0x0b,0x01,0xff,0xcf,0x85,0xcc,0x88,0xcc,0x80,0x00,0x01,0xff,0xcf,0x85, + 0xcc,0x88,0xcc,0x81,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xcf,0x81,0xcc,0x93,0x00, + 0x01,0xff,0xcf,0x81,0xcc,0x94,0x00,0x10,0x09,0x01,0xff,0xcf,0x85,0xcd,0x82,0x00, + 0x01,0xff,0xcf,0x85,0xcc,0x88,0xcd,0x82,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01, + 0xff,0xce,0xa5,0xcc,0x86,0x00,0x01,0xff,0xce,0xa5,0xcc,0x84,0x00,0x10,0x09,0x01, + 0xff,0xce,0xa5,0xcc,0x80,0x00,0x01,0xff,0xce,0xa5,0xcc,0x81,0x00,0xd1,0x12,0x10, + 0x09,0x01,0xff,0xce,0xa1,0xcc,0x94,0x00,0x01,0xff,0xc2,0xa8,0xcc,0x80,0x00,0x10, + 0x09,0x01,0xff,0xc2,0xa8,0xcc,0x81,0x00,0x01,0xff,0x60,0x00,0xd3,0x3b,0xd2,0x18, + 0x51,0x04,0x00,0x00,0x10,0x0b,0x01,0xff,0xcf,0x89,0xcc,0x80,0xcd,0x85,0x00,0x01, + 0xff,0xcf,0x89,0xcd,0x85,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xcf,0x89,0xcc,0x81, + 0xcd,0x85,0x00,0x00,0x00,0x10,0x09,0x01,0xff,0xcf,0x89,0xcd,0x82,0x00,0x01,0xff, + 0xcf,0x89,0xcd,0x82,0xcd,0x85,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xce, + 0x9f,0xcc,0x80,0x00,0x01,0xff,0xce,0x9f,0xcc,0x81,0x00,0x10,0x09,0x01,0xff,0xce, + 0xa9,0xcc,0x80,0x00,0x01,0xff,0xce,0xa9,0xcc,0x81,0x00,0xd1,0x10,0x10,0x09,0x01, + 0xff,0xce,0xa9,0xcd,0x85,0x00,0x01,0xff,0xc2,0xb4,0x00,0x10,0x04,0x01,0x00,0x00, + 0x00,0xe0,0x7e,0x0c,0xcf,0x86,0xe5,0xbb,0x08,0xe4,0x14,0x06,0xe3,0xf7,0x02,0xe2, + 0xbd,0x01,0xd1,0xd0,0xd0,0x4f,0xcf,0x86,0xd5,0x2e,0x94,0x2a,0xd3,0x18,0x92,0x14, + 0x91,0x10,0x10,0x08,0x01,0xff,0xe2,0x80,0x82,0x00,0x01,0xff,0xe2,0x80,0x83,0x00, + 0x01,0x00,0x01,0x00,0x92,0x0d,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xff, + 0x00,0x01,0xff,0x00,0x01,0x00,0x94,0x1b,0x53,0x04,0x01,0x00,0xd2,0x09,0x11,0x04, + 0x01,0x00,0x01,0xff,0x00,0x51,0x05,0x01,0xff,0x00,0x10,0x05,0x01,0xff,0x00,0x04, + 0x00,0x01,0x00,0xcf,0x86,0xd5,0x48,0xd4,0x1c,0xd3,0x10,0x52,0x04,0x01,0x00,0x51, + 0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x06,0x00,0x52,0x04,0x04,0x00,0x11,0x04,0x04, + 0x00,0x06,0x00,0xd3,0x1c,0xd2,0x0c,0x51,0x04,0x06,0x00,0x10,0x04,0x06,0x00,0x07, + 0x00,0xd1,0x08,0x10,0x04,0x07,0x00,0x08,0x00,0x10,0x04,0x08,0x00,0x06,0x00,0x52, + 0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x06,0x00,0xd4,0x23,0xd3, + 0x14,0x52,0x05,0x06,0xff,0x00,0x91,0x0a,0x10,0x05,0x0a,0xff,0x00,0x00,0xff,0x00, + 0x0f,0xff,0x00,0x92,0x0a,0x11,0x05,0x0f,0xff,0x00,0x01,0xff,0x00,0x01,0xff,0x00, + 0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x06,0x00,0x00,0x00,0x01,0x00, + 0x01,0x00,0xd0,0x7e,0xcf,0x86,0xd5,0x34,0xd4,0x14,0x53,0x04,0x01,0x00,0x52,0x04, + 0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0xd3,0x10,0x52,0x04, + 0x08,0x00,0x91,0x08,0x10,0x04,0x08,0x00,0x0c,0x00,0x0c,0x00,0x52,0x04,0x0c,0x00, + 0x91,0x08,0x10,0x04,0x0c,0x00,0x00,0x00,0x00,0x00,0xd4,0x1c,0x53,0x04,0x01,0x00, + 0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x02,0x00,0x91,0x08,0x10,0x04, + 0x03,0x00,0x04,0x00,0x04,0x00,0xd3,0x10,0xd2,0x08,0x11,0x04,0x06,0x00,0x08,0x00, + 0x11,0x04,0x08,0x00,0x0b,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x0b,0x00,0x0c,0x00, + 0x10,0x04,0x0e,0x00,0x10,0x00,0x51,0x04,0x10,0x00,0x10,0x04,0x11,0x00,0x13,0x00, + 0xcf,0x86,0xd5,0x28,0x54,0x04,0x00,0x00,0xd3,0x0c,0x92,0x08,0x11,0x04,0x01,0xe6, + 0x01,0x01,0x01,0xe6,0xd2,0x0c,0x51,0x04,0x01,0x01,0x10,0x04,0x01,0x01,0x01,0xe6, + 0x91,0x08,0x10,0x04,0x01,0xe6,0x01,0x00,0x01,0x00,0xd4,0x30,0xd3,0x1c,0xd2,0x0c, + 0x91,0x08,0x10,0x04,0x01,0x00,0x01,0xe6,0x04,0x00,0xd1,0x08,0x10,0x04,0x06,0x00, + 0x06,0x01,0x10,0x04,0x06,0x01,0x06,0xe6,0x92,0x10,0xd1,0x08,0x10,0x04,0x06,0xdc, + 0x06,0xe6,0x10,0x04,0x06,0x01,0x08,0x01,0x09,0xdc,0x93,0x10,0x92,0x0c,0x91,0x08, + 0x10,0x04,0x0a,0xe6,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd1,0x81,0xd0,0x4f, + 0xcf,0x86,0x55,0x04,0x01,0x00,0xd4,0x29,0xd3,0x13,0x52,0x04,0x01,0x00,0x51,0x04, + 0x01,0x00,0x10,0x07,0x01,0xff,0xce,0xa9,0x00,0x01,0x00,0x92,0x12,0x51,0x04,0x01, + 0x00,0x10,0x06,0x01,0xff,0x4b,0x00,0x01,0xff,0x41,0xcc,0x8a,0x00,0x01,0x00,0x53, + 0x04,0x01,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x01,0x00,0x04,0x00,0x10,0x04,0x04, + 0x00,0x07,0x00,0x91,0x08,0x10,0x04,0x08,0x00,0x06,0x00,0x06,0x00,0xcf,0x86,0x95, + 0x2c,0xd4,0x18,0x53,0x04,0x06,0x00,0x52,0x04,0x06,0x00,0xd1,0x08,0x10,0x04,0x08, + 0x00,0x09,0x00,0x10,0x04,0x09,0x00,0x0a,0x00,0x93,0x10,0x92,0x0c,0x51,0x04,0x0b, + 0x00,0x10,0x04,0x0b,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd0,0x68,0xcf, + 0x86,0xd5,0x48,0xd4,0x28,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01, + 0x00,0x04,0x00,0x91,0x08,0x10,0x04,0x09,0x00,0x0a,0x00,0x0a,0x00,0x92,0x0c,0x91, + 0x08,0x10,0x04,0x0a,0x00,0x0b,0x00,0x11,0x00,0x00,0x00,0x53,0x04,0x01,0x00,0x92, + 0x18,0x51,0x04,0x01,0x00,0x10,0x0a,0x01,0xff,0xe2,0x86,0x90,0xcc,0xb8,0x00,0x01, + 0xff,0xe2,0x86,0x92,0xcc,0xb8,0x00,0x01,0x00,0x94,0x1a,0x53,0x04,0x01,0x00,0x52, + 0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x0a,0x01,0xff,0xe2,0x86,0x94,0xcc,0xb8, + 0x00,0x01,0x00,0x01,0x00,0xcf,0x86,0xd5,0x2e,0x94,0x2a,0x53,0x04,0x01,0x00,0x52, + 0x04,0x01,0x00,0xd1,0x0e,0x10,0x04,0x01,0x00,0x01,0xff,0xe2,0x87,0x90,0xcc,0xb8, + 0x00,0x10,0x0a,0x01,0xff,0xe2,0x87,0x94,0xcc,0xb8,0x00,0x01,0xff,0xe2,0x87,0x92, + 0xcc,0xb8,0x00,0x01,0x00,0xd4,0x14,0x53,0x04,0x01,0x00,0x92,0x0c,0x51,0x04,0x01, + 0x00,0x10,0x04,0x01,0x00,0x04,0x00,0x04,0x00,0x93,0x08,0x12,0x04,0x04,0x00,0x06, + 0x00,0x06,0x00,0xe2,0x38,0x02,0xe1,0x3f,0x01,0xd0,0x68,0xcf,0x86,0xd5,0x3e,0x94, + 0x3a,0xd3,0x16,0x52,0x04,0x01,0x00,0x91,0x0e,0x10,0x0a,0x01,0xff,0xe2,0x88,0x83, + 0xcc,0xb8,0x00,0x01,0x00,0x01,0x00,0xd2,0x12,0x91,0x0e,0x10,0x04,0x01,0x00,0x01, + 0xff,0xe2,0x88,0x88,0xcc,0xb8,0x00,0x01,0x00,0x91,0x0e,0x10,0x0a,0x01,0xff,0xe2, + 0x88,0x8b,0xcc,0xb8,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x94,0x24,0x93,0x20,0x52, + 0x04,0x01,0x00,0xd1,0x0e,0x10,0x0a,0x01,0xff,0xe2,0x88,0xa3,0xcc,0xb8,0x00,0x01, + 0x00,0x10,0x0a,0x01,0xff,0xe2,0x88,0xa5,0xcc,0xb8,0x00,0x01,0x00,0x01,0x00,0x01, + 0x00,0xcf,0x86,0xd5,0x48,0x94,0x44,0xd3,0x2e,0xd2,0x12,0x91,0x0e,0x10,0x04,0x01, + 0x00,0x01,0xff,0xe2,0x88,0xbc,0xcc,0xb8,0x00,0x01,0x00,0xd1,0x0e,0x10,0x0a,0x01, + 0xff,0xe2,0x89,0x83,0xcc,0xb8,0x00,0x01,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0xe2, + 0x89,0x85,0xcc,0xb8,0x00,0x92,0x12,0x91,0x0e,0x10,0x04,0x01,0x00,0x01,0xff,0xe2, + 0x89,0x88,0xcc,0xb8,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4,0x40,0xd3,0x1e,0x92, + 0x1a,0xd1,0x0c,0x10,0x08,0x01,0xff,0x3d,0xcc,0xb8,0x00,0x01,0x00,0x10,0x0a,0x01, + 0xff,0xe2,0x89,0xa1,0xcc,0xb8,0x00,0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00,0xd1, + 0x0e,0x10,0x04,0x01,0x00,0x01,0xff,0xe2,0x89,0x8d,0xcc,0xb8,0x00,0x10,0x08,0x01, + 0xff,0x3c,0xcc,0xb8,0x00,0x01,0xff,0x3e,0xcc,0xb8,0x00,0xd3,0x30,0xd2,0x18,0x91, + 0x14,0x10,0x0a,0x01,0xff,0xe2,0x89,0xa4,0xcc,0xb8,0x00,0x01,0xff,0xe2,0x89,0xa5, + 0xcc,0xb8,0x00,0x01,0x00,0x91,0x14,0x10,0x0a,0x01,0xff,0xe2,0x89,0xb2,0xcc,0xb8, + 0x00,0x01,0xff,0xe2,0x89,0xb3,0xcc,0xb8,0x00,0x01,0x00,0x92,0x18,0x91,0x14,0x10, + 0x0a,0x01,0xff,0xe2,0x89,0xb6,0xcc,0xb8,0x00,0x01,0xff,0xe2,0x89,0xb7,0xcc,0xb8, + 0x00,0x01,0x00,0x01,0x00,0xd0,0x86,0xcf,0x86,0xd5,0x50,0x94,0x4c,0xd3,0x30,0xd2, + 0x18,0x91,0x14,0x10,0x0a,0x01,0xff,0xe2,0x89,0xba,0xcc,0xb8,0x00,0x01,0xff,0xe2, + 0x89,0xbb,0xcc,0xb8,0x00,0x01,0x00,0x91,0x14,0x10,0x0a,0x01,0xff,0xe2,0x8a,0x82, + 0xcc,0xb8,0x00,0x01,0xff,0xe2,0x8a,0x83,0xcc,0xb8,0x00,0x01,0x00,0x92,0x18,0x91, + 0x14,0x10,0x0a,0x01,0xff,0xe2,0x8a,0x86,0xcc,0xb8,0x00,0x01,0xff,0xe2,0x8a,0x87, + 0xcc,0xb8,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x94,0x30,0x53,0x04,0x01,0x00,0x52, + 0x04,0x01,0x00,0xd1,0x14,0x10,0x0a,0x01,0xff,0xe2,0x8a,0xa2,0xcc,0xb8,0x00,0x01, + 0xff,0xe2,0x8a,0xa8,0xcc,0xb8,0x00,0x10,0x0a,0x01,0xff,0xe2,0x8a,0xa9,0xcc,0xb8, + 0x00,0x01,0xff,0xe2,0x8a,0xab,0xcc,0xb8,0x00,0x01,0x00,0xcf,0x86,0x55,0x04,0x01, + 0x00,0xd4,0x5c,0xd3,0x2c,0x92,0x28,0xd1,0x14,0x10,0x0a,0x01,0xff,0xe2,0x89,0xbc, + 0xcc,0xb8,0x00,0x01,0xff,0xe2,0x89,0xbd,0xcc,0xb8,0x00,0x10,0x0a,0x01,0xff,0xe2, + 0x8a,0x91,0xcc,0xb8,0x00,0x01,0xff,0xe2,0x8a,0x92,0xcc,0xb8,0x00,0x01,0x00,0xd2, + 0x18,0x51,0x04,0x01,0x00,0x10,0x0a,0x01,0xff,0xe2,0x8a,0xb2,0xcc,0xb8,0x00,0x01, + 0xff,0xe2,0x8a,0xb3,0xcc,0xb8,0x00,0x91,0x14,0x10,0x0a,0x01,0xff,0xe2,0x8a,0xb4, + 0xcc,0xb8,0x00,0x01,0xff,0xe2,0x8a,0xb5,0xcc,0xb8,0x00,0x01,0x00,0x93,0x0c,0x92, + 0x08,0x11,0x04,0x01,0x00,0x06,0x00,0x06,0x00,0x06,0x00,0xd1,0x64,0xd0,0x3e,0xcf, + 0x86,0xd5,0x18,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x01,0x00,0x04, + 0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x94,0x20,0x53,0x04,0x01,0x00,0x92, + 0x18,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0xe3,0x80,0x88,0x00,0x10,0x08,0x01, + 0xff,0xe3,0x80,0x89,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xcf,0x86,0x55,0x04,0x01, + 0x00,0x54,0x04,0x01,0x00,0x53,0x04,0x01,0x00,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10, + 0x04,0x01,0x00,0x04,0x00,0x91,0x08,0x10,0x04,0x06,0x00,0x04,0x00,0x04,0x00,0xd0, + 0x1e,0xcf,0x86,0x95,0x18,0x54,0x04,0x04,0x00,0x53,0x04,0x04,0x00,0x92,0x0c,0x51, + 0x04,0x04,0x00,0x10,0x04,0x04,0x00,0x06,0x00,0x06,0x00,0x06,0x00,0xcf,0x86,0xd5, + 0x2c,0xd4,0x14,0x53,0x04,0x06,0x00,0x52,0x04,0x06,0x00,0x51,0x04,0x06,0x00,0x10, + 0x04,0x06,0x00,0x07,0x00,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x07,0x00,0x08, + 0x00,0x08,0x00,0x08,0x00,0x12,0x04,0x08,0x00,0x09,0x00,0xd4,0x14,0x53,0x04,0x09, + 0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x0b,0x00,0x0c,0x00,0x0c,0x00,0x0c,0x00,0xd3, + 0x08,0x12,0x04,0x0c,0x00,0x10,0x00,0xd2,0x0c,0x51,0x04,0x10,0x00,0x10,0x04,0x10, + 0x00,0x12,0x00,0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x00,0x13,0x00,0xd3,0xa6,0xd2, + 0x74,0xd1,0x40,0xd0,0x22,0xcf,0x86,0x55,0x04,0x01,0x00,0x94,0x18,0x93,0x14,0x52, + 0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x04,0x00,0x10,0x04,0x04,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0x95,0x18,0x94,0x14,0x53,0x04,0x01,0x00,0x92, + 0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01, + 0x00,0xd0,0x06,0xcf,0x06,0x01,0x00,0xcf,0x86,0x55,0x04,0x01,0x00,0xd4,0x14,0x53, + 0x04,0x01,0x00,0x92,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x06,0x00,0x06, + 0x00,0x53,0x04,0x06,0x00,0x52,0x04,0x06,0x00,0x51,0x04,0x06,0x00,0x10,0x04,0x06, + 0x00,0x07,0x00,0xd1,0x06,0xcf,0x06,0x01,0x00,0xd0,0x1a,0xcf,0x86,0x95,0x14,0x54, + 0x04,0x01,0x00,0x93,0x0c,0x52,0x04,0x01,0x00,0x11,0x04,0x01,0x00,0x06,0x00,0x06, + 0x00,0x01,0x00,0xcf,0x86,0x55,0x04,0x01,0x00,0x54,0x04,0x01,0x00,0x13,0x04,0x04, + 0x00,0x06,0x00,0xd2,0xdc,0xd1,0x48,0xd0,0x26,0xcf,0x86,0x95,0x20,0x54,0x04,0x01, + 0x00,0xd3,0x0c,0x52,0x04,0x01,0x00,0x11,0x04,0x07,0x00,0x06,0x00,0x92,0x0c,0x91, + 0x08,0x10,0x04,0x08,0x00,0x04,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xcf,0x86,0x55, + 0x04,0x01,0x00,0x54,0x04,0x01,0x00,0xd3,0x0c,0x92,0x08,0x11,0x04,0x04,0x00,0x06, + 0x00,0x06,0x00,0x52,0x04,0x06,0x00,0x11,0x04,0x06,0x00,0x08,0x00,0xd0,0x5e,0xcf, + 0x86,0xd5,0x2c,0xd4,0x10,0x53,0x04,0x06,0x00,0x92,0x08,0x11,0x04,0x06,0x00,0x07, + 0x00,0x07,0x00,0xd3,0x0c,0x92,0x08,0x11,0x04,0x07,0x00,0x08,0x00,0x08,0x00,0x52, + 0x04,0x08,0x00,0x91,0x08,0x10,0x04,0x08,0x00,0x0a,0x00,0x0b,0x00,0xd4,0x10,0x93, + 0x0c,0x92,0x08,0x11,0x04,0x07,0x00,0x08,0x00,0x08,0x00,0x08,0x00,0xd3,0x10,0x92, + 0x0c,0x51,0x04,0x08,0x00,0x10,0x04,0x09,0x00,0x0a,0x00,0x0a,0x00,0x52,0x04,0x0a, + 0x00,0x91,0x08,0x10,0x04,0x0a,0x00,0x0b,0x00,0x0b,0x00,0xcf,0x86,0xd5,0x1c,0x94, + 0x18,0xd3,0x08,0x12,0x04,0x0a,0x00,0x0b,0x00,0x52,0x04,0x0b,0x00,0x51,0x04,0x0b, + 0x00,0x10,0x04,0x0c,0x00,0x0b,0x00,0x0b,0x00,0x94,0x14,0x93,0x10,0x92,0x0c,0x51, + 0x04,0x0b,0x00,0x10,0x04,0x0c,0x00,0x0b,0x00,0x0c,0x00,0x0b,0x00,0x0b,0x00,0xd1, + 0xa8,0xd0,0x42,0xcf,0x86,0xd5,0x28,0x94,0x24,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10, + 0x04,0x10,0x00,0x01,0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x0c,0x00,0x01, + 0x00,0x92,0x08,0x11,0x04,0x01,0x00,0x0c,0x00,0x01,0x00,0x01,0x00,0x94,0x14,0x53, + 0x04,0x01,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x0c,0x00,0x01,0x00,0x01,0x00,0x01, + 0x00,0x01,0x00,0xcf,0x86,0xd5,0x40,0xd4,0x18,0x53,0x04,0x01,0x00,0x52,0x04,0x01, + 0x00,0xd1,0x08,0x10,0x04,0x0c,0x00,0x01,0x00,0x10,0x04,0x0c,0x00,0x01,0x00,0xd3, + 0x18,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x0c,0x00,0x51,0x04,0x0c, + 0x00,0x10,0x04,0x01,0x00,0x0b,0x00,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10, + 0x04,0x01,0x00,0x0c,0x00,0xd4,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0c, + 0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x06,0x00,0x93,0x0c,0x52,0x04,0x06,0x00,0x11, + 0x04,0x06,0x00,0x01,0x00,0x01,0x00,0xd0,0x3e,0xcf,0x86,0xd5,0x18,0x54,0x04,0x01, + 0x00,0x93,0x10,0x52,0x04,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x0c,0x00,0x0c, + 0x00,0x01,0x00,0x54,0x04,0x01,0x00,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0c, + 0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10, + 0x04,0x01,0x00,0x0c,0x00,0xcf,0x86,0xd5,0x2c,0x94,0x28,0xd3,0x10,0x52,0x04,0x08, + 0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x09,0x00,0xd2,0x0c,0x51,0x04,0x09, + 0x00,0x10,0x04,0x09,0x00,0x0d,0x00,0x91,0x08,0x10,0x04,0x0a,0x00,0x0d,0x00,0x0c, + 0x00,0x06,0x00,0x94,0x0c,0x53,0x04,0x06,0x00,0x12,0x04,0x06,0x00,0x0a,0x00,0x06, + 0x00,0xe4,0x39,0x01,0xd3,0x0c,0xd2,0x06,0xcf,0x06,0x04,0x00,0xcf,0x06,0x06,0x00, + 0xd2,0x30,0xd1,0x06,0xcf,0x06,0x06,0x00,0xd0,0x06,0xcf,0x06,0x06,0x00,0xcf,0x86, + 0x95,0x1e,0x54,0x04,0x06,0x00,0x53,0x04,0x06,0x00,0x52,0x04,0x06,0x00,0x91,0x0e, + 0x10,0x0a,0x06,0xff,0xe2,0xab,0x9d,0xcc,0xb8,0x00,0x06,0x00,0x06,0x00,0x06,0x00, + 0xd1,0x80,0xd0,0x3a,0xcf,0x86,0xd5,0x28,0xd4,0x10,0x53,0x04,0x07,0x00,0x52,0x04, + 0x07,0x00,0x11,0x04,0x07,0x00,0x08,0x00,0xd3,0x08,0x12,0x04,0x08,0x00,0x09,0x00, + 0x92,0x0c,0x51,0x04,0x09,0x00,0x10,0x04,0x09,0x00,0x0a,0x00,0x0a,0x00,0x94,0x0c, + 0x93,0x08,0x12,0x04,0x09,0x00,0x0a,0x00,0x0a,0x00,0x0a,0x00,0xcf,0x86,0xd5,0x30, + 0xd4,0x14,0x53,0x04,0x0a,0x00,0x52,0x04,0x0a,0x00,0x91,0x08,0x10,0x04,0x0a,0x00, + 0x10,0x00,0x10,0x00,0xd3,0x10,0x52,0x04,0x0a,0x00,0x91,0x08,0x10,0x04,0x0a,0x00, + 0x0b,0x00,0x0b,0x00,0x92,0x08,0x11,0x04,0x0b,0x00,0x10,0x00,0x10,0x00,0x54,0x04, + 0x10,0x00,0x93,0x0c,0x52,0x04,0x10,0x00,0x11,0x04,0x00,0x00,0x10,0x00,0x10,0x00, + 0xd0,0x32,0xcf,0x86,0xd5,0x14,0x54,0x04,0x10,0x00,0x93,0x0c,0x52,0x04,0x10,0x00, + 0x11,0x04,0x10,0x00,0x00,0x00,0x10,0x00,0x54,0x04,0x10,0x00,0x53,0x04,0x10,0x00, + 0xd2,0x08,0x11,0x04,0x10,0x00,0x14,0x00,0x91,0x08,0x10,0x04,0x14,0x00,0x10,0x00, + 0x10,0x00,0xcf,0x86,0xd5,0x28,0xd4,0x14,0x53,0x04,0x10,0x00,0x92,0x0c,0x91,0x08, + 0x10,0x04,0x10,0x00,0x15,0x00,0x10,0x00,0x10,0x00,0x93,0x10,0x92,0x0c,0x51,0x04, + 0x10,0x00,0x10,0x04,0x13,0x00,0x14,0x00,0x14,0x00,0x14,0x00,0xd4,0x0c,0x53,0x04, + 0x14,0x00,0x12,0x04,0x14,0x00,0x11,0x00,0x53,0x04,0x14,0x00,0x52,0x04,0x14,0x00, + 0x51,0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x15,0x00,0xe3,0xb9,0x01,0xd2,0xac,0xd1, + 0x68,0xd0,0x1e,0xcf,0x86,0x55,0x04,0x08,0x00,0x94,0x14,0x53,0x04,0x08,0x00,0x52, + 0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x00,0x00,0x08,0x00,0xcf, + 0x86,0xd5,0x18,0x54,0x04,0x08,0x00,0x53,0x04,0x08,0x00,0x52,0x04,0x08,0x00,0x51, + 0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x00,0x00,0xd4,0x14,0x53,0x04,0x09,0x00,0x52, + 0x04,0x09,0x00,0x91,0x08,0x10,0x04,0x09,0x00,0x0a,0x00,0x0a,0x00,0xd3,0x10,0x92, + 0x0c,0x91,0x08,0x10,0x04,0x0b,0x00,0x0a,0x00,0x0a,0x00,0x09,0x00,0x52,0x04,0x0a, + 0x00,0x11,0x04,0x0a,0x00,0x0b,0x00,0xd0,0x06,0xcf,0x06,0x08,0x00,0xcf,0x86,0x55, + 0x04,0x08,0x00,0xd4,0x1c,0x53,0x04,0x08,0x00,0xd2,0x0c,0x51,0x04,0x08,0x00,0x10, + 0x04,0x08,0x00,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b,0x00,0x0b,0xe6,0xd3, + 0x0c,0x92,0x08,0x11,0x04,0x0b,0xe6,0x0d,0x00,0x00,0x00,0x92,0x0c,0x91,0x08,0x10, + 0x04,0x00,0x00,0x08,0x00,0x08,0x00,0x08,0x00,0xd1,0x6c,0xd0,0x2a,0xcf,0x86,0x55, + 0x04,0x08,0x00,0x94,0x20,0xd3,0x10,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10, + 0x04,0x00,0x00,0x0d,0x00,0x52,0x04,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x0d, + 0x00,0x00,0x00,0x08,0x00,0xcf,0x86,0x55,0x04,0x08,0x00,0xd4,0x1c,0xd3,0x0c,0x52, + 0x04,0x08,0x00,0x11,0x04,0x08,0x00,0x0d,0x00,0x52,0x04,0x00,0x00,0x51,0x04,0x00, + 0x00,0x10,0x04,0x00,0x00,0x08,0x00,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0c, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00,0x10, + 0x04,0x00,0x00,0x0c,0x09,0xd0,0x5a,0xcf,0x86,0xd5,0x18,0x54,0x04,0x08,0x00,0x93, + 0x10,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x00,0x00,0x00, + 0x00,0xd4,0x20,0xd3,0x10,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08, + 0x00,0x00,0x00,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x00, + 0x00,0xd3,0x10,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x00, + 0x00,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x00,0x00,0xcf, + 0x86,0x95,0x40,0xd4,0x20,0xd3,0x10,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10, + 0x04,0x08,0x00,0x00,0x00,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08, + 0x00,0x00,0x00,0xd3,0x10,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08, + 0x00,0x00,0x00,0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x00, + 0x00,0x0a,0xe6,0xd2,0x9c,0xd1,0x68,0xd0,0x32,0xcf,0x86,0xd5,0x14,0x54,0x04,0x08, + 0x00,0x53,0x04,0x08,0x00,0x52,0x04,0x0a,0x00,0x11,0x04,0x08,0x00,0x0a,0x00,0x54, + 0x04,0x0a,0x00,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0a,0x00,0x0b,0x00,0x0d, + 0x00,0x0d,0x00,0x12,0x04,0x0d,0x00,0x10,0x00,0xcf,0x86,0x95,0x30,0x94,0x2c,0xd3, + 0x18,0xd2,0x0c,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x12,0x00,0x91,0x08,0x10, + 0x04,0x12,0x00,0x13,0x00,0x13,0x00,0xd2,0x08,0x11,0x04,0x13,0x00,0x14,0x00,0x51, + 0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x15,0x00,0x00,0x00,0x00,0x00,0xd0,0x1e,0xcf, + 0x86,0x95,0x18,0x54,0x04,0x04,0x00,0x53,0x04,0x04,0x00,0x92,0x0c,0x51,0x04,0x04, + 0x00,0x10,0x04,0x00,0x00,0x04,0x00,0x04,0x00,0x04,0x00,0xcf,0x86,0x55,0x04,0x04, + 0x00,0x54,0x04,0x04,0x00,0x93,0x08,0x12,0x04,0x04,0x00,0x00,0x00,0x00,0x00,0xd1, + 0x06,0xcf,0x06,0x04,0x00,0xd0,0x06,0xcf,0x06,0x04,0x00,0xcf,0x86,0xd5,0x14,0x54, + 0x04,0x04,0x00,0x93,0x0c,0x52,0x04,0x04,0x00,0x11,0x04,0x04,0x00,0x00,0x00,0x00, + 0x00,0x54,0x04,0x00,0x00,0x53,0x04,0x04,0x00,0x12,0x04,0x04,0x00,0x00,0x00,0xcf, + 0x86,0xe5,0xa6,0x05,0xe4,0x9f,0x05,0xe3,0x96,0x04,0xe2,0xe4,0x03,0xe1,0xc0,0x01, + 0xd0,0x3e,0xcf,0x86,0x55,0x04,0x01,0x00,0xd4,0x1c,0x53,0x04,0x01,0x00,0xd2,0x0c, + 0x51,0x04,0x01,0x00,0x10,0x04,0x01,0xda,0x01,0xe4,0x91,0x08,0x10,0x04,0x01,0xe8, + 0x01,0xde,0x01,0xe0,0x53,0x04,0x01,0x00,0xd2,0x0c,0x51,0x04,0x04,0x00,0x10,0x04, + 0x04,0x00,0x06,0x00,0x51,0x04,0x06,0x00,0x10,0x04,0x04,0x00,0x01,0x00,0xcf,0x86, + 0xd5,0xaa,0xd4,0x32,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00, + 0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x81, + 0x8b,0xe3,0x82,0x99,0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x81,0x8d,0xe3,0x82, + 0x99,0x00,0x01,0x00,0xd3,0x3c,0xd2,0x1e,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x81, + 0x8f,0xe3,0x82,0x99,0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x81,0x91,0xe3,0x82, + 0x99,0x00,0x01,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x81,0x93,0xe3,0x82,0x99, + 0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x81,0x95,0xe3,0x82,0x99,0x00,0x01,0x00, + 0xd2,0x1e,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x81,0x97,0xe3,0x82,0x99,0x00,0x01, + 0x00,0x10,0x0b,0x01,0xff,0xe3,0x81,0x99,0xe3,0x82,0x99,0x00,0x01,0x00,0xd1,0x0f, + 0x10,0x0b,0x01,0xff,0xe3,0x81,0x9b,0xe3,0x82,0x99,0x00,0x01,0x00,0x10,0x0b,0x01, + 0xff,0xe3,0x81,0x9d,0xe3,0x82,0x99,0x00,0x01,0x00,0xd4,0x53,0xd3,0x3c,0xd2,0x1e, + 0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x81,0x9f,0xe3,0x82,0x99,0x00,0x01,0x00,0x10, + 0x0b,0x01,0xff,0xe3,0x81,0xa1,0xe3,0x82,0x99,0x00,0x01,0x00,0xd1,0x0f,0x10,0x04, + 0x01,0x00,0x01,0xff,0xe3,0x81,0xa4,0xe3,0x82,0x99,0x00,0x10,0x04,0x01,0x00,0x01, + 0xff,0xe3,0x81,0xa6,0xe3,0x82,0x99,0x00,0x92,0x13,0x91,0x0f,0x10,0x04,0x01,0x00, + 0x01,0xff,0xe3,0x81,0xa8,0xe3,0x82,0x99,0x00,0x01,0x00,0x01,0x00,0xd3,0x4a,0xd2, + 0x25,0xd1,0x16,0x10,0x0b,0x01,0xff,0xe3,0x81,0xaf,0xe3,0x82,0x99,0x00,0x01,0xff, + 0xe3,0x81,0xaf,0xe3,0x82,0x9a,0x00,0x10,0x04,0x01,0x00,0x01,0xff,0xe3,0x81,0xb2, + 0xe3,0x82,0x99,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x81,0xb2,0xe3,0x82,0x9a, + 0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x81,0xb5,0xe3,0x82,0x99,0x00,0x01,0xff, + 0xe3,0x81,0xb5,0xe3,0x82,0x9a,0x00,0xd2,0x1e,0xd1,0x0f,0x10,0x04,0x01,0x00,0x01, + 0xff,0xe3,0x81,0xb8,0xe3,0x82,0x99,0x00,0x10,0x0b,0x01,0xff,0xe3,0x81,0xb8,0xe3, + 0x82,0x9a,0x00,0x01,0x00,0x91,0x16,0x10,0x0b,0x01,0xff,0xe3,0x81,0xbb,0xe3,0x82, + 0x99,0x00,0x01,0xff,0xe3,0x81,0xbb,0xe3,0x82,0x9a,0x00,0x01,0x00,0xd0,0xee,0xcf, + 0x86,0xd5,0x42,0x54,0x04,0x01,0x00,0xd3,0x1b,0x52,0x04,0x01,0x00,0xd1,0x0f,0x10, + 0x0b,0x01,0xff,0xe3,0x81,0x86,0xe3,0x82,0x99,0x00,0x06,0x00,0x10,0x04,0x06,0x00, + 0x00,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x00,0x00,0x01,0x08,0x10,0x04,0x01,0x08, + 0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x82,0x9d,0xe3,0x82,0x99, + 0x00,0x06,0x00,0xd4,0x32,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x06,0x00,0x01, + 0x00,0x01,0x00,0x01,0x00,0x52,0x04,0x01,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3, + 0x82,0xab,0xe3,0x82,0x99,0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x82,0xad,0xe3, + 0x82,0x99,0x00,0x01,0x00,0xd3,0x3c,0xd2,0x1e,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3, + 0x82,0xaf,0xe3,0x82,0x99,0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x82,0xb1,0xe3, + 0x82,0x99,0x00,0x01,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x82,0xb3,0xe3,0x82, + 0x99,0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x82,0xb5,0xe3,0x82,0x99,0x00,0x01, + 0x00,0xd2,0x1e,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x82,0xb7,0xe3,0x82,0x99,0x00, + 0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x82,0xb9,0xe3,0x82,0x99,0x00,0x01,0x00,0xd1, + 0x0f,0x10,0x0b,0x01,0xff,0xe3,0x82,0xbb,0xe3,0x82,0x99,0x00,0x01,0x00,0x10,0x0b, + 0x01,0xff,0xe3,0x82,0xbd,0xe3,0x82,0x99,0x00,0x01,0x00,0xcf,0x86,0xd5,0xd5,0xd4, + 0x53,0xd3,0x3c,0xd2,0x1e,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3,0x82,0xbf,0xe3,0x82, + 0x99,0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x83,0x81,0xe3,0x82,0x99,0x00,0x01, + 0x00,0xd1,0x0f,0x10,0x04,0x01,0x00,0x01,0xff,0xe3,0x83,0x84,0xe3,0x82,0x99,0x00, + 0x10,0x04,0x01,0x00,0x01,0xff,0xe3,0x83,0x86,0xe3,0x82,0x99,0x00,0x92,0x13,0x91, + 0x0f,0x10,0x04,0x01,0x00,0x01,0xff,0xe3,0x83,0x88,0xe3,0x82,0x99,0x00,0x01,0x00, + 0x01,0x00,0xd3,0x4a,0xd2,0x25,0xd1,0x16,0x10,0x0b,0x01,0xff,0xe3,0x83,0x8f,0xe3, + 0x82,0x99,0x00,0x01,0xff,0xe3,0x83,0x8f,0xe3,0x82,0x9a,0x00,0x10,0x04,0x01,0x00, + 0x01,0xff,0xe3,0x83,0x92,0xe3,0x82,0x99,0x00,0xd1,0x0f,0x10,0x0b,0x01,0xff,0xe3, + 0x83,0x92,0xe3,0x82,0x9a,0x00,0x01,0x00,0x10,0x0b,0x01,0xff,0xe3,0x83,0x95,0xe3, + 0x82,0x99,0x00,0x01,0xff,0xe3,0x83,0x95,0xe3,0x82,0x9a,0x00,0xd2,0x1e,0xd1,0x0f, + 0x10,0x04,0x01,0x00,0x01,0xff,0xe3,0x83,0x98,0xe3,0x82,0x99,0x00,0x10,0x0b,0x01, + 0xff,0xe3,0x83,0x98,0xe3,0x82,0x9a,0x00,0x01,0x00,0x91,0x16,0x10,0x0b,0x01,0xff, + 0xe3,0x83,0x9b,0xe3,0x82,0x99,0x00,0x01,0xff,0xe3,0x83,0x9b,0xe3,0x82,0x9a,0x00, + 0x01,0x00,0x54,0x04,0x01,0x00,0xd3,0x22,0x52,0x04,0x01,0x00,0xd1,0x0f,0x10,0x0b, + 0x01,0xff,0xe3,0x82,0xa6,0xe3,0x82,0x99,0x00,0x01,0x00,0x10,0x04,0x01,0x00,0x01, + 0xff,0xe3,0x83,0xaf,0xe3,0x82,0x99,0x00,0xd2,0x25,0xd1,0x16,0x10,0x0b,0x01,0xff, + 0xe3,0x83,0xb0,0xe3,0x82,0x99,0x00,0x01,0xff,0xe3,0x83,0xb1,0xe3,0x82,0x99,0x00, + 0x10,0x0b,0x01,0xff,0xe3,0x83,0xb2,0xe3,0x82,0x99,0x00,0x01,0x00,0x51,0x04,0x01, + 0x00,0x10,0x0b,0x01,0xff,0xe3,0x83,0xbd,0xe3,0x82,0x99,0x00,0x06,0x00,0xd1,0x65, + 0xd0,0x46,0xcf,0x86,0xd5,0x18,0x94,0x14,0x93,0x10,0x52,0x04,0x00,0x00,0x91,0x08, + 0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd4,0x18,0x53,0x04, + 0x01,0x00,0x52,0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x0a,0x00,0x10,0x04, + 0x13,0x00,0x14,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00, + 0x01,0x00,0x01,0x00,0x01,0x00,0xcf,0x86,0x55,0x04,0x01,0x00,0x94,0x15,0x93,0x11, + 0x52,0x04,0x01,0x00,0x91,0x09,0x10,0x05,0x01,0xff,0x00,0x01,0x00,0x01,0x00,0x01, + 0x00,0x01,0x00,0xd0,0x32,0xcf,0x86,0xd5,0x18,0x94,0x14,0x53,0x04,0x01,0x00,0x52, + 0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x54, + 0x04,0x04,0x00,0x53,0x04,0x04,0x00,0x92,0x0c,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c, + 0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0xd5,0x08,0x14,0x04,0x08,0x00,0x0a,0x00,0x94, + 0x0c,0x93,0x08,0x12,0x04,0x0a,0x00,0x00,0x00,0x00,0x00,0x06,0x00,0xd2,0xa4,0xd1, + 0x5c,0xd0,0x22,0xcf,0x86,0x95,0x1c,0x54,0x04,0x01,0x00,0x53,0x04,0x01,0x00,0x52, + 0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x07,0x00,0x10,0x04,0x07,0x00,0x00, + 0x00,0x01,0x00,0xcf,0x86,0xd5,0x20,0xd4,0x0c,0x93,0x08,0x12,0x04,0x01,0x00,0x0b, + 0x00,0x0b,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x07,0x00,0x06,0x00,0x06, + 0x00,0x06,0x00,0x06,0x00,0x54,0x04,0x01,0x00,0x53,0x04,0x01,0x00,0x52,0x04,0x01, + 0x00,0x51,0x04,0x07,0x00,0x10,0x04,0x08,0x00,0x01,0x00,0xd0,0x1e,0xcf,0x86,0x55, + 0x04,0x01,0x00,0x54,0x04,0x01,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x01, + 0x00,0x06,0x00,0x06,0x00,0x06,0x00,0x06,0x00,0xcf,0x86,0xd5,0x10,0x94,0x0c,0x53, + 0x04,0x01,0x00,0x12,0x04,0x01,0x00,0x07,0x00,0x01,0x00,0x54,0x04,0x01,0x00,0x53, + 0x04,0x01,0x00,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x16, + 0x00,0xd1,0x30,0xd0,0x06,0xcf,0x06,0x01,0x00,0xcf,0x86,0x55,0x04,0x01,0x00,0x54, + 0x04,0x01,0x00,0xd3,0x10,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01, + 0x00,0x07,0x00,0x92,0x0c,0x51,0x04,0x07,0x00,0x10,0x04,0x07,0x00,0x01,0x00,0x01, + 0x00,0xd0,0x06,0xcf,0x06,0x01,0x00,0xcf,0x86,0xd5,0x14,0x54,0x04,0x01,0x00,0x53, + 0x04,0x01,0x00,0x52,0x04,0x01,0x00,0x11,0x04,0x01,0x00,0x07,0x00,0x54,0x04,0x01, + 0x00,0x53,0x04,0x01,0x00,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01, + 0x00,0x07,0x00,0xcf,0x06,0x04,0x00,0xcf,0x06,0x04,0x00,0xd1,0x48,0xd0,0x40,0xcf, + 0x86,0xd5,0x06,0xcf,0x06,0x04,0x00,0xd4,0x06,0xcf,0x06,0x04,0x00,0xd3,0x2c,0xd2, + 0x06,0xcf,0x06,0x04,0x00,0xd1,0x06,0xcf,0x06,0x04,0x00,0xd0,0x1a,0xcf,0x86,0x55, + 0x04,0x04,0x00,0x54,0x04,0x04,0x00,0x93,0x0c,0x52,0x04,0x04,0x00,0x11,0x04,0x04, + 0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x07,0x00,0xcf,0x06,0x01,0x00,0xcf,0x86,0xcf, + 0x06,0x01,0x00,0xcf,0x86,0xcf,0x06,0x01,0x00,0xe2,0x71,0x05,0xd1,0x8c,0xd0,0x08, + 0xcf,0x86,0xcf,0x06,0x01,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x01,0x00,0xd4,0x06, + 0xcf,0x06,0x01,0x00,0xd3,0x06,0xcf,0x06,0x01,0x00,0xd2,0x06,0xcf,0x06,0x01,0x00, + 0xd1,0x06,0xcf,0x06,0x01,0x00,0xd0,0x22,0xcf,0x86,0x55,0x04,0x01,0x00,0xd4,0x10, + 0x93,0x0c,0x52,0x04,0x01,0x00,0x11,0x04,0x01,0x00,0x08,0x00,0x08,0x00,0x53,0x04, + 0x08,0x00,0x12,0x04,0x08,0x00,0x0a,0x00,0xcf,0x86,0xd5,0x28,0xd4,0x18,0xd3,0x08, + 0x12,0x04,0x0a,0x00,0x0b,0x00,0x52,0x04,0x0b,0x00,0x91,0x08,0x10,0x04,0x0d,0x00, + 0x11,0x00,0x11,0x00,0x93,0x0c,0x52,0x04,0x11,0x00,0x11,0x04,0x11,0x00,0x13,0x00, + 0x13,0x00,0x94,0x14,0x53,0x04,0x13,0x00,0x92,0x0c,0x51,0x04,0x13,0x00,0x10,0x04, + 0x13,0x00,0x14,0x00,0x14,0x00,0x00,0x00,0xe0,0xdb,0x04,0xcf,0x86,0xe5,0xdf,0x01, + 0xd4,0x06,0xcf,0x06,0x04,0x00,0xd3,0x74,0xd2,0x6e,0xd1,0x06,0xcf,0x06,0x04,0x00, + 0xd0,0x3e,0xcf,0x86,0xd5,0x18,0x94,0x14,0x53,0x04,0x04,0x00,0x52,0x04,0x04,0x00, + 0x91,0x08,0x10,0x04,0x04,0x00,0x00,0x00,0x00,0x00,0x04,0x00,0xd4,0x10,0x93,0x0c, + 0x92,0x08,0x11,0x04,0x04,0x00,0x06,0x00,0x04,0x00,0x04,0x00,0x93,0x10,0x52,0x04, + 0x04,0x00,0x91,0x08,0x10,0x04,0x06,0x00,0x04,0x00,0x04,0x00,0x04,0x00,0xcf,0x86, + 0x95,0x24,0x94,0x20,0x93,0x1c,0xd2,0x0c,0x91,0x08,0x10,0x04,0x04,0x00,0x06,0x00, + 0x04,0x00,0xd1,0x08,0x10,0x04,0x04,0x00,0x06,0x00,0x10,0x04,0x04,0x00,0x00,0x00, + 0x00,0x00,0x0b,0x00,0x0b,0x00,0xcf,0x06,0x0a,0x00,0xd2,0x84,0xd1,0x4c,0xd0,0x16, + 0xcf,0x86,0x55,0x04,0x0a,0x00,0x94,0x0c,0x53,0x04,0x0a,0x00,0x12,0x04,0x0a,0x00, + 0x00,0x00,0x00,0x00,0xcf,0x86,0x55,0x04,0x0a,0x00,0xd4,0x1c,0xd3,0x0c,0x92,0x08, + 0x11,0x04,0x0c,0x00,0x0a,0x00,0x0a,0x00,0x52,0x04,0x0a,0x00,0x51,0x04,0x0a,0x00, + 0x10,0x04,0x0a,0x00,0x0a,0xe6,0xd3,0x08,0x12,0x04,0x0a,0x00,0x0d,0xe6,0x52,0x04, + 0x0d,0xe6,0x11,0x04,0x0a,0xe6,0x0a,0x00,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x54,0x04, + 0x0a,0x00,0x53,0x04,0x0a,0x00,0x52,0x04,0x10,0x00,0x51,0x04,0x10,0x00,0x10,0x04, + 0x11,0xe6,0x0d,0xe6,0x0b,0x00,0xcf,0x86,0x55,0x04,0x0b,0x00,0x54,0x04,0x0b,0x00, + 0x93,0x0c,0x92,0x08,0x11,0x04,0x0b,0xe6,0x0b,0x00,0x0b,0x00,0x00,0x00,0xd1,0x40, + 0xd0,0x3a,0xcf,0x86,0xd5,0x24,0x54,0x04,0x08,0x00,0xd3,0x10,0x52,0x04,0x08,0x00, + 0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x09,0x00,0x92,0x0c,0x51,0x04,0x09,0x00, + 0x10,0x04,0x09,0x00,0x0a,0x00,0x0a,0x00,0x94,0x10,0x93,0x0c,0x92,0x08,0x11,0x04, + 0x09,0x00,0x0a,0x00,0x0a,0x00,0x0a,0x00,0x0a,0x00,0xcf,0x06,0x0a,0x00,0xd0,0x5e, + 0xcf,0x86,0xd5,0x28,0xd4,0x18,0x53,0x04,0x0a,0x00,0x52,0x04,0x0a,0x00,0xd1,0x08, + 0x10,0x04,0x0a,0x00,0x0c,0x00,0x10,0x04,0x0c,0x00,0x11,0x00,0x93,0x0c,0x92,0x08, + 0x11,0x04,0x0c,0x00,0x0d,0x00,0x10,0x00,0x10,0x00,0xd4,0x1c,0x53,0x04,0x0c,0x00, + 0xd2,0x0c,0x51,0x04,0x0c,0x00,0x10,0x04,0x0d,0x00,0x10,0x00,0x51,0x04,0x10,0x00, + 0x10,0x04,0x12,0x00,0x14,0x00,0xd3,0x0c,0x92,0x08,0x11,0x04,0x10,0x00,0x11,0x00, + 0x11,0x00,0x92,0x08,0x11,0x04,0x14,0x00,0x15,0x00,0x15,0x00,0xcf,0x86,0xd5,0x1c, + 0x94,0x18,0x93,0x14,0xd2,0x08,0x11,0x04,0x00,0x00,0x15,0x00,0x51,0x04,0x15,0x00, + 0x10,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x54,0x04,0x00,0x00,0xd3,0x10, + 0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0x92,0x0c, + 0x51,0x04,0x0d,0x00,0x10,0x04,0x0c,0x00,0x0a,0x00,0x0a,0x00,0xe4,0xf2,0x02,0xe3, + 0x65,0x01,0xd2,0x98,0xd1,0x48,0xd0,0x36,0xcf,0x86,0xd5,0x18,0x94,0x14,0x93,0x10, + 0x52,0x04,0x08,0x00,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x09,0x08,0x00,0x08,0x00, + 0x08,0x00,0xd4,0x0c,0x53,0x04,0x08,0x00,0x12,0x04,0x08,0x00,0x00,0x00,0x53,0x04, + 0x0b,0x00,0x92,0x08,0x11,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0x55,0x04, + 0x09,0x00,0x54,0x04,0x09,0x00,0x13,0x04,0x09,0x00,0x00,0x00,0xd0,0x06,0xcf,0x06, + 0x0a,0x00,0xcf,0x86,0xd5,0x2c,0xd4,0x1c,0xd3,0x10,0x52,0x04,0x0a,0x00,0x91,0x08, + 0x10,0x04,0x0a,0x09,0x12,0x00,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00, + 0x0a,0x00,0x53,0x04,0x0a,0x00,0x92,0x08,0x11,0x04,0x0a,0x00,0x00,0x00,0x00,0x00, + 0x54,0x04,0x0b,0xe6,0xd3,0x0c,0x92,0x08,0x11,0x04,0x0b,0xe6,0x0b,0x00,0x0b,0x00, + 0x52,0x04,0x0b,0x00,0x11,0x04,0x11,0x00,0x14,0x00,0xd1,0x60,0xd0,0x22,0xcf,0x86, + 0x55,0x04,0x0a,0x00,0x94,0x18,0x53,0x04,0x0a,0x00,0xd2,0x0c,0x51,0x04,0x0a,0x00, + 0x10,0x04,0x0a,0x00,0x0a,0xdc,0x11,0x04,0x0a,0xdc,0x0a,0x00,0x0a,0x00,0xcf,0x86, + 0xd5,0x24,0x54,0x04,0x0a,0x00,0xd3,0x10,0x92,0x0c,0x51,0x04,0x0a,0x00,0x10,0x04, + 0x0a,0x00,0x0a,0x09,0x00,0x00,0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04, + 0x00,0x00,0x0a,0x00,0x54,0x04,0x0b,0x00,0x53,0x04,0x0b,0x00,0x52,0x04,0x0b,0x00, + 0x91,0x08,0x10,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0xd0,0x1e,0xcf,0x86,0x55,0x04, + 0x0b,0x00,0x54,0x04,0x0b,0x00,0x93,0x10,0x92,0x0c,0x51,0x04,0x0b,0x00,0x10,0x04, + 0x0b,0x00,0x0b,0x07,0x0b,0x00,0x0b,0x00,0xcf,0x86,0xd5,0x34,0xd4,0x20,0xd3,0x10, + 0x92,0x0c,0x91,0x08,0x10,0x04,0x0b,0x09,0x0b,0x00,0x0b,0x00,0x0b,0x00,0x52,0x04, + 0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04,0x00,0x00,0x0b,0x00,0x53,0x04,0x0b,0x00, + 0xd2,0x08,0x11,0x04,0x0b,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x0b,0x00,0x54,0x04, + 0x10,0x00,0x53,0x04,0x10,0x00,0x52,0x04,0x10,0x00,0x51,0x04,0x10,0x00,0x10,0x04, + 0x10,0x00,0x00,0x00,0xd2,0xd0,0xd1,0x50,0xd0,0x1e,0xcf,0x86,0x55,0x04,0x0a,0x00, + 0x54,0x04,0x0a,0x00,0x93,0x10,0x52,0x04,0x0a,0x00,0x51,0x04,0x0a,0x00,0x10,0x04, + 0x0a,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0xd5,0x20,0xd4,0x10,0x53,0x04,0x0a,0x00, + 0x52,0x04,0x0a,0x00,0x11,0x04,0x0a,0x00,0x00,0x00,0x53,0x04,0x0a,0x00,0x92,0x08, + 0x11,0x04,0x0a,0x00,0x00,0x00,0x0a,0x00,0x54,0x04,0x0b,0x00,0x53,0x04,0x0b,0x00, + 0x12,0x04,0x0b,0x00,0x10,0x00,0xd0,0x3a,0xcf,0x86,0x55,0x04,0x0b,0x00,0x54,0x04, + 0x0b,0x00,0xd3,0x1c,0xd2,0x0c,0x91,0x08,0x10,0x04,0x0b,0xe6,0x0b,0x00,0x0b,0xe6, + 0xd1,0x08,0x10,0x04,0x0b,0xdc,0x0b,0x00,0x10,0x04,0x0b,0x00,0x0b,0xe6,0xd2,0x0c, + 0x91,0x08,0x10,0x04,0x0b,0xe6,0x0b,0x00,0x0b,0x00,0x11,0x04,0x0b,0x00,0x0b,0xe6, + 0xcf,0x86,0xd5,0x2c,0xd4,0x18,0x93,0x14,0x92,0x10,0xd1,0x08,0x10,0x04,0x0b,0x00, + 0x0b,0xe6,0x10,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x53,0x04,0x00,0x00, + 0x92,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x0b,0x00,0x0b,0x00,0x54,0x04, + 0x0d,0x00,0x93,0x10,0x52,0x04,0x0d,0x00,0x51,0x04,0x0d,0x00,0x10,0x04,0x0d,0x09, + 0x00,0x00,0x00,0x00,0xd1,0x8c,0xd0,0x72,0xcf,0x86,0xd5,0x4c,0xd4,0x30,0xd3,0x18, + 0xd2,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x0c,0x00,0x0c,0x00,0x51,0x04,0x0c,0x00, + 0x10,0x04,0x0c,0x00,0x00,0x00,0xd2,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x0c,0x00, + 0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00,0x00,0x00,0x93,0x18,0xd2,0x0c, + 0x91,0x08,0x10,0x04,0x00,0x00,0x0c,0x00,0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04, + 0x0c,0x00,0x00,0x00,0x00,0x00,0x94,0x20,0xd3,0x10,0x52,0x04,0x0c,0x00,0x51,0x04, + 0x0c,0x00,0x10,0x04,0x0c,0x00,0x00,0x00,0x52,0x04,0x0c,0x00,0x51,0x04,0x0c,0x00, + 0x10,0x04,0x0c,0x00,0x00,0x00,0x10,0x00,0xcf,0x86,0x55,0x04,0x10,0x00,0x94,0x10, + 0x93,0x0c,0x52,0x04,0x11,0x00,0x11,0x04,0x10,0x00,0x15,0x00,0x00,0x00,0x11,0x00, + 0xd0,0x06,0xcf,0x06,0x11,0x00,0xcf,0x86,0x55,0x04,0x0b,0x00,0xd4,0x14,0x53,0x04, + 0x0b,0x00,0x52,0x04,0x0b,0x00,0x91,0x08,0x10,0x04,0x0b,0x00,0x0b,0x09,0x00,0x00, + 0x53,0x04,0x0b,0x00,0x92,0x08,0x11,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0xcf,0x06, + 0x02,0xff,0xff,0xcf,0x86,0xcf,0x06,0x02,0xff,0xff,0xd1,0x76,0xd0,0x09,0xcf,0x86, + 0xcf,0x06,0x02,0xff,0xff,0xcf,0x86,0x85,0xd4,0x07,0xcf,0x06,0x02,0xff,0xff,0xd3, + 0x07,0xcf,0x06,0x02,0xff,0xff,0xd2,0x07,0xcf,0x06,0x02,0xff,0xff,0xd1,0x07,0xcf, + 0x06,0x02,0xff,0xff,0xd0,0x18,0xcf,0x86,0x55,0x05,0x02,0xff,0xff,0x94,0x0d,0x93, + 0x09,0x12,0x05,0x02,0xff,0xff,0x00,0x00,0x00,0x00,0x0b,0x00,0xcf,0x86,0xd5,0x24, + 0x94,0x20,0xd3,0x10,0x52,0x04,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b,0x00, + 0x00,0x00,0x92,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x0b,0x00,0x0b,0x00, + 0x0b,0x00,0x54,0x04,0x0b,0x00,0x53,0x04,0x0b,0x00,0x12,0x04,0x0b,0x00,0x00,0x00, + 0xd0,0x08,0xcf,0x86,0xcf,0x06,0x01,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x01,0x00, + 0xe4,0x9c,0x10,0xe3,0x16,0x08,0xd2,0x06,0xcf,0x06,0x01,0x00,0xe1,0x08,0x04,0xe0, + 0x04,0x02,0xcf,0x86,0xe5,0x01,0x01,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10, + 0x08,0x01,0xff,0xe8,0xb1,0x88,0x00,0x01,0xff,0xe6,0x9b,0xb4,0x00,0x10,0x08,0x01, + 0xff,0xe8,0xbb,0x8a,0x00,0x01,0xff,0xe8,0xb3,0x88,0x00,0xd1,0x10,0x10,0x08,0x01, + 0xff,0xe6,0xbb,0x91,0x00,0x01,0xff,0xe4,0xb8,0xb2,0x00,0x10,0x08,0x01,0xff,0xe5, + 0x8f,0xa5,0x00,0x01,0xff,0xe9,0xbe,0x9c,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01, + 0xff,0xe9,0xbe,0x9c,0x00,0x01,0xff,0xe5,0xa5,0x91,0x00,0x10,0x08,0x01,0xff,0xe9, + 0x87,0x91,0x00,0x01,0xff,0xe5,0x96,0x87,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe5, + 0xa5,0x88,0x00,0x01,0xff,0xe6,0x87,0xb6,0x00,0x10,0x08,0x01,0xff,0xe7,0x99,0xa9, + 0x00,0x01,0xff,0xe7,0xbe,0x85,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01, + 0xff,0xe8,0x98,0xbf,0x00,0x01,0xff,0xe8,0x9e,0xba,0x00,0x10,0x08,0x01,0xff,0xe8, + 0xa3,0xb8,0x00,0x01,0xff,0xe9,0x82,0x8f,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe6, + 0xa8,0x82,0x00,0x01,0xff,0xe6,0xb4,0x9b,0x00,0x10,0x08,0x01,0xff,0xe7,0x83,0x99, + 0x00,0x01,0xff,0xe7,0x8f,0x9e,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe8, + 0x90,0xbd,0x00,0x01,0xff,0xe9,0x85,0xaa,0x00,0x10,0x08,0x01,0xff,0xe9,0xa7,0xb1, + 0x00,0x01,0xff,0xe4,0xba,0x82,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe5,0x8d,0xb5, + 0x00,0x01,0xff,0xe6,0xac,0x84,0x00,0x10,0x08,0x01,0xff,0xe7,0x88,0x9b,0x00,0x01, + 0xff,0xe8,0x98,0xad,0x00,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01, + 0xff,0xe9,0xb8,0x9e,0x00,0x01,0xff,0xe5,0xb5,0x90,0x00,0x10,0x08,0x01,0xff,0xe6, + 0xbf,0xab,0x00,0x01,0xff,0xe8,0x97,0x8d,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe8, + 0xa5,0xa4,0x00,0x01,0xff,0xe6,0x8b,0x89,0x00,0x10,0x08,0x01,0xff,0xe8,0x87,0x98, + 0x00,0x01,0xff,0xe8,0xa0,0x9f,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe5, + 0xbb,0x8a,0x00,0x01,0xff,0xe6,0x9c,0x97,0x00,0x10,0x08,0x01,0xff,0xe6,0xb5,0xaa, + 0x00,0x01,0xff,0xe7,0x8b,0xbc,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe9,0x83,0x8e, + 0x00,0x01,0xff,0xe4,0xbe,0x86,0x00,0x10,0x08,0x01,0xff,0xe5,0x86,0xb7,0x00,0x01, + 0xff,0xe5,0x8b,0x9e,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe6, + 0x93,0x84,0x00,0x01,0xff,0xe6,0xab,0x93,0x00,0x10,0x08,0x01,0xff,0xe7,0x88,0x90, + 0x00,0x01,0xff,0xe7,0x9b,0xa7,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe8,0x80,0x81, + 0x00,0x01,0xff,0xe8,0x98,0x86,0x00,0x10,0x08,0x01,0xff,0xe8,0x99,0x9c,0x00,0x01, + 0xff,0xe8,0xb7,0xaf,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe9,0x9c,0xb2, + 0x00,0x01,0xff,0xe9,0xad,0xaf,0x00,0x10,0x08,0x01,0xff,0xe9,0xb7,0xba,0x00,0x01, + 0xff,0xe7,0xa2,0x8c,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe7,0xa5,0xbf,0x00,0x01, + 0xff,0xe7,0xb6,0xa0,0x00,0x10,0x08,0x01,0xff,0xe8,0x8f,0x89,0x00,0x01,0xff,0xe9, + 0x8c,0x84,0x00,0xcf,0x86,0xe5,0x01,0x01,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x01,0xff,0xe9,0xb9,0xbf,0x00,0x01,0xff,0xe8,0xab,0x96,0x00,0x10,0x08, + 0x01,0xff,0xe5,0xa3,0x9f,0x00,0x01,0xff,0xe5,0xbc,0x84,0x00,0xd1,0x10,0x10,0x08, + 0x01,0xff,0xe7,0xb1,0xa0,0x00,0x01,0xff,0xe8,0x81,0xbe,0x00,0x10,0x08,0x01,0xff, + 0xe7,0x89,0xa2,0x00,0x01,0xff,0xe7,0xa3,0x8a,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x01,0xff,0xe8,0xb3,0x82,0x00,0x01,0xff,0xe9,0x9b,0xb7,0x00,0x10,0x08,0x01,0xff, + 0xe5,0xa3,0x98,0x00,0x01,0xff,0xe5,0xb1,0xa2,0x00,0xd1,0x10,0x10,0x08,0x01,0xff, + 0xe6,0xa8,0x93,0x00,0x01,0xff,0xe6,0xb7,0x9a,0x00,0x10,0x08,0x01,0xff,0xe6,0xbc, + 0x8f,0x00,0x01,0xff,0xe7,0xb4,0xaf,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x01,0xff,0xe7,0xb8,0xb7,0x00,0x01,0xff,0xe9,0x99,0x8b,0x00,0x10,0x08,0x01,0xff, + 0xe5,0x8b,0x92,0x00,0x01,0xff,0xe8,0x82,0x8b,0x00,0xd1,0x10,0x10,0x08,0x01,0xff, + 0xe5,0x87,0x9c,0x00,0x01,0xff,0xe5,0x87,0x8c,0x00,0x10,0x08,0x01,0xff,0xe7,0xa8, + 0x9c,0x00,0x01,0xff,0xe7,0xb6,0xbe,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff, + 0xe8,0x8f,0xb1,0x00,0x01,0xff,0xe9,0x99,0xb5,0x00,0x10,0x08,0x01,0xff,0xe8,0xae, + 0x80,0x00,0x01,0xff,0xe6,0x8b,0x8f,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe6,0xa8, + 0x82,0x00,0x01,0xff,0xe8,0xab,0xbe,0x00,0x10,0x08,0x01,0xff,0xe4,0xb8,0xb9,0x00, + 0x01,0xff,0xe5,0xaf,0xa7,0x00,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x01,0xff,0xe6,0x80,0x92,0x00,0x01,0xff,0xe7,0x8e,0x87,0x00,0x10,0x08,0x01,0xff, + 0xe7,0x95,0xb0,0x00,0x01,0xff,0xe5,0x8c,0x97,0x00,0xd1,0x10,0x10,0x08,0x01,0xff, + 0xe7,0xa3,0xbb,0x00,0x01,0xff,0xe4,0xbe,0xbf,0x00,0x10,0x08,0x01,0xff,0xe5,0xbe, + 0xa9,0x00,0x01,0xff,0xe4,0xb8,0x8d,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff, + 0xe6,0xb3,0x8c,0x00,0x01,0xff,0xe6,0x95,0xb8,0x00,0x10,0x08,0x01,0xff,0xe7,0xb4, + 0xa2,0x00,0x01,0xff,0xe5,0x8f,0x83,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe5,0xa1, + 0x9e,0x00,0x01,0xff,0xe7,0x9c,0x81,0x00,0x10,0x08,0x01,0xff,0xe8,0x91,0x89,0x00, + 0x01,0xff,0xe8,0xaa,0xaa,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff, + 0xe6,0xae,0xba,0x00,0x01,0xff,0xe8,0xbe,0xb0,0x00,0x10,0x08,0x01,0xff,0xe6,0xb2, + 0x88,0x00,0x01,0xff,0xe6,0x8b,0xbe,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe8,0x8b, + 0xa5,0x00,0x01,0xff,0xe6,0x8e,0xa0,0x00,0x10,0x08,0x01,0xff,0xe7,0x95,0xa5,0x00, + 0x01,0xff,0xe4,0xba,0xae,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe5,0x85, + 0xa9,0x00,0x01,0xff,0xe5,0x87,0x89,0x00,0x10,0x08,0x01,0xff,0xe6,0xa2,0x81,0x00, + 0x01,0xff,0xe7,0xb3,0xa7,0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe8,0x89,0xaf,0x00, + 0x01,0xff,0xe8,0xab,0x92,0x00,0x10,0x08,0x01,0xff,0xe9,0x87,0x8f,0x00,0x01,0xff, + 0xe5,0x8b,0xb5,0x00,0xe0,0x04,0x02,0xcf,0x86,0xe5,0x01,0x01,0xd4,0x80,0xd3,0x40, + 0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe5,0x91,0x82,0x00,0x01,0xff,0xe5,0xa5, + 0xb3,0x00,0x10,0x08,0x01,0xff,0xe5,0xbb,0xac,0x00,0x01,0xff,0xe6,0x97,0x85,0x00, + 0xd1,0x10,0x10,0x08,0x01,0xff,0xe6,0xbf,0xbe,0x00,0x01,0xff,0xe7,0xa4,0xaa,0x00, + 0x10,0x08,0x01,0xff,0xe9,0x96,0xad,0x00,0x01,0xff,0xe9,0xa9,0xaa,0x00,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x01,0xff,0xe9,0xba,0x97,0x00,0x01,0xff,0xe9,0xbb,0x8e,0x00, + 0x10,0x08,0x01,0xff,0xe5,0x8a,0x9b,0x00,0x01,0xff,0xe6,0x9b,0x86,0x00,0xd1,0x10, + 0x10,0x08,0x01,0xff,0xe6,0xad,0xb7,0x00,0x01,0xff,0xe8,0xbd,0xa2,0x00,0x10,0x08, + 0x01,0xff,0xe5,0xb9,0xb4,0x00,0x01,0xff,0xe6,0x86,0x90,0x00,0xd3,0x40,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x01,0xff,0xe6,0x88,0x80,0x00,0x01,0xff,0xe6,0x92,0x9a,0x00, + 0x10,0x08,0x01,0xff,0xe6,0xbc,0xa3,0x00,0x01,0xff,0xe7,0x85,0x89,0x00,0xd1,0x10, + 0x10,0x08,0x01,0xff,0xe7,0x92,0x89,0x00,0x01,0xff,0xe7,0xa7,0x8a,0x00,0x10,0x08, + 0x01,0xff,0xe7,0xb7,0xb4,0x00,0x01,0xff,0xe8,0x81,0xaf,0x00,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x01,0xff,0xe8,0xbc,0xa6,0x00,0x01,0xff,0xe8,0x93,0xae,0x00,0x10,0x08, + 0x01,0xff,0xe9,0x80,0xa3,0x00,0x01,0xff,0xe9,0x8d,0x8a,0x00,0xd1,0x10,0x10,0x08, + 0x01,0xff,0xe5,0x88,0x97,0x00,0x01,0xff,0xe5,0x8a,0xa3,0x00,0x10,0x08,0x01,0xff, + 0xe5,0x92,0xbd,0x00,0x01,0xff,0xe7,0x83,0x88,0x00,0xd4,0x80,0xd3,0x40,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x01,0xff,0xe8,0xa3,0x82,0x00,0x01,0xff,0xe8,0xaa,0xaa,0x00, + 0x10,0x08,0x01,0xff,0xe5,0xbb,0x89,0x00,0x01,0xff,0xe5,0xbf,0xb5,0x00,0xd1,0x10, + 0x10,0x08,0x01,0xff,0xe6,0x8d,0xbb,0x00,0x01,0xff,0xe6,0xae,0xae,0x00,0x10,0x08, + 0x01,0xff,0xe7,0xb0,0xbe,0x00,0x01,0xff,0xe7,0x8d,0xb5,0x00,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x01,0xff,0xe4,0xbb,0xa4,0x00,0x01,0xff,0xe5,0x9b,0xb9,0x00,0x10,0x08, + 0x01,0xff,0xe5,0xaf,0xa7,0x00,0x01,0xff,0xe5,0xb6,0xba,0x00,0xd1,0x10,0x10,0x08, + 0x01,0xff,0xe6,0x80,0x9c,0x00,0x01,0xff,0xe7,0x8e,0xb2,0x00,0x10,0x08,0x01,0xff, + 0xe7,0x91,0xa9,0x00,0x01,0xff,0xe7,0xbe,0x9a,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x01,0xff,0xe8,0x81,0x86,0x00,0x01,0xff,0xe9,0x88,0xb4,0x00,0x10,0x08, + 0x01,0xff,0xe9,0x9b,0xb6,0x00,0x01,0xff,0xe9,0x9d,0x88,0x00,0xd1,0x10,0x10,0x08, + 0x01,0xff,0xe9,0xa0,0x98,0x00,0x01,0xff,0xe4,0xbe,0x8b,0x00,0x10,0x08,0x01,0xff, + 0xe7,0xa6,0xae,0x00,0x01,0xff,0xe9,0x86,0xb4,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x01,0xff,0xe9,0x9a,0xb8,0x00,0x01,0xff,0xe6,0x83,0xa1,0x00,0x10,0x08,0x01,0xff, + 0xe4,0xba,0x86,0x00,0x01,0xff,0xe5,0x83,0x9a,0x00,0xd1,0x10,0x10,0x08,0x01,0xff, + 0xe5,0xaf,0xae,0x00,0x01,0xff,0xe5,0xb0,0xbf,0x00,0x10,0x08,0x01,0xff,0xe6,0x96, + 0x99,0x00,0x01,0xff,0xe6,0xa8,0x82,0x00,0xcf,0x86,0xe5,0x01,0x01,0xd4,0x80,0xd3, + 0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe7,0x87,0x8e,0x00,0x01,0xff,0xe7, + 0x99,0x82,0x00,0x10,0x08,0x01,0xff,0xe8,0x93,0xbc,0x00,0x01,0xff,0xe9,0x81,0xbc, + 0x00,0xd1,0x10,0x10,0x08,0x01,0xff,0xe9,0xbe,0x8d,0x00,0x01,0xff,0xe6,0x9a,0x88, + 0x00,0x10,0x08,0x01,0xff,0xe9,0x98,0xae,0x00,0x01,0xff,0xe5,0x8a,0x89,0x00,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe6,0x9d,0xbb,0x00,0x01,0xff,0xe6,0x9f,0xb3, + 0x00,0x10,0x08,0x01,0xff,0xe6,0xb5,0x81,0x00,0x01,0xff,0xe6,0xba,0x9c,0x00,0xd1, + 0x10,0x10,0x08,0x01,0xff,0xe7,0x90,0x89,0x00,0x01,0xff,0xe7,0x95,0x99,0x00,0x10, + 0x08,0x01,0xff,0xe7,0xa1,0xab,0x00,0x01,0xff,0xe7,0xb4,0x90,0x00,0xd3,0x40,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe9,0xa1,0x9e,0x00,0x01,0xff,0xe5,0x85,0xad, + 0x00,0x10,0x08,0x01,0xff,0xe6,0x88,0xae,0x00,0x01,0xff,0xe9,0x99,0xb8,0x00,0xd1, + 0x10,0x10,0x08,0x01,0xff,0xe5,0x80,0xab,0x00,0x01,0xff,0xe5,0xb4,0x99,0x00,0x10, + 0x08,0x01,0xff,0xe6,0xb7,0xaa,0x00,0x01,0xff,0xe8,0xbc,0xaa,0x00,0xd2,0x20,0xd1, + 0x10,0x10,0x08,0x01,0xff,0xe5,0xbe,0x8b,0x00,0x01,0xff,0xe6,0x85,0x84,0x00,0x10, + 0x08,0x01,0xff,0xe6,0xa0,0x97,0x00,0x01,0xff,0xe7,0x8e,0x87,0x00,0xd1,0x10,0x10, + 0x08,0x01,0xff,0xe9,0x9a,0x86,0x00,0x01,0xff,0xe5,0x88,0xa9,0x00,0x10,0x08,0x01, + 0xff,0xe5,0x90,0x8f,0x00,0x01,0xff,0xe5,0xb1,0xa5,0x00,0xd4,0x80,0xd3,0x40,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x01,0xff,0xe6,0x98,0x93,0x00,0x01,0xff,0xe6,0x9d,0x8e, + 0x00,0x10,0x08,0x01,0xff,0xe6,0xa2,0xa8,0x00,0x01,0xff,0xe6,0xb3,0xa5,0x00,0xd1, + 0x10,0x10,0x08,0x01,0xff,0xe7,0x90,0x86,0x00,0x01,0xff,0xe7,0x97,0xa2,0x00,0x10, + 0x08,0x01,0xff,0xe7,0xbd,0xb9,0x00,0x01,0xff,0xe8,0xa3,0x8f,0x00,0xd2,0x20,0xd1, + 0x10,0x10,0x08,0x01,0xff,0xe8,0xa3,0xa1,0x00,0x01,0xff,0xe9,0x87,0x8c,0x00,0x10, + 0x08,0x01,0xff,0xe9,0x9b,0xa2,0x00,0x01,0xff,0xe5,0x8c,0xbf,0x00,0xd1,0x10,0x10, + 0x08,0x01,0xff,0xe6,0xba,0xba,0x00,0x01,0xff,0xe5,0x90,0x9d,0x00,0x10,0x08,0x01, + 0xff,0xe7,0x87,0x90,0x00,0x01,0xff,0xe7,0x92,0x98,0x00,0xd3,0x40,0xd2,0x20,0xd1, + 0x10,0x10,0x08,0x01,0xff,0xe8,0x97,0xba,0x00,0x01,0xff,0xe9,0x9a,0xa3,0x00,0x10, + 0x08,0x01,0xff,0xe9,0xb1,0x97,0x00,0x01,0xff,0xe9,0xba,0x9f,0x00,0xd1,0x10,0x10, + 0x08,0x01,0xff,0xe6,0x9e,0x97,0x00,0x01,0xff,0xe6,0xb7,0x8b,0x00,0x10,0x08,0x01, + 0xff,0xe8,0x87,0xa8,0x00,0x01,0xff,0xe7,0xab,0x8b,0x00,0xd2,0x20,0xd1,0x10,0x10, + 0x08,0x01,0xff,0xe7,0xac,0xa0,0x00,0x01,0xff,0xe7,0xb2,0x92,0x00,0x10,0x08,0x01, + 0xff,0xe7,0x8b,0x80,0x00,0x01,0xff,0xe7,0x82,0x99,0x00,0xd1,0x10,0x10,0x08,0x01, + 0xff,0xe8,0xad,0x98,0x00,0x01,0xff,0xe4,0xbb,0x80,0x00,0x10,0x08,0x01,0xff,0xe8, + 0x8c,0xb6,0x00,0x01,0xff,0xe5,0x88,0xba,0x00,0xe2,0xad,0x06,0xe1,0xc4,0x03,0xe0, + 0xcb,0x01,0xcf,0x86,0xd5,0xe4,0xd4,0x74,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x01,0xff,0xe5,0x88,0x87,0x00,0x01,0xff,0xe5,0xba,0xa6,0x00,0x10,0x08,0x01,0xff, + 0xe6,0x8b,0x93,0x00,0x01,0xff,0xe7,0xb3,0x96,0x00,0xd1,0x10,0x10,0x08,0x01,0xff, + 0xe5,0xae,0x85,0x00,0x01,0xff,0xe6,0xb4,0x9e,0x00,0x10,0x08,0x01,0xff,0xe6,0x9a, + 0xb4,0x00,0x01,0xff,0xe8,0xbc,0xbb,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x01,0xff, + 0xe8,0xa1,0x8c,0x00,0x01,0xff,0xe9,0x99,0x8d,0x00,0x10,0x08,0x01,0xff,0xe8,0xa6, + 0x8b,0x00,0x01,0xff,0xe5,0xbb,0x93,0x00,0x91,0x10,0x10,0x08,0x01,0xff,0xe5,0x85, + 0x80,0x00,0x01,0xff,0xe5,0x97,0x80,0x00,0x01,0x00,0xd3,0x34,0xd2,0x18,0xd1,0x0c, + 0x10,0x08,0x01,0xff,0xe5,0xa1,0x9a,0x00,0x01,0x00,0x10,0x08,0x01,0xff,0xe6,0x99, + 0xb4,0x00,0x01,0x00,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0xe5,0x87,0x9e,0x00, + 0x10,0x08,0x01,0xff,0xe7,0x8c,0xaa,0x00,0x01,0xff,0xe7,0x9b,0x8a,0x00,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x01,0xff,0xe7,0xa4,0xbc,0x00,0x01,0xff,0xe7,0xa5,0x9e,0x00, + 0x10,0x08,0x01,0xff,0xe7,0xa5,0xa5,0x00,0x01,0xff,0xe7,0xa6,0x8f,0x00,0xd1,0x10, + 0x10,0x08,0x01,0xff,0xe9,0x9d,0x96,0x00,0x01,0xff,0xe7,0xb2,0xbe,0x00,0x10,0x08, + 0x01,0xff,0xe7,0xbe,0xbd,0x00,0x01,0x00,0xd4,0x64,0xd3,0x30,0xd2,0x18,0xd1,0x0c, + 0x10,0x08,0x01,0xff,0xe8,0x98,0x92,0x00,0x01,0x00,0x10,0x08,0x01,0xff,0xe8,0xab, + 0xb8,0x00,0x01,0x00,0xd1,0x0c,0x10,0x04,0x01,0x00,0x01,0xff,0xe9,0x80,0xb8,0x00, + 0x10,0x08,0x01,0xff,0xe9,0x83,0xbd,0x00,0x01,0x00,0xd2,0x14,0x51,0x04,0x01,0x00, + 0x10,0x08,0x01,0xff,0xe9,0xa3,0xaf,0x00,0x01,0xff,0xe9,0xa3,0xbc,0x00,0xd1,0x10, + 0x10,0x08,0x01,0xff,0xe9,0xa4,0xa8,0x00,0x01,0xff,0xe9,0xb6,0xb4,0x00,0x10,0x08, + 0x0d,0xff,0xe9,0x83,0x9e,0x00,0x0d,0xff,0xe9,0x9a,0xb7,0x00,0xd3,0x40,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x06,0xff,0xe4,0xbe,0xae,0x00,0x06,0xff,0xe5,0x83,0xa7,0x00, + 0x10,0x08,0x06,0xff,0xe5,0x85,0x8d,0x00,0x06,0xff,0xe5,0x8b,0x89,0x00,0xd1,0x10, + 0x10,0x08,0x06,0xff,0xe5,0x8b,0xa4,0x00,0x06,0xff,0xe5,0x8d,0x91,0x00,0x10,0x08, + 0x06,0xff,0xe5,0x96,0x9d,0x00,0x06,0xff,0xe5,0x98,0x86,0x00,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x06,0xff,0xe5,0x99,0xa8,0x00,0x06,0xff,0xe5,0xa1,0x80,0x00,0x10,0x08, + 0x06,0xff,0xe5,0xa2,0xa8,0x00,0x06,0xff,0xe5,0xb1,0xa4,0x00,0xd1,0x10,0x10,0x08, + 0x06,0xff,0xe5,0xb1,0xae,0x00,0x06,0xff,0xe6,0x82,0x94,0x00,0x10,0x08,0x06,0xff, + 0xe6,0x85,0xa8,0x00,0x06,0xff,0xe6,0x86,0x8e,0x00,0xcf,0x86,0xe5,0x01,0x01,0xd4, + 0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x06,0xff,0xe6,0x87,0xb2,0x00,0x06, + 0xff,0xe6,0x95,0x8f,0x00,0x10,0x08,0x06,0xff,0xe6,0x97,0xa2,0x00,0x06,0xff,0xe6, + 0x9a,0x91,0x00,0xd1,0x10,0x10,0x08,0x06,0xff,0xe6,0xa2,0x85,0x00,0x06,0xff,0xe6, + 0xb5,0xb7,0x00,0x10,0x08,0x06,0xff,0xe6,0xb8,0x9a,0x00,0x06,0xff,0xe6,0xbc,0xa2, + 0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x06,0xff,0xe7,0x85,0xae,0x00,0x06,0xff,0xe7, + 0x88,0xab,0x00,0x10,0x08,0x06,0xff,0xe7,0x90,0xa2,0x00,0x06,0xff,0xe7,0xa2,0x91, + 0x00,0xd1,0x10,0x10,0x08,0x06,0xff,0xe7,0xa4,0xbe,0x00,0x06,0xff,0xe7,0xa5,0x89, + 0x00,0x10,0x08,0x06,0xff,0xe7,0xa5,0x88,0x00,0x06,0xff,0xe7,0xa5,0x90,0x00,0xd3, + 0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x06,0xff,0xe7,0xa5,0x96,0x00,0x06,0xff,0xe7, + 0xa5,0x9d,0x00,0x10,0x08,0x06,0xff,0xe7,0xa6,0x8d,0x00,0x06,0xff,0xe7,0xa6,0x8e, + 0x00,0xd1,0x10,0x10,0x08,0x06,0xff,0xe7,0xa9,0x80,0x00,0x06,0xff,0xe7,0xaa,0x81, + 0x00,0x10,0x08,0x06,0xff,0xe7,0xaf,0x80,0x00,0x06,0xff,0xe7,0xb7,0xb4,0x00,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x06,0xff,0xe7,0xb8,0x89,0x00,0x06,0xff,0xe7,0xb9,0x81, + 0x00,0x10,0x08,0x06,0xff,0xe7,0xbd,0xb2,0x00,0x06,0xff,0xe8,0x80,0x85,0x00,0xd1, + 0x10,0x10,0x08,0x06,0xff,0xe8,0x87,0xad,0x00,0x06,0xff,0xe8,0x89,0xb9,0x00,0x10, + 0x08,0x06,0xff,0xe8,0x89,0xb9,0x00,0x06,0xff,0xe8,0x91,0x97,0x00,0xd4,0x75,0xd3, + 0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x06,0xff,0xe8,0xa4,0x90,0x00,0x06,0xff,0xe8, + 0xa6,0x96,0x00,0x10,0x08,0x06,0xff,0xe8,0xac,0x81,0x00,0x06,0xff,0xe8,0xac,0xb9, + 0x00,0xd1,0x10,0x10,0x08,0x06,0xff,0xe8,0xb3,0x93,0x00,0x06,0xff,0xe8,0xb4,0x88, + 0x00,0x10,0x08,0x06,0xff,0xe8,0xbe,0xb6,0x00,0x06,0xff,0xe9,0x80,0xb8,0x00,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x06,0xff,0xe9,0x9b,0xa3,0x00,0x06,0xff,0xe9,0x9f,0xbf, + 0x00,0x10,0x08,0x06,0xff,0xe9,0xa0,0xbb,0x00,0x0b,0xff,0xe6,0x81,0xb5,0x00,0x91, + 0x11,0x10,0x09,0x0b,0xff,0xf0,0xa4,0x8b,0xae,0x00,0x0b,0xff,0xe8,0x88,0x98,0x00, + 0x00,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x08,0xff,0xe4,0xb8,0xa6,0x00, + 0x08,0xff,0xe5,0x86,0xb5,0x00,0x10,0x08,0x08,0xff,0xe5,0x85,0xa8,0x00,0x08,0xff, + 0xe4,0xbe,0x80,0x00,0xd1,0x10,0x10,0x08,0x08,0xff,0xe5,0x85,0x85,0x00,0x08,0xff, + 0xe5,0x86,0x80,0x00,0x10,0x08,0x08,0xff,0xe5,0x8b,0x87,0x00,0x08,0xff,0xe5,0x8b, + 0xba,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x08,0xff,0xe5,0x96,0x9d,0x00,0x08,0xff, + 0xe5,0x95,0x95,0x00,0x10,0x08,0x08,0xff,0xe5,0x96,0x99,0x00,0x08,0xff,0xe5,0x97, + 0xa2,0x00,0xd1,0x10,0x10,0x08,0x08,0xff,0xe5,0xa1,0x9a,0x00,0x08,0xff,0xe5,0xa2, + 0xb3,0x00,0x10,0x08,0x08,0xff,0xe5,0xa5,0x84,0x00,0x08,0xff,0xe5,0xa5,0x94,0x00, + 0xe0,0x04,0x02,0xcf,0x86,0xe5,0x01,0x01,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x08,0xff,0xe5,0xa9,0xa2,0x00,0x08,0xff,0xe5,0xac,0xa8,0x00,0x10,0x08, + 0x08,0xff,0xe5,0xbb,0x92,0x00,0x08,0xff,0xe5,0xbb,0x99,0x00,0xd1,0x10,0x10,0x08, + 0x08,0xff,0xe5,0xbd,0xa9,0x00,0x08,0xff,0xe5,0xbe,0xad,0x00,0x10,0x08,0x08,0xff, + 0xe6,0x83,0x98,0x00,0x08,0xff,0xe6,0x85,0x8e,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x08,0xff,0xe6,0x84,0x88,0x00,0x08,0xff,0xe6,0x86,0x8e,0x00,0x10,0x08,0x08,0xff, + 0xe6,0x85,0xa0,0x00,0x08,0xff,0xe6,0x87,0xb2,0x00,0xd1,0x10,0x10,0x08,0x08,0xff, + 0xe6,0x88,0xb4,0x00,0x08,0xff,0xe6,0x8f,0x84,0x00,0x10,0x08,0x08,0xff,0xe6,0x90, + 0x9c,0x00,0x08,0xff,0xe6,0x91,0x92,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x08,0xff,0xe6,0x95,0x96,0x00,0x08,0xff,0xe6,0x99,0xb4,0x00,0x10,0x08,0x08,0xff, + 0xe6,0x9c,0x97,0x00,0x08,0xff,0xe6,0x9c,0x9b,0x00,0xd1,0x10,0x10,0x08,0x08,0xff, + 0xe6,0x9d,0x96,0x00,0x08,0xff,0xe6,0xad,0xb9,0x00,0x10,0x08,0x08,0xff,0xe6,0xae, + 0xba,0x00,0x08,0xff,0xe6,0xb5,0x81,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x08,0xff, + 0xe6,0xbb,0x9b,0x00,0x08,0xff,0xe6,0xbb,0x8b,0x00,0x10,0x08,0x08,0xff,0xe6,0xbc, + 0xa2,0x00,0x08,0xff,0xe7,0x80,0x9e,0x00,0xd1,0x10,0x10,0x08,0x08,0xff,0xe7,0x85, + 0xae,0x00,0x08,0xff,0xe7,0x9e,0xa7,0x00,0x10,0x08,0x08,0xff,0xe7,0x88,0xb5,0x00, + 0x08,0xff,0xe7,0x8a,0xaf,0x00,0xd4,0x80,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x08,0xff,0xe7,0x8c,0xaa,0x00,0x08,0xff,0xe7,0x91,0xb1,0x00,0x10,0x08,0x08,0xff, + 0xe7,0x94,0x86,0x00,0x08,0xff,0xe7,0x94,0xbb,0x00,0xd1,0x10,0x10,0x08,0x08,0xff, + 0xe7,0x98,0x9d,0x00,0x08,0xff,0xe7,0x98,0x9f,0x00,0x10,0x08,0x08,0xff,0xe7,0x9b, + 0x8a,0x00,0x08,0xff,0xe7,0x9b,0x9b,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x08,0xff, + 0xe7,0x9b,0xb4,0x00,0x08,0xff,0xe7,0x9d,0x8a,0x00,0x10,0x08,0x08,0xff,0xe7,0x9d, + 0x80,0x00,0x08,0xff,0xe7,0xa3,0x8c,0x00,0xd1,0x10,0x10,0x08,0x08,0xff,0xe7,0xaa, + 0xb1,0x00,0x08,0xff,0xe7,0xaf,0x80,0x00,0x10,0x08,0x08,0xff,0xe7,0xb1,0xbb,0x00, + 0x08,0xff,0xe7,0xb5,0x9b,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x08,0xff, + 0xe7,0xb7,0xb4,0x00,0x08,0xff,0xe7,0xbc,0xbe,0x00,0x10,0x08,0x08,0xff,0xe8,0x80, + 0x85,0x00,0x08,0xff,0xe8,0x8d,0x92,0x00,0xd1,0x10,0x10,0x08,0x08,0xff,0xe8,0x8f, + 0xaf,0x00,0x08,0xff,0xe8,0x9d,0xb9,0x00,0x10,0x08,0x08,0xff,0xe8,0xa5,0x81,0x00, + 0x08,0xff,0xe8,0xa6,0x86,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x08,0xff,0xe8,0xa6, + 0x96,0x00,0x08,0xff,0xe8,0xaa,0xbf,0x00,0x10,0x08,0x08,0xff,0xe8,0xab,0xb8,0x00, + 0x08,0xff,0xe8,0xab,0x8b,0x00,0xd1,0x10,0x10,0x08,0x08,0xff,0xe8,0xac,0x81,0x00, + 0x08,0xff,0xe8,0xab,0xbe,0x00,0x10,0x08,0x08,0xff,0xe8,0xab,0xad,0x00,0x08,0xff, + 0xe8,0xac,0xb9,0x00,0xcf,0x86,0x95,0xde,0xd4,0x81,0xd3,0x40,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x08,0xff,0xe8,0xae,0x8a,0x00,0x08,0xff,0xe8,0xb4,0x88,0x00,0x10,0x08, + 0x08,0xff,0xe8,0xbc,0xb8,0x00,0x08,0xff,0xe9,0x81,0xb2,0x00,0xd1,0x10,0x10,0x08, + 0x08,0xff,0xe9,0x86,0x99,0x00,0x08,0xff,0xe9,0x89,0xb6,0x00,0x10,0x08,0x08,0xff, + 0xe9,0x99,0xbc,0x00,0x08,0xff,0xe9,0x9b,0xa3,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x08,0xff,0xe9,0x9d,0x96,0x00,0x08,0xff,0xe9,0x9f,0x9b,0x00,0x10,0x08,0x08,0xff, + 0xe9,0x9f,0xbf,0x00,0x08,0xff,0xe9,0xa0,0x8b,0x00,0xd1,0x10,0x10,0x08,0x08,0xff, + 0xe9,0xa0,0xbb,0x00,0x08,0xff,0xe9,0xac,0x92,0x00,0x10,0x08,0x08,0xff,0xe9,0xbe, + 0x9c,0x00,0x08,0xff,0xf0,0xa2,0xa1,0x8a,0x00,0xd3,0x45,0xd2,0x22,0xd1,0x12,0x10, + 0x09,0x08,0xff,0xf0,0xa2,0xa1,0x84,0x00,0x08,0xff,0xf0,0xa3,0x8f,0x95,0x00,0x10, + 0x08,0x08,0xff,0xe3,0xae,0x9d,0x00,0x08,0xff,0xe4,0x80,0x98,0x00,0xd1,0x11,0x10, + 0x08,0x08,0xff,0xe4,0x80,0xb9,0x00,0x08,0xff,0xf0,0xa5,0x89,0x89,0x00,0x10,0x09, + 0x08,0xff,0xf0,0xa5,0xb3,0x90,0x00,0x08,0xff,0xf0,0xa7,0xbb,0x93,0x00,0x92,0x14, + 0x91,0x10,0x10,0x08,0x08,0xff,0xe9,0xbd,0x83,0x00,0x08,0xff,0xe9,0xbe,0x8e,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0xe1,0x94,0x01,0xe0,0x08,0x01,0xcf,0x86,0xd5,0x42, + 0xd4,0x14,0x93,0x10,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00, + 0x00,0x00,0x00,0x00,0xd3,0x10,0x92,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00, + 0x01,0x00,0x01,0x00,0x52,0x04,0x00,0x00,0xd1,0x0d,0x10,0x04,0x00,0x00,0x04,0xff, + 0xd7,0x99,0xd6,0xb4,0x00,0x10,0x04,0x01,0x1a,0x01,0xff,0xd7,0xb2,0xd6,0xb7,0x00, + 0xd4,0x42,0x53,0x04,0x01,0x00,0xd2,0x16,0x51,0x04,0x01,0x00,0x10,0x09,0x01,0xff, + 0xd7,0xa9,0xd7,0x81,0x00,0x01,0xff,0xd7,0xa9,0xd7,0x82,0x00,0xd1,0x16,0x10,0x0b, + 0x01,0xff,0xd7,0xa9,0xd6,0xbc,0xd7,0x81,0x00,0x01,0xff,0xd7,0xa9,0xd6,0xbc,0xd7, + 0x82,0x00,0x10,0x09,0x01,0xff,0xd7,0x90,0xd6,0xb7,0x00,0x01,0xff,0xd7,0x90,0xd6, + 0xb8,0x00,0xd3,0x43,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xd7,0x90,0xd6,0xbc, + 0x00,0x01,0xff,0xd7,0x91,0xd6,0xbc,0x00,0x10,0x09,0x01,0xff,0xd7,0x92,0xd6,0xbc, + 0x00,0x01,0xff,0xd7,0x93,0xd6,0xbc,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xd7,0x94, + 0xd6,0xbc,0x00,0x01,0xff,0xd7,0x95,0xd6,0xbc,0x00,0x10,0x09,0x01,0xff,0xd7,0x96, + 0xd6,0xbc,0x00,0x00,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xd7,0x98,0xd6, + 0xbc,0x00,0x01,0xff,0xd7,0x99,0xd6,0xbc,0x00,0x10,0x09,0x01,0xff,0xd7,0x9a,0xd6, + 0xbc,0x00,0x01,0xff,0xd7,0x9b,0xd6,0xbc,0x00,0xd1,0x0d,0x10,0x09,0x01,0xff,0xd7, + 0x9c,0xd6,0xbc,0x00,0x00,0x00,0x10,0x09,0x01,0xff,0xd7,0x9e,0xd6,0xbc,0x00,0x00, + 0x00,0xcf,0x86,0x95,0x85,0x94,0x81,0xd3,0x3e,0xd2,0x1f,0xd1,0x12,0x10,0x09,0x01, + 0xff,0xd7,0xa0,0xd6,0xbc,0x00,0x01,0xff,0xd7,0xa1,0xd6,0xbc,0x00,0x10,0x04,0x00, + 0x00,0x01,0xff,0xd7,0xa3,0xd6,0xbc,0x00,0xd1,0x0d,0x10,0x09,0x01,0xff,0xd7,0xa4, + 0xd6,0xbc,0x00,0x00,0x00,0x10,0x09,0x01,0xff,0xd7,0xa6,0xd6,0xbc,0x00,0x01,0xff, + 0xd7,0xa7,0xd6,0xbc,0x00,0xd2,0x24,0xd1,0x12,0x10,0x09,0x01,0xff,0xd7,0xa8,0xd6, + 0xbc,0x00,0x01,0xff,0xd7,0xa9,0xd6,0xbc,0x00,0x10,0x09,0x01,0xff,0xd7,0xaa,0xd6, + 0xbc,0x00,0x01,0xff,0xd7,0x95,0xd6,0xb9,0x00,0xd1,0x12,0x10,0x09,0x01,0xff,0xd7, + 0x91,0xd6,0xbf,0x00,0x01,0xff,0xd7,0x9b,0xd6,0xbf,0x00,0x10,0x09,0x01,0xff,0xd7, + 0xa4,0xd6,0xbf,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd0,0x1a,0xcf,0x86,0x55,0x04, + 0x01,0x00,0x54,0x04,0x01,0x00,0x93,0x0c,0x92,0x08,0x11,0x04,0x01,0x00,0x0c,0x00, + 0x0c,0x00,0x0c,0x00,0xcf,0x86,0x95,0x24,0xd4,0x10,0x93,0x0c,0x92,0x08,0x11,0x04, + 0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x93,0x10,0x92,0x0c,0x51,0x04,0x00,0x00, + 0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0xd3,0x5a,0xd2,0x06, + 0xcf,0x06,0x01,0x00,0xd1,0x14,0xd0,0x06,0xcf,0x06,0x01,0x00,0xcf,0x86,0x95,0x08, + 0x14,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0xd0,0x1a,0xcf,0x86,0x95,0x14,0x54,0x04, + 0x01,0x00,0x93,0x0c,0x92,0x08,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00, + 0x01,0x00,0xcf,0x86,0xd5,0x0c,0x94,0x08,0x13,0x04,0x01,0x00,0x00,0x00,0x05,0x00, + 0x54,0x04,0x05,0x00,0x53,0x04,0x01,0x00,0x52,0x04,0x01,0x00,0x91,0x08,0x10,0x04, + 0x06,0x00,0x07,0x00,0x00,0x00,0xd2,0xce,0xd1,0xa5,0xd0,0x37,0xcf,0x86,0xd5,0x15, + 0x54,0x05,0x06,0xff,0x00,0x53,0x04,0x08,0x00,0x92,0x08,0x11,0x04,0x08,0x00,0x00, + 0x00,0x00,0x00,0x94,0x1c,0xd3,0x10,0x52,0x04,0x01,0xe6,0x51,0x04,0x0a,0xe6,0x10, + 0x04,0x0a,0xe6,0x10,0xdc,0x52,0x04,0x10,0xdc,0x11,0x04,0x10,0xdc,0x11,0xe6,0x01, + 0x00,0xcf,0x86,0xd5,0x38,0xd4,0x24,0xd3,0x14,0x52,0x04,0x01,0x00,0xd1,0x08,0x10, + 0x04,0x01,0x00,0x06,0x00,0x10,0x04,0x06,0x00,0x07,0x00,0x92,0x0c,0x91,0x08,0x10, + 0x04,0x07,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x93,0x10,0x92,0x0c,0x51,0x04,0x01, + 0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0xd4,0x18,0xd3,0x10,0x52, + 0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x12,0x04,0x01, + 0x00,0x00,0x00,0x93,0x18,0xd2,0x0c,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x06, + 0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0xd0,0x06,0xcf, + 0x06,0x01,0x00,0xcf,0x86,0x55,0x04,0x01,0x00,0x54,0x04,0x01,0x00,0x53,0x04,0x01, + 0x00,0x52,0x04,0x01,0x00,0xd1,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x10,0x04,0x00, + 0x00,0x01,0xff,0x00,0xd1,0x50,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x94,0x14,0x93,0x10, + 0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01,0x00, + 0x01,0x00,0x01,0x00,0xcf,0x86,0xd5,0x18,0x54,0x04,0x01,0x00,0x53,0x04,0x01,0x00, + 0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x06,0x00,0x94,0x14, + 0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x06,0x00,0x01,0x00,0x01,0x00,0x01,0x00, + 0x01,0x00,0x01,0x00,0xd0,0x2f,0xcf,0x86,0x55,0x04,0x01,0x00,0xd4,0x15,0x93,0x11, + 0x92,0x0d,0x91,0x09,0x10,0x05,0x01,0xff,0x00,0x01,0x00,0x01,0x00,0x01,0x00,0x01, + 0x00,0x53,0x04,0x01,0x00,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01, + 0x00,0x00,0x00,0xcf,0x86,0xd5,0x38,0xd4,0x18,0xd3,0x0c,0x92,0x08,0x11,0x04,0x00, + 0x00,0x01,0x00,0x01,0x00,0x92,0x08,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0xd3, + 0x0c,0x92,0x08,0x11,0x04,0x00,0x00,0x01,0x00,0x01,0x00,0xd2,0x08,0x11,0x04,0x00, + 0x00,0x01,0x00,0x91,0x08,0x10,0x04,0x01,0x00,0x00,0x00,0x00,0x00,0xd4,0x20,0xd3, + 0x10,0x52,0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x52, + 0x04,0x01,0x00,0x51,0x04,0x01,0x00,0x10,0x04,0x01,0x00,0x00,0x00,0x53,0x05,0x00, + 0xff,0x00,0xd2,0x0d,0x91,0x09,0x10,0x05,0x00,0xff,0x00,0x04,0x00,0x04,0x00,0x91, + 0x08,0x10,0x04,0x03,0x00,0x01,0x00,0x01,0x00,0x83,0xe2,0x46,0x3e,0xe1,0x1f,0x3b, + 0xe0,0x9c,0x39,0xcf,0x86,0xe5,0x40,0x26,0xc4,0xe3,0x16,0x14,0xe2,0xef,0x11,0xe1, + 0xd0,0x10,0xe0,0x60,0x07,0xcf,0x86,0xe5,0x53,0x03,0xe4,0x4c,0x02,0xe3,0x3d,0x01, + 0xd2,0x94,0xd1,0x70,0xd0,0x4a,0xcf,0x86,0xd5,0x18,0x94,0x14,0x53,0x04,0x07,0x00, + 0x52,0x04,0x07,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x07,0x00,0x07,0x00,0x07,0x00, + 0xd4,0x14,0x93,0x10,0x52,0x04,0x07,0x00,0x51,0x04,0x07,0x00,0x10,0x04,0x07,0x00, + 0x00,0x00,0x07,0x00,0x53,0x04,0x07,0x00,0xd2,0x0c,0x51,0x04,0x07,0x00,0x10,0x04, + 0x07,0x00,0x00,0x00,0x51,0x04,0x07,0x00,0x10,0x04,0x00,0x00,0x07,0x00,0xcf,0x86, + 0x95,0x20,0xd4,0x10,0x53,0x04,0x07,0x00,0x52,0x04,0x07,0x00,0x11,0x04,0x07,0x00, + 0x00,0x00,0x53,0x04,0x07,0x00,0x52,0x04,0x07,0x00,0x11,0x04,0x07,0x00,0x00,0x00, + 0x00,0x00,0xd0,0x06,0xcf,0x06,0x07,0x00,0xcf,0x86,0x55,0x04,0x07,0x00,0x54,0x04, + 0x07,0x00,0x53,0x04,0x07,0x00,0x92,0x0c,0x51,0x04,0x07,0x00,0x10,0x04,0x07,0x00, + 0x00,0x00,0x00,0x00,0xd1,0x40,0xd0,0x3a,0xcf,0x86,0xd5,0x20,0x94,0x1c,0x93,0x18, + 0xd2,0x0c,0x51,0x04,0x07,0x00,0x10,0x04,0x07,0x00,0x00,0x00,0x51,0x04,0x00,0x00, + 0x10,0x04,0x00,0x00,0x07,0x00,0x07,0x00,0x07,0x00,0x54,0x04,0x07,0x00,0x93,0x10, + 0x52,0x04,0x07,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x07,0x00,0x07,0x00, + 0xcf,0x06,0x08,0x00,0xd0,0x46,0xcf,0x86,0xd5,0x2c,0xd4,0x20,0x53,0x04,0x08,0x00, + 0xd2,0x0c,0x51,0x04,0x08,0x00,0x10,0x04,0x08,0x00,0x10,0x00,0xd1,0x08,0x10,0x04, + 0x10,0x00,0x12,0x00,0x10,0x04,0x12,0x00,0x00,0x00,0x53,0x04,0x0a,0x00,0x12,0x04, + 0x0a,0x00,0x00,0x00,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x10,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0xd5,0x08,0x14,0x04, + 0x00,0x00,0x0a,0x00,0x54,0x04,0x0a,0x00,0x53,0x04,0x0a,0x00,0x52,0x04,0x0a,0x00, + 0x91,0x08,0x10,0x04,0x0a,0x00,0x0a,0xdc,0x00,0x00,0xd2,0x5e,0xd1,0x06,0xcf,0x06, + 0x00,0x00,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x54,0x04,0x0a,0x00,0x53,0x04,0x0a,0x00, + 0x52,0x04,0x0a,0x00,0x91,0x08,0x10,0x04,0x0a,0x00,0x00,0x00,0x00,0x00,0x0a,0x00, + 0xcf,0x86,0xd5,0x18,0x54,0x04,0x0a,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04, + 0x0a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd4,0x14,0x93,0x10,0x92,0x0c, + 0x91,0x08,0x10,0x04,0x10,0xdc,0x10,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x53,0x04, + 0x10,0x00,0x12,0x04,0x10,0x00,0x00,0x00,0xd1,0x70,0xd0,0x36,0xcf,0x86,0xd5,0x18, + 0x54,0x04,0x05,0x00,0x53,0x04,0x05,0x00,0x52,0x04,0x05,0x00,0x51,0x04,0x05,0x00, + 0x10,0x04,0x05,0x00,0x10,0x00,0x94,0x18,0xd3,0x08,0x12,0x04,0x05,0x00,0x00,0x00, + 0x52,0x04,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x13,0x00,0x13,0x00,0x05,0x00, + 0xcf,0x86,0xd5,0x18,0x94,0x14,0x53,0x04,0x05,0x00,0x92,0x0c,0x51,0x04,0x05,0x00, + 0x10,0x04,0x05,0x00,0x00,0x00,0x00,0x00,0x10,0x00,0x54,0x04,0x10,0x00,0xd3,0x0c, + 0x52,0x04,0x10,0x00,0x11,0x04,0x10,0x00,0x10,0xe6,0x92,0x0c,0x51,0x04,0x10,0xe6, + 0x10,0x04,0x10,0xe6,0x00,0x00,0x00,0x00,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x54,0x04, + 0x07,0x00,0x53,0x04,0x07,0x00,0x52,0x04,0x07,0x00,0x51,0x04,0x07,0x00,0x10,0x04, + 0x00,0x00,0x07,0x00,0x08,0x00,0xcf,0x86,0x95,0x1c,0xd4,0x0c,0x93,0x08,0x12,0x04, + 0x08,0x00,0x00,0x00,0x08,0x00,0x93,0x0c,0x52,0x04,0x08,0x00,0x11,0x04,0x08,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0xd3,0xba,0xd2,0x80,0xd1,0x34,0xd0,0x1a,0xcf,0x86, + 0x55,0x04,0x05,0x00,0x94,0x10,0x93,0x0c,0x52,0x04,0x05,0x00,0x11,0x04,0x05,0x00, + 0x07,0x00,0x05,0x00,0x05,0x00,0xcf,0x86,0x95,0x14,0x94,0x10,0x53,0x04,0x05,0x00, + 0x52,0x04,0x05,0x00,0x11,0x04,0x05,0x00,0x07,0x00,0x07,0x00,0x07,0x00,0xd0,0x2a, + 0xcf,0x86,0xd5,0x14,0x54,0x04,0x07,0x00,0x53,0x04,0x07,0x00,0x52,0x04,0x07,0x00, + 0x11,0x04,0x07,0x00,0x00,0x00,0x94,0x10,0x53,0x04,0x07,0x00,0x92,0x08,0x11,0x04, + 0x07,0x00,0x00,0x00,0x00,0x00,0x12,0x00,0xcf,0x86,0xd5,0x10,0x54,0x04,0x12,0x00, + 0x93,0x08,0x12,0x04,0x12,0x00,0x00,0x00,0x12,0x00,0x54,0x04,0x12,0x00,0x53,0x04, + 0x12,0x00,0x12,0x04,0x12,0x00,0x00,0x00,0xd1,0x34,0xd0,0x12,0xcf,0x86,0x55,0x04, + 0x10,0x00,0x94,0x08,0x13,0x04,0x10,0x00,0x00,0x00,0x10,0x00,0xcf,0x86,0x55,0x04, + 0x10,0x00,0x94,0x18,0xd3,0x08,0x12,0x04,0x10,0x00,0x00,0x00,0x52,0x04,0x00,0x00, + 0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0x00,0x00,0xcf,0x06,0x00,0x00, + 0xd2,0x06,0xcf,0x06,0x10,0x00,0xd1,0x40,0xd0,0x1e,0xcf,0x86,0x55,0x04,0x10,0x00, + 0x54,0x04,0x10,0x00,0x93,0x10,0x52,0x04,0x10,0x00,0x51,0x04,0x10,0x00,0x10,0x04, + 0x10,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0xd5,0x14,0x54,0x04,0x10,0x00,0x93,0x0c, + 0x52,0x04,0x10,0x00,0x11,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x94,0x08,0x13,0x04, + 0x10,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xe4,0xce,0x02,0xe3,0x45,0x01, + 0xd2,0xd0,0xd1,0x70,0xd0,0x52,0xcf,0x86,0xd5,0x20,0x94,0x1c,0xd3,0x0c,0x52,0x04, + 0x07,0x00,0x11,0x04,0x07,0x00,0x00,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x07,0x00, + 0x00,0x00,0x07,0x00,0x07,0x00,0x07,0x00,0x54,0x04,0x07,0x00,0xd3,0x10,0x52,0x04, + 0x07,0x00,0x51,0x04,0x07,0x00,0x10,0x04,0x00,0x00,0x07,0x00,0xd2,0x0c,0x91,0x08, + 0x10,0x04,0x07,0x00,0x00,0x00,0x00,0x00,0xd1,0x08,0x10,0x04,0x07,0x00,0x00,0x00, + 0x10,0x04,0x00,0x00,0x07,0x00,0xcf,0x86,0x95,0x18,0x54,0x04,0x0b,0x00,0x93,0x10, + 0x52,0x04,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04,0x00,0x00,0x0b,0x00,0x0b,0x00, + 0x10,0x00,0xd0,0x32,0xcf,0x86,0xd5,0x18,0x54,0x04,0x10,0x00,0x53,0x04,0x10,0x00, + 0x52,0x04,0x10,0x00,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x00,0x00,0x94,0x14, + 0x93,0x10,0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00, + 0x10,0x00,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x11,0x00,0xd3,0x14, + 0xd2,0x0c,0x51,0x04,0x11,0x00,0x10,0x04,0x11,0x00,0x00,0x00,0x11,0x04,0x11,0x00, + 0x00,0x00,0x92,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x11,0x00,0x11,0x00, + 0xd1,0x40,0xd0,0x3a,0xcf,0x86,0xd5,0x1c,0x54,0x04,0x09,0x00,0x53,0x04,0x09,0x00, + 0xd2,0x08,0x11,0x04,0x09,0x00,0x0b,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00, + 0x09,0x00,0x54,0x04,0x0a,0x00,0x53,0x04,0x0a,0x00,0xd2,0x08,0x11,0x04,0x0a,0x00, + 0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x0a,0x00,0xcf,0x06,0x00,0x00, + 0xd0,0x1a,0xcf,0x86,0x55,0x04,0x0d,0x00,0x54,0x04,0x0d,0x00,0x53,0x04,0x0d,0x00, + 0x52,0x04,0x00,0x00,0x11,0x04,0x11,0x00,0x0d,0x00,0xcf,0x86,0x95,0x14,0x54,0x04, + 0x11,0x00,0x93,0x0c,0x92,0x08,0x11,0x04,0x00,0x00,0x11,0x00,0x11,0x00,0x11,0x00, + 0x11,0x00,0xd2,0xec,0xd1,0xa4,0xd0,0x76,0xcf,0x86,0xd5,0x48,0xd4,0x28,0xd3,0x14, + 0x52,0x04,0x08,0x00,0xd1,0x08,0x10,0x04,0x00,0x00,0x08,0x00,0x10,0x04,0x08,0x00, + 0x00,0x00,0x52,0x04,0x00,0x00,0xd1,0x08,0x10,0x04,0x08,0x00,0x08,0xdc,0x10,0x04, + 0x08,0x00,0x08,0xe6,0xd3,0x10,0x52,0x04,0x08,0x00,0x91,0x08,0x10,0x04,0x00,0x00, + 0x08,0x00,0x08,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x08,0x00,0x08,0x00, + 0x08,0x00,0x54,0x04,0x08,0x00,0xd3,0x0c,0x52,0x04,0x08,0x00,0x11,0x04,0x14,0x00, + 0x00,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x08,0xe6,0x08,0x01,0x10,0x04,0x08,0xdc, + 0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x08,0x09,0xcf,0x86,0x95,0x28, + 0xd4,0x14,0x53,0x04,0x08,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x14,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x53,0x04,0x08,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x08,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x0b,0x00,0xd0,0x0a,0xcf,0x86,0x15,0x04,0x10,0x00, + 0x00,0x00,0xcf,0x86,0x55,0x04,0x10,0x00,0xd4,0x24,0xd3,0x14,0x52,0x04,0x10,0x00, + 0xd1,0x08,0x10,0x04,0x10,0x00,0x10,0xe6,0x10,0x04,0x10,0xdc,0x00,0x00,0x92,0x0c, + 0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0x10,0x00,0x93,0x10,0x52,0x04, + 0x10,0x00,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0xd1,0x54, + 0xd0,0x26,0xcf,0x86,0x55,0x04,0x0b,0x00,0x54,0x04,0x0b,0x00,0xd3,0x0c,0x52,0x04, + 0x0b,0x00,0x11,0x04,0x0b,0x00,0x00,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00, + 0x0b,0x00,0x0b,0x00,0x0b,0x00,0xcf,0x86,0xd5,0x14,0x54,0x04,0x0b,0x00,0x93,0x0c, + 0x52,0x04,0x0b,0x00,0x11,0x04,0x0b,0x00,0x00,0x00,0x0b,0x00,0x54,0x04,0x0b,0x00, + 0x93,0x10,0x92,0x0c,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b,0x00,0x00,0x00,0x00,0x00, + 0x0b,0x00,0xd0,0x42,0xcf,0x86,0xd5,0x28,0x54,0x04,0x10,0x00,0xd3,0x0c,0x92,0x08, + 0x11,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0xd2,0x0c,0x91,0x08,0x10,0x04,0x00,0x00, + 0x10,0x00,0x10,0x00,0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x94,0x14, + 0x53,0x04,0x00,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x10,0x00,0x10,0x00, + 0x10,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xd3,0x96,0xd2,0x68,0xd1,0x24,0xd0,0x06, + 0xcf,0x06,0x0b,0x00,0xcf,0x86,0x95,0x18,0x94,0x14,0x53,0x04,0x0b,0x00,0x92,0x0c, + 0x91,0x08,0x10,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0xd0,0x1e,0xcf,0x86,0x55,0x04,0x11,0x00,0x54,0x04,0x11,0x00,0x93,0x10,0x92,0x0c, + 0x51,0x04,0x11,0x00,0x10,0x04,0x11,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86, + 0x55,0x04,0x11,0x00,0x54,0x04,0x11,0x00,0xd3,0x10,0x92,0x0c,0x51,0x04,0x11,0x00, + 0x10,0x04,0x11,0x00,0x00,0x00,0x00,0x00,0x92,0x08,0x11,0x04,0x00,0x00,0x11,0x00, + 0x11,0x00,0xd1,0x28,0xd0,0x22,0xcf,0x86,0x55,0x04,0x14,0x00,0xd4,0x0c,0x93,0x08, + 0x12,0x04,0x14,0x00,0x14,0xe6,0x00,0x00,0x53,0x04,0x14,0x00,0x92,0x08,0x11,0x04, + 0x14,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xd2,0x2a, + 0xd1,0x24,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04, + 0x0b,0x00,0x53,0x04,0x0b,0x00,0x52,0x04,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04, + 0x0b,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xd1,0x58,0xd0,0x12,0xcf,0x86,0x55,0x04, + 0x14,0x00,0x94,0x08,0x13,0x04,0x14,0x00,0x00,0x00,0x14,0x00,0xcf,0x86,0x95,0x40, + 0xd4,0x24,0xd3,0x0c,0x52,0x04,0x14,0x00,0x11,0x04,0x14,0x00,0x14,0xdc,0xd2,0x0c, + 0x51,0x04,0x14,0xe6,0x10,0x04,0x14,0xe6,0x14,0xdc,0x91,0x08,0x10,0x04,0x14,0xe6, + 0x14,0xdc,0x14,0xdc,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x14,0xdc,0x14,0x00, + 0x14,0x00,0x14,0x00,0x92,0x08,0x11,0x04,0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x15,0x00, + 0x93,0x10,0x52,0x04,0x15,0x00,0x51,0x04,0x15,0x00,0x10,0x04,0x15,0x00,0x00,0x00, + 0x00,0x00,0xcf,0x86,0xe5,0x0f,0x06,0xe4,0xf8,0x03,0xe3,0x02,0x02,0xd2,0xfb,0xd1, + 0x4c,0xd0,0x06,0xcf,0x06,0x0c,0x00,0xcf,0x86,0xd5,0x2c,0xd4,0x1c,0xd3,0x10,0x52, + 0x04,0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x09,0x0c,0x00,0x52,0x04,0x0c, + 0x00,0x11,0x04,0x0c,0x00,0x00,0x00,0x93,0x0c,0x92,0x08,0x11,0x04,0x00,0x00,0x0c, + 0x00,0x0c,0x00,0x0c,0x00,0x54,0x04,0x0c,0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00, + 0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x09,0xd0,0x69,0xcf,0x86,0xd5, + 0x32,0x54,0x04,0x0b,0x00,0x53,0x04,0x0b,0x00,0xd2,0x15,0x51,0x04,0x0b,0x00,0x10, + 0x0d,0x0b,0xff,0xf0,0x91,0x82,0x99,0xf0,0x91,0x82,0xba,0x00,0x0b,0x00,0x91,0x11, + 0x10,0x0d,0x0b,0xff,0xf0,0x91,0x82,0x9b,0xf0,0x91,0x82,0xba,0x00,0x0b,0x00,0x0b, + 0x00,0xd4,0x1d,0x53,0x04,0x0b,0x00,0x92,0x15,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b, + 0x00,0x0b,0xff,0xf0,0x91,0x82,0xa5,0xf0,0x91,0x82,0xba,0x00,0x0b,0x00,0x53,0x04, + 0x0b,0x00,0x92,0x10,0xd1,0x08,0x10,0x04,0x0b,0x00,0x0b,0x09,0x10,0x04,0x0b,0x07, + 0x0b,0x00,0x0b,0x00,0xcf,0x86,0xd5,0x20,0x94,0x1c,0xd3,0x0c,0x92,0x08,0x11,0x04, + 0x0b,0x00,0x00,0x00,0x00,0x00,0x52,0x04,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00, + 0x14,0x00,0x00,0x00,0x0d,0x00,0xd4,0x14,0x53,0x04,0x0d,0x00,0x92,0x0c,0x91,0x08, + 0x10,0x04,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x53,0x04,0x0d,0x00,0x92,0x08, + 0x11,0x04,0x0d,0x00,0x00,0x00,0x00,0x00,0xd1,0x96,0xd0,0x5c,0xcf,0x86,0xd5,0x18, + 0x94,0x14,0x93,0x10,0x92,0x0c,0x51,0x04,0x0d,0xe6,0x10,0x04,0x0d,0xe6,0x0d,0x00, + 0x0d,0x00,0x0d,0x00,0x0d,0x00,0xd4,0x26,0x53,0x04,0x0d,0x00,0x52,0x04,0x0d,0x00, + 0x51,0x04,0x0d,0x00,0x10,0x0d,0x0d,0xff,0xf0,0x91,0x84,0xb1,0xf0,0x91,0x84,0xa7, + 0x00,0x0d,0xff,0xf0,0x91,0x84,0xb2,0xf0,0x91,0x84,0xa7,0x00,0x93,0x18,0xd2,0x0c, + 0x51,0x04,0x0d,0x00,0x10,0x04,0x0d,0x00,0x0d,0x09,0x91,0x08,0x10,0x04,0x0d,0x09, + 0x00,0x00,0x0d,0x00,0x0d,0x00,0xcf,0x86,0xd5,0x18,0x94,0x14,0x93,0x10,0x52,0x04, + 0x0d,0x00,0x51,0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x00,0x00,0x00,0x00,0x10,0x00, + 0x54,0x04,0x10,0x00,0x93,0x18,0xd2,0x0c,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00, + 0x10,0x07,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0xd0,0x06, + 0xcf,0x06,0x0d,0x00,0xcf,0x86,0xd5,0x40,0xd4,0x2c,0xd3,0x10,0x92,0x0c,0x91,0x08, + 0x10,0x04,0x0d,0x09,0x0d,0x00,0x0d,0x00,0x0d,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04, + 0x0d,0x00,0x11,0x00,0x10,0x04,0x11,0x07,0x11,0x00,0x91,0x08,0x10,0x04,0x11,0x00, + 0x10,0x00,0x00,0x00,0x53,0x04,0x0d,0x00,0x92,0x0c,0x51,0x04,0x0d,0x00,0x10,0x04, + 0x10,0x00,0x11,0x00,0x11,0x00,0xd4,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04, + 0x00,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x93,0x10,0x52,0x04,0x10,0x00, + 0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd2,0xc8,0xd1,0x48, + 0xd0,0x42,0xcf,0x86,0xd5,0x18,0x54,0x04,0x10,0x00,0x93,0x10,0x92,0x0c,0x51,0x04, + 0x10,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x54,0x04,0x10,0x00, + 0xd3,0x14,0x52,0x04,0x10,0x00,0xd1,0x08,0x10,0x04,0x10,0x00,0x10,0x09,0x10,0x04, + 0x10,0x07,0x10,0x00,0x52,0x04,0x10,0x00,0x51,0x04,0x10,0x00,0x10,0x04,0x12,0x00, + 0x00,0x00,0xcf,0x06,0x00,0x00,0xd0,0x52,0xcf,0x86,0xd5,0x3c,0xd4,0x28,0xd3,0x10, + 0x52,0x04,0x11,0x00,0x51,0x04,0x11,0x00,0x10,0x04,0x11,0x00,0x00,0x00,0xd2,0x0c, + 0x91,0x08,0x10,0x04,0x11,0x00,0x00,0x00,0x11,0x00,0x51,0x04,0x11,0x00,0x10,0x04, + 0x00,0x00,0x11,0x00,0x53,0x04,0x11,0x00,0x52,0x04,0x11,0x00,0x51,0x04,0x11,0x00, + 0x10,0x04,0x00,0x00,0x11,0x00,0x94,0x10,0x53,0x04,0x11,0x00,0x92,0x08,0x11,0x04, + 0x11,0x00,0x00,0x00,0x00,0x00,0x10,0x00,0xcf,0x86,0x55,0x04,0x10,0x00,0xd4,0x18, + 0x53,0x04,0x10,0x00,0x92,0x10,0xd1,0x08,0x10,0x04,0x10,0x00,0x10,0x07,0x10,0x04, + 0x10,0x09,0x00,0x00,0x00,0x00,0x53,0x04,0x10,0x00,0x92,0x08,0x11,0x04,0x10,0x00, + 0x00,0x00,0x00,0x00,0xe1,0x27,0x01,0xd0,0x8a,0xcf,0x86,0xd5,0x44,0xd4,0x2c,0xd3, + 0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x11,0x00,0x10,0x00,0x10,0x00,0x91,0x08,0x10, + 0x04,0x00,0x00,0x10,0x00,0x10,0x00,0x52,0x04,0x10,0x00,0xd1,0x08,0x10,0x04,0x10, + 0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0x93,0x14,0x92,0x10,0xd1,0x08,0x10, + 0x04,0x10,0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0xd4, + 0x14,0x53,0x04,0x10,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x10, + 0x00,0x10,0x00,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x10, + 0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x10,0x00,0x10,0x00,0xd2,0x0c,0x51,0x04,0x10, + 0x00,0x10,0x04,0x00,0x00,0x14,0x07,0x91,0x08,0x10,0x04,0x10,0x07,0x10,0x00,0x10, + 0x00,0xcf,0x86,0xd5,0x6a,0xd4,0x42,0xd3,0x14,0x52,0x04,0x10,0x00,0xd1,0x08,0x10, + 0x04,0x10,0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0xd2,0x19,0xd1,0x08,0x10, + 0x04,0x10,0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0xff,0xf0,0x91,0x8d,0x87,0xf0, + 0x91,0x8c,0xbe,0x00,0x91,0x11,0x10,0x0d,0x10,0xff,0xf0,0x91,0x8d,0x87,0xf0,0x91, + 0x8d,0x97,0x00,0x10,0x09,0x00,0x00,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x11, + 0x00,0x00,0x00,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0x52, + 0x04,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x10,0x00,0x10,0x00,0xd4,0x1c,0xd3, + 0x0c,0x52,0x04,0x10,0x00,0x11,0x04,0x00,0x00,0x10,0xe6,0x52,0x04,0x10,0xe6,0x91, + 0x08,0x10,0x04,0x10,0xe6,0x00,0x00,0x00,0x00,0x93,0x10,0x52,0x04,0x10,0xe6,0x91, + 0x08,0x10,0x04,0x10,0xe6,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xe3, + 0x30,0x01,0xd2,0xb7,0xd1,0x48,0xd0,0x06,0xcf,0x06,0x12,0x00,0xcf,0x86,0x95,0x3c, + 0xd4,0x1c,0x93,0x18,0xd2,0x0c,0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x09,0x12,0x00, + 0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x07,0x12,0x00,0x12,0x00,0x53,0x04,0x12,0x00, + 0xd2,0x0c,0x51,0x04,0x12,0x00,0x10,0x04,0x00,0x00,0x12,0x00,0xd1,0x08,0x10,0x04, + 0x00,0x00,0x12,0x00,0x10,0x04,0x14,0xe6,0x15,0x00,0x00,0x00,0xd0,0x45,0xcf,0x86, + 0x55,0x04,0x10,0x00,0x54,0x04,0x10,0x00,0x53,0x04,0x10,0x00,0xd2,0x15,0x51,0x04, + 0x10,0x00,0x10,0x04,0x10,0x00,0x10,0xff,0xf0,0x91,0x92,0xb9,0xf0,0x91,0x92,0xba, + 0x00,0xd1,0x11,0x10,0x0d,0x10,0xff,0xf0,0x91,0x92,0xb9,0xf0,0x91,0x92,0xb0,0x00, + 0x10,0x00,0x10,0x0d,0x10,0xff,0xf0,0x91,0x92,0xb9,0xf0,0x91,0x92,0xbd,0x00,0x10, + 0x00,0xcf,0x86,0x95,0x24,0xd4,0x14,0x93,0x10,0x92,0x0c,0x51,0x04,0x10,0x00,0x10, + 0x04,0x10,0x09,0x10,0x07,0x10,0x00,0x00,0x00,0x53,0x04,0x10,0x00,0x92,0x08,0x11, + 0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd1,0x06,0xcf,0x06,0x00,0x00,0xd0, + 0x40,0xcf,0x86,0x55,0x04,0x10,0x00,0x54,0x04,0x10,0x00,0xd3,0x0c,0x52,0x04,0x10, + 0x00,0x11,0x04,0x10,0x00,0x00,0x00,0xd2,0x1e,0x51,0x04,0x10,0x00,0x10,0x0d,0x10, + 0xff,0xf0,0x91,0x96,0xb8,0xf0,0x91,0x96,0xaf,0x00,0x10,0xff,0xf0,0x91,0x96,0xb9, + 0xf0,0x91,0x96,0xaf,0x00,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x10,0x09,0xcf, + 0x86,0x95,0x2c,0xd4,0x1c,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x10,0x07,0x10, + 0x00,0x10,0x00,0x10,0x00,0x92,0x08,0x11,0x04,0x10,0x00,0x11,0x00,0x11,0x00,0x53, + 0x04,0x11,0x00,0x52,0x04,0x11,0x00,0x11,0x04,0x11,0x00,0x00,0x00,0x00,0x00,0xd2, + 0xa0,0xd1,0x5c,0xd0,0x1e,0xcf,0x86,0x55,0x04,0x10,0x00,0x54,0x04,0x10,0x00,0x53, + 0x04,0x10,0x00,0x52,0x04,0x10,0x00,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x10, + 0x09,0xcf,0x86,0xd5,0x24,0xd4,0x14,0x93,0x10,0x52,0x04,0x10,0x00,0x91,0x08,0x10, + 0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x53,0x04,0x10,0x00,0x92,0x08,0x11, + 0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x94,0x14,0x53,0x04,0x12,0x00,0x52,0x04,0x12, + 0x00,0x91,0x08,0x10,0x04,0x12,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd0,0x2a,0xcf, + 0x86,0x55,0x04,0x0d,0x00,0x54,0x04,0x0d,0x00,0xd3,0x10,0x52,0x04,0x0d,0x00,0x51, + 0x04,0x0d,0x00,0x10,0x04,0x0d,0x09,0x0d,0x07,0x92,0x0c,0x91,0x08,0x10,0x04,0x15, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0x95,0x14,0x94,0x10,0x53,0x04,0x0d, + 0x00,0x92,0x08,0x11,0x04,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd1, + 0x40,0xd0,0x3a,0xcf,0x86,0xd5,0x20,0x54,0x04,0x11,0x00,0x53,0x04,0x11,0x00,0xd2, + 0x0c,0x51,0x04,0x11,0x00,0x10,0x04,0x14,0x00,0x00,0x00,0x91,0x08,0x10,0x04,0x00, + 0x00,0x11,0x00,0x11,0x00,0x94,0x14,0x53,0x04,0x11,0x00,0x92,0x0c,0x51,0x04,0x11, + 0x00,0x10,0x04,0x11,0x00,0x11,0x09,0x00,0x00,0x11,0x00,0xcf,0x06,0x00,0x00,0xcf, + 0x06,0x00,0x00,0xe4,0x59,0x01,0xd3,0xb2,0xd2,0x5c,0xd1,0x28,0xd0,0x22,0xcf,0x86, + 0x55,0x04,0x14,0x00,0x54,0x04,0x14,0x00,0x53,0x04,0x14,0x00,0x92,0x10,0xd1,0x08, + 0x10,0x04,0x14,0x00,0x14,0x09,0x10,0x04,0x14,0x07,0x14,0x00,0x00,0x00,0xcf,0x06, + 0x00,0x00,0xd0,0x0a,0xcf,0x86,0x15,0x04,0x00,0x00,0x10,0x00,0xcf,0x86,0x55,0x04, + 0x10,0x00,0x54,0x04,0x10,0x00,0xd3,0x10,0x92,0x0c,0x51,0x04,0x10,0x00,0x10,0x04, + 0x10,0x00,0x00,0x00,0x00,0x00,0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04, + 0x00,0x00,0x10,0x00,0xd1,0x06,0xcf,0x06,0x00,0x00,0xd0,0x1a,0xcf,0x86,0x55,0x04, + 0x00,0x00,0x94,0x10,0x53,0x04,0x15,0x00,0x92,0x08,0x11,0x04,0x00,0x00,0x15,0x00, + 0x15,0x00,0x15,0x00,0xcf,0x86,0xd5,0x14,0x54,0x04,0x15,0x00,0x53,0x04,0x15,0x00, + 0x92,0x08,0x11,0x04,0x00,0x00,0x15,0x00,0x15,0x00,0x94,0x1c,0x93,0x18,0xd2,0x0c, + 0x91,0x08,0x10,0x04,0x15,0x09,0x15,0x00,0x15,0x00,0x91,0x08,0x10,0x04,0x15,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd2,0xa0,0xd1,0x3c,0xd0,0x1e,0xcf,0x86, + 0x55,0x04,0x13,0x00,0x54,0x04,0x13,0x00,0x93,0x10,0x52,0x04,0x13,0x00,0x91,0x08, + 0x10,0x04,0x13,0x09,0x13,0x00,0x13,0x00,0x13,0x00,0xcf,0x86,0x95,0x18,0x94,0x14, + 0x93,0x10,0x52,0x04,0x13,0x00,0x51,0x04,0x13,0x00,0x10,0x04,0x13,0x00,0x13,0x09, + 0x00,0x00,0x13,0x00,0x13,0x00,0xd0,0x46,0xcf,0x86,0xd5,0x2c,0xd4,0x10,0x93,0x0c, + 0x52,0x04,0x13,0x00,0x11,0x04,0x15,0x00,0x13,0x00,0x13,0x00,0x53,0x04,0x13,0x00, + 0xd2,0x0c,0x91,0x08,0x10,0x04,0x13,0x00,0x13,0x09,0x13,0x00,0x91,0x08,0x10,0x04, + 0x13,0x00,0x14,0x00,0x13,0x00,0x94,0x14,0x93,0x10,0x92,0x0c,0x51,0x04,0x13,0x00, + 0x10,0x04,0x13,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0x55,0x04, + 0x10,0x00,0x54,0x04,0x10,0x00,0x53,0x04,0x10,0x00,0x92,0x0c,0x91,0x08,0x10,0x04, + 0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xe3,0xa9,0x01,0xd2, + 0xb0,0xd1,0x6c,0xd0,0x3e,0xcf,0x86,0xd5,0x18,0x94,0x14,0x53,0x04,0x12,0x00,0x92, + 0x0c,0x91,0x08,0x10,0x04,0x12,0x00,0x00,0x00,0x12,0x00,0x12,0x00,0x12,0x00,0x54, + 0x04,0x12,0x00,0xd3,0x10,0x52,0x04,0x12,0x00,0x51,0x04,0x12,0x00,0x10,0x04,0x12, + 0x00,0x00,0x00,0x52,0x04,0x12,0x00,0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x00,0x12, + 0x09,0xcf,0x86,0xd5,0x14,0x94,0x10,0x93,0x0c,0x52,0x04,0x12,0x00,0x11,0x04,0x12, + 0x00,0x00,0x00,0x00,0x00,0x12,0x00,0x94,0x14,0x53,0x04,0x12,0x00,0x52,0x04,0x12, + 0x00,0x91,0x08,0x10,0x04,0x12,0x00,0x00,0x00,0x00,0x00,0x12,0x00,0xd0,0x3e,0xcf, + 0x86,0xd5,0x14,0x54,0x04,0x12,0x00,0x93,0x0c,0x92,0x08,0x11,0x04,0x00,0x00,0x12, + 0x00,0x12,0x00,0x12,0x00,0xd4,0x14,0x53,0x04,0x12,0x00,0x92,0x0c,0x91,0x08,0x10, + 0x04,0x00,0x00,0x12,0x00,0x12,0x00,0x12,0x00,0x93,0x10,0x52,0x04,0x12,0x00,0x51, + 0x04,0x12,0x00,0x10,0x04,0x12,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xd1, + 0xa0,0xd0,0x52,0xcf,0x86,0xd5,0x24,0x94,0x20,0xd3,0x10,0x52,0x04,0x13,0x00,0x51, + 0x04,0x13,0x00,0x10,0x04,0x13,0x00,0x00,0x00,0x92,0x0c,0x51,0x04,0x13,0x00,0x10, + 0x04,0x00,0x00,0x13,0x00,0x13,0x00,0x13,0x00,0x54,0x04,0x13,0x00,0xd3,0x10,0x52, + 0x04,0x13,0x00,0x51,0x04,0x13,0x00,0x10,0x04,0x13,0x00,0x00,0x00,0xd2,0x0c,0x51, + 0x04,0x00,0x00,0x10,0x04,0x13,0x00,0x00,0x00,0x51,0x04,0x13,0x00,0x10,0x04,0x00, + 0x00,0x13,0x00,0xcf,0x86,0xd5,0x28,0xd4,0x18,0x93,0x14,0xd2,0x0c,0x51,0x04,0x13, + 0x00,0x10,0x04,0x13,0x07,0x13,0x00,0x11,0x04,0x13,0x09,0x13,0x00,0x00,0x00,0x53, + 0x04,0x13,0x00,0x92,0x08,0x11,0x04,0x13,0x00,0x00,0x00,0x00,0x00,0x94,0x20,0xd3, + 0x10,0x52,0x04,0x14,0x00,0x51,0x04,0x14,0x00,0x10,0x04,0x00,0x00,0x14,0x00,0x92, + 0x0c,0x91,0x08,0x10,0x04,0x14,0x00,0x00,0x00,0x14,0x00,0x14,0x00,0x14,0x00,0xd0, + 0x52,0xcf,0x86,0xd5,0x3c,0xd4,0x14,0x53,0x04,0x14,0x00,0x52,0x04,0x14,0x00,0x51, + 0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x00,0x00,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x14, + 0x00,0x10,0x04,0x00,0x00,0x14,0x00,0x51,0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x14, + 0x09,0x92,0x0c,0x91,0x08,0x10,0x04,0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x94, + 0x10,0x53,0x04,0x14,0x00,0x92,0x08,0x11,0x04,0x14,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0xcf,0x06,0x00,0x00,0xd2,0x2a,0xd1,0x06,0xcf,0x06,0x00,0x00,0xd0,0x06,0xcf, + 0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x14,0x00,0x53,0x04,0x14, + 0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd1, + 0x06,0xcf,0x06,0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x15, + 0x00,0x54,0x04,0x15,0x00,0xd3,0x0c,0x92,0x08,0x11,0x04,0x15,0x00,0x00,0x00,0x00, + 0x00,0x52,0x04,0x00,0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x15,0x00,0xd0, + 0xca,0xcf,0x86,0xd5,0xc2,0xd4,0x54,0xd3,0x06,0xcf,0x06,0x09,0x00,0xd2,0x06,0xcf, + 0x06,0x09,0x00,0xd1,0x24,0xd0,0x06,0xcf,0x06,0x09,0x00,0xcf,0x86,0x55,0x04,0x09, + 0x00,0x94,0x14,0x53,0x04,0x09,0x00,0x52,0x04,0x09,0x00,0x51,0x04,0x09,0x00,0x10, + 0x04,0x09,0x00,0x10,0x00,0x10,0x00,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x54,0x04,0x10, + 0x00,0x53,0x04,0x10,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x10,0x00,0x11,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xd3,0x68,0xd2,0x46,0xd1,0x40,0xd0, + 0x06,0xcf,0x06,0x09,0x00,0xcf,0x86,0x55,0x04,0x09,0x00,0xd4,0x20,0xd3,0x10,0x92, + 0x0c,0x51,0x04,0x09,0x00,0x10,0x04,0x09,0x00,0x10,0x00,0x10,0x00,0x52,0x04,0x10, + 0x00,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x00,0x00,0x93,0x10,0x52,0x04,0x09, + 0x00,0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x11, + 0x00,0xd1,0x1c,0xd0,0x06,0xcf,0x06,0x11,0x00,0xcf,0x86,0x95,0x10,0x94,0x0c,0x93, + 0x08,0x12,0x04,0x11,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00, + 0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x86,0xd5,0x4c,0xd4,0x06,0xcf, + 0x06,0x0b,0x00,0xd3,0x40,0xd2,0x3a,0xd1,0x34,0xd0,0x2e,0xcf,0x86,0x55,0x04,0x0b, + 0x00,0xd4,0x14,0x53,0x04,0x0b,0x00,0x52,0x04,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10, + 0x04,0x0b,0x00,0x00,0x00,0x53,0x04,0x15,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x15, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf, + 0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xd1,0x4c,0xd0,0x44,0xcf, + 0x86,0xd5,0x3c,0xd4,0x06,0xcf,0x06,0x00,0x00,0xd3,0x06,0xcf,0x06,0x11,0x00,0xd2, + 0x2a,0xd1,0x24,0xd0,0x06,0xcf,0x06,0x11,0x00,0xcf,0x86,0x95,0x18,0x94,0x14,0x93, + 0x10,0x52,0x04,0x11,0x00,0x51,0x04,0x11,0x00,0x10,0x04,0x11,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00, + 0x00,0xcf,0x86,0xcf,0x06,0x00,0x00,0xe0,0xd2,0x01,0xcf,0x86,0xd5,0x06,0xcf,0x06, + 0x00,0x00,0xe4,0x0b,0x01,0xd3,0x06,0xcf,0x06,0x0c,0x00,0xd2,0x84,0xd1,0x50,0xd0, + 0x1e,0xcf,0x86,0x55,0x04,0x0c,0x00,0x54,0x04,0x0c,0x00,0x53,0x04,0x0c,0x00,0x92, + 0x0c,0x91,0x08,0x10,0x04,0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0xd5, + 0x18,0x54,0x04,0x10,0x00,0x53,0x04,0x10,0x00,0x52,0x04,0x10,0x00,0x51,0x04,0x10, + 0x00,0x10,0x04,0x10,0x00,0x00,0x00,0x94,0x14,0x53,0x04,0x10,0x00,0xd2,0x08,0x11, + 0x04,0x10,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x10,0x00,0x00,0x00,0xd0,0x06,0xcf, + 0x06,0x00,0x00,0xcf,0x86,0xd5,0x08,0x14,0x04,0x00,0x00,0x10,0x00,0xd4,0x10,0x53, + 0x04,0x10,0x00,0x52,0x04,0x10,0x00,0x11,0x04,0x10,0x00,0x00,0x00,0x93,0x10,0x52, + 0x04,0x10,0x01,0x91,0x08,0x10,0x04,0x10,0x01,0x10,0x00,0x00,0x00,0x00,0x00,0xd1, + 0x6c,0xd0,0x1e,0xcf,0x86,0x55,0x04,0x10,0x00,0x54,0x04,0x10,0x00,0x93,0x10,0x52, + 0x04,0x10,0xe6,0x51,0x04,0x10,0xe6,0x10,0x04,0x10,0xe6,0x10,0x00,0x10,0x00,0xcf, + 0x86,0xd5,0x24,0xd4,0x10,0x93,0x0c,0x52,0x04,0x10,0x00,0x11,0x04,0x10,0x00,0x00, + 0x00,0x00,0x00,0x53,0x04,0x10,0x00,0x92,0x0c,0x51,0x04,0x10,0x00,0x10,0x04,0x00, + 0x00,0x10,0x00,0x10,0x00,0xd4,0x14,0x93,0x10,0x92,0x0c,0x51,0x04,0x10,0x00,0x10, + 0x04,0x00,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x53,0x04,0x10,0x00,0x52,0x04,0x00, + 0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x10,0x00,0x10,0x00,0xd0,0x0e,0xcf,0x86,0x95, + 0x08,0x14,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xd3,0x06,0xcf, + 0x06,0x00,0x00,0xd2,0x30,0xd1,0x0c,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x06,0x14, + 0x00,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x54,0x04,0x14,0x00,0x53,0x04,0x14,0x00,0x92, + 0x0c,0x51,0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf, + 0x06,0x00,0x00,0xd1,0x4c,0xd0,0x06,0xcf,0x06,0x0d,0x00,0xcf,0x86,0xd5,0x2c,0x94, + 0x28,0xd3,0x10,0x52,0x04,0x0d,0x00,0x91,0x08,0x10,0x04,0x0d,0x00,0x15,0x00,0x15, + 0x00,0xd2,0x0c,0x51,0x04,0x15,0x00,0x10,0x04,0x15,0x00,0x00,0x00,0x51,0x04,0x00, + 0x00,0x10,0x04,0x00,0x00,0x15,0x00,0x0d,0x00,0x54,0x04,0x0d,0x00,0x53,0x04,0x0d, + 0x00,0x52,0x04,0x0d,0x00,0x51,0x04,0x0d,0x00,0x10,0x04,0x0d,0x00,0x15,0x00,0xd0, + 0x1e,0xcf,0x86,0x95,0x18,0x94,0x14,0x53,0x04,0x15,0x00,0x52,0x04,0x00,0x00,0x51, + 0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x0d,0x00,0x0d,0x00,0x00,0x00,0xcf,0x86,0x55, + 0x04,0x00,0x00,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x12,0x00,0x13, + 0x00,0x15,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0xcf,0x06,0x12,0x00,0xe2, + 0xc6,0x01,0xd1,0x8e,0xd0,0x86,0xcf,0x86,0xd5,0x48,0xd4,0x06,0xcf,0x06,0x12,0x00, + 0xd3,0x06,0xcf,0x06,0x12,0x00,0xd2,0x06,0xcf,0x06,0x12,0x00,0xd1,0x06,0xcf,0x06, + 0x12,0x00,0xd0,0x06,0xcf,0x06,0x12,0x00,0xcf,0x86,0x55,0x04,0x12,0x00,0xd4,0x14, + 0x53,0x04,0x12,0x00,0x52,0x04,0x12,0x00,0x91,0x08,0x10,0x04,0x12,0x00,0x14,0x00, + 0x14,0x00,0x93,0x0c,0x92,0x08,0x11,0x04,0x14,0x00,0x15,0x00,0x15,0x00,0x00,0x00, + 0xd4,0x36,0xd3,0x06,0xcf,0x06,0x12,0x00,0xd2,0x2a,0xd1,0x06,0xcf,0x06,0x12,0x00, + 0xd0,0x06,0xcf,0x06,0x12,0x00,0xcf,0x86,0x55,0x04,0x12,0x00,0x54,0x04,0x12,0x00, + 0x93,0x10,0x92,0x0c,0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x86,0xcf,0x06,0x00,0x00, + 0xd0,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xcf,0x86,0xd5,0xa2,0xd4,0x9c,0xd3,0x74, + 0xd2,0x26,0xd1,0x20,0xd0,0x1a,0xcf,0x86,0x95,0x14,0x94,0x10,0x93,0x0c,0x92,0x08, + 0x11,0x04,0x0c,0x00,0x13,0x00,0x13,0x00,0x13,0x00,0x13,0x00,0x13,0x00,0xcf,0x06, + 0x13,0x00,0xcf,0x06,0x13,0x00,0xd1,0x48,0xd0,0x1e,0xcf,0x86,0x95,0x18,0x54,0x04, + 0x13,0x00,0x53,0x04,0x13,0x00,0x52,0x04,0x13,0x00,0x51,0x04,0x13,0x00,0x10,0x04, + 0x13,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0xd5,0x18,0x54,0x04,0x00,0x00,0x93,0x10, + 0x92,0x0c,0x51,0x04,0x15,0x00,0x10,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x94,0x0c,0x93,0x08,0x12,0x04,0x00,0x00,0x15,0x00,0x00,0x00,0x13,0x00,0xcf,0x06, + 0x13,0x00,0xd2,0x22,0xd1,0x06,0xcf,0x06,0x13,0x00,0xd0,0x06,0xcf,0x06,0x13,0x00, + 0xcf,0x86,0x55,0x04,0x13,0x00,0x54,0x04,0x13,0x00,0x53,0x04,0x13,0x00,0x12,0x04, + 0x13,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06, + 0x00,0x00,0xd3,0x7f,0xd2,0x79,0xd1,0x34,0xd0,0x06,0xcf,0x06,0x10,0x00,0xcf,0x86, + 0x55,0x04,0x10,0x00,0xd4,0x14,0x53,0x04,0x10,0x00,0x92,0x0c,0x51,0x04,0x10,0x00, + 0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x53,0x04,0x10,0x00,0x52,0x04,0x10,0x00, + 0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0xd0,0x3f,0xcf,0x86,0xd5,0x2c, + 0xd4,0x14,0x53,0x04,0x10,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x53,0x04,0x10,0x00,0xd2,0x08,0x11,0x04,0x10,0x00,0x00,0x00, + 0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x01,0x10,0x00,0x94,0x0d,0x93,0x09,0x12,0x05, + 0x10,0xff,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00, + 0x00,0xcf,0x06,0x00,0x00,0xe1,0x96,0x04,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00, + 0xcf,0x86,0xe5,0x33,0x04,0xe4,0x83,0x02,0xe3,0xf8,0x01,0xd2,0x26,0xd1,0x06,0xcf, + 0x06,0x05,0x00,0xd0,0x06,0xcf,0x06,0x05,0x00,0xcf,0x86,0x55,0x04,0x05,0x00,0x54, + 0x04,0x05,0x00,0x93,0x0c,0x52,0x04,0x05,0x00,0x11,0x04,0x05,0x00,0x00,0x00,0x00, + 0x00,0xd1,0xef,0xd0,0x2a,0xcf,0x86,0x55,0x04,0x05,0x00,0x94,0x20,0xd3,0x10,0x52, + 0x04,0x05,0x00,0x51,0x04,0x05,0x00,0x10,0x04,0x05,0x00,0x00,0x00,0x92,0x0c,0x91, + 0x08,0x10,0x04,0x00,0x00,0x0a,0x00,0x05,0x00,0x05,0x00,0x05,0x00,0xcf,0x86,0xd5, + 0x2a,0x54,0x04,0x05,0x00,0x53,0x04,0x05,0x00,0x52,0x04,0x05,0x00,0x51,0x04,0x05, + 0x00,0x10,0x0d,0x05,0xff,0xf0,0x9d,0x85,0x97,0xf0,0x9d,0x85,0xa5,0x00,0x05,0xff, + 0xf0,0x9d,0x85,0x98,0xf0,0x9d,0x85,0xa5,0x00,0xd4,0x75,0xd3,0x61,0xd2,0x44,0xd1, + 0x22,0x10,0x11,0x05,0xff,0xf0,0x9d,0x85,0x98,0xf0,0x9d,0x85,0xa5,0xf0,0x9d,0x85, + 0xae,0x00,0x05,0xff,0xf0,0x9d,0x85,0x98,0xf0,0x9d,0x85,0xa5,0xf0,0x9d,0x85,0xaf, + 0x00,0x10,0x11,0x05,0xff,0xf0,0x9d,0x85,0x98,0xf0,0x9d,0x85,0xa5,0xf0,0x9d,0x85, + 0xb0,0x00,0x05,0xff,0xf0,0x9d,0x85,0x98,0xf0,0x9d,0x85,0xa5,0xf0,0x9d,0x85,0xb1, + 0x00,0xd1,0x15,0x10,0x11,0x05,0xff,0xf0,0x9d,0x85,0x98,0xf0,0x9d,0x85,0xa5,0xf0, + 0x9d,0x85,0xb2,0x00,0x05,0xd8,0x10,0x04,0x05,0xd8,0x05,0x01,0xd2,0x08,0x11,0x04, + 0x05,0x01,0x05,0x00,0x91,0x08,0x10,0x04,0x05,0x00,0x05,0xe2,0x05,0xd8,0xd3,0x12, + 0x92,0x0d,0x51,0x04,0x05,0xd8,0x10,0x04,0x05,0xd8,0x05,0xff,0x00,0x05,0xff,0x00, + 0x92,0x0e,0x51,0x05,0x05,0xff,0x00,0x10,0x05,0x05,0xff,0x00,0x05,0xdc,0x05,0xdc, + 0xd0,0x97,0xcf,0x86,0xd5,0x28,0x94,0x24,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x05,0xdc, + 0x10,0x04,0x05,0xdc,0x05,0x00,0x91,0x08,0x10,0x04,0x05,0x00,0x05,0xe6,0x05,0xe6, + 0x92,0x08,0x11,0x04,0x05,0xe6,0x05,0xdc,0x05,0x00,0x05,0x00,0xd4,0x14,0x53,0x04, + 0x05,0x00,0xd2,0x08,0x11,0x04,0x05,0x00,0x05,0xe6,0x11,0x04,0x05,0xe6,0x05,0x00, + 0x53,0x04,0x05,0x00,0xd2,0x15,0x51,0x04,0x05,0x00,0x10,0x04,0x05,0x00,0x05,0xff, + 0xf0,0x9d,0x86,0xb9,0xf0,0x9d,0x85,0xa5,0x00,0xd1,0x1e,0x10,0x0d,0x05,0xff,0xf0, + 0x9d,0x86,0xba,0xf0,0x9d,0x85,0xa5,0x00,0x05,0xff,0xf0,0x9d,0x86,0xb9,0xf0,0x9d, + 0x85,0xa5,0xf0,0x9d,0x85,0xae,0x00,0x10,0x11,0x05,0xff,0xf0,0x9d,0x86,0xba,0xf0, + 0x9d,0x85,0xa5,0xf0,0x9d,0x85,0xae,0x00,0x05,0xff,0xf0,0x9d,0x86,0xb9,0xf0,0x9d, + 0x85,0xa5,0xf0,0x9d,0x85,0xaf,0x00,0xcf,0x86,0xd5,0x31,0xd4,0x21,0x93,0x1d,0x92, + 0x19,0x91,0x15,0x10,0x11,0x05,0xff,0xf0,0x9d,0x86,0xba,0xf0,0x9d,0x85,0xa5,0xf0, + 0x9d,0x85,0xaf,0x00,0x05,0x00,0x05,0x00,0x05,0x00,0x05,0x00,0x53,0x04,0x05,0x00, + 0x52,0x04,0x05,0x00,0x11,0x04,0x05,0x00,0x11,0x00,0x94,0x14,0x53,0x04,0x11,0x00, + 0x92,0x0c,0x91,0x08,0x10,0x04,0x11,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0xd2,0x44,0xd1,0x28,0xd0,0x06,0xcf,0x06,0x08,0x00,0xcf,0x86,0x95,0x1c,0x94,0x18, + 0x93,0x14,0xd2,0x08,0x11,0x04,0x08,0x00,0x08,0xe6,0x91,0x08,0x10,0x04,0x08,0xe6, + 0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00, + 0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x14,0x00,0x93,0x08,0x12,0x04,0x14,0x00, + 0x00,0x00,0x00,0x00,0xd1,0x40,0xd0,0x06,0xcf,0x06,0x07,0x00,0xcf,0x86,0xd5,0x18, + 0x54,0x04,0x07,0x00,0x93,0x10,0x52,0x04,0x07,0x00,0x51,0x04,0x07,0x00,0x10,0x04, + 0x07,0x00,0x00,0x00,0x00,0x00,0x54,0x04,0x09,0x00,0xd3,0x0c,0x92,0x08,0x11,0x04, + 0x09,0x00,0x14,0x00,0x14,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x14,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xe3,0x5f,0x01,0xd2,0xb4,0xd1,0x24,0xd0, + 0x06,0xcf,0x06,0x05,0x00,0xcf,0x86,0x95,0x18,0x54,0x04,0x05,0x00,0x93,0x10,0x52, + 0x04,0x05,0x00,0x91,0x08,0x10,0x04,0x05,0x00,0x00,0x00,0x05,0x00,0x05,0x00,0x05, + 0x00,0xd0,0x6a,0xcf,0x86,0xd5,0x18,0x54,0x04,0x05,0x00,0x53,0x04,0x05,0x00,0x52, + 0x04,0x05,0x00,0x91,0x08,0x10,0x04,0x05,0x00,0x00,0x00,0x05,0x00,0xd4,0x34,0xd3, + 0x1c,0xd2,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x05,0x00,0x00,0x00,0xd1,0x08,0x10, + 0x04,0x00,0x00,0x05,0x00,0x10,0x04,0x05,0x00,0x00,0x00,0xd2,0x0c,0x91,0x08,0x10, + 0x04,0x00,0x00,0x05,0x00,0x05,0x00,0x91,0x08,0x10,0x04,0x05,0x00,0x00,0x00,0x05, + 0x00,0x53,0x04,0x05,0x00,0xd2,0x0c,0x51,0x04,0x05,0x00,0x10,0x04,0x00,0x00,0x05, + 0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x05,0x00,0x05,0x00,0xcf,0x86,0x95,0x20,0x94, + 0x1c,0x93,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x05,0x00,0x07,0x00,0x05,0x00,0x91, + 0x08,0x10,0x04,0x00,0x00,0x05,0x00,0x05,0x00,0x05,0x00,0x05,0x00,0x05,0x00,0xd1, + 0xa4,0xd0,0x6a,0xcf,0x86,0xd5,0x48,0xd4,0x28,0xd3,0x10,0x52,0x04,0x05,0x00,0x51, + 0x04,0x05,0x00,0x10,0x04,0x00,0x00,0x05,0x00,0xd2,0x0c,0x51,0x04,0x05,0x00,0x10, + 0x04,0x05,0x00,0x00,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x05,0x00,0x05,0x00,0xd3, + 0x10,0x52,0x04,0x05,0x00,0x91,0x08,0x10,0x04,0x05,0x00,0x00,0x00,0x05,0x00,0x52, + 0x04,0x05,0x00,0x91,0x08,0x10,0x04,0x05,0x00,0x00,0x00,0x05,0x00,0x54,0x04,0x05, + 0x00,0x53,0x04,0x05,0x00,0xd2,0x0c,0x51,0x04,0x05,0x00,0x10,0x04,0x00,0x00,0x05, + 0x00,0x51,0x04,0x05,0x00,0x10,0x04,0x05,0x00,0x00,0x00,0xcf,0x86,0x95,0x34,0xd4, + 0x20,0xd3,0x14,0x52,0x04,0x05,0x00,0xd1,0x08,0x10,0x04,0x05,0x00,0x00,0x00,0x10, + 0x04,0x05,0x00,0x00,0x00,0x92,0x08,0x11,0x04,0x00,0x00,0x05,0x00,0x05,0x00,0x93, + 0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x05,0x00,0x00,0x00,0x05,0x00,0x05,0x00,0x05, + 0x00,0x05,0x00,0xcf,0x06,0x05,0x00,0xd2,0x26,0xd1,0x06,0xcf,0x06,0x05,0x00,0xd0, + 0x1a,0xcf,0x86,0x55,0x04,0x05,0x00,0x94,0x10,0x93,0x0c,0x52,0x04,0x05,0x00,0x11, + 0x04,0x08,0x00,0x00,0x00,0x05,0x00,0x05,0x00,0xcf,0x06,0x05,0x00,0xd1,0x06,0xcf, + 0x06,0x05,0x00,0xd0,0x06,0xcf,0x06,0x05,0x00,0xcf,0x86,0x95,0x18,0x94,0x14,0x53, + 0x04,0x05,0x00,0xd2,0x08,0x11,0x04,0x05,0x00,0x09,0x00,0x11,0x04,0x00,0x00,0x05, + 0x00,0x05,0x00,0x05,0x00,0xd4,0x52,0xd3,0x06,0xcf,0x06,0x11,0x00,0xd2,0x46,0xd1, + 0x06,0xcf,0x06,0x11,0x00,0xd0,0x3a,0xcf,0x86,0xd5,0x20,0xd4,0x0c,0x53,0x04,0x11, + 0x00,0x12,0x04,0x11,0x00,0x00,0x00,0x53,0x04,0x00,0x00,0x92,0x0c,0x51,0x04,0x00, + 0x00,0x10,0x04,0x00,0x00,0x11,0x00,0x11,0x00,0x94,0x14,0x93,0x10,0x92,0x0c,0x91, + 0x08,0x10,0x04,0x00,0x00,0x11,0x00,0x11,0x00,0x11,0x00,0x11,0x00,0x00,0x00,0xcf, + 0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xe0,0xc2,0x03,0xcf,0x86, + 0xe5,0x03,0x01,0xd4,0xfc,0xd3,0xc0,0xd2,0x66,0xd1,0x60,0xd0,0x5a,0xcf,0x86,0xd5, + 0x2c,0xd4,0x14,0x93,0x10,0x52,0x04,0x12,0xe6,0x51,0x04,0x12,0xe6,0x10,0x04,0x12, + 0xe6,0x00,0x00,0x12,0xe6,0x53,0x04,0x12,0xe6,0x92,0x10,0xd1,0x08,0x10,0x04,0x12, + 0xe6,0x00,0x00,0x10,0x04,0x00,0x00,0x12,0xe6,0x12,0xe6,0x94,0x28,0xd3,0x18,0xd2, + 0x0c,0x51,0x04,0x12,0xe6,0x10,0x04,0x00,0x00,0x12,0xe6,0x91,0x08,0x10,0x04,0x12, + 0xe6,0x00,0x00,0x12,0xe6,0x92,0x0c,0x51,0x04,0x12,0xe6,0x10,0x04,0x12,0xe6,0x00, + 0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xd1,0x54,0xd0, + 0x36,0xcf,0x86,0x55,0x04,0x15,0x00,0xd4,0x14,0x53,0x04,0x15,0x00,0x52,0x04,0x15, + 0x00,0x91,0x08,0x10,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0xd3,0x10,0x52,0x04,0x15, + 0xe6,0x51,0x04,0x15,0xe6,0x10,0x04,0x15,0xe6,0x15,0x00,0x52,0x04,0x15,0x00,0x11, + 0x04,0x15,0x00,0x00,0x00,0xcf,0x86,0x95,0x18,0x94,0x14,0x53,0x04,0x15,0x00,0xd2, + 0x08,0x11,0x04,0x15,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x15,0x00,0x00,0x00,0x00, + 0x00,0xcf,0x06,0x00,0x00,0xd2,0x36,0xd1,0x06,0xcf,0x06,0x00,0x00,0xd0,0x06,0xcf, + 0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x15,0x00,0xd4,0x0c,0x53,0x04,0x15,0x00,0x12, + 0x04,0x15,0x00,0x15,0xe6,0x53,0x04,0x15,0x00,0xd2,0x08,0x11,0x04,0x15,0x00,0x00, + 0x00,0x51,0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x15,0x00,0xcf,0x06,0x00,0x00,0xcf, + 0x06,0x00,0x00,0xd4,0x82,0xd3,0x7c,0xd2,0x3e,0xd1,0x06,0xcf,0x06,0x10,0x00,0xd0, + 0x06,0xcf,0x06,0x10,0x00,0xcf,0x86,0x95,0x2c,0xd4,0x18,0x93,0x14,0x52,0x04,0x10, + 0x00,0xd1,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x10,0x00,0x10, + 0x00,0x93,0x10,0x52,0x04,0x10,0xdc,0x51,0x04,0x10,0xdc,0x10,0x04,0x10,0xdc,0x00, + 0x00,0x00,0x00,0x00,0x00,0xd1,0x38,0xd0,0x06,0xcf,0x06,0x12,0x00,0xcf,0x86,0x95, + 0x2c,0xd4,0x18,0xd3,0x08,0x12,0x04,0x12,0x00,0x12,0xe6,0x92,0x0c,0x51,0x04,0x12, + 0xe6,0x10,0x04,0x12,0x07,0x15,0x00,0x00,0x00,0x53,0x04,0x12,0x00,0xd2,0x08,0x11, + 0x04,0x12,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x12,0x00,0x00,0x00,0xcf,0x06,0x00, + 0x00,0xcf,0x06,0x00,0x00,0xd3,0x82,0xd2,0x48,0xd1,0x24,0xd0,0x06,0xcf,0x06,0x00, + 0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00,0x00,0x93,0x10,0x92,0x0c,0x91, + 0x08,0x10,0x04,0x00,0x00,0x14,0x00,0x14,0x00,0x14,0x00,0x14,0x00,0xd0,0x1e,0xcf, + 0x86,0x55,0x04,0x14,0x00,0x54,0x04,0x14,0x00,0x93,0x10,0x52,0x04,0x14,0x00,0x91, + 0x08,0x10,0x04,0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xd1, + 0x34,0xd0,0x2e,0xcf,0x86,0xd5,0x18,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10, + 0x04,0x00,0x00,0x15,0x00,0x15,0x00,0x15,0x00,0x15,0x00,0x15,0x00,0x54,0x04,0x15, + 0x00,0x53,0x04,0x15,0x00,0x52,0x04,0x15,0x00,0x11,0x04,0x15,0x00,0x00,0x00,0xcf, + 0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xe2,0xb2,0x01,0xe1,0x41,0x01,0xd0,0x6e,0xcf, + 0x86,0xd5,0x18,0x94,0x14,0x93,0x10,0x52,0x04,0x0d,0x00,0x91,0x08,0x10,0x04,0x00, + 0x00,0x0d,0x00,0x0d,0x00,0x0d,0x00,0x0d,0x00,0xd4,0x30,0xd3,0x20,0xd2,0x10,0xd1, + 0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x10,0x04,0x0d,0x00,0x00,0x00,0xd1,0x08,0x10, + 0x04,0x0d,0x00,0x00,0x00,0x10,0x04,0x00,0x00,0x0d,0x00,0x92,0x0c,0x91,0x08,0x10, + 0x04,0x00,0x00,0x0d,0x00,0x0d,0x00,0x0d,0x00,0xd3,0x10,0x92,0x0c,0x51,0x04,0x0d, + 0x00,0x10,0x04,0x0d,0x00,0x00,0x00,0x0d,0x00,0x92,0x10,0xd1,0x08,0x10,0x04,0x00, + 0x00,0x0d,0x00,0x10,0x04,0x00,0x00,0x0d,0x00,0x00,0x00,0xcf,0x86,0xd5,0x74,0xd4, + 0x34,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x00,0x00,0x10,0x04,0x0d,0x00,0x00,0x00,0x51, + 0x04,0x00,0x00,0x10,0x04,0x00,0x00,0x0d,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x00, + 0x00,0x0d,0x00,0x10,0x04,0x00,0x00,0x0d,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x0d, + 0x00,0x0d,0x00,0xd3,0x20,0xd2,0x10,0xd1,0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x10, + 0x04,0x0d,0x00,0x00,0x00,0xd1,0x08,0x10,0x04,0x0d,0x00,0x00,0x00,0x10,0x04,0x00, + 0x00,0x0d,0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x10,0x04,0x00, + 0x00,0x0d,0x00,0xd1,0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x10,0x04,0x00,0x00,0x0d, + 0x00,0xd4,0x30,0xd3,0x20,0xd2,0x10,0xd1,0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x10, + 0x04,0x0d,0x00,0x00,0x00,0xd1,0x08,0x10,0x04,0x0d,0x00,0x00,0x00,0x10,0x04,0x00, + 0x00,0x0d,0x00,0x92,0x0c,0x51,0x04,0x0d,0x00,0x10,0x04,0x0d,0x00,0x00,0x00,0x0d, + 0x00,0xd3,0x10,0x92,0x0c,0x51,0x04,0x0d,0x00,0x10,0x04,0x0d,0x00,0x00,0x00,0x0d, + 0x00,0xd2,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x0d,0x00,0xd1,0x08,0x10, + 0x04,0x0d,0x00,0x00,0x00,0x10,0x04,0x0d,0x00,0x00,0x00,0xd0,0x56,0xcf,0x86,0xd5, + 0x20,0xd4,0x14,0x53,0x04,0x0d,0x00,0x92,0x0c,0x51,0x04,0x0d,0x00,0x10,0x04,0x00, + 0x00,0x0d,0x00,0x0d,0x00,0x53,0x04,0x0d,0x00,0x12,0x04,0x0d,0x00,0x00,0x00,0xd4, + 0x28,0xd3,0x18,0xd2,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x0d,0x00,0x91, + 0x08,0x10,0x04,0x00,0x00,0x0d,0x00,0x0d,0x00,0x92,0x0c,0x51,0x04,0x0d,0x00,0x10, + 0x04,0x00,0x00,0x0d,0x00,0x0d,0x00,0x53,0x04,0x0d,0x00,0x12,0x04,0x0d,0x00,0x00, + 0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00,0x00,0x93,0x0c,0x92,0x08,0x11, + 0x04,0x0d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x86,0xe5, + 0x96,0x05,0xe4,0x28,0x03,0xe3,0xed,0x01,0xd2,0xa0,0xd1,0x1c,0xd0,0x16,0xcf,0x86, + 0x55,0x04,0x0a,0x00,0x94,0x0c,0x53,0x04,0x0a,0x00,0x12,0x04,0x0a,0x00,0x00,0x00, + 0x0a,0x00,0xcf,0x06,0x0a,0x00,0xd0,0x46,0xcf,0x86,0xd5,0x10,0x54,0x04,0x0a,0x00, + 0x93,0x08,0x12,0x04,0x0a,0x00,0x00,0x00,0x00,0x00,0xd4,0x14,0x53,0x04,0x0c,0x00, + 0x52,0x04,0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00,0x00,0x00,0xd3,0x10, + 0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x0c,0x00,0x0c,0x00,0x0c,0x00,0x52,0x04, + 0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00,0x10,0x00,0xcf,0x86,0xd5,0x28, + 0xd4,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x0c,0x00,0x0c,0x00, + 0x0c,0x00,0x0c,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x00,0x00,0x0c,0x00, + 0x0c,0x00,0x0c,0x00,0x0c,0x00,0x54,0x04,0x10,0x00,0x93,0x0c,0x52,0x04,0x10,0x00, + 0x11,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0xd1,0xe4,0xd0,0x5a,0xcf,0x86,0xd5,0x20, + 0x94,0x1c,0x53,0x04,0x0b,0x00,0xd2,0x0c,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b,0x00, + 0x10,0x00,0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x0b,0x00,0xd4,0x14, + 0x53,0x04,0x0b,0x00,0x52,0x04,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04,0x0b,0x00, + 0x14,0x00,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0c,0x00,0x0b,0x00,0x0c,0x00, + 0x0c,0x00,0x52,0x04,0x0c,0x00,0xd1,0x08,0x10,0x04,0x0c,0x00,0x0b,0x00,0x10,0x04, + 0x0c,0x00,0x0b,0x00,0xcf,0x86,0xd5,0x4c,0xd4,0x2c,0xd3,0x18,0xd2,0x0c,0x51,0x04, + 0x0c,0x00,0x10,0x04,0x0b,0x00,0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04,0x0b,0x00, + 0x0c,0x00,0xd2,0x08,0x11,0x04,0x0c,0x00,0x0b,0x00,0x51,0x04,0x0b,0x00,0x10,0x04, + 0x0b,0x00,0x0c,0x00,0xd3,0x10,0x52,0x04,0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04, + 0x0c,0x00,0x0b,0x00,0x52,0x04,0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00, + 0x0b,0x00,0xd4,0x18,0x53,0x04,0x0c,0x00,0xd2,0x08,0x11,0x04,0x0c,0x00,0x0d,0x00, + 0x91,0x08,0x10,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0x53,0x04,0x0c,0x00,0xd2,0x10, + 0xd1,0x08,0x10,0x04,0x0c,0x00,0x0b,0x00,0x10,0x04,0x0c,0x00,0x0b,0x00,0xd1,0x08, + 0x10,0x04,0x0b,0x00,0x0c,0x00,0x10,0x04,0x0c,0x00,0x0b,0x00,0xd0,0x4e,0xcf,0x86, + 0xd5,0x34,0xd4,0x14,0x53,0x04,0x0c,0x00,0xd2,0x08,0x11,0x04,0x0c,0x00,0x0b,0x00, + 0x11,0x04,0x0b,0x00,0x0c,0x00,0xd3,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0b,0x00, + 0x0c,0x00,0x0c,0x00,0x0c,0x00,0x92,0x0c,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00, + 0x12,0x00,0x12,0x00,0x94,0x14,0x53,0x04,0x12,0x00,0x52,0x04,0x12,0x00,0x91,0x08, + 0x10,0x04,0x12,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00, + 0x94,0x10,0x93,0x0c,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x0c,0x00,0x0c,0x00, + 0x0c,0x00,0xd2,0x7e,0xd1,0x78,0xd0,0x3e,0xcf,0x86,0xd5,0x1c,0x94,0x18,0x93,0x14, + 0x92,0x10,0xd1,0x08,0x10,0x04,0x0b,0x00,0x0c,0x00,0x10,0x04,0x0c,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x0b,0x00,0x54,0x04,0x0b,0x00,0xd3,0x0c,0x92,0x08,0x11,0x04, + 0x0b,0x00,0x0c,0x00,0x0c,0x00,0x92,0x0c,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00, + 0x12,0x00,0x00,0x00,0xcf,0x86,0xd5,0x24,0xd4,0x14,0x53,0x04,0x0b,0x00,0x92,0x0c, + 0x91,0x08,0x10,0x04,0x0b,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x93,0x0c,0x92,0x08, + 0x11,0x04,0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x94,0x10,0x93,0x0c,0x52,0x04, + 0x13,0x00,0x11,0x04,0x13,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00, + 0xd1,0x58,0xd0,0x3a,0xcf,0x86,0x55,0x04,0x0c,0x00,0xd4,0x20,0xd3,0x10,0x92,0x0c, + 0x91,0x08,0x10,0x04,0x0c,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x52,0x04,0x10,0x00, + 0x91,0x08,0x10,0x04,0x10,0x00,0x11,0x00,0x11,0x00,0x93,0x10,0x52,0x04,0x0c,0x00, + 0x51,0x04,0x0c,0x00,0x10,0x04,0x10,0x00,0x0c,0x00,0x0c,0x00,0xcf,0x86,0x55,0x04, + 0x0c,0x00,0x54,0x04,0x0c,0x00,0x53,0x04,0x0c,0x00,0x52,0x04,0x0c,0x00,0x91,0x08, + 0x10,0x04,0x0c,0x00,0x10,0x00,0x11,0x00,0xd0,0x16,0xcf,0x86,0x95,0x10,0x54,0x04, + 0x0c,0x00,0x93,0x08,0x12,0x04,0x0c,0x00,0x10,0x00,0x10,0x00,0x0c,0x00,0xcf,0x86, + 0xd5,0x34,0xd4,0x28,0xd3,0x10,0x52,0x04,0x0c,0x00,0x91,0x08,0x10,0x04,0x0c,0x00, + 0x10,0x00,0x0c,0x00,0xd2,0x0c,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00,0x10,0x00, + 0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x11,0x00,0x93,0x08,0x12,0x04,0x11,0x00, + 0x10,0x00,0x10,0x00,0x54,0x04,0x0c,0x00,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04, + 0x0c,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x11,0x00,0xd3,0xfc,0xd2,0x6c,0xd1,0x3c, + 0xd0,0x1e,0xcf,0x86,0x55,0x04,0x0c,0x00,0x54,0x04,0x0c,0x00,0x53,0x04,0x0c,0x00, + 0x52,0x04,0x0c,0x00,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00,0x10,0x00,0xcf,0x86, + 0x95,0x18,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0c,0x00,0x10,0x00, + 0x0c,0x00,0x0c,0x00,0x0c,0x00,0x0c,0x00,0x0c,0x00,0xd0,0x06,0xcf,0x06,0x0c,0x00, + 0xcf,0x86,0x55,0x04,0x0c,0x00,0x54,0x04,0x0c,0x00,0x53,0x04,0x0c,0x00,0xd2,0x0c, + 0x91,0x08,0x10,0x04,0x10,0x00,0x0c,0x00,0x0c,0x00,0xd1,0x08,0x10,0x04,0x0c,0x00, + 0x10,0x00,0x10,0x04,0x10,0x00,0x11,0x00,0xd1,0x54,0xd0,0x1a,0xcf,0x86,0x55,0x04, + 0x0c,0x00,0x54,0x04,0x0c,0x00,0x53,0x04,0x0c,0x00,0x52,0x04,0x0c,0x00,0x11,0x04, + 0x0c,0x00,0x10,0x00,0xcf,0x86,0xd5,0x1c,0x94,0x18,0xd3,0x08,0x12,0x04,0x0d,0x00, + 0x10,0x00,0x92,0x0c,0x51,0x04,0x10,0x00,0x10,0x04,0x10,0x00,0x11,0x00,0x11,0x00, + 0x0c,0x00,0xd4,0x08,0x13,0x04,0x0c,0x00,0x10,0x00,0x53,0x04,0x10,0x00,0x92,0x0c, + 0x51,0x04,0x10,0x00,0x10,0x04,0x12,0x00,0x10,0x00,0x10,0x00,0xd0,0x1e,0xcf,0x86, + 0x55,0x04,0x10,0x00,0x94,0x14,0x93,0x10,0x52,0x04,0x10,0x00,0x91,0x08,0x10,0x04, + 0x12,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0x10,0x00,0xcf,0x86,0x55,0x04,0x10,0x00, + 0x54,0x04,0x10,0x00,0x53,0x04,0x10,0x00,0x92,0x0c,0x51,0x04,0x10,0x00,0x10,0x04, + 0x10,0x00,0x0c,0x00,0x0c,0x00,0xe2,0x19,0x01,0xd1,0xa8,0xd0,0x7e,0xcf,0x86,0xd5, + 0x4c,0xd4,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x0d,0x00,0x0c,0x00,0x0c, + 0x00,0x0c,0x00,0x0c,0x00,0xd3,0x1c,0xd2,0x0c,0x91,0x08,0x10,0x04,0x0c,0x00,0x0d, + 0x00,0x0c,0x00,0xd1,0x08,0x10,0x04,0x0c,0x00,0x0d,0x00,0x10,0x04,0x0c,0x00,0x0d, + 0x00,0xd2,0x10,0xd1,0x08,0x10,0x04,0x0c,0x00,0x0d,0x00,0x10,0x04,0x0c,0x00,0x0d, + 0x00,0x51,0x04,0x0c,0x00,0x10,0x04,0x0c,0x00,0x0d,0x00,0xd4,0x1c,0xd3,0x0c,0x52, + 0x04,0x0c,0x00,0x11,0x04,0x0c,0x00,0x0d,0x00,0x52,0x04,0x0c,0x00,0x91,0x08,0x10, + 0x04,0x0d,0x00,0x0c,0x00,0x0d,0x00,0x93,0x10,0x52,0x04,0x0c,0x00,0x91,0x08,0x10, + 0x04,0x0d,0x00,0x0c,0x00,0x0c,0x00,0x0c,0x00,0xcf,0x86,0x95,0x24,0x94,0x20,0x93, + 0x1c,0xd2,0x10,0xd1,0x08,0x10,0x04,0x0c,0x00,0x10,0x00,0x10,0x04,0x10,0x00,0x11, + 0x00,0x91,0x08,0x10,0x04,0x11,0x00,0x0c,0x00,0x0c,0x00,0x0c,0x00,0x10,0x00,0x10, + 0x00,0xd0,0x06,0xcf,0x06,0x0c,0x00,0xcf,0x86,0xd5,0x30,0xd4,0x10,0x93,0x0c,0x52, + 0x04,0x0c,0x00,0x11,0x04,0x0c,0x00,0x10,0x00,0x10,0x00,0x93,0x1c,0xd2,0x10,0xd1, + 0x08,0x10,0x04,0x11,0x00,0x12,0x00,0x10,0x04,0x12,0x00,0x13,0x00,0x91,0x08,0x10, + 0x04,0x13,0x00,0x15,0x00,0x00,0x00,0x00,0x00,0xd4,0x14,0x53,0x04,0x10,0x00,0x52, + 0x04,0x10,0x00,0x91,0x08,0x10,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0xd3,0x10,0x52, + 0x04,0x10,0x00,0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x00,0x13,0x00,0x92,0x10,0xd1, + 0x08,0x10,0x04,0x13,0x00,0x14,0x00,0x10,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0xd1, + 0x1c,0xd0,0x06,0xcf,0x06,0x0c,0x00,0xcf,0x86,0x55,0x04,0x0c,0x00,0x54,0x04,0x0c, + 0x00,0x93,0x08,0x12,0x04,0x0c,0x00,0x00,0x00,0x00,0x00,0xd0,0x06,0xcf,0x06,0x10, + 0x00,0xcf,0x86,0xd5,0x24,0x54,0x04,0x10,0x00,0xd3,0x10,0x52,0x04,0x10,0x00,0x91, + 0x08,0x10,0x04,0x10,0x00,0x14,0x00,0x14,0x00,0x92,0x0c,0x91,0x08,0x10,0x04,0x14, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x94,0x0c,0x53,0x04,0x15,0x00,0x12,0x04,0x15, + 0x00,0x00,0x00,0x00,0x00,0xe4,0x40,0x02,0xe3,0xc9,0x01,0xd2,0x5c,0xd1,0x34,0xd0, + 0x16,0xcf,0x86,0x95,0x10,0x94,0x0c,0x53,0x04,0x10,0x00,0x12,0x04,0x10,0x00,0x00, + 0x00,0x10,0x00,0x10,0x00,0xcf,0x86,0x95,0x18,0xd4,0x08,0x13,0x04,0x10,0x00,0x00, + 0x00,0x53,0x04,0x10,0x00,0x92,0x08,0x11,0x04,0x10,0x00,0x00,0x00,0x00,0x00,0x10, + 0x00,0xd0,0x22,0xcf,0x86,0xd5,0x0c,0x94,0x08,0x13,0x04,0x10,0x00,0x00,0x00,0x10, + 0x00,0x94,0x10,0x53,0x04,0x10,0x00,0x52,0x04,0x10,0x00,0x11,0x04,0x10,0x00,0x00, + 0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xd1,0xc0,0xd0,0x5e,0xcf,0x86,0xd5,0x30,0xd4, + 0x14,0x53,0x04,0x13,0x00,0x52,0x04,0x13,0x00,0x91,0x08,0x10,0x04,0x00,0x00,0x15, + 0x00,0x15,0x00,0x53,0x04,0x11,0x00,0xd2,0x0c,0x91,0x08,0x10,0x04,0x11,0x00,0x12, + 0x00,0x12,0x00,0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x00,0x13,0x00,0xd4,0x08,0x13, + 0x04,0x12,0x00,0x13,0x00,0xd3,0x14,0x92,0x10,0xd1,0x08,0x10,0x04,0x12,0x00,0x13, + 0x00,0x10,0x04,0x13,0x00,0x12,0x00,0x12,0x00,0x52,0x04,0x12,0x00,0x51,0x04,0x12, + 0x00,0x10,0x04,0x12,0x00,0x15,0x00,0xcf,0x86,0xd5,0x28,0xd4,0x14,0x53,0x04,0x12, + 0x00,0x52,0x04,0x12,0x00,0x91,0x08,0x10,0x04,0x13,0x00,0x14,0x00,0x14,0x00,0x53, + 0x04,0x12,0x00,0x52,0x04,0x12,0x00,0x51,0x04,0x12,0x00,0x10,0x04,0x12,0x00,0x13, + 0x00,0xd4,0x0c,0x53,0x04,0x13,0x00,0x12,0x04,0x13,0x00,0x14,0x00,0xd3,0x1c,0xd2, + 0x10,0xd1,0x08,0x10,0x04,0x14,0x00,0x15,0x00,0x10,0x04,0x00,0x00,0x14,0x00,0x51, + 0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x00,0x00,0x92,0x0c,0x51,0x04,0x00,0x00,0x10, + 0x04,0x14,0x00,0x15,0x00,0x14,0x00,0xd0,0x62,0xcf,0x86,0xd5,0x24,0xd4,0x14,0x93, + 0x10,0x52,0x04,0x11,0x00,0x91,0x08,0x10,0x04,0x11,0x00,0x12,0x00,0x12,0x00,0x12, + 0x00,0x93,0x0c,0x92,0x08,0x11,0x04,0x12,0x00,0x13,0x00,0x13,0x00,0x14,0x00,0xd4, + 0x2c,0xd3,0x18,0xd2,0x0c,0x51,0x04,0x14,0x00,0x10,0x04,0x14,0x00,0x00,0x00,0x91, + 0x08,0x10,0x04,0x00,0x00,0x15,0x00,0x15,0x00,0xd2,0x0c,0x51,0x04,0x15,0x00,0x10, + 0x04,0x15,0x00,0x00,0x00,0x11,0x04,0x00,0x00,0x15,0x00,0x53,0x04,0x14,0x00,0x92, + 0x08,0x11,0x04,0x14,0x00,0x15,0x00,0x15,0x00,0xcf,0x86,0xd5,0x30,0x94,0x2c,0xd3, + 0x14,0x92,0x10,0xd1,0x08,0x10,0x04,0x11,0x00,0x14,0x00,0x10,0x04,0x14,0x00,0x15, + 0x00,0x15,0x00,0xd2,0x0c,0x51,0x04,0x15,0x00,0x10,0x04,0x15,0x00,0x00,0x00,0x91, + 0x08,0x10,0x04,0x00,0x00,0x15,0x00,0x15,0x00,0x13,0x00,0x94,0x14,0x93,0x10,0x52, + 0x04,0x13,0x00,0x51,0x04,0x13,0x00,0x10,0x04,0x13,0x00,0x14,0x00,0x14,0x00,0x14, + 0x00,0xd2,0x70,0xd1,0x40,0xd0,0x06,0xcf,0x06,0x15,0x00,0xcf,0x86,0xd5,0x10,0x54, + 0x04,0x15,0x00,0x93,0x08,0x12,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0xd4,0x10,0x53, + 0x04,0x14,0x00,0x52,0x04,0x14,0x00,0x11,0x04,0x14,0x00,0x00,0x00,0xd3,0x08,0x12, + 0x04,0x15,0x00,0x00,0x00,0x92,0x0c,0x51,0x04,0x15,0x00,0x10,0x04,0x15,0x00,0x00, + 0x00,0x00,0x00,0xd0,0x2a,0xcf,0x86,0x95,0x24,0xd4,0x14,0x93,0x10,0x92,0x0c,0x51, + 0x04,0x15,0x00,0x10,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x93,0x0c,0x52, + 0x04,0x15,0x00,0x11,0x04,0x15,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00, + 0x00,0xcf,0x06,0x00,0x00,0xd3,0x06,0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00, + 0x00,0xd1,0x06,0xcf,0x06,0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55, + 0x04,0x00,0x00,0x54,0x04,0x00,0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11, + 0x04,0x00,0x00,0x02,0x00,0xe4,0xf9,0x12,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x05,0x00, + 0xd2,0xc2,0xd1,0x08,0xcf,0x86,0xcf,0x06,0x05,0x00,0xd0,0x44,0xcf,0x86,0xd5,0x3c, + 0xd4,0x06,0xcf,0x06,0x05,0x00,0xd3,0x06,0xcf,0x06,0x05,0x00,0xd2,0x2a,0xd1,0x06, + 0xcf,0x06,0x05,0x00,0xd0,0x06,0xcf,0x06,0x05,0x00,0xcf,0x86,0x95,0x18,0x54,0x04, + 0x05,0x00,0x93,0x10,0x52,0x04,0x05,0x00,0x51,0x04,0x05,0x00,0x10,0x04,0x05,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x0b,0x00,0xcf,0x06,0x0b,0x00,0xcf,0x86, + 0xd5,0x3c,0xd4,0x06,0xcf,0x06,0x0b,0x00,0xd3,0x06,0xcf,0x06,0x0b,0x00,0xd2,0x06, + 0xcf,0x06,0x0b,0x00,0xd1,0x24,0xd0,0x1e,0xcf,0x86,0x55,0x04,0x0b,0x00,0x54,0x04, + 0x0b,0x00,0x93,0x10,0x52,0x04,0x0b,0x00,0x91,0x08,0x10,0x04,0x0b,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0xcf,0x06,0x0c,0x00,0xcf,0x06,0x0c,0x00,0xd4,0x32,0xd3,0x2c, + 0xd2,0x26,0xd1,0x20,0xd0,0x1a,0xcf,0x86,0x95,0x14,0x54,0x04,0x0c,0x00,0x53,0x04, + 0x0c,0x00,0x52,0x04,0x0c,0x00,0x11,0x04,0x0c,0x00,0x00,0x00,0x11,0x00,0xcf,0x06, + 0x11,0x00,0xcf,0x06,0x11,0x00,0xcf,0x06,0x11,0x00,0xcf,0x06,0x11,0x00,0xcf,0x06, + 0x11,0x00,0xd1,0x48,0xd0,0x40,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x11,0x00,0xd4,0x06, + 0xcf,0x06,0x11,0x00,0xd3,0x06,0xcf,0x06,0x11,0x00,0xd2,0x26,0xd1,0x06,0xcf,0x06, + 0x11,0x00,0xd0,0x1a,0xcf,0x86,0x55,0x04,0x11,0x00,0x94,0x10,0x93,0x0c,0x92,0x08, + 0x11,0x04,0x11,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x13,0x00,0xcf,0x06,0x13,0x00, + 0xcf,0x06,0x13,0x00,0xcf,0x86,0xcf,0x06,0x13,0x00,0xd0,0x44,0xcf,0x86,0xd5,0x06, + 0xcf,0x06,0x13,0x00,0xd4,0x36,0xd3,0x06,0xcf,0x06,0x13,0x00,0xd2,0x06,0xcf,0x06, + 0x13,0x00,0xd1,0x06,0xcf,0x06,0x13,0x00,0xd0,0x06,0xcf,0x06,0x13,0x00,0xcf,0x86, + 0x55,0x04,0x13,0x00,0x94,0x14,0x93,0x10,0x92,0x0c,0x91,0x08,0x10,0x04,0x13,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x86, + 0xd5,0x06,0xcf,0x06,0x00,0x00,0xe4,0x68,0x11,0xe3,0x51,0x10,0xe2,0x17,0x08,0xe1, + 0x06,0x04,0xe0,0x03,0x02,0xcf,0x86,0xe5,0x06,0x01,0xd4,0x82,0xd3,0x41,0xd2,0x21, + 0xd1,0x10,0x10,0x08,0x05,0xff,0xe4,0xb8,0xbd,0x00,0x05,0xff,0xe4,0xb8,0xb8,0x00, + 0x10,0x08,0x05,0xff,0xe4,0xb9,0x81,0x00,0x05,0xff,0xf0,0xa0,0x84,0xa2,0x00,0xd1, + 0x10,0x10,0x08,0x05,0xff,0xe4,0xbd,0xa0,0x00,0x05,0xff,0xe4,0xbe,0xae,0x00,0x10, + 0x08,0x05,0xff,0xe4,0xbe,0xbb,0x00,0x05,0xff,0xe5,0x80,0x82,0x00,0xd2,0x20,0xd1, + 0x10,0x10,0x08,0x05,0xff,0xe5,0x81,0xba,0x00,0x05,0xff,0xe5,0x82,0x99,0x00,0x10, + 0x08,0x05,0xff,0xe5,0x83,0xa7,0x00,0x05,0xff,0xe5,0x83,0x8f,0x00,0xd1,0x11,0x10, + 0x08,0x05,0xff,0xe3,0x92,0x9e,0x00,0x05,0xff,0xf0,0xa0,0x98,0xba,0x00,0x10,0x08, + 0x05,0xff,0xe5,0x85,0x8d,0x00,0x05,0xff,0xe5,0x85,0x94,0x00,0xd3,0x42,0xd2,0x21, + 0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0x85,0xa4,0x00,0x05,0xff,0xe5,0x85,0xb7,0x00, + 0x10,0x09,0x05,0xff,0xf0,0xa0,0x94,0x9c,0x00,0x05,0xff,0xe3,0x92,0xb9,0x00,0xd1, + 0x10,0x10,0x08,0x05,0xff,0xe5,0x85,0xa7,0x00,0x05,0xff,0xe5,0x86,0x8d,0x00,0x10, + 0x09,0x05,0xff,0xf0,0xa0,0x95,0x8b,0x00,0x05,0xff,0xe5,0x86,0x97,0x00,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0x86,0xa4,0x00,0x05,0xff,0xe4,0xbb,0x8c,0x00, + 0x10,0x08,0x05,0xff,0xe5,0x86,0xac,0x00,0x05,0xff,0xe5,0x86,0xb5,0x00,0xd1,0x11, + 0x10,0x09,0x05,0xff,0xf0,0xa9,0x87,0x9f,0x00,0x05,0xff,0xe5,0x87,0xb5,0x00,0x10, + 0x08,0x05,0xff,0xe5,0x88,0x83,0x00,0x05,0xff,0xe3,0x93,0x9f,0x00,0xd4,0x80,0xd3, + 0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0x88,0xbb,0x00,0x05,0xff,0xe5, + 0x89,0x86,0x00,0x10,0x08,0x05,0xff,0xe5,0x89,0xb2,0x00,0x05,0xff,0xe5,0x89,0xb7, + 0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe3,0x94,0x95,0x00,0x05,0xff,0xe5,0x8b,0x87, + 0x00,0x10,0x08,0x05,0xff,0xe5,0x8b,0x89,0x00,0x05,0xff,0xe5,0x8b,0xa4,0x00,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0x8b,0xba,0x00,0x05,0xff,0xe5,0x8c,0x85, + 0x00,0x10,0x08,0x05,0xff,0xe5,0x8c,0x86,0x00,0x05,0xff,0xe5,0x8c,0x97,0x00,0xd1, + 0x10,0x10,0x08,0x05,0xff,0xe5,0x8d,0x89,0x00,0x05,0xff,0xe5,0x8d,0x91,0x00,0x10, + 0x08,0x05,0xff,0xe5,0x8d,0x9a,0x00,0x05,0xff,0xe5,0x8d,0xb3,0x00,0xd3,0x39,0xd2, + 0x18,0x91,0x10,0x10,0x08,0x05,0xff,0xe5,0x8d,0xbd,0x00,0x05,0xff,0xe5,0x8d,0xbf, + 0x00,0x05,0xff,0xe5,0x8d,0xbf,0x00,0xd1,0x11,0x10,0x09,0x05,0xff,0xf0,0xa0,0xa8, + 0xac,0x00,0x05,0xff,0xe7,0x81,0xb0,0x00,0x10,0x08,0x05,0xff,0xe5,0x8f,0x8a,0x00, + 0x05,0xff,0xe5,0x8f,0x9f,0x00,0xd2,0x21,0xd1,0x11,0x10,0x09,0x05,0xff,0xf0,0xa0, + 0xad,0xa3,0x00,0x05,0xff,0xe5,0x8f,0xab,0x00,0x10,0x08,0x05,0xff,0xe5,0x8f,0xb1, + 0x00,0x05,0xff,0xe5,0x90,0x86,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0x92,0x9e, + 0x00,0x05,0xff,0xe5,0x90,0xb8,0x00,0x10,0x08,0x05,0xff,0xe5,0x91,0x88,0x00,0x05, + 0xff,0xe5,0x91,0xa8,0x00,0xcf,0x86,0xe5,0x02,0x01,0xd4,0x80,0xd3,0x40,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0x92,0xa2,0x00,0x05,0xff,0xe5,0x93,0xb6,0x00, + 0x10,0x08,0x05,0xff,0xe5,0x94,0x90,0x00,0x05,0xff,0xe5,0x95,0x93,0x00,0xd1,0x10, + 0x10,0x08,0x05,0xff,0xe5,0x95,0xa3,0x00,0x05,0xff,0xe5,0x96,0x84,0x00,0x10,0x08, + 0x05,0xff,0xe5,0x96,0x84,0x00,0x05,0xff,0xe5,0x96,0x99,0x00,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x05,0xff,0xe5,0x96,0xab,0x00,0x05,0xff,0xe5,0x96,0xb3,0x00,0x10,0x08, + 0x05,0xff,0xe5,0x97,0x82,0x00,0x05,0xff,0xe5,0x9c,0x96,0x00,0xd1,0x10,0x10,0x08, + 0x05,0xff,0xe5,0x98,0x86,0x00,0x05,0xff,0xe5,0x9c,0x97,0x00,0x10,0x08,0x05,0xff, + 0xe5,0x99,0x91,0x00,0x05,0xff,0xe5,0x99,0xb4,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x05,0xff,0xe5,0x88,0x87,0x00,0x05,0xff,0xe5,0xa3,0xae,0x00,0x10,0x08, + 0x05,0xff,0xe5,0x9f,0x8e,0x00,0x05,0xff,0xe5,0x9f,0xb4,0x00,0xd1,0x10,0x10,0x08, + 0x05,0xff,0xe5,0xa0,0x8d,0x00,0x05,0xff,0xe5,0x9e,0x8b,0x00,0x10,0x08,0x05,0xff, + 0xe5,0xa0,0xb2,0x00,0x05,0xff,0xe5,0xa0,0xb1,0x00,0xd2,0x21,0xd1,0x11,0x10,0x08, + 0x05,0xff,0xe5,0xa2,0xac,0x00,0x05,0xff,0xf0,0xa1,0x93,0xa4,0x00,0x10,0x08,0x05, + 0xff,0xe5,0xa3,0xb2,0x00,0x05,0xff,0xe5,0xa3,0xb7,0x00,0xd1,0x10,0x10,0x08,0x05, + 0xff,0xe5,0xa4,0x86,0x00,0x05,0xff,0xe5,0xa4,0x9a,0x00,0x10,0x08,0x05,0xff,0xe5, + 0xa4,0xa2,0x00,0x05,0xff,0xe5,0xa5,0xa2,0x00,0xd4,0x7b,0xd3,0x42,0xd2,0x22,0xd1, + 0x12,0x10,0x09,0x05,0xff,0xf0,0xa1,0x9a,0xa8,0x00,0x05,0xff,0xf0,0xa1,0x9b,0xaa, + 0x00,0x10,0x08,0x05,0xff,0xe5,0xa7,0xac,0x00,0x05,0xff,0xe5,0xa8,0x9b,0x00,0xd1, + 0x10,0x10,0x08,0x05,0xff,0xe5,0xa8,0xa7,0x00,0x05,0xff,0xe5,0xa7,0x98,0x00,0x10, + 0x08,0x05,0xff,0xe5,0xa9,0xa6,0x00,0x05,0xff,0xe3,0x9b,0xae,0x00,0xd2,0x18,0x91, + 0x10,0x10,0x08,0x05,0xff,0xe3,0x9b,0xbc,0x00,0x05,0xff,0xe5,0xac,0x88,0x00,0x05, + 0xff,0xe5,0xac,0xbe,0x00,0xd1,0x11,0x10,0x09,0x05,0xff,0xf0,0xa1,0xa7,0x88,0x00, + 0x05,0xff,0xe5,0xaf,0x83,0x00,0x10,0x08,0x05,0xff,0xe5,0xaf,0x98,0x00,0x05,0xff, + 0xe5,0xaf,0xa7,0x00,0xd3,0x41,0xd2,0x21,0xd1,0x11,0x10,0x08,0x05,0xff,0xe5,0xaf, + 0xb3,0x00,0x05,0xff,0xf0,0xa1,0xac,0x98,0x00,0x10,0x08,0x05,0xff,0xe5,0xaf,0xbf, + 0x00,0x05,0xff,0xe5,0xb0,0x86,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0xbd,0x93, + 0x00,0x05,0xff,0xe5,0xb0,0xa2,0x00,0x10,0x08,0x05,0xff,0xe3,0x9e,0x81,0x00,0x05, + 0xff,0xe5,0xb1,0xa0,0x00,0xd2,0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0xb1,0xae, + 0x00,0x05,0xff,0xe5,0xb3,0x80,0x00,0x10,0x08,0x05,0xff,0xe5,0xb2,0x8d,0x00,0x05, + 0xff,0xf0,0xa1,0xb7,0xa4,0x00,0xd1,0x11,0x10,0x08,0x05,0xff,0xe5,0xb5,0x83,0x00, + 0x05,0xff,0xf0,0xa1,0xb7,0xa6,0x00,0x10,0x08,0x05,0xff,0xe5,0xb5,0xae,0x00,0x05, + 0xff,0xe5,0xb5,0xab,0x00,0xe0,0x04,0x02,0xcf,0x86,0xd5,0xfe,0xd4,0x82,0xd3,0x40, + 0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0xb5,0xbc,0x00,0x05,0xff,0xe5,0xb7, + 0xa1,0x00,0x10,0x08,0x05,0xff,0xe5,0xb7,0xa2,0x00,0x05,0xff,0xe3,0xa0,0xaf,0x00, + 0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0xb7,0xbd,0x00,0x05,0xff,0xe5,0xb8,0xa8,0x00, + 0x10,0x08,0x05,0xff,0xe5,0xb8,0xbd,0x00,0x05,0xff,0xe5,0xb9,0xa9,0x00,0xd2,0x21, + 0xd1,0x11,0x10,0x08,0x05,0xff,0xe3,0xa1,0xa2,0x00,0x05,0xff,0xf0,0xa2,0x86,0x83, + 0x00,0x10,0x08,0x05,0xff,0xe3,0xa1,0xbc,0x00,0x05,0xff,0xe5,0xba,0xb0,0x00,0xd1, + 0x10,0x10,0x08,0x05,0xff,0xe5,0xba,0xb3,0x00,0x05,0xff,0xe5,0xba,0xb6,0x00,0x10, + 0x08,0x05,0xff,0xe5,0xbb,0x8a,0x00,0x05,0xff,0xf0,0xaa,0x8e,0x92,0x00,0xd3,0x3b, + 0xd2,0x22,0xd1,0x11,0x10,0x08,0x05,0xff,0xe5,0xbb,0xbe,0x00,0x05,0xff,0xf0,0xa2, + 0x8c,0xb1,0x00,0x10,0x09,0x05,0xff,0xf0,0xa2,0x8c,0xb1,0x00,0x05,0xff,0xe8,0x88, + 0x81,0x00,0x51,0x08,0x05,0xff,0xe5,0xbc,0xa2,0x00,0x10,0x08,0x05,0xff,0xe3,0xa3, + 0x87,0x00,0x05,0xff,0xf0,0xa3,0x8a,0xb8,0x00,0xd2,0x21,0xd1,0x11,0x10,0x09,0x05, + 0xff,0xf0,0xa6,0x87,0x9a,0x00,0x05,0xff,0xe5,0xbd,0xa2,0x00,0x10,0x08,0x05,0xff, + 0xe5,0xbd,0xab,0x00,0x05,0xff,0xe3,0xa3,0xa3,0x00,0xd1,0x10,0x10,0x08,0x05,0xff, + 0xe5,0xbe,0x9a,0x00,0x05,0xff,0xe5,0xbf,0x8d,0x00,0x10,0x08,0x05,0xff,0xe5,0xbf, + 0x97,0x00,0x05,0xff,0xe5,0xbf,0xb9,0x00,0xd4,0x81,0xd3,0x41,0xd2,0x20,0xd1,0x10, + 0x10,0x08,0x05,0xff,0xe6,0x82,0x81,0x00,0x05,0xff,0xe3,0xa4,0xba,0x00,0x10,0x08, + 0x05,0xff,0xe3,0xa4,0x9c,0x00,0x05,0xff,0xe6,0x82,0x94,0x00,0xd1,0x11,0x10,0x09, + 0x05,0xff,0xf0,0xa2,0x9b,0x94,0x00,0x05,0xff,0xe6,0x83,0x87,0x00,0x10,0x08,0x05, + 0xff,0xe6,0x85,0x88,0x00,0x05,0xff,0xe6,0x85,0x8c,0x00,0xd2,0x20,0xd1,0x10,0x10, + 0x08,0x05,0xff,0xe6,0x85,0x8e,0x00,0x05,0xff,0xe6,0x85,0x8c,0x00,0x10,0x08,0x05, + 0xff,0xe6,0x85,0xba,0x00,0x05,0xff,0xe6,0x86,0x8e,0x00,0xd1,0x10,0x10,0x08,0x05, + 0xff,0xe6,0x86,0xb2,0x00,0x05,0xff,0xe6,0x86,0xa4,0x00,0x10,0x08,0x05,0xff,0xe6, + 0x86,0xaf,0x00,0x05,0xff,0xe6,0x87,0x9e,0x00,0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10, + 0x08,0x05,0xff,0xe6,0x87,0xb2,0x00,0x05,0xff,0xe6,0x87,0xb6,0x00,0x10,0x08,0x05, + 0xff,0xe6,0x88,0x90,0x00,0x05,0xff,0xe6,0x88,0x9b,0x00,0xd1,0x10,0x10,0x08,0x05, + 0xff,0xe6,0x89,0x9d,0x00,0x05,0xff,0xe6,0x8a,0xb1,0x00,0x10,0x08,0x05,0xff,0xe6, + 0x8b,0x94,0x00,0x05,0xff,0xe6,0x8d,0x90,0x00,0xd2,0x21,0xd1,0x11,0x10,0x09,0x05, + 0xff,0xf0,0xa2,0xac,0x8c,0x00,0x05,0xff,0xe6,0x8c,0xbd,0x00,0x10,0x08,0x05,0xff, + 0xe6,0x8b,0xbc,0x00,0x05,0xff,0xe6,0x8d,0xa8,0x00,0xd1,0x10,0x10,0x08,0x05,0xff, + 0xe6,0x8e,0x83,0x00,0x05,0xff,0xe6,0x8f,0xa4,0x00,0x10,0x09,0x05,0xff,0xf0,0xa2, + 0xaf,0xb1,0x00,0x05,0xff,0xe6,0x90,0xa2,0x00,0xcf,0x86,0xe5,0x03,0x01,0xd4,0x81, + 0xd3,0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0x8f,0x85,0x00,0x05,0xff, + 0xe6,0x8e,0xa9,0x00,0x10,0x08,0x05,0xff,0xe3,0xa8,0xae,0x00,0x05,0xff,0xe6,0x91, + 0xa9,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0x91,0xbe,0x00,0x05,0xff,0xe6,0x92, + 0x9d,0x00,0x10,0x08,0x05,0xff,0xe6,0x91,0xb7,0x00,0x05,0xff,0xe3,0xa9,0xac,0x00, + 0xd2,0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0x95,0x8f,0x00,0x05,0xff,0xe6,0x95, + 0xac,0x00,0x10,0x09,0x05,0xff,0xf0,0xa3,0x80,0x8a,0x00,0x05,0xff,0xe6,0x97,0xa3, + 0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0x9b,0xb8,0x00,0x05,0xff,0xe6,0x99,0x89, + 0x00,0x10,0x08,0x05,0xff,0xe3,0xac,0x99,0x00,0x05,0xff,0xe6,0x9a,0x91,0x00,0xd3, + 0x40,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe3,0xac,0x88,0x00,0x05,0xff,0xe3, + 0xab,0xa4,0x00,0x10,0x08,0x05,0xff,0xe5,0x86,0x92,0x00,0x05,0xff,0xe5,0x86,0x95, + 0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0x9c,0x80,0x00,0x05,0xff,0xe6,0x9a,0x9c, + 0x00,0x10,0x08,0x05,0xff,0xe8,0x82,0xad,0x00,0x05,0xff,0xe4,0x8f,0x99,0x00,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0x9c,0x97,0x00,0x05,0xff,0xe6,0x9c,0x9b, + 0x00,0x10,0x08,0x05,0xff,0xe6,0x9c,0xa1,0x00,0x05,0xff,0xe6,0x9d,0x9e,0x00,0xd1, + 0x11,0x10,0x08,0x05,0xff,0xe6,0x9d,0x93,0x00,0x05,0xff,0xf0,0xa3,0x8f,0x83,0x00, + 0x10,0x08,0x05,0xff,0xe3,0xad,0x89,0x00,0x05,0xff,0xe6,0x9f,0xba,0x00,0xd4,0x82, + 0xd3,0x41,0xd2,0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0x9e,0x85,0x00,0x05,0xff, + 0xe6,0xa1,0x92,0x00,0x10,0x08,0x05,0xff,0xe6,0xa2,0x85,0x00,0x05,0xff,0xf0,0xa3, + 0x91,0xad,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0xa2,0x8e,0x00,0x05,0xff,0xe6, + 0xa0,0x9f,0x00,0x10,0x08,0x05,0xff,0xe6,0xa4,0x94,0x00,0x05,0xff,0xe3,0xae,0x9d, + 0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0xa5,0x82,0x00,0x05,0xff,0xe6, + 0xa6,0xa3,0x00,0x10,0x08,0x05,0xff,0xe6,0xa7,0xaa,0x00,0x05,0xff,0xe6,0xaa,0xa8, + 0x00,0xd1,0x11,0x10,0x09,0x05,0xff,0xf0,0xa3,0x9a,0xa3,0x00,0x05,0xff,0xe6,0xab, + 0x9b,0x00,0x10,0x08,0x05,0xff,0xe3,0xb0,0x98,0x00,0x05,0xff,0xe6,0xac,0xa1,0x00, + 0xd3,0x42,0xd2,0x21,0xd1,0x11,0x10,0x09,0x05,0xff,0xf0,0xa3,0xa2,0xa7,0x00,0x05, + 0xff,0xe6,0xad,0x94,0x00,0x10,0x08,0x05,0xff,0xe3,0xb1,0x8e,0x00,0x05,0xff,0xe6, + 0xad,0xb2,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0xae,0x9f,0x00,0x05,0xff,0xe6, + 0xae,0xba,0x00,0x10,0x08,0x05,0xff,0xe6,0xae,0xbb,0x00,0x05,0xff,0xf0,0xa3,0xaa, + 0x8d,0x00,0xd2,0x23,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa1,0xb4,0x8b,0x00,0x05, + 0xff,0xf0,0xa3,0xab,0xba,0x00,0x10,0x08,0x05,0xff,0xe6,0xb1,0x8e,0x00,0x05,0xff, + 0xf0,0xa3,0xb2,0xbc,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0xb2,0xbf,0x00,0x05, + 0xff,0xe6,0xb3,0x8d,0x00,0x10,0x08,0x05,0xff,0xe6,0xb1,0xa7,0x00,0x05,0xff,0xe6, + 0xb4,0x96,0x00,0xe1,0x1d,0x04,0xe0,0x0c,0x02,0xcf,0x86,0xe5,0x08,0x01,0xd4,0x82, + 0xd3,0x41,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0xb4,0xbe,0x00,0x05,0xff, + 0xe6,0xb5,0xb7,0x00,0x10,0x08,0x05,0xff,0xe6,0xb5,0x81,0x00,0x05,0xff,0xe6,0xb5, + 0xa9,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0xb5,0xb8,0x00,0x05,0xff,0xe6,0xb6, + 0x85,0x00,0x10,0x09,0x05,0xff,0xf0,0xa3,0xb4,0x9e,0x00,0x05,0xff,0xe6,0xb4,0xb4, + 0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe6,0xb8,0xaf,0x00,0x05,0xff,0xe6, + 0xb9,0xae,0x00,0x10,0x08,0x05,0xff,0xe3,0xb4,0xb3,0x00,0x05,0xff,0xe6,0xbb,0x8b, + 0x00,0xd1,0x11,0x10,0x08,0x05,0xff,0xe6,0xbb,0x87,0x00,0x05,0xff,0xf0,0xa3,0xbb, + 0x91,0x00,0x10,0x08,0x05,0xff,0xe6,0xb7,0xb9,0x00,0x05,0xff,0xe6,0xbd,0xae,0x00, + 0xd3,0x42,0xd2,0x22,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa3,0xbd,0x9e,0x00,0x05, + 0xff,0xf0,0xa3,0xbe,0x8e,0x00,0x10,0x08,0x05,0xff,0xe6,0xbf,0x86,0x00,0x05,0xff, + 0xe7,0x80,0xb9,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe7,0x80,0x9e,0x00,0x05,0xff, + 0xe7,0x80,0x9b,0x00,0x10,0x08,0x05,0xff,0xe3,0xb6,0x96,0x00,0x05,0xff,0xe7,0x81, + 0x8a,0x00,0xd2,0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe7,0x81,0xbd,0x00,0x05,0xff, + 0xe7,0x81,0xb7,0x00,0x10,0x08,0x05,0xff,0xe7,0x82,0xad,0x00,0x05,0xff,0xf0,0xa0, + 0x94,0xa5,0x00,0xd1,0x11,0x10,0x08,0x05,0xff,0xe7,0x85,0x85,0x00,0x05,0xff,0xf0, + 0xa4,0x89,0xa3,0x00,0x10,0x08,0x05,0xff,0xe7,0x86,0x9c,0x00,0x05,0xff,0xf0,0xa4, + 0x8e,0xab,0x00,0xd4,0x7b,0xd3,0x43,0xd2,0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe7, + 0x88,0xa8,0x00,0x05,0xff,0xe7,0x88,0xb5,0x00,0x10,0x08,0x05,0xff,0xe7,0x89,0x90, + 0x00,0x05,0xff,0xf0,0xa4,0x98,0x88,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe7,0x8a, + 0x80,0x00,0x05,0xff,0xe7,0x8a,0x95,0x00,0x10,0x09,0x05,0xff,0xf0,0xa4,0x9c,0xb5, + 0x00,0x05,0xff,0xf0,0xa4,0xa0,0x94,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff, + 0xe7,0x8d,0xba,0x00,0x05,0xff,0xe7,0x8e,0x8b,0x00,0x10,0x08,0x05,0xff,0xe3,0xba, + 0xac,0x00,0x05,0xff,0xe7,0x8e,0xa5,0x00,0x51,0x08,0x05,0xff,0xe3,0xba,0xb8,0x00, + 0x10,0x08,0x05,0xff,0xe7,0x91,0x87,0x00,0x05,0xff,0xe7,0x91,0x9c,0x00,0xd3,0x42, + 0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe7,0x91,0xb1,0x00,0x05,0xff,0xe7,0x92, + 0x85,0x00,0x10,0x08,0x05,0xff,0xe7,0x93,0x8a,0x00,0x05,0xff,0xe3,0xbc,0x9b,0x00, + 0xd1,0x11,0x10,0x08,0x05,0xff,0xe7,0x94,0xa4,0x00,0x05,0xff,0xf0,0xa4,0xb0,0xb6, + 0x00,0x10,0x08,0x05,0xff,0xe7,0x94,0xbe,0x00,0x05,0xff,0xf0,0xa4,0xb2,0x92,0x00, + 0xd2,0x22,0xd1,0x11,0x10,0x08,0x05,0xff,0xe7,0x95,0xb0,0x00,0x05,0xff,0xf0,0xa2, + 0x86,0x9f,0x00,0x10,0x08,0x05,0xff,0xe7,0x98,0x90,0x00,0x05,0xff,0xf0,0xa4,0xbe, + 0xa1,0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa4,0xbe,0xb8,0x00,0x05,0xff,0xf0, + 0xa5,0x81,0x84,0x00,0x10,0x08,0x05,0xff,0xe3,0xbf,0xbc,0x00,0x05,0xff,0xe4,0x80, + 0x88,0x00,0xcf,0x86,0xe5,0x04,0x01,0xd4,0x7d,0xd3,0x3c,0xd2,0x23,0xd1,0x11,0x10, + 0x08,0x05,0xff,0xe7,0x9b,0xb4,0x00,0x05,0xff,0xf0,0xa5,0x83,0xb3,0x00,0x10,0x09, + 0x05,0xff,0xf0,0xa5,0x83,0xb2,0x00,0x05,0xff,0xf0,0xa5,0x84,0x99,0x00,0x91,0x11, + 0x10,0x09,0x05,0xff,0xf0,0xa5,0x84,0xb3,0x00,0x05,0xff,0xe7,0x9c,0x9e,0x00,0x05, + 0xff,0xe7,0x9c,0x9f,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe7,0x9d,0x8a, + 0x00,0x05,0xff,0xe4,0x80,0xb9,0x00,0x10,0x08,0x05,0xff,0xe7,0x9e,0x8b,0x00,0x05, + 0xff,0xe4,0x81,0x86,0x00,0xd1,0x11,0x10,0x08,0x05,0xff,0xe4,0x82,0x96,0x00,0x05, + 0xff,0xf0,0xa5,0x90,0x9d,0x00,0x10,0x08,0x05,0xff,0xe7,0xa1,0x8e,0x00,0x05,0xff, + 0xe7,0xa2,0x8c,0x00,0xd3,0x43,0xd2,0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe7,0xa3, + 0x8c,0x00,0x05,0xff,0xe4,0x83,0xa3,0x00,0x10,0x09,0x05,0xff,0xf0,0xa5,0x98,0xa6, + 0x00,0x05,0xff,0xe7,0xa5,0x96,0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa5,0x9a, + 0x9a,0x00,0x05,0xff,0xf0,0xa5,0x9b,0x85,0x00,0x10,0x08,0x05,0xff,0xe7,0xa6,0x8f, + 0x00,0x05,0xff,0xe7,0xa7,0xab,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe4, + 0x84,0xaf,0x00,0x05,0xff,0xe7,0xa9,0x80,0x00,0x10,0x08,0x05,0xff,0xe7,0xa9,0x8a, + 0x00,0x05,0xff,0xe7,0xa9,0x8f,0x00,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa5,0xa5, + 0xbc,0x00,0x05,0xff,0xf0,0xa5,0xaa,0xa7,0x00,0x10,0x09,0x05,0xff,0xf0,0xa5,0xaa, + 0xa7,0x00,0x05,0xff,0xe7,0xab,0xae,0x00,0xd4,0x83,0xd3,0x42,0xd2,0x21,0xd1,0x11, + 0x10,0x08,0x05,0xff,0xe4,0x88,0x82,0x00,0x05,0xff,0xf0,0xa5,0xae,0xab,0x00,0x10, + 0x08,0x05,0xff,0xe7,0xaf,0x86,0x00,0x05,0xff,0xe7,0xaf,0x89,0x00,0xd1,0x11,0x10, + 0x08,0x05,0xff,0xe4,0x88,0xa7,0x00,0x05,0xff,0xf0,0xa5,0xb2,0x80,0x00,0x10,0x08, + 0x05,0xff,0xe7,0xb3,0x92,0x00,0x05,0xff,0xe4,0x8a,0xa0,0x00,0xd2,0x21,0xd1,0x10, + 0x10,0x08,0x05,0xff,0xe7,0xb3,0xa8,0x00,0x05,0xff,0xe7,0xb3,0xa3,0x00,0x10,0x08, + 0x05,0xff,0xe7,0xb4,0x80,0x00,0x05,0xff,0xf0,0xa5,0xbe,0x86,0x00,0xd1,0x10,0x10, + 0x08,0x05,0xff,0xe7,0xb5,0xa3,0x00,0x05,0xff,0xe4,0x8c,0x81,0x00,0x10,0x08,0x05, + 0xff,0xe7,0xb7,0x87,0x00,0x05,0xff,0xe7,0xb8,0x82,0x00,0xd3,0x44,0xd2,0x22,0xd1, + 0x10,0x10,0x08,0x05,0xff,0xe7,0xb9,0x85,0x00,0x05,0xff,0xe4,0x8c,0xb4,0x00,0x10, + 0x09,0x05,0xff,0xf0,0xa6,0x88,0xa8,0x00,0x05,0xff,0xf0,0xa6,0x89,0x87,0x00,0xd1, + 0x11,0x10,0x08,0x05,0xff,0xe4,0x8d,0x99,0x00,0x05,0xff,0xf0,0xa6,0x8b,0x99,0x00, + 0x10,0x08,0x05,0xff,0xe7,0xbd,0xba,0x00,0x05,0xff,0xf0,0xa6,0x8c,0xbe,0x00,0xd2, + 0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe7,0xbe,0x95,0x00,0x05,0xff,0xe7,0xbf,0xba, + 0x00,0x10,0x08,0x05,0xff,0xe8,0x80,0x85,0x00,0x05,0xff,0xf0,0xa6,0x93,0x9a,0x00, + 0xd1,0x11,0x10,0x09,0x05,0xff,0xf0,0xa6,0x94,0xa3,0x00,0x05,0xff,0xe8,0x81,0xa0, + 0x00,0x10,0x09,0x05,0xff,0xf0,0xa6,0x96,0xa8,0x00,0x05,0xff,0xe8,0x81,0xb0,0x00, + 0xe0,0x11,0x02,0xcf,0x86,0xe5,0x07,0x01,0xd4,0x85,0xd3,0x42,0xd2,0x21,0xd1,0x11, + 0x10,0x09,0x05,0xff,0xf0,0xa3,0x8d,0x9f,0x00,0x05,0xff,0xe4,0x8f,0x95,0x00,0x10, + 0x08,0x05,0xff,0xe8,0x82,0xb2,0x00,0x05,0xff,0xe8,0x84,0x83,0x00,0xd1,0x10,0x10, + 0x08,0x05,0xff,0xe4,0x90,0x8b,0x00,0x05,0xff,0xe8,0x84,0xbe,0x00,0x10,0x08,0x05, + 0xff,0xe5,0xaa,0xb5,0x00,0x05,0xff,0xf0,0xa6,0x9e,0xa7,0x00,0xd2,0x23,0xd1,0x12, + 0x10,0x09,0x05,0xff,0xf0,0xa6,0x9e,0xb5,0x00,0x05,0xff,0xf0,0xa3,0x8e,0x93,0x00, + 0x10,0x09,0x05,0xff,0xf0,0xa3,0x8e,0x9c,0x00,0x05,0xff,0xe8,0x88,0x81,0x00,0xd1, + 0x10,0x10,0x08,0x05,0xff,0xe8,0x88,0x84,0x00,0x05,0xff,0xe8,0xbe,0x9e,0x00,0x10, + 0x08,0x05,0xff,0xe4,0x91,0xab,0x00,0x05,0xff,0xe8,0x8a,0x91,0x00,0xd3,0x41,0xd2, + 0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe8,0x8a,0x8b,0x00,0x05,0xff,0xe8,0x8a,0x9d, + 0x00,0x10,0x08,0x05,0xff,0xe5,0x8a,0xb3,0x00,0x05,0xff,0xe8,0x8a,0xb1,0x00,0xd1, + 0x10,0x10,0x08,0x05,0xff,0xe8,0x8a,0xb3,0x00,0x05,0xff,0xe8,0x8a,0xbd,0x00,0x10, + 0x08,0x05,0xff,0xe8,0x8b,0xa6,0x00,0x05,0xff,0xf0,0xa6,0xac,0xbc,0x00,0xd2,0x20, + 0xd1,0x10,0x10,0x08,0x05,0xff,0xe8,0x8b,0xa5,0x00,0x05,0xff,0xe8,0x8c,0x9d,0x00, + 0x10,0x08,0x05,0xff,0xe8,0x8d,0xa3,0x00,0x05,0xff,0xe8,0x8e,0xad,0x00,0xd1,0x10, + 0x10,0x08,0x05,0xff,0xe8,0x8c,0xa3,0x00,0x05,0xff,0xe8,0x8e,0xbd,0x00,0x10,0x08, + 0x05,0xff,0xe8,0x8f,0xa7,0x00,0x05,0xff,0xe8,0x91,0x97,0x00,0xd4,0x85,0xd3,0x43, + 0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe8,0x8d,0x93,0x00,0x05,0xff,0xe8,0x8f, + 0x8a,0x00,0x10,0x08,0x05,0xff,0xe8,0x8f,0x8c,0x00,0x05,0xff,0xe8,0x8f,0x9c,0x00, + 0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa6,0xb0,0xb6,0x00,0x05,0xff,0xf0,0xa6,0xb5, + 0xab,0x00,0x10,0x09,0x05,0xff,0xf0,0xa6,0xb3,0x95,0x00,0x05,0xff,0xe4,0x94,0xab, + 0x00,0xd2,0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe8,0x93,0xb1,0x00,0x05,0xff,0xe8, + 0x93,0xb3,0x00,0x10,0x08,0x05,0xff,0xe8,0x94,0x96,0x00,0x05,0xff,0xf0,0xa7,0x8f, + 0x8a,0x00,0xd1,0x11,0x10,0x08,0x05,0xff,0xe8,0x95,0xa4,0x00,0x05,0xff,0xf0,0xa6, + 0xbc,0xac,0x00,0x10,0x08,0x05,0xff,0xe4,0x95,0x9d,0x00,0x05,0xff,0xe4,0x95,0xa1, + 0x00,0xd3,0x42,0xd2,0x22,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa6,0xbe,0xb1,0x00, + 0x05,0xff,0xf0,0xa7,0x83,0x92,0x00,0x10,0x08,0x05,0xff,0xe4,0x95,0xab,0x00,0x05, + 0xff,0xe8,0x99,0x90,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe8,0x99,0x9c,0x00,0x05, + 0xff,0xe8,0x99,0xa7,0x00,0x10,0x08,0x05,0xff,0xe8,0x99,0xa9,0x00,0x05,0xff,0xe8, + 0x9a,0xa9,0x00,0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe8,0x9a,0x88,0x00,0x05, + 0xff,0xe8,0x9c,0x8e,0x00,0x10,0x08,0x05,0xff,0xe8,0x9b,0xa2,0x00,0x05,0xff,0xe8, + 0x9d,0xb9,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe8,0x9c,0xa8,0x00,0x05,0xff,0xe8, + 0x9d,0xab,0x00,0x10,0x08,0x05,0xff,0xe8,0x9e,0x86,0x00,0x05,0xff,0xe4,0x97,0x97, + 0x00,0xcf,0x86,0xe5,0x08,0x01,0xd4,0x83,0xd3,0x41,0xd2,0x20,0xd1,0x10,0x10,0x08, + 0x05,0xff,0xe8,0x9f,0xa1,0x00,0x05,0xff,0xe8,0xa0,0x81,0x00,0x10,0x08,0x05,0xff, + 0xe4,0x97,0xb9,0x00,0x05,0xff,0xe8,0xa1,0xa0,0x00,0xd1,0x11,0x10,0x08,0x05,0xff, + 0xe8,0xa1,0xa3,0x00,0x05,0xff,0xf0,0xa7,0x99,0xa7,0x00,0x10,0x08,0x05,0xff,0xe8, + 0xa3,0x97,0x00,0x05,0xff,0xe8,0xa3,0x9e,0x00,0xd2,0x21,0xd1,0x10,0x10,0x08,0x05, + 0xff,0xe4,0x98,0xb5,0x00,0x05,0xff,0xe8,0xa3,0xba,0x00,0x10,0x08,0x05,0xff,0xe3, + 0x92,0xbb,0x00,0x05,0xff,0xf0,0xa7,0xa2,0xae,0x00,0xd1,0x11,0x10,0x09,0x05,0xff, + 0xf0,0xa7,0xa5,0xa6,0x00,0x05,0xff,0xe4,0x9a,0xbe,0x00,0x10,0x08,0x05,0xff,0xe4, + 0x9b,0x87,0x00,0x05,0xff,0xe8,0xaa,0xa0,0x00,0xd3,0x41,0xd2,0x21,0xd1,0x10,0x10, + 0x08,0x05,0xff,0xe8,0xab,0xad,0x00,0x05,0xff,0xe8,0xae,0x8a,0x00,0x10,0x08,0x05, + 0xff,0xe8,0xb1,0x95,0x00,0x05,0xff,0xf0,0xa7,0xb2,0xa8,0x00,0xd1,0x10,0x10,0x08, + 0x05,0xff,0xe8,0xb2,0xab,0x00,0x05,0xff,0xe8,0xb3,0x81,0x00,0x10,0x08,0x05,0xff, + 0xe8,0xb4,0x9b,0x00,0x05,0xff,0xe8,0xb5,0xb7,0x00,0xd2,0x22,0xd1,0x12,0x10,0x09, + 0x05,0xff,0xf0,0xa7,0xbc,0xaf,0x00,0x05,0xff,0xf0,0xa0,0xa0,0x84,0x00,0x10,0x08, + 0x05,0xff,0xe8,0xb7,0x8b,0x00,0x05,0xff,0xe8,0xb6,0xbc,0x00,0xd1,0x11,0x10,0x08, + 0x05,0xff,0xe8,0xb7,0xb0,0x00,0x05,0xff,0xf0,0xa0,0xa3,0x9e,0x00,0x10,0x08,0x05, + 0xff,0xe8,0xbb,0x94,0x00,0x05,0xff,0xe8,0xbc,0xb8,0x00,0xd4,0x84,0xd3,0x43,0xd2, + 0x22,0xd1,0x12,0x10,0x09,0x05,0xff,0xf0,0xa8,0x97,0x92,0x00,0x05,0xff,0xf0,0xa8, + 0x97,0xad,0x00,0x10,0x08,0x05,0xff,0xe9,0x82,0x94,0x00,0x05,0xff,0xe9,0x83,0xb1, + 0x00,0xd1,0x11,0x10,0x08,0x05,0xff,0xe9,0x84,0x91,0x00,0x05,0xff,0xf0,0xa8,0x9c, + 0xae,0x00,0x10,0x08,0x05,0xff,0xe9,0x84,0x9b,0x00,0x05,0xff,0xe9,0x88,0xb8,0x00, + 0xd2,0x20,0xd1,0x10,0x10,0x08,0x05,0xff,0xe9,0x8b,0x97,0x00,0x05,0xff,0xe9,0x8b, + 0x98,0x00,0x10,0x08,0x05,0xff,0xe9,0x89,0xbc,0x00,0x05,0xff,0xe9,0x8f,0xb9,0x00, + 0xd1,0x11,0x10,0x08,0x05,0xff,0xe9,0x90,0x95,0x00,0x05,0xff,0xf0,0xa8,0xaf,0xba, + 0x00,0x10,0x08,0x05,0xff,0xe9,0x96,0x8b,0x00,0x05,0xff,0xe4,0xa6,0x95,0x00,0xd3, + 0x43,0xd2,0x21,0xd1,0x11,0x10,0x08,0x05,0xff,0xe9,0x96,0xb7,0x00,0x05,0xff,0xf0, + 0xa8,0xb5,0xb7,0x00,0x10,0x08,0x05,0xff,0xe4,0xa7,0xa6,0x00,0x05,0xff,0xe9,0x9b, + 0x83,0x00,0xd1,0x10,0x10,0x08,0x05,0xff,0xe5,0xb6,0xb2,0x00,0x05,0xff,0xe9,0x9c, + 0xa3,0x00,0x10,0x09,0x05,0xff,0xf0,0xa9,0x85,0x85,0x00,0x05,0xff,0xf0,0xa9,0x88, + 0x9a,0x00,0xd2,0x21,0xd1,0x10,0x10,0x08,0x05,0xff,0xe4,0xa9,0xae,0x00,0x05,0xff, + 0xe4,0xa9,0xb6,0x00,0x10,0x08,0x05,0xff,0xe9,0x9f,0xa0,0x00,0x05,0xff,0xf0,0xa9, + 0x90,0x8a,0x00,0x91,0x11,0x10,0x08,0x05,0xff,0xe4,0xaa,0xb2,0x00,0x05,0xff,0xf0, + 0xa9,0x92,0x96,0x00,0x05,0xff,0xe9,0xa0,0x8b,0x00,0xe2,0x10,0x01,0xe1,0x09,0x01, + 0xe0,0x02,0x01,0xcf,0x86,0x95,0xfb,0xd4,0x82,0xd3,0x41,0xd2,0x21,0xd1,0x11,0x10, + 0x08,0x05,0xff,0xe9,0xa0,0xa9,0x00,0x05,0xff,0xf0,0xa9,0x96,0xb6,0x00,0x10,0x08, + 0x05,0xff,0xe9,0xa3,0xa2,0x00,0x05,0xff,0xe4,0xac,0xb3,0x00,0xd1,0x10,0x10,0x08, + 0x05,0xff,0xe9,0xa4,0xa9,0x00,0x05,0xff,0xe9,0xa6,0xa7,0x00,0x10,0x08,0x05,0xff, + 0xe9,0xa7,0x82,0x00,0x05,0xff,0xe9,0xa7,0xbe,0x00,0xd2,0x21,0xd1,0x11,0x10,0x08, + 0x05,0xff,0xe4,0xaf,0x8e,0x00,0x05,0xff,0xf0,0xa9,0xac,0xb0,0x00,0x10,0x08,0x05, + 0xff,0xe9,0xac,0x92,0x00,0x05,0xff,0xe9,0xb1,0x80,0x00,0xd1,0x10,0x10,0x08,0x05, + 0xff,0xe9,0xb3,0xbd,0x00,0x05,0xff,0xe4,0xb3,0x8e,0x00,0x10,0x08,0x05,0xff,0xe4, + 0xb3,0xad,0x00,0x05,0xff,0xe9,0xb5,0xa7,0x00,0xd3,0x44,0xd2,0x23,0xd1,0x11,0x10, + 0x09,0x05,0xff,0xf0,0xaa,0x83,0x8e,0x00,0x05,0xff,0xe4,0xb3,0xb8,0x00,0x10,0x09, + 0x05,0xff,0xf0,0xaa,0x84,0x85,0x00,0x05,0xff,0xf0,0xaa,0x88,0x8e,0x00,0xd1,0x11, + 0x10,0x09,0x05,0xff,0xf0,0xaa,0x8a,0x91,0x00,0x05,0xff,0xe9,0xba,0xbb,0x00,0x10, + 0x08,0x05,0xff,0xe4,0xb5,0x96,0x00,0x05,0xff,0xe9,0xbb,0xb9,0x00,0xd2,0x20,0xd1, + 0x10,0x10,0x08,0x05,0xff,0xe9,0xbb,0xbe,0x00,0x05,0xff,0xe9,0xbc,0x85,0x00,0x10, + 0x08,0x05,0xff,0xe9,0xbc,0x8f,0x00,0x05,0xff,0xe9,0xbc,0x96,0x00,0x91,0x11,0x10, + 0x08,0x05,0xff,0xe9,0xbc,0xbb,0x00,0x05,0xff,0xf0,0xaa,0x98,0x80,0x00,0x00,0x00, + 0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xcf,0x06,0x00,0x00,0xd3,0x06, + 0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf,0x06,0x00,0x00, + 0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00,0x00, + 0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02,0x00,0xd3,0x08, + 0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd1,0x08, + 0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xcf,0x86, + 0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00,0x00,0xd3,0x06,0xcf,0x06, + 0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf,0x06,0x00,0x00,0xd0,0x06, + 0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00,0x00,0x53,0x04, + 0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02,0x00,0xcf,0x86,0xd5,0xc0, + 0xd4,0x60,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf,0x06, + 0x00,0x00,0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf,0x06, + 0x00,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00,0x00, + 0xd3,0x06,0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf,0x06, + 0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04, + 0x00,0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02,0x00, + 0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00, + 0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00, + 0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00,0x00,0xd3,0x06, + 0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf,0x06,0x00,0x00, + 0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00,0x00, + 0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02,0x00,0xd4,0x60, + 0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00, + 0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00, + 0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00,0x00,0xd3,0x06, + 0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf,0x06,0x00,0x00, + 0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00,0x00, + 0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02,0x00,0xd3,0x08, + 0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd1,0x08, + 0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xcf,0x86, + 0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00,0x00,0xd3,0x06,0xcf,0x06, + 0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf,0x06,0x00,0x00,0xd0,0x06, + 0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00,0x00,0x53,0x04, + 0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02,0x00,0xe0,0x83,0x01,0xcf, + 0x86,0xd5,0xc0,0xd4,0x60,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf, + 0x86,0xcf,0x06,0x00,0x00,0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf, + 0x86,0xcf,0x06,0x00,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf, + 0x06,0x00,0x00,0xd3,0x06,0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1, + 0x06,0xcf,0x06,0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00, + 0x00,0x54,0x04,0x00,0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00, + 0x00,0x02,0x00,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf, + 0x06,0x00,0x00,0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf, + 0x06,0x00,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00, + 0x00,0xd3,0x06,0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf, + 0x06,0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54, + 0x04,0x00,0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02, + 0x00,0xd4,0x60,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf, + 0x06,0x00,0x00,0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf, + 0x06,0x00,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00, + 0x00,0xd3,0x06,0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf, + 0x06,0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54, + 0x04,0x00,0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02, + 0x00,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf,0x06,0x00, + 0x00,0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x00, + 0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00,0x00,0xd3, + 0x06,0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf,0x06,0x00, + 0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00, + 0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02,0x00,0xcf, + 0x86,0xd5,0xc0,0xd4,0x60,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf, + 0x86,0xcf,0x06,0x00,0x00,0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf, + 0x86,0xcf,0x06,0x00,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf, + 0x06,0x00,0x00,0xd3,0x06,0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1, + 0x06,0xcf,0x06,0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00, + 0x00,0x54,0x04,0x00,0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00, + 0x00,0x02,0x00,0xd3,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf, + 0x06,0x00,0x00,0xd1,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf, + 0x06,0x00,0x00,0xcf,0x86,0xd5,0x06,0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00, + 0x00,0xd3,0x06,0xcf,0x06,0x00,0x00,0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf, + 0x06,0x00,0x00,0xd0,0x06,0xcf,0x06,0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54, + 0x04,0x00,0x00,0x53,0x04,0x00,0x00,0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02, + 0x00,0xd4,0xd9,0xd3,0x81,0xd2,0x79,0xd1,0x71,0xd0,0x69,0xcf,0x86,0xd5,0x60,0xd4, + 0x59,0xd3,0x52,0xd2,0x33,0xd1,0x2c,0xd0,0x25,0xcf,0x86,0x95,0x1e,0x94,0x19,0x93, + 0x14,0x92,0x0f,0x91,0x0a,0x10,0x05,0x00,0xff,0x00,0x05,0xff,0x00,0x00,0xff,0x00, + 0x00,0xff,0x00,0x00,0xff,0x00,0x00,0xff,0x00,0x05,0xff,0x00,0xcf,0x06,0x05,0xff, + 0x00,0xcf,0x06,0x00,0xff,0x00,0xd1,0x07,0xcf,0x06,0x07,0xff,0x00,0xd0,0x07,0xcf, + 0x06,0x07,0xff,0x00,0xcf,0x86,0x55,0x05,0x07,0xff,0x00,0x14,0x05,0x07,0xff,0x00, + 0x00,0xff,0x00,0xcf,0x06,0x00,0xff,0x00,0xcf,0x06,0x00,0xff,0x00,0xcf,0x06,0x00, + 0xff,0x00,0xcf,0x86,0xcf,0x06,0x00,0x00,0xcf,0x86,0xcf,0x06,0x00,0x00,0xcf,0x86, + 0xcf,0x06,0x00,0x00,0xd2,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xd1,0x08,0xcf,0x86, + 0xcf,0x06,0x00,0x00,0xd0,0x08,0xcf,0x86,0xcf,0x06,0x00,0x00,0xcf,0x86,0xd5,0x06, + 0xcf,0x06,0x00,0x00,0xd4,0x06,0xcf,0x06,0x00,0x00,0xd3,0x06,0xcf,0x06,0x00,0x00, + 0xd2,0x06,0xcf,0x06,0x00,0x00,0xd1,0x06,0xcf,0x06,0x00,0x00,0xd0,0x06,0xcf,0x06, + 0x00,0x00,0xcf,0x86,0x55,0x04,0x00,0x00,0x54,0x04,0x00,0x00,0x53,0x04,0x00,0x00, + 0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02,0x00,0xcf,0x86,0xcf,0x06,0x02,0x00, + 0x81,0x80,0xcf,0x86,0x85,0x84,0xcf,0x86,0xcf,0x06,0x02,0x00,0x00,0x00,0x00,0x00 +}; + +const struct utf8data_table utf8_data_table = { + .utf8agetab = utf8agetab, + .utf8agetab_size = ARRAY_SIZE(utf8agetab), + + .utf8nfdicfdata = utf8nfdicfdata, + .utf8nfdicfdata_size = ARRAY_SIZE(utf8nfdicfdata), + + .utf8nfdidata = utf8nfdidata, + .utf8nfdidata_size = ARRAY_SIZE(utf8nfdidata), + + .utf8data = utf8data, +}; +EXPORT_SYMBOL_GPL(utf8_data_table); +MODULE_DESCRIPTION("UTF8 data table"); +MODULE_LICENSE("GPL v2"); diff --git a/fs/unicode/utf8n.h b/fs/unicode/utf8n.h new file mode 100644 index 000000000..fc703aa4b --- /dev/null +++ b/fs/unicode/utf8n.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2014 SGI. + * All rights reserved. + */ + +#ifndef UTF8NORM_H +#define UTF8NORM_H + +#include +#include +#include +#include +#include + +int utf8version_is_supported(const struct unicode_map *um, unsigned int version); + +/* + * Determine the length of the normalized from of the string, + * excluding any terminating NULL byte. + * Returns 0 if only ignorable code points are present. + * Returns -1 if the input is not valid UTF-8. + */ +ssize_t utf8nlen(const struct unicode_map *um, enum utf8_normalization n, + const char *s, size_t len); + +/* Needed in struct utf8cursor below. */ +#define UTF8HANGULLEAF (12) + +/* + * Cursor structure used by the normalizer. + */ +struct utf8cursor { + const struct unicode_map *um; + enum utf8_normalization n; + const char *s; + const char *p; + const char *ss; + const char *sp; + unsigned int len; + unsigned int slen; + short int ccc; + short int nccc; + unsigned char hangul[UTF8HANGULLEAF]; +}; + +/* + * Initialize a utf8cursor to normalize a string. + * Returns 0 on success. + * Returns -1 on failure. + */ +int utf8ncursor(struct utf8cursor *u8c, const struct unicode_map *um, + enum utf8_normalization n, const char *s, size_t len); + +/* + * Get the next byte in the normalization. + * Returns a value > 0 && < 256 on success. + * Returns 0 when the end of the normalization is reached. + * Returns -1 if the string being normalized is not valid UTF-8. + */ +extern int utf8byte(struct utf8cursor *u8c); + +struct utf8data { + unsigned int maxage; + unsigned int offset; +}; + +struct utf8data_table { + const unsigned int *utf8agetab; + int utf8agetab_size; + + const struct utf8data *utf8nfdicfdata; + int utf8nfdicfdata_size; + + const struct utf8data *utf8nfdidata; + int utf8nfdidata_size; + + const unsigned char *utf8data; +}; + +extern const struct utf8data_table utf8_data_table; + +#endif /* UTF8NORM_H */ diff --git a/fs/utimes.c b/fs/utimes.c new file mode 100644 index 000000000..e22664e41 --- /dev/null +++ b/fs/utimes.c @@ -0,0 +1,294 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +static bool nsec_valid(long nsec) +{ + if (nsec == UTIME_OMIT || nsec == UTIME_NOW) + return true; + + return nsec >= 0 && nsec <= 999999999; +} + +int vfs_utimes(const struct path *path, struct timespec64 *times) +{ + int error; + struct iattr newattrs; + struct inode *inode = path->dentry->d_inode; + struct delegated_inode delegated_inode = { }; + + if (times) { + if (!nsec_valid(times[0].tv_nsec) || + !nsec_valid(times[1].tv_nsec)) + return -EINVAL; + if (times[0].tv_nsec == UTIME_NOW && + times[1].tv_nsec == UTIME_NOW) + times = NULL; + } + + error = mnt_want_write(path->mnt); + if (error) + goto out; + + newattrs.ia_valid = ATTR_CTIME | ATTR_MTIME | ATTR_ATIME; + if (times) { + if (times[0].tv_nsec == UTIME_OMIT) + newattrs.ia_valid &= ~ATTR_ATIME; + else if (times[0].tv_nsec != UTIME_NOW) { + newattrs.ia_atime = times[0]; + newattrs.ia_valid |= ATTR_ATIME_SET; + } + + if (times[1].tv_nsec == UTIME_OMIT) + newattrs.ia_valid &= ~ATTR_MTIME; + else if (times[1].tv_nsec != UTIME_NOW) { + newattrs.ia_mtime = times[1]; + newattrs.ia_valid |= ATTR_MTIME_SET; + } + /* + * Tell setattr_prepare(), that this is an explicit time + * update, even if neither ATTR_ATIME_SET nor ATTR_MTIME_SET + * were used. + */ + newattrs.ia_valid |= ATTR_TIMES_SET; + } else { + newattrs.ia_valid |= ATTR_TOUCH; + } +retry_deleg: + inode_lock(inode); + error = notify_change(mnt_idmap(path->mnt), path->dentry, &newattrs, + &delegated_inode); + inode_unlock(inode); + if (is_delegated(&delegated_inode)) { + error = break_deleg_wait(&delegated_inode); + if (!error) + goto retry_deleg; + } + + mnt_drop_write(path->mnt); +out: + return error; +} +EXPORT_SYMBOL_GPL(vfs_utimes); + +static int do_utimes_path(int dfd, const char __user *filename, + struct timespec64 *times, int flags) +{ + struct path path; + int lookup_flags = 0, error; + + if (flags & ~(AT_SYMLINK_NOFOLLOW | AT_EMPTY_PATH)) + return -EINVAL; + + if (!(flags & AT_SYMLINK_NOFOLLOW)) + lookup_flags |= LOOKUP_FOLLOW; + + CLASS(filename_uflags, name)(filename, flags); +retry: + error = filename_lookup(dfd, name, lookup_flags, &path, NULL); + if (error) + return error; + error = vfs_utimes(&path, times); + path_put(&path); + if (retry_estale(error, lookup_flags)) { + lookup_flags |= LOOKUP_REVAL; + goto retry; + } + return error; +} + +static int do_utimes_fd(int fd, struct timespec64 *times, int flags) +{ + if (flags) + return -EINVAL; + + CLASS(fd, f)(fd); + if (fd_empty(f)) + return -EBADF; + return vfs_utimes(&fd_file(f)->f_path, times); +} + +/* + * do_utimes - change times on filename or file descriptor + * @dfd: open file descriptor, -1 or AT_FDCWD + * @filename: path name or NULL + * @times: new times or NULL + * @flags: zero or more flags (only AT_SYMLINK_NOFOLLOW for the moment) + * + * If filename is NULL and dfd refers to an open file, then operate on + * the file. Otherwise look up filename, possibly using dfd as a + * starting point. + * + * If times==NULL, set access and modification to current time, + * must be owner or have write permission. + * Else, update from *times, must be owner or super user. + */ +long do_utimes(int dfd, const char __user *filename, struct timespec64 *times, + int flags) +{ + if (filename == NULL && dfd != AT_FDCWD) + return do_utimes_fd(dfd, times, flags); + return do_utimes_path(dfd, filename, times, flags); +} + +SYSCALL_DEFINE4(utimensat, int, dfd, const char __user *, filename, + struct __kernel_timespec __user *, utimes, int, flags) +{ + struct timespec64 tstimes[2]; + + if (utimes) { + if ((get_timespec64(&tstimes[0], &utimes[0]) || + get_timespec64(&tstimes[1], &utimes[1]))) + return -EFAULT; + + /* Nothing to do, we must not even check the path. */ + if (tstimes[0].tv_nsec == UTIME_OMIT && + tstimes[1].tv_nsec == UTIME_OMIT) + return 0; + } + + return do_utimes(dfd, filename, utimes ? tstimes : NULL, flags); +} + +#ifdef __ARCH_WANT_SYS_UTIME +/* + * futimesat(), utimes() and utime() are older versions of utimensat() + * that are provided for compatibility with traditional C libraries. + * On modern architectures, we always use libc wrappers around + * utimensat() instead. + */ +static long do_futimesat(int dfd, const char __user *filename, + struct __kernel_old_timeval __user *utimes) +{ + struct __kernel_old_timeval times[2]; + struct timespec64 tstimes[2]; + + if (utimes) { + if (copy_from_user(×, utimes, sizeof(times))) + return -EFAULT; + + /* This test is needed to catch all invalid values. If we + would test only in do_utimes we would miss those invalid + values truncated by the multiplication with 1000. Note + that we also catch UTIME_{NOW,OMIT} here which are only + valid for utimensat. */ + if (times[0].tv_usec >= 1000000 || times[0].tv_usec < 0 || + times[1].tv_usec >= 1000000 || times[1].tv_usec < 0) + return -EINVAL; + + tstimes[0].tv_sec = times[0].tv_sec; + tstimes[0].tv_nsec = 1000 * times[0].tv_usec; + tstimes[1].tv_sec = times[1].tv_sec; + tstimes[1].tv_nsec = 1000 * times[1].tv_usec; + } + + return do_utimes(dfd, filename, utimes ? tstimes : NULL, 0); +} + + +SYSCALL_DEFINE3(futimesat, int, dfd, const char __user *, filename, + struct __kernel_old_timeval __user *, utimes) +{ + return do_futimesat(dfd, filename, utimes); +} + +SYSCALL_DEFINE2(utimes, char __user *, filename, + struct __kernel_old_timeval __user *, utimes) +{ + return do_futimesat(AT_FDCWD, filename, utimes); +} + +SYSCALL_DEFINE2(utime, char __user *, filename, struct utimbuf __user *, times) +{ + struct timespec64 tv[2]; + + if (times) { + if (get_user(tv[0].tv_sec, ×->actime) || + get_user(tv[1].tv_sec, ×->modtime)) + return -EFAULT; + tv[0].tv_nsec = 0; + tv[1].tv_nsec = 0; + } + return do_utimes(AT_FDCWD, filename, times ? tv : NULL, 0); +} +#endif + +#ifdef CONFIG_COMPAT_32BIT_TIME +/* + * Not all architectures have sys_utime, so implement this in terms + * of sys_utimes. + */ +#ifdef __ARCH_WANT_SYS_UTIME32 +SYSCALL_DEFINE2(utime32, const char __user *, filename, + struct old_utimbuf32 __user *, t) +{ + struct timespec64 tv[2]; + + if (t) { + if (get_user(tv[0].tv_sec, &t->actime) || + get_user(tv[1].tv_sec, &t->modtime)) + return -EFAULT; + tv[0].tv_nsec = 0; + tv[1].tv_nsec = 0; + } + return do_utimes(AT_FDCWD, filename, t ? tv : NULL, 0); +} +#endif + +SYSCALL_DEFINE4(utimensat_time32, unsigned int, dfd, const char __user *, filename, struct old_timespec32 __user *, t, int, flags) +{ + struct timespec64 tv[2]; + + if (t) { + if (get_old_timespec32(&tv[0], &t[0]) || + get_old_timespec32(&tv[1], &t[1])) + return -EFAULT; + + if (tv[0].tv_nsec == UTIME_OMIT && tv[1].tv_nsec == UTIME_OMIT) + return 0; + } + return do_utimes(dfd, filename, t ? tv : NULL, flags); +} + +#ifdef __ARCH_WANT_SYS_UTIME32 +static long do_compat_futimesat(unsigned int dfd, const char __user *filename, + struct old_timeval32 __user *t) +{ + struct timespec64 tv[2]; + + if (t) { + if (get_user(tv[0].tv_sec, &t[0].tv_sec) || + get_user(tv[0].tv_nsec, &t[0].tv_usec) || + get_user(tv[1].tv_sec, &t[1].tv_sec) || + get_user(tv[1].tv_nsec, &t[1].tv_usec)) + return -EFAULT; + if (tv[0].tv_nsec >= 1000000 || tv[0].tv_nsec < 0 || + tv[1].tv_nsec >= 1000000 || tv[1].tv_nsec < 0) + return -EINVAL; + tv[0].tv_nsec *= 1000; + tv[1].tv_nsec *= 1000; + } + return do_utimes(dfd, filename, t ? tv : NULL, 0); +} + +SYSCALL_DEFINE3(futimesat_time32, unsigned int, dfd, + const char __user *, filename, + struct old_timeval32 __user *, t) +{ + return do_compat_futimesat(dfd, filename, t); +} + +SYSCALL_DEFINE2(utimes_time32, const char __user *, filename, struct old_timeval32 __user *, t) +{ + return do_compat_futimesat(AT_FDCWD, filename, t); +} +#endif +#endif diff --git a/fs/vboxsf/Kconfig b/fs/vboxsf/Kconfig new file mode 100644 index 000000000..d4694026d --- /dev/null +++ b/fs/vboxsf/Kconfig @@ -0,0 +1,10 @@ +config VBOXSF_FS + tristate "VirtualBox guest shared folder (vboxsf) support" + depends on (ARM64 || X86) && VBOXGUEST + select NLS + help + VirtualBox hosts can share folders with guests, this driver + implements the Linux-guest side of this allowing folders exported + by the host to be mounted under Linux. + + If you want to use shared folders in VirtualBox guests, answer Y or M. diff --git a/fs/vboxsf/Makefile b/fs/vboxsf/Makefile new file mode 100644 index 000000000..9e4328e79 --- /dev/null +++ b/fs/vboxsf/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: MIT + +obj-$(CONFIG_VBOXSF_FS) += vboxsf.o + +vboxsf-y := dir.o file.o utils.o vboxsf_wrappers.o super.o diff --git a/fs/vboxsf/dir.c b/fs/vboxsf/dir.c new file mode 100644 index 000000000..0b9eab157 --- /dev/null +++ b/fs/vboxsf/dir.c @@ -0,0 +1,481 @@ +// SPDX-License-Identifier: MIT +/* + * VirtualBox Guest Shared Folders support: Directory inode and file operations + * + * Copyright (C) 2006-2018 Oracle Corporation + */ + +#include +#include +#include "vfsmod.h" + +static int vboxsf_dir_open(struct inode *inode, struct file *file) +{ + struct vboxsf_sbi *sbi = VBOXSF_SBI(inode->i_sb); + struct shfl_createparms params = {}; + struct vboxsf_dir_info *sf_d; + int err; + + sf_d = vboxsf_dir_info_alloc(); + if (!sf_d) + return -ENOMEM; + + params.handle = SHFL_HANDLE_NIL; + params.create_flags = SHFL_CF_DIRECTORY | SHFL_CF_ACT_OPEN_IF_EXISTS | + SHFL_CF_ACT_FAIL_IF_NEW | SHFL_CF_ACCESS_READ; + + err = vboxsf_create_at_dentry(file_dentry(file), ¶ms); + if (err) + goto err_free_dir_info; + + if (params.result != SHFL_FILE_EXISTS) { + err = -ENOENT; + goto err_close; + } + + err = vboxsf_dir_read_all(sbi, sf_d, params.handle); + if (err) + goto err_close; + + vboxsf_close(sbi->root, params.handle); + file->private_data = sf_d; + return 0; + +err_close: + vboxsf_close(sbi->root, params.handle); +err_free_dir_info: + vboxsf_dir_info_free(sf_d); + return err; +} + +static int vboxsf_dir_release(struct inode *inode, struct file *file) +{ + if (file->private_data) + vboxsf_dir_info_free(file->private_data); + + return 0; +} + +static unsigned int vboxsf_get_d_type(u32 mode) +{ + unsigned int d_type; + + switch (mode & SHFL_TYPE_MASK) { + case SHFL_TYPE_FIFO: + d_type = DT_FIFO; + break; + case SHFL_TYPE_DEV_CHAR: + d_type = DT_CHR; + break; + case SHFL_TYPE_DIRECTORY: + d_type = DT_DIR; + break; + case SHFL_TYPE_DEV_BLOCK: + d_type = DT_BLK; + break; + case SHFL_TYPE_FILE: + d_type = DT_REG; + break; + case SHFL_TYPE_SYMLINK: + d_type = DT_LNK; + break; + case SHFL_TYPE_SOCKET: + d_type = DT_SOCK; + break; + case SHFL_TYPE_WHITEOUT: + d_type = DT_WHT; + break; + default: + d_type = DT_UNKNOWN; + break; + } + return d_type; +} + +static bool vboxsf_dir_emit(struct file *dir, struct dir_context *ctx) +{ + struct vboxsf_sbi *sbi = VBOXSF_SBI(file_inode(dir)->i_sb); + struct vboxsf_dir_info *sf_d = dir->private_data; + struct shfl_dirinfo *info; + struct vboxsf_dir_buf *b; + unsigned int d_type; + loff_t i, cur = 0; + ino_t fake_ino; + void *end; + int err; + + list_for_each_entry(b, &sf_d->info_list, head) { +try_next_entry: + if (ctx->pos >= cur + b->entries) { + cur += b->entries; + continue; + } + + /* + * Note the vboxsf_dir_info objects we are iterating over here + * are variable sized, so the info pointer may end up being + * unaligned. This is how we get the data from the host. + * Since vboxsf is only supported on x86 machines this is not + * a problem. + */ + for (i = 0, info = b->buf; i < ctx->pos - cur; i++) { + end = &info->name.string.utf8[info->name.size]; + /* Only happens if the host gives us corrupt data */ + if (WARN_ON(end > (b->buf + b->used))) + return false; + info = end; + } + + end = &info->name.string.utf8[info->name.size]; + if (WARN_ON(end > (b->buf + b->used))) + return false; + + /* Info now points to the right entry, emit it. */ + d_type = vboxsf_get_d_type(info->info.attr.mode); + + /* + * On 32-bit systems pos is 64-bit signed, while ino is 32-bit + * unsigned so fake_ino may overflow, check for this. + */ + if ((ino_t)(ctx->pos + 1) != (u64)(ctx->pos + 1)) { + vbg_err("vboxsf: fake ino overflow, truncating dir\n"); + return false; + } + fake_ino = ctx->pos + 1; + + if (sbi->nls) { + char d_name[NAME_MAX]; + + err = vboxsf_nlscpy(sbi, d_name, NAME_MAX, + info->name.string.utf8, + info->name.length); + if (err) { + /* skip erroneous entry and proceed */ + ctx->pos += 1; + goto try_next_entry; + } + + return dir_emit(ctx, d_name, strlen(d_name), + fake_ino, d_type); + } + + return dir_emit(ctx, info->name.string.utf8, info->name.length, + fake_ino, d_type); + } + + return false; +} + +static int vboxsf_dir_iterate(struct file *dir, struct dir_context *ctx) +{ + bool emitted; + + do { + emitted = vboxsf_dir_emit(dir, ctx); + if (emitted) + ctx->pos += 1; + } while (emitted); + + return 0; +} + +WRAP_DIR_ITER(vboxsf_dir_iterate) // FIXME! +const struct file_operations vboxsf_dir_fops = { + .open = vboxsf_dir_open, + .iterate_shared = shared_vboxsf_dir_iterate, + .release = vboxsf_dir_release, + .read = generic_read_dir, + .llseek = generic_file_llseek, +}; + +/* + * This is called during name resolution/lookup to check if the @dentry in + * the cache is still valid. the job is handled by vboxsf_inode_revalidate. + */ +static int vboxsf_dentry_revalidate(struct inode *dir, const struct qstr *name, + struct dentry *dentry, unsigned int flags) +{ + if (flags & LOOKUP_RCU) + return -ECHILD; + + if (d_really_is_positive(dentry)) + return vboxsf_inode_revalidate(dentry) == 0; + else + return vboxsf_stat_dentry(dentry, NULL) == -ENOENT; +} + +const struct dentry_operations vboxsf_dentry_ops = { + .d_revalidate = vboxsf_dentry_revalidate +}; + +/* iops */ + +static struct dentry *vboxsf_dir_lookup(struct inode *parent, + struct dentry *dentry, + unsigned int flags) +{ + struct vboxsf_sbi *sbi = VBOXSF_SBI(parent->i_sb); + struct shfl_fsobjinfo fsinfo; + struct inode *inode; + int err; + + dentry->d_time = jiffies; + + err = vboxsf_stat_dentry(dentry, &fsinfo); + if (err) { + inode = (err == -ENOENT) ? NULL : ERR_PTR(err); + } else { + inode = vboxsf_new_inode(parent->i_sb); + if (!IS_ERR(inode)) + vboxsf_init_inode(sbi, inode, &fsinfo, false); + } + + return d_splice_alias(inode, dentry); +} + +static int vboxsf_dir_instantiate(struct inode *parent, struct dentry *dentry, + struct shfl_fsobjinfo *info) +{ + struct vboxsf_sbi *sbi = VBOXSF_SBI(parent->i_sb); + struct vboxsf_inode *sf_i; + struct inode *inode; + + inode = vboxsf_new_inode(parent->i_sb); + if (IS_ERR(inode)) + return PTR_ERR(inode); + + sf_i = VBOXSF_I(inode); + /* The host may have given us different attr then requested */ + sf_i->force_restat = 1; + vboxsf_init_inode(sbi, inode, info, false); + + d_instantiate(dentry, inode); + + return 0; +} + +static int vboxsf_dir_create(struct inode *parent, struct dentry *dentry, + umode_t mode, bool is_dir, bool excl, u64 *handle_ret) +{ + struct vboxsf_inode *sf_parent_i = VBOXSF_I(parent); + struct vboxsf_sbi *sbi = VBOXSF_SBI(parent->i_sb); + struct shfl_createparms params = {}; + int err; + + params.handle = SHFL_HANDLE_NIL; + params.create_flags = SHFL_CF_ACT_CREATE_IF_NEW | SHFL_CF_ACCESS_READWRITE; + if (is_dir) + params.create_flags |= SHFL_CF_DIRECTORY; + if (excl) + params.create_flags |= SHFL_CF_ACT_FAIL_IF_EXISTS; + + params.info.attr.mode = (mode & 0777) | + (is_dir ? SHFL_TYPE_DIRECTORY : SHFL_TYPE_FILE); + params.info.attr.additional = SHFLFSOBJATTRADD_NOTHING; + + err = vboxsf_create_at_dentry(dentry, ¶ms); + if (err) + return err; + + if (params.result != SHFL_FILE_CREATED) + return -EPERM; + + err = vboxsf_dir_instantiate(parent, dentry, ¶ms.info); + if (err) + goto out; + + /* parent directory access/change time changed */ + sf_parent_i->force_restat = 1; + +out: + if (err == 0 && handle_ret) + *handle_ret = params.handle; + else + vboxsf_close(sbi->root, params.handle); + + return err; +} + +static int vboxsf_dir_mkfile(struct mnt_idmap *idmap, + struct inode *parent, struct dentry *dentry, + umode_t mode) +{ + return vboxsf_dir_create(parent, dentry, mode, false, true, NULL); +} + +static struct dentry *vboxsf_dir_mkdir(struct mnt_idmap *idmap, + struct inode *parent, struct dentry *dentry, + umode_t mode) +{ + return ERR_PTR(vboxsf_dir_create(parent, dentry, mode, true, true, NULL)); +} + +static int vboxsf_dir_atomic_open(struct inode *parent, struct dentry *dentry, + struct file *file, unsigned int flags, umode_t mode) +{ + struct vboxsf_sbi *sbi = VBOXSF_SBI(parent->i_sb); + struct vboxsf_handle *sf_handle; + u64 handle; + int err; + + if (d_in_lookup(dentry)) { + struct dentry *res = vboxsf_dir_lookup(parent, dentry, 0); + if (res || d_really_is_positive(dentry)) + return finish_no_open(file, res); + } + + /* Only creates */ + if (!(flags & O_CREAT)) + return finish_no_open(file, NULL); + + err = vboxsf_dir_create(parent, dentry, mode, false, flags & O_EXCL, &handle); + if (err) + return err; + + sf_handle = vboxsf_create_sf_handle(d_inode(dentry), handle, SHFL_CF_ACCESS_READWRITE); + if (IS_ERR(sf_handle)) { + vboxsf_close(sbi->root, handle); + return PTR_ERR(sf_handle); + } + + err = finish_open(file, dentry, generic_file_open); + if (err) { + /* This also closes the handle passed to vboxsf_create_sf_handle() */ + vboxsf_release_sf_handle(d_inode(dentry), sf_handle); + return err; + } + + file->private_data = sf_handle; + file->f_mode |= FMODE_CREATED; + return 0; +} + +static int vboxsf_dir_unlink(struct inode *parent, struct dentry *dentry) +{ + struct vboxsf_sbi *sbi = VBOXSF_SBI(parent->i_sb); + struct vboxsf_inode *sf_parent_i = VBOXSF_I(parent); + struct inode *inode = d_inode(dentry); + struct shfl_string *path; + u32 flags; + int err; + + if (S_ISDIR(inode->i_mode)) + flags = SHFL_REMOVE_DIR; + else + flags = SHFL_REMOVE_FILE; + + if (S_ISLNK(inode->i_mode)) + flags |= SHFL_REMOVE_SYMLINK; + + path = vboxsf_path_from_dentry(sbi, dentry); + if (IS_ERR(path)) + return PTR_ERR(path); + + err = vboxsf_remove(sbi->root, path, flags); + __putname(path); + if (err) + return err; + + /* parent directory access/change time changed */ + sf_parent_i->force_restat = 1; + + return 0; +} + +static int vboxsf_dir_rename(struct mnt_idmap *idmap, + struct inode *old_parent, + struct dentry *old_dentry, + struct inode *new_parent, + struct dentry *new_dentry, + unsigned int flags) +{ + struct vboxsf_sbi *sbi = VBOXSF_SBI(old_parent->i_sb); + struct vboxsf_inode *sf_old_parent_i = VBOXSF_I(old_parent); + struct vboxsf_inode *sf_new_parent_i = VBOXSF_I(new_parent); + u32 shfl_flags = SHFL_RENAME_FILE | SHFL_RENAME_REPLACE_IF_EXISTS; + struct shfl_string *old_path, *new_path; + int err; + + if (flags) + return -EINVAL; + + old_path = vboxsf_path_from_dentry(sbi, old_dentry); + if (IS_ERR(old_path)) + return PTR_ERR(old_path); + + new_path = vboxsf_path_from_dentry(sbi, new_dentry); + if (IS_ERR(new_path)) { + err = PTR_ERR(new_path); + goto err_put_old_path; + } + + if (d_inode(old_dentry)->i_mode & S_IFDIR) + shfl_flags = 0; + + err = vboxsf_rename(sbi->root, old_path, new_path, shfl_flags); + if (err == 0) { + /* parent directories access/change time changed */ + sf_new_parent_i->force_restat = 1; + sf_old_parent_i->force_restat = 1; + } + + __putname(new_path); +err_put_old_path: + __putname(old_path); + return err; +} + +static int vboxsf_dir_symlink(struct mnt_idmap *idmap, + struct inode *parent, struct dentry *dentry, + const char *symname) +{ + struct vboxsf_inode *sf_parent_i = VBOXSF_I(parent); + struct vboxsf_sbi *sbi = VBOXSF_SBI(parent->i_sb); + int symname_size = strlen(symname) + 1; + struct shfl_string *path, *ssymname; + struct shfl_fsobjinfo info; + int err; + + path = vboxsf_path_from_dentry(sbi, dentry); + if (IS_ERR(path)) + return PTR_ERR(path); + + ssymname = kmalloc(SHFLSTRING_HEADER_SIZE + symname_size, GFP_KERNEL); + if (!ssymname) { + __putname(path); + return -ENOMEM; + } + ssymname->length = symname_size - 1; + ssymname->size = symname_size; + memcpy(ssymname->string.utf8, symname, symname_size); + + err = vboxsf_symlink(sbi->root, path, ssymname, &info); + kfree(ssymname); + __putname(path); + if (err) { + /* -EROFS means symlinks are note support -> -EPERM */ + return (err == -EROFS) ? -EPERM : err; + } + + err = vboxsf_dir_instantiate(parent, dentry, &info); + if (err) + return err; + + /* parent directory access/change time changed */ + sf_parent_i->force_restat = 1; + return 0; +} + +const struct inode_operations vboxsf_dir_iops = { + .lookup = vboxsf_dir_lookup, + .create = vboxsf_dir_mkfile, + .mkdir = vboxsf_dir_mkdir, + .atomic_open = vboxsf_dir_atomic_open, + .rmdir = vboxsf_dir_unlink, + .unlink = vboxsf_dir_unlink, + .rename = vboxsf_dir_rename, + .symlink = vboxsf_dir_symlink, + .getattr = vboxsf_getattr, + .setattr = vboxsf_setattr, + .fileattr_get = vboxsf_fileattr_get, +}; diff --git a/fs/vboxsf/file.c b/fs/vboxsf/file.c new file mode 100644 index 000000000..943953867 --- /dev/null +++ b/fs/vboxsf/file.c @@ -0,0 +1,395 @@ +// SPDX-License-Identifier: MIT +/* + * VirtualBox Guest Shared Folders support: Regular file inode and file ops. + * + * Copyright (C) 2006-2018 Oracle Corporation + */ + +#include +#include +#include +#include +#include +#include "vfsmod.h" + +struct vboxsf_handle { + u64 handle; + u32 root; + u32 access_flags; + struct kref refcount; + struct list_head head; +}; + +struct vboxsf_handle *vboxsf_create_sf_handle(struct inode *inode, + u64 handle, u32 access_flags) +{ + struct vboxsf_inode *sf_i = VBOXSF_I(inode); + struct vboxsf_handle *sf_handle; + + sf_handle = kmalloc_obj(*sf_handle); + if (!sf_handle) + return ERR_PTR(-ENOMEM); + + /* the host may have given us different attr then requested */ + sf_i->force_restat = 1; + + /* init our handle struct and add it to the inode's handles list */ + sf_handle->handle = handle; + sf_handle->root = VBOXSF_SBI(inode->i_sb)->root; + sf_handle->access_flags = access_flags; + kref_init(&sf_handle->refcount); + + mutex_lock(&sf_i->handle_list_mutex); + list_add(&sf_handle->head, &sf_i->handle_list); + mutex_unlock(&sf_i->handle_list_mutex); + + return sf_handle; +} + +static int vboxsf_file_open(struct inode *inode, struct file *file) +{ + struct vboxsf_sbi *sbi = VBOXSF_SBI(inode->i_sb); + struct shfl_createparms params = {}; + struct vboxsf_handle *sf_handle; + u32 access_flags = 0; + int err; + + /* + * We check the value of params.handle afterwards to find out if + * the call succeeded or failed, as the API does not seem to cleanly + * distinguish error and informational messages. + * + * Furthermore, we must set params.handle to SHFL_HANDLE_NIL to + * make the shared folders host service use our mode parameter. + */ + params.handle = SHFL_HANDLE_NIL; + if (file->f_flags & O_CREAT) { + params.create_flags |= SHFL_CF_ACT_CREATE_IF_NEW; + /* + * We ignore O_EXCL, as the Linux kernel seems to call create + * beforehand itself, so O_EXCL should always fail. + */ + if (file->f_flags & O_TRUNC) + params.create_flags |= SHFL_CF_ACT_OVERWRITE_IF_EXISTS; + else + params.create_flags |= SHFL_CF_ACT_OPEN_IF_EXISTS; + } else { + params.create_flags |= SHFL_CF_ACT_FAIL_IF_NEW; + if (file->f_flags & O_TRUNC) + params.create_flags |= SHFL_CF_ACT_OVERWRITE_IF_EXISTS; + } + + switch (file->f_flags & O_ACCMODE) { + case O_RDONLY: + access_flags |= SHFL_CF_ACCESS_READ; + break; + + case O_WRONLY: + access_flags |= SHFL_CF_ACCESS_WRITE; + break; + + case O_RDWR: + access_flags |= SHFL_CF_ACCESS_READWRITE; + break; + + default: + WARN_ON(1); + } + + if (file->f_flags & O_APPEND) + access_flags |= SHFL_CF_ACCESS_APPEND; + + params.create_flags |= access_flags; + params.info.attr.mode = inode->i_mode; + + err = vboxsf_create_at_dentry(file_dentry(file), ¶ms); + if (err == 0 && params.handle == SHFL_HANDLE_NIL) + err = (params.result == SHFL_FILE_EXISTS) ? -EEXIST : -ENOENT; + if (err) + return err; + + sf_handle = vboxsf_create_sf_handle(inode, params.handle, access_flags); + if (IS_ERR(sf_handle)) { + vboxsf_close(sbi->root, params.handle); + return PTR_ERR(sf_handle); + } + + file->private_data = sf_handle; + return 0; +} + +static void vboxsf_handle_release(struct kref *refcount) +{ + struct vboxsf_handle *sf_handle = + container_of(refcount, struct vboxsf_handle, refcount); + + vboxsf_close(sf_handle->root, sf_handle->handle); + kfree(sf_handle); +} + +void vboxsf_release_sf_handle(struct inode *inode, struct vboxsf_handle *sf_handle) +{ + struct vboxsf_inode *sf_i = VBOXSF_I(inode); + + mutex_lock(&sf_i->handle_list_mutex); + list_del(&sf_handle->head); + mutex_unlock(&sf_i->handle_list_mutex); + + kref_put(&sf_handle->refcount, vboxsf_handle_release); +} + +static int vboxsf_file_release(struct inode *inode, struct file *file) +{ + /* + * When a file is closed on our (the guest) side, we want any subsequent + * accesses done on the host side to see all changes done from our side. + */ + filemap_write_and_wait(inode->i_mapping); + + vboxsf_release_sf_handle(inode, file->private_data); + return 0; +} + +/* + * Write back dirty pages now, because there may not be any suitable + * open files later + */ +static void vboxsf_vma_close(struct vm_area_struct *vma) +{ + filemap_write_and_wait(vma->vm_file->f_mapping); +} + +static const struct vm_operations_struct vboxsf_file_vm_ops = { + .close = vboxsf_vma_close, + .fault = filemap_fault, + .map_pages = filemap_map_pages, +}; + +static int vboxsf_file_mmap_prepare(struct vm_area_desc *desc) +{ + int err; + + err = generic_file_mmap_prepare(desc); + if (!err) + desc->vm_ops = &vboxsf_file_vm_ops; + + return err; +} + +/* + * Note that since we are accessing files on the host's filesystem, files + * may always be changed underneath us by the host! + * + * The vboxsf API between the guest and the host does not offer any functions + * to deal with this. There is no inode-generation to check for changes, no + * events / callback on changes and no way to lock files. + * + * To avoid returning stale data when a file gets *opened* on our (the guest) + * side, we do a "stat" on the host side, then compare the mtime with the + * last known mtime and invalidate the page-cache if they differ. + * This is done from vboxsf_inode_revalidate(). + * + * When reads are done through the read_iter fop, it is possible to do + * further cache revalidation then, there are 3 options to deal with this: + * + * 1) Rely solely on the revalidation done at open time + * 2) Do another "stat" and compare mtime again. Unfortunately the vboxsf + * host API does not allow stat on handles, so we would need to use + * file->f_path.dentry and the stat will then fail if the file was unlinked + * or renamed (and there is no thing like NFS' silly-rename). So we get: + * 2a) "stat" and compare mtime, on stat failure invalidate the cache + * 2b) "stat" and compare mtime, on stat failure do nothing + * 3) Simply always call invalidate_inode_pages2_range on the range of the read + * + * Currently we are keeping things KISS and using option 1. this allows + * directly using generic_file_read_iter without wrapping it. + * + * This means that only data written on the host side before open() on + * the guest side is guaranteed to be seen by the guest. If necessary + * we may provide other read-cache strategies in the future and make this + * configurable through a mount option. + */ +const struct file_operations vboxsf_reg_fops = { + .llseek = generic_file_llseek, + .read_iter = generic_file_read_iter, + .write_iter = generic_file_write_iter, + .mmap_prepare = vboxsf_file_mmap_prepare, + .open = vboxsf_file_open, + .release = vboxsf_file_release, + .fsync = noop_fsync, + .splice_read = filemap_splice_read, +}; + +const struct inode_operations vboxsf_reg_iops = { + .getattr = vboxsf_getattr, + .setattr = vboxsf_setattr, + .fileattr_get = vboxsf_fileattr_get, +}; + +static int vboxsf_read_folio(struct file *file, struct folio *folio) +{ + struct vboxsf_handle *sf_handle = file->private_data; + loff_t off = folio_pos(folio); + u32 nread = PAGE_SIZE; + u8 *buf; + int err; + + buf = kmap_local_folio(folio, 0); + + err = vboxsf_read(sf_handle->root, sf_handle->handle, off, &nread, buf); + buf = folio_zero_tail(folio, nread, buf + nread); + + kunmap_local(buf); + folio_end_read(folio, err == 0); + return err; +} + +static struct vboxsf_handle *vboxsf_get_write_handle(struct vboxsf_inode *sf_i) +{ + struct vboxsf_handle *h, *sf_handle = NULL; + + mutex_lock(&sf_i->handle_list_mutex); + list_for_each_entry(h, &sf_i->handle_list, head) { + if (h->access_flags == SHFL_CF_ACCESS_WRITE || + h->access_flags == SHFL_CF_ACCESS_READWRITE) { + kref_get(&h->refcount); + sf_handle = h; + break; + } + } + mutex_unlock(&sf_i->handle_list_mutex); + + return sf_handle; +} + +static int vboxsf_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct inode *inode = mapping->host; + struct folio *folio = NULL; + struct vboxsf_inode *sf_i = VBOXSF_I(inode); + struct vboxsf_handle *sf_handle; + loff_t size = i_size_read(inode); + int error; + + sf_handle = vboxsf_get_write_handle(sf_i); + if (!sf_handle) + return -EBADF; + + while ((folio = writeback_iter(mapping, wbc, folio, &error))) { + loff_t off = folio_pos(folio); + u32 nwrite = folio_size(folio); + u8 *buf; + + if (nwrite > size - off) + nwrite = size - off; + + buf = kmap_local_folio(folio, 0); + error = vboxsf_write(sf_handle->root, sf_handle->handle, + off, &nwrite, buf); + kunmap_local(buf); + + folio_unlock(folio); + } + + kref_put(&sf_handle->refcount, vboxsf_handle_release); + + /* mtime changed */ + if (error == 0) + sf_i->force_restat = 1; + return error; +} + +static int vboxsf_write_end(const struct kiocb *iocb, + struct address_space *mapping, + loff_t pos, unsigned int len, unsigned int copied, + struct folio *folio, void *fsdata) +{ + struct inode *inode = mapping->host; + struct vboxsf_handle *sf_handle = iocb->ki_filp->private_data; + size_t from = offset_in_folio(folio, pos); + u32 nwritten = len; + u8 *buf; + int err; + + /* zero the stale part of the folio if we did a short copy */ + if (!folio_test_uptodate(folio) && copied < len) + folio_zero_range(folio, from + copied, len - copied); + + buf = kmap(&folio->page); + err = vboxsf_write(sf_handle->root, sf_handle->handle, + pos, &nwritten, buf + from); + kunmap(&folio->page); + + if (err) { + nwritten = 0; + goto out; + } + + /* mtime changed */ + VBOXSF_I(inode)->force_restat = 1; + + if (!folio_test_uptodate(folio) && nwritten == folio_size(folio)) + folio_mark_uptodate(folio); + + pos += nwritten; + if (pos > inode->i_size) + i_size_write(inode, pos); + +out: + folio_unlock(folio); + folio_put(folio); + + return nwritten; +} + +/* + * Note simple_write_begin does not read the page from disk on partial writes + * this is ok since vboxsf_write_end only writes the written parts of the + * page and it does not call folio_mark_uptodate for partial writes. + */ +const struct address_space_operations vboxsf_reg_aops = { + .read_folio = vboxsf_read_folio, + .writepages = vboxsf_writepages, + .dirty_folio = filemap_dirty_folio, + .write_begin = simple_write_begin, + .write_end = vboxsf_write_end, + .migrate_folio = filemap_migrate_folio, +}; + +static const char *vboxsf_get_link(struct dentry *dentry, struct inode *inode, + struct delayed_call *done) +{ + struct vboxsf_sbi *sbi = VBOXSF_SBI(inode->i_sb); + struct shfl_string *path; + char *link; + int err; + + if (!dentry) + return ERR_PTR(-ECHILD); + + path = vboxsf_path_from_dentry(sbi, dentry); + if (IS_ERR(path)) + return ERR_CAST(path); + + link = kzalloc(PATH_MAX, GFP_KERNEL); + if (!link) { + __putname(path); + return ERR_PTR(-ENOMEM); + } + + err = vboxsf_readlink(sbi->root, path, PATH_MAX, link); + __putname(path); + if (err) { + kfree(link); + return ERR_PTR(err); + } + + set_delayed_call(done, kfree_link, link); + return link; +} + +const struct inode_operations vboxsf_lnk_iops = { + .get_link = vboxsf_get_link, + .fileattr_get = vboxsf_fileattr_get, +}; diff --git a/fs/vboxsf/shfl_hostintf.h b/fs/vboxsf/shfl_hostintf.h new file mode 100644 index 000000000..069a019c9 --- /dev/null +++ b/fs/vboxsf/shfl_hostintf.h @@ -0,0 +1,901 @@ +/* SPDX-License-Identifier: MIT */ +/* + * VirtualBox Shared Folders: host interface definition. + * + * Copyright (C) 2006-2018 Oracle Corporation + */ + +#ifndef SHFL_HOSTINTF_H +#define SHFL_HOSTINTF_H + +#include + +/* The max in/out buffer size for a FN_READ or FN_WRITE call */ +#define SHFL_MAX_RW_COUNT (16 * SZ_1M) + +/* + * Structures shared between guest and the service + * can be relocated and use offsets to point to variable + * length parts. + * + * Shared folders protocol works with handles. + * Before doing any action on a file system object, + * one have to obtain the object handle via a SHFL_FN_CREATE + * request. A handle must be closed with SHFL_FN_CLOSE. + */ + +enum { + SHFL_FN_QUERY_MAPPINGS = 1, /* Query mappings changes. */ + SHFL_FN_QUERY_MAP_NAME = 2, /* Query map name. */ + SHFL_FN_CREATE = 3, /* Open/create object. */ + SHFL_FN_CLOSE = 4, /* Close object handle. */ + SHFL_FN_READ = 5, /* Read object content. */ + SHFL_FN_WRITE = 6, /* Write new object content. */ + SHFL_FN_LOCK = 7, /* Lock/unlock a range in the object. */ + SHFL_FN_LIST = 8, /* List object content. */ + SHFL_FN_INFORMATION = 9, /* Query/set object information. */ + /* Note function number 10 is not used! */ + SHFL_FN_REMOVE = 11, /* Remove object */ + SHFL_FN_MAP_FOLDER_OLD = 12, /* Map folder (legacy) */ + SHFL_FN_UNMAP_FOLDER = 13, /* Unmap folder */ + SHFL_FN_RENAME = 14, /* Rename object */ + SHFL_FN_FLUSH = 15, /* Flush file */ + SHFL_FN_SET_UTF8 = 16, /* Select UTF8 filename encoding */ + SHFL_FN_MAP_FOLDER = 17, /* Map folder */ + SHFL_FN_READLINK = 18, /* Read symlink dest (as of VBox 4.0) */ + SHFL_FN_SYMLINK = 19, /* Create symlink (as of VBox 4.0) */ + SHFL_FN_SET_SYMLINKS = 20, /* Ask host to show symlinks (4.0+) */ +}; + +/* Root handles for a mapping are of type u32, Root handles are unique. */ +#define SHFL_ROOT_NIL UINT_MAX + +/* Shared folders handle for an opened object are of type u64. */ +#define SHFL_HANDLE_NIL ULLONG_MAX + +/* Hardcoded maximum length (in chars) of a shared folder name. */ +#define SHFL_MAX_LEN (256) +/* Hardcoded maximum number of shared folder mapping available to the guest. */ +#define SHFL_MAX_MAPPINGS (64) + +/** Shared folder string buffer structure. */ +struct shfl_string { + /** Allocated size of the string member in bytes. */ + u16 size; + + /** Length of string without trailing nul in bytes. */ + u16 length; + + /** UTF-8 or UTF-16 string. Nul terminated. */ + union { + u8 legacy_padding[2]; + DECLARE_FLEX_ARRAY(u8, utf8); + DECLARE_FLEX_ARRAY(u16, utf16); + } string; +}; +VMMDEV_ASSERT_SIZE(shfl_string, 6); + +/* The size of shfl_string w/o the string part. */ +#define SHFLSTRING_HEADER_SIZE 4 + +/* Calculate size of the string. */ +static inline u32 shfl_string_buf_size(const struct shfl_string *string) +{ + return string ? SHFLSTRING_HEADER_SIZE + string->size : 0; +} + +/* Set user id on execution (S_ISUID). */ +#define SHFL_UNIX_ISUID 0004000U +/* Set group id on execution (S_ISGID). */ +#define SHFL_UNIX_ISGID 0002000U +/* Sticky bit (S_ISVTX / S_ISTXT). */ +#define SHFL_UNIX_ISTXT 0001000U + +/* Owner readable (S_IRUSR). */ +#define SHFL_UNIX_IRUSR 0000400U +/* Owner writable (S_IWUSR). */ +#define SHFL_UNIX_IWUSR 0000200U +/* Owner executable (S_IXUSR). */ +#define SHFL_UNIX_IXUSR 0000100U + +/* Group readable (S_IRGRP). */ +#define SHFL_UNIX_IRGRP 0000040U +/* Group writable (S_IWGRP). */ +#define SHFL_UNIX_IWGRP 0000020U +/* Group executable (S_IXGRP). */ +#define SHFL_UNIX_IXGRP 0000010U + +/* Other readable (S_IROTH). */ +#define SHFL_UNIX_IROTH 0000004U +/* Other writable (S_IWOTH). */ +#define SHFL_UNIX_IWOTH 0000002U +/* Other executable (S_IXOTH). */ +#define SHFL_UNIX_IXOTH 0000001U + +/* Named pipe (fifo) (S_IFIFO). */ +#define SHFL_TYPE_FIFO 0010000U +/* Character device (S_IFCHR). */ +#define SHFL_TYPE_DEV_CHAR 0020000U +/* Directory (S_IFDIR). */ +#define SHFL_TYPE_DIRECTORY 0040000U +/* Block device (S_IFBLK). */ +#define SHFL_TYPE_DEV_BLOCK 0060000U +/* Regular file (S_IFREG). */ +#define SHFL_TYPE_FILE 0100000U +/* Symbolic link (S_IFLNK). */ +#define SHFL_TYPE_SYMLINK 0120000U +/* Socket (S_IFSOCK). */ +#define SHFL_TYPE_SOCKET 0140000U +/* Whiteout (S_IFWHT). */ +#define SHFL_TYPE_WHITEOUT 0160000U +/* Type mask (S_IFMT). */ +#define SHFL_TYPE_MASK 0170000U + +/* Checks the mode flags indicate a directory (S_ISDIR). */ +#define SHFL_IS_DIRECTORY(m) (((m) & SHFL_TYPE_MASK) == SHFL_TYPE_DIRECTORY) +/* Checks the mode flags indicate a symbolic link (S_ISLNK). */ +#define SHFL_IS_SYMLINK(m) (((m) & SHFL_TYPE_MASK) == SHFL_TYPE_SYMLINK) + +/** The available additional information in a shfl_fsobjattr object. */ +enum shfl_fsobjattr_add { + /** No additional information is available / requested. */ + SHFLFSOBJATTRADD_NOTHING = 1, + /** + * The additional unix attributes (shfl_fsobjattr::u::unix_attr) are + * available / requested. + */ + SHFLFSOBJATTRADD_UNIX, + /** + * The additional extended attribute size (shfl_fsobjattr::u::size) is + * available / requested. + */ + SHFLFSOBJATTRADD_EASIZE, + /** + * The last valid item (inclusive). + * The valid range is SHFLFSOBJATTRADD_NOTHING thru + * SHFLFSOBJATTRADD_LAST. + */ + SHFLFSOBJATTRADD_LAST = SHFLFSOBJATTRADD_EASIZE, + + /** The usual 32-bit hack. */ + SHFLFSOBJATTRADD_32BIT_SIZE_HACK = 0x7fffffff +}; + +/** + * Additional unix Attributes, these are available when + * shfl_fsobjattr.additional == SHFLFSOBJATTRADD_UNIX. + */ +struct shfl_fsobjattr_unix { + /** + * The user owning the filesystem object (st_uid). + * This field is ~0U if not supported. + */ + u32 uid; + + /** + * The group the filesystem object is assigned (st_gid). + * This field is ~0U if not supported. + */ + u32 gid; + + /** + * Number of hard links to this filesystem object (st_nlink). + * This field is 1 if the filesystem doesn't support hardlinking or + * the information isn't available. + */ + u32 hardlinks; + + /** + * The device number of the device which this filesystem object resides + * on (st_dev). This field is 0 if this information is not available. + */ + u32 inode_id_device; + + /** + * The unique identifier (within the filesystem) of this filesystem + * object (st_ino). Together with inode_id_device, this field can be + * used as a OS wide unique id, when both their values are not 0. + * This field is 0 if the information is not available. + */ + u64 inode_id; + + /** + * User flags (st_flags). + * This field is 0 if this information is not available. + */ + u32 flags; + + /** + * The current generation number (st_gen). + * This field is 0 if this information is not available. + */ + u32 generation_id; + + /** + * The device number of a char. or block device type object (st_rdev). + * This field is 0 if the file isn't a char. or block device or when + * the OS doesn't use the major+minor device idenfication scheme. + */ + u32 device; +} __packed; + +/** Extended attribute size. */ +struct shfl_fsobjattr_easize { + /** Size of EAs. */ + s64 cb; +} __packed; + +/** Shared folder filesystem object attributes. */ +struct shfl_fsobjattr { + /** Mode flags (st_mode). SHFL_UNIX_*, SHFL_TYPE_*, and SHFL_DOS_*. */ + u32 mode; + + /** The additional attributes available. */ + enum shfl_fsobjattr_add additional; + + /** + * Additional attributes. + * + * Unless explicitly specified to an API, the API can provide additional + * data as it is provided by the underlying OS. + */ + union { + struct shfl_fsobjattr_unix unix_attr; + struct shfl_fsobjattr_easize size; + } __packed u; +} __packed; +VMMDEV_ASSERT_SIZE(shfl_fsobjattr, 44); + +struct shfl_timespec { + s64 ns_relative_to_unix_epoch; +}; + +/** Filesystem object information structure. */ +struct shfl_fsobjinfo { + /** + * Logical size (st_size). + * For normal files this is the size of the file. + * For symbolic links, this is the length of the path name contained + * in the symbolic link. + * For other objects this fields needs to be specified. + */ + s64 size; + + /** Disk allocation size (st_blocks * DEV_BSIZE). */ + s64 allocated; + + /** Time of last access (st_atime). */ + struct shfl_timespec access_time; + + /** Time of last data modification (st_mtime). */ + struct shfl_timespec modification_time; + + /** + * Time of last status change (st_ctime). + * If not available this is set to modification_time. + */ + struct shfl_timespec change_time; + + /** + * Time of file birth (st_birthtime). + * If not available this is set to change_time. + */ + struct shfl_timespec birth_time; + + /** Attributes. */ + struct shfl_fsobjattr attr; + +} __packed; +VMMDEV_ASSERT_SIZE(shfl_fsobjinfo, 92); + +/** + * result of an open/create request. + * Along with handle value the result code + * identifies what has happened while + * trying to open the object. + */ +enum shfl_create_result { + SHFL_NO_RESULT, + /** Specified path does not exist. */ + SHFL_PATH_NOT_FOUND, + /** Path to file exists, but the last component does not. */ + SHFL_FILE_NOT_FOUND, + /** File already exists and either has been opened or not. */ + SHFL_FILE_EXISTS, + /** New file was created. */ + SHFL_FILE_CREATED, + /** Existing file was replaced or overwritten. */ + SHFL_FILE_REPLACED +}; + +/* No flags. Initialization value. */ +#define SHFL_CF_NONE (0x00000000) + +/* + * Only lookup the object, do not return a handle. When this is set all other + * flags are ignored. + */ +#define SHFL_CF_LOOKUP (0x00000001) + +/* + * Open parent directory of specified object. + * Useful for the corresponding Windows FSD flag + * and for opening paths like \\dir\\*.* to search the 'dir'. + */ +#define SHFL_CF_OPEN_TARGET_DIRECTORY (0x00000002) + +/* Create/open a directory. */ +#define SHFL_CF_DIRECTORY (0x00000004) + +/* + * Open/create action to do if object exists + * and if the object does not exists. + * REPLACE file means atomically DELETE and CREATE. + * OVERWRITE file means truncating the file to 0 and + * setting new size. + * When opening an existing directory REPLACE and OVERWRITE + * actions are considered invalid, and cause returning + * FILE_EXISTS with NIL handle. + */ +#define SHFL_CF_ACT_MASK_IF_EXISTS (0x000000f0) +#define SHFL_CF_ACT_MASK_IF_NEW (0x00000f00) + +/* What to do if object exists. */ +#define SHFL_CF_ACT_OPEN_IF_EXISTS (0x00000000) +#define SHFL_CF_ACT_FAIL_IF_EXISTS (0x00000010) +#define SHFL_CF_ACT_REPLACE_IF_EXISTS (0x00000020) +#define SHFL_CF_ACT_OVERWRITE_IF_EXISTS (0x00000030) + +/* What to do if object does not exist. */ +#define SHFL_CF_ACT_CREATE_IF_NEW (0x00000000) +#define SHFL_CF_ACT_FAIL_IF_NEW (0x00000100) + +/* Read/write requested access for the object. */ +#define SHFL_CF_ACCESS_MASK_RW (0x00003000) + +/* No access requested. */ +#define SHFL_CF_ACCESS_NONE (0x00000000) +/* Read access requested. */ +#define SHFL_CF_ACCESS_READ (0x00001000) +/* Write access requested. */ +#define SHFL_CF_ACCESS_WRITE (0x00002000) +/* Read/Write access requested. */ +#define SHFL_CF_ACCESS_READWRITE (0x00003000) + +/* Requested share access for the object. */ +#define SHFL_CF_ACCESS_MASK_DENY (0x0000c000) + +/* Allow any access. */ +#define SHFL_CF_ACCESS_DENYNONE (0x00000000) +/* Do not allow read. */ +#define SHFL_CF_ACCESS_DENYREAD (0x00004000) +/* Do not allow write. */ +#define SHFL_CF_ACCESS_DENYWRITE (0x00008000) +/* Do not allow access. */ +#define SHFL_CF_ACCESS_DENYALL (0x0000c000) + +/* Requested access to attributes of the object. */ +#define SHFL_CF_ACCESS_MASK_ATTR (0x00030000) + +/* No access requested. */ +#define SHFL_CF_ACCESS_ATTR_NONE (0x00000000) +/* Read access requested. */ +#define SHFL_CF_ACCESS_ATTR_READ (0x00010000) +/* Write access requested. */ +#define SHFL_CF_ACCESS_ATTR_WRITE (0x00020000) +/* Read/Write access requested. */ +#define SHFL_CF_ACCESS_ATTR_READWRITE (0x00030000) + +/* + * The file is opened in append mode. + * Ignored if SHFL_CF_ACCESS_WRITE is not set. + */ +#define SHFL_CF_ACCESS_APPEND (0x00040000) + +/** Create parameters buffer struct for SHFL_FN_CREATE call */ +struct shfl_createparms { + /** Returned handle of opened object. */ + u64 handle; + + /** Returned result of the operation */ + enum shfl_create_result result; + + /** SHFL_CF_* */ + u32 create_flags; + + /** + * Attributes of object to create and + * returned actual attributes of opened/created object. + */ + struct shfl_fsobjinfo info; +} __packed; + +/** Shared Folder directory information */ +struct shfl_dirinfo { + /** Full information about the object. */ + struct shfl_fsobjinfo info; + /** + * The length of the short field (number of UTF16 chars). + * It is 16-bit for reasons of alignment. + */ + u16 short_name_len; + /** + * The short name for 8.3 compatibility. + * Empty string if not available. + */ + u16 short_name[14]; + struct shfl_string name; +}; + +/** Shared folder filesystem properties. */ +struct shfl_fsproperties { + /** + * The maximum size of a filesystem object name. + * This does not include the '\\0'. + */ + u32 max_component_len; + + /** + * True if the filesystem is remote. + * False if the filesystem is local. + */ + bool remote; + + /** + * True if the filesystem is case sensitive. + * False if the filesystem is case insensitive. + */ + bool case_sensitive; + + /** + * True if the filesystem is mounted read only. + * False if the filesystem is mounted read write. + */ + bool read_only; + + /** + * True if the filesystem can encode unicode object names. + * False if it can't. + */ + bool supports_unicode; + + /** + * True if the filesystem is compresses. + * False if it isn't or we don't know. + */ + bool compressed; + + /** + * True if the filesystem compresses of individual files. + * False if it doesn't or we don't know. + */ + bool file_compression; +}; +VMMDEV_ASSERT_SIZE(shfl_fsproperties, 12); + +struct shfl_volinfo { + s64 total_allocation_bytes; + s64 available_allocation_bytes; + u32 bytes_per_allocation_unit; + u32 bytes_per_sector; + u32 serial; + struct shfl_fsproperties properties; +}; + + +/** SHFL_FN_MAP_FOLDER Parameters structure. */ +struct shfl_map_folder { + /** + * pointer, in: + * Points to struct shfl_string buffer. + */ + struct vmmdev_hgcm_function_parameter path; + + /** + * pointer, out: SHFLROOT (u32) + * Root handle of the mapping which name is queried. + */ + struct vmmdev_hgcm_function_parameter root; + + /** + * pointer, in: UTF16 + * Path delimiter + */ + struct vmmdev_hgcm_function_parameter delimiter; + + /** + * pointer, in: SHFLROOT (u32) + * Case senstive flag + */ + struct vmmdev_hgcm_function_parameter case_sensitive; + +}; + +/* Number of parameters */ +#define SHFL_CPARMS_MAP_FOLDER (4) + + +/** SHFL_FN_UNMAP_FOLDER Parameters structure. */ +struct shfl_unmap_folder { + /** + * pointer, in: SHFLROOT (u32) + * Root handle of the mapping which name is queried. + */ + struct vmmdev_hgcm_function_parameter root; + +}; + +/* Number of parameters */ +#define SHFL_CPARMS_UNMAP_FOLDER (1) + + +/** SHFL_FN_CREATE Parameters structure. */ +struct shfl_create { + /** + * pointer, in: SHFLROOT (u32) + * Root handle of the mapping which name is queried. + */ + struct vmmdev_hgcm_function_parameter root; + + /** + * pointer, in: + * Points to struct shfl_string buffer. + */ + struct vmmdev_hgcm_function_parameter path; + + /** + * pointer, in/out: + * Points to struct shfl_createparms buffer. + */ + struct vmmdev_hgcm_function_parameter parms; + +}; + +/* Number of parameters */ +#define SHFL_CPARMS_CREATE (3) + + +/** SHFL_FN_CLOSE Parameters structure. */ +struct shfl_close { + /** + * pointer, in: SHFLROOT (u32) + * Root handle of the mapping which name is queried. + */ + struct vmmdev_hgcm_function_parameter root; + + /** + * value64, in: + * SHFLHANDLE (u64) of object to close. + */ + struct vmmdev_hgcm_function_parameter handle; + +}; + +/* Number of parameters */ +#define SHFL_CPARMS_CLOSE (2) + + +/** SHFL_FN_READ Parameters structure. */ +struct shfl_read { + /** + * pointer, in: SHFLROOT (u32) + * Root handle of the mapping which name is queried. + */ + struct vmmdev_hgcm_function_parameter root; + + /** + * value64, in: + * SHFLHANDLE (u64) of object to read from. + */ + struct vmmdev_hgcm_function_parameter handle; + + /** + * value64, in: + * Offset to read from. + */ + struct vmmdev_hgcm_function_parameter offset; + + /** + * value64, in/out: + * Bytes to read/How many were read. + */ + struct vmmdev_hgcm_function_parameter cb; + + /** + * pointer, out: + * Buffer to place data to. + */ + struct vmmdev_hgcm_function_parameter buffer; + +}; + +/* Number of parameters */ +#define SHFL_CPARMS_READ (5) + + +/** SHFL_FN_WRITE Parameters structure. */ +struct shfl_write { + /** + * pointer, in: SHFLROOT (u32) + * Root handle of the mapping which name is queried. + */ + struct vmmdev_hgcm_function_parameter root; + + /** + * value64, in: + * SHFLHANDLE (u64) of object to write to. + */ + struct vmmdev_hgcm_function_parameter handle; + + /** + * value64, in: + * Offset to write to. + */ + struct vmmdev_hgcm_function_parameter offset; + + /** + * value64, in/out: + * Bytes to write/How many were written. + */ + struct vmmdev_hgcm_function_parameter cb; + + /** + * pointer, in: + * Data to write. + */ + struct vmmdev_hgcm_function_parameter buffer; + +}; + +/* Number of parameters */ +#define SHFL_CPARMS_WRITE (5) + + +/* + * SHFL_FN_LIST + * Listing information includes variable length RTDIRENTRY[EX] structures. + */ + +#define SHFL_LIST_NONE 0 +#define SHFL_LIST_RETURN_ONE 1 + +/** SHFL_FN_LIST Parameters structure. */ +struct shfl_list { + /** + * pointer, in: SHFLROOT (u32) + * Root handle of the mapping which name is queried. + */ + struct vmmdev_hgcm_function_parameter root; + + /** + * value64, in: + * SHFLHANDLE (u64) of object to be listed. + */ + struct vmmdev_hgcm_function_parameter handle; + + /** + * value32, in: + * List flags SHFL_LIST_*. + */ + struct vmmdev_hgcm_function_parameter flags; + + /** + * value32, in/out: + * Bytes to be used for listing information/How many bytes were used. + */ + struct vmmdev_hgcm_function_parameter cb; + + /** + * pointer, in/optional + * Points to struct shfl_string buffer that specifies a search path. + */ + struct vmmdev_hgcm_function_parameter path; + + /** + * pointer, out: + * Buffer to place listing information to. (struct shfl_dirinfo) + */ + struct vmmdev_hgcm_function_parameter buffer; + + /** + * value32, in/out: + * Indicates a key where the listing must be resumed. + * in: 0 means start from begin of object. + * out: 0 means listing completed. + */ + struct vmmdev_hgcm_function_parameter resume_point; + + /** + * pointer, out: + * Number of files returned + */ + struct vmmdev_hgcm_function_parameter file_count; +}; + +/* Number of parameters */ +#define SHFL_CPARMS_LIST (8) + + +/** SHFL_FN_READLINK Parameters structure. */ +struct shfl_readLink { + /** + * pointer, in: SHFLROOT (u32) + * Root handle of the mapping which name is queried. + */ + struct vmmdev_hgcm_function_parameter root; + + /** + * pointer, in: + * Points to struct shfl_string buffer. + */ + struct vmmdev_hgcm_function_parameter path; + + /** + * pointer, out: + * Buffer to place data to. + */ + struct vmmdev_hgcm_function_parameter buffer; + +}; + +/* Number of parameters */ +#define SHFL_CPARMS_READLINK (3) + + +/* SHFL_FN_INFORMATION */ + +/* Mask of Set/Get bit. */ +#define SHFL_INFO_MODE_MASK (0x1) +/* Get information */ +#define SHFL_INFO_GET (0x0) +/* Set information */ +#define SHFL_INFO_SET (0x1) + +/* Get name of the object. */ +#define SHFL_INFO_NAME (0x2) +/* Set size of object (extend/trucate); only applies to file objects */ +#define SHFL_INFO_SIZE (0x4) +/* Get/Set file object info. */ +#define SHFL_INFO_FILE (0x8) +/* Get volume information. */ +#define SHFL_INFO_VOLUME (0x10) + +/** SHFL_FN_INFORMATION Parameters structure. */ +struct shfl_information { + /** + * pointer, in: SHFLROOT (u32) + * Root handle of the mapping which name is queried. + */ + struct vmmdev_hgcm_function_parameter root; + + /** + * value64, in: + * SHFLHANDLE (u64) of object to be listed. + */ + struct vmmdev_hgcm_function_parameter handle; + + /** + * value32, in: + * SHFL_INFO_* + */ + struct vmmdev_hgcm_function_parameter flags; + + /** + * value32, in/out: + * Bytes to be used for information/How many bytes were used. + */ + struct vmmdev_hgcm_function_parameter cb; + + /** + * pointer, in/out: + * Information to be set/get (shfl_fsobjinfo or shfl_string). Do not + * forget to set the shfl_fsobjinfo::attr::additional for a get + * operation as well. + */ + struct vmmdev_hgcm_function_parameter info; + +}; + +/* Number of parameters */ +#define SHFL_CPARMS_INFORMATION (5) + + +/* SHFL_FN_REMOVE */ + +#define SHFL_REMOVE_FILE (0x1) +#define SHFL_REMOVE_DIR (0x2) +#define SHFL_REMOVE_SYMLINK (0x4) + +/** SHFL_FN_REMOVE Parameters structure. */ +struct shfl_remove { + /** + * pointer, in: SHFLROOT (u32) + * Root handle of the mapping which name is queried. + */ + struct vmmdev_hgcm_function_parameter root; + + /** + * pointer, in: + * Points to struct shfl_string buffer. + */ + struct vmmdev_hgcm_function_parameter path; + + /** + * value32, in: + * remove flags (file/directory) + */ + struct vmmdev_hgcm_function_parameter flags; + +}; + +#define SHFL_CPARMS_REMOVE (3) + + +/* SHFL_FN_RENAME */ + +#define SHFL_RENAME_FILE (0x1) +#define SHFL_RENAME_DIR (0x2) +#define SHFL_RENAME_REPLACE_IF_EXISTS (0x4) + +/** SHFL_FN_RENAME Parameters structure. */ +struct shfl_rename { + /** + * pointer, in: SHFLROOT (u32) + * Root handle of the mapping which name is queried. + */ + struct vmmdev_hgcm_function_parameter root; + + /** + * pointer, in: + * Points to struct shfl_string src. + */ + struct vmmdev_hgcm_function_parameter src; + + /** + * pointer, in: + * Points to struct shfl_string dest. + */ + struct vmmdev_hgcm_function_parameter dest; + + /** + * value32, in: + * rename flags (file/directory) + */ + struct vmmdev_hgcm_function_parameter flags; + +}; + +#define SHFL_CPARMS_RENAME (4) + + +/** SHFL_FN_SYMLINK Parameters structure. */ +struct shfl_symlink { + /** + * pointer, in: SHFLROOT (u32) + * Root handle of the mapping which name is queried. + */ + struct vmmdev_hgcm_function_parameter root; + + /** + * pointer, in: + * Points to struct shfl_string of path for the new symlink. + */ + struct vmmdev_hgcm_function_parameter new_path; + + /** + * pointer, in: + * Points to struct shfl_string of destination for symlink. + */ + struct vmmdev_hgcm_function_parameter old_path; + + /** + * pointer, out: + * Information about created symlink. + */ + struct vmmdev_hgcm_function_parameter info; + +}; + +#define SHFL_CPARMS_SYMLINK (4) + +#endif diff --git a/fs/vboxsf/super.c b/fs/vboxsf/super.c new file mode 100644 index 000000000..a61fbab51 --- /dev/null +++ b/fs/vboxsf/super.c @@ -0,0 +1,484 @@ +// SPDX-License-Identifier: MIT +/* + * VirtualBox Guest Shared Folders support: Virtual File System. + * + * Module initialization/finalization + * File system registration/deregistration + * Superblock reading + * Few utility functions + * + * Copyright (C) 2006-2018 Oracle Corporation + */ + +#include +#include +#include +#include +#include +#include +#include +#include "vfsmod.h" + +#define VBOXSF_SUPER_MAGIC 0x786f4256 /* 'VBox' little endian */ + +static const unsigned char VBSF_MOUNT_SIGNATURE[4] __nonstring = "\000\377\376\375"; + +static int follow_symlinks; +module_param(follow_symlinks, int, 0444); +MODULE_PARM_DESC(follow_symlinks, + "Let host resolve symlinks rather than showing them"); + +static DEFINE_IDA(vboxsf_bdi_ida); +static DEFINE_MUTEX(vboxsf_setup_mutex); +static bool vboxsf_setup_done; +static struct super_operations vboxsf_super_ops; /* forward declaration */ +static struct kmem_cache *vboxsf_inode_cachep; + +static char * const vboxsf_default_nls = CONFIG_NLS_DEFAULT; + +enum { opt_nls, opt_uid, opt_gid, opt_ttl, opt_dmode, opt_fmode, + opt_dmask, opt_fmask }; + +static const struct fs_parameter_spec vboxsf_fs_parameters[] = { + fsparam_string ("nls", opt_nls), + fsparam_uid ("uid", opt_uid), + fsparam_gid ("gid", opt_gid), + fsparam_u32 ("ttl", opt_ttl), + fsparam_u32oct ("dmode", opt_dmode), + fsparam_u32oct ("fmode", opt_fmode), + fsparam_u32oct ("dmask", opt_dmask), + fsparam_u32oct ("fmask", opt_fmask), + {} +}; + +static int vboxsf_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct vboxsf_fs_context *ctx = fc->fs_private; + struct fs_parse_result result; + int opt; + + opt = fs_parse(fc, vboxsf_fs_parameters, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case opt_nls: + if (ctx->nls_name || fc->purpose != FS_CONTEXT_FOR_MOUNT) { + vbg_err("vboxsf: Cannot reconfigure nls option\n"); + return -EINVAL; + } + ctx->nls_name = param->string; + param->string = NULL; + break; + case opt_uid: + ctx->o.uid = result.uid; + break; + case opt_gid: + ctx->o.gid = result.gid; + break; + case opt_ttl: + ctx->o.ttl = msecs_to_jiffies(result.uint_32); + break; + case opt_dmode: + if (result.uint_32 & ~0777) + return -EINVAL; + ctx->o.dmode = result.uint_32; + ctx->o.dmode_set = true; + break; + case opt_fmode: + if (result.uint_32 & ~0777) + return -EINVAL; + ctx->o.fmode = result.uint_32; + ctx->o.fmode_set = true; + break; + case opt_dmask: + if (result.uint_32 & ~07777) + return -EINVAL; + ctx->o.dmask = result.uint_32; + break; + case opt_fmask: + if (result.uint_32 & ~07777) + return -EINVAL; + ctx->o.fmask = result.uint_32; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int vboxsf_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct vboxsf_fs_context *ctx = fc->fs_private; + struct shfl_string *folder_name, root_path; + struct vboxsf_sbi *sbi; + struct dentry *droot; + struct inode *iroot; + char *nls_name; + size_t size; + int err; + + if (!fc->source) + return -EINVAL; + + sbi = kzalloc_obj(*sbi); + if (!sbi) + return -ENOMEM; + + sbi->o = ctx->o; + idr_init(&sbi->ino_idr); + spin_lock_init(&sbi->ino_idr_lock); + sbi->next_generation = 1; + sbi->bdi_id = -1; + + /* Load nls if not utf8 */ + nls_name = ctx->nls_name ? ctx->nls_name : vboxsf_default_nls; + if (strcmp(nls_name, "utf8") != 0) { + if (nls_name == vboxsf_default_nls) + sbi->nls = load_nls_default(); + else + sbi->nls = load_nls(nls_name); + + if (!sbi->nls) { + vbg_err("vboxsf: Count not load '%s' nls\n", nls_name); + err = -EINVAL; + goto fail_destroy_idr; + } + } + + sbi->bdi_id = ida_alloc(&vboxsf_bdi_ida, GFP_KERNEL); + if (sbi->bdi_id < 0) { + err = sbi->bdi_id; + goto fail_free; + } + + err = super_setup_bdi_name(sb, "vboxsf-%d", sbi->bdi_id); + if (err) + goto fail_free; + sb->s_bdi->ra_pages = 0; + sb->s_bdi->io_pages = 0; + + /* Turn source into a shfl_string and map the folder */ + size = strlen(fc->source) + 1; + folder_name = kmalloc(SHFLSTRING_HEADER_SIZE + size, GFP_KERNEL); + if (!folder_name) { + err = -ENOMEM; + goto fail_free; + } + folder_name->size = size; + folder_name->length = size - 1; + strscpy(folder_name->string.utf8, fc->source, size); + err = vboxsf_map_folder(folder_name, &sbi->root); + kfree(folder_name); + if (err) { + vbg_err("vboxsf: Host rejected mount of '%s' with error %d\n", + fc->source, err); + goto fail_free; + } + + root_path.length = 1; + root_path.size = 2; + root_path.string.utf8[0] = '/'; + root_path.string.utf8[1] = 0; + err = vboxsf_stat(sbi, &root_path, &sbi->root_info); + if (err) + goto fail_unmap; + + /* + * A failed query leaves sbi->case_insensitive false, so the + * mount defaults to reporting case-sensitive behavior. Do not + * fail the mount over an advisory attribute. + */ + vboxsf_query_case_sensitive(sbi); + + sb->s_magic = VBOXSF_SUPER_MAGIC; + sb->s_blocksize = 1024; + sb->s_maxbytes = MAX_LFS_FILESIZE; + sb->s_op = &vboxsf_super_ops; + set_default_d_op(sb, &vboxsf_dentry_ops); + + iroot = iget_locked(sb, 0); + if (!iroot) { + err = -ENOMEM; + goto fail_unmap; + } + vboxsf_init_inode(sbi, iroot, &sbi->root_info, false); + unlock_new_inode(iroot); + + droot = d_make_root(iroot); + if (!droot) { + err = -ENOMEM; + goto fail_unmap; + } + + sb->s_root = droot; + sb->s_fs_info = sbi; + return 0; + +fail_unmap: + vboxsf_unmap_folder(sbi->root); +fail_free: + if (sbi->bdi_id >= 0) + ida_free(&vboxsf_bdi_ida, sbi->bdi_id); + if (sbi->nls) + unload_nls(sbi->nls); +fail_destroy_idr: + idr_destroy(&sbi->ino_idr); + kfree(sbi); + return err; +} + +static void vboxsf_inode_init_once(void *data) +{ + struct vboxsf_inode *sf_i = data; + + mutex_init(&sf_i->handle_list_mutex); + inode_init_once(&sf_i->vfs_inode); +} + +static struct inode *vboxsf_alloc_inode(struct super_block *sb) +{ + struct vboxsf_inode *sf_i; + + sf_i = alloc_inode_sb(sb, vboxsf_inode_cachep, GFP_NOFS); + if (!sf_i) + return NULL; + + sf_i->force_restat = 0; + INIT_LIST_HEAD(&sf_i->handle_list); + + return &sf_i->vfs_inode; +} + +static void vboxsf_free_inode(struct inode *inode) +{ + struct vboxsf_sbi *sbi = VBOXSF_SBI(inode->i_sb); + unsigned long flags; + + spin_lock_irqsave(&sbi->ino_idr_lock, flags); + idr_remove(&sbi->ino_idr, inode->i_ino); + spin_unlock_irqrestore(&sbi->ino_idr_lock, flags); + kmem_cache_free(vboxsf_inode_cachep, VBOXSF_I(inode)); +} + +static void vboxsf_put_super(struct super_block *sb) +{ + struct vboxsf_sbi *sbi = VBOXSF_SBI(sb); + + vboxsf_unmap_folder(sbi->root); + if (sbi->bdi_id >= 0) + ida_free(&vboxsf_bdi_ida, sbi->bdi_id); + if (sbi->nls) + unload_nls(sbi->nls); + + /* + * vboxsf_free_inode uses the idr, make sure all delayed rcu free + * inodes are flushed. + */ + rcu_barrier(); + idr_destroy(&sbi->ino_idr); + kfree(sbi); +} + +static int vboxsf_statfs(struct dentry *dentry, struct kstatfs *stat) +{ + struct super_block *sb = dentry->d_sb; + struct shfl_volinfo shfl_volinfo; + struct vboxsf_sbi *sbi; + u32 buf_len; + int err; + + sbi = VBOXSF_SBI(sb); + buf_len = sizeof(shfl_volinfo); + err = vboxsf_fsinfo(sbi->root, 0, SHFL_INFO_GET | SHFL_INFO_VOLUME, + &buf_len, &shfl_volinfo); + if (err) + return err; + + stat->f_type = VBOXSF_SUPER_MAGIC; + stat->f_bsize = shfl_volinfo.bytes_per_allocation_unit; + + do_div(shfl_volinfo.total_allocation_bytes, + shfl_volinfo.bytes_per_allocation_unit); + stat->f_blocks = shfl_volinfo.total_allocation_bytes; + + do_div(shfl_volinfo.available_allocation_bytes, + shfl_volinfo.bytes_per_allocation_unit); + stat->f_bfree = shfl_volinfo.available_allocation_bytes; + stat->f_bavail = shfl_volinfo.available_allocation_bytes; + + stat->f_files = 1000; + /* + * Don't return 0 here since the guest may then think that it is not + * possible to create any more files. + */ + stat->f_ffree = 1000000; + stat->f_fsid.val[0] = 0; + stat->f_fsid.val[1] = 0; + stat->f_namelen = 255; + return 0; +} + +static struct super_operations vboxsf_super_ops = { + .alloc_inode = vboxsf_alloc_inode, + .free_inode = vboxsf_free_inode, + .put_super = vboxsf_put_super, + .statfs = vboxsf_statfs, +}; + +static int vboxsf_setup(void) +{ + int err; + + mutex_lock(&vboxsf_setup_mutex); + + if (vboxsf_setup_done) + goto success; + + vboxsf_inode_cachep = + kmem_cache_create("vboxsf_inode_cache", + sizeof(struct vboxsf_inode), 0, + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, + vboxsf_inode_init_once); + if (!vboxsf_inode_cachep) { + err = -ENOMEM; + goto fail_nomem; + } + + err = vboxsf_connect(); + if (err) { + vbg_err("vboxsf: err %d connecting to guest PCI-device\n", err); + vbg_err("vboxsf: make sure you are inside a VirtualBox VM\n"); + vbg_err("vboxsf: and check dmesg for vboxguest errors\n"); + goto fail_free_cache; + } + + err = vboxsf_set_utf8(); + if (err) { + vbg_err("vboxsf_setutf8 error %d\n", err); + goto fail_disconnect; + } + + if (!follow_symlinks) { + err = vboxsf_set_symlinks(); + if (err) + vbg_warn("vboxsf: Unable to show symlinks: %d\n", err); + } + + vboxsf_setup_done = true; +success: + mutex_unlock(&vboxsf_setup_mutex); + return 0; + +fail_disconnect: + vboxsf_disconnect(); +fail_free_cache: + kmem_cache_destroy(vboxsf_inode_cachep); +fail_nomem: + mutex_unlock(&vboxsf_setup_mutex); + return err; +} + +static int vboxsf_parse_monolithic(struct fs_context *fc, void *data) +{ + if (data && !memcmp(data, VBSF_MOUNT_SIGNATURE, 4)) { + vbg_err("vboxsf: Old binary mount data not supported, remove obsolete mount.vboxsf and/or update your VBoxService.\n"); + return -EINVAL; + } + + return generic_parse_monolithic(fc, data); +} + +static int vboxsf_get_tree(struct fs_context *fc) +{ + int err; + + err = vboxsf_setup(); + if (err) + return err; + + return get_tree_nodev(fc, vboxsf_fill_super); +} + +static int vboxsf_reconfigure(struct fs_context *fc) +{ + struct vboxsf_sbi *sbi = VBOXSF_SBI(fc->root->d_sb); + struct vboxsf_fs_context *ctx = fc->fs_private; + struct inode *iroot = fc->root->d_sb->s_root->d_inode; + + /* Apply changed options to the root inode */ + sbi->o = ctx->o; + vboxsf_init_inode(sbi, iroot, &sbi->root_info, true); + + return 0; +} + +static void vboxsf_free_fc(struct fs_context *fc) +{ + struct vboxsf_fs_context *ctx = fc->fs_private; + + kfree(ctx->nls_name); + kfree(ctx); +} + +static const struct fs_context_operations vboxsf_context_ops = { + .free = vboxsf_free_fc, + .parse_param = vboxsf_parse_param, + .parse_monolithic = vboxsf_parse_monolithic, + .get_tree = vboxsf_get_tree, + .reconfigure = vboxsf_reconfigure, +}; + +static int vboxsf_init_fs_context(struct fs_context *fc) +{ + struct vboxsf_fs_context *ctx; + + ctx = kzalloc_obj(*ctx); + if (!ctx) + return -ENOMEM; + + current_uid_gid(&ctx->o.uid, &ctx->o.gid); + + fc->fs_private = ctx; + fc->ops = &vboxsf_context_ops; + return 0; +} + +static struct file_system_type vboxsf_fs_type = { + .owner = THIS_MODULE, + .name = "vboxsf", + .init_fs_context = vboxsf_init_fs_context, + .kill_sb = kill_anon_super +}; + +/* Module initialization/finalization handlers */ +static int __init vboxsf_init(void) +{ + return register_filesystem(&vboxsf_fs_type); +} + +static void __exit vboxsf_fini(void) +{ + unregister_filesystem(&vboxsf_fs_type); + + mutex_lock(&vboxsf_setup_mutex); + if (vboxsf_setup_done) { + vboxsf_disconnect(); + /* + * Make sure all delayed rcu free inodes are flushed + * before we destroy the cache. + */ + rcu_barrier(); + kmem_cache_destroy(vboxsf_inode_cachep); + } + mutex_unlock(&vboxsf_setup_mutex); +} + +module_init(vboxsf_init); +module_exit(vboxsf_fini); + +MODULE_DESCRIPTION("Oracle VM VirtualBox Module for Host File System Access"); +MODULE_AUTHOR("Oracle Corporation"); +MODULE_LICENSE("GPL v2"); +MODULE_ALIAS_FS("vboxsf"); diff --git a/fs/vboxsf/utils.c b/fs/vboxsf/utils.c new file mode 100644 index 000000000..298bfc932 --- /dev/null +++ b/fs/vboxsf/utils.c @@ -0,0 +1,599 @@ +// SPDX-License-Identifier: MIT +/* + * VirtualBox Guest Shared Folders support: Utility functions. + * Mainly conversion from/to VirtualBox/Linux data structures. + * + * Copyright (C) 2006-2018 Oracle Corporation + */ + +#include +#include +#include +#include +#include +#include +#include "vfsmod.h" + +struct inode *vboxsf_new_inode(struct super_block *sb) +{ + struct vboxsf_sbi *sbi = VBOXSF_SBI(sb); + struct inode *inode; + unsigned long flags; + int cursor, ret; + u32 gen; + + inode = new_inode(sb); + if (!inode) + return ERR_PTR(-ENOMEM); + + idr_preload(GFP_KERNEL); + spin_lock_irqsave(&sbi->ino_idr_lock, flags); + cursor = idr_get_cursor(&sbi->ino_idr); + ret = idr_alloc_cyclic(&sbi->ino_idr, inode, 1, 0, GFP_ATOMIC); + if (ret >= 0 && ret < cursor) + sbi->next_generation++; + gen = sbi->next_generation; + spin_unlock_irqrestore(&sbi->ino_idr_lock, flags); + idr_preload_end(); + + if (ret < 0) { + iput(inode); + return ERR_PTR(ret); + } + + inode->i_ino = ret; + inode->i_generation = gen; + return inode; +} + +/* set [inode] attributes based on [info], uid/gid based on [sbi] */ +int vboxsf_init_inode(struct vboxsf_sbi *sbi, struct inode *inode, + const struct shfl_fsobjinfo *info, bool reinit) +{ + const struct shfl_fsobjattr *attr; + s64 allocated; + umode_t mode; + + attr = &info->attr; + +#define mode_set(r) ((attr->mode & (SHFL_UNIX_##r)) ? (S_##r) : 0) + + mode = mode_set(IRUSR); + mode |= mode_set(IWUSR); + mode |= mode_set(IXUSR); + + mode |= mode_set(IRGRP); + mode |= mode_set(IWGRP); + mode |= mode_set(IXGRP); + + mode |= mode_set(IROTH); + mode |= mode_set(IWOTH); + mode |= mode_set(IXOTH); + +#undef mode_set + + /* We use the host-side values for these */ + inode->i_flags |= S_NOATIME | S_NOCMTIME; + inode->i_mapping->a_ops = &vboxsf_reg_aops; + + if (SHFL_IS_DIRECTORY(attr->mode)) { + if (sbi->o.dmode_set) + mode = sbi->o.dmode; + mode &= ~sbi->o.dmask; + mode |= S_IFDIR; + if (!reinit) { + inode->i_op = &vboxsf_dir_iops; + inode->i_fop = &vboxsf_dir_fops; + /* + * XXX: this probably should be set to the number of entries + * in the directory plus two (. ..) + */ + set_nlink(inode, 1); + } else if (!S_ISDIR(inode->i_mode)) + return -ESTALE; + inode->i_mode = mode; + } else if (SHFL_IS_SYMLINK(attr->mode)) { + if (sbi->o.fmode_set) + mode = sbi->o.fmode; + mode &= ~sbi->o.fmask; + mode |= S_IFLNK; + if (!reinit) { + inode->i_op = &vboxsf_lnk_iops; + set_nlink(inode, 1); + } else if (!S_ISLNK(inode->i_mode)) + return -ESTALE; + inode->i_mode = mode; + } else { + if (sbi->o.fmode_set) + mode = sbi->o.fmode; + mode &= ~sbi->o.fmask; + mode |= S_IFREG; + if (!reinit) { + inode->i_op = &vboxsf_reg_iops; + inode->i_fop = &vboxsf_reg_fops; + set_nlink(inode, 1); + } else if (!S_ISREG(inode->i_mode)) + return -ESTALE; + inode->i_mode = mode; + } + + inode->i_uid = sbi->o.uid; + inode->i_gid = sbi->o.gid; + + inode->i_size = info->size; + inode->i_blkbits = 12; + /* i_blocks always in units of 512 bytes! */ + allocated = info->allocated + 511; + do_div(allocated, 512); + inode->i_blocks = allocated; + + inode_set_atime_to_ts(inode, + ns_to_timespec64(info->access_time.ns_relative_to_unix_epoch)); + inode_set_ctime_to_ts(inode, + ns_to_timespec64(info->change_time.ns_relative_to_unix_epoch)); + inode_set_mtime_to_ts(inode, + ns_to_timespec64(info->modification_time.ns_relative_to_unix_epoch)); + return 0; +} + +int vboxsf_create_at_dentry(struct dentry *dentry, + struct shfl_createparms *params) +{ + struct vboxsf_sbi *sbi = VBOXSF_SBI(dentry->d_sb); + struct shfl_string *path; + int err; + + path = vboxsf_path_from_dentry(sbi, dentry); + if (IS_ERR(path)) + return PTR_ERR(path); + + err = vboxsf_create(sbi->root, path, params); + __putname(path); + + return err; +} + +int vboxsf_stat(struct vboxsf_sbi *sbi, struct shfl_string *path, + struct shfl_fsobjinfo *info) +{ + struct shfl_createparms params = {}; + int err; + + params.handle = SHFL_HANDLE_NIL; + params.create_flags = SHFL_CF_LOOKUP | SHFL_CF_ACT_FAIL_IF_NEW; + + err = vboxsf_create(sbi->root, path, ¶ms); + if (err) + return err; + + if (params.result != SHFL_FILE_EXISTS) + return -ENOENT; + + if (info) + *info = params.info; + + return 0; +} + +int vboxsf_stat_dentry(struct dentry *dentry, struct shfl_fsobjinfo *info) +{ + struct vboxsf_sbi *sbi = VBOXSF_SBI(dentry->d_sb); + struct shfl_string *path; + int err; + + path = vboxsf_path_from_dentry(sbi, dentry); + if (IS_ERR(path)) + return PTR_ERR(path); + + err = vboxsf_stat(sbi, path, info); + __putname(path); + return err; +} + +int vboxsf_inode_revalidate(struct dentry *dentry) +{ + struct vboxsf_sbi *sbi; + struct vboxsf_inode *sf_i; + struct shfl_fsobjinfo info; + struct timespec64 mtime, prev_mtime; + struct inode *inode; + int err; + + if (!dentry || !d_really_is_positive(dentry)) + return -EINVAL; + + inode = d_inode(dentry); + prev_mtime = inode_get_mtime(inode); + sf_i = VBOXSF_I(inode); + sbi = VBOXSF_SBI(dentry->d_sb); + if (!sf_i->force_restat) { + if (time_before(jiffies, dentry->d_time + sbi->o.ttl)) + return 0; + } + + err = vboxsf_stat_dentry(dentry, &info); + if (err) + return err; + + dentry->d_time = jiffies; + sf_i->force_restat = 0; + err = vboxsf_init_inode(sbi, inode, &info, true); + if (err) + return err; + + /* + * If the file was changed on the host side we need to invalidate the + * page-cache for it. Note this also gets triggered by our own writes, + * this is unavoidable. + */ + mtime = inode_get_mtime(inode); + if (timespec64_compare(&mtime, &prev_mtime) > 0) + invalidate_inode_pages2(inode->i_mapping); + + return 0; +} + +int vboxsf_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *kstat, u32 request_mask, unsigned int flags) +{ + int err; + struct dentry *dentry = path->dentry; + struct inode *inode = d_inode(dentry); + struct vboxsf_inode *sf_i = VBOXSF_I(inode); + + switch (flags & AT_STATX_SYNC_TYPE) { + case AT_STATX_DONT_SYNC: + err = 0; + break; + case AT_STATX_FORCE_SYNC: + sf_i->force_restat = 1; + fallthrough; + default: + err = vboxsf_inode_revalidate(dentry); + } + if (err) + return err; + + generic_fillattr(&nop_mnt_idmap, request_mask, d_inode(dentry), kstat); + return 0; +} + +int vboxsf_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *iattr) +{ + struct vboxsf_inode *sf_i = VBOXSF_I(d_inode(dentry)); + struct vboxsf_sbi *sbi = VBOXSF_SBI(dentry->d_sb); + struct shfl_createparms params = {}; + struct shfl_fsobjinfo info = {}; + u32 buf_len; + int err; + + params.handle = SHFL_HANDLE_NIL; + params.create_flags = SHFL_CF_ACT_OPEN_IF_EXISTS | + SHFL_CF_ACT_FAIL_IF_NEW | + SHFL_CF_ACCESS_ATTR_WRITE; + + /* this is at least required for Posix hosts */ + if (iattr->ia_valid & ATTR_SIZE) + params.create_flags |= SHFL_CF_ACCESS_WRITE; + + err = vboxsf_create_at_dentry(dentry, ¶ms); + if (err || params.result != SHFL_FILE_EXISTS) + return err ? err : -ENOENT; + +#define mode_set(r) ((iattr->ia_mode & (S_##r)) ? SHFL_UNIX_##r : 0) + + /* + * Setting the file size and setting the other attributes has to + * be handled separately. + */ + if (iattr->ia_valid & (ATTR_MODE | ATTR_ATIME | ATTR_MTIME)) { + if (iattr->ia_valid & ATTR_MODE) { + info.attr.mode = mode_set(IRUSR); + info.attr.mode |= mode_set(IWUSR); + info.attr.mode |= mode_set(IXUSR); + info.attr.mode |= mode_set(IRGRP); + info.attr.mode |= mode_set(IWGRP); + info.attr.mode |= mode_set(IXGRP); + info.attr.mode |= mode_set(IROTH); + info.attr.mode |= mode_set(IWOTH); + info.attr.mode |= mode_set(IXOTH); + + if (iattr->ia_mode & S_IFDIR) + info.attr.mode |= SHFL_TYPE_DIRECTORY; + else + info.attr.mode |= SHFL_TYPE_FILE; + } + + if (iattr->ia_valid & ATTR_ATIME) + info.access_time.ns_relative_to_unix_epoch = + timespec64_to_ns(&iattr->ia_atime); + + if (iattr->ia_valid & ATTR_MTIME) + info.modification_time.ns_relative_to_unix_epoch = + timespec64_to_ns(&iattr->ia_mtime); + + /* + * Ignore ctime (inode change time) as it can't be set + * from userland anyway. + */ + + buf_len = sizeof(info); + err = vboxsf_fsinfo(sbi->root, params.handle, + SHFL_INFO_SET | SHFL_INFO_FILE, &buf_len, + &info); + if (err) { + vboxsf_close(sbi->root, params.handle); + return err; + } + + /* the host may have given us different attr then requested */ + sf_i->force_restat = 1; + } + +#undef mode_set + + if (iattr->ia_valid & ATTR_SIZE) { + memset(&info, 0, sizeof(info)); + info.size = iattr->ia_size; + buf_len = sizeof(info); + err = vboxsf_fsinfo(sbi->root, params.handle, + SHFL_INFO_SET | SHFL_INFO_SIZE, &buf_len, + &info); + if (err) { + vboxsf_close(sbi->root, params.handle); + return err; + } + + /* the host may have given us different attr then requested */ + sf_i->force_restat = 1; + } + + vboxsf_close(sbi->root, params.handle); + + /* Update the inode with what the host has actually given us. */ + if (sf_i->force_restat) + vboxsf_inode_revalidate(dentry); + + return 0; +} + +/* + * [dentry] contains string encoded in coding system that corresponds + * to [sbi]->nls, we must convert it to UTF8 here. + * Returns a shfl_string allocated through __getname (must be freed using + * __putname), or an ERR_PTR on error. + */ +struct shfl_string *vboxsf_path_from_dentry(struct vboxsf_sbi *sbi, + struct dentry *dentry) +{ + struct shfl_string *shfl_path; + int path_len, out_len, nb; + char *buf, *path; + wchar_t uni; + u8 *out; + + buf = __getname(); + if (!buf) + return ERR_PTR(-ENOMEM); + + path = dentry_path_raw(dentry, buf, PATH_MAX); + if (IS_ERR(path)) { + __putname(buf); + return ERR_CAST(path); + } + path_len = strlen(path); + + if (sbi->nls) { + shfl_path = __getname(); + if (!shfl_path) { + __putname(buf); + return ERR_PTR(-ENOMEM); + } + + out = shfl_path->string.utf8; + out_len = PATH_MAX - SHFLSTRING_HEADER_SIZE - 1; + + while (path_len) { + nb = sbi->nls->char2uni(path, path_len, &uni); + if (nb < 0) { + __putname(shfl_path); + __putname(buf); + return ERR_PTR(-EINVAL); + } + path += nb; + path_len -= nb; + + nb = utf32_to_utf8(uni, out, out_len); + if (nb < 0) { + __putname(shfl_path); + __putname(buf); + return ERR_PTR(-ENAMETOOLONG); + } + out += nb; + out_len -= nb; + } + *out = 0; + shfl_path->length = out - shfl_path->string.utf8; + shfl_path->size = shfl_path->length + 1; + __putname(buf); + } else { + if ((SHFLSTRING_HEADER_SIZE + path_len + 1) > PATH_MAX) { + __putname(buf); + return ERR_PTR(-ENAMETOOLONG); + } + /* + * dentry_path stores the name at the end of buf, but the + * shfl_string string we return must be properly aligned. + */ + shfl_path = (struct shfl_string *)buf; + memmove(shfl_path->string.utf8, path, path_len); + shfl_path->string.utf8[path_len] = 0; + shfl_path->length = path_len; + shfl_path->size = path_len + 1; + } + + return shfl_path; +} + +int vboxsf_nlscpy(struct vboxsf_sbi *sbi, char *name, size_t name_bound_len, + const unsigned char *utf8_name, size_t utf8_len) +{ + const char *in; + char *out; + size_t out_bound_len; + size_t in_bound_len; + + in = utf8_name; + in_bound_len = utf8_len; + + out = name; + /* Reserve space for terminating 0 */ + out_bound_len = name_bound_len - 1; + + while (in_bound_len) { + int nb; + unicode_t uni; + + nb = utf8_to_utf32(in, in_bound_len, &uni); + if (nb < 0) + return -EINVAL; + + in += nb; + in_bound_len -= nb; + + nb = sbi->nls->uni2char(uni, out, out_bound_len); + if (nb < 0) + return nb; + + out += nb; + out_bound_len -= nb; + } + + *out = 0; + + return 0; +} + +static struct vboxsf_dir_buf *vboxsf_dir_buf_alloc(struct list_head *list) +{ + struct vboxsf_dir_buf *b; + + b = kmalloc_obj(*b); + if (!b) + return NULL; + + b->buf = kmalloc(DIR_BUFFER_SIZE, GFP_KERNEL); + if (!b->buf) { + kfree(b); + return NULL; + } + + b->entries = 0; + b->used = 0; + b->free = DIR_BUFFER_SIZE; + list_add(&b->head, list); + + return b; +} + +static void vboxsf_dir_buf_free(struct vboxsf_dir_buf *b) +{ + list_del(&b->head); + kfree(b->buf); + kfree(b); +} + +struct vboxsf_dir_info *vboxsf_dir_info_alloc(void) +{ + struct vboxsf_dir_info *p; + + p = kmalloc_obj(*p); + if (!p) + return NULL; + + INIT_LIST_HEAD(&p->info_list); + return p; +} + +void vboxsf_dir_info_free(struct vboxsf_dir_info *p) +{ + struct list_head *list, *pos, *tmp; + + list = &p->info_list; + list_for_each_safe(pos, tmp, list) { + struct vboxsf_dir_buf *b; + + b = list_entry(pos, struct vboxsf_dir_buf, head); + vboxsf_dir_buf_free(b); + } + kfree(p); +} + +int vboxsf_dir_read_all(struct vboxsf_sbi *sbi, struct vboxsf_dir_info *sf_d, + u64 handle) +{ + struct vboxsf_dir_buf *b; + u32 entries, size; + int err = 0; + void *buf; + + /* vboxsf_dirinfo returns 1 on end of dir */ + while (err == 0) { + b = vboxsf_dir_buf_alloc(&sf_d->info_list); + if (!b) { + err = -ENOMEM; + break; + } + + buf = b->buf; + size = b->free; + + err = vboxsf_dirinfo(sbi->root, handle, NULL, 0, 0, + &size, buf, &entries); + if (err < 0) + break; + + b->entries += entries; + b->free -= size; + b->used += size; + } + + if (b && b->used == 0) + vboxsf_dir_buf_free(b); + + /* -EILSEQ means the host could not translate a filename, ignore */ + if (err > 0 || err == -EILSEQ) + err = 0; + + return err; +} + +int vboxsf_query_case_sensitive(struct vboxsf_sbi *sbi) +{ + struct shfl_volinfo volinfo = {}; + u32 buf_len; + int err; + + buf_len = sizeof(volinfo); + err = vboxsf_fsinfo(sbi->root, 0, SHFL_INFO_GET | SHFL_INFO_VOLUME, + &buf_len, &volinfo); + if (err) + return err; + if (buf_len < sizeof(volinfo)) + return 0; + + sbi->case_insensitive = !volinfo.properties.case_sensitive; + return 0; +} + +int vboxsf_fileattr_get(struct dentry *dentry, struct file_kattr *fa) +{ + struct vboxsf_sbi *sbi = VBOXSF_SBI(dentry->d_sb); + + if (sbi->case_insensitive) { + fa->fsx_xflags |= FS_XFLAG_CASEFOLD; + fa->flags |= FS_CASEFOLD_FL; + } + return 0; +} diff --git a/fs/vboxsf/vboxsf_wrappers.c b/fs/vboxsf/vboxsf_wrappers.c new file mode 100644 index 000000000..5e8d4359e --- /dev/null +++ b/fs/vboxsf/vboxsf_wrappers.c @@ -0,0 +1,371 @@ +// SPDX-License-Identifier: MIT +/* + * Wrapper functions for the shfl host calls. + * + * Copyright (C) 2006-2018 Oracle Corporation + */ + +#include +#include +#include +#include +#include "vfsmod.h" + +#define SHFL_REQUEST \ + (VMMDEV_REQUESTOR_KERNEL | VMMDEV_REQUESTOR_USR_DRV_OTHER | \ + VMMDEV_REQUESTOR_CON_DONT_KNOW | VMMDEV_REQUESTOR_TRUST_NOT_GIVEN) + +static u32 vboxsf_client_id; + +int vboxsf_connect(void) +{ + struct vbg_dev *gdev; + struct vmmdev_hgcm_service_location loc; + int err, vbox_status; + + loc.type = VMMDEV_HGCM_LOC_LOCALHOST_EXISTING; + strcpy(loc.u.localhost.service_name, "VBoxSharedFolders"); + + gdev = vbg_get_gdev(); + if (IS_ERR(gdev)) + return -ENODEV; /* No guest-device */ + + err = vbg_hgcm_connect(gdev, SHFL_REQUEST, &loc, + &vboxsf_client_id, &vbox_status); + vbg_put_gdev(gdev); + + return err ? err : vbg_status_code_to_errno(vbox_status); +} + +void vboxsf_disconnect(void) +{ + struct vbg_dev *gdev; + int vbox_status; + + gdev = vbg_get_gdev(); + if (IS_ERR(gdev)) + return; /* guest-device is gone, already disconnected */ + + vbg_hgcm_disconnect(gdev, SHFL_REQUEST, vboxsf_client_id, &vbox_status); + vbg_put_gdev(gdev); +} + +static int vboxsf_call(u32 function, void *parms, u32 parm_count, int *status) +{ + struct vbg_dev *gdev; + int err, vbox_status; + + gdev = vbg_get_gdev(); + if (IS_ERR(gdev)) + return -ESHUTDOWN; /* guest-dev removed underneath us */ + + err = vbg_hgcm_call(gdev, SHFL_REQUEST, vboxsf_client_id, function, + U32_MAX, parms, parm_count, &vbox_status); + vbg_put_gdev(gdev); + + if (err < 0) + return err; + + if (status) + *status = vbox_status; + + return vbg_status_code_to_errno(vbox_status); +} + +int vboxsf_map_folder(struct shfl_string *folder_name, u32 *root) +{ + struct shfl_map_folder parms; + int err, status; + + parms.path.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL; + parms.path.u.pointer.size = shfl_string_buf_size(folder_name); + parms.path.u.pointer.u.linear_addr = (uintptr_t)folder_name; + + parms.root.type = VMMDEV_HGCM_PARM_TYPE_32BIT; + parms.root.u.value32 = 0; + + parms.delimiter.type = VMMDEV_HGCM_PARM_TYPE_32BIT; + parms.delimiter.u.value32 = '/'; + + parms.case_sensitive.type = VMMDEV_HGCM_PARM_TYPE_32BIT; + parms.case_sensitive.u.value32 = 1; + + err = vboxsf_call(SHFL_FN_MAP_FOLDER, &parms, SHFL_CPARMS_MAP_FOLDER, + &status); + if (err == -ENOSYS && status == VERR_NOT_IMPLEMENTED) + vbg_err("%s: Error host is too old\n", __func__); + + *root = parms.root.u.value32; + return err; +} + +int vboxsf_unmap_folder(u32 root) +{ + struct shfl_unmap_folder parms; + + parms.root.type = VMMDEV_HGCM_PARM_TYPE_32BIT; + parms.root.u.value32 = root; + + return vboxsf_call(SHFL_FN_UNMAP_FOLDER, &parms, + SHFL_CPARMS_UNMAP_FOLDER, NULL); +} + +/** + * vboxsf_create - Create a new file or folder + * @root: Root of the shared folder in which to create the file + * @parsed_path: The path of the file or folder relative to the shared folder + * @create_parms: Parameters for file/folder creation. + * + * Create a new file or folder or open an existing one in a shared folder. + * Note this function always returns 0 / success unless an exceptional condition + * occurs - out of memory, invalid arguments, etc. If the file or folder could + * not be opened or created, create_parms->handle will be set to + * SHFL_HANDLE_NIL on return. In this case the value in create_parms->result + * provides information as to why (e.g. SHFL_FILE_EXISTS), create_parms->result + * is also set on success as additional information. + * + * Returns: + * 0 or negative errno value. + */ +int vboxsf_create(u32 root, struct shfl_string *parsed_path, + struct shfl_createparms *create_parms) +{ + struct shfl_create parms; + + parms.root.type = VMMDEV_HGCM_PARM_TYPE_32BIT; + parms.root.u.value32 = root; + + parms.path.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL; + parms.path.u.pointer.size = shfl_string_buf_size(parsed_path); + parms.path.u.pointer.u.linear_addr = (uintptr_t)parsed_path; + + parms.parms.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL; + parms.parms.u.pointer.size = sizeof(struct shfl_createparms); + parms.parms.u.pointer.u.linear_addr = (uintptr_t)create_parms; + + return vboxsf_call(SHFL_FN_CREATE, &parms, SHFL_CPARMS_CREATE, NULL); +} + +int vboxsf_close(u32 root, u64 handle) +{ + struct shfl_close parms; + + parms.root.type = VMMDEV_HGCM_PARM_TYPE_32BIT; + parms.root.u.value32 = root; + + parms.handle.type = VMMDEV_HGCM_PARM_TYPE_64BIT; + parms.handle.u.value64 = handle; + + return vboxsf_call(SHFL_FN_CLOSE, &parms, SHFL_CPARMS_CLOSE, NULL); +} + +int vboxsf_remove(u32 root, struct shfl_string *parsed_path, u32 flags) +{ + struct shfl_remove parms; + + parms.root.type = VMMDEV_HGCM_PARM_TYPE_32BIT; + parms.root.u.value32 = root; + + parms.path.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL_IN; + parms.path.u.pointer.size = shfl_string_buf_size(parsed_path); + parms.path.u.pointer.u.linear_addr = (uintptr_t)parsed_path; + + parms.flags.type = VMMDEV_HGCM_PARM_TYPE_32BIT; + parms.flags.u.value32 = flags; + + return vboxsf_call(SHFL_FN_REMOVE, &parms, SHFL_CPARMS_REMOVE, NULL); +} + +int vboxsf_rename(u32 root, struct shfl_string *src_path, + struct shfl_string *dest_path, u32 flags) +{ + struct shfl_rename parms; + + parms.root.type = VMMDEV_HGCM_PARM_TYPE_32BIT; + parms.root.u.value32 = root; + + parms.src.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL_IN; + parms.src.u.pointer.size = shfl_string_buf_size(src_path); + parms.src.u.pointer.u.linear_addr = (uintptr_t)src_path; + + parms.dest.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL_IN; + parms.dest.u.pointer.size = shfl_string_buf_size(dest_path); + parms.dest.u.pointer.u.linear_addr = (uintptr_t)dest_path; + + parms.flags.type = VMMDEV_HGCM_PARM_TYPE_32BIT; + parms.flags.u.value32 = flags; + + return vboxsf_call(SHFL_FN_RENAME, &parms, SHFL_CPARMS_RENAME, NULL); +} + +int vboxsf_read(u32 root, u64 handle, u64 offset, u32 *buf_len, u8 *buf) +{ + struct shfl_read parms; + int err; + + parms.root.type = VMMDEV_HGCM_PARM_TYPE_32BIT; + parms.root.u.value32 = root; + + parms.handle.type = VMMDEV_HGCM_PARM_TYPE_64BIT; + parms.handle.u.value64 = handle; + parms.offset.type = VMMDEV_HGCM_PARM_TYPE_64BIT; + parms.offset.u.value64 = offset; + parms.cb.type = VMMDEV_HGCM_PARM_TYPE_32BIT; + parms.cb.u.value32 = *buf_len; + parms.buffer.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL_OUT; + parms.buffer.u.pointer.size = *buf_len; + parms.buffer.u.pointer.u.linear_addr = (uintptr_t)buf; + + err = vboxsf_call(SHFL_FN_READ, &parms, SHFL_CPARMS_READ, NULL); + + *buf_len = parms.cb.u.value32; + return err; +} + +int vboxsf_write(u32 root, u64 handle, u64 offset, u32 *buf_len, u8 *buf) +{ + struct shfl_write parms; + int err; + + parms.root.type = VMMDEV_HGCM_PARM_TYPE_32BIT; + parms.root.u.value32 = root; + + parms.handle.type = VMMDEV_HGCM_PARM_TYPE_64BIT; + parms.handle.u.value64 = handle; + parms.offset.type = VMMDEV_HGCM_PARM_TYPE_64BIT; + parms.offset.u.value64 = offset; + parms.cb.type = VMMDEV_HGCM_PARM_TYPE_32BIT; + parms.cb.u.value32 = *buf_len; + parms.buffer.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL_IN; + parms.buffer.u.pointer.size = *buf_len; + parms.buffer.u.pointer.u.linear_addr = (uintptr_t)buf; + + err = vboxsf_call(SHFL_FN_WRITE, &parms, SHFL_CPARMS_WRITE, NULL); + + *buf_len = parms.cb.u.value32; + return err; +} + +/* Returns 0 on success, 1 on end-of-dir, negative errno otherwise */ +int vboxsf_dirinfo(u32 root, u64 handle, + struct shfl_string *parsed_path, u32 flags, u32 index, + u32 *buf_len, struct shfl_dirinfo *buf, u32 *file_count) +{ + struct shfl_list parms; + int err, status; + + parms.root.type = VMMDEV_HGCM_PARM_TYPE_32BIT; + parms.root.u.value32 = root; + + parms.handle.type = VMMDEV_HGCM_PARM_TYPE_64BIT; + parms.handle.u.value64 = handle; + parms.flags.type = VMMDEV_HGCM_PARM_TYPE_32BIT; + parms.flags.u.value32 = flags; + parms.cb.type = VMMDEV_HGCM_PARM_TYPE_32BIT; + parms.cb.u.value32 = *buf_len; + if (parsed_path) { + parms.path.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL_IN; + parms.path.u.pointer.size = shfl_string_buf_size(parsed_path); + parms.path.u.pointer.u.linear_addr = (uintptr_t)parsed_path; + } else { + parms.path.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_IN; + parms.path.u.pointer.size = 0; + parms.path.u.pointer.u.linear_addr = 0; + } + + parms.buffer.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL_OUT; + parms.buffer.u.pointer.size = *buf_len; + parms.buffer.u.pointer.u.linear_addr = (uintptr_t)buf; + + parms.resume_point.type = VMMDEV_HGCM_PARM_TYPE_32BIT; + parms.resume_point.u.value32 = index; + parms.file_count.type = VMMDEV_HGCM_PARM_TYPE_32BIT; + parms.file_count.u.value32 = 0; /* out parameter only */ + + err = vboxsf_call(SHFL_FN_LIST, &parms, SHFL_CPARMS_LIST, &status); + if (err == -ENODATA && status == VERR_NO_MORE_FILES) + err = 1; + + *buf_len = parms.cb.u.value32; + *file_count = parms.file_count.u.value32; + return err; +} + +int vboxsf_fsinfo(u32 root, u64 handle, u32 flags, + u32 *buf_len, void *buf) +{ + struct shfl_information parms; + int err; + + parms.root.type = VMMDEV_HGCM_PARM_TYPE_32BIT; + parms.root.u.value32 = root; + + parms.handle.type = VMMDEV_HGCM_PARM_TYPE_64BIT; + parms.handle.u.value64 = handle; + parms.flags.type = VMMDEV_HGCM_PARM_TYPE_32BIT; + parms.flags.u.value32 = flags; + parms.cb.type = VMMDEV_HGCM_PARM_TYPE_32BIT; + parms.cb.u.value32 = *buf_len; + parms.info.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL; + parms.info.u.pointer.size = *buf_len; + parms.info.u.pointer.u.linear_addr = (uintptr_t)buf; + + err = vboxsf_call(SHFL_FN_INFORMATION, &parms, SHFL_CPARMS_INFORMATION, + NULL); + + *buf_len = parms.cb.u.value32; + return err; +} + +int vboxsf_readlink(u32 root, struct shfl_string *parsed_path, + u32 buf_len, u8 *buf) +{ + struct shfl_readLink parms; + + parms.root.type = VMMDEV_HGCM_PARM_TYPE_32BIT; + parms.root.u.value32 = root; + + parms.path.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL_IN; + parms.path.u.pointer.size = shfl_string_buf_size(parsed_path); + parms.path.u.pointer.u.linear_addr = (uintptr_t)parsed_path; + + parms.buffer.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL_OUT; + parms.buffer.u.pointer.size = buf_len; + parms.buffer.u.pointer.u.linear_addr = (uintptr_t)buf; + + return vboxsf_call(SHFL_FN_READLINK, &parms, SHFL_CPARMS_READLINK, + NULL); +} + +int vboxsf_symlink(u32 root, struct shfl_string *new_path, + struct shfl_string *old_path, struct shfl_fsobjinfo *buf) +{ + struct shfl_symlink parms; + + parms.root.type = VMMDEV_HGCM_PARM_TYPE_32BIT; + parms.root.u.value32 = root; + + parms.new_path.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL_IN; + parms.new_path.u.pointer.size = shfl_string_buf_size(new_path); + parms.new_path.u.pointer.u.linear_addr = (uintptr_t)new_path; + + parms.old_path.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL_IN; + parms.old_path.u.pointer.size = shfl_string_buf_size(old_path); + parms.old_path.u.pointer.u.linear_addr = (uintptr_t)old_path; + + parms.info.type = VMMDEV_HGCM_PARM_TYPE_LINADDR_KERNEL_OUT; + parms.info.u.pointer.size = sizeof(struct shfl_fsobjinfo); + parms.info.u.pointer.u.linear_addr = (uintptr_t)buf; + + return vboxsf_call(SHFL_FN_SYMLINK, &parms, SHFL_CPARMS_SYMLINK, NULL); +} + +int vboxsf_set_utf8(void) +{ + return vboxsf_call(SHFL_FN_SET_UTF8, NULL, 0, NULL); +} + +int vboxsf_set_symlinks(void) +{ + return vboxsf_call(SHFL_FN_SET_SYMLINKS, NULL, 0, NULL); +} diff --git a/fs/vboxsf/vfsmod.h b/fs/vboxsf/vfsmod.h new file mode 100644 index 000000000..b61afd0ce --- /dev/null +++ b/fs/vboxsf/vfsmod.h @@ -0,0 +1,152 @@ +/* SPDX-License-Identifier: MIT */ +/* + * VirtualBox Guest Shared Folders support: module header. + * + * Copyright (C) 2006-2018 Oracle Corporation + */ + +#ifndef VFSMOD_H +#define VFSMOD_H + +#include +#include +#include "shfl_hostintf.h" + +#define DIR_BUFFER_SIZE SZ_16K + +/* The cast is to prevent assignment of void * to pointers of arbitrary type */ +#define VBOXSF_SBI(sb) ((struct vboxsf_sbi *)(sb)->s_fs_info) +#define VBOXSF_I(i) container_of(i, struct vboxsf_inode, vfs_inode) + +struct vboxsf_handle; + +struct vboxsf_options { + unsigned long ttl; + kuid_t uid; + kgid_t gid; + bool dmode_set; + bool fmode_set; + umode_t dmode; + umode_t fmode; + umode_t dmask; + umode_t fmask; +}; + +struct vboxsf_fs_context { + struct vboxsf_options o; + char *nls_name; +}; + +/* per-shared folder information */ +struct vboxsf_sbi { + struct vboxsf_options o; + struct shfl_fsobjinfo root_info; + struct idr ino_idr; + spinlock_t ino_idr_lock; /* This protects ino_idr */ + struct nls_table *nls; + u32 next_generation; + u32 root; + int bdi_id; + bool case_insensitive; +}; + +/* per-inode information */ +struct vboxsf_inode { + /* some information was changed, update data on next revalidate */ + int force_restat; + /* list of open handles for this inode + lock protecting it */ + struct list_head handle_list; + /* This mutex protects handle_list accesses */ + struct mutex handle_list_mutex; + /* The VFS inode struct */ + struct inode vfs_inode; +}; + +struct vboxsf_dir_info { + struct list_head info_list; +}; + +struct vboxsf_dir_buf { + size_t entries; + size_t free; + size_t used; + void *buf; + struct list_head head; +}; + +/* globals */ +extern const struct inode_operations vboxsf_dir_iops; +extern const struct inode_operations vboxsf_lnk_iops; +extern const struct inode_operations vboxsf_reg_iops; +extern const struct file_operations vboxsf_dir_fops; +extern const struct file_operations vboxsf_reg_fops; +extern const struct address_space_operations vboxsf_reg_aops; +extern const struct dentry_operations vboxsf_dentry_ops; + +/* from file.c */ +struct vboxsf_handle *vboxsf_create_sf_handle(struct inode *inode, + u64 handle, u32 access_flags); +void vboxsf_release_sf_handle(struct inode *inode, struct vboxsf_handle *sf_handle); + +/* from utils.c */ +struct inode *vboxsf_new_inode(struct super_block *sb); +int vboxsf_init_inode(struct vboxsf_sbi *sbi, struct inode *inode, + const struct shfl_fsobjinfo *info, bool reinit); +int vboxsf_create_at_dentry(struct dentry *dentry, + struct shfl_createparms *params); +int vboxsf_stat(struct vboxsf_sbi *sbi, struct shfl_string *path, + struct shfl_fsobjinfo *info); +int vboxsf_stat_dentry(struct dentry *dentry, struct shfl_fsobjinfo *info); +int vboxsf_inode_revalidate(struct dentry *dentry); +int vboxsf_getattr(struct mnt_idmap *idmap, const struct path *path, + struct kstat *kstat, u32 request_mask, + unsigned int query_flags); +int vboxsf_setattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct iattr *iattr); +struct shfl_string *vboxsf_path_from_dentry(struct vboxsf_sbi *sbi, + struct dentry *dentry); +int vboxsf_nlscpy(struct vboxsf_sbi *sbi, char *name, size_t name_bound_len, + const unsigned char *utf8_name, size_t utf8_len); +struct vboxsf_dir_info *vboxsf_dir_info_alloc(void); +void vboxsf_dir_info_free(struct vboxsf_dir_info *p); +int vboxsf_dir_read_all(struct vboxsf_sbi *sbi, struct vboxsf_dir_info *sf_d, + u64 handle); + +int vboxsf_query_case_sensitive(struct vboxsf_sbi *sbi); + +struct file_kattr; +int vboxsf_fileattr_get(struct dentry *dentry, struct file_kattr *fa); + +/* from vboxsf_wrappers.c */ +int vboxsf_connect(void); +void vboxsf_disconnect(void); + +int vboxsf_create(u32 root, struct shfl_string *parsed_path, + struct shfl_createparms *create_parms); + +int vboxsf_close(u32 root, u64 handle); +int vboxsf_remove(u32 root, struct shfl_string *parsed_path, u32 flags); +int vboxsf_rename(u32 root, struct shfl_string *src_path, + struct shfl_string *dest_path, u32 flags); + +int vboxsf_read(u32 root, u64 handle, u64 offset, u32 *buf_len, u8 *buf); +int vboxsf_write(u32 root, u64 handle, u64 offset, u32 *buf_len, u8 *buf); + +int vboxsf_dirinfo(u32 root, u64 handle, + struct shfl_string *parsed_path, u32 flags, u32 index, + u32 *buf_len, struct shfl_dirinfo *buf, u32 *file_count); +int vboxsf_fsinfo(u32 root, u64 handle, u32 flags, + u32 *buf_len, void *buf); + +int vboxsf_map_folder(struct shfl_string *folder_name, u32 *root); +int vboxsf_unmap_folder(u32 root); + +int vboxsf_readlink(u32 root, struct shfl_string *parsed_path, + u32 buf_len, u8 *buf); +int vboxsf_symlink(u32 root, struct shfl_string *new_path, + struct shfl_string *old_path, struct shfl_fsobjinfo *buf); + +int vboxsf_set_utf8(void); +int vboxsf_set_symlinks(void); + +#endif diff --git a/fs/verity/Kconfig b/fs/verity/Kconfig new file mode 100644 index 000000000..b20882963 --- /dev/null +++ b/fs/verity/Kconfig @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: GPL-2.0 + +config FS_VERITY + bool "FS Verity (read-only file-based authenticity protection)" + # Filesystems cache the Merkle tree at a 64K aligned offset in the + # pagecache. That approach assumes the page size is at most 64K. + depends on PAGE_SHIFT <= 16 + select CRYPTO_HASH_INFO + select CRYPTO_LIB_SHA256 + select CRYPTO_LIB_SHA512 + help + This option enables fs-verity. fs-verity is the dm-verity + mechanism implemented at the file level. On supported + filesystems (currently ext4, f2fs, and btrfs), userspace can + use an ioctl to enable verity for a file, which causes the + filesystem to build a Merkle tree for the file. The filesystem + will then transparently verify any data read from the file + against the Merkle tree. The file is also made read-only. + + This serves as an integrity check, but the availability of the + Merkle tree root hash also allows efficiently supporting + various use cases where normally the whole file would need to + be hashed at once, such as: (a) auditing (logging the file's + hash), or (b) authenticity verification (comparing the hash + against a known good value, e.g. from a digital signature). + + fs-verity is especially useful on large files where not all + the contents may actually be needed. Also, fs-verity verifies + data each time it is paged back in, which provides better + protection against malicious disks vs. an ahead-of-time hash. + + If unsure, say N. + +config FS_VERITY_BUILTIN_SIGNATURES + bool "FS Verity builtin signature support" + depends on FS_VERITY + select SYSTEM_DATA_VERIFICATION + help + This option adds support for in-kernel verification of + fs-verity builtin signatures. + + Please take great care before using this feature. It is not + the only way to do signatures with fs-verity, and the + alternatives (such as userspace signature verification, and + IMA appraisal) can be much better. For details about the + limitations of this feature, see + Documentation/filesystems/fsverity.rst. + + If unsure, say N. diff --git a/fs/verity/Makefile b/fs/verity/Makefile new file mode 100644 index 000000000..ddb4a88a0 --- /dev/null +++ b/fs/verity/Makefile @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_FS_VERITY) += enable.o \ + hash_algs.o \ + init.o \ + measure.o \ + open.o \ + pagecache.o \ + read_metadata.o \ + verify.o + +obj-$(CONFIG_FS_VERITY_BUILTIN_SIGNATURES) += signature.o diff --git a/fs/verity/enable.c b/fs/verity/enable.c new file mode 100644 index 000000000..42dfed1ce --- /dev/null +++ b/fs/verity/enable.c @@ -0,0 +1,413 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Ioctl to enable verity on a file + * + * Copyright 2019 Google LLC + */ + +#include "fsverity_private.h" + +#include +#include +#include +#include + +struct block_buffer { + u32 filled; + bool is_root_hash; + u8 *data; +}; + +/* Hash a block, writing the result to the next level's pending block buffer. */ +static int hash_one_block(const struct merkle_tree_params *params, + struct block_buffer *cur) +{ + struct block_buffer *next = cur + 1; + + /* + * Safety check to prevent a buffer overflow in case of a filesystem bug + * that allows the file size to change despite deny_write_access(), or a + * bug in the Merkle tree logic itself + */ + if (WARN_ON_ONCE(next->is_root_hash && next->filled != 0)) + return -EINVAL; + + /* Zero-pad the block if it's shorter than the block size. */ + memset(&cur->data[cur->filled], 0, params->block_size - cur->filled); + + fsverity_hash_block(params, cur->data, &next->data[next->filled]); + next->filled += params->digest_size; + cur->filled = 0; + return 0; +} + +static int write_merkle_tree_block(struct file *file, const u8 *buf, + unsigned long index, + const struct merkle_tree_params *params) +{ + struct inode *inode = file_inode(file); + u64 pos = (u64)index << params->log_blocksize; + int err; + + err = inode->i_sb->s_vop->write_merkle_tree_block(file, buf, pos, + params->block_size); + if (err) + fsverity_err(inode, "Error %d writing Merkle tree block %lu", + err, index); + return err; +} + +/* + * Build the Merkle tree for the given file using the given parameters, and + * return the root hash in @root_hash. + * + * The tree is written to a filesystem-specific location as determined by the + * ->write_merkle_tree_block() method. However, the blocks that comprise the + * tree are the same for all filesystems. + */ +static int build_merkle_tree(struct file *filp, + const struct merkle_tree_params *params, + u8 *root_hash) +{ + struct inode *inode = file_inode(filp); + const u64 data_size = inode->i_size; + const int num_levels = params->num_levels; + struct block_buffer _buffers[1 + FS_VERITY_MAX_LEVELS + 1] = {}; + struct block_buffer *buffers = &_buffers[1]; + unsigned long level_offset[FS_VERITY_MAX_LEVELS]; + int level; + u64 offset; + int err; + + if (data_size == 0) { + /* Empty file is a special case; root hash is all 0's */ + memset(root_hash, 0, params->digest_size); + return 0; + } + + /* + * Allocate the block buffers. Buffer "-1" is for data blocks. + * Buffers 0 <= level < num_levels are for the actual tree levels. + * Buffer 'num_levels' is for the root hash. + */ + for (level = -1; level < num_levels; level++) { + buffers[level].data = kzalloc(params->block_size, GFP_KERNEL); + if (!buffers[level].data) { + err = -ENOMEM; + goto out; + } + } + buffers[num_levels].data = root_hash; + buffers[num_levels].is_root_hash = true; + + BUILD_BUG_ON(sizeof(level_offset) != sizeof(params->level_start)); + memcpy(level_offset, params->level_start, sizeof(level_offset)); + + /* Hash each data block, also hashing the tree blocks as they fill up */ + for (offset = 0; offset < data_size; offset += params->block_size) { + ssize_t bytes_read; + loff_t pos = offset; + + buffers[-1].filled = min_t(u64, params->block_size, + data_size - offset); + bytes_read = __kernel_read(filp, buffers[-1].data, + buffers[-1].filled, &pos); + if (bytes_read < 0) { + err = bytes_read; + fsverity_err(inode, "Error %d reading file data", err); + goto out; + } + if (bytes_read != buffers[-1].filled) { + err = -EINVAL; + fsverity_err(inode, "Short read of file data"); + goto out; + } + err = hash_one_block(params, &buffers[-1]); + if (err) + goto out; + for (level = 0; level < num_levels; level++) { + if (buffers[level].filled + params->digest_size <= + params->block_size) { + /* Next block at @level isn't full yet */ + break; + } + /* Next block at @level is full */ + + err = hash_one_block(params, &buffers[level]); + if (err) + goto out; + err = write_merkle_tree_block(filp, + buffers[level].data, + level_offset[level], + params); + if (err) + goto out; + level_offset[level]++; + } + if (fatal_signal_pending(current)) { + err = -EINTR; + goto out; + } + cond_resched(); + } + /* Finish all nonempty pending tree blocks. */ + for (level = 0; level < num_levels; level++) { + if (buffers[level].filled != 0) { + err = hash_one_block(params, &buffers[level]); + if (err) + goto out; + err = write_merkle_tree_block(filp, + buffers[level].data, + level_offset[level], + params); + if (err) + goto out; + } + } + /* The root hash was filled by the last call to hash_one_block(). */ + if (WARN_ON_ONCE(buffers[num_levels].filled != params->digest_size)) { + err = -EINVAL; + goto out; + } + err = 0; +out: + for (level = -1; level < num_levels; level++) + kfree(buffers[level].data); + return err; +} + +static int enable_verity(struct file *filp, + const struct fsverity_enable_arg *arg) +{ + struct inode *inode = file_inode(filp); + const struct fsverity_operations *vops = inode->i_sb->s_vop; + struct merkle_tree_params params = { }; + struct fsverity_descriptor *desc; + size_t desc_size = struct_size(desc, signature, arg->sig_size); + struct fsverity_info *vi; + int err; + + /* Start initializing the fsverity_descriptor */ + desc = kzalloc(desc_size, GFP_KERNEL); + if (!desc) + return -ENOMEM; + desc->version = 1; + desc->hash_algorithm = arg->hash_algorithm; + desc->log_blocksize = ilog2(arg->block_size); + + /* Get the salt if the user provided one */ + if (arg->salt_size && + copy_from_user(desc->salt, u64_to_user_ptr(arg->salt_ptr), + arg->salt_size)) { + err = -EFAULT; + goto out; + } + desc->salt_size = arg->salt_size; + + /* Get the builtin signature if the user provided one */ + if (arg->sig_size && + copy_from_user(desc->signature, u64_to_user_ptr(arg->sig_ptr), + arg->sig_size)) { + err = -EFAULT; + goto out; + } + desc->sig_size = cpu_to_le32(arg->sig_size); + + desc->data_size = cpu_to_le64(inode->i_size); + + /* Prepare the Merkle tree parameters */ + err = fsverity_init_merkle_tree_params(¶ms, inode, + arg->hash_algorithm, + desc->log_blocksize, + desc->salt, desc->salt_size); + if (err) + goto out; + + trace_fsverity_enable(inode, ¶ms); + + /* + * Start enabling verity on this file, serialized by the inode lock. + * Fail if verity is already enabled or is already being enabled. + */ + inode_lock(inode); + if (IS_VERITY(inode)) + err = -EEXIST; + else + err = vops->begin_enable_verity(filp); + inode_unlock(inode); + if (err) + goto out; + + /* + * Build the Merkle tree. Don't hold the inode lock during this, since + * on huge files this may take a very long time and we don't want to + * force unrelated syscalls like chown() to block forever. We don't + * need the inode lock here because deny_write_access() already prevents + * the file from being written to or truncated, and we still serialize + * ->begin_enable_verity() and ->end_enable_verity() using the inode + * lock and only allow one process to be here at a time on a given file. + */ + BUILD_BUG_ON(sizeof(desc->root_hash) < FS_VERITY_MAX_DIGEST_SIZE); + err = build_merkle_tree(filp, ¶ms, desc->root_hash); + if (err) { + fsverity_err(inode, "Error %d building Merkle tree", err); + goto rollback; + } + + /* + * Create the fsverity_info. Don't bother trying to save work by + * reusing the merkle_tree_params from above. Instead, just create the + * fsverity_info from the fsverity_descriptor as if it were just loaded + * from disk. This is simpler, and it serves as an extra check that the + * metadata we're writing is valid before actually enabling verity. + */ + vi = fsverity_create_info(inode, desc); + if (IS_ERR(vi)) { + err = PTR_ERR(vi); + goto rollback; + } + + trace_fsverity_tree_done(inode, vi, ¶ms); + + /* + * Add the fsverity_info into the hash table before finishing the + * initialization so that we don't have to undo the enabling when memory + * allocation for the hash table fails. This is safe because looking up + * the fsverity_info always first checks the S_VERITY flag on the inode, + * which will only be set at the very end of the ->end_enable_verity + * method. + */ + err = fsverity_set_info(vi); + if (err) { + fsverity_free_info(vi); + goto rollback; + } + + /* + * Tell the filesystem to finish enabling verity on the file. + * Serialized with ->begin_enable_verity() by the inode lock. The file + * system needs to set the S_VERITY flag on the inode at the very end of + * the method, at which point the fsverity information can be accessed + * by other threads. + */ + inode_lock(inode); + err = vops->end_enable_verity(filp, desc, desc_size, params.tree_size); + inode_unlock(inode); + if (err) { + fsverity_err(inode, "%ps() failed with err %d", + vops->end_enable_verity, err); + fsverity_remove_info(vi); + } else if (WARN_ON_ONCE(!IS_VERITY(inode))) { + fsverity_remove_info(vi); + err = -EINVAL; + } +out: + kfree(params.hashstate); + kfree(desc); + return err; + +rollback: + inode_lock(inode); + (void)vops->end_enable_verity(filp, NULL, 0, params.tree_size); + inode_unlock(inode); + goto out; +} + +/** + * fsverity_ioctl_enable() - enable verity on a file + * @filp: file to enable verity on + * @uarg: user pointer to fsverity_enable_arg + * + * Enable fs-verity on a file. See the "FS_IOC_ENABLE_VERITY" section of + * Documentation/filesystems/fsverity.rst for the documentation. + * + * Return: 0 on success, -errno on failure + */ +int fsverity_ioctl_enable(struct file *filp, const void __user *uarg) +{ + struct inode *inode = file_inode(filp); + struct fsverity_enable_arg arg; + int err; + + if (copy_from_user(&arg, uarg, sizeof(arg))) + return -EFAULT; + + if (arg.version != 1) + return -EINVAL; + + if (arg.__reserved1 || + memchr_inv(arg.__reserved2, 0, sizeof(arg.__reserved2))) + return -EINVAL; + + if (!is_power_of_2(arg.block_size)) + return -EINVAL; + + if (arg.salt_size > sizeof_field(struct fsverity_descriptor, salt)) + return -EMSGSIZE; + + if (arg.sig_size > FS_VERITY_MAX_SIGNATURE_SIZE) + return -EMSGSIZE; + + /* + * Require a regular file with write access. But the actual fd must + * still be readonly so that we can lock out all writers. This is + * needed to guarantee that no writable fds exist to the file once it + * has verity enabled, and to stabilize the data being hashed. + */ + + err = file_permission(filp, MAY_WRITE); + if (err) + return err; + /* + * __kernel_read() is used while building the Merkle tree. So, we can't + * allow file descriptors that were opened for ioctl access only, using + * the special nonstandard access mode 3. O_RDONLY only, please! + */ + if (!(filp->f_mode & FMODE_READ)) + return -EBADF; + + if (IS_APPEND(inode)) + return -EPERM; + + if (S_ISDIR(inode->i_mode)) + return -EISDIR; + + if (!S_ISREG(inode->i_mode)) + return -EINVAL; + + err = mnt_want_write_file(filp); + if (err) /* -EROFS */ + return err; + + err = deny_write_access(filp); + if (err) /* -ETXTBSY */ + goto out_drop_write; + + err = enable_verity(filp, &arg); + + /* + * We no longer drop the inode's pagecache after enabling verity. This + * used to be done to try to avoid a race condition where pages could be + * evicted after being used in the Merkle tree construction, then + * re-instantiated by a concurrent read. Such pages are unverified, and + * the backing storage could have filled them with different content, so + * they shouldn't be used to fulfill reads once verity is enabled. + * + * But, dropping the pagecache has a big performance impact, and it + * doesn't fully solve the race condition anyway. So for those reasons, + * and also because this race condition isn't very important relatively + * speaking (especially for small-ish files, where the chance of a page + * being used, evicted, *and* re-instantiated all while enabling verity + * is quite small), we no longer drop the inode's pagecache. + */ + + /* + * allow_write_access() is needed to pair with deny_write_access(). + * Regardless, the filesystem won't allow writing to verity files. + */ + allow_write_access(filp); +out_drop_write: + mnt_drop_write_file(filp); + return err; +} +EXPORT_SYMBOL_GPL(fsverity_ioctl_enable); diff --git a/fs/verity/fsverity_private.h b/fs/verity/fsverity_private.h new file mode 100644 index 000000000..881d46f25 --- /dev/null +++ b/fs/verity/fsverity_private.h @@ -0,0 +1,171 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * fs-verity: read-only file-based authenticity protection + * + * Copyright 2019 Google LLC + */ + +#ifndef _FSVERITY_PRIVATE_H +#define _FSVERITY_PRIVATE_H + +#define pr_fmt(fmt) "fs-verity: " fmt + +#include +#include + +/* + * Implementation limit: maximum depth of the Merkle tree. For now 8 is plenty; + * it's enough for over U64_MAX bytes of data using SHA-256 and 4K blocks. + */ +#define FS_VERITY_MAX_LEVELS 8 + +/* A hash algorithm supported by fs-verity */ +struct fsverity_hash_alg { + const char *name; /* crypto API name, e.g. sha256 */ + unsigned int digest_size; /* digest size in bytes, e.g. 32 for SHA-256 */ + unsigned int block_size; /* block size in bytes, e.g. 64 for SHA-256 */ + /* + * The HASH_ALGO_* constant for this algorithm. This is different from + * FS_VERITY_HASH_ALG_*, which uses a different numbering scheme. + */ + enum hash_algo algo_id; +}; + +union fsverity_hash_ctx { + struct sha256_ctx sha256; + struct sha512_ctx sha512; +}; + +/* Merkle tree parameters: hash algorithm, initial hash state, and topology */ +struct merkle_tree_params { + const struct fsverity_hash_alg *hash_alg; /* the hash algorithm */ + /* initial hash state if salted, NULL if unsalted */ + const union fsverity_hash_ctx *hashstate; + unsigned int digest_size; /* same as hash_alg->digest_size */ + unsigned int block_size; /* size of data and tree blocks */ + unsigned int hashes_per_block; /* number of hashes per tree block */ + unsigned int blocks_per_page; /* PAGE_SIZE / block_size */ + u8 log_digestsize; /* log2(digest_size) */ + u8 log_blocksize; /* log2(block_size) */ + u8 log_arity; /* log2(hashes_per_block) */ + u8 log_blocks_per_page; /* log2(blocks_per_page) */ + unsigned int num_levels; /* number of levels in Merkle tree */ + u64 tree_size; /* Merkle tree size in bytes */ + unsigned long tree_pages; /* Merkle tree size in pages */ + + /* the hash of an all-zeroes block */ + u8 zero_digest[FS_VERITY_MAX_DIGEST_SIZE]; + + /* + * Starting block index for each tree level, ordered from leaf level (0) + * to root level ('num_levels - 1') + */ + unsigned long level_start[FS_VERITY_MAX_LEVELS]; +}; + +/* + * fsverity_info - cached verity metadata for an inode + * + * When a verity file is first opened, an instance of this struct is allocated + * and a pointer to it is stored in the global hash table, indexed by the inode + * pointer value. It remains alive until the inode is evicted. It caches + * information about the Merkle tree that's needed to efficiently verify data + * read from the file. It also caches the file digest. The Merkle tree pages + * themselves are not cached here, but the filesystem may cache them. + */ +struct fsverity_info { + struct rhash_head rhash_head; + struct merkle_tree_params tree_params; + u8 root_hash[FS_VERITY_MAX_DIGEST_SIZE]; + u8 file_digest[FS_VERITY_MAX_DIGEST_SIZE]; + struct inode *inode; + unsigned long *hash_block_verified; +}; + +#define FS_VERITY_MAX_SIGNATURE_SIZE (FS_VERITY_MAX_DESCRIPTOR_SIZE - \ + sizeof(struct fsverity_descriptor)) + +/* hash_algs.c */ + +extern const struct fsverity_hash_alg fsverity_hash_algs[]; + +const struct fsverity_hash_alg *fsverity_get_hash_alg(const struct inode *inode, + unsigned int num); +union fsverity_hash_ctx * +fsverity_prepare_hash_state(const struct fsverity_hash_alg *alg, + const u8 *salt, size_t salt_size); +void fsverity_hash_block(const struct merkle_tree_params *params, + const void *data, u8 *out); +void fsverity_hash_buffer(const struct fsverity_hash_alg *alg, + const void *data, size_t size, u8 *out); +void __init fsverity_check_hash_algs(void); + +/* init.c */ + +void __printf(3, 4) __cold +fsverity_msg(const struct inode *inode, const char *level, + const char *fmt, ...); + +#define fsverity_warn(inode, fmt, ...) \ + fsverity_msg((inode), KERN_WARNING, fmt, ##__VA_ARGS__) +#define fsverity_err(inode, fmt, ...) \ + fsverity_msg((inode), KERN_ERR, fmt, ##__VA_ARGS__) + +/* measure.c */ + +#ifdef CONFIG_BPF_SYSCALL +void __init fsverity_init_bpf(void); +#else +static inline void fsverity_init_bpf(void) +{ +} +#endif + +/* open.c */ + +int fsverity_init_merkle_tree_params(struct merkle_tree_params *params, + const struct inode *inode, + unsigned int hash_algorithm, + unsigned int log_blocksize, + const u8 *salt, size_t salt_size); + +struct fsverity_info *fsverity_create_info(struct inode *inode, + struct fsverity_descriptor *desc); + +int fsverity_set_info(struct fsverity_info *vi); +void fsverity_free_info(struct fsverity_info *vi); +void fsverity_remove_info(struct fsverity_info *vi); + +int fsverity_get_descriptor(struct inode *inode, + struct fsverity_descriptor **desc_ret); + +void __init fsverity_init_info_cache(void); + +/* signature.c */ + +#ifdef CONFIG_FS_VERITY_BUILTIN_SIGNATURES +extern int fsverity_require_signatures; +int fsverity_verify_signature(const struct fsverity_info *vi, + const u8 *signature, size_t sig_size); + +void __init fsverity_init_signature(void); +#else /* !CONFIG_FS_VERITY_BUILTIN_SIGNATURES */ +static inline int +fsverity_verify_signature(const struct fsverity_info *vi, + const u8 *signature, size_t sig_size) +{ + return 0; +} + +static inline void fsverity_init_signature(void) +{ +} +#endif /* !CONFIG_FS_VERITY_BUILTIN_SIGNATURES */ + +/* verify.c */ + +void __init fsverity_init_workqueue(void); + +#include + +#endif /* _FSVERITY_PRIVATE_H */ diff --git a/fs/verity/hash_algs.c b/fs/verity/hash_algs.c new file mode 100644 index 000000000..de53e14c8 --- /dev/null +++ b/fs/verity/hash_algs.c @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs-verity hash algorithms + * + * Copyright 2019 Google LLC + */ + +#include "fsverity_private.h" + +/* The hash algorithms supported by fs-verity */ +const struct fsverity_hash_alg fsverity_hash_algs[] = { + [FS_VERITY_HASH_ALG_SHA256] = { + .name = "sha256", + .digest_size = SHA256_DIGEST_SIZE, + .block_size = SHA256_BLOCK_SIZE, + .algo_id = HASH_ALGO_SHA256, + }, + [FS_VERITY_HASH_ALG_SHA512] = { + .name = "sha512", + .digest_size = SHA512_DIGEST_SIZE, + .block_size = SHA512_BLOCK_SIZE, + .algo_id = HASH_ALGO_SHA512, + }, +}; + +/** + * fsverity_get_hash_alg() - get a hash algorithm by number + * @inode: optional inode for logging purposes + * @num: the hash algorithm number + * + * Get the struct fsverity_hash_alg for the given hash algorithm number. + * + * Return: pointer to the hash alg if it's known, otherwise NULL. + */ +const struct fsverity_hash_alg *fsverity_get_hash_alg(const struct inode *inode, + unsigned int num) +{ + if (num >= ARRAY_SIZE(fsverity_hash_algs) || + !fsverity_hash_algs[num].name) { + fsverity_warn(inode, "Unknown hash algorithm number: %u", num); + return NULL; + } + return &fsverity_hash_algs[num]; +} + +/** + * fsverity_prepare_hash_state() - precompute the initial hash state + * @alg: hash algorithm + * @salt: a salt which is to be prepended to all data to be hashed + * @salt_size: salt size in bytes + * + * Return: the kmalloc()'ed initial hash state, or NULL if out of memory. + */ +union fsverity_hash_ctx * +fsverity_prepare_hash_state(const struct fsverity_hash_alg *alg, + const u8 *salt, size_t salt_size) +{ + u8 *padded_salt = NULL; + size_t padded_salt_size; + union fsverity_hash_ctx ctx; + void *res = NULL; + + /* + * Zero-pad the salt to the next multiple of the input size of the hash + * algorithm's compression function, e.g. 64 bytes for SHA-256 or 128 + * bytes for SHA-512. This ensures that the hash algorithm won't have + * any bytes buffered internally after processing the salt, thus making + * salted hashing just as fast as unsalted hashing. + */ + padded_salt_size = round_up(salt_size, alg->block_size); + padded_salt = kzalloc(padded_salt_size, GFP_KERNEL); + if (!padded_salt) + return NULL; + memcpy(padded_salt, salt, salt_size); + + switch (alg->algo_id) { + case HASH_ALGO_SHA256: + sha256_init(&ctx.sha256); + sha256_update(&ctx.sha256, padded_salt, padded_salt_size); + res = kmemdup(&ctx.sha256, sizeof(ctx.sha256), GFP_KERNEL); + break; + case HASH_ALGO_SHA512: + sha512_init(&ctx.sha512); + sha512_update(&ctx.sha512, padded_salt, padded_salt_size); + res = kmemdup(&ctx.sha512, sizeof(ctx.sha512), GFP_KERNEL); + break; + default: + WARN_ON_ONCE(1); + } + kfree(padded_salt); + return res; +} + +/** + * fsverity_hash_block() - hash a single data or hash block + * @params: the Merkle tree's parameters + * @data: virtual address of a buffer containing the block to hash + * @out: output digest, size 'params->digest_size' bytes + * + * Hash a single data or hash block. The hash is salted if a salt is specified + * in the Merkle tree parameters. + */ +void fsverity_hash_block(const struct merkle_tree_params *params, + const void *data, u8 *out) +{ + union fsverity_hash_ctx ctx; + + if (!params->hashstate) { + fsverity_hash_buffer(params->hash_alg, data, params->block_size, + out); + return; + } + + switch (params->hash_alg->algo_id) { + case HASH_ALGO_SHA256: + ctx.sha256 = params->hashstate->sha256; + sha256_update(&ctx.sha256, data, params->block_size); + sha256_final(&ctx.sha256, out); + return; + case HASH_ALGO_SHA512: + ctx.sha512 = params->hashstate->sha512; + sha512_update(&ctx.sha512, data, params->block_size); + sha512_final(&ctx.sha512, out); + return; + default: + BUG(); + } +} + +/** + * fsverity_hash_buffer() - hash some data + * @alg: the hash algorithm to use + * @data: the data to hash + * @size: size of data to hash, in bytes + * @out: output digest, size 'alg->digest_size' bytes + */ +void fsverity_hash_buffer(const struct fsverity_hash_alg *alg, + const void *data, size_t size, u8 *out) +{ + switch (alg->algo_id) { + case HASH_ALGO_SHA256: + sha256(data, size, out); + return; + case HASH_ALGO_SHA512: + sha512(data, size, out); + return; + default: + BUG(); + } +} + +void __init fsverity_check_hash_algs(void) +{ + size_t i; + + /* + * Sanity check the hash algorithms (could be a build-time check, but + * they're in an array) + */ + for (i = 0; i < ARRAY_SIZE(fsverity_hash_algs); i++) { + const struct fsverity_hash_alg *alg = &fsverity_hash_algs[i]; + + if (!alg->name) + continue; + + /* + * 0 must never be allocated as an FS_VERITY_HASH_ALG_* value, + * as it is reserved for users that use 0 to mean unspecified or + * a default value. fs/verity/ itself doesn't care and doesn't + * have a default algorithm, but some users make use of this. + */ + BUG_ON(i == 0); + + BUG_ON(alg->digest_size > FS_VERITY_MAX_DIGEST_SIZE); + + /* + * For efficiency, the implementation currently assumes the + * digest and block sizes are powers of 2. This limitation can + * be lifted if the code is updated to handle other values. + */ + BUG_ON(!is_power_of_2(alg->digest_size)); + BUG_ON(!is_power_of_2(alg->block_size)); + + /* Verify that there is a valid mapping to HASH_ALGO_*. */ + BUG_ON(alg->algo_id == 0); + BUG_ON(alg->digest_size != hash_digest_size[alg->algo_id]); + } +} diff --git a/fs/verity/init.c b/fs/verity/init.c new file mode 100644 index 000000000..3aa55dec8 --- /dev/null +++ b/fs/verity/init.c @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fs-verity module initialization and logging + * + * Copyright 2019 Google LLC + */ + +#define CREATE_TRACE_POINTS +#include "fsverity_private.h" + +#include + +#ifdef CONFIG_SYSCTL +static const struct ctl_table fsverity_sysctl_table[] = { +#ifdef CONFIG_FS_VERITY_BUILTIN_SIGNATURES + { + .procname = "require_signatures", + .data = &fsverity_require_signatures, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE, + }, +#endif +}; + +static void __init fsverity_init_sysctl(void) +{ + register_sysctl_init("fs/verity", fsverity_sysctl_table); +} +#else /* CONFIG_SYSCTL */ +static inline void fsverity_init_sysctl(void) +{ +} +#endif /* !CONFIG_SYSCTL */ + +void fsverity_msg(const struct inode *inode, const char *level, + const char *fmt, ...) +{ + static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + struct va_format vaf; + va_list args; + + if (!__ratelimit(&rs)) + return; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + if (inode) + printk("%sfs-verity (%s, inode %llu): %pV\n", + level, inode->i_sb->s_id, inode->i_ino, &vaf); + else + printk("%sfs-verity: %pV\n", level, &vaf); + va_end(args); +} + +static int __init fsverity_init(void) +{ + fsverity_check_hash_algs(); + fsverity_init_info_cache(); + fsverity_init_workqueue(); + fsverity_init_sysctl(); + fsverity_init_signature(); + fsverity_init_bpf(); + return 0; +} +late_initcall(fsverity_init) diff --git a/fs/verity/measure.c b/fs/verity/measure.c new file mode 100644 index 000000000..68dfccb69 --- /dev/null +++ b/fs/verity/measure.c @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Ioctl to get a verity file's digest + * + * Copyright 2019 Google LLC + */ + +#include "fsverity_private.h" + +#include +#include +#include +#include + +/** + * fsverity_ioctl_measure() - get a verity file's digest + * @filp: file to get digest of + * @_uarg: user pointer to fsverity_digest + * + * Retrieve the file digest that the kernel is enforcing for reads from a verity + * file. See the "FS_IOC_MEASURE_VERITY" section of + * Documentation/filesystems/fsverity.rst for the documentation. + * + * Return: 0 on success, -errno on failure + */ +int fsverity_ioctl_measure(struct file *filp, void __user *_uarg) +{ + const struct inode *inode = file_inode(filp); + struct fsverity_digest __user *uarg = _uarg; + const struct fsverity_info *vi; + const struct fsverity_hash_alg *hash_alg; + struct fsverity_digest arg; + + vi = fsverity_get_info(inode); + if (!vi) + return -ENODATA; /* not a verity file */ + hash_alg = vi->tree_params.hash_alg; + + /* + * The user specifies the digest_size their buffer has space for; we can + * return the digest if it fits in the available space. We write back + * the actual size, which may be shorter than the user-specified size. + */ + + if (get_user(arg.digest_size, &uarg->digest_size)) + return -EFAULT; + if (arg.digest_size < hash_alg->digest_size) + return -EOVERFLOW; + + memset(&arg, 0, sizeof(arg)); + arg.digest_algorithm = hash_alg - fsverity_hash_algs; + arg.digest_size = hash_alg->digest_size; + + if (copy_to_user(uarg, &arg, sizeof(arg))) + return -EFAULT; + + if (copy_to_user(uarg->digest, vi->file_digest, hash_alg->digest_size)) + return -EFAULT; + + return 0; +} +EXPORT_SYMBOL_GPL(fsverity_ioctl_measure); + +/** + * fsverity_get_digest() - get a verity file's digest + * @inode: inode to get digest of + * @raw_digest: (out) the raw file digest + * @alg: (out) the digest's algorithm, as a FS_VERITY_HASH_ALG_* value + * @halg: (out) the digest's algorithm, as a HASH_ALGO_* value + * + * Retrieves the fsverity digest of the given file. The + * fsverity_ensure_verity_info() must be called on the inode beforehand; + * otherwise this function will not recognize when fsverity is enabled. + * + * The file's fsverity digest consists of @raw_digest in combination with either + * @alg or @halg. (The caller can choose which one of @alg or @halg to use.) + * + * IMPORTANT: Callers *must* make use of one of the two algorithm IDs, since + * @raw_digest is meaningless without knowing which algorithm it uses! fsverity + * provides no security guarantee for users who ignore the algorithm ID, even if + * they use the digest size (since algorithms can share the same digest size). + * + * Return: The size of the raw digest in bytes, or 0 if the file doesn't have + * fsverity enabled. + */ +int fsverity_get_digest(struct inode *inode, + u8 raw_digest[FS_VERITY_MAX_DIGEST_SIZE], + u8 *alg, enum hash_algo *halg) +{ + const struct fsverity_info *vi; + const struct fsverity_hash_alg *hash_alg; + + vi = fsverity_get_info(inode); + if (!vi) + return 0; /* not a verity file */ + + hash_alg = vi->tree_params.hash_alg; + memcpy(raw_digest, vi->file_digest, hash_alg->digest_size); + if (alg) + *alg = hash_alg - fsverity_hash_algs; + if (halg) + *halg = hash_alg->algo_id; + return hash_alg->digest_size; +} +EXPORT_SYMBOL_GPL(fsverity_get_digest); + +#ifdef CONFIG_BPF_SYSCALL + +/* bpf kfuncs */ +__bpf_kfunc_start_defs(); + +/** + * bpf_get_fsverity_digest: read fsverity digest of file + * @file: file to get digest from + * @digest_p: (out) dynptr for struct fsverity_digest + * + * Read fsverity_digest of *file* into *digest_ptr*. + * + * Return: 0 on success, a negative value on error. + */ +__bpf_kfunc int bpf_get_fsverity_digest(struct file *file, const struct bpf_dynptr *digest_p) +{ + const struct bpf_dynptr_kern *digest_ptr = (struct bpf_dynptr_kern *)digest_p; + const struct inode *inode = file_inode(file); + u64 dynptr_sz = __bpf_dynptr_size(digest_ptr); + struct fsverity_digest *arg; + const struct fsverity_info *vi; + const struct fsverity_hash_alg *hash_alg; + u64 out_digest_sz; + + if (dynptr_sz < sizeof(struct fsverity_digest)) + return -EINVAL; + + arg = __bpf_dynptr_data_rw(digest_ptr, dynptr_sz); + if (!arg) + return -EINVAL; + + if (!IS_ALIGNED((uintptr_t)arg, __alignof__(*arg))) + return -EINVAL; + + vi = fsverity_get_info(inode); + if (!vi) + return -ENODATA; /* not a verity file */ + + hash_alg = vi->tree_params.hash_alg; + + out_digest_sz = dynptr_sz - sizeof(struct fsverity_digest); + if (out_digest_sz < hash_alg->digest_size) + return -EOVERFLOW; + + arg->digest_algorithm = hash_alg - fsverity_hash_algs; + arg->digest_size = hash_alg->digest_size; + + /* copy digest */ + memcpy(arg->digest, vi->file_digest, hash_alg->digest_size); + + /* fill the extra buffer with zeros */ + if (out_digest_sz > hash_alg->digest_size) + memset(arg->digest + hash_alg->digest_size, 0, + out_digest_sz - hash_alg->digest_size); + + return 0; +} + +__bpf_kfunc_end_defs(); + +BTF_KFUNCS_START(fsverity_set_ids) +BTF_ID_FLAGS(func, bpf_get_fsverity_digest) +BTF_KFUNCS_END(fsverity_set_ids) + +static int bpf_get_fsverity_digest_filter(const struct bpf_prog *prog, u32 kfunc_id) +{ + if (!btf_id_set8_contains(&fsverity_set_ids, kfunc_id)) + return 0; + + /* Only allow to attach from LSM hooks, to avoid recursion */ + return prog->type != BPF_PROG_TYPE_LSM ? -EACCES : 0; +} + +static const struct btf_kfunc_id_set bpf_fsverity_set = { + .owner = THIS_MODULE, + .set = &fsverity_set_ids, + .filter = bpf_get_fsverity_digest_filter, +}; + +void __init fsverity_init_bpf(void) +{ + register_btf_kfunc_id_set(BPF_PROG_TYPE_LSM, &bpf_fsverity_set); +} + +#endif /* CONFIG_BPF_SYSCALL */ diff --git a/fs/verity/open.c b/fs/verity/open.c new file mode 100644 index 000000000..d0c56a7fa --- /dev/null +++ b/fs/verity/open.c @@ -0,0 +1,425 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Opening fs-verity files + * + * Copyright 2019 Google LLC + */ + +#include "fsverity_private.h" + +#include +#include +#include + +static struct kmem_cache *fsverity_info_cachep; +static struct rhashtable fsverity_info_hash; + +static const struct rhashtable_params fsverity_info_hash_params = { + .key_len = sizeof_field(struct fsverity_info, inode), + .key_offset = offsetof(struct fsverity_info, inode), + .head_offset = offsetof(struct fsverity_info, rhash_head), + .automatic_shrinking = true, +}; + +/** + * fsverity_init_merkle_tree_params() - initialize Merkle tree parameters + * @params: the parameters struct to initialize + * @inode: the inode for which the Merkle tree is being built + * @hash_algorithm: number of hash algorithm to use + * @log_blocksize: log base 2 of block size to use + * @salt: pointer to salt (optional) + * @salt_size: size of salt, possibly 0 + * + * Validate the hash algorithm and block size, then compute the tree topology + * (num levels, num blocks in each level, etc.) and initialize @params. + * + * Return: 0 on success, -errno on failure + */ +int fsverity_init_merkle_tree_params(struct merkle_tree_params *params, + const struct inode *inode, + unsigned int hash_algorithm, + unsigned int log_blocksize, + const u8 *salt, size_t salt_size) +{ + const struct fsverity_hash_alg *hash_alg; + int err; + u64 blocks; + u64 blocks_in_level[FS_VERITY_MAX_LEVELS]; + u64 offset; + int level; + + memset(params, 0, sizeof(*params)); + + hash_alg = fsverity_get_hash_alg(inode, hash_algorithm); + if (!hash_alg) + return -EINVAL; + params->hash_alg = hash_alg; + params->digest_size = hash_alg->digest_size; + + if (salt_size) { + params->hashstate = + fsverity_prepare_hash_state(hash_alg, salt, salt_size); + if (!params->hashstate) { + err = -ENOMEM; + goto out_err; + } + } + + /* + * fs/verity/ directly assumes that the Merkle tree block size is a + * power of 2 less than or equal to PAGE_SIZE. Another restriction + * arises from the interaction between fs/verity/ and the filesystems + * themselves: filesystems expect to be able to verify a single + * filesystem block of data at a time. Therefore, the Merkle tree block + * size must also be less than or equal to the filesystem block size. + * + * The above are the only hard limitations, so in theory the Merkle tree + * block size could be as small as twice the digest size. However, + * that's not useful, and it would result in some unusually deep and + * large Merkle trees. So we currently require that the Merkle tree + * block size be at least 1024 bytes. That's small enough to test the + * sub-page block case on systems with 4K pages, but not too small. + */ + if (log_blocksize < 10 || log_blocksize > PAGE_SHIFT || + log_blocksize > inode->i_blkbits) { + fsverity_warn(inode, "Unsupported log_blocksize: %u", + log_blocksize); + err = -EINVAL; + goto out_err; + } + params->log_blocksize = log_blocksize; + params->block_size = 1 << log_blocksize; + params->log_blocks_per_page = PAGE_SHIFT - log_blocksize; + params->blocks_per_page = 1 << params->log_blocks_per_page; + + if (WARN_ON_ONCE(!is_power_of_2(params->digest_size))) { + err = -EINVAL; + goto out_err; + } + if (params->block_size < 2 * params->digest_size) { + fsverity_warn(inode, + "Merkle tree block size (%u) too small for hash algorithm \"%s\"", + params->block_size, hash_alg->name); + err = -EINVAL; + goto out_err; + } + params->log_digestsize = ilog2(params->digest_size); + params->log_arity = log_blocksize - params->log_digestsize; + params->hashes_per_block = 1 << params->log_arity; + + /* + * Compute the number of levels in the Merkle tree and create a map from + * level to the starting block of that level. Level 'num_levels - 1' is + * the root and is stored first. Level 0 is the level directly "above" + * the data blocks and is stored last. + */ + + /* Compute number of levels and the number of blocks in each level */ + blocks = ((u64)inode->i_size + params->block_size - 1) >> log_blocksize; + while (blocks > 1) { + if (params->num_levels >= FS_VERITY_MAX_LEVELS) { + fsverity_err(inode, "Too many levels in Merkle tree"); + err = -EFBIG; + goto out_err; + } + blocks = (blocks + params->hashes_per_block - 1) >> + params->log_arity; + blocks_in_level[params->num_levels++] = blocks; + } + + /* Compute the starting block of each level */ + offset = 0; + for (level = (int)params->num_levels - 1; level >= 0; level--) { + params->level_start[level] = offset; + offset += blocks_in_level[level]; + } + + /* + * With block_size != PAGE_SIZE, an in-memory bitmap will need to be + * allocated to track the "verified" status of hash blocks. Don't allow + * this bitmap to get too large. For now, limit it to 1 MiB, which + * limits the file size to about 4.4 TB with SHA-256 and 4K blocks. + * + * Together with the fact that the data, and thus also the Merkle tree, + * cannot have more than ULONG_MAX pages, this implies that hash block + * indices can always fit in an 'unsigned long'. But to be safe, we + * explicitly check for that too. Note, this is only for hash block + * indices; data block indices might not fit in an 'unsigned long'. + */ + if ((params->block_size != PAGE_SIZE && offset > 1 << 23) || + offset > ULONG_MAX) { + fsverity_err(inode, "Too many blocks in Merkle tree"); + err = -EFBIG; + goto out_err; + } + + fsverity_hash_block(params, page_address(ZERO_PAGE(0)), + params->zero_digest); + + params->tree_size = offset << log_blocksize; + params->tree_pages = PAGE_ALIGN(params->tree_size) >> PAGE_SHIFT; + return 0; + +out_err: + kfree(params->hashstate); + memset(params, 0, sizeof(*params)); + return err; +} + +/* + * Compute the file digest by hashing the fsverity_descriptor excluding the + * builtin signature and with the sig_size field set to 0. + */ +static void compute_file_digest(const struct fsverity_hash_alg *hash_alg, + struct fsverity_descriptor *desc, + u8 *file_digest) +{ + __le32 sig_size = desc->sig_size; + + desc->sig_size = 0; + fsverity_hash_buffer(hash_alg, desc, sizeof(*desc), file_digest); + desc->sig_size = sig_size; +} + +/* + * Create a new fsverity_info from the given fsverity_descriptor (with optional + * appended builtin signature), and check the signature if present. The + * fsverity_descriptor must have already undergone basic validation. + */ +struct fsverity_info *fsverity_create_info(struct inode *inode, + struct fsverity_descriptor *desc) +{ + struct fsverity_info *vi; + int err; + + vi = kmem_cache_zalloc(fsverity_info_cachep, GFP_KERNEL); + if (!vi) + return ERR_PTR(-ENOMEM); + vi->inode = inode; + + err = fsverity_init_merkle_tree_params(&vi->tree_params, inode, + desc->hash_algorithm, + desc->log_blocksize, + desc->salt, desc->salt_size); + if (err) { + fsverity_err(inode, + "Error %d initializing Merkle tree parameters", + err); + goto fail; + } + + memcpy(vi->root_hash, desc->root_hash, vi->tree_params.digest_size); + + compute_file_digest(vi->tree_params.hash_alg, desc, vi->file_digest); + + err = fsverity_verify_signature(vi, desc->signature, + le32_to_cpu(desc->sig_size)); + if (err) + goto fail; + + if (vi->tree_params.block_size != PAGE_SIZE) { + /* + * When the Merkle tree block size and page size differ, we use + * a bitmap to keep track of which hash blocks have been + * verified. This bitmap must contain one bit per hash block, + * including alignment to a page boundary at the end. + * + * Eventually, to support extremely large files in an efficient + * way, it might be necessary to make pages of this bitmap + * reclaimable. But for now, simply allocating the whole bitmap + * is a simple solution that works well on the files on which + * fsverity is realistically used. E.g., with SHA-256 and 4K + * blocks, a 100MB file only needs a 24-byte bitmap, and the + * bitmap for any file under 17GB fits in a 4K page. + */ + unsigned long num_bits = + vi->tree_params.tree_pages << + vi->tree_params.log_blocks_per_page; + + vi->hash_block_verified = kvcalloc(BITS_TO_LONGS(num_bits), + sizeof(unsigned long), + GFP_KERNEL); + if (!vi->hash_block_verified) { + err = -ENOMEM; + goto fail; + } + } + + return vi; + +fail: + fsverity_free_info(vi); + return ERR_PTR(err); +} + +int fsverity_set_info(struct fsverity_info *vi) +{ + return rhashtable_lookup_insert_fast(&fsverity_info_hash, + &vi->rhash_head, + fsverity_info_hash_params); +} + +struct fsverity_info *__fsverity_get_info(const struct inode *inode) +{ + return rhashtable_lookup_fast(&fsverity_info_hash, &inode, + fsverity_info_hash_params); +} +EXPORT_SYMBOL_GPL(__fsverity_get_info); + +static bool validate_fsverity_descriptor(struct inode *inode, + const struct fsverity_descriptor *desc, + size_t desc_size) +{ + if (desc_size < sizeof(*desc)) { + fsverity_err(inode, "Unrecognized descriptor size: %zu bytes", + desc_size); + return false; + } + + if (desc->version != 1) { + fsverity_err(inode, "Unrecognized descriptor version: %u", + desc->version); + return false; + } + + if (memchr_inv(desc->__reserved, 0, sizeof(desc->__reserved))) { + fsverity_err(inode, "Reserved bits set in descriptor"); + return false; + } + + if (desc->salt_size > sizeof(desc->salt)) { + fsverity_err(inode, "Invalid salt_size: %u", desc->salt_size); + return false; + } + + if (le64_to_cpu(desc->data_size) != inode->i_size) { + fsverity_err(inode, + "Wrong data_size: %llu (desc) != %lld (inode)", + le64_to_cpu(desc->data_size), inode->i_size); + return false; + } + + if (le32_to_cpu(desc->sig_size) > desc_size - sizeof(*desc)) { + fsverity_err(inode, "Signature overflows verity descriptor"); + return false; + } + + return true; +} + +/* + * Read the inode's fsverity_descriptor (with optional appended builtin + * signature) from the filesystem, and do basic validation of it. + */ +int fsverity_get_descriptor(struct inode *inode, + struct fsverity_descriptor **desc_ret) +{ + int res; + struct fsverity_descriptor *desc; + + res = inode->i_sb->s_vop->get_verity_descriptor(inode, NULL, 0); + if (res < 0) { + fsverity_err(inode, + "Error %d getting verity descriptor size", res); + return res; + } + if (res > FS_VERITY_MAX_DESCRIPTOR_SIZE) { + fsverity_err(inode, "Verity descriptor is too large (%d bytes)", + res); + return -EMSGSIZE; + } + desc = kmalloc(res, GFP_KERNEL); + if (!desc) + return -ENOMEM; + res = inode->i_sb->s_vop->get_verity_descriptor(inode, desc, res); + if (res < 0) { + fsverity_err(inode, "Error %d reading verity descriptor", res); + kfree(desc); + return res; + } + + if (!validate_fsverity_descriptor(inode, desc, res)) { + kfree(desc); + return -EINVAL; + } + + *desc_ret = desc; + return 0; +} + +static int ensure_verity_info(struct inode *inode) +{ + struct fsverity_info *vi = fsverity_get_info(inode), *found; + struct fsverity_descriptor *desc; + int err; + + if (vi) + return 0; + + err = fsverity_get_descriptor(inode, &desc); + if (err) + return err; + + vi = fsverity_create_info(inode, desc); + if (IS_ERR(vi)) { + err = PTR_ERR(vi); + goto out_free_desc; + } + + /* + * Multiple tasks may race to set the inode's verity info, in which case + * we might find an existing fsverity_info in the hash table. + */ + found = rhashtable_lookup_get_insert_fast(&fsverity_info_hash, + &vi->rhash_head, + fsverity_info_hash_params); + if (found) { + fsverity_free_info(vi); + if (IS_ERR(found)) + err = PTR_ERR(found); + } + +out_free_desc: + kfree(desc); + return err; +} + +int __fsverity_file_open(struct inode *inode, struct file *filp) +{ + if (filp->f_mode & FMODE_WRITE) + return -EPERM; + return ensure_verity_info(inode); +} +EXPORT_SYMBOL_GPL(__fsverity_file_open); + +void fsverity_free_info(struct fsverity_info *vi) +{ + kfree(vi->tree_params.hashstate); + kvfree(vi->hash_block_verified); + kmem_cache_free(fsverity_info_cachep, vi); +} + +void fsverity_remove_info(struct fsverity_info *vi) +{ + rhashtable_remove_fast(&fsverity_info_hash, &vi->rhash_head, + fsverity_info_hash_params); + fsverity_free_info(vi); +} + +void fsverity_cleanup_inode(struct inode *inode) +{ + struct fsverity_info *vi = fsverity_get_info(inode); + + if (vi) + fsverity_remove_info(vi); +} + +void __init fsverity_init_info_cache(void) +{ + if (rhashtable_init(&fsverity_info_hash, &fsverity_info_hash_params)) + panic("failed to initialize fsverity hash\n"); + fsverity_info_cachep = KMEM_CACHE_USERCOPY( + fsverity_info, + SLAB_RECLAIM_ACCOUNT | SLAB_PANIC, + file_digest); +} diff --git a/fs/verity/pagecache.c b/fs/verity/pagecache.c new file mode 100644 index 000000000..99f5f53ee --- /dev/null +++ b/fs/verity/pagecache.c @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2019 Google LLC + */ +#include "fsverity_private.h" + +#include +#include +#include + +/** + * generic_read_merkle_tree_page - generic ->read_merkle_tree_page helper + * @inode: inode containing the Merkle tree + * @index: 0-based index of the Merkle tree page in the inode + * + * The caller needs to adjust @index from the Merkle-tree relative index passed + * to ->read_merkle_tree_page to the actual index where the Merkle tree is + * stored in the page cache for @inode. + */ +struct page *generic_read_merkle_tree_page(struct inode *inode, pgoff_t index) +{ + struct folio *folio; + + folio = read_mapping_folio(inode->i_mapping, index, NULL); + if (IS_ERR(folio)) + return ERR_CAST(folio); + return folio_file_page(folio, index); +} +EXPORT_SYMBOL_GPL(generic_read_merkle_tree_page); + +/** + * generic_readahead_merkle_tree() - generic ->readahead_merkle_tree helper + * @inode: inode containing the Merkle tree + * @index: 0-based index of the first Merkle tree page to read ahead in the + * inode + * @nr_pages: the number of Merkle tree pages that should be read ahead + * + * The caller needs to adjust @index from the Merkle-tree relative index passed + * to ->read_merkle_tree_page to the actual index where the Merkle tree is + * stored in the page cache for @inode. + */ +void generic_readahead_merkle_tree(struct inode *inode, pgoff_t index, + unsigned long nr_pages) +{ + struct folio *folio; + + lockdep_assert_held(&inode->i_mapping->invalidate_lock); + + folio = __filemap_get_folio(inode->i_mapping, index, FGP_ACCESSED, 0); + if (folio == ERR_PTR(-ENOENT) || + (!IS_ERR(folio) && !folio_test_uptodate(folio))) { + DEFINE_READAHEAD(ractl, NULL, NULL, inode->i_mapping, index); + + page_cache_ra_unbounded(&ractl, nr_pages, 0); + } + if (!IS_ERR(folio)) + folio_put(folio); +} +EXPORT_SYMBOL_GPL(generic_readahead_merkle_tree); + +/** + * fsverity_fill_zerohash() - fill folio with hashes of zero data block + * @folio: folio to fill + * @offset: offset in the folio to start + * @len: length of the range to fill with hashes + * @vi: fsverity info + */ +void fsverity_fill_zerohash(struct folio *folio, size_t offset, size_t len, + struct fsverity_info *vi) +{ + size_t off = offset; + + WARN_ON_ONCE(!IS_ALIGNED(offset, vi->tree_params.digest_size)); + WARN_ON_ONCE(!IS_ALIGNED(len, vi->tree_params.digest_size)); + + for (; off < (offset + len); off += vi->tree_params.digest_size) + memcpy_to_folio(folio, off, vi->tree_params.zero_digest, + vi->tree_params.digest_size); +} +EXPORT_SYMBOL_GPL(fsverity_fill_zerohash); diff --git a/fs/verity/read_metadata.c b/fs/verity/read_metadata.c new file mode 100644 index 000000000..b4c089243 --- /dev/null +++ b/fs/verity/read_metadata.c @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Ioctl to read verity metadata + * + * Copyright 2021 Google LLC + */ + +#include "fsverity_private.h" + +#include +#include +#include +#include +#include + +static int fsverity_read_merkle_tree(struct inode *inode, + const struct fsverity_info *vi, + void __user *buf, u64 offset, int length) +{ + const struct fsverity_operations *vops = inode->i_sb->s_vop; + u64 end_offset; + unsigned int offs_in_page; + pgoff_t index, last_index; + int retval = 0; + int err = 0; + + end_offset = min(offset + length, vi->tree_params.tree_size); + if (offset >= end_offset) + return 0; + offs_in_page = offset_in_page(offset); + index = offset >> PAGE_SHIFT; + last_index = (end_offset - 1) >> PAGE_SHIFT; + + /* + * Kick off readahead for the range we are going to read to ensure a + * single large sequential read instead of lots of small ones. + */ + if (inode->i_sb->s_vop->readahead_merkle_tree) { + filemap_invalidate_lock_shared(inode->i_mapping); + inode->i_sb->s_vop->readahead_merkle_tree( + inode, index, last_index - index + 1); + filemap_invalidate_unlock_shared(inode->i_mapping); + } + + /* + * Iterate through each Merkle tree page in the requested range and copy + * the requested portion to userspace. Note that the Merkle tree block + * size isn't important here, as we are returning a byte stream; i.e., + * we can just work with pages even if the tree block size != PAGE_SIZE. + */ + for (; index <= last_index; index++) { + unsigned int bytes_to_copy = min_t(u64, end_offset - offset, + PAGE_SIZE - offs_in_page); + struct page *page; + const void *virt; + + page = vops->read_merkle_tree_page(inode, index); + if (IS_ERR(page)) { + err = PTR_ERR(page); + fsverity_err(inode, + "Error %d reading Merkle tree page %lu", + err, index); + break; + } + + virt = kmap_local_page(page); + if (copy_to_user(buf, virt + offs_in_page, bytes_to_copy)) { + kunmap_local(virt); + put_page(page); + err = -EFAULT; + break; + } + kunmap_local(virt); + put_page(page); + + retval += bytes_to_copy; + buf += bytes_to_copy; + offset += bytes_to_copy; + + if (fatal_signal_pending(current)) { + err = -EINTR; + break; + } + cond_resched(); + offs_in_page = 0; + } + return retval ? retval : err; +} + +/* Copy the requested portion of the buffer to userspace. */ +static int fsverity_read_buffer(void __user *dst, u64 offset, int length, + const void *src, size_t src_length) +{ + if (offset >= src_length) + return 0; + src += offset; + src_length -= offset; + + length = min_t(size_t, length, src_length); + + if (copy_to_user(dst, src, length)) + return -EFAULT; + + return length; +} + +static int fsverity_read_descriptor(struct inode *inode, + void __user *buf, u64 offset, int length) +{ + struct fsverity_descriptor *desc; + size_t desc_size; + int res; + + res = fsverity_get_descriptor(inode, &desc); + if (res) + return res; + + /* don't include the builtin signature */ + desc_size = offsetof(struct fsverity_descriptor, signature); + desc->sig_size = 0; + + res = fsverity_read_buffer(buf, offset, length, desc, desc_size); + + kfree(desc); + return res; +} + +static int fsverity_read_signature(struct inode *inode, + void __user *buf, u64 offset, int length) +{ + struct fsverity_descriptor *desc; + int res; + + res = fsverity_get_descriptor(inode, &desc); + if (res) + return res; + + if (desc->sig_size == 0) { + res = -ENODATA; + goto out; + } + + /* + * Include only the builtin signature. fsverity_get_descriptor() + * already verified that sig_size is in-bounds. + */ + res = fsverity_read_buffer(buf, offset, length, desc->signature, + le32_to_cpu(desc->sig_size)); +out: + kfree(desc); + return res; +} + +/** + * fsverity_ioctl_read_metadata() - read verity metadata from a file + * @filp: file to read the metadata from + * @uarg: user pointer to fsverity_read_metadata_arg + * + * Return: length read on success, 0 on EOF, -errno on failure + */ +int fsverity_ioctl_read_metadata(struct file *filp, const void __user *uarg) +{ + struct inode *inode = file_inode(filp); + const struct fsverity_info *vi; + struct fsverity_read_metadata_arg arg; + int length; + void __user *buf; + + vi = fsverity_get_info(inode); + if (!vi) + return -ENODATA; /* not a verity file */ + /* + * Note that we don't have to explicitly check that the file is open for + * reading, since verity files can only be opened for reading. + */ + + if (copy_from_user(&arg, uarg, sizeof(arg))) + return -EFAULT; + + if (arg.__reserved) + return -EINVAL; + + /* offset + length must not overflow. */ + if (arg.offset + arg.length < arg.offset) + return -EINVAL; + + /* Ensure that the return value will fit in INT_MAX. */ + length = min_t(u64, arg.length, INT_MAX); + + buf = u64_to_user_ptr(arg.buf_ptr); + + switch (arg.metadata_type) { + case FS_VERITY_METADATA_TYPE_MERKLE_TREE: + return fsverity_read_merkle_tree(inode, vi, buf, arg.offset, + length); + case FS_VERITY_METADATA_TYPE_DESCRIPTOR: + return fsverity_read_descriptor(inode, buf, arg.offset, length); + case FS_VERITY_METADATA_TYPE_SIGNATURE: + return fsverity_read_signature(inode, buf, arg.offset, length); + default: + return -EINVAL; + } +} +EXPORT_SYMBOL_GPL(fsverity_ioctl_read_metadata); diff --git a/fs/verity/signature.c b/fs/verity/signature.c new file mode 100644 index 000000000..0302a4e50 --- /dev/null +++ b/fs/verity/signature.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Verification of builtin signatures + * + * Copyright 2019 Google LLC + */ + +/* + * This file implements verification of fs-verity builtin signatures. Please + * take great care before using this feature. It is not the only way to do + * signatures with fs-verity, and the alternatives (such as userspace signature + * verification, and IMA appraisal) can be much better. For details about the + * limitations of this feature, see Documentation/filesystems/fsverity.rst. + */ + +#include "fsverity_private.h" + +#include +#include +#include +#include +#include + +/* + * /proc/sys/fs/verity/require_signatures + * If 1, all verity files must have a valid builtin signature. + */ +int fsverity_require_signatures; + +/* + * Keyring that contains the trusted X.509 certificates. + * + * Only root (kuid=0) can modify this. Also, root may use + * keyctl_restrict_keyring() to prevent any more additions. + */ +static struct key *fsverity_keyring; + +/** + * fsverity_verify_signature() - check a verity file's signature + * @vi: the file's fsverity_info + * @signature: the file's built-in signature + * @sig_size: size of signature in bytes, or 0 if no signature + * + * If the file includes a signature of its fs-verity file digest, verify it + * against the certificates in the fs-verity keyring. Note that signatures + * are verified regardless of the state of the 'fsverity_require_signatures' + * variable and the LSM subsystem relies on this behavior to help enforce + * file integrity policies. Please discuss changes with the LSM list + * (thank you!). + * + * Return: 0 on success (signature valid or not required); -errno on failure + */ +int fsverity_verify_signature(const struct fsverity_info *vi, + const u8 *signature, size_t sig_size) +{ + const struct inode *inode = vi->inode; + const struct fsverity_hash_alg *hash_alg = vi->tree_params.hash_alg; + struct fsverity_formatted_digest *d; + int err; + + if (sig_size == 0) { + if (fsverity_require_signatures) { + fsverity_err(inode, + "require_signatures=1, rejecting unsigned file!"); + return -EPERM; + } + return 0; + } + + if (fsverity_keyring->keys.nr_leaves_on_tree == 0) { + /* + * The ".fs-verity" keyring is empty, due to builtin signatures + * being supported by the kernel but not actually being used. + * In this case, verify_pkcs7_signature() would always return an + * error, usually ENOKEY. It could also be EBADMSG if the + * PKCS#7 is malformed, but that isn't very important to + * distinguish. So, just skip to ENOKEY to avoid the attack + * surface of the PKCS#7 parser, which would otherwise be + * reachable by any task able to execute FS_IOC_ENABLE_VERITY. + */ + fsverity_err(inode, + "fs-verity keyring is empty, rejecting signed file!"); + return -ENOKEY; + } + + d = kzalloc(sizeof(*d) + hash_alg->digest_size, GFP_KERNEL); + if (!d) + return -ENOMEM; + memcpy(d->magic, "FSVerity", 8); + d->digest_algorithm = cpu_to_le16(hash_alg - fsverity_hash_algs); + d->digest_size = cpu_to_le16(hash_alg->digest_size); + memcpy(d->digest, vi->file_digest, hash_alg->digest_size); + + err = verify_pkcs7_signature(d, sizeof(*d) + hash_alg->digest_size, + signature, sig_size, fsverity_keyring, + VERIFYING_UNSPECIFIED_SIGNATURE, + NULL, NULL); + kfree(d); + + if (err) { + if (err == -ENOKEY) + fsverity_err(inode, + "File's signing cert isn't in the fs-verity keyring"); + else if (err == -EKEYREJECTED) + fsverity_err(inode, "Incorrect file signature"); + else if (err == -EBADMSG) + fsverity_err(inode, "Malformed file signature"); + else + fsverity_err(inode, "Error %d verifying file signature", + err); + return err; + } + + err = security_inode_setintegrity(inode, + LSM_INT_FSVERITY_BUILTINSIG_VALID, + signature, + sig_size); + + if (err) { + fsverity_err(inode, "Error %d exposing file signature to LSMs", + err); + return err; + } + + return 0; +} + +void __init fsverity_init_signature(void) +{ + fsverity_keyring = + keyring_alloc(".fs-verity", KUIDT_INIT(0), KGIDT_INIT(0), + current_cred(), KEY_POS_SEARCH | + KEY_USR_VIEW | KEY_USR_READ | KEY_USR_WRITE | + KEY_USR_SEARCH | KEY_USR_SETATTR, + KEY_ALLOC_NOT_IN_QUOTA, NULL, NULL); + if (IS_ERR(fsverity_keyring)) + panic("failed to allocate \".fs-verity\" keyring"); +} diff --git a/fs/verity/verify.c b/fs/verity/verify.c new file mode 100644 index 000000000..4004a1d42 --- /dev/null +++ b/fs/verity/verify.c @@ -0,0 +1,499 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Data verification functions, i.e. hooks for ->readahead() + * + * Copyright 2019 Google LLC + */ + +#include "fsverity_private.h" + +#include +#include + +#define FS_VERITY_MAX_PENDING_BLOCKS 2 + +struct fsverity_pending_block { + const void *data; + u64 pos; + u8 real_hash[FS_VERITY_MAX_DIGEST_SIZE]; +}; + +struct fsverity_verification_context { + struct fsverity_info *vi; + + /* + * This is the queue of data blocks that are pending verification. When + * the crypto layer supports interleaved hashing, we allow multiple + * blocks to be queued up in order to utilize it. This can improve + * performance significantly vs. sequential hashing of each block. + */ + int num_pending; + int max_pending; + struct fsverity_pending_block + pending_blocks[FS_VERITY_MAX_PENDING_BLOCKS]; +}; + +static struct workqueue_struct *fsverity_read_workqueue; + +/** + * fsverity_readahead() - kick off readahead on fsverity hashes + * @vi: fsverity_info for the inode to be read + * @index: first file data page index that is being read + * @nr_pages: number of file data pages to be read + * + * Start readahead on the fsverity hashes that are needed to verify the file + * data in the range from @index to @index + @nr_pages (exclusive upper bound). + * + * To be called from the file systems' ->read_folio and ->readahead methods to + * ensure that the hashes are already cached on completion of the file data + * read if possible. + */ +void fsverity_readahead(struct fsverity_info *vi, pgoff_t index, + unsigned long nr_pages) +{ + struct inode *inode = vi->inode; + const struct merkle_tree_params *params = &vi->tree_params; + u64 start_hidx = (u64)index << params->log_blocks_per_page; + u64 end_hidx = + (((u64)index + nr_pages) << params->log_blocks_per_page) - 1; + int level; + + if (!inode->i_sb->s_vop->readahead_merkle_tree) + return; + + for (level = 0; level < params->num_levels; level++) { + unsigned long level_start = params->level_start[level]; + unsigned long next_start_hidx = start_hidx >> params->log_arity; + unsigned long next_end_hidx = end_hidx >> params->log_arity; + pgoff_t start_idx = (level_start + next_start_hidx) >> + params->log_blocks_per_page; + pgoff_t end_idx = (level_start + next_end_hidx) >> + params->log_blocks_per_page; + + inode->i_sb->s_vop->readahead_merkle_tree( + inode, start_idx, end_idx - start_idx + 1); + + start_hidx = next_start_hidx; + end_hidx = next_end_hidx; + } +} +EXPORT_SYMBOL_GPL(fsverity_readahead); + +/* + * Returns true if the hash block with index @hblock_idx in the tree, located in + * @hpage, has already been verified. + */ +static bool is_hash_block_verified(struct fsverity_info *vi, struct page *hpage, + unsigned long hblock_idx) +{ + unsigned int blocks_per_page; + unsigned int i; + + /* + * When the Merkle tree block size and page size are the same, then the + * ->hash_block_verified bitmap isn't allocated, and we use PG_checked + * to directly indicate whether the page's block has been verified. + * + * Using PG_checked also guarantees that we re-verify hash pages that + * get evicted and re-instantiated from the backing storage, as new + * pages always start out with PG_checked cleared. + */ + if (!vi->hash_block_verified) + return PageChecked(hpage); + + /* + * When the Merkle tree block size and page size differ, we use a bitmap + * to indicate whether each hash block has been verified. + * + * However, we still need to ensure that hash pages that get evicted and + * re-instantiated from the backing storage are re-verified. To do + * this, we use PG_checked again, but now it doesn't really mean + * "checked". Instead, now it just serves as an indicator for whether + * the hash page is newly instantiated or not. If the page is new, as + * indicated by PG_checked=0, we clear the bitmap bits for the page's + * blocks since they are untrustworthy, then set PG_checked=1. + * Otherwise we return the bitmap bit for the requested block. + * + * Multiple threads may execute this code concurrently on the same page. + * This is safe because we use memory barriers to ensure that if a + * thread sees PG_checked=1, then it also sees the associated bitmap + * clearing to have occurred. Also, all writes and their corresponding + * reads are atomic, and all writes are safe to repeat in the event that + * multiple threads get into the PG_checked=0 section. (Clearing a + * bitmap bit again at worst causes a hash block to be verified + * redundantly. That event should be very rare, so it's not worth using + * a lock to avoid. Setting PG_checked again has no effect.) + */ + if (PageChecked(hpage)) { + /* + * A read memory barrier is needed here to give ACQUIRE + * semantics to the above PageChecked() test. + */ + smp_rmb(); + return test_bit(hblock_idx, vi->hash_block_verified); + } + blocks_per_page = vi->tree_params.blocks_per_page; + hblock_idx = round_down(hblock_idx, blocks_per_page); + for (i = 0; i < blocks_per_page; i++) + clear_bit(hblock_idx + i, vi->hash_block_verified); + /* + * A write memory barrier is needed here to give RELEASE semantics to + * the below SetPageChecked() operation. + */ + smp_wmb(); + SetPageChecked(hpage); + return false; +} + +/* + * Verify the hash of a single data block against the file's Merkle tree. + * + * In principle, we need to verify the entire path to the root node. However, + * for efficiency the filesystem may cache the hash blocks. Therefore we need + * only ascend the tree until an already-verified hash block is seen, and then + * verify the path to that block. + * + * Return: %true if the data block is valid, else %false. + */ +static bool verify_data_block(struct fsverity_info *vi, + const struct fsverity_pending_block *dblock) +{ + struct inode *inode = vi->inode; + const u64 data_pos = dblock->pos; + const struct merkle_tree_params *params = &vi->tree_params; + const unsigned int hsize = params->digest_size; + int level; + u8 _want_hash[FS_VERITY_MAX_DIGEST_SIZE]; + const u8 *want_hash; + u8 real_hash[FS_VERITY_MAX_DIGEST_SIZE]; + /* The hash blocks that are traversed, indexed by level */ + struct { + /* Page containing the hash block */ + struct page *page; + /* Mapped address of the hash block (will be within @page) */ + const void *addr; + /* Index of the hash block in the tree overall */ + unsigned long index; + /* Byte offset of the wanted hash relative to @addr */ + unsigned int hoffset; + } hblocks[FS_VERITY_MAX_LEVELS]; + + trace_fsverity_verify_data_block(inode, params, data_pos); + + /* + * The index of the previous level's block within that level; also the + * index of that block's hash within the current level. + */ + u64 hidx = data_pos >> params->log_blocksize; + + /* + * Up to FS_VERITY_MAX_PENDING_BLOCKS + FS_VERITY_MAX_LEVELS pages may + * be mapped at once. + */ + static_assert(FS_VERITY_MAX_PENDING_BLOCKS + FS_VERITY_MAX_LEVELS <= + KM_MAX_IDX); + + if (unlikely(data_pos >= inode->i_size)) { + /* + * This can happen in the data page spanning EOF when the Merkle + * tree block size is less than the page size. The Merkle tree + * doesn't cover data blocks fully past EOF. But the entire + * page spanning EOF can be visible to userspace via a mmap, and + * any part past EOF should be all zeroes. Therefore, we need + * to verify that any data blocks fully past EOF are all zeroes. + */ + if (memchr_inv(dblock->data, 0, params->block_size)) { + fsverity_err(inode, + "FILE CORRUPTED! Data past EOF is not zeroed"); + return false; + } + return true; + } + + /* + * Starting at the leaf level, ascend the tree saving hash blocks along + * the way until we find a hash block that has already been verified, or + * until we reach the root. + */ + for (level = 0; level < params->num_levels; level++) { + unsigned long next_hidx; + unsigned long hblock_idx; + pgoff_t hpage_idx; + unsigned int hblock_offset_in_page; + unsigned int hoffset; + struct page *hpage; + const void *haddr; + + /* + * The index of the block in the current level; also the index + * of that block's hash within the next level. + */ + next_hidx = hidx >> params->log_arity; + + /* Index of the hash block in the tree overall */ + hblock_idx = params->level_start[level] + next_hidx; + + /* Index of the hash page in the tree overall */ + hpage_idx = hblock_idx >> params->log_blocks_per_page; + + /* Byte offset of the hash block within the page */ + hblock_offset_in_page = + (hblock_idx << params->log_blocksize) & ~PAGE_MASK; + + /* Byte offset of the hash within the block */ + hoffset = (hidx << params->log_digestsize) & + (params->block_size - 1); + + hpage = inode->i_sb->s_vop->read_merkle_tree_page(inode, + hpage_idx); + if (IS_ERR(hpage)) { + fsverity_err(inode, + "Error %ld reading Merkle tree page %lu", + PTR_ERR(hpage), hpage_idx); + goto error; + } + haddr = kmap_local_page(hpage) + hblock_offset_in_page; + if (is_hash_block_verified(vi, hpage, hblock_idx)) { + memcpy(_want_hash, haddr + hoffset, hsize); + want_hash = _want_hash; + kunmap_local(haddr); + put_page(hpage); + trace_fsverity_merkle_hit(inode, data_pos, hblock_idx, + level, + hoffset >> params->log_digestsize); + goto descend; + } + hblocks[level].page = hpage; + hblocks[level].addr = haddr; + hblocks[level].index = hblock_idx; + hblocks[level].hoffset = hoffset; + hidx = next_hidx; + } + + want_hash = vi->root_hash; +descend: + /* Descend the tree verifying hash blocks. */ + for (; level > 0; level--) { + struct page *hpage = hblocks[level - 1].page; + const void *haddr = hblocks[level - 1].addr; + unsigned long hblock_idx = hblocks[level - 1].index; + unsigned int hoffset = hblocks[level - 1].hoffset; + + trace_fsverity_verify_merkle_block(inode, hblock_idx, + level, hoffset >> params->log_digestsize); + + fsverity_hash_block(params, haddr, real_hash); + if (memcmp(want_hash, real_hash, hsize) != 0) + goto corrupted; + /* + * Mark the hash block as verified. This must be atomic and + * idempotent, as the same hash block might be verified by + * multiple threads concurrently. + */ + if (vi->hash_block_verified) + set_bit(hblock_idx, vi->hash_block_verified); + else + SetPageChecked(hpage); + memcpy(_want_hash, haddr + hoffset, hsize); + want_hash = _want_hash; + kunmap_local(haddr); + put_page(hpage); + } + + /* Finally, verify the hash of the data block. */ + if (memcmp(want_hash, dblock->real_hash, hsize) != 0) + goto corrupted; + return true; + +corrupted: + fsverity_err( + inode, + "FILE CORRUPTED! pos=%llu, level=%d, want_hash=%s:%*phN, real_hash=%s:%*phN", + data_pos, level - 1, params->hash_alg->name, hsize, want_hash, + params->hash_alg->name, hsize, + level == 0 ? dblock->real_hash : real_hash); +error: + for (; level > 0; level--) { + kunmap_local(hblocks[level - 1].addr); + put_page(hblocks[level - 1].page); + } + return false; +} + +static void +fsverity_init_verification_context(struct fsverity_verification_context *ctx, + struct fsverity_info *vi) +{ + ctx->vi = vi; + ctx->num_pending = 0; + if (vi->tree_params.hash_alg->algo_id == HASH_ALGO_SHA256 && + sha256_finup_2x_is_optimized()) + ctx->max_pending = 2; + else + ctx->max_pending = 1; +} + +static void +fsverity_clear_pending_blocks(struct fsverity_verification_context *ctx) +{ + int i; + + for (i = ctx->num_pending - 1; i >= 0; i--) { + kunmap_local(ctx->pending_blocks[i].data); + ctx->pending_blocks[i].data = NULL; + } + ctx->num_pending = 0; +} + +static bool +fsverity_verify_pending_blocks(struct fsverity_verification_context *ctx) +{ + struct fsverity_info *vi = ctx->vi; + const struct merkle_tree_params *params = &vi->tree_params; + int i; + + if (ctx->num_pending == 2) { + /* num_pending == 2 implies that the algorithm is SHA-256 */ + sha256_finup_2x(params->hashstate ? ¶ms->hashstate->sha256 : + NULL, + ctx->pending_blocks[0].data, + ctx->pending_blocks[1].data, params->block_size, + ctx->pending_blocks[0].real_hash, + ctx->pending_blocks[1].real_hash); + } else { + for (i = 0; i < ctx->num_pending; i++) + fsverity_hash_block(params, ctx->pending_blocks[i].data, + ctx->pending_blocks[i].real_hash); + } + + for (i = 0; i < ctx->num_pending; i++) { + if (!verify_data_block(vi, &ctx->pending_blocks[i])) + return false; + } + fsverity_clear_pending_blocks(ctx); + return true; +} + +static bool fsverity_add_data_blocks(struct fsverity_verification_context *ctx, + struct folio *data_folio, size_t len, + size_t offset) +{ + struct fsverity_info *vi = ctx->vi; + const struct merkle_tree_params *params = &vi->tree_params; + const unsigned int block_size = params->block_size; + u64 pos = (u64)data_folio->index << PAGE_SHIFT; + + if (WARN_ON_ONCE(len <= 0 || !IS_ALIGNED(len | offset, block_size))) + return false; + if (WARN_ON_ONCE(!folio_test_locked(data_folio) || + folio_test_uptodate(data_folio))) + return false; + do { + ctx->pending_blocks[ctx->num_pending].data = + kmap_local_folio(data_folio, offset); + ctx->pending_blocks[ctx->num_pending].pos = pos + offset; + if (++ctx->num_pending == ctx->max_pending && + !fsverity_verify_pending_blocks(ctx)) + return false; + offset += block_size; + len -= block_size; + } while (len); + return true; +} + +/** + * fsverity_verify_blocks() - verify data in a folio + * @vi: fsverity_info for the inode to be read + * @folio: the folio containing the data to verify + * @len: the length of the data to verify in the folio + * @offset: the offset of the data to verify in the folio + * + * Verify data that has just been read from a verity file. The data must be + * located in a pagecache folio that is still locked and not yet uptodate. The + * length and offset of the data must be Merkle tree block size aligned. + * + * Return: %true if the data is valid, else %false. + */ +bool fsverity_verify_blocks(struct fsverity_info *vi, struct folio *folio, + size_t len, size_t offset) +{ + struct fsverity_verification_context ctx; + + fsverity_init_verification_context(&ctx, vi); + + if (fsverity_add_data_blocks(&ctx, folio, len, offset) && + fsverity_verify_pending_blocks(&ctx)) + return true; + fsverity_clear_pending_blocks(&ctx); + return false; +} +EXPORT_SYMBOL_GPL(fsverity_verify_blocks); + +#ifdef CONFIG_BLOCK +/** + * fsverity_verify_bio() - verify a 'read' bio that has just completed + * @vi: fsverity_info for the inode to be read + * @bio: the bio to verify + * + * Verify the bio's data against the file's Merkle tree. All bio data segments + * must be aligned to the file's Merkle tree block size. If any data fails + * verification, then bio->bi_status is set to an error status. + * + * This is a helper function for use by the ->readahead() method of filesystems + * that issue bios to read data directly into the page cache. Filesystems that + * populate the page cache without issuing bios (e.g. non block-based + * filesystems) must instead call fsverity_verify_blocks() directly. All + * filesystems must also call fsverity_verify_blocks() on holes. + */ +void fsverity_verify_bio(struct fsverity_info *vi, struct bio *bio) +{ + struct fsverity_verification_context ctx; + struct folio_iter fi; + + fsverity_init_verification_context(&ctx, vi); + + bio_for_each_folio_all(fi, bio) { + if (!fsverity_add_data_blocks(&ctx, fi.folio, fi.length, + fi.offset)) + goto ioerr; + } + + if (!fsverity_verify_pending_blocks(&ctx)) + goto ioerr; + return; + +ioerr: + fsverity_clear_pending_blocks(&ctx); + bio->bi_status = BLK_STS_IOERR; +} +EXPORT_SYMBOL_GPL(fsverity_verify_bio); +#endif /* CONFIG_BLOCK */ + +/** + * fsverity_enqueue_verify_work() - enqueue work on the fs-verity workqueue + * @work: the work to enqueue + * + * Enqueue verification work for asynchronous processing. + */ +void fsverity_enqueue_verify_work(struct work_struct *work) +{ + queue_work(fsverity_read_workqueue, work); +} +EXPORT_SYMBOL_GPL(fsverity_enqueue_verify_work); + +void __init fsverity_init_workqueue(void) +{ + /* + * Use a high-priority workqueue to prioritize verification work, which + * blocks reads from completing, over regular application tasks. + * + * For performance reasons, don't use an unbound workqueue. Using an + * unbound workqueue for crypto operations causes excessive scheduler + * latency on ARM64. + */ + fsverity_read_workqueue = alloc_workqueue("fsverity_read_queue", + WQ_HIGHPRI | WQ_PERCPU, + num_online_cpus()); + if (!fsverity_read_workqueue) + panic("failed to allocate fsverity_read_queue"); +} diff --git a/fs/xattr.c b/fs/xattr.c new file mode 100644 index 000000000..d58979115 --- /dev/null +++ b/fs/xattr.c @@ -0,0 +1,1761 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + File: fs/xattr.c + + Extended attribute handling. + + Copyright (C) 2001 by Andreas Gruenbacher + Copyright (C) 2001 SGI - Silicon Graphics, Inc + Copyright (c) 2004 Red Hat, Inc., James Morris + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "internal.h" + +struct sx_key { + const struct list_head *parent; + const char *name; +}; + +static const char * +strcmp_prefix(const char *a, const char *a_prefix) +{ + while (*a_prefix && *a == *a_prefix) { + a++; + a_prefix++; + } + return *a_prefix ? NULL : a; +} + +/* + * In order to implement different sets of xattr operations for each xattr + * prefix, a filesystem should create a null-terminated array of struct + * xattr_handler (one for each prefix) and hang a pointer to it off of the + * s_xattr field of the superblock. + */ +#define for_each_xattr_handler(handlers, handler) \ + if (handlers) \ + for ((handler) = *(handlers)++; \ + (handler) != NULL; \ + (handler) = *(handlers)++) + +/* + * Find the xattr_handler with the matching prefix. + */ +static const struct xattr_handler * +xattr_resolve_name(struct inode *inode, const char **name) +{ + const struct xattr_handler * const *handlers = inode->i_sb->s_xattr; + const struct xattr_handler *handler; + + if (!(inode->i_opflags & IOP_XATTR)) { + if (unlikely(is_bad_inode(inode))) + return ERR_PTR(-EIO); + return ERR_PTR(-EOPNOTSUPP); + } + for_each_xattr_handler(handlers, handler) { + const char *n; + + n = strcmp_prefix(*name, xattr_prefix(handler)); + if (n) { + if (!handler->prefix ^ !*n) { + if (*n) + continue; + return ERR_PTR(-EINVAL); + } + *name = n; + return handler; + } + } + return ERR_PTR(-EOPNOTSUPP); +} + +/** + * may_write_xattr - check whether inode allows writing xattr + * @idmap: idmap of the mount the inode was found from + * @inode: the inode on which to set an xattr + * + * Check whether the inode allows writing xattrs. Specifically, we can never + * set or remove an extended attribute on a read-only filesystem or on an + * immutable / append-only inode. + * + * We also need to ensure that the inode has a mapping in the mount to + * not risk writing back invalid i_{g,u}id values. + * + * Return: On success zero is returned. On error a negative errno is returned. + */ +int may_write_xattr(struct mnt_idmap *idmap, struct inode *inode) +{ + if (IS_IMMUTABLE(inode)) + return -EPERM; + if (IS_APPEND(inode)) + return -EPERM; + if (HAS_UNMAPPED_ID(idmap, inode)) + return -EPERM; + return 0; +} + +static inline int xattr_permission_error(int mask) +{ + if (mask & MAY_WRITE) + return -EPERM; + return -ENODATA; +} + +/* + * Check permissions for extended attribute access. This is a bit complicated + * because different namespaces have very different rules. + */ +static int +xattr_permission(struct mnt_idmap *idmap, struct inode *inode, + const char *name, int mask) +{ + if (mask & MAY_WRITE) { + int ret; + + ret = may_write_xattr(idmap, inode); + if (ret) + return ret; + } + + /* + * No restriction for security.* and system.* from the VFS. Decision + * on these is left to the underlying filesystem / security module. + */ + if (!strncmp(name, XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN) || + !strncmp(name, XATTR_SYSTEM_PREFIX, XATTR_SYSTEM_PREFIX_LEN)) + return 0; + + /* + * The trusted.* namespace can only be accessed by privileged users. + */ + if (!strncmp(name, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN)) { + if (!capable(CAP_SYS_ADMIN)) + return xattr_permission_error(mask); + return 0; + } + + /* + * In the user.* namespace, only regular files and directories can have + * extended attributes. For sticky directories, only the owner and + * privileged users can write attributes. + */ + if (!strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN)) { + switch (inode->i_mode & S_IFMT) { + case S_IFREG: + break; + case S_IFDIR: + if (!(inode->i_mode & S_ISVTX)) + break; + if (!(mask & MAY_WRITE)) + break; + if (inode_owner_or_capable(idmap, inode)) + break; + return -EPERM; + case S_IFSOCK: + break; + default: + return xattr_permission_error(mask); + } + } + + return inode_permission(idmap, inode, mask); +} + +/* + * Look for any handler that deals with the specified namespace. + */ +int +xattr_supports_user_prefix(struct inode *inode) +{ + const struct xattr_handler * const *handlers = inode->i_sb->s_xattr; + const struct xattr_handler *handler; + + if (!(inode->i_opflags & IOP_XATTR)) { + if (unlikely(is_bad_inode(inode))) + return -EIO; + return -EOPNOTSUPP; + } + + for_each_xattr_handler(handlers, handler) { + if (!strncmp(xattr_prefix(handler), XATTR_USER_PREFIX, + XATTR_USER_PREFIX_LEN)) + return 0; + } + + return -EOPNOTSUPP; +} +EXPORT_SYMBOL(xattr_supports_user_prefix); + +int +__vfs_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct inode *inode, const char *name, const void *value, + size_t size, int flags) +{ + const struct xattr_handler *handler; + + if (is_posix_acl_xattr(name)) + return -EOPNOTSUPP; + + handler = xattr_resolve_name(inode, &name); + if (IS_ERR(handler)) + return PTR_ERR(handler); + if (!handler->set) + return -EOPNOTSUPP; + if (size == 0) + value = ""; /* empty EA, do not remove */ + return handler->set(handler, idmap, dentry, inode, name, value, + size, flags); +} +EXPORT_SYMBOL(__vfs_setxattr); + +/** + * __vfs_setxattr_noperm - perform setxattr operation without performing + * permission checks. + * + * @idmap: idmap of the mount the inode was found from + * @dentry: object to perform setxattr on + * @name: xattr name to set + * @value: value to set @name to + * @size: size of @value + * @flags: flags to pass into filesystem operations + * + * returns the result of the internal setxattr or setsecurity operations. + * + * This function requires the caller to lock the inode's i_rwsem before it + * is executed. It also assumes that the caller will make the appropriate + * permission checks. + */ +int __vfs_setxattr_noperm(struct mnt_idmap *idmap, + struct dentry *dentry, const char *name, + const void *value, size_t size, int flags) +{ + struct inode *inode = dentry->d_inode; + int error = -EAGAIN; + int issec = !strncmp(name, XATTR_SECURITY_PREFIX, + XATTR_SECURITY_PREFIX_LEN); + + if (issec) + inode->i_flags &= ~S_NOSEC; + if (inode->i_opflags & IOP_XATTR) { + error = __vfs_setxattr(idmap, dentry, inode, name, value, + size, flags); + if (!error) { + fsnotify_xattr(dentry); + security_inode_post_setxattr(dentry, name, value, + size, flags); + } + } else { + if (unlikely(is_bad_inode(inode))) + return -EIO; + } + if (error == -EAGAIN) { + error = -EOPNOTSUPP; + + if (issec) { + const char *suffix = name + XATTR_SECURITY_PREFIX_LEN; + + error = security_inode_setsecurity(inode, suffix, value, + size, flags); + if (!error) + fsnotify_xattr(dentry); + } + } + + return error; +} + +/** + * __vfs_setxattr_locked - set an extended attribute while holding the inode + * lock + * + * @idmap: idmap of the mount of the target inode + * @dentry: object to perform setxattr on + * @name: xattr name to set + * @value: value to set @name to + * @size: size of @value + * @flags: flags to pass into filesystem operations + * @delegated_inode: on return, will contain an inode pointer that + * a delegation was broken on, NULL if none. + */ +int +__vfs_setxattr_locked(struct mnt_idmap *idmap, struct dentry *dentry, + const char *name, const void *value, size_t size, + int flags, struct delegated_inode *delegated_inode) +{ + struct inode *inode = dentry->d_inode; + int error; + + error = xattr_permission(idmap, inode, name, MAY_WRITE); + if (error) + return error; + + error = security_inode_setxattr(idmap, dentry, name, value, size, + flags); + if (error) + goto out; + + error = try_break_deleg(inode, 0, delegated_inode); + if (error) + goto out; + + error = __vfs_setxattr_noperm(idmap, dentry, name, value, + size, flags); + +out: + return error; +} +EXPORT_SYMBOL_GPL(__vfs_setxattr_locked); + +int +vfs_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, + const char *name, const void *value, size_t size, int flags) +{ + struct inode *inode = dentry->d_inode; + struct delegated_inode delegated_inode = { }; + const void *orig_value = value; + int error; + + if (size && strcmp(name, XATTR_NAME_CAPS) == 0) { + error = cap_convert_nscap(idmap, dentry, &value, size); + if (error < 0) + return error; + size = error; + } + +retry_deleg: + inode_lock(inode); + error = __vfs_setxattr_locked(idmap, dentry, name, value, size, + flags, &delegated_inode); + inode_unlock(inode); + + if (is_delegated(&delegated_inode)) { + error = break_deleg_wait(&delegated_inode); + if (!error) + goto retry_deleg; + } + if (value != orig_value) + kfree(value); + + return error; +} +EXPORT_SYMBOL_GPL(vfs_setxattr); + +static ssize_t +xattr_getsecurity(struct mnt_idmap *idmap, struct inode *inode, + const char *name, void *value, size_t size) +{ + void *buffer = NULL; + ssize_t len; + + if (!value || !size) { + len = security_inode_getsecurity(idmap, inode, name, + &buffer, false); + goto out_noalloc; + } + + len = security_inode_getsecurity(idmap, inode, name, &buffer, + true); + if (len < 0) + return len; + if (size < len) { + len = -ERANGE; + goto out; + } + memcpy(value, buffer, len); +out: + kfree(buffer); +out_noalloc: + return len; +} + +/* + * vfs_getxattr_alloc - allocate memory, if necessary, before calling getxattr + * + * Allocate memory, if not already allocated, or re-allocate correct size, + * before retrieving the extended attribute. The xattr value buffer should + * always be freed by the caller, even on error. + * + * Returns the result of alloc, if failed, or the getxattr operation. + */ +int +vfs_getxattr_alloc(struct mnt_idmap *idmap, struct dentry *dentry, + const char *name, char **xattr_value, size_t xattr_size, + gfp_t flags) +{ + const struct xattr_handler *handler; + struct inode *inode = dentry->d_inode; + char *value = *xattr_value; + int error; + + error = xattr_permission(idmap, inode, name, MAY_READ); + if (error) + return error; + + handler = xattr_resolve_name(inode, &name); + if (IS_ERR(handler)) + return PTR_ERR(handler); + if (!handler->get) + return -EOPNOTSUPP; + error = handler->get(handler, dentry, inode, name, NULL, 0); + if (error < 0) + return error; + + if (!value || (error > xattr_size)) { + value = krealloc(*xattr_value, error + 1, flags); + if (!value) + return -ENOMEM; + memset(value, 0, error + 1); + } + + error = handler->get(handler, dentry, inode, name, value, error); + *xattr_value = value; + return error; +} + +ssize_t +__vfs_getxattr(struct dentry *dentry, struct inode *inode, const char *name, + void *value, size_t size) +{ + const struct xattr_handler *handler; + + if (is_posix_acl_xattr(name)) + return -EOPNOTSUPP; + + handler = xattr_resolve_name(inode, &name); + if (IS_ERR(handler)) + return PTR_ERR(handler); + if (!handler->get) + return -EOPNOTSUPP; + return handler->get(handler, dentry, inode, name, value, size); +} +EXPORT_SYMBOL(__vfs_getxattr); + +ssize_t +vfs_getxattr(struct mnt_idmap *idmap, struct dentry *dentry, + const char *name, void *value, size_t size) +{ + struct inode *inode = dentry->d_inode; + int error; + + error = xattr_permission(idmap, inode, name, MAY_READ); + if (error) + return error; + + error = security_inode_getxattr(dentry, name); + if (error) + return error; + + if (!strncmp(name, XATTR_SECURITY_PREFIX, + XATTR_SECURITY_PREFIX_LEN)) { + const char *suffix = name + XATTR_SECURITY_PREFIX_LEN; + int ret = xattr_getsecurity(idmap, inode, suffix, value, + size); + /* + * Only overwrite the return value if a security module + * is actually active. + */ + if (ret == -EOPNOTSUPP) + goto nolsm; + return ret; + } +nolsm: + return __vfs_getxattr(dentry, inode, name, value, size); +} +EXPORT_SYMBOL_GPL(vfs_getxattr); + +/** + * vfs_listxattr - retrieve \0 separated list of xattr names + * @dentry: the dentry from whose inode the xattr names are retrieved + * @list: buffer to store xattr names into + * @size: size of the buffer + * + * This function returns the names of all xattrs associated with the + * inode of @dentry. + * + * Note, for legacy reasons the vfs_listxattr() function lists POSIX + * ACLs as well. Since POSIX ACLs are decoupled from IOP_XATTR the + * vfs_listxattr() function doesn't check for this flag since a + * filesystem could implement POSIX ACLs without implementing any other + * xattrs. + * + * However, since all codepaths that remove IOP_XATTR also assign of + * inode operations that either don't implement or implement a stub + * ->listxattr() operation. + * + * Return: On success, the size of the buffer that was used. On error a + * negative error code. + */ +ssize_t +vfs_listxattr(struct dentry *dentry, char *list, size_t size) +{ + struct inode *inode = d_inode(dentry); + ssize_t error; + + error = security_inode_listxattr(dentry); + if (error) + return error; + + if (inode->i_op->listxattr) { + error = inode->i_op->listxattr(dentry, list, size); + } else { + ssize_t remaining = size; + + error = security_inode_listsecurity(inode, &list, &remaining); + if (error) + return error; + error = size - remaining; + } + return error; +} +EXPORT_SYMBOL_GPL(vfs_listxattr); + +int +__vfs_removexattr(struct mnt_idmap *idmap, struct dentry *dentry, + const char *name) +{ + struct inode *inode = d_inode(dentry); + const struct xattr_handler *handler; + + if (is_posix_acl_xattr(name)) + return -EOPNOTSUPP; + + handler = xattr_resolve_name(inode, &name); + if (IS_ERR(handler)) + return PTR_ERR(handler); + if (!handler->set) + return -EOPNOTSUPP; + return handler->set(handler, idmap, dentry, inode, name, NULL, 0, + XATTR_REPLACE); +} +EXPORT_SYMBOL(__vfs_removexattr); + +/** + * __vfs_removexattr_locked - set an extended attribute while holding the inode + * lock + * + * @idmap: idmap of the mount of the target inode + * @dentry: object to perform setxattr on + * @name: name of xattr to remove + * @delegated_inode: on return, will contain an inode pointer that + * a delegation was broken on, NULL if none. + */ +int +__vfs_removexattr_locked(struct mnt_idmap *idmap, + struct dentry *dentry, const char *name, + struct delegated_inode *delegated_inode) +{ + struct inode *inode = dentry->d_inode; + int error; + + error = xattr_permission(idmap, inode, name, MAY_WRITE); + if (error) + return error; + + error = security_inode_removexattr(idmap, dentry, name); + if (error) + goto out; + + error = try_break_deleg(inode, 0, delegated_inode); + if (error) + goto out; + + error = __vfs_removexattr(idmap, dentry, name); + if (error) + return error; + + fsnotify_xattr(dentry); + security_inode_post_removexattr(dentry, name); + +out: + return error; +} +EXPORT_SYMBOL_GPL(__vfs_removexattr_locked); + +int +vfs_removexattr(struct mnt_idmap *idmap, struct dentry *dentry, + const char *name) +{ + struct inode *inode = dentry->d_inode; + struct delegated_inode delegated_inode = { }; + int error; + +retry_deleg: + inode_lock(inode); + error = __vfs_removexattr_locked(idmap, dentry, + name, &delegated_inode); + inode_unlock(inode); + + if (is_delegated(&delegated_inode)) { + error = break_deleg_wait(&delegated_inode); + if (!error) + goto retry_deleg; + } + + return error; +} +EXPORT_SYMBOL_GPL(vfs_removexattr); + +int import_xattr_name(struct xattr_name *kname, const char __user *name) +{ + int error = strncpy_from_user(kname->name, name, + sizeof(kname->name)); + if (error == 0 || error == sizeof(kname->name)) + return -ERANGE; + if (error < 0) + return error; + return 0; +} + +/* + * Extended attribute SET operations + */ + +int setxattr_copy(const char __user *name, struct kernel_xattr_ctx *ctx) +{ + int error; + + if (ctx->flags & ~(XATTR_CREATE|XATTR_REPLACE)) + return -EINVAL; + + error = import_xattr_name(ctx->kname, name); + if (error) + return error; + + if (ctx->size) { + if (ctx->size > XATTR_SIZE_MAX) + return -E2BIG; + + ctx->kvalue = vmemdup_user(ctx->cvalue, ctx->size); + if (IS_ERR(ctx->kvalue)) { + error = PTR_ERR(ctx->kvalue); + ctx->kvalue = NULL; + } + } + + return error; +} + +static int do_setxattr(struct mnt_idmap *idmap, struct dentry *dentry, + struct kernel_xattr_ctx *ctx) +{ + if (is_posix_acl_xattr(ctx->kname->name)) + return do_set_acl(idmap, dentry, ctx->kname->name, + ctx->kvalue, ctx->size); + + return vfs_setxattr(idmap, dentry, ctx->kname->name, + ctx->kvalue, ctx->size, ctx->flags); +} + +int file_setxattr(struct file *f, struct kernel_xattr_ctx *ctx) +{ + int error = mnt_want_write_file(f); + + if (!error) { + audit_file(f); + error = do_setxattr(file_mnt_idmap(f), f->f_path.dentry, ctx); + mnt_drop_write_file(f); + } + return error; +} + +int filename_setxattr(int dfd, struct filename *filename, + unsigned int lookup_flags, struct kernel_xattr_ctx *ctx) +{ + struct path path; + int error; + +retry: + error = filename_lookup(dfd, filename, lookup_flags, &path, NULL); + if (error) + return error; + error = mnt_want_write(path.mnt); + if (!error) { + error = do_setxattr(mnt_idmap(path.mnt), path.dentry, ctx); + mnt_drop_write(path.mnt); + } + path_put(&path); + if (retry_estale(error, lookup_flags)) { + lookup_flags |= LOOKUP_REVAL; + goto retry; + } + return error; +} + +static int path_setxattrat(int dfd, const char __user *pathname, + unsigned int at_flags, const char __user *name, + const void __user *value, size_t size, int flags) +{ + struct xattr_name kname; + struct kernel_xattr_ctx ctx = { + .cvalue = value, + .kvalue = NULL, + .size = size, + .kname = &kname, + .flags = flags, + }; + unsigned int lookup_flags = 0; + int error; + + if ((at_flags & ~(AT_SYMLINK_NOFOLLOW | AT_EMPTY_PATH)) != 0) + return -EINVAL; + + if (!(at_flags & AT_SYMLINK_NOFOLLOW)) + lookup_flags = LOOKUP_FOLLOW; + + error = setxattr_copy(name, &ctx); + if (error) + return error; + + CLASS(filename_maybe_null, filename)(pathname, at_flags); + if (!filename && dfd >= 0) { + CLASS(fd, f)(dfd); + if (fd_empty(f)) + error = -EBADF; + else + error = file_setxattr(fd_file(f), &ctx); + } else { + error = filename_setxattr(dfd, filename, lookup_flags, &ctx); + } + kvfree(ctx.kvalue); + return error; +} + +SYSCALL_DEFINE6(setxattrat, int, dfd, const char __user *, pathname, unsigned int, at_flags, + const char __user *, name, const struct xattr_args __user *, uargs, + size_t, usize) +{ + struct xattr_args args = {}; + int error; + + BUILD_BUG_ON(sizeof(struct xattr_args) < XATTR_ARGS_SIZE_VER0); + BUILD_BUG_ON(sizeof(struct xattr_args) != XATTR_ARGS_SIZE_LATEST); + + if (unlikely(usize < XATTR_ARGS_SIZE_VER0)) + return -EINVAL; + if (usize > PAGE_SIZE) + return -E2BIG; + + error = copy_struct_from_user(&args, sizeof(args), uargs, usize); + if (error) + return error; + + return path_setxattrat(dfd, pathname, at_flags, name, + u64_to_user_ptr(args.value), args.size, + args.flags); +} + +SYSCALL_DEFINE5(setxattr, const char __user *, pathname, + const char __user *, name, const void __user *, value, + size_t, size, int, flags) +{ + return path_setxattrat(AT_FDCWD, pathname, 0, name, value, size, flags); +} + +SYSCALL_DEFINE5(lsetxattr, const char __user *, pathname, + const char __user *, name, const void __user *, value, + size_t, size, int, flags) +{ + return path_setxattrat(AT_FDCWD, pathname, AT_SYMLINK_NOFOLLOW, name, + value, size, flags); +} + +SYSCALL_DEFINE5(fsetxattr, int, fd, const char __user *, name, + const void __user *,value, size_t, size, int, flags) +{ + return path_setxattrat(fd, NULL, AT_EMPTY_PATH, name, + value, size, flags); +} + +/* + * Extended attribute GET operations + */ +static ssize_t +do_getxattr(struct mnt_idmap *idmap, struct dentry *d, + struct kernel_xattr_ctx *ctx) +{ + ssize_t error; + char *kname = ctx->kname->name; + void *kvalue = NULL; + + if (ctx->size) { + if (ctx->size > XATTR_SIZE_MAX) + ctx->size = XATTR_SIZE_MAX; + kvalue = kvzalloc(ctx->size, GFP_KERNEL); + if (!kvalue) + return -ENOMEM; + } + + if (is_posix_acl_xattr(kname)) + error = do_get_acl(idmap, d, kname, kvalue, ctx->size); + else + error = vfs_getxattr(idmap, d, kname, kvalue, ctx->size); + if (error > 0) { + if (ctx->size && copy_to_user(ctx->value, kvalue, error)) + error = -EFAULT; + } else if (error == -ERANGE && ctx->size >= XATTR_SIZE_MAX) { + /* The file system tried to returned a value bigger + than XATTR_SIZE_MAX bytes. Not possible. */ + error = -E2BIG; + } + + kvfree(kvalue); + return error; +} + +ssize_t file_getxattr(struct file *f, struct kernel_xattr_ctx *ctx) +{ + audit_file(f); + return do_getxattr(file_mnt_idmap(f), f->f_path.dentry, ctx); +} + +ssize_t filename_getxattr(int dfd, struct filename *filename, + unsigned int lookup_flags, struct kernel_xattr_ctx *ctx) +{ + struct path path; + ssize_t error; +retry: + error = filename_lookup(dfd, filename, lookup_flags, &path, NULL); + if (error) + return error; + error = do_getxattr(mnt_idmap(path.mnt), path.dentry, ctx); + path_put(&path); + if (retry_estale(error, lookup_flags)) { + lookup_flags |= LOOKUP_REVAL; + goto retry; + } + return error; +} + +static ssize_t path_getxattrat(int dfd, const char __user *pathname, + unsigned int at_flags, const char __user *name, + void __user *value, size_t size) +{ + struct xattr_name kname; + struct kernel_xattr_ctx ctx = { + .value = value, + .size = size, + .kname = &kname, + .flags = 0, + }; + ssize_t error; + + if ((at_flags & ~(AT_SYMLINK_NOFOLLOW | AT_EMPTY_PATH)) != 0) + return -EINVAL; + + error = import_xattr_name(&kname, name); + if (error) + return error; + + CLASS(filename_maybe_null, filename)(pathname, at_flags); + if (!filename && dfd >= 0) { + CLASS(fd, f)(dfd); + if (fd_empty(f)) + return -EBADF; + return file_getxattr(fd_file(f), &ctx); + } else { + int lookup_flags = 0; + if (!(at_flags & AT_SYMLINK_NOFOLLOW)) + lookup_flags = LOOKUP_FOLLOW; + return filename_getxattr(dfd, filename, lookup_flags, &ctx); + } +} + +SYSCALL_DEFINE6(getxattrat, int, dfd, const char __user *, pathname, unsigned int, at_flags, + const char __user *, name, struct xattr_args __user *, uargs, size_t, usize) +{ + struct xattr_args args = {}; + int error; + + BUILD_BUG_ON(sizeof(struct xattr_args) < XATTR_ARGS_SIZE_VER0); + BUILD_BUG_ON(sizeof(struct xattr_args) != XATTR_ARGS_SIZE_LATEST); + + if (unlikely(usize < XATTR_ARGS_SIZE_VER0)) + return -EINVAL; + if (usize > PAGE_SIZE) + return -E2BIG; + + error = copy_struct_from_user(&args, sizeof(args), uargs, usize); + if (error) + return error; + + if (args.flags != 0) + return -EINVAL; + + return path_getxattrat(dfd, pathname, at_flags, name, + u64_to_user_ptr(args.value), args.size); +} + +SYSCALL_DEFINE4(getxattr, const char __user *, pathname, + const char __user *, name, void __user *, value, size_t, size) +{ + return path_getxattrat(AT_FDCWD, pathname, 0, name, value, size); +} + +SYSCALL_DEFINE4(lgetxattr, const char __user *, pathname, + const char __user *, name, void __user *, value, size_t, size) +{ + return path_getxattrat(AT_FDCWD, pathname, AT_SYMLINK_NOFOLLOW, name, + value, size); +} + +SYSCALL_DEFINE4(fgetxattr, int, fd, const char __user *, name, + void __user *, value, size_t, size) +{ + return path_getxattrat(fd, NULL, AT_EMPTY_PATH, name, value, size); +} + +/* + * Extended attribute LIST operations + */ +static ssize_t +listxattr(struct dentry *d, char __user *list, size_t size) +{ + ssize_t error; + char *klist = NULL; + + if (size) { + if (size > XATTR_LIST_MAX) + size = XATTR_LIST_MAX; + klist = kvmalloc(size, GFP_KERNEL); + if (!klist) + return -ENOMEM; + } + + error = vfs_listxattr(d, klist, size); + if (error > 0) { + if (size && copy_to_user(list, klist, error)) + error = -EFAULT; + } else if (error == -ERANGE && size >= XATTR_LIST_MAX) { + /* The file system tried to returned a list bigger + than XATTR_LIST_MAX bytes. Not possible. */ + error = -E2BIG; + } + + kvfree(klist); + + return error; +} + +static +ssize_t file_listxattr(struct file *f, char __user *list, size_t size) +{ + audit_file(f); + return listxattr(f->f_path.dentry, list, size); +} + +static +ssize_t filename_listxattr(int dfd, struct filename *filename, + unsigned int lookup_flags, + char __user *list, size_t size) +{ + struct path path; + ssize_t error; +retry: + error = filename_lookup(dfd, filename, lookup_flags, &path, NULL); + if (error) + return error; + error = listxattr(path.dentry, list, size); + path_put(&path); + if (retry_estale(error, lookup_flags)) { + lookup_flags |= LOOKUP_REVAL; + goto retry; + } + return error; +} + +static ssize_t path_listxattrat(int dfd, const char __user *pathname, + unsigned int at_flags, char __user *list, + size_t size) +{ + int lookup_flags; + + if ((at_flags & ~(AT_SYMLINK_NOFOLLOW | AT_EMPTY_PATH)) != 0) + return -EINVAL; + + CLASS(filename_maybe_null, filename)(pathname, at_flags); + if (!filename) { + CLASS(fd, f)(dfd); + if (fd_empty(f)) + return -EBADF; + return file_listxattr(fd_file(f), list, size); + } + + lookup_flags = (at_flags & AT_SYMLINK_NOFOLLOW) ? 0 : LOOKUP_FOLLOW; + return filename_listxattr(dfd, filename, lookup_flags, list, size); +} + +SYSCALL_DEFINE5(listxattrat, int, dfd, const char __user *, pathname, + unsigned int, at_flags, + char __user *, list, size_t, size) +{ + return path_listxattrat(dfd, pathname, at_flags, list, size); +} + +SYSCALL_DEFINE3(listxattr, const char __user *, pathname, char __user *, list, + size_t, size) +{ + return path_listxattrat(AT_FDCWD, pathname, 0, list, size); +} + +SYSCALL_DEFINE3(llistxattr, const char __user *, pathname, char __user *, list, + size_t, size) +{ + return path_listxattrat(AT_FDCWD, pathname, AT_SYMLINK_NOFOLLOW, list, size); +} + +SYSCALL_DEFINE3(flistxattr, int, fd, char __user *, list, size_t, size) +{ + return path_listxattrat(fd, NULL, AT_EMPTY_PATH, list, size); +} + +/* + * Extended attribute REMOVE operations + */ +static long +removexattr(struct mnt_idmap *idmap, struct dentry *d, const char *name) +{ + if (is_posix_acl_xattr(name)) + return vfs_remove_acl(idmap, d, name); + return vfs_removexattr(idmap, d, name); +} + +static int file_removexattr(struct file *f, struct xattr_name *kname) +{ + int error = mnt_want_write_file(f); + + if (!error) { + audit_file(f); + error = removexattr(file_mnt_idmap(f), + f->f_path.dentry, kname->name); + mnt_drop_write_file(f); + } + return error; +} + +static int filename_removexattr(int dfd, struct filename *filename, + unsigned int lookup_flags, struct xattr_name *kname) +{ + struct path path; + int error; + +retry: + error = filename_lookup(dfd, filename, lookup_flags, &path, NULL); + if (error) + return error; + error = mnt_want_write(path.mnt); + if (!error) { + error = removexattr(mnt_idmap(path.mnt), path.dentry, kname->name); + mnt_drop_write(path.mnt); + } + path_put(&path); + if (retry_estale(error, lookup_flags)) { + lookup_flags |= LOOKUP_REVAL; + goto retry; + } + return error; +} + +static int path_removexattrat(int dfd, const char __user *pathname, + unsigned int at_flags, const char __user *name) +{ + struct xattr_name kname; + unsigned int lookup_flags; + int error; + + if ((at_flags & ~(AT_SYMLINK_NOFOLLOW | AT_EMPTY_PATH)) != 0) + return -EINVAL; + + error = import_xattr_name(&kname, name); + if (error) + return error; + + CLASS(filename_maybe_null, filename)(pathname, at_flags); + if (!filename) { + CLASS(fd, f)(dfd); + if (fd_empty(f)) + return -EBADF; + return file_removexattr(fd_file(f), &kname); + } + lookup_flags = (at_flags & AT_SYMLINK_NOFOLLOW) ? 0 : LOOKUP_FOLLOW; + return filename_removexattr(dfd, filename, lookup_flags, &kname); +} + +SYSCALL_DEFINE4(removexattrat, int, dfd, const char __user *, pathname, + unsigned int, at_flags, const char __user *, name) +{ + return path_removexattrat(dfd, pathname, at_flags, name); +} + +SYSCALL_DEFINE2(removexattr, const char __user *, pathname, + const char __user *, name) +{ + return path_removexattrat(AT_FDCWD, pathname, 0, name); +} + +SYSCALL_DEFINE2(lremovexattr, const char __user *, pathname, + const char __user *, name) +{ + return path_removexattrat(AT_FDCWD, pathname, AT_SYMLINK_NOFOLLOW, name); +} + +SYSCALL_DEFINE2(fremovexattr, int, fd, const char __user *, name) +{ + return path_removexattrat(fd, NULL, AT_EMPTY_PATH, name); +} + +int xattr_list_one(char **buffer, ssize_t *remaining_size, const char *name) +{ + size_t len; + + len = strlen(name) + 1; + if (*buffer) { + if (*remaining_size < len) + return -ERANGE; + memcpy(*buffer, name, len); + *buffer += len; + } + *remaining_size -= len; + return 0; +} + +/** + * generic_listxattr - run through a dentry's xattr list() operations + * @dentry: dentry to list the xattrs + * @buffer: result buffer + * @buffer_size: size of @buffer + * + * Combine the results of the list() operation from every xattr_handler in the + * xattr_handler stack. + * + * Note that this will not include the entries for POSIX ACLs. + */ +ssize_t +generic_listxattr(struct dentry *dentry, char *buffer, size_t buffer_size) +{ + const struct xattr_handler *handler, * const *handlers = dentry->d_sb->s_xattr; + ssize_t remaining_size = buffer_size; + + for_each_xattr_handler(handlers, handler) { + int err; + + if (!handler->name || (handler->list && !handler->list(dentry))) + continue; + err = xattr_list_one(&buffer, &remaining_size, handler->name); + if (err) + return err; + } + + return buffer_size - remaining_size; +} +EXPORT_SYMBOL(generic_listxattr); + +/** + * xattr_full_name - Compute full attribute name from suffix + * + * @handler: handler of the xattr_handler operation + * @name: name passed to the xattr_handler operation + * + * The get and set xattr handler operations are called with the remainder of + * the attribute name after skipping the handler's prefix: for example, "foo" + * is passed to the get operation of a handler with prefix "user." to get + * attribute "user.foo". The full name is still "there" in the name though. + * + * Note: the list xattr handler operation when called from the vfs is passed a + * NULL name; some file systems use this operation internally, with varying + * semantics. + */ +const char *xattr_full_name(const struct xattr_handler *handler, + const char *name) +{ + size_t prefix_len = strlen(xattr_prefix(handler)); + + return name - prefix_len; +} +EXPORT_SYMBOL(xattr_full_name); + +/** + * simple_xattr_space - estimate the memory used by a simple xattr + * @name: the full name of the xattr + * @size: the size of its value + * + * This takes no account of how much larger the two slab objects actually are: + * that would depend on the slab implementation, when what is required is a + * deterministic number, which grows with name length and size and quantity. + * + * Return: The approximate number of bytes of memory used by such an xattr. + */ +size_t simple_xattr_space(const char *name, size_t size) +{ + /* + * Use "40" instead of sizeof(struct simple_xattr), to return the + * same result on 32-bit and 64-bit, and even if simple_xattr grows. + */ + return 40 + size + strlen(name); +} + +/** + * simple_xattr_free - free an xattr object + * @xattr: the xattr object + * + * Free the xattr object. Can handle @xattr being NULL. + */ +void simple_xattr_free(struct simple_xattr *xattr) +{ + if (xattr) + kfree(xattr->name); + kvfree(xattr); +} + +static void simple_xattr_rcu_free(struct rcu_head *head) +{ + struct simple_xattr *xattr = container_of(head, struct simple_xattr, rcu); + + simple_xattr_free(xattr); +} + +/** + * simple_xattr_free_rcu - free an xattr object with RCU delay + * @xattr: the xattr object + * + * Free the xattr object after an RCU grace period. This must be used when + * the xattr was removed from a data structure that concurrent RCU readers + * may still be traversing. Can handle @xattr being NULL. + */ +void simple_xattr_free_rcu(struct simple_xattr *xattr) +{ + if (xattr) + call_rcu(&xattr->rcu, simple_xattr_rcu_free); +} + +/** + * simple_xattr_alloc - allocate new xattr object + * @value: value of the xattr object + * @size: size of @value + * + * Allocate a new xattr object and initialize respective members. The caller is + * responsible for handling the name of the xattr. + * + * Return: New xattr object on success, NULL if @value is NULL, ERR_PTR on + * failure. + */ +struct simple_xattr *simple_xattr_alloc(const void *value, size_t size) +{ + struct simple_xattr *new_xattr; + size_t len; + + if (!value) + return NULL; + + /* wrap around? */ + len = sizeof(*new_xattr) + size; + if (len < sizeof(*new_xattr)) + return ERR_PTR(-ENOMEM); + + new_xattr = kvmalloc(len, GFP_KERNEL_ACCOUNT); + if (!new_xattr) + return ERR_PTR(-ENOMEM); + + new_xattr->size = size; + memcpy(new_xattr->value, value, size); + return new_xattr; +} + +static u32 sx_hashfn(const char *name, const struct list_head *parent, u32 seed) +{ + return jhash(name, strlen(name), jhash(&parent, sizeof(parent), seed)); +} + +static u32 simple_xattr_hashfn(const void *data, u32 len, u32 seed) +{ + const struct sx_key *key = data; + + return sx_hashfn(key->name, key->parent, seed); +} + +static u32 simple_xattr_obj_hashfn(const void *obj, u32 len, u32 seed) +{ + const struct simple_xattr *xattr = obj; + + return sx_hashfn(xattr->name, xattr->parent, seed); +} + +static int simple_xattr_obj_cmpfn(struct rhashtable_compare_arg *arg, + const void *obj) +{ + const struct simple_xattr *xattr = obj; + const struct sx_key *key = arg->key; + + return xattr->parent != key->parent || strcmp(xattr->name, key->name); +} + +static const struct rhashtable_params simple_xattr_params = { + .head_offset = offsetof(struct simple_xattr, hash_node), + .hashfn = simple_xattr_hashfn, + .obj_hashfn = simple_xattr_obj_hashfn, + .obj_cmpfn = simple_xattr_obj_cmpfn, + .automatic_shrinking = true, +}; + +/** + * simple_xattr_get - get an xattr object + * @cache: anchor for the hash table + * @xattrs: the header of the xattr object + * @name: the name of the xattr to retrieve + * @buffer: the buffer to store the value into + * @size: the size of @buffer + * + * Try to find and retrieve the xattr object associated with @name. + * If @buffer is provided store the value of @xattr in @buffer + * otherwise just return the length. The size of @buffer is limited + * to XATTR_SIZE_MAX which currently is 65536. + * + * Return: On success the length of the xattr value is returned. On error a + * negative error code is returned. + */ +int simple_xattr_get(struct simple_xattr_cache *cache, struct list_head *xattrs, + const char *name, void *buffer, size_t size) +{ + struct simple_xattr *xattr; + struct sx_key key = { .parent = xattrs, .name = name }; + struct rhashtable *ht = READ_ONCE(cache->ht); + int ret = -ENODATA; + + if (!ht) + return ret; + + guard(rcu)(); + xattr = rhashtable_lookup(ht, &key, simple_xattr_params); + if (xattr) { + ret = xattr->size; + if (buffer) { + if (size < xattr->size) + ret = -ERANGE; + else + memcpy(buffer, xattr->value, xattr->size); + } + } + return ret; +} + +static struct rhashtable *simple_xattrs_lazy_alloc(struct simple_xattr_cache *cache, + const void *value, int flags) +{ + struct rhashtable *oldht, *ht = READ_ONCE(cache->ht); + int err; + + if (unlikely(!ht)) { + if (!value) + return (flags & XATTR_REPLACE) ? ERR_PTR(-ENODATA) : NULL; + + ht = kzalloc_obj(*ht); + if (!ht) + return ERR_PTR(-ENOMEM); + + err = rhashtable_init(ht, &simple_xattr_params); + if (err) { + kfree(ht); + return ERR_PTR(err); + } + + /* + * Provides release semantics on success, so that use of a + * non-NULL READ_ONCE(cache->ht) will be ordered relative to the + * above initialization, due to implicit address dependency. + */ + oldht = cmpxchg_release(&cache->ht, NULL, ht); + if (oldht) { + /* Race lost */ + rhashtable_destroy(ht); + kfree(ht); + ht = oldht; + } + } + return ht; +} + +/** + * simple_xattr_set - set an xattr object + * @cache: anchor for the hash table + * @xattrs: the header of the xattr object + * @name: the name of the xattr to retrieve + * @value: the value to store along the xattr + * @size: the size of @value + * @flags: the flags determining how to set the xattr + * + * Set a new xattr object. + * If @value is passed a new xattr object will be allocated. If XATTR_REPLACE + * is specified in @flags a matching xattr object for @name must already exist. + * If it does it will be replaced with the new xattr object. If it doesn't we + * fail. If XATTR_CREATE is specified and a matching xattr does already exist + * we fail. If it doesn't we create a new xattr. If @flags is zero we simply + * insert the new xattr replacing any existing one. + * + * If @value is empty and a matching xattr object is found we delete it if + * XATTR_REPLACE is specified in @flags or @flags is zero. + * + * If @value is empty and no matching xattr object for @name is found we do + * nothing if XATTR_CREATE is specified in @flags or @flags is zero. For + * XATTR_REPLACE we fail as mentioned above. + * + * Note: Callers must externally serialize writes. All current callers hold + * the inode lock for write operations. The lookup->replace/remove sequence + * is not atomic with respect to the rhashtable's per-bucket locking, but + * is safe because writes are serialized by the caller. + * + * Return: On success, the removed or replaced xattr is returned, to be freed + * by the caller; or NULL if none. On failure a negative error code is returned. + */ +struct simple_xattr *simple_xattr_set(struct simple_xattr_cache *cache, struct list_head *xattrs, + const char *name, const void *value, + size_t size, int flags) +{ + struct sx_key key = { .parent = xattrs, .name = name }; + struct simple_xattr *old_xattr = NULL; + struct rhashtable *ht; + int err; + + ht = simple_xattrs_lazy_alloc(cache, value, flags); + if (IS_ERR_OR_NULL(ht)) + return ERR_CAST(ht); + + CLASS(simple_xattr, new_xattr)(value, size); + if (IS_ERR(new_xattr)) + return new_xattr; + + if (new_xattr) { + new_xattr->parent = xattrs; + new_xattr->name = kstrdup(name, GFP_KERNEL_ACCOUNT); + if (!new_xattr->name) + return ERR_PTR(-ENOMEM); + } + + /* + * Hash table lookup/replace/remove will grab RCU read lock themselves. + * This makes sure that hash table lookup is safe against concurrent + * modification on another inode. + */ + old_xattr = rhashtable_lookup_fast(ht, &key, simple_xattr_params); + if (old_xattr) { + /* Fail if XATTR_CREATE is requested and the xattr exists. */ + if (flags & XATTR_CREATE) + return ERR_PTR(-EEXIST); + + if (new_xattr) { + err = rhashtable_replace_fast(ht, + &old_xattr->hash_node, + &new_xattr->hash_node, + simple_xattr_params); + if (err) + return ERR_PTR(err); + + list_replace_rcu(&old_xattr->node, &new_xattr->node); + } else { + err = rhashtable_remove_fast(ht, + &old_xattr->hash_node, + simple_xattr_params); + if (err) + return ERR_PTR(err); + + list_del_rcu(&old_xattr->node); + } + } else { + /* Fail if XATTR_REPLACE is requested but no xattr is found. */ + if (flags & XATTR_REPLACE) + return ERR_PTR(-ENODATA); + + /* + * If XATTR_CREATE or no flags are specified together with a + * new value simply insert it. + */ + if (new_xattr) { + err = rhashtable_insert_fast(ht, + &new_xattr->hash_node, + simple_xattr_params); + if (err) + return ERR_PTR(err); + + list_add_tail_rcu(&new_xattr->node, xattrs); + } + + /* + * If XATTR_CREATE or no flags are specified and neither an + * old or new xattr exist then we don't need to do anything. + */ + } + + retain_and_null_ptr(new_xattr); + return old_xattr; +} + +static inline void simple_xattr_limits_dec(struct simple_xattr_limits *limits, + size_t size) +{ + atomic_sub(size, &limits->xattr_size); + atomic_dec(&limits->nr_xattrs); +} + +static inline int simple_xattr_limits_inc(struct simple_xattr_limits *limits, + size_t size) +{ + if (atomic_inc_return(&limits->nr_xattrs) > SIMPLE_XATTR_MAX_NR) { + atomic_dec(&limits->nr_xattrs); + return -ENOSPC; + } + + if (atomic_add_return(size, &limits->xattr_size) <= SIMPLE_XATTR_MAX_SIZE) + return 0; + + simple_xattr_limits_dec(limits, size); + return -ENOSPC; +} + +/** + * simple_xattr_set_limited - set an xattr with per-inode user.* limits + * @cache: anchor for the hash table + * @xattrs: the header of the xattr object + * @limits: per-inode limit counters for user.* xattrs + * @name: the name of the xattr to set or remove + * @value: the value to store (NULL to remove) + * @size: the size of @value + * @flags: XATTR_CREATE, XATTR_REPLACE, or 0 + * + * Like simple_xattr_set(), but enforces per-inode count and total value size + * limits for user.* xattrs. Uses speculative pre-increment of the atomic + * counters to avoid races without requiring external locks. + * + * Return: On success zero is returned. On failure a negative error code is + * returned. + */ +int simple_xattr_set_limited(struct simple_xattr_cache *cache, struct list_head *xattrs, + struct simple_xattr_limits *limits, + const char *name, const void *value, + size_t size, int flags) +{ + struct simple_xattr *old_xattr; + int ret; + + if (value) { + ret = simple_xattr_limits_inc(limits, size); + if (ret) + return ret; + } + + old_xattr = simple_xattr_set(cache, xattrs, name, value, size, flags); + if (IS_ERR(old_xattr)) { + if (value) + simple_xattr_limits_dec(limits, size); + return PTR_ERR(old_xattr); + } + if (old_xattr) { + simple_xattr_limits_dec(limits, old_xattr->size); + simple_xattr_free_rcu(old_xattr); + } + return 0; +} + +static bool xattr_is_trusted(const char *name) +{ + return !strncmp(name, XATTR_TRUSTED_PREFIX, XATTR_TRUSTED_PREFIX_LEN); +} + +static bool xattr_is_maclabel(const char *name) +{ + const char *suffix = name + XATTR_SECURITY_PREFIX_LEN; + + return !strncmp(name, XATTR_SECURITY_PREFIX, + XATTR_SECURITY_PREFIX_LEN) && + security_ismaclabel(suffix); +} + +/** + * simple_xattr_list - list all xattr objects + * @inode: inode from which to get the xattrs + * @xattrs: the header of the xattr object + * @buffer: the buffer to store all xattrs into + * @size: the size of @buffer + * + * List all xattrs associated with @inode. If @buffer is NULL we returned + * the required size of the buffer. If @buffer is provided we store the + * xattrs value into it provided it is big enough. + * + * Note, the number of xattr names that can be listed with listxattr(2) is + * limited to XATTR_LIST_MAX aka 65536 bytes. If a larger buffer is passed + * then vfs_listxattr() caps it to XATTR_LIST_MAX and if more xattr names + * are found it will return -E2BIG. + * + * Return: On success the required size or the size of the copied xattrs is + * returned. On error a negative error code is returned. + */ +ssize_t simple_xattr_list(struct inode *inode, struct list_head *xattrs, + char *buffer, size_t size) +{ + bool trusted = ns_capable_noaudit(&init_user_ns, CAP_SYS_ADMIN); + struct simple_xattr *xattr; + ssize_t remaining_size = size; + int err = 0; + + err = posix_acl_listxattr(inode, &buffer, &remaining_size); + if (err) + return err; + + err = security_inode_listsecurity(inode, &buffer, &remaining_size); + if (err < 0) + return err; + + if (buffer) { + if (remaining_size < err) + return -ERANGE; + buffer += err; + } + remaining_size -= err; + err = 0; + + if (!xattrs) + return size - remaining_size; + + rcu_read_lock(); + list_for_each_entry_rcu(xattr, xattrs, node) { + /* skip "trusted." attributes for unprivileged callers */ + if (!trusted && xattr_is_trusted(xattr->name)) + continue; + + /* skip MAC labels; these are provided by LSM above */ + if (xattr_is_maclabel(xattr->name)) + continue; + + err = xattr_list_one(&buffer, &remaining_size, xattr->name); + if (err) + break; + } + rcu_read_unlock(); + + return err ? err : size - remaining_size; +} + +/** + * simple_xattr_add - add xattr objects + * @cache: anchor for the hash table + * @xattrs: the header of the xattr object + * @new_xattr: the xattr object to add + * + * Add an xattr object to @xattrs. This assumes no replacement or removal + * of matching xattrs is wanted. Should only be called during inode + * initialization when a few distinct initial xattrs are supposed to be set. + * + * Return: On success zero is returned. On failure a negative error code is + * returned. + */ +int simple_xattr_add(struct simple_xattr_cache *cache, struct list_head *xattrs, + struct simple_xattr *new_xattr) +{ + struct rhashtable *ht; + int err; + + ht = simple_xattrs_lazy_alloc(cache, new_xattr->value, 0); + if (IS_ERR(ht)) + return PTR_ERR(ht); + + new_xattr->parent = xattrs; + err = rhashtable_insert_fast(ht, &new_xattr->hash_node, simple_xattr_params); + if (err) + return err; + + list_add_tail_rcu(&new_xattr->node, xattrs); + return 0; +} + +/** + * simple_xattr_add_limited - add an xattr object, charging per-inode limits + * @cache: anchor for the hash table + * @xattrs: the header of the xattr object + * @limits: per-inode limit counters + * @new_xattr: the xattr object to add + * + * Like simple_xattr_add(), but also accounts @new_xattr against @limits so + * that a later removal or replacement of it through simple_xattr_set_limited() + * decrements counters that were actually incremented, rather than underflowing + * them. Use this instead of simple_xattr_add() when seeding initial xattrs + * that share a namespace with the limited set/remove path. + * + * Return: On success zero is returned. On failure a negative error code is + * returned. + */ +int simple_xattr_add_limited(struct simple_xattr_cache *cache, + struct list_head *xattrs, + struct simple_xattr_limits *limits, + struct simple_xattr *new_xattr) +{ + int err; + + err = simple_xattr_limits_inc(limits, new_xattr->size); + if (err) + return err; + + err = simple_xattr_add(cache, xattrs, new_xattr); + if (err) + simple_xattr_limits_dec(limits, new_xattr->size); + return err; +} + +/** + * simple_xattrs_free - free xattrs + * @cache: anchor for the hash table + * @xattrs: xattr header whose xattrs to destroy + * @freed_space: approximate number of bytes of memory freed from @xattrs + * + * Destroy all xattrs in @xattrs. When this is called no one can hold a + * reference to any of the xattrs anymore. + */ +void simple_xattrs_free(struct simple_xattr_cache *cache, struct list_head *xattrs, + size_t *freed_space) +{ + if (freed_space) + *freed_space = 0; + + while (!list_empty(xattrs)) { + struct simple_xattr *xattr = list_first_entry(xattrs, typeof(*xattr), node); + + rhashtable_remove_fast(cache->ht, &xattr->hash_node, simple_xattr_params); + list_del(&xattr->node); + if (freed_space) + *freed_space += simple_xattr_space(xattr->name, xattr->size); + /* + * Free with RCU, since the xattr might still get accessed by + * the hash compare function + */ + simple_xattr_free_rcu(xattr); + } +} + +/** + * simple_xattr_cache_cleanup - free the cache + * @cache: anchor for the hash table + * + * Destroy the cache table, which was lazily allocated on adding the first xattr. + */ +void simple_xattr_cache_cleanup(struct simple_xattr_cache *cache) +{ + if (cache->ht) { + WARN_ON(atomic_read(&cache->ht->nelems)); + rhashtable_destroy(cache->ht); + kfree(cache->ht); + cache->ht = NULL; + } +} diff --git a/fs/xfs/Kconfig b/fs/xfs/Kconfig new file mode 100644 index 000000000..b99da294e --- /dev/null +++ b/fs/xfs/Kconfig @@ -0,0 +1,243 @@ +# SPDX-License-Identifier: GPL-2.0-only +config XFS_FS + tristate "XFS filesystem support" + depends on BLOCK + select EXPORTFS + select CRC32 + select FS_IOMAP + help + XFS is a high performance journaling filesystem which originated + on the SGI IRIX platform. It is completely multi-threaded, can + support large files and large filesystems, extended attributes, + variable block sizes, is extent based, and makes extensive use of + Btrees (directories, extents, free space) to aid both performance + and scalability. + + Refer to the documentation at + for complete details. This implementation is on-disk compatible + with the IRIX version of XFS. + + To compile this file system support as a module, choose M here: the + module will be called xfs. Be aware, however, that if the file + system of your root partition is compiled as a module, you'll need + to use an initial ramdisk (initrd) to boot. + +config XFS_SUPPORT_V4 + bool "Support deprecated V4 (crc=0) format" + depends on XFS_FS + default n + help + The V4 filesystem format lacks certain features that are supported + by the V5 format, such as metadata checksumming, strengthened + metadata verification, and the ability to store timestamps past the + year 2038. Because of this, the V4 format is deprecated. All users + should upgrade by backing up their files, reformatting, and restoring + from the backup. + + Administrators and users can detect a V4 filesystem by running + xfs_info against a filesystem mountpoint and checking for a string + beginning with "crc=". If the string "crc=0" is found, the + filesystem is a V4 filesystem. If no such string is found, please + upgrade xfsprogs to the latest version and try again. + + This option became default N in September 2025. Support for the + V4 format will be removed entirely in September 2030. Distributors + can say N here to withdraw support earlier. + + To continue supporting the old V4 format (crc=0), say Y. + To close off an attack surface, say N. + +config XFS_SUPPORT_ASCII_CI + bool "Support deprecated case-insensitive ascii (ascii-ci=1) format" + depends on XFS_FS + default n + help + The ASCII case insensitivity filesystem feature only works correctly + on systems that have been coerced into using ISO 8859-1, and it does + not work on extended attributes. The kernel has no visibility into + the locale settings in userspace, so it corrupts UTF-8 names. + Enabling this feature makes XFS vulnerable to mixed case sensitivity + attacks. Because of this, the feature is deprecated. All users + should upgrade by backing up their files, reformatting, and restoring + from the backup. + + Administrators and users can detect such a filesystem by running + xfs_info against a filesystem mountpoint and checking for a string + beginning with "ascii-ci=". If the string "ascii-ci=1" is found, the + filesystem is a case-insensitive filesystem. If no such string is + found, please upgrade xfsprogs to the latest version and try again. + + This option became default N in September 2025. Support for the + feature will be removed entirely in September 2030. Distributors + can say N here to withdraw support earlier. + + To continue supporting case-insensitivity (ascii-ci=1), say Y. + To close off an attack surface, say N. + +config XFS_QUOTA + bool "XFS Quota support" + depends on XFS_FS + select QUOTACTL + help + If you say Y here, you will be able to set limits for disk usage on + a per user and/or a per group basis under XFS. XFS considers quota + information as filesystem metadata and uses journaling to provide a + higher level guarantee of consistency. The on-disk data format for + quota is also compatible with the IRIX version of XFS, allowing a + filesystem to be migrated between Linux and IRIX without any need + for conversion. + + If unsure, say N. More comprehensive documentation can be found in + README.quota in the xfsprogs package. XFS quota can be used either + with or without the generic quota support enabled (CONFIG_QUOTA) - + they are completely independent subsystems. + +config XFS_POSIX_ACL + bool "XFS POSIX ACL support" + depends on XFS_FS + select FS_POSIX_ACL + help + POSIX Access Control Lists (ACLs) support permissions for users and + groups beyond the owner/group/world scheme. + + If you don't know what Access Control Lists are, say N. + +config XFS_RT + bool "XFS Realtime subvolume support" + depends on XFS_FS + default BLK_DEV_ZONED + help + If you say Y here you will be able to mount and use XFS filesystems + which contain a realtime subvolume. The realtime subvolume is a + separate area of disk space where only file data is stored. It was + originally designed to provide deterministic data rates suitable + for media streaming applications, but is also useful as a generic + mechanism for ensuring data and metadata/log I/Os are completely + separated. Regular file I/Os are isolated to a separate device + from all other requests, and this can be done quite transparently + to applications via the inherit-realtime directory inode flag. + + See the xfs man page in section 5 for additional information. + + This option is mandatory to support zoned block devices. For these + devices, the realtime subvolume must be backed by a zoned block + device and a regular block device used as the main device (for + metadata). If the zoned block device is a host-managed SMR hard-disk + containing conventional zones at the beginning of its address space, + XFS will use the disk conventional zones as the main device and the + remaining sequential write required zones as the backing storage for + the realtime subvolume. + + If unsure, say N. + +config XFS_DRAIN_INTENTS + bool + select JUMP_LABEL if HAVE_ARCH_JUMP_LABEL + +config XFS_LIVE_HOOKS + bool + select JUMP_LABEL if HAVE_ARCH_JUMP_LABEL + +config XFS_MEMORY_BUFS + bool + +config XFS_BTREE_IN_MEM + bool + +config XFS_ONLINE_SCRUB + bool "XFS online metadata check support" + default y + depends on XFS_FS + depends on TMPFS && SHMEM + select XFS_LIVE_HOOKS + select XFS_DRAIN_INTENTS + select XFS_MEMORY_BUFS + help + If you say Y here you will be able to check metadata on a + mounted XFS filesystem. This feature is intended to reduce + filesystem downtime by supplementing xfs_repair. The key + advantage here is to look for problems proactively so that + they can be dealt with in a controlled manner. + + See the xfs_scrub man page in section 8 for additional information. + +config XFS_ONLINE_SCRUB_STATS + bool "XFS online metadata check usage data collection" + default y + depends on XFS_ONLINE_SCRUB + depends on DEBUG_FS + help + If you say Y here, the kernel will gather usage data about + the online metadata check subsystem. This includes the number + of invocations, the outcomes, and the results of repairs, if any. + This may slow down scrub slightly due to the use of high precision + timers and the need to merge per-invocation information into the + filesystem counters. + + Usage data are collected in /sys/kernel/debug/xfs/scrub. + +config XFS_ONLINE_REPAIR + bool "XFS online metadata repair support" + default y + depends on XFS_FS && XFS_ONLINE_SCRUB + select XFS_BTREE_IN_MEM + help + If you say Y here you will be able to repair metadata on a + mounted XFS filesystem. This feature is intended to reduce + filesystem downtime by fixing minor problems before they cause the + filesystem to go down. However, it requires that the filesystem be + formatted with secondary metadata, such as reverse mappings and inode + parent pointers. + + See the xfs_scrub man page in section 8 for additional information. + +config XFS_WARN + bool "XFS Verbose Warnings" + depends on XFS_FS && !XFS_DEBUG + help + Say Y here to get an XFS build with many additional warnings. + It converts ASSERT checks to WARN, so will log any out-of-bounds + conditions that occur that would otherwise be missed. It is much + lighter weight than XFS_DEBUG and does not modify algorithms and will + not cause the kernel to panic on non-fatal errors. + + However, similar to XFS_DEBUG, it is only advisable to use this if you + are debugging a particular problem. + +config XFS_DEBUG + bool "XFS Debugging support" + depends on XFS_FS + help + Say Y here to get an XFS build with many debugging features, + including ASSERT checks, function wrappers around macros, + and extra sanity-checking functions in various code paths. + + Note that the resulting code will be HUGE and SLOW, and probably + not useful unless you are debugging a particular problem. + + Say N unless you are an XFS developer, or you play one on TV. + +config XFS_DEBUG_EXPENSIVE + bool "XFS expensive debugging checks" + depends on XFS_FS && XFS_DEBUG + help + Say Y here to get an XFS build with expensive debugging checks + enabled. These checks may affect performance significantly. + + Note that the resulting code will be HUGER and SLOWER, and probably + not useful unless you are debugging a particular problem. + + Say N unless you are an XFS developer, or you play one on TV. + +config XFS_ASSERT_FATAL + bool "XFS fatal asserts" + default y + depends on XFS_FS && XFS_DEBUG + help + Set the default DEBUG mode ASSERT failure behavior. + + Say Y here to cause DEBUG mode ASSERT failures to result in fatal + errors that BUG() the kernel by default. If you say N, ASSERT failures + result in warnings. + + This behavior can be modified at runtime via sysfs. diff --git a/fs/xfs/Makefile b/fs/xfs/Makefile new file mode 100644 index 000000000..9f7133e02 --- /dev/null +++ b/fs/xfs/Makefile @@ -0,0 +1,256 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (c) 2000-2005 Silicon Graphics, Inc. +# All Rights Reserved. +# + +ccflags-y += -I $(src) # needed for trace events +ccflags-y += -I $(src)/libxfs + +obj-$(CONFIG_XFS_FS) += xfs.o + +# this one should be compiled first, as the tracing macros can easily blow up +xfs-y += xfs_trace.o + +# build the libxfs code first +xfs-y += $(addprefix libxfs/, \ + xfs_group.o \ + xfs_ag.o \ + xfs_ag_resv.o \ + xfs_alloc.o \ + xfs_alloc_btree.o \ + xfs_attr.o \ + xfs_attr_leaf.o \ + xfs_attr_remote.o \ + xfs_bit.o \ + xfs_bmap.o \ + xfs_bmap_btree.o \ + xfs_btree.o \ + xfs_btree_staging.o \ + xfs_da_btree.o \ + xfs_defer.o \ + xfs_dir2.o \ + xfs_dir2_block.o \ + xfs_dir2_data.o \ + xfs_dir2_leaf.o \ + xfs_dir2_node.o \ + xfs_dir2_sf.o \ + xfs_dquot_buf.o \ + xfs_exchmaps.o \ + xfs_ialloc.o \ + xfs_ialloc_btree.o \ + xfs_iext_tree.o \ + xfs_inode_fork.o \ + xfs_inode_buf.o \ + xfs_inode_util.o \ + xfs_log_rlimit.o \ + xfs_metadir.o \ + xfs_metafile.o \ + xfs_parent.o \ + xfs_rmap.o \ + xfs_rmap_btree.o \ + xfs_refcount.o \ + xfs_refcount_btree.o \ + xfs_rtrefcount_btree.o \ + xfs_rtrmap_btree.o \ + xfs_sb.o \ + xfs_symlink_remote.o \ + xfs_trans_inode.o \ + xfs_trans_resv.o \ + xfs_trans_space.o \ + xfs_types.o \ + ) +# xfs_rtbitmap is shared with libxfs +xfs-$(CONFIG_XFS_RT) += $(addprefix libxfs/, \ + xfs_rtbitmap.o \ + xfs_rtgroup.o \ + xfs_zones.o \ + ) + +# highlevel code +xfs-y += xfs_aops.o \ + xfs_attr_inactive.o \ + xfs_attr_list.o \ + xfs_bmap_util.o \ + xfs_bio_io.o \ + xfs_buf.o \ + xfs_dahash_test.o \ + xfs_dir2_readdir.o \ + xfs_discard.o \ + xfs_error.o \ + xfs_exchrange.o \ + xfs_export.o \ + xfs_extent_busy.o \ + xfs_file.o \ + xfs_filestream.o \ + xfs_fsmap.o \ + xfs_fsops.o \ + xfs_globals.o \ + xfs_handle.o \ + xfs_health.o \ + xfs_healthmon.o \ + xfs_icache.o \ + xfs_ioctl.o \ + xfs_iomap.o \ + xfs_iops.o \ + xfs_inode.o \ + xfs_itable.o \ + xfs_iwalk.o \ + xfs_message.o \ + xfs_mount.o \ + xfs_mru_cache.o \ + xfs_pwork.o \ + xfs_reflink.o \ + xfs_stats.o \ + xfs_super.o \ + xfs_symlink.o \ + xfs_sysfs.o \ + xfs_trans.o \ + xfs_verify_media.o \ + xfs_xattr.o + +# low-level transaction/log code +xfs-y += xfs_log.o \ + xfs_log_cil.o \ + xfs_bmap_item.o \ + xfs_buf_item.o \ + xfs_buf_item_recover.o \ + xfs_dquot_item_recover.o \ + xfs_exchmaps_item.o \ + xfs_extfree_item.o \ + xfs_attr_item.o \ + xfs_icreate_item.o \ + xfs_inode_item.o \ + xfs_inode_item_recover.o \ + xfs_iunlink_item.o \ + xfs_refcount_item.o \ + xfs_rmap_item.o \ + xfs_log_recover.o \ + xfs_trans_ail.o \ + xfs_trans_buf.o + +# optional features +xfs-$(CONFIG_XFS_QUOTA) += xfs_dquot.o \ + xfs_dquot_item.o \ + xfs_trans_dquot.o \ + xfs_qm_syscalls.o \ + xfs_qm_bhv.o \ + xfs_qm.o \ + xfs_quotaops.o + +# xfs_rtbitmap is shared with libxfs +xfs-$(CONFIG_XFS_RT) += xfs_rtalloc.o \ + xfs_zone_alloc.o \ + xfs_zone_gc.o \ + xfs_zone_info.o \ + xfs_zone_space_resv.o + +xfs-$(CONFIG_XFS_POSIX_ACL) += xfs_acl.o +xfs-$(CONFIG_SYSCTL) += xfs_sysctl.o +xfs-$(CONFIG_COMPAT) += xfs_ioctl32.o +xfs-$(CONFIG_EXPORTFS_BLOCK_OPS) += xfs_pnfs.o + +# notify failure +ifeq ($(CONFIG_MEMORY_FAILURE),y) +xfs-$(CONFIG_FS_DAX) += xfs_notify_failure.o +endif + +xfs-$(CONFIG_XFS_DRAIN_INTENTS) += xfs_drain.o +xfs-$(CONFIG_XFS_LIVE_HOOKS) += xfs_hooks.o +xfs-$(CONFIG_XFS_MEMORY_BUFS) += xfs_buf_mem.o +xfs-$(CONFIG_XFS_BTREE_IN_MEM) += libxfs/xfs_btree_mem.o + +# online scrub/repair +ifeq ($(CONFIG_XFS_ONLINE_SCRUB),y) + +# Tracepoints like to blow up, so build that before everything else + +xfs-y += $(addprefix scrub/, \ + trace.o \ + agb_bitmap.o \ + agheader.o \ + alloc.o \ + attr.o \ + bitmap.o \ + bmap.o \ + btree.o \ + common.o \ + dabtree.o \ + dir.o \ + dirtree.o \ + fscounters.o \ + health.o \ + ialloc.o \ + inode.o \ + iscan.o \ + listxattr.o \ + metapath.o \ + nlinks.o \ + parent.o \ + readdir.o \ + refcount.o \ + rmap.o \ + scrub.o \ + symlink.o \ + xfarray.o \ + xfblob.o \ + xfile.o \ + ) + +xfs-$(CONFIG_XFS_ONLINE_SCRUB_STATS) += scrub/stats.o + +xfs-$(CONFIG_XFS_RT) += $(addprefix scrub/, \ + rgsuper.o \ + rtbitmap.o \ + rtrefcount.o \ + rtrmap.o \ + rtsummary.o \ + ) + +xfs-$(CONFIG_XFS_QUOTA) += $(addprefix scrub/, \ + dqiterate.o \ + quota.o \ + quotacheck.o \ + ) + +# online repair +ifeq ($(CONFIG_XFS_ONLINE_REPAIR),y) +xfs-y += $(addprefix scrub/, \ + agheader_repair.o \ + alloc_repair.o \ + attr_repair.o \ + bmap_repair.o \ + cow_repair.o \ + dir_repair.o \ + dirtree_repair.o \ + findparent.o \ + fscounters_repair.o \ + ialloc_repair.o \ + inode_repair.o \ + newbt.o \ + nlinks_repair.o \ + orphanage.o \ + parent_repair.o \ + rcbag_btree.o \ + rcbag.o \ + reap.o \ + refcount_repair.o \ + repair.o \ + rmap_repair.o \ + symlink_repair.o \ + tempfile.o \ + ) + +xfs-$(CONFIG_XFS_RT) += $(addprefix scrub/, \ + rtbitmap_repair.o \ + rtrefcount_repair.o \ + rtrmap_repair.o \ + rtsummary_repair.o \ + ) + +xfs-$(CONFIG_XFS_QUOTA) += $(addprefix scrub/, \ + quota_repair.o \ + quotacheck_repair.o \ + ) +endif +endif diff --git a/fs/xfs/libxfs/xfs_ag.c b/fs/xfs/libxfs/xfs_ag.c new file mode 100644 index 000000000..6cab66781 --- /dev/null +++ b/fs/xfs/libxfs/xfs_ag.c @@ -0,0 +1,995 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * Copyright (c) 2018 Red Hat, Inc. + * All rights reserved. + */ + +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_sb.h" +#include "xfs_mount.h" +#include "xfs_btree.h" +#include "xfs_alloc_btree.h" +#include "xfs_rmap_btree.h" +#include "xfs_alloc.h" +#include "xfs_ialloc.h" +#include "xfs_rmap.h" +#include "xfs_ag.h" +#include "xfs_ag_resv.h" +#include "xfs_health.h" +#include "xfs_error.h" +#include "xfs_bmap.h" +#include "xfs_defer.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_trace.h" +#include "xfs_inode.h" +#include "xfs_icache.h" +#include "xfs_group.h" + +/* + * xfs_initialize_perag_data + * + * Read in each per-ag structure so we can count up the number of + * allocated inodes, free inodes and used filesystem blocks as this + * information is no longer persistent in the superblock. Once we have + * this information, write it into the in-core superblock structure. + */ +int +xfs_initialize_perag_data( + struct xfs_mount *mp, + xfs_agnumber_t agcount) +{ + xfs_agnumber_t index; + struct xfs_perag *pag; + struct xfs_sb *sbp = &mp->m_sb; + uint64_t ifree = 0; + uint64_t ialloc = 0; + uint64_t bfree = 0; + uint64_t bfreelst = 0; + uint64_t btree = 0; + uint64_t fdblocks; + int error = 0; + + for (index = 0; index < agcount; index++) { + /* + * Read the AGF and AGI buffers to populate the per-ag + * structures for us. + */ + pag = xfs_perag_get(mp, index); + error = xfs_alloc_read_agf(pag, NULL, 0, NULL); + if (!error) + error = xfs_ialloc_read_agi(pag, NULL, 0, NULL); + if (error) { + xfs_perag_put(pag); + return error; + } + + ifree += pag->pagi_freecount; + ialloc += pag->pagi_count; + bfree += pag->pagf_freeblks; + bfreelst += pag->pagf_flcount; + btree += pag->pagf_btreeblks; + xfs_perag_put(pag); + } + fdblocks = bfree + bfreelst + btree; + + /* + * If the new summary counts are obviously incorrect, fail the + * mount operation because that implies the AGFs are also corrupt. + * Clear FS_COUNTERS so that we don't unmount with a dirty log, which + * will prevent xfs_repair from fixing anything. + */ + if (fdblocks > sbp->sb_dblocks || ifree > ialloc) { + xfs_alert(mp, "AGF corruption. Please run xfs_repair."); + xfs_fs_mark_sick(mp, XFS_SICK_FS_COUNTERS); + error = -EFSCORRUPTED; + goto out; + } + + /* Overwrite incore superblock counters with just-read data */ + spin_lock(&mp->m_sb_lock); + sbp->sb_ifree = ifree; + sbp->sb_icount = ialloc; + sbp->sb_fdblocks = fdblocks; + spin_unlock(&mp->m_sb_lock); + + xfs_reinit_percpu_counters(mp); +out: + xfs_fs_mark_healthy(mp, XFS_SICK_FS_COUNTERS); + return error; +} + +static void +xfs_perag_uninit( + struct xfs_group *xg) +{ +#ifdef __KERNEL__ + cancel_delayed_work_sync(&to_perag(xg)->pag_blockgc_work); +#endif +} + +/* + * Free up the per-ag resources within the specified AG range. + */ +void +xfs_free_perag_range( + struct xfs_mount *mp, + xfs_agnumber_t first_agno, + xfs_agnumber_t end_agno) + +{ + xfs_agnumber_t agno; + + for (agno = first_agno; agno < end_agno; agno++) + xfs_group_free(mp, agno, XG_TYPE_AG, xfs_perag_uninit); +} + +/* Find the size of the AG, in blocks. */ +static xfs_agblock_t +__xfs_ag_block_count( + struct xfs_mount *mp, + xfs_agnumber_t agno, + xfs_agnumber_t agcount, + xfs_rfsblock_t dblocks) +{ + ASSERT(agno < agcount); + + if (agno < agcount - 1) + return mp->m_sb.sb_agblocks; + return dblocks - (agno * mp->m_sb.sb_agblocks); +} + +xfs_agblock_t +xfs_ag_block_count( + struct xfs_mount *mp, + xfs_agnumber_t agno) +{ + return __xfs_ag_block_count(mp, agno, mp->m_sb.sb_agcount, + mp->m_sb.sb_dblocks); +} + +/* Calculate the first and last possible inode number in an AG. */ +static void +__xfs_agino_range( + struct xfs_mount *mp, + xfs_agblock_t eoag, + xfs_agino_t *first, + xfs_agino_t *last) +{ + xfs_agblock_t bno; + + /* + * Calculate the first inode, which will be in the first + * cluster-aligned block after the AGFL. + */ + bno = round_up(XFS_AGFL_BLOCK(mp) + 1, M_IGEO(mp)->cluster_align); + *first = XFS_AGB_TO_AGINO(mp, bno); + + /* + * Calculate the last inode, which will be at the end of the + * last (aligned) cluster that can be allocated in the AG. + */ + bno = round_down(eoag, M_IGEO(mp)->cluster_align); + *last = XFS_AGB_TO_AGINO(mp, bno) - 1; +} + +void +xfs_agino_range( + struct xfs_mount *mp, + xfs_agnumber_t agno, + xfs_agino_t *first, + xfs_agino_t *last) +{ + return __xfs_agino_range(mp, xfs_ag_block_count(mp, agno), first, last); +} + +/* + * Update the perag of the previous tail AG if it has been changed during + * recovery (i.e. recovery of a growfs). + */ +int +xfs_update_last_ag_size( + struct xfs_mount *mp, + xfs_agnumber_t prev_agcount) +{ + struct xfs_perag *pag = xfs_perag_grab(mp, prev_agcount - 1); + + if (!pag) + return -EFSCORRUPTED; + pag_group(pag)->xg_block_count = __xfs_ag_block_count(mp, + prev_agcount - 1, mp->m_sb.sb_agcount, + mp->m_sb.sb_dblocks); + __xfs_agino_range(mp, pag_group(pag)->xg_block_count, &pag->agino_min, + &pag->agino_max); + xfs_perag_rele(pag); + return 0; +} + +static int +xfs_perag_alloc( + struct xfs_mount *mp, + xfs_agnumber_t index, + xfs_agnumber_t agcount, + xfs_rfsblock_t dblocks) +{ + struct xfs_perag *pag; + int error; + + pag = kzalloc_obj(*pag); + if (!pag) + return -ENOMEM; + +#ifdef __KERNEL__ + /* Place kernel structure only init below this point. */ + spin_lock_init(&pag->pag_ici_lock); + INIT_DELAYED_WORK(&pag->pag_blockgc_work, xfs_blockgc_worker); + INIT_RADIX_TREE(&pag->pag_ici_root, GFP_ATOMIC); +#endif /* __KERNEL__ */ + + /* + * Pre-calculated geometry + */ + pag_group(pag)->xg_block_count = __xfs_ag_block_count(mp, index, agcount, + dblocks); + pag_group(pag)->xg_min_gbno = XFS_AGFL_BLOCK(mp) + 1; + __xfs_agino_range(mp, pag_group(pag)->xg_block_count, &pag->agino_min, + &pag->agino_max); + + error = xfs_group_insert(mp, pag_group(pag), index, XG_TYPE_AG); + if (error) + goto out_free_perag; + + return 0; + +out_free_perag: + kfree(pag); + return error; +} + +int +xfs_initialize_perag( + struct xfs_mount *mp, + xfs_agnumber_t orig_agcount, + xfs_agnumber_t new_agcount, + xfs_rfsblock_t dblocks, + xfs_agnumber_t *maxagi) +{ + xfs_agnumber_t index; + int error; + + if (orig_agcount >= new_agcount) + return 0; + + for (index = orig_agcount; index < new_agcount; index++) { + error = xfs_perag_alloc(mp, index, new_agcount, dblocks); + if (error) + goto out_unwind_new_pags; + } + + *maxagi = xfs_set_inode_alloc(mp, new_agcount); + mp->m_ag_prealloc_blocks = xfs_prealloc_blocks(mp); + return 0; + +out_unwind_new_pags: + xfs_free_perag_range(mp, orig_agcount, index); + return error; +} + +static int +xfs_get_aghdr_buf( + struct xfs_mount *mp, + xfs_daddr_t blkno, + size_t numblks, + struct xfs_buf **bpp, + const struct xfs_buf_ops *ops) +{ + struct xfs_buf *bp; + int error; + + error = xfs_buf_get_uncached(mp->m_ddev_targp, numblks, &bp); + if (error) + return error; + + bp->b_maps[0].bm_bn = blkno; + bp->b_ops = ops; + + *bpp = bp; + return 0; +} + +/* + * Generic btree root block init function + */ +static void +xfs_btroot_init( + struct xfs_mount *mp, + struct xfs_buf *bp, + struct aghdr_init_data *id) +{ + xfs_btree_init_buf(mp, bp, id->bc_ops, 0, 0, id->agno); +} + +/* Finish initializing a free space btree. */ +static void +xfs_freesp_init_recs( + struct xfs_mount *mp, + struct xfs_buf *bp, + struct aghdr_init_data *id) +{ + struct xfs_alloc_rec *arec; + struct xfs_btree_block *block = XFS_BUF_TO_BLOCK(bp); + + arec = XFS_ALLOC_REC_ADDR(mp, XFS_BUF_TO_BLOCK(bp), 1); + arec->ar_startblock = cpu_to_be32(mp->m_ag_prealloc_blocks); + + if (xfs_ag_contains_log(mp, id->agno)) { + struct xfs_alloc_rec *nrec; + xfs_agblock_t start = XFS_FSB_TO_AGBNO(mp, + mp->m_sb.sb_logstart); + + ASSERT(start >= mp->m_ag_prealloc_blocks); + if (start != mp->m_ag_prealloc_blocks) { + /* + * Modify first record to pad stripe align of log and + * bump the record count. + */ + arec->ar_blockcount = cpu_to_be32(start - + mp->m_ag_prealloc_blocks); + be16_add_cpu(&block->bb_numrecs, 1); + nrec = arec + 1; + + /* + * Insert second record at start of internal log + * which then gets trimmed. + */ + nrec->ar_startblock = cpu_to_be32( + be32_to_cpu(arec->ar_startblock) + + be32_to_cpu(arec->ar_blockcount)); + arec = nrec; + } + /* + * Change record start to after the internal log + */ + be32_add_cpu(&arec->ar_startblock, mp->m_sb.sb_logblocks); + } + + /* + * Calculate the block count of this record; if it is nonzero, + * increment the record count. + */ + arec->ar_blockcount = cpu_to_be32(id->agsize - + be32_to_cpu(arec->ar_startblock)); + if (arec->ar_blockcount) + be16_add_cpu(&block->bb_numrecs, 1); +} + +/* + * bnobt/cntbt btree root block init functions + */ +static void +xfs_bnoroot_init( + struct xfs_mount *mp, + struct xfs_buf *bp, + struct aghdr_init_data *id) +{ + xfs_btree_init_buf(mp, bp, id->bc_ops, 0, 0, id->agno); + xfs_freesp_init_recs(mp, bp, id); +} + +/* + * Reverse map root block init + */ +static void +xfs_rmaproot_init( + struct xfs_mount *mp, + struct xfs_buf *bp, + struct aghdr_init_data *id) +{ + struct xfs_btree_block *block = XFS_BUF_TO_BLOCK(bp); + struct xfs_rmap_rec *rrec; + + xfs_btree_init_buf(mp, bp, id->bc_ops, 0, 4, id->agno); + + /* + * mark the AG header regions as static metadata The BNO + * btree block is the first block after the headers, so + * it's location defines the size of region the static + * metadata consumes. + * + * Note: unlike mkfs, we never have to account for log + * space when growing the data regions + */ + rrec = XFS_RMAP_REC_ADDR(block, 1); + rrec->rm_startblock = 0; + rrec->rm_blockcount = cpu_to_be32(XFS_BNO_BLOCK(mp)); + rrec->rm_owner = cpu_to_be64(XFS_RMAP_OWN_FS); + rrec->rm_offset = 0; + + /* account freespace btree root blocks */ + rrec = XFS_RMAP_REC_ADDR(block, 2); + rrec->rm_startblock = cpu_to_be32(XFS_BNO_BLOCK(mp)); + rrec->rm_blockcount = cpu_to_be32(2); + rrec->rm_owner = cpu_to_be64(XFS_RMAP_OWN_AG); + rrec->rm_offset = 0; + + /* account inode btree root blocks */ + rrec = XFS_RMAP_REC_ADDR(block, 3); + rrec->rm_startblock = cpu_to_be32(XFS_IBT_BLOCK(mp)); + rrec->rm_blockcount = cpu_to_be32(XFS_RMAP_BLOCK(mp) - + XFS_IBT_BLOCK(mp)); + rrec->rm_owner = cpu_to_be64(XFS_RMAP_OWN_INOBT); + rrec->rm_offset = 0; + + /* account for rmap btree root */ + rrec = XFS_RMAP_REC_ADDR(block, 4); + rrec->rm_startblock = cpu_to_be32(XFS_RMAP_BLOCK(mp)); + rrec->rm_blockcount = cpu_to_be32(1); + rrec->rm_owner = cpu_to_be64(XFS_RMAP_OWN_AG); + rrec->rm_offset = 0; + + /* account for refc btree root */ + if (xfs_has_reflink(mp)) { + rrec = XFS_RMAP_REC_ADDR(block, 5); + rrec->rm_startblock = cpu_to_be32(xfs_refc_block(mp)); + rrec->rm_blockcount = cpu_to_be32(1); + rrec->rm_owner = cpu_to_be64(XFS_RMAP_OWN_REFC); + rrec->rm_offset = 0; + be16_add_cpu(&block->bb_numrecs, 1); + } + + /* account for the log space */ + if (xfs_ag_contains_log(mp, id->agno)) { + rrec = XFS_RMAP_REC_ADDR(block, + be16_to_cpu(block->bb_numrecs) + 1); + rrec->rm_startblock = cpu_to_be32( + XFS_FSB_TO_AGBNO(mp, mp->m_sb.sb_logstart)); + rrec->rm_blockcount = cpu_to_be32(mp->m_sb.sb_logblocks); + rrec->rm_owner = cpu_to_be64(XFS_RMAP_OWN_LOG); + rrec->rm_offset = 0; + be16_add_cpu(&block->bb_numrecs, 1); + } +} + +/* + * Initialise new secondary superblocks with the pre-grow geometry, but mark + * them as "in progress" so we know they haven't yet been activated. This will + * get cleared when the update with the new geometry information is done after + * changes to the primary are committed. This isn't strictly necessary, but we + * get it for free with the delayed buffer write lists and it means we can tell + * if a grow operation didn't complete properly after the fact. + */ +static void +xfs_sbblock_init( + struct xfs_mount *mp, + struct xfs_buf *bp, + struct aghdr_init_data *id) +{ + struct xfs_dsb *dsb = bp->b_addr; + + xfs_sb_to_disk(dsb, &mp->m_sb); + dsb->sb_inprogress = 1; +} + +static void +xfs_agfblock_init( + struct xfs_mount *mp, + struct xfs_buf *bp, + struct aghdr_init_data *id) +{ + struct xfs_agf *agf = bp->b_addr; + xfs_extlen_t tmpsize; + + agf->agf_magicnum = cpu_to_be32(XFS_AGF_MAGIC); + agf->agf_versionnum = cpu_to_be32(XFS_AGF_VERSION); + agf->agf_seqno = cpu_to_be32(id->agno); + agf->agf_length = cpu_to_be32(id->agsize); + agf->agf_bno_root = cpu_to_be32(XFS_BNO_BLOCK(mp)); + agf->agf_cnt_root = cpu_to_be32(XFS_CNT_BLOCK(mp)); + agf->agf_bno_level = cpu_to_be32(1); + agf->agf_cnt_level = cpu_to_be32(1); + if (xfs_has_rmapbt(mp)) { + agf->agf_rmap_root = cpu_to_be32(XFS_RMAP_BLOCK(mp)); + agf->agf_rmap_level = cpu_to_be32(1); + agf->agf_rmap_blocks = cpu_to_be32(1); + } + + agf->agf_flfirst = cpu_to_be32(1); + agf->agf_fllast = 0; + agf->agf_flcount = 0; + tmpsize = id->agsize - mp->m_ag_prealloc_blocks; + agf->agf_freeblks = cpu_to_be32(tmpsize); + agf->agf_longest = cpu_to_be32(tmpsize); + if (xfs_has_crc(mp)) + uuid_copy(&agf->agf_uuid, &mp->m_sb.sb_meta_uuid); + if (xfs_has_reflink(mp)) { + agf->agf_refcount_root = cpu_to_be32( + xfs_refc_block(mp)); + agf->agf_refcount_level = cpu_to_be32(1); + agf->agf_refcount_blocks = cpu_to_be32(1); + } + + if (xfs_ag_contains_log(mp, id->agno)) { + int64_t logblocks = mp->m_sb.sb_logblocks; + + be32_add_cpu(&agf->agf_freeblks, -logblocks); + agf->agf_longest = cpu_to_be32(id->agsize - + XFS_FSB_TO_AGBNO(mp, mp->m_sb.sb_logstart) - logblocks); + } +} + +static void +xfs_agflblock_init( + struct xfs_mount *mp, + struct xfs_buf *bp, + struct aghdr_init_data *id) +{ + struct xfs_agfl *agfl = XFS_BUF_TO_AGFL(bp); + __be32 *agfl_bno; + int bucket; + + if (xfs_has_crc(mp)) { + agfl->agfl_magicnum = cpu_to_be32(XFS_AGFL_MAGIC); + agfl->agfl_seqno = cpu_to_be32(id->agno); + uuid_copy(&agfl->agfl_uuid, &mp->m_sb.sb_meta_uuid); + } + + agfl_bno = xfs_buf_to_agfl_bno(bp); + for (bucket = 0; bucket < xfs_agfl_size(mp); bucket++) + agfl_bno[bucket] = cpu_to_be32(NULLAGBLOCK); +} + +static void +xfs_agiblock_init( + struct xfs_mount *mp, + struct xfs_buf *bp, + struct aghdr_init_data *id) +{ + struct xfs_agi *agi = bp->b_addr; + int bucket; + + agi->agi_magicnum = cpu_to_be32(XFS_AGI_MAGIC); + agi->agi_versionnum = cpu_to_be32(XFS_AGI_VERSION); + agi->agi_seqno = cpu_to_be32(id->agno); + agi->agi_length = cpu_to_be32(id->agsize); + agi->agi_count = 0; + agi->agi_root = cpu_to_be32(XFS_IBT_BLOCK(mp)); + agi->agi_level = cpu_to_be32(1); + agi->agi_freecount = 0; + agi->agi_newino = cpu_to_be32(NULLAGINO); + agi->agi_dirino = cpu_to_be32(NULLAGINO); + if (xfs_has_crc(mp)) + uuid_copy(&agi->agi_uuid, &mp->m_sb.sb_meta_uuid); + if (xfs_has_finobt(mp)) { + agi->agi_free_root = cpu_to_be32(XFS_FIBT_BLOCK(mp)); + agi->agi_free_level = cpu_to_be32(1); + } + for (bucket = 0; bucket < XFS_AGI_UNLINKED_BUCKETS; bucket++) + agi->agi_unlinked[bucket] = cpu_to_be32(NULLAGINO); + if (xfs_has_inobtcounts(mp)) { + agi->agi_iblocks = cpu_to_be32(1); + if (xfs_has_finobt(mp)) + agi->agi_fblocks = cpu_to_be32(1); + } +} + +typedef void (*aghdr_init_work_f)(struct xfs_mount *mp, struct xfs_buf *bp, + struct aghdr_init_data *id); +static int +xfs_ag_init_hdr( + struct xfs_mount *mp, + struct aghdr_init_data *id, + aghdr_init_work_f work, + const struct xfs_buf_ops *ops) +{ + struct xfs_buf *bp; + int error; + + error = xfs_get_aghdr_buf(mp, id->daddr, id->numblks, &bp, ops); + if (error) + return error; + + (*work)(mp, bp, id); + + xfs_buf_delwri_queue(bp, &id->buffer_list); + xfs_buf_relse(bp); + return 0; +} + +struct xfs_aghdr_grow_data { + xfs_daddr_t daddr; + size_t numblks; + const struct xfs_buf_ops *ops; + aghdr_init_work_f work; + const struct xfs_btree_ops *bc_ops; + bool need_init; +}; + +/* + * Prepare new AG headers to be written to disk. We use uncached buffers here, + * as it is assumed these new AG headers are currently beyond the currently + * valid filesystem address space. Using cached buffers would trip over EOFS + * corruption detection alogrithms in the buffer cache lookup routines. + * + * This is a non-transactional function, but the prepared buffers are added to a + * delayed write buffer list supplied by the caller so they can submit them to + * disk and wait on them as required. + */ +int +xfs_ag_init_headers( + struct xfs_mount *mp, + struct aghdr_init_data *id) + +{ + struct xfs_aghdr_grow_data aghdr_data[] = { + { /* SB */ + .daddr = XFS_AG_DADDR(mp, id->agno, XFS_SB_DADDR), + .numblks = XFS_FSS_TO_BB(mp, 1), + .ops = &xfs_sb_buf_ops, + .work = &xfs_sbblock_init, + .need_init = true + }, + { /* AGF */ + .daddr = XFS_AG_DADDR(mp, id->agno, XFS_AGF_DADDR(mp)), + .numblks = XFS_FSS_TO_BB(mp, 1), + .ops = &xfs_agf_buf_ops, + .work = &xfs_agfblock_init, + .need_init = true + }, + { /* AGFL */ + .daddr = XFS_AG_DADDR(mp, id->agno, XFS_AGFL_DADDR(mp)), + .numblks = XFS_FSS_TO_BB(mp, 1), + .ops = &xfs_agfl_buf_ops, + .work = &xfs_agflblock_init, + .need_init = true + }, + { /* AGI */ + .daddr = XFS_AG_DADDR(mp, id->agno, XFS_AGI_DADDR(mp)), + .numblks = XFS_FSS_TO_BB(mp, 1), + .ops = &xfs_agi_buf_ops, + .work = &xfs_agiblock_init, + .need_init = true + }, + { /* BNO root block */ + .daddr = XFS_AGB_TO_DADDR(mp, id->agno, XFS_BNO_BLOCK(mp)), + .numblks = BTOBB(mp->m_sb.sb_blocksize), + .ops = &xfs_bnobt_buf_ops, + .work = &xfs_bnoroot_init, + .bc_ops = &xfs_bnobt_ops, + .need_init = true + }, + { /* CNT root block */ + .daddr = XFS_AGB_TO_DADDR(mp, id->agno, XFS_CNT_BLOCK(mp)), + .numblks = BTOBB(mp->m_sb.sb_blocksize), + .ops = &xfs_cntbt_buf_ops, + .work = &xfs_bnoroot_init, + .bc_ops = &xfs_cntbt_ops, + .need_init = true + }, + { /* INO root block */ + .daddr = XFS_AGB_TO_DADDR(mp, id->agno, XFS_IBT_BLOCK(mp)), + .numblks = BTOBB(mp->m_sb.sb_blocksize), + .ops = &xfs_inobt_buf_ops, + .work = &xfs_btroot_init, + .bc_ops = &xfs_inobt_ops, + .need_init = true + }, + { /* FINO root block */ + .daddr = XFS_AGB_TO_DADDR(mp, id->agno, XFS_FIBT_BLOCK(mp)), + .numblks = BTOBB(mp->m_sb.sb_blocksize), + .ops = &xfs_finobt_buf_ops, + .work = &xfs_btroot_init, + .bc_ops = &xfs_finobt_ops, + .need_init = xfs_has_finobt(mp) + }, + { /* RMAP root block */ + .daddr = XFS_AGB_TO_DADDR(mp, id->agno, XFS_RMAP_BLOCK(mp)), + .numblks = BTOBB(mp->m_sb.sb_blocksize), + .ops = &xfs_rmapbt_buf_ops, + .work = &xfs_rmaproot_init, + .bc_ops = &xfs_rmapbt_ops, + .need_init = xfs_has_rmapbt(mp) + }, + { /* REFC root block */ + .daddr = XFS_AGB_TO_DADDR(mp, id->agno, xfs_refc_block(mp)), + .numblks = BTOBB(mp->m_sb.sb_blocksize), + .ops = &xfs_refcountbt_buf_ops, + .work = &xfs_btroot_init, + .bc_ops = &xfs_refcountbt_ops, + .need_init = xfs_has_reflink(mp) + }, + { /* NULL terminating block */ + .daddr = XFS_BUF_DADDR_NULL, + } + }; + struct xfs_aghdr_grow_data *dp; + int error = 0; + + /* Account for AG free space in new AG */ + id->nfree += id->agsize - mp->m_ag_prealloc_blocks; + for (dp = &aghdr_data[0]; dp->daddr != XFS_BUF_DADDR_NULL; dp++) { + if (!dp->need_init) + continue; + + id->daddr = dp->daddr; + id->numblks = dp->numblks; + id->bc_ops = dp->bc_ops; + error = xfs_ag_init_hdr(mp, id, dp->work, dp->ops); + if (error) + break; + } + return error; +} + +int +xfs_ag_shrink_space( + struct xfs_perag *pag, + struct xfs_trans **tpp, + xfs_extlen_t delta) +{ + struct xfs_mount *mp = pag_mount(pag); + struct xfs_alloc_arg args = { + .tp = *tpp, + .mp = mp, + .pag = pag, + .minlen = delta, + .maxlen = delta, + .oinfo = XFS_RMAP_OINFO_SKIP_UPDATE, + .resv = XFS_AG_RESV_NONE, + .prod = 1 + }; + struct xfs_buf *agibp, *agfbp; + struct xfs_agi *agi; + struct xfs_agf *agf; + xfs_agblock_t aglen; + int error, err2; + + ASSERT(pag_agno(pag) == mp->m_sb.sb_agcount - 1); + error = xfs_ialloc_read_agi(pag, *tpp, 0, &agibp); + if (error) + return error; + + agi = agibp->b_addr; + + error = xfs_alloc_read_agf(pag, *tpp, 0, &agfbp); + if (error) + return error; + + agf = agfbp->b_addr; + aglen = be32_to_cpu(agi->agi_length); + /* some extra paranoid checks before we shrink the ag */ + if (XFS_IS_CORRUPT(mp, agf->agf_length != agi->agi_length)) { + xfs_ag_mark_sick(pag, XFS_SICK_AG_AGF); + return -EFSCORRUPTED; + } + if (delta >= aglen) + return -EINVAL; + + /* + * Make sure that the last inode cluster cannot overlap with the new + * end of the AG, even if it's sparse. + */ + error = xfs_ialloc_check_shrink(pag, *tpp, agibp, aglen - delta); + if (error) + return error; + + /* + * Disable perag reservations so it doesn't cause the allocation request + * to fail. We'll reestablish reservation before we return. + */ + xfs_ag_resv_free(pag); + + /* internal log shouldn't also show up in the free space btrees */ + error = xfs_alloc_vextent_exact_bno(&args, + xfs_agbno_to_fsb(pag, aglen - delta)); + if (!error && args.agbno == NULLAGBLOCK) + error = -ENOSPC; + + if (error) { + /* + * If extent allocation fails, need to roll the transaction to + * ensure that the AGFL fixup has been committed anyway. + * + * We need to hold the AGF across the roll to ensure nothing can + * access the AG for allocation until the shrink is fully + * cleaned up. And due to the resetting of the AG block + * reservation space needing to lock the AGI, we also have to + * hold that so we don't get AGI/AGF lock order inversions in + * the error handling path. + */ + xfs_trans_bhold(*tpp, agfbp); + xfs_trans_bhold(*tpp, agibp); + err2 = xfs_trans_roll(tpp); + if (err2) + return err2; + xfs_trans_bjoin(*tpp, agfbp); + xfs_trans_bjoin(*tpp, agibp); + goto resv_init_out; + } + + /* + * if successfully deleted from freespace btrees, need to confirm + * per-AG reservation works as expected. + */ + be32_add_cpu(&agi->agi_length, -delta); + be32_add_cpu(&agf->agf_length, -delta); + + err2 = xfs_ag_resv_init(pag, *tpp); + if (err2) { + be32_add_cpu(&agi->agi_length, delta); + be32_add_cpu(&agf->agf_length, delta); + if (err2 != -ENOSPC) + goto resv_err; + + err2 = xfs_free_extent_later(*tpp, args.fsbno, delta, NULL, + XFS_AG_RESV_NONE, XFS_FREE_EXTENT_SKIP_DISCARD); + if (err2) + goto resv_err; + + /* + * Roll the transaction before trying to re-init the per-ag + * reservation. The new transaction is clean so it will cancel + * without any side effects. + */ + error = xfs_defer_finish(tpp); + if (error) + return error; + + error = -ENOSPC; + goto resv_init_out; + } + + /* Update perag geometry */ + pag_group(pag)->xg_block_count -= delta; + __xfs_agino_range(mp, pag_group(pag)->xg_block_count, &pag->agino_min, + &pag->agino_max); + + xfs_ialloc_log_agi(*tpp, agibp, XFS_AGI_LENGTH); + xfs_alloc_log_agf(*tpp, agfbp, XFS_AGF_LENGTH); + return 0; + +resv_init_out: + err2 = xfs_ag_resv_init(pag, *tpp); + if (!err2) + return error; +resv_err: + xfs_warn(mp, "Error %d reserving per-AG metadata reserve pool.", err2); + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); + return err2; +} + +/* + * Return the agcount for the new file system size passed in *nb and adjust *nb + * when it has to be reduced because of maximum AG count or because it would + * create a below minimum size AG. + */ +xfs_agnumber_t +xfs_growfs_compute_agcount( + struct xfs_mount *mp, + xfs_rfsblock_t *nb) +{ + uint64_t agcount; /* 64-bits wide to catch overflows */ + xfs_extlen_t remainder; + + agcount = div_u64_rem(*nb, mp->m_sb.sb_agblocks, &remainder); + if (agcount >= XFS_MAX_AGNUMBER + 1) { + agcount = XFS_MAX_AGNUMBER + 1; + remainder = 0; + } + *nb = (xfs_rfsblock_t)agcount * mp->m_sb.sb_agblocks; + if (remainder >= XFS_MIN_AG_BLOCKS) { + *nb += remainder; + agcount++; + } + return agcount; +} + +/* + * Extent the AG indicated by the @id by the length passed in + */ +int +xfs_ag_extend_space( + struct xfs_perag *pag, + struct xfs_trans *tp, + xfs_extlen_t len) +{ + struct xfs_mount *mp = pag_mount(pag); + struct xfs_buf *bp; + struct xfs_agi *agi; + struct xfs_agf *agf; + int error; + + ASSERT(pag_agno(pag) == mp->m_sb.sb_agcount - 1); + + error = xfs_ialloc_read_agi(pag, tp, 0, &bp); + if (error) + return error; + + agi = bp->b_addr; + be32_add_cpu(&agi->agi_length, len); + xfs_ialloc_log_agi(tp, bp, XFS_AGI_LENGTH); + + /* + * Change agf length. + */ + error = xfs_alloc_read_agf(pag, tp, 0, &bp); + if (error) + return error; + + agf = bp->b_addr; + be32_add_cpu(&agf->agf_length, len); + ASSERT(agf->agf_length == agi->agi_length); + xfs_alloc_log_agf(tp, bp, XFS_AGF_LENGTH); + + /* + * Free the new space. + * + * XFS_RMAP_OINFO_SKIP_UPDATE is used here to tell the rmap btree that + * this doesn't actually exist in the rmap btree. + */ + error = xfs_rmap_free(tp, bp, pag, be32_to_cpu(agf->agf_length) - len, + len, &XFS_RMAP_OINFO_SKIP_UPDATE); + if (error) + return error; + + error = xfs_free_extent(tp, pag, be32_to_cpu(agf->agf_length) - len, + len, &XFS_RMAP_OINFO_SKIP_UPDATE, XFS_AG_RESV_NONE); + if (error) + return error; + + /* Update perag geometry */ + pag_group(pag)->xg_block_count = be32_to_cpu(agf->agf_length); + __xfs_agino_range(mp, pag_group(pag)->xg_block_count, &pag->agino_min, + &pag->agino_max); + return 0; +} + +/* Retrieve AG geometry. */ +int +xfs_ag_get_geometry( + struct xfs_perag *pag, + struct xfs_ag_geometry *ageo) +{ + struct xfs_buf *agi_bp; + struct xfs_buf *agf_bp; + struct xfs_agi *agi; + struct xfs_agf *agf; + unsigned int freeblks; + int error; + + /* Lock the AG headers. */ + error = xfs_ialloc_read_agi(pag, NULL, 0, &agi_bp); + if (error) + return error; + error = xfs_alloc_read_agf(pag, NULL, 0, &agf_bp); + if (error) + goto out_agi; + + /* Fill out form. */ + memset(ageo, 0, sizeof(*ageo)); + ageo->ag_number = pag_agno(pag); + + agi = agi_bp->b_addr; + ageo->ag_icount = be32_to_cpu(agi->agi_count); + ageo->ag_ifree = be32_to_cpu(agi->agi_freecount); + + agf = agf_bp->b_addr; + ageo->ag_length = be32_to_cpu(agf->agf_length); + freeblks = pag->pagf_freeblks + + pag->pagf_flcount + + pag->pagf_btreeblks - + xfs_ag_resv_needed(pag, XFS_AG_RESV_NONE); + ageo->ag_freeblks = freeblks; + xfs_ag_geom_health(pag, ageo); + + /* Release resources. */ + xfs_buf_relse(agf_bp); +out_agi: + xfs_buf_relse(agi_bp); + return error; +} diff --git a/fs/xfs/libxfs/xfs_ag.h b/fs/xfs/libxfs/xfs_ag.h new file mode 100644 index 000000000..fd22fe598 --- /dev/null +++ b/fs/xfs/libxfs/xfs_ag.h @@ -0,0 +1,362 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2018 Red Hat, Inc. + * All rights reserved. + */ + +#ifndef __LIBXFS_AG_H +#define __LIBXFS_AG_H 1 + +#include "xfs_group.h" + +struct xfs_mount; +struct xfs_trans; +struct xfs_perag; + +/* + * Per-ag infrastructure + */ + +/* per-AG block reservation data structures*/ +struct xfs_ag_resv { + /* number of blocks originally reserved here */ + xfs_extlen_t ar_orig_reserved; + /* number of blocks reserved here */ + xfs_extlen_t ar_reserved; + /* number of blocks originally asked for */ + xfs_extlen_t ar_asked; +}; + +/* + * Per-ag incore structure, copies of information in agf and agi, to improve the + * performance of allocation group selection. + */ +struct xfs_perag { + struct xfs_group pag_group; + unsigned long pag_opstate; + uint8_t pagf_bno_level; /* # of levels in bno btree */ + uint8_t pagf_cnt_level; /* # of levels in cnt btree */ + uint8_t pagf_rmap_level;/* # of levels in rmap btree */ + uint32_t pagf_flcount; /* count of blocks in freelist */ + xfs_extlen_t pagf_freeblks; /* total free blocks */ + xfs_extlen_t pagf_longest; /* longest free space */ + uint32_t pagf_btreeblks; /* # of blocks held in AGF btrees */ + xfs_agino_t pagi_freecount; /* number of free inodes */ + xfs_agino_t pagi_count; /* number of allocated inodes */ + + /* + * Inode allocation search lookup optimisation. + * If the pagino matches, the search for new inodes + * doesn't need to search the near ones again straight away + */ + xfs_agino_t pagl_pagino; + xfs_agino_t pagl_leftrec; + xfs_agino_t pagl_rightrec; + + uint8_t pagf_refcount_level; /* recount btree height */ + + /* Blocks reserved for all kinds of metadata. */ + struct xfs_ag_resv pag_meta_resv; + /* Blocks reserved for the reverse mapping btree. */ + struct xfs_ag_resv pag_rmapbt_resv; + + /* Precalculated geometry info */ + xfs_agino_t agino_min; + xfs_agino_t agino_max; + +#ifdef __KERNEL__ + /* -- kernel only structures below this line -- */ + +#ifdef CONFIG_XFS_ONLINE_REPAIR + /* + * Alternate btree heights so that online repair won't trip the write + * verifiers while rebuilding the AG btrees. + */ + uint8_t pagf_repair_bno_level; + uint8_t pagf_repair_cnt_level; + uint8_t pagf_repair_refcount_level; + uint8_t pagf_repair_rmap_level; +#endif + + atomic_t pagf_fstrms; /* # of filestreams active in this AG */ + + spinlock_t pag_ici_lock; /* incore inode cache lock */ + struct radix_tree_root pag_ici_root; /* incore inode cache root */ + int pag_ici_reclaimable; /* reclaimable inodes */ + unsigned long pag_ici_reclaim_cursor; /* reclaim restart point */ + + /* background prealloc block trimming */ + struct delayed_work pag_blockgc_work; +#endif /* __KERNEL__ */ +}; + +static inline struct xfs_perag *to_perag(struct xfs_group *xg) +{ + return container_of(xg, struct xfs_perag, pag_group); +} + +static inline struct xfs_group *pag_group(struct xfs_perag *pag) +{ + return &pag->pag_group; +} + +static inline struct xfs_mount *pag_mount(const struct xfs_perag *pag) +{ + return pag->pag_group.xg_mount; +} + +static inline xfs_agnumber_t pag_agno(const struct xfs_perag *pag) +{ + return pag->pag_group.xg_gno; +} + +/* + * Per-AG operational state. These are atomic flag bits. + */ +#define XFS_AGSTATE_AGF_INIT 0 +#define XFS_AGSTATE_AGI_INIT 1 +#define XFS_AGSTATE_PREFERS_METADATA 2 +#define XFS_AGSTATE_ALLOWS_INODES 3 +#define XFS_AGSTATE_AGFL_NEEDS_RESET 4 + +#define __XFS_AG_OPSTATE(name, NAME) \ +static inline bool xfs_perag_ ## name (struct xfs_perag *pag) \ +{ \ + return test_bit(XFS_AGSTATE_ ## NAME, &pag->pag_opstate); \ +} + +__XFS_AG_OPSTATE(initialised_agf, AGF_INIT) +__XFS_AG_OPSTATE(initialised_agi, AGI_INIT) +__XFS_AG_OPSTATE(prefers_metadata, PREFERS_METADATA) +__XFS_AG_OPSTATE(allows_inodes, ALLOWS_INODES) +__XFS_AG_OPSTATE(agfl_needs_reset, AGFL_NEEDS_RESET) + +int xfs_initialize_perag(struct xfs_mount *mp, xfs_agnumber_t orig_agcount, + xfs_agnumber_t new_agcount, xfs_rfsblock_t dcount, + xfs_agnumber_t *maxagi); +void xfs_free_perag_range(struct xfs_mount *mp, xfs_agnumber_t first_agno, + xfs_agnumber_t end_agno); +int xfs_initialize_perag_data(struct xfs_mount *mp, xfs_agnumber_t agno); +int xfs_update_last_ag_size(struct xfs_mount *mp, xfs_agnumber_t prev_agcount); + +/* Passive AG references */ +static inline struct xfs_perag * +xfs_perag_get( + struct xfs_mount *mp, + xfs_agnumber_t agno) +{ + return to_perag(xfs_group_get(mp, agno, XG_TYPE_AG)); +} + +static inline struct xfs_perag * +xfs_perag_hold( + struct xfs_perag *pag) +{ + return to_perag(xfs_group_hold(pag_group(pag))); +} + +static inline void +xfs_perag_put( + struct xfs_perag *pag) +{ + xfs_group_put(pag_group(pag)); +} + +/* Active AG references */ +static inline struct xfs_perag * +xfs_perag_grab( + struct xfs_mount *mp, + xfs_agnumber_t agno) +{ + return to_perag(xfs_group_grab(mp, agno, XG_TYPE_AG)); +} + +static inline void +xfs_perag_rele( + struct xfs_perag *pag) +{ + xfs_group_rele(pag_group(pag)); +} + +static inline struct xfs_perag * +xfs_perag_next_range( + struct xfs_mount *mp, + struct xfs_perag *pag, + xfs_agnumber_t start_agno, + xfs_agnumber_t end_agno) +{ + return to_perag(xfs_group_next_range(mp, pag ? pag_group(pag) : NULL, + start_agno, end_agno, XG_TYPE_AG)); +} + +static inline struct xfs_perag * +xfs_perag_next_from( + struct xfs_mount *mp, + struct xfs_perag *pag, + xfs_agnumber_t start_agno) +{ + return xfs_perag_next_range(mp, pag, start_agno, mp->m_sb.sb_agcount - 1); +} + +static inline struct xfs_perag * +xfs_perag_next( + struct xfs_mount *mp, + struct xfs_perag *pag) +{ + return xfs_perag_next_from(mp, pag, 0); +} + +/* + * Per-ag geometry infomation and validation + */ +xfs_agblock_t xfs_ag_block_count(struct xfs_mount *mp, xfs_agnumber_t agno); +void xfs_agino_range(struct xfs_mount *mp, xfs_agnumber_t agno, + xfs_agino_t *first, xfs_agino_t *last); + +static inline bool +xfs_verify_agbno(struct xfs_perag *pag, xfs_agblock_t agbno) +{ + return xfs_verify_gbno(pag_group(pag), agbno); +} + +static inline bool +xfs_verify_agbext( + struct xfs_perag *pag, + xfs_agblock_t agbno, + xfs_agblock_t len) +{ + return xfs_verify_gbext(pag_group(pag), agbno, len); +} + +/* + * Verify that an AG inode number pointer neither points outside the AG + * nor points at static metadata. + */ +static inline bool +xfs_verify_agino(struct xfs_perag *pag, xfs_agino_t agino) +{ + if (agino < pag->agino_min) + return false; + if (agino > pag->agino_max) + return false; + return true; +} + +/* + * Verify that an AG inode number pointer neither points outside the AG + * nor points at static metadata, or is NULLAGINO. + */ +static inline bool +xfs_verify_agino_or_null(struct xfs_perag *pag, xfs_agino_t agino) +{ + if (agino == NULLAGINO) + return true; + return xfs_verify_agino(pag, agino); +} + +static inline bool +xfs_ag_contains_log(struct xfs_mount *mp, xfs_agnumber_t agno) +{ + return mp->m_sb.sb_logstart > 0 && + agno == XFS_FSB_TO_AGNO(mp, mp->m_sb.sb_logstart); +} + +static inline struct xfs_perag * +xfs_perag_next_wrap( + struct xfs_perag *pag, + xfs_agnumber_t *agno, + xfs_agnumber_t stop_agno, + xfs_agnumber_t restart_agno, + xfs_agnumber_t wrap_agno) +{ + struct xfs_mount *mp = pag_mount(pag); + + *agno = pag_agno(pag) + 1; + xfs_perag_rele(pag); + while (*agno != stop_agno) { + if (*agno >= wrap_agno) { + if (restart_agno >= stop_agno) + break; + *agno = restart_agno; + } + + pag = xfs_perag_grab(mp, *agno); + if (pag) + return pag; + (*agno)++; + } + return NULL; +} + +/* + * Iterate all AGs from start_agno through wrap_agno, then restart_agno through + * (start_agno - 1). + */ +#define for_each_perag_wrap_range(mp, start_agno, restart_agno, wrap_agno, agno, pag) \ + for ((agno) = (start_agno), (pag) = xfs_perag_grab((mp), (agno)); \ + (pag) != NULL; \ + (pag) = xfs_perag_next_wrap((pag), &(agno), (start_agno), \ + (restart_agno), (wrap_agno))) +/* + * Iterate all AGs from start_agno through wrap_agno, then 0 through + * (start_agno - 1). + */ +#define for_each_perag_wrap_at(mp, start_agno, wrap_agno, agno, pag) \ + for_each_perag_wrap_range((mp), (start_agno), 0, (wrap_agno), (agno), (pag)) + +/* + * Iterate all AGs from start_agno through to the end of the filesystem, then 0 + * through (start_agno - 1). + */ +#define for_each_perag_wrap(mp, start_agno, agno, pag) \ + for_each_perag_wrap_at((mp), (start_agno), (mp)->m_sb.sb_agcount, \ + (agno), (pag)) + + +struct aghdr_init_data { + /* per ag data */ + xfs_agblock_t agno; /* ag to init */ + xfs_extlen_t agsize; /* new AG size */ + struct list_head buffer_list; /* buffer writeback list */ + xfs_rfsblock_t nfree; /* cumulative new free space */ + + /* per header data */ + xfs_daddr_t daddr; /* header location */ + size_t numblks; /* size of header */ + const struct xfs_btree_ops *bc_ops; /* btree ops */ +}; + +int xfs_ag_init_headers(struct xfs_mount *mp, struct aghdr_init_data *id); +int xfs_ag_shrink_space(struct xfs_perag *pag, struct xfs_trans **tpp, + xfs_extlen_t delta); +int xfs_ag_extend_space(struct xfs_perag *pag, struct xfs_trans *tp, + xfs_extlen_t len); +int xfs_ag_get_geometry(struct xfs_perag *pag, struct xfs_ag_geometry *ageo); +xfs_agnumber_t xfs_growfs_compute_agcount(struct xfs_mount *mp, + xfs_rfsblock_t *nb); + +static inline xfs_fsblock_t +xfs_agbno_to_fsb( + struct xfs_perag *pag, + xfs_agblock_t agbno) +{ + return XFS_AGB_TO_FSB(pag_mount(pag), pag_agno(pag), agbno); +} + +static inline xfs_daddr_t +xfs_agbno_to_daddr( + struct xfs_perag *pag, + xfs_agblock_t agbno) +{ + return XFS_AGB_TO_DADDR(pag_mount(pag), pag_agno(pag), agbno); +} + +static inline xfs_ino_t +xfs_agino_to_ino( + struct xfs_perag *pag, + xfs_agino_t agino) +{ + return XFS_AGINO_TO_INO(pag_mount(pag), pag_agno(pag), agino); +} + +#endif /* __LIBXFS_AG_H */ diff --git a/fs/xfs/libxfs/xfs_ag_resv.c b/fs/xfs/libxfs/xfs_ag_resv.c new file mode 100644 index 000000000..c4cdcc570 --- /dev/null +++ b/fs/xfs/libxfs/xfs_ag_resv.c @@ -0,0 +1,418 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2016 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_alloc.h" +#include "xfs_errortag.h" +#include "xfs_error.h" +#include "xfs_trace.h" +#include "xfs_trans.h" +#include "xfs_rmap_btree.h" +#include "xfs_btree.h" +#include "xfs_refcount_btree.h" +#include "xfs_ialloc_btree.h" +#include "xfs_ag.h" +#include "xfs_ag_resv.h" + +/* + * Per-AG Block Reservations + * + * For some kinds of allocation group metadata structures, it is advantageous + * to reserve a small number of blocks in each AG so that future expansions of + * that data structure do not encounter ENOSPC because errors during a btree + * split cause the filesystem to go offline. + * + * Prior to the introduction of reflink, this wasn't an issue because the free + * space btrees maintain a reserve of space (the AGFL) to handle any expansion + * that may be necessary; and allocations of other metadata (inodes, BMBT, + * dir/attr) aren't restricted to a single AG. However, with reflink it is + * possible to allocate all the space in an AG, have subsequent reflink/CoW + * activity expand the refcount btree, and discover that there's no space left + * to handle that expansion. Since we can calculate the maximum size of the + * refcount btree, we can reserve space for it and avoid ENOSPC. + * + * Handling per-AG reservations consists of three changes to the allocator's + * behavior: First, because these reservations are always needed, we decrease + * the ag_max_usable counter to reflect the size of the AG after the reserved + * blocks are taken. Second, the reservations must be reflected in the + * fdblocks count to maintain proper accounting. Third, each AG must maintain + * its own reserved block counter so that we can calculate the amount of space + * that must remain free to maintain the reservations. Fourth, the "remaining + * reserved blocks" count must be used when calculating the length of the + * longest free extent in an AG and to clamp maxlen in the per-AG allocation + * functions. In other words, we maintain a virtual allocation via in-core + * accounting tricks so that we don't have to clean up after a crash. :) + * + * Reserved blocks can be managed by passing one of the enum xfs_ag_resv_type + * values via struct xfs_alloc_arg or directly to the xfs_free_extent + * function. It might seem a little funny to maintain a reservoir of blocks + * to feed another reservoir, but the AGFL only holds enough blocks to get + * through the next transaction. The per-AG reservation is to ensure (we + * hope) that each AG never runs out of blocks. Each data structure wanting + * to use the reservation system should update ask/used in xfs_ag_resv_init. + */ + +/* + * Are we critically low on blocks? For now we'll define that as the number + * of blocks we can get our hands on being less than 10% of what we reserved + * or less than some arbitrary number (maximum btree height). + */ +bool +xfs_ag_resv_critical( + struct xfs_perag *pag, + enum xfs_ag_resv_type type) +{ + struct xfs_mount *mp = pag_mount(pag); + xfs_extlen_t avail; + xfs_extlen_t orig; + + switch (type) { + case XFS_AG_RESV_METADATA: + avail = pag->pagf_freeblks - pag->pag_rmapbt_resv.ar_reserved; + orig = pag->pag_meta_resv.ar_asked; + break; + case XFS_AG_RESV_RMAPBT: + avail = pag->pagf_freeblks + pag->pagf_flcount - + pag->pag_meta_resv.ar_reserved; + orig = pag->pag_rmapbt_resv.ar_asked; + break; + default: + ASSERT(0); + return false; + } + + trace_xfs_ag_resv_critical(pag, type, avail); + + /* Critically low if less than 10% or max btree height remains. */ + return avail < orig / 10 || avail < mp->m_agbtree_maxlevels || + XFS_TEST_ERROR(mp, XFS_ERRTAG_AG_RESV_CRITICAL); +} + +/* + * How many blocks are reserved but not used, and therefore must not be + * allocated away? + */ +xfs_extlen_t +xfs_ag_resv_needed( + struct xfs_perag *pag, + enum xfs_ag_resv_type type) +{ + xfs_extlen_t len; + + len = pag->pag_meta_resv.ar_reserved + pag->pag_rmapbt_resv.ar_reserved; + switch (type) { + case XFS_AG_RESV_METADATA: + case XFS_AG_RESV_RMAPBT: + len -= xfs_perag_resv(pag, type)->ar_reserved; + break; + case XFS_AG_RESV_METAFILE: + case XFS_AG_RESV_NONE: + /* empty */ + break; + default: + ASSERT(0); + } + + trace_xfs_ag_resv_needed(pag, type, len); + + return len; +} + +/* Clean out a reservation */ +static void +__xfs_ag_resv_free( + struct xfs_perag *pag, + enum xfs_ag_resv_type type) +{ + struct xfs_ag_resv *resv; + xfs_extlen_t oldresv; + + trace_xfs_ag_resv_free(pag, type, 0); + + resv = xfs_perag_resv(pag, type); + if (pag_agno(pag) == 0) + pag_mount(pag)->m_ag_max_usable += resv->ar_asked; + /* + * RMAPBT blocks come from the AGFL and AGFL blocks are always + * considered "free", so whatever was reserved at mount time must be + * given back at umount. + */ + if (type == XFS_AG_RESV_RMAPBT) + oldresv = resv->ar_orig_reserved; + else + oldresv = resv->ar_reserved; + xfs_add_fdblocks(pag_mount(pag), oldresv); + resv->ar_reserved = 0; + resv->ar_asked = 0; + resv->ar_orig_reserved = 0; +} + +/* Free a per-AG reservation. */ +void +xfs_ag_resv_free( + struct xfs_perag *pag) +{ + __xfs_ag_resv_free(pag, XFS_AG_RESV_RMAPBT); + __xfs_ag_resv_free(pag, XFS_AG_RESV_METADATA); +} + +static int +__xfs_ag_resv_init( + struct xfs_perag *pag, + enum xfs_ag_resv_type type, + xfs_extlen_t ask, + xfs_extlen_t used) +{ + struct xfs_mount *mp = pag_mount(pag); + struct xfs_ag_resv *resv; + int error; + xfs_extlen_t hidden_space; + + if (used > ask) + ask = used; + + switch (type) { + case XFS_AG_RESV_RMAPBT: + /* + * Space taken by the rmapbt is not subtracted from fdblocks + * because the rmapbt lives in the free space. Here we must + * subtract the entire reservation from fdblocks so that we + * always have blocks available for rmapbt expansion. + */ + hidden_space = ask; + break; + case XFS_AG_RESV_METADATA: + /* + * Space taken by all other metadata btrees are accounted + * on-disk as used space. We therefore only hide the space + * that is reserved but not used by the trees. + */ + hidden_space = ask - used; + break; + default: + ASSERT(0); + return -EINVAL; + } + + if (XFS_TEST_ERROR(mp, XFS_ERRTAG_AG_RESV_FAIL)) + error = -ENOSPC; + else + error = xfs_dec_fdblocks(mp, hidden_space, true); + if (error) { + trace_xfs_ag_resv_init_error(pag, error, _RET_IP_); + xfs_warn(mp, +"Per-AG reservation for AG %u failed. Filesystem may run out of space.", + pag_agno(pag)); + return error; + } + + /* + * Reduce the maximum per-AG allocation length by however much we're + * trying to reserve for an AG. Since this is a filesystem-wide + * counter, we only make the adjustment for AG 0. This assumes that + * there aren't any AGs hungrier for per-AG reservation than AG 0. + */ + if (pag_agno(pag) == 0) + mp->m_ag_max_usable -= ask; + + resv = xfs_perag_resv(pag, type); + resv->ar_asked = ask; + resv->ar_orig_reserved = hidden_space; + resv->ar_reserved = ask - used; + + trace_xfs_ag_resv_init(pag, type, ask); + return 0; +} + +/* Create a per-AG block reservation. */ +int +xfs_ag_resv_init( + struct xfs_perag *pag, + struct xfs_trans *tp) +{ + struct xfs_mount *mp = pag_mount(pag); + xfs_extlen_t ask; + xfs_extlen_t used; + int error = 0, error2; + bool has_resv = false; + + /* Create the metadata reservation. */ + if (pag->pag_meta_resv.ar_asked == 0) { + ask = used = 0; + + error = xfs_refcountbt_calc_reserves(mp, tp, pag, &ask, &used); + if (error) + goto out; + + error = xfs_finobt_calc_reserves(pag, tp, &ask, &used); + if (error) + goto out; + + error = __xfs_ag_resv_init(pag, XFS_AG_RESV_METADATA, + ask, used); + if (error) { + /* + * Because we didn't have per-AG reservations when the + * finobt feature was added we might not be able to + * reserve all needed blocks. Warn and fall back to the + * old and potentially buggy code in that case, but + * ensure we do have the reservation for the refcountbt. + */ + ask = used = 0; + + mp->m_finobt_nores = true; + + error = xfs_refcountbt_calc_reserves(mp, tp, pag, &ask, + &used); + if (error) + goto out; + + error = __xfs_ag_resv_init(pag, XFS_AG_RESV_METADATA, + ask, used); + if (error) + goto out; + } + if (ask) + has_resv = true; + } + + /* Create the RMAPBT metadata reservation */ + if (pag->pag_rmapbt_resv.ar_asked == 0) { + ask = used = 0; + + error = xfs_rmapbt_calc_reserves(mp, tp, pag, &ask, &used); + if (error) + goto out; + + error = __xfs_ag_resv_init(pag, XFS_AG_RESV_RMAPBT, ask, used); + if (error) + goto out; + if (ask) + has_resv = true; + } + +out: + /* + * Initialize the pagf if we have at least one active reservation on the + * AG. This may have occurred already via reservation calculation, but + * fall back to an explicit init to ensure the in-core allocbt usage + * counters are initialized as soon as possible. This is important + * because filesystems with large perag reservations are susceptible to + * free space reservation problems that the allocbt counter is used to + * address. + */ + if (has_resv) { + error2 = xfs_alloc_read_agf(pag, tp, 0, NULL); + if (error2) + return error2; + + /* + * If there isn't enough space in the AG to satisfy the + * reservation, let the caller know that there wasn't enough + * space. Callers are responsible for deciding what to do + * next, since (in theory) we can stumble along with + * insufficient reservation if data blocks are being freed to + * replenish the AG's free space. + */ + if (!error && + xfs_perag_resv(pag, XFS_AG_RESV_METADATA)->ar_reserved + + xfs_perag_resv(pag, XFS_AG_RESV_RMAPBT)->ar_reserved > + pag->pagf_freeblks + pag->pagf_flcount) + error = -ENOSPC; + } + + return error; +} + +/* Allocate a block from the reservation. */ +void +xfs_ag_resv_alloc_extent( + struct xfs_perag *pag, + enum xfs_ag_resv_type type, + struct xfs_alloc_arg *args) +{ + struct xfs_ag_resv *resv; + xfs_extlen_t len; + uint field; + + trace_xfs_ag_resv_alloc_extent(pag, type, args->len); + + switch (type) { + case XFS_AG_RESV_AGFL: + case XFS_AG_RESV_METAFILE: + return; + case XFS_AG_RESV_METADATA: + case XFS_AG_RESV_RMAPBT: + resv = xfs_perag_resv(pag, type); + break; + default: + ASSERT(0); + fallthrough; + case XFS_AG_RESV_NONE: + field = args->wasdel ? XFS_TRANS_SB_RES_FDBLOCKS : + XFS_TRANS_SB_FDBLOCKS; + xfs_trans_mod_sb(args->tp, field, -(int64_t)args->len); + return; + } + + len = min_t(xfs_extlen_t, args->len, resv->ar_reserved); + resv->ar_reserved -= len; + if (type == XFS_AG_RESV_RMAPBT) + return; + /* Allocations of reserved blocks only need on-disk sb updates... */ + xfs_trans_mod_sb(args->tp, XFS_TRANS_SB_RES_FDBLOCKS, -(int64_t)len); + /* ...but non-reserved blocks need in-core and on-disk updates. */ + if (args->len > len) + xfs_trans_mod_sb(args->tp, XFS_TRANS_SB_FDBLOCKS, + -((int64_t)args->len - len)); +} + +/* Free a block to the reservation. */ +void +xfs_ag_resv_free_extent( + struct xfs_perag *pag, + enum xfs_ag_resv_type type, + struct xfs_trans *tp, + xfs_extlen_t len) +{ + xfs_extlen_t leftover; + struct xfs_ag_resv *resv; + + trace_xfs_ag_resv_free_extent(pag, type, len); + + switch (type) { + case XFS_AG_RESV_AGFL: + case XFS_AG_RESV_METAFILE: + return; + case XFS_AG_RESV_METADATA: + case XFS_AG_RESV_RMAPBT: + resv = xfs_perag_resv(pag, type); + break; + default: + ASSERT(0); + fallthrough; + case XFS_AG_RESV_NONE: + xfs_trans_mod_sb(tp, XFS_TRANS_SB_FDBLOCKS, (int64_t)len); + fallthrough; + case XFS_AG_RESV_IGNORE: + return; + } + + leftover = min_t(xfs_extlen_t, len, resv->ar_asked - resv->ar_reserved); + resv->ar_reserved += leftover; + if (type == XFS_AG_RESV_RMAPBT) + return; + /* Freeing into the reserved pool only requires on-disk update... */ + xfs_trans_mod_sb(tp, XFS_TRANS_SB_RES_FDBLOCKS, len); + /* ...but freeing beyond that requires in-core and on-disk update. */ + if (len > leftover) + xfs_trans_mod_sb(tp, XFS_TRANS_SB_FDBLOCKS, len - leftover); +} diff --git a/fs/xfs/libxfs/xfs_ag_resv.h b/fs/xfs/libxfs/xfs_ag_resv.h new file mode 100644 index 000000000..f247eeff7 --- /dev/null +++ b/fs/xfs/libxfs/xfs_ag_resv.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright (C) 2016 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_AG_RESV_H__ +#define __XFS_AG_RESV_H__ + +void xfs_ag_resv_free(struct xfs_perag *pag); +int xfs_ag_resv_init(struct xfs_perag *pag, struct xfs_trans *tp); + +bool xfs_ag_resv_critical(struct xfs_perag *pag, enum xfs_ag_resv_type type); +xfs_extlen_t xfs_ag_resv_needed(struct xfs_perag *pag, + enum xfs_ag_resv_type type); + +void xfs_ag_resv_alloc_extent(struct xfs_perag *pag, enum xfs_ag_resv_type type, + struct xfs_alloc_arg *args); +void xfs_ag_resv_free_extent(struct xfs_perag *pag, enum xfs_ag_resv_type type, + struct xfs_trans *tp, xfs_extlen_t len); + +static inline struct xfs_ag_resv * +xfs_perag_resv( + struct xfs_perag *pag, + enum xfs_ag_resv_type type) +{ + switch (type) { + case XFS_AG_RESV_METADATA: + return &pag->pag_meta_resv; + case XFS_AG_RESV_RMAPBT: + return &pag->pag_rmapbt_resv; + default: + return NULL; + } +} + +#endif /* __XFS_AG_RESV_H__ */ diff --git a/fs/xfs/libxfs/xfs_alloc.c b/fs/xfs/libxfs/xfs_alloc.c new file mode 100644 index 000000000..d99602bcc --- /dev/null +++ b/fs/xfs/libxfs/xfs_alloc.c @@ -0,0 +1,4192 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_shared.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_btree.h" +#include "xfs_rmap.h" +#include "xfs_alloc_btree.h" +#include "xfs_alloc.h" +#include "xfs_extent_busy.h" +#include "xfs_errortag.h" +#include "xfs_error.h" +#include "xfs_trace.h" +#include "xfs_trans.h" +#include "xfs_buf_item.h" +#include "xfs_log.h" +#include "xfs_ag.h" +#include "xfs_ag_resv.h" +#include "xfs_bmap.h" +#include "xfs_health.h" +#include "xfs_extfree_item.h" + +struct kmem_cache *xfs_extfree_item_cache; + +struct workqueue_struct *xfs_alloc_wq; + +#define XFSA_FIXUP_BNO_OK 1 +#define XFSA_FIXUP_CNT_OK 2 + +/* + * Size of the AGFL. For CRC-enabled filesystes we steal a couple of slots in + * the beginning of the block for a proper header with the location information + * and CRC. + */ +unsigned int +xfs_agfl_size( + struct xfs_mount *mp) +{ + unsigned int size = mp->m_sb.sb_sectsize; + + if (xfs_has_crc(mp)) + size -= sizeof(struct xfs_agfl); + + return size / sizeof(xfs_agblock_t); +} + +unsigned int +xfs_refc_block( + struct xfs_mount *mp) +{ + if (xfs_has_rmapbt(mp)) + return XFS_RMAP_BLOCK(mp) + 1; + if (xfs_has_finobt(mp)) + return XFS_FIBT_BLOCK(mp) + 1; + return XFS_IBT_BLOCK(mp) + 1; +} + +xfs_extlen_t +xfs_prealloc_blocks( + struct xfs_mount *mp) +{ + if (xfs_has_reflink(mp)) + return xfs_refc_block(mp) + 1; + if (xfs_has_rmapbt(mp)) + return XFS_RMAP_BLOCK(mp) + 1; + if (xfs_has_finobt(mp)) + return XFS_FIBT_BLOCK(mp) + 1; + return XFS_IBT_BLOCK(mp) + 1; +} + +/* + * The number of blocks per AG that we withhold from xfs_dec_fdblocks to + * guarantee that we can refill the AGFL prior to allocating space in a nearly + * full AG. Although the space described by the free space btrees, the + * blocks used by the freesp btrees themselves, and the blocks owned by the + * AGFL are counted in the ondisk fdblocks, it's a mistake to let the ondisk + * free space in the AG drop so low that the free space btrees cannot refill an + * empty AGFL up to the minimum level. Rather than grind through empty AGs + * until the fs goes down, we subtract this many AG blocks from the incore + * fdblocks to ensure user allocation does not overcommit the space the + * filesystem needs for the AGFLs. The rmap btree uses a per-AG reservation to + * withhold space from xfs_dec_fdblocks, so we do not account for that here. + */ +#define XFS_ALLOCBT_AGFL_RESERVE 4 + +/* + * Compute the number of blocks that we set aside to guarantee the ability to + * refill the AGFL and handle a full bmap btree split. + * + * In order to avoid ENOSPC-related deadlock caused by out-of-order locking of + * AGF buffer (PV 947395), we place constraints on the relationship among + * actual allocations for data blocks, freelist blocks, and potential file data + * bmap btree blocks. However, these restrictions may result in no actual space + * allocated for a delayed extent, for example, a data block in a certain AG is + * allocated but there is no additional block for the additional bmap btree + * block due to a split of the bmap btree of the file. The result of this may + * lead to an infinite loop when the file gets flushed to disk and all delayed + * extents need to be actually allocated. To get around this, we explicitly set + * aside a few blocks which will not be reserved in delayed allocation. + * + * For each AG, we need to reserve enough blocks to replenish a totally empty + * AGFL and 4 more to handle a potential split of the file's bmap btree. + */ +unsigned int +xfs_alloc_set_aside( + struct xfs_mount *mp) +{ + return mp->m_sb.sb_agcount * (XFS_ALLOCBT_AGFL_RESERVE + 4); +} + +/* + * When deciding how much space to allocate out of an AG, we limit the + * allocation maximum size to the size the AG. However, we cannot use all the + * blocks in the AG - some are permanently used by metadata. These + * blocks are generally: + * - the AG superblock, AGF, AGI and AGFL + * - the AGF (bno and cnt) and AGI btree root blocks, and optionally + * the AGI free inode and rmap btree root blocks. + * - blocks on the AGFL according to xfs_alloc_set_aside() limits + * - the rmapbt root block + * + * The AG headers are sector sized, so the amount of space they take up is + * dependent on filesystem geometry. The others are all single blocks. + */ +unsigned int +xfs_alloc_ag_max_usable( + struct xfs_mount *mp) +{ + unsigned int blocks; + + blocks = XFS_BB_TO_FSB(mp, XFS_FSS_TO_BB(mp, 4)); /* ag headers */ + blocks += XFS_ALLOCBT_AGFL_RESERVE; + blocks += 3; /* AGF, AGI btree root blocks */ + if (xfs_has_finobt(mp)) + blocks++; /* finobt root block */ + if (xfs_has_rmapbt(mp)) + blocks++; /* rmap root block */ + if (xfs_has_reflink(mp)) + blocks++; /* refcount root block */ + + return mp->m_sb.sb_agblocks - blocks; +} + + +static int +xfs_alloc_lookup( + struct xfs_btree_cur *cur, + xfs_lookup_t dir, + xfs_agblock_t bno, + xfs_extlen_t len, + int *stat) +{ + int error; + + cur->bc_rec.a.ar_startblock = bno; + cur->bc_rec.a.ar_blockcount = len; + error = xfs_btree_lookup(cur, dir, stat); + if (*stat == 1) + cur->bc_flags |= XFS_BTREE_ALLOCBT_ACTIVE; + else + cur->bc_flags &= ~XFS_BTREE_ALLOCBT_ACTIVE; + return error; +} + +/* + * Lookup the record equal to [bno, len] in the btree given by cur. + */ +static inline int /* error */ +xfs_alloc_lookup_eq( + struct xfs_btree_cur *cur, /* btree cursor */ + xfs_agblock_t bno, /* starting block of extent */ + xfs_extlen_t len, /* length of extent */ + int *stat) /* success/failure */ +{ + return xfs_alloc_lookup(cur, XFS_LOOKUP_EQ, bno, len, stat); +} + +/* + * Lookup the first record greater than or equal to [bno, len] + * in the btree given by cur. + */ +int /* error */ +xfs_alloc_lookup_ge( + struct xfs_btree_cur *cur, /* btree cursor */ + xfs_agblock_t bno, /* starting block of extent */ + xfs_extlen_t len, /* length of extent */ + int *stat) /* success/failure */ +{ + return xfs_alloc_lookup(cur, XFS_LOOKUP_GE, bno, len, stat); +} + +/* + * Lookup the first record less than or equal to [bno, len] + * in the btree given by cur. + */ +int /* error */ +xfs_alloc_lookup_le( + struct xfs_btree_cur *cur, /* btree cursor */ + xfs_agblock_t bno, /* starting block of extent */ + xfs_extlen_t len, /* length of extent */ + int *stat) /* success/failure */ +{ + return xfs_alloc_lookup(cur, XFS_LOOKUP_LE, bno, len, stat); +} + +static inline bool +xfs_alloc_cur_active( + struct xfs_btree_cur *cur) +{ + return cur && (cur->bc_flags & XFS_BTREE_ALLOCBT_ACTIVE); +} + +/* + * Update the record referred to by cur to the value given + * by [bno, len]. + * This either works (return 0) or gets an EFSCORRUPTED error. + */ +STATIC int /* error */ +xfs_alloc_update( + struct xfs_btree_cur *cur, /* btree cursor */ + xfs_agblock_t bno, /* starting block of extent */ + xfs_extlen_t len) /* length of extent */ +{ + union xfs_btree_rec rec; + + rec.alloc.ar_startblock = cpu_to_be32(bno); + rec.alloc.ar_blockcount = cpu_to_be32(len); + return xfs_btree_update(cur, &rec); +} + +/* Convert the ondisk btree record to its incore representation. */ +void +xfs_alloc_btrec_to_irec( + const union xfs_btree_rec *rec, + struct xfs_alloc_rec_incore *irec) +{ + irec->ar_startblock = be32_to_cpu(rec->alloc.ar_startblock); + irec->ar_blockcount = be32_to_cpu(rec->alloc.ar_blockcount); +} + +/* Simple checks for free space records. */ +xfs_failaddr_t +xfs_alloc_check_irec( + struct xfs_perag *pag, + const struct xfs_alloc_rec_incore *irec) +{ + if (irec->ar_blockcount == 0) + return __this_address; + + /* check for valid extent range, including overflow */ + if (!xfs_verify_agbext(pag, irec->ar_startblock, irec->ar_blockcount)) + return __this_address; + + return NULL; +} + +static inline int +xfs_alloc_complain_bad_rec( + struct xfs_btree_cur *cur, + xfs_failaddr_t fa, + const struct xfs_alloc_rec_incore *irec) +{ + struct xfs_mount *mp = cur->bc_mp; + + xfs_warn(mp, + "%sbt record corruption in AG %d detected at %pS!", + cur->bc_ops->name, cur->bc_group->xg_gno, fa); + xfs_warn(mp, + "start block 0x%x block count 0x%x", irec->ar_startblock, + irec->ar_blockcount); + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; +} + +/* + * Get the data from the pointed-to record. + */ +int /* error */ +xfs_alloc_get_rec( + struct xfs_btree_cur *cur, /* btree cursor */ + xfs_agblock_t *bno, /* output: starting block of extent */ + xfs_extlen_t *len, /* output: length of extent */ + int *stat) /* output: success/failure */ +{ + struct xfs_alloc_rec_incore irec; + union xfs_btree_rec *rec; + xfs_failaddr_t fa; + int error; + + error = xfs_btree_get_rec(cur, &rec, stat); + if (error || !(*stat)) + return error; + + xfs_alloc_btrec_to_irec(rec, &irec); + fa = xfs_alloc_check_irec(to_perag(cur->bc_group), &irec); + if (fa) + return xfs_alloc_complain_bad_rec(cur, fa, &irec); + + *bno = irec.ar_startblock; + *len = irec.ar_blockcount; + return 0; +} + +/* + * Compute aligned version of the found extent. + * Takes alignment and min length into account. + */ +STATIC bool +xfs_alloc_compute_aligned( + xfs_alloc_arg_t *args, /* allocation argument structure */ + xfs_agblock_t foundbno, /* starting block in found extent */ + xfs_extlen_t foundlen, /* length in found extent */ + xfs_agblock_t *resbno, /* result block number */ + xfs_extlen_t *reslen, /* result length */ + unsigned *busy_gen) +{ + xfs_agblock_t bno = foundbno; + xfs_extlen_t len = foundlen; + xfs_extlen_t diff; + bool busy; + + /* Trim busy sections out of found extent */ + busy = xfs_extent_busy_trim(pag_group(args->pag), args->minlen, + args->maxlen, &bno, &len, busy_gen); + + /* + * If we have a largish extent that happens to start before min_agbno, + * see if we can shift it into range... + */ + if (bno < args->min_agbno && bno + len > args->min_agbno) { + diff = args->min_agbno - bno; + if (len > diff) { + bno += diff; + len -= diff; + } + } + + if (args->alignment > 1 && len >= args->minlen) { + xfs_agblock_t aligned_bno = roundup(bno, args->alignment); + + diff = aligned_bno - bno; + + *resbno = aligned_bno; + *reslen = diff >= len ? 0 : len - diff; + } else { + *resbno = bno; + *reslen = len; + } + + return busy; +} + +/* + * Compute best start block and diff for "near" allocations. + * freelen >= wantlen already checked by caller. + */ +STATIC xfs_extlen_t /* difference value (absolute) */ +xfs_alloc_compute_diff( + xfs_agblock_t wantbno, /* target starting block */ + xfs_extlen_t wantlen, /* target length */ + xfs_extlen_t alignment, /* target alignment */ + int datatype, /* are we allocating data? */ + xfs_agblock_t freebno, /* freespace's starting block */ + xfs_extlen_t freelen, /* freespace's length */ + xfs_agblock_t *newbnop) /* result: best start block from free */ +{ + xfs_agblock_t freeend; /* end of freespace extent */ + xfs_agblock_t newbno1; /* return block number */ + xfs_agblock_t newbno2; /* other new block number */ + xfs_extlen_t newlen1 = 0; /* length with newbno1 */ + xfs_extlen_t newlen2 = 0; /* length with newbno2 */ + xfs_agblock_t wantend; /* end of target extent */ + bool userdata = datatype & XFS_ALLOC_USERDATA; + + ASSERT(freelen >= wantlen); + freeend = freebno + freelen; + wantend = wantbno + wantlen; + /* + * We want to allocate from the start of a free extent if it is past + * the desired block or if we are allocating user data and the free + * extent is before desired block. The second case is there to allow + * for contiguous allocation from the remaining free space if the file + * grows in the short term. + */ + if (freebno >= wantbno || (userdata && freeend < wantend)) { + if ((newbno1 = roundup(freebno, alignment)) >= freeend) + newbno1 = NULLAGBLOCK; + } else if (freeend >= wantend && alignment > 1) { + newbno1 = roundup(wantbno, alignment); + newbno2 = newbno1 - alignment; + if (newbno1 >= freeend) + newbno1 = NULLAGBLOCK; + else + newlen1 = XFS_EXTLEN_MIN(wantlen, freeend - newbno1); + if (newbno2 < freebno) + newbno2 = NULLAGBLOCK; + else + newlen2 = XFS_EXTLEN_MIN(wantlen, freeend - newbno2); + if (newbno1 != NULLAGBLOCK && newbno2 != NULLAGBLOCK) { + if (newlen1 < newlen2 || + (newlen1 == newlen2 && + abs_diff(newbno1, wantbno) > + abs_diff(newbno2, wantbno))) + newbno1 = newbno2; + } else if (newbno2 != NULLAGBLOCK) + newbno1 = newbno2; + } else if (freeend >= wantend) { + newbno1 = wantbno; + } else if (alignment > 1) { + newbno1 = roundup(freeend - wantlen, alignment); + if (newbno1 > freeend - wantlen && + newbno1 - alignment >= freebno) + newbno1 -= alignment; + else if (newbno1 >= freeend) + newbno1 = NULLAGBLOCK; + } else + newbno1 = freeend - wantlen; + *newbnop = newbno1; + return newbno1 == NULLAGBLOCK ? 0 : abs_diff(newbno1, wantbno); +} + +/* + * Fix up the length, based on mod and prod. + * len should be k * prod + mod for some k. + * If len is too small it is returned unchanged. + * If len hits maxlen it is left alone. + */ +STATIC void +xfs_alloc_fix_len( + xfs_alloc_arg_t *args) /* allocation argument structure */ +{ + xfs_extlen_t k; + xfs_extlen_t rlen; + + ASSERT(args->mod < args->prod); + rlen = args->len; + ASSERT(rlen >= args->minlen); + ASSERT(rlen <= args->maxlen); + if (args->prod <= 1 || rlen < args->mod || rlen == args->maxlen || + (args->mod == 0 && rlen < args->prod)) + return; + k = rlen % args->prod; + if (k == args->mod) + return; + if (k > args->mod) + rlen = rlen - (k - args->mod); + else + rlen = rlen - args->prod + (args->mod - k); + /* casts to (int) catch length underflows */ + if ((int)rlen < (int)args->minlen) + return; + ASSERT(rlen >= args->minlen && rlen <= args->maxlen); + ASSERT(rlen % args->prod == args->mod); + ASSERT(args->pag->pagf_freeblks + args->pag->pagf_flcount >= + rlen + args->minleft); + args->len = rlen; +} + +/* + * Determine if the cursor points to the block that contains the right-most + * block of records in the by-count btree. This block contains the largest + * contiguous free extent in the AG, so if we modify a record in this block we + * need to call xfs_alloc_fixup_longest() once the modifications are done to + * ensure the agf->agf_longest field is kept up to date with the longest free + * extent tracked by the by-count btree. + */ +static bool +xfs_alloc_cursor_at_lastrec( + struct xfs_btree_cur *cnt_cur) +{ + struct xfs_btree_block *block; + union xfs_btree_ptr ptr; + struct xfs_buf *bp; + + block = xfs_btree_get_block(cnt_cur, 0, &bp); + + xfs_btree_get_sibling(cnt_cur, block, &ptr, XFS_BB_RIGHTSIB); + return xfs_btree_ptr_is_null(cnt_cur, &ptr); +} + +/* + * Find the rightmost record of the cntbt, and return the longest free space + * recorded in it. Simply set both the block number and the length to their + * maximum values before searching. + */ +static int +xfs_cntbt_longest( + struct xfs_btree_cur *cnt_cur, + xfs_extlen_t *longest) +{ + struct xfs_alloc_rec_incore irec; + union xfs_btree_rec *rec; + int stat = 0; + int error; + + memset(&cnt_cur->bc_rec, 0xFF, sizeof(cnt_cur->bc_rec)); + error = xfs_btree_lookup(cnt_cur, XFS_LOOKUP_LE, &stat); + if (error) + return error; + if (!stat) { + /* totally empty tree */ + *longest = 0; + return 0; + } + + error = xfs_btree_get_rec(cnt_cur, &rec, &stat); + if (error) + return error; + if (XFS_IS_CORRUPT(cnt_cur->bc_mp, !stat)) { + xfs_btree_mark_sick(cnt_cur); + return -EFSCORRUPTED; + } + + xfs_alloc_btrec_to_irec(rec, &irec); + *longest = irec.ar_blockcount; + return 0; +} + +/* + * Update the longest contiguous free extent in the AG from the by-count cursor + * that is passed to us. This should be done at the end of any allocation or + * freeing operation that touches the longest extent in the btree. + * + * Needing to update the longest extent can be determined by calling + * xfs_alloc_cursor_at_lastrec() after the cursor is positioned for record + * modification but before the modification begins. + */ +static int +xfs_alloc_fixup_longest( + struct xfs_btree_cur *cnt_cur) +{ + struct xfs_perag *pag = to_perag(cnt_cur->bc_group); + struct xfs_buf *bp = cnt_cur->bc_ag.agbp; + struct xfs_agf *agf = bp->b_addr; + xfs_extlen_t longest = 0; + int error; + + /* Lookup last rec in order to update AGF. */ + error = xfs_cntbt_longest(cnt_cur, &longest); + if (error) + return error; + + pag->pagf_longest = longest; + agf->agf_longest = cpu_to_be32(pag->pagf_longest); + xfs_alloc_log_agf(cnt_cur->bc_tp, bp, XFS_AGF_LONGEST); + + return 0; +} + +/* + * Update the two btrees, logically removing from freespace the extent + * starting at rbno, rlen blocks. The extent is contained within the + * actual (current) free extent fbno for flen blocks. + * Flags are passed in indicating whether the cursors are set to the + * relevant records. + */ +STATIC int /* error code */ +xfs_alloc_fixup_trees( + struct xfs_btree_cur *cnt_cur, /* cursor for by-size btree */ + struct xfs_btree_cur *bno_cur, /* cursor for by-block btree */ + xfs_agblock_t fbno, /* starting block of free extent */ + xfs_extlen_t flen, /* length of free extent */ + xfs_agblock_t rbno, /* starting block of returned extent */ + xfs_extlen_t rlen, /* length of returned extent */ + int flags) /* flags, XFSA_FIXUP_... */ +{ + int error; /* error code */ + int i; /* operation results */ + xfs_agblock_t nfbno1; /* first new free startblock */ + xfs_agblock_t nfbno2; /* second new free startblock */ + xfs_extlen_t nflen1 = 0; /* first new free length */ + xfs_extlen_t nflen2 = 0; /* second new free length */ + struct xfs_mount *mp; + bool fixup_longest = false; + + mp = cnt_cur->bc_mp; + + /* + * Look up the record in the by-size tree if necessary. + */ + if (flags & XFSA_FIXUP_CNT_OK) { +#ifdef DEBUG + if ((error = xfs_alloc_get_rec(cnt_cur, &nfbno1, &nflen1, &i))) + return error; + if (XFS_IS_CORRUPT(mp, + i != 1 || + nfbno1 != fbno || + nflen1 != flen)) { + xfs_btree_mark_sick(cnt_cur); + return -EFSCORRUPTED; + } +#endif + } else { + if ((error = xfs_alloc_lookup_eq(cnt_cur, fbno, flen, &i))) + return error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cnt_cur); + return -EFSCORRUPTED; + } + } + /* + * Look up the record in the by-block tree if necessary. + */ + if (flags & XFSA_FIXUP_BNO_OK) { +#ifdef DEBUG + if ((error = xfs_alloc_get_rec(bno_cur, &nfbno1, &nflen1, &i))) + return error; + if (XFS_IS_CORRUPT(mp, + i != 1 || + nfbno1 != fbno || + nflen1 != flen)) { + xfs_btree_mark_sick(bno_cur); + return -EFSCORRUPTED; + } +#endif + } else { + if ((error = xfs_alloc_lookup_eq(bno_cur, fbno, flen, &i))) + return error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(bno_cur); + return -EFSCORRUPTED; + } + } + +#ifdef DEBUG + if (bno_cur->bc_nlevels == 1 && cnt_cur->bc_nlevels == 1) { + struct xfs_btree_block *bnoblock; + struct xfs_btree_block *cntblock; + + bnoblock = XFS_BUF_TO_BLOCK(bno_cur->bc_levels[0].bp); + cntblock = XFS_BUF_TO_BLOCK(cnt_cur->bc_levels[0].bp); + + if (XFS_IS_CORRUPT(mp, + bnoblock->bb_numrecs != + cntblock->bb_numrecs)) { + xfs_btree_mark_sick(bno_cur); + return -EFSCORRUPTED; + } + } +#endif + + /* + * Deal with all four cases: the allocated record is contained + * within the freespace record, so we can have new freespace + * at either (or both) end, or no freespace remaining. + */ + if (rbno == fbno && rlen == flen) + nfbno1 = nfbno2 = NULLAGBLOCK; + else if (rbno == fbno) { + nfbno1 = rbno + rlen; + nflen1 = flen - rlen; + nfbno2 = NULLAGBLOCK; + } else if (rbno + rlen == fbno + flen) { + nfbno1 = fbno; + nflen1 = flen - rlen; + nfbno2 = NULLAGBLOCK; + } else { + nfbno1 = fbno; + nflen1 = rbno - fbno; + nfbno2 = rbno + rlen; + nflen2 = (fbno + flen) - nfbno2; + } + + if (xfs_alloc_cursor_at_lastrec(cnt_cur)) + fixup_longest = true; + + /* + * Delete the entry from the by-size btree. + */ + if ((error = xfs_btree_delete(cnt_cur, &i))) + return error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cnt_cur); + return -EFSCORRUPTED; + } + /* + * Add new by-size btree entry(s). + */ + if (nfbno1 != NULLAGBLOCK) { + if ((error = xfs_alloc_lookup_eq(cnt_cur, nfbno1, nflen1, &i))) + return error; + if (XFS_IS_CORRUPT(mp, i != 0)) { + xfs_btree_mark_sick(cnt_cur); + return -EFSCORRUPTED; + } + if ((error = xfs_btree_insert(cnt_cur, &i))) + return error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cnt_cur); + return -EFSCORRUPTED; + } + } + if (nfbno2 != NULLAGBLOCK) { + if ((error = xfs_alloc_lookup_eq(cnt_cur, nfbno2, nflen2, &i))) + return error; + if (XFS_IS_CORRUPT(mp, i != 0)) { + xfs_btree_mark_sick(cnt_cur); + return -EFSCORRUPTED; + } + if ((error = xfs_btree_insert(cnt_cur, &i))) + return error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cnt_cur); + return -EFSCORRUPTED; + } + } + /* + * Fix up the by-block btree entry(s). + */ + if (nfbno1 == NULLAGBLOCK) { + /* + * No remaining freespace, just delete the by-block tree entry. + */ + if ((error = xfs_btree_delete(bno_cur, &i))) + return error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(bno_cur); + return -EFSCORRUPTED; + } + } else { + /* + * Update the by-block entry to start later|be shorter. + */ + if ((error = xfs_alloc_update(bno_cur, nfbno1, nflen1))) + return error; + } + if (nfbno2 != NULLAGBLOCK) { + /* + * 2 resulting free entries, need to add one. + */ + if ((error = xfs_alloc_lookup_eq(bno_cur, nfbno2, nflen2, &i))) + return error; + if (XFS_IS_CORRUPT(mp, i != 0)) { + xfs_btree_mark_sick(bno_cur); + return -EFSCORRUPTED; + } + if ((error = xfs_btree_insert(bno_cur, &i))) + return error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(bno_cur); + return -EFSCORRUPTED; + } + } + + if (fixup_longest) + return xfs_alloc_fixup_longest(cnt_cur); + + return 0; +} + +/* + * We do not verify the AGFL contents against AGF-based index counters here, + * even though we may have access to the perag that contains shadow copies. We + * don't know if the AGF based counters have been checked, and if they have they + * still may be inconsistent because they haven't yet been reset on the first + * allocation after the AGF has been read in. + * + * This means we can only check that all agfl entries contain valid or null + * values because we can't reliably determine the active range to exclude + * NULLAGBNO as a valid value. + * + * However, we can't even do that for v4 format filesystems because there are + * old versions of mkfs out there that does not initialise the AGFL to known, + * verifiable values. HEnce we can't tell the difference between a AGFL block + * allocated by mkfs and a corrupted AGFL block here on v4 filesystems. + * + * As a result, we can only fully validate AGFL block numbers when we pull them + * from the freelist in xfs_alloc_get_freelist(). + */ +static xfs_failaddr_t +xfs_agfl_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_agfl *agfl = XFS_BUF_TO_AGFL(bp); + __be32 *agfl_bno = xfs_buf_to_agfl_bno(bp); + int i; + + if (!xfs_has_crc(mp)) + return NULL; + + if (!xfs_verify_magic(bp, agfl->agfl_magicnum)) + return __this_address; + if (!uuid_equal(&agfl->agfl_uuid, &mp->m_sb.sb_meta_uuid)) + return __this_address; + /* + * during growfs operations, the perag is not fully initialised, + * so we can't use it for any useful checking. growfs ensures we can't + * use it by using uncached buffers that don't have the perag attached + * so we can detect and avoid this problem. + */ + if (bp->b_pag && be32_to_cpu(agfl->agfl_seqno) != pag_agno((bp->b_pag))) + return __this_address; + + for (i = 0; i < xfs_agfl_size(mp); i++) { + if (be32_to_cpu(agfl_bno[i]) != NULLAGBLOCK && + be32_to_cpu(agfl_bno[i]) >= mp->m_sb.sb_agblocks) + return __this_address; + } + + if (!xfs_log_check_lsn(mp, be64_to_cpu(XFS_BUF_TO_AGFL(bp)->agfl_lsn))) + return __this_address; + return NULL; +} + +static void +xfs_agfl_read_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + xfs_failaddr_t fa; + + /* + * There is no verification of non-crc AGFLs because mkfs does not + * initialise the AGFL to zero or NULL. Hence the only valid part of the + * AGFL is what the AGF says is active. We can't get to the AGF, so we + * can't verify just those entries are valid. + */ + if (!xfs_has_crc(mp)) + return; + + if (!xfs_buf_verify_cksum(bp, XFS_AGFL_CRC_OFF)) + xfs_verifier_error(bp, -EFSBADCRC, __this_address); + else { + fa = xfs_agfl_verify(bp); + if (fa) + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + } +} + +static void +xfs_agfl_write_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_buf_log_item *bip = bp->b_log_item; + xfs_failaddr_t fa; + + /* no verification of non-crc AGFLs */ + if (!xfs_has_crc(mp)) + return; + + fa = xfs_agfl_verify(bp); + if (fa) { + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + return; + } + + if (bip) + XFS_BUF_TO_AGFL(bp)->agfl_lsn = cpu_to_be64(bip->bli_item.li_lsn); + + xfs_buf_update_cksum(bp, XFS_AGFL_CRC_OFF); +} + +const struct xfs_buf_ops xfs_agfl_buf_ops = { + .name = "xfs_agfl", + .magic = { cpu_to_be32(XFS_AGFL_MAGIC), cpu_to_be32(XFS_AGFL_MAGIC) }, + .verify_read = xfs_agfl_read_verify, + .verify_write = xfs_agfl_write_verify, + .verify_struct = xfs_agfl_verify, +}; + +/* + * Read in the allocation group free block array. + */ +int +xfs_alloc_read_agfl( + struct xfs_perag *pag, + struct xfs_trans *tp, + struct xfs_buf **bpp) +{ + struct xfs_mount *mp = pag_mount(pag); + struct xfs_buf *bp; + int error; + + error = xfs_trans_read_buf(mp, tp, mp->m_ddev_targp, + XFS_AG_DADDR(mp, pag_agno(pag), XFS_AGFL_DADDR(mp)), + XFS_FSS_TO_BB(mp, 1), 0, &bp, &xfs_agfl_buf_ops); + if (xfs_metadata_is_sick(error)) + xfs_ag_mark_sick(pag, XFS_SICK_AG_AGFL); + if (error) + return error; + xfs_buf_set_ref(bp, XFS_AGFL_REF); + *bpp = bp; + return 0; +} + +STATIC int +xfs_alloc_update_counters( + struct xfs_trans *tp, + struct xfs_buf *agbp, + long len) +{ + struct xfs_agf *agf = agbp->b_addr; + + agbp->b_pag->pagf_freeblks += len; + be32_add_cpu(&agf->agf_freeblks, len); + + if (unlikely(be32_to_cpu(agf->agf_freeblks) > + be32_to_cpu(agf->agf_length))) { + xfs_buf_mark_corrupt(agbp); + xfs_ag_mark_sick(agbp->b_pag, XFS_SICK_AG_AGF); + return -EFSCORRUPTED; + } + + xfs_alloc_log_agf(tp, agbp, XFS_AGF_FREEBLKS); + return 0; +} + +/* + * Block allocation algorithm and data structures. + */ +struct xfs_alloc_cur { + struct xfs_btree_cur *cnt; /* btree cursors */ + struct xfs_btree_cur *bnolt; + struct xfs_btree_cur *bnogt; + xfs_extlen_t cur_len;/* current search length */ + xfs_agblock_t rec_bno;/* extent startblock */ + xfs_extlen_t rec_len;/* extent length */ + xfs_agblock_t bno; /* alloc bno */ + xfs_extlen_t len; /* alloc len */ + xfs_extlen_t diff; /* diff from search bno */ + unsigned int busy_gen;/* busy state */ + bool busy; +}; + +/* + * Set up cursors, etc. in the extent allocation cursor. This function can be + * called multiple times to reset an initialized structure without having to + * reallocate cursors. + */ +static int +xfs_alloc_cur_setup( + struct xfs_alloc_arg *args, + struct xfs_alloc_cur *acur) +{ + int error; + int i; + + acur->cur_len = args->maxlen; + acur->rec_bno = 0; + acur->rec_len = 0; + acur->bno = 0; + acur->len = 0; + acur->diff = -1; + acur->busy = false; + acur->busy_gen = 0; + + /* + * Perform an initial cntbt lookup to check for availability of maxlen + * extents. If this fails, we'll return -ENOSPC to signal the caller to + * attempt a small allocation. + */ + if (!acur->cnt) + acur->cnt = xfs_cntbt_init_cursor(args->mp, args->tp, + args->agbp, args->pag); + error = xfs_alloc_lookup_ge(acur->cnt, 0, args->maxlen, &i); + if (error) + return error; + + /* + * Allocate the bnobt left and right search cursors. + */ + if (!acur->bnolt) + acur->bnolt = xfs_bnobt_init_cursor(args->mp, args->tp, + args->agbp, args->pag); + if (!acur->bnogt) + acur->bnogt = xfs_bnobt_init_cursor(args->mp, args->tp, + args->agbp, args->pag); + return i == 1 ? 0 : -ENOSPC; +} + +static void +xfs_alloc_cur_close( + struct xfs_alloc_cur *acur, + bool error) +{ + int cur_error = XFS_BTREE_NOERROR; + + if (error) + cur_error = XFS_BTREE_ERROR; + + if (acur->cnt) + xfs_btree_del_cursor(acur->cnt, cur_error); + if (acur->bnolt) + xfs_btree_del_cursor(acur->bnolt, cur_error); + if (acur->bnogt) + xfs_btree_del_cursor(acur->bnogt, cur_error); + acur->cnt = acur->bnolt = acur->bnogt = NULL; +} + +/* + * Check an extent for allocation and track the best available candidate in the + * allocation structure. The cursor is deactivated if it has entered an out of + * range state based on allocation arguments. Optionally return the extent + * extent geometry and allocation status if requested by the caller. + */ +static int +xfs_alloc_cur_check( + struct xfs_alloc_arg *args, + struct xfs_alloc_cur *acur, + struct xfs_btree_cur *cur, + int *new) +{ + int error, i; + xfs_agblock_t bno, bnoa, bnew; + xfs_extlen_t len, lena, diff = -1; + bool busy; + unsigned busy_gen = 0; + bool deactivate = false; + bool isbnobt = xfs_btree_is_bno(cur->bc_ops); + + *new = 0; + + error = xfs_alloc_get_rec(cur, &bno, &len, &i); + if (error) + return error; + if (XFS_IS_CORRUPT(args->mp, i != 1)) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + + /* + * Check minlen and deactivate a cntbt cursor if out of acceptable size + * range (i.e., walking backwards looking for a minlen extent). + */ + if (len < args->minlen) { + deactivate = !isbnobt; + goto out; + } + + busy = xfs_alloc_compute_aligned(args, bno, len, &bnoa, &lena, + &busy_gen); + acur->busy |= busy; + if (busy) + acur->busy_gen = busy_gen; + /* deactivate a bnobt cursor outside of locality range */ + if (bnoa < args->min_agbno || bnoa > args->max_agbno) { + deactivate = isbnobt; + goto out; + } + if (lena < args->minlen) + goto out; + + args->len = XFS_EXTLEN_MIN(lena, args->maxlen); + xfs_alloc_fix_len(args); + ASSERT(args->len >= args->minlen); + if (args->len < acur->len) + goto out; + + /* + * We have an aligned record that satisfies minlen and beats or matches + * the candidate extent size. Compare locality for near allocation mode. + */ + diff = xfs_alloc_compute_diff(args->agbno, args->len, + args->alignment, args->datatype, + bnoa, lena, &bnew); + if (bnew == NULLAGBLOCK) + goto out; + + /* + * Deactivate a bnobt cursor with worse locality than the current best. + */ + if (diff > acur->diff) { + deactivate = isbnobt; + goto out; + } + + ASSERT(args->len > acur->len || + (args->len == acur->len && diff <= acur->diff)); + acur->rec_bno = bno; + acur->rec_len = len; + acur->bno = bnew; + acur->len = args->len; + acur->diff = diff; + *new = 1; + + /* + * We're done if we found a perfect allocation. This only deactivates + * the current cursor, but this is just an optimization to terminate a + * cntbt search that otherwise runs to the edge of the tree. + */ + if (acur->diff == 0 && acur->len == args->maxlen) + deactivate = true; +out: + if (deactivate) + cur->bc_flags &= ~XFS_BTREE_ALLOCBT_ACTIVE; + trace_xfs_alloc_cur_check(cur, bno, len, diff, *new); + return 0; +} + +/* + * Complete an allocation of a candidate extent. Remove the extent from both + * trees and update the args structure. + */ +STATIC int +xfs_alloc_cur_finish( + struct xfs_alloc_arg *args, + struct xfs_alloc_cur *acur) +{ + int error; + + ASSERT(acur->cnt && acur->bnolt); + ASSERT(acur->bno >= acur->rec_bno); + ASSERT(acur->bno + acur->len <= acur->rec_bno + acur->rec_len); + ASSERT(xfs_verify_agbext(args->pag, acur->rec_bno, acur->rec_len)); + + error = xfs_alloc_fixup_trees(acur->cnt, acur->bnolt, acur->rec_bno, + acur->rec_len, acur->bno, acur->len, 0); + if (error) + return error; + + args->agbno = acur->bno; + args->len = acur->len; + args->wasfromfl = 0; + + trace_xfs_alloc_cur(args); + return 0; +} + +/* + * Locality allocation lookup algorithm. This expects a cntbt cursor and uses + * bno optimized lookup to search for extents with ideal size and locality. + */ +STATIC int +xfs_alloc_cntbt_iter( + struct xfs_alloc_arg *args, + struct xfs_alloc_cur *acur) +{ + struct xfs_btree_cur *cur = acur->cnt; + xfs_agblock_t bno; + xfs_extlen_t len, cur_len; + int error; + int i; + + if (!xfs_alloc_cur_active(cur)) + return 0; + + /* locality optimized lookup */ + cur_len = acur->cur_len; + error = xfs_alloc_lookup_ge(cur, args->agbno, cur_len, &i); + if (error) + return error; + if (i == 0) + return 0; + error = xfs_alloc_get_rec(cur, &bno, &len, &i); + if (error) + return error; + + /* check the current record and update search length from it */ + error = xfs_alloc_cur_check(args, acur, cur, &i); + if (error) + return error; + ASSERT(len >= acur->cur_len); + acur->cur_len = len; + + /* + * We looked up the first record >= [agbno, len] above. The agbno is a + * secondary key and so the current record may lie just before or after + * agbno. If it is past agbno, check the previous record too so long as + * the length matches as it may be closer. Don't check a smaller record + * because that could deactivate our cursor. + */ + if (bno > args->agbno) { + error = xfs_btree_decrement(cur, 0, &i); + if (!error && i) { + error = xfs_alloc_get_rec(cur, &bno, &len, &i); + if (!error && i && len == acur->cur_len) + error = xfs_alloc_cur_check(args, acur, cur, + &i); + } + if (error) + return error; + } + + /* + * Increment the search key until we find at least one allocation + * candidate or if the extent we found was larger. Otherwise, double the + * search key to optimize the search. Efficiency is more important here + * than absolute best locality. + */ + cur_len <<= 1; + if (!acur->len || acur->cur_len >= cur_len) + acur->cur_len++; + else + acur->cur_len = cur_len; + + return error; +} + +/* + * Deal with the case where only small freespaces remain. Either return the + * contents of the last freespace record, or allocate space from the freelist if + * there is nothing in the tree. + */ +STATIC int /* error */ +xfs_alloc_ag_vextent_small( + struct xfs_alloc_arg *args, /* allocation argument structure */ + struct xfs_btree_cur *ccur, /* optional by-size cursor */ + xfs_agblock_t *fbnop, /* result block number */ + xfs_extlen_t *flenp, /* result length */ + int *stat) /* status: 0-freelist, 1-normal/none */ +{ + struct xfs_agf *agf = args->agbp->b_addr; + int error = 0; + xfs_agblock_t fbno = NULLAGBLOCK; + xfs_extlen_t flen = 0; + int i = 0; + + /* + * If a cntbt cursor is provided, try to allocate the largest record in + * the tree. Try the AGFL if the cntbt is empty, otherwise fail the + * allocation. Make sure to respect minleft even when pulling from the + * freelist. + */ + if (ccur) + error = xfs_btree_decrement(ccur, 0, &i); + if (error) + goto error; + if (i) { + error = xfs_alloc_get_rec(ccur, &fbno, &flen, &i); + if (error) + goto error; + if (XFS_IS_CORRUPT(args->mp, i != 1)) { + xfs_btree_mark_sick(ccur); + error = -EFSCORRUPTED; + goto error; + } + goto out; + } + + if (args->minlen != 1 || args->alignment != 1 || + args->resv == XFS_AG_RESV_AGFL || + be32_to_cpu(agf->agf_flcount) <= args->minleft) + goto out; + + error = xfs_alloc_get_freelist(args->pag, args->tp, args->agbp, + &fbno, 0); + if (error) + goto error; + if (fbno == NULLAGBLOCK) + goto out; + + xfs_extent_busy_reuse(pag_group(args->pag), fbno, 1, + (args->datatype & XFS_ALLOC_NOBUSY)); + + if (args->datatype & XFS_ALLOC_USERDATA) { + struct xfs_buf *bp; + + error = xfs_trans_get_buf(args->tp, args->mp->m_ddev_targp, + xfs_agbno_to_daddr(args->pag, fbno), + args->mp->m_bsize, 0, &bp); + if (error) + goto error; + xfs_trans_binval(args->tp, bp); + } + *fbnop = args->agbno = fbno; + *flenp = args->len = 1; + if (XFS_IS_CORRUPT(args->mp, fbno >= be32_to_cpu(agf->agf_length))) { + xfs_btree_mark_sick(ccur); + error = -EFSCORRUPTED; + goto error; + } + args->wasfromfl = 1; + trace_xfs_alloc_small_freelist(args); + + /* + * If we're feeding an AGFL block to something that doesn't live in the + * free space, we need to clear out the OWN_AG rmap. + */ + error = xfs_rmap_free(args->tp, args->agbp, args->pag, fbno, 1, + &XFS_RMAP_OINFO_AG); + if (error) + goto error; + + *stat = 0; + return 0; + +out: + /* + * Can't do the allocation, give up. + */ + if (flen < args->minlen) { + args->agbno = NULLAGBLOCK; + trace_xfs_alloc_small_notenough(args); + flen = 0; + } + *fbnop = fbno; + *flenp = flen; + *stat = 1; + trace_xfs_alloc_small_done(args); + return 0; + +error: + trace_xfs_alloc_small_error(args); + return error; +} + +/* + * Allocate a variable extent at exactly agno/bno. + * Extent's length (returned in *len) will be between minlen and maxlen, + * and of the form k * prod + mod unless there's nothing that large. + * Return the starting a.g. block (bno), or NULLAGBLOCK if we can't do it. + */ +STATIC int /* error */ +xfs_alloc_ag_vextent_exact( + xfs_alloc_arg_t *args) /* allocation argument structure */ +{ + struct xfs_btree_cur *bno_cur;/* by block-number btree cursor */ + struct xfs_btree_cur *cnt_cur;/* by count btree cursor */ + int error; + xfs_agblock_t fbno; /* start block of found extent */ + xfs_extlen_t flen; /* length of found extent */ + xfs_agblock_t tbno; /* start block of busy extent */ + xfs_extlen_t tlen; /* length of busy extent */ + xfs_agblock_t tend; /* end block of busy extent */ + int i; /* success/failure of operation */ + unsigned busy_gen; + + ASSERT(args->alignment == 1); + + /* + * Allocate/initialize a cursor for the by-number freespace btree. + */ + bno_cur = xfs_bnobt_init_cursor(args->mp, args->tp, args->agbp, + args->pag); + + /* + * Lookup bno and minlen in the btree (minlen is irrelevant, really). + * Look for the closest free block <= bno, it must contain bno + * if any free block does. + */ + error = xfs_alloc_lookup_le(bno_cur, args->agbno, args->minlen, &i); + if (error) + goto error0; + if (!i) + goto not_found; + + /* + * Grab the freespace record. + */ + error = xfs_alloc_get_rec(bno_cur, &fbno, &flen, &i); + if (error) + goto error0; + if (XFS_IS_CORRUPT(args->mp, i != 1)) { + xfs_btree_mark_sick(bno_cur); + error = -EFSCORRUPTED; + goto error0; + } + ASSERT(fbno <= args->agbno); + + /* + * Check for overlapping busy extents. + */ + tbno = fbno; + tlen = flen; + xfs_extent_busy_trim(pag_group(args->pag), args->minlen, args->maxlen, + &tbno, &tlen, &busy_gen); + + /* + * Give up if the start of the extent is busy, or the freespace isn't + * long enough for the minimum request. + */ + if (tbno > args->agbno) + goto not_found; + if (tlen < args->minlen) + goto not_found; + tend = tbno + tlen; + if (tend < args->agbno + args->minlen) + goto not_found; + + /* + * End of extent will be smaller of the freespace end and the + * maximal requested end. + * + * Fix the length according to mod and prod if given. + */ + args->len = XFS_AGBLOCK_MIN(tend, args->agbno + args->maxlen) + - args->agbno; + xfs_alloc_fix_len(args); + ASSERT(args->agbno + args->len <= tend); + + /* + * We are allocating agbno for args->len + * Allocate/initialize a cursor for the by-size btree. + */ + cnt_cur = xfs_cntbt_init_cursor(args->mp, args->tp, args->agbp, + args->pag); + ASSERT(xfs_verify_agbext(args->pag, args->agbno, args->len)); + error = xfs_alloc_fixup_trees(cnt_cur, bno_cur, fbno, flen, args->agbno, + args->len, XFSA_FIXUP_BNO_OK); + if (error) { + xfs_btree_del_cursor(cnt_cur, XFS_BTREE_ERROR); + goto error0; + } + + xfs_btree_del_cursor(bno_cur, XFS_BTREE_NOERROR); + xfs_btree_del_cursor(cnt_cur, XFS_BTREE_NOERROR); + + args->wasfromfl = 0; + trace_xfs_alloc_exact_done(args); + return 0; + +not_found: + /* Didn't find it, return null. */ + xfs_btree_del_cursor(bno_cur, XFS_BTREE_NOERROR); + args->agbno = NULLAGBLOCK; + trace_xfs_alloc_exact_notfound(args); + return 0; + +error0: + xfs_btree_del_cursor(bno_cur, XFS_BTREE_ERROR); + trace_xfs_alloc_exact_error(args); + return error; +} + +/* + * Search a given number of btree records in a given direction. Check each + * record against the good extent we've already found. + */ +STATIC int +xfs_alloc_walk_iter( + struct xfs_alloc_arg *args, + struct xfs_alloc_cur *acur, + struct xfs_btree_cur *cur, + bool increment, + bool find_one, /* quit on first candidate */ + int count, /* rec count (-1 for infinite) */ + int *stat) +{ + int error; + int i; + + *stat = 0; + + /* + * Search so long as the cursor is active or we find a better extent. + * The cursor is deactivated if it extends beyond the range of the + * current allocation candidate. + */ + while (xfs_alloc_cur_active(cur) && count) { + error = xfs_alloc_cur_check(args, acur, cur, &i); + if (error) + return error; + if (i == 1) { + *stat = 1; + if (find_one) + break; + } + if (!xfs_alloc_cur_active(cur)) + break; + + if (increment) + error = xfs_btree_increment(cur, 0, &i); + else + error = xfs_btree_decrement(cur, 0, &i); + if (error) + return error; + if (i == 0) + cur->bc_flags &= ~XFS_BTREE_ALLOCBT_ACTIVE; + + if (count > 0) + count--; + } + + return 0; +} + +/* + * Search the by-bno and by-size btrees in parallel in search of an extent with + * ideal locality based on the NEAR mode ->agbno locality hint. + */ +STATIC int +xfs_alloc_ag_vextent_locality( + struct xfs_alloc_arg *args, + struct xfs_alloc_cur *acur, + int *stat) +{ + struct xfs_btree_cur *fbcur = NULL; + int error; + int i; + bool fbinc; + + ASSERT(acur->len == 0); + + *stat = 0; + + error = xfs_alloc_lookup_ge(acur->cnt, args->agbno, acur->cur_len, &i); + if (error) + return error; + error = xfs_alloc_lookup_le(acur->bnolt, args->agbno, 0, &i); + if (error) + return error; + error = xfs_alloc_lookup_ge(acur->bnogt, args->agbno, 0, &i); + if (error) + return error; + + /* + * Search the bnobt and cntbt in parallel. Search the bnobt left and + * right and lookup the closest extent to the locality hint for each + * extent size key in the cntbt. The entire search terminates + * immediately on a bnobt hit because that means we've found best case + * locality. Otherwise the search continues until the cntbt cursor runs + * off the end of the tree. If no allocation candidate is found at this + * point, give up on locality, walk backwards from the end of the cntbt + * and take the first available extent. + * + * The parallel tree searches balance each other out to provide fairly + * consistent performance for various situations. The bnobt search can + * have pathological behavior in the worst case scenario of larger + * allocation requests and fragmented free space. On the other hand, the + * bnobt is able to satisfy most smaller allocation requests much more + * quickly than the cntbt. The cntbt search can sift through fragmented + * free space and sets of free extents for larger allocation requests + * more quickly than the bnobt. Since the locality hint is just a hint + * and we don't want to scan the entire bnobt for perfect locality, the + * cntbt search essentially bounds the bnobt search such that we can + * find good enough locality at reasonable performance in most cases. + */ + while (xfs_alloc_cur_active(acur->bnolt) || + xfs_alloc_cur_active(acur->bnogt) || + xfs_alloc_cur_active(acur->cnt)) { + + trace_xfs_alloc_cur_lookup(args); + + /* + * Search the bnobt left and right. In the case of a hit, finish + * the search in the opposite direction and we're done. + */ + error = xfs_alloc_walk_iter(args, acur, acur->bnolt, false, + true, 1, &i); + if (error) + return error; + if (i == 1) { + trace_xfs_alloc_cur_left(args); + fbcur = acur->bnogt; + fbinc = true; + break; + } + error = xfs_alloc_walk_iter(args, acur, acur->bnogt, true, true, + 1, &i); + if (error) + return error; + if (i == 1) { + trace_xfs_alloc_cur_right(args); + fbcur = acur->bnolt; + fbinc = false; + break; + } + + /* + * Check the extent with best locality based on the current + * extent size search key and keep track of the best candidate. + */ + error = xfs_alloc_cntbt_iter(args, acur); + if (error) + return error; + if (!xfs_alloc_cur_active(acur->cnt)) { + trace_xfs_alloc_cur_lookup_done(args); + break; + } + } + + /* + * If we failed to find anything due to busy extents, return empty + * handed so the caller can flush and retry. If no busy extents were + * found, walk backwards from the end of the cntbt as a last resort. + */ + if (!xfs_alloc_cur_active(acur->cnt) && !acur->len && !acur->busy) { + error = xfs_btree_decrement(acur->cnt, 0, &i); + if (error) + return error; + if (i) { + acur->cnt->bc_flags |= XFS_BTREE_ALLOCBT_ACTIVE; + fbcur = acur->cnt; + fbinc = false; + } + } + + /* + * Search in the opposite direction for a better entry in the case of + * a bnobt hit or walk backwards from the end of the cntbt. + */ + if (fbcur) { + error = xfs_alloc_walk_iter(args, acur, fbcur, fbinc, true, -1, + &i); + if (error) + return error; + } + + if (acur->len) + *stat = 1; + + return 0; +} + +/* Check the last block of the cnt btree for allocations. */ +static int +xfs_alloc_ag_vextent_lastblock( + struct xfs_alloc_arg *args, + struct xfs_alloc_cur *acur, + xfs_agblock_t *bno, + xfs_extlen_t *len, + bool *allocated) +{ + int error; + int i; + +#ifdef DEBUG + /* Randomly don't execute the first algorithm. */ + if (get_random_u32_below(2)) + return 0; +#endif + + /* + * Start from the entry that lookup found, sequence through all larger + * free blocks. If we're actually pointing at a record smaller than + * maxlen, go to the start of this block, and skip all those smaller + * than minlen. + */ + if (*len || args->alignment > 1) { + acur->cnt->bc_levels[0].ptr = 1; + do { + error = xfs_alloc_get_rec(acur->cnt, bno, len, &i); + if (error) + return error; + if (XFS_IS_CORRUPT(args->mp, i != 1)) { + xfs_btree_mark_sick(acur->cnt); + return -EFSCORRUPTED; + } + if (*len >= args->minlen) + break; + error = xfs_btree_increment(acur->cnt, 0, &i); + if (error) + return error; + } while (i); + ASSERT(*len >= args->minlen); + if (!i) + return 0; + } + + error = xfs_alloc_walk_iter(args, acur, acur->cnt, true, false, -1, &i); + if (error) + return error; + + /* + * It didn't work. We COULD be in a case where there's a good record + * somewhere, so try again. + */ + if (acur->len == 0) + return 0; + + trace_xfs_alloc_near_first(args); + *allocated = true; + return 0; +} + +/* + * Allocate a variable extent near bno in the allocation group agno. + * Extent's length (returned in len) will be between minlen and maxlen, + * and of the form k * prod + mod unless there's nothing that large. + * Return the starting a.g. block, or NULLAGBLOCK if we can't do it. + */ +STATIC int +xfs_alloc_ag_vextent_near( + struct xfs_alloc_arg *args, + uint32_t alloc_flags) +{ + struct xfs_alloc_cur acur = {}; + int error; /* error code */ + int i; /* result code, temporary */ + xfs_agblock_t bno; + xfs_extlen_t len; + + /* handle uninitialized agbno range so caller doesn't have to */ + if (!args->min_agbno && !args->max_agbno) + args->max_agbno = args->mp->m_sb.sb_agblocks - 1; + ASSERT(args->min_agbno <= args->max_agbno); + + /* clamp agbno to the range if it's outside */ + if (args->agbno < args->min_agbno) + args->agbno = args->min_agbno; + if (args->agbno > args->max_agbno) + args->agbno = args->max_agbno; + + /* Retry once quickly if we find busy extents before blocking. */ + alloc_flags |= XFS_ALLOC_FLAG_TRYFLUSH; +restart: + len = 0; + + /* + * Set up cursors and see if there are any free extents as big as + * maxlen. If not, pick the last entry in the tree unless the tree is + * empty. + */ + error = xfs_alloc_cur_setup(args, &acur); + if (error == -ENOSPC) { + error = xfs_alloc_ag_vextent_small(args, acur.cnt, &bno, + &len, &i); + if (error) + goto out; + if (i == 0 || len == 0) { + trace_xfs_alloc_near_noentry(args); + goto out; + } + ASSERT(i == 1); + } else if (error) { + goto out; + } + + /* + * First algorithm. + * If the requested extent is large wrt the freespaces available + * in this a.g., then the cursor will be pointing to a btree entry + * near the right edge of the tree. If it's in the last btree leaf + * block, then we just examine all the entries in that block + * that are big enough, and pick the best one. + */ + if (xfs_btree_islastblock(acur.cnt, 0)) { + bool allocated = false; + + error = xfs_alloc_ag_vextent_lastblock(args, &acur, &bno, &len, + &allocated); + if (error) + goto out; + if (allocated) + goto alloc_finish; + } + + /* + * Second algorithm. Combined cntbt and bnobt search to find ideal + * locality. + */ + error = xfs_alloc_ag_vextent_locality(args, &acur, &i); + if (error) + goto out; + + /* + * If we couldn't get anything, give up. + */ + if (!acur.len) { + if (acur.busy) { + /* + * Our only valid extents must have been busy. Flush and + * retry the allocation again. If we get an -EAGAIN + * error, we're being told that a deadlock was avoided + * and the current transaction needs committing before + * the allocation can be retried. + */ + trace_xfs_alloc_near_busy(args); + error = xfs_extent_busy_flush(args->tp, + pag_group(args->pag), acur.busy_gen, + alloc_flags); + if (error) + goto out; + + alloc_flags &= ~XFS_ALLOC_FLAG_TRYFLUSH; + goto restart; + } + trace_xfs_alloc_size_neither(args); + args->agbno = NULLAGBLOCK; + goto out; + } + +alloc_finish: + /* fix up btrees on a successful allocation */ + error = xfs_alloc_cur_finish(args, &acur); + +out: + xfs_alloc_cur_close(&acur, error); + return error; +} + +/* + * Allocate a variable extent anywhere in the allocation group agno. + * Extent's length (returned in len) will be between minlen and maxlen, + * and of the form k * prod + mod unless there's nothing that large. + * Return the starting a.g. block, or NULLAGBLOCK if we can't do it. + */ +static int +xfs_alloc_ag_vextent_size( + struct xfs_alloc_arg *args, + uint32_t alloc_flags) +{ + struct xfs_agf *agf = args->agbp->b_addr; + struct xfs_btree_cur *bno_cur; + struct xfs_btree_cur *cnt_cur; + xfs_agblock_t fbno; /* start of found freespace */ + xfs_extlen_t flen; /* length of found freespace */ + xfs_agblock_t rbno; /* returned block number */ + xfs_extlen_t rlen; /* length of returned extent */ + bool busy; + unsigned busy_gen; + int error; + int i; + + /* Retry once quickly if we find busy extents before blocking. */ + alloc_flags |= XFS_ALLOC_FLAG_TRYFLUSH; +restart: + /* + * Allocate and initialize a cursor for the by-size btree. + */ + cnt_cur = xfs_cntbt_init_cursor(args->mp, args->tp, args->agbp, + args->pag); + bno_cur = NULL; + + /* + * Look for an entry >= maxlen+alignment-1 blocks. + */ + if ((error = xfs_alloc_lookup_ge(cnt_cur, 0, + args->maxlen + args->alignment - 1, &i))) + goto error0; + + /* + * If none then we have to settle for a smaller extent. In the case that + * there are no large extents, this will return the last entry in the + * tree unless the tree is empty. In the case that there are only busy + * large extents, this will return the largest small extent unless there + * are no smaller extents available. + */ + if (!i) { + error = xfs_alloc_ag_vextent_small(args, cnt_cur, + &fbno, &flen, &i); + if (error) + goto error0; + if (i == 0 || flen == 0) { + xfs_btree_del_cursor(cnt_cur, XFS_BTREE_NOERROR); + trace_xfs_alloc_size_noentry(args); + return 0; + } + ASSERT(i == 1); + busy = xfs_alloc_compute_aligned(args, fbno, flen, &rbno, + &rlen, &busy_gen); + } else { + /* + * Search for a non-busy extent that is large enough. + */ + for (;;) { + error = xfs_alloc_get_rec(cnt_cur, &fbno, &flen, &i); + if (error) + goto error0; + if (XFS_IS_CORRUPT(args->mp, i != 1)) { + xfs_btree_mark_sick(cnt_cur); + error = -EFSCORRUPTED; + goto error0; + } + + busy = xfs_alloc_compute_aligned(args, fbno, flen, + &rbno, &rlen, &busy_gen); + + if (rlen >= args->maxlen) + break; + + error = xfs_btree_increment(cnt_cur, 0, &i); + if (error) + goto error0; + if (i) + continue; + + /* + * Our only valid extents must have been busy. Flush and + * retry the allocation again. If we get an -EAGAIN + * error, we're being told that a deadlock was avoided + * and the current transaction needs committing before + * the allocation can be retried. + */ + trace_xfs_alloc_size_busy(args); + error = xfs_extent_busy_flush(args->tp, + pag_group(args->pag), busy_gen, + alloc_flags); + if (error) + goto error0; + + alloc_flags &= ~XFS_ALLOC_FLAG_TRYFLUSH; + xfs_btree_del_cursor(cnt_cur, XFS_BTREE_NOERROR); + goto restart; + } + } + + /* + * In the first case above, we got the last entry in the + * by-size btree. Now we check to see if the space hits maxlen + * once aligned; if not, we search left for something better. + * This can't happen in the second case above. + */ + rlen = XFS_EXTLEN_MIN(args->maxlen, rlen); + if (XFS_IS_CORRUPT(args->mp, + rlen != 0 && + (rlen > flen || + rbno + rlen > fbno + flen))) { + xfs_btree_mark_sick(cnt_cur); + error = -EFSCORRUPTED; + goto error0; + } + if (rlen < args->maxlen) { + xfs_agblock_t bestfbno; + xfs_extlen_t bestflen; + xfs_agblock_t bestrbno; + xfs_extlen_t bestrlen; + + bestrlen = rlen; + bestrbno = rbno; + bestflen = flen; + bestfbno = fbno; + for (;;) { + if ((error = xfs_btree_decrement(cnt_cur, 0, &i))) + goto error0; + if (i == 0) + break; + if ((error = xfs_alloc_get_rec(cnt_cur, &fbno, &flen, + &i))) + goto error0; + if (XFS_IS_CORRUPT(args->mp, i != 1)) { + xfs_btree_mark_sick(cnt_cur); + error = -EFSCORRUPTED; + goto error0; + } + if (flen <= bestrlen) + break; + busy = xfs_alloc_compute_aligned(args, fbno, flen, + &rbno, &rlen, &busy_gen); + rlen = XFS_EXTLEN_MIN(args->maxlen, rlen); + if (XFS_IS_CORRUPT(args->mp, + rlen != 0 && + (rlen > flen || + rbno + rlen > fbno + flen))) { + xfs_btree_mark_sick(cnt_cur); + error = -EFSCORRUPTED; + goto error0; + } + if (rlen > bestrlen) { + bestrlen = rlen; + bestrbno = rbno; + bestflen = flen; + bestfbno = fbno; + if (rlen == args->maxlen) + break; + } + } + if ((error = xfs_alloc_lookup_eq(cnt_cur, bestfbno, bestflen, + &i))) + goto error0; + if (XFS_IS_CORRUPT(args->mp, i != 1)) { + xfs_btree_mark_sick(cnt_cur); + error = -EFSCORRUPTED; + goto error0; + } + rlen = bestrlen; + rbno = bestrbno; + flen = bestflen; + fbno = bestfbno; + } + args->wasfromfl = 0; + /* + * Fix up the length. + */ + args->len = rlen; + if (rlen < args->minlen) { + if (busy) { + /* + * Our only valid extents must have been busy. Flush and + * retry the allocation again. If we get an -EAGAIN + * error, we're being told that a deadlock was avoided + * and the current transaction needs committing before + * the allocation can be retried. + */ + trace_xfs_alloc_size_busy(args); + error = xfs_extent_busy_flush(args->tp, + pag_group(args->pag), busy_gen, + alloc_flags); + if (error) + goto error0; + + alloc_flags &= ~XFS_ALLOC_FLAG_TRYFLUSH; + xfs_btree_del_cursor(cnt_cur, XFS_BTREE_NOERROR); + goto restart; + } + goto out_nominleft; + } + xfs_alloc_fix_len(args); + + rlen = args->len; + if (XFS_IS_CORRUPT(args->mp, rlen > flen)) { + xfs_btree_mark_sick(cnt_cur); + error = -EFSCORRUPTED; + goto error0; + } + /* + * Allocate and initialize a cursor for the by-block tree. + */ + bno_cur = xfs_bnobt_init_cursor(args->mp, args->tp, args->agbp, + args->pag); + if ((error = xfs_alloc_fixup_trees(cnt_cur, bno_cur, fbno, flen, + rbno, rlen, XFSA_FIXUP_CNT_OK))) + goto error0; + xfs_btree_del_cursor(cnt_cur, XFS_BTREE_NOERROR); + xfs_btree_del_cursor(bno_cur, XFS_BTREE_NOERROR); + cnt_cur = bno_cur = NULL; + args->len = rlen; + args->agbno = rbno; + if (XFS_IS_CORRUPT(args->mp, + args->agbno + args->len > + be32_to_cpu(agf->agf_length))) { + xfs_ag_mark_sick(args->pag, XFS_SICK_AG_BNOBT); + error = -EFSCORRUPTED; + goto error0; + } + trace_xfs_alloc_size_done(args); + return 0; + +error0: + trace_xfs_alloc_size_error(args); + if (cnt_cur) + xfs_btree_del_cursor(cnt_cur, XFS_BTREE_ERROR); + if (bno_cur) + xfs_btree_del_cursor(bno_cur, XFS_BTREE_ERROR); + return error; + +out_nominleft: + xfs_btree_del_cursor(cnt_cur, XFS_BTREE_NOERROR); + trace_xfs_alloc_size_nominleft(args); + args->agbno = NULLAGBLOCK; + return 0; +} + +/* + * Free the extent starting at agno/bno for length. + */ +int +xfs_free_ag_extent( + struct xfs_trans *tp, + struct xfs_buf *agbp, + xfs_agblock_t bno, + xfs_extlen_t len, + const struct xfs_owner_info *oinfo, + enum xfs_ag_resv_type type) +{ + struct xfs_mount *mp; + struct xfs_btree_cur *bno_cur; + struct xfs_btree_cur *cnt_cur; + xfs_agblock_t gtbno; /* start of right neighbor */ + xfs_extlen_t gtlen; /* length of right neighbor */ + xfs_agblock_t ltbno; /* start of left neighbor */ + xfs_extlen_t ltlen; /* length of left neighbor */ + xfs_agblock_t nbno; /* new starting block of freesp */ + xfs_extlen_t nlen; /* new length of freespace */ + int haveleft; /* have a left neighbor */ + int haveright; /* have a right neighbor */ + int i; + int error; + struct xfs_perag *pag = agbp->b_pag; + bool fixup_longest = false; + + bno_cur = cnt_cur = NULL; + mp = tp->t_mountp; + + if (!xfs_rmap_should_skip_owner_update(oinfo)) { + error = xfs_rmap_free(tp, agbp, pag, bno, len, oinfo); + if (error) + goto error0; + } + + /* + * Allocate and initialize a cursor for the by-block btree. + */ + bno_cur = xfs_bnobt_init_cursor(mp, tp, agbp, pag); + /* + * Look for a neighboring block on the left (lower block numbers) + * that is contiguous with this space. + */ + if ((error = xfs_alloc_lookup_le(bno_cur, bno, len, &haveleft))) + goto error0; + if (haveleft) { + /* + * There is a block to our left. + */ + if ((error = xfs_alloc_get_rec(bno_cur, <bno, <len, &i))) + goto error0; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(bno_cur); + error = -EFSCORRUPTED; + goto error0; + } + /* + * It's not contiguous, though. + */ + if (ltbno + ltlen < bno) + haveleft = 0; + else { + /* + * If this failure happens the request to free this + * space was invalid, it's (partly) already free. + * Very bad. + */ + if (XFS_IS_CORRUPT(mp, ltbno + ltlen > bno)) { + xfs_btree_mark_sick(bno_cur); + error = -EFSCORRUPTED; + goto error0; + } + } + } + /* + * Look for a neighboring block on the right (higher block numbers) + * that is contiguous with this space. + */ + if ((error = xfs_btree_increment(bno_cur, 0, &haveright))) + goto error0; + if (haveright) { + /* + * There is a block to our right. + */ + if ((error = xfs_alloc_get_rec(bno_cur, >bno, >len, &i))) + goto error0; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(bno_cur); + error = -EFSCORRUPTED; + goto error0; + } + /* + * It's not contiguous, though. + */ + if (bno + len < gtbno) + haveright = 0; + else { + /* + * If this failure happens the request to free this + * space was invalid, it's (partly) already free. + * Very bad. + */ + if (XFS_IS_CORRUPT(mp, bno + len > gtbno)) { + xfs_btree_mark_sick(bno_cur); + error = -EFSCORRUPTED; + goto error0; + } + } + } + /* + * Now allocate and initialize a cursor for the by-size tree. + */ + cnt_cur = xfs_cntbt_init_cursor(mp, tp, agbp, pag); + /* + * Have both left and right contiguous neighbors. + * Merge all three into a single free block. + */ + if (haveleft && haveright) { + /* + * Delete the old by-size entry on the left. + */ + if ((error = xfs_alloc_lookup_eq(cnt_cur, ltbno, ltlen, &i))) + goto error0; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cnt_cur); + error = -EFSCORRUPTED; + goto error0; + } + if ((error = xfs_btree_delete(cnt_cur, &i))) + goto error0; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cnt_cur); + error = -EFSCORRUPTED; + goto error0; + } + /* + * Delete the old by-size entry on the right. + */ + if ((error = xfs_alloc_lookup_eq(cnt_cur, gtbno, gtlen, &i))) + goto error0; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cnt_cur); + error = -EFSCORRUPTED; + goto error0; + } + if ((error = xfs_btree_delete(cnt_cur, &i))) + goto error0; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cnt_cur); + error = -EFSCORRUPTED; + goto error0; + } + /* + * Delete the old by-block entry for the right block. + */ + if ((error = xfs_btree_delete(bno_cur, &i))) + goto error0; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(bno_cur); + error = -EFSCORRUPTED; + goto error0; + } + /* + * Move the by-block cursor back to the left neighbor. + */ + if ((error = xfs_btree_decrement(bno_cur, 0, &i))) + goto error0; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(bno_cur); + error = -EFSCORRUPTED; + goto error0; + } +#ifdef DEBUG + /* + * Check that this is the right record: delete didn't + * mangle the cursor. + */ + { + xfs_agblock_t xxbno; + xfs_extlen_t xxlen; + + if ((error = xfs_alloc_get_rec(bno_cur, &xxbno, &xxlen, + &i))) + goto error0; + if (XFS_IS_CORRUPT(mp, + i != 1 || + xxbno != ltbno || + xxlen != ltlen)) { + xfs_btree_mark_sick(bno_cur); + error = -EFSCORRUPTED; + goto error0; + } + } +#endif + /* + * Update remaining by-block entry to the new, joined block. + */ + nbno = ltbno; + nlen = len + ltlen + gtlen; + if ((error = xfs_alloc_update(bno_cur, nbno, nlen))) + goto error0; + } + /* + * Have only a left contiguous neighbor. + * Merge it together with the new freespace. + */ + else if (haveleft) { + /* + * Delete the old by-size entry on the left. + */ + if ((error = xfs_alloc_lookup_eq(cnt_cur, ltbno, ltlen, &i))) + goto error0; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cnt_cur); + error = -EFSCORRUPTED; + goto error0; + } + if ((error = xfs_btree_delete(cnt_cur, &i))) + goto error0; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cnt_cur); + error = -EFSCORRUPTED; + goto error0; + } + /* + * Back up the by-block cursor to the left neighbor, and + * update its length. + */ + if ((error = xfs_btree_decrement(bno_cur, 0, &i))) + goto error0; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(bno_cur); + error = -EFSCORRUPTED; + goto error0; + } + nbno = ltbno; + nlen = len + ltlen; + if ((error = xfs_alloc_update(bno_cur, nbno, nlen))) + goto error0; + } + /* + * Have only a right contiguous neighbor. + * Merge it together with the new freespace. + */ + else if (haveright) { + /* + * Delete the old by-size entry on the right. + */ + if ((error = xfs_alloc_lookup_eq(cnt_cur, gtbno, gtlen, &i))) + goto error0; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cnt_cur); + error = -EFSCORRUPTED; + goto error0; + } + if ((error = xfs_btree_delete(cnt_cur, &i))) + goto error0; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cnt_cur); + error = -EFSCORRUPTED; + goto error0; + } + /* + * Update the starting block and length of the right + * neighbor in the by-block tree. + */ + nbno = bno; + nlen = len + gtlen; + if ((error = xfs_alloc_update(bno_cur, nbno, nlen))) + goto error0; + } + /* + * No contiguous neighbors. + * Insert the new freespace into the by-block tree. + */ + else { + nbno = bno; + nlen = len; + if ((error = xfs_btree_insert(bno_cur, &i))) + goto error0; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(bno_cur); + error = -EFSCORRUPTED; + goto error0; + } + } + xfs_btree_del_cursor(bno_cur, XFS_BTREE_NOERROR); + bno_cur = NULL; + + /* + * In all cases we need to insert the new freespace in the by-size tree. + * + * If this new freespace is being inserted in the block that contains + * the largest free space in the btree, make sure we also fix up the + * agf->agf-longest tracker field. + */ + if ((error = xfs_alloc_lookup_eq(cnt_cur, nbno, nlen, &i))) + goto error0; + if (XFS_IS_CORRUPT(mp, i != 0)) { + xfs_btree_mark_sick(cnt_cur); + error = -EFSCORRUPTED; + goto error0; + } + if (xfs_alloc_cursor_at_lastrec(cnt_cur)) + fixup_longest = true; + if ((error = xfs_btree_insert(cnt_cur, &i))) + goto error0; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cnt_cur); + error = -EFSCORRUPTED; + goto error0; + } + if (fixup_longest) { + error = xfs_alloc_fixup_longest(cnt_cur); + if (error) + goto error0; + } + + xfs_btree_del_cursor(cnt_cur, XFS_BTREE_NOERROR); + cnt_cur = NULL; + + /* + * Update the freespace totals in the ag and superblock. + */ + error = xfs_alloc_update_counters(tp, agbp, len); + xfs_ag_resv_free_extent(pag, type, tp, len); + if (error) + goto error0; + + XFS_STATS_INC(mp, xs_freex); + XFS_STATS_ADD(mp, xs_freeb, len); + + trace_xfs_free_extent(pag, bno, len, type, haveleft, haveright); + + return 0; + + error0: + trace_xfs_free_extent(pag, bno, len, type, -1, -1); + if (bno_cur) + xfs_btree_del_cursor(bno_cur, XFS_BTREE_ERROR); + if (cnt_cur) + xfs_btree_del_cursor(cnt_cur, XFS_BTREE_ERROR); + return error; +} + +/* + * Visible (exported) allocation/free functions. + * Some of these are used just by xfs_alloc_btree.c and this file. + */ + +/* + * Compute and fill in value of m_alloc_maxlevels. + */ +void +xfs_alloc_compute_maxlevels( + xfs_mount_t *mp) /* file system mount structure */ +{ + mp->m_alloc_maxlevels = xfs_btree_compute_maxlevels(mp->m_alloc_mnr, + (mp->m_sb.sb_agblocks + 1) / 2); + ASSERT(mp->m_alloc_maxlevels <= xfs_allocbt_maxlevels_ondisk()); +} + +/* + * Find the length of the longest extent in an AG. The 'need' parameter + * specifies how much space we're going to need for the AGFL and the + * 'reserved' parameter tells us how many blocks in this AG are reserved for + * other callers. + */ +xfs_extlen_t +xfs_alloc_longest_free_extent( + struct xfs_perag *pag, + xfs_extlen_t need, + xfs_extlen_t reserved) +{ + xfs_extlen_t delta = 0; + + /* + * If the AGFL needs a recharge, we'll have to subtract that from the + * longest extent. + */ + if (need > pag->pagf_flcount) + delta = need - pag->pagf_flcount; + + /* + * If we cannot maintain others' reservations with space from the + * not-longest freesp extents, we'll have to subtract /that/ from + * the longest extent too. + */ + if (pag->pagf_freeblks - pag->pagf_longest < reserved) + delta += reserved - (pag->pagf_freeblks - pag->pagf_longest); + + /* + * If the longest extent is long enough to satisfy all the + * reservations and AGFL rules in place, we can return this extent. + */ + if (pag->pagf_longest > delta) + return min_t(xfs_extlen_t, pag_mount(pag)->m_ag_max_usable, + pag->pagf_longest - delta); + + /* Otherwise, let the caller try for 1 block if there's space. */ + return pag->pagf_flcount > 0 || pag->pagf_longest > 0; +} + +/* + * Compute the minimum length of the AGFL in the given AG. If @pag is NULL, + * return the largest possible minimum length. + */ +unsigned int +xfs_alloc_min_freelist( + struct xfs_mount *mp, + struct xfs_perag *pag) +{ + /* AG btrees have at least 1 level. */ + const unsigned int bno_level = pag ? pag->pagf_bno_level : 1; + const unsigned int cnt_level = pag ? pag->pagf_cnt_level : 1; + const unsigned int rmap_level = pag ? pag->pagf_rmap_level : 1; + unsigned int min_free; + + ASSERT(mp->m_alloc_maxlevels > 0); + + /* + * For a btree shorter than the maximum height, the worst case is that + * every level gets split and a new level is added, then while inserting + * another entry to refill the AGFL, every level under the old root gets + * split again. This is: + * + * (full height split reservation) + (AGFL refill split height) + * = (current height + 1) + (current height - 1) + * = (new height) + (new height - 2) + * = 2 * new height - 2 + * + * For a btree of maximum height, the worst case is that every level + * under the root gets split, then while inserting another entry to + * refill the AGFL, every level under the root gets split again. This is + * also: + * + * 2 * (current height - 1) + * = 2 * (new height - 1) + * = 2 * new height - 2 + */ + + /* space needed by-bno freespace btree */ + min_free = min(bno_level + 1, mp->m_alloc_maxlevels) * 2 - 2; + /* space needed by-size freespace btree */ + min_free += min(cnt_level + 1, mp->m_alloc_maxlevels) * 2 - 2; + /* space needed reverse mapping used space btree */ + if (xfs_has_rmapbt(mp)) + min_free += min(rmap_level + 1, mp->m_rmap_maxlevels) * 2 - 2; + return min_free; +} + +/* + * Check if the operation we are fixing up the freelist for should go ahead or + * not. If we are freeing blocks, we always allow it, otherwise the allocation + * is dependent on whether the size and shape of free space available will + * permit the requested allocation to take place. + */ +static bool +xfs_alloc_space_available( + struct xfs_alloc_arg *args, + xfs_extlen_t min_free, + int flags) +{ + struct xfs_perag *pag = args->pag; + xfs_extlen_t alloc_len, longest; + xfs_extlen_t reservation; /* blocks that are still reserved */ + int available; + xfs_extlen_t agflcount; + + if (flags & XFS_ALLOC_FLAG_FREEING) + return true; + + reservation = xfs_ag_resv_needed(pag, args->resv); + + /* do we have enough contiguous free space for the allocation? */ + alloc_len = args->minlen + (args->alignment - 1) + args->minalignslop; + longest = xfs_alloc_longest_free_extent(pag, min_free, reservation); + if (longest < alloc_len) + return false; + + /* + * Do we have enough free space remaining for the allocation? Don't + * account extra agfl blocks because we are about to defer free them, + * making them unavailable until the current transaction commits. + */ + agflcount = min_t(xfs_extlen_t, pag->pagf_flcount, min_free); + available = (int)(pag->pagf_freeblks + agflcount - + reservation - min_free - args->minleft); + if (available < (int)max(args->total, alloc_len)) + return false; + + /* + * Clamp maxlen to the amount of free space available for the actual + * extent allocation. + */ + if (available < (int)args->maxlen && !(flags & XFS_ALLOC_FLAG_CHECK)) { + args->maxlen = available; + ASSERT(args->maxlen > 0); + ASSERT(args->maxlen >= args->minlen); + } + + return true; +} + +/* + * Check the agfl fields of the agf for inconsistency or corruption. + * + * The original purpose was to detect an agfl header padding mismatch between + * current and early v5 kernels. This problem manifests as a 1-slot size + * difference between the on-disk flcount and the active [first, last] range of + * a wrapped agfl. + * + * However, we need to use these same checks to catch agfl count corruptions + * unrelated to padding. This could occur on any v4 or v5 filesystem, so either + * way, we need to reset the agfl and warn the user. + * + * Return true if a reset is required before the agfl can be used, false + * otherwise. + */ +static bool +xfs_agfl_needs_reset( + struct xfs_mount *mp, + struct xfs_agf *agf) +{ + uint32_t f = be32_to_cpu(agf->agf_flfirst); + uint32_t l = be32_to_cpu(agf->agf_fllast); + uint32_t c = be32_to_cpu(agf->agf_flcount); + int agfl_size = xfs_agfl_size(mp); + int active; + + /* + * The agf read verifier catches severe corruption of these fields. + * Repeat some sanity checks to cover a packed -> unpacked mismatch if + * the verifier allows it. + */ + if (f >= agfl_size || l >= agfl_size) + return true; + if (c > agfl_size) + return true; + + /* + * Check consistency between the on-disk count and the active range. An + * agfl padding mismatch manifests as an inconsistent flcount. + */ + if (c && l >= f) + active = l - f + 1; + else if (c) + active = agfl_size - f + l + 1; + else + active = 0; + + return active != c; +} + +/* + * Reset the agfl to an empty state. Ignore/drop any existing blocks since the + * agfl content cannot be trusted. Warn the user that a repair is required to + * recover leaked blocks. + * + * The purpose of this mechanism is to handle filesystems affected by the agfl + * header padding mismatch problem. A reset keeps the filesystem online with a + * relatively minor free space accounting inconsistency rather than suffer the + * inevitable crash from use of an invalid agfl block. + */ +static void +xfs_agfl_reset( + struct xfs_trans *tp, + struct xfs_buf *agbp, + struct xfs_perag *pag) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_agf *agf = agbp->b_addr; + + ASSERT(xfs_perag_agfl_needs_reset(pag)); + trace_xfs_agfl_reset(mp, agf, 0, _RET_IP_); + + xfs_warn(mp, + "WARNING: Reset corrupted AGFL on AG %u. %d blocks leaked. " + "Please unmount and run xfs_repair.", + pag_agno(pag), pag->pagf_flcount); + + agf->agf_flfirst = 0; + agf->agf_fllast = cpu_to_be32(xfs_agfl_size(mp) - 1); + agf->agf_flcount = 0; + xfs_alloc_log_agf(tp, agbp, XFS_AGF_FLFIRST | XFS_AGF_FLLAST | + XFS_AGF_FLCOUNT); + + pag->pagf_flcount = 0; + clear_bit(XFS_AGSTATE_AGFL_NEEDS_RESET, &pag->pag_opstate); +} + +/* + * Add the extent to the list of extents to be free at transaction end. + * The list is maintained sorted (by block number). + */ +static int +xfs_defer_extent_free( + struct xfs_trans *tp, + xfs_fsblock_t bno, + xfs_filblks_t len, + const struct xfs_owner_info *oinfo, + enum xfs_ag_resv_type type, + unsigned int free_flags, + struct xfs_defer_pending **dfpp) +{ + struct xfs_extent_free_item *xefi; + struct xfs_mount *mp = tp->t_mountp; + + ASSERT(len <= XFS_MAX_BMBT_EXTLEN); + ASSERT(!isnullstartblock(bno)); + ASSERT(!(free_flags & ~XFS_FREE_EXTENT_ALL_FLAGS)); + + if (free_flags & XFS_FREE_EXTENT_REALTIME) { + if (type != XFS_AG_RESV_NONE) { + ASSERT(type == XFS_AG_RESV_NONE); + return -EFSCORRUPTED; + } + if (XFS_IS_CORRUPT(mp, !xfs_verify_rtbext(mp, bno, len))) + return -EFSCORRUPTED; + } else { + if (XFS_IS_CORRUPT(mp, !xfs_verify_fsbext(mp, bno, len))) + return -EFSCORRUPTED; + } + + xefi = kmem_cache_zalloc(xfs_extfree_item_cache, + GFP_KERNEL | __GFP_NOFAIL); + xefi->xefi_startblock = bno; + xefi->xefi_blockcount = (xfs_extlen_t)len; + xefi->xefi_agresv = type; + if (free_flags & XFS_FREE_EXTENT_SKIP_DISCARD) + xefi->xefi_flags |= XFS_EFI_SKIP_DISCARD; + if (free_flags & XFS_FREE_EXTENT_REALTIME) + xefi->xefi_flags |= XFS_EFI_REALTIME; + if (oinfo) { + ASSERT(oinfo->oi_offset == 0); + + if (oinfo->oi_flags & XFS_OWNER_INFO_ATTR_FORK) + xefi->xefi_flags |= XFS_EFI_ATTR_FORK; + if (oinfo->oi_flags & XFS_OWNER_INFO_BMBT_BLOCK) + xefi->xefi_flags |= XFS_EFI_BMBT_BLOCK; + xefi->xefi_owner = oinfo->oi_owner; + } else { + xefi->xefi_owner = XFS_RMAP_OWN_NULL; + } + + xfs_extent_free_defer_add(tp, xefi, dfpp); + return 0; +} + +int +xfs_free_extent_later( + struct xfs_trans *tp, + xfs_fsblock_t bno, + xfs_filblks_t len, + const struct xfs_owner_info *oinfo, + enum xfs_ag_resv_type type, + unsigned int free_flags) +{ + struct xfs_defer_pending *dontcare = NULL; + + return xfs_defer_extent_free(tp, bno, len, oinfo, type, free_flags, + &dontcare); +} + +/* + * Set up automatic freeing of unwritten space in the filesystem. + * + * This function attached a paused deferred extent free item to the + * transaction. Pausing means that the EFI will be logged in the next + * transaction commit, but the pending EFI will not be finished until the + * pending item is unpaused. + * + * If the system goes down after the EFI has been persisted to the log but + * before the pending item is unpaused, log recovery will find the EFI, fail to + * find the EFD, and free the space. + * + * If the pending item is unpaused, the next transaction commit will log an EFD + * without freeing the space. + * + * Caller must ensure that the tp, fsbno, len, oinfo, and resv flags of the + * @args structure are set to the relevant values. + */ +int +xfs_alloc_schedule_autoreap( + const struct xfs_alloc_arg *args, + unsigned int free_flags, + struct xfs_alloc_autoreap *aarp) +{ + int error; + + error = xfs_defer_extent_free(args->tp, args->fsbno, args->len, + &args->oinfo, args->resv, free_flags, &aarp->dfp); + if (error) + return error; + + xfs_defer_item_pause(args->tp, aarp->dfp); + return 0; +} + +/* + * Cancel automatic freeing of unwritten space in the filesystem. + * + * Earlier, we created a paused deferred extent free item and attached it to + * this transaction so that we could automatically roll back a new space + * allocation if the system went down. Now we want to cancel the paused work + * item by marking the EFI stale so we don't actually free the space, unpausing + * the pending item and logging an EFD. + * + * The caller generally should have already mapped the space into the ondisk + * filesystem. If the reserved space was partially used, the caller must call + * xfs_free_extent_later to create a new EFI to free the unused space. + */ +void +xfs_alloc_cancel_autoreap( + struct xfs_trans *tp, + struct xfs_alloc_autoreap *aarp) +{ + struct xfs_defer_pending *dfp = aarp->dfp; + struct xfs_extent_free_item *xefi; + + if (!dfp) + return; + + list_for_each_entry(xefi, &dfp->dfp_work, xefi_list) + xefi->xefi_flags |= XFS_EFI_CANCELLED; + + xfs_defer_item_unpause(tp, dfp); +} + +/* + * Commit automatic freeing of unwritten space in the filesystem. + * + * This unpauses an earlier _schedule_autoreap and commits to freeing the + * allocated space. Call this if none of the reserved space was used. + */ +void +xfs_alloc_commit_autoreap( + struct xfs_trans *tp, + struct xfs_alloc_autoreap *aarp) +{ + if (aarp->dfp) + xfs_defer_item_unpause(tp, aarp->dfp); +} + +/* + * Check if an AGF has a free extent record whose length is equal to + * args->minlen. + */ +STATIC int +xfs_exact_minlen_extent_available( + struct xfs_alloc_arg *args, + struct xfs_buf *agbp, + int *stat) +{ + struct xfs_btree_cur *cnt_cur; + xfs_agblock_t fbno; + xfs_extlen_t flen; + int error = 0; + + cnt_cur = xfs_cntbt_init_cursor(args->mp, args->tp, agbp, + args->pag); + error = xfs_alloc_lookup_ge(cnt_cur, 0, args->minlen, stat); + if (error) + goto out; + + if (*stat == 0) { + xfs_btree_mark_sick(cnt_cur); + error = -EFSCORRUPTED; + goto out; + } + + error = xfs_alloc_get_rec(cnt_cur, &fbno, &flen, stat); + if (error) + goto out; + + if (*stat == 1 && flen != args->minlen) + *stat = 0; + +out: + xfs_btree_del_cursor(cnt_cur, error); + + return error; +} + +/* + * Decide whether to use this allocation group for this allocation. + * If so, fix up the btree freelist's size. + */ +int /* error */ +xfs_alloc_fix_freelist( + struct xfs_alloc_arg *args, /* allocation argument structure */ + uint32_t alloc_flags) +{ + struct xfs_mount *mp = args->mp; + struct xfs_perag *pag = args->pag; + struct xfs_trans *tp = args->tp; + struct xfs_buf *agbp = NULL; + struct xfs_buf *agflbp = NULL; + struct xfs_alloc_arg targs; /* local allocation arguments */ + xfs_agblock_t bno; /* freelist block */ + xfs_extlen_t need; /* total blocks needed in freelist */ + int error = 0; + + /* deferred ops (AGFL block frees) require permanent transactions */ + ASSERT(tp->t_flags & XFS_TRANS_PERM_LOG_RES); + + if (!xfs_perag_initialised_agf(pag)) { + error = xfs_alloc_read_agf(pag, tp, alloc_flags, &agbp); + if (error) { + /* Couldn't lock the AGF so skip this AG. */ + if (error == -EAGAIN) + error = 0; + goto out_no_agbp; + } + } + + /* + * If this is a metadata preferred pag and we are user data then try + * somewhere else if we are not being asked to try harder at this + * point + */ + if (xfs_perag_prefers_metadata(pag) && + (args->datatype & XFS_ALLOC_USERDATA) && + (alloc_flags & XFS_ALLOC_FLAG_TRYLOCK)) { + ASSERT(!(alloc_flags & XFS_ALLOC_FLAG_FREEING)); + goto out_agbp_relse; + } + + need = xfs_alloc_min_freelist(mp, pag); + if (!xfs_alloc_space_available(args, need, alloc_flags | + XFS_ALLOC_FLAG_CHECK)) + goto out_agbp_relse; + + /* + * Get the a.g. freespace buffer. + * Can fail if we're not blocking on locks, and it's held. + */ + if (!agbp) { + error = xfs_alloc_read_agf(pag, tp, alloc_flags, &agbp); + if (error) { + /* Couldn't lock the AGF so skip this AG. */ + if (error == -EAGAIN) + error = 0; + goto out_no_agbp; + } + } + + /* reset a padding mismatched agfl before final free space check */ + if (xfs_perag_agfl_needs_reset(pag)) + xfs_agfl_reset(tp, agbp, pag); + + /* If there isn't enough total space or single-extent, reject it. */ + need = xfs_alloc_min_freelist(mp, pag); + if (!xfs_alloc_space_available(args, need, alloc_flags)) + goto out_agbp_relse; + + if (IS_ENABLED(CONFIG_XFS_DEBUG) && args->alloc_minlen_only) { + int stat; + + error = xfs_exact_minlen_extent_available(args, agbp, &stat); + if (error || !stat) + goto out_agbp_relse; + } + + /* + * Make the freelist shorter if it's too long. + * + * Note that from this point onwards, we will always release the agf and + * agfl buffers on error. This handles the case where we error out and + * the buffers are clean or may not have been joined to the transaction + * and hence need to be released manually. If they have been joined to + * the transaction, then xfs_trans_brelse() will handle them + * appropriately based on the recursion count and dirty state of the + * buffer. + * + * XXX (dgc): When we have lots of free space, does this buy us + * anything other than extra overhead when we need to put more blocks + * back on the free list? Maybe we should only do this when space is + * getting low or the AGFL is more than half full? + * + * The NOSHRINK flag prevents the AGFL from being shrunk if it's too + * big; the NORMAP flag prevents AGFL expand/shrink operations from + * updating the rmapbt. Both flags are used in xfs_repair while we're + * rebuilding the rmapbt, and neither are used by the kernel. They're + * both required to ensure that rmaps are correctly recorded for the + * regenerated AGFL, bnobt, and cntbt. See repair/phase5.c and + * repair/rmap.c in xfsprogs for details. + */ + memset(&targs, 0, sizeof(targs)); + /* struct copy below */ + if (alloc_flags & XFS_ALLOC_FLAG_NORMAP) + targs.oinfo = XFS_RMAP_OINFO_SKIP_UPDATE; + else + targs.oinfo = XFS_RMAP_OINFO_AG; + while (!(alloc_flags & XFS_ALLOC_FLAG_NOSHRINK) && + pag->pagf_flcount > need) { + error = xfs_alloc_get_freelist(pag, tp, agbp, &bno, 0); + if (error) + goto out_agbp_relse; + + /* + * Defer the AGFL block free. + * + * This helps to prevent log reservation overruns due to too + * many allocation operations in a transaction. AGFL frees are + * prone to this problem because for one they are always freed + * one at a time. Further, an immediate AGFL block free can + * cause a btree join and require another block free before the + * real allocation can proceed. + * Deferring the free disconnects freeing up the AGFL slot from + * freeing the block. + */ + error = xfs_free_extent_later(tp, xfs_agbno_to_fsb(pag, bno), + 1, &targs.oinfo, XFS_AG_RESV_AGFL, 0); + if (error) + goto out_agbp_relse; + } + + targs.tp = tp; + targs.mp = mp; + targs.agbp = agbp; + targs.agno = args->agno; + targs.alignment = targs.minlen = targs.prod = 1; + targs.pag = pag; + error = xfs_alloc_read_agfl(pag, tp, &agflbp); + if (error) + goto out_agbp_relse; + + /* Make the freelist longer if it's too short. */ + while (pag->pagf_flcount < need) { + targs.agbno = 0; + targs.maxlen = need - pag->pagf_flcount; + targs.resv = XFS_AG_RESV_AGFL; + + /* Allocate as many blocks as possible at once. */ + error = xfs_alloc_ag_vextent_size(&targs, alloc_flags); + if (error) + goto out_agflbp_relse; + + /* + * Stop if we run out. Won't happen if callers are obeying + * the restrictions correctly. Can happen for free calls + * on a completely full ag. + */ + if (targs.agbno == NULLAGBLOCK) { + if (alloc_flags & XFS_ALLOC_FLAG_FREEING) + break; + goto out_agflbp_relse; + } + + if (!xfs_rmap_should_skip_owner_update(&targs.oinfo)) { + error = xfs_rmap_alloc(tp, agbp, pag, + targs.agbno, targs.len, &targs.oinfo); + if (error) + goto out_agflbp_relse; + } + error = xfs_alloc_update_counters(tp, agbp, + -((long)(targs.len))); + if (error) + goto out_agflbp_relse; + + /* + * Put each allocated block on the list. + */ + for (bno = targs.agbno; bno < targs.agbno + targs.len; bno++) { + error = xfs_alloc_put_freelist(pag, tp, agbp, + agflbp, bno, 0); + if (error) + goto out_agflbp_relse; + } + } + xfs_trans_brelse(tp, agflbp); + args->agbp = agbp; + return 0; + +out_agflbp_relse: + xfs_trans_brelse(tp, agflbp); +out_agbp_relse: + if (agbp) + xfs_trans_brelse(tp, agbp); +out_no_agbp: + args->agbp = NULL; + return error; +} + +/* + * Get a block from the freelist. + * Returns with the buffer for the block gotten. + */ +int +xfs_alloc_get_freelist( + struct xfs_perag *pag, + struct xfs_trans *tp, + struct xfs_buf *agbp, + xfs_agblock_t *bnop, + int btreeblk) +{ + struct xfs_agf *agf = agbp->b_addr; + struct xfs_buf *agflbp; + xfs_agblock_t bno; + __be32 *agfl_bno; + int error; + uint32_t logflags; + struct xfs_mount *mp = tp->t_mountp; + + /* + * Freelist is empty, give up. + */ + if (!agf->agf_flcount) { + *bnop = NULLAGBLOCK; + return 0; + } + /* + * Read the array of free blocks. + */ + error = xfs_alloc_read_agfl(pag, tp, &agflbp); + if (error) + return error; + + + /* + * Get the block number and update the data structures. + */ + agfl_bno = xfs_buf_to_agfl_bno(agflbp); + bno = be32_to_cpu(agfl_bno[be32_to_cpu(agf->agf_flfirst)]); + if (XFS_IS_CORRUPT(tp->t_mountp, !xfs_verify_agbno(pag, bno))) + return -EFSCORRUPTED; + + be32_add_cpu(&agf->agf_flfirst, 1); + xfs_trans_brelse(tp, agflbp); + if (be32_to_cpu(agf->agf_flfirst) == xfs_agfl_size(mp)) + agf->agf_flfirst = 0; + + ASSERT(!xfs_perag_agfl_needs_reset(pag)); + be32_add_cpu(&agf->agf_flcount, -1); + pag->pagf_flcount--; + + logflags = XFS_AGF_FLFIRST | XFS_AGF_FLCOUNT; + if (btreeblk) { + be32_add_cpu(&agf->agf_btreeblks, 1); + pag->pagf_btreeblks++; + logflags |= XFS_AGF_BTREEBLKS; + } + + xfs_alloc_log_agf(tp, agbp, logflags); + *bnop = bno; + + return 0; +} + +/* + * Log the given fields from the agf structure. + */ +void +xfs_alloc_log_agf( + struct xfs_trans *tp, + struct xfs_buf *bp, + uint32_t fields) +{ + int first; /* first byte offset */ + int last; /* last byte offset */ + static const short offsets[] = { + offsetof(xfs_agf_t, agf_magicnum), + offsetof(xfs_agf_t, agf_versionnum), + offsetof(xfs_agf_t, agf_seqno), + offsetof(xfs_agf_t, agf_length), + offsetof(xfs_agf_t, agf_bno_root), /* also cnt/rmap root */ + offsetof(xfs_agf_t, agf_bno_level), /* also cnt/rmap levels */ + offsetof(xfs_agf_t, agf_flfirst), + offsetof(xfs_agf_t, agf_fllast), + offsetof(xfs_agf_t, agf_flcount), + offsetof(xfs_agf_t, agf_freeblks), + offsetof(xfs_agf_t, agf_longest), + offsetof(xfs_agf_t, agf_btreeblks), + offsetof(xfs_agf_t, agf_uuid), + offsetof(xfs_agf_t, agf_rmap_blocks), + offsetof(xfs_agf_t, agf_refcount_blocks), + offsetof(xfs_agf_t, agf_refcount_root), + offsetof(xfs_agf_t, agf_refcount_level), + /* needed so that we don't log the whole rest of the structure: */ + offsetof(xfs_agf_t, agf_spare64), + sizeof(xfs_agf_t) + }; + + trace_xfs_agf(tp->t_mountp, bp->b_addr, fields, _RET_IP_); + + xfs_trans_buf_set_type(tp, bp, XFS_BLFT_AGF_BUF); + + xfs_btree_offsets(fields, offsets, XFS_AGF_NUM_BITS, &first, &last); + xfs_trans_log_buf(tp, bp, (uint)first, (uint)last); +} + +/* + * Put the block on the freelist for the allocation group. + */ +int +xfs_alloc_put_freelist( + struct xfs_perag *pag, + struct xfs_trans *tp, + struct xfs_buf *agbp, + struct xfs_buf *agflbp, + xfs_agblock_t bno, + int btreeblk) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_agf *agf = agbp->b_addr; + __be32 *blockp; + int error; + uint32_t logflags; + __be32 *agfl_bno; + int startoff; + + if (!agflbp) { + error = xfs_alloc_read_agfl(pag, tp, &agflbp); + if (error) + return error; + } + + be32_add_cpu(&agf->agf_fllast, 1); + if (be32_to_cpu(agf->agf_fllast) == xfs_agfl_size(mp)) + agf->agf_fllast = 0; + + ASSERT(!xfs_perag_agfl_needs_reset(pag)); + be32_add_cpu(&agf->agf_flcount, 1); + pag->pagf_flcount++; + + logflags = XFS_AGF_FLLAST | XFS_AGF_FLCOUNT; + if (btreeblk) { + be32_add_cpu(&agf->agf_btreeblks, -1); + pag->pagf_btreeblks--; + logflags |= XFS_AGF_BTREEBLKS; + } + + ASSERT(be32_to_cpu(agf->agf_flcount) <= xfs_agfl_size(mp)); + + agfl_bno = xfs_buf_to_agfl_bno(agflbp); + blockp = &agfl_bno[be32_to_cpu(agf->agf_fllast)]; + *blockp = cpu_to_be32(bno); + startoff = (char *)blockp - (char *)agflbp->b_addr; + + xfs_alloc_log_agf(tp, agbp, logflags); + + xfs_trans_buf_set_type(tp, agflbp, XFS_BLFT_AGFL_BUF); + xfs_trans_log_buf(tp, agflbp, startoff, + startoff + sizeof(xfs_agblock_t) - 1); + return 0; +} + +/* + * Check that this AGF/AGI header's sequence number and length matches the AG + * number and size in fsblocks. + */ +xfs_failaddr_t +xfs_validate_ag_length( + struct xfs_buf *bp, + uint32_t seqno, + uint32_t length) +{ + struct xfs_mount *mp = bp->b_mount; + /* + * During growfs operations, the perag is not fully initialised, + * so we can't use it for any useful checking. growfs ensures we can't + * use it by using uncached buffers that don't have the perag attached + * so we can detect and avoid this problem. + */ + if (bp->b_pag && seqno != pag_agno(bp->b_pag)) + return __this_address; + + /* + * Only the last AG in the filesystem is allowed to be shorter + * than the AG size recorded in the superblock. + */ + if (length != mp->m_sb.sb_agblocks) { + /* + * During growfs, the new last AG can get here before we + * have updated the superblock. Give it a pass on the seqno + * check. + */ + if (bp->b_pag && seqno != mp->m_sb.sb_agcount - 1) + return __this_address; + if (length < XFS_MIN_AG_BLOCKS) + return __this_address; + if (length > mp->m_sb.sb_agblocks) + return __this_address; + } + + return NULL; +} + +/* + * Verify the AGF is consistent. + * + * We do not verify the AGFL indexes in the AGF are fully consistent here + * because of issues with variable on-disk structure sizes. Instead, we check + * the agfl indexes for consistency when we initialise the perag from the AGF + * information after a read completes. + * + * If the index is inconsistent, then we mark the perag as needing an AGFL + * reset. The first AGFL update performed then resets the AGFL indexes and + * refills the AGFL with known good free blocks, allowing the filesystem to + * continue operating normally at the cost of a few leaked free space blocks. + */ +static xfs_failaddr_t +xfs_agf_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_agf *agf = bp->b_addr; + xfs_failaddr_t fa; + uint32_t agf_seqno = be32_to_cpu(agf->agf_seqno); + uint32_t agf_length = be32_to_cpu(agf->agf_length); + + if (xfs_has_crc(mp)) { + if (!uuid_equal(&agf->agf_uuid, &mp->m_sb.sb_meta_uuid)) + return __this_address; + if (!xfs_log_check_lsn(mp, be64_to_cpu(agf->agf_lsn))) + return __this_address; + } + + if (!xfs_verify_magic(bp, agf->agf_magicnum)) + return __this_address; + + if (!XFS_AGF_GOOD_VERSION(be32_to_cpu(agf->agf_versionnum))) + return __this_address; + + /* + * Both agf_seqno and agf_length need to validated before anything else + * block number related in the AGF or AGFL can be checked. + */ + fa = xfs_validate_ag_length(bp, agf_seqno, agf_length); + if (fa) + return fa; + + if (be32_to_cpu(agf->agf_flfirst) >= xfs_agfl_size(mp)) + return __this_address; + if (be32_to_cpu(agf->agf_fllast) >= xfs_agfl_size(mp)) + return __this_address; + if (be32_to_cpu(agf->agf_flcount) > xfs_agfl_size(mp)) + return __this_address; + + if (be32_to_cpu(agf->agf_freeblks) < be32_to_cpu(agf->agf_longest) || + be32_to_cpu(agf->agf_freeblks) > agf_length) + return __this_address; + + if (be32_to_cpu(agf->agf_bno_level) < 1 || + be32_to_cpu(agf->agf_cnt_level) < 1 || + be32_to_cpu(agf->agf_bno_level) > mp->m_alloc_maxlevels || + be32_to_cpu(agf->agf_cnt_level) > mp->m_alloc_maxlevels) + return __this_address; + + if (xfs_has_lazysbcount(mp) && + be32_to_cpu(agf->agf_btreeblks) > agf_length) + return __this_address; + + if (xfs_has_rmapbt(mp)) { + if (be32_to_cpu(agf->agf_rmap_blocks) > agf_length) + return __this_address; + + if (be32_to_cpu(agf->agf_rmap_level) < 1 || + be32_to_cpu(agf->agf_rmap_level) > mp->m_rmap_maxlevels) + return __this_address; + } + + if (xfs_has_reflink(mp)) { + if (be32_to_cpu(agf->agf_refcount_blocks) > agf_length) + return __this_address; + + if (be32_to_cpu(agf->agf_refcount_level) < 1 || + be32_to_cpu(agf->agf_refcount_level) > mp->m_refc_maxlevels) + return __this_address; + } + + return NULL; +} + +static void +xfs_agf_read_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + xfs_failaddr_t fa; + + if (xfs_has_crc(mp) && + !xfs_buf_verify_cksum(bp, XFS_AGF_CRC_OFF)) + xfs_verifier_error(bp, -EFSBADCRC, __this_address); + else { + fa = xfs_agf_verify(bp); + if (fa || XFS_TEST_ERROR(mp, XFS_ERRTAG_ALLOC_READ_AGF)) + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + } +} + +static void +xfs_agf_write_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_buf_log_item *bip = bp->b_log_item; + struct xfs_agf *agf = bp->b_addr; + xfs_failaddr_t fa; + + fa = xfs_agf_verify(bp); + if (fa) { + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + return; + } + + if (!xfs_has_crc(mp)) + return; + + if (bip) + agf->agf_lsn = cpu_to_be64(bip->bli_item.li_lsn); + + xfs_buf_update_cksum(bp, XFS_AGF_CRC_OFF); +} + +const struct xfs_buf_ops xfs_agf_buf_ops = { + .name = "xfs_agf", + .magic = { cpu_to_be32(XFS_AGF_MAGIC), cpu_to_be32(XFS_AGF_MAGIC) }, + .verify_read = xfs_agf_read_verify, + .verify_write = xfs_agf_write_verify, + .verify_struct = xfs_agf_verify, +}; + +/* + * Read in the allocation group header (free/alloc section). + */ +int +xfs_read_agf( + struct xfs_perag *pag, + struct xfs_trans *tp, + int flags, + struct xfs_buf **agfbpp) +{ + struct xfs_mount *mp = pag_mount(pag); + int error; + + trace_xfs_read_agf(pag); + + error = xfs_trans_read_buf(mp, tp, mp->m_ddev_targp, + XFS_AG_DADDR(mp, pag_agno(pag), XFS_AGF_DADDR(mp)), + XFS_FSS_TO_BB(mp, 1), flags, agfbpp, &xfs_agf_buf_ops); + if (xfs_metadata_is_sick(error)) + xfs_ag_mark_sick(pag, XFS_SICK_AG_AGF); + if (error) + return error; + + xfs_buf_set_ref(*agfbpp, XFS_AGF_REF); + return 0; +} + +/* + * Read in the allocation group header (free/alloc section) and initialise the + * perag structure if necessary. If the caller provides @agfbpp, then return the + * locked buffer to the caller, otherwise free it. + */ +int +xfs_alloc_read_agf( + struct xfs_perag *pag, + struct xfs_trans *tp, + int flags, + struct xfs_buf **agfbpp) +{ + struct xfs_mount *mp = pag_mount(pag); + struct xfs_buf *agfbp; + struct xfs_agf *agf; + int error; + int allocbt_blks; + + trace_xfs_alloc_read_agf(pag); + + /* We don't support trylock when freeing. */ + ASSERT((flags & (XFS_ALLOC_FLAG_FREEING | XFS_ALLOC_FLAG_TRYLOCK)) != + (XFS_ALLOC_FLAG_FREEING | XFS_ALLOC_FLAG_TRYLOCK)); + error = xfs_read_agf(pag, tp, + (flags & XFS_ALLOC_FLAG_TRYLOCK) ? XBF_TRYLOCK : 0, + &agfbp); + if (error) + return error; + + agf = agfbp->b_addr; + if (!xfs_perag_initialised_agf(pag)) { + pag->pagf_freeblks = be32_to_cpu(agf->agf_freeblks); + pag->pagf_btreeblks = be32_to_cpu(agf->agf_btreeblks); + pag->pagf_flcount = be32_to_cpu(agf->agf_flcount); + pag->pagf_longest = be32_to_cpu(agf->agf_longest); + pag->pagf_bno_level = be32_to_cpu(agf->agf_bno_level); + pag->pagf_cnt_level = be32_to_cpu(agf->agf_cnt_level); + pag->pagf_rmap_level = be32_to_cpu(agf->agf_rmap_level); + pag->pagf_refcount_level = be32_to_cpu(agf->agf_refcount_level); + if (xfs_agfl_needs_reset(mp, agf)) + set_bit(XFS_AGSTATE_AGFL_NEEDS_RESET, &pag->pag_opstate); + else + clear_bit(XFS_AGSTATE_AGFL_NEEDS_RESET, &pag->pag_opstate); + + /* + * Update the in-core allocbt counter. Filter out the rmapbt + * subset of the btreeblks counter because the rmapbt is managed + * by perag reservation. Subtract one for the rmapbt root block + * because the rmap counter includes it while the btreeblks + * counter only tracks non-root blocks. + */ + allocbt_blks = pag->pagf_btreeblks; + if (xfs_has_rmapbt(mp)) + allocbt_blks -= be32_to_cpu(agf->agf_rmap_blocks) - 1; + if (allocbt_blks > 0) + atomic64_add(allocbt_blks, &mp->m_allocbt_blks); + + set_bit(XFS_AGSTATE_AGF_INIT, &pag->pag_opstate); + } + +#ifdef DEBUG + /* + * It's possible for the AGF to be out of sync if the block device is + * silently dropping writes. This can happen in fstests with dmflakey + * enabled, which allows the buffer to be cleaned and reclaimed by + * memory pressure and then re-read from disk here. We will get a + * stale version of the AGF from disk, and nothing good can happen from + * here. Hence if we detect this situation, immediately shut down the + * filesystem. + * + * This can also happen if we are already in the middle of a forced + * shutdown, so don't bother checking if we are already shut down. + */ + if (!xfs_is_shutdown(pag_mount(pag))) { + bool ok = true; + + ok &= pag->pagf_freeblks == be32_to_cpu(agf->agf_freeblks); + ok &= pag->pagf_freeblks == be32_to_cpu(agf->agf_freeblks); + ok &= pag->pagf_btreeblks == be32_to_cpu(agf->agf_btreeblks); + ok &= pag->pagf_flcount == be32_to_cpu(agf->agf_flcount); + ok &= pag->pagf_longest == be32_to_cpu(agf->agf_longest); + ok &= pag->pagf_bno_level == be32_to_cpu(agf->agf_bno_level); + ok &= pag->pagf_cnt_level == be32_to_cpu(agf->agf_cnt_level); + + if (XFS_IS_CORRUPT(pag_mount(pag), !ok)) { + xfs_ag_mark_sick(pag, XFS_SICK_AG_AGF); + xfs_trans_brelse(tp, agfbp); + xfs_force_shutdown(pag_mount(pag), + SHUTDOWN_CORRUPT_ONDISK); + return -EFSCORRUPTED; + } + } +#endif /* DEBUG */ + + if (agfbpp) + *agfbpp = agfbp; + else + xfs_trans_brelse(tp, agfbp); + return 0; +} + +/* + * Pre-proces allocation arguments to set initial state that we don't require + * callers to set up correctly, as well as bounds check the allocation args + * that are set up. + */ +static int +xfs_alloc_vextent_check_args( + struct xfs_alloc_arg *args, + xfs_fsblock_t target, + xfs_agnumber_t *minimum_agno) +{ + struct xfs_mount *mp = args->mp; + xfs_agblock_t agsize; + + args->fsbno = NULLFSBLOCK; + + *minimum_agno = 0; + if (args->tp->t_highest_agno != NULLAGNUMBER) + *minimum_agno = args->tp->t_highest_agno; + + /* + * Just fix this up, for the case where the last a.g. is shorter + * (or there's only one a.g.) and the caller couldn't easily figure + * that out (xfs_bmap_alloc). + */ + agsize = mp->m_sb.sb_agblocks; + if (args->maxlen > agsize) + args->maxlen = agsize; + if (args->alignment == 0) + args->alignment = 1; + + ASSERT(args->minlen > 0); + ASSERT(args->maxlen > 0); + ASSERT(args->alignment > 0); + ASSERT(args->resv != XFS_AG_RESV_AGFL); + + ASSERT(XFS_FSB_TO_AGNO(mp, target) < mp->m_sb.sb_agcount); + ASSERT(XFS_FSB_TO_AGBNO(mp, target) < agsize); + ASSERT(args->minlen <= args->maxlen); + ASSERT(args->minlen <= agsize); + ASSERT(args->mod < args->prod); + + if (XFS_FSB_TO_AGNO(mp, target) >= mp->m_sb.sb_agcount || + XFS_FSB_TO_AGBNO(mp, target) >= agsize || + args->minlen > args->maxlen || args->minlen > agsize || + args->mod >= args->prod) { + trace_xfs_alloc_vextent_badargs(args); + return -ENOSPC; + } + + if (args->agno != NULLAGNUMBER && *minimum_agno > args->agno) { + trace_xfs_alloc_vextent_skip_deadlock(args); + return -ENOSPC; + } + return 0; + +} + +/* + * Prepare an AG for allocation. If the AG is not prepared to accept the + * allocation, return failure. + * + * XXX(dgc): The complexity of "need_pag" will go away as all caller paths are + * modified to hold their own perag references. + */ +static int +xfs_alloc_vextent_prepare_ag( + struct xfs_alloc_arg *args, + uint32_t alloc_flags) +{ + bool need_pag = !args->pag; + int error; + + if (need_pag) + args->pag = xfs_perag_get(args->mp, args->agno); + + args->agbp = NULL; + error = xfs_alloc_fix_freelist(args, alloc_flags); + if (error) { + trace_xfs_alloc_vextent_nofix(args); + if (need_pag) + xfs_perag_put(args->pag); + args->agbno = NULLAGBLOCK; + return error; + } + if (!args->agbp) { + /* cannot allocate in this AG at all */ + trace_xfs_alloc_vextent_noagbp(args); + args->agbno = NULLAGBLOCK; + return 0; + } + args->wasfromfl = 0; + return 0; +} + +/* + * Post-process allocation results to account for the allocation if it succeed + * and set the allocated block number correctly for the caller. + * + * XXX: we should really be returning ENOSPC for ENOSPC, not + * hiding it behind a "successful" NULLFSBLOCK allocation. + */ +static int +xfs_alloc_vextent_finish( + struct xfs_alloc_arg *args, + xfs_agnumber_t minimum_agno, + int alloc_error, + bool drop_perag) +{ + struct xfs_mount *mp = args->mp; + int error = 0; + + /* + * We can end up here with a locked AGF. If we failed, the caller is + * likely going to try to allocate again with different parameters, and + * that can widen the AGs that are searched for free space. If we have + * to do BMBT block allocation, we have to do a new allocation. + * + * Hence leaving this function with the AGF locked opens up potential + * ABBA AGF deadlocks because a future allocation attempt in this + * transaction may attempt to lock a lower number AGF. + * + * We can't release the AGF until the transaction is commited, so at + * this point we must update the "first allocation" tracker to point at + * this AG if the tracker is empty or points to a lower AG. This allows + * the next allocation attempt to be modified appropriately to avoid + * deadlocks. + */ + if (args->agbp && + (args->tp->t_highest_agno == NULLAGNUMBER || + args->agno > minimum_agno)) + args->tp->t_highest_agno = args->agno; + + /* + * If the allocation failed with an error or we had an ENOSPC result, + * preserve the returned error whilst also marking the allocation result + * as "no extent allocated". This ensures that callers that fail to + * capture the error will still treat it as a failed allocation. + */ + if (alloc_error || args->agbno == NULLAGBLOCK) { + args->fsbno = NULLFSBLOCK; + error = alloc_error; + goto out_drop_perag; + } + + args->fsbno = xfs_agbno_to_fsb(args->pag, args->agbno); + + ASSERT(args->len >= args->minlen); + ASSERT(args->len <= args->maxlen); + ASSERT(args->agbno % args->alignment == 0); + XFS_AG_CHECK_DADDR(mp, XFS_FSB_TO_DADDR(mp, args->fsbno), args->len); + + /* if not file data, insert new block into the reverse map btree */ + if (!xfs_rmap_should_skip_owner_update(&args->oinfo)) { + error = xfs_rmap_alloc(args->tp, args->agbp, args->pag, + args->agbno, args->len, &args->oinfo); + if (error) + goto out_drop_perag; + } + + if (!args->wasfromfl) { + error = xfs_alloc_update_counters(args->tp, args->agbp, + -((long)(args->len))); + if (error) + goto out_drop_perag; + + ASSERT(!xfs_extent_busy_search(pag_group(args->pag), + args->agbno, args->len)); + } + + xfs_ag_resv_alloc_extent(args->pag, args->resv, args); + + XFS_STATS_INC(mp, xs_allocx); + XFS_STATS_ADD(mp, xs_allocb, args->len); + + trace_xfs_alloc_vextent_finish(args); + +out_drop_perag: + if (drop_perag && args->pag) { + xfs_perag_rele(args->pag); + args->pag = NULL; + } + return error; +} + +/* + * Allocate within a single AG only. This uses a best-fit length algorithm so if + * you need an exact sized allocation without locality constraints, this is the + * fastest way to do it. + * + * Caller is expected to hold a perag reference in args->pag. + */ +int +xfs_alloc_vextent_this_ag( + struct xfs_alloc_arg *args, + xfs_agnumber_t agno) +{ + xfs_agnumber_t minimum_agno; + uint32_t alloc_flags = 0; + int error; + + ASSERT(args->pag != NULL); + ASSERT(pag_agno(args->pag) == agno); + + args->agno = agno; + args->agbno = 0; + + trace_xfs_alloc_vextent_this_ag(args); + + error = xfs_alloc_vextent_check_args(args, + xfs_agbno_to_fsb(args->pag, 0), &minimum_agno); + if (error) { + if (error == -ENOSPC) + return 0; + return error; + } + + error = xfs_alloc_vextent_prepare_ag(args, alloc_flags); + if (!error && args->agbp) + error = xfs_alloc_ag_vextent_size(args, alloc_flags); + + return xfs_alloc_vextent_finish(args, minimum_agno, error, false); +} + +/* + * Iterate all AGs trying to allocate an extent starting from @start_ag. + * + * If the incoming allocation type is XFS_ALLOCTYPE_NEAR_BNO, it means the + * allocation attempts in @start_agno have locality information. If we fail to + * allocate in that AG, then we revert to anywhere-in-AG for all the other AGs + * we attempt to allocation in as there is no locality optimisation possible for + * those allocations. + * + * On return, args->pag may be left referenced if we finish before the "all + * failed" return point. The allocation finish still needs the perag, and + * so the caller will release it once they've finished the allocation. + * + * When we wrap the AG iteration at the end of the filesystem, we have to be + * careful not to wrap into AGs below ones we already have locked in the + * transaction if we are doing a blocking iteration. This will result in an + * out-of-order locking of AGFs and hence can cause deadlocks. + */ +static int +xfs_alloc_vextent_iterate_ags( + struct xfs_alloc_arg *args, + xfs_agnumber_t minimum_agno, + xfs_agnumber_t start_agno, + xfs_agblock_t target_agbno, + uint32_t alloc_flags) +{ + struct xfs_mount *mp = args->mp; + xfs_agnumber_t restart_agno = minimum_agno; + xfs_agnumber_t agno; + int error = 0; + + if (alloc_flags & XFS_ALLOC_FLAG_TRYLOCK) + restart_agno = 0; +restart: + for_each_perag_wrap_range(mp, start_agno, restart_agno, + mp->m_sb.sb_agcount, agno, args->pag) { + args->agno = agno; + error = xfs_alloc_vextent_prepare_ag(args, alloc_flags); + if (error) + break; + if (!args->agbp) { + trace_xfs_alloc_vextent_loopfailed(args); + continue; + } + + /* + * Allocation is supposed to succeed now, so break out of the + * loop regardless of whether we succeed or not. + */ + if (args->agno == start_agno && target_agbno) { + args->agbno = target_agbno; + error = xfs_alloc_ag_vextent_near(args, alloc_flags); + } else { + args->agbno = 0; + error = xfs_alloc_ag_vextent_size(args, alloc_flags); + } + break; + } + if (error) { + xfs_perag_rele(args->pag); + args->pag = NULL; + return error; + } + if (args->agbp) + return 0; + + /* + * We didn't find an AG we can alloation from. If we were given + * constraining flags by the caller, drop them and retry the allocation + * without any constraints being set. + */ + if (alloc_flags & XFS_ALLOC_FLAG_TRYLOCK) { + alloc_flags &= ~XFS_ALLOC_FLAG_TRYLOCK; + restart_agno = minimum_agno; + goto restart; + } + + ASSERT(args->pag == NULL); + trace_xfs_alloc_vextent_allfailed(args); + return 0; +} + +/* + * Iterate from the AGs from the start AG to the end of the filesystem, trying + * to allocate blocks. It starts with a near allocation attempt in the initial + * AG, then falls back to anywhere-in-ag after the first AG fails. It will wrap + * back to zero if allowed by previous allocations in this transaction, + * otherwise will wrap back to the start AG and run a second blocking pass to + * the end of the filesystem. + */ +int +xfs_alloc_vextent_start_ag( + struct xfs_alloc_arg *args, + xfs_fsblock_t target) +{ + struct xfs_mount *mp = args->mp; + xfs_agnumber_t minimum_agno; + xfs_agnumber_t start_agno; + xfs_agnumber_t rotorstep = xfs_rotorstep; + bool bump_rotor = false; + uint32_t alloc_flags = XFS_ALLOC_FLAG_TRYLOCK; + int error; + + ASSERT(args->pag == NULL); + + args->agno = NULLAGNUMBER; + args->agbno = NULLAGBLOCK; + + trace_xfs_alloc_vextent_start_ag(args); + + error = xfs_alloc_vextent_check_args(args, target, &minimum_agno); + if (error) { + if (error == -ENOSPC) + return 0; + return error; + } + + if ((args->datatype & XFS_ALLOC_INITIAL_USER_DATA) && + xfs_is_inode32(mp)) { + target = XFS_AGB_TO_FSB(mp, + ((mp->m_agfrotor / rotorstep) % + mp->m_sb.sb_agcount), 0); + bump_rotor = 1; + } + + start_agno = max(minimum_agno, XFS_FSB_TO_AGNO(mp, target)); + error = xfs_alloc_vextent_iterate_ags(args, minimum_agno, start_agno, + XFS_FSB_TO_AGBNO(mp, target), alloc_flags); + + if (bump_rotor) { + if (args->agno == start_agno) + mp->m_agfrotor = (mp->m_agfrotor + 1) % + (mp->m_sb.sb_agcount * rotorstep); + else + mp->m_agfrotor = (args->agno * rotorstep + 1) % + (mp->m_sb.sb_agcount * rotorstep); + } + + return xfs_alloc_vextent_finish(args, minimum_agno, error, true); +} + +/* + * Iterate from the agno indicated via @target through to the end of the + * filesystem attempting blocking allocation. This does not wrap or try a second + * pass, so will not recurse into AGs lower than indicated by the target. + */ +int +xfs_alloc_vextent_first_ag( + struct xfs_alloc_arg *args, + xfs_fsblock_t target) + { + struct xfs_mount *mp = args->mp; + xfs_agnumber_t minimum_agno; + xfs_agnumber_t start_agno; + uint32_t alloc_flags = XFS_ALLOC_FLAG_TRYLOCK; + int error; + + ASSERT(args->pag == NULL); + + args->agno = NULLAGNUMBER; + args->agbno = NULLAGBLOCK; + + trace_xfs_alloc_vextent_first_ag(args); + + error = xfs_alloc_vextent_check_args(args, target, &minimum_agno); + if (error) { + if (error == -ENOSPC) + return 0; + return error; + } + + start_agno = max(minimum_agno, XFS_FSB_TO_AGNO(mp, target)); + error = xfs_alloc_vextent_iterate_ags(args, minimum_agno, start_agno, + XFS_FSB_TO_AGBNO(mp, target), alloc_flags); + return xfs_alloc_vextent_finish(args, minimum_agno, error, true); +} + +/* + * Allocate at the exact block target or fail. Caller is expected to hold a + * perag reference in args->pag. + */ +int +xfs_alloc_vextent_exact_bno( + struct xfs_alloc_arg *args, + xfs_fsblock_t target) +{ + struct xfs_mount *mp = args->mp; + xfs_agnumber_t minimum_agno; + int error; + + ASSERT(args->pag != NULL); + ASSERT(pag_agno(args->pag) == XFS_FSB_TO_AGNO(mp, target)); + + args->agno = XFS_FSB_TO_AGNO(mp, target); + args->agbno = XFS_FSB_TO_AGBNO(mp, target); + + trace_xfs_alloc_vextent_exact_bno(args); + + error = xfs_alloc_vextent_check_args(args, target, &minimum_agno); + if (error) { + if (error == -ENOSPC) + return 0; + return error; + } + + error = xfs_alloc_vextent_prepare_ag(args, 0); + if (!error && args->agbp) + error = xfs_alloc_ag_vextent_exact(args); + + return xfs_alloc_vextent_finish(args, minimum_agno, error, false); +} + +/* + * Allocate an extent as close to the target as possible. If there are not + * viable candidates in the AG, then fail the allocation. + * + * Caller may or may not have a per-ag reference in args->pag. + */ +int +xfs_alloc_vextent_near_bno( + struct xfs_alloc_arg *args, + xfs_fsblock_t target) +{ + struct xfs_mount *mp = args->mp; + xfs_agnumber_t minimum_agno; + bool needs_perag = args->pag == NULL; + uint32_t alloc_flags = 0; + int error; + + if (!needs_perag) + ASSERT(pag_agno(args->pag) == XFS_FSB_TO_AGNO(mp, target)); + + args->agno = XFS_FSB_TO_AGNO(mp, target); + args->agbno = XFS_FSB_TO_AGBNO(mp, target); + + trace_xfs_alloc_vextent_near_bno(args); + + error = xfs_alloc_vextent_check_args(args, target, &minimum_agno); + if (error) { + if (error == -ENOSPC) + return 0; + return error; + } + + if (needs_perag) + args->pag = xfs_perag_grab(mp, args->agno); + + error = xfs_alloc_vextent_prepare_ag(args, alloc_flags); + if (!error && args->agbp) + error = xfs_alloc_ag_vextent_near(args, alloc_flags); + + return xfs_alloc_vextent_finish(args, minimum_agno, error, needs_perag); +} + +/* Ensure that the freelist is at full capacity. */ +int +xfs_free_extent_fix_freelist( + struct xfs_trans *tp, + struct xfs_perag *pag, + struct xfs_buf **agbp) +{ + struct xfs_alloc_arg args; + int error; + + memset(&args, 0, sizeof(struct xfs_alloc_arg)); + args.tp = tp; + args.mp = tp->t_mountp; + args.agno = pag_agno(pag); + args.pag = pag; + + /* + * validate that the block number is legal - the enables us to detect + * and handle a silent filesystem corruption rather than crashing. + */ + if (args.agno >= args.mp->m_sb.sb_agcount) + return -EFSCORRUPTED; + + error = xfs_alloc_fix_freelist(&args, XFS_ALLOC_FLAG_FREEING); + if (error) + return error; + + *agbp = args.agbp; + return 0; +} + +/* + * Free an extent. + * Just break up the extent address and hand off to xfs_free_ag_extent + * after fixing up the freelist. + */ +int +__xfs_free_extent( + struct xfs_trans *tp, + struct xfs_perag *pag, + xfs_agblock_t agbno, + xfs_extlen_t len, + const struct xfs_owner_info *oinfo, + enum xfs_ag_resv_type type, + bool skip_discard) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_buf *agbp; + struct xfs_agf *agf; + int error; + unsigned int busy_flags = 0; + + ASSERT(len != 0); + ASSERT(type != XFS_AG_RESV_AGFL); + + if (XFS_TEST_ERROR(mp, XFS_ERRTAG_FREE_EXTENT)) + return -EIO; + + error = xfs_free_extent_fix_freelist(tp, pag, &agbp); + if (error) { + if (xfs_metadata_is_sick(error)) + xfs_ag_mark_sick(pag, XFS_SICK_AG_BNOBT); + return error; + } + + agf = agbp->b_addr; + + if (XFS_IS_CORRUPT(mp, agbno >= mp->m_sb.sb_agblocks)) { + xfs_ag_mark_sick(pag, XFS_SICK_AG_BNOBT); + error = -EFSCORRUPTED; + goto err_release; + } + + /* validate the extent size is legal now we have the agf locked */ + if (XFS_IS_CORRUPT(mp, agbno + len > be32_to_cpu(agf->agf_length))) { + xfs_ag_mark_sick(pag, XFS_SICK_AG_BNOBT); + error = -EFSCORRUPTED; + goto err_release; + } + + error = xfs_free_ag_extent(tp, agbp, agbno, len, oinfo, type); + if (error) + goto err_release; + + if (skip_discard) + busy_flags |= XFS_EXTENT_BUSY_SKIP_DISCARD; + xfs_extent_busy_insert(tp, pag_group(pag), agbno, len, busy_flags); + return 0; + +err_release: + xfs_trans_brelse(tp, agbp); + return error; +} + +struct xfs_alloc_query_range_info { + xfs_alloc_query_range_fn fn; + void *priv; +}; + +/* Format btree record and pass to our callback. */ +STATIC int +xfs_alloc_query_range_helper( + struct xfs_btree_cur *cur, + const union xfs_btree_rec *rec, + void *priv) +{ + struct xfs_alloc_query_range_info *query = priv; + struct xfs_alloc_rec_incore irec; + xfs_failaddr_t fa; + + xfs_alloc_btrec_to_irec(rec, &irec); + fa = xfs_alloc_check_irec(to_perag(cur->bc_group), &irec); + if (fa) + return xfs_alloc_complain_bad_rec(cur, fa, &irec); + + return query->fn(cur, &irec, query->priv); +} + +/* Find all free space within a given range of blocks. */ +int +xfs_alloc_query_range( + struct xfs_btree_cur *cur, + const struct xfs_alloc_rec_incore *low_rec, + const struct xfs_alloc_rec_incore *high_rec, + xfs_alloc_query_range_fn fn, + void *priv) +{ + union xfs_btree_irec low_brec = { .a = *low_rec }; + union xfs_btree_irec high_brec = { .a = *high_rec }; + struct xfs_alloc_query_range_info query = { .priv = priv, .fn = fn }; + + ASSERT(xfs_btree_is_bno(cur->bc_ops)); + return xfs_btree_query_range(cur, &low_brec, &high_brec, + xfs_alloc_query_range_helper, &query); +} + +/* Find all free space records. */ +int +xfs_alloc_query_all( + struct xfs_btree_cur *cur, + xfs_alloc_query_range_fn fn, + void *priv) +{ + struct xfs_alloc_query_range_info query; + + ASSERT(xfs_btree_is_bno(cur->bc_ops)); + query.priv = priv; + query.fn = fn; + return xfs_btree_query_all(cur, xfs_alloc_query_range_helper, &query); +} + +/* + * Scan part of the keyspace of the free space and tell us if the area has no + * records, is fully mapped by records, or is partially filled. + */ +int +xfs_alloc_has_records( + struct xfs_btree_cur *cur, + xfs_agblock_t bno, + xfs_extlen_t len, + enum xbtree_recpacking *outcome) +{ + union xfs_btree_irec low; + union xfs_btree_irec high; + + memset(&low, 0, sizeof(low)); + low.a.ar_startblock = bno; + memset(&high, 0xFF, sizeof(high)); + high.a.ar_startblock = bno + len - 1; + + return xfs_btree_has_records(cur, &low, &high, NULL, outcome); +} + +/* + * Walk all the blocks in the AGFL. The @walk_fn can return any negative + * error code or XFS_ITER_*. + */ +int +xfs_agfl_walk( + struct xfs_mount *mp, + struct xfs_agf *agf, + struct xfs_buf *agflbp, + xfs_agfl_walk_fn walk_fn, + void *priv) +{ + __be32 *agfl_bno; + unsigned int i; + int error; + + agfl_bno = xfs_buf_to_agfl_bno(agflbp); + i = be32_to_cpu(agf->agf_flfirst); + + /* Nothing to walk in an empty AGFL. */ + if (agf->agf_flcount == cpu_to_be32(0)) + return 0; + + /* Otherwise, walk from first to last, wrapping as needed. */ + for (;;) { + error = walk_fn(mp, be32_to_cpu(agfl_bno[i]), priv); + if (error) + return error; + if (i == be32_to_cpu(agf->agf_fllast)) + break; + if (++i == xfs_agfl_size(mp)) + i = 0; + } + + return 0; +} + +int __init +xfs_extfree_intent_init_cache(void) +{ + xfs_extfree_item_cache = kmem_cache_create("xfs_extfree_intent", + sizeof(struct xfs_extent_free_item), + 0, 0, NULL); + + return xfs_extfree_item_cache != NULL ? 0 : -ENOMEM; +} + +void +xfs_extfree_intent_destroy_cache(void) +{ + kmem_cache_destroy(xfs_extfree_item_cache); + xfs_extfree_item_cache = NULL; +} diff --git a/fs/xfs/libxfs/xfs_alloc.h b/fs/xfs/libxfs/xfs_alloc.h new file mode 100644 index 000000000..50ef79a1e --- /dev/null +++ b/fs/xfs/libxfs/xfs_alloc.h @@ -0,0 +1,290 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_ALLOC_H__ +#define __XFS_ALLOC_H__ + +struct xfs_buf; +struct xfs_btree_cur; +struct xfs_mount; +struct xfs_perag; +struct xfs_trans; + +extern struct workqueue_struct *xfs_alloc_wq; + +unsigned int xfs_agfl_size(struct xfs_mount *mp); + +/* + * Flags for xfs_alloc_fix_freelist. + */ +#define XFS_ALLOC_FLAG_TRYLOCK (1U << 0) /* use trylock for buffer locking */ +#define XFS_ALLOC_FLAG_FREEING (1U << 1) /* indicate caller is freeing extents*/ +#define XFS_ALLOC_FLAG_NORMAP (1U << 2) /* don't modify the rmapbt */ +#define XFS_ALLOC_FLAG_NOSHRINK (1U << 3) /* don't shrink the freelist */ +#define XFS_ALLOC_FLAG_CHECK (1U << 4) /* test only, don't modify args */ +#define XFS_ALLOC_FLAG_TRYFLUSH (1U << 5) /* don't wait in busy extent flush */ + +/* + * Argument structure for xfs_alloc routines. + * This is turned into a structure to avoid having 20 arguments passed + * down several levels of the stack. + */ +typedef struct xfs_alloc_arg { + struct xfs_trans *tp; /* transaction pointer */ + struct xfs_mount *mp; /* file system mount point */ + struct xfs_buf *agbp; /* buffer for a.g. freelist header */ + struct xfs_perag *pag; /* per-ag struct for this agno */ + xfs_fsblock_t fsbno; /* file system block number */ + xfs_agnumber_t agno; /* allocation group number */ + xfs_agblock_t agbno; /* allocation group-relative block # */ + xfs_extlen_t minlen; /* minimum size of extent */ + xfs_extlen_t maxlen; /* maximum size of extent */ + xfs_extlen_t mod; /* mod value for extent size */ + xfs_extlen_t prod; /* prod value for extent size */ + xfs_extlen_t minleft; /* min blocks must be left after us */ + xfs_extlen_t total; /* total blocks needed in xaction */ + xfs_extlen_t alignment; /* align answer to multiple of this */ + xfs_extlen_t minalignslop; /* slop for minlen+alignment calcs */ + xfs_agblock_t min_agbno; /* set an agbno range for NEAR allocs */ + xfs_agblock_t max_agbno; /* ... */ + xfs_extlen_t len; /* output: actual size of extent */ + int datatype; /* mask defining data type treatment */ + char wasdel; /* set if allocation was prev delayed */ + char wasfromfl; /* set if allocation is from freelist */ + bool alloc_minlen_only; /* allocate exact minlen extent */ + struct xfs_owner_info oinfo; /* owner of blocks being allocated */ + enum xfs_ag_resv_type resv; /* block reservation to use */ +} xfs_alloc_arg_t; + +/* + * Defines for datatype + */ +#define XFS_ALLOC_USERDATA (1 << 0)/* allocation is for user data*/ +#define XFS_ALLOC_INITIAL_USER_DATA (1 << 1)/* special case start of file */ +#define XFS_ALLOC_NOBUSY (1 << 2)/* Busy extents not allowed */ + +/* freespace limit calculations */ +unsigned int xfs_alloc_set_aside(struct xfs_mount *mp); +unsigned int xfs_alloc_ag_max_usable(struct xfs_mount *mp); + +xfs_extlen_t xfs_alloc_longest_free_extent(struct xfs_perag *pag, + xfs_extlen_t need, xfs_extlen_t reserved); +unsigned int xfs_alloc_min_freelist(struct xfs_mount *mp, + struct xfs_perag *pag); +int xfs_alloc_get_freelist(struct xfs_perag *pag, struct xfs_trans *tp, + struct xfs_buf *agfbp, xfs_agblock_t *bnop, int btreeblk); +int xfs_alloc_put_freelist(struct xfs_perag *pag, struct xfs_trans *tp, + struct xfs_buf *agfbp, struct xfs_buf *agflbp, + xfs_agblock_t bno, int btreeblk); +int xfs_free_ag_extent(struct xfs_trans *tp, struct xfs_buf *agbp, + xfs_agblock_t bno, xfs_extlen_t len, + const struct xfs_owner_info *oinfo, enum xfs_ag_resv_type type); + +/* + * Compute and fill in value of m_alloc_maxlevels. + */ +void +xfs_alloc_compute_maxlevels( + struct xfs_mount *mp); /* file system mount structure */ + +/* + * Log the given fields from the agf structure. + */ +void +xfs_alloc_log_agf( + struct xfs_trans *tp, /* transaction pointer */ + struct xfs_buf *bp, /* buffer for a.g. freelist header */ + uint32_t fields);/* mask of fields to be logged (XFS_AGF_...) */ + +/* + * Allocate an extent anywhere in the specific AG given. If there is no + * space matching the requirements in that AG, then the allocation will fail. + */ +int xfs_alloc_vextent_this_ag(struct xfs_alloc_arg *args, xfs_agnumber_t agno); + +/* + * Allocate an extent as close to the target as possible. If there are not + * viable candidates in the AG, then fail the allocation. + */ +int xfs_alloc_vextent_near_bno(struct xfs_alloc_arg *args, + xfs_fsblock_t target); + +/* + * Allocate an extent exactly at the target given. If this is not possible + * then the allocation fails. + */ +int xfs_alloc_vextent_exact_bno(struct xfs_alloc_arg *args, + xfs_fsblock_t target); + +/* + * Best effort full filesystem allocation scan. + * + * Locality aware allocation will be attempted in the initial AG, but on failure + * non-localised attempts will be made. The AGs are constrained by previous + * allocations in the current transaction. Two passes will be made - the first + * non-blocking, the second blocking. + */ +int xfs_alloc_vextent_start_ag(struct xfs_alloc_arg *args, + xfs_fsblock_t target); + +/* + * Iterate from the AG indicated from args->fsbno through to the end of the + * filesystem attempting blocking allocation. This is for use in last + * resort allocation attempts when everything else has failed. + */ +int xfs_alloc_vextent_first_ag(struct xfs_alloc_arg *args, + xfs_fsblock_t target); + +/* + * Free an extent. + */ +int /* error */ +__xfs_free_extent( + struct xfs_trans *tp, /* transaction pointer */ + struct xfs_perag *pag, + xfs_agblock_t agbno, + xfs_extlen_t len, /* length of extent */ + const struct xfs_owner_info *oinfo, /* extent owner */ + enum xfs_ag_resv_type type, /* block reservation type */ + bool skip_discard); + +static inline int +xfs_free_extent( + struct xfs_trans *tp, + struct xfs_perag *pag, + xfs_agblock_t agbno, + xfs_extlen_t len, + const struct xfs_owner_info *oinfo, + enum xfs_ag_resv_type type) +{ + return __xfs_free_extent(tp, pag, agbno, len, oinfo, type, false); +} + +int /* error */ +xfs_alloc_lookup_le( + struct xfs_btree_cur *cur, /* btree cursor */ + xfs_agblock_t bno, /* starting block of extent */ + xfs_extlen_t len, /* length of extent */ + int *stat); /* success/failure */ + +int /* error */ +xfs_alloc_lookup_ge( + struct xfs_btree_cur *cur, /* btree cursor */ + xfs_agblock_t bno, /* starting block of extent */ + xfs_extlen_t len, /* length of extent */ + int *stat); /* success/failure */ + +int /* error */ +xfs_alloc_get_rec( + struct xfs_btree_cur *cur, /* btree cursor */ + xfs_agblock_t *bno, /* output: starting block of extent */ + xfs_extlen_t *len, /* output: length of extent */ + int *stat); /* output: success/failure */ + +union xfs_btree_rec; +void xfs_alloc_btrec_to_irec(const union xfs_btree_rec *rec, + struct xfs_alloc_rec_incore *irec); +xfs_failaddr_t xfs_alloc_check_irec(struct xfs_perag *pag, + const struct xfs_alloc_rec_incore *irec); + +int xfs_read_agf(struct xfs_perag *pag, struct xfs_trans *tp, int flags, + struct xfs_buf **agfbpp); +int xfs_alloc_read_agf(struct xfs_perag *pag, struct xfs_trans *tp, int flags, + struct xfs_buf **agfbpp); +int xfs_alloc_read_agfl(struct xfs_perag *pag, struct xfs_trans *tp, + struct xfs_buf **bpp); +int xfs_alloc_fix_freelist(struct xfs_alloc_arg *args, uint32_t alloc_flags); +int xfs_free_extent_fix_freelist(struct xfs_trans *tp, struct xfs_perag *pag, + struct xfs_buf **agbp); + +xfs_extlen_t xfs_prealloc_blocks(struct xfs_mount *mp); + +typedef int (*xfs_alloc_query_range_fn)( + struct xfs_btree_cur *cur, + const struct xfs_alloc_rec_incore *rec, + void *priv); + +int xfs_alloc_query_range(struct xfs_btree_cur *cur, + const struct xfs_alloc_rec_incore *low_rec, + const struct xfs_alloc_rec_incore *high_rec, + xfs_alloc_query_range_fn fn, void *priv); +int xfs_alloc_query_all(struct xfs_btree_cur *cur, xfs_alloc_query_range_fn fn, + void *priv); + +int xfs_alloc_has_records(struct xfs_btree_cur *cur, xfs_agblock_t bno, + xfs_extlen_t len, enum xbtree_recpacking *outcome); + +typedef int (*xfs_agfl_walk_fn)(struct xfs_mount *mp, xfs_agblock_t bno, + void *priv); +int xfs_agfl_walk(struct xfs_mount *mp, struct xfs_agf *agf, + struct xfs_buf *agflbp, xfs_agfl_walk_fn walk_fn, void *priv); + +static inline __be32 * +xfs_buf_to_agfl_bno( + struct xfs_buf *bp) +{ + if (xfs_has_crc(bp->b_mount)) + return bp->b_addr + sizeof(struct xfs_agfl); + return bp->b_addr; +} + +int xfs_free_extent_later(struct xfs_trans *tp, xfs_fsblock_t bno, + xfs_filblks_t len, const struct xfs_owner_info *oinfo, + enum xfs_ag_resv_type type, unsigned int free_flags); + +/* Don't issue a discard for the blocks freed. */ +#define XFS_FREE_EXTENT_SKIP_DISCARD (1U << 0) + +/* Free blocks on the realtime device. */ +#define XFS_FREE_EXTENT_REALTIME (1U << 1) + +#define XFS_FREE_EXTENT_ALL_FLAGS (XFS_FREE_EXTENT_SKIP_DISCARD | \ + XFS_FREE_EXTENT_REALTIME) + +/* + * List of extents to be free "later". + * The list is kept sorted on xbf_startblock. + */ +struct xfs_extent_free_item { + struct list_head xefi_list; + uint64_t xefi_owner; + xfs_fsblock_t xefi_startblock;/* starting fs block number */ + xfs_extlen_t xefi_blockcount;/* number of blocks in extent */ + struct xfs_group *xefi_group; + unsigned int xefi_flags; + enum xfs_ag_resv_type xefi_agresv; +}; + +#define XFS_EFI_SKIP_DISCARD (1U << 0) /* don't issue discard */ +#define XFS_EFI_ATTR_FORK (1U << 1) /* freeing attr fork block */ +#define XFS_EFI_BMBT_BLOCK (1U << 2) /* freeing bmap btree block */ +#define XFS_EFI_CANCELLED (1U << 3) /* dont actually free the space */ +#define XFS_EFI_REALTIME (1U << 4) /* freeing realtime extent */ + +static inline bool xfs_efi_is_realtime(const struct xfs_extent_free_item *xefi) +{ + return xefi->xefi_flags & XFS_EFI_REALTIME; +} + +struct xfs_alloc_autoreap { + struct xfs_defer_pending *dfp; +}; + +int xfs_alloc_schedule_autoreap(const struct xfs_alloc_arg *args, + unsigned int free_flags, struct xfs_alloc_autoreap *aarp); +void xfs_alloc_cancel_autoreap(struct xfs_trans *tp, + struct xfs_alloc_autoreap *aarp); +void xfs_alloc_commit_autoreap(struct xfs_trans *tp, + struct xfs_alloc_autoreap *aarp); + +extern struct kmem_cache *xfs_extfree_item_cache; + +int __init xfs_extfree_intent_init_cache(void); +void xfs_extfree_intent_destroy_cache(void); + +xfs_failaddr_t xfs_validate_ag_length(struct xfs_buf *bp, uint32_t seqno, + uint32_t length); + +#endif /* __XFS_ALLOC_H__ */ diff --git a/fs/xfs/libxfs/xfs_alloc_btree.c b/fs/xfs/libxfs/xfs_alloc_btree.c new file mode 100644 index 000000000..29f6ec1c3 --- /dev/null +++ b/fs/xfs/libxfs/xfs_alloc_btree.c @@ -0,0 +1,619 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2001,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_btree.h" +#include "xfs_btree_staging.h" +#include "xfs_alloc_btree.h" +#include "xfs_alloc.h" +#include "xfs_extent_busy.h" +#include "xfs_error.h" +#include "xfs_health.h" +#include "xfs_trace.h" +#include "xfs_trans.h" +#include "xfs_ag.h" + +static struct kmem_cache *xfs_allocbt_cur_cache; + +STATIC struct xfs_btree_cur * +xfs_bnobt_dup_cursor( + struct xfs_btree_cur *cur) +{ + return xfs_bnobt_init_cursor(cur->bc_mp, cur->bc_tp, cur->bc_ag.agbp, + to_perag(cur->bc_group)); +} + +STATIC struct xfs_btree_cur * +xfs_cntbt_dup_cursor( + struct xfs_btree_cur *cur) +{ + return xfs_cntbt_init_cursor(cur->bc_mp, cur->bc_tp, cur->bc_ag.agbp, + to_perag(cur->bc_group)); +} + +STATIC void +xfs_allocbt_set_root( + struct xfs_btree_cur *cur, + const union xfs_btree_ptr *ptr, + int inc) +{ + struct xfs_perag *pag = to_perag(cur->bc_group); + struct xfs_buf *agbp = cur->bc_ag.agbp; + struct xfs_agf *agf = agbp->b_addr; + + ASSERT(ptr->s != 0); + + if (xfs_btree_is_bno(cur->bc_ops)) { + agf->agf_bno_root = ptr->s; + be32_add_cpu(&agf->agf_bno_level, inc); + pag->pagf_bno_level += inc; + } else { + agf->agf_cnt_root = ptr->s; + be32_add_cpu(&agf->agf_cnt_level, inc); + pag->pagf_cnt_level += inc; + } + + xfs_alloc_log_agf(cur->bc_tp, agbp, XFS_AGF_ROOTS | XFS_AGF_LEVELS); +} + +STATIC int +xfs_allocbt_alloc_block( + struct xfs_btree_cur *cur, + const union xfs_btree_ptr *start, + union xfs_btree_ptr *new, + int *stat) +{ + int error; + xfs_agblock_t bno; + + /* Allocate the new block from the freelist. If we can't, give up. */ + error = xfs_alloc_get_freelist(to_perag(cur->bc_group), cur->bc_tp, + cur->bc_ag.agbp, &bno, 1); + if (error) + return error; + + if (bno == NULLAGBLOCK) { + *stat = 0; + return 0; + } + + atomic64_inc(&cur->bc_mp->m_allocbt_blks); + xfs_extent_busy_reuse(cur->bc_group, bno, 1, false); + + new->s = cpu_to_be32(bno); + + *stat = 1; + return 0; +} + +STATIC int +xfs_allocbt_free_block( + struct xfs_btree_cur *cur, + struct xfs_buf *bp) +{ + struct xfs_buf *agbp = cur->bc_ag.agbp; + xfs_agblock_t bno; + int error; + + bno = xfs_daddr_to_agbno(cur->bc_mp, xfs_buf_daddr(bp)); + error = xfs_alloc_put_freelist(to_perag(cur->bc_group), cur->bc_tp, + agbp, NULL, bno, 1); + if (error) + return error; + + atomic64_dec(&cur->bc_mp->m_allocbt_blks); + xfs_extent_busy_insert(cur->bc_tp, pag_group(agbp->b_pag), bno, 1, + XFS_EXTENT_BUSY_SKIP_DISCARD); + return 0; +} + +STATIC int +xfs_allocbt_get_minrecs( + struct xfs_btree_cur *cur, + int level) +{ + return cur->bc_mp->m_alloc_mnr[level != 0]; +} + +STATIC int +xfs_allocbt_get_maxrecs( + struct xfs_btree_cur *cur, + int level) +{ + return cur->bc_mp->m_alloc_mxr[level != 0]; +} + +STATIC void +xfs_allocbt_init_key_from_rec( + union xfs_btree_key *key, + const union xfs_btree_rec *rec) +{ + key->alloc.ar_startblock = rec->alloc.ar_startblock; + key->alloc.ar_blockcount = rec->alloc.ar_blockcount; +} + +STATIC void +xfs_bnobt_init_high_key_from_rec( + union xfs_btree_key *key, + const union xfs_btree_rec *rec) +{ + __u32 x; + + x = be32_to_cpu(rec->alloc.ar_startblock); + x += be32_to_cpu(rec->alloc.ar_blockcount) - 1; + key->alloc.ar_startblock = cpu_to_be32(x); + key->alloc.ar_blockcount = 0; +} + +STATIC void +xfs_cntbt_init_high_key_from_rec( + union xfs_btree_key *key, + const union xfs_btree_rec *rec) +{ + key->alloc.ar_blockcount = rec->alloc.ar_blockcount; + key->alloc.ar_startblock = 0; +} + +STATIC void +xfs_allocbt_init_rec_from_cur( + struct xfs_btree_cur *cur, + union xfs_btree_rec *rec) +{ + rec->alloc.ar_startblock = cpu_to_be32(cur->bc_rec.a.ar_startblock); + rec->alloc.ar_blockcount = cpu_to_be32(cur->bc_rec.a.ar_blockcount); +} + +STATIC void +xfs_allocbt_init_ptr_from_cur( + struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr) +{ + struct xfs_agf *agf = cur->bc_ag.agbp->b_addr; + + ASSERT(cur->bc_group->xg_gno == be32_to_cpu(agf->agf_seqno)); + + if (xfs_btree_is_bno(cur->bc_ops)) + ptr->s = agf->agf_bno_root; + else + ptr->s = agf->agf_cnt_root; +} + +STATIC int +xfs_bnobt_cmp_key_with_cur( + struct xfs_btree_cur *cur, + const union xfs_btree_key *key) +{ + struct xfs_alloc_rec_incore *rec = &cur->bc_rec.a; + const struct xfs_alloc_rec *kp = &key->alloc; + + return cmp_int(be32_to_cpu(kp->ar_startblock), + rec->ar_startblock); +} + +STATIC int +xfs_cntbt_cmp_key_with_cur( + struct xfs_btree_cur *cur, + const union xfs_btree_key *key) +{ + struct xfs_alloc_rec_incore *rec = &cur->bc_rec.a; + const struct xfs_alloc_rec *kp = &key->alloc; + + return cmp_int(be32_to_cpu(kp->ar_blockcount), rec->ar_blockcount) ?: + cmp_int(be32_to_cpu(kp->ar_startblock), rec->ar_startblock); +} + +STATIC int +xfs_bnobt_cmp_two_keys( + struct xfs_btree_cur *cur, + const union xfs_btree_key *k1, + const union xfs_btree_key *k2, + const union xfs_btree_key *mask) +{ + ASSERT(!mask || mask->alloc.ar_startblock); + + return cmp_int(be32_to_cpu(k1->alloc.ar_startblock), + be32_to_cpu(k2->alloc.ar_startblock)); +} + +STATIC int +xfs_cntbt_cmp_two_keys( + struct xfs_btree_cur *cur, + const union xfs_btree_key *k1, + const union xfs_btree_key *k2, + const union xfs_btree_key *mask) +{ + ASSERT(!mask || (mask->alloc.ar_blockcount && + mask->alloc.ar_startblock)); + + return cmp_int(be32_to_cpu(k1->alloc.ar_blockcount), + be32_to_cpu(k2->alloc.ar_blockcount)) ?: + cmp_int(be32_to_cpu(k1->alloc.ar_startblock), + be32_to_cpu(k2->alloc.ar_startblock)); +} + +static xfs_failaddr_t +xfs_allocbt_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_btree_block *block = XFS_BUF_TO_BLOCK(bp); + struct xfs_perag *pag = bp->b_pag; + xfs_failaddr_t fa; + unsigned int level; + + if (!xfs_verify_magic(bp, block->bb_magic)) + return __this_address; + + if (xfs_has_crc(mp)) { + fa = xfs_btree_agblock_v5hdr_verify(bp); + if (fa) + return fa; + } + + /* + * The perag may not be attached during grow operations or fully + * initialized from the AGF during log recovery. Therefore we can only + * check against maximum tree depth from those contexts. + * + * Otherwise check against the per-tree limit. Peek at one of the + * verifier magic values to determine the type of tree we're verifying + * against. + */ + level = be16_to_cpu(block->bb_level); + if (pag && xfs_perag_initialised_agf(pag)) { + unsigned int maxlevel, repair_maxlevel = 0; + + /* + * Online repair could be rewriting the free space btrees, so + * we'll validate against the larger of either tree while this + * is going on. + */ + if (bp->b_ops->magic[0] == cpu_to_be32(XFS_ABTC_MAGIC)) { + maxlevel = pag->pagf_cnt_level; +#ifdef CONFIG_XFS_ONLINE_REPAIR + repair_maxlevel = pag->pagf_repair_cnt_level; +#endif + } else { + maxlevel = pag->pagf_bno_level; +#ifdef CONFIG_XFS_ONLINE_REPAIR + repair_maxlevel = pag->pagf_repair_bno_level; +#endif + } + + if (level >= max(maxlevel, repair_maxlevel)) + return __this_address; + } else if (level >= mp->m_alloc_maxlevels) + return __this_address; + + return xfs_btree_agblock_verify(bp, mp->m_alloc_mxr[level != 0]); +} + +static void +xfs_allocbt_read_verify( + struct xfs_buf *bp) +{ + xfs_failaddr_t fa; + + if (!xfs_btree_agblock_verify_crc(bp)) + xfs_verifier_error(bp, -EFSBADCRC, __this_address); + else { + fa = xfs_allocbt_verify(bp); + if (fa) + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + } + + if (bp->b_error) + trace_xfs_btree_corrupt(bp, _RET_IP_); +} + +static void +xfs_allocbt_write_verify( + struct xfs_buf *bp) +{ + xfs_failaddr_t fa; + + fa = xfs_allocbt_verify(bp); + if (fa) { + trace_xfs_btree_corrupt(bp, _RET_IP_); + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + return; + } + xfs_btree_agblock_calc_crc(bp); + +} + +const struct xfs_buf_ops xfs_bnobt_buf_ops = { + .name = "xfs_bnobt", + .magic = { cpu_to_be32(XFS_ABTB_MAGIC), + cpu_to_be32(XFS_ABTB_CRC_MAGIC) }, + .verify_read = xfs_allocbt_read_verify, + .verify_write = xfs_allocbt_write_verify, + .verify_struct = xfs_allocbt_verify, +}; + +const struct xfs_buf_ops xfs_cntbt_buf_ops = { + .name = "xfs_cntbt", + .magic = { cpu_to_be32(XFS_ABTC_MAGIC), + cpu_to_be32(XFS_ABTC_CRC_MAGIC) }, + .verify_read = xfs_allocbt_read_verify, + .verify_write = xfs_allocbt_write_verify, + .verify_struct = xfs_allocbt_verify, +}; + +STATIC int +xfs_bnobt_keys_inorder( + struct xfs_btree_cur *cur, + const union xfs_btree_key *k1, + const union xfs_btree_key *k2) +{ + return be32_to_cpu(k1->alloc.ar_startblock) < + be32_to_cpu(k2->alloc.ar_startblock); +} + +STATIC int +xfs_bnobt_recs_inorder( + struct xfs_btree_cur *cur, + const union xfs_btree_rec *r1, + const union xfs_btree_rec *r2) +{ + return be32_to_cpu(r1->alloc.ar_startblock) + + be32_to_cpu(r1->alloc.ar_blockcount) <= + be32_to_cpu(r2->alloc.ar_startblock); +} + +STATIC int +xfs_cntbt_keys_inorder( + struct xfs_btree_cur *cur, + const union xfs_btree_key *k1, + const union xfs_btree_key *k2) +{ + return be32_to_cpu(k1->alloc.ar_blockcount) < + be32_to_cpu(k2->alloc.ar_blockcount) || + (k1->alloc.ar_blockcount == k2->alloc.ar_blockcount && + be32_to_cpu(k1->alloc.ar_startblock) < + be32_to_cpu(k2->alloc.ar_startblock)); +} + +STATIC int +xfs_cntbt_recs_inorder( + struct xfs_btree_cur *cur, + const union xfs_btree_rec *r1, + const union xfs_btree_rec *r2) +{ + return be32_to_cpu(r1->alloc.ar_blockcount) < + be32_to_cpu(r2->alloc.ar_blockcount) || + (r1->alloc.ar_blockcount == r2->alloc.ar_blockcount && + be32_to_cpu(r1->alloc.ar_startblock) < + be32_to_cpu(r2->alloc.ar_startblock)); +} + +STATIC enum xbtree_key_contig +xfs_allocbt_keys_contiguous( + struct xfs_btree_cur *cur, + const union xfs_btree_key *key1, + const union xfs_btree_key *key2, + const union xfs_btree_key *mask) +{ + ASSERT(!mask || mask->alloc.ar_startblock); + + return xbtree_key_contig(be32_to_cpu(key1->alloc.ar_startblock), + be32_to_cpu(key2->alloc.ar_startblock)); +} + +const struct xfs_btree_ops xfs_bnobt_ops = { + .name = "bno", + .type = XFS_BTREE_TYPE_AG, + + .rec_len = sizeof(xfs_alloc_rec_t), + .key_len = sizeof(xfs_alloc_key_t), + .ptr_len = XFS_BTREE_SHORT_PTR_LEN, + + .lru_refs = XFS_ALLOC_BTREE_REF, + .statoff = XFS_STATS_CALC_INDEX(xs_abtb_2), + .sick_mask = XFS_SICK_AG_BNOBT, + + .dup_cursor = xfs_bnobt_dup_cursor, + .set_root = xfs_allocbt_set_root, + .alloc_block = xfs_allocbt_alloc_block, + .free_block = xfs_allocbt_free_block, + .get_minrecs = xfs_allocbt_get_minrecs, + .get_maxrecs = xfs_allocbt_get_maxrecs, + .init_key_from_rec = xfs_allocbt_init_key_from_rec, + .init_high_key_from_rec = xfs_bnobt_init_high_key_from_rec, + .init_rec_from_cur = xfs_allocbt_init_rec_from_cur, + .init_ptr_from_cur = xfs_allocbt_init_ptr_from_cur, + .cmp_key_with_cur = xfs_bnobt_cmp_key_with_cur, + .buf_ops = &xfs_bnobt_buf_ops, + .cmp_two_keys = xfs_bnobt_cmp_two_keys, + .keys_inorder = xfs_bnobt_keys_inorder, + .recs_inorder = xfs_bnobt_recs_inorder, + .keys_contiguous = xfs_allocbt_keys_contiguous, +}; + +const struct xfs_btree_ops xfs_cntbt_ops = { + .name = "cnt", + .type = XFS_BTREE_TYPE_AG, + + .rec_len = sizeof(xfs_alloc_rec_t), + .key_len = sizeof(xfs_alloc_key_t), + .ptr_len = XFS_BTREE_SHORT_PTR_LEN, + + .lru_refs = XFS_ALLOC_BTREE_REF, + .statoff = XFS_STATS_CALC_INDEX(xs_abtc_2), + .sick_mask = XFS_SICK_AG_CNTBT, + + .dup_cursor = xfs_cntbt_dup_cursor, + .set_root = xfs_allocbt_set_root, + .alloc_block = xfs_allocbt_alloc_block, + .free_block = xfs_allocbt_free_block, + .get_minrecs = xfs_allocbt_get_minrecs, + .get_maxrecs = xfs_allocbt_get_maxrecs, + .init_key_from_rec = xfs_allocbt_init_key_from_rec, + .init_high_key_from_rec = xfs_cntbt_init_high_key_from_rec, + .init_rec_from_cur = xfs_allocbt_init_rec_from_cur, + .init_ptr_from_cur = xfs_allocbt_init_ptr_from_cur, + .cmp_key_with_cur = xfs_cntbt_cmp_key_with_cur, + .buf_ops = &xfs_cntbt_buf_ops, + .cmp_two_keys = xfs_cntbt_cmp_two_keys, + .keys_inorder = xfs_cntbt_keys_inorder, + .recs_inorder = xfs_cntbt_recs_inorder, + .keys_contiguous = NULL, /* not needed right now */ +}; + +/* + * Allocate a new bnobt cursor. + * + * For staging cursors tp and agbp are NULL. + */ +struct xfs_btree_cur * +xfs_bnobt_init_cursor( + struct xfs_mount *mp, + struct xfs_trans *tp, + struct xfs_buf *agbp, + struct xfs_perag *pag) +{ + struct xfs_btree_cur *cur; + + cur = xfs_btree_alloc_cursor(mp, tp, &xfs_bnobt_ops, + mp->m_alloc_maxlevels, xfs_allocbt_cur_cache); + cur->bc_group = xfs_group_hold(pag_group(pag)); + cur->bc_ag.agbp = agbp; + if (agbp) { + struct xfs_agf *agf = agbp->b_addr; + + cur->bc_nlevels = be32_to_cpu(agf->agf_bno_level); + } + return cur; +} + +/* + * Allocate a new cntbt cursor. + * + * For staging cursors tp and agbp are NULL. + */ +struct xfs_btree_cur * +xfs_cntbt_init_cursor( + struct xfs_mount *mp, + struct xfs_trans *tp, + struct xfs_buf *agbp, + struct xfs_perag *pag) +{ + struct xfs_btree_cur *cur; + + cur = xfs_btree_alloc_cursor(mp, tp, &xfs_cntbt_ops, + mp->m_alloc_maxlevels, xfs_allocbt_cur_cache); + cur->bc_group = xfs_group_hold(pag_group(pag)); + cur->bc_ag.agbp = agbp; + if (agbp) { + struct xfs_agf *agf = agbp->b_addr; + + cur->bc_nlevels = be32_to_cpu(agf->agf_cnt_level); + } + return cur; +} + +/* + * Install a new free space btree root. Caller is responsible for invalidating + * and freeing the old btree blocks. + */ +void +xfs_allocbt_commit_staged_btree( + struct xfs_btree_cur *cur, + struct xfs_trans *tp, + struct xfs_buf *agbp) +{ + struct xfs_agf *agf = agbp->b_addr; + struct xbtree_afakeroot *afake = cur->bc_ag.afake; + + ASSERT(cur->bc_flags & XFS_BTREE_STAGING); + + if (xfs_btree_is_bno(cur->bc_ops)) { + agf->agf_bno_root = cpu_to_be32(afake->af_root); + agf->agf_bno_level = cpu_to_be32(afake->af_levels); + } else { + agf->agf_cnt_root = cpu_to_be32(afake->af_root); + agf->agf_cnt_level = cpu_to_be32(afake->af_levels); + } + xfs_alloc_log_agf(tp, agbp, XFS_AGF_ROOTS | XFS_AGF_LEVELS); + + xfs_btree_commit_afakeroot(cur, tp, agbp); +} + +/* Calculate number of records in an alloc btree block. */ +static inline unsigned int +xfs_allocbt_block_maxrecs( + unsigned int blocklen, + bool leaf) +{ + if (leaf) + return blocklen / sizeof(xfs_alloc_rec_t); + return blocklen / (sizeof(xfs_alloc_key_t) + sizeof(xfs_alloc_ptr_t)); +} + +/* + * Calculate number of records in an alloc btree block. + */ +unsigned int +xfs_allocbt_maxrecs( + struct xfs_mount *mp, + unsigned int blocklen, + bool leaf) +{ + blocklen -= XFS_ALLOC_BLOCK_LEN(mp); + return xfs_allocbt_block_maxrecs(blocklen, leaf); +} + +/* Free space btrees are at their largest when every other block is free. */ +#define XFS_MAX_FREESP_RECORDS ((XFS_MAX_AG_BLOCKS + 1) / 2) + +/* Compute the max possible height for free space btrees. */ +unsigned int +xfs_allocbt_maxlevels_ondisk(void) +{ + unsigned int minrecs[2]; + unsigned int blocklen; + + blocklen = min(XFS_MIN_BLOCKSIZE - XFS_BTREE_SBLOCK_LEN, + XFS_MIN_CRC_BLOCKSIZE - XFS_BTREE_SBLOCK_CRC_LEN); + + minrecs[0] = xfs_allocbt_block_maxrecs(blocklen, true) / 2; + minrecs[1] = xfs_allocbt_block_maxrecs(blocklen, false) / 2; + + return xfs_btree_compute_maxlevels(minrecs, XFS_MAX_FREESP_RECORDS); +} + +/* Calculate the freespace btree size for some records. */ +xfs_extlen_t +xfs_allocbt_calc_size( + struct xfs_mount *mp, + unsigned long long len) +{ + return xfs_btree_calc_size(mp->m_alloc_mnr, len); +} + +int __init +xfs_allocbt_init_cur_cache(void) +{ + xfs_allocbt_cur_cache = kmem_cache_create("xfs_bnobt_cur", + xfs_btree_cur_sizeof(xfs_allocbt_maxlevels_ondisk()), + 0, 0, NULL); + + if (!xfs_allocbt_cur_cache) + return -ENOMEM; + return 0; +} + +void +xfs_allocbt_destroy_cur_cache(void) +{ + kmem_cache_destroy(xfs_allocbt_cur_cache); + xfs_allocbt_cur_cache = NULL; +} diff --git a/fs/xfs/libxfs/xfs_alloc_btree.h b/fs/xfs/libxfs/xfs_alloc_btree.h new file mode 100644 index 000000000..12647f9aa --- /dev/null +++ b/fs/xfs/libxfs/xfs_alloc_btree.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2000,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_ALLOC_BTREE_H__ +#define __XFS_ALLOC_BTREE_H__ + +/* + * Freespace on-disk structures + */ + +struct xfs_buf; +struct xfs_btree_cur; +struct xfs_mount; +struct xfs_perag; +struct xbtree_afakeroot; + +/* + * Btree block header size depends on a superblock flag. + */ +#define XFS_ALLOC_BLOCK_LEN(mp) \ + (xfs_has_crc(((mp))) ? \ + XFS_BTREE_SBLOCK_CRC_LEN : XFS_BTREE_SBLOCK_LEN) + +/* + * Record, key, and pointer address macros for btree blocks. + * + * (note that some of these may appear unused, but they are used in userspace) + */ +#define XFS_ALLOC_REC_ADDR(mp, block, index) \ + ((xfs_alloc_rec_t *) \ + ((char *)(block) + \ + XFS_ALLOC_BLOCK_LEN(mp) + \ + (((index) - 1) * sizeof(xfs_alloc_rec_t)))) + +#define XFS_ALLOC_KEY_ADDR(mp, block, index) \ + ((xfs_alloc_key_t *) \ + ((char *)(block) + \ + XFS_ALLOC_BLOCK_LEN(mp) + \ + ((index) - 1) * sizeof(xfs_alloc_key_t))) + +#define XFS_ALLOC_PTR_ADDR(mp, block, index, maxrecs) \ + ((xfs_alloc_ptr_t *) \ + ((char *)(block) + \ + XFS_ALLOC_BLOCK_LEN(mp) + \ + (maxrecs) * sizeof(xfs_alloc_key_t) + \ + ((index) - 1) * sizeof(xfs_alloc_ptr_t))) + +struct xfs_btree_cur *xfs_bnobt_init_cursor(struct xfs_mount *mp, + struct xfs_trans *tp, struct xfs_buf *bp, + struct xfs_perag *pag); +struct xfs_btree_cur *xfs_cntbt_init_cursor(struct xfs_mount *mp, + struct xfs_trans *tp, struct xfs_buf *bp, + struct xfs_perag *pag); +unsigned int xfs_allocbt_maxrecs(struct xfs_mount *mp, unsigned int blocklen, + bool leaf); +extern xfs_extlen_t xfs_allocbt_calc_size(struct xfs_mount *mp, + unsigned long long len); + +void xfs_allocbt_commit_staged_btree(struct xfs_btree_cur *cur, + struct xfs_trans *tp, struct xfs_buf *agbp); + +unsigned int xfs_allocbt_maxlevels_ondisk(void); + +int __init xfs_allocbt_init_cur_cache(void); +void xfs_allocbt_destroy_cur_cache(void); + +#endif /* __XFS_ALLOC_BTREE_H__ */ diff --git a/fs/xfs/libxfs/xfs_attr.c b/fs/xfs/libxfs/xfs_attr.c new file mode 100644 index 000000000..b3f7b2c34 --- /dev/null +++ b/fs/xfs/libxfs/xfs_attr.c @@ -0,0 +1,1661 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_attr_sf.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_bmap.h" +#include "xfs_bmap_btree.h" +#include "xfs_attr.h" +#include "xfs_attr_leaf.h" +#include "xfs_attr_remote.h" +#include "xfs_quota.h" +#include "xfs_trans_space.h" +#include "xfs_trace.h" +#include "xfs_attr_item.h" +#include "xfs_xattr.h" +#include "xfs_parent.h" + +struct kmem_cache *xfs_attr_intent_cache; + +/* + * xfs_attr.c + * + * Provide the external interfaces to manage attribute lists. + */ + +/*======================================================================== + * Function prototypes for the kernel. + *========================================================================*/ + +/* + * Internal routines when attribute list fits inside the inode. + */ +STATIC int xfs_attr_shortform_addname(xfs_da_args_t *args); + +/* + * Internal routines when attribute list is one block. + */ +STATIC int xfs_attr_leaf_get(xfs_da_args_t *args); +STATIC int xfs_attr_leaf_removename(xfs_da_args_t *args); + +/* + * Internal routines when attribute list is more than one block. + */ +STATIC int xfs_attr_node_get(xfs_da_args_t *args); +STATIC void xfs_attr_restore_rmt_blk(struct xfs_da_args *args); +static int xfs_attr_node_try_addname(struct xfs_attr_intent *attr); +STATIC int xfs_attr_node_addname_find_attr(struct xfs_attr_intent *attr); +STATIC int xfs_attr_node_remove_attr(struct xfs_attr_intent *attr); +STATIC int xfs_attr_node_lookup(struct xfs_da_args *args, + struct xfs_da_state *state); + +int +xfs_inode_hasattr( + struct xfs_inode *ip) +{ + if (!xfs_inode_has_attr_fork(ip)) + return 0; + if (ip->i_af.if_format == XFS_DINODE_FMT_EXTENTS && + ip->i_af.if_nextents == 0) + return 0; + return 1; +} + +/* + * Returns true if the there is exactly only block in the attr fork, in which + * case the attribute fork consists of a single leaf block entry. + */ +bool +xfs_attr_is_leaf( + struct xfs_inode *ip) +{ + struct xfs_ifork *ifp = &ip->i_af; + struct xfs_iext_cursor icur; + struct xfs_bmbt_irec imap; + + ASSERT(!xfs_need_iread_extents(ifp)); + + if (ifp->if_nextents != 1 || ifp->if_format != XFS_DINODE_FMT_EXTENTS) + return false; + + xfs_iext_first(ifp, &icur); + xfs_iext_get_extent(ifp, &icur, &imap); + return imap.br_startoff == 0 && imap.br_blockcount == 1; +} + +/* + * XXX (dchinner): name path state saving and refilling is an optimisation to + * avoid needing to look up name entries after rolling transactions removing + * remote xattr blocks between the name entry lookup and name entry removal. + * This optimisation got sidelined when combining the set and remove state + * machines, but the code has been left in place because it is worthwhile to + * restore the optimisation once the combined state machine paths have settled. + * + * This comment is a public service announcement to remind Future Dave that he + * still needs to restore this code to working order. + */ +#if 0 +/* + * Fill in the disk block numbers in the state structure for the buffers + * that are attached to the state structure. + * This is done so that we can quickly reattach ourselves to those buffers + * after some set of transaction commits have released these buffers. + */ +static int +xfs_attr_fillstate(xfs_da_state_t *state) +{ + xfs_da_state_path_t *path; + xfs_da_state_blk_t *blk; + int level; + + trace_xfs_attr_fillstate(state->args); + + /* + * Roll down the "path" in the state structure, storing the on-disk + * block number for those buffers in the "path". + */ + path = &state->path; + ASSERT((path->active >= 0) && (path->active < XFS_DA_NODE_MAXDEPTH)); + for (blk = path->blk, level = 0; level < path->active; blk++, level++) { + if (blk->bp) { + blk->disk_blkno = xfs_buf_daddr(blk->bp); + blk->bp = NULL; + } else { + blk->disk_blkno = 0; + } + } + + /* + * Roll down the "altpath" in the state structure, storing the on-disk + * block number for those buffers in the "altpath". + */ + path = &state->altpath; + ASSERT((path->active >= 0) && (path->active < XFS_DA_NODE_MAXDEPTH)); + for (blk = path->blk, level = 0; level < path->active; blk++, level++) { + if (blk->bp) { + blk->disk_blkno = xfs_buf_daddr(blk->bp); + blk->bp = NULL; + } else { + blk->disk_blkno = 0; + } + } + + return 0; +} + +/* + * Reattach the buffers to the state structure based on the disk block + * numbers stored in the state structure. + * This is done after some set of transaction commits have released those + * buffers from our grip. + */ +static int +xfs_attr_refillstate(xfs_da_state_t *state) +{ + xfs_da_state_path_t *path; + xfs_da_state_blk_t *blk; + int level, error; + + trace_xfs_attr_refillstate(state->args); + + /* + * Roll down the "path" in the state structure, storing the on-disk + * block number for those buffers in the "path". + */ + path = &state->path; + ASSERT((path->active >= 0) && (path->active < XFS_DA_NODE_MAXDEPTH)); + for (blk = path->blk, level = 0; level < path->active; blk++, level++) { + if (blk->disk_blkno) { + error = xfs_da3_node_read_mapped(state->args->trans, + state->args->dp, blk->disk_blkno, + &blk->bp, XFS_ATTR_FORK); + if (error) + return error; + } else { + blk->bp = NULL; + } + } + + /* + * Roll down the "altpath" in the state structure, storing the on-disk + * block number for those buffers in the "altpath". + */ + path = &state->altpath; + ASSERT((path->active >= 0) && (path->active < XFS_DA_NODE_MAXDEPTH)); + for (blk = path->blk, level = 0; level < path->active; blk++, level++) { + if (blk->disk_blkno) { + error = xfs_da3_node_read_mapped(state->args->trans, + state->args->dp, blk->disk_blkno, + &blk->bp, XFS_ATTR_FORK); + if (error) + return error; + } else { + blk->bp = NULL; + } + } + + return 0; +} +#else +static int xfs_attr_fillstate(xfs_da_state_t *state) { return 0; } +#endif + +/*======================================================================== + * Overall external interface routines. + *========================================================================*/ + +/* + * Retrieve an extended attribute and its value. Must have ilock. + * Returns 0 on successful retrieval, otherwise an error. + */ +int +xfs_attr_get_ilocked( + struct xfs_da_args *args) +{ + int error; + + xfs_assert_ilocked(args->dp, XFS_ILOCK_SHARED | XFS_ILOCK_EXCL); + + if (!xfs_inode_hasattr(args->dp)) + return -ENOATTR; + + /* + * The incore attr fork iext tree must be loaded for xfs_attr_is_leaf + * to work correctly. + */ + error = xfs_iread_extents(args->trans, args->dp, XFS_ATTR_FORK); + if (error) + return error; + + if (args->dp->i_af.if_format == XFS_DINODE_FMT_LOCAL) + return xfs_attr_shortform_getvalue(args); + if (xfs_attr_is_leaf(args->dp)) + return xfs_attr_leaf_get(args); + return xfs_attr_node_get(args); +} + +/* + * Retrieve an extended attribute by name, and its value if requested. + * + * If args->valuelen is zero, then the caller does not want the value, just an + * indication whether the attribute exists and the size of the value if it + * exists. The size is returned in args.valuelen. + * + * If args->value is NULL but args->valuelen is non-zero, allocate the buffer + * for the value after existence of the attribute has been determined. The + * caller always has to free args->value if it is set, no matter if this + * function was successful or not. + * + * If the attribute is found, but exceeds the size limit set by the caller in + * args->valuelen, return -ERANGE with the size of the attribute that was found + * in args->valuelen. + */ +int +xfs_attr_get( + struct xfs_da_args *args) +{ + uint lock_mode; + int error; + + XFS_STATS_INC(args->dp->i_mount, xs_attr_get); + + if (xfs_is_shutdown(args->dp->i_mount)) + return -EIO; + + if (!args->owner) + args->owner = I_INO(args->dp); + args->geo = args->dp->i_mount->m_attr_geo; + args->whichfork = XFS_ATTR_FORK; + xfs_attr_sethash(args); + + /* Entirely possible to look up a name which doesn't exist */ + args->op_flags = XFS_DA_OP_OKNOENT; + + lock_mode = xfs_ilock_attr_map_shared(args->dp); + error = xfs_attr_get_ilocked(args); + xfs_iunlock(args->dp, lock_mode); + + return error; +} + +/* + * Calculate how many blocks we need for the new attribute, + */ +int +xfs_attr_calc_size( + struct xfs_da_args *args, + int *local) +{ + struct xfs_mount *mp = args->dp->i_mount; + int size; + int nblks; + + /* + * Determine space new attribute will use, and if it would be + * "local" or "remote" (note: local != inline). + */ + size = xfs_attr_leaf_newentsize(args, local); + nblks = XFS_DAENTER_SPACE_RES(mp, XFS_ATTR_FORK); + if (*local) { + if (size > (args->geo->blksize / 2)) { + /* Double split possible */ + nblks *= 2; + } + } else { + /* + * Out of line attribute, cannot double split, but + * make room for the attribute value itself. + */ + uint dblocks = xfs_attr3_rmt_blocks(mp, args->valuelen); + nblks += dblocks; + nblks += XFS_NEXTENTADD_SPACE_RES(mp, dblocks, XFS_ATTR_FORK); + } + + return nblks; +} + +/* Initialize transaction reservation for an xattr set/replace/upsert */ +inline struct xfs_trans_res +xfs_attr_set_resv( + const struct xfs_da_args *args) +{ + struct xfs_mount *mp = args->dp->i_mount; + struct xfs_trans_res ret = { + .tr_logres = M_RES(mp)->tr_attrsetm.tr_logres + + M_RES(mp)->tr_attrsetrt.tr_logres * args->total, + .tr_logcount = XFS_ATTRSET_LOG_COUNT, + .tr_logflags = XFS_TRANS_PERM_LOG_RES, + }; + + return ret; +} + +/* + * Add an attr to a shortform fork. If there is no space, + * xfs_attr_shortform_addname() will convert to leaf format and return -ENOSPC. + * to use. + */ +STATIC int +xfs_attr_try_sf_addname( + struct xfs_da_args *args) +{ + int error; + + /* + * Build initial attribute list (if required). + */ + if (args->dp->i_af.if_format == XFS_DINODE_FMT_EXTENTS) + xfs_attr_shortform_create(args); + + error = xfs_attr_shortform_addname(args); + if (error == -ENOSPC) + return error; + + /* + * Commit the shortform mods, and we're done. + * NOTE: this is also the error path (EEXIST, etc). + */ + if (!error) + xfs_trans_ichgtime(args->trans, args->dp, XFS_ICHGTIME_CHG); + + if (xfs_has_wsync(args->dp->i_mount)) + xfs_trans_set_sync(args->trans); + + return error; +} + +static int +xfs_attr_sf_addname( + struct xfs_attr_intent *attr) +{ + struct xfs_da_args *args = attr->xattri_da_args; + int error = 0; + + error = xfs_attr_try_sf_addname(args); + if (error != -ENOSPC) { + ASSERT(!error || error == -EEXIST); + attr->xattri_dela_state = XFS_DAS_DONE; + goto out; + } + + /* + * It won't fit in the shortform, transform to a leaf block. GROT: + * another possible req'mt for a double-split btree op. + */ + error = xfs_attr_shortform_to_leaf(args); + if (error) + return error; + + attr->xattri_dela_state = XFS_DAS_LEAF_ADD; +out: + trace_xfs_attr_sf_addname_return(attr->xattri_dela_state, args->dp); + return error; +} + +/* Compute the hash value for a user/root/secure extended attribute */ +xfs_dahash_t +xfs_attr_hashname( + const uint8_t *name, + int namelen) +{ + return xfs_da_hashname(name, namelen); +} + +/* Compute the hash value for any extended attribute from any namespace. */ +xfs_dahash_t +xfs_attr_hashval( + struct xfs_mount *mp, + unsigned int attr_flags, + const uint8_t *name, + int namelen, + const void *value, + int valuelen) +{ + ASSERT(xfs_attr_check_namespace(attr_flags)); + + if (attr_flags & XFS_ATTR_PARENT) + return xfs_parent_hashattr(mp, name, namelen, value, valuelen); + + return xfs_attr_hashname(name, namelen); +} + +/* Save the current remote block info and clear the current pointers. */ +static void +xfs_attr_save_rmt_blk( + struct xfs_da_args *args) +{ + args->blkno2 = args->blkno; + args->index2 = args->index; + args->rmtblkno2 = args->rmtblkno; + args->rmtblkcnt2 = args->rmtblkcnt; + args->rmtvaluelen2 = args->rmtvaluelen; + args->rmtblkno = 0; + args->rmtblkcnt = 0; + args->rmtvaluelen = 0; +} + +/* Set stored info about a remote block */ +static void +xfs_attr_restore_rmt_blk( + struct xfs_da_args *args) +{ + args->blkno = args->blkno2; + args->index = args->index2; + args->rmtblkno = args->rmtblkno2; + args->rmtblkcnt = args->rmtblkcnt2; + args->rmtvaluelen = args->rmtvaluelen2; +} + +/* + * PPTR_REPLACE operations require the caller to set the old and new names and + * values explicitly. Update the canonical fields to the new name and value + * here now that the removal phase has finished. + */ +static void +xfs_attr_update_pptr_replace_args( + struct xfs_da_args *args) +{ + ASSERT(args->new_namelen > 0); + args->name = args->new_name; + args->namelen = args->new_namelen; + args->value = args->new_value; + args->valuelen = args->new_valuelen; + xfs_attr_sethash(args); +} + +/* + * Handle the state change on completion of a multi-state attr operation. + * + * If the XFS_DA_OP_REPLACE flag is set, this means the operation was the first + * modification in a attr replace operation and we still have to do the second + * state, indicated by @replace_state. + * + * We consume the XFS_DA_OP_REPLACE flag so that when we are called again on + * completion of the second half of the attr replace operation we correctly + * signal that it is done. + */ +static enum xfs_delattr_state +xfs_attr_complete_op( + struct xfs_attr_intent *attr, + enum xfs_delattr_state replace_state) +{ + struct xfs_da_args *args = attr->xattri_da_args; + + if (!(args->op_flags & XFS_DA_OP_REPLACE)) + replace_state = XFS_DAS_DONE; + else if (xfs_attr_intent_op(attr) == XFS_ATTRI_OP_FLAGS_PPTR_REPLACE) + xfs_attr_update_pptr_replace_args(args); + + args->op_flags &= ~XFS_DA_OP_REPLACE; + args->attr_filter &= ~XFS_ATTR_INCOMPLETE; + return replace_state; +} + +/* + * Try to add an attribute to an inode in leaf form. + */ +static int +xfs_attr_leaf_addname( + struct xfs_attr_intent *attr) +{ + struct xfs_da_args *args = attr->xattri_da_args; + struct xfs_buf *bp; + int error; + + ASSERT(xfs_attr_is_leaf(args->dp)); + + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, 0, &bp); + if (error) + return error; + + /* + * Look up the xattr name to set the insertion point for the new xattr. + */ + error = xfs_attr3_leaf_lookup_int(bp, args); + switch (error) { + case -ENOATTR: + if (args->op_flags & XFS_DA_OP_REPLACE) + goto out_brelse; + break; + case -EEXIST: + if (!(args->op_flags & XFS_DA_OP_REPLACE)) + goto out_brelse; + + trace_xfs_attr_leaf_replace(args); + /* + * Save the existing remote attr state so that the current + * values reflect the state of the new attribute we are about to + * add, not the attribute we just found and will remove later. + */ + xfs_attr_save_rmt_blk(args); + break; + case 0: + break; + default: + goto out_brelse; + } + + /* + * We need to commit and roll if we need to allocate remote xattr blocks + * or perform more xattr manipulations. Otherwise there is nothing more + * to do and we can return success. + */ + if (!xfs_attr3_leaf_add(bp, args)) { + error = xfs_attr3_leaf_to_node(args); + if (error) + return error; + + attr->xattri_dela_state = XFS_DAS_NODE_ADD; + } else if (args->rmtblkno) { + attr->xattri_dela_state = XFS_DAS_LEAF_SET_RMT; + } else { + attr->xattri_dela_state = + xfs_attr_complete_op(attr, XFS_DAS_LEAF_REPLACE); + } + + trace_xfs_attr_leaf_addname_return(attr->xattri_dela_state, args->dp); + return 0; + +out_brelse: + xfs_trans_brelse(args->trans, bp); + return error; +} + +/* + * Add an entry to a node format attr tree. + * + * Note that we might still have a leaf here - xfs_attr_is_leaf() cannot tell + * the difference between leaf + remote attr blocks and a node format tree, + * so we may still end up having to convert from leaf to node format here. + */ +static int +xfs_attr_node_addname( + struct xfs_attr_intent *attr) +{ + struct xfs_da_args *args = attr->xattri_da_args; + int error; + + error = xfs_attr_node_addname_find_attr(attr); + if (error) + return error; + + error = xfs_attr_node_try_addname(attr); + if (error == 1) { + error = xfs_attr3_leaf_to_node(args); + if (error) + return error; + /* + * No state change, we really are in node form now + * but we need the transaction rolled to continue. + */ + goto out; + } + if (error) + return error; + + if (args->rmtblkno) + attr->xattri_dela_state = XFS_DAS_NODE_SET_RMT; + else + attr->xattri_dela_state = xfs_attr_complete_op(attr, + XFS_DAS_NODE_REPLACE); +out: + trace_xfs_attr_node_addname_return(attr->xattri_dela_state, args->dp); + return error; +} + +static int +xfs_attr_rmtval_alloc( + struct xfs_attr_intent *attr) +{ + struct xfs_da_args *args = attr->xattri_da_args; + int error = 0; + + /* + * If there was an out-of-line value, allocate the blocks we + * identified for its storage and copy the value. This is done + * after we create the attribute so that we don't overflow the + * maximum size of a transaction and/or hit a deadlock. + */ + if (attr->xattri_blkcnt > 0) { + error = xfs_attr_rmtval_set_blk(attr); + if (error) + return error; + /* Roll the transaction only if there is more to allocate. */ + if (attr->xattri_blkcnt > 0) + goto out; + } + + error = xfs_attr_rmtval_set_value(args); + if (error) + return error; + + attr->xattri_dela_state = xfs_attr_complete_op(attr, + ++attr->xattri_dela_state); + /* + * If we are not doing a rename, we've finished the operation but still + * have to clear the incomplete flag protecting the new attr from + * exposing partially initialised state if we crash during creation. + */ + if (attr->xattri_dela_state == XFS_DAS_DONE) + error = xfs_attr3_leaf_clearflag(args); +out: + trace_xfs_attr_rmtval_alloc(attr->xattri_dela_state, args->dp); + return error; +} + +/* + * Mark an attribute entry INCOMPLETE and save pointers to the relevant buffers + * for later deletion of the entry. + */ +static int +xfs_attr_leaf_mark_incomplete( + struct xfs_da_args *args, + struct xfs_da_state *state) +{ + int error; + + /* + * Fill in disk block numbers in the state structure + * so that we can get the buffers back after we commit + * several transactions in the following calls. + */ + error = xfs_attr_fillstate(state); + if (error) + return error; + + /* + * Mark the attribute as INCOMPLETE + */ + return xfs_attr3_leaf_setflag(args); +} + +/* Ensure the da state of an xattr deferred work item is ready to go. */ +static inline void +xfs_attr_item_init_da_state( + struct xfs_attr_intent *attr) +{ + struct xfs_da_args *args = attr->xattri_da_args; + + if (!attr->xattri_da_state) + attr->xattri_da_state = xfs_da_state_alloc(args); + else + xfs_da_state_reset(attr->xattri_da_state, args); +} + +/* + * Initial setup for xfs_attr_node_removename. Make sure the attr is there and + * the blocks are valid. Attr keys with remote blocks will be marked + * incomplete. + */ +static +int xfs_attr_node_removename_setup( + struct xfs_attr_intent *attr) +{ + struct xfs_da_args *args = attr->xattri_da_args; + struct xfs_da_state *state; + int error; + + xfs_attr_item_init_da_state(attr); + error = xfs_attr_node_lookup(args, attr->xattri_da_state); + if (error != -EEXIST) + goto out; + error = 0; + + state = attr->xattri_da_state; + ASSERT(state->path.blk[state->path.active - 1].bp != NULL); + ASSERT(state->path.blk[state->path.active - 1].magic == + XFS_ATTR_LEAF_MAGIC); + + error = xfs_attr_leaf_mark_incomplete(args, state); + if (error) + goto out; + if (args->rmtblkno > 0) + error = xfs_attr_rmtval_invalidate(args); +out: + if (error) { + xfs_da_state_free(attr->xattri_da_state); + attr->xattri_da_state = NULL; + } + + return error; +} + +/* + * Remove the original attr we have just replaced. This is dependent on the + * original lookup and insert placing the old attr in args->blkno/args->index + * and the new attr in args->blkno2/args->index2. + */ +static int +xfs_attr_leaf_remove_attr( + struct xfs_attr_intent *attr) +{ + struct xfs_da_args *args = attr->xattri_da_args; + struct xfs_inode *dp = args->dp; + struct xfs_buf *bp = NULL; + int forkoff; + int error; + + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, + args->blkno, &bp); + if (error) + return error; + + xfs_attr3_leaf_remove(bp, args); + + forkoff = xfs_attr_shortform_allfit(bp, dp); + if (forkoff) + error = xfs_attr3_leaf_to_shortform(bp, args, forkoff); + /* bp is gone due to xfs_da_shrink_inode */ + + return error; +} + +/* + * Shrink an attribute from leaf to shortform. Used by the node format remove + * path when the node format collapses to a single block and so we have to check + * if it can be collapsed further. + */ +static int +xfs_attr_leaf_shrink( + struct xfs_da_args *args) +{ + struct xfs_inode *dp = args->dp; + struct xfs_buf *bp; + int forkoff; + int error; + + if (!xfs_attr_is_leaf(dp)) + return 0; + + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, 0, &bp); + if (error) + return error; + + forkoff = xfs_attr_shortform_allfit(bp, dp); + if (forkoff) { + error = xfs_attr3_leaf_to_shortform(bp, args, forkoff); + /* bp is gone due to xfs_da_shrink_inode */ + } else { + xfs_trans_brelse(args->trans, bp); + } + + return error; +} + +/* + * Run the attribute operation specified in @attr. + * + * This routine is meant to function as a delayed operation and will set the + * state to XFS_DAS_DONE when the operation is complete. Calling functions will + * need to handle this, and recall the function until either an error or + * XFS_DAS_DONE is detected. + */ +int +xfs_attr_set_iter( + struct xfs_attr_intent *attr) +{ + struct xfs_da_args *args = attr->xattri_da_args; + int error = 0; + + /* State machine switch */ +next_state: + switch (attr->xattri_dela_state) { + case XFS_DAS_UNINIT: + ASSERT(0); + return -EFSCORRUPTED; + case XFS_DAS_SF_ADD: + return xfs_attr_sf_addname(attr); + case XFS_DAS_LEAF_ADD: + return xfs_attr_leaf_addname(attr); + case XFS_DAS_NODE_ADD: + return xfs_attr_node_addname(attr); + + case XFS_DAS_SF_REMOVE: + error = xfs_attr_sf_removename(args); + attr->xattri_dela_state = xfs_attr_complete_op(attr, + xfs_attr_init_add_state(args)); + break; + case XFS_DAS_LEAF_REMOVE: + error = xfs_attr_leaf_removename(args); + attr->xattri_dela_state = xfs_attr_complete_op(attr, + xfs_attr_init_add_state(args)); + break; + case XFS_DAS_NODE_REMOVE: + error = xfs_attr_node_removename_setup(attr); + if (error == -ENOATTR && + (args->op_flags & XFS_DA_OP_RECOVERY)) { + attr->xattri_dela_state = xfs_attr_complete_op(attr, + xfs_attr_init_add_state(args)); + error = 0; + break; + } + if (error) + return error; + attr->xattri_dela_state = XFS_DAS_NODE_REMOVE_RMT; + if (args->rmtblkno == 0) + attr->xattri_dela_state++; + break; + + case XFS_DAS_LEAF_SET_RMT: + case XFS_DAS_NODE_SET_RMT: + error = xfs_attr_rmtval_find_space(attr); + if (error) + return error; + attr->xattri_dela_state++; + fallthrough; + + case XFS_DAS_LEAF_ALLOC_RMT: + case XFS_DAS_NODE_ALLOC_RMT: + error = xfs_attr_rmtval_alloc(attr); + if (error) + return error; + if (attr->xattri_dela_state == XFS_DAS_DONE) + break; + goto next_state; + + case XFS_DAS_LEAF_REPLACE: + case XFS_DAS_NODE_REPLACE: + /* + * We must "flip" the incomplete flags on the "new" and "old" + * attribute/value pairs so that one disappears and one appears + * atomically. + */ + error = xfs_attr3_leaf_flipflags(args); + if (error) + return error; + /* + * We must commit the flag value change now to make it atomic + * and then we can start the next trans in series at REMOVE_OLD. + */ + attr->xattri_dela_state++; + break; + + case XFS_DAS_LEAF_REMOVE_OLD: + case XFS_DAS_NODE_REMOVE_OLD: + /* + * If we have a remote attr, start the process of removing it + * by invalidating any cached buffers. + * + * If we don't have a remote attr, we skip the remote block + * removal state altogether with a second state increment. + */ + xfs_attr_restore_rmt_blk(args); + if (args->rmtblkno) { + error = xfs_attr_rmtval_invalidate(args); + if (error) + return error; + } else { + attr->xattri_dela_state++; + } + + attr->xattri_dela_state++; + goto next_state; + + case XFS_DAS_LEAF_REMOVE_RMT: + case XFS_DAS_NODE_REMOVE_RMT: + error = xfs_attr_rmtval_remove(attr); + if (error == -EAGAIN) { + error = 0; + break; + } + if (error) + return error; + + /* + * We've finished removing the remote attr blocks, so commit the + * transaction and move on to removing the attr name from the + * leaf/node block. Removing the attr might require a full + * transaction reservation for btree block freeing, so we + * can't do that in the same transaction where we removed the + * remote attr blocks. + */ + attr->xattri_dela_state++; + break; + + case XFS_DAS_LEAF_REMOVE_ATTR: + error = xfs_attr_leaf_remove_attr(attr); + attr->xattri_dela_state = xfs_attr_complete_op(attr, + xfs_attr_init_add_state(args)); + break; + + case XFS_DAS_NODE_REMOVE_ATTR: + error = xfs_attr_node_remove_attr(attr); + if (!error) + error = xfs_attr_leaf_shrink(args); + attr->xattri_dela_state = xfs_attr_complete_op(attr, + xfs_attr_init_add_state(args)); + break; + default: + ASSERT(0); + break; + } + + trace_xfs_attr_set_iter_return(attr->xattri_dela_state, args->dp); + return error; +} + + +/* + * Return EEXIST if attr is found, or ENOATTR if not + */ +static int +xfs_attr_lookup( + struct xfs_da_args *args) +{ + struct xfs_inode *dp = args->dp; + struct xfs_buf *bp = NULL; + struct xfs_da_state *state; + int error; + + if (!xfs_inode_hasattr(dp)) + return -ENOATTR; + + if (dp->i_af.if_format == XFS_DINODE_FMT_LOCAL) { + if (xfs_attr_sf_findname(args)) + return -EEXIST; + return -ENOATTR; + } + + /* Prerequisite for xfs_attr_is_leaf */ + error = xfs_iread_extents(args->trans, args->dp, XFS_ATTR_FORK); + if (error) + return error; + + if (xfs_attr_is_leaf(dp)) { + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, + 0, &bp); + if (error) + return error; + error = xfs_attr3_leaf_lookup_int(bp, args); + xfs_trans_brelse(args->trans, bp); + return error; + } + + state = xfs_da_state_alloc(args); + error = xfs_attr_node_lookup(args, state); + xfs_da_state_free(state); + return error; +} + +int +xfs_attr_add_fork( + struct xfs_inode *ip, /* incore inode pointer */ + int size, /* space new attribute needs */ + int rsvd) /* xact may use reserved blks */ +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_trans *tp; /* transaction pointer */ + unsigned int blks; /* space reservation */ + int error; /* error return value */ + + if (!xfs_is_metadir_inode(ip)) + ASSERT(!XFS_NOT_DQATTACHED(mp, ip)); + + blks = XFS_ADDAFORK_SPACE_RES(mp); + + error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_addafork, blks, 0, + rsvd, &tp); + if (error) + return error; + + if (xfs_inode_has_attr_fork(ip)) + goto trans_cancel; + + error = xfs_bmap_add_attrfork(tp, ip, size, rsvd); + if (error) + goto trans_cancel; + + error = xfs_trans_commit(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; + +trans_cancel: + xfs_trans_cancel(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; +} + +/* + * Decide if it is theoretically possible to try to bypass the attr intent + * mechanism for better performance. Other constraints (e.g. available space + * in the existing structure) are not considered here. + */ +static inline bool +xfs_attr_can_shortcut( + const struct xfs_inode *ip) +{ + return xfs_inode_has_attr_fork(ip) && xfs_attr_is_shortform(ip); +} + +/* Try to set an attr in one transaction or fall back to attr intents. */ +int +xfs_attr_setname( + struct xfs_da_args *args, + int rmt_blks) +{ + int error; + + if (!rmt_blks && xfs_attr_can_shortcut(args->dp)) { + args->op_flags |= XFS_DA_OP_ADDNAME; + + error = xfs_attr_try_sf_addname(args); + if (error != -ENOSPC) + return error; + } + + xfs_attr_defer_add(args, XFS_ATTR_DEFER_SET); + return 0; +} + +/* Try to remove an attr in one transaction or fall back to attr intents. */ +int +xfs_attr_removename( + struct xfs_da_args *args) +{ + if (xfs_attr_can_shortcut(args->dp)) + return xfs_attr_sf_removename(args); + + xfs_attr_defer_add(args, XFS_ATTR_DEFER_REMOVE); + return 0; +} + +/* Try to replace an attr in one transaction or fall back to attr intents. */ +int +xfs_attr_replacename( + struct xfs_da_args *args, + int rmt_blks) +{ + int error; + + if (rmt_blks || !xfs_attr_can_shortcut(args->dp)) { + xfs_attr_defer_add(args, XFS_ATTR_DEFER_REPLACE); + return 0; + } + + error = xfs_attr_shortform_replace(args); + if (error != -ENOSPC) + return error; + + args->op_flags |= XFS_DA_OP_ADDNAME | XFS_DA_OP_REPLACE; + + error = xfs_attr_sf_removename(args); + if (error) + return error; + + if (args->attr_filter & XFS_ATTR_PARENT) { + /* + * Move the new name/value to the regular name/value slots and + * zero out the new name/value slots because we don't need to + * log them for a PPTR_SET operation. + */ + xfs_attr_update_pptr_replace_args(args); + args->new_name = NULL; + args->new_namelen = 0; + args->new_value = NULL; + args->new_valuelen = 0; + } + args->op_flags &= ~XFS_DA_OP_REPLACE; + + error = xfs_attr_try_sf_addname(args); + if (error != -ENOSPC) + return error; + + xfs_attr_defer_add(args, XFS_ATTR_DEFER_SET); + return 0; +} + +/* + * Make a change to the xattr structure. + * + * The caller must have initialized @args, attached dquots, and must not hold + * any ILOCKs. Reserved data blocks may be used if @rsvd is set. + * + * Returns -EEXIST for XFS_ATTRUPDATE_CREATE if the name already exists. + * Returns -ENOATTR for XFS_ATTRUPDATE_REMOVE if the name does not exist. + * Returns 0 on success, or a negative errno if something else went wrong. + */ +int +xfs_attr_set( + struct xfs_da_args *args, + enum xfs_attr_update op, + bool rsvd) +{ + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + struct xfs_trans_res tres; + int error, local; + int rmt_blks = 0; + unsigned int total = 0; + + ASSERT(!args->trans); + + switch (op) { + case XFS_ATTRUPDATE_UPSERT: + case XFS_ATTRUPDATE_CREATE: + case XFS_ATTRUPDATE_REPLACE: + XFS_STATS_INC(mp, xs_attr_set); + args->total = xfs_attr_calc_size(args, &local); + + /* + * If the inode doesn't have an attribute fork, add one. + * (inode must not be locked when we call this routine) + */ + if (xfs_inode_has_attr_fork(dp) == 0) { + int sf_size = sizeof(struct xfs_attr_sf_hdr) + + xfs_attr_sf_entsize_byname(args->namelen, + args->valuelen); + + error = xfs_attr_add_fork(dp, sf_size, rsvd); + if (error) + return error; + } + + if (!local) + rmt_blks = xfs_attr3_rmt_blocks(mp, args->valuelen); + + tres = xfs_attr_set_resv(args); + total = args->total; + break; + case XFS_ATTRUPDATE_REMOVE: + XFS_STATS_INC(mp, xs_attr_remove); + rmt_blks = xfs_attr3_max_rmt_blocks(mp); + tres = M_RES(mp)->tr_attrrm; + total = XFS_ATTRRM_SPACE_RES(mp); + break; + } + + /* + * Root fork attributes can use reserved data blocks for this + * operation if necessary + */ + error = xfs_trans_alloc_inode(dp, &tres, total, 0, rsvd, &args->trans); + if (error) + return error; + + if (op != XFS_ATTRUPDATE_REMOVE || xfs_inode_hasattr(dp)) { + error = xfs_iext_count_extend(args->trans, dp, XFS_ATTR_FORK, + XFS_IEXT_ATTR_MANIP_CNT(rmt_blks)); + if (error) + goto out_trans_cancel; + } + + error = xfs_attr_lookup(args); + switch (error) { + case -EEXIST: + if (op == XFS_ATTRUPDATE_REMOVE) { + /* if no value, we are performing a remove operation */ + error = xfs_attr_removename(args); + if (error) + goto out_trans_cancel; + break; + } + + /* Pure create fails if the attr already exists */ + if (op == XFS_ATTRUPDATE_CREATE) + goto out_trans_cancel; + + error = xfs_attr_replacename(args, rmt_blks); + if (error) + goto out_trans_cancel; + break; + case -ENOATTR: + /* Can't remove what isn't there. */ + if (op == XFS_ATTRUPDATE_REMOVE) + goto out_trans_cancel; + + /* Pure replace fails if no existing attr to replace. */ + if (op == XFS_ATTRUPDATE_REPLACE) + goto out_trans_cancel; + + error = xfs_attr_setname(args, rmt_blks); + if (error) + goto out_trans_cancel; + break; + default: + goto out_trans_cancel; + } + + /* + * If this is a synchronous mount, make sure that the + * transaction goes to disk before returning to the user. + */ + if (xfs_has_wsync(mp)) + xfs_trans_set_sync(args->trans); + + xfs_trans_ichgtime(args->trans, dp, XFS_ICHGTIME_CHG); + + /* + * Commit the last in the sequence of transactions. + */ + xfs_trans_log_inode(args->trans, dp, XFS_ILOG_CORE); + error = xfs_trans_commit(args->trans); +out_unlock: + xfs_iunlock(dp, XFS_ILOCK_EXCL); + args->trans = NULL; + return error; + +out_trans_cancel: + if (args->trans) + xfs_trans_cancel(args->trans); + goto out_unlock; +} + +/*======================================================================== + * External routines when attribute list is inside the inode + *========================================================================*/ + +int xfs_attr_sf_totsize(struct xfs_inode *dp) +{ + struct xfs_attr_sf_hdr *sf = dp->i_af.if_data; + + return be16_to_cpu(sf->totsize); +} + +/* + * Add a name to the shortform attribute list structure + * This is the external routine. + */ +static int +xfs_attr_shortform_addname( + struct xfs_da_args *args) +{ + int newsize, forkoff; + + trace_xfs_attr_sf_addname(args); + + if (xfs_attr_sf_findname(args)) { + int error; + + ASSERT(args->op_flags & XFS_DA_OP_REPLACE); + + error = xfs_attr_sf_removename(args); + if (error) + return error; + + /* + * Since we have removed the old attr, clear XFS_DA_OP_REPLACE + * so that the new attr doesn't fit in shortform format, the + * leaf format add routine won't trip over the attr not being + * around. + */ + args->op_flags &= ~XFS_DA_OP_REPLACE; + } else { + ASSERT(!(args->op_flags & XFS_DA_OP_REPLACE)); + } + + if (args->namelen >= XFS_ATTR_SF_ENTSIZE_MAX || + args->valuelen >= XFS_ATTR_SF_ENTSIZE_MAX) + return -ENOSPC; + + newsize = xfs_attr_sf_totsize(args->dp); + newsize += xfs_attr_sf_entsize_byname(args->namelen, args->valuelen); + + forkoff = xfs_attr_shortform_bytesfit(args->dp, newsize); + if (!forkoff) + return -ENOSPC; + + xfs_attr_shortform_add(args, forkoff); + return 0; +} + + +/*======================================================================== + * External routines when attribute list is one block + *========================================================================*/ + +/* + * Remove a name from the leaf attribute list structure + * + * This leaf block cannot have a "remote" value, we only call this routine + * if bmap_one_block() says there is only one block (ie: no remote blks). + */ +STATIC int +xfs_attr_leaf_removename( + struct xfs_da_args *args) +{ + struct xfs_inode *dp = args->dp; + int error, forkoff; + struct xfs_buf *bp; + + trace_xfs_attr_leaf_removename(args); + + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, 0, &bp); + if (error) + return error; + error = xfs_attr3_leaf_lookup_int(bp, args); + if (error != -EEXIST) { + xfs_trans_brelse(args->trans, bp); + if (error == -ENOATTR && (args->op_flags & XFS_DA_OP_RECOVERY)) + return 0; + return error; + } + + xfs_attr3_leaf_remove(bp, args); + + /* + * If the result is small enough, shrink it all into the inode. + */ + forkoff = xfs_attr_shortform_allfit(bp, dp); + if (forkoff) + return xfs_attr3_leaf_to_shortform(bp, args, forkoff); + /* bp is gone due to xfs_da_shrink_inode */ + + return 0; +} + +/* + * Look up a name in a leaf attribute list structure. + * + * This leaf block cannot have a "remote" value, we only call this routine + * if bmap_one_block() says there is only one block (ie: no remote blks). + * + * Returns 0 on successful retrieval, otherwise an error. + */ +STATIC int +xfs_attr_leaf_get( + struct xfs_da_args *args) +{ + struct xfs_buf *bp; + int error; + + trace_xfs_attr_leaf_get(args); + + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, 0, &bp); + if (error) + return error; + error = xfs_attr3_leaf_lookup_int(bp, args); + if (error == -EEXIST) + error = xfs_attr3_leaf_getvalue(bp, args); + xfs_trans_brelse(args->trans, bp); + return error; +} + +/* Return EEXIST if attr is found, or ENOATTR if not. */ +STATIC int +xfs_attr_node_lookup( + struct xfs_da_args *args, + struct xfs_da_state *state) +{ + int retval, error; + + /* + * Search to see if name exists, and get back a pointer to it. + */ + error = xfs_da3_node_lookup_int(state, &retval); + if (error) + return error; + + return retval; +} + +/*======================================================================== + * External routines when attribute list size > geo->blksize + *========================================================================*/ + +STATIC int +xfs_attr_node_addname_find_attr( + struct xfs_attr_intent *attr) +{ + struct xfs_da_args *args = attr->xattri_da_args; + int error; + + /* + * Search to see if name already exists, and get back a pointer + * to where it should go. + */ + xfs_attr_item_init_da_state(attr); + error = xfs_attr_node_lookup(args, attr->xattri_da_state); + switch (error) { + case -ENOATTR: + if (args->op_flags & XFS_DA_OP_REPLACE) + goto error; + break; + case -EEXIST: + if (!(args->op_flags & XFS_DA_OP_REPLACE)) + goto error; + + + trace_xfs_attr_node_replace(args); + /* + * Save the existing remote attr state so that the current + * values reflect the state of the new attribute we are about to + * add, not the attribute we just found and will remove later. + */ + xfs_attr_save_rmt_blk(args); + break; + case 0: + break; + default: + goto error; + } + + return 0; +error: + if (attr->xattri_da_state) { + xfs_da_state_free(attr->xattri_da_state); + attr->xattri_da_state = NULL; + } + return error; +} + +/* + * Add a name to a Btree-format attribute list. + * + * This will involve walking down the Btree, and may involve splitting leaf + * nodes and even splitting intermediate nodes up to and including the root + * node (a special case of an intermediate node). + * + * If the tree was still in single leaf format and needs to converted to + * real node format return 1 and let the caller handle that. + */ +static int +xfs_attr_node_try_addname( + struct xfs_attr_intent *attr) +{ + struct xfs_da_state *state = attr->xattri_da_state; + struct xfs_da_state_blk *blk; + int error = 0; + + trace_xfs_attr_node_addname(state->args); + + blk = &state->path.blk[state->path.active-1]; + ASSERT(blk->magic == XFS_ATTR_LEAF_MAGIC); + + if (!xfs_attr3_leaf_add(blk->bp, state->args)) { + if (state->path.active == 1) { + /* + * Its really a single leaf node, but it had + * out-of-line values so it looked like it *might* + * have been a b-tree. Let the caller deal with this. + */ + error = 1; + goto out; + } + + /* + * Split as many Btree elements as required. + * This code tracks the new and old attr's location + * in the index/blkno/rmtblkno/rmtblkcnt fields and + * in the index2/blkno2/rmtblkno2/rmtblkcnt2 fields. + */ + error = xfs_da3_split(state); + if (error) + goto out; + } else { + /* + * Addition succeeded, update Btree hashvals. + */ + xfs_da3_fixhashpath(state, &state->path); + } + +out: + xfs_da_state_free(state); + attr->xattri_da_state = NULL; + return error; +} + +static int +xfs_attr_node_removename( + struct xfs_da_args *args, + struct xfs_da_state *state) +{ + struct xfs_da_state_blk *blk; + int retval; + + /* + * Remove the name and update the hashvals in the tree. + */ + blk = &state->path.blk[state->path.active-1]; + ASSERT(blk->magic == XFS_ATTR_LEAF_MAGIC); + retval = xfs_attr3_leaf_remove(blk->bp, args); + xfs_da3_fixhashpath(state, &state->path); + + return retval; +} + +static int +xfs_attr_node_remove_attr( + struct xfs_attr_intent *attr) +{ + struct xfs_da_args *args = attr->xattri_da_args; + struct xfs_da_state *state = xfs_da_state_alloc(args); + int retval = 0; + int error = 0; + + /* + * The attr we are removing has already been marked incomplete, so + * we need to set the filter appropriately to re-find the "old" + * attribute entry after any split ops. + */ + args->attr_filter |= XFS_ATTR_INCOMPLETE; + error = xfs_da3_node_lookup_int(state, &retval); + if (error) + goto out; + + error = xfs_attr_node_removename(args, state); + + /* + * Check to see if the tree needs to be collapsed. + */ + if (retval && (state->path.active > 1)) { + error = xfs_da3_join(state); + if (error) + goto out; + } + retval = error = 0; + +out: + xfs_da_state_free(state); + if (error) + return error; + return retval; +} + +/* + * Retrieve the attribute data from a node attribute list. + * + * This routine gets called for any attribute fork that has more than one + * block, ie: both true Btree attr lists and for single-leaf-blocks with + * "remote" values taking up more blocks. + * + * Returns 0 on successful retrieval, otherwise an error. + */ +STATIC int +xfs_attr_node_get( + struct xfs_da_args *args) +{ + struct xfs_da_state *state; + struct xfs_da_state_blk *blk; + int i; + int error; + + trace_xfs_attr_node_get(args); + + /* + * Search to see if name exists, and get back a pointer to it. + */ + state = xfs_da_state_alloc(args); + error = xfs_attr_node_lookup(args, state); + if (error != -EEXIST) + goto out_release; + + /* + * Get the value, local or "remote" + */ + blk = &state->path.blk[state->path.active - 1]; + error = xfs_attr3_leaf_getvalue(blk->bp, args); + + /* + * If not in a transaction, we have to release all the buffers. + */ +out_release: + for (i = 0; i < state->path.active; i++) { + xfs_trans_brelse(args->trans, state->path.blk[i].bp); + state->path.blk[i].bp = NULL; + } + + xfs_da_state_free(state); + return error; +} + +/* Enforce that there is at most one namespace bit per attr. */ +inline bool xfs_attr_check_namespace(unsigned int attr_flags) +{ + return hweight32(attr_flags & XFS_ATTR_NSP_ONDISK_MASK) < 2; +} + +/* Returns true if the attribute entry name is valid. */ +bool +xfs_attr_namecheck( + unsigned int attr_flags, + const void *name, + size_t length) +{ + /* Only one namespace bit allowed. */ + if (!xfs_attr_check_namespace(attr_flags)) + return false; + + /* + * MAXNAMELEN includes the trailing null, but (name/length) leave it + * out, so use >= for the length check. + */ + if (length >= MAXNAMELEN) + return false; + + /* Parent pointers have their own validation. */ + if (attr_flags & XFS_ATTR_PARENT) + return xfs_parent_namecheck(attr_flags, name, length); + + /* There shouldn't be any nulls here */ + return !memchr(name, 0, length); +} + +int __init +xfs_attr_intent_init_cache(void) +{ + xfs_attr_intent_cache = kmem_cache_create("xfs_attr_intent", + sizeof(struct xfs_attr_intent), + 0, 0, NULL); + + return xfs_attr_intent_cache != NULL ? 0 : -ENOMEM; +} + +void +xfs_attr_intent_destroy_cache(void) +{ + kmem_cache_destroy(xfs_attr_intent_cache); + xfs_attr_intent_cache = NULL; +} diff --git a/fs/xfs/libxfs/xfs_attr.h b/fs/xfs/libxfs/xfs_attr.h new file mode 100644 index 000000000..67fd9c75a --- /dev/null +++ b/fs/xfs/libxfs/xfs_attr.h @@ -0,0 +1,657 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2000,2002-2003,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_ATTR_H__ +#define __XFS_ATTR_H__ + +struct xfs_inode; +struct xfs_da_args; +struct xfs_attr_list_context; + +/* + * Large attribute lists are structured around Btrees where all the data + * elements are in the leaf nodes. Attribute names are hashed into an int, + * then that int is used as the index into the Btree. Since the hashval + * of an attribute name may not be unique, we may have duplicate keys. + * The internal links in the Btree are logical block offsets into the file. + * + * Small attribute lists use a different format and are packed as tightly + * as possible so as to fit into the literal area of the inode. + */ + +/* + * The maximum size (into the kernel or returned from the kernel) of an + * attribute value or the buffer used for an attr_list() call. Larger + * sizes will result in an ERANGE return code. + */ +#define ATTR_MAX_VALUELEN (64*1024) /* max length of a value */ + +/* + * Kernel-internal version of the attrlist cursor. + */ +struct xfs_attrlist_cursor_kern { + __u32 hashval; /* hash value of next entry to add */ + __u32 blkno; /* block containing entry (suggestion) */ + __u32 offset; /* offset in list of equal-hashvals */ + __u16 pad1; /* padding to match user-level */ + __u8 pad2; /* padding to match user-level */ + __u8 initted; /* T/F: cursor has been initialized */ +}; + + +/*======================================================================== + * Structure used to pass context around among the routines. + *========================================================================*/ + + +/* void; state communicated via *context */ +typedef void (*put_listent_func_t)(struct xfs_attr_list_context *context, + int flags, unsigned char *name, int namelen, void *value, + int valuelen); + +struct xfs_attr_list_context { + struct xfs_trans *tp; + struct xfs_inode *dp; /* inode */ + struct xfs_attrlist_cursor_kern cursor; /* position in list */ + /* output buffer */ + void *buffer __counted_by_ptr(bufsize); + + /* + * Abort attribute list iteration if non-zero. Can be used to pass + * error values to the xfs_attr_list caller. + */ + int seen_enough; + bool allow_incomplete; + + ssize_t count; /* num used entries */ + int dupcnt; /* count dup hashvals seen */ + int bufsize; /* total buffer size */ + int firstu; /* first used byte in buffer */ + unsigned int attr_filter; /* XFS_ATTR_{ROOT,SECURE} */ + int resynch; /* T/F: resynch with cursor */ + put_listent_func_t put_listent; /* list output fmt function */ + int index; /* index into output buffer */ +}; + + +/* + * ======================================================================== + * Structure used to pass context around among the delayed routines. + * ======================================================================== + */ + +/* + * Below is a state machine diagram for attr remove operations. The XFS_DAS_* + * states indicate places where the function would return -EAGAIN, and then + * immediately resume from after being called by the calling function. States + * marked as a "subroutine state" indicate that they belong to a subroutine, and + * so the calling function needs to pass them back to that subroutine to allow + * it to finish where it left off. But they otherwise do not have a role in the + * calling function other than just passing through. + * + * xfs_attr_remove_iter() + * │ + * v + * have attr to remove? ──n──> done + * │ + * y + * │ + * v + * are we short form? ──y──> xfs_attr_shortform_remove ──> done + * │ + * n + * │ + * V + * are we leaf form? ──y──> xfs_attr_leaf_removename ──> done + * │ + * n + * │ + * V + * ┌── need to setup state? + * │ │ + * n y + * │ │ + * │ v + * │ find attr and get state + * │ attr has remote blks? ──n─┐ + * │ │ v + * │ │ find and invalidate + * │ y the remote blocks. + * │ │ mark attr incomplete + * │ ├────────────────┘ + * └──────────┤ + * │ + * v + * Have remote blks to remove? ───y─────┐ + * │ ^ remove the blks + * │ │ │ + * │ │ v + * │ XFS_DAS_RMTBLK <─n── done? + * │ re-enter with │ + * │ one less blk to y + * │ remove │ + * │ V + * │ refill the state + * n │ + * │ v + * │ XFS_DAS_RM_NAME + * │ │ + * ├─────────────────────────┘ + * │ + * v + * remove leaf and + * update hash with + * xfs_attr_node_remove_cleanup + * │ + * v + * need to + * shrink tree? ─n─┐ + * │ │ + * y │ + * │ │ + * v │ + * join leaf │ + * │ │ + * v │ + * XFS_DAS_RM_SHRINK │ + * │ │ + * v │ + * do the shrink │ + * │ │ + * v │ + * free state <──┘ + * │ + * v + * done + * + * + * Below is a state machine diagram for attr set operations. + * + * It seems the challenge with understanding this system comes from trying to + * absorb the state machine all at once, when really one should only be looking + * at it with in the context of a single function. Once a state sensitive + * function is called, the idea is that it "takes ownership" of the + * state machine. It isn't concerned with the states that may have belonged to + * it's calling parent. Only the states relevant to itself or any other + * subroutines there in. Once a calling function hands off the state machine to + * a subroutine, it needs to respect the simple rule that it doesn't "own" the + * state machine anymore, and it's the responsibility of that calling function + * to propagate the -EAGAIN back up the call stack. Upon reentry, it is + * committed to re-calling that subroutine until it returns something other than + * -EAGAIN. Once that subroutine signals completion (by returning anything other + * than -EAGAIN), the calling function can resume using the state machine. + * + * xfs_attr_set_iter() + * │ + * v + * ┌─y─ has an attr fork? + * │ | + * │ n + * │ | + * │ V + * │ add a fork + * │ │ + * └──────────┤ + * │ + * V + * ┌─── is shortform? + * │ │ + * │ y + * │ │ + * │ V + * │ xfs_attr_set_fmt + * │ | + * │ V + * │ xfs_attr_try_sf_addname + * │ │ + * │ V + * │ had enough ──y──> done + * │ space? + * n │ + * │ n + * │ │ + * │ V + * │ transform to leaf + * │ │ + * │ V + * │ hold the leaf buffer + * │ │ + * │ V + * │ return -EAGAIN + * │ Re-enter in + * │ leaf form + * │ + * └─> release leaf buffer + * if needed + * │ + * V + * ┌───n── fork has + * │ only 1 blk? + * │ │ + * │ y + * │ │ + * │ v + * │ xfs_attr_leaf_try_add() + * │ │ + * │ v + * │ had enough ──────────────y─────────────┐ + * │ space? │ + * │ │ │ + * │ n │ + * │ │ │ + * │ v │ + * │ return -EAGAIN │ + * │ re-enter in │ + * │ node form │ + * │ │ │ + * └──────────┤ │ + * │ │ + * V │ + * xfs_attr_node_addname_find_attr │ + * determines if this │ + * is create or rename │ + * find space to store attr │ + * │ │ + * v │ + * xfs_attr_node_addname │ + * │ │ + * v │ + * fits in a node leaf? ────n─────┐ │ + * │ ^ v │ + * │ │ single leaf node? │ + * │ │ │ │ │ + * y │ y n │ + * │ │ │ │ │ + * v │ v v │ + * update │ grow the leaf split if │ + * hashvals └── return -EAGAIN needed │ + * │ retry leaf add │ │ + * │ on reentry │ │ + * ├────────────────────────────┘ │ + * │ │ + * v │ + * need to alloc │ + * ┌─y── or flip flag? │ + * │ │ │ + * │ n │ + * │ │ │ + * │ v │ + * │ done │ + * │ │ + * │ │ + * │ XFS_DAS_FOUND_LBLK <────────────────┘ + * │ │ + * │ V + * │ xfs_attr_leaf_addname() + * │ │ + * │ v + * │ ┌──first time through? + * │ │ │ + * │ │ y + * │ │ │ + * │ n v + * │ │ if we have rmt blks + * │ │ find space for them + * │ │ │ + * │ └──────────┤ + * │ │ + * │ v + * │ still have + * │ ┌─n─ blks to alloc? <──┐ + * │ │ │ │ + * │ │ y │ + * │ │ │ │ + * │ │ v │ + * │ │ alloc one blk │ + * │ │ return -EAGAIN ──┘ + * │ │ re-enter with one + * │ │ less blk to alloc + * │ │ + * │ │ + * │ └───> set the rmt + * │ value + * │ │ + * │ v + * │ was this + * │ a rename? ──n─┐ + * │ │ │ + * │ y │ + * │ │ │ + * │ v │ + * │ flip incomplete │ + * │ flag │ + * │ │ │ + * │ v │ + * │ XFS_DAS_FLIP_LFLAG │ + * │ │ │ + * │ v │ + * │ need to remove │ + * │ old bks? ──n──┤ + * │ │ │ + * │ y │ + * │ │ │ + * │ V │ + * │ remove │ + * │ ┌───> old blks │ + * │ │ │ │ + * │ XFS_DAS_RM_LBLK │ │ + * │ ^ │ │ + * │ │ v │ + * │ └──y── more to │ + * │ remove? │ + * │ │ │ + * │ n │ + * │ │ │ + * │ v │ + * │ XFS_DAS_RD_LEAF │ + * │ │ │ + * │ v │ + * │ remove leaf │ + * │ │ │ + * │ v │ + * │ shrink to sf │ + * │ if needed │ + * │ │ │ + * │ v │ + * │ done <──────┘ + * │ + * └──────> XFS_DAS_FOUND_NBLK + * │ + * v + * ┌─────n── need to + * │ alloc blks? + * │ │ + * │ y + * │ │ + * │ v + * │ find space + * │ │ + * │ v + * │ ┌─>XFS_DAS_ALLOC_NODE + * │ │ │ + * │ │ v + * │ │ alloc blk + * │ │ │ + * │ │ v + * │ └──y── need to alloc + * │ more blocks? + * │ │ + * │ n + * │ │ + * │ v + * │ set the rmt value + * │ │ + * │ v + * │ was this + * └────────> a rename? ──n─┐ + * │ │ + * y │ + * │ │ + * v │ + * flip incomplete │ + * flag │ + * │ │ + * v │ + * XFS_DAS_FLIP_NFLAG │ + * │ │ + * v │ + * need to │ + * remove blks? ─n──┤ + * │ │ + * y │ + * │ │ + * v │ + * remove │ + * ┌────────> old blks │ + * │ │ │ + * XFS_DAS_RM_NBLK │ │ + * ^ │ │ + * │ v │ + * └──────y── more to │ + * remove │ + * │ │ + * n │ + * │ │ + * v │ + * XFS_DAS_CLR_FLAG │ + * │ │ + * v │ + * clear flags │ + * │ │ + * ├──────────┘ + * │ + * v + * done + */ + +/* + * Enum values for xfs_attr_intent.xattri_da_state + * + * These values are used by delayed attribute operations to keep track of where + * they were before they returned -EAGAIN. A return code of -EAGAIN signals the + * calling function to roll the transaction, and then call the subroutine to + * finish the operation. The enum is then used by the subroutine to jump back + * to where it was and resume executing where it left off. + */ +enum xfs_delattr_state { + XFS_DAS_UNINIT = 0, /* No state has been set yet */ + + /* + * Initial sequence states. The replace setup code relies on the + * ADD and REMOVE states for a specific format to be sequential so + * that we can transform the initial operation to be performed + * according to the xfs_has_larp() state easily. + */ + XFS_DAS_SF_ADD, /* Initial sf add state */ + XFS_DAS_SF_REMOVE, /* Initial sf replace/remove state */ + + XFS_DAS_LEAF_ADD, /* Initial leaf add state */ + XFS_DAS_LEAF_REMOVE, /* Initial leaf replace/remove state */ + + XFS_DAS_NODE_ADD, /* Initial node add state */ + XFS_DAS_NODE_REMOVE, /* Initial node replace/remove state */ + + /* Leaf state set/replace/remove sequence */ + XFS_DAS_LEAF_SET_RMT, /* set a remote xattr from a leaf */ + XFS_DAS_LEAF_ALLOC_RMT, /* We are allocating remote blocks */ + XFS_DAS_LEAF_REPLACE, /* Perform replace ops on a leaf */ + XFS_DAS_LEAF_REMOVE_OLD, /* Start removing old attr from leaf */ + XFS_DAS_LEAF_REMOVE_RMT, /* A rename is removing remote blocks */ + XFS_DAS_LEAF_REMOVE_ATTR, /* Remove the old attr from a leaf */ + + /* Node state sequence, must match leaf state above */ + XFS_DAS_NODE_SET_RMT, /* set a remote xattr from a node */ + XFS_DAS_NODE_ALLOC_RMT, /* We are allocating remote blocks */ + XFS_DAS_NODE_REPLACE, /* Perform replace ops on a node */ + XFS_DAS_NODE_REMOVE_OLD, /* Start removing old attr from node */ + XFS_DAS_NODE_REMOVE_RMT, /* A rename is removing remote blocks */ + XFS_DAS_NODE_REMOVE_ATTR, /* Remove the old attr from a node */ + + XFS_DAS_DONE, /* finished operation */ +}; + +#define XFS_DAS_STRINGS \ + { XFS_DAS_UNINIT, "XFS_DAS_UNINIT" }, \ + { XFS_DAS_SF_ADD, "XFS_DAS_SF_ADD" }, \ + { XFS_DAS_SF_REMOVE, "XFS_DAS_SF_REMOVE" }, \ + { XFS_DAS_LEAF_ADD, "XFS_DAS_LEAF_ADD" }, \ + { XFS_DAS_LEAF_REMOVE, "XFS_DAS_LEAF_REMOVE" }, \ + { XFS_DAS_NODE_ADD, "XFS_DAS_NODE_ADD" }, \ + { XFS_DAS_NODE_REMOVE, "XFS_DAS_NODE_REMOVE" }, \ + { XFS_DAS_LEAF_SET_RMT, "XFS_DAS_LEAF_SET_RMT" }, \ + { XFS_DAS_LEAF_ALLOC_RMT, "XFS_DAS_LEAF_ALLOC_RMT" }, \ + { XFS_DAS_LEAF_REPLACE, "XFS_DAS_LEAF_REPLACE" }, \ + { XFS_DAS_LEAF_REMOVE_OLD, "XFS_DAS_LEAF_REMOVE_OLD" }, \ + { XFS_DAS_LEAF_REMOVE_RMT, "XFS_DAS_LEAF_REMOVE_RMT" }, \ + { XFS_DAS_LEAF_REMOVE_ATTR, "XFS_DAS_LEAF_REMOVE_ATTR" }, \ + { XFS_DAS_NODE_SET_RMT, "XFS_DAS_NODE_SET_RMT" }, \ + { XFS_DAS_NODE_ALLOC_RMT, "XFS_DAS_NODE_ALLOC_RMT" }, \ + { XFS_DAS_NODE_REPLACE, "XFS_DAS_NODE_REPLACE" }, \ + { XFS_DAS_NODE_REMOVE_OLD, "XFS_DAS_NODE_REMOVE_OLD" }, \ + { XFS_DAS_NODE_REMOVE_RMT, "XFS_DAS_NODE_REMOVE_RMT" }, \ + { XFS_DAS_NODE_REMOVE_ATTR, "XFS_DAS_NODE_REMOVE_ATTR" }, \ + { XFS_DAS_DONE, "XFS_DAS_DONE" } + +struct xfs_attri_log_nameval; + +/* + * Context used for keeping track of delayed attribute operations + */ +struct xfs_attr_intent { + /* + * used to log this item to an intent containing a list of attrs to + * commit later + */ + struct list_head xattri_list; + + /* Used in xfs_attr_node_removename to roll through removing blocks */ + struct xfs_da_state *xattri_da_state; + + struct xfs_da_args *xattri_da_args; + + /* + * Shared buffer containing the attr name, new name, and value so that + * the logging code can share large memory buffers between log items. + */ + struct xfs_attri_log_nameval *xattri_nameval; + + /* Used to keep track of current state of delayed operation */ + enum xfs_delattr_state xattri_dela_state; + + /* + * Attr operation being performed - XFS_ATTRI_OP_FLAGS_* + */ + unsigned int xattri_op_flags; + + /* Used in xfs_attr_rmtval_set_blk to roll through allocating blocks */ + xfs_dablk_t xattri_lblkno; + int xattri_blkcnt; + struct xfs_bmbt_irec xattri_map; +}; + +static inline unsigned int +xfs_attr_intent_op(const struct xfs_attr_intent *attr) +{ + return attr->xattri_op_flags & XFS_ATTRI_OP_FLAGS_TYPE_MASK; +} + +/*======================================================================== + * Function prototypes for the kernel. + *========================================================================*/ + +/* + * Overall external interface routines. + */ +int xfs_attr_inactive(struct xfs_inode *dp); +int xfs_attr_list_ilocked(struct xfs_attr_list_context *); +int xfs_attr_list(struct xfs_attr_list_context *); +int xfs_inode_hasattr(struct xfs_inode *ip); +bool xfs_attr_is_leaf(struct xfs_inode *ip); +int xfs_attr_get_ilocked(struct xfs_da_args *args); +int xfs_attr_get(struct xfs_da_args *args); + +enum xfs_attr_update { + XFS_ATTRUPDATE_REMOVE, /* remove attr */ + XFS_ATTRUPDATE_UPSERT, /* set value, replace any existing attr */ + XFS_ATTRUPDATE_CREATE, /* set value, fail if attr already exists */ + XFS_ATTRUPDATE_REPLACE, /* set value, fail if attr does not exist */ +}; + +int xfs_attr_set(struct xfs_da_args *args, enum xfs_attr_update op, bool rsvd); +int xfs_attr_set_iter(struct xfs_attr_intent *attr); +int xfs_attr_remove_iter(struct xfs_attr_intent *attr); +bool xfs_attr_check_namespace(unsigned int attr_flags); +bool xfs_attr_namecheck(unsigned int attr_flags, const void *name, + size_t length); +int xfs_attr_calc_size(struct xfs_da_args *args, int *local); +struct xfs_trans_res xfs_attr_set_resv(const struct xfs_da_args *args); + +/* + * Check to see if the attr should be upgraded from non-existent or shortform to + * single-leaf-block attribute list. + */ +static inline bool +xfs_attr_is_shortform( + const struct xfs_inode *ip) +{ + return ip->i_af.if_format == XFS_DINODE_FMT_LOCAL || + (ip->i_af.if_format == XFS_DINODE_FMT_EXTENTS && + ip->i_af.if_nextents == 0); +} + +static inline enum xfs_delattr_state +xfs_attr_init_add_state(struct xfs_da_args *args) +{ + /* + * When called from the completion of a attr remove to determine the + * next state, the attribute fork may be null. This can occur only occur + * on a pure remove, but we grab the next state before we check if a + * replace operation is being performed. If we are called from any other + * context, i_af is guaranteed to exist. Hence if the attr fork is + * null, we were called from a pure remove operation and so we are done. + */ + if (!xfs_inode_has_attr_fork(args->dp)) + return XFS_DAS_DONE; + + args->op_flags |= XFS_DA_OP_ADDNAME; + if (xfs_attr_is_shortform(args->dp)) + return XFS_DAS_SF_ADD; + if (xfs_attr_is_leaf(args->dp)) + return XFS_DAS_LEAF_ADD; + return XFS_DAS_NODE_ADD; +} + +static inline enum xfs_delattr_state +xfs_attr_init_remove_state(struct xfs_da_args *args) +{ + if (xfs_attr_is_shortform(args->dp)) + return XFS_DAS_SF_REMOVE; + if (xfs_attr_is_leaf(args->dp)) + return XFS_DAS_LEAF_REMOVE; + return XFS_DAS_NODE_REMOVE; +} + +/* + * If we are logging the attributes, then we have to start with removal of the + * old attribute so that there is always consistent state that we can recover + * from if the system goes down part way through. We always log the new attr + * value, so even when we remove the attr first we still have the information in + * the log to finish the replace operation atomically. + */ +static inline enum xfs_delattr_state +xfs_attr_init_replace_state(struct xfs_da_args *args) +{ + args->op_flags |= XFS_DA_OP_ADDNAME | XFS_DA_OP_REPLACE; + if (args->op_flags & XFS_DA_OP_LOGGED) + return xfs_attr_init_remove_state(args); + return xfs_attr_init_add_state(args); +} + +xfs_dahash_t xfs_attr_hashname(const uint8_t *name, int namelen); + +xfs_dahash_t xfs_attr_hashval(struct xfs_mount *mp, unsigned int attr_flags, + const uint8_t *name, int namelen, const void *value, + int valuelen); + +/* Set the hash value for any extended attribute from any namespace. */ +static inline void xfs_attr_sethash(struct xfs_da_args *args) +{ + args->hashval = xfs_attr_hashval(args->dp->i_mount, args->attr_filter, + args->name, args->namelen, + args->value, args->valuelen); +} + +extern struct kmem_cache *xfs_attr_intent_cache; +int __init xfs_attr_intent_init_cache(void); +void xfs_attr_intent_destroy_cache(void); + +int xfs_attr_sf_totsize(struct xfs_inode *dp); +int xfs_attr_add_fork(struct xfs_inode *ip, int size, int rsvd); + +int xfs_attr_setname(struct xfs_da_args *args, int rmt_blks); +int xfs_attr_removename(struct xfs_da_args *args); +int xfs_attr_replacename(struct xfs_da_args *args, int rmt_blks); + +#endif /* __XFS_ATTR_H__ */ diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c new file mode 100644 index 000000000..b6288395f --- /dev/null +++ b/fs/xfs/libxfs/xfs_attr_leaf.c @@ -0,0 +1,3161 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * Copyright (c) 2013 Red Hat, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_sb.h" +#include "xfs_mount.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_bmap_btree.h" +#include "xfs_bmap.h" +#include "xfs_attr_sf.h" +#include "xfs_attr.h" +#include "xfs_attr_remote.h" +#include "xfs_attr_leaf.h" +#include "xfs_error.h" +#include "xfs_trace.h" +#include "xfs_buf_item.h" +#include "xfs_dir2.h" +#include "xfs_log.h" +#include "xfs_ag.h" +#include "xfs_errortag.h" +#include "xfs_health.h" + + +/* + * xfs_attr_leaf.c + * + * Routines to implement leaf blocks of attributes as Btrees of hashed names. + */ + +/*======================================================================== + * Function prototypes for the kernel. + *========================================================================*/ + +/* + * Routines used for growing the Btree. + */ +STATIC int xfs_attr3_leaf_create(struct xfs_da_args *args, + xfs_dablk_t which_block, struct xfs_buf **bpp); +STATIC void xfs_attr3_leaf_add_work(struct xfs_buf *leaf_buffer, + struct xfs_attr3_icleaf_hdr *ichdr, + struct xfs_da_args *args, int freemap_index); +STATIC void xfs_attr3_leaf_compact(struct xfs_da_args *args, + struct xfs_attr3_icleaf_hdr *ichdr, + struct xfs_buf *leaf_buffer); +STATIC void xfs_attr3_leaf_rebalance(xfs_da_state_t *state, + xfs_da_state_blk_t *blk1, + xfs_da_state_blk_t *blk2); +STATIC int xfs_attr3_leaf_figure_balance(xfs_da_state_t *state, + xfs_da_state_blk_t *leaf_blk_1, + struct xfs_attr3_icleaf_hdr *ichdr1, + xfs_da_state_blk_t *leaf_blk_2, + struct xfs_attr3_icleaf_hdr *ichdr2, + int *number_entries_in_blk1, + int *number_usedbytes_in_blk1); + +/* + * Utility routines. + */ +STATIC void xfs_attr3_leaf_moveents(struct xfs_da_args *args, + struct xfs_attr_leafblock *src_leaf, + struct xfs_attr3_icleaf_hdr *src_ichdr, int src_start, + struct xfs_attr_leafblock *dst_leaf, + struct xfs_attr3_icleaf_hdr *dst_ichdr, int dst_start, + int move_count); +STATIC int xfs_attr_leaf_entsize(xfs_attr_leafblock_t *leaf, int index); + +/* Compute the byte offset of the end of the leaf entry array. */ +static inline int +xfs_attr_leaf_entries_end( + unsigned int hdrcount, + const struct xfs_attr_leafblock *leaf) +{ + return hdrcount * sizeof(struct xfs_attr_leaf_entry) + + xfs_attr3_leaf_hdr_size(leaf); +} + +static inline bool +ichdr_freemaps_overlap( + const struct xfs_attr3_icleaf_hdr *ichdr, + unsigned int x, + unsigned int y) +{ + const unsigned int xend = + ichdr->freemap[x].base + ichdr->freemap[x].size; + const unsigned int yend = + ichdr->freemap[y].base + ichdr->freemap[y].size; + + /* empty slots do not overlap */ + if (!ichdr->freemap[x].size || !ichdr->freemap[y].size) + return false; + + return ichdr->freemap[x].base < yend && xend > ichdr->freemap[y].base; +} + +static inline xfs_failaddr_t +xfs_attr_leaf_ichdr_freemaps_verify( + const struct xfs_attr3_icleaf_hdr *ichdr, + const struct xfs_attr_leafblock *leaf) +{ + unsigned int entries_end = + xfs_attr_leaf_entries_end(ichdr->count, leaf); + int i; + + if (ichdr_freemaps_overlap(ichdr, 0, 1)) + return __this_address; + if (ichdr_freemaps_overlap(ichdr, 0, 2)) + return __this_address; + if (ichdr_freemaps_overlap(ichdr, 1, 2)) + return __this_address; + + for (i = 0; i < XFS_ATTR_LEAF_MAPSIZE; i++) { + if (ichdr->freemap[i].size > 0 && + ichdr->freemap[i].base < entries_end) + return __this_address; + } + + return NULL; +} + +/* + * attr3 block 'firstused' conversion helpers. + * + * firstused refers to the offset of the first used byte of the nameval region + * of an attr leaf block. The region starts at the tail of the block and expands + * backwards towards the middle. As such, firstused is initialized to the block + * size for an empty leaf block and is reduced from there. + * + * The attr3 block size is pegged to the fsb size and the maximum fsb is 64k. + * The in-core firstused field is 32-bit and thus supports the maximum fsb size. + * The on-disk field is only 16-bit, however, and overflows at 64k. Since this + * only occurs at exactly 64k, we use zero as a magic on-disk value to represent + * the attr block size. The following helpers manage the conversion between the + * in-core and on-disk formats. + */ + +static void +xfs_attr3_leaf_firstused_from_disk( + struct xfs_da_geometry *geo, + struct xfs_attr3_icleaf_hdr *to, + struct xfs_attr_leafblock *from) +{ + struct xfs_attr3_leaf_hdr *hdr3; + + if (from->hdr.info.magic == cpu_to_be16(XFS_ATTR3_LEAF_MAGIC)) { + hdr3 = (struct xfs_attr3_leaf_hdr *) from; + to->firstused = be16_to_cpu(hdr3->firstused); + } else { + to->firstused = be16_to_cpu(from->hdr.firstused); + } + + /* + * Convert from the magic fsb size value to actual blocksize. This + * should only occur for empty blocks when the block size overflows + * 16-bits. + */ + if (to->firstused == XFS_ATTR3_LEAF_NULLOFF) { + ASSERT(!to->count && !to->usedbytes); + ASSERT(geo->blksize > USHRT_MAX); + to->firstused = geo->blksize; + } +} + +static void +xfs_attr3_leaf_firstused_to_disk( + struct xfs_da_geometry *geo, + struct xfs_attr_leafblock *to, + struct xfs_attr3_icleaf_hdr *from) +{ + struct xfs_attr3_leaf_hdr *hdr3; + uint32_t firstused; + + /* magic value should only be seen on disk */ + ASSERT(from->firstused != XFS_ATTR3_LEAF_NULLOFF); + + /* + * Scale down the 32-bit in-core firstused value to the 16-bit on-disk + * value. This only overflows at the max supported value of 64k. Use the + * magic on-disk value to represent block size in this case. + */ + firstused = from->firstused; + if (firstused > USHRT_MAX) { + ASSERT(from->firstused == geo->blksize); + firstused = XFS_ATTR3_LEAF_NULLOFF; + } + + if (from->magic == XFS_ATTR3_LEAF_MAGIC) { + hdr3 = (struct xfs_attr3_leaf_hdr *) to; + hdr3->firstused = cpu_to_be16(firstused); + } else { + to->hdr.firstused = cpu_to_be16(firstused); + } +} + +void +xfs_attr3_leaf_hdr_from_disk( + struct xfs_da_geometry *geo, + struct xfs_attr3_icleaf_hdr *to, + struct xfs_attr_leafblock *from) +{ + int i; + + ASSERT(from->hdr.info.magic == cpu_to_be16(XFS_ATTR_LEAF_MAGIC) || + from->hdr.info.magic == cpu_to_be16(XFS_ATTR3_LEAF_MAGIC)); + + if (from->hdr.info.magic == cpu_to_be16(XFS_ATTR3_LEAF_MAGIC)) { + struct xfs_attr3_leaf_hdr *hdr3 = (struct xfs_attr3_leaf_hdr *)from; + + to->forw = be32_to_cpu(hdr3->info.hdr.forw); + to->back = be32_to_cpu(hdr3->info.hdr.back); + to->magic = be16_to_cpu(hdr3->info.hdr.magic); + to->count = be16_to_cpu(hdr3->count); + to->usedbytes = be16_to_cpu(hdr3->usedbytes); + xfs_attr3_leaf_firstused_from_disk(geo, to, from); + to->holes = hdr3->holes; + + for (i = 0; i < XFS_ATTR_LEAF_MAPSIZE; i++) { + to->freemap[i].base = be16_to_cpu(hdr3->freemap[i].base); + to->freemap[i].size = be16_to_cpu(hdr3->freemap[i].size); + } + return; + } + to->forw = be32_to_cpu(from->hdr.info.forw); + to->back = be32_to_cpu(from->hdr.info.back); + to->magic = be16_to_cpu(from->hdr.info.magic); + to->count = be16_to_cpu(from->hdr.count); + to->usedbytes = be16_to_cpu(from->hdr.usedbytes); + xfs_attr3_leaf_firstused_from_disk(geo, to, from); + to->holes = from->hdr.holes; + + for (i = 0; i < XFS_ATTR_LEAF_MAPSIZE; i++) { + to->freemap[i].base = be16_to_cpu(from->hdr.freemap[i].base); + to->freemap[i].size = be16_to_cpu(from->hdr.freemap[i].size); + } +} + +void +xfs_attr3_leaf_hdr_to_disk( + struct xfs_da_geometry *geo, + struct xfs_attr_leafblock *to, + struct xfs_attr3_icleaf_hdr *from) +{ + int i; + + ASSERT(from->magic == XFS_ATTR_LEAF_MAGIC || + from->magic == XFS_ATTR3_LEAF_MAGIC); + + if (from->magic == XFS_ATTR3_LEAF_MAGIC) { + struct xfs_attr3_leaf_hdr *hdr3 = (struct xfs_attr3_leaf_hdr *)to; + + hdr3->info.hdr.forw = cpu_to_be32(from->forw); + hdr3->info.hdr.back = cpu_to_be32(from->back); + hdr3->info.hdr.magic = cpu_to_be16(from->magic); + hdr3->count = cpu_to_be16(from->count); + hdr3->usedbytes = cpu_to_be16(from->usedbytes); + xfs_attr3_leaf_firstused_to_disk(geo, to, from); + hdr3->holes = from->holes; + hdr3->pad1 = 0; + + for (i = 0; i < XFS_ATTR_LEAF_MAPSIZE; i++) { + hdr3->freemap[i].base = cpu_to_be16(from->freemap[i].base); + hdr3->freemap[i].size = cpu_to_be16(from->freemap[i].size); + } + + ASSERT(xfs_attr_leaf_ichdr_freemaps_verify(from, to) == NULL); + return; + } + to->hdr.info.forw = cpu_to_be32(from->forw); + to->hdr.info.back = cpu_to_be32(from->back); + to->hdr.info.magic = cpu_to_be16(from->magic); + to->hdr.count = cpu_to_be16(from->count); + to->hdr.usedbytes = cpu_to_be16(from->usedbytes); + xfs_attr3_leaf_firstused_to_disk(geo, to, from); + to->hdr.holes = from->holes; + to->hdr.pad1 = 0; + + for (i = 0; i < XFS_ATTR_LEAF_MAPSIZE; i++) { + to->hdr.freemap[i].base = cpu_to_be16(from->freemap[i].base); + to->hdr.freemap[i].size = cpu_to_be16(from->freemap[i].size); + } + + ASSERT(xfs_attr_leaf_ichdr_freemaps_verify(from, to) == NULL); +} + +static xfs_failaddr_t +xfs_attr3_leaf_verify_entry( + struct xfs_mount *mp, + char *buf_end, + struct xfs_attr_leafblock *leaf, + struct xfs_attr3_icleaf_hdr *leafhdr, + struct xfs_attr_leaf_entry *ent, + int idx, + __u32 *last_hashval) +{ + struct xfs_attr_leaf_name_local *lentry; + struct xfs_attr_leaf_name_remote *rentry; + char *name_end; + unsigned int nameidx; + unsigned int namesize; + __u32 hashval; + + /* hash order check */ + hashval = be32_to_cpu(ent->hashval); + if (hashval < *last_hashval) + return __this_address; + *last_hashval = hashval; + + nameidx = be16_to_cpu(ent->nameidx); + if (nameidx < leafhdr->firstused || nameidx >= mp->m_attr_geo->blksize) + return __this_address; + + /* + * Check the name information. The namelen fields are u8 so we can't + * possibly exceed the maximum name length of 255 bytes. + */ + if (ent->flags & XFS_ATTR_LOCAL) { + lentry = xfs_attr3_leaf_name_local(leaf, idx); + + /* Validate lentry pointer is within bounds before field access */ + if ((char *)lentry >= buf_end) + return __this_address; + if ((char *)lentry + offsetof(struct xfs_attr_leaf_name_local, nameval) > buf_end) + return __this_address; + + namesize = xfs_attr_leaf_entsize_local(lentry->namelen, + be16_to_cpu(lentry->valuelen)); + name_end = (char *)lentry + namesize; + if (lentry->namelen == 0) + return __this_address; + } else { + rentry = xfs_attr3_leaf_name_remote(leaf, idx); + + /* Validate rentry pointer is within bounds before field access */ + if ((char *)rentry >= buf_end) + return __this_address; + if ((char *)rentry + offsetof(struct xfs_attr_leaf_name_remote, name) > buf_end) + return __this_address; + + namesize = xfs_attr_leaf_entsize_remote(rentry->namelen); + name_end = (char *)rentry + namesize; + if (rentry->namelen == 0) + return __this_address; + if (!(ent->flags & XFS_ATTR_INCOMPLETE) && + rentry->valueblk == 0) + return __this_address; + } + + if (name_end > buf_end) + return __this_address; + + return NULL; +} + +/* + * Validate an attribute leaf block. + * + * Empty leaf blocks can occur under the following circumstances: + * + * 1. setxattr adds a new extended attribute to a file; + * 2. The file has zero existing attributes; + * 3. The attribute is too large to fit in the attribute fork; + * 4. The attribute is small enough to fit in a leaf block; + * 5. A log flush occurs after committing the transaction that creates + * the (empty) leaf block; and + * 6. The filesystem goes down after the log flush but before the new + * attribute can be committed to the leaf block. + * + * Hence we need to ensure that we don't fail the validation purely + * because the leaf is empty. + */ +static xfs_failaddr_t +xfs_attr3_leaf_verify( + struct xfs_buf *bp) +{ + struct xfs_attr3_icleaf_hdr ichdr; + struct xfs_mount *mp = bp->b_mount; + struct xfs_attr_leafblock *leaf = bp->b_addr; + struct xfs_attr_leaf_entry *entries; + struct xfs_attr_leaf_entry *ent; + char *buf_end; + uint32_t end; /* must be 32bit - see below */ + __u32 last_hashval = 0; + int i; + xfs_failaddr_t fa; + + xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, &ichdr, leaf); + + fa = xfs_da3_blkinfo_verify(bp, bp->b_addr); + if (fa) + return fa; + + /* + * firstused is the block offset of the first name info structure. + * Make sure it doesn't go off the block or crash into the header. + */ + if (ichdr.firstused > mp->m_attr_geo->blksize) + return __this_address; + if (ichdr.firstused < xfs_attr3_leaf_hdr_size(leaf)) + return __this_address; + + /* Make sure the entries array doesn't crash into the name info. */ + entries = xfs_attr3_leaf_entryp(bp->b_addr); + if ((char *)&entries[ichdr.count] > + (char *)bp->b_addr + ichdr.firstused) + return __this_address; + + /* + * NOTE: This verifier historically failed empty leaf buffers because + * we expect the fork to be in another format. Empty attr fork format + * conversions are possible during xattr set, however, and format + * conversion is not atomic with the xattr set that triggers it. We + * cannot assume leaf blocks are non-empty until that is addressed. + */ + buf_end = (char *)bp->b_addr + mp->m_attr_geo->blksize; + for (i = 0, ent = entries; i < ichdr.count; ent++, i++) { + fa = xfs_attr3_leaf_verify_entry(mp, buf_end, leaf, &ichdr, + ent, i, &last_hashval); + if (fa) + return fa; + } + + /* + * Quickly check the freemap information. Attribute data has to be + * aligned to 4-byte boundaries, and likewise for the free space. + * + * Note that for 64k block size filesystems, the freemap entries cannot + * overflow as they are only be16 fields. However, when checking end + * pointer of the freemap, we have to be careful to detect overflows and + * so use uint32_t for those checks. + */ + for (i = 0; i < XFS_ATTR_LEAF_MAPSIZE; i++) { + if (ichdr.freemap[i].base > mp->m_attr_geo->blksize) + return __this_address; + if (ichdr.freemap[i].base & 0x3) + return __this_address; + if (ichdr.freemap[i].size > mp->m_attr_geo->blksize) + return __this_address; + if (ichdr.freemap[i].size & 0x3) + return __this_address; + + /* be care of 16 bit overflows here */ + end = (uint32_t)ichdr.freemap[i].base + ichdr.freemap[i].size; + if (end < ichdr.freemap[i].base) + return __this_address; + if (end > mp->m_attr_geo->blksize) + return __this_address; + } + + fa = xfs_attr_leaf_ichdr_freemaps_verify(&ichdr, leaf); + if (fa) + return fa; + + return NULL; +} + +xfs_failaddr_t +xfs_attr3_leaf_header_check( + struct xfs_buf *bp, + xfs_ino_t owner) +{ + struct xfs_mount *mp = bp->b_mount; + + if (xfs_has_crc(mp)) { + struct xfs_attr3_leafblock *hdr3 = bp->b_addr; + + if (hdr3->hdr.info.hdr.magic != + cpu_to_be16(XFS_ATTR3_LEAF_MAGIC)) + return __this_address; + + if (be64_to_cpu(hdr3->hdr.info.owner) != owner) + return __this_address; + } + + return NULL; +} + +static void +xfs_attr3_leaf_write_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_buf_log_item *bip = bp->b_log_item; + struct xfs_attr3_leaf_hdr *hdr3 = bp->b_addr; + xfs_failaddr_t fa; + + fa = xfs_attr3_leaf_verify(bp); + if (fa) { + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + return; + } + + if (!xfs_has_crc(mp)) + return; + + if (bip) + hdr3->info.lsn = cpu_to_be64(bip->bli_item.li_lsn); + + xfs_buf_update_cksum(bp, XFS_ATTR3_LEAF_CRC_OFF); +} + +/* + * leaf/node format detection on trees is sketchy, so a node read can be done on + * leaf level blocks when detection identifies the tree as a node format tree + * incorrectly. In this case, we need to swap the verifier to match the correct + * format of the block being read. + */ +static void +xfs_attr3_leaf_read_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + xfs_failaddr_t fa; + + if (xfs_has_crc(mp) && + !xfs_buf_verify_cksum(bp, XFS_ATTR3_LEAF_CRC_OFF)) + xfs_verifier_error(bp, -EFSBADCRC, __this_address); + else { + fa = xfs_attr3_leaf_verify(bp); + if (fa) + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + } +} + +const struct xfs_buf_ops xfs_attr3_leaf_buf_ops = { + .name = "xfs_attr3_leaf", + .magic16 = { cpu_to_be16(XFS_ATTR_LEAF_MAGIC), + cpu_to_be16(XFS_ATTR3_LEAF_MAGIC) }, + .verify_read = xfs_attr3_leaf_read_verify, + .verify_write = xfs_attr3_leaf_write_verify, + .verify_struct = xfs_attr3_leaf_verify, +}; + +int +xfs_attr3_leaf_read( + struct xfs_trans *tp, + struct xfs_inode *dp, + xfs_ino_t owner, + xfs_dablk_t bno, + struct xfs_buf **bpp) +{ + xfs_failaddr_t fa; + int err; + + err = xfs_da_read_buf(tp, dp, bno, 0, bpp, XFS_ATTR_FORK, + &xfs_attr3_leaf_buf_ops); + if (err || !(*bpp)) + return err; + + fa = xfs_attr3_leaf_header_check(*bpp, owner); + if (fa) { + __xfs_buf_mark_corrupt(*bpp, fa); + xfs_trans_brelse(tp, *bpp); + *bpp = NULL; + xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); + return -EFSCORRUPTED; + } + + if (tp) + xfs_trans_buf_set_type(tp, *bpp, XFS_BLFT_ATTR_LEAF_BUF); + return 0; +} + +/*======================================================================== + * Namespace helper routines + *========================================================================*/ + +/* + * If we are in log recovery, then we want the lookup to ignore the INCOMPLETE + * flag on disk - if there's an incomplete attr then recovery needs to tear it + * down. If there's no incomplete attr, then recovery needs to tear that attr + * down to replace it with the attr that has been logged. In this case, the + * INCOMPLETE flag will not be set in attr->attr_filter, but rather + * XFS_DA_OP_RECOVERY will be set in args->op_flags. + */ +static inline unsigned int xfs_attr_match_mask(const struct xfs_da_args *args) +{ + if (args->op_flags & XFS_DA_OP_RECOVERY) + return XFS_ATTR_NSP_ONDISK_MASK; + return XFS_ATTR_NSP_ONDISK_MASK | XFS_ATTR_INCOMPLETE; +} + +static inline bool +xfs_attr_parent_match( + const struct xfs_da_args *args, + const void *value, + unsigned int valuelen) +{ + ASSERT(args->value != NULL); + + /* Parent pointers do not use remote values */ + if (!value) + return false; + + /* + * The only value we support is a parent rec. However, we'll accept + * any valuelen so that offline repair can delete ATTR_PARENT values + * that are not parent pointers. + */ + if (valuelen != args->valuelen) + return false; + + return memcmp(args->value, value, valuelen) == 0; +} + +static bool +xfs_attr_match( + struct xfs_da_args *args, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + const void *value, + unsigned int valuelen) +{ + unsigned int mask = xfs_attr_match_mask(args); + + if (args->namelen != namelen) + return false; + if ((args->attr_filter & mask) != (attr_flags & mask)) + return false; + if (memcmp(args->name, name, namelen) != 0) + return false; + + if (attr_flags & XFS_ATTR_PARENT) + return xfs_attr_parent_match(args, value, valuelen); + + return true; +} + +static int +xfs_attr_copy_value( + struct xfs_da_args *args, + unsigned char *value, + int valuelen) +{ + /* + * Parent pointer lookups require the caller to specify the name and + * value, so don't copy anything. + */ + if (args->attr_filter & XFS_ATTR_PARENT) + return 0; + + /* + * No copy if all we have to do is get the length + */ + if (!args->valuelen) { + args->valuelen = valuelen; + return 0; + } + + /* + * No copy if the length of the existing buffer is too small + */ + if (args->valuelen < valuelen) { + args->valuelen = valuelen; + return -ERANGE; + } + + if (!args->value) { + args->value = kvmalloc(valuelen, GFP_KERNEL | __GFP_NOLOCKDEP); + if (!args->value) + return -ENOMEM; + } + args->valuelen = valuelen; + + /* remote block xattr requires IO for copy-in */ + if (args->rmtblkno) + return xfs_attr_rmtval_get(args); + + /* + * This is to prevent a GCC warning because the remote xattr case + * doesn't have a value to pass in. In that case, we never reach here, + * but GCC can't work that out and so throws a "passing NULL to + * memcpy" warning. + */ + if (!value) + return -EINVAL; + memcpy(args->value, value, valuelen); + return 0; +} + +/*======================================================================== + * External routines when attribute fork size < XFS_LITINO(mp). + *========================================================================*/ + +/* + * Query whether the total requested number of attr fork bytes of extended + * attribute space will be able to fit inline. + * + * Returns zero if not, else the i_forkoff fork offset to be used in the + * literal area for attribute data once the new bytes have been added. + * + * i_forkoff must be 8 byte aligned, hence is stored as a >>3 value; + * special case for dev/uuid inodes, they have fixed size data forks. + */ +int +xfs_attr_shortform_bytesfit( + struct xfs_inode *dp, + int bytes) +{ + struct xfs_mount *mp = dp->i_mount; + int64_t dsize; + int minforkoff; + int maxforkoff; + int offset; + + /* + * Check if the new size could fit at all first: + */ + if (bytes > XFS_LITINO(mp)) + return 0; + + /* rounded down */ + offset = (XFS_LITINO(mp) - bytes) >> 3; + + if (dp->i_df.if_format == XFS_DINODE_FMT_DEV) { + minforkoff = roundup(sizeof(xfs_dev_t), 8) >> 3; + return (offset >= minforkoff) ? minforkoff : 0; + } + + /* + * If the requested numbers of bytes is smaller or equal to the + * current attribute fork size we can always proceed. + * + * Note that if_bytes in the data fork might actually be larger than + * the current data fork size is due to delalloc extents. In that + * case either the extent count will go down when they are converted + * to real extents, or the delalloc conversion will take care of the + * literal area rebalancing. + */ + if (bytes <= xfs_inode_attr_fork_size(dp)) + return dp->i_forkoff; + + /* + * For attr2 we can try to move the forkoff if there is space in the + * literal area + */ + dsize = dp->i_df.if_bytes; + + switch (dp->i_df.if_format) { + case XFS_DINODE_FMT_EXTENTS: + /* + * If there is no attr fork and the data fork is extents, + * determine if creating the default attr fork will result + * in the extents form migrating to btree. If so, the + * minimum offset only needs to be the space required for + * the btree root. + */ + if (!dp->i_forkoff && dp->i_df.if_bytes > + xfs_default_attroffset(dp)) + dsize = xfs_bmdr_space_calc(MINDBTPTRS); + break; + case XFS_DINODE_FMT_BTREE: + /* + * If we have a data btree then keep forkoff if we have one, + * otherwise we are adding a new attr, so then we set + * minforkoff to where the btree root can finish so we have + * plenty of room for attrs + */ + if (dp->i_forkoff) { + if (offset < dp->i_forkoff) + return 0; + return dp->i_forkoff; + } + dsize = xfs_bmap_bmdr_space(dp->i_df.if_broot); + break; + } + + /* + * A data fork btree root must have space for at least + * MINDBTPTRS key/ptr pairs if the data fork is small or empty. + */ + minforkoff = max_t(int64_t, dsize, xfs_bmdr_space_calc(MINDBTPTRS)); + minforkoff = roundup(minforkoff, 8) >> 3; + + /* attr fork btree root can have at least this many key/ptr pairs */ + maxforkoff = XFS_LITINO(mp) - xfs_bmdr_space_calc(MINABTPTRS); + maxforkoff = maxforkoff >> 3; /* rounded down */ + + if (offset >= maxforkoff) + return maxforkoff; + if (offset >= minforkoff) + return offset; + return 0; +} + +/* + * Switch on the ATTR2 superblock bit (implies also FEATURES2) unless + * on-disk version bit says it is already set + */ +STATIC void +xfs_sbversion_add_attr2( + struct xfs_mount *mp, + struct xfs_trans *tp) +{ + if (mp->m_sb.sb_features2 & XFS_SB_VERSION2_ATTR2BIT) + return; + + spin_lock(&mp->m_sb_lock); + xfs_add_attr2(mp); + spin_unlock(&mp->m_sb_lock); + xfs_log_sb(tp); +} + +/* + * Create the initial contents of a shortform attribute list. + */ +void +xfs_attr_shortform_create( + struct xfs_da_args *args) +{ + struct xfs_inode *dp = args->dp; + struct xfs_ifork *ifp = &dp->i_af; + struct xfs_attr_sf_hdr *hdr; + + trace_xfs_attr_sf_create(args); + + ASSERT(ifp->if_bytes == 0); + if (ifp->if_format == XFS_DINODE_FMT_EXTENTS) + ifp->if_format = XFS_DINODE_FMT_LOCAL; + + hdr = xfs_idata_realloc(dp, sizeof(*hdr), XFS_ATTR_FORK); + memset(hdr, 0, sizeof(*hdr)); + hdr->totsize = cpu_to_be16(sizeof(*hdr)); + xfs_trans_log_inode(args->trans, dp, XFS_ILOG_CORE | XFS_ILOG_ADATA); +} + +/* + * Return the entry if the attr in args is found, or NULL if not. + */ +struct xfs_attr_sf_entry * +xfs_attr_sf_findname( + struct xfs_da_args *args) +{ + struct xfs_attr_sf_hdr *sf = args->dp->i_af.if_data; + struct xfs_attr_sf_entry *sfe; + + for (sfe = xfs_attr_sf_firstentry(sf); + sfe < xfs_attr_sf_endptr(sf); + sfe = xfs_attr_sf_nextentry(sfe)) { + if (xfs_attr_match(args, sfe->flags, sfe->nameval, + sfe->namelen, &sfe->nameval[sfe->namelen], + sfe->valuelen)) + return sfe; + } + + return NULL; +} + +/* + * Replace a shortform xattr if it's the right length. Returns 0 on success, + * -ENOSPC if the length is wrong, or -ENOATTR if the attr was not found. + */ +int +xfs_attr_shortform_replace( + struct xfs_da_args *args) +{ + struct xfs_attr_sf_entry *sfe; + + ASSERT(args->dp->i_af.if_format == XFS_DINODE_FMT_LOCAL); + + trace_xfs_attr_sf_replace(args); + + sfe = xfs_attr_sf_findname(args); + if (!sfe) + return -ENOATTR; + + if (args->attr_filter & XFS_ATTR_PARENT) { + if (sfe->namelen != args->new_namelen || + sfe->valuelen != args->new_valuelen) + return -ENOSPC; + + memcpy(sfe->nameval, args->new_name, sfe->namelen); + memcpy(&sfe->nameval[sfe->namelen], args->new_value, + sfe->valuelen); + } else { + if (sfe->valuelen != args->valuelen) + return -ENOSPC; + memcpy(&sfe->nameval[sfe->namelen], args->value, + sfe->valuelen); + } + + xfs_trans_log_inode(args->trans, args->dp, + XFS_ILOG_CORE | XFS_ILOG_ADATA); + return 0; +} + +/* + * Add a name/value pair to the shortform attribute list. + * Overflow from the inode has already been checked for. + */ +void +xfs_attr_shortform_add( + struct xfs_da_args *args, + int forkoff) +{ + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + struct xfs_ifork *ifp = &dp->i_af; + struct xfs_attr_sf_hdr *sf = ifp->if_data; + struct xfs_attr_sf_entry *sfe; + int size; + + trace_xfs_attr_sf_add(args); + + dp->i_forkoff = forkoff; + + ASSERT(ifp->if_format == XFS_DINODE_FMT_LOCAL); + ASSERT(!xfs_attr_sf_findname(args)); + + size = xfs_attr_sf_entsize_byname(args->namelen, args->valuelen); + sf = xfs_idata_realloc(dp, size, XFS_ATTR_FORK); + + sfe = xfs_attr_sf_endptr(sf); + sfe->namelen = args->namelen; + sfe->valuelen = args->valuelen; + sfe->flags = args->attr_filter; + memcpy(sfe->nameval, args->name, args->namelen); + memcpy(&sfe->nameval[args->namelen], args->value, args->valuelen); + sf->count++; + be16_add_cpu(&sf->totsize, size); + xfs_trans_log_inode(args->trans, dp, XFS_ILOG_CORE | XFS_ILOG_ADATA); + + xfs_sbversion_add_attr2(mp, args->trans); +} + +/* + * After the last attribute is removed revert to original inode format, + * making all literal area available to the data fork once more. + */ +void +xfs_attr_fork_remove( + struct xfs_inode *ip, + struct xfs_trans *tp) +{ + ASSERT(ip->i_af.if_nextents == 0); + + xfs_ifork_zap_attr(ip); + ip->i_forkoff = 0; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); +} + +/* + * Remove an attribute from the shortform attribute list structure. + */ +int +xfs_attr_sf_removename( + struct xfs_da_args *args) +{ + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + struct xfs_attr_sf_hdr *sf = dp->i_af.if_data; + struct xfs_attr_sf_entry *sfe; + uint16_t totsize = be16_to_cpu(sf->totsize); + void *next, *end; + int size = 0; + + trace_xfs_attr_sf_remove(args); + + sfe = xfs_attr_sf_findname(args); + if (!sfe) { + /* + * If we are recovering an operation, finding nothing to remove + * is not an error, it just means there was nothing to clean up. + */ + if (args->op_flags & XFS_DA_OP_RECOVERY) + return 0; + return -ENOATTR; + } + + /* + * Fix up the attribute fork data, covering the hole + */ + size = xfs_attr_sf_entsize(sfe); + next = xfs_attr_sf_nextentry(sfe); + end = xfs_attr_sf_endptr(sf); + if (next < end) + memmove(sfe, next, end - next); + sf->count--; + totsize -= size; + sf->totsize = cpu_to_be16(totsize); + + /* + * Fix up the start offset of the attribute fork + */ + if (totsize == sizeof(struct xfs_attr_sf_hdr) && + (dp->i_df.if_format != XFS_DINODE_FMT_BTREE) && + !(args->op_flags & (XFS_DA_OP_ADDNAME | XFS_DA_OP_REPLACE)) && + !xfs_has_parent(mp)) { + xfs_attr_fork_remove(dp, args->trans); + } else { + xfs_idata_realloc(dp, -size, XFS_ATTR_FORK); + dp->i_forkoff = xfs_attr_shortform_bytesfit(dp, totsize); + ASSERT(dp->i_forkoff); + ASSERT(totsize > sizeof(struct xfs_attr_sf_hdr) || + (args->op_flags & XFS_DA_OP_ADDNAME) || + dp->i_df.if_format == XFS_DINODE_FMT_BTREE || + xfs_has_parent(mp)); + xfs_trans_log_inode(args->trans, dp, + XFS_ILOG_CORE | XFS_ILOG_ADATA); + } + + xfs_sbversion_add_attr2(mp, args->trans); + + return 0; +} + +/* + * Retrieve the attribute value and length. + * + * If args->valuelen is zero, only the length needs to be returned. Unlike a + * lookup, we only return an error if the attribute does not exist or we can't + * retrieve the value. + */ +int +xfs_attr_shortform_getvalue( + struct xfs_da_args *args) +{ + struct xfs_attr_sf_entry *sfe; + + ASSERT(args->dp->i_af.if_format == XFS_DINODE_FMT_LOCAL); + + trace_xfs_attr_sf_lookup(args); + + sfe = xfs_attr_sf_findname(args); + if (!sfe) + return -ENOATTR; + return xfs_attr_copy_value(args, &sfe->nameval[args->namelen], + sfe->valuelen); +} + +/* Convert from using the shortform to the leaf format. */ +int +xfs_attr_shortform_to_leaf( + struct xfs_da_args *args) +{ + struct xfs_inode *dp = args->dp; + struct xfs_ifork *ifp = &dp->i_af; + struct xfs_attr_sf_hdr *sf = ifp->if_data; + struct xfs_attr_sf_entry *sfe; + int size = be16_to_cpu(sf->totsize); + struct xfs_da_args nargs; + char *tmpbuffer; + int error, i; + xfs_dablk_t blkno; + struct xfs_buf *bp; + + trace_xfs_attr_sf_to_leaf(args); + + tmpbuffer = kmalloc(size, GFP_KERNEL | __GFP_NOFAIL); + memcpy(tmpbuffer, ifp->if_data, size); + sf = (struct xfs_attr_sf_hdr *)tmpbuffer; + + xfs_idata_realloc(dp, -size, XFS_ATTR_FORK); + xfs_bmap_local_to_extents_empty(args->trans, dp, XFS_ATTR_FORK); + + bp = NULL; + error = xfs_da_grow_inode(args, &blkno); + if (error) + goto out; + + ASSERT(blkno == 0); + error = xfs_attr3_leaf_create(args, blkno, &bp); + if (error) + goto out; + + memset((char *)&nargs, 0, sizeof(nargs)); + nargs.dp = dp; + nargs.geo = args->geo; + nargs.total = args->total; + nargs.whichfork = XFS_ATTR_FORK; + nargs.trans = args->trans; + nargs.op_flags = XFS_DA_OP_OKNOENT; + nargs.owner = args->owner; + + sfe = xfs_attr_sf_firstentry(sf); + for (i = 0; i < sf->count; i++) { + nargs.name = sfe->nameval; + nargs.namelen = sfe->namelen; + nargs.value = &sfe->nameval[nargs.namelen]; + nargs.valuelen = sfe->valuelen; + nargs.attr_filter = sfe->flags & XFS_ATTR_NSP_ONDISK_MASK; + if (!xfs_attr_check_namespace(sfe->flags)) { + xfs_da_mark_sick(args); + error = -EFSCORRUPTED; + goto out; + } + xfs_attr_sethash(&nargs); + error = xfs_attr3_leaf_lookup_int(bp, &nargs); /* set a->index */ + ASSERT(error == -ENOATTR); + if (!xfs_attr3_leaf_add(bp, &nargs)) + ASSERT(0); + sfe = xfs_attr_sf_nextentry(sfe); + } + error = 0; +out: + kfree(tmpbuffer); + return error; +} + +/* + * Check a leaf attribute block to see if all the entries would fit into + * a shortform attribute list. + */ +int +xfs_attr_shortform_allfit( + struct xfs_buf *bp, + struct xfs_inode *dp) +{ + struct xfs_attr_leafblock *leaf; + struct xfs_attr_leaf_entry *entry; + xfs_attr_leaf_name_local_t *name_loc; + struct xfs_attr3_icleaf_hdr leafhdr; + int bytes; + int i; + struct xfs_mount *mp = bp->b_mount; + + leaf = bp->b_addr; + xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, &leafhdr, leaf); + entry = xfs_attr3_leaf_entryp(leaf); + + bytes = sizeof(struct xfs_attr_sf_hdr); + for (i = 0; i < leafhdr.count; entry++, i++) { + if (entry->flags & XFS_ATTR_INCOMPLETE) + continue; /* don't copy partial entries */ + if (!(entry->flags & XFS_ATTR_LOCAL)) + return 0; + name_loc = xfs_attr3_leaf_name_local(leaf, i); + if (name_loc->namelen >= XFS_ATTR_SF_ENTSIZE_MAX) + return 0; + if (be16_to_cpu(name_loc->valuelen) >= XFS_ATTR_SF_ENTSIZE_MAX) + return 0; + bytes += xfs_attr_sf_entsize_byname(name_loc->namelen, + be16_to_cpu(name_loc->valuelen)); + } + if ((dp->i_df.if_format != XFS_DINODE_FMT_BTREE) && + (bytes == sizeof(struct xfs_attr_sf_hdr))) + return -1; + return xfs_attr_shortform_bytesfit(dp, bytes); +} + +/* Verify the consistency of a raw inline attribute fork. */ +xfs_failaddr_t +xfs_attr_shortform_verify( + struct xfs_attr_sf_hdr *sfp, + size_t size) +{ + struct xfs_attr_sf_entry *sfep = xfs_attr_sf_firstentry(sfp); + struct xfs_attr_sf_entry *next_sfep; + char *endp; + int i; + + /* + * Give up if the attribute is way too short. + */ + if (size < sizeof(struct xfs_attr_sf_hdr)) + return __this_address; + + endp = (char *)sfp + size; + + /* Check all reported entries */ + for (i = 0; i < sfp->count; i++) { + /* + * struct xfs_attr_sf_entry has a variable length. + * Check the fixed-offset parts of the structure are + * within the data buffer. + * xfs_attr_sf_entry is defined with a 1-byte variable + * array at the end, so we must subtract that off. + */ + if (((char *)sfep + sizeof(*sfep)) >= endp) + return __this_address; + + /* Don't allow names with known bad length. */ + if (sfep->namelen == 0) + return __this_address; + + /* + * Check that the variable-length part of the structure is + * within the data buffer. The next entry starts after the + * name component, so nextentry is an acceptable test. + */ + next_sfep = xfs_attr_sf_nextentry(sfep); + if ((char *)next_sfep > endp) + return __this_address; + + /* + * Check for unknown flags. Short form doesn't support + * the incomplete or local bits, so we can use the namespace + * mask here. + */ + if (sfep->flags & ~XFS_ATTR_NSP_ONDISK_MASK) + return __this_address; + + /* + * Check for invalid namespace combinations. We only allow + * one namespace flag per xattr, so we can just count the + * bits (i.e. hweight) here. + */ + if (!xfs_attr_check_namespace(sfep->flags)) + return __this_address; + + sfep = next_sfep; + } + if ((void *)sfep != (void *)endp) + return __this_address; + + return NULL; +} + +/* + * Convert a leaf attribute list to shortform attribute list + */ +int +xfs_attr3_leaf_to_shortform( + struct xfs_buf *bp, + struct xfs_da_args *args, + int forkoff) +{ + struct xfs_attr_leafblock *leaf; + struct xfs_attr3_icleaf_hdr ichdr; + struct xfs_attr_leaf_entry *entry; + struct xfs_attr_leaf_name_local *name_loc; + struct xfs_da_args nargs; + struct xfs_inode *dp = args->dp; + char *tmpbuffer; + int error; + int i; + + trace_xfs_attr_leaf_to_sf(args); + + tmpbuffer = kvmalloc(args->geo->blksize, GFP_KERNEL | __GFP_NOFAIL); + memcpy(tmpbuffer, bp->b_addr, args->geo->blksize); + + leaf = (xfs_attr_leafblock_t *)tmpbuffer; + xfs_attr3_leaf_hdr_from_disk(args->geo, &ichdr, leaf); + entry = xfs_attr3_leaf_entryp(leaf); + + /* XXX (dgc): buffer is about to be marked stale - why zero it? */ + memset(bp->b_addr, 0, args->geo->blksize); + + /* + * Clean out the prior contents of the attribute list. + */ + error = xfs_da_shrink_inode(args, 0, bp); + if (error) + goto out; + + if (forkoff == -1) { + /* + * Don't remove the attr fork if this operation is the first + * part of a attr replace operations. We're going to add a new + * attr immediately, so we need to keep the attr fork around in + * this case. + */ + if (!(args->op_flags & XFS_DA_OP_REPLACE)) { + ASSERT(dp->i_df.if_format != XFS_DINODE_FMT_BTREE); + xfs_attr_fork_remove(dp, args->trans); + } + goto out; + } + + xfs_attr_shortform_create(args); + + /* + * Copy the attributes + */ + memset((char *)&nargs, 0, sizeof(nargs)); + nargs.geo = args->geo; + nargs.dp = dp; + nargs.total = args->total; + nargs.whichfork = XFS_ATTR_FORK; + nargs.trans = args->trans; + nargs.op_flags = XFS_DA_OP_OKNOENT; + nargs.owner = args->owner; + + for (i = 0; i < ichdr.count; entry++, i++) { + if (entry->flags & XFS_ATTR_INCOMPLETE) + continue; /* don't copy partial entries */ + if (!entry->nameidx) + continue; + ASSERT(entry->flags & XFS_ATTR_LOCAL); + name_loc = xfs_attr3_leaf_name_local(leaf, i); + nargs.name = name_loc->nameval; + nargs.namelen = name_loc->namelen; + nargs.value = &name_loc->nameval[nargs.namelen]; + nargs.valuelen = be16_to_cpu(name_loc->valuelen); + nargs.hashval = be32_to_cpu(entry->hashval); + nargs.attr_filter = entry->flags & XFS_ATTR_NSP_ONDISK_MASK; + xfs_attr_shortform_add(&nargs, forkoff); + } + error = 0; + +out: + kvfree(tmpbuffer); + return error; +} + +/* + * Convert from using a single leaf to a root node and a leaf. + */ +int +xfs_attr3_leaf_to_node( + struct xfs_da_args *args) +{ + struct xfs_attr_leafblock *leaf; + struct xfs_attr3_icleaf_hdr icleafhdr; + struct xfs_attr_leaf_entry *entries; + struct xfs_da3_icnode_hdr icnodehdr; + struct xfs_da_intnode *node; + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + struct xfs_buf *bp1 = NULL; + struct xfs_buf *bp2 = NULL; + xfs_dablk_t blkno; + int error; + + trace_xfs_attr_leaf_to_node(args); + + if (XFS_TEST_ERROR(mp, XFS_ERRTAG_ATTR_LEAF_TO_NODE)) { + error = -EIO; + goto out; + } + + error = xfs_da_grow_inode(args, &blkno); + if (error) + goto out; + error = xfs_attr3_leaf_read(args->trans, dp, args->owner, 0, &bp1); + if (error) + goto out; + + error = xfs_da_get_buf(args->trans, dp, blkno, &bp2, XFS_ATTR_FORK); + if (error) + goto out; + + /* + * Copy leaf to new buffer and log it. + */ + xfs_da_buf_copy(bp2, bp1, args->geo->blksize); + xfs_trans_log_buf(args->trans, bp2, 0, args->geo->blksize - 1); + + /* + * Set up the new root node. + */ + error = xfs_da3_node_create(args, 0, 1, &bp1, XFS_ATTR_FORK); + if (error) + goto out; + node = bp1->b_addr; + xfs_da3_node_hdr_from_disk(mp, &icnodehdr, node); + + leaf = bp2->b_addr; + xfs_attr3_leaf_hdr_from_disk(args->geo, &icleafhdr, leaf); + entries = xfs_attr3_leaf_entryp(leaf); + + /* both on-disk, don't endian-flip twice */ + icnodehdr.btree[0].hashval = entries[icleafhdr.count - 1].hashval; + icnodehdr.btree[0].before = cpu_to_be32(blkno); + icnodehdr.count = 1; + xfs_da3_node_hdr_to_disk(dp->i_mount, node, &icnodehdr); + xfs_trans_log_buf(args->trans, bp1, 0, args->geo->blksize - 1); + error = 0; +out: + return error; +} + +/*======================================================================== + * Routines used for growing the Btree. + *========================================================================*/ + +/* + * Create the initial contents of a leaf attribute list + * or a leaf in a node attribute list. + */ +STATIC int +xfs_attr3_leaf_create( + struct xfs_da_args *args, + xfs_dablk_t blkno, + struct xfs_buf **bpp) +{ + struct xfs_attr_leafblock *leaf; + struct xfs_attr3_icleaf_hdr ichdr; + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + struct xfs_buf *bp; + int error; + + trace_xfs_attr_leaf_create(args); + + error = xfs_da_get_buf(args->trans, args->dp, blkno, &bp, + XFS_ATTR_FORK); + if (error) + return error; + bp->b_ops = &xfs_attr3_leaf_buf_ops; + xfs_trans_buf_set_type(args->trans, bp, XFS_BLFT_ATTR_LEAF_BUF); + leaf = bp->b_addr; + memset(leaf, 0, args->geo->blksize); + + memset(&ichdr, 0, sizeof(ichdr)); + ichdr.firstused = args->geo->blksize; + + if (xfs_has_crc(mp)) { + struct xfs_da3_blkinfo *hdr3 = bp->b_addr; + + ichdr.magic = XFS_ATTR3_LEAF_MAGIC; + + hdr3->blkno = cpu_to_be64(xfs_buf_daddr(bp)); + hdr3->owner = cpu_to_be64(args->owner); + uuid_copy(&hdr3->uuid, &mp->m_sb.sb_meta_uuid); + + ichdr.freemap[0].base = sizeof(struct xfs_attr3_leaf_hdr); + } else { + ichdr.magic = XFS_ATTR_LEAF_MAGIC; + ichdr.freemap[0].base = sizeof(struct xfs_attr_leaf_hdr); + } + ichdr.freemap[0].size = ichdr.firstused - ichdr.freemap[0].base; + + xfs_attr3_leaf_hdr_to_disk(args->geo, leaf, &ichdr); + xfs_trans_log_buf(args->trans, bp, 0, args->geo->blksize - 1); + + *bpp = bp; + return 0; +} + +/* + * Reinitialize an existing attr fork block as an empty leaf, and attach + * the buffer to tp. + */ +int +xfs_attr3_leaf_init( + struct xfs_trans *tp, + struct xfs_inode *dp, + xfs_dablk_t blkno) +{ + struct xfs_buf *bp = NULL; + struct xfs_da_args args = { + .trans = tp, + .dp = dp, + .owner = I_INO(dp), + .geo = dp->i_mount->m_attr_geo, + }; + + ASSERT(tp != NULL); + + return xfs_attr3_leaf_create(&args, blkno, &bp); +} +/* + * Split the leaf node, rebalance, then add the new entry. + * + * Returns 0 if the entry was added, 1 if a further split is needed or a + * negative error number otherwise. + */ +int +xfs_attr3_leaf_split( + struct xfs_da_state *state, + struct xfs_da_state_blk *oldblk, + struct xfs_da_state_blk *newblk) +{ + bool added; + xfs_dablk_t blkno; + int error; + + trace_xfs_attr_leaf_split(state->args); + + /* + * Allocate space for a new leaf node. + */ + ASSERT(oldblk->magic == XFS_ATTR_LEAF_MAGIC); + error = xfs_da_grow_inode(state->args, &blkno); + if (error) + return error; + error = xfs_attr3_leaf_create(state->args, blkno, &newblk->bp); + if (error) + return error; + newblk->blkno = blkno; + newblk->magic = XFS_ATTR_LEAF_MAGIC; + + /* + * Rebalance the entries across the two leaves. + * NOTE: rebalance() currently depends on the 2nd block being empty. + */ + xfs_attr3_leaf_rebalance(state, oldblk, newblk); + error = xfs_da3_blk_link(state, oldblk, newblk); + if (error) + return error; + + /* + * Save info on "old" attribute for "atomic rename" ops, leaf_add() + * modifies the index/blkno/rmtblk/rmtblkcnt fields to show the + * "new" attrs info. Will need the "old" info to remove it later. + * + * Insert the "new" entry in the correct block. + */ + if (state->inleaf) { + trace_xfs_attr_leaf_add_old(state->args); + added = xfs_attr3_leaf_add(oldblk->bp, state->args); + } else { + trace_xfs_attr_leaf_add_new(state->args); + added = xfs_attr3_leaf_add(newblk->bp, state->args); + } + + /* + * Update last hashval in each block since we added the name. + */ + oldblk->hashval = xfs_attr_leaf_lasthash(oldblk->bp, NULL); + newblk->hashval = xfs_attr_leaf_lasthash(newblk->bp, NULL); + if (!added) + return 1; + return 0; +} + +/* + * Add a name to the leaf attribute list structure. + */ +bool +xfs_attr3_leaf_add( + struct xfs_buf *bp, + struct xfs_da_args *args) +{ + struct xfs_attr_leafblock *leaf; + struct xfs_attr3_icleaf_hdr ichdr; + int tablesize; + int entsize; + bool added = true; + int sum; + int tmp; + int i; + + trace_xfs_attr_leaf_add(args); + + leaf = bp->b_addr; + xfs_attr3_leaf_hdr_from_disk(args->geo, &ichdr, leaf); + ASSERT(args->index >= 0 && args->index <= ichdr.count); + entsize = xfs_attr_leaf_newentsize(args, NULL); + + /* + * Search through freemap for first-fit on new name length. + * (may need to figure in size of entry struct too) + */ + tablesize = xfs_attr_leaf_entries_end(ichdr.count + 1, leaf); + for (sum = 0, i = XFS_ATTR_LEAF_MAPSIZE - 1; i >= 0; i--) { + if (tablesize > ichdr.firstused) { + sum += ichdr.freemap[i].size; + continue; + } + if (!ichdr.freemap[i].size) + continue; /* no space in this map */ + tmp = entsize; + if (ichdr.freemap[i].base < ichdr.firstused) + tmp += sizeof(xfs_attr_leaf_entry_t); + if (ichdr.freemap[i].size >= tmp) { + xfs_attr3_leaf_add_work(bp, &ichdr, args, i); + goto out_log_hdr; + } + sum += ichdr.freemap[i].size; + } + + /* + * If there are no holes in the address space of the block, + * and we don't have enough freespace, then compaction will do us + * no good and we should just give up. + */ + if (!ichdr.holes && sum < entsize) + return false; + + /* + * Compact the entries to coalesce free space. + * This may change the hdr->count via dropping INCOMPLETE entries. + */ + xfs_attr3_leaf_compact(args, &ichdr, bp); + + /* + * After compaction, the block is guaranteed to have only one + * free region, in freemap[0]. If it is not big enough, give up. + */ + if (ichdr.freemap[0].size < (entsize + sizeof(xfs_attr_leaf_entry_t))) { + added = false; + goto out_log_hdr; + } + + xfs_attr3_leaf_add_work(bp, &ichdr, args, 0); + +out_log_hdr: + xfs_attr3_leaf_hdr_to_disk(args->geo, leaf, &ichdr); + xfs_trans_log_buf(args->trans, bp, + XFS_DA_LOGRANGE(leaf, &leaf->hdr, + xfs_attr3_leaf_hdr_size(leaf))); + return added; +} + +/* + * Add a name to a leaf attribute list structure. + */ +STATIC void +xfs_attr3_leaf_add_work( + struct xfs_buf *bp, + struct xfs_attr3_icleaf_hdr *ichdr, + struct xfs_da_args *args, + int mapindex) +{ + struct xfs_attr_leafblock *leaf; + struct xfs_attr_leaf_entry *entry; + struct xfs_attr_leaf_name_local *name_loc; + struct xfs_attr_leaf_name_remote *name_rmt; + struct xfs_mount *mp; + int old_end, new_end; + int tmp; + int i; + + trace_xfs_attr_leaf_add_work(args); + + leaf = bp->b_addr; + ASSERT(mapindex >= 0 && mapindex < XFS_ATTR_LEAF_MAPSIZE); + ASSERT(args->index >= 0 && args->index <= ichdr->count); + + /* + * Force open some space in the entry array and fill it in. + */ + entry = &xfs_attr3_leaf_entryp(leaf)[args->index]; + if (args->index < ichdr->count) { + tmp = ichdr->count - args->index; + tmp *= sizeof(xfs_attr_leaf_entry_t); + memmove(entry + 1, entry, tmp); + xfs_trans_log_buf(args->trans, bp, + XFS_DA_LOGRANGE(leaf, entry, tmp + sizeof(*entry))); + } + ichdr->count++; + + /* + * Allocate space for the new string (at the end of the run). + */ + mp = args->trans->t_mountp; + ASSERT(ichdr->freemap[mapindex].base < args->geo->blksize); + ASSERT((ichdr->freemap[mapindex].base & 0x3) == 0); + ASSERT(ichdr->freemap[mapindex].size >= + xfs_attr_leaf_newentsize(args, NULL)); + ASSERT(ichdr->freemap[mapindex].size < args->geo->blksize); + ASSERT((ichdr->freemap[mapindex].size & 0x3) == 0); + + ichdr->freemap[mapindex].size -= xfs_attr_leaf_newentsize(args, &tmp); + + entry->nameidx = cpu_to_be16(ichdr->freemap[mapindex].base + + ichdr->freemap[mapindex].size); + entry->hashval = cpu_to_be32(args->hashval); + entry->flags = args->attr_filter; + if (tmp) + entry->flags |= XFS_ATTR_LOCAL; + if (args->op_flags & XFS_DA_OP_REPLACE) { + if (!(args->op_flags & XFS_DA_OP_LOGGED)) + entry->flags |= XFS_ATTR_INCOMPLETE; + if ((args->blkno2 == args->blkno) && + (args->index2 <= args->index)) { + args->index2++; + } + } + xfs_trans_log_buf(args->trans, bp, + XFS_DA_LOGRANGE(leaf, entry, sizeof(*entry))); + ASSERT((args->index == 0) || + (be32_to_cpu(entry->hashval) >= be32_to_cpu((entry-1)->hashval))); + ASSERT((args->index == ichdr->count - 1) || + (be32_to_cpu(entry->hashval) <= be32_to_cpu((entry+1)->hashval))); + + /* + * For "remote" attribute values, simply note that we need to + * allocate space for the "remote" value. We can't actually + * allocate the extents in this transaction, and we can't decide + * which blocks they should be as we might allocate more blocks + * as part of this transaction (a split operation for example). + */ + if (entry->flags & XFS_ATTR_LOCAL) { + name_loc = xfs_attr3_leaf_name_local(leaf, args->index); + name_loc->namelen = args->namelen; + name_loc->valuelen = cpu_to_be16(args->valuelen); + memcpy((char *)name_loc->nameval, args->name, args->namelen); + memcpy((char *)&name_loc->nameval[args->namelen], args->value, + be16_to_cpu(name_loc->valuelen)); + } else { + name_rmt = xfs_attr3_leaf_name_remote(leaf, args->index); + name_rmt->namelen = args->namelen; + memcpy((char *)name_rmt->name, args->name, args->namelen); + entry->flags |= XFS_ATTR_INCOMPLETE; + /* just in case */ + name_rmt->valuelen = 0; + name_rmt->valueblk = 0; + args->rmtblkno = 1; + args->rmtblkcnt = xfs_attr3_rmt_blocks(mp, args->valuelen); + args->rmtvaluelen = args->valuelen; + } + xfs_trans_log_buf(args->trans, bp, + XFS_DA_LOGRANGE(leaf, xfs_attr3_leaf_name(leaf, args->index), + xfs_attr_leaf_entsize(leaf, args->index))); + + /* + * Update the control info for this leaf node + */ + if (be16_to_cpu(entry->nameidx) < ichdr->firstused) + ichdr->firstused = be16_to_cpu(entry->nameidx); + + new_end = xfs_attr_leaf_entries_end(ichdr->count, leaf); + old_end = new_end - sizeof(struct xfs_attr_leaf_entry); + + ASSERT(ichdr->firstused >= new_end); + + for (i = 0; i < XFS_ATTR_LEAF_MAPSIZE; i++) { + int diff = 0; + + if (ichdr->freemap[i].base == old_end) { + /* + * This freemap entry starts at the old end of the + * leaf entry array, so we need to adjust its base + * upward to accomodate the larger array. + */ + diff = sizeof(struct xfs_attr_leaf_entry); + } else if (ichdr->freemap[i].size > 0 && + ichdr->freemap[i].base < new_end) { + /* + * This freemap entry starts in the space claimed by + * the new leaf entry. Adjust its base upward to + * reflect that. + */ + diff = new_end - ichdr->freemap[i].base; + } + + if (diff) { + ichdr->freemap[i].base += diff; + ichdr->freemap[i].size -= + min_t(uint16_t, ichdr->freemap[i].size, diff); + } + + /* + * Don't leave zero-length freemaps with nonzero base lying + * around, because we don't want the code in _remove that + * matches on base address to get confused and create + * overlapping freemaps. If we end up with no freemap entries + * then the next _add will compact the leaf block and + * regenerate the freemaps. + */ + if (ichdr->freemap[i].size == 0 && ichdr->freemap[i].base > 0) { + ichdr->freemap[i].base = 0; + ichdr->holes = 1; + } + } + ichdr->usedbytes += xfs_attr_leaf_entsize(leaf, args->index); +} + +/* + * Garbage collect a leaf attribute list block by copying it to a new buffer. + */ +STATIC void +xfs_attr3_leaf_compact( + struct xfs_da_args *args, + struct xfs_attr3_icleaf_hdr *ichdr_dst, + struct xfs_buf *bp) +{ + struct xfs_attr_leafblock *leaf_src; + struct xfs_attr_leafblock *leaf_dst; + struct xfs_attr3_icleaf_hdr ichdr_src; + struct xfs_trans *trans = args->trans; + char *tmpbuffer; + + trace_xfs_attr_leaf_compact(args); + + tmpbuffer = kvmalloc(args->geo->blksize, GFP_KERNEL | __GFP_NOFAIL); + memcpy(tmpbuffer, bp->b_addr, args->geo->blksize); + memset(bp->b_addr, 0, args->geo->blksize); + leaf_src = (xfs_attr_leafblock_t *)tmpbuffer; + leaf_dst = bp->b_addr; + + /* + * Copy the on-disk header back into the destination buffer to ensure + * all the information in the header that is not part of the incore + * header structure is preserved. + */ + memcpy(bp->b_addr, tmpbuffer, xfs_attr3_leaf_hdr_size(leaf_src)); + + /* Initialise the incore headers */ + ichdr_src = *ichdr_dst; /* struct copy */ + ichdr_dst->firstused = args->geo->blksize; + ichdr_dst->usedbytes = 0; + ichdr_dst->count = 0; + ichdr_dst->holes = 0; + ichdr_dst->freemap[0].base = xfs_attr3_leaf_hdr_size(leaf_src); + ichdr_dst->freemap[0].size = ichdr_dst->firstused - + ichdr_dst->freemap[0].base; + ichdr_dst->freemap[1].base = 0; + ichdr_dst->freemap[2].base = 0; + ichdr_dst->freemap[1].size = 0; + ichdr_dst->freemap[2].size = 0; + + /* write the header back to initialise the underlying buffer */ + xfs_attr3_leaf_hdr_to_disk(args->geo, leaf_dst, ichdr_dst); + + /* + * Copy all entry's in the same (sorted) order, + * but allocate name/value pairs packed and in sequence. + */ + xfs_attr3_leaf_moveents(args, leaf_src, &ichdr_src, 0, + leaf_dst, ichdr_dst, 0, ichdr_src.count); + /* + * this logs the entire buffer, but the caller must write the header + * back to the buffer when it is finished modifying it. + */ + xfs_trans_log_buf(trans, bp, 0, args->geo->blksize - 1); + + kvfree(tmpbuffer); +} + +/* + * Compare two leaf blocks "order". + * Return 0 unless leaf2 should go before leaf1. + */ +static int +xfs_attr3_leaf_order( + struct xfs_buf *leaf1_bp, + struct xfs_attr3_icleaf_hdr *leaf1hdr, + struct xfs_buf *leaf2_bp, + struct xfs_attr3_icleaf_hdr *leaf2hdr) +{ + struct xfs_attr_leaf_entry *entries1; + struct xfs_attr_leaf_entry *entries2; + + entries1 = xfs_attr3_leaf_entryp(leaf1_bp->b_addr); + entries2 = xfs_attr3_leaf_entryp(leaf2_bp->b_addr); + if (leaf1hdr->count > 0 && leaf2hdr->count > 0 && + ((be32_to_cpu(entries2[0].hashval) < + be32_to_cpu(entries1[0].hashval)) || + (be32_to_cpu(entries2[leaf2hdr->count - 1].hashval) < + be32_to_cpu(entries1[leaf1hdr->count - 1].hashval)))) { + return 1; + } + return 0; +} + +int +xfs_attr_leaf_order( + struct xfs_buf *leaf1_bp, + struct xfs_buf *leaf2_bp) +{ + struct xfs_attr3_icleaf_hdr ichdr1; + struct xfs_attr3_icleaf_hdr ichdr2; + struct xfs_mount *mp = leaf1_bp->b_mount; + + xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, &ichdr1, leaf1_bp->b_addr); + xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, &ichdr2, leaf2_bp->b_addr); + return xfs_attr3_leaf_order(leaf1_bp, &ichdr1, leaf2_bp, &ichdr2); +} + +/* + * Redistribute the attribute list entries between two leaf nodes, + * taking into account the size of the new entry. + * + * NOTE: if new block is empty, then it will get the upper half of the + * old block. At present, all (one) callers pass in an empty second block. + * + * This code adjusts the args->index/blkno and args->index2/blkno2 fields + * to match what it is doing in splitting the attribute leaf block. Those + * values are used in "atomic rename" operations on attributes. Note that + * the "new" and "old" values can end up in different blocks. + */ +STATIC void +xfs_attr3_leaf_rebalance( + struct xfs_da_state *state, + struct xfs_da_state_blk *blk1, + struct xfs_da_state_blk *blk2) +{ + struct xfs_da_args *args; + struct xfs_attr_leafblock *leaf1; + struct xfs_attr_leafblock *leaf2; + struct xfs_attr3_icleaf_hdr ichdr1; + struct xfs_attr3_icleaf_hdr ichdr2; + struct xfs_attr_leaf_entry *entries1; + struct xfs_attr_leaf_entry *entries2; + int count; + int totallen; + int max; + int space; + int swap; + + /* + * Set up environment. + */ + ASSERT(blk1->magic == XFS_ATTR_LEAF_MAGIC); + ASSERT(blk2->magic == XFS_ATTR_LEAF_MAGIC); + leaf1 = blk1->bp->b_addr; + leaf2 = blk2->bp->b_addr; + xfs_attr3_leaf_hdr_from_disk(state->args->geo, &ichdr1, leaf1); + xfs_attr3_leaf_hdr_from_disk(state->args->geo, &ichdr2, leaf2); + ASSERT(ichdr2.count == 0); + args = state->args; + + trace_xfs_attr_leaf_rebalance(args); + + /* + * Check ordering of blocks, reverse if it makes things simpler. + * + * NOTE: Given that all (current) callers pass in an empty + * second block, this code should never set "swap". + */ + swap = 0; + if (xfs_attr3_leaf_order(blk1->bp, &ichdr1, blk2->bp, &ichdr2)) { + swap(blk1, blk2); + + /* swap structures rather than reconverting them */ + swap(ichdr1, ichdr2); + + leaf1 = blk1->bp->b_addr; + leaf2 = blk2->bp->b_addr; + swap = 1; + } + + /* + * Examine entries until we reduce the absolute difference in + * byte usage between the two blocks to a minimum. Then get + * the direction to copy and the number of elements to move. + * + * "inleaf" is true if the new entry should be inserted into blk1. + * If "swap" is also true, then reverse the sense of "inleaf". + */ + state->inleaf = xfs_attr3_leaf_figure_balance(state, blk1, &ichdr1, + blk2, &ichdr2, + &count, &totallen); + if (swap) + state->inleaf = !state->inleaf; + + /* + * Move any entries required from leaf to leaf: + */ + if (count < ichdr1.count) { + /* + * Figure the total bytes to be added to the destination leaf. + */ + /* number entries being moved */ + count = ichdr1.count - count; + space = ichdr1.usedbytes - totallen; + space += count * sizeof(xfs_attr_leaf_entry_t); + + /* + * leaf2 is the destination, compact it if it looks tight. + */ + max = ichdr2.firstused - + xfs_attr_leaf_entries_end(ichdr2.count, leaf1); + if (space > max) + xfs_attr3_leaf_compact(args, &ichdr2, blk2->bp); + + /* + * Move high entries from leaf1 to low end of leaf2. + */ + xfs_attr3_leaf_moveents(args, leaf1, &ichdr1, + ichdr1.count - count, leaf2, &ichdr2, 0, count); + + } else if (count > ichdr1.count) { + /* + * I assert that since all callers pass in an empty + * second buffer, this code should never execute. + */ + ASSERT(0); + + /* + * Figure the total bytes to be added to the destination leaf. + */ + /* number entries being moved */ + count -= ichdr1.count; + space = totallen - ichdr1.usedbytes; + space += count * sizeof(xfs_attr_leaf_entry_t); + + /* + * leaf1 is the destination, compact it if it looks tight. + */ + max = ichdr1.firstused - + xfs_attr_leaf_entries_end(ichdr1.count, leaf1); + if (space > max) + xfs_attr3_leaf_compact(args, &ichdr1, blk1->bp); + + /* + * Move low entries from leaf2 to high end of leaf1. + */ + xfs_attr3_leaf_moveents(args, leaf2, &ichdr2, 0, leaf1, &ichdr1, + ichdr1.count, count); + } + + xfs_attr3_leaf_hdr_to_disk(state->args->geo, leaf1, &ichdr1); + xfs_attr3_leaf_hdr_to_disk(state->args->geo, leaf2, &ichdr2); + xfs_trans_log_buf(args->trans, blk1->bp, 0, args->geo->blksize - 1); + xfs_trans_log_buf(args->trans, blk2->bp, 0, args->geo->blksize - 1); + + /* + * Copy out last hashval in each block for B-tree code. + */ + entries1 = xfs_attr3_leaf_entryp(leaf1); + entries2 = xfs_attr3_leaf_entryp(leaf2); + blk1->hashval = be32_to_cpu(entries1[ichdr1.count - 1].hashval); + blk2->hashval = be32_to_cpu(entries2[ichdr2.count - 1].hashval); + + /* + * Adjust the expected index for insertion. + * NOTE: this code depends on the (current) situation that the + * second block was originally empty. + * + * If the insertion point moved to the 2nd block, we must adjust + * the index. We must also track the entry just following the + * new entry for use in an "atomic rename" operation, that entry + * is always the "old" entry and the "new" entry is what we are + * inserting. The index/blkno fields refer to the "old" entry, + * while the index2/blkno2 fields refer to the "new" entry. + */ + if (blk1->index > ichdr1.count) { + ASSERT(state->inleaf == 0); + blk2->index = blk1->index - ichdr1.count; + args->index = args->index2 = blk2->index; + args->blkno = args->blkno2 = blk2->blkno; + } else if (blk1->index == ichdr1.count) { + if (state->inleaf) { + args->index = blk1->index; + args->blkno = blk1->blkno; + args->index2 = 0; + args->blkno2 = blk2->blkno; + } else { + /* + * On a double leaf split, the original attr location + * is already stored in blkno2/index2, so don't + * overwrite it overwise we corrupt the tree. + */ + blk2->index = blk1->index - ichdr1.count; + args->index = blk2->index; + args->blkno = blk2->blkno; + if (!state->extravalid) { + /* + * set the new attr location to match the old + * one and let the higher level split code + * decide where in the leaf to place it. + */ + args->index2 = blk2->index; + args->blkno2 = blk2->blkno; + } + } + } else { + ASSERT(state->inleaf == 1); + args->index = args->index2 = blk1->index; + args->blkno = args->blkno2 = blk1->blkno; + } +} + +/* + * Examine entries until we reduce the absolute difference in + * byte usage between the two blocks to a minimum. + * GROT: Is this really necessary? With other than a 512 byte blocksize, + * GROT: there will always be enough room in either block for a new entry. + * GROT: Do a double-split for this case? + */ +STATIC int +xfs_attr3_leaf_figure_balance( + struct xfs_da_state *state, + struct xfs_da_state_blk *blk1, + struct xfs_attr3_icleaf_hdr *ichdr1, + struct xfs_da_state_blk *blk2, + struct xfs_attr3_icleaf_hdr *ichdr2, + int *countarg, + int *usedbytesarg) +{ + struct xfs_attr_leafblock *leaf1 = blk1->bp->b_addr; + struct xfs_attr_leafblock *leaf2 = blk2->bp->b_addr; + struct xfs_attr_leaf_entry *entry; + int count; + int max; + int index; + int totallen = 0; + int half; + int lastdelta; + int foundit = 0; + int tmp; + + /* + * Examine entries until we reduce the absolute difference in + * byte usage between the two blocks to a minimum. + */ + max = ichdr1->count + ichdr2->count; + half = (max + 1) * sizeof(*entry); + half += ichdr1->usedbytes + ichdr2->usedbytes + + xfs_attr_leaf_newentsize(state->args, NULL); + half /= 2; + lastdelta = state->args->geo->blksize; + entry = xfs_attr3_leaf_entryp(leaf1); + for (count = index = 0; count < max; entry++, index++, count++) { + +#define XFS_ATTR_ABS(A) (((A) < 0) ? -(A) : (A)) + /* + * The new entry is in the first block, account for it. + */ + if (count == blk1->index) { + tmp = totallen + sizeof(*entry) + + xfs_attr_leaf_newentsize(state->args, NULL); + if (XFS_ATTR_ABS(half - tmp) > lastdelta) + break; + lastdelta = XFS_ATTR_ABS(half - tmp); + totallen = tmp; + foundit = 1; + } + + /* + * Wrap around into the second block if necessary. + */ + if (count == ichdr1->count) { + leaf1 = leaf2; + entry = xfs_attr3_leaf_entryp(leaf1); + index = 0; + } + + /* + * Figure out if next leaf entry would be too much. + */ + tmp = totallen + sizeof(*entry) + xfs_attr_leaf_entsize(leaf1, + index); + if (XFS_ATTR_ABS(half - tmp) > lastdelta) + break; + lastdelta = XFS_ATTR_ABS(half - tmp); + totallen = tmp; +#undef XFS_ATTR_ABS + } + + /* + * Calculate the number of usedbytes that will end up in lower block. + * If new entry not in lower block, fix up the count. + */ + totallen -= count * sizeof(*entry); + if (foundit) { + totallen -= sizeof(*entry) + + xfs_attr_leaf_newentsize(state->args, NULL); + } + + *countarg = count; + *usedbytesarg = totallen; + return foundit; +} + +/*======================================================================== + * Routines used for shrinking the Btree. + *========================================================================*/ + +/* + * Check a leaf block and its neighbors to see if the block should be + * collapsed into one or the other neighbor. Always keep the block + * with the smaller block number. + * If the current block is over 50% full, don't try to join it, return 0. + * If the block is empty, fill in the state structure and return 2. + * If it can be collapsed, fill in the state structure and return 1. + * If nothing can be done, return 0. + * + * GROT: allow for INCOMPLETE entries in calculation. + */ +int +xfs_attr3_leaf_toosmall( + struct xfs_da_state *state, + int *action) +{ + struct xfs_attr_leafblock *leaf; + struct xfs_da_state_blk *blk; + struct xfs_attr3_icleaf_hdr ichdr; + struct xfs_buf *bp; + xfs_dablk_t blkno; + int bytes; + int forward; + int error; + int retval; + int i; + + trace_xfs_attr_leaf_toosmall(state->args); + + /* + * Check for the degenerate case of the block being over 50% full. + * If so, it's not worth even looking to see if we might be able + * to coalesce with a sibling. + */ + blk = &state->path.blk[ state->path.active-1 ]; + leaf = blk->bp->b_addr; + xfs_attr3_leaf_hdr_from_disk(state->args->geo, &ichdr, leaf); + bytes = xfs_attr_leaf_entries_end(ichdr.count, leaf) + ichdr.usedbytes; + if (bytes > (state->args->geo->blksize >> 1)) { + *action = 0; /* blk over 50%, don't try to join */ + return 0; + } + + /* + * Check for the degenerate case of the block being empty. + * If the block is empty, we'll simply delete it, no need to + * coalesce it with a sibling block. We choose (arbitrarily) + * to merge with the forward block unless it is NULL. + */ + if (ichdr.count == 0) { + /* + * Make altpath point to the block we want to keep and + * path point to the block we want to drop (this one). + */ + forward = (ichdr.forw != 0); + memcpy(&state->altpath, &state->path, sizeof(state->path)); + error = xfs_da3_path_shift(state, &state->altpath, forward, + 0, &retval); + if (error) + return error; + if (retval) { + *action = 0; + } else { + *action = 2; + } + return 0; + } + + /* + * Examine each sibling block to see if we can coalesce with + * at least 25% free space to spare. We need to figure out + * whether to merge with the forward or the backward block. + * We prefer coalescing with the lower numbered sibling so as + * to shrink an attribute list over time. + */ + /* start with smaller blk num */ + forward = ichdr.forw < ichdr.back; + for (i = 0; i < 2; forward = !forward, i++) { + struct xfs_attr3_icleaf_hdr ichdr2; + if (forward) + blkno = ichdr.forw; + else + blkno = ichdr.back; + if (blkno == 0) + continue; + error = xfs_attr3_leaf_read(state->args->trans, state->args->dp, + state->args->owner, blkno, &bp); + if (error) + return error; + + xfs_attr3_leaf_hdr_from_disk(state->args->geo, &ichdr2, bp->b_addr); + + bytes = state->args->geo->blksize - + (state->args->geo->blksize >> 2) - + ichdr.usedbytes - ichdr2.usedbytes - + xfs_attr_leaf_entries_end(ichdr.count + ichdr2.count, + leaf); + + xfs_trans_brelse(state->args->trans, bp); + if (bytes >= 0) + break; /* fits with at least 25% to spare */ + } + if (i >= 2) { + *action = 0; + return 0; + } + + /* + * Make altpath point to the block we want to keep (the lower + * numbered block) and path point to the block we want to drop. + */ + memcpy(&state->altpath, &state->path, sizeof(state->path)); + if (blkno < blk->blkno) { + error = xfs_da3_path_shift(state, &state->altpath, forward, + 0, &retval); + } else { + error = xfs_da3_path_shift(state, &state->path, forward, + 0, &retval); + } + if (error) + return error; + if (retval) { + *action = 0; + } else { + *action = 1; + } + return 0; +} + +/* + * Remove a name from the leaf attribute list structure. + * + * Return 1 if leaf is less than 37% full, 0 if >= 37% full. + * If two leaves are 37% full, when combined they will leave 25% free. + */ +int +xfs_attr3_leaf_remove( + struct xfs_buf *bp, + struct xfs_da_args *args) +{ + struct xfs_attr_leafblock *leaf; + struct xfs_attr3_icleaf_hdr ichdr; + struct xfs_attr_leaf_entry *entry; + int before; + int after; + int smallest; + int entsize; + int tablesize; + int tmp; + int i; + + trace_xfs_attr_leaf_remove(args); + + leaf = bp->b_addr; + xfs_attr3_leaf_hdr_from_disk(args->geo, &ichdr, leaf); + + ASSERT(ichdr.count > 0 && ichdr.count < args->geo->blksize / 8); + ASSERT(args->index >= 0 && args->index < ichdr.count); + ASSERT(ichdr.firstused >= xfs_attr_leaf_entries_end(ichdr.count, leaf)); + + entry = &xfs_attr3_leaf_entryp(leaf)[args->index]; + + ASSERT(be16_to_cpu(entry->nameidx) >= ichdr.firstused); + ASSERT(be16_to_cpu(entry->nameidx) < args->geo->blksize); + + /* + * Scan through free region table: + * check for adjacency of free'd entry with an existing one, + * find smallest free region in case we need to replace it, + * adjust any map that borders the entry table, + */ + tablesize = xfs_attr_leaf_entries_end(ichdr.count, leaf); + tmp = ichdr.freemap[0].size; + before = after = -1; + smallest = XFS_ATTR_LEAF_MAPSIZE - 1; + entsize = xfs_attr_leaf_entsize(leaf, args->index); + for (i = 0; i < XFS_ATTR_LEAF_MAPSIZE; i++) { + ASSERT(ichdr.freemap[i].base < args->geo->blksize); + ASSERT(ichdr.freemap[i].size < args->geo->blksize); + if (ichdr.freemap[i].base == tablesize) { + ichdr.freemap[i].base -= sizeof(xfs_attr_leaf_entry_t); + ichdr.freemap[i].size += sizeof(xfs_attr_leaf_entry_t); + } + + if (ichdr.freemap[i].base + ichdr.freemap[i].size == + be16_to_cpu(entry->nameidx)) { + before = i; + } else if (ichdr.freemap[i].base == + (be16_to_cpu(entry->nameidx) + entsize)) { + after = i; + } else if (ichdr.freemap[i].size < tmp) { + tmp = ichdr.freemap[i].size; + smallest = i; + } + } + + /* + * Coalesce adjacent freemap regions, + * or replace the smallest region. + */ + if ((before >= 0) || (after >= 0)) { + if ((before >= 0) && (after >= 0)) { + ichdr.freemap[before].size += entsize; + ichdr.freemap[before].size += ichdr.freemap[after].size; + ichdr.freemap[after].base = 0; + ichdr.freemap[after].size = 0; + } else if (before >= 0) { + ichdr.freemap[before].size += entsize; + } else { + ichdr.freemap[after].base = be16_to_cpu(entry->nameidx); + ichdr.freemap[after].size += entsize; + } + } else { + /* + * Replace smallest region (if it is smaller than free'd entry) + */ + if (ichdr.freemap[smallest].size < entsize) { + ichdr.freemap[smallest].base = be16_to_cpu(entry->nameidx); + ichdr.freemap[smallest].size = entsize; + } + } + + /* + * Did we remove the first entry? + */ + if (be16_to_cpu(entry->nameidx) == ichdr.firstused) + smallest = 1; + else + smallest = 0; + + /* + * Compress the remaining entries and zero out the removed stuff. + */ + memset(xfs_attr3_leaf_name(leaf, args->index), 0, entsize); + ichdr.usedbytes -= entsize; + xfs_trans_log_buf(args->trans, bp, + XFS_DA_LOGRANGE(leaf, xfs_attr3_leaf_name(leaf, args->index), + entsize)); + + tmp = (ichdr.count - args->index) * sizeof(xfs_attr_leaf_entry_t); + memmove(entry, entry + 1, tmp); + ichdr.count--; + xfs_trans_log_buf(args->trans, bp, + XFS_DA_LOGRANGE(leaf, entry, tmp + sizeof(xfs_attr_leaf_entry_t))); + + entry = &xfs_attr3_leaf_entryp(leaf)[ichdr.count]; + memset(entry, 0, sizeof(xfs_attr_leaf_entry_t)); + + /* + * If we removed the first entry, re-find the first used byte + * in the name area. Note that if the entry was the "firstused", + * then we don't have a "hole" in our block resulting from + * removing the name. + */ + if (smallest) { + tmp = args->geo->blksize; + entry = xfs_attr3_leaf_entryp(leaf); + for (i = ichdr.count - 1; i >= 0; entry++, i--) { + ASSERT(be16_to_cpu(entry->nameidx) >= ichdr.firstused); + ASSERT(be16_to_cpu(entry->nameidx) < args->geo->blksize); + + if (be16_to_cpu(entry->nameidx) < tmp) + tmp = be16_to_cpu(entry->nameidx); + } + ichdr.firstused = tmp; + ASSERT(ichdr.firstused != 0); + } else { + ichdr.holes = 1; /* mark as needing compaction */ + } + xfs_attr3_leaf_hdr_to_disk(args->geo, leaf, &ichdr); + xfs_trans_log_buf(args->trans, bp, + XFS_DA_LOGRANGE(leaf, &leaf->hdr, + xfs_attr3_leaf_hdr_size(leaf))); + + /* + * Check if leaf is less than 50% full, caller may want to + * "join" the leaf with a sibling if so. + */ + tmp = ichdr.usedbytes + xfs_attr_leaf_entries_end(ichdr.count, leaf); + + return tmp < args->geo->magicpct; /* leaf is < 37% full */ +} + +/* + * Move all the attribute list entries from drop_leaf into save_leaf. + */ +void +xfs_attr3_leaf_unbalance( + struct xfs_da_state *state, + struct xfs_da_state_blk *drop_blk, + struct xfs_da_state_blk *save_blk) +{ + struct xfs_attr_leafblock *drop_leaf = drop_blk->bp->b_addr; + struct xfs_attr_leafblock *save_leaf = save_blk->bp->b_addr; + struct xfs_attr3_icleaf_hdr drophdr; + struct xfs_attr3_icleaf_hdr savehdr; + struct xfs_attr_leaf_entry *entry; + + trace_xfs_attr_leaf_unbalance(state->args); + + xfs_attr3_leaf_hdr_from_disk(state->args->geo, &drophdr, drop_leaf); + xfs_attr3_leaf_hdr_from_disk(state->args->geo, &savehdr, save_leaf); + entry = xfs_attr3_leaf_entryp(drop_leaf); + + /* + * Save last hashval from dying block for later Btree fixup. + */ + drop_blk->hashval = be32_to_cpu(entry[drophdr.count - 1].hashval); + + /* + * Check if we need a temp buffer, or can we do it in place. + * Note that we don't check "leaf" for holes because we will + * always be dropping it, toosmall() decided that for us already. + */ + if (savehdr.holes == 0) { + /* + * dest leaf has no holes, so we add there. May need + * to make some room in the entry array. + */ + if (xfs_attr3_leaf_order(save_blk->bp, &savehdr, + drop_blk->bp, &drophdr)) { + xfs_attr3_leaf_moveents(state->args, + drop_leaf, &drophdr, 0, + save_leaf, &savehdr, 0, + drophdr.count); + } else { + xfs_attr3_leaf_moveents(state->args, + drop_leaf, &drophdr, 0, + save_leaf, &savehdr, + savehdr.count, drophdr.count); + } + } else { + /* + * Destination has holes, so we make a temporary copy + * of the leaf and add them both to that. + */ + struct xfs_attr_leafblock *tmp_leaf; + struct xfs_attr3_icleaf_hdr tmphdr; + + tmp_leaf = kvzalloc(state->args->geo->blksize, + GFP_KERNEL | __GFP_NOFAIL); + + /* + * Copy the header into the temp leaf so that all the stuff + * not in the incore header is present and gets copied back in + * once we've moved all the entries. + */ + memcpy(tmp_leaf, save_leaf, xfs_attr3_leaf_hdr_size(save_leaf)); + + memset(&tmphdr, 0, sizeof(tmphdr)); + tmphdr.magic = savehdr.magic; + tmphdr.forw = savehdr.forw; + tmphdr.back = savehdr.back; + tmphdr.firstused = state->args->geo->blksize; + + /* write the header to the temp buffer to initialise it */ + xfs_attr3_leaf_hdr_to_disk(state->args->geo, tmp_leaf, &tmphdr); + + if (xfs_attr3_leaf_order(save_blk->bp, &savehdr, + drop_blk->bp, &drophdr)) { + xfs_attr3_leaf_moveents(state->args, + drop_leaf, &drophdr, 0, + tmp_leaf, &tmphdr, 0, + drophdr.count); + xfs_attr3_leaf_moveents(state->args, + save_leaf, &savehdr, 0, + tmp_leaf, &tmphdr, tmphdr.count, + savehdr.count); + } else { + xfs_attr3_leaf_moveents(state->args, + save_leaf, &savehdr, 0, + tmp_leaf, &tmphdr, 0, + savehdr.count); + xfs_attr3_leaf_moveents(state->args, + drop_leaf, &drophdr, 0, + tmp_leaf, &tmphdr, tmphdr.count, + drophdr.count); + } + memcpy(save_leaf, tmp_leaf, state->args->geo->blksize); + savehdr = tmphdr; /* struct copy */ + kvfree(tmp_leaf); + } + + xfs_attr3_leaf_hdr_to_disk(state->args->geo, save_leaf, &savehdr); + xfs_trans_log_buf(state->args->trans, save_blk->bp, 0, + state->args->geo->blksize - 1); + + /* + * Copy out last hashval in each block for B-tree code. + */ + entry = xfs_attr3_leaf_entryp(save_leaf); + save_blk->hashval = be32_to_cpu(entry[savehdr.count - 1].hashval); +} + +/*======================================================================== + * Routines used for finding things in the Btree. + *========================================================================*/ + +/* + * Look up a name in a leaf attribute list structure. + * This is the internal routine, it uses the caller's buffer. + * + * Note that duplicate keys are allowed, but only check within the + * current leaf node. The Btree code must check in adjacent leaf nodes. + * + * Return in args->index the index into the entry[] array of either + * the found entry, or where the entry should have been (insert before + * that entry). + * + * Don't change the args->value unless we find the attribute. + */ +int +xfs_attr3_leaf_lookup_int( + struct xfs_buf *bp, + struct xfs_da_args *args) +{ + struct xfs_attr_leafblock *leaf; + struct xfs_attr3_icleaf_hdr ichdr; + struct xfs_attr_leaf_entry *entry; + struct xfs_attr_leaf_entry *entries; + struct xfs_attr_leaf_name_local *name_loc; + struct xfs_attr_leaf_name_remote *name_rmt; + xfs_dahash_t hashval; + int probe; + int span; + + trace_xfs_attr_leaf_lookup(args); + + leaf = bp->b_addr; + xfs_attr3_leaf_hdr_from_disk(args->geo, &ichdr, leaf); + entries = xfs_attr3_leaf_entryp(leaf); + if (ichdr.count >= args->geo->blksize / 8) { + xfs_buf_mark_corrupt(bp); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + + /* + * Binary search. (note: small blocks will skip this loop) + */ + hashval = args->hashval; + probe = span = ichdr.count / 2; + for (entry = &entries[probe]; span > 4; entry = &entries[probe]) { + span /= 2; + if (be32_to_cpu(entry->hashval) < hashval) + probe += span; + else if (be32_to_cpu(entry->hashval) > hashval) + probe -= span; + else + break; + } + if (!(probe >= 0 && (!ichdr.count || probe < ichdr.count))) { + xfs_buf_mark_corrupt(bp); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + if (!(span <= 4 || be32_to_cpu(entry->hashval) == hashval)) { + xfs_buf_mark_corrupt(bp); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + + /* + * Since we may have duplicate hashval's, find the first matching + * hashval in the leaf. + */ + while (probe > 0 && be32_to_cpu(entry->hashval) >= hashval) { + entry--; + probe--; + } + while (probe < ichdr.count && + be32_to_cpu(entry->hashval) < hashval) { + entry++; + probe++; + } + if (probe == ichdr.count || be32_to_cpu(entry->hashval) != hashval) { + args->index = probe; + return -ENOATTR; + } + + /* + * Duplicate keys may be present, so search all of them for a match. + */ + for (; probe < ichdr.count && (be32_to_cpu(entry->hashval) == hashval); + entry++, probe++) { +/* + * GROT: Add code to remove incomplete entries. + */ + if (entry->flags & XFS_ATTR_LOCAL) { + name_loc = xfs_attr3_leaf_name_local(leaf, probe); + if (!xfs_attr_match(args, entry->flags, + name_loc->nameval, name_loc->namelen, + &name_loc->nameval[name_loc->namelen], + be16_to_cpu(name_loc->valuelen))) + continue; + args->index = probe; + return -EEXIST; + } else { + unsigned int valuelen; + + name_rmt = xfs_attr3_leaf_name_remote(leaf, probe); + valuelen = be32_to_cpu(name_rmt->valuelen); + if (!xfs_attr_match(args, entry->flags, name_rmt->name, + name_rmt->namelen, NULL, valuelen)) + continue; + args->index = probe; + args->rmtvaluelen = valuelen; + args->rmtblkno = be32_to_cpu(name_rmt->valueblk); + args->rmtblkcnt = xfs_attr3_rmt_blocks( + args->dp->i_mount, + args->rmtvaluelen); + return -EEXIST; + } + } + args->index = probe; + return -ENOATTR; +} + +/* + * Get the value associated with an attribute name from a leaf attribute + * list structure. + * + * If args->valuelen is zero, only the length needs to be returned. Unlike a + * lookup, we only return an error if the attribute does not exist or we can't + * retrieve the value. + */ +int +xfs_attr3_leaf_getvalue( + struct xfs_buf *bp, + struct xfs_da_args *args) +{ + struct xfs_attr_leafblock *leaf; + struct xfs_attr3_icleaf_hdr ichdr; + struct xfs_attr_leaf_entry *entry; + struct xfs_attr_leaf_name_local *name_loc; + struct xfs_attr_leaf_name_remote *name_rmt; + + leaf = bp->b_addr; + xfs_attr3_leaf_hdr_from_disk(args->geo, &ichdr, leaf); + ASSERT(ichdr.count < args->geo->blksize / 8); + ASSERT(args->index < ichdr.count); + + entry = &xfs_attr3_leaf_entryp(leaf)[args->index]; + if (entry->flags & XFS_ATTR_LOCAL) { + name_loc = xfs_attr3_leaf_name_local(leaf, args->index); + ASSERT(name_loc->namelen == args->namelen); + ASSERT(memcmp(args->name, name_loc->nameval, args->namelen) == 0); + return xfs_attr_copy_value(args, + &name_loc->nameval[args->namelen], + be16_to_cpu(name_loc->valuelen)); + } + + name_rmt = xfs_attr3_leaf_name_remote(leaf, args->index); + ASSERT(name_rmt->namelen == args->namelen); + ASSERT(memcmp(args->name, name_rmt->name, args->namelen) == 0); + args->rmtvaluelen = be32_to_cpu(name_rmt->valuelen); + args->rmtblkno = be32_to_cpu(name_rmt->valueblk); + args->rmtblkcnt = xfs_attr3_rmt_blocks(args->dp->i_mount, + args->rmtvaluelen); + return xfs_attr_copy_value(args, NULL, args->rmtvaluelen); +} + +/*======================================================================== + * Utility routines. + *========================================================================*/ + +/* + * Move the indicated entries from one leaf to another. + * NOTE: this routine modifies both source and destination leaves. + */ +/*ARGSUSED*/ +STATIC void +xfs_attr3_leaf_moveents( + struct xfs_da_args *args, + struct xfs_attr_leafblock *leaf_s, + struct xfs_attr3_icleaf_hdr *ichdr_s, + int start_s, + struct xfs_attr_leafblock *leaf_d, + struct xfs_attr3_icleaf_hdr *ichdr_d, + int start_d, + int count) +{ + struct xfs_attr_leaf_entry *entry_s; + struct xfs_attr_leaf_entry *entry_d; + int desti; + int tmp; + int i; + + /* + * Check for nothing to do. + */ + if (count == 0) + return; + + /* + * Set up environment. + */ + ASSERT(ichdr_s->magic == XFS_ATTR_LEAF_MAGIC || + ichdr_s->magic == XFS_ATTR3_LEAF_MAGIC); + ASSERT(ichdr_s->magic == ichdr_d->magic); + ASSERT(ichdr_s->count > 0 && ichdr_s->count < args->geo->blksize / 8); + ASSERT(ichdr_s->firstused >= + xfs_attr_leaf_entries_end(ichdr_s->count, leaf_s)); + ASSERT(ichdr_d->count < args->geo->blksize / 8); + ASSERT(ichdr_d->firstused >= + xfs_attr_leaf_entries_end(ichdr_d->count, leaf_d)); + + ASSERT(start_s < ichdr_s->count); + ASSERT(start_d <= ichdr_d->count); + ASSERT(count <= ichdr_s->count); + + + /* + * Move the entries in the destination leaf up to make a hole? + */ + if (start_d < ichdr_d->count) { + tmp = ichdr_d->count - start_d; + tmp *= sizeof(xfs_attr_leaf_entry_t); + entry_s = &xfs_attr3_leaf_entryp(leaf_d)[start_d]; + entry_d = &xfs_attr3_leaf_entryp(leaf_d)[start_d + count]; + memmove(entry_d, entry_s, tmp); + } + + /* + * Copy all entry's in the same (sorted) order, + * but allocate attribute info packed and in sequence. + */ + entry_s = &xfs_attr3_leaf_entryp(leaf_s)[start_s]; + entry_d = &xfs_attr3_leaf_entryp(leaf_d)[start_d]; + desti = start_d; + for (i = 0; i < count; entry_s++, entry_d++, desti++, i++) { + ASSERT(be16_to_cpu(entry_s->nameidx) >= ichdr_s->firstused); + tmp = xfs_attr_leaf_entsize(leaf_s, start_s + i); +#ifdef GROT + /* + * Code to drop INCOMPLETE entries. Difficult to use as we + * may also need to change the insertion index. Code turned + * off for 6.2, should be revisited later. + */ + if (entry_s->flags & XFS_ATTR_INCOMPLETE) { /* skip partials? */ + memset(xfs_attr3_leaf_name(leaf_s, start_s + i), 0, tmp); + ichdr_s->usedbytes -= tmp; + ichdr_s->count -= 1; + entry_d--; /* to compensate for ++ in loop hdr */ + desti--; + if ((start_s + i) < offset) + result++; /* insertion index adjustment */ + } else { +#endif /* GROT */ + ichdr_d->firstused -= tmp; + /* both on-disk, don't endian flip twice */ + entry_d->hashval = entry_s->hashval; + entry_d->nameidx = cpu_to_be16(ichdr_d->firstused); + entry_d->flags = entry_s->flags; + ASSERT(be16_to_cpu(entry_d->nameidx) + tmp + <= args->geo->blksize); + memmove(xfs_attr3_leaf_name(leaf_d, desti), + xfs_attr3_leaf_name(leaf_s, start_s + i), tmp); + ASSERT(be16_to_cpu(entry_s->nameidx) + tmp + <= args->geo->blksize); + memset(xfs_attr3_leaf_name(leaf_s, start_s + i), 0, tmp); + ichdr_s->usedbytes -= tmp; + ichdr_d->usedbytes += tmp; + ichdr_s->count -= 1; + ichdr_d->count += 1; + tmp = xfs_attr_leaf_entries_end(ichdr_d->count, leaf_d); + ASSERT(ichdr_d->firstused >= tmp); +#ifdef GROT + } +#endif /* GROT */ + } + + /* + * Zero out the entries we just copied. + */ + if (start_s == ichdr_s->count) { + tmp = count * sizeof(xfs_attr_leaf_entry_t); + entry_s = &xfs_attr3_leaf_entryp(leaf_s)[start_s]; + ASSERT(((char *)entry_s + tmp) <= + ((char *)leaf_s + args->geo->blksize)); + memset(entry_s, 0, tmp); + } else { + /* + * Move the remaining entries down to fill the hole, + * then zero the entries at the top. + */ + tmp = (ichdr_s->count - count) * sizeof(xfs_attr_leaf_entry_t); + entry_s = &xfs_attr3_leaf_entryp(leaf_s)[start_s + count]; + entry_d = &xfs_attr3_leaf_entryp(leaf_s)[start_s]; + memmove(entry_d, entry_s, tmp); + + tmp = count * sizeof(xfs_attr_leaf_entry_t); + entry_s = &xfs_attr3_leaf_entryp(leaf_s)[ichdr_s->count]; + ASSERT(((char *)entry_s + tmp) <= + ((char *)leaf_s + args->geo->blksize)); + memset(entry_s, 0, tmp); + } + + /* + * Fill in the freemap information + */ + ichdr_d->freemap[0].base = + xfs_attr_leaf_entries_end(ichdr_d->count, leaf_d); + ichdr_d->freemap[0].size = ichdr_d->firstused - ichdr_d->freemap[0].base; + ichdr_d->freemap[1].base = 0; + ichdr_d->freemap[2].base = 0; + ichdr_d->freemap[1].size = 0; + ichdr_d->freemap[2].size = 0; + ichdr_s->holes = 1; /* leaf may not be compact */ +} + +/* + * Pick up the last hashvalue from a leaf block. + */ +xfs_dahash_t +xfs_attr_leaf_lasthash( + struct xfs_buf *bp, + int *count) +{ + struct xfs_attr3_icleaf_hdr ichdr; + struct xfs_attr_leaf_entry *entries; + struct xfs_mount *mp = bp->b_mount; + + xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, &ichdr, bp->b_addr); + entries = xfs_attr3_leaf_entryp(bp->b_addr); + if (count) + *count = ichdr.count; + if (!ichdr.count) + return 0; + return be32_to_cpu(entries[ichdr.count - 1].hashval); +} + +/* + * Calculate the number of bytes used to store the indicated attribute + * (whether local or remote only calculate bytes in this block). + */ +STATIC int +xfs_attr_leaf_entsize(xfs_attr_leafblock_t *leaf, int index) +{ + struct xfs_attr_leaf_entry *entries; + xfs_attr_leaf_name_local_t *name_loc; + xfs_attr_leaf_name_remote_t *name_rmt; + int size; + + entries = xfs_attr3_leaf_entryp(leaf); + if (entries[index].flags & XFS_ATTR_LOCAL) { + name_loc = xfs_attr3_leaf_name_local(leaf, index); + size = xfs_attr_leaf_entsize_local(name_loc->namelen, + be16_to_cpu(name_loc->valuelen)); + } else { + name_rmt = xfs_attr3_leaf_name_remote(leaf, index); + size = xfs_attr_leaf_entsize_remote(name_rmt->namelen); + } + return size; +} + +/* + * Calculate the number of bytes that would be required to store the new + * attribute (whether local or remote only calculate bytes in this block). + * This routine decides as a side effect whether the attribute will be + * a "local" or a "remote" attribute. + */ +int +xfs_attr_leaf_newentsize( + struct xfs_da_args *args, + int *local) +{ + int size; + + size = xfs_attr_leaf_entsize_local(args->namelen, args->valuelen); + if (size < xfs_attr_leaf_entsize_local_max(args->geo->blksize)) { + if (local) + *local = 1; + return size; + } + if (local) + *local = 0; + return xfs_attr_leaf_entsize_remote(args->namelen); +} + + +/*======================================================================== + * Manage the INCOMPLETE flag in a leaf entry + *========================================================================*/ + +/* + * Clear the INCOMPLETE flag on an entry in a leaf block. + */ +int +xfs_attr3_leaf_clearflag( + struct xfs_da_args *args) +{ + struct xfs_attr_leafblock *leaf; + struct xfs_attr_leaf_entry *entry; + struct xfs_attr_leaf_name_remote *name_rmt; + struct xfs_buf *bp; + int error; +#ifdef DEBUG + struct xfs_attr3_icleaf_hdr ichdr; + xfs_attr_leaf_name_local_t *name_loc; + int namelen; + char *name; +#endif /* DEBUG */ + + trace_xfs_attr_leaf_clearflag(args); + /* + * Set up the operation. + */ + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, + args->blkno, &bp); + if (error) + return error; + + leaf = bp->b_addr; + entry = &xfs_attr3_leaf_entryp(leaf)[args->index]; + ASSERT(entry->flags & XFS_ATTR_INCOMPLETE); + +#ifdef DEBUG + xfs_attr3_leaf_hdr_from_disk(args->geo, &ichdr, leaf); + ASSERT(args->index < ichdr.count); + ASSERT(args->index >= 0); + + if (entry->flags & XFS_ATTR_LOCAL) { + name_loc = xfs_attr3_leaf_name_local(leaf, args->index); + namelen = name_loc->namelen; + name = (char *)name_loc->nameval; + } else { + name_rmt = xfs_attr3_leaf_name_remote(leaf, args->index); + namelen = name_rmt->namelen; + name = (char *)name_rmt->name; + } + ASSERT(be32_to_cpu(entry->hashval) == args->hashval); + ASSERT(namelen == args->namelen); + ASSERT(memcmp(name, args->name, namelen) == 0); +#endif /* DEBUG */ + + entry->flags &= ~XFS_ATTR_INCOMPLETE; + xfs_trans_log_buf(args->trans, bp, + XFS_DA_LOGRANGE(leaf, entry, sizeof(*entry))); + + if (args->rmtblkno) { + ASSERT((entry->flags & XFS_ATTR_LOCAL) == 0); + name_rmt = xfs_attr3_leaf_name_remote(leaf, args->index); + name_rmt->valueblk = cpu_to_be32(args->rmtblkno); + name_rmt->valuelen = cpu_to_be32(args->rmtvaluelen); + xfs_trans_log_buf(args->trans, bp, + XFS_DA_LOGRANGE(leaf, name_rmt, sizeof(*name_rmt))); + } + + return 0; +} + +/* + * Set the INCOMPLETE flag on an entry in a leaf block. + */ +int +xfs_attr3_leaf_setflag( + struct xfs_da_args *args) +{ + struct xfs_attr_leafblock *leaf; + struct xfs_attr_leaf_entry *entry; + struct xfs_attr_leaf_name_remote *name_rmt; + struct xfs_buf *bp; + int error; +#ifdef DEBUG + struct xfs_attr3_icleaf_hdr ichdr; +#endif + + trace_xfs_attr_leaf_setflag(args); + + /* + * Set up the operation. + */ + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, + args->blkno, &bp); + if (error) + return error; + + leaf = bp->b_addr; +#ifdef DEBUG + xfs_attr3_leaf_hdr_from_disk(args->geo, &ichdr, leaf); + ASSERT(args->index < ichdr.count); + ASSERT(args->index >= 0); +#endif + entry = &xfs_attr3_leaf_entryp(leaf)[args->index]; + + ASSERT((entry->flags & XFS_ATTR_INCOMPLETE) == 0); + entry->flags |= XFS_ATTR_INCOMPLETE; + xfs_trans_log_buf(args->trans, bp, + XFS_DA_LOGRANGE(leaf, entry, sizeof(*entry))); + if ((entry->flags & XFS_ATTR_LOCAL) == 0) { + name_rmt = xfs_attr3_leaf_name_remote(leaf, args->index); + name_rmt->valueblk = 0; + name_rmt->valuelen = 0; + xfs_trans_log_buf(args->trans, bp, + XFS_DA_LOGRANGE(leaf, name_rmt, sizeof(*name_rmt))); + } + + return 0; +} + +/* + * In a single transaction, clear the INCOMPLETE flag on the leaf entry + * given by args->blkno/index and set the INCOMPLETE flag on the leaf + * entry given by args->blkno2/index2. + * + * Note that they could be in different blocks, or in the same block. + */ +int +xfs_attr3_leaf_flipflags( + struct xfs_da_args *args) +{ + struct xfs_attr_leafblock *leaf1; + struct xfs_attr_leafblock *leaf2; + struct xfs_attr_leaf_entry *entry1; + struct xfs_attr_leaf_entry *entry2; + struct xfs_attr_leaf_name_remote *name_rmt; + struct xfs_buf *bp1; + struct xfs_buf *bp2; + int error; +#ifdef DEBUG + struct xfs_attr3_icleaf_hdr ichdr1; + struct xfs_attr3_icleaf_hdr ichdr2; + xfs_attr_leaf_name_local_t *name_loc; + int namelen1, namelen2; + char *name1, *name2; +#endif /* DEBUG */ + + trace_xfs_attr_leaf_flipflags(args); + + /* + * Read the block containing the "old" attr + */ + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, + args->blkno, &bp1); + if (error) + return error; + + /* + * Read the block containing the "new" attr, if it is different + */ + if (args->blkno2 != args->blkno) { + error = xfs_attr3_leaf_read(args->trans, args->dp, args->owner, + args->blkno2, &bp2); + if (error) + return error; + } else { + bp2 = bp1; + } + + leaf1 = bp1->b_addr; + entry1 = &xfs_attr3_leaf_entryp(leaf1)[args->index]; + + leaf2 = bp2->b_addr; + entry2 = &xfs_attr3_leaf_entryp(leaf2)[args->index2]; + +#ifdef DEBUG + xfs_attr3_leaf_hdr_from_disk(args->geo, &ichdr1, leaf1); + ASSERT(args->index < ichdr1.count); + ASSERT(args->index >= 0); + + xfs_attr3_leaf_hdr_from_disk(args->geo, &ichdr2, leaf2); + ASSERT(args->index2 < ichdr2.count); + ASSERT(args->index2 >= 0); + + if (entry1->flags & XFS_ATTR_LOCAL) { + name_loc = xfs_attr3_leaf_name_local(leaf1, args->index); + namelen1 = name_loc->namelen; + name1 = (char *)name_loc->nameval; + } else { + name_rmt = xfs_attr3_leaf_name_remote(leaf1, args->index); + namelen1 = name_rmt->namelen; + name1 = (char *)name_rmt->name; + } + if (entry2->flags & XFS_ATTR_LOCAL) { + name_loc = xfs_attr3_leaf_name_local(leaf2, args->index2); + namelen2 = name_loc->namelen; + name2 = (char *)name_loc->nameval; + } else { + name_rmt = xfs_attr3_leaf_name_remote(leaf2, args->index2); + namelen2 = name_rmt->namelen; + name2 = (char *)name_rmt->name; + } + ASSERT(be32_to_cpu(entry1->hashval) == be32_to_cpu(entry2->hashval)); + ASSERT(namelen1 == namelen2); + ASSERT(memcmp(name1, name2, namelen1) == 0); +#endif /* DEBUG */ + + ASSERT(entry1->flags & XFS_ATTR_INCOMPLETE); + ASSERT((entry2->flags & XFS_ATTR_INCOMPLETE) == 0); + + entry1->flags &= ~XFS_ATTR_INCOMPLETE; + xfs_trans_log_buf(args->trans, bp1, + XFS_DA_LOGRANGE(leaf1, entry1, sizeof(*entry1))); + if (args->rmtblkno) { + ASSERT((entry1->flags & XFS_ATTR_LOCAL) == 0); + name_rmt = xfs_attr3_leaf_name_remote(leaf1, args->index); + name_rmt->valueblk = cpu_to_be32(args->rmtblkno); + name_rmt->valuelen = cpu_to_be32(args->rmtvaluelen); + xfs_trans_log_buf(args->trans, bp1, + XFS_DA_LOGRANGE(leaf1, name_rmt, sizeof(*name_rmt))); + } + + entry2->flags |= XFS_ATTR_INCOMPLETE; + xfs_trans_log_buf(args->trans, bp2, + XFS_DA_LOGRANGE(leaf2, entry2, sizeof(*entry2))); + if ((entry2->flags & XFS_ATTR_LOCAL) == 0) { + name_rmt = xfs_attr3_leaf_name_remote(leaf2, args->index2); + name_rmt->valueblk = 0; + name_rmt->valuelen = 0; + xfs_trans_log_buf(args->trans, bp2, + XFS_DA_LOGRANGE(leaf2, name_rmt, sizeof(*name_rmt))); + } + + return 0; +} diff --git a/fs/xfs/libxfs/xfs_attr_leaf.h b/fs/xfs/libxfs/xfs_attr_leaf.h new file mode 100644 index 000000000..72639efe6 --- /dev/null +++ b/fs/xfs/libxfs/xfs_attr_leaf.h @@ -0,0 +1,115 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2000,2002-2003,2005 Silicon Graphics, Inc. + * Copyright (c) 2013 Red Hat, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_ATTR_LEAF_H__ +#define __XFS_ATTR_LEAF_H__ + +struct attrlist; +struct xfs_attr_list_context; +struct xfs_da_args; +struct xfs_da_state; +struct xfs_da_state_blk; +struct xfs_inode; +struct xfs_trans; + +/* + * Incore version of the attribute leaf header. + */ +struct xfs_attr3_icleaf_hdr { + uint32_t forw; + uint32_t back; + uint16_t magic; + uint16_t count; + uint16_t usedbytes; + /* + * Firstused is 32-bit here instead of 16-bit like the on-disk variant + * to support maximum fsb size of 64k without overflow issues throughout + * the attr code. Instead, the overflow condition is handled on + * conversion to/from disk. + */ + uint32_t firstused; + __u8 holes; + struct { + uint16_t base; + uint16_t size; + } freemap[XFS_ATTR_LEAF_MAPSIZE]; +}; + +/*======================================================================== + * Function prototypes for the kernel. + *========================================================================*/ + +/* + * Internal routines when attribute fork size < XFS_LITINO(mp). + */ +void xfs_attr_shortform_create(struct xfs_da_args *args); +int xfs_attr_shortform_replace(struct xfs_da_args *args); +void xfs_attr_shortform_add(struct xfs_da_args *args, int forkoff); +int xfs_attr_shortform_getvalue(struct xfs_da_args *args); +int xfs_attr_shortform_to_leaf(struct xfs_da_args *args); +int xfs_attr_sf_removename(struct xfs_da_args *args); +struct xfs_attr_sf_entry *xfs_attr_sf_findname(struct xfs_da_args *args); +int xfs_attr_shortform_allfit(struct xfs_buf *bp, struct xfs_inode *dp); +int xfs_attr_shortform_bytesfit(struct xfs_inode *dp, int bytes); +xfs_failaddr_t xfs_attr_shortform_verify(struct xfs_attr_sf_hdr *sfp, + size_t size); +void xfs_attr_fork_remove(struct xfs_inode *ip, struct xfs_trans *tp); + +/* + * Internal routines when attribute fork size == XFS_LBSIZE(mp). + */ +int xfs_attr3_leaf_to_node(struct xfs_da_args *args); +int xfs_attr3_leaf_to_shortform(struct xfs_buf *bp, + struct xfs_da_args *args, int forkoff); +int xfs_attr3_leaf_clearflag(struct xfs_da_args *args); +int xfs_attr3_leaf_setflag(struct xfs_da_args *args); +int xfs_attr3_leaf_flipflags(struct xfs_da_args *args); + +/* + * Routines used for growing the Btree. + */ +int xfs_attr3_leaf_split(struct xfs_da_state *state, + struct xfs_da_state_blk *oldblk, + struct xfs_da_state_blk *newblk); +int xfs_attr3_leaf_lookup_int(struct xfs_buf *leaf, + struct xfs_da_args *args); +int xfs_attr3_leaf_getvalue(struct xfs_buf *bp, struct xfs_da_args *args); +bool xfs_attr3_leaf_add(struct xfs_buf *leaf_buffer, + struct xfs_da_args *args); +int xfs_attr3_leaf_remove(struct xfs_buf *leaf_buffer, + struct xfs_da_args *args); +int xfs_attr3_leaf_list_int(struct xfs_buf *bp, + struct xfs_attr_list_context *context); + +/* + * Routines used for shrinking the Btree. + */ + +int xfs_attr3_leaf_init(struct xfs_trans *tp, struct xfs_inode *dp, + xfs_dablk_t blkno); +int xfs_attr3_leaf_toosmall(struct xfs_da_state *state, int *retval); +void xfs_attr3_leaf_unbalance(struct xfs_da_state *state, + struct xfs_da_state_blk *drop_blk, + struct xfs_da_state_blk *save_blk); +/* + * Utility routines. + */ +xfs_dahash_t xfs_attr_leaf_lasthash(struct xfs_buf *bp, int *count); +int xfs_attr_leaf_order(struct xfs_buf *leaf1_bp, + struct xfs_buf *leaf2_bp); +int xfs_attr_leaf_newentsize(struct xfs_da_args *args, int *local); +int xfs_attr3_leaf_read(struct xfs_trans *tp, struct xfs_inode *dp, + xfs_ino_t owner, xfs_dablk_t bno, struct xfs_buf **bpp); +void xfs_attr3_leaf_hdr_from_disk(struct xfs_da_geometry *geo, + struct xfs_attr3_icleaf_hdr *to, + struct xfs_attr_leafblock *from); +void xfs_attr3_leaf_hdr_to_disk(struct xfs_da_geometry *geo, + struct xfs_attr_leafblock *to, + struct xfs_attr3_icleaf_hdr *from); +xfs_failaddr_t xfs_attr3_leaf_header_check(struct xfs_buf *bp, + xfs_ino_t owner); + +#endif /* __XFS_ATTR_LEAF_H__ */ diff --git a/fs/xfs/libxfs/xfs_attr_remote.c b/fs/xfs/libxfs/xfs_attr_remote.c new file mode 100644 index 000000000..e6c8dd1a9 --- /dev/null +++ b/fs/xfs/libxfs/xfs_attr_remote.c @@ -0,0 +1,738 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * Copyright (c) 2013 Red Hat, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_bmap.h" +#include "xfs_attr.h" +#include "xfs_attr_remote.h" +#include "xfs_trace.h" +#include "xfs_error.h" +#include "xfs_health.h" + +#define ATTR_RMTVALUE_MAPSIZE 1 /* # of map entries at once */ + +/* + * Remote Attribute Values + * ======================= + * + * Remote extended attribute values are conceptually simple -- they're written + * to data blocks mapped by an inode's attribute fork, and they have an upper + * size limit of 64k. Setting a value does not involve the XFS log. + * + * However, on a v5 filesystem, maximally sized remote attr values require one + * block more than 64k worth of space to hold both the remote attribute value + * header (64 bytes). On a 4k block filesystem this results in a 68k buffer; + * on a 64k block filesystem, this would be a 128k buffer. Note that the log + * format can only handle a dirty buffer of XFS_MAX_BLOCKSIZE length (64k). + * Therefore, we /must/ ensure that remote attribute value buffers never touch + * the logging system and therefore never have a log item. + */ + +/* How many bytes can be stored in a remote value buffer? */ +inline unsigned int +xfs_attr3_rmt_buf_space( + struct xfs_mount *mp) +{ + unsigned int blocksize = mp->m_attr_geo->blksize; + + if (xfs_has_crc(mp)) + return blocksize - sizeof(struct xfs_attr3_rmt_hdr); + + return blocksize; +} + +/* Compute number of fsblocks needed to store a remote attr value */ +unsigned int +xfs_attr3_rmt_blocks( + struct xfs_mount *mp, + unsigned int attrlen) +{ + /* + * Each contiguous block has a header, so it is not just a simple + * attribute length to FSB conversion. + */ + if (xfs_has_crc(mp)) + return howmany(attrlen, xfs_attr3_rmt_buf_space(mp)); + + return XFS_B_TO_FSB(mp, attrlen); +} + +/* + * Checking of the remote attribute header is split into two parts. The verifier + * does CRC, location and bounds checking, the unpacking function checks the + * attribute parameters and owner. + */ +static xfs_failaddr_t +xfs_attr3_rmt_hdr_ok( + void *ptr, + xfs_ino_t ino, + uint32_t offset, + uint32_t size, + xfs_daddr_t bno) +{ + struct xfs_attr3_rmt_hdr *rmt = ptr; + + if (bno != be64_to_cpu(rmt->rm_blkno)) + return __this_address; + if (offset != be32_to_cpu(rmt->rm_offset)) + return __this_address; + if (size != be32_to_cpu(rmt->rm_bytes)) + return __this_address; + if (ino != be64_to_cpu(rmt->rm_owner)) + return __this_address; + + /* ok */ + return NULL; +} + +static xfs_failaddr_t +xfs_attr3_rmt_verify( + struct xfs_mount *mp, + struct xfs_buf *bp, + void *ptr, + xfs_daddr_t bno) +{ + struct xfs_attr3_rmt_hdr *rmt = ptr; + + if (!xfs_verify_magic(bp, rmt->rm_magic)) + return __this_address; + if (!uuid_equal(&rmt->rm_uuid, &mp->m_sb.sb_meta_uuid)) + return __this_address; + if (be64_to_cpu(rmt->rm_blkno) != bno) + return __this_address; + if (be32_to_cpu(rmt->rm_bytes) > mp->m_attr_geo->blksize - sizeof(*rmt)) + return __this_address; + if (be32_to_cpu(rmt->rm_offset) + + be32_to_cpu(rmt->rm_bytes) > XFS_XATTR_SIZE_MAX) + return __this_address; + if (rmt->rm_owner == 0) + return __this_address; + + return NULL; +} + +static int +__xfs_attr3_rmt_read_verify( + struct xfs_buf *bp, + bool check_crc, + xfs_failaddr_t *failaddr) +{ + struct xfs_mount *mp = bp->b_mount; + char *ptr; + unsigned int len; + xfs_daddr_t bno; + unsigned int blksize = mp->m_attr_geo->blksize; + + /* no verification of non-crc buffers */ + if (!xfs_has_crc(mp)) + return 0; + + ptr = bp->b_addr; + bno = xfs_buf_daddr(bp); + len = BBTOB(bp->b_length); + ASSERT(len >= blksize); + + while (len > 0) { + if (check_crc && + !xfs_verify_cksum(ptr, blksize, XFS_ATTR3_RMT_CRC_OFF)) { + *failaddr = __this_address; + return -EFSBADCRC; + } + *failaddr = xfs_attr3_rmt_verify(mp, bp, ptr, bno); + if (*failaddr) + return -EFSCORRUPTED; + len -= blksize; + ptr += blksize; + bno += BTOBB(blksize); + } + + if (len != 0) { + *failaddr = __this_address; + return -EFSCORRUPTED; + } + + return 0; +} + +static void +xfs_attr3_rmt_read_verify( + struct xfs_buf *bp) +{ + xfs_failaddr_t fa; + int error; + + error = __xfs_attr3_rmt_read_verify(bp, true, &fa); + if (error) + xfs_verifier_error(bp, error, fa); +} + +static xfs_failaddr_t +xfs_attr3_rmt_verify_struct( + struct xfs_buf *bp) +{ + xfs_failaddr_t fa; + int error; + + error = __xfs_attr3_rmt_read_verify(bp, false, &fa); + return error ? fa : NULL; +} + +static void +xfs_attr3_rmt_write_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + xfs_failaddr_t fa; + unsigned int blksize = mp->m_attr_geo->blksize; + char *ptr; + int len; + xfs_daddr_t bno; + + /* no verification of non-crc buffers */ + if (!xfs_has_crc(mp)) + return; + + ptr = bp->b_addr; + bno = xfs_buf_daddr(bp); + len = BBTOB(bp->b_length); + ASSERT(len >= blksize); + + while (len > 0) { + struct xfs_attr3_rmt_hdr *rmt = (struct xfs_attr3_rmt_hdr *)ptr; + + fa = xfs_attr3_rmt_verify(mp, bp, ptr, bno); + if (fa) { + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + return; + } + + /* + * Ensure we aren't writing bogus LSNs to disk. See + * xfs_attr3_rmt_hdr_set() for the explanation. + */ + if (rmt->rm_lsn != cpu_to_be64(NULLCOMMITLSN)) { + xfs_verifier_error(bp, -EFSCORRUPTED, __this_address); + return; + } + xfs_update_cksum(ptr, blksize, XFS_ATTR3_RMT_CRC_OFF); + + len -= blksize; + ptr += blksize; + bno += BTOBB(blksize); + } + + if (len != 0) + xfs_verifier_error(bp, -EFSCORRUPTED, __this_address); +} + +const struct xfs_buf_ops xfs_attr3_rmt_buf_ops = { + .name = "xfs_attr3_rmt", + .magic = { 0, cpu_to_be32(XFS_ATTR3_RMT_MAGIC) }, + .verify_read = xfs_attr3_rmt_read_verify, + .verify_write = xfs_attr3_rmt_write_verify, + .verify_struct = xfs_attr3_rmt_verify_struct, +}; + +STATIC int +xfs_attr3_rmt_hdr_set( + struct xfs_mount *mp, + void *ptr, + xfs_ino_t ino, + uint32_t offset, + uint32_t size, + xfs_daddr_t bno) +{ + struct xfs_attr3_rmt_hdr *rmt = ptr; + + if (!xfs_has_crc(mp)) + return 0; + + rmt->rm_magic = cpu_to_be32(XFS_ATTR3_RMT_MAGIC); + rmt->rm_offset = cpu_to_be32(offset); + rmt->rm_bytes = cpu_to_be32(size); + uuid_copy(&rmt->rm_uuid, &mp->m_sb.sb_meta_uuid); + rmt->rm_owner = cpu_to_be64(ino); + rmt->rm_blkno = cpu_to_be64(bno); + + /* + * Remote attribute blocks are written synchronously, so we don't + * have an LSN that we can stamp in them that makes any sense to log + * recovery. To ensure that log recovery handles overwrites of these + * blocks sanely (i.e. once they've been freed and reallocated as some + * other type of metadata) we need to ensure that the LSN has a value + * that tells log recovery to ignore the LSN and overwrite the buffer + * with whatever is in it's log. To do this, we use the magic + * NULLCOMMITLSN to indicate that the LSN is invalid. + */ + rmt->rm_lsn = cpu_to_be64(NULLCOMMITLSN); + + return sizeof(struct xfs_attr3_rmt_hdr); +} + +/* + * Helper functions to copy attribute data in and out of the one disk extents + */ +STATIC int +xfs_attr_rmtval_copyout( + struct xfs_mount *mp, + struct xfs_buf *bp, + struct xfs_inode *dp, + xfs_ino_t owner, + unsigned int *offset, + unsigned int *valuelen, + uint8_t **dst) +{ + char *src = bp->b_addr; + xfs_daddr_t bno = xfs_buf_daddr(bp); + unsigned int len = BBTOB(bp->b_length); + unsigned int blksize = mp->m_attr_geo->blksize; + + ASSERT(len >= blksize); + + while (len > 0 && *valuelen > 0) { + unsigned int hdr_size = 0; + unsigned int byte_cnt = xfs_attr3_rmt_buf_space(mp); + + byte_cnt = min(*valuelen, byte_cnt); + + if (xfs_has_crc(mp)) { + if (xfs_attr3_rmt_hdr_ok(src, owner, *offset, + byte_cnt, bno)) { + xfs_alert(mp, +"remote attribute header mismatch bno/off/len/owner (0x%llx/0x%x/Ox%x/0x%llx)", + bno, *offset, byte_cnt, owner); + xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); + return -EFSCORRUPTED; + } + hdr_size = sizeof(struct xfs_attr3_rmt_hdr); + } + + memcpy(*dst, src + hdr_size, byte_cnt); + + /* roll buffer forwards */ + len -= blksize; + src += blksize; + bno += BTOBB(blksize); + + /* roll attribute data forwards */ + *valuelen -= byte_cnt; + *dst += byte_cnt; + *offset += byte_cnt; + } + return 0; +} + +STATIC void +xfs_attr_rmtval_copyin( + struct xfs_mount *mp, + struct xfs_buf *bp, + xfs_ino_t ino, + unsigned int *offset, + unsigned int *valuelen, + uint8_t **src) +{ + char *dst = bp->b_addr; + xfs_daddr_t bno = xfs_buf_daddr(bp); + unsigned int len = BBTOB(bp->b_length); + unsigned int blksize = mp->m_attr_geo->blksize; + + ASSERT(len >= blksize); + + while (len > 0 && *valuelen > 0) { + unsigned int hdr_size; + unsigned int byte_cnt = xfs_attr3_rmt_buf_space(mp); + + byte_cnt = min(*valuelen, byte_cnt); + hdr_size = xfs_attr3_rmt_hdr_set(mp, dst, ino, *offset, + byte_cnt, bno); + + memcpy(dst + hdr_size, *src, byte_cnt); + + /* + * If this is the last block, zero the remainder of it. + * Check that we are actually the last block, too. + */ + if (byte_cnt + hdr_size < blksize) { + ASSERT(*valuelen - byte_cnt == 0); + ASSERT(len == blksize); + memset(dst + hdr_size + byte_cnt, 0, + blksize - hdr_size - byte_cnt); + } + + /* roll buffer forwards */ + len -= blksize; + dst += blksize; + bno += BTOBB(blksize); + + /* roll attribute data forwards */ + *valuelen -= byte_cnt; + *src += byte_cnt; + *offset += byte_cnt; + } +} + +/* + * Read the value associated with an attribute from the out-of-line buffer + * that we stored it in. + * + * Returns 0 on successful retrieval, otherwise an error. + */ +int +xfs_attr_rmtval_get( + struct xfs_da_args *args) +{ + struct xfs_bmbt_irec map[ATTR_RMTVALUE_MAPSIZE]; + struct xfs_mount *mp = args->dp->i_mount; + struct xfs_buf *bp; + xfs_dablk_t lblkno = args->rmtblkno; + uint8_t *dst = args->value; + unsigned int valuelen; + int nmap; + int error; + unsigned int blkcnt = args->rmtblkcnt; + int i; + unsigned int offset = 0; + + trace_xfs_attr_rmtval_get(args); + + ASSERT(args->valuelen != 0); + ASSERT(args->rmtvaluelen == args->valuelen); + + valuelen = args->rmtvaluelen; + while (valuelen > 0) { + nmap = ATTR_RMTVALUE_MAPSIZE; + error = xfs_bmapi_read(args->dp, (xfs_fileoff_t)lblkno, + blkcnt, map, &nmap, + XFS_BMAPI_ATTRFORK); + if (error) + return error; + ASSERT(nmap >= 1); + + for (i = 0; (i < nmap) && (valuelen > 0); i++) { + xfs_daddr_t dblkno; + int dblkcnt; + + ASSERT((map[i].br_startblock != DELAYSTARTBLOCK) && + (map[i].br_startblock != HOLESTARTBLOCK)); + dblkno = XFS_FSB_TO_DADDR(mp, map[i].br_startblock); + dblkcnt = XFS_FSB_TO_BB(mp, map[i].br_blockcount); + error = xfs_buf_read(mp->m_ddev_targp, dblkno, dblkcnt, + 0, &bp, &xfs_attr3_rmt_buf_ops); + if (xfs_metadata_is_sick(error)) + xfs_dirattr_mark_sick(args->dp, XFS_ATTR_FORK); + /* + * ENODATA from disk implies a disk medium failure; + * ENODATA for xattrs means attribute not found, so + * disambiguate that here. + */ + if (error == -ENODATA) + error = -EIO; + if (error) + return error; + + error = xfs_attr_rmtval_copyout(mp, bp, args->dp, + args->owner, &offset, &valuelen, &dst); + xfs_buf_relse(bp); + if (error) + return error; + + /* roll attribute extent map forwards */ + lblkno += map[i].br_blockcount; + blkcnt -= map[i].br_blockcount; + } + } + ASSERT(valuelen == 0); + return 0; +} + +/* + * Find a "hole" in the attribute address space large enough for us to drop the + * new attributes value into + */ +int +xfs_attr_rmt_find_hole( + struct xfs_da_args *args) +{ + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + int error; + unsigned int blkcnt; + xfs_fileoff_t lfileoff = 0; + + /* + * Because CRC enable attributes have headers, we can't just do a + * straight byte to FSB conversion and have to take the header space + * into account. + */ + blkcnt = xfs_attr3_rmt_blocks(mp, args->rmtvaluelen); + error = xfs_bmap_first_unused(args->trans, args->dp, blkcnt, &lfileoff, + XFS_ATTR_FORK); + if (error) + return error; + + args->rmtblkno = (xfs_dablk_t)lfileoff; + args->rmtblkcnt = blkcnt; + + return 0; +} + +int +xfs_attr_rmtval_set_value( + struct xfs_da_args *args) +{ + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + struct xfs_bmbt_irec map; + xfs_dablk_t lblkno; + uint8_t *src = args->value; + unsigned int blkcnt; + unsigned int valuelen; + int nmap; + int error; + unsigned int offset = 0; + + /* + * Roll through the "value", copying the attribute value to the + * already-allocated blocks. Blocks are written synchronously + * so that we can know they are all on disk before we turn off + * the INCOMPLETE flag. + */ + lblkno = args->rmtblkno; + blkcnt = args->rmtblkcnt; + valuelen = args->rmtvaluelen; + while (valuelen > 0) { + struct xfs_buf *bp; + xfs_daddr_t dblkno; + int dblkcnt; + + ASSERT(blkcnt > 0); + + nmap = 1; + error = xfs_bmapi_read(dp, (xfs_fileoff_t)lblkno, + blkcnt, &map, &nmap, + XFS_BMAPI_ATTRFORK); + if (error) + return error; + ASSERT(nmap == 1); + ASSERT((map.br_startblock != DELAYSTARTBLOCK) && + (map.br_startblock != HOLESTARTBLOCK)); + + dblkno = XFS_FSB_TO_DADDR(mp, map.br_startblock), + dblkcnt = XFS_FSB_TO_BB(mp, map.br_blockcount); + + error = xfs_buf_get(mp->m_ddev_targp, dblkno, dblkcnt, &bp); + if (error) + return error; + bp->b_ops = &xfs_attr3_rmt_buf_ops; + + xfs_attr_rmtval_copyin(mp, bp, args->owner, &offset, &valuelen, + &src); + + error = xfs_bwrite(bp); /* GROT: NOTE: synchronous write */ + xfs_buf_relse(bp); + if (error) + return error; + + + /* roll attribute extent map forwards */ + lblkno += map.br_blockcount; + blkcnt -= map.br_blockcount; + } + ASSERT(valuelen == 0); + return 0; +} + +/* Mark stale any incore buffers for the remote value. */ +int +xfs_attr_rmtval_stale( + struct xfs_inode *ip, + struct xfs_bmbt_irec *map, + xfs_buf_flags_t incore_flags) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_buf *bp; + int error; + + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + + if (XFS_IS_CORRUPT(mp, map->br_startblock == DELAYSTARTBLOCK) || + XFS_IS_CORRUPT(mp, map->br_startblock == HOLESTARTBLOCK)) { + xfs_bmap_mark_sick(ip, XFS_ATTR_FORK); + return -EFSCORRUPTED; + } + + error = xfs_buf_incore(mp->m_ddev_targp, + XFS_FSB_TO_DADDR(mp, map->br_startblock), + XFS_FSB_TO_BB(mp, map->br_blockcount), + incore_flags, &bp); + if (error) { + if (error == -ENOENT) + return 0; + return error; + } + + xfs_buf_stale(bp); + xfs_buf_relse(bp); + return 0; +} + +/* + * Find a hole for the attr and store it in the delayed attr context. This + * initializes the context to roll through allocating an attr extent for a + * delayed attr operation + */ +int +xfs_attr_rmtval_find_space( + struct xfs_attr_intent *attr) +{ + struct xfs_da_args *args = attr->xattri_da_args; + struct xfs_bmbt_irec *map = &attr->xattri_map; + int error; + + attr->xattri_lblkno = 0; + attr->xattri_blkcnt = 0; + args->rmtblkcnt = 0; + args->rmtblkno = 0; + memset(map, 0, sizeof(struct xfs_bmbt_irec)); + + error = xfs_attr_rmt_find_hole(args); + if (error) + return error; + + attr->xattri_blkcnt = args->rmtblkcnt; + attr->xattri_lblkno = args->rmtblkno; + + return 0; +} + +/* + * Write one block of the value associated with an attribute into the + * out-of-line buffer that we have defined for it. This is similar to a subset + * of xfs_attr_rmtval_set, but records the current block to the delayed attr + * context, and leaves transaction handling to the caller. + */ +int +xfs_attr_rmtval_set_blk( + struct xfs_attr_intent *attr) +{ + struct xfs_da_args *args = attr->xattri_da_args; + struct xfs_inode *dp = args->dp; + struct xfs_bmbt_irec *map = &attr->xattri_map; + int nmap; + int error; + + nmap = 1; + error = xfs_bmapi_write(args->trans, dp, + (xfs_fileoff_t)attr->xattri_lblkno, + attr->xattri_blkcnt, XFS_BMAPI_ATTRFORK, args->total, + map, &nmap); + if (error) + return error; + + ASSERT((map->br_startblock != DELAYSTARTBLOCK) && + (map->br_startblock != HOLESTARTBLOCK)); + + /* roll attribute extent map forwards */ + attr->xattri_lblkno += map->br_blockcount; + attr->xattri_blkcnt -= map->br_blockcount; + + return 0; +} + +/* + * Remove the value associated with an attribute by deleting the + * out-of-line buffer that it is stored on. + */ +int +xfs_attr_rmtval_invalidate( + struct xfs_da_args *args) +{ + xfs_dablk_t lblkno; + unsigned int blkcnt; + int error; + + /* + * Roll through the "value", invalidating the attribute value's blocks. + */ + lblkno = args->rmtblkno; + blkcnt = args->rmtblkcnt; + while (blkcnt > 0) { + struct xfs_bmbt_irec map; + int nmap; + + /* + * Try to remember where we decided to put the value. + */ + nmap = 1; + error = xfs_bmapi_read(args->dp, (xfs_fileoff_t)lblkno, + blkcnt, &map, &nmap, XFS_BMAPI_ATTRFORK); + if (error) + return error; + if (XFS_IS_CORRUPT(args->dp->i_mount, nmap != 1)) { + xfs_bmap_mark_sick(args->dp, XFS_ATTR_FORK); + return -EFSCORRUPTED; + } + error = xfs_attr_rmtval_stale(args->dp, &map, XBF_TRYLOCK); + if (error) + return error; + + lblkno += map.br_blockcount; + blkcnt -= map.br_blockcount; + } + return 0; +} + +/* + * Remove the value associated with an attribute by deleting the out-of-line + * buffer that it is stored on. Returns -EAGAIN for the caller to refresh the + * transaction and re-call the function. Callers should keep calling this + * routine until it returns something other than -EAGAIN. + */ +int +xfs_attr_rmtval_remove( + struct xfs_attr_intent *attr) +{ + struct xfs_da_args *args = attr->xattri_da_args; + int error, done; + + /* + * Unmap value blocks for this attr. + */ + error = xfs_bunmapi(args->trans, args->dp, args->rmtblkno, + args->rmtblkcnt, XFS_BMAPI_ATTRFORK, 1, &done); + if (error) + return error; + + /* + * We don't need an explicit state here to pick up where we left off. We + * can figure it out using the !done return code. The actual value of + * attr->xattri_dela_state may be some value reminiscent of the calling + * function, but it's value is irrelevant with in the context of this + * function. Once we are done here, the next state is set as needed by + * the parent + */ + if (!done) { + trace_xfs_attr_rmtval_remove_return(attr->xattri_dela_state, + args->dp); + return -EAGAIN; + } + + args->rmtblkno = 0; + args->rmtblkcnt = 0; + return 0; +} diff --git a/fs/xfs/libxfs/xfs_attr_remote.h b/fs/xfs/libxfs/xfs_attr_remote.h new file mode 100644 index 000000000..e3c6c7d77 --- /dev/null +++ b/fs/xfs/libxfs/xfs_attr_remote.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2013 Red Hat, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_ATTR_REMOTE_H__ +#define __XFS_ATTR_REMOTE_H__ + +unsigned int xfs_attr3_rmt_blocks(struct xfs_mount *mp, unsigned int attrlen); + +/* Number of rmt blocks needed to store the maximally sized attr value */ +static inline unsigned int xfs_attr3_max_rmt_blocks(struct xfs_mount *mp) +{ + return xfs_attr3_rmt_blocks(mp, XFS_XATTR_SIZE_MAX); +} + +int xfs_attr_rmtval_get(struct xfs_da_args *args); +int xfs_attr_rmtval_stale(struct xfs_inode *ip, struct xfs_bmbt_irec *map, + xfs_buf_flags_t incore_flags); +int xfs_attr_rmtval_invalidate(struct xfs_da_args *args); +int xfs_attr_rmtval_remove(struct xfs_attr_intent *attr); +int xfs_attr_rmt_find_hole(struct xfs_da_args *args); +int xfs_attr_rmtval_set_value(struct xfs_da_args *args); +int xfs_attr_rmtval_set_blk(struct xfs_attr_intent *attr); +int xfs_attr_rmtval_find_space(struct xfs_attr_intent *attr); +#endif /* __XFS_ATTR_REMOTE_H__ */ diff --git a/fs/xfs/libxfs/xfs_attr_sf.h b/fs/xfs/libxfs/xfs_attr_sf.h new file mode 100644 index 000000000..73bdc0e55 --- /dev/null +++ b/fs/xfs/libxfs/xfs_attr_sf.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2000,2002,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_ATTR_SF_H__ +#define __XFS_ATTR_SF_H__ + +/* + * We generate this then sort it, attr_list() must return things in hash-order. + */ +typedef struct xfs_attr_sf_sort { + uint8_t entno; /* entry number in original list */ + uint8_t namelen; /* length of name value (no null) */ + uint8_t valuelen; /* length of value */ + uint8_t flags; /* flags bits (see xfs_attr_leaf.h) */ + xfs_dahash_t hash; /* this entry's hash value */ + unsigned char *name; /* name value, pointer into buffer */ + void *value; +} xfs_attr_sf_sort_t; + +#define XFS_ATTR_SF_ENTSIZE_MAX /* max space for name&value */ \ + ((1 << (NBBY*(int)sizeof(uint8_t))) - 1) + +/* space name/value uses */ +static inline int xfs_attr_sf_entsize_byname(uint8_t nlen, uint8_t vlen) +{ + return sizeof(struct xfs_attr_sf_entry) + nlen + vlen; +} + +/* space an entry uses */ +static inline int xfs_attr_sf_entsize(struct xfs_attr_sf_entry *sfep) +{ + return struct_size(sfep, nameval, sfep->namelen + sfep->valuelen); +} + +/* first entry in the SF attr fork */ +static inline struct xfs_attr_sf_entry * +xfs_attr_sf_firstentry(struct xfs_attr_sf_hdr *hdr) +{ + return (struct xfs_attr_sf_entry *)(hdr + 1); +} + +/* next entry after sfep */ +static inline struct xfs_attr_sf_entry * +xfs_attr_sf_nextentry(struct xfs_attr_sf_entry *sfep) +{ + return (void *)sfep + xfs_attr_sf_entsize(sfep); +} + +/* pointer to the space after the last entry, e.g. for adding a new one */ +static inline struct xfs_attr_sf_entry * +xfs_attr_sf_endptr(struct xfs_attr_sf_hdr *sf) +{ + return (void *)sf + be16_to_cpu(sf->totsize); +} + +#endif /* __XFS_ATTR_SF_H__ */ diff --git a/fs/xfs/libxfs/xfs_bit.c b/fs/xfs/libxfs/xfs_bit.c new file mode 100644 index 000000000..f05a07c0f --- /dev/null +++ b/fs/xfs/libxfs/xfs_bit.c @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_log_format.h" +#include "xfs_bit.h" + +/* + * XFS bit manipulation routines, used in non-realtime code. + */ + +/* + * Return whether bitmap is empty. + * Size is number of words in the bitmap, which is padded to word boundary + * Returns 1 for empty, 0 for non-empty. + */ +int +xfs_bitmap_empty(uint *map, uint size) +{ + uint i; + + for (i = 0; i < size; i++) { + if (map[i] != 0) + return 0; + } + + return 1; +} + +/* + * Count the number of contiguous bits set in the bitmap starting with bit + * start_bit. Size is the size of the bitmap in words. + */ +int +xfs_contig_bits(uint *map, uint size, uint start_bit) +{ + uint * p = ((unsigned int *) map) + (start_bit >> BIT_TO_WORD_SHIFT); + uint result = 0; + uint tmp; + + size <<= BIT_TO_WORD_SHIFT; + + ASSERT(start_bit < size); + size -= start_bit & ~(NBWORD - 1); + start_bit &= (NBWORD - 1); + if (start_bit) { + tmp = *p++; + /* set to one first offset bits prior to start */ + tmp |= (~0U >> (NBWORD-start_bit)); + if (tmp != ~0U) + goto found; + result += NBWORD; + size -= NBWORD; + } + while (size) { + if ((tmp = *p++) != ~0U) + goto found; + result += NBWORD; + size -= NBWORD; + } + return result - start_bit; +found: + return result + ffz(tmp) - start_bit; +} + +/* + * This takes the bit number to start looking from and + * returns the next set bit from there. It returns -1 + * if there are no more bits set or the start bit is + * beyond the end of the bitmap. + * + * Size is the number of words, not bytes, in the bitmap. + */ +int xfs_next_bit(uint *map, uint size, uint start_bit) +{ + uint * p = ((unsigned int *) map) + (start_bit >> BIT_TO_WORD_SHIFT); + uint result = start_bit & ~(NBWORD - 1); + uint tmp; + + size <<= BIT_TO_WORD_SHIFT; + + if (start_bit >= size) + return -1; + size -= result; + start_bit &= (NBWORD - 1); + if (start_bit) { + tmp = *p++; + /* set to zero first offset bits prior to start */ + tmp &= (~0U << start_bit); + if (tmp != 0U) + goto found; + result += NBWORD; + size -= NBWORD; + } + while (size) { + if ((tmp = *p++) != 0U) + goto found; + result += NBWORD; + size -= NBWORD; + } + return -1; +found: + return result + ffs(tmp) - 1; +} diff --git a/fs/xfs/libxfs/xfs_bit.h b/fs/xfs/libxfs/xfs_bit.h new file mode 100644 index 000000000..a04f266ae --- /dev/null +++ b/fs/xfs/libxfs/xfs_bit.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2000,2002,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_BIT_H__ +#define __XFS_BIT_H__ + +/* + * XFS bit manipulation routines. + */ + +/* + * masks with n high/low bits set, 64-bit values + */ +static inline uint64_t xfs_mask64hi(int n) +{ + return (uint64_t)-1 << (64 - (n)); +} +static inline uint32_t xfs_mask32lo(int n) +{ + return ((uint32_t)1 << (n)) - 1; +} +static inline uint64_t xfs_mask64lo(int n) +{ + return ((uint64_t)1 << (n)) - 1; +} + +/* Get high bit set out of 32-bit argument, -1 if none set */ +static inline int xfs_highbit32(uint32_t v) +{ + return fls(v) - 1; +} + +/* Get high bit set out of 64-bit argument, -1 if none set */ +static inline int xfs_highbit64(uint64_t v) +{ + return fls64(v) - 1; +} + +/* Get low bit set out of 32-bit argument, -1 if none set */ +static inline int xfs_lowbit32(uint32_t v) +{ + return ffs(v) - 1; +} + +/* Get low bit set out of 64-bit argument, -1 if none set */ +static inline int xfs_lowbit64(uint64_t v) +{ + uint32_t w = (uint32_t)v; + int n = 0; + + if (w) { /* lower bits */ + n = ffs(w); + } else { /* upper bits */ + w = (uint32_t)(v >> 32); + if (w) { + n = ffs(w); + if (n) + n += 32; + } + } + return n - 1; +} + +/* Return whether bitmap is empty (1 == empty) */ +extern int xfs_bitmap_empty(uint *map, uint size); + +/* Count continuous one bits in map starting with start_bit */ +extern int xfs_contig_bits(uint *map, uint size, uint start_bit); + +/* Find next set bit in map */ +extern int xfs_next_bit(uint *map, uint size, uint start_bit); + +#endif /* __XFS_BIT_H__ */ diff --git a/fs/xfs/libxfs/xfs_bmap.c b/fs/xfs/libxfs/xfs_bmap.c new file mode 100644 index 000000000..d64defeda --- /dev/null +++ b/fs/xfs/libxfs/xfs_bmap.c @@ -0,0 +1,6264 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2006 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_sb.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_dir2.h" +#include "xfs_inode.h" +#include "xfs_btree.h" +#include "xfs_trans.h" +#include "xfs_alloc.h" +#include "xfs_bmap.h" +#include "xfs_bmap_util.h" +#include "xfs_bmap_btree.h" +#include "xfs_rtbitmap.h" +#include "xfs_errortag.h" +#include "xfs_error.h" +#include "xfs_quota.h" +#include "xfs_trans_space.h" +#include "xfs_buf_item.h" +#include "xfs_trace.h" +#include "xfs_attr_leaf.h" +#include "xfs_filestream.h" +#include "xfs_rmap.h" +#include "xfs_ag.h" +#include "xfs_ag_resv.h" +#include "xfs_refcount.h" +#include "xfs_iomap.h" +#include "xfs_health.h" +#include "xfs_bmap_item.h" +#include "xfs_symlink_remote.h" +#include "xfs_inode_util.h" +#include "xfs_rtgroup.h" +#include "xfs_zone_alloc.h" + +struct kmem_cache *xfs_bmap_intent_cache; + +/* + * Miscellaneous helper functions + */ + +/* + * Compute and fill in the value of the maximum depth of a bmap btree + * in this filesystem. Done once, during mount. + */ +void +xfs_bmap_compute_maxlevels( + xfs_mount_t *mp, /* file system mount structure */ + int whichfork) /* data or attr fork */ +{ + uint64_t maxblocks; /* max blocks at this level */ + xfs_extnum_t maxleafents; /* max leaf entries possible */ + int level; /* btree level */ + int maxrootrecs; /* max records in root block */ + int minleafrecs; /* min records in leaf block */ + int minnoderecs; /* min records in node block */ + int sz; /* root block size */ + + /* + * The maximum number of extents in a fork, hence the maximum number of + * leaf entries, is controlled by the size of the on-disk extent count. + * + * Note that we can no longer assume that if we are in ATTR1 that the + * fork offset of all the inodes will be + * (xfs_default_attroffset(ip) >> 3) because we could have mounted with + * ATTR2 and then mounted back with ATTR1, keeping the i_forkoff's fixed + * but probably at various positions. Therefore, for both ATTR1 and + * ATTR2 we have to assume the worst case scenario of a minimum size + * available. + */ + maxleafents = xfs_iext_max_nextents(xfs_has_large_extent_counts(mp), + whichfork); + if (whichfork == XFS_DATA_FORK) + sz = xfs_bmdr_space_calc(MINDBTPTRS); + else + sz = xfs_bmdr_space_calc(MINABTPTRS); + + maxrootrecs = xfs_bmdr_maxrecs(sz, 0); + minleafrecs = mp->m_bmap_dmnr[0]; + minnoderecs = mp->m_bmap_dmnr[1]; + maxblocks = howmany_64(maxleafents, minleafrecs); + for (level = 1; maxblocks > 1; level++) { + if (maxblocks <= maxrootrecs) + maxblocks = 1; + else + maxblocks = howmany_64(maxblocks, minnoderecs); + } + mp->m_bm_maxlevels[whichfork] = level; + ASSERT(mp->m_bm_maxlevels[whichfork] <= xfs_bmbt_maxlevels_ondisk()); +} + +unsigned int +xfs_bmap_compute_attr_offset( + struct xfs_mount *mp) +{ + if (mp->m_sb.sb_inodesize == 256) + return XFS_LITINO(mp) - xfs_bmdr_space_calc(MINABTPTRS); + return xfs_bmdr_space_calc(6 * MINABTPTRS); +} + +STATIC int /* error */ +xfs_bmbt_lookup_eq( + struct xfs_btree_cur *cur, + struct xfs_bmbt_irec *irec, + int *stat) /* success/failure */ +{ + cur->bc_rec.b = *irec; + return xfs_btree_lookup(cur, XFS_LOOKUP_EQ, stat); +} + +STATIC int /* error */ +xfs_bmbt_lookup_first( + struct xfs_btree_cur *cur, + int *stat) /* success/failure */ +{ + cur->bc_rec.b.br_startoff = 0; + cur->bc_rec.b.br_startblock = 0; + cur->bc_rec.b.br_blockcount = 0; + return xfs_btree_lookup(cur, XFS_LOOKUP_GE, stat); +} + +/* + * Check if the inode needs to be converted to btree format. + */ +static inline bool xfs_bmap_needs_btree(struct xfs_inode *ip, int whichfork) +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + + return whichfork != XFS_COW_FORK && + ifp->if_format == XFS_DINODE_FMT_EXTENTS && + ifp->if_nextents > XFS_IFORK_MAXEXT(ip, whichfork); +} + +/* + * Check if the inode should be converted to extent format. + */ +static inline bool xfs_bmap_wants_extents(struct xfs_inode *ip, int whichfork) +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + + return whichfork != XFS_COW_FORK && + ifp->if_format == XFS_DINODE_FMT_BTREE && + ifp->if_nextents <= XFS_IFORK_MAXEXT(ip, whichfork); +} + +/* + * Update the record referred to by cur to the value given by irec + * This either works (return 0) or gets an EFSCORRUPTED error. + */ +STATIC int +xfs_bmbt_update( + struct xfs_btree_cur *cur, + struct xfs_bmbt_irec *irec) +{ + union xfs_btree_rec rec; + + xfs_bmbt_disk_set_all(&rec.bmbt, irec); + return xfs_btree_update(cur, &rec); +} + +/* + * Compute the worst-case number of indirect blocks that will be used + * for ip's delayed extent of length "len". + */ +xfs_filblks_t +xfs_bmap_worst_indlen( + struct xfs_inode *ip, /* incore inode pointer */ + xfs_filblks_t len) /* delayed extent length */ +{ + struct xfs_mount *mp = ip->i_mount; + int maxrecs = mp->m_bmap_dmxr[0]; + int level; + xfs_filblks_t rval; + + for (level = 0, rval = 0; + level < XFS_BM_MAXLEVELS(mp, XFS_DATA_FORK); + level++) { + len += maxrecs - 1; + do_div(len, maxrecs); + rval += len; + if (len == 1) + return rval + XFS_BM_MAXLEVELS(mp, XFS_DATA_FORK) - + level - 1; + if (level == 0) + maxrecs = mp->m_bmap_dmxr[1]; + } + return rval; +} + +/* + * Calculate the default attribute fork offset for newly created inodes. + */ +uint +xfs_default_attroffset( + struct xfs_inode *ip) +{ + if (ip->i_df.if_format == XFS_DINODE_FMT_DEV) + return roundup(sizeof(xfs_dev_t), 8); + return M_IGEO(ip->i_mount)->attr_fork_offset; +} + +/* + * Helper routine to reset inode i_forkoff field when switching attribute fork + * from local to extent format - we reset it where possible to make space + * available for inline data fork extents. + */ +STATIC void +xfs_bmap_forkoff_reset( + xfs_inode_t *ip, + int whichfork) +{ + if (whichfork == XFS_ATTR_FORK && + ip->i_df.if_format != XFS_DINODE_FMT_DEV && + ip->i_df.if_format != XFS_DINODE_FMT_BTREE) { + uint dfl_forkoff = xfs_default_attroffset(ip) >> 3; + + if (dfl_forkoff > ip->i_forkoff) + ip->i_forkoff = dfl_forkoff; + } +} + +static int +xfs_bmap_read_buf( + struct xfs_mount *mp, /* file system mount point */ + struct xfs_trans *tp, /* transaction pointer */ + xfs_fsblock_t fsbno, /* file system block number */ + struct xfs_buf **bpp) /* buffer for fsbno */ +{ + struct xfs_buf *bp; /* return value */ + int error; + + if (!xfs_verify_fsbno(mp, fsbno)) + return -EFSCORRUPTED; + error = xfs_trans_read_buf(mp, tp, mp->m_ddev_targp, + XFS_FSB_TO_DADDR(mp, fsbno), mp->m_bsize, 0, &bp, + &xfs_bmbt_buf_ops); + if (!error) { + xfs_buf_set_ref(bp, XFS_BMAP_BTREE_REF); + *bpp = bp; + } + return error; +} + +#ifdef DEBUG +STATIC struct xfs_buf * +xfs_bmap_get_bp( + struct xfs_btree_cur *cur, + xfs_fsblock_t bno) +{ + struct xfs_log_item *lip; + int i; + + if (!cur) + return NULL; + + for (i = 0; i < cur->bc_maxlevels; i++) { + if (!cur->bc_levels[i].bp) + break; + if (xfs_buf_daddr(cur->bc_levels[i].bp) == bno) + return cur->bc_levels[i].bp; + } + + /* Chase down all the log items to see if the bp is there */ + list_for_each_entry(lip, &cur->bc_tp->t_items, li_trans) { + struct xfs_buf_log_item *bip = (struct xfs_buf_log_item *)lip; + + if (bip->bli_item.li_type == XFS_LI_BUF && + xfs_buf_daddr(bip->bli_buf) == bno) + return bip->bli_buf; + } + + return NULL; +} + +STATIC void +xfs_check_block( + struct xfs_btree_block *block, + xfs_mount_t *mp, + int root, + short sz) +{ + int i, j, dmxr; + __be64 *pp, *thispa; /* pointer to block address */ + xfs_bmbt_key_t *prevp, *keyp; + + ASSERT(be16_to_cpu(block->bb_level) > 0); + + prevp = NULL; + for( i = 1; i <= xfs_btree_get_numrecs(block); i++) { + dmxr = mp->m_bmap_dmxr[0]; + keyp = xfs_bmbt_key_addr(mp, block, i); + + if (prevp) { + ASSERT(be64_to_cpu(prevp->br_startoff) < + be64_to_cpu(keyp->br_startoff)); + } + prevp = keyp; + + /* + * Compare the block numbers to see if there are dups. + */ + if (root) + pp = xfs_bmap_broot_ptr_addr(mp, block, i, sz); + else + pp = xfs_bmbt_ptr_addr(mp, block, i, dmxr); + + for (j = i+1; j <= be16_to_cpu(block->bb_numrecs); j++) { + if (root) + thispa = xfs_bmap_broot_ptr_addr(mp, block, j, sz); + else + thispa = xfs_bmbt_ptr_addr(mp, block, j, dmxr); + if (*thispa == *pp) { + xfs_warn(mp, "%s: thispa(%d) == pp(%d) %lld", + __func__, j, i, + (unsigned long long)be64_to_cpu(*thispa)); + xfs_err(mp, "%s: ptrs are equal in node\n", + __func__); + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); + } + } + } +} + +/* + * Check that the extents for the inode ip are in the right order in all + * btree leaves. THis becomes prohibitively expensive for large extent count + * files, so don't bother with inodes that have more than 10,000 extents in + * them. The btree record ordering checks will still be done, so for such large + * bmapbt constructs that is going to catch most corruptions. + */ +STATIC void +xfs_bmap_check_leaf_extents( + struct xfs_btree_cur *cur, /* btree cursor or null */ + xfs_inode_t *ip, /* incore inode pointer */ + int whichfork) /* data or attr fork */ +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + struct xfs_btree_block *block; /* current btree block */ + xfs_fsblock_t bno; /* block # of "block" */ + struct xfs_buf *bp; /* buffer for "block" */ + int error; /* error return value */ + xfs_extnum_t i=0, j; /* index into the extents list */ + int level; /* btree level, for checking */ + __be64 *pp; /* pointer to block address */ + xfs_bmbt_rec_t *ep; /* pointer to current extent */ + xfs_bmbt_rec_t last = {0, 0}; /* last extent in prev block */ + xfs_bmbt_rec_t *nextp; /* pointer to next extent */ + int bp_release = 0; + + if (ifp->if_format != XFS_DINODE_FMT_BTREE) + return; + + /* skip large extent count inodes */ + if (ip->i_df.if_nextents > 10000) + return; + + bno = NULLFSBLOCK; + block = ifp->if_broot; + /* + * Root level must use BMAP_BROOT_PTR_ADDR macro to get ptr out. + */ + level = be16_to_cpu(block->bb_level); + ASSERT(level > 0); + xfs_check_block(block, mp, 1, ifp->if_broot_bytes); + pp = xfs_bmap_broot_ptr_addr(mp, block, 1, ifp->if_broot_bytes); + bno = be64_to_cpu(*pp); + + ASSERT(bno != NULLFSBLOCK); + ASSERT(XFS_FSB_TO_AGNO(mp, bno) < mp->m_sb.sb_agcount); + ASSERT(XFS_FSB_TO_AGBNO(mp, bno) < mp->m_sb.sb_agblocks); + + /* + * Go down the tree until leaf level is reached, following the first + * pointer (leftmost) at each level. + */ + while (level-- > 0) { + /* See if buf is in cur first */ + bp_release = 0; + bp = xfs_bmap_get_bp(cur, XFS_FSB_TO_DADDR(mp, bno)); + if (!bp) { + bp_release = 1; + error = xfs_bmap_read_buf(mp, NULL, bno, &bp); + if (xfs_metadata_is_sick(error)) + xfs_btree_mark_sick(cur); + if (error) + goto error_norelse; + } + block = XFS_BUF_TO_BLOCK(bp); + if (level == 0) + break; + + /* + * Check this block for basic sanity (increasing keys and + * no duplicate blocks). + */ + + xfs_check_block(block, mp, 0, 0); + pp = xfs_bmbt_ptr_addr(mp, block, 1, mp->m_bmap_dmxr[1]); + bno = be64_to_cpu(*pp); + if (XFS_IS_CORRUPT(mp, !xfs_verify_fsbno(mp, bno))) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error0; + } + if (bp_release) { + bp_release = 0; + xfs_trans_brelse(NULL, bp); + } + } + + /* + * Here with bp and block set to the leftmost leaf node in the tree. + */ + i = 0; + + /* + * Loop over all leaf nodes checking that all extents are in the right order. + */ + for (;;) { + xfs_fsblock_t nextbno; + xfs_extnum_t num_recs; + + + num_recs = xfs_btree_get_numrecs(block); + + /* + * Read-ahead the next leaf block, if any. + */ + + nextbno = be64_to_cpu(block->bb_u.l.bb_rightsib); + + /* + * Check all the extents to make sure they are OK. + * If we had a previous block, the last entry should + * conform with the first entry in this one. + */ + + ep = xfs_bmbt_rec_addr(mp, block, 1); + if (i) { + ASSERT(xfs_bmbt_disk_get_startoff(&last) + + xfs_bmbt_disk_get_blockcount(&last) <= + xfs_bmbt_disk_get_startoff(ep)); + } + for (j = 1; j < num_recs; j++) { + nextp = xfs_bmbt_rec_addr(mp, block, j + 1); + ASSERT(xfs_bmbt_disk_get_startoff(ep) + + xfs_bmbt_disk_get_blockcount(ep) <= + xfs_bmbt_disk_get_startoff(nextp)); + ep = nextp; + } + + last = *ep; + i += num_recs; + if (bp_release) { + bp_release = 0; + xfs_trans_brelse(NULL, bp); + } + bno = nextbno; + /* + * If we've reached the end, stop. + */ + if (bno == NULLFSBLOCK) + break; + + bp_release = 0; + bp = xfs_bmap_get_bp(cur, XFS_FSB_TO_DADDR(mp, bno)); + if (!bp) { + bp_release = 1; + error = xfs_bmap_read_buf(mp, NULL, bno, &bp); + if (xfs_metadata_is_sick(error)) + xfs_btree_mark_sick(cur); + if (error) + goto error_norelse; + } + block = XFS_BUF_TO_BLOCK(bp); + } + + return; + +error0: + xfs_warn(mp, "%s: at error0", __func__); + if (bp_release) + xfs_trans_brelse(NULL, bp); +error_norelse: + xfs_warn(mp, "%s: BAD after btree leaves for %llu extents", + __func__, i); + xfs_err(mp, "%s: CORRUPTED BTREE OR SOMETHING", __func__); + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); + return; +} + +/* + * Validate that the bmbt_irecs being returned from bmapi are valid + * given the caller's original parameters. Specifically check the + * ranges of the returned irecs to ensure that they only extend beyond + * the given parameters if the XFS_BMAPI_ENTIRE flag was set. + */ +STATIC void +xfs_bmap_validate_ret( + xfs_fileoff_t bno, + xfs_filblks_t len, + uint32_t flags, + xfs_bmbt_irec_t *mval, + int nmap, + int ret_nmap) +{ + int i; /* index to map values */ + + ASSERT(ret_nmap <= nmap); + + for (i = 0; i < ret_nmap; i++) { + ASSERT(mval[i].br_blockcount > 0); + if (!(flags & XFS_BMAPI_ENTIRE)) { + ASSERT(mval[i].br_startoff >= bno); + ASSERT(mval[i].br_blockcount <= len); + ASSERT(mval[i].br_startoff + mval[i].br_blockcount <= + bno + len); + } else { + ASSERT(mval[i].br_startoff < bno + len); + ASSERT(mval[i].br_startoff + mval[i].br_blockcount > + bno); + } + ASSERT(i == 0 || + mval[i - 1].br_startoff + mval[i - 1].br_blockcount == + mval[i].br_startoff); + ASSERT(mval[i].br_startblock != DELAYSTARTBLOCK && + mval[i].br_startblock != HOLESTARTBLOCK); + ASSERT(mval[i].br_state == XFS_EXT_NORM || + mval[i].br_state == XFS_EXT_UNWRITTEN); + } +} + +#else +#define xfs_bmap_check_leaf_extents(cur, ip, whichfork) do { } while (0) +#define xfs_bmap_validate_ret(bno,len,flags,mval,onmap,nmap) do { } while (0) +#endif /* DEBUG */ + +/* + * Inode fork format manipulation functions + */ + +/* + * Convert the inode format to extent format if it currently is in btree format, + * but the extent list is small enough that it fits into the extent format. + * + * Since the extents are already in-core, all we have to do is give up the space + * for the btree root and pitch the leaf block. + */ +STATIC int /* error */ +xfs_bmap_btree_to_extents( + struct xfs_trans *tp, /* transaction pointer */ + struct xfs_inode *ip, /* incore inode pointer */ + struct xfs_btree_cur *cur, /* btree cursor */ + int *logflagsp, /* inode logging flags */ + int whichfork) /* data or attr fork */ +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + struct xfs_mount *mp = ip->i_mount; + struct xfs_btree_block *rblock = ifp->if_broot; + struct xfs_btree_block *cblock;/* child btree block */ + xfs_fsblock_t cbno; /* child block number */ + struct xfs_buf *cbp; /* child block's buffer */ + int error; /* error return value */ + __be64 *pp; /* ptr to block address */ + struct xfs_owner_info oinfo; + + /* check if we actually need the extent format first: */ + if (!xfs_bmap_wants_extents(ip, whichfork)) + return 0; + + ASSERT(cur); + ASSERT(whichfork != XFS_COW_FORK); + ASSERT(ifp->if_format == XFS_DINODE_FMT_BTREE); + ASSERT(be16_to_cpu(rblock->bb_level) == 1); + ASSERT(be16_to_cpu(rblock->bb_numrecs) == 1); + ASSERT(xfs_bmbt_maxrecs(mp, ifp->if_broot_bytes, false) == 1); + + pp = xfs_bmap_broot_ptr_addr(mp, rblock, 1, ifp->if_broot_bytes); + cbno = be64_to_cpu(*pp); +#ifdef DEBUG + if (XFS_IS_CORRUPT(cur->bc_mp, !xfs_verify_fsbno(mp, cbno))) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } +#endif + error = xfs_bmap_read_buf(mp, tp, cbno, &cbp); + if (xfs_metadata_is_sick(error)) + xfs_btree_mark_sick(cur); + if (error) + return error; + cblock = XFS_BUF_TO_BLOCK(cbp); + if ((error = xfs_btree_check_block(cur, cblock, 0, cbp))) + return error; + + xfs_rmap_inode_bmbt_owner(&oinfo, ip, whichfork); + error = xfs_free_extent_later(cur->bc_tp, cbno, 1, &oinfo, + XFS_AG_RESV_NONE, 0); + if (error) + return error; + + ip->i_nblocks--; + xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_BCOUNT, -1L); + xfs_trans_binval(tp, cbp); + if (cur->bc_levels[0].bp == cbp) + cur->bc_levels[0].bp = NULL; + xfs_bmap_broot_realloc(ip, whichfork, 0); + ASSERT(ifp->if_broot == NULL); + ifp->if_format = XFS_DINODE_FMT_EXTENTS; + *logflagsp |= XFS_ILOG_CORE | xfs_ilog_fext(whichfork); + return 0; +} + +/* + * Convert an extents-format file into a btree-format file. + * The new file will have a root block (in the inode) and a single child block. + */ +STATIC int /* error */ +xfs_bmap_extents_to_btree( + struct xfs_trans *tp, /* transaction pointer */ + struct xfs_inode *ip, /* incore inode pointer */ + struct xfs_btree_cur **curp, /* cursor returned to caller */ + int wasdel, /* converting a delayed alloc */ + int *logflagsp, /* inode logging flags */ + int whichfork) /* data or attr fork */ +{ + struct xfs_btree_block *ablock; /* allocated (child) bt block */ + struct xfs_buf *abp; /* buffer for ablock */ + struct xfs_alloc_arg args; /* allocation arguments */ + struct xfs_bmbt_rec *arp; /* child record pointer */ + struct xfs_btree_block *block; /* btree root block */ + struct xfs_btree_cur *cur; /* bmap btree cursor */ + int error; /* error return value */ + struct xfs_ifork *ifp; /* inode fork pointer */ + struct xfs_bmbt_key *kp; /* root block key pointer */ + struct xfs_mount *mp; /* mount structure */ + xfs_bmbt_ptr_t *pp; /* root block address pointer */ + struct xfs_iext_cursor icur; + struct xfs_bmbt_irec rec; + xfs_extnum_t cnt = 0; + + mp = ip->i_mount; + ASSERT(whichfork != XFS_COW_FORK); + ifp = xfs_ifork_ptr(ip, whichfork); + ASSERT(ifp->if_format == XFS_DINODE_FMT_EXTENTS); + + /* + * Make space in the inode incore. This needs to be undone if we fail + * to expand the root. + */ + block = xfs_bmap_broot_realloc(ip, whichfork, 1); + + /* + * Fill in the root. + */ + xfs_bmbt_init_block(ip, block, NULL, 1, 1); + /* + * Need a cursor. Can't allocate until bb_level is filled in. + */ + cur = xfs_bmbt_init_cursor(mp, tp, ip, whichfork); + if (wasdel) + cur->bc_flags |= XFS_BTREE_BMBT_WASDEL; + /* + * Convert to a btree with two levels, one record in root. + */ + ifp->if_format = XFS_DINODE_FMT_BTREE; + memset(&args, 0, sizeof(args)); + args.tp = tp; + args.mp = mp; + xfs_rmap_inode_bmbt_owner(&args.oinfo, ip, whichfork); + + args.minlen = args.maxlen = args.prod = 1; + args.wasdel = wasdel; + *logflagsp = 0; + error = xfs_alloc_vextent_start_ag(&args, XFS_INODE_TO_FSB(ip)); + if (error) + goto out_root_realloc; + + /* + * Allocation can't fail, the space was reserved. + */ + if (WARN_ON_ONCE(args.fsbno == NULLFSBLOCK)) { + error = -ENOSPC; + goto out_root_realloc; + } + + cur->bc_bmap.allocated++; + ip->i_nblocks++; + xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_BCOUNT, 1L); + error = xfs_trans_get_buf(tp, mp->m_ddev_targp, + XFS_FSB_TO_DADDR(mp, args.fsbno), + mp->m_bsize, 0, &abp); + if (error) + goto out_unreserve_dquot; + + /* + * Fill in the child block. + */ + ablock = XFS_BUF_TO_BLOCK(abp); + xfs_bmbt_init_block(ip, ablock, abp, 0, 0); + + for_each_xfs_iext(ifp, &icur, &rec) { + if (isnullstartblock(rec.br_startblock)) + continue; + arp = xfs_bmbt_rec_addr(mp, ablock, 1 + cnt); + xfs_bmbt_disk_set_all(arp, &rec); + cnt++; + } + ASSERT(cnt == ifp->if_nextents); + xfs_btree_set_numrecs(ablock, cnt); + + /* + * Fill in the root key and pointer. + */ + kp = xfs_bmbt_key_addr(mp, block, 1); + arp = xfs_bmbt_rec_addr(mp, ablock, 1); + kp->br_startoff = cpu_to_be64(xfs_bmbt_disk_get_startoff(arp)); + pp = xfs_bmbt_ptr_addr(mp, block, 1, xfs_bmbt_get_maxrecs(cur, + be16_to_cpu(block->bb_level))); + *pp = cpu_to_be64(args.fsbno); + + /* + * Do all this logging at the end so that + * the root is at the right level. + */ + xfs_btree_log_block(cur, abp, XFS_BB_ALL_BITS); + xfs_btree_log_recs(cur, abp, 1, be16_to_cpu(ablock->bb_numrecs)); + ASSERT(*curp == NULL); + *curp = cur; + *logflagsp = XFS_ILOG_CORE | xfs_ilog_fbroot(whichfork); + return 0; + +out_unreserve_dquot: + xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_BCOUNT, -1L); +out_root_realloc: + xfs_bmap_broot_realloc(ip, whichfork, 0); + ifp->if_format = XFS_DINODE_FMT_EXTENTS; + ASSERT(ifp->if_broot == NULL); + xfs_btree_del_cursor(cur, XFS_BTREE_ERROR); + + return error; +} + +/* + * Convert a local file to an extents file. + * This code is out of bounds for data forks of regular files, + * since the file data needs to get logged so things will stay consistent. + * (The bmap-level manipulations are ok, though). + */ +void +xfs_bmap_local_to_extents_empty( + struct xfs_trans *tp, + struct xfs_inode *ip, + int whichfork) +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + + ASSERT(whichfork != XFS_COW_FORK); + ASSERT(ifp->if_format == XFS_DINODE_FMT_LOCAL); + ASSERT(ifp->if_bytes == 0); + ASSERT(ifp->if_nextents == 0); + + xfs_bmap_forkoff_reset(ip, whichfork); + ifp->if_data = NULL; + ifp->if_height = 0; + ifp->if_format = XFS_DINODE_FMT_EXTENTS; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); +} + + +int /* error */ +xfs_bmap_local_to_extents( + xfs_trans_t *tp, /* transaction pointer */ + xfs_inode_t *ip, /* incore inode pointer */ + xfs_extlen_t total, /* total blocks needed by transaction */ + int *logflagsp, /* inode logging flags */ + int whichfork, + void (*init_fn)(struct xfs_trans *tp, + struct xfs_buf *bp, + struct xfs_inode *ip, + struct xfs_ifork *ifp, void *priv), + void *priv) +{ + int error = 0; + int flags; /* logging flags returned */ + struct xfs_ifork *ifp; /* inode fork pointer */ + xfs_alloc_arg_t args; /* allocation arguments */ + struct xfs_buf *bp; /* buffer for extent block */ + struct xfs_bmbt_irec rec; + struct xfs_iext_cursor icur; + + /* + * We don't want to deal with the case of keeping inode data inline yet. + * So sending the data fork of a regular inode is invalid. + */ + ASSERT(!(S_ISREG(VFS_I(ip)->i_mode) && whichfork == XFS_DATA_FORK)); + ifp = xfs_ifork_ptr(ip, whichfork); + ASSERT(ifp->if_format == XFS_DINODE_FMT_LOCAL); + + if (!ifp->if_bytes) { + xfs_bmap_local_to_extents_empty(tp, ip, whichfork); + flags = XFS_ILOG_CORE; + goto done; + } + + flags = 0; + error = 0; + memset(&args, 0, sizeof(args)); + args.tp = tp; + args.mp = ip->i_mount; + args.total = total; + args.minlen = args.maxlen = args.prod = 1; + xfs_rmap_inode_owner(&args.oinfo, ip, whichfork, 0); + + /* + * Allocate a block. We know we need only one, since the + * file currently fits in an inode. + */ + args.total = total; + args.minlen = args.maxlen = args.prod = 1; + error = xfs_alloc_vextent_start_ag(&args, XFS_INODE_TO_FSB(ip)); + if (error) + goto done; + + /* Can't fail, the space was reserved. */ + ASSERT(args.fsbno != NULLFSBLOCK); + ASSERT(args.len == 1); + error = xfs_trans_get_buf(tp, args.mp->m_ddev_targp, + XFS_FSB_TO_DADDR(args.mp, args.fsbno), + args.mp->m_bsize, 0, &bp); + if (error) + goto done; + + /* + * Initialize the block, copy the data and log the remote buffer. + * + * The callout is responsible for logging because the remote format + * might differ from the local format and thus we don't know how much to + * log here. Note that init_fn must also set the buffer log item type + * correctly. + */ + init_fn(tp, bp, ip, ifp, priv); + + /* account for the change in fork size */ + xfs_idata_realloc(ip, -ifp->if_bytes, whichfork); + xfs_bmap_local_to_extents_empty(tp, ip, whichfork); + flags |= XFS_ILOG_CORE; + + ifp->if_data = NULL; + ifp->if_height = 0; + + rec.br_startoff = 0; + rec.br_startblock = args.fsbno; + rec.br_blockcount = 1; + rec.br_state = XFS_EXT_NORM; + xfs_iext_first(ifp, &icur); + xfs_iext_insert(ip, &icur, &rec, 0); + + ifp->if_nextents = 1; + ip->i_nblocks = 1; + xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_BCOUNT, 1L); + flags |= xfs_ilog_fext(whichfork); + +done: + *logflagsp = flags; + return error; +} + +/* + * Called from xfs_bmap_add_attrfork to handle btree format files. + */ +STATIC int /* error */ +xfs_bmap_add_attrfork_btree( + xfs_trans_t *tp, /* transaction pointer */ + xfs_inode_t *ip, /* incore inode pointer */ + int *flags) /* inode logging flags */ +{ + struct xfs_btree_block *block = ip->i_df.if_broot; + struct xfs_btree_cur *cur; /* btree cursor */ + int error; /* error return value */ + xfs_mount_t *mp; /* file system mount struct */ + int stat; /* newroot status */ + + mp = ip->i_mount; + + if (xfs_bmap_bmdr_space(block) <= xfs_inode_data_fork_size(ip)) + *flags |= XFS_ILOG_DBROOT; + else { + cur = xfs_bmbt_init_cursor(mp, tp, ip, XFS_DATA_FORK); + error = xfs_bmbt_lookup_first(cur, &stat); + if (error) + goto error0; + /* must be at least one entry */ + if (XFS_IS_CORRUPT(mp, stat != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error0; + } + if ((error = xfs_btree_new_iroot(cur, flags, &stat))) + goto error0; + if (stat == 0) { + xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR); + return -ENOSPC; + } + cur->bc_bmap.allocated = 0; + xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR); + } + return 0; +error0: + xfs_btree_del_cursor(cur, XFS_BTREE_ERROR); + return error; +} + +/* + * Called from xfs_bmap_add_attrfork to handle extents format files. + */ +STATIC int /* error */ +xfs_bmap_add_attrfork_extents( + struct xfs_trans *tp, /* transaction pointer */ + struct xfs_inode *ip, /* incore inode pointer */ + int *flags) /* inode logging flags */ +{ + struct xfs_btree_cur *cur; /* bmap btree cursor */ + int error; /* error return value */ + + if (ip->i_df.if_nextents * sizeof(struct xfs_bmbt_rec) <= + xfs_inode_data_fork_size(ip)) + return 0; + cur = NULL; + error = xfs_bmap_extents_to_btree(tp, ip, &cur, 0, flags, + XFS_DATA_FORK); + if (cur) { + cur->bc_bmap.allocated = 0; + xfs_btree_del_cursor(cur, error); + } + return error; +} + +/* + * Called from xfs_bmap_add_attrfork to handle local format files. Each + * different data fork content type needs a different callout to do the + * conversion. Some are basic and only require special block initialisation + * callouts for the data formating, others (directories) are so specialised they + * handle everything themselves. + * + * XXX (dgc): investigate whether directory conversion can use the generic + * formatting callout. It should be possible - it's just a very complex + * formatter. + */ +STATIC int /* error */ +xfs_bmap_add_attrfork_local( + struct xfs_trans *tp, /* transaction pointer */ + struct xfs_inode *ip, /* incore inode pointer */ + int *flags) /* inode logging flags */ +{ + struct xfs_da_args dargs; /* args for dir/attr code */ + + if (ip->i_df.if_bytes <= xfs_inode_data_fork_size(ip)) + return 0; + + if (S_ISDIR(VFS_I(ip)->i_mode)) { + memset(&dargs, 0, sizeof(dargs)); + dargs.geo = ip->i_mount->m_dir_geo; + dargs.dp = ip; + dargs.total = dargs.geo->fsbcount; + dargs.whichfork = XFS_DATA_FORK; + dargs.trans = tp; + dargs.owner = I_INO(ip); + return xfs_dir2_sf_to_block(&dargs); + } + + if (S_ISLNK(VFS_I(ip)->i_mode)) + return xfs_bmap_local_to_extents(tp, ip, 1, flags, + XFS_DATA_FORK, xfs_symlink_local_to_remote, + NULL); + + /* should only be called for types that support local format data */ + ASSERT(0); + xfs_bmap_mark_sick(ip, XFS_ATTR_FORK); + return -EFSCORRUPTED; +} + +/* + * Set an inode attr fork offset based on the format of the data fork. + */ +static int +xfs_bmap_set_attrforkoff( + struct xfs_inode *ip, + int size) +{ + int default_size = xfs_default_attroffset(ip) >> 3; + + switch (ip->i_df.if_format) { + case XFS_DINODE_FMT_DEV: + ip->i_forkoff = default_size; + break; + case XFS_DINODE_FMT_LOCAL: + case XFS_DINODE_FMT_EXTENTS: + case XFS_DINODE_FMT_BTREE: + ip->i_forkoff = xfs_attr_shortform_bytesfit(ip, size); + if (!ip->i_forkoff) + ip->i_forkoff = default_size; + break; + default: + ASSERT(0); + return -EINVAL; + } + + return 0; +} + +/* + * Convert inode from non-attributed to attributed. Caller must hold the + * ILOCK_EXCL and the file cannot have an attr fork. + */ +int /* error code */ +xfs_bmap_add_attrfork( + struct xfs_trans *tp, + struct xfs_inode *ip, /* incore inode pointer */ + int size, /* space new attribute needs */ + int rsvd) /* xact may use reserved blks */ +{ + struct xfs_mount *mp = tp->t_mountp; + int logflags; /* logging flags */ + int error; /* error return value */ + + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + if (!xfs_is_metadir_inode(ip)) + ASSERT(!XFS_NOT_DQATTACHED(mp, ip)); + ASSERT(!xfs_inode_has_attr_fork(ip)); + + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + error = xfs_bmap_set_attrforkoff(ip, size); + if (error) + return error; + + xfs_ifork_init_attr(ip, XFS_DINODE_FMT_EXTENTS, 0); + logflags = 0; + switch (ip->i_df.if_format) { + case XFS_DINODE_FMT_LOCAL: + error = xfs_bmap_add_attrfork_local(tp, ip, &logflags); + break; + case XFS_DINODE_FMT_EXTENTS: + error = xfs_bmap_add_attrfork_extents(tp, ip, &logflags); + break; + case XFS_DINODE_FMT_BTREE: + error = xfs_bmap_add_attrfork_btree(tp, ip, &logflags); + break; + default: + error = 0; + break; + } + if (logflags) + xfs_trans_log_inode(tp, ip, logflags); + if (error) + return error; + if (!xfs_has_attr(mp)) { + bool log_sb = false; + + spin_lock(&mp->m_sb_lock); + if (!xfs_has_attr(mp)) { + xfs_add_attr(mp); + xfs_add_attr2(mp); + log_sb = true; + } + spin_unlock(&mp->m_sb_lock); + if (log_sb) + xfs_log_sb(tp); + } + + return 0; +} + +/* + * Internal and external extent tree search functions. + */ + +struct xfs_iread_state { + struct xfs_iext_cursor icur; + xfs_extnum_t loaded; +}; + +int +xfs_bmap_complain_bad_rec( + struct xfs_inode *ip, + int whichfork, + xfs_failaddr_t fa, + const struct xfs_bmbt_irec *irec) +{ + struct xfs_mount *mp = ip->i_mount; + const char *forkname; + + switch (whichfork) { + case XFS_DATA_FORK: forkname = "data"; break; + case XFS_ATTR_FORK: forkname = "attr"; break; + case XFS_COW_FORK: forkname = "CoW"; break; + default: forkname = "???"; break; + } + + xfs_warn(mp, + "Bmap BTree record corruption in inode 0x%llx %s fork detected at %pS!", + I_INO(ip), forkname, fa); + xfs_warn(mp, + "Offset 0x%llx, start block 0x%llx, block count 0x%llx state 0x%x", + irec->br_startoff, irec->br_startblock, irec->br_blockcount, + irec->br_state); + + return -EFSCORRUPTED; +} + +/* Stuff every bmbt record from this block into the incore extent map. */ +static int +xfs_iread_bmbt_block( + struct xfs_btree_cur *cur, + int level, + void *priv) +{ + struct xfs_iread_state *ir = priv; + struct xfs_mount *mp = cur->bc_mp; + struct xfs_inode *ip = cur->bc_ino.ip; + struct xfs_btree_block *block; + struct xfs_buf *bp; + struct xfs_bmbt_rec *frp; + xfs_extnum_t num_recs; + xfs_extnum_t j; + int whichfork = cur->bc_ino.whichfork; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + + block = xfs_btree_get_block(cur, level, &bp); + + /* Abort if we find more records than nextents. */ + num_recs = xfs_btree_get_numrecs(block); + if (unlikely(ir->loaded + num_recs > ifp->if_nextents)) { + xfs_warn(ip->i_mount, "corrupt dinode %llu, (btree extents).", + (unsigned long long)I_INO(ip)); + xfs_inode_verifier_error(ip, -EFSCORRUPTED, __func__, block, + sizeof(*block), __this_address); + xfs_bmap_mark_sick(ip, whichfork); + return -EFSCORRUPTED; + } + + /* Copy records into the incore cache. */ + frp = xfs_bmbt_rec_addr(mp, block, 1); + for (j = 0; j < num_recs; j++, frp++, ir->loaded++) { + struct xfs_bmbt_irec new; + xfs_failaddr_t fa; + + xfs_bmbt_disk_get_all(frp, &new); + fa = xfs_bmap_validate_extent(ip, whichfork, &new); + if (fa) { + xfs_inode_verifier_error(ip, -EFSCORRUPTED, + "xfs_iread_extents(2)", frp, + sizeof(*frp), fa); + xfs_bmap_mark_sick(ip, whichfork); + return xfs_bmap_complain_bad_rec(ip, whichfork, fa, + &new); + } + xfs_iext_insert(ip, &ir->icur, &new, + xfs_bmap_fork_to_state(whichfork)); + trace_xfs_read_extent(ip, &ir->icur, + xfs_bmap_fork_to_state(whichfork), _THIS_IP_); + xfs_iext_next(ifp, &ir->icur); + } + + return 0; +} + +/* + * Read in extents from a btree-format inode. + */ +int +xfs_iread_extents( + struct xfs_trans *tp, + struct xfs_inode *ip, + int whichfork) +{ + struct xfs_iread_state ir; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + struct xfs_mount *mp = ip->i_mount; + struct xfs_btree_cur *cur; + int error; + + if (!xfs_need_iread_extents(ifp)) + return 0; + + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + + ir.loaded = 0; + xfs_iext_first(ifp, &ir.icur); + cur = xfs_bmbt_init_cursor(mp, tp, ip, whichfork); + error = xfs_btree_visit_blocks(cur, xfs_iread_bmbt_block, + XFS_BTREE_VISIT_RECORDS, &ir); + xfs_btree_del_cursor(cur, error); + if (error) + goto out; + + if (XFS_IS_CORRUPT(mp, ir.loaded != ifp->if_nextents)) { + xfs_bmap_mark_sick(ip, whichfork); + error = -EFSCORRUPTED; + goto out; + } + ASSERT(ir.loaded == xfs_iext_count(ifp)); + /* + * Use release semantics so that we can use acquire semantics in + * xfs_need_iread_extents and be guaranteed to see a valid mapping tree + * after that load. + */ + smp_store_release(&ifp->if_needextents, 0); + return 0; +out: + if (xfs_metadata_is_sick(error)) + xfs_bmap_mark_sick(ip, whichfork); + xfs_iext_destroy(ifp); + return error; +} + +/* + * Returns the relative block number of the first unused block(s) in the given + * fork with at least "len" logically contiguous blocks free. This is the + * lowest-address hole if the fork has holes, else the first block past the end + * of fork. Return 0 if the fork is currently local (in-inode). + */ +int /* error */ +xfs_bmap_first_unused( + struct xfs_trans *tp, /* transaction pointer */ + struct xfs_inode *ip, /* incore inode */ + xfs_extlen_t len, /* size of hole to find */ + xfs_fileoff_t *first_unused, /* unused block */ + int whichfork) /* data or attr fork */ +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + struct xfs_bmbt_irec got; + struct xfs_iext_cursor icur; + xfs_fileoff_t lastaddr = 0; + xfs_fileoff_t lowest, max; + int error; + + if (ifp->if_format == XFS_DINODE_FMT_LOCAL) { + *first_unused = 0; + return 0; + } + + ASSERT(xfs_ifork_has_extents(ifp)); + + error = xfs_iread_extents(tp, ip, whichfork); + if (error) + return error; + + lowest = max = *first_unused; + for_each_xfs_iext(ifp, &icur, &got) { + /* + * See if the hole before this extent will work. + */ + if (got.br_startoff >= lowest + len && + got.br_startoff - max >= len) + break; + lastaddr = got.br_startoff + got.br_blockcount; + max = XFS_FILEOFF_MAX(lastaddr, lowest); + } + + *first_unused = max; + return 0; +} + +/* + * Returns the file-relative block number of the last block - 1 before + * last_block (input value) in the file. + * This is not based on i_size, it is based on the extent records. + * Returns 0 for local files, as they do not have extent records. + */ +int /* error */ +xfs_bmap_last_before( + struct xfs_trans *tp, /* transaction pointer */ + struct xfs_inode *ip, /* incore inode */ + xfs_fileoff_t *last_block, /* last block */ + int whichfork) /* data or attr fork */ +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + struct xfs_bmbt_irec got; + struct xfs_iext_cursor icur; + int error; + + switch (ifp->if_format) { + case XFS_DINODE_FMT_LOCAL: + *last_block = 0; + return 0; + case XFS_DINODE_FMT_BTREE: + case XFS_DINODE_FMT_EXTENTS: + break; + default: + ASSERT(0); + xfs_bmap_mark_sick(ip, whichfork); + return -EFSCORRUPTED; + } + + error = xfs_iread_extents(tp, ip, whichfork); + if (error) + return error; + + if (!xfs_iext_lookup_extent_before(ip, ifp, last_block, &icur, &got)) + *last_block = 0; + return 0; +} + +int +xfs_bmap_last_extent( + struct xfs_trans *tp, + struct xfs_inode *ip, + int whichfork, + struct xfs_bmbt_irec *rec, + int *is_empty) +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + struct xfs_iext_cursor icur; + int error; + + error = xfs_iread_extents(tp, ip, whichfork); + if (error) + return error; + + xfs_iext_last(ifp, &icur); + if (!xfs_iext_get_extent(ifp, &icur, rec)) + *is_empty = 1; + else + *is_empty = 0; + return 0; +} + +/* + * Check the last inode extent to determine whether this allocation will result + * in blocks being allocated at the end of the file. When we allocate new data + * blocks at the end of the file which do not start at the previous data block, + * we will try to align the new blocks at stripe unit boundaries. + * + * Returns 1 in bma->aeof if the file (fork) is empty as any new write will be + * at, or past the EOF. + */ +STATIC int +xfs_bmap_isaeof( + struct xfs_bmalloca *bma, + int whichfork) +{ + struct xfs_bmbt_irec rec; + int is_empty; + int error; + + bma->aeof = false; + error = xfs_bmap_last_extent(NULL, bma->ip, whichfork, &rec, + &is_empty); + if (error) + return error; + + if (is_empty) { + bma->aeof = true; + return 0; + } + + /* + * Check if we are allocation or past the last extent, or at least into + * the last delayed allocated extent. + */ + bma->aeof = bma->offset >= rec.br_startoff + rec.br_blockcount || + (bma->offset >= rec.br_startoff && + isnullstartblock(rec.br_startblock)); + return 0; +} + +/* + * Returns the file-relative block number of the first block past eof in + * the file. This is not based on i_size, it is based on the extent records. + * Returns 0 for local files, as they do not have extent records. + */ +int +xfs_bmap_last_offset( + struct xfs_inode *ip, + xfs_fileoff_t *last_block, + int whichfork) +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + struct xfs_bmbt_irec rec; + int is_empty; + int error; + + *last_block = 0; + + if (ifp->if_format == XFS_DINODE_FMT_LOCAL) + return 0; + + if (XFS_IS_CORRUPT(ip->i_mount, !xfs_ifork_has_extents(ifp))) { + xfs_bmap_mark_sick(ip, whichfork); + return -EFSCORRUPTED; + } + + error = xfs_bmap_last_extent(NULL, ip, whichfork, &rec, &is_empty); + if (error || is_empty) + return error; + + *last_block = rec.br_startoff + rec.br_blockcount; + return 0; +} + +/* + * Extent tree manipulation functions used during allocation. + */ + +static inline bool +xfs_bmap_same_rtgroup( + struct xfs_inode *ip, + int whichfork, + struct xfs_bmbt_irec *left, + struct xfs_bmbt_irec *right) +{ + struct xfs_mount *mp = ip->i_mount; + + if (xfs_ifork_is_realtime(ip, whichfork) && xfs_has_rtgroups(mp)) { + if (xfs_rtb_to_rgno(mp, left->br_startblock) != + xfs_rtb_to_rgno(mp, right->br_startblock)) + return false; + } + + return true; +} + +/* + * Convert a delayed allocation to a real allocation. + */ +STATIC int /* error */ +xfs_bmap_add_extent_delay_real( + struct xfs_bmalloca *bma, + int whichfork) +{ + struct xfs_mount *mp = bma->ip->i_mount; + struct xfs_ifork *ifp = xfs_ifork_ptr(bma->ip, whichfork); + struct xfs_bmbt_irec *new = &bma->got; + int error; /* error return value */ + int i; /* temp state */ + xfs_fileoff_t new_endoff; /* end offset of new entry */ + xfs_bmbt_irec_t r[3]; /* neighbor extent entries */ + /* left is 0, right is 1, prev is 2 */ + int rval=0; /* return value (logging flags) */ + uint32_t state = xfs_bmap_fork_to_state(whichfork); + xfs_filblks_t da_new; /* new count del alloc blocks used */ + xfs_filblks_t da_old; /* old count del alloc blocks used */ + xfs_filblks_t temp=0; /* value for da_new calculations */ + int tmp_rval; /* partial logging flags */ + struct xfs_bmbt_irec old; + + ASSERT(whichfork != XFS_ATTR_FORK); + ASSERT(!isnullstartblock(new->br_startblock)); + ASSERT(!bma->cur || (bma->cur->bc_flags & XFS_BTREE_BMBT_WASDEL)); + + XFS_STATS_INC(mp, xs_add_exlist); + +#define LEFT r[0] +#define RIGHT r[1] +#define PREV r[2] + + /* + * Set up a bunch of variables to make the tests simpler. + */ + xfs_iext_get_extent(ifp, &bma->icur, &PREV); + new_endoff = new->br_startoff + new->br_blockcount; + ASSERT(isnullstartblock(PREV.br_startblock)); + ASSERT(PREV.br_startoff <= new->br_startoff); + ASSERT(PREV.br_startoff + PREV.br_blockcount >= new_endoff); + + da_old = startblockval(PREV.br_startblock); + da_new = 0; + + /* + * Set flags determining what part of the previous delayed allocation + * extent is being replaced by a real allocation. + */ + if (PREV.br_startoff == new->br_startoff) + state |= BMAP_LEFT_FILLING; + if (PREV.br_startoff + PREV.br_blockcount == new_endoff) + state |= BMAP_RIGHT_FILLING; + + /* + * Check and set flags if this segment has a left neighbor. + * Don't set contiguous if the combined extent would be too large. + */ + if (xfs_iext_peek_prev_extent(ifp, &bma->icur, &LEFT)) { + state |= BMAP_LEFT_VALID; + if (isnullstartblock(LEFT.br_startblock)) + state |= BMAP_LEFT_DELAY; + } + + if ((state & BMAP_LEFT_VALID) && !(state & BMAP_LEFT_DELAY) && + LEFT.br_startoff + LEFT.br_blockcount == new->br_startoff && + LEFT.br_startblock + LEFT.br_blockcount == new->br_startblock && + LEFT.br_state == new->br_state && + LEFT.br_blockcount + new->br_blockcount <= XFS_MAX_BMBT_EXTLEN && + xfs_bmap_same_rtgroup(bma->ip, whichfork, &LEFT, new)) + state |= BMAP_LEFT_CONTIG; + + /* + * Check and set flags if this segment has a right neighbor. + * Don't set contiguous if the combined extent would be too large. + * Also check for all-three-contiguous being too large. + */ + if (xfs_iext_peek_next_extent(ifp, &bma->icur, &RIGHT)) { + state |= BMAP_RIGHT_VALID; + if (isnullstartblock(RIGHT.br_startblock)) + state |= BMAP_RIGHT_DELAY; + } + + if ((state & BMAP_RIGHT_VALID) && !(state & BMAP_RIGHT_DELAY) && + new_endoff == RIGHT.br_startoff && + new->br_startblock + new->br_blockcount == RIGHT.br_startblock && + new->br_state == RIGHT.br_state && + new->br_blockcount + RIGHT.br_blockcount <= XFS_MAX_BMBT_EXTLEN && + ((state & (BMAP_LEFT_CONTIG | BMAP_LEFT_FILLING | + BMAP_RIGHT_FILLING)) != + (BMAP_LEFT_CONTIG | BMAP_LEFT_FILLING | + BMAP_RIGHT_FILLING) || + LEFT.br_blockcount + new->br_blockcount + RIGHT.br_blockcount + <= XFS_MAX_BMBT_EXTLEN) && + xfs_bmap_same_rtgroup(bma->ip, whichfork, new, &RIGHT)) + state |= BMAP_RIGHT_CONTIG; + + error = 0; + /* + * Switch out based on the FILLING and CONTIG state bits. + */ + switch (state & (BMAP_LEFT_FILLING | BMAP_LEFT_CONTIG | + BMAP_RIGHT_FILLING | BMAP_RIGHT_CONTIG)) { + case BMAP_LEFT_FILLING | BMAP_LEFT_CONTIG | + BMAP_RIGHT_FILLING | BMAP_RIGHT_CONTIG: + /* + * Filling in all of a previously delayed allocation extent. + * The left and right neighbors are both contiguous with new. + */ + LEFT.br_blockcount += PREV.br_blockcount + RIGHT.br_blockcount; + + xfs_iext_remove(bma->ip, &bma->icur, state); + xfs_iext_remove(bma->ip, &bma->icur, state); + xfs_iext_prev(ifp, &bma->icur); + xfs_iext_update_extent(bma->ip, state, &bma->icur, &LEFT); + ifp->if_nextents--; + + if (bma->cur == NULL) + rval = XFS_ILOG_CORE | XFS_ILOG_DEXT; + else { + rval = XFS_ILOG_CORE; + error = xfs_bmbt_lookup_eq(bma->cur, &RIGHT, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(bma->cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_btree_delete(bma->cur, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(bma->cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_btree_decrement(bma->cur, 0, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(bma->cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_bmbt_update(bma->cur, &LEFT); + if (error) + goto done; + } + ASSERT(da_new <= da_old); + break; + + case BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING | BMAP_LEFT_CONTIG: + /* + * Filling in all of a previously delayed allocation extent. + * The left neighbor is contiguous, the right is not. + */ + old = LEFT; + LEFT.br_blockcount += PREV.br_blockcount; + + xfs_iext_remove(bma->ip, &bma->icur, state); + xfs_iext_prev(ifp, &bma->icur); + xfs_iext_update_extent(bma->ip, state, &bma->icur, &LEFT); + + if (bma->cur == NULL) + rval = XFS_ILOG_DEXT; + else { + rval = 0; + error = xfs_bmbt_lookup_eq(bma->cur, &old, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(bma->cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_bmbt_update(bma->cur, &LEFT); + if (error) + goto done; + } + ASSERT(da_new <= da_old); + break; + + case BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING | BMAP_RIGHT_CONTIG: + /* + * Filling in all of a previously delayed allocation extent. + * The right neighbor is contiguous, the left is not. Take care + * with delay -> unwritten extent allocation here because the + * delalloc record we are overwriting is always written. + */ + PREV.br_startblock = new->br_startblock; + PREV.br_blockcount += RIGHT.br_blockcount; + PREV.br_state = new->br_state; + + xfs_iext_next(ifp, &bma->icur); + xfs_iext_remove(bma->ip, &bma->icur, state); + xfs_iext_prev(ifp, &bma->icur); + xfs_iext_update_extent(bma->ip, state, &bma->icur, &PREV); + + if (bma->cur == NULL) + rval = XFS_ILOG_DEXT; + else { + rval = 0; + error = xfs_bmbt_lookup_eq(bma->cur, &RIGHT, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(bma->cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_bmbt_update(bma->cur, &PREV); + if (error) + goto done; + } + ASSERT(da_new <= da_old); + break; + + case BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING: + /* + * Filling in all of a previously delayed allocation extent. + * Neither the left nor right neighbors are contiguous with + * the new one. + */ + PREV.br_startblock = new->br_startblock; + PREV.br_state = new->br_state; + xfs_iext_update_extent(bma->ip, state, &bma->icur, &PREV); + ifp->if_nextents++; + + if (bma->cur == NULL) + rval = XFS_ILOG_CORE | XFS_ILOG_DEXT; + else { + rval = XFS_ILOG_CORE; + error = xfs_bmbt_lookup_eq(bma->cur, new, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 0)) { + xfs_btree_mark_sick(bma->cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_btree_insert(bma->cur, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(bma->cur); + error = -EFSCORRUPTED; + goto done; + } + } + ASSERT(da_new <= da_old); + break; + + case BMAP_LEFT_FILLING | BMAP_LEFT_CONTIG: + /* + * Filling in the first part of a previous delayed allocation. + * The left neighbor is contiguous. + */ + old = LEFT; + temp = PREV.br_blockcount - new->br_blockcount; + da_new = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(bma->ip, temp), + startblockval(PREV.br_startblock)); + + LEFT.br_blockcount += new->br_blockcount; + + PREV.br_blockcount = temp; + PREV.br_startoff += new->br_blockcount; + PREV.br_startblock = nullstartblock(da_new); + + xfs_iext_update_extent(bma->ip, state, &bma->icur, &PREV); + xfs_iext_prev(ifp, &bma->icur); + xfs_iext_update_extent(bma->ip, state, &bma->icur, &LEFT); + + if (bma->cur == NULL) + rval = XFS_ILOG_DEXT; + else { + rval = 0; + error = xfs_bmbt_lookup_eq(bma->cur, &old, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(bma->cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_bmbt_update(bma->cur, &LEFT); + if (error) + goto done; + } + ASSERT(da_new <= da_old); + break; + + case BMAP_LEFT_FILLING: + /* + * Filling in the first part of a previous delayed allocation. + * The left neighbor is not contiguous. + */ + xfs_iext_update_extent(bma->ip, state, &bma->icur, new); + ifp->if_nextents++; + + if (bma->cur == NULL) + rval = XFS_ILOG_CORE | XFS_ILOG_DEXT; + else { + rval = XFS_ILOG_CORE; + error = xfs_bmbt_lookup_eq(bma->cur, new, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 0)) { + xfs_btree_mark_sick(bma->cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_btree_insert(bma->cur, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(bma->cur); + error = -EFSCORRUPTED; + goto done; + } + } + + if (xfs_bmap_needs_btree(bma->ip, whichfork)) { + error = xfs_bmap_extents_to_btree(bma->tp, bma->ip, + &bma->cur, 1, &tmp_rval, whichfork); + rval |= tmp_rval; + if (error) + goto done; + } + + temp = PREV.br_blockcount - new->br_blockcount; + da_new = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(bma->ip, temp), + startblockval(PREV.br_startblock) - + (bma->cur ? bma->cur->bc_bmap.allocated : 0)); + + PREV.br_startoff = new_endoff; + PREV.br_blockcount = temp; + PREV.br_startblock = nullstartblock(da_new); + xfs_iext_next(ifp, &bma->icur); + xfs_iext_insert(bma->ip, &bma->icur, &PREV, state); + xfs_iext_prev(ifp, &bma->icur); + break; + + case BMAP_RIGHT_FILLING | BMAP_RIGHT_CONTIG: + /* + * Filling in the last part of a previous delayed allocation. + * The right neighbor is contiguous with the new allocation. + */ + old = RIGHT; + RIGHT.br_startoff = new->br_startoff; + RIGHT.br_startblock = new->br_startblock; + RIGHT.br_blockcount += new->br_blockcount; + + if (bma->cur == NULL) + rval = XFS_ILOG_DEXT; + else { + rval = 0; + error = xfs_bmbt_lookup_eq(bma->cur, &old, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(bma->cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_bmbt_update(bma->cur, &RIGHT); + if (error) + goto done; + } + + temp = PREV.br_blockcount - new->br_blockcount; + da_new = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(bma->ip, temp), + startblockval(PREV.br_startblock)); + + PREV.br_blockcount = temp; + PREV.br_startblock = nullstartblock(da_new); + + xfs_iext_update_extent(bma->ip, state, &bma->icur, &PREV); + xfs_iext_next(ifp, &bma->icur); + xfs_iext_update_extent(bma->ip, state, &bma->icur, &RIGHT); + ASSERT(da_new <= da_old); + break; + + case BMAP_RIGHT_FILLING: + /* + * Filling in the last part of a previous delayed allocation. + * The right neighbor is not contiguous. + */ + xfs_iext_update_extent(bma->ip, state, &bma->icur, new); + ifp->if_nextents++; + + if (bma->cur == NULL) + rval = XFS_ILOG_CORE | XFS_ILOG_DEXT; + else { + rval = XFS_ILOG_CORE; + error = xfs_bmbt_lookup_eq(bma->cur, new, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 0)) { + xfs_btree_mark_sick(bma->cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_btree_insert(bma->cur, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(bma->cur); + error = -EFSCORRUPTED; + goto done; + } + } + + if (xfs_bmap_needs_btree(bma->ip, whichfork)) { + error = xfs_bmap_extents_to_btree(bma->tp, bma->ip, + &bma->cur, 1, &tmp_rval, whichfork); + rval |= tmp_rval; + if (error) + goto done; + } + + temp = PREV.br_blockcount - new->br_blockcount; + da_new = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(bma->ip, temp), + startblockval(PREV.br_startblock) - + (bma->cur ? bma->cur->bc_bmap.allocated : 0)); + + PREV.br_startblock = nullstartblock(da_new); + PREV.br_blockcount = temp; + xfs_iext_insert(bma->ip, &bma->icur, &PREV, state); + xfs_iext_next(ifp, &bma->icur); + ASSERT(da_new <= da_old); + break; + + case 0: + /* + * Filling in the middle part of a previous delayed allocation. + * Contiguity is impossible here. + * This case is avoided almost all the time. + * + * We start with a delayed allocation: + * + * +ddddddddddddddddddddddddddddddddddddddddddddddddddddddd+ + * PREV @ idx + * + * and we are allocating: + * +rrrrrrrrrrrrrrrrr+ + * new + * + * and we set it up for insertion as: + * +ddddddddddddddddddd+rrrrrrrrrrrrrrrrr+ddddddddddddddddd+ + * new + * PREV @ idx LEFT RIGHT + * inserted at idx + 1 + */ + old = PREV; + + /* LEFT is the new middle */ + LEFT = *new; + + /* RIGHT is the new right */ + RIGHT.br_state = PREV.br_state; + RIGHT.br_startoff = new_endoff; + RIGHT.br_blockcount = + PREV.br_startoff + PREV.br_blockcount - new_endoff; + RIGHT.br_startblock = + nullstartblock(xfs_bmap_worst_indlen(bma->ip, + RIGHT.br_blockcount)); + + /* truncate PREV */ + PREV.br_blockcount = new->br_startoff - PREV.br_startoff; + PREV.br_startblock = + nullstartblock(xfs_bmap_worst_indlen(bma->ip, + PREV.br_blockcount)); + xfs_iext_update_extent(bma->ip, state, &bma->icur, &PREV); + + xfs_iext_next(ifp, &bma->icur); + xfs_iext_insert(bma->ip, &bma->icur, &RIGHT, state); + xfs_iext_insert(bma->ip, &bma->icur, &LEFT, state); + ifp->if_nextents++; + + if (bma->cur == NULL) + rval = XFS_ILOG_CORE | XFS_ILOG_DEXT; + else { + rval = XFS_ILOG_CORE; + error = xfs_bmbt_lookup_eq(bma->cur, new, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 0)) { + xfs_btree_mark_sick(bma->cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_btree_insert(bma->cur, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(bma->cur); + error = -EFSCORRUPTED; + goto done; + } + } + + if (xfs_bmap_needs_btree(bma->ip, whichfork)) { + error = xfs_bmap_extents_to_btree(bma->tp, bma->ip, + &bma->cur, 1, &tmp_rval, whichfork); + rval |= tmp_rval; + if (error) + goto done; + } + + da_new = startblockval(PREV.br_startblock) + + startblockval(RIGHT.br_startblock); + break; + + case BMAP_LEFT_FILLING | BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG: + case BMAP_RIGHT_FILLING | BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG: + case BMAP_LEFT_FILLING | BMAP_RIGHT_CONTIG: + case BMAP_RIGHT_FILLING | BMAP_LEFT_CONTIG: + case BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG: + case BMAP_LEFT_CONTIG: + case BMAP_RIGHT_CONTIG: + /* + * These cases are all impossible. + */ + ASSERT(0); + } + + /* add reverse mapping unless caller opted out */ + if (!(bma->flags & XFS_BMAPI_NORMAP)) + xfs_rmap_map_extent(bma->tp, bma->ip, whichfork, new); + + /* convert to a btree if necessary */ + if (xfs_bmap_needs_btree(bma->ip, whichfork)) { + int tmp_logflags; /* partial log flag return val */ + + ASSERT(bma->cur == NULL); + error = xfs_bmap_extents_to_btree(bma->tp, bma->ip, + &bma->cur, da_old > 0, &tmp_logflags, + whichfork); + bma->logflags |= tmp_logflags; + if (error) + goto done; + } + + if (da_new != da_old) + xfs_mod_delalloc(bma->ip, 0, (int64_t)da_new - da_old); + + if (bma->cur) { + da_new += bma->cur->bc_bmap.allocated; + bma->cur->bc_bmap.allocated = 0; + } + + /* adjust for changes in reserved delayed indirect blocks */ + if (da_new < da_old) + xfs_add_fdblocks(mp, da_old - da_new); + else if (da_new > da_old) + error = xfs_dec_fdblocks(mp, da_new - da_old, true); + + xfs_bmap_check_leaf_extents(bma->cur, bma->ip, whichfork); +done: + if (whichfork != XFS_COW_FORK) + bma->logflags |= rval; + return error; +#undef LEFT +#undef RIGHT +#undef PREV +} + +/* + * Convert an unwritten allocation to a real allocation or vice versa. + */ +int /* error */ +xfs_bmap_add_extent_unwritten_real( + struct xfs_trans *tp, + xfs_inode_t *ip, /* incore inode pointer */ + int whichfork, + struct xfs_iext_cursor *icur, + struct xfs_btree_cur **curp, /* if *curp is null, not a btree */ + xfs_bmbt_irec_t *new, /* new data to add to file extents */ + int *logflagsp) /* inode logging flags */ +{ + struct xfs_btree_cur *cur; /* btree cursor */ + int error; /* error return value */ + int i; /* temp state */ + struct xfs_ifork *ifp; /* inode fork pointer */ + xfs_fileoff_t new_endoff; /* end offset of new entry */ + xfs_bmbt_irec_t r[3]; /* neighbor extent entries */ + /* left is 0, right is 1, prev is 2 */ + int rval=0; /* return value (logging flags) */ + uint32_t state = xfs_bmap_fork_to_state(whichfork); + struct xfs_mount *mp = ip->i_mount; + struct xfs_bmbt_irec old; + + *logflagsp = 0; + + cur = *curp; + ifp = xfs_ifork_ptr(ip, whichfork); + + ASSERT(!isnullstartblock(new->br_startblock)); + + XFS_STATS_INC(mp, xs_add_exlist); + +#define LEFT r[0] +#define RIGHT r[1] +#define PREV r[2] + + /* + * Set up a bunch of variables to make the tests simpler. + */ + error = 0; + xfs_iext_get_extent(ifp, icur, &PREV); + ASSERT(new->br_state != PREV.br_state); + new_endoff = new->br_startoff + new->br_blockcount; + ASSERT(PREV.br_startoff <= new->br_startoff); + ASSERT(PREV.br_startoff + PREV.br_blockcount >= new_endoff); + + /* + * Set flags determining what part of the previous oldext allocation + * extent is being replaced by a newext allocation. + */ + if (PREV.br_startoff == new->br_startoff) + state |= BMAP_LEFT_FILLING; + if (PREV.br_startoff + PREV.br_blockcount == new_endoff) + state |= BMAP_RIGHT_FILLING; + + /* + * Check and set flags if this segment has a left neighbor. + * Don't set contiguous if the combined extent would be too large. + */ + if (xfs_iext_peek_prev_extent(ifp, icur, &LEFT)) { + state |= BMAP_LEFT_VALID; + if (isnullstartblock(LEFT.br_startblock)) + state |= BMAP_LEFT_DELAY; + } + + if ((state & BMAP_LEFT_VALID) && !(state & BMAP_LEFT_DELAY) && + LEFT.br_startoff + LEFT.br_blockcount == new->br_startoff && + LEFT.br_startblock + LEFT.br_blockcount == new->br_startblock && + LEFT.br_state == new->br_state && + LEFT.br_blockcount + new->br_blockcount <= XFS_MAX_BMBT_EXTLEN && + xfs_bmap_same_rtgroup(ip, whichfork, &LEFT, new)) + state |= BMAP_LEFT_CONTIG; + + /* + * Check and set flags if this segment has a right neighbor. + * Don't set contiguous if the combined extent would be too large. + * Also check for all-three-contiguous being too large. + */ + if (xfs_iext_peek_next_extent(ifp, icur, &RIGHT)) { + state |= BMAP_RIGHT_VALID; + if (isnullstartblock(RIGHT.br_startblock)) + state |= BMAP_RIGHT_DELAY; + } + + if ((state & BMAP_RIGHT_VALID) && !(state & BMAP_RIGHT_DELAY) && + new_endoff == RIGHT.br_startoff && + new->br_startblock + new->br_blockcount == RIGHT.br_startblock && + new->br_state == RIGHT.br_state && + new->br_blockcount + RIGHT.br_blockcount <= XFS_MAX_BMBT_EXTLEN && + ((state & (BMAP_LEFT_CONTIG | BMAP_LEFT_FILLING | + BMAP_RIGHT_FILLING)) != + (BMAP_LEFT_CONTIG | BMAP_LEFT_FILLING | + BMAP_RIGHT_FILLING) || + LEFT.br_blockcount + new->br_blockcount + RIGHT.br_blockcount + <= XFS_MAX_BMBT_EXTLEN) && + xfs_bmap_same_rtgroup(ip, whichfork, new, &RIGHT)) + state |= BMAP_RIGHT_CONTIG; + + /* + * Switch out based on the FILLING and CONTIG state bits. + */ + switch (state & (BMAP_LEFT_FILLING | BMAP_LEFT_CONTIG | + BMAP_RIGHT_FILLING | BMAP_RIGHT_CONTIG)) { + case BMAP_LEFT_FILLING | BMAP_LEFT_CONTIG | + BMAP_RIGHT_FILLING | BMAP_RIGHT_CONTIG: + /* + * Setting all of a previous oldext extent to newext. + * The left and right neighbors are both contiguous with new. + */ + LEFT.br_blockcount += PREV.br_blockcount + RIGHT.br_blockcount; + + xfs_iext_remove(ip, icur, state); + xfs_iext_remove(ip, icur, state); + xfs_iext_prev(ifp, icur); + xfs_iext_update_extent(ip, state, icur, &LEFT); + ifp->if_nextents -= 2; + if (cur == NULL) + rval = XFS_ILOG_CORE | XFS_ILOG_DEXT; + else { + rval = XFS_ILOG_CORE; + error = xfs_bmbt_lookup_eq(cur, &RIGHT, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + if ((error = xfs_btree_delete(cur, &i))) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + if ((error = xfs_btree_decrement(cur, 0, &i))) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + if ((error = xfs_btree_delete(cur, &i))) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + if ((error = xfs_btree_decrement(cur, 0, &i))) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_bmbt_update(cur, &LEFT); + if (error) + goto done; + } + break; + + case BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING | BMAP_LEFT_CONTIG: + /* + * Setting all of a previous oldext extent to newext. + * The left neighbor is contiguous, the right is not. + */ + LEFT.br_blockcount += PREV.br_blockcount; + + xfs_iext_remove(ip, icur, state); + xfs_iext_prev(ifp, icur); + xfs_iext_update_extent(ip, state, icur, &LEFT); + ifp->if_nextents--; + if (cur == NULL) + rval = XFS_ILOG_CORE | XFS_ILOG_DEXT; + else { + rval = XFS_ILOG_CORE; + error = xfs_bmbt_lookup_eq(cur, &PREV, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + if ((error = xfs_btree_delete(cur, &i))) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + if ((error = xfs_btree_decrement(cur, 0, &i))) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_bmbt_update(cur, &LEFT); + if (error) + goto done; + } + break; + + case BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING | BMAP_RIGHT_CONTIG: + /* + * Setting all of a previous oldext extent to newext. + * The right neighbor is contiguous, the left is not. + */ + PREV.br_blockcount += RIGHT.br_blockcount; + PREV.br_state = new->br_state; + + xfs_iext_next(ifp, icur); + xfs_iext_remove(ip, icur, state); + xfs_iext_prev(ifp, icur); + xfs_iext_update_extent(ip, state, icur, &PREV); + ifp->if_nextents--; + + if (cur == NULL) + rval = XFS_ILOG_CORE | XFS_ILOG_DEXT; + else { + rval = XFS_ILOG_CORE; + error = xfs_bmbt_lookup_eq(cur, &RIGHT, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + if ((error = xfs_btree_delete(cur, &i))) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + if ((error = xfs_btree_decrement(cur, 0, &i))) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_bmbt_update(cur, &PREV); + if (error) + goto done; + } + break; + + case BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING: + /* + * Setting all of a previous oldext extent to newext. + * Neither the left nor right neighbors are contiguous with + * the new one. + */ + PREV.br_state = new->br_state; + xfs_iext_update_extent(ip, state, icur, &PREV); + + if (cur == NULL) + rval = XFS_ILOG_DEXT; + else { + rval = 0; + error = xfs_bmbt_lookup_eq(cur, new, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_bmbt_update(cur, &PREV); + if (error) + goto done; + } + break; + + case BMAP_LEFT_FILLING | BMAP_LEFT_CONTIG: + /* + * Setting the first part of a previous oldext extent to newext. + * The left neighbor is contiguous. + */ + LEFT.br_blockcount += new->br_blockcount; + + old = PREV; + PREV.br_startoff += new->br_blockcount; + PREV.br_startblock += new->br_blockcount; + PREV.br_blockcount -= new->br_blockcount; + + xfs_iext_update_extent(ip, state, icur, &PREV); + xfs_iext_prev(ifp, icur); + xfs_iext_update_extent(ip, state, icur, &LEFT); + + if (cur == NULL) + rval = XFS_ILOG_DEXT; + else { + rval = 0; + error = xfs_bmbt_lookup_eq(cur, &old, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_bmbt_update(cur, &PREV); + if (error) + goto done; + error = xfs_btree_decrement(cur, 0, &i); + if (error) + goto done; + error = xfs_bmbt_update(cur, &LEFT); + if (error) + goto done; + } + break; + + case BMAP_LEFT_FILLING: + /* + * Setting the first part of a previous oldext extent to newext. + * The left neighbor is not contiguous. + */ + old = PREV; + PREV.br_startoff += new->br_blockcount; + PREV.br_startblock += new->br_blockcount; + PREV.br_blockcount -= new->br_blockcount; + + xfs_iext_update_extent(ip, state, icur, &PREV); + xfs_iext_insert(ip, icur, new, state); + ifp->if_nextents++; + + if (cur == NULL) + rval = XFS_ILOG_CORE | XFS_ILOG_DEXT; + else { + rval = XFS_ILOG_CORE; + error = xfs_bmbt_lookup_eq(cur, &old, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_bmbt_update(cur, &PREV); + if (error) + goto done; + cur->bc_rec.b = *new; + if ((error = xfs_btree_insert(cur, &i))) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + } + break; + + case BMAP_RIGHT_FILLING | BMAP_RIGHT_CONTIG: + /* + * Setting the last part of a previous oldext extent to newext. + * The right neighbor is contiguous with the new allocation. + */ + old = PREV; + PREV.br_blockcount -= new->br_blockcount; + + RIGHT.br_startoff = new->br_startoff; + RIGHT.br_startblock = new->br_startblock; + RIGHT.br_blockcount += new->br_blockcount; + + xfs_iext_update_extent(ip, state, icur, &PREV); + xfs_iext_next(ifp, icur); + xfs_iext_update_extent(ip, state, icur, &RIGHT); + + if (cur == NULL) + rval = XFS_ILOG_DEXT; + else { + rval = 0; + error = xfs_bmbt_lookup_eq(cur, &old, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_bmbt_update(cur, &PREV); + if (error) + goto done; + error = xfs_btree_increment(cur, 0, &i); + if (error) + goto done; + error = xfs_bmbt_update(cur, &RIGHT); + if (error) + goto done; + } + break; + + case BMAP_RIGHT_FILLING: + /* + * Setting the last part of a previous oldext extent to newext. + * The right neighbor is not contiguous. + */ + old = PREV; + PREV.br_blockcount -= new->br_blockcount; + + xfs_iext_update_extent(ip, state, icur, &PREV); + xfs_iext_next(ifp, icur); + xfs_iext_insert(ip, icur, new, state); + ifp->if_nextents++; + + if (cur == NULL) + rval = XFS_ILOG_CORE | XFS_ILOG_DEXT; + else { + rval = XFS_ILOG_CORE; + error = xfs_bmbt_lookup_eq(cur, &old, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_bmbt_update(cur, &PREV); + if (error) + goto done; + error = xfs_bmbt_lookup_eq(cur, new, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 0)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + if ((error = xfs_btree_insert(cur, &i))) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + } + break; + + case 0: + /* + * Setting the middle part of a previous oldext extent to + * newext. Contiguity is impossible here. + * One extent becomes three extents. + */ + old = PREV; + PREV.br_blockcount = new->br_startoff - PREV.br_startoff; + + r[0] = *new; + r[1].br_startoff = new_endoff; + r[1].br_blockcount = + old.br_startoff + old.br_blockcount - new_endoff; + r[1].br_startblock = new->br_startblock + new->br_blockcount; + r[1].br_state = PREV.br_state; + + xfs_iext_update_extent(ip, state, icur, &PREV); + xfs_iext_next(ifp, icur); + xfs_iext_insert(ip, icur, &r[1], state); + xfs_iext_insert(ip, icur, &r[0], state); + ifp->if_nextents += 2; + + if (cur == NULL) + rval = XFS_ILOG_CORE | XFS_ILOG_DEXT; + else { + rval = XFS_ILOG_CORE; + error = xfs_bmbt_lookup_eq(cur, &old, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + /* new right extent - oldext */ + error = xfs_bmbt_update(cur, &r[1]); + if (error) + goto done; + /* new left extent - oldext */ + cur->bc_rec.b = PREV; + if ((error = xfs_btree_insert(cur, &i))) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + /* + * Reset the cursor to the position of the new extent + * we are about to insert as we can't trust it after + * the previous insert. + */ + error = xfs_bmbt_lookup_eq(cur, new, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 0)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + /* new middle extent - newext */ + if ((error = xfs_btree_insert(cur, &i))) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + } + break; + + case BMAP_LEFT_FILLING | BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG: + case BMAP_RIGHT_FILLING | BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG: + case BMAP_LEFT_FILLING | BMAP_RIGHT_CONTIG: + case BMAP_RIGHT_FILLING | BMAP_LEFT_CONTIG: + case BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG: + case BMAP_LEFT_CONTIG: + case BMAP_RIGHT_CONTIG: + /* + * These cases are all impossible. + */ + ASSERT(0); + } + + /* update reverse mappings */ + xfs_rmap_convert_extent(mp, tp, ip, whichfork, new); + + /* convert to a btree if necessary */ + if (xfs_bmap_needs_btree(ip, whichfork)) { + int tmp_logflags; /* partial log flag return val */ + + ASSERT(cur == NULL); + error = xfs_bmap_extents_to_btree(tp, ip, &cur, 0, + &tmp_logflags, whichfork); + *logflagsp |= tmp_logflags; + if (error) + goto done; + } + + /* clear out the allocated field, done with it now in any case. */ + if (cur) { + cur->bc_bmap.allocated = 0; + *curp = cur; + } + + xfs_bmap_check_leaf_extents(*curp, ip, whichfork); +done: + *logflagsp |= rval; + return error; +#undef LEFT +#undef RIGHT +#undef PREV +} + +/* + * Convert a hole to a real allocation. + */ +STATIC int /* error */ +xfs_bmap_add_extent_hole_real( + struct xfs_trans *tp, + struct xfs_inode *ip, + int whichfork, + struct xfs_iext_cursor *icur, + struct xfs_btree_cur **curp, + struct xfs_bmbt_irec *new, + int *logflagsp, + uint32_t flags) +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + struct xfs_mount *mp = ip->i_mount; + struct xfs_btree_cur *cur = *curp; + int error; /* error return value */ + int i; /* temp state */ + xfs_bmbt_irec_t left; /* left neighbor extent entry */ + xfs_bmbt_irec_t right; /* right neighbor extent entry */ + int rval=0; /* return value (logging flags) */ + uint32_t state = xfs_bmap_fork_to_state(whichfork); + struct xfs_bmbt_irec old; + + ASSERT(!isnullstartblock(new->br_startblock)); + ASSERT(!cur || !(cur->bc_flags & XFS_BTREE_BMBT_WASDEL)); + + XFS_STATS_INC(mp, xs_add_exlist); + + /* + * Check and set flags if this segment has a left neighbor. + */ + if (xfs_iext_peek_prev_extent(ifp, icur, &left)) { + state |= BMAP_LEFT_VALID; + if (isnullstartblock(left.br_startblock)) + state |= BMAP_LEFT_DELAY; + } + + /* + * Check and set flags if this segment has a current value. + * Not true if we're inserting into the "hole" at eof. + */ + if (xfs_iext_get_extent(ifp, icur, &right)) { + state |= BMAP_RIGHT_VALID; + if (isnullstartblock(right.br_startblock)) + state |= BMAP_RIGHT_DELAY; + } + + /* + * We're inserting a real allocation between "left" and "right". + * Set the contiguity flags. Don't let extents get too large. + */ + if ((state & BMAP_LEFT_VALID) && !(state & BMAP_LEFT_DELAY) && + left.br_startoff + left.br_blockcount == new->br_startoff && + left.br_startblock + left.br_blockcount == new->br_startblock && + left.br_state == new->br_state && + left.br_blockcount + new->br_blockcount <= XFS_MAX_BMBT_EXTLEN && + xfs_bmap_same_rtgroup(ip, whichfork, &left, new)) + state |= BMAP_LEFT_CONTIG; + + if ((state & BMAP_RIGHT_VALID) && !(state & BMAP_RIGHT_DELAY) && + new->br_startoff + new->br_blockcount == right.br_startoff && + new->br_startblock + new->br_blockcount == right.br_startblock && + new->br_state == right.br_state && + new->br_blockcount + right.br_blockcount <= XFS_MAX_BMBT_EXTLEN && + (!(state & BMAP_LEFT_CONTIG) || + left.br_blockcount + new->br_blockcount + + right.br_blockcount <= XFS_MAX_BMBT_EXTLEN) && + xfs_bmap_same_rtgroup(ip, whichfork, new, &right)) + state |= BMAP_RIGHT_CONTIG; + + error = 0; + /* + * Select which case we're in here, and implement it. + */ + switch (state & (BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG)) { + case BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG: + /* + * New allocation is contiguous with real allocations on the + * left and on the right. + * Merge all three into a single extent record. + */ + left.br_blockcount += new->br_blockcount + right.br_blockcount; + + xfs_iext_remove(ip, icur, state); + xfs_iext_prev(ifp, icur); + xfs_iext_update_extent(ip, state, icur, &left); + ifp->if_nextents--; + + if (cur == NULL) { + rval = XFS_ILOG_CORE | xfs_ilog_fext(whichfork); + } else { + rval = XFS_ILOG_CORE; + error = xfs_bmbt_lookup_eq(cur, &right, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_btree_delete(cur, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_btree_decrement(cur, 0, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_bmbt_update(cur, &left); + if (error) + goto done; + } + break; + + case BMAP_LEFT_CONTIG: + /* + * New allocation is contiguous with a real allocation + * on the left. + * Merge the new allocation with the left neighbor. + */ + old = left; + left.br_blockcount += new->br_blockcount; + + xfs_iext_prev(ifp, icur); + xfs_iext_update_extent(ip, state, icur, &left); + + if (cur == NULL) { + rval = xfs_ilog_fext(whichfork); + } else { + rval = 0; + error = xfs_bmbt_lookup_eq(cur, &old, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_bmbt_update(cur, &left); + if (error) + goto done; + } + break; + + case BMAP_RIGHT_CONTIG: + /* + * New allocation is contiguous with a real allocation + * on the right. + * Merge the new allocation with the right neighbor. + */ + old = right; + + right.br_startoff = new->br_startoff; + right.br_startblock = new->br_startblock; + right.br_blockcount += new->br_blockcount; + xfs_iext_update_extent(ip, state, icur, &right); + + if (cur == NULL) { + rval = xfs_ilog_fext(whichfork); + } else { + rval = 0; + error = xfs_bmbt_lookup_eq(cur, &old, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_bmbt_update(cur, &right); + if (error) + goto done; + } + break; + + case 0: + /* + * New allocation is not contiguous with another + * real allocation. + * Insert a new entry. + */ + xfs_iext_insert(ip, icur, new, state); + ifp->if_nextents++; + + if (cur == NULL) { + rval = XFS_ILOG_CORE | xfs_ilog_fext(whichfork); + } else { + rval = XFS_ILOG_CORE; + error = xfs_bmbt_lookup_eq(cur, new, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 0)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_btree_insert(cur, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + } + break; + } + + /* add reverse mapping unless caller opted out */ + if (!(flags & XFS_BMAPI_NORMAP)) + xfs_rmap_map_extent(tp, ip, whichfork, new); + + /* convert to a btree if necessary */ + if (xfs_bmap_needs_btree(ip, whichfork)) { + int tmp_logflags; /* partial log flag return val */ + + ASSERT(cur == NULL); + error = xfs_bmap_extents_to_btree(tp, ip, curp, 0, + &tmp_logflags, whichfork); + *logflagsp |= tmp_logflags; + cur = *curp; + if (error) + goto done; + } + + /* clear out the allocated field, done with it now in any case. */ + if (cur) + cur->bc_bmap.allocated = 0; + + xfs_bmap_check_leaf_extents(cur, ip, whichfork); +done: + *logflagsp |= rval; + return error; +} + +/* + * Functions used in the extent read, allocate and remove paths + */ + +/* + * Adjust the size of the new extent based on i_extsize and rt extsize. + */ +int +xfs_bmap_extsize_align( + xfs_mount_t *mp, + xfs_bmbt_irec_t *gotp, /* next extent pointer */ + xfs_bmbt_irec_t *prevp, /* previous extent pointer */ + xfs_extlen_t extsz, /* align to this extent size */ + int rt, /* is this a realtime inode? */ + int eof, /* is extent at end-of-file? */ + int delay, /* creating delalloc extent? */ + int convert, /* overwriting unwritten extent? */ + xfs_fileoff_t *offp, /* in/out: aligned offset */ + xfs_extlen_t *lenp) /* in/out: aligned length */ +{ + xfs_fileoff_t orig_off; /* original offset */ + xfs_extlen_t orig_alen; /* original length */ + xfs_fileoff_t orig_end; /* original off+len */ + xfs_fileoff_t nexto; /* next file offset */ + xfs_fileoff_t prevo; /* previous file offset */ + xfs_fileoff_t align_off; /* temp for offset */ + xfs_extlen_t align_alen; /* temp for length */ + xfs_extlen_t temp; /* temp for calculations */ + + if (convert) + return 0; + + orig_off = align_off = *offp; + orig_alen = align_alen = *lenp; + orig_end = orig_off + orig_alen; + + /* + * If this request overlaps an existing extent, then don't + * attempt to perform any additional alignment. + */ + if (!delay && !eof && + (orig_off >= gotp->br_startoff) && + (orig_end <= gotp->br_startoff + gotp->br_blockcount)) { + return 0; + } + + /* + * If the file offset is unaligned vs. the extent size + * we need to align it. This will be possible unless + * the file was previously written with a kernel that didn't + * perform this alignment, or if a truncate shot us in the + * foot. + */ + div_u64_rem(orig_off, extsz, &temp); + if (temp) { + align_alen += temp; + align_off -= temp; + } + + /* Same adjustment for the end of the requested area. */ + temp = (align_alen % extsz); + if (temp) + align_alen += extsz - temp; + + /* + * For large extent hint sizes, the aligned extent might be larger than + * XFS_BMBT_MAX_EXTLEN. In that case, reduce the size by an extsz so + * that it pulls the length back under XFS_BMBT_MAX_EXTLEN. The outer + * allocation loops handle short allocation just fine, so it is safe to + * do this. We only want to do it when we are forced to, though, because + * it means more allocation operations are required. + */ + while (align_alen > XFS_MAX_BMBT_EXTLEN) + align_alen -= extsz; + ASSERT(align_alen <= XFS_MAX_BMBT_EXTLEN); + + /* + * If the previous block overlaps with this proposed allocation + * then move the start forward without adjusting the length. + */ + if (prevp->br_startoff != NULLFILEOFF) { + if (prevp->br_startblock == HOLESTARTBLOCK) + prevo = prevp->br_startoff; + else + prevo = prevp->br_startoff + prevp->br_blockcount; + } else + prevo = 0; + if (align_off != orig_off && align_off < prevo) + align_off = prevo; + /* + * If the next block overlaps with this proposed allocation + * then move the start back without adjusting the length, + * but not before offset 0. + * This may of course make the start overlap previous block, + * and if we hit the offset 0 limit then the next block + * can still overlap too. + */ + if (!eof && gotp->br_startoff != NULLFILEOFF) { + if ((delay && gotp->br_startblock == HOLESTARTBLOCK) || + (!delay && gotp->br_startblock == DELAYSTARTBLOCK)) + nexto = gotp->br_startoff + gotp->br_blockcount; + else + nexto = gotp->br_startoff; + } else + nexto = NULLFILEOFF; + if (!eof && + align_off + align_alen != orig_end && + align_off + align_alen > nexto) + align_off = nexto > align_alen ? nexto - align_alen : 0; + /* + * If we're now overlapping the next or previous extent that + * means we can't fit an extsz piece in this hole. Just move + * the start forward to the first valid spot and set + * the length so we hit the end. + */ + if (align_off != orig_off && align_off < prevo) + align_off = prevo; + if (align_off + align_alen != orig_end && + align_off + align_alen > nexto && + nexto != NULLFILEOFF) { + ASSERT(nexto > prevo); + align_alen = nexto - align_off; + } + + /* + * If realtime, and the result isn't a multiple of the realtime + * extent size we need to remove blocks until it is. + */ + if (rt && (temp = xfs_extlen_to_rtxmod(mp, align_alen))) { + /* + * We're not covering the original request, or + * we won't be able to once we fix the length. + */ + if (orig_off < align_off || + orig_end > align_off + align_alen || + align_alen - temp < orig_alen) + return -EINVAL; + /* + * Try to fix it by moving the start up. + */ + if (align_off + temp <= orig_off) { + align_alen -= temp; + align_off += temp; + } + /* + * Try to fix it by moving the end in. + */ + else if (align_off + align_alen - temp >= orig_end) + align_alen -= temp; + /* + * Set the start to the minimum then trim the length. + */ + else { + align_alen -= orig_off - align_off; + align_off = orig_off; + align_alen -= xfs_extlen_to_rtxmod(mp, align_alen); + } + /* + * Result doesn't cover the request, fail it. + */ + if (orig_off < align_off || orig_end > align_off + align_alen) + return -EINVAL; + } else { + ASSERT(orig_off >= align_off); + /* see XFS_BMBT_MAX_EXTLEN handling above */ + ASSERT(orig_end <= align_off + align_alen || + align_alen + extsz > XFS_MAX_BMBT_EXTLEN); + } + +#ifdef DEBUG + if (!eof && gotp->br_startoff != NULLFILEOFF) + ASSERT(align_off + align_alen <= gotp->br_startoff); + if (prevp->br_startoff != NULLFILEOFF) + ASSERT(align_off >= prevp->br_startoff + prevp->br_blockcount); +#endif + + *lenp = align_alen; + *offp = align_off; + return 0; +} + +static inline bool +xfs_bmap_adjacent_valid( + struct xfs_bmalloca *ap, + xfs_fsblock_t x, + xfs_fsblock_t y) +{ + struct xfs_mount *mp = ap->ip->i_mount; + + if (XFS_IS_REALTIME_INODE(ap->ip) && + (ap->datatype & XFS_ALLOC_USERDATA)) { + if (!xfs_has_rtgroups(mp)) + return x < mp->m_sb.sb_rblocks; + + return xfs_rtb_to_rgno(mp, x) == xfs_rtb_to_rgno(mp, y) && + xfs_rtb_to_rgno(mp, x) < mp->m_sb.sb_rgcount && + xfs_rtb_to_rtx(mp, x) < mp->m_sb.sb_rgextents; + + } + + return XFS_FSB_TO_AGNO(mp, x) == XFS_FSB_TO_AGNO(mp, y) && + XFS_FSB_TO_AGNO(mp, x) < mp->m_sb.sb_agcount && + XFS_FSB_TO_AGBNO(mp, x) < mp->m_sb.sb_agblocks; +} + +#define XFS_ALLOC_GAP_UNITS 4 + +/* returns true if ap->blkno was modified */ +bool +xfs_bmap_adjacent( + struct xfs_bmalloca *ap) /* bmap alloc argument struct */ +{ + xfs_fsblock_t adjust; /* adjustment to block numbers */ + + /* + * If allocating at eof, and there's a previous real block, + * try to use its last block as our starting point. + */ + if (ap->eof && ap->prev.br_startoff != NULLFILEOFF && + !isnullstartblock(ap->prev.br_startblock) && + xfs_bmap_adjacent_valid(ap, + ap->prev.br_startblock + ap->prev.br_blockcount, + ap->prev.br_startblock)) { + ap->blkno = ap->prev.br_startblock + ap->prev.br_blockcount; + /* + * Adjust for the gap between prevp and us. + */ + adjust = ap->offset - + (ap->prev.br_startoff + ap->prev.br_blockcount); + if (adjust && xfs_bmap_adjacent_valid(ap, ap->blkno + adjust, + ap->prev.br_startblock)) + ap->blkno += adjust; + return true; + } + /* + * If not at eof, then compare the two neighbor blocks. + * Figure out whether either one gives us a good starting point, + * and pick the better one. + */ + if (!ap->eof) { + xfs_fsblock_t gotbno; /* right side block number */ + xfs_fsblock_t gotdiff=0; /* right side difference */ + xfs_fsblock_t prevbno; /* left side block number */ + xfs_fsblock_t prevdiff=0; /* left side difference */ + + /* + * If there's a previous (left) block, select a requested + * start block based on it. + */ + if (ap->prev.br_startoff != NULLFILEOFF && + !isnullstartblock(ap->prev.br_startblock) && + (prevbno = ap->prev.br_startblock + + ap->prev.br_blockcount) && + xfs_bmap_adjacent_valid(ap, prevbno, + ap->prev.br_startblock)) { + /* + * Calculate gap to end of previous block. + */ + adjust = prevdiff = ap->offset - + (ap->prev.br_startoff + + ap->prev.br_blockcount); + /* + * Figure the startblock based on the previous block's + * end and the gap size. + * Heuristic! + * If the gap is large relative to the piece we're + * allocating, or using it gives us an invalid block + * number, then just use the end of the previous block. + */ + if (prevdiff <= XFS_ALLOC_GAP_UNITS * ap->length && + xfs_bmap_adjacent_valid(ap, prevbno + prevdiff, + ap->prev.br_startblock)) + prevbno += adjust; + else + prevdiff += adjust; + } + /* + * No previous block or can't follow it, just default. + */ + else + prevbno = NULLFSBLOCK; + /* + * If there's a following (right) block, select a requested + * start block based on it. + */ + if (!isnullstartblock(ap->got.br_startblock)) { + /* + * Calculate gap to start of next block. + */ + adjust = gotdiff = ap->got.br_startoff - ap->offset; + /* + * Figure the startblock based on the next block's + * start and the gap size. + */ + gotbno = ap->got.br_startblock; + /* + * Heuristic! + * If the gap is large relative to the piece we're + * allocating, or using it gives us an invalid block + * number, then just use the start of the next block + * offset by our length. + */ + if (gotdiff <= XFS_ALLOC_GAP_UNITS * ap->length && + xfs_bmap_adjacent_valid(ap, gotbno - gotdiff, + gotbno)) + gotbno -= adjust; + else if (xfs_bmap_adjacent_valid(ap, gotbno - ap->length, + gotbno)) { + gotbno -= ap->length; + gotdiff += adjust - ap->length; + } else + gotdiff += adjust; + } + /* + * No next block, just default. + */ + else + gotbno = NULLFSBLOCK; + /* + * If both valid, pick the better one, else the only good + * one, else ap->blkno is already set (to 0 or the inode block). + */ + if (prevbno != NULLFSBLOCK && gotbno != NULLFSBLOCK) { + ap->blkno = prevdiff <= gotdiff ? prevbno : gotbno; + return true; + } + if (prevbno != NULLFSBLOCK) { + ap->blkno = prevbno; + return true; + } + if (gotbno != NULLFSBLOCK) { + ap->blkno = gotbno; + return true; + } + } + + return false; +} + +int +xfs_bmap_longest_free_extent( + struct xfs_perag *pag, + struct xfs_trans *tp, + xfs_extlen_t *blen) +{ + xfs_extlen_t longest; + int error = 0; + + if (!xfs_perag_initialised_agf(pag)) { + error = xfs_alloc_read_agf(pag, tp, XFS_ALLOC_FLAG_TRYLOCK, + NULL); + if (error) + return error; + } + + longest = xfs_alloc_longest_free_extent(pag, + xfs_alloc_min_freelist(pag_mount(pag), pag), + xfs_ag_resv_needed(pag, XFS_AG_RESV_NONE)); + if (*blen < longest) + *blen = longest; + + return 0; +} + +static xfs_extlen_t +xfs_bmap_select_minlen( + struct xfs_bmalloca *ap, + struct xfs_alloc_arg *args, + xfs_extlen_t blen) +{ + + /* + * Since we used XFS_ALLOC_FLAG_TRYLOCK in _longest_free_extent(), it is + * possible that there is enough contiguous free space for this request. + */ + if (blen < ap->minlen) + return ap->minlen; + + /* + * If the best seen length is less than the request length, + * use the best as the minimum, otherwise we've got the maxlen we + * were asked for. + */ + if (blen < args->maxlen) + return blen; + return args->maxlen; +} + +static int +xfs_bmap_btalloc_select_lengths( + struct xfs_bmalloca *ap, + struct xfs_alloc_arg *args, + xfs_extlen_t *blen) +{ + struct xfs_mount *mp = args->mp; + struct xfs_perag *pag; + xfs_agnumber_t agno, startag; + int error = 0; + + if (ap->tp->t_flags & XFS_TRANS_LOWMODE) { + args->total = ap->minlen; + args->minlen = ap->minlen; + return 0; + } + + args->total = ap->total; + startag = XFS_FSB_TO_AGNO(mp, ap->blkno); + if (startag == NULLAGNUMBER) + startag = 0; + + *blen = 0; + for_each_perag_wrap(mp, startag, agno, pag) { + error = xfs_bmap_longest_free_extent(pag, args->tp, blen); + if (error && error != -EAGAIN) + break; + error = 0; + if (*blen >= args->maxlen) + break; + } + if (pag) + xfs_perag_rele(pag); + + args->minlen = xfs_bmap_select_minlen(ap, args, *blen); + return error; +} + +/* Update all inode and quota accounting for the allocation we just did. */ +void +xfs_bmap_alloc_account( + struct xfs_bmalloca *ap) +{ + bool isrt = XFS_IS_REALTIME_INODE(ap->ip) && + !(ap->flags & XFS_BMAPI_ATTRFORK); + uint fld; + + if (ap->flags & XFS_BMAPI_COWFORK) { + /* + * COW fork blocks are in-core only and thus are treated as + * in-core quota reservation (like delalloc blocks) even when + * converted to real blocks. The quota reservation is not + * accounted to disk until blocks are remapped to the data + * fork. So if these blocks were previously delalloc, we + * already have quota reservation and there's nothing to do + * yet. + */ + if (ap->wasdel) { + xfs_mod_delalloc(ap->ip, -(int64_t)ap->length, 0); + return; + } + + /* + * Otherwise, we've allocated blocks in a hole. The transaction + * has acquired in-core quota reservation for this extent. + * Rather than account these as real blocks, however, we reduce + * the transaction quota reservation based on the allocation. + * This essentially transfers the transaction quota reservation + * to that of a delalloc extent. + */ + ap->ip->i_delayed_blks += ap->length; + xfs_trans_mod_dquot_byino(ap->tp, ap->ip, isrt ? + XFS_TRANS_DQ_RES_RTBLKS : XFS_TRANS_DQ_RES_BLKS, + -(long)ap->length); + return; + } + + /* data/attr fork only */ + ap->ip->i_nblocks += ap->length; + xfs_trans_log_inode(ap->tp, ap->ip, XFS_ILOG_CORE); + if (ap->wasdel) { + ap->ip->i_delayed_blks -= ap->length; + xfs_mod_delalloc(ap->ip, -(int64_t)ap->length, 0); + fld = isrt ? XFS_TRANS_DQ_DELRTBCOUNT : XFS_TRANS_DQ_DELBCOUNT; + } else { + fld = isrt ? XFS_TRANS_DQ_RTBCOUNT : XFS_TRANS_DQ_BCOUNT; + } + + xfs_trans_mod_dquot_byino(ap->tp, ap->ip, fld, ap->length); +} + +static int +xfs_bmap_compute_alignments( + struct xfs_bmalloca *ap, + struct xfs_alloc_arg *args) +{ + struct xfs_mount *mp = args->mp; + xfs_extlen_t align = 0; /* minimum allocation alignment */ + int stripe_align = 0; + + /* stripe alignment for allocation is determined by mount parameters */ + if (mp->m_swidth && xfs_has_swalloc(mp)) + stripe_align = mp->m_swidth; + else if (mp->m_dalign) + stripe_align = mp->m_dalign; + + if (ap->flags & XFS_BMAPI_COWFORK) + align = xfs_get_cowextsz_hint(ap->ip); + else if (ap->datatype & XFS_ALLOC_USERDATA) + align = xfs_get_extsz_hint(ap->ip); + + /* Try to align start block to any minimum allocation alignment */ + if (align > 1 && (ap->flags & XFS_BMAPI_EXTSZALIGN)) + args->alignment = align; + + if (align) { + if (xfs_bmap_extsize_align(mp, &ap->got, &ap->prev, align, 0, + ap->eof, 0, ap->conv, &ap->offset, + &ap->length)) + ASSERT(0); + ASSERT(ap->length); + } + + /* apply extent size hints if obtained earlier */ + if (align) { + args->prod = align; + div_u64_rem(ap->offset, args->prod, &args->mod); + if (args->mod) + args->mod = args->prod - args->mod; + } else if (mp->m_sb.sb_blocksize >= PAGE_SIZE) { + args->prod = 1; + args->mod = 0; + } else { + args->prod = PAGE_SIZE >> mp->m_sb.sb_blocklog; + div_u64_rem(ap->offset, args->prod, &args->mod); + if (args->mod) + args->mod = args->prod - args->mod; + } + + return stripe_align; +} + +static void +xfs_bmap_process_allocated_extent( + struct xfs_bmalloca *ap, + struct xfs_alloc_arg *args, + xfs_fileoff_t orig_offset, + xfs_extlen_t orig_length) +{ + ap->blkno = args->fsbno; + ap->length = args->len; + /* + * If the extent size hint is active, we tried to round the + * caller's allocation request offset down to extsz and the + * length up to another extsz boundary. If we found a free + * extent we mapped it in starting at this new offset. If the + * newly mapped space isn't long enough to cover any of the + * range of offsets that was originally requested, move the + * mapping up so that we can fill as much of the caller's + * original request as possible. Free space is apparently + * very fragmented so we're unlikely to be able to satisfy the + * hints anyway. + */ + if (ap->length <= orig_length) + ap->offset = orig_offset; + else if (ap->offset + ap->length < orig_offset + orig_length) + ap->offset = orig_offset + orig_length - ap->length; + xfs_bmap_alloc_account(ap); +} + +static int +xfs_bmap_exact_minlen_extent_alloc( + struct xfs_bmalloca *ap, + struct xfs_alloc_arg *args) +{ + if (ap->minlen != 1) { + args->fsbno = NULLFSBLOCK; + return 0; + } + + args->alloc_minlen_only = 1; + args->minlen = args->maxlen = ap->minlen; + args->total = ap->total; + + /* + * Unlike the longest extent available in an AG, we don't track + * the length of an AG's shortest extent. + * XFS_ERRTAG_BMAP_ALLOC_MINLEN_EXTENT is a debug only knob and + * hence we can afford to start traversing from the 0th AG since + * we need not be concerned about a drop in performance in + * "debug only" code paths. + */ + ap->blkno = XFS_AGB_TO_FSB(ap->ip->i_mount, 0, 0); + + /* + * Call xfs_bmap_btalloc_low_space here as it first does a "normal" AG + * iteration and then drops args->total to args->minlen, which might be + * required to find an allocation for the transaction reservation when + * the file system is very full. + */ + return xfs_bmap_btalloc_low_space(ap, args); +} + +/* + * If we are not low on available data blocks and we are allocating at + * EOF, optimise allocation for contiguous file extension and/or stripe + * alignment of the new extent. + * + * NOTE: ap->aeof is only set if the allocation length is >= the + * stripe unit and the allocation offset is at the end of file. + */ +static int +xfs_bmap_btalloc_at_eof( + struct xfs_bmalloca *ap, + struct xfs_alloc_arg *args, + xfs_extlen_t blen, + int stripe_align, + bool ag_only) +{ + struct xfs_mount *mp = args->mp; + struct xfs_perag *caller_pag = args->pag; + int error; + + /* + * If there are already extents in the file, and xfs_bmap_adjacent() has + * given a better blkno, try an exact EOF block allocation to extend the + * file as a contiguous extent. If that fails, or it's the first + * allocation in a file, just try for a stripe aligned allocation. + */ + if (ap->eof) { + xfs_extlen_t nextminlen = 0; + + /* + * Compute the minlen+alignment for the next case. Set slop so + * that the value of minlen+alignment+slop doesn't go up between + * the calls. + */ + args->alignment = 1; + if (blen > stripe_align && blen <= args->maxlen) + nextminlen = blen - stripe_align; + else + nextminlen = args->minlen; + if (nextminlen + stripe_align > args->minlen + 1) + args->minalignslop = nextminlen + stripe_align - + args->minlen - 1; + else + args->minalignslop = 0; + + if (!caller_pag) + args->pag = xfs_perag_get(mp, XFS_FSB_TO_AGNO(mp, ap->blkno)); + error = xfs_alloc_vextent_exact_bno(args, ap->blkno); + if (!caller_pag) { + xfs_perag_put(args->pag); + args->pag = NULL; + } + if (error) + return error; + + if (args->fsbno != NULLFSBLOCK) + return 0; + /* + * Exact allocation failed. Reset to try an aligned allocation + * according to the original allocation specification. + */ + args->alignment = stripe_align; + args->minlen = nextminlen; + args->minalignslop = 0; + } else { + /* + * Adjust minlen to try and preserve alignment if we + * can't guarantee an aligned maxlen extent. + */ + args->alignment = stripe_align; + if (blen > args->alignment && + blen <= args->maxlen + args->alignment) + args->minlen = blen - args->alignment; + args->minalignslop = 0; + } + + if (ag_only) { + error = xfs_alloc_vextent_near_bno(args, ap->blkno); + } else { + args->pag = NULL; + error = xfs_alloc_vextent_start_ag(args, ap->blkno); + ASSERT(args->pag == NULL); + args->pag = caller_pag; + } + if (error) + return error; + + if (args->fsbno != NULLFSBLOCK) + return 0; + + /* + * Allocation failed, so turn return the allocation args to their + * original non-aligned state so the caller can proceed on allocation + * failure as if this function was never called. + */ + args->alignment = 1; + return 0; +} + +/* + * We have failed multiple allocation attempts so now are in a low space + * allocation situation. Try a locality first full filesystem minimum length + * allocation whilst still maintaining necessary total block reservation + * requirements. + * + * If that fails, we are now critically low on space, so perform a last resort + * allocation attempt: no reserve, no locality, blocking, minimum length, full + * filesystem free space scan. We also indicate to future allocations in this + * transaction that we are critically low on space so they don't waste time on + * allocation modes that are unlikely to succeed. + */ +int +xfs_bmap_btalloc_low_space( + struct xfs_bmalloca *ap, + struct xfs_alloc_arg *args) +{ + int error; + + if (args->minlen > ap->minlen) { + args->minlen = ap->minlen; + error = xfs_alloc_vextent_start_ag(args, ap->blkno); + if (error || args->fsbno != NULLFSBLOCK) + return error; + } + + /* Last ditch attempt before failure is declared. */ + args->total = ap->minlen; + error = xfs_alloc_vextent_first_ag(args, 0); + if (error) + return error; + ap->tp->t_flags |= XFS_TRANS_LOWMODE; + return 0; +} + +static int +xfs_bmap_btalloc_filestreams( + struct xfs_bmalloca *ap, + struct xfs_alloc_arg *args, + int stripe_align) +{ + xfs_extlen_t blen = 0; + int error = 0; + + + error = xfs_filestream_select_ag(ap, args, &blen); + if (error) + return error; + ASSERT(args->pag); + + /* + * If we are in low space mode, then optimal allocation will fail so + * prepare for minimal allocation and jump to the low space algorithm + * immediately. + */ + if (ap->tp->t_flags & XFS_TRANS_LOWMODE) { + args->minlen = ap->minlen; + ASSERT(args->fsbno == NULLFSBLOCK); + goto out_low_space; + } + + args->minlen = xfs_bmap_select_minlen(ap, args, blen); + if (ap->aeof) + error = xfs_bmap_btalloc_at_eof(ap, args, blen, stripe_align, + true); + + if (!error && args->fsbno == NULLFSBLOCK) + error = xfs_alloc_vextent_near_bno(args, ap->blkno); + +out_low_space: + /* + * We are now done with the perag reference for the filestreams + * association provided by xfs_filestream_select_ag(). Release it now as + * we've either succeeded, had a fatal error or we are out of space and + * need to do a full filesystem scan for free space which will take it's + * own references. + */ + xfs_perag_rele(args->pag); + args->pag = NULL; + if (error || args->fsbno != NULLFSBLOCK) + return error; + + return xfs_bmap_btalloc_low_space(ap, args); +} + +static int +xfs_bmap_btalloc_best_length( + struct xfs_bmalloca *ap, + struct xfs_alloc_arg *args, + int stripe_align) +{ + xfs_extlen_t blen = 0; + int error; + + ap->blkno = XFS_INODE_TO_FSB(ap->ip); + if (!xfs_bmap_adjacent(ap)) + ap->eof = false; + + /* + * Search for an allocation group with a single extent large enough for + * the request. If one isn't found, then adjust the minimum allocation + * size to the largest space found. + */ + error = xfs_bmap_btalloc_select_lengths(ap, args, &blen); + if (error) + return error; + + /* + * Don't attempt optimal EOF allocation if previous allocations barely + * succeeded due to being near ENOSPC. It is highly unlikely we'll get + * optimal or even aligned allocations in this case, so don't waste time + * trying. + */ + if (ap->aeof && !(ap->tp->t_flags & XFS_TRANS_LOWMODE)) { + error = xfs_bmap_btalloc_at_eof(ap, args, blen, stripe_align, + false); + if (error || args->fsbno != NULLFSBLOCK) + return error; + } + + error = xfs_alloc_vextent_start_ag(args, ap->blkno); + if (error || args->fsbno != NULLFSBLOCK) + return error; + + return xfs_bmap_btalloc_low_space(ap, args); +} + +static int +xfs_bmap_btalloc( + struct xfs_bmalloca *ap) +{ + struct xfs_mount *mp = ap->ip->i_mount; + struct xfs_alloc_arg args = { + .tp = ap->tp, + .mp = mp, + .fsbno = NULLFSBLOCK, + .oinfo = XFS_RMAP_OINFO_SKIP_UPDATE, + .minleft = ap->minleft, + .wasdel = ap->wasdel, + .resv = XFS_AG_RESV_NONE, + .datatype = ap->datatype, + .alignment = 1, + .minalignslop = 0, + }; + xfs_fileoff_t orig_offset; + xfs_extlen_t orig_length; + int error; + int stripe_align; + + ASSERT(ap->length); + orig_offset = ap->offset; + orig_length = ap->length; + + stripe_align = xfs_bmap_compute_alignments(ap, &args); + + /* Trim the allocation back to the maximum an AG can fit. */ + args.maxlen = min(ap->length, mp->m_ag_max_usable); + + if (unlikely(XFS_TEST_ERROR(mp, XFS_ERRTAG_BMAP_ALLOC_MINLEN_EXTENT))) + error = xfs_bmap_exact_minlen_extent_alloc(ap, &args); + else if ((ap->datatype & XFS_ALLOC_USERDATA) && + xfs_inode_is_filestream(ap->ip)) + error = xfs_bmap_btalloc_filestreams(ap, &args, stripe_align); + else + error = xfs_bmap_btalloc_best_length(ap, &args, stripe_align); + if (error) + return error; + + if (args.fsbno != NULLFSBLOCK) { + xfs_bmap_process_allocated_extent(ap, &args, orig_offset, + orig_length); + } else { + ap->blkno = NULLFSBLOCK; + ap->length = 0; + } + return 0; +} + +/* Trim extent to fit a logical block range. */ +void +xfs_trim_extent( + struct xfs_bmbt_irec *irec, + xfs_fileoff_t bno, + xfs_filblks_t len) +{ + xfs_fileoff_t distance; + xfs_fileoff_t end = bno + len; + + if (irec->br_startoff + irec->br_blockcount <= bno || + irec->br_startoff >= end) { + irec->br_blockcount = 0; + return; + } + + if (irec->br_startoff < bno) { + distance = bno - irec->br_startoff; + if (isnullstartblock(irec->br_startblock)) + irec->br_startblock = DELAYSTARTBLOCK; + if (irec->br_startblock != DELAYSTARTBLOCK && + irec->br_startblock != HOLESTARTBLOCK) + irec->br_startblock += distance; + irec->br_startoff += distance; + irec->br_blockcount -= distance; + } + + if (end < irec->br_startoff + irec->br_blockcount) { + distance = irec->br_startoff + irec->br_blockcount - end; + irec->br_blockcount -= distance; + } +} + +/* + * Trim the returned map to the required bounds + */ +STATIC void +xfs_bmapi_trim_map( + struct xfs_bmbt_irec *mval, + struct xfs_bmbt_irec *got, + xfs_fileoff_t *bno, + xfs_filblks_t len, + xfs_fileoff_t obno, + xfs_fileoff_t end, + int n, + uint32_t flags) +{ + if ((flags & XFS_BMAPI_ENTIRE) || + got->br_startoff + got->br_blockcount <= obno) { + *mval = *got; + if (isnullstartblock(got->br_startblock)) + mval->br_startblock = DELAYSTARTBLOCK; + return; + } + + if (obno > *bno) + *bno = obno; + ASSERT((*bno >= obno) || (n == 0)); + ASSERT(*bno < end); + mval->br_startoff = *bno; + if (isnullstartblock(got->br_startblock)) + mval->br_startblock = DELAYSTARTBLOCK; + else + mval->br_startblock = got->br_startblock + + (*bno - got->br_startoff); + /* + * Return the minimum of what we got and what we asked for for + * the length. We can use the len variable here because it is + * modified below and we could have been there before coming + * here if the first part of the allocation didn't overlap what + * was asked for. + */ + mval->br_blockcount = XFS_FILBLKS_MIN(end - *bno, + got->br_blockcount - (*bno - got->br_startoff)); + mval->br_state = got->br_state; + ASSERT(mval->br_blockcount <= len); + return; +} + +/* + * Update and validate the extent map to return + */ +STATIC void +xfs_bmapi_update_map( + struct xfs_bmbt_irec **map, + xfs_fileoff_t *bno, + xfs_filblks_t *len, + xfs_fileoff_t obno, + xfs_fileoff_t end, + int *n, + uint32_t flags) +{ + xfs_bmbt_irec_t *mval = *map; + + ASSERT((flags & XFS_BMAPI_ENTIRE) || + ((mval->br_startoff + mval->br_blockcount) <= end)); + ASSERT((flags & XFS_BMAPI_ENTIRE) || (mval->br_blockcount <= *len) || + (mval->br_startoff < obno)); + + *bno = mval->br_startoff + mval->br_blockcount; + *len = end - *bno; + if (*n > 0 && mval->br_startoff == mval[-1].br_startoff) { + /* update previous map with new information */ + ASSERT(mval->br_startblock == mval[-1].br_startblock); + ASSERT(mval->br_blockcount > mval[-1].br_blockcount); + ASSERT(mval->br_state == mval[-1].br_state); + mval[-1].br_blockcount = mval->br_blockcount; + mval[-1].br_state = mval->br_state; + } else if (*n > 0 && mval->br_startblock != DELAYSTARTBLOCK && + mval[-1].br_startblock != DELAYSTARTBLOCK && + mval[-1].br_startblock != HOLESTARTBLOCK && + mval->br_startblock == mval[-1].br_startblock + + mval[-1].br_blockcount && + mval[-1].br_state == mval->br_state) { + ASSERT(mval->br_startoff == + mval[-1].br_startoff + mval[-1].br_blockcount); + mval[-1].br_blockcount += mval->br_blockcount; + } else if (*n > 0 && + mval->br_startblock == DELAYSTARTBLOCK && + mval[-1].br_startblock == DELAYSTARTBLOCK && + mval->br_startoff == + mval[-1].br_startoff + mval[-1].br_blockcount) { + mval[-1].br_blockcount += mval->br_blockcount; + mval[-1].br_state = mval->br_state; + } else if (!((*n == 0) && + ((mval->br_startoff + mval->br_blockcount) <= + obno))) { + mval++; + (*n)++; + } + *map = mval; +} + +/* + * Map file blocks to filesystem blocks without allocation. + */ +int +xfs_bmapi_read( + struct xfs_inode *ip, + xfs_fileoff_t bno, + xfs_filblks_t len, + struct xfs_bmbt_irec *mval, + int *nmap, + uint32_t flags) +{ + struct xfs_mount *mp = ip->i_mount; + int whichfork = xfs_bmapi_whichfork(flags); + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + struct xfs_bmbt_irec got; + xfs_fileoff_t obno; + xfs_fileoff_t end; + struct xfs_iext_cursor icur; + int error; + bool eof = false; + int n = 0; + + ASSERT(*nmap >= 1); + ASSERT(!(flags & ~(XFS_BMAPI_ATTRFORK | XFS_BMAPI_ENTIRE))); + xfs_assert_ilocked(ip, XFS_ILOCK_SHARED | XFS_ILOCK_EXCL); + + if (WARN_ON_ONCE(!ifp)) { + xfs_bmap_mark_sick(ip, whichfork); + return -EFSCORRUPTED; + } + + if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ifp)) || + XFS_TEST_ERROR(mp, XFS_ERRTAG_BMAPIFORMAT)) { + xfs_bmap_mark_sick(ip, whichfork); + return -EFSCORRUPTED; + } + + if (xfs_is_shutdown(mp)) + return -EIO; + + XFS_STATS_INC(mp, xs_blk_mapr); + + error = xfs_iread_extents(NULL, ip, whichfork); + if (error) + return error; + + if (!xfs_iext_lookup_extent(ip, ifp, bno, &icur, &got)) + eof = true; + end = bno + len; + obno = bno; + + while (bno < end && n < *nmap) { + /* Reading past eof, act as though there's a hole up to end. */ + if (eof) + got.br_startoff = end; + if (got.br_startoff > bno) { + /* Reading in a hole. */ + mval->br_startoff = bno; + mval->br_startblock = HOLESTARTBLOCK; + mval->br_blockcount = + XFS_FILBLKS_MIN(len, got.br_startoff - bno); + mval->br_state = XFS_EXT_NORM; + bno += mval->br_blockcount; + len -= mval->br_blockcount; + mval++; + n++; + continue; + } + + /* set up the extent map to return. */ + xfs_bmapi_trim_map(mval, &got, &bno, len, obno, end, n, flags); + xfs_bmapi_update_map(&mval, &bno, &len, obno, end, &n, flags); + + /* If we're done, stop now. */ + if (bno >= end || n >= *nmap) + break; + + /* Else go on to the next record. */ + if (!xfs_iext_next_extent(ifp, &icur, &got)) + eof = true; + } + *nmap = n; + return 0; +} + +static int +xfs_bmapi_allocate( + struct xfs_bmalloca *bma) +{ + struct xfs_mount *mp = bma->ip->i_mount; + int whichfork = xfs_bmapi_whichfork(bma->flags); + struct xfs_ifork *ifp = xfs_ifork_ptr(bma->ip, whichfork); + int error; + + ASSERT(bma->length > 0); + ASSERT(bma->length <= XFS_MAX_BMBT_EXTLEN); + + if (bma->flags & XFS_BMAPI_CONTIG) + bma->minlen = bma->length; + else + bma->minlen = 1; + + if (!(bma->flags & XFS_BMAPI_METADATA)) { + /* + * For the data and COW fork, the first data in the file is + * treated differently to all other allocations. For the + * attribute fork, we only need to ensure the allocated range + * is not on the busy list. + */ + bma->datatype = XFS_ALLOC_NOBUSY; + if (whichfork == XFS_DATA_FORK || whichfork == XFS_COW_FORK) { + bma->datatype |= XFS_ALLOC_USERDATA; + if (bma->offset == 0) + bma->datatype |= XFS_ALLOC_INITIAL_USER_DATA; + + if (mp->m_dalign && bma->length >= mp->m_dalign) { + error = xfs_bmap_isaeof(bma, whichfork); + if (error) + return error; + } + } + } + + if ((bma->datatype & XFS_ALLOC_USERDATA) && + XFS_IS_REALTIME_INODE(bma->ip)) + error = xfs_bmap_rtalloc(bma); + else + error = xfs_bmap_btalloc(bma); + if (error) + return error; + if (bma->blkno == NULLFSBLOCK) + return -ENOSPC; + + if (WARN_ON_ONCE(!xfs_valid_startblock(bma->ip, bma->blkno))) { + xfs_bmap_mark_sick(bma->ip, whichfork); + return -EFSCORRUPTED; + } + + if (bma->flags & XFS_BMAPI_ZERO) { + error = xfs_zero_extent(bma->ip, bma->blkno, bma->length); + if (error) + return error; + } + + if (ifp->if_format == XFS_DINODE_FMT_BTREE && !bma->cur) + bma->cur = xfs_bmbt_init_cursor(mp, bma->tp, bma->ip, whichfork); + /* + * Bump the number of extents we've allocated + * in this call. + */ + bma->nallocs++; + + if (bma->cur && bma->wasdel) + bma->cur->bc_flags |= XFS_BTREE_BMBT_WASDEL; + + bma->got.br_startoff = bma->offset; + bma->got.br_startblock = bma->blkno; + bma->got.br_blockcount = bma->length; + bma->got.br_state = XFS_EXT_NORM; + + if (bma->flags & XFS_BMAPI_PREALLOC) + bma->got.br_state = XFS_EXT_UNWRITTEN; + + if (bma->wasdel) + error = xfs_bmap_add_extent_delay_real(bma, whichfork); + else + error = xfs_bmap_add_extent_hole_real(bma->tp, bma->ip, + whichfork, &bma->icur, &bma->cur, &bma->got, + &bma->logflags, bma->flags); + if (error) + return error; + + /* + * Update our extent pointer, given that xfs_bmap_add_extent_delay_real + * or xfs_bmap_add_extent_hole_real might have merged it into one of + * the neighbouring ones. + */ + xfs_iext_get_extent(ifp, &bma->icur, &bma->got); + + ASSERT(bma->got.br_startoff <= bma->offset); + ASSERT(bma->got.br_startoff + bma->got.br_blockcount >= + bma->offset + bma->length); + ASSERT(bma->got.br_state == XFS_EXT_NORM || + bma->got.br_state == XFS_EXT_UNWRITTEN); + return 0; +} + +STATIC int +xfs_bmapi_convert_unwritten( + struct xfs_bmalloca *bma, + struct xfs_bmbt_irec *mval, + xfs_filblks_t len, + uint32_t flags) +{ + int whichfork = xfs_bmapi_whichfork(flags); + struct xfs_ifork *ifp = xfs_ifork_ptr(bma->ip, whichfork); + int tmp_logflags = 0; + int error; + + /* check if we need to do unwritten->real conversion */ + if (mval->br_state == XFS_EXT_UNWRITTEN && + (flags & XFS_BMAPI_PREALLOC)) + return 0; + + /* check if we need to do real->unwritten conversion */ + if (mval->br_state == XFS_EXT_NORM && + (flags & (XFS_BMAPI_PREALLOC | XFS_BMAPI_CONVERT)) != + (XFS_BMAPI_PREALLOC | XFS_BMAPI_CONVERT)) + return 0; + + /* + * Modify (by adding) the state flag, if writing. + */ + ASSERT(mval->br_blockcount <= len); + if (ifp->if_format == XFS_DINODE_FMT_BTREE && !bma->cur) { + bma->cur = xfs_bmbt_init_cursor(bma->ip->i_mount, bma->tp, + bma->ip, whichfork); + } + mval->br_state = (mval->br_state == XFS_EXT_UNWRITTEN) + ? XFS_EXT_NORM : XFS_EXT_UNWRITTEN; + + /* + * Before insertion into the bmbt, zero the range being converted + * if required. + */ + if (flags & XFS_BMAPI_ZERO) { + error = xfs_zero_extent(bma->ip, mval->br_startblock, + mval->br_blockcount); + if (error) + return error; + } + + error = xfs_bmap_add_extent_unwritten_real(bma->tp, bma->ip, whichfork, + &bma->icur, &bma->cur, mval, &tmp_logflags); + /* + * Log the inode core unconditionally in the unwritten extent conversion + * path because the conversion might not have done so (e.g., if the + * extent count hasn't changed). We need to make sure the inode is dirty + * in the transaction for the sake of fsync(), even if nothing has + * changed, because fsync() will not force the log for this transaction + * unless it sees the inode pinned. + * + * Note: If we're only converting cow fork extents, there aren't + * any on-disk updates to make, so we don't need to log anything. + */ + if (whichfork != XFS_COW_FORK) + bma->logflags |= tmp_logflags | XFS_ILOG_CORE; + if (error) + return error; + + /* + * Update our extent pointer, given that + * xfs_bmap_add_extent_unwritten_real might have merged it into one + * of the neighbouring ones. + */ + xfs_iext_get_extent(ifp, &bma->icur, &bma->got); + + /* + * We may have combined previously unwritten space with written space, + * so generate another request. + */ + if (mval->br_blockcount < len) + return -EAGAIN; + return 0; +} + +xfs_extlen_t +xfs_bmapi_minleft( + struct xfs_trans *tp, + struct xfs_inode *ip, + int fork) +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, fork); + + if (tp && tp->t_highest_agno != NULLAGNUMBER) + return 0; + if (ifp->if_format != XFS_DINODE_FMT_BTREE) + return 1; + return be16_to_cpu(ifp->if_broot->bb_level) + 1; +} + +/* + * Log whatever the flags say, even if error. Otherwise we might miss detecting + * a case where the data is changed, there's an error, and it's not logged so we + * don't shutdown when we should. Don't bother logging extents/btree changes if + * we converted to the other format. + */ +static void +xfs_bmapi_finish( + struct xfs_bmalloca *bma, + int whichfork, + int error) +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(bma->ip, whichfork); + + if ((bma->logflags & xfs_ilog_fext(whichfork)) && + ifp->if_format != XFS_DINODE_FMT_EXTENTS) + bma->logflags &= ~xfs_ilog_fext(whichfork); + else if ((bma->logflags & xfs_ilog_fbroot(whichfork)) && + ifp->if_format != XFS_DINODE_FMT_BTREE) + bma->logflags &= ~xfs_ilog_fbroot(whichfork); + + if (bma->logflags) + xfs_trans_log_inode(bma->tp, bma->ip, bma->logflags); + if (bma->cur) + xfs_btree_del_cursor(bma->cur, error); +} + +/* + * Map file blocks to filesystem blocks, and allocate blocks or convert the + * extent state if necessary. Details behaviour is controlled by the flags + * parameter. Only allocates blocks from a single allocation group, to avoid + * locking problems. + * + * Returns 0 on success and places the extent mappings in mval. nmaps is used + * as an input/output parameter where the caller specifies the maximum number + * of mappings that may be returned and xfs_bmapi_write passes back the number + * of mappings (including existing mappings) it found. + * + * Returns a negative error code on failure, including -ENOSPC when it could not + * allocate any blocks and -ENOSR when it did allocate blocks to convert a + * delalloc range, but those blocks were before the passed in range. + */ +int +xfs_bmapi_write( + struct xfs_trans *tp, /* transaction pointer */ + struct xfs_inode *ip, /* incore inode */ + xfs_fileoff_t bno, /* starting file offs. mapped */ + xfs_filblks_t len, /* length to map in file */ + uint32_t flags, /* XFS_BMAPI_... */ + xfs_extlen_t total, /* total blocks needed */ + struct xfs_bmbt_irec *mval, /* output: map values */ + int *nmap) /* i/o: mval size/count */ +{ + struct xfs_bmalloca bma = { + .tp = tp, + .ip = ip, + .total = total, + }; + struct xfs_mount *mp = ip->i_mount; + int whichfork = xfs_bmapi_whichfork(flags); + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + xfs_fileoff_t end; /* end of mapped file region */ + bool eof = false; /* after the end of extents */ + int error; /* error return */ + int n; /* current extent index */ + xfs_fileoff_t obno; /* old block number (offset) */ + +#ifdef DEBUG + xfs_fileoff_t orig_bno; /* original block number value */ + int orig_flags; /* original flags arg value */ + xfs_filblks_t orig_len; /* original value of len arg */ + struct xfs_bmbt_irec *orig_mval; /* original value of mval */ + int orig_nmap; /* original value of *nmap */ + + orig_bno = bno; + orig_len = len; + orig_flags = flags; + orig_mval = mval; + orig_nmap = *nmap; +#endif + + ASSERT(*nmap >= 1); + ASSERT(*nmap <= XFS_BMAP_MAX_NMAP); + ASSERT(tp != NULL); + ASSERT(len > 0); + ASSERT(ifp->if_format != XFS_DINODE_FMT_LOCAL); + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + ASSERT(!(flags & XFS_BMAPI_REMAP)); + + /* zeroing is for currently only for data extents, not metadata */ + ASSERT((flags & (XFS_BMAPI_METADATA | XFS_BMAPI_ZERO)) != + (XFS_BMAPI_METADATA | XFS_BMAPI_ZERO)); + /* + * we can allocate unwritten extents or pre-zero allocated blocks, + * but it makes no sense to do both at once. This would result in + * zeroing the unwritten extent twice, but it still being an + * unwritten extent.... + */ + ASSERT((flags & (XFS_BMAPI_PREALLOC | XFS_BMAPI_ZERO)) != + (XFS_BMAPI_PREALLOC | XFS_BMAPI_ZERO)); + + if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ifp)) || + XFS_TEST_ERROR(mp, XFS_ERRTAG_BMAPIFORMAT)) { + xfs_bmap_mark_sick(ip, whichfork); + return -EFSCORRUPTED; + } + + if (xfs_is_shutdown(mp)) + return -EIO; + + XFS_STATS_INC(mp, xs_blk_mapw); + + error = xfs_iread_extents(tp, ip, whichfork); + if (error) + goto error0; + + if (!xfs_iext_lookup_extent(ip, ifp, bno, &bma.icur, &bma.got)) + eof = true; + if (!xfs_iext_peek_prev_extent(ifp, &bma.icur, &bma.prev)) + bma.prev.br_startoff = NULLFILEOFF; + bma.minleft = xfs_bmapi_minleft(tp, ip, whichfork); + + n = 0; + end = bno + len; + obno = bno; + while (bno < end && n < *nmap) { + bool need_alloc = false, wasdelay = false; + + /* in hole or beyond EOF? */ + if (eof || bma.got.br_startoff > bno) { + /* + * CoW fork conversions should /never/ hit EOF or + * holes. There should always be something for us + * to work on. + */ + ASSERT(!((flags & XFS_BMAPI_CONVERT) && + (flags & XFS_BMAPI_COWFORK))); + + need_alloc = true; + } else if (isnullstartblock(bma.got.br_startblock)) { + wasdelay = true; + } + + /* + * First, deal with the hole before the allocated space + * that we found, if any. + */ + if (need_alloc || wasdelay) { + bma.eof = eof; + bma.conv = !!(flags & XFS_BMAPI_CONVERT); + bma.wasdel = wasdelay; + bma.offset = bno; + bma.flags = flags; + + /* + * There's a 32/64 bit type mismatch between the + * allocation length request (which can be 64 bits in + * length) and the bma length request, which is + * xfs_extlen_t and therefore 32 bits. Hence we have to + * be careful and do the min() using the larger type to + * avoid overflows. + */ + bma.length = XFS_FILBLKS_MIN(len, XFS_MAX_BMBT_EXTLEN); + + if (wasdelay) { + bma.length = XFS_FILBLKS_MIN(bma.length, + bma.got.br_blockcount - + (bno - bma.got.br_startoff)); + } else { + if (!eof) + bma.length = XFS_FILBLKS_MIN(bma.length, + bma.got.br_startoff - bno); + } + + ASSERT(bma.length > 0); + error = xfs_bmapi_allocate(&bma); + if (error) { + /* + * If we already allocated space in a previous + * iteration return what we go so far when + * running out of space. + */ + if (error == -ENOSPC && bma.nallocs) + break; + goto error0; + } + + /* + * If this is a CoW allocation, record the data in + * the refcount btree for orphan recovery. + */ + if (whichfork == XFS_COW_FORK) + xfs_refcount_alloc_cow_extent(tp, + XFS_IS_REALTIME_INODE(ip), + bma.blkno, bma.length); + } + + /* Deal with the allocated space we found. */ + xfs_bmapi_trim_map(mval, &bma.got, &bno, len, obno, + end, n, flags); + + /* Execute unwritten extent conversion if necessary */ + error = xfs_bmapi_convert_unwritten(&bma, mval, len, flags); + if (error == -EAGAIN) + continue; + if (error) + goto error0; + + /* update the extent map to return */ + xfs_bmapi_update_map(&mval, &bno, &len, obno, end, &n, flags); + + /* + * If we're done, stop now. Stop when we've allocated + * XFS_BMAP_MAX_NMAP extents no matter what. Otherwise + * the transaction may get too big. + */ + if (bno >= end || n >= *nmap || bma.nallocs >= *nmap) + break; + + /* Else go on to the next record. */ + bma.prev = bma.got; + if (!xfs_iext_next_extent(ifp, &bma.icur, &bma.got)) + eof = true; + } + + error = xfs_bmap_btree_to_extents(tp, ip, bma.cur, &bma.logflags, + whichfork); + if (error) + goto error0; + + ASSERT(ifp->if_format != XFS_DINODE_FMT_BTREE || + ifp->if_nextents > XFS_IFORK_MAXEXT(ip, whichfork)); + xfs_bmapi_finish(&bma, whichfork, 0); + xfs_bmap_validate_ret(orig_bno, orig_len, orig_flags, orig_mval, + orig_nmap, n); + + /* + * When converting delayed allocations, xfs_bmapi_allocate ignores + * the passed in bno and always converts from the start of the found + * delalloc extent. + * + * To avoid a successful return with *nmap set to 0, return the magic + * -ENOSR error code for this particular case so that the caller can + * handle it. + */ + if (!n) { + ASSERT(bma.nallocs >= *nmap); + return -ENOSR; + } + *nmap = n; + return 0; +error0: + xfs_bmapi_finish(&bma, whichfork, error); + return error; +} + +/* + * Convert an existing delalloc extent to real blocks based on file offset. This + * attempts to allocate the entire delalloc extent and may require multiple + * invocations to allocate the target offset if a large enough physical extent + * is not available. + */ +static int +xfs_bmapi_convert_one_delalloc( + struct xfs_inode *ip, + int whichfork, + xfs_off_t offset, + struct iomap *iomap, + unsigned int *seq) +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + struct xfs_mount *mp = ip->i_mount; + xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); + struct xfs_bmalloca bma = { NULL }; + uint16_t flags = 0; + struct xfs_trans *tp; + int error; + + if (whichfork == XFS_COW_FORK) + flags |= IOMAP_F_SHARED; + + /* + * Space for the extent and indirect blocks was reserved when the + * delalloc extent was created so there's no need to do so here. + */ + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, 0, 0, + XFS_TRANS_RESERVE, &tp); + if (error) + return error; + + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, 0); + + error = xfs_iext_count_extend(tp, ip, whichfork, + XFS_IEXT_ADD_NOSPLIT_CNT); + if (error) + goto out_trans_cancel; + + if (!xfs_iext_lookup_extent(ip, ifp, offset_fsb, &bma.icur, &bma.got) || + bma.got.br_startoff > offset_fsb) { + /* + * No extent found in the range we are trying to convert. This + * should only happen for the COW fork, where another thread + * might have moved the extent to the data fork in the meantime. + */ + WARN_ON_ONCE(whichfork != XFS_COW_FORK); + error = -EAGAIN; + goto out_trans_cancel; + } + + /* + * If we find a real extent here we raced with another thread converting + * the extent. Just return the real extent at this offset. + */ + if (!isnullstartblock(bma.got.br_startblock)) { + xfs_bmbt_to_iomap(ip, iomap, &bma.got, 0, flags, + xfs_iomap_inode_sequence(ip, flags)); + if (seq) + *seq = READ_ONCE(ifp->if_seq); + goto out_trans_cancel; + } + + bma.tp = tp; + bma.ip = ip; + bma.wasdel = true; + bma.minleft = xfs_bmapi_minleft(tp, ip, whichfork); + + /* + * Always allocate convert from the start of the delalloc extent even if + * that is outside the passed in range to create large contiguous + * extents on disk. + */ + bma.offset = bma.got.br_startoff; + bma.length = bma.got.br_blockcount; + + /* + * When we're converting the delalloc reservations backing dirty pages + * in the page cache, we must be careful about how we create the new + * extents: + * + * New CoW fork extents are created unwritten, turned into real extents + * when we're about to write the data to disk, and mapped into the data + * fork after the write finishes. End of story. + * + * New data fork extents must be mapped in as unwritten and converted + * to real extents after the write succeeds to avoid exposing stale + * disk contents if we crash. + */ + bma.flags = XFS_BMAPI_PREALLOC; + if (whichfork == XFS_COW_FORK) + bma.flags |= XFS_BMAPI_COWFORK; + + if (!xfs_iext_peek_prev_extent(ifp, &bma.icur, &bma.prev)) + bma.prev.br_startoff = NULLFILEOFF; + + error = xfs_bmapi_allocate(&bma); + if (error) + goto out_finish; + + XFS_STATS_ADD(mp, xs_xstrat_bytes, XFS_FSB_TO_B(mp, bma.length)); + XFS_STATS_INC(mp, xs_xstrat_quick); + + ASSERT(!isnullstartblock(bma.got.br_startblock)); + xfs_bmbt_to_iomap(ip, iomap, &bma.got, 0, flags, + xfs_iomap_inode_sequence(ip, flags)); + if (seq) + *seq = READ_ONCE(ifp->if_seq); + + if (whichfork == XFS_COW_FORK) + xfs_refcount_alloc_cow_extent(tp, XFS_IS_REALTIME_INODE(ip), + bma.blkno, bma.length); + + error = xfs_bmap_btree_to_extents(tp, ip, bma.cur, &bma.logflags, + whichfork); + if (error) + goto out_finish; + + xfs_bmapi_finish(&bma, whichfork, 0); + error = xfs_trans_commit(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; + +out_finish: + xfs_bmapi_finish(&bma, whichfork, error); +out_trans_cancel: + xfs_trans_cancel(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; +} + +/* + * Pass in a dellalloc extent and convert it to real extents, return the real + * extent that maps offset_fsb in iomap. + */ +int +xfs_bmapi_convert_delalloc( + struct xfs_inode *ip, + int whichfork, + loff_t offset, + struct iomap *iomap, + unsigned int *seq) +{ + int error; + + /* + * Attempt to allocate whatever delalloc extent currently backs offset + * and put the result into iomap. Allocate in a loop because it may + * take several attempts to allocate real blocks for a contiguous + * delalloc extent if free space is sufficiently fragmented. + */ + do { + error = xfs_bmapi_convert_one_delalloc(ip, whichfork, offset, + iomap, seq); + if (error) + return error; + } while (iomap->offset + iomap->length <= offset); + + return 0; +} + +int +xfs_bmapi_remap( + struct xfs_trans *tp, + struct xfs_inode *ip, + xfs_fileoff_t bno, + xfs_filblks_t len, + xfs_fsblock_t startblock, + uint32_t flags) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_ifork *ifp; + struct xfs_btree_cur *cur = NULL; + struct xfs_bmbt_irec got; + struct xfs_iext_cursor icur; + int whichfork = xfs_bmapi_whichfork(flags); + int logflags = 0, error; + + ifp = xfs_ifork_ptr(ip, whichfork); + ASSERT(len > 0); + ASSERT(len <= (xfs_filblks_t)XFS_MAX_BMBT_EXTLEN); + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + ASSERT(!(flags & ~(XFS_BMAPI_ATTRFORK | XFS_BMAPI_PREALLOC | + XFS_BMAPI_NORMAP))); + ASSERT((flags & (XFS_BMAPI_ATTRFORK | XFS_BMAPI_PREALLOC)) != + (XFS_BMAPI_ATTRFORK | XFS_BMAPI_PREALLOC)); + + if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ifp)) || + XFS_TEST_ERROR(mp, XFS_ERRTAG_BMAPIFORMAT)) { + xfs_bmap_mark_sick(ip, whichfork); + return -EFSCORRUPTED; + } + + if (xfs_is_shutdown(mp)) + return -EIO; + + error = xfs_iread_extents(tp, ip, whichfork); + if (error) + return error; + + if (xfs_iext_lookup_extent(ip, ifp, bno, &icur, &got)) { + /* make sure we only reflink into a hole. */ + ASSERT(got.br_startoff > bno); + ASSERT(got.br_startoff - bno >= len); + } + + ip->i_nblocks += len; + ip->i_delayed_blks -= len; /* see xfs_bmap_defer_add */ + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + + if (ifp->if_format == XFS_DINODE_FMT_BTREE) + cur = xfs_bmbt_init_cursor(mp, tp, ip, whichfork); + + got.br_startoff = bno; + got.br_startblock = startblock; + got.br_blockcount = len; + if (flags & XFS_BMAPI_PREALLOC) + got.br_state = XFS_EXT_UNWRITTEN; + else + got.br_state = XFS_EXT_NORM; + + error = xfs_bmap_add_extent_hole_real(tp, ip, whichfork, &icur, + &cur, &got, &logflags, flags); + if (error) + goto error0; + + error = xfs_bmap_btree_to_extents(tp, ip, cur, &logflags, whichfork); + +error0: + if (ip->i_df.if_format != XFS_DINODE_FMT_EXTENTS) + logflags &= ~XFS_ILOG_DEXT; + else if (ip->i_df.if_format != XFS_DINODE_FMT_BTREE) + logflags &= ~XFS_ILOG_DBROOT; + + if (logflags) + xfs_trans_log_inode(tp, ip, logflags); + if (cur) + xfs_btree_del_cursor(cur, error); + return error; +} + +/* + * When a delalloc extent is split (e.g., due to a hole punch), the original + * indlen reservation must be shared across the two new extents that are left + * behind. + * + * Given the original reservation and the worst case indlen for the two new + * extents (as calculated by xfs_bmap_worst_indlen()), split the original + * reservation fairly across the two new extents. If necessary, steal available + * blocks from a deleted extent to make up a reservation deficiency (e.g., if + * ores == 1). The number of stolen blocks is returned. The availability and + * subsequent accounting of stolen blocks is the responsibility of the caller. + */ +static void +xfs_bmap_split_indlen( + xfs_filblks_t ores, /* original res. */ + xfs_filblks_t *indlen1, /* ext1 worst indlen */ + xfs_filblks_t *indlen2) /* ext2 worst indlen */ +{ + xfs_filblks_t len1 = *indlen1; + xfs_filblks_t len2 = *indlen2; + xfs_filblks_t nres = len1 + len2; /* new total res. */ + xfs_filblks_t resfactor; + + /* + * We can't meet the total required reservation for the two extents. + * Calculate the percent of the overall shortage between both extents + * and apply this percentage to each of the requested indlen values. + * This distributes the shortage fairly and reduces the chances that one + * of the two extents is left with nothing when extents are repeatedly + * split. + */ + resfactor = (ores * 100); + do_div(resfactor, nres); + len1 *= resfactor; + do_div(len1, 100); + len2 *= resfactor; + do_div(len2, 100); + ASSERT(len1 + len2 <= ores); + ASSERT(len1 < *indlen1 && len2 < *indlen2); + + /* + * Hand out the remainder to each extent. If one of the two reservations + * is zero, we want to make sure that one gets a block first. The loop + * below starts with len1, so hand len2 a block right off the bat if it + * is zero. + */ + ores -= (len1 + len2); + ASSERT((*indlen1 - len1) + (*indlen2 - len2) >= ores); + if (ores && !len2 && *indlen2) { + len2++; + ores--; + } + while (ores) { + if (len1 < *indlen1) { + len1++; + ores--; + } + if (!ores) + break; + if (len2 < *indlen2) { + len2++; + ores--; + } + } + + *indlen1 = len1; + *indlen2 = len2; +} + +void +xfs_bmap_del_extent_delay( + struct xfs_inode *ip, + int whichfork, + struct xfs_iext_cursor *icur, + struct xfs_bmbt_irec *got, + struct xfs_bmbt_irec *del, + uint32_t bflags) /* bmapi flags */ +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + struct xfs_bmbt_irec new; + int64_t da_old, da_new, da_diff = 0; + xfs_fileoff_t del_endoff, got_endoff; + xfs_filblks_t got_indlen, new_indlen, stolen = 0; + uint32_t state = xfs_bmap_fork_to_state(whichfork); + uint64_t fdblocks; + bool isrt; + + XFS_STATS_INC(mp, xs_del_exlist); + + isrt = xfs_ifork_is_realtime(ip, whichfork); + del_endoff = del->br_startoff + del->br_blockcount; + got_endoff = got->br_startoff + got->br_blockcount; + da_old = startblockval(got->br_startblock); + da_new = 0; + + ASSERT(del->br_blockcount > 0); + ASSERT(got->br_startoff <= del->br_startoff); + ASSERT(got_endoff >= del_endoff); + + /* + * Update the inode delalloc counter now and wait to update the + * sb counters as we might have to borrow some blocks for the + * indirect block accounting. + */ + xfs_quota_unreserve_blkres(ip, del->br_blockcount); + ip->i_delayed_blks -= del->br_blockcount; + + if (got->br_startoff == del->br_startoff) + state |= BMAP_LEFT_FILLING; + if (got_endoff == del_endoff) + state |= BMAP_RIGHT_FILLING; + + switch (state & (BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING)) { + case BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING: + /* + * Matches the whole extent. Delete the entry. + */ + xfs_iext_remove(ip, icur, state); + xfs_iext_prev(ifp, icur); + break; + case BMAP_LEFT_FILLING: + /* + * Deleting the first part of the extent. + */ + got->br_startoff = del_endoff; + got->br_blockcount -= del->br_blockcount; + da_new = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(ip, + got->br_blockcount), da_old); + got->br_startblock = nullstartblock((int)da_new); + xfs_iext_update_extent(ip, state, icur, got); + break; + case BMAP_RIGHT_FILLING: + /* + * Deleting the last part of the extent. + */ + got->br_blockcount = got->br_blockcount - del->br_blockcount; + da_new = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(ip, + got->br_blockcount), da_old); + got->br_startblock = nullstartblock((int)da_new); + xfs_iext_update_extent(ip, state, icur, got); + break; + case 0: + /* + * Deleting the middle of the extent. + * + * Distribute the original indlen reservation across the two new + * extents. Steal blocks from the deleted extent if necessary. + * Stealing blocks simply fudges the fdblocks accounting below. + * Warn if either of the new indlen reservations is zero as this + * can lead to delalloc problems. + */ + got->br_blockcount = del->br_startoff - got->br_startoff; + got_indlen = xfs_bmap_worst_indlen(ip, got->br_blockcount); + + new.br_blockcount = got_endoff - del_endoff; + new_indlen = xfs_bmap_worst_indlen(ip, new.br_blockcount); + + WARN_ON_ONCE(!got_indlen || !new_indlen); + /* + * Steal as many blocks as we can to try and satisfy the worst + * case indlen for both new extents. + * + * However, we can't just steal reservations from the data + * blocks if this is an RT inodes as the data and metadata + * blocks come from different pools. We'll have to live with + * under-filled indirect reservation in this case. + */ + da_new = got_indlen + new_indlen; + if (da_new > da_old && !isrt) { + stolen = XFS_FILBLKS_MIN(da_new - da_old, + del->br_blockcount); + da_old += stolen; + } + if (da_new > da_old) + xfs_bmap_split_indlen(da_old, &got_indlen, &new_indlen); + da_new = got_indlen + new_indlen; + + got->br_startblock = nullstartblock((int)got_indlen); + + new.br_startoff = del_endoff; + new.br_state = got->br_state; + new.br_startblock = nullstartblock((int)new_indlen); + + xfs_iext_update_extent(ip, state, icur, got); + xfs_iext_next(ifp, icur); + xfs_iext_insert(ip, icur, &new, state); + + del->br_blockcount -= stolen; + break; + } + + ASSERT(da_old >= da_new); + da_diff = da_old - da_new; + fdblocks = da_diff; + + if (bflags & XFS_BMAPI_REMAP) { + ; + } else if (isrt) { + xfs_rtbxlen_t rtxlen; + + rtxlen = xfs_blen_to_rtbxlen(mp, del->br_blockcount); + if (xfs_is_zoned_inode(ip)) + xfs_zoned_add_available(mp, rtxlen); + xfs_add_frextents(mp, rtxlen); + } else { + fdblocks += del->br_blockcount; + } + + xfs_add_fdblocks(mp, fdblocks); + xfs_mod_delalloc(ip, -(int64_t)del->br_blockcount, -da_diff); +} + +void +xfs_bmap_del_extent_cow( + struct xfs_inode *ip, + struct xfs_iext_cursor *icur, + struct xfs_bmbt_irec *got, + struct xfs_bmbt_irec *del) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_COW_FORK); + struct xfs_bmbt_irec new; + xfs_fileoff_t del_endoff, got_endoff; + uint32_t state = BMAP_COWFORK; + + XFS_STATS_INC(mp, xs_del_exlist); + + del_endoff = del->br_startoff + del->br_blockcount; + got_endoff = got->br_startoff + got->br_blockcount; + + ASSERT(del->br_blockcount > 0); + ASSERT(got->br_startoff <= del->br_startoff); + ASSERT(got_endoff >= del_endoff); + ASSERT(!isnullstartblock(got->br_startblock)); + + if (got->br_startoff == del->br_startoff) + state |= BMAP_LEFT_FILLING; + if (got_endoff == del_endoff) + state |= BMAP_RIGHT_FILLING; + + switch (state & (BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING)) { + case BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING: + /* + * Matches the whole extent. Delete the entry. + */ + xfs_iext_remove(ip, icur, state); + xfs_iext_prev(ifp, icur); + break; + case BMAP_LEFT_FILLING: + /* + * Deleting the first part of the extent. + */ + got->br_startoff = del_endoff; + got->br_blockcount -= del->br_blockcount; + got->br_startblock = del->br_startblock + del->br_blockcount; + xfs_iext_update_extent(ip, state, icur, got); + break; + case BMAP_RIGHT_FILLING: + /* + * Deleting the last part of the extent. + */ + got->br_blockcount -= del->br_blockcount; + xfs_iext_update_extent(ip, state, icur, got); + break; + case 0: + /* + * Deleting the middle of the extent. + */ + got->br_blockcount = del->br_startoff - got->br_startoff; + + new.br_startoff = del_endoff; + new.br_blockcount = got_endoff - del_endoff; + new.br_state = got->br_state; + new.br_startblock = del->br_startblock + del->br_blockcount; + + xfs_iext_update_extent(ip, state, icur, got); + xfs_iext_next(ifp, icur); + xfs_iext_insert(ip, icur, &new, state); + break; + } + ip->i_delayed_blks -= del->br_blockcount; +} + +static int +xfs_bmap_free_rtblocks( + struct xfs_trans *tp, + struct xfs_bmbt_irec *del) +{ + struct xfs_rtgroup *rtg; + int error; + + rtg = xfs_rtgroup_grab(tp->t_mountp, 0); + if (!rtg) + return -EIO; + + /* + * Ensure the bitmap and summary inodes are locked and joined to the + * transaction before modifying them. + */ + if (!(tp->t_flags & XFS_TRANS_RTBITMAP_LOCKED)) { + tp->t_flags |= XFS_TRANS_RTBITMAP_LOCKED; + xfs_rtgroup_lock(rtg, XFS_RTGLOCK_BITMAP); + xfs_rtgroup_trans_join(tp, rtg, XFS_RTGLOCK_BITMAP); + } + + error = xfs_rtfree_blocks(tp, rtg, del->br_startblock, + del->br_blockcount); + xfs_rtgroup_rele(rtg); + return error; +} + +/* + * Called by xfs_bmapi to update file extent records and the btree + * after removing space. + */ +STATIC int /* error */ +xfs_bmap_del_extent_real( + xfs_inode_t *ip, /* incore inode pointer */ + xfs_trans_t *tp, /* current transaction pointer */ + struct xfs_iext_cursor *icur, + struct xfs_btree_cur *cur, /* if null, not a btree */ + xfs_bmbt_irec_t *del, /* data to remove from extents */ + int *logflagsp, /* inode logging flags */ + int whichfork, /* data or attr fork */ + uint32_t bflags) /* bmapi flags */ +{ + xfs_fsblock_t del_endblock=0; /* first block past del */ + xfs_fileoff_t del_endoff; /* first offset past del */ + int error = 0; /* error return value */ + struct xfs_bmbt_irec got; /* current extent entry */ + xfs_fileoff_t got_endoff; /* first offset past got */ + int i; /* temp state */ + struct xfs_ifork *ifp; /* inode fork pointer */ + xfs_mount_t *mp; /* mount structure */ + xfs_filblks_t nblks; /* quota/sb block count */ + xfs_bmbt_irec_t new; /* new record to be inserted */ + /* REFERENCED */ + uint qfield; /* quota field to update */ + uint32_t state = xfs_bmap_fork_to_state(whichfork); + struct xfs_bmbt_irec old; + + *logflagsp = 0; + + mp = ip->i_mount; + XFS_STATS_INC(mp, xs_del_exlist); + + ifp = xfs_ifork_ptr(ip, whichfork); + ASSERT(del->br_blockcount > 0); + xfs_iext_get_extent(ifp, icur, &got); + ASSERT(got.br_startoff <= del->br_startoff); + del_endoff = del->br_startoff + del->br_blockcount; + got_endoff = got.br_startoff + got.br_blockcount; + ASSERT(got_endoff >= del_endoff); + ASSERT(!isnullstartblock(got.br_startblock)); + qfield = 0; + + /* + * If it's the case where the directory code is running with no block + * reservation, and the deleted block is in the middle of its extent, + * and the resulting insert of an extent would cause transformation to + * btree format, then reject it. The calling code will then swap blocks + * around instead. We have to do this now, rather than waiting for the + * conversion to btree format, since the transaction will be dirty then. + */ + if (tp->t_blk_res == 0 && + ifp->if_format == XFS_DINODE_FMT_EXTENTS && + ifp->if_nextents >= XFS_IFORK_MAXEXT(ip, whichfork) && + del->br_startoff > got.br_startoff && del_endoff < got_endoff) + return -ENOSPC; + + *logflagsp = XFS_ILOG_CORE; + if (xfs_ifork_is_realtime(ip, whichfork)) + qfield = XFS_TRANS_DQ_RTBCOUNT; + else + qfield = XFS_TRANS_DQ_BCOUNT; + nblks = del->br_blockcount; + + del_endblock = del->br_startblock + del->br_blockcount; + if (cur) { + error = xfs_bmbt_lookup_eq(cur, &got, &i); + if (error) + return error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + } + + if (got.br_startoff == del->br_startoff) + state |= BMAP_LEFT_FILLING; + if (got_endoff == del_endoff) + state |= BMAP_RIGHT_FILLING; + + switch (state & (BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING)) { + case BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING: + /* + * Matches the whole extent. Delete the entry. + */ + xfs_iext_remove(ip, icur, state); + xfs_iext_prev(ifp, icur); + ifp->if_nextents--; + + *logflagsp |= XFS_ILOG_CORE; + if (!cur) { + *logflagsp |= xfs_ilog_fext(whichfork); + break; + } + if ((error = xfs_btree_delete(cur, &i))) + return error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + break; + case BMAP_LEFT_FILLING: + /* + * Deleting the first part of the extent. + */ + got.br_startoff = del_endoff; + got.br_startblock = del_endblock; + got.br_blockcount -= del->br_blockcount; + xfs_iext_update_extent(ip, state, icur, &got); + if (!cur) { + *logflagsp |= xfs_ilog_fext(whichfork); + break; + } + error = xfs_bmbt_update(cur, &got); + if (error) + return error; + break; + case BMAP_RIGHT_FILLING: + /* + * Deleting the last part of the extent. + */ + got.br_blockcount -= del->br_blockcount; + xfs_iext_update_extent(ip, state, icur, &got); + if (!cur) { + *logflagsp |= xfs_ilog_fext(whichfork); + break; + } + error = xfs_bmbt_update(cur, &got); + if (error) + return error; + break; + case 0: + /* + * Deleting the middle of the extent. + */ + + old = got; + + got.br_blockcount = del->br_startoff - got.br_startoff; + xfs_iext_update_extent(ip, state, icur, &got); + + new.br_startoff = del_endoff; + new.br_blockcount = got_endoff - del_endoff; + new.br_state = got.br_state; + new.br_startblock = del_endblock; + + *logflagsp |= XFS_ILOG_CORE; + if (cur) { + error = xfs_bmbt_update(cur, &got); + if (error) + return error; + error = xfs_btree_increment(cur, 0, &i); + if (error) + return error; + cur->bc_rec.b = new; + error = xfs_btree_insert(cur, &i); + if (error && error != -ENOSPC) + return error; + /* + * If get no-space back from btree insert, it tried a + * split, and we have a zero block reservation. Fix up + * our state and return the error. + */ + if (error == -ENOSPC) { + /* + * Reset the cursor, don't trust it after any + * insert operation. + */ + error = xfs_bmbt_lookup_eq(cur, &got, &i); + if (error) + return error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + /* + * Update the btree record back + * to the original value. + */ + error = xfs_bmbt_update(cur, &old); + if (error) + return error; + /* + * Reset the extent record back + * to the original value. + */ + xfs_iext_update_extent(ip, state, icur, &old); + *logflagsp = 0; + return -ENOSPC; + } + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + } else + *logflagsp |= xfs_ilog_fext(whichfork); + + ifp->if_nextents++; + xfs_iext_next(ifp, icur); + xfs_iext_insert(ip, icur, &new, state); + break; + } + + /* remove reverse mapping */ + xfs_rmap_unmap_extent(tp, ip, whichfork, del); + + /* + * If we need to, add to list of extents to delete. + */ + if (!(bflags & XFS_BMAPI_REMAP)) { + bool isrt = xfs_ifork_is_realtime(ip, whichfork); + + if (xfs_is_reflink_inode(ip) && whichfork == XFS_DATA_FORK) { + xfs_refcount_decrease_extent(tp, isrt, del); + } else if (isrt && !xfs_has_rtgroups(mp)) { + error = xfs_bmap_free_rtblocks(tp, del); + } else { + unsigned int efi_flags = 0; + + if ((bflags & XFS_BMAPI_NODISCARD) || + del->br_state == XFS_EXT_UNWRITTEN) + efi_flags |= XFS_FREE_EXTENT_SKIP_DISCARD; + + /* + * Historically, we did not use EFIs to free realtime + * extents. However, when reverse mapping is enabled, + * we must maintain the same order of operations as the + * data device, which is: Remove the file mapping, + * remove the reverse mapping, and then free the + * blocks. Reflink for realtime volumes requires the + * same sort of ordering. Both features rely on + * rtgroups, so let's gate rt EFI usage on rtgroups. + */ + if (isrt) + efi_flags |= XFS_FREE_EXTENT_REALTIME; + + error = xfs_free_extent_later(tp, del->br_startblock, + del->br_blockcount, NULL, + XFS_AG_RESV_NONE, efi_flags); + } + if (error) + return error; + } + + /* + * Adjust inode # blocks in the file. + */ + if (nblks) + ip->i_nblocks -= nblks; + /* + * Adjust quota data. + */ + if (qfield && !(bflags & XFS_BMAPI_REMAP)) + xfs_trans_mod_dquot_byino(tp, ip, qfield, (long)-nblks); + + return 0; +} + +/* + * Unmap (remove) blocks from a file. + * If nexts is nonzero then the number of extents to remove is limited to + * that value. If not all extents in the block range can be removed then + * *done is set. + */ +static int +__xfs_bunmapi( + struct xfs_trans *tp, /* transaction pointer */ + struct xfs_inode *ip, /* incore inode */ + xfs_fileoff_t start, /* first file offset deleted */ + xfs_filblks_t *rlen, /* i/o: amount remaining */ + uint32_t flags, /* misc flags */ + xfs_extnum_t nexts) /* number of extents max */ +{ + struct xfs_btree_cur *cur; /* bmap btree cursor */ + struct xfs_bmbt_irec del; /* extent being deleted */ + int error; /* error return value */ + xfs_extnum_t extno; /* extent number in list */ + struct xfs_bmbt_irec got; /* current extent record */ + struct xfs_ifork *ifp; /* inode fork pointer */ + int isrt; /* freeing in rt area */ + int logflags; /* transaction logging flags */ + xfs_extlen_t mod; /* rt extent offset */ + struct xfs_mount *mp = ip->i_mount; + int tmp_logflags; /* partial logging flags */ + int wasdel; /* was a delayed alloc extent */ + int whichfork; /* data or attribute fork */ + xfs_filblks_t len = *rlen; /* length to unmap in file */ + xfs_fileoff_t end; + struct xfs_iext_cursor icur; + bool done = false; + + trace_xfs_bunmap(ip, start, len, flags, _RET_IP_); + + whichfork = xfs_bmapi_whichfork(flags); + ASSERT(whichfork != XFS_COW_FORK); + ifp = xfs_ifork_ptr(ip, whichfork); + if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ifp))) { + xfs_bmap_mark_sick(ip, whichfork); + return -EFSCORRUPTED; + } + if (xfs_is_shutdown(mp)) + return -EIO; + + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + ASSERT(len > 0); + ASSERT(nexts >= 0); + + error = xfs_iread_extents(tp, ip, whichfork); + if (error) + return error; + + if (xfs_iext_count(ifp) == 0) { + *rlen = 0; + return 0; + } + XFS_STATS_INC(mp, xs_blk_unmap); + isrt = xfs_ifork_is_realtime(ip, whichfork); + end = start + len; + + if (!xfs_iext_lookup_extent_before(ip, ifp, &end, &icur, &got)) { + *rlen = 0; + return 0; + } + end--; + + logflags = 0; + if (ifp->if_format == XFS_DINODE_FMT_BTREE) { + ASSERT(ifp->if_format == XFS_DINODE_FMT_BTREE); + cur = xfs_bmbt_init_cursor(mp, tp, ip, whichfork); + } else + cur = NULL; + + extno = 0; + while (end != (xfs_fileoff_t)-1 && end >= start && + (nexts == 0 || extno < nexts)) { + /* + * Is the found extent after a hole in which end lives? + * Just back up to the previous extent, if so. + */ + if (got.br_startoff > end && + !xfs_iext_prev_extent(ifp, &icur, &got)) { + done = true; + break; + } + /* + * Is the last block of this extent before the range + * we're supposed to delete? If so, we're done. + */ + end = XFS_FILEOFF_MIN(end, + got.br_startoff + got.br_blockcount - 1); + if (end < start) + break; + /* + * Then deal with the (possibly delayed) allocated space + * we found. + */ + del = got; + wasdel = isnullstartblock(del.br_startblock); + + if (got.br_startoff < start) { + del.br_startoff = start; + del.br_blockcount -= start - got.br_startoff; + if (!wasdel) + del.br_startblock += start - got.br_startoff; + } + if (del.br_startoff + del.br_blockcount > end + 1) + del.br_blockcount = end + 1 - del.br_startoff; + + if (!isrt || (flags & XFS_BMAPI_REMAP)) + goto delete; + + mod = xfs_rtb_to_rtxoff(mp, + del.br_startblock + del.br_blockcount); + if (mod) { + /* + * Realtime extent not lined up at the end. + * The extent could have been split into written + * and unwritten pieces, or we could just be + * unmapping part of it. But we can't really + * get rid of part of a realtime extent. + */ + if (del.br_state == XFS_EXT_UNWRITTEN) { + /* + * This piece is unwritten, or we're not + * using unwritten extents. Skip over it. + */ + ASSERT((flags & XFS_BMAPI_REMAP) || end >= mod); + end -= mod > del.br_blockcount ? + del.br_blockcount : mod; + if (end < got.br_startoff && + !xfs_iext_prev_extent(ifp, &icur, &got)) { + done = true; + break; + } + continue; + } + /* + * It's written, turn it unwritten. + * This is better than zeroing it. + */ + ASSERT(del.br_state == XFS_EXT_NORM); + ASSERT(tp->t_blk_res > 0); + /* + * If this spans a realtime extent boundary, + * chop it back to the start of the one we end at. + */ + if (del.br_blockcount > mod) { + del.br_startoff += del.br_blockcount - mod; + del.br_startblock += del.br_blockcount - mod; + del.br_blockcount = mod; + } + del.br_state = XFS_EXT_UNWRITTEN; + error = xfs_bmap_add_extent_unwritten_real(tp, ip, + whichfork, &icur, &cur, &del, + &logflags); + if (error) + goto error0; + goto nodelete; + } + + mod = xfs_rtb_to_rtxoff(mp, del.br_startblock); + if (mod) { + xfs_extlen_t off = mp->m_sb.sb_rextsize - mod; + + /* + * Realtime extent is lined up at the end but not + * at the front. We'll get rid of full extents if + * we can. + */ + if (del.br_blockcount > off) { + del.br_blockcount -= off; + del.br_startoff += off; + del.br_startblock += off; + } else if (del.br_startoff == start && + (del.br_state == XFS_EXT_UNWRITTEN || + tp->t_blk_res == 0)) { + /* + * Can't make it unwritten. There isn't + * a full extent here so just skip it. + */ + ASSERT(end >= del.br_blockcount); + end -= del.br_blockcount; + if (got.br_startoff > end && + !xfs_iext_prev_extent(ifp, &icur, &got)) { + done = true; + break; + } + continue; + } else if (del.br_state == XFS_EXT_UNWRITTEN) { + struct xfs_bmbt_irec prev; + xfs_fileoff_t unwrite_start; + + /* + * This one is already unwritten. + * It must have a written left neighbor. + * Unwrite the killed part of that one and + * try again. + */ + if (!xfs_iext_prev_extent(ifp, &icur, &prev)) + ASSERT(0); + ASSERT(prev.br_state == XFS_EXT_NORM); + ASSERT(!isnullstartblock(prev.br_startblock)); + ASSERT(del.br_startblock == + prev.br_startblock + prev.br_blockcount); + unwrite_start = max3(start, + del.br_startoff - mod, + prev.br_startoff); + mod = unwrite_start - prev.br_startoff; + prev.br_startoff = unwrite_start; + prev.br_startblock += mod; + prev.br_blockcount -= mod; + prev.br_state = XFS_EXT_UNWRITTEN; + error = xfs_bmap_add_extent_unwritten_real(tp, + ip, whichfork, &icur, &cur, + &prev, &logflags); + if (error) + goto error0; + goto nodelete; + } else { + ASSERT(del.br_state == XFS_EXT_NORM); + del.br_state = XFS_EXT_UNWRITTEN; + error = xfs_bmap_add_extent_unwritten_real(tp, + ip, whichfork, &icur, &cur, + &del, &logflags); + if (error) + goto error0; + goto nodelete; + } + } + +delete: + if (wasdel) { + xfs_bmap_del_extent_delay(ip, whichfork, &icur, &got, + &del, flags); + } else { + error = xfs_bmap_del_extent_real(ip, tp, &icur, cur, + &del, &tmp_logflags, whichfork, + flags); + logflags |= tmp_logflags; + if (error) + goto error0; + } + + end = del.br_startoff - 1; +nodelete: + /* + * If not done go on to the next (previous) record. + */ + if (end != (xfs_fileoff_t)-1 && end >= start) { + if (!xfs_iext_get_extent(ifp, &icur, &got) || + (got.br_startoff > end && + !xfs_iext_prev_extent(ifp, &icur, &got))) { + done = true; + break; + } + extno++; + } + } + if (done || end == (xfs_fileoff_t)-1 || end < start) + *rlen = 0; + else + *rlen = end - start + 1; + + /* + * Convert to a btree if necessary. + */ + if (xfs_bmap_needs_btree(ip, whichfork)) { + ASSERT(cur == NULL); + error = xfs_bmap_extents_to_btree(tp, ip, &cur, 0, + &tmp_logflags, whichfork); + logflags |= tmp_logflags; + } else { + error = xfs_bmap_btree_to_extents(tp, ip, cur, &logflags, + whichfork); + } + +error0: + /* + * Log everything. Do this after conversion, there's no point in + * logging the extent records if we've converted to btree format. + */ + if ((logflags & xfs_ilog_fext(whichfork)) && + ifp->if_format != XFS_DINODE_FMT_EXTENTS) + logflags &= ~xfs_ilog_fext(whichfork); + else if ((logflags & xfs_ilog_fbroot(whichfork)) && + ifp->if_format != XFS_DINODE_FMT_BTREE) + logflags &= ~xfs_ilog_fbroot(whichfork); + /* + * Log inode even in the error case, if the transaction + * is dirty we'll need to shut down the filesystem. + */ + if (logflags) + xfs_trans_log_inode(tp, ip, logflags); + if (cur) { + if (!error) + cur->bc_bmap.allocated = 0; + xfs_btree_del_cursor(cur, error); + } + return error; +} + +/* Unmap a range of a file. */ +int +xfs_bunmapi( + xfs_trans_t *tp, + struct xfs_inode *ip, + xfs_fileoff_t bno, + xfs_filblks_t len, + uint32_t flags, + xfs_extnum_t nexts, + int *done) +{ + int error; + + error = __xfs_bunmapi(tp, ip, bno, &len, flags, nexts); + *done = (len == 0); + return error; +} + +/* + * Determine whether an extent shift can be accomplished by a merge with the + * extent that precedes the target hole of the shift. + */ +STATIC bool +xfs_bmse_can_merge( + struct xfs_inode *ip, + int whichfork, + struct xfs_bmbt_irec *left, /* preceding extent */ + struct xfs_bmbt_irec *got, /* current extent to shift */ + xfs_fileoff_t shift) /* shift fsb */ +{ + xfs_fileoff_t startoff; + + startoff = got->br_startoff - shift; + + /* + * The extent, once shifted, must be adjacent in-file and on-disk with + * the preceding extent. + */ + if ((left->br_startoff + left->br_blockcount != startoff) || + (left->br_startblock + left->br_blockcount != got->br_startblock) || + (left->br_state != got->br_state) || + (left->br_blockcount + got->br_blockcount > XFS_MAX_BMBT_EXTLEN) || + !xfs_bmap_same_rtgroup(ip, whichfork, left, got)) + return false; + + return true; +} + +/* + * A bmap extent shift adjusts the file offset of an extent to fill a preceding + * hole in the file. If an extent shift would result in the extent being fully + * adjacent to the extent that currently precedes the hole, we can merge with + * the preceding extent rather than do the shift. + * + * This function assumes the caller has verified a shift-by-merge is possible + * with the provided extents via xfs_bmse_can_merge(). + */ +STATIC int +xfs_bmse_merge( + struct xfs_trans *tp, + struct xfs_inode *ip, + int whichfork, + xfs_fileoff_t shift, /* shift fsb */ + struct xfs_iext_cursor *icur, + struct xfs_bmbt_irec *got, /* extent to shift */ + struct xfs_bmbt_irec *left, /* preceding extent */ + struct xfs_btree_cur *cur, + int *logflags) /* output */ +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + struct xfs_bmbt_irec new; + xfs_filblks_t blockcount; + int error, i; + struct xfs_mount *mp = ip->i_mount; + + blockcount = left->br_blockcount + got->br_blockcount; + + xfs_assert_ilocked(ip, XFS_IOLOCK_EXCL | XFS_ILOCK_EXCL); + ASSERT(xfs_bmse_can_merge(ip, whichfork, left, got, shift)); + + new = *left; + new.br_blockcount = blockcount; + + /* + * Update the on-disk extent count, the btree if necessary and log the + * inode. + */ + ifp->if_nextents--; + *logflags |= XFS_ILOG_CORE; + if (!cur) { + *logflags |= XFS_ILOG_DEXT; + goto done; + } + + /* lookup and remove the extent to merge */ + error = xfs_bmbt_lookup_eq(cur, got, &i); + if (error) + return error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + + error = xfs_btree_delete(cur, &i); + if (error) + return error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + + /* lookup and update size of the previous extent */ + error = xfs_bmbt_lookup_eq(cur, left, &i); + if (error) + return error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + + error = xfs_bmbt_update(cur, &new); + if (error) + return error; + + /* change to extent format if required after extent removal */ + error = xfs_bmap_btree_to_extents(tp, ip, cur, logflags, whichfork); + if (error) + return error; + +done: + xfs_iext_remove(ip, icur, 0); + xfs_iext_prev(ifp, icur); + xfs_iext_update_extent(ip, xfs_bmap_fork_to_state(whichfork), icur, + &new); + + /* update reverse mapping. rmap functions merge the rmaps for us */ + xfs_rmap_unmap_extent(tp, ip, whichfork, got); + memcpy(&new, got, sizeof(new)); + new.br_startoff = left->br_startoff + left->br_blockcount; + xfs_rmap_map_extent(tp, ip, whichfork, &new); + return 0; +} + +static int +xfs_bmap_shift_update_extent( + struct xfs_trans *tp, + struct xfs_inode *ip, + int whichfork, + struct xfs_iext_cursor *icur, + struct xfs_bmbt_irec *got, + struct xfs_btree_cur *cur, + int *logflags, + xfs_fileoff_t startoff) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_bmbt_irec prev = *got; + int error, i; + + *logflags |= XFS_ILOG_CORE; + + got->br_startoff = startoff; + + if (cur) { + error = xfs_bmbt_lookup_eq(cur, &prev, &i); + if (error) + return error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + + error = xfs_bmbt_update(cur, got); + if (error) + return error; + } else { + *logflags |= XFS_ILOG_DEXT; + } + + xfs_iext_update_extent(ip, xfs_bmap_fork_to_state(whichfork), icur, + got); + + /* update reverse mapping */ + xfs_rmap_unmap_extent(tp, ip, whichfork, &prev); + xfs_rmap_map_extent(tp, ip, whichfork, got); + return 0; +} + +int +xfs_bmap_collapse_extents( + struct xfs_trans *tp, + struct xfs_inode *ip, + xfs_fileoff_t *next_fsb, + xfs_fileoff_t offset_shift_fsb, + bool *done) +{ + int whichfork = XFS_DATA_FORK; + struct xfs_mount *mp = ip->i_mount; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + struct xfs_btree_cur *cur = NULL; + struct xfs_bmbt_irec got, prev; + struct xfs_iext_cursor icur; + xfs_fileoff_t new_startoff; + int error = 0; + int logflags = 0; + + if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ifp)) || + XFS_TEST_ERROR(mp, XFS_ERRTAG_BMAPIFORMAT)) { + xfs_bmap_mark_sick(ip, whichfork); + return -EFSCORRUPTED; + } + + if (xfs_is_shutdown(mp)) + return -EIO; + + xfs_assert_ilocked(ip, XFS_IOLOCK_EXCL | XFS_ILOCK_EXCL); + + error = xfs_iread_extents(tp, ip, whichfork); + if (error) + return error; + + if (ifp->if_format == XFS_DINODE_FMT_BTREE) + cur = xfs_bmbt_init_cursor(mp, tp, ip, whichfork); + + if (!xfs_iext_lookup_extent(ip, ifp, *next_fsb, &icur, &got)) { + *done = true; + goto del_cursor; + } + if (XFS_IS_CORRUPT(mp, isnullstartblock(got.br_startblock))) { + xfs_bmap_mark_sick(ip, whichfork); + error = -EFSCORRUPTED; + goto del_cursor; + } + + new_startoff = got.br_startoff - offset_shift_fsb; + if (xfs_iext_peek_prev_extent(ifp, &icur, &prev)) { + if (new_startoff < prev.br_startoff + prev.br_blockcount) { + error = -EINVAL; + goto del_cursor; + } + + if (xfs_bmse_can_merge(ip, whichfork, &prev, &got, + offset_shift_fsb)) { + error = xfs_bmse_merge(tp, ip, whichfork, + offset_shift_fsb, &icur, &got, &prev, + cur, &logflags); + if (error) + goto del_cursor; + goto done; + } + } else { + if (got.br_startoff < offset_shift_fsb) { + error = -EINVAL; + goto del_cursor; + } + } + + error = xfs_bmap_shift_update_extent(tp, ip, whichfork, &icur, &got, + cur, &logflags, new_startoff); + if (error) + goto del_cursor; + +done: + if (!xfs_iext_next_extent(ifp, &icur, &got)) { + *done = true; + goto del_cursor; + } + + *next_fsb = got.br_startoff; +del_cursor: + if (cur) + xfs_btree_del_cursor(cur, error); + if (logflags) + xfs_trans_log_inode(tp, ip, logflags); + return error; +} + +/* Make sure we won't be right-shifting an extent past the maximum bound. */ +int +xfs_bmap_can_insert_extents( + struct xfs_inode *ip, + xfs_fileoff_t off, + xfs_fileoff_t shift) +{ + struct xfs_bmbt_irec got; + int is_empty; + int error = 0; + + xfs_assert_ilocked(ip, XFS_IOLOCK_EXCL); + + if (xfs_is_shutdown(ip->i_mount)) + return -EIO; + + xfs_ilock(ip, XFS_ILOCK_EXCL); + error = xfs_bmap_last_extent(NULL, ip, XFS_DATA_FORK, &got, &is_empty); + if (!error && !is_empty && got.br_startoff >= off && + ((got.br_startoff + shift) & BMBT_STARTOFF_MASK) < got.br_startoff) + error = -EINVAL; + xfs_iunlock(ip, XFS_ILOCK_EXCL); + + return error; +} + +int +xfs_bmap_insert_extents( + struct xfs_trans *tp, + struct xfs_inode *ip, + xfs_fileoff_t *next_fsb, + xfs_fileoff_t offset_shift_fsb, + bool *done, + xfs_fileoff_t stop_fsb) +{ + int whichfork = XFS_DATA_FORK; + struct xfs_mount *mp = ip->i_mount; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + struct xfs_btree_cur *cur = NULL; + struct xfs_bmbt_irec got, next; + struct xfs_iext_cursor icur; + xfs_fileoff_t new_startoff; + int error = 0; + int logflags = 0; + + if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ifp)) || + XFS_TEST_ERROR(mp, XFS_ERRTAG_BMAPIFORMAT)) { + xfs_bmap_mark_sick(ip, whichfork); + return -EFSCORRUPTED; + } + + if (xfs_is_shutdown(mp)) + return -EIO; + + xfs_assert_ilocked(ip, XFS_IOLOCK_EXCL | XFS_ILOCK_EXCL); + + error = xfs_iread_extents(tp, ip, whichfork); + if (error) + return error; + + if (ifp->if_format == XFS_DINODE_FMT_BTREE) + cur = xfs_bmbt_init_cursor(mp, tp, ip, whichfork); + + if (*next_fsb == NULLFSBLOCK) { + xfs_iext_last(ifp, &icur); + if (!xfs_iext_get_extent(ifp, &icur, &got) || + stop_fsb > got.br_startoff) { + *done = true; + goto del_cursor; + } + } else { + if (!xfs_iext_lookup_extent(ip, ifp, *next_fsb, &icur, &got)) { + *done = true; + goto del_cursor; + } + } + if (XFS_IS_CORRUPT(mp, isnullstartblock(got.br_startblock))) { + xfs_bmap_mark_sick(ip, whichfork); + error = -EFSCORRUPTED; + goto del_cursor; + } + + if (XFS_IS_CORRUPT(mp, stop_fsb > got.br_startoff)) { + xfs_bmap_mark_sick(ip, whichfork); + error = -EFSCORRUPTED; + goto del_cursor; + } + + new_startoff = got.br_startoff + offset_shift_fsb; + if (xfs_iext_peek_next_extent(ifp, &icur, &next)) { + if (new_startoff + got.br_blockcount > next.br_startoff) { + error = -EINVAL; + goto del_cursor; + } + + /* + * Unlike a left shift (which involves a hole punch), a right + * shift does not modify extent neighbors in any way. We should + * never find mergeable extents in this scenario. Check anyways + * and warn if we encounter two extents that could be one. + */ + if (xfs_bmse_can_merge(ip, whichfork, &got, &next, + offset_shift_fsb)) + WARN_ON_ONCE(1); + } + + error = xfs_bmap_shift_update_extent(tp, ip, whichfork, &icur, &got, + cur, &logflags, new_startoff); + if (error) + goto del_cursor; + + if (!xfs_iext_prev_extent(ifp, &icur, &got) || + stop_fsb >= got.br_startoff + got.br_blockcount) { + *done = true; + goto del_cursor; + } + + *next_fsb = got.br_startoff; +del_cursor: + if (cur) + xfs_btree_del_cursor(cur, error); + if (logflags) + xfs_trans_log_inode(tp, ip, logflags); + return error; +} + +/* + * Splits an extent into two extents at split_fsb block such that it is the + * first block of the current_ext. @ext is a target extent to be split. + * @split_fsb is a block where the extents is split. If split_fsb lies in a + * hole or the first block of extents, just return 0. + */ +int +xfs_bmap_split_extent( + struct xfs_trans *tp, + struct xfs_inode *ip, + xfs_fileoff_t split_fsb) +{ + int whichfork = XFS_DATA_FORK; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + struct xfs_btree_cur *cur = NULL; + struct xfs_bmbt_irec got; + struct xfs_bmbt_irec new; /* split extent */ + struct xfs_mount *mp = ip->i_mount; + xfs_fsblock_t gotblkcnt; /* new block count for got */ + struct xfs_iext_cursor icur; + int error = 0; + int logflags = 0; + int i = 0; + + if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(ifp)) || + XFS_TEST_ERROR(mp, XFS_ERRTAG_BMAPIFORMAT)) { + xfs_bmap_mark_sick(ip, whichfork); + return -EFSCORRUPTED; + } + + if (xfs_is_shutdown(mp)) + return -EIO; + + /* Read in all the extents */ + error = xfs_iread_extents(tp, ip, whichfork); + if (error) + return error; + + /* + * If there are not extents, or split_fsb lies in a hole we are done. + */ + if (!xfs_iext_lookup_extent(ip, ifp, split_fsb, &icur, &got) || + got.br_startoff >= split_fsb) + return 0; + + gotblkcnt = split_fsb - got.br_startoff; + new.br_startoff = split_fsb; + new.br_startblock = got.br_startblock + gotblkcnt; + new.br_blockcount = got.br_blockcount - gotblkcnt; + new.br_state = got.br_state; + + if (ifp->if_format == XFS_DINODE_FMT_BTREE) { + cur = xfs_bmbt_init_cursor(mp, tp, ip, whichfork); + error = xfs_bmbt_lookup_eq(cur, &got, &i); + if (error) + goto del_cursor; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto del_cursor; + } + } + + got.br_blockcount = gotblkcnt; + xfs_iext_update_extent(ip, xfs_bmap_fork_to_state(whichfork), &icur, + &got); + + logflags = XFS_ILOG_CORE; + if (cur) { + error = xfs_bmbt_update(cur, &got); + if (error) + goto del_cursor; + } else + logflags |= XFS_ILOG_DEXT; + + /* Add new extent */ + xfs_iext_next(ifp, &icur); + xfs_iext_insert(ip, &icur, &new, 0); + ifp->if_nextents++; + + if (cur) { + error = xfs_bmbt_lookup_eq(cur, &new, &i); + if (error) + goto del_cursor; + if (XFS_IS_CORRUPT(mp, i != 0)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto del_cursor; + } + error = xfs_btree_insert(cur, &i); + if (error) + goto del_cursor; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto del_cursor; + } + } + + /* + * Convert to a btree if necessary. + */ + if (xfs_bmap_needs_btree(ip, whichfork)) { + int tmp_logflags; /* partial log flag return val */ + + ASSERT(cur == NULL); + error = xfs_bmap_extents_to_btree(tp, ip, &cur, 0, + &tmp_logflags, whichfork); + logflags |= tmp_logflags; + } + +del_cursor: + if (cur) { + cur->bc_bmap.allocated = 0; + xfs_btree_del_cursor(cur, error); + } + + if (logflags) + xfs_trans_log_inode(tp, ip, logflags); + return error; +} + +/* Record a bmap intent. */ +static inline void +__xfs_bmap_add( + struct xfs_trans *tp, + enum xfs_bmap_intent_type type, + struct xfs_inode *ip, + int whichfork, + struct xfs_bmbt_irec *bmap) +{ + struct xfs_bmap_intent *bi; + + if ((whichfork != XFS_DATA_FORK && whichfork != XFS_ATTR_FORK) || + bmap->br_startblock == HOLESTARTBLOCK || + bmap->br_startblock == DELAYSTARTBLOCK) + return; + + bi = kmem_cache_alloc(xfs_bmap_intent_cache, GFP_KERNEL | __GFP_NOFAIL); + INIT_LIST_HEAD(&bi->bi_list); + bi->bi_type = type; + bi->bi_owner = ip; + bi->bi_whichfork = whichfork; + bi->bi_bmap = *bmap; + + xfs_bmap_defer_add(tp, bi); +} + +/* Map an extent into a file. */ +void +xfs_bmap_map_extent( + struct xfs_trans *tp, + struct xfs_inode *ip, + int whichfork, + struct xfs_bmbt_irec *PREV) +{ + __xfs_bmap_add(tp, XFS_BMAP_MAP, ip, whichfork, PREV); +} + +/* Unmap an extent out of a file. */ +void +xfs_bmap_unmap_extent( + struct xfs_trans *tp, + struct xfs_inode *ip, + int whichfork, + struct xfs_bmbt_irec *PREV) +{ + __xfs_bmap_add(tp, XFS_BMAP_UNMAP, ip, whichfork, PREV); +} + +/* + * Process one of the deferred bmap operations. We pass back the + * btree cursor to maintain our lock on the bmapbt between calls. + */ +int +xfs_bmap_finish_one( + struct xfs_trans *tp, + struct xfs_bmap_intent *bi) +{ + struct xfs_bmbt_irec *bmap = &bi->bi_bmap; + int error = 0; + int flags = 0; + + if (bi->bi_whichfork == XFS_ATTR_FORK) + flags |= XFS_BMAPI_ATTRFORK; + + ASSERT(tp->t_highest_agno == NULLAGNUMBER); + + trace_xfs_bmap_deferred(bi); + + if (XFS_TEST_ERROR(tp->t_mountp, XFS_ERRTAG_BMAP_FINISH_ONE)) + return -EIO; + + switch (bi->bi_type) { + case XFS_BMAP_MAP: + if (bi->bi_bmap.br_state == XFS_EXT_UNWRITTEN) + flags |= XFS_BMAPI_PREALLOC; + error = xfs_bmapi_remap(tp, bi->bi_owner, bmap->br_startoff, + bmap->br_blockcount, bmap->br_startblock, + flags); + bmap->br_blockcount = 0; + break; + case XFS_BMAP_UNMAP: + error = __xfs_bunmapi(tp, bi->bi_owner, bmap->br_startoff, + &bmap->br_blockcount, flags | XFS_BMAPI_REMAP, + 1); + break; + default: + ASSERT(0); + xfs_bmap_mark_sick(bi->bi_owner, bi->bi_whichfork); + error = -EFSCORRUPTED; + } + + return error; +} + +/* Check that an extent does not have invalid flags or bad ranges. */ +xfs_failaddr_t +xfs_bmap_validate_extent_raw( + struct xfs_mount *mp, + bool rtfile, + int whichfork, + struct xfs_bmbt_irec *irec) +{ + if (!xfs_verify_fileext(mp, irec->br_startoff, irec->br_blockcount)) + return __this_address; + + if (rtfile && whichfork == XFS_DATA_FORK) { + if (!xfs_verify_rtbext(mp, irec->br_startblock, + irec->br_blockcount)) + return __this_address; + } else { + if (!xfs_verify_fsbext(mp, irec->br_startblock, + irec->br_blockcount)) + return __this_address; + } + if (irec->br_state != XFS_EXT_NORM && whichfork != XFS_DATA_FORK) + return __this_address; + return NULL; +} + +int __init +xfs_bmap_intent_init_cache(void) +{ + xfs_bmap_intent_cache = kmem_cache_create("xfs_bmap_intent", + sizeof(struct xfs_bmap_intent), + 0, 0, NULL); + + return xfs_bmap_intent_cache != NULL ? 0 : -ENOMEM; +} + +void +xfs_bmap_intent_destroy_cache(void) +{ + kmem_cache_destroy(xfs_bmap_intent_cache); + xfs_bmap_intent_cache = NULL; +} + +/* Check that an inode's extent does not have invalid flags or bad ranges. */ +xfs_failaddr_t +xfs_bmap_validate_extent( + struct xfs_inode *ip, + int whichfork, + struct xfs_bmbt_irec *irec) +{ + return xfs_bmap_validate_extent_raw(ip->i_mount, + XFS_IS_REALTIME_INODE(ip), whichfork, irec); +} + +/* + * Used in xfs_itruncate_extents(). This is the maximum number of extents + * freed from a file in a single transaction. + */ +#define XFS_ITRUNC_MAX_EXTENTS 2 + +/* + * Unmap every extent in part of an inode's fork. We don't do any higher level + * invalidation work at all. + */ +int +xfs_bunmapi_range( + struct xfs_trans **tpp, + struct xfs_inode *ip, + uint32_t flags, + xfs_fileoff_t startoff, + xfs_fileoff_t endoff) +{ + xfs_filblks_t unmap_len = endoff - startoff + 1; + int error = 0; + + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + + while (unmap_len > 0) { + ASSERT((*tpp)->t_highest_agno == NULLAGNUMBER); + error = __xfs_bunmapi(*tpp, ip, startoff, &unmap_len, flags, + XFS_ITRUNC_MAX_EXTENTS); + if (error) + goto out; + + /* free the just unmapped extents */ + error = xfs_defer_finish(tpp); + if (error) + goto out; + cond_resched(); + } +out: + return error; +} + +struct xfs_bmap_query_range { + xfs_bmap_query_range_fn fn; + void *priv; +}; + +/* Format btree record and pass to our callback. */ +STATIC int +xfs_bmap_query_range_helper( + struct xfs_btree_cur *cur, + const union xfs_btree_rec *rec, + void *priv) +{ + struct xfs_bmap_query_range *query = priv; + struct xfs_bmbt_irec irec; + xfs_failaddr_t fa; + + xfs_bmbt_disk_get_all(&rec->bmbt, &irec); + fa = xfs_bmap_validate_extent(cur->bc_ino.ip, cur->bc_ino.whichfork, + &irec); + if (fa) { + xfs_btree_mark_sick(cur); + return xfs_bmap_complain_bad_rec(cur->bc_ino.ip, + cur->bc_ino.whichfork, fa, &irec); + } + + return query->fn(cur, &irec, query->priv); +} + +/* Find all bmaps. */ +int +xfs_bmap_query_all( + struct xfs_btree_cur *cur, + xfs_bmap_query_range_fn fn, + void *priv) +{ + struct xfs_bmap_query_range query = { + .priv = priv, + .fn = fn, + }; + + return xfs_btree_query_all(cur, xfs_bmap_query_range_helper, &query); +} + +/* Helper function to extract extent size hint from inode */ +xfs_extlen_t +xfs_get_extsz_hint( + struct xfs_inode *ip) +{ + /* + * No point in aligning allocations if we need to COW to actually + * write to them. + */ + if (!xfs_is_always_cow_inode(ip) && + (ip->i_diflags & XFS_DIFLAG_EXTSIZE) && ip->i_extsize) + return ip->i_extsize; + if (XFS_IS_REALTIME_INODE(ip) && + ip->i_mount->m_sb.sb_rextsize > 1) + return ip->i_mount->m_sb.sb_rextsize; + return 0; +} + +/* + * Helper function to extract CoW extent size hint from inode. + * Between the extent size hint and the CoW extent size hint, we + * return the greater of the two. If the value is zero (automatic), + * use the default size. + */ +xfs_extlen_t +xfs_get_cowextsz_hint( + struct xfs_inode *ip) +{ + xfs_extlen_t a, b; + + a = 0; + if (ip->i_diflags2 & XFS_DIFLAG2_COWEXTSIZE) + a = ip->i_cowextsize; + if (XFS_IS_REALTIME_INODE(ip)) { + b = 0; + if (ip->i_diflags & XFS_DIFLAG_EXTSIZE) + b = ip->i_extsize; + } else { + b = xfs_get_extsz_hint(ip); + } + + a = max(a, b); + if (a == 0) + return XFS_DEFAULT_COWEXTSZ_HINT; + return a; +} diff --git a/fs/xfs/libxfs/xfs_bmap.h b/fs/xfs/libxfs/xfs_bmap.h new file mode 100644 index 000000000..d5f272930 --- /dev/null +++ b/fs/xfs/libxfs/xfs_bmap.h @@ -0,0 +1,303 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2000-2006 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_BMAP_H__ +#define __XFS_BMAP_H__ + +struct getbmap; +struct xfs_bmbt_irec; +struct xfs_ifork; +struct xfs_inode; +struct xfs_mount; +struct xfs_trans; +struct xfs_alloc_arg; + +/* + * Argument structure for xfs_bmap_alloc. + */ +struct xfs_bmalloca { + struct xfs_trans *tp; /* transaction pointer */ + struct xfs_inode *ip; /* incore inode pointer */ + struct xfs_bmbt_irec prev; /* extent before the new one */ + struct xfs_bmbt_irec got; /* extent after, or delayed */ + + xfs_fileoff_t offset; /* offset in file filling in */ + xfs_extlen_t length; /* i/o length asked/allocated */ + xfs_fsblock_t blkno; /* starting block of new extent */ + + struct xfs_btree_cur *cur; /* btree cursor */ + struct xfs_iext_cursor icur; /* incore extent cursor */ + int nallocs;/* number of extents alloc'd */ + int logflags;/* flags for transaction logging */ + + xfs_extlen_t total; /* total blocks needed for xaction */ + xfs_extlen_t minlen; /* minimum allocation size (blocks) */ + xfs_extlen_t minleft; /* amount must be left after alloc */ + bool eof; /* set if allocating past last extent */ + bool wasdel; /* replacing a delayed allocation */ + bool aeof; /* allocated space at eof */ + bool conv; /* overwriting unwritten extents */ + int datatype;/* data type being allocated */ + uint32_t flags; +}; + +#define XFS_BMAP_MAX_NMAP 4 + +/* + * Flags for xfs_bmapi_* + */ +#define XFS_BMAPI_ENTIRE (1u << 0) /* return entire extent untrimmed */ +#define XFS_BMAPI_METADATA (1u << 1) /* mapping metadata not user data */ +#define XFS_BMAPI_ATTRFORK (1u << 2) /* use attribute fork not data */ +#define XFS_BMAPI_PREALLOC (1u << 3) /* preallocating unwritten space */ +#define XFS_BMAPI_CONTIG (1u << 4) /* must allocate only one extent */ +/* + * unwritten extent conversion - this needs write cache flushing and no additional + * allocation alignments. When specified with XFS_BMAPI_PREALLOC it converts + * from written to unwritten, otherwise convert from unwritten to written. + */ +#define XFS_BMAPI_CONVERT (1u << 5) + +/* + * allocate zeroed extents - this requires all newly allocated user data extents + * to be initialised to zero. It will be ignored if XFS_BMAPI_METADATA is set. + * Use in conjunction with XFS_BMAPI_CONVERT to convert unwritten extents found + * during the allocation range to zeroed written extents. + */ +#define XFS_BMAPI_ZERO (1u << 6) + +/* + * Map the inode offset to the block given in ap->firstblock. Primarily + * used for reflink. The range must be in a hole, and this flag cannot be + * turned on with PREALLOC or CONVERT, and cannot be used on the attr fork. + * + * For bunmapi, this flag unmaps the range without adjusting quota, reducing + * refcount, or freeing the blocks. + */ +#define XFS_BMAPI_REMAP (1u << 7) + +/* Map something in the CoW fork. */ +#define XFS_BMAPI_COWFORK (1u << 8) + +/* Skip online discard of freed extents */ +#define XFS_BMAPI_NODISCARD (1u << 9) + +/* Do not update the rmap btree. Used for reconstructing bmbt from rmapbt. */ +#define XFS_BMAPI_NORMAP (1u << 10) + +/* Try to align allocations to the extent size hint */ +#define XFS_BMAPI_EXTSZALIGN (1u << 11) + +#define XFS_BMAPI_FLAGS \ + { XFS_BMAPI_ENTIRE, "ENTIRE" }, \ + { XFS_BMAPI_METADATA, "METADATA" }, \ + { XFS_BMAPI_ATTRFORK, "ATTRFORK" }, \ + { XFS_BMAPI_PREALLOC, "PREALLOC" }, \ + { XFS_BMAPI_CONTIG, "CONTIG" }, \ + { XFS_BMAPI_CONVERT, "CONVERT" }, \ + { XFS_BMAPI_ZERO, "ZERO" }, \ + { XFS_BMAPI_REMAP, "REMAP" }, \ + { XFS_BMAPI_COWFORK, "COWFORK" }, \ + { XFS_BMAPI_NODISCARD, "NODISCARD" }, \ + { XFS_BMAPI_NORMAP, "NORMAP" },\ + { XFS_BMAPI_EXTSZALIGN, "EXTSZALIGN" } + + +static inline int xfs_bmapi_aflag(int w) +{ + return (w == XFS_ATTR_FORK ? XFS_BMAPI_ATTRFORK : + (w == XFS_COW_FORK ? XFS_BMAPI_COWFORK : 0)); +} + +static inline int xfs_bmapi_whichfork(uint32_t bmapi_flags) +{ + if (bmapi_flags & XFS_BMAPI_COWFORK) + return XFS_COW_FORK; + else if (bmapi_flags & XFS_BMAPI_ATTRFORK) + return XFS_ATTR_FORK; + return XFS_DATA_FORK; +} + +void xfs_bmap_alloc_account(struct xfs_bmalloca *ap); + +/* + * Special values for xfs_bmbt_irec_t br_startblock field. + */ +#define DELAYSTARTBLOCK ((xfs_fsblock_t)-1LL) +#define HOLESTARTBLOCK ((xfs_fsblock_t)-2LL) + +/* + * Flags for xfs_bmap_add_extent*. + */ +#define BMAP_LEFT_CONTIG (1u << 0) +#define BMAP_RIGHT_CONTIG (1u << 1) +#define BMAP_LEFT_FILLING (1u << 2) +#define BMAP_RIGHT_FILLING (1u << 3) +#define BMAP_LEFT_DELAY (1u << 4) +#define BMAP_RIGHT_DELAY (1u << 5) +#define BMAP_LEFT_VALID (1u << 6) +#define BMAP_RIGHT_VALID (1u << 7) +#define BMAP_ATTRFORK (1u << 8) +#define BMAP_COWFORK (1u << 9) + +#define XFS_BMAP_EXT_FLAGS \ + { BMAP_LEFT_CONTIG, "LC" }, \ + { BMAP_RIGHT_CONTIG, "RC" }, \ + { BMAP_LEFT_FILLING, "LF" }, \ + { BMAP_RIGHT_FILLING, "RF" }, \ + { BMAP_ATTRFORK, "ATTR" }, \ + { BMAP_COWFORK, "COW" } + +/* Return true if the extent is an allocated extent, written or not. */ +static inline bool xfs_bmap_is_real_extent(const struct xfs_bmbt_irec *irec) +{ + return irec->br_startblock != HOLESTARTBLOCK && + irec->br_startblock != DELAYSTARTBLOCK && + !isnullstartblock(irec->br_startblock); +} + +/* + * Return true if the extent is a real, allocated extent, or false if it is a + * delayed allocation, and unwritten extent or a hole. + */ +static inline bool xfs_bmap_is_written_extent(const struct xfs_bmbt_irec *irec) +{ + return xfs_bmap_is_real_extent(irec) && + irec->br_state != XFS_EXT_UNWRITTEN; +} + +/* + * Check the mapping for obviously garbage allocations that could trash the + * filesystem immediately. + */ +#define xfs_valid_startblock(ip, startblock) \ + ((startblock) != 0 || XFS_IS_REALTIME_INODE(ip)) + +int xfs_bmap_longest_free_extent(struct xfs_perag *pag, + struct xfs_trans *tp, xfs_extlen_t *blen); +void xfs_trim_extent(struct xfs_bmbt_irec *irec, xfs_fileoff_t bno, + xfs_filblks_t len); +unsigned int xfs_bmap_compute_attr_offset(struct xfs_mount *mp); +int xfs_bmap_add_attrfork(struct xfs_trans *tp, struct xfs_inode *ip, + int size, int rsvd); +void xfs_bmap_local_to_extents_empty(struct xfs_trans *tp, + struct xfs_inode *ip, int whichfork); +int xfs_bmap_local_to_extents(struct xfs_trans *tp, struct xfs_inode *ip, + xfs_extlen_t total, int *logflagsp, int whichfork, + void (*init_fn)(struct xfs_trans *tp, struct xfs_buf *bp, + struct xfs_inode *ip, struct xfs_ifork *ifp, + void *priv), + void *priv); +void xfs_bmap_compute_maxlevels(struct xfs_mount *mp, int whichfork); +int xfs_bmap_first_unused(struct xfs_trans *tp, struct xfs_inode *ip, + xfs_extlen_t len, xfs_fileoff_t *unused, int whichfork); +int xfs_bmap_last_before(struct xfs_trans *tp, struct xfs_inode *ip, + xfs_fileoff_t *last_block, int whichfork); +int xfs_bmap_last_offset(struct xfs_inode *ip, xfs_fileoff_t *unused, + int whichfork); +int xfs_bmapi_read(struct xfs_inode *ip, xfs_fileoff_t bno, + xfs_filblks_t len, struct xfs_bmbt_irec *mval, + int *nmap, uint32_t flags); +int xfs_bmapi_write(struct xfs_trans *tp, struct xfs_inode *ip, + xfs_fileoff_t bno, xfs_filblks_t len, uint32_t flags, + xfs_extlen_t total, struct xfs_bmbt_irec *mval, int *nmap); +int xfs_bunmapi(struct xfs_trans *tp, struct xfs_inode *ip, + xfs_fileoff_t bno, xfs_filblks_t len, uint32_t flags, + xfs_extnum_t nexts, int *done); +void xfs_bmap_del_extent_delay(struct xfs_inode *ip, int whichfork, + struct xfs_iext_cursor *cur, struct xfs_bmbt_irec *got, + struct xfs_bmbt_irec *del, uint32_t bflags); +void xfs_bmap_del_extent_cow(struct xfs_inode *ip, + struct xfs_iext_cursor *cur, struct xfs_bmbt_irec *got, + struct xfs_bmbt_irec *del); +uint xfs_default_attroffset(struct xfs_inode *ip); +int xfs_bmap_collapse_extents(struct xfs_trans *tp, struct xfs_inode *ip, + xfs_fileoff_t *next_fsb, xfs_fileoff_t offset_shift_fsb, + bool *done); +int xfs_bmap_can_insert_extents(struct xfs_inode *ip, xfs_fileoff_t off, + xfs_fileoff_t shift); +int xfs_bmap_insert_extents(struct xfs_trans *tp, struct xfs_inode *ip, + xfs_fileoff_t *next_fsb, xfs_fileoff_t offset_shift_fsb, + bool *done, xfs_fileoff_t stop_fsb); +int xfs_bmap_split_extent(struct xfs_trans *tp, struct xfs_inode *ip, + xfs_fileoff_t split_offset); +int xfs_bmapi_convert_delalloc(struct xfs_inode *ip, int whichfork, + xfs_off_t offset, struct iomap *iomap, unsigned int *seq); +int xfs_bmap_add_extent_unwritten_real(struct xfs_trans *tp, + struct xfs_inode *ip, int whichfork, + struct xfs_iext_cursor *icur, struct xfs_btree_cur **curp, + struct xfs_bmbt_irec *new, int *logflagsp); +xfs_extlen_t xfs_bmapi_minleft(struct xfs_trans *tp, struct xfs_inode *ip, + int fork); +int xfs_bmap_btalloc_low_space(struct xfs_bmalloca *ap, + struct xfs_alloc_arg *args); +xfs_filblks_t xfs_bmap_worst_indlen(struct xfs_inode *ip, xfs_filblks_t len); + +enum xfs_bmap_intent_type { + XFS_BMAP_MAP = 1, + XFS_BMAP_UNMAP, +}; + +#define XFS_BMAP_INTENT_STRINGS \ + { XFS_BMAP_MAP, "map" }, \ + { XFS_BMAP_UNMAP, "unmap" } + +struct xfs_bmap_intent { + struct list_head bi_list; + enum xfs_bmap_intent_type bi_type; + int bi_whichfork; + struct xfs_inode *bi_owner; + struct xfs_group *bi_group; + struct xfs_bmbt_irec bi_bmap; +}; + +int xfs_bmap_finish_one(struct xfs_trans *tp, struct xfs_bmap_intent *bi); +void xfs_bmap_map_extent(struct xfs_trans *tp, struct xfs_inode *ip, + int whichfork, struct xfs_bmbt_irec *imap); +void xfs_bmap_unmap_extent(struct xfs_trans *tp, struct xfs_inode *ip, + int whichfork, struct xfs_bmbt_irec *imap); + +static inline uint32_t xfs_bmap_fork_to_state(int whichfork) +{ + switch (whichfork) { + case XFS_ATTR_FORK: + return BMAP_ATTRFORK; + case XFS_COW_FORK: + return BMAP_COWFORK; + default: + return 0; + } +} + +xfs_failaddr_t xfs_bmap_validate_extent_raw(struct xfs_mount *mp, bool rtfile, + int whichfork, struct xfs_bmbt_irec *irec); +xfs_failaddr_t xfs_bmap_validate_extent(struct xfs_inode *ip, int whichfork, + struct xfs_bmbt_irec *irec); +int xfs_bmap_complain_bad_rec(struct xfs_inode *ip, int whichfork, + xfs_failaddr_t fa, const struct xfs_bmbt_irec *irec); + +int xfs_bmapi_remap(struct xfs_trans *tp, struct xfs_inode *ip, + xfs_fileoff_t bno, xfs_filblks_t len, xfs_fsblock_t startblock, + uint32_t flags); +int xfs_bunmapi_range(struct xfs_trans **tpp, struct xfs_inode *ip, + uint32_t flags, xfs_fileoff_t startoff, xfs_fileoff_t endoff); + +extern struct kmem_cache *xfs_bmap_intent_cache; + +int __init xfs_bmap_intent_init_cache(void); +void xfs_bmap_intent_destroy_cache(void); + +typedef int (*xfs_bmap_query_range_fn)( + struct xfs_btree_cur *cur, + struct xfs_bmbt_irec *rec, + void *priv); + +int xfs_bmap_query_all(struct xfs_btree_cur *cur, xfs_bmap_query_range_fn fn, + void *priv); + +xfs_extlen_t xfs_get_extsz_hint(struct xfs_inode *ip); +xfs_extlen_t xfs_get_cowextsz_hint(struct xfs_inode *ip); + +#endif /* __XFS_BMAP_H__ */ diff --git a/fs/xfs/libxfs/xfs_bmap_btree.c b/fs/xfs/libxfs/xfs_bmap_btree.c new file mode 100644 index 000000000..758a4b1cc --- /dev/null +++ b/fs/xfs/libxfs/xfs_bmap_btree.c @@ -0,0 +1,860 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2003,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_alloc.h" +#include "xfs_btree.h" +#include "xfs_btree_staging.h" +#include "xfs_bmap_btree.h" +#include "xfs_bmap.h" +#include "xfs_error.h" +#include "xfs_quota.h" +#include "xfs_trace.h" +#include "xfs_rmap.h" +#include "xfs_ag.h" + +static struct kmem_cache *xfs_bmbt_cur_cache; + +void +xfs_bmbt_init_block( + struct xfs_inode *ip, + struct xfs_btree_block *buf, + struct xfs_buf *bp, + __u16 level, + __u16 numrecs) +{ + if (bp) + xfs_btree_init_buf(ip->i_mount, bp, &xfs_bmbt_ops, level, + numrecs, I_INO(ip)); + else + xfs_btree_init_block(ip->i_mount, buf, &xfs_bmbt_ops, level, + numrecs, I_INO(ip)); +} + +/* + * Convert on-disk form of btree root to in-memory form. + */ +void +xfs_bmdr_to_bmbt( + struct xfs_inode *ip, + xfs_bmdr_block_t *dblock, + int dblocklen, + struct xfs_btree_block *rblock, + int rblocklen) +{ + struct xfs_mount *mp = ip->i_mount; + int dmxr; + xfs_bmbt_key_t *fkp; + __be64 *fpp; + xfs_bmbt_key_t *tkp; + __be64 *tpp; + + xfs_bmbt_init_block(ip, rblock, NULL, 0, 0); + rblock->bb_level = dblock->bb_level; + ASSERT(be16_to_cpu(rblock->bb_level) > 0); + rblock->bb_numrecs = dblock->bb_numrecs; + dmxr = xfs_bmdr_maxrecs(dblocklen, 0); + fkp = xfs_bmdr_key_addr(dblock, 1); + tkp = xfs_bmbt_key_addr(mp, rblock, 1); + fpp = xfs_bmdr_ptr_addr(dblock, 1, dmxr); + tpp = xfs_bmap_broot_ptr_addr(mp, rblock, 1, rblocklen); + dmxr = be16_to_cpu(dblock->bb_numrecs); + memcpy(tkp, fkp, sizeof(*fkp) * dmxr); + memcpy(tpp, fpp, sizeof(*fpp) * dmxr); +} + +void +xfs_bmbt_disk_get_all( + const struct xfs_bmbt_rec *rec, + struct xfs_bmbt_irec *irec) +{ + uint64_t l0 = get_unaligned_be64(&rec->l0); + uint64_t l1 = get_unaligned_be64(&rec->l1); + + irec->br_startoff = (l0 & xfs_mask64lo(64 - BMBT_EXNTFLAG_BITLEN)) >> 9; + irec->br_startblock = ((l0 & xfs_mask64lo(9)) << 43) | (l1 >> 21); + irec->br_blockcount = l1 & xfs_mask64lo(21); + if (l0 >> (64 - BMBT_EXNTFLAG_BITLEN)) + irec->br_state = XFS_EXT_UNWRITTEN; + else + irec->br_state = XFS_EXT_NORM; +} + +/* + * Extract the blockcount field from an on disk bmap extent record. + */ +xfs_filblks_t +xfs_bmbt_disk_get_blockcount( + const struct xfs_bmbt_rec *r) +{ + return (xfs_filblks_t)(be64_to_cpu(r->l1) & xfs_mask64lo(21)); +} + +/* + * Extract the startoff field from a disk format bmap extent record. + */ +xfs_fileoff_t +xfs_bmbt_disk_get_startoff( + const struct xfs_bmbt_rec *r) +{ + return ((xfs_fileoff_t)be64_to_cpu(r->l0) & + xfs_mask64lo(64 - BMBT_EXNTFLAG_BITLEN)) >> 9; +} + +/* + * Set all the fields in a bmap extent record from the uncompressed form. + */ +void +xfs_bmbt_disk_set_all( + struct xfs_bmbt_rec *r, + struct xfs_bmbt_irec *s) +{ + int extent_flag = (s->br_state != XFS_EXT_NORM); + + ASSERT(s->br_state == XFS_EXT_NORM || s->br_state == XFS_EXT_UNWRITTEN); + ASSERT(!(s->br_startoff & xfs_mask64hi(64-BMBT_STARTOFF_BITLEN))); + ASSERT(!(s->br_blockcount & xfs_mask64hi(64-BMBT_BLOCKCOUNT_BITLEN))); + ASSERT(!(s->br_startblock & xfs_mask64hi(64-BMBT_STARTBLOCK_BITLEN))); + + put_unaligned_be64( + ((xfs_bmbt_rec_base_t)extent_flag << 63) | + ((xfs_bmbt_rec_base_t)s->br_startoff << 9) | + ((xfs_bmbt_rec_base_t)s->br_startblock >> 43), &r->l0); + put_unaligned_be64( + ((xfs_bmbt_rec_base_t)s->br_startblock << 21) | + ((xfs_bmbt_rec_base_t)s->br_blockcount & + (xfs_bmbt_rec_base_t)xfs_mask64lo(21)), &r->l1); +} + +/* + * Convert in-memory form of btree root to on-disk form. + */ +void +xfs_bmbt_to_bmdr( + struct xfs_mount *mp, + struct xfs_btree_block *rblock, + int rblocklen, + xfs_bmdr_block_t *dblock, + int dblocklen) +{ + int dmxr; + xfs_bmbt_key_t *fkp; + __be64 *fpp; + xfs_bmbt_key_t *tkp; + __be64 *tpp; + + if (xfs_has_crc(mp)) { + ASSERT(rblock->bb_magic == cpu_to_be32(XFS_BMAP_CRC_MAGIC)); + ASSERT(uuid_equal(&rblock->bb_u.l.bb_uuid, + &mp->m_sb.sb_meta_uuid)); + ASSERT(rblock->bb_u.l.bb_blkno == + cpu_to_be64(XFS_BUF_DADDR_NULL)); + } else + ASSERT(rblock->bb_magic == cpu_to_be32(XFS_BMAP_MAGIC)); + ASSERT(rblock->bb_u.l.bb_leftsib == cpu_to_be64(NULLFSBLOCK)); + ASSERT(rblock->bb_u.l.bb_rightsib == cpu_to_be64(NULLFSBLOCK)); + ASSERT(rblock->bb_level != 0); + dblock->bb_level = rblock->bb_level; + dblock->bb_numrecs = rblock->bb_numrecs; + dmxr = xfs_bmdr_maxrecs(dblocklen, 0); + fkp = xfs_bmbt_key_addr(mp, rblock, 1); + tkp = xfs_bmdr_key_addr(dblock, 1); + fpp = xfs_bmap_broot_ptr_addr(mp, rblock, 1, rblocklen); + tpp = xfs_bmdr_ptr_addr(dblock, 1, dmxr); + dmxr = be16_to_cpu(dblock->bb_numrecs); + memcpy(tkp, fkp, sizeof(*fkp) * dmxr); + memcpy(tpp, fpp, sizeof(*fpp) * dmxr); +} + +STATIC struct xfs_btree_cur * +xfs_bmbt_dup_cursor( + struct xfs_btree_cur *cur) +{ + struct xfs_btree_cur *new; + + new = xfs_bmbt_init_cursor(cur->bc_mp, cur->bc_tp, + cur->bc_ino.ip, cur->bc_ino.whichfork); + new->bc_flags |= (cur->bc_flags & + (XFS_BTREE_BMBT_INVALID_OWNER | XFS_BTREE_BMBT_WASDEL)); + return new; +} + +STATIC void +xfs_bmbt_update_cursor( + struct xfs_btree_cur *src, + struct xfs_btree_cur *dst) +{ + ASSERT((dst->bc_tp->t_highest_agno != NULLAGNUMBER) || + (dst->bc_ino.ip->i_diflags & XFS_DIFLAG_REALTIME)); + + dst->bc_bmap.allocated += src->bc_bmap.allocated; + dst->bc_tp->t_highest_agno = src->bc_tp->t_highest_agno; + + src->bc_bmap.allocated = 0; +} + +STATIC int +xfs_bmbt_alloc_block( + struct xfs_btree_cur *cur, + const union xfs_btree_ptr *start, + union xfs_btree_ptr *new, + int *stat) +{ + struct xfs_alloc_arg args; + int error; + + memset(&args, 0, sizeof(args)); + args.tp = cur->bc_tp; + args.mp = cur->bc_mp; + xfs_rmap_inode_bmbt_owner(&args.oinfo, cur->bc_ino.ip, + cur->bc_ino.whichfork); + args.minlen = args.maxlen = args.prod = 1; + args.wasdel = cur->bc_flags & XFS_BTREE_BMBT_WASDEL; + if (!args.wasdel && args.tp->t_blk_res == 0) + return -ENOSPC; + + /* + * If we are coming here from something like unwritten extent + * conversion, there has been no data extent allocation already done, so + * we have to ensure that we attempt to locate the entire set of bmbt + * allocations in the same AG, as xfs_bmapi_write() would have reserved. + */ + if (cur->bc_tp->t_highest_agno == NULLAGNUMBER) + args.minleft = xfs_bmapi_minleft(cur->bc_tp, cur->bc_ino.ip, + cur->bc_ino.whichfork); + + error = xfs_alloc_vextent_start_ag(&args, be64_to_cpu(start->l)); + if (error) + return error; + + if (args.fsbno == NULLFSBLOCK && args.minleft) { + /* + * Could not find an AG with enough free space to satisfy + * a full btree split. Try again and if + * successful activate the lowspace algorithm. + */ + args.minleft = 0; + error = xfs_alloc_vextent_start_ag(&args, 0); + if (error) + return error; + cur->bc_tp->t_flags |= XFS_TRANS_LOWMODE; + } + if (WARN_ON_ONCE(args.fsbno == NULLFSBLOCK)) { + *stat = 0; + return 0; + } + + ASSERT(args.len == 1); + cur->bc_bmap.allocated++; + cur->bc_ino.ip->i_nblocks++; + xfs_trans_log_inode(args.tp, cur->bc_ino.ip, XFS_ILOG_CORE); + xfs_trans_mod_dquot_byino(args.tp, cur->bc_ino.ip, + XFS_TRANS_DQ_BCOUNT, 1L); + + new->l = cpu_to_be64(args.fsbno); + + *stat = 1; + return 0; +} + +STATIC int +xfs_bmbt_free_block( + struct xfs_btree_cur *cur, + struct xfs_buf *bp) +{ + struct xfs_mount *mp = cur->bc_mp; + struct xfs_inode *ip = cur->bc_ino.ip; + struct xfs_trans *tp = cur->bc_tp; + xfs_fsblock_t fsbno = XFS_DADDR_TO_FSB(mp, xfs_buf_daddr(bp)); + struct xfs_owner_info oinfo; + int error; + + xfs_rmap_inode_bmbt_owner(&oinfo, ip, cur->bc_ino.whichfork); + error = xfs_free_extent_later(cur->bc_tp, fsbno, 1, &oinfo, + XFS_AG_RESV_NONE, 0); + if (error) + return error; + + ip->i_nblocks--; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_BCOUNT, -1L); + return 0; +} + +STATIC int +xfs_bmbt_get_minrecs( + struct xfs_btree_cur *cur, + int level) +{ + if (level == cur->bc_nlevels - 1) { + struct xfs_ifork *ifp = xfs_btree_ifork_ptr(cur); + + return xfs_bmbt_maxrecs(cur->bc_mp, + ifp->if_broot_bytes, level == 0) / 2; + } + + return cur->bc_mp->m_bmap_dmnr[level != 0]; +} + +int +xfs_bmbt_get_maxrecs( + struct xfs_btree_cur *cur, + int level) +{ + if (level == cur->bc_nlevels - 1) { + struct xfs_ifork *ifp = xfs_btree_ifork_ptr(cur); + + return xfs_bmbt_maxrecs(cur->bc_mp, + ifp->if_broot_bytes, level == 0); + } + + return cur->bc_mp->m_bmap_dmxr[level != 0]; + +} + +/* + * Get the maximum records we could store in the on-disk format. + * + * For non-root nodes this is equivalent to xfs_bmbt_get_maxrecs, but + * for the root node this checks the available space in the dinode fork + * so that we can resize the in-memory buffer to match it. After a + * resize to the maximum size this function returns the same value + * as xfs_bmbt_get_maxrecs for the root node, too. + */ +STATIC int +xfs_bmbt_get_dmaxrecs( + struct xfs_btree_cur *cur, + int level) +{ + if (level != cur->bc_nlevels - 1) + return cur->bc_mp->m_bmap_dmxr[level != 0]; + return xfs_bmdr_maxrecs(cur->bc_ino.forksize, level == 0); +} + +STATIC void +xfs_bmbt_init_key_from_rec( + union xfs_btree_key *key, + const union xfs_btree_rec *rec) +{ + key->bmbt.br_startoff = + cpu_to_be64(xfs_bmbt_disk_get_startoff(&rec->bmbt)); +} + +STATIC void +xfs_bmbt_init_high_key_from_rec( + union xfs_btree_key *key, + const union xfs_btree_rec *rec) +{ + key->bmbt.br_startoff = cpu_to_be64( + xfs_bmbt_disk_get_startoff(&rec->bmbt) + + xfs_bmbt_disk_get_blockcount(&rec->bmbt) - 1); +} + +STATIC void +xfs_bmbt_init_rec_from_cur( + struct xfs_btree_cur *cur, + union xfs_btree_rec *rec) +{ + xfs_bmbt_disk_set_all(&rec->bmbt, &cur->bc_rec.b); +} + +STATIC int +xfs_bmbt_cmp_key_with_cur( + struct xfs_btree_cur *cur, + const union xfs_btree_key *key) +{ + return cmp_int(be64_to_cpu(key->bmbt.br_startoff), + cur->bc_rec.b.br_startoff); +} + +STATIC int +xfs_bmbt_cmp_two_keys( + struct xfs_btree_cur *cur, + const union xfs_btree_key *k1, + const union xfs_btree_key *k2, + const union xfs_btree_key *mask) +{ + ASSERT(!mask || mask->bmbt.br_startoff); + + return cmp_int(be64_to_cpu(k1->bmbt.br_startoff), + be64_to_cpu(k2->bmbt.br_startoff)); +} + +static xfs_failaddr_t +xfs_bmbt_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_btree_block *block = XFS_BUF_TO_BLOCK(bp); + xfs_failaddr_t fa; + unsigned int level; + + if (!xfs_verify_magic(bp, block->bb_magic)) + return __this_address; + + if (xfs_has_crc(mp)) { + /* + * XXX: need a better way of verifying the owner here. Right now + * just make sure there has been one set. + */ + fa = xfs_btree_fsblock_v5hdr_verify(bp, XFS_RMAP_OWN_UNKNOWN); + if (fa) + return fa; + } + + /* + * numrecs and level verification. + * + * We don't know what fork we belong to, so just verify that the level + * is less than the maximum of the two. Later checks will be more + * precise. + */ + level = be16_to_cpu(block->bb_level); + if (level > max(mp->m_bm_maxlevels[0], mp->m_bm_maxlevels[1])) + return __this_address; + + return xfs_btree_fsblock_verify(bp, mp->m_bmap_dmxr[level != 0]); +} + +static void +xfs_bmbt_read_verify( + struct xfs_buf *bp) +{ + xfs_failaddr_t fa; + + if (!xfs_btree_fsblock_verify_crc(bp)) + xfs_verifier_error(bp, -EFSBADCRC, __this_address); + else { + fa = xfs_bmbt_verify(bp); + if (fa) + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + } + + if (bp->b_error) + trace_xfs_btree_corrupt(bp, _RET_IP_); +} + +static void +xfs_bmbt_write_verify( + struct xfs_buf *bp) +{ + xfs_failaddr_t fa; + + fa = xfs_bmbt_verify(bp); + if (fa) { + trace_xfs_btree_corrupt(bp, _RET_IP_); + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + return; + } + xfs_btree_fsblock_calc_crc(bp); +} + +const struct xfs_buf_ops xfs_bmbt_buf_ops = { + .name = "xfs_bmbt", + .magic = { cpu_to_be32(XFS_BMAP_MAGIC), + cpu_to_be32(XFS_BMAP_CRC_MAGIC) }, + .verify_read = xfs_bmbt_read_verify, + .verify_write = xfs_bmbt_write_verify, + .verify_struct = xfs_bmbt_verify, +}; + + +STATIC int +xfs_bmbt_keys_inorder( + struct xfs_btree_cur *cur, + const union xfs_btree_key *k1, + const union xfs_btree_key *k2) +{ + return be64_to_cpu(k1->bmbt.br_startoff) < + be64_to_cpu(k2->bmbt.br_startoff); +} + +STATIC int +xfs_bmbt_recs_inorder( + struct xfs_btree_cur *cur, + const union xfs_btree_rec *r1, + const union xfs_btree_rec *r2) +{ + return xfs_bmbt_disk_get_startoff(&r1->bmbt) + + xfs_bmbt_disk_get_blockcount(&r1->bmbt) <= + xfs_bmbt_disk_get_startoff(&r2->bmbt); +} + +STATIC enum xbtree_key_contig +xfs_bmbt_keys_contiguous( + struct xfs_btree_cur *cur, + const union xfs_btree_key *key1, + const union xfs_btree_key *key2, + const union xfs_btree_key *mask) +{ + ASSERT(!mask || mask->bmbt.br_startoff); + + return xbtree_key_contig(be64_to_cpu(key1->bmbt.br_startoff), + be64_to_cpu(key2->bmbt.br_startoff)); +} + +static inline void +xfs_bmbt_move_ptrs( + struct xfs_mount *mp, + struct xfs_btree_block *broot, + short old_size, + size_t new_size, + unsigned int numrecs) +{ + void *dptr; + void *sptr; + + sptr = xfs_bmap_broot_ptr_addr(mp, broot, 1, old_size); + dptr = xfs_bmap_broot_ptr_addr(mp, broot, 1, new_size); + memmove(dptr, sptr, numrecs * sizeof(xfs_bmbt_ptr_t)); +} + +/* + * Reallocate the space for if_broot based on the number of records. Move the + * records and pointers in if_broot to fit the new size. When shrinking this + * will eliminate holes between the records and pointers created by the caller. + * When growing this will create holes to be filled in by the caller. + * + * The caller must not request to add more records than would fit in the + * on-disk inode root. If the if_broot is currently NULL, then if we are + * adding records, one will be allocated. The caller must also not request + * that the number of records go below zero, although it can go to zero. + * + * ip -- the inode whose if_broot area is changing + * whichfork -- which inode fork to change + * new_numrecs -- the new number of records requested for the if_broot array + * + * Returns the incore btree root block. + */ +struct xfs_btree_block * +xfs_bmap_broot_realloc( + struct xfs_inode *ip, + int whichfork, + unsigned int new_numrecs) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + struct xfs_btree_block *broot; + unsigned int new_size; + unsigned int old_size = ifp->if_broot_bytes; + + /* + * Block mapping btrees do not support storing zero records; if this + * happens, the fork is being changed to FMT_EXTENTS. Free the broot + * and get out. + */ + if (new_numrecs == 0) + return xfs_broot_realloc(ifp, 0); + + new_size = xfs_bmap_broot_space_calc(mp, new_numrecs); + + /* Handle the nop case quietly. */ + if (new_size == old_size) + return ifp->if_broot; + + if (new_size > old_size) { + unsigned int old_numrecs; + + /* + * If there wasn't any memory allocated before, just + * allocate it now and get out. + */ + if (old_size == 0) + return xfs_broot_realloc(ifp, new_size); + + /* + * If there is already an existing if_broot, then we need + * to realloc() it and shift the pointers to their new + * location. The records don't change location because + * they are kept butted up against the btree block header. + */ + old_numrecs = xfs_bmbt_maxrecs(mp, old_size, false); + broot = xfs_broot_realloc(ifp, new_size); + ASSERT(xfs_bmap_bmdr_space(broot) <= + xfs_inode_fork_size(ip, whichfork)); + xfs_bmbt_move_ptrs(mp, broot, old_size, new_size, old_numrecs); + return broot; + } + + /* + * We're reducing, but not totally eliminating, numrecs. In this case, + * we are shrinking the if_broot buffer, so it must already exist. + */ + ASSERT(ifp->if_broot != NULL && old_size > 0 && new_size > 0); + + /* + * Shrink the btree root by moving the bmbt pointers, since they are + * not butted up against the btree block header, then reallocating + * broot. + */ + xfs_bmbt_move_ptrs(mp, ifp->if_broot, old_size, new_size, new_numrecs); + broot = xfs_broot_realloc(ifp, new_size); + ASSERT(xfs_bmap_bmdr_space(broot) <= + xfs_inode_fork_size(ip, whichfork)); + return broot; +} + +static struct xfs_btree_block * +xfs_bmbt_broot_realloc( + struct xfs_btree_cur *cur, + unsigned int new_numrecs) +{ + return xfs_bmap_broot_realloc(cur->bc_ino.ip, cur->bc_ino.whichfork, + new_numrecs); +} + +const struct xfs_btree_ops xfs_bmbt_ops = { + .name = "bmap", + .type = XFS_BTREE_TYPE_INODE, + + .rec_len = sizeof(xfs_bmbt_rec_t), + .key_len = sizeof(xfs_bmbt_key_t), + .ptr_len = XFS_BTREE_LONG_PTR_LEN, + + .lru_refs = XFS_BMAP_BTREE_REF, + .statoff = XFS_STATS_CALC_INDEX(xs_bmbt_2), + + .dup_cursor = xfs_bmbt_dup_cursor, + .update_cursor = xfs_bmbt_update_cursor, + .alloc_block = xfs_bmbt_alloc_block, + .free_block = xfs_bmbt_free_block, + .get_maxrecs = xfs_bmbt_get_maxrecs, + .get_minrecs = xfs_bmbt_get_minrecs, + .get_dmaxrecs = xfs_bmbt_get_dmaxrecs, + .init_key_from_rec = xfs_bmbt_init_key_from_rec, + .init_high_key_from_rec = xfs_bmbt_init_high_key_from_rec, + .init_rec_from_cur = xfs_bmbt_init_rec_from_cur, + .cmp_key_with_cur = xfs_bmbt_cmp_key_with_cur, + .cmp_two_keys = xfs_bmbt_cmp_two_keys, + .buf_ops = &xfs_bmbt_buf_ops, + .keys_inorder = xfs_bmbt_keys_inorder, + .recs_inorder = xfs_bmbt_recs_inorder, + .keys_contiguous = xfs_bmbt_keys_contiguous, + .broot_realloc = xfs_bmbt_broot_realloc, +}; + +/* + * Create a new bmap btree cursor. + * + * For staging cursors -1 in passed in whichfork. + */ +struct xfs_btree_cur * +xfs_bmbt_init_cursor( + struct xfs_mount *mp, + struct xfs_trans *tp, + struct xfs_inode *ip, + int whichfork) +{ + struct xfs_btree_cur *cur; + unsigned int maxlevels; + + ASSERT(whichfork != XFS_COW_FORK); + + /* + * The Data fork always has larger maxlevel, so use that for staging + * cursors. + */ + switch (whichfork) { + case XFS_STAGING_FORK: + maxlevels = mp->m_bm_maxlevels[XFS_DATA_FORK]; + break; + default: + maxlevels = mp->m_bm_maxlevels[whichfork]; + break; + } + cur = xfs_btree_alloc_cursor(mp, tp, &xfs_bmbt_ops, maxlevels, + xfs_bmbt_cur_cache); + cur->bc_ino.ip = ip; + cur->bc_ino.whichfork = whichfork; + cur->bc_bmap.allocated = 0; + if (whichfork != XFS_STAGING_FORK) { + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + + cur->bc_nlevels = be16_to_cpu(ifp->if_broot->bb_level) + 1; + cur->bc_ino.forksize = xfs_inode_fork_size(ip, whichfork); + } + return cur; +} + +/* Calculate number of records in a block mapping btree block. */ +static inline unsigned int +xfs_bmbt_block_maxrecs( + unsigned int blocklen, + bool leaf) +{ + if (leaf) + return blocklen / sizeof(xfs_bmbt_rec_t); + return blocklen / (sizeof(xfs_bmbt_key_t) + sizeof(xfs_bmbt_ptr_t)); +} + +/* + * Swap in the new inode fork root. Once we pass this point the newly rebuilt + * mappings are in place and we have to kill off any old btree blocks. + */ +void +xfs_bmbt_commit_staged_btree( + struct xfs_btree_cur *cur, + struct xfs_trans *tp, + int whichfork) +{ + struct xbtree_ifakeroot *ifake = cur->bc_ino.ifake; + struct xfs_ifork *ifp; + static const short brootflag[2] = {XFS_ILOG_DBROOT, XFS_ILOG_ABROOT}; + static const short extflag[2] = {XFS_ILOG_DEXT, XFS_ILOG_AEXT}; + int flags = XFS_ILOG_CORE; + + ASSERT(cur->bc_flags & XFS_BTREE_STAGING); + ASSERT(whichfork != XFS_COW_FORK); + + /* + * Free any resources hanging off the real fork, then shallow-copy the + * staging fork's contents into the real fork to transfer everything + * we just built. + */ + ifp = xfs_ifork_ptr(cur->bc_ino.ip, whichfork); + xfs_idestroy_fork(ifp); + memcpy(ifp, ifake->if_fork, sizeof(struct xfs_ifork)); + + switch (ifp->if_format) { + case XFS_DINODE_FMT_EXTENTS: + flags |= extflag[whichfork]; + break; + case XFS_DINODE_FMT_BTREE: + flags |= brootflag[whichfork]; + break; + default: + ASSERT(0); + break; + } + xfs_trans_log_inode(tp, cur->bc_ino.ip, flags); + xfs_btree_commit_ifakeroot(cur, tp, whichfork); +} + +/* + * Calculate number of records in a bmap btree block. + */ +unsigned int +xfs_bmbt_maxrecs( + struct xfs_mount *mp, + unsigned int blocklen, + bool leaf) +{ + blocklen -= xfs_bmbt_block_len(mp); + return xfs_bmbt_block_maxrecs(blocklen, leaf); +} + +/* + * Calculate the maximum possible height of the btree that the on-disk format + * supports. This is used for sizing structures large enough to support every + * possible configuration of a filesystem that might get mounted. + */ +unsigned int +xfs_bmbt_maxlevels_ondisk(void) +{ + unsigned int minrecs[2]; + unsigned int blocklen; + + blocklen = min(XFS_MIN_BLOCKSIZE - XFS_BTREE_SBLOCK_LEN, + XFS_MIN_CRC_BLOCKSIZE - XFS_BTREE_SBLOCK_CRC_LEN); + + minrecs[0] = xfs_bmbt_block_maxrecs(blocklen, true) / 2; + minrecs[1] = xfs_bmbt_block_maxrecs(blocklen, false) / 2; + + /* One extra level for the inode root. */ + return xfs_btree_compute_maxlevels(minrecs, + XFS_MAX_EXTCNT_DATA_FORK_LARGE) + 1; +} + +/* + * Calculate number of records in a bmap btree inode root. + */ +int +xfs_bmdr_maxrecs( + int blocklen, + int leaf) +{ + blocklen -= sizeof(xfs_bmdr_block_t); + + if (leaf) + return blocklen / sizeof(xfs_bmdr_rec_t); + return blocklen / (sizeof(xfs_bmdr_key_t) + sizeof(xfs_bmdr_ptr_t)); +} + +/* + * Change the owner of a btree format fork fo the inode passed in. Change it to + * the owner of that is passed in so that we can change owners before or after + * we switch forks between inodes. The operation that the caller is doing will + * determine whether is needs to change owner before or after the switch. + * + * For demand paged transactional modification, the fork switch should be done + * after reading in all the blocks, modifying them and pinning them in the + * transaction. For modification when the buffers are already pinned in memory, + * the fork switch can be done before changing the owner as we won't need to + * validate the owner until the btree buffers are unpinned and writes can occur + * again. + * + * For recovery based ownership change, there is no transactional context and + * so a buffer list must be supplied so that we can record the buffers that we + * modified for the caller to issue IO on. + */ +int +xfs_bmbt_change_owner( + struct xfs_trans *tp, + struct xfs_inode *ip, + int whichfork, + xfs_ino_t new_owner, + struct list_head *buffer_list) +{ + struct xfs_btree_cur *cur; + int error; + + ASSERT(tp || buffer_list); + ASSERT(!(tp && buffer_list)); + ASSERT(xfs_ifork_ptr(ip, whichfork)->if_format == XFS_DINODE_FMT_BTREE); + + cur = xfs_bmbt_init_cursor(ip->i_mount, tp, ip, whichfork); + cur->bc_flags |= XFS_BTREE_BMBT_INVALID_OWNER; + + error = xfs_btree_change_owner(cur, new_owner, buffer_list); + xfs_btree_del_cursor(cur, error); + return error; +} + +/* Calculate the bmap btree size for some records. */ +unsigned long long +xfs_bmbt_calc_size( + struct xfs_mount *mp, + unsigned long long len) +{ + return xfs_btree_calc_size(mp->m_bmap_dmnr, len); +} + +int __init +xfs_bmbt_init_cur_cache(void) +{ + xfs_bmbt_cur_cache = kmem_cache_create("xfs_bmbt_cur", + xfs_btree_cur_sizeof(xfs_bmbt_maxlevels_ondisk()), + 0, 0, NULL); + + if (!xfs_bmbt_cur_cache) + return -ENOMEM; + return 0; +} + +void +xfs_bmbt_destroy_cur_cache(void) +{ + kmem_cache_destroy(xfs_bmbt_cur_cache); + xfs_bmbt_cur_cache = NULL; +} diff --git a/fs/xfs/libxfs/xfs_bmap_btree.h b/fs/xfs/libxfs/xfs_bmap_btree.h new file mode 100644 index 000000000..e0c870bea --- /dev/null +++ b/fs/xfs/libxfs/xfs_bmap_btree.h @@ -0,0 +1,204 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2000,2002-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_BMAP_BTREE_H__ +#define __XFS_BMAP_BTREE_H__ + +struct xfs_btree_cur; +struct xfs_btree_block; +struct xfs_mount; +struct xfs_inode; +struct xfs_trans; +struct xbtree_ifakeroot; + +/* + * Maximum number of bmap btree levels. + */ +#define XFS_BM_MAXLEVELS(mp,w) ((mp)->m_bm_maxlevels[(w)]) + +/* + * Prototypes for xfs_bmap.c to call. + */ +extern void xfs_bmdr_to_bmbt(struct xfs_inode *, xfs_bmdr_block_t *, int, + struct xfs_btree_block *, int); + +void xfs_bmbt_disk_set_all(struct xfs_bmbt_rec *r, struct xfs_bmbt_irec *s); +extern xfs_filblks_t xfs_bmbt_disk_get_blockcount(const struct xfs_bmbt_rec *r); +extern xfs_fileoff_t xfs_bmbt_disk_get_startoff(const struct xfs_bmbt_rec *r); +void xfs_bmbt_disk_get_all(const struct xfs_bmbt_rec *r, + struct xfs_bmbt_irec *s); + +extern void xfs_bmbt_to_bmdr(struct xfs_mount *, struct xfs_btree_block *, int, + xfs_bmdr_block_t *, int); + +extern int xfs_bmbt_get_maxrecs(struct xfs_btree_cur *, int level); +extern int xfs_bmdr_maxrecs(int blocklen, int leaf); +unsigned int xfs_bmbt_maxrecs(struct xfs_mount *mp, unsigned int blocklen, + bool leaf); + +extern int xfs_bmbt_change_owner(struct xfs_trans *tp, struct xfs_inode *ip, + int whichfork, xfs_ino_t new_owner, + struct list_head *buffer_list); + +extern struct xfs_btree_cur *xfs_bmbt_init_cursor(struct xfs_mount *, + struct xfs_trans *, struct xfs_inode *, int); +void xfs_bmbt_commit_staged_btree(struct xfs_btree_cur *cur, + struct xfs_trans *tp, int whichfork); + +extern unsigned long long xfs_bmbt_calc_size(struct xfs_mount *mp, + unsigned long long len); + +unsigned int xfs_bmbt_maxlevels_ondisk(void); + +int __init xfs_bmbt_init_cur_cache(void); +void xfs_bmbt_destroy_cur_cache(void); + +void xfs_bmbt_init_block(struct xfs_inode *ip, struct xfs_btree_block *buf, + struct xfs_buf *bp, __u16 level, __u16 numrecs); + +/* + * Btree block header size depends on a superblock flag. + */ +static inline size_t +xfs_bmbt_block_len(struct xfs_mount *mp) +{ + return xfs_has_crc(mp) ? + XFS_BTREE_LBLOCK_CRC_LEN : XFS_BTREE_LBLOCK_LEN; +} + +/* Addresses of key, pointers, and records within an incore bmbt block. */ + +static inline struct xfs_bmbt_rec * +xfs_bmbt_rec_addr( + struct xfs_mount *mp, + struct xfs_btree_block *block, + unsigned int index) +{ + return (struct xfs_bmbt_rec *) + ((char *)block + xfs_bmbt_block_len(mp) + + (index - 1) * sizeof(struct xfs_bmbt_rec)); +} + +static inline struct xfs_bmbt_key * +xfs_bmbt_key_addr( + struct xfs_mount *mp, + struct xfs_btree_block *block, + unsigned int index) +{ + return (struct xfs_bmbt_key *) + ((char *)block + xfs_bmbt_block_len(mp) + + (index - 1) * sizeof(struct xfs_bmbt_key)); +} + +static inline xfs_bmbt_ptr_t * +xfs_bmbt_ptr_addr( + struct xfs_mount *mp, + struct xfs_btree_block *block, + unsigned int index, + unsigned int maxrecs) +{ + return (xfs_bmbt_ptr_t *) + ((char *)block + xfs_bmbt_block_len(mp) + + maxrecs * sizeof(struct xfs_bmbt_key) + + (index - 1) * sizeof(xfs_bmbt_ptr_t)); +} + +/* Addresses of key, pointers, and records within an ondisk bmbt block. */ + +static inline struct xfs_bmbt_rec * +xfs_bmdr_rec_addr( + struct xfs_bmdr_block *block, + unsigned int index) +{ + return (struct xfs_bmbt_rec *) + ((char *)(block + 1) + + (index - 1) * sizeof(struct xfs_bmbt_rec)); +} + +static inline struct xfs_bmbt_key * +xfs_bmdr_key_addr( + struct xfs_bmdr_block *block, + unsigned int index) +{ + return (struct xfs_bmbt_key *) + ((char *)(block + 1) + + (index - 1) * sizeof(struct xfs_bmbt_key)); +} + +static inline xfs_bmbt_ptr_t * +xfs_bmdr_ptr_addr( + struct xfs_bmdr_block *block, + unsigned int index, + unsigned int maxrecs) +{ + return (xfs_bmbt_ptr_t *) + ((char *)(block + 1) + + maxrecs * sizeof(struct xfs_bmbt_key) + + (index - 1) * sizeof(xfs_bmbt_ptr_t)); +} + +/* + * Address of pointers within the incore btree root. + * + * These are to be used when we know the size of the block and + * we don't have a cursor. + */ +static inline xfs_bmbt_ptr_t * +xfs_bmap_broot_ptr_addr( + struct xfs_mount *mp, + struct xfs_btree_block *bb, + unsigned int i, + unsigned int sz) +{ + return xfs_bmbt_ptr_addr(mp, bb, i, xfs_bmbt_maxrecs(mp, sz, false)); +} + +/* + * Compute the space required for the incore btree root containing the given + * number of records. + */ +static inline size_t +xfs_bmap_broot_space_calc( + struct xfs_mount *mp, + unsigned int nrecs) +{ + return xfs_bmbt_block_len(mp) + + (nrecs * (sizeof(struct xfs_bmbt_key) + sizeof(xfs_bmbt_ptr_t))); +} + +/* + * Compute the space required for the incore btree root given the ondisk + * btree root block. + */ +static inline size_t +xfs_bmap_broot_space( + struct xfs_mount *mp, + struct xfs_bmdr_block *bb) +{ + return xfs_bmap_broot_space_calc(mp, be16_to_cpu(bb->bb_numrecs)); +} + +/* Compute the space required for the ondisk root block. */ +static inline size_t +xfs_bmdr_space_calc(unsigned int nrecs) +{ + return sizeof(struct xfs_bmdr_block) + + (nrecs * (sizeof(struct xfs_bmbt_key) + sizeof(xfs_bmbt_ptr_t))); +} + +/* + * Compute the space required for the ondisk root block given an incore root + * block. + */ +static inline size_t +xfs_bmap_bmdr_space(struct xfs_btree_block *bb) +{ + return xfs_bmdr_space_calc(be16_to_cpu(bb->bb_numrecs)); +} + +struct xfs_btree_block *xfs_bmap_broot_realloc(struct xfs_inode *ip, + int whichfork, unsigned int new_numrecs); + +#endif /* __XFS_BMAP_BTREE_H__ */ diff --git a/fs/xfs/libxfs/xfs_btree.c b/fs/xfs/libxfs/xfs_btree.c new file mode 100644 index 000000000..60ef7f08b --- /dev/null +++ b/fs/xfs/libxfs/xfs_btree.c @@ -0,0 +1,5650 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_buf_item.h" +#include "xfs_btree.h" +#include "xfs_errortag.h" +#include "xfs_error.h" +#include "xfs_trace.h" +#include "xfs_alloc.h" +#include "xfs_log.h" +#include "xfs_btree_staging.h" +#include "xfs_ag.h" +#include "xfs_alloc_btree.h" +#include "xfs_ialloc_btree.h" +#include "xfs_bmap_btree.h" +#include "xfs_rmap_btree.h" +#include "xfs_refcount_btree.h" +#include "xfs_health.h" +#include "xfs_buf_mem.h" +#include "xfs_btree_mem.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_bmap.h" +#include "xfs_rmap.h" +#include "xfs_quota.h" +#include "xfs_metafile.h" +#include "xfs_rtrefcount_btree.h" + +/* + * Btree magic numbers. + */ +uint32_t +xfs_btree_magic( + struct xfs_mount *mp, + const struct xfs_btree_ops *ops) +{ + int idx = xfs_has_crc(mp) ? 1 : 0; + __be32 magic = ops->buf_ops->magic[idx]; + + /* Ensure we asked for crc for crc-only magics. */ + ASSERT(magic != 0); + return be32_to_cpu(magic); +} + +/* + * These sibling pointer checks are optimised for null sibling pointers. This + * happens a lot, and we don't need to byte swap at runtime if the sibling + * pointer is NULL. + * + * These are explicitly marked at inline because the cost of calling them as + * functions instead of inlining them is about 36 bytes extra code per call site + * on x86-64. Yes, gcc-11 fails to inline them, and explicit inlining of these + * two sibling check functions reduces the compiled code size by over 300 + * bytes. + */ +static inline xfs_failaddr_t +xfs_btree_check_fsblock_siblings( + struct xfs_mount *mp, + xfs_fsblock_t fsb, + __be64 dsibling) +{ + xfs_fsblock_t sibling; + + if (dsibling == cpu_to_be64(NULLFSBLOCK)) + return NULL; + + sibling = be64_to_cpu(dsibling); + if (sibling == fsb) + return __this_address; + if (!xfs_verify_fsbno(mp, sibling)) + return __this_address; + return NULL; +} + +static inline xfs_failaddr_t +xfs_btree_check_memblock_siblings( + struct xfs_buftarg *btp, + xfbno_t bno, + __be64 dsibling) +{ + xfbno_t sibling; + + if (dsibling == cpu_to_be64(NULLFSBLOCK)) + return NULL; + + sibling = be64_to_cpu(dsibling); + if (sibling == bno) + return __this_address; + if (!xmbuf_verify_daddr(btp, xfbno_to_daddr(sibling))) + return __this_address; + return NULL; +} + +static inline xfs_failaddr_t +xfs_btree_check_agblock_siblings( + struct xfs_perag *pag, + xfs_agblock_t agbno, + __be32 dsibling) +{ + xfs_agblock_t sibling; + + if (dsibling == cpu_to_be32(NULLAGBLOCK)) + return NULL; + + sibling = be32_to_cpu(dsibling); + if (sibling == agbno) + return __this_address; + if (!xfs_verify_agbno(pag, sibling)) + return __this_address; + return NULL; +} + +static xfs_failaddr_t +__xfs_btree_check_lblock_hdr( + struct xfs_btree_cur *cur, + struct xfs_btree_block *block, + int level, + struct xfs_buf *bp) +{ + struct xfs_mount *mp = cur->bc_mp; + + if (xfs_has_crc(mp)) { + if (!uuid_equal(&block->bb_u.l.bb_uuid, &mp->m_sb.sb_meta_uuid)) + return __this_address; + if (block->bb_u.l.bb_blkno != + cpu_to_be64(bp ? xfs_buf_daddr(bp) : XFS_BUF_DADDR_NULL)) + return __this_address; + if (block->bb_u.l.bb_pad != cpu_to_be32(0)) + return __this_address; + } + + if (be32_to_cpu(block->bb_magic) != xfs_btree_magic(mp, cur->bc_ops)) + return __this_address; + if (be16_to_cpu(block->bb_level) != level) + return __this_address; + if (be16_to_cpu(block->bb_numrecs) > + cur->bc_ops->get_maxrecs(cur, level)) + return __this_address; + + return NULL; +} + +/* + * Check a long btree block header. Return the address of the failing check, + * or NULL if everything is ok. + */ +static xfs_failaddr_t +__xfs_btree_check_fsblock( + struct xfs_btree_cur *cur, + struct xfs_btree_block *block, + int level, + struct xfs_buf *bp) +{ + struct xfs_mount *mp = cur->bc_mp; + xfs_failaddr_t fa; + xfs_fsblock_t fsb; + + fa = __xfs_btree_check_lblock_hdr(cur, block, level, bp); + if (fa) + return fa; + + /* + * For inode-rooted btrees, the root block sits in the inode fork. In + * that case bp is NULL, and the block must not have any siblings. + */ + if (!bp) { + if (block->bb_u.l.bb_leftsib != cpu_to_be64(NULLFSBLOCK)) + return __this_address; + if (block->bb_u.l.bb_rightsib != cpu_to_be64(NULLFSBLOCK)) + return __this_address; + return NULL; + } + + fsb = XFS_DADDR_TO_FSB(mp, xfs_buf_daddr(bp)); + fa = xfs_btree_check_fsblock_siblings(mp, fsb, + block->bb_u.l.bb_leftsib); + if (!fa) + fa = xfs_btree_check_fsblock_siblings(mp, fsb, + block->bb_u.l.bb_rightsib); + return fa; +} + +/* + * Check an in-memory btree block header. Return the address of the failing + * check, or NULL if everything is ok. + */ +static xfs_failaddr_t +__xfs_btree_check_memblock( + struct xfs_btree_cur *cur, + struct xfs_btree_block *block, + int level, + struct xfs_buf *bp) +{ + struct xfs_buftarg *btp = cur->bc_mem.xfbtree->target; + xfs_failaddr_t fa; + xfbno_t bno; + + fa = __xfs_btree_check_lblock_hdr(cur, block, level, bp); + if (fa) + return fa; + + bno = xfs_daddr_to_xfbno(xfs_buf_daddr(bp)); + fa = xfs_btree_check_memblock_siblings(btp, bno, + block->bb_u.l.bb_leftsib); + if (!fa) + fa = xfs_btree_check_memblock_siblings(btp, bno, + block->bb_u.l.bb_rightsib); + return fa; +} + +/* + * Check a short btree block header. Return the address of the failing check, + * or NULL if everything is ok. + */ +static xfs_failaddr_t +__xfs_btree_check_agblock( + struct xfs_btree_cur *cur, + struct xfs_btree_block *block, + int level, + struct xfs_buf *bp) +{ + struct xfs_mount *mp = cur->bc_mp; + struct xfs_perag *pag = to_perag(cur->bc_group); + xfs_failaddr_t fa; + xfs_agblock_t agbno; + + if (xfs_has_crc(mp)) { + if (!uuid_equal(&block->bb_u.s.bb_uuid, &mp->m_sb.sb_meta_uuid)) + return __this_address; + if (block->bb_u.s.bb_blkno != cpu_to_be64(xfs_buf_daddr(bp))) + return __this_address; + } + + if (be32_to_cpu(block->bb_magic) != xfs_btree_magic(mp, cur->bc_ops)) + return __this_address; + if (be16_to_cpu(block->bb_level) != level) + return __this_address; + if (be16_to_cpu(block->bb_numrecs) > + cur->bc_ops->get_maxrecs(cur, level)) + return __this_address; + + agbno = xfs_daddr_to_agbno(mp, xfs_buf_daddr(bp)); + fa = xfs_btree_check_agblock_siblings(pag, agbno, + block->bb_u.s.bb_leftsib); + if (!fa) + fa = xfs_btree_check_agblock_siblings(pag, agbno, + block->bb_u.s.bb_rightsib); + return fa; +} + +/* + * Internal btree block check. + * + * Return NULL if the block is ok or the address of the failed check otherwise. + */ +xfs_failaddr_t +__xfs_btree_check_block( + struct xfs_btree_cur *cur, + struct xfs_btree_block *block, + int level, + struct xfs_buf *bp) +{ + switch (cur->bc_ops->type) { + case XFS_BTREE_TYPE_MEM: + return __xfs_btree_check_memblock(cur, block, level, bp); + case XFS_BTREE_TYPE_AG: + return __xfs_btree_check_agblock(cur, block, level, bp); + case XFS_BTREE_TYPE_INODE: + return __xfs_btree_check_fsblock(cur, block, level, bp); + default: + ASSERT(0); + return __this_address; + } +} + +static inline unsigned int xfs_btree_block_errtag(struct xfs_btree_cur *cur) +{ + if (cur->bc_ops->ptr_len == XFS_BTREE_SHORT_PTR_LEN) + return XFS_ERRTAG_BTREE_CHECK_SBLOCK; + return XFS_ERRTAG_BTREE_CHECK_LBLOCK; +} + +/* + * Debug routine: check that block header is ok. + */ +int +xfs_btree_check_block( + struct xfs_btree_cur *cur, /* btree cursor */ + struct xfs_btree_block *block, /* generic btree block pointer */ + int level, /* level of the btree block */ + struct xfs_buf *bp) /* buffer containing block, if any */ +{ + struct xfs_mount *mp = cur->bc_mp; + xfs_failaddr_t fa; + + fa = __xfs_btree_check_block(cur, block, level, bp); + if (XFS_IS_CORRUPT(mp, fa != NULL) || + XFS_TEST_ERROR(mp, xfs_btree_block_errtag(cur))) { + if (bp) + trace_xfs_btree_corrupt(bp, _RET_IP_); + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + return 0; +} + +int +__xfs_btree_check_ptr( + struct xfs_btree_cur *cur, + const union xfs_btree_ptr *ptr, + int index, + int level) +{ + if (level <= 0) + return -EFSCORRUPTED; + + switch (cur->bc_ops->type) { + case XFS_BTREE_TYPE_MEM: + if (!xfbtree_verify_bno(cur->bc_mem.xfbtree, + be64_to_cpu((&ptr->l)[index]))) + return -EFSCORRUPTED; + break; + case XFS_BTREE_TYPE_INODE: + if (!xfs_verify_fsbno(cur->bc_mp, + be64_to_cpu((&ptr->l)[index]))) + return -EFSCORRUPTED; + break; + case XFS_BTREE_TYPE_AG: + if (!xfs_verify_agbno(to_perag(cur->bc_group), + be32_to_cpu((&ptr->s)[index]))) + return -EFSCORRUPTED; + break; + } + + return 0; +} + +/* + * Check that a given (indexed) btree pointer at a certain level of a + * btree is valid and doesn't point past where it should. + */ +static int +xfs_btree_check_ptr( + struct xfs_btree_cur *cur, + const union xfs_btree_ptr *ptr, + int index, + int level) +{ + int error; + + error = __xfs_btree_check_ptr(cur, ptr, index, level); + if (error) { + switch (cur->bc_ops->type) { + case XFS_BTREE_TYPE_MEM: + xfs_err(cur->bc_mp, +"In-memory: Corrupt %sbt flags 0x%x pointer at level %d index %d fa %pS.", + cur->bc_ops->name, cur->bc_flags, level, index, + __this_address); + break; + case XFS_BTREE_TYPE_INODE: + xfs_err(cur->bc_mp, +"Inode %llu fork %d: Corrupt %sbt pointer at level %d index %d.", + I_INO(cur->bc_ino.ip), + cur->bc_ino.whichfork, cur->bc_ops->name, + level, index); + break; + case XFS_BTREE_TYPE_AG: + xfs_err(cur->bc_mp, +"AG %u: Corrupt %sbt pointer at level %d index %d.", + cur->bc_group->xg_gno, cur->bc_ops->name, + level, index); + break; + } + xfs_btree_mark_sick(cur); + } + + return error; +} + +#ifdef DEBUG +# define xfs_btree_debug_check_ptr xfs_btree_check_ptr +#else +# define xfs_btree_debug_check_ptr(...) (0) +#endif + +/* + * Calculate CRC on the whole btree block and stuff it into the + * long-form btree header. + * + * Prior to calculting the CRC, pull the LSN out of the buffer log item and put + * it into the buffer so recovery knows what the last modification was that made + * it to disk. + */ +void +xfs_btree_fsblock_calc_crc( + struct xfs_buf *bp) +{ + struct xfs_btree_block *block = XFS_BUF_TO_BLOCK(bp); + struct xfs_buf_log_item *bip = bp->b_log_item; + + if (!xfs_has_crc(bp->b_mount)) + return; + if (bip) + block->bb_u.l.bb_lsn = cpu_to_be64(bip->bli_item.li_lsn); + xfs_buf_update_cksum(bp, XFS_BTREE_LBLOCK_CRC_OFF); +} + +bool +xfs_btree_fsblock_verify_crc( + struct xfs_buf *bp) +{ + struct xfs_btree_block *block = XFS_BUF_TO_BLOCK(bp); + struct xfs_mount *mp = bp->b_mount; + + if (xfs_has_crc(mp)) { + if (!xfs_log_check_lsn(mp, be64_to_cpu(block->bb_u.l.bb_lsn))) + return false; + return xfs_buf_verify_cksum(bp, XFS_BTREE_LBLOCK_CRC_OFF); + } + + return true; +} + +/* + * Calculate CRC on the whole btree block and stuff it into the + * short-form btree header. + * + * Prior to calculting the CRC, pull the LSN out of the buffer log item and put + * it into the buffer so recovery knows what the last modification was that made + * it to disk. + */ +void +xfs_btree_agblock_calc_crc( + struct xfs_buf *bp) +{ + struct xfs_btree_block *block = XFS_BUF_TO_BLOCK(bp); + struct xfs_buf_log_item *bip = bp->b_log_item; + + if (!xfs_has_crc(bp->b_mount)) + return; + if (bip) + block->bb_u.s.bb_lsn = cpu_to_be64(bip->bli_item.li_lsn); + xfs_buf_update_cksum(bp, XFS_BTREE_SBLOCK_CRC_OFF); +} + +bool +xfs_btree_agblock_verify_crc( + struct xfs_buf *bp) +{ + struct xfs_btree_block *block = XFS_BUF_TO_BLOCK(bp); + struct xfs_mount *mp = bp->b_mount; + + if (xfs_has_crc(mp)) { + if (!xfs_log_check_lsn(mp, be64_to_cpu(block->bb_u.s.bb_lsn))) + return false; + return xfs_buf_verify_cksum(bp, XFS_BTREE_SBLOCK_CRC_OFF); + } + + return true; +} + +static int +xfs_btree_free_block( + struct xfs_btree_cur *cur, + struct xfs_buf *bp) +{ + int error; + + trace_xfs_btree_free_block(cur, bp); + + /* + * Don't allow block freeing for a staging cursor, because staging + * cursors do not support regular btree modifications. + */ + if (unlikely(cur->bc_flags & XFS_BTREE_STAGING)) { + ASSERT(0); + return -EFSCORRUPTED; + } + + error = cur->bc_ops->free_block(cur, bp); + if (!error) { + xfs_trans_binval(cur->bc_tp, bp); + XFS_BTREE_STATS_INC(cur, free); + } + return error; +} + +/* + * Delete the btree cursor. + */ +void +xfs_btree_del_cursor( + struct xfs_btree_cur *cur, /* btree cursor */ + int error) /* del because of error */ +{ + int i; /* btree level */ + + /* + * Clear the buffer pointers and release the buffers. If we're doing + * this because of an error, inspect all of the entries in the bc_bufs + * array for buffers to be unlocked. This is because some of the btree + * code works from level n down to 0, and if we get an error along the + * way we won't have initialized all the entries down to 0. + */ + for (i = 0; i < cur->bc_nlevels; i++) { + if (cur->bc_levels[i].bp) + xfs_trans_brelse(cur->bc_tp, cur->bc_levels[i].bp); + else if (!error) + break; + } + + /* + * If we are doing a BMBT update, the number of unaccounted blocks + * allocated during this cursor life time should be zero. If it's not + * zero, then we should be shut down or on our way to shutdown due to + * cancelling a dirty transaction on error. + */ + ASSERT(!xfs_btree_is_bmap(cur->bc_ops) || cur->bc_bmap.allocated == 0 || + xfs_is_shutdown(cur->bc_mp) || error != 0); + + if (cur->bc_group) + xfs_group_put(cur->bc_group); + kmem_cache_free(cur->bc_cache, cur); +} + +/* Return the buffer target for this btree's buffer. */ +static inline struct xfs_buftarg * +xfs_btree_buftarg( + struct xfs_btree_cur *cur) +{ + if (cur->bc_ops->type == XFS_BTREE_TYPE_MEM) + return cur->bc_mem.xfbtree->target; + return cur->bc_mp->m_ddev_targp; +} + +/* Return the block size (in units of 512b sectors) for this btree. */ +static inline unsigned int +xfs_btree_bbsize( + struct xfs_btree_cur *cur) +{ + if (cur->bc_ops->type == XFS_BTREE_TYPE_MEM) + return XFBNO_BBSIZE; + return cur->bc_mp->m_bsize; +} + +/* + * Duplicate the btree cursor. + * Allocate a new one, copy the record, re-get the buffers. + */ +int /* error */ +xfs_btree_dup_cursor( + struct xfs_btree_cur *cur, /* input cursor */ + struct xfs_btree_cur **ncur) /* output cursor */ +{ + struct xfs_mount *mp = cur->bc_mp; + struct xfs_trans *tp = cur->bc_tp; + struct xfs_buf *bp; + struct xfs_btree_cur *new; + int error; + int i; + + /* + * Don't allow staging cursors to be duplicated because they're supposed + * to be kept private to a single thread. + */ + if (unlikely(cur->bc_flags & XFS_BTREE_STAGING)) { + ASSERT(0); + return -EFSCORRUPTED; + } + + /* + * Allocate a new cursor like the old one. + */ + new = cur->bc_ops->dup_cursor(cur); + + /* + * Copy the record currently in the cursor. + */ + new->bc_rec = cur->bc_rec; + + /* + * For each level current, re-get the buffer and copy the ptr value. + */ + for (i = 0; i < new->bc_nlevels; i++) { + new->bc_levels[i].ptr = cur->bc_levels[i].ptr; + new->bc_levels[i].ra = cur->bc_levels[i].ra; + bp = cur->bc_levels[i].bp; + if (bp) { + error = xfs_trans_read_buf(mp, tp, + xfs_btree_buftarg(cur), + xfs_buf_daddr(bp), + xfs_btree_bbsize(cur), 0, &bp, + cur->bc_ops->buf_ops); + if (xfs_metadata_is_sick(error)) + xfs_btree_mark_sick(new); + if (error) { + xfs_btree_del_cursor(new, error); + *ncur = NULL; + return error; + } + } + new->bc_levels[i].bp = bp; + } + *ncur = new; + return 0; +} + +/* + * XFS btree block layout and addressing: + * + * There are two types of blocks in the btree: leaf and non-leaf blocks. + * + * The leaf record start with a header then followed by records containing + * the values. A non-leaf block also starts with the same header, and + * then first contains lookup keys followed by an equal number of pointers + * to the btree blocks at the previous level. + * + * +--------+-------+-------+-------+-------+-------+-------+ + * Leaf: | header | rec 1 | rec 2 | rec 3 | rec 4 | rec 5 | rec N | + * +--------+-------+-------+-------+-------+-------+-------+ + * + * +--------+-------+-------+-------+-------+-------+-------+ + * Non-Leaf: | header | key 1 | key 2 | key N | ptr 1 | ptr 2 | ptr N | + * +--------+-------+-------+-------+-------+-------+-------+ + * + * The header is called struct xfs_btree_block for reasons better left unknown + * and comes in different versions for short (32bit) and long (64bit) block + * pointers. The record and key structures are defined by the btree instances + * and opaque to the btree core. The block pointers are simple disk endian + * integers, available in a short (32bit) and long (64bit) variant. + * + * The helpers below calculate the offset of a given record, key or pointer + * into a btree block (xfs_btree_*_offset) or return a pointer to the given + * record, key or pointer (xfs_btree_*_addr). Note that all addressing + * inside the btree block is done using indices starting at one, not zero! + * + * If XFS_BTGEO_OVERLAPPING is set, then this btree supports keys containing + * overlapping intervals. In such a tree, records are still sorted lowest to + * highest and indexed by the smallest key value that refers to the record. + * However, nodes are different: each pointer has two associated keys -- one + * indexing the lowest key available in the block(s) below (the same behavior + * as the key in a regular btree) and another indexing the highest key + * available in the block(s) below. Because records are /not/ sorted by the + * highest key, all leaf block updates require us to compute the highest key + * that matches any record in the leaf and to recursively update the high keys + * in the nodes going further up in the tree, if necessary. Nodes look like + * this: + * + * +--------+-----+-----+-----+-----+-----+-------+-------+-----+ + * Non-Leaf: | header | lo1 | hi1 | lo2 | hi2 | ... | ptr 1 | ptr 2 | ... | + * +--------+-----+-----+-----+-----+-----+-------+-------+-----+ + * + * To perform an interval query on an overlapped tree, perform the usual + * depth-first search and use the low and high keys to decide if we can skip + * that particular node. If a leaf node is reached, return the records that + * intersect the interval. Note that an interval query may return numerous + * entries. For a non-overlapped tree, simply search for the record associated + * with the lowest key and iterate forward until a non-matching record is + * found. Section 14.3 ("Interval Trees") of _Introduction to Algorithms_ by + * Cormen, Leiserson, Rivest, and Stein (2nd or 3rd ed. only) discuss this in + * more detail. + * + * Why do we care about overlapping intervals? Let's say you have a bunch of + * reverse mapping records on a reflink filesystem: + * + * 1: +- file A startblock B offset C length D -----------+ + * 2: +- file E startblock F offset G length H --------------+ + * 3: +- file I startblock F offset J length K --+ + * 4: +- file L... --+ + * + * Now say we want to map block (B+D) into file A at offset (C+D). Ideally, + * we'd simply increment the length of record 1. But how do we find the record + * that ends at (B+D-1) (i.e. record 1)? A LE lookup of (B+D-1) would return + * record 3 because the keys are ordered first by startblock. An interval + * query would return records 1 and 2 because they both overlap (B+D-1), and + * from that we can pick out record 1 as the appropriate left neighbor. + * + * In the non-overlapped case you can do a LE lookup and decrement the cursor + * because a record's interval must end before the next record. + */ + +/* + * Return size of the btree block header for this btree instance. + */ +static inline size_t xfs_btree_block_len(struct xfs_btree_cur *cur) +{ + if (cur->bc_ops->ptr_len == XFS_BTREE_LONG_PTR_LEN) { + if (xfs_has_crc(cur->bc_mp)) + return XFS_BTREE_LBLOCK_CRC_LEN; + return XFS_BTREE_LBLOCK_LEN; + } + if (xfs_has_crc(cur->bc_mp)) + return XFS_BTREE_SBLOCK_CRC_LEN; + return XFS_BTREE_SBLOCK_LEN; +} + +/* + * Calculate offset of the n-th record in a btree block. + */ +STATIC size_t +xfs_btree_rec_offset( + struct xfs_btree_cur *cur, + int n) +{ + return xfs_btree_block_len(cur) + + (n - 1) * cur->bc_ops->rec_len; +} + +/* + * Calculate offset of the n-th key in a btree block. + */ +STATIC size_t +xfs_btree_key_offset( + struct xfs_btree_cur *cur, + int n) +{ + return xfs_btree_block_len(cur) + + (n - 1) * cur->bc_ops->key_len; +} + +/* + * Calculate offset of the n-th high key in a btree block. + */ +STATIC size_t +xfs_btree_high_key_offset( + struct xfs_btree_cur *cur, + int n) +{ + return xfs_btree_block_len(cur) + + (n - 1) * cur->bc_ops->key_len + (cur->bc_ops->key_len / 2); +} + +/* + * Calculate offset of the n-th block pointer in a btree block. + */ +STATIC size_t +xfs_btree_ptr_offset( + struct xfs_btree_cur *cur, + int n, + int level) +{ + return xfs_btree_block_len(cur) + + cur->bc_ops->get_maxrecs(cur, level) * cur->bc_ops->key_len + + (n - 1) * cur->bc_ops->ptr_len; +} + +/* + * Return a pointer to the n-th record in the btree block. + */ +union xfs_btree_rec * +xfs_btree_rec_addr( + struct xfs_btree_cur *cur, + int n, + struct xfs_btree_block *block) +{ + return (union xfs_btree_rec *) + ((char *)block + xfs_btree_rec_offset(cur, n)); +} + +/* + * Return a pointer to the n-th key in the btree block. + */ +union xfs_btree_key * +xfs_btree_key_addr( + struct xfs_btree_cur *cur, + int n, + struct xfs_btree_block *block) +{ + return (union xfs_btree_key *) + ((char *)block + xfs_btree_key_offset(cur, n)); +} + +/* + * Return a pointer to the n-th high key in the btree block. + */ +union xfs_btree_key * +xfs_btree_high_key_addr( + struct xfs_btree_cur *cur, + int n, + struct xfs_btree_block *block) +{ + return (union xfs_btree_key *) + ((char *)block + xfs_btree_high_key_offset(cur, n)); +} + +/* + * Return a pointer to the n-th block pointer in the btree block. + */ +union xfs_btree_ptr * +xfs_btree_ptr_addr( + struct xfs_btree_cur *cur, + int n, + struct xfs_btree_block *block) +{ + int level = xfs_btree_get_level(block); + + ASSERT(block->bb_level != 0); + + return (union xfs_btree_ptr *) + ((char *)block + xfs_btree_ptr_offset(cur, n, level)); +} + +struct xfs_ifork * +xfs_btree_ifork_ptr( + struct xfs_btree_cur *cur) +{ + ASSERT(cur->bc_ops->type == XFS_BTREE_TYPE_INODE); + + if (cur->bc_flags & XFS_BTREE_STAGING) + return cur->bc_ino.ifake->if_fork; + return xfs_ifork_ptr(cur->bc_ino.ip, cur->bc_ino.whichfork); +} + +/* + * Get the root block which is stored in the inode. + * + * For now this btree implementation assumes the btree root is always + * stored in the if_broot field of an inode fork. + */ +STATIC struct xfs_btree_block * +xfs_btree_get_iroot( + struct xfs_btree_cur *cur) +{ + struct xfs_ifork *ifp = xfs_btree_ifork_ptr(cur); + + return (struct xfs_btree_block *)ifp->if_broot; +} + +/* + * Retrieve the block pointer from the cursor at the given level. + * This may be an inode btree root or from a buffer. + */ +struct xfs_btree_block * /* generic btree block pointer */ +xfs_btree_get_block( + struct xfs_btree_cur *cur, /* btree cursor */ + int level, /* level in btree */ + struct xfs_buf **bpp) /* buffer containing the block */ +{ + if (xfs_btree_at_iroot(cur, level)) { + *bpp = NULL; + return xfs_btree_get_iroot(cur); + } + + *bpp = cur->bc_levels[level].bp; + return XFS_BUF_TO_BLOCK(*bpp); +} + +/* + * Change the cursor to point to the first record at the given level. + * Other levels are unaffected. + */ +STATIC int /* success=1, failure=0 */ +xfs_btree_firstrec( + struct xfs_btree_cur *cur, /* btree cursor */ + int level) /* level to change */ +{ + struct xfs_btree_block *block; /* generic btree block pointer */ + struct xfs_buf *bp; /* buffer containing block */ + + /* + * Get the block pointer for this level. + */ + block = xfs_btree_get_block(cur, level, &bp); + if (xfs_btree_check_block(cur, block, level, bp)) + return 0; + /* + * It's empty, there is no such record. + */ + if (!block->bb_numrecs) + return 0; + /* + * Set the ptr value to 1, that's the first record/key. + */ + cur->bc_levels[level].ptr = 1; + return 1; +} + +/* + * Change the cursor to point to the last record in the current block + * at the given level. Other levels are unaffected. + */ +STATIC int /* success=1, failure=0 */ +xfs_btree_lastrec( + struct xfs_btree_cur *cur, /* btree cursor */ + int level) /* level to change */ +{ + struct xfs_btree_block *block; /* generic btree block pointer */ + struct xfs_buf *bp; /* buffer containing block */ + + /* + * Get the block pointer for this level. + */ + block = xfs_btree_get_block(cur, level, &bp); + if (xfs_btree_check_block(cur, block, level, bp)) + return 0; + /* + * It's empty, there is no such record. + */ + if (!block->bb_numrecs) + return 0; + /* + * Set the ptr value to numrecs, that's the last record/key. + */ + cur->bc_levels[level].ptr = be16_to_cpu(block->bb_numrecs); + return 1; +} + +/* + * Compute first and last byte offsets for the fields given. + * Interprets the offsets table, which contains struct field offsets. + */ +void +xfs_btree_offsets( + uint32_t fields, /* bitmask of fields */ + const short *offsets, /* table of field offsets */ + int nbits, /* number of bits to inspect */ + int *first, /* output: first byte offset */ + int *last) /* output: last byte offset */ +{ + int i; /* current bit number */ + uint32_t imask; /* mask for current bit number */ + + ASSERT(fields != 0); + /* + * Find the lowest bit, so the first byte offset. + */ + for (i = 0, imask = 1u; ; i++, imask <<= 1) { + if (imask & fields) { + *first = offsets[i]; + break; + } + } + /* + * Find the highest bit, so the last byte offset. + */ + for (i = nbits - 1, imask = 1u << i; ; i--, imask >>= 1) { + if (imask & fields) { + *last = offsets[i + 1] - 1; + break; + } + } +} + +STATIC int +xfs_btree_readahead_fsblock( + struct xfs_btree_cur *cur, + int lr, + struct xfs_btree_block *block) +{ + struct xfs_mount *mp = cur->bc_mp; + xfs_fsblock_t left = be64_to_cpu(block->bb_u.l.bb_leftsib); + xfs_fsblock_t right = be64_to_cpu(block->bb_u.l.bb_rightsib); + int rval = 0; + + if ((lr & XFS_BTCUR_LEFTRA) && left != NULLFSBLOCK) { + xfs_buf_readahead(mp->m_ddev_targp, XFS_FSB_TO_DADDR(mp, left), + mp->m_bsize, cur->bc_ops->buf_ops); + rval++; + } + + if ((lr & XFS_BTCUR_RIGHTRA) && right != NULLFSBLOCK) { + xfs_buf_readahead(mp->m_ddev_targp, XFS_FSB_TO_DADDR(mp, right), + mp->m_bsize, cur->bc_ops->buf_ops); + rval++; + } + + return rval; +} + +STATIC int +xfs_btree_readahead_memblock( + struct xfs_btree_cur *cur, + int lr, + struct xfs_btree_block *block) +{ + struct xfs_buftarg *btp = cur->bc_mem.xfbtree->target; + xfbno_t left = be64_to_cpu(block->bb_u.l.bb_leftsib); + xfbno_t right = be64_to_cpu(block->bb_u.l.bb_rightsib); + int rval = 0; + + if ((lr & XFS_BTCUR_LEFTRA) && left != NULLFSBLOCK) { + xfs_buf_readahead(btp, xfbno_to_daddr(left), XFBNO_BBSIZE, + cur->bc_ops->buf_ops); + rval++; + } + + if ((lr & XFS_BTCUR_RIGHTRA) && right != NULLFSBLOCK) { + xfs_buf_readahead(btp, xfbno_to_daddr(right), XFBNO_BBSIZE, + cur->bc_ops->buf_ops); + rval++; + } + + return rval; +} + +STATIC int +xfs_btree_readahead_agblock( + struct xfs_btree_cur *cur, + int lr, + struct xfs_btree_block *block) +{ + struct xfs_mount *mp = cur->bc_mp; + struct xfs_perag *pag = to_perag(cur->bc_group); + xfs_agblock_t left = be32_to_cpu(block->bb_u.s.bb_leftsib); + xfs_agblock_t right = be32_to_cpu(block->bb_u.s.bb_rightsib); + int rval = 0; + + if ((lr & XFS_BTCUR_LEFTRA) && left != NULLAGBLOCK) { + xfs_buf_readahead(mp->m_ddev_targp, + xfs_agbno_to_daddr(pag, left), mp->m_bsize, + cur->bc_ops->buf_ops); + rval++; + } + + if ((lr & XFS_BTCUR_RIGHTRA) && right != NULLAGBLOCK) { + xfs_buf_readahead(mp->m_ddev_targp, + xfs_agbno_to_daddr(pag, right), mp->m_bsize, + cur->bc_ops->buf_ops); + rval++; + } + + return rval; +} + +/* + * Read-ahead btree blocks, at the given level. + * Bits in lr are set from XFS_BTCUR_{LEFT,RIGHT}RA. + */ +STATIC int +xfs_btree_readahead( + struct xfs_btree_cur *cur, /* btree cursor */ + int lev, /* level in btree */ + int lr) /* left/right bits */ +{ + struct xfs_btree_block *block; + + /* + * No readahead needed if we are at the root level and the + * btree root is stored in the inode. + */ + if (xfs_btree_at_iroot(cur, lev)) + return 0; + + if ((cur->bc_levels[lev].ra | lr) == cur->bc_levels[lev].ra) + return 0; + + cur->bc_levels[lev].ra |= lr; + block = XFS_BUF_TO_BLOCK(cur->bc_levels[lev].bp); + + switch (cur->bc_ops->type) { + case XFS_BTREE_TYPE_AG: + return xfs_btree_readahead_agblock(cur, lr, block); + case XFS_BTREE_TYPE_INODE: + return xfs_btree_readahead_fsblock(cur, lr, block); + case XFS_BTREE_TYPE_MEM: + return xfs_btree_readahead_memblock(cur, lr, block); + default: + ASSERT(0); + return 0; + } +} + +STATIC int +xfs_btree_ptr_to_daddr( + struct xfs_btree_cur *cur, + const union xfs_btree_ptr *ptr, + xfs_daddr_t *daddr) +{ + int error; + + error = xfs_btree_check_ptr(cur, ptr, 0, 1); + if (error) + return error; + + switch (cur->bc_ops->type) { + case XFS_BTREE_TYPE_AG: + *daddr = xfs_agbno_to_daddr(to_perag(cur->bc_group), + be32_to_cpu(ptr->s)); + break; + case XFS_BTREE_TYPE_INODE: + *daddr = XFS_FSB_TO_DADDR(cur->bc_mp, be64_to_cpu(ptr->l)); + break; + case XFS_BTREE_TYPE_MEM: + *daddr = xfbno_to_daddr(be64_to_cpu(ptr->l)); + break; + } + return 0; +} + +/* + * Readahead @count btree blocks at the given @ptr location. + * + * We don't need to care about long or short form btrees here as we have a + * method of converting the ptr directly to a daddr available to us. + */ +STATIC void +xfs_btree_readahead_ptr( + struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr, + xfs_extlen_t count) +{ + xfs_daddr_t daddr; + + if (xfs_btree_ptr_to_daddr(cur, ptr, &daddr)) + return; + xfs_buf_readahead(xfs_btree_buftarg(cur), daddr, + xfs_btree_bbsize(cur) * count, + cur->bc_ops->buf_ops); +} + +/* + * Set the buffer for level "lev" in the cursor to bp, releasing + * any previous buffer. + */ +STATIC void +xfs_btree_setbuf( + struct xfs_btree_cur *cur, /* btree cursor */ + int lev, /* level in btree */ + struct xfs_buf *bp) /* new buffer to set */ +{ + struct xfs_btree_block *b; /* btree block */ + + if (cur->bc_levels[lev].bp) + xfs_trans_brelse(cur->bc_tp, cur->bc_levels[lev].bp); + cur->bc_levels[lev].bp = bp; + cur->bc_levels[lev].ra = 0; + + b = XFS_BUF_TO_BLOCK(bp); + if (cur->bc_ops->ptr_len == XFS_BTREE_LONG_PTR_LEN) { + if (b->bb_u.l.bb_leftsib == cpu_to_be64(NULLFSBLOCK)) + cur->bc_levels[lev].ra |= XFS_BTCUR_LEFTRA; + if (b->bb_u.l.bb_rightsib == cpu_to_be64(NULLFSBLOCK)) + cur->bc_levels[lev].ra |= XFS_BTCUR_RIGHTRA; + } else { + if (b->bb_u.s.bb_leftsib == cpu_to_be32(NULLAGBLOCK)) + cur->bc_levels[lev].ra |= XFS_BTCUR_LEFTRA; + if (b->bb_u.s.bb_rightsib == cpu_to_be32(NULLAGBLOCK)) + cur->bc_levels[lev].ra |= XFS_BTCUR_RIGHTRA; + } +} + +bool +xfs_btree_ptr_is_null( + struct xfs_btree_cur *cur, + const union xfs_btree_ptr *ptr) +{ + if (cur->bc_ops->ptr_len == XFS_BTREE_LONG_PTR_LEN) + return ptr->l == cpu_to_be64(NULLFSBLOCK); + else + return ptr->s == cpu_to_be32(NULLAGBLOCK); +} + +void +xfs_btree_set_ptr_null( + struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr) +{ + if (cur->bc_ops->ptr_len == XFS_BTREE_LONG_PTR_LEN) + ptr->l = cpu_to_be64(NULLFSBLOCK); + else + ptr->s = cpu_to_be32(NULLAGBLOCK); +} + +static inline bool +xfs_btree_ptrs_equal( + struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr1, + union xfs_btree_ptr *ptr2) +{ + if (cur->bc_ops->ptr_len == XFS_BTREE_LONG_PTR_LEN) + return ptr1->l == ptr2->l; + return ptr1->s == ptr2->s; +} + +/* + * Get/set/init sibling pointers + */ +void +xfs_btree_get_sibling( + struct xfs_btree_cur *cur, + struct xfs_btree_block *block, + union xfs_btree_ptr *ptr, + int lr) +{ + ASSERT(lr == XFS_BB_LEFTSIB || lr == XFS_BB_RIGHTSIB); + + if (cur->bc_ops->ptr_len == XFS_BTREE_LONG_PTR_LEN) { + if (lr == XFS_BB_RIGHTSIB) + ptr->l = block->bb_u.l.bb_rightsib; + else + ptr->l = block->bb_u.l.bb_leftsib; + } else { + if (lr == XFS_BB_RIGHTSIB) + ptr->s = block->bb_u.s.bb_rightsib; + else + ptr->s = block->bb_u.s.bb_leftsib; + } +} + +void +xfs_btree_set_sibling( + struct xfs_btree_cur *cur, + struct xfs_btree_block *block, + const union xfs_btree_ptr *ptr, + int lr) +{ + ASSERT(lr == XFS_BB_LEFTSIB || lr == XFS_BB_RIGHTSIB); + + if (cur->bc_ops->ptr_len == XFS_BTREE_LONG_PTR_LEN) { + if (lr == XFS_BB_RIGHTSIB) + block->bb_u.l.bb_rightsib = ptr->l; + else + block->bb_u.l.bb_leftsib = ptr->l; + } else { + if (lr == XFS_BB_RIGHTSIB) + block->bb_u.s.bb_rightsib = ptr->s; + else + block->bb_u.s.bb_leftsib = ptr->s; + } +} + +static void +__xfs_btree_init_block( + struct xfs_mount *mp, + struct xfs_btree_block *buf, + const struct xfs_btree_ops *ops, + xfs_daddr_t blkno, + __u16 level, + __u16 numrecs, + __u64 owner) +{ + bool crc = xfs_has_crc(mp); + __u32 magic = xfs_btree_magic(mp, ops); + + buf->bb_magic = cpu_to_be32(magic); + buf->bb_level = cpu_to_be16(level); + buf->bb_numrecs = cpu_to_be16(numrecs); + + if (ops->ptr_len == XFS_BTREE_LONG_PTR_LEN) { + buf->bb_u.l.bb_leftsib = cpu_to_be64(NULLFSBLOCK); + buf->bb_u.l.bb_rightsib = cpu_to_be64(NULLFSBLOCK); + if (crc) { + buf->bb_u.l.bb_blkno = cpu_to_be64(blkno); + buf->bb_u.l.bb_owner = cpu_to_be64(owner); + uuid_copy(&buf->bb_u.l.bb_uuid, &mp->m_sb.sb_meta_uuid); + buf->bb_u.l.bb_pad = 0; + buf->bb_u.l.bb_lsn = 0; + } + } else { + buf->bb_u.s.bb_leftsib = cpu_to_be32(NULLAGBLOCK); + buf->bb_u.s.bb_rightsib = cpu_to_be32(NULLAGBLOCK); + if (crc) { + buf->bb_u.s.bb_blkno = cpu_to_be64(blkno); + /* owner is a 32 bit value on short blocks */ + buf->bb_u.s.bb_owner = cpu_to_be32((__u32)owner); + uuid_copy(&buf->bb_u.s.bb_uuid, &mp->m_sb.sb_meta_uuid); + buf->bb_u.s.bb_lsn = 0; + } + } +} + +void +xfs_btree_init_block( + struct xfs_mount *mp, + struct xfs_btree_block *block, + const struct xfs_btree_ops *ops, + __u16 level, + __u16 numrecs, + __u64 owner) +{ + __xfs_btree_init_block(mp, block, ops, XFS_BUF_DADDR_NULL, level, + numrecs, owner); +} + +void +xfs_btree_init_buf( + struct xfs_mount *mp, + struct xfs_buf *bp, + const struct xfs_btree_ops *ops, + __u16 level, + __u16 numrecs, + __u64 owner) +{ + __xfs_btree_init_block(mp, XFS_BUF_TO_BLOCK(bp), ops, + xfs_buf_daddr(bp), level, numrecs, owner); + bp->b_ops = ops->buf_ops; +} + +static inline __u64 +xfs_btree_owner( + struct xfs_btree_cur *cur) +{ + switch (cur->bc_ops->type) { + case XFS_BTREE_TYPE_MEM: + return cur->bc_mem.xfbtree->owner; + case XFS_BTREE_TYPE_INODE: + return I_INO(cur->bc_ino.ip); + case XFS_BTREE_TYPE_AG: + return cur->bc_group->xg_gno; + default: + ASSERT(0); + return 0; + } +} + +void +xfs_btree_init_block_cur( + struct xfs_btree_cur *cur, + struct xfs_buf *bp, + int level, + int numrecs) +{ + xfs_btree_init_buf(cur->bc_mp, bp, cur->bc_ops, level, numrecs, + xfs_btree_owner(cur)); +} + +STATIC void +xfs_btree_buf_to_ptr( + struct xfs_btree_cur *cur, + struct xfs_buf *bp, + union xfs_btree_ptr *ptr) +{ + switch (cur->bc_ops->type) { + case XFS_BTREE_TYPE_AG: + ptr->s = cpu_to_be32(xfs_daddr_to_agbno(cur->bc_mp, + xfs_buf_daddr(bp))); + break; + case XFS_BTREE_TYPE_INODE: + ptr->l = cpu_to_be64(XFS_DADDR_TO_FSB(cur->bc_mp, + xfs_buf_daddr(bp))); + break; + case XFS_BTREE_TYPE_MEM: + ptr->l = cpu_to_be64(xfs_daddr_to_xfbno(xfs_buf_daddr(bp))); + break; + } +} + +static inline void +xfs_btree_set_refs( + struct xfs_btree_cur *cur, + struct xfs_buf *bp) +{ + xfs_buf_set_ref(bp, cur->bc_ops->lru_refs); +} + +int +xfs_btree_get_buf_block( + struct xfs_btree_cur *cur, + const union xfs_btree_ptr *ptr, + struct xfs_btree_block **block, + struct xfs_buf **bpp) +{ + xfs_daddr_t d; + int error; + + error = xfs_btree_ptr_to_daddr(cur, ptr, &d); + if (error) + return error; + error = xfs_trans_get_buf(cur->bc_tp, xfs_btree_buftarg(cur), d, + xfs_btree_bbsize(cur), 0, bpp); + if (error) + return error; + + (*bpp)->b_ops = cur->bc_ops->buf_ops; + *block = XFS_BUF_TO_BLOCK(*bpp); + return 0; +} + +/* + * Read in the buffer at the given ptr and return the buffer and + * the block pointer within the buffer. + */ +int +xfs_btree_read_buf_block( + struct xfs_btree_cur *cur, + const union xfs_btree_ptr *ptr, + int flags, + struct xfs_btree_block **block, + struct xfs_buf **bpp) +{ + struct xfs_mount *mp = cur->bc_mp; + xfs_daddr_t d; + int error; + + /* need to sort out how callers deal with failures first */ + ASSERT(!(flags & XBF_TRYLOCK)); + + error = xfs_btree_ptr_to_daddr(cur, ptr, &d); + if (error) + return error; + error = xfs_trans_read_buf(mp, cur->bc_tp, xfs_btree_buftarg(cur), d, + xfs_btree_bbsize(cur), flags, bpp, + cur->bc_ops->buf_ops); + if (xfs_metadata_is_sick(error)) + xfs_btree_mark_sick(cur); + if (error) + return error; + + xfs_btree_set_refs(cur, *bpp); + *block = XFS_BUF_TO_BLOCK(*bpp); + return 0; +} + +/* + * Copy keys from one btree block to another. + */ +void +xfs_btree_copy_keys( + struct xfs_btree_cur *cur, + union xfs_btree_key *dst_key, + const union xfs_btree_key *src_key, + int numkeys) +{ + ASSERT(numkeys >= 0); + memcpy(dst_key, src_key, numkeys * cur->bc_ops->key_len); +} + +/* + * Copy records from one btree block to another. + */ +STATIC void +xfs_btree_copy_recs( + struct xfs_btree_cur *cur, + union xfs_btree_rec *dst_rec, + union xfs_btree_rec *src_rec, + int numrecs) +{ + ASSERT(numrecs >= 0); + memcpy(dst_rec, src_rec, numrecs * cur->bc_ops->rec_len); +} + +/* + * Copy block pointers from one btree block to another. + */ +void +xfs_btree_copy_ptrs( + struct xfs_btree_cur *cur, + union xfs_btree_ptr *dst_ptr, + const union xfs_btree_ptr *src_ptr, + int numptrs) +{ + ASSERT(numptrs >= 0); + memcpy(dst_ptr, src_ptr, numptrs * cur->bc_ops->ptr_len); +} + +/* + * Shift keys one index left/right inside a single btree block. + */ +STATIC void +xfs_btree_shift_keys( + struct xfs_btree_cur *cur, + union xfs_btree_key *key, + int dir, + int numkeys) +{ + char *dst_key; + + ASSERT(numkeys >= 0); + ASSERT(dir == 1 || dir == -1); + + dst_key = (char *)key + (dir * cur->bc_ops->key_len); + memmove(dst_key, key, numkeys * cur->bc_ops->key_len); +} + +/* + * Shift records one index left/right inside a single btree block. + */ +STATIC void +xfs_btree_shift_recs( + struct xfs_btree_cur *cur, + union xfs_btree_rec *rec, + int dir, + int numrecs) +{ + char *dst_rec; + + ASSERT(numrecs >= 0); + ASSERT(dir == 1 || dir == -1); + + dst_rec = (char *)rec + (dir * cur->bc_ops->rec_len); + memmove(dst_rec, rec, numrecs * cur->bc_ops->rec_len); +} + +/* + * Shift block pointers one index left/right inside a single btree block. + */ +STATIC void +xfs_btree_shift_ptrs( + struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr, + int dir, + int numptrs) +{ + char *dst_ptr; + + ASSERT(numptrs >= 0); + ASSERT(dir == 1 || dir == -1); + + dst_ptr = (char *)ptr + (dir * cur->bc_ops->ptr_len); + memmove(dst_ptr, ptr, numptrs * cur->bc_ops->ptr_len); +} + +/* + * Log key values from the btree block. + */ +STATIC void +xfs_btree_log_keys( + struct xfs_btree_cur *cur, + struct xfs_buf *bp, + int first, + int last) +{ + + if (bp) { + xfs_trans_buf_set_type(cur->bc_tp, bp, XFS_BLFT_BTREE_BUF); + xfs_trans_log_buf(cur->bc_tp, bp, + xfs_btree_key_offset(cur, first), + xfs_btree_key_offset(cur, last + 1) - 1); + } else { + xfs_trans_log_inode(cur->bc_tp, cur->bc_ino.ip, + xfs_ilog_fbroot(cur->bc_ino.whichfork)); + } +} + +/* + * Log record values from the btree block. + */ +void +xfs_btree_log_recs( + struct xfs_btree_cur *cur, + struct xfs_buf *bp, + int first, + int last) +{ + if (!bp) { + xfs_trans_log_inode(cur->bc_tp, cur->bc_ino.ip, + xfs_ilog_fbroot(cur->bc_ino.whichfork)); + return; + } + + xfs_trans_buf_set_type(cur->bc_tp, bp, XFS_BLFT_BTREE_BUF); + xfs_trans_log_buf(cur->bc_tp, bp, + xfs_btree_rec_offset(cur, first), + xfs_btree_rec_offset(cur, last + 1) - 1); +} + +/* + * Log block pointer fields from a btree block (nonleaf). + */ +STATIC void +xfs_btree_log_ptrs( + struct xfs_btree_cur *cur, /* btree cursor */ + struct xfs_buf *bp, /* buffer containing btree block */ + int first, /* index of first pointer to log */ + int last) /* index of last pointer to log */ +{ + + if (bp) { + struct xfs_btree_block *block = XFS_BUF_TO_BLOCK(bp); + int level = xfs_btree_get_level(block); + + xfs_trans_buf_set_type(cur->bc_tp, bp, XFS_BLFT_BTREE_BUF); + xfs_trans_log_buf(cur->bc_tp, bp, + xfs_btree_ptr_offset(cur, first, level), + xfs_btree_ptr_offset(cur, last + 1, level) - 1); + } else { + xfs_trans_log_inode(cur->bc_tp, cur->bc_ino.ip, + xfs_ilog_fbroot(cur->bc_ino.whichfork)); + } + +} + +/* + * Log fields from a btree block header. + */ +void +xfs_btree_log_block( + struct xfs_btree_cur *cur, /* btree cursor */ + struct xfs_buf *bp, /* buffer containing btree block */ + uint32_t fields) /* mask of fields: XFS_BB_... */ +{ + int first; /* first byte offset logged */ + int last; /* last byte offset logged */ + static const short soffsets[] = { /* table of offsets (short) */ + offsetof(struct xfs_btree_block, bb_magic), + offsetof(struct xfs_btree_block, bb_level), + offsetof(struct xfs_btree_block, bb_numrecs), + offsetof(struct xfs_btree_block, bb_u.s.bb_leftsib), + offsetof(struct xfs_btree_block, bb_u.s.bb_rightsib), + offsetof(struct xfs_btree_block, bb_u.s.bb_blkno), + offsetof(struct xfs_btree_block, bb_u.s.bb_lsn), + offsetof(struct xfs_btree_block, bb_u.s.bb_uuid), + offsetof(struct xfs_btree_block, bb_u.s.bb_owner), + offsetof(struct xfs_btree_block, bb_u.s.bb_crc), + XFS_BTREE_SBLOCK_CRC_LEN + }; + static const short loffsets[] = { /* table of offsets (long) */ + offsetof(struct xfs_btree_block, bb_magic), + offsetof(struct xfs_btree_block, bb_level), + offsetof(struct xfs_btree_block, bb_numrecs), + offsetof(struct xfs_btree_block, bb_u.l.bb_leftsib), + offsetof(struct xfs_btree_block, bb_u.l.bb_rightsib), + offsetof(struct xfs_btree_block, bb_u.l.bb_blkno), + offsetof(struct xfs_btree_block, bb_u.l.bb_lsn), + offsetof(struct xfs_btree_block, bb_u.l.bb_uuid), + offsetof(struct xfs_btree_block, bb_u.l.bb_owner), + offsetof(struct xfs_btree_block, bb_u.l.bb_crc), + offsetof(struct xfs_btree_block, bb_u.l.bb_pad), + XFS_BTREE_LBLOCK_CRC_LEN + }; + + if (bp) { + int nbits; + + if (xfs_has_crc(cur->bc_mp)) { + /* + * We don't log the CRC when updating a btree + * block but instead recreate it during log + * recovery. As the log buffers have checksums + * of their own this is safe and avoids logging a crc + * update in a lot of places. + */ + if (fields == XFS_BB_ALL_BITS) + fields = XFS_BB_ALL_BITS_CRC; + nbits = XFS_BB_NUM_BITS_CRC; + } else { + nbits = XFS_BB_NUM_BITS; + } + xfs_btree_offsets(fields, + (cur->bc_ops->ptr_len == XFS_BTREE_LONG_PTR_LEN) ? + loffsets : soffsets, + nbits, &first, &last); + xfs_trans_buf_set_type(cur->bc_tp, bp, XFS_BLFT_BTREE_BUF); + xfs_trans_log_buf(cur->bc_tp, bp, first, last); + } else { + xfs_trans_log_inode(cur->bc_tp, cur->bc_ino.ip, + xfs_ilog_fbroot(cur->bc_ino.whichfork)); + } +} + +/* + * Increment cursor by one record at the level. + * For nonzero levels the leaf-ward information is untouched. + */ +int /* error */ +xfs_btree_increment( + struct xfs_btree_cur *cur, + int level, + int *stat) /* success/failure */ +{ + struct xfs_btree_block *block; + union xfs_btree_ptr ptr; + struct xfs_buf *bp; + int error; /* error return value */ + int lev; + + ASSERT(level < cur->bc_nlevels); + + /* Read-ahead to the right at this level. */ + xfs_btree_readahead(cur, level, XFS_BTCUR_RIGHTRA); + + /* Get a pointer to the btree block. */ + block = xfs_btree_get_block(cur, level, &bp); + +#ifdef DEBUG + error = xfs_btree_check_block(cur, block, level, bp); + if (error) + goto error0; +#endif + + /* We're done if we remain in the block after the increment. */ + if (++cur->bc_levels[level].ptr <= xfs_btree_get_numrecs(block)) + goto out1; + + /* Fail if we just went off the right edge of the tree. */ + xfs_btree_get_sibling(cur, block, &ptr, XFS_BB_RIGHTSIB); + if (xfs_btree_ptr_is_null(cur, &ptr)) + goto out0; + + XFS_BTREE_STATS_INC(cur, increment); + + /* + * March up the tree incrementing pointers. + * Stop when we don't go off the right edge of a block. + */ + for (lev = level + 1; lev < cur->bc_nlevels; lev++) { + block = xfs_btree_get_block(cur, lev, &bp); + +#ifdef DEBUG + error = xfs_btree_check_block(cur, block, lev, bp); + if (error) + goto error0; +#endif + + if (++cur->bc_levels[lev].ptr <= xfs_btree_get_numrecs(block)) + break; + + /* Read-ahead the right block for the next loop. */ + xfs_btree_readahead(cur, lev, XFS_BTCUR_RIGHTRA); + } + + /* + * If we went off the root then we are either seriously + * confused or have the tree root in an inode. + */ + if (lev == cur->bc_nlevels) { + if (cur->bc_ops->type == XFS_BTREE_TYPE_INODE) + goto out0; + ASSERT(0); + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error0; + } + ASSERT(lev < cur->bc_nlevels); + + /* + * Now walk back down the tree, fixing up the cursor's buffer + * pointers and key numbers. + */ + for (block = xfs_btree_get_block(cur, lev, &bp); lev > level; ) { + union xfs_btree_ptr *ptrp; + + ptrp = xfs_btree_ptr_addr(cur, cur->bc_levels[lev].ptr, block); + --lev; + error = xfs_btree_read_buf_block(cur, ptrp, 0, &block, &bp); + if (error) + goto error0; + + xfs_btree_setbuf(cur, lev, bp); + cur->bc_levels[lev].ptr = 1; + } +out1: + *stat = 1; + return 0; + +out0: + *stat = 0; + return 0; + +error0: + return error; +} + +/* + * Decrement cursor by one record at the level. + * For nonzero levels the leaf-ward information is untouched. + */ +int /* error */ +xfs_btree_decrement( + struct xfs_btree_cur *cur, + int level, + int *stat) /* success/failure */ +{ + struct xfs_btree_block *block; + struct xfs_buf *bp; + int error; /* error return value */ + int lev; + union xfs_btree_ptr ptr; + + ASSERT(level < cur->bc_nlevels); + + /* Read-ahead to the left at this level. */ + xfs_btree_readahead(cur, level, XFS_BTCUR_LEFTRA); + + /* We're done if we remain in the block after the decrement. */ + if (--cur->bc_levels[level].ptr > 0) + goto out1; + + /* Get a pointer to the btree block. */ + block = xfs_btree_get_block(cur, level, &bp); + +#ifdef DEBUG + error = xfs_btree_check_block(cur, block, level, bp); + if (error) + goto error0; +#endif + + /* Fail if we just went off the left edge of the tree. */ + xfs_btree_get_sibling(cur, block, &ptr, XFS_BB_LEFTSIB); + if (xfs_btree_ptr_is_null(cur, &ptr)) + goto out0; + + XFS_BTREE_STATS_INC(cur, decrement); + + /* + * March up the tree decrementing pointers. + * Stop when we don't go off the left edge of a block. + */ + for (lev = level + 1; lev < cur->bc_nlevels; lev++) { + if (--cur->bc_levels[lev].ptr > 0) + break; + /* Read-ahead the left block for the next loop. */ + xfs_btree_readahead(cur, lev, XFS_BTCUR_LEFTRA); + } + + /* + * If we went off the root then we are seriously confused. + * or the root of the tree is in an inode. + */ + if (lev == cur->bc_nlevels) { + if (cur->bc_ops->type == XFS_BTREE_TYPE_INODE) + goto out0; + ASSERT(0); + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error0; + } + ASSERT(lev < cur->bc_nlevels); + + /* + * Now walk back down the tree, fixing up the cursor's buffer + * pointers and key numbers. + */ + for (block = xfs_btree_get_block(cur, lev, &bp); lev > level; ) { + union xfs_btree_ptr *ptrp; + + ptrp = xfs_btree_ptr_addr(cur, cur->bc_levels[lev].ptr, block); + --lev; + error = xfs_btree_read_buf_block(cur, ptrp, 0, &block, &bp); + if (error) + goto error0; + xfs_btree_setbuf(cur, lev, bp); + cur->bc_levels[lev].ptr = xfs_btree_get_numrecs(block); + } +out1: + *stat = 1; + return 0; + +out0: + *stat = 0; + return 0; + +error0: + return error; +} + +/* + * Check the btree block owner now that we have the context to know who the + * real owner is. + */ +static inline xfs_failaddr_t +xfs_btree_check_block_owner( + struct xfs_btree_cur *cur, + struct xfs_btree_block *block) +{ + __u64 owner; + + if (!xfs_has_crc(cur->bc_mp) || + (cur->bc_flags & XFS_BTREE_BMBT_INVALID_OWNER)) + return NULL; + + owner = xfs_btree_owner(cur); + if (cur->bc_ops->ptr_len == XFS_BTREE_LONG_PTR_LEN) { + if (be64_to_cpu(block->bb_u.l.bb_owner) != owner) + return __this_address; + } else { + if (be32_to_cpu(block->bb_u.s.bb_owner) != owner) + return __this_address; + } + + return NULL; +} + +int +xfs_btree_lookup_get_block( + struct xfs_btree_cur *cur, /* btree cursor */ + int level, /* level in the btree */ + const union xfs_btree_ptr *pp, /* ptr to btree block */ + struct xfs_btree_block **blkp) /* return btree block */ +{ + struct xfs_buf *bp; /* buffer pointer for btree block */ + xfs_daddr_t daddr; + int error = 0; + + /* special case the root block if in an inode */ + if (xfs_btree_at_iroot(cur, level)) { + *blkp = xfs_btree_get_iroot(cur); + return 0; + } + + /* + * If the old buffer at this level for the disk address we are + * looking for re-use it. + * + * Otherwise throw it away and get a new one. + */ + bp = cur->bc_levels[level].bp; + error = xfs_btree_ptr_to_daddr(cur, pp, &daddr); + if (error) + return error; + if (bp && xfs_buf_daddr(bp) == daddr) { + *blkp = XFS_BUF_TO_BLOCK(bp); + return 0; + } + + error = xfs_btree_read_buf_block(cur, pp, 0, blkp, &bp); + if (error) + return error; + + /* Check the inode owner since the verifiers don't. */ + if (xfs_btree_check_block_owner(cur, *blkp) != NULL) + goto out_bad; + + /* Did we get the level we were looking for? */ + if (be16_to_cpu((*blkp)->bb_level) != level) + goto out_bad; + + /* Check that internal nodes have at least one record. */ + if (level != 0 && be16_to_cpu((*blkp)->bb_numrecs) == 0) + goto out_bad; + + xfs_btree_setbuf(cur, level, bp); + return 0; + +out_bad: + *blkp = NULL; + xfs_buf_mark_corrupt(bp); + xfs_trans_brelse(cur->bc_tp, bp); + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; +} + +/* + * Get current search key. For level 0 we don't actually have a key + * structure so we make one up from the record. For all other levels + * we just return the right key. + */ +STATIC union xfs_btree_key * +xfs_lookup_get_search_key( + struct xfs_btree_cur *cur, + int level, + int keyno, + struct xfs_btree_block *block, + union xfs_btree_key *kp) +{ + if (level == 0) { + cur->bc_ops->init_key_from_rec(kp, + xfs_btree_rec_addr(cur, keyno, block)); + return kp; + } + + return xfs_btree_key_addr(cur, keyno, block); +} + +/* + * Initialize a pointer to the root block. + */ +void +xfs_btree_init_ptr_from_cur( + struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr) +{ + if (cur->bc_ops->type == XFS_BTREE_TYPE_INODE) { + /* + * Inode-rooted btrees call xfs_btree_get_iroot to find the root + * in xfs_btree_lookup_get_block and don't need a pointer here. + */ + ptr->l = 0; + } else if (cur->bc_flags & XFS_BTREE_STAGING) { + ptr->s = cpu_to_be32(cur->bc_ag.afake->af_root); + } else { + cur->bc_ops->init_ptr_from_cur(cur, ptr); + } +} + +/* + * Lookup the record. The cursor is made to point to it, based on dir. + * stat is set to 0 if can't find any such record, 1 for success. + */ +int /* error */ +xfs_btree_lookup( + struct xfs_btree_cur *cur, /* btree cursor */ + xfs_lookup_t dir, /* <=, ==, or >= */ + int *stat) /* success/failure */ +{ + struct xfs_btree_block *block; /* current btree block */ + int cmp_r; /* current key comparison result */ + int error; /* error return value */ + int keyno; /* current key number */ + int level; /* level in the btree */ + union xfs_btree_ptr *pp; /* ptr to btree block */ + union xfs_btree_ptr ptr; /* ptr to btree block */ + + XFS_BTREE_STATS_INC(cur, lookup); + + /* No such thing as a zero-level tree. */ + if (XFS_IS_CORRUPT(cur->bc_mp, cur->bc_nlevels == 0)) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + + block = NULL; + keyno = 0; + + /* initialise start pointer from cursor */ + xfs_btree_init_ptr_from_cur(cur, &ptr); + pp = &ptr; + + /* + * Iterate over each level in the btree, starting at the root. + * For each level above the leaves, find the key we need, based + * on the lookup record, then follow the corresponding block + * pointer down to the next level. + */ + for (level = cur->bc_nlevels - 1, cmp_r = 1; level >= 0; level--) { + /* Get the block we need to do the lookup on. */ + error = xfs_btree_lookup_get_block(cur, level, pp, &block); + if (error) + goto error0; + + if (cmp_r == 0) { + /* + * If we already had a key match at a higher level, we + * know we need to use the first entry in this block. + */ + keyno = 1; + } else { + /* Otherwise search this block. Do a binary search. */ + + int high; /* high entry number */ + int low; /* low entry number */ + + /* Set low and high entry numbers, 1-based. */ + low = 1; + high = xfs_btree_get_numrecs(block); + if (!high) { + /* Block is empty, must be an empty leaf. */ + if (level != 0 || cur->bc_nlevels != 1) { + XFS_CORRUPTION_ERROR(__func__, + XFS_ERRLEVEL_LOW, + cur->bc_mp, block, + sizeof(*block)); + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + + cur->bc_levels[0].ptr = dir != XFS_LOOKUP_LE; + *stat = 0; + return 0; + } + + /* Binary search the block. */ + while (low <= high) { + union xfs_btree_key key; + union xfs_btree_key *kp; + + XFS_BTREE_STATS_INC(cur, compare); + + /* keyno is average of low and high. */ + keyno = (low + high) >> 1; + + /* Get current search key */ + kp = xfs_lookup_get_search_key(cur, level, + keyno, block, &key); + + /* + * Compute comparison result to get next + * direction: + * - less than, move right + * - greater than, move left + * - equal, we're done + */ + cmp_r = cur->bc_ops->cmp_key_with_cur(cur, kp); + if (cmp_r < 0) + low = keyno + 1; + else if (cmp_r > 0) + high = keyno - 1; + else + break; + } + } + + /* + * If there are more levels, set up for the next level + * by getting the block number and filling in the cursor. + */ + if (level > 0) { + /* + * If we moved left, need the previous key number, + * unless there isn't one. + */ + if (cmp_r > 0 && --keyno < 1) + keyno = 1; + pp = xfs_btree_ptr_addr(cur, keyno, block); + + error = xfs_btree_debug_check_ptr(cur, pp, 0, level); + if (error) + goto error0; + + cur->bc_levels[level].ptr = keyno; + } + } + + /* Done with the search. See if we need to adjust the results. */ + if (dir != XFS_LOOKUP_LE && cmp_r < 0) { + keyno++; + /* + * If ge search and we went off the end of the block, but it's + * not the last block, we're in the wrong block. + */ + xfs_btree_get_sibling(cur, block, &ptr, XFS_BB_RIGHTSIB); + if (dir == XFS_LOOKUP_GE && + keyno > xfs_btree_get_numrecs(block) && + !xfs_btree_ptr_is_null(cur, &ptr)) { + int i; + + cur->bc_levels[0].ptr = keyno; + error = xfs_btree_increment(cur, 0, &i); + if (error) + goto error0; + if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + *stat = 1; + return 0; + } + } else if (dir == XFS_LOOKUP_LE && cmp_r > 0) + keyno--; + cur->bc_levels[0].ptr = keyno; + + /* Return if we succeeded or not. */ + if (keyno == 0 || keyno > xfs_btree_get_numrecs(block)) + *stat = 0; + else if (dir != XFS_LOOKUP_EQ || cmp_r == 0) + *stat = 1; + else + *stat = 0; + return 0; + +error0: + return error; +} + +/* Find the high key storage area from a regular key. */ +union xfs_btree_key * +xfs_btree_high_key_from_key( + struct xfs_btree_cur *cur, + union xfs_btree_key *key) +{ + ASSERT(cur->bc_ops->geom_flags & XFS_BTGEO_OVERLAPPING); + return (union xfs_btree_key *)((char *)key + + (cur->bc_ops->key_len / 2)); +} + +/* Determine the low (and high if overlapped) keys of a leaf block */ +STATIC void +xfs_btree_get_leaf_keys( + struct xfs_btree_cur *cur, + struct xfs_btree_block *block, + union xfs_btree_key *key) +{ + union xfs_btree_key max_hkey; + union xfs_btree_key hkey; + union xfs_btree_rec *rec; + union xfs_btree_key *high; + int n; + + rec = xfs_btree_rec_addr(cur, 1, block); + cur->bc_ops->init_key_from_rec(key, rec); + + if (cur->bc_ops->geom_flags & XFS_BTGEO_OVERLAPPING) { + + cur->bc_ops->init_high_key_from_rec(&max_hkey, rec); + for (n = 2; n <= xfs_btree_get_numrecs(block); n++) { + rec = xfs_btree_rec_addr(cur, n, block); + cur->bc_ops->init_high_key_from_rec(&hkey, rec); + if (xfs_btree_keycmp_gt(cur, &hkey, &max_hkey)) + max_hkey = hkey; + } + + high = xfs_btree_high_key_from_key(cur, key); + memcpy(high, &max_hkey, cur->bc_ops->key_len / 2); + } +} + +/* Determine the low (and high if overlapped) keys of a node block */ +STATIC void +xfs_btree_get_node_keys( + struct xfs_btree_cur *cur, + struct xfs_btree_block *block, + union xfs_btree_key *key) +{ + union xfs_btree_key *hkey; + union xfs_btree_key *max_hkey; + union xfs_btree_key *high; + int n; + + if (cur->bc_ops->geom_flags & XFS_BTGEO_OVERLAPPING) { + memcpy(key, xfs_btree_key_addr(cur, 1, block), + cur->bc_ops->key_len / 2); + + max_hkey = xfs_btree_high_key_addr(cur, 1, block); + for (n = 2; n <= xfs_btree_get_numrecs(block); n++) { + hkey = xfs_btree_high_key_addr(cur, n, block); + if (xfs_btree_keycmp_gt(cur, hkey, max_hkey)) + max_hkey = hkey; + } + + high = xfs_btree_high_key_from_key(cur, key); + memcpy(high, max_hkey, cur->bc_ops->key_len / 2); + } else { + memcpy(key, xfs_btree_key_addr(cur, 1, block), + cur->bc_ops->key_len); + } +} + +/* Derive the keys for any btree block. */ +void +xfs_btree_get_keys( + struct xfs_btree_cur *cur, + struct xfs_btree_block *block, + union xfs_btree_key *key) +{ + if (be16_to_cpu(block->bb_level) == 0) + xfs_btree_get_leaf_keys(cur, block, key); + else + xfs_btree_get_node_keys(cur, block, key); +} + +/* + * Decide if we need to update the parent keys of a btree block. For + * a standard btree this is only necessary if we're updating the first + * record/key. For an overlapping btree, we must always update the + * keys because the highest key can be in any of the records or keys + * in the block. + */ +static inline bool +xfs_btree_needs_key_update( + struct xfs_btree_cur *cur, + int ptr) +{ + return (cur->bc_ops->geom_flags & XFS_BTGEO_OVERLAPPING) || ptr == 1; +} + +/* + * Update the low and high parent keys of the given level, progressing + * towards the root. If force_all is false, stop if the keys for a given + * level do not need updating. + */ +STATIC int +__xfs_btree_updkeys( + struct xfs_btree_cur *cur, + int level, + struct xfs_btree_block *block, + struct xfs_buf *bp0, + bool force_all) +{ + union xfs_btree_key key; /* keys from current level */ + union xfs_btree_key *lkey; /* keys from the next level up */ + union xfs_btree_key *hkey; + union xfs_btree_key *nlkey; /* keys from the next level up */ + union xfs_btree_key *nhkey; + struct xfs_buf *bp; + int ptr; + + ASSERT(cur->bc_ops->geom_flags & XFS_BTGEO_OVERLAPPING); + + /* Exit if there aren't any parent levels to update. */ + if (level + 1 >= cur->bc_nlevels) + return 0; + + trace_xfs_btree_updkeys(cur, level, bp0); + + lkey = &key; + hkey = xfs_btree_high_key_from_key(cur, lkey); + xfs_btree_get_keys(cur, block, lkey); + for (level++; level < cur->bc_nlevels; level++) { +#ifdef DEBUG + int error; +#endif + block = xfs_btree_get_block(cur, level, &bp); + trace_xfs_btree_updkeys(cur, level, bp); +#ifdef DEBUG + error = xfs_btree_check_block(cur, block, level, bp); + if (error) + return error; +#endif + ptr = cur->bc_levels[level].ptr; + nlkey = xfs_btree_key_addr(cur, ptr, block); + nhkey = xfs_btree_high_key_addr(cur, ptr, block); + if (!force_all && + xfs_btree_keycmp_eq(cur, nlkey, lkey) && + xfs_btree_keycmp_eq(cur, nhkey, hkey)) + break; + xfs_btree_copy_keys(cur, nlkey, lkey, 1); + xfs_btree_log_keys(cur, bp, ptr, ptr); + if (level + 1 >= cur->bc_nlevels) + break; + xfs_btree_get_node_keys(cur, block, lkey); + } + + return 0; +} + +/* Update all the keys from some level in cursor back to the root. */ +STATIC int +xfs_btree_updkeys_force( + struct xfs_btree_cur *cur, + int level) +{ + struct xfs_buf *bp; + struct xfs_btree_block *block; + + block = xfs_btree_get_block(cur, level, &bp); + return __xfs_btree_updkeys(cur, level, block, bp, true); +} + +/* + * Update the parent keys of the given level, progressing towards the root. + */ +STATIC int +xfs_btree_update_keys( + struct xfs_btree_cur *cur, + int level) +{ + struct xfs_btree_block *block; + struct xfs_buf *bp; + union xfs_btree_key *kp; + union xfs_btree_key key; + int ptr; + + ASSERT(level >= 0); + + block = xfs_btree_get_block(cur, level, &bp); + if (cur->bc_ops->geom_flags & XFS_BTGEO_OVERLAPPING) + return __xfs_btree_updkeys(cur, level, block, bp, false); + + /* + * Go up the tree from this level toward the root. + * At each level, update the key value to the value input. + * Stop when we reach a level where the cursor isn't pointing + * at the first entry in the block. + */ + xfs_btree_get_keys(cur, block, &key); + for (level++, ptr = 1; ptr == 1 && level < cur->bc_nlevels; level++) { +#ifdef DEBUG + int error; +#endif + block = xfs_btree_get_block(cur, level, &bp); +#ifdef DEBUG + error = xfs_btree_check_block(cur, block, level, bp); + if (error) + return error; +#endif + ptr = cur->bc_levels[level].ptr; + kp = xfs_btree_key_addr(cur, ptr, block); + xfs_btree_copy_keys(cur, kp, &key, 1); + xfs_btree_log_keys(cur, bp, ptr, ptr); + } + + return 0; +} + +/* + * Update the record referred to by cur to the value in the + * given record. This either works (return 0) or gets an + * EFSCORRUPTED error. + */ +int +xfs_btree_update( + struct xfs_btree_cur *cur, + union xfs_btree_rec *rec) +{ + struct xfs_btree_block *block; + struct xfs_buf *bp; + int error; + int ptr; + union xfs_btree_rec *rp; + + /* Pick up the current block. */ + block = xfs_btree_get_block(cur, 0, &bp); + +#ifdef DEBUG + error = xfs_btree_check_block(cur, block, 0, bp); + if (error) + goto error0; +#endif + /* Get the address of the rec to be updated. */ + ptr = cur->bc_levels[0].ptr; + rp = xfs_btree_rec_addr(cur, ptr, block); + + /* Fill in the new contents and log them. */ + xfs_btree_copy_recs(cur, rp, rec, 1); + xfs_btree_log_recs(cur, bp, ptr, ptr); + + /* Pass new key value up to our parent. */ + if (xfs_btree_needs_key_update(cur, ptr)) { + error = xfs_btree_update_keys(cur, 0); + if (error) + goto error0; + } + + return 0; + +error0: + return error; +} + +/* + * Move 1 record left from cur/level if possible. + * Update cur to reflect the new path. + */ +STATIC int /* error */ +xfs_btree_lshift( + struct xfs_btree_cur *cur, + int level, + int *stat) /* success/failure */ +{ + struct xfs_buf *lbp; /* left buffer pointer */ + struct xfs_btree_block *left; /* left btree block */ + int lrecs; /* left record count */ + struct xfs_buf *rbp; /* right buffer pointer */ + struct xfs_btree_block *right; /* right btree block */ + struct xfs_btree_cur *tcur; /* temporary btree cursor */ + int rrecs; /* right record count */ + union xfs_btree_ptr lptr; /* left btree pointer */ + union xfs_btree_key *rkp = NULL; /* right btree key */ + union xfs_btree_ptr *rpp = NULL; /* right address pointer */ + union xfs_btree_rec *rrp = NULL; /* right record pointer */ + int error; /* error return value */ + int i; + + if (xfs_btree_at_iroot(cur, level)) + goto out0; + + /* Set up variables for this block as "right". */ + right = xfs_btree_get_block(cur, level, &rbp); + +#ifdef DEBUG + error = xfs_btree_check_block(cur, right, level, rbp); + if (error) + goto error0; +#endif + + /* If we've got no left sibling then we can't shift an entry left. */ + xfs_btree_get_sibling(cur, right, &lptr, XFS_BB_LEFTSIB); + if (xfs_btree_ptr_is_null(cur, &lptr)) + goto out0; + + /* + * If the cursor entry is the one that would be moved, don't + * do it... it's too complicated. + */ + if (cur->bc_levels[level].ptr <= 1) + goto out0; + + /* Set up the left neighbor as "left". */ + error = xfs_btree_read_buf_block(cur, &lptr, 0, &left, &lbp); + if (error) + goto error0; + + /* If it's full, it can't take another entry. */ + lrecs = xfs_btree_get_numrecs(left); + if (lrecs == cur->bc_ops->get_maxrecs(cur, level)) + goto out0; + + rrecs = xfs_btree_get_numrecs(right); + + /* + * We add one entry to the left side and remove one for the right side. + * Account for it here, the changes will be updated on disk and logged + * later. + */ + lrecs++; + rrecs--; + + XFS_BTREE_STATS_INC(cur, lshift); + XFS_BTREE_STATS_ADD(cur, moves, 1); + + /* + * If non-leaf, copy a key and a ptr to the left block. + * Log the changes to the left block. + */ + if (level > 0) { + /* It's a non-leaf. Move keys and pointers. */ + union xfs_btree_key *lkp; /* left btree key */ + union xfs_btree_ptr *lpp; /* left address pointer */ + + lkp = xfs_btree_key_addr(cur, lrecs, left); + rkp = xfs_btree_key_addr(cur, 1, right); + + lpp = xfs_btree_ptr_addr(cur, lrecs, left); + rpp = xfs_btree_ptr_addr(cur, 1, right); + + error = xfs_btree_debug_check_ptr(cur, rpp, 0, level); + if (error) + goto error0; + + xfs_btree_copy_keys(cur, lkp, rkp, 1); + xfs_btree_copy_ptrs(cur, lpp, rpp, 1); + + xfs_btree_log_keys(cur, lbp, lrecs, lrecs); + xfs_btree_log_ptrs(cur, lbp, lrecs, lrecs); + + ASSERT(cur->bc_ops->keys_inorder(cur, + xfs_btree_key_addr(cur, lrecs - 1, left), lkp)); + } else { + /* It's a leaf. Move records. */ + union xfs_btree_rec *lrp; /* left record pointer */ + + lrp = xfs_btree_rec_addr(cur, lrecs, left); + rrp = xfs_btree_rec_addr(cur, 1, right); + + xfs_btree_copy_recs(cur, lrp, rrp, 1); + xfs_btree_log_recs(cur, lbp, lrecs, lrecs); + + ASSERT(cur->bc_ops->recs_inorder(cur, + xfs_btree_rec_addr(cur, lrecs - 1, left), lrp)); + } + + xfs_btree_set_numrecs(left, lrecs); + xfs_btree_log_block(cur, lbp, XFS_BB_NUMRECS); + + xfs_btree_set_numrecs(right, rrecs); + xfs_btree_log_block(cur, rbp, XFS_BB_NUMRECS); + + /* + * Slide the contents of right down one entry. + */ + XFS_BTREE_STATS_ADD(cur, moves, rrecs - 1); + if (level > 0) { + /* It's a nonleaf. operate on keys and ptrs */ + for (i = 0; i < rrecs; i++) { + error = xfs_btree_debug_check_ptr(cur, rpp, i + 1, level); + if (error) + goto error0; + } + + xfs_btree_shift_keys(cur, + xfs_btree_key_addr(cur, 2, right), + -1, rrecs); + xfs_btree_shift_ptrs(cur, + xfs_btree_ptr_addr(cur, 2, right), + -1, rrecs); + + xfs_btree_log_keys(cur, rbp, 1, rrecs); + xfs_btree_log_ptrs(cur, rbp, 1, rrecs); + } else { + /* It's a leaf. operate on records */ + xfs_btree_shift_recs(cur, + xfs_btree_rec_addr(cur, 2, right), + -1, rrecs); + xfs_btree_log_recs(cur, rbp, 1, rrecs); + } + + /* + * Using a temporary cursor, update the parent key values of the + * block on the left. + */ + if (cur->bc_ops->geom_flags & XFS_BTGEO_OVERLAPPING) { + error = xfs_btree_dup_cursor(cur, &tcur); + if (error) + goto error0; + i = xfs_btree_firstrec(tcur, level); + if (XFS_IS_CORRUPT(tcur->bc_mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error0; + } + + error = xfs_btree_decrement(tcur, level, &i); + if (error) + goto error1; + + /* Update the parent high keys of the left block, if needed. */ + error = xfs_btree_update_keys(tcur, level); + if (error) + goto error1; + + xfs_btree_del_cursor(tcur, XFS_BTREE_NOERROR); + } + + /* Update the parent keys of the right block. */ + error = xfs_btree_update_keys(cur, level); + if (error) + goto error0; + + /* Slide the cursor value left one. */ + cur->bc_levels[level].ptr--; + + *stat = 1; + return 0; + +out0: + *stat = 0; + return 0; + +error0: + return error; + +error1: + xfs_btree_del_cursor(tcur, XFS_BTREE_ERROR); + return error; +} + +/* + * Move 1 record right from cur/level if possible. + * Update cur to reflect the new path. + */ +STATIC int /* error */ +xfs_btree_rshift( + struct xfs_btree_cur *cur, + int level, + int *stat) /* success/failure */ +{ + struct xfs_buf *lbp; /* left buffer pointer */ + struct xfs_btree_block *left; /* left btree block */ + struct xfs_buf *rbp; /* right buffer pointer */ + struct xfs_btree_block *right; /* right btree block */ + struct xfs_btree_cur *tcur; /* temporary btree cursor */ + union xfs_btree_ptr rptr; /* right block pointer */ + union xfs_btree_key *rkp; /* right btree key */ + int rrecs; /* right record count */ + int lrecs; /* left record count */ + int error; /* error return value */ + int i; /* loop counter */ + + if (xfs_btree_at_iroot(cur, level)) + goto out0; + + /* Set up variables for this block as "left". */ + left = xfs_btree_get_block(cur, level, &lbp); + +#ifdef DEBUG + error = xfs_btree_check_block(cur, left, level, lbp); + if (error) + goto error0; +#endif + + /* If we've got no right sibling then we can't shift an entry right. */ + xfs_btree_get_sibling(cur, left, &rptr, XFS_BB_RIGHTSIB); + if (xfs_btree_ptr_is_null(cur, &rptr)) + goto out0; + + /* + * If the cursor entry is the one that would be moved, don't + * do it... it's too complicated. + */ + lrecs = xfs_btree_get_numrecs(left); + if (cur->bc_levels[level].ptr >= lrecs) + goto out0; + + /* Set up the right neighbor as "right". */ + error = xfs_btree_read_buf_block(cur, &rptr, 0, &right, &rbp); + if (error) + goto error0; + + /* If it's full, it can't take another entry. */ + rrecs = xfs_btree_get_numrecs(right); + if (rrecs == cur->bc_ops->get_maxrecs(cur, level)) + goto out0; + + XFS_BTREE_STATS_INC(cur, rshift); + XFS_BTREE_STATS_ADD(cur, moves, rrecs); + + /* + * Make a hole at the start of the right neighbor block, then + * copy the last left block entry to the hole. + */ + if (level > 0) { + /* It's a nonleaf. make a hole in the keys and ptrs */ + union xfs_btree_key *lkp; + union xfs_btree_ptr *lpp; + union xfs_btree_ptr *rpp; + + lkp = xfs_btree_key_addr(cur, lrecs, left); + lpp = xfs_btree_ptr_addr(cur, lrecs, left); + rkp = xfs_btree_key_addr(cur, 1, right); + rpp = xfs_btree_ptr_addr(cur, 1, right); + + for (i = rrecs - 1; i >= 0; i--) { + error = xfs_btree_debug_check_ptr(cur, rpp, i, level); + if (error) + goto error0; + } + + xfs_btree_shift_keys(cur, rkp, 1, rrecs); + xfs_btree_shift_ptrs(cur, rpp, 1, rrecs); + + error = xfs_btree_debug_check_ptr(cur, lpp, 0, level); + if (error) + goto error0; + + /* Now put the new data in, and log it. */ + xfs_btree_copy_keys(cur, rkp, lkp, 1); + xfs_btree_copy_ptrs(cur, rpp, lpp, 1); + + xfs_btree_log_keys(cur, rbp, 1, rrecs + 1); + xfs_btree_log_ptrs(cur, rbp, 1, rrecs + 1); + + ASSERT(cur->bc_ops->keys_inorder(cur, rkp, + xfs_btree_key_addr(cur, 2, right))); + } else { + /* It's a leaf. make a hole in the records */ + union xfs_btree_rec *lrp; + union xfs_btree_rec *rrp; + + lrp = xfs_btree_rec_addr(cur, lrecs, left); + rrp = xfs_btree_rec_addr(cur, 1, right); + + xfs_btree_shift_recs(cur, rrp, 1, rrecs); + + /* Now put the new data in, and log it. */ + xfs_btree_copy_recs(cur, rrp, lrp, 1); + xfs_btree_log_recs(cur, rbp, 1, rrecs + 1); + } + + /* + * Decrement and log left's numrecs, bump and log right's numrecs. + */ + xfs_btree_set_numrecs(left, --lrecs); + xfs_btree_log_block(cur, lbp, XFS_BB_NUMRECS); + + xfs_btree_set_numrecs(right, ++rrecs); + xfs_btree_log_block(cur, rbp, XFS_BB_NUMRECS); + + /* + * Using a temporary cursor, update the parent key values of the + * block on the right. + */ + error = xfs_btree_dup_cursor(cur, &tcur); + if (error) + goto error0; + i = xfs_btree_lastrec(tcur, level); + if (XFS_IS_CORRUPT(tcur->bc_mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error0; + } + + error = xfs_btree_increment(tcur, level, &i); + if (error) + goto error1; + + /* Update the parent high keys of the left block, if needed. */ + if (cur->bc_ops->geom_flags & XFS_BTGEO_OVERLAPPING) { + error = xfs_btree_update_keys(cur, level); + if (error) + goto error1; + } + + /* Update the parent keys of the right block. */ + error = xfs_btree_update_keys(tcur, level); + if (error) + goto error1; + + xfs_btree_del_cursor(tcur, XFS_BTREE_NOERROR); + + *stat = 1; + return 0; + +out0: + *stat = 0; + return 0; + +error0: + return error; + +error1: + xfs_btree_del_cursor(tcur, XFS_BTREE_ERROR); + return error; +} + +static inline int +xfs_btree_alloc_block( + struct xfs_btree_cur *cur, + const union xfs_btree_ptr *hint_block, + union xfs_btree_ptr *new_block, + int *stat) +{ + int error; + + /* + * Don't allow block allocation for a staging cursor, because staging + * cursors do not support regular btree modifications. + * + * Bulk loading uses a separate callback to obtain new blocks from a + * preallocated list, which prevents ENOSPC failures during loading. + */ + if (unlikely(cur->bc_flags & XFS_BTREE_STAGING)) { + ASSERT(0); + return -EFSCORRUPTED; + } + + error = cur->bc_ops->alloc_block(cur, hint_block, new_block, stat); + trace_xfs_btree_alloc_block(cur, new_block, *stat, error); + return error; +} + +/* + * Split cur/level block in half. + * Return new block number and the key to its first + * record (to be inserted into parent). + */ +STATIC int /* error */ +__xfs_btree_split( + struct xfs_btree_cur *cur, + int level, + union xfs_btree_ptr *ptrp, + union xfs_btree_key *key, + struct xfs_btree_cur **curp, + int *stat) /* success/failure */ +{ + union xfs_btree_ptr lptr; /* left sibling block ptr */ + struct xfs_buf *lbp; /* left buffer pointer */ + struct xfs_btree_block *left; /* left btree block */ + union xfs_btree_ptr rptr; /* right sibling block ptr */ + struct xfs_buf *rbp; /* right buffer pointer */ + struct xfs_btree_block *right; /* right btree block */ + union xfs_btree_ptr rrptr; /* right-right sibling ptr */ + struct xfs_buf *rrbp; /* right-right buffer pointer */ + struct xfs_btree_block *rrblock; /* right-right btree block */ + int lrecs; + int rrecs; + int src_index; + int error; /* error return value */ + int i; + + XFS_BTREE_STATS_INC(cur, split); + + /* Set up left block (current one). */ + left = xfs_btree_get_block(cur, level, &lbp); + +#ifdef DEBUG + error = xfs_btree_check_block(cur, left, level, lbp); + if (error) + goto error0; +#endif + + xfs_btree_buf_to_ptr(cur, lbp, &lptr); + + /* Allocate the new block. If we can't do it, we're toast. Give up. */ + error = xfs_btree_alloc_block(cur, &lptr, &rptr, stat); + if (error) + goto error0; + if (*stat == 0) + goto out0; + XFS_BTREE_STATS_INC(cur, alloc); + + /* Set up the new block as "right". */ + error = xfs_btree_get_buf_block(cur, &rptr, &right, &rbp); + if (error) + goto error0; + + /* Fill in the btree header for the new right block. */ + xfs_btree_init_block_cur(cur, rbp, xfs_btree_get_level(left), 0); + + /* + * Split the entries between the old and the new block evenly. + * Make sure that if there's an odd number of entries now, that + * each new block will have the same number of entries. + */ + lrecs = xfs_btree_get_numrecs(left); + rrecs = lrecs / 2; + if ((lrecs & 1) && cur->bc_levels[level].ptr <= rrecs + 1) + rrecs++; + src_index = (lrecs - rrecs + 1); + + XFS_BTREE_STATS_ADD(cur, moves, rrecs); + + /* Adjust numrecs for the later get_*_keys() calls. */ + lrecs -= rrecs; + xfs_btree_set_numrecs(left, lrecs); + xfs_btree_set_numrecs(right, xfs_btree_get_numrecs(right) + rrecs); + + /* + * Copy btree block entries from the left block over to the + * new block, the right. Update the right block and log the + * changes. + */ + if (level > 0) { + /* It's a non-leaf. Move keys and pointers. */ + union xfs_btree_key *lkp; /* left btree key */ + union xfs_btree_ptr *lpp; /* left address pointer */ + union xfs_btree_key *rkp; /* right btree key */ + union xfs_btree_ptr *rpp; /* right address pointer */ + + lkp = xfs_btree_key_addr(cur, src_index, left); + lpp = xfs_btree_ptr_addr(cur, src_index, left); + rkp = xfs_btree_key_addr(cur, 1, right); + rpp = xfs_btree_ptr_addr(cur, 1, right); + + for (i = src_index; i < rrecs; i++) { + error = xfs_btree_debug_check_ptr(cur, lpp, i, level); + if (error) + goto error0; + } + + /* Copy the keys & pointers to the new block. */ + xfs_btree_copy_keys(cur, rkp, lkp, rrecs); + xfs_btree_copy_ptrs(cur, rpp, lpp, rrecs); + + xfs_btree_log_keys(cur, rbp, 1, rrecs); + xfs_btree_log_ptrs(cur, rbp, 1, rrecs); + + /* Stash the keys of the new block for later insertion. */ + xfs_btree_get_node_keys(cur, right, key); + } else { + /* It's a leaf. Move records. */ + union xfs_btree_rec *lrp; /* left record pointer */ + union xfs_btree_rec *rrp; /* right record pointer */ + + lrp = xfs_btree_rec_addr(cur, src_index, left); + rrp = xfs_btree_rec_addr(cur, 1, right); + + /* Copy records to the new block. */ + xfs_btree_copy_recs(cur, rrp, lrp, rrecs); + xfs_btree_log_recs(cur, rbp, 1, rrecs); + + /* Stash the keys of the new block for later insertion. */ + xfs_btree_get_leaf_keys(cur, right, key); + } + + /* + * Find the left block number by looking in the buffer. + * Adjust sibling pointers. + */ + xfs_btree_get_sibling(cur, left, &rrptr, XFS_BB_RIGHTSIB); + xfs_btree_set_sibling(cur, right, &rrptr, XFS_BB_RIGHTSIB); + xfs_btree_set_sibling(cur, right, &lptr, XFS_BB_LEFTSIB); + xfs_btree_set_sibling(cur, left, &rptr, XFS_BB_RIGHTSIB); + + xfs_btree_log_block(cur, rbp, XFS_BB_ALL_BITS); + xfs_btree_log_block(cur, lbp, XFS_BB_NUMRECS | XFS_BB_RIGHTSIB); + + /* + * If there's a block to the new block's right, make that block + * point back to right instead of to left. + */ + if (!xfs_btree_ptr_is_null(cur, &rrptr)) { + error = xfs_btree_read_buf_block(cur, &rrptr, + 0, &rrblock, &rrbp); + if (error) + goto error0; + xfs_btree_set_sibling(cur, rrblock, &rptr, XFS_BB_LEFTSIB); + xfs_btree_log_block(cur, rrbp, XFS_BB_LEFTSIB); + } + + /* Update the parent high keys of the left block, if needed. */ + if (cur->bc_ops->geom_flags & XFS_BTGEO_OVERLAPPING) { + error = xfs_btree_update_keys(cur, level); + if (error) + goto error0; + } + + /* + * If the cursor is really in the right block, move it there. + * If it's just pointing past the last entry in left, then we'll + * insert there, so don't change anything in that case. + */ + if (cur->bc_levels[level].ptr > lrecs + 1) { + xfs_btree_setbuf(cur, level, rbp); + cur->bc_levels[level].ptr -= lrecs; + } + /* + * If there are more levels, we'll need another cursor which refers + * the right block, no matter where this cursor was. + */ + if (level + 1 < cur->bc_nlevels) { + error = xfs_btree_dup_cursor(cur, curp); + if (error) + goto error0; + (*curp)->bc_levels[level + 1].ptr++; + } + *ptrp = rptr; + *stat = 1; + return 0; +out0: + *stat = 0; + return 0; + +error0: + return error; +} + +#ifdef __KERNEL__ +struct xfs_btree_split_args { + struct xfs_btree_cur *cur; + int level; + union xfs_btree_ptr *ptrp; + union xfs_btree_key *key; + struct xfs_btree_cur **curp; + int *stat; /* success/failure */ + int result; + bool kswapd; /* allocation in kswapd context */ + struct completion *done; + struct work_struct work; +}; + +/* + * Stack switching interfaces for allocation + */ +static void +xfs_btree_split_worker( + struct work_struct *work) +{ + struct xfs_btree_split_args *args = container_of(work, + struct xfs_btree_split_args, work); + unsigned long pflags; + unsigned long new_pflags = 0; + + /* + * we are in a transaction context here, but may also be doing work + * in kswapd context, and hence we may need to inherit that state + * temporarily to ensure that we don't block waiting for memory reclaim + * in any way. + */ + if (args->kswapd) + new_pflags |= PF_MEMALLOC | PF_KSWAPD; + + current_set_flags_nested(&pflags, new_pflags); + xfs_trans_set_context(args->cur->bc_tp); + + args->result = __xfs_btree_split(args->cur, args->level, args->ptrp, + args->key, args->curp, args->stat); + + xfs_trans_clear_context(args->cur->bc_tp); + current_restore_flags_nested(&pflags, new_pflags); + + /* + * Do not access args after complete() has run here. We don't own args + * and the owner may run and free args before we return here. + */ + complete(args->done); + +} + +/* + * BMBT split requests often come in with little stack to work on so we push + * them off to a worker thread so there is lots of stack to use. For the other + * btree types, just call directly to avoid the context switch overhead here. + * + * Care must be taken here - the work queue rescuer thread introduces potential + * AGF <> worker queue deadlocks if the BMBT block allocation has to lock new + * AGFs to allocate blocks. A task being run by the rescuer could attempt to + * lock an AGF that is already locked by a task queued to run by the rescuer, + * resulting in an ABBA deadlock as the rescuer cannot run the lock holder to + * release it until the current thread it is running gains the lock. + * + * To avoid this issue, we only ever queue BMBT splits that don't have an AGF + * already locked to allocate from. The only place that doesn't hold an AGF + * locked is unwritten extent conversion at IO completion, but that has already + * been offloaded to a worker thread and hence has no stack consumption issues + * we have to worry about. + */ +STATIC int /* error */ +xfs_btree_split( + struct xfs_btree_cur *cur, + int level, + union xfs_btree_ptr *ptrp, + union xfs_btree_key *key, + struct xfs_btree_cur **curp, + int *stat) /* success/failure */ +{ + struct xfs_btree_split_args args; + DECLARE_COMPLETION_ONSTACK(done); + + if (!xfs_btree_is_bmap(cur->bc_ops) || + cur->bc_tp->t_highest_agno == NULLAGNUMBER) + return __xfs_btree_split(cur, level, ptrp, key, curp, stat); + + args.cur = cur; + args.level = level; + args.ptrp = ptrp; + args.key = key; + args.curp = curp; + args.stat = stat; + args.done = &done; + args.kswapd = current_is_kswapd(); + INIT_WORK_ONSTACK(&args.work, xfs_btree_split_worker); + queue_work(xfs_alloc_wq, &args.work); + wait_for_completion(&done); + destroy_work_on_stack(&args.work); + return args.result; +} +#else +#define xfs_btree_split __xfs_btree_split +#endif /* __KERNEL__ */ + +/* Move the records from a root leaf block to a separate block. */ +STATIC void +xfs_btree_promote_leaf_iroot( + struct xfs_btree_cur *cur, + struct xfs_btree_block *block, + struct xfs_buf *cbp, + union xfs_btree_ptr *cptr, + struct xfs_btree_block *cblock) +{ + union xfs_btree_rec *rp; + union xfs_btree_rec *crp; + union xfs_btree_key *kp; + union xfs_btree_ptr *pp; + struct xfs_btree_block *broot; + int numrecs = xfs_btree_get_numrecs(block); + + /* Copy the records from the leaf broot into the new child block. */ + rp = xfs_btree_rec_addr(cur, 1, block); + crp = xfs_btree_rec_addr(cur, 1, cblock); + xfs_btree_copy_recs(cur, crp, rp, numrecs); + + /* + * Increment the tree height. + * + * Trickery here: The amount of memory that we need per record for the + * ifork's btree root block may change when we convert the broot from a + * leaf to a node block. Free the existing leaf broot so that nobody + * thinks we need to migrate node pointers when we realloc the broot + * buffer after bumping nlevels. + */ + cur->bc_ops->broot_realloc(cur, 0); + cur->bc_nlevels++; + cur->bc_levels[1].ptr = 1; + + /* + * Allocate a new node broot and initialize it to point to the new + * child block. + */ + broot = cur->bc_ops->broot_realloc(cur, 1); + xfs_btree_init_block(cur->bc_mp, broot, cur->bc_ops, + cur->bc_nlevels - 1, 1, I_INO(cur->bc_ino.ip)); + + pp = xfs_btree_ptr_addr(cur, 1, broot); + kp = xfs_btree_key_addr(cur, 1, broot); + xfs_btree_copy_ptrs(cur, pp, cptr, 1); + xfs_btree_get_keys(cur, cblock, kp); + + /* Attach the new block to the cursor and log it. */ + xfs_btree_setbuf(cur, 0, cbp); + xfs_btree_log_block(cur, cbp, XFS_BB_ALL_BITS); + xfs_btree_log_recs(cur, cbp, 1, numrecs); +} + +/* + * Move the keys and pointers from a root block to a separate block. + * + * Since the keyptr size does not change, all we have to do is increase the + * tree height, copy the keyptrs to the new internal node (cblock), shrink + * the root, and copy the pointers there. + */ +STATIC int +xfs_btree_promote_node_iroot( + struct xfs_btree_cur *cur, + struct xfs_btree_block *block, + int level, + struct xfs_buf *cbp, + union xfs_btree_ptr *cptr, + struct xfs_btree_block *cblock) +{ + union xfs_btree_key *ckp; + union xfs_btree_key *kp; + union xfs_btree_ptr *cpp; + union xfs_btree_ptr *pp; + int i; + int error; + int numrecs = xfs_btree_get_numrecs(block); + + /* + * Increase tree height, adjusting the root block level to match. + * We cannot change the root btree node size until we've copied the + * block contents to the new child block. + */ + be16_add_cpu(&block->bb_level, 1); + cur->bc_nlevels++; + cur->bc_levels[level + 1].ptr = 1; + + /* + * Adjust the root btree record count, then copy the keys from the old + * root to the new child block. + */ + xfs_btree_set_numrecs(block, 1); + kp = xfs_btree_key_addr(cur, 1, block); + ckp = xfs_btree_key_addr(cur, 1, cblock); + xfs_btree_copy_keys(cur, ckp, kp, numrecs); + + /* Check the pointers and copy them to the new child block. */ + pp = xfs_btree_ptr_addr(cur, 1, block); + cpp = xfs_btree_ptr_addr(cur, 1, cblock); + for (i = 0; i < numrecs; i++) { + error = xfs_btree_debug_check_ptr(cur, pp, i, level); + if (error) + return error; + } + xfs_btree_copy_ptrs(cur, cpp, pp, numrecs); + + /* + * Set the first keyptr to point to the new child block, then shrink + * the memory buffer for the root block. + */ + error = xfs_btree_debug_check_ptr(cur, cptr, 0, level); + if (error) + return error; + xfs_btree_copy_ptrs(cur, pp, cptr, 1); + xfs_btree_get_keys(cur, cblock, kp); + + cur->bc_ops->broot_realloc(cur, 1); + + /* Attach the new block to the cursor and log it. */ + xfs_btree_setbuf(cur, level, cbp); + xfs_btree_log_block(cur, cbp, XFS_BB_ALL_BITS); + xfs_btree_log_keys(cur, cbp, 1, numrecs); + xfs_btree_log_ptrs(cur, cbp, 1, numrecs); + return 0; +} + +/* + * Copy the old inode root contents into a real block and make the + * broot point to it. + */ +int /* error */ +xfs_btree_new_iroot( + struct xfs_btree_cur *cur, /* btree cursor */ + int *logflags, /* logging flags for inode */ + int *stat) /* return status - 0 fail */ +{ + struct xfs_buf *cbp; /* buffer for cblock */ + struct xfs_btree_block *block; /* btree block */ + struct xfs_btree_block *cblock; /* child btree block */ + union xfs_btree_ptr aptr; + union xfs_btree_ptr nptr; /* new block addr */ + int level; /* btree level */ + int error; /* error return code */ + + XFS_BTREE_STATS_INC(cur, newroot); + + ASSERT(cur->bc_ops->type == XFS_BTREE_TYPE_INODE); + + level = cur->bc_nlevels - 1; + + block = xfs_btree_get_iroot(cur); + ASSERT(level > 0 || (cur->bc_ops->geom_flags & XFS_BTGEO_IROOT_RECORDS)); + if (level > 0) + aptr = *xfs_btree_ptr_addr(cur, 1, block); + else + aptr.l = cpu_to_be64(XFS_INODE_TO_FSB(cur->bc_ino.ip)); + + /* Allocate the new block. If we can't do it, we're toast. Give up. */ + error = xfs_btree_alloc_block(cur, &aptr, &nptr, stat); + if (error) + goto error0; + if (*stat == 0) + return 0; + + XFS_BTREE_STATS_INC(cur, alloc); + + /* Copy the root into a real block. */ + error = xfs_btree_get_buf_block(cur, &nptr, &cblock, &cbp); + if (error) + goto error0; + + /* + * we can't just memcpy() the root in for CRC enabled btree blocks. + * In that case have to also ensure the blkno remains correct + */ + memcpy(cblock, block, xfs_btree_block_len(cur)); + if (xfs_has_crc(cur->bc_mp)) { + __be64 bno = cpu_to_be64(xfs_buf_daddr(cbp)); + if (cur->bc_ops->ptr_len == XFS_BTREE_LONG_PTR_LEN) + cblock->bb_u.l.bb_blkno = bno; + else + cblock->bb_u.s.bb_blkno = bno; + } + + if (level > 0) { + error = xfs_btree_promote_node_iroot(cur, block, level, cbp, + &nptr, cblock); + if (error) + goto error0; + } else { + xfs_btree_promote_leaf_iroot(cur, block, cbp, &nptr, cblock); + } + + *logflags |= XFS_ILOG_CORE | xfs_ilog_fbroot(cur->bc_ino.whichfork); + *stat = 1; + return 0; +error0: + return error; +} + +static void +xfs_btree_set_root( + struct xfs_btree_cur *cur, + const union xfs_btree_ptr *ptr, + int inc) +{ + if (cur->bc_flags & XFS_BTREE_STAGING) { + /* Update the btree root information for a per-AG fake root. */ + cur->bc_ag.afake->af_root = be32_to_cpu(ptr->s); + cur->bc_ag.afake->af_levels += inc; + } else { + cur->bc_ops->set_root(cur, ptr, inc); + } +} + +/* + * Allocate a new root block, fill it in. + */ +STATIC int /* error */ +xfs_btree_new_root( + struct xfs_btree_cur *cur, /* btree cursor */ + int *stat) /* success/failure */ +{ + struct xfs_btree_block *block; /* one half of the old root block */ + struct xfs_buf *bp; /* buffer containing block */ + int error; /* error return value */ + struct xfs_buf *lbp; /* left buffer pointer */ + struct xfs_btree_block *left; /* left btree block */ + struct xfs_buf *nbp; /* new (root) buffer */ + struct xfs_btree_block *new; /* new (root) btree block */ + int nptr; /* new value for key index, 1 or 2 */ + struct xfs_buf *rbp; /* right buffer pointer */ + struct xfs_btree_block *right; /* right btree block */ + union xfs_btree_ptr rptr; + union xfs_btree_ptr lptr; + + XFS_BTREE_STATS_INC(cur, newroot); + + /* initialise our start point from the cursor */ + xfs_btree_init_ptr_from_cur(cur, &rptr); + + /* Allocate the new block. If we can't do it, we're toast. Give up. */ + error = xfs_btree_alloc_block(cur, &rptr, &lptr, stat); + if (error) + goto error0; + if (*stat == 0) + goto out0; + XFS_BTREE_STATS_INC(cur, alloc); + + /* Set up the new block. */ + error = xfs_btree_get_buf_block(cur, &lptr, &new, &nbp); + if (error) + goto error0; + + /* Set the root in the holding structure increasing the level by 1. */ + xfs_btree_set_root(cur, &lptr, 1); + + /* + * At the previous root level there are now two blocks: the old root, + * and the new block generated when it was split. We don't know which + * one the cursor is pointing at, so we set up variables "left" and + * "right" for each case. + */ + block = xfs_btree_get_block(cur, cur->bc_nlevels - 1, &bp); + +#ifdef DEBUG + error = xfs_btree_check_block(cur, block, cur->bc_nlevels - 1, bp); + if (error) + goto error0; +#endif + + xfs_btree_get_sibling(cur, block, &rptr, XFS_BB_RIGHTSIB); + if (!xfs_btree_ptr_is_null(cur, &rptr)) { + /* Our block is left, pick up the right block. */ + lbp = bp; + xfs_btree_buf_to_ptr(cur, lbp, &lptr); + left = block; + error = xfs_btree_read_buf_block(cur, &rptr, 0, &right, &rbp); + if (error) + goto error0; + bp = rbp; + nptr = 1; + } else { + /* Our block is right, pick up the left block. */ + rbp = bp; + xfs_btree_buf_to_ptr(cur, rbp, &rptr); + right = block; + xfs_btree_get_sibling(cur, right, &lptr, XFS_BB_LEFTSIB); + error = xfs_btree_read_buf_block(cur, &lptr, 0, &left, &lbp); + if (error) + goto error0; + bp = lbp; + nptr = 2; + } + + /* Fill in the new block's btree header and log it. */ + xfs_btree_init_block_cur(cur, nbp, cur->bc_nlevels, 2); + xfs_btree_log_block(cur, nbp, XFS_BB_ALL_BITS); + ASSERT(!xfs_btree_ptr_is_null(cur, &lptr) && + !xfs_btree_ptr_is_null(cur, &rptr)); + + /* Fill in the key data in the new root. */ + if (xfs_btree_get_level(left) > 0) { + /* + * Get the keys for the left block's keys and put them directly + * in the parent block. Do the same for the right block. + */ + xfs_btree_get_node_keys(cur, left, + xfs_btree_key_addr(cur, 1, new)); + xfs_btree_get_node_keys(cur, right, + xfs_btree_key_addr(cur, 2, new)); + } else { + /* + * Get the keys for the left block's records and put them + * directly in the parent block. Do the same for the right + * block. + */ + xfs_btree_get_leaf_keys(cur, left, + xfs_btree_key_addr(cur, 1, new)); + xfs_btree_get_leaf_keys(cur, right, + xfs_btree_key_addr(cur, 2, new)); + } + xfs_btree_log_keys(cur, nbp, 1, 2); + + /* Fill in the pointer data in the new root. */ + xfs_btree_copy_ptrs(cur, + xfs_btree_ptr_addr(cur, 1, new), &lptr, 1); + xfs_btree_copy_ptrs(cur, + xfs_btree_ptr_addr(cur, 2, new), &rptr, 1); + xfs_btree_log_ptrs(cur, nbp, 1, 2); + + /* Fix up the cursor. */ + xfs_btree_setbuf(cur, cur->bc_nlevels, nbp); + cur->bc_levels[cur->bc_nlevels].ptr = nptr; + cur->bc_nlevels++; + ASSERT(cur->bc_nlevels <= cur->bc_maxlevels); + *stat = 1; + return 0; +error0: + return error; +out0: + *stat = 0; + return 0; +} + +STATIC int +xfs_btree_make_block_unfull( + struct xfs_btree_cur *cur, /* btree cursor */ + int level, /* btree level */ + int numrecs,/* # of recs in block */ + int *oindex,/* old tree index */ + int *index, /* new tree index */ + union xfs_btree_ptr *nptr, /* new btree ptr */ + struct xfs_btree_cur **ncur, /* new btree cursor */ + union xfs_btree_key *key, /* key of new block */ + int *stat) +{ + int error = 0; + + if (xfs_btree_at_iroot(cur, level)) { + struct xfs_inode *ip = cur->bc_ino.ip; + + if (numrecs < cur->bc_ops->get_dmaxrecs(cur, level)) { + /* A root block that can be made bigger. */ + cur->bc_ops->broot_realloc(cur, numrecs + 1); + *stat = 1; + } else { + /* A root block that needs replacing */ + int logflags = 0; + + error = xfs_btree_new_iroot(cur, &logflags, stat); + if (error || *stat == 0) + return error; + + xfs_trans_log_inode(cur->bc_tp, ip, logflags); + } + + return 0; + } + + /* First, try shifting an entry to the right neighbor. */ + error = xfs_btree_rshift(cur, level, stat); + if (error || *stat) + return error; + + /* Next, try shifting an entry to the left neighbor. */ + error = xfs_btree_lshift(cur, level, stat); + if (error) + return error; + + if (*stat) { + *oindex = *index = cur->bc_levels[level].ptr; + return 0; + } + + /* + * Next, try splitting the current block in half. + * + * If this works we have to re-set our variables because we + * could be in a different block now. + */ + error = xfs_btree_split(cur, level, nptr, key, ncur, stat); + if (error || *stat == 0) + return error; + + + *index = cur->bc_levels[level].ptr; + return 0; +} + +/* + * Insert one record/level. Return information to the caller + * allowing the next level up to proceed if necessary. + */ +STATIC int +xfs_btree_insrec( + struct xfs_btree_cur *cur, /* btree cursor */ + int level, /* level to insert record at */ + union xfs_btree_ptr *ptrp, /* i/o: block number inserted */ + union xfs_btree_rec *rec, /* record to insert */ + union xfs_btree_key *key, /* i/o: block key for ptrp */ + struct xfs_btree_cur **curp, /* output: new cursor replacing cur */ + int *stat) /* success/failure */ +{ + struct xfs_btree_block *block; /* btree block */ + struct xfs_buf *bp; /* buffer for block */ + union xfs_btree_ptr nptr; /* new block ptr */ + struct xfs_btree_cur *ncur = NULL; /* new btree cursor */ + union xfs_btree_key nkey; /* new block key */ + union xfs_btree_key *lkey; + int optr; /* old key/record index */ + int ptr; /* key/record index */ + int numrecs;/* number of records */ + int error; /* error return value */ + int i; + xfs_daddr_t old_bn; + + ncur = NULL; + lkey = &nkey; + + /* + * If we have an external root pointer, and we've made it to the + * root level, allocate a new root block and we're done. + */ + if (cur->bc_ops->type != XFS_BTREE_TYPE_INODE && + level >= cur->bc_nlevels) { + error = xfs_btree_new_root(cur, stat); + xfs_btree_set_ptr_null(cur, ptrp); + + return error; + } + + /* If we're off the left edge, return failure. */ + ptr = cur->bc_levels[level].ptr; + if (ptr == 0) { + *stat = 0; + return 0; + } + + optr = ptr; + + XFS_BTREE_STATS_INC(cur, insrec); + + /* Get pointers to the btree buffer and block. */ + block = xfs_btree_get_block(cur, level, &bp); + old_bn = bp ? xfs_buf_daddr(bp) : XFS_BUF_DADDR_NULL; + numrecs = xfs_btree_get_numrecs(block); + +#ifdef DEBUG + error = xfs_btree_check_block(cur, block, level, bp); + if (error) + goto error0; + + /* Check that the new entry is being inserted in the right place. */ + if (ptr <= numrecs) { + if (level == 0) { + ASSERT(cur->bc_ops->recs_inorder(cur, rec, + xfs_btree_rec_addr(cur, ptr, block))); + } else { + ASSERT(cur->bc_ops->keys_inorder(cur, key, + xfs_btree_key_addr(cur, ptr, block))); + } + } +#endif + + /* + * If the block is full, we can't insert the new entry until we + * make the block un-full. + */ + xfs_btree_set_ptr_null(cur, &nptr); + if (numrecs == cur->bc_ops->get_maxrecs(cur, level)) { + error = xfs_btree_make_block_unfull(cur, level, numrecs, + &optr, &ptr, &nptr, &ncur, lkey, stat); + if (error || *stat == 0) + goto error0; + } + + /* + * The current block may have changed if the block was + * previously full and we have just made space in it. + */ + block = xfs_btree_get_block(cur, level, &bp); + numrecs = xfs_btree_get_numrecs(block); + +#ifdef DEBUG + error = xfs_btree_check_block(cur, block, level, bp); + if (error) + goto error0; +#endif + + /* + * At this point we know there's room for our new entry in the block + * we're pointing at. + */ + XFS_BTREE_STATS_ADD(cur, moves, numrecs - ptr + 1); + + if (level > 0) { + /* It's a nonleaf. make a hole in the keys and ptrs */ + union xfs_btree_key *kp; + union xfs_btree_ptr *pp; + + kp = xfs_btree_key_addr(cur, ptr, block); + pp = xfs_btree_ptr_addr(cur, ptr, block); + + for (i = numrecs - ptr; i >= 0; i--) { + error = xfs_btree_debug_check_ptr(cur, pp, i, level); + if (error) + goto error0; + } + + xfs_btree_shift_keys(cur, kp, 1, numrecs - ptr + 1); + xfs_btree_shift_ptrs(cur, pp, 1, numrecs - ptr + 1); + + error = xfs_btree_debug_check_ptr(cur, ptrp, 0, level); + if (error) + goto error0; + + /* Now put the new data in, bump numrecs and log it. */ + xfs_btree_copy_keys(cur, kp, key, 1); + xfs_btree_copy_ptrs(cur, pp, ptrp, 1); + numrecs++; + xfs_btree_set_numrecs(block, numrecs); + xfs_btree_log_ptrs(cur, bp, ptr, numrecs); + xfs_btree_log_keys(cur, bp, ptr, numrecs); +#ifdef DEBUG + if (ptr < numrecs) { + ASSERT(cur->bc_ops->keys_inorder(cur, kp, + xfs_btree_key_addr(cur, ptr + 1, block))); + } +#endif + } else { + /* It's a leaf. make a hole in the records */ + union xfs_btree_rec *rp; + + rp = xfs_btree_rec_addr(cur, ptr, block); + + xfs_btree_shift_recs(cur, rp, 1, numrecs - ptr + 1); + + /* Now put the new data in, bump numrecs and log it. */ + xfs_btree_copy_recs(cur, rp, rec, 1); + xfs_btree_set_numrecs(block, ++numrecs); + xfs_btree_log_recs(cur, bp, ptr, numrecs); +#ifdef DEBUG + if (ptr < numrecs) { + ASSERT(cur->bc_ops->recs_inorder(cur, rp, + xfs_btree_rec_addr(cur, ptr + 1, block))); + } +#endif + } + + /* Log the new number of records in the btree header. */ + xfs_btree_log_block(cur, bp, XFS_BB_NUMRECS); + + /* + * Update btree keys to reflect the newly added record or keyptr. + * There are three cases here to be aware of. Normally, all we have to + * do is walk towards the root, updating keys as necessary. + * + * If the caller had us target a full block for the insertion, we dealt + * with that by calling the _make_block_unfull function. If the + * "make unfull" function splits the block, it'll hand us back the key + * and pointer of the new block. We haven't yet added the new block to + * the next level up, so if we decide to add the new record to the new + * block (bp->b_bn != old_bn), we have to update the caller's pointer + * so that the caller adds the new block with the correct key. + * + * However, there is a third possibility-- if the selected block is the + * root block of an inode-rooted btree and cannot be expanded further, + * the "make unfull" function moves the root block contents to a new + * block and updates the root block to point to the new block. In this + * case, no block pointer is passed back because the block has already + * been added to the btree. In this case, we need to use the regular + * key update function, just like the first case. This is critical for + * overlapping btrees, because the high key must be updated to reflect + * the entire tree, not just the subtree accessible through the first + * child of the root (which is now two levels down from the root). + */ + if (!xfs_btree_ptr_is_null(cur, &nptr) && + bp && xfs_buf_daddr(bp) != old_bn) { + xfs_btree_get_keys(cur, block, lkey); + } else if (xfs_btree_needs_key_update(cur, optr)) { + error = xfs_btree_update_keys(cur, level); + if (error) + goto error0; + } + + /* + * Return the new block number, if any. + * If there is one, give back a record value and a cursor too. + */ + *ptrp = nptr; + if (!xfs_btree_ptr_is_null(cur, &nptr)) { + xfs_btree_copy_keys(cur, key, lkey, 1); + *curp = ncur; + } + + *stat = 1; + return 0; + +error0: + if (ncur) + xfs_btree_del_cursor(ncur, error); + return error; +} + +/* + * Insert the record at the point referenced by cur. + * + * A multi-level split of the tree on insert will invalidate the original + * cursor. All callers of this function should assume that the cursor is + * no longer valid and revalidate it. + */ +int +xfs_btree_insert( + struct xfs_btree_cur *cur, + int *stat) +{ + int error; /* error return value */ + int i; /* result value, 0 for failure */ + int level; /* current level number in btree */ + union xfs_btree_ptr nptr; /* new block number (split result) */ + struct xfs_btree_cur *ncur; /* new cursor (split result) */ + struct xfs_btree_cur *pcur; /* previous level's cursor */ + union xfs_btree_key bkey; /* key of block to insert */ + union xfs_btree_key *key; + union xfs_btree_rec rec; /* record to insert */ + + level = 0; + ncur = NULL; + pcur = cur; + key = &bkey; + + xfs_btree_set_ptr_null(cur, &nptr); + + /* Make a key out of the record data to be inserted, and save it. */ + cur->bc_ops->init_rec_from_cur(cur, &rec); + cur->bc_ops->init_key_from_rec(key, &rec); + + /* + * Loop going up the tree, starting at the leaf level. + * Stop when we don't get a split block, that must mean that + * the insert is finished with this level. + */ + do { + /* + * Insert nrec/nptr into this level of the tree. + * Note if we fail, nptr will be null. + */ + error = xfs_btree_insrec(pcur, level, &nptr, &rec, key, + &ncur, &i); + if (error) { + if (pcur != cur) + xfs_btree_del_cursor(pcur, XFS_BTREE_ERROR); + goto error0; + } + + if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error0; + } + level++; + + /* + * See if the cursor we just used is trash. + * Can't trash the caller's cursor, but otherwise we should + * if ncur is a new cursor or we're about to be done. + */ + if (pcur != cur && + (ncur || xfs_btree_ptr_is_null(cur, &nptr))) { + /* Save the state from the cursor before we trash it */ + if (cur->bc_ops->update_cursor && + !(cur->bc_flags & XFS_BTREE_STAGING)) + cur->bc_ops->update_cursor(pcur, cur); + cur->bc_nlevels = pcur->bc_nlevels; + xfs_btree_del_cursor(pcur, XFS_BTREE_NOERROR); + } + /* If we got a new cursor, switch to it. */ + if (ncur) { + pcur = ncur; + ncur = NULL; + } + } while (!xfs_btree_ptr_is_null(cur, &nptr)); + + *stat = i; + return 0; +error0: + return error; +} + +/* Move the records from a child leaf block to the root block. */ +STATIC void +xfs_btree_demote_leaf_child( + struct xfs_btree_cur *cur, + struct xfs_btree_block *cblock, + int numrecs) +{ + union xfs_btree_rec *rp; + union xfs_btree_rec *crp; + struct xfs_btree_block *broot; + + /* + * Decrease the tree height. + * + * Trickery here: The amount of memory that we need per record for the + * ifork's btree root block may change when we convert the broot from a + * node to a leaf. Free the old node broot so that we can get a fresh + * leaf broot. + */ + cur->bc_ops->broot_realloc(cur, 0); + cur->bc_nlevels--; + + /* + * Allocate a new leaf broot and copy the records from the old child. + * Detach the old child from the cursor. + */ + broot = cur->bc_ops->broot_realloc(cur, numrecs); + xfs_btree_init_block(cur->bc_mp, broot, cur->bc_ops, 0, numrecs, + I_INO(cur->bc_ino.ip)); + + rp = xfs_btree_rec_addr(cur, 1, broot); + crp = xfs_btree_rec_addr(cur, 1, cblock); + xfs_btree_copy_recs(cur, rp, crp, numrecs); + + cur->bc_levels[0].bp = NULL; +} + +/* + * Move the keyptrs from a child node block to the root block. + * + * Since the keyptr size does not change, all we have to do is increase the + * tree height, copy the keyptrs to the new internal node (cblock), shrink + * the root, and copy the pointers there. + */ +STATIC int +xfs_btree_demote_node_child( + struct xfs_btree_cur *cur, + struct xfs_btree_block *cblock, + int level, + int numrecs) +{ + struct xfs_btree_block *block; + union xfs_btree_key *ckp; + union xfs_btree_key *kp; + union xfs_btree_ptr *cpp; + union xfs_btree_ptr *pp; + int i; + int error; + + /* + * Adjust the root btree node size and the record count to match the + * doomed child so that we can copy the keyptrs ahead of changing the + * tree shape. + */ + block = cur->bc_ops->broot_realloc(cur, numrecs); + + xfs_btree_set_numrecs(block, numrecs); + ASSERT(block->bb_numrecs == cblock->bb_numrecs); + + /* Copy keys from the doomed block. */ + kp = xfs_btree_key_addr(cur, 1, block); + ckp = xfs_btree_key_addr(cur, 1, cblock); + xfs_btree_copy_keys(cur, kp, ckp, numrecs); + + /* Copy pointers from the doomed block. */ + pp = xfs_btree_ptr_addr(cur, 1, block); + cpp = xfs_btree_ptr_addr(cur, 1, cblock); + for (i = 0; i < numrecs; i++) { + error = xfs_btree_debug_check_ptr(cur, cpp, i, level - 1); + if (error) + return error; + } + xfs_btree_copy_ptrs(cur, pp, cpp, numrecs); + + /* Decrease tree height, adjusting the root block level to match. */ + cur->bc_levels[level - 1].bp = NULL; + be16_add_cpu(&block->bb_level, -1); + cur->bc_nlevels--; + return 0; +} + +/* + * Try to merge a non-leaf block back into the inode root. + * + * Note: the killroot names comes from the fact that we're effectively + * killing the old root block. But because we can't just delete the + * inode we have to copy the single block it was pointing to into the + * inode. + */ +STATIC int +xfs_btree_kill_iroot( + struct xfs_btree_cur *cur) +{ + struct xfs_inode *ip = cur->bc_ino.ip; + struct xfs_btree_block *block; + struct xfs_btree_block *cblock; + struct xfs_buf *cbp; + int level; + int numrecs; + int error; +#ifdef DEBUG + union xfs_btree_ptr ptr; +#endif + + ASSERT(cur->bc_ops->type == XFS_BTREE_TYPE_INODE); + ASSERT((cur->bc_ops->geom_flags & XFS_BTGEO_IROOT_RECORDS) || + cur->bc_nlevels > 1); + + /* + * Don't deal with the root block needs to be a leaf case. + * We're just going to turn the thing back into extents anyway. + */ + level = cur->bc_nlevels - 1; + if (level == 1 && !(cur->bc_ops->geom_flags & XFS_BTGEO_IROOT_RECORDS)) + goto out0; + + /* If we're already a leaf, jump out. */ + if (level == 0) + goto out0; + + /* + * Give up if the root has multiple children. + */ + block = xfs_btree_get_iroot(cur); + if (xfs_btree_get_numrecs(block) != 1) + goto out0; + + cblock = xfs_btree_get_block(cur, level - 1, &cbp); + numrecs = xfs_btree_get_numrecs(cblock); + + /* + * Only do this if the next level will fit. + * Then the data must be copied up to the inode, + * instead of freeing the root you free the next level. + */ + if (numrecs > cur->bc_ops->get_dmaxrecs(cur, level)) + goto out0; + + XFS_BTREE_STATS_INC(cur, killroot); + +#ifdef DEBUG + xfs_btree_get_sibling(cur, block, &ptr, XFS_BB_LEFTSIB); + ASSERT(xfs_btree_ptr_is_null(cur, &ptr)); + xfs_btree_get_sibling(cur, block, &ptr, XFS_BB_RIGHTSIB); + ASSERT(xfs_btree_ptr_is_null(cur, &ptr)); +#endif + + if (level > 1) { + error = xfs_btree_demote_node_child(cur, cblock, level, + numrecs); + if (error) + return error; + } else + xfs_btree_demote_leaf_child(cur, cblock, numrecs); + + error = xfs_btree_free_block(cur, cbp); + if (error) + return error; + + xfs_trans_log_inode(cur->bc_tp, ip, + XFS_ILOG_CORE | xfs_ilog_fbroot(cur->bc_ino.whichfork)); +out0: + return 0; +} + +/* + * Kill the current root node, and replace it with it's only child node. + */ +STATIC int +xfs_btree_kill_root( + struct xfs_btree_cur *cur, + struct xfs_buf *bp, + int level, + union xfs_btree_ptr *newroot) +{ + int error; + + XFS_BTREE_STATS_INC(cur, killroot); + + /* + * Update the root pointer, decreasing the level by 1 and then + * free the old root. + */ + xfs_btree_set_root(cur, newroot, -1); + + error = xfs_btree_free_block(cur, bp); + if (error) + return error; + + cur->bc_levels[level].bp = NULL; + cur->bc_levels[level].ra = 0; + cur->bc_nlevels--; + + return 0; +} + +STATIC int +xfs_btree_dec_cursor( + struct xfs_btree_cur *cur, + int level, + int *stat) +{ + int error; + int i; + + if (level > 0) { + error = xfs_btree_decrement(cur, level, &i); + if (error) + return error; + } + + *stat = 1; + return 0; +} + +/* + * Single level of the btree record deletion routine. + * Delete record pointed to by cur/level. + * Remove the record from its block then rebalance the tree. + * Return 0 for error, 1 for done, 2 to go on to the next level. + */ +STATIC int /* error */ +xfs_btree_delrec( + struct xfs_btree_cur *cur, /* btree cursor */ + int level, /* level removing record from */ + int *stat) /* fail/done/go-on */ +{ + struct xfs_btree_block *block; /* btree block */ + union xfs_btree_ptr cptr; /* current block ptr */ + struct xfs_buf *bp; /* buffer for block */ + int error; /* error return value */ + int i; /* loop counter */ + union xfs_btree_ptr lptr; /* left sibling block ptr */ + struct xfs_buf *lbp; /* left buffer pointer */ + struct xfs_btree_block *left; /* left btree block */ + int lrecs = 0; /* left record count */ + int ptr; /* key/record index */ + union xfs_btree_ptr rptr; /* right sibling block ptr */ + struct xfs_buf *rbp; /* right buffer pointer */ + struct xfs_btree_block *right; /* right btree block */ + struct xfs_btree_block *rrblock; /* right-right btree block */ + struct xfs_buf *rrbp; /* right-right buffer pointer */ + int rrecs = 0; /* right record count */ + struct xfs_btree_cur *tcur; /* temporary btree cursor */ + int numrecs; /* temporary numrec count */ + + tcur = NULL; + + /* Get the index of the entry being deleted, check for nothing there. */ + ptr = cur->bc_levels[level].ptr; + if (ptr == 0) { + *stat = 0; + return 0; + } + + /* Get the buffer & block containing the record or key/ptr. */ + block = xfs_btree_get_block(cur, level, &bp); + numrecs = xfs_btree_get_numrecs(block); + +#ifdef DEBUG + error = xfs_btree_check_block(cur, block, level, bp); + if (error) + goto error0; +#endif + + /* Fail if we're off the end of the block. */ + if (ptr > numrecs) { + *stat = 0; + return 0; + } + + XFS_BTREE_STATS_INC(cur, delrec); + XFS_BTREE_STATS_ADD(cur, moves, numrecs - ptr); + + /* Excise the entries being deleted. */ + if (level > 0) { + /* It's a nonleaf. operate on keys and ptrs */ + union xfs_btree_key *lkp; + union xfs_btree_ptr *lpp; + + lkp = xfs_btree_key_addr(cur, ptr + 1, block); + lpp = xfs_btree_ptr_addr(cur, ptr + 1, block); + + for (i = 0; i < numrecs - ptr; i++) { + error = xfs_btree_debug_check_ptr(cur, lpp, i, level); + if (error) + goto error0; + } + + if (ptr < numrecs) { + xfs_btree_shift_keys(cur, lkp, -1, numrecs - ptr); + xfs_btree_shift_ptrs(cur, lpp, -1, numrecs - ptr); + xfs_btree_log_keys(cur, bp, ptr, numrecs - 1); + xfs_btree_log_ptrs(cur, bp, ptr, numrecs - 1); + } + } else { + /* It's a leaf. operate on records */ + if (ptr < numrecs) { + xfs_btree_shift_recs(cur, + xfs_btree_rec_addr(cur, ptr + 1, block), + -1, numrecs - ptr); + xfs_btree_log_recs(cur, bp, ptr, numrecs - 1); + } + } + + /* + * Decrement and log the number of entries in the block. + */ + xfs_btree_set_numrecs(block, --numrecs); + xfs_btree_log_block(cur, bp, XFS_BB_NUMRECS); + + /* + * We're at the root level. First, shrink the root block in-memory. + * Try to get rid of the next level down. If we can't then there's + * nothing left to do. numrecs was decremented above. + */ + if (xfs_btree_at_iroot(cur, level)) { + cur->bc_ops->broot_realloc(cur, numrecs); + + error = xfs_btree_kill_iroot(cur); + if (error) + goto error0; + + error = xfs_btree_dec_cursor(cur, level, stat); + if (error) + goto error0; + *stat = 1; + return 0; + } + + /* + * If this is the root level, and there's only one entry left, and it's + * NOT the leaf level, then we can get rid of this level. + */ + if (level == cur->bc_nlevels - 1) { + if (numrecs == 1 && level > 0) { + union xfs_btree_ptr *pp; + /* + * pp is still set to the first pointer in the block. + * Make it the new root of the btree. + */ + pp = xfs_btree_ptr_addr(cur, 1, block); + error = xfs_btree_kill_root(cur, bp, level, pp); + if (error) + goto error0; + } else if (level > 0) { + error = xfs_btree_dec_cursor(cur, level, stat); + if (error) + goto error0; + } + *stat = 1; + return 0; + } + + /* + * If we deleted the leftmost entry in the block, update the + * key values above us in the tree. + */ + if (xfs_btree_needs_key_update(cur, ptr)) { + error = xfs_btree_update_keys(cur, level); + if (error) + goto error0; + } + + /* + * If the number of records remaining in the block is at least + * the minimum, we're done. + */ + if (numrecs >= cur->bc_ops->get_minrecs(cur, level)) { + error = xfs_btree_dec_cursor(cur, level, stat); + if (error) + goto error0; + return 0; + } + + /* + * Otherwise, we have to move some records around to keep the + * tree balanced. Look at the left and right sibling blocks to + * see if we can re-balance by moving only one record. + */ + xfs_btree_get_sibling(cur, block, &rptr, XFS_BB_RIGHTSIB); + xfs_btree_get_sibling(cur, block, &lptr, XFS_BB_LEFTSIB); + + if (cur->bc_ops->type == XFS_BTREE_TYPE_INODE) { + /* + * One child of root, need to get a chance to copy its contents + * into the root and delete it. Can't go up to next level, + * there's nothing to delete there. + */ + if (xfs_btree_ptr_is_null(cur, &rptr) && + xfs_btree_ptr_is_null(cur, &lptr) && + level == cur->bc_nlevels - 2) { + error = xfs_btree_kill_iroot(cur); + if (!error) + error = xfs_btree_dec_cursor(cur, level, stat); + if (error) + goto error0; + return 0; + } + } + + ASSERT(!xfs_btree_ptr_is_null(cur, &rptr) || + !xfs_btree_ptr_is_null(cur, &lptr)); + + /* + * Duplicate the cursor so our btree manipulations here won't + * disrupt the next level up. + */ + error = xfs_btree_dup_cursor(cur, &tcur); + if (error) + goto error0; + + /* + * If there's a right sibling, see if it's ok to shift an entry + * out of it. + */ + if (!xfs_btree_ptr_is_null(cur, &rptr)) { + /* + * Move the temp cursor to the last entry in the next block. + * Actually any entry but the first would suffice. + */ + i = xfs_btree_lastrec(tcur, level); + if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error0; + } + + error = xfs_btree_increment(tcur, level, &i); + if (error) + goto error0; + if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error0; + } + + i = xfs_btree_lastrec(tcur, level); + if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error0; + } + + /* Grab a pointer to the block. */ + right = xfs_btree_get_block(tcur, level, &rbp); +#ifdef DEBUG + error = xfs_btree_check_block(tcur, right, level, rbp); + if (error) + goto error0; +#endif + /* Grab the current block number, for future use. */ + xfs_btree_get_sibling(tcur, right, &cptr, XFS_BB_LEFTSIB); + + /* + * If right block is full enough so that removing one entry + * won't make it too empty, and left-shifting an entry out + * of right to us works, we're done. + */ + if (xfs_btree_get_numrecs(right) - 1 >= + cur->bc_ops->get_minrecs(tcur, level)) { + error = xfs_btree_lshift(tcur, level, &i); + if (error) + goto error0; + if (i) { + ASSERT(xfs_btree_get_numrecs(block) >= + cur->bc_ops->get_minrecs(tcur, level)); + + xfs_btree_del_cursor(tcur, XFS_BTREE_NOERROR); + tcur = NULL; + + error = xfs_btree_dec_cursor(cur, level, stat); + if (error) + goto error0; + return 0; + } + } + + /* + * Otherwise, grab the number of records in right for + * future reference, and fix up the temp cursor to point + * to our block again (last record). + */ + rrecs = xfs_btree_get_numrecs(right); + if (!xfs_btree_ptr_is_null(cur, &lptr)) { + i = xfs_btree_firstrec(tcur, level); + if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error0; + } + + error = xfs_btree_decrement(tcur, level, &i); + if (error) + goto error0; + if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error0; + } + } + } + + /* + * If there's a left sibling, see if it's ok to shift an entry + * out of it. + */ + if (!xfs_btree_ptr_is_null(cur, &lptr)) { + /* + * Move the temp cursor to the first entry in the + * previous block. + */ + i = xfs_btree_firstrec(tcur, level); + if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error0; + } + + error = xfs_btree_decrement(tcur, level, &i); + if (error) + goto error0; + i = xfs_btree_firstrec(tcur, level); + if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error0; + } + + /* Grab a pointer to the block. */ + left = xfs_btree_get_block(tcur, level, &lbp); +#ifdef DEBUG + error = xfs_btree_check_block(cur, left, level, lbp); + if (error) + goto error0; +#endif + /* Grab the current block number, for future use. */ + xfs_btree_get_sibling(tcur, left, &cptr, XFS_BB_RIGHTSIB); + + /* + * If left block is full enough so that removing one entry + * won't make it too empty, and right-shifting an entry out + * of left to us works, we're done. + */ + if (xfs_btree_get_numrecs(left) - 1 >= + cur->bc_ops->get_minrecs(tcur, level)) { + error = xfs_btree_rshift(tcur, level, &i); + if (error) + goto error0; + if (i) { + ASSERT(xfs_btree_get_numrecs(block) >= + cur->bc_ops->get_minrecs(tcur, level)); + xfs_btree_del_cursor(tcur, XFS_BTREE_NOERROR); + tcur = NULL; + if (level == 0) + cur->bc_levels[0].ptr++; + + *stat = 1; + return 0; + } + } + + /* + * Otherwise, grab the number of records in right for + * future reference. + */ + lrecs = xfs_btree_get_numrecs(left); + } + + /* Delete the temp cursor, we're done with it. */ + xfs_btree_del_cursor(tcur, XFS_BTREE_NOERROR); + tcur = NULL; + + /* If here, we need to do a join to keep the tree balanced. */ + ASSERT(!xfs_btree_ptr_is_null(cur, &cptr)); + + if (!xfs_btree_ptr_is_null(cur, &lptr) && + lrecs + xfs_btree_get_numrecs(block) <= + cur->bc_ops->get_maxrecs(cur, level)) { + /* + * Set "right" to be the starting block, + * "left" to be the left neighbor. + */ + rptr = cptr; + right = block; + rbp = bp; + error = xfs_btree_read_buf_block(cur, &lptr, 0, &left, &lbp); + if (error) + goto error0; + + /* + * If that won't work, see if we can join with the right neighbor block. + */ + } else if (!xfs_btree_ptr_is_null(cur, &rptr) && + rrecs + xfs_btree_get_numrecs(block) <= + cur->bc_ops->get_maxrecs(cur, level)) { + /* + * Set "left" to be the starting block, + * "right" to be the right neighbor. + */ + lptr = cptr; + left = block; + lbp = bp; + error = xfs_btree_read_buf_block(cur, &rptr, 0, &right, &rbp); + if (error) + goto error0; + + /* + * Otherwise, we can't fix the imbalance. + * Just return. This is probably a logic error, but it's not fatal. + */ + } else { + error = xfs_btree_dec_cursor(cur, level, stat); + if (error) + goto error0; + return 0; + } + + rrecs = xfs_btree_get_numrecs(right); + lrecs = xfs_btree_get_numrecs(left); + + /* + * We're now going to join "left" and "right" by moving all the stuff + * in "right" to "left" and deleting "right". + */ + XFS_BTREE_STATS_ADD(cur, moves, rrecs); + if (level > 0) { + /* It's a non-leaf. Move keys and pointers. */ + union xfs_btree_key *lkp; /* left btree key */ + union xfs_btree_ptr *lpp; /* left address pointer */ + union xfs_btree_key *rkp; /* right btree key */ + union xfs_btree_ptr *rpp; /* right address pointer */ + + lkp = xfs_btree_key_addr(cur, lrecs + 1, left); + lpp = xfs_btree_ptr_addr(cur, lrecs + 1, left); + rkp = xfs_btree_key_addr(cur, 1, right); + rpp = xfs_btree_ptr_addr(cur, 1, right); + + for (i = 1; i < rrecs; i++) { + error = xfs_btree_debug_check_ptr(cur, rpp, i, level); + if (error) + goto error0; + } + + xfs_btree_copy_keys(cur, lkp, rkp, rrecs); + xfs_btree_copy_ptrs(cur, lpp, rpp, rrecs); + + xfs_btree_log_keys(cur, lbp, lrecs + 1, lrecs + rrecs); + xfs_btree_log_ptrs(cur, lbp, lrecs + 1, lrecs + rrecs); + } else { + /* It's a leaf. Move records. */ + union xfs_btree_rec *lrp; /* left record pointer */ + union xfs_btree_rec *rrp; /* right record pointer */ + + lrp = xfs_btree_rec_addr(cur, lrecs + 1, left); + rrp = xfs_btree_rec_addr(cur, 1, right); + + xfs_btree_copy_recs(cur, lrp, rrp, rrecs); + xfs_btree_log_recs(cur, lbp, lrecs + 1, lrecs + rrecs); + } + + XFS_BTREE_STATS_INC(cur, join); + + /* + * Fix up the number of records and right block pointer in the + * surviving block, and log it. + */ + xfs_btree_set_numrecs(left, lrecs + rrecs); + xfs_btree_get_sibling(cur, right, &cptr, XFS_BB_RIGHTSIB); + xfs_btree_set_sibling(cur, left, &cptr, XFS_BB_RIGHTSIB); + xfs_btree_log_block(cur, lbp, XFS_BB_NUMRECS | XFS_BB_RIGHTSIB); + + /* If there is a right sibling, point it to the remaining block. */ + xfs_btree_get_sibling(cur, left, &cptr, XFS_BB_RIGHTSIB); + if (!xfs_btree_ptr_is_null(cur, &cptr)) { + error = xfs_btree_read_buf_block(cur, &cptr, 0, &rrblock, &rrbp); + if (error) + goto error0; + xfs_btree_set_sibling(cur, rrblock, &lptr, XFS_BB_LEFTSIB); + xfs_btree_log_block(cur, rrbp, XFS_BB_LEFTSIB); + } + + /* Free the deleted block. */ + error = xfs_btree_free_block(cur, rbp); + if (error) + goto error0; + + /* + * If we joined with the left neighbor, set the buffer in the + * cursor to the left block, and fix up the index. + */ + if (bp != lbp) { + cur->bc_levels[level].bp = lbp; + cur->bc_levels[level].ptr += lrecs; + cur->bc_levels[level].ra = 0; + } + /* + * If we joined with the right neighbor and there's a level above + * us, increment the cursor at that level. + */ + else if (cur->bc_ops->type == XFS_BTREE_TYPE_INODE || + level + 1 < cur->bc_nlevels) { + error = xfs_btree_increment(cur, level + 1, &i); + if (error) + goto error0; + } + + /* + * Readjust the ptr at this level if it's not a leaf, since it's + * still pointing at the deletion point, which makes the cursor + * inconsistent. If this makes the ptr 0, the caller fixes it up. + * We can't use decrement because it would change the next level up. + */ + if (level > 0) + cur->bc_levels[level].ptr--; + + /* + * We combined blocks, so we have to update the parent keys if the + * btree supports overlapped intervals. However, + * bc_levels[level + 1].ptr points to the old block so that the caller + * knows which record to delete. Therefore, the caller must be savvy + * enough to call updkeys for us if we return stat == 2. The other + * exit points from this function don't require deletions further up + * the tree, so they can call updkeys directly. + */ + + /* Return value means the next level up has something to do. */ + *stat = 2; + return 0; + +error0: + if (tcur) + xfs_btree_del_cursor(tcur, XFS_BTREE_ERROR); + return error; +} + +/* + * Delete the record pointed to by cur. + * The cursor refers to the place where the record was (could be inserted) + * when the operation returns. + */ +int /* error */ +xfs_btree_delete( + struct xfs_btree_cur *cur, + int *stat) /* success/failure */ +{ + int error; /* error return value */ + int level; + int i; + bool joined = false; + + /* + * Go up the tree, starting at leaf level. + * + * If 2 is returned then a join was done; go to the next level. + * Otherwise we are done. + */ + for (level = 0, i = 2; i == 2; level++) { + error = xfs_btree_delrec(cur, level, &i); + if (error) + goto error0; + if (i == 2) + joined = true; + } + + /* + * If we combined blocks as part of deleting the record, delrec won't + * have updated the parent high keys so we have to do that here. + */ + if (joined && (cur->bc_ops->geom_flags & XFS_BTGEO_OVERLAPPING)) { + error = xfs_btree_updkeys_force(cur, 0); + if (error) + goto error0; + } + + if (i == 0) { + for (level = 1; level < cur->bc_nlevels; level++) { + if (cur->bc_levels[level].ptr == 0) { + error = xfs_btree_decrement(cur, level, &i); + if (error) + goto error0; + break; + } + } + } + + *stat = i; + return 0; +error0: + return error; +} + +/* + * Get the data from the pointed-to record. + */ +int /* error */ +xfs_btree_get_rec( + struct xfs_btree_cur *cur, /* btree cursor */ + union xfs_btree_rec **recp, /* output: btree record */ + int *stat) /* output: success/failure */ +{ + struct xfs_btree_block *block; /* btree block */ + struct xfs_buf *bp; /* buffer pointer */ + int ptr; /* record number */ +#ifdef DEBUG + int error; /* error return value */ +#endif + + ptr = cur->bc_levels[0].ptr; + block = xfs_btree_get_block(cur, 0, &bp); + +#ifdef DEBUG + error = xfs_btree_check_block(cur, block, 0, bp); + if (error) + return error; +#endif + + /* + * Off the right end or left end, return failure. + */ + if (ptr > xfs_btree_get_numrecs(block) || ptr <= 0) { + *stat = 0; + return 0; + } + + /* + * Point to the record and extract its data. + */ + *recp = xfs_btree_rec_addr(cur, ptr, block); + *stat = 1; + return 0; +} + +/* Visit a block in a btree. */ +STATIC int +xfs_btree_visit_block( + struct xfs_btree_cur *cur, + int level, + xfs_btree_visit_blocks_fn fn, + void *data) +{ + struct xfs_btree_block *block; + struct xfs_buf *bp; + union xfs_btree_ptr rptr, bufptr; + int error; + + /* do right sibling readahead */ + xfs_btree_readahead(cur, level, XFS_BTCUR_RIGHTRA); + block = xfs_btree_get_block(cur, level, &bp); + + /* process the block */ + error = fn(cur, level, data); + if (error) + return error; + + /* now read rh sibling block for next iteration */ + xfs_btree_get_sibling(cur, block, &rptr, XFS_BB_RIGHTSIB); + if (xfs_btree_ptr_is_null(cur, &rptr)) + return -ENOENT; + + /* + * We only visit blocks once in this walk, so we have to avoid the + * internal xfs_btree_lookup_get_block() optimisation where it will + * return the same block without checking if the right sibling points + * back to us and creates a cyclic reference in the btree. + */ + xfs_btree_buf_to_ptr(cur, bp, &bufptr); + if (xfs_btree_ptrs_equal(cur, &rptr, &bufptr)) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + + return xfs_btree_lookup_get_block(cur, level, &rptr, &block); +} + + +/* Visit every block in a btree. */ +int +xfs_btree_visit_blocks( + struct xfs_btree_cur *cur, + xfs_btree_visit_blocks_fn fn, + unsigned int flags, + void *data) +{ + union xfs_btree_ptr lptr; + int level; + struct xfs_btree_block *block = NULL; + int error = 0; + + xfs_btree_init_ptr_from_cur(cur, &lptr); + + /* for each level */ + for (level = cur->bc_nlevels - 1; level >= 0; level--) { + /* grab the left hand block */ + error = xfs_btree_lookup_get_block(cur, level, &lptr, &block); + if (error) + return error; + + /* readahead the left most block for the next level down */ + if (level > 0) { + union xfs_btree_ptr *ptr; + + ptr = xfs_btree_ptr_addr(cur, 1, block); + xfs_btree_readahead_ptr(cur, ptr, 1); + + /* save for the next iteration of the loop */ + xfs_btree_copy_ptrs(cur, &lptr, ptr, 1); + + if (!(flags & XFS_BTREE_VISIT_LEAVES)) + continue; + } else if (!(flags & XFS_BTREE_VISIT_RECORDS)) { + continue; + } + + /* for each buffer in the level */ + do { + error = xfs_btree_visit_block(cur, level, fn, data); + } while (!error); + + if (error != -ENOENT) + return error; + } + + return 0; +} + +/* + * Change the owner of a btree. + * + * The mechanism we use here is ordered buffer logging. Because we don't know + * how many buffers were are going to need to modify, we don't really want to + * have to make transaction reservations for the worst case of every buffer in a + * full size btree as that may be more space that we can fit in the log.... + * + * We do the btree walk in the most optimal manner possible - we have sibling + * pointers so we can just walk all the blocks on each level from left to right + * in a single pass, and then move to the next level and do the same. We can + * also do readahead on the sibling pointers to get IO moving more quickly, + * though for slow disks this is unlikely to make much difference to performance + * as the amount of CPU work we have to do before moving to the next block is + * relatively small. + * + * For each btree block that we load, modify the owner appropriately, set the + * buffer as an ordered buffer and log it appropriately. We need to ensure that + * we mark the region we change dirty so that if the buffer is relogged in + * a subsequent transaction the changes we make here as an ordered buffer are + * correctly relogged in that transaction. If we are in recovery context, then + * just queue the modified buffer as delayed write buffer so the transaction + * recovery completion writes the changes to disk. + */ +struct xfs_btree_block_change_owner_info { + uint64_t new_owner; + struct list_head *buffer_list; +}; + +static int +xfs_btree_block_change_owner( + struct xfs_btree_cur *cur, + int level, + void *data) +{ + struct xfs_btree_block_change_owner_info *bbcoi = data; + struct xfs_btree_block *block; + struct xfs_buf *bp; + + /* modify the owner */ + block = xfs_btree_get_block(cur, level, &bp); + if (cur->bc_ops->ptr_len == XFS_BTREE_LONG_PTR_LEN) { + if (block->bb_u.l.bb_owner == cpu_to_be64(bbcoi->new_owner)) + return 0; + block->bb_u.l.bb_owner = cpu_to_be64(bbcoi->new_owner); + } else { + if (block->bb_u.s.bb_owner == cpu_to_be32(bbcoi->new_owner)) + return 0; + block->bb_u.s.bb_owner = cpu_to_be32(bbcoi->new_owner); + } + + /* + * If the block is a root block hosted in an inode, we might not have a + * buffer pointer here and we shouldn't attempt to log the change as the + * information is already held in the inode and discarded when the root + * block is formatted into the on-disk inode fork. We still change it, + * though, so everything is consistent in memory. + */ + if (!bp) { + ASSERT(cur->bc_ops->type == XFS_BTREE_TYPE_INODE); + ASSERT(level == cur->bc_nlevels - 1); + return 0; + } + + if (cur->bc_tp) { + if (!xfs_trans_ordered_buf(cur->bc_tp, bp)) { + xfs_btree_log_block(cur, bp, XFS_BB_OWNER); + return -EAGAIN; + } + } else { + xfs_buf_delwri_queue(bp, bbcoi->buffer_list); + } + + return 0; +} + +int +xfs_btree_change_owner( + struct xfs_btree_cur *cur, + uint64_t new_owner, + struct list_head *buffer_list) +{ + struct xfs_btree_block_change_owner_info bbcoi; + + bbcoi.new_owner = new_owner; + bbcoi.buffer_list = buffer_list; + + return xfs_btree_visit_blocks(cur, xfs_btree_block_change_owner, + XFS_BTREE_VISIT_ALL, &bbcoi); +} + +/* Verify the v5 fields of a long-format btree block. */ +xfs_failaddr_t +xfs_btree_fsblock_v5hdr_verify( + struct xfs_buf *bp, + uint64_t owner) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_btree_block *block = XFS_BUF_TO_BLOCK(bp); + + if (!xfs_has_crc(mp)) + return __this_address; + if (!uuid_equal(&block->bb_u.l.bb_uuid, &mp->m_sb.sb_meta_uuid)) + return __this_address; + if (block->bb_u.l.bb_blkno != cpu_to_be64(xfs_buf_daddr(bp))) + return __this_address; + if (owner != XFS_RMAP_OWN_UNKNOWN && + be64_to_cpu(block->bb_u.l.bb_owner) != owner) + return __this_address; + return NULL; +} + +/* Verify a long-format btree block. */ +xfs_failaddr_t +xfs_btree_fsblock_verify( + struct xfs_buf *bp, + unsigned int max_recs) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_btree_block *block = XFS_BUF_TO_BLOCK(bp); + xfs_fsblock_t fsb; + xfs_failaddr_t fa; + + ASSERT(!xfs_buftarg_is_mem(bp->b_target)); + + /* numrecs verification */ + if (be16_to_cpu(block->bb_numrecs) > max_recs) + return __this_address; + + /* sibling pointer verification */ + fsb = XFS_DADDR_TO_FSB(mp, xfs_buf_daddr(bp)); + fa = xfs_btree_check_fsblock_siblings(mp, fsb, + block->bb_u.l.bb_leftsib); + if (!fa) + fa = xfs_btree_check_fsblock_siblings(mp, fsb, + block->bb_u.l.bb_rightsib); + return fa; +} + +/* Verify an in-memory btree block. */ +xfs_failaddr_t +xfs_btree_memblock_verify( + struct xfs_buf *bp, + unsigned int max_recs) +{ + struct xfs_btree_block *block = XFS_BUF_TO_BLOCK(bp); + struct xfs_buftarg *btp = bp->b_target; + xfs_failaddr_t fa; + xfbno_t bno; + + ASSERT(xfs_buftarg_is_mem(bp->b_target)); + + /* numrecs verification */ + if (be16_to_cpu(block->bb_numrecs) > max_recs) + return __this_address; + + /* sibling pointer verification */ + bno = xfs_daddr_to_xfbno(xfs_buf_daddr(bp)); + fa = xfs_btree_check_memblock_siblings(btp, bno, + block->bb_u.l.bb_leftsib); + if (fa) + return fa; + fa = xfs_btree_check_memblock_siblings(btp, bno, + block->bb_u.l.bb_rightsib); + if (fa) + return fa; + + return NULL; +} +/** + * xfs_btree_agblock_v5hdr_verify() -- verify the v5 fields of a short-format + * btree block + * + * @bp: buffer containing the btree block + */ +xfs_failaddr_t +xfs_btree_agblock_v5hdr_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_btree_block *block = XFS_BUF_TO_BLOCK(bp); + struct xfs_perag *pag = bp->b_pag; + + if (!xfs_has_crc(mp)) + return __this_address; + if (!uuid_equal(&block->bb_u.s.bb_uuid, &mp->m_sb.sb_meta_uuid)) + return __this_address; + if (block->bb_u.s.bb_blkno != cpu_to_be64(xfs_buf_daddr(bp))) + return __this_address; + if (pag && be32_to_cpu(block->bb_u.s.bb_owner) != pag_agno(pag)) + return __this_address; + return NULL; +} + +/** + * xfs_btree_agblock_verify() -- verify a short-format btree block + * + * @bp: buffer containing the btree block + * @max_recs: maximum records allowed in this btree node + */ +xfs_failaddr_t +xfs_btree_agblock_verify( + struct xfs_buf *bp, + unsigned int max_recs) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_btree_block *block = XFS_BUF_TO_BLOCK(bp); + xfs_agblock_t agbno; + xfs_failaddr_t fa; + + ASSERT(!xfs_buftarg_is_mem(bp->b_target)); + + /* numrecs verification */ + if (be16_to_cpu(block->bb_numrecs) > max_recs) + return __this_address; + + /* sibling pointer verification */ + agbno = xfs_daddr_to_agbno(mp, xfs_buf_daddr(bp)); + fa = xfs_btree_check_agblock_siblings(bp->b_pag, agbno, + block->bb_u.s.bb_leftsib); + if (!fa) + fa = xfs_btree_check_agblock_siblings(bp->b_pag, agbno, + block->bb_u.s.bb_rightsib); + return fa; +} + +/* + * For the given limits on leaf and keyptr records per block, calculate the + * height of the tree needed to index the number of leaf records. + */ +unsigned int +xfs_btree_compute_maxlevels( + const unsigned int *limits, + unsigned long long records) +{ + unsigned long long level_blocks = howmany_64(records, limits[0]); + unsigned int height = 1; + + while (level_blocks > 1) { + level_blocks = howmany_64(level_blocks, limits[1]); + height++; + } + + return height; +} + +/* + * For the given limits on leaf and keyptr records per block, calculate the + * number of blocks needed to index the given number of leaf records. + */ +unsigned long long +xfs_btree_calc_size( + const unsigned int *limits, + unsigned long long records) +{ + unsigned long long level_blocks = howmany_64(records, limits[0]); + unsigned long long blocks = level_blocks; + + while (level_blocks > 1) { + level_blocks = howmany_64(level_blocks, limits[1]); + blocks += level_blocks; + } + + return blocks; +} + +/* + * Given a number of available blocks for the btree to consume with records and + * pointers, calculate the height of the tree needed to index all the records + * that space can hold based on the number of pointers each interior node + * holds. + * + * We start by assuming a single level tree consumes a single block, then track + * the number of blocks each node level consumes until we no longer have space + * to store the next node level. At this point, we are indexing all the leaf + * blocks in the space, and there's no more free space to split the tree any + * further. That's our maximum btree height. + */ +unsigned int +xfs_btree_space_to_height( + const unsigned int *limits, + unsigned long long leaf_blocks) +{ + /* + * The root btree block can have fewer than minrecs pointers in it + * because the tree might not be big enough to require that amount of + * fanout. Hence it has a minimum size of 2 pointers, not limits[1]. + */ + unsigned long long node_blocks = 2; + unsigned long long blocks_left = leaf_blocks - 1; + unsigned int height = 1; + + if (leaf_blocks < 1) + return 0; + + while (node_blocks < blocks_left) { + blocks_left -= node_blocks; + node_blocks *= limits[1]; + height++; + } + + return height; +} + +/* + * Query a regular btree for all records overlapping a given interval. + * Start with a LE lookup of the key of low_rec and return all records + * until we find a record with a key greater than the key of high_rec. + */ +STATIC int +xfs_btree_simple_query_range( + struct xfs_btree_cur *cur, + const union xfs_btree_key *low_key, + const union xfs_btree_key *high_key, + xfs_btree_query_range_fn fn, + void *priv) +{ + union xfs_btree_rec *recp; + union xfs_btree_key rec_key; + int stat; + bool firstrec = true; + int error; + + ASSERT(cur->bc_ops->init_high_key_from_rec); + ASSERT(cur->bc_ops->cmp_two_keys); + + /* + * Find the leftmost record. The btree cursor must be set + * to the low record used to generate low_key. + */ + stat = 0; + error = xfs_btree_lookup(cur, XFS_LOOKUP_LE, &stat); + if (error) + goto out; + + /* Nothing? See if there's anything to the right. */ + if (!stat) { + error = xfs_btree_increment(cur, 0, &stat); + if (error) + goto out; + } + + while (stat) { + /* Find the record. */ + error = xfs_btree_get_rec(cur, &recp, &stat); + if (error || !stat) + break; + + /* Skip if low_key > high_key(rec). */ + if (firstrec) { + cur->bc_ops->init_high_key_from_rec(&rec_key, recp); + firstrec = false; + if (xfs_btree_keycmp_gt(cur, low_key, &rec_key)) + goto advloop; + } + + /* Stop if low_key(rec) > high_key. */ + cur->bc_ops->init_key_from_rec(&rec_key, recp); + if (xfs_btree_keycmp_gt(cur, &rec_key, high_key)) + break; + + /* Callback */ + error = fn(cur, recp, priv); + if (error) + break; + +advloop: + /* Move on to the next record. */ + error = xfs_btree_increment(cur, 0, &stat); + if (error) + break; + } + +out: + return error; +} + +/* + * Query an overlapped interval btree for all records overlapping a given + * interval. This function roughly follows the algorithm given in + * "Interval Trees" of _Introduction to Algorithms_, which is section + * 14.3 in the 2nd and 3rd editions. + * + * First, generate keys for the low and high records passed in. + * + * For any leaf node, generate the high and low keys for the record. + * If the record keys overlap with the query low/high keys, pass the + * record to the function iterator. + * + * For any internal node, compare the low and high keys of each + * pointer against the query low/high keys. If there's an overlap, + * follow the pointer. + * + * As an optimization, we stop scanning a block when we find a low key + * that is greater than the query's high key. + */ +STATIC int +xfs_btree_overlapped_query_range( + struct xfs_btree_cur *cur, + const union xfs_btree_key *low_key, + const union xfs_btree_key *high_key, + xfs_btree_query_range_fn fn, + void *priv) +{ + union xfs_btree_ptr ptr; + union xfs_btree_ptr *pp; + union xfs_btree_key rec_key; + union xfs_btree_key rec_hkey; + union xfs_btree_key *lkp; + union xfs_btree_key *hkp; + union xfs_btree_rec *recp; + struct xfs_btree_block *block; + int level; + struct xfs_buf *bp; + int i; + int error; + + /* Load the root of the btree. */ + level = cur->bc_nlevels - 1; + xfs_btree_init_ptr_from_cur(cur, &ptr); + error = xfs_btree_lookup_get_block(cur, level, &ptr, &block); + if (error) + return error; + xfs_btree_get_block(cur, level, &bp); + trace_xfs_btree_overlapped_query_range(cur, level, bp); +#ifdef DEBUG + error = xfs_btree_check_block(cur, block, level, bp); + if (error) + goto out; +#endif + cur->bc_levels[level].ptr = 1; + + while (level < cur->bc_nlevels) { + block = xfs_btree_get_block(cur, level, &bp); + + /* End of node, pop back towards the root. */ + if (cur->bc_levels[level].ptr > + be16_to_cpu(block->bb_numrecs)) { +pop_up: + if (level < cur->bc_nlevels - 1) + cur->bc_levels[level + 1].ptr++; + level++; + continue; + } + + if (level == 0) { + /* Handle a leaf node. */ + recp = xfs_btree_rec_addr(cur, cur->bc_levels[0].ptr, + block); + + cur->bc_ops->init_high_key_from_rec(&rec_hkey, recp); + cur->bc_ops->init_key_from_rec(&rec_key, recp); + + /* + * If (query's high key < record's low key), then there + * are no more interesting records in this block. Pop + * up to the leaf level to find more record blocks. + * + * If (record's high key >= query's low key) and + * (query's high key >= record's low key), then + * this record overlaps the query range; callback. + */ + if (xfs_btree_keycmp_lt(cur, high_key, &rec_key)) + goto pop_up; + if (xfs_btree_keycmp_ge(cur, &rec_hkey, low_key)) { + error = fn(cur, recp, priv); + if (error) + break; + } + cur->bc_levels[level].ptr++; + continue; + } + + /* Handle an internal node. */ + lkp = xfs_btree_key_addr(cur, cur->bc_levels[level].ptr, block); + hkp = xfs_btree_high_key_addr(cur, cur->bc_levels[level].ptr, + block); + pp = xfs_btree_ptr_addr(cur, cur->bc_levels[level].ptr, block); + + /* + * If (query's high key < pointer's low key), then there are no + * more interesting keys in this block. Pop up one leaf level + * to continue looking for records. + * + * If (pointer's high key >= query's low key) and + * (query's high key >= pointer's low key), then + * this record overlaps the query range; follow pointer. + */ + if (xfs_btree_keycmp_lt(cur, high_key, lkp)) + goto pop_up; + if (xfs_btree_keycmp_ge(cur, hkp, low_key)) { + level--; + error = xfs_btree_lookup_get_block(cur, level, pp, + &block); + if (error) + goto out; + xfs_btree_get_block(cur, level, &bp); + trace_xfs_btree_overlapped_query_range(cur, level, bp); +#ifdef DEBUG + error = xfs_btree_check_block(cur, block, level, bp); + if (error) + goto out; +#endif + cur->bc_levels[level].ptr = 1; + continue; + } + cur->bc_levels[level].ptr++; + } + +out: + /* + * If we don't end this function with the cursor pointing at a record + * block, a subsequent non-error cursor deletion will not release + * node-level buffers, causing a buffer leak. This is quite possible + * with a zero-results range query, so release the buffers if we + * failed to return any results. + */ + if (cur->bc_levels[0].bp == NULL) { + for (i = 0; i < cur->bc_nlevels; i++) { + if (cur->bc_levels[i].bp) { + xfs_trans_brelse(cur->bc_tp, + cur->bc_levels[i].bp); + cur->bc_levels[i].bp = NULL; + cur->bc_levels[i].ptr = 0; + cur->bc_levels[i].ra = 0; + } + } + } + + return error; +} + +static inline void +xfs_btree_key_from_irec( + struct xfs_btree_cur *cur, + union xfs_btree_key *key, + const union xfs_btree_irec *irec) +{ + union xfs_btree_rec rec; + + cur->bc_rec = *irec; + cur->bc_ops->init_rec_from_cur(cur, &rec); + cur->bc_ops->init_key_from_rec(key, &rec); +} + +/* + * Query a btree for all records overlapping a given interval of keys. The + * supplied function will be called with each record found; return one of the + * XFS_BTREE_QUERY_RANGE_{CONTINUE,ABORT} values or the usual negative error + * code. This function returns -ECANCELED, zero, or a negative error code. + */ +int +xfs_btree_query_range( + struct xfs_btree_cur *cur, + const union xfs_btree_irec *low_rec, + const union xfs_btree_irec *high_rec, + xfs_btree_query_range_fn fn, + void *priv) +{ + union xfs_btree_key low_key; + union xfs_btree_key high_key; + + /* Find the keys of both ends of the interval. */ + xfs_btree_key_from_irec(cur, &high_key, high_rec); + xfs_btree_key_from_irec(cur, &low_key, low_rec); + + /* Enforce low key <= high key. */ + if (!xfs_btree_keycmp_le(cur, &low_key, &high_key)) + return -EINVAL; + + if (!(cur->bc_ops->geom_flags & XFS_BTGEO_OVERLAPPING)) + return xfs_btree_simple_query_range(cur, &low_key, + &high_key, fn, priv); + return xfs_btree_overlapped_query_range(cur, &low_key, &high_key, + fn, priv); +} + +/* Query a btree for all records. */ +int +xfs_btree_query_all( + struct xfs_btree_cur *cur, + xfs_btree_query_range_fn fn, + void *priv) +{ + union xfs_btree_key low_key; + union xfs_btree_key high_key; + + memset(&cur->bc_rec, 0, sizeof(cur->bc_rec)); + memset(&low_key, 0, sizeof(low_key)); + memset(&high_key, 0xFF, sizeof(high_key)); + + return xfs_btree_simple_query_range(cur, &low_key, &high_key, fn, priv); +} + +static int +xfs_btree_count_blocks_helper( + struct xfs_btree_cur *cur, + int level, + void *data) +{ + xfs_filblks_t *blocks = data; + (*blocks)++; + + return 0; +} + +/* Count the blocks in a btree and return the result in *blocks. */ +int +xfs_btree_count_blocks( + struct xfs_btree_cur *cur, + xfs_filblks_t *blocks) +{ + *blocks = 0; + return xfs_btree_visit_blocks(cur, xfs_btree_count_blocks_helper, + XFS_BTREE_VISIT_ALL, blocks); +} + +/* Compare two btree pointers. */ +int +xfs_btree_cmp_two_ptrs( + struct xfs_btree_cur *cur, + const union xfs_btree_ptr *a, + const union xfs_btree_ptr *b) +{ + if (cur->bc_ops->ptr_len == XFS_BTREE_LONG_PTR_LEN) + return cmp_int(be64_to_cpu(a->l), be64_to_cpu(b->l)); + return cmp_int(be32_to_cpu(a->s), be32_to_cpu(b->s)); +} + +struct xfs_btree_has_records { + /* Keys for the start and end of the range we want to know about. */ + union xfs_btree_key start_key; + union xfs_btree_key end_key; + + /* Mask for key comparisons, if desired. */ + const union xfs_btree_key *key_mask; + + /* Highest record key we've seen so far. */ + union xfs_btree_key high_key; + + enum xbtree_recpacking outcome; +}; + +STATIC int +xfs_btree_has_records_helper( + struct xfs_btree_cur *cur, + const union xfs_btree_rec *rec, + void *priv) +{ + union xfs_btree_key rec_key; + union xfs_btree_key rec_high_key; + struct xfs_btree_has_records *info = priv; + enum xbtree_key_contig key_contig; + + cur->bc_ops->init_key_from_rec(&rec_key, rec); + + if (info->outcome == XBTREE_RECPACKING_EMPTY) { + info->outcome = XBTREE_RECPACKING_SPARSE; + + /* + * If the first record we find does not overlap the start key, + * then there is a hole at the start of the search range. + * Classify this as sparse and stop immediately. + */ + if (xfs_btree_masked_keycmp_lt(cur, &info->start_key, &rec_key, + info->key_mask)) + return -ECANCELED; + } else { + /* + * If a subsequent record does not overlap with the any record + * we've seen so far, there is a hole in the middle of the + * search range. Classify this as sparse and stop. + * If the keys overlap and this btree does not allow overlap, + * signal corruption. + */ + key_contig = cur->bc_ops->keys_contiguous(cur, &info->high_key, + &rec_key, info->key_mask); + if (key_contig == XBTREE_KEY_OVERLAP && + !(cur->bc_ops->geom_flags & XFS_BTGEO_OVERLAPPING)) + return -EFSCORRUPTED; + if (key_contig == XBTREE_KEY_GAP) + return -ECANCELED; + } + + /* + * If high_key(rec) is larger than any other high key we've seen, + * remember it for later. + */ + cur->bc_ops->init_high_key_from_rec(&rec_high_key, rec); + if (xfs_btree_masked_keycmp_gt(cur, &rec_high_key, &info->high_key, + info->key_mask)) + info->high_key = rec_high_key; /* struct copy */ + + return 0; +} + +/* + * Scan part of the keyspace of a btree and tell us if that keyspace does not + * map to any records; is fully mapped to records; or is partially mapped to + * records. This is the btree record equivalent to determining if a file is + * sparse. + * + * For most btree types, the record scan should use all available btree key + * fields to compare the keys encountered. These callers should pass NULL for + * @mask. However, some callers (e.g. scanning physical space in the rmapbt) + * want to ignore some part of the btree record keyspace when performing the + * comparison. These callers should pass in a union xfs_btree_key object with + * the fields that *should* be a part of the comparison set to any nonzero + * value, and the rest zeroed. + */ +int +xfs_btree_has_records( + struct xfs_btree_cur *cur, + const union xfs_btree_irec *low, + const union xfs_btree_irec *high, + const union xfs_btree_key *mask, + enum xbtree_recpacking *outcome) +{ + struct xfs_btree_has_records info = { + .outcome = XBTREE_RECPACKING_EMPTY, + .key_mask = mask, + }; + int error; + + /* Not all btrees support this operation. */ + if (!cur->bc_ops->keys_contiguous) { + ASSERT(0); + return -EOPNOTSUPP; + } + + xfs_btree_key_from_irec(cur, &info.start_key, low); + xfs_btree_key_from_irec(cur, &info.end_key, high); + + error = xfs_btree_query_range(cur, low, high, + xfs_btree_has_records_helper, &info); + if (error == -ECANCELED) + goto out; + if (error) + return error; + + if (info.outcome == XBTREE_RECPACKING_EMPTY) + goto out; + + /* + * If the largest high_key(rec) we saw during the walk is greater than + * the end of the search range, classify this as full. Otherwise, + * there is a hole at the end of the search range. + */ + if (xfs_btree_masked_keycmp_ge(cur, &info.high_key, &info.end_key, + mask)) + info.outcome = XBTREE_RECPACKING_FULL; + +out: + *outcome = info.outcome; + return 0; +} + +/* Are there more records in this btree? */ +bool +xfs_btree_has_more_records( + struct xfs_btree_cur *cur) +{ + struct xfs_btree_block *block; + struct xfs_buf *bp; + + block = xfs_btree_get_block(cur, 0, &bp); + + /* There are still records in this block. */ + if (cur->bc_levels[0].ptr < xfs_btree_get_numrecs(block)) + return true; + + /* There are more record blocks. */ + if (cur->bc_ops->ptr_len == XFS_BTREE_LONG_PTR_LEN) + return block->bb_u.l.bb_rightsib != cpu_to_be64(NULLFSBLOCK); + else + return block->bb_u.s.bb_rightsib != cpu_to_be32(NULLAGBLOCK); +} + +/* Set up all the btree cursor caches. */ +int __init +xfs_btree_init_cur_caches(void) +{ + int error; + + error = xfs_allocbt_init_cur_cache(); + if (error) + return error; + error = xfs_inobt_init_cur_cache(); + if (error) + goto err; + error = xfs_bmbt_init_cur_cache(); + if (error) + goto err; + error = xfs_rmapbt_init_cur_cache(); + if (error) + goto err; + error = xfs_refcountbt_init_cur_cache(); + if (error) + goto err; + error = xfs_rtrmapbt_init_cur_cache(); + if (error) + goto err; + error = xfs_rtrefcountbt_init_cur_cache(); + if (error) + goto err; + + return 0; +err: + xfs_btree_destroy_cur_caches(); + return error; +} + +/* Destroy all the btree cursor caches, if they've been allocated. */ +void +xfs_btree_destroy_cur_caches(void) +{ + xfs_allocbt_destroy_cur_cache(); + xfs_inobt_destroy_cur_cache(); + xfs_bmbt_destroy_cur_cache(); + xfs_rmapbt_destroy_cur_cache(); + xfs_refcountbt_destroy_cur_cache(); + xfs_rtrmapbt_destroy_cur_cache(); + xfs_rtrefcountbt_destroy_cur_cache(); +} + +/* Move the btree cursor before the first record. */ +int +xfs_btree_goto_left_edge( + struct xfs_btree_cur *cur) +{ + int stat = 0; + int error; + + memset(&cur->bc_rec, 0, sizeof(cur->bc_rec)); + error = xfs_btree_lookup(cur, XFS_LOOKUP_LE, &stat); + if (error) + return error; + if (!stat) + return 0; + + error = xfs_btree_decrement(cur, 0, &stat); + if (error) + return error; + if (stat != 0) { + ASSERT(0); + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + + return 0; +} + +/* Allocate a block for an inode-rooted metadata btree. */ +int +xfs_btree_alloc_metafile_block( + struct xfs_btree_cur *cur, + const union xfs_btree_ptr *start, + union xfs_btree_ptr *new, + int *stat) +{ + struct xfs_alloc_arg args = { + .mp = cur->bc_mp, + .tp = cur->bc_tp, + .resv = XFS_AG_RESV_METAFILE, + .minlen = 1, + .maxlen = 1, + .prod = 1, + }; + struct xfs_inode *ip = cur->bc_ino.ip; + int error; + + ASSERT(xfs_is_metadir_inode(ip)); + + xfs_rmap_inode_bmbt_owner(&args.oinfo, ip, cur->bc_ino.whichfork); + error = xfs_alloc_vextent_start_ag(&args, XFS_INODE_TO_FSB(ip)); + if (error) + return error; + if (args.fsbno == NULLFSBLOCK) { + *stat = 0; + return 0; + } + ASSERT(args.len == 1); + + xfs_metafile_resv_alloc_space(ip, &args); + + new->l = cpu_to_be64(args.fsbno); + *stat = 1; + return 0; +} + +/* Free a block from an inode-rooted metadata btree. */ +int +xfs_btree_free_metafile_block( + struct xfs_btree_cur *cur, + struct xfs_buf *bp) +{ + struct xfs_owner_info oinfo; + struct xfs_mount *mp = cur->bc_mp; + struct xfs_inode *ip = cur->bc_ino.ip; + struct xfs_trans *tp = cur->bc_tp; + xfs_fsblock_t fsbno = XFS_DADDR_TO_FSB(mp, xfs_buf_daddr(bp)); + int error; + + ASSERT(xfs_is_metadir_inode(ip)); + + xfs_rmap_inode_bmbt_owner(&oinfo, ip, cur->bc_ino.whichfork); + error = xfs_free_extent_later(tp, fsbno, 1, &oinfo, XFS_AG_RESV_METAFILE, + 0); + if (error) + return error; + + xfs_metafile_resv_free_space(ip, tp, 1); + return 0; +} diff --git a/fs/xfs/libxfs/xfs_btree.h b/fs/xfs/libxfs/xfs_btree.h new file mode 100644 index 000000000..60e78572e --- /dev/null +++ b/fs/xfs/libxfs/xfs_btree.h @@ -0,0 +1,715 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2000-2001,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_BTREE_H__ +#define __XFS_BTREE_H__ + +struct xfs_buf; +struct xfs_inode; +struct xfs_mount; +struct xfs_trans; +struct xfs_ifork; +struct xfs_perag; + +/* + * Generic key, ptr and record wrapper structures. + * + * These are disk format structures, and are converted where necessary + * by the btree specific code that needs to interpret them. + */ +union xfs_btree_ptr { + __be32 s; /* short form ptr */ + __be64 l; /* long form ptr */ +}; + +/* + * The in-core btree key. Overlapping btrees actually store two keys + * per pointer, so we reserve enough memory to hold both. The __*bigkey + * items should never be accessed directly. + */ +union xfs_btree_key { + struct xfs_bmbt_key bmbt; + xfs_bmdr_key_t bmbr; /* bmbt root block */ + xfs_alloc_key_t alloc; + struct xfs_inobt_key inobt; + struct xfs_rmap_key rmap; + struct xfs_rmap_key __rmap_bigkey[2]; + struct xfs_refcount_key refc; +}; + +union xfs_btree_rec { + struct xfs_bmbt_rec bmbt; + xfs_bmdr_rec_t bmbr; /* bmbt root block */ + struct xfs_alloc_rec alloc; + struct xfs_inobt_rec inobt; + struct xfs_rmap_rec rmap; + struct xfs_refcount_rec refc; +}; + +/* + * This nonsense is to make -wlint happy. + */ +#define XFS_LOOKUP_EQ ((xfs_lookup_t)XFS_LOOKUP_EQi) +#define XFS_LOOKUP_LE ((xfs_lookup_t)XFS_LOOKUP_LEi) +#define XFS_LOOKUP_GE ((xfs_lookup_t)XFS_LOOKUP_GEi) + +struct xfs_btree_ops; +uint32_t xfs_btree_magic(struct xfs_mount *mp, const struct xfs_btree_ops *ops); + +/* + * For logging record fields. + */ +#define XFS_BB_MAGIC (1u << 0) +#define XFS_BB_LEVEL (1u << 1) +#define XFS_BB_NUMRECS (1u << 2) +#define XFS_BB_LEFTSIB (1u << 3) +#define XFS_BB_RIGHTSIB (1u << 4) +#define XFS_BB_BLKNO (1u << 5) +#define XFS_BB_LSN (1u << 6) +#define XFS_BB_UUID (1u << 7) +#define XFS_BB_OWNER (1u << 8) +#define XFS_BB_NUM_BITS 5 +#define XFS_BB_ALL_BITS ((1u << XFS_BB_NUM_BITS) - 1) +#define XFS_BB_NUM_BITS_CRC 9 +#define XFS_BB_ALL_BITS_CRC ((1u << XFS_BB_NUM_BITS_CRC) - 1) + +/* + * Generic stats interface + */ +#define XFS_BTREE_STATS_INC(cur, stat) \ + XFS_STATS_INC_OFF((cur)->bc_mp, \ + (cur)->bc_ops->statoff + __XBTS_ ## stat) +#define XFS_BTREE_STATS_ADD(cur, stat, val) \ + XFS_STATS_ADD_OFF((cur)->bc_mp, \ + (cur)->bc_ops->statoff + __XBTS_ ## stat, val) + +enum xbtree_key_contig { + XBTREE_KEY_GAP = 0, + XBTREE_KEY_CONTIGUOUS, + XBTREE_KEY_OVERLAP, +}; + +/* + * Decide if these two numeric btree key fields are contiguous, overlapping, + * or if there's a gap between them. @x should be the field from the high + * key and @y should be the field from the low key. + */ +static inline enum xbtree_key_contig xbtree_key_contig(uint64_t x, uint64_t y) +{ + x++; + if (x < y) + return XBTREE_KEY_GAP; + if (x == y) + return XBTREE_KEY_CONTIGUOUS; + return XBTREE_KEY_OVERLAP; +} + +#define XFS_BTREE_LONG_PTR_LEN (sizeof(__be64)) +#define XFS_BTREE_SHORT_PTR_LEN (sizeof(__be32)) + +enum xfs_btree_type { + XFS_BTREE_TYPE_AG, + XFS_BTREE_TYPE_INODE, + XFS_BTREE_TYPE_MEM, +}; + +struct xfs_btree_ops { + const char *name; + + /* Type of btree - AG-rooted or inode-rooted */ + enum xfs_btree_type type; + + /* XFS_BTGEO_* flags that determine the geometry of the btree */ + unsigned int geom_flags; + + /* size of the key, pointer, and record structures */ + size_t key_len; + size_t ptr_len; + size_t rec_len; + + /* LRU refcount to set on each btree buffer created */ + unsigned int lru_refs; + + /* offset of btree stats array */ + unsigned int statoff; + + /* sick mask for health reporting (not for bmap btrees) */ + unsigned int sick_mask; + + /* cursor operations */ + struct xfs_btree_cur *(*dup_cursor)(struct xfs_btree_cur *); + void (*update_cursor)(struct xfs_btree_cur *src, + struct xfs_btree_cur *dst); + + /* update btree root pointer */ + void (*set_root)(struct xfs_btree_cur *cur, + const union xfs_btree_ptr *nptr, int level_change); + + /* block allocation / freeing */ + int (*alloc_block)(struct xfs_btree_cur *cur, + const union xfs_btree_ptr *start_bno, + union xfs_btree_ptr *new_bno, + int *stat); + int (*free_block)(struct xfs_btree_cur *cur, struct xfs_buf *bp); + + /* records in block/level */ + int (*get_minrecs)(struct xfs_btree_cur *cur, int level); + int (*get_maxrecs)(struct xfs_btree_cur *cur, int level); + + /* records on disk. Matter for the root in inode case. */ + int (*get_dmaxrecs)(struct xfs_btree_cur *cur, int level); + + /* init values of btree structures */ + void (*init_key_from_rec)(union xfs_btree_key *key, + const union xfs_btree_rec *rec); + void (*init_rec_from_cur)(struct xfs_btree_cur *cur, + union xfs_btree_rec *rec); + void (*init_ptr_from_cur)(struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr); + void (*init_high_key_from_rec)(union xfs_btree_key *key, + const union xfs_btree_rec *rec); + + /* + * Compare key value and cursor value -- positive if key > cur, + * negative if key < cur, and zero if equal. + */ + int (*cmp_key_with_cur)(struct xfs_btree_cur *cur, + const union xfs_btree_key *key); + + /* + * Compare key1 and key2 -- positive if key1 > key2, negative if + * key1 < key2, and zero if equal. If the @mask parameter is non NULL, + * each key field to be used in the comparison must contain a nonzero + * value. + */ + int (*cmp_two_keys)(struct xfs_btree_cur *cur, + const union xfs_btree_key *key1, + const union xfs_btree_key *key2, + const union xfs_btree_key *mask); + + const struct xfs_buf_ops *buf_ops; + + /* check that k1 is lower than k2 */ + int (*keys_inorder)(struct xfs_btree_cur *cur, + const union xfs_btree_key *k1, + const union xfs_btree_key *k2); + + /* check that r1 is lower than r2 */ + int (*recs_inorder)(struct xfs_btree_cur *cur, + const union xfs_btree_rec *r1, + const union xfs_btree_rec *r2); + + /* + * Are these two btree keys immediately adjacent? + * + * Given two btree keys @key1 and @key2, decide if it is impossible for + * there to be a third btree key K satisfying the relationship + * @key1 < K < @key2. To determine if two btree records are + * immediately adjacent, @key1 should be the high key of the first + * record and @key2 should be the low key of the second record. + * If the @mask parameter is non NULL, each key field to be used in the + * comparison must contain a nonzero value. + */ + enum xbtree_key_contig (*keys_contiguous)(struct xfs_btree_cur *cur, + const union xfs_btree_key *key1, + const union xfs_btree_key *key2, + const union xfs_btree_key *mask); + + /* + * Reallocate the space for if_broot to fit the number of records. + * Move the records and pointers in if_broot to fit the new size. When + * shrinking this will eliminate holes between the records and pointers + * created by the caller. When growing this will create holes to be + * filled in by the caller. + * + * The caller must not request to add more records than would fit in + * the on-disk inode root. If the if_broot is currently NULL, then if + * we are adding records, one will be allocated. The caller must also + * not request that the number of records go below zero, although it + * can go to zero. + */ + struct xfs_btree_block *(*broot_realloc)(struct xfs_btree_cur *cur, + unsigned int new_numrecs); +}; + +/* btree geometry flags */ +#define XFS_BTGEO_OVERLAPPING (1U << 0) /* overlapping intervals */ +#define XFS_BTGEO_IROOT_RECORDS (1U << 1) /* iroot can store records */ + +union xfs_btree_irec { + struct xfs_alloc_rec_incore a; + struct xfs_bmbt_irec b; + struct xfs_inobt_rec_incore i; + struct xfs_rmap_irec r; + struct xfs_refcount_irec rc; +}; + +struct xfs_btree_level { + /* buffer pointer */ + struct xfs_buf *bp; + + /* key/record number */ + uint16_t ptr; + + /* readahead info */ +#define XFS_BTCUR_LEFTRA (1 << 0) /* left sibling has been read-ahead */ +#define XFS_BTCUR_RIGHTRA (1 << 1) /* right sibling has been read-ahead */ + uint16_t ra; +}; + +/* + * Btree cursor structure. + * This collects all information needed by the btree code in one place. + */ +struct xfs_btree_cur +{ + struct xfs_trans *bc_tp; /* transaction we're in, if any */ + struct xfs_mount *bc_mp; /* file system mount struct */ + const struct xfs_btree_ops *bc_ops; + struct kmem_cache *bc_cache; /* cursor cache */ + unsigned int bc_flags; /* btree features - below */ + union xfs_btree_irec bc_rec; /* current insert/search record value */ + uint8_t bc_nlevels; /* number of levels in the tree */ + uint8_t bc_maxlevels; /* maximum levels for this btree type */ + struct xfs_group *bc_group; + + /* per-type information */ + union { + struct { + struct xfs_inode *ip; + short forksize; + char whichfork; + struct xbtree_ifakeroot *ifake; /* for staging cursor */ + } bc_ino; + struct { + struct xfs_buf *agbp; + struct xbtree_afakeroot *afake; /* for staging cursor */ + } bc_ag; + struct { + struct xfbtree *xfbtree; + } bc_mem; + }; + + /* per-format private data */ + union { + struct { + int allocated; + } bc_bmap; /* bmapbt */ + struct { + unsigned int nr_ops; /* # record updates */ + unsigned int shape_changes; /* # of extent splits */ + } bc_refc; /* refcountbt/rtrefcountbt */ + }; + + /* Must be at the end of the struct! */ + struct xfs_btree_level bc_levels[]; +}; + +/* + * Compute the size of a btree cursor that can handle a btree of a given + * height. The bc_levels array handles node and leaf blocks, so its size + * is exactly nlevels. + */ +static inline size_t +xfs_btree_cur_sizeof(unsigned int nlevels) +{ + return struct_size_t(struct xfs_btree_cur, bc_levels, nlevels); +} + +/* cursor state flags */ +/* + * The root of this btree is a fakeroot structure so that we can stage a btree + * rebuild without leaving it accessible via primary metadata. The ops struct + * is dynamically allocated and must be freed when the cursor is deleted. + */ +#define XFS_BTREE_STAGING (1U << 0) + +/* We are converting a delalloc reservation (only for bmbt btrees) */ +#define XFS_BTREE_BMBT_WASDEL (1U << 1) + +/* For extent swap, ignore owner check in verifier (only for bmbt btrees) */ +#define XFS_BTREE_BMBT_INVALID_OWNER (1U << 2) + +/* Cursor is active (only for allocbt btrees) */ +#define XFS_BTREE_ALLOCBT_ACTIVE (1U << 3) + +#define XFS_BTREE_NOERROR 0 +#define XFS_BTREE_ERROR 1 + +/* + * Convert from buffer to btree block header. + */ +#define XFS_BUF_TO_BLOCK(bp) ((struct xfs_btree_block *)((bp)->b_addr)) + +xfs_failaddr_t __xfs_btree_check_block(struct xfs_btree_cur *cur, + struct xfs_btree_block *block, int level, struct xfs_buf *bp); +int __xfs_btree_check_ptr(struct xfs_btree_cur *cur, + const union xfs_btree_ptr *ptr, int index, int level); + +/* + * Check that block header is ok. + */ +int +xfs_btree_check_block( + struct xfs_btree_cur *cur, /* btree cursor */ + struct xfs_btree_block *block, /* generic btree block pointer */ + int level, /* level of the btree block */ + struct xfs_buf *bp); /* buffer containing block, if any */ + +/* + * Delete the btree cursor. + */ +void +xfs_btree_del_cursor( + struct xfs_btree_cur *cur, /* btree cursor */ + int error); /* del because of error */ + +/* + * Duplicate the btree cursor. + * Allocate a new one, copy the record, re-get the buffers. + */ +int /* error */ +xfs_btree_dup_cursor( + struct xfs_btree_cur *cur, /* input cursor */ + struct xfs_btree_cur **ncur);/* output cursor */ + +/* + * Compute first and last byte offsets for the fields given. + * Interprets the offsets table, which contains struct field offsets. + */ +void +xfs_btree_offsets( + uint32_t fields, /* bitmask of fields */ + const short *offsets,/* table of field offsets */ + int nbits, /* number of bits to inspect */ + int *first, /* output: first byte offset */ + int *last); /* output: last byte offset */ + +/* + * Initialise a new btree block header + */ +void xfs_btree_init_buf(struct xfs_mount *mp, struct xfs_buf *bp, + const struct xfs_btree_ops *ops, __u16 level, __u16 numrecs, + __u64 owner); +void xfs_btree_init_block(struct xfs_mount *mp, + struct xfs_btree_block *buf, const struct xfs_btree_ops *ops, + __u16 level, __u16 numrecs, __u64 owner); + +/* + * Common btree core entry points. + */ +int xfs_btree_increment(struct xfs_btree_cur *, int, int *); +int xfs_btree_decrement(struct xfs_btree_cur *, int, int *); +int xfs_btree_lookup(struct xfs_btree_cur *, xfs_lookup_t, int *); +int xfs_btree_update(struct xfs_btree_cur *, union xfs_btree_rec *); +int xfs_btree_new_iroot(struct xfs_btree_cur *, int *, int *); +int xfs_btree_insert(struct xfs_btree_cur *, int *); +int xfs_btree_delete(struct xfs_btree_cur *, int *); +int xfs_btree_get_rec(struct xfs_btree_cur *, union xfs_btree_rec **, int *); +int xfs_btree_change_owner(struct xfs_btree_cur *cur, uint64_t new_owner, + struct list_head *buffer_list); + +/* + * btree block CRC helpers + */ +void xfs_btree_fsblock_calc_crc(struct xfs_buf *); +bool xfs_btree_fsblock_verify_crc(struct xfs_buf *); +void xfs_btree_agblock_calc_crc(struct xfs_buf *); +bool xfs_btree_agblock_verify_crc(struct xfs_buf *); + +/* + * Internal btree helpers also used by xfs_bmap.c. + */ +void xfs_btree_log_block(struct xfs_btree_cur *, struct xfs_buf *, uint32_t); +void xfs_btree_log_recs(struct xfs_btree_cur *, struct xfs_buf *, int, int); + +/* + * Helpers. + */ +static inline int xfs_btree_get_numrecs(const struct xfs_btree_block *block) +{ + return be16_to_cpu(block->bb_numrecs); +} + +static inline void xfs_btree_set_numrecs(struct xfs_btree_block *block, + uint16_t numrecs) +{ + block->bb_numrecs = cpu_to_be16(numrecs); +} + +static inline int xfs_btree_get_level(const struct xfs_btree_block *block) +{ + return be16_to_cpu(block->bb_level); +} + + +/* + * Min and max functions for extlen, agblock, fileoff, and filblks types. + */ +#define XFS_EXTLEN_MIN(a,b) min_t(xfs_extlen_t, (a), (b)) +#define XFS_EXTLEN_MAX(a,b) max_t(xfs_extlen_t, (a), (b)) +#define XFS_AGBLOCK_MIN(a,b) min_t(xfs_agblock_t, (a), (b)) +#define XFS_AGBLOCK_MAX(a,b) max_t(xfs_agblock_t, (a), (b)) +#define XFS_FILEOFF_MIN(a,b) min_t(xfs_fileoff_t, (a), (b)) +#define XFS_FILEOFF_MAX(a,b) max_t(xfs_fileoff_t, (a), (b)) +#define XFS_FILBLKS_MIN(a,b) min_t(xfs_filblks_t, (a), (b)) +#define XFS_FILBLKS_MAX(a,b) max_t(xfs_filblks_t, (a), (b)) + +xfs_failaddr_t xfs_btree_agblock_v5hdr_verify(struct xfs_buf *bp); +xfs_failaddr_t xfs_btree_agblock_verify(struct xfs_buf *bp, + unsigned int max_recs); +xfs_failaddr_t xfs_btree_fsblock_v5hdr_verify(struct xfs_buf *bp, + uint64_t owner); +xfs_failaddr_t xfs_btree_fsblock_verify(struct xfs_buf *bp, + unsigned int max_recs); +xfs_failaddr_t xfs_btree_memblock_verify(struct xfs_buf *bp, + unsigned int max_recs); + +unsigned int xfs_btree_compute_maxlevels(const unsigned int *limits, + unsigned long long records); +unsigned long long xfs_btree_calc_size(const unsigned int *limits, + unsigned long long records); +unsigned int xfs_btree_space_to_height(const unsigned int *limits, + unsigned long long blocks); + +/* + * Return codes for the query range iterator function are 0 to continue + * iterating, and non-zero to stop iterating. Any non-zero value will be + * passed up to the _query_range caller. The special value -ECANCELED can be + * used to stop iteration, because _query_range never generates that error + * code on its own. + */ +typedef int (*xfs_btree_query_range_fn)(struct xfs_btree_cur *cur, + const union xfs_btree_rec *rec, void *priv); + +int xfs_btree_query_range(struct xfs_btree_cur *cur, + const union xfs_btree_irec *low_rec, + const union xfs_btree_irec *high_rec, + xfs_btree_query_range_fn fn, void *priv); +int xfs_btree_query_all(struct xfs_btree_cur *cur, xfs_btree_query_range_fn fn, + void *priv); + +typedef int (*xfs_btree_visit_blocks_fn)(struct xfs_btree_cur *cur, int level, + void *data); +/* Visit record blocks. */ +#define XFS_BTREE_VISIT_RECORDS (1 << 0) +/* Visit leaf blocks. */ +#define XFS_BTREE_VISIT_LEAVES (1 << 1) +/* Visit all blocks. */ +#define XFS_BTREE_VISIT_ALL (XFS_BTREE_VISIT_RECORDS | \ + XFS_BTREE_VISIT_LEAVES) +int xfs_btree_visit_blocks(struct xfs_btree_cur *cur, + xfs_btree_visit_blocks_fn fn, unsigned int flags, void *data); + +int xfs_btree_count_blocks(struct xfs_btree_cur *cur, xfs_filblks_t *blocks); + +union xfs_btree_rec *xfs_btree_rec_addr(struct xfs_btree_cur *cur, int n, + struct xfs_btree_block *block); +union xfs_btree_key *xfs_btree_key_addr(struct xfs_btree_cur *cur, int n, + struct xfs_btree_block *block); +union xfs_btree_key *xfs_btree_high_key_addr(struct xfs_btree_cur *cur, int n, + struct xfs_btree_block *block); +union xfs_btree_ptr *xfs_btree_ptr_addr(struct xfs_btree_cur *cur, int n, + struct xfs_btree_block *block); +int xfs_btree_lookup_get_block(struct xfs_btree_cur *cur, int level, + const union xfs_btree_ptr *pp, struct xfs_btree_block **blkp); +struct xfs_btree_block *xfs_btree_get_block(struct xfs_btree_cur *cur, + int level, struct xfs_buf **bpp); +bool xfs_btree_ptr_is_null(struct xfs_btree_cur *cur, + const union xfs_btree_ptr *ptr); +int xfs_btree_cmp_two_ptrs(struct xfs_btree_cur *cur, + const union xfs_btree_ptr *a, + const union xfs_btree_ptr *b); +void xfs_btree_get_sibling(struct xfs_btree_cur *cur, + struct xfs_btree_block *block, + union xfs_btree_ptr *ptr, int lr); +void xfs_btree_get_keys(struct xfs_btree_cur *cur, + struct xfs_btree_block *block, union xfs_btree_key *key); +union xfs_btree_key *xfs_btree_high_key_from_key(struct xfs_btree_cur *cur, + union xfs_btree_key *key); +typedef bool (*xfs_btree_key_gap_fn)(struct xfs_btree_cur *cur, + const union xfs_btree_key *key1, + const union xfs_btree_key *key2); + +int xfs_btree_has_records(struct xfs_btree_cur *cur, + const union xfs_btree_irec *low, + const union xfs_btree_irec *high, + const union xfs_btree_key *mask, + enum xbtree_recpacking *outcome); + +bool xfs_btree_has_more_records(struct xfs_btree_cur *cur); +struct xfs_ifork *xfs_btree_ifork_ptr(struct xfs_btree_cur *cur); + +/* Key comparison helpers */ +static inline bool +xfs_btree_keycmp_lt( + struct xfs_btree_cur *cur, + const union xfs_btree_key *key1, + const union xfs_btree_key *key2) +{ + return cur->bc_ops->cmp_two_keys(cur, key1, key2, NULL) < 0; +} + +static inline bool +xfs_btree_keycmp_gt( + struct xfs_btree_cur *cur, + const union xfs_btree_key *key1, + const union xfs_btree_key *key2) +{ + return cur->bc_ops->cmp_two_keys(cur, key1, key2, NULL) > 0; +} + +static inline bool +xfs_btree_keycmp_eq( + struct xfs_btree_cur *cur, + const union xfs_btree_key *key1, + const union xfs_btree_key *key2) +{ + return cur->bc_ops->cmp_two_keys(cur, key1, key2, NULL) == 0; +} + +static inline bool +xfs_btree_keycmp_le( + struct xfs_btree_cur *cur, + const union xfs_btree_key *key1, + const union xfs_btree_key *key2) +{ + return !xfs_btree_keycmp_gt(cur, key1, key2); +} + +static inline bool +xfs_btree_keycmp_ge( + struct xfs_btree_cur *cur, + const union xfs_btree_key *key1, + const union xfs_btree_key *key2) +{ + return !xfs_btree_keycmp_lt(cur, key1, key2); +} + +static inline bool +xfs_btree_keycmp_ne( + struct xfs_btree_cur *cur, + const union xfs_btree_key *key1, + const union xfs_btree_key *key2) +{ + return !xfs_btree_keycmp_eq(cur, key1, key2); +} + +/* Masked key comparison helpers */ +static inline bool +xfs_btree_masked_keycmp_lt( + struct xfs_btree_cur *cur, + const union xfs_btree_key *key1, + const union xfs_btree_key *key2, + const union xfs_btree_key *mask) +{ + return cur->bc_ops->cmp_two_keys(cur, key1, key2, mask) < 0; +} + +static inline bool +xfs_btree_masked_keycmp_gt( + struct xfs_btree_cur *cur, + const union xfs_btree_key *key1, + const union xfs_btree_key *key2, + const union xfs_btree_key *mask) +{ + return cur->bc_ops->cmp_two_keys(cur, key1, key2, mask) > 0; +} + +static inline bool +xfs_btree_masked_keycmp_ge( + struct xfs_btree_cur *cur, + const union xfs_btree_key *key1, + const union xfs_btree_key *key2, + const union xfs_btree_key *mask) +{ + return !xfs_btree_masked_keycmp_lt(cur, key1, key2, mask); +} + +/* Does this cursor point to the last block in the given level? */ +static inline bool +xfs_btree_islastblock( + struct xfs_btree_cur *cur, + int level) +{ + struct xfs_btree_block *block; + struct xfs_buf *bp; + + block = xfs_btree_get_block(cur, level, &bp); + + if (cur->bc_ops->ptr_len == XFS_BTREE_LONG_PTR_LEN) + return block->bb_u.l.bb_rightsib == cpu_to_be64(NULLFSBLOCK); + return block->bb_u.s.bb_rightsib == cpu_to_be32(NULLAGBLOCK); +} + +void xfs_btree_set_ptr_null(struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr); +int xfs_btree_get_buf_block(struct xfs_btree_cur *cur, + const union xfs_btree_ptr *ptr, struct xfs_btree_block **block, + struct xfs_buf **bpp); +int xfs_btree_read_buf_block(struct xfs_btree_cur *cur, + const union xfs_btree_ptr *ptr, int flags, + struct xfs_btree_block **block, struct xfs_buf **bpp); +void xfs_btree_set_sibling(struct xfs_btree_cur *cur, + struct xfs_btree_block *block, const union xfs_btree_ptr *ptr, + int lr); +void xfs_btree_init_block_cur(struct xfs_btree_cur *cur, + struct xfs_buf *bp, int level, int numrecs); +void xfs_btree_copy_ptrs(struct xfs_btree_cur *cur, + union xfs_btree_ptr *dst_ptr, + const union xfs_btree_ptr *src_ptr, int numptrs); +void xfs_btree_copy_keys(struct xfs_btree_cur *cur, + union xfs_btree_key *dst_key, + const union xfs_btree_key *src_key, int numkeys); +void xfs_btree_init_ptr_from_cur(struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr); + +static inline struct xfs_btree_cur * +xfs_btree_alloc_cursor( + struct xfs_mount *mp, + struct xfs_trans *tp, + const struct xfs_btree_ops *ops, + uint8_t maxlevels, + struct kmem_cache *cache) +{ + struct xfs_btree_cur *cur; + + ASSERT(ops->ptr_len == XFS_BTREE_LONG_PTR_LEN || + ops->ptr_len == XFS_BTREE_SHORT_PTR_LEN); + + /* BMBT allocations can come through from non-transactional context. */ + cur = kmem_cache_zalloc(cache, + GFP_KERNEL | __GFP_NOLOCKDEP | __GFP_NOFAIL); + cur->bc_ops = ops; + cur->bc_tp = tp; + cur->bc_mp = mp; + cur->bc_maxlevels = maxlevels; + cur->bc_cache = cache; + + return cur; +} + +int __init xfs_btree_init_cur_caches(void); +void xfs_btree_destroy_cur_caches(void); + +int xfs_btree_goto_left_edge(struct xfs_btree_cur *cur); + +/* Does this level of the cursor point to the inode root (and not a block)? */ +static inline bool +xfs_btree_at_iroot( + const struct xfs_btree_cur *cur, + int level) +{ + return cur->bc_ops->type == XFS_BTREE_TYPE_INODE && + level == cur->bc_nlevels - 1; +} + +int xfs_btree_alloc_metafile_block(struct xfs_btree_cur *cur, + const union xfs_btree_ptr *start, union xfs_btree_ptr *newp, + int *stat); +int xfs_btree_free_metafile_block(struct xfs_btree_cur *cur, + struct xfs_buf *bp); + +#endif /* __XFS_BTREE_H__ */ diff --git a/fs/xfs/libxfs/xfs_btree_mem.c b/fs/xfs/libxfs/xfs_btree_mem.c new file mode 100644 index 000000000..37136a70e --- /dev/null +++ b/fs/xfs/libxfs/xfs_btree_mem.c @@ -0,0 +1,346 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2021-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_trans.h" +#include "xfs_btree.h" +#include "xfs_error.h" +#include "xfs_buf_mem.h" +#include "xfs_btree_mem.h" +#include "xfs_ag.h" +#include "xfs_buf_item.h" +#include "xfs_trace.h" +#include "xfs_rtgroup.h" + +/* Set the root of an in-memory btree. */ +void +xfbtree_set_root( + struct xfs_btree_cur *cur, + const union xfs_btree_ptr *ptr, + int inc) +{ + ASSERT(cur->bc_ops->type == XFS_BTREE_TYPE_MEM); + + cur->bc_mem.xfbtree->root = *ptr; + cur->bc_mem.xfbtree->nlevels += inc; +} + +/* Initialize a pointer from the in-memory btree header. */ +void +xfbtree_init_ptr_from_cur( + struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr) +{ + ASSERT(cur->bc_ops->type == XFS_BTREE_TYPE_MEM); + + *ptr = cur->bc_mem.xfbtree->root; +} + +/* Duplicate an in-memory btree cursor. */ +struct xfs_btree_cur * +xfbtree_dup_cursor( + struct xfs_btree_cur *cur) +{ + struct xfs_btree_cur *ncur; + + ASSERT(cur->bc_ops->type == XFS_BTREE_TYPE_MEM); + + ncur = xfs_btree_alloc_cursor(cur->bc_mp, cur->bc_tp, cur->bc_ops, + cur->bc_maxlevels, cur->bc_cache); + ncur->bc_flags = cur->bc_flags; + ncur->bc_nlevels = cur->bc_nlevels; + ncur->bc_mem.xfbtree = cur->bc_mem.xfbtree; + if (cur->bc_group) + ncur->bc_group = xfs_group_hold(cur->bc_group); + return ncur; +} + +/* Close the btree xfile and release all resources. */ +void +xfbtree_destroy( + struct xfbtree *xfbt) +{ + xfs_buftarg_drain(xfbt->target); +} + +/* Compute the number of bytes available for records. */ +static inline unsigned int +xfbtree_rec_bytes( + struct xfs_mount *mp, + const struct xfs_btree_ops *ops) +{ + return XMBUF_BLOCKSIZE - XFS_BTREE_LBLOCK_CRC_LEN; +} + +/* Initialize an empty leaf block as the btree root. */ +STATIC int +xfbtree_init_leaf_block( + struct xfs_mount *mp, + struct xfbtree *xfbt, + const struct xfs_btree_ops *ops) +{ + struct xfs_buf *bp; + xfbno_t bno = xfbt->highest_bno++; + int error; + + error = xfs_buf_get(xfbt->target, xfbno_to_daddr(bno), XFBNO_BBSIZE, + &bp); + if (error) + return error; + + trace_xfbtree_create_root_buf(xfbt, bp); + + bp->b_ops = ops->buf_ops; + xfs_btree_init_buf(mp, bp, ops, 0, 0, xfbt->owner); + xfs_buf_relse(bp); + + xfbt->root.l = cpu_to_be64(bno); + return 0; +} + +/* + * Create an in-memory btree root that can be used with the given xmbuf. + * Callers must set xfbt->owner. + */ +int +xfbtree_init( + struct xfs_mount *mp, + struct xfbtree *xfbt, + struct xfs_buftarg *btp, + const struct xfs_btree_ops *ops) +{ + unsigned int blocklen = xfbtree_rec_bytes(mp, ops); + unsigned int keyptr_len; + int error; + + /* Requires a long-format CRC-format btree */ + if (!xfs_has_crc(mp)) { + ASSERT(xfs_has_crc(mp)); + return -EINVAL; + } + if (ops->ptr_len != XFS_BTREE_LONG_PTR_LEN) { + ASSERT(ops->ptr_len == XFS_BTREE_LONG_PTR_LEN); + return -EINVAL; + } + + memset(xfbt, 0, sizeof(*xfbt)); + xfbt->target = btp; + + /* Set up min/maxrecs for this btree. */ + keyptr_len = ops->key_len + sizeof(__be64); + xfbt->maxrecs[0] = blocklen / ops->rec_len; + xfbt->maxrecs[1] = blocklen / keyptr_len; + xfbt->minrecs[0] = xfbt->maxrecs[0] / 2; + xfbt->minrecs[1] = xfbt->maxrecs[1] / 2; + xfbt->highest_bno = 0; + xfbt->nlevels = 1; + + /* Initialize the empty btree. */ + error = xfbtree_init_leaf_block(mp, xfbt, ops); + if (error) + goto err_freesp; + + trace_xfbtree_init(mp, xfbt, ops); + + return 0; + +err_freesp: + xfs_buftarg_drain(xfbt->target); + return error; +} + +/* Allocate a block to our in-memory btree. */ +int +xfbtree_alloc_block( + struct xfs_btree_cur *cur, + const union xfs_btree_ptr *start, + union xfs_btree_ptr *new, + int *stat) +{ + struct xfbtree *xfbt = cur->bc_mem.xfbtree; + xfbno_t bno = xfbt->highest_bno++; + + ASSERT(cur->bc_ops->type == XFS_BTREE_TYPE_MEM); + + trace_xfbtree_alloc_block(xfbt, cur, bno); + + /* Fail if the block address exceeds the maximum for the buftarg. */ + if (!xfbtree_verify_bno(xfbt, bno)) { + ASSERT(xfbtree_verify_bno(xfbt, bno)); + *stat = 0; + return 0; + } + + new->l = cpu_to_be64(bno); + *stat = 1; + return 0; +} + +/* Free a block from our in-memory btree. */ +int +xfbtree_free_block( + struct xfs_btree_cur *cur, + struct xfs_buf *bp) +{ + struct xfbtree *xfbt = cur->bc_mem.xfbtree; + xfs_daddr_t daddr = xfs_buf_daddr(bp); + xfbno_t bno = xfs_daddr_to_xfbno(daddr); + + ASSERT(cur->bc_ops->type == XFS_BTREE_TYPE_MEM); + + trace_xfbtree_free_block(xfbt, cur, bno); + + if (bno + 1 == xfbt->highest_bno) + xfbt->highest_bno--; + + return 0; +} + +/* Return the minimum number of records for a btree block. */ +int +xfbtree_get_minrecs( + struct xfs_btree_cur *cur, + int level) +{ + struct xfbtree *xfbt = cur->bc_mem.xfbtree; + + return xfbt->minrecs[level != 0]; +} + +/* Return the maximum number of records for a btree block. */ +int +xfbtree_get_maxrecs( + struct xfs_btree_cur *cur, + int level) +{ + struct xfbtree *xfbt = cur->bc_mem.xfbtree; + + return xfbt->maxrecs[level != 0]; +} + +/* If this log item is a buffer item that came from the xfbtree, return it. */ +static inline struct xfs_buf * +xfbtree_buf_match( + struct xfbtree *xfbt, + const struct xfs_log_item *lip) +{ + const struct xfs_buf_log_item *bli; + struct xfs_buf *bp; + + if (lip->li_type != XFS_LI_BUF) + return NULL; + + bli = container_of(lip, struct xfs_buf_log_item, bli_item); + bp = bli->bli_buf; + if (bp->b_target != xfbt->target) + return NULL; + + return bp; +} + +/* + * Commit changes to the incore btree immediately by writing all dirty xfbtree + * buffers to the backing xfile. This detaches all xfbtree buffers from the + * transaction, even on failure. The buffer locks are dropped between the + * delwri queue and submit, so the caller must synchronize btree access. + * + * Normally we'd let the buffers commit with the transaction and get written to + * the xfile via the log, but online repair stages ephemeral btrees in memory + * and uses the btree_staging functions to write new btrees to disk atomically. + * The in-memory btree (and its backing store) are discarded at the end of the + * repair phase, which means that xfbtree buffers cannot commit with the rest + * of a transaction. + * + * In other words, online repair only needs the transaction to collect buffer + * pointers and to avoid buffer deadlocks, not to guarantee consistency of + * updates. + */ +int +xfbtree_trans_commit( + struct xfbtree *xfbt, + struct xfs_trans *tp) +{ + struct xfs_log_item *lip, *n; + bool tp_dirty = false; + int error = 0; + + /* + * For each xfbtree buffer attached to the transaction, write the dirty + * buffers to the xfile and release them. + */ + list_for_each_entry_safe(lip, n, &tp->t_items, li_trans) { + struct xfs_buf *bp = xfbtree_buf_match(xfbt, lip); + + if (!bp) { + if (test_bit(XFS_LI_DIRTY, &lip->li_flags)) + tp_dirty |= true; + continue; + } + + trace_xfbtree_trans_commit_buf(xfbt, bp); + + xmbuf_trans_bdetach(tp, bp); + + /* + * If the buffer fails verification, note the failure but + * continue walking the transaction items so that we remove all + * ephemeral btree buffers. + */ + if (!error) + error = xmbuf_finalize(bp); + + xfs_buf_relse(bp); + } + + /* + * Reset the transaction's dirty flag to reflect the dirty state of the + * log items that are still attached. + */ + tp->t_flags = (tp->t_flags & ~XFS_TRANS_DIRTY) | + (tp_dirty ? XFS_TRANS_DIRTY : 0); + + return error; +} + +/* + * Cancel changes to the incore btree by detaching all the xfbtree buffers. + * Changes are not undone, so callers must not access the btree ever again. + */ +void +xfbtree_trans_cancel( + struct xfbtree *xfbt, + struct xfs_trans *tp) +{ + struct xfs_log_item *lip, *n; + bool tp_dirty = false; + + list_for_each_entry_safe(lip, n, &tp->t_items, li_trans) { + struct xfs_buf *bp = xfbtree_buf_match(xfbt, lip); + + if (!bp) { + if (test_bit(XFS_LI_DIRTY, &lip->li_flags)) + tp_dirty |= true; + continue; + } + + trace_xfbtree_trans_cancel_buf(xfbt, bp); + + xmbuf_trans_bdetach(tp, bp); + xfs_buf_relse(bp); + } + + /* + * Reset the transaction's dirty flag to reflect the dirty state of the + * log items that are still attached. + */ + tp->t_flags = (tp->t_flags & ~XFS_TRANS_DIRTY) | + (tp_dirty ? XFS_TRANS_DIRTY : 0); +} diff --git a/fs/xfs/libxfs/xfs_btree_mem.h b/fs/xfs/libxfs/xfs_btree_mem.h new file mode 100644 index 000000000..1c3825786 --- /dev/null +++ b/fs/xfs/libxfs/xfs_btree_mem.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2021-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_BTREE_MEM_H__ +#define __XFS_BTREE_MEM_H__ + +typedef uint64_t xfbno_t; + +#define XFBNO_BLOCKSIZE (XMBUF_BLOCKSIZE) +#define XFBNO_BBSHIFT (XMBUF_BLOCKSHIFT - BBSHIFT) +#define XFBNO_BBSIZE (XFBNO_BLOCKSIZE >> BBSHIFT) + +static inline xfs_daddr_t xfbno_to_daddr(xfbno_t blkno) +{ + return blkno << XFBNO_BBSHIFT; +} + +static inline xfbno_t xfs_daddr_to_xfbno(xfs_daddr_t daddr) +{ + return daddr >> XFBNO_BBSHIFT; +} + +struct xfbtree { + /* buffer cache target for this in-memory btree */ + struct xfs_buftarg *target; + + /* Highest block number that has been written to. */ + xfbno_t highest_bno; + + /* Owner of this btree. */ + unsigned long long owner; + + /* Btree header */ + union xfs_btree_ptr root; + unsigned int nlevels; + + /* Minimum and maximum records per block. */ + unsigned int maxrecs[2]; + unsigned int minrecs[2]; +}; + +#ifdef CONFIG_XFS_BTREE_IN_MEM +static inline bool xfbtree_verify_bno(struct xfbtree *xfbt, xfbno_t bno) +{ + return xmbuf_verify_daddr(xfbt->target, xfbno_to_daddr(bno)); +} + +void xfbtree_set_root(struct xfs_btree_cur *cur, + const union xfs_btree_ptr *ptr, int inc); +void xfbtree_init_ptr_from_cur(struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr); +struct xfs_btree_cur *xfbtree_dup_cursor(struct xfs_btree_cur *cur); + +int xfbtree_get_minrecs(struct xfs_btree_cur *cur, int level); +int xfbtree_get_maxrecs(struct xfs_btree_cur *cur, int level); + +int xfbtree_alloc_block(struct xfs_btree_cur *cur, + const union xfs_btree_ptr *start, union xfs_btree_ptr *ptr, + int *stat); +int xfbtree_free_block(struct xfs_btree_cur *cur, struct xfs_buf *bp); + +/* Callers must set xfbt->target and xfbt->owner before calling this */ +int xfbtree_init(struct xfs_mount *mp, struct xfbtree *xfbt, + struct xfs_buftarg *btp, const struct xfs_btree_ops *ops); +void xfbtree_destroy(struct xfbtree *xfbt); + +int xfbtree_trans_commit(struct xfbtree *xfbt, struct xfs_trans *tp); +void xfbtree_trans_cancel(struct xfbtree *xfbt, struct xfs_trans *tp); +#else +# define xfbtree_verify_bno(...) (false) +#endif /* CONFIG_XFS_BTREE_IN_MEM */ + +#endif /* __XFS_BTREE_MEM_H__ */ diff --git a/fs/xfs/libxfs/xfs_btree_staging.c b/fs/xfs/libxfs/xfs_btree_staging.c new file mode 100644 index 000000000..7314dab4b --- /dev/null +++ b/fs/xfs/libxfs/xfs_btree_staging.c @@ -0,0 +1,819 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2020 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_btree.h" +#include "xfs_trace.h" +#include "xfs_btree_staging.h" + +/* + * Staging Cursors and Fake Roots for Btrees + * ========================================= + * + * A staging btree cursor is a special type of btree cursor that callers must + * use to construct a new btree index using the btree bulk loader code. The + * bulk loading code uses the staging btree cursor to abstract the details of + * initializing new btree blocks and filling them with records or key/ptr + * pairs. Regular btree operations (e.g. queries and modifications) are not + * supported with staging cursors, and callers must not invoke them. + * + * Fake root structures contain all the information about a btree that is under + * construction by the bulk loading code. Staging btree cursors point to fake + * root structures instead of the usual AG header or inode structure. + * + * Callers are expected to initialize a fake root structure and pass it into + * the _stage_cursor function for a specific btree type. When bulk loading is + * complete, callers should call the _commit_staged_btree function for that + * specific btree type to commit the new btree into the filesystem. + */ + +/* + * Bulk Loading for AG Btrees + * ========================== + * + * For a btree rooted in an AG header, pass a xbtree_afakeroot structure to the + * staging cursor. Callers should initialize this to zero. + * + * The _stage_cursor() function for a specific btree type should call + * xfs_btree_stage_afakeroot to set up the in-memory cursor as a staging + * cursor. The corresponding _commit_staged_btree() function should log the + * new root and call xfs_btree_commit_afakeroot() to transform the staging + * cursor into a regular btree cursor. + */ + +/* + * Initialize a AG-rooted btree cursor with the given AG btree fake root. + */ +void +xfs_btree_stage_afakeroot( + struct xfs_btree_cur *cur, + struct xbtree_afakeroot *afake) +{ + ASSERT(!(cur->bc_flags & XFS_BTREE_STAGING)); + ASSERT(cur->bc_ops->type != XFS_BTREE_TYPE_INODE); + ASSERT(cur->bc_tp == NULL); + + cur->bc_ag.afake = afake; + cur->bc_nlevels = afake->af_levels; + cur->bc_flags |= XFS_BTREE_STAGING; +} + +/* + * Transform an AG-rooted staging btree cursor back into a regular cursor by + * substituting a real btree root for the fake one and restoring normal btree + * cursor ops. The caller must log the btree root change prior to calling + * this. + */ +void +xfs_btree_commit_afakeroot( + struct xfs_btree_cur *cur, + struct xfs_trans *tp, + struct xfs_buf *agbp) +{ + ASSERT(cur->bc_flags & XFS_BTREE_STAGING); + ASSERT(cur->bc_tp == NULL); + + trace_xfs_btree_commit_afakeroot(cur); + + cur->bc_ag.afake = NULL; + cur->bc_ag.agbp = agbp; + cur->bc_flags &= ~XFS_BTREE_STAGING; + cur->bc_tp = tp; +} + +/* + * Bulk Loading for Inode-Rooted Btrees + * ==================================== + * + * For a btree rooted in an inode fork, pass a xbtree_ifakeroot structure to + * the staging cursor. This structure should be initialized as follows: + * + * - if_fork_size field should be set to the number of bytes available to the + * fork in the inode. + * + * - if_fork should point to a freshly allocated struct xfs_ifork. + * + * - if_format should be set to the appropriate fork type (e.g. + * XFS_DINODE_FMT_BTREE). + * + * All other fields must be zero. + * + * The _stage_cursor() function for a specific btree type should call + * xfs_btree_stage_ifakeroot to set up the in-memory cursor as a staging + * cursor. The corresponding _commit_staged_btree() function should log the + * new root and call xfs_btree_commit_ifakeroot() to transform the staging + * cursor into a regular btree cursor. + */ + +/* + * Initialize an inode-rooted btree cursor with the given inode btree fake + * root. The btree cursor's bc_ops will be overridden as needed to make the + * staging functionality work. If new_ops is not NULL, these new ops will be + * passed out to the caller for further overriding. + */ +void +xfs_btree_stage_ifakeroot( + struct xfs_btree_cur *cur, + struct xbtree_ifakeroot *ifake) +{ + ASSERT(!(cur->bc_flags & XFS_BTREE_STAGING)); + ASSERT(cur->bc_ops->type == XFS_BTREE_TYPE_INODE); + ASSERT(cur->bc_tp == NULL); + + cur->bc_ino.ifake = ifake; + cur->bc_nlevels = ifake->if_levels; + cur->bc_ino.forksize = ifake->if_fork_size; + cur->bc_ino.whichfork = XFS_STAGING_FORK; + cur->bc_flags |= XFS_BTREE_STAGING; +} + +/* + * Transform an inode-rooted staging btree cursor back into a regular cursor by + * substituting a real btree root for the fake one and restoring normal btree + * cursor ops. The caller must log the btree root change prior to calling + * this. + */ +void +xfs_btree_commit_ifakeroot( + struct xfs_btree_cur *cur, + struct xfs_trans *tp, + int whichfork) +{ + ASSERT(cur->bc_flags & XFS_BTREE_STAGING); + ASSERT(cur->bc_tp == NULL); + + trace_xfs_btree_commit_ifakeroot(cur); + + cur->bc_ino.ifake = NULL; + cur->bc_ino.whichfork = whichfork; + cur->bc_flags &= ~XFS_BTREE_STAGING; + cur->bc_tp = tp; +} + +/* + * Bulk Loading of Staged Btrees + * ============================= + * + * This interface is used with a staged btree cursor to create a totally new + * btree with a large number of records (i.e. more than what would fit in a + * single root block). When the creation is complete, the new root can be + * linked atomically into the filesystem by committing the staged cursor. + * + * Creation of a new btree proceeds roughly as follows: + * + * The first step is to initialize an appropriate fake btree root structure and + * then construct a staged btree cursor. Refer to the block comments about + * "Bulk Loading for AG Btrees" and "Bulk Loading for Inode-Rooted Btrees" for + * more information about how to do this. + * + * The second step is to initialize a struct xfs_btree_bload context as + * documented in the structure definition. + * + * The third step is to call xfs_btree_bload_compute_geometry to compute the + * height of and the number of blocks needed to construct the btree. See the + * section "Computing the Geometry of the New Btree" for details about this + * computation. + * + * In step four, the caller must allocate xfs_btree_bload.nr_blocks blocks and + * save them for later use by ->claim_block(). Bulk loading requires all + * blocks to be allocated beforehand to avoid ENOSPC failures midway through a + * rebuild, and to minimize seek distances of the new btree. + * + * Step five is to call xfs_btree_bload() to start constructing the btree. + * + * The final step is to commit the staging btree cursor, which logs the new + * btree root and turns the staging cursor into a regular cursor. The caller + * is responsible for cleaning up the previous btree blocks, if any. + * + * Computing the Geometry of the New Btree + * ======================================= + * + * The number of items placed in each btree block is computed via the following + * algorithm: For leaf levels, the number of items for the level is nr_records + * in the bload structure. For node levels, the number of items for the level + * is the number of blocks in the next lower level of the tree. For each + * level, the desired number of items per block is defined as: + * + * desired = max(minrecs, maxrecs - slack factor) + * + * The number of blocks for the level is defined to be: + * + * blocks = floor(nr_items / desired) + * + * Note this is rounded down so that the npb calculation below will never fall + * below minrecs. The number of items that will actually be loaded into each + * btree block is defined as: + * + * npb = nr_items / blocks + * + * Some of the leftmost blocks in the level will contain one extra record as + * needed to handle uneven division. If the number of records in any block + * would exceed maxrecs for that level, blocks is incremented and npb is + * recalculated. + * + * In other words, we compute the number of blocks needed to satisfy a given + * loading level, then spread the items as evenly as possible. + * + * The height and number of fs blocks required to create the btree are computed + * and returned via btree_height and nr_blocks. + */ + +/* + * Put a btree block that we're loading onto the ordered list and release it. + * The btree blocks will be written to disk when bulk loading is finished. + * If we reach the dirty buffer threshold, flush them to disk before + * continuing. + */ +static int +xfs_btree_bload_drop_buf( + struct xfs_btree_bload *bbl, + struct list_head *buffers_list, + struct xfs_buf **bpp) +{ + struct xfs_buf *bp = *bpp; + int error; + + if (!bp) + return 0; + + /* + * Mark this buffer uptodate so that a subsequent xfs_buf_read will + * not pointlessly reread the contents from the disk. + */ + xfs_buf_set_uptodate(bp); + xfs_buf_delwri_queue_here(bp, buffers_list); + xfs_buf_relse(bp); + *bpp = NULL; + bbl->nr_dirty++; + + if (!bbl->max_dirty || bbl->nr_dirty < bbl->max_dirty) + return 0; + + error = xfs_buf_delwri_submit(buffers_list); + if (error) + return error; + + bbl->nr_dirty = 0; + return 0; +} + +/* + * Allocate and initialize one btree block for bulk loading. + * + * The new btree block will have its level and numrecs fields set to the values + * of the level and nr_this_block parameters, respectively. + * + * The caller should ensure that ptrp, bpp, and blockp refer to the left + * sibling of the new block, if there is any. On exit, ptrp, bpp, and blockp + * will all point to the new block. + */ +STATIC int +xfs_btree_bload_prep_block( + struct xfs_btree_cur *cur, + struct xfs_btree_bload *bbl, + struct list_head *buffers_list, + unsigned int level, + unsigned int nr_this_block, + union xfs_btree_ptr *ptrp, /* in/out */ + struct xfs_buf **bpp, /* in/out */ + struct xfs_btree_block **blockp, /* in/out */ + void *priv) +{ + union xfs_btree_ptr new_ptr; + struct xfs_buf *new_bp; + struct xfs_btree_block *new_block; + int ret; + + if (xfs_btree_at_iroot(cur, level)) { + struct xfs_ifork *ifp = xfs_btree_ifork_ptr(cur); + size_t new_size; + + ASSERT(*bpp == NULL); + + /* Allocate a new incore btree root block. */ + new_size = bbl->iroot_size(cur, level, nr_this_block, priv); + ifp->if_broot = kzalloc(new_size, GFP_KERNEL | __GFP_NOFAIL); + ifp->if_broot_bytes = (int)new_size; + + /* Initialize it and send it out. */ + xfs_btree_init_block(cur->bc_mp, ifp->if_broot, cur->bc_ops, + level, nr_this_block, I_INO(cur->bc_ino.ip)); + + *bpp = NULL; + *blockp = ifp->if_broot; + xfs_btree_set_ptr_null(cur, ptrp); + return 0; + } + + /* Claim one of the caller's preallocated blocks. */ + xfs_btree_set_ptr_null(cur, &new_ptr); + ret = bbl->claim_block(cur, &new_ptr, priv); + if (ret) + return ret; + + ASSERT(!xfs_btree_ptr_is_null(cur, &new_ptr)); + + ret = xfs_btree_get_buf_block(cur, &new_ptr, &new_block, &new_bp); + if (ret) + return ret; + + /* + * The previous block (if any) is the left sibling of the new block, + * so set its right sibling pointer to the new block and drop it. + */ + if (*blockp) + xfs_btree_set_sibling(cur, *blockp, &new_ptr, XFS_BB_RIGHTSIB); + + ret = xfs_btree_bload_drop_buf(bbl, buffers_list, bpp); + if (ret) + return ret; + + /* Initialize the new btree block. */ + xfs_btree_init_block_cur(cur, new_bp, level, nr_this_block); + xfs_btree_set_sibling(cur, new_block, ptrp, XFS_BB_LEFTSIB); + + /* Set the out parameters. */ + *bpp = new_bp; + *blockp = new_block; + xfs_btree_copy_ptrs(cur, ptrp, &new_ptr, 1); + return 0; +} + +/* Load one leaf block. */ +STATIC int +xfs_btree_bload_leaf( + struct xfs_btree_cur *cur, + unsigned int recs_this_block, + xfs_btree_bload_get_records_fn get_records, + struct xfs_btree_block *block, + void *priv) +{ + unsigned int j = 1; + int ret; + + /* Fill the leaf block with records. */ + while (j <= recs_this_block) { + ret = get_records(cur, j, block, recs_this_block - j + 1, priv); + if (ret < 0) + return ret; + j += ret; + } + + return 0; +} + +/* + * Load one node block with key/ptr pairs. + * + * child_ptr must point to a block within the next level down in the tree. A + * key/ptr entry will be created in the new node block to the block pointed to + * by child_ptr. On exit, child_ptr points to the next block on the child + * level that needs processing. + */ +STATIC int +xfs_btree_bload_node( + struct xfs_btree_cur *cur, + unsigned int recs_this_block, + union xfs_btree_ptr *child_ptr, + struct xfs_btree_block *block) +{ + unsigned int j; + int ret; + + /* Fill the node block with keys and pointers. */ + for (j = 1; j <= recs_this_block; j++) { + union xfs_btree_key child_key; + union xfs_btree_ptr *block_ptr; + union xfs_btree_key *block_key; + struct xfs_btree_block *child_block; + struct xfs_buf *child_bp; + + ASSERT(!xfs_btree_ptr_is_null(cur, child_ptr)); + + /* + * Read the lower-level block in case the buffer for it has + * been reclaimed. LRU refs will be set on the block, which is + * desirable if the new btree commits. + */ + ret = xfs_btree_read_buf_block(cur, child_ptr, 0, &child_block, + &child_bp); + if (ret) + return ret; + + block_ptr = xfs_btree_ptr_addr(cur, j, block); + xfs_btree_copy_ptrs(cur, block_ptr, child_ptr, 1); + + block_key = xfs_btree_key_addr(cur, j, block); + xfs_btree_get_keys(cur, child_block, &child_key); + xfs_btree_copy_keys(cur, block_key, &child_key, 1); + + xfs_btree_get_sibling(cur, child_block, child_ptr, + XFS_BB_RIGHTSIB); + xfs_buf_relse(child_bp); + } + + return 0; +} + +/* + * Compute the maximum number of records (or keyptrs) per block that we want to + * install at this level in the btree. Caller is responsible for having set + * @cur->bc_ino.forksize to the desired fork size, if appropriate. + */ +STATIC unsigned int +xfs_btree_bload_max_npb( + struct xfs_btree_cur *cur, + struct xfs_btree_bload *bbl, + unsigned int level) +{ + unsigned int ret; + + if (level == cur->bc_nlevels - 1 && cur->bc_ops->get_dmaxrecs) + return cur->bc_ops->get_dmaxrecs(cur, level); + + ret = cur->bc_ops->get_maxrecs(cur, level); + if (level == 0) + ret -= bbl->leaf_slack; + else + ret -= bbl->node_slack; + return ret; +} + +/* + * Compute the desired number of records (or keyptrs) per block that we want to + * install at this level in the btree, which must be somewhere between minrecs + * and max_npb. The caller is free to install fewer records per block. + */ +STATIC unsigned int +xfs_btree_bload_desired_npb( + struct xfs_btree_cur *cur, + struct xfs_btree_bload *bbl, + unsigned int level) +{ + unsigned int npb = xfs_btree_bload_max_npb(cur, bbl, level); + + /* Root blocks are not subject to minrecs rules. */ + if (level == cur->bc_nlevels - 1) + return max(1U, npb); + + return max_t(unsigned int, cur->bc_ops->get_minrecs(cur, level), npb); +} + +/* + * Compute the number of records to be stored in each block at this level and + * the number of blocks for this level. For leaf levels, we must populate an + * empty root block even if there are no records, so we have to have at least + * one block. + */ +STATIC void +xfs_btree_bload_level_geometry( + struct xfs_btree_cur *cur, + struct xfs_btree_bload *bbl, + unsigned int level, + uint64_t nr_this_level, + unsigned int *avg_per_block, + uint64_t *blocks, + uint64_t *blocks_with_extra) +{ + uint64_t npb; + uint64_t dontcare; + unsigned int desired_npb; + unsigned int maxnr; + + /* + * Compute the absolute maximum number of records that we can store in + * the ondisk block or inode root. + */ + if (cur->bc_ops->get_dmaxrecs) + maxnr = cur->bc_ops->get_dmaxrecs(cur, level); + else + maxnr = cur->bc_ops->get_maxrecs(cur, level); + + /* + * Compute the number of blocks we need to fill each block with the + * desired number of records/keyptrs per block. Because desired_npb + * could be minrecs, we use regular integer division (which rounds + * the block count down) so that in the next step the effective # of + * items per block will never be less than desired_npb. + */ + desired_npb = xfs_btree_bload_desired_npb(cur, bbl, level); + *blocks = div64_u64_rem(nr_this_level, desired_npb, &dontcare); + *blocks = max(1ULL, *blocks); + + /* + * Compute the number of records that we will actually put in each + * block, assuming that we want to spread the records evenly between + * the blocks. Take care that the effective # of items per block (npb) + * won't exceed maxrecs even for the blocks that get an extra record, + * since desired_npb could be maxrecs, and in the previous step we + * rounded the block count down. + */ + npb = div64_u64_rem(nr_this_level, *blocks, blocks_with_extra); + if (npb > maxnr || (npb == maxnr && *blocks_with_extra > 0)) { + (*blocks)++; + npb = div64_u64_rem(nr_this_level, *blocks, blocks_with_extra); + } + + *avg_per_block = min_t(uint64_t, npb, nr_this_level); + + trace_xfs_btree_bload_level_geometry(cur, level, nr_this_level, + *avg_per_block, desired_npb, *blocks, + *blocks_with_extra); +} + +/* + * Ensure a slack value is appropriate for the btree. + * + * If the slack value is negative, set slack so that we fill the block to + * halfway between minrecs and maxrecs. Make sure the slack is never so large + * that we can underflow minrecs. + */ +static void +xfs_btree_bload_ensure_slack( + struct xfs_btree_cur *cur, + int *slack, + int level) +{ + int maxr; + int minr; + + maxr = cur->bc_ops->get_maxrecs(cur, level); + minr = cur->bc_ops->get_minrecs(cur, level); + + /* + * If slack is negative, automatically set slack so that we load the + * btree block approximately halfway between minrecs and maxrecs. + * Generally, this will net us 75% loading. + */ + if (*slack < 0) + *slack = maxr - ((maxr + minr) >> 1); + + *slack = min(*slack, maxr - minr); +} + +/* + * Prepare a btree cursor for a bulk load operation by computing the geometry + * fields in bbl. Caller must ensure that the btree cursor is a staging + * cursor. This function can be called multiple times. + */ +int +xfs_btree_bload_compute_geometry( + struct xfs_btree_cur *cur, + struct xfs_btree_bload *bbl, + uint64_t nr_records) +{ + const struct xfs_btree_ops *ops = cur->bc_ops; + uint64_t nr_blocks = 0; + uint64_t nr_this_level; + + ASSERT(cur->bc_flags & XFS_BTREE_STAGING); + + /* + * Make sure that the slack values make sense for traditional leaf and + * node blocks. Inode-rooted btrees will return different minrecs and + * maxrecs values for the root block (bc_nlevels == level - 1). We're + * checking levels 0 and 1 here, so set bc_nlevels such that the btree + * code doesn't interpret either as the root level. + */ + cur->bc_nlevels = cur->bc_maxlevels - 1; + xfs_btree_bload_ensure_slack(cur, &bbl->leaf_slack, 0); + xfs_btree_bload_ensure_slack(cur, &bbl->node_slack, 1); + + bbl->nr_records = nr_this_level = nr_records; + for (cur->bc_nlevels = 1; cur->bc_nlevels <= cur->bc_maxlevels;) { + uint64_t level_blocks; + uint64_t dontcare64; + unsigned int level = cur->bc_nlevels - 1; + unsigned int avg_per_block; + + xfs_btree_bload_level_geometry(cur, bbl, level, nr_this_level, + &avg_per_block, &level_blocks, &dontcare64); + + if (ops->type == XFS_BTREE_TYPE_INODE) { + /* + * If all the items we want to store at this level + * would fit in the inode root block, then we have our + * btree root and are done. + * + * Note that bmap btrees forbid records in the root. + */ + if ((level != 0 || + (ops->geom_flags & XFS_BTGEO_IROOT_RECORDS)) && + nr_this_level <= avg_per_block) { + nr_blocks++; + break; + } + + /* + * Otherwise, we have to store all the items for this + * level in traditional btree blocks and therefore need + * another level of btree to point to those blocks. + * + * We have to re-compute the geometry for each level of + * an inode-rooted btree because the geometry differs + * between a btree root in an inode fork and a + * traditional btree block. + * + * This distinction is made in the btree code based on + * whether level == bc_nlevels - 1. Based on the + * previous root block size check against the root + * block geometry, we know that we aren't yet ready to + * populate the root. Increment bc_nevels and + * recalculate the geometry for a traditional + * block-based btree level. + */ + cur->bc_nlevels++; + ASSERT(cur->bc_nlevels <= cur->bc_maxlevels); + xfs_btree_bload_level_geometry(cur, bbl, level, + nr_this_level, &avg_per_block, + &level_blocks, &dontcare64); + } else { + /* + * If all the items we want to store at this level + * would fit in a single root block, we're done. + */ + if (nr_this_level <= avg_per_block) { + nr_blocks++; + break; + } + + /* Otherwise, we need another level of btree. */ + cur->bc_nlevels++; + ASSERT(cur->bc_nlevels <= cur->bc_maxlevels); + } + + nr_blocks += level_blocks; + nr_this_level = level_blocks; + } + + if (cur->bc_nlevels > cur->bc_maxlevels) + return -EOVERFLOW; + + bbl->btree_height = cur->bc_nlevels; + if (ops->type == XFS_BTREE_TYPE_INODE) + bbl->nr_blocks = nr_blocks - 1; + else + bbl->nr_blocks = nr_blocks; + return 0; +} + +/* Bulk load a btree given the parameters and geometry established in bbl. */ +int +xfs_btree_bload( + struct xfs_btree_cur *cur, + struct xfs_btree_bload *bbl, + void *priv) +{ + struct list_head buffers_list; + union xfs_btree_ptr child_ptr; + union xfs_btree_ptr ptr; + struct xfs_buf *bp = NULL; + struct xfs_btree_block *block = NULL; + uint64_t nr_this_level = bbl->nr_records; + uint64_t blocks; + uint64_t i; + uint64_t blocks_with_extra; + uint64_t total_blocks = 0; + unsigned int avg_per_block; + unsigned int level = 0; + int ret; + + ASSERT(cur->bc_flags & XFS_BTREE_STAGING); + + INIT_LIST_HEAD(&buffers_list); + cur->bc_nlevels = bbl->btree_height; + xfs_btree_set_ptr_null(cur, &child_ptr); + xfs_btree_set_ptr_null(cur, &ptr); + bbl->nr_dirty = 0; + + xfs_btree_bload_level_geometry(cur, bbl, level, nr_this_level, + &avg_per_block, &blocks, &blocks_with_extra); + + /* Load each leaf block. */ + for (i = 0; i < blocks; i++) { + unsigned int nr_this_block = avg_per_block; + + /* + * Due to rounding, btree blocks will not be evenly populated + * in most cases. blocks_with_extra tells us how many blocks + * will receive an extra record to distribute the excess across + * the current level as evenly as possible. + */ + if (i < blocks_with_extra) + nr_this_block++; + + ret = xfs_btree_bload_prep_block(cur, bbl, &buffers_list, level, + nr_this_block, &ptr, &bp, &block, priv); + if (ret) + goto out; + + trace_xfs_btree_bload_block(cur, level, i, blocks, &ptr, + nr_this_block); + + ret = xfs_btree_bload_leaf(cur, nr_this_block, bbl->get_records, + block, priv); + if (ret) + goto out; + + /* + * Record the leftmost leaf pointer so we know where to start + * with the first node level. + */ + if (i == 0) + xfs_btree_copy_ptrs(cur, &child_ptr, &ptr, 1); + } + total_blocks += blocks; + + ret = xfs_btree_bload_drop_buf(bbl, &buffers_list, &bp); + if (ret) + goto out; + + /* Populate the internal btree nodes. */ + for (level = 1; level < cur->bc_nlevels; level++) { + union xfs_btree_ptr first_ptr; + + nr_this_level = blocks; + block = NULL; + xfs_btree_set_ptr_null(cur, &ptr); + + xfs_btree_bload_level_geometry(cur, bbl, level, nr_this_level, + &avg_per_block, &blocks, &blocks_with_extra); + + /* Load each node block. */ + for (i = 0; i < blocks; i++) { + unsigned int nr_this_block = avg_per_block; + + if (i < blocks_with_extra) + nr_this_block++; + + ret = xfs_btree_bload_prep_block(cur, bbl, + &buffers_list, level, nr_this_block, + &ptr, &bp, &block, priv); + if (ret) + goto out; + + trace_xfs_btree_bload_block(cur, level, i, blocks, + &ptr, nr_this_block); + + ret = xfs_btree_bload_node(cur, nr_this_block, + &child_ptr, block); + if (ret) + goto out; + + /* + * Record the leftmost node pointer so that we know + * where to start the next node level above this one. + */ + if (i == 0) + xfs_btree_copy_ptrs(cur, &first_ptr, &ptr, 1); + } + total_blocks += blocks; + + ret = xfs_btree_bload_drop_buf(bbl, &buffers_list, &bp); + if (ret) + goto out; + + xfs_btree_copy_ptrs(cur, &child_ptr, &first_ptr, 1); + } + + /* Initialize the new root. */ + if (cur->bc_ops->type == XFS_BTREE_TYPE_INODE) { + ASSERT(xfs_btree_ptr_is_null(cur, &ptr)); + cur->bc_ino.ifake->if_levels = cur->bc_nlevels; + cur->bc_ino.ifake->if_blocks = total_blocks - 1; + } else { + cur->bc_ag.afake->af_root = be32_to_cpu(ptr.s); + cur->bc_ag.afake->af_levels = cur->bc_nlevels; + cur->bc_ag.afake->af_blocks = total_blocks; + } + + /* + * Write the new blocks to disk. If the ordered list isn't empty after + * that, then something went wrong and we have to fail. This should + * never happen, but we'll check anyway. + */ + ret = xfs_buf_delwri_submit(&buffers_list); + if (ret) + goto out; + if (!list_empty(&buffers_list)) { + ASSERT(list_empty(&buffers_list)); + ret = -EIO; + } + +out: + xfs_buf_delwri_cancel(&buffers_list); + if (bp) + xfs_buf_relse(bp); + return ret; +} diff --git a/fs/xfs/libxfs/xfs_btree_staging.h b/fs/xfs/libxfs/xfs_btree_staging.h new file mode 100644 index 000000000..0c9c2ffb1 --- /dev/null +++ b/fs/xfs/libxfs/xfs_btree_staging.h @@ -0,0 +1,130 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2020 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_BTREE_STAGING_H__ +#define __XFS_BTREE_STAGING_H__ + +/* Fake root for an AG-rooted btree. */ +struct xbtree_afakeroot { + /* AG block number of the new btree root. */ + xfs_agblock_t af_root; + + /* Height of the new btree. */ + unsigned int af_levels; + + /* Number of blocks used by the btree. */ + unsigned int af_blocks; +}; + +/* Cursor interactions with fake roots for AG-rooted btrees. */ +void xfs_btree_stage_afakeroot(struct xfs_btree_cur *cur, + struct xbtree_afakeroot *afake); +void xfs_btree_commit_afakeroot(struct xfs_btree_cur *cur, struct xfs_trans *tp, + struct xfs_buf *agbp); + +/* Fake root for an inode-rooted btree. */ +struct xbtree_ifakeroot { + /* Fake inode fork. */ + struct xfs_ifork *if_fork; + + /* Number of blocks used by the btree. */ + int64_t if_blocks; + + /* Height of the new btree. */ + unsigned int if_levels; + + /* Number of bytes available for this fork in the inode. */ + unsigned int if_fork_size; +}; + +/* Cursor interactions with fake roots for inode-rooted btrees. */ +void xfs_btree_stage_ifakeroot(struct xfs_btree_cur *cur, + struct xbtree_ifakeroot *ifake); +void xfs_btree_commit_ifakeroot(struct xfs_btree_cur *cur, struct xfs_trans *tp, + int whichfork); + +/* Bulk loading of staged btrees. */ +typedef int (*xfs_btree_bload_get_records_fn)(struct xfs_btree_cur *cur, + unsigned int idx, struct xfs_btree_block *block, + unsigned int nr_wanted, void *priv); +typedef int (*xfs_btree_bload_claim_block_fn)(struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr, void *priv); +typedef size_t (*xfs_btree_bload_iroot_size_fn)(struct xfs_btree_cur *cur, + unsigned int level, unsigned int nr_this_level, void *priv); + +struct xfs_btree_bload { + /* + * This function will be called to load @nr_wanted records into the + * btree. The implementation does this by setting the cursor's bc_rec + * field in in-core format and using init_rec_from_cur to set the + * records in the btree block. Records must be returned in sort order. + * The function must return the number of records loaded or the usual + * negative errno. + */ + xfs_btree_bload_get_records_fn get_records; + + /* + * This function will be called nr_blocks times to obtain a pointer + * to a new btree block on disk. Callers must preallocate all space + * for the new btree before calling xfs_btree_bload, and this function + * is what claims that reservation. + */ + xfs_btree_bload_claim_block_fn claim_block; + + /* + * This function should return the size of the in-core btree root + * block. It is only necessary for XFS_BTREE_TYPE_INODE btrees. + */ + xfs_btree_bload_iroot_size_fn iroot_size; + + /* + * The caller should set this to the number of records that will be + * stored in the new btree. + */ + uint64_t nr_records; + + /* + * Number of free records to leave in each leaf block. If the caller + * sets this to -1, the slack value will be calculated to be halfway + * between maxrecs and minrecs. This typically leaves the block 75% + * full. Note that slack values are not enforced on inode root blocks. + */ + int leaf_slack; + + /* + * Number of free key/ptrs pairs to leave in each node block. This + * field has the same semantics as leaf_slack. + */ + int node_slack; + + /* + * The xfs_btree_bload_compute_geometry function will set this to the + * number of btree blocks needed to store nr_records records. + */ + uint64_t nr_blocks; + + /* + * The xfs_btree_bload_compute_geometry function will set this to the + * height of the new btree. + */ + unsigned int btree_height; + + /* + * Flush the new btree block buffer list to disk after this many blocks + * have been formatted. Zero prohibits writing any buffers until all + * blocks have been formatted. + */ + uint16_t max_dirty; + + /* Number of dirty buffers. */ + uint16_t nr_dirty; +}; + +int xfs_btree_bload_compute_geometry(struct xfs_btree_cur *cur, + struct xfs_btree_bload *bbl, uint64_t nr_records); +int xfs_btree_bload(struct xfs_btree_cur *cur, struct xfs_btree_bload *bbl, + void *priv); + +#endif /* __XFS_BTREE_STAGING_H__ */ diff --git a/fs/xfs/libxfs/xfs_cksum.h b/fs/xfs/libxfs/xfs_cksum.h new file mode 100644 index 000000000..999a290cf --- /dev/null +++ b/fs/xfs/libxfs/xfs_cksum.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _XFS_CKSUM_H +#define _XFS_CKSUM_H 1 + +#define XFS_CRC_SEED (~(uint32_t)0) + +/* + * Calculate the intermediate checksum for a buffer that has the CRC field + * inside it. The offset of the 32bit crc fields is passed as the + * cksum_offset parameter. We do not modify the buffer during verification, + * hence we have to split the CRC calculation across the cksum_offset. + */ +static inline uint32_t +xfs_start_cksum_safe(char *buffer, size_t length, unsigned long cksum_offset) +{ + uint32_t zero = 0; + uint32_t crc; + + /* Calculate CRC up to the checksum. */ + crc = crc32c(XFS_CRC_SEED, buffer, cksum_offset); + + /* Skip checksum field */ + crc = crc32c(crc, &zero, sizeof(__u32)); + + /* Calculate the rest of the CRC. */ + return crc32c(crc, &buffer[cksum_offset + sizeof(__be32)], + length - (cksum_offset + sizeof(__be32))); +} + +/* + * Fast CRC method where the buffer is modified. Callers must have exclusive + * access to the buffer while the calculation takes place. + */ +static inline uint32_t +xfs_start_cksum_update(char *buffer, size_t length, unsigned long cksum_offset) +{ + /* zero the CRC field */ + *(__le32 *)(buffer + cksum_offset) = 0; + + /* single pass CRC calculation for the entire buffer */ + return crc32c(XFS_CRC_SEED, buffer, length); +} + +/* + * Convert the intermediate checksum to the final ondisk format. + * + * The CRC32c calculation uses LE format even on BE machines, but returns the + * result in host endian format. Hence we need to byte swap it back to LE format + * so that it is consistent on disk. + */ +static inline __le32 +xfs_end_cksum(uint32_t crc) +{ + return ~cpu_to_le32(crc); +} + +/* + * Helper to generate the checksum for a buffer. + * + * This modifies the buffer temporarily - callers must have exclusive + * access to the buffer while the calculation takes place. + */ +static inline void +xfs_update_cksum(char *buffer, size_t length, unsigned long cksum_offset) +{ + uint32_t crc = xfs_start_cksum_update(buffer, length, cksum_offset); + + *(__le32 *)(buffer + cksum_offset) = xfs_end_cksum(crc); +} + +/* + * Helper to verify the checksum for a buffer. + */ +static inline int +xfs_verify_cksum(char *buffer, size_t length, unsigned long cksum_offset) +{ + uint32_t crc = xfs_start_cksum_safe(buffer, length, cksum_offset); + + return *(__le32 *)(buffer + cksum_offset) == xfs_end_cksum(crc); +} + +#endif /* _XFS_CKSUM_H */ diff --git a/fs/xfs/libxfs/xfs_da_btree.c b/fs/xfs/libxfs/xfs_da_btree.c new file mode 100644 index 000000000..f190c0885 --- /dev/null +++ b/fs/xfs/libxfs/xfs_da_btree.c @@ -0,0 +1,2912 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * Copyright (c) 2013 Red Hat, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_dir2.h" +#include "xfs_dir2_priv.h" +#include "xfs_trans.h" +#include "xfs_bmap.h" +#include "xfs_attr_leaf.h" +#include "xfs_error.h" +#include "xfs_trace.h" +#include "xfs_buf_item.h" +#include "xfs_log.h" +#include "xfs_errortag.h" +#include "xfs_health.h" + +/* + * xfs_da_btree.c + * + * Routines to implement directories as Btrees of hashed names. + */ + +/*======================================================================== + * Function prototypes for the kernel. + *========================================================================*/ + +/* + * Routines used for growing the Btree. + */ +STATIC int xfs_da3_root_split(xfs_da_state_t *state, + xfs_da_state_blk_t *existing_root, + xfs_da_state_blk_t *new_child); +STATIC int xfs_da3_node_split(xfs_da_state_t *state, + xfs_da_state_blk_t *existing_blk, + xfs_da_state_blk_t *split_blk, + xfs_da_state_blk_t *blk_to_add, + int treelevel, + int *result); +STATIC void xfs_da3_node_rebalance(xfs_da_state_t *state, + xfs_da_state_blk_t *node_blk_1, + xfs_da_state_blk_t *node_blk_2); +STATIC void xfs_da3_node_add(xfs_da_state_t *state, + xfs_da_state_blk_t *old_node_blk, + xfs_da_state_blk_t *new_node_blk); + +/* + * Routines used for shrinking the Btree. + */ +STATIC int xfs_da3_root_join(xfs_da_state_t *state, + xfs_da_state_blk_t *root_blk); +STATIC int xfs_da3_node_toosmall(xfs_da_state_t *state, int *retval); +STATIC void xfs_da3_node_remove(xfs_da_state_t *state, + xfs_da_state_blk_t *drop_blk); +STATIC void xfs_da3_node_unbalance(xfs_da_state_t *state, + xfs_da_state_blk_t *src_node_blk, + xfs_da_state_blk_t *dst_node_blk); + +/* + * Utility routines. + */ +STATIC int xfs_da3_blk_unlink(xfs_da_state_t *state, + xfs_da_state_blk_t *drop_blk, + xfs_da_state_blk_t *save_blk); + + +struct kmem_cache *xfs_da_state_cache; /* anchor for dir/attr state */ + +/* + * Allocate a dir-state structure. + * We don't put them on the stack since they're large. + */ +struct xfs_da_state * +xfs_da_state_alloc( + struct xfs_da_args *args) +{ + struct xfs_da_state *state; + + state = kmem_cache_zalloc(xfs_da_state_cache, + GFP_KERNEL | __GFP_NOLOCKDEP | __GFP_NOFAIL); + state->args = args; + state->mp = args->dp->i_mount; + return state; +} + +/* + * Kill the altpath contents of a da-state structure. + */ +STATIC void +xfs_da_state_kill_altpath(xfs_da_state_t *state) +{ + int i; + + for (i = 0; i < state->altpath.active; i++) + state->altpath.blk[i].bp = NULL; + state->altpath.active = 0; +} + +/* + * Free a da-state structure. + */ +void +xfs_da_state_free(xfs_da_state_t *state) +{ + xfs_da_state_kill_altpath(state); +#ifdef DEBUG + memset((char *)state, 0, sizeof(*state)); +#endif /* DEBUG */ + kmem_cache_free(xfs_da_state_cache, state); +} + +void +xfs_da_state_reset( + struct xfs_da_state *state, + struct xfs_da_args *args) +{ + xfs_da_state_kill_altpath(state); + memset(state, 0, sizeof(struct xfs_da_state)); + state->args = args; + state->mp = state->args->dp->i_mount; +} + +static inline int xfs_dabuf_nfsb(struct xfs_mount *mp, int whichfork) +{ + if (whichfork == XFS_DATA_FORK) + return mp->m_dir_geo->fsbcount; + return mp->m_attr_geo->fsbcount; +} + +void +xfs_da3_node_hdr_from_disk( + struct xfs_mount *mp, + struct xfs_da3_icnode_hdr *to, + struct xfs_da_intnode *from) +{ + if (xfs_has_crc(mp)) { + struct xfs_da3_intnode *from3 = (struct xfs_da3_intnode *)from; + + to->forw = be32_to_cpu(from3->hdr.info.hdr.forw); + to->back = be32_to_cpu(from3->hdr.info.hdr.back); + to->magic = be16_to_cpu(from3->hdr.info.hdr.magic); + to->count = be16_to_cpu(from3->hdr.__count); + to->level = be16_to_cpu(from3->hdr.__level); + to->btree = from3->__btree; + ASSERT(to->magic == XFS_DA3_NODE_MAGIC); + } else { + to->forw = be32_to_cpu(from->hdr.info.forw); + to->back = be32_to_cpu(from->hdr.info.back); + to->magic = be16_to_cpu(from->hdr.info.magic); + to->count = be16_to_cpu(from->hdr.__count); + to->level = be16_to_cpu(from->hdr.__level); + to->btree = from->__btree; + ASSERT(to->magic == XFS_DA_NODE_MAGIC); + } +} + +void +xfs_da3_node_hdr_to_disk( + struct xfs_mount *mp, + struct xfs_da_intnode *to, + struct xfs_da3_icnode_hdr *from) +{ + if (xfs_has_crc(mp)) { + struct xfs_da3_intnode *to3 = (struct xfs_da3_intnode *)to; + + ASSERT(from->magic == XFS_DA3_NODE_MAGIC); + to3->hdr.info.hdr.forw = cpu_to_be32(from->forw); + to3->hdr.info.hdr.back = cpu_to_be32(from->back); + to3->hdr.info.hdr.magic = cpu_to_be16(from->magic); + to3->hdr.__count = cpu_to_be16(from->count); + to3->hdr.__level = cpu_to_be16(from->level); + } else { + ASSERT(from->magic == XFS_DA_NODE_MAGIC); + to->hdr.info.forw = cpu_to_be32(from->forw); + to->hdr.info.back = cpu_to_be32(from->back); + to->hdr.info.magic = cpu_to_be16(from->magic); + to->hdr.__count = cpu_to_be16(from->count); + to->hdr.__level = cpu_to_be16(from->level); + } +} + +/* + * Verify an xfs_da3_blkinfo structure. Note that the da3 fields are only + * accessible on v5 filesystems. This header format is common across da node, + * attr leaf and dir leaf blocks. + */ +xfs_failaddr_t +xfs_da3_blkinfo_verify( + struct xfs_buf *bp, + struct xfs_da3_blkinfo *hdr3) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_da_blkinfo *hdr = &hdr3->hdr; + + if (!xfs_verify_magic16(bp, hdr->magic)) + return __this_address; + + if (xfs_has_crc(mp)) { + if (!uuid_equal(&hdr3->uuid, &mp->m_sb.sb_meta_uuid)) + return __this_address; + if (be64_to_cpu(hdr3->blkno) != xfs_buf_daddr(bp)) + return __this_address; + if (!xfs_log_check_lsn(mp, be64_to_cpu(hdr3->lsn))) + return __this_address; + } + + return NULL; +} + +static xfs_failaddr_t +xfs_da3_node_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_da_intnode *hdr = bp->b_addr; + struct xfs_da3_icnode_hdr ichdr; + xfs_failaddr_t fa; + + xfs_da3_node_hdr_from_disk(mp, &ichdr, hdr); + + fa = xfs_da3_blkinfo_verify(bp, bp->b_addr); + if (fa) + return fa; + + if (ichdr.level == 0) + return __this_address; + if (ichdr.level > XFS_DA_NODE_MAXDEPTH) + return __this_address; + if (ichdr.count == 0) + return __this_address; + + /* + * we don't know if the node is for and attribute or directory tree, + * so only fail if the count is outside both bounds + */ + if (ichdr.count > mp->m_dir_geo->node_ents && + ichdr.count > mp->m_attr_geo->node_ents) + return __this_address; + + /* XXX: hash order check? */ + + return NULL; +} + +xfs_failaddr_t +xfs_da3_node_header_check( + struct xfs_buf *bp, + xfs_ino_t owner) +{ + struct xfs_mount *mp = bp->b_mount; + + if (xfs_has_crc(mp)) { + struct xfs_da3_blkinfo *hdr3 = bp->b_addr; + + if (hdr3->hdr.magic != cpu_to_be16(XFS_DA3_NODE_MAGIC)) + return __this_address; + + if (be64_to_cpu(hdr3->owner) != owner) + return __this_address; + } + + return NULL; +} + +xfs_failaddr_t +xfs_da3_header_check( + struct xfs_buf *bp, + xfs_ino_t owner) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_da_blkinfo *hdr = bp->b_addr; + + if (!xfs_has_crc(mp)) + return NULL; + + switch (hdr->magic) { + case cpu_to_be16(XFS_ATTR3_LEAF_MAGIC): + return xfs_attr3_leaf_header_check(bp, owner); + case cpu_to_be16(XFS_DA3_NODE_MAGIC): + return xfs_da3_node_header_check(bp, owner); + case cpu_to_be16(XFS_DIR3_LEAF1_MAGIC): + case cpu_to_be16(XFS_DIR3_LEAFN_MAGIC): + return xfs_dir3_leaf_header_check(bp, owner); + } + + ASSERT(0); + return NULL; +} + +static void +xfs_da3_node_write_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_buf_log_item *bip = bp->b_log_item; + struct xfs_da3_node_hdr *hdr3 = bp->b_addr; + xfs_failaddr_t fa; + + fa = xfs_da3_node_verify(bp); + if (fa) { + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + return; + } + + if (!xfs_has_crc(mp)) + return; + + if (bip) + hdr3->info.lsn = cpu_to_be64(bip->bli_item.li_lsn); + + xfs_buf_update_cksum(bp, XFS_DA3_NODE_CRC_OFF); +} + +/* + * leaf/node format detection on trees is sketchy, so a node read can be done on + * leaf level blocks when detection identifies the tree as a node format tree + * incorrectly. In this case, we need to swap the verifier to match the correct + * format of the block being read. + */ +static void +xfs_da3_node_read_verify( + struct xfs_buf *bp) +{ + struct xfs_da_blkinfo *info = bp->b_addr; + xfs_failaddr_t fa; + + switch (be16_to_cpu(info->magic)) { + case XFS_DA3_NODE_MAGIC: + if (!xfs_buf_verify_cksum(bp, XFS_DA3_NODE_CRC_OFF)) { + xfs_verifier_error(bp, -EFSBADCRC, + __this_address); + break; + } + fallthrough; + case XFS_DA_NODE_MAGIC: + fa = xfs_da3_node_verify(bp); + if (fa) + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + return; + case XFS_ATTR_LEAF_MAGIC: + case XFS_ATTR3_LEAF_MAGIC: + bp->b_ops = &xfs_attr3_leaf_buf_ops; + bp->b_ops->verify_read(bp); + return; + case XFS_DIR2_LEAFN_MAGIC: + case XFS_DIR3_LEAFN_MAGIC: + bp->b_ops = &xfs_dir3_leafn_buf_ops; + bp->b_ops->verify_read(bp); + return; + default: + xfs_verifier_error(bp, -EFSCORRUPTED, __this_address); + break; + } +} + +/* Verify the structure of a da3 block. */ +static xfs_failaddr_t +xfs_da3_node_verify_struct( + struct xfs_buf *bp) +{ + struct xfs_da_blkinfo *info = bp->b_addr; + + switch (be16_to_cpu(info->magic)) { + case XFS_DA3_NODE_MAGIC: + case XFS_DA_NODE_MAGIC: + return xfs_da3_node_verify(bp); + case XFS_ATTR_LEAF_MAGIC: + case XFS_ATTR3_LEAF_MAGIC: + bp->b_ops = &xfs_attr3_leaf_buf_ops; + return bp->b_ops->verify_struct(bp); + case XFS_DIR2_LEAFN_MAGIC: + case XFS_DIR3_LEAFN_MAGIC: + bp->b_ops = &xfs_dir3_leafn_buf_ops; + return bp->b_ops->verify_struct(bp); + default: + return __this_address; + } +} + +const struct xfs_buf_ops xfs_da3_node_buf_ops = { + .name = "xfs_da3_node", + .magic16 = { cpu_to_be16(XFS_DA_NODE_MAGIC), + cpu_to_be16(XFS_DA3_NODE_MAGIC) }, + .verify_read = xfs_da3_node_read_verify, + .verify_write = xfs_da3_node_write_verify, + .verify_struct = xfs_da3_node_verify_struct, +}; + +static int +xfs_da3_node_set_type( + struct xfs_trans *tp, + struct xfs_inode *dp, + int whichfork, + struct xfs_buf *bp) +{ + struct xfs_da_blkinfo *info = bp->b_addr; + + switch (be16_to_cpu(info->magic)) { + case XFS_DA_NODE_MAGIC: + case XFS_DA3_NODE_MAGIC: + xfs_trans_buf_set_type(tp, bp, XFS_BLFT_DA_NODE_BUF); + return 0; + case XFS_ATTR_LEAF_MAGIC: + case XFS_ATTR3_LEAF_MAGIC: + xfs_trans_buf_set_type(tp, bp, XFS_BLFT_ATTR_LEAF_BUF); + return 0; + case XFS_DIR2_LEAFN_MAGIC: + case XFS_DIR3_LEAFN_MAGIC: + xfs_trans_buf_set_type(tp, bp, XFS_BLFT_DIR_LEAFN_BUF); + return 0; + default: + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, tp->t_mountp, + info, sizeof(*info)); + xfs_trans_brelse(tp, bp); + xfs_dirattr_mark_sick(dp, whichfork); + return -EFSCORRUPTED; + } +} + +int +xfs_da3_node_read( + struct xfs_trans *tp, + struct xfs_inode *dp, + xfs_dablk_t bno, + struct xfs_buf **bpp, + int whichfork) +{ + int error; + + error = xfs_da_read_buf(tp, dp, bno, 0, bpp, whichfork, + &xfs_da3_node_buf_ops); + if (error || !*bpp || !tp) + return error; + return xfs_da3_node_set_type(tp, dp, whichfork, *bpp); +} + +int +xfs_da3_node_read_mapped( + struct xfs_trans *tp, + struct xfs_inode *dp, + xfs_daddr_t mappedbno, + struct xfs_buf **bpp, + int whichfork) +{ + struct xfs_mount *mp = dp->i_mount; + int error; + + error = xfs_trans_read_buf(mp, tp, mp->m_ddev_targp, mappedbno, + XFS_FSB_TO_BB(mp, xfs_dabuf_nfsb(mp, whichfork)), 0, + bpp, &xfs_da3_node_buf_ops); + if (xfs_metadata_is_sick(error)) + xfs_dirattr_mark_sick(dp, whichfork); + if (error || !*bpp) + return error; + + if (whichfork == XFS_ATTR_FORK) + xfs_buf_set_ref(*bpp, XFS_ATTR_BTREE_REF); + else + xfs_buf_set_ref(*bpp, XFS_DIR_BTREE_REF); + + if (!tp) + return 0; + return xfs_da3_node_set_type(tp, dp, whichfork, *bpp); +} + +/* + * Copy src directory/attr leaf/node buffer to the dst. + * For v5 file systems make sure the right blkno is stamped in. + */ +void +xfs_da_buf_copy( + struct xfs_buf *dst, + struct xfs_buf *src, + size_t size) +{ + struct xfs_da3_blkinfo *da3 = dst->b_addr; + + memcpy(dst->b_addr, src->b_addr, size); + dst->b_ops = src->b_ops; + xfs_trans_buf_copy_type(dst, src); + if (xfs_has_crc(dst->b_mount)) + da3->blkno = cpu_to_be64(xfs_buf_daddr(dst)); +} + +/*======================================================================== + * Routines used for growing the Btree. + *========================================================================*/ + +/* + * Create the initial contents of an intermediate node. + */ +int +xfs_da3_node_create( + struct xfs_da_args *args, + xfs_dablk_t blkno, + int level, + struct xfs_buf **bpp, + int whichfork) +{ + struct xfs_da_intnode *node; + struct xfs_trans *tp = args->trans; + struct xfs_mount *mp = tp->t_mountp; + struct xfs_da3_icnode_hdr ichdr = {0}; + struct xfs_buf *bp; + int error; + struct xfs_inode *dp = args->dp; + + trace_xfs_da_node_create(args); + ASSERT(level <= XFS_DA_NODE_MAXDEPTH); + + error = xfs_da_get_buf(tp, dp, blkno, &bp, whichfork); + if (error) + return error; + bp->b_ops = &xfs_da3_node_buf_ops; + xfs_trans_buf_set_type(tp, bp, XFS_BLFT_DA_NODE_BUF); + node = bp->b_addr; + + if (xfs_has_crc(mp)) { + struct xfs_da3_node_hdr *hdr3 = bp->b_addr; + + memset(hdr3, 0, sizeof(struct xfs_da3_node_hdr)); + ichdr.magic = XFS_DA3_NODE_MAGIC; + hdr3->info.blkno = cpu_to_be64(xfs_buf_daddr(bp)); + hdr3->info.owner = cpu_to_be64(args->owner); + uuid_copy(&hdr3->info.uuid, &mp->m_sb.sb_meta_uuid); + } else { + ichdr.magic = XFS_DA_NODE_MAGIC; + } + ichdr.level = level; + + xfs_da3_node_hdr_to_disk(dp->i_mount, node, &ichdr); + xfs_trans_log_buf(tp, bp, + XFS_DA_LOGRANGE(node, &node->hdr, args->geo->node_hdr_size)); + + *bpp = bp; + return 0; +} + +/* + * Split a leaf node, rebalance, then possibly split + * intermediate nodes, rebalance, etc. + */ +int /* error */ +xfs_da3_split( + struct xfs_da_state *state) +{ + struct xfs_da_state_blk *oldblk; + struct xfs_da_state_blk *newblk; + struct xfs_da_state_blk *addblk; + struct xfs_da_intnode *node; + int max; + int action = 0; + int error; + int i; + + trace_xfs_da_split(state->args); + + if (XFS_TEST_ERROR(state->mp, XFS_ERRTAG_DA_LEAF_SPLIT)) + return -EIO; + + /* + * Walk back up the tree splitting/inserting/adjusting as necessary. + * If we need to insert and there isn't room, split the node, then + * decide which fragment to insert the new block from below into. + * Note that we may split the root this way, but we need more fixup. + */ + max = state->path.active - 1; + ASSERT((max >= 0) && (max < XFS_DA_NODE_MAXDEPTH)); + ASSERT(state->path.blk[max].magic == XFS_ATTR_LEAF_MAGIC || + state->path.blk[max].magic == XFS_DIR2_LEAFN_MAGIC); + + addblk = &state->path.blk[max]; /* initial dummy value */ + for (i = max; (i >= 0) && addblk; state->path.active--, i--) { + oldblk = &state->path.blk[i]; + newblk = &state->altpath.blk[i]; + + /* + * If a leaf node then + * Allocate a new leaf node, then rebalance across them. + * else if an intermediate node then + * We split on the last layer, must we split the node? + */ + switch (oldblk->magic) { + case XFS_ATTR_LEAF_MAGIC: + error = xfs_attr3_leaf_split(state, oldblk, newblk); + if (error < 0) + return error; /* GROT: attr is inconsistent */ + if (!error) { + addblk = newblk; + break; + } + /* + * Entry wouldn't fit, split the leaf again. The new + * extrablk will be consumed by xfs_da3_node_split if + * the node is split. + */ + state->extravalid = 1; + if (state->inleaf) { + state->extraafter = 0; /* before newblk */ + trace_xfs_attr_leaf_split_before(state->args); + error = xfs_attr3_leaf_split(state, oldblk, + &state->extrablk); + } else { + state->extraafter = 1; /* after newblk */ + trace_xfs_attr_leaf_split_after(state->args); + error = xfs_attr3_leaf_split(state, newblk, + &state->extrablk); + } + if (error == 1) + return -ENOSPC; + if (error) + return error; /* GROT: attr inconsistent */ + addblk = newblk; + break; + case XFS_DIR2_LEAFN_MAGIC: + error = xfs_dir2_leafn_split(state, oldblk, newblk); + if (error) + return error; + addblk = newblk; + break; + case XFS_DA_NODE_MAGIC: + error = xfs_da3_node_split(state, oldblk, newblk, addblk, + max - i, &action); + addblk->bp = NULL; + if (error) + return error; /* GROT: dir is inconsistent */ + /* + * Record the newly split block for the next time thru? + */ + if (action) + addblk = newblk; + else + addblk = NULL; + break; + } + + /* + * Update the btree to show the new hashval for this child. + */ + xfs_da3_fixhashpath(state, &state->path); + } + if (!addblk) + return 0; + + /* + * xfs_da3_node_split() should have consumed any extra blocks we added + * during a double leaf split in the attr fork. This is guaranteed as + * we can't be here if the attr fork only has a single leaf block. + */ + ASSERT(state->extravalid == 0 || + state->path.blk[max].magic == XFS_DIR2_LEAFN_MAGIC); + + /* + * Split the root node. + */ + ASSERT(state->path.active == 0); + oldblk = &state->path.blk[0]; + error = xfs_da3_root_split(state, oldblk, addblk); + if (error) + goto out; + + /* + * Update pointers to the node which used to be block 0 and just got + * bumped because of the addition of a new root node. Note that the + * original block 0 could be at any position in the list of blocks in + * the tree. + * + * Note: the magic numbers and sibling pointers are in the same physical + * place for both v2 and v3 headers (by design). Hence it doesn't matter + * which version of the xfs_da_intnode structure we use here as the + * result will be the same using either structure. + */ + node = oldblk->bp->b_addr; + if (node->hdr.info.forw) { + if (be32_to_cpu(node->hdr.info.forw) != addblk->blkno) { + xfs_buf_mark_corrupt(oldblk->bp); + xfs_da_mark_sick(state->args); + error = -EFSCORRUPTED; + goto out; + } + node = addblk->bp->b_addr; + node->hdr.info.back = cpu_to_be32(oldblk->blkno); + xfs_trans_log_buf(state->args->trans, addblk->bp, + XFS_DA_LOGRANGE(node, &node->hdr.info, + sizeof(node->hdr.info))); + } + node = oldblk->bp->b_addr; + if (node->hdr.info.back) { + if (be32_to_cpu(node->hdr.info.back) != addblk->blkno) { + xfs_buf_mark_corrupt(oldblk->bp); + xfs_da_mark_sick(state->args); + error = -EFSCORRUPTED; + goto out; + } + node = addblk->bp->b_addr; + node->hdr.info.forw = cpu_to_be32(oldblk->blkno); + xfs_trans_log_buf(state->args->trans, addblk->bp, + XFS_DA_LOGRANGE(node, &node->hdr.info, + sizeof(node->hdr.info))); + } +out: + addblk->bp = NULL; + return error; +} + +/* + * Split the root. We have to create a new root and point to the two + * parts (the split old root) that we just created. Copy block zero to + * the EOF, extending the inode in process. + */ +STATIC int /* error */ +xfs_da3_root_split( + struct xfs_da_state *state, + struct xfs_da_state_blk *blk1, + struct xfs_da_state_blk *blk2) +{ + struct xfs_da_intnode *node; + struct xfs_da_intnode *oldroot; + struct xfs_da_node_entry *btree; + struct xfs_da3_icnode_hdr nodehdr; + struct xfs_da_args *args; + struct xfs_buf *bp; + struct xfs_inode *dp; + struct xfs_trans *tp; + struct xfs_dir2_leaf *leaf; + xfs_dablk_t blkno; + int level; + int error; + int size; + + trace_xfs_da_root_split(state->args); + + /* + * Copy the existing (incorrect) block from the root node position + * to a free space somewhere. + */ + args = state->args; + error = xfs_da_grow_inode(args, &blkno); + if (error) + return error; + + dp = args->dp; + tp = args->trans; + error = xfs_da_get_buf(tp, dp, blkno, &bp, args->whichfork); + if (error) + return error; + node = bp->b_addr; + oldroot = blk1->bp->b_addr; + if (oldroot->hdr.info.magic == cpu_to_be16(XFS_DA_NODE_MAGIC) || + oldroot->hdr.info.magic == cpu_to_be16(XFS_DA3_NODE_MAGIC)) { + struct xfs_da3_icnode_hdr icnodehdr; + + xfs_da3_node_hdr_from_disk(dp->i_mount, &icnodehdr, oldroot); + btree = icnodehdr.btree; + size = (int)((char *)&btree[icnodehdr.count] - (char *)oldroot); + level = icnodehdr.level; + } else { + struct xfs_dir3_icleaf_hdr leafhdr; + + leaf = (xfs_dir2_leaf_t *)oldroot; + xfs_dir2_leaf_hdr_from_disk(dp->i_mount, &leafhdr, leaf); + + ASSERT(leafhdr.magic == XFS_DIR2_LEAFN_MAGIC || + leafhdr.magic == XFS_DIR3_LEAFN_MAGIC); + size = (int)((char *)&leafhdr.ents[leafhdr.count] - + (char *)leaf); + level = 0; + } + + /* + * Copy old root to new buffer and log it. + */ + xfs_da_buf_copy(bp, blk1->bp, size); + xfs_trans_log_buf(tp, bp, 0, size - 1); + + /* + * Update blk1 to point to new buffer. + */ + blk1->bp = bp; + blk1->blkno = blkno; + + /* + * Set up the new root node. + */ + error = xfs_da3_node_create(args, + (args->whichfork == XFS_DATA_FORK) ? args->geo->leafblk : 0, + level + 1, &bp, args->whichfork); + if (error) + return error; + + node = bp->b_addr; + xfs_da3_node_hdr_from_disk(dp->i_mount, &nodehdr, node); + btree = nodehdr.btree; + btree[0].hashval = cpu_to_be32(blk1->hashval); + btree[0].before = cpu_to_be32(blk1->blkno); + btree[1].hashval = cpu_to_be32(blk2->hashval); + btree[1].before = cpu_to_be32(blk2->blkno); + nodehdr.count = 2; + xfs_da3_node_hdr_to_disk(dp->i_mount, node, &nodehdr); + +#ifdef DEBUG + if (oldroot->hdr.info.magic == cpu_to_be16(XFS_DIR2_LEAFN_MAGIC) || + oldroot->hdr.info.magic == cpu_to_be16(XFS_DIR3_LEAFN_MAGIC)) { + ASSERT(blk1->blkno >= args->geo->leafblk && + blk1->blkno < args->geo->freeblk); + ASSERT(blk2->blkno >= args->geo->leafblk && + blk2->blkno < args->geo->freeblk); + } +#endif + + /* Header is already logged by xfs_da_node_create */ + xfs_trans_log_buf(tp, bp, + XFS_DA_LOGRANGE(node, btree, sizeof(xfs_da_node_entry_t) * 2)); + + return 0; +} + +/* + * Split the node, rebalance, then add the new entry. + */ +STATIC int /* error */ +xfs_da3_node_split( + struct xfs_da_state *state, + struct xfs_da_state_blk *oldblk, + struct xfs_da_state_blk *newblk, + struct xfs_da_state_blk *addblk, + int treelevel, + int *result) +{ + struct xfs_da_intnode *node; + struct xfs_da3_icnode_hdr nodehdr; + xfs_dablk_t blkno; + int newcount; + int error; + int useextra; + struct xfs_inode *dp = state->args->dp; + + trace_xfs_da_node_split(state->args); + + node = oldblk->bp->b_addr; + xfs_da3_node_hdr_from_disk(dp->i_mount, &nodehdr, node); + + /* + * With V2 dirs the extra block is data or freespace. + */ + useextra = state->extravalid && state->args->whichfork == XFS_ATTR_FORK; + newcount = 1 + useextra; + /* + * Do we have to split the node? + */ + if (nodehdr.count + newcount > state->args->geo->node_ents) { + /* + * Allocate a new node, add to the doubly linked chain of + * nodes, then move some of our excess entries into it. + */ + error = xfs_da_grow_inode(state->args, &blkno); + if (error) + return error; /* GROT: dir is inconsistent */ + + error = xfs_da3_node_create(state->args, blkno, treelevel, + &newblk->bp, state->args->whichfork); + if (error) + return error; /* GROT: dir is inconsistent */ + newblk->blkno = blkno; + newblk->magic = XFS_DA_NODE_MAGIC; + xfs_da3_node_rebalance(state, oldblk, newblk); + error = xfs_da3_blk_link(state, oldblk, newblk); + if (error) + return error; + *result = 1; + } else { + *result = 0; + } + + /* + * Insert the new entry(s) into the correct block + * (updating last hashval in the process). + * + * xfs_da3_node_add() inserts BEFORE the given index, + * and as a result of using node_lookup_int() we always + * point to a valid entry (not after one), but a split + * operation always results in a new block whose hashvals + * FOLLOW the current block. + * + * If we had double-split op below us, then add the extra block too. + */ + node = oldblk->bp->b_addr; + xfs_da3_node_hdr_from_disk(dp->i_mount, &nodehdr, node); + if (oldblk->index <= nodehdr.count) { + oldblk->index++; + xfs_da3_node_add(state, oldblk, addblk); + if (useextra) { + if (state->extraafter) + oldblk->index++; + xfs_da3_node_add(state, oldblk, &state->extrablk); + state->extravalid = 0; + } + } else { + newblk->index++; + xfs_da3_node_add(state, newblk, addblk); + if (useextra) { + if (state->extraafter) + newblk->index++; + xfs_da3_node_add(state, newblk, &state->extrablk); + state->extravalid = 0; + } + } + + return 0; +} + +/* + * Balance the btree elements between two intermediate nodes, + * usually one full and one empty. + * + * NOTE: if blk2 is empty, then it will get the upper half of blk1. + */ +STATIC void +xfs_da3_node_rebalance( + struct xfs_da_state *state, + struct xfs_da_state_blk *blk1, + struct xfs_da_state_blk *blk2) +{ + struct xfs_da_intnode *node1; + struct xfs_da_intnode *node2; + struct xfs_da_node_entry *btree1; + struct xfs_da_node_entry *btree2; + struct xfs_da_node_entry *btree_s; + struct xfs_da_node_entry *btree_d; + struct xfs_da3_icnode_hdr nodehdr1; + struct xfs_da3_icnode_hdr nodehdr2; + struct xfs_trans *tp; + int count; + int tmp; + int swap = 0; + struct xfs_inode *dp = state->args->dp; + + trace_xfs_da_node_rebalance(state->args); + + node1 = blk1->bp->b_addr; + node2 = blk2->bp->b_addr; + xfs_da3_node_hdr_from_disk(dp->i_mount, &nodehdr1, node1); + xfs_da3_node_hdr_from_disk(dp->i_mount, &nodehdr2, node2); + btree1 = nodehdr1.btree; + btree2 = nodehdr2.btree; + + /* + * Figure out how many entries need to move, and in which direction. + * Swap the nodes around if that makes it simpler. + */ + if (nodehdr1.count > 0 && nodehdr2.count > 0 && + ((be32_to_cpu(btree2[0].hashval) < be32_to_cpu(btree1[0].hashval)) || + (be32_to_cpu(btree2[nodehdr2.count - 1].hashval) < + be32_to_cpu(btree1[nodehdr1.count - 1].hashval)))) { + swap(node1, node2); + xfs_da3_node_hdr_from_disk(dp->i_mount, &nodehdr1, node1); + xfs_da3_node_hdr_from_disk(dp->i_mount, &nodehdr2, node2); + btree1 = nodehdr1.btree; + btree2 = nodehdr2.btree; + swap = 1; + } + + count = (nodehdr1.count - nodehdr2.count) / 2; + if (count == 0) + return; + tp = state->args->trans; + /* + * Two cases: high-to-low and low-to-high. + */ + if (count > 0) { + /* + * Move elements in node2 up to make a hole. + */ + tmp = nodehdr2.count; + if (tmp > 0) { + tmp *= (uint)sizeof(xfs_da_node_entry_t); + btree_s = &btree2[0]; + btree_d = &btree2[count]; + memmove(btree_d, btree_s, tmp); + } + + /* + * Move the req'd B-tree elements from high in node1 to + * low in node2. + */ + nodehdr2.count += count; + tmp = count * (uint)sizeof(xfs_da_node_entry_t); + btree_s = &btree1[nodehdr1.count - count]; + btree_d = &btree2[0]; + memcpy(btree_d, btree_s, tmp); + nodehdr1.count -= count; + } else { + /* + * Move the req'd B-tree elements from low in node2 to + * high in node1. + */ + count = -count; + tmp = count * (uint)sizeof(xfs_da_node_entry_t); + btree_s = &btree2[0]; + btree_d = &btree1[nodehdr1.count]; + memcpy(btree_d, btree_s, tmp); + nodehdr1.count += count; + + xfs_trans_log_buf(tp, blk1->bp, + XFS_DA_LOGRANGE(node1, btree_d, tmp)); + + /* + * Move elements in node2 down to fill the hole. + */ + tmp = nodehdr2.count - count; + tmp *= (uint)sizeof(xfs_da_node_entry_t); + btree_s = &btree2[count]; + btree_d = &btree2[0]; + memmove(btree_d, btree_s, tmp); + nodehdr2.count -= count; + } + + /* + * Log header of node 1 and all current bits of node 2. + */ + xfs_da3_node_hdr_to_disk(dp->i_mount, node1, &nodehdr1); + xfs_trans_log_buf(tp, blk1->bp, + XFS_DA_LOGRANGE(node1, &node1->hdr, + state->args->geo->node_hdr_size)); + + xfs_da3_node_hdr_to_disk(dp->i_mount, node2, &nodehdr2); + xfs_trans_log_buf(tp, blk2->bp, + XFS_DA_LOGRANGE(node2, &node2->hdr, + state->args->geo->node_hdr_size + + (sizeof(btree2[0]) * nodehdr2.count))); + + /* + * Record the last hashval from each block for upward propagation. + * (note: don't use the swapped node pointers) + */ + if (swap) { + node1 = blk1->bp->b_addr; + node2 = blk2->bp->b_addr; + xfs_da3_node_hdr_from_disk(dp->i_mount, &nodehdr1, node1); + xfs_da3_node_hdr_from_disk(dp->i_mount, &nodehdr2, node2); + btree1 = nodehdr1.btree; + btree2 = nodehdr2.btree; + } + blk1->hashval = be32_to_cpu(btree1[nodehdr1.count - 1].hashval); + blk2->hashval = be32_to_cpu(btree2[nodehdr2.count - 1].hashval); + + /* + * Adjust the expected index for insertion. + */ + if (blk1->index >= nodehdr1.count) { + blk2->index = blk1->index - nodehdr1.count; + blk1->index = nodehdr1.count + 1; /* make it invalid */ + } +} + +/* + * Add a new entry to an intermediate node. + */ +STATIC void +xfs_da3_node_add( + struct xfs_da_state *state, + struct xfs_da_state_blk *oldblk, + struct xfs_da_state_blk *newblk) +{ + struct xfs_da_intnode *node; + struct xfs_da3_icnode_hdr nodehdr; + struct xfs_da_node_entry *btree; + int tmp; + struct xfs_inode *dp = state->args->dp; + + trace_xfs_da_node_add(state->args); + + node = oldblk->bp->b_addr; + xfs_da3_node_hdr_from_disk(dp->i_mount, &nodehdr, node); + btree = nodehdr.btree; + + ASSERT(oldblk->index >= 0 && oldblk->index <= nodehdr.count); + ASSERT(newblk->blkno != 0); + if (state->args->whichfork == XFS_DATA_FORK) + ASSERT(newblk->blkno >= state->args->geo->leafblk && + newblk->blkno < state->args->geo->freeblk); + + /* + * We may need to make some room before we insert the new node. + */ + tmp = 0; + if (oldblk->index < nodehdr.count) { + tmp = (nodehdr.count - oldblk->index) * (uint)sizeof(*btree); + memmove(&btree[oldblk->index + 1], &btree[oldblk->index], tmp); + } + btree[oldblk->index].hashval = cpu_to_be32(newblk->hashval); + btree[oldblk->index].before = cpu_to_be32(newblk->blkno); + xfs_trans_log_buf(state->args->trans, oldblk->bp, + XFS_DA_LOGRANGE(node, &btree[oldblk->index], + tmp + sizeof(*btree))); + + nodehdr.count += 1; + xfs_da3_node_hdr_to_disk(dp->i_mount, node, &nodehdr); + xfs_trans_log_buf(state->args->trans, oldblk->bp, + XFS_DA_LOGRANGE(node, &node->hdr, + state->args->geo->node_hdr_size)); + + /* + * Copy the last hash value from the oldblk to propagate upwards. + */ + oldblk->hashval = be32_to_cpu(btree[nodehdr.count - 1].hashval); +} + +/*======================================================================== + * Routines used for shrinking the Btree. + *========================================================================*/ + +/* + * Deallocate an empty leaf node, remove it from its parent, + * possibly deallocating that block, etc... + */ +int +xfs_da3_join( + struct xfs_da_state *state) +{ + struct xfs_da_state_blk *drop_blk; + struct xfs_da_state_blk *save_blk; + int action = 0; + int error; + + trace_xfs_da_join(state->args); + + drop_blk = &state->path.blk[ state->path.active-1 ]; + save_blk = &state->altpath.blk[ state->path.active-1 ]; + ASSERT(state->path.blk[0].magic == XFS_DA_NODE_MAGIC); + ASSERT(drop_blk->magic == XFS_ATTR_LEAF_MAGIC || + drop_blk->magic == XFS_DIR2_LEAFN_MAGIC); + + /* + * Walk back up the tree joining/deallocating as necessary. + * When we stop dropping blocks, break out. + */ + for ( ; state->path.active >= 2; drop_blk--, save_blk--, + state->path.active--) { + /* + * See if we can combine the block with a neighbor. + * (action == 0) => no options, just leave + * (action == 1) => coalesce, then unlink + * (action == 2) => block empty, unlink it + */ + switch (drop_blk->magic) { + case XFS_ATTR_LEAF_MAGIC: + error = xfs_attr3_leaf_toosmall(state, &action); + if (error) + return error; + if (action == 0) + return 0; + xfs_attr3_leaf_unbalance(state, drop_blk, save_blk); + break; + case XFS_DIR2_LEAFN_MAGIC: + error = xfs_dir2_leafn_toosmall(state, &action); + if (error) + return error; + if (action == 0) + return 0; + xfs_dir2_leafn_unbalance(state, drop_blk, save_blk); + break; + case XFS_DA_NODE_MAGIC: + /* + * Remove the offending node, fixup hashvals, + * check for a toosmall neighbor. + */ + xfs_da3_node_remove(state, drop_blk); + xfs_da3_fixhashpath(state, &state->path); + error = xfs_da3_node_toosmall(state, &action); + if (error) + return error; + if (action == 0) + return 0; + xfs_da3_node_unbalance(state, drop_blk, save_blk); + break; + } + xfs_da3_fixhashpath(state, &state->altpath); + error = xfs_da3_blk_unlink(state, drop_blk, save_blk); + xfs_da_state_kill_altpath(state); + if (error) + return error; + error = xfs_da_shrink_inode(state->args, drop_blk->blkno, + drop_blk->bp); + drop_blk->bp = NULL; + if (error) + return error; + } + /* + * We joined all the way to the top. If it turns out that + * we only have one entry in the root, make the child block + * the new root. + */ + xfs_da3_node_remove(state, drop_blk); + xfs_da3_fixhashpath(state, &state->path); + error = xfs_da3_root_join(state, &state->path.blk[0]); + return error; +} + +#ifdef DEBUG +static void +xfs_da_blkinfo_onlychild_validate(struct xfs_da_blkinfo *blkinfo, __u16 level) +{ + __be16 magic = blkinfo->magic; + + if (level == 1) { + ASSERT(magic == cpu_to_be16(XFS_DIR2_LEAFN_MAGIC) || + magic == cpu_to_be16(XFS_DIR3_LEAFN_MAGIC) || + magic == cpu_to_be16(XFS_ATTR_LEAF_MAGIC) || + magic == cpu_to_be16(XFS_ATTR3_LEAF_MAGIC)); + } else { + ASSERT(magic == cpu_to_be16(XFS_DA_NODE_MAGIC) || + magic == cpu_to_be16(XFS_DA3_NODE_MAGIC)); + } + ASSERT(!blkinfo->forw); + ASSERT(!blkinfo->back); +} +#else /* !DEBUG */ +#define xfs_da_blkinfo_onlychild_validate(blkinfo, level) +#endif /* !DEBUG */ + +/* + * We have only one entry in the root. Copy the only remaining child of + * the old root to block 0 as the new root node. + */ +STATIC int +xfs_da3_root_join( + struct xfs_da_state *state, + struct xfs_da_state_blk *root_blk) +{ + struct xfs_da_intnode *oldroot; + struct xfs_da_args *args; + xfs_dablk_t child; + struct xfs_buf *bp; + struct xfs_da3_icnode_hdr oldroothdr; + int error; + struct xfs_inode *dp = state->args->dp; + xfs_failaddr_t fa; + + trace_xfs_da_root_join(state->args); + + ASSERT(root_blk->magic == XFS_DA_NODE_MAGIC); + + args = state->args; + oldroot = root_blk->bp->b_addr; + xfs_da3_node_hdr_from_disk(dp->i_mount, &oldroothdr, oldroot); + ASSERT(oldroothdr.forw == 0); + ASSERT(oldroothdr.back == 0); + + /* + * If the root has more than one child, then don't do anything. + */ + if (oldroothdr.count > 1) + return 0; + + /* + * Read in the (only) child block, then copy those bytes into + * the root block's buffer and free the original child block. + */ + child = be32_to_cpu(oldroothdr.btree[0].before); + ASSERT(child != 0); + error = xfs_da3_node_read(args->trans, dp, child, &bp, args->whichfork); + if (error) + return error; + fa = xfs_da3_header_check(bp, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(bp, fa); + xfs_trans_brelse(args->trans, bp); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + xfs_da_blkinfo_onlychild_validate(bp->b_addr, oldroothdr.level); + + /* + * Copy child to root buffer and log it. + */ + xfs_da_buf_copy(root_blk->bp, bp, args->geo->blksize); + xfs_trans_log_buf(args->trans, root_blk->bp, 0, + args->geo->blksize - 1); + /* + * Now we can drop the child buffer. + */ + error = xfs_da_shrink_inode(args, child, bp); + return error; +} + +/* + * Check a node block and its neighbors to see if the block should be + * collapsed into one or the other neighbor. Always keep the block + * with the smaller block number. + * If the current block is over 50% full, don't try to join it, return 0. + * If the block is empty, fill in the state structure and return 2. + * If it can be collapsed, fill in the state structure and return 1. + * If nothing can be done, return 0. + */ +STATIC int +xfs_da3_node_toosmall( + struct xfs_da_state *state, + int *action) +{ + struct xfs_da_intnode *node; + struct xfs_da_state_blk *blk; + struct xfs_da_blkinfo *info; + xfs_dablk_t blkno; + struct xfs_buf *bp; + xfs_failaddr_t fa; + struct xfs_da3_icnode_hdr nodehdr; + int count; + int forward; + int error; + int retval; + int i; + struct xfs_inode *dp = state->args->dp; + + trace_xfs_da_node_toosmall(state->args); + + /* + * Check for the degenerate case of the block being over 50% full. + * If so, it's not worth even looking to see if we might be able + * to coalesce with a sibling. + */ + blk = &state->path.blk[ state->path.active-1 ]; + info = blk->bp->b_addr; + node = (xfs_da_intnode_t *)info; + xfs_da3_node_hdr_from_disk(dp->i_mount, &nodehdr, node); + if (nodehdr.count > (state->args->geo->node_ents >> 1)) { + *action = 0; /* blk over 50%, don't try to join */ + return 0; /* blk over 50%, don't try to join */ + } + + /* + * Check for the degenerate case of the block being empty. + * If the block is empty, we'll simply delete it, no need to + * coalesce it with a sibling block. We choose (arbitrarily) + * to merge with the forward block unless it is NULL. + */ + if (nodehdr.count == 0) { + /* + * Make altpath point to the block we want to keep and + * path point to the block we want to drop (this one). + */ + forward = (info->forw != 0); + memcpy(&state->altpath, &state->path, sizeof(state->path)); + error = xfs_da3_path_shift(state, &state->altpath, forward, + 0, &retval); + if (error) + return error; + if (retval) { + *action = 0; + } else { + *action = 2; + } + return 0; + } + + /* + * Examine each sibling block to see if we can coalesce with + * at least 25% free space to spare. We need to figure out + * whether to merge with the forward or the backward block. + * We prefer coalescing with the lower numbered sibling so as + * to shrink a directory over time. + */ + count = state->args->geo->node_ents; + count -= state->args->geo->node_ents >> 2; + count -= nodehdr.count; + + /* start with smaller blk num */ + forward = nodehdr.forw < nodehdr.back; + for (i = 0; i < 2; forward = !forward, i++) { + struct xfs_da3_icnode_hdr thdr; + if (forward) + blkno = nodehdr.forw; + else + blkno = nodehdr.back; + if (blkno == 0) + continue; + error = xfs_da3_node_read(state->args->trans, dp, blkno, &bp, + state->args->whichfork); + if (error) + return error; + fa = xfs_da3_node_header_check(bp, state->args->owner); + if (fa) { + __xfs_buf_mark_corrupt(bp, fa); + xfs_trans_brelse(state->args->trans, bp); + xfs_da_mark_sick(state->args); + return -EFSCORRUPTED; + } + + node = bp->b_addr; + xfs_da3_node_hdr_from_disk(dp->i_mount, &thdr, node); + xfs_trans_brelse(state->args->trans, bp); + + if (count - thdr.count >= 0) + break; /* fits with at least 25% to spare */ + } + if (i >= 2) { + *action = 0; + return 0; + } + + /* + * Make altpath point to the block we want to keep (the lower + * numbered block) and path point to the block we want to drop. + */ + memcpy(&state->altpath, &state->path, sizeof(state->path)); + if (blkno < blk->blkno) { + error = xfs_da3_path_shift(state, &state->altpath, forward, + 0, &retval); + } else { + error = xfs_da3_path_shift(state, &state->path, forward, + 0, &retval); + } + if (error) + return error; + if (retval) { + *action = 0; + return 0; + } + *action = 1; + return 0; +} + +/* + * Pick up the last hashvalue from an intermediate node. + */ +STATIC uint +xfs_da3_node_lasthash( + struct xfs_inode *dp, + struct xfs_buf *bp, + int *count) +{ + struct xfs_da3_icnode_hdr nodehdr; + + xfs_da3_node_hdr_from_disk(dp->i_mount, &nodehdr, bp->b_addr); + if (count) + *count = nodehdr.count; + if (!nodehdr.count) + return 0; + return be32_to_cpu(nodehdr.btree[nodehdr.count - 1].hashval); +} + +/* + * Walk back up the tree adjusting hash values as necessary, + * when we stop making changes, return. + */ +void +xfs_da3_fixhashpath( + struct xfs_da_state *state, + struct xfs_da_state_path *path) +{ + struct xfs_da_state_blk *blk; + struct xfs_da_intnode *node; + struct xfs_da_node_entry *btree; + xfs_dahash_t lasthash=0; + int level; + int count; + struct xfs_inode *dp = state->args->dp; + + trace_xfs_da_fixhashpath(state->args); + + level = path->active-1; + blk = &path->blk[ level ]; + switch (blk->magic) { + case XFS_ATTR_LEAF_MAGIC: + lasthash = xfs_attr_leaf_lasthash(blk->bp, &count); + if (count == 0) + return; + break; + case XFS_DIR2_LEAFN_MAGIC: + lasthash = xfs_dir2_leaf_lasthash(dp, blk->bp, &count); + if (count == 0) + return; + break; + case XFS_DA_NODE_MAGIC: + lasthash = xfs_da3_node_lasthash(dp, blk->bp, &count); + if (count == 0) + return; + break; + } + for (blk--, level--; level >= 0; blk--, level--) { + struct xfs_da3_icnode_hdr nodehdr; + + node = blk->bp->b_addr; + xfs_da3_node_hdr_from_disk(dp->i_mount, &nodehdr, node); + btree = nodehdr.btree; + if (be32_to_cpu(btree[blk->index].hashval) == lasthash) + break; + blk->hashval = lasthash; + btree[blk->index].hashval = cpu_to_be32(lasthash); + xfs_trans_log_buf(state->args->trans, blk->bp, + XFS_DA_LOGRANGE(node, &btree[blk->index], + sizeof(*btree))); + + lasthash = be32_to_cpu(btree[nodehdr.count - 1].hashval); + } +} + +/* + * Internal implementation to remove an entry from an intermediate node. + */ +STATIC void +__xfs_da3_node_remove( + struct xfs_trans *tp, + struct xfs_inode *dp, + struct xfs_da_geometry *geo, + struct xfs_da_state_blk *drop_blk) +{ + struct xfs_da_intnode *node; + struct xfs_da3_icnode_hdr nodehdr; + struct xfs_da_node_entry *btree; + int index; + int tmp; + + node = drop_blk->bp->b_addr; + xfs_da3_node_hdr_from_disk(dp->i_mount, &nodehdr, node); + ASSERT(drop_blk->index < nodehdr.count); + ASSERT(drop_blk->index >= 0); + + /* + * Copy over the offending entry, or just zero it out. + */ + index = drop_blk->index; + btree = nodehdr.btree; + if (index < nodehdr.count - 1) { + tmp = nodehdr.count - index - 1; + tmp *= (uint)sizeof(xfs_da_node_entry_t); + memmove(&btree[index], &btree[index + 1], tmp); + xfs_trans_log_buf(tp, drop_blk->bp, + XFS_DA_LOGRANGE(node, &btree[index], tmp)); + index = nodehdr.count - 1; + } + memset(&btree[index], 0, sizeof(xfs_da_node_entry_t)); + xfs_trans_log_buf(tp, drop_blk->bp, + XFS_DA_LOGRANGE(node, &btree[index], sizeof(btree[index]))); + nodehdr.count -= 1; + xfs_da3_node_hdr_to_disk(dp->i_mount, node, &nodehdr); + xfs_trans_log_buf(tp, drop_blk->bp, + XFS_DA_LOGRANGE(node, &node->hdr, geo->node_hdr_size)); + + /* + * Copy the last hash value from the block to propagate upwards. + */ + drop_blk->hashval = be32_to_cpu(btree[index - 1].hashval); +} + +/* + * Remove an entry from an intermediate node. + */ +STATIC void +xfs_da3_node_remove( + struct xfs_da_state *state, + struct xfs_da_state_blk *drop_blk) +{ + trace_xfs_da_node_remove(state->args); + + __xfs_da3_node_remove(state->args->trans, state->args->dp, + state->args->geo, drop_blk); +} + +/* + * Remove an entry from an intermediate attr node at the specified index. + */ +void +xfs_attr3_node_entry_remove( + struct xfs_trans *tp, + struct xfs_inode *dp, + struct xfs_buf *bp, + int index) +{ + struct xfs_da_state_blk blk = { + .index = index, + .bp = bp, + }; + + __xfs_da3_node_remove(tp, dp, dp->i_mount->m_attr_geo, &blk); +} + +/* + * Unbalance the elements between two intermediate nodes, + * move all Btree elements from one node into another. + */ +STATIC void +xfs_da3_node_unbalance( + struct xfs_da_state *state, + struct xfs_da_state_blk *drop_blk, + struct xfs_da_state_blk *save_blk) +{ + struct xfs_da_intnode *drop_node; + struct xfs_da_intnode *save_node; + struct xfs_da_node_entry *drop_btree; + struct xfs_da_node_entry *save_btree; + struct xfs_da3_icnode_hdr drop_hdr; + struct xfs_da3_icnode_hdr save_hdr; + struct xfs_trans *tp; + int sindex; + int tmp; + struct xfs_inode *dp = state->args->dp; + + trace_xfs_da_node_unbalance(state->args); + + drop_node = drop_blk->bp->b_addr; + save_node = save_blk->bp->b_addr; + xfs_da3_node_hdr_from_disk(dp->i_mount, &drop_hdr, drop_node); + xfs_da3_node_hdr_from_disk(dp->i_mount, &save_hdr, save_node); + drop_btree = drop_hdr.btree; + save_btree = save_hdr.btree; + tp = state->args->trans; + + /* + * If the dying block has lower hashvals, then move all the + * elements in the remaining block up to make a hole. + */ + if ((be32_to_cpu(drop_btree[0].hashval) < + be32_to_cpu(save_btree[0].hashval)) || + (be32_to_cpu(drop_btree[drop_hdr.count - 1].hashval) < + be32_to_cpu(save_btree[save_hdr.count - 1].hashval))) { + /* XXX: check this - is memmove dst correct? */ + tmp = save_hdr.count * sizeof(xfs_da_node_entry_t); + memmove(&save_btree[drop_hdr.count], &save_btree[0], tmp); + + sindex = 0; + xfs_trans_log_buf(tp, save_blk->bp, + XFS_DA_LOGRANGE(save_node, &save_btree[0], + (save_hdr.count + drop_hdr.count) * + sizeof(xfs_da_node_entry_t))); + } else { + sindex = save_hdr.count; + xfs_trans_log_buf(tp, save_blk->bp, + XFS_DA_LOGRANGE(save_node, &save_btree[sindex], + drop_hdr.count * sizeof(xfs_da_node_entry_t))); + } + + /* + * Move all the B-tree elements from drop_blk to save_blk. + */ + tmp = drop_hdr.count * (uint)sizeof(xfs_da_node_entry_t); + memcpy(&save_btree[sindex], &drop_btree[0], tmp); + save_hdr.count += drop_hdr.count; + + xfs_da3_node_hdr_to_disk(dp->i_mount, save_node, &save_hdr); + xfs_trans_log_buf(tp, save_blk->bp, + XFS_DA_LOGRANGE(save_node, &save_node->hdr, + state->args->geo->node_hdr_size)); + + /* + * Save the last hashval in the remaining block for upward propagation. + */ + save_blk->hashval = be32_to_cpu(save_btree[save_hdr.count - 1].hashval); +} + +/*======================================================================== + * Routines used for finding things in the Btree. + *========================================================================*/ + +/* + * Walk down the Btree looking for a particular filename, filling + * in the state structure as we go. + * + * We will set the state structure to point to each of the elements + * in each of the nodes where either the hashval is or should be. + * + * We support duplicate hashval's so for each entry in the current + * node that could contain the desired hashval, descend. This is a + * pruned depth-first tree search. + */ +int /* error */ +xfs_da3_node_lookup_int( + struct xfs_da_state *state, + int *result) +{ + struct xfs_da_state_blk *blk; + struct xfs_da_blkinfo *curr; + struct xfs_da_intnode *node; + struct xfs_da_node_entry *btree; + struct xfs_da3_icnode_hdr nodehdr; + struct xfs_da_args *args; + xfs_failaddr_t fa; + xfs_dablk_t blkno; + xfs_dahash_t hashval; + xfs_dahash_t btreehashval; + int probe; + int span; + int max; + int error; + int retval; + unsigned int expected_level = 0; + uint16_t magic; + struct xfs_inode *dp = state->args->dp; + + args = state->args; + + /* + * Descend thru the B-tree searching each level for the right + * node to use, until the right hashval is found. + */ + blkno = args->geo->leafblk; + for (blk = &state->path.blk[0], state->path.active = 1; + state->path.active <= XFS_DA_NODE_MAXDEPTH; + blk++, state->path.active++) { + /* + * Read the next node down in the tree. + */ + blk->blkno = blkno; + error = xfs_da3_node_read(args->trans, args->dp, blkno, + &blk->bp, args->whichfork); + if (error) { + blk->blkno = 0; + state->path.active--; + return error; + } + curr = blk->bp->b_addr; + magic = be16_to_cpu(curr->magic); + + if (magic == XFS_ATTR_LEAF_MAGIC || + magic == XFS_ATTR3_LEAF_MAGIC) { + fa = xfs_attr3_leaf_header_check(blk->bp, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(blk->bp, fa); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + blk->magic = XFS_ATTR_LEAF_MAGIC; + blk->hashval = xfs_attr_leaf_lasthash(blk->bp, NULL); + break; + } + + if (magic == XFS_DIR2_LEAFN_MAGIC || + magic == XFS_DIR3_LEAFN_MAGIC) { + fa = xfs_dir3_leaf_header_check(blk->bp, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(blk->bp, fa); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + blk->magic = XFS_DIR2_LEAFN_MAGIC; + blk->hashval = xfs_dir2_leaf_lasthash(args->dp, + blk->bp, NULL); + break; + } + + if (magic != XFS_DA_NODE_MAGIC && magic != XFS_DA3_NODE_MAGIC) { + xfs_buf_mark_corrupt(blk->bp); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + + fa = xfs_da3_node_header_check(blk->bp, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(blk->bp, fa); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + + blk->magic = XFS_DA_NODE_MAGIC; + + /* + * Search an intermediate node for a match. + */ + node = blk->bp->b_addr; + xfs_da3_node_hdr_from_disk(dp->i_mount, &nodehdr, node); + btree = nodehdr.btree; + + /* Tree taller than we can handle; bail out! */ + if (nodehdr.level >= XFS_DA_NODE_MAXDEPTH) { + xfs_buf_mark_corrupt(blk->bp); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + + /* Check the level from the root. */ + if (blkno == args->geo->leafblk) + expected_level = nodehdr.level - 1; + else if (expected_level != nodehdr.level) { + xfs_buf_mark_corrupt(blk->bp); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } else + expected_level--; + + max = nodehdr.count; + blk->hashval = be32_to_cpu(btree[max - 1].hashval); + + /* + * Binary search. (note: small blocks will skip loop) + */ + probe = span = max / 2; + hashval = args->hashval; + while (span > 4) { + span /= 2; + btreehashval = be32_to_cpu(btree[probe].hashval); + if (btreehashval < hashval) + probe += span; + else if (btreehashval > hashval) + probe -= span; + else + break; + } + ASSERT((probe >= 0) && (probe < max)); + ASSERT((span <= 4) || + (be32_to_cpu(btree[probe].hashval) == hashval)); + + /* + * Since we may have duplicate hashval's, find the first + * matching hashval in the node. + */ + while (probe > 0 && + be32_to_cpu(btree[probe].hashval) >= hashval) { + probe--; + } + while (probe < max && + be32_to_cpu(btree[probe].hashval) < hashval) { + probe++; + } + + /* + * Pick the right block to descend on. + */ + if (probe == max) { + blk->index = max - 1; + blkno = be32_to_cpu(btree[max - 1].before); + } else { + blk->index = probe; + blkno = be32_to_cpu(btree[probe].before); + } + + /* We can't point back to the root. */ + if (XFS_IS_CORRUPT(dp->i_mount, blkno == args->geo->leafblk)) { + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + } + + if (XFS_IS_CORRUPT(dp->i_mount, expected_level != 0)) { + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + + /* + * A leaf block that ends in the hashval that we are interested in + * (final hashval == search hashval) means that the next block may + * contain more entries with the same hashval, shift upward to the + * next leaf and keep searching. + */ + for (;;) { + if (blk->magic == XFS_DIR2_LEAFN_MAGIC) { + retval = xfs_dir2_leafn_lookup_int(blk->bp, args, + &blk->index, state); + } else if (blk->magic == XFS_ATTR_LEAF_MAGIC) { + retval = xfs_attr3_leaf_lookup_int(blk->bp, args); + blk->index = args->index; + args->blkno = blk->blkno; + } else { + ASSERT(0); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + if (((retval == -ENOENT) || (retval == -ENOATTR)) && + (blk->hashval == args->hashval)) { + error = xfs_da3_path_shift(state, &state->path, 1, 1, + &retval); + if (error) + return error; + if (retval == 0) { + continue; + } else if (blk->magic == XFS_ATTR_LEAF_MAGIC) { + /* path_shift() gives ENOENT */ + retval = -ENOATTR; + } + } + break; + } + *result = retval; + return 0; +} + +/*======================================================================== + * Utility routines. + *========================================================================*/ + +/* + * Compare two intermediate nodes for "order". + */ +STATIC int +xfs_da3_node_order( + struct xfs_inode *dp, + struct xfs_buf *node1_bp, + struct xfs_buf *node2_bp) +{ + struct xfs_da_intnode *node1; + struct xfs_da_intnode *node2; + struct xfs_da_node_entry *btree1; + struct xfs_da_node_entry *btree2; + struct xfs_da3_icnode_hdr node1hdr; + struct xfs_da3_icnode_hdr node2hdr; + + node1 = node1_bp->b_addr; + node2 = node2_bp->b_addr; + xfs_da3_node_hdr_from_disk(dp->i_mount, &node1hdr, node1); + xfs_da3_node_hdr_from_disk(dp->i_mount, &node2hdr, node2); + btree1 = node1hdr.btree; + btree2 = node2hdr.btree; + + if (node1hdr.count > 0 && node2hdr.count > 0 && + ((be32_to_cpu(btree2[0].hashval) < be32_to_cpu(btree1[0].hashval)) || + (be32_to_cpu(btree2[node2hdr.count - 1].hashval) < + be32_to_cpu(btree1[node1hdr.count - 1].hashval)))) { + return 1; + } + return 0; +} + +/* + * Link a new block into a doubly linked list of blocks (of whatever type). + */ +int /* error */ +xfs_da3_blk_link( + struct xfs_da_state *state, + struct xfs_da_state_blk *old_blk, + struct xfs_da_state_blk *new_blk) +{ + struct xfs_da_blkinfo *old_info; + struct xfs_da_blkinfo *new_info; + struct xfs_da_blkinfo *tmp_info; + struct xfs_da_args *args; + struct xfs_buf *bp; + xfs_failaddr_t fa; + int before = 0; + int error; + struct xfs_inode *dp = state->args->dp; + + /* + * Set up environment. + */ + args = state->args; + ASSERT(args != NULL); + old_info = old_blk->bp->b_addr; + new_info = new_blk->bp->b_addr; + ASSERT(old_blk->magic == XFS_DA_NODE_MAGIC || + old_blk->magic == XFS_DIR2_LEAFN_MAGIC || + old_blk->magic == XFS_ATTR_LEAF_MAGIC); + + switch (old_blk->magic) { + case XFS_ATTR_LEAF_MAGIC: + before = xfs_attr_leaf_order(old_blk->bp, new_blk->bp); + break; + case XFS_DIR2_LEAFN_MAGIC: + before = xfs_dir2_leafn_order(dp, old_blk->bp, new_blk->bp); + break; + case XFS_DA_NODE_MAGIC: + before = xfs_da3_node_order(dp, old_blk->bp, new_blk->bp); + break; + } + + /* + * Link blocks in appropriate order. + */ + if (before) { + /* + * Link new block in before existing block. + */ + trace_xfs_da_link_before(args); + new_info->forw = cpu_to_be32(old_blk->blkno); + new_info->back = old_info->back; + if (old_info->back) { + error = xfs_da3_node_read(args->trans, dp, + be32_to_cpu(old_info->back), + &bp, args->whichfork); + if (error) + return error; + fa = xfs_da3_header_check(bp, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(bp, fa); + xfs_trans_brelse(args->trans, bp); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + ASSERT(bp != NULL); + tmp_info = bp->b_addr; + ASSERT(tmp_info->magic == old_info->magic); + ASSERT(be32_to_cpu(tmp_info->forw) == old_blk->blkno); + tmp_info->forw = cpu_to_be32(new_blk->blkno); + xfs_trans_log_buf(args->trans, bp, 0, sizeof(*tmp_info)-1); + } + old_info->back = cpu_to_be32(new_blk->blkno); + } else { + /* + * Link new block in after existing block. + */ + trace_xfs_da_link_after(args); + new_info->forw = old_info->forw; + new_info->back = cpu_to_be32(old_blk->blkno); + if (old_info->forw) { + error = xfs_da3_node_read(args->trans, dp, + be32_to_cpu(old_info->forw), + &bp, args->whichfork); + if (error) + return error; + fa = xfs_da3_header_check(bp, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(bp, fa); + xfs_trans_brelse(args->trans, bp); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + ASSERT(bp != NULL); + tmp_info = bp->b_addr; + ASSERT(tmp_info->magic == old_info->magic); + ASSERT(be32_to_cpu(tmp_info->back) == old_blk->blkno); + tmp_info->back = cpu_to_be32(new_blk->blkno); + xfs_trans_log_buf(args->trans, bp, 0, sizeof(*tmp_info)-1); + } + old_info->forw = cpu_to_be32(new_blk->blkno); + } + + xfs_trans_log_buf(args->trans, old_blk->bp, 0, sizeof(*tmp_info) - 1); + xfs_trans_log_buf(args->trans, new_blk->bp, 0, sizeof(*tmp_info) - 1); + return 0; +} + +/* + * Unlink a block from a doubly linked list of blocks. + */ +STATIC int /* error */ +xfs_da3_blk_unlink( + struct xfs_da_state *state, + struct xfs_da_state_blk *drop_blk, + struct xfs_da_state_blk *save_blk) +{ + struct xfs_da_blkinfo *drop_info; + struct xfs_da_blkinfo *save_info; + struct xfs_da_blkinfo *tmp_info; + struct xfs_da_args *args; + struct xfs_buf *bp; + xfs_failaddr_t fa; + int error; + + /* + * Set up environment. + */ + args = state->args; + ASSERT(args != NULL); + save_info = save_blk->bp->b_addr; + drop_info = drop_blk->bp->b_addr; + ASSERT(save_blk->magic == XFS_DA_NODE_MAGIC || + save_blk->magic == XFS_DIR2_LEAFN_MAGIC || + save_blk->magic == XFS_ATTR_LEAF_MAGIC); + ASSERT(save_blk->magic == drop_blk->magic); + ASSERT((be32_to_cpu(save_info->forw) == drop_blk->blkno) || + (be32_to_cpu(save_info->back) == drop_blk->blkno)); + ASSERT((be32_to_cpu(drop_info->forw) == save_blk->blkno) || + (be32_to_cpu(drop_info->back) == save_blk->blkno)); + + /* + * Unlink the leaf block from the doubly linked chain of leaves. + */ + if (be32_to_cpu(save_info->back) == drop_blk->blkno) { + trace_xfs_da_unlink_back(args); + save_info->back = drop_info->back; + if (drop_info->back) { + error = xfs_da3_node_read(args->trans, args->dp, + be32_to_cpu(drop_info->back), + &bp, args->whichfork); + if (error) + return error; + fa = xfs_da3_header_check(bp, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(bp, fa); + xfs_trans_brelse(args->trans, bp); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + ASSERT(bp != NULL); + tmp_info = bp->b_addr; + ASSERT(tmp_info->magic == save_info->magic); + ASSERT(be32_to_cpu(tmp_info->forw) == drop_blk->blkno); + tmp_info->forw = cpu_to_be32(save_blk->blkno); + xfs_trans_log_buf(args->trans, bp, 0, + sizeof(*tmp_info) - 1); + } + } else { + trace_xfs_da_unlink_forward(args); + save_info->forw = drop_info->forw; + if (drop_info->forw) { + error = xfs_da3_node_read(args->trans, args->dp, + be32_to_cpu(drop_info->forw), + &bp, args->whichfork); + if (error) + return error; + fa = xfs_da3_header_check(bp, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(bp, fa); + xfs_trans_brelse(args->trans, bp); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + ASSERT(bp != NULL); + tmp_info = bp->b_addr; + ASSERT(tmp_info->magic == save_info->magic); + ASSERT(be32_to_cpu(tmp_info->back) == drop_blk->blkno); + tmp_info->back = cpu_to_be32(save_blk->blkno); + xfs_trans_log_buf(args->trans, bp, 0, + sizeof(*tmp_info) - 1); + } + } + + xfs_trans_log_buf(args->trans, save_blk->bp, 0, sizeof(*save_info) - 1); + return 0; +} + +/* + * Move a path "forward" or "!forward" one block at the current level. + * + * This routine will adjust a "path" to point to the next block + * "forward" (higher hashvalues) or "!forward" (lower hashvals) in the + * Btree, including updating pointers to the intermediate nodes between + * the new bottom and the root. + */ +int /* error */ +xfs_da3_path_shift( + struct xfs_da_state *state, + struct xfs_da_state_path *path, + int forward, + int release, + int *result) +{ + struct xfs_da_state_blk *blk; + struct xfs_da_blkinfo *info; + struct xfs_da_args *args; + struct xfs_da_node_entry *btree; + struct xfs_da3_icnode_hdr nodehdr; + struct xfs_buf *bp; + xfs_failaddr_t fa; + xfs_dablk_t blkno = 0; + int level; + int error; + struct xfs_inode *dp = state->args->dp; + + trace_xfs_da_path_shift(state->args); + + /* + * Roll up the Btree looking for the first block where our + * current index is not at the edge of the block. Note that + * we skip the bottom layer because we want the sibling block. + */ + args = state->args; + ASSERT(args != NULL); + ASSERT(path != NULL); + ASSERT((path->active > 0) && (path->active < XFS_DA_NODE_MAXDEPTH)); + level = (path->active-1) - 1; /* skip bottom layer in path */ + for (; level >= 0; level--) { + blk = &path->blk[level]; + xfs_da3_node_hdr_from_disk(dp->i_mount, &nodehdr, + blk->bp->b_addr); + + if (forward && (blk->index < nodehdr.count - 1)) { + blk->index++; + blkno = be32_to_cpu(nodehdr.btree[blk->index].before); + break; + } else if (!forward && (blk->index > 0)) { + blk->index--; + blkno = be32_to_cpu(nodehdr.btree[blk->index].before); + break; + } + } + if (level < 0) { + *result = -ENOENT; /* we're out of our tree */ + ASSERT(args->op_flags & XFS_DA_OP_OKNOENT); + return 0; + } + + /* + * Roll down the edge of the subtree until we reach the + * same depth we were at originally. + */ + for (blk++, level++; level < path->active; blk++, level++) { + /* + * Read the next child block into a local buffer. + */ + error = xfs_da3_node_read(args->trans, dp, blkno, &bp, + args->whichfork); + if (error) + return error; + + /* + * Release the old block (if it's dirty, the trans doesn't + * actually let go) and swap the local buffer into the path + * structure. This ensures failure of the above read doesn't set + * a NULL buffer in an active slot in the path. + */ + if (release) + xfs_trans_brelse(args->trans, blk->bp); + blk->blkno = blkno; + blk->bp = bp; + + info = blk->bp->b_addr; + ASSERT(info->magic == cpu_to_be16(XFS_DA_NODE_MAGIC) || + info->magic == cpu_to_be16(XFS_DA3_NODE_MAGIC) || + info->magic == cpu_to_be16(XFS_DIR2_LEAFN_MAGIC) || + info->magic == cpu_to_be16(XFS_DIR3_LEAFN_MAGIC) || + info->magic == cpu_to_be16(XFS_ATTR_LEAF_MAGIC) || + info->magic == cpu_to_be16(XFS_ATTR3_LEAF_MAGIC)); + + + /* + * Note: we flatten the magic number to a single type so we + * don't have to compare against crc/non-crc types elsewhere. + */ + switch (be16_to_cpu(info->magic)) { + case XFS_DA_NODE_MAGIC: + case XFS_DA3_NODE_MAGIC: + fa = xfs_da3_node_header_check(blk->bp, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(blk->bp, fa); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + blk->magic = XFS_DA_NODE_MAGIC; + xfs_da3_node_hdr_from_disk(dp->i_mount, &nodehdr, + bp->b_addr); + btree = nodehdr.btree; + blk->hashval = be32_to_cpu(btree[nodehdr.count - 1].hashval); + if (forward) + blk->index = 0; + else + blk->index = nodehdr.count - 1; + blkno = be32_to_cpu(btree[blk->index].before); + break; + case XFS_ATTR_LEAF_MAGIC: + case XFS_ATTR3_LEAF_MAGIC: + fa = xfs_attr3_leaf_header_check(blk->bp, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(blk->bp, fa); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + blk->magic = XFS_ATTR_LEAF_MAGIC; + ASSERT(level == path->active-1); + blk->index = 0; + blk->hashval = xfs_attr_leaf_lasthash(blk->bp, NULL); + break; + case XFS_DIR2_LEAFN_MAGIC: + case XFS_DIR3_LEAFN_MAGIC: + fa = xfs_dir3_leaf_header_check(blk->bp, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(blk->bp, fa); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + blk->magic = XFS_DIR2_LEAFN_MAGIC; + ASSERT(level == path->active-1); + blk->index = 0; + blk->hashval = xfs_dir2_leaf_lasthash(args->dp, + blk->bp, NULL); + break; + default: + ASSERT(0); + break; + } + } + *result = 0; + return 0; +} + + +/*======================================================================== + * Utility routines. + *========================================================================*/ + +/* + * Implement a simple hash on a character string. + * Rotate the hash value by 7 bits, then XOR each character in. + * This is implemented with some source-level loop unrolling. + */ +xfs_dahash_t +xfs_da_hashname(const uint8_t *name, int namelen) +{ + xfs_dahash_t hash; + + /* + * Do four characters at a time as long as we can. + */ + for (hash = 0; namelen >= 4; namelen -= 4, name += 4) + hash = (name[0] << 21) ^ (name[1] << 14) ^ (name[2] << 7) ^ + (name[3] << 0) ^ rol32(hash, 7 * 4); + + /* + * Now do the rest of the characters. + */ + switch (namelen) { + case 3: + return (name[0] << 14) ^ (name[1] << 7) ^ (name[2] << 0) ^ + rol32(hash, 7 * 3); + case 2: + return (name[0] << 7) ^ (name[1] << 0) ^ rol32(hash, 7 * 2); + case 1: + return (name[0] << 0) ^ rol32(hash, 7 * 1); + default: /* case 0: */ + return hash; + } +} + +enum xfs_dacmp +xfs_da_compname( + struct xfs_da_args *args, + const unsigned char *name, + int len) +{ + return (args->namelen == len && memcmp(args->name, name, len) == 0) ? + XFS_CMP_EXACT : XFS_CMP_DIFFERENT; +} + +int +xfs_da_grow_inode_int( + struct xfs_da_args *args, + xfs_fileoff_t *bno, + int count) +{ + struct xfs_trans *tp = args->trans; + struct xfs_inode *dp = args->dp; + int w = args->whichfork; + xfs_rfsblock_t nblks = dp->i_nblocks; + struct xfs_bmbt_irec map, *mapp = ↦ + int nmap, error, got, i, mapi = 1; + + /* + * Find a spot in the file space to put the new block. + */ + error = xfs_bmap_first_unused(tp, dp, count, bno, w); + if (error) + return error; + + /* + * Try mapping it in one filesystem block. + */ + nmap = 1; + error = xfs_bmapi_write(tp, dp, *bno, count, + xfs_bmapi_aflag(w)|XFS_BMAPI_METADATA|XFS_BMAPI_CONTIG, + args->total, &map, &nmap); + if (error == -ENOSPC && count > 1) { + xfs_fileoff_t b; + int c; + + /* + * If we didn't get it and the block might work if fragmented, + * try without the CONTIG flag. Loop until we get it all. + */ + mapp = kmalloc_objs(*mapp, count, GFP_KERNEL | __GFP_NOFAIL); + for (b = *bno, mapi = 0; b < *bno + count; ) { + c = (int)(*bno + count - b); + nmap = min(XFS_BMAP_MAX_NMAP, c); + error = xfs_bmapi_write(tp, dp, b, c, + xfs_bmapi_aflag(w)|XFS_BMAPI_METADATA, + args->total, &mapp[mapi], &nmap); + if (error) + goto out_free_map; + mapi += nmap; + b = mapp[mapi - 1].br_startoff + + mapp[mapi - 1].br_blockcount; + } + } + if (error) + goto out_free_map; + + /* + * Count the blocks we got, make sure it matches the total. + */ + for (i = 0, got = 0; i < mapi; i++) + got += mapp[i].br_blockcount; + if (got != count || mapp[0].br_startoff != *bno || + mapp[mapi - 1].br_startoff + mapp[mapi - 1].br_blockcount != + *bno + count) { + error = -ENOSPC; + goto out_free_map; + } + + /* account for newly allocated blocks in reserved blocks total */ + args->total -= dp->i_nblocks - nblks; + +out_free_map: + if (mapp != &map) + kfree(mapp); + return error; +} + +/* + * Add a block to the btree ahead of the file. + * Return the new block number to the caller. + */ +int +xfs_da_grow_inode( + struct xfs_da_args *args, + xfs_dablk_t *new_blkno) +{ + xfs_fileoff_t bno; + int error; + + trace_xfs_da_grow_inode(args); + + bno = args->geo->leafblk; + error = xfs_da_grow_inode_int(args, &bno, args->geo->fsbcount); + if (!error) + *new_blkno = (xfs_dablk_t)bno; + return error; +} + +/* + * Ick. We need to always be able to remove a btree block, even + * if there's no space reservation because the filesystem is full. + * This is called if xfs_bunmapi on a btree block fails due to ENOSPC. + * It swaps the target block with the last block in the file. The + * last block in the file can always be removed since it can't cause + * a bmap btree split to do that. + */ +STATIC int +xfs_da3_swap_lastblock( + struct xfs_da_args *args, + xfs_dablk_t *dead_blknop, + struct xfs_buf **dead_bufp) +{ + struct xfs_da_blkinfo *dead_info; + struct xfs_da_blkinfo *sib_info; + struct xfs_da_intnode *par_node; + struct xfs_da_intnode *dead_node; + struct xfs_dir2_leaf *dead_leaf2; + struct xfs_da_node_entry *btree; + struct xfs_da3_icnode_hdr par_hdr; + struct xfs_inode *dp; + struct xfs_trans *tp; + struct xfs_mount *mp; + struct xfs_buf *dead_buf; + struct xfs_buf *last_buf; + struct xfs_buf *sib_buf; + struct xfs_buf *par_buf; + xfs_failaddr_t fa; + xfs_dahash_t dead_hash; + xfs_fileoff_t lastoff; + xfs_dablk_t dead_blkno; + xfs_dablk_t last_blkno; + xfs_dablk_t sib_blkno; + xfs_dablk_t par_blkno; + int error; + int w; + int entno; + int level; + int dead_level; + + trace_xfs_da_swap_lastblock(args); + + dead_buf = *dead_bufp; + dead_blkno = *dead_blknop; + tp = args->trans; + dp = args->dp; + w = args->whichfork; + ASSERT(w == XFS_DATA_FORK); + mp = dp->i_mount; + lastoff = args->geo->freeblk; + error = xfs_bmap_last_before(tp, dp, &lastoff, w); + if (error) + return error; + if (XFS_IS_CORRUPT(mp, lastoff == 0)) { + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + /* + * Read the last block in the btree space. + */ + last_blkno = (xfs_dablk_t)lastoff - args->geo->fsbcount; + error = xfs_da3_node_read(tp, dp, last_blkno, &last_buf, w); + if (error) + return error; + fa = xfs_da3_header_check(last_buf, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(last_buf, fa); + xfs_trans_brelse(tp, last_buf); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + + /* + * Copy the last block into the dead buffer and log it. + */ + xfs_da_buf_copy(dead_buf, last_buf, args->geo->blksize); + xfs_trans_log_buf(tp, dead_buf, 0, args->geo->blksize - 1); + dead_info = dead_buf->b_addr; + + /* + * Get values from the moved block. + */ + if (dead_info->magic == cpu_to_be16(XFS_DIR2_LEAFN_MAGIC) || + dead_info->magic == cpu_to_be16(XFS_DIR3_LEAFN_MAGIC)) { + struct xfs_dir3_icleaf_hdr leafhdr; + struct xfs_dir2_leaf_entry *ents; + + dead_leaf2 = (xfs_dir2_leaf_t *)dead_info; + xfs_dir2_leaf_hdr_from_disk(dp->i_mount, &leafhdr, + dead_leaf2); + ents = leafhdr.ents; + dead_level = 0; + dead_hash = be32_to_cpu(ents[leafhdr.count - 1].hashval); + } else { + struct xfs_da3_icnode_hdr deadhdr; + + dead_node = (xfs_da_intnode_t *)dead_info; + xfs_da3_node_hdr_from_disk(dp->i_mount, &deadhdr, dead_node); + btree = deadhdr.btree; + dead_level = deadhdr.level; + dead_hash = be32_to_cpu(btree[deadhdr.count - 1].hashval); + } + sib_buf = par_buf = NULL; + /* + * If the moved block has a left sibling, fix up the pointers. + */ + if ((sib_blkno = be32_to_cpu(dead_info->back))) { + error = xfs_da3_node_read(tp, dp, sib_blkno, &sib_buf, w); + if (error) + goto done; + fa = xfs_da3_header_check(sib_buf, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(sib_buf, fa); + xfs_da_mark_sick(args); + error = -EFSCORRUPTED; + goto done; + } + sib_info = sib_buf->b_addr; + if (XFS_IS_CORRUPT(mp, + be32_to_cpu(sib_info->forw) != last_blkno || + sib_info->magic != dead_info->magic)) { + xfs_da_mark_sick(args); + error = -EFSCORRUPTED; + goto done; + } + sib_info->forw = cpu_to_be32(dead_blkno); + xfs_trans_log_buf(tp, sib_buf, + XFS_DA_LOGRANGE(sib_info, &sib_info->forw, + sizeof(sib_info->forw))); + sib_buf = NULL; + } + /* + * If the moved block has a right sibling, fix up the pointers. + */ + if ((sib_blkno = be32_to_cpu(dead_info->forw))) { + error = xfs_da3_node_read(tp, dp, sib_blkno, &sib_buf, w); + if (error) + goto done; + fa = xfs_da3_header_check(sib_buf, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(sib_buf, fa); + xfs_da_mark_sick(args); + error = -EFSCORRUPTED; + goto done; + } + sib_info = sib_buf->b_addr; + if (XFS_IS_CORRUPT(mp, + be32_to_cpu(sib_info->back) != last_blkno || + sib_info->magic != dead_info->magic)) { + xfs_da_mark_sick(args); + error = -EFSCORRUPTED; + goto done; + } + sib_info->back = cpu_to_be32(dead_blkno); + xfs_trans_log_buf(tp, sib_buf, + XFS_DA_LOGRANGE(sib_info, &sib_info->back, + sizeof(sib_info->back))); + sib_buf = NULL; + } + par_blkno = args->geo->leafblk; + level = -1; + /* + * Walk down the tree looking for the parent of the moved block. + */ + for (;;) { + error = xfs_da3_node_read(tp, dp, par_blkno, &par_buf, w); + if (error) + goto done; + fa = xfs_da3_node_header_check(par_buf, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(par_buf, fa); + xfs_da_mark_sick(args); + error = -EFSCORRUPTED; + goto done; + } + par_node = par_buf->b_addr; + xfs_da3_node_hdr_from_disk(dp->i_mount, &par_hdr, par_node); + if (XFS_IS_CORRUPT(mp, + level >= 0 && level != par_hdr.level + 1)) { + xfs_da_mark_sick(args); + error = -EFSCORRUPTED; + goto done; + } + level = par_hdr.level; + btree = par_hdr.btree; + for (entno = 0; + entno < par_hdr.count && + be32_to_cpu(btree[entno].hashval) < dead_hash; + entno++) + continue; + if (XFS_IS_CORRUPT(mp, entno == par_hdr.count)) { + xfs_da_mark_sick(args); + error = -EFSCORRUPTED; + goto done; + } + par_blkno = be32_to_cpu(btree[entno].before); + if (level == dead_level + 1) + break; + xfs_trans_brelse(tp, par_buf); + par_buf = NULL; + } + /* + * We're in the right parent block. + * Look for the right entry. + */ + for (;;) { + for (; + entno < par_hdr.count && + be32_to_cpu(btree[entno].before) != last_blkno; + entno++) + continue; + if (entno < par_hdr.count) + break; + par_blkno = par_hdr.forw; + xfs_trans_brelse(tp, par_buf); + par_buf = NULL; + if (XFS_IS_CORRUPT(mp, par_blkno == 0)) { + xfs_da_mark_sick(args); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_da3_node_read(tp, dp, par_blkno, &par_buf, w); + if (error) + goto done; + fa = xfs_da3_node_header_check(par_buf, args->owner); + if (fa) { + __xfs_buf_mark_corrupt(par_buf, fa); + xfs_da_mark_sick(args); + error = -EFSCORRUPTED; + goto done; + } + par_node = par_buf->b_addr; + xfs_da3_node_hdr_from_disk(dp->i_mount, &par_hdr, par_node); + if (XFS_IS_CORRUPT(mp, par_hdr.level != level)) { + xfs_da_mark_sick(args); + error = -EFSCORRUPTED; + goto done; + } + btree = par_hdr.btree; + entno = 0; + } + /* + * Update the parent entry pointing to the moved block. + */ + btree[entno].before = cpu_to_be32(dead_blkno); + xfs_trans_log_buf(tp, par_buf, + XFS_DA_LOGRANGE(par_node, &btree[entno].before, + sizeof(btree[entno].before))); + *dead_blknop = last_blkno; + *dead_bufp = last_buf; + return 0; +done: + if (par_buf) + xfs_trans_brelse(tp, par_buf); + if (sib_buf) + xfs_trans_brelse(tp, sib_buf); + xfs_trans_brelse(tp, last_buf); + return error; +} + +/* + * Remove a btree block from a directory or attribute. + */ +int +xfs_da_shrink_inode( + struct xfs_da_args *args, + xfs_dablk_t dead_blkno, + struct xfs_buf *dead_buf) +{ + struct xfs_inode *dp; + int done, error, w, count; + struct xfs_trans *tp; + + trace_xfs_da_shrink_inode(args); + + dp = args->dp; + w = args->whichfork; + tp = args->trans; + count = args->geo->fsbcount; + for (;;) { + /* + * Remove extents. If we get ENOSPC for a dir we have to move + * the last block to the place we want to kill. + */ + error = xfs_bunmapi(tp, dp, dead_blkno, count, + xfs_bmapi_aflag(w), 0, &done); + if (error == -ENOSPC) { + if (w != XFS_DATA_FORK) + break; + error = xfs_da3_swap_lastblock(args, &dead_blkno, + &dead_buf); + if (error) + break; + } else { + break; + } + } + xfs_trans_binval(tp, dead_buf); + return error; +} + +static int +xfs_dabuf_map( + struct xfs_inode *dp, + xfs_dablk_t bno, + unsigned int flags, + int whichfork, + struct xfs_buf_map **mapp, + int *nmaps) +{ + struct xfs_mount *mp = dp->i_mount; + int nfsb = xfs_dabuf_nfsb(mp, whichfork); + struct xfs_bmbt_irec irec, *irecs = &irec; + struct xfs_buf_map *map = *mapp; + xfs_fileoff_t off = bno; + int error = 0, nirecs, i; + + if (nfsb > 1) + irecs = kzalloc(sizeof(irec) * nfsb, + GFP_KERNEL | __GFP_NOLOCKDEP | __GFP_NOFAIL); + + nirecs = nfsb; + error = xfs_bmapi_read(dp, bno, nfsb, irecs, &nirecs, + xfs_bmapi_aflag(whichfork)); + if (error) + goto out_free_irecs; + + /* + * Use the caller provided map for the single map case, else allocate a + * larger one that needs to be free by the caller. + */ + if (nirecs > 1) { + map = kcalloc(nirecs, sizeof(struct xfs_buf_map), + GFP_KERNEL | __GFP_NOLOCKDEP | __GFP_NOFAIL); + *mapp = map; + } + + for (i = 0; i < nirecs; i++) { + if (irecs[i].br_startblock == HOLESTARTBLOCK || + irecs[i].br_startblock == DELAYSTARTBLOCK) + goto invalid_mapping; + if (off != irecs[i].br_startoff) + goto invalid_mapping; + + map[i].bm_bn = XFS_FSB_TO_DADDR(mp, irecs[i].br_startblock); + map[i].bm_len = XFS_FSB_TO_BB(mp, irecs[i].br_blockcount); + off += irecs[i].br_blockcount; + } + + if (off != bno + nfsb) + goto invalid_mapping; + + *nmaps = nirecs; +out_free_irecs: + if (irecs != &irec) + kfree(irecs); + return error; + +invalid_mapping: + /* Caller ok with no mapping. */ + if (XFS_IS_CORRUPT(mp, !(flags & XFS_DABUF_MAP_HOLE_OK))) { + xfs_dirattr_mark_sick(dp, whichfork); + error = -EFSCORRUPTED; + if (xfs_error_level >= XFS_ERRLEVEL_LOW) { + xfs_alert(mp, "%s: bno %u inode %llu", + __func__, bno, I_INO(dp)); + + for (i = 0; i < nirecs; i++) { + xfs_alert(mp, +"[%02d] br_startoff %lld br_startblock %lld br_blockcount %lld br_state %d", + i, irecs[i].br_startoff, + irecs[i].br_startblock, + irecs[i].br_blockcount, + irecs[i].br_state); + } + } + } else { + *nmaps = 0; + } + goto out_free_irecs; +} + +/* + * Get a buffer for the dir/attr block. + */ +int +xfs_da_get_buf( + struct xfs_trans *tp, + struct xfs_inode *dp, + xfs_dablk_t bno, + struct xfs_buf **bpp, + int whichfork) +{ + struct xfs_mount *mp = dp->i_mount; + struct xfs_buf *bp; + struct xfs_buf_map map, *mapp = ↦ + int nmap = 1; + int error; + + *bpp = NULL; + error = xfs_dabuf_map(dp, bno, 0, whichfork, &mapp, &nmap); + if (error || nmap == 0) + goto out_free; + + error = xfs_trans_get_buf_map(tp, mp->m_ddev_targp, mapp, nmap, 0, &bp); + if (error) + goto out_free; + + *bpp = bp; + +out_free: + if (mapp != &map) + kfree(mapp); + + return error; +} + +/* + * Get a buffer for the dir/attr block, fill in the contents. + */ +int +xfs_da_read_buf( + struct xfs_trans *tp, + struct xfs_inode *dp, + xfs_dablk_t bno, + unsigned int flags, + struct xfs_buf **bpp, + int whichfork, + const struct xfs_buf_ops *ops) +{ + struct xfs_mount *mp = dp->i_mount; + struct xfs_buf *bp; + struct xfs_buf_map map, *mapp = ↦ + int nmap = 1; + int error; + + *bpp = NULL; + error = xfs_dabuf_map(dp, bno, flags, whichfork, &mapp, &nmap); + if (error || !nmap) + goto out_free; + + error = xfs_trans_read_buf_map(mp, tp, mp->m_ddev_targp, mapp, nmap, 0, + &bp, ops); + if (xfs_metadata_is_sick(error)) + xfs_dirattr_mark_sick(dp, whichfork); + /* + * ENODATA from disk implies a disk medium failure; ENODATA for + * xattrs means attribute not found, so disambiguate that here. + */ + if (error == -ENODATA && whichfork == XFS_ATTR_FORK) + error = -EIO; + if (error) + goto out_free; + + if (whichfork == XFS_ATTR_FORK) + xfs_buf_set_ref(bp, XFS_ATTR_BTREE_REF); + else + xfs_buf_set_ref(bp, XFS_DIR_BTREE_REF); + *bpp = bp; +out_free: + if (mapp != &map) + kfree(mapp); + + return error; +} + +/* + * Readahead the dir/attr block. + */ +int +xfs_da_reada_buf( + struct xfs_inode *dp, + xfs_dablk_t bno, + unsigned int flags, + int whichfork, + const struct xfs_buf_ops *ops) +{ + struct xfs_buf_map map; + struct xfs_buf_map *mapp; + int nmap; + int error; + + mapp = ↦ + nmap = 1; + error = xfs_dabuf_map(dp, bno, flags, whichfork, &mapp, &nmap); + if (error || !nmap) + goto out_free; + + xfs_buf_readahead_map(dp->i_mount->m_ddev_targp, mapp, nmap, ops); + +out_free: + if (mapp != &map) + kfree(mapp); + + return error; +} diff --git a/fs/xfs/libxfs/xfs_da_btree.h b/fs/xfs/libxfs/xfs_da_btree.h new file mode 100644 index 000000000..afcf2d3c7 --- /dev/null +++ b/fs/xfs/libxfs/xfs_da_btree.h @@ -0,0 +1,247 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2000,2002,2005 Silicon Graphics, Inc. + * Copyright (c) 2013 Red Hat, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_DA_BTREE_H__ +#define __XFS_DA_BTREE_H__ + +struct xfs_inode; +struct xfs_trans; + +/* + * Directory/attribute geometry information. There will be one of these for each + * data fork type, and it will be passed around via the xfs_da_args. Global + * structures will be attached to the xfs_mount. + */ +struct xfs_da_geometry { + unsigned int blksize; /* da block size in bytes */ + unsigned int fsbcount; /* da block size in filesystem blocks */ + uint8_t fsblog; /* log2 of _filesystem_ block size */ + uint8_t blklog; /* log2 of da block size */ + unsigned int node_hdr_size; /* danode header size in bytes */ + unsigned int node_ents; /* # of entries in a danode */ + unsigned int magicpct; /* 37% of block size in bytes */ + xfs_dablk_t datablk; /* blockno of dir data v2 */ + unsigned int leaf_hdr_size; /* dir2 leaf header size */ + unsigned int leaf_max_ents; /* # of entries in dir2 leaf */ + xfs_dablk_t leafblk; /* blockno of leaf data v2 */ + unsigned int free_hdr_size; /* dir2 free header size */ + unsigned int free_max_bests; /* # of bests entries in dir2 free */ + xfs_dablk_t freeblk; /* blockno of free data v2 */ + xfs_extnum_t max_extents; /* Max. extents in corresponding fork */ + + xfs_dir2_data_aoff_t data_first_offset; + size_t data_entry_offset; +}; + +/*======================================================================== + * Btree searching and modification structure definitions. + *========================================================================*/ + +/* + * Search comparison results + */ +enum xfs_dacmp { + XFS_CMP_DIFFERENT, /* names are completely different */ + XFS_CMP_EXACT, /* names are exactly the same */ + XFS_CMP_CASE /* names are same but differ in case */ +}; + +/* + * Structure to ease passing around component names. + */ +typedef struct xfs_da_args { + struct xfs_da_geometry *geo; /* da block geometry */ + const uint8_t *name; /* string (maybe not NULL terminated) */ + const uint8_t *new_name; /* new attr name */ + void *value; /* set of bytes (maybe contain NULLs) */ + void *new_value; /* new xattr value (may contain NULLs) */ + struct xfs_inode *dp; /* directory inode to manipulate */ + struct xfs_trans *trans; /* current trans (changes over time) */ + + xfs_ino_t inumber; /* input/output inode number */ + xfs_ino_t owner; /* inode that owns the dir/attr data */ + + int valuelen; /* length of value */ + int new_valuelen; /* length of new_value */ + uint8_t filetype; /* filetype of inode for directories */ + uint8_t op_flags; /* operation flags */ + uint8_t attr_filter; /* XFS_ATTR_{ROOT,SECURE,INCOMPLETE} */ + short namelen; /* length of string (maybe no NULL) */ + short new_namelen; /* length of new attr name */ + xfs_dahash_t hashval; /* hash value of name */ + xfs_extlen_t total; /* total blocks needed, for 1st bmap */ + int whichfork; /* data or attribute fork */ + xfs_dablk_t blkno; /* blkno of attr leaf of interest */ + int index; /* index of attr of interest in blk */ + xfs_dablk_t rmtblkno; /* remote attr value starting blkno */ + int rmtblkcnt; /* remote attr value block count */ + int rmtvaluelen; /* remote attr value length in bytes */ + xfs_dablk_t blkno2; /* blkno of 2nd attr leaf of interest */ + int index2; /* index of 2nd attr in blk */ + xfs_dablk_t rmtblkno2; /* remote attr value starting blkno */ + int rmtblkcnt2; /* remote attr value block count */ + int rmtvaluelen2; /* remote attr value length in bytes */ + enum xfs_dacmp cmpresult; /* name compare result for lookups */ +} xfs_da_args_t; + +/* + * Operation flags: + */ +#define XFS_DA_OP_JUSTCHECK (1u << 0) /* check for ok with no space */ +#define XFS_DA_OP_REPLACE (1u << 1) /* this is an atomic replace op */ +#define XFS_DA_OP_ADDNAME (1u << 2) /* this is an add operation */ +#define XFS_DA_OP_OKNOENT (1u << 3) /* lookup op, ENOENT ok, else die */ +#define XFS_DA_OP_CILOOKUP (1u << 4) /* lookup returns CI name if found */ +#define XFS_DA_OP_RECOVERY (1u << 5) /* Log recovery operation */ +#define XFS_DA_OP_LOGGED (1u << 6) /* Use intent items to track op */ + +#define XFS_DA_OP_FLAGS \ + { XFS_DA_OP_JUSTCHECK, "JUSTCHECK" }, \ + { XFS_DA_OP_REPLACE, "REPLACE" }, \ + { XFS_DA_OP_ADDNAME, "ADDNAME" }, \ + { XFS_DA_OP_OKNOENT, "OKNOENT" }, \ + { XFS_DA_OP_CILOOKUP, "CILOOKUP" }, \ + { XFS_DA_OP_RECOVERY, "RECOVERY" }, \ + { XFS_DA_OP_LOGGED, "LOGGED" } + +/* + * Storage for holding state during Btree searches and split/join ops. + * + * Only need space for 5 intermediate nodes. With a minimum of 62-way + * fanout to the Btree, we can support over 900 million directory blocks, + * which is slightly more than enough. + */ +typedef struct xfs_da_state_blk { + struct xfs_buf *bp; /* buffer containing block */ + xfs_dablk_t blkno; /* filesystem blkno of buffer */ + xfs_daddr_t disk_blkno; /* on-disk blkno (in BBs) of buffer */ + int index; /* relevant index into block */ + xfs_dahash_t hashval; /* last hash value in block */ + int magic; /* blk's magic number, ie: blk type */ +} xfs_da_state_blk_t; + +typedef struct xfs_da_state_path { + int active; /* number of active levels */ + xfs_da_state_blk_t blk[XFS_DA_NODE_MAXDEPTH]; +} xfs_da_state_path_t; + +typedef struct xfs_da_state { + xfs_da_args_t *args; /* filename arguments */ + struct xfs_mount *mp; /* filesystem mount point */ + xfs_da_state_path_t path; /* search/split paths */ + xfs_da_state_path_t altpath; /* alternate path for join */ + unsigned char inleaf; /* insert into 1->lf, 0->splf */ + unsigned char extravalid; /* T/F: extrablk is in use */ + unsigned char extraafter; /* T/F: extrablk is after new */ + xfs_da_state_blk_t extrablk; /* for double-splits on leaves */ + /* for dirv2 extrablk is data */ +} xfs_da_state_t; + +/* + * In-core version of the node header to abstract the differences in the v2 and + * v3 disk format of the headers. Callers need to convert to/from disk format as + * appropriate. + */ +struct xfs_da3_icnode_hdr { + uint32_t forw; + uint32_t back; + uint16_t magic; + uint16_t count; + uint16_t level; + + /* + * Pointer to the on-disk format entries, which are behind the + * variable size (v4 vs v5) header in the on-disk block. + */ + struct xfs_da_node_entry *btree; +}; + +/* + * Utility macros to aid in logging changed structure fields. + */ +#define XFS_DA_LOGOFF(BASE, ADDR) ((char *)(ADDR) - (char *)(BASE)) +#define XFS_DA_LOGRANGE(BASE, ADDR, SIZE) \ + (uint)(XFS_DA_LOGOFF(BASE, ADDR)), \ + (uint)(XFS_DA_LOGOFF(BASE, ADDR)+(SIZE)-1) + +/*======================================================================== + * Function prototypes. + *========================================================================*/ + +/* + * Routines used for growing the Btree. + */ +int xfs_da3_node_create(struct xfs_da_args *args, xfs_dablk_t blkno, + int level, struct xfs_buf **bpp, int whichfork); +int xfs_da3_split(xfs_da_state_t *state); + +/* + * Routines used for shrinking the Btree. + */ +int xfs_da3_join(xfs_da_state_t *state); +void xfs_da3_fixhashpath(struct xfs_da_state *state, + struct xfs_da_state_path *path_to_to_fix); +void xfs_attr3_node_entry_remove(struct xfs_trans *tp, struct xfs_inode *dp, + struct xfs_buf *bp, int index); + +/* + * Routines used for finding things in the Btree. + */ +int xfs_da3_node_lookup_int(xfs_da_state_t *state, int *result); +int xfs_da3_path_shift(xfs_da_state_t *state, xfs_da_state_path_t *path, + int forward, int release, int *result); +/* + * Utility routines. + */ +int xfs_da3_blk_link(xfs_da_state_t *state, xfs_da_state_blk_t *old_blk, + xfs_da_state_blk_t *new_blk); +int xfs_da3_node_read(struct xfs_trans *tp, struct xfs_inode *dp, + xfs_dablk_t bno, struct xfs_buf **bpp, int whichfork); +int xfs_da3_node_read_mapped(struct xfs_trans *tp, struct xfs_inode *dp, + xfs_daddr_t mappedbno, struct xfs_buf **bpp, + int whichfork); + +/* + * Utility routines. + */ + +#define XFS_DABUF_MAP_HOLE_OK (1u << 0) + +int xfs_da_grow_inode(xfs_da_args_t *args, xfs_dablk_t *new_blkno); +int xfs_da_grow_inode_int(struct xfs_da_args *args, xfs_fileoff_t *bno, + int count); +int xfs_da_get_buf(struct xfs_trans *trans, struct xfs_inode *dp, + xfs_dablk_t bno, struct xfs_buf **bp, int whichfork); +int xfs_da_read_buf(struct xfs_trans *trans, struct xfs_inode *dp, + xfs_dablk_t bno, unsigned int flags, struct xfs_buf **bpp, + int whichfork, const struct xfs_buf_ops *ops); +int xfs_da_reada_buf(struct xfs_inode *dp, xfs_dablk_t bno, + unsigned int flags, int whichfork, + const struct xfs_buf_ops *ops); +int xfs_da_shrink_inode(xfs_da_args_t *args, xfs_dablk_t dead_blkno, + struct xfs_buf *dead_buf); +void xfs_da_buf_copy(struct xfs_buf *dst, struct xfs_buf *src, + size_t size); + +uint xfs_da_hashname(const uint8_t *name_string, int name_length); +enum xfs_dacmp xfs_da_compname(struct xfs_da_args *args, + const unsigned char *name, int len); + + +struct xfs_da_state *xfs_da_state_alloc(struct xfs_da_args *args); +void xfs_da_state_free(xfs_da_state_t *state); +void xfs_da_state_reset(struct xfs_da_state *state, struct xfs_da_args *args); + +void xfs_da3_node_hdr_from_disk(struct xfs_mount *mp, + struct xfs_da3_icnode_hdr *to, struct xfs_da_intnode *from); +void xfs_da3_node_hdr_to_disk(struct xfs_mount *mp, + struct xfs_da_intnode *to, struct xfs_da3_icnode_hdr *from); +xfs_failaddr_t xfs_da3_header_check(struct xfs_buf *bp, xfs_ino_t owner); +xfs_failaddr_t xfs_da3_node_header_check(struct xfs_buf *bp, xfs_ino_t owner); + +extern struct kmem_cache *xfs_da_state_cache; + +#endif /* __XFS_DA_BTREE_H__ */ diff --git a/fs/xfs/libxfs/xfs_da_format.h b/fs/xfs/libxfs/xfs_da_format.h new file mode 100644 index 000000000..7d55307e6 --- /dev/null +++ b/fs/xfs/libxfs/xfs_da_format.h @@ -0,0 +1,907 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2000-2001,2005 Silicon Graphics, Inc. + * Copyright (c) 2013 Red Hat, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_DA_FORMAT_H__ +#define __XFS_DA_FORMAT_H__ + +/* + * This structure is common to both leaf nodes and non-leaf nodes in the Btree. + * + * It is used to manage a doubly linked list of all blocks at the same + * level in the Btree, and to identify which type of block this is. + */ +#define XFS_DA_NODE_MAGIC 0xfebe /* magic number: non-leaf blocks */ +#define XFS_ATTR_LEAF_MAGIC 0xfbee /* magic number: attribute leaf blks */ +#define XFS_DIR2_LEAF1_MAGIC 0xd2f1 /* magic number: v2 dirlf single blks */ +#define XFS_DIR2_LEAFN_MAGIC 0xd2ff /* magic number: v2 dirlf multi blks */ + +typedef struct xfs_da_blkinfo { + __be32 forw; /* previous block in list */ + __be32 back; /* following block in list */ + __be16 magic; /* validity check on block */ + __be16 pad; /* unused */ +} xfs_da_blkinfo_t; + +/* + * CRC enabled directory structure types + * + * The headers change size for the additional verification information, but + * otherwise the tree layouts and contents are unchanged. Hence the da btree + * code can use the struct xfs_da_blkinfo for manipulating the tree links and + * magic numbers without modification for both v2 and v3 nodes. + */ +#define XFS_DA3_NODE_MAGIC 0x3ebe /* magic number: non-leaf blocks */ +#define XFS_ATTR3_LEAF_MAGIC 0x3bee /* magic number: attribute leaf blks */ +#define XFS_DIR3_LEAF1_MAGIC 0x3df1 /* magic number: v3 dirlf single blks */ +#define XFS_DIR3_LEAFN_MAGIC 0x3dff /* magic number: v3 dirlf multi blks */ + +struct xfs_da3_blkinfo { + /* + * the node link manipulation code relies on the fact that the first + * element of this structure is the struct xfs_da_blkinfo so it can + * ignore the differences in the rest of the structures. + */ + struct xfs_da_blkinfo hdr; + __be32 crc; /* CRC of block */ + __be64 blkno; /* first block of the buffer */ + __be64 lsn; /* sequence number of last write */ + uuid_t uuid; /* filesystem we belong to */ + __be64 owner; /* inode that owns the block */ +}; + +/* + * This is the structure of the root and intermediate nodes in the Btree. + * The leaf nodes are defined above. + * + * Entries are not packed. + * + * Since we have duplicate keys, use a binary search but always follow + * all match in the block, not just the first match found. + */ +#define XFS_DA_NODE_MAXDEPTH 5 /* max depth of Btree */ + +typedef struct xfs_da_node_hdr { + struct xfs_da_blkinfo info; /* block type, links, etc. */ + __be16 __count; /* count of active entries */ + __be16 __level; /* level above leaves (leaf == 0) */ +} xfs_da_node_hdr_t; + +struct xfs_da3_node_hdr { + struct xfs_da3_blkinfo info; /* block type, links, etc. */ + __be16 __count; /* count of active entries */ + __be16 __level; /* level above leaves (leaf == 0) */ + __be32 __pad32; +}; + +#define XFS_DA3_NODE_CRC_OFF (offsetof(struct xfs_da3_node_hdr, info.crc)) + +typedef struct xfs_da_node_entry { + __be32 hashval; /* hash value for this descendant */ + __be32 before; /* Btree block before this key */ +} xfs_da_node_entry_t; + +typedef struct xfs_da_intnode { + struct xfs_da_node_hdr hdr; + struct xfs_da_node_entry __btree[]; +} xfs_da_intnode_t; + +struct xfs_da3_intnode { + struct xfs_da3_node_hdr hdr; + struct xfs_da_node_entry __btree[]; +}; + +/* + * Directory version 2. + * + * There are 4 possible formats: + * - shortform - embedded into the inode + * - single block - data with embedded leaf at the end + * - multiple data blocks, single leaf+freeindex block + * - data blocks, node and leaf blocks (btree), freeindex blocks + * + * Note: many node blocks structures and constants are shared with the attr + * code and defined in xfs_da_btree.h. + */ + +#define XFS_DIR2_BLOCK_MAGIC 0x58443242 /* XD2B: single block dirs */ +#define XFS_DIR2_DATA_MAGIC 0x58443244 /* XD2D: multiblock dirs */ +#define XFS_DIR2_FREE_MAGIC 0x58443246 /* XD2F: free index blocks */ + +/* + * Directory Version 3 With CRCs. + * + * The tree formats are the same as for version 2 directories. The difference + * is in the block header and dirent formats. In many cases the v3 structures + * use v2 definitions as they are no different and this makes code sharing much + * easier. + * + * Also, the xfs_dir3_*() functions handle both v2 and v3 formats - if the + * format is v2 then they switch to the existing v2 code, or the format is v3 + * they implement the v3 functionality. This means the existing dir2 is a mix of + * xfs_dir2/xfs_dir3 calls and functions. The xfs_dir3 functions are called + * where there is a difference in the formats, otherwise the code is unchanged. + * + * Where it is possible, the code decides what to do based on the magic numbers + * in the blocks rather than feature bits in the superblock. This means the code + * is as independent of the external XFS code as possible as doesn't require + * passing struct xfs_mount pointers into places where it isn't really + * necessary. + * + * Version 3 includes: + * + * - a larger block header for CRC and identification purposes and so the + * offsets of all the structures inside the blocks are different. + * + * - new magic numbers to be able to detect the v2/v3 types on the fly. + */ + +#define XFS_DIR3_BLOCK_MAGIC 0x58444233 /* XDB3: single block dirs */ +#define XFS_DIR3_DATA_MAGIC 0x58444433 /* XDD3: multiblock dirs */ +#define XFS_DIR3_FREE_MAGIC 0x58444633 /* XDF3: free index blocks */ + +/* + * Dirents in version 3 directories have a file type field. Additions to this + * list are an on-disk format change, requiring feature bits. Valid values + * are as follows: + */ +#define XFS_DIR3_FT_UNKNOWN 0 +#define XFS_DIR3_FT_REG_FILE 1 +#define XFS_DIR3_FT_DIR 2 +#define XFS_DIR3_FT_CHRDEV 3 +#define XFS_DIR3_FT_BLKDEV 4 +#define XFS_DIR3_FT_FIFO 5 +#define XFS_DIR3_FT_SOCK 6 +#define XFS_DIR3_FT_SYMLINK 7 +#define XFS_DIR3_FT_WHT 8 + +#define XFS_DIR3_FT_MAX 9 + +#define XFS_DIR3_FTYPE_STR \ + { XFS_DIR3_FT_UNKNOWN, "unknown" }, \ + { XFS_DIR3_FT_REG_FILE, "file" }, \ + { XFS_DIR3_FT_DIR, "directory" }, \ + { XFS_DIR3_FT_CHRDEV, "char" }, \ + { XFS_DIR3_FT_BLKDEV, "block" }, \ + { XFS_DIR3_FT_FIFO, "fifo" }, \ + { XFS_DIR3_FT_SOCK, "sock" }, \ + { XFS_DIR3_FT_SYMLINK, "symlink" }, \ + { XFS_DIR3_FT_WHT, "whiteout" } + +/* + * Byte offset in data block and shortform entry. + */ +typedef uint16_t xfs_dir2_data_off_t; +#define NULLDATAOFF 0xffffU +typedef uint xfs_dir2_data_aoff_t; /* argument form */ + +/* + * Offset in data space of a data entry. + */ +typedef uint32_t xfs_dir2_dataptr_t; +#define XFS_DIR2_MAX_DATAPTR ((xfs_dir2_dataptr_t)0xffffffff) +#define XFS_DIR2_NULL_DATAPTR ((xfs_dir2_dataptr_t)0) + +/* + * Byte offset in a directory. + */ +typedef xfs_off_t xfs_dir2_off_t; + +/* + * Directory block number (logical dirblk in file) + */ +typedef uint32_t xfs_dir2_db_t; + +#define XFS_INO32_SIZE 4 +#define XFS_INO64_SIZE 8 +#define XFS_INO64_DIFF (XFS_INO64_SIZE - XFS_INO32_SIZE) + +#define XFS_DIR2_MAX_SHORT_INUM ((xfs_ino_t)0xffffffffULL) + +/* + * Directory layout when stored internal to an inode. + * + * Small directories are packed as tightly as possible so as to fit into the + * literal area of the inode. These "shortform" directories consist of a + * single xfs_dir2_sf_hdr header followed by zero or more xfs_dir2_sf_entry + * structures. Due the different inode number storage size and the variable + * length name field in the xfs_dir2_sf_entry all these structure are + * variable length, and the accessors in this file should be used to iterate + * over them. + */ +typedef struct xfs_dir2_sf_hdr { + uint8_t count; /* count of entries */ + uint8_t i8count; /* count of 8-byte inode #s */ + uint8_t parent[8]; /* parent dir inode number */ +} __packed xfs_dir2_sf_hdr_t; + +typedef struct xfs_dir2_sf_entry { + __u8 namelen; /* actual name length */ + __u8 offset[2]; /* saved offset */ + __u8 name[]; /* name, variable size */ + /* + * A single byte containing the file type field follows the inode + * number for version 3 directory entries. + * + * A 64-bit or 32-bit inode number follows here, at a variable offset + * after the name. + */ +} __packed xfs_dir2_sf_entry_t; + +static inline int xfs_dir2_sf_hdr_size(int i8count) +{ + return sizeof(struct xfs_dir2_sf_hdr) - + (i8count == 0) * XFS_INO64_DIFF; +} + +static inline xfs_dir2_data_aoff_t +xfs_dir2_sf_get_offset(xfs_dir2_sf_entry_t *sfep) +{ + return get_unaligned_be16(sfep->offset); +} + +static inline void +xfs_dir2_sf_put_offset(xfs_dir2_sf_entry_t *sfep, xfs_dir2_data_aoff_t off) +{ + put_unaligned_be16(off, sfep->offset); +} + +static inline struct xfs_dir2_sf_entry * +xfs_dir2_sf_firstentry(struct xfs_dir2_sf_hdr *hdr) +{ + return (struct xfs_dir2_sf_entry *) + ((char *)hdr + xfs_dir2_sf_hdr_size(hdr->i8count)); +} + +/* + * Data block structures. + * + * A pure data block looks like the following drawing on disk: + * + * +-------------------------------------------------+ + * | xfs_dir2_data_hdr_t | + * +-------------------------------------------------+ + * | xfs_dir2_data_entry_t OR xfs_dir2_data_unused_t | + * | xfs_dir2_data_entry_t OR xfs_dir2_data_unused_t | + * | xfs_dir2_data_entry_t OR xfs_dir2_data_unused_t | + * | ... | + * +-------------------------------------------------+ + * | unused space | + * +-------------------------------------------------+ + * + * As all the entries are variable size structures the accessors below should + * be used to iterate over them. + * + * In addition to the pure data blocks for the data and node formats, + * most structures are also used for the combined data/freespace "block" + * format below. + */ + +#define XFS_DIR2_DATA_ALIGN_LOG 3 /* i.e., 8 bytes */ +#define XFS_DIR2_DATA_ALIGN (1 << XFS_DIR2_DATA_ALIGN_LOG) +#define XFS_DIR2_DATA_FREE_TAG 0xffff +#define XFS_DIR2_DATA_FD_COUNT 3 + +/* + * Directory address space divided into sections, + * spaces separated by 32GB. + */ +#define XFS_DIR2_MAX_SPACES 3 +#define XFS_DIR2_SPACE_SIZE (1ULL << (32 + XFS_DIR2_DATA_ALIGN_LOG)) +#define XFS_DIR2_DATA_SPACE 0 +#define XFS_DIR2_DATA_OFFSET (XFS_DIR2_DATA_SPACE * XFS_DIR2_SPACE_SIZE) + +/* + * Describe a free area in the data block. + * + * The freespace will be formatted as a xfs_dir2_data_unused_t. + */ +typedef struct xfs_dir2_data_free { + __be16 offset; /* start of freespace */ + __be16 length; /* length of freespace */ +} xfs_dir2_data_free_t; + +/* + * Header for the data blocks. + * + * The code knows that XFS_DIR2_DATA_FD_COUNT is 3. + */ +typedef struct xfs_dir2_data_hdr { + __be32 magic; /* XFS_DIR2_DATA_MAGIC or */ + /* XFS_DIR2_BLOCK_MAGIC */ + xfs_dir2_data_free_t bestfree[XFS_DIR2_DATA_FD_COUNT]; +} xfs_dir2_data_hdr_t; + +/* + * define a structure for all the verification fields we are adding to the + * directory block structures. This will be used in several structures. + * The magic number must be the first entry to align with all the dir2 + * structures so we determine how to decode them just by the magic number. + */ +struct xfs_dir3_blk_hdr { + __be32 magic; /* magic number */ + __be32 crc; /* CRC of block */ + __be64 blkno; /* first block of the buffer */ + __be64 lsn; /* sequence number of last write */ + uuid_t uuid; /* filesystem we belong to */ + __be64 owner; /* inode that owns the block */ +}; + +struct xfs_dir3_data_hdr { + struct xfs_dir3_blk_hdr hdr; + xfs_dir2_data_free_t best_free[XFS_DIR2_DATA_FD_COUNT]; + __be32 pad; /* 64 bit alignment */ +}; + +#define XFS_DIR3_DATA_CRC_OFF offsetof(struct xfs_dir3_data_hdr, hdr.crc) + +/* + * Active entry in a data block. + * + * Aligned to 8 bytes. After the variable length name field there is a + * 2 byte tag field, which can be accessed using xfs_dir3_data_entry_tag_p. + * + * For dir3 structures, there is file type field between the name and the tag. + * This can only be manipulated by helper functions. It is packed hard against + * the end of the name so any padding for rounding is between the file type and + * the tag. + */ +typedef struct xfs_dir2_data_entry { + __be64 inumber; /* inode number */ + __u8 namelen; /* name length */ + __u8 name[]; /* name bytes, no null */ + /* __u8 filetype; */ /* type of inode we point to */ + /* __be16 tag; */ /* starting offset of us */ +} xfs_dir2_data_entry_t; + +/* + * Unused entry in a data block. + * + * Aligned to 8 bytes. Tag appears as the last 2 bytes and must be accessed + * using xfs_dir2_data_unused_tag_p. + */ +typedef struct xfs_dir2_data_unused { + __be16 freetag; /* XFS_DIR2_DATA_FREE_TAG */ + __be16 length; /* total free length */ + /* variable offset */ + __be16 tag; /* starting offset of us */ +} xfs_dir2_data_unused_t; + +/* + * Pointer to a freespace's tag word. + */ +static inline __be16 * +xfs_dir2_data_unused_tag_p(struct xfs_dir2_data_unused *dup) +{ + return (__be16 *)((char *)dup + + be16_to_cpu(dup->length) - sizeof(__be16)); +} + +/* + * Leaf block structures. + * + * A pure leaf block looks like the following drawing on disk: + * + * +---------------------------+ + * | xfs_dir2_leaf_hdr_t | + * +---------------------------+ + * | xfs_dir2_leaf_entry_t | + * | xfs_dir2_leaf_entry_t | + * | xfs_dir2_leaf_entry_t | + * | xfs_dir2_leaf_entry_t | + * | ... | + * +---------------------------+ + * | xfs_dir2_data_off_t | + * | xfs_dir2_data_off_t | + * | xfs_dir2_data_off_t | + * | ... | + * +---------------------------+ + * | xfs_dir2_leaf_tail_t | + * +---------------------------+ + * + * The xfs_dir2_data_off_t members (bests) and tail are at the end of the block + * for single-leaf (magic = XFS_DIR2_LEAF1_MAGIC) blocks only, but not present + * for directories with separate leaf nodes and free space blocks + * (magic = XFS_DIR2_LEAFN_MAGIC). + * + * As all the entries are variable size structures the accessors below should + * be used to iterate over them. + */ + +/* + * Offset of the leaf/node space. First block in this space + * is the btree root. + */ +#define XFS_DIR2_LEAF_SPACE 1 +#define XFS_DIR2_LEAF_OFFSET (XFS_DIR2_LEAF_SPACE * XFS_DIR2_SPACE_SIZE) + +/* + * Leaf block header. + */ +typedef struct xfs_dir2_leaf_hdr { + xfs_da_blkinfo_t info; /* header for da routines */ + __be16 count; /* count of entries */ + __be16 stale; /* count of stale entries */ +} xfs_dir2_leaf_hdr_t; + +struct xfs_dir3_leaf_hdr { + struct xfs_da3_blkinfo info; /* header for da routines */ + __be16 count; /* count of entries */ + __be16 stale; /* count of stale entries */ + __be32 pad; /* 64 bit alignment */ +}; + +/* + * Leaf block entry. + */ +typedef struct xfs_dir2_leaf_entry { + __be32 hashval; /* hash value of name */ + __be32 address; /* address of data entry */ +} xfs_dir2_leaf_entry_t; + +/* + * Leaf block tail. + */ +typedef struct xfs_dir2_leaf_tail { + __be32 bestcount; +} xfs_dir2_leaf_tail_t; + +/* + * Leaf block. + */ +typedef struct xfs_dir2_leaf { + xfs_dir2_leaf_hdr_t hdr; /* leaf header */ + xfs_dir2_leaf_entry_t __ents[]; /* entries */ +} xfs_dir2_leaf_t; + +struct xfs_dir3_leaf { + struct xfs_dir3_leaf_hdr hdr; /* leaf header */ + struct xfs_dir2_leaf_entry __ents[]; /* entries */ +}; + +#define XFS_DIR3_LEAF_CRC_OFF offsetof(struct xfs_dir3_leaf_hdr, info.crc) + +/* + * Get address of the bests array in the single-leaf block. + */ +static inline __be16 * +xfs_dir2_leaf_bests_p(struct xfs_dir2_leaf_tail *ltp) +{ + return (__be16 *)ltp - be32_to_cpu(ltp->bestcount); +} + +/* + * Free space block definitions for the node format. + */ + +/* + * Offset of the freespace index. + */ +#define XFS_DIR2_FREE_SPACE 2 +#define XFS_DIR2_FREE_OFFSET (XFS_DIR2_FREE_SPACE * XFS_DIR2_SPACE_SIZE) + +typedef struct xfs_dir2_free_hdr { + __be32 magic; /* XFS_DIR2_FREE_MAGIC */ + __be32 firstdb; /* db of first entry */ + __be32 nvalid; /* count of valid entries */ + __be32 nused; /* count of used entries */ +} xfs_dir2_free_hdr_t; + +typedef struct xfs_dir2_free { + xfs_dir2_free_hdr_t hdr; /* block header */ + __be16 bests[]; /* best free counts */ + /* unused entries are -1 */ +} xfs_dir2_free_t; + +struct xfs_dir3_free_hdr { + struct xfs_dir3_blk_hdr hdr; + __be32 firstdb; /* db of first entry */ + __be32 nvalid; /* count of valid entries */ + __be32 nused; /* count of used entries */ + __be32 pad; /* 64 bit alignment */ +}; + +struct xfs_dir3_free { + struct xfs_dir3_free_hdr hdr; + __be16 bests[]; /* best free counts */ + /* unused entries are -1 */ +}; + +#define XFS_DIR3_FREE_CRC_OFF offsetof(struct xfs_dir3_free, hdr.hdr.crc) + +/* + * Single block format. + * + * The single block format looks like the following drawing on disk: + * + * +-------------------------------------------------+ + * | xfs_dir2_data_hdr_t | + * +-------------------------------------------------+ + * | xfs_dir2_data_entry_t OR xfs_dir2_data_unused_t | + * | xfs_dir2_data_entry_t OR xfs_dir2_data_unused_t | + * | xfs_dir2_data_entry_t OR xfs_dir2_data_unused_t : + * | ... | + * +-------------------------------------------------+ + * | unused space | + * +-------------------------------------------------+ + * | ... | + * | xfs_dir2_leaf_entry_t | + * | xfs_dir2_leaf_entry_t | + * +-------------------------------------------------+ + * | xfs_dir2_block_tail_t | + * +-------------------------------------------------+ + * + * As all the entries are variable size structures the accessors below should + * be used to iterate over them. + */ + +typedef struct xfs_dir2_block_tail { + __be32 count; /* count of leaf entries */ + __be32 stale; /* count of stale lf entries */ +} xfs_dir2_block_tail_t; + +/* + * Pointer to the leaf entries embedded in a data block (1-block format) + */ +static inline struct xfs_dir2_leaf_entry * +xfs_dir2_block_leaf_p(struct xfs_dir2_block_tail *btp) +{ + return ((struct xfs_dir2_leaf_entry *)btp) - be32_to_cpu(btp->count); +} + + +/* + * Attribute storage layout + * + * Attribute lists are structured around Btrees where all the data + * elements are in the leaf nodes. Attribute names are hashed into an int, + * then that int is used as the index into the Btree. Since the hashval + * of an attribute name may not be unique, we may have duplicate keys. The + * internal links in the Btree are logical block offsets into the file. + * + * Struct leaf_entry's are packed from the top. Name/values grow from the + * bottom but are not packed. The freemap contains run-length-encoded entries + * for the free bytes after the leaf_entry's, but only the N largest such, + * smaller runs are dropped. When the freemap doesn't show enough space + * for an allocation, we compact the name/value area and try again. If we + * still don't have enough space, then we have to split the block. The + * name/value structs (both local and remote versions) must be 32bit aligned. + * + * Since we have duplicate hash keys, for each key that matches, compare + * the actual name string. The root and intermediate node search always + * takes the first-in-the-block key match found, so we should only have + * to work "forw"ard. If none matches, continue with the "forw"ard leaf + * nodes until the hash key changes or the attribute name is found. + * + * We store the fact that an attribute is a ROOT/USER/SECURE attribute in + * the leaf_entry. The namespaces are independent only because we also look + * at the namespace bit when we are looking for a matching attribute name. + * + * We also store an "incomplete" bit in the leaf_entry. It shows that an + * attribute is in the middle of being created and should not be shown to + * the user if we crash during the time that the bit is set. We clear the + * bit when we have finished setting up the attribute. We do this because + * we cannot create some large attributes inside a single transaction, and we + * need some indication that we weren't finished if we crash in the middle. + */ +#define XFS_ATTR_LEAF_MAPSIZE 3 /* how many freespace slots */ + +/* + * Attribute storage when stored inside the inode. + * + * Small attribute lists are packed as tightly as possible so as to fit into the + * literal area of the inode. + * + * These "shortform" attribute forks consist of a single xfs_attr_sf_hdr header + * followed by zero or more xfs_attr_sf_entry structures. + */ +struct xfs_attr_sf_hdr { /* constant-structure header block */ + __be16 totsize; /* total bytes in shortform list */ + __u8 count; /* count of active entries */ + __u8 padding; +}; + +struct xfs_attr_sf_entry { + __u8 namelen; /* actual length of name (no NULL) */ + __u8 valuelen; /* actual length of value (no NULL) */ + __u8 flags; /* flags bits (XFS_ATTR_*) */ + __u8 nameval[]; /* name & value bytes concatenated */ +}; + +typedef struct xfs_attr_leaf_map { /* RLE map of free bytes */ + __be16 base; /* base of free region */ + __be16 size; /* length of free region */ +} xfs_attr_leaf_map_t; + +typedef struct xfs_attr_leaf_hdr { /* constant-structure header block */ + xfs_da_blkinfo_t info; /* block type, links, etc. */ + __be16 count; /* count of active leaf_entry's */ + __be16 usedbytes; /* num bytes of names/values stored */ + __be16 firstused; /* first used byte in name area */ + __u8 holes; /* != 0 if blk needs compaction */ + __u8 pad1; + xfs_attr_leaf_map_t freemap[XFS_ATTR_LEAF_MAPSIZE]; + /* N largest free regions */ +} xfs_attr_leaf_hdr_t; + +typedef struct xfs_attr_leaf_entry { /* sorted on key, not name */ + __be32 hashval; /* hash value of name */ + __be16 nameidx; /* index into buffer of name/value */ + __u8 flags; /* LOCAL/ROOT/SECURE/INCOMPLETE flag */ + __u8 pad2; /* unused pad byte */ +} xfs_attr_leaf_entry_t; + +typedef struct xfs_attr_leaf_name_local { + __be16 valuelen; /* number of bytes in value */ + __u8 namelen; /* length of name bytes */ + /* + * In Linux 6.5 this flex array was converted from nameval[1] to + * nameval[]. Be very careful here about extra padding at the end; + * see xfs_attr_leaf_entsize_local() for details. + */ + __u8 nameval[]; /* name/value bytes */ +} xfs_attr_leaf_name_local_t; + +typedef struct xfs_attr_leaf_name_remote { + __be32 valueblk; /* block number of value bytes */ + __be32 valuelen; /* number of bytes in value */ + __u8 namelen; /* length of name bytes */ + /* + * In Linux 6.5 this flex array was converted from name[1] to name[]. + * Be very careful here about extra padding at the end; see + * xfs_attr_leaf_entsize_remote() for details. + */ + __u8 name[]; /* name bytes */ +} xfs_attr_leaf_name_remote_t; + +typedef struct xfs_attr_leafblock { + xfs_attr_leaf_hdr_t hdr; /* constant-structure header block */ + xfs_attr_leaf_entry_t entries[]; /* sorted on key, not name */ + /* + * The rest of the block contains the following structures after the + * leaf entries, growing from the bottom up. The variables are never + * referenced and definining them can actually make gcc optimize away + * accesses to the 'entries' array above index 0 so don't do that. + * + * xfs_attr_leaf_name_local_t namelist; + * xfs_attr_leaf_name_remote_t valuelist; + */ +} xfs_attr_leafblock_t; + +/* + * CRC enabled leaf structures. Called "version 3" structures to match the + * version number of the directory and dablk structures for this feature, and + * attr2 is already taken by the variable inode attribute fork size feature. + */ +struct xfs_attr3_leaf_hdr { + struct xfs_da3_blkinfo info; + __be16 count; + __be16 usedbytes; + __be16 firstused; + __u8 holes; + __u8 pad1; + struct xfs_attr_leaf_map freemap[XFS_ATTR_LEAF_MAPSIZE]; + __be32 pad2; /* 64 bit alignment */ +}; + +#define XFS_ATTR3_LEAF_CRC_OFF (offsetof(struct xfs_attr3_leaf_hdr, info.crc)) + +struct xfs_attr3_leafblock { + struct xfs_attr3_leaf_hdr hdr; + struct xfs_attr_leaf_entry entries[]; + + /* + * The rest of the block contains the following structures after the + * leaf entries, growing from the bottom up. The variables are never + * referenced, the locations accessed purely from helper functions. + * + * struct xfs_attr_leaf_name_local + * struct xfs_attr_leaf_name_remote + */ +}; + +/* + * Special value to represent fs block size in the leaf header firstused field. + * Only used when block size overflows the 2-bytes available on disk. + */ +#define XFS_ATTR3_LEAF_NULLOFF 0 + +/* + * Flags used in the leaf_entry[i].flags field. + */ +#define XFS_ATTR_LOCAL_BIT 0 /* attr is stored locally */ +#define XFS_ATTR_ROOT_BIT 1 /* limit access to trusted attrs */ +#define XFS_ATTR_SECURE_BIT 2 /* limit access to secure attrs */ +#define XFS_ATTR_PARENT_BIT 3 /* parent pointer attrs */ +#define XFS_ATTR_INCOMPLETE_BIT 7 /* attr in middle of create/delete */ +#define XFS_ATTR_LOCAL (1u << XFS_ATTR_LOCAL_BIT) +#define XFS_ATTR_ROOT (1u << XFS_ATTR_ROOT_BIT) +#define XFS_ATTR_SECURE (1u << XFS_ATTR_SECURE_BIT) +#define XFS_ATTR_PARENT (1u << XFS_ATTR_PARENT_BIT) +#define XFS_ATTR_INCOMPLETE (1u << XFS_ATTR_INCOMPLETE_BIT) + +#define XFS_ATTR_NSP_ONDISK_MASK (XFS_ATTR_ROOT | \ + XFS_ATTR_SECURE | \ + XFS_ATTR_PARENT) + +/* Private attr namespaces not exposed to userspace */ +#define XFS_ATTR_PRIVATE_NSP_MASK (XFS_ATTR_PARENT) + +#define XFS_ATTR_ONDISK_MASK (XFS_ATTR_NSP_ONDISK_MASK | \ + XFS_ATTR_LOCAL | \ + XFS_ATTR_INCOMPLETE) + +#define XFS_ATTR_NAMESPACE_STR \ + { XFS_ATTR_LOCAL, "local" }, \ + { XFS_ATTR_ROOT, "root" }, \ + { XFS_ATTR_SECURE, "secure" }, \ + { XFS_ATTR_PARENT, "parent" } + +/* + * Alignment for namelist and valuelist entries (since they are mixed + * there can be only one alignment value) + */ +#define XFS_ATTR_LEAF_NAME_ALIGN ((uint)sizeof(xfs_dablk_t)) + +static inline int +xfs_attr3_leaf_hdr_size(const struct xfs_attr_leafblock *leafp) +{ + if (leafp->hdr.info.magic == cpu_to_be16(XFS_ATTR3_LEAF_MAGIC)) + return sizeof(struct xfs_attr3_leaf_hdr); + return sizeof(struct xfs_attr_leaf_hdr); +} + +static inline struct xfs_attr_leaf_entry * +xfs_attr3_leaf_entryp(xfs_attr_leafblock_t *leafp) +{ + if (leafp->hdr.info.magic == cpu_to_be16(XFS_ATTR3_LEAF_MAGIC)) + return &((struct xfs_attr3_leafblock *)leafp)->entries[0]; + return &leafp->entries[0]; +} + +/* + * Cast typed pointers for "local" and "remote" name/value structs. + */ +static inline char * +xfs_attr3_leaf_name(xfs_attr_leafblock_t *leafp, int idx) +{ + struct xfs_attr_leaf_entry *entries = xfs_attr3_leaf_entryp(leafp); + + return &((char *)leafp)[be16_to_cpu(entries[idx].nameidx)]; +} + +static inline xfs_attr_leaf_name_remote_t * +xfs_attr3_leaf_name_remote(xfs_attr_leafblock_t *leafp, int idx) +{ + return (xfs_attr_leaf_name_remote_t *)xfs_attr3_leaf_name(leafp, idx); +} + +static inline xfs_attr_leaf_name_local_t * +xfs_attr3_leaf_name_local(xfs_attr_leafblock_t *leafp, int idx) +{ + return (xfs_attr_leaf_name_local_t *)xfs_attr3_leaf_name(leafp, idx); +} + +/* + * Calculate total bytes used (including trailing pad for alignment) for + * a "local" name/value structure, a "remote" name/value structure, and + * a pointer which might be either. + */ +static inline int xfs_attr_leaf_entsize_remote(int nlen) +{ + /* + * Prior to Linux 6.5, struct xfs_attr_leaf_name_remote ended with + * name[1], which was used as a flexarray. The layout of this struct + * is 9 bytes of fixed-length fields followed by a __u8 flex array at + * offset 9. + * + * On most architectures, struct xfs_attr_leaf_name_remote had two + * bytes of implicit padding at the end of the struct to make the + * struct length 12. After converting name[1] to name[], there are + * three implicit padding bytes and the struct size remains 12. + * However, there are compiler configurations that do not add implicit + * padding at all (m68k) and have been broken for years. + * + * This entsize computation historically added (the xattr name length) + * to (the padded struct length - 1) and rounded that sum up to the + * nearest multiple of 4 (NAME_ALIGN). IOWs, round_up(11 + nlen, 4). + * This is encoded in the ondisk format, so we cannot change this. + * + * Compute the entsize from offsetof of the flexarray and manually + * adding bytes for the implicit padding. + */ + const size_t remotesize = + offsetof(struct xfs_attr_leaf_name_remote, name) + 2; + + return round_up(remotesize + nlen, XFS_ATTR_LEAF_NAME_ALIGN); +} + +static inline int xfs_attr_leaf_entsize_local(int nlen, int vlen) +{ + /* + * Prior to Linux 6.5, struct xfs_attr_leaf_name_local ended with + * nameval[1], which was used as a flexarray. The layout of this + * struct is 3 bytes of fixed-length fields followed by a __u8 flex + * array at offset 3. + * + * struct xfs_attr_leaf_name_local had zero bytes of implicit padding + * at the end of the struct to make the struct length 4. On most + * architectures, after converting nameval[1] to nameval[], there is + * one implicit padding byte and the struct size remains 4. However, + * there are compiler configurations that do not add implicit padding + * at all (m68k) and would break. + * + * This entsize computation historically added (the xattr name and + * value length) to (the padded struct length - 1) and rounded that sum + * up to the nearest multiple of 4 (NAME_ALIGN). IOWs, the formula is + * round_up(3 + nlen + vlen, 4). This is encoded in the ondisk format, + * so we cannot change this. + * + * Compute the entsize from offsetof of the flexarray and manually + * adding bytes for the implicit padding. + */ + const size_t localsize = + offsetof(struct xfs_attr_leaf_name_local, nameval); + + return round_up(localsize + nlen + vlen, XFS_ATTR_LEAF_NAME_ALIGN); +} + +static inline int xfs_attr_leaf_entsize_local_max(int bsize) +{ + return (((bsize) >> 1) + ((bsize) >> 2)); +} + + + +/* + * Remote attribute block format definition + * + * There is one of these headers per filesystem block in a remote attribute. + * This is done to ensure there is a 1:1 mapping between the attribute value + * length and the number of blocks needed to store the attribute. This makes the + * verification of a buffer a little more complex, but greatly simplifies the + * allocation, reading and writing of these attributes as we don't have to guess + * the number of blocks needed to store the attribute data. + */ +#define XFS_ATTR3_RMT_MAGIC 0x5841524d /* XARM */ + +struct xfs_attr3_rmt_hdr { + __be32 rm_magic; + __be32 rm_offset; + __be32 rm_bytes; + __be32 rm_crc; + uuid_t rm_uuid; + __be64 rm_owner; + __be64 rm_blkno; + __be64 rm_lsn; +}; + +#define XFS_ATTR3_RMT_CRC_OFF offsetof(struct xfs_attr3_rmt_hdr, rm_crc) + +unsigned int xfs_attr3_rmt_buf_space(struct xfs_mount *mp); + +/* Number of bytes in a directory block. */ +static inline unsigned int xfs_dir2_dirblock_bytes(struct xfs_sb *sbp) +{ + return 1 << (sbp->sb_blocklog + sbp->sb_dirblklog); +} + +xfs_failaddr_t xfs_da3_blkinfo_verify(struct xfs_buf *bp, + struct xfs_da3_blkinfo *hdr3); + +/* + * Parent pointer attribute format definition + * + * The xattr name contains the dirent name. + * The xattr value encodes the parent inode number and generation to ease + * opening parents by handle. + * The xattr hashval is xfs_dir2_namehash() ^ p_ino + */ +struct xfs_parent_rec { + __be64 p_ino; + __be32 p_gen; +} __packed; + +#endif /* __XFS_DA_FORMAT_H__ */ diff --git a/fs/xfs/libxfs/xfs_defer.c b/fs/xfs/libxfs/xfs_defer.c new file mode 100644 index 000000000..89501e8bd --- /dev/null +++ b/fs/xfs/libxfs/xfs_defer.c @@ -0,0 +1,1245 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2016 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_trans.h" +#include "xfs_trans_priv.h" +#include "xfs_buf_item.h" +#include "xfs_inode.h" +#include "xfs_inode_item.h" +#include "xfs_trace.h" +#include "xfs_icache.h" +#include "xfs_log.h" +#include "xfs_log_priv.h" +#include "xfs_rmap.h" +#include "xfs_refcount.h" +#include "xfs_bmap.h" +#include "xfs_alloc.h" +#include "xfs_buf.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_attr.h" +#include "xfs_exchmaps.h" + +static struct kmem_cache *xfs_defer_pending_cache; + +/* + * Deferred Operations in XFS + * + * Due to the way locking rules work in XFS, certain transactions (block + * mapping and unmapping, typically) have permanent reservations so that + * we can roll the transaction to adhere to AG locking order rules and + * to unlock buffers between metadata updates. Prior to rmap/reflink, + * the mapping code had a mechanism to perform these deferrals for + * extents that were going to be freed; this code makes that facility + * more generic. + * + * When adding the reverse mapping and reflink features, it became + * necessary to perform complex remapping multi-transactions to comply + * with AG locking order rules, and to be able to spread a single + * refcount update operation (an operation on an n-block extent can + * update as many as n records!) among multiple transactions. XFS can + * roll a transaction to facilitate this, but using this facility + * requires us to log "intent" items in case log recovery needs to + * redo the operation, and to log "done" items to indicate that redo + * is not necessary. + * + * Deferred work is tracked in xfs_defer_pending items. Each pending + * item tracks one type of deferred work. Incoming work items (which + * have not yet had an intent logged) are attached to a pending item + * on the dop_intake list, where they wait for the caller to finish + * the deferred operations. + * + * Finishing a set of deferred operations is an involved process. To + * start, we define "rolling a deferred-op transaction" as follows: + * + * > For each xfs_defer_pending item on the dop_intake list, + * - Sort the work items in AG order. XFS locking + * order rules require us to lock buffers in AG order. + * - Create a log intent item for that type. + * - Attach it to the pending item. + * - Move the pending item from the dop_intake list to the + * dop_pending list. + * > Roll the transaction. + * + * NOTE: To avoid exceeding the transaction reservation, we limit the + * number of items that we attach to a given xfs_defer_pending. + * + * The actual finishing process looks like this: + * + * > For each xfs_defer_pending in the dop_pending list, + * - Roll the deferred-op transaction as above. + * - Create a log done item for that type, and attach it to the + * log intent item. + * - For each work item attached to the log intent item, + * * Perform the described action. + * * Attach the work item to the log done item. + * * If the result of doing the work was -EAGAIN, ->finish work + * wants a new transaction. See the "Requesting a Fresh + * Transaction while Finishing Deferred Work" section below for + * details. + * + * The key here is that we must log an intent item for all pending + * work items every time we roll the transaction, and that we must log + * a done item as soon as the work is completed. With this mechanism + * we can perform complex remapping operations, chaining intent items + * as needed. + * + * Requesting a Fresh Transaction while Finishing Deferred Work + * + * If ->finish_item decides that it needs a fresh transaction to + * finish the work, it must ask its caller (xfs_defer_finish) for a + * continuation. The most likely cause of this circumstance are the + * refcount adjust functions deciding that they've logged enough items + * to be at risk of exceeding the transaction reservation. + * + * To get a fresh transaction, we want to log the existing log done + * item to prevent the log intent item from replaying, immediately log + * a new log intent item with the unfinished work items, roll the + * transaction, and re-call ->finish_item wherever it left off. The + * log done item and the new log intent item must be in the same + * transaction or atomicity cannot be guaranteed; defer_finish ensures + * that this happens. + * + * This requires some coordination between ->finish_item and + * defer_finish. Upon deciding to request a new transaction, + * ->finish_item should update the current work item to reflect the + * unfinished work. Next, it should reset the log done item's list + * count to the number of items finished, and return -EAGAIN. + * defer_finish sees the -EAGAIN, logs the new log intent item + * with the remaining work items, and leaves the xfs_defer_pending + * item at the head of the dop_work queue. Then it rolls the + * transaction and picks up processing where it left off. It is + * required that ->finish_item must be careful to leave enough + * transaction reservation to fit the new log intent item. + * + * This is an example of remapping the extent (E, E+B) into file X at + * offset A and dealing with the extent (C, C+B) already being mapped + * there: + * +-------------------------------------------------+ + * | Unmap file X startblock C offset A length B | t0 + * | Intent to reduce refcount for extent (C, B) | + * | Intent to remove rmap (X, C, A, B) | + * | Intent to free extent (D, 1) (bmbt block) | + * | Intent to map (X, A, B) at startblock E | + * +-------------------------------------------------+ + * | Map file X startblock E offset A length B | t1 + * | Done mapping (X, E, A, B) | + * | Intent to increase refcount for extent (E, B) | + * | Intent to add rmap (X, E, A, B) | + * +-------------------------------------------------+ + * | Reduce refcount for extent (C, B) | t2 + * | Done reducing refcount for extent (C, 9) | + * | Intent to reduce refcount for extent (C+9, B-9) | + * | (ran out of space after 9 refcount updates) | + * +-------------------------------------------------+ + * | Reduce refcount for extent (C+9, B+9) | t3 + * | Done reducing refcount for extent (C+9, B-9) | + * | Increase refcount for extent (E, B) | + * | Done increasing refcount for extent (E, B) | + * | Intent to free extent (C, B) | + * | Intent to free extent (F, 1) (refcountbt block) | + * | Intent to remove rmap (F, 1, REFC) | + * +-------------------------------------------------+ + * | Remove rmap (X, C, A, B) | t4 + * | Done removing rmap (X, C, A, B) | + * | Add rmap (X, E, A, B) | + * | Done adding rmap (X, E, A, B) | + * | Remove rmap (F, 1, REFC) | + * | Done removing rmap (F, 1, REFC) | + * +-------------------------------------------------+ + * | Free extent (C, B) | t5 + * | Done freeing extent (C, B) | + * | Free extent (D, 1) | + * | Done freeing extent (D, 1) | + * | Free extent (F, 1) | + * | Done freeing extent (F, 1) | + * +-------------------------------------------------+ + * + * If we should crash before t2 commits, log recovery replays + * the following intent items: + * + * - Intent to reduce refcount for extent (C, B) + * - Intent to remove rmap (X, C, A, B) + * - Intent to free extent (D, 1) (bmbt block) + * - Intent to increase refcount for extent (E, B) + * - Intent to add rmap (X, E, A, B) + * + * In the process of recovering, it should also generate and take care + * of these intent items: + * + * - Intent to free extent (C, B) + * - Intent to free extent (F, 1) (refcountbt block) + * - Intent to remove rmap (F, 1, REFC) + * + * Note that the continuation requested between t2 and t3 is likely to + * reoccur. + */ +STATIC struct xfs_log_item * +xfs_defer_barrier_create_intent( + struct xfs_trans *tp, + struct list_head *items, + unsigned int count, + bool sort) +{ + return NULL; +} + +STATIC void +xfs_defer_barrier_abort_intent( + struct xfs_log_item *intent) +{ + /* empty */ +} + +STATIC struct xfs_log_item * +xfs_defer_barrier_create_done( + struct xfs_trans *tp, + struct xfs_log_item *intent, + unsigned int count) +{ + return NULL; +} + +STATIC int +xfs_defer_barrier_finish_item( + struct xfs_trans *tp, + struct xfs_log_item *done, + struct list_head *item, + struct xfs_btree_cur **state) +{ + ASSERT(0); + return -EFSCORRUPTED; +} + +STATIC void +xfs_defer_barrier_cancel_item( + struct list_head *item) +{ + ASSERT(0); +} + +static const struct xfs_defer_op_type xfs_barrier_defer_type = { + .max_items = 1, + .create_intent = xfs_defer_barrier_create_intent, + .abort_intent = xfs_defer_barrier_abort_intent, + .create_done = xfs_defer_barrier_create_done, + .finish_item = xfs_defer_barrier_finish_item, + .cancel_item = xfs_defer_barrier_cancel_item, +}; + +/* Create a log intent done item for a log intent item. */ +static inline void +xfs_defer_create_done( + struct xfs_trans *tp, + struct xfs_defer_pending *dfp) +{ + struct xfs_log_item *lip; + + /* If there is no log intent item, there can be no log done item. */ + if (!dfp->dfp_intent) + return; + + /* + * Mark the transaction dirty, even on error. This ensures the + * transaction is aborted, which: + * + * 1.) releases the log intent item and frees the log done item + * 2.) shuts down the filesystem + */ + tp->t_flags |= XFS_TRANS_DIRTY; + lip = dfp->dfp_ops->create_done(tp, dfp->dfp_intent, dfp->dfp_count); + if (!lip) + return; + + tp->t_flags |= XFS_TRANS_HAS_INTENT_DONE; + xfs_trans_add_item(tp, lip); + set_bit(XFS_LI_DIRTY, &lip->li_flags); + dfp->dfp_done = lip; +} + +/* + * Ensure there's a log intent item associated with this deferred work item if + * the operation must be restarted on crash. Returns 1 if there's a log item; + * 0 if there isn't; or a negative errno. + */ +static int +xfs_defer_create_intent( + struct xfs_trans *tp, + struct xfs_defer_pending *dfp, + bool sort) +{ + struct xfs_log_item *lip; + + if (dfp->dfp_intent) + return 1; + + lip = dfp->dfp_ops->create_intent(tp, &dfp->dfp_work, dfp->dfp_count, + sort); + if (!lip) + return 0; + if (IS_ERR(lip)) + return PTR_ERR(lip); + + tp->t_flags |= XFS_TRANS_DIRTY; + xfs_trans_add_item(tp, lip); + set_bit(XFS_LI_DIRTY, &lip->li_flags); + dfp->dfp_intent = lip; + return 1; +} + +/* + * For each pending item in the intake list, log its intent item and the + * associated extents, then add the entire intake list to the end of + * the pending list. + * + * Returns 1 if at least one log item was associated with the deferred work; + * 0 if there are no log items; or a negative errno. + */ +static int +xfs_defer_create_intents( + struct xfs_trans *tp) +{ + struct xfs_defer_pending *dfp; + int ret = 0; + + list_for_each_entry(dfp, &tp->t_dfops, dfp_list) { + int ret2; + + trace_xfs_defer_create_intent(tp->t_mountp, dfp); + ret2 = xfs_defer_create_intent(tp, dfp, true); + if (ret2 < 0) + return ret2; + ret |= ret2; + } + return ret; +} + +static inline void +xfs_defer_pending_abort( + struct xfs_mount *mp, + struct xfs_defer_pending *dfp) +{ + trace_xfs_defer_pending_abort(mp, dfp); + + if (dfp->dfp_intent && !dfp->dfp_done) { + dfp->dfp_ops->abort_intent(dfp->dfp_intent); + dfp->dfp_intent = NULL; + } +} + +static inline void +xfs_defer_pending_cancel_work( + struct xfs_mount *mp, + struct xfs_defer_pending *dfp) +{ + struct list_head *pwi; + struct list_head *n; + + trace_xfs_defer_cancel_list(mp, dfp); + + list_del(&dfp->dfp_list); + list_for_each_safe(pwi, n, &dfp->dfp_work) { + list_del(pwi); + dfp->dfp_count--; + trace_xfs_defer_cancel_item(mp, dfp, pwi); + dfp->dfp_ops->cancel_item(pwi); + } + ASSERT(dfp->dfp_count == 0); + kmem_cache_free(xfs_defer_pending_cache, dfp); +} + +STATIC void +xfs_defer_pending_abort_list( + struct xfs_mount *mp, + struct list_head *dop_list) +{ + struct xfs_defer_pending *dfp; + + /* Abort intent items that don't have a done item. */ + list_for_each_entry(dfp, dop_list, dfp_list) + xfs_defer_pending_abort(mp, dfp); +} + +/* Abort all the intents that were committed. */ +STATIC void +xfs_defer_trans_abort( + struct xfs_trans *tp, + struct list_head *dop_pending) +{ + trace_xfs_defer_trans_abort(tp, _RET_IP_); + xfs_defer_pending_abort_list(tp->t_mountp, dop_pending); +} + +/* + * Capture resources that the caller said not to release ("held") when the + * transaction commits. Caller is responsible for zero-initializing @dres. + */ +static int +xfs_defer_save_resources( + struct xfs_defer_resources *dres, + struct xfs_trans *tp) +{ + struct xfs_buf_log_item *bli; + struct xfs_inode_log_item *ili; + struct xfs_log_item *lip; + + BUILD_BUG_ON(NBBY * sizeof(dres->dr_ordered) < XFS_DEFER_OPS_NR_BUFS); + + list_for_each_entry(lip, &tp->t_items, li_trans) { + switch (lip->li_type) { + case XFS_LI_BUF: + bli = container_of(lip, struct xfs_buf_log_item, + bli_item); + if (bli->bli_flags & XFS_BLI_HOLD) { + if (dres->dr_bufs >= XFS_DEFER_OPS_NR_BUFS) { + ASSERT(0); + return -EFSCORRUPTED; + } + if (bli->bli_flags & XFS_BLI_ORDERED) + dres->dr_ordered |= + (1U << dres->dr_bufs); + else + xfs_trans_dirty_buf(tp, bli->bli_buf); + dres->dr_bp[dres->dr_bufs++] = bli->bli_buf; + } + break; + case XFS_LI_INODE: + ili = container_of(lip, struct xfs_inode_log_item, + ili_item); + if (ili->ili_lock_flags == 0) { + if (dres->dr_inos >= XFS_DEFER_OPS_NR_INODES) { + ASSERT(0); + return -EFSCORRUPTED; + } + xfs_trans_log_inode(tp, ili->ili_inode, + XFS_ILOG_CORE); + dres->dr_ip[dres->dr_inos++] = ili->ili_inode; + } + break; + default: + break; + } + } + + return 0; +} + +/* Attach the held resources to the transaction. */ +static void +xfs_defer_restore_resources( + struct xfs_trans *tp, + struct xfs_defer_resources *dres) +{ + unsigned short i; + + /* Rejoin the joined inodes. */ + for (i = 0; i < dres->dr_inos; i++) + xfs_trans_ijoin(tp, dres->dr_ip[i], 0); + + /* Rejoin the buffers and dirty them so the log moves forward. */ + for (i = 0; i < dres->dr_bufs; i++) { + xfs_trans_bjoin(tp, dres->dr_bp[i]); + if (dres->dr_ordered & (1U << i)) + xfs_trans_ordered_buf(tp, dres->dr_bp[i]); + xfs_trans_bhold(tp, dres->dr_bp[i]); + } +} + +/* Roll a transaction so we can do some deferred op processing. */ +STATIC int +xfs_defer_trans_roll( + struct xfs_trans **tpp) +{ + struct xfs_defer_resources dres = { }; + int error; + + error = xfs_defer_save_resources(&dres, *tpp); + if (error) + return error; + + trace_xfs_defer_trans_roll(*tpp, _RET_IP_); + + /* + * Roll the transaction. Rolling always given a new transaction (even + * if committing the old one fails!) to hand back to the caller, so we + * join the held resources to the new transaction so that we always + * return with the held resources joined to @tpp, no matter what + * happened. + */ + error = xfs_trans_roll(tpp); + + xfs_defer_restore_resources(*tpp, &dres); + + if (error) + trace_xfs_defer_trans_roll_error(*tpp, error); + return error; +} + +/* + * Free up any items left in the list. + */ +static void +xfs_defer_cancel_list( + struct xfs_mount *mp, + struct list_head *dop_list) +{ + struct xfs_defer_pending *dfp; + struct xfs_defer_pending *pli; + + /* + * Free the pending items. Caller should already have arranged + * for the intent items to be released. + */ + list_for_each_entry_safe(dfp, pli, dop_list, dfp_list) + xfs_defer_pending_cancel_work(mp, dfp); +} + +static inline void +xfs_defer_relog_intent( + struct xfs_trans *tp, + struct xfs_defer_pending *dfp) +{ + struct xfs_log_item *lip; + + xfs_defer_create_done(tp, dfp); + + lip = dfp->dfp_ops->relog_intent(tp, dfp->dfp_intent, dfp->dfp_done); + if (lip) { + xfs_trans_add_item(tp, lip); + set_bit(XFS_LI_DIRTY, &lip->li_flags); + } + dfp->dfp_done = NULL; + dfp->dfp_intent = lip; +} + +/* + * Prevent a log intent item from pinning the tail of the log by logging a + * done item to release the intent item; and then log a new intent item. + * The caller should provide a fresh transaction and roll it after we're done. + */ +static void +xfs_defer_relog( + struct xfs_trans **tpp, + struct list_head *dfops) +{ + struct xlog *log = (*tpp)->t_mountp->m_log; + struct xfs_defer_pending *dfp; + xfs_lsn_t threshold_lsn = NULLCOMMITLSN; + + + ASSERT((*tpp)->t_flags & XFS_TRANS_PERM_LOG_RES); + + list_for_each_entry(dfp, dfops, dfp_list) { + /* + * If the log intent item for this deferred op is not a part of + * the current log checkpoint, relog the intent item to keep + * the log tail moving forward. We're ok with this being racy + * because an incorrect decision means we'll be a little slower + * at pushing the tail. + */ + if (dfp->dfp_intent == NULL || + xfs_log_item_in_current_chkpt(dfp->dfp_intent)) + continue; + + /* + * Figure out where we need the tail to be in order to maintain + * the minimum required free space in the log. Only sample + * the log threshold once per call. + */ + if (threshold_lsn == NULLCOMMITLSN) { + threshold_lsn = xfs_ail_get_push_target(log->l_ailp); + if (threshold_lsn == NULLCOMMITLSN) + break; + } + if (XFS_LSN_CMP(dfp->dfp_intent->li_lsn, threshold_lsn) >= 0) + continue; + + trace_xfs_defer_relog_intent((*tpp)->t_mountp, dfp); + XFS_STATS_INC((*tpp)->t_mountp, xs_defer_relog); + + xfs_defer_relog_intent(*tpp, dfp); + } +} + +/* + * Log an intent-done item for the first pending intent, and finish the work + * items. + */ +int +xfs_defer_finish_one( + struct xfs_trans *tp, + struct xfs_defer_pending *dfp) +{ + const struct xfs_defer_op_type *ops = dfp->dfp_ops; + struct xfs_btree_cur *state = NULL; + struct list_head *li, *n; + int error; + + trace_xfs_defer_pending_finish(tp->t_mountp, dfp); + + xfs_defer_create_done(tp, dfp); + list_for_each_safe(li, n, &dfp->dfp_work) { + list_del(li); + dfp->dfp_count--; + trace_xfs_defer_finish_item(tp->t_mountp, dfp, li); + error = ops->finish_item(tp, dfp->dfp_done, li, &state); + if (error == -EAGAIN) { + int ret; + + /* + * Caller wants a fresh transaction; put the work item + * back on the list and log a new log intent item to + * replace the old one. See "Requesting a Fresh + * Transaction while Finishing Deferred Work" above. + */ + list_add(li, &dfp->dfp_work); + dfp->dfp_count++; + dfp->dfp_done = NULL; + dfp->dfp_intent = NULL; + ret = xfs_defer_create_intent(tp, dfp, false); + if (ret < 0) + error = ret; + } + + if (error) + goto out; + } + + /* Done with the dfp, free it. */ + list_del(&dfp->dfp_list); + kmem_cache_free(xfs_defer_pending_cache, dfp); +out: + if (ops->finish_cleanup) + ops->finish_cleanup(tp, state, error); + return error; +} + +/* Move all paused deferred work from @tp to @paused_list. */ +static void +xfs_defer_isolate_paused( + struct xfs_trans *tp, + struct list_head *paused_list) +{ + struct xfs_defer_pending *dfp; + struct xfs_defer_pending *pli; + + list_for_each_entry_safe(dfp, pli, &tp->t_dfops, dfp_list) { + if (!(dfp->dfp_flags & XFS_DEFER_PAUSED)) + continue; + + list_move_tail(&dfp->dfp_list, paused_list); + trace_xfs_defer_isolate_paused(tp->t_mountp, dfp); + } +} + +/* + * Finish all the pending work. This involves logging intent items for + * any work items that wandered in since the last transaction roll (if + * one has even happened), rolling the transaction, and finishing the + * work items in the first item on the logged-and-pending list. + * + * If an inode is provided, relog it to the new transaction. + */ +int +xfs_defer_finish_noroll( + struct xfs_trans **tp) +{ + struct xfs_defer_pending *dfp = NULL; + int error = 0; + LIST_HEAD(dop_pending); + LIST_HEAD(dop_paused); + + ASSERT((*tp)->t_flags & XFS_TRANS_PERM_LOG_RES); + + trace_xfs_defer_finish(*tp, _RET_IP_); + + /* Until we run out of pending work to finish... */ + while (!list_empty(&dop_pending) || !list_empty(&(*tp)->t_dfops)) { + /* + * Deferred items that are created in the process of finishing + * other deferred work items should be queued at the head of + * the pending list, which puts them ahead of the deferred work + * that was created by the caller. This keeps the number of + * pending work items to a minimum, which decreases the amount + * of time that any one intent item can stick around in memory, + * pinning the log tail. + */ + int has_intents = xfs_defer_create_intents(*tp); + + xfs_defer_isolate_paused(*tp, &dop_paused); + + list_splice_init(&(*tp)->t_dfops, &dop_pending); + + if (has_intents < 0) { + error = has_intents; + goto out_shutdown; + } + if (has_intents || dfp) { + error = xfs_defer_trans_roll(tp); + if (error) + goto out_shutdown; + + /* Relog intent items to keep the log moving. */ + xfs_defer_relog(tp, &dop_pending); + xfs_defer_relog(tp, &dop_paused); + + if ((*tp)->t_flags & XFS_TRANS_DIRTY) { + error = xfs_defer_trans_roll(tp); + if (error) + goto out_shutdown; + } + } + + dfp = list_first_entry_or_null(&dop_pending, + struct xfs_defer_pending, dfp_list); + if (!dfp) + break; + error = xfs_defer_finish_one(*tp, dfp); + if (error && error != -EAGAIN) + goto out_shutdown; + } + + /* Requeue the paused items in the outgoing transaction. */ + list_splice_tail_init(&dop_paused, &(*tp)->t_dfops); + + trace_xfs_defer_finish_done(*tp, _RET_IP_); + return 0; + +out_shutdown: + list_splice_tail_init(&dop_paused, &dop_pending); + xfs_defer_trans_abort(*tp, &dop_pending); + xfs_force_shutdown((*tp)->t_mountp, SHUTDOWN_CORRUPT_INCORE); + trace_xfs_defer_finish_error(*tp, error); + xfs_defer_cancel_list((*tp)->t_mountp, &dop_pending); + xfs_defer_cancel(*tp); + return error; +} + +int +xfs_defer_finish( + struct xfs_trans **tp) +{ +#ifdef DEBUG + struct xfs_defer_pending *dfp; +#endif + int error; + + /* + * Finish and roll the transaction once more to avoid returning to the + * caller with a dirty transaction. + */ + error = xfs_defer_finish_noroll(tp); + if (error) + return error; + if ((*tp)->t_flags & XFS_TRANS_DIRTY) { + error = xfs_defer_trans_roll(tp); + if (error) { + xfs_force_shutdown((*tp)->t_mountp, + SHUTDOWN_CORRUPT_INCORE); + return error; + } + } + + /* Reset LOWMODE now that we've finished all the dfops. */ +#ifdef DEBUG + list_for_each_entry(dfp, &(*tp)->t_dfops, dfp_list) + ASSERT(dfp->dfp_flags & XFS_DEFER_PAUSED); +#endif + (*tp)->t_flags &= ~XFS_TRANS_LOWMODE; + return 0; +} + +void +xfs_defer_cancel( + struct xfs_trans *tp) +{ + struct xfs_mount *mp = tp->t_mountp; + + trace_xfs_defer_cancel(tp, _RET_IP_); + xfs_defer_trans_abort(tp, &tp->t_dfops); + xfs_defer_cancel_list(mp, &tp->t_dfops); +} + +/* + * Return the last pending work item attached to this transaction if it matches + * the deferred op type. + */ +static inline struct xfs_defer_pending * +xfs_defer_find_last( + struct xfs_trans *tp, + const struct xfs_defer_op_type *ops) +{ + struct xfs_defer_pending *dfp = NULL; + + /* No dfops at all? */ + if (list_empty(&tp->t_dfops)) + return NULL; + + dfp = list_last_entry(&tp->t_dfops, struct xfs_defer_pending, + dfp_list); + + /* Wrong type? */ + if (dfp->dfp_ops != ops) + return NULL; + return dfp; +} + +/* + * Decide if we can add a deferred work item to the last dfops item attached + * to the transaction. + */ +static inline bool +xfs_defer_can_append( + struct xfs_defer_pending *dfp, + const struct xfs_defer_op_type *ops) +{ + /* Already logged? */ + if (dfp->dfp_intent) + return false; + + /* Paused items cannot absorb more work */ + if (dfp->dfp_flags & XFS_DEFER_PAUSED) + return false; + + /* Already full? */ + if (ops->max_items && dfp->dfp_count >= ops->max_items) + return false; + + return true; +} + +/* Create a new pending item at the end of the transaction list. */ +static inline struct xfs_defer_pending * +xfs_defer_alloc( + struct list_head *dfops, + const struct xfs_defer_op_type *ops) +{ + struct xfs_defer_pending *dfp; + + dfp = kmem_cache_zalloc(xfs_defer_pending_cache, + GFP_KERNEL | __GFP_NOFAIL); + dfp->dfp_ops = ops; + INIT_LIST_HEAD(&dfp->dfp_work); + list_add_tail(&dfp->dfp_list, dfops); + + return dfp; +} + +/* Add an item for later deferred processing. */ +struct xfs_defer_pending * +xfs_defer_add( + struct xfs_trans *tp, + struct list_head *li, + const struct xfs_defer_op_type *ops) +{ + struct xfs_defer_pending *dfp = NULL; + + ASSERT(tp->t_flags & XFS_TRANS_PERM_LOG_RES); + + if (!ops->finish_item) { + ASSERT(ops->finish_item != NULL); + xfs_force_shutdown(tp->t_mountp, SHUTDOWN_CORRUPT_INCORE); + return NULL; + } + + dfp = xfs_defer_find_last(tp, ops); + if (!dfp || !xfs_defer_can_append(dfp, ops)) + dfp = xfs_defer_alloc(&tp->t_dfops, ops); + + xfs_defer_add_item(dfp, li); + trace_xfs_defer_add_item(tp->t_mountp, dfp, li); + return dfp; +} + +/* + * Add a defer ops barrier to force two otherwise adjacent deferred work items + * to be tracked separately and have separate log items. + */ +void +xfs_defer_add_barrier( + struct xfs_trans *tp) +{ + struct xfs_defer_pending *dfp; + + ASSERT(tp->t_flags & XFS_TRANS_PERM_LOG_RES); + + /* If the last defer op added was a barrier, we're done. */ + dfp = xfs_defer_find_last(tp, &xfs_barrier_defer_type); + if (dfp) + return; + + dfp = xfs_defer_alloc(&tp->t_dfops, &xfs_barrier_defer_type); + + trace_xfs_defer_add_item(tp->t_mountp, dfp, NULL); +} + +/* + * Create a pending deferred work item to replay the recovered intent item + * and add it to the list. + */ +void +xfs_defer_start_recovery( + struct xfs_log_item *lip, + struct list_head *r_dfops, + const struct xfs_defer_op_type *ops) +{ + struct xfs_defer_pending *dfp = xfs_defer_alloc(r_dfops, ops); + + dfp->dfp_intent = lip; +} + +/* + * Cancel a deferred work item created to recover a log intent item. @dfp + * will be freed after this function returns. + */ +void +xfs_defer_cancel_recovery( + struct xfs_mount *mp, + struct xfs_defer_pending *dfp) +{ + xfs_defer_pending_abort(mp, dfp); + xfs_defer_pending_cancel_work(mp, dfp); +} + +/* Replay the deferred work item created from a recovered log intent item. */ +int +xfs_defer_finish_recovery( + struct xfs_mount *mp, + struct xfs_defer_pending *dfp, + struct list_head *capture_list) +{ + const struct xfs_defer_op_type *ops = dfp->dfp_ops; + int error; + + /* dfp is freed by recover_work and must not be accessed afterwards */ + error = ops->recover_work(dfp, capture_list); + if (error) + trace_xlog_intent_recovery_failed(mp, ops, error); + return error; +} + +/* + * Move deferred ops from one transaction to another and reset the source to + * initial state. This is primarily used to carry state forward across + * transaction rolls with pending dfops. + */ +void +xfs_defer_move( + struct xfs_trans *dtp, + struct xfs_trans *stp) +{ + list_splice_init(&stp->t_dfops, &dtp->t_dfops); + + /* + * Low free space mode was historically controlled by a dfops field. + * This meant that low mode state potentially carried across multiple + * transaction rolls. Transfer low mode on a dfops move to preserve + * that behavior. + */ + dtp->t_flags |= (stp->t_flags & XFS_TRANS_LOWMODE); + stp->t_flags &= ~XFS_TRANS_LOWMODE; +} + +/* + * Prepare a chain of fresh deferred ops work items to be completed later. Log + * recovery requires the ability to put off until later the actual finishing + * work so that it can process unfinished items recovered from the log in + * correct order. + * + * Create and log intent items for all the work that we're capturing so that we + * can be assured that the items will get replayed if the system goes down + * before log recovery gets a chance to finish the work it put off. The entire + * deferred ops state is transferred to the capture structure and the + * transaction is then ready for the caller to commit it. If there are no + * intent items to capture, this function returns NULL. + * + * If capture_ip is not NULL, the capture structure will obtain an extra + * reference to the inode. + */ +static struct xfs_defer_capture * +xfs_defer_ops_capture( + struct xfs_trans *tp) +{ + struct xfs_defer_capture *dfc; + unsigned short i; + int error; + + if (list_empty(&tp->t_dfops)) + return NULL; + + error = xfs_defer_create_intents(tp); + if (error < 0) + return ERR_PTR(error); + + /* Create an object to capture the defer ops. */ + dfc = kzalloc_obj(*dfc, GFP_KERNEL | __GFP_NOFAIL); + INIT_LIST_HEAD(&dfc->dfc_list); + INIT_LIST_HEAD(&dfc->dfc_dfops); + + /* Move the dfops chain and transaction state to the capture struct. */ + list_splice_init(&tp->t_dfops, &dfc->dfc_dfops); + dfc->dfc_tpflags = tp->t_flags & XFS_TRANS_LOWMODE; + tp->t_flags &= ~XFS_TRANS_LOWMODE; + + /* Capture the remaining block reservations along with the dfops. */ + dfc->dfc_blkres = tp->t_blk_res - tp->t_blk_res_used; + dfc->dfc_rtxres = tp->t_rtx_res - tp->t_rtx_res_used; + + /* Preserve the log reservation size. */ + dfc->dfc_logres = tp->t_log_res; + + error = xfs_defer_save_resources(&dfc->dfc_held, tp); + if (error) { + /* + * Resource capture should never fail, but if it does, we + * still have to shut down the log and release things + * properly. + */ + xfs_force_shutdown(tp->t_mountp, SHUTDOWN_CORRUPT_INCORE); + } + + /* + * Grab extra references to the inodes and buffers because callers are + * expected to release their held references after we commit the + * transaction. + */ + for (i = 0; i < dfc->dfc_held.dr_inos; i++) { + xfs_assert_ilocked(dfc->dfc_held.dr_ip[i], XFS_ILOCK_EXCL); + ihold(VFS_I(dfc->dfc_held.dr_ip[i])); + } + + for (i = 0; i < dfc->dfc_held.dr_bufs; i++) + xfs_buf_hold(dfc->dfc_held.dr_bp[i]); + + return dfc; +} + +/* Release all resources that we used to capture deferred ops. */ +void +xfs_defer_ops_capture_abort( + struct xfs_mount *mp, + struct xfs_defer_capture *dfc) +{ + unsigned short i; + + xfs_defer_pending_abort_list(mp, &dfc->dfc_dfops); + xfs_defer_cancel_list(mp, &dfc->dfc_dfops); + + for (i = 0; i < dfc->dfc_held.dr_bufs; i++) + xfs_buf_relse(dfc->dfc_held.dr_bp[i]); + + for (i = 0; i < dfc->dfc_held.dr_inos; i++) + xfs_irele(dfc->dfc_held.dr_ip[i]); + + kfree(dfc); +} + +/* + * Capture any deferred ops and commit the transaction. This is the last step + * needed to finish a log intent item that we recovered from the log. If any + * of the deferred ops operate on an inode, the caller must pass in that inode + * so that the reference can be transferred to the capture structure. The + * caller must hold ILOCK_EXCL on the inode, and must unlock it before calling + * xfs_defer_ops_continue. + */ +int +xfs_defer_ops_capture_and_commit( + struct xfs_trans *tp, + struct list_head *capture_list) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_defer_capture *dfc; + int error; + + /* If we don't capture anything, commit transaction and exit. */ + dfc = xfs_defer_ops_capture(tp); + if (IS_ERR(dfc)) { + xfs_trans_cancel(tp); + return PTR_ERR(dfc); + } + if (!dfc) + return xfs_trans_commit(tp); + + /* Commit the transaction and add the capture structure to the list. */ + error = xfs_trans_commit(tp); + if (error) { + xfs_defer_ops_capture_abort(mp, dfc); + return error; + } + + list_add_tail(&dfc->dfc_list, capture_list); + return 0; +} + +/* + * Attach a chain of captured deferred ops to a new transaction and free the + * capture structure. If an inode was captured, it will be passed back to the + * caller with ILOCK_EXCL held and joined to the transaction with lockflags==0. + * The caller now owns the inode reference. + */ +void +xfs_defer_ops_continue( + struct xfs_defer_capture *dfc, + struct xfs_trans *tp, + struct xfs_defer_resources *dres) +{ + unsigned int i; + + ASSERT(tp->t_flags & XFS_TRANS_PERM_LOG_RES); + ASSERT(!(tp->t_flags & XFS_TRANS_DIRTY)); + + /* Lock the captured resources to the new transaction. */ + if (dfc->dfc_held.dr_inos > 2) { + xfs_sort_inodes(dfc->dfc_held.dr_ip, dfc->dfc_held.dr_inos); + xfs_lock_inodes(dfc->dfc_held.dr_ip, dfc->dfc_held.dr_inos, + XFS_ILOCK_EXCL); + } else if (dfc->dfc_held.dr_inos == 2) + xfs_lock_two_inodes(dfc->dfc_held.dr_ip[0], XFS_ILOCK_EXCL, + dfc->dfc_held.dr_ip[1], XFS_ILOCK_EXCL); + else if (dfc->dfc_held.dr_inos == 1) + xfs_ilock(dfc->dfc_held.dr_ip[0], XFS_ILOCK_EXCL); + + for (i = 0; i < dfc->dfc_held.dr_bufs; i++) + xfs_buf_lock(dfc->dfc_held.dr_bp[i]); + + /* Join the captured resources to the new transaction. */ + xfs_defer_restore_resources(tp, &dfc->dfc_held); + memcpy(dres, &dfc->dfc_held, sizeof(struct xfs_defer_resources)); + dres->dr_bufs = 0; + + /* Move captured dfops chain and state to the transaction. */ + list_splice_init(&dfc->dfc_dfops, &tp->t_dfops); + tp->t_flags |= dfc->dfc_tpflags; + + kfree(dfc); +} + +/* Release the resources captured and continued during recovery. */ +void +xfs_defer_resources_rele( + struct xfs_defer_resources *dres) +{ + unsigned short i; + + for (i = 0; i < dres->dr_inos; i++) { + xfs_iunlock(dres->dr_ip[i], XFS_ILOCK_EXCL); + xfs_irele(dres->dr_ip[i]); + dres->dr_ip[i] = NULL; + } + + for (i = 0; i < dres->dr_bufs; i++) { + xfs_buf_relse(dres->dr_bp[i]); + dres->dr_bp[i] = NULL; + } + + dres->dr_inos = 0; + dres->dr_bufs = 0; + dres->dr_ordered = 0; +} + +static inline int __init +xfs_defer_init_cache(void) +{ + xfs_defer_pending_cache = kmem_cache_create("xfs_defer_pending", + sizeof(struct xfs_defer_pending), + 0, 0, NULL); + + return xfs_defer_pending_cache != NULL ? 0 : -ENOMEM; +} + +static inline void +xfs_defer_destroy_cache(void) +{ + kmem_cache_destroy(xfs_defer_pending_cache); + xfs_defer_pending_cache = NULL; +} + +/* Set up caches for deferred work items. */ +int __init +xfs_defer_init_item_caches(void) +{ + int error; + + error = xfs_defer_init_cache(); + if (error) + return error; + error = xfs_rmap_intent_init_cache(); + if (error) + goto err; + error = xfs_refcount_intent_init_cache(); + if (error) + goto err; + error = xfs_bmap_intent_init_cache(); + if (error) + goto err; + error = xfs_extfree_intent_init_cache(); + if (error) + goto err; + error = xfs_attr_intent_init_cache(); + if (error) + goto err; + error = xfs_exchmaps_intent_init_cache(); + if (error) + goto err; + + return 0; +err: + xfs_defer_destroy_item_caches(); + return error; +} + +/* Destroy all the deferred work item caches, if they've been allocated. */ +void +xfs_defer_destroy_item_caches(void) +{ + xfs_exchmaps_intent_destroy_cache(); + xfs_attr_intent_destroy_cache(); + xfs_extfree_intent_destroy_cache(); + xfs_bmap_intent_destroy_cache(); + xfs_refcount_intent_destroy_cache(); + xfs_rmap_intent_destroy_cache(); + xfs_defer_destroy_cache(); +} + +/* + * Mark a deferred work item so that it will be requeued indefinitely without + * being finished. Caller must ensure there are no data dependencies on this + * work item in the meantime. + */ +void +xfs_defer_item_pause( + struct xfs_trans *tp, + struct xfs_defer_pending *dfp) +{ + ASSERT(!(dfp->dfp_flags & XFS_DEFER_PAUSED)); + + dfp->dfp_flags |= XFS_DEFER_PAUSED; + + trace_xfs_defer_item_pause(tp->t_mountp, dfp); +} + +/* + * Release a paused deferred work item so that it will be finished during the + * next transaction roll. + */ +void +xfs_defer_item_unpause( + struct xfs_trans *tp, + struct xfs_defer_pending *dfp) +{ + ASSERT(dfp->dfp_flags & XFS_DEFER_PAUSED); + + dfp->dfp_flags &= ~XFS_DEFER_PAUSED; + + trace_xfs_defer_item_unpause(tp->t_mountp, dfp); +} diff --git a/fs/xfs/libxfs/xfs_defer.h b/fs/xfs/libxfs/xfs_defer.h new file mode 100644 index 000000000..9effd95dd --- /dev/null +++ b/fs/xfs/libxfs/xfs_defer.h @@ -0,0 +1,165 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright (C) 2016 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_DEFER_H__ +#define __XFS_DEFER_H__ + +struct xfs_btree_cur; +struct xfs_defer_op_type; +struct xfs_defer_capture; + +/* + * Save a log intent item and a list of extents, so that we can replay + * whatever action had to happen to the extent list and file the log done + * item. + */ +struct xfs_defer_pending { + struct list_head dfp_list; /* pending items */ + struct list_head dfp_work; /* work items */ + struct xfs_log_item *dfp_intent; /* log intent item */ + struct xfs_log_item *dfp_done; /* log done item */ + const struct xfs_defer_op_type *dfp_ops; + unsigned int dfp_count; /* # extent items */ + unsigned int dfp_flags; +}; + +/* + * Create a log intent item for this deferred item, but don't actually finish + * the work. Caller must clear this before the final transaction commit. + */ +#define XFS_DEFER_PAUSED (1U << 0) + +#define XFS_DEFER_PENDING_STRINGS \ + { XFS_DEFER_PAUSED, "paused" } + +void xfs_defer_item_pause(struct xfs_trans *tp, struct xfs_defer_pending *dfp); +void xfs_defer_item_unpause(struct xfs_trans *tp, struct xfs_defer_pending *dfp); + +struct xfs_defer_pending *xfs_defer_add(struct xfs_trans *tp, struct list_head *h, + const struct xfs_defer_op_type *ops); +int xfs_defer_finish_noroll(struct xfs_trans **tp); +int xfs_defer_finish(struct xfs_trans **tp); +int xfs_defer_finish_one(struct xfs_trans *tp, struct xfs_defer_pending *dfp); +void xfs_defer_cancel(struct xfs_trans *); +void xfs_defer_move(struct xfs_trans *dtp, struct xfs_trans *stp); + +/* Description of a deferred type. */ +struct xfs_defer_op_type { + const char *name; + unsigned int max_items; + struct xfs_log_item *(*create_intent)(struct xfs_trans *tp, + struct list_head *items, unsigned int count, bool sort); + void (*abort_intent)(struct xfs_log_item *intent); + struct xfs_log_item *(*create_done)(struct xfs_trans *tp, + struct xfs_log_item *intent, unsigned int count); + int (*finish_item)(struct xfs_trans *tp, struct xfs_log_item *done, + struct list_head *item, struct xfs_btree_cur **state); + void (*finish_cleanup)(struct xfs_trans *tp, + struct xfs_btree_cur *state, int error); + void (*cancel_item)(struct list_head *item); + int (*recover_work)(struct xfs_defer_pending *dfp, + struct list_head *capture_list); + struct xfs_log_item *(*relog_intent)(struct xfs_trans *tp, + struct xfs_log_item *intent, + struct xfs_log_item *done_item); +}; + +extern const struct xfs_defer_op_type xfs_bmap_update_defer_type; +extern const struct xfs_defer_op_type xfs_refcount_update_defer_type; +extern const struct xfs_defer_op_type xfs_rtrefcount_update_defer_type; +extern const struct xfs_defer_op_type xfs_rmap_update_defer_type; +extern const struct xfs_defer_op_type xfs_rtrmap_update_defer_type; +extern const struct xfs_defer_op_type xfs_extent_free_defer_type; +extern const struct xfs_defer_op_type xfs_agfl_free_defer_type; +extern const struct xfs_defer_op_type xfs_rtextent_free_defer_type; +extern const struct xfs_defer_op_type xfs_attr_defer_type; +extern const struct xfs_defer_op_type xfs_exchmaps_defer_type; + +/* + * Deferred operation item relogging limits. + */ + +/* + * Rename w/ parent pointers can require up to 5 inodes with deferred ops to + * be joined to the transaction: src_dp, target_dp, src_ip, target_ip, and wip. + * These inodes are locked in sorted order by their inode numbers + */ +#define XFS_DEFER_OPS_NR_INODES 5 +#define XFS_DEFER_OPS_NR_BUFS 2 /* join up to two buffers */ + +/* Resources that must be held across a transaction roll. */ +struct xfs_defer_resources { + /* held buffers */ + struct xfs_buf *dr_bp[XFS_DEFER_OPS_NR_BUFS]; + + /* inodes with no unlock flags */ + struct xfs_inode *dr_ip[XFS_DEFER_OPS_NR_INODES]; + + /* number of held buffers */ + unsigned short dr_bufs; + + /* bitmap of ordered buffers */ + unsigned short dr_ordered; + + /* number of held inodes */ + unsigned short dr_inos; +}; + +/* + * This structure enables a dfops user to detach the chain of deferred + * operations from a transaction so that they can be continued later. + */ +struct xfs_defer_capture { + /* List of other capture structures. */ + struct list_head dfc_list; + + /* Deferred ops state saved from the transaction. */ + struct list_head dfc_dfops; + unsigned int dfc_tpflags; + + /* Block reservations for the data and rt devices. */ + unsigned int dfc_blkres; + unsigned int dfc_rtxres; + + /* Log reservation saved from the transaction. */ + unsigned int dfc_logres; + + struct xfs_defer_resources dfc_held; +}; + +/* + * Functions to capture a chain of deferred operations and continue them later. + * This doesn't normally happen except log recovery. + */ +int xfs_defer_ops_capture_and_commit(struct xfs_trans *tp, + struct list_head *capture_list); +void xfs_defer_ops_continue(struct xfs_defer_capture *d, struct xfs_trans *tp, + struct xfs_defer_resources *dres); +void xfs_defer_ops_capture_abort(struct xfs_mount *mp, + struct xfs_defer_capture *d); +void xfs_defer_resources_rele(struct xfs_defer_resources *dres); + +void xfs_defer_start_recovery(struct xfs_log_item *lip, + struct list_head *r_dfops, const struct xfs_defer_op_type *ops); +void xfs_defer_cancel_recovery(struct xfs_mount *mp, + struct xfs_defer_pending *dfp); +int xfs_defer_finish_recovery(struct xfs_mount *mp, + struct xfs_defer_pending *dfp, struct list_head *capture_list); + +static inline void +xfs_defer_add_item( + struct xfs_defer_pending *dfp, + struct list_head *work) +{ + list_add_tail(work, &dfp->dfp_work); + dfp->dfp_count++; +} + +int __init xfs_defer_init_item_caches(void); +void xfs_defer_destroy_item_caches(void); + +void xfs_defer_add_barrier(struct xfs_trans *tp); + +#endif /* __XFS_DEFER_H__ */ diff --git a/fs/xfs/libxfs/xfs_dir2.c b/fs/xfs/libxfs/xfs_dir2.c new file mode 100644 index 000000000..0c0402a29 --- /dev/null +++ b/fs/xfs/libxfs/xfs_dir2.c @@ -0,0 +1,1411 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2001,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_bmap.h" +#include "xfs_dir2.h" +#include "xfs_dir2_priv.h" +#include "xfs_errortag.h" +#include "xfs_error.h" +#include "xfs_trace.h" +#include "xfs_health.h" +#include "xfs_bmap_btree.h" +#include "xfs_trans_space.h" +#include "xfs_parent.h" +#include "xfs_ag.h" +#include "xfs_ialloc.h" + +const struct xfs_name xfs_name_dotdot = { + .name = (const unsigned char *)"..", + .len = 2, + .type = XFS_DIR3_FT_DIR, +}; + +const struct xfs_name xfs_name_dot = { + .name = (const unsigned char *)".", + .len = 1, + .type = XFS_DIR3_FT_DIR, +}; + +/* + * Convert inode mode to directory entry filetype + */ +unsigned char +xfs_mode_to_ftype( + int mode) +{ + switch (mode & S_IFMT) { + case S_IFREG: + return XFS_DIR3_FT_REG_FILE; + case S_IFDIR: + return XFS_DIR3_FT_DIR; + case S_IFCHR: + return XFS_DIR3_FT_CHRDEV; + case S_IFBLK: + return XFS_DIR3_FT_BLKDEV; + case S_IFIFO: + return XFS_DIR3_FT_FIFO; + case S_IFSOCK: + return XFS_DIR3_FT_SOCK; + case S_IFLNK: + return XFS_DIR3_FT_SYMLINK; + default: + return XFS_DIR3_FT_UNKNOWN; + } +} + +/* + * ASCII case-insensitive (ie. A-Z) support for directories that was + * used in IRIX. + */ +xfs_dahash_t +xfs_ascii_ci_hashname( + const struct xfs_name *name) +{ + xfs_dahash_t hash; + int i; + + for (i = 0, hash = 0; i < name->len; i++) + hash = xfs_ascii_ci_xfrm(name->name[i]) ^ rol32(hash, 7); + + return hash; +} + +enum xfs_dacmp +xfs_ascii_ci_compname( + struct xfs_da_args *args, + const unsigned char *name, + int len) +{ + enum xfs_dacmp result; + int i; + + if (args->namelen != len) + return XFS_CMP_DIFFERENT; + + result = XFS_CMP_EXACT; + for (i = 0; i < len; i++) { + if (args->name[i] == name[i]) + continue; + if (xfs_ascii_ci_xfrm(args->name[i]) != + xfs_ascii_ci_xfrm(name[i])) + return XFS_CMP_DIFFERENT; + result = XFS_CMP_CASE; + } + + return result; +} + +int +xfs_da_mount( + struct xfs_mount *mp) +{ + struct xfs_da_geometry *dageo; + + + ASSERT(mp->m_sb.sb_versionnum & XFS_SB_VERSION_DIRV2BIT); + ASSERT(xfs_dir2_dirblock_bytes(&mp->m_sb) <= XFS_MAX_BLOCKSIZE); + + mp->m_dir_geo = kzalloc_obj(struct xfs_da_geometry, + GFP_KERNEL | __GFP_RETRY_MAYFAIL); + mp->m_attr_geo = kzalloc_obj(struct xfs_da_geometry, + GFP_KERNEL | __GFP_RETRY_MAYFAIL); + if (!mp->m_dir_geo || !mp->m_attr_geo) { + kfree(mp->m_dir_geo); + kfree(mp->m_attr_geo); + return -ENOMEM; + } + + /* set up directory geometry */ + dageo = mp->m_dir_geo; + dageo->blklog = mp->m_sb.sb_blocklog + mp->m_sb.sb_dirblklog; + dageo->fsblog = mp->m_sb.sb_blocklog; + dageo->blksize = xfs_dir2_dirblock_bytes(&mp->m_sb); + dageo->fsbcount = 1 << mp->m_sb.sb_dirblklog; + if (xfs_has_crc(mp)) { + dageo->node_hdr_size = sizeof(struct xfs_da3_node_hdr); + dageo->leaf_hdr_size = sizeof(struct xfs_dir3_leaf_hdr); + dageo->free_hdr_size = sizeof(struct xfs_dir3_free_hdr); + dageo->data_entry_offset = + sizeof(struct xfs_dir3_data_hdr); + } else { + dageo->node_hdr_size = sizeof(struct xfs_da_node_hdr); + dageo->leaf_hdr_size = sizeof(struct xfs_dir2_leaf_hdr); + dageo->free_hdr_size = sizeof(struct xfs_dir2_free_hdr); + dageo->data_entry_offset = + sizeof(struct xfs_dir2_data_hdr); + } + dageo->leaf_max_ents = (dageo->blksize - dageo->leaf_hdr_size) / + sizeof(struct xfs_dir2_leaf_entry); + dageo->free_max_bests = (dageo->blksize - dageo->free_hdr_size) / + sizeof(xfs_dir2_data_off_t); + + dageo->data_first_offset = dageo->data_entry_offset + + xfs_dir2_data_entsize(mp, 1) + + xfs_dir2_data_entsize(mp, 2); + + /* + * Now we've set up the block conversion variables, we can calculate the + * segment block constants using the geometry structure. + */ + dageo->datablk = xfs_dir2_byte_to_da(dageo, XFS_DIR2_DATA_OFFSET); + dageo->leafblk = xfs_dir2_byte_to_da(dageo, XFS_DIR2_LEAF_OFFSET); + dageo->freeblk = xfs_dir2_byte_to_da(dageo, XFS_DIR2_FREE_OFFSET); + dageo->node_ents = (dageo->blksize - dageo->node_hdr_size) / + (uint)sizeof(xfs_da_node_entry_t); + dageo->max_extents = (XFS_DIR2_MAX_SPACES * XFS_DIR2_SPACE_SIZE) >> + mp->m_sb.sb_blocklog; + dageo->magicpct = (dageo->blksize * 37) / 100; + + /* set up attribute geometry - single fsb only */ + dageo = mp->m_attr_geo; + dageo->blklog = mp->m_sb.sb_blocklog; + dageo->fsblog = mp->m_sb.sb_blocklog; + dageo->blksize = 1 << dageo->blklog; + dageo->fsbcount = 1; + dageo->node_hdr_size = mp->m_dir_geo->node_hdr_size; + dageo->node_ents = (dageo->blksize - dageo->node_hdr_size) / + (uint)sizeof(xfs_da_node_entry_t); + + if (xfs_has_large_extent_counts(mp)) + dageo->max_extents = XFS_MAX_EXTCNT_ATTR_FORK_LARGE; + else + dageo->max_extents = XFS_MAX_EXTCNT_ATTR_FORK_SMALL; + + dageo->magicpct = (dageo->blksize * 37) / 100; + return 0; +} + +void +xfs_da_unmount( + struct xfs_mount *mp) +{ + kfree(mp->m_dir_geo); + kfree(mp->m_attr_geo); +} + +/* + * Return 1 if directory contains only "." and "..". + */ +static bool +xfs_dir_isempty( + xfs_inode_t *dp) +{ + xfs_dir2_sf_hdr_t *sfp; + + ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); + if (dp->i_disk_size == 0) /* might happen during shutdown. */ + return true; + if (dp->i_disk_size > xfs_inode_data_fork_size(dp)) + return false; + sfp = dp->i_df.if_data; + return !sfp->count; +} + +/* + * Validate a given inode number. + */ +int +xfs_dir_ino_validate( + xfs_mount_t *mp, + xfs_ino_t ino) +{ + bool ino_ok = xfs_verify_dir_ino(mp, ino); + + if (XFS_IS_CORRUPT(mp, !ino_ok) || + XFS_TEST_ERROR(mp, XFS_ERRTAG_DIR_INO_VALIDATE)) { + xfs_warn(mp, "Invalid inode number 0x%Lx", + (unsigned long long) ino); + return -EFSCORRUPTED; + } + return 0; +} + +/* + * Initialize a directory with its "." and ".." entries. + */ +int +xfs_dir_init( + xfs_trans_t *tp, + xfs_inode_t *dp, + xfs_inode_t *pdp) +{ + struct xfs_da_args *args; + int error; + + ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); + error = xfs_dir_ino_validate(tp->t_mountp, I_INO(pdp)); + if (error) + return error; + + args = kzalloc_obj(*args, GFP_KERNEL | __GFP_NOFAIL); + if (!args) + return -ENOMEM; + + args->geo = dp->i_mount->m_dir_geo; + args->dp = dp; + args->trans = tp; + args->owner = I_INO(dp); + error = xfs_dir2_sf_create(args, I_INO(pdp)); + kfree(args); + return error; +} + +enum xfs_dir2_fmt +xfs_dir2_format( + struct xfs_da_args *args, + int *error) +{ + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + struct xfs_da_geometry *geo = mp->m_dir_geo; + xfs_fileoff_t eof; + + xfs_assert_ilocked(dp, XFS_ILOCK_SHARED | XFS_ILOCK_EXCL); + + *error = 0; + if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) + return XFS_DIR2_FMT_SF; + + *error = xfs_bmap_last_offset(dp, &eof, XFS_DATA_FORK); + if (*error) + return XFS_DIR2_FMT_ERROR; + + if (eof == XFS_B_TO_FSB(mp, geo->blksize)) { + if (XFS_IS_CORRUPT(mp, dp->i_disk_size != geo->blksize)) { + xfs_da_mark_sick(args); + *error = -EFSCORRUPTED; + return XFS_DIR2_FMT_ERROR; + } + return XFS_DIR2_FMT_BLOCK; + } + if (eof == geo->leafblk + geo->fsbcount) + return XFS_DIR2_FMT_LEAF; + return XFS_DIR2_FMT_NODE; +} + +int +xfs_dir_createname_args( + struct xfs_da_args *args) +{ + int error; + + if (!args->inumber) + args->op_flags |= XFS_DA_OP_JUSTCHECK; + + switch (xfs_dir2_format(args, &error)) { + case XFS_DIR2_FMT_SF: + return xfs_dir2_sf_addname(args); + case XFS_DIR2_FMT_BLOCK: + return xfs_dir2_block_addname(args); + case XFS_DIR2_FMT_LEAF: + return xfs_dir2_leaf_addname(args); + case XFS_DIR2_FMT_NODE: + return xfs_dir2_node_addname(args); + default: + return error; + } +} + +/* + * Enter a name in a directory, or check for available space. + * If inum is 0, only the available space test is performed. + */ +int +xfs_dir_createname( + struct xfs_trans *tp, + struct xfs_inode *dp, + const struct xfs_name *name, + xfs_ino_t inum, /* new entry inode number */ + xfs_extlen_t total) /* bmap's total block count */ +{ + struct xfs_da_args *args; + int rval; + + ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); + + if (inum) { + rval = xfs_dir_ino_validate(tp->t_mountp, inum); + if (rval) + return rval; + XFS_STATS_INC(dp->i_mount, xs_dir_create); + } + + args = kzalloc_obj(*args, GFP_KERNEL | __GFP_NOFAIL); + if (!args) + return -ENOMEM; + + args->geo = dp->i_mount->m_dir_geo; + args->name = name->name; + args->namelen = name->len; + args->filetype = name->type; + args->hashval = xfs_dir2_hashname(dp->i_mount, name); + args->inumber = inum; + args->dp = dp; + args->total = total; + args->whichfork = XFS_DATA_FORK; + args->trans = tp; + args->op_flags = XFS_DA_OP_ADDNAME | XFS_DA_OP_OKNOENT; + args->owner = I_INO(dp); + + rval = xfs_dir_createname_args(args); + kfree(args); + return rval; +} + +/* + * If doing a CI lookup and case-insensitive match, dup actual name into + * args.value. Return EEXIST for success (ie. name found) or an error. + */ +int +xfs_dir_cilookup_result( + struct xfs_da_args *args, + const unsigned char *name, + int len) +{ + if (args->cmpresult == XFS_CMP_DIFFERENT) + return -ENOENT; + if (args->cmpresult != XFS_CMP_CASE || + !(args->op_flags & XFS_DA_OP_CILOOKUP)) + return -EEXIST; + + args->value = kmemdup(name, len, + GFP_KERNEL | __GFP_NOLOCKDEP | __GFP_RETRY_MAYFAIL); + if (!args->value) + return -ENOMEM; + + args->valuelen = len; + return -EEXIST; +} + +int +xfs_dir_lookup_args( + struct xfs_da_args *args) +{ + int error; + + switch (xfs_dir2_format(args, &error)) { + case XFS_DIR2_FMT_SF: + error = xfs_dir2_sf_lookup(args); + break; + case XFS_DIR2_FMT_BLOCK: + error = xfs_dir2_block_lookup(args); + break; + case XFS_DIR2_FMT_LEAF: + error = xfs_dir2_leaf_lookup(args); + break; + case XFS_DIR2_FMT_NODE: + error = xfs_dir2_node_lookup(args); + break; + default: + break; + } + + if (error != -EEXIST) + return error; + return 0; +} + +/* + * Lookup a name in a directory, give back the inode number. + * If ci_name is not NULL, returns the actual name in ci_name if it differs + * to name, or ci_name->name is set to NULL for an exact match. + */ + +int +xfs_dir_lookup( + struct xfs_trans *tp, + struct xfs_inode *dp, + const struct xfs_name *name, + xfs_ino_t *inum, /* out: inode number */ + struct xfs_name *ci_name) /* out: actual name if CI match */ +{ + struct xfs_da_args *args; + int rval; + int lock_mode; + + ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); + XFS_STATS_INC(dp->i_mount, xs_dir_lookup); + + args = kzalloc_obj(*args, GFP_KERNEL | __GFP_NOLOCKDEP | __GFP_NOFAIL); + args->geo = dp->i_mount->m_dir_geo; + args->name = name->name; + args->namelen = name->len; + args->filetype = name->type; + args->hashval = xfs_dir2_hashname(dp->i_mount, name); + args->dp = dp; + args->whichfork = XFS_DATA_FORK; + args->trans = tp; + args->op_flags = XFS_DA_OP_OKNOENT; + args->owner = I_INO(dp); + if (ci_name) + args->op_flags |= XFS_DA_OP_CILOOKUP; + + lock_mode = xfs_ilock_data_map_shared(dp); + rval = xfs_dir_lookup_args(args); + if (!rval) { + *inum = args->inumber; + if (ci_name) { + ci_name->name = args->value; + ci_name->len = args->valuelen; + } + } + xfs_iunlock(dp, lock_mode); + kfree(args); + return rval; +} + +int +xfs_dir_removename_args( + struct xfs_da_args *args) +{ + int error; + + switch (xfs_dir2_format(args, &error)) { + case XFS_DIR2_FMT_SF: + return xfs_dir2_sf_removename(args); + case XFS_DIR2_FMT_BLOCK: + return xfs_dir2_block_removename(args); + case XFS_DIR2_FMT_LEAF: + return xfs_dir2_leaf_removename(args); + case XFS_DIR2_FMT_NODE: + return xfs_dir2_node_removename(args); + default: + return error; + } +} + +/* + * Remove an entry from a directory. + */ +int +xfs_dir_removename( + struct xfs_trans *tp, + struct xfs_inode *dp, + const struct xfs_name *name, + xfs_ino_t ino, + xfs_extlen_t total) /* bmap's total block count */ +{ + struct xfs_da_args *args; + int rval; + + ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); + XFS_STATS_INC(dp->i_mount, xs_dir_remove); + + args = kzalloc_obj(*args, GFP_KERNEL | __GFP_NOFAIL); + if (!args) + return -ENOMEM; + + args->geo = dp->i_mount->m_dir_geo; + args->name = name->name; + args->namelen = name->len; + args->filetype = name->type; + args->hashval = xfs_dir2_hashname(dp->i_mount, name); + args->inumber = ino; + args->dp = dp; + args->total = total; + args->whichfork = XFS_DATA_FORK; + args->trans = tp; + args->owner = I_INO(dp); + rval = xfs_dir_removename_args(args); + kfree(args); + return rval; +} + +int +xfs_dir_replace_args( + struct xfs_da_args *args) +{ + int error; + + switch (xfs_dir2_format(args, &error)) { + case XFS_DIR2_FMT_SF: + return xfs_dir2_sf_replace(args); + case XFS_DIR2_FMT_BLOCK: + return xfs_dir2_block_replace(args); + case XFS_DIR2_FMT_LEAF: + return xfs_dir2_leaf_replace(args); + case XFS_DIR2_FMT_NODE: + return xfs_dir2_node_replace(args); + default: + return error; + } +} + +/* + * Replace the inode number of a directory entry. + */ +int +xfs_dir_replace( + struct xfs_trans *tp, + struct xfs_inode *dp, + const struct xfs_name *name, /* name of entry to replace */ + xfs_ino_t inum, /* new inode number */ + xfs_extlen_t total) /* bmap's total block count */ +{ + struct xfs_da_args *args; + int rval; + + ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); + + rval = xfs_dir_ino_validate(tp->t_mountp, inum); + if (rval) + return rval; + + args = kzalloc_obj(*args, GFP_KERNEL | __GFP_NOFAIL); + if (!args) + return -ENOMEM; + + args->geo = dp->i_mount->m_dir_geo; + args->name = name->name; + args->namelen = name->len; + args->filetype = name->type; + args->hashval = xfs_dir2_hashname(dp->i_mount, name); + args->inumber = inum; + args->dp = dp; + args->total = total; + args->whichfork = XFS_DATA_FORK; + args->trans = tp; + args->owner = I_INO(dp); + rval = xfs_dir_replace_args(args); + kfree(args); + return rval; +} + +/* + * See if this entry can be added to the directory without allocating space. + */ +int +xfs_dir_canenter( + struct xfs_trans *tp, + struct xfs_inode *dp, + const struct xfs_name *name) /* name of entry to add */ +{ + return xfs_dir_createname(tp, dp, name, 0, 0); +} + +/* + * Utility routines. + */ + +/* + * Add a block to the directory. + * + * This routine is for data and free blocks, not leaf/node blocks which are + * handled by xfs_da_grow_inode. + */ +int +xfs_dir2_grow_inode( + struct xfs_da_args *args, + int space, /* v2 dir's space XFS_DIR2_xxx_SPACE */ + xfs_dir2_db_t *dbp) /* out: block number added */ +{ + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + xfs_fileoff_t bno; /* directory offset of new block */ + int count; /* count of filesystem blocks */ + int error; + + trace_xfs_dir2_grow_inode(args, space); + + /* + * Set lowest possible block in the space requested. + */ + bno = XFS_B_TO_FSBT(mp, space * XFS_DIR2_SPACE_SIZE); + count = args->geo->fsbcount; + + error = xfs_da_grow_inode_int(args, &bno, count); + if (error) + return error; + + *dbp = xfs_dir2_da_to_db(args->geo, (xfs_dablk_t)bno); + + /* + * Update file's size if this is the data space and it grew. + */ + if (space == XFS_DIR2_DATA_SPACE) { + xfs_fsize_t size; /* directory file (data) size */ + + size = XFS_FSB_TO_B(mp, bno + count); + if (size > dp->i_disk_size) { + dp->i_disk_size = size; + xfs_trans_log_inode(args->trans, dp, XFS_ILOG_CORE); + } + } + return 0; +} + +/* + * Remove the given block from the directory. + * This routine is used for data and free blocks, leaf/node are done + * by xfs_da_shrink_inode. + */ +int +xfs_dir2_shrink_inode( + struct xfs_da_args *args, + xfs_dir2_db_t db, + struct xfs_buf *bp) +{ + xfs_fileoff_t bno; /* directory file offset */ + xfs_dablk_t da; /* directory file offset */ + int done; /* bunmap is finished */ + struct xfs_inode *dp; + int error; + struct xfs_mount *mp; + struct xfs_trans *tp; + + trace_xfs_dir2_shrink_inode(args, db); + + dp = args->dp; + mp = dp->i_mount; + tp = args->trans; + da = xfs_dir2_db_to_da(args->geo, db); + + /* Unmap the fsblock(s). */ + error = xfs_bunmapi(tp, dp, da, args->geo->fsbcount, 0, 0, &done); + if (error) { + /* + * ENOSPC actually can happen if we're in a removename with no + * space reservation, and the resulting block removal would + * cause a bmap btree split or conversion from extents to btree. + * This can only happen for un-fragmented directory blocks, + * since you need to be punching out the middle of an extent. + * In this case we need to leave the block in the file, and not + * binval it. So the block has to be in a consistent empty + * state and appropriately logged. We don't free up the buffer, + * the caller can tell it hasn't happened since it got an error + * back. + */ + return error; + } + ASSERT(done); + /* + * Invalidate the buffer from the transaction. + */ + xfs_trans_binval(tp, bp); + /* + * If it's not a data block, we're done. + */ + if (db >= xfs_dir2_byte_to_db(args->geo, XFS_DIR2_LEAF_OFFSET)) + return 0; + /* + * If the block isn't the last one in the directory, we're done. + */ + if (dp->i_disk_size > xfs_dir2_db_off_to_byte(args->geo, db + 1, 0)) + return 0; + bno = da; + if ((error = xfs_bmap_last_before(tp, dp, &bno, XFS_DATA_FORK))) { + /* + * This can't really happen unless there's kernel corruption. + */ + return error; + } + if (db == args->geo->datablk) + ASSERT(bno == 0); + else + ASSERT(bno > 0); + /* + * Set the size to the new last block. + */ + dp->i_disk_size = XFS_FSB_TO_B(mp, bno); + xfs_trans_log_inode(tp, dp, XFS_ILOG_CORE); + return 0; +} + +/* Returns true if the directory entry name is valid. */ +bool +xfs_dir2_namecheck( + const void *name, + size_t length) +{ + /* + * MAXNAMELEN includes the trailing null, but (name/length) leave it + * out, so use >= for the length check. + */ + if (length >= MAXNAMELEN) + return false; + + /* There shouldn't be any slashes or nulls here */ + return !memchr(name, '/', length) && !memchr(name, 0, length); +} + +xfs_dahash_t +xfs_dir2_hashname( + struct xfs_mount *mp, + const struct xfs_name *name) +{ + if (unlikely(xfs_has_asciici(mp))) + return xfs_ascii_ci_hashname(name); + return xfs_da_hashname(name->name, name->len); +} + +enum xfs_dacmp +xfs_dir2_compname( + struct xfs_da_args *args, + const unsigned char *name, + int len) +{ + if (unlikely(xfs_has_asciici(args->dp->i_mount))) + return xfs_ascii_ci_compname(args, name, len); + return xfs_da_compname(args, name, len); +} + +#ifdef CONFIG_XFS_LIVE_HOOKS +/* + * Use a static key here to reduce the overhead of directory live update hooks. + * If the compiler supports jump labels, the static branch will be replaced by + * a nop sled when there are no hook users. Online fsck is currently the only + * caller, so this is a reasonable tradeoff. + * + * Note: Patching the kernel code requires taking the cpu hotplug lock. Other + * parts of the kernel allocate memory with that lock held, which means that + * XFS callers cannot hold any locks that might be used by memory reclaim or + * writeback when calling the static_branch_{inc,dec} functions. + */ +DEFINE_STATIC_XFS_HOOK_SWITCH(xfs_dir_hooks_switch); + +void +xfs_dir_hook_disable(void) +{ + xfs_hooks_switch_off(&xfs_dir_hooks_switch); +} + +void +xfs_dir_hook_enable(void) +{ + xfs_hooks_switch_on(&xfs_dir_hooks_switch); +} + +/* Call hooks for a directory update relating to a child dirent update. */ +inline void +xfs_dir_update_hook( + struct xfs_inode *dp, + struct xfs_inode *ip, + int delta, + const struct xfs_name *name) +{ + if (xfs_hooks_switched_on(&xfs_dir_hooks_switch)) { + struct xfs_dir_update_params p = { + .dp = dp, + .ip = ip, + .delta = delta, + .name = name, + }; + struct xfs_mount *mp = ip->i_mount; + + xfs_hooks_call(&mp->m_dir_update_hooks, 0, &p); + } +} + +/* Call the specified function during a directory update. */ +int +xfs_dir_hook_add( + struct xfs_mount *mp, + struct xfs_dir_hook *hook) +{ + return xfs_hooks_add(&mp->m_dir_update_hooks, &hook->dirent_hook); +} + +/* Stop calling the specified function during a directory update. */ +void +xfs_dir_hook_del( + struct xfs_mount *mp, + struct xfs_dir_hook *hook) +{ + xfs_hooks_del(&mp->m_dir_update_hooks, &hook->dirent_hook); +} + +/* Configure directory update hook functions. */ +void +xfs_dir_hook_setup( + struct xfs_dir_hook *hook, + notifier_fn_t mod_fn) +{ + xfs_hook_setup(&hook->dirent_hook, mod_fn); +} +#endif /* CONFIG_XFS_LIVE_HOOKS */ + +/* + * Given a directory @dp, a newly allocated inode @ip, and a @name, link @ip + * into @dp under the given @name. If @ip is a directory, it will be + * initialized. Both inodes must have the ILOCK held and the transaction must + * have sufficient blocks reserved. + */ +int +xfs_dir_create_child( + struct xfs_trans *tp, + unsigned int resblks, + struct xfs_dir_update *du) +{ + struct xfs_inode *dp = du->dp; + const struct xfs_name *name = du->name; + struct xfs_inode *ip = du->ip; + int error; + + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + xfs_assert_ilocked(dp, XFS_ILOCK_EXCL); + + error = xfs_dir_createname(tp, dp, name, I_INO(ip), resblks); + if (error) { + ASSERT(error != -ENOSPC); + return error; + } + + xfs_trans_ichgtime(tp, dp, XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG); + xfs_trans_log_inode(tp, dp, XFS_ILOG_CORE); + + if (S_ISDIR(VFS_I(ip)->i_mode)) { + error = xfs_dir_init(tp, ip, dp); + if (error) + return error; + + xfs_bumplink(tp, dp); + } + + /* + * If we have parent pointers, we need to add the attribute containing + * the parent information now. + */ + if (du->ppargs) { + error = xfs_parent_addname(tp, du->ppargs, dp, name, ip); + if (error) + return error; + } + + xfs_dir_update_hook(dp, ip, 1, name); + return 0; +} + +/* + * Given a directory @dp, an existing non-directory inode @ip, and a @name, + * link @ip into @dp under the given @name. Both inodes must have the ILOCK + * held. + */ +int +xfs_dir_add_child( + struct xfs_trans *tp, + unsigned int resblks, + struct xfs_dir_update *du) +{ + struct xfs_inode *dp = du->dp; + const struct xfs_name *name = du->name; + struct xfs_inode *ip = du->ip; + struct xfs_mount *mp = tp->t_mountp; + int error; + + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + xfs_assert_ilocked(dp, XFS_ILOCK_EXCL); + ASSERT(!S_ISDIR(VFS_I(ip)->i_mode)); + + if (!resblks) { + error = xfs_dir_canenter(tp, dp, name); + if (error) + return error; + } + + /* + * Handle initial link state of O_TMPFILE inode + */ + if (VFS_I(ip)->i_nlink == 0) { + struct xfs_perag *pag; + + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(ip)); + error = xfs_iunlink_remove(tp, pag, ip); + xfs_perag_put(pag); + if (error) + return error; + } + + error = xfs_dir_createname(tp, dp, name, I_INO(ip), resblks); + if (error) + return error; + + xfs_trans_ichgtime(tp, dp, XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG); + xfs_trans_log_inode(tp, dp, XFS_ILOG_CORE); + + xfs_bumplink(tp, ip); + + /* + * If we have parent pointers, we now need to add the parent record to + * the attribute fork of the inode. If this is the initial parent + * attribute, we need to create it correctly, otherwise we can just add + * the parent to the inode. + */ + if (du->ppargs) { + error = xfs_parent_addname(tp, du->ppargs, dp, name, ip); + if (error) + return error; + } + + xfs_dir_update_hook(dp, ip, 1, name); + return 0; +} + +/* + * Given a directory @dp, a child @ip, and a @name, remove the (@name, @ip) + * entry from the directory. Both inodes must have the ILOCK held. + */ +int +xfs_dir_remove_child( + struct xfs_trans *tp, + unsigned int resblks, + struct xfs_dir_update *du) +{ + struct xfs_inode *dp = du->dp; + const struct xfs_name *name = du->name; + struct xfs_inode *ip = du->ip; + int error; + + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + xfs_assert_ilocked(dp, XFS_ILOCK_EXCL); + + /* + * If we're removing a directory perform some additional validation. + */ + if (S_ISDIR(VFS_I(ip)->i_mode)) { + ASSERT(VFS_I(ip)->i_nlink >= 2); + if (VFS_I(ip)->i_nlink != 2) + return -ENOTEMPTY; + if (!xfs_dir_isempty(ip)) + return -ENOTEMPTY; + + /* Drop the link from ip's "..". */ + error = xfs_droplink(tp, dp); + if (error) + return error; + + /* Drop the "." link from ip to self. */ + error = xfs_droplink(tp, ip); + if (error) + return error; + + /* + * Point the unlinked child directory's ".." entry to the root + * directory to eliminate back-references to inodes that may + * get freed before the child directory is closed. If the fs + * gets shrunk, this can lead to dirent inode validation errors. + */ + if (I_INO(dp) != tp->t_mountp->m_sb.sb_rootino) { + error = xfs_dir_replace(tp, ip, &xfs_name_dotdot, + tp->t_mountp->m_sb.sb_rootino, 0); + if (error) + return error; + } + } else { + /* + * When removing a non-directory we need to log the parent + * inode here. For a directory this is done implicitly + * by the xfs_droplink call for the ".." entry. + */ + xfs_trans_log_inode(tp, dp, XFS_ILOG_CORE); + } + xfs_trans_ichgtime(tp, dp, XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG); + + /* Drop the link from dp to ip. */ + error = xfs_droplink(tp, ip); + if (error) + return error; + + error = xfs_dir_removename(tp, dp, name, I_INO(ip), resblks); + if (error) { + ASSERT(error != -ENOENT); + return error; + } + + /* Remove parent pointer. */ + if (du->ppargs) { + error = xfs_parent_removename(tp, du->ppargs, dp, name, ip); + if (error) + return error; + } + + xfs_dir_update_hook(dp, ip, -1, name); + return 0; +} + +/* + * Exchange the entry (@name1, @ip1) in directory @dp1 with the entry (@name2, + * @ip2) in directory @dp2, and update '..' @ip1 and @ip2's entries as needed. + * @ip1 and @ip2 need not be of the same type. + * + * All inodes must have the ILOCK held, and both entries must already exist. + */ +int +xfs_dir_exchange_children( + struct xfs_trans *tp, + struct xfs_dir_update *du1, + struct xfs_dir_update *du2, + unsigned int spaceres) +{ + struct xfs_inode *dp1 = du1->dp; + const struct xfs_name *name1 = du1->name; + struct xfs_inode *ip1 = du1->ip; + struct xfs_inode *dp2 = du2->dp; + const struct xfs_name *name2 = du2->name; + struct xfs_inode *ip2 = du2->ip; + int ip1_flags = 0; + int ip2_flags = 0; + int dp2_flags = 0; + int error; + + /* Swap inode number for dirent in first parent */ + error = xfs_dir_replace(tp, dp1, name1, I_INO(ip2), spaceres); + if (error) + return error; + + /* Swap inode number for dirent in second parent */ + error = xfs_dir_replace(tp, dp2, name2, I_INO(ip1), spaceres); + if (error) + return error; + + /* + * If we're renaming one or more directories across different parents, + * update the respective ".." entries (and link counts) to match the new + * parents. + */ + if (dp1 != dp2) { + dp2_flags = XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG; + + if (S_ISDIR(VFS_I(ip2)->i_mode)) { + error = xfs_dir_replace(tp, ip2, &xfs_name_dotdot, + I_INO(dp1), spaceres); + if (error) + return error; + + /* transfer ip2 ".." reference to dp1 */ + if (!S_ISDIR(VFS_I(ip1)->i_mode)) { + error = xfs_droplink(tp, dp2); + if (error) + return error; + xfs_bumplink(tp, dp1); + } + + /* + * Although ip1 isn't changed here, userspace needs + * to be warned about the change, so that applications + * relying on it (like backup ones), will properly + * notify the change + */ + ip1_flags |= XFS_ICHGTIME_CHG; + ip2_flags |= XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG; + } + + if (S_ISDIR(VFS_I(ip1)->i_mode)) { + error = xfs_dir_replace(tp, ip1, &xfs_name_dotdot, + I_INO(dp2), spaceres); + if (error) + return error; + + /* transfer ip1 ".." reference to dp2 */ + if (!S_ISDIR(VFS_I(ip2)->i_mode)) { + error = xfs_droplink(tp, dp1); + if (error) + return error; + xfs_bumplink(tp, dp2); + } + + /* + * Although ip2 isn't changed here, userspace needs + * to be warned about the change, so that applications + * relying on it (like backup ones), will properly + * notify the change + */ + ip1_flags |= XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG; + ip2_flags |= XFS_ICHGTIME_CHG; + } + } + + if (ip1_flags) { + xfs_trans_ichgtime(tp, ip1, ip1_flags); + xfs_trans_log_inode(tp, ip1, XFS_ILOG_CORE); + } + if (ip2_flags) { + xfs_trans_ichgtime(tp, ip2, ip2_flags); + xfs_trans_log_inode(tp, ip2, XFS_ILOG_CORE); + } + if (dp2_flags) { + xfs_trans_ichgtime(tp, dp2, dp2_flags); + xfs_trans_log_inode(tp, dp2, XFS_ILOG_CORE); + } + xfs_trans_ichgtime(tp, dp1, XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG); + xfs_trans_log_inode(tp, dp1, XFS_ILOG_CORE); + + /* Schedule parent pointer replacements */ + if (du1->ppargs) { + error = xfs_parent_replacename(tp, du1->ppargs, dp1, name1, + dp2, name2, ip1); + if (error) + return error; + } + + if (du2->ppargs) { + error = xfs_parent_replacename(tp, du2->ppargs, dp2, name2, + dp1, name1, ip2); + if (error) + return error; + } + + /* + * Inform our hook clients that we've finished an exchange operation as + * follows: removed the source and target files from their directories; + * added the target to the source directory; and added the source to + * the target directory. All inodes are locked, so it's ok to model a + * rename this way so long as we say we deleted entries before we add + * new ones. + */ + xfs_dir_update_hook(dp1, ip1, -1, name1); + xfs_dir_update_hook(dp2, ip2, -1, name2); + xfs_dir_update_hook(dp1, ip2, 1, name1); + xfs_dir_update_hook(dp2, ip1, 1, name2); + return 0; +} + +/* + * Given an entry (@src_name, @src_ip) in directory @src_dp, make the entry + * @target_name in directory @target_dp point to @src_ip and remove the + * original entry, cleaning up everything left behind. + * + * Cleanup involves dropping a link count on @target_ip, and either removing + * the (@src_name, @src_ip) entry from @src_dp or simply replacing the entry + * with (@src_name, @wip) if a whiteout inode @wip is supplied. + * + * All inodes must have the ILOCK held. We assume that if @src_ip is a + * directory then its '..' doesn't already point to @target_dp, and that @wip + * is a freshly allocated whiteout. + */ +int +xfs_dir_rename_children( + struct xfs_trans *tp, + struct xfs_dir_update *du_src, + struct xfs_dir_update *du_tgt, + unsigned int spaceres, + struct xfs_dir_update *du_wip) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_inode *src_dp = du_src->dp; + const struct xfs_name *src_name = du_src->name; + struct xfs_inode *src_ip = du_src->ip; + struct xfs_inode *target_dp = du_tgt->dp; + const struct xfs_name *target_name = du_tgt->name; + struct xfs_inode *target_ip = du_tgt->ip; + bool new_parent = (src_dp != target_dp); + bool src_is_directory; + int error; + + src_is_directory = S_ISDIR(VFS_I(src_ip)->i_mode); + + /* + * Check for expected errors before we dirty the transaction + * so we can return an error without a transaction abort. + */ + if (target_ip == NULL) { + /* + * If there's no space reservation, check the entry will + * fit before actually inserting it. + */ + if (!spaceres) { + error = xfs_dir_canenter(tp, target_dp, target_name); + if (error) + return error; + } + } else { + /* + * If target exists and it's a directory, check that whether + * it can be destroyed. + */ + if (S_ISDIR(VFS_I(target_ip)->i_mode) && + (!xfs_dir_isempty(target_ip) || + (VFS_I(target_ip)->i_nlink > 2))) + return -EEXIST; + } + + /* + * Directory entry creation below may acquire the AGF. Remove + * the whiteout from the unlinked list first to preserve correct + * AGI/AGF locking order. This dirties the transaction so failures + * after this point will abort and log recovery will clean up the + * mess. + * + * For whiteouts, we need to bump the link count on the whiteout + * inode. After this point, we have a real link, clear the tmpfile + * state flag from the inode so it doesn't accidentally get misused + * in future. + */ + if (du_wip->ip) { + struct xfs_perag *pag; + + ASSERT(VFS_I(du_wip->ip)->i_nlink == 0); + + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(du_wip->ip)); + error = xfs_iunlink_remove(tp, pag, du_wip->ip); + xfs_perag_put(pag); + if (error) + return error; + + xfs_bumplink(tp, du_wip->ip); + } + + /* + * Set up the target. + */ + if (target_ip == NULL) { + /* + * If target does not exist and the rename crosses + * directories, adjust the target directory link count + * to account for the ".." reference from the new entry. + */ + error = xfs_dir_createname(tp, target_dp, target_name, + I_INO(src_ip), spaceres); + if (error) + return error; + + xfs_trans_ichgtime(tp, target_dp, + XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG); + + if (new_parent && src_is_directory) { + xfs_bumplink(tp, target_dp); + } + } else { /* target_ip != NULL */ + /* + * Link the source inode under the target name. + * If the source inode is a directory and we are moving + * it across directories, its ".." entry will be + * inconsistent until we replace that down below. + * + * In case there is already an entry with the same + * name at the destination directory, remove it first. + */ + error = xfs_dir_replace(tp, target_dp, target_name, + I_INO(src_ip), spaceres); + if (error) + return error; + + xfs_trans_ichgtime(tp, target_dp, + XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG); + + /* + * Decrement the link count on the target since the target + * dir no longer points to it. + */ + error = xfs_droplink(tp, target_ip); + if (error) + return error; + + if (src_is_directory) { + /* + * Drop the link from the old "." entry. + */ + error = xfs_droplink(tp, target_ip); + if (error) + return error; + } + } /* target_ip != NULL */ + + /* + * Remove the source. + */ + if (new_parent && src_is_directory) { + /* + * Rewrite the ".." entry to point to the new + * directory. + */ + error = xfs_dir_replace(tp, src_ip, &xfs_name_dotdot, + I_INO(target_dp), spaceres); + ASSERT(error != -EEXIST); + if (error) + return error; + } + + /* + * We always want to hit the ctime on the source inode. + * + * This isn't strictly required by the standards since the source + * inode isn't really being changed, but old unix file systems did + * it and some incremental backup programs won't work without it. + */ + xfs_trans_ichgtime(tp, src_ip, XFS_ICHGTIME_CHG); + xfs_trans_log_inode(tp, src_ip, XFS_ILOG_CORE); + + /* + * Adjust the link count on src_dp. This is necessary when + * renaming a directory, either within one parent when + * the target existed, or across two parent directories. + */ + if (src_is_directory && (new_parent || target_ip != NULL)) { + + /* + * Decrement link count on src_directory since the + * entry that's moved no longer points to it. + */ + error = xfs_droplink(tp, src_dp); + if (error) + return error; + } + + /* + * For whiteouts, we only need to update the source dirent with the + * inode number of the whiteout inode rather than removing it + * altogether. + */ + if (du_wip->ip) + error = xfs_dir_replace(tp, src_dp, src_name, I_INO(du_wip->ip), + spaceres); + else + error = xfs_dir_removename(tp, src_dp, src_name, I_INO(src_ip), + spaceres); + if (error) + return error; + + xfs_trans_ichgtime(tp, src_dp, XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG); + xfs_trans_log_inode(tp, src_dp, XFS_ILOG_CORE); + if (new_parent) + xfs_trans_log_inode(tp, target_dp, XFS_ILOG_CORE); + + /* Schedule parent pointer updates. */ + if (du_wip->ppargs) { + error = xfs_parent_addname(tp, du_wip->ppargs, src_dp, + src_name, du_wip->ip); + if (error) + return error; + } + + if (du_src->ppargs) { + error = xfs_parent_replacename(tp, du_src->ppargs, src_dp, + src_name, target_dp, target_name, src_ip); + if (error) + return error; + } + + if (du_tgt->ppargs) { + error = xfs_parent_removename(tp, du_tgt->ppargs, target_dp, + target_name, target_ip); + if (error) + return error; + } + + /* + * Inform our hook clients that we've finished a rename operation as + * follows: removed the source and target files from their directories; + * that we've added the source to the target directory; and finally + * that we've added the whiteout, if there was one. All inodes are + * locked, so it's ok to model a rename this way so long as we say we + * deleted entries before we add new ones. + */ + if (target_ip) + xfs_dir_update_hook(target_dp, target_ip, -1, target_name); + xfs_dir_update_hook(src_dp, src_ip, -1, src_name); + xfs_dir_update_hook(target_dp, src_ip, 1, target_name); + if (du_wip->ip) + xfs_dir_update_hook(src_dp, du_wip->ip, 1, src_name); + return 0; +} diff --git a/fs/xfs/libxfs/xfs_dir2.h b/fs/xfs/libxfs/xfs_dir2.h new file mode 100644 index 000000000..a6594a5a9 --- /dev/null +++ b/fs/xfs/libxfs/xfs_dir2.h @@ -0,0 +1,358 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2000-2001,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_DIR2_H__ +#define __XFS_DIR2_H__ + +#include "xfs_da_format.h" +#include "xfs_da_btree.h" + +struct xfs_da_args; +struct xfs_inode; +struct xfs_mount; +struct xfs_trans; +struct xfs_dir2_sf_hdr; +struct xfs_dir2_sf_entry; +struct xfs_dir2_data_hdr; +struct xfs_dir2_data_entry; +struct xfs_dir2_data_unused; +struct xfs_dir3_icfree_hdr; +struct xfs_dir3_icleaf_hdr; + +extern const struct xfs_name xfs_name_dotdot; +extern const struct xfs_name xfs_name_dot; + +static inline bool +xfs_dir2_samename( + const struct xfs_name *n1, + const struct xfs_name *n2) +{ + if (n1 == n2) + return true; + if (n1->len != n2->len) + return false; + return !memcmp(n1->name, n2->name, n1->len); +} + +enum xfs_dir2_fmt { + XFS_DIR2_FMT_SF, + XFS_DIR2_FMT_BLOCK, + XFS_DIR2_FMT_LEAF, + XFS_DIR2_FMT_NODE, + XFS_DIR2_FMT_ERROR, +}; + +enum xfs_dir2_fmt xfs_dir2_format(struct xfs_da_args *args, int *error); + +/* + * Convert inode mode to directory entry filetype + */ +extern unsigned char xfs_mode_to_ftype(int mode); + +/* + * Generic directory interface routines + */ +extern void xfs_dir_startup(void); +extern int xfs_da_mount(struct xfs_mount *mp); +extern void xfs_da_unmount(struct xfs_mount *mp); + +extern int xfs_dir_init(struct xfs_trans *tp, struct xfs_inode *dp, + struct xfs_inode *pdp); +extern int xfs_dir_createname(struct xfs_trans *tp, struct xfs_inode *dp, + const struct xfs_name *name, xfs_ino_t inum, + xfs_extlen_t tot); +extern int xfs_dir_lookup(struct xfs_trans *tp, struct xfs_inode *dp, + const struct xfs_name *name, xfs_ino_t *inum, + struct xfs_name *ci_name); +extern int xfs_dir_removename(struct xfs_trans *tp, struct xfs_inode *dp, + const struct xfs_name *name, xfs_ino_t ino, + xfs_extlen_t tot); +extern int xfs_dir_replace(struct xfs_trans *tp, struct xfs_inode *dp, + const struct xfs_name *name, xfs_ino_t inum, + xfs_extlen_t tot); +extern int xfs_dir_canenter(struct xfs_trans *tp, struct xfs_inode *dp, + const struct xfs_name *name); + +int xfs_dir_lookup_args(struct xfs_da_args *args); +int xfs_dir_createname_args(struct xfs_da_args *args); +int xfs_dir_removename_args(struct xfs_da_args *args); +int xfs_dir_replace_args(struct xfs_da_args *args); + +/* + * Direct call from the bmap code, bypassing the generic directory layer. + */ +extern int xfs_dir2_sf_to_block(struct xfs_da_args *args); + +/* + * Interface routines used by userspace utilities + */ +extern int xfs_dir2_shrink_inode(struct xfs_da_args *args, xfs_dir2_db_t db, + struct xfs_buf *bp); + +extern void xfs_dir2_data_freescan(struct xfs_mount *mp, + struct xfs_dir2_data_hdr *hdr, int *loghead); +extern void xfs_dir2_data_log_entry(struct xfs_da_args *args, + struct xfs_buf *bp, struct xfs_dir2_data_entry *dep); +extern void xfs_dir2_data_log_header(struct xfs_da_args *args, + struct xfs_buf *bp); +extern void xfs_dir2_data_log_unused(struct xfs_da_args *args, + struct xfs_buf *bp, struct xfs_dir2_data_unused *dup); +extern void xfs_dir2_data_make_free(struct xfs_da_args *args, + struct xfs_buf *bp, xfs_dir2_data_aoff_t offset, + xfs_dir2_data_aoff_t len, int *needlogp, int *needscanp); +extern int xfs_dir2_data_use_free(struct xfs_da_args *args, + struct xfs_buf *bp, struct xfs_dir2_data_unused *dup, + xfs_dir2_data_aoff_t offset, xfs_dir2_data_aoff_t len, + int *needlogp, int *needscanp); + +extern struct xfs_dir2_data_free *xfs_dir2_data_freefind( + struct xfs_dir2_data_hdr *hdr, struct xfs_dir2_data_free *bf, + struct xfs_dir2_data_unused *dup); + +extern int xfs_dir_ino_validate(struct xfs_mount *mp, xfs_ino_t ino); + +xfs_failaddr_t xfs_dir3_leaf_header_check(struct xfs_buf *bp, xfs_ino_t owner); +xfs_failaddr_t xfs_dir3_data_header_check(struct xfs_buf *bp, xfs_ino_t owner); +xfs_failaddr_t xfs_dir3_block_header_check(struct xfs_buf *bp, xfs_ino_t owner); + +extern const struct xfs_buf_ops xfs_dir3_block_buf_ops; +extern const struct xfs_buf_ops xfs_dir3_leafn_buf_ops; +extern const struct xfs_buf_ops xfs_dir3_leaf1_buf_ops; +extern const struct xfs_buf_ops xfs_dir3_free_buf_ops; +extern const struct xfs_buf_ops xfs_dir3_data_buf_ops; + +/* + * Directory offset/block conversion functions. + * + * DB blocks here are logical directory block numbers, not filesystem blocks. + */ + +/* + * Convert dataptr to byte in file space + */ +static inline xfs_dir2_off_t +xfs_dir2_dataptr_to_byte(xfs_dir2_dataptr_t dp) +{ + return (xfs_dir2_off_t)dp << XFS_DIR2_DATA_ALIGN_LOG; +} + +/* + * Convert byte in file space to dataptr. It had better be aligned. + */ +static inline xfs_dir2_dataptr_t +xfs_dir2_byte_to_dataptr(xfs_dir2_off_t by) +{ + return (xfs_dir2_dataptr_t)(by >> XFS_DIR2_DATA_ALIGN_LOG); +} + +/* + * Convert byte in space to (DB) block + */ +static inline xfs_dir2_db_t +xfs_dir2_byte_to_db(struct xfs_da_geometry *geo, xfs_dir2_off_t by) +{ + return (xfs_dir2_db_t)(by >> geo->blklog); +} + +/* + * Convert dataptr to a block number + */ +static inline xfs_dir2_db_t +xfs_dir2_dataptr_to_db(struct xfs_da_geometry *geo, xfs_dir2_dataptr_t dp) +{ + return xfs_dir2_byte_to_db(geo, xfs_dir2_dataptr_to_byte(dp)); +} + +/* + * Convert byte in space to offset in a block + */ +static inline xfs_dir2_data_aoff_t +xfs_dir2_byte_to_off(struct xfs_da_geometry *geo, xfs_dir2_off_t by) +{ + return (xfs_dir2_data_aoff_t)(by & (geo->blksize - 1)); +} + +/* + * Convert dataptr to a byte offset in a block + */ +static inline xfs_dir2_data_aoff_t +xfs_dir2_dataptr_to_off(struct xfs_da_geometry *geo, xfs_dir2_dataptr_t dp) +{ + return xfs_dir2_byte_to_off(geo, xfs_dir2_dataptr_to_byte(dp)); +} + +/* + * Convert block and offset to byte in space + */ +static inline xfs_dir2_off_t +xfs_dir2_db_off_to_byte(struct xfs_da_geometry *geo, xfs_dir2_db_t db, + xfs_dir2_data_aoff_t o) +{ + return ((xfs_dir2_off_t)db << geo->blklog) + o; +} + +/* + * Convert block (DB) to block (dablk) + */ +static inline xfs_dablk_t +xfs_dir2_db_to_da(struct xfs_da_geometry *geo, xfs_dir2_db_t db) +{ + return (xfs_dablk_t)(db << (geo->blklog - geo->fsblog)); +} + +/* + * Convert byte in space to (DA) block + */ +static inline xfs_dablk_t +xfs_dir2_byte_to_da(struct xfs_da_geometry *geo, xfs_dir2_off_t by) +{ + return xfs_dir2_db_to_da(geo, xfs_dir2_byte_to_db(geo, by)); +} + +/* + * Convert block and offset to dataptr + */ +static inline xfs_dir2_dataptr_t +xfs_dir2_db_off_to_dataptr(struct xfs_da_geometry *geo, xfs_dir2_db_t db, + xfs_dir2_data_aoff_t o) +{ + return xfs_dir2_byte_to_dataptr(xfs_dir2_db_off_to_byte(geo, db, o)); +} + +/* + * Convert block (dablk) to block (DB) + */ +static inline xfs_dir2_db_t +xfs_dir2_da_to_db(struct xfs_da_geometry *geo, xfs_dablk_t da) +{ + return (xfs_dir2_db_t)(da >> (geo->blklog - geo->fsblog)); +} + +/* + * Convert block (dablk) to byte offset in space + */ +static inline xfs_dir2_off_t +xfs_dir2_da_to_byte(struct xfs_da_geometry *geo, xfs_dablk_t da) +{ + return xfs_dir2_db_off_to_byte(geo, xfs_dir2_da_to_db(geo, da), 0); +} + +/* + * Directory tail pointer accessor functions. Based on block geometry. + */ +static inline struct xfs_dir2_block_tail * +xfs_dir2_block_tail_p(struct xfs_da_geometry *geo, struct xfs_dir2_data_hdr *hdr) +{ + return ((struct xfs_dir2_block_tail *) + ((char *)hdr + geo->blksize)) - 1; +} + +static inline struct xfs_dir2_leaf_tail * +xfs_dir2_leaf_tail_p(struct xfs_da_geometry *geo, struct xfs_dir2_leaf *lp) +{ + return (struct xfs_dir2_leaf_tail *) + ((char *)lp + geo->blksize - + sizeof(struct xfs_dir2_leaf_tail)); +} + +/* + * The Linux API doesn't pass down the total size of the buffer + * we read into down to the filesystem. With the filldir concept + * it's not needed for correct information, but the XFS dir2 leaf + * code wants an estimate of the buffer size to calculate it's + * readahead window and size the buffers used for mapping to + * physical blocks. + * + * Try to give it an estimate that's good enough, maybe at some + * point we can change the ->readdir prototype to include the + * buffer size. For now we use the current glibc buffer size. + * musl libc hardcodes 2k and dietlibc uses PAGE_SIZE. + */ +#define XFS_READDIR_BUFSIZE (32768) + +unsigned char xfs_dir3_get_dtype(struct xfs_mount *mp, uint8_t filetype); +unsigned int xfs_dir3_data_end_offset(struct xfs_da_geometry *geo, + struct xfs_dir2_data_hdr *hdr); +bool xfs_dir2_namecheck(const void *name, size_t length); + +/* + * The "ascii-ci" feature was created to speed up case-insensitive lookups for + * a Samba product. Because of the inherent problems with CI and UTF-8 + * encoding, etc, it was decided that Samba would be configured to export + * latin1/iso 8859-1 encodings as that covered >90% of the target markets for + * the product. Hence the "ascii-ci" casefolding code could be encoded into + * the XFS directory operations and remove all the overhead of casefolding from + * Samba. + * + * To provide consistent hashing behavior between the userspace and kernel, + * these functions prepare names for hashing by transforming specific bytes + * to other bytes. Robustness with other encodings is not guaranteed. + */ +static inline bool xfs_ascii_ci_need_xfrm(unsigned char c) +{ + if (c >= 0x41 && c <= 0x5a) /* A-Z */ + return true; + if (c >= 0xc0 && c <= 0xd6) /* latin A-O with accents */ + return true; + if (c >= 0xd8 && c <= 0xde) /* latin O-Y with accents */ + return true; + return false; +} + +static inline unsigned char xfs_ascii_ci_xfrm(unsigned char c) +{ + if (xfs_ascii_ci_need_xfrm(c)) + c -= 'A' - 'a'; + return c; +} + +struct xfs_dir_update_params { + const struct xfs_inode *dp; + const struct xfs_inode *ip; + const struct xfs_name *name; + int delta; +}; + +#ifdef CONFIG_XFS_LIVE_HOOKS +void xfs_dir_update_hook(struct xfs_inode *dp, struct xfs_inode *ip, + int delta, const struct xfs_name *name); + +struct xfs_dir_hook { + struct xfs_hook dirent_hook; +}; + +void xfs_dir_hook_disable(void); +void xfs_dir_hook_enable(void); + +int xfs_dir_hook_add(struct xfs_mount *mp, struct xfs_dir_hook *hook); +void xfs_dir_hook_del(struct xfs_mount *mp, struct xfs_dir_hook *hook); +void xfs_dir_hook_setup(struct xfs_dir_hook *hook, notifier_fn_t mod_fn); +#else +# define xfs_dir_update_hook(dp, ip, delta, name) ((void)0) +#endif /* CONFIG_XFS_LIVE_HOOKS */ + +struct xfs_parent_args; + +struct xfs_dir_update { + struct xfs_inode *dp; + const struct xfs_name *name; + struct xfs_inode *ip; + struct xfs_parent_args *ppargs; +}; + +int xfs_dir_create_child(struct xfs_trans *tp, unsigned int resblks, + struct xfs_dir_update *du); +int xfs_dir_add_child(struct xfs_trans *tp, unsigned int resblks, + struct xfs_dir_update *du); +int xfs_dir_remove_child(struct xfs_trans *tp, unsigned int resblks, + struct xfs_dir_update *du); + +int xfs_dir_exchange_children(struct xfs_trans *tp, struct xfs_dir_update *du1, + struct xfs_dir_update *du2, unsigned int spaceres); +int xfs_dir_rename_children(struct xfs_trans *tp, struct xfs_dir_update *du_src, + struct xfs_dir_update *du_tgt, unsigned int spaceres, + struct xfs_dir_update *du_wip); + +#endif /* __XFS_DIR2_H__ */ diff --git a/fs/xfs/libxfs/xfs_dir2_block.c b/fs/xfs/libxfs/xfs_dir2_block.c new file mode 100644 index 000000000..6d70e6b42 --- /dev/null +++ b/fs/xfs/libxfs/xfs_dir2_block.c @@ -0,0 +1,1281 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2003,2005 Silicon Graphics, Inc. + * Copyright (c) 2013 Red Hat, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_bmap.h" +#include "xfs_buf_item.h" +#include "xfs_dir2.h" +#include "xfs_dir2_priv.h" +#include "xfs_error.h" +#include "xfs_trace.h" +#include "xfs_log.h" +#include "xfs_health.h" + +/* + * Local function prototypes. + */ +static void xfs_dir2_block_log_leaf(xfs_trans_t *tp, struct xfs_buf *bp, + int first, int last); +static void xfs_dir2_block_log_tail(xfs_trans_t *tp, struct xfs_buf *bp); +static int xfs_dir2_block_lookup_int(xfs_da_args_t *args, struct xfs_buf **bpp, + int *entno); +static int xfs_dir2_block_sort(const void *a, const void *b); + +static xfs_dahash_t xfs_dir_hash_dot, xfs_dir_hash_dotdot; + +/* + * One-time startup routine called from xfs_init(). + */ +void +xfs_dir_startup(void) +{ + xfs_dir_hash_dot = xfs_da_hashname((unsigned char *)".", 1); + xfs_dir_hash_dotdot = xfs_da_hashname((unsigned char *)"..", 2); +} + +static xfs_failaddr_t +xfs_dir3_block_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_dir3_blk_hdr *hdr3 = bp->b_addr; + + if (!xfs_verify_magic(bp, hdr3->magic)) + return __this_address; + + if (xfs_has_crc(mp)) { + if (!uuid_equal(&hdr3->uuid, &mp->m_sb.sb_meta_uuid)) + return __this_address; + if (be64_to_cpu(hdr3->blkno) != xfs_buf_daddr(bp)) + return __this_address; + if (!xfs_log_check_lsn(mp, be64_to_cpu(hdr3->lsn))) + return __this_address; + } + return __xfs_dir3_data_check(NULL, bp); +} + +static void +xfs_dir3_block_read_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + xfs_failaddr_t fa; + + if (xfs_has_crc(mp) && + !xfs_buf_verify_cksum(bp, XFS_DIR3_DATA_CRC_OFF)) + xfs_verifier_error(bp, -EFSBADCRC, __this_address); + else { + fa = xfs_dir3_block_verify(bp); + if (fa) + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + } +} + +static void +xfs_dir3_block_write_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_buf_log_item *bip = bp->b_log_item; + struct xfs_dir3_blk_hdr *hdr3 = bp->b_addr; + xfs_failaddr_t fa; + + fa = xfs_dir3_block_verify(bp); + if (fa) { + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + return; + } + + if (!xfs_has_crc(mp)) + return; + + if (bip) + hdr3->lsn = cpu_to_be64(bip->bli_item.li_lsn); + + xfs_buf_update_cksum(bp, XFS_DIR3_DATA_CRC_OFF); +} + +const struct xfs_buf_ops xfs_dir3_block_buf_ops = { + .name = "xfs_dir3_block", + .magic = { cpu_to_be32(XFS_DIR2_BLOCK_MAGIC), + cpu_to_be32(XFS_DIR3_BLOCK_MAGIC) }, + .verify_read = xfs_dir3_block_read_verify, + .verify_write = xfs_dir3_block_write_verify, + .verify_struct = xfs_dir3_block_verify, +}; + +xfs_failaddr_t +xfs_dir3_block_header_check( + struct xfs_buf *bp, + xfs_ino_t owner) +{ + struct xfs_mount *mp = bp->b_mount; + + if (xfs_has_crc(mp)) { + struct xfs_dir3_blk_hdr *hdr3 = bp->b_addr; + + if (hdr3->magic != cpu_to_be32(XFS_DIR3_BLOCK_MAGIC)) + return __this_address; + + if (be64_to_cpu(hdr3->owner) != owner) + return __this_address; + } + + return NULL; +} + +int +xfs_dir3_block_read( + struct xfs_trans *tp, + struct xfs_inode *dp, + xfs_ino_t owner, + struct xfs_buf **bpp) +{ + struct xfs_mount *mp = dp->i_mount; + xfs_failaddr_t fa; + int err; + + err = xfs_da_read_buf(tp, dp, mp->m_dir_geo->datablk, 0, bpp, + XFS_DATA_FORK, &xfs_dir3_block_buf_ops); + if (err || !*bpp) + return err; + + /* Check things that we can't do in the verifier. */ + fa = xfs_dir3_block_header_check(*bpp, owner); + if (fa) { + __xfs_buf_mark_corrupt(*bpp, fa); + xfs_trans_brelse(tp, *bpp); + *bpp = NULL; + xfs_dirattr_mark_sick(dp, XFS_DATA_FORK); + return -EFSCORRUPTED; + } + + xfs_trans_buf_set_type(tp, *bpp, XFS_BLFT_DIR_BLOCK_BUF); + return err; +} + +static void +xfs_dir3_block_init( + struct xfs_da_args *args, + struct xfs_buf *bp) +{ + struct xfs_trans *tp = args->trans; + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + struct xfs_dir3_blk_hdr *hdr3 = bp->b_addr; + + bp->b_ops = &xfs_dir3_block_buf_ops; + xfs_trans_buf_set_type(tp, bp, XFS_BLFT_DIR_BLOCK_BUF); + + if (xfs_has_crc(mp)) { + memset(hdr3, 0, sizeof(*hdr3)); + hdr3->magic = cpu_to_be32(XFS_DIR3_BLOCK_MAGIC); + hdr3->blkno = cpu_to_be64(xfs_buf_daddr(bp)); + hdr3->owner = cpu_to_be64(args->owner); + uuid_copy(&hdr3->uuid, &mp->m_sb.sb_meta_uuid); + return; + + } + hdr3->magic = cpu_to_be32(XFS_DIR2_BLOCK_MAGIC); +} + +static void +xfs_dir2_block_need_space( + struct xfs_inode *dp, + struct xfs_dir2_data_hdr *hdr, + struct xfs_dir2_block_tail *btp, + struct xfs_dir2_leaf_entry *blp, + __be16 **tagpp, + struct xfs_dir2_data_unused **dupp, + struct xfs_dir2_data_unused **enddupp, + int *compact, + int len) +{ + struct xfs_dir2_data_free *bf; + __be16 *tagp = NULL; + struct xfs_dir2_data_unused *dup = NULL; + struct xfs_dir2_data_unused *enddup = NULL; + + *compact = 0; + bf = xfs_dir2_data_bestfree_p(dp->i_mount, hdr); + + /* + * If there are stale entries we'll use one for the leaf. + */ + if (btp->stale) { + if (be16_to_cpu(bf[0].length) >= len) { + /* + * The biggest entry enough to avoid compaction. + */ + dup = (xfs_dir2_data_unused_t *) + ((char *)hdr + be16_to_cpu(bf[0].offset)); + goto out; + } + + /* + * Will need to compact to make this work. + * Tag just before the first leaf entry. + */ + *compact = 1; + tagp = (__be16 *)blp - 1; + + /* Data object just before the first leaf entry. */ + dup = (xfs_dir2_data_unused_t *)((char *)hdr + be16_to_cpu(*tagp)); + + /* + * If it's not free then the data will go where the + * leaf data starts now, if it works at all. + */ + if (be16_to_cpu(dup->freetag) == XFS_DIR2_DATA_FREE_TAG) { + if (be16_to_cpu(dup->length) + (be32_to_cpu(btp->stale) - 1) * + (uint)sizeof(*blp) < len) + dup = NULL; + } else if ((be32_to_cpu(btp->stale) - 1) * (uint)sizeof(*blp) < len) + dup = NULL; + else + dup = (xfs_dir2_data_unused_t *)blp; + goto out; + } + + /* + * no stale entries, so just use free space. + * Tag just before the first leaf entry. + */ + tagp = (__be16 *)blp - 1; + + /* Data object just before the first leaf entry. */ + enddup = (xfs_dir2_data_unused_t *)((char *)hdr + be16_to_cpu(*tagp)); + + /* + * If it's not free then can't do this add without cleaning up: + * the space before the first leaf entry needs to be free so it + * can be expanded to hold the pointer to the new entry. + */ + if (be16_to_cpu(enddup->freetag) == XFS_DIR2_DATA_FREE_TAG) { + /* + * Check out the biggest freespace and see if it's the same one. + */ + dup = (xfs_dir2_data_unused_t *) + ((char *)hdr + be16_to_cpu(bf[0].offset)); + if (dup != enddup) { + /* + * Not the same free entry, just check its length. + */ + if (be16_to_cpu(dup->length) < len) + dup = NULL; + goto out; + } + + /* + * It is the biggest freespace, can it hold the leaf too? + */ + if (be16_to_cpu(dup->length) < len + (uint)sizeof(*blp)) { + /* + * Yes, use the second-largest entry instead if it works. + */ + if (be16_to_cpu(bf[1].length) >= len) + dup = (xfs_dir2_data_unused_t *) + ((char *)hdr + be16_to_cpu(bf[1].offset)); + else + dup = NULL; + } + } +out: + *tagpp = tagp; + *dupp = dup; + *enddupp = enddup; +} + +/* + * compact the leaf entries. + * Leave the highest-numbered stale entry stale. + * XXX should be the one closest to mid but mid is not yet computed. + */ +static void +xfs_dir2_block_compact( + struct xfs_da_args *args, + struct xfs_buf *bp, + struct xfs_dir2_data_hdr *hdr, + struct xfs_dir2_block_tail *btp, + struct xfs_dir2_leaf_entry *blp, + int *needlog, + int *lfloghigh, + int *lfloglow) +{ + int fromidx; /* source leaf index */ + int toidx; /* target leaf index */ + int needscan = 0; + int highstale; /* high stale index */ + + fromidx = toidx = be32_to_cpu(btp->count) - 1; + highstale = *lfloghigh = -1; + for (; fromidx >= 0; fromidx--) { + if (blp[fromidx].address == cpu_to_be32(XFS_DIR2_NULL_DATAPTR)) { + if (highstale == -1) + highstale = toidx; + else { + if (*lfloghigh == -1) + *lfloghigh = toidx; + continue; + } + } + if (fromidx < toidx) + blp[toidx] = blp[fromidx]; + toidx--; + } + *lfloglow = toidx + 1 - (be32_to_cpu(btp->stale) - 1); + *lfloghigh -= be32_to_cpu(btp->stale) - 1; + be32_add_cpu(&btp->count, -(be32_to_cpu(btp->stale) - 1)); + xfs_dir2_data_make_free(args, bp, + (xfs_dir2_data_aoff_t)((char *)blp - (char *)hdr), + (xfs_dir2_data_aoff_t)((be32_to_cpu(btp->stale) - 1) * sizeof(*blp)), + needlog, &needscan); + btp->stale = cpu_to_be32(1); + /* + * If we now need to rebuild the bestfree map, do so. + * This needs to happen before the next call to use_free. + */ + if (needscan) + xfs_dir2_data_freescan(args->dp->i_mount, hdr, needlog); +} + +/* + * Add an entry to a block directory. + */ +int /* error */ +xfs_dir2_block_addname( + xfs_da_args_t *args) /* directory op arguments */ +{ + xfs_dir2_data_hdr_t *hdr; /* block header */ + xfs_dir2_leaf_entry_t *blp; /* block leaf entries */ + struct xfs_buf *bp; /* buffer for block */ + xfs_dir2_block_tail_t *btp; /* block tail */ + int compact; /* need to compact leaf ents */ + xfs_dir2_data_entry_t *dep; /* block data entry */ + xfs_inode_t *dp; /* directory inode */ + xfs_dir2_data_unused_t *dup; /* block unused entry */ + int error; /* error return value */ + xfs_dir2_data_unused_t *enddup=NULL; /* unused at end of data */ + xfs_dahash_t hash; /* hash value of found entry */ + int high; /* high index for binary srch */ + int highstale; /* high stale index */ + int lfloghigh=0; /* last final leaf to log */ + int lfloglow=0; /* first final leaf to log */ + int len; /* length of the new entry */ + int low; /* low index for binary srch */ + int lowstale; /* low stale index */ + int mid=0; /* midpoint for binary srch */ + int needlog; /* need to log header */ + int needscan; /* need to rescan freespace */ + __be16 *tagp; /* pointer to tag value */ + xfs_trans_t *tp; /* transaction structure */ + + trace_xfs_dir2_block_addname(args); + + dp = args->dp; + tp = args->trans; + + /* Read the (one and only) directory block into bp. */ + error = xfs_dir3_block_read(tp, dp, args->owner, &bp); + if (error) + return error; + + len = xfs_dir2_data_entsize(dp->i_mount, args->namelen); + + /* + * Set up pointers to parts of the block. + */ + hdr = bp->b_addr; + btp = xfs_dir2_block_tail_p(args->geo, hdr); + blp = xfs_dir2_block_leaf_p(btp); + + /* + * Find out if we can reuse stale entries or whether we need extra + * space for entry and new leaf. + */ + xfs_dir2_block_need_space(dp, hdr, btp, blp, &tagp, &dup, + &enddup, &compact, len); + + /* + * Done everything we need for a space check now. + */ + if (args->op_flags & XFS_DA_OP_JUSTCHECK) { + xfs_trans_brelse(tp, bp); + if (!dup) + return -ENOSPC; + return 0; + } + + /* + * If we don't have space for the new entry & leaf ... + */ + if (!dup) { + /* Don't have a space reservation: return no-space. */ + if (args->total == 0) + return -ENOSPC; + /* + * Convert to the next larger format. + * Then add the new entry in that format. + */ + error = xfs_dir2_block_to_leaf(args, bp); + if (error) + return error; + return xfs_dir2_leaf_addname(args); + } + + needlog = needscan = 0; + + /* + * If need to compact the leaf entries, do it now. + */ + if (compact) { + xfs_dir2_block_compact(args, bp, hdr, btp, blp, &needlog, + &lfloghigh, &lfloglow); + /* recalculate blp post-compaction */ + blp = xfs_dir2_block_leaf_p(btp); + } else if (btp->stale) { + /* + * Set leaf logging boundaries to impossible state. + * For the no-stale case they're set explicitly. + */ + lfloglow = be32_to_cpu(btp->count); + lfloghigh = -1; + } + + /* + * Find the slot that's first lower than our hash value, -1 if none. + */ + for (low = 0, high = be32_to_cpu(btp->count) - 1; low <= high; ) { + mid = (low + high) >> 1; + if ((hash = be32_to_cpu(blp[mid].hashval)) == args->hashval) + break; + if (hash < args->hashval) + low = mid + 1; + else + high = mid - 1; + } + while (mid >= 0 && be32_to_cpu(blp[mid].hashval) >= args->hashval) { + mid--; + } + /* + * No stale entries, will use enddup space to hold new leaf. + */ + if (!btp->stale) { + xfs_dir2_data_aoff_t aoff; + + /* + * Mark the space needed for the new leaf entry, now in use. + */ + aoff = (xfs_dir2_data_aoff_t)((char *)enddup - (char *)hdr + + be16_to_cpu(enddup->length) - sizeof(*blp)); + error = xfs_dir2_data_use_free(args, bp, enddup, aoff, + (xfs_dir2_data_aoff_t)sizeof(*blp), &needlog, + &needscan); + if (error) + return error; + + /* + * Update the tail (entry count). + */ + be32_add_cpu(&btp->count, 1); + /* + * If we now need to rebuild the bestfree map, do so. + * This needs to happen before the next call to use_free. + */ + if (needscan) { + xfs_dir2_data_freescan(dp->i_mount, hdr, &needlog); + needscan = 0; + } + /* + * Adjust pointer to the first leaf entry, we're about to move + * the table up one to open up space for the new leaf entry. + * Then adjust our index to match. + */ + blp--; + mid++; + if (mid) + memmove(blp, &blp[1], mid * sizeof(*blp)); + lfloglow = 0; + lfloghigh = mid; + } + /* + * Use a stale leaf for our new entry. + */ + else { + for (lowstale = mid; + lowstale >= 0 && + blp[lowstale].address != + cpu_to_be32(XFS_DIR2_NULL_DATAPTR); + lowstale--) + continue; + for (highstale = mid + 1; + highstale < be32_to_cpu(btp->count) && + blp[highstale].address != + cpu_to_be32(XFS_DIR2_NULL_DATAPTR) && + (lowstale < 0 || mid - lowstale > highstale - mid); + highstale++) + continue; + /* + * Move entries toward the low-numbered stale entry. + */ + if (lowstale >= 0 && + (highstale == be32_to_cpu(btp->count) || + mid - lowstale <= highstale - mid)) { + if (mid - lowstale) + memmove(&blp[lowstale], &blp[lowstale + 1], + (mid - lowstale) * sizeof(*blp)); + lfloglow = min(lowstale, lfloglow); + lfloghigh = max(mid, lfloghigh); + } + /* + * Move entries toward the high-numbered stale entry. + */ + else { + ASSERT(highstale < be32_to_cpu(btp->count)); + mid++; + if (highstale - mid) + memmove(&blp[mid + 1], &blp[mid], + (highstale - mid) * sizeof(*blp)); + lfloglow = min(mid, lfloglow); + lfloghigh = max(highstale, lfloghigh); + } + be32_add_cpu(&btp->stale, -1); + } + /* + * Point to the new data entry. + */ + dep = (xfs_dir2_data_entry_t *)dup; + /* + * Fill in the leaf entry. + */ + blp[mid].hashval = cpu_to_be32(args->hashval); + blp[mid].address = cpu_to_be32(xfs_dir2_byte_to_dataptr( + (char *)dep - (char *)hdr)); + xfs_dir2_block_log_leaf(tp, bp, lfloglow, lfloghigh); + /* + * Mark space for the data entry used. + */ + error = xfs_dir2_data_use_free(args, bp, dup, + (xfs_dir2_data_aoff_t)((char *)dup - (char *)hdr), + (xfs_dir2_data_aoff_t)len, &needlog, &needscan); + if (error) + return error; + /* + * Create the new data entry. + */ + dep->inumber = cpu_to_be64(args->inumber); + dep->namelen = args->namelen; + memcpy(dep->name, args->name, args->namelen); + xfs_dir2_data_put_ftype(dp->i_mount, dep, args->filetype); + tagp = xfs_dir2_data_entry_tag_p(dp->i_mount, dep); + *tagp = cpu_to_be16((char *)dep - (char *)hdr); + /* + * Clean up the bestfree array and log the header, tail, and entry. + */ + if (needscan) + xfs_dir2_data_freescan(dp->i_mount, hdr, &needlog); + if (needlog) + xfs_dir2_data_log_header(args, bp); + xfs_dir2_block_log_tail(tp, bp); + xfs_dir2_data_log_entry(args, bp, dep); + xfs_dir3_data_check(dp, bp); + return 0; +} + +/* + * Log leaf entries from the block. + */ +static void +xfs_dir2_block_log_leaf( + xfs_trans_t *tp, /* transaction structure */ + struct xfs_buf *bp, /* block buffer */ + int first, /* index of first logged leaf */ + int last) /* index of last logged leaf */ +{ + xfs_dir2_data_hdr_t *hdr = bp->b_addr; + xfs_dir2_leaf_entry_t *blp; + xfs_dir2_block_tail_t *btp; + + btp = xfs_dir2_block_tail_p(tp->t_mountp->m_dir_geo, hdr); + blp = xfs_dir2_block_leaf_p(btp); + xfs_trans_log_buf(tp, bp, (uint)((char *)&blp[first] - (char *)hdr), + (uint)((char *)&blp[last + 1] - (char *)hdr - 1)); +} + +/* + * Log the block tail. + */ +static void +xfs_dir2_block_log_tail( + xfs_trans_t *tp, /* transaction structure */ + struct xfs_buf *bp) /* block buffer */ +{ + xfs_dir2_data_hdr_t *hdr = bp->b_addr; + xfs_dir2_block_tail_t *btp; + + btp = xfs_dir2_block_tail_p(tp->t_mountp->m_dir_geo, hdr); + xfs_trans_log_buf(tp, bp, (uint)((char *)btp - (char *)hdr), + (uint)((char *)(btp + 1) - (char *)hdr - 1)); +} + +/* + * Look up an entry in the block. This is the external routine, + * xfs_dir2_block_lookup_int does the real work. + */ +int /* error */ +xfs_dir2_block_lookup( + xfs_da_args_t *args) /* dir lookup arguments */ +{ + xfs_dir2_data_hdr_t *hdr; /* block header */ + xfs_dir2_leaf_entry_t *blp; /* block leaf entries */ + struct xfs_buf *bp; /* block buffer */ + xfs_dir2_block_tail_t *btp; /* block tail */ + xfs_dir2_data_entry_t *dep; /* block data entry */ + xfs_inode_t *dp; /* incore inode */ + int ent; /* entry index */ + int error; /* error return value */ + + trace_xfs_dir2_block_lookup(args); + + /* + * Get the buffer, look up the entry. + * If not found (ENOENT) then return, have no buffer. + */ + if ((error = xfs_dir2_block_lookup_int(args, &bp, &ent))) + return error; + dp = args->dp; + hdr = bp->b_addr; + xfs_dir3_data_check(dp, bp); + btp = xfs_dir2_block_tail_p(args->geo, hdr); + blp = xfs_dir2_block_leaf_p(btp); + /* + * Get the offset from the leaf entry, to point to the data. + */ + dep = (xfs_dir2_data_entry_t *)((char *)hdr + + xfs_dir2_dataptr_to_off(args->geo, + be32_to_cpu(blp[ent].address))); + /* + * Fill in inode number, CI name if appropriate, release the block. + */ + args->inumber = be64_to_cpu(dep->inumber); + args->filetype = xfs_dir2_data_get_ftype(dp->i_mount, dep); + error = xfs_dir_cilookup_result(args, dep->name, dep->namelen); + xfs_trans_brelse(args->trans, bp); + return error; +} + +/* + * Internal block lookup routine. + */ +static int /* error */ +xfs_dir2_block_lookup_int( + xfs_da_args_t *args, /* dir lookup arguments */ + struct xfs_buf **bpp, /* returned block buffer */ + int *entno) /* returned entry number */ +{ + xfs_dir2_dataptr_t addr; /* data entry address */ + xfs_dir2_data_hdr_t *hdr; /* block header */ + xfs_dir2_leaf_entry_t *blp; /* block leaf entries */ + struct xfs_buf *bp; /* block buffer */ + xfs_dir2_block_tail_t *btp; /* block tail */ + xfs_dir2_data_entry_t *dep; /* block data entry */ + xfs_inode_t *dp; /* incore inode */ + int error; /* error return value */ + xfs_dahash_t hash; /* found hash value */ + int high; /* binary search high index */ + int low; /* binary search low index */ + int mid; /* binary search current idx */ + xfs_trans_t *tp; /* transaction pointer */ + enum xfs_dacmp cmp; /* comparison result */ + + dp = args->dp; + tp = args->trans; + + error = xfs_dir3_block_read(tp, dp, args->owner, &bp); + if (error) + return error; + + hdr = bp->b_addr; + xfs_dir3_data_check(dp, bp); + btp = xfs_dir2_block_tail_p(args->geo, hdr); + blp = xfs_dir2_block_leaf_p(btp); + /* + * Loop doing a binary search for our hash value. + * Find our entry, ENOENT if it's not there. + */ + for (low = 0, high = be32_to_cpu(btp->count) - 1; ; ) { + ASSERT(low <= high); + mid = (low + high) >> 1; + if ((hash = be32_to_cpu(blp[mid].hashval)) == args->hashval) + break; + if (hash < args->hashval) + low = mid + 1; + else + high = mid - 1; + if (low > high) { + ASSERT(args->op_flags & XFS_DA_OP_OKNOENT); + xfs_trans_brelse(tp, bp); + return -ENOENT; + } + } + /* + * Back up to the first one with the right hash value. + */ + while (mid > 0 && be32_to_cpu(blp[mid - 1].hashval) == args->hashval) { + mid--; + } + /* + * Now loop forward through all the entries with the + * right hash value looking for our name. + */ + do { + if ((addr = be32_to_cpu(blp[mid].address)) == XFS_DIR2_NULL_DATAPTR) + continue; + /* + * Get pointer to the entry from the leaf. + */ + dep = (xfs_dir2_data_entry_t *) + ((char *)hdr + xfs_dir2_dataptr_to_off(args->geo, addr)); + /* + * Compare name and if it's an exact match, return the index + * and buffer. If it's the first case-insensitive match, store + * the index and buffer and continue looking for an exact match. + */ + cmp = xfs_dir2_compname(args, dep->name, dep->namelen); + if (cmp != XFS_CMP_DIFFERENT && cmp != args->cmpresult) { + args->cmpresult = cmp; + *bpp = bp; + *entno = mid; + if (cmp == XFS_CMP_EXACT) + return 0; + } + } while (++mid < be32_to_cpu(btp->count) && + be32_to_cpu(blp[mid].hashval) == hash); + + ASSERT(args->op_flags & XFS_DA_OP_OKNOENT); + /* + * Here, we can only be doing a lookup (not a rename or replace). + * If a case-insensitive match was found earlier, return success. + */ + if (args->cmpresult == XFS_CMP_CASE) + return 0; + /* + * No match, release the buffer and return ENOENT. + */ + xfs_trans_brelse(tp, bp); + return -ENOENT; +} + +/* + * Remove an entry from a block format directory. + * If that makes the block small enough to fit in shortform, transform it. + */ +int /* error */ +xfs_dir2_block_removename( + xfs_da_args_t *args) /* directory operation args */ +{ + xfs_dir2_data_hdr_t *hdr; /* block header */ + xfs_dir2_leaf_entry_t *blp; /* block leaf pointer */ + struct xfs_buf *bp; /* block buffer */ + xfs_dir2_block_tail_t *btp; /* block tail */ + xfs_dir2_data_entry_t *dep; /* block data entry */ + xfs_inode_t *dp; /* incore inode */ + int ent; /* block leaf entry index */ + int error; /* error return value */ + int needlog; /* need to log block header */ + int needscan; /* need to fixup bestfree */ + xfs_dir2_sf_hdr_t sfh; /* shortform header */ + int size; /* shortform size */ + xfs_trans_t *tp; /* transaction pointer */ + + trace_xfs_dir2_block_removename(args); + + /* + * Look up the entry in the block. Gets the buffer and entry index. + * It will always be there, the vnodeops level does a lookup first. + */ + if ((error = xfs_dir2_block_lookup_int(args, &bp, &ent))) { + return error; + } + dp = args->dp; + tp = args->trans; + hdr = bp->b_addr; + btp = xfs_dir2_block_tail_p(args->geo, hdr); + blp = xfs_dir2_block_leaf_p(btp); + /* + * Point to the data entry using the leaf entry. + */ + dep = (xfs_dir2_data_entry_t *)((char *)hdr + + xfs_dir2_dataptr_to_off(args->geo, + be32_to_cpu(blp[ent].address))); + /* + * Mark the data entry's space free. + */ + needlog = needscan = 0; + xfs_dir2_data_make_free(args, bp, + (xfs_dir2_data_aoff_t)((char *)dep - (char *)hdr), + xfs_dir2_data_entsize(dp->i_mount, dep->namelen), &needlog, + &needscan); + /* + * Fix up the block tail. + */ + be32_add_cpu(&btp->stale, 1); + xfs_dir2_block_log_tail(tp, bp); + /* + * Remove the leaf entry by marking it stale. + */ + blp[ent].address = cpu_to_be32(XFS_DIR2_NULL_DATAPTR); + xfs_dir2_block_log_leaf(tp, bp, ent, ent); + /* + * Fix up bestfree, log the header if necessary. + */ + if (needscan) + xfs_dir2_data_freescan(dp->i_mount, hdr, &needlog); + if (needlog) + xfs_dir2_data_log_header(args, bp); + xfs_dir3_data_check(dp, bp); + /* + * See if the size as a shortform is good enough. + */ + size = xfs_dir2_block_sfsize(dp, hdr, &sfh); + if (size > xfs_inode_data_fork_size(dp)) + return 0; + + /* + * If it works, do the conversion. + */ + return xfs_dir2_block_to_sf(args, bp, size, &sfh); +} + +/* + * Replace an entry in a V2 block directory. + * Change the inode number to the new value. + */ +int /* error */ +xfs_dir2_block_replace( + xfs_da_args_t *args) /* directory operation args */ +{ + xfs_dir2_data_hdr_t *hdr; /* block header */ + xfs_dir2_leaf_entry_t *blp; /* block leaf entries */ + struct xfs_buf *bp; /* block buffer */ + xfs_dir2_block_tail_t *btp; /* block tail */ + xfs_dir2_data_entry_t *dep; /* block data entry */ + xfs_inode_t *dp; /* incore inode */ + int ent; /* leaf entry index */ + int error; /* error return value */ + + trace_xfs_dir2_block_replace(args); + + /* + * Lookup the entry in the directory. Get buffer and entry index. + * This will always succeed since the caller has already done a lookup. + */ + if ((error = xfs_dir2_block_lookup_int(args, &bp, &ent))) { + return error; + } + dp = args->dp; + hdr = bp->b_addr; + btp = xfs_dir2_block_tail_p(args->geo, hdr); + blp = xfs_dir2_block_leaf_p(btp); + /* + * Point to the data entry we need to change. + */ + dep = (xfs_dir2_data_entry_t *)((char *)hdr + + xfs_dir2_dataptr_to_off(args->geo, + be32_to_cpu(blp[ent].address))); + ASSERT(be64_to_cpu(dep->inumber) != args->inumber); + /* + * Change the inode number to the new value. + */ + dep->inumber = cpu_to_be64(args->inumber); + xfs_dir2_data_put_ftype(dp->i_mount, dep, args->filetype); + xfs_dir2_data_log_entry(args, bp, dep); + xfs_dir3_data_check(dp, bp); + return 0; +} + +/* + * Qsort comparison routine for the block leaf entries. + */ +static int /* sort order */ +xfs_dir2_block_sort( + const void *a, /* first leaf entry */ + const void *b) /* second leaf entry */ +{ + const xfs_dir2_leaf_entry_t *la; /* first leaf entry */ + const xfs_dir2_leaf_entry_t *lb; /* second leaf entry */ + + la = a; + lb = b; + return be32_to_cpu(la->hashval) < be32_to_cpu(lb->hashval) ? -1 : + (be32_to_cpu(la->hashval) > be32_to_cpu(lb->hashval) ? 1 : 0); +} + +/* + * Convert a V2 leaf directory to a V2 block directory if possible. + */ +int /* error */ +xfs_dir2_leaf_to_block( + xfs_da_args_t *args, /* operation arguments */ + struct xfs_buf *lbp, /* leaf buffer */ + struct xfs_buf *dbp) /* data buffer */ +{ + __be16 *bestsp; /* leaf bests table */ + xfs_dir2_data_hdr_t *hdr; /* block header */ + xfs_dir2_block_tail_t *btp; /* block tail */ + xfs_inode_t *dp; /* incore directory inode */ + xfs_dir2_data_unused_t *dup; /* unused data entry */ + int error; /* error return value */ + int from; /* leaf from index */ + xfs_dir2_leaf_t *leaf; /* leaf structure */ + xfs_dir2_leaf_entry_t *lep; /* leaf entry */ + xfs_dir2_leaf_tail_t *ltp; /* leaf tail structure */ + xfs_mount_t *mp; /* file system mount point */ + int needlog; /* need to log data header */ + int needscan; /* need to scan for bestfree */ + xfs_dir2_sf_hdr_t sfh; /* shortform header */ + int size; /* bytes used */ + __be16 *tagp; /* end of entry (tag) */ + int to; /* block/leaf to index */ + xfs_trans_t *tp; /* transaction pointer */ + struct xfs_dir3_icleaf_hdr leafhdr; + + trace_xfs_dir2_leaf_to_block(args); + + dp = args->dp; + tp = args->trans; + mp = dp->i_mount; + leaf = lbp->b_addr; + xfs_dir2_leaf_hdr_from_disk(mp, &leafhdr, leaf); + ltp = xfs_dir2_leaf_tail_p(args->geo, leaf); + + ASSERT(leafhdr.magic == XFS_DIR2_LEAF1_MAGIC || + leafhdr.magic == XFS_DIR3_LEAF1_MAGIC); + /* + * If there are data blocks other than the first one, take this + * opportunity to remove trailing empty data blocks that may have + * been left behind during no-space-reservation operations. + * These will show up in the leaf bests table. + */ + while (dp->i_disk_size > args->geo->blksize) { + int hdrsz; + + hdrsz = args->geo->data_entry_offset; + bestsp = xfs_dir2_leaf_bests_p(ltp); + if (be16_to_cpu(bestsp[be32_to_cpu(ltp->bestcount) - 1]) == + args->geo->blksize - hdrsz) { + if ((error = + xfs_dir2_leaf_trim_data(args, lbp, + (xfs_dir2_db_t)(be32_to_cpu(ltp->bestcount) - 1)))) + return error; + } else + return 0; + } + /* + * Read the data block if we don't already have it, give up if it fails. + */ + if (!dbp) { + error = xfs_dir3_data_read(tp, dp, args->owner, + args->geo->datablk, 0, &dbp); + if (error) + return error; + } + hdr = dbp->b_addr; + ASSERT(hdr->magic == cpu_to_be32(XFS_DIR2_DATA_MAGIC) || + hdr->magic == cpu_to_be32(XFS_DIR3_DATA_MAGIC)); + + /* + * Size of the "leaf" area in the block. + */ + size = (uint)sizeof(xfs_dir2_block_tail_t) + + (uint)sizeof(*lep) * (leafhdr.count - leafhdr.stale); + /* + * Look at the last data entry. + */ + tagp = (__be16 *)((char *)hdr + args->geo->blksize) - 1; + dup = (xfs_dir2_data_unused_t *)((char *)hdr + be16_to_cpu(*tagp)); + /* + * If it's not free or is too short we can't do it. + */ + if (be16_to_cpu(dup->freetag) != XFS_DIR2_DATA_FREE_TAG || + be16_to_cpu(dup->length) < size) + return 0; + + /* + * Start converting it to block form. + */ + xfs_dir3_block_init(args, dbp); + + needlog = 1; + needscan = 0; + /* + * Use up the space at the end of the block (blp/btp). + */ + error = xfs_dir2_data_use_free(args, dbp, dup, + args->geo->blksize - size, size, &needlog, &needscan); + if (error) + return error; + /* + * Initialize the block tail. + */ + btp = xfs_dir2_block_tail_p(args->geo, hdr); + btp->count = cpu_to_be32(leafhdr.count - leafhdr.stale); + btp->stale = 0; + xfs_dir2_block_log_tail(tp, dbp); + /* + * Initialize the block leaf area. We compact out stale entries. + */ + lep = xfs_dir2_block_leaf_p(btp); + for (from = to = 0; from < leafhdr.count; from++) { + if (leafhdr.ents[from].address == + cpu_to_be32(XFS_DIR2_NULL_DATAPTR)) + continue; + lep[to++] = leafhdr.ents[from]; + } + ASSERT(to == be32_to_cpu(btp->count)); + xfs_dir2_block_log_leaf(tp, dbp, 0, be32_to_cpu(btp->count) - 1); + /* + * Scan the bestfree if we need it and log the data block header. + */ + if (needscan) + xfs_dir2_data_freescan(dp->i_mount, hdr, &needlog); + if (needlog) + xfs_dir2_data_log_header(args, dbp); + /* + * Pitch the old leaf block. + */ + error = xfs_da_shrink_inode(args, args->geo->leafblk, lbp); + if (error) + return error; + + /* + * Now see if the resulting block can be shrunken to shortform. + */ + size = xfs_dir2_block_sfsize(dp, hdr, &sfh); + if (size > xfs_inode_data_fork_size(dp)) + return 0; + + return xfs_dir2_block_to_sf(args, dbp, size, &sfh); +} + +/* + * Convert the shortform directory to block form. + */ +int /* error */ +xfs_dir2_sf_to_block( + struct xfs_da_args *args) +{ + struct xfs_trans *tp = args->trans; + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + struct xfs_ifork *ifp = xfs_ifork_ptr(dp, XFS_DATA_FORK); + struct xfs_da_geometry *geo = args->geo; + xfs_dir2_db_t blkno; /* dir-relative block # (0) */ + xfs_dir2_data_hdr_t *hdr; /* block header */ + xfs_dir2_leaf_entry_t *blp; /* block leaf entries */ + struct xfs_buf *bp; /* block buffer */ + xfs_dir2_block_tail_t *btp; /* block tail pointer */ + xfs_dir2_data_entry_t *dep; /* data entry pointer */ + int dummy; /* trash */ + xfs_dir2_data_unused_t *dup; /* unused entry pointer */ + int endoffset; /* end of data objects */ + int error; /* error return value */ + int i; /* index */ + int needlog; /* need to log block header */ + int needscan; /* need to scan block freespc */ + int newoffset; /* offset from current entry */ + unsigned int offset = geo->data_entry_offset; + xfs_dir2_sf_entry_t *sfep; /* sf entry pointer */ + struct xfs_dir2_sf_hdr *oldsfp = ifp->if_data; + xfs_dir2_sf_hdr_t *sfp; /* shortform header */ + __be16 *tagp; /* end of data entry */ + struct xfs_name name; + + trace_xfs_dir2_sf_to_block(args); + + ASSERT(ifp->if_format == XFS_DINODE_FMT_LOCAL); + ASSERT(dp->i_disk_size >= offsetof(struct xfs_dir2_sf_hdr, parent)); + + ASSERT(ifp->if_bytes == dp->i_disk_size); + ASSERT(oldsfp != NULL); + ASSERT(dp->i_disk_size >= xfs_dir2_sf_hdr_size(oldsfp->i8count)); + ASSERT(dp->i_df.if_nextents == 0); + + /* + * Copy the directory into a temporary buffer. + * Then pitch the incore inode data so we can make extents. + */ + sfp = kmalloc(ifp->if_bytes, GFP_KERNEL | __GFP_NOFAIL); + memcpy(sfp, oldsfp, ifp->if_bytes); + + xfs_idata_realloc(dp, -ifp->if_bytes, XFS_DATA_FORK); + xfs_bmap_local_to_extents_empty(tp, dp, XFS_DATA_FORK); + dp->i_disk_size = 0; + + /* + * Add block 0 to the inode. + */ + error = xfs_dir2_grow_inode(args, XFS_DIR2_DATA_SPACE, &blkno); + if (error) + goto out_free; + /* + * Initialize the data block, then convert it to block format. + */ + error = xfs_dir3_data_init(args, blkno, &bp); + if (error) + goto out_free; + xfs_dir3_block_init(args, bp); + hdr = bp->b_addr; + + /* + * Compute size of block "tail" area. + */ + i = (uint)sizeof(*btp) + + (sfp->count + 2) * (uint)sizeof(xfs_dir2_leaf_entry_t); + /* + * The whole thing is initialized to free by the init routine. + * Say we're using the leaf and tail area. + */ + dup = bp->b_addr + offset; + needlog = needscan = 0; + error = xfs_dir2_data_use_free(args, bp, dup, args->geo->blksize - i, + i, &needlog, &needscan); + if (error) + goto out_free; + ASSERT(needscan == 0); + /* + * Fill in the tail. + */ + btp = xfs_dir2_block_tail_p(args->geo, hdr); + btp->count = cpu_to_be32(sfp->count + 2); /* ., .. */ + btp->stale = 0; + blp = xfs_dir2_block_leaf_p(btp); + endoffset = (uint)((char *)blp - (char *)hdr); + /* + * Remove the freespace, we'll manage it. + */ + error = xfs_dir2_data_use_free(args, bp, dup, + (xfs_dir2_data_aoff_t)((char *)dup - (char *)hdr), + be16_to_cpu(dup->length), &needlog, &needscan); + if (error) + goto out_free; + + /* + * Create entry for . + */ + dep = bp->b_addr + offset; + dep->inumber = cpu_to_be64(args->owner); + dep->namelen = 1; + dep->name[0] = '.'; + xfs_dir2_data_put_ftype(mp, dep, XFS_DIR3_FT_DIR); + tagp = xfs_dir2_data_entry_tag_p(mp, dep); + *tagp = cpu_to_be16(offset); + xfs_dir2_data_log_entry(args, bp, dep); + blp[0].hashval = cpu_to_be32(xfs_dir_hash_dot); + blp[0].address = cpu_to_be32(xfs_dir2_byte_to_dataptr(offset)); + offset += xfs_dir2_data_entsize(mp, dep->namelen); + + /* + * Create entry for .. + */ + dep = bp->b_addr + offset; + dep->inumber = cpu_to_be64(xfs_dir2_sf_get_parent_ino(sfp)); + dep->namelen = 2; + dep->name[0] = dep->name[1] = '.'; + xfs_dir2_data_put_ftype(mp, dep, XFS_DIR3_FT_DIR); + tagp = xfs_dir2_data_entry_tag_p(mp, dep); + *tagp = cpu_to_be16(offset); + xfs_dir2_data_log_entry(args, bp, dep); + blp[1].hashval = cpu_to_be32(xfs_dir_hash_dotdot); + blp[1].address = cpu_to_be32(xfs_dir2_byte_to_dataptr(offset)); + offset += xfs_dir2_data_entsize(mp, dep->namelen); + + /* + * Loop over existing entries, stuff them in. + */ + i = 0; + if (!sfp->count) + sfep = NULL; + else + sfep = xfs_dir2_sf_firstentry(sfp); + + /* + * Need to preserve the existing offset values in the sf directory. + * Insert holes (unused entries) where necessary. + */ + while (offset < endoffset) { + /* + * sfep is null when we reach the end of the list. + */ + if (sfep == NULL) + newoffset = endoffset; + else + newoffset = xfs_dir2_sf_get_offset(sfep); + /* + * There should be a hole here, make one. + */ + if (offset < newoffset) { + dup = bp->b_addr + offset; + dup->freetag = cpu_to_be16(XFS_DIR2_DATA_FREE_TAG); + dup->length = cpu_to_be16(newoffset - offset); + *xfs_dir2_data_unused_tag_p(dup) = cpu_to_be16(offset); + xfs_dir2_data_log_unused(args, bp, dup); + xfs_dir2_data_freeinsert(hdr, + xfs_dir2_data_bestfree_p(mp, hdr), + dup, &dummy); + offset += be16_to_cpu(dup->length); + continue; + } + /* + * Copy a real entry. + */ + dep = bp->b_addr + newoffset; + dep->inumber = cpu_to_be64(xfs_dir2_sf_get_ino(mp, sfp, sfep)); + dep->namelen = sfep->namelen; + xfs_dir2_data_put_ftype(mp, dep, + xfs_dir2_sf_get_ftype(mp, sfep)); + memcpy(dep->name, sfep->name, dep->namelen); + tagp = xfs_dir2_data_entry_tag_p(mp, dep); + *tagp = cpu_to_be16(newoffset); + xfs_dir2_data_log_entry(args, bp, dep); + name.name = sfep->name; + name.len = sfep->namelen; + blp[2 + i].hashval = cpu_to_be32(xfs_dir2_hashname(mp, &name)); + blp[2 + i].address = + cpu_to_be32(xfs_dir2_byte_to_dataptr(newoffset)); + offset = (int)((char *)(tagp + 1) - (char *)hdr); + if (++i == sfp->count) + sfep = NULL; + else + sfep = xfs_dir2_sf_nextentry(mp, sfp, sfep); + } + /* Done with the temporary buffer */ + kfree(sfp); + /* + * Sort the leaf entries by hash value. + */ + xfs_sort(blp, be32_to_cpu(btp->count), sizeof(*blp), xfs_dir2_block_sort); + /* + * Log the leaf entry area and tail. + * Already logged the header in data_init, ignore needlog. + */ + ASSERT(needscan == 0); + xfs_dir2_block_log_leaf(tp, bp, 0, be32_to_cpu(btp->count) - 1); + xfs_dir2_block_log_tail(tp, bp); + xfs_dir3_data_check(dp, bp); + return 0; +out_free: + kfree(sfp); + return error; +} diff --git a/fs/xfs/libxfs/xfs_dir2_data.c b/fs/xfs/libxfs/xfs_dir2_data.c new file mode 100644 index 000000000..aecbab610 --- /dev/null +++ b/fs/xfs/libxfs/xfs_dir2_data.c @@ -0,0 +1,1255 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc. + * Copyright (c) 2013 Red Hat, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_dir2.h" +#include "xfs_dir2_priv.h" +#include "xfs_error.h" +#include "xfs_trans.h" +#include "xfs_buf_item.h" +#include "xfs_log.h" +#include "xfs_health.h" + +static xfs_failaddr_t xfs_dir2_data_freefind_verify( + struct xfs_dir2_data_hdr *hdr, struct xfs_dir2_data_free *bf, + struct xfs_dir2_data_unused *dup, + struct xfs_dir2_data_free **bf_ent); + +struct xfs_dir2_data_free * +xfs_dir2_data_bestfree_p( + struct xfs_mount *mp, + struct xfs_dir2_data_hdr *hdr) +{ + if (xfs_has_crc(mp)) + return ((struct xfs_dir3_data_hdr *)hdr)->best_free; + return hdr->bestfree; +} + +/* + * Pointer to an entry's tag word. + */ +__be16 * +xfs_dir2_data_entry_tag_p( + struct xfs_mount *mp, + struct xfs_dir2_data_entry *dep) +{ + return (__be16 *)((char *)dep + + xfs_dir2_data_entsize(mp, dep->namelen) - sizeof(__be16)); +} + +uint8_t +xfs_dir2_data_get_ftype( + struct xfs_mount *mp, + struct xfs_dir2_data_entry *dep) +{ + if (xfs_has_ftype(mp)) { + uint8_t ftype = dep->name[dep->namelen]; + + if (likely(ftype < XFS_DIR3_FT_MAX)) + return ftype; + } + + return XFS_DIR3_FT_UNKNOWN; +} + +void +xfs_dir2_data_put_ftype( + struct xfs_mount *mp, + struct xfs_dir2_data_entry *dep, + uint8_t ftype) +{ + ASSERT(ftype < XFS_DIR3_FT_MAX); + ASSERT(dep->namelen != 0); + + if (xfs_has_ftype(mp)) + dep->name[dep->namelen] = ftype; +} + +/* + * The number of leaf entries is limited by the size of the block and the amount + * of space used by the data entries. We don't know how much space is used by + * the data entries yet, so just ensure that the count falls somewhere inside + * the block right now. + */ +static inline unsigned int +xfs_dir2_data_max_leaf_entries( + struct xfs_da_geometry *geo) +{ + return (geo->blksize - sizeof(struct xfs_dir2_block_tail) - + geo->data_entry_offset) / + sizeof(struct xfs_dir2_leaf_entry); +} + +/* + * Check the consistency of the data block. + * The input can also be a block-format directory. + * Return NULL if the buffer is good, otherwise the address of the error. + */ +xfs_failaddr_t +__xfs_dir3_data_check( + struct xfs_inode *dp, /* incore inode pointer */ + struct xfs_buf *bp) /* data block's buffer */ +{ + xfs_dir2_dataptr_t addr; /* addr for leaf lookup */ + xfs_dir2_data_free_t *bf; /* bestfree table */ + xfs_dir2_block_tail_t *btp=NULL; /* block tail */ + int count; /* count of entries found */ + xfs_dir2_data_hdr_t *hdr; /* data block header */ + xfs_dir2_data_free_t *dfp; /* bestfree entry */ + int freeseen; /* mask of bestfrees seen */ + xfs_dahash_t hash; /* hash of current name */ + int i; /* leaf index */ + int lastfree; /* last entry was unused */ + xfs_dir2_leaf_entry_t *lep=NULL; /* block leaf entries */ + struct xfs_mount *mp = bp->b_mount; + int stale; /* count of stale leaves */ + struct xfs_name name; + unsigned int offset; + unsigned int end; + struct xfs_da_geometry *geo = mp->m_dir_geo; + + /* + * If this isn't a directory, something is seriously wrong. Bail out. + */ + if (dp && !S_ISDIR(VFS_I(dp)->i_mode)) + return __this_address; + + hdr = bp->b_addr; + offset = geo->data_entry_offset; + + switch (hdr->magic) { + case cpu_to_be32(XFS_DIR3_BLOCK_MAGIC): + case cpu_to_be32(XFS_DIR2_BLOCK_MAGIC): + btp = xfs_dir2_block_tail_p(geo, hdr); + lep = xfs_dir2_block_leaf_p(btp); + + if (be32_to_cpu(btp->count) >= + xfs_dir2_data_max_leaf_entries(geo)) + return __this_address; + break; + case cpu_to_be32(XFS_DIR3_DATA_MAGIC): + case cpu_to_be32(XFS_DIR2_DATA_MAGIC): + break; + default: + return __this_address; + } + end = xfs_dir3_data_end_offset(geo, hdr); + if (!end) + return __this_address; + + /* + * Account for zero bestfree entries. + */ + bf = xfs_dir2_data_bestfree_p(mp, hdr); + count = lastfree = freeseen = 0; + if (!bf[0].length) { + if (bf[0].offset) + return __this_address; + freeseen |= 1 << 0; + } + if (!bf[1].length) { + if (bf[1].offset) + return __this_address; + freeseen |= 1 << 1; + } + if (!bf[2].length) { + if (bf[2].offset) + return __this_address; + freeseen |= 1 << 2; + } + + if (be16_to_cpu(bf[0].length) < be16_to_cpu(bf[1].length)) + return __this_address; + if (be16_to_cpu(bf[1].length) < be16_to_cpu(bf[2].length)) + return __this_address; + /* + * Loop over the data/unused entries. + */ + while (offset < end) { + struct xfs_dir2_data_unused *dup = bp->b_addr + offset; + struct xfs_dir2_data_entry *dep = bp->b_addr + offset; + unsigned int reclen; + + /* + * Are the remaining bytes large enough to hold an + * unused entry? + */ + if (offset > end - xfs_dir2_data_unusedsize(1)) + return __this_address; + + /* + * If it's unused, look for the space in the bestfree table. + * If we find it, account for that, else make sure it + * doesn't need to be there. + */ + if (be16_to_cpu(dup->freetag) == XFS_DIR2_DATA_FREE_TAG) { + xfs_failaddr_t fa; + + reclen = xfs_dir2_data_unusedsize( + be16_to_cpu(dup->length)); + if (lastfree != 0) + return __this_address; + if (be16_to_cpu(dup->length) != reclen) + return __this_address; + if (offset + reclen > end) + return __this_address; + if (be16_to_cpu(*xfs_dir2_data_unused_tag_p(dup)) != + offset) + return __this_address; + fa = xfs_dir2_data_freefind_verify(hdr, bf, dup, &dfp); + if (fa) + return fa; + if (dfp) { + i = (int)(dfp - bf); + if ((freeseen & (1 << i)) != 0) + return __this_address; + freeseen |= 1 << i; + } else { + if (be16_to_cpu(dup->length) > + be16_to_cpu(bf[2].length)) + return __this_address; + } + offset += reclen; + lastfree = 1; + continue; + } + + /* + * This is not an unused entry. Are the remaining bytes + * large enough for a dirent with a single-byte name? + */ + if (offset > end - xfs_dir2_data_entsize(mp, 1)) + return __this_address; + + /* + * It's a real entry. Validate the fields. + * If this is a block directory then make sure it's + * in the leaf section of the block. + * The linear search is crude but this is DEBUG code. + */ + if (dep->namelen == 0) + return __this_address; + reclen = xfs_dir2_data_entsize(mp, dep->namelen); + if (offset + reclen > end) + return __this_address; + if (!xfs_verify_dir_ino(mp, be64_to_cpu(dep->inumber))) + return __this_address; + if (be16_to_cpu(*xfs_dir2_data_entry_tag_p(mp, dep)) != offset) + return __this_address; + if (xfs_dir2_data_get_ftype(mp, dep) >= XFS_DIR3_FT_MAX) + return __this_address; + count++; + lastfree = 0; + if (hdr->magic == cpu_to_be32(XFS_DIR2_BLOCK_MAGIC) || + hdr->magic == cpu_to_be32(XFS_DIR3_BLOCK_MAGIC)) { + addr = xfs_dir2_db_off_to_dataptr(geo, geo->datablk, + (xfs_dir2_data_aoff_t) + ((char *)dep - (char *)hdr)); + name.name = dep->name; + name.len = dep->namelen; + hash = xfs_dir2_hashname(mp, &name); + for (i = 0; i < be32_to_cpu(btp->count); i++) { + if (be32_to_cpu(lep[i].address) == addr && + be32_to_cpu(lep[i].hashval) == hash) + break; + } + if (i >= be32_to_cpu(btp->count)) + return __this_address; + } + offset += reclen; + } + /* + * Need to have seen all the entries and all the bestfree slots. + */ + if (freeseen != 7) + return __this_address; + if (hdr->magic == cpu_to_be32(XFS_DIR2_BLOCK_MAGIC) || + hdr->magic == cpu_to_be32(XFS_DIR3_BLOCK_MAGIC)) { + for (i = stale = 0; i < be32_to_cpu(btp->count); i++) { + if (lep[i].address == + cpu_to_be32(XFS_DIR2_NULL_DATAPTR)) + stale++; + if (i > 0 && be32_to_cpu(lep[i].hashval) < + be32_to_cpu(lep[i - 1].hashval)) + return __this_address; + } + if (count != be32_to_cpu(btp->count) - be32_to_cpu(btp->stale)) + return __this_address; + if (stale != be32_to_cpu(btp->stale)) + return __this_address; + } + return NULL; +} + +#ifdef DEBUG +void +xfs_dir3_data_check( + struct xfs_inode *dp, + struct xfs_buf *bp) +{ + xfs_failaddr_t fa; + + fa = __xfs_dir3_data_check(dp, bp); + if (!fa) + return; + xfs_corruption_error(__func__, XFS_ERRLEVEL_LOW, dp->i_mount, + bp->b_addr, BBTOB(bp->b_length), __FILE__, __LINE__, + fa); + ASSERT(0); +} +#endif + +static xfs_failaddr_t +xfs_dir3_data_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_dir3_blk_hdr *hdr3 = bp->b_addr; + + if (!xfs_verify_magic(bp, hdr3->magic)) + return __this_address; + + if (xfs_has_crc(mp)) { + if (!uuid_equal(&hdr3->uuid, &mp->m_sb.sb_meta_uuid)) + return __this_address; + if (be64_to_cpu(hdr3->blkno) != xfs_buf_daddr(bp)) + return __this_address; + if (!xfs_log_check_lsn(mp, be64_to_cpu(hdr3->lsn))) + return __this_address; + } + return __xfs_dir3_data_check(NULL, bp); +} + +/* + * Readahead of the first block of the directory when it is opened is completely + * oblivious to the format of the directory. Hence we can either get a block + * format buffer or a data format buffer on readahead. + */ +static void +xfs_dir3_data_reada_verify( + struct xfs_buf *bp) +{ + struct xfs_dir2_data_hdr *hdr = bp->b_addr; + + switch (hdr->magic) { + case cpu_to_be32(XFS_DIR2_BLOCK_MAGIC): + case cpu_to_be32(XFS_DIR3_BLOCK_MAGIC): + bp->b_ops = &xfs_dir3_block_buf_ops; + bp->b_ops->verify_read(bp); + return; + case cpu_to_be32(XFS_DIR2_DATA_MAGIC): + case cpu_to_be32(XFS_DIR3_DATA_MAGIC): + bp->b_ops = &xfs_dir3_data_buf_ops; + bp->b_ops->verify_read(bp); + return; + default: + xfs_verifier_error(bp, -EFSCORRUPTED, __this_address); + break; + } +} + +static void +xfs_dir3_data_read_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + xfs_failaddr_t fa; + + if (xfs_has_crc(mp) && + !xfs_buf_verify_cksum(bp, XFS_DIR3_DATA_CRC_OFF)) + xfs_verifier_error(bp, -EFSBADCRC, __this_address); + else { + fa = xfs_dir3_data_verify(bp); + if (fa) + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + } +} + +static void +xfs_dir3_data_write_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_buf_log_item *bip = bp->b_log_item; + struct xfs_dir3_blk_hdr *hdr3 = bp->b_addr; + struct xfs_dir3_data_hdr *datahdr3 = bp->b_addr; + xfs_failaddr_t fa; + + fa = xfs_dir3_data_verify(bp); + if (fa) { + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + return; + } + + if (!xfs_has_crc(mp)) + return; + + if (bip) + hdr3->lsn = cpu_to_be64(bip->bli_item.li_lsn); + + /* + * Zero padding that may be stale from old kernels. + */ + datahdr3->pad = 0; + + xfs_buf_update_cksum(bp, XFS_DIR3_DATA_CRC_OFF); +} + +const struct xfs_buf_ops xfs_dir3_data_buf_ops = { + .name = "xfs_dir3_data", + .magic = { cpu_to_be32(XFS_DIR2_DATA_MAGIC), + cpu_to_be32(XFS_DIR3_DATA_MAGIC) }, + .verify_read = xfs_dir3_data_read_verify, + .verify_write = xfs_dir3_data_write_verify, + .verify_struct = xfs_dir3_data_verify, +}; + +static const struct xfs_buf_ops xfs_dir3_data_reada_buf_ops = { + .name = "xfs_dir3_data_reada", + .magic = { cpu_to_be32(XFS_DIR2_DATA_MAGIC), + cpu_to_be32(XFS_DIR3_DATA_MAGIC) }, + .verify_read = xfs_dir3_data_reada_verify, + .verify_write = xfs_dir3_data_write_verify, +}; + +xfs_failaddr_t +xfs_dir3_data_header_check( + struct xfs_buf *bp, + xfs_ino_t owner) +{ + struct xfs_mount *mp = bp->b_mount; + + if (xfs_has_crc(mp)) { + struct xfs_dir3_data_hdr *hdr3 = bp->b_addr; + + if (hdr3->hdr.magic != cpu_to_be32(XFS_DIR3_DATA_MAGIC)) + return __this_address; + + if (be64_to_cpu(hdr3->hdr.owner) != owner) + return __this_address; + } + + return NULL; +} + +int +xfs_dir3_data_read( + struct xfs_trans *tp, + struct xfs_inode *dp, + xfs_ino_t owner, + xfs_dablk_t bno, + unsigned int flags, + struct xfs_buf **bpp) +{ + xfs_failaddr_t fa; + int err; + + err = xfs_da_read_buf(tp, dp, bno, flags, bpp, XFS_DATA_FORK, + &xfs_dir3_data_buf_ops); + if (err || !*bpp) + return err; + + /* Check things that we can't do in the verifier. */ + fa = xfs_dir3_data_header_check(*bpp, owner); + if (fa) { + __xfs_buf_mark_corrupt(*bpp, fa); + xfs_trans_brelse(tp, *bpp); + *bpp = NULL; + xfs_dirattr_mark_sick(dp, XFS_DATA_FORK); + return -EFSCORRUPTED; + } + + xfs_trans_buf_set_type(tp, *bpp, XFS_BLFT_DIR_DATA_BUF); + return err; +} + +int +xfs_dir3_data_readahead( + struct xfs_inode *dp, + xfs_dablk_t bno, + unsigned int flags) +{ + return xfs_da_reada_buf(dp, bno, flags, XFS_DATA_FORK, + &xfs_dir3_data_reada_buf_ops); +} + +/* + * Find the bestfree entry that exactly coincides with unused directory space + * or a verifier error because the bestfree data are bad. + */ +static xfs_failaddr_t +xfs_dir2_data_freefind_verify( + struct xfs_dir2_data_hdr *hdr, + struct xfs_dir2_data_free *bf, + struct xfs_dir2_data_unused *dup, + struct xfs_dir2_data_free **bf_ent) +{ + struct xfs_dir2_data_free *dfp; + xfs_dir2_data_aoff_t off; + bool matched = false; + bool seenzero = false; + + *bf_ent = NULL; + off = (xfs_dir2_data_aoff_t)((char *)dup - (char *)hdr); + + /* + * Validate some consistency in the bestfree table. + * Check order, non-overlapping entries, and if we find the + * one we're looking for it has to be exact. + */ + for (dfp = &bf[0]; dfp < &bf[XFS_DIR2_DATA_FD_COUNT]; dfp++) { + if (!dfp->offset) { + if (dfp->length) + return __this_address; + seenzero = true; + continue; + } + if (seenzero) + return __this_address; + if (be16_to_cpu(dfp->offset) == off) { + matched = true; + if (dfp->length != dup->length) + return __this_address; + } else if (be16_to_cpu(dfp->offset) > off) { + if (off + be16_to_cpu(dup->length) > + be16_to_cpu(dfp->offset)) + return __this_address; + } else { + if (be16_to_cpu(dfp->offset) + + be16_to_cpu(dfp->length) > off) + return __this_address; + } + if (!matched && + be16_to_cpu(dfp->length) < be16_to_cpu(dup->length)) + return __this_address; + if (dfp > &bf[0] && + be16_to_cpu(dfp[-1].length) < be16_to_cpu(dfp[0].length)) + return __this_address; + } + + /* Looks ok so far; now try to match up with a bestfree entry. */ + *bf_ent = xfs_dir2_data_freefind(hdr, bf, dup); + return NULL; +} + +/* + * Given a data block and an unused entry from that block, + * return the bestfree entry if any that corresponds to it. + */ +xfs_dir2_data_free_t * +xfs_dir2_data_freefind( + struct xfs_dir2_data_hdr *hdr, /* data block header */ + struct xfs_dir2_data_free *bf, /* bestfree table pointer */ + struct xfs_dir2_data_unused *dup) /* unused space */ +{ + xfs_dir2_data_free_t *dfp; /* bestfree entry */ + xfs_dir2_data_aoff_t off; /* offset value needed */ + + off = (xfs_dir2_data_aoff_t)((char *)dup - (char *)hdr); + + /* + * If this is smaller than the smallest bestfree entry, + * it can't be there since they're sorted. + */ + if (be16_to_cpu(dup->length) < + be16_to_cpu(bf[XFS_DIR2_DATA_FD_COUNT - 1].length)) + return NULL; + /* + * Look at the three bestfree entries for our guy. + */ + for (dfp = &bf[0]; dfp < &bf[XFS_DIR2_DATA_FD_COUNT]; dfp++) { + if (!dfp->offset) + return NULL; + if (be16_to_cpu(dfp->offset) == off) + return dfp; + } + /* + * Didn't find it. This only happens if there are duplicate lengths. + */ + return NULL; +} + +/* + * Insert an unused-space entry into the bestfree table. + */ +xfs_dir2_data_free_t * /* entry inserted */ +xfs_dir2_data_freeinsert( + struct xfs_dir2_data_hdr *hdr, /* data block pointer */ + struct xfs_dir2_data_free *dfp, /* bestfree table pointer */ + struct xfs_dir2_data_unused *dup, /* unused space */ + int *loghead) /* log the data header (out) */ +{ + xfs_dir2_data_free_t new; /* new bestfree entry */ + + ASSERT(hdr->magic == cpu_to_be32(XFS_DIR2_DATA_MAGIC) || + hdr->magic == cpu_to_be32(XFS_DIR2_BLOCK_MAGIC) || + hdr->magic == cpu_to_be32(XFS_DIR3_DATA_MAGIC) || + hdr->magic == cpu_to_be32(XFS_DIR3_BLOCK_MAGIC)); + + new.length = dup->length; + new.offset = cpu_to_be16((char *)dup - (char *)hdr); + + /* + * Insert at position 0, 1, or 2; or not at all. + */ + if (be16_to_cpu(new.length) > be16_to_cpu(dfp[0].length)) { + dfp[2] = dfp[1]; + dfp[1] = dfp[0]; + dfp[0] = new; + *loghead = 1; + return &dfp[0]; + } + if (be16_to_cpu(new.length) > be16_to_cpu(dfp[1].length)) { + dfp[2] = dfp[1]; + dfp[1] = new; + *loghead = 1; + return &dfp[1]; + } + if (be16_to_cpu(new.length) > be16_to_cpu(dfp[2].length)) { + dfp[2] = new; + *loghead = 1; + return &dfp[2]; + } + return NULL; +} + +/* + * Remove a bestfree entry from the table. + */ +STATIC void +xfs_dir2_data_freeremove( + struct xfs_dir2_data_hdr *hdr, /* data block header */ + struct xfs_dir2_data_free *bf, /* bestfree table pointer */ + struct xfs_dir2_data_free *dfp, /* bestfree entry pointer */ + int *loghead) /* out: log data header */ +{ + + ASSERT(hdr->magic == cpu_to_be32(XFS_DIR2_DATA_MAGIC) || + hdr->magic == cpu_to_be32(XFS_DIR2_BLOCK_MAGIC) || + hdr->magic == cpu_to_be32(XFS_DIR3_DATA_MAGIC) || + hdr->magic == cpu_to_be32(XFS_DIR3_BLOCK_MAGIC)); + + /* + * It's the first entry, slide the next 2 up. + */ + if (dfp == &bf[0]) { + bf[0] = bf[1]; + bf[1] = bf[2]; + } + /* + * It's the second entry, slide the 3rd entry up. + */ + else if (dfp == &bf[1]) + bf[1] = bf[2]; + /* + * Must be the last entry. + */ + else + ASSERT(dfp == &bf[2]); + /* + * Clear the 3rd entry, must be zero now. + */ + bf[2].length = 0; + bf[2].offset = 0; + *loghead = 1; +} + +/* + * Given a data block, reconstruct its bestfree map. + */ +void +xfs_dir2_data_freescan( + struct xfs_mount *mp, + struct xfs_dir2_data_hdr *hdr, + int *loghead) +{ + struct xfs_da_geometry *geo = mp->m_dir_geo; + struct xfs_dir2_data_free *bf = xfs_dir2_data_bestfree_p(mp, hdr); + void *addr = hdr; + unsigned int offset = geo->data_entry_offset; + unsigned int end; + + ASSERT(hdr->magic == cpu_to_be32(XFS_DIR2_DATA_MAGIC) || + hdr->magic == cpu_to_be32(XFS_DIR3_DATA_MAGIC) || + hdr->magic == cpu_to_be32(XFS_DIR2_BLOCK_MAGIC) || + hdr->magic == cpu_to_be32(XFS_DIR3_BLOCK_MAGIC)); + + /* + * Start by clearing the table. + */ + memset(bf, 0, sizeof(*bf) * XFS_DIR2_DATA_FD_COUNT); + *loghead = 1; + + end = xfs_dir3_data_end_offset(geo, addr); + while (offset < end) { + struct xfs_dir2_data_unused *dup = addr + offset; + struct xfs_dir2_data_entry *dep = addr + offset; + + /* + * If it's a free entry, insert it. + */ + if (be16_to_cpu(dup->freetag) == XFS_DIR2_DATA_FREE_TAG) { + ASSERT(offset == + be16_to_cpu(*xfs_dir2_data_unused_tag_p(dup))); + xfs_dir2_data_freeinsert(hdr, bf, dup, loghead); + offset += be16_to_cpu(dup->length); + continue; + } + + /* + * For active entries, check their tags and skip them. + */ + ASSERT(offset == + be16_to_cpu(*xfs_dir2_data_entry_tag_p(mp, dep))); + offset += xfs_dir2_data_entsize(mp, dep->namelen); + } +} + +/* + * Initialize a data block at the given block number in the directory. + * Give back the buffer for the created block. + */ +int /* error */ +xfs_dir3_data_init( + struct xfs_da_args *args, /* directory operation args */ + xfs_dir2_db_t blkno, /* logical dir block number */ + struct xfs_buf **bpp) /* output block buffer */ +{ + struct xfs_trans *tp = args->trans; + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + struct xfs_da_geometry *geo = args->geo; + struct xfs_buf *bp; + struct xfs_dir2_data_hdr *hdr; + struct xfs_dir2_data_unused *dup; + struct xfs_dir2_data_free *bf; + int error; + + /* + * Get the buffer set up for the block. + */ + error = xfs_da_get_buf(tp, dp, xfs_dir2_db_to_da(args->geo, blkno), + &bp, XFS_DATA_FORK); + if (error) + return error; + bp->b_ops = &xfs_dir3_data_buf_ops; + xfs_trans_buf_set_type(tp, bp, XFS_BLFT_DIR_DATA_BUF); + + /* + * Initialize the whole directory header region to zero + * so that all padding, bestfree entries, and any + * future header fields are clean. + */ + hdr = bp->b_addr; + memset(hdr, 0, geo->data_entry_offset); + + if (xfs_has_crc(mp)) { + struct xfs_dir3_blk_hdr *hdr3 = bp->b_addr; + + hdr3->magic = cpu_to_be32(XFS_DIR3_DATA_MAGIC); + hdr3->blkno = cpu_to_be64(xfs_buf_daddr(bp)); + hdr3->owner = cpu_to_be64(args->owner); + uuid_copy(&hdr3->uuid, &mp->m_sb.sb_meta_uuid); + + } else + hdr->magic = cpu_to_be32(XFS_DIR2_DATA_MAGIC); + + bf = xfs_dir2_data_bestfree_p(mp, hdr); + bf[0].offset = cpu_to_be16(geo->data_entry_offset); + bf[0].length = cpu_to_be16(geo->blksize - geo->data_entry_offset); + + /* + * Set up an unused entry for the block's body. + */ + dup = bp->b_addr + geo->data_entry_offset; + dup->freetag = cpu_to_be16(XFS_DIR2_DATA_FREE_TAG); + dup->length = bf[0].length; + *xfs_dir2_data_unused_tag_p(dup) = cpu_to_be16((char *)dup - (char *)hdr); + + /* + * Log it and return it. + */ + xfs_dir2_data_log_header(args, bp); + xfs_dir2_data_log_unused(args, bp, dup); + *bpp = bp; + return 0; +} + +/* + * Log an active data entry from the block. + */ +void +xfs_dir2_data_log_entry( + struct xfs_da_args *args, + struct xfs_buf *bp, + xfs_dir2_data_entry_t *dep) /* data entry pointer */ +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_dir2_data_hdr *hdr = bp->b_addr; + + ASSERT(hdr->magic == cpu_to_be32(XFS_DIR2_DATA_MAGIC) || + hdr->magic == cpu_to_be32(XFS_DIR3_DATA_MAGIC) || + hdr->magic == cpu_to_be32(XFS_DIR2_BLOCK_MAGIC) || + hdr->magic == cpu_to_be32(XFS_DIR3_BLOCK_MAGIC)); + + xfs_trans_log_buf(args->trans, bp, (uint)((char *)dep - (char *)hdr), + (uint)((char *)(xfs_dir2_data_entry_tag_p(mp, dep) + 1) - + (char *)hdr - 1)); +} + +/* + * Log a data block header. + */ +void +xfs_dir2_data_log_header( + struct xfs_da_args *args, + struct xfs_buf *bp) +{ +#ifdef DEBUG + struct xfs_dir2_data_hdr *hdr = bp->b_addr; + + ASSERT(hdr->magic == cpu_to_be32(XFS_DIR2_DATA_MAGIC) || + hdr->magic == cpu_to_be32(XFS_DIR3_DATA_MAGIC) || + hdr->magic == cpu_to_be32(XFS_DIR2_BLOCK_MAGIC) || + hdr->magic == cpu_to_be32(XFS_DIR3_BLOCK_MAGIC)); +#endif + + xfs_trans_log_buf(args->trans, bp, 0, args->geo->data_entry_offset - 1); +} + +/* + * Log a data unused entry. + */ +void +xfs_dir2_data_log_unused( + struct xfs_da_args *args, + struct xfs_buf *bp, + xfs_dir2_data_unused_t *dup) /* data unused pointer */ +{ + xfs_dir2_data_hdr_t *hdr = bp->b_addr; + + ASSERT(hdr->magic == cpu_to_be32(XFS_DIR2_DATA_MAGIC) || + hdr->magic == cpu_to_be32(XFS_DIR3_DATA_MAGIC) || + hdr->magic == cpu_to_be32(XFS_DIR2_BLOCK_MAGIC) || + hdr->magic == cpu_to_be32(XFS_DIR3_BLOCK_MAGIC)); + + /* + * Log the first part of the unused entry. + */ + xfs_trans_log_buf(args->trans, bp, (uint)((char *)dup - (char *)hdr), + (uint)((char *)&dup->length + sizeof(dup->length) - + 1 - (char *)hdr)); + /* + * Log the end (tag) of the unused entry. + */ + xfs_trans_log_buf(args->trans, bp, + (uint)((char *)xfs_dir2_data_unused_tag_p(dup) - (char *)hdr), + (uint)((char *)xfs_dir2_data_unused_tag_p(dup) - (char *)hdr + + sizeof(xfs_dir2_data_off_t) - 1)); +} + +/* + * Make a byte range in the data block unused. + * Its current contents are unimportant. + */ +void +xfs_dir2_data_make_free( + struct xfs_da_args *args, + struct xfs_buf *bp, + xfs_dir2_data_aoff_t offset, /* starting byte offset */ + xfs_dir2_data_aoff_t len, /* length in bytes */ + int *needlogp, /* out: log header */ + int *needscanp) /* out: regen bestfree */ +{ + xfs_dir2_data_hdr_t *hdr; /* data block pointer */ + xfs_dir2_data_free_t *dfp; /* bestfree pointer */ + int needscan; /* need to regen bestfree */ + xfs_dir2_data_unused_t *newdup; /* new unused entry */ + xfs_dir2_data_unused_t *postdup; /* unused entry after us */ + xfs_dir2_data_unused_t *prevdup; /* unused entry before us */ + unsigned int end; + struct xfs_dir2_data_free *bf; + + hdr = bp->b_addr; + + /* + * Figure out where the end of the data area is. + */ + end = xfs_dir3_data_end_offset(args->geo, hdr); + ASSERT(end != 0); + + /* + * If this isn't the start of the block, then back up to + * the previous entry and see if it's free. + */ + if (offset > args->geo->data_entry_offset) { + __be16 *tagp; /* tag just before us */ + + tagp = (__be16 *)((char *)hdr + offset) - 1; + prevdup = (xfs_dir2_data_unused_t *)((char *)hdr + be16_to_cpu(*tagp)); + if (be16_to_cpu(prevdup->freetag) != XFS_DIR2_DATA_FREE_TAG) + prevdup = NULL; + } else + prevdup = NULL; + /* + * If this isn't the end of the block, see if the entry after + * us is free. + */ + if (offset + len < end) { + postdup = + (xfs_dir2_data_unused_t *)((char *)hdr + offset + len); + if (be16_to_cpu(postdup->freetag) != XFS_DIR2_DATA_FREE_TAG) + postdup = NULL; + } else + postdup = NULL; + ASSERT(*needscanp == 0); + needscan = 0; + /* + * Previous and following entries are both free, + * merge everything into a single free entry. + */ + bf = xfs_dir2_data_bestfree_p(args->dp->i_mount, hdr); + if (prevdup && postdup) { + xfs_dir2_data_free_t *dfp2; /* another bestfree pointer */ + + /* + * See if prevdup and/or postdup are in bestfree table. + */ + dfp = xfs_dir2_data_freefind(hdr, bf, prevdup); + dfp2 = xfs_dir2_data_freefind(hdr, bf, postdup); + /* + * We need a rescan unless there are exactly 2 free entries + * namely our two. Then we know what's happening, otherwise + * since the third bestfree is there, there might be more + * entries. + */ + needscan = (bf[2].length != 0); + /* + * Fix up the new big freespace. + */ + be16_add_cpu(&prevdup->length, len + be16_to_cpu(postdup->length)); + *xfs_dir2_data_unused_tag_p(prevdup) = + cpu_to_be16((char *)prevdup - (char *)hdr); + xfs_dir2_data_log_unused(args, bp, prevdup); + if (!needscan) { + /* + * Has to be the case that entries 0 and 1 are + * dfp and dfp2 (don't know which is which), and + * entry 2 is empty. + * Remove entry 1 first then entry 0. + */ + ASSERT(dfp && dfp2); + if (dfp == &bf[1]) { + dfp = &bf[0]; + ASSERT(dfp2 == dfp); + dfp2 = &bf[1]; + } + xfs_dir2_data_freeremove(hdr, bf, dfp2, needlogp); + xfs_dir2_data_freeremove(hdr, bf, dfp, needlogp); + /* + * Now insert the new entry. + */ + dfp = xfs_dir2_data_freeinsert(hdr, bf, prevdup, + needlogp); + ASSERT(dfp == &bf[0]); + ASSERT(dfp->length == prevdup->length); + ASSERT(!dfp[1].length); + ASSERT(!dfp[2].length); + } + } + /* + * The entry before us is free, merge with it. + */ + else if (prevdup) { + dfp = xfs_dir2_data_freefind(hdr, bf, prevdup); + be16_add_cpu(&prevdup->length, len); + *xfs_dir2_data_unused_tag_p(prevdup) = + cpu_to_be16((char *)prevdup - (char *)hdr); + xfs_dir2_data_log_unused(args, bp, prevdup); + /* + * If the previous entry was in the table, the new entry + * is longer, so it will be in the table too. Remove + * the old one and add the new one. + */ + if (dfp) { + xfs_dir2_data_freeremove(hdr, bf, dfp, needlogp); + xfs_dir2_data_freeinsert(hdr, bf, prevdup, needlogp); + } + /* + * Otherwise we need a scan if the new entry is big enough. + */ + else { + needscan = be16_to_cpu(prevdup->length) > + be16_to_cpu(bf[2].length); + } + } + /* + * The following entry is free, merge with it. + */ + else if (postdup) { + dfp = xfs_dir2_data_freefind(hdr, bf, postdup); + newdup = (xfs_dir2_data_unused_t *)((char *)hdr + offset); + newdup->freetag = cpu_to_be16(XFS_DIR2_DATA_FREE_TAG); + newdup->length = cpu_to_be16(len + be16_to_cpu(postdup->length)); + *xfs_dir2_data_unused_tag_p(newdup) = + cpu_to_be16((char *)newdup - (char *)hdr); + xfs_dir2_data_log_unused(args, bp, newdup); + /* + * If the following entry was in the table, the new entry + * is longer, so it will be in the table too. Remove + * the old one and add the new one. + */ + if (dfp) { + xfs_dir2_data_freeremove(hdr, bf, dfp, needlogp); + xfs_dir2_data_freeinsert(hdr, bf, newdup, needlogp); + } + /* + * Otherwise we need a scan if the new entry is big enough. + */ + else { + needscan = be16_to_cpu(newdup->length) > + be16_to_cpu(bf[2].length); + } + } + /* + * Neither neighbor is free. Make a new entry. + */ + else { + newdup = (xfs_dir2_data_unused_t *)((char *)hdr + offset); + newdup->freetag = cpu_to_be16(XFS_DIR2_DATA_FREE_TAG); + newdup->length = cpu_to_be16(len); + *xfs_dir2_data_unused_tag_p(newdup) = + cpu_to_be16((char *)newdup - (char *)hdr); + xfs_dir2_data_log_unused(args, bp, newdup); + xfs_dir2_data_freeinsert(hdr, bf, newdup, needlogp); + } + *needscanp = needscan; +} + +/* Check our free data for obvious signs of corruption. */ +static inline xfs_failaddr_t +xfs_dir2_data_check_free( + struct xfs_dir2_data_hdr *hdr, + struct xfs_dir2_data_unused *dup, + xfs_dir2_data_aoff_t offset, + xfs_dir2_data_aoff_t len) +{ + if (hdr->magic != cpu_to_be32(XFS_DIR2_DATA_MAGIC) && + hdr->magic != cpu_to_be32(XFS_DIR3_DATA_MAGIC) && + hdr->magic != cpu_to_be32(XFS_DIR2_BLOCK_MAGIC) && + hdr->magic != cpu_to_be32(XFS_DIR3_BLOCK_MAGIC)) + return __this_address; + if (be16_to_cpu(dup->freetag) != XFS_DIR2_DATA_FREE_TAG) + return __this_address; + if (offset < (char *)dup - (char *)hdr) + return __this_address; + if (offset + len > (char *)dup + be16_to_cpu(dup->length) - (char *)hdr) + return __this_address; + if ((char *)dup - (char *)hdr != + be16_to_cpu(*xfs_dir2_data_unused_tag_p(dup))) + return __this_address; + return NULL; +} + +/* Sanity-check a new bestfree entry. */ +static inline xfs_failaddr_t +xfs_dir2_data_check_new_free( + struct xfs_dir2_data_hdr *hdr, + struct xfs_dir2_data_free *dfp, + struct xfs_dir2_data_unused *newdup) +{ + if (dfp == NULL) + return __this_address; + if (dfp->length != newdup->length) + return __this_address; + if (be16_to_cpu(dfp->offset) != (char *)newdup - (char *)hdr) + return __this_address; + return NULL; +} + +/* + * Take a byte range out of an existing unused space and make it un-free. + */ +int +xfs_dir2_data_use_free( + struct xfs_da_args *args, + struct xfs_buf *bp, + xfs_dir2_data_unused_t *dup, /* unused entry */ + xfs_dir2_data_aoff_t offset, /* starting offset to use */ + xfs_dir2_data_aoff_t len, /* length to use */ + int *needlogp, /* out: need to log header */ + int *needscanp) /* out: need regen bestfree */ +{ + xfs_dir2_data_hdr_t *hdr; /* data block header */ + xfs_dir2_data_free_t *dfp; /* bestfree pointer */ + xfs_dir2_data_unused_t *newdup; /* new unused entry */ + xfs_dir2_data_unused_t *newdup2; /* another new unused entry */ + struct xfs_dir2_data_free *bf; + xfs_failaddr_t fa; + int matchback; /* matches end of freespace */ + int matchfront; /* matches start of freespace */ + int needscan; /* need to regen bestfree */ + int oldlen; /* old unused entry's length */ + + hdr = bp->b_addr; + fa = xfs_dir2_data_check_free(hdr, dup, offset, len); + if (fa) + goto corrupt; + /* + * Look up the entry in the bestfree table. + */ + oldlen = be16_to_cpu(dup->length); + bf = xfs_dir2_data_bestfree_p(args->dp->i_mount, hdr); + dfp = xfs_dir2_data_freefind(hdr, bf, dup); + ASSERT(dfp || oldlen <= be16_to_cpu(bf[2].length)); + /* + * Check for alignment with front and back of the entry. + */ + matchfront = (char *)dup - (char *)hdr == offset; + matchback = (char *)dup + oldlen - (char *)hdr == offset + len; + ASSERT(*needscanp == 0); + needscan = 0; + /* + * If we matched it exactly we just need to get rid of it from + * the bestfree table. + */ + if (matchfront && matchback) { + if (dfp) { + needscan = (bf[2].offset != 0); + if (!needscan) + xfs_dir2_data_freeremove(hdr, bf, dfp, + needlogp); + } + } + /* + * We match the first part of the entry. + * Make a new entry with the remaining freespace. + */ + else if (matchfront) { + newdup = (xfs_dir2_data_unused_t *)((char *)hdr + offset + len); + newdup->freetag = cpu_to_be16(XFS_DIR2_DATA_FREE_TAG); + newdup->length = cpu_to_be16(oldlen - len); + *xfs_dir2_data_unused_tag_p(newdup) = + cpu_to_be16((char *)newdup - (char *)hdr); + xfs_dir2_data_log_unused(args, bp, newdup); + /* + * If it was in the table, remove it and add the new one. + */ + if (dfp) { + xfs_dir2_data_freeremove(hdr, bf, dfp, needlogp); + dfp = xfs_dir2_data_freeinsert(hdr, bf, newdup, + needlogp); + fa = xfs_dir2_data_check_new_free(hdr, dfp, newdup); + if (fa) + goto corrupt; + /* + * If we got inserted at the last slot, + * that means we don't know if there was a better + * choice for the last slot, or not. Rescan. + */ + needscan = dfp == &bf[2]; + } + } + /* + * We match the last part of the entry. + * Trim the allocated space off the tail of the entry. + */ + else if (matchback) { + newdup = dup; + newdup->length = cpu_to_be16(((char *)hdr + offset) - (char *)newdup); + *xfs_dir2_data_unused_tag_p(newdup) = + cpu_to_be16((char *)newdup - (char *)hdr); + xfs_dir2_data_log_unused(args, bp, newdup); + /* + * If it was in the table, remove it and add the new one. + */ + if (dfp) { + xfs_dir2_data_freeremove(hdr, bf, dfp, needlogp); + dfp = xfs_dir2_data_freeinsert(hdr, bf, newdup, + needlogp); + fa = xfs_dir2_data_check_new_free(hdr, dfp, newdup); + if (fa) + goto corrupt; + /* + * If we got inserted at the last slot, + * that means we don't know if there was a better + * choice for the last slot, or not. Rescan. + */ + needscan = dfp == &bf[2]; + } + } + /* + * Poking out the middle of an entry. + * Make two new entries. + */ + else { + newdup = dup; + newdup->length = cpu_to_be16(((char *)hdr + offset) - (char *)newdup); + *xfs_dir2_data_unused_tag_p(newdup) = + cpu_to_be16((char *)newdup - (char *)hdr); + xfs_dir2_data_log_unused(args, bp, newdup); + newdup2 = (xfs_dir2_data_unused_t *)((char *)hdr + offset + len); + newdup2->freetag = cpu_to_be16(XFS_DIR2_DATA_FREE_TAG); + newdup2->length = cpu_to_be16(oldlen - len - be16_to_cpu(newdup->length)); + *xfs_dir2_data_unused_tag_p(newdup2) = + cpu_to_be16((char *)newdup2 - (char *)hdr); + xfs_dir2_data_log_unused(args, bp, newdup2); + /* + * If the old entry was in the table, we need to scan + * if the 3rd entry was valid, since these entries + * are smaller than the old one. + * If we don't need to scan that means there were 1 or 2 + * entries in the table, and removing the old and adding + * the 2 new will work. + */ + if (dfp) { + needscan = (bf[2].length != 0); + if (!needscan) { + xfs_dir2_data_freeremove(hdr, bf, dfp, + needlogp); + xfs_dir2_data_freeinsert(hdr, bf, newdup, + needlogp); + xfs_dir2_data_freeinsert(hdr, bf, newdup2, + needlogp); + } + } + } + *needscanp = needscan; + return 0; +corrupt: + xfs_corruption_error(__func__, XFS_ERRLEVEL_LOW, args->dp->i_mount, + hdr, sizeof(*hdr), __FILE__, __LINE__, fa); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; +} + +/* Find the end of the entry data in a data/block format dir block. */ +unsigned int +xfs_dir3_data_end_offset( + struct xfs_da_geometry *geo, + struct xfs_dir2_data_hdr *hdr) +{ + void *p; + + switch (hdr->magic) { + case cpu_to_be32(XFS_DIR3_BLOCK_MAGIC): + case cpu_to_be32(XFS_DIR2_BLOCK_MAGIC): + p = xfs_dir2_block_leaf_p(xfs_dir2_block_tail_p(geo, hdr)); + return p - (void *)hdr; + case cpu_to_be32(XFS_DIR3_DATA_MAGIC): + case cpu_to_be32(XFS_DIR2_DATA_MAGIC): + return geo->blksize; + default: + return 0; + } +} diff --git a/fs/xfs/libxfs/xfs_dir2_leaf.c b/fs/xfs/libxfs/xfs_dir2_leaf.c new file mode 100644 index 000000000..bc909543e --- /dev/null +++ b/fs/xfs/libxfs/xfs_dir2_leaf.c @@ -0,0 +1,1881 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2003,2005 Silicon Graphics, Inc. + * Copyright (c) 2013 Red Hat, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_bmap.h" +#include "xfs_dir2.h" +#include "xfs_dir2_priv.h" +#include "xfs_error.h" +#include "xfs_trace.h" +#include "xfs_trans.h" +#include "xfs_buf_item.h" +#include "xfs_health.h" + +/* + * Local function declarations. + */ +static int xfs_dir2_leaf_lookup_int(xfs_da_args_t *args, struct xfs_buf **lbpp, + int *indexp, struct xfs_buf **dbpp, + struct xfs_dir3_icleaf_hdr *leafhdr); +static void xfs_dir3_leaf_log_bests(struct xfs_da_args *args, + struct xfs_buf *bp, int first, int last); +static void xfs_dir3_leaf_log_tail(struct xfs_da_args *args, + struct xfs_buf *bp); + +void +xfs_dir2_leaf_hdr_from_disk( + struct xfs_mount *mp, + struct xfs_dir3_icleaf_hdr *to, + struct xfs_dir2_leaf *from) +{ + if (xfs_has_crc(mp)) { + struct xfs_dir3_leaf *from3 = (struct xfs_dir3_leaf *)from; + + to->forw = be32_to_cpu(from3->hdr.info.hdr.forw); + to->back = be32_to_cpu(from3->hdr.info.hdr.back); + to->magic = be16_to_cpu(from3->hdr.info.hdr.magic); + to->count = be16_to_cpu(from3->hdr.count); + to->stale = be16_to_cpu(from3->hdr.stale); + to->ents = from3->__ents; + + ASSERT(to->magic == XFS_DIR3_LEAF1_MAGIC || + to->magic == XFS_DIR3_LEAFN_MAGIC); + } else { + to->forw = be32_to_cpu(from->hdr.info.forw); + to->back = be32_to_cpu(from->hdr.info.back); + to->magic = be16_to_cpu(from->hdr.info.magic); + to->count = be16_to_cpu(from->hdr.count); + to->stale = be16_to_cpu(from->hdr.stale); + to->ents = from->__ents; + + ASSERT(to->magic == XFS_DIR2_LEAF1_MAGIC || + to->magic == XFS_DIR2_LEAFN_MAGIC); + } +} + +void +xfs_dir2_leaf_hdr_to_disk( + struct xfs_mount *mp, + struct xfs_dir2_leaf *to, + struct xfs_dir3_icleaf_hdr *from) +{ + if (xfs_has_crc(mp)) { + struct xfs_dir3_leaf *to3 = (struct xfs_dir3_leaf *)to; + + ASSERT(from->magic == XFS_DIR3_LEAF1_MAGIC || + from->magic == XFS_DIR3_LEAFN_MAGIC); + + to3->hdr.info.hdr.forw = cpu_to_be32(from->forw); + to3->hdr.info.hdr.back = cpu_to_be32(from->back); + to3->hdr.info.hdr.magic = cpu_to_be16(from->magic); + to3->hdr.count = cpu_to_be16(from->count); + to3->hdr.stale = cpu_to_be16(from->stale); + } else { + ASSERT(from->magic == XFS_DIR2_LEAF1_MAGIC || + from->magic == XFS_DIR2_LEAFN_MAGIC); + + to->hdr.info.forw = cpu_to_be32(from->forw); + to->hdr.info.back = cpu_to_be32(from->back); + to->hdr.info.magic = cpu_to_be16(from->magic); + to->hdr.count = cpu_to_be16(from->count); + to->hdr.stale = cpu_to_be16(from->stale); + } +} + +/* + * Check the internal consistency of a leaf1 block. + * Pop an assert if something is wrong. + */ +#ifdef DEBUG +static xfs_failaddr_t +xfs_dir3_leaf1_check( + struct xfs_inode *dp, + struct xfs_buf *bp) +{ + struct xfs_dir2_leaf *leaf = bp->b_addr; + struct xfs_dir3_icleaf_hdr leafhdr; + + xfs_dir2_leaf_hdr_from_disk(dp->i_mount, &leafhdr, leaf); + + if (leafhdr.magic == XFS_DIR3_LEAF1_MAGIC) { + struct xfs_dir3_leaf_hdr *leaf3 = bp->b_addr; + if (be64_to_cpu(leaf3->info.blkno) != xfs_buf_daddr(bp)) + return __this_address; + } else if (leafhdr.magic != XFS_DIR2_LEAF1_MAGIC) + return __this_address; + + return xfs_dir3_leaf_check_int(dp->i_mount, &leafhdr, leaf, false); +} + +static inline void +xfs_dir3_leaf_check( + struct xfs_inode *dp, + struct xfs_buf *bp) +{ + xfs_failaddr_t fa; + + fa = xfs_dir3_leaf1_check(dp, bp); + if (!fa) + return; + xfs_corruption_error(__func__, XFS_ERRLEVEL_LOW, dp->i_mount, + bp->b_addr, BBTOB(bp->b_length), __FILE__, __LINE__, + fa); + ASSERT(0); +} +#else +#define xfs_dir3_leaf_check(dp, bp) +#endif + +xfs_failaddr_t +xfs_dir3_leaf_check_int( + struct xfs_mount *mp, + struct xfs_dir3_icleaf_hdr *hdr, + struct xfs_dir2_leaf *leaf, + bool expensive_checking) +{ + struct xfs_da_geometry *geo = mp->m_dir_geo; + xfs_dir2_leaf_tail_t *ltp; + int stale; + int i; + bool isleaf1 = (hdr->magic == XFS_DIR2_LEAF1_MAGIC || + hdr->magic == XFS_DIR3_LEAF1_MAGIC); + + ltp = xfs_dir2_leaf_tail_p(geo, leaf); + + /* + * XXX (dgc): This value is not restrictive enough. + * Should factor in the size of the bests table as well. + * We can deduce a value for that from i_disk_size. + */ + if (hdr->count > geo->leaf_max_ents) + return __this_address; + + /* Leaves and bests don't overlap in leaf format. */ + if (isleaf1 && + (char *)&hdr->ents[hdr->count] > (char *)xfs_dir2_leaf_bests_p(ltp)) + return __this_address; + + if (!expensive_checking) + return NULL; + + /* Check hash value order, count stale entries. */ + for (i = stale = 0; i < hdr->count; i++) { + if (i + 1 < hdr->count) { + if (be32_to_cpu(hdr->ents[i].hashval) > + be32_to_cpu(hdr->ents[i + 1].hashval)) + return __this_address; + } + if (hdr->ents[i].address == cpu_to_be32(XFS_DIR2_NULL_DATAPTR)) + stale++; + if (isleaf1 && xfs_dir2_dataptr_to_db(geo, + be32_to_cpu(hdr->ents[i].address)) >= + be32_to_cpu(ltp->bestcount)) + return __this_address; + } + if (hdr->stale != stale) + return __this_address; + return NULL; +} + +/* + * We verify the magic numbers before decoding the leaf header so that on debug + * kernels we don't get assertion failures in xfs_dir3_leaf_hdr_from_disk() due + * to incorrect magic numbers. + */ +static xfs_failaddr_t +xfs_dir3_leaf_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_dir3_icleaf_hdr leafhdr; + xfs_failaddr_t fa; + + fa = xfs_da3_blkinfo_verify(bp, bp->b_addr); + if (fa) + return fa; + + xfs_dir2_leaf_hdr_from_disk(mp, &leafhdr, bp->b_addr); + return xfs_dir3_leaf_check_int(mp, &leafhdr, bp->b_addr, true); +} + +xfs_failaddr_t +xfs_dir3_leaf_header_check( + struct xfs_buf *bp, + xfs_ino_t owner) +{ + struct xfs_mount *mp = bp->b_mount; + + if (xfs_has_crc(mp)) { + struct xfs_dir3_leaf *hdr3 = bp->b_addr; + + if (hdr3->hdr.info.hdr.magic != + cpu_to_be16(XFS_DIR3_LEAF1_MAGIC) && + hdr3->hdr.info.hdr.magic != + cpu_to_be16(XFS_DIR3_LEAFN_MAGIC)) + return __this_address; + + if (be64_to_cpu(hdr3->hdr.info.owner) != owner) + return __this_address; + } + + return NULL; +} + +static void +xfs_dir3_leaf_read_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + xfs_failaddr_t fa; + + if (xfs_has_crc(mp) && + !xfs_buf_verify_cksum(bp, XFS_DIR3_LEAF_CRC_OFF)) + xfs_verifier_error(bp, -EFSBADCRC, __this_address); + else { + fa = xfs_dir3_leaf_verify(bp); + if (fa) + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + } +} + +static void +xfs_dir3_leaf_write_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_buf_log_item *bip = bp->b_log_item; + struct xfs_dir3_leaf_hdr *hdr3 = bp->b_addr; + xfs_failaddr_t fa; + + fa = xfs_dir3_leaf_verify(bp); + if (fa) { + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + return; + } + + if (!xfs_has_crc(mp)) + return; + + if (bip) + hdr3->info.lsn = cpu_to_be64(bip->bli_item.li_lsn); + + xfs_buf_update_cksum(bp, XFS_DIR3_LEAF_CRC_OFF); +} + +const struct xfs_buf_ops xfs_dir3_leaf1_buf_ops = { + .name = "xfs_dir3_leaf1", + .magic16 = { cpu_to_be16(XFS_DIR2_LEAF1_MAGIC), + cpu_to_be16(XFS_DIR3_LEAF1_MAGIC) }, + .verify_read = xfs_dir3_leaf_read_verify, + .verify_write = xfs_dir3_leaf_write_verify, + .verify_struct = xfs_dir3_leaf_verify, +}; + +const struct xfs_buf_ops xfs_dir3_leafn_buf_ops = { + .name = "xfs_dir3_leafn", + .magic16 = { cpu_to_be16(XFS_DIR2_LEAFN_MAGIC), + cpu_to_be16(XFS_DIR3_LEAFN_MAGIC) }, + .verify_read = xfs_dir3_leaf_read_verify, + .verify_write = xfs_dir3_leaf_write_verify, + .verify_struct = xfs_dir3_leaf_verify, +}; + +int +xfs_dir3_leaf_read( + struct xfs_trans *tp, + struct xfs_inode *dp, + xfs_ino_t owner, + xfs_dablk_t fbno, + struct xfs_buf **bpp) +{ + xfs_failaddr_t fa; + int err; + + err = xfs_da_read_buf(tp, dp, fbno, 0, bpp, XFS_DATA_FORK, + &xfs_dir3_leaf1_buf_ops); + if (err || !(*bpp)) + return err; + + fa = xfs_dir3_leaf_header_check(*bpp, owner); + if (fa) { + __xfs_buf_mark_corrupt(*bpp, fa); + xfs_trans_brelse(tp, *bpp); + *bpp = NULL; + xfs_dirattr_mark_sick(dp, XFS_DATA_FORK); + return -EFSCORRUPTED; + } + + if (tp) + xfs_trans_buf_set_type(tp, *bpp, XFS_BLFT_DIR_LEAF1_BUF); + return 0; +} + +int +xfs_dir3_leafn_read( + struct xfs_trans *tp, + struct xfs_inode *dp, + xfs_ino_t owner, + xfs_dablk_t fbno, + struct xfs_buf **bpp) +{ + xfs_failaddr_t fa; + int err; + + err = xfs_da_read_buf(tp, dp, fbno, 0, bpp, XFS_DATA_FORK, + &xfs_dir3_leafn_buf_ops); + if (err || !(*bpp)) + return err; + + fa = xfs_dir3_leaf_header_check(*bpp, owner); + if (fa) { + __xfs_buf_mark_corrupt(*bpp, fa); + xfs_trans_brelse(tp, *bpp); + *bpp = NULL; + xfs_dirattr_mark_sick(dp, XFS_DATA_FORK); + return -EFSCORRUPTED; + } + + if (tp) + xfs_trans_buf_set_type(tp, *bpp, XFS_BLFT_DIR_LEAFN_BUF); + return 0; +} + +/* + * Initialize a new leaf block, leaf1 or leafn magic accepted. + */ +static void +xfs_dir3_leaf_init( + struct xfs_da_args *args, + struct xfs_buf *bp, + uint16_t type) +{ + struct xfs_mount *mp = args->dp->i_mount; + struct xfs_trans *tp = args->trans; + struct xfs_dir2_leaf *leaf = bp->b_addr; + + ASSERT(type == XFS_DIR2_LEAF1_MAGIC || type == XFS_DIR2_LEAFN_MAGIC); + + if (xfs_has_crc(mp)) { + struct xfs_dir3_leaf_hdr *leaf3 = bp->b_addr; + + memset(leaf3, 0, sizeof(*leaf3)); + + leaf3->info.hdr.magic = (type == XFS_DIR2_LEAF1_MAGIC) + ? cpu_to_be16(XFS_DIR3_LEAF1_MAGIC) + : cpu_to_be16(XFS_DIR3_LEAFN_MAGIC); + leaf3->info.blkno = cpu_to_be64(xfs_buf_daddr(bp)); + leaf3->info.owner = cpu_to_be64(args->owner); + uuid_copy(&leaf3->info.uuid, &mp->m_sb.sb_meta_uuid); + } else { + memset(leaf, 0, sizeof(*leaf)); + leaf->hdr.info.magic = cpu_to_be16(type); + } + + /* + * If it's a leaf-format directory initialize the tail. + * Caller is responsible for initialising the bests table. + */ + if (type == XFS_DIR2_LEAF1_MAGIC) { + struct xfs_dir2_leaf_tail *ltp; + + ltp = xfs_dir2_leaf_tail_p(mp->m_dir_geo, leaf); + ltp->bestcount = 0; + bp->b_ops = &xfs_dir3_leaf1_buf_ops; + xfs_trans_buf_set_type(tp, bp, XFS_BLFT_DIR_LEAF1_BUF); + } else { + bp->b_ops = &xfs_dir3_leafn_buf_ops; + xfs_trans_buf_set_type(tp, bp, XFS_BLFT_DIR_LEAFN_BUF); + } +} + +int +xfs_dir3_leaf_get_buf( + xfs_da_args_t *args, + xfs_dir2_db_t bno, + struct xfs_buf **bpp, + uint16_t magic) +{ + struct xfs_inode *dp = args->dp; + struct xfs_trans *tp = args->trans; + struct xfs_buf *bp; + int error; + + ASSERT(magic == XFS_DIR2_LEAF1_MAGIC || magic == XFS_DIR2_LEAFN_MAGIC); + ASSERT(bno >= xfs_dir2_byte_to_db(args->geo, XFS_DIR2_LEAF_OFFSET) && + bno < xfs_dir2_byte_to_db(args->geo, XFS_DIR2_FREE_OFFSET)); + + error = xfs_da_get_buf(tp, dp, xfs_dir2_db_to_da(args->geo, bno), + &bp, XFS_DATA_FORK); + if (error) + return error; + + xfs_dir3_leaf_init(args, bp, magic); + xfs_dir3_leaf_log_header(args, bp); + if (magic == XFS_DIR2_LEAF1_MAGIC) + xfs_dir3_leaf_log_tail(args, bp); + *bpp = bp; + return 0; +} + +/* + * Convert a block form directory to a leaf form directory. + */ +int /* error */ +xfs_dir2_block_to_leaf( + xfs_da_args_t *args, /* operation arguments */ + struct xfs_buf *dbp) /* input block's buffer */ +{ + __be16 *bestsp; /* leaf's bestsp entries */ + xfs_dablk_t blkno; /* leaf block's bno */ + xfs_dir2_data_hdr_t *hdr; /* block header */ + xfs_dir2_leaf_entry_t *blp; /* block's leaf entries */ + xfs_dir2_block_tail_t *btp; /* block's tail */ + xfs_inode_t *dp; /* incore directory inode */ + int error; /* error return code */ + struct xfs_buf *lbp; /* leaf block's buffer */ + xfs_dir2_db_t ldb; /* leaf block's bno */ + xfs_dir2_leaf_t *leaf; /* leaf structure */ + xfs_dir2_leaf_tail_t *ltp; /* leaf's tail */ + int needlog; /* need to log block header */ + int needscan; /* need to rescan bestfree */ + xfs_trans_t *tp; /* transaction pointer */ + struct xfs_dir2_data_free *bf; + struct xfs_dir3_icleaf_hdr leafhdr; + + trace_xfs_dir2_block_to_leaf(args); + + dp = args->dp; + tp = args->trans; + /* + * Add the leaf block to the inode. + * This interface will only put blocks in the leaf/node range. + * Since that's empty now, we'll get the root (block 0 in range). + */ + if ((error = xfs_da_grow_inode(args, &blkno))) { + return error; + } + ldb = xfs_dir2_da_to_db(args->geo, blkno); + ASSERT(ldb == xfs_dir2_byte_to_db(args->geo, XFS_DIR2_LEAF_OFFSET)); + /* + * Initialize the leaf block, get a buffer for it. + */ + error = xfs_dir3_leaf_get_buf(args, ldb, &lbp, XFS_DIR2_LEAF1_MAGIC); + if (error) + return error; + + leaf = lbp->b_addr; + hdr = dbp->b_addr; + xfs_dir3_data_check(dp, dbp); + btp = xfs_dir2_block_tail_p(args->geo, hdr); + blp = xfs_dir2_block_leaf_p(btp); + bf = xfs_dir2_data_bestfree_p(dp->i_mount, hdr); + + /* + * Set the counts in the leaf header. + */ + xfs_dir2_leaf_hdr_from_disk(dp->i_mount, &leafhdr, leaf); + leafhdr.count = be32_to_cpu(btp->count); + leafhdr.stale = be32_to_cpu(btp->stale); + xfs_dir2_leaf_hdr_to_disk(dp->i_mount, leaf, &leafhdr); + xfs_dir3_leaf_log_header(args, lbp); + + /* + * Could compact these but I think we always do the conversion + * after squeezing out stale entries. + */ + memcpy(leafhdr.ents, blp, + be32_to_cpu(btp->count) * sizeof(struct xfs_dir2_leaf_entry)); + xfs_dir3_leaf_log_ents(args, &leafhdr, lbp, 0, leafhdr.count - 1); + needscan = 0; + needlog = 1; + /* + * Make the space formerly occupied by the leaf entries and block + * tail be free. + */ + xfs_dir2_data_make_free(args, dbp, + (xfs_dir2_data_aoff_t)((char *)blp - (char *)hdr), + (xfs_dir2_data_aoff_t)((char *)hdr + args->geo->blksize - + (char *)blp), + &needlog, &needscan); + /* + * Fix up the block header, make it a data block. + */ + dbp->b_ops = &xfs_dir3_data_buf_ops; + xfs_trans_buf_set_type(tp, dbp, XFS_BLFT_DIR_DATA_BUF); + if (hdr->magic == cpu_to_be32(XFS_DIR2_BLOCK_MAGIC)) + hdr->magic = cpu_to_be32(XFS_DIR2_DATA_MAGIC); + else + hdr->magic = cpu_to_be32(XFS_DIR3_DATA_MAGIC); + + if (needscan) + xfs_dir2_data_freescan(dp->i_mount, hdr, &needlog); + /* + * Set up leaf tail and bests table. + */ + ltp = xfs_dir2_leaf_tail_p(args->geo, leaf); + ltp->bestcount = cpu_to_be32(1); + bestsp = xfs_dir2_leaf_bests_p(ltp); + bestsp[0] = bf[0].length; + /* + * Log the data header and leaf bests table. + */ + if (needlog) + xfs_dir2_data_log_header(args, dbp); + xfs_dir3_leaf_check(dp, lbp); + xfs_dir3_data_check(dp, dbp); + xfs_dir3_leaf_log_bests(args, lbp, 0, 0); + return 0; +} + +STATIC void +xfs_dir3_leaf_find_stale( + struct xfs_dir3_icleaf_hdr *leafhdr, + struct xfs_dir2_leaf_entry *ents, + int index, + int *lowstale, + int *highstale) +{ + /* + * Find the first stale entry before our index, if any. + */ + for (*lowstale = index - 1; *lowstale >= 0; --*lowstale) { + if (ents[*lowstale].address == + cpu_to_be32(XFS_DIR2_NULL_DATAPTR)) + break; + } + + /* + * Find the first stale entry at or after our index, if any. + * Stop if the result would require moving more entries than using + * lowstale. + */ + for (*highstale = index; *highstale < leafhdr->count; ++*highstale) { + if (ents[*highstale].address == + cpu_to_be32(XFS_DIR2_NULL_DATAPTR)) + break; + if (*lowstale >= 0 && index - *lowstale <= *highstale - index) + break; + } +} + +struct xfs_dir2_leaf_entry * +xfs_dir3_leaf_find_entry( + struct xfs_dir3_icleaf_hdr *leafhdr, + struct xfs_dir2_leaf_entry *ents, + int index, /* leaf table position */ + int compact, /* need to compact leaves */ + int lowstale, /* index of prev stale leaf */ + int highstale, /* index of next stale leaf */ + int *lfloglow, /* low leaf logging index */ + int *lfloghigh) /* high leaf logging index */ +{ + if (!leafhdr->stale) { + xfs_dir2_leaf_entry_t *lep; /* leaf entry table pointer */ + + /* + * Now we need to make room to insert the leaf entry. + * + * If there are no stale entries, just insert a hole at index. + */ + lep = &ents[index]; + if (index < leafhdr->count) + memmove(lep + 1, lep, + (leafhdr->count - index) * sizeof(*lep)); + + /* + * Record low and high logging indices for the leaf. + */ + *lfloglow = index; + *lfloghigh = leafhdr->count++; + return lep; + } + + /* + * There are stale entries. + * + * We will use one of them for the new entry. It's probably not at + * the right location, so we'll have to shift some up or down first. + * + * If we didn't compact before, we need to find the nearest stale + * entries before and after our insertion point. + */ + if (compact == 0) + xfs_dir3_leaf_find_stale(leafhdr, ents, index, + &lowstale, &highstale); + + /* + * If the low one is better, use it. + */ + if (lowstale >= 0 && + (highstale == leafhdr->count || + index - lowstale - 1 < highstale - index)) { + ASSERT(index - lowstale - 1 >= 0); + ASSERT(ents[lowstale].address == + cpu_to_be32(XFS_DIR2_NULL_DATAPTR)); + + /* + * Copy entries up to cover the stale entry and make room + * for the new entry. + */ + if (index - lowstale - 1 > 0) { + memmove(&ents[lowstale], &ents[lowstale + 1], + (index - lowstale - 1) * + sizeof(xfs_dir2_leaf_entry_t)); + } + *lfloglow = min(lowstale, *lfloglow); + *lfloghigh = max(index - 1, *lfloghigh); + leafhdr->stale--; + return &ents[index - 1]; + } + + /* + * The high one is better, so use that one. + */ + ASSERT(highstale - index >= 0); + ASSERT(ents[highstale].address == cpu_to_be32(XFS_DIR2_NULL_DATAPTR)); + + /* + * Copy entries down to cover the stale entry and make room for the + * new entry. + */ + if (highstale - index > 0) { + memmove(&ents[index + 1], &ents[index], + (highstale - index) * sizeof(xfs_dir2_leaf_entry_t)); + } + *lfloglow = min(index, *lfloglow); + *lfloghigh = max(highstale, *lfloghigh); + leafhdr->stale--; + return &ents[index]; +} + +/* + * Add an entry to a leaf form directory. + */ +int /* error */ +xfs_dir2_leaf_addname( + struct xfs_da_args *args) /* operation arguments */ +{ + struct xfs_dir3_icleaf_hdr leafhdr; + struct xfs_trans *tp = args->trans; + __be16 *bestsp; /* freespace table in leaf */ + __be16 *tagp; /* end of data entry */ + struct xfs_buf *dbp; /* data block buffer */ + struct xfs_buf *lbp; /* leaf's buffer */ + struct xfs_dir2_leaf *leaf; /* leaf structure */ + struct xfs_inode *dp = args->dp; /* incore directory inode */ + struct xfs_dir2_data_hdr *hdr; /* data block header */ + struct xfs_dir2_data_entry *dep; /* data block entry */ + struct xfs_dir2_leaf_entry *lep; /* leaf entry table pointer */ + struct xfs_dir2_leaf_entry *ents; + struct xfs_dir2_data_unused *dup; /* data unused entry */ + struct xfs_dir2_leaf_tail *ltp; /* leaf tail pointer */ + struct xfs_dir2_data_free *bf; /* bestfree table */ + int compact; /* need to compact leaves */ + int error; /* error return value */ + int grown; /* allocated new data block */ + int highstale = 0; /* index of next stale leaf */ + int i; /* temporary, index */ + int index; /* leaf table position */ + int length; /* length of new entry */ + int lfloglow; /* low leaf logging index */ + int lfloghigh; /* high leaf logging index */ + int lowstale = 0; /* index of prev stale leaf */ + int needbytes; /* leaf block bytes needed */ + int needlog; /* need to log data header */ + int needscan; /* need to rescan data free */ + xfs_dir2_db_t use_block; /* data block number */ + + trace_xfs_dir2_leaf_addname(args); + + error = xfs_dir3_leaf_read(tp, dp, args->owner, args->geo->leafblk, + &lbp); + if (error) + return error; + + /* + * Look up the entry by hash value and name. + * We know it's not there, our caller has already done a lookup. + * So the index is of the entry to insert in front of. + * But if there are dup hash values the index is of the first of those. + */ + index = xfs_dir2_leaf_search_hash(args, lbp); + leaf = lbp->b_addr; + ltp = xfs_dir2_leaf_tail_p(args->geo, leaf); + xfs_dir2_leaf_hdr_from_disk(dp->i_mount, &leafhdr, leaf); + ents = leafhdr.ents; + bestsp = xfs_dir2_leaf_bests_p(ltp); + length = xfs_dir2_data_entsize(dp->i_mount, args->namelen); + + /* + * See if there are any entries with the same hash value + * and space in their block for the new entry. + * This is good because it puts multiple same-hash value entries + * in a data block, improving the lookup of those entries. + */ + for (use_block = -1, lep = &ents[index]; + index < leafhdr.count && be32_to_cpu(lep->hashval) == args->hashval; + index++, lep++) { + if (be32_to_cpu(lep->address) == XFS_DIR2_NULL_DATAPTR) + continue; + i = xfs_dir2_dataptr_to_db(args->geo, be32_to_cpu(lep->address)); + ASSERT(i < be32_to_cpu(ltp->bestcount)); + ASSERT(bestsp[i] != cpu_to_be16(NULLDATAOFF)); + if (be16_to_cpu(bestsp[i]) >= length) { + use_block = i; + break; + } + } + /* + * Didn't find a block yet, linear search all the data blocks. + */ + if (use_block == -1) { + for (i = 0; i < be32_to_cpu(ltp->bestcount); i++) { + /* + * Remember a block we see that's missing. + */ + if (bestsp[i] == cpu_to_be16(NULLDATAOFF) && + use_block == -1) + use_block = i; + else if (be16_to_cpu(bestsp[i]) >= length) { + use_block = i; + break; + } + } + } + /* + * How many bytes do we need in the leaf block? + */ + needbytes = 0; + if (!leafhdr.stale) + needbytes += sizeof(xfs_dir2_leaf_entry_t); + if (use_block == -1) + needbytes += sizeof(xfs_dir2_data_off_t); + + /* + * Now kill use_block if it refers to a missing block, so we + * can use it as an indication of allocation needed. + */ + if (use_block != -1 && bestsp[use_block] == cpu_to_be16(NULLDATAOFF)) + use_block = -1; + /* + * If we don't have enough free bytes but we can make enough + * by compacting out stale entries, we'll do that. + */ + if ((char *)bestsp - (char *)&ents[leafhdr.count] < needbytes && + leafhdr.stale > 1) + compact = 1; + + /* + * Otherwise if we don't have enough free bytes we need to + * convert to node form. + */ + else if ((char *)bestsp - (char *)&ents[leafhdr.count] < needbytes) { + /* + * Just checking or no space reservation, give up. + */ + if ((args->op_flags & XFS_DA_OP_JUSTCHECK) || + args->total == 0) { + xfs_trans_brelse(tp, lbp); + return -ENOSPC; + } + /* + * Convert to node form. + */ + error = xfs_dir2_leaf_to_node(args, lbp); + if (error) + return error; + /* + * Then add the new entry. + */ + return xfs_dir2_node_addname(args); + } + /* + * Otherwise it will fit without compaction. + */ + else + compact = 0; + /* + * If just checking, then it will fit unless we needed to allocate + * a new data block. + */ + if (args->op_flags & XFS_DA_OP_JUSTCHECK) { + xfs_trans_brelse(tp, lbp); + return use_block == -1 ? -ENOSPC : 0; + } + /* + * If no allocations are allowed, return now before we've + * changed anything. + */ + if (args->total == 0 && use_block == -1) { + xfs_trans_brelse(tp, lbp); + return -ENOSPC; + } + /* + * Need to compact the leaf entries, removing stale ones. + * Leave one stale entry behind - the one closest to our + * insertion index - and we'll shift that one to our insertion + * point later. + */ + if (compact) { + xfs_dir3_leaf_compact_x1(&leafhdr, ents, &index, &lowstale, + &highstale, &lfloglow, &lfloghigh); + } + /* + * There are stale entries, so we'll need log-low and log-high + * impossibly bad values later. + */ + else if (leafhdr.stale) { + lfloglow = leafhdr.count; + lfloghigh = -1; + } + /* + * If there was no data block space found, we need to allocate + * a new one. + */ + if (use_block == -1) { + /* + * Add the new data block. + */ + if ((error = xfs_dir2_grow_inode(args, XFS_DIR2_DATA_SPACE, + &use_block))) { + xfs_trans_brelse(tp, lbp); + return error; + } + /* + * Initialize the block. + */ + if ((error = xfs_dir3_data_init(args, use_block, &dbp))) { + xfs_trans_brelse(tp, lbp); + return error; + } + /* + * If we're adding a new data block on the end we need to + * extend the bests table. Copy it up one entry. + */ + if (use_block >= be32_to_cpu(ltp->bestcount)) { + bestsp--; + memmove(&bestsp[0], &bestsp[1], + be32_to_cpu(ltp->bestcount) * sizeof(bestsp[0])); + be32_add_cpu(<p->bestcount, 1); + xfs_dir3_leaf_log_tail(args, lbp); + xfs_dir3_leaf_log_bests(args, lbp, 0, + be32_to_cpu(ltp->bestcount) - 1); + } + /* + * If we're filling in a previously empty block just log it. + */ + else + xfs_dir3_leaf_log_bests(args, lbp, use_block, use_block); + hdr = dbp->b_addr; + bf = xfs_dir2_data_bestfree_p(dp->i_mount, hdr); + bestsp[use_block] = bf[0].length; + grown = 1; + } else { + /* + * Already had space in some data block. + * Just read that one in. + */ + error = xfs_dir3_data_read(tp, dp, args->owner, + xfs_dir2_db_to_da(args->geo, use_block), 0, + &dbp); + if (error) { + xfs_trans_brelse(tp, lbp); + return error; + } + hdr = dbp->b_addr; + bf = xfs_dir2_data_bestfree_p(dp->i_mount, hdr); + grown = 0; + } + /* + * Point to the biggest freespace in our data block. + */ + dup = (xfs_dir2_data_unused_t *) + ((char *)hdr + be16_to_cpu(bf[0].offset)); + needscan = needlog = 0; + /* + * Mark the initial part of our freespace in use for the new entry. + */ + error = xfs_dir2_data_use_free(args, dbp, dup, + (xfs_dir2_data_aoff_t)((char *)dup - (char *)hdr), + length, &needlog, &needscan); + if (error) { + xfs_trans_brelse(tp, lbp); + return error; + } + /* + * Initialize our new entry (at last). + */ + dep = (xfs_dir2_data_entry_t *)dup; + dep->inumber = cpu_to_be64(args->inumber); + dep->namelen = args->namelen; + memcpy(dep->name, args->name, dep->namelen); + xfs_dir2_data_put_ftype(dp->i_mount, dep, args->filetype); + tagp = xfs_dir2_data_entry_tag_p(dp->i_mount, dep); + *tagp = cpu_to_be16((char *)dep - (char *)hdr); + /* + * Need to scan fix up the bestfree table. + */ + if (needscan) + xfs_dir2_data_freescan(dp->i_mount, hdr, &needlog); + /* + * Need to log the data block's header. + */ + if (needlog) + xfs_dir2_data_log_header(args, dbp); + xfs_dir2_data_log_entry(args, dbp, dep); + /* + * If the bests table needs to be changed, do it. + * Log the change unless we've already done that. + */ + if (be16_to_cpu(bestsp[use_block]) != be16_to_cpu(bf[0].length)) { + bestsp[use_block] = bf[0].length; + if (!grown) + xfs_dir3_leaf_log_bests(args, lbp, use_block, use_block); + } + + lep = xfs_dir3_leaf_find_entry(&leafhdr, ents, index, compact, lowstale, + highstale, &lfloglow, &lfloghigh); + + /* + * Fill in the new leaf entry. + */ + lep->hashval = cpu_to_be32(args->hashval); + lep->address = cpu_to_be32( + xfs_dir2_db_off_to_dataptr(args->geo, use_block, + be16_to_cpu(*tagp))); + /* + * Log the leaf fields and give up the buffers. + */ + xfs_dir2_leaf_hdr_to_disk(dp->i_mount, leaf, &leafhdr); + xfs_dir3_leaf_log_header(args, lbp); + xfs_dir3_leaf_log_ents(args, &leafhdr, lbp, lfloglow, lfloghigh); + xfs_dir3_leaf_check(dp, lbp); + xfs_dir3_data_check(dp, dbp); + return 0; +} + +/* + * Compact out any stale entries in the leaf. + * Log the header and changed leaf entries, if any. + */ +void +xfs_dir3_leaf_compact( + xfs_da_args_t *args, /* operation arguments */ + struct xfs_dir3_icleaf_hdr *leafhdr, + struct xfs_buf *bp) /* leaf buffer */ +{ + int from; /* source leaf index */ + xfs_dir2_leaf_t *leaf; /* leaf structure */ + int loglow; /* first leaf entry to log */ + int to; /* target leaf index */ + struct xfs_inode *dp = args->dp; + + leaf = bp->b_addr; + if (!leafhdr->stale) + return; + + /* + * Compress out the stale entries in place. + */ + for (from = to = 0, loglow = -1; from < leafhdr->count; from++) { + if (leafhdr->ents[from].address == + cpu_to_be32(XFS_DIR2_NULL_DATAPTR)) + continue; + /* + * Only actually copy the entries that are different. + */ + if (from > to) { + if (loglow == -1) + loglow = to; + leafhdr->ents[to] = leafhdr->ents[from]; + } + to++; + } + /* + * Update and log the header, log the leaf entries. + */ + ASSERT(leafhdr->stale == from - to); + leafhdr->count -= leafhdr->stale; + leafhdr->stale = 0; + + xfs_dir2_leaf_hdr_to_disk(dp->i_mount, leaf, leafhdr); + xfs_dir3_leaf_log_header(args, bp); + if (loglow != -1) + xfs_dir3_leaf_log_ents(args, leafhdr, bp, loglow, to - 1); +} + +/* + * Compact the leaf entries, removing stale ones. + * Leave one stale entry behind - the one closest to our + * insertion index - and the caller will shift that one to our insertion + * point later. + * Return new insertion index, where the remaining stale entry is, + * and leaf logging indices. + */ +void +xfs_dir3_leaf_compact_x1( + struct xfs_dir3_icleaf_hdr *leafhdr, + struct xfs_dir2_leaf_entry *ents, + int *indexp, /* insertion index */ + int *lowstalep, /* out: stale entry before us */ + int *highstalep, /* out: stale entry after us */ + int *lowlogp, /* out: low log index */ + int *highlogp) /* out: high log index */ +{ + int from; /* source copy index */ + int highstale; /* stale entry at/after index */ + int index; /* insertion index */ + int keepstale; /* source index of kept stale */ + int lowstale; /* stale entry before index */ + int newindex=0; /* new insertion index */ + int to; /* destination copy index */ + + ASSERT(leafhdr->stale > 1); + index = *indexp; + + xfs_dir3_leaf_find_stale(leafhdr, ents, index, &lowstale, &highstale); + + /* + * Pick the better of lowstale and highstale. + */ + if (lowstale >= 0 && + (highstale == leafhdr->count || + index - lowstale <= highstale - index)) + keepstale = lowstale; + else + keepstale = highstale; + /* + * Copy the entries in place, removing all the stale entries + * except keepstale. + */ + for (from = to = 0; from < leafhdr->count; from++) { + /* + * Notice the new value of index. + */ + if (index == from) + newindex = to; + if (from != keepstale && + ents[from].address == cpu_to_be32(XFS_DIR2_NULL_DATAPTR)) { + if (from == to) + *lowlogp = to; + continue; + } + /* + * Record the new keepstale value for the insertion. + */ + if (from == keepstale) + lowstale = highstale = to; + /* + * Copy only the entries that have moved. + */ + if (from > to) + ents[to] = ents[from]; + to++; + } + ASSERT(from > to); + /* + * If the insertion point was past the last entry, + * set the new insertion point accordingly. + */ + if (index == from) + newindex = to; + *indexp = newindex; + /* + * Adjust the leaf header values. + */ + leafhdr->count -= from - to; + leafhdr->stale = 1; + /* + * Remember the low/high stale value only in the "right" + * direction. + */ + if (lowstale >= newindex) + lowstale = -1; + else + highstale = leafhdr->count; + *highlogp = leafhdr->count - 1; + *lowstalep = lowstale; + *highstalep = highstale; +} + +/* + * Log the bests entries indicated from a leaf1 block. + */ +static void +xfs_dir3_leaf_log_bests( + struct xfs_da_args *args, + struct xfs_buf *bp, /* leaf buffer */ + int first, /* first entry to log */ + int last) /* last entry to log */ +{ + __be16 *firstb; /* pointer to first entry */ + __be16 *lastb; /* pointer to last entry */ + struct xfs_dir2_leaf *leaf = bp->b_addr; + xfs_dir2_leaf_tail_t *ltp; /* leaf tail structure */ + + ASSERT(leaf->hdr.info.magic == cpu_to_be16(XFS_DIR2_LEAF1_MAGIC) || + leaf->hdr.info.magic == cpu_to_be16(XFS_DIR3_LEAF1_MAGIC)); + + ltp = xfs_dir2_leaf_tail_p(args->geo, leaf); + firstb = xfs_dir2_leaf_bests_p(ltp) + first; + lastb = xfs_dir2_leaf_bests_p(ltp) + last; + xfs_trans_log_buf(args->trans, bp, + (uint)((char *)firstb - (char *)leaf), + (uint)((char *)lastb - (char *)leaf + sizeof(*lastb) - 1)); +} + +/* + * Log the leaf entries indicated from a leaf1 or leafn block. + */ +void +xfs_dir3_leaf_log_ents( + struct xfs_da_args *args, + struct xfs_dir3_icleaf_hdr *hdr, + struct xfs_buf *bp, + int first, + int last) +{ + xfs_dir2_leaf_entry_t *firstlep; /* pointer to first entry */ + xfs_dir2_leaf_entry_t *lastlep; /* pointer to last entry */ + struct xfs_dir2_leaf *leaf = bp->b_addr; + + ASSERT(leaf->hdr.info.magic == cpu_to_be16(XFS_DIR2_LEAF1_MAGIC) || + leaf->hdr.info.magic == cpu_to_be16(XFS_DIR3_LEAF1_MAGIC) || + leaf->hdr.info.magic == cpu_to_be16(XFS_DIR2_LEAFN_MAGIC) || + leaf->hdr.info.magic == cpu_to_be16(XFS_DIR3_LEAFN_MAGIC)); + + firstlep = &hdr->ents[first]; + lastlep = &hdr->ents[last]; + xfs_trans_log_buf(args->trans, bp, + (uint)((char *)firstlep - (char *)leaf), + (uint)((char *)lastlep - (char *)leaf + sizeof(*lastlep) - 1)); +} + +/* + * Log the header of the leaf1 or leafn block. + */ +void +xfs_dir3_leaf_log_header( + struct xfs_da_args *args, + struct xfs_buf *bp) +{ + struct xfs_dir2_leaf *leaf = bp->b_addr; + + ASSERT(leaf->hdr.info.magic == cpu_to_be16(XFS_DIR2_LEAF1_MAGIC) || + leaf->hdr.info.magic == cpu_to_be16(XFS_DIR3_LEAF1_MAGIC) || + leaf->hdr.info.magic == cpu_to_be16(XFS_DIR2_LEAFN_MAGIC) || + leaf->hdr.info.magic == cpu_to_be16(XFS_DIR3_LEAFN_MAGIC)); + + xfs_trans_log_buf(args->trans, bp, + (uint)((char *)&leaf->hdr - (char *)leaf), + args->geo->leaf_hdr_size - 1); +} + +/* + * Log the tail of the leaf1 block. + */ +STATIC void +xfs_dir3_leaf_log_tail( + struct xfs_da_args *args, + struct xfs_buf *bp) +{ + struct xfs_dir2_leaf *leaf = bp->b_addr; + xfs_dir2_leaf_tail_t *ltp; /* leaf tail structure */ + + ASSERT(leaf->hdr.info.magic == cpu_to_be16(XFS_DIR2_LEAF1_MAGIC) || + leaf->hdr.info.magic == cpu_to_be16(XFS_DIR3_LEAF1_MAGIC) || + leaf->hdr.info.magic == cpu_to_be16(XFS_DIR2_LEAFN_MAGIC) || + leaf->hdr.info.magic == cpu_to_be16(XFS_DIR3_LEAFN_MAGIC)); + + ltp = xfs_dir2_leaf_tail_p(args->geo, leaf); + xfs_trans_log_buf(args->trans, bp, (uint)((char *)ltp - (char *)leaf), + (uint)(args->geo->blksize - 1)); +} + +/* + * Look up the entry referred to by args in the leaf format directory. + * Most of the work is done by the xfs_dir2_leaf_lookup_int routine which + * is also used by the node-format code. + */ +int +xfs_dir2_leaf_lookup( + xfs_da_args_t *args) /* operation arguments */ +{ + struct xfs_buf *dbp; /* data block buffer */ + xfs_dir2_data_entry_t *dep; /* data block entry */ + xfs_inode_t *dp; /* incore directory inode */ + int error; /* error return code */ + int index; /* found entry index */ + struct xfs_buf *lbp; /* leaf buffer */ + xfs_dir2_leaf_entry_t *lep; /* leaf entry */ + xfs_trans_t *tp; /* transaction pointer */ + struct xfs_dir3_icleaf_hdr leafhdr; + + trace_xfs_dir2_leaf_lookup(args); + + /* + * Look up name in the leaf block, returning both buffers and index. + */ + error = xfs_dir2_leaf_lookup_int(args, &lbp, &index, &dbp, &leafhdr); + if (error) + return error; + + tp = args->trans; + dp = args->dp; + xfs_dir3_leaf_check(dp, lbp); + + /* + * Get to the leaf entry and contained data entry address. + */ + lep = &leafhdr.ents[index]; + + /* + * Point to the data entry. + */ + dep = (xfs_dir2_data_entry_t *) + ((char *)dbp->b_addr + + xfs_dir2_dataptr_to_off(args->geo, be32_to_cpu(lep->address))); + /* + * Return the found inode number & CI name if appropriate + */ + args->inumber = be64_to_cpu(dep->inumber); + args->filetype = xfs_dir2_data_get_ftype(dp->i_mount, dep); + error = xfs_dir_cilookup_result(args, dep->name, dep->namelen); + xfs_trans_brelse(tp, dbp); + xfs_trans_brelse(tp, lbp); + return error; +} + +/* + * Look up name/hash in the leaf block. + * Fill in indexp with the found index, and dbpp with the data buffer. + * If not found dbpp will be NULL, and ENOENT comes back. + * lbpp will always be filled in with the leaf buffer unless there's an error. + */ +static int /* error */ +xfs_dir2_leaf_lookup_int( + xfs_da_args_t *args, /* operation arguments */ + struct xfs_buf **lbpp, /* out: leaf buffer */ + int *indexp, /* out: index in leaf block */ + struct xfs_buf **dbpp, /* out: data buffer */ + struct xfs_dir3_icleaf_hdr *leafhdr) +{ + xfs_dir2_db_t curdb = -1; /* current data block number */ + struct xfs_buf *dbp = NULL; /* data buffer */ + xfs_dir2_data_entry_t *dep; /* data entry */ + xfs_inode_t *dp; /* incore directory inode */ + int error; /* error return code */ + int index; /* index in leaf block */ + struct xfs_buf *lbp; /* leaf buffer */ + xfs_dir2_leaf_entry_t *lep; /* leaf entry */ + xfs_dir2_leaf_t *leaf; /* leaf structure */ + xfs_mount_t *mp; /* filesystem mount point */ + xfs_dir2_db_t newdb; /* new data block number */ + xfs_trans_t *tp; /* transaction pointer */ + xfs_dir2_db_t cidb = -1; /* case match data block no. */ + enum xfs_dacmp cmp; /* name compare result */ + + dp = args->dp; + tp = args->trans; + mp = dp->i_mount; + + error = xfs_dir3_leaf_read(tp, dp, args->owner, args->geo->leafblk, + &lbp); + if (error) + return error; + + *lbpp = lbp; + leaf = lbp->b_addr; + xfs_dir3_leaf_check(dp, lbp); + xfs_dir2_leaf_hdr_from_disk(mp, leafhdr, leaf); + + /* + * Look for the first leaf entry with our hash value. + */ + index = xfs_dir2_leaf_search_hash(args, lbp); + /* + * Loop over all the entries with the right hash value + * looking to match the name. + */ + for (lep = &leafhdr->ents[index]; + index < leafhdr->count && + be32_to_cpu(lep->hashval) == args->hashval; + lep++, index++) { + /* + * Skip over stale leaf entries. + */ + if (be32_to_cpu(lep->address) == XFS_DIR2_NULL_DATAPTR) + continue; + /* + * Get the new data block number. + */ + newdb = xfs_dir2_dataptr_to_db(args->geo, + be32_to_cpu(lep->address)); + /* + * If it's not the same as the old data block number, + * need to pitch the old one and read the new one. + */ + if (newdb != curdb) { + if (dbp) + xfs_trans_brelse(tp, dbp); + error = xfs_dir3_data_read(tp, dp, args->owner, + xfs_dir2_db_to_da(args->geo, newdb), 0, + &dbp); + if (error) { + xfs_trans_brelse(tp, lbp); + return error; + } + curdb = newdb; + } + /* + * Point to the data entry. + */ + dep = (xfs_dir2_data_entry_t *)((char *)dbp->b_addr + + xfs_dir2_dataptr_to_off(args->geo, + be32_to_cpu(lep->address))); + /* + * Compare name and if it's an exact match, return the index + * and buffer. If it's the first case-insensitive match, store + * the index and buffer and continue looking for an exact match. + */ + cmp = xfs_dir2_compname(args, dep->name, dep->namelen); + if (cmp != XFS_CMP_DIFFERENT && cmp != args->cmpresult) { + args->cmpresult = cmp; + *indexp = index; + /* case exact match: return the current buffer. */ + if (cmp == XFS_CMP_EXACT) { + *dbpp = dbp; + return 0; + } + cidb = curdb; + } + } + ASSERT(args->op_flags & XFS_DA_OP_OKNOENT); + /* + * Here, we can only be doing a lookup (not a rename or remove). + * If a case-insensitive match was found earlier, re-read the + * appropriate data block if required and return it. + */ + if (args->cmpresult == XFS_CMP_CASE) { + ASSERT(cidb != -1); + if (cidb != curdb) { + xfs_trans_brelse(tp, dbp); + error = xfs_dir3_data_read(tp, dp, args->owner, + xfs_dir2_db_to_da(args->geo, cidb), 0, + &dbp); + if (error) { + xfs_trans_brelse(tp, lbp); + return error; + } + } + *dbpp = dbp; + return 0; + } + /* + * No match found, return -ENOENT. + */ + ASSERT(cidb == -1); + if (dbp) + xfs_trans_brelse(tp, dbp); + xfs_trans_brelse(tp, lbp); + return -ENOENT; +} + +/* + * Remove an entry from a leaf format directory. + */ +int /* error */ +xfs_dir2_leaf_removename( + xfs_da_args_t *args) /* operation arguments */ +{ + struct xfs_da_geometry *geo = args->geo; + __be16 *bestsp; /* leaf block best freespace */ + xfs_dir2_data_hdr_t *hdr; /* data block header */ + xfs_dir2_db_t db; /* data block number */ + struct xfs_buf *dbp; /* data block buffer */ + xfs_dir2_data_entry_t *dep; /* data entry structure */ + xfs_inode_t *dp; /* incore directory inode */ + int error; /* error return code */ + xfs_dir2_db_t i; /* temporary data block # */ + int index; /* index into leaf entries */ + struct xfs_buf *lbp; /* leaf buffer */ + xfs_dir2_leaf_t *leaf; /* leaf structure */ + xfs_dir2_leaf_entry_t *lep; /* leaf entry */ + xfs_dir2_leaf_tail_t *ltp; /* leaf tail structure */ + int needlog; /* need to log data header */ + int needscan; /* need to rescan data frees */ + xfs_dir2_data_off_t oldbest; /* old value of best free */ + struct xfs_dir2_data_free *bf; /* bestfree table */ + struct xfs_dir3_icleaf_hdr leafhdr; + + trace_xfs_dir2_leaf_removename(args); + + /* + * Lookup the leaf entry, get the leaf and data blocks read in. + */ + error = xfs_dir2_leaf_lookup_int(args, &lbp, &index, &dbp, &leafhdr); + if (error) + return error; + + dp = args->dp; + leaf = lbp->b_addr; + hdr = dbp->b_addr; + xfs_dir3_data_check(dp, dbp); + bf = xfs_dir2_data_bestfree_p(dp->i_mount, hdr); + + /* + * Point to the leaf entry, use that to point to the data entry. + */ + lep = &leafhdr.ents[index]; + db = xfs_dir2_dataptr_to_db(geo, be32_to_cpu(lep->address)); + dep = (xfs_dir2_data_entry_t *)((char *)hdr + + xfs_dir2_dataptr_to_off(geo, be32_to_cpu(lep->address))); + needscan = needlog = 0; + oldbest = be16_to_cpu(bf[0].length); + ltp = xfs_dir2_leaf_tail_p(geo, leaf); + bestsp = xfs_dir2_leaf_bests_p(ltp); + if (be16_to_cpu(bestsp[db]) != oldbest) { + xfs_buf_mark_corrupt(lbp); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + + /* + * Mark the former data entry unused. + */ + xfs_dir2_data_make_free(args, dbp, + (xfs_dir2_data_aoff_t)((char *)dep - (char *)hdr), + xfs_dir2_data_entsize(dp->i_mount, dep->namelen), &needlog, + &needscan); + /* + * We just mark the leaf entry stale by putting a null in it. + */ + leafhdr.stale++; + xfs_dir2_leaf_hdr_to_disk(dp->i_mount, leaf, &leafhdr); + xfs_dir3_leaf_log_header(args, lbp); + + lep->address = cpu_to_be32(XFS_DIR2_NULL_DATAPTR); + xfs_dir3_leaf_log_ents(args, &leafhdr, lbp, index, index); + + /* + * Scan the freespace in the data block again if necessary, + * log the data block header if necessary. + */ + if (needscan) + xfs_dir2_data_freescan(dp->i_mount, hdr, &needlog); + if (needlog) + xfs_dir2_data_log_header(args, dbp); + /* + * If the longest freespace in the data block has changed, + * put the new value in the bests table and log that. + */ + if (be16_to_cpu(bf[0].length) != oldbest) { + bestsp[db] = bf[0].length; + xfs_dir3_leaf_log_bests(args, lbp, db, db); + } + xfs_dir3_data_check(dp, dbp); + /* + * If the data block is now empty then get rid of the data block. + */ + if (be16_to_cpu(bf[0].length) == + geo->blksize - geo->data_entry_offset) { + ASSERT(db != geo->datablk); + if ((error = xfs_dir2_shrink_inode(args, db, dbp))) { + /* + * Nope, can't get rid of it because it caused + * allocation of a bmap btree block to do so. + * Just go on, returning success, leaving the + * empty block in place. + */ + if (error == -ENOSPC && args->total == 0) + error = 0; + xfs_dir3_leaf_check(dp, lbp); + return error; + } + dbp = NULL; + /* + * If this is the last data block then compact the + * bests table by getting rid of entries. + */ + if (db == be32_to_cpu(ltp->bestcount) - 1) { + /* + * Look for the last active entry (i). + */ + for (i = db - 1; i > 0; i--) { + if (bestsp[i] != cpu_to_be16(NULLDATAOFF)) + break; + } + /* + * Copy the table down so inactive entries at the + * end are removed. + */ + memmove(&bestsp[db - i], bestsp, + (be32_to_cpu(ltp->bestcount) - (db - i)) * sizeof(*bestsp)); + be32_add_cpu(<p->bestcount, -(db - i)); + xfs_dir3_leaf_log_tail(args, lbp); + xfs_dir3_leaf_log_bests(args, lbp, 0, + be32_to_cpu(ltp->bestcount) - 1); + } else + bestsp[db] = cpu_to_be16(NULLDATAOFF); + } + /* + * If the data block was not the first one, drop it. + */ + else if (db != geo->datablk) + dbp = NULL; + + xfs_dir3_leaf_check(dp, lbp); + /* + * See if we can convert to block form. + */ + return xfs_dir2_leaf_to_block(args, lbp, dbp); +} + +/* + * Replace the inode number in a leaf format directory entry. + */ +int /* error */ +xfs_dir2_leaf_replace( + xfs_da_args_t *args) /* operation arguments */ +{ + struct xfs_buf *dbp; /* data block buffer */ + xfs_dir2_data_entry_t *dep; /* data block entry */ + xfs_inode_t *dp; /* incore directory inode */ + int error; /* error return code */ + int index; /* index of leaf entry */ + struct xfs_buf *lbp; /* leaf buffer */ + xfs_dir2_leaf_entry_t *lep; /* leaf entry */ + xfs_trans_t *tp; /* transaction pointer */ + struct xfs_dir3_icleaf_hdr leafhdr; + + trace_xfs_dir2_leaf_replace(args); + + /* + * Look up the entry. + */ + error = xfs_dir2_leaf_lookup_int(args, &lbp, &index, &dbp, &leafhdr); + if (error) + return error; + + dp = args->dp; + /* + * Point to the leaf entry, get data address from it. + */ + lep = &leafhdr.ents[index]; + /* + * Point to the data entry. + */ + dep = (xfs_dir2_data_entry_t *) + ((char *)dbp->b_addr + + xfs_dir2_dataptr_to_off(args->geo, be32_to_cpu(lep->address))); + ASSERT(args->inumber != be64_to_cpu(dep->inumber)); + /* + * Put the new inode number in, log it. + */ + dep->inumber = cpu_to_be64(args->inumber); + xfs_dir2_data_put_ftype(dp->i_mount, dep, args->filetype); + tp = args->trans; + xfs_dir2_data_log_entry(args, dbp, dep); + xfs_dir3_leaf_check(dp, lbp); + xfs_trans_brelse(tp, lbp); + return 0; +} + +/* + * Return index in the leaf block (lbp) which is either the first + * one with this hash value, or if there are none, the insert point + * for that hash value. + */ +int /* index value */ +xfs_dir2_leaf_search_hash( + xfs_da_args_t *args, /* operation arguments */ + struct xfs_buf *lbp) /* leaf buffer */ +{ + xfs_dahash_t hash=0; /* hash from this entry */ + xfs_dahash_t hashwant; /* hash value looking for */ + int high; /* high leaf index */ + int low; /* low leaf index */ + xfs_dir2_leaf_entry_t *lep; /* leaf entry */ + int mid=0; /* current leaf index */ + struct xfs_dir3_icleaf_hdr leafhdr; + + xfs_dir2_leaf_hdr_from_disk(args->dp->i_mount, &leafhdr, lbp->b_addr); + + /* + * Note, the table cannot be empty, so we have to go through the loop. + * Binary search the leaf entries looking for our hash value. + */ + for (lep = leafhdr.ents, low = 0, high = leafhdr.count - 1, + hashwant = args->hashval; + low <= high; ) { + mid = (low + high) >> 1; + if ((hash = be32_to_cpu(lep[mid].hashval)) == hashwant) + break; + if (hash < hashwant) + low = mid + 1; + else + high = mid - 1; + } + /* + * Found one, back up through all the equal hash values. + */ + if (hash == hashwant) { + while (mid > 0 && be32_to_cpu(lep[mid - 1].hashval) == hashwant) { + mid--; + } + } + /* + * Need to point to an entry higher than ours. + */ + else if (hash < hashwant) + mid++; + return mid; +} + +/* + * Trim off a trailing data block. We know it's empty since the leaf + * freespace table says so. + */ +int /* error */ +xfs_dir2_leaf_trim_data( + xfs_da_args_t *args, /* operation arguments */ + struct xfs_buf *lbp, /* leaf buffer */ + xfs_dir2_db_t db) /* data block number */ +{ + struct xfs_da_geometry *geo = args->geo; + __be16 *bestsp; /* leaf bests table */ + struct xfs_buf *dbp; /* data block buffer */ + xfs_inode_t *dp; /* incore directory inode */ + int error; /* error return value */ + xfs_dir2_leaf_t *leaf; /* leaf structure */ + xfs_dir2_leaf_tail_t *ltp; /* leaf tail structure */ + xfs_trans_t *tp; /* transaction pointer */ + + dp = args->dp; + tp = args->trans; + /* + * Read the offending data block. We need its buffer. + */ + error = xfs_dir3_data_read(tp, dp, args->owner, + xfs_dir2_db_to_da(geo, db), 0, &dbp); + if (error) + return error; + + leaf = lbp->b_addr; + ltp = xfs_dir2_leaf_tail_p(geo, leaf); + +#ifdef DEBUG +{ + struct xfs_dir2_data_hdr *hdr = dbp->b_addr; + struct xfs_dir2_data_free *bf = + xfs_dir2_data_bestfree_p(dp->i_mount, hdr); + + ASSERT(hdr->magic == cpu_to_be32(XFS_DIR2_DATA_MAGIC) || + hdr->magic == cpu_to_be32(XFS_DIR3_DATA_MAGIC)); + ASSERT(be16_to_cpu(bf[0].length) == + geo->blksize - geo->data_entry_offset); + ASSERT(db == be32_to_cpu(ltp->bestcount) - 1); +} +#endif + + /* + * Get rid of the data block. + */ + if ((error = xfs_dir2_shrink_inode(args, db, dbp))) { + ASSERT(error != -ENOSPC); + xfs_trans_brelse(tp, dbp); + return error; + } + /* + * Eliminate the last bests entry from the table. + */ + bestsp = xfs_dir2_leaf_bests_p(ltp); + be32_add_cpu(<p->bestcount, -1); + memmove(&bestsp[1], &bestsp[0], be32_to_cpu(ltp->bestcount) * sizeof(*bestsp)); + xfs_dir3_leaf_log_tail(args, lbp); + xfs_dir3_leaf_log_bests(args, lbp, 0, be32_to_cpu(ltp->bestcount) - 1); + return 0; +} + +static inline size_t +xfs_dir3_leaf_size( + struct xfs_dir3_icleaf_hdr *hdr, + int counts) +{ + int entries; + int hdrsize; + + entries = hdr->count - hdr->stale; + if (hdr->magic == XFS_DIR2_LEAF1_MAGIC || + hdr->magic == XFS_DIR2_LEAFN_MAGIC) + hdrsize = sizeof(struct xfs_dir2_leaf_hdr); + else + hdrsize = sizeof(struct xfs_dir3_leaf_hdr); + + return hdrsize + entries * sizeof(xfs_dir2_leaf_entry_t) + + counts * sizeof(xfs_dir2_data_off_t) + + sizeof(xfs_dir2_leaf_tail_t); +} + +/* + * Convert node form directory to leaf form directory. + * The root of the node form dir needs to already be a LEAFN block. + * Just return if we can't do anything. + */ +int /* error */ +xfs_dir2_node_to_leaf( + xfs_da_state_t *state) /* directory operation state */ +{ + xfs_da_args_t *args; /* operation arguments */ + xfs_inode_t *dp; /* incore directory inode */ + int error; /* error return code */ + struct xfs_buf *fbp; /* buffer for freespace block */ + xfs_fileoff_t fo; /* freespace file offset */ + struct xfs_buf *lbp; /* buffer for leaf block */ + xfs_dir2_leaf_tail_t *ltp; /* tail of leaf structure */ + xfs_dir2_leaf_t *leaf; /* leaf structure */ + xfs_mount_t *mp; /* filesystem mount point */ + int rval; /* successful free trim? */ + xfs_trans_t *tp; /* transaction pointer */ + struct xfs_dir3_icleaf_hdr leafhdr; + struct xfs_dir3_icfree_hdr freehdr; + + /* + * There's more than a leaf level in the btree, so there must + * be multiple leafn blocks. Give up. + */ + if (state->path.active > 1) + return 0; + args = state->args; + + trace_xfs_dir2_node_to_leaf(args); + + mp = state->mp; + dp = args->dp; + tp = args->trans; + /* + * Get the last offset in the file. + */ + if ((error = xfs_bmap_last_offset(dp, &fo, XFS_DATA_FORK))) { + return error; + } + fo -= args->geo->fsbcount; + /* + * If there are freespace blocks other than the first one, + * take this opportunity to remove trailing empty freespace blocks + * that may have been left behind during no-space-reservation + * operations. + */ + while (fo > args->geo->freeblk) { + if ((error = xfs_dir2_node_trim_free(args, fo, &rval))) { + return error; + } + if (rval) + fo -= args->geo->fsbcount; + else + return 0; + } + /* + * Now find the block just before the freespace block. + */ + if ((error = xfs_bmap_last_before(tp, dp, &fo, XFS_DATA_FORK))) { + return error; + } + /* + * If it's not the single leaf block, give up. + */ + if (XFS_FSB_TO_B(mp, fo) > XFS_DIR2_LEAF_OFFSET + args->geo->blksize) + return 0; + lbp = state->path.blk[0].bp; + leaf = lbp->b_addr; + xfs_dir2_leaf_hdr_from_disk(mp, &leafhdr, leaf); + + ASSERT(leafhdr.magic == XFS_DIR2_LEAFN_MAGIC || + leafhdr.magic == XFS_DIR3_LEAFN_MAGIC); + + /* + * Read the freespace block. + */ + error = xfs_dir2_free_read(tp, dp, args->owner, args->geo->freeblk, + &fbp); + if (error) + return error; + xfs_dir2_free_hdr_from_disk(mp, &freehdr, fbp->b_addr); + + ASSERT(!freehdr.firstdb); + + /* + * Now see if the leafn and free data will fit in a leaf1. + * If not, release the buffer and give up. + */ + if (xfs_dir3_leaf_size(&leafhdr, freehdr.nvalid) > args->geo->blksize) { + xfs_trans_brelse(tp, fbp); + return 0; + } + + /* + * If the leaf has any stale entries in it, compress them out. + */ + if (leafhdr.stale) + xfs_dir3_leaf_compact(args, &leafhdr, lbp); + + lbp->b_ops = &xfs_dir3_leaf1_buf_ops; + xfs_trans_buf_set_type(tp, lbp, XFS_BLFT_DIR_LEAF1_BUF); + leafhdr.magic = (leafhdr.magic == XFS_DIR2_LEAFN_MAGIC) + ? XFS_DIR2_LEAF1_MAGIC + : XFS_DIR3_LEAF1_MAGIC; + + /* + * Set up the leaf tail from the freespace block. + */ + ltp = xfs_dir2_leaf_tail_p(args->geo, leaf); + ltp->bestcount = cpu_to_be32(freehdr.nvalid); + + /* + * Set up the leaf bests table. + */ + memcpy(xfs_dir2_leaf_bests_p(ltp), freehdr.bests, + freehdr.nvalid * sizeof(xfs_dir2_data_off_t)); + + xfs_dir2_leaf_hdr_to_disk(mp, leaf, &leafhdr); + xfs_dir3_leaf_log_header(args, lbp); + xfs_dir3_leaf_log_bests(args, lbp, 0, be32_to_cpu(ltp->bestcount) - 1); + xfs_dir3_leaf_log_tail(args, lbp); + xfs_dir3_leaf_check(dp, lbp); + + /* + * Get rid of the freespace block. + */ + error = xfs_dir2_shrink_inode(args, + xfs_dir2_byte_to_db(args->geo, XFS_DIR2_FREE_OFFSET), + fbp); + if (error) { + /* + * This can't fail here because it can only happen when + * punching out the middle of an extent, and this is an + * isolated block. + */ + ASSERT(error != -ENOSPC); + return error; + } + fbp = NULL; + /* + * Now see if we can convert the single-leaf directory + * down to a block form directory. + * This routine always kills the dabuf for the leaf, so + * eliminate it from the path. + */ + error = xfs_dir2_leaf_to_block(args, lbp, NULL); + state->path.blk[0].bp = NULL; + return error; +} diff --git a/fs/xfs/libxfs/xfs_dir2_node.c b/fs/xfs/libxfs/xfs_dir2_node.c new file mode 100644 index 000000000..19cb536e5 --- /dev/null +++ b/fs/xfs/libxfs/xfs_dir2_node.c @@ -0,0 +1,2348 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * Copyright (c) 2013 Red Hat, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_bmap.h" +#include "xfs_dir2.h" +#include "xfs_dir2_priv.h" +#include "xfs_error.h" +#include "xfs_trace.h" +#include "xfs_trans.h" +#include "xfs_buf_item.h" +#include "xfs_log.h" +#include "xfs_health.h" + +/* + * Function declarations. + */ +static int xfs_dir2_leafn_add(struct xfs_buf *bp, xfs_da_args_t *args, + int index); +static void xfs_dir2_leafn_rebalance(xfs_da_state_t *state, + xfs_da_state_blk_t *blk1, + xfs_da_state_blk_t *blk2); +static int xfs_dir2_leafn_remove(xfs_da_args_t *args, struct xfs_buf *bp, + int index, xfs_da_state_blk_t *dblk, + int *rval); + +/* + * Convert data space db to the corresponding free db. + */ +static xfs_dir2_db_t +xfs_dir2_db_to_fdb(struct xfs_da_geometry *geo, xfs_dir2_db_t db) +{ + return xfs_dir2_byte_to_db(geo, XFS_DIR2_FREE_OFFSET) + + (db / geo->free_max_bests); +} + +/* + * Convert data space db to the corresponding index in a free db. + */ +static int +xfs_dir2_db_to_fdindex(struct xfs_da_geometry *geo, xfs_dir2_db_t db) +{ + return db % geo->free_max_bests; +} + +/* + * Check internal consistency of a leafn block. + */ +#ifdef DEBUG +static xfs_failaddr_t +xfs_dir3_leafn_check( + struct xfs_inode *dp, + struct xfs_buf *bp) +{ + struct xfs_dir2_leaf *leaf = bp->b_addr; + struct xfs_dir3_icleaf_hdr leafhdr; + + xfs_dir2_leaf_hdr_from_disk(dp->i_mount, &leafhdr, leaf); + + if (leafhdr.magic == XFS_DIR3_LEAFN_MAGIC) { + struct xfs_dir3_leaf_hdr *leaf3 = bp->b_addr; + if (be64_to_cpu(leaf3->info.blkno) != xfs_buf_daddr(bp)) + return __this_address; + } else if (leafhdr.magic != XFS_DIR2_LEAFN_MAGIC) + return __this_address; + + return xfs_dir3_leaf_check_int(dp->i_mount, &leafhdr, leaf, false); +} + +static inline void +xfs_dir3_leaf_check( + struct xfs_inode *dp, + struct xfs_buf *bp) +{ + xfs_failaddr_t fa; + + fa = xfs_dir3_leafn_check(dp, bp); + if (!fa) + return; + xfs_corruption_error(__func__, XFS_ERRLEVEL_LOW, dp->i_mount, + bp->b_addr, BBTOB(bp->b_length), __FILE__, __LINE__, + fa); + ASSERT(0); +} +#else +#define xfs_dir3_leaf_check(dp, bp) +#endif + +static xfs_failaddr_t +xfs_dir3_free_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_dir2_free_hdr *hdr = bp->b_addr; + + if (!xfs_verify_magic(bp, hdr->magic)) + return __this_address; + + if (xfs_has_crc(mp)) { + struct xfs_dir3_blk_hdr *hdr3 = bp->b_addr; + + if (!uuid_equal(&hdr3->uuid, &mp->m_sb.sb_meta_uuid)) + return __this_address; + if (be64_to_cpu(hdr3->blkno) != xfs_buf_daddr(bp)) + return __this_address; + if (!xfs_log_check_lsn(mp, be64_to_cpu(hdr3->lsn))) + return __this_address; + } + + /* XXX: should bounds check the xfs_dir3_icfree_hdr here */ + + return NULL; +} + +static void +xfs_dir3_free_read_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + xfs_failaddr_t fa; + + if (xfs_has_crc(mp) && + !xfs_buf_verify_cksum(bp, XFS_DIR3_FREE_CRC_OFF)) + xfs_verifier_error(bp, -EFSBADCRC, __this_address); + else { + fa = xfs_dir3_free_verify(bp); + if (fa) + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + } +} + +static void +xfs_dir3_free_write_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_buf_log_item *bip = bp->b_log_item; + struct xfs_dir3_blk_hdr *hdr3 = bp->b_addr; + xfs_failaddr_t fa; + + fa = xfs_dir3_free_verify(bp); + if (fa) { + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + return; + } + + if (!xfs_has_crc(mp)) + return; + + if (bip) + hdr3->lsn = cpu_to_be64(bip->bli_item.li_lsn); + + xfs_buf_update_cksum(bp, XFS_DIR3_FREE_CRC_OFF); +} + +const struct xfs_buf_ops xfs_dir3_free_buf_ops = { + .name = "xfs_dir3_free", + .magic = { cpu_to_be32(XFS_DIR2_FREE_MAGIC), + cpu_to_be32(XFS_DIR3_FREE_MAGIC) }, + .verify_read = xfs_dir3_free_read_verify, + .verify_write = xfs_dir3_free_write_verify, + .verify_struct = xfs_dir3_free_verify, +}; + +/* Everything ok in the free block header? */ +static xfs_failaddr_t +xfs_dir3_free_header_check( + struct xfs_buf *bp, + xfs_ino_t owner, + xfs_dablk_t fbno) +{ + struct xfs_mount *mp = bp->b_mount; + int maxbests = mp->m_dir_geo->free_max_bests; + unsigned int firstdb; + + firstdb = (xfs_dir2_da_to_db(mp->m_dir_geo, fbno) - + xfs_dir2_byte_to_db(mp->m_dir_geo, XFS_DIR2_FREE_OFFSET)) * + maxbests; + if (xfs_has_crc(mp)) { + struct xfs_dir3_free_hdr *hdr3 = bp->b_addr; + + if (be32_to_cpu(hdr3->firstdb) != firstdb) + return __this_address; + if (be32_to_cpu(hdr3->nvalid) > maxbests) + return __this_address; + if (be32_to_cpu(hdr3->nvalid) < be32_to_cpu(hdr3->nused)) + return __this_address; + if (be64_to_cpu(hdr3->hdr.owner) != owner) + return __this_address; + } else { + struct xfs_dir2_free_hdr *hdr = bp->b_addr; + + if (be32_to_cpu(hdr->firstdb) != firstdb) + return __this_address; + if (be32_to_cpu(hdr->nvalid) > maxbests) + return __this_address; + if (be32_to_cpu(hdr->nvalid) < be32_to_cpu(hdr->nused)) + return __this_address; + } + return NULL; +} + +static int +__xfs_dir3_free_read( + struct xfs_trans *tp, + struct xfs_inode *dp, + xfs_ino_t owner, + xfs_dablk_t fbno, + unsigned int flags, + struct xfs_buf **bpp) +{ + xfs_failaddr_t fa; + int err; + + err = xfs_da_read_buf(tp, dp, fbno, flags, bpp, XFS_DATA_FORK, + &xfs_dir3_free_buf_ops); + if (err || !*bpp) + return err; + + /* Check things that we can't do in the verifier. */ + fa = xfs_dir3_free_header_check(*bpp, owner, fbno); + if (fa) { + __xfs_buf_mark_corrupt(*bpp, fa); + xfs_trans_brelse(tp, *bpp); + *bpp = NULL; + xfs_dirattr_mark_sick(dp, XFS_DATA_FORK); + return -EFSCORRUPTED; + } + + /* try read returns without an error or *bpp if it lands in a hole */ + if (tp) + xfs_trans_buf_set_type(tp, *bpp, XFS_BLFT_DIR_FREE_BUF); + + return 0; +} + +void +xfs_dir2_free_hdr_from_disk( + struct xfs_mount *mp, + struct xfs_dir3_icfree_hdr *to, + struct xfs_dir2_free *from) +{ + if (xfs_has_crc(mp)) { + struct xfs_dir3_free *from3 = (struct xfs_dir3_free *)from; + + to->magic = be32_to_cpu(from3->hdr.hdr.magic); + to->firstdb = be32_to_cpu(from3->hdr.firstdb); + to->nvalid = be32_to_cpu(from3->hdr.nvalid); + to->nused = be32_to_cpu(from3->hdr.nused); + to->bests = from3->bests; + + ASSERT(to->magic == XFS_DIR3_FREE_MAGIC); + } else { + to->magic = be32_to_cpu(from->hdr.magic); + to->firstdb = be32_to_cpu(from->hdr.firstdb); + to->nvalid = be32_to_cpu(from->hdr.nvalid); + to->nused = be32_to_cpu(from->hdr.nused); + to->bests = from->bests; + + ASSERT(to->magic == XFS_DIR2_FREE_MAGIC); + } +} + +static void +xfs_dir2_free_hdr_to_disk( + struct xfs_mount *mp, + struct xfs_dir2_free *to, + struct xfs_dir3_icfree_hdr *from) +{ + if (xfs_has_crc(mp)) { + struct xfs_dir3_free *to3 = (struct xfs_dir3_free *)to; + + ASSERT(from->magic == XFS_DIR3_FREE_MAGIC); + + to3->hdr.hdr.magic = cpu_to_be32(from->magic); + to3->hdr.firstdb = cpu_to_be32(from->firstdb); + to3->hdr.nvalid = cpu_to_be32(from->nvalid); + to3->hdr.nused = cpu_to_be32(from->nused); + } else { + ASSERT(from->magic == XFS_DIR2_FREE_MAGIC); + + to->hdr.magic = cpu_to_be32(from->magic); + to->hdr.firstdb = cpu_to_be32(from->firstdb); + to->hdr.nvalid = cpu_to_be32(from->nvalid); + to->hdr.nused = cpu_to_be32(from->nused); + } +} + +int +xfs_dir2_free_read( + struct xfs_trans *tp, + struct xfs_inode *dp, + xfs_ino_t owner, + xfs_dablk_t fbno, + struct xfs_buf **bpp) +{ + return __xfs_dir3_free_read(tp, dp, owner, fbno, 0, bpp); +} + +static int +xfs_dir2_free_try_read( + struct xfs_trans *tp, + struct xfs_inode *dp, + xfs_ino_t owner, + xfs_dablk_t fbno, + struct xfs_buf **bpp) +{ + return __xfs_dir3_free_read(tp, dp, owner, fbno, XFS_DABUF_MAP_HOLE_OK, + bpp); +} + +static int +xfs_dir3_free_get_buf( + xfs_da_args_t *args, + xfs_dir2_db_t fbno, + struct xfs_buf **bpp) +{ + struct xfs_trans *tp = args->trans; + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + struct xfs_buf *bp; + int error; + struct xfs_dir3_icfree_hdr hdr; + + error = xfs_da_get_buf(tp, dp, xfs_dir2_db_to_da(args->geo, fbno), + &bp, XFS_DATA_FORK); + if (error) + return error; + + xfs_trans_buf_set_type(tp, bp, XFS_BLFT_DIR_FREE_BUF); + bp->b_ops = &xfs_dir3_free_buf_ops; + + /* + * Initialize the new block to be empty, and remember + * its first slot as our empty slot. + */ + memset(bp->b_addr, 0, sizeof(struct xfs_dir3_free_hdr)); + memset(&hdr, 0, sizeof(hdr)); + + if (xfs_has_crc(mp)) { + struct xfs_dir3_free_hdr *hdr3 = bp->b_addr; + + hdr.magic = XFS_DIR3_FREE_MAGIC; + + hdr3->hdr.blkno = cpu_to_be64(xfs_buf_daddr(bp)); + hdr3->hdr.owner = cpu_to_be64(args->owner); + uuid_copy(&hdr3->hdr.uuid, &mp->m_sb.sb_meta_uuid); + } else + hdr.magic = XFS_DIR2_FREE_MAGIC; + xfs_dir2_free_hdr_to_disk(mp, bp->b_addr, &hdr); + *bpp = bp; + return 0; +} + +/* + * Log entries from a freespace block. + */ +STATIC void +xfs_dir2_free_log_bests( + struct xfs_da_args *args, + struct xfs_dir3_icfree_hdr *hdr, + struct xfs_buf *bp, + int first, /* first entry to log */ + int last) /* last entry to log */ +{ + struct xfs_dir2_free *free = bp->b_addr; + + ASSERT(free->hdr.magic == cpu_to_be32(XFS_DIR2_FREE_MAGIC) || + free->hdr.magic == cpu_to_be32(XFS_DIR3_FREE_MAGIC)); + xfs_trans_log_buf(args->trans, bp, + (char *)&hdr->bests[first] - (char *)free, + (char *)&hdr->bests[last] - (char *)free + + sizeof(hdr->bests[0]) - 1); +} + +/* + * Log header from a freespace block. + */ +static void +xfs_dir2_free_log_header( + struct xfs_da_args *args, + struct xfs_buf *bp) +{ +#ifdef DEBUG + xfs_dir2_free_t *free; /* freespace structure */ + + free = bp->b_addr; + ASSERT(free->hdr.magic == cpu_to_be32(XFS_DIR2_FREE_MAGIC) || + free->hdr.magic == cpu_to_be32(XFS_DIR3_FREE_MAGIC)); +#endif + xfs_trans_log_buf(args->trans, bp, 0, + args->geo->free_hdr_size - 1); +} + +/* + * Convert a leaf-format directory to a node-format directory. + * We need to change the magic number of the leaf block, and copy + * the freespace table out of the leaf block into its own block. + */ +int /* error */ +xfs_dir2_leaf_to_node( + xfs_da_args_t *args, /* operation arguments */ + struct xfs_buf *lbp) /* leaf buffer */ +{ + xfs_inode_t *dp; /* incore directory inode */ + int error; /* error return value */ + struct xfs_buf *fbp; /* freespace buffer */ + xfs_dir2_db_t fdb; /* freespace block number */ + __be16 *from; /* pointer to freespace entry */ + int i; /* leaf freespace index */ + xfs_dir2_leaf_t *leaf; /* leaf structure */ + xfs_dir2_leaf_tail_t *ltp; /* leaf tail structure */ + int n; /* count of live freespc ents */ + xfs_dir2_data_off_t off; /* freespace entry value */ + xfs_trans_t *tp; /* transaction pointer */ + struct xfs_dir3_icfree_hdr freehdr; + + trace_xfs_dir2_leaf_to_node(args); + + dp = args->dp; + tp = args->trans; + /* + * Add a freespace block to the directory. + */ + if ((error = xfs_dir2_grow_inode(args, XFS_DIR2_FREE_SPACE, &fdb))) { + return error; + } + ASSERT(fdb == xfs_dir2_byte_to_db(args->geo, XFS_DIR2_FREE_OFFSET)); + /* + * Get the buffer for the new freespace block. + */ + error = xfs_dir3_free_get_buf(args, fdb, &fbp); + if (error) + return error; + + xfs_dir2_free_hdr_from_disk(dp->i_mount, &freehdr, fbp->b_addr); + leaf = lbp->b_addr; + ltp = xfs_dir2_leaf_tail_p(args->geo, leaf); + if (be32_to_cpu(ltp->bestcount) > + (uint)dp->i_disk_size / args->geo->blksize) { + xfs_buf_mark_corrupt(lbp); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + + /* + * Copy freespace entries from the leaf block to the new block. + * Count active entries. + */ + from = xfs_dir2_leaf_bests_p(ltp); + for (i = n = 0; i < be32_to_cpu(ltp->bestcount); i++, from++) { + off = be16_to_cpu(*from); + if (off != NULLDATAOFF) + n++; + freehdr.bests[i] = cpu_to_be16(off); + } + + /* + * Now initialize the freespace block header. + */ + freehdr.nused = n; + freehdr.nvalid = be32_to_cpu(ltp->bestcount); + + xfs_dir2_free_hdr_to_disk(dp->i_mount, fbp->b_addr, &freehdr); + xfs_dir2_free_log_bests(args, &freehdr, fbp, 0, freehdr.nvalid - 1); + xfs_dir2_free_log_header(args, fbp); + + /* + * Converting the leaf to a leafnode is just a matter of changing the + * magic number and the ops. Do the change directly to the buffer as + * it's less work (and less code) than decoding the header to host + * format and back again. + */ + if (leaf->hdr.info.magic == cpu_to_be16(XFS_DIR2_LEAF1_MAGIC)) + leaf->hdr.info.magic = cpu_to_be16(XFS_DIR2_LEAFN_MAGIC); + else + leaf->hdr.info.magic = cpu_to_be16(XFS_DIR3_LEAFN_MAGIC); + lbp->b_ops = &xfs_dir3_leafn_buf_ops; + xfs_trans_buf_set_type(tp, lbp, XFS_BLFT_DIR_LEAFN_BUF); + xfs_dir3_leaf_log_header(args, lbp); + xfs_dir3_leaf_check(dp, lbp); + return 0; +} + +/* + * Add a leaf entry to a leaf block in a node-form directory. + * The other work necessary is done from the caller. + */ +static int /* error */ +xfs_dir2_leafn_add( + struct xfs_buf *bp, /* leaf buffer */ + struct xfs_da_args *args, /* operation arguments */ + int index) /* insertion pt for new entry */ +{ + struct xfs_dir3_icleaf_hdr leafhdr; + struct xfs_inode *dp = args->dp; + struct xfs_dir2_leaf *leaf = bp->b_addr; + struct xfs_dir2_leaf_entry *lep; + struct xfs_dir2_leaf_entry *ents; + int compact; /* compacting stale leaves */ + int highstale = 0; /* next stale entry */ + int lfloghigh; /* high leaf entry logging */ + int lfloglow; /* low leaf entry logging */ + int lowstale = 0; /* previous stale entry */ + + trace_xfs_dir2_leafn_add(args, index); + + xfs_dir2_leaf_hdr_from_disk(dp->i_mount, &leafhdr, leaf); + ents = leafhdr.ents; + + /* + * Quick check just to make sure we are not going to index + * into other peoples memory + */ + if (index < 0) { + xfs_buf_mark_corrupt(bp); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + + /* + * If there are already the maximum number of leaf entries in + * the block, if there are no stale entries it won't fit. + * Caller will do a split. If there are stale entries we'll do + * a compact. + */ + + if (leafhdr.count == args->geo->leaf_max_ents) { + if (!leafhdr.stale) + return -ENOSPC; + compact = leafhdr.stale > 1; + } else + compact = 0; + ASSERT(index == 0 || be32_to_cpu(ents[index - 1].hashval) <= args->hashval); + ASSERT(index == leafhdr.count || + be32_to_cpu(ents[index].hashval) >= args->hashval); + + if (args->op_flags & XFS_DA_OP_JUSTCHECK) + return 0; + + /* + * Compact out all but one stale leaf entry. Leaves behind + * the entry closest to index. + */ + if (compact) + xfs_dir3_leaf_compact_x1(&leafhdr, ents, &index, &lowstale, + &highstale, &lfloglow, &lfloghigh); + else if (leafhdr.stale) { + /* + * Set impossible logging indices for this case. + */ + lfloglow = leafhdr.count; + lfloghigh = -1; + } + + /* + * Insert the new entry, log everything. + */ + lep = xfs_dir3_leaf_find_entry(&leafhdr, ents, index, compact, lowstale, + highstale, &lfloglow, &lfloghigh); + + lep->hashval = cpu_to_be32(args->hashval); + lep->address = cpu_to_be32(xfs_dir2_db_off_to_dataptr(args->geo, + args->blkno, args->index)); + + xfs_dir2_leaf_hdr_to_disk(dp->i_mount, leaf, &leafhdr); + xfs_dir3_leaf_log_header(args, bp); + xfs_dir3_leaf_log_ents(args, &leafhdr, bp, lfloglow, lfloghigh); + xfs_dir3_leaf_check(dp, bp); + return 0; +} + +#ifdef DEBUG +static void +xfs_dir2_free_hdr_check( + struct xfs_inode *dp, + struct xfs_buf *bp, + xfs_dir2_db_t db) +{ + struct xfs_dir3_icfree_hdr hdr; + + xfs_dir2_free_hdr_from_disk(dp->i_mount, &hdr, bp->b_addr); + + ASSERT((hdr.firstdb % dp->i_mount->m_dir_geo->free_max_bests) == 0); + ASSERT(hdr.firstdb <= db); + ASSERT(db < hdr.firstdb + hdr.nvalid); +} +#else +#define xfs_dir2_free_hdr_check(dp, bp, db) +#endif /* DEBUG */ + +/* + * Return the last hash value in the leaf. + * Stale entries are ok. + */ +xfs_dahash_t /* hash value */ +xfs_dir2_leaf_lasthash( + struct xfs_inode *dp, + struct xfs_buf *bp, /* leaf buffer */ + int *count) /* count of entries in leaf */ +{ + struct xfs_dir3_icleaf_hdr leafhdr; + + xfs_dir2_leaf_hdr_from_disk(dp->i_mount, &leafhdr, bp->b_addr); + + ASSERT(leafhdr.magic == XFS_DIR2_LEAFN_MAGIC || + leafhdr.magic == XFS_DIR3_LEAFN_MAGIC || + leafhdr.magic == XFS_DIR2_LEAF1_MAGIC || + leafhdr.magic == XFS_DIR3_LEAF1_MAGIC); + + if (count) + *count = leafhdr.count; + if (!leafhdr.count) + return 0; + return be32_to_cpu(leafhdr.ents[leafhdr.count - 1].hashval); +} + +/* + * Look up a leaf entry for space to add a name in a node-format leaf block. + * The extrablk in state is a freespace block. + */ +STATIC int +xfs_dir2_leafn_lookup_for_addname( + struct xfs_buf *bp, /* leaf buffer */ + xfs_da_args_t *args, /* operation arguments */ + int *indexp, /* out: leaf entry index */ + xfs_da_state_t *state) /* state to fill in */ +{ + struct xfs_buf *curbp = NULL; /* current data/free buffer */ + xfs_dir2_db_t curdb = -1; /* current data block number */ + xfs_dir2_db_t curfdb = -1; /* current free block number */ + xfs_inode_t *dp; /* incore directory inode */ + int error; /* error return value */ + int fi; /* free entry index */ + xfs_dir2_free_t *free = NULL; /* free block structure */ + int index; /* leaf entry index */ + xfs_dir2_leaf_t *leaf; /* leaf structure */ + int length; /* length of new data entry */ + xfs_dir2_leaf_entry_t *lep; /* leaf entry */ + xfs_mount_t *mp; /* filesystem mount point */ + xfs_dir2_db_t newdb; /* new data block number */ + xfs_dir2_db_t newfdb; /* new free block number */ + xfs_trans_t *tp; /* transaction pointer */ + struct xfs_dir3_icleaf_hdr leafhdr; + + dp = args->dp; + tp = args->trans; + mp = dp->i_mount; + leaf = bp->b_addr; + xfs_dir2_leaf_hdr_from_disk(mp, &leafhdr, leaf); + + xfs_dir3_leaf_check(dp, bp); + ASSERT(leafhdr.count > 0); + + /* + * Look up the hash value in the leaf entries. + */ + index = xfs_dir2_leaf_search_hash(args, bp); + /* + * Do we have a buffer coming in? + */ + if (state->extravalid) { + /* If so, it's a free block buffer, get the block number. */ + curbp = state->extrablk.bp; + curfdb = state->extrablk.blkno; + free = curbp->b_addr; + ASSERT(free->hdr.magic == cpu_to_be32(XFS_DIR2_FREE_MAGIC) || + free->hdr.magic == cpu_to_be32(XFS_DIR3_FREE_MAGIC)); + } + length = xfs_dir2_data_entsize(mp, args->namelen); + /* + * Loop over leaf entries with the right hash value. + */ + for (lep = &leafhdr.ents[index]; + index < leafhdr.count && be32_to_cpu(lep->hashval) == args->hashval; + lep++, index++) { + /* + * Skip stale leaf entries. + */ + if (be32_to_cpu(lep->address) == XFS_DIR2_NULL_DATAPTR) + continue; + /* + * Pull the data block number from the entry. + */ + newdb = xfs_dir2_dataptr_to_db(args->geo, + be32_to_cpu(lep->address)); + /* + * For addname, we're looking for a place to put the new entry. + * We want to use a data block with an entry of equal + * hash value to ours if there is one with room. + * + * If this block isn't the data block we already have + * in hand, take a look at it. + */ + if (newdb != curdb) { + struct xfs_dir3_icfree_hdr freehdr; + + curdb = newdb; + /* + * Convert the data block to the free block + * holding its freespace information. + */ + newfdb = xfs_dir2_db_to_fdb(args->geo, newdb); + /* + * If it's not the one we have in hand, read it in. + */ + if (newfdb != curfdb) { + /* + * If we had one before, drop it. + */ + if (curbp) + xfs_trans_brelse(tp, curbp); + + error = xfs_dir2_free_read(tp, dp, args->owner, + xfs_dir2_db_to_da(args->geo, + newfdb), + &curbp); + if (error) + return error; + free = curbp->b_addr; + + xfs_dir2_free_hdr_check(dp, curbp, curdb); + } + /* + * Get the index for our entry. + */ + fi = xfs_dir2_db_to_fdindex(args->geo, curdb); + /* + * If it has room, return it. + */ + xfs_dir2_free_hdr_from_disk(mp, &freehdr, free); + if (XFS_IS_CORRUPT(mp, + freehdr.bests[fi] == + cpu_to_be16(NULLDATAOFF))) { + if (curfdb != newfdb) + xfs_trans_brelse(tp, curbp); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + curfdb = newfdb; + if (be16_to_cpu(freehdr.bests[fi]) >= length) + goto out; + } + } + /* Didn't find any space */ + fi = -1; +out: + ASSERT(args->op_flags & XFS_DA_OP_OKNOENT); + if (curbp) { + /* Giving back a free block. */ + state->extravalid = 1; + state->extrablk.bp = curbp; + state->extrablk.index = fi; + state->extrablk.blkno = curfdb; + + /* + * Important: this magic number is not in the buffer - it's for + * buffer type information and therefore only the free/data type + * matters here, not whether CRCs are enabled or not. + */ + state->extrablk.magic = XFS_DIR2_FREE_MAGIC; + } else { + state->extravalid = 0; + } + /* + * Return the index, that will be the insertion point. + */ + *indexp = index; + return -ENOENT; +} + +/* + * Look up a leaf entry in a node-format leaf block. + * The extrablk in state a data block. + */ +STATIC int +xfs_dir2_leafn_lookup_for_entry( + struct xfs_buf *bp, /* leaf buffer */ + xfs_da_args_t *args, /* operation arguments */ + int *indexp, /* out: leaf entry index */ + xfs_da_state_t *state) /* state to fill in */ +{ + struct xfs_buf *curbp = NULL; /* current data/free buffer */ + xfs_dir2_db_t curdb = -1; /* current data block number */ + xfs_dir2_data_entry_t *dep; /* data block entry */ + xfs_inode_t *dp; /* incore directory inode */ + int error; /* error return value */ + int index; /* leaf entry index */ + xfs_dir2_leaf_t *leaf; /* leaf structure */ + xfs_dir2_leaf_entry_t *lep; /* leaf entry */ + xfs_mount_t *mp; /* filesystem mount point */ + xfs_dir2_db_t newdb; /* new data block number */ + xfs_trans_t *tp; /* transaction pointer */ + enum xfs_dacmp cmp; /* comparison result */ + struct xfs_dir3_icleaf_hdr leafhdr; + + dp = args->dp; + tp = args->trans; + mp = dp->i_mount; + leaf = bp->b_addr; + xfs_dir2_leaf_hdr_from_disk(mp, &leafhdr, leaf); + + xfs_dir3_leaf_check(dp, bp); + if (leafhdr.count <= 0) { + xfs_buf_mark_corrupt(bp); + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + + /* + * Look up the hash value in the leaf entries. + */ + index = xfs_dir2_leaf_search_hash(args, bp); + /* + * Do we have a buffer coming in? + */ + if (state->extravalid) { + curbp = state->extrablk.bp; + curdb = state->extrablk.blkno; + } + /* + * Loop over leaf entries with the right hash value. + */ + for (lep = &leafhdr.ents[index]; + index < leafhdr.count && be32_to_cpu(lep->hashval) == args->hashval; + lep++, index++) { + /* + * Skip stale leaf entries. + */ + if (be32_to_cpu(lep->address) == XFS_DIR2_NULL_DATAPTR) + continue; + /* + * Pull the data block number from the entry. + */ + newdb = xfs_dir2_dataptr_to_db(args->geo, + be32_to_cpu(lep->address)); + /* + * Not adding a new entry, so we really want to find + * the name given to us. + * + * If it's a different data block, go get it. + */ + if (newdb != curdb) { + /* + * If we had a block before that we aren't saving + * for a CI name, drop it + */ + if (curbp && (args->cmpresult == XFS_CMP_DIFFERENT || + curdb != state->extrablk.blkno)) + xfs_trans_brelse(tp, curbp); + /* + * If needing the block that is saved with a CI match, + * use it otherwise read in the new data block. + */ + if (args->cmpresult != XFS_CMP_DIFFERENT && + newdb == state->extrablk.blkno) { + ASSERT(state->extravalid); + curbp = state->extrablk.bp; + } else { + error = xfs_dir3_data_read(tp, dp, args->owner, + xfs_dir2_db_to_da(args->geo, + newdb), + 0, &curbp); + if (error) + return error; + } + xfs_dir3_data_check(dp, curbp); + curdb = newdb; + } + /* + * Point to the data entry. + */ + dep = (xfs_dir2_data_entry_t *)((char *)curbp->b_addr + + xfs_dir2_dataptr_to_off(args->geo, + be32_to_cpu(lep->address))); + /* + * Compare the entry and if it's an exact match, return + * EEXIST immediately. If it's the first case-insensitive + * match, store the block & inode number and continue looking. + */ + cmp = xfs_dir2_compname(args, dep->name, dep->namelen); + if (cmp != XFS_CMP_DIFFERENT && cmp != args->cmpresult) { + /* If there is a CI match block, drop it */ + if (args->cmpresult != XFS_CMP_DIFFERENT && + curdb != state->extrablk.blkno) + xfs_trans_brelse(tp, state->extrablk.bp); + args->cmpresult = cmp; + args->inumber = be64_to_cpu(dep->inumber); + args->filetype = xfs_dir2_data_get_ftype(mp, dep); + *indexp = index; + state->extravalid = 1; + state->extrablk.bp = curbp; + state->extrablk.blkno = curdb; + state->extrablk.index = (int)((char *)dep - + (char *)curbp->b_addr); + state->extrablk.magic = XFS_DIR2_DATA_MAGIC; + curbp->b_ops = &xfs_dir3_data_buf_ops; + xfs_trans_buf_set_type(tp, curbp, XFS_BLFT_DIR_DATA_BUF); + if (cmp == XFS_CMP_EXACT) + return -EEXIST; + } + } + ASSERT(index == leafhdr.count || (args->op_flags & XFS_DA_OP_OKNOENT)); + if (curbp) { + if (args->cmpresult == XFS_CMP_DIFFERENT) { + /* Giving back last used data block. */ + state->extravalid = 1; + state->extrablk.bp = curbp; + state->extrablk.index = -1; + state->extrablk.blkno = curdb; + state->extrablk.magic = XFS_DIR2_DATA_MAGIC; + curbp->b_ops = &xfs_dir3_data_buf_ops; + xfs_trans_buf_set_type(tp, curbp, XFS_BLFT_DIR_DATA_BUF); + } else { + /* If the curbp is not the CI match block, drop it */ + if (state->extrablk.bp != curbp) + xfs_trans_brelse(tp, curbp); + } + } else { + state->extravalid = 0; + } + *indexp = index; + return -ENOENT; +} + +/* + * Look up a leaf entry in a node-format leaf block. + * If this is an addname then the extrablk in state is a freespace block, + * otherwise it's a data block. + */ +int +xfs_dir2_leafn_lookup_int( + struct xfs_buf *bp, /* leaf buffer */ + xfs_da_args_t *args, /* operation arguments */ + int *indexp, /* out: leaf entry index */ + xfs_da_state_t *state) /* state to fill in */ +{ + if (args->op_flags & XFS_DA_OP_ADDNAME) + return xfs_dir2_leafn_lookup_for_addname(bp, args, indexp, + state); + return xfs_dir2_leafn_lookup_for_entry(bp, args, indexp, state); +} + +/* + * Move count leaf entries from source to destination leaf. + * Log entries and headers. Stale entries are preserved. + */ +static void +xfs_dir3_leafn_moveents( + xfs_da_args_t *args, /* operation arguments */ + struct xfs_buf *bp_s, /* source */ + struct xfs_dir3_icleaf_hdr *shdr, + struct xfs_dir2_leaf_entry *sents, + int start_s,/* source leaf index */ + struct xfs_buf *bp_d, /* destination */ + struct xfs_dir3_icleaf_hdr *dhdr, + struct xfs_dir2_leaf_entry *dents, + int start_d,/* destination leaf index */ + int count) /* count of leaves to copy */ +{ + int stale; /* count stale leaves copied */ + + trace_xfs_dir2_leafn_moveents(args, start_s, start_d, count); + + /* + * Silently return if nothing to do. + */ + if (count == 0) + return; + + /* + * If the destination index is not the end of the current + * destination leaf entries, open up a hole in the destination + * to hold the new entries. + */ + if (start_d < dhdr->count) { + memmove(&dents[start_d + count], &dents[start_d], + (dhdr->count - start_d) * sizeof(xfs_dir2_leaf_entry_t)); + xfs_dir3_leaf_log_ents(args, dhdr, bp_d, start_d + count, + count + dhdr->count - 1); + } + /* + * If the source has stale leaves, count the ones in the copy range + * so we can update the header correctly. + */ + if (shdr->stale) { + int i; /* temp leaf index */ + + for (i = start_s, stale = 0; i < start_s + count; i++) { + if (sents[i].address == + cpu_to_be32(XFS_DIR2_NULL_DATAPTR)) + stale++; + } + } else + stale = 0; + /* + * Copy the leaf entries from source to destination. + */ + memcpy(&dents[start_d], &sents[start_s], + count * sizeof(xfs_dir2_leaf_entry_t)); + xfs_dir3_leaf_log_ents(args, dhdr, bp_d, start_d, start_d + count - 1); + + /* + * If there are source entries after the ones we copied, + * delete the ones we copied by sliding the next ones down. + */ + if (start_s + count < shdr->count) { + memmove(&sents[start_s], &sents[start_s + count], + count * sizeof(xfs_dir2_leaf_entry_t)); + xfs_dir3_leaf_log_ents(args, shdr, bp_s, start_s, + start_s + count - 1); + } + + /* + * Update the headers and log them. + */ + shdr->count -= count; + shdr->stale -= stale; + dhdr->count += count; + dhdr->stale += stale; +} + +/* + * Determine the sort order of two leaf blocks. + * Returns 1 if both are valid and leaf2 should be before leaf1, else 0. + */ +int /* sort order */ +xfs_dir2_leafn_order( + struct xfs_inode *dp, + struct xfs_buf *leaf1_bp, /* leaf1 buffer */ + struct xfs_buf *leaf2_bp) /* leaf2 buffer */ +{ + struct xfs_dir2_leaf *leaf1 = leaf1_bp->b_addr; + struct xfs_dir2_leaf *leaf2 = leaf2_bp->b_addr; + struct xfs_dir2_leaf_entry *ents1; + struct xfs_dir2_leaf_entry *ents2; + struct xfs_dir3_icleaf_hdr hdr1; + struct xfs_dir3_icleaf_hdr hdr2; + + xfs_dir2_leaf_hdr_from_disk(dp->i_mount, &hdr1, leaf1); + xfs_dir2_leaf_hdr_from_disk(dp->i_mount, &hdr2, leaf2); + ents1 = hdr1.ents; + ents2 = hdr2.ents; + + if (hdr1.count > 0 && hdr2.count > 0 && + (be32_to_cpu(ents2[0].hashval) < be32_to_cpu(ents1[0].hashval) || + be32_to_cpu(ents2[hdr2.count - 1].hashval) < + be32_to_cpu(ents1[hdr1.count - 1].hashval))) + return 1; + return 0; +} + +/* + * Rebalance leaf entries between two leaf blocks. + * This is actually only called when the second block is new, + * though the code deals with the general case. + * A new entry will be inserted in one of the blocks, and that + * entry is taken into account when balancing. + */ +static void +xfs_dir2_leafn_rebalance( + xfs_da_state_t *state, /* btree cursor */ + xfs_da_state_blk_t *blk1, /* first btree block */ + xfs_da_state_blk_t *blk2) /* second btree block */ +{ + xfs_da_args_t *args; /* operation arguments */ + int count; /* count (& direction) leaves */ + int isleft; /* new goes in left leaf */ + xfs_dir2_leaf_t *leaf1; /* first leaf structure */ + xfs_dir2_leaf_t *leaf2; /* second leaf structure */ + int mid; /* midpoint leaf index */ +#if defined(DEBUG) || defined(XFS_WARN) + int oldstale; /* old count of stale leaves */ +#endif + int oldsum; /* old total leaf count */ + int swap_blocks; /* swapped leaf blocks */ + struct xfs_dir2_leaf_entry *ents1; + struct xfs_dir2_leaf_entry *ents2; + struct xfs_dir3_icleaf_hdr hdr1; + struct xfs_dir3_icleaf_hdr hdr2; + struct xfs_inode *dp = state->args->dp; + + args = state->args; + /* + * If the block order is wrong, swap the arguments. + */ + swap_blocks = xfs_dir2_leafn_order(dp, blk1->bp, blk2->bp); + if (swap_blocks) + swap(blk1, blk2); + + leaf1 = blk1->bp->b_addr; + leaf2 = blk2->bp->b_addr; + xfs_dir2_leaf_hdr_from_disk(dp->i_mount, &hdr1, leaf1); + xfs_dir2_leaf_hdr_from_disk(dp->i_mount, &hdr2, leaf2); + ents1 = hdr1.ents; + ents2 = hdr2.ents; + + oldsum = hdr1.count + hdr2.count; +#if defined(DEBUG) || defined(XFS_WARN) + oldstale = hdr1.stale + hdr2.stale; +#endif + mid = oldsum >> 1; + + /* + * If the old leaf count was odd then the new one will be even, + * so we need to divide the new count evenly. + */ + if (oldsum & 1) { + xfs_dahash_t midhash; /* middle entry hash value */ + + if (mid >= hdr1.count) + midhash = be32_to_cpu(ents2[mid - hdr1.count].hashval); + else + midhash = be32_to_cpu(ents1[mid].hashval); + isleft = args->hashval <= midhash; + } + /* + * If the old count is even then the new count is odd, so there's + * no preferred side for the new entry. + * Pick the left one. + */ + else + isleft = 1; + /* + * Calculate moved entry count. Positive means left-to-right, + * negative means right-to-left. Then move the entries. + */ + count = hdr1.count - mid + (isleft == 0); + if (count > 0) + xfs_dir3_leafn_moveents(args, blk1->bp, &hdr1, ents1, + hdr1.count - count, blk2->bp, + &hdr2, ents2, 0, count); + else if (count < 0) + xfs_dir3_leafn_moveents(args, blk2->bp, &hdr2, ents2, 0, + blk1->bp, &hdr1, ents1, + hdr1.count, count); + + ASSERT(hdr1.count + hdr2.count == oldsum); + ASSERT(hdr1.stale + hdr2.stale == oldstale); + + /* log the changes made when moving the entries */ + xfs_dir2_leaf_hdr_to_disk(dp->i_mount, leaf1, &hdr1); + xfs_dir2_leaf_hdr_to_disk(dp->i_mount, leaf2, &hdr2); + xfs_dir3_leaf_log_header(args, blk1->bp); + xfs_dir3_leaf_log_header(args, blk2->bp); + + xfs_dir3_leaf_check(dp, blk1->bp); + xfs_dir3_leaf_check(dp, blk2->bp); + + /* + * Mark whether we're inserting into the old or new leaf. + */ + if (hdr1.count < hdr2.count) + state->inleaf = swap_blocks; + else if (hdr1.count > hdr2.count) + state->inleaf = !swap_blocks; + else + state->inleaf = swap_blocks ^ (blk1->index <= hdr1.count); + /* + * Adjust the expected index for insertion. + */ + if (!state->inleaf) + blk2->index = blk1->index - hdr1.count; + + /* + * Finally sanity check just to make sure we are not returning a + * negative index + */ + if (blk2->index < 0) { + state->inleaf = 1; + blk2->index = 0; + xfs_alert(dp->i_mount, + "%s: picked the wrong leaf? reverting original leaf: blk1->index %d", + __func__, blk1->index); + } +} + +static int +xfs_dir3_data_block_free( + xfs_da_args_t *args, + struct xfs_dir2_data_hdr *hdr, + struct xfs_dir2_free *free, + xfs_dir2_db_t fdb, + int findex, + struct xfs_buf *fbp, + int longest) +{ + int logfree = 0; + struct xfs_dir3_icfree_hdr freehdr; + struct xfs_inode *dp = args->dp; + + xfs_dir2_free_hdr_from_disk(dp->i_mount, &freehdr, free); + if (hdr) { + /* + * Data block is not empty, just set the free entry to the new + * value. + */ + freehdr.bests[findex] = cpu_to_be16(longest); + xfs_dir2_free_log_bests(args, &freehdr, fbp, findex, findex); + return 0; + } + + /* One less used entry in the free table. */ + freehdr.nused--; + + /* + * If this was the last entry in the table, we can trim the table size + * back. There might be other entries at the end referring to + * non-existent data blocks, get those too. + */ + if (findex == freehdr.nvalid - 1) { + int i; /* free entry index */ + + for (i = findex - 1; i >= 0; i--) { + if (freehdr.bests[i] != cpu_to_be16(NULLDATAOFF)) + break; + } + freehdr.nvalid = i + 1; + logfree = 0; + } else { + /* Not the last entry, just punch it out. */ + freehdr.bests[findex] = cpu_to_be16(NULLDATAOFF); + logfree = 1; + } + + xfs_dir2_free_hdr_to_disk(dp->i_mount, free, &freehdr); + xfs_dir2_free_log_header(args, fbp); + + /* + * If there are no useful entries left in the block, get rid of the + * block if we can. + */ + if (!freehdr.nused) { + int error; + + error = xfs_dir2_shrink_inode(args, fdb, fbp); + if (error == 0) { + fbp = NULL; + logfree = 0; + } else if (error != -ENOSPC || args->total != 0) + return error; + /* + * It's possible to get ENOSPC if there is no + * space reservation. In this case some one + * else will eventually get rid of this block. + */ + } + + /* Log the free entry that changed, unless we got rid of it. */ + if (logfree) + xfs_dir2_free_log_bests(args, &freehdr, fbp, findex, findex); + return 0; +} + +/* + * Remove an entry from a node directory. + * This removes the leaf entry and the data entry, + * and updates the free block if necessary. + */ +static int /* error */ +xfs_dir2_leafn_remove( + xfs_da_args_t *args, /* operation arguments */ + struct xfs_buf *bp, /* leaf buffer */ + int index, /* leaf entry index */ + xfs_da_state_blk_t *dblk, /* data block */ + int *rval) /* resulting block needs join */ +{ + struct xfs_da_geometry *geo = args->geo; + xfs_dir2_data_hdr_t *hdr; /* data block header */ + xfs_dir2_db_t db; /* data block number */ + struct xfs_buf *dbp; /* data block buffer */ + xfs_dir2_data_entry_t *dep; /* data block entry */ + xfs_inode_t *dp; /* incore directory inode */ + xfs_dir2_leaf_t *leaf; /* leaf structure */ + xfs_dir2_leaf_entry_t *lep; /* leaf entry */ + int longest; /* longest data free entry */ + int off; /* data block entry offset */ + int needlog; /* need to log data header */ + int needscan; /* need to rescan data frees */ + xfs_trans_t *tp; /* transaction pointer */ + struct xfs_dir2_data_free *bf; /* bestfree table */ + struct xfs_dir3_icleaf_hdr leafhdr; + + trace_xfs_dir2_leafn_remove(args, index); + + dp = args->dp; + tp = args->trans; + leaf = bp->b_addr; + xfs_dir2_leaf_hdr_from_disk(dp->i_mount, &leafhdr, leaf); + + /* + * Point to the entry we're removing. + */ + lep = &leafhdr.ents[index]; + + /* + * Extract the data block and offset from the entry. + */ + db = xfs_dir2_dataptr_to_db(geo, be32_to_cpu(lep->address)); + ASSERT(dblk->blkno == db); + off = xfs_dir2_dataptr_to_off(geo, be32_to_cpu(lep->address)); + ASSERT(dblk->index == off); + + /* + * Kill the leaf entry by marking it stale. + * Log the leaf block changes. + */ + leafhdr.stale++; + xfs_dir2_leaf_hdr_to_disk(dp->i_mount, leaf, &leafhdr); + xfs_dir3_leaf_log_header(args, bp); + + lep->address = cpu_to_be32(XFS_DIR2_NULL_DATAPTR); + xfs_dir3_leaf_log_ents(args, &leafhdr, bp, index, index); + + /* + * Make the data entry free. Keep track of the longest freespace + * in the data block in case it changes. + */ + dbp = dblk->bp; + hdr = dbp->b_addr; + dep = (xfs_dir2_data_entry_t *)((char *)hdr + off); + bf = xfs_dir2_data_bestfree_p(dp->i_mount, hdr); + longest = be16_to_cpu(bf[0].length); + needlog = needscan = 0; + xfs_dir2_data_make_free(args, dbp, off, + xfs_dir2_data_entsize(dp->i_mount, dep->namelen), &needlog, + &needscan); + /* + * Rescan the data block freespaces for bestfree. + * Log the data block header if needed. + */ + if (needscan) + xfs_dir2_data_freescan(dp->i_mount, hdr, &needlog); + if (needlog) + xfs_dir2_data_log_header(args, dbp); + xfs_dir3_data_check(dp, dbp); + /* + * If the longest data block freespace changes, need to update + * the corresponding freeblock entry. + */ + if (longest < be16_to_cpu(bf[0].length)) { + int error; /* error return value */ + struct xfs_buf *fbp; /* freeblock buffer */ + xfs_dir2_db_t fdb; /* freeblock block number */ + int findex; /* index in freeblock entries */ + xfs_dir2_free_t *free; /* freeblock structure */ + + /* + * Convert the data block number to a free block, + * read in the free block. + */ + fdb = xfs_dir2_db_to_fdb(geo, db); + error = xfs_dir2_free_read(tp, dp, args->owner, + xfs_dir2_db_to_da(geo, fdb), &fbp); + if (error) + return error; + free = fbp->b_addr; +#ifdef DEBUG + { + struct xfs_dir3_icfree_hdr freehdr; + + xfs_dir2_free_hdr_from_disk(dp->i_mount, &freehdr, free); + ASSERT(freehdr.firstdb == geo->free_max_bests * + (fdb - xfs_dir2_byte_to_db(geo, XFS_DIR2_FREE_OFFSET))); + } +#endif + /* + * Calculate which entry we need to fix. + */ + findex = xfs_dir2_db_to_fdindex(geo, db); + longest = be16_to_cpu(bf[0].length); + /* + * If the data block is now empty we can get rid of it + * (usually). + */ + if (longest == geo->blksize - geo->data_entry_offset) { + /* + * Try to punch out the data block. + */ + error = xfs_dir2_shrink_inode(args, db, dbp); + if (error == 0) { + dblk->bp = NULL; + hdr = NULL; + } + /* + * We can get ENOSPC if there's no space reservation. + * In this case just drop the buffer and some one else + * will eventually get rid of the empty block. + */ + else if (!(error == -ENOSPC && args->total == 0)) + return error; + } + /* + * If we got rid of the data block, we can eliminate that entry + * in the free block. + */ + error = xfs_dir3_data_block_free(args, hdr, free, + fdb, findex, fbp, longest); + if (error) + return error; + } + + xfs_dir3_leaf_check(dp, bp); + /* + * Return indication of whether this leaf block is empty enough + * to justify trying to join it with a neighbor. + */ + *rval = (geo->leaf_hdr_size + + (uint)sizeof(leafhdr.ents) * (leafhdr.count - leafhdr.stale)) < + geo->magicpct; + return 0; +} + +/* + * Split the leaf entries in the old block into old and new blocks. + */ +int /* error */ +xfs_dir2_leafn_split( + xfs_da_state_t *state, /* btree cursor */ + xfs_da_state_blk_t *oldblk, /* original block */ + xfs_da_state_blk_t *newblk) /* newly created block */ +{ + xfs_da_args_t *args; /* operation arguments */ + xfs_dablk_t blkno; /* new leaf block number */ + int error; /* error return value */ + struct xfs_inode *dp; + + /* + * Allocate space for a new leaf node. + */ + args = state->args; + dp = args->dp; + ASSERT(oldblk->magic == XFS_DIR2_LEAFN_MAGIC); + error = xfs_da_grow_inode(args, &blkno); + if (error) { + return error; + } + /* + * Initialize the new leaf block. + */ + error = xfs_dir3_leaf_get_buf(args, xfs_dir2_da_to_db(args->geo, blkno), + &newblk->bp, XFS_DIR2_LEAFN_MAGIC); + if (error) + return error; + + newblk->blkno = blkno; + newblk->magic = XFS_DIR2_LEAFN_MAGIC; + /* + * Rebalance the entries across the two leaves, link the new + * block into the leaves. + */ + xfs_dir2_leafn_rebalance(state, oldblk, newblk); + error = xfs_da3_blk_link(state, oldblk, newblk); + if (error) { + return error; + } + /* + * Insert the new entry in the correct block. + */ + if (state->inleaf) + error = xfs_dir2_leafn_add(oldblk->bp, args, oldblk->index); + else + error = xfs_dir2_leafn_add(newblk->bp, args, newblk->index); + /* + * Update last hashval in each block since we added the name. + */ + oldblk->hashval = xfs_dir2_leaf_lasthash(dp, oldblk->bp, NULL); + newblk->hashval = xfs_dir2_leaf_lasthash(dp, newblk->bp, NULL); + xfs_dir3_leaf_check(dp, oldblk->bp); + xfs_dir3_leaf_check(dp, newblk->bp); + return error; +} + +/* + * Check a leaf block and its neighbors to see if the block should be + * collapsed into one or the other neighbor. Always keep the block + * with the smaller block number. + * If the current block is over 50% full, don't try to join it, return 0. + * If the block is empty, fill in the state structure and return 2. + * If it can be collapsed, fill in the state structure and return 1. + * If nothing can be done, return 0. + */ +int /* error */ +xfs_dir2_leafn_toosmall( + xfs_da_state_t *state, /* btree cursor */ + int *action) /* resulting action to take */ +{ + xfs_da_state_blk_t *blk; /* leaf block */ + xfs_dablk_t blkno; /* leaf block number */ + struct xfs_buf *bp; /* leaf buffer */ + int bytes; /* bytes in use */ + int count; /* leaf live entry count */ + int error; /* error return value */ + int forward; /* sibling block direction */ + int i; /* sibling counter */ + xfs_dir2_leaf_t *leaf; /* leaf structure */ + int rval; /* result from path_shift */ + struct xfs_dir3_icleaf_hdr leafhdr; + struct xfs_dir2_leaf_entry *ents; + struct xfs_inode *dp = state->args->dp; + + /* + * Check for the degenerate case of the block being over 50% full. + * If so, it's not worth even looking to see if we might be able + * to coalesce with a sibling. + */ + blk = &state->path.blk[state->path.active - 1]; + leaf = blk->bp->b_addr; + xfs_dir2_leaf_hdr_from_disk(dp->i_mount, &leafhdr, leaf); + ents = leafhdr.ents; + xfs_dir3_leaf_check(dp, blk->bp); + + count = leafhdr.count - leafhdr.stale; + bytes = state->args->geo->leaf_hdr_size + count * sizeof(ents[0]); + if (bytes > (state->args->geo->blksize >> 1)) { + /* + * Blk over 50%, don't try to join. + */ + *action = 0; + return 0; + } + /* + * Check for the degenerate case of the block being empty. + * If the block is empty, we'll simply delete it, no need to + * coalesce it with a sibling block. We choose (arbitrarily) + * to merge with the forward block unless it is NULL. + */ + if (count == 0) { + /* + * Make altpath point to the block we want to keep and + * path point to the block we want to drop (this one). + */ + forward = (leafhdr.forw != 0); + memcpy(&state->altpath, &state->path, sizeof(state->path)); + error = xfs_da3_path_shift(state, &state->altpath, forward, 0, + &rval); + if (error) + return error; + *action = rval ? 2 : 0; + return 0; + } + /* + * Examine each sibling block to see if we can coalesce with + * at least 25% free space to spare. We need to figure out + * whether to merge with the forward or the backward block. + * We prefer coalescing with the lower numbered sibling so as + * to shrink a directory over time. + */ + forward = leafhdr.forw < leafhdr.back; + for (i = 0, bp = NULL; i < 2; forward = !forward, i++) { + struct xfs_dir3_icleaf_hdr hdr2; + + blkno = forward ? leafhdr.forw : leafhdr.back; + if (blkno == 0) + continue; + /* + * Read the sibling leaf block. + */ + error = xfs_dir3_leafn_read(state->args->trans, dp, + state->args->owner, blkno, &bp); + if (error) + return error; + + /* + * Count bytes in the two blocks combined. + */ + count = leafhdr.count - leafhdr.stale; + bytes = state->args->geo->blksize - + (state->args->geo->blksize >> 2); + + leaf = bp->b_addr; + xfs_dir2_leaf_hdr_from_disk(dp->i_mount, &hdr2, leaf); + ents = hdr2.ents; + count += hdr2.count - hdr2.stale; + bytes -= count * sizeof(ents[0]); + + /* + * Fits with at least 25% to spare. + */ + if (bytes >= 0) + break; + xfs_trans_brelse(state->args->trans, bp); + } + /* + * Didn't like either block, give up. + */ + if (i >= 2) { + *action = 0; + return 0; + } + + /* + * Make altpath point to the block we want to keep (the lower + * numbered block) and path point to the block we want to drop. + */ + memcpy(&state->altpath, &state->path, sizeof(state->path)); + if (blkno < blk->blkno) + error = xfs_da3_path_shift(state, &state->altpath, forward, 0, + &rval); + else + error = xfs_da3_path_shift(state, &state->path, forward, 0, + &rval); + if (error) { + return error; + } + *action = rval ? 0 : 1; + return 0; +} + +/* + * Move all the leaf entries from drop_blk to save_blk. + * This is done as part of a join operation. + */ +void +xfs_dir2_leafn_unbalance( + xfs_da_state_t *state, /* cursor */ + xfs_da_state_blk_t *drop_blk, /* dead block */ + xfs_da_state_blk_t *save_blk) /* surviving block */ +{ + xfs_da_args_t *args; /* operation arguments */ + xfs_dir2_leaf_t *drop_leaf; /* dead leaf structure */ + xfs_dir2_leaf_t *save_leaf; /* surviving leaf structure */ + struct xfs_dir3_icleaf_hdr savehdr; + struct xfs_dir3_icleaf_hdr drophdr; + struct xfs_dir2_leaf_entry *sents; + struct xfs_dir2_leaf_entry *dents; + struct xfs_inode *dp = state->args->dp; + + args = state->args; + ASSERT(drop_blk->magic == XFS_DIR2_LEAFN_MAGIC); + ASSERT(save_blk->magic == XFS_DIR2_LEAFN_MAGIC); + drop_leaf = drop_blk->bp->b_addr; + save_leaf = save_blk->bp->b_addr; + + xfs_dir2_leaf_hdr_from_disk(dp->i_mount, &savehdr, save_leaf); + xfs_dir2_leaf_hdr_from_disk(dp->i_mount, &drophdr, drop_leaf); + sents = savehdr.ents; + dents = drophdr.ents; + + /* + * If there are any stale leaf entries, take this opportunity + * to purge them. + */ + if (drophdr.stale) + xfs_dir3_leaf_compact(args, &drophdr, drop_blk->bp); + if (savehdr.stale) + xfs_dir3_leaf_compact(args, &savehdr, save_blk->bp); + + /* + * Move the entries from drop to the appropriate end of save. + */ + drop_blk->hashval = be32_to_cpu(dents[drophdr.count - 1].hashval); + if (xfs_dir2_leafn_order(dp, save_blk->bp, drop_blk->bp)) + xfs_dir3_leafn_moveents(args, drop_blk->bp, &drophdr, dents, 0, + save_blk->bp, &savehdr, sents, 0, + drophdr.count); + else + xfs_dir3_leafn_moveents(args, drop_blk->bp, &drophdr, dents, 0, + save_blk->bp, &savehdr, sents, + savehdr.count, drophdr.count); + save_blk->hashval = be32_to_cpu(sents[savehdr.count - 1].hashval); + + /* log the changes made when moving the entries */ + xfs_dir2_leaf_hdr_to_disk(dp->i_mount, save_leaf, &savehdr); + xfs_dir2_leaf_hdr_to_disk(dp->i_mount, drop_leaf, &drophdr); + xfs_dir3_leaf_log_header(args, save_blk->bp); + xfs_dir3_leaf_log_header(args, drop_blk->bp); + + xfs_dir3_leaf_check(dp, save_blk->bp); + xfs_dir3_leaf_check(dp, drop_blk->bp); +} + +/* + * Add a new data block to the directory at the free space index that the caller + * has specified. + */ +static int +xfs_dir2_node_add_datablk( + struct xfs_da_args *args, + struct xfs_da_state_blk *fblk, + xfs_dir2_db_t *dbno, + struct xfs_buf **dbpp, + struct xfs_buf **fbpp, + struct xfs_dir3_icfree_hdr *hdr, + int *findex) +{ + struct xfs_inode *dp = args->dp; + struct xfs_trans *tp = args->trans; + struct xfs_mount *mp = dp->i_mount; + struct xfs_dir2_data_free *bf; + xfs_dir2_db_t fbno; + struct xfs_buf *fbp; + struct xfs_buf *dbp; + int error; + + /* Not allowed to allocate, return failure. */ + if (args->total == 0) + return -ENOSPC; + + /* Allocate and initialize the new data block. */ + error = xfs_dir2_grow_inode(args, XFS_DIR2_DATA_SPACE, dbno); + if (error) + return error; + error = xfs_dir3_data_init(args, *dbno, &dbp); + if (error) + return error; + + /* + * Get the freespace block corresponding to the data block + * that was just allocated. + */ + fbno = xfs_dir2_db_to_fdb(args->geo, *dbno); + error = xfs_dir2_free_try_read(tp, dp, args->owner, + xfs_dir2_db_to_da(args->geo, fbno), &fbp); + if (error) + return error; + + /* + * If there wasn't a freespace block, the read will + * return a NULL fbp. Allocate and initialize a new one. + */ + if (!fbp) { + error = xfs_dir2_grow_inode(args, XFS_DIR2_FREE_SPACE, &fbno); + if (error) + return error; + + if (XFS_IS_CORRUPT(mp, + xfs_dir2_db_to_fdb(args->geo, *dbno) != + fbno)) { + xfs_alert(mp, +"%s: dir ino %llu needed freesp block %lld for data block %lld, got %lld", + __func__, (unsigned long long)I_INO(dp), + (long long)xfs_dir2_db_to_fdb(args->geo, *dbno), + (long long)*dbno, (long long)fbno); + if (fblk) { + xfs_alert(mp, + " fblk "PTR_FMT" blkno %llu index %d magic 0x%x", + fblk, (unsigned long long)fblk->blkno, + fblk->index, fblk->magic); + } else { + xfs_alert(mp, " ... fblk is NULL"); + } + xfs_da_mark_sick(args); + return -EFSCORRUPTED; + } + + /* Get a buffer for the new block. */ + error = xfs_dir3_free_get_buf(args, fbno, &fbp); + if (error) + return error; + xfs_dir2_free_hdr_from_disk(mp, hdr, fbp->b_addr); + + /* Remember the first slot as our empty slot. */ + hdr->firstdb = (fbno - xfs_dir2_byte_to_db(args->geo, + XFS_DIR2_FREE_OFFSET)) * + args->geo->free_max_bests; + } else { + xfs_dir2_free_hdr_from_disk(mp, hdr, fbp->b_addr); + } + + /* Set the freespace block index from the data block number. */ + *findex = xfs_dir2_db_to_fdindex(args->geo, *dbno); + + /* Extend the freespace table if the new data block is off the end. */ + if (*findex >= hdr->nvalid) { + ASSERT(*findex < args->geo->free_max_bests); + hdr->nvalid = *findex + 1; + hdr->bests[*findex] = cpu_to_be16(NULLDATAOFF); + } + + /* + * If this entry was for an empty data block (this should always be + * true) then update the header. + */ + if (hdr->bests[*findex] == cpu_to_be16(NULLDATAOFF)) { + hdr->nused++; + xfs_dir2_free_hdr_to_disk(mp, fbp->b_addr, hdr); + xfs_dir2_free_log_header(args, fbp); + } + + /* Update the freespace value for the new block in the table. */ + bf = xfs_dir2_data_bestfree_p(mp, dbp->b_addr); + hdr->bests[*findex] = bf[0].length; + + *dbpp = dbp; + *fbpp = fbp; + return 0; +} + +static int +xfs_dir2_node_find_freeblk( + struct xfs_da_args *args, + struct xfs_da_state_blk *fblk, + xfs_dir2_db_t *dbnop, + struct xfs_buf **fbpp, + struct xfs_dir3_icfree_hdr *hdr, + int *findexp, + int length) +{ + struct xfs_inode *dp = args->dp; + struct xfs_trans *tp = args->trans; + struct xfs_buf *fbp = NULL; + xfs_dir2_db_t firstfbno; + xfs_dir2_db_t lastfbno; + xfs_dir2_db_t ifbno = -1; + xfs_dir2_db_t dbno = -1; + xfs_dir2_db_t fbno; + xfs_fileoff_t fo; + int findex = 0; + int error; + + /* + * If we came in with a freespace block that means that lookup + * found an entry with our hash value. This is the freespace + * block for that data entry. + */ + if (fblk) { + fbp = fblk->bp; + findex = fblk->index; + xfs_dir2_free_hdr_from_disk(dp->i_mount, hdr, fbp->b_addr); + if (findex >= 0) { + /* caller already found the freespace for us. */ + ASSERT(findex < hdr->nvalid); + ASSERT(be16_to_cpu(hdr->bests[findex]) != NULLDATAOFF); + ASSERT(be16_to_cpu(hdr->bests[findex]) >= length); + dbno = hdr->firstdb + findex; + goto found_block; + } + + /* + * The data block looked at didn't have enough room. + * We'll start at the beginning of the freespace entries. + */ + ifbno = fblk->blkno; + xfs_trans_brelse(tp, fbp); + fbp = NULL; + fblk->bp = NULL; + } + + /* + * If we don't have a data block yet, we're going to scan the freespace + * data for a data block with enough free space in it. + */ + error = xfs_bmap_last_offset(dp, &fo, XFS_DATA_FORK); + if (error) + return error; + lastfbno = xfs_dir2_da_to_db(args->geo, (xfs_dablk_t)fo); + firstfbno = xfs_dir2_byte_to_db(args->geo, XFS_DIR2_FREE_OFFSET); + + for (fbno = lastfbno - 1; fbno >= firstfbno; fbno--) { + /* If it's ifbno we already looked at it. */ + if (fbno == ifbno) + continue; + + /* + * Read the block. There can be holes in the freespace blocks, + * so this might not succeed. This should be really rare, so + * there's no reason to avoid it. + */ + error = xfs_dir2_free_try_read(tp, dp, args->owner, + xfs_dir2_db_to_da(args->geo, fbno), + &fbp); + if (error) + return error; + if (!fbp) + continue; + + xfs_dir2_free_hdr_from_disk(dp->i_mount, hdr, fbp->b_addr); + + /* Scan the free entry array for a large enough free space. */ + for (findex = hdr->nvalid - 1; findex >= 0; findex--) { + if (be16_to_cpu(hdr->bests[findex]) != NULLDATAOFF && + be16_to_cpu(hdr->bests[findex]) >= length) { + dbno = hdr->firstdb + findex; + goto found_block; + } + } + + /* Didn't find free space, go on to next free block */ + xfs_trans_brelse(tp, fbp); + } + +found_block: + *dbnop = dbno; + *fbpp = fbp; + *findexp = findex; + return 0; +} + +/* + * Add the data entry for a node-format directory name addition. + * The leaf entry is added in xfs_dir2_leafn_add. + * We may enter with a freespace block that the lookup found. + */ +static int +xfs_dir2_node_addname_int( + struct xfs_da_args *args, /* operation arguments */ + struct xfs_da_state_blk *fblk) /* optional freespace block */ +{ + struct xfs_dir2_data_unused *dup; /* data unused entry pointer */ + struct xfs_dir2_data_entry *dep; /* data entry pointer */ + struct xfs_dir2_data_hdr *hdr; /* data block header */ + struct xfs_dir2_data_free *bf; + struct xfs_trans *tp = args->trans; + struct xfs_inode *dp = args->dp; + struct xfs_dir3_icfree_hdr freehdr; + struct xfs_buf *dbp; /* data block buffer */ + struct xfs_buf *fbp; /* freespace buffer */ + xfs_dir2_data_aoff_t aoff; + xfs_dir2_db_t dbno; /* data block number */ + int error; /* error return value */ + int findex; /* freespace entry index */ + int length; /* length of the new entry */ + int logfree = 0; /* need to log free entry */ + int needlog = 0; /* need to log data header */ + int needscan = 0; /* need to rescan data frees */ + __be16 *tagp; /* data entry tag pointer */ + + length = xfs_dir2_data_entsize(dp->i_mount, args->namelen); + error = xfs_dir2_node_find_freeblk(args, fblk, &dbno, &fbp, &freehdr, + &findex, length); + if (error) + return error; + + /* + * Now we know if we must allocate blocks, so if we are checking whether + * we can insert without allocation then we can return now. + */ + if (args->op_flags & XFS_DA_OP_JUSTCHECK) { + if (dbno == -1) + return -ENOSPC; + return 0; + } + + /* + * If we don't have a data block, we need to allocate one and make + * the freespace entries refer to it. + */ + if (dbno == -1) { + /* we're going to have to log the free block index later */ + logfree = 1; + error = xfs_dir2_node_add_datablk(args, fblk, &dbno, &dbp, &fbp, + &freehdr, &findex); + } else { + /* Read the data block in. */ + error = xfs_dir3_data_read(tp, dp, args->owner, + xfs_dir2_db_to_da(args->geo, dbno), 0, &dbp); + } + if (error) + return error; + + /* setup for data block up now */ + hdr = dbp->b_addr; + bf = xfs_dir2_data_bestfree_p(dp->i_mount, hdr); + ASSERT(be16_to_cpu(bf[0].length) >= length); + + /* Point to the existing unused space. */ + dup = (xfs_dir2_data_unused_t *) + ((char *)hdr + be16_to_cpu(bf[0].offset)); + + /* Mark the first part of the unused space, inuse for us. */ + aoff = (xfs_dir2_data_aoff_t)((char *)dup - (char *)hdr); + error = xfs_dir2_data_use_free(args, dbp, dup, aoff, length, + &needlog, &needscan); + if (error) { + xfs_trans_brelse(tp, dbp); + return error; + } + + /* Fill in the new entry and log it. */ + dep = (xfs_dir2_data_entry_t *)dup; + dep->inumber = cpu_to_be64(args->inumber); + dep->namelen = args->namelen; + memcpy(dep->name, args->name, dep->namelen); + xfs_dir2_data_put_ftype(dp->i_mount, dep, args->filetype); + tagp = xfs_dir2_data_entry_tag_p(dp->i_mount, dep); + *tagp = cpu_to_be16((char *)dep - (char *)hdr); + xfs_dir2_data_log_entry(args, dbp, dep); + + /* Rescan the freespace and log the data block if needed. */ + if (needscan) + xfs_dir2_data_freescan(dp->i_mount, hdr, &needlog); + if (needlog) + xfs_dir2_data_log_header(args, dbp); + + /* If the freespace block entry is now wrong, update it. */ + if (freehdr.bests[findex] != bf[0].length) { + freehdr.bests[findex] = bf[0].length; + logfree = 1; + } + + /* Log the freespace entry if needed. */ + if (logfree) + xfs_dir2_free_log_bests(args, &freehdr, fbp, findex, findex); + + /* Return the data block and offset in args. */ + args->blkno = (xfs_dablk_t)dbno; + args->index = be16_to_cpu(*tagp); + return 0; +} + +/* + * Top-level node form directory addname routine. + */ +int /* error */ +xfs_dir2_node_addname( + xfs_da_args_t *args) /* operation arguments */ +{ + xfs_da_state_blk_t *blk; /* leaf block for insert */ + int error; /* error return value */ + int rval; /* sub-return value */ + xfs_da_state_t *state; /* btree cursor */ + + trace_xfs_dir2_node_addname(args); + + /* + * Allocate and initialize the state (btree cursor). + */ + state = xfs_da_state_alloc(args); + /* + * Look up the name. We're not supposed to find it, but + * this gives us the insertion point. + */ + error = xfs_da3_node_lookup_int(state, &rval); + if (error) + rval = error; + if (rval != -ENOENT) { + goto done; + } + /* + * Add the data entry to a data block. + * Extravalid is set to a freeblock found by lookup. + */ + rval = xfs_dir2_node_addname_int(args, + state->extravalid ? &state->extrablk : NULL); + if (rval) { + goto done; + } + blk = &state->path.blk[state->path.active - 1]; + ASSERT(blk->magic == XFS_DIR2_LEAFN_MAGIC); + /* + * Add the new leaf entry. + */ + rval = xfs_dir2_leafn_add(blk->bp, args, blk->index); + if (rval == 0) { + /* + * It worked, fix the hash values up the btree. + */ + if (!(args->op_flags & XFS_DA_OP_JUSTCHECK)) + xfs_da3_fixhashpath(state, &state->path); + } else { + /* + * It didn't work, we need to split the leaf block. + */ + if (args->total == 0) { + ASSERT(rval == -ENOSPC); + goto done; + } + /* + * Split the leaf block and insert the new entry. + */ + rval = xfs_da3_split(state); + } +done: + xfs_da_state_free(state); + return rval; +} + +/* + * Lookup an entry in a node-format directory. + * All the real work happens in xfs_da3_node_lookup_int. + * The only real output is the inode number of the entry. + */ +int /* error */ +xfs_dir2_node_lookup( + xfs_da_args_t *args) /* operation arguments */ +{ + int error; /* error return value */ + int i; /* btree level */ + int rval; /* operation return value */ + xfs_da_state_t *state; /* btree cursor */ + + trace_xfs_dir2_node_lookup(args); + + /* + * Allocate and initialize the btree cursor. + */ + state = xfs_da_state_alloc(args); + + /* + * Fill in the path to the entry in the cursor. + */ + error = xfs_da3_node_lookup_int(state, &rval); + if (error) + rval = error; + else if (rval == -ENOENT && args->cmpresult == XFS_CMP_CASE) { + /* If a CI match, dup the actual name and return -EEXIST */ + xfs_dir2_data_entry_t *dep; + + dep = (xfs_dir2_data_entry_t *) + ((char *)state->extrablk.bp->b_addr + + state->extrablk.index); + rval = xfs_dir_cilookup_result(args, dep->name, dep->namelen); + } + /* + * Release the btree blocks and leaf block. + */ + for (i = 0; i < state->path.active; i++) { + xfs_trans_brelse(args->trans, state->path.blk[i].bp); + state->path.blk[i].bp = NULL; + } + /* + * Release the data block if we have it. + */ + if (state->extravalid && state->extrablk.bp) { + xfs_trans_brelse(args->trans, state->extrablk.bp); + state->extrablk.bp = NULL; + } + xfs_da_state_free(state); + return rval; +} + +/* + * Remove an entry from a node-format directory. + */ +int /* error */ +xfs_dir2_node_removename( + struct xfs_da_args *args) /* operation arguments */ +{ + struct xfs_da_state_blk *blk; /* leaf block */ + int error; /* error return value */ + int rval; /* operation return value */ + struct xfs_da_state *state; /* btree cursor */ + + trace_xfs_dir2_node_removename(args); + + /* + * Allocate and initialize the btree cursor. + */ + state = xfs_da_state_alloc(args); + + /* Look up the entry we're deleting, set up the cursor. */ + error = xfs_da3_node_lookup_int(state, &rval); + if (error) + goto out_free; + + /* Didn't find it, upper layer screwed up. */ + if (rval != -EEXIST) { + error = rval; + goto out_free; + } + + blk = &state->path.blk[state->path.active - 1]; + ASSERT(blk->magic == XFS_DIR2_LEAFN_MAGIC); + ASSERT(state->extravalid); + /* + * Remove the leaf and data entries. + * Extrablk refers to the data block. + */ + error = xfs_dir2_leafn_remove(args, blk->bp, blk->index, + &state->extrablk, &rval); + if (error) + goto out_free; + /* + * Fix the hash values up the btree. + */ + xfs_da3_fixhashpath(state, &state->path); + /* + * If we need to join leaf blocks, do it. + */ + if (rval && state->path.active > 1) + error = xfs_da3_join(state); + /* + * If no errors so far, try conversion to leaf format. + */ + if (!error) + error = xfs_dir2_node_to_leaf(state); +out_free: + xfs_da_state_free(state); + return error; +} + +/* + * Replace an entry's inode number in a node-format directory. + */ +int /* error */ +xfs_dir2_node_replace( + xfs_da_args_t *args) /* operation arguments */ +{ + xfs_da_state_blk_t *blk; /* leaf block */ + xfs_dir2_data_hdr_t *hdr; /* data block header */ + xfs_dir2_data_entry_t *dep; /* data entry changed */ + int error; /* error return value */ + int i; /* btree level */ + xfs_ino_t inum; /* new inode number */ + int ftype; /* new file type */ + int rval; /* internal return value */ + xfs_da_state_t *state; /* btree cursor */ + + trace_xfs_dir2_node_replace(args); + + /* + * Allocate and initialize the btree cursor. + */ + state = xfs_da_state_alloc(args); + + /* + * We have to save new inode number and ftype since + * xfs_da3_node_lookup_int() is going to overwrite them + */ + inum = args->inumber; + ftype = args->filetype; + + /* + * Lookup the entry to change in the btree. + */ + error = xfs_da3_node_lookup_int(state, &rval); + if (error) { + rval = error; + } + /* + * It should be found, since the vnodeops layer has looked it up + * and locked it. But paranoia is good. + */ + if (rval == -EEXIST) { + struct xfs_dir3_icleaf_hdr leafhdr; + + /* + * Find the leaf entry. + */ + blk = &state->path.blk[state->path.active - 1]; + ASSERT(blk->magic == XFS_DIR2_LEAFN_MAGIC); + ASSERT(state->extravalid); + + xfs_dir2_leaf_hdr_from_disk(state->mp, &leafhdr, + blk->bp->b_addr); + /* + * Point to the data entry. + */ + hdr = state->extrablk.bp->b_addr; + ASSERT(hdr->magic == cpu_to_be32(XFS_DIR2_DATA_MAGIC) || + hdr->magic == cpu_to_be32(XFS_DIR3_DATA_MAGIC)); + dep = (xfs_dir2_data_entry_t *) + ((char *)hdr + + xfs_dir2_dataptr_to_off(args->geo, + be32_to_cpu(leafhdr.ents[blk->index].address))); + ASSERT(inum != be64_to_cpu(dep->inumber)); + /* + * Fill in the new inode number and log the entry. + */ + dep->inumber = cpu_to_be64(inum); + xfs_dir2_data_put_ftype(state->mp, dep, ftype); + xfs_dir2_data_log_entry(args, state->extrablk.bp, dep); + rval = 0; + } + /* + * Didn't find it, and we're holding a data block. Drop it. + */ + else if (state->extravalid) { + xfs_trans_brelse(args->trans, state->extrablk.bp); + state->extrablk.bp = NULL; + } + /* + * Release all the buffers in the cursor. + */ + for (i = 0; i < state->path.active; i++) { + xfs_trans_brelse(args->trans, state->path.blk[i].bp); + state->path.blk[i].bp = NULL; + } + xfs_da_state_free(state); + return rval; +} + +/* + * Trim off a trailing empty freespace block. + * Return (in rvalp) 1 if we did it, 0 if not. + */ +int /* error */ +xfs_dir2_node_trim_free( + xfs_da_args_t *args, /* operation arguments */ + xfs_fileoff_t fo, /* free block number */ + int *rvalp) /* out: did something */ +{ + struct xfs_buf *bp; /* freespace buffer */ + xfs_inode_t *dp; /* incore directory inode */ + int error; /* error return code */ + xfs_dir2_free_t *free; /* freespace structure */ + xfs_trans_t *tp; /* transaction pointer */ + struct xfs_dir3_icfree_hdr freehdr; + + dp = args->dp; + tp = args->trans; + + *rvalp = 0; + + /* + * Read the freespace block. + */ + error = xfs_dir2_free_try_read(tp, dp, args->owner, fo, &bp); + if (error) + return error; + /* + * There can be holes in freespace. If fo is a hole, there's + * nothing to do. + */ + if (!bp) + return 0; + free = bp->b_addr; + xfs_dir2_free_hdr_from_disk(dp->i_mount, &freehdr, free); + + /* + * If there are used entries, there's nothing to do. + */ + if (freehdr.nused > 0) { + xfs_trans_brelse(tp, bp); + return 0; + } + /* + * Blow the block away. + */ + error = xfs_dir2_shrink_inode(args, + xfs_dir2_da_to_db(args->geo, (xfs_dablk_t)fo), bp); + if (error) { + /* + * Can't fail with ENOSPC since that only happens with no + * space reservation, when breaking up an extent into two + * pieces. This is the last block of an extent. + */ + ASSERT(error != -ENOSPC); + xfs_trans_brelse(tp, bp); + return error; + } + /* + * Return that we succeeded. + */ + *rvalp = 1; + return 0; +} diff --git a/fs/xfs/libxfs/xfs_dir2_priv.h b/fs/xfs/libxfs/xfs_dir2_priv.h new file mode 100644 index 000000000..100413502 --- /dev/null +++ b/fs/xfs/libxfs/xfs_dir2_priv.h @@ -0,0 +1,218 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2000-2001,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_DIR2_PRIV_H__ +#define __XFS_DIR2_PRIV_H__ + +struct dir_context; + +/* + * In-core version of the leaf and free block headers to abstract the + * differences in the v2 and v3 disk format of the headers. + */ +struct xfs_dir3_icleaf_hdr { + uint32_t forw; + uint32_t back; + uint16_t magic; + uint16_t count; + uint16_t stale; + + /* + * Pointer to the on-disk format entries, which are behind the + * variable size (v4 vs v5) header in the on-disk block. + */ + struct xfs_dir2_leaf_entry *ents; +}; + +struct xfs_dir3_icfree_hdr { + uint32_t magic; + uint32_t firstdb; + uint32_t nvalid; + uint32_t nused; + + /* + * Pointer to the on-disk format entries, which are behind the + * variable size (v4 vs v5) header in the on-disk block. + */ + __be16 *bests; +}; + +/* xfs_dir2.c */ +xfs_dahash_t xfs_ascii_ci_hashname(const struct xfs_name *name); +enum xfs_dacmp xfs_ascii_ci_compname(struct xfs_da_args *args, + const unsigned char *name, int len); +extern int xfs_dir2_grow_inode(struct xfs_da_args *args, int space, + xfs_dir2_db_t *dbp); +extern int xfs_dir_cilookup_result(struct xfs_da_args *args, + const unsigned char *name, int len); + + +/* xfs_dir2_block.c */ +int xfs_dir3_block_read(struct xfs_trans *tp, struct xfs_inode *dp, + xfs_ino_t owner, struct xfs_buf **bpp); +extern int xfs_dir2_block_addname(struct xfs_da_args *args); +extern int xfs_dir2_block_lookup(struct xfs_da_args *args); +extern int xfs_dir2_block_removename(struct xfs_da_args *args); +extern int xfs_dir2_block_replace(struct xfs_da_args *args); +extern int xfs_dir2_leaf_to_block(struct xfs_da_args *args, + struct xfs_buf *lbp, struct xfs_buf *dbp); + +/* xfs_dir2_data.c */ +struct xfs_dir2_data_free *xfs_dir2_data_bestfree_p(struct xfs_mount *mp, + struct xfs_dir2_data_hdr *hdr); +__be16 *xfs_dir2_data_entry_tag_p(struct xfs_mount *mp, + struct xfs_dir2_data_entry *dep); +uint8_t xfs_dir2_data_get_ftype(struct xfs_mount *mp, + struct xfs_dir2_data_entry *dep); +void xfs_dir2_data_put_ftype(struct xfs_mount *mp, + struct xfs_dir2_data_entry *dep, uint8_t ftype); + +#ifdef DEBUG +extern void xfs_dir3_data_check(struct xfs_inode *dp, struct xfs_buf *bp); +#else +#define xfs_dir3_data_check(dp,bp) +#endif + +extern xfs_failaddr_t __xfs_dir3_data_check(struct xfs_inode *dp, + struct xfs_buf *bp); +int xfs_dir3_data_read(struct xfs_trans *tp, struct xfs_inode *dp, + xfs_ino_t owner, xfs_dablk_t bno, unsigned int flags, + struct xfs_buf **bpp); +int xfs_dir3_data_readahead(struct xfs_inode *dp, xfs_dablk_t bno, + unsigned int flags); + +extern struct xfs_dir2_data_free * +xfs_dir2_data_freeinsert(struct xfs_dir2_data_hdr *hdr, + struct xfs_dir2_data_free *bf, struct xfs_dir2_data_unused *dup, + int *loghead); +extern int xfs_dir3_data_init(struct xfs_da_args *args, xfs_dir2_db_t blkno, + struct xfs_buf **bpp); + +/* xfs_dir2_leaf.c */ +void xfs_dir2_leaf_hdr_from_disk(struct xfs_mount *mp, + struct xfs_dir3_icleaf_hdr *to, struct xfs_dir2_leaf *from); +void xfs_dir2_leaf_hdr_to_disk(struct xfs_mount *mp, struct xfs_dir2_leaf *to, + struct xfs_dir3_icleaf_hdr *from); +int xfs_dir3_leaf_read(struct xfs_trans *tp, struct xfs_inode *dp, + xfs_ino_t owner, xfs_dablk_t fbno, struct xfs_buf **bpp); +int xfs_dir3_leafn_read(struct xfs_trans *tp, struct xfs_inode *dp, + xfs_ino_t owner, xfs_dablk_t fbno, struct xfs_buf **bpp); +extern int xfs_dir2_block_to_leaf(struct xfs_da_args *args, + struct xfs_buf *dbp); +extern int xfs_dir2_leaf_addname(struct xfs_da_args *args); +extern void xfs_dir3_leaf_compact(struct xfs_da_args *args, + struct xfs_dir3_icleaf_hdr *leafhdr, struct xfs_buf *bp); +extern void xfs_dir3_leaf_compact_x1(struct xfs_dir3_icleaf_hdr *leafhdr, + struct xfs_dir2_leaf_entry *ents, int *indexp, + int *lowstalep, int *highstalep, int *lowlogp, int *highlogp); +extern int xfs_dir3_leaf_get_buf(struct xfs_da_args *args, xfs_dir2_db_t bno, + struct xfs_buf **bpp, uint16_t magic); +extern void xfs_dir3_leaf_log_ents(struct xfs_da_args *args, + struct xfs_dir3_icleaf_hdr *hdr, struct xfs_buf *bp, int first, + int last); +extern void xfs_dir3_leaf_log_header(struct xfs_da_args *args, + struct xfs_buf *bp); +extern int xfs_dir2_leaf_lookup(struct xfs_da_args *args); +extern int xfs_dir2_leaf_removename(struct xfs_da_args *args); +extern int xfs_dir2_leaf_replace(struct xfs_da_args *args); +extern int xfs_dir2_leaf_search_hash(struct xfs_da_args *args, + struct xfs_buf *lbp); +extern int xfs_dir2_leaf_trim_data(struct xfs_da_args *args, + struct xfs_buf *lbp, xfs_dir2_db_t db); +extern struct xfs_dir2_leaf_entry * +xfs_dir3_leaf_find_entry(struct xfs_dir3_icleaf_hdr *leafhdr, + struct xfs_dir2_leaf_entry *ents, int index, int compact, + int lowstale, int highstale, int *lfloglow, int *lfloghigh); +extern int xfs_dir2_node_to_leaf(struct xfs_da_state *state); + +extern xfs_failaddr_t xfs_dir3_leaf_check_int(struct xfs_mount *mp, + struct xfs_dir3_icleaf_hdr *hdr, struct xfs_dir2_leaf *leaf, + bool expensive_checks); + +/* xfs_dir2_node.c */ +void xfs_dir2_free_hdr_from_disk(struct xfs_mount *mp, + struct xfs_dir3_icfree_hdr *to, struct xfs_dir2_free *from); +extern int xfs_dir2_leaf_to_node(struct xfs_da_args *args, + struct xfs_buf *lbp); +extern xfs_dahash_t xfs_dir2_leaf_lasthash(struct xfs_inode *dp, + struct xfs_buf *bp, int *count); +extern int xfs_dir2_leafn_lookup_int(struct xfs_buf *bp, + struct xfs_da_args *args, int *indexp, + struct xfs_da_state *state); +extern int xfs_dir2_leafn_order(struct xfs_inode *dp, struct xfs_buf *leaf1_bp, + struct xfs_buf *leaf2_bp); +extern int xfs_dir2_leafn_split(struct xfs_da_state *state, + struct xfs_da_state_blk *oldblk, struct xfs_da_state_blk *newblk); +extern int xfs_dir2_leafn_toosmall(struct xfs_da_state *state, int *action); +extern void xfs_dir2_leafn_unbalance(struct xfs_da_state *state, + struct xfs_da_state_blk *drop_blk, + struct xfs_da_state_blk *save_blk); +extern int xfs_dir2_node_addname(struct xfs_da_args *args); +extern int xfs_dir2_node_lookup(struct xfs_da_args *args); +extern int xfs_dir2_node_removename(struct xfs_da_args *args); +extern int xfs_dir2_node_replace(struct xfs_da_args *args); +extern int xfs_dir2_node_trim_free(struct xfs_da_args *args, xfs_fileoff_t fo, + int *rvalp); +int xfs_dir2_free_read(struct xfs_trans *tp, struct xfs_inode *dp, + xfs_ino_t owner, xfs_dablk_t fbno, struct xfs_buf **bpp); + +/* xfs_dir2_sf.c */ +xfs_ino_t xfs_dir2_sf_get_ino(struct xfs_mount *mp, struct xfs_dir2_sf_hdr *hdr, + struct xfs_dir2_sf_entry *sfep); +xfs_ino_t xfs_dir2_sf_get_parent_ino(struct xfs_dir2_sf_hdr *hdr); +void xfs_dir2_sf_put_parent_ino(struct xfs_dir2_sf_hdr *hdr, xfs_ino_t ino); +uint8_t xfs_dir2_sf_get_ftype(struct xfs_mount *mp, + struct xfs_dir2_sf_entry *sfep); +struct xfs_dir2_sf_entry *xfs_dir2_sf_nextentry(struct xfs_mount *mp, + struct xfs_dir2_sf_hdr *hdr, struct xfs_dir2_sf_entry *sfep); +extern int xfs_dir2_block_sfsize(struct xfs_inode *dp, + struct xfs_dir2_data_hdr *block, struct xfs_dir2_sf_hdr *sfhp); +extern int xfs_dir2_block_to_sf(struct xfs_da_args *args, struct xfs_buf *bp, + int size, xfs_dir2_sf_hdr_t *sfhp); +extern int xfs_dir2_sf_addname(struct xfs_da_args *args); +extern int xfs_dir2_sf_create(struct xfs_da_args *args, xfs_ino_t pino); +extern int xfs_dir2_sf_lookup(struct xfs_da_args *args); +extern int xfs_dir2_sf_removename(struct xfs_da_args *args); +extern int xfs_dir2_sf_replace(struct xfs_da_args *args); +xfs_failaddr_t xfs_dir2_sf_verify(struct xfs_mount *mp, + struct xfs_dir2_sf_hdr *sfp, int64_t size); +int xfs_dir2_sf_entsize(struct xfs_mount *mp, + struct xfs_dir2_sf_hdr *hdr, int len); +void xfs_dir2_sf_put_ino(struct xfs_mount *mp, struct xfs_dir2_sf_hdr *hdr, + struct xfs_dir2_sf_entry *sfep, xfs_ino_t ino); +void xfs_dir2_sf_put_ftype(struct xfs_mount *mp, + struct xfs_dir2_sf_entry *sfep, uint8_t ftype); + +/* xfs_dir2_readdir.c */ +extern int xfs_readdir(struct xfs_trans *tp, struct xfs_inode *dp, + struct dir_context *ctx, size_t bufsize); + +static inline unsigned int +xfs_dir2_data_unusedsize( + unsigned int len) +{ + return round_up(len, XFS_DIR2_DATA_ALIGN); +} + +static inline unsigned int +xfs_dir2_data_entsize( + struct xfs_mount *mp, + unsigned int namelen) +{ + unsigned int len; + + len = offsetof(struct xfs_dir2_data_entry, name[0]) + namelen + + sizeof(xfs_dir2_data_off_t) /* tag */; + if (xfs_has_ftype(mp)) + len += sizeof(uint8_t); + return round_up(len, XFS_DIR2_DATA_ALIGN); +} + +xfs_dahash_t xfs_dir2_hashname(struct xfs_mount *mp, + const struct xfs_name *name); +enum xfs_dacmp xfs_dir2_compname(struct xfs_da_args *args, + const unsigned char *name, int len); + +#endif /* __XFS_DIR2_PRIV_H__ */ diff --git a/fs/xfs/libxfs/xfs_dir2_sf.c b/fs/xfs/libxfs/xfs_dir2_sf.c new file mode 100644 index 000000000..0567cf8b9 --- /dev/null +++ b/fs/xfs/libxfs/xfs_dir2_sf.c @@ -0,0 +1,1268 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2003,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_dir2.h" +#include "xfs_dir2_priv.h" +#include "xfs_trace.h" + +/* + * Prototypes for internal functions. + */ +static void xfs_dir2_sf_addname_easy(xfs_da_args_t *args, + xfs_dir2_sf_entry_t *sfep, + xfs_dir2_data_aoff_t offset, + int new_isize); +static void xfs_dir2_sf_addname_hard(xfs_da_args_t *args, int objchange, + int new_isize); +static int xfs_dir2_sf_addname_pick(xfs_da_args_t *args, int objchange, + xfs_dir2_sf_entry_t **sfepp, + xfs_dir2_data_aoff_t *offsetp); +#ifdef DEBUG +static void xfs_dir2_sf_check(xfs_da_args_t *args); +#else +#define xfs_dir2_sf_check(args) +#endif /* DEBUG */ + +static void xfs_dir2_sf_toino4(xfs_da_args_t *args); +static void xfs_dir2_sf_toino8(xfs_da_args_t *args); + +int +xfs_dir2_sf_entsize( + struct xfs_mount *mp, + struct xfs_dir2_sf_hdr *hdr, + int len) +{ + int count = len; + + count += sizeof(struct xfs_dir2_sf_entry); /* namelen + offset */ + count += hdr->i8count ? XFS_INO64_SIZE : XFS_INO32_SIZE; /* ino # */ + + if (xfs_has_ftype(mp)) + count += sizeof(uint8_t); + return count; +} + +struct xfs_dir2_sf_entry * +xfs_dir2_sf_nextentry( + struct xfs_mount *mp, + struct xfs_dir2_sf_hdr *hdr, + struct xfs_dir2_sf_entry *sfep) +{ + return (void *)sfep + xfs_dir2_sf_entsize(mp, hdr, sfep->namelen); +} + +/* + * In short-form directory entries the inode numbers are stored at variable + * offset behind the entry name. If the entry stores a filetype value, then it + * sits between the name and the inode number. The actual inode numbers can + * come in two formats as well, either 4 bytes or 8 bytes wide. + */ +xfs_ino_t +xfs_dir2_sf_get_ino( + struct xfs_mount *mp, + struct xfs_dir2_sf_hdr *hdr, + struct xfs_dir2_sf_entry *sfep) +{ + uint8_t *from = sfep->name + sfep->namelen; + + if (xfs_has_ftype(mp)) + from++; + + if (!hdr->i8count) + return get_unaligned_be32(from); + return get_unaligned_be64(from) & XFS_MAXINUMBER; +} + +void +xfs_dir2_sf_put_ino( + struct xfs_mount *mp, + struct xfs_dir2_sf_hdr *hdr, + struct xfs_dir2_sf_entry *sfep, + xfs_ino_t ino) +{ + uint8_t *to = sfep->name + sfep->namelen; + + ASSERT(ino <= XFS_MAXINUMBER); + + if (xfs_has_ftype(mp)) + to++; + + if (hdr->i8count) + put_unaligned_be64(ino, to); + else + put_unaligned_be32(ino, to); +} + +xfs_ino_t +xfs_dir2_sf_get_parent_ino( + struct xfs_dir2_sf_hdr *hdr) +{ + if (!hdr->i8count) + return get_unaligned_be32(hdr->parent); + return get_unaligned_be64(hdr->parent) & XFS_MAXINUMBER; +} + +void +xfs_dir2_sf_put_parent_ino( + struct xfs_dir2_sf_hdr *hdr, + xfs_ino_t ino) +{ + ASSERT(ino <= XFS_MAXINUMBER); + + if (hdr->i8count) + put_unaligned_be64(ino, hdr->parent); + else + put_unaligned_be32(ino, hdr->parent); +} + +/* + * The file type field is stored at the end of the name for filetype enabled + * shortform directories, or not at all otherwise. + */ +uint8_t +xfs_dir2_sf_get_ftype( + struct xfs_mount *mp, + struct xfs_dir2_sf_entry *sfep) +{ + if (xfs_has_ftype(mp)) { + uint8_t ftype = sfep->name[sfep->namelen]; + + if (ftype < XFS_DIR3_FT_MAX) + return ftype; + } + + return XFS_DIR3_FT_UNKNOWN; +} + +void +xfs_dir2_sf_put_ftype( + struct xfs_mount *mp, + struct xfs_dir2_sf_entry *sfep, + uint8_t ftype) +{ + ASSERT(ftype < XFS_DIR3_FT_MAX); + + if (xfs_has_ftype(mp)) + sfep->name[sfep->namelen] = ftype; +} + +/* + * Given a block directory (dp/block), calculate its size as a shortform (sf) + * directory and a header for the sf directory, if it will fit it the + * space currently present in the inode. If it won't fit, the output + * size is too big (but not accurate). + */ +int /* size for sf form */ +xfs_dir2_block_sfsize( + xfs_inode_t *dp, /* incore inode pointer */ + xfs_dir2_data_hdr_t *hdr, /* block directory data */ + xfs_dir2_sf_hdr_t *sfhp) /* output: header for sf form */ +{ + xfs_dir2_dataptr_t addr; /* data entry address */ + xfs_dir2_leaf_entry_t *blp; /* leaf area of the block */ + xfs_dir2_block_tail_t *btp; /* tail area of the block */ + int count; /* shortform entry count */ + xfs_dir2_data_entry_t *dep; /* data entry in the block */ + int i; /* block entry index */ + int i8count; /* count of big-inode entries */ + int isdot; /* entry is "." */ + int isdotdot; /* entry is ".." */ + xfs_mount_t *mp; /* mount structure pointer */ + int namelen; /* total name bytes */ + xfs_ino_t parent = 0; /* parent inode number */ + int size=0; /* total computed size */ + int has_ftype; + struct xfs_da_geometry *geo; + + mp = dp->i_mount; + geo = mp->m_dir_geo; + + /* + * if there is a filetype field, add the extra byte to the namelen + * for each entry that we see. + */ + has_ftype = xfs_has_ftype(mp) ? 1 : 0; + + count = i8count = namelen = 0; + btp = xfs_dir2_block_tail_p(geo, hdr); + blp = xfs_dir2_block_leaf_p(btp); + + /* + * Iterate over the block's data entries by using the leaf pointers. + */ + for (i = 0; i < be32_to_cpu(btp->count); i++) { + if ((addr = be32_to_cpu(blp[i].address)) == XFS_DIR2_NULL_DATAPTR) + continue; + /* + * Calculate the pointer to the entry at hand. + */ + dep = (xfs_dir2_data_entry_t *)((char *)hdr + + xfs_dir2_dataptr_to_off(geo, addr)); + /* + * Detect . and .., so we can special-case them. + * . is not included in sf directories. + * .. is included by just the parent inode number. + */ + isdot = dep->namelen == 1 && dep->name[0] == '.'; + isdotdot = + dep->namelen == 2 && + dep->name[0] == '.' && dep->name[1] == '.'; + + if (!isdot) + i8count += be64_to_cpu(dep->inumber) > XFS_DIR2_MAX_SHORT_INUM; + + /* take into account the file type field */ + if (!isdot && !isdotdot) { + count++; + namelen += dep->namelen + has_ftype; + } else if (isdotdot) + parent = be64_to_cpu(dep->inumber); + /* + * Calculate the new size, see if we should give up yet. + */ + size = xfs_dir2_sf_hdr_size(i8count) + /* header */ + count * 3 * sizeof(u8) + /* namelen + offset */ + namelen + /* name */ + (i8count ? /* inumber */ + count * XFS_INO64_SIZE : + count * XFS_INO32_SIZE); + if (size > xfs_inode_data_fork_size(dp)) + return size; /* size value is a failure */ + } + /* + * Create the output header, if it worked. + */ + sfhp->count = count; + sfhp->i8count = i8count; + xfs_dir2_sf_put_parent_ino(sfhp, parent); + return size; +} + +/* + * Convert a block format directory to shortform. + * Caller has already checked that it will fit, and built us a header. + */ +int /* error */ +xfs_dir2_block_to_sf( + struct xfs_da_args *args, /* operation arguments */ + struct xfs_buf *bp, + int size, /* shortform directory size */ + struct xfs_dir2_sf_hdr *sfhp) /* shortform directory hdr */ +{ + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + int error; /* error return value */ + int logflags; /* inode logging flags */ + struct xfs_dir2_sf_entry *sfep; /* shortform entry */ + struct xfs_dir2_sf_hdr *sfp; /* shortform directory header */ + unsigned int offset = args->geo->data_entry_offset; + unsigned int end; + + trace_xfs_dir2_block_to_sf(args); + + /* + * Allocate a temporary destination buffer the size of the inode to + * format the data into. Once we have formatted the data, we can free + * the block and copy the formatted data into the inode literal area. + */ + sfp = kmalloc(mp->m_sb.sb_inodesize, GFP_KERNEL | __GFP_NOFAIL); + memcpy(sfp, sfhp, xfs_dir2_sf_hdr_size(sfhp->i8count)); + + /* + * Loop over the active and unused entries. Stop when we reach the + * leaf/tail portion of the block. + */ + end = xfs_dir3_data_end_offset(args->geo, bp->b_addr); + sfep = xfs_dir2_sf_firstentry(sfp); + while (offset < end) { + struct xfs_dir2_data_unused *dup = bp->b_addr + offset; + struct xfs_dir2_data_entry *dep = bp->b_addr + offset; + + /* + * If it's unused, just skip over it. + */ + if (be16_to_cpu(dup->freetag) == XFS_DIR2_DATA_FREE_TAG) { + offset += be16_to_cpu(dup->length); + continue; + } + + /* + * Skip . + */ + if (dep->namelen == 1 && dep->name[0] == '.') + ASSERT(be64_to_cpu(dep->inumber) == I_INO(dp)); + /* + * Skip .., but make sure the inode number is right. + */ + else if (dep->namelen == 2 && + dep->name[0] == '.' && dep->name[1] == '.') + ASSERT(be64_to_cpu(dep->inumber) == + xfs_dir2_sf_get_parent_ino(sfp)); + /* + * Normal entry, copy it into shortform. + */ + else { + sfep->namelen = dep->namelen; + xfs_dir2_sf_put_offset(sfep, offset); + memcpy(sfep->name, dep->name, dep->namelen); + xfs_dir2_sf_put_ino(mp, sfp, sfep, + be64_to_cpu(dep->inumber)); + xfs_dir2_sf_put_ftype(mp, sfep, + xfs_dir2_data_get_ftype(mp, dep)); + + sfep = xfs_dir2_sf_nextentry(mp, sfp, sfep); + } + offset += xfs_dir2_data_entsize(mp, dep->namelen); + } + ASSERT((char *)sfep - (char *)sfp == size); + + /* now we are done with the block, we can shrink the inode */ + logflags = XFS_ILOG_CORE; + error = xfs_dir2_shrink_inode(args, args->geo->datablk, bp); + if (error) { + ASSERT(error != -ENOSPC); + goto out; + } + + /* + * The buffer is now unconditionally gone, whether + * xfs_dir2_shrink_inode worked or not. + * + * Convert the inode to local format and copy the data in. + */ + ASSERT(dp->i_df.if_bytes == 0); + xfs_init_local_fork(dp, XFS_DATA_FORK, sfp, size); + dp->i_df.if_format = XFS_DINODE_FMT_LOCAL; + dp->i_disk_size = size; + + logflags |= XFS_ILOG_DDATA; + xfs_dir2_sf_check(args); +out: + xfs_trans_log_inode(args->trans, dp, logflags); + kfree(sfp); + return error; +} + +/* + * Add a name to a shortform directory. + * There are two algorithms, "easy" and "hard" which we decide on + * before changing anything. + * Convert to block form if necessary, if the new entry won't fit. + */ +int /* error */ +xfs_dir2_sf_addname( + xfs_da_args_t *args) /* operation arguments */ +{ + struct xfs_inode *dp = args->dp; + struct xfs_dir2_sf_hdr *sfp = dp->i_df.if_data; + int error; /* error return value */ + int incr_isize; /* total change in size */ + int new_isize; /* size after adding name */ + int objchange; /* changing to 8-byte inodes */ + xfs_dir2_data_aoff_t offset = 0; /* offset for new entry */ + int pick; /* which algorithm to use */ + xfs_dir2_sf_entry_t *sfep = NULL; /* shortform entry */ + + trace_xfs_dir2_sf_addname(args); + + ASSERT(xfs_dir2_sf_lookup(args) == -ENOENT); + ASSERT(dp->i_df.if_format == XFS_DINODE_FMT_LOCAL); + ASSERT(dp->i_disk_size >= offsetof(struct xfs_dir2_sf_hdr, parent)); + ASSERT(dp->i_df.if_bytes == dp->i_disk_size); + ASSERT(sfp != NULL); + ASSERT(dp->i_disk_size >= xfs_dir2_sf_hdr_size(sfp->i8count)); + /* + * Compute entry (and change in) size. + */ + incr_isize = xfs_dir2_sf_entsize(dp->i_mount, sfp, args->namelen); + objchange = 0; + + /* + * Do we have to change to 8 byte inodes? + */ + if (args->inumber > XFS_DIR2_MAX_SHORT_INUM && sfp->i8count == 0) { + /* + * Yes, adjust the inode size. old count + (parent + new) + */ + incr_isize += (sfp->count + 2) * XFS_INO64_DIFF; + objchange = 1; + } + + new_isize = (int)dp->i_disk_size + incr_isize; + /* + * Won't fit as shortform any more (due to size), + * or the pick routine says it won't (due to offset values). + */ + if (new_isize > xfs_inode_data_fork_size(dp) || + (pick = + xfs_dir2_sf_addname_pick(args, objchange, &sfep, &offset)) == 0) { + /* + * Just checking or no space reservation, it doesn't fit. + */ + if ((args->op_flags & XFS_DA_OP_JUSTCHECK) || args->total == 0) + return -ENOSPC; + /* + * Convert to block form then add the name. + */ + error = xfs_dir2_sf_to_block(args); + if (error) + return error; + return xfs_dir2_block_addname(args); + } + /* + * Just checking, it fits. + */ + if (args->op_flags & XFS_DA_OP_JUSTCHECK) + return 0; + /* + * Do it the easy way - just add it at the end. + */ + if (pick == 1) + xfs_dir2_sf_addname_easy(args, sfep, offset, new_isize); + /* + * Do it the hard way - look for a place to insert the new entry. + * Convert to 8 byte inode numbers first if necessary. + */ + else { + ASSERT(pick == 2); + if (objchange) + xfs_dir2_sf_toino8(args); + xfs_dir2_sf_addname_hard(args, objchange, new_isize); + } + xfs_trans_log_inode(args->trans, dp, XFS_ILOG_CORE | XFS_ILOG_DDATA); + return 0; +} + +/* + * Add the new entry the "easy" way. + * This is copying the old directory and adding the new entry at the end. + * Since it's sorted by "offset" we need room after the last offset + * that's already there, and then room to convert to a block directory. + * This is already checked by the pick routine. + */ +static void +xfs_dir2_sf_addname_easy( + xfs_da_args_t *args, /* operation arguments */ + xfs_dir2_sf_entry_t *sfep, /* pointer to new entry */ + xfs_dir2_data_aoff_t offset, /* offset to use for new ent */ + int new_isize) /* new directory size */ +{ + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + struct xfs_dir2_sf_hdr *sfp = dp->i_df.if_data; + int byteoff = (int)((char *)sfep - (char *)sfp); + + /* + * Grow the in-inode space. + */ + sfp = xfs_idata_realloc(dp, xfs_dir2_sf_entsize(mp, sfp, args->namelen), + XFS_DATA_FORK); + /* + * Need to set up again due to realloc of the inode data. + */ + sfep = (xfs_dir2_sf_entry_t *)((char *)sfp + byteoff); + /* + * Fill in the new entry. + */ + sfep->namelen = args->namelen; + xfs_dir2_sf_put_offset(sfep, offset); + memcpy(sfep->name, args->name, sfep->namelen); + xfs_dir2_sf_put_ino(mp, sfp, sfep, args->inumber); + xfs_dir2_sf_put_ftype(mp, sfep, args->filetype); + + /* + * Update the header and inode. + */ + sfp->count++; + if (args->inumber > XFS_DIR2_MAX_SHORT_INUM) + sfp->i8count++; + dp->i_disk_size = new_isize; + xfs_dir2_sf_check(args); +} + +/* + * Add the new entry the "hard" way. + * The caller has already converted to 8 byte inode numbers if necessary, + * in which case we need to leave the i8count at 1. + * Find a hole that the new entry will fit into, and copy + * the first part of the entries, the new entry, and the last part of + * the entries. + */ +/* ARGSUSED */ +static void +xfs_dir2_sf_addname_hard( + xfs_da_args_t *args, /* operation arguments */ + int objchange, /* changing inode number size */ + int new_isize) /* new directory size */ +{ + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + int add_datasize; /* data size need for new ent */ + char *buf; /* buffer for old */ + int eof; /* reached end of old dir */ + int nbytes; /* temp for byte copies */ + xfs_dir2_data_aoff_t new_offset; /* next offset value */ + xfs_dir2_data_aoff_t offset; /* current offset value */ + int old_isize; /* previous size */ + xfs_dir2_sf_entry_t *oldsfep; /* entry in original dir */ + xfs_dir2_sf_hdr_t *oldsfp; /* original shortform dir */ + xfs_dir2_sf_entry_t *sfep; /* entry in new dir */ + xfs_dir2_sf_hdr_t *sfp; /* new shortform dir */ + + /* + * Copy the old directory to the stack buffer. + */ + old_isize = (int)dp->i_disk_size; + buf = kmalloc(old_isize, GFP_KERNEL | __GFP_NOFAIL); + oldsfp = (xfs_dir2_sf_hdr_t *)buf; + memcpy(oldsfp, dp->i_df.if_data, old_isize); + /* + * Loop over the old directory finding the place we're going + * to insert the new entry. + * If it's going to end up at the end then oldsfep will point there. + */ + for (offset = args->geo->data_first_offset, + oldsfep = xfs_dir2_sf_firstentry(oldsfp), + add_datasize = xfs_dir2_data_entsize(mp, args->namelen), + eof = (char *)oldsfep == &buf[old_isize]; + !eof; + offset = new_offset + xfs_dir2_data_entsize(mp, oldsfep->namelen), + oldsfep = xfs_dir2_sf_nextentry(mp, oldsfp, oldsfep), + eof = (char *)oldsfep == &buf[old_isize]) { + new_offset = xfs_dir2_sf_get_offset(oldsfep); + if (offset + add_datasize <= new_offset) + break; + } + /* + * Get rid of the old directory, then allocate space for + * the new one. We do this so xfs_idata_realloc won't copy + * the data. + */ + xfs_idata_realloc(dp, -old_isize, XFS_DATA_FORK); + sfp = xfs_idata_realloc(dp, new_isize, XFS_DATA_FORK); + + /* + * Copy the first part of the directory, including the header. + */ + nbytes = (int)((char *)oldsfep - (char *)oldsfp); + memcpy(sfp, oldsfp, nbytes); + sfep = (xfs_dir2_sf_entry_t *)((char *)sfp + nbytes); + /* + * Fill in the new entry, and update the header counts. + */ + sfep->namelen = args->namelen; + xfs_dir2_sf_put_offset(sfep, offset); + memcpy(sfep->name, args->name, sfep->namelen); + xfs_dir2_sf_put_ino(mp, sfp, sfep, args->inumber); + xfs_dir2_sf_put_ftype(mp, sfep, args->filetype); + sfp->count++; + if (args->inumber > XFS_DIR2_MAX_SHORT_INUM && !objchange) + sfp->i8count++; + /* + * If there's more left to copy, do that. + */ + if (!eof) { + sfep = xfs_dir2_sf_nextentry(mp, sfp, sfep); + memcpy(sfep, oldsfep, old_isize - nbytes); + } + kfree(buf); + dp->i_disk_size = new_isize; + xfs_dir2_sf_check(args); +} + +/* + * Decide if the new entry will fit at all. + * If it will fit, pick between adding the new entry to the end (easy) + * or somewhere else (hard). + * Return 0 (won't fit), 1 (easy), 2 (hard). + */ +/*ARGSUSED*/ +static int /* pick result */ +xfs_dir2_sf_addname_pick( + xfs_da_args_t *args, /* operation arguments */ + int objchange, /* inode # size changes */ + xfs_dir2_sf_entry_t **sfepp, /* out(1): new entry ptr */ + xfs_dir2_data_aoff_t *offsetp) /* out(1): new offset */ +{ + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + int holefit; /* found hole it will fit in */ + int i; /* entry number */ + xfs_dir2_data_aoff_t offset; /* data block offset */ + xfs_dir2_sf_entry_t *sfep; /* shortform entry */ + struct xfs_dir2_sf_hdr *sfp = dp->i_df.if_data; + int size; /* entry's data size */ + int used; /* data bytes used */ + + size = xfs_dir2_data_entsize(mp, args->namelen); + offset = args->geo->data_first_offset; + sfep = xfs_dir2_sf_firstentry(sfp); + holefit = 0; + /* + * Loop over sf entries. + * Keep track of data offset and whether we've seen a place + * to insert the new entry. + */ + for (i = 0; i < sfp->count; i++) { + if (!holefit) + holefit = offset + size <= xfs_dir2_sf_get_offset(sfep); + offset = xfs_dir2_sf_get_offset(sfep) + + xfs_dir2_data_entsize(mp, sfep->namelen); + sfep = xfs_dir2_sf_nextentry(mp, sfp, sfep); + } + /* + * Calculate data bytes used excluding the new entry, if this + * was a data block (block form directory). + */ + used = offset + + (sfp->count + 3) * (uint)sizeof(xfs_dir2_leaf_entry_t) + + (uint)sizeof(xfs_dir2_block_tail_t); + /* + * If it won't fit in a block form then we can't insert it, + * we'll go back, convert to block, then try the insert and convert + * to leaf. + */ + if (used + (holefit ? 0 : size) > args->geo->blksize) + return 0; + /* + * If changing the inode number size, do it the hard way. + */ + if (objchange) + return 2; + /* + * If it won't fit at the end then do it the hard way (use the hole). + */ + if (used + size > args->geo->blksize) + return 2; + /* + * Do it the easy way. + */ + *sfepp = sfep; + *offsetp = offset; + return 1; +} + +#ifdef DEBUG +/* + * Check consistency of shortform directory, assert if bad. + */ +static void +xfs_dir2_sf_check( + xfs_da_args_t *args) /* operation arguments */ +{ + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + struct xfs_dir2_sf_hdr *sfp = dp->i_df.if_data; + int i; /* entry number */ + int i8count; /* number of big inode#s */ + xfs_ino_t ino; /* entry inode number */ + int offset; /* data offset */ + xfs_dir2_sf_entry_t *sfep; /* shortform dir entry */ + + offset = args->geo->data_first_offset; + ino = xfs_dir2_sf_get_parent_ino(sfp); + i8count = ino > XFS_DIR2_MAX_SHORT_INUM; + + for (i = 0, sfep = xfs_dir2_sf_firstentry(sfp); + i < sfp->count; + i++, sfep = xfs_dir2_sf_nextentry(mp, sfp, sfep)) { + ASSERT(xfs_dir2_sf_get_offset(sfep) >= offset); + ino = xfs_dir2_sf_get_ino(mp, sfp, sfep); + i8count += ino > XFS_DIR2_MAX_SHORT_INUM; + offset = + xfs_dir2_sf_get_offset(sfep) + + xfs_dir2_data_entsize(mp, sfep->namelen); + ASSERT(xfs_dir2_sf_get_ftype(mp, sfep) < XFS_DIR3_FT_MAX); + } + ASSERT(i8count == sfp->i8count); + ASSERT((char *)sfep - (char *)sfp == dp->i_disk_size); + ASSERT(offset + + (sfp->count + 2) * (uint)sizeof(xfs_dir2_leaf_entry_t) + + (uint)sizeof(xfs_dir2_block_tail_t) <= args->geo->blksize); +} +#endif /* DEBUG */ + +/* Verify the consistency of an inline directory. */ +xfs_failaddr_t +xfs_dir2_sf_verify( + struct xfs_mount *mp, + struct xfs_dir2_sf_hdr *sfp, + int64_t size) +{ + struct xfs_dir2_sf_entry *sfep; + struct xfs_dir2_sf_entry *next_sfep; + char *endp; + xfs_ino_t ino; + int i; + int i8count; + int offset; + int error; + uint8_t filetype; + + /* + * Give up if the directory is way too short. + */ + if (size <= offsetof(struct xfs_dir2_sf_hdr, parent) || + size < xfs_dir2_sf_hdr_size(sfp->i8count)) + return __this_address; + + endp = (char *)sfp + size; + + /* Check .. entry */ + ino = xfs_dir2_sf_get_parent_ino(sfp); + i8count = ino > XFS_DIR2_MAX_SHORT_INUM; + error = xfs_dir_ino_validate(mp, ino); + if (error) + return __this_address; + offset = mp->m_dir_geo->data_first_offset; + + /* Check all reported entries */ + sfep = xfs_dir2_sf_firstentry(sfp); + for (i = 0; i < sfp->count; i++) { + /* + * struct xfs_dir2_sf_entry has a variable length. + * Check the fixed-offset parts of the structure are + * within the data buffer. + */ + if (((char *)sfep + sizeof(*sfep)) >= endp) + return __this_address; + + /* Don't allow names with known bad length. */ + if (sfep->namelen == 0) + return __this_address; + + /* + * Check that the variable-length part of the structure is + * within the data buffer. The next entry starts after the + * name component, so nextentry is an acceptable test. + */ + next_sfep = xfs_dir2_sf_nextentry(mp, sfp, sfep); + if (endp < (char *)next_sfep) + return __this_address; + + /* Check that the offsets always increase. */ + if (xfs_dir2_sf_get_offset(sfep) < offset) + return __this_address; + + /* Check the inode number. */ + ino = xfs_dir2_sf_get_ino(mp, sfp, sfep); + i8count += ino > XFS_DIR2_MAX_SHORT_INUM; + error = xfs_dir_ino_validate(mp, ino); + if (error) + return __this_address; + + /* Check the file type. */ + filetype = xfs_dir2_sf_get_ftype(mp, sfep); + if (filetype >= XFS_DIR3_FT_MAX) + return __this_address; + + offset = xfs_dir2_sf_get_offset(sfep) + + xfs_dir2_data_entsize(mp, sfep->namelen); + + sfep = next_sfep; + } + if (i8count != sfp->i8count) + return __this_address; + if ((void *)sfep != (void *)endp) + return __this_address; + + /* Make sure this whole thing ought to be in local format. */ + if (offset + (sfp->count + 2) * (uint)sizeof(xfs_dir2_leaf_entry_t) + + (uint)sizeof(xfs_dir2_block_tail_t) > mp->m_dir_geo->blksize) + return __this_address; + + return NULL; +} + +/* + * Create a new (shortform) directory. + */ +int /* error, always 0 */ +xfs_dir2_sf_create( + xfs_da_args_t *args, /* operation arguments */ + xfs_ino_t pino) /* parent inode number */ +{ + xfs_inode_t *dp; /* incore directory inode */ + int i8count; /* parent inode is an 8-byte number */ + xfs_dir2_sf_hdr_t *sfp; /* shortform structure */ + int size; /* directory size */ + + trace_xfs_dir2_sf_create(args); + + dp = args->dp; + + ASSERT(dp != NULL); + ASSERT(dp->i_disk_size == 0); + /* + * If it's currently a zero-length extent file, + * convert it to local format. + */ + if (dp->i_df.if_format == XFS_DINODE_FMT_EXTENTS) { + dp->i_df.if_format = XFS_DINODE_FMT_LOCAL; + xfs_trans_log_inode(args->trans, dp, XFS_ILOG_CORE); + } + ASSERT(dp->i_df.if_format == XFS_DINODE_FMT_LOCAL); + ASSERT(dp->i_df.if_bytes == 0); + i8count = pino > XFS_DIR2_MAX_SHORT_INUM; + size = xfs_dir2_sf_hdr_size(i8count); + + /* + * Make a buffer for the data and fill in the header. + */ + sfp = xfs_idata_realloc(dp, size, XFS_DATA_FORK); + sfp->i8count = i8count; + + /* + * Now can put in the inode number, since i8count is set. + */ + xfs_dir2_sf_put_parent_ino(sfp, pino); + sfp->count = 0; + dp->i_disk_size = size; + xfs_dir2_sf_check(args); + xfs_trans_log_inode(args->trans, dp, XFS_ILOG_CORE | XFS_ILOG_DDATA); + return 0; +} + +/* + * Lookup an entry in a shortform directory. + * Returns EEXIST if found, ENOENT if not found. + */ +int /* error */ +xfs_dir2_sf_lookup( + xfs_da_args_t *args) /* operation arguments */ +{ + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + struct xfs_dir2_sf_hdr *sfp = dp->i_df.if_data; + int i; /* entry index */ + xfs_dir2_sf_entry_t *sfep; /* shortform directory entry */ + enum xfs_dacmp cmp; /* comparison result */ + xfs_dir2_sf_entry_t *ci_sfep; /* case-insens. entry */ + + trace_xfs_dir2_sf_lookup(args); + + xfs_dir2_sf_check(args); + + ASSERT(dp->i_df.if_format == XFS_DINODE_FMT_LOCAL); + ASSERT(dp->i_disk_size >= offsetof(struct xfs_dir2_sf_hdr, parent)); + ASSERT(dp->i_df.if_bytes == dp->i_disk_size); + ASSERT(sfp != NULL); + ASSERT(dp->i_disk_size >= xfs_dir2_sf_hdr_size(sfp->i8count)); + /* + * Special case for . + */ + if (args->namelen == 1 && args->name[0] == '.') { + args->inumber = I_INO(dp); + args->cmpresult = XFS_CMP_EXACT; + args->filetype = XFS_DIR3_FT_DIR; + return -EEXIST; + } + /* + * Special case for .. + */ + if (args->namelen == 2 && + args->name[0] == '.' && args->name[1] == '.') { + args->inumber = xfs_dir2_sf_get_parent_ino(sfp); + args->cmpresult = XFS_CMP_EXACT; + args->filetype = XFS_DIR3_FT_DIR; + return -EEXIST; + } + /* + * Loop over all the entries trying to match ours. + */ + ci_sfep = NULL; + for (i = 0, sfep = xfs_dir2_sf_firstentry(sfp); i < sfp->count; + i++, sfep = xfs_dir2_sf_nextentry(mp, sfp, sfep)) { + /* + * Compare name and if it's an exact match, return the inode + * number. If it's the first case-insensitive match, store the + * inode number and continue looking for an exact match. + */ + cmp = xfs_dir2_compname(args, sfep->name, sfep->namelen); + if (cmp != XFS_CMP_DIFFERENT && cmp != args->cmpresult) { + args->cmpresult = cmp; + args->inumber = xfs_dir2_sf_get_ino(mp, sfp, sfep); + args->filetype = xfs_dir2_sf_get_ftype(mp, sfep); + if (cmp == XFS_CMP_EXACT) + return -EEXIST; + ci_sfep = sfep; + } + } + ASSERT(args->op_flags & XFS_DA_OP_OKNOENT); + /* + * Here, we can only be doing a lookup (not a rename or replace). + * If a case-insensitive match was not found, return -ENOENT. + */ + if (!ci_sfep) + return -ENOENT; + /* otherwise process the CI match as required by the caller */ + return xfs_dir_cilookup_result(args, ci_sfep->name, ci_sfep->namelen); +} + +/* + * Remove an entry from a shortform directory. + */ +int /* error */ +xfs_dir2_sf_removename( + xfs_da_args_t *args) +{ + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + struct xfs_dir2_sf_hdr *sfp = dp->i_df.if_data; + int byteoff; /* offset of removed entry */ + int entsize; /* this entry's size */ + int i; /* shortform entry index */ + int newsize; /* new inode size */ + int oldsize; /* old inode size */ + xfs_dir2_sf_entry_t *sfep; /* shortform directory entry */ + + trace_xfs_dir2_sf_removename(args); + + ASSERT(dp->i_df.if_format == XFS_DINODE_FMT_LOCAL); + oldsize = (int)dp->i_disk_size; + ASSERT(oldsize >= offsetof(struct xfs_dir2_sf_hdr, parent)); + ASSERT(dp->i_df.if_bytes == oldsize); + ASSERT(sfp != NULL); + ASSERT(oldsize >= xfs_dir2_sf_hdr_size(sfp->i8count)); + /* + * Loop over the old directory entries. + * Find the one we're deleting. + */ + for (i = 0, sfep = xfs_dir2_sf_firstentry(sfp); i < sfp->count; + i++, sfep = xfs_dir2_sf_nextentry(mp, sfp, sfep)) { + if (xfs_da_compname(args, sfep->name, sfep->namelen) == + XFS_CMP_EXACT) { + ASSERT(xfs_dir2_sf_get_ino(mp, sfp, sfep) == + args->inumber); + break; + } + } + /* + * Didn't find it. + */ + if (i == sfp->count) + return -ENOENT; + /* + * Calculate sizes. + */ + byteoff = (int)((char *)sfep - (char *)sfp); + entsize = xfs_dir2_sf_entsize(mp, sfp, args->namelen); + newsize = oldsize - entsize; + /* + * Copy the part if any after the removed entry, sliding it down. + */ + if (byteoff + entsize < oldsize) + memmove((char *)sfp + byteoff, (char *)sfp + byteoff + entsize, + oldsize - (byteoff + entsize)); + /* + * Fix up the header and file size. + */ + sfp->count--; + dp->i_disk_size = newsize; + + /* + * Reallocate, making it smaller. + */ + sfp = xfs_idata_realloc(dp, newsize - oldsize, XFS_DATA_FORK); + + /* + * Are we changing inode number size? + */ + if (args->inumber > XFS_DIR2_MAX_SHORT_INUM) { + if (sfp->i8count == 1) + xfs_dir2_sf_toino4(args); + else + sfp->i8count--; + } + xfs_dir2_sf_check(args); + xfs_trans_log_inode(args->trans, dp, XFS_ILOG_CORE | XFS_ILOG_DDATA); + return 0; +} + +/* + * Check whether the sf dir replace operation need more blocks. + */ +static bool +xfs_dir2_sf_replace_needblock( + struct xfs_inode *dp, + xfs_ino_t inum) +{ + struct xfs_dir2_sf_hdr *sfp = dp->i_df.if_data; + int newsize; + + if (dp->i_df.if_format != XFS_DINODE_FMT_LOCAL) + return false; + + newsize = dp->i_df.if_bytes + (sfp->count + 1) * XFS_INO64_DIFF; + + return inum > XFS_DIR2_MAX_SHORT_INUM && + sfp->i8count == 0 && newsize > xfs_inode_data_fork_size(dp); +} + +/* + * Replace the inode number of an entry in a shortform directory. + */ +int /* error */ +xfs_dir2_sf_replace( + xfs_da_args_t *args) /* operation arguments */ +{ + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + struct xfs_dir2_sf_hdr *sfp = dp->i_df.if_data; + int i; /* entry index */ + xfs_ino_t ino=0; /* entry old inode number */ + int i8elevated; /* sf_toino8 set i8count=1 */ + xfs_dir2_sf_entry_t *sfep; /* shortform directory entry */ + + trace_xfs_dir2_sf_replace(args); + + ASSERT(dp->i_df.if_format == XFS_DINODE_FMT_LOCAL); + ASSERT(dp->i_disk_size >= offsetof(struct xfs_dir2_sf_hdr, parent)); + ASSERT(dp->i_df.if_bytes == dp->i_disk_size); + ASSERT(sfp != NULL); + ASSERT(dp->i_disk_size >= xfs_dir2_sf_hdr_size(sfp->i8count)); + + /* + * New inode number is large, and need to convert to 8-byte inodes. + */ + if (args->inumber > XFS_DIR2_MAX_SHORT_INUM && sfp->i8count == 0) { + int error; /* error return value */ + + /* + * Won't fit as shortform, convert to block then do replace. + */ + if (xfs_dir2_sf_replace_needblock(dp, args->inumber)) { + error = xfs_dir2_sf_to_block(args); + if (error) + return error; + return xfs_dir2_block_replace(args); + } + /* + * Still fits, convert to 8-byte now. + */ + xfs_dir2_sf_toino8(args); + i8elevated = 1; + sfp = dp->i_df.if_data; + } else + i8elevated = 0; + + ASSERT(args->namelen != 1 || args->name[0] != '.'); + /* + * Replace ..'s entry. + */ + if (args->namelen == 2 && + args->name[0] == '.' && args->name[1] == '.') { + ino = xfs_dir2_sf_get_parent_ino(sfp); + ASSERT(args->inumber != ino); + xfs_dir2_sf_put_parent_ino(sfp, args->inumber); + } + /* + * Normal entry, look for the name. + */ + else { + for (i = 0, sfep = xfs_dir2_sf_firstentry(sfp); i < sfp->count; + i++, sfep = xfs_dir2_sf_nextentry(mp, sfp, sfep)) { + if (xfs_da_compname(args, sfep->name, sfep->namelen) == + XFS_CMP_EXACT) { + ino = xfs_dir2_sf_get_ino(mp, sfp, sfep); + ASSERT(args->inumber != ino); + xfs_dir2_sf_put_ino(mp, sfp, sfep, + args->inumber); + xfs_dir2_sf_put_ftype(mp, sfep, args->filetype); + break; + } + } + /* + * Didn't find it. + */ + if (i == sfp->count) { + ASSERT(args->op_flags & XFS_DA_OP_OKNOENT); + if (i8elevated) + xfs_dir2_sf_toino4(args); + return -ENOENT; + } + } + /* + * See if the old number was large, the new number is small. + */ + if (ino > XFS_DIR2_MAX_SHORT_INUM && + args->inumber <= XFS_DIR2_MAX_SHORT_INUM) { + /* + * And the old count was one, so need to convert to small. + */ + if (sfp->i8count == 1) + xfs_dir2_sf_toino4(args); + else + sfp->i8count--; + } + /* + * See if the old number was small, the new number is large. + */ + if (ino <= XFS_DIR2_MAX_SHORT_INUM && + args->inumber > XFS_DIR2_MAX_SHORT_INUM) { + /* + * add to the i8count unless we just converted to 8-byte + * inodes (which does an implied i8count = 1) + */ + ASSERT(sfp->i8count != 0); + if (!i8elevated) + sfp->i8count++; + } + xfs_dir2_sf_check(args); + xfs_trans_log_inode(args->trans, dp, XFS_ILOG_DDATA); + return 0; +} + +/* + * Convert from 8-byte inode numbers to 4-byte inode numbers. + * The last 8-byte inode number is gone, but the count is still 1. + */ +static void +xfs_dir2_sf_toino4( + xfs_da_args_t *args) /* operation arguments */ +{ + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + struct xfs_dir2_sf_hdr *oldsfp = dp->i_df.if_data; + char *buf; /* old dir's buffer */ + int i; /* entry index */ + int newsize; /* new inode size */ + xfs_dir2_sf_entry_t *oldsfep; /* old sf entry */ + int oldsize; /* old inode size */ + xfs_dir2_sf_entry_t *sfep; /* new sf entry */ + xfs_dir2_sf_hdr_t *sfp; /* new sf directory */ + + trace_xfs_dir2_sf_toino4(args); + + /* + * Copy the old directory to the buffer. + * Then nuke it from the inode, and add the new buffer to the inode. + * Don't want xfs_idata_realloc copying the data here. + */ + oldsize = dp->i_df.if_bytes; + buf = kmalloc(oldsize, GFP_KERNEL | __GFP_NOFAIL); + ASSERT(oldsfp->i8count == 1); + memcpy(buf, oldsfp, oldsize); + /* + * Compute the new inode size. + */ + newsize = oldsize - (oldsfp->count + 1) * XFS_INO64_DIFF; + xfs_idata_realloc(dp, -oldsize, XFS_DATA_FORK); + xfs_idata_realloc(dp, newsize, XFS_DATA_FORK); + /* + * Reset our pointers, the data has moved. + */ + oldsfp = (xfs_dir2_sf_hdr_t *)buf; + sfp = dp->i_df.if_data; + /* + * Fill in the new header. + */ + sfp->count = oldsfp->count; + sfp->i8count = 0; + xfs_dir2_sf_put_parent_ino(sfp, xfs_dir2_sf_get_parent_ino(oldsfp)); + /* + * Copy the entries field by field. + */ + for (i = 0, sfep = xfs_dir2_sf_firstentry(sfp), + oldsfep = xfs_dir2_sf_firstentry(oldsfp); + i < sfp->count; + i++, sfep = xfs_dir2_sf_nextentry(mp, sfp, sfep), + oldsfep = xfs_dir2_sf_nextentry(mp, oldsfp, oldsfep)) { + sfep->namelen = oldsfep->namelen; + memcpy(sfep->offset, oldsfep->offset, sizeof(sfep->offset)); + memcpy(sfep->name, oldsfep->name, sfep->namelen); + xfs_dir2_sf_put_ino(mp, sfp, sfep, + xfs_dir2_sf_get_ino(mp, oldsfp, oldsfep)); + xfs_dir2_sf_put_ftype(mp, sfep, + xfs_dir2_sf_get_ftype(mp, oldsfep)); + } + /* + * Clean up the inode. + */ + kfree(buf); + dp->i_disk_size = newsize; + xfs_trans_log_inode(args->trans, dp, XFS_ILOG_CORE | XFS_ILOG_DDATA); +} + +/* + * Convert existing entries from 4-byte inode numbers to 8-byte inode numbers. + * The new entry w/ an 8-byte inode number is not there yet; we leave with + * i8count set to 1, but no corresponding 8-byte entry. + */ +static void +xfs_dir2_sf_toino8( + xfs_da_args_t *args) /* operation arguments */ +{ + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + struct xfs_dir2_sf_hdr *oldsfp = dp->i_df.if_data; + char *buf; /* old dir's buffer */ + int i; /* entry index */ + int newsize; /* new inode size */ + xfs_dir2_sf_entry_t *oldsfep; /* old sf entry */ + int oldsize; /* old inode size */ + xfs_dir2_sf_entry_t *sfep; /* new sf entry */ + xfs_dir2_sf_hdr_t *sfp; /* new sf directory */ + + trace_xfs_dir2_sf_toino8(args); + + /* + * Copy the old directory to the buffer. + * Then nuke it from the inode, and add the new buffer to the inode. + * Don't want xfs_idata_realloc copying the data here. + */ + oldsize = dp->i_df.if_bytes; + buf = kmalloc(oldsize, GFP_KERNEL | __GFP_NOFAIL); + ASSERT(oldsfp->i8count == 0); + memcpy(buf, oldsfp, oldsize); + /* + * Compute the new inode size (nb: entry count + 1 for parent) + */ + newsize = oldsize + (oldsfp->count + 1) * XFS_INO64_DIFF; + xfs_idata_realloc(dp, -oldsize, XFS_DATA_FORK); + xfs_idata_realloc(dp, newsize, XFS_DATA_FORK); + /* + * Reset our pointers, the data has moved. + */ + oldsfp = (xfs_dir2_sf_hdr_t *)buf; + sfp = dp->i_df.if_data; + /* + * Fill in the new header. + */ + sfp->count = oldsfp->count; + sfp->i8count = 1; + xfs_dir2_sf_put_parent_ino(sfp, xfs_dir2_sf_get_parent_ino(oldsfp)); + /* + * Copy the entries field by field. + */ + for (i = 0, sfep = xfs_dir2_sf_firstentry(sfp), + oldsfep = xfs_dir2_sf_firstentry(oldsfp); + i < sfp->count; + i++, sfep = xfs_dir2_sf_nextentry(mp, sfp, sfep), + oldsfep = xfs_dir2_sf_nextentry(mp, oldsfp, oldsfep)) { + sfep->namelen = oldsfep->namelen; + memcpy(sfep->offset, oldsfep->offset, sizeof(sfep->offset)); + memcpy(sfep->name, oldsfep->name, sfep->namelen); + xfs_dir2_sf_put_ino(mp, sfp, sfep, + xfs_dir2_sf_get_ino(mp, oldsfp, oldsfep)); + xfs_dir2_sf_put_ftype(mp, sfep, + xfs_dir2_sf_get_ftype(mp, oldsfep)); + } + /* + * Clean up the inode. + */ + kfree(buf); + dp->i_disk_size = newsize; + xfs_trans_log_inode(args->trans, dp, XFS_ILOG_CORE | XFS_ILOG_DDATA); +} diff --git a/fs/xfs/libxfs/xfs_dquot_buf.c b/fs/xfs/libxfs/xfs_dquot_buf.c new file mode 100644 index 000000000..77954d1d9 --- /dev/null +++ b/fs/xfs/libxfs/xfs_dquot_buf.c @@ -0,0 +1,506 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2006 Silicon Graphics, Inc. + * Copyright (c) 2013 Red Hat, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_quota.h" +#include "xfs_trans.h" +#include "xfs_qm.h" +#include "xfs_error.h" +#include "xfs_health.h" +#include "xfs_metadir.h" +#include "xfs_metafile.h" + +int +xfs_calc_dquots_per_chunk( + unsigned int nbblks) /* basic block units */ +{ + ASSERT(nbblks > 0); + return BBTOB(nbblks) / sizeof(struct xfs_dqblk); +} + +/* + * Do some primitive error checking on ondisk dquot data structures. + * + * The xfs_dqblk structure /contains/ the xfs_disk_dquot structure; + * we verify them separately because at some points we have only the + * smaller xfs_disk_dquot structure available. + */ + +xfs_failaddr_t +xfs_dquot_verify( + struct xfs_mount *mp, + struct xfs_disk_dquot *ddq, + xfs_dqid_t id) /* used only during quotacheck */ +{ + __u8 ddq_type; + + /* + * We can encounter an uninitialized dquot buffer for 2 reasons: + * 1. If we crash while deleting the quotainode(s), and those blks got + * used for user data. This is because we take the path of regular + * file deletion; however, the size field of quotainodes is never + * updated, so all the tricks that we play in itruncate_finish + * don't quite matter. + * + * 2. We don't play the quota buffers when there's a quotaoff logitem. + * But the allocation will be replayed so we'll end up with an + * uninitialized quota block. + * + * This is all fine; things are still consistent, and we haven't lost + * any quota information. Just don't complain about bad dquot blks. + */ + if (ddq->d_magic != cpu_to_be16(XFS_DQUOT_MAGIC)) + return __this_address; + if (ddq->d_version != XFS_DQUOT_VERSION) + return __this_address; + + if (ddq->d_type & ~XFS_DQTYPE_ANY) + return __this_address; + ddq_type = ddq->d_type & XFS_DQTYPE_REC_MASK; + if (ddq_type != XFS_DQTYPE_USER && + ddq_type != XFS_DQTYPE_PROJ && + ddq_type != XFS_DQTYPE_GROUP) + return __this_address; + + if ((ddq->d_type & XFS_DQTYPE_BIGTIME) && + !xfs_has_bigtime(mp)) + return __this_address; + + if ((ddq->d_type & XFS_DQTYPE_BIGTIME) && !ddq->d_id) + return __this_address; + + if (id != -1 && id != be32_to_cpu(ddq->d_id)) + return __this_address; + + if (!ddq->d_id) + return NULL; + + if (ddq->d_blk_softlimit && + be64_to_cpu(ddq->d_bcount) > be64_to_cpu(ddq->d_blk_softlimit) && + !ddq->d_btimer) + return __this_address; + + if (ddq->d_ino_softlimit && + be64_to_cpu(ddq->d_icount) > be64_to_cpu(ddq->d_ino_softlimit) && + !ddq->d_itimer) + return __this_address; + + if (ddq->d_rtb_softlimit && + be64_to_cpu(ddq->d_rtbcount) > be64_to_cpu(ddq->d_rtb_softlimit) && + !ddq->d_rtbtimer) + return __this_address; + + return NULL; +} + +xfs_failaddr_t +xfs_dqblk_verify( + struct xfs_mount *mp, + struct xfs_dqblk *dqb, + xfs_dqid_t id) /* used only during quotacheck */ +{ + if (xfs_has_crc(mp) && + !uuid_equal(&dqb->dd_uuid, &mp->m_sb.sb_meta_uuid)) + return __this_address; + + return xfs_dquot_verify(mp, &dqb->dd_diskdq, id); +} + +/* + * Do some primitive error checking on ondisk dquot data structures. + */ +void +xfs_dqblk_repair( + struct xfs_mount *mp, + struct xfs_dqblk *dqb, + xfs_dqid_t id, + xfs_dqtype_t type) +{ + /* + * Typically, a repair is only requested by quotacheck. + */ + ASSERT(id != -1); + memset(dqb, 0, sizeof(struct xfs_dqblk)); + + dqb->dd_diskdq.d_magic = cpu_to_be16(XFS_DQUOT_MAGIC); + dqb->dd_diskdq.d_version = XFS_DQUOT_VERSION; + dqb->dd_diskdq.d_type = type; + dqb->dd_diskdq.d_id = cpu_to_be32(id); + + if (xfs_has_crc(mp)) { + uuid_copy(&dqb->dd_uuid, &mp->m_sb.sb_meta_uuid); + xfs_update_cksum((char *)dqb, sizeof(struct xfs_dqblk), + XFS_DQUOT_CRC_OFF); + } +} + +STATIC bool +xfs_dquot_buf_verify_crc( + struct xfs_mount *mp, + struct xfs_buf *bp, + bool readahead) +{ + struct xfs_dqblk *d = (struct xfs_dqblk *)bp->b_addr; + int ndquots; + int i; + + if (!xfs_has_crc(mp)) + return true; + + /* + * if we are in log recovery, the quota subsystem has not been + * initialised so we have no quotainfo structure. In that case, we need + * to manually calculate the number of dquots in the buffer. + */ + if (mp->m_quotainfo) + ndquots = mp->m_quotainfo->qi_dqperchunk; + else + ndquots = xfs_calc_dquots_per_chunk(bp->b_length); + + for (i = 0; i < ndquots; i++, d++) { + if (!xfs_verify_cksum((char *)d, sizeof(struct xfs_dqblk), + XFS_DQUOT_CRC_OFF)) { + if (!readahead) + xfs_buf_verifier_error(bp, -EFSBADCRC, __func__, + d, sizeof(*d), __this_address); + return false; + } + } + return true; +} + +STATIC xfs_failaddr_t +xfs_dquot_buf_verify( + struct xfs_mount *mp, + struct xfs_buf *bp, + bool readahead) +{ + struct xfs_dqblk *dqb = bp->b_addr; + xfs_failaddr_t fa; + xfs_dqid_t id = 0; + int ndquots; + int i; + + /* + * if we are in log recovery, the quota subsystem has not been + * initialised so we have no quotainfo structure. In that case, we need + * to manually calculate the number of dquots in the buffer. + */ + if (mp->m_quotainfo) + ndquots = mp->m_quotainfo->qi_dqperchunk; + else + ndquots = xfs_calc_dquots_per_chunk(bp->b_length); + + /* + * On the first read of the buffer, verify that each dquot is valid. + * We don't know what the id of the dquot is supposed to be, just that + * they should be increasing monotonically within the buffer. If the + * first id is corrupt, then it will fail on the second dquot in the + * buffer so corruptions could point to the wrong dquot in this case. + */ + for (i = 0; i < ndquots; i++) { + struct xfs_disk_dquot *ddq; + + ddq = &dqb[i].dd_diskdq; + + if (i == 0) + id = be32_to_cpu(ddq->d_id); + + fa = xfs_dqblk_verify(mp, &dqb[i], id + i); + if (fa) { + if (!readahead) + xfs_buf_verifier_error(bp, -EFSCORRUPTED, + __func__, &dqb[i], + sizeof(struct xfs_dqblk), fa); + return fa; + } + } + + return NULL; +} + +static xfs_failaddr_t +xfs_dquot_buf_verify_struct( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + + return xfs_dquot_buf_verify(mp, bp, false); +} + +static void +xfs_dquot_buf_read_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + + if (!xfs_dquot_buf_verify_crc(mp, bp, false)) + return; + xfs_dquot_buf_verify(mp, bp, false); +} + +/* + * readahead errors are silent and simply leave the buffer as !done so a real + * read will then be run with the xfs_dquot_buf_ops verifier. See + * xfs_inode_buf_verify() for why we use EIO here rather than reporting the + * failure. + */ +static void +xfs_dquot_buf_readahead_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + + if (!xfs_dquot_buf_verify_crc(mp, bp, true) || + xfs_dquot_buf_verify(mp, bp, true) != NULL) + xfs_buf_ioerror(bp, -EIO); +} + +/* + * we don't calculate the CRC here as that is done when the dquot is flushed to + * the buffer after the update is done. This ensures that the dquot in the + * buffer always has an up-to-date CRC value. + */ +static void +xfs_dquot_buf_write_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + + xfs_dquot_buf_verify(mp, bp, false); +} + +const struct xfs_buf_ops xfs_dquot_buf_ops = { + .name = "xfs_dquot", + .magic16 = { cpu_to_be16(XFS_DQUOT_MAGIC), + cpu_to_be16(XFS_DQUOT_MAGIC) }, + .verify_read = xfs_dquot_buf_read_verify, + .verify_write = xfs_dquot_buf_write_verify, + .verify_struct = xfs_dquot_buf_verify_struct, +}; + +const struct xfs_buf_ops xfs_dquot_buf_ra_ops = { + .name = "xfs_dquot_ra", + .magic16 = { cpu_to_be16(XFS_DQUOT_MAGIC), + cpu_to_be16(XFS_DQUOT_MAGIC) }, + .verify_read = xfs_dquot_buf_readahead_verify, + .verify_write = xfs_dquot_buf_write_verify, +}; + +/* Convert an on-disk timer value into an incore timer value. */ +time64_t +xfs_dquot_from_disk_ts( + struct xfs_disk_dquot *ddq, + __be32 dtimer) +{ + uint32_t t = be32_to_cpu(dtimer); + + if (t != 0 && (ddq->d_type & XFS_DQTYPE_BIGTIME)) + return xfs_dq_bigtime_to_unix(t); + + return t; +} + +/* Convert an incore timer value into an on-disk timer value. */ +__be32 +xfs_dquot_to_disk_ts( + struct xfs_dquot *dqp, + time64_t timer) +{ + uint32_t t = timer; + + if (timer != 0 && (dqp->q_type & XFS_DQTYPE_BIGTIME)) + t = xfs_dq_unix_to_bigtime(timer); + + return cpu_to_be32(t); +} + +inline unsigned int +xfs_dqinode_sick_mask(xfs_dqtype_t type) +{ + switch (type) { + case XFS_DQTYPE_USER: + return XFS_SICK_FS_UQUOTA; + case XFS_DQTYPE_GROUP: + return XFS_SICK_FS_GQUOTA; + case XFS_DQTYPE_PROJ: + return XFS_SICK_FS_PQUOTA; + } + + ASSERT(0); + return 0; +} + +/* + * Load the inode for a given type of quota, assuming that the sb fields have + * been sorted out. This is not true when switching quota types on a V4 + * filesystem, so do not use this function for that. If metadir is enabled, + * @dp must be the /quota metadir. + * + * Returns -ENOENT if the quota inode field is NULLFSINO; 0 and an inode on + * success; or a negative errno. + */ +int +xfs_dqinode_load( + struct xfs_trans *tp, + struct xfs_inode *dp, + xfs_dqtype_t type, + struct xfs_inode **ipp) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_inode *ip; + enum xfs_metafile_type metafile_type = xfs_dqinode_metafile_type(type); + int error; + + if (!xfs_has_metadir(mp)) { + xfs_ino_t ino; + + switch (type) { + case XFS_DQTYPE_USER: + ino = mp->m_sb.sb_uquotino; + break; + case XFS_DQTYPE_GROUP: + ino = mp->m_sb.sb_gquotino; + break; + case XFS_DQTYPE_PROJ: + ino = mp->m_sb.sb_pquotino; + break; + default: + ASSERT(0); + return -EFSCORRUPTED; + } + + /* Should have set 0 to NULLFSINO when loading superblock */ + if (ino == NULLFSINO) + return -ENOENT; + + error = xfs_trans_metafile_iget(tp, ino, metafile_type, &ip); + } else { + error = xfs_metadir_load(tp, dp, xfs_dqinode_path(type), + metafile_type, &ip); + if (error == -ENOENT) + return error; + } + if (error) { + if (xfs_metadata_is_sick(error)) + xfs_fs_mark_sick(mp, xfs_dqinode_sick_mask(type)); + return error; + } + + if (XFS_IS_CORRUPT(mp, ip->i_df.if_format != XFS_DINODE_FMT_EXTENTS && + ip->i_df.if_format != XFS_DINODE_FMT_BTREE)) { + xfs_irele(ip); + xfs_fs_mark_sick(mp, xfs_dqinode_sick_mask(type)); + return -EFSCORRUPTED; + } + + if (XFS_IS_CORRUPT(mp, ip->i_projid != 0)) { + xfs_irele(ip); + xfs_fs_mark_sick(mp, xfs_dqinode_sick_mask(type)); + return -EFSCORRUPTED; + } + + *ipp = ip; + return 0; +} + +static int +xfs_dqinode_init( + struct xfs_metadir_update *upd, + void *priv) +{ + xfs_trans_log_inode(upd->tp, upd->ip, XFS_ILOG_CORE); + return 0; +} + +/* Create a metadata directory quota inode. */ +int +xfs_dqinode_metadir_create( + struct xfs_inode *dp, + xfs_dqtype_t type, + struct xfs_inode **ipp) +{ + struct xfs_metadir_update upd = { + .dp = dp, + .metafile_type = xfs_dqinode_metafile_type(type), + .path = xfs_dqinode_path(type), + }; + + return xfs_metadir_create_file(&upd, S_IFREG, xfs_dqinode_init, NULL, + ipp); +} + +#ifndef __KERNEL__ +/* Link a metadata directory quota inode. */ +int +xfs_dqinode_metadir_link( + struct xfs_inode *dp, + xfs_dqtype_t type, + struct xfs_inode *ip) +{ + struct xfs_metadir_update upd = { + .dp = dp, + .metafile_type = xfs_dqinode_metafile_type(type), + .path = xfs_dqinode_path(type), + .ip = ip, + }; + int error; + + error = xfs_metadir_start_link(&upd); + if (error) + return error; + + error = xfs_metadir_link(&upd); + if (error) + return error; + + xfs_trans_log_inode(upd.tp, upd.ip, XFS_ILOG_CORE); + + return xfs_metadir_commit(&upd); +} +#endif /* __KERNEL__ */ + +/* Create the parent directory for all quota inodes and load it. */ +int +xfs_dqinode_mkdir_parent( + struct xfs_mount *mp, + struct xfs_inode **dpp) +{ + if (!mp->m_metadirip) { + xfs_fs_mark_sick(mp, XFS_SICK_FS_METADIR); + return -EFSCORRUPTED; + } + + return xfs_metadir_mkdir(mp->m_metadirip, "quota", dpp); +} + +/* + * Load the parent directory of all quota inodes. Pass the inode to the caller + * because quota functions (e.g. QUOTARM) can be called on the quota files even + * if quotas are not enabled. + */ +int +xfs_dqinode_load_parent( + struct xfs_trans *tp, + struct xfs_inode **dpp) +{ + struct xfs_mount *mp = tp->t_mountp; + + if (!mp->m_metadirip) { + xfs_fs_mark_sick(mp, XFS_SICK_FS_METADIR); + return -EFSCORRUPTED; + } + + return xfs_metadir_load(tp, mp->m_metadirip, "quota", XFS_METAFILE_DIR, + dpp); +} diff --git a/fs/xfs/libxfs/xfs_errortag.h b/fs/xfs/libxfs/xfs_errortag.h new file mode 100644 index 000000000..6de207fed --- /dev/null +++ b/fs/xfs/libxfs/xfs_errortag.h @@ -0,0 +1,143 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc. + * Copyright (C) 2017 Oracle. + * All Rights Reserved. + */ +#if !defined(__XFS_ERRORTAG_H_) || defined(XFS_ERRTAG) +#define __XFS_ERRORTAG_H_ + +/* + * There are two ways to use this header file. The first way is to #include it + * bare, which will define all the XFS_ERRTAG_* error injection knobs for use + * with the XFS_TEST_ERROR macro. The second way is to enclose the #include + * with a #define for an XFS_ERRTAG macro, in which case the header will define + " an XFS_ERRTAGS macro that expands to invoke that XFS_ERRTAG macro for each + * defined error injection knob. + */ + +/* + * These are the actual error injection tags. The numbers should be consecutive + * because arrays are sized based on the maximum. + */ +#define XFS_ERRTAG_NOERROR 0 +#define XFS_ERRTAG_IFLUSH_1 1 +#define XFS_ERRTAG_IFLUSH_2 2 +#define XFS_ERRTAG_IFLUSH_3 3 +#define XFS_ERRTAG_IFLUSH_4 4 +#define XFS_ERRTAG_IFLUSH_5 5 +#define XFS_ERRTAG_IFLUSH_6 6 +#define XFS_ERRTAG_DA_READ_BUF 7 +#define XFS_ERRTAG_BTREE_CHECK_LBLOCK 8 +#define XFS_ERRTAG_BTREE_CHECK_SBLOCK 9 +#define XFS_ERRTAG_ALLOC_READ_AGF 10 +#define XFS_ERRTAG_IALLOC_READ_AGI 11 +#define XFS_ERRTAG_ITOBP_INOTOBP 12 +#define XFS_ERRTAG_IUNLINK 13 +#define XFS_ERRTAG_IUNLINK_REMOVE 14 +#define XFS_ERRTAG_DIR_INO_VALIDATE 15 +#define XFS_ERRTAG_BULKSTAT_READ_CHUNK 16 +#define XFS_ERRTAG_IODONE_IOERR 17 +#define XFS_ERRTAG_STRATREAD_IOERR 18 +#define XFS_ERRTAG_STRATCMPL_IOERR 19 +#define XFS_ERRTAG_DIOWRITE_IOERR 20 +#define XFS_ERRTAG_BMAPIFORMAT 21 +#define XFS_ERRTAG_FREE_EXTENT 22 +#define XFS_ERRTAG_RMAP_FINISH_ONE 23 +#define XFS_ERRTAG_REFCOUNT_CONTINUE_UPDATE 24 +#define XFS_ERRTAG_REFCOUNT_FINISH_ONE 25 +#define XFS_ERRTAG_BMAP_FINISH_ONE 26 +#define XFS_ERRTAG_AG_RESV_CRITICAL 27 + +/* + * Drop-writes support removed because write error handling cannot trash + * pre-existing delalloc extents in any useful way anymore. We retain the + * definition so that we can reject it as an invalid value in + * xfs_errortag_add(). + */ +#define XFS_ERRTAG_DROP_WRITES 28 +#define XFS_ERRTAG_LOG_BAD_CRC 29 +#define XFS_ERRTAG_LOG_ITEM_PIN 30 +#define XFS_ERRTAG_BUF_LRU_REF 31 +#define XFS_ERRTAG_FORCE_SCRUB_REPAIR 32 +#define XFS_ERRTAG_FORCE_SUMMARY_RECALC 33 +#define XFS_ERRTAG_IUNLINK_FALLBACK 34 +#define XFS_ERRTAG_BUF_IOERROR 35 +#define XFS_ERRTAG_REDUCE_MAX_IEXTENTS 36 +#define XFS_ERRTAG_BMAP_ALLOC_MINLEN_EXTENT 37 +#define XFS_ERRTAG_AG_RESV_FAIL 38 +#define XFS_ERRTAG_LARP 39 +#define XFS_ERRTAG_DA_LEAF_SPLIT 40 +#define XFS_ERRTAG_ATTR_LEAF_TO_NODE 41 +#define XFS_ERRTAG_WB_DELAY_MS 42 +#define XFS_ERRTAG_WRITE_DELAY_MS 43 +#define XFS_ERRTAG_EXCHMAPS_FINISH_ONE 44 +#define XFS_ERRTAG_METAFILE_RESV_CRITICAL 45 +#define XFS_ERRTAG_FORCE_ZERO_RANGE 46 +#define XFS_ERRTAG_ZONE_RESET 47 +#define XFS_ERRTAG_MAX 48 + +/* + * Random factors for above tags, 1 means always, 2 means 1/2 time, etc. + */ +#define XFS_RANDOM_DEFAULT 100 + +/* + * Table of errror injection knobs. The parameters to the XFS_ERRTAG macro are: + * 1. The XFS_ERRTAG_ flag but without the prefix; + * 2. The name of the sysfs knob; and + * 3. The default value for the knob. + */ +#ifdef XFS_ERRTAG +# undef XFS_ERRTAGS +# define XFS_ERRTAGS \ +XFS_ERRTAG(NOERROR, noerror, XFS_RANDOM_DEFAULT) \ +XFS_ERRTAG(IFLUSH_1, iflush1, XFS_RANDOM_DEFAULT) \ +XFS_ERRTAG(IFLUSH_2, iflush2, XFS_RANDOM_DEFAULT) \ +XFS_ERRTAG(IFLUSH_3, iflush3, XFS_RANDOM_DEFAULT) \ +XFS_ERRTAG(IFLUSH_4, iflush4, XFS_RANDOM_DEFAULT) \ +XFS_ERRTAG(IFLUSH_5, iflush5, XFS_RANDOM_DEFAULT) \ +XFS_ERRTAG(IFLUSH_6, iflush6, XFS_RANDOM_DEFAULT) \ +XFS_ERRTAG(DA_READ_BUF, dareadbuf, XFS_RANDOM_DEFAULT) \ +XFS_ERRTAG(BTREE_CHECK_LBLOCK, btree_chk_lblk, XFS_RANDOM_DEFAULT/4) \ +XFS_ERRTAG(BTREE_CHECK_SBLOCK, btree_chk_sblk, XFS_RANDOM_DEFAULT) \ +XFS_ERRTAG(ALLOC_READ_AGF, readagf, XFS_RANDOM_DEFAULT) \ +XFS_ERRTAG(IALLOC_READ_AGI, readagi, XFS_RANDOM_DEFAULT) \ +XFS_ERRTAG(ITOBP_INOTOBP, itobp, XFS_RANDOM_DEFAULT) \ +XFS_ERRTAG(IUNLINK, iunlink, XFS_RANDOM_DEFAULT) \ +XFS_ERRTAG(IUNLINK_REMOVE, iunlinkrm, XFS_RANDOM_DEFAULT) \ +XFS_ERRTAG(DIR_INO_VALIDATE, dirinovalid, XFS_RANDOM_DEFAULT) \ +XFS_ERRTAG(BULKSTAT_READ_CHUNK, bulkstat, XFS_RANDOM_DEFAULT) \ +XFS_ERRTAG(IODONE_IOERR, logiodone, XFS_RANDOM_DEFAULT/10) \ +XFS_ERRTAG(STRATREAD_IOERR, stratread, XFS_RANDOM_DEFAULT/10) \ +XFS_ERRTAG(STRATCMPL_IOERR, stratcmpl, XFS_RANDOM_DEFAULT/10) \ +XFS_ERRTAG(DIOWRITE_IOERR, diowrite, XFS_RANDOM_DEFAULT/10) \ +XFS_ERRTAG(BMAPIFORMAT, bmapifmt, XFS_RANDOM_DEFAULT) \ +XFS_ERRTAG(FREE_EXTENT, free_extent, 1) \ +XFS_ERRTAG(RMAP_FINISH_ONE, rmap_finish_one, 1) \ +XFS_ERRTAG(REFCOUNT_CONTINUE_UPDATE, refcount_continue_update, 1) \ +XFS_ERRTAG(REFCOUNT_FINISH_ONE, refcount_finish_one, 1) \ +XFS_ERRTAG(BMAP_FINISH_ONE, bmap_finish_one, 1) \ +XFS_ERRTAG(AG_RESV_CRITICAL, ag_resv_critical, 4) \ +XFS_ERRTAG(LOG_BAD_CRC, log_bad_crc, 1) \ +XFS_ERRTAG(LOG_ITEM_PIN, log_item_pin, 1) \ +XFS_ERRTAG(BUF_LRU_REF, buf_lru_ref, 2) \ +XFS_ERRTAG(FORCE_SCRUB_REPAIR, force_repair, 1) \ +XFS_ERRTAG(FORCE_SUMMARY_RECALC, bad_summary, 1) \ +XFS_ERRTAG(IUNLINK_FALLBACK, iunlink_fallback, XFS_RANDOM_DEFAULT/10) \ +XFS_ERRTAG(BUF_IOERROR, buf_ioerror, XFS_RANDOM_DEFAULT) \ +XFS_ERRTAG(REDUCE_MAX_IEXTENTS, reduce_max_iextents, 1) \ +XFS_ERRTAG(BMAP_ALLOC_MINLEN_EXTENT, bmap_alloc_minlen_extent, 1) \ +XFS_ERRTAG(AG_RESV_FAIL, ag_resv_fail, 1) \ +XFS_ERRTAG(LARP, larp, 1) \ +XFS_ERRTAG(DA_LEAF_SPLIT, da_leaf_split, 1) \ +XFS_ERRTAG(ATTR_LEAF_TO_NODE, attr_leaf_to_node, 1) \ +XFS_ERRTAG(WB_DELAY_MS, wb_delay_ms, 3000) \ +XFS_ERRTAG(WRITE_DELAY_MS, write_delay_ms, 3000) \ +XFS_ERRTAG(EXCHMAPS_FINISH_ONE, exchmaps_finish_one, 1) \ +XFS_ERRTAG(METAFILE_RESV_CRITICAL, metafile_resv_crit, 4) \ +XFS_ERRTAG(FORCE_ZERO_RANGE, force_zero_range, 4) \ +XFS_ERRTAG(ZONE_RESET, zone_reset, 1) +#endif /* XFS_ERRTAG */ + +#endif /* __XFS_ERRORTAG_H_ */ diff --git a/fs/xfs/libxfs/xfs_exchmaps.c b/fs/xfs/libxfs/xfs_exchmaps.c new file mode 100644 index 000000000..3efed37cb --- /dev/null +++ b/fs/xfs/libxfs/xfs_exchmaps.c @@ -0,0 +1,1247 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_bmap.h" +#include "xfs_icache.h" +#include "xfs_quota.h" +#include "xfs_exchmaps.h" +#include "xfs_trace.h" +#include "xfs_bmap_btree.h" +#include "xfs_trans_space.h" +#include "xfs_error.h" +#include "xfs_errortag.h" +#include "xfs_health.h" +#include "xfs_exchmaps_item.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_attr_leaf.h" +#include "xfs_attr.h" +#include "xfs_dir2_priv.h" +#include "xfs_dir2.h" +#include "xfs_symlink_remote.h" + +struct kmem_cache *xfs_exchmaps_intent_cache; + +/* bmbt mappings adjacent to a pair of records. */ +struct xfs_exchmaps_adjacent { + struct xfs_bmbt_irec left1; + struct xfs_bmbt_irec right1; + struct xfs_bmbt_irec left2; + struct xfs_bmbt_irec right2; +}; + +#define ADJACENT_INIT { \ + .left1 = { .br_startblock = HOLESTARTBLOCK }, \ + .right1 = { .br_startblock = HOLESTARTBLOCK }, \ + .left2 = { .br_startblock = HOLESTARTBLOCK }, \ + .right2 = { .br_startblock = HOLESTARTBLOCK }, \ +} + +/* Information to reset reflink flag / CoW fork state after an exchange. */ + +/* + * If the reflink flag is set on either inode, make sure it has an incore CoW + * fork, since all reflink inodes must have them. If there's a CoW fork and it + * has mappings in it, make sure the inodes are tagged appropriately so that + * speculative preallocations can be GC'd if we run low of space. + */ +static inline void +xfs_exchmaps_ensure_cowfork( + struct xfs_inode *ip) +{ + struct xfs_ifork *cfork; + + if (xfs_is_reflink_inode(ip)) + xfs_ifork_init_cow(ip); + + cfork = xfs_ifork_ptr(ip, XFS_COW_FORK); + if (!cfork) + return; + if (cfork->if_bytes > 0) + xfs_inode_set_cowblocks_tag(ip); + else + xfs_inode_clear_cowblocks_tag(ip); +} + +/* + * Adjust the on-disk inode size upwards if needed so that we never add + * mappings into the file past EOF. This is crucial so that log recovery won't + * get confused by the sudden appearance of post-eof mappings. + */ +STATIC void +xfs_exchmaps_update_size( + struct xfs_trans *tp, + struct xfs_inode *ip, + struct xfs_bmbt_irec *imap, + xfs_fsize_t new_isize) +{ + struct xfs_mount *mp = tp->t_mountp; + xfs_fsize_t len; + + if (new_isize < 0) + return; + + len = min(XFS_FSB_TO_B(mp, imap->br_startoff + imap->br_blockcount), + new_isize); + + if (len <= ip->i_disk_size) + return; + + trace_xfs_exchmaps_update_inode_size(ip, len); + + ip->i_disk_size = len; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); +} + +/* Advance the incore state tracking after exchanging a mapping. */ +static inline void +xmi_advance( + struct xfs_exchmaps_intent *xmi, + const struct xfs_bmbt_irec *irec) +{ + xmi->xmi_startoff1 += irec->br_blockcount; + xmi->xmi_startoff2 += irec->br_blockcount; + xmi->xmi_blockcount -= irec->br_blockcount; +} + +/* Do we still have more mappings to exchange? */ +static inline bool +xmi_has_more_exchange_work(const struct xfs_exchmaps_intent *xmi) +{ + return xmi->xmi_blockcount > 0; +} + +/* Do we have post-operation cleanups to perform? */ +static inline bool +xmi_has_postop_work(const struct xfs_exchmaps_intent *xmi) +{ + return xmi->xmi_flags & (XFS_EXCHMAPS_CLEAR_INO1_REFLINK | + XFS_EXCHMAPS_CLEAR_INO2_REFLINK | + __XFS_EXCHMAPS_INO2_SHORTFORM); +} + +/* Check all mappings to make sure we can actually exchange them. */ +int +xfs_exchmaps_check_forks( + struct xfs_mount *mp, + const struct xfs_exchmaps_req *req) +{ + struct xfs_ifork *ifp1, *ifp2; + int whichfork = xfs_exchmaps_reqfork(req); + + /* No fork? */ + ifp1 = xfs_ifork_ptr(req->ip1, whichfork); + ifp2 = xfs_ifork_ptr(req->ip2, whichfork); + if (!ifp1 || !ifp2) + return -EINVAL; + + /* We don't know how to exchange local format forks. */ + if (ifp1->if_format == XFS_DINODE_FMT_LOCAL || + ifp2->if_format == XFS_DINODE_FMT_LOCAL) + return -EINVAL; + + return 0; +} + +#ifdef CONFIG_XFS_QUOTA +/* Log the actual updates to the quota accounting. */ +static inline void +xfs_exchmaps_update_quota( + struct xfs_trans *tp, + struct xfs_exchmaps_intent *xmi, + struct xfs_bmbt_irec *irec1, + struct xfs_bmbt_irec *irec2) +{ + int64_t ip1_delta = 0, ip2_delta = 0; + unsigned int qflag; + + qflag = XFS_IS_REALTIME_INODE(xmi->xmi_ip1) ? XFS_TRANS_DQ_RTBCOUNT : + XFS_TRANS_DQ_BCOUNT; + + if (xfs_bmap_is_real_extent(irec1)) { + ip1_delta -= irec1->br_blockcount; + ip2_delta += irec1->br_blockcount; + } + + if (xfs_bmap_is_real_extent(irec2)) { + ip1_delta += irec2->br_blockcount; + ip2_delta -= irec2->br_blockcount; + } + + xfs_trans_mod_dquot_byino(tp, xmi->xmi_ip1, qflag, ip1_delta); + xfs_trans_mod_dquot_byino(tp, xmi->xmi_ip2, qflag, ip2_delta); +} +#else +# define xfs_exchmaps_update_quota(tp, xmi, irec1, irec2) ((void)0) +#endif + +/* Decide if we want to skip this mapping from file1. */ +static inline bool +xfs_exchmaps_can_skip_mapping( + struct xfs_exchmaps_intent *xmi, + struct xfs_bmbt_irec *irec) +{ + struct xfs_mount *mp = xmi->xmi_ip1->i_mount; + + /* Do not skip this mapping if the caller did not tell us to. */ + if (!(xmi->xmi_flags & XFS_EXCHMAPS_INO1_WRITTEN)) + return false; + + /* Do not skip mapped, written mappings. */ + if (xfs_bmap_is_written_extent(irec)) + return false; + + /* + * The mapping is unwritten or a hole. It cannot be a delalloc + * reservation because we already excluded those. It cannot be an + * unwritten extent with dirty page cache because we flushed the page + * cache. For files where the allocation unit is 1FSB (files on the + * data dev, rt files if the extent size is 1FSB), we can safely + * skip this mapping. + */ + if (!xfs_inode_has_bigrtalloc(xmi->xmi_ip1)) + return true; + + /* + * For a realtime file with a multi-fsb allocation unit, the decision + * is trickier because we can only swap full allocation units. + * Unwritten mappings can appear in the middle of an rtx if the rtx is + * partially written, but they can also appear for preallocations. + * + * If the mapping is a hole, skip it entirely. Holes should align with + * rtx boundaries. + */ + if (!xfs_bmap_is_real_extent(irec)) + return true; + + /* + * All mappings below this point are unwritten. + * + * - If the beginning is not aligned to an rtx, trim the end of the + * mapping so that it does not cross an rtx boundary, and swap it. + * + * - If both ends are aligned to an rtx, skip the entire mapping. + */ + if (!isaligned_64(irec->br_startoff, mp->m_sb.sb_rextsize)) { + xfs_fileoff_t new_end; + + new_end = roundup_64(irec->br_startoff, mp->m_sb.sb_rextsize); + irec->br_blockcount = min(irec->br_blockcount, + new_end - irec->br_startoff); + return false; + } + if (isaligned_64(irec->br_blockcount, mp->m_sb.sb_rextsize)) + return true; + + /* + * All mappings below this point are unwritten, start on an rtx + * boundary, and do not end on an rtx boundary. + * + * - If the mapping is longer than one rtx, trim the end of the mapping + * down to an rtx boundary and skip it. + * + * - The mapping is shorter than one rtx. Swap it. + */ + if (irec->br_blockcount > mp->m_sb.sb_rextsize) { + xfs_fileoff_t new_end; + + new_end = rounddown_64(irec->br_startoff + irec->br_blockcount, + mp->m_sb.sb_rextsize); + irec->br_blockcount = new_end - irec->br_startoff; + return true; + } + + return false; +} + +/* + * Walk forward through the file ranges in @xmi until we find two different + * mappings to exchange. If there is work to do, return the mappings; + * otherwise we've reached the end of the range and xmi_blockcount will be + * zero. + * + * If the walk skips over a pair of mappings to the same storage, save them as + * the left records in @adj (if provided) so that the simulation phase can + * avoid an extra lookup. + */ +static int +xfs_exchmaps_find_mappings( + struct xfs_exchmaps_intent *xmi, + struct xfs_bmbt_irec *irec1, + struct xfs_bmbt_irec *irec2, + struct xfs_exchmaps_adjacent *adj) +{ + int nimaps; + int bmap_flags; + int error; + + bmap_flags = xfs_bmapi_aflag(xfs_exchmaps_whichfork(xmi)); + + for (; xmi_has_more_exchange_work(xmi); xmi_advance(xmi, irec1)) { + /* Read mapping from the first file */ + nimaps = 1; + error = xfs_bmapi_read(xmi->xmi_ip1, xmi->xmi_startoff1, + xmi->xmi_blockcount, irec1, &nimaps, + bmap_flags); + if (error) + return error; + if (nimaps != 1 || + irec1->br_startblock == DELAYSTARTBLOCK || + irec1->br_startoff != xmi->xmi_startoff1) { + /* + * We should never get no mapping or a delalloc mapping + * or something that doesn't match what we asked for, + * since the caller flushed both inodes and we hold the + * ILOCKs for both inodes. + */ + ASSERT(0); + return -EINVAL; + } + + if (xfs_exchmaps_can_skip_mapping(xmi, irec1)) { + trace_xfs_exchmaps_mapping1_skip(xmi->xmi_ip1, irec1); + continue; + } + + /* Read mapping from the second file */ + nimaps = 1; + error = xfs_bmapi_read(xmi->xmi_ip2, xmi->xmi_startoff2, + irec1->br_blockcount, irec2, &nimaps, + bmap_flags); + if (error) + return error; + if (nimaps != 1 || + irec2->br_startblock == DELAYSTARTBLOCK || + irec2->br_startoff != xmi->xmi_startoff2) { + /* + * We should never get no mapping or a delalloc mapping + * or something that doesn't match what we asked for, + * since the caller flushed both inodes and we hold the + * ILOCKs for both inodes. + */ + ASSERT(0); + return -EINVAL; + } + + /* + * We can only exchange as many blocks as the smaller of the + * two mapping maps. + */ + irec1->br_blockcount = min(irec1->br_blockcount, + irec2->br_blockcount); + + trace_xfs_exchmaps_mapping1(xmi->xmi_ip1, irec1); + trace_xfs_exchmaps_mapping2(xmi->xmi_ip2, irec2); + + /* We found something to exchange, so return it. */ + if (irec1->br_startblock != irec2->br_startblock) + return 0; + + /* + * Two mappings pointing to the same physical block must not + * have different states; that's filesystem corruption. Move + * on to the next mapping if they're both holes or both point + * to the same physical space extent. + */ + if (irec1->br_state != irec2->br_state) { + xfs_bmap_mark_sick(xmi->xmi_ip1, + xfs_exchmaps_whichfork(xmi)); + xfs_bmap_mark_sick(xmi->xmi_ip2, + xfs_exchmaps_whichfork(xmi)); + return -EFSCORRUPTED; + } + + /* + * Save the mappings if we're estimating work and skipping + * these identical mappings. + */ + if (adj) { + memcpy(&adj->left1, irec1, sizeof(*irec1)); + memcpy(&adj->left2, irec2, sizeof(*irec2)); + } + } + + return 0; +} + +/* Exchange these two mappings. */ +static void +xfs_exchmaps_one_step( + struct xfs_trans *tp, + struct xfs_exchmaps_intent *xmi, + struct xfs_bmbt_irec *irec1, + struct xfs_bmbt_irec *irec2) +{ + int whichfork = xfs_exchmaps_whichfork(xmi); + + xfs_exchmaps_update_quota(tp, xmi, irec1, irec2); + + /* Remove both mappings. */ + xfs_bmap_unmap_extent(tp, xmi->xmi_ip1, whichfork, irec1); + xfs_bmap_unmap_extent(tp, xmi->xmi_ip2, whichfork, irec2); + + /* + * Re-add both mappings. We exchange the file offsets between the two + * maps and add the opposite map, which has the effect of filling the + * logical offsets we just unmapped, but with with the physical mapping + * information exchanged. + */ + swap(irec1->br_startoff, irec2->br_startoff); + xfs_bmap_map_extent(tp, xmi->xmi_ip1, whichfork, irec2); + xfs_bmap_map_extent(tp, xmi->xmi_ip2, whichfork, irec1); + + /* Make sure we're not adding mappings past EOF. */ + if (whichfork == XFS_DATA_FORK) { + xfs_exchmaps_update_size(tp, xmi->xmi_ip1, irec2, + xmi->xmi_isize1); + xfs_exchmaps_update_size(tp, xmi->xmi_ip2, irec1, + xmi->xmi_isize2); + } + + /* + * Advance our cursor and exit. The caller (either defer ops or log + * recovery) will log the XMD item, and if *blockcount is nonzero, it + * will log a new XMI item for the remainder and call us back. + */ + xmi_advance(xmi, irec1); +} + +/* Convert inode2's leaf attr fork back to shortform, if possible.. */ +STATIC int +xfs_exchmaps_attr_to_sf( + struct xfs_trans *tp, + struct xfs_exchmaps_intent *xmi) +{ + struct xfs_da_args args = { + .dp = xmi->xmi_ip2, + .geo = tp->t_mountp->m_attr_geo, + .whichfork = XFS_ATTR_FORK, + .trans = tp, + .owner = I_INO(xmi->xmi_ip2), + }; + struct xfs_buf *bp; + int forkoff; + int error; + + if (!xfs_attr_is_leaf(xmi->xmi_ip2)) + return 0; + + error = xfs_attr3_leaf_read(tp, xmi->xmi_ip2, I_INO(xmi->xmi_ip2), 0, + &bp); + if (error) + return error; + + forkoff = xfs_attr_shortform_allfit(bp, xmi->xmi_ip2); + if (forkoff == 0) + return 0; + + return xfs_attr3_leaf_to_shortform(bp, &args, forkoff); +} + +/* Convert inode2's block dir fork back to shortform, if possible.. */ +STATIC int +xfs_exchmaps_dir_to_sf( + struct xfs_trans *tp, + struct xfs_exchmaps_intent *xmi) +{ + struct xfs_da_args args = { + .dp = xmi->xmi_ip2, + .geo = tp->t_mountp->m_dir_geo, + .whichfork = XFS_DATA_FORK, + .trans = tp, + .owner = I_INO(xmi->xmi_ip2), + }; + struct xfs_dir2_sf_hdr sfh; + struct xfs_buf *bp; + int size; + int error = 0; + + if (xfs_dir2_format(&args, &error) != XFS_DIR2_FMT_BLOCK) + return error; + + error = xfs_dir3_block_read(tp, xmi->xmi_ip2, I_INO(xmi->xmi_ip2), &bp); + if (error) + return error; + + size = xfs_dir2_block_sfsize(xmi->xmi_ip2, bp->b_addr, &sfh); + if (size > xfs_inode_data_fork_size(xmi->xmi_ip2)) + return 0; + + return xfs_dir2_block_to_sf(&args, bp, size, &sfh); +} + +/* Convert inode2's remote symlink target back to shortform, if possible. */ +STATIC int +xfs_exchmaps_link_to_sf( + struct xfs_trans *tp, + struct xfs_exchmaps_intent *xmi) +{ + struct xfs_inode *ip = xmi->xmi_ip2; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK); + char *buf; + int error; + + if (ifp->if_format == XFS_DINODE_FMT_LOCAL || + ip->i_disk_size > xfs_inode_data_fork_size(ip)) + return 0; + + /* Read the current symlink target into a buffer. */ + buf = kmalloc(ip->i_disk_size + 1, + GFP_KERNEL | __GFP_NOLOCKDEP | __GFP_NOFAIL); + if (!buf) { + ASSERT(0); + return -ENOMEM; + } + + error = xfs_symlink_remote_read(ip, buf); + if (error) + goto free; + + /* Remove the blocks. */ + error = xfs_symlink_remote_truncate(tp, ip); + if (error) + goto free; + + /* Convert fork to local format and log our changes. */ + xfs_idestroy_fork(ifp); + ifp->if_bytes = 0; + ifp->if_format = XFS_DINODE_FMT_LOCAL; + xfs_init_local_fork(ip, XFS_DATA_FORK, buf, ip->i_disk_size); + xfs_trans_log_inode(tp, ip, XFS_ILOG_DDATA | XFS_ILOG_CORE); +free: + kfree(buf); + return error; +} + +/* Clear the reflink flag after an exchange. */ +static inline void +xfs_exchmaps_clear_reflink( + struct xfs_trans *tp, + struct xfs_inode *ip) +{ + trace_xfs_reflink_unset_inode_flag(ip); + + ip->i_diflags2 &= ~XFS_DIFLAG2_REFLINK; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); +} + +/* Finish whatever work might come after an exchange operation. */ +static int +xfs_exchmaps_do_postop_work( + struct xfs_trans *tp, + struct xfs_exchmaps_intent *xmi) +{ + if (xmi->xmi_flags & __XFS_EXCHMAPS_INO2_SHORTFORM) { + int error = 0; + + if (xmi->xmi_flags & XFS_EXCHMAPS_ATTR_FORK) + error = xfs_exchmaps_attr_to_sf(tp, xmi); + else if (S_ISDIR(VFS_I(xmi->xmi_ip2)->i_mode)) + error = xfs_exchmaps_dir_to_sf(tp, xmi); + else if (S_ISLNK(VFS_I(xmi->xmi_ip2)->i_mode)) + error = xfs_exchmaps_link_to_sf(tp, xmi); + xmi->xmi_flags &= ~__XFS_EXCHMAPS_INO2_SHORTFORM; + if (error) + return error; + } + + if (xmi->xmi_flags & XFS_EXCHMAPS_CLEAR_INO1_REFLINK) { + xfs_exchmaps_clear_reflink(tp, xmi->xmi_ip1); + xmi->xmi_flags &= ~XFS_EXCHMAPS_CLEAR_INO1_REFLINK; + } + + if (xmi->xmi_flags & XFS_EXCHMAPS_CLEAR_INO2_REFLINK) { + xfs_exchmaps_clear_reflink(tp, xmi->xmi_ip2); + xmi->xmi_flags &= ~XFS_EXCHMAPS_CLEAR_INO2_REFLINK; + } + + return 0; +} + +/* Finish one step in a mapping exchange operation, possibly relogging. */ +int +xfs_exchmaps_finish_one( + struct xfs_trans *tp, + struct xfs_exchmaps_intent *xmi) +{ + struct xfs_bmbt_irec irec1, irec2; + int error; + + if (xmi_has_more_exchange_work(xmi)) { + /* + * If the operation state says that some range of the files + * have not yet been exchanged, look for mappings in that range + * to exchange. If we find some mappings, exchange them. + */ + error = xfs_exchmaps_find_mappings(xmi, &irec1, &irec2, NULL); + if (error) + return error; + + if (xmi_has_more_exchange_work(xmi)) + xfs_exchmaps_one_step(tp, xmi, &irec1, &irec2); + + /* + * If the caller asked us to exchange the file sizes after the + * exchange and either we just exchanged the last mappings in + * the range or we didn't find anything to exchange, update the + * ondisk file sizes. + */ + if ((xmi->xmi_flags & XFS_EXCHMAPS_SET_SIZES) && + !xmi_has_more_exchange_work(xmi)) { + xmi->xmi_ip1->i_disk_size = xmi->xmi_isize1; + xmi->xmi_ip2->i_disk_size = xmi->xmi_isize2; + + xfs_trans_log_inode(tp, xmi->xmi_ip1, XFS_ILOG_CORE); + xfs_trans_log_inode(tp, xmi->xmi_ip2, XFS_ILOG_CORE); + } + } else if (xmi_has_postop_work(xmi)) { + /* + * Now that we're finished with the exchange operation, + * complete the post-op cleanup work. + */ + error = xfs_exchmaps_do_postop_work(tp, xmi); + if (error) + return error; + } + + if (XFS_TEST_ERROR(tp->t_mountp, XFS_ERRTAG_EXCHMAPS_FINISH_ONE)) + return -EIO; + + /* If we still have work to do, ask for a new transaction. */ + if (xmi_has_more_exchange_work(xmi) || xmi_has_postop_work(xmi)) { + trace_xfs_exchmaps_defer(tp->t_mountp, xmi); + return -EAGAIN; + } + + /* + * If we reach here, we've finished all the exchange work and the post + * operation work. The last thing we need to do before returning to + * the caller is to make sure that COW forks are set up correctly. + */ + if (!(xmi->xmi_flags & XFS_EXCHMAPS_ATTR_FORK)) { + xfs_exchmaps_ensure_cowfork(xmi->xmi_ip1); + xfs_exchmaps_ensure_cowfork(xmi->xmi_ip2); + } + + return 0; +} + +/* + * Compute the amount of bmbt blocks we should reserve for each file. In the + * worst case, each exchange will fill a hole with a new mapping, which could + * result in a btree split every time we add a new leaf block. + */ +static inline uint64_t +xfs_exchmaps_bmbt_blocks( + struct xfs_mount *mp, + const struct xfs_exchmaps_req *req) +{ + return howmany_64(req->nr_exchanges, + XFS_MAX_CONTIG_BMAPS_PER_BLOCK(mp)) * + XFS_EXTENTADD_SPACE_RES(mp, xfs_exchmaps_reqfork(req)); +} + +/* Compute the space we should reserve for the rmap btree expansions. */ +static inline uint64_t +xfs_exchmaps_rmapbt_blocks( + struct xfs_mount *mp, + const struct xfs_exchmaps_req *req) +{ + if (!xfs_has_rmapbt(mp)) + return 0; + if (XFS_IS_REALTIME_INODE(req->ip1)) + return howmany_64(req->nr_exchanges, + XFS_MAX_CONTIG_RTRMAPS_PER_BLOCK(mp)) * + XFS_RTRMAPADD_SPACE_RES(mp); + + return howmany_64(req->nr_exchanges, + XFS_MAX_CONTIG_RMAPS_PER_BLOCK(mp)) * + XFS_RMAPADD_SPACE_RES(mp); +} + +/* Estimate the bmbt and rmapbt overhead required to exchange mappings. */ +int +xfs_exchmaps_estimate_overhead( + struct xfs_exchmaps_req *req) +{ + struct xfs_mount *mp = req->ip1->i_mount; + xfs_filblks_t bmbt_blocks; + xfs_filblks_t rmapbt_blocks; + xfs_filblks_t resblks = req->resblks; + + /* + * Compute the number of bmbt and rmapbt blocks we might need to handle + * the estimated number of exchanges. + */ + bmbt_blocks = xfs_exchmaps_bmbt_blocks(mp, req); + rmapbt_blocks = xfs_exchmaps_rmapbt_blocks(mp, req); + + trace_xfs_exchmaps_overhead(mp, bmbt_blocks, rmapbt_blocks); + + /* Make sure the change in file block count doesn't overflow. */ + if (check_add_overflow(req->ip1_bcount, bmbt_blocks, &req->ip1_bcount)) + return -EFBIG; + if (check_add_overflow(req->ip2_bcount, bmbt_blocks, &req->ip2_bcount)) + return -EFBIG; + + /* + * Add together the number of blocks we need to handle btree growth, + * then add it to the number of blocks we need to reserve to this + * transaction. + */ + if (check_add_overflow(resblks, bmbt_blocks, &resblks)) + return -ENOSPC; + if (check_add_overflow(resblks, bmbt_blocks, &resblks)) + return -ENOSPC; + if (check_add_overflow(resblks, rmapbt_blocks, &resblks)) + return -ENOSPC; + if (check_add_overflow(resblks, rmapbt_blocks, &resblks)) + return -ENOSPC; + + /* Can't actually reserve more than UINT_MAX blocks. */ + if (resblks > UINT_MAX) + return -ENOSPC; + + req->resblks = resblks; + trace_xfs_exchmaps_final_estimate(req); + return 0; +} + +/* Decide if we can merge two real mappings. */ +static inline bool +xmi_can_merge( + const struct xfs_bmbt_irec *b1, + const struct xfs_bmbt_irec *b2) +{ + /* Don't merge holes. */ + if (b1->br_startblock == HOLESTARTBLOCK || + b2->br_startblock == HOLESTARTBLOCK) + return false; + + /* We don't merge holes. */ + if (!xfs_bmap_is_real_extent(b1) || !xfs_bmap_is_real_extent(b2)) + return false; + + if (b1->br_startoff + b1->br_blockcount == b2->br_startoff && + b1->br_startblock + b1->br_blockcount == b2->br_startblock && + b1->br_state == b2->br_state && + b1->br_blockcount + b2->br_blockcount <= XFS_MAX_BMBT_EXTLEN) + return true; + + return false; +} + +/* + * Decide if we can merge three mappings. Caller must ensure all three + * mappings must not be holes or delalloc reservations. + */ +static inline bool +xmi_can_merge_all( + const struct xfs_bmbt_irec *l, + const struct xfs_bmbt_irec *m, + const struct xfs_bmbt_irec *r) +{ + xfs_filblks_t new_len; + + new_len = l->br_blockcount + m->br_blockcount + r->br_blockcount; + return new_len <= XFS_MAX_BMBT_EXTLEN; +} + +#define CLEFT_CONTIG 0x01 +#define CRIGHT_CONTIG 0x02 +#define CHOLE 0x04 +#define CBOTH_CONTIG (CLEFT_CONTIG | CRIGHT_CONTIG) + +#define NLEFT_CONTIG 0x10 +#define NRIGHT_CONTIG 0x20 +#define NHOLE 0x40 +#define NBOTH_CONTIG (NLEFT_CONTIG | NRIGHT_CONTIG) + +/* Estimate the effect of a single exchange on mapping count. */ +static inline int +xmi_delta_nextents_step( + struct xfs_mount *mp, + const struct xfs_bmbt_irec *left, + const struct xfs_bmbt_irec *curr, + const struct xfs_bmbt_irec *new, + const struct xfs_bmbt_irec *right) +{ + bool lhole, rhole, chole, nhole; + unsigned int state = 0; + int ret = 0; + + lhole = left->br_startblock == HOLESTARTBLOCK; + rhole = right->br_startblock == HOLESTARTBLOCK; + chole = curr->br_startblock == HOLESTARTBLOCK; + nhole = new->br_startblock == HOLESTARTBLOCK; + + if (chole) + state |= CHOLE; + if (!lhole && !chole && xmi_can_merge(left, curr)) + state |= CLEFT_CONTIG; + if (!rhole && !chole && xmi_can_merge(curr, right)) + state |= CRIGHT_CONTIG; + if ((state & CBOTH_CONTIG) == CBOTH_CONTIG && + !xmi_can_merge_all(left, curr, right)) + state &= ~CRIGHT_CONTIG; + + if (nhole) + state |= NHOLE; + if (!lhole && !nhole && xmi_can_merge(left, new)) + state |= NLEFT_CONTIG; + if (!rhole && !nhole && xmi_can_merge(new, right)) + state |= NRIGHT_CONTIG; + if ((state & NBOTH_CONTIG) == NBOTH_CONTIG && + !xmi_can_merge_all(left, new, right)) + state &= ~NRIGHT_CONTIG; + + switch (state & (CLEFT_CONTIG | CRIGHT_CONTIG | CHOLE)) { + case CLEFT_CONTIG | CRIGHT_CONTIG: + /* + * left/curr/right are the same mapping, so deleting curr + * causes 2 new mappings to be created. + */ + ret += 2; + break; + case 0: + /* + * curr is not contiguous with any mapping, so we remove curr + * completely + */ + ret--; + break; + case CHOLE: + /* hole, do nothing */ + break; + case CLEFT_CONTIG: + case CRIGHT_CONTIG: + /* trim either left or right, no change */ + break; + } + + switch (state & (NLEFT_CONTIG | NRIGHT_CONTIG | NHOLE)) { + case NLEFT_CONTIG | NRIGHT_CONTIG: + /* + * left/curr/right will become the same mapping, so adding + * curr causes the deletion of right. + */ + ret--; + break; + case 0: + /* new is not contiguous with any mapping */ + ret++; + break; + case NHOLE: + /* hole, do nothing. */ + break; + case NLEFT_CONTIG: + case NRIGHT_CONTIG: + /* new is absorbed into left or right, no change */ + break; + } + + trace_xfs_exchmaps_delta_nextents_step(mp, left, curr, new, right, ret, + state); + return ret; +} + +/* Make sure we don't overflow the extent (mapping) counters. */ +static inline int +xmi_ensure_delta_nextents( + struct xfs_exchmaps_req *req, + struct xfs_inode *ip, + int64_t delta) +{ + struct xfs_mount *mp = ip->i_mount; + int whichfork = xfs_exchmaps_reqfork(req); + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + uint64_t new_nextents; + xfs_extnum_t max_nextents; + + if (delta < 0) + return 0; + + /* + * It's always an error if the delta causes integer overflow. delta + * needs an explicit cast here to avoid warnings about implicit casts + * coded into the overflow check. + */ + if (check_add_overflow(ifp->if_nextents, (uint64_t)delta, + &new_nextents)) + return -EFBIG; + + if (XFS_TEST_ERROR(mp, XFS_ERRTAG_REDUCE_MAX_IEXTENTS) && + new_nextents > 10) + return -EFBIG; + + /* + * We always promote both inodes to have large extent counts if the + * superblock feature is enabled, so we only need to check against the + * theoretical maximum. + */ + max_nextents = xfs_iext_max_nextents(xfs_has_large_extent_counts(mp), + whichfork); + if (new_nextents > max_nextents) + return -EFBIG; + + return 0; +} + +/* Find the next mapping after irec. */ +static inline int +xmi_next( + struct xfs_inode *ip, + int bmap_flags, + const struct xfs_bmbt_irec *irec, + struct xfs_bmbt_irec *nrec) +{ + xfs_fileoff_t off; + xfs_filblks_t blockcount; + int nimaps = 1; + int error; + + off = irec->br_startoff + irec->br_blockcount; + blockcount = XFS_MAX_FILEOFF - off; + error = xfs_bmapi_read(ip, off, blockcount, nrec, &nimaps, bmap_flags); + if (error) + return error; + if (nrec->br_startblock == DELAYSTARTBLOCK || + nrec->br_startoff != off) { + /* + * If we don't get the mapping we want, return a zero-length + * mapping, which our estimator function will pretend is a hole. + * We shouldn't get delalloc reservations. + */ + nrec->br_startblock = HOLESTARTBLOCK; + } + + return 0; +} + +int __init +xfs_exchmaps_intent_init_cache(void) +{ + xfs_exchmaps_intent_cache = kmem_cache_create("xfs_exchmaps_intent", + sizeof(struct xfs_exchmaps_intent), + 0, 0, NULL); + + return xfs_exchmaps_intent_cache != NULL ? 0 : -ENOMEM; +} + +void +xfs_exchmaps_intent_destroy_cache(void) +{ + kmem_cache_destroy(xfs_exchmaps_intent_cache); + xfs_exchmaps_intent_cache = NULL; +} + +/* + * Decide if we will exchange the reflink flags between the two files after the + * exchange. The only time we want to do this is if we're exchanging all + * mappings under EOF and the inode reflink flags have different states. + */ +static inline bool +xmi_can_exchange_reflink_flags( + const struct xfs_exchmaps_req *req, + unsigned int reflink_state) +{ + struct xfs_mount *mp = req->ip1->i_mount; + + /* + * The INO1_WRITTEN optimization can skip exchanging hole and + * unwritten mappings, which means we cannot guarantee that all + * shared extents actually moved to the other file. Clearing the + * reflink flag of an inode that still holds shared extents breaks + * the CoW write path, so refuse to exchange the flags in that case. + */ + if (req->flags & XFS_EXCHMAPS_INO1_WRITTEN) + return false; + + if (hweight32(reflink_state) != 1) + return false; + if (req->startoff1 != 0 || req->startoff2 != 0) + return false; + if (req->blockcount != XFS_B_TO_FSB(mp, req->ip1->i_disk_size)) + return false; + if (req->blockcount != XFS_B_TO_FSB(mp, req->ip2->i_disk_size)) + return false; + return true; +} + + +/* Allocate and initialize a new incore intent item from a request. */ +struct xfs_exchmaps_intent * +xfs_exchmaps_init_intent( + const struct xfs_exchmaps_req *req) +{ + struct xfs_exchmaps_intent *xmi; + unsigned int rs = 0; + + xmi = kmem_cache_zalloc(xfs_exchmaps_intent_cache, + GFP_NOFS | __GFP_NOFAIL); + INIT_LIST_HEAD(&xmi->xmi_list); + xmi->xmi_ip1 = req->ip1; + xmi->xmi_ip2 = req->ip2; + xmi->xmi_startoff1 = req->startoff1; + xmi->xmi_startoff2 = req->startoff2; + xmi->xmi_blockcount = req->blockcount; + xmi->xmi_isize1 = xmi->xmi_isize2 = -1; + xmi->xmi_flags = req->flags & XFS_EXCHMAPS_PARAMS; + + if (xfs_exchmaps_whichfork(xmi) == XFS_ATTR_FORK) { + xmi->xmi_flags |= __XFS_EXCHMAPS_INO2_SHORTFORM; + return xmi; + } + + if (req->flags & XFS_EXCHMAPS_SET_SIZES) { + xmi->xmi_flags |= XFS_EXCHMAPS_SET_SIZES; + xmi->xmi_isize1 = req->ip2->i_disk_size; + xmi->xmi_isize2 = req->ip1->i_disk_size; + } + + /* Record the state of each inode's reflink flag before the op. */ + if (xfs_is_reflink_inode(req->ip1)) + rs |= 1; + if (xfs_is_reflink_inode(req->ip2)) + rs |= 2; + + /* + * Figure out if we're clearing the reflink flags (which effectively + * exchanges them) after the operation. + */ + if (xmi_can_exchange_reflink_flags(req, rs)) { + if (rs & 1) + xmi->xmi_flags |= XFS_EXCHMAPS_CLEAR_INO1_REFLINK; + if (rs & 2) + xmi->xmi_flags |= XFS_EXCHMAPS_CLEAR_INO2_REFLINK; + } + + if (S_ISDIR(VFS_I(xmi->xmi_ip2)->i_mode) || + S_ISLNK(VFS_I(xmi->xmi_ip2)->i_mode)) + xmi->xmi_flags |= __XFS_EXCHMAPS_INO2_SHORTFORM; + + return xmi; +} + +/* + * Estimate the number of exchange operations and the number of file blocks + * in each file that will be affected by the exchange operation. + */ +int +xfs_exchmaps_estimate( + struct xfs_exchmaps_req *req) +{ + struct xfs_exchmaps_intent *xmi; + struct xfs_bmbt_irec irec1, irec2; + struct xfs_exchmaps_adjacent adj = ADJACENT_INIT; + xfs_filblks_t ip1_blocks = 0, ip2_blocks = 0; + int64_t d_nexts1, d_nexts2; + int bmap_flags; + int error; + + ASSERT(!(req->flags & ~XFS_EXCHMAPS_PARAMS)); + + bmap_flags = xfs_bmapi_aflag(xfs_exchmaps_reqfork(req)); + xmi = xfs_exchmaps_init_intent(req); + + /* + * To guard against the possibility of overflowing the extent counters, + * we have to estimate an upper bound on the potential increase in that + * counter. We can split the mapping at each end of the range, and for + * each step of the exchange we can split the mapping that we're + * working on if the mappings do not align. + */ + d_nexts1 = d_nexts2 = 3; + + while (xmi_has_more_exchange_work(xmi)) { + /* + * Walk through the file ranges until we find something to + * exchange. Because we're simulating the exchange, pass in + * adj to capture skipped mappings for correct estimation of + * bmbt record merges. + */ + error = xfs_exchmaps_find_mappings(xmi, &irec1, &irec2, &adj); + if (error) + goto out_free; + if (!xmi_has_more_exchange_work(xmi)) + break; + + /* Update accounting. */ + if (xfs_bmap_is_real_extent(&irec1)) + ip1_blocks += irec1.br_blockcount; + if (xfs_bmap_is_real_extent(&irec2)) + ip2_blocks += irec2.br_blockcount; + req->nr_exchanges++; + + /* Read the next mappings from both files. */ + error = xmi_next(req->ip1, bmap_flags, &irec1, &adj.right1); + if (error) + goto out_free; + + error = xmi_next(req->ip2, bmap_flags, &irec2, &adj.right2); + if (error) + goto out_free; + + /* Update extent count deltas. */ + d_nexts1 += xmi_delta_nextents_step(req->ip1->i_mount, + &adj.left1, &irec1, &irec2, &adj.right1); + + d_nexts2 += xmi_delta_nextents_step(req->ip1->i_mount, + &adj.left2, &irec2, &irec1, &adj.right2); + + /* Now pretend we exchanged the mappings. */ + if (xmi_can_merge(&adj.left2, &irec1)) + adj.left2.br_blockcount += irec1.br_blockcount; + else + memcpy(&adj.left2, &irec1, sizeof(irec1)); + + if (xmi_can_merge(&adj.left1, &irec2)) + adj.left1.br_blockcount += irec2.br_blockcount; + else + memcpy(&adj.left1, &irec2, sizeof(irec2)); + + xmi_advance(xmi, &irec1); + } + + /* Account for the blocks that are being exchanged. */ + if (XFS_IS_REALTIME_INODE(req->ip1) && + xfs_exchmaps_reqfork(req) == XFS_DATA_FORK) { + req->ip1_rtbcount = ip1_blocks; + req->ip2_rtbcount = ip2_blocks; + } else { + req->ip1_bcount = ip1_blocks; + req->ip2_bcount = ip2_blocks; + } + + /* + * Make sure that both forks have enough slack left in their extent + * counters that the exchange operation will not overflow. + */ + trace_xfs_exchmaps_delta_nextents(req, d_nexts1, d_nexts2); + if (req->ip1 == req->ip2) { + error = xmi_ensure_delta_nextents(req, req->ip1, + d_nexts1 + d_nexts2); + } else { + error = xmi_ensure_delta_nextents(req, req->ip1, d_nexts1); + if (error) + goto out_free; + error = xmi_ensure_delta_nextents(req, req->ip2, d_nexts2); + } + if (error) + goto out_free; + + trace_xfs_exchmaps_initial_estimate(req); + error = xfs_exchmaps_estimate_overhead(req); +out_free: + kmem_cache_free(xfs_exchmaps_intent_cache, xmi); + return error; +} + +/* Set the reflink flag before an operation. */ +static inline void +xfs_exchmaps_set_reflink( + struct xfs_trans *tp, + struct xfs_inode *ip) +{ + trace_xfs_reflink_set_inode_flag(ip); + + ip->i_diflags2 |= XFS_DIFLAG2_REFLINK; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); +} + +/* + * If either file has shared blocks and we're exchanging data forks, we must + * flag the other file as having shared blocks so that we get the shared-block + * rmap functions if we need to fix up the rmaps. + */ +void +xfs_exchmaps_ensure_reflink( + struct xfs_trans *tp, + const struct xfs_exchmaps_intent *xmi) +{ + unsigned int rs = 0; + + if (xfs_is_reflink_inode(xmi->xmi_ip1)) + rs |= 1; + if (xfs_is_reflink_inode(xmi->xmi_ip2)) + rs |= 2; + + if ((rs & 1) && !xfs_is_reflink_inode(xmi->xmi_ip2)) + xfs_exchmaps_set_reflink(tp, xmi->xmi_ip2); + + if ((rs & 2) && !xfs_is_reflink_inode(xmi->xmi_ip1)) + xfs_exchmaps_set_reflink(tp, xmi->xmi_ip1); +} + +/* Set the large extent count flag before an operation if needed. */ +static inline void +xfs_exchmaps_ensure_large_extent_counts( + struct xfs_trans *tp, + struct xfs_inode *ip) +{ + if (xfs_inode_has_large_extent_counts(ip)) + return; + + ip->i_diflags2 |= XFS_DIFLAG2_NREXT64; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); +} + +/* Widen the extent counter fields of both inodes if necessary. */ +void +xfs_exchmaps_upgrade_extent_counts( + struct xfs_trans *tp, + const struct xfs_exchmaps_intent *xmi) +{ + if (!xfs_has_large_extent_counts(tp->t_mountp)) + return; + + xfs_exchmaps_ensure_large_extent_counts(tp, xmi->xmi_ip1); + xfs_exchmaps_ensure_large_extent_counts(tp, xmi->xmi_ip2); +} + +/* + * Schedule an exchange a range of mappings from one inode to another. + * + * The use of file mapping exchange log intent items ensures the operation can + * be resumed even if the system goes down. The caller must commit the + * transaction to start the work. + * + * The caller must ensure the inodes must be joined to the transaction and + * ILOCKd; they will still be joined to the transaction at exit. + */ +void +xfs_exchange_mappings( + struct xfs_trans *tp, + const struct xfs_exchmaps_req *req) +{ + struct xfs_exchmaps_intent *xmi; + + BUILD_BUG_ON(XFS_EXCHMAPS_INTERNAL_FLAGS & XFS_EXCHMAPS_LOGGED_FLAGS); + + xfs_assert_ilocked(req->ip1, XFS_ILOCK_EXCL); + xfs_assert_ilocked(req->ip2, XFS_ILOCK_EXCL); + ASSERT(!(req->flags & ~XFS_EXCHMAPS_LOGGED_FLAGS)); + if (req->flags & XFS_EXCHMAPS_SET_SIZES) + ASSERT(!(req->flags & XFS_EXCHMAPS_ATTR_FORK)); + ASSERT(xfs_has_exchange_range(tp->t_mountp)); + + if (req->blockcount == 0) + return; + + xmi = xfs_exchmaps_init_intent(req); + xfs_exchmaps_defer_add(tp, xmi); + xfs_exchmaps_ensure_reflink(tp, xmi); + xfs_exchmaps_upgrade_extent_counts(tp, xmi); +} diff --git a/fs/xfs/libxfs/xfs_exchmaps.h b/fs/xfs/libxfs/xfs_exchmaps.h new file mode 100644 index 000000000..fa822dff2 --- /dev/null +++ b/fs/xfs/libxfs/xfs_exchmaps.h @@ -0,0 +1,124 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_EXCHMAPS_H__ +#define __XFS_EXCHMAPS_H__ + +/* In-core deferred operation info about a file mapping exchange request. */ +struct xfs_exchmaps_intent { + /* List of other incore deferred work. */ + struct list_head xmi_list; + + /* Inodes participating in the operation. */ + struct xfs_inode *xmi_ip1; + struct xfs_inode *xmi_ip2; + + /* File offset range information. */ + xfs_fileoff_t xmi_startoff1; + xfs_fileoff_t xmi_startoff2; + xfs_filblks_t xmi_blockcount; + + /* Set these file sizes after the operation, unless negative. */ + xfs_fsize_t xmi_isize1; + xfs_fsize_t xmi_isize2; + + uint64_t xmi_flags; /* XFS_EXCHMAPS_* flags */ +}; + +/* Try to convert inode2 from block to short format at the end, if possible. */ +#define __XFS_EXCHMAPS_INO2_SHORTFORM (1ULL << 63) + +#define XFS_EXCHMAPS_INTERNAL_FLAGS (__XFS_EXCHMAPS_INO2_SHORTFORM) + +/* flags that can be passed to xfs_exchmaps_{estimate,mappings} */ +#define XFS_EXCHMAPS_PARAMS (XFS_EXCHMAPS_ATTR_FORK | \ + XFS_EXCHMAPS_SET_SIZES | \ + XFS_EXCHMAPS_INO1_WRITTEN) + +static inline int +xfs_exchmaps_whichfork(const struct xfs_exchmaps_intent *xmi) +{ + if (xmi->xmi_flags & XFS_EXCHMAPS_ATTR_FORK) + return XFS_ATTR_FORK; + return XFS_DATA_FORK; +} + +/* Parameters for a mapping exchange request. */ +struct xfs_exchmaps_req { + /* Inodes participating in the operation. */ + struct xfs_inode *ip1; + struct xfs_inode *ip2; + + /* File offset range information. */ + xfs_fileoff_t startoff1; + xfs_fileoff_t startoff2; + xfs_filblks_t blockcount; + + /* XFS_EXCHMAPS_* operation flags */ + uint64_t flags; + + /* + * Fields below this line are filled out by xfs_exchmaps_estimate; + * callers should initialize this part of the struct to zero. + */ + + /* + * Data device blocks to be moved out of ip1, and free space needed to + * handle the bmbt changes. + */ + xfs_filblks_t ip1_bcount; + + /* + * Data device blocks to be moved out of ip2, and free space needed to + * handle the bmbt changes. + */ + xfs_filblks_t ip2_bcount; + + /* rt blocks to be moved out of ip1. */ + xfs_filblks_t ip1_rtbcount; + + /* rt blocks to be moved out of ip2. */ + xfs_filblks_t ip2_rtbcount; + + /* Free space needed to handle the bmbt changes */ + unsigned long long resblks; + + /* Number of exchanges needed to complete the operation */ + unsigned long long nr_exchanges; +}; + +static inline int +xfs_exchmaps_reqfork(const struct xfs_exchmaps_req *req) +{ + if (req->flags & XFS_EXCHMAPS_ATTR_FORK) + return XFS_ATTR_FORK; + return XFS_DATA_FORK; +} + +int xfs_exchmaps_estimate_overhead(struct xfs_exchmaps_req *req); +int xfs_exchmaps_estimate(struct xfs_exchmaps_req *req); + +extern struct kmem_cache *xfs_exchmaps_intent_cache; + +int __init xfs_exchmaps_intent_init_cache(void); +void xfs_exchmaps_intent_destroy_cache(void); + +struct xfs_exchmaps_intent *xfs_exchmaps_init_intent( + const struct xfs_exchmaps_req *req); +void xfs_exchmaps_ensure_reflink(struct xfs_trans *tp, + const struct xfs_exchmaps_intent *xmi); +void xfs_exchmaps_upgrade_extent_counts(struct xfs_trans *tp, + const struct xfs_exchmaps_intent *xmi); + +int xfs_exchmaps_finish_one(struct xfs_trans *tp, + struct xfs_exchmaps_intent *xmi); + +int xfs_exchmaps_check_forks(struct xfs_mount *mp, + const struct xfs_exchmaps_req *req); + +void xfs_exchange_mappings(struct xfs_trans *tp, + const struct xfs_exchmaps_req *req); + +#endif /* __XFS_EXCHMAPS_H__ */ diff --git a/fs/xfs/libxfs/xfs_format.h b/fs/xfs/libxfs/xfs_format.h new file mode 100644 index 000000000..dd0ed046f --- /dev/null +++ b/fs/xfs/libxfs/xfs_format.h @@ -0,0 +1,2030 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_FORMAT_H__ +#define __XFS_FORMAT_H__ + +/* + * XFS On Disk Format Definitions + * + * This header file defines all the on-disk format definitions for + * general XFS objects. Directory and attribute related objects are defined in + * xfs_da_format.h, which log and log item formats are defined in + * xfs_log_format.h. Everything else goes here. + */ + +struct xfs_mount; +struct xfs_trans; +struct xfs_inode; +struct xfs_buf; +struct xfs_ifork; + +/* + * Super block + * Fits into a sector-sized buffer at address 0 of each allocation group. + * Only the first of these is ever updated except during growfs. + */ +#define XFS_SB_MAGIC 0x58465342 /* 'XFSB' */ +#define XFS_SB_VERSION_1 1 /* 5.3, 6.0.1, 6.1 */ +#define XFS_SB_VERSION_2 2 /* 6.2 - attributes */ +#define XFS_SB_VERSION_3 3 /* 6.2 - new inode version */ +#define XFS_SB_VERSION_4 4 /* 6.2+ - bitmask version */ +#define XFS_SB_VERSION_5 5 /* CRC enabled filesystem */ +#define XFS_SB_VERSION_NUMBITS 0x000f +#define XFS_SB_VERSION_ALLFBITS 0xfff0 +#define XFS_SB_VERSION_ATTRBIT 0x0010 +#define XFS_SB_VERSION_NLINKBIT 0x0020 +#define XFS_SB_VERSION_QUOTABIT 0x0040 +#define XFS_SB_VERSION_ALIGNBIT 0x0080 +#define XFS_SB_VERSION_DALIGNBIT 0x0100 +#define XFS_SB_VERSION_SHAREDBIT 0x0200 +#define XFS_SB_VERSION_LOGV2BIT 0x0400 +#define XFS_SB_VERSION_SECTORBIT 0x0800 +#define XFS_SB_VERSION_EXTFLGBIT 0x1000 +#define XFS_SB_VERSION_DIRV2BIT 0x2000 +#define XFS_SB_VERSION_BORGBIT 0x4000 /* ASCII only case-insens. */ +#define XFS_SB_VERSION_MOREBITSBIT 0x8000 + +/* + * The size of a single extended attribute on disk is limited by + * the size of index values within the attribute entries themselves. + * These are be16 fields, so we can only support attribute data + * sizes up to 2^16 bytes in length. + */ +#define XFS_XATTR_SIZE_MAX (1 << 16) + +/* + * Supported feature bit list is just all bits in the versionnum field because + * we've used them all up and understand them all. Except, of course, for the + * shared superblock bit, which nobody knows what it does and so is unsupported. + */ +#define XFS_SB_VERSION_OKBITS \ + ((XFS_SB_VERSION_NUMBITS | XFS_SB_VERSION_ALLFBITS) & \ + ~XFS_SB_VERSION_SHAREDBIT) + +/* + * There are two words to hold XFS "feature" bits: the original + * word, sb_versionnum, and sb_features2. Whenever a bit is set in + * sb_features2, the feature bit XFS_SB_VERSION_MOREBITSBIT must be set. + * + * These defines represent bits in sb_features2. + */ +#define XFS_SB_VERSION2_RESERVED1BIT 0x00000001 +#define XFS_SB_VERSION2_LAZYSBCOUNTBIT 0x00000002 /* Superblk counters */ +#define XFS_SB_VERSION2_RESERVED4BIT 0x00000004 +#define XFS_SB_VERSION2_ATTR2BIT 0x00000008 /* Inline attr rework */ +#define XFS_SB_VERSION2_PARENTBIT 0x00000010 /* parent pointers */ +#define XFS_SB_VERSION2_PROJID32BIT 0x00000080 /* 32 bit project id */ +#define XFS_SB_VERSION2_CRCBIT 0x00000100 /* metadata CRCs */ +#define XFS_SB_VERSION2_FTYPE 0x00000200 /* inode type in dir */ + +#define XFS_SB_VERSION2_OKBITS \ + (XFS_SB_VERSION2_LAZYSBCOUNTBIT | \ + XFS_SB_VERSION2_ATTR2BIT | \ + XFS_SB_VERSION2_PROJID32BIT | \ + XFS_SB_VERSION2_FTYPE) + +/* Maximum size of the xfs filesystem label, no terminating NULL */ +#define XFSLABEL_MAX 12 + +/* + * Superblock - in core version. Must be padded to 64 bit alignment. + */ +typedef struct xfs_sb { + uint32_t sb_magicnum; /* magic number == XFS_SB_MAGIC */ + uint32_t sb_blocksize; /* logical block size, bytes */ + xfs_rfsblock_t sb_dblocks; /* number of data blocks */ + xfs_rfsblock_t sb_rblocks; /* number of realtime blocks */ + xfs_rtbxlen_t sb_rextents; /* number of realtime extents */ + uuid_t sb_uuid; /* user-visible file system unique id */ + xfs_fsblock_t sb_logstart; /* starting block of log if internal */ + xfs_ino_t sb_rootino; /* root inode number */ + xfs_ino_t sb_rbmino; /* bitmap inode for realtime extents */ + xfs_ino_t sb_rsumino; /* summary inode for rt bitmap */ + xfs_agblock_t sb_rextsize; /* realtime extent size, blocks */ + xfs_agblock_t sb_agblocks; /* size of an allocation group */ + xfs_agnumber_t sb_agcount; /* number of allocation groups */ + xfs_extlen_t sb_rbmblocks; /* number of rt bitmap blocks */ + xfs_extlen_t sb_logblocks; /* number of log blocks */ + uint16_t sb_versionnum; /* header version == XFS_SB_VERSION */ + uint16_t sb_sectsize; /* volume sector size, bytes */ + uint16_t sb_inodesize; /* inode size, bytes */ + uint16_t sb_inopblock; /* inodes per block */ + char sb_fname[XFSLABEL_MAX] __nonstring; /* file system name */ + uint8_t sb_blocklog; /* log2 of sb_blocksize */ + uint8_t sb_sectlog; /* log2 of sb_sectsize */ + uint8_t sb_inodelog; /* log2 of sb_inodesize */ + uint8_t sb_inopblog; /* log2 of sb_inopblock */ + uint8_t sb_agblklog; /* log2 of sb_agblocks (rounded up) */ + uint8_t sb_rextslog; /* log2 of sb_rextents */ + uint8_t sb_inprogress; /* mkfs is in progress, don't mount */ + uint8_t sb_imax_pct; /* max % of fs for inode space */ + /* statistics */ + /* + * These fields must remain contiguous. If you really + * want to change their layout, make sure you fix the + * code in xfs_trans_apply_sb_deltas(). + */ + uint64_t sb_icount; /* allocated inodes */ + uint64_t sb_ifree; /* free inodes */ + uint64_t sb_fdblocks; /* free data blocks */ + uint64_t sb_frextents; /* free realtime extents */ + /* + * End contiguous fields. + */ + xfs_ino_t sb_uquotino; /* user quota inode */ + xfs_ino_t sb_gquotino; /* group quota inode */ + uint16_t sb_qflags; /* quota flags */ + uint8_t sb_flags; /* misc. flags */ + uint8_t sb_shared_vn; /* shared version number */ + xfs_extlen_t sb_inoalignmt; /* inode chunk alignment, fsblocks */ + uint32_t sb_unit; /* stripe or raid unit */ + uint32_t sb_width; /* stripe or raid width */ + uint8_t sb_dirblklog; /* log2 of dir block size (fsbs) */ + uint8_t sb_logsectlog; /* log2 of the log sector size */ + uint16_t sb_logsectsize; /* sector size for the log, bytes */ + uint32_t sb_logsunit; /* stripe unit size for the log */ + uint32_t sb_features2; /* additional feature bits */ + + /* + * bad features2 field as a result of failing to pad the sb structure to + * 64 bits. Some machines will be using this field for features2 bits. + * Easiest just to mark it bad and not use it for anything else. + * + * This is not kept up to date in memory; it is always overwritten by + * the value in sb_features2 when formatting the incore superblock to + * the disk buffer. + */ + uint32_t sb_bad_features2; + + /* version 5 superblock fields start here */ + + /* feature masks */ + uint32_t sb_features_compat; + uint32_t sb_features_ro_compat; + uint32_t sb_features_incompat; + uint32_t sb_features_log_incompat; + + uint32_t sb_crc; /* superblock crc */ + xfs_extlen_t sb_spino_align; /* sparse inode chunk alignment */ + + xfs_ino_t sb_pquotino; /* project quota inode */ + xfs_lsn_t sb_lsn; /* last write sequence */ + uuid_t sb_meta_uuid; /* metadata file system unique id */ + + xfs_ino_t sb_metadirino; /* metadata directory tree root */ + + xfs_rgnumber_t sb_rgcount; /* number of realtime groups */ + xfs_rtxlen_t sb_rgextents; /* size of a realtime group in rtx */ + uint8_t sb_rgblklog; /* rt group number shift */ + uint8_t sb_pad[7]; /* zeroes */ + xfs_rfsblock_t sb_rtstart; /* start of internal RT section (FSB) */ + xfs_filblks_t sb_rtreserved; /* reserved (zoned) RT blocks */ + + /* must be padded to 64 bit alignment */ +} xfs_sb_t; + +/* + * Superblock - on disk version. + * Must be padded to 64 bit alignment. + */ +struct xfs_dsb { + __be32 sb_magicnum; /* magic number == XFS_SB_MAGIC */ + __be32 sb_blocksize; /* logical block size, bytes */ + __be64 sb_dblocks; /* number of data blocks */ + __be64 sb_rblocks; /* number of realtime blocks */ + __be64 sb_rextents; /* number of realtime extents */ + uuid_t sb_uuid; /* user-visible file system unique id */ + __be64 sb_logstart; /* starting block of log if internal */ + __be64 sb_rootino; /* root inode number */ + __be64 sb_rbmino; /* bitmap inode for realtime extents */ + __be64 sb_rsumino; /* summary inode for rt bitmap */ + __be32 sb_rextsize; /* realtime extent size, blocks */ + __be32 sb_agblocks; /* size of an allocation group */ + __be32 sb_agcount; /* number of allocation groups */ + __be32 sb_rbmblocks; /* number of rt bitmap blocks */ + __be32 sb_logblocks; /* number of log blocks */ + __be16 sb_versionnum; /* header version == XFS_SB_VERSION */ + __be16 sb_sectsize; /* volume sector size, bytes */ + __be16 sb_inodesize; /* inode size, bytes */ + __be16 sb_inopblock; /* inodes per block */ + char sb_fname[XFSLABEL_MAX]; /* file system name */ + __u8 sb_blocklog; /* log2 of sb_blocksize */ + __u8 sb_sectlog; /* log2 of sb_sectsize */ + __u8 sb_inodelog; /* log2 of sb_inodesize */ + __u8 sb_inopblog; /* log2 of sb_inopblock */ + __u8 sb_agblklog; /* log2 of sb_agblocks (rounded up) */ + __u8 sb_rextslog; /* log2 of sb_rextents */ + __u8 sb_inprogress; /* mkfs is in progress, don't mount */ + __u8 sb_imax_pct; /* max % of fs for inode space */ + /* statistics */ + /* + * These fields must remain contiguous. If you really + * want to change their layout, make sure you fix the + * code in xfs_trans_apply_sb_deltas(). + */ + __be64 sb_icount; /* allocated inodes */ + __be64 sb_ifree; /* free inodes */ + __be64 sb_fdblocks; /* free data blocks */ + __be64 sb_frextents; /* free realtime extents */ + /* + * End contiguous fields. + */ + __be64 sb_uquotino; /* user quota inode */ + __be64 sb_gquotino; /* group quota inode */ + __be16 sb_qflags; /* quota flags */ + __u8 sb_flags; /* misc. flags */ + __u8 sb_shared_vn; /* shared version number */ + __be32 sb_inoalignmt; /* inode chunk alignment, fsblocks */ + __be32 sb_unit; /* stripe or raid unit */ + __be32 sb_width; /* stripe or raid width */ + __u8 sb_dirblklog; /* log2 of dir block size (fsbs) */ + __u8 sb_logsectlog; /* log2 of the log sector size */ + __be16 sb_logsectsize; /* sector size for the log, bytes */ + __be32 sb_logsunit; /* stripe unit size for the log */ + __be32 sb_features2; /* additional feature bits */ + /* + * bad features2 field as a result of failing to pad the sb + * structure to 64 bits. Some machines will be using this field + * for features2 bits. Easiest just to mark it bad and not use + * it for anything else. + */ + __be32 sb_bad_features2; + + /* version 5 superblock fields start here */ + + /* feature masks */ + __be32 sb_features_compat; + __be32 sb_features_ro_compat; + __be32 sb_features_incompat; + __be32 sb_features_log_incompat; + + __le32 sb_crc; /* superblock crc */ + __be32 sb_spino_align; /* sparse inode chunk alignment */ + + __be64 sb_pquotino; /* project quota inode */ + __be64 sb_lsn; /* last write sequence */ + uuid_t sb_meta_uuid; /* metadata file system unique id */ + + __be64 sb_metadirino; /* metadata directory tree root */ + __be32 sb_rgcount; /* # of realtime groups */ + __be32 sb_rgextents; /* size of rtgroup in rtx */ + __u8 sb_rgblklog; /* rt group number shift */ + __u8 sb_pad[7]; /* zeroes */ + __be64 sb_rtstart; /* start of internal RT section (FSB) */ + __be64 sb_rtreserved; /* reserved (zoned) RT blocks */ + + /* + * The size of this structure must be padded to 64 bit alignment. + * + * NOTE: Don't forget to update secondary_sb_whack in xfs_repair when + * adding new fields here. + */ +}; + +#define XFS_SB_CRC_OFF offsetof(struct xfs_dsb, sb_crc) + +/* + * Misc. Flags - warning - these will be cleared by xfs_repair unless + * a feature bit is set when the flag is used. + */ +#define XFS_SBF_NOFLAGS 0x00 /* no flags set */ +#define XFS_SBF_READONLY 0x01 /* only read-only mounts allowed */ + +/* + * define max. shared version we can interoperate with + */ +#define XFS_SB_MAX_SHARED_VN 0 + +#define XFS_SB_VERSION_NUM(sbp) ((sbp)->sb_versionnum & XFS_SB_VERSION_NUMBITS) + +static inline bool xfs_sb_is_v5(const struct xfs_sb *sbp) +{ + return XFS_SB_VERSION_NUM(sbp) == XFS_SB_VERSION_5; +} + +/* + * Detect a mismatched features2 field. Older kernels read/wrote + * this into the wrong slot, so to be safe we keep them in sync. + */ +static inline bool xfs_sb_has_mismatched_features2(const struct xfs_sb *sbp) +{ + return sbp->sb_bad_features2 != sbp->sb_features2; +} + +static inline bool xfs_sb_version_hasmorebits(const struct xfs_sb *sbp) +{ + return xfs_sb_is_v5(sbp) || + (sbp->sb_versionnum & XFS_SB_VERSION_MOREBITSBIT); +} + +static inline void xfs_sb_version_addattr(struct xfs_sb *sbp) +{ + sbp->sb_versionnum |= XFS_SB_VERSION_ATTRBIT; +} + +static inline void xfs_sb_version_addquota(struct xfs_sb *sbp) +{ + sbp->sb_versionnum |= XFS_SB_VERSION_QUOTABIT; +} + +static inline void xfs_sb_version_addattr2(struct xfs_sb *sbp) +{ + sbp->sb_versionnum |= XFS_SB_VERSION_MOREBITSBIT; + sbp->sb_features2 |= XFS_SB_VERSION2_ATTR2BIT; +} + +static inline void xfs_sb_version_addprojid32(struct xfs_sb *sbp) +{ + sbp->sb_versionnum |= XFS_SB_VERSION_MOREBITSBIT; + sbp->sb_features2 |= XFS_SB_VERSION2_PROJID32BIT; +} + +/* + * Extended v5 superblock feature masks. These are to be used for new v5 + * superblock features only. + * + * Compat features are new features that old kernels will not notice or affect + * and so can mount read-write without issues. + * + * RO-Compat (read only) are features that old kernels can read but will break + * if they write. Hence only read-only mounts of such filesystems are allowed on + * kernels that don't support the feature bit. + * + * InCompat features are features which old kernels will not understand and so + * must not mount. + * + * Log-InCompat features are for changes to log formats or new transactions that + * can't be replayed on older kernels. The fields are set when the filesystem is + * mounted, and a clean unmount clears the fields. + */ +#define XFS_SB_FEAT_COMPAT_ALL 0 +#define XFS_SB_FEAT_COMPAT_UNKNOWN ~XFS_SB_FEAT_COMPAT_ALL +static inline bool +xfs_sb_has_compat_feature( + const struct xfs_sb *sbp, + uint32_t feature) +{ + return (sbp->sb_features_compat & feature) != 0; +} + +#define XFS_SB_FEAT_RO_COMPAT_FINOBT (1 << 0) /* free inode btree */ +#define XFS_SB_FEAT_RO_COMPAT_RMAPBT (1 << 1) /* reverse map btree */ +#define XFS_SB_FEAT_RO_COMPAT_REFLINK (1 << 2) /* reflinked files */ +#define XFS_SB_FEAT_RO_COMPAT_INOBTCNT (1 << 3) /* inobt block counts */ +#define XFS_SB_FEAT_RO_COMPAT_ALL \ + (XFS_SB_FEAT_RO_COMPAT_FINOBT | \ + XFS_SB_FEAT_RO_COMPAT_RMAPBT | \ + XFS_SB_FEAT_RO_COMPAT_REFLINK| \ + XFS_SB_FEAT_RO_COMPAT_INOBTCNT) +#define XFS_SB_FEAT_RO_COMPAT_UNKNOWN ~XFS_SB_FEAT_RO_COMPAT_ALL +static inline bool +xfs_sb_has_ro_compat_feature( + const struct xfs_sb *sbp, + uint32_t feature) +{ + return (sbp->sb_features_ro_compat & feature) != 0; +} + +#define XFS_SB_FEAT_INCOMPAT_FTYPE (1 << 0) /* filetype in dirent */ +#define XFS_SB_FEAT_INCOMPAT_SPINODES (1 << 1) /* sparse inode chunks */ +#define XFS_SB_FEAT_INCOMPAT_META_UUID (1 << 2) /* metadata UUID */ +#define XFS_SB_FEAT_INCOMPAT_BIGTIME (1 << 3) /* large timestamps */ +#define XFS_SB_FEAT_INCOMPAT_NEEDSREPAIR (1 << 4) /* needs xfs_repair */ +#define XFS_SB_FEAT_INCOMPAT_NREXT64 (1 << 5) /* large extent counters */ +#define XFS_SB_FEAT_INCOMPAT_EXCHRANGE (1 << 6) /* exchangerange supported */ +#define XFS_SB_FEAT_INCOMPAT_PARENT (1 << 7) /* parent pointers */ +#define XFS_SB_FEAT_INCOMPAT_METADIR (1 << 8) /* metadata dir tree */ +#define XFS_SB_FEAT_INCOMPAT_ZONED (1 << 9) /* zoned RT allocator */ +#define XFS_SB_FEAT_INCOMPAT_ZONE_GAPS (1 << 10) /* RTGs have LBA gaps */ + +#define XFS_SB_FEAT_INCOMPAT_ALL \ + (XFS_SB_FEAT_INCOMPAT_FTYPE | \ + XFS_SB_FEAT_INCOMPAT_SPINODES | \ + XFS_SB_FEAT_INCOMPAT_META_UUID | \ + XFS_SB_FEAT_INCOMPAT_BIGTIME | \ + XFS_SB_FEAT_INCOMPAT_NEEDSREPAIR | \ + XFS_SB_FEAT_INCOMPAT_NREXT64 | \ + XFS_SB_FEAT_INCOMPAT_EXCHRANGE | \ + XFS_SB_FEAT_INCOMPAT_PARENT | \ + XFS_SB_FEAT_INCOMPAT_METADIR | \ + XFS_SB_FEAT_INCOMPAT_ZONED | \ + XFS_SB_FEAT_INCOMPAT_ZONE_GAPS) + +#define XFS_SB_FEAT_INCOMPAT_UNKNOWN ~XFS_SB_FEAT_INCOMPAT_ALL +static inline bool +xfs_sb_has_incompat_feature( + const struct xfs_sb *sbp, + uint32_t feature) +{ + return (sbp->sb_features_incompat & feature) != 0; +} + +#define XFS_SB_FEAT_INCOMPAT_LOG_XATTRS (1 << 0) /* Delayed Attributes */ +#define XFS_SB_FEAT_INCOMPAT_LOG_ALL \ + (XFS_SB_FEAT_INCOMPAT_LOG_XATTRS) +#define XFS_SB_FEAT_INCOMPAT_LOG_UNKNOWN ~XFS_SB_FEAT_INCOMPAT_LOG_ALL +static inline bool +xfs_sb_has_incompat_log_feature( + const struct xfs_sb *sbp, + uint32_t feature) +{ + return (sbp->sb_features_log_incompat & feature) != 0; +} + +static inline void +xfs_sb_remove_incompat_log_features( + struct xfs_sb *sbp) +{ + sbp->sb_features_log_incompat &= ~XFS_SB_FEAT_INCOMPAT_LOG_ALL; +} + +static inline void +xfs_sb_add_incompat_log_features( + struct xfs_sb *sbp, + unsigned int features) +{ + sbp->sb_features_log_incompat |= features; +} + +static inline bool xfs_sb_version_haslogxattrs(const struct xfs_sb *sbp) +{ + return xfs_sb_is_v5(sbp) && (sbp->sb_features_log_incompat & + XFS_SB_FEAT_INCOMPAT_LOG_XATTRS); +} + +static inline bool +xfs_is_quota_inode(struct xfs_sb *sbp, xfs_ino_t ino) +{ + return (ino == sbp->sb_uquotino || + ino == sbp->sb_gquotino || + ino == sbp->sb_pquotino); +} + +#define XFS_SB_DADDR ((xfs_daddr_t)0) /* daddr in filesystem/ag */ +#define XFS_SB_BLOCK(mp) XFS_HDR_BLOCK(mp, XFS_SB_DADDR) + +#define XFS_HDR_BLOCK(mp,d) ((xfs_agblock_t)XFS_BB_TO_FSBT(mp,d)) +#define XFS_DADDR_TO_FSB(mp,d) XFS_AGB_TO_FSB(mp, \ + xfs_daddr_to_agno(mp,d), xfs_daddr_to_agbno(mp,d)) +#define XFS_FSB_TO_DADDR(mp,fsbno) XFS_AGB_TO_DADDR(mp, \ + XFS_FSB_TO_AGNO(mp,fsbno), XFS_FSB_TO_AGBNO(mp,fsbno)) + +/* + * File system sector to basic block conversions. + */ +#define XFS_FSS_TO_BB(mp,sec) ((sec) << (mp)->m_sectbb_log) + +/* + * File system block to basic block conversions. + */ +#define XFS_FSB_TO_BB(mp,fsbno) ((fsbno) << (mp)->m_blkbb_log) +#define XFS_BB_TO_FSB(mp,bb) \ + (((bb) + (XFS_FSB_TO_BB(mp,1) - 1)) >> (mp)->m_blkbb_log) +#define XFS_BB_TO_FSBT(mp,bb) ((bb) >> (mp)->m_blkbb_log) + +/* + * File system block to byte conversions. + */ +#define XFS_FSB_TO_B(mp,fsbno) ((xfs_fsize_t)(fsbno) << (mp)->m_sb.sb_blocklog) +#define XFS_B_TO_FSB(mp,b) \ + ((((uint64_t)(b)) + (mp)->m_blockmask) >> (mp)->m_sb.sb_blocklog) +#define XFS_B_TO_FSBT(mp,b) (((uint64_t)(b)) >> (mp)->m_sb.sb_blocklog) + +/* + * Allocation group header + * + * This is divided into three structures, placed in sequential 512-byte + * buffers after a copy of the superblock (also in a 512-byte buffer). + */ +#define XFS_AGF_MAGIC 0x58414746 /* 'XAGF' */ +#define XFS_AGI_MAGIC 0x58414749 /* 'XAGI' */ +#define XFS_AGFL_MAGIC 0x5841464c /* 'XAFL' */ +#define XFS_AGF_VERSION 1 +#define XFS_AGI_VERSION 1 + +#define XFS_AGF_GOOD_VERSION(v) ((v) == XFS_AGF_VERSION) +#define XFS_AGI_GOOD_VERSION(v) ((v) == XFS_AGI_VERSION) + +/* + * agf_cnt_level in the first AGF overlaps the EFS superblock's magic number. + * Since the magic numbers valid for EFS are > 64k, our value cannot be confused + * for an EFS superblock. + */ + +typedef struct xfs_agf { + /* + * Common allocation group header information + */ + __be32 agf_magicnum; /* magic number == XFS_AGF_MAGIC */ + __be32 agf_versionnum; /* header version == XFS_AGF_VERSION */ + __be32 agf_seqno; /* sequence # starting from 0 */ + __be32 agf_length; /* size in blocks of a.g. */ + /* + * Freespace and rmap information + */ + __be32 agf_bno_root; /* bnobt root block */ + __be32 agf_cnt_root; /* cntbt root block */ + __be32 agf_rmap_root; /* rmapbt root block */ + + __be32 agf_bno_level; /* bnobt btree levels */ + __be32 agf_cnt_level; /* cntbt btree levels */ + __be32 agf_rmap_level; /* rmapbt btree levels */ + + __be32 agf_flfirst; /* first freelist block's index */ + __be32 agf_fllast; /* last freelist block's index */ + __be32 agf_flcount; /* count of blocks in freelist */ + __be32 agf_freeblks; /* total free blocks */ + + __be32 agf_longest; /* longest free space */ + __be32 agf_btreeblks; /* # of blocks held in AGF btrees */ + uuid_t agf_uuid; /* uuid of filesystem */ + + __be32 agf_rmap_blocks; /* rmapbt blocks used */ + __be32 agf_refcount_blocks; /* refcountbt blocks used */ + + __be32 agf_refcount_root; /* refcount tree root block */ + __be32 agf_refcount_level; /* refcount btree levels */ + + /* + * reserve some contiguous space for future logged fields before we add + * the unlogged fields. This makes the range logging via flags and + * structure offsets much simpler. + */ + __be64 agf_spare64[14]; + + /* unlogged fields, written during buffer writeback. */ + __be64 agf_lsn; /* last write sequence */ + __be32 agf_crc; /* crc of agf sector */ + __be32 agf_spare2; + + /* structure must be padded to 64 bit alignment */ +} xfs_agf_t; + +#define XFS_AGF_CRC_OFF offsetof(struct xfs_agf, agf_crc) + +#define XFS_AGF_MAGICNUM (1u << 0) +#define XFS_AGF_VERSIONNUM (1u << 1) +#define XFS_AGF_SEQNO (1u << 2) +#define XFS_AGF_LENGTH (1u << 3) +#define XFS_AGF_ROOTS (1u << 4) +#define XFS_AGF_LEVELS (1u << 5) +#define XFS_AGF_FLFIRST (1u << 6) +#define XFS_AGF_FLLAST (1u << 7) +#define XFS_AGF_FLCOUNT (1u << 8) +#define XFS_AGF_FREEBLKS (1u << 9) +#define XFS_AGF_LONGEST (1u << 10) +#define XFS_AGF_BTREEBLKS (1u << 11) +#define XFS_AGF_UUID (1u << 12) +#define XFS_AGF_RMAP_BLOCKS (1u << 13) +#define XFS_AGF_REFCOUNT_BLOCKS (1u << 14) +#define XFS_AGF_REFCOUNT_ROOT (1u << 15) +#define XFS_AGF_REFCOUNT_LEVEL (1u << 16) +#define XFS_AGF_SPARE64 (1u << 17) +#define XFS_AGF_NUM_BITS 18 +#define XFS_AGF_ALL_BITS ((1u << XFS_AGF_NUM_BITS) - 1) + +#define XFS_AGF_FLAGS \ + { XFS_AGF_MAGICNUM, "MAGICNUM" }, \ + { XFS_AGF_VERSIONNUM, "VERSIONNUM" }, \ + { XFS_AGF_SEQNO, "SEQNO" }, \ + { XFS_AGF_LENGTH, "LENGTH" }, \ + { XFS_AGF_ROOTS, "ROOTS" }, \ + { XFS_AGF_LEVELS, "LEVELS" }, \ + { XFS_AGF_FLFIRST, "FLFIRST" }, \ + { XFS_AGF_FLLAST, "FLLAST" }, \ + { XFS_AGF_FLCOUNT, "FLCOUNT" }, \ + { XFS_AGF_FREEBLKS, "FREEBLKS" }, \ + { XFS_AGF_LONGEST, "LONGEST" }, \ + { XFS_AGF_BTREEBLKS, "BTREEBLKS" }, \ + { XFS_AGF_UUID, "UUID" }, \ + { XFS_AGF_RMAP_BLOCKS, "RMAP_BLOCKS" }, \ + { XFS_AGF_REFCOUNT_BLOCKS, "REFCOUNT_BLOCKS" }, \ + { XFS_AGF_REFCOUNT_ROOT, "REFCOUNT_ROOT" }, \ + { XFS_AGF_REFCOUNT_LEVEL, "REFCOUNT_LEVEL" }, \ + { XFS_AGF_SPARE64, "SPARE64" } + +/* disk block (xfs_daddr_t) in the AG */ +#define XFS_AGF_DADDR(mp) ((xfs_daddr_t)(1 << (mp)->m_sectbb_log)) +#define XFS_AGF_BLOCK(mp) XFS_HDR_BLOCK(mp, XFS_AGF_DADDR(mp)) + +/* + * Size of the unlinked inode hash table in the agi. + */ +#define XFS_AGI_UNLINKED_BUCKETS 64 + +typedef struct xfs_agi { + /* + * Common allocation group header information + */ + __be32 agi_magicnum; /* magic number == XFS_AGI_MAGIC */ + __be32 agi_versionnum; /* header version == XFS_AGI_VERSION */ + __be32 agi_seqno; /* sequence # starting from 0 */ + __be32 agi_length; /* size in blocks of a.g. */ + /* + * Inode information + * Inodes are mapped by interpreting the inode number, so no + * mapping data is needed here. + */ + __be32 agi_count; /* count of allocated inodes */ + __be32 agi_root; /* root of inode btree */ + __be32 agi_level; /* levels in inode btree */ + __be32 agi_freecount; /* number of free inodes */ + + __be32 agi_newino; /* new inode just allocated */ + __be32 agi_dirino; /* last directory inode chunk */ + /* + * Hash table of inodes which have been unlinked but are + * still being referenced. + */ + __be32 agi_unlinked[XFS_AGI_UNLINKED_BUCKETS]; + /* + * This marks the end of logging region 1 and start of logging region 2. + */ + uuid_t agi_uuid; /* uuid of filesystem */ + __be32 agi_crc; /* crc of agi sector */ + __be32 agi_pad32; + __be64 agi_lsn; /* last write sequence */ + + __be32 agi_free_root; /* root of the free inode btree */ + __be32 agi_free_level;/* levels in free inode btree */ + + __be32 agi_iblocks; /* inobt blocks used */ + __be32 agi_fblocks; /* finobt blocks used */ + + /* structure must be padded to 64 bit alignment */ +} xfs_agi_t; + +#define XFS_AGI_CRC_OFF offsetof(struct xfs_agi, agi_crc) + +#define XFS_AGI_MAGICNUM (1u << 0) +#define XFS_AGI_VERSIONNUM (1u << 1) +#define XFS_AGI_SEQNO (1u << 2) +#define XFS_AGI_LENGTH (1u << 3) +#define XFS_AGI_COUNT (1u << 4) +#define XFS_AGI_ROOT (1u << 5) +#define XFS_AGI_LEVEL (1u << 6) +#define XFS_AGI_FREECOUNT (1u << 7) +#define XFS_AGI_NEWINO (1u << 8) +#define XFS_AGI_DIRINO (1u << 9) +#define XFS_AGI_UNLINKED (1u << 10) +#define XFS_AGI_NUM_BITS_R1 11 /* end of the 1st agi logging region */ +#define XFS_AGI_ALL_BITS_R1 ((1u << XFS_AGI_NUM_BITS_R1) - 1) +#define XFS_AGI_FREE_ROOT (1u << 11) +#define XFS_AGI_FREE_LEVEL (1u << 12) +#define XFS_AGI_IBLOCKS (1u << 13) /* both inobt/finobt block counters */ +#define XFS_AGI_NUM_BITS_R2 14 + +/* disk block (xfs_daddr_t) in the AG */ +#define XFS_AGI_DADDR(mp) ((xfs_daddr_t)(2 << (mp)->m_sectbb_log)) +#define XFS_AGI_BLOCK(mp) XFS_HDR_BLOCK(mp, XFS_AGI_DADDR(mp)) + +/* + * The third a.g. block contains the a.g. freelist, an array + * of block pointers to blocks owned by the allocation btree code. + */ +#define XFS_AGFL_DADDR(mp) ((xfs_daddr_t)(3 << (mp)->m_sectbb_log)) +#define XFS_AGFL_BLOCK(mp) XFS_HDR_BLOCK(mp, XFS_AGFL_DADDR(mp)) +#define XFS_BUF_TO_AGFL(bp) ((struct xfs_agfl *)((bp)->b_addr)) + +struct xfs_agfl { + __be32 agfl_magicnum; + __be32 agfl_seqno; + uuid_t agfl_uuid; + __be64 agfl_lsn; + __be32 agfl_crc; +} __attribute__((packed)); + +#define XFS_AGFL_CRC_OFF offsetof(struct xfs_agfl, agfl_crc) + +#define XFS_AGB_TO_FSB(mp,agno,agbno) \ + (((xfs_fsblock_t)(agno) << (mp)->m_sb.sb_agblklog) | (agbno)) +#define XFS_FSB_TO_AGNO(mp,fsbno) \ + ((xfs_agnumber_t)((fsbno) >> (mp)->m_sb.sb_agblklog)) +#define XFS_FSB_TO_AGBNO(mp,fsbno) \ + ((xfs_agblock_t)((fsbno) & xfs_mask32lo((mp)->m_sb.sb_agblklog))) +#define XFS_AGB_TO_DADDR(mp,agno,agbno) \ + ((xfs_daddr_t)XFS_FSB_TO_BB(mp, \ + (xfs_fsblock_t)(agno) * (mp)->m_sb.sb_agblocks + (agbno))) +#define XFS_AG_DADDR(mp,agno,d) (XFS_AGB_TO_DADDR(mp, agno, 0) + (d)) + +/* + * For checking for bad ranges of xfs_daddr_t's, covering multiple + * allocation groups or a single xfs_daddr_t that's a superblock copy. + */ +#define XFS_AG_CHECK_DADDR(mp,d,len) \ + ((len) == 1 ? \ + ASSERT((d) == XFS_SB_DADDR || \ + xfs_daddr_to_agbno(mp, d) != XFS_SB_DADDR) : \ + ASSERT(xfs_daddr_to_agno(mp, d) == \ + xfs_daddr_to_agno(mp, (d) + (len) - 1))) + +/* + * Realtime bitmap information is accessed by the word, which is currently + * stored in host-endian format. Starting with the realtime groups feature, + * the words are stored in be32 ondisk. + */ +union xfs_rtword_raw { + __u32 old; + __be32 rtg; +}; + +/* + * Realtime summary counts are accessed by the word, which is currently + * stored in host-endian format. Starting with the realtime groups feature, + * the words are stored in be32 ondisk. + */ +union xfs_suminfo_raw { + __u32 old; + __be32 rtg; +}; + +/* + * Realtime allocation groups break the rt section into multiple pieces that + * could be locked independently. Realtime block group numbers are 32-bit + * quantities. Block numbers within a group are also 32-bit quantities, but + * the upper bit must never be set. rtgroup 0 might have a superblock in it, + * so the minimum size of an rtgroup is 2 rtx. + */ +#define XFS_MAX_RGBLOCKS ((xfs_rgblock_t)(1U << 31) - 1) +#define XFS_MIN_RGEXTENTS ((xfs_rtxlen_t)2) +#define XFS_MAX_RGNUMBER ((xfs_rgnumber_t)(-1U)) + +#define XFS_RTSB_MAGIC 0x46726F67 /* 'Frog' */ + +/* + * Realtime superblock - on disk version. Must be padded to 64 bit alignment. + * The first block of the realtime volume contains this superblock. + */ +struct xfs_rtsb { + __be32 rsb_magicnum; /* magic number == XFS_RTSB_MAGIC */ + __le32 rsb_crc; /* superblock crc */ + + __be32 rsb_pad; /* zero */ + unsigned char rsb_fname[XFSLABEL_MAX]; /* file system name */ + + uuid_t rsb_uuid; /* user-visible file system unique id */ + uuid_t rsb_meta_uuid; /* metadata file system unique id */ + + /* must be padded to 64 bit alignment */ +}; + +#define XFS_RTSB_CRC_OFF offsetof(struct xfs_rtsb, rsb_crc) +#define XFS_RTSB_DADDR ((xfs_daddr_t)0) /* daddr in rt section */ + +/* + * XFS Timestamps + * ============== + * + * Traditional ondisk inode timestamps consist of signed 32-bit counters for + * seconds and nanoseconds; time zero is the Unix epoch, Jan 1 00:00:00 UTC + * 1970, which means that the timestamp epoch is the same as the Unix epoch. + * Therefore, the ondisk min and max defined here can be used directly to + * constrain the incore timestamps on a Unix system. Note that we actually + * encode a __be64 value on disk. + * + * When the bigtime feature is enabled, ondisk inode timestamps become an + * unsigned 64-bit nanoseconds counter. This means that the bigtime inode + * timestamp epoch is the start of the classic timestamp range, which is + * Dec 13 20:45:52 UTC 1901. Because the epochs are not the same, callers + * /must/ use the bigtime conversion functions when encoding and decoding raw + * timestamps. + */ +typedef __be64 xfs_timestamp_t; + +/* Legacy timestamp encoding format. */ +struct xfs_legacy_timestamp { + __be32 t_sec; /* timestamp seconds */ + __be32 t_nsec; /* timestamp nanoseconds */ +}; + +/* + * Smallest possible ondisk seconds value with traditional timestamps. This + * corresponds exactly with the incore timestamp Dec 13 20:45:52 UTC 1901. + */ +#define XFS_LEGACY_TIME_MIN ((int64_t)S32_MIN) + +/* + * Largest possible ondisk seconds value with traditional timestamps. This + * corresponds exactly with the incore timestamp Jan 19 03:14:07 UTC 2038. + */ +#define XFS_LEGACY_TIME_MAX ((int64_t)S32_MAX) + +/* + * Smallest possible ondisk seconds value with bigtime timestamps. This + * corresponds (after conversion to a Unix timestamp) with the traditional + * minimum timestamp of Dec 13 20:45:52 UTC 1901. + */ +#define XFS_BIGTIME_TIME_MIN ((int64_t)0) + +/* + * Largest supported ondisk seconds value with bigtime timestamps. This + * corresponds (after conversion to a Unix timestamp) with an incore timestamp + * of Jul 2 20:20:24 UTC 2486. + * + * We round down the ondisk limit so that the bigtime quota and inode max + * timestamps will be the same. + */ +#define XFS_BIGTIME_TIME_MAX ((int64_t)((-1ULL / NSEC_PER_SEC) & ~0x3ULL)) + +/* + * Bigtime epoch is set exactly to the minimum time value that a traditional + * 32-bit timestamp can represent when using the Unix epoch as a reference. + * Hence the Unix epoch is at a fixed offset into the supported bigtime + * timestamp range. + * + * The bigtime epoch also matches the minimum value an on-disk 32-bit XFS + * timestamp can represent so we will not lose any fidelity in converting + * to/from unix and bigtime timestamps. + * + * The following conversion factor converts a seconds counter from the Unix + * epoch to the bigtime epoch. + */ +#define XFS_BIGTIME_EPOCH_OFFSET (-(int64_t)S32_MIN) + +/* Convert a timestamp from the Unix epoch to the bigtime epoch. */ +static inline uint64_t xfs_unix_to_bigtime(time64_t unix_seconds) +{ + return (uint64_t)unix_seconds + XFS_BIGTIME_EPOCH_OFFSET; +} + +/* Convert a timestamp from the bigtime epoch to the Unix epoch. */ +static inline time64_t xfs_bigtime_to_unix(uint64_t ondisk_seconds) +{ + return (time64_t)ondisk_seconds - XFS_BIGTIME_EPOCH_OFFSET; +} + +enum xfs_metafile_type { + XFS_METAFILE_UNKNOWN, /* unknown */ + XFS_METAFILE_DIR, /* metadir directory */ + XFS_METAFILE_USRQUOTA, /* user quota */ + XFS_METAFILE_GRPQUOTA, /* group quota */ + XFS_METAFILE_PRJQUOTA, /* project quota */ + XFS_METAFILE_RTBITMAP, /* rt bitmap */ + XFS_METAFILE_RTSUMMARY, /* rt summary */ + XFS_METAFILE_RTRMAP, /* rt rmap */ + XFS_METAFILE_RTREFCOUNT, /* rt refcount */ + + XFS_METAFILE_MAX +} __packed; + +#define XFS_METAFILE_TYPE_STR \ + { XFS_METAFILE_UNKNOWN, "unknown" }, \ + { XFS_METAFILE_DIR, "dir" }, \ + { XFS_METAFILE_USRQUOTA, "usrquota" }, \ + { XFS_METAFILE_GRPQUOTA, "grpquota" }, \ + { XFS_METAFILE_PRJQUOTA, "prjquota" }, \ + { XFS_METAFILE_RTBITMAP, "rtbitmap" }, \ + { XFS_METAFILE_RTSUMMARY, "rtsummary" }, \ + { XFS_METAFILE_RTRMAP, "rtrmap" }, \ + { XFS_METAFILE_RTREFCOUNT, "rtrefcount" } + +/* + * On-disk inode structure. + * + * This is just the header or "dinode core", the inode is expanded to fill a + * variable size the leftover area split into a data and an attribute fork. + * The format of the data and attribute fork depends on the format of the + * inode as indicated by di_format and di_aformat. To access the data and + * attribute use the XFS_DFORK_DPTR, XFS_DFORK_APTR, and XFS_DFORK_PTR macros + * below. + * + * There is a very similar struct xfs_log_dinode which matches the layout of + * this structure, but is kept in native format instead of big endian. + * + * Note: di_flushiter is only used by v1/2 inodes - it's effectively a zeroed + * padding field for v3 inodes. + */ +#define XFS_DINODE_MAGIC 0x494e /* 'IN' */ +struct xfs_dinode { + __be16 di_magic; /* inode magic # = XFS_DINODE_MAGIC */ + __be16 di_mode; /* mode and type of file */ + __u8 di_version; /* inode version */ + __u8 di_format; /* format of di_c data */ + __be16 di_metatype; /* XFS_METAFILE_*; was di_onlink */ + __be32 di_uid; /* owner's user id */ + __be32 di_gid; /* owner's group id */ + __be32 di_nlink; /* number of links to file */ + __be16 di_projid_lo; /* lower part of owner's project id */ + __be16 di_projid_hi; /* higher part owner's project id */ + union { + /* Number of data fork extents if NREXT64 is set */ + __be64 di_big_nextents; + + /* Padding for V3 inodes without NREXT64 set. */ + __be64 di_v3_pad; + + /* Padding and inode flush counter for V2 inodes. */ + struct { + __u8 di_v2_pad[6]; + __be16 di_flushiter; + }; + }; + xfs_timestamp_t di_atime; /* time last accessed */ + xfs_timestamp_t di_mtime; /* time last modified */ + xfs_timestamp_t di_ctime; /* time created/inode modified */ + __be64 di_size; /* number of bytes in file */ + __be64 di_nblocks; /* # of direct & btree blocks used */ + __be32 di_extsize; /* basic/minimum extent size for file */ + union { + /* + * For V2 inodes and V3 inodes without NREXT64 set, this + * is the number of data and attr fork extents. + */ + struct { + __be32 di_nextents; + __be16 di_anextents; + } __packed; + + /* Number of attr fork extents if NREXT64 is set. */ + struct { + __be32 di_big_anextents; + __be16 di_nrext64_pad; + } __packed; + } __packed; + __u8 di_forkoff; /* attr fork offs, <<3 for 64b align */ + __s8 di_aformat; /* format of attr fork's data */ + __be32 di_dmevmask; /* DMIG event mask */ + __be16 di_dmstate; /* DMIG state info */ + __be16 di_flags; /* random flags, XFS_DIFLAG_... */ + __be32 di_gen; /* generation number */ + + /* di_next_unlinked is the only non-core field in the old dinode */ + __be32 di_next_unlinked;/* agi unlinked list ptr */ + + /* start of the extended dinode, writable fields */ + __le32 di_crc; /* CRC of the inode */ + __be64 di_changecount; /* number of attribute changes */ + __be64 di_lsn; /* flush sequence */ + __be64 di_flags2; /* more random flags */ + union { + /* basic cow extent size for (regular) file */ + __be32 di_cowextsize; + /* used blocks in RTG for (zoned) rtrmap inode */ + __be32 di_used_blocks; + }; + __u8 di_pad2[12]; /* more padding for future expansion */ + + /* fields only written to during inode creation */ + xfs_timestamp_t di_crtime; /* time created */ + __be64 di_ino; /* inode number */ + uuid_t di_uuid; /* UUID of the filesystem */ + + /* structure must be padded to 64 bit alignment */ +}; + +#define XFS_DINODE_CRC_OFF offsetof(struct xfs_dinode, di_crc) + +#define DI_MAX_FLUSH 0xffff + +/* + * Size of the core inode on disk. Version 1 and 2 inodes have + * the same size, but version 3 has grown a few additional fields. + */ +static inline uint xfs_dinode_size(int version) +{ + if (version == 3) + return sizeof(struct xfs_dinode); + return offsetof(struct xfs_dinode, di_crc); +} + +/* + * The 32 bit link count in the inode theoretically maxes out at UINT_MAX. + * Since the pathconf interface is signed, we use 2^31 - 1 instead. + */ +#define XFS_MAXLINK ((1U << 31) - 1U) + +/* + * Any file that hits the maximum ondisk link count should be pinned to avoid + * a use-after-free situation. + */ +#define XFS_NLINK_PINNED (~0U) + +/* + * Values for di_format + * + * This enum is used in string mapping in xfs_trace.h; please keep the + * TRACE_DEFINE_ENUMs for it up to date. + */ +enum xfs_dinode_fmt { + XFS_DINODE_FMT_DEV, /* xfs_dev_t */ + XFS_DINODE_FMT_LOCAL, /* bulk data */ + XFS_DINODE_FMT_EXTENTS, /* struct xfs_bmbt_rec */ + XFS_DINODE_FMT_BTREE, /* struct xfs_bmdr_block */ + XFS_DINODE_FMT_UUID, /* added long ago, but never used */ + XFS_DINODE_FMT_META_BTREE, /* metadata btree */ +}; + +#define XFS_INODE_FORMAT_STR \ + { XFS_DINODE_FMT_DEV, "dev" }, \ + { XFS_DINODE_FMT_LOCAL, "local" }, \ + { XFS_DINODE_FMT_EXTENTS, "extent" }, \ + { XFS_DINODE_FMT_BTREE, "btree" }, \ + { XFS_DINODE_FMT_UUID, "uuid" }, \ + { XFS_DINODE_FMT_META_BTREE, "meta_btree" } + +/* + * Max values for extnum and aextnum. + * + * The original on-disk extent counts were held in signed fields, resulting in + * maximum extent counts of 2^31 and 2^15 for the data and attr forks + * respectively. Similarly the maximum extent length is limited to 2^21 blocks + * by the 21-bit wide blockcount field of a BMBT extent record. + * + * The newly introduced data fork extent counter can hold a 64-bit value, + * however the maximum number of extents in a file is also limited to 2^54 + * extents by the 54-bit wide startoff field of a BMBT extent record. + * + * It is further limited by the maximum supported file size of 2^63 + * *bytes*. This leads to a maximum extent count for maximally sized filesystem + * blocks (64kB) of: + * + * 2^63 bytes / 2^16 bytes per block = 2^47 blocks + * + * Rounding up 47 to the nearest multiple of bits-per-byte results in 48. Hence + * 2^48 was chosen as the maximum data fork extent count. + * + * The maximum file size that can be represented by the data fork extent counter + * in the worst case occurs when all extents are 1 block in length and each + * block is 1KB in size. + * + * With XFS_MAX_EXTCNT_DATA_FORK_SMALL representing maximum extent count and + * with 1KB sized blocks, a file can reach upto, + * 1KB * (2^31) = 2TB + * + * This is much larger than the theoretical maximum size of a directory + * i.e. XFS_DIR2_SPACE_SIZE * XFS_DIR2_MAX_SPACES = ~96GB. + * + * Hence, a directory inode can never overflow its data fork extent counter. + */ +#define XFS_MAX_EXTCNT_DATA_FORK_LARGE ((xfs_extnum_t)((1ULL << 48) - 1)) +#define XFS_MAX_EXTCNT_ATTR_FORK_LARGE ((xfs_extnum_t)((1ULL << 32) - 1)) +#define XFS_MAX_EXTCNT_DATA_FORK_SMALL ((xfs_extnum_t)((1ULL << 31) - 1)) +#define XFS_MAX_EXTCNT_ATTR_FORK_SMALL ((xfs_extnum_t)((1ULL << 15) - 1)) + +/* + * When we upgrade an inode to the large extent counts, the maximum value by + * which the extent count can increase is bound by the change in size of the + * on-disk field. No upgrade operation should ever be adding more than a few + * tens of extents, so if we get a really large value it is a sign of a code bug + * or corruption. + */ +#define XFS_MAX_EXTCNT_UPGRADE_NR \ + min(XFS_MAX_EXTCNT_ATTR_FORK_LARGE - XFS_MAX_EXTCNT_ATTR_FORK_SMALL, \ + XFS_MAX_EXTCNT_DATA_FORK_LARGE - XFS_MAX_EXTCNT_DATA_FORK_SMALL) + +/* + * Inode minimum and maximum sizes. + */ +#define XFS_DINODE_MIN_LOG 8 +#define XFS_DINODE_MAX_LOG 11 +#define XFS_DINODE_MIN_SIZE (1 << XFS_DINODE_MIN_LOG) +#define XFS_DINODE_MAX_SIZE (1 << XFS_DINODE_MAX_LOG) + +/* + * Inode size for given fs. + */ +#define XFS_DINODE_SIZE(mp) \ + (xfs_has_v3inodes(mp) ? \ + sizeof(struct xfs_dinode) : \ + offsetof(struct xfs_dinode, di_crc)) +#define XFS_LITINO(mp) \ + ((mp)->m_sb.sb_inodesize - XFS_DINODE_SIZE(mp)) + +/* + * Inode data & attribute fork sizes, per inode. + */ +#define XFS_DFORK_BOFF(dip) ((int)((dip)->di_forkoff << 3)) + +#define XFS_DFORK_DSIZE(dip,mp) \ + ((dip)->di_forkoff ? XFS_DFORK_BOFF(dip) : XFS_LITINO(mp)) +#define XFS_DFORK_ASIZE(dip,mp) \ + ((dip)->di_forkoff ? XFS_LITINO(mp) - XFS_DFORK_BOFF(dip) : 0) +#define XFS_DFORK_SIZE(dip,mp,w) \ + ((w) == XFS_DATA_FORK ? \ + XFS_DFORK_DSIZE(dip, mp) : \ + XFS_DFORK_ASIZE(dip, mp)) + +#define XFS_DFORK_MAXEXT(dip, mp, w) \ + (XFS_DFORK_SIZE(dip, mp, w) / sizeof(struct xfs_bmbt_rec)) + +/* + * Return pointers to the data or attribute forks. + */ +#define XFS_DFORK_DPTR(dip) \ + ((void *)dip + xfs_dinode_size(dip->di_version)) +#define XFS_DFORK_APTR(dip) \ + (XFS_DFORK_DPTR(dip) + XFS_DFORK_BOFF(dip)) +#define XFS_DFORK_PTR(dip,w) \ + ((w) == XFS_DATA_FORK ? XFS_DFORK_DPTR(dip) : XFS_DFORK_APTR(dip)) + +#define XFS_DFORK_FORMAT(dip,w) \ + ((w) == XFS_DATA_FORK ? \ + (dip)->di_format : \ + (dip)->di_aformat) + +/* + * For block and character special files the 32bit dev_t is stored at the + * beginning of the data fork. + */ +static inline xfs_dev_t xfs_dinode_get_rdev(struct xfs_dinode *dip) +{ + return be32_to_cpu(*(__be32 *)XFS_DFORK_DPTR(dip)); +} + +static inline void xfs_dinode_put_rdev(struct xfs_dinode *dip, xfs_dev_t rdev) +{ + *(__be32 *)XFS_DFORK_DPTR(dip) = cpu_to_be32(rdev); +} + +/* + * Values for di_flags + */ +#define XFS_DIFLAG_REALTIME_BIT 0 /* file's blocks come from rt area */ +#define XFS_DIFLAG_PREALLOC_BIT 1 /* file space has been preallocated */ +#define XFS_DIFLAG_NEWRTBM_BIT 2 /* for rtbitmap inode, new format */ +#define XFS_DIFLAG_IMMUTABLE_BIT 3 /* inode is immutable */ +#define XFS_DIFLAG_APPEND_BIT 4 /* inode is append-only */ +#define XFS_DIFLAG_SYNC_BIT 5 /* inode is written synchronously */ +#define XFS_DIFLAG_NOATIME_BIT 6 /* do not update atime */ +#define XFS_DIFLAG_NODUMP_BIT 7 /* do not dump */ +#define XFS_DIFLAG_RTINHERIT_BIT 8 /* create with realtime bit set */ +#define XFS_DIFLAG_PROJINHERIT_BIT 9 /* create with parents projid */ +#define XFS_DIFLAG_NOSYMLINKS_BIT 10 /* disallow symlink creation */ +#define XFS_DIFLAG_EXTSIZE_BIT 11 /* inode extent size allocator hint */ +#define XFS_DIFLAG_EXTSZINHERIT_BIT 12 /* inherit inode extent size */ +#define XFS_DIFLAG_NODEFRAG_BIT 13 /* do not reorganize/defragment */ +#define XFS_DIFLAG_FILESTREAM_BIT 14 /* use filestream allocator */ +/* Do not use bit 15, di_flags is legacy and unchanging now */ + +#define XFS_DIFLAG_REALTIME (1 << XFS_DIFLAG_REALTIME_BIT) +#define XFS_DIFLAG_PREALLOC (1 << XFS_DIFLAG_PREALLOC_BIT) +#define XFS_DIFLAG_NEWRTBM (1 << XFS_DIFLAG_NEWRTBM_BIT) +#define XFS_DIFLAG_IMMUTABLE (1 << XFS_DIFLAG_IMMUTABLE_BIT) +#define XFS_DIFLAG_APPEND (1 << XFS_DIFLAG_APPEND_BIT) +#define XFS_DIFLAG_SYNC (1 << XFS_DIFLAG_SYNC_BIT) +#define XFS_DIFLAG_NOATIME (1 << XFS_DIFLAG_NOATIME_BIT) +#define XFS_DIFLAG_NODUMP (1 << XFS_DIFLAG_NODUMP_BIT) +#define XFS_DIFLAG_RTINHERIT (1 << XFS_DIFLAG_RTINHERIT_BIT) +#define XFS_DIFLAG_PROJINHERIT (1 << XFS_DIFLAG_PROJINHERIT_BIT) +#define XFS_DIFLAG_NOSYMLINKS (1 << XFS_DIFLAG_NOSYMLINKS_BIT) +#define XFS_DIFLAG_EXTSIZE (1 << XFS_DIFLAG_EXTSIZE_BIT) +#define XFS_DIFLAG_EXTSZINHERIT (1 << XFS_DIFLAG_EXTSZINHERIT_BIT) +#define XFS_DIFLAG_NODEFRAG (1 << XFS_DIFLAG_NODEFRAG_BIT) +#define XFS_DIFLAG_FILESTREAM (1 << XFS_DIFLAG_FILESTREAM_BIT) + +#define XFS_DIFLAG_ANY \ + (XFS_DIFLAG_REALTIME | XFS_DIFLAG_PREALLOC | XFS_DIFLAG_NEWRTBM | \ + XFS_DIFLAG_IMMUTABLE | XFS_DIFLAG_APPEND | XFS_DIFLAG_SYNC | \ + XFS_DIFLAG_NOATIME | XFS_DIFLAG_NODUMP | XFS_DIFLAG_RTINHERIT | \ + XFS_DIFLAG_PROJINHERIT | XFS_DIFLAG_NOSYMLINKS | XFS_DIFLAG_EXTSIZE | \ + XFS_DIFLAG_EXTSZINHERIT | XFS_DIFLAG_NODEFRAG | XFS_DIFLAG_FILESTREAM) + +/* + * Values for di_flags2 These start by being exposed to userspace in the upper + * 16 bits of the XFS_XFLAG_s range. + */ +/* use DAX for this inode */ +#define XFS_DIFLAG2_DAX_BIT 0 + +/* file's blocks may be shared */ +#define XFS_DIFLAG2_REFLINK_BIT 1 + +/* copy on write extent size hint */ +#define XFS_DIFLAG2_COWEXTSIZE_BIT 2 + +/* big timestamps */ +#define XFS_DIFLAG2_BIGTIME_BIT 3 + +/* large extent counters */ +#define XFS_DIFLAG2_NREXT64_BIT 4 + +/* + * The inode contains filesystem metadata and can be found through the metadata + * directory tree. Metadata inodes must satisfy the following constraints: + * + * - V5 filesystem (and ftype) are enabled; + * - The only valid modes are regular files and directories; + * - The access bits must be zero; + * - DMAPI event and state masks are zero; + * - The user and group IDs must be zero; + * - The project ID can be used as a u32 annotation; + * - The immutable, sync, noatime, nodump, nodefrag flags must be set. + * - The dax flag must not be set. + * - Directories must have nosymlinks set. + * + * These requirements are chosen defensively to minimize the ability of + * userspace to read or modify the contents, should a metadata file ever + * escape to userspace. + * + * There are further constraints on the directory tree itself: + * + * - Metadata inodes must never be resolvable through the root directory; + * - They must never be accessed by userspace; + * - Metadata directory entries must have correct ftype. + * + * Superblock-rooted metadata files must have the METADATA iflag set even + * though they do not have a parent directory. + */ +#define XFS_DIFLAG2_METADATA_BIT 5 + +#define XFS_DIFLAG2_DAX (1ULL << XFS_DIFLAG2_DAX_BIT) +#define XFS_DIFLAG2_REFLINK (1ULL << XFS_DIFLAG2_REFLINK_BIT) +#define XFS_DIFLAG2_COWEXTSIZE (1ULL << XFS_DIFLAG2_COWEXTSIZE_BIT) +#define XFS_DIFLAG2_BIGTIME (1ULL << XFS_DIFLAG2_BIGTIME_BIT) +#define XFS_DIFLAG2_NREXT64 (1ULL << XFS_DIFLAG2_NREXT64_BIT) +#define XFS_DIFLAG2_METADATA (1ULL << XFS_DIFLAG2_METADATA_BIT) + +#define XFS_DIFLAG2_ANY \ + (XFS_DIFLAG2_DAX | XFS_DIFLAG2_REFLINK | XFS_DIFLAG2_COWEXTSIZE | \ + XFS_DIFLAG2_BIGTIME | XFS_DIFLAG2_NREXT64 | XFS_DIFLAG2_METADATA) + +static inline bool xfs_dinode_has_bigtime(const struct xfs_dinode *dip) +{ + return dip->di_version >= 3 && + (dip->di_flags2 & cpu_to_be64(XFS_DIFLAG2_BIGTIME)); +} + +static inline bool xfs_dinode_has_large_extent_counts( + const struct xfs_dinode *dip) +{ + return dip->di_version >= 3 && + (dip->di_flags2 & cpu_to_be64(XFS_DIFLAG2_NREXT64)); +} + +static inline bool xfs_dinode_is_metadir(const struct xfs_dinode *dip) +{ + return dip->di_version >= 3 && + (dip->di_flags2 & cpu_to_be64(XFS_DIFLAG2_METADATA)); +} + +/* + * Inode number format: + * low inopblog bits - offset in block + * next agblklog bits - block number in ag + * next agno_log bits - ag number + * high agno_log-agblklog-inopblog bits - 0 + */ +#define XFS_INO_MASK(k) (uint32_t)((1ULL << (k)) - 1) +#define XFS_INO_OFFSET_BITS(mp) (mp)->m_sb.sb_inopblog +#define XFS_INO_AGBNO_BITS(mp) (mp)->m_sb.sb_agblklog +#define XFS_INO_AGINO_BITS(mp) ((mp)->m_ino_geo.agino_log) +#define XFS_INO_AGNO_BITS(mp) (mp)->m_agno_log +#define XFS_INO_BITS(mp) \ + XFS_INO_AGNO_BITS(mp) + XFS_INO_AGINO_BITS(mp) +#define XFS_INO_TO_AGNO(mp,i) \ + ((xfs_agnumber_t)((i) >> XFS_INO_AGINO_BITS(mp))) +#define XFS_INODE_TO_AGNO(ip) \ + XFS_INO_TO_AGNO((ip)->i_mount, I_INO(ip)) +#define XFS_INO_TO_AGINO(mp,i) \ + ((xfs_agino_t)(i) & XFS_INO_MASK(XFS_INO_AGINO_BITS(mp))) +#define XFS_INODE_TO_AGINO(ip) \ + XFS_INO_TO_AGINO((ip)->i_mount, I_INO(ip)) +#define XFS_INO_TO_AGBNO(mp,i) \ + (((xfs_agblock_t)(i) >> XFS_INO_OFFSET_BITS(mp)) & \ + XFS_INO_MASK(XFS_INO_AGBNO_BITS(mp))) +#define XFS_INO_TO_OFFSET(mp,i) \ + ((int)(i) & XFS_INO_MASK(XFS_INO_OFFSET_BITS(mp))) +#define XFS_INO_TO_FSB(mp,i) \ + XFS_AGB_TO_FSB(mp, XFS_INO_TO_AGNO(mp,i), XFS_INO_TO_AGBNO(mp,i)) +#define XFS_INODE_TO_FSB(ip) \ + XFS_INO_TO_FSB((ip)->i_mount, I_INO(ip)) +#define XFS_AGINO_TO_INO(mp,a,i) \ + (((xfs_ino_t)(a) << XFS_INO_AGINO_BITS(mp)) | (i)) +#define XFS_AGINO_TO_AGBNO(mp,i) ((i) >> XFS_INO_OFFSET_BITS(mp)) +#define XFS_AGINO_TO_OFFSET(mp,i) \ + ((i) & XFS_INO_MASK(XFS_INO_OFFSET_BITS(mp))) +#define XFS_OFFBNO_TO_AGINO(mp,b,o) \ + ((xfs_agino_t)(((b) << XFS_INO_OFFSET_BITS(mp)) | (o))) +#define XFS_FSB_TO_INO(mp, b) ((xfs_ino_t)((b) << XFS_INO_OFFSET_BITS(mp))) +#define XFS_AGB_TO_AGINO(mp, b) ((xfs_agino_t)((b) << XFS_INO_OFFSET_BITS(mp))) + +#define XFS_MAXINUMBER ((xfs_ino_t)((1ULL << 56) - 1ULL)) +#define XFS_MAXINUMBER_32 ((xfs_ino_t)((1ULL << 32) - 1ULL)) + +/* + * RealTime Device format definitions + */ + +/* Min and max rt extent sizes, specified in bytes */ +#define XFS_MAX_RTEXTSIZE (1024 * 1024 * 1024) /* 1GB */ +#define XFS_DFL_RTEXTSIZE (64 * 1024) /* 64kB */ +#define XFS_MIN_RTEXTSIZE (4 * 1024) /* 4kB */ + +/* + * RT bit manipulation macros. + */ +#define XFS_RTBITMAP_MAGIC 0x424D505A /* BMPZ */ +#define XFS_RTSUMMARY_MAGIC 0x53554D59 /* SUMY */ + +struct xfs_rtbuf_blkinfo { + __be32 rt_magic; /* validity check on block */ + __be32 rt_crc; /* CRC of block */ + __be64 rt_owner; /* inode that owns the block */ + __be64 rt_blkno; /* first block of the buffer */ + __be64 rt_lsn; /* sequence number of last write */ + uuid_t rt_uuid; /* filesystem we belong to */ +}; + +#define XFS_RTBUF_CRC_OFF \ + offsetof(struct xfs_rtbuf_blkinfo, rt_crc) + +/* + * Dquot and dquot block format definitions + */ +#define XFS_DQUOT_MAGIC 0x4451 /* 'DQ' */ +#define XFS_DQUOT_VERSION (uint8_t)0x01 /* latest version number */ + +#define XFS_DQTYPE_USER (1u << 0) /* user dquot record */ +#define XFS_DQTYPE_PROJ (1u << 1) /* project dquot record */ +#define XFS_DQTYPE_GROUP (1u << 2) /* group dquot record */ +#define XFS_DQTYPE_BIGTIME (1u << 7) /* large expiry timestamps */ + +/* bitmask to determine if this is a user/group/project dquot */ +#define XFS_DQTYPE_REC_MASK (XFS_DQTYPE_USER | \ + XFS_DQTYPE_PROJ | \ + XFS_DQTYPE_GROUP) + +#define XFS_DQTYPE_ANY (XFS_DQTYPE_REC_MASK | \ + XFS_DQTYPE_BIGTIME) + +/* + * XFS Quota Timers + * ================ + * + * Traditional quota grace period expiration timers are an unsigned 32-bit + * seconds counter; time zero is the Unix epoch, Jan 1 00:00:01 UTC 1970. + * Note that an expiration value of zero means that the quota limit has not + * been reached, and therefore no expiration has been set. Therefore, the + * ondisk min and max defined here can be used directly to constrain the incore + * quota expiration timestamps on a Unix system. + * + * When bigtime is enabled, we trade two bits of precision to expand the + * expiration timeout range to match that of big inode timestamps. The min and + * max recorded here are the on-disk limits, not a Unix timestamp. + * + * The grace period for each quota type is stored in the root dquot (id = 0) + * and is applied to a non-root dquot when it exceeds the soft or hard limits. + * The length of quota grace periods are unsigned 32-bit quantities measured in + * units of seconds. A value of zero means to use the default period. + */ + +/* + * Smallest possible ondisk quota expiration value with traditional timestamps. + * This corresponds exactly with the incore expiration Jan 1 00:00:01 UTC 1970. + */ +#define XFS_DQ_LEGACY_EXPIRY_MIN ((int64_t)1) + +/* + * Largest possible ondisk quota expiration value with traditional timestamps. + * This corresponds exactly with the incore expiration Feb 7 06:28:15 UTC 2106. + */ +#define XFS_DQ_LEGACY_EXPIRY_MAX ((int64_t)U32_MAX) + +/* + * Smallest possible ondisk quota expiration value with bigtime timestamps. + * This corresponds (after conversion to a Unix timestamp) with the incore + * expiration of Jan 1 00:00:04 UTC 1970. + */ +#define XFS_DQ_BIGTIME_EXPIRY_MIN (XFS_DQ_LEGACY_EXPIRY_MIN) + +/* + * Largest supported ondisk quota expiration value with bigtime timestamps. + * This corresponds (after conversion to a Unix timestamp) with an incore + * expiration of Jul 2 20:20:24 UTC 2486. + * + * The ondisk field supports values up to -1U, which corresponds to an incore + * expiration in 2514. This is beyond the maximum the bigtime inode timestamp, + * so we cap the maximum bigtime quota expiration to the max inode timestamp. + */ +#define XFS_DQ_BIGTIME_EXPIRY_MAX ((int64_t)4074815106U) + +/* + * The following conversion factors assist in converting a quota expiration + * timestamp between the incore and ondisk formats. + */ +#define XFS_DQ_BIGTIME_SHIFT (2) +#define XFS_DQ_BIGTIME_SLACK ((int64_t)(1ULL << XFS_DQ_BIGTIME_SHIFT) - 1) + +/* Convert an incore quota expiration timestamp to an ondisk bigtime value. */ +static inline uint32_t xfs_dq_unix_to_bigtime(time64_t unix_seconds) +{ + /* + * Round the expiration timestamp up to the nearest bigtime timestamp + * that we can store, to give users the most time to fix problems. + */ + return ((uint64_t)unix_seconds + XFS_DQ_BIGTIME_SLACK) >> + XFS_DQ_BIGTIME_SHIFT; +} + +/* Convert an ondisk bigtime quota expiration value to an incore timestamp. */ +static inline time64_t xfs_dq_bigtime_to_unix(uint32_t ondisk_seconds) +{ + return (time64_t)ondisk_seconds << XFS_DQ_BIGTIME_SHIFT; +} + +/* + * Default quota grace periods, ranging from zero (use the compiled defaults) + * to ~136 years. These are applied to a non-root dquot that has exceeded + * either limit. + */ +#define XFS_DQ_GRACE_MIN ((int64_t)0) +#define XFS_DQ_GRACE_MAX ((int64_t)U32_MAX) + +/* Maximum id value for a quota record */ +#define XFS_DQ_ID_MAX (U32_MAX) + +/* + * This is the main portion of the on-disk representation of quota information + * for a user. We pad this with some more expansion room to construct the on + * disk structure. + */ +struct xfs_disk_dquot { + __be16 d_magic; /* dquot magic = XFS_DQUOT_MAGIC */ + __u8 d_version; /* dquot version */ + __u8 d_type; /* XFS_DQTYPE_USER/PROJ/GROUP */ + __be32 d_id; /* user,project,group id */ + __be64 d_blk_hardlimit;/* absolute limit on disk blks */ + __be64 d_blk_softlimit;/* preferred limit on disk blks */ + __be64 d_ino_hardlimit;/* maximum # allocated inodes */ + __be64 d_ino_softlimit;/* preferred inode limit */ + __be64 d_bcount; /* disk blocks owned by the user */ + __be64 d_icount; /* inodes owned by the user */ + __be32 d_itimer; /* zero if within inode limits if not, + this is when we refuse service */ + __be32 d_btimer; /* similar to above; for disk blocks */ + __be16 d_iwarns; /* warnings issued wrt num inodes */ + __be16 d_bwarns; /* warnings issued wrt disk blocks */ + __be32 d_pad0; /* 64 bit align */ + __be64 d_rtb_hardlimit;/* absolute limit on realtime blks */ + __be64 d_rtb_softlimit;/* preferred limit on RT disk blks */ + __be64 d_rtbcount; /* realtime blocks owned */ + __be32 d_rtbtimer; /* similar to above; for RT disk blocks */ + __be16 d_rtbwarns; /* warnings issued wrt RT disk blocks */ + __be16 d_pad; +}; + +/* + * This is what goes on disk. This is separated from the xfs_disk_dquot because + * carrying the unnecessary padding would be a waste of memory. + */ +struct xfs_dqblk { + struct xfs_disk_dquot dd_diskdq; /* portion living incore as well */ + char dd_fill[4];/* filling for posterity */ + + /* + * These two are only present on filesystems with the CRC bits set. + */ + __be32 dd_crc; /* checksum */ + __be64 dd_lsn; /* last modification in log */ + uuid_t dd_uuid; /* location information */ +}; + +#define XFS_DQUOT_CRC_OFF offsetof(struct xfs_dqblk, dd_crc) + +/* + * This defines the unit of allocation of dquots. + * + * Currently, it is just one file system block, and a 4K blk contains 30 + * (136 * 30 = 4080) dquots. It's probably not worth trying to make + * this more dynamic. + * + * However, if this number is changed, we have to make sure that we don't + * implicitly assume that we do allocations in chunks of a single filesystem + * block in the dquot/xqm code. + * + * This is part of the ondisk format because the structure size is not a power + * of two, which leaves slack at the end of the disk block. + */ +#define XFS_DQUOT_CLUSTER_SIZE_FSB (xfs_filblks_t)1 + +/* + * Remote symlink format and access functions. + */ +#define XFS_SYMLINK_MAGIC 0x58534c4d /* XSLM */ + +struct xfs_dsymlink_hdr { + __be32 sl_magic; + __be32 sl_offset; + __be32 sl_bytes; + __be32 sl_crc; + uuid_t sl_uuid; + __be64 sl_owner; + __be64 sl_blkno; + __be64 sl_lsn; +}; + +#define XFS_SYMLINK_CRC_OFF offsetof(struct xfs_dsymlink_hdr, sl_crc) + +#define XFS_SYMLINK_MAXLEN 1024 +/* + * The maximum pathlen is 1024 bytes. Since the minimum file system + * blocksize is 512 bytes, we can get a max of 3 extents back from + * bmapi when crc headers are taken into account. + */ +#define XFS_SYMLINK_MAPS 3 + +#define XFS_SYMLINK_BUF_SPACE(mp, bufsize) \ + ((bufsize) - (xfs_has_crc((mp)) ? \ + sizeof(struct xfs_dsymlink_hdr) : 0)) + + +/* + * Allocation Btree format definitions + * + * There are two on-disk btrees, one sorted by blockno and one sorted + * by blockcount and blockno. All blocks look the same to make the code + * simpler; if we have time later, we'll make the optimizations. + */ +#define XFS_ABTB_MAGIC 0x41425442 /* 'ABTB' for bno tree */ +#define XFS_ABTB_CRC_MAGIC 0x41423342 /* 'AB3B' */ +#define XFS_ABTC_MAGIC 0x41425443 /* 'ABTC' for cnt tree */ +#define XFS_ABTC_CRC_MAGIC 0x41423343 /* 'AB3C' */ + +/* + * Data record/key structure + */ +typedef struct xfs_alloc_rec { + __be32 ar_startblock; /* starting block number */ + __be32 ar_blockcount; /* count of free blocks */ +} xfs_alloc_rec_t, xfs_alloc_key_t; + +typedef struct xfs_alloc_rec_incore { + xfs_agblock_t ar_startblock; /* starting block number */ + xfs_extlen_t ar_blockcount; /* count of free blocks */ +} xfs_alloc_rec_incore_t; + +/* btree pointer type */ +typedef __be32 xfs_alloc_ptr_t; + +/* + * Block numbers in the AG: + * SB is sector 0, AGF is sector 1, AGI is sector 2, AGFL is sector 3. + */ +#define XFS_BNO_BLOCK(mp) ((xfs_agblock_t)(XFS_AGFL_BLOCK(mp) + 1)) +#define XFS_CNT_BLOCK(mp) ((xfs_agblock_t)(XFS_BNO_BLOCK(mp) + 1)) + + +/* + * Inode Allocation Btree format definitions + * + * There is a btree for the inode map per allocation group. + */ +#define XFS_IBT_MAGIC 0x49414254 /* 'IABT' */ +#define XFS_IBT_CRC_MAGIC 0x49414233 /* 'IAB3' */ +#define XFS_FIBT_MAGIC 0x46494254 /* 'FIBT' */ +#define XFS_FIBT_CRC_MAGIC 0x46494233 /* 'FIB3' */ + +typedef uint64_t xfs_inofree_t; +#define XFS_INODES_PER_CHUNK (NBBY * sizeof(xfs_inofree_t)) +#define XFS_INODES_PER_CHUNK_LOG (XFS_NBBYLOG + 3) +#define XFS_INOBT_ALL_FREE ((xfs_inofree_t)-1) +#define XFS_INOBT_MASK(i) ((xfs_inofree_t)1 << (i)) + +#define XFS_INOBT_HOLEMASK_FULL 0 /* holemask for full chunk */ +#define XFS_INOBT_HOLEMASK_BITS (NBBY * sizeof(uint16_t)) +#define XFS_INODES_PER_HOLEMASK_BIT \ + (XFS_INODES_PER_CHUNK / (NBBY * sizeof(uint16_t))) + +static inline xfs_inofree_t xfs_inobt_maskn(int i, int n) +{ + return ((n >= XFS_INODES_PER_CHUNK ? 0 : XFS_INOBT_MASK(n)) - 1) << i; +} + +/* + * The on-disk inode record structure has two formats. The original "full" + * format uses a 4-byte freecount. The "sparse" format uses a 1-byte freecount + * and replaces the 3 high-order freecount bytes wth the holemask and inode + * count. + * + * The holemask of the sparse record format allows an inode chunk to have holes + * that refer to blocks not owned by the inode record. This facilitates inode + * allocation in the event of severe free space fragmentation. + */ +typedef struct xfs_inobt_rec { + __be32 ir_startino; /* starting inode number */ + union { + struct { + __be32 ir_freecount; /* count of free inodes */ + } f; + struct { + __be16 ir_holemask;/* hole mask for sparse chunks */ + __u8 ir_count; /* total inode count */ + __u8 ir_freecount; /* count of free inodes */ + } sp; + } ir_u; + __be64 ir_free; /* free inode mask */ +} xfs_inobt_rec_t; + +typedef struct xfs_inobt_rec_incore { + xfs_agino_t ir_startino; /* starting inode number */ + uint16_t ir_holemask; /* hole mask for sparse chunks */ + uint8_t ir_count; /* total inode count */ + uint8_t ir_freecount; /* count of free inodes (set bits) */ + xfs_inofree_t ir_free; /* free inode mask */ +} xfs_inobt_rec_incore_t; + +static inline bool xfs_inobt_issparse(uint16_t holemask) +{ + /* non-zero holemask represents a sparse rec. */ + return holemask; +} + +/* + * Key structure + */ +typedef struct xfs_inobt_key { + __be32 ir_startino; /* starting inode number */ +} xfs_inobt_key_t; + +/* btree pointer type */ +typedef __be32 xfs_inobt_ptr_t; + +/* + * block numbers in the AG. + */ +#define XFS_IBT_BLOCK(mp) ((xfs_agblock_t)(XFS_CNT_BLOCK(mp) + 1)) +#define XFS_FIBT_BLOCK(mp) ((xfs_agblock_t)(XFS_IBT_BLOCK(mp) + 1)) + +/* + * Reverse mapping btree format definitions + * + * There is a btree for the reverse map per allocation group + */ +#define XFS_RMAP_CRC_MAGIC 0x524d4233 /* 'RMB3' */ + +/* + * Ownership info for an extent. This is used to create reverse-mapping + * entries. + */ +#define XFS_OWNER_INFO_ATTR_FORK (1 << 0) +#define XFS_OWNER_INFO_BMBT_BLOCK (1 << 1) +struct xfs_owner_info { + uint64_t oi_owner; + xfs_fileoff_t oi_offset; + unsigned int oi_flags; +}; + +/* + * Special owner types. + * + * Seeing as we only support up to 8EB, we have the upper bit of the owner field + * to tell us we have a special owner value. We use these for static metadata + * allocated at mkfs/growfs time, as well as for freespace management metadata. + */ +#define XFS_RMAP_OWN_NULL (-1ULL) /* No owner, for growfs */ +#define XFS_RMAP_OWN_UNKNOWN (-2ULL) /* Unknown owner, for EFI recovery */ +#define XFS_RMAP_OWN_FS (-3ULL) /* static fs metadata */ +#define XFS_RMAP_OWN_LOG (-4ULL) /* static fs metadata */ +#define XFS_RMAP_OWN_AG (-5ULL) /* AG freespace btree blocks */ +#define XFS_RMAP_OWN_INOBT (-6ULL) /* Inode btree blocks */ +#define XFS_RMAP_OWN_INODES (-7ULL) /* Inode chunk */ +#define XFS_RMAP_OWN_REFC (-8ULL) /* refcount tree */ +#define XFS_RMAP_OWN_COW (-9ULL) /* cow allocations */ +#define XFS_RMAP_OWN_MIN (-10ULL) /* guard */ + +#define XFS_RMAP_NON_INODE_OWNER(owner) (!!((owner) & (1ULL << 63))) + +/* + * Data record structure + */ +struct xfs_rmap_rec { + __be32 rm_startblock; /* extent start block */ + __be32 rm_blockcount; /* extent length */ + __be64 rm_owner; /* extent owner */ + __be64 rm_offset; /* offset within the owner */ +}; + +/* + * rmap btree record + * rm_offset:63 is the attribute fork flag + * rm_offset:62 is the bmbt block flag + * rm_offset:61 is the unwritten extent flag (same as l0:63 in bmbt) + * rm_offset:54-60 aren't used and should be zero + * rm_offset:0-53 is the block offset within the inode + */ +#define XFS_RMAP_OFF_ATTR_FORK ((uint64_t)1ULL << 63) +#define XFS_RMAP_OFF_BMBT_BLOCK ((uint64_t)1ULL << 62) +#define XFS_RMAP_OFF_UNWRITTEN ((uint64_t)1ULL << 61) + +#define XFS_RMAP_LEN_MAX ((uint32_t)~0U) +#define XFS_RMAP_OFF_FLAGS (XFS_RMAP_OFF_ATTR_FORK | \ + XFS_RMAP_OFF_BMBT_BLOCK | \ + XFS_RMAP_OFF_UNWRITTEN) +#define XFS_RMAP_OFF_MASK ((uint64_t)0x3FFFFFFFFFFFFFULL) + +#define XFS_RMAP_OFF(off) ((off) & XFS_RMAP_OFF_MASK) + +#define XFS_RMAP_IS_BMBT_BLOCK(off) (!!((off) & XFS_RMAP_OFF_BMBT_BLOCK)) +#define XFS_RMAP_IS_ATTR_FORK(off) (!!((off) & XFS_RMAP_OFF_ATTR_FORK)) +#define XFS_RMAP_IS_UNWRITTEN(len) (!!((off) & XFS_RMAP_OFF_UNWRITTEN)) + +#define RMAPBT_STARTBLOCK_BITLEN 32 +#define RMAPBT_BLOCKCOUNT_BITLEN 32 +#define RMAPBT_OWNER_BITLEN 64 +#define RMAPBT_ATTRFLAG_BITLEN 1 +#define RMAPBT_BMBTFLAG_BITLEN 1 +#define RMAPBT_EXNTFLAG_BITLEN 1 +#define RMAPBT_UNUSED_OFFSET_BITLEN 7 +#define RMAPBT_OFFSET_BITLEN 54 + +/* + * Key structure + * + * We don't use the length for lookups + */ +struct xfs_rmap_key { + __be32 rm_startblock; /* extent start block */ + __be64 rm_owner; /* extent owner */ + __be64 rm_offset; /* offset within the owner */ +} __attribute__((packed)); + +/* btree pointer type */ +typedef __be32 xfs_rmap_ptr_t; + +#define XFS_RMAP_BLOCK(mp) \ + (xfs_has_finobt(((mp))) ? \ + XFS_FIBT_BLOCK(mp) + 1 : \ + XFS_IBT_BLOCK(mp) + 1) + +/* + * Realtime Reverse mapping btree format definitions + * + * This is a btree for reverse mapping records for realtime volumes + */ +#define XFS_RTRMAP_CRC_MAGIC 0x4d415052 /* 'MAPR' */ + +/* + * rtrmap root header, on-disk form only. + */ +struct xfs_rtrmap_root { + __be16 bb_level; /* 0 is a leaf */ + __be16 bb_numrecs; /* current # of data records */ +}; + +/* inode-based btree pointer type */ +typedef __be64 xfs_rtrmap_ptr_t; + +/* + * Reference Count Btree format definitions + * + */ +#define XFS_REFC_CRC_MAGIC 0x52334643 /* 'R3FC' */ + +unsigned int xfs_refc_block(struct xfs_mount *mp); + +/* + * Data record/key structure + * + * Each record associates a range of physical blocks (starting at + * rc_startblock and ending rc_blockcount blocks later) with a reference + * count (rc_refcount). Extents that are being used to stage a copy on + * write (CoW) operation are recorded in the refcount btree with a + * refcount of 1. All other records must have a refcount > 1 and must + * track an extent mapped only by file data forks. + * + * Extents with a single owner (attributes, metadata, non-shared file + * data) are not tracked here. Free space is also not tracked here. + * This is consistent with pre-reflink XFS. + */ + +/* + * Extents that are being used to stage a copy on write are stored + * in the refcount btree with a refcount of 1 and the upper bit set + * on the startblock. This speeds up mount time deletion of stale + * staging extents because they're all at the right side of the tree. + */ +#define XFS_REFC_COWFLAG (1U << 31) +#define REFCNTBT_COWFLAG_BITLEN 1 +#define REFCNTBT_AGBLOCK_BITLEN 31 + +struct xfs_refcount_rec { + __be32 rc_startblock; /* starting block number */ + __be32 rc_blockcount; /* count of blocks */ + __be32 rc_refcount; /* number of inodes linked here */ +}; + +struct xfs_refcount_key { + __be32 rc_startblock; /* starting block number */ +}; + +#define XFS_REFC_REFCOUNT_MAX ((xfs_nlink_t)~0U) +#define XFS_REFC_LEN_MAX ((xfs_extlen_t)~0U) + +/* btree pointer type */ +typedef __be32 xfs_refcount_ptr_t; + +/* + * Realtime Reference Count btree format definitions + * + * This is a btree for reference count records for realtime volumes + */ +#define XFS_RTREFC_CRC_MAGIC 0x52434e54 /* 'RCNT' */ + +/* + * rt refcount root header, on-disk form only. + */ +struct xfs_rtrefcount_root { + __be16 bb_level; /* 0 is a leaf */ + __be16 bb_numrecs; /* current # of data records */ +}; + +/* inode-rooted btree pointer type */ +typedef __be64 xfs_rtrefcount_ptr_t; + +/* + * BMAP Btree format definitions + * + * This includes both the root block definition that sits inside an inode fork + * and the record/pointer formats for the leaf/node in the blocks. + */ +#define XFS_BMAP_MAGIC 0x424d4150 /* 'BMAP' */ +#define XFS_BMAP_CRC_MAGIC 0x424d4133 /* 'BMA3' */ + +/* + * Bmap root header, on-disk form only. + */ +typedef struct xfs_bmdr_block { + __be16 bb_level; /* 0 is a leaf */ + __be16 bb_numrecs; /* current # of data records */ +} xfs_bmdr_block_t; + +/* + * Bmap btree record and extent descriptor. + * l0:63 is an extent flag (value 1 indicates non-normal). + * l0:9-62 are startoff. + * l0:0-8 and l1:21-63 are startblock. + * l1:0-20 are blockcount. + */ +#define BMBT_EXNTFLAG_BITLEN 1 +#define BMBT_STARTOFF_BITLEN 54 +#define BMBT_STARTBLOCK_BITLEN 52 +#define BMBT_BLOCKCOUNT_BITLEN 21 + +#define BMBT_STARTOFF_MASK ((1ULL << BMBT_STARTOFF_BITLEN) - 1) +#define BMBT_BLOCKCOUNT_MASK ((1ULL << BMBT_BLOCKCOUNT_BITLEN) - 1) + +#define XFS_MAX_BMBT_EXTLEN ((xfs_extlen_t)(BMBT_BLOCKCOUNT_MASK)) + +/* + * bmbt records have a file offset (block) field that is 54 bits wide, so this + * is the largest xfs_fileoff_t that we ever expect to see. + */ +#define XFS_MAX_FILEOFF (BMBT_STARTOFF_MASK + BMBT_BLOCKCOUNT_MASK) + +typedef struct xfs_bmbt_rec { + __be64 l0, l1; +} xfs_bmbt_rec_t; + +typedef uint64_t xfs_bmbt_rec_base_t; /* use this for casts */ +typedef xfs_bmbt_rec_t xfs_bmdr_rec_t; + +/* + * Values and macros for delayed-allocation startblock fields. + */ +#define STARTBLOCKVALBITS 17 +#define STARTBLOCKMASKBITS (15 + 20) +#define STARTBLOCKMASK \ + (((((xfs_fsblock_t)1) << STARTBLOCKMASKBITS) - 1) << STARTBLOCKVALBITS) + +static inline int isnullstartblock(xfs_fsblock_t x) +{ + return ((x) & STARTBLOCKMASK) == STARTBLOCKMASK; +} + +static inline xfs_fsblock_t nullstartblock(int k) +{ + ASSERT(k < (1 << STARTBLOCKVALBITS)); + return STARTBLOCKMASK | (k); +} + +static inline xfs_filblks_t startblockval(xfs_fsblock_t x) +{ + return (xfs_filblks_t)((x) & ~STARTBLOCKMASK); +} + +/* + * Key structure for non-leaf levels of the tree. + */ +typedef struct xfs_bmbt_key { + __be64 br_startoff; /* starting file offset */ +} xfs_bmbt_key_t, xfs_bmdr_key_t; + +/* btree pointer type */ +typedef __be64 xfs_bmbt_ptr_t, xfs_bmdr_ptr_t; + + +/* + * Generic Btree block format definitions + * + * This is a combination of the actual format used on disk for short and long + * format btrees. The first three fields are shared by both format, but the + * pointers are different and should be used with care. + * + * To get the size of the actual short or long form headers please use the size + * macros below. Never use sizeof(xfs_btree_block). + * + * The blkno, crc, lsn, owner and uuid fields are only available in filesystems + * with the crc feature bit, and all accesses to them must be conditional on + * that flag. + */ +/* short form block header */ +struct xfs_btree_block_shdr { + __be32 bb_leftsib; + __be32 bb_rightsib; + + __be64 bb_blkno; + __be64 bb_lsn; + uuid_t bb_uuid; + __be32 bb_owner; + __le32 bb_crc; +}; + +/* long form block header */ +struct xfs_btree_block_lhdr { + __be64 bb_leftsib; + __be64 bb_rightsib; + + __be64 bb_blkno; + __be64 bb_lsn; + uuid_t bb_uuid; + __be64 bb_owner; + __le32 bb_crc; + __be32 bb_pad; /* padding for alignment */ +}; + +struct xfs_btree_block { + __be32 bb_magic; /* magic number for block type */ + __be16 bb_level; /* 0 is a leaf */ + __be16 bb_numrecs; /* current # of data records */ + union { + struct xfs_btree_block_shdr s; + struct xfs_btree_block_lhdr l; + } bb_u; /* rest */ +}; + +/* size of a short form block */ +#define XFS_BTREE_SBLOCK_LEN \ + (offsetof(struct xfs_btree_block, bb_u) + \ + offsetof(struct xfs_btree_block_shdr, bb_blkno)) +/* size of a long form block */ +#define XFS_BTREE_LBLOCK_LEN \ + (offsetof(struct xfs_btree_block, bb_u) + \ + offsetof(struct xfs_btree_block_lhdr, bb_blkno)) + +/* sizes of CRC enabled btree blocks */ +#define XFS_BTREE_SBLOCK_CRC_LEN \ + (offsetof(struct xfs_btree_block, bb_u) + \ + sizeof(struct xfs_btree_block_shdr)) +#define XFS_BTREE_LBLOCK_CRC_LEN \ + (offsetof(struct xfs_btree_block, bb_u) + \ + sizeof(struct xfs_btree_block_lhdr)) + +#define XFS_BTREE_SBLOCK_CRC_OFF \ + offsetof(struct xfs_btree_block, bb_u.s.bb_crc) +#define XFS_BTREE_LBLOCK_CRC_OFF \ + offsetof(struct xfs_btree_block, bb_u.l.bb_crc) + +/* + * On-disk XFS access control list structure. + */ +struct xfs_acl_entry { + __be32 ae_tag; + __be32 ae_id; + __be16 ae_perm; + __be16 ae_pad; /* fill the implicit hole in the structure */ +}; + +struct xfs_acl { + __be32 acl_cnt; + struct xfs_acl_entry acl_entry[]; +}; + +/* + * The number of ACL entries allowed is defined by the on-disk format. + * For v4 superblocks, that is limited to 25 entries. For v5 superblocks, it is + * limited only by the maximum size of the xattr that stores the information. + */ +#define XFS_ACL_MAX_ENTRIES(mp) \ + (xfs_has_crc(mp) \ + ? (XFS_XATTR_SIZE_MAX - sizeof(struct xfs_acl)) / \ + sizeof(struct xfs_acl_entry) \ + : 25) + +#define XFS_ACL_SIZE(cnt) \ + (sizeof(struct xfs_acl) + \ + sizeof(struct xfs_acl_entry) * cnt) + +#define XFS_ACL_MAX_SIZE(mp) \ + XFS_ACL_SIZE(XFS_ACL_MAX_ENTRIES((mp))) + + +/* On-disk XFS extended attribute names */ +#define SGI_ACL_FILE "SGI_ACL_FILE" +#define SGI_ACL_DEFAULT "SGI_ACL_DEFAULT" +#define SGI_ACL_FILE_SIZE (sizeof(SGI_ACL_FILE)-1) +#define SGI_ACL_DEFAULT_SIZE (sizeof(SGI_ACL_DEFAULT)-1) + +#endif /* __XFS_FORMAT_H__ */ diff --git a/fs/xfs/libxfs/xfs_fs.h b/fs/xfs/libxfs/xfs_fs.h new file mode 100644 index 000000000..185f09f32 --- /dev/null +++ b/fs/xfs/libxfs/xfs_fs.h @@ -0,0 +1,1301 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +/* + * Copyright (c) 1995-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_FS_H__ +#define __XFS_FS_H__ + +/* + * SGI's XFS filesystem's major stuff (constants, structures) + * NOTE: This file must be compile-able with C++ compilers. + */ + +/* + * Direct I/O attribute record used with XFS_IOC_DIOINFO + * d_miniosz is the min xfer size, xfer size multiple and file seek offset + * alignment. + */ +#ifndef HAVE_DIOATTR +struct dioattr { + __u32 d_mem; /* data buffer memory alignment */ + __u32 d_miniosz; /* min xfer size */ + __u32 d_maxiosz; /* max xfer size */ +}; +#endif + +/* + * Structure for XFS_IOC_GETBMAP. + * On input, fill in bmv_offset and bmv_length of the first structure + * to indicate the area of interest in the file, and bmv_entries with + * the number of array elements given back. The first structure is + * updated on return to give the offset and length for the next call. + */ +#ifndef HAVE_GETBMAP +struct getbmap { + __s64 bmv_offset; /* file offset of segment in blocks */ + __s64 bmv_block; /* starting block (64-bit daddr_t) */ + __s64 bmv_length; /* length of segment, blocks */ + __s32 bmv_count; /* # of entries in array incl. 1st */ + __s32 bmv_entries; /* # of entries filled in (output) */ +}; +#endif + +/* + * Structure for XFS_IOC_GETBMAPX. Fields bmv_offset through bmv_entries + * are used exactly as in the getbmap structure. The getbmapx structure + * has additional bmv_iflags and bmv_oflags fields. The bmv_iflags field + * is only used for the first structure. It contains input flags + * specifying XFS_IOC_GETBMAPX actions. The bmv_oflags field is filled + * in by the XFS_IOC_GETBMAPX command for each returned structure after + * the first. + */ +#ifndef HAVE_GETBMAPX +struct getbmapx { + __s64 bmv_offset; /* file offset of segment in blocks */ + __s64 bmv_block; /* starting block (64-bit daddr_t) */ + __s64 bmv_length; /* length of segment, blocks */ + __s32 bmv_count; /* # of entries in array incl. 1st */ + __s32 bmv_entries; /* # of entries filled in (output). */ + __s32 bmv_iflags; /* input flags (1st structure) */ + __s32 bmv_oflags; /* output flags (after 1st structure)*/ + __s32 bmv_unused1; /* future use */ + __s32 bmv_unused2; /* future use */ +}; +#endif + +/* bmv_iflags values - set by XFS_IOC_GETBMAPX caller. */ +#define BMV_IF_ATTRFORK 0x1 /* return attr fork rather than data */ +#define BMV_IF_NO_DMAPI_READ 0x2 /* Deprecated */ +#define BMV_IF_PREALLOC 0x4 /* rtn status BMV_OF_PREALLOC if req */ +#define BMV_IF_DELALLOC 0x8 /* rtn status BMV_OF_DELALLOC if req */ +#define BMV_IF_NO_HOLES 0x10 /* Do not return holes */ +#define BMV_IF_COWFORK 0x20 /* return CoW fork rather than data */ +#define BMV_IF_VALID \ + (BMV_IF_ATTRFORK|BMV_IF_NO_DMAPI_READ|BMV_IF_PREALLOC| \ + BMV_IF_DELALLOC|BMV_IF_NO_HOLES|BMV_IF_COWFORK) + +/* bmv_oflags values - returned for each non-header segment */ +#define BMV_OF_PREALLOC 0x1 /* segment = unwritten pre-allocation */ +#define BMV_OF_DELALLOC 0x2 /* segment = delayed allocation */ +#define BMV_OF_LAST 0x4 /* segment is the last in the file */ +#define BMV_OF_SHARED 0x8 /* segment shared with another file */ + +/* fmr_owner special values for FS_IOC_GETFSMAP */ +#define XFS_FMR_OWN_FREE FMR_OWN_FREE /* free space */ +#define XFS_FMR_OWN_UNKNOWN FMR_OWN_UNKNOWN /* unknown owner */ +#define XFS_FMR_OWN_FS FMR_OWNER('X', 1) /* static fs metadata */ +#define XFS_FMR_OWN_LOG FMR_OWNER('X', 2) /* journalling log */ +#define XFS_FMR_OWN_AG FMR_OWNER('X', 3) /* per-AG metadata */ +#define XFS_FMR_OWN_INOBT FMR_OWNER('X', 4) /* inode btree blocks */ +#define XFS_FMR_OWN_INODES FMR_OWNER('X', 5) /* inodes */ +#define XFS_FMR_OWN_REFC FMR_OWNER('X', 6) /* refcount tree */ +#define XFS_FMR_OWN_COW FMR_OWNER('X', 7) /* cow staging */ +#define XFS_FMR_OWN_DEFECTIVE FMR_OWNER('X', 8) /* bad blocks */ + +/* + * File segment locking set data type for 64 bit access. + * Also used for all the RESV/FREE interfaces. + */ +typedef struct xfs_flock64 { + __s16 l_type; + __s16 l_whence; + __s64 l_start; + __s64 l_len; /* len == 0 means until end of file */ + __s32 l_sysid; + __u32 l_pid; + __s32 l_pad[4]; /* reserve area */ +} xfs_flock64_t; + +/* + * Output for XFS_IOC_FSGEOMETRY_V1 + */ +struct xfs_fsop_geom_v1 { + __u32 blocksize; /* filesystem (data) block size */ + __u32 rtextsize; /* realtime extent size */ + __u32 agblocks; /* fsblocks in an AG */ + __u32 agcount; /* number of allocation groups */ + __u32 logblocks; /* fsblocks in the log */ + __u32 sectsize; /* (data) sector size, bytes */ + __u32 inodesize; /* inode size in bytes */ + __u32 imaxpct; /* max allowed inode space(%) */ + __u64 datablocks; /* fsblocks in data subvolume */ + __u64 rtblocks; /* fsblocks in realtime subvol */ + __u64 rtextents; /* rt extents in realtime subvol*/ + __u64 logstart; /* starting fsblock of the log */ + unsigned char uuid[16]; /* unique id of the filesystem */ + __u32 sunit; /* stripe unit, fsblocks */ + __u32 swidth; /* stripe width, fsblocks */ + __s32 version; /* structure version */ + __u32 flags; /* superblock version flags */ + __u32 logsectsize; /* log sector size, bytes */ + __u32 rtsectsize; /* realtime sector size, bytes */ + __u32 dirblocksize; /* directory block size, bytes */ +}; + +/* + * Output for XFS_IOC_FSGEOMETRY_V4 + */ +struct xfs_fsop_geom_v4 { + __u32 blocksize; /* filesystem (data) block size */ + __u32 rtextsize; /* realtime extent size */ + __u32 agblocks; /* fsblocks in an AG */ + __u32 agcount; /* number of allocation groups */ + __u32 logblocks; /* fsblocks in the log */ + __u32 sectsize; /* (data) sector size, bytes */ + __u32 inodesize; /* inode size in bytes */ + __u32 imaxpct; /* max allowed inode space(%) */ + __u64 datablocks; /* fsblocks in data subvolume */ + __u64 rtblocks; /* fsblocks in realtime subvol */ + __u64 rtextents; /* rt extents in realtime subvol*/ + __u64 logstart; /* starting fsblock of the log */ + unsigned char uuid[16]; /* unique id of the filesystem */ + __u32 sunit; /* stripe unit, fsblocks */ + __u32 swidth; /* stripe width, fsblocks */ + __s32 version; /* structure version */ + __u32 flags; /* superblock version flags */ + __u32 logsectsize; /* log sector size, bytes */ + __u32 rtsectsize; /* realtime sector size, bytes */ + __u32 dirblocksize; /* directory block size, bytes */ + __u32 logsunit; /* log stripe unit, bytes */ +}; + +/* + * Output for XFS_IOC_FSGEOMETRY + */ +struct xfs_fsop_geom { + __u32 blocksize; /* filesystem (data) block size */ + __u32 rtextsize; /* realtime extent size */ + __u32 agblocks; /* fsblocks in an AG */ + __u32 agcount; /* number of allocation groups */ + __u32 logblocks; /* fsblocks in the log */ + __u32 sectsize; /* (data) sector size, bytes */ + __u32 inodesize; /* inode size in bytes */ + __u32 imaxpct; /* max allowed inode space(%) */ + __u64 datablocks; /* fsblocks in data subvolume */ + __u64 rtblocks; /* fsblocks in realtime subvol */ + __u64 rtextents; /* rt extents in realtime subvol*/ + __u64 logstart; /* starting fsblock of the log */ + unsigned char uuid[16]; /* unique id of the filesystem */ + __u32 sunit; /* stripe unit, fsblocks */ + __u32 swidth; /* stripe width, fsblocks */ + __s32 version; /* structure version */ + __u32 flags; /* superblock version flags */ + __u32 logsectsize; /* log sector size, bytes */ + __u32 rtsectsize; /* realtime sector size, bytes */ + __u32 dirblocksize; /* directory block size, bytes */ + __u32 logsunit; /* log stripe unit, bytes */ + uint32_t sick; /* o: unhealthy fs & rt metadata */ + uint32_t checked; /* o: checked fs & rt metadata */ + __u32 rgextents; /* rt extents in a realtime group */ + __u32 rgcount; /* number of realtime groups */ + __u64 rtstart; /* start of internal rt section */ + __u64 rtreserved; /* RT (zoned) reserved blocks */ + __u64 reserved[14]; /* reserved space */ +}; + +#define XFS_FSOP_GEOM_SICK_COUNTERS (1 << 0) /* summary counters */ +#define XFS_FSOP_GEOM_SICK_UQUOTA (1 << 1) /* user quota */ +#define XFS_FSOP_GEOM_SICK_GQUOTA (1 << 2) /* group quota */ +#define XFS_FSOP_GEOM_SICK_PQUOTA (1 << 3) /* project quota */ +#define XFS_FSOP_GEOM_SICK_RT_BITMAP (1 << 4) /* realtime bitmap */ +#define XFS_FSOP_GEOM_SICK_RT_SUMMARY (1 << 5) /* realtime summary */ +#define XFS_FSOP_GEOM_SICK_QUOTACHECK (1 << 6) /* quota counts */ +#define XFS_FSOP_GEOM_SICK_NLINKS (1 << 7) /* inode link counts */ +#define XFS_FSOP_GEOM_SICK_METADIR (1 << 8) /* metadata directory */ +#define XFS_FSOP_GEOM_SICK_METAPATH (1 << 9) /* metadir tree path */ + +/* Output for XFS_FS_COUNTS */ +typedef struct xfs_fsop_counts { + __u64 freedata; /* free data section blocks */ + __u64 freertx; /* free rt extents */ + __u64 freeino; /* free inodes */ + __u64 allocino; /* total allocated inodes */ +} xfs_fsop_counts_t; + +/* Input/Output for XFS_GET_RESBLKS and XFS_SET_RESBLKS */ +typedef struct xfs_fsop_resblks { + __u64 resblks; + __u64 resblks_avail; +} xfs_fsop_resblks_t; + +#define XFS_FSOP_GEOM_VERSION 0 +#define XFS_FSOP_GEOM_VERSION_V5 5 + +#define XFS_FSOP_GEOM_FLAGS_ATTR (1 << 0) /* attributes in use */ +#define XFS_FSOP_GEOM_FLAGS_NLINK (1 << 1) /* 32-bit nlink values */ +#define XFS_FSOP_GEOM_FLAGS_QUOTA (1 << 2) /* quotas enabled */ +#define XFS_FSOP_GEOM_FLAGS_IALIGN (1 << 3) /* inode alignment */ +#define XFS_FSOP_GEOM_FLAGS_DALIGN (1 << 4) /* large data alignment */ +#define XFS_FSOP_GEOM_FLAGS_SHARED (1 << 5) /* read-only shared */ +#define XFS_FSOP_GEOM_FLAGS_EXTFLG (1 << 6) /* special extent flag */ +#define XFS_FSOP_GEOM_FLAGS_DIRV2 (1 << 7) /* directory version 2 */ +#define XFS_FSOP_GEOM_FLAGS_LOGV2 (1 << 8) /* log format version 2 */ +#define XFS_FSOP_GEOM_FLAGS_SECTOR (1 << 9) /* sector sizes >1BB */ +#define XFS_FSOP_GEOM_FLAGS_ATTR2 (1 << 10) /* inline attributes rework */ +#define XFS_FSOP_GEOM_FLAGS_PROJID32 (1 << 11) /* 32-bit project IDs */ +#define XFS_FSOP_GEOM_FLAGS_DIRV2CI (1 << 12) /* ASCII only CI names */ + /* -- Do not use -- (1 << 13) SGI parent pointers */ +#define XFS_FSOP_GEOM_FLAGS_LAZYSB (1 << 14) /* lazy superblock counters */ +#define XFS_FSOP_GEOM_FLAGS_V5SB (1 << 15) /* version 5 superblock */ +#define XFS_FSOP_GEOM_FLAGS_FTYPE (1 << 16) /* inode directory types */ +#define XFS_FSOP_GEOM_FLAGS_FINOBT (1 << 17) /* free inode btree */ +#define XFS_FSOP_GEOM_FLAGS_SPINODES (1 << 18) /* sparse inode chunks */ +#define XFS_FSOP_GEOM_FLAGS_RMAPBT (1 << 19) /* reverse mapping btree */ +#define XFS_FSOP_GEOM_FLAGS_REFLINK (1 << 20) /* files can share blocks */ +#define XFS_FSOP_GEOM_FLAGS_BIGTIME (1 << 21) /* 64-bit nsec timestamps */ +#define XFS_FSOP_GEOM_FLAGS_INOBTCNT (1 << 22) /* inobt btree counter */ +#define XFS_FSOP_GEOM_FLAGS_NREXT64 (1 << 23) /* large extent counters */ +#define XFS_FSOP_GEOM_FLAGS_EXCHANGE_RANGE (1 << 24) /* exchange range */ +#define XFS_FSOP_GEOM_FLAGS_PARENT (1 << 25) /* linux parent pointers */ +#define XFS_FSOP_GEOM_FLAGS_METADIR (1 << 26) /* metadata directories */ +#define XFS_FSOP_GEOM_FLAGS_ZONED (1 << 27) /* zoned rt device */ + +/* + * Minimum and maximum sizes need for growth checks. + * + * Block counts are in units of filesystem blocks, not basic blocks. + */ +#define XFS_MIN_AG_BLOCKS 64 +#define XFS_MIN_LOG_BLOCKS 512ULL +#define XFS_MAX_LOG_BLOCKS (1024 * 1024ULL) +#define XFS_MIN_LOG_BYTES (10 * 1024 * 1024ULL) + +/* + * Limits on sb_agblocks/sb_agblklog -- mkfs won't format AGs smaller than + * 16MB or larger than 1TB. + */ +#define XFS_MIN_AG_BYTES (1ULL << 24) /* 16 MB */ +#define XFS_MAX_AG_BYTES (1ULL << 40) /* 1 TB */ +#define XFS_MAX_AG_BLOCKS (XFS_MAX_AG_BYTES / XFS_MIN_BLOCKSIZE) +#define XFS_MAX_CRC_AG_BLOCKS (XFS_MAX_AG_BYTES / XFS_MIN_CRC_BLOCKSIZE) + +#define XFS_MAX_AGNUMBER ((xfs_agnumber_t)(NULLAGNUMBER - 1)) + +/* keep the maximum size under 2^31 by a small amount */ +#define XFS_MAX_LOG_BYTES \ + ((2 * 1024 * 1024 * 1024ULL) - XFS_MIN_LOG_BYTES) + +/* Used for sanity checks on superblock */ +#define XFS_MAX_DBLOCKS(s) ((xfs_rfsblock_t)(s)->sb_agcount * (s)->sb_agblocks) +#define XFS_MIN_DBLOCKS(s) ((xfs_rfsblock_t)((s)->sb_agcount - 1) * \ + (s)->sb_agblocks + XFS_MIN_AG_BLOCKS) + +/* + * Output for XFS_IOC_AG_GEOMETRY + */ +struct xfs_ag_geometry { + uint32_t ag_number; /* i/o: AG number */ + uint32_t ag_length; /* o: length in blocks */ + uint32_t ag_freeblks; /* o: free space */ + uint32_t ag_icount; /* o: inodes allocated */ + uint32_t ag_ifree; /* o: inodes free */ + uint32_t ag_sick; /* o: sick things in ag */ + uint32_t ag_checked; /* o: checked metadata in ag */ + uint32_t ag_flags; /* i/o: flags for this ag */ + uint64_t ag_reserved[12];/* o: zero */ +}; +#define XFS_AG_GEOM_SICK_SB (1 << 0) /* superblock */ +#define XFS_AG_GEOM_SICK_AGF (1 << 1) /* AGF header */ +#define XFS_AG_GEOM_SICK_AGFL (1 << 2) /* AGFL header */ +#define XFS_AG_GEOM_SICK_AGI (1 << 3) /* AGI header */ +#define XFS_AG_GEOM_SICK_BNOBT (1 << 4) /* free space by block */ +#define XFS_AG_GEOM_SICK_CNTBT (1 << 5) /* free space by length */ +#define XFS_AG_GEOM_SICK_INOBT (1 << 6) /* inode index */ +#define XFS_AG_GEOM_SICK_FINOBT (1 << 7) /* free inode index */ +#define XFS_AG_GEOM_SICK_RMAPBT (1 << 8) /* reverse mappings */ +#define XFS_AG_GEOM_SICK_REFCNTBT (1 << 9) /* reference counts */ +#define XFS_AG_GEOM_SICK_INODES (1 << 10) /* bad inodes were seen */ + +/* + * Structures for XFS_IOC_FSGROWFSDATA, XFS_IOC_FSGROWFSLOG & XFS_IOC_FSGROWFSRT + */ +typedef struct xfs_growfs_data { + __u64 newblocks; /* new data subvol size, fsblocks */ + __u32 imaxpct; /* new inode space percentage limit */ +} xfs_growfs_data_t; + +typedef struct xfs_growfs_log { + __u32 newblocks; /* new log size, fsblocks */ + __u32 isint; /* 1 if new log is internal */ +} xfs_growfs_log_t; + +typedef struct xfs_growfs_rt { + __u64 newblocks; /* new realtime size, fsblocks */ + __u32 extsize; /* new realtime extent size, fsblocks */ +} xfs_growfs_rt_t; + + +/* + * Structures returned from ioctl XFS_IOC_FSBULKSTAT & XFS_IOC_FSBULKSTAT_SINGLE + */ +typedef struct xfs_bstime { + __kernel_long_t tv_sec; /* seconds */ + __s32 tv_nsec; /* and nanoseconds */ +} xfs_bstime_t; + +struct xfs_bstat { + __u64 bs_ino; /* inode number */ + __u16 bs_mode; /* type and mode */ + __u16 bs_nlink; /* number of links */ + __u32 bs_uid; /* user id */ + __u32 bs_gid; /* group id */ + __u32 bs_rdev; /* device value */ + __s32 bs_blksize; /* block size */ + __s64 bs_size; /* file size */ + xfs_bstime_t bs_atime; /* access time */ + xfs_bstime_t bs_mtime; /* modify time */ + xfs_bstime_t bs_ctime; /* inode change time */ + int64_t bs_blocks; /* number of blocks */ + __u32 bs_xflags; /* extended flags */ + __s32 bs_extsize; /* extent size */ + __s32 bs_extents; /* number of extents */ + __u32 bs_gen; /* generation count */ + __u16 bs_projid_lo; /* lower part of project id */ +#define bs_projid bs_projid_lo /* (previously just bs_projid) */ + __u16 bs_forkoff; /* inode fork offset in bytes */ + __u16 bs_projid_hi; /* higher part of project id */ + uint16_t bs_sick; /* sick inode metadata */ + uint16_t bs_checked; /* checked inode metadata */ + unsigned char bs_pad[2]; /* pad space, unused */ + __u32 bs_cowextsize; /* cow extent size */ + __u32 bs_dmevmask; /* DMIG event mask */ + __u16 bs_dmstate; /* DMIG state info */ + __u16 bs_aextents; /* attribute number of extents */ +}; + +/* New bulkstat structure that reports v5 features and fixes padding issues */ +struct xfs_bulkstat { + uint64_t bs_ino; /* inode number */ + uint64_t bs_size; /* file size */ + + uint64_t bs_blocks; /* number of blocks */ + uint64_t bs_xflags; /* extended flags */ + + int64_t bs_atime; /* access time, seconds */ + int64_t bs_mtime; /* modify time, seconds */ + + int64_t bs_ctime; /* inode change time, seconds */ + int64_t bs_btime; /* creation time, seconds */ + + uint32_t bs_gen; /* generation count */ + uint32_t bs_uid; /* user id */ + uint32_t bs_gid; /* group id */ + uint32_t bs_projectid; /* project id */ + + uint32_t bs_atime_nsec; /* access time, nanoseconds */ + uint32_t bs_mtime_nsec; /* modify time, nanoseconds */ + uint32_t bs_ctime_nsec; /* inode change time, nanoseconds */ + uint32_t bs_btime_nsec; /* creation time, nanoseconds */ + + uint32_t bs_blksize; /* block size */ + uint32_t bs_rdev; /* device value */ + uint32_t bs_cowextsize_blks; /* cow extent size hint, blocks */ + uint32_t bs_extsize_blks; /* extent size hint, blocks */ + + uint32_t bs_nlink; /* number of links */ + uint32_t bs_extents; /* 32-bit data fork extent counter */ + uint32_t bs_aextents; /* attribute number of extents */ + uint16_t bs_version; /* structure version */ + uint16_t bs_forkoff; /* inode fork offset in bytes */ + + uint16_t bs_sick; /* sick inode metadata */ + uint16_t bs_checked; /* checked inode metadata */ + uint16_t bs_mode; /* type and mode */ + uint16_t bs_pad2; /* zeroed */ + uint64_t bs_extents64; /* 64-bit data fork extent counter */ + + uint64_t bs_pad[6]; /* zeroed */ +}; + +#define XFS_BULKSTAT_VERSION_V1 (1) +#define XFS_BULKSTAT_VERSION_V5 (5) + +/* bs_sick flags */ +#define XFS_BS_SICK_INODE (1 << 0) /* inode core */ +#define XFS_BS_SICK_BMBTD (1 << 1) /* data fork */ +#define XFS_BS_SICK_BMBTA (1 << 2) /* attr fork */ +#define XFS_BS_SICK_BMBTC (1 << 3) /* cow fork */ +#define XFS_BS_SICK_DIR (1 << 4) /* directory */ +#define XFS_BS_SICK_XATTR (1 << 5) /* extended attributes */ +#define XFS_BS_SICK_SYMLINK (1 << 6) /* symbolic link remote target */ +#define XFS_BS_SICK_PARENT (1 << 7) /* parent pointers */ +#define XFS_BS_SICK_DIRTREE (1 << 8) /* directory tree structure */ + +/* + * Project quota id helpers (previously projid was 16bit only + * and using two 16bit values to hold new 32bit projid was chosen + * to retain compatibility with "old" filesystems). + */ +static inline uint32_t +bstat_get_projid(const struct xfs_bstat *bs) +{ + return (uint32_t)bs->bs_projid_hi << 16 | bs->bs_projid_lo; +} + +/* + * The user-level BulkStat Request interface structure. + */ +struct xfs_fsop_bulkreq { + __u64 __user *lastip; /* last inode # pointer */ + __s32 icount; /* count of entries in buffer */ + void __user *ubuffer;/* user buffer for inode desc. */ + __s32 __user *ocount; /* output count pointer */ +}; + +/* + * Structures returned from xfs_inumbers routine (XFS_IOC_FSINUMBERS). + */ +struct xfs_inogrp { + __u64 xi_startino; /* starting inode number */ + __s32 xi_alloccount; /* # bits set in allocmask */ + __u64 xi_allocmask; /* mask of allocated inodes */ +}; + +/* New inumbers structure that reports v5 features and fixes padding issues */ +struct xfs_inumbers { + uint64_t xi_startino; /* starting inode number */ + uint64_t xi_allocmask; /* mask of allocated inodes */ + uint8_t xi_alloccount; /* # bits set in allocmask */ + uint8_t xi_version; /* version */ + uint8_t xi_padding[6]; /* zero */ +}; + +#define XFS_INUMBERS_VERSION_V1 (1) +#define XFS_INUMBERS_VERSION_V5 (5) + +/* Header for bulk inode requests. */ +struct xfs_bulk_ireq { + uint64_t ino; /* I/O: start with this inode */ + uint32_t flags; /* I/O: operation flags */ + uint32_t icount; /* I: count of entries in buffer */ + uint32_t ocount; /* O: count of entries filled out */ + uint32_t agno; /* I: see comment for IREQ_AGNO */ + uint64_t reserved[5]; /* must be zero */ +}; + +/* + * Only return results from the specified @agno. If @ino is zero, start + * with the first inode of @agno. + */ +#define XFS_BULK_IREQ_AGNO (1U << 0) + +/* + * Return bulkstat information for a single inode, where @ino value is a + * special value, not a literal inode number. See the XFS_BULK_IREQ_SPECIAL_* + * values below. Not compatible with XFS_BULK_IREQ_AGNO. + */ +#define XFS_BULK_IREQ_SPECIAL (1U << 1) + +/* + * Return data fork extent count via xfs_bulkstat->bs_extents64 field and assign + * 0 to xfs_bulkstat->bs_extents when the flag is set. Otherwise, use + * xfs_bulkstat->bs_extents for returning data fork extent count and set + * xfs_bulkstat->bs_extents64 to 0. In the second case, return -EOVERFLOW and + * assign 0 to xfs_bulkstat->bs_extents if data fork extent count is larger than + * XFS_MAX_EXTCNT_DATA_FORK_OLD. + */ +#define XFS_BULK_IREQ_NREXT64 (1U << 2) + +/* + * Allow bulkstat to return information about metadata directories. This + * enables xfs_scrub to find them for scanning, as they are otherwise ordinary + * directories. + */ +#define XFS_BULK_IREQ_METADIR (1U << 3) + +#define XFS_BULK_IREQ_FLAGS_ALL (XFS_BULK_IREQ_AGNO | \ + XFS_BULK_IREQ_SPECIAL | \ + XFS_BULK_IREQ_NREXT64 | \ + XFS_BULK_IREQ_METADIR) + +/* Operate on the root directory inode. */ +#define XFS_BULK_IREQ_SPECIAL_ROOT (1) + +/* + * ioctl structures for v5 bulkstat and inumbers requests + */ +struct xfs_bulkstat_req { + struct xfs_bulk_ireq hdr; + struct xfs_bulkstat bulkstat[]; +}; +#define XFS_BULKSTAT_REQ_SIZE(nr) (sizeof(struct xfs_bulkstat_req) + \ + (nr) * sizeof(struct xfs_bulkstat)) + +struct xfs_inumbers_req { + struct xfs_bulk_ireq hdr; + struct xfs_inumbers inumbers[]; +}; +#define XFS_INUMBERS_REQ_SIZE(nr) (sizeof(struct xfs_inumbers_req) + \ + (nr) * sizeof(struct xfs_inumbers)) + +/* + * Error injection. + */ +typedef struct xfs_error_injection { + __s32 fd; + __s32 errtag; +} xfs_error_injection_t; + + +/* + * Speculative preallocation trimming. + */ +#define XFS_EOFBLOCKS_VERSION 1 +struct xfs_fs_eofblocks { + __u32 eof_version; + __u32 eof_flags; + uid_t eof_uid; + gid_t eof_gid; + prid_t eof_prid; + __u32 pad32; + __u64 eof_min_file_size; + __u64 pad64[12]; +}; + +/* eof_flags values */ +#define XFS_EOF_FLAGS_SYNC (1 << 0) /* sync/wait mode scan */ +#define XFS_EOF_FLAGS_UID (1 << 1) /* filter by uid */ +#define XFS_EOF_FLAGS_GID (1 << 2) /* filter by gid */ +#define XFS_EOF_FLAGS_PRID (1 << 3) /* filter by project id */ +#define XFS_EOF_FLAGS_MINFILESIZE (1 << 4) /* filter by min file size */ +#define XFS_EOF_FLAGS_UNION (1 << 5) /* union filter algorithm; + * kernel only, not included in + * valid mask */ +#define XFS_EOF_FLAGS_VALID \ + (XFS_EOF_FLAGS_SYNC | \ + XFS_EOF_FLAGS_UID | \ + XFS_EOF_FLAGS_GID | \ + XFS_EOF_FLAGS_PRID | \ + XFS_EOF_FLAGS_MINFILESIZE) + + +/* + * The user-level Handle Request interface structure. + */ +typedef struct xfs_fsop_handlereq { + __u32 fd; /* fd for FD_TO_HANDLE */ + void __user *path; /* user pathname */ + __u32 oflags; /* open flags */ + void __user *ihandle;/* user supplied handle */ + __u32 ihandlen; /* user supplied length */ + void __user *ohandle;/* user buffer for handle */ + __u32 __user *ohandlen;/* user buffer length */ +} xfs_fsop_handlereq_t; + +/* + * Compound structures for passing args through Handle Request interfaces + * xfs_attrlist_by_handle, xfs_attrmulti_by_handle + * - ioctls: XFS_IOC_ATTRLIST_BY_HANDLE, and XFS_IOC_ATTRMULTI_BY_HANDLE + */ + +/* + * Flags passed in xfs_attr_multiop.am_flags for the attr ioctl interface. + * + * NOTE: Must match the values declared in libattr without the XFS_IOC_ prefix. + */ +#define XFS_IOC_ATTR_ROOT 0x0002 /* use attrs in root namespace */ +#define XFS_IOC_ATTR_SECURE 0x0008 /* use attrs in security namespace */ +#define XFS_IOC_ATTR_CREATE 0x0010 /* fail if attr already exists */ +#define XFS_IOC_ATTR_REPLACE 0x0020 /* fail if attr does not exist */ + +typedef struct xfs_attrlist_cursor { + __u32 opaque[4]; +} xfs_attrlist_cursor_t; + +/* + * Define how lists of attribute names are returned to userspace from the + * XFS_IOC_ATTRLIST_BY_HANDLE ioctl. struct xfs_attrlist is the header at the + * beginning of the returned buffer, and a each entry in al_offset contains the + * relative offset of an xfs_attrlist_ent containing the actual entry. + * + * NOTE: struct xfs_attrlist must match struct attrlist defined in libattr, and + * struct xfs_attrlist_ent must match struct attrlist_ent defined in libattr. + */ +struct xfs_attrlist { + __s32 al_count; /* number of entries in attrlist */ + __s32 al_more; /* T/F: more attrs (do call again) */ + __s32 al_offset[]; /* byte offsets of attrs [var-sized] */ +}; + +struct xfs_attrlist_ent { /* data from attr_list() */ + __u32 a_valuelen; /* number bytes in value of attr */ + char a_name[]; /* attr name (NULL terminated) */ +}; + +typedef struct xfs_fsop_attrlist_handlereq { + struct xfs_fsop_handlereq hreq; /* handle interface structure */ + struct xfs_attrlist_cursor pos; /* opaque cookie, list offset */ + __u32 flags; /* which namespace to use */ + __u32 buflen; /* length of buffer supplied */ + void __user *buffer; /* returned names */ +} xfs_fsop_attrlist_handlereq_t; + +typedef struct xfs_attr_multiop { + __u32 am_opcode; +#define ATTR_OP_GET 1 /* return the indicated attr's value */ +#define ATTR_OP_SET 2 /* set/create the indicated attr/value pair */ +#define ATTR_OP_REMOVE 3 /* remove the indicated attr */ + __s32 am_error; + void __user *am_attrname; + void __user *am_attrvalue; + __u32 am_length; + __u32 am_flags; /* XFS_IOC_ATTR_* */ +} xfs_attr_multiop_t; + +typedef struct xfs_fsop_attrmulti_handlereq { + struct xfs_fsop_handlereq hreq; /* handle interface structure */ + __u32 opcount;/* count of following multiop */ + struct xfs_attr_multiop __user *ops; /* attr_multi data */ +} xfs_fsop_attrmulti_handlereq_t; + +/* + * per machine unique filesystem identifier types. + */ +typedef struct xfs_fsid { + __u32 val[2]; /* file system id type */ +} xfs_fsid_t; + +typedef struct xfs_fid { + __u16 fid_len; /* length of remainder */ + __u16 fid_pad; + __u32 fid_gen; /* generation number */ + __u64 fid_ino; /* 64 bits inode number */ +} xfs_fid_t; + +typedef struct xfs_handle { + union { + __s64 align; /* force alignment of ha_fid */ + xfs_fsid_t _ha_fsid; /* unique file system identifier */ + } ha_u; + xfs_fid_t ha_fid; /* file system specific file ID */ +} xfs_handle_t; +#define ha_fsid ha_u._ha_fsid + +/* + * Structure passed to XFS_IOC_SWAPEXT + */ +typedef struct xfs_swapext +{ + int64_t sx_version; /* version */ +#define XFS_SX_VERSION 0 + int64_t sx_fdtarget; /* fd of target file */ + int64_t sx_fdtmp; /* fd of tmp file */ + xfs_off_t sx_offset; /* offset into file */ + xfs_off_t sx_length; /* leng from offset */ + char sx_pad[16]; /* pad space, unused */ + struct xfs_bstat sx_stat; /* stat of target b4 copy */ +} xfs_swapext_t; + +/* + * Flags for going down operation + */ +#define XFS_FSOP_GOING_FLAGS_DEFAULT 0x0 /* going down */ +#define XFS_FSOP_GOING_FLAGS_LOGFLUSH 0x1 /* flush log but not data */ +#define XFS_FSOP_GOING_FLAGS_NOLOGFLUSH 0x2 /* don't flush log nor data */ + +/* metadata scrubbing */ +struct xfs_scrub_metadata { + __u32 sm_type; /* What to check? */ + __u32 sm_flags; /* flags; see below. */ + __u64 sm_ino; /* inode number. */ + __u32 sm_gen; /* inode generation. */ + __u32 sm_agno; /* ag number. */ + __u64 sm_reserved[5]; /* pad to 64 bytes */ +}; + +/* + * Metadata types and flags for scrub operation. + */ + +/* Scrub subcommands. */ +#define XFS_SCRUB_TYPE_PROBE 0 /* presence test ioctl */ +#define XFS_SCRUB_TYPE_SB 1 /* superblock */ +#define XFS_SCRUB_TYPE_AGF 2 /* AG free header */ +#define XFS_SCRUB_TYPE_AGFL 3 /* AG free list */ +#define XFS_SCRUB_TYPE_AGI 4 /* AG inode header */ +#define XFS_SCRUB_TYPE_BNOBT 5 /* freesp by block btree */ +#define XFS_SCRUB_TYPE_CNTBT 6 /* freesp by length btree */ +#define XFS_SCRUB_TYPE_INOBT 7 /* inode btree */ +#define XFS_SCRUB_TYPE_FINOBT 8 /* free inode btree */ +#define XFS_SCRUB_TYPE_RMAPBT 9 /* reverse mapping btree */ +#define XFS_SCRUB_TYPE_REFCNTBT 10 /* reference count btree */ +#define XFS_SCRUB_TYPE_INODE 11 /* inode record */ +#define XFS_SCRUB_TYPE_BMBTD 12 /* data fork block mapping */ +#define XFS_SCRUB_TYPE_BMBTA 13 /* attr fork block mapping */ +#define XFS_SCRUB_TYPE_BMBTC 14 /* CoW fork block mapping */ +#define XFS_SCRUB_TYPE_DIR 15 /* directory */ +#define XFS_SCRUB_TYPE_XATTR 16 /* extended attribute */ +#define XFS_SCRUB_TYPE_SYMLINK 17 /* symbolic link */ +#define XFS_SCRUB_TYPE_PARENT 18 /* parent pointers */ +#define XFS_SCRUB_TYPE_RTBITMAP 19 /* realtime bitmap */ +#define XFS_SCRUB_TYPE_RTSUM 20 /* realtime summary */ +#define XFS_SCRUB_TYPE_UQUOTA 21 /* user quotas */ +#define XFS_SCRUB_TYPE_GQUOTA 22 /* group quotas */ +#define XFS_SCRUB_TYPE_PQUOTA 23 /* project quotas */ +#define XFS_SCRUB_TYPE_FSCOUNTERS 24 /* fs summary counters */ +#define XFS_SCRUB_TYPE_QUOTACHECK 25 /* quota counters */ +#define XFS_SCRUB_TYPE_NLINKS 26 /* inode link counts */ +#define XFS_SCRUB_TYPE_HEALTHY 27 /* everything checked out ok */ +#define XFS_SCRUB_TYPE_DIRTREE 28 /* directory tree structure */ +#define XFS_SCRUB_TYPE_METAPATH 29 /* metadata directory tree paths */ +#define XFS_SCRUB_TYPE_RGSUPER 30 /* realtime superblock */ +#define XFS_SCRUB_TYPE_RTRMAPBT 31 /* rtgroup reverse mapping btree */ +#define XFS_SCRUB_TYPE_RTREFCBT 32 /* realtime reference count btree */ + +/* Number of scrub subcommands. */ +#define XFS_SCRUB_TYPE_NR 33 + +/* + * This special type code only applies to the vectored scrub implementation. + * + * If any of the previous scrub vectors recorded runtime errors or have + * sv_flags bits set that match the OFLAG bits in the barrier vector's + * sv_flags, set the barrier's sv_ret to -ECANCELED and return to userspace. + */ +#define XFS_SCRUB_TYPE_BARRIER (0xFFFFFFFF) + +/* i: Repair this metadata. */ +#define XFS_SCRUB_IFLAG_REPAIR (1u << 0) + +/* o: Metadata object needs repair. */ +#define XFS_SCRUB_OFLAG_CORRUPT (1u << 1) + +/* + * o: Metadata object could be optimized. It's not corrupt, but + * we could improve on it somehow. + */ +#define XFS_SCRUB_OFLAG_PREEN (1u << 2) + +/* o: Cross-referencing failed. */ +#define XFS_SCRUB_OFLAG_XFAIL (1u << 3) + +/* o: Metadata object disagrees with cross-referenced metadata. */ +#define XFS_SCRUB_OFLAG_XCORRUPT (1u << 4) + +/* o: Scan was not complete. */ +#define XFS_SCRUB_OFLAG_INCOMPLETE (1u << 5) + +/* o: Metadata object looked funny but isn't corrupt. */ +#define XFS_SCRUB_OFLAG_WARNING (1u << 6) + +/* + * o: IFLAG_REPAIR was set but metadata object did not need fixing or + * optimization and has therefore not been altered. + */ +#define XFS_SCRUB_OFLAG_NO_REPAIR_NEEDED (1u << 7) + +/* i: Rebuild the data structure. */ +#define XFS_SCRUB_IFLAG_FORCE_REBUILD (1u << 8) + +#define XFS_SCRUB_FLAGS_IN (XFS_SCRUB_IFLAG_REPAIR | \ + XFS_SCRUB_IFLAG_FORCE_REBUILD) +#define XFS_SCRUB_FLAGS_OUT (XFS_SCRUB_OFLAG_CORRUPT | \ + XFS_SCRUB_OFLAG_PREEN | \ + XFS_SCRUB_OFLAG_XFAIL | \ + XFS_SCRUB_OFLAG_XCORRUPT | \ + XFS_SCRUB_OFLAG_INCOMPLETE | \ + XFS_SCRUB_OFLAG_WARNING | \ + XFS_SCRUB_OFLAG_NO_REPAIR_NEEDED) +#define XFS_SCRUB_FLAGS_ALL (XFS_SCRUB_FLAGS_IN | XFS_SCRUB_FLAGS_OUT) + +/* Vectored scrub calls to reduce the number of kernel transitions. */ + +struct xfs_scrub_vec { + __u32 sv_type; /* XFS_SCRUB_TYPE_* */ + __u32 sv_flags; /* XFS_SCRUB_FLAGS_* */ + __s32 sv_ret; /* 0 or a negative error code */ + __u32 sv_reserved; /* must be zero */ +}; + +/* Vectored metadata scrub control structure. */ +struct xfs_scrub_vec_head { + __u64 svh_ino; /* inode number. */ + __u32 svh_gen; /* inode generation. */ + __u32 svh_agno; /* ag number. */ + __u32 svh_flags; /* XFS_SCRUB_VEC_FLAGS_* */ + __u16 svh_rest_us; /* wait this much time between vector items */ + __u16 svh_nr; /* number of svh_vectors */ + __u64 svh_reserved; /* must be zero */ + __u64 svh_vectors; /* pointer to buffer of xfs_scrub_vec */ +}; + +#define XFS_SCRUB_VEC_FLAGS_ALL (0) + +/* + * i: sm_ino values for XFS_SCRUB_TYPE_METAPATH to select a metadata file for + * path checking. + */ +#define XFS_SCRUB_METAPATH_PROBE (0) /* do we have a metapath scrubber? */ +#define XFS_SCRUB_METAPATH_RTDIR (1) /* rtrgroups metadir */ +#define XFS_SCRUB_METAPATH_RTBITMAP (2) /* per-rtg bitmap */ +#define XFS_SCRUB_METAPATH_RTSUMMARY (3) /* per-rtg summary */ +#define XFS_SCRUB_METAPATH_QUOTADIR (4) /* quota metadir */ +#define XFS_SCRUB_METAPATH_USRQUOTA (5) /* user quota */ +#define XFS_SCRUB_METAPATH_GRPQUOTA (6) /* group quota */ +#define XFS_SCRUB_METAPATH_PRJQUOTA (7) /* project quota */ +#define XFS_SCRUB_METAPATH_RTRMAPBT (8) /* realtime reverse mapping */ +#define XFS_SCRUB_METAPATH_RTREFCOUNTBT (9) /* realtime refcount */ + +/* Number of metapath sm_ino values */ +#define XFS_SCRUB_METAPATH_NR (10) + +/* + * ioctl limits + */ +#ifdef XATTR_LIST_MAX +# define XFS_XATTR_LIST_MAX XATTR_LIST_MAX +#else +# define XFS_XATTR_LIST_MAX 65536 +#endif + +/* + * Exchange part of file1 with part of the file that this ioctl that is being + * called against (which we'll call file2). Filesystems must be able to + * restart and complete the operation even after the system goes down. + */ +struct xfs_exchange_range { + __s32 file1_fd; + __u32 pad; /* must be zeroes */ + __u64 file1_offset; /* file1 offset, bytes */ + __u64 file2_offset; /* file2 offset, bytes */ + __u64 length; /* bytes to exchange */ + + __u64 flags; /* see XFS_EXCHANGE_RANGE_* below */ +}; + +/* + * Using the same definition of file2 as struct xfs_exchange_range, commit the + * contents of file1 into file2 if file2 has the same inode number, mtime, and + * ctime as the arguments provided to the call. The old contents of file2 will + * be moved to file1. + * + * Returns -EBUSY if there isn't an exact match for the file2 fields. + * + * Filesystems must be able to restart and complete the operation even after + * the system goes down. + */ +struct xfs_commit_range { + __s32 file1_fd; + __u32 pad; /* must be zeroes */ + __u64 file1_offset; /* file1 offset, bytes */ + __u64 file2_offset; /* file2 offset, bytes */ + __u64 length; /* bytes to exchange */ + + __u64 flags; /* see XFS_EXCHANGE_RANGE_* below */ + + /* opaque file2 metadata for freshness checks */ + __u64 file2_freshness[6]; +}; + +/* + * Exchange file data all the way to the ends of both files, and then exchange + * the file sizes. This flag can be used to replace a file's contents with a + * different amount of data. length will be ignored. + */ +#define XFS_EXCHANGE_RANGE_TO_EOF (1ULL << 0) + +/* Flush all changes in file data and file metadata to disk before returning. */ +#define XFS_EXCHANGE_RANGE_DSYNC (1ULL << 1) + +/* Dry run; do all the parameter verification but do not change anything. */ +#define XFS_EXCHANGE_RANGE_DRY_RUN (1ULL << 2) + +/* + * Exchange only the parts of the two files where the file allocation units + * mapped to file1's range have been written to. This can accelerate + * scatter-gather atomic writes with a temp file if all writes are aligned to + * the file allocation unit. + */ +#define XFS_EXCHANGE_RANGE_FILE1_WRITTEN (1ULL << 3) + +#define XFS_EXCHANGE_RANGE_ALL_FLAGS (XFS_EXCHANGE_RANGE_TO_EOF | \ + XFS_EXCHANGE_RANGE_DSYNC | \ + XFS_EXCHANGE_RANGE_DRY_RUN | \ + XFS_EXCHANGE_RANGE_FILE1_WRITTEN) + +/* Iterating parent pointers of files. */ + +/* target was the root directory */ +#define XFS_GETPARENTS_OFLAG_ROOT (1U << 0) + +/* Cursor is done iterating pptrs */ +#define XFS_GETPARENTS_OFLAG_DONE (1U << 1) + +#define XFS_GETPARENTS_OFLAGS_ALL (XFS_GETPARENTS_OFLAG_ROOT | \ + XFS_GETPARENTS_OFLAG_DONE) + +#define XFS_GETPARENTS_IFLAGS_ALL (0) + +struct xfs_getparents_rec { + struct xfs_handle gpr_parent; /* Handle to parent */ + __u32 gpr_reclen; /* Length of entire record */ + __u32 gpr_reserved; /* zero */ + char gpr_name[]; /* Null-terminated filename */ +}; + +/* Iterate through this file's directory parent pointers */ +struct xfs_getparents { + /* + * Structure to track progress in iterating the parent pointers. + * Must be initialized to zeroes before the first ioctl call, and + * not touched by callers after that. + */ + struct xfs_attrlist_cursor gp_cursor; + + /* Input flags: XFS_GETPARENTS_IFLAG* */ + __u16 gp_iflags; + + /* Output flags: XFS_GETPARENTS_OFLAG* */ + __u16 gp_oflags; + + /* Size of the gp_buffer in bytes */ + __u32 gp_bufsize; + + /* Must be set to zero */ + __u64 gp_reserved; + + /* Pointer to a buffer in which to place xfs_getparents_rec */ + __u64 gp_buffer; +}; + +static inline struct xfs_getparents_rec * +xfs_getparents_first_rec(struct xfs_getparents *gp) +{ + return (struct xfs_getparents_rec *)(uintptr_t)gp->gp_buffer; +} + +static inline struct xfs_getparents_rec * +xfs_getparents_next_rec(struct xfs_getparents *gp, + struct xfs_getparents_rec *gpr) +{ + void *next = ((char *)gpr + gpr->gpr_reclen); + void *end = (void *)(uintptr_t)(gp->gp_buffer + gp->gp_bufsize); + + if (next >= end) + return NULL; + + return (struct xfs_getparents_rec *)next; +} + +/* Iterate through this file handle's directory parent pointers. */ +struct xfs_getparents_by_handle { + /* Handle to file whose parents we want. */ + struct xfs_handle gph_handle; + + struct xfs_getparents gph_request; +}; + +/* + * Output for XFS_IOC_RTGROUP_GEOMETRY + */ +struct xfs_rtgroup_geometry { + __u32 rg_number; /* i/o: rtgroup number */ + __u32 rg_length; /* o: length in blocks */ + __u32 rg_sick; /* o: sick things in ag */ + __u32 rg_checked; /* o: checked metadata in ag */ + __u32 rg_flags; /* i/o: flags for this ag */ + __u32 rg_writepointer; /* o: write pointer block offset for zoned */ + __u32 rg_reserved[26]; /* o: zero */ +}; +#define XFS_RTGROUP_GEOM_SICK_SUPER (1U << 0) /* superblock */ +#define XFS_RTGROUP_GEOM_SICK_BITMAP (1U << 1) /* rtbitmap */ +#define XFS_RTGROUP_GEOM_SICK_SUMMARY (1U << 2) /* rtsummary */ +#define XFS_RTGROUP_GEOM_SICK_RMAPBT (1U << 3) /* reverse mappings */ +#define XFS_RTGROUP_GEOM_SICK_REFCNTBT (1U << 4) /* reference counts */ + +#define XFS_RTGROUP_GEOM_WRITEPOINTER (1U << 0) /* write pointer */ + +/* Health monitor event domains */ + +/* affects the whole fs */ +#define XFS_HEALTH_MONITOR_DOMAIN_MOUNT (0) + +/* metadata health events */ +#define XFS_HEALTH_MONITOR_DOMAIN_FS (1) +#define XFS_HEALTH_MONITOR_DOMAIN_AG (2) +#define XFS_HEALTH_MONITOR_DOMAIN_INODE (3) +#define XFS_HEALTH_MONITOR_DOMAIN_RTGROUP (4) + +/* disk events */ +#define XFS_HEALTH_MONITOR_DOMAIN_DATADEV (5) +#define XFS_HEALTH_MONITOR_DOMAIN_RTDEV (6) +#define XFS_HEALTH_MONITOR_DOMAIN_LOGDEV (7) + +/* file range events */ +#define XFS_HEALTH_MONITOR_DOMAIN_FILERANGE (8) + +/* Health monitor event types */ + +/* status of the monitor itself */ +#define XFS_HEALTH_MONITOR_TYPE_RUNNING (0) +#define XFS_HEALTH_MONITOR_TYPE_LOST (1) + +/* filesystem was unmounted */ +#define XFS_HEALTH_MONITOR_TYPE_UNMOUNT (2) + +/* metadata health events */ +#define XFS_HEALTH_MONITOR_TYPE_SICK (3) +#define XFS_HEALTH_MONITOR_TYPE_CORRUPT (4) +#define XFS_HEALTH_MONITOR_TYPE_HEALTHY (5) + +/* filesystem shutdown */ +#define XFS_HEALTH_MONITOR_TYPE_SHUTDOWN (6) + +/* media errors */ +#define XFS_HEALTH_MONITOR_TYPE_MEDIA_ERROR (7) + +/* pagecache I/O to a file range failed */ +#define XFS_HEALTH_MONITOR_TYPE_BUFREAD (8) +#define XFS_HEALTH_MONITOR_TYPE_BUFWRITE (9) + +/* direct I/O to a file range failed */ +#define XFS_HEALTH_MONITOR_TYPE_DIOREAD (10) +#define XFS_HEALTH_MONITOR_TYPE_DIOWRITE (11) + +/* out of band media error reported for a file range */ +#define XFS_HEALTH_MONITOR_TYPE_DATALOST (12) + +/* lost events */ +struct xfs_health_monitor_lost { + __u64 count; +}; + +/* fs/rt metadata */ +struct xfs_health_monitor_fs { + /* XFS_FSOP_GEOM_SICK_* flags */ + __u32 mask; +}; + +/* ag/rtgroup metadata */ +struct xfs_health_monitor_group { + /* XFS_{AG,RTGROUP}_SICK_* flags */ + __u32 mask; + __u32 gno; +}; + +/* inode metadata */ +struct xfs_health_monitor_inode { + /* XFS_BS_SICK_* flags */ + __u32 mask; + __u32 gen; + __u64 ino; +}; + +/* shutdown reasons */ +#define XFS_HEALTH_SHUTDOWN_META_IO_ERROR (1u << 0) +#define XFS_HEALTH_SHUTDOWN_LOG_IO_ERROR (1u << 1) +#define XFS_HEALTH_SHUTDOWN_FORCE_UMOUNT (1u << 2) +#define XFS_HEALTH_SHUTDOWN_CORRUPT_INCORE (1u << 3) +#define XFS_HEALTH_SHUTDOWN_CORRUPT_ONDISK (1u << 4) +#define XFS_HEALTH_SHUTDOWN_DEVICE_REMOVED (1u << 5) + +/* shutdown */ +struct xfs_health_monitor_shutdown { + /* XFS_HEALTH_SHUTDOWN_* flags */ + __u32 reasons; +}; + +/* file range events */ +struct xfs_health_monitor_filerange { + __u64 pos; + __u64 len; + __u64 ino; + __u32 gen; + __u32 error; +}; + +/* disk media errors */ +struct xfs_health_monitor_media { + __u64 daddr; + __u64 bbcount; +}; + +struct xfs_health_monitor_event { + /* XFS_HEALTH_MONITOR_DOMAIN_* */ + __u32 domain; + + /* XFS_HEALTH_MONITOR_TYPE_* */ + __u32 type; + + /* Timestamp of the event, in nanoseconds since the Unix epoch */ + __u64 time_ns; + + /* + * Details of the event. The primary clients are written in python + * and rust, so break this up because bindgen hates anonymous structs + * and unions. + */ + union { + struct xfs_health_monitor_lost lost; + struct xfs_health_monitor_fs fs; + struct xfs_health_monitor_group group; + struct xfs_health_monitor_inode inode; + struct xfs_health_monitor_shutdown shutdown; + struct xfs_health_monitor_media media; + struct xfs_health_monitor_filerange filerange; + } e; + + /* zeroes */ + __u64 pad[2]; +}; + +struct xfs_health_monitor { + __u64 flags; /* flags */ + __u8 format; /* output format */ + __u8 pad[23]; /* zeroes */ +}; + +/* Return all health status events, not just deltas */ +#define XFS_HEALTH_MONITOR_VERBOSE (1ULL << 0) + +#define XFS_HEALTH_MONITOR_ALL (XFS_HEALTH_MONITOR_VERBOSE) + +/* Initial return format version */ +#define XFS_HEALTH_MONITOR_FMT_V0 (0) + +/* + * Check that a given fd points to the same filesystem that the health monitor + * is monitoring. + */ +struct xfs_health_file_on_monitored_fs { + __s32 fd; + __u32 flags; /* zero for now */ +}; + +/* Verify the media of the underlying devices */ +struct xfs_verify_media { + __u32 me_dev; /* I: XFS_DEV_{DATA,LOG,RT} */ + __u32 me_flags; /* I: XFS_VERIFY_MEDIA_* */ + + /* + * IO: inclusive start of disk range to verify, in 512b blocks. + * Will be adjusted upwards as media verification succeeds. + */ + __u64 me_start_daddr; + + /* + * IO: exclusive end of the disk range to verify, in 512b blocks. + * Can be adjusted downwards to match device size. + */ + __u64 me_end_daddr; + + __u32 me_ioerror; /* O: I/O error (positive) */ + __u32 me_max_io_size; /* I: maximum IO size in bytes */ + + __u32 me_rest_us; /* I: rest time between IOs, usecs */ + __u32 me_pad; /* zero */ +}; + +#define XFS_VERIFY_MEDIA_REPORT (1 << 0) /* report to fsnotify */ + +#define XFS_VERIFY_MEDIA_FLAGS (XFS_VERIFY_MEDIA_REPORT) + +/* + * ioctl commands that are used by Linux filesystems + */ +#define XFS_IOC_GETXFLAGS FS_IOC_GETFLAGS +#define XFS_IOC_SETXFLAGS FS_IOC_SETFLAGS +#define XFS_IOC_GETVERSION FS_IOC_GETVERSION + +/* + * ioctl commands that replace IRIX fcntl()'s + * For 'documentation' purposed more than anything else, + * the "cmd #" field reflects the IRIX fcntl number. + */ +/* XFS_IOC_ALLOCSP ------- deprecated 10 */ +/* XFS_IOC_FREESP -------- deprecated 11 */ +#define XFS_IOC_DIOINFO _IOR ('X', 30, struct dioattr) +#define XFS_IOC_FSGETXATTR FS_IOC_FSGETXATTR +#define XFS_IOC_FSSETXATTR FS_IOC_FSSETXATTR +/* XFS_IOC_ALLOCSP64 ----- deprecated 36 */ +/* XFS_IOC_FREESP64 ------ deprecated 37 */ +#define XFS_IOC_GETBMAP _IOWR('X', 38, struct getbmap) +/* XFS_IOC_FSSETDM ------- deprecated 39 */ +#define XFS_IOC_RESVSP _IOW ('X', 40, struct xfs_flock64) +#define XFS_IOC_UNRESVSP _IOW ('X', 41, struct xfs_flock64) +#define XFS_IOC_RESVSP64 _IOW ('X', 42, struct xfs_flock64) +#define XFS_IOC_UNRESVSP64 _IOW ('X', 43, struct xfs_flock64) +#define XFS_IOC_GETBMAPA _IOWR('X', 44, struct getbmap) +#define XFS_IOC_FSGETXATTRA _IOR ('X', 45, struct fsxattr) +/* XFS_IOC_SETBIOSIZE ---- deprecated 46 */ +/* XFS_IOC_GETBIOSIZE ---- deprecated 47 */ +#define XFS_IOC_GETBMAPX _IOWR('X', 56, struct getbmap) +#define XFS_IOC_ZERO_RANGE _IOW ('X', 57, struct xfs_flock64) +#define XFS_IOC_FREE_EOFBLOCKS _IOR ('X', 58, struct xfs_fs_eofblocks) +/* XFS_IOC_GETFSMAP ------ hoisted 59 */ +#define XFS_IOC_SCRUB_METADATA _IOWR('X', 60, struct xfs_scrub_metadata) +#define XFS_IOC_AG_GEOMETRY _IOWR('X', 61, struct xfs_ag_geometry) +#define XFS_IOC_GETPARENTS _IOWR('X', 62, struct xfs_getparents) +#define XFS_IOC_GETPARENTS_BY_HANDLE _IOWR('X', 63, struct xfs_getparents_by_handle) +#define XFS_IOC_SCRUBV_METADATA _IOWR('X', 64, struct xfs_scrub_vec_head) +#define XFS_IOC_RTGROUP_GEOMETRY _IOWR('X', 65, struct xfs_rtgroup_geometry) +#define XFS_IOC_HEALTH_MONITOR _IOW ('X', 68, struct xfs_health_monitor) +#define XFS_IOC_HEALTH_FD_ON_MONITORED_FS \ + _IOW ('X', 69, struct xfs_health_file_on_monitored_fs) +#define XFS_IOC_VERIFY_MEDIA _IOWR('X', 70, struct xfs_verify_media) + +/* + * ioctl commands that replace IRIX syssgi()'s + */ +#define XFS_IOC_FSGEOMETRY_V1 _IOR ('X', 100, struct xfs_fsop_geom_v1) +#define XFS_IOC_FSBULKSTAT _IOWR('X', 101, struct xfs_fsop_bulkreq) +#define XFS_IOC_FSBULKSTAT_SINGLE _IOWR('X', 102, struct xfs_fsop_bulkreq) +#define XFS_IOC_FSINUMBERS _IOWR('X', 103, struct xfs_fsop_bulkreq) +#define XFS_IOC_PATH_TO_FSHANDLE _IOWR('X', 104, struct xfs_fsop_handlereq) +#define XFS_IOC_PATH_TO_HANDLE _IOWR('X', 105, struct xfs_fsop_handlereq) +#define XFS_IOC_FD_TO_HANDLE _IOWR('X', 106, struct xfs_fsop_handlereq) +#define XFS_IOC_OPEN_BY_HANDLE _IOWR('X', 107, struct xfs_fsop_handlereq) +#define XFS_IOC_READLINK_BY_HANDLE _IOWR('X', 108, struct xfs_fsop_handlereq) +#define XFS_IOC_SWAPEXT _IOWR('X', 109, struct xfs_swapext) +#define XFS_IOC_FSGROWFSDATA _IOW ('X', 110, struct xfs_growfs_data) +#define XFS_IOC_FSGROWFSLOG _IOW ('X', 111, struct xfs_growfs_log) +#define XFS_IOC_FSGROWFSRT _IOW ('X', 112, struct xfs_growfs_rt) +#define XFS_IOC_FSCOUNTS _IOR ('X', 113, struct xfs_fsop_counts) +#define XFS_IOC_SET_RESBLKS _IOWR('X', 114, struct xfs_fsop_resblks) +#define XFS_IOC_GET_RESBLKS _IOR ('X', 115, struct xfs_fsop_resblks) +#define XFS_IOC_ERROR_INJECTION _IOW ('X', 116, struct xfs_error_injection) +#define XFS_IOC_ERROR_CLEARALL _IOW ('X', 117, struct xfs_error_injection) +/* XFS_IOC_ATTRCTL_BY_HANDLE -- deprecated 118 */ + +#define XFS_IOC_FREEZE _IOWR('X', 119, int) /* aka FIFREEZE */ +#define XFS_IOC_THAW _IOWR('X', 120, int) /* aka FITHAW */ + +/* XFS_IOC_FSSETDM_BY_HANDLE -- deprecated 121 */ +#define XFS_IOC_ATTRLIST_BY_HANDLE _IOW ('X', 122, struct xfs_fsop_attrlist_handlereq) +#define XFS_IOC_ATTRMULTI_BY_HANDLE _IOW ('X', 123, struct xfs_fsop_attrmulti_handlereq) +#define XFS_IOC_FSGEOMETRY_V4 _IOR ('X', 124, struct xfs_fsop_geom_v4) +#define XFS_IOC_GOINGDOWN _IOR ('X', 125, uint32_t) +#define XFS_IOC_FSGEOMETRY _IOR ('X', 126, struct xfs_fsop_geom) +#define XFS_IOC_BULKSTAT _IOR ('X', 127, struct xfs_bulkstat_req) +#define XFS_IOC_INUMBERS _IOR ('X', 128, struct xfs_inumbers_req) +#define XFS_IOC_EXCHANGE_RANGE _IOW ('X', 129, struct xfs_exchange_range) +#define XFS_IOC_START_COMMIT _IOR ('X', 130, struct xfs_commit_range) +#define XFS_IOC_COMMIT_RANGE _IOW ('X', 131, struct xfs_commit_range) +/* XFS_IOC_GETFSUUID ---------- deprecated 140 */ + +/* + * Devices supported by a single XFS file system. Reported in fsmaps fmr_device + * when using internal RT devices. + */ +enum xfs_device { + XFS_DEV_DATA = 1, + XFS_DEV_LOG = 2, + XFS_DEV_RT = 3, +}; + +#ifndef HAVE_BBMACROS +/* + * Block I/O parameterization. A basic block (BB) is the lowest size of + * filesystem allocation, and must equal 512. Length units given to bio + * routines are in BB's. + */ +#define BBSHIFT 9 +#define BBSIZE (1<> BBSHIFT) +#define BTOBBT(bytes) ((__u64)(bytes) >> BBSHIFT) +#define BBTOB(bbs) ((bbs) << BBSHIFT) +#endif + +#endif /* __XFS_FS_H__ */ diff --git a/fs/xfs/libxfs/xfs_group.c b/fs/xfs/libxfs/xfs_group.c new file mode 100644 index 000000000..2ff9d1e56 --- /dev/null +++ b/fs/xfs/libxfs/xfs_group.c @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018 Red Hat, Inc. + */ + +#include "xfs_platform.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_error.h" +#include "xfs_trace.h" +#include "xfs_extent_busy.h" +#include "xfs_group.h" + +/* + * Groups can have passive and active references. + * + * For passive references the code freeing a group is responsible for cleaning + * up objects that hold the passive references (e.g. cached buffers). + * Routines manipulating passive references are xfs_group_get, xfs_group_hold + * and xfs_group_put. + * + * Active references are for short term access to the group for walking trees or + * accessing state. If a group is being shrunk or offlined, the lookup will fail + * to find that group and return NULL instead. + * Routines manipulating active references are xfs_group_grab and + * xfs_group_rele. + */ + +struct xfs_group * +xfs_group_get( + struct xfs_mount *mp, + uint32_t index, + enum xfs_group_type type) +{ + struct xfs_group *xg; + + rcu_read_lock(); + xg = xa_load(&mp->m_groups[type].xa, index); + if (xg) { + trace_xfs_group_get(xg, _RET_IP_); + ASSERT(atomic_read(&xg->xg_ref) >= 0); + atomic_inc(&xg->xg_ref); + } + rcu_read_unlock(); + return xg; +} + +struct xfs_group * +xfs_group_hold( + struct xfs_group *xg) +{ + ASSERT(atomic_read(&xg->xg_ref) > 0 || + atomic_read(&xg->xg_active_ref) > 0); + + trace_xfs_group_hold(xg, _RET_IP_); + atomic_inc(&xg->xg_ref); + return xg; +} + +void +xfs_group_put( + struct xfs_group *xg) +{ + trace_xfs_group_put(xg, _RET_IP_); + + ASSERT(atomic_read(&xg->xg_ref) > 0); + atomic_dec(&xg->xg_ref); +} + +struct xfs_group * +xfs_group_grab( + struct xfs_mount *mp, + uint32_t index, + enum xfs_group_type type) +{ + struct xfs_group *xg; + + rcu_read_lock(); + xg = xa_load(&mp->m_groups[type].xa, index); + if (xg) { + trace_xfs_group_grab(xg, _RET_IP_); + if (!atomic_inc_not_zero(&xg->xg_active_ref)) + xg = NULL; + } + rcu_read_unlock(); + return xg; +} + +/* + * Iterate to the next group. To start the iteration at @start_index, a %NULL + * @xg is passed, else the previous group returned from this function. The + * caller should break out of the loop when this returns %NULL. If the caller + * wants to break out of a loop that did not finish it needs to release the + * active reference to @xg using xfs_group_rele() itself. + */ +struct xfs_group * +xfs_group_next_range( + struct xfs_mount *mp, + struct xfs_group *xg, + uint32_t start_index, + uint32_t end_index, + enum xfs_group_type type) +{ + uint32_t index = start_index; + + if (xg) { + index = xg->xg_gno + 1; + xfs_group_rele(xg); + } + if (index > end_index) + return NULL; + return xfs_group_grab(mp, index, type); +} + +/* + * Find the next group after @xg, or the first group if @xg is NULL. + */ +struct xfs_group * +xfs_group_grab_next_mark( + struct xfs_mount *mp, + struct xfs_group *xg, + xa_mark_t mark, + enum xfs_group_type type) +{ + unsigned long index = 0; + + if (xg) { + index = xg->xg_gno + 1; + xfs_group_rele(xg); + } + + rcu_read_lock(); + xg = xa_find(&mp->m_groups[type].xa, &index, ULONG_MAX, mark); + if (xg) { + trace_xfs_group_grab_next_tag(xg, _RET_IP_); + if (!atomic_inc_not_zero(&xg->xg_active_ref)) + xg = NULL; + } + rcu_read_unlock(); + return xg; +} + +void +xfs_group_rele( + struct xfs_group *xg) +{ + trace_xfs_group_rele(xg, _RET_IP_); + atomic_dec(&xg->xg_active_ref); +} + +void +xfs_group_free( + struct xfs_mount *mp, + uint32_t index, + enum xfs_group_type type, + void (*uninit)(struct xfs_group *xg)) +{ + struct xfs_group *xg = xa_erase(&mp->m_groups[type].xa, index); + + XFS_IS_CORRUPT(mp, atomic_read(&xg->xg_ref) != 0); + + xfs_defer_drain_free(&xg->xg_intents_drain); +#ifdef __KERNEL__ + if (xfs_group_has_extent_busy(xg->xg_mount, xg->xg_type)) + kfree(xg->xg_busy_extents); +#endif + + if (uninit) + uninit(xg); + + /* drop the mount's active reference */ + xfs_group_rele(xg); + XFS_IS_CORRUPT(mp, atomic_read(&xg->xg_active_ref) > 0); + XFS_IS_CORRUPT(mp, atomic_read(&xg->xg_active_ref) < 0); + kfree_rcu_mightsleep(xg); +} + +int +xfs_group_insert( + struct xfs_mount *mp, + struct xfs_group *xg, + uint32_t index, + enum xfs_group_type type) +{ + int error; + + xg->xg_mount = mp; + xg->xg_gno = index; + xg->xg_type = type; + +#ifdef __KERNEL__ + if (xfs_group_has_extent_busy(mp, type)) { + xg->xg_busy_extents = xfs_extent_busy_alloc(); + if (!xg->xg_busy_extents) + return -ENOMEM; + } + spin_lock_init(&xg->xg_state_lock); + xfs_hooks_init(&xg->xg_rmap_update_hooks); +#endif + xfs_defer_drain_init(&xg->xg_intents_drain); + + /* Active ref owned by mount indicates group is online. */ + atomic_set(&xg->xg_active_ref, 1); + + error = xa_insert(&mp->m_groups[type].xa, index, xg, GFP_KERNEL); + if (error) { + WARN_ON_ONCE(error == -EBUSY); + goto out_drain; + } + + return 0; +out_drain: + xfs_defer_drain_free(&xg->xg_intents_drain); +#ifdef __KERNEL__ + if (xfs_group_has_extent_busy(xg->xg_mount, xg->xg_type)) + kfree(xg->xg_busy_extents); +#endif + return error; +} + +struct xfs_group * +xfs_group_get_by_fsb( + struct xfs_mount *mp, + xfs_fsblock_t fsbno, + enum xfs_group_type type) +{ + return xfs_group_get(mp, xfs_fsb_to_gno(mp, fsbno, type), type); +} diff --git a/fs/xfs/libxfs/xfs_group.h b/fs/xfs/libxfs/xfs_group.h new file mode 100644 index 000000000..4ae638f1c --- /dev/null +++ b/fs/xfs/libxfs/xfs_group.h @@ -0,0 +1,192 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2018 Red Hat, Inc. + */ +#ifndef __LIBXFS_GROUP_H +#define __LIBXFS_GROUP_H 1 + +struct xfs_group { + struct xfs_mount *xg_mount; + uint32_t xg_gno; + enum xfs_group_type xg_type; + atomic_t xg_ref; /* passive reference count */ + atomic_t xg_active_ref; /* active reference count */ + + /* Precalculated geometry info */ + uint32_t xg_block_count; /* max usable gbno */ + uint32_t xg_min_gbno; /* min usable gbno */ + +#ifdef __KERNEL__ + /* -- kernel only structures below this line -- */ + + union { + /* + * For perags and non-zoned RT groups: + * Track freed but not yet committed extents. + */ + struct xfs_extent_busy_tree *xg_busy_extents; + + /* + * For zoned RT groups: + * List of groups that need a zone reset. + * + * The zonegc code forces a log flush of the rtrmap inode before + * resetting the write pointer, so there is no need for + * individual busy extent tracking. + */ + struct xfs_group *xg_next_reset; + }; + + /* + * Bitsets of per-ag metadata that have been checked and/or are sick. + * Callers should hold xg_state_lock before accessing this field. + */ + uint16_t xg_checked; + uint16_t xg_sick; + spinlock_t xg_state_lock; + + /* + * We use xfs_drain to track the number of deferred log intent items + * that have been queued (but not yet processed) so that waiters (e.g. + * scrub) will not lock resources when other threads are in the middle + * of processing a chain of intent items only to find momentary + * inconsistencies. + */ + struct xfs_defer_drain xg_intents_drain; + + /* + * Hook to feed rmapbt updates to an active online repair. + */ + struct xfs_hooks xg_rmap_update_hooks; +#endif /* __KERNEL__ */ +}; + +struct xfs_group *xfs_group_get(struct xfs_mount *mp, uint32_t index, + enum xfs_group_type type); +struct xfs_group *xfs_group_get_by_fsb(struct xfs_mount *mp, + xfs_fsblock_t fsbno, enum xfs_group_type type); +struct xfs_group *xfs_group_hold(struct xfs_group *xg); +void xfs_group_put(struct xfs_group *xg); + +struct xfs_group *xfs_group_grab(struct xfs_mount *mp, uint32_t index, + enum xfs_group_type type); +struct xfs_group *xfs_group_next_range(struct xfs_mount *mp, + struct xfs_group *xg, uint32_t start_index, uint32_t end_index, + enum xfs_group_type type); +struct xfs_group *xfs_group_grab_next_mark(struct xfs_mount *mp, + struct xfs_group *xg, xa_mark_t mark, enum xfs_group_type type); +void xfs_group_rele(struct xfs_group *xg); + +void xfs_group_free(struct xfs_mount *mp, uint32_t index, + enum xfs_group_type type, void (*uninit)(struct xfs_group *xg)); +int xfs_group_insert(struct xfs_mount *mp, struct xfs_group *xg, + uint32_t index, enum xfs_group_type); + +#define xfs_group_set_mark(_xg, _mark) \ + xa_set_mark(&(_xg)->xg_mount->m_groups[(_xg)->xg_type].xa, \ + (_xg)->xg_gno, (_mark)) +#define xfs_group_clear_mark(_xg, _mark) \ + xa_clear_mark(&(_xg)->xg_mount->m_groups[(_xg)->xg_type].xa, \ + (_xg)->xg_gno, (_mark)) +#define xfs_group_marked(_mp, _type, _mark) \ + xa_marked(&(_mp)->m_groups[(_type)].xa, (_mark)) + +static inline xfs_agblock_t +xfs_group_max_blocks( + struct xfs_group *xg) +{ + return xg->xg_mount->m_groups[xg->xg_type].blocks; +} + +static inline xfs_rfsblock_t +xfs_groups_to_rfsbs( + struct xfs_mount *mp, + uint32_t nr_groups, + enum xfs_group_type type) +{ + return (xfs_rfsblock_t)mp->m_groups[type].blocks * nr_groups; +} + +static inline xfs_fsblock_t +xfs_group_start_fsb( + struct xfs_group *xg) +{ + return ((xfs_fsblock_t)xg->xg_gno) << + xg->xg_mount->m_groups[xg->xg_type].blklog; +} + +static inline xfs_fsblock_t +xfs_gbno_to_fsb( + struct xfs_group *xg, + xfs_agblock_t gbno) +{ + return xfs_group_start_fsb(xg) | gbno; +} + +static inline xfs_daddr_t +xfs_gbno_to_daddr( + struct xfs_group *xg, + xfs_agblock_t gbno) +{ + struct xfs_mount *mp = xg->xg_mount; + struct xfs_groups *g = &mp->m_groups[xg->xg_type]; + xfs_fsblock_t fsbno; + + if (g->has_daddr_gaps) + fsbno = xfs_gbno_to_fsb(xg, gbno); + else + fsbno = (xfs_fsblock_t)xg->xg_gno * g->blocks + gbno; + + return XFS_FSB_TO_BB(mp, g->start_fsb + fsbno); +} + +static inline uint32_t +xfs_fsb_to_gno( + struct xfs_mount *mp, + xfs_fsblock_t fsbno, + enum xfs_group_type type) +{ + if (!mp->m_groups[type].blklog) + return 0; + return fsbno >> mp->m_groups[type].blklog; +} + +static inline xfs_agblock_t +xfs_fsb_to_gbno( + struct xfs_mount *mp, + xfs_fsblock_t fsbno, + enum xfs_group_type type) +{ + return fsbno & mp->m_groups[type].blkmask; +} + +static inline bool +xfs_verify_gbno( + struct xfs_group *xg, + uint32_t gbno) +{ + if (gbno >= xg->xg_block_count) + return false; + if (gbno < xg->xg_min_gbno) + return false; + return true; +} + +static inline bool +xfs_verify_gbext( + struct xfs_group *xg, + uint32_t gbno, + uint32_t glen) +{ + uint32_t end; + + if (!xfs_verify_gbno(xg, gbno)) + return false; + if (glen == 0 || check_add_overflow(gbno, glen - 1, &end)) + return false; + if (!xfs_verify_gbno(xg, end)) + return false; + return true; +} + +#endif /* __LIBXFS_GROUP_H */ diff --git a/fs/xfs/libxfs/xfs_health.h b/fs/xfs/libxfs/xfs_health.h new file mode 100644 index 000000000..1d45cf578 --- /dev/null +++ b/fs/xfs/libxfs/xfs_health.h @@ -0,0 +1,297 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright (C) 2019 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_HEALTH_H__ +#define __XFS_HEALTH_H__ + +struct xfs_group; + +/* + * In-Core Filesystem Health Assessments + * ===================================== + * + * We'd like to be able to summarize the current health status of the + * filesystem so that the administrator knows when it's necessary to schedule + * some downtime for repairs. Until then, we would also like to avoid abrupt + * shutdowns due to corrupt metadata. + * + * The online scrub feature evaluates the health of all filesystem metadata. + * When scrub detects corruption in a piece of metadata it will set the + * corresponding sickness flag, and repair will clear it if successful. If + * problems remain at unmount time, we can also request manual intervention by + * logging a notice to run xfs_repair. + * + * Each health tracking group uses a pair of fields for reporting. The + * "checked" field tell us if a given piece of metadata has ever been examined, + * and the "sick" field tells us if that piece was found to need repairs. + * Therefore we can conclude that for a given sick flag value: + * + * - checked && sick => metadata needs repair + * - checked && !sick => metadata is ok + * - !checked && sick => errors have been observed during normal operation, + * but the metadata has not been checked thoroughly + * - !checked && !sick => has not been examined since mount + * + * Evidence of health problems can be sorted into three basic categories: + * + * a) Primary evidence, which signals that something is defective within the + * general grouping of metadata. + * + * b) Secondary evidence, which are side effects of primary problem but are + * not themselves problems. These can be forgotten when the primary + * health problems are addressed. + * + * c) Indirect evidence, which points to something being wrong in another + * group, but we had to release resources and this is all that's left of + * that state. + */ + +struct xfs_mount; +struct xfs_perag; +struct xfs_inode; +struct xfs_fsop_geom; +struct xfs_btree_cur; +struct xfs_da_args; +struct xfs_rtgroup; + +/* Observable health issues for metadata spanning the entire filesystem. */ +#define XFS_SICK_FS_COUNTERS (1 << 0) /* summary counters */ +#define XFS_SICK_FS_UQUOTA (1 << 1) /* user quota */ +#define XFS_SICK_FS_GQUOTA (1 << 2) /* group quota */ +#define XFS_SICK_FS_PQUOTA (1 << 3) /* project quota */ +#define XFS_SICK_FS_QUOTACHECK (1 << 4) /* quota counts */ +#define XFS_SICK_FS_NLINKS (1 << 5) /* inode link counts */ +#define XFS_SICK_FS_METADIR (1 << 6) /* metadata directory tree */ +#define XFS_SICK_FS_METAPATH (1 << 7) /* metadata directory tree path */ + +/* Observable health issues for realtime group metadata. */ +#define XFS_SICK_RG_SUPER (1 << 0) /* rt group superblock */ +#define XFS_SICK_RG_BITMAP (1 << 1) /* rt group bitmap */ +#define XFS_SICK_RG_SUMMARY (1 << 2) /* rt groups summary */ +#define XFS_SICK_RG_RMAPBT (1 << 3) /* reverse mappings */ +#define XFS_SICK_RG_REFCNTBT (1 << 4) /* reference counts */ + +/* Observable health issues for AG metadata. */ +#define XFS_SICK_AG_SB (1 << 0) /* superblock */ +#define XFS_SICK_AG_AGF (1 << 1) /* AGF header */ +#define XFS_SICK_AG_AGFL (1 << 2) /* AGFL header */ +#define XFS_SICK_AG_AGI (1 << 3) /* AGI header */ +#define XFS_SICK_AG_BNOBT (1 << 4) /* free space by block */ +#define XFS_SICK_AG_CNTBT (1 << 5) /* free space by length */ +#define XFS_SICK_AG_INOBT (1 << 6) /* inode index */ +#define XFS_SICK_AG_FINOBT (1 << 7) /* free inode index */ +#define XFS_SICK_AG_RMAPBT (1 << 8) /* reverse mappings */ +#define XFS_SICK_AG_REFCNTBT (1 << 9) /* reference counts */ +#define XFS_SICK_AG_INODES (1 << 10) /* inactivated bad inodes */ + +/* Observable health issues for inode metadata. */ +#define XFS_SICK_INO_CORE (1 << 0) /* inode core */ +#define XFS_SICK_INO_BMBTD (1 << 1) /* data fork */ +#define XFS_SICK_INO_BMBTA (1 << 2) /* attr fork */ +#define XFS_SICK_INO_BMBTC (1 << 3) /* cow fork */ +#define XFS_SICK_INO_DIR (1 << 4) /* directory */ +#define XFS_SICK_INO_XATTR (1 << 5) /* extended attributes */ +#define XFS_SICK_INO_SYMLINK (1 << 6) /* symbolic link remote target */ +#define XFS_SICK_INO_PARENT (1 << 7) /* parent pointers */ + +#define XFS_SICK_INO_BMBTD_ZAPPED (1 << 8) /* data fork erased */ +#define XFS_SICK_INO_BMBTA_ZAPPED (1 << 9) /* attr fork erased */ +#define XFS_SICK_INO_DIR_ZAPPED (1 << 10) /* directory erased */ +#define XFS_SICK_INO_SYMLINK_ZAPPED (1 << 11) /* symlink erased */ + +/* Don't propagate sick status to ag health summary during inactivation */ +#define XFS_SICK_INO_FORGET (1 << 12) +#define XFS_SICK_INO_DIRTREE (1 << 13) /* directory tree structure */ + +/* Primary evidence of health problems in a given group. */ +#define XFS_SICK_FS_PRIMARY (XFS_SICK_FS_COUNTERS | \ + XFS_SICK_FS_UQUOTA | \ + XFS_SICK_FS_GQUOTA | \ + XFS_SICK_FS_PQUOTA | \ + XFS_SICK_FS_QUOTACHECK | \ + XFS_SICK_FS_NLINKS | \ + XFS_SICK_FS_METADIR | \ + XFS_SICK_FS_METAPATH) + +#define XFS_SICK_RG_PRIMARY (XFS_SICK_RG_SUPER | \ + XFS_SICK_RG_BITMAP | \ + XFS_SICK_RG_SUMMARY | \ + XFS_SICK_RG_RMAPBT | \ + XFS_SICK_RG_REFCNTBT) + +#define XFS_SICK_AG_PRIMARY (XFS_SICK_AG_SB | \ + XFS_SICK_AG_AGF | \ + XFS_SICK_AG_AGFL | \ + XFS_SICK_AG_AGI | \ + XFS_SICK_AG_BNOBT | \ + XFS_SICK_AG_CNTBT | \ + XFS_SICK_AG_INOBT | \ + XFS_SICK_AG_FINOBT | \ + XFS_SICK_AG_RMAPBT | \ + XFS_SICK_AG_REFCNTBT) + +#define XFS_SICK_INO_PRIMARY (XFS_SICK_INO_CORE | \ + XFS_SICK_INO_BMBTD | \ + XFS_SICK_INO_BMBTA | \ + XFS_SICK_INO_BMBTC | \ + XFS_SICK_INO_DIR | \ + XFS_SICK_INO_XATTR | \ + XFS_SICK_INO_SYMLINK | \ + XFS_SICK_INO_PARENT | \ + XFS_SICK_INO_DIRTREE) + +#define XFS_SICK_INO_ZAPPED (XFS_SICK_INO_BMBTD_ZAPPED | \ + XFS_SICK_INO_BMBTA_ZAPPED | \ + XFS_SICK_INO_DIR_ZAPPED | \ + XFS_SICK_INO_SYMLINK_ZAPPED) + +/* Secondary state related to (but not primary evidence of) health problems. */ +#define XFS_SICK_FS_SECONDARY (0) +#define XFS_SICK_RG_SECONDARY (0) +#define XFS_SICK_AG_SECONDARY (0) +#define XFS_SICK_INO_SECONDARY (XFS_SICK_INO_FORGET) + +/* Evidence of health problems elsewhere. */ +#define XFS_SICK_FS_INDIRECT (0) +#define XFS_SICK_RG_INDIRECT (0) +#define XFS_SICK_AG_INDIRECT (XFS_SICK_AG_INODES) +#define XFS_SICK_INO_INDIRECT (0) + +/* All health masks. */ +#define XFS_SICK_FS_ALL (XFS_SICK_FS_PRIMARY | \ + XFS_SICK_FS_SECONDARY | \ + XFS_SICK_FS_INDIRECT) + +#define XFS_SICK_RG_ALL (XFS_SICK_RG_PRIMARY | \ + XFS_SICK_RG_SECONDARY | \ + XFS_SICK_RG_INDIRECT) + +#define XFS_SICK_AG_ALL (XFS_SICK_AG_PRIMARY | \ + XFS_SICK_AG_SECONDARY | \ + XFS_SICK_AG_INDIRECT) + +#define XFS_SICK_INO_ALL (XFS_SICK_INO_PRIMARY | \ + XFS_SICK_INO_SECONDARY | \ + XFS_SICK_INO_INDIRECT | \ + XFS_SICK_INO_ZAPPED) + +/* + * These functions must be provided by the xfs implementation. Function + * behavior with respect to the first argument should be as follows: + * + * xfs_*_mark_sick: Set the sick flags and do not set checked flags. + * Runtime code should call this upon encountering + * a corruption. + * + * xfs_*_mark_corrupt: Set the sick and checked flags simultaneously. + * Fsck tools should call this when corruption is + * found. + * + * xfs_*_mark_healthy: Clear the sick flags and set the checked flags. + * Fsck tools should call this after correcting errors. + * + * xfs_*_measure_sickness: Return the sick and check status in the provided + * out parameters. + */ + +void xfs_fs_mark_sick(struct xfs_mount *mp, unsigned int mask); +void xfs_fs_mark_corrupt(struct xfs_mount *mp, unsigned int mask); +void xfs_fs_mark_healthy(struct xfs_mount *mp, unsigned int mask); +void xfs_fs_measure_sickness(struct xfs_mount *mp, unsigned int *sick, + unsigned int *checked); + +void xfs_rgno_mark_sick(struct xfs_mount *mp, xfs_rgnumber_t rgno, + unsigned int mask); + +void xfs_agno_mark_sick(struct xfs_mount *mp, xfs_agnumber_t agno, + unsigned int mask); +void xfs_group_mark_sick(struct xfs_group *xg, unsigned int mask); +#define xfs_ag_mark_sick(pag, mask) \ + xfs_group_mark_sick(pag_group(pag), (mask)) +void xfs_group_mark_corrupt(struct xfs_group *xg, unsigned int mask); +void xfs_group_mark_healthy(struct xfs_group *xg, unsigned int mask); +void xfs_group_measure_sickness(struct xfs_group *xg, unsigned int *sick, + unsigned int *checked); + +void xfs_inode_mark_sick(struct xfs_inode *ip, unsigned int mask); +void xfs_inode_mark_corrupt(struct xfs_inode *ip, unsigned int mask); +void xfs_inode_mark_healthy(struct xfs_inode *ip, unsigned int mask); +void xfs_inode_measure_sickness(struct xfs_inode *ip, unsigned int *sick, + unsigned int *checked); + +void xfs_health_unmount(struct xfs_mount *mp); +void xfs_bmap_mark_sick(struct xfs_inode *ip, int whichfork); +void xfs_btree_mark_sick(struct xfs_btree_cur *cur); +void xfs_dirattr_mark_sick(struct xfs_inode *ip, int whichfork); +void xfs_da_mark_sick(struct xfs_da_args *args); + +/* Now some helpers. */ + +static inline bool +xfs_fs_has_sickness(struct xfs_mount *mp, unsigned int mask) +{ + unsigned int sick, checked; + + xfs_fs_measure_sickness(mp, &sick, &checked); + return sick & mask; +} + +static inline bool +xfs_group_has_sickness( + struct xfs_group *xg, + unsigned int mask) +{ + unsigned int sick, checked; + + xfs_group_measure_sickness(xg, &sick, &checked); + return sick & mask; +} + +#define xfs_ag_has_sickness(pag, mask) \ + xfs_group_has_sickness(pag_group(pag), (mask)) +#define xfs_ag_is_healthy(pag) \ + (!xfs_ag_has_sickness((pag), UINT_MAX)) + +#define xfs_rtgroup_has_sickness(rtg, mask) \ + xfs_group_has_sickness(rtg_group(rtg), (mask)) +#define xfs_rtgroup_is_healthy(rtg) \ + (!xfs_rtgroup_has_sickness((rtg), UINT_MAX)) + +static inline bool +xfs_inode_has_sickness(struct xfs_inode *ip, unsigned int mask) +{ + unsigned int sick, checked; + + xfs_inode_measure_sickness(ip, &sick, &checked); + return sick & mask; +} + +static inline bool +xfs_fs_is_healthy(struct xfs_mount *mp) +{ + return !xfs_fs_has_sickness(mp, -1U); +} + +static inline bool +xfs_inode_is_healthy(struct xfs_inode *ip) +{ + return !xfs_inode_has_sickness(ip, -1U); +} + +void xfs_fsop_geom_health(struct xfs_mount *mp, struct xfs_fsop_geom *geo); +void xfs_ag_geom_health(struct xfs_perag *pag, struct xfs_ag_geometry *ageo); +void xfs_rtgroup_geom_health(struct xfs_rtgroup *rtg, + struct xfs_rtgroup_geometry *rgeo); +void xfs_bulkstat_health(struct xfs_inode *ip, struct xfs_bulkstat *bs); + +#define xfs_metadata_is_sick(error) \ + (unlikely((error) == -EFSCORRUPTED || (error) == -EFSBADCRC)) + +unsigned int xfs_healthmon_inode_mask(unsigned int sick_mask); +unsigned int xfs_healthmon_rtgroup_mask(unsigned int sick_mask); +unsigned int xfs_healthmon_perag_mask(unsigned int sick_mask); +unsigned int xfs_healthmon_fs_mask(unsigned int sick_mask); + +#endif /* __XFS_HEALTH_H__ */ diff --git a/fs/xfs/libxfs/xfs_ialloc.c b/fs/xfs/libxfs/xfs_ialloc.c new file mode 100644 index 000000000..58dac4d50 --- /dev/null +++ b/fs/xfs/libxfs/xfs_ialloc.c @@ -0,0 +1,3185 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_btree.h" +#include "xfs_ialloc.h" +#include "xfs_ialloc_btree.h" +#include "xfs_alloc.h" +#include "xfs_errortag.h" +#include "xfs_error.h" +#include "xfs_bmap.h" +#include "xfs_trans.h" +#include "xfs_buf_item.h" +#include "xfs_icreate_item.h" +#include "xfs_icache.h" +#include "xfs_trace.h" +#include "xfs_log.h" +#include "xfs_rmap.h" +#include "xfs_ag.h" +#include "xfs_health.h" + +/* + * Lookup a record by ino in the btree given by cur. + */ +int /* error */ +xfs_inobt_lookup( + struct xfs_btree_cur *cur, /* btree cursor */ + xfs_agino_t ino, /* starting inode of chunk */ + xfs_lookup_t dir, /* <=, >=, == */ + int *stat) /* success/failure */ +{ + cur->bc_rec.i.ir_startino = ino; + cur->bc_rec.i.ir_holemask = 0; + cur->bc_rec.i.ir_count = 0; + cur->bc_rec.i.ir_freecount = 0; + cur->bc_rec.i.ir_free = 0; + return xfs_btree_lookup(cur, dir, stat); +} + +/* + * Update the record referred to by cur to the value given. + * This either works (return 0) or gets an EFSCORRUPTED error. + */ +STATIC int /* error */ +xfs_inobt_update( + struct xfs_btree_cur *cur, /* btree cursor */ + xfs_inobt_rec_incore_t *irec) /* btree record */ +{ + union xfs_btree_rec rec; + + rec.inobt.ir_startino = cpu_to_be32(irec->ir_startino); + if (xfs_has_sparseinodes(cur->bc_mp)) { + rec.inobt.ir_u.sp.ir_holemask = cpu_to_be16(irec->ir_holemask); + rec.inobt.ir_u.sp.ir_count = irec->ir_count; + rec.inobt.ir_u.sp.ir_freecount = irec->ir_freecount; + } else { + /* ir_holemask/ir_count not supported on-disk */ + rec.inobt.ir_u.f.ir_freecount = cpu_to_be32(irec->ir_freecount); + } + rec.inobt.ir_free = cpu_to_be64(irec->ir_free); + return xfs_btree_update(cur, &rec); +} + +/* Convert on-disk btree record to incore inobt record. */ +void +xfs_inobt_btrec_to_irec( + struct xfs_mount *mp, + const union xfs_btree_rec *rec, + struct xfs_inobt_rec_incore *irec) +{ + irec->ir_startino = be32_to_cpu(rec->inobt.ir_startino); + if (xfs_has_sparseinodes(mp)) { + irec->ir_holemask = be16_to_cpu(rec->inobt.ir_u.sp.ir_holemask); + irec->ir_count = rec->inobt.ir_u.sp.ir_count; + irec->ir_freecount = rec->inobt.ir_u.sp.ir_freecount; + } else { + /* + * ir_holemask/ir_count not supported on-disk. Fill in hardcoded + * values for full inode chunks. + */ + irec->ir_holemask = XFS_INOBT_HOLEMASK_FULL; + irec->ir_count = XFS_INODES_PER_CHUNK; + irec->ir_freecount = + be32_to_cpu(rec->inobt.ir_u.f.ir_freecount); + } + irec->ir_free = be64_to_cpu(rec->inobt.ir_free); +} + +/* Compute the freecount of an incore inode record. */ +uint8_t +xfs_inobt_rec_freecount( + const struct xfs_inobt_rec_incore *irec) +{ + uint64_t realfree = irec->ir_free; + + if (xfs_inobt_issparse(irec->ir_holemask)) + realfree &= xfs_inobt_irec_to_allocmask(irec); + return hweight64(realfree); +} + +/* Simple checks for inode records. */ +xfs_failaddr_t +xfs_inobt_check_irec( + struct xfs_perag *pag, + const struct xfs_inobt_rec_incore *irec) +{ + /* Record has to be properly aligned within the AG. */ + if (!xfs_verify_agino(pag, irec->ir_startino)) + return __this_address; + if (!xfs_verify_agino(pag, + irec->ir_startino + XFS_INODES_PER_CHUNK - 1)) + return __this_address; + if (irec->ir_count < XFS_INODES_PER_HOLEMASK_BIT || + irec->ir_count > XFS_INODES_PER_CHUNK) + return __this_address; + if (irec->ir_freecount > XFS_INODES_PER_CHUNK) + return __this_address; + + if (xfs_inobt_rec_freecount(irec) != irec->ir_freecount) + return __this_address; + + return NULL; +} + +static inline int +xfs_inobt_complain_bad_rec( + struct xfs_btree_cur *cur, + xfs_failaddr_t fa, + const struct xfs_inobt_rec_incore *irec) +{ + struct xfs_mount *mp = cur->bc_mp; + + xfs_warn(mp, + "%sbt record corruption in AG %d detected at %pS!", + cur->bc_ops->name, cur->bc_group->xg_gno, fa); + xfs_warn(mp, +"start inode 0x%x, count 0x%x, free 0x%x freemask 0x%llx, holemask 0x%x", + irec->ir_startino, irec->ir_count, irec->ir_freecount, + irec->ir_free, irec->ir_holemask); + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; +} + +/* + * Get the data from the pointed-to record. + */ +int +xfs_inobt_get_rec( + struct xfs_btree_cur *cur, + struct xfs_inobt_rec_incore *irec, + int *stat) +{ + struct xfs_mount *mp = cur->bc_mp; + union xfs_btree_rec *rec; + xfs_failaddr_t fa; + int error; + + error = xfs_btree_get_rec(cur, &rec, stat); + if (error || *stat == 0) + return error; + + xfs_inobt_btrec_to_irec(mp, rec, irec); + fa = xfs_inobt_check_irec(to_perag(cur->bc_group), irec); + if (fa) + return xfs_inobt_complain_bad_rec(cur, fa, irec); + + return 0; +} + +/* + * Insert a single inobt record. Cursor must already point to desired location. + */ +int +xfs_inobt_insert_rec( + struct xfs_btree_cur *cur, + uint16_t holemask, + uint8_t count, + int32_t freecount, + xfs_inofree_t free, + int *stat) +{ + cur->bc_rec.i.ir_holemask = holemask; + cur->bc_rec.i.ir_count = count; + cur->bc_rec.i.ir_freecount = freecount; + cur->bc_rec.i.ir_free = free; + return xfs_btree_insert(cur, stat); +} + +/* + * Insert records describing a newly allocated inode chunk into the inobt. + */ +STATIC int +xfs_inobt_insert( + struct xfs_perag *pag, + struct xfs_trans *tp, + struct xfs_buf *agbp, + xfs_agino_t newino, + xfs_agino_t newlen, + bool is_finobt) +{ + struct xfs_btree_cur *cur; + xfs_agino_t thisino; + int i; + int error; + + if (is_finobt) + cur = xfs_finobt_init_cursor(pag, tp, agbp); + else + cur = xfs_inobt_init_cursor(pag, tp, agbp); + + for (thisino = newino; + thisino < newino + newlen; + thisino += XFS_INODES_PER_CHUNK) { + error = xfs_inobt_lookup(cur, thisino, XFS_LOOKUP_EQ, &i); + if (error) { + xfs_btree_del_cursor(cur, XFS_BTREE_ERROR); + return error; + } + ASSERT(i == 0); + + error = xfs_inobt_insert_rec(cur, XFS_INOBT_HOLEMASK_FULL, + XFS_INODES_PER_CHUNK, + XFS_INODES_PER_CHUNK, + XFS_INOBT_ALL_FREE, &i); + if (error) { + xfs_btree_del_cursor(cur, XFS_BTREE_ERROR); + return error; + } + ASSERT(i == 1); + } + + xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR); + + return 0; +} + +/* + * Verify that the number of free inodes in the AGI is correct. + */ +#ifdef DEBUG +static int +xfs_check_agi_freecount( + struct xfs_btree_cur *cur) +{ + if (cur->bc_nlevels == 1) { + xfs_inobt_rec_incore_t rec; + int freecount = 0; + int error; + int i; + + error = xfs_inobt_lookup(cur, 0, XFS_LOOKUP_GE, &i); + if (error) + return error; + + do { + error = xfs_inobt_get_rec(cur, &rec, &i); + if (error) + return error; + + if (i) { + freecount += rec.ir_freecount; + error = xfs_btree_increment(cur, 0, &i); + if (error) + return error; + } + } while (i == 1); + + if (!xfs_is_shutdown(cur->bc_mp)) { + ASSERT(freecount == + to_perag(cur->bc_group)->pagi_freecount); + } + } + return 0; +} +#else +#define xfs_check_agi_freecount(cur) 0 +#endif + +/* + * Initialise a new set of inodes. When called without a transaction context + * (e.g. from recovery) we initiate a delayed write of the inode buffers rather + * than logging them (which in a transaction context puts them into the AIL + * for writeback rather than the xfsbufd queue). + */ +int +xfs_ialloc_inode_init( + struct xfs_mount *mp, + struct xfs_trans *tp, + struct list_head *buffer_list, + int icount, + xfs_agnumber_t agno, + xfs_agblock_t agbno, + xfs_agblock_t length, + unsigned int gen) +{ + struct xfs_buf *fbuf; + struct xfs_dinode *free; + int nbufs; + int version; + int i, j; + xfs_daddr_t d; + xfs_ino_t ino = 0; + int error; + + /* + * Loop over the new block(s), filling in the inodes. For small block + * sizes, manipulate the inodes in buffers which are multiples of the + * blocks size. + */ + nbufs = length / M_IGEO(mp)->blocks_per_cluster; + + /* + * Figure out what version number to use in the inodes we create. If + * the superblock version has caught up to the one that supports the new + * inode format, then use the new inode version. Otherwise use the old + * version so that old kernels will continue to be able to use the file + * system. + * + * For v3 inodes, we also need to write the inode number into the inode, + * so calculate the first inode number of the chunk here as + * XFS_AGB_TO_AGINO() only works within a filesystem block, not + * across multiple filesystem blocks (such as a cluster) and so cannot + * be used in the cluster buffer loop below. + * + * Further, because we are writing the inode directly into the buffer + * and calculating a CRC on the entire inode, we have ot log the entire + * inode so that the entire range the CRC covers is present in the log. + * That means for v3 inode we log the entire buffer rather than just the + * inode cores. + */ + if (xfs_has_v3inodes(mp)) { + version = 3; + ino = XFS_AGINO_TO_INO(mp, agno, XFS_AGB_TO_AGINO(mp, agbno)); + + /* + * log the initialisation that is about to take place as an + * logical operation. This means the transaction does not + * need to log the physical changes to the inode buffers as log + * recovery will know what initialisation is actually needed. + * Hence we only need to log the buffers as "ordered" buffers so + * they track in the AIL as if they were physically logged. + */ + if (tp) + xfs_icreate_log(tp, agno, agbno, icount, + mp->m_sb.sb_inodesize, length, gen); + } else + version = 2; + + for (j = 0; j < nbufs; j++) { + /* + * Get the block. + */ + d = XFS_AGB_TO_DADDR(mp, agno, agbno + + (j * M_IGEO(mp)->blocks_per_cluster)); + error = xfs_trans_get_buf(tp, mp->m_ddev_targp, d, + mp->m_bsize * M_IGEO(mp)->blocks_per_cluster, + 0, &fbuf); + if (error) + return error; + + /* Initialize the inode buffers and log them appropriately. */ + fbuf->b_ops = &xfs_inode_buf_ops; + xfs_buf_zero(fbuf, 0, BBTOB(fbuf->b_length)); + for (i = 0; i < M_IGEO(mp)->inodes_per_cluster; i++) { + int ioffset = i << mp->m_sb.sb_inodelog; + + free = xfs_make_iptr(mp, fbuf, i); + free->di_magic = cpu_to_be16(XFS_DINODE_MAGIC); + free->di_version = version; + free->di_gen = cpu_to_be32(gen); + free->di_next_unlinked = cpu_to_be32(NULLAGINO); + + if (version == 3) { + free->di_ino = cpu_to_be64(ino); + ino++; + uuid_copy(&free->di_uuid, + &mp->m_sb.sb_meta_uuid); + xfs_dinode_calc_crc(mp, free); + } else if (tp) { + /* just log the inode core */ + xfs_trans_log_buf(tp, fbuf, ioffset, + ioffset + XFS_DINODE_SIZE(mp) - 1); + } + } + + if (tp) { + /* + * Mark the buffer as an inode allocation buffer so it + * sticks in AIL at the point of this allocation + * transaction. This ensures the they are on disk before + * the tail of the log can be moved past this + * transaction (i.e. by preventing relogging from moving + * it forward in the log). + */ + xfs_trans_inode_alloc_buf(tp, fbuf); + if (version == 3) { + /* + * Mark the buffer as ordered so that they are + * not physically logged in the transaction but + * still tracked in the AIL as part of the + * transaction and pin the log appropriately. + */ + xfs_trans_ordered_buf(tp, fbuf); + } + } else { + xfs_buf_set_uptodate(fbuf); + xfs_buf_delwri_queue(fbuf, buffer_list); + xfs_buf_relse(fbuf); + } + } + return 0; +} + +/* + * Align startino and allocmask for a recently allocated sparse chunk such that + * they are fit for insertion (or merge) into the on-disk inode btrees. + * + * Background: + * + * When enabled, sparse inode support increases the inode alignment from cluster + * size to inode chunk size. This means that the minimum range between two + * non-adjacent inode records in the inobt is large enough for a full inode + * record. This allows for cluster sized, cluster aligned block allocation + * without need to worry about whether the resulting inode record overlaps with + * another record in the tree. Without this basic rule, we would have to deal + * with the consequences of overlap by potentially undoing recent allocations in + * the inode allocation codepath. + * + * Because of this alignment rule (which is enforced on mount), there are two + * inobt possibilities for newly allocated sparse chunks. One is that the + * aligned inode record for the chunk covers a range of inodes not already + * covered in the inobt (i.e., it is safe to insert a new sparse record). The + * other is that a record already exists at the aligned startino that considers + * the newly allocated range as sparse. In the latter case, record content is + * merged in hope that sparse inode chunks fill to full chunks over time. + */ +STATIC void +xfs_align_sparse_ino( + struct xfs_mount *mp, + xfs_agino_t *startino, + uint16_t *allocmask) +{ + xfs_agblock_t agbno; + xfs_agblock_t mod; + int offset; + + agbno = XFS_AGINO_TO_AGBNO(mp, *startino); + mod = agbno % mp->m_sb.sb_inoalignmt; + if (!mod) + return; + + /* calculate the inode offset and align startino */ + offset = XFS_AGB_TO_AGINO(mp, mod); + *startino -= offset; + + /* + * Since startino has been aligned down, left shift allocmask such that + * it continues to represent the same physical inodes relative to the + * new startino. + */ + *allocmask <<= offset / XFS_INODES_PER_HOLEMASK_BIT; +} + +/* + * Determine whether the source inode record can merge into the target. Both + * records must be sparse, the inode ranges must match and there must be no + * allocation overlap between the records. + */ +STATIC bool +__xfs_inobt_can_merge( + struct xfs_inobt_rec_incore *trec, /* tgt record */ + struct xfs_inobt_rec_incore *srec) /* src record */ +{ + uint64_t talloc; + uint64_t salloc; + + /* records must cover the same inode range */ + if (trec->ir_startino != srec->ir_startino) + return false; + + /* both records must be sparse */ + if (!xfs_inobt_issparse(trec->ir_holemask) || + !xfs_inobt_issparse(srec->ir_holemask)) + return false; + + /* both records must track some inodes */ + if (!trec->ir_count || !srec->ir_count) + return false; + + /* can't exceed capacity of a full record */ + if (trec->ir_count + srec->ir_count > XFS_INODES_PER_CHUNK) + return false; + + /* verify there is no allocation overlap */ + talloc = xfs_inobt_irec_to_allocmask(trec); + salloc = xfs_inobt_irec_to_allocmask(srec); + if (talloc & salloc) + return false; + + return true; +} + +/* + * Merge the source inode record into the target. The caller must call + * __xfs_inobt_can_merge() to ensure the merge is valid. + */ +STATIC void +__xfs_inobt_rec_merge( + struct xfs_inobt_rec_incore *trec, /* target */ + struct xfs_inobt_rec_incore *srec) /* src */ +{ + ASSERT(trec->ir_startino == srec->ir_startino); + + /* combine the counts */ + trec->ir_count += srec->ir_count; + trec->ir_freecount += srec->ir_freecount; + + /* + * Merge the holemask and free mask. For both fields, 0 bits refer to + * allocated inodes. We combine the allocated ranges with bitwise AND. + */ + trec->ir_holemask &= srec->ir_holemask; + trec->ir_free &= srec->ir_free; +} + +/* + * Insert a new sparse inode chunk into the associated inode allocation btree. + * The inode record for the sparse chunk is pre-aligned to a startino that + * should match any pre-existing sparse inode record in the tree. This allows + * sparse chunks to fill over time. + * + * If no preexisting record exists, the provided record is inserted. + * If there is a preexisting record, the provided record is merged with the + * existing record and updated in place. The merged record is returned in nrec. + * + * It is considered corruption if a merge is requested and not possible. Given + * the sparse inode alignment constraints, this should never happen. + */ +STATIC int +xfs_inobt_insert_sprec( + struct xfs_perag *pag, + struct xfs_trans *tp, + struct xfs_buf *agbp, + struct xfs_inobt_rec_incore *nrec) /* in/out: new/merged rec. */ +{ + struct xfs_mount *mp = pag_mount(pag); + struct xfs_btree_cur *cur; + int error; + int i; + struct xfs_inobt_rec_incore rec; + + cur = xfs_inobt_init_cursor(pag, tp, agbp); + + /* the new record is pre-aligned so we know where to look */ + error = xfs_inobt_lookup(cur, nrec->ir_startino, XFS_LOOKUP_EQ, &i); + if (error) + goto error; + /* if nothing there, insert a new record and return */ + if (i == 0) { + error = xfs_inobt_insert_rec(cur, nrec->ir_holemask, + nrec->ir_count, nrec->ir_freecount, + nrec->ir_free, &i); + if (error) + goto error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error; + } + + goto out; + } + + /* + * A record exists at this startino. Merge the records. + */ + error = xfs_inobt_get_rec(cur, &rec, &i); + if (error) + goto error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error; + } + if (XFS_IS_CORRUPT(mp, rec.ir_startino != nrec->ir_startino)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error; + } + + /* + * This should never fail. If we have coexisting records that + * cannot merge, something is seriously wrong. + */ + if (XFS_IS_CORRUPT(mp, !__xfs_inobt_can_merge(nrec, &rec))) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error; + } + + trace_xfs_irec_merge_pre(pag, &rec, nrec); + + /* merge to nrec to output the updated record */ + __xfs_inobt_rec_merge(nrec, &rec); + + trace_xfs_irec_merge_post(pag, nrec); + + error = xfs_inobt_rec_check_count(mp, nrec); + if (error) + goto error; + + error = xfs_inobt_update(cur, nrec); + if (error) + goto error; + +out: + xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR); + return 0; +error: + xfs_btree_del_cursor(cur, XFS_BTREE_ERROR); + return error; +} + +/* + * Insert a new sparse inode chunk into the free inode btree. The inode + * record for the sparse chunk is pre-aligned to a startino that should match + * any pre-existing sparse inode record in the tree. This allows sparse chunks + * to fill over time. + * + * The new record is always inserted, overwriting a pre-existing record if + * there is one. + */ +STATIC int +xfs_finobt_insert_sprec( + struct xfs_perag *pag, + struct xfs_trans *tp, + struct xfs_buf *agbp, + struct xfs_inobt_rec_incore *nrec) /* in/out: new rec. */ +{ + struct xfs_mount *mp = pag_mount(pag); + struct xfs_btree_cur *cur; + int error; + int i; + + cur = xfs_finobt_init_cursor(pag, tp, agbp); + + /* the new record is pre-aligned so we know where to look */ + error = xfs_inobt_lookup(cur, nrec->ir_startino, XFS_LOOKUP_EQ, &i); + if (error) + goto error; + /* if nothing there, insert a new record and return */ + if (i == 0) { + error = xfs_inobt_insert_rec(cur, nrec->ir_holemask, + nrec->ir_count, nrec->ir_freecount, + nrec->ir_free, &i); + if (error) + goto error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error; + } + } else { + error = xfs_inobt_update(cur, nrec); + if (error) + goto error; + } + + xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR); + return 0; +error: + xfs_btree_del_cursor(cur, XFS_BTREE_ERROR); + return error; +} + + +/* + * Allocate new inodes in the allocation group specified by agbp. Returns 0 if + * inodes were allocated in this AG; -EAGAIN if there was no space in this AG so + * the caller knows it can try another AG, a hard -ENOSPC when over the maximum + * inode count threshold, or the usual negative error code for other errors. + */ +STATIC int +xfs_ialloc_ag_alloc( + struct xfs_perag *pag, + struct xfs_trans *tp, + struct xfs_buf *agbp) +{ + struct xfs_agi *agi; + struct xfs_alloc_arg args; + int error; + xfs_agino_t newino; /* new first inode's number */ + xfs_agino_t newlen; /* new number of inodes */ + int isaligned = 0; /* inode allocation at stripe */ + /* unit boundary */ + /* init. to full chunk */ + struct xfs_inobt_rec_incore rec; + struct xfs_ino_geometry *igeo = M_IGEO(tp->t_mountp); + uint16_t allocmask = (uint16_t) -1; + int do_sparse = 0; + + memset(&args, 0, sizeof(args)); + args.tp = tp; + args.mp = tp->t_mountp; + args.fsbno = NULLFSBLOCK; + args.oinfo = XFS_RMAP_OINFO_INODES; + args.pag = pag; + +#ifdef DEBUG + /* randomly do sparse inode allocations */ + if (xfs_has_sparseinodes(tp->t_mountp) && + igeo->ialloc_min_blks < igeo->ialloc_blks) + do_sparse = get_random_u32_below(2); +#endif + + /* + * Locking will ensure that we don't have two callers in here + * at one time. + */ + newlen = igeo->ialloc_inos; + if (igeo->maxicount && + percpu_counter_read_positive(&args.mp->m_icount) + newlen > + igeo->maxicount) + return -ENOSPC; + args.minlen = args.maxlen = igeo->ialloc_blks; + /* + * First try to allocate inodes contiguous with the last-allocated + * chunk of inodes. If the filesystem is striped, this will fill + * an entire stripe unit with inodes. + */ + agi = agbp->b_addr; + newino = be32_to_cpu(agi->agi_newino); + args.agbno = XFS_AGINO_TO_AGBNO(args.mp, newino) + + igeo->ialloc_blks; + if (do_sparse) + goto sparse_alloc; + if (likely(newino != NULLAGINO && + (args.agbno < be32_to_cpu(agi->agi_length)))) { + args.prod = 1; + + /* + * We need to take into account alignment here to ensure that + * we don't modify the free list if we fail to have an exact + * block. If we don't have an exact match, and every oher + * attempt allocation attempt fails, we'll end up cancelling + * a dirty transaction and shutting down. + * + * For an exact allocation, alignment must be 1, + * however we need to take cluster alignment into account when + * fixing up the freelist. Use the minalignslop field to + * indicate that extra blocks might be required for alignment, + * but not to use them in the actual exact allocation. + */ + args.alignment = 1; + args.minalignslop = igeo->cluster_align - 1; + + /* Allow space for the inode btree to split. */ + args.minleft = igeo->inobt_maxlevels; + error = xfs_alloc_vextent_exact_bno(&args, + xfs_agbno_to_fsb(pag, args.agbno)); + if (error) + return error; + + /* + * This request might have dirtied the transaction if the AG can + * satisfy the request, but the exact block was not available. + * If the allocation did fail, subsequent requests will relax + * the exact agbno requirement and increase the alignment + * instead. It is critical that the total size of the request + * (len + alignment + slop) does not increase from this point + * on, so reset minalignslop to ensure it is not included in + * subsequent requests. + */ + args.minalignslop = 0; + } + + if (unlikely(args.fsbno == NULLFSBLOCK)) { + /* + * Set the alignment for the allocation. + * If stripe alignment is turned on then align at stripe unit + * boundary. + * If the cluster size is smaller than a filesystem block + * then we're doing I/O for inodes in filesystem block size + * pieces, so don't need alignment anyway. + */ + isaligned = 0; + if (igeo->ialloc_align) { + ASSERT(!xfs_has_noalign(args.mp)); + args.alignment = args.mp->m_dalign; + isaligned = 1; + } else + args.alignment = igeo->cluster_align; + /* + * Allocate a fixed-size extent of inodes. + */ + args.prod = 1; + /* + * Allow space for the inode btree to split. + */ + args.minleft = igeo->inobt_maxlevels; + error = xfs_alloc_vextent_near_bno(&args, + xfs_agbno_to_fsb(pag, + be32_to_cpu(agi->agi_root))); + if (error) + return error; + } + + /* + * If stripe alignment is turned on, then try again with cluster + * alignment. + */ + if (isaligned && args.fsbno == NULLFSBLOCK) { + args.alignment = igeo->cluster_align; + error = xfs_alloc_vextent_near_bno(&args, + xfs_agbno_to_fsb(pag, + be32_to_cpu(agi->agi_root))); + if (error) + return error; + } + + /* + * Finally, try a sparse allocation if the filesystem supports it and + * the sparse allocation length is smaller than a full chunk. + */ + if (xfs_has_sparseinodes(args.mp) && + igeo->ialloc_min_blks < igeo->ialloc_blks && + args.fsbno == NULLFSBLOCK) { +sparse_alloc: + args.alignment = args.mp->m_sb.sb_spino_align; + args.prod = 1; + + args.minlen = igeo->ialloc_min_blks; + args.maxlen = args.minlen; + + /* + * The inode record will be aligned to full chunk size. We must + * prevent sparse allocation from AG boundaries that result in + * invalid inode records, such as records that start at agbno 0 + * or extend beyond the AG. + * + * Set min agbno to the first chunk aligned, non-zero agbno and + * max to one less than the last chunk aligned agbno from the + * end of the AG. We subtract 1 from max so that the cluster + * allocation alignment takes over and allows allocation within + * the last full inode chunk in the AG. + */ + args.min_agbno = args.mp->m_sb.sb_inoalignmt; + args.max_agbno = round_down(xfs_ag_block_count(args.mp, + pag_agno(pag)), + args.mp->m_sb.sb_inoalignmt) - 1; + + error = xfs_alloc_vextent_near_bno(&args, + xfs_agbno_to_fsb(pag, + be32_to_cpu(agi->agi_root))); + if (error) + return error; + + newlen = XFS_AGB_TO_AGINO(args.mp, args.len); + ASSERT(newlen <= XFS_INODES_PER_CHUNK); + allocmask = (1 << (newlen / XFS_INODES_PER_HOLEMASK_BIT)) - 1; + } + + if (args.fsbno == NULLFSBLOCK) + return -EAGAIN; + + ASSERT(args.len == args.minlen); + + /* + * Stamp and write the inode buffers. + * + * Seed the new inode cluster with a random generation number. This + * prevents short-term reuse of generation numbers if a chunk is + * freed and then immediately reallocated. We use random numbers + * rather than a linear progression to prevent the next generation + * number from being easily guessable. + */ + error = xfs_ialloc_inode_init(args.mp, tp, NULL, newlen, pag_agno(pag), + args.agbno, args.len, get_random_u32()); + + if (error) + return error; + /* + * Convert the results. + */ + newino = XFS_AGB_TO_AGINO(args.mp, args.agbno); + + if (xfs_inobt_issparse(~allocmask)) { + /* + * We've allocated a sparse chunk. Align the startino and mask. + */ + xfs_align_sparse_ino(args.mp, &newino, &allocmask); + + rec.ir_startino = newino; + rec.ir_holemask = ~allocmask; + rec.ir_count = newlen; + rec.ir_freecount = newlen; + rec.ir_free = XFS_INOBT_ALL_FREE; + + /* + * Insert the sparse record into the inobt and allow for a merge + * if necessary. If a merge does occur, rec is updated to the + * merged record. + */ + error = xfs_inobt_insert_sprec(pag, tp, agbp, &rec); + if (error == -EFSCORRUPTED) { + xfs_alert(args.mp, + "invalid sparse inode record: ino 0x%llx holemask 0x%x count %u", + xfs_agino_to_ino(pag, rec.ir_startino), + rec.ir_holemask, rec.ir_count); + xfs_force_shutdown(args.mp, SHUTDOWN_CORRUPT_INCORE); + } + if (error) + return error; + + /* + * We can't merge the part we've just allocated as for the inobt + * due to finobt semantics. The original record may or may not + * exist independent of whether physical inodes exist in this + * sparse chunk. + * + * We must update the finobt record based on the inobt record. + * rec contains the fully merged and up to date inobt record + * from the previous call. Set merge false to replace any + * existing record with this one. + */ + if (xfs_has_finobt(args.mp)) { + error = xfs_finobt_insert_sprec(pag, tp, agbp, &rec); + if (error) + return error; + } + } else { + /* full chunk - insert new records to both btrees */ + error = xfs_inobt_insert(pag, tp, agbp, newino, newlen, false); + if (error) + return error; + + if (xfs_has_finobt(args.mp)) { + error = xfs_inobt_insert(pag, tp, agbp, newino, + newlen, true); + if (error) + return error; + } + } + + /* + * Update AGI counts and newino. + */ + be32_add_cpu(&agi->agi_count, newlen); + be32_add_cpu(&agi->agi_freecount, newlen); + pag->pagi_freecount += newlen; + pag->pagi_count += newlen; + agi->agi_newino = cpu_to_be32(newino); + + /* + * Log allocation group header fields + */ + xfs_ialloc_log_agi(tp, agbp, + XFS_AGI_COUNT | XFS_AGI_FREECOUNT | XFS_AGI_NEWINO); + /* + * Modify/log superblock values for inode count and inode free count. + */ + xfs_trans_mod_sb(tp, XFS_TRANS_SB_ICOUNT, (long)newlen); + xfs_trans_mod_sb(tp, XFS_TRANS_SB_IFREE, (long)newlen); + return 0; +} + +/* + * Try to retrieve the next record to the left/right from the current one. + */ +STATIC int +xfs_ialloc_next_rec( + struct xfs_btree_cur *cur, + xfs_inobt_rec_incore_t *rec, + int *done, + int left) +{ + int error; + int i; + + if (left) + error = xfs_btree_decrement(cur, 0, &i); + else + error = xfs_btree_increment(cur, 0, &i); + + if (error) + return error; + *done = !i; + if (i) { + error = xfs_inobt_get_rec(cur, rec, &i); + if (error) + return error; + if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + } + + return 0; +} + +STATIC int +xfs_ialloc_get_rec( + struct xfs_btree_cur *cur, + xfs_agino_t agino, + xfs_inobt_rec_incore_t *rec, + int *done) +{ + int error; + int i; + + error = xfs_inobt_lookup(cur, agino, XFS_LOOKUP_EQ, &i); + if (error) + return error; + *done = !i; + if (i) { + error = xfs_inobt_get_rec(cur, rec, &i); + if (error) + return error; + if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + } + + return 0; +} + +/* + * Return the offset of the first free inode in the record. If the inode chunk + * is sparsely allocated, we convert the record holemask to inode granularity + * and mask off the unallocated regions from the inode free mask. + */ +STATIC int +xfs_inobt_first_free_inode( + struct xfs_inobt_rec_incore *rec) +{ + xfs_inofree_t realfree; + + /* if there are no holes, return the first available offset */ + if (!xfs_inobt_issparse(rec->ir_holemask)) + return xfs_lowbit64(rec->ir_free); + + realfree = xfs_inobt_irec_to_allocmask(rec); + realfree &= rec->ir_free; + + return xfs_lowbit64(realfree); +} + +/* + * If this AG has corrupt inodes, check if allocating this inode would fail + * with corruption errors. Returns 0 if we're clear, or EAGAIN to try again + * somewhere else. + */ +static int +xfs_dialloc_check_ino( + struct xfs_perag *pag, + struct xfs_trans *tp, + xfs_ino_t ino) +{ + struct xfs_imap imap; + struct xfs_buf *bp; + int error; + + error = xfs_imap(pag, tp, ino, &imap, 0); + if (error) + return -EAGAIN; + + error = xfs_read_icluster(pag, tp, imap.im_agbno, &bp); + if (error) + return -EAGAIN; + + xfs_trans_brelse(tp, bp); + return 0; +} + +/* + * Allocate an inode using the inobt-only algorithm. + */ +STATIC int +xfs_dialloc_ag_inobt( + struct xfs_perag *pag, + struct xfs_trans *tp, + struct xfs_buf *agbp, + xfs_ino_t parent, + xfs_ino_t *inop) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_agi *agi = agbp->b_addr; + xfs_agnumber_t pagno = XFS_INO_TO_AGNO(mp, parent); + xfs_agino_t pagino = XFS_INO_TO_AGINO(mp, parent); + struct xfs_btree_cur *cur, *tcur; + struct xfs_inobt_rec_incore rec, trec; + xfs_ino_t ino; + int error; + int offset; + int i, j; + int searchdistance = 10; + + ASSERT(xfs_perag_initialised_agi(pag)); + ASSERT(xfs_perag_allows_inodes(pag)); + ASSERT(pag->pagi_freecount > 0); + + restart_pagno: + cur = xfs_inobt_init_cursor(pag, tp, agbp); + /* + * If pagino is 0 (this is the root inode allocation) use newino. + * This must work because we've just allocated some. + */ + if (!pagino) + pagino = be32_to_cpu(agi->agi_newino); + + error = xfs_check_agi_freecount(cur); + if (error) + goto error0; + + /* + * If in the same AG as the parent, try to get near the parent. + */ + if (pagno == pag_agno(pag)) { + int doneleft; /* done, to the left */ + int doneright; /* done, to the right */ + + error = xfs_inobt_lookup(cur, pagino, XFS_LOOKUP_LE, &i); + if (error) + goto error0; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error0; + } + + error = xfs_inobt_get_rec(cur, &rec, &j); + if (error) + goto error0; + if (XFS_IS_CORRUPT(mp, j != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error0; + } + + if (rec.ir_freecount > 0) { + /* + * Found a free inode in the same chunk + * as the parent, done. + */ + goto alloc_inode; + } + + + /* + * In the same AG as parent, but parent's chunk is full. + */ + + /* duplicate the cursor, search left & right simultaneously */ + error = xfs_btree_dup_cursor(cur, &tcur); + if (error) + goto error0; + + /* + * Skip to last blocks looked up if same parent inode. + */ + if (pagino != NULLAGINO && + pag->pagl_pagino == pagino && + pag->pagl_leftrec != NULLAGINO && + pag->pagl_rightrec != NULLAGINO) { + error = xfs_ialloc_get_rec(tcur, pag->pagl_leftrec, + &trec, &doneleft); + if (error) + goto error1; + + error = xfs_ialloc_get_rec(cur, pag->pagl_rightrec, + &rec, &doneright); + if (error) + goto error1; + } else { + /* search left with tcur, back up 1 record */ + error = xfs_ialloc_next_rec(tcur, &trec, &doneleft, 1); + if (error) + goto error1; + + /* search right with cur, go forward 1 record. */ + error = xfs_ialloc_next_rec(cur, &rec, &doneright, 0); + if (error) + goto error1; + } + + /* + * Loop until we find an inode chunk with a free inode. + */ + while (--searchdistance > 0 && (!doneleft || !doneright)) { + int useleft; /* using left inode chunk this time */ + + /* figure out the closer block if both are valid. */ + if (!doneleft && !doneright) { + useleft = pagino - + (trec.ir_startino + XFS_INODES_PER_CHUNK - 1) < + rec.ir_startino - pagino; + } else { + useleft = !doneleft; + } + + /* free inodes to the left? */ + if (useleft && trec.ir_freecount) { + xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR); + cur = tcur; + + pag->pagl_leftrec = trec.ir_startino; + pag->pagl_rightrec = rec.ir_startino; + pag->pagl_pagino = pagino; + rec = trec; + goto alloc_inode; + } + + /* free inodes to the right? */ + if (!useleft && rec.ir_freecount) { + xfs_btree_del_cursor(tcur, XFS_BTREE_NOERROR); + + pag->pagl_leftrec = trec.ir_startino; + pag->pagl_rightrec = rec.ir_startino; + pag->pagl_pagino = pagino; + goto alloc_inode; + } + + /* get next record to check */ + if (useleft) { + error = xfs_ialloc_next_rec(tcur, &trec, + &doneleft, 1); + } else { + error = xfs_ialloc_next_rec(cur, &rec, + &doneright, 0); + } + if (error) + goto error1; + } + + if (searchdistance <= 0) { + /* + * Not in range - save last search + * location and allocate a new inode + */ + xfs_btree_del_cursor(tcur, XFS_BTREE_NOERROR); + pag->pagl_leftrec = trec.ir_startino; + pag->pagl_rightrec = rec.ir_startino; + pag->pagl_pagino = pagino; + + } else { + /* + * We've reached the end of the btree. because + * we are only searching a small chunk of the + * btree each search, there is obviously free + * inodes closer to the parent inode than we + * are now. restart the search again. + */ + pag->pagl_pagino = NULLAGINO; + pag->pagl_leftrec = NULLAGINO; + pag->pagl_rightrec = NULLAGINO; + xfs_btree_del_cursor(tcur, XFS_BTREE_NOERROR); + xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR); + goto restart_pagno; + } + } + + /* + * In a different AG from the parent. + * See if the most recently allocated block has any free. + */ + if (agi->agi_newino != cpu_to_be32(NULLAGINO)) { + error = xfs_inobt_lookup(cur, be32_to_cpu(agi->agi_newino), + XFS_LOOKUP_EQ, &i); + if (error) + goto error0; + + if (i == 1) { + error = xfs_inobt_get_rec(cur, &rec, &j); + if (error) + goto error0; + + if (j == 1 && rec.ir_freecount > 0) { + /* + * The last chunk allocated in the group + * still has a free inode. + */ + goto alloc_inode; + } + } + } + + /* + * None left in the last group, search the whole AG + */ + error = xfs_inobt_lookup(cur, 0, XFS_LOOKUP_GE, &i); + if (error) + goto error0; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error0; + } + + for (;;) { + error = xfs_inobt_get_rec(cur, &rec, &i); + if (error) + goto error0; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error0; + } + if (rec.ir_freecount > 0) + break; + error = xfs_btree_increment(cur, 0, &i); + if (error) + goto error0; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error0; + } + } + +alloc_inode: + offset = xfs_inobt_first_free_inode(&rec); + ASSERT(offset >= 0); + ASSERT(offset < XFS_INODES_PER_CHUNK); + ASSERT((XFS_AGINO_TO_OFFSET(mp, rec.ir_startino) % + XFS_INODES_PER_CHUNK) == 0); + ino = xfs_agino_to_ino(pag, rec.ir_startino + offset); + + if (xfs_ag_has_sickness(pag, XFS_SICK_AG_INODES)) { + error = xfs_dialloc_check_ino(pag, tp, ino); + if (error) + goto error0; + } + + rec.ir_free &= ~XFS_INOBT_MASK(offset); + rec.ir_freecount--; + error = xfs_inobt_update(cur, &rec); + if (error) + goto error0; + be32_add_cpu(&agi->agi_freecount, -1); + xfs_ialloc_log_agi(tp, agbp, XFS_AGI_FREECOUNT); + pag->pagi_freecount--; + + error = xfs_check_agi_freecount(cur); + if (error) + goto error0; + + xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR); + xfs_trans_mod_sb(tp, XFS_TRANS_SB_IFREE, -1); + *inop = ino; + return 0; +error1: + xfs_btree_del_cursor(tcur, XFS_BTREE_ERROR); +error0: + xfs_btree_del_cursor(cur, XFS_BTREE_ERROR); + return error; +} + +/* + * Use the free inode btree to allocate an inode based on distance from the + * parent. Note that the provided cursor may be deleted and replaced. + */ +STATIC int +xfs_dialloc_ag_finobt_near( + xfs_agino_t pagino, + struct xfs_btree_cur **ocur, + struct xfs_inobt_rec_incore *rec) +{ + struct xfs_btree_cur *lcur = *ocur; /* left search cursor */ + struct xfs_btree_cur *rcur; /* right search cursor */ + struct xfs_inobt_rec_incore rrec; + int error; + int i, j; + + error = xfs_inobt_lookup(lcur, pagino, XFS_LOOKUP_LE, &i); + if (error) + return error; + + if (i == 1) { + error = xfs_inobt_get_rec(lcur, rec, &i); + if (error) + return error; + if (XFS_IS_CORRUPT(lcur->bc_mp, i != 1)) { + xfs_btree_mark_sick(lcur); + return -EFSCORRUPTED; + } + + /* + * See if we've landed in the parent inode record. The finobt + * only tracks chunks with at least one free inode, so record + * existence is enough. + */ + if (pagino >= rec->ir_startino && + pagino < (rec->ir_startino + XFS_INODES_PER_CHUNK)) + return 0; + } + + error = xfs_btree_dup_cursor(lcur, &rcur); + if (error) + return error; + + error = xfs_inobt_lookup(rcur, pagino, XFS_LOOKUP_GE, &j); + if (error) + goto error_rcur; + if (j == 1) { + error = xfs_inobt_get_rec(rcur, &rrec, &j); + if (error) + goto error_rcur; + if (XFS_IS_CORRUPT(lcur->bc_mp, j != 1)) { + xfs_btree_mark_sick(lcur); + error = -EFSCORRUPTED; + goto error_rcur; + } + } + + if (XFS_IS_CORRUPT(lcur->bc_mp, i != 1 && j != 1)) { + xfs_btree_mark_sick(lcur); + error = -EFSCORRUPTED; + goto error_rcur; + } + if (i == 1 && j == 1) { + /* + * Both the left and right records are valid. Choose the closer + * inode chunk to the target. + */ + if ((pagino - rec->ir_startino + XFS_INODES_PER_CHUNK - 1) > + (rrec.ir_startino - pagino)) { + *rec = rrec; + xfs_btree_del_cursor(lcur, XFS_BTREE_NOERROR); + *ocur = rcur; + } else { + xfs_btree_del_cursor(rcur, XFS_BTREE_NOERROR); + } + } else if (j == 1) { + /* only the right record is valid */ + *rec = rrec; + xfs_btree_del_cursor(lcur, XFS_BTREE_NOERROR); + *ocur = rcur; + } else if (i == 1) { + /* only the left record is valid */ + xfs_btree_del_cursor(rcur, XFS_BTREE_NOERROR); + } + + return 0; + +error_rcur: + xfs_btree_del_cursor(rcur, XFS_BTREE_ERROR); + return error; +} + +/* + * Use the free inode btree to find a free inode based on a newino hint. If + * the hint is NULL, find the first free inode in the AG. + */ +STATIC int +xfs_dialloc_ag_finobt_newino( + struct xfs_agi *agi, + struct xfs_btree_cur *cur, + struct xfs_inobt_rec_incore *rec) +{ + int error; + int i; + + if (agi->agi_newino != cpu_to_be32(NULLAGINO)) { + error = xfs_inobt_lookup(cur, be32_to_cpu(agi->agi_newino), + XFS_LOOKUP_EQ, &i); + if (error) + return error; + if (i == 1) { + error = xfs_inobt_get_rec(cur, rec, &i); + if (error) + return error; + if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + return 0; + } + } + + /* + * Find the first inode available in the AG. + */ + error = xfs_inobt_lookup(cur, 0, XFS_LOOKUP_GE, &i); + if (error) + return error; + if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + + error = xfs_inobt_get_rec(cur, rec, &i); + if (error) + return error; + if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + + return 0; +} + +/* + * Update the inobt based on a modification made to the finobt. Also ensure that + * the records from both trees are equivalent post-modification. + */ +STATIC int +xfs_dialloc_ag_update_inobt( + struct xfs_btree_cur *cur, /* inobt cursor */ + struct xfs_inobt_rec_incore *frec, /* finobt record */ + int offset) /* inode offset */ +{ + struct xfs_inobt_rec_incore rec; + int error; + int i; + + error = xfs_inobt_lookup(cur, frec->ir_startino, XFS_LOOKUP_EQ, &i); + if (error) + return error; + if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + + error = xfs_inobt_get_rec(cur, &rec, &i); + if (error) + return error; + if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + ASSERT((XFS_AGINO_TO_OFFSET(cur->bc_mp, rec.ir_startino) % + XFS_INODES_PER_CHUNK) == 0); + + rec.ir_free &= ~XFS_INOBT_MASK(offset); + rec.ir_freecount--; + + if (XFS_IS_CORRUPT(cur->bc_mp, + rec.ir_free != frec->ir_free || + rec.ir_freecount != frec->ir_freecount)) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + + return xfs_inobt_update(cur, &rec); +} + +/* + * Allocate an inode using the free inode btree, if available. Otherwise, fall + * back to the inobt search algorithm. + * + * The caller selected an AG for us, and made sure that free inodes are + * available. + */ +static int +xfs_dialloc_ag( + struct xfs_perag *pag, + struct xfs_trans *tp, + struct xfs_buf *agbp, + xfs_ino_t parent, + xfs_ino_t *inop) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_agi *agi = agbp->b_addr; + xfs_agnumber_t pagno = XFS_INO_TO_AGNO(mp, parent); + xfs_agino_t pagino = XFS_INO_TO_AGINO(mp, parent); + struct xfs_btree_cur *cur; /* finobt cursor */ + struct xfs_btree_cur *icur; /* inobt cursor */ + struct xfs_inobt_rec_incore rec; + xfs_ino_t ino; + int error; + int offset; + int i; + + if (!xfs_has_finobt(mp)) + return xfs_dialloc_ag_inobt(pag, tp, agbp, parent, inop); + + /* + * If pagino is 0 (this is the root inode allocation) use newino. + * This must work because we've just allocated some. + */ + if (!pagino) + pagino = be32_to_cpu(agi->agi_newino); + + cur = xfs_finobt_init_cursor(pag, tp, agbp); + + error = xfs_check_agi_freecount(cur); + if (error) + goto error_cur; + + /* + * The search algorithm depends on whether we're in the same AG as the + * parent. If so, find the closest available inode to the parent. If + * not, consider the agi hint or find the first free inode in the AG. + */ + if (pag_agno(pag) == pagno) + error = xfs_dialloc_ag_finobt_near(pagino, &cur, &rec); + else + error = xfs_dialloc_ag_finobt_newino(agi, cur, &rec); + if (error) + goto error_cur; + + offset = xfs_inobt_first_free_inode(&rec); + ASSERT(offset >= 0); + ASSERT(offset < XFS_INODES_PER_CHUNK); + ASSERT((XFS_AGINO_TO_OFFSET(mp, rec.ir_startino) % + XFS_INODES_PER_CHUNK) == 0); + ino = xfs_agino_to_ino(pag, rec.ir_startino + offset); + + if (xfs_ag_has_sickness(pag, XFS_SICK_AG_INODES)) { + error = xfs_dialloc_check_ino(pag, tp, ino); + if (error) + goto error_cur; + } + + /* + * Modify or remove the finobt record. + */ + rec.ir_free &= ~XFS_INOBT_MASK(offset); + rec.ir_freecount--; + if (rec.ir_freecount) + error = xfs_inobt_update(cur, &rec); + else + error = xfs_btree_delete(cur, &i); + if (error) + goto error_cur; + + /* + * The finobt has now been updated appropriately. We haven't updated the + * agi and superblock yet, so we can create an inobt cursor and validate + * the original freecount. If all is well, make the equivalent update to + * the inobt using the finobt record and offset information. + */ + icur = xfs_inobt_init_cursor(pag, tp, agbp); + + error = xfs_check_agi_freecount(icur); + if (error) + goto error_icur; + + error = xfs_dialloc_ag_update_inobt(icur, &rec, offset); + if (error) + goto error_icur; + + /* + * Both trees have now been updated. We must update the perag and + * superblock before we can check the freecount for each btree. + */ + be32_add_cpu(&agi->agi_freecount, -1); + xfs_ialloc_log_agi(tp, agbp, XFS_AGI_FREECOUNT); + pag->pagi_freecount--; + + xfs_trans_mod_sb(tp, XFS_TRANS_SB_IFREE, -1); + + error = xfs_check_agi_freecount(icur); + if (error) + goto error_icur; + error = xfs_check_agi_freecount(cur); + if (error) + goto error_icur; + + xfs_btree_del_cursor(icur, XFS_BTREE_NOERROR); + xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR); + *inop = ino; + return 0; + +error_icur: + xfs_btree_del_cursor(icur, XFS_BTREE_ERROR); +error_cur: + xfs_btree_del_cursor(cur, XFS_BTREE_ERROR); + return error; +} + +static int +xfs_dialloc_roll( + struct xfs_trans **tpp, + struct xfs_buf *agibp) +{ + struct xfs_trans *tp = *tpp; + struct xfs_dquot_acct *dqinfo; + int error; + + /* + * Hold to on to the agibp across the commit so no other allocation can + * come in and take the free inodes we just allocated for our caller. + */ + xfs_trans_bhold(tp, agibp); + + /* + * We want the quota changes to be associated with the next transaction, + * NOT this one. So, detach the dqinfo from this and attach it to the + * next transaction. + */ + dqinfo = tp->t_dqinfo; + tp->t_dqinfo = NULL; + + error = xfs_trans_roll(&tp); + + /* Re-attach the quota info that we detached from prev trx. */ + tp->t_dqinfo = dqinfo; + + /* + * Join the buffer even on commit error so that the buffer is released + * when the caller cancels the transaction and doesn't have to handle + * this error case specially. + */ + xfs_trans_bjoin(tp, agibp); + *tpp = tp; + return error; +} + +static bool +xfs_dialloc_good_ag( + struct xfs_perag *pag, + struct xfs_trans *tp, + umode_t mode, + int flags, + bool ok_alloc) +{ + struct xfs_mount *mp = tp->t_mountp; + xfs_extlen_t ineed; + xfs_extlen_t longest = 0; + int needspace; + int error; + + if (!pag) + return false; + if (!xfs_perag_allows_inodes(pag)) + return false; + + if (!xfs_perag_initialised_agi(pag)) { + error = xfs_ialloc_read_agi(pag, tp, 0, NULL); + if (error) + return false; + } + + if (pag->pagi_freecount) + return true; + if (!ok_alloc) + return false; + + if (!xfs_perag_initialised_agf(pag)) { + error = xfs_alloc_read_agf(pag, tp, flags, NULL); + if (error) + return false; + } + + /* + * Check that there is enough free space for the file plus a chunk of + * inodes if we need to allocate some. If this is the first pass across + * the AGs, take into account the potential space needed for alignment + * of inode chunks when checking the longest contiguous free space in + * the AG - this prevents us from getting ENOSPC because we have free + * space larger than ialloc_blks but alignment constraints prevent us + * from using it. + * + * If we can't find an AG with space for full alignment slack to be + * taken into account, we must be near ENOSPC in all AGs. Hence we + * don't include alignment for the second pass and so if we fail + * allocation due to alignment issues then it is most likely a real + * ENOSPC condition. + * + * XXX(dgc): this calculation is now bogus thanks to the per-ag + * reservations that xfs_alloc_fix_freelist() now does via + * xfs_alloc_space_available(). When the AG fills up, pagf_freeblks will + * be more than large enough for the check below to succeed, but + * xfs_alloc_space_available() will fail because of the non-zero + * metadata reservation and hence we won't actually be able to allocate + * more inodes in this AG. We do soooo much unnecessary work near ENOSPC + * because of this. + */ + ineed = M_IGEO(mp)->ialloc_min_blks; + if (flags && ineed > 1) + ineed += M_IGEO(mp)->cluster_align; + longest = pag->pagf_longest; + if (!longest) + longest = pag->pagf_flcount > 0; + needspace = S_ISDIR(mode) || S_ISREG(mode) || S_ISLNK(mode); + + if (pag->pagf_freeblks < needspace + ineed || longest < ineed) + return false; + return true; +} + +static int +xfs_dialloc_try_ag( + struct xfs_perag *pag, + struct xfs_trans **tpp, + xfs_ino_t parent, + xfs_ino_t *new_ino, + bool ok_alloc) +{ + struct xfs_buf *agbp; + xfs_ino_t ino; + int error; + + /* + * Then read in the AGI buffer and recheck with the AGI buffer + * lock held. + */ + error = xfs_ialloc_read_agi(pag, *tpp, 0, &agbp); + if (error) + return error; + + if (!pag->pagi_freecount) { + if (!ok_alloc) { + error = -EAGAIN; + goto out_release; + } + + error = xfs_ialloc_ag_alloc(pag, *tpp, agbp); + if (error < 0) + goto out_release; + + /* + * We successfully allocated space for an inode cluster in this + * AG. Roll the transaction so that we can allocate one of the + * new inodes. + */ + ASSERT(pag->pagi_freecount > 0); + error = xfs_dialloc_roll(tpp, agbp); + if (error) + goto out_release; + } + + /* Allocate an inode in the found AG */ + error = xfs_dialloc_ag(pag, *tpp, agbp, parent, &ino); + if (!error) + *new_ino = ino; + return error; + +out_release: + xfs_trans_brelse(*tpp, agbp); + return error; +} + +/* + * Pick an AG for the new inode. + * + * Directories, symlinks, and regular files frequently allocate at least one + * block, so factor that potential expansion when we examine whether an AG has + * enough space for file creation. Try to keep metadata files all in the same + * AG. + */ +static inline xfs_agnumber_t +xfs_dialloc_pick_ag( + struct xfs_mount *mp, + struct xfs_inode *dp, + umode_t mode) +{ + xfs_agnumber_t start_agno; + + if (!dp) + return 0; + if (xfs_is_metadir_inode(dp)) { + if (mp->m_sb.sb_logstart) + return XFS_FSB_TO_AGNO(mp, mp->m_sb.sb_logstart); + return 0; + } + + if (S_ISDIR(mode)) + return (atomic_inc_return(&mp->m_agirotor) - 1) % mp->m_maxagi; + + start_agno = XFS_INODE_TO_AGNO(dp); + if (start_agno >= mp->m_maxagi) + start_agno = 0; + + return start_agno; +} + +/* + * Allocate an on-disk inode. + * + * Mode is used to tell whether the new inode is a directory and hence where to + * locate it. The on-disk inode that is allocated will be returned in @new_ino + * on success, otherwise an error will be set to indicate the failure (e.g. + * -ENOSPC). + */ +int +xfs_dialloc( + struct xfs_trans **tpp, + const struct xfs_icreate_args *args, + xfs_ino_t *new_ino) +{ + struct xfs_mount *mp = (*tpp)->t_mountp; + struct xfs_perag *pag; + struct xfs_ino_geometry *igeo = M_IGEO(mp); + xfs_ino_t ino = NULLFSINO; + xfs_ino_t parent = args->pip ? I_INO(args->pip) : 0; + xfs_agnumber_t agno; + xfs_agnumber_t start_agno; + umode_t mode = args->mode & S_IFMT; + bool ok_alloc = true; + bool low_space = false; + int flags; + int error = 0; + + start_agno = xfs_dialloc_pick_ag(mp, args->pip, mode); + + /* + * If we have already hit the ceiling of inode blocks then clear + * ok_alloc so we scan all available agi structures for a free + * inode. + * + * Read rough value of mp->m_icount by percpu_counter_read_positive, + * which will sacrifice the preciseness but improve the performance. + */ + if (igeo->maxicount && + percpu_counter_read_positive(&mp->m_icount) + igeo->ialloc_inos + > igeo->maxicount) { + ok_alloc = false; + } + + /* + * If we are near to ENOSPC, we want to prefer allocation from AGs that + * have free inodes in them rather than use up free space allocating new + * inode chunks. Hence we turn off allocation for the first non-blocking + * pass through the AGs if we are near ENOSPC to consume free inodes + * that we can immediately allocate, but then we allow allocation on the + * second pass if we fail to find an AG with free inodes in it. + */ + if (xfs_estimate_freecounter(mp, XC_FREE_BLOCKS) < + mp->m_low_space[XFS_LOWSP_1_PCNT]) { + ok_alloc = false; + low_space = true; + } + + /* + * Loop until we find an allocation group that either has free inodes + * or in which we can allocate some inodes. Iterate through the + * allocation groups upward, wrapping at the end. + */ + flags = XFS_ALLOC_FLAG_TRYLOCK; +retry: + for_each_perag_wrap_at(mp, start_agno, mp->m_maxagi, agno, pag) { + if (xfs_dialloc_good_ag(pag, *tpp, mode, flags, ok_alloc)) { + error = xfs_dialloc_try_ag(pag, tpp, parent, + &ino, ok_alloc); + if (error != -EAGAIN) + break; + error = 0; + } + + if (xfs_is_shutdown(mp)) { + error = -EFSCORRUPTED; + break; + } + } + if (pag) + xfs_perag_rele(pag); + if (error) + return error; + if (ino == NULLFSINO) { + if (flags) { + flags = 0; + if (low_space) + ok_alloc = true; + goto retry; + } + return -ENOSPC; + } + + /* + * Protect against obviously corrupt allocation btree records. Later + * xfs_iget checks will catch re-allocation of other active in-memory + * and on-disk inodes. If we don't catch reallocating the parent inode + * here we will deadlock in xfs_iget() so we have to do these checks + * first. + */ + if (ino == parent || !xfs_verify_dir_ino(mp, ino)) { + xfs_alert(mp, "Allocated a known in-use inode 0x%llx!", ino); + xfs_agno_mark_sick(mp, XFS_INO_TO_AGNO(mp, ino), + XFS_SICK_AG_INOBT); + return -EFSCORRUPTED; + } + + *new_ino = ino; + return 0; +} + +/* + * Free the blocks of an inode chunk. We must consider that the inode chunk + * might be sparse and only free the regions that are allocated as part of the + * chunk. + */ +static int +xfs_difree_inode_chunk( + struct xfs_trans *tp, + struct xfs_perag *pag, + struct xfs_inobt_rec_incore *rec) +{ + struct xfs_mount *mp = tp->t_mountp; + xfs_agblock_t sagbno = XFS_AGINO_TO_AGBNO(mp, + rec->ir_startino); + int startidx, endidx; + int nextbit; + xfs_agblock_t agbno; + int contigblk; + DECLARE_BITMAP(holemask, XFS_INOBT_HOLEMASK_BITS); + + if (!xfs_inobt_issparse(rec->ir_holemask)) { + /* not sparse, calculate extent info directly */ + return xfs_free_extent_later(tp, xfs_agbno_to_fsb(pag, sagbno), + M_IGEO(mp)->ialloc_blks, &XFS_RMAP_OINFO_INODES, + XFS_AG_RESV_NONE, 0); + } + + /* holemask is only 16-bits (fits in an unsigned long) */ + ASSERT(sizeof(rec->ir_holemask) <= sizeof(holemask[0])); + holemask[0] = rec->ir_holemask; + + /* + * Find contiguous ranges of zeroes (i.e., allocated regions) in the + * holemask and convert the start/end index of each range to an extent. + * We start with the start and end index both pointing at the first 0 in + * the mask. + */ + startidx = endidx = find_first_zero_bit(holemask, + XFS_INOBT_HOLEMASK_BITS); + nextbit = startidx + 1; + while (startidx < XFS_INOBT_HOLEMASK_BITS) { + int error; + + nextbit = find_next_zero_bit(holemask, XFS_INOBT_HOLEMASK_BITS, + nextbit); + /* + * If the next zero bit is contiguous, update the end index of + * the current range and continue. + */ + if (nextbit != XFS_INOBT_HOLEMASK_BITS && + nextbit == endidx + 1) { + endidx = nextbit; + goto next; + } + + /* + * nextbit is not contiguous with the current end index. Convert + * the current start/end to an extent and add it to the free + * list. + */ + agbno = sagbno + (startidx * XFS_INODES_PER_HOLEMASK_BIT) / + mp->m_sb.sb_inopblock; + contigblk = ((endidx - startidx + 1) * + XFS_INODES_PER_HOLEMASK_BIT) / + mp->m_sb.sb_inopblock; + + ASSERT(agbno % mp->m_sb.sb_spino_align == 0); + ASSERT(contigblk % mp->m_sb.sb_spino_align == 0); + error = xfs_free_extent_later(tp, xfs_agbno_to_fsb(pag, agbno), + contigblk, &XFS_RMAP_OINFO_INODES, + XFS_AG_RESV_NONE, 0); + if (error) + return error; + + /* reset range to current bit and carry on... */ + startidx = endidx = nextbit; + +next: + nextbit++; + } + return 0; +} + +STATIC int +xfs_difree_inobt( + struct xfs_perag *pag, + struct xfs_trans *tp, + struct xfs_buf *agbp, + xfs_agino_t agino, + struct xfs_icluster *xic, + struct xfs_inobt_rec_incore *orec) +{ + struct xfs_mount *mp = pag_mount(pag); + struct xfs_agi *agi = agbp->b_addr; + struct xfs_btree_cur *cur; + struct xfs_inobt_rec_incore rec; + int ilen; + int error; + int i; + int off; + + ASSERT(agi->agi_magicnum == cpu_to_be32(XFS_AGI_MAGIC)); + ASSERT(XFS_AGINO_TO_AGBNO(mp, agino) < be32_to_cpu(agi->agi_length)); + + /* + * Initialize the cursor. + */ + cur = xfs_inobt_init_cursor(pag, tp, agbp); + + error = xfs_check_agi_freecount(cur); + if (error) + goto error0; + + /* + * Look for the entry describing this inode. + */ + if ((error = xfs_inobt_lookup(cur, agino, XFS_LOOKUP_LE, &i))) { + xfs_warn(mp, "%s: xfs_inobt_lookup() returned error %d.", + __func__, error); + goto error0; + } + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error0; + } + error = xfs_inobt_get_rec(cur, &rec, &i); + if (error) { + xfs_warn(mp, "%s: xfs_inobt_get_rec() returned error %d.", + __func__, error); + goto error0; + } + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error0; + } + /* + * Get the offset in the inode chunk. + */ + off = agino - rec.ir_startino; + ASSERT(off >= 0 && off < XFS_INODES_PER_CHUNK); + ASSERT(!(rec.ir_free & XFS_INOBT_MASK(off))); + /* + * Mark the inode free & increment the count. + */ + rec.ir_free |= XFS_INOBT_MASK(off); + rec.ir_freecount++; + + /* + * When an inode chunk is free, it becomes eligible for removal. Don't + * remove the chunk if the block size is large enough for multiple inode + * chunks (that might not be free). + */ + if (rec.ir_free == XFS_INOBT_ALL_FREE && + mp->m_sb.sb_inopblock <= XFS_INODES_PER_CHUNK) { + xic->deleted = true; + xic->first_ino = xfs_agino_to_ino(pag, rec.ir_startino); + xic->alloc = xfs_inobt_irec_to_allocmask(&rec); + + /* + * Remove the inode cluster from the AGI B+Tree, adjust the + * AGI and Superblock inode counts, and mark the disk space + * to be freed when the transaction is committed. + */ + ilen = rec.ir_freecount; + be32_add_cpu(&agi->agi_count, -ilen); + be32_add_cpu(&agi->agi_freecount, -(ilen - 1)); + xfs_ialloc_log_agi(tp, agbp, XFS_AGI_COUNT | XFS_AGI_FREECOUNT); + pag->pagi_freecount -= ilen - 1; + pag->pagi_count -= ilen; + xfs_trans_mod_sb(tp, XFS_TRANS_SB_ICOUNT, -ilen); + xfs_trans_mod_sb(tp, XFS_TRANS_SB_IFREE, -(ilen - 1)); + + if ((error = xfs_btree_delete(cur, &i))) { + xfs_warn(mp, "%s: xfs_btree_delete returned error %d.", + __func__, error); + goto error0; + } + + error = xfs_difree_inode_chunk(tp, pag, &rec); + if (error) + goto error0; + } else { + xic->deleted = false; + + error = xfs_inobt_update(cur, &rec); + if (error) { + xfs_warn(mp, "%s: xfs_inobt_update returned error %d.", + __func__, error); + goto error0; + } + + /* + * Change the inode free counts and log the ag/sb changes. + */ + be32_add_cpu(&agi->agi_freecount, 1); + xfs_ialloc_log_agi(tp, agbp, XFS_AGI_FREECOUNT); + pag->pagi_freecount++; + xfs_trans_mod_sb(tp, XFS_TRANS_SB_IFREE, 1); + } + + error = xfs_check_agi_freecount(cur); + if (error) + goto error0; + + *orec = rec; + xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR); + return 0; + +error0: + xfs_btree_del_cursor(cur, XFS_BTREE_ERROR); + return error; +} + +/* + * Free an inode in the free inode btree. + */ +STATIC int +xfs_difree_finobt( + struct xfs_perag *pag, + struct xfs_trans *tp, + struct xfs_buf *agbp, + xfs_agino_t agino, + struct xfs_inobt_rec_incore *ibtrec) /* inobt record */ +{ + struct xfs_mount *mp = pag_mount(pag); + struct xfs_btree_cur *cur; + struct xfs_inobt_rec_incore rec; + int offset = agino - ibtrec->ir_startino; + int error; + int i; + + cur = xfs_finobt_init_cursor(pag, tp, agbp); + + error = xfs_inobt_lookup(cur, ibtrec->ir_startino, XFS_LOOKUP_EQ, &i); + if (error) + goto error; + if (i == 0) { + /* + * If the record does not exist in the finobt, we must have just + * freed an inode in a previously fully allocated chunk. If not, + * something is out of sync. + */ + if (XFS_IS_CORRUPT(mp, ibtrec->ir_freecount != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error; + } + + error = xfs_inobt_insert_rec(cur, ibtrec->ir_holemask, + ibtrec->ir_count, + ibtrec->ir_freecount, + ibtrec->ir_free, &i); + if (error) + goto error; + ASSERT(i == 1); + + goto out; + } + + /* + * Read and update the existing record. We could just copy the ibtrec + * across here, but that would defeat the purpose of having redundant + * metadata. By making the modifications independently, we can catch + * corruptions that we wouldn't see if we just copied from one record + * to another. + */ + error = xfs_inobt_get_rec(cur, &rec, &i); + if (error) + goto error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error; + } + + rec.ir_free |= XFS_INOBT_MASK(offset); + rec.ir_freecount++; + + if (XFS_IS_CORRUPT(mp, + rec.ir_free != ibtrec->ir_free || + rec.ir_freecount != ibtrec->ir_freecount)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto error; + } + + /* + * The content of inobt records should always match between the inobt + * and finobt. The lifecycle of records in the finobt is different from + * the inobt in that the finobt only tracks records with at least one + * free inode. Hence, if all of the inodes are free and we aren't + * keeping inode chunks permanently on disk, remove the record. + * Otherwise, update the record with the new information. + * + * Note that we currently can't free chunks when the block size is large + * enough for multiple chunks. Leave the finobt record to remain in sync + * with the inobt. + */ + if (rec.ir_free == XFS_INOBT_ALL_FREE && + mp->m_sb.sb_inopblock <= XFS_INODES_PER_CHUNK) { + error = xfs_btree_delete(cur, &i); + if (error) + goto error; + ASSERT(i == 1); + } else { + error = xfs_inobt_update(cur, &rec); + if (error) + goto error; + } + +out: + error = xfs_check_agi_freecount(cur); + if (error) + goto error; + + xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR); + return 0; + +error: + xfs_btree_del_cursor(cur, XFS_BTREE_ERROR); + return error; +} + +/* + * Free disk inode. Carefully avoids touching the incore inode, all + * manipulations incore are the caller's responsibility. + * The on-disk inode is not changed by this operation, only the + * btree (free inode mask) is changed. + */ +int +xfs_difree( + struct xfs_trans *tp, + struct xfs_perag *pag, + xfs_ino_t inode, + struct xfs_icluster *xic) +{ + /* REFERENCED */ + xfs_agblock_t agbno; /* block number containing inode */ + struct xfs_buf *agbp; /* buffer for allocation group header */ + xfs_agino_t agino; /* allocation group inode number */ + int error; /* error return value */ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_inobt_rec_incore rec;/* btree record */ + + /* + * Break up inode number into its components. + */ + if (pag_agno(pag) != XFS_INO_TO_AGNO(mp, inode)) { + xfs_warn(mp, "%s: agno != pag_agno(pag) (%d != %d).", + __func__, XFS_INO_TO_AGNO(mp, inode), pag_agno(pag)); + ASSERT(0); + return -EINVAL; + } + agino = XFS_INO_TO_AGINO(mp, inode); + if (inode != xfs_agino_to_ino(pag, agino)) { + xfs_warn(mp, "%s: inode != xfs_agino_to_ino() (%llu != %llu).", + __func__, (unsigned long long)inode, + (unsigned long long)xfs_agino_to_ino(pag, agino)); + ASSERT(0); + return -EINVAL; + } + agbno = XFS_AGINO_TO_AGBNO(mp, agino); + if (agbno >= xfs_ag_block_count(mp, pag_agno(pag))) { + xfs_warn(mp, "%s: agbno >= xfs_ag_block_count (%d >= %d).", + __func__, agbno, xfs_ag_block_count(mp, pag_agno(pag))); + ASSERT(0); + return -EINVAL; + } + /* + * Get the allocation group header. + */ + error = xfs_ialloc_read_agi(pag, tp, 0, &agbp); + if (error) { + xfs_warn(mp, "%s: xfs_ialloc_read_agi() returned error %d.", + __func__, error); + return error; + } + + /* + * Fix up the inode allocation btree. + */ + error = xfs_difree_inobt(pag, tp, agbp, agino, xic, &rec); + if (error) + goto error0; + + /* + * Fix up the free inode btree. + */ + if (xfs_has_finobt(mp)) { + error = xfs_difree_finobt(pag, tp, agbp, agino, &rec); + if (error) + goto error0; + } + + return 0; + +error0: + return error; +} + +STATIC int +xfs_imap_lookup( + struct xfs_perag *pag, + struct xfs_trans *tp, + xfs_agino_t agino, + xfs_agblock_t agbno, + xfs_agblock_t *chunk_agbno, + xfs_agblock_t *offset_agbno, + int flags) +{ + struct xfs_mount *mp = pag_mount(pag); + struct xfs_inobt_rec_incore rec; + struct xfs_btree_cur *cur; + struct xfs_buf *agbp; + int error; + int i; + + error = xfs_ialloc_read_agi(pag, tp, 0, &agbp); + if (error) { + xfs_alert(mp, + "%s: xfs_ialloc_read_agi() returned error %d, agno %d", + __func__, error, pag_agno(pag)); + return error; + } + + /* + * Lookup the inode record for the given agino. If the record cannot be + * found, then it's an invalid inode number and we should abort. Once + * we have a record, we need to ensure it contains the inode number + * we are looking up. + */ + cur = xfs_inobt_init_cursor(pag, tp, agbp); + error = xfs_inobt_lookup(cur, agino, XFS_LOOKUP_LE, &i); + if (!error) { + if (i) + error = xfs_inobt_get_rec(cur, &rec, &i); + if (!error && i == 0) + error = -EINVAL; + } + + xfs_trans_brelse(tp, agbp); + xfs_btree_del_cursor(cur, error); + if (error) + return error; + + /* check that the returned record contains the required inode */ + if (rec.ir_startino > agino || + rec.ir_startino + M_IGEO(mp)->ialloc_inos <= agino) + return -EINVAL; + + /* for untrusted inodes check it is allocated first */ + if ((flags & XFS_IGET_UNTRUSTED) && + (rec.ir_free & XFS_INOBT_MASK(agino - rec.ir_startino))) + return -EINVAL; + + *chunk_agbno = XFS_AGINO_TO_AGBNO(mp, rec.ir_startino); + *offset_agbno = agbno - *chunk_agbno; + return 0; +} + +/* + * Return the location of the inode in imap, for mapping it into a buffer. + */ +int +xfs_imap( + struct xfs_perag *pag, + struct xfs_trans *tp, + xfs_ino_t ino, /* inode to locate */ + struct xfs_imap *imap, /* location map structure */ + uint flags) /* flags for inode btree lookup */ +{ + struct xfs_mount *mp = pag_mount(pag); + xfs_agblock_t agbno; /* block number of inode in the alloc group */ + xfs_agino_t agino; /* inode number within alloc group */ + xfs_agblock_t chunk_agbno; /* first block in inode chunk */ + xfs_agblock_t cluster_agbno; /* first block in inode cluster */ + int error; /* error code */ + int offset; /* index of inode in its buffer */ + xfs_agblock_t offset_agbno; /* blks from chunk start to inode */ + + ASSERT(ino != NULLFSINO); + + /* + * Split up the inode number into its parts. + */ + agino = XFS_INO_TO_AGINO(mp, ino); + agbno = XFS_AGINO_TO_AGBNO(mp, agino); + if (agbno >= xfs_ag_block_count(mp, pag_agno(pag)) || + ino != xfs_agino_to_ino(pag, agino)) { + error = -EINVAL; +#ifdef DEBUG + /* + * Don't output diagnostic information for untrusted inodes + * as they can be invalid without implying corruption. + */ + if (flags & XFS_IGET_UNTRUSTED) + return error; + if (agbno >= xfs_ag_block_count(mp, pag_agno(pag))) { + xfs_alert(mp, + "%s: agbno (0x%llx) >= mp->m_sb.sb_agblocks (0x%lx)", + __func__, (unsigned long long)agbno, + (unsigned long)xfs_ag_block_count(mp, + pag_agno(pag))); + } + if (ino != xfs_agino_to_ino(pag, agino)) { + xfs_alert(mp, + "%s: ino (0x%llx) != xfs_agino_to_ino() (0x%llx)", + __func__, ino, + xfs_agino_to_ino(pag, agino)); + } + xfs_stack_trace(); +#endif /* DEBUG */ + return error; + } + + /* + * For bulkstat and handle lookups, we have an untrusted inode number + * that we have to verify is valid. We cannot do this just by reading + * the inode buffer as it may have been unlinked and removed leaving + * inodes in stale state on disk. Hence we have to do a btree lookup + * in all cases where an untrusted inode number is passed. + */ + if (!(flags & XFS_IGET_UNTRUSTED)) { + /* + * If the inode cluster size is the same or smaller than the + * blocksize, get to the buffer by simple arithmetics. + */ + if (M_IGEO(mp)->blocks_per_cluster == 1) { + cluster_agbno = agbno; + offset = XFS_INO_TO_OFFSET(mp, ino); + ASSERT(offset < mp->m_sb.sb_inopblock); + goto out; + } + + /* + * If the inode chunks are aligned, use simple maths to find the + * location. + */ + if (M_IGEO(mp)->inoalign_mask) { + offset_agbno = agbno & M_IGEO(mp)->inoalign_mask; + chunk_agbno = agbno - offset_agbno; + goto out_map; + } + + /* + * Otherwise we have to do a btree lookup to find the location. + */ + } + + error = xfs_imap_lookup(pag, tp, agino, agbno, &chunk_agbno, + &offset_agbno, flags); + if (error) + return error; + +out_map: + ASSERT(agbno >= chunk_agbno); + cluster_agbno = chunk_agbno + + ((offset_agbno / M_IGEO(mp)->blocks_per_cluster) * + M_IGEO(mp)->blocks_per_cluster); + offset = ((agbno - cluster_agbno) * mp->m_sb.sb_inopblock) + + XFS_INO_TO_OFFSET(mp, ino); +out: + imap->im_agbno = cluster_agbno; + imap->im_boffset = (unsigned short)(offset << mp->m_sb.sb_inodelog); + if (imap->im_agbno + M_IGEO(mp)->blocks_per_cluster > + pag_group(pag)->xg_block_count) { + xfs_alert(mp, "inode cluster out of range: %u/%u > %u", + imap->im_agbno, M_IGEO(mp)->blocks_per_cluster, + pag_group(pag)->xg_block_count); + return -EINVAL; + } + return 0; +} + +/* + * Log specified fields for the ag hdr (inode section). The growth of the agi + * structure over time requires that we interpret the buffer as two logical + * regions delineated by the end of the unlinked list. This is due to the size + * of the hash table and its location in the middle of the agi. + * + * For example, a request to log a field before agi_unlinked and a field after + * agi_unlinked could cause us to log the entire hash table and use an excessive + * amount of log space. To avoid this behavior, log the region up through + * agi_unlinked in one call and the region after agi_unlinked through the end of + * the structure in another. + */ +void +xfs_ialloc_log_agi( + struct xfs_trans *tp, + struct xfs_buf *bp, + uint32_t fields) +{ + int first; /* first byte number */ + int last; /* last byte number */ + static const short offsets[] = { /* field starting offsets */ + /* keep in sync with bit definitions */ + offsetof(xfs_agi_t, agi_magicnum), + offsetof(xfs_agi_t, agi_versionnum), + offsetof(xfs_agi_t, agi_seqno), + offsetof(xfs_agi_t, agi_length), + offsetof(xfs_agi_t, agi_count), + offsetof(xfs_agi_t, agi_root), + offsetof(xfs_agi_t, agi_level), + offsetof(xfs_agi_t, agi_freecount), + offsetof(xfs_agi_t, agi_newino), + offsetof(xfs_agi_t, agi_dirino), + offsetof(xfs_agi_t, agi_unlinked), + offsetof(xfs_agi_t, agi_free_root), + offsetof(xfs_agi_t, agi_free_level), + offsetof(xfs_agi_t, agi_iblocks), + sizeof(xfs_agi_t) + }; +#ifdef DEBUG + struct xfs_agi *agi = bp->b_addr; + + ASSERT(agi->agi_magicnum == cpu_to_be32(XFS_AGI_MAGIC)); +#endif + + /* + * Compute byte offsets for the first and last fields in the first + * region and log the agi buffer. This only logs up through + * agi_unlinked. + */ + if (fields & XFS_AGI_ALL_BITS_R1) { + xfs_btree_offsets(fields, offsets, XFS_AGI_NUM_BITS_R1, + &first, &last); + xfs_trans_log_buf(tp, bp, first, last); + } + + /* + * Mask off the bits in the first region and calculate the first and + * last field offsets for any bits in the second region. + */ + fields &= ~XFS_AGI_ALL_BITS_R1; + if (fields) { + xfs_btree_offsets(fields, offsets, XFS_AGI_NUM_BITS_R2, + &first, &last); + xfs_trans_log_buf(tp, bp, first, last); + } +} + +static xfs_failaddr_t +xfs_agi_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_agi *agi = bp->b_addr; + xfs_failaddr_t fa; + uint32_t agi_seqno = be32_to_cpu(agi->agi_seqno); + uint32_t agi_length = be32_to_cpu(agi->agi_length); + int i; + + if (xfs_has_crc(mp)) { + if (!uuid_equal(&agi->agi_uuid, &mp->m_sb.sb_meta_uuid)) + return __this_address; + if (!xfs_log_check_lsn(mp, be64_to_cpu(agi->agi_lsn))) + return __this_address; + } + + /* + * Validate the magic number of the agi block. + */ + if (!xfs_verify_magic(bp, agi->agi_magicnum)) + return __this_address; + if (!XFS_AGI_GOOD_VERSION(be32_to_cpu(agi->agi_versionnum))) + return __this_address; + + fa = xfs_validate_ag_length(bp, agi_seqno, agi_length); + if (fa) + return fa; + + if (be32_to_cpu(agi->agi_level) < 1 || + be32_to_cpu(agi->agi_level) > M_IGEO(mp)->inobt_maxlevels) + return __this_address; + + if (xfs_has_finobt(mp) && + (be32_to_cpu(agi->agi_free_level) < 1 || + be32_to_cpu(agi->agi_free_level) > M_IGEO(mp)->inobt_maxlevels)) + return __this_address; + + for (i = 0; i < XFS_AGI_UNLINKED_BUCKETS; i++) { + if (agi->agi_unlinked[i] == cpu_to_be32(NULLAGINO)) + continue; + if (!xfs_verify_ino(mp, be32_to_cpu(agi->agi_unlinked[i]))) + return __this_address; + } + + return NULL; +} + +static void +xfs_agi_read_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + xfs_failaddr_t fa; + + if (xfs_has_crc(mp) && + !xfs_buf_verify_cksum(bp, XFS_AGI_CRC_OFF)) + xfs_verifier_error(bp, -EFSBADCRC, __this_address); + else { + fa = xfs_agi_verify(bp); + if (fa || XFS_TEST_ERROR(mp, XFS_ERRTAG_IALLOC_READ_AGI)) + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + } +} + +static void +xfs_agi_write_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_buf_log_item *bip = bp->b_log_item; + struct xfs_agi *agi = bp->b_addr; + xfs_failaddr_t fa; + + fa = xfs_agi_verify(bp); + if (fa) { + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + return; + } + + if (!xfs_has_crc(mp)) + return; + + if (bip) + agi->agi_lsn = cpu_to_be64(bip->bli_item.li_lsn); + xfs_buf_update_cksum(bp, XFS_AGI_CRC_OFF); +} + +const struct xfs_buf_ops xfs_agi_buf_ops = { + .name = "xfs_agi", + .magic = { cpu_to_be32(XFS_AGI_MAGIC), cpu_to_be32(XFS_AGI_MAGIC) }, + .verify_read = xfs_agi_read_verify, + .verify_write = xfs_agi_write_verify, + .verify_struct = xfs_agi_verify, +}; + +/* + * Read in the allocation group header (inode allocation section) + */ +int +xfs_read_agi( + struct xfs_perag *pag, + struct xfs_trans *tp, + xfs_buf_flags_t flags, + struct xfs_buf **agibpp) +{ + struct xfs_mount *mp = pag_mount(pag); + int error; + + trace_xfs_read_agi(pag); + + error = xfs_trans_read_buf(mp, tp, mp->m_ddev_targp, + XFS_AG_DADDR(mp, pag_agno(pag), XFS_AGI_DADDR(mp)), + XFS_FSS_TO_BB(mp, 1), flags, agibpp, &xfs_agi_buf_ops); + if (xfs_metadata_is_sick(error)) + xfs_ag_mark_sick(pag, XFS_SICK_AG_AGI); + if (error) + return error; + if (tp) + xfs_trans_buf_set_type(tp, *agibpp, XFS_BLFT_AGI_BUF); + + xfs_buf_set_ref(*agibpp, XFS_AGI_REF); + return 0; +} + +/* + * Read in the agi and initialise the per-ag data. If the caller supplies a + * @agibpp, return the locked AGI buffer to them, otherwise release it. + */ +int +xfs_ialloc_read_agi( + struct xfs_perag *pag, + struct xfs_trans *tp, + int flags, + struct xfs_buf **agibpp) +{ + struct xfs_buf *agibp; + struct xfs_agi *agi; + int error; + + trace_xfs_ialloc_read_agi(pag); + + error = xfs_read_agi(pag, tp, + (flags & XFS_IALLOC_FLAG_TRYLOCK) ? XBF_TRYLOCK : 0, + &agibp); + if (error) + return error; + + agi = agibp->b_addr; + if (!xfs_perag_initialised_agi(pag)) { + pag->pagi_freecount = be32_to_cpu(agi->agi_freecount); + pag->pagi_count = be32_to_cpu(agi->agi_count); + set_bit(XFS_AGSTATE_AGI_INIT, &pag->pag_opstate); + } + +#ifdef DEBUG + /* + * It's possible for the AGF to be out of sync if the block device is + * silently dropping writes. This can happen in fstests with dmflakey + * enabled, which allows the buffer to be cleaned and reclaimed by + * memory pressure and then re-read from disk here. We will get a + * stale version of the AGF from disk, and nothing good can happen from + * here. Hence if we detect this situation, immediately shut down the + * filesystem. + * + * This can also happen if we are already in the middle of a forced + * shutdown, so don't bother checking if we are already shut down. + */ + if (!xfs_is_shutdown(pag_mount(pag))) { + bool ok = true; + + ok &= pag->pagi_freecount == be32_to_cpu(agi->agi_freecount); + ok &= pag->pagi_count == be32_to_cpu(agi->agi_count); + + if (XFS_IS_CORRUPT(pag_mount(pag), !ok)) { + xfs_ag_mark_sick(pag, XFS_SICK_AG_AGI); + xfs_trans_brelse(tp, agibp); + xfs_force_shutdown(pag_mount(pag), + SHUTDOWN_CORRUPT_ONDISK); + return -EFSCORRUPTED; + } + } +#endif /* DEBUG */ + + if (agibpp) + *agibpp = agibp; + else + xfs_trans_brelse(tp, agibp); + return 0; +} + +/* How many inodes are backed by inode clusters ondisk? */ +STATIC int +xfs_ialloc_count_ondisk( + struct xfs_btree_cur *cur, + xfs_agino_t low, + xfs_agino_t high, + unsigned int *allocated) +{ + struct xfs_inobt_rec_incore irec; + unsigned int ret = 0; + int has_record; + int error; + + error = xfs_inobt_lookup(cur, low, XFS_LOOKUP_LE, &has_record); + if (error) + return error; + + while (has_record) { + unsigned int i, hole_idx; + + error = xfs_inobt_get_rec(cur, &irec, &has_record); + if (error) + return error; + if (irec.ir_startino > high) + break; + + for (i = 0; i < XFS_INODES_PER_CHUNK; i++) { + if (irec.ir_startino + i < low) + continue; + if (irec.ir_startino + i > high) + break; + + hole_idx = i / XFS_INODES_PER_HOLEMASK_BIT; + if (!(irec.ir_holemask & (1U << hole_idx))) + ret++; + } + + error = xfs_btree_increment(cur, 0, &has_record); + if (error) + return error; + } + + *allocated = ret; + return 0; +} + +/* Is there an inode record covering a given extent? */ +int +xfs_ialloc_has_inodes_at_extent( + struct xfs_btree_cur *cur, + xfs_agblock_t bno, + xfs_extlen_t len, + enum xbtree_recpacking *outcome) +{ + xfs_agino_t agino; + xfs_agino_t last_agino; + unsigned int allocated; + int error; + + agino = XFS_AGB_TO_AGINO(cur->bc_mp, bno); + last_agino = XFS_AGB_TO_AGINO(cur->bc_mp, bno + len) - 1; + + error = xfs_ialloc_count_ondisk(cur, agino, last_agino, &allocated); + if (error) + return error; + + if (allocated == 0) + *outcome = XBTREE_RECPACKING_EMPTY; + else if (allocated == last_agino - agino + 1) + *outcome = XBTREE_RECPACKING_FULL; + else + *outcome = XBTREE_RECPACKING_SPARSE; + return 0; +} + +struct xfs_ialloc_count_inodes { + xfs_agino_t count; + xfs_agino_t freecount; +}; + +/* Record inode counts across all inobt records. */ +STATIC int +xfs_ialloc_count_inodes_rec( + struct xfs_btree_cur *cur, + const union xfs_btree_rec *rec, + void *priv) +{ + struct xfs_inobt_rec_incore irec; + struct xfs_ialloc_count_inodes *ci = priv; + xfs_failaddr_t fa; + + xfs_inobt_btrec_to_irec(cur->bc_mp, rec, &irec); + fa = xfs_inobt_check_irec(to_perag(cur->bc_group), &irec); + if (fa) + return xfs_inobt_complain_bad_rec(cur, fa, &irec); + + ci->count += irec.ir_count; + ci->freecount += irec.ir_freecount; + + return 0; +} + +/* Count allocated and free inodes under an inobt. */ +int +xfs_ialloc_count_inodes( + struct xfs_btree_cur *cur, + xfs_agino_t *count, + xfs_agino_t *freecount) +{ + struct xfs_ialloc_count_inodes ci = {0}; + int error; + + ASSERT(xfs_btree_is_ino(cur->bc_ops)); + error = xfs_btree_query_all(cur, xfs_ialloc_count_inodes_rec, &ci); + if (error) + return error; + + *count = ci.count; + *freecount = ci.freecount; + return 0; +} + +/* + * Initialize inode-related geometry information. + * + * Compute the inode btree min and max levels and set maxicount. + * + * Set the inode cluster size. This may still be overridden by the file + * system block size if it is larger than the chosen cluster size. + * + * For v5 filesystems, scale the cluster size with the inode size to keep a + * constant ratio of inode per cluster buffer, but only if mkfs has set the + * inode alignment value appropriately for larger cluster sizes. + * + * Then compute the inode cluster alignment information. + */ +void +xfs_ialloc_setup_geometry( + struct xfs_mount *mp) +{ + struct xfs_sb *sbp = &mp->m_sb; + struct xfs_ino_geometry *igeo = M_IGEO(mp); + uint64_t icount; + uint inodes; + + igeo->new_diflags2 = 0; + if (xfs_has_bigtime(mp)) + igeo->new_diflags2 |= XFS_DIFLAG2_BIGTIME; + if (xfs_has_large_extent_counts(mp)) + igeo->new_diflags2 |= XFS_DIFLAG2_NREXT64; + + /* Compute inode btree geometry. */ + igeo->agino_log = sbp->sb_inopblog + sbp->sb_agblklog; + igeo->inobt_mxr[0] = xfs_inobt_maxrecs(mp, sbp->sb_blocksize, true); + igeo->inobt_mxr[1] = xfs_inobt_maxrecs(mp, sbp->sb_blocksize, false); + igeo->inobt_mnr[0] = igeo->inobt_mxr[0] / 2; + igeo->inobt_mnr[1] = igeo->inobt_mxr[1] / 2; + + igeo->ialloc_inos = max_t(uint16_t, XFS_INODES_PER_CHUNK, + sbp->sb_inopblock); + igeo->ialloc_blks = igeo->ialloc_inos >> sbp->sb_inopblog; + + if (sbp->sb_spino_align) + igeo->ialloc_min_blks = sbp->sb_spino_align; + else + igeo->ialloc_min_blks = igeo->ialloc_blks; + + /* Compute and fill in value of m_ino_geo.inobt_maxlevels. */ + inodes = (1LL << XFS_INO_AGINO_BITS(mp)) >> XFS_INODES_PER_CHUNK_LOG; + igeo->inobt_maxlevels = xfs_btree_compute_maxlevels(igeo->inobt_mnr, + inodes); + ASSERT(igeo->inobt_maxlevels <= xfs_iallocbt_maxlevels_ondisk()); + + /* + * Set the maximum inode count for this filesystem, being careful not + * to use obviously garbage sb_inopblog/sb_inopblock values. Regular + * users should never get here due to failing sb verification, but + * certain users (xfs_db) need to be usable even with corrupt metadata. + */ + if (sbp->sb_imax_pct && igeo->ialloc_blks) { + /* + * Make sure the maximum inode count is a multiple + * of the units we allocate inodes in. + */ + icount = sbp->sb_dblocks * sbp->sb_imax_pct; + do_div(icount, 100); + do_div(icount, igeo->ialloc_blks); + igeo->maxicount = XFS_FSB_TO_INO(mp, + icount * igeo->ialloc_blks); + } else { + igeo->maxicount = 0; + } + + /* + * Compute the desired size of an inode cluster buffer size, which + * starts at 8K and (on v5 filesystems) scales up with larger inode + * sizes. + * + * Preserve the desired inode cluster size because the sparse inodes + * feature uses that desired size (not the actual size) to compute the + * sparse inode alignment. The mount code validates this value, so we + * cannot change the behavior. + */ + igeo->inode_cluster_size_raw = XFS_INODE_BIG_CLUSTER_SIZE; + if (xfs_has_v3inodes(mp)) { + int new_size = igeo->inode_cluster_size_raw; + + new_size *= mp->m_sb.sb_inodesize / XFS_DINODE_MIN_SIZE; + if (mp->m_sb.sb_inoalignmt >= XFS_B_TO_FSBT(mp, new_size)) + igeo->inode_cluster_size_raw = new_size; + } + + /* Calculate inode cluster ratios. */ + if (igeo->inode_cluster_size_raw > mp->m_sb.sb_blocksize) + igeo->blocks_per_cluster = XFS_B_TO_FSBT(mp, + igeo->inode_cluster_size_raw); + else + igeo->blocks_per_cluster = 1; + igeo->inode_cluster_size = XFS_FSB_TO_B(mp, igeo->blocks_per_cluster); + igeo->inodes_per_cluster = XFS_FSB_TO_INO(mp, igeo->blocks_per_cluster); + + /* Calculate inode cluster alignment. */ + if (xfs_has_align(mp) && + mp->m_sb.sb_inoalignmt >= igeo->blocks_per_cluster) + igeo->cluster_align = mp->m_sb.sb_inoalignmt; + else + igeo->cluster_align = 1; + igeo->inoalign_mask = igeo->cluster_align - 1; + igeo->cluster_align_inodes = XFS_FSB_TO_INO(mp, igeo->cluster_align); + + /* + * If we are using stripe alignment, check whether + * the stripe unit is a multiple of the inode alignment + */ + if (mp->m_dalign && igeo->inoalign_mask && + !(mp->m_dalign & igeo->inoalign_mask)) + igeo->ialloc_align = mp->m_dalign; + else + igeo->ialloc_align = 0; + + if (mp->m_sb.sb_blocksize > PAGE_SIZE) + igeo->min_folio_order = mp->m_sb.sb_blocklog - PAGE_SHIFT; + else + igeo->min_folio_order = 0; +} + +/* Compute the location of the root directory inode that is laid out by mkfs. */ +xfs_ino_t +xfs_ialloc_calc_rootino( + struct xfs_mount *mp, + int sunit) +{ + struct xfs_ino_geometry *igeo = M_IGEO(mp); + xfs_agblock_t first_bno; + + /* + * Pre-calculate the geometry of AG 0. We know what it looks like + * because libxfs knows how to create allocation groups now. + * + * first_bno is the first block in which mkfs could possibly have + * allocated the root directory inode, once we factor in the metadata + * that mkfs formats before it. Namely, the four AG headers... + */ + first_bno = howmany(4 * mp->m_sb.sb_sectsize, mp->m_sb.sb_blocksize); + + /* ...the two free space btree roots... */ + first_bno += 2; + + /* ...the inode btree root... */ + first_bno += 1; + + /* ...the initial AGFL... */ + first_bno += xfs_alloc_min_freelist(mp, NULL); + + /* ...the free inode btree root... */ + if (xfs_has_finobt(mp)) + first_bno++; + + /* ...the reverse mapping btree root... */ + if (xfs_has_rmapbt(mp)) + first_bno++; + + /* ...the reference count btree... */ + if (xfs_has_reflink(mp)) + first_bno++; + + /* + * ...and the log, if it is allocated in the first allocation group. + * + * This can happen with filesystems that only have a single + * allocation group, or very odd geometries created by old mkfs + * versions on very small filesystems. + */ + if (xfs_ag_contains_log(mp, 0)) + first_bno += mp->m_sb.sb_logblocks; + + /* + * Now round first_bno up to whatever allocation alignment is given + * by the filesystem or was passed in. + */ + if (xfs_has_dalign(mp) && igeo->ialloc_align > 0) + first_bno = roundup(first_bno, sunit); + else if (xfs_has_align(mp) && + mp->m_sb.sb_inoalignmt > 1) + first_bno = roundup(first_bno, mp->m_sb.sb_inoalignmt); + + return XFS_AGINO_TO_INO(mp, 0, XFS_AGB_TO_AGINO(mp, first_bno)); +} + +/* + * Ensure there are not sparse inode clusters that cross the new EOAG. + * + * This is a no-op for non-spinode filesystems since clusters are always fully + * allocated and checking the bnobt suffices. However, a spinode filesystem + * could have a record where the upper inodes are free blocks. If those blocks + * were removed from the filesystem, the inode record would extend beyond EOAG, + * which will be flagged as corruption. + */ +int +xfs_ialloc_check_shrink( + struct xfs_perag *pag, + struct xfs_trans *tp, + struct xfs_buf *agibp, + xfs_agblock_t new_length) +{ + struct xfs_inobt_rec_incore rec; + struct xfs_btree_cur *cur; + xfs_agino_t agino; + int has; + int error; + + if (!xfs_has_sparseinodes(pag_mount(pag))) + return 0; + + cur = xfs_inobt_init_cursor(pag, tp, agibp); + + /* Look up the inobt record that would correspond to the new EOFS. */ + agino = XFS_AGB_TO_AGINO(pag_mount(pag), new_length); + error = xfs_inobt_lookup(cur, agino, XFS_LOOKUP_LE, &has); + if (error || !has) + goto out; + + error = xfs_inobt_get_rec(cur, &rec, &has); + if (error) + goto out; + + if (!has) { + xfs_ag_mark_sick(pag, XFS_SICK_AG_INOBT); + error = -EFSCORRUPTED; + goto out; + } + + /* If the record covers inodes that would be beyond EOFS, bail out. */ + if (rec.ir_startino + XFS_INODES_PER_CHUNK > agino) { + error = -ENOSPC; + goto out; + } +out: + xfs_btree_del_cursor(cur, error); + return error; +} diff --git a/fs/xfs/libxfs/xfs_ialloc.h b/fs/xfs/libxfs/xfs_ialloc.h new file mode 100644 index 000000000..3a1323155 --- /dev/null +++ b/fs/xfs/libxfs/xfs_ialloc.h @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_IALLOC_H__ +#define __XFS_IALLOC_H__ + +struct xfs_buf; +struct xfs_dinode; +struct xfs_imap; +struct xfs_mount; +struct xfs_trans; +struct xfs_btree_cur; +struct xfs_perag; + +/* Move inodes in clusters of this size */ +#define XFS_INODE_BIG_CLUSTER_SIZE 8192 + +struct xfs_icluster { + bool deleted; /* record is deleted */ + xfs_ino_t first_ino; /* first inode number */ + uint64_t alloc; /* inode phys. allocation bitmap for + * sparse chunks */ +}; + +/* + * Make an inode pointer out of the buffer/offset. + */ +static inline struct xfs_dinode * +xfs_make_iptr(struct xfs_mount *mp, struct xfs_buf *b, int o) +{ + return xfs_buf_offset(b, o << (mp)->m_sb.sb_inodelog); +} + +struct xfs_icreate_args; + +/* + * Allocate an inode on disk. Mode is used to tell whether the new inode will + * need space, and whether it is a directory. + */ +int xfs_dialloc(struct xfs_trans **tpp, const struct xfs_icreate_args *args, + xfs_ino_t *new_ino); + +int xfs_difree(struct xfs_trans *tp, struct xfs_perag *pag, + xfs_ino_t ino, struct xfs_icluster *ifree); + +/* + * Return the location of the inode in imap, for mapping it into a buffer. + */ +int +xfs_imap( + struct xfs_perag *pag, + struct xfs_trans *tp, /* transaction pointer */ + xfs_ino_t ino, /* inode to locate */ + struct xfs_imap *imap, /* location map structure */ + uint flags); /* flags for inode btree lookup */ + +/* + * Log specified fields for the ag hdr (inode section) + */ +void +xfs_ialloc_log_agi( + struct xfs_trans *tp, /* transaction pointer */ + struct xfs_buf *bp, /* allocation group header buffer */ + uint32_t fields); /* bitmask of fields to log */ + +int xfs_read_agi(struct xfs_perag *pag, struct xfs_trans *tp, xfs_buf_flags_t flags, + struct xfs_buf **agibpp); +int xfs_ialloc_read_agi(struct xfs_perag *pag, struct xfs_trans *tp, + int flags, struct xfs_buf **agibpp); +#define XFS_IALLOC_FLAG_TRYLOCK (1U << 0) /* use trylock for buffer locking */ + +/* + * Lookup a record by ino in the btree given by cur. + */ +int xfs_inobt_lookup(struct xfs_btree_cur *cur, xfs_agino_t ino, + xfs_lookup_t dir, int *stat); + +/* + * Get the data from the pointed-to record. + */ +int xfs_inobt_get_rec(struct xfs_btree_cur *cur, + xfs_inobt_rec_incore_t *rec, int *stat); +uint8_t xfs_inobt_rec_freecount(const struct xfs_inobt_rec_incore *irec); + +/* + * Inode chunk initialisation routine + */ +int xfs_ialloc_inode_init(struct xfs_mount *mp, struct xfs_trans *tp, + struct list_head *buffer_list, int icount, + xfs_agnumber_t agno, xfs_agblock_t agbno, + xfs_agblock_t length, unsigned int gen); + + +union xfs_btree_rec; +void xfs_inobt_btrec_to_irec(struct xfs_mount *mp, + const union xfs_btree_rec *rec, + struct xfs_inobt_rec_incore *irec); +xfs_failaddr_t xfs_inobt_check_irec(struct xfs_perag *pag, + const struct xfs_inobt_rec_incore *irec); +int xfs_ialloc_has_inodes_at_extent(struct xfs_btree_cur *cur, + xfs_agblock_t bno, xfs_extlen_t len, + enum xbtree_recpacking *outcome); +int xfs_ialloc_count_inodes(struct xfs_btree_cur *cur, xfs_agino_t *count, + xfs_agino_t *freecount); +int xfs_inobt_insert_rec(struct xfs_btree_cur *cur, uint16_t holemask, + uint8_t count, int32_t freecount, xfs_inofree_t free, + int *stat); + +int xfs_ialloc_cluster_alignment(struct xfs_mount *mp); +void xfs_ialloc_setup_geometry(struct xfs_mount *mp); +xfs_ino_t xfs_ialloc_calc_rootino(struct xfs_mount *mp, int sunit); + +int xfs_ialloc_check_shrink(struct xfs_perag *pag, struct xfs_trans *tp, + struct xfs_buf *agibp, xfs_agblock_t new_length); + +#endif /* __XFS_IALLOC_H__ */ diff --git a/fs/xfs/libxfs/xfs_ialloc_btree.c b/fs/xfs/libxfs/xfs_ialloc_btree.c new file mode 100644 index 000000000..1376e8630 --- /dev/null +++ b/fs/xfs/libxfs/xfs_ialloc_btree.c @@ -0,0 +1,838 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2001,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_mount.h" +#include "xfs_btree.h" +#include "xfs_btree_staging.h" +#include "xfs_ialloc.h" +#include "xfs_ialloc_btree.h" +#include "xfs_alloc.h" +#include "xfs_error.h" +#include "xfs_health.h" +#include "xfs_trace.h" +#include "xfs_trans.h" +#include "xfs_rmap.h" +#include "xfs_ag.h" + +static struct kmem_cache *xfs_inobt_cur_cache; + +STATIC int +xfs_inobt_get_minrecs( + struct xfs_btree_cur *cur, + int level) +{ + return M_IGEO(cur->bc_mp)->inobt_mnr[level != 0]; +} + +STATIC struct xfs_btree_cur * +xfs_inobt_dup_cursor( + struct xfs_btree_cur *cur) +{ + return xfs_inobt_init_cursor(to_perag(cur->bc_group), cur->bc_tp, + cur->bc_ag.agbp); +} + +STATIC struct xfs_btree_cur * +xfs_finobt_dup_cursor( + struct xfs_btree_cur *cur) +{ + return xfs_finobt_init_cursor(to_perag(cur->bc_group), cur->bc_tp, + cur->bc_ag.agbp); +} + +STATIC void +xfs_inobt_set_root( + struct xfs_btree_cur *cur, + const union xfs_btree_ptr *nptr, + int inc) /* level change */ +{ + struct xfs_buf *agbp = cur->bc_ag.agbp; + struct xfs_agi *agi = agbp->b_addr; + + agi->agi_root = nptr->s; + be32_add_cpu(&agi->agi_level, inc); + xfs_ialloc_log_agi(cur->bc_tp, agbp, XFS_AGI_ROOT | XFS_AGI_LEVEL); +} + +STATIC void +xfs_finobt_set_root( + struct xfs_btree_cur *cur, + const union xfs_btree_ptr *nptr, + int inc) /* level change */ +{ + struct xfs_buf *agbp = cur->bc_ag.agbp; + struct xfs_agi *agi = agbp->b_addr; + + agi->agi_free_root = nptr->s; + be32_add_cpu(&agi->agi_free_level, inc); + xfs_ialloc_log_agi(cur->bc_tp, agbp, + XFS_AGI_FREE_ROOT | XFS_AGI_FREE_LEVEL); +} + +/* Update the inode btree block counter for this btree. */ +static inline void +xfs_inobt_mod_blockcount( + struct xfs_btree_cur *cur, + int howmuch) +{ + struct xfs_buf *agbp = cur->bc_ag.agbp; + struct xfs_agi *agi = agbp->b_addr; + + if (!xfs_has_inobtcounts(cur->bc_mp)) + return; + + if (xfs_btree_is_fino(cur->bc_ops)) + be32_add_cpu(&agi->agi_fblocks, howmuch); + else + be32_add_cpu(&agi->agi_iblocks, howmuch); + xfs_ialloc_log_agi(cur->bc_tp, agbp, XFS_AGI_IBLOCKS); +} + +STATIC int +__xfs_inobt_alloc_block( + struct xfs_btree_cur *cur, + const union xfs_btree_ptr *start, + union xfs_btree_ptr *new, + int *stat, + enum xfs_ag_resv_type resv) +{ + xfs_alloc_arg_t args; /* block allocation args */ + int error; /* error return value */ + xfs_agblock_t sbno = be32_to_cpu(start->s); + + memset(&args, 0, sizeof(args)); + args.tp = cur->bc_tp; + args.mp = cur->bc_mp; + args.pag = to_perag(cur->bc_group); + args.oinfo = XFS_RMAP_OINFO_INOBT; + args.minlen = 1; + args.maxlen = 1; + args.prod = 1; + args.resv = resv; + + error = xfs_alloc_vextent_near_bno(&args, + xfs_agbno_to_fsb(args.pag, sbno)); + if (error) + return error; + + if (args.fsbno == NULLFSBLOCK) { + *stat = 0; + return 0; + } + ASSERT(args.len == 1); + + new->s = cpu_to_be32(XFS_FSB_TO_AGBNO(args.mp, args.fsbno)); + *stat = 1; + xfs_inobt_mod_blockcount(cur, 1); + return 0; +} + +STATIC int +xfs_inobt_alloc_block( + struct xfs_btree_cur *cur, + const union xfs_btree_ptr *start, + union xfs_btree_ptr *new, + int *stat) +{ + return __xfs_inobt_alloc_block(cur, start, new, stat, XFS_AG_RESV_NONE); +} + +STATIC int +xfs_finobt_alloc_block( + struct xfs_btree_cur *cur, + const union xfs_btree_ptr *start, + union xfs_btree_ptr *new, + int *stat) +{ + if (cur->bc_mp->m_finobt_nores) + return xfs_inobt_alloc_block(cur, start, new, stat); + return __xfs_inobt_alloc_block(cur, start, new, stat, + XFS_AG_RESV_METADATA); +} + +STATIC int +__xfs_inobt_free_block( + struct xfs_btree_cur *cur, + struct xfs_buf *bp, + enum xfs_ag_resv_type resv) +{ + xfs_fsblock_t fsbno; + + xfs_inobt_mod_blockcount(cur, -1); + fsbno = XFS_DADDR_TO_FSB(cur->bc_mp, xfs_buf_daddr(bp)); + return xfs_free_extent_later(cur->bc_tp, fsbno, 1, + &XFS_RMAP_OINFO_INOBT, resv, 0); +} + +STATIC int +xfs_inobt_free_block( + struct xfs_btree_cur *cur, + struct xfs_buf *bp) +{ + return __xfs_inobt_free_block(cur, bp, XFS_AG_RESV_NONE); +} + +STATIC int +xfs_finobt_free_block( + struct xfs_btree_cur *cur, + struct xfs_buf *bp) +{ + if (cur->bc_mp->m_finobt_nores) + return xfs_inobt_free_block(cur, bp); + return __xfs_inobt_free_block(cur, bp, XFS_AG_RESV_METADATA); +} + +STATIC int +xfs_inobt_get_maxrecs( + struct xfs_btree_cur *cur, + int level) +{ + return M_IGEO(cur->bc_mp)->inobt_mxr[level != 0]; +} + +STATIC void +xfs_inobt_init_key_from_rec( + union xfs_btree_key *key, + const union xfs_btree_rec *rec) +{ + key->inobt.ir_startino = rec->inobt.ir_startino; +} + +STATIC void +xfs_inobt_init_high_key_from_rec( + union xfs_btree_key *key, + const union xfs_btree_rec *rec) +{ + __u32 x; + + x = be32_to_cpu(rec->inobt.ir_startino); + x += XFS_INODES_PER_CHUNK - 1; + key->inobt.ir_startino = cpu_to_be32(x); +} + +STATIC void +xfs_inobt_init_rec_from_cur( + struct xfs_btree_cur *cur, + union xfs_btree_rec *rec) +{ + rec->inobt.ir_startino = cpu_to_be32(cur->bc_rec.i.ir_startino); + if (xfs_has_sparseinodes(cur->bc_mp)) { + rec->inobt.ir_u.sp.ir_holemask = + cpu_to_be16(cur->bc_rec.i.ir_holemask); + rec->inobt.ir_u.sp.ir_count = cur->bc_rec.i.ir_count; + rec->inobt.ir_u.sp.ir_freecount = cur->bc_rec.i.ir_freecount; + } else { + /* ir_holemask/ir_count not supported on-disk */ + rec->inobt.ir_u.f.ir_freecount = + cpu_to_be32(cur->bc_rec.i.ir_freecount); + } + rec->inobt.ir_free = cpu_to_be64(cur->bc_rec.i.ir_free); +} + +/* + * initial value of ptr for lookup + */ +STATIC void +xfs_inobt_init_ptr_from_cur( + struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr) +{ + struct xfs_agi *agi = cur->bc_ag.agbp->b_addr; + + ASSERT(cur->bc_group->xg_gno == be32_to_cpu(agi->agi_seqno)); + + ptr->s = agi->agi_root; +} + +STATIC void +xfs_finobt_init_ptr_from_cur( + struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr) +{ + struct xfs_agi *agi = cur->bc_ag.agbp->b_addr; + + ASSERT(cur->bc_group->xg_gno == be32_to_cpu(agi->agi_seqno)); + + ptr->s = agi->agi_free_root; +} + +STATIC int +xfs_inobt_cmp_key_with_cur( + struct xfs_btree_cur *cur, + const union xfs_btree_key *key) +{ + return cmp_int(be32_to_cpu(key->inobt.ir_startino), + cur->bc_rec.i.ir_startino); +} + +STATIC int +xfs_inobt_cmp_two_keys( + struct xfs_btree_cur *cur, + const union xfs_btree_key *k1, + const union xfs_btree_key *k2, + const union xfs_btree_key *mask) +{ + ASSERT(!mask || mask->inobt.ir_startino); + + return cmp_int(be32_to_cpu(k1->inobt.ir_startino), + be32_to_cpu(k2->inobt.ir_startino)); +} + +static xfs_failaddr_t +xfs_inobt_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_btree_block *block = XFS_BUF_TO_BLOCK(bp); + xfs_failaddr_t fa; + unsigned int level; + + if (!xfs_verify_magic(bp, block->bb_magic)) + return __this_address; + + /* + * During growfs operations, we can't verify the exact owner as the + * perag is not fully initialised and hence not attached to the buffer. + * + * Similarly, during log recovery we will have a perag structure + * attached, but the agi information will not yet have been initialised + * from the on disk AGI. We don't currently use any of this information, + * but beware of the landmine (i.e. need to check + * xfs_perag_initialised_agi(pag)) if we ever do. + */ + if (xfs_has_crc(mp)) { + fa = xfs_btree_agblock_v5hdr_verify(bp); + if (fa) + return fa; + } + + /* level verification */ + level = be16_to_cpu(block->bb_level); + if (level >= M_IGEO(mp)->inobt_maxlevels) + return __this_address; + + return xfs_btree_agblock_verify(bp, + M_IGEO(mp)->inobt_mxr[level != 0]); +} + +static void +xfs_inobt_read_verify( + struct xfs_buf *bp) +{ + xfs_failaddr_t fa; + + if (!xfs_btree_agblock_verify_crc(bp)) + xfs_verifier_error(bp, -EFSBADCRC, __this_address); + else { + fa = xfs_inobt_verify(bp); + if (fa) + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + } + + if (bp->b_error) + trace_xfs_btree_corrupt(bp, _RET_IP_); +} + +static void +xfs_inobt_write_verify( + struct xfs_buf *bp) +{ + xfs_failaddr_t fa; + + fa = xfs_inobt_verify(bp); + if (fa) { + trace_xfs_btree_corrupt(bp, _RET_IP_); + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + return; + } + xfs_btree_agblock_calc_crc(bp); + +} + +const struct xfs_buf_ops xfs_inobt_buf_ops = { + .name = "xfs_inobt", + .magic = { cpu_to_be32(XFS_IBT_MAGIC), cpu_to_be32(XFS_IBT_CRC_MAGIC) }, + .verify_read = xfs_inobt_read_verify, + .verify_write = xfs_inobt_write_verify, + .verify_struct = xfs_inobt_verify, +}; + +const struct xfs_buf_ops xfs_finobt_buf_ops = { + .name = "xfs_finobt", + .magic = { cpu_to_be32(XFS_FIBT_MAGIC), + cpu_to_be32(XFS_FIBT_CRC_MAGIC) }, + .verify_read = xfs_inobt_read_verify, + .verify_write = xfs_inobt_write_verify, + .verify_struct = xfs_inobt_verify, +}; + +STATIC int +xfs_inobt_keys_inorder( + struct xfs_btree_cur *cur, + const union xfs_btree_key *k1, + const union xfs_btree_key *k2) +{ + return be32_to_cpu(k1->inobt.ir_startino) < + be32_to_cpu(k2->inobt.ir_startino); +} + +STATIC int +xfs_inobt_recs_inorder( + struct xfs_btree_cur *cur, + const union xfs_btree_rec *r1, + const union xfs_btree_rec *r2) +{ + return be32_to_cpu(r1->inobt.ir_startino) + XFS_INODES_PER_CHUNK <= + be32_to_cpu(r2->inobt.ir_startino); +} + +STATIC enum xbtree_key_contig +xfs_inobt_keys_contiguous( + struct xfs_btree_cur *cur, + const union xfs_btree_key *key1, + const union xfs_btree_key *key2, + const union xfs_btree_key *mask) +{ + ASSERT(!mask || mask->inobt.ir_startino); + + return xbtree_key_contig(be32_to_cpu(key1->inobt.ir_startino), + be32_to_cpu(key2->inobt.ir_startino)); +} + +const struct xfs_btree_ops xfs_inobt_ops = { + .name = "ino", + .type = XFS_BTREE_TYPE_AG, + + .rec_len = sizeof(xfs_inobt_rec_t), + .key_len = sizeof(xfs_inobt_key_t), + .ptr_len = XFS_BTREE_SHORT_PTR_LEN, + + .lru_refs = XFS_INO_BTREE_REF, + .statoff = XFS_STATS_CALC_INDEX(xs_ibt_2), + .sick_mask = XFS_SICK_AG_INOBT, + + .dup_cursor = xfs_inobt_dup_cursor, + .set_root = xfs_inobt_set_root, + .alloc_block = xfs_inobt_alloc_block, + .free_block = xfs_inobt_free_block, + .get_minrecs = xfs_inobt_get_minrecs, + .get_maxrecs = xfs_inobt_get_maxrecs, + .init_key_from_rec = xfs_inobt_init_key_from_rec, + .init_high_key_from_rec = xfs_inobt_init_high_key_from_rec, + .init_rec_from_cur = xfs_inobt_init_rec_from_cur, + .init_ptr_from_cur = xfs_inobt_init_ptr_from_cur, + .cmp_key_with_cur = xfs_inobt_cmp_key_with_cur, + .buf_ops = &xfs_inobt_buf_ops, + .cmp_two_keys = xfs_inobt_cmp_two_keys, + .keys_inorder = xfs_inobt_keys_inorder, + .recs_inorder = xfs_inobt_recs_inorder, + .keys_contiguous = xfs_inobt_keys_contiguous, +}; + +const struct xfs_btree_ops xfs_finobt_ops = { + .name = "fino", + .type = XFS_BTREE_TYPE_AG, + + .rec_len = sizeof(xfs_inobt_rec_t), + .key_len = sizeof(xfs_inobt_key_t), + .ptr_len = XFS_BTREE_SHORT_PTR_LEN, + + .lru_refs = XFS_INO_BTREE_REF, + .statoff = XFS_STATS_CALC_INDEX(xs_fibt_2), + .sick_mask = XFS_SICK_AG_FINOBT, + + .dup_cursor = xfs_finobt_dup_cursor, + .set_root = xfs_finobt_set_root, + .alloc_block = xfs_finobt_alloc_block, + .free_block = xfs_finobt_free_block, + .get_minrecs = xfs_inobt_get_minrecs, + .get_maxrecs = xfs_inobt_get_maxrecs, + .init_key_from_rec = xfs_inobt_init_key_from_rec, + .init_high_key_from_rec = xfs_inobt_init_high_key_from_rec, + .init_rec_from_cur = xfs_inobt_init_rec_from_cur, + .init_ptr_from_cur = xfs_finobt_init_ptr_from_cur, + .cmp_key_with_cur = xfs_inobt_cmp_key_with_cur, + .buf_ops = &xfs_finobt_buf_ops, + .cmp_two_keys = xfs_inobt_cmp_two_keys, + .keys_inorder = xfs_inobt_keys_inorder, + .recs_inorder = xfs_inobt_recs_inorder, + .keys_contiguous = xfs_inobt_keys_contiguous, +}; + +/* + * Create an inode btree cursor. + * + * For staging cursors tp and agbp are NULL. + */ +struct xfs_btree_cur * +xfs_inobt_init_cursor( + struct xfs_perag *pag, + struct xfs_trans *tp, + struct xfs_buf *agbp) +{ + struct xfs_mount *mp = pag_mount(pag); + struct xfs_btree_cur *cur; + + cur = xfs_btree_alloc_cursor(mp, tp, &xfs_inobt_ops, + M_IGEO(mp)->inobt_maxlevels, xfs_inobt_cur_cache); + cur->bc_group = xfs_group_hold(pag_group(pag)); + cur->bc_ag.agbp = agbp; + if (agbp) { + struct xfs_agi *agi = agbp->b_addr; + + cur->bc_nlevels = be32_to_cpu(agi->agi_level); + } + return cur; +} + +/* + * Create a free inode btree cursor. + * + * For staging cursors tp and agbp are NULL. + */ +struct xfs_btree_cur * +xfs_finobt_init_cursor( + struct xfs_perag *pag, + struct xfs_trans *tp, + struct xfs_buf *agbp) +{ + struct xfs_mount *mp = pag_mount(pag); + struct xfs_btree_cur *cur; + + cur = xfs_btree_alloc_cursor(mp, tp, &xfs_finobt_ops, + M_IGEO(mp)->inobt_maxlevels, xfs_inobt_cur_cache); + cur->bc_group = xfs_group_hold(pag_group(pag)); + cur->bc_ag.agbp = agbp; + if (agbp) { + struct xfs_agi *agi = agbp->b_addr; + + cur->bc_nlevels = be32_to_cpu(agi->agi_free_level); + } + return cur; +} + +/* + * Install a new inobt btree root. Caller is responsible for invalidating + * and freeing the old btree blocks. + */ +void +xfs_inobt_commit_staged_btree( + struct xfs_btree_cur *cur, + struct xfs_trans *tp, + struct xfs_buf *agbp) +{ + struct xfs_agi *agi = agbp->b_addr; + struct xbtree_afakeroot *afake = cur->bc_ag.afake; + int fields; + + ASSERT(cur->bc_flags & XFS_BTREE_STAGING); + + if (xfs_btree_is_ino(cur->bc_ops)) { + fields = XFS_AGI_ROOT | XFS_AGI_LEVEL; + agi->agi_root = cpu_to_be32(afake->af_root); + agi->agi_level = cpu_to_be32(afake->af_levels); + if (xfs_has_inobtcounts(cur->bc_mp)) { + agi->agi_iblocks = cpu_to_be32(afake->af_blocks); + fields |= XFS_AGI_IBLOCKS; + } + xfs_ialloc_log_agi(tp, agbp, fields); + xfs_btree_commit_afakeroot(cur, tp, agbp); + } else { + fields = XFS_AGI_FREE_ROOT | XFS_AGI_FREE_LEVEL; + agi->agi_free_root = cpu_to_be32(afake->af_root); + agi->agi_free_level = cpu_to_be32(afake->af_levels); + if (xfs_has_inobtcounts(cur->bc_mp)) { + agi->agi_fblocks = cpu_to_be32(afake->af_blocks); + fields |= XFS_AGI_IBLOCKS; + } + xfs_ialloc_log_agi(tp, agbp, fields); + xfs_btree_commit_afakeroot(cur, tp, agbp); + } +} + +/* Calculate number of records in an inode btree block. */ +static inline unsigned int +xfs_inobt_block_maxrecs( + unsigned int blocklen, + bool leaf) +{ + if (leaf) + return blocklen / sizeof(xfs_inobt_rec_t); + return blocklen / (sizeof(xfs_inobt_key_t) + sizeof(xfs_inobt_ptr_t)); +} + +/* + * Calculate number of records in an inobt btree block. + */ +unsigned int +xfs_inobt_maxrecs( + struct xfs_mount *mp, + unsigned int blocklen, + bool leaf) +{ + blocklen -= XFS_INOBT_BLOCK_LEN(mp); + return xfs_inobt_block_maxrecs(blocklen, leaf); +} + +/* + * Maximum number of inode btree records per AG. Pretend that we can fill an + * entire AG completely full of inodes except for the AG headers. + */ +#define XFS_MAX_INODE_RECORDS \ + ((XFS_MAX_AG_BYTES - (4 * BBSIZE)) / XFS_DINODE_MIN_SIZE) / \ + XFS_INODES_PER_CHUNK + +/* Compute the max possible height for the inode btree. */ +static inline unsigned int +xfs_inobt_maxlevels_ondisk(void) +{ + unsigned int minrecs[2]; + unsigned int blocklen; + + blocklen = min(XFS_MIN_BLOCKSIZE - XFS_BTREE_SBLOCK_LEN, + XFS_MIN_CRC_BLOCKSIZE - XFS_BTREE_SBLOCK_CRC_LEN); + + minrecs[0] = xfs_inobt_block_maxrecs(blocklen, true) / 2; + minrecs[1] = xfs_inobt_block_maxrecs(blocklen, false) / 2; + + return xfs_btree_compute_maxlevels(minrecs, XFS_MAX_INODE_RECORDS); +} + +/* Compute the max possible height for the free inode btree. */ +static inline unsigned int +xfs_finobt_maxlevels_ondisk(void) +{ + unsigned int minrecs[2]; + unsigned int blocklen; + + blocklen = XFS_MIN_CRC_BLOCKSIZE - XFS_BTREE_SBLOCK_CRC_LEN; + + minrecs[0] = xfs_inobt_block_maxrecs(blocklen, true) / 2; + minrecs[1] = xfs_inobt_block_maxrecs(blocklen, false) / 2; + + return xfs_btree_compute_maxlevels(minrecs, XFS_MAX_INODE_RECORDS); +} + +/* Compute the max possible height for either inode btree. */ +unsigned int +xfs_iallocbt_maxlevels_ondisk(void) +{ + return max(xfs_inobt_maxlevels_ondisk(), + xfs_finobt_maxlevels_ondisk()); +} + +/* + * Convert the inode record holemask to an inode allocation bitmap. The inode + * allocation bitmap is inode granularity and specifies whether an inode is + * physically allocated on disk (not whether the inode is considered allocated + * or free by the fs). + * + * A bit value of 1 means the inode is allocated, a value of 0 means it is free. + */ +uint64_t +xfs_inobt_irec_to_allocmask( + const struct xfs_inobt_rec_incore *rec) +{ + uint64_t bitmap = 0; + uint64_t inodespbit; + int nextbit; + uint allocbitmap; + + /* + * The holemask has 16-bits for a 64 inode record. Therefore each + * holemask bit represents multiple inodes. Create a mask of bits to set + * in the allocmask for each holemask bit. + */ + inodespbit = (1 << XFS_INODES_PER_HOLEMASK_BIT) - 1; + + /* + * Allocated inodes are represented by 0 bits in holemask. Invert the 0 + * bits to 1 and convert to a uint so we can use xfs_next_bit(). Mask + * anything beyond the 16 holemask bits since this casts to a larger + * type. + */ + allocbitmap = ~rec->ir_holemask & ((1 << XFS_INOBT_HOLEMASK_BITS) - 1); + + /* + * allocbitmap is the inverted holemask so every set bit represents + * allocated inodes. To expand from 16-bit holemask granularity to + * 64-bit (e.g., bit-per-inode), set inodespbit bits in the target + * bitmap for every holemask bit. + */ + nextbit = xfs_next_bit(&allocbitmap, 1, 0); + while (nextbit != -1) { + ASSERT(nextbit < (sizeof(rec->ir_holemask) * NBBY)); + + bitmap |= (inodespbit << + (nextbit * XFS_INODES_PER_HOLEMASK_BIT)); + + nextbit = xfs_next_bit(&allocbitmap, 1, nextbit + 1); + } + + return bitmap; +} + +#if defined(DEBUG) || defined(XFS_WARN) +/* + * Verify that an in-core inode record has a valid inode count. + */ +int +xfs_inobt_rec_check_count( + struct xfs_mount *mp, + struct xfs_inobt_rec_incore *rec) +{ + int inocount = 0; + int nextbit = 0; + uint64_t allocbmap; + int wordsz; + + wordsz = sizeof(allocbmap) / sizeof(unsigned int); + allocbmap = xfs_inobt_irec_to_allocmask(rec); + + nextbit = xfs_next_bit((uint *) &allocbmap, wordsz, nextbit); + while (nextbit != -1) { + inocount++; + nextbit = xfs_next_bit((uint *) &allocbmap, wordsz, + nextbit + 1); + } + + if (inocount != rec->ir_count) + return -EFSCORRUPTED; + + return 0; +} +#endif /* DEBUG */ + +static xfs_extlen_t +xfs_inobt_max_size( + struct xfs_perag *pag) +{ + struct xfs_mount *mp = pag_mount(pag); + xfs_agblock_t agblocks = pag_group(pag)->xg_block_count; + + /* Bail out if we're uninitialized, which can happen in mkfs. */ + if (M_IGEO(mp)->inobt_mxr[0] == 0) + return 0; + + /* + * The log is permanently allocated, so the space it occupies will + * never be available for the kinds of things that would require btree + * expansion. We therefore can pretend the space isn't there. + */ + if (xfs_ag_contains_log(mp, pag_agno(pag))) + agblocks -= mp->m_sb.sb_logblocks; + + return xfs_btree_calc_size(M_IGEO(mp)->inobt_mnr, + (uint64_t)agblocks * mp->m_sb.sb_inopblock / + XFS_INODES_PER_CHUNK); +} + +static int +xfs_finobt_count_blocks( + struct xfs_perag *pag, + struct xfs_trans *tp, + xfs_extlen_t *tree_blocks) +{ + struct xfs_buf *agbp = NULL; + struct xfs_btree_cur *cur; + xfs_filblks_t blocks; + int error; + + error = xfs_ialloc_read_agi(pag, tp, 0, &agbp); + if (error) + return error; + + cur = xfs_finobt_init_cursor(pag, tp, agbp); + error = xfs_btree_count_blocks(cur, &blocks); + xfs_btree_del_cursor(cur, error); + xfs_trans_brelse(tp, agbp); + *tree_blocks = blocks; + + return error; +} + +/* Read finobt block count from AGI header. */ +static int +xfs_finobt_read_blocks( + struct xfs_perag *pag, + struct xfs_trans *tp, + xfs_extlen_t *tree_blocks) +{ + struct xfs_buf *agbp; + struct xfs_agi *agi; + int error; + + error = xfs_ialloc_read_agi(pag, tp, 0, &agbp); + if (error) + return error; + + agi = agbp->b_addr; + *tree_blocks = be32_to_cpu(agi->agi_fblocks); + xfs_trans_brelse(tp, agbp); + return 0; +} + +/* + * Figure out how many blocks to reserve and how many are used by this btree. + */ +int +xfs_finobt_calc_reserves( + struct xfs_perag *pag, + struct xfs_trans *tp, + xfs_extlen_t *ask, + xfs_extlen_t *used) +{ + xfs_extlen_t tree_len = 0; + int error; + + if (!xfs_has_finobt(pag_mount(pag))) + return 0; + + if (xfs_has_inobtcounts(pag_mount(pag))) + error = xfs_finobt_read_blocks(pag, tp, &tree_len); + else + error = xfs_finobt_count_blocks(pag, tp, &tree_len); + if (error) + return error; + + *ask += xfs_inobt_max_size(pag); + *used += tree_len; + return 0; +} + +/* Calculate the inobt btree size for some records. */ +xfs_extlen_t +xfs_iallocbt_calc_size( + struct xfs_mount *mp, + unsigned long long len) +{ + return xfs_btree_calc_size(M_IGEO(mp)->inobt_mnr, len); +} + +int __init +xfs_inobt_init_cur_cache(void) +{ + xfs_inobt_cur_cache = kmem_cache_create("xfs_inobt_cur", + xfs_btree_cur_sizeof(xfs_inobt_maxlevels_ondisk()), + 0, 0, NULL); + + if (!xfs_inobt_cur_cache) + return -ENOMEM; + return 0; +} + +void +xfs_inobt_destroy_cur_cache(void) +{ + kmem_cache_destroy(xfs_inobt_cur_cache); + xfs_inobt_cur_cache = NULL; +} diff --git a/fs/xfs/libxfs/xfs_ialloc_btree.h b/fs/xfs/libxfs/xfs_ialloc_btree.h new file mode 100644 index 000000000..300edf5bc --- /dev/null +++ b/fs/xfs/libxfs/xfs_ialloc_btree.h @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_IALLOC_BTREE_H__ +#define __XFS_IALLOC_BTREE_H__ + +/* + * Inode map on-disk structures + */ + +struct xfs_buf; +struct xfs_btree_cur; +struct xfs_mount; +struct xfs_perag; + +/* + * Btree block header size depends on a superblock flag. + */ +#define XFS_INOBT_BLOCK_LEN(mp) \ + (xfs_has_crc(((mp))) ? \ + XFS_BTREE_SBLOCK_CRC_LEN : XFS_BTREE_SBLOCK_LEN) + +/* + * Record, key, and pointer address macros for btree blocks. + * + * (note that some of these may appear unused, but they are used in userspace) + */ +#define XFS_INOBT_REC_ADDR(mp, block, index) \ + ((xfs_inobt_rec_t *) \ + ((char *)(block) + \ + XFS_INOBT_BLOCK_LEN(mp) + \ + (((index) - 1) * sizeof(xfs_inobt_rec_t)))) + +#define XFS_INOBT_KEY_ADDR(mp, block, index) \ + ((xfs_inobt_key_t *) \ + ((char *)(block) + \ + XFS_INOBT_BLOCK_LEN(mp) + \ + ((index) - 1) * sizeof(xfs_inobt_key_t))) + +#define XFS_INOBT_PTR_ADDR(mp, block, index, maxrecs) \ + ((xfs_inobt_ptr_t *) \ + ((char *)(block) + \ + XFS_INOBT_BLOCK_LEN(mp) + \ + (maxrecs) * sizeof(xfs_inobt_key_t) + \ + ((index) - 1) * sizeof(xfs_inobt_ptr_t))) + +struct xfs_btree_cur *xfs_inobt_init_cursor(struct xfs_perag *pag, + struct xfs_trans *tp, struct xfs_buf *agbp); +struct xfs_btree_cur *xfs_finobt_init_cursor(struct xfs_perag *pag, + struct xfs_trans *tp, struct xfs_buf *agbp); +unsigned int xfs_inobt_maxrecs(struct xfs_mount *mp, unsigned int blocklen, + bool leaf); + +/* ir_holemask to inode allocation bitmap conversion */ +uint64_t xfs_inobt_irec_to_allocmask(const struct xfs_inobt_rec_incore *irec); + +#if defined(DEBUG) || defined(XFS_WARN) +int xfs_inobt_rec_check_count(struct xfs_mount *, + struct xfs_inobt_rec_incore *); +#else +#define xfs_inobt_rec_check_count(mp, rec) 0 +#endif /* DEBUG */ + +int xfs_finobt_calc_reserves(struct xfs_perag *perag, struct xfs_trans *tp, + xfs_extlen_t *ask, xfs_extlen_t *used); +extern xfs_extlen_t xfs_iallocbt_calc_size(struct xfs_mount *mp, + unsigned long long len); + +void xfs_inobt_commit_staged_btree(struct xfs_btree_cur *cur, + struct xfs_trans *tp, struct xfs_buf *agbp); + +unsigned int xfs_iallocbt_maxlevels_ondisk(void); + +int __init xfs_inobt_init_cur_cache(void); +void xfs_inobt_destroy_cur_cache(void); + +#endif /* __XFS_IALLOC_BTREE_H__ */ diff --git a/fs/xfs/libxfs/xfs_iext_tree.c b/fs/xfs/libxfs/xfs_iext_tree.c new file mode 100644 index 000000000..5b2b926ab --- /dev/null +++ b/fs/xfs/libxfs/xfs_iext_tree.c @@ -0,0 +1,1067 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2017 Christoph Hellwig. + */ + +#include "xfs_platform.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_bit.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_trace.h" + +/* + * In-core extent record layout: + * + * +-------+----------------------------+ + * | 00:53 | all 54 bits of startoff | + * | 54:63 | low 10 bits of startblock | + * +-------+----------------------------+ + * | 00:20 | all 21 bits of length | + * | 21 | unwritten extent bit | + * | 22:63 | high 42 bits of startblock | + * +-------+----------------------------+ + */ +#define XFS_IEXT_STARTOFF_MASK xfs_mask64lo(BMBT_STARTOFF_BITLEN) +#define XFS_IEXT_LENGTH_MASK xfs_mask64lo(BMBT_BLOCKCOUNT_BITLEN) +#define XFS_IEXT_STARTBLOCK_MASK xfs_mask64lo(BMBT_STARTBLOCK_BITLEN) + +struct xfs_iext_rec { + uint64_t lo; + uint64_t hi; +}; + +/* + * Given that the length can't be a zero, only an empty hi value indicates an + * unused record. + */ +static bool xfs_iext_rec_is_empty(struct xfs_iext_rec *rec) +{ + return rec->hi == 0; +} + +static inline void xfs_iext_rec_clear(struct xfs_iext_rec *rec) +{ + rec->lo = 0; + rec->hi = 0; +} + +static void +xfs_iext_set( + struct xfs_iext_rec *rec, + struct xfs_bmbt_irec *irec) +{ + ASSERT((irec->br_startoff & ~XFS_IEXT_STARTOFF_MASK) == 0); + ASSERT((irec->br_blockcount & ~XFS_IEXT_LENGTH_MASK) == 0); + ASSERT((irec->br_startblock & ~XFS_IEXT_STARTBLOCK_MASK) == 0); + + rec->lo = irec->br_startoff & XFS_IEXT_STARTOFF_MASK; + rec->hi = irec->br_blockcount & XFS_IEXT_LENGTH_MASK; + + rec->lo |= (irec->br_startblock << 54); + rec->hi |= ((irec->br_startblock & ~xfs_mask64lo(10)) << (22 - 10)); + + if (irec->br_state == XFS_EXT_UNWRITTEN) + rec->hi |= (1 << 21); +} + +static void +xfs_iext_get( + struct xfs_bmbt_irec *irec, + struct xfs_iext_rec *rec) +{ + irec->br_startoff = rec->lo & XFS_IEXT_STARTOFF_MASK; + irec->br_blockcount = rec->hi & XFS_IEXT_LENGTH_MASK; + + irec->br_startblock = rec->lo >> 54; + irec->br_startblock |= (rec->hi & xfs_mask64hi(42)) >> (22 - 10); + + if (rec->hi & (1 << 21)) + irec->br_state = XFS_EXT_UNWRITTEN; + else + irec->br_state = XFS_EXT_NORM; +} + +enum { + NODE_SIZE = 256, + KEYS_PER_NODE = NODE_SIZE / (sizeof(uint64_t) + sizeof(void *)), + RECS_PER_LEAF = (NODE_SIZE - (2 * sizeof(struct xfs_iext_leaf *))) / + sizeof(struct xfs_iext_rec), +}; + +/* + * In-core extent btree block layout: + * + * There are two types of blocks in the btree: leaf and inner (non-leaf) blocks. + * + * The leaf blocks are made up by %KEYS_PER_NODE extent records, which each + * contain the startoffset, blockcount, startblock and unwritten extent flag. + * See above for the exact format, followed by pointers to the previous and next + * leaf blocks (if there are any). + * + * The inner (non-leaf) blocks first contain KEYS_PER_NODE lookup keys, followed + * by an equal number of pointers to the btree blocks at the next lower level. + * + * +-------+-------+-------+-------+-------+----------+----------+ + * Leaf: | rec 1 | rec 2 | rec 3 | rec 4 | rec N | prev-ptr | next-ptr | + * +-------+-------+-------+-------+-------+----------+----------+ + * + * +-------+-------+-------+-------+-------+-------+------+-------+ + * Inner: | key 1 | key 2 | key 3 | key N | ptr 1 | ptr 2 | ptr3 | ptr N | + * +-------+-------+-------+-------+-------+-------+------+-------+ + */ +struct xfs_iext_node { + uint64_t keys[KEYS_PER_NODE]; +#define XFS_IEXT_KEY_INVALID (1ULL << 63) + void *ptrs[KEYS_PER_NODE]; +}; + +struct xfs_iext_leaf { + struct xfs_iext_rec recs[RECS_PER_LEAF]; + struct xfs_iext_leaf *prev; + struct xfs_iext_leaf *next; +}; + +inline xfs_extnum_t xfs_iext_count(struct xfs_ifork *ifp) +{ + return ifp->if_bytes / sizeof(struct xfs_iext_rec); +} + +static inline int xfs_iext_max_recs(struct xfs_ifork *ifp) +{ + if (ifp->if_height == 1) + return xfs_iext_count(ifp); + return RECS_PER_LEAF; +} + +static inline struct xfs_iext_rec *cur_rec(struct xfs_iext_cursor *cur) +{ + return &cur->leaf->recs[cur->pos]; +} + +static inline bool xfs_iext_valid(struct xfs_ifork *ifp, + struct xfs_iext_cursor *cur) +{ + if (!cur->leaf) + return false; + if (cur->pos < 0 || cur->pos >= xfs_iext_max_recs(ifp)) + return false; + if (xfs_iext_rec_is_empty(cur_rec(cur))) + return false; + return true; +} + +static void * +xfs_iext_find_first_leaf( + struct xfs_ifork *ifp) +{ + struct xfs_iext_node *node = ifp->if_data; + int height; + + if (!ifp->if_height) + return NULL; + + for (height = ifp->if_height; height > 1; height--) { + node = node->ptrs[0]; + ASSERT(node); + } + + return node; +} + +static void * +xfs_iext_find_last_leaf( + struct xfs_ifork *ifp) +{ + struct xfs_iext_node *node = ifp->if_data; + int height, i; + + if (!ifp->if_height) + return NULL; + + for (height = ifp->if_height; height > 1; height--) { + for (i = 1; i < KEYS_PER_NODE; i++) + if (!node->ptrs[i]) + break; + node = node->ptrs[i - 1]; + ASSERT(node); + } + + return node; +} + +void +xfs_iext_first( + struct xfs_ifork *ifp, + struct xfs_iext_cursor *cur) +{ + cur->pos = 0; + cur->leaf = xfs_iext_find_first_leaf(ifp); +} + +void +xfs_iext_last( + struct xfs_ifork *ifp, + struct xfs_iext_cursor *cur) +{ + int i; + + cur->leaf = xfs_iext_find_last_leaf(ifp); + if (!cur->leaf) { + cur->pos = 0; + return; + } + + for (i = 1; i < xfs_iext_max_recs(ifp); i++) { + if (xfs_iext_rec_is_empty(&cur->leaf->recs[i])) + break; + } + cur->pos = i - 1; +} + +void +xfs_iext_next( + struct xfs_ifork *ifp, + struct xfs_iext_cursor *cur) +{ + if (!cur->leaf) { + ASSERT(cur->pos <= 0 || cur->pos >= RECS_PER_LEAF); + xfs_iext_first(ifp, cur); + return; + } + + ASSERT(cur->pos >= 0); + ASSERT(cur->pos < xfs_iext_max_recs(ifp)); + + cur->pos++; + if (ifp->if_height > 1 && !xfs_iext_valid(ifp, cur) && + cur->leaf->next) { + cur->leaf = cur->leaf->next; + cur->pos = 0; + } +} + +void +xfs_iext_prev( + struct xfs_ifork *ifp, + struct xfs_iext_cursor *cur) +{ + if (!cur->leaf) { + ASSERT(cur->pos <= 0 || cur->pos >= RECS_PER_LEAF); + xfs_iext_last(ifp, cur); + return; + } + + ASSERT(cur->pos >= 0); + ASSERT(cur->pos <= RECS_PER_LEAF); + +recurse: + do { + cur->pos--; + if (xfs_iext_valid(ifp, cur)) + return; + } while (cur->pos > 0); + + if (ifp->if_height > 1 && cur->leaf->prev) { + cur->leaf = cur->leaf->prev; + cur->pos = RECS_PER_LEAF; + goto recurse; + } +} + +static inline int +xfs_iext_key_cmp( + struct xfs_iext_node *node, + int n, + xfs_fileoff_t offset) +{ + if (node->keys[n] > offset) + return 1; + if (node->keys[n] < offset) + return -1; + return 0; +} + +static inline int +xfs_iext_rec_cmp( + struct xfs_iext_rec *rec, + xfs_fileoff_t offset) +{ + uint64_t rec_offset = rec->lo & XFS_IEXT_STARTOFF_MASK; + uint32_t rec_len = rec->hi & XFS_IEXT_LENGTH_MASK; + + if (rec_offset > offset) + return 1; + if (rec_offset + rec_len <= offset) + return -1; + return 0; +} + +static void * +xfs_iext_find_level( + struct xfs_ifork *ifp, + xfs_fileoff_t offset, + int level) +{ + struct xfs_iext_node *node = ifp->if_data; + int height, i; + + if (!ifp->if_height) + return NULL; + + for (height = ifp->if_height; height > level; height--) { + for (i = 1; i < KEYS_PER_NODE; i++) + if (xfs_iext_key_cmp(node, i, offset) > 0) + break; + + node = node->ptrs[i - 1]; + if (!node) + break; + } + + return node; +} + +static int +xfs_iext_node_pos( + struct xfs_iext_node *node, + xfs_fileoff_t offset) +{ + int i; + + for (i = 1; i < KEYS_PER_NODE; i++) { + if (xfs_iext_key_cmp(node, i, offset) > 0) + break; + } + + return i - 1; +} + +static int +xfs_iext_node_insert_pos( + struct xfs_iext_node *node, + xfs_fileoff_t offset) +{ + int i; + + for (i = 0; i < KEYS_PER_NODE; i++) { + if (xfs_iext_key_cmp(node, i, offset) > 0) + return i; + } + + return KEYS_PER_NODE; +} + +static int +xfs_iext_node_nr_entries( + struct xfs_iext_node *node, + int start) +{ + int i; + + for (i = start; i < KEYS_PER_NODE; i++) { + if (node->keys[i] == XFS_IEXT_KEY_INVALID) + break; + } + + return i; +} + +static int +xfs_iext_leaf_nr_entries( + struct xfs_ifork *ifp, + struct xfs_iext_leaf *leaf, + int start) +{ + int i; + + for (i = start; i < xfs_iext_max_recs(ifp); i++) { + if (xfs_iext_rec_is_empty(&leaf->recs[i])) + break; + } + + return i; +} + +static inline uint64_t +xfs_iext_leaf_key( + struct xfs_iext_leaf *leaf, + int n) +{ + return leaf->recs[n].lo & XFS_IEXT_STARTOFF_MASK; +} + +static inline void * +xfs_iext_alloc_node( + int size) +{ + return kzalloc(size, GFP_KERNEL | __GFP_NOLOCKDEP | __GFP_NOFAIL); +} + +static void +xfs_iext_grow( + struct xfs_ifork *ifp) +{ + struct xfs_iext_node *node = xfs_iext_alloc_node(NODE_SIZE); + int i; + + if (ifp->if_height == 1) { + struct xfs_iext_leaf *prev = ifp->if_data; + + node->keys[0] = xfs_iext_leaf_key(prev, 0); + node->ptrs[0] = prev; + } else { + struct xfs_iext_node *prev = ifp->if_data; + + ASSERT(ifp->if_height > 1); + + node->keys[0] = prev->keys[0]; + node->ptrs[0] = prev; + } + + for (i = 1; i < KEYS_PER_NODE; i++) + node->keys[i] = XFS_IEXT_KEY_INVALID; + + ifp->if_data = node; + ifp->if_height++; +} + +static void +xfs_iext_update_node( + struct xfs_ifork *ifp, + xfs_fileoff_t old_offset, + xfs_fileoff_t new_offset, + int level, + void *ptr) +{ + struct xfs_iext_node *node = ifp->if_data; + int height, i; + + for (height = ifp->if_height; height > level; height--) { + for (i = 0; i < KEYS_PER_NODE; i++) { + if (i > 0 && xfs_iext_key_cmp(node, i, old_offset) > 0) + break; + if (node->keys[i] == old_offset) + node->keys[i] = new_offset; + } + node = node->ptrs[i - 1]; + ASSERT(node); + } + + ASSERT(node == ptr); +} + +static struct xfs_iext_node * +xfs_iext_split_node( + struct xfs_iext_node **nodep, + int *pos, + int *nr_entries) +{ + struct xfs_iext_node *node = *nodep; + struct xfs_iext_node *new = xfs_iext_alloc_node(NODE_SIZE); + const int nr_move = KEYS_PER_NODE / 2; + int nr_keep = nr_move + (KEYS_PER_NODE & 1); + int i = 0; + + /* for sequential append operations just spill over into the new node */ + if (*pos == KEYS_PER_NODE) { + *nodep = new; + *pos = 0; + *nr_entries = 0; + goto done; + } + + + for (i = 0; i < nr_move; i++) { + new->keys[i] = node->keys[nr_keep + i]; + new->ptrs[i] = node->ptrs[nr_keep + i]; + + node->keys[nr_keep + i] = XFS_IEXT_KEY_INVALID; + node->ptrs[nr_keep + i] = NULL; + } + + if (*pos >= nr_keep) { + *nodep = new; + *pos -= nr_keep; + *nr_entries = nr_move; + } else { + *nr_entries = nr_keep; + } +done: + for (; i < KEYS_PER_NODE; i++) + new->keys[i] = XFS_IEXT_KEY_INVALID; + return new; +} + +static void +xfs_iext_insert_node( + struct xfs_ifork *ifp, + uint64_t offset, + void *ptr, + int level) +{ + struct xfs_iext_node *node, *new; + int i, pos, nr_entries; + +again: + if (ifp->if_height < level) + xfs_iext_grow(ifp); + + new = NULL; + node = xfs_iext_find_level(ifp, offset, level); + pos = xfs_iext_node_insert_pos(node, offset); + nr_entries = xfs_iext_node_nr_entries(node, pos); + + ASSERT(pos >= nr_entries || xfs_iext_key_cmp(node, pos, offset) != 0); + ASSERT(nr_entries <= KEYS_PER_NODE); + + if (nr_entries == KEYS_PER_NODE) + new = xfs_iext_split_node(&node, &pos, &nr_entries); + + /* + * Update the pointers in higher levels if the first entry changes + * in an existing node. + */ + if (node != new && pos == 0 && nr_entries > 0) + xfs_iext_update_node(ifp, node->keys[0], offset, level, node); + + for (i = nr_entries; i > pos; i--) { + node->keys[i] = node->keys[i - 1]; + node->ptrs[i] = node->ptrs[i - 1]; + } + node->keys[pos] = offset; + node->ptrs[pos] = ptr; + + if (new) { + offset = new->keys[0]; + ptr = new; + level++; + goto again; + } +} + +static struct xfs_iext_leaf * +xfs_iext_split_leaf( + struct xfs_iext_cursor *cur, + int *nr_entries) +{ + struct xfs_iext_leaf *leaf = cur->leaf; + struct xfs_iext_leaf *new = xfs_iext_alloc_node(NODE_SIZE); + const int nr_move = RECS_PER_LEAF / 2; + int nr_keep = nr_move + (RECS_PER_LEAF & 1); + int i; + + /* for sequential append operations just spill over into the new node */ + if (cur->pos == RECS_PER_LEAF) { + cur->leaf = new; + cur->pos = 0; + *nr_entries = 0; + goto done; + } + + for (i = 0; i < nr_move; i++) { + new->recs[i] = leaf->recs[nr_keep + i]; + xfs_iext_rec_clear(&leaf->recs[nr_keep + i]); + } + + if (cur->pos >= nr_keep) { + cur->leaf = new; + cur->pos -= nr_keep; + *nr_entries = nr_move; + } else { + *nr_entries = nr_keep; + } +done: + if (leaf->next) + leaf->next->prev = new; + new->next = leaf->next; + new->prev = leaf; + leaf->next = new; + return new; +} + +static void +xfs_iext_alloc_root( + struct xfs_ifork *ifp, + struct xfs_iext_cursor *cur) +{ + ASSERT(ifp->if_bytes == 0); + + ifp->if_data = xfs_iext_alloc_node(sizeof(struct xfs_iext_rec)); + ifp->if_height = 1; + + /* now that we have a node step into it */ + cur->leaf = ifp->if_data; + cur->pos = 0; +} + +static void +xfs_iext_realloc_root( + struct xfs_ifork *ifp, + struct xfs_iext_cursor *cur) +{ + int64_t new_size = ifp->if_bytes + sizeof(struct xfs_iext_rec); + void *new; + + /* account for the prev/next pointers */ + if (new_size / sizeof(struct xfs_iext_rec) == RECS_PER_LEAF) + new_size = NODE_SIZE; + + new = krealloc(ifp->if_data, new_size, + GFP_KERNEL | __GFP_NOLOCKDEP | __GFP_NOFAIL); + memset(new + ifp->if_bytes, 0, new_size - ifp->if_bytes); + ifp->if_data = new; + cur->leaf = new; +} + +/* + * Increment the sequence counter on extent tree changes. If we are on a COW + * fork, this allows the writeback code to skip looking for a COW extent if the + * COW fork hasn't changed. We use WRITE_ONCE here to ensure the update to the + * sequence counter is seen before the modifications to the extent tree itself + * take effect. + */ +static inline void xfs_iext_inc_seq(struct xfs_ifork *ifp) +{ + WRITE_ONCE(ifp->if_seq, READ_ONCE(ifp->if_seq) + 1); +} + +void +xfs_iext_insert_raw( + struct xfs_ifork *ifp, + struct xfs_iext_cursor *cur, + struct xfs_bmbt_irec *irec) +{ + xfs_fileoff_t offset = irec->br_startoff; + struct xfs_iext_leaf *new = NULL; + int nr_entries, i; + + xfs_iext_inc_seq(ifp); + + if (ifp->if_height == 0) + xfs_iext_alloc_root(ifp, cur); + else if (ifp->if_height == 1) + xfs_iext_realloc_root(ifp, cur); + + nr_entries = xfs_iext_leaf_nr_entries(ifp, cur->leaf, cur->pos); + ASSERT(nr_entries <= RECS_PER_LEAF); + ASSERT(cur->pos >= nr_entries || + xfs_iext_rec_cmp(cur_rec(cur), irec->br_startoff) != 0); + + if (nr_entries == RECS_PER_LEAF) + new = xfs_iext_split_leaf(cur, &nr_entries); + + /* + * Update the pointers in higher levels if the first entry changes + * in an existing node. + */ + if (cur->leaf != new && cur->pos == 0 && nr_entries > 0) { + xfs_iext_update_node(ifp, xfs_iext_leaf_key(cur->leaf, 0), + offset, 1, cur->leaf); + } + + for (i = nr_entries; i > cur->pos; i--) + cur->leaf->recs[i] = cur->leaf->recs[i - 1]; + xfs_iext_set(cur_rec(cur), irec); + ifp->if_bytes += sizeof(struct xfs_iext_rec); + + if (new) + xfs_iext_insert_node(ifp, xfs_iext_leaf_key(new, 0), new, 2); +} + +void +xfs_iext_insert( + struct xfs_inode *ip, + struct xfs_iext_cursor *cur, + struct xfs_bmbt_irec *irec, + int state) +{ + struct xfs_ifork *ifp = xfs_iext_state_to_fork(ip, state); + + xfs_iext_insert_raw(ifp, cur, irec); + trace_xfs_iext_insert(ip, cur, state, _RET_IP_); +} + +static struct xfs_iext_node * +xfs_iext_rebalance_node( + struct xfs_iext_node *parent, + int *pos, + struct xfs_iext_node *node, + int nr_entries) +{ + /* + * If the neighbouring nodes are completely full, or have different + * parents, we might never be able to merge our node, and will only + * delete it once the number of entries hits zero. + */ + if (nr_entries == 0) + return node; + + if (*pos > 0) { + struct xfs_iext_node *prev = parent->ptrs[*pos - 1]; + int nr_prev = xfs_iext_node_nr_entries(prev, 0), i; + + if (nr_prev + nr_entries <= KEYS_PER_NODE) { + for (i = 0; i < nr_entries; i++) { + prev->keys[nr_prev + i] = node->keys[i]; + prev->ptrs[nr_prev + i] = node->ptrs[i]; + } + return node; + } + } + + if (*pos + 1 < xfs_iext_node_nr_entries(parent, *pos)) { + struct xfs_iext_node *next = parent->ptrs[*pos + 1]; + int nr_next = xfs_iext_node_nr_entries(next, 0), i; + + if (nr_entries + nr_next <= KEYS_PER_NODE) { + /* + * Merge the next node into this node so that we don't + * have to do an additional update of the keys in the + * higher levels. + */ + for (i = 0; i < nr_next; i++) { + node->keys[nr_entries + i] = next->keys[i]; + node->ptrs[nr_entries + i] = next->ptrs[i]; + } + + ++*pos; + return next; + } + } + + return NULL; +} + +static void +xfs_iext_remove_node( + struct xfs_ifork *ifp, + xfs_fileoff_t offset, + void *victim) +{ + struct xfs_iext_node *node, *parent; + int level = 2, pos, nr_entries, i; + + ASSERT(level <= ifp->if_height); + node = xfs_iext_find_level(ifp, offset, level); + pos = xfs_iext_node_pos(node, offset); +again: + ASSERT(node->ptrs[pos]); + ASSERT(node->ptrs[pos] == victim); + kfree(victim); + + nr_entries = xfs_iext_node_nr_entries(node, pos) - 1; + offset = node->keys[0]; + for (i = pos; i < nr_entries; i++) { + node->keys[i] = node->keys[i + 1]; + node->ptrs[i] = node->ptrs[i + 1]; + } + node->keys[nr_entries] = XFS_IEXT_KEY_INVALID; + node->ptrs[nr_entries] = NULL; + + if (pos == 0 && nr_entries > 0) { + xfs_iext_update_node(ifp, offset, node->keys[0], level, node); + offset = node->keys[0]; + } + + if (nr_entries >= KEYS_PER_NODE / 2) + return; + + if (level < ifp->if_height) { + /* + * If we aren't at the root yet try to find a neighbour node to + * merge with (or delete the node if it is empty), and then + * recurse up to the next level. + */ + level++; + parent = xfs_iext_find_level(ifp, offset, level); + pos = xfs_iext_node_pos(parent, offset); + + ASSERT(pos != KEYS_PER_NODE); + ASSERT(parent->ptrs[pos] == node); + + node = xfs_iext_rebalance_node(parent, &pos, node, nr_entries); + if (node) { + victim = node; + node = parent; + goto again; + } + } else if (nr_entries == 1) { + /* + * If we are at the root and only one entry is left we can just + * free this node and update the root pointer. + */ + ASSERT(node == ifp->if_data); + ifp->if_data = node->ptrs[0]; + ifp->if_height--; + kfree(node); + } +} + +static void +xfs_iext_rebalance_leaf( + struct xfs_ifork *ifp, + struct xfs_iext_cursor *cur, + struct xfs_iext_leaf *leaf, + xfs_fileoff_t offset, + int nr_entries) +{ + /* + * If the neighbouring nodes are completely full we might never be able + * to merge our node, and will only delete it once the number of + * entries hits zero. + */ + if (nr_entries == 0) + goto remove_node; + + if (leaf->prev) { + int nr_prev = xfs_iext_leaf_nr_entries(ifp, leaf->prev, 0), i; + + if (nr_prev + nr_entries <= RECS_PER_LEAF) { + for (i = 0; i < nr_entries; i++) + leaf->prev->recs[nr_prev + i] = leaf->recs[i]; + + if (cur->leaf == leaf) { + cur->leaf = leaf->prev; + cur->pos += nr_prev; + } + goto remove_node; + } + } + + if (leaf->next) { + int nr_next = xfs_iext_leaf_nr_entries(ifp, leaf->next, 0), i; + + if (nr_entries + nr_next <= RECS_PER_LEAF) { + /* + * Merge the next node into this node so that we don't + * have to do an additional update of the keys in the + * higher levels. + */ + for (i = 0; i < nr_next; i++) { + leaf->recs[nr_entries + i] = + leaf->next->recs[i]; + } + + if (cur->leaf == leaf->next) { + cur->leaf = leaf; + cur->pos += nr_entries; + } + + offset = xfs_iext_leaf_key(leaf->next, 0); + leaf = leaf->next; + goto remove_node; + } + } + + return; +remove_node: + if (leaf->prev) + leaf->prev->next = leaf->next; + if (leaf->next) + leaf->next->prev = leaf->prev; + xfs_iext_remove_node(ifp, offset, leaf); +} + +static void +xfs_iext_free_last_leaf( + struct xfs_ifork *ifp) +{ + ifp->if_height--; + kfree(ifp->if_data); + ifp->if_data = NULL; +} + +void +xfs_iext_remove( + struct xfs_inode *ip, + struct xfs_iext_cursor *cur, + int state) +{ + struct xfs_ifork *ifp = xfs_iext_state_to_fork(ip, state); + struct xfs_iext_leaf *leaf = cur->leaf; + xfs_fileoff_t offset = xfs_iext_leaf_key(leaf, 0); + int i, nr_entries; + + trace_xfs_iext_remove(ip, cur, state, _RET_IP_); + + ASSERT(ifp->if_height > 0); + ASSERT(ifp->if_data != NULL); + ASSERT(xfs_iext_valid(ifp, cur)); + + xfs_iext_inc_seq(ifp); + + nr_entries = xfs_iext_leaf_nr_entries(ifp, leaf, cur->pos) - 1; + for (i = cur->pos; i < nr_entries; i++) + leaf->recs[i] = leaf->recs[i + 1]; + xfs_iext_rec_clear(&leaf->recs[nr_entries]); + ifp->if_bytes -= sizeof(struct xfs_iext_rec); + + if (cur->pos == 0 && nr_entries > 0) { + xfs_iext_update_node(ifp, offset, xfs_iext_leaf_key(leaf, 0), 1, + leaf); + offset = xfs_iext_leaf_key(leaf, 0); + } else if (cur->pos == nr_entries) { + if (ifp->if_height > 1 && leaf->next) + cur->leaf = leaf->next; + else + cur->leaf = NULL; + cur->pos = 0; + } + + if (nr_entries >= RECS_PER_LEAF / 2) + return; + + if (ifp->if_height > 1) + xfs_iext_rebalance_leaf(ifp, cur, leaf, offset, nr_entries); + else if (nr_entries == 0) + xfs_iext_free_last_leaf(ifp); +} + +/* + * Lookup the extent covering bno. + * + * If there is an extent covering bno return the extent index, and store the + * expanded extent structure in *gotp, and the extent cursor in *cur. + * If there is no extent covering bno, but there is an extent after it (e.g. + * it lies in a hole) return that extent in *gotp and its cursor in *cur + * instead. + * If bno is beyond the last extent return false, and return an invalid + * cursor value. + */ +bool +xfs_iext_lookup_extent( + struct xfs_inode *ip, + struct xfs_ifork *ifp, + xfs_fileoff_t offset, + struct xfs_iext_cursor *cur, + struct xfs_bmbt_irec *gotp) +{ + XFS_STATS_INC(ip->i_mount, xs_look_exlist); + + cur->leaf = xfs_iext_find_level(ifp, offset, 1); + if (!cur->leaf) { + cur->pos = 0; + return false; + } + + for (cur->pos = 0; cur->pos < xfs_iext_max_recs(ifp); cur->pos++) { + struct xfs_iext_rec *rec = cur_rec(cur); + + if (xfs_iext_rec_is_empty(rec)) + break; + if (xfs_iext_rec_cmp(rec, offset) >= 0) + goto found; + } + + /* Try looking in the next node for an entry > offset */ + if (ifp->if_height == 1 || !cur->leaf->next) + return false; + cur->leaf = cur->leaf->next; + cur->pos = 0; + if (!xfs_iext_valid(ifp, cur)) + return false; +found: + xfs_iext_get(gotp, cur_rec(cur)); + return true; +} + +/* + * Returns the last extent before end, and if this extent doesn't cover + * end, update end to the end of the extent. + */ +bool +xfs_iext_lookup_extent_before( + struct xfs_inode *ip, + struct xfs_ifork *ifp, + xfs_fileoff_t *end, + struct xfs_iext_cursor *cur, + struct xfs_bmbt_irec *gotp) +{ + /* could be optimized to not even look up the next on a match.. */ + if (xfs_iext_lookup_extent(ip, ifp, *end - 1, cur, gotp) && + gotp->br_startoff <= *end - 1) + return true; + if (!xfs_iext_prev_extent(ifp, cur, gotp)) + return false; + *end = gotp->br_startoff + gotp->br_blockcount; + return true; +} + +void +xfs_iext_update_extent( + struct xfs_inode *ip, + int state, + struct xfs_iext_cursor *cur, + struct xfs_bmbt_irec *new) +{ + struct xfs_ifork *ifp = xfs_iext_state_to_fork(ip, state); + + xfs_iext_inc_seq(ifp); + + if (cur->pos == 0) { + struct xfs_bmbt_irec old; + + xfs_iext_get(&old, cur_rec(cur)); + if (new->br_startoff != old.br_startoff) { + xfs_iext_update_node(ifp, old.br_startoff, + new->br_startoff, 1, cur->leaf); + } + } + + trace_xfs_bmap_pre_update(ip, cur, state, _RET_IP_); + xfs_iext_set(cur_rec(cur), new); + trace_xfs_bmap_post_update(ip, cur, state, _RET_IP_); +} + +/* + * Return true if the cursor points at an extent and return the extent structure + * in gotp. Else return false. + */ +bool +xfs_iext_get_extent( + struct xfs_ifork *ifp, + struct xfs_iext_cursor *cur, + struct xfs_bmbt_irec *gotp) +{ + if (!xfs_iext_valid(ifp, cur)) + return false; + xfs_iext_get(gotp, cur_rec(cur)); + return true; +} + +/* + * This is a recursive function, because of that we need to be extremely + * careful with stack usage. + */ +static void +xfs_iext_destroy_node( + struct xfs_iext_node *node, + int level) +{ + int i; + + if (level > 1) { + for (i = 0; i < KEYS_PER_NODE; i++) { + if (node->keys[i] == XFS_IEXT_KEY_INVALID) + break; + xfs_iext_destroy_node(node->ptrs[i], level - 1); + } + } + + kfree(node); +} + +void +xfs_iext_destroy( + struct xfs_ifork *ifp) +{ + xfs_iext_destroy_node(ifp->if_data, ifp->if_height); + + ifp->if_bytes = 0; + ifp->if_height = 0; + ifp->if_data = NULL; +} diff --git a/fs/xfs/libxfs/xfs_inode_buf.c b/fs/xfs/libxfs/xfs_inode_buf.c new file mode 100644 index 000000000..e4c3f7b24 --- /dev/null +++ b/fs/xfs/libxfs/xfs_inode_buf.c @@ -0,0 +1,975 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2006 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_ag.h" +#include "xfs_inode.h" +#include "xfs_errortag.h" +#include "xfs_error.h" +#include "xfs_icache.h" +#include "xfs_trans.h" +#include "xfs_ialloc.h" +#include "xfs_dir2.h" +#include "xfs_health.h" +#include "xfs_metafile.h" + +#include + +/* + * If we are doing readahead on an inode buffer, we might be in log recovery + * reading an inode allocation buffer that hasn't yet been replayed, and hence + * has not had the inode cores stamped into it. Hence for readahead, the buffer + * may be potentially invalid. + * + * If the readahead buffer is invalid, we need to mark it with an error so that a + * followup read will re-read it from disk. + * + * We don't report the error otherwise to avoid warnings during log recovery and + * we don't get unnecessary panics on debug kernels. Use EIO here because all + * we want to do is say readahead failed; there is no-one to report the error + * to, so this will distinguish it from a non-ra verifier failure. + * + * Changes to this readahead error behaviour also need to be reflected in + * xfs_dquot_buf_readahead_verify(). + */ +static void +xfs_inode_buf_verify( + struct xfs_buf *bp, + bool readahead) +{ + struct xfs_mount *mp = bp->b_mount; + int i; + int ni; + + /* + * Validate the magic number and version of every inode in the buffer + */ + ni = XFS_BB_TO_FSB(mp, bp->b_length) * mp->m_sb.sb_inopblock; + for (i = 0; i < ni; i++) { + struct xfs_dinode *dip; + xfs_agino_t unlinked_ino; + int di_ok; + + dip = xfs_buf_offset(bp, (i << mp->m_sb.sb_inodelog)); + unlinked_ino = be32_to_cpu(dip->di_next_unlinked); + di_ok = xfs_verify_magic16(bp, dip->di_magic) && + xfs_dinode_good_version(mp, dip->di_version) && + xfs_verify_agino_or_null(bp->b_pag, unlinked_ino); + if (unlikely(!di_ok || + XFS_TEST_ERROR(mp, XFS_ERRTAG_ITOBP_INOTOBP))) { + if (readahead) { + xfs_buf_ioerror(bp, -EIO); + return; + } + +#ifdef DEBUG + xfs_alert(mp, + "bad inode magic/vsn daddr %lld #%d (magic=%x)", + (unsigned long long)xfs_buf_daddr(bp), i, + be16_to_cpu(dip->di_magic)); +#endif + xfs_buf_verifier_error(bp, -EFSCORRUPTED, + __func__, dip, sizeof(*dip), + NULL); + return; + } + } +} + + +static void +xfs_inode_buf_read_verify( + struct xfs_buf *bp) +{ + xfs_inode_buf_verify(bp, false); +} + +static void +xfs_inode_buf_readahead_verify( + struct xfs_buf *bp) +{ + xfs_inode_buf_verify(bp, true); +} + +static void +xfs_inode_buf_write_verify( + struct xfs_buf *bp) +{ + xfs_inode_buf_verify(bp, false); +} + +const struct xfs_buf_ops xfs_inode_buf_ops = { + .name = "xfs_inode", + .magic16 = { cpu_to_be16(XFS_DINODE_MAGIC), + cpu_to_be16(XFS_DINODE_MAGIC) }, + .verify_read = xfs_inode_buf_read_verify, + .verify_write = xfs_inode_buf_write_verify, +}; + +const struct xfs_buf_ops xfs_inode_buf_ra_ops = { + .name = "xfs_inode_ra", + .magic16 = { cpu_to_be16(XFS_DINODE_MAGIC), + cpu_to_be16(XFS_DINODE_MAGIC) }, + .verify_read = xfs_inode_buf_readahead_verify, + .verify_write = xfs_inode_buf_write_verify, +}; + + +/* + * Read the inode cluster at @bno and return it in @bpp. + */ +int +xfs_read_icluster( + struct xfs_perag *pag, + struct xfs_trans *tp, + xfs_agblock_t agbno, + struct xfs_buf **bpp) +{ + struct xfs_mount *mp = pag_mount(pag); + int error; + + error = xfs_trans_read_buf(mp, tp, mp->m_ddev_targp, + xfs_agbno_to_daddr(pag, agbno), + XFS_FSB_TO_BB(mp, M_IGEO(mp)->blocks_per_cluster), + 0, bpp, &xfs_inode_buf_ops); + if (xfs_metadata_is_sick(error)) + xfs_agno_mark_sick(mp, pag_agno(pag), XFS_SICK_AG_INODES); + return error; +} + +static inline struct timespec64 xfs_inode_decode_bigtime(uint64_t ts) +{ + struct timespec64 tv; + uint32_t n; + + tv.tv_sec = xfs_bigtime_to_unix(div_u64_rem(ts, NSEC_PER_SEC, &n)); + tv.tv_nsec = n; + + return tv; +} + +/* Convert an ondisk timestamp to an incore timestamp. */ +struct timespec64 +xfs_inode_from_disk_ts( + struct xfs_dinode *dip, + const xfs_timestamp_t ts) +{ + struct timespec64 tv; + struct xfs_legacy_timestamp *lts; + + if (xfs_dinode_has_bigtime(dip)) + return xfs_inode_decode_bigtime(be64_to_cpu(ts)); + + lts = (struct xfs_legacy_timestamp *)&ts; + tv.tv_sec = (int)be32_to_cpu(lts->t_sec); + tv.tv_nsec = (int)be32_to_cpu(lts->t_nsec); + + return tv; +} + +int +xfs_inode_from_disk( + struct xfs_inode *ip, + struct xfs_dinode *from) +{ + struct inode *inode = VFS_I(ip); + int error; + xfs_failaddr_t fa; + + ASSERT(ip->i_cowfp == NULL); + + fa = xfs_dinode_verify(ip->i_mount, I_INO(ip), from); + if (fa) { + xfs_inode_verifier_error(ip, -EFSCORRUPTED, "dinode", from, + sizeof(*from), fa); + return -EFSCORRUPTED; + } + + /* + * First get the permanent information that is needed to allocate an + * inode. If the inode is unused, mode is zero and we shouldn't mess + * with the uninitialized part of it. + */ + if (!xfs_has_v3inodes(ip->i_mount)) + ip->i_flushiter = be16_to_cpu(from->di_flushiter); + inode->i_generation = be32_to_cpu(from->di_gen); + inode->i_mode = be16_to_cpu(from->di_mode); + if (!inode->i_mode) + return 0; + + /* + * Convert v1 inodes immediately to v2 inode format as this is the + * minimum inode version format we support in the rest of the code. + * They will also be unconditionally written back to disk as v2 inodes. + */ + if (unlikely(from->di_version == 1)) { + /* di_metatype used to be di_onlink */ + set_nlink(inode, be16_to_cpu(from->di_metatype)); + ip->i_projid = 0; + } else { + set_nlink(inode, be32_to_cpu(from->di_nlink)); + ip->i_projid = (prid_t)be16_to_cpu(from->di_projid_hi) << 16 | + be16_to_cpu(from->di_projid_lo); + if (xfs_dinode_is_metadir(from)) + ip->i_metatype = be16_to_cpu(from->di_metatype); + } + + i_uid_write(inode, be32_to_cpu(from->di_uid)); + i_gid_write(inode, be32_to_cpu(from->di_gid)); + + /* + * Time is signed, so need to convert to signed 32 bit before + * storing in inode timestamp which may be 64 bit. Otherwise + * a time before epoch is converted to a time long after epoch + * on 64 bit systems. + */ + inode_set_atime_to_ts(inode, + xfs_inode_from_disk_ts(from, from->di_atime)); + inode_set_mtime_to_ts(inode, + xfs_inode_from_disk_ts(from, from->di_mtime)); + inode_set_ctime_to_ts(inode, + xfs_inode_from_disk_ts(from, from->di_ctime)); + + ip->i_disk_size = be64_to_cpu(from->di_size); + ip->i_nblocks = be64_to_cpu(from->di_nblocks); + ip->i_extsize = be32_to_cpu(from->di_extsize); + ip->i_forkoff = from->di_forkoff; + ip->i_diflags = be16_to_cpu(from->di_flags); + ip->i_next_unlinked = be32_to_cpu(from->di_next_unlinked); + + if (from->di_dmevmask || from->di_dmstate) + xfs_iflags_set(ip, XFS_IPRESERVE_DM_FIELDS); + + if (xfs_has_v3inodes(ip->i_mount)) { + inode_set_iversion_queried(inode, + be64_to_cpu(from->di_changecount)); + ip->i_crtime = xfs_inode_from_disk_ts(from, from->di_crtime); + ip->i_diflags2 = be64_to_cpu(from->di_flags2); + /* also covers the di_used_blocks union arm: */ + ip->i_cowextsize = be32_to_cpu(from->di_cowextsize); + BUILD_BUG_ON(sizeof(from->di_cowextsize) != + sizeof(from->di_used_blocks)); + } + + error = xfs_iformat_data_fork(ip, from); + if (error) + return error; + if (from->di_forkoff) { + error = xfs_iformat_attr_fork(ip, from); + if (error) + goto out_destroy_data_fork; + } + if (xfs_is_reflink_inode(ip)) + xfs_ifork_init_cow(ip); + if (xfs_is_metadir_inode(ip)) { + XFS_STATS_DEC(ip->i_mount, xs_inodes_active); + XFS_STATS_INC(ip->i_mount, xs_inodes_meta); + } + return 0; + +out_destroy_data_fork: + xfs_idestroy_fork(&ip->i_df); + return error; +} + +/* Convert an incore timestamp to an ondisk timestamp. */ +static inline xfs_timestamp_t +xfs_inode_to_disk_ts( + struct xfs_inode *ip, + const struct timespec64 tv) +{ + struct xfs_legacy_timestamp *lts; + xfs_timestamp_t ts; + + if (xfs_inode_has_bigtime(ip)) + return cpu_to_be64(xfs_inode_encode_bigtime(tv)); + + lts = (struct xfs_legacy_timestamp *)&ts; + lts->t_sec = cpu_to_be32(tv.tv_sec); + lts->t_nsec = cpu_to_be32(tv.tv_nsec); + + return ts; +} + +static inline void +xfs_inode_to_disk_iext_counters( + struct xfs_inode *ip, + struct xfs_dinode *to) +{ + if (xfs_inode_has_large_extent_counts(ip)) { + to->di_big_nextents = cpu_to_be64(xfs_ifork_nextents(&ip->i_df)); + to->di_big_anextents = cpu_to_be32(xfs_ifork_nextents(&ip->i_af)); + /* + * We might be upgrading the inode to use larger extent counters + * than was previously used. Hence zero the unused field. + */ + to->di_nrext64_pad = cpu_to_be16(0); + } else { + to->di_nextents = cpu_to_be32(xfs_ifork_nextents(&ip->i_df)); + to->di_anextents = cpu_to_be16(xfs_ifork_nextents(&ip->i_af)); + } +} + +void +xfs_inode_to_disk( + struct xfs_inode *ip, + struct xfs_dinode *to, + xfs_lsn_t lsn) +{ + struct inode *inode = VFS_I(ip); + + to->di_magic = cpu_to_be16(XFS_DINODE_MAGIC); + if (xfs_is_metadir_inode(ip)) + to->di_metatype = cpu_to_be16(ip->i_metatype); + else + to->di_metatype = 0; + + to->di_format = xfs_ifork_format(&ip->i_df); + to->di_uid = cpu_to_be32(i_uid_read(inode)); + to->di_gid = cpu_to_be32(i_gid_read(inode)); + to->di_projid_lo = cpu_to_be16(ip->i_projid & 0xffff); + to->di_projid_hi = cpu_to_be16(ip->i_projid >> 16); + + to->di_atime = xfs_inode_to_disk_ts(ip, inode_get_atime(inode)); + to->di_mtime = xfs_inode_to_disk_ts(ip, inode_get_mtime(inode)); + to->di_ctime = xfs_inode_to_disk_ts(ip, inode_get_ctime(inode)); + to->di_nlink = cpu_to_be32(inode->i_nlink); + to->di_gen = cpu_to_be32(inode->i_generation); + to->di_mode = cpu_to_be16(inode->i_mode); + + to->di_size = cpu_to_be64(ip->i_disk_size); + to->di_nblocks = cpu_to_be64(ip->i_nblocks); + to->di_extsize = cpu_to_be32(ip->i_extsize); + to->di_forkoff = ip->i_forkoff; + to->di_aformat = xfs_ifork_format(&ip->i_af); + to->di_flags = cpu_to_be16(ip->i_diflags); + + if (xfs_has_v3inodes(ip->i_mount)) { + to->di_version = 3; + to->di_changecount = cpu_to_be64(inode_peek_iversion(inode)); + to->di_crtime = xfs_inode_to_disk_ts(ip, ip->i_crtime); + to->di_flags2 = cpu_to_be64(ip->i_diflags2); + /* also covers the di_used_blocks union arm: */ + to->di_cowextsize = cpu_to_be32(ip->i_cowextsize); + to->di_ino = cpu_to_be64(I_INO(ip)); + to->di_lsn = cpu_to_be64(lsn); + memset(to->di_pad2, 0, sizeof(to->di_pad2)); + uuid_copy(&to->di_uuid, &ip->i_mount->m_sb.sb_meta_uuid); + to->di_v3_pad = 0; + } else { + to->di_version = 2; + to->di_flushiter = cpu_to_be16(ip->i_flushiter); + memset(to->di_v2_pad, 0, sizeof(to->di_v2_pad)); + } + + xfs_inode_to_disk_iext_counters(ip, to); +} + +static xfs_failaddr_t +xfs_dinode_verify_fork( + struct xfs_dinode *dip, + struct xfs_mount *mp, + int whichfork) +{ + xfs_extnum_t di_nextents; + xfs_extnum_t max_extents; + mode_t mode = be16_to_cpu(dip->di_mode); + uint32_t fork_size = XFS_DFORK_SIZE(dip, mp, whichfork); + uint32_t fork_format = XFS_DFORK_FORMAT(dip, whichfork); + + di_nextents = xfs_dfork_nextents(dip, whichfork); + + /* + * For fork types that can contain local data, check that the fork + * format matches the size of local data contained within the fork. + */ + if (whichfork == XFS_DATA_FORK) { + /* + * A directory small enough to fit in the inode must be stored + * in local format. The directory sf <-> extents conversion + * code updates the directory size accordingly. Directories + * being truncated have zero size and are not subject to this + * check. + */ + if (S_ISDIR(mode)) { + if (dip->di_size && + be64_to_cpu(dip->di_size) <= fork_size && + fork_format != XFS_DINODE_FMT_LOCAL) + return __this_address; + } + + /* + * A symlink with a target small enough to fit in the inode can + * be stored in extents format if xattrs were added (thus + * converting the data fork from shortform to remote format) + * and then removed. + */ + if (S_ISLNK(mode)) { + if (be64_to_cpu(dip->di_size) <= fork_size && + fork_format != XFS_DINODE_FMT_EXTENTS && + fork_format != XFS_DINODE_FMT_LOCAL) + return __this_address; + } + + /* + * For all types, check that when the size says the fork should + * be in extent or btree format, the inode isn't claiming to be + * in local format. + */ + if (be64_to_cpu(dip->di_size) > fork_size && + fork_format == XFS_DINODE_FMT_LOCAL) + return __this_address; + } + + switch (fork_format) { + case XFS_DINODE_FMT_LOCAL: + /* + * No local regular files yet. + */ + if (S_ISREG(mode) && whichfork == XFS_DATA_FORK) + return __this_address; + if (di_nextents) + return __this_address; + break; + case XFS_DINODE_FMT_EXTENTS: + if (di_nextents > XFS_DFORK_MAXEXT(dip, mp, whichfork)) + return __this_address; + break; + case XFS_DINODE_FMT_BTREE: + max_extents = xfs_iext_max_nextents( + xfs_dinode_has_large_extent_counts(dip), + whichfork); + if (di_nextents > max_extents) + return __this_address; + break; + case XFS_DINODE_FMT_META_BTREE: + if (!xfs_has_metadir(mp)) + return __this_address; + if (!(dip->di_flags2 & cpu_to_be64(XFS_DIFLAG2_METADATA))) + return __this_address; + switch (be16_to_cpu(dip->di_metatype)) { + case XFS_METAFILE_RTRMAP: + /* + * growfs must create the rtrmap inodes before adding a + * realtime volume to the filesystem, so we cannot use + * the rtrmapbt predicate here. + */ + if (!xfs_has_rmapbt(mp)) + return __this_address; + break; + case XFS_METAFILE_RTREFCOUNT: + /* same comment about growfs and rmap inodes applies */ + if (!xfs_has_reflink(mp)) + return __this_address; + break; + default: + return __this_address; + } + break; + default: + return __this_address; + } + return NULL; +} + +static xfs_failaddr_t +xfs_dinode_verify_forkoff( + struct xfs_dinode *dip, + struct xfs_mount *mp) +{ + if (!dip->di_forkoff) + return NULL; + + switch (dip->di_format) { + case XFS_DINODE_FMT_DEV: + if (dip->di_forkoff != (roundup(sizeof(xfs_dev_t), 8) >> 3)) + return __this_address; + break; + case XFS_DINODE_FMT_META_BTREE: + if (!xfs_has_metadir(mp) || !xfs_has_parent(mp)) + return __this_address; + fallthrough; + case XFS_DINODE_FMT_LOCAL: /* fall through ... */ + case XFS_DINODE_FMT_EXTENTS: /* fall through ... */ + case XFS_DINODE_FMT_BTREE: + if (dip->di_forkoff >= (XFS_LITINO(mp) >> 3)) + return __this_address; + break; + default: + return __this_address; + } + return NULL; +} + +static xfs_failaddr_t +xfs_dinode_verify_nrext64( + struct xfs_mount *mp, + struct xfs_dinode *dip) +{ + if (xfs_dinode_has_large_extent_counts(dip)) { + if (!xfs_has_large_extent_counts(mp)) + return __this_address; + if (dip->di_nrext64_pad != 0) + return __this_address; + } else if (dip->di_version >= 3) { + if (dip->di_v3_pad != 0) + return __this_address; + } + + return NULL; +} + +/* + * Validate all the picky requirements we have for a file that claims to be + * filesystem metadata. + */ +xfs_failaddr_t +xfs_dinode_verify_metadir( + struct xfs_mount *mp, + struct xfs_dinode *dip, + uint16_t mode, + uint16_t flags, + uint64_t flags2) +{ + if (!xfs_has_metadir(mp)) + return __this_address; + + /* V5 filesystem only */ + if (dip->di_version < 3) + return __this_address; + + if (be16_to_cpu(dip->di_metatype) >= XFS_METAFILE_MAX) + return __this_address; + + /* V3 inode fields that are always zero */ + if ((flags2 & XFS_DIFLAG2_NREXT64) && dip->di_nrext64_pad) + return __this_address; + if (!(flags2 & XFS_DIFLAG2_NREXT64) && dip->di_flushiter) + return __this_address; + + /* Metadata files can only be directories or regular files */ + if (!S_ISDIR(mode) && !S_ISREG(mode)) + return __this_address; + + /* They must have zero access permissions */ + if (mode & 0777) + return __this_address; + + /* DMAPI event and state masks are zero */ + if (dip->di_dmevmask || dip->di_dmstate) + return __this_address; + + /* + * User and group IDs must be zero. The project ID is used for + * grouping inodes. Metadata inodes are never accounted to quotas. + */ + if (dip->di_uid || dip->di_gid) + return __this_address; + + /* Mandatory inode flags must be set */ + if (S_ISDIR(mode)) { + if ((flags & XFS_METADIR_DIFLAGS) != XFS_METADIR_DIFLAGS) + return __this_address; + } else { + if ((flags & XFS_METAFILE_DIFLAGS) != XFS_METAFILE_DIFLAGS) + return __this_address; + } + + /* dax flags2 must not be set */ + if (flags2 & XFS_DIFLAG2_DAX) + return __this_address; + + return NULL; +} + +xfs_failaddr_t +xfs_dinode_verify( + struct xfs_mount *mp, + xfs_ino_t ino, + struct xfs_dinode *dip) +{ + xfs_failaddr_t fa; + uint16_t mode; + uint16_t flags; + uint64_t flags2; + uint64_t di_size; + xfs_extnum_t nextents; + xfs_extnum_t naextents; + xfs_filblks_t nblocks; + + if (dip->di_magic != cpu_to_be16(XFS_DINODE_MAGIC)) + return __this_address; + + /* Verify v3 integrity information first */ + if (dip->di_version >= 3) { + if (!xfs_has_v3inodes(mp)) + return __this_address; + if (!xfs_verify_cksum((char *)dip, mp->m_sb.sb_inodesize, + XFS_DINODE_CRC_OFF)) + return __this_address; + if (be64_to_cpu(dip->di_ino) != ino) + return __this_address; + if (!uuid_equal(&dip->di_uuid, &mp->m_sb.sb_meta_uuid)) + return __this_address; + } + + /* + * Historical note: xfsprogs in the 3.2 era set up its incore inodes to + * have di_nlink track the link count, even if the actual filesystem + * only supported V1 inodes (i.e. di_onlink). When writing out the + * ondisk inode, it would set both the ondisk di_nlink and di_onlink to + * the the incore di_nlink value, which is why we cannot check for + * di_nlink==0 on a V1 inode. V2/3 inodes would get written out with + * di_onlink==0, so we can check that. + */ + if (dip->di_version == 2) { + if (dip->di_metatype) + return __this_address; + } else if (dip->di_version >= 3) { + if (!xfs_dinode_is_metadir(dip) && dip->di_metatype) + return __this_address; + } + + /* don't allow invalid i_size */ + di_size = be64_to_cpu(dip->di_size); + if (di_size & (1ULL << 63)) + return __this_address; + + mode = be16_to_cpu(dip->di_mode); + if (mode && xfs_mode_to_ftype(mode) == XFS_DIR3_FT_UNKNOWN) + return __this_address; + + /* + * No zero-length symlinks/dirs unless they're unlinked and hence being + * inactivated. + */ + if ((S_ISLNK(mode) || S_ISDIR(mode)) && di_size == 0) { + if (dip->di_version > 1) { + if (dip->di_nlink) + return __this_address; + } else { + /* di_metatype used to be di_onlink */ + if (dip->di_metatype) + return __this_address; + } + } + + fa = xfs_dinode_verify_nrext64(mp, dip); + if (fa) + return fa; + + nextents = xfs_dfork_data_extents(dip); + naextents = xfs_dfork_attr_extents(dip); + nblocks = be64_to_cpu(dip->di_nblocks); + + /* Fork checks carried over from xfs_iformat_fork */ + if (mode && nextents + naextents > nblocks) + return __this_address; + + if (S_ISDIR(mode) && nextents > mp->m_dir_geo->max_extents) + return __this_address; + + if (mode && XFS_DFORK_BOFF(dip) > mp->m_sb.sb_inodesize) + return __this_address; + + flags = be16_to_cpu(dip->di_flags); + + if (mode && (flags & XFS_DIFLAG_REALTIME) && !mp->m_rtdev_targp) + return __this_address; + + /* check for illegal values of forkoff */ + fa = xfs_dinode_verify_forkoff(dip, mp); + if (fa) + return fa; + + /* Do we have appropriate data fork formats for the mode? */ + switch (mode & S_IFMT) { + case S_IFIFO: + case S_IFCHR: + case S_IFBLK: + case S_IFSOCK: + if (dip->di_format != XFS_DINODE_FMT_DEV) + return __this_address; + break; + case S_IFREG: + case S_IFLNK: + case S_IFDIR: + fa = xfs_dinode_verify_fork(dip, mp, XFS_DATA_FORK); + if (fa) + return fa; + break; + case 0: + /* Uninitialized inode ok. */ + break; + default: + return __this_address; + } + + if (dip->di_forkoff) { + fa = xfs_dinode_verify_fork(dip, mp, XFS_ATTR_FORK); + if (fa) + return fa; + } else { + /* + * If there is no fork offset, this may be a freshly-made inode + * in a new disk cluster, in which case di_aformat is zeroed. + * Otherwise, such an inode must be in EXTENTS format; this goes + * for freed inodes as well. + */ + switch (dip->di_aformat) { + case 0: + case XFS_DINODE_FMT_EXTENTS: + break; + default: + return __this_address; + } + if (naextents) + return __this_address; + } + + /* extent size hint validation */ + fa = xfs_inode_validate_extsize(mp, be32_to_cpu(dip->di_extsize), + mode, flags); + if (fa) + return fa; + + /* only version 3 or greater inodes are extensively verified here */ + if (dip->di_version < 3) + return NULL; + + flags2 = be64_to_cpu(dip->di_flags2); + + /* don't allow reflink/cowextsize if we don't have reflink */ + if ((flags2 & (XFS_DIFLAG2_REFLINK | XFS_DIFLAG2_COWEXTSIZE)) && + !xfs_has_reflink(mp)) + return __this_address; + + /* only regular files get reflink */ + if ((flags2 & XFS_DIFLAG2_REFLINK) && (mode & S_IFMT) != S_IFREG) + return __this_address; + + /* don't let reflink and realtime mix */ + if ((flags2 & XFS_DIFLAG2_REFLINK) && (flags & XFS_DIFLAG_REALTIME) && + !xfs_has_rtreflink(mp)) + return __this_address; + + if (xfs_has_zoned(mp) && + dip->di_metatype == cpu_to_be16(XFS_METAFILE_RTRMAP)) { + if (be32_to_cpu(dip->di_used_blocks) > mp->m_sb.sb_rgextents) + return __this_address; + } else { + /* COW extent size hint validation */ + fa = xfs_inode_validate_cowextsize(mp, + be32_to_cpu(dip->di_cowextsize), + mode, flags, flags2); + if (fa) + return fa; + } + + /* bigtime iflag can only happen on bigtime filesystems */ + if (xfs_dinode_has_bigtime(dip) && + !xfs_has_bigtime(mp)) + return __this_address; + + if (flags2 & XFS_DIFLAG2_METADATA) { + fa = xfs_dinode_verify_metadir(mp, dip, mode, flags, flags2); + if (fa) + return fa; + } + + /* metadata inodes containing btrees always have zero extent count */ + if (XFS_DFORK_FORMAT(dip, XFS_DATA_FORK) != XFS_DINODE_FMT_META_BTREE) { + if (nextents + naextents == 0 && nblocks != 0) + return __this_address; + } + + return NULL; +} + +void +xfs_dinode_calc_crc( + struct xfs_mount *mp, + struct xfs_dinode *dip) +{ + uint32_t crc; + + if (dip->di_version < 3) + return; + + ASSERT(xfs_has_crc(mp)); + crc = xfs_start_cksum_update((char *)dip, mp->m_sb.sb_inodesize, + XFS_DINODE_CRC_OFF); + dip->di_crc = xfs_end_cksum(crc); +} + +/* + * Validate di_extsize hint. + * + * 1. Extent size hint is only valid for directories and regular files. + * 2. FS_XFLAG_EXTSIZE is only valid for regular files. + * 3. FS_XFLAG_EXTSZINHERIT is only valid for directories. + * 4. Hint cannot be larger than MAXTEXTLEN. + * 5. Can be changed on directories at any time. + * 6. Hint value of 0 turns off hints, clears inode flags. + * 7. Extent size must be a multiple of the appropriate block size. + * For realtime files, this is the rt extent size. + * 8. For non-realtime files, the extent size hint must be limited + * to half the AG size to avoid alignment extending the extent beyond the + * limits of the AG. + */ +xfs_failaddr_t +xfs_inode_validate_extsize( + struct xfs_mount *mp, + uint32_t extsize, + uint16_t mode, + uint16_t flags) +{ + bool rt_flag; + bool hint_flag; + bool inherit_flag; + uint32_t extsize_bytes; + uint32_t blocksize_bytes; + + rt_flag = (flags & XFS_DIFLAG_REALTIME); + hint_flag = (flags & XFS_DIFLAG_EXTSIZE); + inherit_flag = (flags & XFS_DIFLAG_EXTSZINHERIT); + extsize_bytes = XFS_FSB_TO_B(mp, extsize); + + /* + * This comment describes a historic gap in this verifier function. + * + * For a directory with both RTINHERIT and EXTSZINHERIT flags set, this + * function has never checked that the extent size hint is an integer + * multiple of the realtime extent size. Since we allow users to set + * this combination on non-rt filesystems /and/ to change the rt + * extent size when adding a rt device to a filesystem, the net effect + * is that users can configure a filesystem anticipating one rt + * geometry and change their minds later. Directories do not use the + * extent size hint, so this is harmless for them. + * + * If a directory with a misaligned extent size hint is allowed to + * propagate that hint into a new regular realtime file, the result + * is that the inode cluster buffer verifier will trigger a corruption + * shutdown the next time it is run, because the verifier has always + * enforced the alignment rule for regular files. + * + * Because we allow administrators to set a new rt extent size when + * adding a rt section, we cannot add a check to this verifier because + * that will result a new source of directory corruption errors when + * reading an existing filesystem. Instead, we rely on callers to + * decide when alignment checks are appropriate, and fix things up as + * needed. + */ + + if (rt_flag) + blocksize_bytes = XFS_FSB_TO_B(mp, mp->m_sb.sb_rextsize); + else + blocksize_bytes = mp->m_sb.sb_blocksize; + + if ((hint_flag || inherit_flag) && !(S_ISDIR(mode) || S_ISREG(mode))) + return __this_address; + + if (hint_flag && !S_ISREG(mode)) + return __this_address; + + if (inherit_flag && !S_ISDIR(mode)) + return __this_address; + + if ((hint_flag || inherit_flag) && extsize == 0) + return __this_address; + + /* free inodes get flags set to zero but extsize remains */ + if (mode && !(hint_flag || inherit_flag) && extsize != 0) + return __this_address; + + if (extsize_bytes % blocksize_bytes) + return __this_address; + + if (extsize > XFS_MAX_BMBT_EXTLEN) + return __this_address; + + if (!rt_flag && extsize > mp->m_sb.sb_agblocks / 2) + return __this_address; + + return NULL; +} + +/* + * Validate di_cowextsize hint. + * + * 1. CoW extent size hint can only be set if reflink is enabled on the fs. + * The inode does not have to have any shared blocks, but it must be a v3. + * 2. FS_XFLAG_COWEXTSIZE is only valid for directories and regular files; + * for a directory, the hint is propagated to new files. + * 3. Can be changed on files & directories at any time. + * 4. Hint value of 0 turns off hints, clears inode flags. + * 5. Extent size must be a multiple of the appropriate block size. + * 6. The extent size hint must be limited to half the AG size to avoid + * alignment extending the extent beyond the limits of the AG. + */ +xfs_failaddr_t +xfs_inode_validate_cowextsize( + struct xfs_mount *mp, + uint32_t cowextsize, + uint16_t mode, + uint16_t flags, + uint64_t flags2) +{ + bool rt_flag; + bool hint_flag; + uint32_t cowextsize_bytes; + uint32_t blocksize_bytes; + + rt_flag = (flags & XFS_DIFLAG_REALTIME); + hint_flag = (flags2 & XFS_DIFLAG2_COWEXTSIZE); + cowextsize_bytes = XFS_FSB_TO_B(mp, cowextsize); + + /* + * Similar to extent size hints, a directory can be configured to + * propagate realtime status and a CoW extent size hint to newly + * created files even if there is no realtime device, and the hints on + * disk can become misaligned if the sysadmin changes the rt extent + * size while adding the realtime device. + * + * Therefore, we can only enforce the rextsize alignment check against + * regular realtime files, and rely on callers to decide when alignment + * checks are appropriate, and fix things up as needed. + */ + + if (rt_flag) + blocksize_bytes = XFS_FSB_TO_B(mp, mp->m_sb.sb_rextsize); + else + blocksize_bytes = mp->m_sb.sb_blocksize; + + if (hint_flag && !xfs_has_reflink(mp)) + return __this_address; + + if (hint_flag && !(S_ISDIR(mode) || S_ISREG(mode))) + return __this_address; + + if (hint_flag && cowextsize == 0) + return __this_address; + + /* free inodes get flags set to zero but cowextsize remains */ + if (mode && !hint_flag && cowextsize != 0) + return __this_address; + + if (cowextsize_bytes % blocksize_bytes) + return __this_address; + + if (cowextsize > XFS_MAX_BMBT_EXTLEN) + return __this_address; + + if (!rt_flag && cowextsize > mp->m_sb.sb_agblocks / 2) + return __this_address; + + return NULL; +} diff --git a/fs/xfs/libxfs/xfs_inode_buf.h b/fs/xfs/libxfs/xfs_inode_buf.h new file mode 100644 index 000000000..f3624532b --- /dev/null +++ b/fs/xfs/libxfs/xfs_inode_buf.h @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2003,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_INODE_BUF_H__ +#define __XFS_INODE_BUF_H__ + +struct xfs_inode; +struct xfs_dinode; + +/* + * Inode location information. Stored in the inode and passed to + * xfs_read_icluster() to get a buffer and dinode for a given inode. + */ +struct xfs_imap { + xfs_agblock_t im_agbno; /* starting agbno of inode cluster */ + unsigned short im_boffset; /* offset in inode cluster in bytes */ +} __packed; + +int xfs_read_icluster(struct xfs_perag *pag, struct xfs_trans *tp, + xfs_agblock_t agbno, struct xfs_buf **bpp); +void xfs_dinode_calc_crc(struct xfs_mount *mp, struct xfs_dinode *dip); +void xfs_inode_to_disk(struct xfs_inode *ip, struct xfs_dinode *to, + xfs_lsn_t lsn); +int xfs_inode_from_disk(struct xfs_inode *ip, struct xfs_dinode *from); + +xfs_failaddr_t xfs_dinode_verify(struct xfs_mount *mp, xfs_ino_t ino, + struct xfs_dinode *dip); +xfs_failaddr_t xfs_dinode_verify_metadir(struct xfs_mount *mp, + struct xfs_dinode *dip, uint16_t mode, uint16_t flags, + uint64_t flags2); +xfs_failaddr_t xfs_inode_validate_extsize(struct xfs_mount *mp, + uint32_t extsize, uint16_t mode, uint16_t flags); +xfs_failaddr_t xfs_inode_validate_cowextsize(struct xfs_mount *mp, + uint32_t cowextsize, uint16_t mode, uint16_t flags, + uint64_t flags2); + +static inline uint64_t xfs_inode_encode_bigtime(struct timespec64 tv) +{ + return xfs_unix_to_bigtime(tv.tv_sec) * NSEC_PER_SEC + tv.tv_nsec; +} + +struct timespec64 xfs_inode_from_disk_ts(struct xfs_dinode *dip, + const xfs_timestamp_t ts); + +static inline bool +xfs_dinode_good_version(struct xfs_mount *mp, uint8_t version) +{ + if (xfs_has_v3inodes(mp)) + return version == 3; + return version == 1 || version == 2; +} + + +#endif /* __XFS_INODE_BUF_H__ */ diff --git a/fs/xfs/libxfs/xfs_inode_fork.c b/fs/xfs/libxfs/xfs_inode_fork.c new file mode 100644 index 000000000..606a36526 --- /dev/null +++ b/fs/xfs/libxfs/xfs_inode_fork.c @@ -0,0 +1,778 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2006 Silicon Graphics, Inc. + * All Rights Reserved. + */ + +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_inode_item.h" +#include "xfs_btree.h" +#include "xfs_bmap_btree.h" +#include "xfs_bmap.h" +#include "xfs_error.h" +#include "xfs_trace.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_dir2_priv.h" +#include "xfs_attr_leaf.h" +#include "xfs_types.h" +#include "xfs_errortag.h" +#include "xfs_health.h" +#include "xfs_symlink_remote.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_rtrefcount_btree.h" + +struct kmem_cache *xfs_ifork_cache; + +void +xfs_init_local_fork( + struct xfs_inode *ip, + int whichfork, + const void *data, + int64_t size) +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + int mem_size = size; + bool zero_terminate; + + /* + * If we are using the local fork to store a symlink body we need to + * zero-terminate it so that we can pass it back to the VFS directly. + * Overallocate the in-memory fork by one for that and add a zero + * to terminate it below. + */ + zero_terminate = S_ISLNK(VFS_I(ip)->i_mode); + if (zero_terminate) + mem_size++; + + if (size) { + char *new_data = kmalloc(mem_size, + GFP_KERNEL | __GFP_NOLOCKDEP | __GFP_NOFAIL); + + memcpy(new_data, data, size); + if (zero_terminate) + new_data[size] = '\0'; + + ifp->if_data = new_data; + } else { + ifp->if_data = NULL; + } + + ifp->if_bytes = size; +} + +/* + * The file is in-lined in the on-disk inode. + */ +STATIC int +xfs_iformat_local( + struct xfs_inode *ip, + struct xfs_dinode *dip, + int whichfork, + int size) +{ + /* + * If the size is unreasonable, then something + * is wrong and we just bail out rather than crash in + * kmalloc() or memcpy() below. + */ + if (unlikely(size > XFS_DFORK_SIZE(dip, ip->i_mount, whichfork))) { + xfs_warn(ip->i_mount, + "corrupt inode %llu (bad size %d for local fork, size = %zd).", + (unsigned long long)I_INO(ip), size, + XFS_DFORK_SIZE(dip, ip->i_mount, whichfork)); + xfs_inode_verifier_error(ip, -EFSCORRUPTED, + "xfs_iformat_local", dip, sizeof(*dip), + __this_address); + xfs_inode_mark_sick(ip, XFS_SICK_INO_CORE); + return -EFSCORRUPTED; + } + + xfs_init_local_fork(ip, whichfork, XFS_DFORK_PTR(dip, whichfork), size); + return 0; +} + +/* + * The file consists of a set of extents all of which fit into the on-disk + * inode. + */ +STATIC int +xfs_iformat_extents( + struct xfs_inode *ip, + struct xfs_dinode *dip, + int whichfork) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + int state = xfs_bmap_fork_to_state(whichfork); + xfs_extnum_t nex = xfs_dfork_nextents(dip, whichfork); + int size = nex * sizeof(xfs_bmbt_rec_t); + struct xfs_iext_cursor icur; + struct xfs_bmbt_rec *dp; + struct xfs_bmbt_irec new; + int i; + + /* + * If the number of extents is unreasonable, then something is wrong and + * we just bail out rather than crash in kmalloc() or memcpy() below. + */ + if (unlikely(size < 0 || size > XFS_DFORK_SIZE(dip, mp, whichfork))) { + xfs_warn(ip->i_mount, "corrupt inode %llu ((a)extents = %llu).", + I_INO(ip), nex); + xfs_inode_verifier_error(ip, -EFSCORRUPTED, + "xfs_iformat_extents(1)", dip, sizeof(*dip), + __this_address); + xfs_inode_mark_sick(ip, XFS_SICK_INO_CORE); + return -EFSCORRUPTED; + } + + ifp->if_bytes = 0; + ifp->if_data = NULL; + ifp->if_height = 0; + if (size) { + dp = (xfs_bmbt_rec_t *) XFS_DFORK_PTR(dip, whichfork); + + xfs_iext_first(ifp, &icur); + for (i = 0; i < nex; i++, dp++) { + xfs_failaddr_t fa; + + xfs_bmbt_disk_get_all(dp, &new); + fa = xfs_bmap_validate_extent(ip, whichfork, &new); + if (fa) { + xfs_inode_verifier_error(ip, -EFSCORRUPTED, + "xfs_iformat_extents(2)", + dp, sizeof(*dp), fa); + xfs_inode_mark_sick(ip, XFS_SICK_INO_CORE); + return xfs_bmap_complain_bad_rec(ip, whichfork, + fa, &new); + } + + xfs_iext_insert(ip, &icur, &new, state); + trace_xfs_read_extent(ip, &icur, state, _THIS_IP_); + xfs_iext_next(ifp, &icur); + } + } + return 0; +} + +/* + * The file has too many extents to fit into + * the inode, so they are in B-tree format. + * Allocate a buffer for the root of the B-tree + * and copy the root into it. The i_extents + * field will remain NULL until all of the + * extents are read in (when they are needed). + */ +STATIC int +xfs_iformat_btree( + struct xfs_inode *ip, + struct xfs_dinode *dip, + int whichfork) +{ + struct xfs_mount *mp = ip->i_mount; + xfs_bmdr_block_t *dfp; + struct xfs_ifork *ifp; + struct xfs_btree_block *broot; + int nrecs; + int size; + int level; + + ifp = xfs_ifork_ptr(ip, whichfork); + dfp = (xfs_bmdr_block_t *)XFS_DFORK_PTR(dip, whichfork); + size = xfs_bmap_broot_space(mp, dfp); + nrecs = be16_to_cpu(dfp->bb_numrecs); + level = be16_to_cpu(dfp->bb_level); + + /* + * blow out if -- fork has less extents than can fit in + * fork (fork shouldn't be a btree format), root btree + * block has more records than can fit into the fork, + * or the number of extents is greater than the number of + * blocks. + */ + if (unlikely(ifp->if_nextents <= XFS_IFORK_MAXEXT(ip, whichfork) || + nrecs == 0 || + xfs_bmdr_space_calc(nrecs) > + XFS_DFORK_SIZE(dip, mp, whichfork) || + ifp->if_nextents > ip->i_nblocks) || + level == 0 || level > XFS_BM_MAXLEVELS(mp, whichfork)) { + xfs_warn(mp, "corrupt inode %llu (btree).", + (unsigned long long)I_INO(ip)); + xfs_inode_verifier_error(ip, -EFSCORRUPTED, + "xfs_iformat_btree", dfp, size, + __this_address); + xfs_inode_mark_sick(ip, XFS_SICK_INO_CORE); + return -EFSCORRUPTED; + } + + broot = xfs_broot_alloc(ifp, size); + /* + * Copy and convert from the on-disk structure + * to the in-memory structure. + */ + xfs_bmdr_to_bmbt(ip, dfp, XFS_DFORK_SIZE(dip, ip->i_mount, whichfork), + broot, size); + + ifp->if_bytes = 0; + ifp->if_data = NULL; + ifp->if_height = 0; + return 0; +} + +int +xfs_iformat_data_fork( + struct xfs_inode *ip, + struct xfs_dinode *dip) +{ + struct inode *inode = VFS_I(ip); + int error; + + /* + * Initialize the extent count early, as the per-format routines may + * depend on it. Use release semantics to set needextents /after/ we + * set the format. This ensures that we can use acquire semantics on + * needextents in xfs_need_iread_extents() and be guaranteed to see a + * valid format value after that load. + */ + ip->i_df.if_format = dip->di_format; + ip->i_df.if_nextents = xfs_dfork_data_extents(dip); + smp_store_release(&ip->i_df.if_needextents, + ip->i_df.if_format == XFS_DINODE_FMT_BTREE ? 1 : 0); + + switch (inode->i_mode & S_IFMT) { + case S_IFIFO: + case S_IFCHR: + case S_IFBLK: + case S_IFSOCK: + ip->i_disk_size = 0; + inode->i_rdev = xfs_to_linux_dev_t(xfs_dinode_get_rdev(dip)); + return 0; + case S_IFREG: + case S_IFLNK: + case S_IFDIR: + switch (ip->i_df.if_format) { + case XFS_DINODE_FMT_LOCAL: + error = xfs_iformat_local(ip, dip, XFS_DATA_FORK, + be64_to_cpu(dip->di_size)); + if (!error) + error = xfs_ifork_verify_local_data(ip); + return error; + case XFS_DINODE_FMT_EXTENTS: + return xfs_iformat_extents(ip, dip, XFS_DATA_FORK); + case XFS_DINODE_FMT_BTREE: + return xfs_iformat_btree(ip, dip, XFS_DATA_FORK); + case XFS_DINODE_FMT_META_BTREE: + switch (ip->i_metatype) { + case XFS_METAFILE_RTRMAP: + return xfs_iformat_rtrmap(ip, dip); + case XFS_METAFILE_RTREFCOUNT: + return xfs_iformat_rtrefcount(ip, dip); + default: + break; + } + fallthrough; + default: + xfs_inode_verifier_error(ip, -EFSCORRUPTED, __func__, + dip, sizeof(*dip), __this_address); + xfs_inode_mark_sick(ip, XFS_SICK_INO_CORE); + return -EFSCORRUPTED; + } + break; + default: + xfs_inode_verifier_error(ip, -EFSCORRUPTED, __func__, dip, + sizeof(*dip), __this_address); + xfs_inode_mark_sick(ip, XFS_SICK_INO_CORE); + return -EFSCORRUPTED; + } +} + +static uint16_t +xfs_dfork_attr_shortform_size( + struct xfs_dinode *dip) +{ + struct xfs_attr_sf_hdr *sf = XFS_DFORK_APTR(dip); + + return be16_to_cpu(sf->totsize); +} + +void +xfs_ifork_init_attr( + struct xfs_inode *ip, + enum xfs_dinode_fmt format, + xfs_extnum_t nextents) +{ + /* + * Initialize the extent count early, as the per-format routines may + * depend on it. Use release semantics to set needextents /after/ we + * set the format. This ensures that we can use acquire semantics on + * needextents in xfs_need_iread_extents() and be guaranteed to see a + * valid format value after that load. + */ + ip->i_af.if_format = format; + ip->i_af.if_nextents = nextents; + smp_store_release(&ip->i_af.if_needextents, + ip->i_af.if_format == XFS_DINODE_FMT_BTREE ? 1 : 0); +} + +void +xfs_ifork_zap_attr( + struct xfs_inode *ip) +{ + xfs_idestroy_fork(&ip->i_af); + memset(&ip->i_af, 0, sizeof(struct xfs_ifork)); + ip->i_af.if_format = XFS_DINODE_FMT_EXTENTS; +} + +int +xfs_iformat_attr_fork( + struct xfs_inode *ip, + struct xfs_dinode *dip) +{ + xfs_extnum_t naextents = xfs_dfork_attr_extents(dip); + int error = 0; + + /* + * Initialize the extent count early, as the per-format routines may + * depend on it. + */ + xfs_ifork_init_attr(ip, dip->di_aformat, naextents); + + switch (ip->i_af.if_format) { + case XFS_DINODE_FMT_LOCAL: + error = xfs_iformat_local(ip, dip, XFS_ATTR_FORK, + xfs_dfork_attr_shortform_size(dip)); + if (!error) + error = xfs_ifork_verify_local_attr(ip); + break; + case XFS_DINODE_FMT_EXTENTS: + error = xfs_iformat_extents(ip, dip, XFS_ATTR_FORK); + break; + case XFS_DINODE_FMT_BTREE: + error = xfs_iformat_btree(ip, dip, XFS_ATTR_FORK); + break; + default: + xfs_inode_verifier_error(ip, error, __func__, dip, + sizeof(*dip), __this_address); + xfs_inode_mark_sick(ip, XFS_SICK_INO_CORE); + error = -EFSCORRUPTED; + break; + } + + if (error) + xfs_ifork_zap_attr(ip); + return error; +} + +/* + * Allocate the if_broot component of an inode fork so that it is @new_size + * bytes in size, using __GFP_NOLOCKDEP like all the other code that + * initializes a broot during inode load. Returns if_broot. + */ +struct xfs_btree_block * +xfs_broot_alloc( + struct xfs_ifork *ifp, + size_t new_size) +{ + ASSERT(ifp->if_broot == NULL); + + ifp->if_broot = kmalloc(new_size, + GFP_KERNEL | __GFP_NOLOCKDEP | __GFP_NOFAIL); + ifp->if_broot_bytes = new_size; + return ifp->if_broot; +} + +/* + * Reallocate the if_broot component of an inode fork so that it is @new_size + * bytes in size. Returns if_broot. + */ +struct xfs_btree_block * +xfs_broot_realloc( + struct xfs_ifork *ifp, + size_t new_size) +{ + /* No size change? No action needed. */ + if (new_size == ifp->if_broot_bytes) + return ifp->if_broot; + + /* New size is zero, free it. */ + if (new_size == 0) { + ifp->if_broot_bytes = 0; + kfree(ifp->if_broot); + ifp->if_broot = NULL; + return NULL; + } + + /* + * Shrinking the iroot means we allocate a new smaller object and copy + * it. We don't trust krealloc not to nop on realloc-down. + */ + if (ifp->if_broot_bytes > 0 && ifp->if_broot_bytes > new_size) { + struct xfs_btree_block *old_broot = ifp->if_broot; + + ifp->if_broot = kmalloc(new_size, GFP_KERNEL | __GFP_NOFAIL); + ifp->if_broot_bytes = new_size; + memcpy(ifp->if_broot, old_broot, new_size); + kfree(old_broot); + return ifp->if_broot; + } + + /* + * Growing the iroot means we can krealloc. This may get us the same + * object. + */ + ifp->if_broot = krealloc(ifp->if_broot, new_size, + GFP_KERNEL | __GFP_NOFAIL); + ifp->if_broot_bytes = new_size; + return ifp->if_broot; +} + +/* + * This is called when the amount of space needed for if_data + * is increased or decreased. The change in size is indicated by + * the number of bytes that need to be added or deleted in the + * byte_diff parameter. + * + * If the amount of space needed has decreased below the size of the + * inline buffer, then switch to using the inline buffer. Otherwise, + * use krealloc() or kmalloc() to adjust the size of the buffer + * to what is needed. + * + * ip -- the inode whose if_data area is changing + * byte_diff -- the change in the number of bytes, positive or negative, + * requested for the if_data array. + */ +void * +xfs_idata_realloc( + struct xfs_inode *ip, + int64_t byte_diff, + int whichfork) +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + int64_t new_size = ifp->if_bytes + byte_diff; + + ASSERT(new_size >= 0); + ASSERT(new_size <= xfs_inode_fork_size(ip, whichfork)); + + if (byte_diff) { + ifp->if_data = krealloc(ifp->if_data, new_size, + GFP_KERNEL | __GFP_NOFAIL); + if (new_size == 0) + ifp->if_data = NULL; + ifp->if_bytes = new_size; + } + + return ifp->if_data; +} + +/* Free all memory and reset a fork back to its initial state. */ +void +xfs_idestroy_fork( + struct xfs_ifork *ifp) +{ + if (ifp->if_broot != NULL) { + kfree(ifp->if_broot); + ifp->if_broot = NULL; + } + + switch (ifp->if_format) { + case XFS_DINODE_FMT_LOCAL: + kfree(ifp->if_data); + ifp->if_data = NULL; + break; + case XFS_DINODE_FMT_EXTENTS: + case XFS_DINODE_FMT_BTREE: + if (ifp->if_height) + xfs_iext_destroy(ifp); + break; + } +} + +/* + * Convert in-core extents to on-disk form + * + * In the case of the data fork, the in-core and on-disk fork sizes can be + * different due to delayed allocation extents. We only copy on-disk extents + * here, so callers must always use the physical fork size to determine the + * size of the buffer passed to this routine. We will return the size actually + * used. + */ +int +xfs_iextents_copy( + struct xfs_inode *ip, + struct xfs_bmbt_rec *dp, + int whichfork) +{ + int state = xfs_bmap_fork_to_state(whichfork); + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + struct xfs_iext_cursor icur; + struct xfs_bmbt_irec rec; + int64_t copied = 0; + + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL | XFS_ILOCK_SHARED); + ASSERT(ifp->if_bytes > 0); + + for_each_xfs_iext(ifp, &icur, &rec) { + if (isnullstartblock(rec.br_startblock)) + continue; + ASSERT(xfs_bmap_validate_extent(ip, whichfork, &rec) == NULL); + xfs_bmbt_disk_set_all(dp, &rec); + trace_xfs_write_extent(ip, &icur, state, _RET_IP_); + copied += sizeof(struct xfs_bmbt_rec); + dp++; + } + + ASSERT(copied > 0); + ASSERT(copied <= ifp->if_bytes); + return copied; +} + +/* + * Each of the following cases stores data into the same region + * of the on-disk inode, so only one of them can be valid at + * any given time. While it is possible to have conflicting formats + * and log flags, e.g. having XFS_ILOG_?DATA set when the fork is + * in EXTENTS format, this can only happen when the fork has + * changed formats after being modified but before being flushed. + * In these cases, the format always takes precedence, because the + * format indicates the current state of the fork. + */ +void +xfs_iflush_fork( + struct xfs_inode *ip, + struct xfs_dinode *dip, + struct xfs_inode_log_item *iip, + int whichfork) +{ + char *cp; + struct xfs_ifork *ifp; + xfs_mount_t *mp; + static const short brootflag[2] = + { XFS_ILOG_DBROOT, XFS_ILOG_ABROOT }; + static const short dataflag[2] = + { XFS_ILOG_DDATA, XFS_ILOG_ADATA }; + static const short extflag[2] = + { XFS_ILOG_DEXT, XFS_ILOG_AEXT }; + + if (!iip) + return; + ifp = xfs_ifork_ptr(ip, whichfork); + /* + * This can happen if we gave up in iformat in an error path, + * for the attribute fork. + */ + if (!ifp) { + ASSERT(whichfork == XFS_ATTR_FORK); + return; + } + cp = XFS_DFORK_PTR(dip, whichfork); + mp = ip->i_mount; + switch (ifp->if_format) { + case XFS_DINODE_FMT_LOCAL: + if ((iip->ili_fields & dataflag[whichfork]) && + (ifp->if_bytes > 0)) { + ASSERT(ifp->if_data != NULL); + ASSERT(ifp->if_bytes <= xfs_inode_fork_size(ip, whichfork)); + memcpy(cp, ifp->if_data, ifp->if_bytes); + } + break; + + case XFS_DINODE_FMT_EXTENTS: + if ((iip->ili_fields & extflag[whichfork]) && + (ifp->if_bytes > 0)) { + ASSERT(ifp->if_nextents > 0); + (void)xfs_iextents_copy(ip, (xfs_bmbt_rec_t *)cp, + whichfork); + } + break; + + case XFS_DINODE_FMT_BTREE: + if ((iip->ili_fields & brootflag[whichfork]) && + (ifp->if_broot_bytes > 0)) { + ASSERT(ifp->if_broot != NULL); + ASSERT(xfs_bmap_bmdr_space(ifp->if_broot) <= + xfs_inode_fork_size(ip, whichfork)); + xfs_bmbt_to_bmdr(mp, ifp->if_broot, ifp->if_broot_bytes, + (xfs_bmdr_block_t *)cp, + XFS_DFORK_SIZE(dip, mp, whichfork)); + } + break; + + case XFS_DINODE_FMT_DEV: + if (iip->ili_fields & XFS_ILOG_DEV) { + ASSERT(whichfork == XFS_DATA_FORK); + xfs_dinode_put_rdev(dip, + linux_to_xfs_dev_t(VFS_I(ip)->i_rdev)); + } + break; + + case XFS_DINODE_FMT_META_BTREE: + ASSERT(whichfork == XFS_DATA_FORK); + + if (!(iip->ili_fields & brootflag[whichfork])) + break; + + switch (ip->i_metatype) { + case XFS_METAFILE_RTRMAP: + xfs_iflush_rtrmap(ip, dip); + break; + case XFS_METAFILE_RTREFCOUNT: + xfs_iflush_rtrefcount(ip, dip); + break; + default: + ASSERT(0); + break; + } + break; + + default: + ASSERT(0); + break; + } +} + +/* Convert bmap state flags to an inode fork. */ +struct xfs_ifork * +xfs_iext_state_to_fork( + struct xfs_inode *ip, + int state) +{ + if (state & BMAP_COWFORK) + return ip->i_cowfp; + else if (state & BMAP_ATTRFORK) + return &ip->i_af; + return &ip->i_df; +} + +/* + * Initialize an inode's copy-on-write fork. + */ +void +xfs_ifork_init_cow( + struct xfs_inode *ip) +{ + if (ip->i_cowfp) + return; + + ip->i_cowfp = kmem_cache_zalloc(xfs_ifork_cache, + GFP_KERNEL | __GFP_NOLOCKDEP | __GFP_NOFAIL); + ip->i_cowfp->if_format = XFS_DINODE_FMT_EXTENTS; +} + +/* Verify the inline contents of the data fork of an inode. */ +int +xfs_ifork_verify_local_data( + struct xfs_inode *ip) +{ + xfs_failaddr_t fa = NULL; + + switch (VFS_I(ip)->i_mode & S_IFMT) { + case S_IFDIR: { + struct xfs_mount *mp = ip->i_mount; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK); + struct xfs_dir2_sf_hdr *sfp = ifp->if_data; + + fa = xfs_dir2_sf_verify(mp, sfp, ifp->if_bytes); + break; + } + case S_IFLNK: { + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK); + + fa = xfs_symlink_shortform_verify(ifp->if_data, ifp->if_bytes); + break; + } + default: + break; + } + + if (fa) { + xfs_inode_verifier_error(ip, -EFSCORRUPTED, "data fork", + ip->i_df.if_data, ip->i_df.if_bytes, fa); + return -EFSCORRUPTED; + } + + return 0; +} + +/* Verify the inline contents of the attr fork of an inode. */ +int +xfs_ifork_verify_local_attr( + struct xfs_inode *ip) +{ + struct xfs_ifork *ifp = &ip->i_af; + xfs_failaddr_t fa; + + if (!xfs_inode_has_attr_fork(ip)) { + fa = __this_address; + } else { + struct xfs_ifork *ifp = &ip->i_af; + + ASSERT(ifp->if_format == XFS_DINODE_FMT_LOCAL); + fa = xfs_attr_shortform_verify(ifp->if_data, ifp->if_bytes); + } + if (fa) { + xfs_inode_verifier_error(ip, -EFSCORRUPTED, "attr fork", + ifp->if_data, ifp->if_bytes, fa); + return -EFSCORRUPTED; + } + + return 0; +} + +/* + * Check if the inode fork supports adding nr_to_add more extents. + * + * If it doesn't but we can upgrade it to large extent counters, do the upgrade. + * If we can't upgrade or are already using big counters but still can't fit the + * additional extents, return -EFBIG. + */ +int +xfs_iext_count_extend( + struct xfs_trans *tp, + struct xfs_inode *ip, + int whichfork, + uint nr_to_add) +{ + struct xfs_mount *mp = ip->i_mount; + bool has_large = + xfs_inode_has_large_extent_counts(ip); + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + uint64_t nr_exts; + + ASSERT(nr_to_add <= XFS_MAX_EXTCNT_UPGRADE_NR); + + if (whichfork == XFS_COW_FORK) + return 0; + + /* no point in upgrading if if_nextents overflows */ + nr_exts = ifp->if_nextents + nr_to_add; + if (nr_exts < ifp->if_nextents) + return -EFBIG; + + if (XFS_TEST_ERROR(mp, XFS_ERRTAG_REDUCE_MAX_IEXTENTS) && nr_exts > 10) + return -EFBIG; + + if (nr_exts > xfs_iext_max_nextents(has_large, whichfork)) { + if (has_large || !xfs_has_large_extent_counts(mp)) + return -EFBIG; + ip->i_diflags2 |= XFS_DIFLAG2_NREXT64; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + } + return 0; +} + +/* Decide if a file mapping is on the realtime device or not. */ +bool +xfs_ifork_is_realtime( + struct xfs_inode *ip, + int whichfork) +{ + return XFS_IS_REALTIME_INODE(ip) && whichfork != XFS_ATTR_FORK; +} diff --git a/fs/xfs/libxfs/xfs_inode_fork.h b/fs/xfs/libxfs/xfs_inode_fork.h new file mode 100644 index 000000000..69ed0919d --- /dev/null +++ b/fs/xfs/libxfs/xfs_inode_fork.h @@ -0,0 +1,274 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2003,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_INODE_FORK_H__ +#define __XFS_INODE_FORK_H__ + +struct xfs_inode_log_item; +struct xfs_dinode; + +/* + * File incore extent information, present for each of data & attr forks. + */ +struct xfs_ifork { + int64_t if_bytes; /* bytes in if_data */ + struct xfs_btree_block *if_broot; /* file's incore btree root */ + unsigned int if_seq; /* fork mod counter */ + int if_height; /* height of the extent tree */ + void *if_data; /* extent tree root or + inline data */ + xfs_extnum_t if_nextents; /* # of extents in this fork */ + short if_broot_bytes; /* bytes allocated for root */ + int8_t if_format; /* format of this fork */ + uint8_t if_needextents; /* extents have not been read */ +}; + +/* + * Worst-case increase in the fork extent count when we're adding a single + * extent to a fork and there's no possibility of splitting an existing mapping. + */ +#define XFS_IEXT_ADD_NOSPLIT_CNT (1) + +/* + * Punching out an extent from the middle of an existing extent can cause the + * extent count to increase by 1. + * i.e. | Old extent | Hole | Old extent | + */ +#define XFS_IEXT_PUNCH_HOLE_CNT (1) + +/* + * Adding/removing an xattr can cause XFS_DA_NODE_MAXDEPTH extents to + * be added. One extra extent for dabtree in case a local attr is + * large enough to cause a double split. It can also cause extent + * count to increase proportional to the size of a remote xattr's + * value. + */ +#define XFS_IEXT_ATTR_MANIP_CNT(rmt_blks) \ + (XFS_DA_NODE_MAXDEPTH + max(1, rmt_blks)) + +/* + * A write to a sub-interval of an existing unwritten extent causes the original + * extent to be split into 3 extents + * i.e. | Unwritten | Real | Unwritten | + * Hence extent count can increase by 2. + */ +#define XFS_IEXT_WRITE_UNWRITTEN_CNT (2) + + +/* + * Moving an extent to data fork can cause a sub-interval of an existing extent + * to be unmapped. This will increase extent count by 1. Mapping in the new + * extent can increase the extent count by 1 again i.e. + * | Old extent | New extent | Old extent | + * Hence number of extents increases by 2. + */ +#define XFS_IEXT_REFLINK_END_COW_CNT (2) + +/* + * Removing an initial range of source/donor file's extent and adding a new + * extent (from donor/source file) in its place will cause extent count to + * increase by 1. + */ +#define XFS_IEXT_SWAP_RMAP_CNT (1) + +/* + * Fork handling. + */ +#define XFS_IFORK_MAXEXT(ip, w) \ + (xfs_inode_fork_size(ip, w) / sizeof(xfs_bmbt_rec_t)) + +static inline bool xfs_ifork_has_extents(struct xfs_ifork *ifp) +{ + return ifp->if_format == XFS_DINODE_FMT_EXTENTS || + ifp->if_format == XFS_DINODE_FMT_BTREE; +} + +static inline xfs_extnum_t xfs_ifork_nextents(struct xfs_ifork *ifp) +{ + if (!ifp) + return 0; + return ifp->if_nextents; +} + +static inline int8_t xfs_ifork_format(struct xfs_ifork *ifp) +{ + if (!ifp) + return XFS_DINODE_FMT_EXTENTS; + return ifp->if_format; +} + +static inline xfs_extnum_t xfs_iext_max_nextents(bool has_large_extent_counts, + int whichfork) +{ + switch (whichfork) { + case XFS_DATA_FORK: + case XFS_COW_FORK: + if (has_large_extent_counts) + return XFS_MAX_EXTCNT_DATA_FORK_LARGE; + return XFS_MAX_EXTCNT_DATA_FORK_SMALL; + + case XFS_ATTR_FORK: + if (has_large_extent_counts) + return XFS_MAX_EXTCNT_ATTR_FORK_LARGE; + return XFS_MAX_EXTCNT_ATTR_FORK_SMALL; + + default: + ASSERT(0); + return 0; + } +} + +static inline xfs_extnum_t +xfs_dfork_data_extents( + struct xfs_dinode *dip) +{ + if (xfs_dinode_has_large_extent_counts(dip)) + return be64_to_cpu(dip->di_big_nextents); + + return be32_to_cpu(dip->di_nextents); +} + +static inline xfs_extnum_t +xfs_dfork_attr_extents( + struct xfs_dinode *dip) +{ + if (xfs_dinode_has_large_extent_counts(dip)) + return be32_to_cpu(dip->di_big_anextents); + + return be16_to_cpu(dip->di_anextents); +} + +static inline xfs_extnum_t +xfs_dfork_nextents( + struct xfs_dinode *dip, + int whichfork) +{ + switch (whichfork) { + case XFS_DATA_FORK: + return xfs_dfork_data_extents(dip); + case XFS_ATTR_FORK: + return xfs_dfork_attr_extents(dip); + default: + ASSERT(0); + break; + } + + return 0; +} + +void xfs_ifork_zap_attr(struct xfs_inode *ip); +void xfs_ifork_init_attr(struct xfs_inode *ip, enum xfs_dinode_fmt format, + xfs_extnum_t nextents); +struct xfs_ifork *xfs_iext_state_to_fork(struct xfs_inode *ip, int state); + +int xfs_iformat_data_fork(struct xfs_inode *, struct xfs_dinode *); +int xfs_iformat_attr_fork(struct xfs_inode *, struct xfs_dinode *); +void xfs_iflush_fork(struct xfs_inode *, struct xfs_dinode *, + struct xfs_inode_log_item *, int); +void xfs_idestroy_fork(struct xfs_ifork *ifp); +void * xfs_idata_realloc(struct xfs_inode *ip, int64_t byte_diff, + int whichfork); +struct xfs_btree_block *xfs_broot_alloc(struct xfs_ifork *ifp, + size_t new_size); +struct xfs_btree_block *xfs_broot_realloc(struct xfs_ifork *ifp, + size_t new_size); + +int xfs_iread_extents(struct xfs_trans *, struct xfs_inode *, int); +int xfs_iextents_copy(struct xfs_inode *, struct xfs_bmbt_rec *, + int); +void xfs_init_local_fork(struct xfs_inode *ip, int whichfork, + const void *data, int64_t size); + +xfs_extnum_t xfs_iext_count(struct xfs_ifork *ifp); +void xfs_iext_insert_raw(struct xfs_ifork *ifp, + struct xfs_iext_cursor *cur, + struct xfs_bmbt_irec *irec); +void xfs_iext_insert(struct xfs_inode *, struct xfs_iext_cursor *cur, + struct xfs_bmbt_irec *, int); +void xfs_iext_remove(struct xfs_inode *, struct xfs_iext_cursor *, + int); +void xfs_iext_destroy(struct xfs_ifork *); + +bool xfs_iext_lookup_extent(struct xfs_inode *ip, + struct xfs_ifork *ifp, xfs_fileoff_t bno, + struct xfs_iext_cursor *cur, + struct xfs_bmbt_irec *gotp); +bool xfs_iext_lookup_extent_before(struct xfs_inode *ip, + struct xfs_ifork *ifp, xfs_fileoff_t *end, + struct xfs_iext_cursor *cur, + struct xfs_bmbt_irec *gotp); +bool xfs_iext_get_extent(struct xfs_ifork *ifp, + struct xfs_iext_cursor *cur, + struct xfs_bmbt_irec *gotp); +void xfs_iext_update_extent(struct xfs_inode *ip, int state, + struct xfs_iext_cursor *cur, + struct xfs_bmbt_irec *gotp); + +void xfs_iext_first(struct xfs_ifork *, struct xfs_iext_cursor *); +void xfs_iext_last(struct xfs_ifork *, struct xfs_iext_cursor *); +void xfs_iext_next(struct xfs_ifork *, struct xfs_iext_cursor *); +void xfs_iext_prev(struct xfs_ifork *, struct xfs_iext_cursor *); + +static inline bool xfs_iext_next_extent(struct xfs_ifork *ifp, + struct xfs_iext_cursor *cur, struct xfs_bmbt_irec *gotp) +{ + xfs_iext_next(ifp, cur); + return xfs_iext_get_extent(ifp, cur, gotp); +} + +static inline bool xfs_iext_prev_extent(struct xfs_ifork *ifp, + struct xfs_iext_cursor *cur, struct xfs_bmbt_irec *gotp) +{ + xfs_iext_prev(ifp, cur); + return xfs_iext_get_extent(ifp, cur, gotp); +} + +/* + * Return the extent after cur in gotp without updating the cursor. + */ +static inline bool xfs_iext_peek_next_extent(struct xfs_ifork *ifp, + struct xfs_iext_cursor *cur, struct xfs_bmbt_irec *gotp) +{ + struct xfs_iext_cursor ncur = *cur; + + xfs_iext_next(ifp, &ncur); + return xfs_iext_get_extent(ifp, &ncur, gotp); +} + +/* + * Return the extent before cur in gotp without updating the cursor. + */ +static inline bool xfs_iext_peek_prev_extent(struct xfs_ifork *ifp, + struct xfs_iext_cursor *cur, struct xfs_bmbt_irec *gotp) +{ + struct xfs_iext_cursor ncur = *cur; + + xfs_iext_prev(ifp, &ncur); + return xfs_iext_get_extent(ifp, &ncur, gotp); +} + +#define for_each_xfs_iext(ifp, ext, got) \ + for (xfs_iext_first((ifp), (ext)); \ + xfs_iext_get_extent((ifp), (ext), (got)); \ + xfs_iext_next((ifp), (ext))) + +extern struct kmem_cache *xfs_ifork_cache; + +extern void xfs_ifork_init_cow(struct xfs_inode *ip); + +int xfs_ifork_verify_local_data(struct xfs_inode *ip); +int xfs_ifork_verify_local_attr(struct xfs_inode *ip); +int xfs_iext_count_extend(struct xfs_trans *tp, struct xfs_inode *ip, + int whichfork, uint nr_to_add); +bool xfs_ifork_is_realtime(struct xfs_inode *ip, int whichfork); + +/* returns true if the fork has extents but they are not read in yet. */ +static inline bool xfs_need_iread_extents(const struct xfs_ifork *ifp) +{ + /* see xfs_iformat_{data,attr}_fork() for needextents semantics */ + return smp_load_acquire(&ifp->if_needextents) != 0; +} + +#endif /* __XFS_INODE_FORK_H__ */ diff --git a/fs/xfs/libxfs/xfs_inode_util.c b/fs/xfs/libxfs/xfs_inode_util.c new file mode 100644 index 000000000..258ac3d0d --- /dev/null +++ b/fs/xfs/libxfs/xfs_inode_util.c @@ -0,0 +1,742 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2006 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_sb.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_inode_util.h" +#include "xfs_trans.h" +#include "xfs_ialloc.h" +#include "xfs_health.h" +#include "xfs_bmap.h" +#include "xfs_error.h" +#include "xfs_trace.h" +#include "xfs_ag.h" +#include "xfs_iunlink_item.h" +#include "xfs_inode_item.h" + +uint16_t +xfs_flags2diflags( + struct xfs_inode *ip, + unsigned int xflags) +{ + /* can't set PREALLOC this way, just preserve it */ + uint16_t di_flags = + (ip->i_diflags & XFS_DIFLAG_PREALLOC); + + if (xflags & FS_XFLAG_IMMUTABLE) + di_flags |= XFS_DIFLAG_IMMUTABLE; + if (xflags & FS_XFLAG_APPEND) + di_flags |= XFS_DIFLAG_APPEND; + if (xflags & FS_XFLAG_SYNC) + di_flags |= XFS_DIFLAG_SYNC; + if (xflags & FS_XFLAG_NOATIME) + di_flags |= XFS_DIFLAG_NOATIME; + if (xflags & FS_XFLAG_NODUMP) + di_flags |= XFS_DIFLAG_NODUMP; + if (xflags & FS_XFLAG_NODEFRAG) + di_flags |= XFS_DIFLAG_NODEFRAG; + if (xflags & FS_XFLAG_FILESTREAM) + di_flags |= XFS_DIFLAG_FILESTREAM; + if (S_ISDIR(VFS_I(ip)->i_mode)) { + if (xflags & FS_XFLAG_RTINHERIT) + di_flags |= XFS_DIFLAG_RTINHERIT; + if (xflags & FS_XFLAG_NOSYMLINKS) + di_flags |= XFS_DIFLAG_NOSYMLINKS; + if (xflags & FS_XFLAG_EXTSZINHERIT) + di_flags |= XFS_DIFLAG_EXTSZINHERIT; + if (xflags & FS_XFLAG_PROJINHERIT) + di_flags |= XFS_DIFLAG_PROJINHERIT; + } else if (S_ISREG(VFS_I(ip)->i_mode)) { + if (xflags & FS_XFLAG_REALTIME) + di_flags |= XFS_DIFLAG_REALTIME; + if (xflags & FS_XFLAG_EXTSIZE) + di_flags |= XFS_DIFLAG_EXTSIZE; + } + + return di_flags; +} + +uint64_t +xfs_flags2diflags2( + struct xfs_inode *ip, + unsigned int xflags) +{ + uint64_t di_flags2 = + (ip->i_diflags2 & (XFS_DIFLAG2_REFLINK | + XFS_DIFLAG2_BIGTIME | + XFS_DIFLAG2_NREXT64)); + + if (xflags & FS_XFLAG_DAX) + di_flags2 |= XFS_DIFLAG2_DAX; + if (xflags & FS_XFLAG_COWEXTSIZE) + di_flags2 |= XFS_DIFLAG2_COWEXTSIZE; + + return di_flags2; +} + +uint32_t +xfs_ip2xflags( + struct xfs_inode *ip) +{ + uint32_t flags = 0; + + if (ip->i_diflags & XFS_DIFLAG_ANY) { + if (ip->i_diflags & XFS_DIFLAG_REALTIME) + flags |= FS_XFLAG_REALTIME; + if (ip->i_diflags & XFS_DIFLAG_PREALLOC) + flags |= FS_XFLAG_PREALLOC; + if (ip->i_diflags & XFS_DIFLAG_IMMUTABLE) + flags |= FS_XFLAG_IMMUTABLE; + if (ip->i_diflags & XFS_DIFLAG_APPEND) + flags |= FS_XFLAG_APPEND; + if (ip->i_diflags & XFS_DIFLAG_SYNC) + flags |= FS_XFLAG_SYNC; + if (ip->i_diflags & XFS_DIFLAG_NOATIME) + flags |= FS_XFLAG_NOATIME; + if (ip->i_diflags & XFS_DIFLAG_NODUMP) + flags |= FS_XFLAG_NODUMP; + if (ip->i_diflags & XFS_DIFLAG_RTINHERIT) + flags |= FS_XFLAG_RTINHERIT; + if (ip->i_diflags & XFS_DIFLAG_PROJINHERIT) + flags |= FS_XFLAG_PROJINHERIT; + if (ip->i_diflags & XFS_DIFLAG_NOSYMLINKS) + flags |= FS_XFLAG_NOSYMLINKS; + if (ip->i_diflags & XFS_DIFLAG_EXTSIZE) + flags |= FS_XFLAG_EXTSIZE; + if (ip->i_diflags & XFS_DIFLAG_EXTSZINHERIT) + flags |= FS_XFLAG_EXTSZINHERIT; + if (ip->i_diflags & XFS_DIFLAG_NODEFRAG) + flags |= FS_XFLAG_NODEFRAG; + if (ip->i_diflags & XFS_DIFLAG_FILESTREAM) + flags |= FS_XFLAG_FILESTREAM; + } + + if (ip->i_diflags2 & XFS_DIFLAG2_ANY) { + if (ip->i_diflags2 & XFS_DIFLAG2_DAX) + flags |= FS_XFLAG_DAX; + if (ip->i_diflags2 & XFS_DIFLAG2_COWEXTSIZE) + flags |= FS_XFLAG_COWEXTSIZE; + } + + if (xfs_inode_has_attr_fork(ip)) + flags |= FS_XFLAG_HASATTR; + if (xfs_has_asciici(ip->i_mount)) + flags |= FS_XFLAG_CASEFOLD; + return flags; +} + +prid_t +xfs_get_initial_prid(struct xfs_inode *dp) +{ + if (dp->i_diflags & XFS_DIFLAG_PROJINHERIT) + return dp->i_projid; + + /* Assign to the root project by default. */ + return 0; +} + +/* Propagate di_flags from a parent inode to a child inode. */ +static inline void +xfs_inode_inherit_flags( + struct xfs_inode *ip, + const struct xfs_inode *pip) +{ + unsigned int di_flags = 0; + xfs_failaddr_t failaddr; + umode_t mode = VFS_I(ip)->i_mode; + + if (S_ISDIR(mode)) { + if (pip->i_diflags & XFS_DIFLAG_RTINHERIT) + di_flags |= XFS_DIFLAG_RTINHERIT; + if (pip->i_diflags & XFS_DIFLAG_EXTSZINHERIT) { + di_flags |= XFS_DIFLAG_EXTSZINHERIT; + ip->i_extsize = pip->i_extsize; + } + if (pip->i_diflags & XFS_DIFLAG_PROJINHERIT) + di_flags |= XFS_DIFLAG_PROJINHERIT; + } else if (S_ISREG(mode)) { + if ((pip->i_diflags & XFS_DIFLAG_RTINHERIT) && + xfs_has_realtime(ip->i_mount)) + di_flags |= XFS_DIFLAG_REALTIME; + if (pip->i_diflags & XFS_DIFLAG_EXTSZINHERIT) { + di_flags |= XFS_DIFLAG_EXTSIZE; + ip->i_extsize = pip->i_extsize; + } + } + if ((pip->i_diflags & XFS_DIFLAG_NOATIME) && + xfs_inherit_noatime) + di_flags |= XFS_DIFLAG_NOATIME; + if ((pip->i_diflags & XFS_DIFLAG_NODUMP) && + xfs_inherit_nodump) + di_flags |= XFS_DIFLAG_NODUMP; + if ((pip->i_diflags & XFS_DIFLAG_SYNC) && + xfs_inherit_sync) + di_flags |= XFS_DIFLAG_SYNC; + if ((pip->i_diflags & XFS_DIFLAG_NOSYMLINKS) && + xfs_inherit_nosymlinks) + di_flags |= XFS_DIFLAG_NOSYMLINKS; + if ((pip->i_diflags & XFS_DIFLAG_NODEFRAG) && + xfs_inherit_nodefrag) + di_flags |= XFS_DIFLAG_NODEFRAG; + if (pip->i_diflags & XFS_DIFLAG_FILESTREAM) + di_flags |= XFS_DIFLAG_FILESTREAM; + + ip->i_diflags |= di_flags; + + /* + * Inode verifiers on older kernels only check that the extent size + * hint is an integer multiple of the rt extent size on realtime files. + * They did not check the hint alignment on a directory with both + * rtinherit and extszinherit flags set. If the misaligned hint is + * propagated from a directory into a new realtime file, new file + * allocations will fail due to math errors in the rt allocator and/or + * trip the verifiers. Validate the hint settings in the new file so + * that we don't let broken hints propagate. + */ + failaddr = xfs_inode_validate_extsize(ip->i_mount, ip->i_extsize, + VFS_I(ip)->i_mode, ip->i_diflags); + if (failaddr) { + ip->i_diflags &= ~(XFS_DIFLAG_EXTSIZE | + XFS_DIFLAG_EXTSZINHERIT); + ip->i_extsize = 0; + } +} + +/* Propagate di_flags2 from a parent inode to a child inode. */ +static inline void +xfs_inode_inherit_flags2( + struct xfs_inode *ip, + const struct xfs_inode *pip) +{ + xfs_failaddr_t failaddr; + + if (pip->i_diflags2 & XFS_DIFLAG2_COWEXTSIZE) { + ip->i_diflags2 |= XFS_DIFLAG2_COWEXTSIZE; + ip->i_cowextsize = pip->i_cowextsize; + } + if (pip->i_diflags2 & XFS_DIFLAG2_DAX) + ip->i_diflags2 |= XFS_DIFLAG2_DAX; + if (xfs_is_metadir_inode(pip)) + ip->i_diflags2 |= XFS_DIFLAG2_METADATA; + + /* Don't let invalid cowextsize hints propagate. */ + failaddr = xfs_inode_validate_cowextsize(ip->i_mount, ip->i_cowextsize, + VFS_I(ip)->i_mode, ip->i_diflags, ip->i_diflags2); + if (failaddr) { + ip->i_diflags2 &= ~XFS_DIFLAG2_COWEXTSIZE; + ip->i_cowextsize = 0; + } +} + +/* + * If we need to create attributes immediately after allocating the inode, + * initialise an empty attribute fork right now. We use the default fork offset + * for attributes here as we don't know exactly what size or how many + * attributes we might be adding. We can do this safely here because we know + * the data fork is completely empty and this saves us from needing to run a + * separate transaction to set the fork offset in the immediate future. + * + * If we have parent pointers and the caller hasn't told us that the file will + * never be linked into a directory tree, we /must/ create the attr fork. + */ +static inline bool +xfs_icreate_want_attrfork( + struct xfs_mount *mp, + const struct xfs_icreate_args *args) +{ + if (args->flags & XFS_ICREATE_INIT_XATTRS) + return true; + + if (!(args->flags & XFS_ICREATE_UNLINKABLE) && xfs_has_parent(mp)) + return true; + + return false; +} + +/* Initialise an inode's attributes. */ +void +xfs_inode_init( + struct xfs_trans *tp, + const struct xfs_icreate_args *args, + struct xfs_inode *ip) +{ + struct xfs_inode *pip = args->pip; + struct inode *dir = pip ? VFS_I(pip) : NULL; + struct xfs_mount *mp = tp->t_mountp; + struct inode *inode = VFS_I(ip); + unsigned int flags; + int times = XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG | + XFS_ICHGTIME_ACCESS; + + if (args->flags & XFS_ICREATE_TMPFILE) + set_nlink(inode, 0); + else if (S_ISDIR(args->mode)) + set_nlink(inode, 2); + else + set_nlink(inode, 1); + inode->i_rdev = args->rdev; + + if (!args->idmap || pip == NULL) { + /* creating a tree root, sb rooted, or detached file */ + inode->i_uid = GLOBAL_ROOT_UID; + inode->i_gid = GLOBAL_ROOT_GID; + ip->i_projid = 0; + inode->i_mode = args->mode; + } else { + /* creating a child in the directory tree */ + if (dir && !(dir->i_mode & S_ISGID) && xfs_has_grpid(mp)) { + inode_fsuid_set(inode, args->idmap); + inode->i_gid = dir->i_gid; + inode->i_mode = args->mode; + } else { + inode_init_owner(args->idmap, inode, dir, args->mode); + } + ip->i_projid = xfs_get_initial_prid(pip); + } + + ip->i_disk_size = 0; + ip->i_df.if_nextents = 0; + ASSERT(ip->i_nblocks == 0); + + ip->i_extsize = 0; + ip->i_diflags = 0; + + if (xfs_has_v3inodes(mp)) { + inode_set_iversion(inode, 1); + /* also covers the di_used_blocks union arm: */ + ip->i_cowextsize = 0; + times |= XFS_ICHGTIME_CREATE; + } + + xfs_trans_ichgtime(tp, ip, times); + + flags = XFS_ILOG_CORE; + switch (args->mode & S_IFMT) { + case S_IFIFO: + case S_IFCHR: + case S_IFBLK: + case S_IFSOCK: + ip->i_df.if_format = XFS_DINODE_FMT_DEV; + flags |= XFS_ILOG_DEV; + break; + case S_IFREG: + case S_IFDIR: + if (pip && (pip->i_diflags & XFS_DIFLAG_ANY)) + xfs_inode_inherit_flags(ip, pip); + if (pip && (pip->i_diflags2 & XFS_DIFLAG2_ANY)) + xfs_inode_inherit_flags2(ip, pip); + fallthrough; + case S_IFLNK: + ip->i_df.if_format = XFS_DINODE_FMT_EXTENTS; + ip->i_df.if_bytes = 0; + ip->i_df.if_data = NULL; + break; + default: + ASSERT(0); + } + + if (xfs_icreate_want_attrfork(mp, args)) { + ip->i_forkoff = xfs_default_attroffset(ip) >> 3; + xfs_ifork_init_attr(ip, XFS_DINODE_FMT_EXTENTS, 0); + + if (!xfs_has_attr(mp)) { + spin_lock(&mp->m_sb_lock); + xfs_add_attr(mp); + spin_unlock(&mp->m_sb_lock); + xfs_log_sb(tp); + } + } + + xfs_trans_log_inode(tp, ip, flags); +} + +/* + * In-Core Unlinked List Lookups + * ============================= + * + * Every inode is supposed to be reachable from some other piece of metadata + * with the exception of the root directory. Inodes with a connection to a + * file descriptor but not linked from anywhere in the on-disk directory tree + * are collectively known as unlinked inodes, though the filesystem itself + * maintains links to these inodes so that on-disk metadata are consistent. + * + * XFS implements a per-AG on-disk hash table of unlinked inodes. The AGI + * header contains a number of buckets that point to an inode, and each inode + * record has a pointer to the next inode in the hash chain. This + * singly-linked list causes scaling problems in the iunlink remove function + * because we must walk that list to find the inode that points to the inode + * being removed from the unlinked hash bucket list. + * + * Hence we keep an in-memory double linked list to link each inode on an + * unlinked list. Because there are 64 unlinked lists per AGI, keeping pointer + * based lists would require having 64 list heads in the perag, one for each + * list. This is expensive in terms of memory (think millions of AGs) and cache + * misses on lookups. Instead, use the fact that inodes on the unlinked list + * must be referenced at the VFS level to keep them on the list and hence we + * have an existence guarantee for inodes on the unlinked list. + * + * Given we have an existence guarantee, we can use lockless inode cache lookups + * to resolve aginos to xfs inodes. This means we only need 8 bytes per inode + * for the double linked unlinked list, and we don't need any extra locking to + * keep the list safe as all manipulations are done under the AGI buffer lock. + * Keeping the list up to date does not require memory allocation, just finding + * the XFS inode and updating the next/prev unlinked list aginos. + */ + +/* + * Update the prev pointer of the next agino. Returns -ENOLINK if the inode + * is not in cache. + */ +static int +xfs_iunlink_update_backref( + struct xfs_perag *pag, + xfs_agino_t prev_agino, + xfs_agino_t next_agino) +{ + struct xfs_inode *ip; + + /* No update necessary if we are at the end of the list. */ + if (next_agino == NULLAGINO) + return 0; + + ip = xfs_iunlink_lookup(pag, next_agino); + if (!ip) + return -ENOLINK; + + ip->i_prev_unlinked = prev_agino; + return 0; +} + +/* + * Point the AGI unlinked bucket at an inode and log the results. The caller + * is responsible for validating the old value. + */ +STATIC int +xfs_iunlink_update_bucket( + struct xfs_trans *tp, + struct xfs_perag *pag, + struct xfs_buf *agibp, + unsigned int bucket_index, + xfs_agino_t new_agino) +{ + struct xfs_agi *agi = agibp->b_addr; + xfs_agino_t old_value; + int offset; + + ASSERT(xfs_verify_agino_or_null(pag, new_agino)); + + old_value = be32_to_cpu(agi->agi_unlinked[bucket_index]); + trace_xfs_iunlink_update_bucket(pag, bucket_index, old_value, + new_agino); + + /* + * We should never find the head of the list already set to the value + * passed in because either we're adding or removing ourselves from the + * head of the list. + */ + if (old_value == new_agino) { + xfs_buf_mark_corrupt(agibp); + xfs_ag_mark_sick(pag, XFS_SICK_AG_AGI); + return -EFSCORRUPTED; + } + + agi->agi_unlinked[bucket_index] = cpu_to_be32(new_agino); + offset = offsetof(struct xfs_agi, agi_unlinked) + + (sizeof(xfs_agino_t) * bucket_index); + xfs_trans_log_buf(tp, agibp, offset, offset + sizeof(xfs_agino_t) - 1); + return 0; +} + +static int +xfs_iunlink_insert_inode( + struct xfs_trans *tp, + struct xfs_perag *pag, + struct xfs_buf *agibp, + struct xfs_inode *ip) +{ + struct xfs_agi *agi = agibp->b_addr; + xfs_agino_t next_agino; + xfs_agino_t agino = XFS_INODE_TO_AGINO(ip); + short bucket_index = agino % XFS_AGI_UNLINKED_BUCKETS; + int error; + + /* + * Get the index into the agi hash table for the list this inode will + * go on. Make sure the pointer isn't garbage and that this inode + * isn't already on the list. + */ + next_agino = be32_to_cpu(agi->agi_unlinked[bucket_index]); + if (next_agino == agino || + !xfs_verify_agino_or_null(pag, next_agino)) { + xfs_buf_mark_corrupt(agibp); + xfs_ag_mark_sick(pag, XFS_SICK_AG_AGI); + return -EFSCORRUPTED; + } + + /* + * Update the prev pointer in the next inode to point back to this + * inode. + */ + error = xfs_iunlink_update_backref(pag, agino, next_agino); + if (error == -ENOLINK) + error = xfs_iunlink_reload_next(tp, agibp, agino, next_agino); + if (error) + return error; + + if (next_agino != NULLAGINO) { + /* + * There is already another inode in the bucket, so point this + * inode to the current head of the list. + */ + error = xfs_iunlink_log_inode(tp, ip, pag, next_agino); + if (error) + return error; + ip->i_next_unlinked = next_agino; + } + + /* Point the head of the list to point to this inode. */ + ip->i_prev_unlinked = NULLAGINO; + return xfs_iunlink_update_bucket(tp, pag, agibp, bucket_index, agino); +} + +/* + * This is called when the inode's link count has gone to 0 or we are creating + * a tmpfile via O_TMPFILE. The inode @ip must have nlink == 0. + * + * We place the on-disk inode on a list in the AGI. It will be pulled from this + * list when the inode is freed. + */ +int +xfs_iunlink( + struct xfs_trans *tp, + struct xfs_inode *ip) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_perag *pag; + struct xfs_buf *agibp; + int error; + + ASSERT(VFS_I(ip)->i_nlink == 0); + ASSERT(VFS_I(ip)->i_mode != 0); + trace_xfs_iunlink(ip); + + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(ip)); + + /* Get the agi buffer first. It ensures lock ordering on the list. */ + error = xfs_read_agi(pag, tp, 0, &agibp); + if (error) + goto out; + + error = xfs_iunlink_insert_inode(tp, pag, agibp, ip); +out: + xfs_perag_put(pag); + return error; +} + +static int +xfs_iunlink_remove_inode( + struct xfs_trans *tp, + struct xfs_perag *pag, + struct xfs_buf *agibp, + struct xfs_inode *ip) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_agi *agi = agibp->b_addr; + xfs_agino_t agino = XFS_INODE_TO_AGINO(ip); + xfs_agino_t head_agino; + short bucket_index = agino % XFS_AGI_UNLINKED_BUCKETS; + int error; + + trace_xfs_iunlink_remove(ip); + + /* + * Get the index into the agi hash table for the list this inode will + * go on. Make sure the head pointer isn't garbage. + */ + head_agino = be32_to_cpu(agi->agi_unlinked[bucket_index]); + if (!xfs_verify_agino(pag, head_agino)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + agi, sizeof(*agi)); + xfs_ag_mark_sick(pag, XFS_SICK_AG_AGI); + return -EFSCORRUPTED; + } + + /* + * Set our inode's next_unlinked pointer to NULL and then return + * the old pointer value so that we can update whatever was previous + * to us in the list to point to whatever was next in the list. + */ + error = xfs_iunlink_log_inode(tp, ip, pag, NULLAGINO); + if (error) + return error; + + /* + * Update the prev pointer in the next inode to point back to previous + * inode in the chain. + */ + error = xfs_iunlink_update_backref(pag, ip->i_prev_unlinked, + ip->i_next_unlinked); + if (error == -ENOLINK) + error = xfs_iunlink_reload_next(tp, agibp, ip->i_prev_unlinked, + ip->i_next_unlinked); + if (error) + return error; + + if (head_agino != agino) { + struct xfs_inode *prev_ip; + + prev_ip = xfs_iunlink_lookup(pag, ip->i_prev_unlinked); + if (!prev_ip) { + xfs_inode_mark_sick(ip, XFS_SICK_INO_CORE); + return -EFSCORRUPTED; + } + + error = xfs_iunlink_log_inode(tp, prev_ip, pag, + ip->i_next_unlinked); + prev_ip->i_next_unlinked = ip->i_next_unlinked; + } else { + /* Point the head of the list to the next unlinked inode. */ + error = xfs_iunlink_update_bucket(tp, pag, agibp, bucket_index, + ip->i_next_unlinked); + } + + ip->i_next_unlinked = NULLAGINO; + ip->i_prev_unlinked = 0; + return error; +} + +/* + * Pull the on-disk inode from the AGI unlinked list. + */ +int +xfs_iunlink_remove( + struct xfs_trans *tp, + struct xfs_perag *pag, + struct xfs_inode *ip) +{ + struct xfs_buf *agibp; + int error; + + trace_xfs_iunlink_remove(ip); + + /* Get the agi buffer first. It ensures lock ordering on the list. */ + error = xfs_read_agi(pag, tp, 0, &agibp); + if (error) + return error; + + return xfs_iunlink_remove_inode(tp, pag, agibp, ip); +} + +/* + * Decrement the link count on an inode & log the change. If this causes the + * link count to go to zero, move the inode to AGI unlinked list so that it can + * be freed when the last active reference goes away via xfs_inactive(). + */ +int +xfs_droplink( + struct xfs_trans *tp, + struct xfs_inode *ip) +{ + struct inode *inode = VFS_I(ip); + + xfs_trans_ichgtime(tp, ip, XFS_ICHGTIME_CHG); + + if (inode->i_nlink == 0) { + xfs_info_ratelimited(tp->t_mountp, + "Inode 0x%llx link count dropped below zero. Pinning link count.", + I_INO(ip)); + set_nlink(inode, XFS_NLINK_PINNED); + } + if (inode->i_nlink != XFS_NLINK_PINNED) + drop_nlink(inode); + + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + + if (inode->i_nlink) + return 0; + + return xfs_iunlink(tp, ip); +} + +/* + * Increment the link count on an inode & log the change. + */ +void +xfs_bumplink( + struct xfs_trans *tp, + struct xfs_inode *ip) +{ + struct inode *inode = VFS_I(ip); + + xfs_trans_ichgtime(tp, ip, XFS_ICHGTIME_CHG); + + if (inode->i_nlink == XFS_NLINK_PINNED - 1) + xfs_info_ratelimited(tp->t_mountp, + "Inode 0x%llx link count exceeded maximum. Pinning link count.", + I_INO(ip)); + if (inode->i_nlink != XFS_NLINK_PINNED) + inc_nlink(inode); + + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); +} + +/* Free an inode in the ondisk index and zero it out. */ +int +xfs_inode_uninit( + struct xfs_trans *tp, + struct xfs_perag *pag, + struct xfs_inode *ip, + struct xfs_icluster *xic) +{ + struct xfs_mount *mp = ip->i_mount; + int error; + + /* + * Free the inode first so that we guarantee that the AGI lock is going + * to be taken before we remove the inode from the unlinked list. This + * makes the AGI lock -> unlinked list modification order the same as + * used in O_TMPFILE creation. + */ + error = xfs_difree(tp, pag, I_INO(ip), xic); + if (error) + return error; + + error = xfs_iunlink_remove(tp, pag, ip); + if (error) + return error; + + /* + * Free any local-format data sitting around before we reset the + * data fork to extents format. Note that the attr fork data has + * already been freed by xfs_attr_inactive. + */ + if (ip->i_df.if_format == XFS_DINODE_FMT_LOCAL) { + kfree(ip->i_df.if_data); + ip->i_df.if_data = NULL; + ip->i_df.if_bytes = 0; + } + + VFS_I(ip)->i_mode = 0; /* mark incore inode as free */ + ip->i_diflags = 0; + ip->i_diflags2 = mp->m_ino_geo.new_diflags2; + ip->i_forkoff = 0; /* mark the attr fork not in use */ + ip->i_df.if_format = XFS_DINODE_FMT_EXTENTS; + + /* + * Bump the generation count so no one will be confused + * by reincarnations of this inode. + */ + VFS_I(ip)->i_generation++; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + return 0; +} diff --git a/fs/xfs/libxfs/xfs_inode_util.h b/fs/xfs/libxfs/xfs_inode_util.h new file mode 100644 index 000000000..060242998 --- /dev/null +++ b/fs/xfs/libxfs/xfs_inode_util.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2000-2003,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_INODE_UTIL_H__ +#define __XFS_INODE_UTIL_H__ + +struct xfs_icluster; + +uint16_t xfs_flags2diflags(struct xfs_inode *ip, unsigned int xflags); +uint64_t xfs_flags2diflags2(struct xfs_inode *ip, unsigned int xflags); +uint32_t xfs_dic2xflags(struct xfs_inode *ip); +uint32_t xfs_ip2xflags(struct xfs_inode *ip); + +prid_t xfs_get_initial_prid(struct xfs_inode *dp); + +/* + * File creation context. + * + * Due to our only partial reliance on the VFS to propagate uid and gid values + * according to accepted Unix behaviors, callers must initialize idmap to the + * correct idmapping structure to get the correct inheritance behaviors when + * XFS_MOUNT_GRPID is set. + * + * To create files detached from the directory tree (e.g. quota inodes), set + * idmap to NULL. To create a tree root, set pip to NULL. + */ +struct xfs_icreate_args { + struct mnt_idmap *idmap; + struct xfs_inode *pip; /* parent inode or null */ + dev_t rdev; + umode_t mode; + +#define XFS_ICREATE_TMPFILE (1U << 0) /* create an unlinked file */ +#define XFS_ICREATE_INIT_XATTRS (1U << 1) /* will set xattrs immediately */ +#define XFS_ICREATE_UNLINKABLE (1U << 2) /* cannot link into dir tree */ + uint16_t flags; +}; + +/* + * Flags for xfs_trans_ichgtime(). + */ +#define XFS_ICHGTIME_MOD 0x1 /* data fork modification timestamp */ +#define XFS_ICHGTIME_CHG 0x2 /* inode field change timestamp */ +#define XFS_ICHGTIME_CREATE 0x4 /* inode create timestamp */ +#define XFS_ICHGTIME_ACCESS 0x8 /* last access timestamp */ +void xfs_trans_ichgtime(struct xfs_trans *tp, struct xfs_inode *ip, int flags); + +void xfs_inode_init(struct xfs_trans *tp, const struct xfs_icreate_args *args, + struct xfs_inode *ip); + +int xfs_inode_uninit(struct xfs_trans *tp, struct xfs_perag *pag, + struct xfs_inode *ip, struct xfs_icluster *xic); + +int xfs_iunlink(struct xfs_trans *tp, struct xfs_inode *ip); +int xfs_iunlink_remove(struct xfs_trans *tp, struct xfs_perag *pag, + struct xfs_inode *ip); +int xfs_droplink(struct xfs_trans *tp, struct xfs_inode *ip); +void xfs_bumplink(struct xfs_trans *tp, struct xfs_inode *ip); + +#endif /* __XFS_INODE_UTIL_H__ */ diff --git a/fs/xfs/libxfs/xfs_log_format.h b/fs/xfs/libxfs/xfs_log_format.h new file mode 100644 index 000000000..a4e1b3eb4 --- /dev/null +++ b/fs/xfs/libxfs/xfs_log_format.h @@ -0,0 +1,1086 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2003,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_LOG_FORMAT_H__ +#define __XFS_LOG_FORMAT_H__ + +struct xfs_mount; +struct xfs_trans_res; + +/* + * On-disk Log Format definitions. + * + * This file contains all the on-disk format definitions used within the log. It + * includes the physical log structure itself, as well as all the log item + * format structures that are written into the log and intepreted by log + * recovery. We start with the physical log format definitions, and then work + * through all the log items definitions and everything they encode into the + * log. + */ +typedef uint32_t xlog_tid_t; + +#define XLOG_MIN_ICLOGS 2 +#define XLOG_MAX_ICLOGS 8 +#define XLOG_HEADER_MAGIC_NUM 0xFEEDbabe /* Invalid cycle number */ +#define XLOG_VERSION_1 1 +#define XLOG_VERSION_2 2 /* Large IClogs, Log sunit */ +#define XLOG_VERSION_OKBITS (XLOG_VERSION_1 | XLOG_VERSION_2) +#define XLOG_MIN_RECORD_BSIZE (16*1024) /* eventually 32k */ +#define XLOG_BIG_RECORD_BSIZE (32*1024) /* 32k buffers */ +#define XLOG_MAX_RECORD_BSIZE (256*1024) +#define XLOG_HEADER_CYCLE_SIZE (32*1024) /* cycle data in header */ +#define XLOG_CYCLE_DATA_SIZE (XLOG_HEADER_CYCLE_SIZE / BBSIZE) +#define XLOG_MIN_RECORD_BSHIFT 14 /* 16384 == 1 << 14 */ +#define XLOG_BIG_RECORD_BSHIFT 15 /* 32k == 1 << 15 */ +#define XLOG_MAX_RECORD_BSHIFT 18 /* 256k == 1 << 18 */ + +#define XLOG_HEADER_SIZE 512 + +/* Minimum number of transactions that must fit in the log (defined by mkfs) */ +#define XFS_MIN_LOG_FACTOR 3 + +#define XLOG_REC_SHIFT(log) \ + BTOBB(1 << (xfs_has_logv2(log->l_mp) ? \ + XLOG_MAX_RECORD_BSHIFT : XLOG_BIG_RECORD_BSHIFT)) +#define XLOG_TOTAL_REC_SHIFT(log) \ + BTOBB(XLOG_MAX_ICLOGS << (xfs_has_logv2(log->l_mp) ? \ + XLOG_MAX_RECORD_BSHIFT : XLOG_BIG_RECORD_BSHIFT)) + +/* get lsn fields */ +#define CYCLE_LSN(lsn) ((uint)((lsn)>>32)) +#define BLOCK_LSN(lsn) ((uint)(lsn)) + +/* + * By comparing each component, we don't have to worry about extra endian issues + * in treating two 32 bit numbers as one 64 bit number + */ +static inline xfs_lsn_t XFS_LSN_CMP(xfs_lsn_t lsn1, xfs_lsn_t lsn2) +{ + if (CYCLE_LSN(lsn1) != CYCLE_LSN(lsn2)) + return CYCLE_LSN(lsn1) < CYCLE_LSN(lsn2) ? -999 : 999; + if (BLOCK_LSN(lsn1) != BLOCK_LSN(lsn2)) + return BLOCK_LSN(lsn1) < BLOCK_LSN(lsn2) ? -999 : 999; + return 0; +} + +/* this is used in a spot where we might otherwise double-endian-flip */ +#define CYCLE_LSN_DISK(lsn) (((__be32 *)&(lsn))[0]) + +static inline xfs_lsn_t xlog_assign_lsn(uint cycle, uint block) +{ + return ((xfs_lsn_t)cycle << 32) | block; +} + +static inline uint xlog_get_cycle(char *ptr) +{ + if (be32_to_cpu(*(__be32 *)ptr) == XLOG_HEADER_MAGIC_NUM) + return be32_to_cpu(*((__be32 *)ptr + 1)); + else + return be32_to_cpu(*(__be32 *)ptr); +} + +/* Log Clients */ +#define XFS_TRANSACTION 0x69 +#define XFS_LOG 0xaa + +#define XLOG_UNMOUNT_TYPE 0x556e /* Un for Unmount */ + +/* + * Log item for unmount records. + * + * The unmount record used to have a string "Unmount filesystem--" in the + * data section where the "Un" was really a magic number (XLOG_UNMOUNT_TYPE). + * We just write the magic number now; see xfs_log_unmount_write. + */ +struct xfs_unmount_log_format { + uint16_t magic; /* XLOG_UNMOUNT_TYPE */ + uint16_t pad1; + uint32_t pad2; /* may as well make it 64 bits */ +}; + +/* + * Flags to log operation header + * + * The first write of a new transaction will be preceded with a start + * record, XLOG_START_TRANS. Once a transaction is committed, a commit + * record is written, XLOG_COMMIT_TRANS. If a single region can not fit into + * the remainder of the current active in-core log, it is split up into + * multiple regions. Each partial region will be marked with a + * XLOG_CONTINUE_TRANS until the last one, which gets marked with XLOG_END_TRANS. + * + */ +#define XLOG_START_TRANS 0x01 /* Start a new transaction */ +#define XLOG_COMMIT_TRANS 0x02 /* Commit this transaction */ +#define XLOG_CONTINUE_TRANS 0x04 /* Cont this trans into new region */ +#define XLOG_WAS_CONT_TRANS 0x08 /* Cont this trans into new region */ +#define XLOG_END_TRANS 0x10 /* End a continued transaction */ +#define XLOG_UNMOUNT_TRANS 0x20 /* Unmount a filesystem transaction */ + +struct xlog_op_header { + __be32 oh_tid; /* transaction id of operation : 4 b */ + __be32 oh_len; /* bytes in data region : 4 b */ + __u8 oh_clientid; /* who sent me this : 1 b */ + __u8 oh_flags; /* : 1 b */ + __u16 oh_res2; /* 32 bit align : 2 b */ +}; + +/* valid values for h_fmt */ +#define XLOG_FMT_UNKNOWN 0 +#define XLOG_FMT_LINUX_LE 1 +#define XLOG_FMT_LINUX_BE 2 +#define XLOG_FMT_IRIX_BE 3 + +/* our fmt */ +#ifdef XFS_NATIVE_HOST +#define XLOG_FMT XLOG_FMT_LINUX_BE +#else +#define XLOG_FMT XLOG_FMT_LINUX_LE +#endif + +struct xlog_rec_ext_header { + __be32 xh_cycle; /* write cycle of log */ + __be32 xh_cycle_data[XLOG_CYCLE_DATA_SIZE]; + __u8 xh_reserved[252]; +}; + +/* actual ext header payload size for checksumming */ +#define XLOG_REC_EXT_SIZE \ + offsetofend(struct xlog_rec_ext_header, xh_cycle_data) + +struct xlog_rec_header { + __be32 h_magicno; /* log record (LR) identifier : 4 */ + __be32 h_cycle; /* write cycle of log : 4 */ + __be32 h_version; /* LR version : 4 */ + __be32 h_len; /* len in bytes; should be 64-bit aligned: 4 */ + __be64 h_lsn; /* lsn of this LR : 8 */ + __be64 h_tail_lsn; /* lsn of 1st LR w/ buffers not committed: 8 */ + __le32 h_crc; /* crc of log record : 4 */ + __be32 h_prev_block; /* block number to previous LR : 4 */ + __be32 h_num_logops; /* number of log operations in this LR : 4 */ + __be32 h_cycle_data[XLOG_CYCLE_DATA_SIZE]; + + /* fields added by the Linux port: */ + __be32 h_fmt; /* format of log record : 4 */ + uuid_t h_fs_uuid; /* uuid of FS : 16 */ + + /* fields added for log v2: */ + __be32 h_size; /* iclog size : 4 */ + + /* + * When h_size added for log v2 support, it caused structure to have + * a different size on i386 vs all other architectures because the + * sum of the size ofthe member is not aligned by that of the largest + * __be64-sized member, and i386 has really odd struct alignment rules. + * + * Due to the way the log headers are placed out on-disk that alone is + * not a problem becaue the xlog_rec_header always sits alone in a + * BBSIZEs area, and the rest of that area is padded with zeroes. + * But xlog_cksum used to calculate the checksum based on the structure + * size, and thus gives different checksums for i386 vs the rest. + * We now do two checksum validation passes for both sizes to allow + * moving v5 file systems with unclean logs between i386 and other + * (little-endian) architectures. + */ + __u32 h_pad0; + + __u8 h_reserved[184]; + struct xlog_rec_ext_header h_ext[]; +}; + +#ifdef __i386__ +#define XLOG_REC_SIZE offsetofend(struct xlog_rec_header, h_size) +#define XLOG_REC_SIZE_OTHER offsetofend(struct xlog_rec_header, h_pad0) +#else +#define XLOG_REC_SIZE offsetofend(struct xlog_rec_header, h_pad0) +#define XLOG_REC_SIZE_OTHER offsetofend(struct xlog_rec_header, h_size) +#endif /* __i386__ */ + +/* + * Transaction Header definitions. + * + * This is the structure written in the log at the head of every transaction. It + * identifies the type and id of the transaction, and contains the number of + * items logged by the transaction so we know how many to expect during + * recovery. + * + * Do not change the below structure without redoing the code in + * xlog_recover_add_to_trans() and xlog_recover_add_to_cont_trans(). + */ +struct xfs_trans_header { + uint th_magic; /* magic number */ + uint th_type; /* transaction type */ + int32_t th_tid; /* transaction id (unused) */ + uint th_num_items; /* num items logged by trans */ +}; + +#define XFS_TRANS_HEADER_MAGIC 0x5452414e /* TRAN */ + +/* + * The only type valid for th_type in CIL-enabled file system logs: + */ +#define XFS_TRANS_CHECKPOINT 40 + +/* + * Log item types. + */ +#define XFS_LI_EFI 0x1236 +#define XFS_LI_EFD 0x1237 +#define XFS_LI_IUNLINK 0x1238 +#define XFS_LI_INODE 0x123b /* aligned ino chunks, var-size ibufs */ +#define XFS_LI_BUF 0x123c /* v2 bufs, variable sized inode bufs */ +#define XFS_LI_DQUOT 0x123d +#define XFS_LI_QUOTAOFF 0x123e +#define XFS_LI_ICREATE 0x123f +#define XFS_LI_RUI 0x1240 /* rmap update intent */ +#define XFS_LI_RUD 0x1241 +#define XFS_LI_CUI 0x1242 /* refcount update intent */ +#define XFS_LI_CUD 0x1243 +#define XFS_LI_BUI 0x1244 /* bmbt update intent */ +#define XFS_LI_BUD 0x1245 +#define XFS_LI_ATTRI 0x1246 /* attr set/remove intent*/ +#define XFS_LI_ATTRD 0x1247 /* attr set/remove done */ +#define XFS_LI_XMI 0x1248 /* mapping exchange intent */ +#define XFS_LI_XMD 0x1249 /* mapping exchange done */ +#define XFS_LI_EFI_RT 0x124a /* realtime extent free intent */ +#define XFS_LI_EFD_RT 0x124b /* realtime extent free done */ +#define XFS_LI_RUI_RT 0x124c /* realtime rmap update intent */ +#define XFS_LI_RUD_RT 0x124d /* realtime rmap update done */ +#define XFS_LI_CUI_RT 0x124e /* realtime refcount update intent */ +#define XFS_LI_CUD_RT 0x124f /* realtime refcount update done */ + +#define XFS_LI_TYPE_DESC \ + { XFS_LI_EFI, "XFS_LI_EFI" }, \ + { XFS_LI_EFD, "XFS_LI_EFD" }, \ + { XFS_LI_IUNLINK, "XFS_LI_IUNLINK" }, \ + { XFS_LI_INODE, "XFS_LI_INODE" }, \ + { XFS_LI_BUF, "XFS_LI_BUF" }, \ + { XFS_LI_DQUOT, "XFS_LI_DQUOT" }, \ + { XFS_LI_QUOTAOFF, "XFS_LI_QUOTAOFF" }, \ + { XFS_LI_ICREATE, "XFS_LI_ICREATE" }, \ + { XFS_LI_RUI, "XFS_LI_RUI" }, \ + { XFS_LI_RUD, "XFS_LI_RUD" }, \ + { XFS_LI_CUI, "XFS_LI_CUI" }, \ + { XFS_LI_CUD, "XFS_LI_CUD" }, \ + { XFS_LI_BUI, "XFS_LI_BUI" }, \ + { XFS_LI_BUD, "XFS_LI_BUD" }, \ + { XFS_LI_ATTRI, "XFS_LI_ATTRI" }, \ + { XFS_LI_ATTRD, "XFS_LI_ATTRD" }, \ + { XFS_LI_XMI, "XFS_LI_XMI" }, \ + { XFS_LI_XMD, "XFS_LI_XMD" }, \ + { XFS_LI_EFI_RT, "XFS_LI_EFI_RT" }, \ + { XFS_LI_EFD_RT, "XFS_LI_EFD_RT" }, \ + { XFS_LI_RUI_RT, "XFS_LI_RUI_RT" }, \ + { XFS_LI_RUD_RT, "XFS_LI_RUD_RT" }, \ + { XFS_LI_CUI_RT, "XFS_LI_CUI_RT" }, \ + { XFS_LI_CUD_RT, "XFS_LI_CUD_RT" } + +/* + * Inode Log Item Format definitions. + * + * This is the structure used to lay out an inode log item in the + * log. The size of the inline data/extents/b-tree root to be logged + * (if any) is indicated in the ilf_dsize field. Changes to this structure + * must be added on to the end. + */ +struct xfs_inode_log_format { + uint16_t ilf_type; /* inode log item type */ + uint16_t ilf_size; /* size of this item */ + uint32_t ilf_fields; /* flags for fields logged */ + uint16_t ilf_asize; /* size of attr d/ext/root */ + uint16_t ilf_dsize; /* size of data/ext/root */ + uint32_t ilf_pad; /* pad for 64 bit boundary */ + uint64_t ilf_ino; /* inode number */ + union { + uint32_t ilfu_rdev; /* rdev value for dev inode*/ + uint8_t __pad[16]; /* unused */ + } ilf_u; + int64_t ilf_blkno; /* blkno of inode buffer */ + int32_t ilf_len; /* len of inode buffer */ + int32_t ilf_boffset; /* off of inode in buffer */ +}; + +/* + * Old 32 bit systems will log in this format without the 64 bit + * alignment padding. Recovery will detect this and convert it to the + * correct format. + */ +struct xfs_inode_log_format_32 { + uint16_t ilf_type; /* inode log item type */ + uint16_t ilf_size; /* size of this item */ + uint32_t ilf_fields; /* flags for fields logged */ + uint16_t ilf_asize; /* size of attr d/ext/root */ + uint16_t ilf_dsize; /* size of data/ext/root */ + uint64_t ilf_ino; /* inode number */ + union { + uint32_t ilfu_rdev; /* rdev value for dev inode*/ + uint8_t __pad[16]; /* unused */ + } ilf_u; + int64_t ilf_blkno; /* blkno of inode buffer */ + int32_t ilf_len; /* len of inode buffer */ + int32_t ilf_boffset; /* off of inode in buffer */ +} __attribute__((packed)); + + +/* + * Flags for xfs_trans_log_inode flags field. + */ +#define XFS_ILOG_CORE 0x001 /* log standard inode fields */ +#define XFS_ILOG_DDATA 0x002 /* log i_df.if_data */ +#define XFS_ILOG_DEXT 0x004 /* log i_df.if_extents */ +#define XFS_ILOG_DBROOT 0x008 /* log i_df.i_broot */ +#define XFS_ILOG_DEV 0x010 /* log the dev field */ +#define XFS_ILOG_UUID 0x020 /* added long ago, but never used */ +#define XFS_ILOG_ADATA 0x040 /* log i_af.if_data */ +#define XFS_ILOG_AEXT 0x080 /* log i_af.if_extents */ +#define XFS_ILOG_ABROOT 0x100 /* log i_af.i_broot */ +#define XFS_ILOG_DOWNER 0x200 /* change the data fork owner on replay */ +#define XFS_ILOG_AOWNER 0x400 /* change the attr fork owner on replay */ + +/* + * The timestamps are dirty, but not necessarily anything else in the inode + * core. Unlike the other fields above this one must never make it to disk + * in the ilf_fields of the inode_log_format, but is purely store in-memory in + * ili_fields in the inode_log_item. + */ +#define XFS_ILOG_TIMESTAMP 0x4000 + +/* + * The version field has been changed, but not necessarily anything else of + * interest. This must never make it to disk - it is used purely to ensure that + * the inode item ->precommit operation can update the fsync flag triggers + * in the inode item correctly. + */ +#define XFS_ILOG_IVERSION 0x8000 + +#define XFS_ILOG_DFORK (XFS_ILOG_DDATA | XFS_ILOG_DEXT | \ + XFS_ILOG_DBROOT) + +#define XFS_ILOG_AFORK (XFS_ILOG_ADATA | XFS_ILOG_AEXT | \ + XFS_ILOG_ABROOT) + +#define XFS_ILOG_ALL (XFS_ILOG_CORE | XFS_ILOG_DDATA | \ + XFS_ILOG_DEXT | XFS_ILOG_DBROOT | \ + XFS_ILOG_DEV | XFS_ILOG_ADATA | \ + XFS_ILOG_AEXT | XFS_ILOG_ABROOT | \ + XFS_ILOG_TIMESTAMP | XFS_ILOG_DOWNER | \ + XFS_ILOG_AOWNER) + +static inline int xfs_ilog_fbroot(int w) +{ + return (w == XFS_DATA_FORK ? XFS_ILOG_DBROOT : XFS_ILOG_ABROOT); +} + +static inline int xfs_ilog_fext(int w) +{ + return (w == XFS_DATA_FORK ? XFS_ILOG_DEXT : XFS_ILOG_AEXT); +} + +static inline int xfs_ilog_fdata(int w) +{ + return (w == XFS_DATA_FORK ? XFS_ILOG_DDATA : XFS_ILOG_ADATA); +} + +/* + * Incore version of the on-disk inode core structures. We log this directly + * into the journal in host CPU format (for better or worse) and as such + * directly mirrors the xfs_dinode structure as it must contain all the same + * information. + */ +typedef uint64_t xfs_log_timestamp_t; + +/* Legacy timestamp encoding format. */ +struct xfs_log_legacy_timestamp { + int32_t t_sec; /* timestamp seconds */ + int32_t t_nsec; /* timestamp nanoseconds */ +}; + +/* + * Define the format of the inode core that is logged. This structure must be + * kept identical to struct xfs_dinode except for the endianness annotations. + */ +struct xfs_log_dinode { + uint16_t di_magic; /* inode magic # = XFS_DINODE_MAGIC */ + uint16_t di_mode; /* mode and type of file */ + int8_t di_version; /* inode version */ + int8_t di_format; /* format of di_c data */ + uint16_t di_metatype; /* metadata type, if DIFLAG2_METADATA */ + uint32_t di_uid; /* owner's user id */ + uint32_t di_gid; /* owner's group id */ + uint32_t di_nlink; /* number of links to file */ + uint16_t di_projid_lo; /* lower part of owner's project id */ + uint16_t di_projid_hi; /* higher part of owner's project id */ + union { + /* Number of data fork extents if NREXT64 is set */ + uint64_t di_big_nextents; + + /* Padding for V3 inodes without NREXT64 set. */ + uint64_t di_v3_pad; + + /* Padding and inode flush counter for V2 inodes. */ + struct { + uint8_t di_v2_pad[6]; /* V2 inode zeroed space */ + uint16_t di_flushiter; /* V2 inode incremented on flush */ + }; + }; + xfs_log_timestamp_t di_atime; /* time last accessed */ + xfs_log_timestamp_t di_mtime; /* time last modified */ + xfs_log_timestamp_t di_ctime; /* time created/inode modified */ + xfs_fsize_t di_size; /* number of bytes in file */ + xfs_rfsblock_t di_nblocks; /* # of direct & btree blocks used */ + xfs_extlen_t di_extsize; /* basic/minimum extent size for file */ + union { + /* + * For V2 inodes and V3 inodes without NREXT64 set, this + * is the number of data and attr fork extents. + */ + struct { + uint32_t di_nextents; + uint16_t di_anextents; + } __packed; + + /* Number of attr fork extents if NREXT64 is set. */ + struct { + uint32_t di_big_anextents; + uint16_t di_nrext64_pad; + } __packed; + } __packed; + uint8_t di_forkoff; /* attr fork offs, <<3 for 64b align */ + int8_t di_aformat; /* format of attr fork's data */ + uint32_t di_dmevmask; /* DMIG event mask */ + uint16_t di_dmstate; /* DMIG state info */ + uint16_t di_flags; /* random flags, XFS_DIFLAG_... */ + uint32_t di_gen; /* generation number */ + + /* di_next_unlinked is the only non-core field in the old dinode */ + xfs_agino_t di_next_unlinked;/* agi unlinked list ptr */ + + /* start of the extended dinode, writable fields */ + uint32_t di_crc; /* CRC of the inode */ + uint64_t di_changecount; /* number of attribute changes */ + + /* + * The LSN we write to this field during formatting is not a reflection + * of the current on-disk LSN. It should never be used for recovery + * sequencing, nor should it be recovered into the on-disk inode at all. + * See xlog_recover_inode_commit_pass2() and xfs_log_dinode_to_disk() + * for details. + */ + xfs_lsn_t di_lsn; + + uint64_t di_flags2; /* more random flags */ + union { + /* basic cow extent size for (regular) file */ + uint32_t di_cowextsize; + /* used blocks in RTG for (zoned) rtrmap inode */ + uint32_t di_used_blocks; + }; + uint8_t di_pad2[12]; /* more padding for future expansion */ + + /* fields only written to during inode creation */ + xfs_log_timestamp_t di_crtime; /* time created */ + xfs_ino_t di_ino; /* inode number */ + uuid_t di_uuid; /* UUID of the filesystem */ + + /* structure must be padded to 64 bit alignment */ +}; + +#define xfs_log_dinode_size(mp) \ + (xfs_has_v3inodes((mp)) ? \ + sizeof(struct xfs_log_dinode) : \ + offsetof(struct xfs_log_dinode, di_next_unlinked)) + +/* + * Buffer Log Format definitions + * + * These are the physical dirty bitmap definitions for the log format structure. + */ +#define XFS_BLF_CHUNK 128 +#define XFS_BLF_SHIFT 7 +#define BIT_TO_WORD_SHIFT 5 +#define NBWORD (NBBY * sizeof(unsigned int)) + +/* + * This flag indicates that the buffer contains on disk inodes + * and requires special recovery handling. + */ +#define XFS_BLF_INODE_BUF (1<<0) + +/* + * This flag indicates that the buffer should not be replayed + * during recovery because its blocks are being freed. + */ +#define XFS_BLF_CANCEL (1<<1) + +/* + * This flag indicates that the buffer contains on disk + * user or group dquots and may require special recovery handling. + */ +#define XFS_BLF_UDQUOT_BUF (1<<2) +#define XFS_BLF_PDQUOT_BUF (1<<3) +#define XFS_BLF_GDQUOT_BUF (1<<4) + +/* + * This is the structure used to lay out a buf log item in the log. The data + * map describes which 128 byte chunks of the buffer have been logged. + * + * The placement of blf_map_size causes blf_data_map to start at an odd + * multiple of sizeof(unsigned int) offset within the struct. Because the data + * bitmap size will always be an even number, the end of the data_map (and + * therefore the structure) will also be at an odd multiple of sizeof(unsigned + * int). Some 64-bit compilers will insert padding at the end of the struct to + * ensure 64-bit alignment of blf_blkno, but 32-bit ones will not. Therefore, + * XFS_BLF_DATAMAP_SIZE must be an odd number to make the padding explicit and + * keep the structure size consistent between 32-bit and 64-bit platforms. + */ +#define __XFS_BLF_DATAMAP_SIZE ((XFS_MAX_BLOCKSIZE / XFS_BLF_CHUNK) / NBWORD) +#define XFS_BLF_DATAMAP_SIZE (__XFS_BLF_DATAMAP_SIZE + 1) + +struct xfs_buf_log_format { + unsigned short blf_type; /* buf log item type indicator */ + unsigned short blf_size; /* size of this item */ + unsigned short blf_flags; /* misc state */ + unsigned short blf_len; /* number of blocks in this buf */ + int64_t blf_blkno; /* starting blkno of this buf */ + unsigned int blf_map_size; /* used size of data bitmap in words */ + unsigned int blf_data_map[XFS_BLF_DATAMAP_SIZE]; /* dirty bitmap */ +}; + +/* + * All buffers now need to tell recovery where the magic number + * is so that it can verify and calculate the CRCs on the buffer correctly + * once the changes have been replayed into the buffer. + * + * The type value is held in the upper 5 bits of the blf_flags field, which is + * an unsigned 16 bit field. Hence we need to shift it 11 bits up and down. + */ +#define XFS_BLFT_BITS 5 +#define XFS_BLFT_SHIFT 11 +#define XFS_BLFT_MASK (((1 << XFS_BLFT_BITS) - 1) << XFS_BLFT_SHIFT) + +enum xfs_blft { + XFS_BLFT_UNKNOWN_BUF = 0, + XFS_BLFT_UDQUOT_BUF, + XFS_BLFT_PDQUOT_BUF, + XFS_BLFT_GDQUOT_BUF, + XFS_BLFT_BTREE_BUF, + XFS_BLFT_AGF_BUF, + XFS_BLFT_AGFL_BUF, + XFS_BLFT_AGI_BUF, + XFS_BLFT_DINO_BUF, + XFS_BLFT_SYMLINK_BUF, + XFS_BLFT_DIR_BLOCK_BUF, + XFS_BLFT_DIR_DATA_BUF, + XFS_BLFT_DIR_FREE_BUF, + XFS_BLFT_DIR_LEAF1_BUF, + XFS_BLFT_DIR_LEAFN_BUF, + XFS_BLFT_DA_NODE_BUF, + XFS_BLFT_ATTR_LEAF_BUF, + XFS_BLFT_ATTR_RMT_BUF, + XFS_BLFT_SB_BUF, + XFS_BLFT_RTBITMAP_BUF, + XFS_BLFT_RTSUMMARY_BUF, + XFS_BLFT_MAX_BUF = (1 << XFS_BLFT_BITS), +}; + +static inline void +xfs_blft_to_flags(struct xfs_buf_log_format *blf, enum xfs_blft type) +{ + ASSERT(type > XFS_BLFT_UNKNOWN_BUF && type < XFS_BLFT_MAX_BUF); + blf->blf_flags &= ~XFS_BLFT_MASK; + blf->blf_flags |= ((type << XFS_BLFT_SHIFT) & XFS_BLFT_MASK); +} + +static inline uint16_t +xfs_blft_from_flags(struct xfs_buf_log_format *blf) +{ + return (blf->blf_flags & XFS_BLFT_MASK) >> XFS_BLFT_SHIFT; +} + +/* + * EFI/EFD log format definitions + */ +struct xfs_extent { + xfs_fsblock_t ext_start; + xfs_extlen_t ext_len; +}; + +/* + * Since the structures in struct xfs_extent add up to 96 bytes, it has + * different alignments on i386 vs all other architectures, because i386 + * does not pad structures to their natural alignment. + * + * Provide the different variants for use by a conversion routine. + */ +struct xfs_extent_32 { + uint64_t ext_start; + uint32_t ext_len; +} __attribute__((packed)); + +struct xfs_extent_64 { + uint64_t ext_start; + uint32_t ext_len; + uint32_t ext_pad; +}; + +/* + * This is the structure used to lay out an efi log item in the + * log. The efi_extents field is a variable size array whose + * size is given by efi_nextents. + */ +struct xfs_efi_log_format { + uint16_t efi_type; /* efi log item type */ + uint16_t efi_size; /* size of this item */ + uint32_t efi_nextents; /* # extents to free */ + uint64_t efi_id; /* efi identifier */ + struct xfs_extent efi_extents[]; /* array of extents to free */ +}; + +static inline size_t +xfs_efi_log_format_sizeof( + unsigned int nr) +{ + return sizeof(struct xfs_efi_log_format) + + nr * sizeof(struct xfs_extent); +} + +struct xfs_efi_log_format_32 { + uint16_t efi_type; /* efi log item type */ + uint16_t efi_size; /* size of this item */ + uint32_t efi_nextents; /* # extents to free */ + uint64_t efi_id; /* efi identifier */ + struct xfs_extent_32 efi_extents[]; /* array of extents to free */ +} __attribute__((packed)); + +static inline size_t +xfs_efi_log_format32_sizeof( + unsigned int nr) +{ + return sizeof(struct xfs_efi_log_format_32) + + nr * sizeof(struct xfs_extent_32); +} + +struct xfs_efi_log_format_64 { + uint16_t efi_type; /* efi log item type */ + uint16_t efi_size; /* size of this item */ + uint32_t efi_nextents; /* # extents to free */ + uint64_t efi_id; /* efi identifier */ + struct xfs_extent_64 efi_extents[]; /* array of extents to free */ +}; + +static inline size_t +xfs_efi_log_format64_sizeof( + unsigned int nr) +{ + return sizeof(struct xfs_efi_log_format_64) + + nr * sizeof(struct xfs_extent_64); +} + +/* + * This is the structure used to lay out an efd log item in the + * log. The efd_extents array is a variable size array whose + * size is given by efd_nextents; + */ +struct xfs_efd_log_format { + uint16_t efd_type; /* efd log item type */ + uint16_t efd_size; /* size of this item */ + uint32_t efd_nextents; /* # of extents freed */ + uint64_t efd_efi_id; /* id of corresponding efi */ + struct xfs_extent efd_extents[]; /* array of extents freed */ +}; + +static inline size_t +xfs_efd_log_format_sizeof( + unsigned int nr) +{ + return sizeof(struct xfs_efd_log_format) + + nr * sizeof(struct xfs_extent); +} + +struct xfs_efd_log_format_32 { + uint16_t efd_type; /* efd log item type */ + uint16_t efd_size; /* size of this item */ + uint32_t efd_nextents; /* # of extents freed */ + uint64_t efd_efi_id; /* id of corresponding efi */ + struct xfs_extent_32 efd_extents[]; /* array of extents freed */ +} __attribute__((packed)); + +static inline size_t +xfs_efd_log_format32_sizeof( + unsigned int nr) +{ + return sizeof(struct xfs_efd_log_format_32) + + nr * sizeof(struct xfs_extent_32); +} + +struct xfs_efd_log_format_64 { + uint16_t efd_type; /* efd log item type */ + uint16_t efd_size; /* size of this item */ + uint32_t efd_nextents; /* # of extents freed */ + uint64_t efd_efi_id; /* id of corresponding efi */ + struct xfs_extent_64 efd_extents[]; /* array of extents freed */ +}; + +static inline size_t +xfs_efd_log_format64_sizeof( + unsigned int nr) +{ + return sizeof(struct xfs_efd_log_format_64) + + nr * sizeof(struct xfs_extent_64); +} + +/* + * RUI/RUD (reverse mapping) log format definitions + */ +struct xfs_map_extent { + uint64_t me_owner; + uint64_t me_startblock; + uint64_t me_startoff; + uint32_t me_len; + uint32_t me_flags; +}; + +/* rmap me_flags: upper bits are flags, lower byte is type code */ +#define XFS_RMAP_EXTENT_MAP 1 +#define XFS_RMAP_EXTENT_MAP_SHARED 2 +#define XFS_RMAP_EXTENT_UNMAP 3 +#define XFS_RMAP_EXTENT_UNMAP_SHARED 4 +#define XFS_RMAP_EXTENT_CONVERT 5 +#define XFS_RMAP_EXTENT_CONVERT_SHARED 6 +#define XFS_RMAP_EXTENT_ALLOC 7 +#define XFS_RMAP_EXTENT_FREE 8 +#define XFS_RMAP_EXTENT_TYPE_MASK 0xFF + +#define XFS_RMAP_EXTENT_ATTR_FORK (1U << 31) +#define XFS_RMAP_EXTENT_BMBT_BLOCK (1U << 30) +#define XFS_RMAP_EXTENT_UNWRITTEN (1U << 29) + +#define XFS_RMAP_EXTENT_FLAGS (XFS_RMAP_EXTENT_TYPE_MASK | \ + XFS_RMAP_EXTENT_ATTR_FORK | \ + XFS_RMAP_EXTENT_BMBT_BLOCK | \ + XFS_RMAP_EXTENT_UNWRITTEN) + +/* + * This is the structure used to lay out an rui log item in the + * log. The rui_extents field is a variable size array whose + * size is given by rui_nextents. + */ +struct xfs_rui_log_format { + uint16_t rui_type; /* rui log item type */ + uint16_t rui_size; /* size of this item */ + uint32_t rui_nextents; /* # extents to free */ + uint64_t rui_id; /* rui identifier */ + struct xfs_map_extent rui_extents[]; /* array of extents to rmap */ +}; + +static inline size_t +xfs_rui_log_format_sizeof( + unsigned int nr) +{ + return sizeof(struct xfs_rui_log_format) + + nr * sizeof(struct xfs_map_extent); +} + +/* + * This is the structure used to lay out an rud log item in the + * log. The rud_extents array is a variable size array whose + * size is given by rud_nextents; + */ +struct xfs_rud_log_format { + uint16_t rud_type; /* rud log item type */ + uint16_t rud_size; /* size of this item */ + uint32_t __pad; + uint64_t rud_rui_id; /* id of corresponding rui */ +}; + +/* + * CUI/CUD (refcount update) log format definitions + */ +struct xfs_phys_extent { + uint64_t pe_startblock; + uint32_t pe_len; + uint32_t pe_flags; +}; + +/* refcount pe_flags: upper bits are flags, lower byte is type code */ +/* Type codes are taken directly from enum xfs_refcount_intent_type. */ +#define XFS_REFCOUNT_EXTENT_TYPE_MASK 0xFF + +#define XFS_REFCOUNT_EXTENT_FLAGS (XFS_REFCOUNT_EXTENT_TYPE_MASK) + +/* + * This is the structure used to lay out a cui log item in the + * log. The cui_extents field is a variable size array whose + * size is given by cui_nextents. + */ +struct xfs_cui_log_format { + uint16_t cui_type; /* cui log item type */ + uint16_t cui_size; /* size of this item */ + uint32_t cui_nextents; /* # extents to free */ + uint64_t cui_id; /* cui identifier */ + struct xfs_phys_extent cui_extents[]; /* array of extents */ +}; + +static inline size_t +xfs_cui_log_format_sizeof( + unsigned int nr) +{ + return sizeof(struct xfs_cui_log_format) + + nr * sizeof(struct xfs_phys_extent); +} + +/* + * This is the structure used to lay out a cud log item in the + * log. The cud_extents array is a variable size array whose + * size is given by cud_nextents; + */ +struct xfs_cud_log_format { + uint16_t cud_type; /* cud log item type */ + uint16_t cud_size; /* size of this item */ + uint32_t __pad; + uint64_t cud_cui_id; /* id of corresponding cui */ +}; + +/* + * BUI/BUD (inode block mapping) log format definitions + */ + +/* bmbt me_flags: upper bits are flags, lower byte is type code */ +/* Type codes are taken directly from enum xfs_bmap_intent_type. */ +#define XFS_BMAP_EXTENT_TYPE_MASK 0xFF + +#define XFS_BMAP_EXTENT_ATTR_FORK (1U << 31) +#define XFS_BMAP_EXTENT_UNWRITTEN (1U << 30) +#define XFS_BMAP_EXTENT_REALTIME (1U << 29) + +#define XFS_BMAP_EXTENT_FLAGS (XFS_BMAP_EXTENT_TYPE_MASK | \ + XFS_BMAP_EXTENT_ATTR_FORK | \ + XFS_BMAP_EXTENT_UNWRITTEN | \ + XFS_BMAP_EXTENT_REALTIME) + +/* + * This is the structure used to lay out an bui log item in the + * log. The bui_extents field is a variable size array whose + * size is given by bui_nextents. + */ +struct xfs_bui_log_format { + uint16_t bui_type; /* bui log item type */ + uint16_t bui_size; /* size of this item */ + uint32_t bui_nextents; /* # extents to free */ + uint64_t bui_id; /* bui identifier */ + struct xfs_map_extent bui_extents[]; /* array of extents to bmap */ +}; + +static inline size_t +xfs_bui_log_format_sizeof( + unsigned int nr) +{ + return sizeof(struct xfs_bui_log_format) + + nr * sizeof(struct xfs_map_extent); +} + +/* + * This is the structure used to lay out an bud log item in the + * log. The bud_extents array is a variable size array whose + * size is given by bud_nextents; + */ +struct xfs_bud_log_format { + uint16_t bud_type; /* bud log item type */ + uint16_t bud_size; /* size of this item */ + uint32_t __pad; + uint64_t bud_bui_id; /* id of corresponding bui */ +}; + +/* + * XMI/XMD (file mapping exchange) log format definitions + */ + +/* This is the structure used to lay out an mapping exchange log item. */ +struct xfs_xmi_log_format { + uint16_t xmi_type; /* xmi log item type */ + uint16_t xmi_size; /* size of this item */ + uint32_t __pad; /* must be zero */ + uint64_t xmi_id; /* xmi identifier */ + + uint64_t xmi_inode1; /* inumber of first file */ + uint64_t xmi_inode2; /* inumber of second file */ + uint32_t xmi_igen1; /* generation of first file */ + uint32_t xmi_igen2; /* generation of second file */ + uint64_t xmi_startoff1; /* block offset into file1 */ + uint64_t xmi_startoff2; /* block offset into file2 */ + uint64_t xmi_blockcount; /* number of blocks */ + uint64_t xmi_flags; /* XFS_EXCHMAPS_* */ + uint64_t xmi_isize1; /* intended file1 size */ + uint64_t xmi_isize2; /* intended file2 size */ +}; + +/* Exchange mappings between extended attribute forks instead of data forks. */ +#define XFS_EXCHMAPS_ATTR_FORK (1ULL << 0) + +/* Set the file sizes when finished. */ +#define XFS_EXCHMAPS_SET_SIZES (1ULL << 1) + +/* + * Exchange the mappings of the two files only if the file allocation units + * mapped to file1's range have been written. + */ +#define XFS_EXCHMAPS_INO1_WRITTEN (1ULL << 2) + +/* Clear the reflink flag from inode1 after the operation. */ +#define XFS_EXCHMAPS_CLEAR_INO1_REFLINK (1ULL << 3) + +/* Clear the reflink flag from inode2 after the operation. */ +#define XFS_EXCHMAPS_CLEAR_INO2_REFLINK (1ULL << 4) + +#define XFS_EXCHMAPS_LOGGED_FLAGS (XFS_EXCHMAPS_ATTR_FORK | \ + XFS_EXCHMAPS_SET_SIZES | \ + XFS_EXCHMAPS_INO1_WRITTEN | \ + XFS_EXCHMAPS_CLEAR_INO1_REFLINK | \ + XFS_EXCHMAPS_CLEAR_INO2_REFLINK) + +/* This is the structure used to lay out an mapping exchange done log item. */ +struct xfs_xmd_log_format { + uint16_t xmd_type; /* xmd log item type */ + uint16_t xmd_size; /* size of this item */ + uint32_t __pad; + uint64_t xmd_xmi_id; /* id of corresponding xmi */ +}; + +/* + * Dquot Log format definitions. + * + * The first two fields must be the type and size fitting into + * 32 bits : log_recovery code assumes that. + */ +struct xfs_dq_logformat { + uint16_t qlf_type; /* dquot log item type */ + uint16_t qlf_size; /* size of this item */ + xfs_dqid_t qlf_id; /* usr/grp/proj id : 32 bits */ + int64_t qlf_blkno; /* blkno of dquot buffer */ + int32_t qlf_len; /* len of dquot buffer */ + uint32_t qlf_boffset; /* off of dquot in buffer */ +}; + +/* + * log format struct for QUOTAOFF records. + * The first two fields must be the type and size fitting into + * 32 bits : log_recovery code assumes that. + * We write two LI_QUOTAOFF logitems per quotaoff, the last one keeps a pointer + * to the first and ensures that the first logitem is taken out of the AIL + * only when the last one is securely committed. + */ +struct xfs_qoff_logformat { + unsigned short qf_type; /* quotaoff log item type */ + unsigned short qf_size; /* size of this item */ + unsigned int qf_flags; /* USR and/or GRP */ + char qf_pad[12]; /* padding for future */ +}; + +/* + * Disk quotas status in m_qflags, and also sb_qflags. 16 bits. + */ +#define XFS_UQUOTA_ACCT 0x0001 /* user quota accounting ON */ +#define XFS_UQUOTA_ENFD 0x0002 /* user quota limits enforced */ +#define XFS_UQUOTA_CHKD 0x0004 /* quotacheck run on usr quotas */ +#define XFS_PQUOTA_ACCT 0x0008 /* project quota accounting ON */ +#define XFS_OQUOTA_ENFD 0x0010 /* other (grp/prj) quota limits enforced */ +#define XFS_OQUOTA_CHKD 0x0020 /* quotacheck run on other (grp/prj) quotas */ +#define XFS_GQUOTA_ACCT 0x0040 /* group quota accounting ON */ + +/* + * Conversion to and from the combined OQUOTA flag (if necessary) + * is done only in xfs_sb_qflags_to_disk() and xfs_sb_qflags_from_disk() + */ +#define XFS_GQUOTA_ENFD 0x0080 /* group quota limits enforced */ +#define XFS_GQUOTA_CHKD 0x0100 /* quotacheck run on group quotas */ +#define XFS_PQUOTA_ENFD 0x0200 /* project quota limits enforced */ +#define XFS_PQUOTA_CHKD 0x0400 /* quotacheck run on project quotas */ + +#define XFS_ALL_QUOTA_ACCT \ + (XFS_UQUOTA_ACCT | XFS_GQUOTA_ACCT | XFS_PQUOTA_ACCT) +#define XFS_ALL_QUOTA_ENFD \ + (XFS_UQUOTA_ENFD | XFS_GQUOTA_ENFD | XFS_PQUOTA_ENFD) +#define XFS_ALL_QUOTA_CHKD \ + (XFS_UQUOTA_CHKD | XFS_GQUOTA_CHKD | XFS_PQUOTA_CHKD) + +#define XFS_MOUNT_QUOTA_ALL (XFS_UQUOTA_ACCT|XFS_UQUOTA_ENFD|\ + XFS_UQUOTA_CHKD|XFS_GQUOTA_ACCT|\ + XFS_GQUOTA_ENFD|XFS_GQUOTA_CHKD|\ + XFS_PQUOTA_ACCT|XFS_PQUOTA_ENFD|\ + XFS_PQUOTA_CHKD) + +/* + * Inode create log item structure + * + * Log recovery assumes the first two entries are the type and size and they fit + * in 32 bits. Also in host order (ugh) so they have to be 32 bit aligned so + * decoding can be done correctly. + */ +struct xfs_icreate_log { + uint16_t icl_type; /* type of log format structure */ + uint16_t icl_size; /* size of log format structure */ + __be32 icl_ag; /* ag being allocated in */ + __be32 icl_agbno; /* start block of inode range */ + __be32 icl_count; /* number of inodes to initialise */ + __be32 icl_isize; /* size of inodes */ + __be32 icl_length; /* length of extent to initialise */ + __be32 icl_gen; /* inode generation number to use */ +}; + +/* + * Flags for deferred attribute operations. + * Upper bits are flags, lower byte is type code + */ +#define XFS_ATTRI_OP_FLAGS_SET 1 /* Set the attribute */ +#define XFS_ATTRI_OP_FLAGS_REMOVE 2 /* Remove the attribute */ +#define XFS_ATTRI_OP_FLAGS_REPLACE 3 /* Replace the attribute */ +#define XFS_ATTRI_OP_FLAGS_PPTR_SET 4 /* Set parent pointer */ +#define XFS_ATTRI_OP_FLAGS_PPTR_REMOVE 5 /* Remove parent pointer */ +#define XFS_ATTRI_OP_FLAGS_PPTR_REPLACE 6 /* Replace parent pointer */ +#define XFS_ATTRI_OP_FLAGS_TYPE_MASK 0xFF /* Flags type mask */ + +/* + * alfi_attr_filter captures the state of xfs_da_args.attr_filter, so it should + * never have any other bits set. + */ +#define XFS_ATTRI_FILTER_MASK (XFS_ATTR_ROOT | \ + XFS_ATTR_SECURE | \ + XFS_ATTR_PARENT | \ + XFS_ATTR_INCOMPLETE) + +/* + * This is the structure used to lay out an attr log item in the + * log. + */ +struct xfs_attri_log_format { + uint16_t alfi_type; /* attri log item type */ + uint16_t alfi_size; /* size of this item */ + uint32_t alfi_igen; /* generation of alfi_ino for pptr ops */ + uint64_t alfi_id; /* attri identifier */ + uint64_t alfi_ino; /* the inode for this attr operation */ + uint32_t alfi_op_flags; /* marks the op as a set or remove */ + union { + uint32_t alfi_name_len; /* attr name length */ + struct { + /* + * For PPTR_REPLACE, these are the lengths of the old + * and new attr names. The new and old values must + * have the same length. + */ + uint16_t alfi_old_name_len; + uint16_t alfi_new_name_len; + }; + }; + uint32_t alfi_value_len; /* attr value length */ + uint32_t alfi_attr_filter;/* attr filter flags */ +}; + +struct xfs_attrd_log_format { + uint16_t alfd_type; /* attrd log item type */ + uint16_t alfd_size; /* size of this item */ + uint32_t __pad; /* pad to 64 bit aligned */ + uint64_t alfd_alf_id; /* id of corresponding attri */ +}; + +#endif /* __XFS_LOG_FORMAT_H__ */ diff --git a/fs/xfs/libxfs/xfs_log_recover.h b/fs/xfs/libxfs/xfs_log_recover.h new file mode 100644 index 000000000..9e712e623 --- /dev/null +++ b/fs/xfs/libxfs/xfs_log_recover.h @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_LOG_RECOVER_H__ +#define __XFS_LOG_RECOVER_H__ + +/* + * Each log item type (XFS_LI_*) gets its own xlog_recover_item_ops to + * define how recovery should work for that type of log item. + */ +struct xlog_recover_item; +struct xfs_defer_op_type; + +/* Sorting hat for log items as they're read in. */ +enum xlog_recover_reorder { + XLOG_REORDER_BUFFER_LIST, + XLOG_REORDER_ITEM_LIST, + XLOG_REORDER_INODE_BUFFER_LIST, + XLOG_REORDER_CANCEL_LIST, +}; + +struct xlog_recover_item_ops { + uint16_t item_type; /* XFS_LI_* type code. */ + + /* + * Help sort recovered log items into the order required to replay them + * correctly. Log item types that always use XLOG_REORDER_ITEM_LIST do + * not have to supply a function here. See the comment preceding + * xlog_recover_reorder_trans for more details about what the return + * values mean. + */ + enum xlog_recover_reorder (*reorder)(struct xlog_recover_item *item); + + /* Start readahead for pass2, if provided. */ + void (*ra_pass2)(struct xlog *log, struct xlog_recover_item *item); + + /* Do whatever work we need to do for pass1, if provided. */ + int (*commit_pass1)(struct xlog *log, struct xlog_recover_item *item); + + /* + * This function should do whatever work is needed for pass2 of log + * recovery, if provided. + * + * If the recovered item is an intent item, this function should parse + * the recovered item to construct an in-core log intent item and + * insert it into the AIL. The in-core log intent item should have 1 + * refcount so that the item is freed either (a) when we commit the + * recovered log item for the intent-done item; (b) replay the work and + * log a new intent-done item; or (c) recovery fails and we have to + * abort. + * + * If the recovered item is an intent-done item, this function should + * parse the recovered item to find the id of the corresponding intent + * log item. Next, it should find the in-core log intent item in the + * AIL and release it. + */ + int (*commit_pass2)(struct xlog *log, struct list_head *buffer_list, + struct xlog_recover_item *item, xfs_lsn_t lsn); +}; + +extern const struct xlog_recover_item_ops xlog_icreate_item_ops; +extern const struct xlog_recover_item_ops xlog_buf_item_ops; +extern const struct xlog_recover_item_ops xlog_inode_item_ops; +extern const struct xlog_recover_item_ops xlog_dquot_item_ops; +extern const struct xlog_recover_item_ops xlog_quotaoff_item_ops; +extern const struct xlog_recover_item_ops xlog_bui_item_ops; +extern const struct xlog_recover_item_ops xlog_bud_item_ops; +extern const struct xlog_recover_item_ops xlog_efi_item_ops; +extern const struct xlog_recover_item_ops xlog_efd_item_ops; +extern const struct xlog_recover_item_ops xlog_rui_item_ops; +extern const struct xlog_recover_item_ops xlog_rud_item_ops; +extern const struct xlog_recover_item_ops xlog_cui_item_ops; +extern const struct xlog_recover_item_ops xlog_cud_item_ops; +extern const struct xlog_recover_item_ops xlog_attri_item_ops; +extern const struct xlog_recover_item_ops xlog_attrd_item_ops; +extern const struct xlog_recover_item_ops xlog_xmi_item_ops; +extern const struct xlog_recover_item_ops xlog_xmd_item_ops; +extern const struct xlog_recover_item_ops xlog_rtefi_item_ops; +extern const struct xlog_recover_item_ops xlog_rtefd_item_ops; +extern const struct xlog_recover_item_ops xlog_rtrui_item_ops; +extern const struct xlog_recover_item_ops xlog_rtrud_item_ops; +extern const struct xlog_recover_item_ops xlog_rtcui_item_ops; +extern const struct xlog_recover_item_ops xlog_rtcud_item_ops; + +/* + * Macros, structures, prototypes for internal log manager use. + */ + +#define XLOG_RHASH_BITS 4 +#define XLOG_RHASH_SIZE 16 +#define XLOG_RHASH_SHIFT 2 +#define XLOG_RHASH(tid) \ + ((((uint32_t)tid)>>XLOG_RHASH_SHIFT) & (XLOG_RHASH_SIZE-1)) + +#define XLOG_MAX_REGIONS_IN_ITEM (XFS_MAX_BLOCKSIZE / XFS_BLF_CHUNK / 2 + 1) + + +/* + * item headers are in ri_buf[0]. Additional buffers follow. + */ +struct xlog_recover_item { + struct list_head ri_list; + int ri_cnt; /* count of regions found */ + int ri_total; /* total regions */ + struct kvec *ri_buf; /* ptr to regions buffer */ + const struct xlog_recover_item_ops *ri_ops; +}; + +struct xlog_recover { + struct hlist_node r_list; + xlog_tid_t r_log_tid; /* log's transaction id */ + struct xfs_trans_header r_theader; /* trans header for partial */ + int r_state; /* not needed */ + xfs_lsn_t r_lsn; /* xact lsn */ + struct list_head r_itemq; /* q for items */ +}; + +#define ITEM_TYPE(i) (*(unsigned short *)(i)->ri_buf[0].iov_base) + +#define XLOG_RECOVER_CRCPASS 0 +#define XLOG_RECOVER_PASS1 1 +#define XLOG_RECOVER_PASS2 2 + +void xlog_buf_readahead(struct xlog *log, xfs_daddr_t blkno, uint len, + const struct xfs_buf_ops *ops); +bool xlog_is_buffer_cancelled(struct xlog *log, xfs_daddr_t blkno, uint len); + +int xlog_recover_iget(struct xfs_mount *mp, xfs_ino_t ino, + struct xfs_inode **ipp); +int xlog_recover_iget_handle(struct xfs_mount *mp, xfs_ino_t ino, uint32_t gen, + struct xfs_inode **ipp); +void xlog_recover_release_intent(struct xlog *log, unsigned short intent_type, + uint64_t intent_id); +int xlog_alloc_buf_cancel_table(struct xlog *log); +void xlog_free_buf_cancel_table(struct xlog *log); + +#ifdef DEBUG +void xlog_check_buf_cancel_table(struct xlog *log); +#else +#define xlog_check_buf_cancel_table(log) do { } while (0) +#endif + +/* + * Transform a regular reservation into one suitable for recovery of a log + * intent item. + * + * Intent recovery only runs a single step of the transaction chain and defers + * the rest to a separate transaction. Therefore, we reduce logcount to 1 here + * to avoid livelocks if the log grant space is nearly exhausted due to the + * recovered intent pinning the tail. Keep the same logflags to avoid tripping + * asserts elsewhere. Struct copies abound below. + */ +static inline struct xfs_trans_res +xlog_recover_resv(const struct xfs_trans_res *r) +{ + struct xfs_trans_res ret = { + .tr_logres = r->tr_logres, + .tr_logcount = 1, + .tr_logflags = r->tr_logflags, + }; + + return ret; +} + +struct xfs_defer_pending; + +void xlog_recover_intent_item(struct xlog *log, struct xfs_log_item *lip, + xfs_lsn_t lsn, const struct xfs_defer_op_type *ops); +int xlog_recover_finish_intent(struct xfs_trans *tp, + struct xfs_defer_pending *dfp); + +#endif /* __XFS_LOG_RECOVER_H__ */ diff --git a/fs/xfs/libxfs/xfs_log_rlimit.c b/fs/xfs/libxfs/xfs_log_rlimit.c new file mode 100644 index 000000000..37712b2f8 --- /dev/null +++ b/fs/xfs/libxfs/xfs_log_rlimit.c @@ -0,0 +1,251 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2013 Jie Liu. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_da_format.h" +#include "xfs_trans_space.h" +#include "xfs_da_btree.h" +#include "xfs_bmap_btree.h" +#include "xfs_trace.h" + +/* + * Shortly after enabling the large extents count feature in 2023, longstanding + * bugs were found in the code that computes the minimum log size. Luckily, + * the bugs resulted in over-estimates of that size, so there's no impact to + * existing users. However, we don't want to reduce the minimum log size + * because that can create the situation where a newer mkfs writes a new + * filesystem that an older kernel won't mount. + * + * Several years prior, we also discovered that the transaction reservations + * for rmap and reflink operations were unnecessarily large. That was fixed, + * but the minimum log size computation was left alone to avoid the + * compatibility problems noted above. Fix that too. + * + * Therefore, we only may correct the computation starting with filesystem + * features that didn't exist in 2023. In other words, only turn this on if + * the filesystem has parent pointers. + * + * This function can be called before the XFS_HAS_* flags have been set up, + * (e.g. mkfs) so we must check the ondisk superblock. + */ +static inline bool +xfs_want_minlogsize_fixes( + struct xfs_sb *sb) +{ + return xfs_sb_is_v5(sb) && + xfs_sb_has_incompat_feature(sb, XFS_SB_FEAT_INCOMPAT_PARENT); +} + +/* + * Calculate the maximum length in bytes that would be required for a local + * attribute value as large attributes out of line are not logged. + */ +STATIC int +xfs_log_calc_max_attrsetm_res( + struct xfs_mount *mp) +{ + int size; + int nblks; + + size = xfs_attr_leaf_entsize_local_max(mp->m_attr_geo->blksize) - + MAXNAMELEN - 1; + nblks = XFS_DAENTER_SPACE_RES(mp, XFS_ATTR_FORK); + nblks += XFS_B_TO_FSB(mp, size); + + /* + * If the feature set is new enough, correct a unit conversion error in + * the xattr transaction reservation code that resulted in oversized + * minimum log size computations. + */ + if (xfs_want_minlogsize_fixes(&mp->m_sb)) + size = XFS_B_TO_FSB(mp, size); + + nblks += XFS_NEXTENTADD_SPACE_RES(mp, size, XFS_ATTR_FORK); + + return M_RES(mp)->tr_attrsetm.tr_logres + + M_RES(mp)->tr_attrsetrt.tr_logres * nblks; +} + +/* + * Compute an alternate set of log reservation sizes for use exclusively with + * minimum log size calculations. + */ +static void +xfs_log_calc_trans_resv_for_minlogblocks( + struct xfs_mount *mp, + struct xfs_trans_resv *resv) +{ + unsigned int rmap_maxlevels = mp->m_rmap_maxlevels; + + /* + * If the feature set is new enough, drop the oversized minimum log + * size computation introduced by the original reflink code. + */ + if (xfs_want_minlogsize_fixes(&mp->m_sb)) { + xfs_trans_resv_calc(mp, resv); + resv->tr_atomic_ioend = M_RES(mp)->tr_atomic_ioend; + return; + } + + /* + * In the early days of rmap+reflink, we always set the rmap maxlevels + * to 9 even if the AG was small enough that it would never grow to + * that height. Transaction reservation sizes influence the minimum + * log size calculation, which influences the size of the log that mkfs + * creates. Use the old value here to ensure that newly formatted + * small filesystems will mount on older kernels. + */ + if (xfs_has_rmapbt(mp) && xfs_has_reflink(mp)) + mp->m_rmap_maxlevels = XFS_OLD_REFLINK_RMAP_MAXLEVELS; + + xfs_trans_resv_calc(mp, resv); + + /* Copy the dynamic transaction reservation types from the running fs */ + resv->tr_atomic_ioend = M_RES(mp)->tr_atomic_ioend; + + if (xfs_has_reflink(mp)) { + /* + * In the early days of reflink, typical log operation counts + * were greatly overestimated. + */ + resv->tr_write.tr_logcount = XFS_WRITE_LOG_COUNT_REFLINK; + resv->tr_itruncate.tr_logcount = + XFS_ITRUNCATE_LOG_COUNT_REFLINK; + resv->tr_qm_dqalloc.tr_logcount = XFS_WRITE_LOG_COUNT_REFLINK; + } else if (xfs_has_rmapbt(mp)) { + /* + * In the early days of non-reflink rmap, the impact of rmapbt + * updates on log counts were not taken into account at all. + */ + resv->tr_write.tr_logcount = XFS_WRITE_LOG_COUNT; + resv->tr_itruncate.tr_logcount = XFS_ITRUNCATE_LOG_COUNT; + resv->tr_qm_dqalloc.tr_logcount = XFS_WRITE_LOG_COUNT; + } + + /* + * In the early days of reflink, we did not use deferred refcount + * update log items, so log reservations must be recomputed using the + * old calculations. + */ + resv->tr_write.tr_logres = + xfs_calc_write_reservation_minlogsize(mp); + resv->tr_itruncate.tr_logres = + xfs_calc_itruncate_reservation_minlogsize(mp); + resv->tr_qm_dqalloc.tr_logres = + xfs_calc_qm_dqalloc_reservation_minlogsize(mp); + + /* Put everything back the way it was. This goes at the end. */ + mp->m_rmap_maxlevels = rmap_maxlevels; +} + +/* + * Iterate over the log space reservation table to figure out and return + * the maximum one in terms of the pre-calculated values which were done + * at mount time. + */ +void +xfs_log_get_max_trans_res( + struct xfs_mount *mp, + struct xfs_trans_res *max_resp) +{ + struct xfs_trans_resv resv = {}; + struct xfs_trans_res *resp; + struct xfs_trans_res *end_resp; + unsigned int i; + int log_space = 0; + int attr_space; + + attr_space = xfs_log_calc_max_attrsetm_res(mp); + + xfs_log_calc_trans_resv_for_minlogblocks(mp, &resv); + + resp = (struct xfs_trans_res *)&resv; + end_resp = (struct xfs_trans_res *)(&resv + 1); + for (i = 0; resp < end_resp; i++, resp++) { + int tmp = resp->tr_logcount > 1 ? + resp->tr_logres * resp->tr_logcount : + resp->tr_logres; + + trace_xfs_trans_resv_calc_minlogsize(mp, i, resp); + if (log_space < tmp) { + log_space = tmp; + *max_resp = *resp; /* struct copy */ + } + } + + if (attr_space > log_space) { + *max_resp = resv.tr_attrsetm; /* struct copy */ + max_resp->tr_logres = attr_space; + } + trace_xfs_log_get_max_trans_res(mp, max_resp); +} + +/* + * Calculate the minimum valid log size for the given superblock configuration. + * Used to calculate the minimum log size at mkfs time, and to determine if + * the log is large enough or not at mount time. Returns the minimum size in + * filesystem block size units. + */ +int +xfs_log_calc_minimum_size( + struct xfs_mount *mp) +{ + struct xfs_trans_res tres = {0}; + int max_logres; + int min_logblks = 0; + int lsunit = 0; + + xfs_log_get_max_trans_res(mp, &tres); + + max_logres = xfs_log_calc_unit_res(mp, tres.tr_logres); + if (tres.tr_logcount > 1) + max_logres *= tres.tr_logcount; + + if (xfs_has_logv2(mp) && mp->m_sb.sb_logsunit > 1) + lsunit = BTOBB(mp->m_sb.sb_logsunit); + + /* + * Two factors should be taken into account for calculating the minimum + * log space. + * 1) The fundamental limitation is that no single transaction can be + * larger than half size of the log. + * + * From mkfs.xfs, this is considered by the XFS_MIN_LOG_FACTOR + * define, which is set to 3. That means we can definitely fit + * maximally sized 2 transactions in the log. We'll use this same + * value here. + * + * 2) If the lsunit option is specified, a transaction requires 2 LSU + * for the reservation because there are two log writes that can + * require padding - the transaction data and the commit record which + * are written separately and both can require padding to the LSU. + * Consider that we can have an active CIL reservation holding 2*LSU, + * but the CIL is not over a push threshold, in this case, if we + * don't have enough log space for at one new transaction, which + * includes another 2*LSU in the reservation, we will run into dead + * loop situation in log space grant procedure. i.e. + * xlog_grant_head_wait(). + * + * Hence the log size needs to be able to contain two maximally sized + * and padded transactions, which is (2 * (2 * LSU + maxlres)). + * + * Also, the log size should be a multiple of the log stripe unit, round + * it up to lsunit boundary if lsunit is specified. + */ + if (lsunit) { + min_logblks = roundup_64(BTOBB(max_logres), lsunit) + + 2 * lsunit; + } else + min_logblks = BTOBB(max_logres) + 2 * BBSIZE; + min_logblks *= XFS_MIN_LOG_FACTOR; + + return XFS_BB_TO_FSB(mp, min_logblks); +} diff --git a/fs/xfs/libxfs/xfs_metadir.c b/fs/xfs/libxfs/xfs_metadir.c new file mode 100644 index 000000000..7c6b086b7 --- /dev/null +++ b/fs/xfs/libxfs/xfs_metadir.c @@ -0,0 +1,501 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2018-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_sb.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_trans.h" +#include "xfs_metafile.h" +#include "xfs_metadir.h" +#include "xfs_trace.h" +#include "xfs_inode.h" +#include "xfs_quota.h" +#include "xfs_ialloc.h" +#include "xfs_bmap_btree.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_trans_space.h" +#include "xfs_ag.h" +#include "xfs_dir2.h" +#include "xfs_dir2_priv.h" +#include "xfs_parent.h" +#include "xfs_health.h" +#include "xfs_errortag.h" +#include "xfs_error.h" +#include "xfs_btree.h" +#include "xfs_alloc.h" + +/* + * Metadata Directory Tree + * ======================= + * + * These functions provide an abstraction layer for looking up, creating, and + * deleting metadata inodes that live within a special metadata directory tree. + * + * This code does not manage the five existing metadata inodes: real time + * bitmap & summary; and the user, group, and quotas. All other metadata + * inodes must use only the xfs_meta{dir,file}_* functions. + * + * Callers wishing to create or hardlink a metadata inode must create an + * xfs_metadir_update structure, call the appropriate xfs_metadir* function, + * and then call xfs_metadir_commit or xfs_metadir_cancel to commit or cancel + * the update. Files in the metadata directory tree currently cannot be + * unlinked. + * + * When the metadir feature is enabled, all metadata inodes must have the + * "metadata" inode flag set to prevent them from being exposed to the outside + * world. + * + * Callers must take the ILOCK of any inode in the metadata directory tree to + * synchronize access to that inode. It is never necessary to take the IOLOCK + * or the MMAPLOCK since metadata inodes must not be exposed to user space. + */ + +static inline void +xfs_metadir_set_xname( + struct xfs_name *xname, + const char *path, + unsigned char ftype) +{ + xname->name = (const unsigned char *)path; + xname->len = strlen(path); + xname->type = ftype; +} + +/* + * Given a parent directory @dp and a metadata inode path component @xname, + * Look up the inode number in the directory, returning it in @ino. + * @xname.type must match the directory entry's ftype. + * + * Caller must hold ILOCK_EXCL. + */ +static inline int +xfs_metadir_lookup( + struct xfs_trans *tp, + struct xfs_inode *dp, + struct xfs_name *xname, + xfs_ino_t *ino) +{ + struct xfs_mount *mp = dp->i_mount; + struct xfs_da_args args = { + .trans = tp, + .dp = dp, + .geo = mp->m_dir_geo, + .name = xname->name, + .namelen = xname->len, + .hashval = xfs_dir2_hashname(mp, xname), + .whichfork = XFS_DATA_FORK, + .op_flags = XFS_DA_OP_OKNOENT, + .owner = I_INO(dp), + }; + int error; + + if (!S_ISDIR(VFS_I(dp)->i_mode)) { + xfs_fs_mark_sick(mp, XFS_SICK_FS_METADIR); + return -EFSCORRUPTED; + } + if (xfs_is_shutdown(mp)) + return -EIO; + + error = xfs_dir_lookup_args(&args); + if (error) + return error; + + if (!xfs_verify_ino(mp, args.inumber)) { + xfs_fs_mark_sick(mp, XFS_SICK_FS_METADIR); + return -EFSCORRUPTED; + } + if (xname->type != XFS_DIR3_FT_UNKNOWN && xname->type != args.filetype) { + xfs_fs_mark_sick(mp, XFS_SICK_FS_METADIR); + return -EFSCORRUPTED; + } + + trace_xfs_metadir_lookup(dp, xname, args.inumber); + *ino = args.inumber; + return 0; +} + +/* + * Look up and read a metadata inode from the metadata directory. If the path + * component doesn't exist, return -ENOENT. + */ +int +xfs_metadir_load( + struct xfs_trans *tp, + struct xfs_inode *dp, + const char *path, + enum xfs_metafile_type metafile_type, + struct xfs_inode **ipp) +{ + struct xfs_name xname; + xfs_ino_t ino; + int error; + + xfs_metadir_set_xname(&xname, path, XFS_DIR3_FT_UNKNOWN); + + xfs_ilock(dp, XFS_ILOCK_EXCL); + error = xfs_metadir_lookup(tp, dp, &xname, &ino); + xfs_iunlock(dp, XFS_ILOCK_EXCL); + if (error) + return error; + return xfs_trans_metafile_iget(tp, ino, metafile_type, ipp); +} + +/* + * Unlock and release resources after committing (or cancelling) a metadata + * directory tree operation. The caller retains its reference to @upd->ip + * and must release it explicitly. + */ +static inline void +xfs_metadir_teardown( + struct xfs_metadir_update *upd, + int error) +{ + trace_xfs_metadir_teardown(upd, error); + + if (upd->ppargs) { + xfs_parent_finish(upd->dp->i_mount, upd->ppargs); + upd->ppargs = NULL; + } + + if (upd->ip) { + if (upd->ip_locked) + xfs_iunlock(upd->ip, XFS_ILOCK_EXCL); + upd->ip_locked = false; + } + + if (upd->dp_locked) + xfs_iunlock(upd->dp, XFS_ILOCK_EXCL); + upd->dp_locked = false; +} + +/* + * Begin the process of creating a metadata file by allocating transactions + * and taking whatever resources we're going to need. + */ +static int +xfs_metadir_start_create( + struct xfs_metadir_update *upd) +{ + struct xfs_mount *mp = upd->dp->i_mount; + int error; + + ASSERT(upd->dp != NULL); + ASSERT(upd->ip == NULL); + ASSERT(xfs_has_metadir(mp)); + ASSERT(upd->metafile_type != XFS_METAFILE_UNKNOWN); + + error = xfs_parent_start(mp, &upd->ppargs); + if (error) + return error; + + /* + * If we ever need the ability to create rt metadata files on a + * pre-metadir filesystem, we'll need to dqattach the parent here. + * Currently we assume that mkfs will create the files and quotacheck + * will account for them. + */ + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_create, + xfs_create_space_res(mp, MAXNAMELEN), 0, 0, &upd->tp); + if (error) + goto out_teardown; + + /* + * Lock the parent directory if there is one. We can't ijoin it to + * the transaction until after the child file has been created. + */ + xfs_ilock(upd->dp, XFS_ILOCK_EXCL | XFS_ILOCK_PARENT); + upd->dp_locked = true; + + trace_xfs_metadir_start_create(upd); + return 0; +out_teardown: + xfs_metadir_teardown(upd, error); + return error; +} + +/* + * Create a metadata inode with the given @mode, and insert it into the + * metadata directory tree at the given @upd->path. The path up to the final + * component must already exist. The final path component must not exist. + * + * The new metadata inode will be attached to the update structure @upd->ip, + * with the ILOCK held until the caller releases it. + * + * NOTE: This function may return a new inode to the caller even if it returns + * a negative error code. If an inode is passed back, the caller must finish + * setting up the inode before releasing it. + */ +static int +xfs_metadir_create( + struct xfs_metadir_update *upd, + umode_t mode) +{ + struct xfs_icreate_args args = { + .pip = upd->dp, + .mode = mode, + }; + struct xfs_name xname; + struct xfs_dir_update du = { + .dp = upd->dp, + .name = &xname, + .ppargs = upd->ppargs, + }; + struct xfs_mount *mp = upd->dp->i_mount; + xfs_ino_t ino; + unsigned int resblks; + int error; + + xfs_assert_ilocked(upd->dp, XFS_ILOCK_EXCL); + + /* Check that the name does not already exist in the directory. */ + xfs_metadir_set_xname(&xname, upd->path, XFS_DIR3_FT_UNKNOWN); + error = xfs_metadir_lookup(upd->tp, upd->dp, &xname, &ino); + switch (error) { + case -ENOENT: + break; + case 0: + error = -EEXIST; + fallthrough; + default: + return error; + } + + /* + * A newly created regular or special file just has one directory + * entry pointing to them, but a directory also the "." entry + * pointing to itself. + */ + error = xfs_dialloc(&upd->tp, &args, &ino); + if (error) + return error; + error = xfs_icreate(upd->tp, ino, &args, &upd->ip); + if (error) + return error; + du.ip = upd->ip; + xfs_metafile_set_iflag(upd->tp, upd->ip, upd->metafile_type); + upd->ip_locked = true; + + /* + * Join the directory inode to the transaction. We do not do it + * earlier because xfs_dialloc rolls the transaction. + */ + xfs_trans_ijoin(upd->tp, upd->dp, 0); + + /* Create the entry. */ + if (S_ISDIR(args.mode)) + resblks = xfs_mkdir_space_res(mp, xname.len); + else + resblks = xfs_create_space_res(mp, xname.len); + xname.type = xfs_mode_to_ftype(args.mode); + + trace_xfs_metadir_try_create(upd); + + error = xfs_dir_create_child(upd->tp, resblks, &du); + if (error) + return error; + + /* Metadir files are not accounted to quota. */ + + trace_xfs_metadir_create(upd); + + return 0; +} + +#ifndef __KERNEL__ +/* + * Begin the process of linking a metadata file by allocating transactions + * and locking whatever resources we're going to need. + */ +int +xfs_metadir_start_link( + struct xfs_metadir_update *upd) +{ + struct xfs_mount *mp = upd->dp->i_mount; + unsigned int resblks; + int nospace_error = 0; + int error; + + ASSERT(upd->dp != NULL); + ASSERT(upd->ip != NULL); + ASSERT(xfs_has_metadir(mp)); + + error = xfs_parent_start(mp, &upd->ppargs); + if (error) + return error; + + resblks = xfs_link_space_res(mp, MAXNAMELEN); + error = xfs_trans_alloc_dir(upd->dp, &M_RES(mp)->tr_link, upd->ip, + &resblks, &upd->tp, &nospace_error); + if (error) + goto out_teardown; + if (!resblks) { + /* We don't allow reservationless updates. */ + xfs_trans_cancel(upd->tp); + upd->tp = NULL; + xfs_iunlock(upd->dp, XFS_ILOCK_EXCL); + xfs_iunlock(upd->ip, XFS_ILOCK_EXCL); + error = nospace_error; + goto out_teardown; + } + + upd->dp_locked = true; + upd->ip_locked = true; + + trace_xfs_metadir_start_link(upd); + return 0; +out_teardown: + xfs_metadir_teardown(upd, error); + return error; +} + +/* + * Link the metadata directory given by @path to the inode @upd->ip. + * The path (up to the final component) must already exist, but the final + * component must not already exist. + */ +int +xfs_metadir_link( + struct xfs_metadir_update *upd) +{ + struct xfs_name xname; + struct xfs_dir_update du = { + .dp = upd->dp, + .name = &xname, + .ip = upd->ip, + .ppargs = upd->ppargs, + }; + struct xfs_mount *mp = upd->dp->i_mount; + xfs_ino_t ino; + unsigned int resblks; + int error; + + xfs_assert_ilocked(upd->dp, XFS_ILOCK_EXCL); + xfs_assert_ilocked(upd->ip, XFS_ILOCK_EXCL); + + /* Look up the name in the current directory. */ + xfs_metadir_set_xname(&xname, upd->path, + xfs_mode_to_ftype(VFS_I(upd->ip)->i_mode)); + error = xfs_metadir_lookup(upd->tp, upd->dp, &xname, &ino); + switch (error) { + case -ENOENT: + break; + case 0: + error = -EEXIST; + fallthrough; + default: + return error; + } + + resblks = xfs_link_space_res(mp, xname.len); + error = xfs_dir_add_child(upd->tp, resblks, &du); + if (error) + return error; + + trace_xfs_metadir_link(upd); + + return 0; +} +#endif /* ! __KERNEL__ */ + +/* Commit a metadir update and unlock/drop all resources. */ +int +xfs_metadir_commit( + struct xfs_metadir_update *upd) +{ + int error; + + trace_xfs_metadir_commit(upd); + + error = xfs_trans_commit(upd->tp); + upd->tp = NULL; + + xfs_metadir_teardown(upd, error); + return error; +} + +/* Cancel a metadir update and unlock/drop all resources. */ +static void +xfs_metadir_cancel( + struct xfs_metadir_update *upd, + int error) +{ + trace_xfs_metadir_cancel(upd); + + xfs_trans_cancel(upd->tp); + upd->tp = NULL; + + xfs_metadir_teardown(upd, error); +} + +int +xfs_metadir_create_file( + struct xfs_metadir_update *upd, + umode_t mode, + xfs_metadir_createfn create, + void *priv, + struct xfs_inode **ipp) +{ + int error; + + if (xfs_is_shutdown(upd->dp->i_mount)) + return -EIO; + + error = xfs_metadir_start_create(upd); + if (error) + return error; + + error = xfs_metadir_create(upd, mode); + if (error) + goto out_cancel; + + if (create) { + error = create(upd, priv); + if (error) + goto out_cancel; + } + + error = xfs_metadir_commit(upd); + if (error) + goto out_irele; + + xfs_finish_inode_setup(upd->ip); + *ipp = upd->ip; + return 0; + +out_cancel: + xfs_metadir_cancel(upd, error); +out_irele: + /* Have to finish setting up the inode to ensure it's deleted. */ + if (upd->ip) { + xfs_finish_inode_setup(upd->ip); + xfs_irele(upd->ip); + } + return error; +} + +/* Create a metadata for the last component of the path. */ +int +xfs_metadir_mkdir( + struct xfs_inode *dp, + const char *path, + struct xfs_inode **ipp) +{ + struct xfs_metadir_update upd = { + .dp = dp, + .path = path, + .metafile_type = XFS_METAFILE_DIR, + }; + + return xfs_metadir_create_file(&upd, S_IFDIR, NULL, NULL, ipp); +} diff --git a/fs/xfs/libxfs/xfs_metadir.h b/fs/xfs/libxfs/xfs_metadir.h new file mode 100644 index 000000000..e434b9d1c --- /dev/null +++ b/fs/xfs/libxfs/xfs_metadir.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2018-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_METADIR_H__ +#define __XFS_METADIR_H__ + +/* Cleanup widget for metadata inode creation and deletion. */ +struct xfs_metadir_update { + /* Parent directory */ + struct xfs_inode *dp; + + /* Path to metadata file */ + const char *path; + + /* Parent pointer update context */ + struct xfs_parent_args *ppargs; + + /* Child metadata file */ + struct xfs_inode *ip; + + struct xfs_trans *tp; + + enum xfs_metafile_type metafile_type; + + unsigned int dp_locked:1; + unsigned int ip_locked:1; +}; + +int xfs_metadir_load(struct xfs_trans *tp, struct xfs_inode *dp, + const char *path, enum xfs_metafile_type metafile_type, + struct xfs_inode **ipp); + +typedef int (*xfs_metadir_createfn)(struct xfs_metadir_update *upd, void *priv); + +int xfs_metadir_create_file(struct xfs_metadir_update *upd, umode_t mode, + xfs_metadir_createfn create, void *priv, + struct xfs_inode **ipp); + +int xfs_metadir_start_link(struct xfs_metadir_update *upd); +int xfs_metadir_link(struct xfs_metadir_update *upd); + +int xfs_metadir_commit(struct xfs_metadir_update *upd); + +int xfs_metadir_mkdir(struct xfs_inode *dp, const char *path, + struct xfs_inode **ipp); + +#endif /* __XFS_METADIR_H__ */ diff --git a/fs/xfs/libxfs/xfs_metafile.c b/fs/xfs/libxfs/xfs_metafile.c new file mode 100644 index 000000000..71f004e9d --- /dev/null +++ b/fs/xfs/libxfs/xfs_metafile.c @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2018-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_sb.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_trans.h" +#include "xfs_metafile.h" +#include "xfs_trace.h" +#include "xfs_inode.h" +#include "xfs_quota.h" +#include "xfs_errortag.h" +#include "xfs_error.h" +#include "xfs_alloc.h" +#include "xfs_rtgroup.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_rtrefcount_btree.h" + +static const struct { + enum xfs_metafile_type mtype; + const char *name; +} xfs_metafile_type_strs[] = { XFS_METAFILE_TYPE_STR }; + +const char * +xfs_metafile_type_str(enum xfs_metafile_type metatype) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(xfs_metafile_type_strs); i++) { + if (xfs_metafile_type_strs[i].mtype == metatype) + return xfs_metafile_type_strs[i].name; + } + + return NULL; +} + +/* Set up an inode to be recognized as a metadata directory inode. */ +void +xfs_metafile_set_iflag( + struct xfs_trans *tp, + struct xfs_inode *ip, + enum xfs_metafile_type metafile_type) +{ + VFS_I(ip)->i_mode &= ~0777; + VFS_I(ip)->i_uid = GLOBAL_ROOT_UID; + VFS_I(ip)->i_gid = GLOBAL_ROOT_GID; + if (S_ISDIR(VFS_I(ip)->i_mode)) + ip->i_diflags |= XFS_METADIR_DIFLAGS; + else + ip->i_diflags |= XFS_METAFILE_DIFLAGS; + ip->i_diflags2 &= ~XFS_DIFLAG2_DAX; + ip->i_diflags2 |= XFS_DIFLAG2_METADATA; + ip->i_metatype = metafile_type; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + + XFS_STATS_DEC(ip->i_mount, xs_inodes_active); + XFS_STATS_INC(ip->i_mount, xs_inodes_meta); +} + +/* Clear the metadata directory inode flag. */ +void +xfs_metafile_clear_iflag( + struct xfs_trans *tp, + struct xfs_inode *ip) +{ + ASSERT(xfs_is_metadir_inode(ip)); + ASSERT(VFS_I(ip)->i_nlink == 0); + + ip->i_diflags2 &= ~XFS_DIFLAG2_METADATA; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + XFS_STATS_INC(ip->i_mount, xs_inodes_active); + XFS_STATS_DEC(ip->i_mount, xs_inodes_meta); +} + +/* + * Is the metafile reservations at or beneath a certain threshold? + */ +static inline bool +xfs_metafile_resv_can_cover( + struct xfs_mount *mp, + int64_t rhs) +{ + /* + * The amount of space that can be allocated to this metadata file is + * the remaining reservation for the particular metadata file + the + * global free block count. Take care of the first case to avoid + * touching the per-cpu counter. + */ + if (mp->m_metafile_resv_avail >= rhs) + return true; + + /* + * There aren't enough blocks left in the inode's reservation, but it + * isn't critical unless there also isn't enough free space. + */ + return xfs_compare_freecounter(mp, XC_FREE_BLOCKS, + rhs - mp->m_metafile_resv_avail, 2048) >= 0; +} + +/* + * Is the metafile reservation critically low on blocks? For now we'll define + * that as the number of blocks we can get our hands on being less than 10% of + * what we reserved or less than some arbitrary number (maximum btree height). + */ +bool +xfs_metafile_resv_critical( + struct xfs_mount *mp) +{ + ASSERT(xfs_has_metadir(mp)); + + trace_xfs_metafile_resv_critical(mp, 0); + + if (!xfs_metafile_resv_can_cover(mp, mp->m_rtbtree_maxlevels)) + return true; + + if (!xfs_metafile_resv_can_cover(mp, + div_u64(mp->m_metafile_resv_target, 10))) + return true; + + return XFS_TEST_ERROR(mp, XFS_ERRTAG_METAFILE_RESV_CRITICAL); +} + +/* Allocate a block from the metadata file's reservation. */ +void +xfs_metafile_resv_alloc_space( + struct xfs_inode *ip, + struct xfs_alloc_arg *args) +{ + struct xfs_mount *mp = ip->i_mount; + int64_t len = args->len; + + ASSERT(xfs_is_metadir_inode(ip)); + ASSERT(args->resv == XFS_AG_RESV_METAFILE); + + trace_xfs_metafile_resv_alloc_space(mp, args->len); + + /* + * Allocate the blocks from the metadata inode's block reservation + * and update the ondisk sb counter. + */ + mutex_lock(&mp->m_metafile_resv_lock); + if (mp->m_metafile_resv_avail > 0) { + int64_t from_resv; + + from_resv = min_t(int64_t, len, mp->m_metafile_resv_avail); + mp->m_metafile_resv_avail -= from_resv; + xfs_mod_delalloc(ip, 0, -from_resv); + xfs_trans_mod_sb(args->tp, XFS_TRANS_SB_RES_FDBLOCKS, + -from_resv); + len -= from_resv; + } + + /* + * Any allocation in excess of the reservation requires in-core and + * on-disk fdblocks updates. If we can grab @len blocks from the + * in-core fdblocks then all we need to do is update the on-disk + * superblock; if not, then try to steal some from the transaction's + * block reservation. Overruns are only expected for rmap btrees. + */ + if (len) { + unsigned int field; + int error; + + error = xfs_dec_fdblocks(ip->i_mount, len, true); + if (error) + field = XFS_TRANS_SB_FDBLOCKS; + else + field = XFS_TRANS_SB_RES_FDBLOCKS; + + xfs_trans_mod_sb(args->tp, field, -len); + } + + mp->m_metafile_resv_used += args->len; + mutex_unlock(&mp->m_metafile_resv_lock); + + ip->i_nblocks += args->len; + xfs_trans_log_inode(args->tp, ip, XFS_ILOG_CORE); +} + +/* Free a block to the metadata file's reservation. */ +void +xfs_metafile_resv_free_space( + struct xfs_inode *ip, + struct xfs_trans *tp, + xfs_filblks_t len) +{ + struct xfs_mount *mp = ip->i_mount; + int64_t to_resv; + + ASSERT(xfs_is_metadir_inode(ip)); + + trace_xfs_metafile_resv_free_space(mp, len); + + ip->i_nblocks -= len; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + + mutex_lock(&mp->m_metafile_resv_lock); + mp->m_metafile_resv_used -= len; + + /* + * Add the freed blocks back into the inode's delalloc reservation + * until it reaches the maximum size. Update the ondisk fdblocks only. + */ + to_resv = mp->m_metafile_resv_target - + (mp->m_metafile_resv_used + mp->m_metafile_resv_avail); + if (to_resv > 0) { + to_resv = min_t(int64_t, to_resv, len); + mp->m_metafile_resv_avail += to_resv; + xfs_mod_delalloc(ip, 0, to_resv); + xfs_trans_mod_sb(tp, XFS_TRANS_SB_RES_FDBLOCKS, to_resv); + len -= to_resv; + } + mutex_unlock(&mp->m_metafile_resv_lock); + + /* + * Everything else goes back to the filesystem, so update the in-core + * and on-disk counters. + */ + if (len) + xfs_trans_mod_sb(tp, XFS_TRANS_SB_FDBLOCKS, len); +} + +static void +__xfs_metafile_resv_free( + struct xfs_mount *mp) +{ + if (mp->m_metafile_resv_avail) { + xfs_mod_sb_delalloc(mp, -(int64_t)mp->m_metafile_resv_avail); + xfs_add_fdblocks(mp, mp->m_metafile_resv_avail); + } + mp->m_metafile_resv_avail = 0; + mp->m_metafile_resv_used = 0; + mp->m_metafile_resv_target = 0; +} + +/* Release unused metafile space reservation. */ +void +xfs_metafile_resv_free( + struct xfs_mount *mp) +{ + if (!xfs_has_metadir(mp)) + return; + + trace_xfs_metafile_resv_free(mp, 0); + + mutex_lock(&mp->m_metafile_resv_lock); + __xfs_metafile_resv_free(mp); + mutex_unlock(&mp->m_metafile_resv_lock); +} + +/* Set up a metafile space reservation. */ +int +xfs_metafile_resv_init( + struct xfs_mount *mp) +{ + struct xfs_rtgroup *rtg = NULL; + xfs_filblks_t used = 0, target = 0; + xfs_filblks_t hidden_space; + xfs_rfsblock_t dblocks_avail = mp->m_sb.sb_dblocks / 4; + int error = 0; + + if (!xfs_has_metadir(mp)) + return 0; + + /* + * Free any previous reservation to have a clean slate. + */ + mutex_lock(&mp->m_metafile_resv_lock); + __xfs_metafile_resv_free(mp); + + /* + * Currently the only btree metafiles that require reservations are the + * rtrmap and the rtrefcount. Anything new will have to be added here + * as well. + */ + while ((rtg = xfs_rtgroup_next(mp, rtg))) { + if (xfs_has_rtrmapbt(mp)) { + used += rtg_rmap(rtg)->i_nblocks; + target += xfs_rtrmapbt_calc_reserves(mp); + } + if (xfs_has_rtreflink(mp)) { + used += rtg_refcount(rtg)->i_nblocks; + target += xfs_rtrefcountbt_calc_reserves(mp); + } + } + + if (!target) + goto out_unlock; + + /* + * Space taken by the per-AG metadata btrees are accounted on-disk as + * used space. We therefore only hide the space that is reserved but + * not used by the trees. + */ + if (used > target) + target = used; + else if (target > dblocks_avail) + target = dblocks_avail; + hidden_space = target - used; + + error = xfs_dec_fdblocks(mp, hidden_space, true); + if (error) { + trace_xfs_metafile_resv_init_error(mp, 0); + goto out_unlock; + } + + xfs_mod_sb_delalloc(mp, hidden_space); + + mp->m_metafile_resv_target = target; + mp->m_metafile_resv_used = used; + mp->m_metafile_resv_avail = hidden_space; + + trace_xfs_metafile_resv_init(mp, target); + +out_unlock: + mutex_unlock(&mp->m_metafile_resv_lock); + return error; +} diff --git a/fs/xfs/libxfs/xfs_metafile.h b/fs/xfs/libxfs/xfs_metafile.h new file mode 100644 index 000000000..ae6f9e779 --- /dev/null +++ b/fs/xfs/libxfs/xfs_metafile.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2018-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_METAFILE_H__ +#define __XFS_METAFILE_H__ + +const char *xfs_metafile_type_str(enum xfs_metafile_type metatype); + +/* All metadata files must have these flags set. */ +#define XFS_METAFILE_DIFLAGS (XFS_DIFLAG_IMMUTABLE | \ + XFS_DIFLAG_SYNC | \ + XFS_DIFLAG_NOATIME | \ + XFS_DIFLAG_NODUMP | \ + XFS_DIFLAG_NODEFRAG) + +/* All metadata directories must have these flags set. */ +#define XFS_METADIR_DIFLAGS (XFS_METAFILE_DIFLAGS | \ + XFS_DIFLAG_NOSYMLINKS) + +void xfs_metafile_set_iflag(struct xfs_trans *tp, struct xfs_inode *ip, + enum xfs_metafile_type metafile_type); +void xfs_metafile_clear_iflag(struct xfs_trans *tp, struct xfs_inode *ip); + +/* Space reservations for metadata inodes. */ +struct xfs_alloc_arg; + +bool xfs_metafile_resv_critical(struct xfs_mount *mp); +void xfs_metafile_resv_alloc_space(struct xfs_inode *ip, + struct xfs_alloc_arg *args); +void xfs_metafile_resv_free_space(struct xfs_inode *ip, struct xfs_trans *tp, + xfs_filblks_t len); +void xfs_metafile_resv_free(struct xfs_mount *mp); +int xfs_metafile_resv_init(struct xfs_mount *mp); + +/* Code specific to kernel/userspace; must be provided externally. */ + +int xfs_trans_metafile_iget(struct xfs_trans *tp, xfs_ino_t ino, + enum xfs_metafile_type metafile_type, struct xfs_inode **ipp); +int xfs_metafile_iget(struct xfs_mount *mp, xfs_ino_t ino, + enum xfs_metafile_type metafile_type, struct xfs_inode **ipp); + +#endif /* __XFS_METAFILE_H__ */ diff --git a/fs/xfs/libxfs/xfs_ondisk.h b/fs/xfs/libxfs/xfs_ondisk.h new file mode 100644 index 000000000..23cde1248 --- /dev/null +++ b/fs/xfs/libxfs/xfs_ondisk.h @@ -0,0 +1,326 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2016 Oracle. + * All Rights Reserved. + */ +#ifndef __XFS_ONDISK_H +#define __XFS_ONDISK_H + +#define XFS_CHECK_STRUCT_SIZE(structname, size) \ + static_assert(sizeof(structname) == (size), \ + "XFS: sizeof(" #structname ") is wrong, expected " #size) + +#define XFS_CHECK_OFFSET(structname, member, off) \ + static_assert(offsetof(structname, member) == (off), \ + "XFS: offsetof(" #structname ", " #member ") is wrong, " \ + "expected " #off) + +#define XFS_CHECK_VALUE(value, expected) \ + static_assert((value) == (expected), \ + "XFS: value of " #value " is wrong, expected " #expected) + +#define XFS_CHECK_SB_OFFSET(field, offset) \ + XFS_CHECK_OFFSET(struct xfs_dsb, field, offset); \ + XFS_CHECK_OFFSET(struct xfs_sb, field, offset); + +static inline void __init +xfs_check_ondisk_structs(void) +{ + /* file structures */ + XFS_CHECK_STRUCT_SIZE(struct xfs_acl, 4); + XFS_CHECK_STRUCT_SIZE(struct xfs_acl_entry, 12); + XFS_CHECK_STRUCT_SIZE(struct xfs_bmbt_key, 8); + XFS_CHECK_STRUCT_SIZE(struct xfs_bmbt_rec, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_bmdr_block, 4); + XFS_CHECK_STRUCT_SIZE(struct xfs_dinode, 176); + XFS_CHECK_STRUCT_SIZE(struct xfs_disk_dquot, 104); + XFS_CHECK_STRUCT_SIZE(struct xfs_dqblk, 136); + XFS_CHECK_STRUCT_SIZE(struct xfs_dsymlink_hdr, 56); + XFS_CHECK_STRUCT_SIZE(xfs_timestamp_t, 8); + XFS_CHECK_STRUCT_SIZE(struct xfs_legacy_timestamp, 8); + + /* space btrees */ + XFS_CHECK_STRUCT_SIZE(struct xfs_agf, 224); + XFS_CHECK_STRUCT_SIZE(struct xfs_agfl, 36); + XFS_CHECK_STRUCT_SIZE(struct xfs_agi, 344); + XFS_CHECK_STRUCT_SIZE(struct xfs_alloc_rec, 8); + XFS_CHECK_STRUCT_SIZE(struct xfs_btree_block, 72); + XFS_CHECK_STRUCT_SIZE(struct xfs_btree_block_lhdr, 64); + XFS_CHECK_STRUCT_SIZE(struct xfs_btree_block_shdr, 48); + XFS_CHECK_STRUCT_SIZE(struct xfs_inobt_key, 4); + XFS_CHECK_STRUCT_SIZE(struct xfs_inobt_rec, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_refcount_key, 4); + XFS_CHECK_STRUCT_SIZE(struct xfs_refcount_rec, 12); + XFS_CHECK_STRUCT_SIZE(struct xfs_rmap_key, 20); + XFS_CHECK_STRUCT_SIZE(struct xfs_rmap_rec, 24); + XFS_CHECK_STRUCT_SIZE(xfs_alloc_key_t, 8); + XFS_CHECK_STRUCT_SIZE(xfs_alloc_ptr_t, 4); + XFS_CHECK_STRUCT_SIZE(xfs_inobt_ptr_t, 4); + XFS_CHECK_STRUCT_SIZE(xfs_refcount_ptr_t, 4); + XFS_CHECK_STRUCT_SIZE(xfs_rmap_ptr_t, 4); + XFS_CHECK_STRUCT_SIZE(xfs_bmdr_key_t, 8); + + /* dir/attr trees */ + XFS_CHECK_STRUCT_SIZE(struct xfs_attr3_leaf_hdr, 80); + XFS_CHECK_STRUCT_SIZE(struct xfs_attr3_leafblock, 80); + XFS_CHECK_STRUCT_SIZE(struct xfs_attr3_rmt_hdr, 56); + XFS_CHECK_STRUCT_SIZE(struct xfs_da3_blkinfo, 56); + XFS_CHECK_STRUCT_SIZE(struct xfs_da3_intnode, 64); + XFS_CHECK_STRUCT_SIZE(struct xfs_da3_node_hdr, 64); + XFS_CHECK_STRUCT_SIZE(struct xfs_dir3_blk_hdr, 48); + XFS_CHECK_STRUCT_SIZE(struct xfs_dir3_data_hdr, 64); + XFS_CHECK_STRUCT_SIZE(struct xfs_dir3_free, 64); + XFS_CHECK_STRUCT_SIZE(struct xfs_dir3_free_hdr, 64); + XFS_CHECK_STRUCT_SIZE(struct xfs_dir3_leaf, 64); + XFS_CHECK_STRUCT_SIZE(struct xfs_dir3_leaf_hdr, 64); + XFS_CHECK_STRUCT_SIZE(struct xfs_attr_leaf_entry, 8); + XFS_CHECK_STRUCT_SIZE(struct xfs_attr_leaf_hdr, 32); + XFS_CHECK_STRUCT_SIZE(struct xfs_attr_leaf_map, 4); + XFS_CHECK_STRUCT_SIZE(struct xfs_attr_leaf_name_local, 4); + + /* realtime structures */ + XFS_CHECK_STRUCT_SIZE(struct xfs_rtsb, 56); + XFS_CHECK_STRUCT_SIZE(union xfs_rtword_raw, 4); + XFS_CHECK_STRUCT_SIZE(union xfs_suminfo_raw, 4); + XFS_CHECK_STRUCT_SIZE(struct xfs_rtbuf_blkinfo, 48); + XFS_CHECK_STRUCT_SIZE(xfs_rtrmap_ptr_t, 8); + XFS_CHECK_STRUCT_SIZE(struct xfs_rtrmap_root, 4); + XFS_CHECK_STRUCT_SIZE(xfs_rtrefcount_ptr_t, 8); + XFS_CHECK_STRUCT_SIZE(struct xfs_rtrefcount_root, 4); + + /* + * m68k has problems with struct xfs_attr_leaf_name_remote, but we pad + * it to 4 bytes anyway so it's not obviously a problem. Hence for the + * moment we don't check this structure. This can be re-instated when + * the attr definitions are updated to use c99 VLA definitions. + * + XFS_CHECK_STRUCT_SIZE(struct xfs_attr_leaf_name_remote, 12); + */ + + XFS_CHECK_OFFSET(struct xfs_attr_leaf_name_local, valuelen, 0); + XFS_CHECK_OFFSET(struct xfs_attr_leaf_name_local, namelen, 2); + XFS_CHECK_OFFSET(struct xfs_attr_leaf_name_local, nameval, 3); + XFS_CHECK_OFFSET(struct xfs_attr_leaf_name_remote, valueblk, 0); + XFS_CHECK_OFFSET(struct xfs_attr_leaf_name_remote, valuelen, 4); + XFS_CHECK_OFFSET(struct xfs_attr_leaf_name_remote, namelen, 8); + XFS_CHECK_OFFSET(struct xfs_attr_leaf_name_remote, name, 9); + XFS_CHECK_STRUCT_SIZE(struct xfs_attr_leafblock, 32); + XFS_CHECK_STRUCT_SIZE(struct xfs_attr_sf_hdr, 4); + XFS_CHECK_OFFSET(struct xfs_attr_sf_hdr, totsize, 0); + XFS_CHECK_OFFSET(struct xfs_attr_sf_hdr, count, 2); + XFS_CHECK_OFFSET(struct xfs_attr_sf_entry, namelen, 0); + XFS_CHECK_OFFSET(struct xfs_attr_sf_entry, valuelen, 1); + XFS_CHECK_OFFSET(struct xfs_attr_sf_entry, flags, 2); + XFS_CHECK_OFFSET(struct xfs_attr_sf_entry, nameval, 3); + XFS_CHECK_STRUCT_SIZE(struct xfs_da_blkinfo, 12); + XFS_CHECK_STRUCT_SIZE(struct xfs_da_intnode, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_da_node_entry, 8); + XFS_CHECK_STRUCT_SIZE(struct xfs_da_node_hdr, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_dir2_data_free, 4); + XFS_CHECK_STRUCT_SIZE(struct xfs_dir2_data_hdr, 16); + XFS_CHECK_OFFSET(struct xfs_dir2_data_unused, freetag, 0); + XFS_CHECK_OFFSET(struct xfs_dir2_data_unused, length, 2); + XFS_CHECK_STRUCT_SIZE(struct xfs_dir2_free_hdr, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_dir2_free, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_dir2_leaf_entry, 8); + XFS_CHECK_STRUCT_SIZE(struct xfs_dir2_leaf_hdr, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_dir2_leaf, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_dir2_leaf_tail, 4); + XFS_CHECK_STRUCT_SIZE(struct xfs_dir2_sf_entry, 3); + XFS_CHECK_OFFSET(struct xfs_dir2_sf_entry, namelen, 0); + XFS_CHECK_OFFSET(struct xfs_dir2_sf_entry, offset, 1); + XFS_CHECK_OFFSET(struct xfs_dir2_sf_entry, name, 3); + XFS_CHECK_STRUCT_SIZE(struct xfs_dir2_sf_hdr, 10); + XFS_CHECK_STRUCT_SIZE(struct xfs_parent_rec, 12); + + /* ondisk dir/attr structures from xfs/122 */ + XFS_CHECK_STRUCT_SIZE(struct xfs_attr_sf_entry, 3); + XFS_CHECK_STRUCT_SIZE(struct xfs_dir2_data_free, 4); + XFS_CHECK_STRUCT_SIZE(struct xfs_dir2_data_unused, 6); + + /* log structures */ + XFS_CHECK_STRUCT_SIZE(struct xfs_buf_log_format, 88); + XFS_CHECK_STRUCT_SIZE(struct xfs_dq_logformat, 24); + XFS_CHECK_STRUCT_SIZE(struct xfs_efd_log_format_32, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_efd_log_format_64, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_efi_log_format_32, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_efi_log_format_64, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_extent_32, 12); + XFS_CHECK_STRUCT_SIZE(struct xfs_extent_64, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_log_dinode, 176); + XFS_CHECK_STRUCT_SIZE(struct xfs_icreate_log, 28); + XFS_CHECK_STRUCT_SIZE(xfs_log_timestamp_t, 8); + XFS_CHECK_STRUCT_SIZE(struct xfs_log_legacy_timestamp, 8); + XFS_CHECK_STRUCT_SIZE(struct xfs_inode_log_format_32, 52); + XFS_CHECK_STRUCT_SIZE(struct xfs_inode_log_format, 56); + XFS_CHECK_STRUCT_SIZE(struct xfs_qoff_logformat, 20); + XFS_CHECK_STRUCT_SIZE(struct xfs_trans_header, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_attri_log_format, 40); + XFS_CHECK_STRUCT_SIZE(struct xfs_attrd_log_format, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_bui_log_format, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_bud_log_format, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_cui_log_format, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_cud_log_format, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_rui_log_format, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_rud_log_format, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_map_extent, 32); + XFS_CHECK_STRUCT_SIZE(struct xfs_phys_extent, 16); + XFS_CHECK_STRUCT_SIZE(struct xlog_rec_header, 512); + XFS_CHECK_STRUCT_SIZE(struct xlog_rec_ext_header, 512); + + XFS_CHECK_OFFSET(struct xlog_rec_header, h_reserved, 328); + XFS_CHECK_OFFSET(struct xlog_rec_ext_header, xh_reserved, 260); + XFS_CHECK_OFFSET(struct xfs_bui_log_format, bui_extents, 16); + XFS_CHECK_OFFSET(struct xfs_cui_log_format, cui_extents, 16); + XFS_CHECK_OFFSET(struct xfs_rui_log_format, rui_extents, 16); + XFS_CHECK_OFFSET(struct xfs_efi_log_format, efi_extents, 16); + XFS_CHECK_OFFSET(struct xfs_efi_log_format_32, efi_extents, 16); + XFS_CHECK_OFFSET(struct xfs_efi_log_format_64, efi_extents, 16); + + /* ondisk log structures from xfs/122 */ + XFS_CHECK_STRUCT_SIZE(struct xfs_unmount_log_format, 8); + XFS_CHECK_STRUCT_SIZE(struct xfs_xmd_log_format, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_xmi_log_format, 88); + + /* parent pointer ioctls */ + XFS_CHECK_STRUCT_SIZE(struct xfs_getparents_rec, 32); + XFS_CHECK_STRUCT_SIZE(struct xfs_getparents, 40); + XFS_CHECK_STRUCT_SIZE(struct xfs_getparents_by_handle, 64); + + /* + * The v5 superblock format extended several v4 header structures with + * additional data. While new fields are only accessible on v5 + * superblocks, it's important that the v5 structures place original v4 + * fields/headers in the correct location on-disk. For example, we must + * be able to find magic values at the same location in certain blocks + * regardless of superblock version. + * + * The following checks ensure that various v5 data structures place the + * subset of v4 metadata associated with the same type of block at the + * start of the on-disk block. If there is no data structure definition + * for certain types of v4 blocks, traverse down to the first field of + * common metadata (e.g., magic value) and make sure it is at offset + * zero. + */ + XFS_CHECK_OFFSET(struct xfs_dir3_leaf, hdr.info.hdr, 0); + XFS_CHECK_OFFSET(struct xfs_da3_intnode, hdr.info.hdr, 0); + XFS_CHECK_OFFSET(struct xfs_dir3_data_hdr, hdr.magic, 0); + XFS_CHECK_OFFSET(struct xfs_dir3_free, hdr.hdr.magic, 0); + XFS_CHECK_OFFSET(struct xfs_attr3_leafblock, hdr.info.hdr, 0); + + /* + * Make sure the incore inode timestamp range corresponds to hand + * converted values based on the ondisk format specification. + */ + XFS_CHECK_VALUE(XFS_BIGTIME_TIME_MIN - XFS_BIGTIME_EPOCH_OFFSET, + XFS_LEGACY_TIME_MIN); + XFS_CHECK_VALUE(XFS_BIGTIME_TIME_MAX - XFS_BIGTIME_EPOCH_OFFSET, + 16299260424LL); + + /* Do the same with the incore quota expiration range. */ + XFS_CHECK_VALUE(XFS_DQ_BIGTIME_EXPIRY_MIN << XFS_DQ_BIGTIME_SHIFT, 4); + XFS_CHECK_VALUE(XFS_DQ_BIGTIME_EXPIRY_MAX << XFS_DQ_BIGTIME_SHIFT, + 16299260424LL); + + /* superblock field checks we got from xfs/122 */ + XFS_CHECK_STRUCT_SIZE(struct xfs_dsb, 304); + XFS_CHECK_STRUCT_SIZE(struct xfs_sb, 304); + XFS_CHECK_SB_OFFSET(sb_magicnum, 0); + XFS_CHECK_SB_OFFSET(sb_blocksize, 4); + XFS_CHECK_SB_OFFSET(sb_dblocks, 8); + XFS_CHECK_SB_OFFSET(sb_rblocks, 16); + XFS_CHECK_SB_OFFSET(sb_rextents, 24); + XFS_CHECK_SB_OFFSET(sb_uuid, 32); + XFS_CHECK_SB_OFFSET(sb_logstart, 48); + XFS_CHECK_SB_OFFSET(sb_rootino, 56); + XFS_CHECK_SB_OFFSET(sb_rbmino, 64); + XFS_CHECK_SB_OFFSET(sb_rsumino, 72); + XFS_CHECK_SB_OFFSET(sb_rextsize, 80); + XFS_CHECK_SB_OFFSET(sb_agblocks, 84); + XFS_CHECK_SB_OFFSET(sb_agcount, 88); + XFS_CHECK_SB_OFFSET(sb_rbmblocks, 92); + XFS_CHECK_SB_OFFSET(sb_logblocks, 96); + XFS_CHECK_SB_OFFSET(sb_versionnum, 100); + XFS_CHECK_SB_OFFSET(sb_sectsize, 102); + XFS_CHECK_SB_OFFSET(sb_inodesize, 104); + XFS_CHECK_SB_OFFSET(sb_inopblock, 106); + XFS_CHECK_SB_OFFSET(sb_blocklog, 120); + XFS_CHECK_SB_OFFSET(sb_fname[12], 120); + XFS_CHECK_SB_OFFSET(sb_sectlog, 121); + XFS_CHECK_SB_OFFSET(sb_inodelog, 122); + XFS_CHECK_SB_OFFSET(sb_inopblog, 123); + XFS_CHECK_SB_OFFSET(sb_agblklog, 124); + XFS_CHECK_SB_OFFSET(sb_rextslog, 125); + XFS_CHECK_SB_OFFSET(sb_inprogress, 126); + XFS_CHECK_SB_OFFSET(sb_imax_pct, 127); + XFS_CHECK_SB_OFFSET(sb_icount, 128); + XFS_CHECK_SB_OFFSET(sb_ifree, 136); + XFS_CHECK_SB_OFFSET(sb_fdblocks, 144); + XFS_CHECK_SB_OFFSET(sb_frextents, 152); + XFS_CHECK_SB_OFFSET(sb_uquotino, 160); + XFS_CHECK_SB_OFFSET(sb_gquotino, 168); + XFS_CHECK_SB_OFFSET(sb_qflags, 176); + XFS_CHECK_SB_OFFSET(sb_flags, 178); + XFS_CHECK_SB_OFFSET(sb_shared_vn, 179); + XFS_CHECK_SB_OFFSET(sb_inoalignmt, 180); + XFS_CHECK_SB_OFFSET(sb_unit, 184); + XFS_CHECK_SB_OFFSET(sb_width, 188); + XFS_CHECK_SB_OFFSET(sb_dirblklog, 192); + XFS_CHECK_SB_OFFSET(sb_logsectlog, 193); + XFS_CHECK_SB_OFFSET(sb_logsectsize, 194); + XFS_CHECK_SB_OFFSET(sb_logsunit, 196); + XFS_CHECK_SB_OFFSET(sb_features2, 200); + XFS_CHECK_SB_OFFSET(sb_bad_features2, 204); + XFS_CHECK_SB_OFFSET(sb_features_compat, 208); + XFS_CHECK_SB_OFFSET(sb_features_ro_compat, 212); + XFS_CHECK_SB_OFFSET(sb_features_incompat, 216); + XFS_CHECK_SB_OFFSET(sb_features_log_incompat, 220); + XFS_CHECK_SB_OFFSET(sb_crc, 224); + XFS_CHECK_SB_OFFSET(sb_spino_align, 228); + XFS_CHECK_SB_OFFSET(sb_pquotino, 232); + XFS_CHECK_SB_OFFSET(sb_lsn, 240); + XFS_CHECK_SB_OFFSET(sb_meta_uuid, 248); + XFS_CHECK_SB_OFFSET(sb_metadirino, 264); + XFS_CHECK_SB_OFFSET(sb_rgcount, 272); + XFS_CHECK_SB_OFFSET(sb_rgextents, 276); + XFS_CHECK_SB_OFFSET(sb_rgblklog, 280); + XFS_CHECK_SB_OFFSET(sb_pad, 281); + XFS_CHECK_SB_OFFSET(sb_rtstart, 288); + XFS_CHECK_SB_OFFSET(sb_rtreserved, 296); + + /* + * ioctl UABI + * + * Due to different padding/alignment requirements across + * different architectures, some structures are ommited from + * the size checks. In addition, structures with architecture + * dependent size fields are also ommited (e.g. __kernel_long_t). + */ + XFS_CHECK_STRUCT_SIZE(struct xfs_bulkstat, 192); + XFS_CHECK_STRUCT_SIZE(struct xfs_inumbers, 24); + XFS_CHECK_STRUCT_SIZE(struct xfs_bulkstat_req, 64); + XFS_CHECK_STRUCT_SIZE(struct xfs_inumbers_req, 64); + XFS_CHECK_STRUCT_SIZE(struct dioattr, 12); + XFS_CHECK_STRUCT_SIZE(struct getbmap, 32); + XFS_CHECK_STRUCT_SIZE(struct getbmapx, 48); + XFS_CHECK_STRUCT_SIZE(struct xfs_attrlist_cursor, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_attrlist, 8); + XFS_CHECK_STRUCT_SIZE(struct xfs_attrlist, 8); + XFS_CHECK_STRUCT_SIZE(struct xfs_attrlist_ent, 4); + XFS_CHECK_STRUCT_SIZE(struct xfs_ag_geometry, 128); + XFS_CHECK_STRUCT_SIZE(struct xfs_rtgroup_geometry, 128); + XFS_CHECK_STRUCT_SIZE(struct xfs_error_injection, 8); + XFS_CHECK_STRUCT_SIZE(struct xfs_fsop_geom, 256); + XFS_CHECK_STRUCT_SIZE(struct xfs_fsop_geom_v4, 112); + XFS_CHECK_STRUCT_SIZE(struct xfs_fsop_counts, 32); + XFS_CHECK_STRUCT_SIZE(struct xfs_fsop_resblks, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_growfs_log, 8); + XFS_CHECK_STRUCT_SIZE(struct xfs_bulk_ireq, 64); + XFS_CHECK_STRUCT_SIZE(struct xfs_fs_eofblocks, 128); + XFS_CHECK_STRUCT_SIZE(struct xfs_fsid, 8); + XFS_CHECK_STRUCT_SIZE(struct xfs_scrub_metadata, 64); + XFS_CHECK_STRUCT_SIZE(struct xfs_scrub_vec, 16); + XFS_CHECK_STRUCT_SIZE(struct xfs_scrub_vec_head, 40); +} + +#endif /* __XFS_ONDISK_H */ diff --git a/fs/xfs/libxfs/xfs_parent.c b/fs/xfs/libxfs/xfs_parent.c new file mode 100644 index 000000000..8d111c9b6 --- /dev/null +++ b/fs/xfs/libxfs/xfs_parent.c @@ -0,0 +1,380 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2022-2024 Oracle. + * All rights reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_format.h" +#include "xfs_da_format.h" +#include "xfs_log_format.h" +#include "xfs_shared.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_bmap_btree.h" +#include "xfs_inode.h" +#include "xfs_error.h" +#include "xfs_trace.h" +#include "xfs_trans.h" +#include "xfs_da_btree.h" +#include "xfs_attr.h" +#include "xfs_dir2.h" +#include "xfs_dir2_priv.h" +#include "xfs_attr_sf.h" +#include "xfs_bmap.h" +#include "xfs_defer.h" +#include "xfs_xattr.h" +#include "xfs_parent.h" +#include "xfs_trans_space.h" +#include "xfs_attr_item.h" +#include "xfs_health.h" +#include "xfs_attr_leaf.h" + +struct kmem_cache *xfs_parent_args_cache; + +/* + * Parent pointer attribute handling. + * + * Because the attribute name is a filename component, it will never be longer + * than 255 bytes and must not contain nulls or slashes. These are roughly the + * same constraints that apply to attribute names. + * + * The attribute value must always be a struct xfs_parent_rec. This means the + * attribute will never be in remote format because 12 bytes is nowhere near + * xfs_attr_leaf_entsize_local_max() (~75% of block size). + * + * Creating a new parent attribute will always create a new attribute - there + * should never, ever be an existing attribute in the tree for a new inode. + * ENOSPC behavior is problematic - creating the inode without the parent + * pointer is effectively a corruption, so we allow parent attribute creation + * to dip into the reserve block pool to avoid unexpected ENOSPC errors from + * occurring. + */ + +/* Return true if parent pointer attr name is valid. */ +bool +xfs_parent_namecheck( + unsigned int attr_flags, + const void *name, + size_t length) +{ + /* + * Parent pointers always use logged operations, so there should never + * be incomplete xattrs. + */ + if (attr_flags & XFS_ATTR_INCOMPLETE) + return false; + + return xfs_dir2_namecheck(name, length); +} + +/* Return true if parent pointer attr value is valid. */ +bool +xfs_parent_valuecheck( + struct xfs_mount *mp, + const void *value, + size_t valuelen) +{ + const struct xfs_parent_rec *rec = value; + + if (!xfs_has_parent(mp)) + return false; + + /* The xattr value must be a parent record. */ + if (valuelen != sizeof(struct xfs_parent_rec)) + return false; + + /* The parent record must be local. */ + if (value == NULL) + return false; + + /* The parent inumber must be valid. */ + if (!xfs_verify_dir_ino(mp, be64_to_cpu(rec->p_ino))) + return false; + + return true; +} + +/* Compute the attribute name hash for a parent pointer. */ +xfs_dahash_t +xfs_parent_hashval( + struct xfs_mount *mp, + const uint8_t *name, + int namelen, + xfs_ino_t parent_ino) +{ + struct xfs_name xname = { + .name = name, + .len = namelen, + }; + + /* + * Use the same dirent name hash as would be used on the directory, but + * mix in the parent inode number to avoid collisions on hardlinked + * files with identical names but different parents. + */ + return xfs_dir2_hashname(mp, &xname) ^ + upper_32_bits(parent_ino) ^ lower_32_bits(parent_ino); +} + +/* Compute the attribute name hash from the xattr components. */ +xfs_dahash_t +xfs_parent_hashattr( + struct xfs_mount *mp, + const uint8_t *name, + int namelen, + const void *value, + int valuelen) +{ + const struct xfs_parent_rec *rec = value; + + /* Requires a local attr value in xfs_parent_rec format */ + if (valuelen != sizeof(struct xfs_parent_rec)) { + ASSERT(valuelen == sizeof(struct xfs_parent_rec)); + return 0; + } + + if (!value) { + ASSERT(value != NULL); + return 0; + } + + return xfs_parent_hashval(mp, name, namelen, be64_to_cpu(rec->p_ino)); +} + +/* + * Initialize the parent pointer arguments structure. Caller must have zeroed + * the contents of @args. @tp is only required for updates. + */ +static void +xfs_parent_da_args_init( + struct xfs_da_args *args, + struct xfs_trans *tp, + struct xfs_parent_rec *rec, + struct xfs_inode *child, + xfs_ino_t owner, + const struct xfs_name *parent_name) +{ + args->geo = child->i_mount->m_attr_geo; + args->whichfork = XFS_ATTR_FORK; + args->attr_filter = XFS_ATTR_PARENT; + args->op_flags = XFS_DA_OP_LOGGED | XFS_DA_OP_OKNOENT; + args->trans = tp; + args->dp = child; + args->owner = owner; + args->name = parent_name->name; + args->namelen = parent_name->len; + args->value = rec; + args->valuelen = sizeof(struct xfs_parent_rec); + xfs_attr_sethash(args); +} + +/* Make sure the incore state is ready for a parent pointer query/update. */ +static inline int +xfs_parent_iread_extents( + struct xfs_trans *tp, + struct xfs_inode *child) +{ + /* Parent pointers require that the attr fork must exist. */ + if (XFS_IS_CORRUPT(child->i_mount, !xfs_inode_has_attr_fork(child))) { + xfs_inode_mark_sick(child, XFS_SICK_INO_PARENT); + return -EFSCORRUPTED; + } + + return xfs_iread_extents(tp, child, XFS_ATTR_FORK); +} + +/* Add a parent pointer to reflect a dirent addition. */ +int +xfs_parent_addname( + struct xfs_trans *tp, + struct xfs_parent_args *ppargs, + struct xfs_inode *dp, + const struct xfs_name *parent_name, + struct xfs_inode *child) +{ + int error; + + error = xfs_parent_iread_extents(tp, child); + if (error) + return error; + + xfs_inode_to_parent_rec(&ppargs->rec, dp); + xfs_parent_da_args_init(&ppargs->args, tp, &ppargs->rec, child, + I_INO(child), parent_name); + + return xfs_attr_setname(&ppargs->args, 0); +} + +/* Remove a parent pointer to reflect a dirent removal. */ +int +xfs_parent_removename( + struct xfs_trans *tp, + struct xfs_parent_args *ppargs, + struct xfs_inode *dp, + const struct xfs_name *parent_name, + struct xfs_inode *child) +{ + int error; + + error = xfs_parent_iread_extents(tp, child); + if (error) + return error; + + xfs_inode_to_parent_rec(&ppargs->rec, dp); + xfs_parent_da_args_init(&ppargs->args, tp, &ppargs->rec, child, + I_INO(child), parent_name); + + return xfs_attr_removename(&ppargs->args); +} + +/* Replace one parent pointer with another to reflect a rename. */ +int +xfs_parent_replacename( + struct xfs_trans *tp, + struct xfs_parent_args *ppargs, + struct xfs_inode *old_dp, + const struct xfs_name *old_name, + struct xfs_inode *new_dp, + const struct xfs_name *new_name, + struct xfs_inode *child) +{ + int error; + + error = xfs_parent_iread_extents(tp, child); + if (error) + return error; + + xfs_inode_to_parent_rec(&ppargs->rec, old_dp); + xfs_parent_da_args_init(&ppargs->args, tp, &ppargs->rec, child, + I_INO(child), old_name); + + xfs_inode_to_parent_rec(&ppargs->new_rec, new_dp); + + ppargs->args.new_name = new_name->name; + ppargs->args.new_namelen = new_name->len; + ppargs->args.new_value = &ppargs->new_rec; + ppargs->args.new_valuelen = sizeof(struct xfs_parent_rec); + + return xfs_attr_replacename(&ppargs->args, 0); +} + +/* + * Extract parent pointer information from any parent pointer xattr into + * @parent_ino/gen. The last two parameters can be NULL pointers. + * + * Returns 0 if this is not a parent pointer xattr at all; or -EFSCORRUPTED for + * garbage. + */ +int +xfs_parent_from_attr( + struct xfs_mount *mp, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + const void *value, + unsigned int valuelen, + xfs_ino_t *parent_ino, + uint32_t *parent_gen) +{ + const struct xfs_parent_rec *rec = value; + + ASSERT(attr_flags & XFS_ATTR_PARENT); + + if (!xfs_parent_namecheck(attr_flags, name, namelen)) + return -EFSCORRUPTED; + if (!xfs_parent_valuecheck(mp, value, valuelen)) + return -EFSCORRUPTED; + + if (parent_ino) + *parent_ino = be64_to_cpu(rec->p_ino); + if (parent_gen) + *parent_gen = be32_to_cpu(rec->p_gen); + return 0; +} + +/* + * Look up a parent pointer record (@parent_name -> @pptr) of @ip. + * + * Caller must hold at least ILOCK_SHARED. The scratchpad need not be + * initialized. + * + * Returns 0 if the pointer is found, -ENOATTR if there is no match, or a + * negative errno. + */ +int +xfs_parent_lookup( + struct xfs_trans *tp, + struct xfs_inode *ip, + const struct xfs_name *parent_name, + struct xfs_parent_rec *pptr, + struct xfs_da_args *scratch) +{ + memset(scratch, 0, sizeof(struct xfs_da_args)); + xfs_parent_da_args_init(scratch, tp, pptr, ip, I_INO(ip), parent_name); + return xfs_attr_get_ilocked(scratch); +} + +/* Sanity-check a parent pointer before we try to perform repairs. */ +static inline bool +xfs_parent_sanity_check( + struct xfs_mount *mp, + const struct xfs_name *parent_name, + const struct xfs_parent_rec *pptr) +{ + if (!xfs_parent_namecheck(XFS_ATTR_PARENT, parent_name->name, + parent_name->len)) + return false; + + if (!xfs_parent_valuecheck(mp, pptr, sizeof(*pptr))) + return false; + + return true; +} + + +/* + * Attach the parent pointer (@parent_name -> @pptr) to @ip immediately. + * Caller must not have a transaction or hold the ILOCK. This is for + * specialized repair functions only. The scratchpad need not be initialized. + */ +int +xfs_parent_set( + struct xfs_inode *ip, + xfs_ino_t owner, + const struct xfs_name *parent_name, + struct xfs_parent_rec *pptr, + struct xfs_da_args *scratch) +{ + if (!xfs_parent_sanity_check(ip->i_mount, parent_name, pptr)) { + ASSERT(0); + return -EFSCORRUPTED; + } + + memset(scratch, 0, sizeof(struct xfs_da_args)); + xfs_parent_da_args_init(scratch, NULL, pptr, ip, owner, parent_name); + return xfs_attr_set(scratch, XFS_ATTRUPDATE_CREATE, false); +} + +/* + * Remove the parent pointer (@parent_name -> @pptr) from @ip immediately. + * Caller must not have a transaction or hold the ILOCK. This is for + * specialized repair functions only. The scratchpad need not be initialized. + */ +int +xfs_parent_unset( + struct xfs_inode *ip, + xfs_ino_t owner, + const struct xfs_name *parent_name, + struct xfs_parent_rec *pptr, + struct xfs_da_args *scratch) +{ + if (!xfs_parent_sanity_check(ip->i_mount, parent_name, pptr)) { + ASSERT(0); + return -EFSCORRUPTED; + } + + memset(scratch, 0, sizeof(struct xfs_da_args)); + xfs_parent_da_args_init(scratch, NULL, pptr, ip, owner, parent_name); + return xfs_attr_set(scratch, XFS_ATTRUPDATE_REMOVE, false); +} diff --git a/fs/xfs/libxfs/xfs_parent.h b/fs/xfs/libxfs/xfs_parent.h new file mode 100644 index 000000000..8eb4de9c5 --- /dev/null +++ b/fs/xfs/libxfs/xfs_parent.h @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2022-2024 Oracle. + * All Rights Reserved. + */ +#ifndef __XFS_PARENT_H__ +#define __XFS_PARENT_H__ + +/* Metadata validators */ +bool xfs_parent_namecheck(unsigned int attr_flags, const void *name, + size_t length); +bool xfs_parent_valuecheck(struct xfs_mount *mp, const void *value, + size_t valuelen); + +xfs_dahash_t xfs_parent_hashval(struct xfs_mount *mp, const uint8_t *name, + int namelen, xfs_ino_t parent_ino); +xfs_dahash_t xfs_parent_hashattr(struct xfs_mount *mp, const uint8_t *name, + int namelen, const void *value, int valuelen); + +/* Initializes a xfs_parent_rec to be stored as an attribute name. */ +static inline void +xfs_parent_rec_init( + struct xfs_parent_rec *rec, + xfs_ino_t ino, + uint32_t gen) +{ + rec->p_ino = cpu_to_be64(ino); + rec->p_gen = cpu_to_be32(gen); +} + +/* Initializes a xfs_parent_rec to be stored as an attribute name. */ +static inline void +xfs_inode_to_parent_rec( + struct xfs_parent_rec *rec, + const struct xfs_inode *dp) +{ + xfs_parent_rec_init(rec, I_INO(dp), VFS_IC(dp)->i_generation); +} + +extern struct kmem_cache *xfs_parent_args_cache; + +/* + * Parent pointer information needed to pass around the deferred xattr update + * machinery. + */ +struct xfs_parent_args { + struct xfs_parent_rec rec; + struct xfs_parent_rec new_rec; + struct xfs_da_args args; +}; + +/* + * Start a parent pointer update by allocating the context object we need to + * perform a parent pointer update. + */ +static inline int +xfs_parent_start( + struct xfs_mount *mp, + struct xfs_parent_args **ppargsp) +{ + if (!xfs_has_parent(mp)) { + *ppargsp = NULL; + return 0; + } + + *ppargsp = kmem_cache_zalloc(xfs_parent_args_cache, GFP_KERNEL); + if (!*ppargsp) + return -ENOMEM; + return 0; +} + +/* Finish a parent pointer update by freeing the context object. */ +static inline void +xfs_parent_finish( + struct xfs_mount *mp, + struct xfs_parent_args *ppargs) +{ + if (ppargs) + kmem_cache_free(xfs_parent_args_cache, ppargs); +} + +int xfs_parent_addname(struct xfs_trans *tp, struct xfs_parent_args *ppargs, + struct xfs_inode *dp, const struct xfs_name *parent_name, + struct xfs_inode *child); +int xfs_parent_removename(struct xfs_trans *tp, struct xfs_parent_args *ppargs, + struct xfs_inode *dp, const struct xfs_name *parent_name, + struct xfs_inode *child); +int xfs_parent_replacename(struct xfs_trans *tp, + struct xfs_parent_args *ppargs, + struct xfs_inode *old_dp, const struct xfs_name *old_name, + struct xfs_inode *new_dp, const struct xfs_name *new_name, + struct xfs_inode *child); + +int xfs_parent_from_attr(struct xfs_mount *mp, unsigned int attr_flags, + const unsigned char *name, unsigned int namelen, + const void *value, unsigned int valuelen, + xfs_ino_t *parent_ino, uint32_t *parent_gen); + +/* Repair functions */ +int xfs_parent_lookup(struct xfs_trans *tp, struct xfs_inode *ip, + const struct xfs_name *name, struct xfs_parent_rec *pptr, + struct xfs_da_args *scratch); +int xfs_parent_set(struct xfs_inode *ip, xfs_ino_t owner, + const struct xfs_name *name, struct xfs_parent_rec *pptr, + struct xfs_da_args *scratch); +int xfs_parent_unset(struct xfs_inode *ip, xfs_ino_t owner, + const struct xfs_name *name, struct xfs_parent_rec *pptr, + struct xfs_da_args *scratch); + +#endif /* __XFS_PARENT_H__ */ diff --git a/fs/xfs/libxfs/xfs_quota_defs.h b/fs/xfs/libxfs/xfs_quota_defs.h new file mode 100644 index 000000000..551d7ae46 --- /dev/null +++ b/fs/xfs/libxfs/xfs_quota_defs.h @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_QUOTA_DEFS_H__ +#define __XFS_QUOTA_DEFS_H__ + +/* + * Quota definitions shared between user and kernel source trees. + */ + +/* + * Even though users may not have quota limits occupying all 64-bits, + * they may need 64-bit accounting. Hence, 64-bit quota-counters, + * and quota-limits. This is a waste in the common case, but hey ... + */ +typedef uint64_t xfs_qcnt_t; + +typedef uint8_t xfs_dqtype_t; + +#define XFS_DQTYPE_STRINGS \ + { XFS_DQTYPE_USER, "USER" }, \ + { XFS_DQTYPE_PROJ, "PROJ" }, \ + { XFS_DQTYPE_GROUP, "GROUP" }, \ + { XFS_DQTYPE_BIGTIME, "BIGTIME" } + +/* + * flags for q_flags field in the dquot. + */ +#define XFS_DQFLAG_DIRTY (1u << 0) /* dquot is dirty */ + +#define XFS_DQFLAG_STRINGS \ + { XFS_DQFLAG_DIRTY, "DIRTY" } + +/* + * We have the possibility of all three quota types being active at once, and + * hence free space modification requires modification of all three current + * dquots in a single transaction. For this case we need to have a reservation + * of at least 3 dquots. + * + * However, a chmod operation can change both UID and GID in a single + * transaction, resulting in requiring {old, new} x {uid, gid} dquots to be + * modified. Hence for this case we need to reserve space for at least 4 dquots. + * + * And in the worst case, there's a rename operation that can be modifying up to + * 4 inodes with dquots attached to them. In reality, the only inodes that can + * have their dquots modified are the source and destination directory inodes + * due to directory name creation and removal. That can require space allocation + * and/or freeing on both directory inodes, and hence all three dquots on each + * inode can be modified. And if the directories are world writeable, all the + * dquots can be unique and so 6 dquots can be modified.... + * + * And, of course, we also need to take into account the dquot log format item + * used to describe each dquot. + */ +#define XFS_DQUOT_LOGRES \ + ((sizeof(struct xfs_dq_logformat) + sizeof(struct xfs_disk_dquot)) * 6) + +#define XFS_IS_QUOTA_ON(mp) ((mp)->m_qflags & XFS_ALL_QUOTA_ACCT) +#define XFS_IS_UQUOTA_ON(mp) ((mp)->m_qflags & XFS_UQUOTA_ACCT) +#define XFS_IS_PQUOTA_ON(mp) ((mp)->m_qflags & XFS_PQUOTA_ACCT) +#define XFS_IS_GQUOTA_ON(mp) ((mp)->m_qflags & XFS_GQUOTA_ACCT) +#define XFS_IS_UQUOTA_ENFORCED(mp) ((mp)->m_qflags & XFS_UQUOTA_ENFD) +#define XFS_IS_GQUOTA_ENFORCED(mp) ((mp)->m_qflags & XFS_GQUOTA_ENFD) +#define XFS_IS_PQUOTA_ENFORCED(mp) ((mp)->m_qflags & XFS_PQUOTA_ENFD) + +/* + * Flags to tell various functions what to do. Not all of these are meaningful + * to a single function. None of these XFS_QMOPT_* flags are meant to have + * persistent values (ie. their values can and will change between versions) + */ +#define XFS_QMOPT_UQUOTA (1u << 0) /* user dquot requested */ +#define XFS_QMOPT_GQUOTA (1u << 1) /* group dquot requested */ +#define XFS_QMOPT_PQUOTA (1u << 2) /* project dquot requested */ +#define XFS_QMOPT_FORCE_RES (1u << 3) /* ignore quota limits */ +#define XFS_QMOPT_SBVERSION (1u << 4) /* change superblock version num */ + +/* + * flags to xfs_trans_mod_dquot to indicate which field needs to be + * modified. + */ +#define XFS_QMOPT_RES_REGBLKS (1u << 7) +#define XFS_QMOPT_RES_RTBLKS (1u << 8) +#define XFS_QMOPT_BCOUNT (1u << 9) +#define XFS_QMOPT_ICOUNT (1u << 10) +#define XFS_QMOPT_RTBCOUNT (1u << 11) +#define XFS_QMOPT_DELBCOUNT (1u << 12) +#define XFS_QMOPT_DELRTBCOUNT (1u << 13) +#define XFS_QMOPT_RES_INOS (1u << 14) + +/* + * flags for dqalloc. + */ +#define XFS_QMOPT_INHERIT (1u << 31) + +#define XFS_QMOPT_FLAGS \ + { XFS_QMOPT_UQUOTA, "UQUOTA" }, \ + { XFS_QMOPT_PQUOTA, "PQUOTA" }, \ + { XFS_QMOPT_FORCE_RES, "FORCE_RES" }, \ + { XFS_QMOPT_SBVERSION, "SBVERSION" }, \ + { XFS_QMOPT_GQUOTA, "GQUOTA" }, \ + { XFS_QMOPT_INHERIT, "INHERIT" }, \ + { XFS_QMOPT_RES_REGBLKS, "RES_REGBLKS" }, \ + { XFS_QMOPT_RES_RTBLKS, "RES_RTBLKS" }, \ + { XFS_QMOPT_BCOUNT, "BCOUNT" }, \ + { XFS_QMOPT_ICOUNT, "ICOUNT" }, \ + { XFS_QMOPT_RTBCOUNT, "RTBCOUNT" }, \ + { XFS_QMOPT_DELBCOUNT, "DELBCOUNT" }, \ + { XFS_QMOPT_DELRTBCOUNT, "DELRTBCOUNT" }, \ + { XFS_QMOPT_RES_INOS, "RES_INOS" } + +/* + * flags to xfs_trans_mod_dquot. + */ +#define XFS_TRANS_DQ_RES_BLKS XFS_QMOPT_RES_REGBLKS +#define XFS_TRANS_DQ_RES_RTBLKS XFS_QMOPT_RES_RTBLKS +#define XFS_TRANS_DQ_RES_INOS XFS_QMOPT_RES_INOS +#define XFS_TRANS_DQ_BCOUNT XFS_QMOPT_BCOUNT +#define XFS_TRANS_DQ_DELBCOUNT XFS_QMOPT_DELBCOUNT +#define XFS_TRANS_DQ_ICOUNT XFS_QMOPT_ICOUNT +#define XFS_TRANS_DQ_RTBCOUNT XFS_QMOPT_RTBCOUNT +#define XFS_TRANS_DQ_DELRTBCOUNT XFS_QMOPT_DELRTBCOUNT + + +#define XFS_QMOPT_QUOTALL \ + (XFS_QMOPT_UQUOTA | XFS_QMOPT_PQUOTA | XFS_QMOPT_GQUOTA) +#define XFS_QMOPT_RESBLK_MASK (XFS_QMOPT_RES_REGBLKS | XFS_QMOPT_RES_RTBLKS) + + +extern xfs_failaddr_t xfs_dquot_verify(struct xfs_mount *mp, + struct xfs_disk_dquot *ddq, xfs_dqid_t id); +extern xfs_failaddr_t xfs_dqblk_verify(struct xfs_mount *mp, + struct xfs_dqblk *dqb, xfs_dqid_t id); +extern int xfs_calc_dquots_per_chunk(unsigned int nbblks); +extern void xfs_dqblk_repair(struct xfs_mount *mp, struct xfs_dqblk *dqb, + xfs_dqid_t id, xfs_dqtype_t type); + +struct xfs_dquot; +time64_t xfs_dquot_from_disk_ts(struct xfs_disk_dquot *ddq, + __be32 dtimer); +__be32 xfs_dquot_to_disk_ts(struct xfs_dquot *ddq, time64_t timer); + +static inline const char * +xfs_dqinode_path(xfs_dqtype_t type) +{ + switch (type) { + case XFS_DQTYPE_USER: + return "user"; + case XFS_DQTYPE_GROUP: + return "group"; + case XFS_DQTYPE_PROJ: + return "project"; + } + + ASSERT(0); + return NULL; +} + +static inline enum xfs_metafile_type +xfs_dqinode_metafile_type(xfs_dqtype_t type) +{ + switch (type) { + case XFS_DQTYPE_USER: + return XFS_METAFILE_USRQUOTA; + case XFS_DQTYPE_GROUP: + return XFS_METAFILE_GRPQUOTA; + case XFS_DQTYPE_PROJ: + return XFS_METAFILE_PRJQUOTA; + } + + ASSERT(0); + return XFS_METAFILE_UNKNOWN; +} + +unsigned int xfs_dqinode_sick_mask(xfs_dqtype_t type); + +int xfs_dqinode_load(struct xfs_trans *tp, struct xfs_inode *dp, + xfs_dqtype_t type, struct xfs_inode **ipp); +int xfs_dqinode_metadir_create(struct xfs_inode *dp, xfs_dqtype_t type, + struct xfs_inode **ipp); +int xfs_dqinode_metadir_link(struct xfs_inode *dp, xfs_dqtype_t type, + struct xfs_inode *ip); +int xfs_dqinode_mkdir_parent(struct xfs_mount *mp, struct xfs_inode **dpp); +int xfs_dqinode_load_parent(struct xfs_trans *tp, struct xfs_inode **dpp); + +#endif /* __XFS_QUOTA_H__ */ diff --git a/fs/xfs/libxfs/xfs_refcount.c b/fs/xfs/libxfs/xfs_refcount.c new file mode 100644 index 000000000..0ec6ccd8b --- /dev/null +++ b/fs/xfs/libxfs/xfs_refcount.c @@ -0,0 +1,2244 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2016 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_btree.h" +#include "xfs_bmap.h" +#include "xfs_refcount_btree.h" +#include "xfs_alloc.h" +#include "xfs_errortag.h" +#include "xfs_error.h" +#include "xfs_trace.h" +#include "xfs_trans.h" +#include "xfs_bit.h" +#include "xfs_refcount.h" +#include "xfs_rmap.h" +#include "xfs_ag.h" +#include "xfs_health.h" +#include "xfs_refcount_item.h" +#include "xfs_rtgroup.h" +#include "xfs_rtalloc.h" +#include "xfs_rtrefcount_btree.h" + +struct kmem_cache *xfs_refcount_intent_cache; + +/* Allowable refcount adjustment amounts. */ +enum xfs_refc_adjust_op { + XFS_REFCOUNT_ADJUST_INCREASE = 1, + XFS_REFCOUNT_ADJUST_DECREASE = -1, + XFS_REFCOUNT_ADJUST_COW_ALLOC = 0, + XFS_REFCOUNT_ADJUST_COW_FREE = -1, +}; + +STATIC int __xfs_refcount_cow_alloc(struct xfs_btree_cur *rcur, + xfs_agblock_t agbno, xfs_extlen_t aglen); +STATIC int __xfs_refcount_cow_free(struct xfs_btree_cur *rcur, + xfs_agblock_t agbno, xfs_extlen_t aglen); + +/* + * Look up the first record less than or equal to [bno, len] in the btree + * given by cur. + */ +int +xfs_refcount_lookup_le( + struct xfs_btree_cur *cur, + enum xfs_refc_domain domain, + xfs_agblock_t bno, + int *stat) +{ + trace_xfs_refcount_lookup(cur, + xfs_refcount_encode_startblock(bno, domain), + XFS_LOOKUP_LE); + cur->bc_rec.rc.rc_startblock = bno; + cur->bc_rec.rc.rc_blockcount = 0; + cur->bc_rec.rc.rc_domain = domain; + return xfs_btree_lookup(cur, XFS_LOOKUP_LE, stat); +} + +/* + * Look up the first record greater than or equal to [bno, len] in the btree + * given by cur. + */ +int +xfs_refcount_lookup_ge( + struct xfs_btree_cur *cur, + enum xfs_refc_domain domain, + xfs_agblock_t bno, + int *stat) +{ + trace_xfs_refcount_lookup(cur, + xfs_refcount_encode_startblock(bno, domain), + XFS_LOOKUP_GE); + cur->bc_rec.rc.rc_startblock = bno; + cur->bc_rec.rc.rc_blockcount = 0; + cur->bc_rec.rc.rc_domain = domain; + return xfs_btree_lookup(cur, XFS_LOOKUP_GE, stat); +} + +/* + * Look up the first record equal to [bno, len] in the btree + * given by cur. + */ +int +xfs_refcount_lookup_eq( + struct xfs_btree_cur *cur, + enum xfs_refc_domain domain, + xfs_agblock_t bno, + int *stat) +{ + trace_xfs_refcount_lookup(cur, + xfs_refcount_encode_startblock(bno, domain), + XFS_LOOKUP_LE); + cur->bc_rec.rc.rc_startblock = bno; + cur->bc_rec.rc.rc_blockcount = 0; + cur->bc_rec.rc.rc_domain = domain; + return xfs_btree_lookup(cur, XFS_LOOKUP_EQ, stat); +} + +/* Convert on-disk record to in-core format. */ +void +xfs_refcount_btrec_to_irec( + const union xfs_btree_rec *rec, + struct xfs_refcount_irec *irec) +{ + uint32_t start; + + start = be32_to_cpu(rec->refc.rc_startblock); + if (start & XFS_REFC_COWFLAG) { + start &= ~XFS_REFC_COWFLAG; + irec->rc_domain = XFS_REFC_DOMAIN_COW; + } else { + irec->rc_domain = XFS_REFC_DOMAIN_SHARED; + } + + irec->rc_startblock = start; + irec->rc_blockcount = be32_to_cpu(rec->refc.rc_blockcount); + irec->rc_refcount = be32_to_cpu(rec->refc.rc_refcount); +} + +/* Simple checks for refcount records. */ +xfs_failaddr_t +xfs_refcount_check_irec( + struct xfs_perag *pag, + const struct xfs_refcount_irec *irec) +{ + if (irec->rc_blockcount == 0 || irec->rc_blockcount > XFS_REFC_LEN_MAX) + return __this_address; + + if (!xfs_refcount_check_domain(irec)) + return __this_address; + + /* check for valid extent range, including overflow */ + if (!xfs_verify_agbext(pag, irec->rc_startblock, irec->rc_blockcount)) + return __this_address; + + if (irec->rc_refcount == 0 || irec->rc_refcount > XFS_REFC_REFCOUNT_MAX) + return __this_address; + + return NULL; +} + +xfs_failaddr_t +xfs_rtrefcount_check_irec( + struct xfs_rtgroup *rtg, + const struct xfs_refcount_irec *irec) +{ + if (irec->rc_blockcount == 0 || irec->rc_blockcount > XFS_REFC_LEN_MAX) + return __this_address; + + if (!xfs_refcount_check_domain(irec)) + return __this_address; + + /* check for valid extent range, including overflow */ + if (!xfs_verify_rgbext(rtg, irec->rc_startblock, irec->rc_blockcount)) + return __this_address; + + if (irec->rc_refcount == 0 || irec->rc_refcount > XFS_REFC_REFCOUNT_MAX) + return __this_address; + + return NULL; +} + +static inline xfs_failaddr_t +xfs_refcount_check_btrec( + struct xfs_btree_cur *cur, + const struct xfs_refcount_irec *irec) +{ + if (xfs_btree_is_rtrefcount(cur->bc_ops)) + return xfs_rtrefcount_check_irec(to_rtg(cur->bc_group), irec); + return xfs_refcount_check_irec(to_perag(cur->bc_group), irec); +} + +static inline int +xfs_refcount_complain_bad_rec( + struct xfs_btree_cur *cur, + xfs_failaddr_t fa, + const struct xfs_refcount_irec *irec) +{ + struct xfs_mount *mp = cur->bc_mp; + + if (xfs_btree_is_rtrefcount(cur->bc_ops)) { + xfs_warn(mp, + "RT Refcount BTree record corruption in rtgroup %u detected at %pS!", + cur->bc_group->xg_gno, fa); + } else { + xfs_warn(mp, + "Refcount BTree record corruption in AG %d detected at %pS!", + cur->bc_group->xg_gno, fa); + } + xfs_warn(mp, + "Start block 0x%x, block count 0x%x, references 0x%x", + irec->rc_startblock, irec->rc_blockcount, irec->rc_refcount); + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; +} + +/* + * Get the data from the pointed-to record. + */ +int +xfs_refcount_get_rec( + struct xfs_btree_cur *cur, + struct xfs_refcount_irec *irec, + int *stat) +{ + union xfs_btree_rec *rec; + xfs_failaddr_t fa; + int error; + + error = xfs_btree_get_rec(cur, &rec, stat); + if (error || !*stat) + return error; + + xfs_refcount_btrec_to_irec(rec, irec); + fa = xfs_refcount_check_btrec(cur, irec); + if (fa) + return xfs_refcount_complain_bad_rec(cur, fa, irec); + + trace_xfs_refcount_get(cur, irec); + return 0; +} + +/* + * Update the record referred to by cur to the value given + * by [bno, len, refcount]. + * This either works (return 0) or gets an EFSCORRUPTED error. + */ +STATIC int +xfs_refcount_update( + struct xfs_btree_cur *cur, + struct xfs_refcount_irec *irec) +{ + union xfs_btree_rec rec; + uint32_t start; + int error; + + trace_xfs_refcount_update(cur, irec); + + start = xfs_refcount_encode_startblock(irec->rc_startblock, + irec->rc_domain); + rec.refc.rc_startblock = cpu_to_be32(start); + rec.refc.rc_blockcount = cpu_to_be32(irec->rc_blockcount); + rec.refc.rc_refcount = cpu_to_be32(irec->rc_refcount); + + error = xfs_btree_update(cur, &rec); + if (error) + trace_xfs_refcount_update_error(cur, error, _RET_IP_); + return error; +} + +/* + * Insert the record referred to by cur to the value given + * by [bno, len, refcount]. + * This either works (return 0) or gets an EFSCORRUPTED error. + */ +int +xfs_refcount_insert( + struct xfs_btree_cur *cur, + struct xfs_refcount_irec *irec, + int *i) +{ + int error; + + trace_xfs_refcount_insert(cur, irec); + + cur->bc_rec.rc.rc_startblock = irec->rc_startblock; + cur->bc_rec.rc.rc_blockcount = irec->rc_blockcount; + cur->bc_rec.rc.rc_refcount = irec->rc_refcount; + cur->bc_rec.rc.rc_domain = irec->rc_domain; + + error = xfs_btree_insert(cur, i); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, *i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + +out_error: + if (error) + trace_xfs_refcount_insert_error(cur, error, _RET_IP_); + return error; +} + +/* + * Remove the record referred to by cur, then set the pointer to the spot + * where the record could be re-inserted, in case we want to increment or + * decrement the cursor. + * This either works (return 0) or gets an EFSCORRUPTED error. + */ +STATIC int +xfs_refcount_delete( + struct xfs_btree_cur *cur, + int *i) +{ + struct xfs_refcount_irec irec; + int found_rec; + int error; + + error = xfs_refcount_get_rec(cur, &irec, &found_rec); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, found_rec != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + trace_xfs_refcount_delete(cur, &irec); + error = xfs_btree_delete(cur, i); + if (XFS_IS_CORRUPT(cur->bc_mp, *i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + if (error) + goto out_error; + error = xfs_refcount_lookup_ge(cur, irec.rc_domain, irec.rc_startblock, + &found_rec); +out_error: + if (error) + trace_xfs_refcount_delete_error(cur, error, _RET_IP_); + return error; +} + +/* + * Adjusting the Reference Count + * + * As stated elsewhere, the reference count btree (refcbt) stores + * >1 reference counts for extents of physical blocks. In this + * operation, we're either raising or lowering the reference count of + * some subrange stored in the tree: + * + * <------ adjustment range ------> + * ----+ +---+-----+ +--+--------+--------- + * 2 | | 3 | 4 | |17| 55 | 10 + * ----+ +---+-----+ +--+--------+--------- + * X axis is physical blocks number; + * reference counts are the numbers inside the rectangles + * + * The first thing we need to do is to ensure that there are no + * refcount extents crossing either boundary of the range to be + * adjusted. For any extent that does cross a boundary, split it into + * two extents so that we can increment the refcount of one of the + * pieces later: + * + * <------ adjustment range ------> + * ----+ +---+-----+ +--+--------+----+---- + * 2 | | 3 | 2 | |17| 55 | 10 | 10 + * ----+ +---+-----+ +--+--------+----+---- + * + * For this next step, let's assume that all the physical blocks in + * the adjustment range are mapped to a file and are therefore in use + * at least once. Therefore, we can infer that any gap in the + * refcount tree within the adjustment range represents a physical + * extent with refcount == 1: + * + * <------ adjustment range ------> + * ----+---+---+-----+-+--+--------+----+---- + * 2 |"1"| 3 | 2 |1|17| 55 | 10 | 10 + * ----+---+---+-----+-+--+--------+----+---- + * ^ + * + * For each extent that falls within the interval range, figure out + * which extent is to the left or the right of that extent. Now we + * have a left, current, and right extent. If the new reference count + * of the center extent enables us to merge left, center, and right + * into one record covering all three, do so. If the center extent is + * at the left end of the range, abuts the left extent, and its new + * reference count matches the left extent's record, then merge them. + * If the center extent is at the right end of the range, abuts the + * right extent, and the reference counts match, merge those. In the + * example, we can left merge (assuming an increment operation): + * + * <------ adjustment range ------> + * --------+---+-----+-+--+--------+----+---- + * 2 | 3 | 2 |1|17| 55 | 10 | 10 + * --------+---+-----+-+--+--------+----+---- + * ^ + * + * For all other extents within the range, adjust the reference count + * or delete it if the refcount falls below 2. If we were + * incrementing, the end result looks like this: + * + * <------ adjustment range ------> + * --------+---+-----+-+--+--------+----+---- + * 2 | 4 | 3 |2|18| 56 | 11 | 10 + * --------+---+-----+-+--+--------+----+---- + * + * The result of a decrement operation looks as such: + * + * <------ adjustment range ------> + * ----+ +---+ +--+--------+----+---- + * 2 | | 2 | |16| 54 | 9 | 10 + * ----+ +---+ +--+--------+----+---- + * DDDD 111111DD + * + * The blocks marked "D" are freed; the blocks marked "1" are only + * referenced once and therefore the record is removed from the + * refcount btree. + */ + +/* Next block after this extent. */ +static inline xfs_agblock_t +xfs_refc_next( + struct xfs_refcount_irec *rc) +{ + return rc->rc_startblock + rc->rc_blockcount; +} + +/* + * Split a refcount extent that crosses agbno. + */ +STATIC int +xfs_refcount_split_extent( + struct xfs_btree_cur *cur, + enum xfs_refc_domain domain, + xfs_agblock_t agbno, + bool *shape_changed) +{ + struct xfs_refcount_irec rcext, tmp; + int found_rec; + int error; + + *shape_changed = false; + error = xfs_refcount_lookup_le(cur, domain, agbno, &found_rec); + if (error) + goto out_error; + if (!found_rec) + return 0; + + error = xfs_refcount_get_rec(cur, &rcext, &found_rec); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, found_rec != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + if (rcext.rc_domain != domain) + return 0; + if (rcext.rc_startblock == agbno || xfs_refc_next(&rcext) <= agbno) + return 0; + + *shape_changed = true; + trace_xfs_refcount_split_extent(cur, &rcext, agbno); + + /* Establish the right extent. */ + tmp = rcext; + tmp.rc_startblock = agbno; + tmp.rc_blockcount -= (agbno - rcext.rc_startblock); + error = xfs_refcount_update(cur, &tmp); + if (error) + goto out_error; + + /* Insert the left extent. */ + tmp = rcext; + tmp.rc_blockcount = agbno - rcext.rc_startblock; + error = xfs_refcount_insert(cur, &tmp, &found_rec); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, found_rec != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + return error; + +out_error: + trace_xfs_refcount_split_extent_error(cur, error, _RET_IP_); + return error; +} + +/* + * Merge the left, center, and right extents. + */ +STATIC int +xfs_refcount_merge_center_extents( + struct xfs_btree_cur *cur, + struct xfs_refcount_irec *left, + struct xfs_refcount_irec *center, + struct xfs_refcount_irec *right, + unsigned long long extlen, + xfs_extlen_t *aglen) +{ + int error; + int found_rec; + + trace_xfs_refcount_merge_center_extents(cur, left, center, right); + + ASSERT(left->rc_domain == center->rc_domain); + ASSERT(right->rc_domain == center->rc_domain); + + /* + * Make sure the center and right extents are not in the btree. + * If the center extent was synthesized, the first delete call + * removes the right extent and we skip the second deletion. + * If center and right were in the btree, then the first delete + * call removes the center and the second one removes the right + * extent. + */ + error = xfs_refcount_lookup_ge(cur, center->rc_domain, + center->rc_startblock, &found_rec); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, found_rec != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + + error = xfs_refcount_delete(cur, &found_rec); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, found_rec != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + + if (center->rc_refcount > 1) { + error = xfs_refcount_delete(cur, &found_rec); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, found_rec != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + } + + /* Enlarge the left extent. */ + error = xfs_refcount_lookup_le(cur, left->rc_domain, + left->rc_startblock, &found_rec); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, found_rec != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + + left->rc_blockcount = extlen; + error = xfs_refcount_update(cur, left); + if (error) + goto out_error; + + *aglen = 0; + return error; + +out_error: + trace_xfs_refcount_merge_center_extents_error(cur, error, _RET_IP_); + return error; +} + +/* + * Merge with the left extent. + */ +STATIC int +xfs_refcount_merge_left_extent( + struct xfs_btree_cur *cur, + struct xfs_refcount_irec *left, + struct xfs_refcount_irec *cleft, + xfs_agblock_t *agbno, + xfs_extlen_t *aglen) +{ + int error; + int found_rec; + + trace_xfs_refcount_merge_left_extent(cur, left, cleft); + + ASSERT(left->rc_domain == cleft->rc_domain); + + /* If the extent at agbno (cleft) wasn't synthesized, remove it. */ + if (cleft->rc_refcount > 1) { + error = xfs_refcount_lookup_le(cur, cleft->rc_domain, + cleft->rc_startblock, &found_rec); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, found_rec != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + + error = xfs_refcount_delete(cur, &found_rec); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, found_rec != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + } + + /* Enlarge the left extent. */ + error = xfs_refcount_lookup_le(cur, left->rc_domain, + left->rc_startblock, &found_rec); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, found_rec != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + + left->rc_blockcount += cleft->rc_blockcount; + error = xfs_refcount_update(cur, left); + if (error) + goto out_error; + + *agbno += cleft->rc_blockcount; + *aglen -= cleft->rc_blockcount; + return error; + +out_error: + trace_xfs_refcount_merge_left_extent_error(cur, error, _RET_IP_); + return error; +} + +/* + * Merge with the right extent. + */ +STATIC int +xfs_refcount_merge_right_extent( + struct xfs_btree_cur *cur, + struct xfs_refcount_irec *right, + struct xfs_refcount_irec *cright, + xfs_extlen_t *aglen) +{ + int error; + int found_rec; + + trace_xfs_refcount_merge_right_extent(cur, cright, right); + + ASSERT(right->rc_domain == cright->rc_domain); + + /* + * If the extent ending at agbno+aglen (cright) wasn't synthesized, + * remove it. + */ + if (cright->rc_refcount > 1) { + error = xfs_refcount_lookup_le(cur, cright->rc_domain, + cright->rc_startblock, &found_rec); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, found_rec != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + + error = xfs_refcount_delete(cur, &found_rec); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, found_rec != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + } + + /* Enlarge the right extent. */ + error = xfs_refcount_lookup_le(cur, right->rc_domain, + right->rc_startblock, &found_rec); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, found_rec != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + + right->rc_startblock -= cright->rc_blockcount; + right->rc_blockcount += cright->rc_blockcount; + error = xfs_refcount_update(cur, right); + if (error) + goto out_error; + + *aglen -= cright->rc_blockcount; + return error; + +out_error: + trace_xfs_refcount_merge_right_extent_error(cur, error, _RET_IP_); + return error; +} + +/* + * Find the left extent and the one after it (cleft). This function assumes + * that we've already split any extent crossing agbno. + */ +STATIC int +xfs_refcount_find_left_extents( + struct xfs_btree_cur *cur, + struct xfs_refcount_irec *left, + struct xfs_refcount_irec *cleft, + enum xfs_refc_domain domain, + xfs_agblock_t agbno, + xfs_extlen_t aglen) +{ + struct xfs_refcount_irec tmp; + int error; + int found_rec; + + left->rc_startblock = cleft->rc_startblock = NULLAGBLOCK; + error = xfs_refcount_lookup_le(cur, domain, agbno - 1, &found_rec); + if (error) + goto out_error; + if (!found_rec) + return 0; + + error = xfs_refcount_get_rec(cur, &tmp, &found_rec); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, found_rec != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + + if (tmp.rc_domain != domain) + return 0; + if (xfs_refc_next(&tmp) != agbno) + return 0; + /* We have a left extent; retrieve (or invent) the next right one */ + *left = tmp; + + error = xfs_btree_increment(cur, 0, &found_rec); + if (error) + goto out_error; + if (found_rec) { + error = xfs_refcount_get_rec(cur, &tmp, &found_rec); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, found_rec != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + + if (tmp.rc_domain != domain) + goto not_found; + + /* if tmp starts at the end of our range, just use that */ + if (tmp.rc_startblock == agbno) + *cleft = tmp; + else { + /* + * There's a gap in the refcntbt at the start of the + * range we're interested in (refcount == 1) so + * synthesize the implied extent and pass it back. + * We assume here that the agbno/aglen range was + * passed in from a data fork extent mapping and + * therefore is allocated to exactly one owner. + */ + cleft->rc_startblock = agbno; + cleft->rc_blockcount = min(aglen, + tmp.rc_startblock - agbno); + cleft->rc_refcount = 1; + cleft->rc_domain = domain; + } + } else { +not_found: + /* + * No extents, so pretend that there's one covering the whole + * range. + */ + cleft->rc_startblock = agbno; + cleft->rc_blockcount = aglen; + cleft->rc_refcount = 1; + cleft->rc_domain = domain; + } + trace_xfs_refcount_find_left_extent(cur, left, cleft, agbno); + return error; + +out_error: + trace_xfs_refcount_find_left_extent_error(cur, error, _RET_IP_); + return error; +} + +/* + * Find the right extent and the one before it (cright). This function + * assumes that we've already split any extents crossing agbno + aglen. + */ +STATIC int +xfs_refcount_find_right_extents( + struct xfs_btree_cur *cur, + struct xfs_refcount_irec *right, + struct xfs_refcount_irec *cright, + enum xfs_refc_domain domain, + xfs_agblock_t agbno, + xfs_extlen_t aglen) +{ + struct xfs_refcount_irec tmp; + int error; + int found_rec; + + right->rc_startblock = cright->rc_startblock = NULLAGBLOCK; + error = xfs_refcount_lookup_ge(cur, domain, agbno + aglen, &found_rec); + if (error) + goto out_error; + if (!found_rec) + return 0; + + error = xfs_refcount_get_rec(cur, &tmp, &found_rec); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, found_rec != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + + if (tmp.rc_domain != domain) + return 0; + if (tmp.rc_startblock != agbno + aglen) + return 0; + /* We have a right extent; retrieve (or invent) the next left one */ + *right = tmp; + + error = xfs_btree_decrement(cur, 0, &found_rec); + if (error) + goto out_error; + if (found_rec) { + error = xfs_refcount_get_rec(cur, &tmp, &found_rec); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, found_rec != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + + if (tmp.rc_domain != domain) + goto not_found; + + /* if tmp ends at the end of our range, just use that */ + if (xfs_refc_next(&tmp) == agbno + aglen) + *cright = tmp; + else { + /* + * There's a gap in the refcntbt at the end of the + * range we're interested in (refcount == 1) so + * create the implied extent and pass it back. + * We assume here that the agbno/aglen range was + * passed in from a data fork extent mapping and + * therefore is allocated to exactly one owner. + */ + cright->rc_startblock = max(agbno, xfs_refc_next(&tmp)); + cright->rc_blockcount = right->rc_startblock - + cright->rc_startblock; + cright->rc_refcount = 1; + cright->rc_domain = domain; + } + } else { +not_found: + /* + * No extents, so pretend that there's one covering the whole + * range. + */ + cright->rc_startblock = agbno; + cright->rc_blockcount = aglen; + cright->rc_refcount = 1; + cright->rc_domain = domain; + } + trace_xfs_refcount_find_right_extent(cur, cright, right, + agbno + aglen); + return error; + +out_error: + trace_xfs_refcount_find_right_extent_error(cur, error, _RET_IP_); + return error; +} + +/* Is this extent valid? */ +static inline bool +xfs_refc_valid( + const struct xfs_refcount_irec *rc) +{ + return rc->rc_startblock != NULLAGBLOCK; +} + +static inline xfs_nlink_t +xfs_refc_merge_refcount( + const struct xfs_refcount_irec *irec, + enum xfs_refc_adjust_op adjust) +{ + /* Once a record hits XFS_REFC_REFCOUNT_MAX, it is pinned forever */ + if (irec->rc_refcount == XFS_REFC_REFCOUNT_MAX) + return XFS_REFC_REFCOUNT_MAX; + return irec->rc_refcount + adjust; +} + +static inline bool +xfs_refc_want_merge_center( + const struct xfs_refcount_irec *left, + const struct xfs_refcount_irec *cleft, + const struct xfs_refcount_irec *cright, + const struct xfs_refcount_irec *right, + bool cleft_is_cright, + enum xfs_refc_adjust_op adjust, + unsigned long long *ulenp) +{ + unsigned long long ulen = left->rc_blockcount; + xfs_nlink_t new_refcount; + + /* + * To merge with a center record, both shoulder records must be + * adjacent to the record we want to adjust. This is only true if + * find_left and find_right made all four records valid. + */ + if (!xfs_refc_valid(left) || !xfs_refc_valid(right) || + !xfs_refc_valid(cleft) || !xfs_refc_valid(cright)) + return false; + + /* There must only be one record for the entire range. */ + if (!cleft_is_cright) + return false; + + /* The shoulder record refcounts must match the new refcount. */ + new_refcount = xfs_refc_merge_refcount(cleft, adjust); + if (left->rc_refcount != new_refcount) + return false; + if (right->rc_refcount != new_refcount) + return false; + + /* + * The new record cannot exceed the max length. ulen is a ULL as the + * individual record block counts can be up to (u32 - 1) in length + * hence we need to catch u32 addition overflows here. + */ + ulen += cleft->rc_blockcount + right->rc_blockcount; + if (ulen >= XFS_REFC_LEN_MAX) + return false; + + *ulenp = ulen; + return true; +} + +static inline bool +xfs_refc_want_merge_left( + const struct xfs_refcount_irec *left, + const struct xfs_refcount_irec *cleft, + enum xfs_refc_adjust_op adjust) +{ + unsigned long long ulen = left->rc_blockcount; + xfs_nlink_t new_refcount; + + /* + * For a left merge, the left shoulder record must be adjacent to the + * start of the range. If this is true, find_left made left and cleft + * contain valid contents. + */ + if (!xfs_refc_valid(left) || !xfs_refc_valid(cleft)) + return false; + + /* Left shoulder record refcount must match the new refcount. */ + new_refcount = xfs_refc_merge_refcount(cleft, adjust); + if (left->rc_refcount != new_refcount) + return false; + + /* + * The new record cannot exceed the max length. ulen is a ULL as the + * individual record block counts can be up to (u32 - 1) in length + * hence we need to catch u32 addition overflows here. + */ + ulen += cleft->rc_blockcount; + if (ulen >= XFS_REFC_LEN_MAX) + return false; + + return true; +} + +static inline bool +xfs_refc_want_merge_right( + const struct xfs_refcount_irec *cright, + const struct xfs_refcount_irec *right, + enum xfs_refc_adjust_op adjust) +{ + unsigned long long ulen = right->rc_blockcount; + xfs_nlink_t new_refcount; + + /* + * For a right merge, the right shoulder record must be adjacent to the + * end of the range. If this is true, find_right made cright and right + * contain valid contents. + */ + if (!xfs_refc_valid(right) || !xfs_refc_valid(cright)) + return false; + + /* Right shoulder record refcount must match the new refcount. */ + new_refcount = xfs_refc_merge_refcount(cright, adjust); + if (right->rc_refcount != new_refcount) + return false; + + /* + * The new record cannot exceed the max length. ulen is a ULL as the + * individual record block counts can be up to (u32 - 1) in length + * hence we need to catch u32 addition overflows here. + */ + ulen += cright->rc_blockcount; + if (ulen >= XFS_REFC_LEN_MAX) + return false; + + return true; +} + +/* + * Try to merge with any extents on the boundaries of the adjustment range. + */ +STATIC int +xfs_refcount_merge_extents( + struct xfs_btree_cur *cur, + enum xfs_refc_domain domain, + xfs_agblock_t *agbno, + xfs_extlen_t *aglen, + enum xfs_refc_adjust_op adjust, + bool *shape_changed) +{ + struct xfs_refcount_irec left = {0}, cleft = {0}; + struct xfs_refcount_irec cright = {0}, right = {0}; + int error; + unsigned long long ulen; + bool cequal; + + *shape_changed = false; + /* + * Find the extent just below agbno [left], just above agbno [cleft], + * just below (agbno + aglen) [cright], and just above (agbno + aglen) + * [right]. + */ + error = xfs_refcount_find_left_extents(cur, &left, &cleft, domain, + *agbno, *aglen); + if (error) + return error; + error = xfs_refcount_find_right_extents(cur, &right, &cright, domain, + *agbno, *aglen); + if (error) + return error; + + /* No left or right extent to merge; exit. */ + if (!xfs_refc_valid(&left) && !xfs_refc_valid(&right)) + return 0; + + cequal = (cleft.rc_startblock == cright.rc_startblock) && + (cleft.rc_blockcount == cright.rc_blockcount); + + /* Try to merge left, cleft, and right. cleft must == cright. */ + if (xfs_refc_want_merge_center(&left, &cleft, &cright, &right, cequal, + adjust, &ulen)) { + *shape_changed = true; + return xfs_refcount_merge_center_extents(cur, &left, &cleft, + &right, ulen, aglen); + } + + /* Try to merge left and cleft. */ + if (xfs_refc_want_merge_left(&left, &cleft, adjust)) { + *shape_changed = true; + error = xfs_refcount_merge_left_extent(cur, &left, &cleft, + agbno, aglen); + if (error) + return error; + + /* + * If we just merged left + cleft and cleft == cright, + * we no longer have a cright to merge with right. We're done. + */ + if (cequal) + return 0; + } + + /* Try to merge cright and right. */ + if (xfs_refc_want_merge_right(&cright, &right, adjust)) { + *shape_changed = true; + return xfs_refcount_merge_right_extent(cur, &right, &cright, + aglen); + } + + return 0; +} + +/* + * XXX: This is a pretty hand-wavy estimate. The penalty for guessing + * true incorrectly is a shutdown FS; the penalty for guessing false + * incorrectly is more transaction rolls than might be necessary. + * Be conservative here. + */ +static bool +xfs_refcount_still_have_space( + struct xfs_btree_cur *cur) +{ + unsigned long overhead; + + /* + * Worst case estimate: full splits of the free space and rmap btrees + * to handle each of the shape changes to the refcount btree. + */ + overhead = xfs_allocfree_block_count(cur->bc_mp, + cur->bc_refc.shape_changes); + overhead += cur->bc_maxlevels; + overhead *= cur->bc_mp->m_sb.sb_blocksize; + + /* + * Only allow 2 refcount extent updates per transaction if the + * refcount continue update "error" has been injected. + */ + if (cur->bc_refc.nr_ops > 2 && + XFS_TEST_ERROR(cur->bc_mp, XFS_ERRTAG_REFCOUNT_CONTINUE_UPDATE)) + return false; + + if (cur->bc_refc.nr_ops == 0) + return true; + else if (overhead > cur->bc_tp->t_log_res) + return false; + return cur->bc_tp->t_log_res - overhead > + cur->bc_refc.nr_ops * XFS_REFCOUNT_ITEM_OVERHEAD; +} + +/* Schedule an extent free. */ +static int +xrefc_free_extent( + struct xfs_btree_cur *cur, + struct xfs_refcount_irec *rec) +{ + unsigned int flags = 0; + + if (xfs_btree_is_rtrefcount(cur->bc_ops)) + flags |= XFS_FREE_EXTENT_REALTIME; + + return xfs_free_extent_later(cur->bc_tp, + xfs_gbno_to_fsb(cur->bc_group, rec->rc_startblock), + rec->rc_blockcount, NULL, XFS_AG_RESV_NONE, flags); +} + +/* + * Adjust the refcounts of middle extents. At this point we should have + * split extents that crossed the adjustment range; merged with adjacent + * extents; and updated agbno/aglen to reflect the merges. Therefore, + * all we have to do is update the extents inside [agbno, agbno + aglen]. + */ +STATIC int +xfs_refcount_adjust_extents( + struct xfs_btree_cur *cur, + xfs_agblock_t *agbno, + xfs_extlen_t *aglen, + enum xfs_refc_adjust_op adj) +{ + struct xfs_refcount_irec ext, tmp; + int error; + int found_rec, found_tmp; + + /* Merging did all the work already. */ + if (*aglen == 0) + return 0; + + error = xfs_refcount_lookup_ge(cur, XFS_REFC_DOMAIN_SHARED, *agbno, + &found_rec); + if (error) + goto out_error; + + while (*aglen > 0 && xfs_refcount_still_have_space(cur)) { + error = xfs_refcount_get_rec(cur, &ext, &found_rec); + if (error) + goto out_error; + if (!found_rec || ext.rc_domain != XFS_REFC_DOMAIN_SHARED) { + ext.rc_startblock = xfs_group_max_blocks(cur->bc_group); + ext.rc_blockcount = 0; + ext.rc_refcount = 0; + ext.rc_domain = XFS_REFC_DOMAIN_SHARED; + } + + /* + * Deal with a hole in the refcount tree; if a file maps to + * these blocks and there's no refcountbt record, pretend that + * there is one with refcount == 1. + */ + if (ext.rc_startblock != *agbno) { + tmp.rc_startblock = *agbno; + tmp.rc_blockcount = min(*aglen, + ext.rc_startblock - *agbno); + tmp.rc_refcount = 1 + adj; + tmp.rc_domain = XFS_REFC_DOMAIN_SHARED; + + trace_xfs_refcount_modify_extent(cur, &tmp); + + /* + * Either cover the hole (increment) or + * delete the range (decrement). + */ + cur->bc_refc.nr_ops++; + if (tmp.rc_refcount) { + error = xfs_refcount_insert(cur, &tmp, + &found_tmp); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, + found_tmp != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + } else { + error = xrefc_free_extent(cur, &tmp); + if (error) + goto out_error; + } + + (*agbno) += tmp.rc_blockcount; + (*aglen) -= tmp.rc_blockcount; + + /* Stop if there's nothing left to modify */ + if (*aglen == 0 || !xfs_refcount_still_have_space(cur)) + break; + + /* Move the cursor to the start of ext. */ + error = xfs_refcount_lookup_ge(cur, + XFS_REFC_DOMAIN_SHARED, *agbno, + &found_rec); + if (error) + goto out_error; + } + + /* + * A previous step trimmed agbno/aglen such that the end of the + * range would not be in the middle of the record. If this is + * no longer the case, something is seriously wrong with the + * btree. Make sure we never feed the synthesized record into + * the processing loop below. + */ + if (XFS_IS_CORRUPT(cur->bc_mp, ext.rc_blockcount == 0) || + XFS_IS_CORRUPT(cur->bc_mp, ext.rc_blockcount > *aglen)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + + /* + * Adjust the reference count and either update the tree + * (incr) or free the blocks (decr). + */ + if (ext.rc_refcount == XFS_REFC_REFCOUNT_MAX) + goto skip; + ext.rc_refcount += adj; + trace_xfs_refcount_modify_extent(cur, &ext); + cur->bc_refc.nr_ops++; + if (ext.rc_refcount > 1) { + error = xfs_refcount_update(cur, &ext); + if (error) + goto out_error; + } else if (ext.rc_refcount == 1) { + error = xfs_refcount_delete(cur, &found_rec); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, found_rec != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + goto advloop; + } else { + error = xrefc_free_extent(cur, &ext); + if (error) + goto out_error; + } + +skip: + error = xfs_btree_increment(cur, 0, &found_rec); + if (error) + goto out_error; + +advloop: + (*agbno) += ext.rc_blockcount; + (*aglen) -= ext.rc_blockcount; + } + + return error; +out_error: + trace_xfs_refcount_modify_extent_error(cur, error, _RET_IP_); + return error; +} + +/* Adjust the reference count of a range of AG blocks. */ +STATIC int +xfs_refcount_adjust( + struct xfs_btree_cur *cur, + xfs_agblock_t *agbno, + xfs_extlen_t *aglen, + enum xfs_refc_adjust_op adj) +{ + bool shape_changed; + int shape_changes = 0; + int error; + + if (adj == XFS_REFCOUNT_ADJUST_INCREASE) + trace_xfs_refcount_increase(cur, *agbno, *aglen); + else + trace_xfs_refcount_decrease(cur, *agbno, *aglen); + + /* + * Ensure that no rcextents cross the boundary of the adjustment range. + */ + error = xfs_refcount_split_extent(cur, XFS_REFC_DOMAIN_SHARED, + *agbno, &shape_changed); + if (error) + goto out_error; + if (shape_changed) + shape_changes++; + + error = xfs_refcount_split_extent(cur, XFS_REFC_DOMAIN_SHARED, + *agbno + *aglen, &shape_changed); + if (error) + goto out_error; + if (shape_changed) + shape_changes++; + + /* + * Try to merge with the left or right extents of the range. + */ + error = xfs_refcount_merge_extents(cur, XFS_REFC_DOMAIN_SHARED, + agbno, aglen, adj, &shape_changed); + if (error) + goto out_error; + if (shape_changed) + shape_changes++; + if (shape_changes) + cur->bc_refc.shape_changes++; + + /* Now that we've taken care of the ends, adjust the middle extents */ + error = xfs_refcount_adjust_extents(cur, agbno, aglen, adj); + if (error) + goto out_error; + + return 0; + +out_error: + trace_xfs_refcount_adjust_error(cur, error, _RET_IP_); + return error; +} + +/* + * Set up a continuation a deferred refcount operation by updating the intent. + * Checks to make sure we're not going to run off the end of the AG. + */ +static inline int +xfs_refcount_continue_op( + struct xfs_btree_cur *cur, + struct xfs_refcount_intent *ri, + xfs_agblock_t new_agbno) +{ + struct xfs_mount *mp = cur->bc_mp; + struct xfs_perag *pag = to_perag(cur->bc_group); + + if (XFS_IS_CORRUPT(mp, !xfs_verify_agbext(pag, new_agbno, + ri->ri_blockcount))) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + + ri->ri_startblock = xfs_agbno_to_fsb(pag, new_agbno); + + ASSERT(xfs_verify_fsbext(mp, ri->ri_startblock, ri->ri_blockcount)); + ASSERT(pag_agno(pag) == XFS_FSB_TO_AGNO(mp, ri->ri_startblock)); + + return 0; +} + +/* + * Process one of the deferred refcount operations. We pass back the + * btree cursor to maintain our lock on the btree between calls. + * This saves time and eliminates a buffer deadlock between the + * superblock and the AGF because we'll always grab them in the same + * order. + */ +int +xfs_refcount_finish_one( + struct xfs_trans *tp, + struct xfs_refcount_intent *ri, + struct xfs_btree_cur **pcur) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_btree_cur *rcur = *pcur; + struct xfs_buf *agbp = NULL; + int error = 0; + xfs_agblock_t bno; + unsigned long nr_ops = 0; + int shape_changes = 0; + + bno = XFS_FSB_TO_AGBNO(mp, ri->ri_startblock); + + trace_xfs_refcount_deferred(mp, ri); + + if (XFS_TEST_ERROR(mp, XFS_ERRTAG_REFCOUNT_FINISH_ONE)) + return -EIO; + + /* + * If we haven't gotten a cursor or the cursor AG doesn't match + * the startblock, get one now. + */ + if (rcur != NULL && rcur->bc_group != ri->ri_group) { + nr_ops = rcur->bc_refc.nr_ops; + shape_changes = rcur->bc_refc.shape_changes; + xfs_btree_del_cursor(rcur, 0); + rcur = NULL; + *pcur = NULL; + } + if (rcur == NULL) { + struct xfs_perag *pag = to_perag(ri->ri_group); + + error = xfs_alloc_read_agf(pag, tp, 0, &agbp); + if (error) + return error; + + *pcur = rcur = xfs_refcountbt_init_cursor(mp, tp, agbp, pag); + rcur->bc_refc.nr_ops = nr_ops; + rcur->bc_refc.shape_changes = shape_changes; + } + + switch (ri->ri_type) { + case XFS_REFCOUNT_INCREASE: + error = xfs_refcount_adjust(rcur, &bno, &ri->ri_blockcount, + XFS_REFCOUNT_ADJUST_INCREASE); + if (error) + return error; + if (ri->ri_blockcount > 0) + error = xfs_refcount_continue_op(rcur, ri, bno); + break; + case XFS_REFCOUNT_DECREASE: + error = xfs_refcount_adjust(rcur, &bno, &ri->ri_blockcount, + XFS_REFCOUNT_ADJUST_DECREASE); + if (error) + return error; + if (ri->ri_blockcount > 0) + error = xfs_refcount_continue_op(rcur, ri, bno); + break; + case XFS_REFCOUNT_ALLOC_COW: + error = __xfs_refcount_cow_alloc(rcur, bno, ri->ri_blockcount); + if (error) + return error; + ri->ri_blockcount = 0; + break; + case XFS_REFCOUNT_FREE_COW: + error = __xfs_refcount_cow_free(rcur, bno, ri->ri_blockcount); + if (error) + return error; + ri->ri_blockcount = 0; + break; + default: + ASSERT(0); + return -EFSCORRUPTED; + } + if (!error && ri->ri_blockcount > 0) + trace_xfs_refcount_finish_one_leftover(mp, ri); + return error; +} + +/* + * Set up a continuation a deferred rtrefcount operation by updating the + * intent. Checks to make sure we're not going to run off the end of the + * rtgroup. + */ +static inline int +xfs_rtrefcount_continue_op( + struct xfs_btree_cur *cur, + struct xfs_refcount_intent *ri, + xfs_agblock_t new_agbno) +{ + struct xfs_mount *mp = cur->bc_mp; + struct xfs_rtgroup *rtg = to_rtg(ri->ri_group); + + if (XFS_IS_CORRUPT(mp, !xfs_verify_rgbext(rtg, new_agbno, + ri->ri_blockcount))) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + + ri->ri_startblock = xfs_rgbno_to_rtb(rtg, new_agbno); + + ASSERT(xfs_verify_rtbext(mp, ri->ri_startblock, ri->ri_blockcount)); + return 0; +} + +/* + * Process one of the deferred realtime refcount operations. We pass back the + * btree cursor to maintain our lock on the btree between calls. + */ +int +xfs_rtrefcount_finish_one( + struct xfs_trans *tp, + struct xfs_refcount_intent *ri, + struct xfs_btree_cur **pcur) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_rtgroup *rtg = to_rtg(ri->ri_group); + struct xfs_btree_cur *rcur = *pcur; + int error = 0; + xfs_rgblock_t bno; + unsigned long nr_ops = 0; + int shape_changes = 0; + + bno = xfs_rtb_to_rgbno(mp, ri->ri_startblock); + + trace_xfs_refcount_deferred(mp, ri); + + if (XFS_TEST_ERROR(mp, XFS_ERRTAG_REFCOUNT_FINISH_ONE)) + return -EIO; + + /* + * If we haven't gotten a cursor or the cursor AG doesn't match + * the startblock, get one now. + */ + if (rcur != NULL && rcur->bc_group != ri->ri_group) { + nr_ops = rcur->bc_refc.nr_ops; + shape_changes = rcur->bc_refc.shape_changes; + xfs_btree_del_cursor(rcur, 0); + rcur = NULL; + *pcur = NULL; + } + if (rcur == NULL) { + xfs_rtgroup_lock(rtg, XFS_RTGLOCK_REFCOUNT); + xfs_rtgroup_trans_join(tp, rtg, XFS_RTGLOCK_REFCOUNT); + *pcur = rcur = xfs_rtrefcountbt_init_cursor(tp, rtg); + + rcur->bc_refc.nr_ops = nr_ops; + rcur->bc_refc.shape_changes = shape_changes; + } + + switch (ri->ri_type) { + case XFS_REFCOUNT_INCREASE: + error = xfs_refcount_adjust(rcur, &bno, &ri->ri_blockcount, + XFS_REFCOUNT_ADJUST_INCREASE); + if (error) + return error; + if (ri->ri_blockcount > 0) + error = xfs_rtrefcount_continue_op(rcur, ri, bno); + break; + case XFS_REFCOUNT_DECREASE: + error = xfs_refcount_adjust(rcur, &bno, &ri->ri_blockcount, + XFS_REFCOUNT_ADJUST_DECREASE); + if (error) + return error; + if (ri->ri_blockcount > 0) + error = xfs_rtrefcount_continue_op(rcur, ri, bno); + break; + case XFS_REFCOUNT_ALLOC_COW: + error = __xfs_refcount_cow_alloc(rcur, bno, ri->ri_blockcount); + if (error) + return error; + ri->ri_blockcount = 0; + break; + case XFS_REFCOUNT_FREE_COW: + error = __xfs_refcount_cow_free(rcur, bno, ri->ri_blockcount); + if (error) + return error; + ri->ri_blockcount = 0; + break; + default: + ASSERT(0); + return -EFSCORRUPTED; + } + if (!error && ri->ri_blockcount > 0) + trace_xfs_refcount_finish_one_leftover(mp, ri); + return error; +} + +/* + * Record a refcount intent for later processing. + */ +static void +__xfs_refcount_add( + struct xfs_trans *tp, + enum xfs_refcount_intent_type type, + bool isrt, + xfs_fsblock_t startblock, + xfs_extlen_t blockcount) +{ + struct xfs_refcount_intent *ri; + + ri = kmem_cache_alloc(xfs_refcount_intent_cache, + GFP_KERNEL | __GFP_NOFAIL); + INIT_LIST_HEAD(&ri->ri_list); + ri->ri_type = type; + ri->ri_startblock = startblock; + ri->ri_blockcount = blockcount; + ri->ri_realtime = isrt; + + xfs_refcount_defer_add(tp, ri); +} + +/* + * Increase the reference count of the blocks backing a file's extent. + */ +void +xfs_refcount_increase_extent( + struct xfs_trans *tp, + bool isrt, + struct xfs_bmbt_irec *PREV) +{ + if (!xfs_has_reflink(tp->t_mountp)) + return; + + __xfs_refcount_add(tp, XFS_REFCOUNT_INCREASE, isrt, PREV->br_startblock, + PREV->br_blockcount); +} + +/* + * Decrease the reference count of the blocks backing a file's extent. + */ +void +xfs_refcount_decrease_extent( + struct xfs_trans *tp, + bool isrt, + struct xfs_bmbt_irec *PREV) +{ + if (!xfs_has_reflink(tp->t_mountp)) + return; + + __xfs_refcount_add(tp, XFS_REFCOUNT_DECREASE, isrt, PREV->br_startblock, + PREV->br_blockcount); +} + +/* + * Given an AG extent, find the lowest-numbered run of shared blocks + * within that range and return the range in fbno/flen. If + * find_end_of_shared is set, return the longest contiguous extent of + * shared blocks; if not, just return the first extent we find. If no + * shared blocks are found, fbno and flen will be set to NULLAGBLOCK + * and 0, respectively. + */ +int +xfs_refcount_find_shared( + struct xfs_btree_cur *cur, + xfs_agblock_t agbno, + xfs_extlen_t aglen, + xfs_agblock_t *fbno, + xfs_extlen_t *flen, + bool find_end_of_shared) +{ + struct xfs_refcount_irec tmp; + int i; + int have; + int error; + + trace_xfs_refcount_find_shared(cur, agbno, aglen); + + /* By default, skip the whole range */ + *fbno = NULLAGBLOCK; + *flen = 0; + + /* Try to find a refcount extent that crosses the start */ + error = xfs_refcount_lookup_le(cur, XFS_REFC_DOMAIN_SHARED, agbno, + &have); + if (error) + goto out_error; + if (!have) { + /* No left extent, look at the next one */ + error = xfs_btree_increment(cur, 0, &have); + if (error) + goto out_error; + if (!have) + goto done; + } + error = xfs_refcount_get_rec(cur, &tmp, &i); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + if (tmp.rc_domain != XFS_REFC_DOMAIN_SHARED) + goto done; + + /* If the extent ends before the start, look at the next one */ + if (tmp.rc_startblock + tmp.rc_blockcount <= agbno) { + error = xfs_btree_increment(cur, 0, &have); + if (error) + goto out_error; + if (!have) + goto done; + error = xfs_refcount_get_rec(cur, &tmp, &i); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + if (tmp.rc_domain != XFS_REFC_DOMAIN_SHARED) + goto done; + } + + /* If the extent starts after the range we want, bail out */ + if (tmp.rc_startblock >= agbno + aglen) + goto done; + + /* We found the start of a shared extent! */ + if (tmp.rc_startblock < agbno) { + tmp.rc_blockcount -= (agbno - tmp.rc_startblock); + tmp.rc_startblock = agbno; + } + + *fbno = tmp.rc_startblock; + *flen = min(tmp.rc_blockcount, agbno + aglen - *fbno); + if (!find_end_of_shared) + goto done; + + /* Otherwise, find the end of this shared extent */ + while (*fbno + *flen < agbno + aglen) { + error = xfs_btree_increment(cur, 0, &have); + if (error) + goto out_error; + if (!have) + break; + error = xfs_refcount_get_rec(cur, &tmp, &i); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + if (tmp.rc_domain != XFS_REFC_DOMAIN_SHARED || + tmp.rc_startblock >= agbno + aglen || + tmp.rc_startblock != *fbno + *flen) + break; + *flen = min(*flen + tmp.rc_blockcount, agbno + aglen - *fbno); + } + +done: + trace_xfs_refcount_find_shared_result(cur, *fbno, *flen); + +out_error: + if (error) + trace_xfs_refcount_find_shared_error(cur, error, _RET_IP_); + return error; +} + +/* + * Recovering CoW Blocks After a Crash + * + * Due to the way that the copy on write mechanism works, there's a window of + * opportunity in which we can lose track of allocated blocks during a crash. + * Because CoW uses delayed allocation in the in-core CoW fork, writeback + * causes blocks to be allocated and stored in the CoW fork. The blocks are + * no longer in the free space btree but are not otherwise recorded anywhere + * until the write completes and the blocks are mapped into the file. A crash + * in between allocation and remapping results in the replacement blocks being + * lost. This situation is exacerbated by the CoW extent size hint because + * allocations can hang around for long time. + * + * However, there is a place where we can record these allocations before they + * become mappings -- the reference count btree. The btree does not record + * extents with refcount == 1, so we can record allocations with a refcount of + * 1. Blocks being used for CoW writeout cannot be shared, so there should be + * no conflict with shared block records. These mappings should be created + * when we allocate blocks to the CoW fork and deleted when they're removed + * from the CoW fork. + * + * Minor nit: records for in-progress CoW allocations and records for shared + * extents must never be merged, to preserve the property that (except for CoW + * allocations) there are no refcount btree entries with refcount == 1. The + * only time this could potentially happen is when unsharing a block that's + * adjacent to CoW allocations, so we must be careful to avoid this. + * + * At mount time we recover lost CoW allocations by searching the refcount + * btree for these refcount == 1 mappings. These represent CoW allocations + * that were in progress at the time the filesystem went down, so we can free + * them to get the space back. + * + * This mechanism is superior to creating EFIs for unmapped CoW extents for + * several reasons -- first, EFIs pin the tail of the log and would have to be + * periodically relogged to avoid filling up the log. Second, CoW completions + * will have to file an EFD and create new EFIs for whatever remains in the + * CoW fork; this partially takes care of (1) but extent-size reservations + * will have to periodically relog even if there's no writeout in progress. + * This can happen if the CoW extent size hint is set, which you really want. + * Third, EFIs cannot currently be automatically relogged into newer + * transactions to advance the log tail. Fourth, stuffing the log full of + * EFIs places an upper bound on the number of CoW allocations that can be + * held filesystem-wide at any given time. Recording them in the refcount + * btree doesn't require us to maintain any state in memory and doesn't pin + * the log. + */ +/* + * Adjust the refcounts of CoW allocations. These allocations are "magic" + * in that they're not referenced anywhere else in the filesystem, so we + * stash them in the refcount btree with a refcount of 1 until either file + * remapping (or CoW cancellation) happens. + */ +STATIC int +xfs_refcount_adjust_cow_extents( + struct xfs_btree_cur *cur, + xfs_agblock_t agbno, + xfs_extlen_t aglen, + enum xfs_refc_adjust_op adj) +{ + struct xfs_refcount_irec ext, tmp; + int error; + int found_rec, found_tmp; + + if (aglen == 0) + return 0; + + /* Find any overlapping refcount records */ + error = xfs_refcount_lookup_ge(cur, XFS_REFC_DOMAIN_COW, agbno, + &found_rec); + if (error) + goto out_error; + error = xfs_refcount_get_rec(cur, &ext, &found_rec); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, found_rec && + ext.rc_domain != XFS_REFC_DOMAIN_COW)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + if (!found_rec) { + ext.rc_startblock = xfs_group_max_blocks(cur->bc_group); + ext.rc_blockcount = 0; + ext.rc_refcount = 0; + ext.rc_domain = XFS_REFC_DOMAIN_COW; + } + + switch (adj) { + case XFS_REFCOUNT_ADJUST_COW_ALLOC: + /* Adding a CoW reservation, there should be nothing here. */ + if (XFS_IS_CORRUPT(cur->bc_mp, + agbno + aglen > ext.rc_startblock)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + + tmp.rc_startblock = agbno; + tmp.rc_blockcount = aglen; + tmp.rc_refcount = 1; + tmp.rc_domain = XFS_REFC_DOMAIN_COW; + + trace_xfs_refcount_modify_extent(cur, &tmp); + + error = xfs_refcount_insert(cur, &tmp, + &found_tmp); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, found_tmp != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + break; + case XFS_REFCOUNT_ADJUST_COW_FREE: + /* Removing a CoW reservation, there should be one extent. */ + if (XFS_IS_CORRUPT(cur->bc_mp, ext.rc_startblock != agbno)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + if (XFS_IS_CORRUPT(cur->bc_mp, ext.rc_blockcount != aglen)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + if (XFS_IS_CORRUPT(cur->bc_mp, ext.rc_refcount != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + + ext.rc_refcount = 0; + trace_xfs_refcount_modify_extent(cur, &ext); + error = xfs_refcount_delete(cur, &found_rec); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(cur->bc_mp, found_rec != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + break; + default: + ASSERT(0); + } + + return error; +out_error: + trace_xfs_refcount_modify_extent_error(cur, error, _RET_IP_); + return error; +} + +/* + * Add or remove refcount btree entries for CoW reservations. + */ +STATIC int +xfs_refcount_adjust_cow( + struct xfs_btree_cur *cur, + xfs_agblock_t agbno, + xfs_extlen_t aglen, + enum xfs_refc_adjust_op adj) +{ + bool shape_changed; + int error; + + /* + * Ensure that no rcextents cross the boundary of the adjustment range. + */ + error = xfs_refcount_split_extent(cur, XFS_REFC_DOMAIN_COW, + agbno, &shape_changed); + if (error) + goto out_error; + + error = xfs_refcount_split_extent(cur, XFS_REFC_DOMAIN_COW, + agbno + aglen, &shape_changed); + if (error) + goto out_error; + + /* + * Try to merge with the left or right extents of the range. + */ + error = xfs_refcount_merge_extents(cur, XFS_REFC_DOMAIN_COW, &agbno, + &aglen, adj, &shape_changed); + if (error) + goto out_error; + + /* Now that we've taken care of the ends, adjust the middle extents */ + error = xfs_refcount_adjust_cow_extents(cur, agbno, aglen, adj); + if (error) + goto out_error; + + return 0; + +out_error: + trace_xfs_refcount_adjust_cow_error(cur, error, _RET_IP_); + return error; +} + +/* + * Record a CoW allocation in the refcount btree. + */ +STATIC int +__xfs_refcount_cow_alloc( + struct xfs_btree_cur *rcur, + xfs_agblock_t agbno, + xfs_extlen_t aglen) +{ + trace_xfs_refcount_cow_increase(rcur, agbno, aglen); + + /* Add refcount btree reservation */ + return xfs_refcount_adjust_cow(rcur, agbno, aglen, + XFS_REFCOUNT_ADJUST_COW_ALLOC); +} + +/* + * Remove a CoW allocation from the refcount btree. + */ +STATIC int +__xfs_refcount_cow_free( + struct xfs_btree_cur *rcur, + xfs_agblock_t agbno, + xfs_extlen_t aglen) +{ + trace_xfs_refcount_cow_decrease(rcur, agbno, aglen); + + /* Remove refcount btree reservation */ + return xfs_refcount_adjust_cow(rcur, agbno, aglen, + XFS_REFCOUNT_ADJUST_COW_FREE); +} + +/* Record a CoW staging extent in the refcount btree. */ +void +xfs_refcount_alloc_cow_extent( + struct xfs_trans *tp, + bool isrt, + xfs_fsblock_t fsb, + xfs_extlen_t len) +{ + struct xfs_mount *mp = tp->t_mountp; + + if (!xfs_has_reflink(mp)) + return; + + __xfs_refcount_add(tp, XFS_REFCOUNT_ALLOC_COW, isrt, fsb, len); + + /* Add rmap entry */ + xfs_rmap_alloc_extent(tp, isrt, fsb, len, XFS_RMAP_OWN_COW); +} + +/* Forget a CoW staging event in the refcount btree. */ +void +xfs_refcount_free_cow_extent( + struct xfs_trans *tp, + bool isrt, + xfs_fsblock_t fsb, + xfs_extlen_t len) +{ + struct xfs_mount *mp = tp->t_mountp; + + if (!xfs_has_reflink(mp)) + return; + + /* Remove rmap entry */ + xfs_rmap_free_extent(tp, isrt, fsb, len, XFS_RMAP_OWN_COW); + __xfs_refcount_add(tp, XFS_REFCOUNT_FREE_COW, isrt, fsb, len); +} + +struct xfs_refcount_recovery { + struct list_head rr_list; + struct xfs_refcount_irec rr_rrec; +}; + +/* Stuff an extent on the recovery list. */ +STATIC int +xfs_refcount_recover_extent( + struct xfs_btree_cur *cur, + const union xfs_btree_rec *rec, + void *priv) +{ + struct list_head *debris = priv; + struct xfs_refcount_recovery *rr; + + if (XFS_IS_CORRUPT(cur->bc_mp, + be32_to_cpu(rec->refc.rc_refcount) != 1)) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + + rr = kmalloc_obj(struct xfs_refcount_recovery, + GFP_KERNEL | __GFP_NOFAIL); + INIT_LIST_HEAD(&rr->rr_list); + xfs_refcount_btrec_to_irec(rec, &rr->rr_rrec); + + if (xfs_refcount_check_btrec(cur, &rr->rr_rrec) != NULL || + XFS_IS_CORRUPT(cur->bc_mp, + rr->rr_rrec.rc_domain != XFS_REFC_DOMAIN_COW)) { + xfs_btree_mark_sick(cur); + kfree(rr); + return -EFSCORRUPTED; + } + + list_add_tail(&rr->rr_list, debris); + return 0; +} + +/* Find and remove leftover CoW reservations. */ +int +xfs_refcount_recover_cow_leftovers( + struct xfs_group *xg) +{ + struct xfs_mount *mp = xg->xg_mount; + bool isrt = xg->xg_type == XG_TYPE_RTG; + struct xfs_trans *tp; + struct xfs_btree_cur *cur; + struct xfs_buf *agbp = NULL; + struct xfs_refcount_recovery *rr, *n; + struct list_head debris; + union xfs_btree_irec low = { + .rc.rc_domain = XFS_REFC_DOMAIN_COW, + }; + union xfs_btree_irec high = { + .rc.rc_domain = XFS_REFC_DOMAIN_COW, + .rc.rc_startblock = -1U, + }; + xfs_fsblock_t fsb; + int error; + + /* reflink filesystems must not have groups larger than 2^31-1 blocks */ + BUILD_BUG_ON(XFS_MAX_RGBLOCKS >= XFS_REFC_COWFLAG); + BUILD_BUG_ON(XFS_MAX_CRC_AG_BLOCKS >= XFS_REFC_COWFLAG); + + if (isrt) { + if (!xfs_has_rtgroups(mp)) + return 0; + if (xfs_group_max_blocks(xg) >= XFS_MAX_RGBLOCKS) + return -EOPNOTSUPP; + } else { + if (xfs_group_max_blocks(xg) > XFS_MAX_CRC_AG_BLOCKS) + return -EOPNOTSUPP; + } + + INIT_LIST_HEAD(&debris); + + /* + * In this first part, we use an empty transaction to gather up + * all the leftover CoW extents so that we can subsequently + * delete them. The empty transaction is used to avoid + * a buffer lock deadlock if there happens to be a loop in the + * refcountbt because we're allowed to re-grab a buffer that is + * already attached to our transaction. When we're done + * recording the CoW debris we cancel the (empty) transaction + * and everything goes away cleanly. + */ + tp = xfs_trans_alloc_empty(mp); + + if (isrt) { + xfs_rtgroup_lock(to_rtg(xg), XFS_RTGLOCK_REFCOUNT); + cur = xfs_rtrefcountbt_init_cursor(tp, to_rtg(xg)); + } else { + error = xfs_alloc_read_agf(to_perag(xg), tp, 0, &agbp); + if (error) + goto out_trans; + cur = xfs_refcountbt_init_cursor(mp, tp, agbp, to_perag(xg)); + } + + /* Find all the leftover CoW staging extents. */ + error = xfs_btree_query_range(cur, &low, &high, + xfs_refcount_recover_extent, &debris); + xfs_btree_del_cursor(cur, error); + if (agbp) + xfs_trans_brelse(tp, agbp); + else + xfs_rtgroup_unlock(to_rtg(xg), XFS_RTGLOCK_REFCOUNT); + xfs_trans_cancel(tp); + if (error) + goto out_free; + + /* Now iterate the list to free the leftovers */ + list_for_each_entry_safe(rr, n, &debris, rr_list) { + /* Set up transaction. */ + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, 0, 0, 0, &tp); + if (error) + goto out_free; + + /* Free the orphan record */ + fsb = xfs_gbno_to_fsb(xg, rr->rr_rrec.rc_startblock); + xfs_refcount_free_cow_extent(tp, isrt, fsb, + rr->rr_rrec.rc_blockcount); + + /* Free the block. */ + error = xfs_free_extent_later(tp, fsb, + rr->rr_rrec.rc_blockcount, NULL, + XFS_AG_RESV_NONE, + isrt ? XFS_FREE_EXTENT_REALTIME : 0); + if (error) + goto out_trans; + + error = xfs_trans_commit(tp); + if (error) + goto out_free; + + list_del(&rr->rr_list); + kfree(rr); + } + + return error; +out_trans: + xfs_trans_cancel(tp); +out_free: + /* Free the leftover list */ + list_for_each_entry_safe(rr, n, &debris, rr_list) { + list_del(&rr->rr_list); + kfree(rr); + } + return error; +} + +/* + * Scan part of the keyspace of the refcount records and tell us if the area + * has no records, is fully mapped by records, or is partially filled. + */ +int +xfs_refcount_has_records( + struct xfs_btree_cur *cur, + enum xfs_refc_domain domain, + xfs_agblock_t bno, + xfs_extlen_t len, + enum xbtree_recpacking *outcome) +{ + union xfs_btree_irec low; + union xfs_btree_irec high; + + memset(&low, 0, sizeof(low)); + low.rc.rc_startblock = bno; + memset(&high, 0xFF, sizeof(high)); + high.rc.rc_startblock = bno + len - 1; + low.rc.rc_domain = high.rc.rc_domain = domain; + + return xfs_btree_has_records(cur, &low, &high, NULL, outcome); +} + +struct xfs_refcount_query_range_info { + xfs_refcount_query_range_fn fn; + void *priv; +}; + +/* Format btree record and pass to our callback. */ +STATIC int +xfs_refcount_query_range_helper( + struct xfs_btree_cur *cur, + const union xfs_btree_rec *rec, + void *priv) +{ + struct xfs_refcount_query_range_info *query = priv; + struct xfs_refcount_irec irec; + xfs_failaddr_t fa; + + xfs_refcount_btrec_to_irec(rec, &irec); + fa = xfs_refcount_check_btrec(cur, &irec); + if (fa) + return xfs_refcount_complain_bad_rec(cur, fa, &irec); + + return query->fn(cur, &irec, query->priv); +} + +/* Find all refcount records between two keys. */ +int +xfs_refcount_query_range( + struct xfs_btree_cur *cur, + const struct xfs_refcount_irec *low_rec, + const struct xfs_refcount_irec *high_rec, + xfs_refcount_query_range_fn fn, + void *priv) +{ + union xfs_btree_irec low_brec = { .rc = *low_rec }; + union xfs_btree_irec high_brec = { .rc = *high_rec }; + struct xfs_refcount_query_range_info query = { .priv = priv, .fn = fn }; + + return xfs_btree_query_range(cur, &low_brec, &high_brec, + xfs_refcount_query_range_helper, &query); +} + +int __init +xfs_refcount_intent_init_cache(void) +{ + xfs_refcount_intent_cache = kmem_cache_create("xfs_refc_intent", + sizeof(struct xfs_refcount_intent), + 0, 0, NULL); + + return xfs_refcount_intent_cache != NULL ? 0 : -ENOMEM; +} + +void +xfs_refcount_intent_destroy_cache(void) +{ + kmem_cache_destroy(xfs_refcount_intent_cache); + xfs_refcount_intent_cache = NULL; +} diff --git a/fs/xfs/libxfs/xfs_refcount.h b/fs/xfs/libxfs/xfs_refcount.h new file mode 100644 index 000000000..f2e299a71 --- /dev/null +++ b/fs/xfs/libxfs/xfs_refcount.h @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2016 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_REFCOUNT_H__ +#define __XFS_REFCOUNT_H__ + +struct xfs_trans; +struct xfs_mount; +struct xfs_perag; +struct xfs_btree_cur; +struct xfs_bmbt_irec; +struct xfs_refcount_irec; +struct xfs_rtgroup; + +extern int xfs_refcount_lookup_le(struct xfs_btree_cur *cur, + enum xfs_refc_domain domain, xfs_agblock_t bno, int *stat); +extern int xfs_refcount_lookup_ge(struct xfs_btree_cur *cur, + enum xfs_refc_domain domain, xfs_agblock_t bno, int *stat); +extern int xfs_refcount_lookup_eq(struct xfs_btree_cur *cur, + enum xfs_refc_domain domain, xfs_agblock_t bno, int *stat); +extern int xfs_refcount_get_rec(struct xfs_btree_cur *cur, + struct xfs_refcount_irec *irec, int *stat); + +static inline uint32_t +xfs_refcount_encode_startblock( + xfs_agblock_t startblock, + enum xfs_refc_domain domain) +{ + uint32_t start; + + /* + * low level btree operations need to handle the generic btree range + * query functions (which set rc_domain == -1U), so we check that the + * domain is /not/ shared. + */ + start = startblock & ~XFS_REFC_COWFLAG; + if (domain != XFS_REFC_DOMAIN_SHARED) + start |= XFS_REFC_COWFLAG; + + return start; +} + +enum xfs_refcount_intent_type { + XFS_REFCOUNT_INCREASE = 1, + XFS_REFCOUNT_DECREASE, + XFS_REFCOUNT_ALLOC_COW, + XFS_REFCOUNT_FREE_COW, +}; + +#define XFS_REFCOUNT_INTENT_STRINGS \ + { XFS_REFCOUNT_INCREASE, "incr" }, \ + { XFS_REFCOUNT_DECREASE, "decr" }, \ + { XFS_REFCOUNT_ALLOC_COW, "alloc_cow" }, \ + { XFS_REFCOUNT_FREE_COW, "free_cow" } + +struct xfs_refcount_intent { + struct list_head ri_list; + struct xfs_group *ri_group; + enum xfs_refcount_intent_type ri_type; + xfs_extlen_t ri_blockcount; + xfs_fsblock_t ri_startblock; + bool ri_realtime; +}; + +/* Check that the refcount is appropriate for the record domain. */ +static inline bool +xfs_refcount_check_domain( + const struct xfs_refcount_irec *irec) +{ + if (irec->rc_domain == XFS_REFC_DOMAIN_COW && irec->rc_refcount != 1) + return false; + if (irec->rc_domain == XFS_REFC_DOMAIN_SHARED && irec->rc_refcount < 2) + return false; + return true; +} + +void xfs_refcount_increase_extent(struct xfs_trans *tp, bool isrt, + struct xfs_bmbt_irec *irec); +void xfs_refcount_decrease_extent(struct xfs_trans *tp, bool isrt, + struct xfs_bmbt_irec *irec); + +int xfs_refcount_finish_one(struct xfs_trans *tp, + struct xfs_refcount_intent *ri, struct xfs_btree_cur **pcur); +int xfs_rtrefcount_finish_one(struct xfs_trans *tp, + struct xfs_refcount_intent *ri, struct xfs_btree_cur **pcur); + +extern int xfs_refcount_find_shared(struct xfs_btree_cur *cur, + xfs_agblock_t agbno, xfs_extlen_t aglen, xfs_agblock_t *fbno, + xfs_extlen_t *flen, bool find_end_of_shared); + +void xfs_refcount_alloc_cow_extent(struct xfs_trans *tp, bool isrt, + xfs_fsblock_t fsb, xfs_extlen_t len); +void xfs_refcount_free_cow_extent(struct xfs_trans *tp, bool isrt, + xfs_fsblock_t fsb, xfs_extlen_t len); +int xfs_refcount_recover_cow_leftovers(struct xfs_group *xg); + +/* + * While we're adjusting the refcounts records of an extent, we have + * to keep an eye on the number of extents we're dirtying -- run too + * many in a single transaction and we'll exceed the transaction's + * reservation and crash the fs. Each record adds 12 bytes to the + * log (plus any key updates) so we'll conservatively assume 32 bytes + * per record. We must also leave space for btree splits on both ends + * of the range and space for the CUD and a new CUI. + * + * Each EFI that we attach to the transaction is assumed to consume ~32 bytes. + * This is a low estimate for an EFI tracking a single extent (16 bytes for the + * EFI header, 16 for the extent, and 12 for the xlog op header), but the + * estimate is acceptable if there's more than one extent being freed. + * In the worst case of freeing every other block during a refcount decrease + * operation, we amortize the space used for one EFI log item across 16 + * extents. + */ +#define XFS_REFCOUNT_ITEM_OVERHEAD 32 + +extern int xfs_refcount_has_records(struct xfs_btree_cur *cur, + enum xfs_refc_domain domain, xfs_agblock_t bno, + xfs_extlen_t len, enum xbtree_recpacking *outcome); +union xfs_btree_rec; +extern void xfs_refcount_btrec_to_irec(const union xfs_btree_rec *rec, + struct xfs_refcount_irec *irec); +xfs_failaddr_t xfs_refcount_check_irec(struct xfs_perag *pag, + const struct xfs_refcount_irec *irec); +xfs_failaddr_t xfs_rtrefcount_check_irec(struct xfs_rtgroup *rtg, + const struct xfs_refcount_irec *irec); +extern int xfs_refcount_insert(struct xfs_btree_cur *cur, + struct xfs_refcount_irec *irec, int *stat); + +extern struct kmem_cache *xfs_refcount_intent_cache; + +int __init xfs_refcount_intent_init_cache(void); +void xfs_refcount_intent_destroy_cache(void); + +typedef int (*xfs_refcount_query_range_fn)( + struct xfs_btree_cur *cur, + const struct xfs_refcount_irec *rec, + void *priv); + +int xfs_refcount_query_range(struct xfs_btree_cur *cur, + const struct xfs_refcount_irec *low_rec, + const struct xfs_refcount_irec *high_rec, + xfs_refcount_query_range_fn fn, void *priv); + +#endif /* __XFS_REFCOUNT_H__ */ diff --git a/fs/xfs/libxfs/xfs_refcount_btree.c b/fs/xfs/libxfs/xfs_refcount_btree.c new file mode 100644 index 000000000..7e5f92c1a --- /dev/null +++ b/fs/xfs/libxfs/xfs_refcount_btree.c @@ -0,0 +1,543 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2016 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_btree.h" +#include "xfs_btree_staging.h" +#include "xfs_refcount_btree.h" +#include "xfs_refcount.h" +#include "xfs_alloc.h" +#include "xfs_error.h" +#include "xfs_health.h" +#include "xfs_trace.h" +#include "xfs_trans.h" +#include "xfs_bit.h" +#include "xfs_rmap.h" +#include "xfs_ag.h" + +static struct kmem_cache *xfs_refcountbt_cur_cache; + +static struct xfs_btree_cur * +xfs_refcountbt_dup_cursor( + struct xfs_btree_cur *cur) +{ + return xfs_refcountbt_init_cursor(cur->bc_mp, cur->bc_tp, + cur->bc_ag.agbp, to_perag(cur->bc_group)); +} + +STATIC void +xfs_refcountbt_set_root( + struct xfs_btree_cur *cur, + const union xfs_btree_ptr *ptr, + int inc) +{ + struct xfs_buf *agbp = cur->bc_ag.agbp; + struct xfs_agf *agf = agbp->b_addr; + struct xfs_perag *pag = agbp->b_pag; + + ASSERT(ptr->s != 0); + + agf->agf_refcount_root = ptr->s; + be32_add_cpu(&agf->agf_refcount_level, inc); + pag->pagf_refcount_level += inc; + + xfs_alloc_log_agf(cur->bc_tp, agbp, + XFS_AGF_REFCOUNT_ROOT | XFS_AGF_REFCOUNT_LEVEL); +} + +STATIC int +xfs_refcountbt_alloc_block( + struct xfs_btree_cur *cur, + const union xfs_btree_ptr *start, + union xfs_btree_ptr *new, + int *stat) +{ + struct xfs_buf *agbp = cur->bc_ag.agbp; + struct xfs_agf *agf = agbp->b_addr; + struct xfs_alloc_arg args; /* block allocation args */ + int error; /* error return value */ + + memset(&args, 0, sizeof(args)); + args.tp = cur->bc_tp; + args.mp = cur->bc_mp; + args.pag = to_perag(cur->bc_group); + args.oinfo = XFS_RMAP_OINFO_REFC; + args.minlen = args.maxlen = args.prod = 1; + args.resv = XFS_AG_RESV_METADATA; + + error = xfs_alloc_vextent_near_bno(&args, + xfs_agbno_to_fsb(args.pag, xfs_refc_block(args.mp))); + if (error) + goto out_error; + if (args.fsbno == NULLFSBLOCK) { + *stat = 0; + return 0; + } + ASSERT(args.agno == cur->bc_group->xg_gno); + ASSERT(args.len == 1); + + new->s = cpu_to_be32(args.agbno); + be32_add_cpu(&agf->agf_refcount_blocks, 1); + xfs_alloc_log_agf(cur->bc_tp, agbp, XFS_AGF_REFCOUNT_BLOCKS); + + *stat = 1; + return 0; + +out_error: + return error; +} + +STATIC int +xfs_refcountbt_free_block( + struct xfs_btree_cur *cur, + struct xfs_buf *bp) +{ + struct xfs_mount *mp = cur->bc_mp; + struct xfs_buf *agbp = cur->bc_ag.agbp; + struct xfs_agf *agf = agbp->b_addr; + xfs_fsblock_t fsbno = XFS_DADDR_TO_FSB(mp, xfs_buf_daddr(bp)); + + be32_add_cpu(&agf->agf_refcount_blocks, -1); + xfs_alloc_log_agf(cur->bc_tp, agbp, XFS_AGF_REFCOUNT_BLOCKS); + return xfs_free_extent_later(cur->bc_tp, fsbno, 1, + &XFS_RMAP_OINFO_REFC, XFS_AG_RESV_METADATA, 0); +} + +STATIC int +xfs_refcountbt_get_minrecs( + struct xfs_btree_cur *cur, + int level) +{ + return cur->bc_mp->m_refc_mnr[level != 0]; +} + +STATIC int +xfs_refcountbt_get_maxrecs( + struct xfs_btree_cur *cur, + int level) +{ + return cur->bc_mp->m_refc_mxr[level != 0]; +} + +STATIC void +xfs_refcountbt_init_key_from_rec( + union xfs_btree_key *key, + const union xfs_btree_rec *rec) +{ + key->refc.rc_startblock = rec->refc.rc_startblock; +} + +STATIC void +xfs_refcountbt_init_high_key_from_rec( + union xfs_btree_key *key, + const union xfs_btree_rec *rec) +{ + __u32 x; + + x = be32_to_cpu(rec->refc.rc_startblock); + x += be32_to_cpu(rec->refc.rc_blockcount) - 1; + key->refc.rc_startblock = cpu_to_be32(x); +} + +STATIC void +xfs_refcountbt_init_rec_from_cur( + struct xfs_btree_cur *cur, + union xfs_btree_rec *rec) +{ + const struct xfs_refcount_irec *irec = &cur->bc_rec.rc; + uint32_t start; + + start = xfs_refcount_encode_startblock(irec->rc_startblock, + irec->rc_domain); + rec->refc.rc_startblock = cpu_to_be32(start); + rec->refc.rc_blockcount = cpu_to_be32(cur->bc_rec.rc.rc_blockcount); + rec->refc.rc_refcount = cpu_to_be32(cur->bc_rec.rc.rc_refcount); +} + +STATIC void +xfs_refcountbt_init_ptr_from_cur( + struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr) +{ + struct xfs_agf *agf = cur->bc_ag.agbp->b_addr; + + ASSERT(cur->bc_group->xg_gno == be32_to_cpu(agf->agf_seqno)); + + ptr->s = agf->agf_refcount_root; +} + +STATIC int +xfs_refcountbt_cmp_key_with_cur( + struct xfs_btree_cur *cur, + const union xfs_btree_key *key) +{ + const struct xfs_refcount_key *kp = &key->refc; + const struct xfs_refcount_irec *irec = &cur->bc_rec.rc; + uint32_t start; + + start = xfs_refcount_encode_startblock(irec->rc_startblock, + irec->rc_domain); + return cmp_int(be32_to_cpu(kp->rc_startblock), start); +} + +STATIC int +xfs_refcountbt_cmp_two_keys( + struct xfs_btree_cur *cur, + const union xfs_btree_key *k1, + const union xfs_btree_key *k2, + const union xfs_btree_key *mask) +{ + ASSERT(!mask || mask->refc.rc_startblock); + + return cmp_int(be32_to_cpu(k1->refc.rc_startblock), + be32_to_cpu(k2->refc.rc_startblock)); +} + +STATIC xfs_failaddr_t +xfs_refcountbt_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_btree_block *block = XFS_BUF_TO_BLOCK(bp); + struct xfs_perag *pag = bp->b_pag; + xfs_failaddr_t fa; + unsigned int level; + + if (!xfs_verify_magic(bp, block->bb_magic)) + return __this_address; + + if (!xfs_has_reflink(mp)) + return __this_address; + fa = xfs_btree_agblock_v5hdr_verify(bp); + if (fa) + return fa; + + level = be16_to_cpu(block->bb_level); + if (pag && xfs_perag_initialised_agf(pag)) { + unsigned int maxlevel = pag->pagf_refcount_level; + +#ifdef CONFIG_XFS_ONLINE_REPAIR + /* + * Online repair could be rewriting the refcount btree, so + * we'll validate against the larger of either tree while this + * is going on. + */ + maxlevel = max_t(unsigned int, maxlevel, + pag->pagf_repair_refcount_level); +#endif + if (level >= maxlevel) + return __this_address; + } else if (level >= mp->m_refc_maxlevels) + return __this_address; + + return xfs_btree_agblock_verify(bp, mp->m_refc_mxr[level != 0]); +} + +STATIC void +xfs_refcountbt_read_verify( + struct xfs_buf *bp) +{ + xfs_failaddr_t fa; + + if (!xfs_btree_agblock_verify_crc(bp)) + xfs_verifier_error(bp, -EFSBADCRC, __this_address); + else { + fa = xfs_refcountbt_verify(bp); + if (fa) + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + } + + if (bp->b_error) + trace_xfs_btree_corrupt(bp, _RET_IP_); +} + +STATIC void +xfs_refcountbt_write_verify( + struct xfs_buf *bp) +{ + xfs_failaddr_t fa; + + fa = xfs_refcountbt_verify(bp); + if (fa) { + trace_xfs_btree_corrupt(bp, _RET_IP_); + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + return; + } + xfs_btree_agblock_calc_crc(bp); + +} + +const struct xfs_buf_ops xfs_refcountbt_buf_ops = { + .name = "xfs_refcountbt", + .magic = { 0, cpu_to_be32(XFS_REFC_CRC_MAGIC) }, + .verify_read = xfs_refcountbt_read_verify, + .verify_write = xfs_refcountbt_write_verify, + .verify_struct = xfs_refcountbt_verify, +}; + +STATIC int +xfs_refcountbt_keys_inorder( + struct xfs_btree_cur *cur, + const union xfs_btree_key *k1, + const union xfs_btree_key *k2) +{ + return be32_to_cpu(k1->refc.rc_startblock) < + be32_to_cpu(k2->refc.rc_startblock); +} + +STATIC int +xfs_refcountbt_recs_inorder( + struct xfs_btree_cur *cur, + const union xfs_btree_rec *r1, + const union xfs_btree_rec *r2) +{ + return be32_to_cpu(r1->refc.rc_startblock) + + be32_to_cpu(r1->refc.rc_blockcount) <= + be32_to_cpu(r2->refc.rc_startblock); +} + +STATIC enum xbtree_key_contig +xfs_refcountbt_keys_contiguous( + struct xfs_btree_cur *cur, + const union xfs_btree_key *key1, + const union xfs_btree_key *key2, + const union xfs_btree_key *mask) +{ + ASSERT(!mask || mask->refc.rc_startblock); + + return xbtree_key_contig(be32_to_cpu(key1->refc.rc_startblock), + be32_to_cpu(key2->refc.rc_startblock)); +} + +const struct xfs_btree_ops xfs_refcountbt_ops = { + .name = "refcount", + .type = XFS_BTREE_TYPE_AG, + + .rec_len = sizeof(struct xfs_refcount_rec), + .key_len = sizeof(struct xfs_refcount_key), + .ptr_len = XFS_BTREE_SHORT_PTR_LEN, + + .lru_refs = XFS_REFC_BTREE_REF, + .statoff = XFS_STATS_CALC_INDEX(xs_refcbt_2), + .sick_mask = XFS_SICK_AG_REFCNTBT, + + .dup_cursor = xfs_refcountbt_dup_cursor, + .set_root = xfs_refcountbt_set_root, + .alloc_block = xfs_refcountbt_alloc_block, + .free_block = xfs_refcountbt_free_block, + .get_minrecs = xfs_refcountbt_get_minrecs, + .get_maxrecs = xfs_refcountbt_get_maxrecs, + .init_key_from_rec = xfs_refcountbt_init_key_from_rec, + .init_high_key_from_rec = xfs_refcountbt_init_high_key_from_rec, + .init_rec_from_cur = xfs_refcountbt_init_rec_from_cur, + .init_ptr_from_cur = xfs_refcountbt_init_ptr_from_cur, + .cmp_key_with_cur = xfs_refcountbt_cmp_key_with_cur, + .buf_ops = &xfs_refcountbt_buf_ops, + .cmp_two_keys = xfs_refcountbt_cmp_two_keys, + .keys_inorder = xfs_refcountbt_keys_inorder, + .recs_inorder = xfs_refcountbt_recs_inorder, + .keys_contiguous = xfs_refcountbt_keys_contiguous, +}; + +/* + * Create a new refcount btree cursor. + * + * For staging cursors tp and agbp are NULL. + */ +struct xfs_btree_cur * +xfs_refcountbt_init_cursor( + struct xfs_mount *mp, + struct xfs_trans *tp, + struct xfs_buf *agbp, + struct xfs_perag *pag) +{ + struct xfs_btree_cur *cur; + + ASSERT(pag_agno(pag) < mp->m_sb.sb_agcount); + + cur = xfs_btree_alloc_cursor(mp, tp, &xfs_refcountbt_ops, + mp->m_refc_maxlevels, xfs_refcountbt_cur_cache); + cur->bc_group = xfs_group_hold(pag_group(pag)); + cur->bc_refc.nr_ops = 0; + cur->bc_refc.shape_changes = 0; + cur->bc_ag.agbp = agbp; + if (agbp) { + struct xfs_agf *agf = agbp->b_addr; + + cur->bc_nlevels = be32_to_cpu(agf->agf_refcount_level); + } + return cur; +} + +/* + * Swap in the new btree root. Once we pass this point the newly rebuilt btree + * is in place and we have to kill off all the old btree blocks. + */ +void +xfs_refcountbt_commit_staged_btree( + struct xfs_btree_cur *cur, + struct xfs_trans *tp, + struct xfs_buf *agbp) +{ + struct xfs_agf *agf = agbp->b_addr; + struct xbtree_afakeroot *afake = cur->bc_ag.afake; + + ASSERT(cur->bc_flags & XFS_BTREE_STAGING); + + agf->agf_refcount_root = cpu_to_be32(afake->af_root); + agf->agf_refcount_level = cpu_to_be32(afake->af_levels); + agf->agf_refcount_blocks = cpu_to_be32(afake->af_blocks); + xfs_alloc_log_agf(tp, agbp, XFS_AGF_REFCOUNT_BLOCKS | + XFS_AGF_REFCOUNT_ROOT | + XFS_AGF_REFCOUNT_LEVEL); + xfs_btree_commit_afakeroot(cur, tp, agbp); +} + +/* Calculate number of records in a refcount btree block. */ +static inline unsigned int +xfs_refcountbt_block_maxrecs( + unsigned int blocklen, + bool leaf) +{ + if (leaf) + return blocklen / sizeof(struct xfs_refcount_rec); + return blocklen / (sizeof(struct xfs_refcount_key) + + sizeof(xfs_refcount_ptr_t)); +} + +/* + * Calculate the number of records in a refcount btree block. + */ +unsigned int +xfs_refcountbt_maxrecs( + struct xfs_mount *mp, + unsigned int blocklen, + bool leaf) +{ + blocklen -= XFS_REFCOUNT_BLOCK_LEN; + return xfs_refcountbt_block_maxrecs(blocklen, leaf); +} + +/* Compute the max possible height of the maximally sized refcount btree. */ +unsigned int +xfs_refcountbt_maxlevels_ondisk(void) +{ + unsigned int minrecs[2]; + unsigned int blocklen; + + blocklen = XFS_MIN_CRC_BLOCKSIZE - XFS_BTREE_SBLOCK_CRC_LEN; + + minrecs[0] = xfs_refcountbt_block_maxrecs(blocklen, true) / 2; + minrecs[1] = xfs_refcountbt_block_maxrecs(blocklen, false) / 2; + + return xfs_btree_compute_maxlevels(minrecs, XFS_MAX_CRC_AG_BLOCKS); +} + +/* Compute the maximum height of a refcount btree. */ +void +xfs_refcountbt_compute_maxlevels( + struct xfs_mount *mp) +{ + if (!xfs_has_reflink(mp)) { + mp->m_refc_maxlevels = 0; + return; + } + + mp->m_refc_maxlevels = xfs_btree_compute_maxlevels( + mp->m_refc_mnr, mp->m_sb.sb_agblocks); + ASSERT(mp->m_refc_maxlevels <= xfs_refcountbt_maxlevels_ondisk()); +} + +/* Calculate the refcount btree size for some records. */ +xfs_extlen_t +xfs_refcountbt_calc_size( + struct xfs_mount *mp, + unsigned long long len) +{ + return xfs_btree_calc_size(mp->m_refc_mnr, len); +} + +/* + * Calculate the maximum refcount btree size. + */ +xfs_extlen_t +xfs_refcountbt_max_size( + struct xfs_mount *mp, + xfs_agblock_t agblocks) +{ + /* Bail out if we're uninitialized, which can happen in mkfs. */ + if (mp->m_refc_mxr[0] == 0) + return 0; + + return xfs_refcountbt_calc_size(mp, agblocks); +} + +/* + * Figure out how many blocks to reserve and how many are used by this btree. + */ +int +xfs_refcountbt_calc_reserves( + struct xfs_mount *mp, + struct xfs_trans *tp, + struct xfs_perag *pag, + xfs_extlen_t *ask, + xfs_extlen_t *used) +{ + struct xfs_buf *agbp; + struct xfs_agf *agf; + xfs_agblock_t agblocks; + xfs_extlen_t tree_len; + int error; + + if (!xfs_has_reflink(mp)) + return 0; + + error = xfs_alloc_read_agf(pag, tp, 0, &agbp); + if (error) + return error; + + agf = agbp->b_addr; + agblocks = be32_to_cpu(agf->agf_length); + tree_len = be32_to_cpu(agf->agf_refcount_blocks); + xfs_trans_brelse(tp, agbp); + + /* + * The log is permanently allocated, so the space it occupies will + * never be available for the kinds of things that would require btree + * expansion. We therefore can pretend the space isn't there. + */ + if (xfs_ag_contains_log(mp, pag_agno(pag))) + agblocks -= mp->m_sb.sb_logblocks; + + *ask += xfs_refcountbt_max_size(mp, agblocks); + *used += tree_len; + + return error; +} + +int __init +xfs_refcountbt_init_cur_cache(void) +{ + xfs_refcountbt_cur_cache = kmem_cache_create("xfs_refcbt_cur", + xfs_btree_cur_sizeof(xfs_refcountbt_maxlevels_ondisk()), + 0, 0, NULL); + + if (!xfs_refcountbt_cur_cache) + return -ENOMEM; + return 0; +} + +void +xfs_refcountbt_destroy_cur_cache(void) +{ + kmem_cache_destroy(xfs_refcountbt_cur_cache); + xfs_refcountbt_cur_cache = NULL; +} diff --git a/fs/xfs/libxfs/xfs_refcount_btree.h b/fs/xfs/libxfs/xfs_refcount_btree.h new file mode 100644 index 000000000..beb93bef6 --- /dev/null +++ b/fs/xfs/libxfs/xfs_refcount_btree.h @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2016 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_REFCOUNT_BTREE_H__ +#define __XFS_REFCOUNT_BTREE_H__ + +/* + * Reference Count Btree on-disk structures + */ + +struct xfs_buf; +struct xfs_btree_cur; +struct xfs_mount; +struct xfs_perag; +struct xbtree_afakeroot; + +/* + * Btree block header size + */ +#define XFS_REFCOUNT_BLOCK_LEN XFS_BTREE_SBLOCK_CRC_LEN + +/* + * Record, key, and pointer address macros for btree blocks. + * + * (note that some of these may appear unused, but they are used in userspace) + */ +#define XFS_REFCOUNT_REC_ADDR(block, index) \ + ((struct xfs_refcount_rec *) \ + ((char *)(block) + \ + XFS_REFCOUNT_BLOCK_LEN + \ + (((index) - 1) * sizeof(struct xfs_refcount_rec)))) + +#define XFS_REFCOUNT_KEY_ADDR(block, index) \ + ((struct xfs_refcount_key *) \ + ((char *)(block) + \ + XFS_REFCOUNT_BLOCK_LEN + \ + ((index) - 1) * sizeof(struct xfs_refcount_key))) + +#define XFS_REFCOUNT_PTR_ADDR(block, index, maxrecs) \ + ((xfs_refcount_ptr_t *) \ + ((char *)(block) + \ + XFS_REFCOUNT_BLOCK_LEN + \ + (maxrecs) * sizeof(struct xfs_refcount_key) + \ + ((index) - 1) * sizeof(xfs_refcount_ptr_t))) + +extern struct xfs_btree_cur *xfs_refcountbt_init_cursor(struct xfs_mount *mp, + struct xfs_trans *tp, struct xfs_buf *agbp, + struct xfs_perag *pag); +unsigned int xfs_refcountbt_maxrecs(struct xfs_mount *mp, unsigned int blocklen, + bool leaf); +extern void xfs_refcountbt_compute_maxlevels(struct xfs_mount *mp); + +extern xfs_extlen_t xfs_refcountbt_calc_size(struct xfs_mount *mp, + unsigned long long len); +extern xfs_extlen_t xfs_refcountbt_max_size(struct xfs_mount *mp, + xfs_agblock_t agblocks); + +extern int xfs_refcountbt_calc_reserves(struct xfs_mount *mp, + struct xfs_trans *tp, struct xfs_perag *pag, xfs_extlen_t *ask, + xfs_extlen_t *used); + +void xfs_refcountbt_commit_staged_btree(struct xfs_btree_cur *cur, + struct xfs_trans *tp, struct xfs_buf *agbp); + +unsigned int xfs_refcountbt_maxlevels_ondisk(void); + +int __init xfs_refcountbt_init_cur_cache(void); +void xfs_refcountbt_destroy_cur_cache(void); + +#endif /* __XFS_REFCOUNT_BTREE_H__ */ diff --git a/fs/xfs/libxfs/xfs_rmap.c b/fs/xfs/libxfs/xfs_rmap.c new file mode 100644 index 000000000..34d218de2 --- /dev/null +++ b/fs/xfs/libxfs/xfs_rmap.c @@ -0,0 +1,3142 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2014 Red Hat, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_mount.h" +#include "xfs_sb.h" +#include "xfs_defer.h" +#include "xfs_btree.h" +#include "xfs_trans.h" +#include "xfs_alloc.h" +#include "xfs_rmap.h" +#include "xfs_rmap_btree.h" +#include "xfs_trace.h" +#include "xfs_errortag.h" +#include "xfs_error.h" +#include "xfs_inode.h" +#include "xfs_ag.h" +#include "xfs_health.h" +#include "xfs_rmap_item.h" +#include "xfs_rtgroup.h" +#include "xfs_rtrmap_btree.h" + +struct kmem_cache *xfs_rmap_intent_cache; + +/* + * Lookup the first record less than or equal to [bno, len, owner, offset] + * in the btree given by cur. + */ +int +xfs_rmap_lookup_le( + struct xfs_btree_cur *cur, + xfs_agblock_t bno, + uint64_t owner, + uint64_t offset, + unsigned int flags, + struct xfs_rmap_irec *irec, + int *stat) +{ + int get_stat = 0; + int error; + + cur->bc_rec.r.rm_startblock = bno; + cur->bc_rec.r.rm_blockcount = 0; + cur->bc_rec.r.rm_owner = owner; + cur->bc_rec.r.rm_offset = offset; + cur->bc_rec.r.rm_flags = flags; + + error = xfs_btree_lookup(cur, XFS_LOOKUP_LE, stat); + if (error || !(*stat) || !irec) + return error; + + error = xfs_rmap_get_rec(cur, irec, &get_stat); + if (error) + return error; + if (!get_stat) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + + return 0; +} + +/* + * Lookup the record exactly matching [bno, len, owner, offset] + * in the btree given by cur. + */ +int +xfs_rmap_lookup_eq( + struct xfs_btree_cur *cur, + xfs_agblock_t bno, + xfs_extlen_t len, + uint64_t owner, + uint64_t offset, + unsigned int flags, + int *stat) +{ + cur->bc_rec.r.rm_startblock = bno; + cur->bc_rec.r.rm_blockcount = len; + cur->bc_rec.r.rm_owner = owner; + cur->bc_rec.r.rm_offset = offset; + cur->bc_rec.r.rm_flags = flags; + return xfs_btree_lookup(cur, XFS_LOOKUP_EQ, stat); +} + +/* + * Update the record referred to by cur to the value given + * by [bno, len, owner, offset]. + * This either works (return 0) or gets an EFSCORRUPTED error. + */ +STATIC int +xfs_rmap_update( + struct xfs_btree_cur *cur, + struct xfs_rmap_irec *irec) +{ + union xfs_btree_rec rec; + int error; + + trace_xfs_rmap_update(cur, irec->rm_startblock, irec->rm_blockcount, + irec->rm_owner, irec->rm_offset, irec->rm_flags); + + rec.rmap.rm_startblock = cpu_to_be32(irec->rm_startblock); + rec.rmap.rm_blockcount = cpu_to_be32(irec->rm_blockcount); + rec.rmap.rm_owner = cpu_to_be64(irec->rm_owner); + rec.rmap.rm_offset = cpu_to_be64( + xfs_rmap_irec_offset_pack(irec)); + error = xfs_btree_update(cur, &rec); + if (error) + trace_xfs_rmap_update_error(cur, error, _RET_IP_); + return error; +} + +int +xfs_rmap_insert( + struct xfs_btree_cur *rcur, + xfs_agblock_t agbno, + xfs_extlen_t len, + uint64_t owner, + uint64_t offset, + unsigned int flags) +{ + int i; + int error; + + trace_xfs_rmap_insert(rcur, agbno, len, owner, offset, flags); + + error = xfs_rmap_lookup_eq(rcur, agbno, len, owner, offset, flags, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(rcur->bc_mp, i != 0)) { + xfs_btree_mark_sick(rcur); + error = -EFSCORRUPTED; + goto done; + } + + rcur->bc_rec.r.rm_startblock = agbno; + rcur->bc_rec.r.rm_blockcount = len; + rcur->bc_rec.r.rm_owner = owner; + rcur->bc_rec.r.rm_offset = offset; + rcur->bc_rec.r.rm_flags = flags; + error = xfs_btree_insert(rcur, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(rcur->bc_mp, i != 1)) { + xfs_btree_mark_sick(rcur); + error = -EFSCORRUPTED; + goto done; + } +done: + if (error) + trace_xfs_rmap_insert_error(rcur, error, _RET_IP_); + return error; +} + +STATIC int +xfs_rmap_delete( + struct xfs_btree_cur *rcur, + xfs_agblock_t agbno, + xfs_extlen_t len, + uint64_t owner, + uint64_t offset, + unsigned int flags) +{ + int i; + int error; + + trace_xfs_rmap_delete(rcur, agbno, len, owner, offset, flags); + + error = xfs_rmap_lookup_eq(rcur, agbno, len, owner, offset, flags, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(rcur->bc_mp, i != 1)) { + xfs_btree_mark_sick(rcur); + error = -EFSCORRUPTED; + goto done; + } + + error = xfs_btree_delete(rcur, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(rcur->bc_mp, i != 1)) { + xfs_btree_mark_sick(rcur); + error = -EFSCORRUPTED; + goto done; + } +done: + if (error) + trace_xfs_rmap_delete_error(rcur, error, _RET_IP_); + return error; +} + +/* Convert an internal btree record to an rmap record. */ +xfs_failaddr_t +xfs_rmap_btrec_to_irec( + const union xfs_btree_rec *rec, + struct xfs_rmap_irec *irec) +{ + irec->rm_startblock = be32_to_cpu(rec->rmap.rm_startblock); + irec->rm_blockcount = be32_to_cpu(rec->rmap.rm_blockcount); + irec->rm_owner = be64_to_cpu(rec->rmap.rm_owner); + return xfs_rmap_irec_offset_unpack(be64_to_cpu(rec->rmap.rm_offset), + irec); +} + +/* Simple checks for rmap records. */ +xfs_failaddr_t +xfs_rmap_check_irec( + struct xfs_perag *pag, + const struct xfs_rmap_irec *irec) +{ + struct xfs_mount *mp = pag_mount(pag); + bool is_inode; + bool is_unwritten; + bool is_bmbt; + bool is_attr; + + if (irec->rm_blockcount == 0) + return __this_address; + if (irec->rm_startblock <= XFS_AGFL_BLOCK(mp)) { + if (irec->rm_owner != XFS_RMAP_OWN_FS) + return __this_address; + if (irec->rm_blockcount != XFS_AGFL_BLOCK(mp) + 1) + return __this_address; + } else { + /* check for valid extent range, including overflow */ + if (!xfs_verify_agbext(pag, irec->rm_startblock, + irec->rm_blockcount)) + return __this_address; + } + + if (!(xfs_verify_ino(mp, irec->rm_owner) || + (irec->rm_owner <= XFS_RMAP_OWN_FS && + irec->rm_owner >= XFS_RMAP_OWN_MIN))) + return __this_address; + + /* Check flags. */ + is_inode = !XFS_RMAP_NON_INODE_OWNER(irec->rm_owner); + is_bmbt = irec->rm_flags & XFS_RMAP_BMBT_BLOCK; + is_attr = irec->rm_flags & XFS_RMAP_ATTR_FORK; + is_unwritten = irec->rm_flags & XFS_RMAP_UNWRITTEN; + + if (is_bmbt && irec->rm_offset != 0) + return __this_address; + + if (!is_inode && irec->rm_offset != 0) + return __this_address; + + if (is_unwritten && (is_bmbt || !is_inode || is_attr)) + return __this_address; + + if (!is_inode && (is_bmbt || is_unwritten || is_attr)) + return __this_address; + + /* Check for a valid fork offset, if applicable. */ + if (is_inode && !is_bmbt && + !xfs_verify_fileext(mp, irec->rm_offset, irec->rm_blockcount)) + return __this_address; + + return NULL; +} + +static xfs_failaddr_t +xfs_rtrmap_check_meta_irec( + struct xfs_rtgroup *rtg, + const struct xfs_rmap_irec *irec) +{ + struct xfs_mount *mp = rtg_mount(rtg); + + if (irec->rm_offset != 0) + return __this_address; + if (irec->rm_flags & XFS_RMAP_UNWRITTEN) + return __this_address; + + switch (irec->rm_owner) { + case XFS_RMAP_OWN_FS: + if (irec->rm_startblock != 0) + return __this_address; + if (irec->rm_blockcount != mp->m_sb.sb_rextsize) + return __this_address; + return NULL; + case XFS_RMAP_OWN_COW: + if (!xfs_has_rtreflink(mp)) + return __this_address; + if (!xfs_verify_rgbext(rtg, irec->rm_startblock, + irec->rm_blockcount)) + return __this_address; + return NULL; + default: + return __this_address; + } + + return NULL; +} + +static xfs_failaddr_t +xfs_rtrmap_check_inode_irec( + struct xfs_rtgroup *rtg, + const struct xfs_rmap_irec *irec) +{ + struct xfs_mount *mp = rtg_mount(rtg); + + if (!xfs_verify_ino(mp, irec->rm_owner)) + return __this_address; + if (!xfs_verify_rgbext(rtg, irec->rm_startblock, irec->rm_blockcount)) + return __this_address; + if (!xfs_verify_fileext(mp, irec->rm_offset, irec->rm_blockcount)) + return __this_address; + return NULL; +} + +xfs_failaddr_t +xfs_rtrmap_check_irec( + struct xfs_rtgroup *rtg, + const struct xfs_rmap_irec *irec) +{ + if (irec->rm_blockcount == 0) + return __this_address; + if (irec->rm_flags & (XFS_RMAP_BMBT_BLOCK | XFS_RMAP_ATTR_FORK)) + return __this_address; + if (XFS_RMAP_NON_INODE_OWNER(irec->rm_owner)) + return xfs_rtrmap_check_meta_irec(rtg, irec); + return xfs_rtrmap_check_inode_irec(rtg, irec); +} + +static inline xfs_failaddr_t +xfs_rmap_check_btrec( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *irec) +{ + if (xfs_btree_is_rtrmap(cur->bc_ops) || + xfs_btree_is_mem_rtrmap(cur->bc_ops)) + return xfs_rtrmap_check_irec(to_rtg(cur->bc_group), irec); + return xfs_rmap_check_irec(to_perag(cur->bc_group), irec); +} + +static inline int +xfs_rmap_complain_bad_rec( + struct xfs_btree_cur *cur, + xfs_failaddr_t fa, + const struct xfs_rmap_irec *irec) +{ + struct xfs_mount *mp = cur->bc_mp; + + if (xfs_btree_is_mem_rmap(cur->bc_ops)) + xfs_warn(mp, + "In-Memory Reverse Mapping BTree record corruption detected at %pS!", fa); + else if (xfs_btree_is_rtrmap(cur->bc_ops)) + xfs_warn(mp, + "RT Reverse Mapping BTree record corruption in rtgroup %u detected at %pS!", + cur->bc_group->xg_gno, fa); + else + xfs_warn(mp, + "Reverse Mapping BTree record corruption in AG %d detected at %pS!", + cur->bc_group->xg_gno, fa); + xfs_warn(mp, + "Owner 0x%llx, flags 0x%x, start block 0x%x block count 0x%x", + irec->rm_owner, irec->rm_flags, irec->rm_startblock, + irec->rm_blockcount); + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; +} + +/* + * Get the data from the pointed-to record. + */ +int +xfs_rmap_get_rec( + struct xfs_btree_cur *cur, + struct xfs_rmap_irec *irec, + int *stat) +{ + union xfs_btree_rec *rec; + xfs_failaddr_t fa; + int error; + + error = xfs_btree_get_rec(cur, &rec, stat); + if (error || !*stat) + return error; + + fa = xfs_rmap_btrec_to_irec(rec, irec); + if (!fa) + fa = xfs_rmap_check_btrec(cur, irec); + if (fa) + return xfs_rmap_complain_bad_rec(cur, fa, irec); + + return 0; +} + +struct xfs_find_left_neighbor_info { + struct xfs_rmap_irec high; + struct xfs_rmap_irec *irec; +}; + +/* For each rmap given, figure out if it matches the key we want. */ +STATIC int +xfs_rmap_find_left_neighbor_helper( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv) +{ + struct xfs_find_left_neighbor_info *info = priv; + + trace_xfs_rmap_find_left_neighbor_candidate(cur, rec->rm_startblock, + rec->rm_blockcount, rec->rm_owner, rec->rm_offset, + rec->rm_flags); + + if (rec->rm_owner != info->high.rm_owner) + return 0; + if (!XFS_RMAP_NON_INODE_OWNER(rec->rm_owner) && + !(rec->rm_flags & XFS_RMAP_BMBT_BLOCK) && + rec->rm_offset + rec->rm_blockcount - 1 != info->high.rm_offset) + return 0; + + *info->irec = *rec; + return -ECANCELED; +} + +/* + * Find the record to the left of the given extent, being careful only to + * return a match with the same owner and adjacent physical and logical + * block ranges. + */ +STATIC int +xfs_rmap_find_left_neighbor( + struct xfs_btree_cur *cur, + xfs_agblock_t bno, + uint64_t owner, + uint64_t offset, + unsigned int flags, + struct xfs_rmap_irec *irec, + int *stat) +{ + struct xfs_find_left_neighbor_info info; + int found = 0; + int error; + + *stat = 0; + if (bno == 0) + return 0; + info.high.rm_startblock = bno - 1; + info.high.rm_owner = owner; + if (!XFS_RMAP_NON_INODE_OWNER(owner) && + !(flags & XFS_RMAP_BMBT_BLOCK)) { + if (offset == 0) + return 0; + info.high.rm_offset = offset - 1; + } else + info.high.rm_offset = 0; + info.high.rm_flags = flags; + info.high.rm_blockcount = 0; + info.irec = irec; + + trace_xfs_rmap_find_left_neighbor_query(cur, bno, 0, owner, offset, + flags); + + /* + * Historically, we always used the range query to walk every reverse + * mapping that could possibly overlap the key that the caller asked + * for, and filter out the ones that don't. That is very slow when + * there are a lot of records. + * + * However, there are two scenarios where the classic btree search can + * produce correct results -- if the index contains a record that is an + * exact match for the lookup key; and if there are no other records + * between the record we want and the key we supplied. + * + * As an optimization, try a non-overlapped lookup first. This makes + * extent conversion and remap operations run a bit faster if the + * physical extents aren't being shared. If we don't find what we + * want, we fall back to the overlapped query. + */ + error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, irec, + &found); + if (error) + return error; + if (found) + error = xfs_rmap_find_left_neighbor_helper(cur, irec, &info); + if (!error) + error = xfs_rmap_query_range(cur, &info.high, &info.high, + xfs_rmap_find_left_neighbor_helper, &info); + if (error != -ECANCELED) + return error; + + *stat = 1; + trace_xfs_rmap_find_left_neighbor_result(cur, irec->rm_startblock, + irec->rm_blockcount, irec->rm_owner, irec->rm_offset, + irec->rm_flags); + return 0; +} + +/* For each rmap given, figure out if it matches the key we want. */ +STATIC int +xfs_rmap_lookup_le_range_helper( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv) +{ + struct xfs_find_left_neighbor_info *info = priv; + + trace_xfs_rmap_lookup_le_range_candidate(cur, rec->rm_startblock, + rec->rm_blockcount, rec->rm_owner, rec->rm_offset, + rec->rm_flags); + + if (rec->rm_owner != info->high.rm_owner) + return 0; + if (!XFS_RMAP_NON_INODE_OWNER(rec->rm_owner) && + !(rec->rm_flags & XFS_RMAP_BMBT_BLOCK) && + (rec->rm_offset > info->high.rm_offset || + rec->rm_offset + rec->rm_blockcount <= info->high.rm_offset)) + return 0; + + *info->irec = *rec; + return -ECANCELED; +} + +/* + * Find the record to the left of the given extent, being careful only to + * return a match with the same owner and overlapping physical and logical + * block ranges. This is the overlapping-interval version of + * xfs_rmap_lookup_le. + */ +int +xfs_rmap_lookup_le_range( + struct xfs_btree_cur *cur, + xfs_agblock_t bno, + uint64_t owner, + uint64_t offset, + unsigned int flags, + struct xfs_rmap_irec *irec, + int *stat) +{ + struct xfs_find_left_neighbor_info info; + int found = 0; + int error; + + info.high.rm_startblock = bno; + info.high.rm_owner = owner; + if (!XFS_RMAP_NON_INODE_OWNER(owner) && !(flags & XFS_RMAP_BMBT_BLOCK)) + info.high.rm_offset = offset; + else + info.high.rm_offset = 0; + info.high.rm_flags = flags; + info.high.rm_blockcount = 0; + *stat = 0; + info.irec = irec; + + trace_xfs_rmap_lookup_le_range(cur, bno, 0, owner, offset, flags); + + /* + * Historically, we always used the range query to walk every reverse + * mapping that could possibly overlap the key that the caller asked + * for, and filter out the ones that don't. That is very slow when + * there are a lot of records. + * + * However, there are two scenarios where the classic btree search can + * produce correct results -- if the index contains a record that is an + * exact match for the lookup key; and if there are no other records + * between the record we want and the key we supplied. + * + * As an optimization, try a non-overlapped lookup first. This makes + * scrub run much faster on most filesystems because bmbt records are + * usually an exact match for rmap records. If we don't find what we + * want, we fall back to the overlapped query. + */ + error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, irec, + &found); + if (error) + return error; + if (found) + error = xfs_rmap_lookup_le_range_helper(cur, irec, &info); + if (!error) + error = xfs_rmap_query_range(cur, &info.high, &info.high, + xfs_rmap_lookup_le_range_helper, &info); + if (error != -ECANCELED) + return error; + + *stat = 1; + trace_xfs_rmap_lookup_le_range_result(cur, irec->rm_startblock, + irec->rm_blockcount, irec->rm_owner, irec->rm_offset, + irec->rm_flags); + return 0; +} + +/* + * Perform all the relevant owner checks for a removal op. If we're doing an + * unknown-owner removal then we have no owner information to check. + */ +static int +xfs_rmap_free_check_owner( + struct xfs_btree_cur *cur, + uint64_t ltoff, + struct xfs_rmap_irec *rec, + xfs_extlen_t len, + uint64_t owner, + uint64_t offset, + unsigned int flags) +{ + struct xfs_mount *mp = cur->bc_mp; + int error = 0; + + if (owner == XFS_RMAP_OWN_UNKNOWN) + return 0; + + /* Make sure the unwritten flag matches. */ + if (XFS_IS_CORRUPT(mp, + (flags & XFS_RMAP_UNWRITTEN) != + (rec->rm_flags & XFS_RMAP_UNWRITTEN))) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out; + } + + /* Make sure the owner matches what we expect to find in the tree. */ + if (XFS_IS_CORRUPT(mp, owner != rec->rm_owner)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out; + } + + /* Check the offset, if necessary. */ + if (XFS_RMAP_NON_INODE_OWNER(owner)) + goto out; + + if (flags & XFS_RMAP_BMBT_BLOCK) { + if (XFS_IS_CORRUPT(mp, + !(rec->rm_flags & XFS_RMAP_BMBT_BLOCK))) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out; + } + } else { + if (XFS_IS_CORRUPT(mp, rec->rm_offset > offset)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out; + } + if (XFS_IS_CORRUPT(mp, + offset + len > ltoff + rec->rm_blockcount)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out; + } + } + +out: + return error; +} + +/* + * Find the extent in the rmap btree and remove it. + * + * The record we find should always be an exact match for the extent that we're + * looking for, since we insert them into the btree without modification. + * + * Special Case #1: when growing the filesystem, we "free" an extent when + * growing the last AG. This extent is new space and so it is not tracked as + * used space in the btree. The growfs code will pass in an owner of + * XFS_RMAP_OWN_NULL to indicate that it expected that there is no owner of this + * extent. We verify that - the extent lookup result in a record that does not + * overlap. + * + * Special Case #2: EFIs do not record the owner of the extent, so when + * recovering EFIs from the log we pass in XFS_RMAP_OWN_UNKNOWN to tell the rmap + * btree to ignore the owner (i.e. wildcard match) so we don't trigger + * corruption checks during log recovery. + */ +STATIC int +xfs_rmap_unmap( + struct xfs_btree_cur *cur, + xfs_agblock_t bno, + xfs_extlen_t len, + bool unwritten, + const struct xfs_owner_info *oinfo) +{ + struct xfs_mount *mp = cur->bc_mp; + struct xfs_rmap_irec ltrec; + uint64_t ltoff; + int error = 0; + int i; + uint64_t owner; + uint64_t offset; + unsigned int flags; + bool ignore_off; + + xfs_owner_info_unpack(oinfo, &owner, &offset, &flags); + ignore_off = XFS_RMAP_NON_INODE_OWNER(owner) || + (flags & XFS_RMAP_BMBT_BLOCK); + if (unwritten) + flags |= XFS_RMAP_UNWRITTEN; + trace_xfs_rmap_unmap(cur, bno, len, unwritten, oinfo); + + /* + * We should always have a left record because there's a static record + * for the AG headers at rm_startblock == 0 created by mkfs/growfs that + * will not ever be removed from the tree. + */ + error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, <rec, &i); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + + trace_xfs_rmap_lookup_le_range_result(cur, ltrec.rm_startblock, + ltrec.rm_blockcount, ltrec.rm_owner, ltrec.rm_offset, + ltrec.rm_flags); + ltoff = ltrec.rm_offset; + + /* + * For growfs, the incoming extent must be beyond the left record we + * just found as it is new space and won't be used by anyone. This is + * just a corruption check as we don't actually do anything with this + * extent. Note that we need to use >= instead of > because it might + * be the case that the "left" extent goes all the way to EOFS. + */ + if (owner == XFS_RMAP_OWN_NULL) { + if (XFS_IS_CORRUPT(mp, + bno < + ltrec.rm_startblock + ltrec.rm_blockcount)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + goto out_done; + } + + /* + * If we're doing an unknown-owner removal for EFI recovery, we expect + * to find the full range in the rmapbt or nothing at all. If we + * don't find any rmaps overlapping either end of the range, we're + * done. Hopefully this means that the EFI creator already queued + * (and finished) a RUI to remove the rmap. + */ + if (owner == XFS_RMAP_OWN_UNKNOWN && + ltrec.rm_startblock + ltrec.rm_blockcount <= bno) { + struct xfs_rmap_irec rtrec; + + error = xfs_btree_increment(cur, 0, &i); + if (error) + goto out_error; + if (i == 0) + goto out_done; + error = xfs_rmap_get_rec(cur, &rtrec, &i); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + if (rtrec.rm_startblock >= bno + len) + goto out_done; + } + + /* Make sure the extent we found covers the entire freeing range. */ + if (XFS_IS_CORRUPT(mp, + ltrec.rm_startblock > bno || + ltrec.rm_startblock + ltrec.rm_blockcount < + bno + len)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + + /* Check owner information. */ + error = xfs_rmap_free_check_owner(cur, ltoff, <rec, len, owner, + offset, flags); + if (error) + goto out_error; + + if (ltrec.rm_startblock == bno && ltrec.rm_blockcount == len) { + /* exact match, simply remove the record from rmap tree */ + trace_xfs_rmap_delete(cur, ltrec.rm_startblock, + ltrec.rm_blockcount, ltrec.rm_owner, + ltrec.rm_offset, ltrec.rm_flags); + error = xfs_btree_delete(cur, &i); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + } else if (ltrec.rm_startblock == bno) { + /* + * overlap left hand side of extent: move the start, trim the + * length and update the current record. + * + * ltbno ltlen + * Orig: |oooooooooooooooooooo| + * Freeing: |fffffffff| + * Result: |rrrrrrrrrr| + * bno len + */ + ltrec.rm_startblock += len; + ltrec.rm_blockcount -= len; + if (!ignore_off) + ltrec.rm_offset += len; + error = xfs_rmap_update(cur, <rec); + if (error) + goto out_error; + } else if (ltrec.rm_startblock + ltrec.rm_blockcount == bno + len) { + /* + * overlap right hand side of extent: trim the length and update + * the current record. + * + * ltbno ltlen + * Orig: |oooooooooooooooooooo| + * Freeing: |fffffffff| + * Result: |rrrrrrrrrr| + * bno len + */ + ltrec.rm_blockcount -= len; + error = xfs_rmap_update(cur, <rec); + if (error) + goto out_error; + } else { + + /* + * overlap middle of extent: trim the length of the existing + * record to the length of the new left-extent size, increment + * the insertion position so we can insert a new record + * containing the remaining right-extent space. + * + * ltbno ltlen + * Orig: |oooooooooooooooooooo| + * Freeing: |fffffffff| + * Result: |rrrrr| |rrrr| + * bno len + */ + xfs_extlen_t orig_len = ltrec.rm_blockcount; + + ltrec.rm_blockcount = bno - ltrec.rm_startblock; + error = xfs_rmap_update(cur, <rec); + if (error) + goto out_error; + + error = xfs_btree_increment(cur, 0, &i); + if (error) + goto out_error; + + cur->bc_rec.r.rm_startblock = bno + len; + cur->bc_rec.r.rm_blockcount = orig_len - len - + ltrec.rm_blockcount; + cur->bc_rec.r.rm_owner = ltrec.rm_owner; + if (ignore_off) + cur->bc_rec.r.rm_offset = 0; + else + cur->bc_rec.r.rm_offset = offset + len; + cur->bc_rec.r.rm_flags = flags; + trace_xfs_rmap_insert(cur, cur->bc_rec.r.rm_startblock, + cur->bc_rec.r.rm_blockcount, + cur->bc_rec.r.rm_owner, + cur->bc_rec.r.rm_offset, + cur->bc_rec.r.rm_flags); + error = xfs_btree_insert(cur, &i); + if (error) + goto out_error; + } + +out_done: + trace_xfs_rmap_unmap_done(cur, bno, len, unwritten, oinfo); +out_error: + if (error) + trace_xfs_rmap_unmap_error(cur, error, _RET_IP_); + return error; +} + +#ifdef CONFIG_XFS_LIVE_HOOKS +/* + * Use a static key here to reduce the overhead of rmapbt live updates. If + * the compiler supports jump labels, the static branch will be replaced by a + * nop sled when there are no hook users. Online fsck is currently the only + * caller, so this is a reasonable tradeoff. + * + * Note: Patching the kernel code requires taking the cpu hotplug lock. Other + * parts of the kernel allocate memory with that lock held, which means that + * XFS callers cannot hold any locks that might be used by memory reclaim or + * writeback when calling the static_branch_{inc,dec} functions. + */ +DEFINE_STATIC_XFS_HOOK_SWITCH(xfs_rmap_hooks_switch); + +void +xfs_rmap_hook_disable(void) +{ + xfs_hooks_switch_off(&xfs_rmap_hooks_switch); +} + +void +xfs_rmap_hook_enable(void) +{ + xfs_hooks_switch_on(&xfs_rmap_hooks_switch); +} + +/* Call downstream hooks for a reverse mapping update. */ +static inline void +xfs_rmap_update_hook( + struct xfs_trans *tp, + struct xfs_group *xg, + enum xfs_rmap_intent_type op, + xfs_agblock_t startblock, + xfs_extlen_t blockcount, + bool unwritten, + const struct xfs_owner_info *oinfo) +{ + if (xfs_hooks_switched_on(&xfs_rmap_hooks_switch)) { + struct xfs_rmap_update_params p = { + .startblock = startblock, + .blockcount = blockcount, + .unwritten = unwritten, + .oinfo = *oinfo, /* struct copy */ + }; + + if (xg) + xfs_hooks_call(&xg->xg_rmap_update_hooks, op, &p); + } +} + +/* Call the specified function during a reverse mapping update. */ +int +xfs_rmap_hook_add( + struct xfs_group *xg, + struct xfs_rmap_hook *hook) +{ + return xfs_hooks_add(&xg->xg_rmap_update_hooks, &hook->rmap_hook); +} + +/* Stop calling the specified function during a reverse mapping update. */ +void +xfs_rmap_hook_del( + struct xfs_group *xg, + struct xfs_rmap_hook *hook) +{ + xfs_hooks_del(&xg->xg_rmap_update_hooks, &hook->rmap_hook); +} + +/* Configure rmap update hook functions. */ +void +xfs_rmap_hook_setup( + struct xfs_rmap_hook *hook, + notifier_fn_t mod_fn) +{ + xfs_hook_setup(&hook->rmap_hook, mod_fn); +} +#else +# define xfs_rmap_update_hook(t, p, o, s, b, u, oi) do { } while (0) +#endif /* CONFIG_XFS_LIVE_HOOKS */ + +/* + * Remove a reference to an extent in the rmap btree. + */ +int +xfs_rmap_free( + struct xfs_trans *tp, + struct xfs_buf *agbp, + struct xfs_perag *pag, + xfs_agblock_t bno, + xfs_extlen_t len, + const struct xfs_owner_info *oinfo) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_btree_cur *cur; + int error; + + if (!xfs_has_rmapbt(mp)) + return 0; + + cur = xfs_rmapbt_init_cursor(mp, tp, agbp, pag); + xfs_rmap_update_hook(tp, pag_group(pag), XFS_RMAP_UNMAP, bno, len, + false, oinfo); + error = xfs_rmap_unmap(cur, bno, len, false, oinfo); + + xfs_btree_del_cursor(cur, error); + return error; +} + +/* + * A mergeable rmap must have the same owner and the same values for + * the unwritten, attr_fork, and bmbt flags. The startblock and + * offset are checked separately. + */ +static bool +xfs_rmap_is_mergeable( + struct xfs_rmap_irec *irec, + uint64_t owner, + unsigned int flags) +{ + if (irec->rm_owner == XFS_RMAP_OWN_NULL) + return false; + if (irec->rm_owner != owner) + return false; + if ((flags & XFS_RMAP_UNWRITTEN) ^ + (irec->rm_flags & XFS_RMAP_UNWRITTEN)) + return false; + if ((flags & XFS_RMAP_ATTR_FORK) ^ + (irec->rm_flags & XFS_RMAP_ATTR_FORK)) + return false; + if ((flags & XFS_RMAP_BMBT_BLOCK) ^ + (irec->rm_flags & XFS_RMAP_BMBT_BLOCK)) + return false; + return true; +} + +/* + * When we allocate a new block, the first thing we do is add a reference to + * the extent in the rmap btree. This takes the form of a [agbno, length, + * owner, offset] record. Flags are encoded in the high bits of the offset + * field. + */ +STATIC int +xfs_rmap_map( + struct xfs_btree_cur *cur, + xfs_agblock_t bno, + xfs_extlen_t len, + bool unwritten, + const struct xfs_owner_info *oinfo) +{ + struct xfs_mount *mp = cur->bc_mp; + struct xfs_rmap_irec ltrec; + struct xfs_rmap_irec gtrec; + int have_gt; + int have_lt; + int error = 0; + int i; + uint64_t owner; + uint64_t offset; + unsigned int flags = 0; + bool ignore_off; + + xfs_owner_info_unpack(oinfo, &owner, &offset, &flags); + ASSERT(owner != 0); + ignore_off = XFS_RMAP_NON_INODE_OWNER(owner) || + (flags & XFS_RMAP_BMBT_BLOCK); + if (unwritten) + flags |= XFS_RMAP_UNWRITTEN; + trace_xfs_rmap_map(cur, bno, len, unwritten, oinfo); + ASSERT(!xfs_rmap_should_skip_owner_update(oinfo)); + + /* + * For the initial lookup, look for an exact match or the left-adjacent + * record for our insertion point. This will also give us the record for + * start block contiguity tests. + */ + error = xfs_rmap_lookup_le(cur, bno, owner, offset, flags, <rec, + &have_lt); + if (error) + goto out_error; + if (have_lt) { + trace_xfs_rmap_lookup_le_range_result(cur, ltrec.rm_startblock, + ltrec.rm_blockcount, ltrec.rm_owner, + ltrec.rm_offset, ltrec.rm_flags); + + if (!xfs_rmap_is_mergeable(<rec, owner, flags)) + have_lt = 0; + } + + if (XFS_IS_CORRUPT(mp, + have_lt != 0 && + ltrec.rm_startblock + ltrec.rm_blockcount > bno)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + + /* + * Increment the cursor to see if we have a right-adjacent record to our + * insertion point. This will give us the record for end block + * contiguity tests. + */ + error = xfs_btree_increment(cur, 0, &have_gt); + if (error) + goto out_error; + if (have_gt) { + error = xfs_rmap_get_rec(cur, >rec, &have_gt); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(mp, have_gt != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + if (XFS_IS_CORRUPT(mp, bno + len > gtrec.rm_startblock)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + trace_xfs_rmap_find_right_neighbor_result(cur, + gtrec.rm_startblock, gtrec.rm_blockcount, + gtrec.rm_owner, gtrec.rm_offset, + gtrec.rm_flags); + if (!xfs_rmap_is_mergeable(>rec, owner, flags)) + have_gt = 0; + } + + /* + * Note: cursor currently points one record to the right of ltrec, even + * if there is no record in the tree to the right. + */ + if (have_lt && + ltrec.rm_startblock + ltrec.rm_blockcount == bno && + (ignore_off || ltrec.rm_offset + ltrec.rm_blockcount == offset)) { + /* + * left edge contiguous, merge into left record. + * + * ltbno ltlen + * orig: |ooooooooo| + * adding: |aaaaaaaaa| + * result: |rrrrrrrrrrrrrrrrrrr| + * bno len + */ + ltrec.rm_blockcount += len; + if (have_gt && + bno + len == gtrec.rm_startblock && + (ignore_off || offset + len == gtrec.rm_offset) && + (unsigned long)ltrec.rm_blockcount + len + + gtrec.rm_blockcount <= XFS_RMAP_LEN_MAX) { + /* + * right edge also contiguous, delete right record + * and merge into left record. + * + * ltbno ltlen gtbno gtlen + * orig: |ooooooooo| |ooooooooo| + * adding: |aaaaaaaaa| + * result: |rrrrrrrrrrrrrrrrrrrrrrrrrrrrr| + */ + ltrec.rm_blockcount += gtrec.rm_blockcount; + trace_xfs_rmap_delete(cur, gtrec.rm_startblock, + gtrec.rm_blockcount, gtrec.rm_owner, + gtrec.rm_offset, gtrec.rm_flags); + error = xfs_btree_delete(cur, &i); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + } + + /* point the cursor back to the left record and update */ + error = xfs_btree_decrement(cur, 0, &have_gt); + if (error) + goto out_error; + error = xfs_rmap_update(cur, <rec); + if (error) + goto out_error; + } else if (have_gt && + bno + len == gtrec.rm_startblock && + (ignore_off || offset + len == gtrec.rm_offset)) { + /* + * right edge contiguous, merge into right record. + * + * gtbno gtlen + * Orig: |ooooooooo| + * adding: |aaaaaaaaa| + * Result: |rrrrrrrrrrrrrrrrrrr| + * bno len + */ + gtrec.rm_startblock = bno; + gtrec.rm_blockcount += len; + if (!ignore_off) + gtrec.rm_offset = offset; + error = xfs_rmap_update(cur, >rec); + if (error) + goto out_error; + } else { + /* + * no contiguous edge with identical owner, insert + * new record at current cursor position. + */ + cur->bc_rec.r.rm_startblock = bno; + cur->bc_rec.r.rm_blockcount = len; + cur->bc_rec.r.rm_owner = owner; + cur->bc_rec.r.rm_offset = offset; + cur->bc_rec.r.rm_flags = flags; + trace_xfs_rmap_insert(cur, bno, len, owner, offset, flags); + error = xfs_btree_insert(cur, &i); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + } + + trace_xfs_rmap_map_done(cur, bno, len, unwritten, oinfo); +out_error: + if (error) + trace_xfs_rmap_map_error(cur, error, _RET_IP_); + return error; +} + +/* + * Add a reference to an extent in the rmap btree. + */ +int +xfs_rmap_alloc( + struct xfs_trans *tp, + struct xfs_buf *agbp, + struct xfs_perag *pag, + xfs_agblock_t bno, + xfs_extlen_t len, + const struct xfs_owner_info *oinfo) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_btree_cur *cur; + int error; + + if (!xfs_has_rmapbt(mp)) + return 0; + + cur = xfs_rmapbt_init_cursor(mp, tp, agbp, pag); + xfs_rmap_update_hook(tp, pag_group(pag), XFS_RMAP_MAP, bno, len, false, + oinfo); + error = xfs_rmap_map(cur, bno, len, false, oinfo); + + xfs_btree_del_cursor(cur, error); + return error; +} + +#define RMAP_LEFT_CONTIG (1 << 0) +#define RMAP_RIGHT_CONTIG (1 << 1) +#define RMAP_LEFT_FILLING (1 << 2) +#define RMAP_RIGHT_FILLING (1 << 3) +#define RMAP_LEFT_VALID (1 << 6) +#define RMAP_RIGHT_VALID (1 << 7) + +#define LEFT r[0] +#define RIGHT r[1] +#define PREV r[2] +#define NEW r[3] + +/* + * Convert an unwritten extent to a real extent or vice versa. + * Does not handle overlapping extents. + */ +STATIC int +xfs_rmap_convert( + struct xfs_btree_cur *cur, + xfs_agblock_t bno, + xfs_extlen_t len, + bool unwritten, + const struct xfs_owner_info *oinfo) +{ + struct xfs_mount *mp = cur->bc_mp; + struct xfs_rmap_irec r[4]; /* neighbor extent entries */ + /* left is 0, right is 1, */ + /* prev is 2, new is 3 */ + uint64_t owner; + uint64_t offset; + uint64_t new_endoff; + unsigned int oldext; + unsigned int newext; + unsigned int flags = 0; + int i; + int state = 0; + int error; + + xfs_owner_info_unpack(oinfo, &owner, &offset, &flags); + ASSERT(!(XFS_RMAP_NON_INODE_OWNER(owner) || + (flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK)))); + oldext = unwritten ? XFS_RMAP_UNWRITTEN : 0; + new_endoff = offset + len; + trace_xfs_rmap_convert(cur, bno, len, unwritten, oinfo); + + /* + * For the initial lookup, look for an exact match or the left-adjacent + * record for our insertion point. This will also give us the record for + * start block contiguity tests. + */ + error = xfs_rmap_lookup_le(cur, bno, owner, offset, oldext, &PREV, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + + trace_xfs_rmap_lookup_le_range_result(cur, PREV.rm_startblock, + PREV.rm_blockcount, PREV.rm_owner, PREV.rm_offset, + PREV.rm_flags); + + ASSERT(PREV.rm_offset <= offset); + ASSERT(PREV.rm_offset + PREV.rm_blockcount >= new_endoff); + ASSERT((PREV.rm_flags & XFS_RMAP_UNWRITTEN) == oldext); + newext = ~oldext & XFS_RMAP_UNWRITTEN; + + /* + * Set flags determining what part of the previous oldext allocation + * extent is being replaced by a newext allocation. + */ + if (PREV.rm_offset == offset) + state |= RMAP_LEFT_FILLING; + if (PREV.rm_offset + PREV.rm_blockcount == new_endoff) + state |= RMAP_RIGHT_FILLING; + + /* + * Decrement the cursor to see if we have a left-adjacent record to our + * insertion point. This will give us the record for end block + * contiguity tests. + */ + error = xfs_btree_decrement(cur, 0, &i); + if (error) + goto done; + if (i) { + state |= RMAP_LEFT_VALID; + error = xfs_rmap_get_rec(cur, &LEFT, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + if (XFS_IS_CORRUPT(mp, + LEFT.rm_startblock + LEFT.rm_blockcount > + bno)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + trace_xfs_rmap_find_left_neighbor_result(cur, + LEFT.rm_startblock, LEFT.rm_blockcount, + LEFT.rm_owner, LEFT.rm_offset, LEFT.rm_flags); + if (LEFT.rm_startblock + LEFT.rm_blockcount == bno && + LEFT.rm_offset + LEFT.rm_blockcount == offset && + xfs_rmap_is_mergeable(&LEFT, owner, newext)) + state |= RMAP_LEFT_CONTIG; + } + + /* + * Increment the cursor to see if we have a right-adjacent record to our + * insertion point. This will give us the record for end block + * contiguity tests. + */ + error = xfs_btree_increment(cur, 0, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_btree_increment(cur, 0, &i); + if (error) + goto done; + if (i) { + state |= RMAP_RIGHT_VALID; + error = xfs_rmap_get_rec(cur, &RIGHT, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + if (XFS_IS_CORRUPT(mp, bno + len > RIGHT.rm_startblock)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + trace_xfs_rmap_find_right_neighbor_result(cur, + RIGHT.rm_startblock, RIGHT.rm_blockcount, + RIGHT.rm_owner, RIGHT.rm_offset, + RIGHT.rm_flags); + if (bno + len == RIGHT.rm_startblock && + offset + len == RIGHT.rm_offset && + xfs_rmap_is_mergeable(&RIGHT, owner, newext)) + state |= RMAP_RIGHT_CONTIG; + } + + /* check that left + prev + right is not too long */ + if ((state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG | + RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) == + (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG | + RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG) && + (unsigned long)LEFT.rm_blockcount + len + + RIGHT.rm_blockcount > XFS_RMAP_LEN_MAX) + state &= ~RMAP_RIGHT_CONTIG; + + trace_xfs_rmap_convert_state(cur, state, _RET_IP_); + + /* reset the cursor back to PREV */ + error = xfs_rmap_lookup_le(cur, bno, owner, offset, oldext, NULL, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + + /* + * Switch out based on the FILLING and CONTIG state bits. + */ + switch (state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG | + RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) { + case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG | + RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG: + /* + * Setting all of a previous oldext extent to newext. + * The left and right neighbors are both contiguous with new. + */ + error = xfs_btree_increment(cur, 0, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + trace_xfs_rmap_delete(cur, RIGHT.rm_startblock, + RIGHT.rm_blockcount, RIGHT.rm_owner, + RIGHT.rm_offset, RIGHT.rm_flags); + error = xfs_btree_delete(cur, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_btree_decrement(cur, 0, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + trace_xfs_rmap_delete(cur, PREV.rm_startblock, + PREV.rm_blockcount, PREV.rm_owner, + PREV.rm_offset, PREV.rm_flags); + error = xfs_btree_delete(cur, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_btree_decrement(cur, 0, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + NEW = LEFT; + NEW.rm_blockcount += PREV.rm_blockcount + RIGHT.rm_blockcount; + error = xfs_rmap_update(cur, &NEW); + if (error) + goto done; + break; + + case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG: + /* + * Setting all of a previous oldext extent to newext. + * The left neighbor is contiguous, the right is not. + */ + trace_xfs_rmap_delete(cur, PREV.rm_startblock, + PREV.rm_blockcount, PREV.rm_owner, + PREV.rm_offset, PREV.rm_flags); + error = xfs_btree_delete(cur, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_btree_decrement(cur, 0, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + NEW = LEFT; + NEW.rm_blockcount += PREV.rm_blockcount; + error = xfs_rmap_update(cur, &NEW); + if (error) + goto done; + break; + + case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG: + /* + * Setting all of a previous oldext extent to newext. + * The right neighbor is contiguous, the left is not. + */ + error = xfs_btree_increment(cur, 0, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + trace_xfs_rmap_delete(cur, RIGHT.rm_startblock, + RIGHT.rm_blockcount, RIGHT.rm_owner, + RIGHT.rm_offset, RIGHT.rm_flags); + error = xfs_btree_delete(cur, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + error = xfs_btree_decrement(cur, 0, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + NEW = PREV; + NEW.rm_blockcount = len + RIGHT.rm_blockcount; + NEW.rm_flags = newext; + error = xfs_rmap_update(cur, &NEW); + if (error) + goto done; + break; + + case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING: + /* + * Setting all of a previous oldext extent to newext. + * Neither the left nor right neighbors are contiguous with + * the new one. + */ + NEW = PREV; + NEW.rm_flags = newext; + error = xfs_rmap_update(cur, &NEW); + if (error) + goto done; + break; + + case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG: + /* + * Setting the first part of a previous oldext extent to newext. + * The left neighbor is contiguous. + */ + NEW = PREV; + NEW.rm_offset += len; + NEW.rm_startblock += len; + NEW.rm_blockcount -= len; + error = xfs_rmap_update(cur, &NEW); + if (error) + goto done; + error = xfs_btree_decrement(cur, 0, &i); + if (error) + goto done; + NEW = LEFT; + NEW.rm_blockcount += len; + error = xfs_rmap_update(cur, &NEW); + if (error) + goto done; + break; + + case RMAP_LEFT_FILLING: + /* + * Setting the first part of a previous oldext extent to newext. + * The left neighbor is not contiguous. + */ + NEW = PREV; + NEW.rm_startblock += len; + NEW.rm_offset += len; + NEW.rm_blockcount -= len; + error = xfs_rmap_update(cur, &NEW); + if (error) + goto done; + NEW.rm_startblock = bno; + NEW.rm_owner = owner; + NEW.rm_offset = offset; + NEW.rm_blockcount = len; + NEW.rm_flags = newext; + cur->bc_rec.r = NEW; + trace_xfs_rmap_insert(cur, bno, len, owner, offset, newext); + error = xfs_btree_insert(cur, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + break; + + case RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG: + /* + * Setting the last part of a previous oldext extent to newext. + * The right neighbor is contiguous with the new allocation. + */ + NEW = PREV; + NEW.rm_blockcount -= len; + error = xfs_rmap_update(cur, &NEW); + if (error) + goto done; + error = xfs_btree_increment(cur, 0, &i); + if (error) + goto done; + NEW = RIGHT; + NEW.rm_offset = offset; + NEW.rm_startblock = bno; + NEW.rm_blockcount += len; + error = xfs_rmap_update(cur, &NEW); + if (error) + goto done; + break; + + case RMAP_RIGHT_FILLING: + /* + * Setting the last part of a previous oldext extent to newext. + * The right neighbor is not contiguous. + */ + NEW = PREV; + NEW.rm_blockcount -= len; + error = xfs_rmap_update(cur, &NEW); + if (error) + goto done; + error = xfs_rmap_lookup_eq(cur, bno, len, owner, offset, + oldext, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 0)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + NEW.rm_startblock = bno; + NEW.rm_owner = owner; + NEW.rm_offset = offset; + NEW.rm_blockcount = len; + NEW.rm_flags = newext; + cur->bc_rec.r = NEW; + trace_xfs_rmap_insert(cur, bno, len, owner, offset, newext); + error = xfs_btree_insert(cur, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + break; + + case 0: + /* + * Setting the middle part of a previous oldext extent to + * newext. Contiguity is impossible here. + * One extent becomes three extents. + */ + /* new right extent - oldext */ + NEW.rm_startblock = bno + len; + NEW.rm_owner = owner; + NEW.rm_offset = new_endoff; + NEW.rm_blockcount = PREV.rm_offset + PREV.rm_blockcount - + new_endoff; + NEW.rm_flags = PREV.rm_flags; + error = xfs_rmap_update(cur, &NEW); + if (error) + goto done; + /* new left extent - oldext */ + NEW = PREV; + NEW.rm_blockcount = offset - PREV.rm_offset; + cur->bc_rec.r = NEW; + trace_xfs_rmap_insert(cur, NEW.rm_startblock, + NEW.rm_blockcount, NEW.rm_owner, NEW.rm_offset, + NEW.rm_flags); + error = xfs_btree_insert(cur, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + /* + * Reset the cursor to the position of the new extent + * we are about to insert as we can't trust it after + * the previous insert. + */ + error = xfs_rmap_lookup_eq(cur, bno, len, owner, offset, + oldext, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 0)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + /* new middle extent - newext */ + cur->bc_rec.r.rm_flags &= ~XFS_RMAP_UNWRITTEN; + cur->bc_rec.r.rm_flags |= newext; + trace_xfs_rmap_insert(cur, bno, len, owner, offset, newext); + error = xfs_btree_insert(cur, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + break; + + case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG: + case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG: + case RMAP_LEFT_FILLING | RMAP_RIGHT_CONTIG: + case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG: + case RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG: + case RMAP_LEFT_CONTIG: + case RMAP_RIGHT_CONTIG: + /* + * These cases are all impossible. + */ + ASSERT(0); + } + + trace_xfs_rmap_convert_done(cur, bno, len, unwritten, oinfo); +done: + if (error) + trace_xfs_rmap_convert_error(cur, error, _RET_IP_); + return error; +} + +/* + * Convert an unwritten extent to a real extent or vice versa. If there is no + * possibility of overlapping extents, delegate to the simpler convert + * function. + */ +STATIC int +xfs_rmap_convert_shared( + struct xfs_btree_cur *cur, + xfs_agblock_t bno, + xfs_extlen_t len, + bool unwritten, + const struct xfs_owner_info *oinfo) +{ + struct xfs_mount *mp = cur->bc_mp; + struct xfs_rmap_irec r[4]; /* neighbor extent entries */ + /* left is 0, right is 1, */ + /* prev is 2, new is 3 */ + uint64_t owner; + uint64_t offset; + uint64_t new_endoff; + unsigned int oldext; + unsigned int newext; + unsigned int flags = 0; + int i; + int state = 0; + int error; + + xfs_owner_info_unpack(oinfo, &owner, &offset, &flags); + ASSERT(!(XFS_RMAP_NON_INODE_OWNER(owner) || + (flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK)))); + oldext = unwritten ? XFS_RMAP_UNWRITTEN : 0; + new_endoff = offset + len; + trace_xfs_rmap_convert(cur, bno, len, unwritten, oinfo); + + /* + * For the initial lookup, look for and exact match or the left-adjacent + * record for our insertion point. This will also give us the record for + * start block contiguity tests. + */ + error = xfs_rmap_lookup_le_range(cur, bno, owner, offset, oldext, + &PREV, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + + ASSERT(PREV.rm_offset <= offset); + ASSERT(PREV.rm_offset + PREV.rm_blockcount >= new_endoff); + ASSERT((PREV.rm_flags & XFS_RMAP_UNWRITTEN) == oldext); + newext = ~oldext & XFS_RMAP_UNWRITTEN; + + /* + * Set flags determining what part of the previous oldext allocation + * extent is being replaced by a newext allocation. + */ + if (PREV.rm_offset == offset) + state |= RMAP_LEFT_FILLING; + if (PREV.rm_offset + PREV.rm_blockcount == new_endoff) + state |= RMAP_RIGHT_FILLING; + + /* Is there a left record that abuts our range? */ + error = xfs_rmap_find_left_neighbor(cur, bno, owner, offset, newext, + &LEFT, &i); + if (error) + goto done; + if (i) { + state |= RMAP_LEFT_VALID; + if (XFS_IS_CORRUPT(mp, + LEFT.rm_startblock + LEFT.rm_blockcount > + bno)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + if (xfs_rmap_is_mergeable(&LEFT, owner, newext)) + state |= RMAP_LEFT_CONTIG; + } + + /* Is there a right record that abuts our range? */ + error = xfs_rmap_lookup_eq(cur, bno + len, len, owner, offset + len, + newext, &i); + if (error) + goto done; + if (i) { + state |= RMAP_RIGHT_VALID; + error = xfs_rmap_get_rec(cur, &RIGHT, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + if (XFS_IS_CORRUPT(mp, bno + len > RIGHT.rm_startblock)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + trace_xfs_rmap_find_right_neighbor_result(cur, + RIGHT.rm_startblock, RIGHT.rm_blockcount, + RIGHT.rm_owner, RIGHT.rm_offset, + RIGHT.rm_flags); + if (xfs_rmap_is_mergeable(&RIGHT, owner, newext)) + state |= RMAP_RIGHT_CONTIG; + } + + /* check that left + prev + right is not too long */ + if ((state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG | + RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) == + (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG | + RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG) && + (unsigned long)LEFT.rm_blockcount + len + + RIGHT.rm_blockcount > XFS_RMAP_LEN_MAX) + state &= ~RMAP_RIGHT_CONTIG; + + trace_xfs_rmap_convert_state(cur, state, _RET_IP_); + /* + * Switch out based on the FILLING and CONTIG state bits. + */ + switch (state & (RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG | + RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG)) { + case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG | + RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG: + /* + * Setting all of a previous oldext extent to newext. + * The left and right neighbors are both contiguous with new. + */ + error = xfs_rmap_delete(cur, RIGHT.rm_startblock, + RIGHT.rm_blockcount, RIGHT.rm_owner, + RIGHT.rm_offset, RIGHT.rm_flags); + if (error) + goto done; + error = xfs_rmap_delete(cur, PREV.rm_startblock, + PREV.rm_blockcount, PREV.rm_owner, + PREV.rm_offset, PREV.rm_flags); + if (error) + goto done; + NEW = LEFT; + error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock, + NEW.rm_blockcount, NEW.rm_owner, + NEW.rm_offset, NEW.rm_flags, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + NEW.rm_blockcount += PREV.rm_blockcount + RIGHT.rm_blockcount; + error = xfs_rmap_update(cur, &NEW); + if (error) + goto done; + break; + + case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG: + /* + * Setting all of a previous oldext extent to newext. + * The left neighbor is contiguous, the right is not. + */ + error = xfs_rmap_delete(cur, PREV.rm_startblock, + PREV.rm_blockcount, PREV.rm_owner, + PREV.rm_offset, PREV.rm_flags); + if (error) + goto done; + NEW = LEFT; + error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock, + NEW.rm_blockcount, NEW.rm_owner, + NEW.rm_offset, NEW.rm_flags, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + NEW.rm_blockcount += PREV.rm_blockcount; + error = xfs_rmap_update(cur, &NEW); + if (error) + goto done; + break; + + case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG: + /* + * Setting all of a previous oldext extent to newext. + * The right neighbor is contiguous, the left is not. + */ + error = xfs_rmap_delete(cur, RIGHT.rm_startblock, + RIGHT.rm_blockcount, RIGHT.rm_owner, + RIGHT.rm_offset, RIGHT.rm_flags); + if (error) + goto done; + NEW = PREV; + error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock, + NEW.rm_blockcount, NEW.rm_owner, + NEW.rm_offset, NEW.rm_flags, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + NEW.rm_blockcount += RIGHT.rm_blockcount; + NEW.rm_flags = RIGHT.rm_flags; + error = xfs_rmap_update(cur, &NEW); + if (error) + goto done; + break; + + case RMAP_LEFT_FILLING | RMAP_RIGHT_FILLING: + /* + * Setting all of a previous oldext extent to newext. + * Neither the left nor right neighbors are contiguous with + * the new one. + */ + NEW = PREV; + error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock, + NEW.rm_blockcount, NEW.rm_owner, + NEW.rm_offset, NEW.rm_flags, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + NEW.rm_flags = newext; + error = xfs_rmap_update(cur, &NEW); + if (error) + goto done; + break; + + case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG: + /* + * Setting the first part of a previous oldext extent to newext. + * The left neighbor is contiguous. + */ + NEW = PREV; + error = xfs_rmap_delete(cur, NEW.rm_startblock, + NEW.rm_blockcount, NEW.rm_owner, + NEW.rm_offset, NEW.rm_flags); + if (error) + goto done; + NEW.rm_offset += len; + NEW.rm_startblock += len; + NEW.rm_blockcount -= len; + error = xfs_rmap_insert(cur, NEW.rm_startblock, + NEW.rm_blockcount, NEW.rm_owner, + NEW.rm_offset, NEW.rm_flags); + if (error) + goto done; + NEW = LEFT; + error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock, + NEW.rm_blockcount, NEW.rm_owner, + NEW.rm_offset, NEW.rm_flags, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + NEW.rm_blockcount += len; + error = xfs_rmap_update(cur, &NEW); + if (error) + goto done; + break; + + case RMAP_LEFT_FILLING: + /* + * Setting the first part of a previous oldext extent to newext. + * The left neighbor is not contiguous. + */ + NEW = PREV; + error = xfs_rmap_delete(cur, NEW.rm_startblock, + NEW.rm_blockcount, NEW.rm_owner, + NEW.rm_offset, NEW.rm_flags); + if (error) + goto done; + NEW.rm_offset += len; + NEW.rm_startblock += len; + NEW.rm_blockcount -= len; + error = xfs_rmap_insert(cur, NEW.rm_startblock, + NEW.rm_blockcount, NEW.rm_owner, + NEW.rm_offset, NEW.rm_flags); + if (error) + goto done; + error = xfs_rmap_insert(cur, bno, len, owner, offset, newext); + if (error) + goto done; + break; + + case RMAP_RIGHT_FILLING | RMAP_RIGHT_CONTIG: + /* + * Setting the last part of a previous oldext extent to newext. + * The right neighbor is contiguous with the new allocation. + */ + NEW = PREV; + error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock, + NEW.rm_blockcount, NEW.rm_owner, + NEW.rm_offset, NEW.rm_flags, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + NEW.rm_blockcount = offset - NEW.rm_offset; + error = xfs_rmap_update(cur, &NEW); + if (error) + goto done; + NEW = RIGHT; + error = xfs_rmap_delete(cur, NEW.rm_startblock, + NEW.rm_blockcount, NEW.rm_owner, + NEW.rm_offset, NEW.rm_flags); + if (error) + goto done; + NEW.rm_offset = offset; + NEW.rm_startblock = bno; + NEW.rm_blockcount += len; + error = xfs_rmap_insert(cur, NEW.rm_startblock, + NEW.rm_blockcount, NEW.rm_owner, + NEW.rm_offset, NEW.rm_flags); + if (error) + goto done; + break; + + case RMAP_RIGHT_FILLING: + /* + * Setting the last part of a previous oldext extent to newext. + * The right neighbor is not contiguous. + */ + NEW = PREV; + error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock, + NEW.rm_blockcount, NEW.rm_owner, + NEW.rm_offset, NEW.rm_flags, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + NEW.rm_blockcount -= len; + error = xfs_rmap_update(cur, &NEW); + if (error) + goto done; + error = xfs_rmap_insert(cur, bno, len, owner, offset, newext); + if (error) + goto done; + break; + + case 0: + /* + * Setting the middle part of a previous oldext extent to + * newext. Contiguity is impossible here. + * One extent becomes three extents. + */ + /* new right extent - oldext */ + NEW.rm_startblock = bno + len; + NEW.rm_owner = owner; + NEW.rm_offset = new_endoff; + NEW.rm_blockcount = PREV.rm_offset + PREV.rm_blockcount - + new_endoff; + NEW.rm_flags = PREV.rm_flags; + error = xfs_rmap_insert(cur, NEW.rm_startblock, + NEW.rm_blockcount, NEW.rm_owner, NEW.rm_offset, + NEW.rm_flags); + if (error) + goto done; + /* new left extent - oldext */ + NEW = PREV; + error = xfs_rmap_lookup_eq(cur, NEW.rm_startblock, + NEW.rm_blockcount, NEW.rm_owner, + NEW.rm_offset, NEW.rm_flags, &i); + if (error) + goto done; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto done; + } + NEW.rm_blockcount = offset - NEW.rm_offset; + error = xfs_rmap_update(cur, &NEW); + if (error) + goto done; + /* new middle extent - newext */ + NEW.rm_startblock = bno; + NEW.rm_blockcount = len; + NEW.rm_owner = owner; + NEW.rm_offset = offset; + NEW.rm_flags = newext; + error = xfs_rmap_insert(cur, NEW.rm_startblock, + NEW.rm_blockcount, NEW.rm_owner, NEW.rm_offset, + NEW.rm_flags); + if (error) + goto done; + break; + + case RMAP_LEFT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG: + case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG: + case RMAP_LEFT_FILLING | RMAP_RIGHT_CONTIG: + case RMAP_RIGHT_FILLING | RMAP_LEFT_CONTIG: + case RMAP_LEFT_CONTIG | RMAP_RIGHT_CONTIG: + case RMAP_LEFT_CONTIG: + case RMAP_RIGHT_CONTIG: + /* + * These cases are all impossible. + */ + ASSERT(0); + } + + trace_xfs_rmap_convert_done(cur, bno, len, unwritten, oinfo); +done: + if (error) + trace_xfs_rmap_convert_error(cur, error, _RET_IP_); + return error; +} + +#undef NEW +#undef LEFT +#undef RIGHT +#undef PREV + +/* + * Find an extent in the rmap btree and unmap it. For rmap extent types that + * can overlap (data fork rmaps on reflink filesystems) we must be careful + * that the prev/next records in the btree might belong to another owner. + * Therefore we must use delete+insert to alter any of the key fields. + * + * For every other situation there can only be one owner for a given extent, + * so we can call the regular _free function. + */ +STATIC int +xfs_rmap_unmap_shared( + struct xfs_btree_cur *cur, + xfs_agblock_t bno, + xfs_extlen_t len, + bool unwritten, + const struct xfs_owner_info *oinfo) +{ + struct xfs_mount *mp = cur->bc_mp; + struct xfs_rmap_irec ltrec; + uint64_t ltoff; + int error = 0; + int i; + uint64_t owner; + uint64_t offset; + unsigned int flags; + + xfs_owner_info_unpack(oinfo, &owner, &offset, &flags); + if (unwritten) + flags |= XFS_RMAP_UNWRITTEN; + trace_xfs_rmap_unmap(cur, bno, len, unwritten, oinfo); + + /* + * We should always have a left record because there's a static record + * for the AG headers at rm_startblock == 0 created by mkfs/growfs that + * will not ever be removed from the tree. + */ + error = xfs_rmap_lookup_le_range(cur, bno, owner, offset, flags, + <rec, &i); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + ltoff = ltrec.rm_offset; + + /* Make sure the extent we found covers the entire freeing range. */ + if (XFS_IS_CORRUPT(mp, + ltrec.rm_startblock > bno || + ltrec.rm_startblock + ltrec.rm_blockcount < + bno + len)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + + /* Make sure the owner matches what we expect to find in the tree. */ + if (XFS_IS_CORRUPT(mp, owner != ltrec.rm_owner)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + + /* Make sure the unwritten flag matches. */ + if (XFS_IS_CORRUPT(mp, + (flags & XFS_RMAP_UNWRITTEN) != + (ltrec.rm_flags & XFS_RMAP_UNWRITTEN))) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + + /* Check the offset. */ + if (XFS_IS_CORRUPT(mp, ltrec.rm_offset > offset)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + if (XFS_IS_CORRUPT(mp, offset > ltoff + ltrec.rm_blockcount)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + + if (ltrec.rm_startblock == bno && ltrec.rm_blockcount == len) { + /* Exact match, simply remove the record from rmap tree. */ + error = xfs_rmap_delete(cur, ltrec.rm_startblock, + ltrec.rm_blockcount, ltrec.rm_owner, + ltrec.rm_offset, ltrec.rm_flags); + if (error) + goto out_error; + } else if (ltrec.rm_startblock == bno) { + /* + * Overlap left hand side of extent: move the start, trim the + * length and update the current record. + * + * ltbno ltlen + * Orig: |oooooooooooooooooooo| + * Freeing: |fffffffff| + * Result: |rrrrrrrrrr| + * bno len + */ + + /* Delete prev rmap. */ + error = xfs_rmap_delete(cur, ltrec.rm_startblock, + ltrec.rm_blockcount, ltrec.rm_owner, + ltrec.rm_offset, ltrec.rm_flags); + if (error) + goto out_error; + + /* Add an rmap at the new offset. */ + ltrec.rm_startblock += len; + ltrec.rm_blockcount -= len; + ltrec.rm_offset += len; + error = xfs_rmap_insert(cur, ltrec.rm_startblock, + ltrec.rm_blockcount, ltrec.rm_owner, + ltrec.rm_offset, ltrec.rm_flags); + if (error) + goto out_error; + } else if (ltrec.rm_startblock + ltrec.rm_blockcount == bno + len) { + /* + * Overlap right hand side of extent: trim the length and + * update the current record. + * + * ltbno ltlen + * Orig: |oooooooooooooooooooo| + * Freeing: |fffffffff| + * Result: |rrrrrrrrrr| + * bno len + */ + error = xfs_rmap_lookup_eq(cur, ltrec.rm_startblock, + ltrec.rm_blockcount, ltrec.rm_owner, + ltrec.rm_offset, ltrec.rm_flags, &i); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + ltrec.rm_blockcount -= len; + error = xfs_rmap_update(cur, <rec); + if (error) + goto out_error; + } else { + /* + * Overlap middle of extent: trim the length of the existing + * record to the length of the new left-extent size, increment + * the insertion position so we can insert a new record + * containing the remaining right-extent space. + * + * ltbno ltlen + * Orig: |oooooooooooooooooooo| + * Freeing: |fffffffff| + * Result: |rrrrr| |rrrr| + * bno len + */ + xfs_extlen_t orig_len = ltrec.rm_blockcount; + + /* Shrink the left side of the rmap */ + error = xfs_rmap_lookup_eq(cur, ltrec.rm_startblock, + ltrec.rm_blockcount, ltrec.rm_owner, + ltrec.rm_offset, ltrec.rm_flags, &i); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + ltrec.rm_blockcount = bno - ltrec.rm_startblock; + error = xfs_rmap_update(cur, <rec); + if (error) + goto out_error; + + /* Add an rmap at the new offset */ + error = xfs_rmap_insert(cur, bno + len, + orig_len - len - ltrec.rm_blockcount, + ltrec.rm_owner, offset + len, + ltrec.rm_flags); + if (error) + goto out_error; + } + + trace_xfs_rmap_unmap_done(cur, bno, len, unwritten, oinfo); +out_error: + if (error) + trace_xfs_rmap_unmap_error(cur, error, _RET_IP_); + return error; +} + +/* + * Find an extent in the rmap btree and map it. For rmap extent types that + * can overlap (data fork rmaps on reflink filesystems) we must be careful + * that the prev/next records in the btree might belong to another owner. + * Therefore we must use delete+insert to alter any of the key fields. + * + * For every other situation there can only be one owner for a given extent, + * so we can call the regular _alloc function. + */ +STATIC int +xfs_rmap_map_shared( + struct xfs_btree_cur *cur, + xfs_agblock_t bno, + xfs_extlen_t len, + bool unwritten, + const struct xfs_owner_info *oinfo) +{ + struct xfs_mount *mp = cur->bc_mp; + struct xfs_rmap_irec ltrec; + struct xfs_rmap_irec gtrec; + int have_gt; + int have_lt; + int error = 0; + int i; + uint64_t owner; + uint64_t offset; + unsigned int flags = 0; + + xfs_owner_info_unpack(oinfo, &owner, &offset, &flags); + if (unwritten) + flags |= XFS_RMAP_UNWRITTEN; + trace_xfs_rmap_map(cur, bno, len, unwritten, oinfo); + + /* Is there a left record that abuts our range? */ + error = xfs_rmap_find_left_neighbor(cur, bno, owner, offset, flags, + <rec, &have_lt); + if (error) + goto out_error; + if (have_lt && + !xfs_rmap_is_mergeable(<rec, owner, flags)) + have_lt = 0; + + /* Is there a right record that abuts our range? */ + error = xfs_rmap_lookup_eq(cur, bno + len, len, owner, offset + len, + flags, &have_gt); + if (error) + goto out_error; + if (have_gt) { + error = xfs_rmap_get_rec(cur, >rec, &have_gt); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(mp, have_gt != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + trace_xfs_rmap_find_right_neighbor_result(cur, + gtrec.rm_startblock, gtrec.rm_blockcount, + gtrec.rm_owner, gtrec.rm_offset, + gtrec.rm_flags); + + if (!xfs_rmap_is_mergeable(>rec, owner, flags)) + have_gt = 0; + } + + if (have_lt && + ltrec.rm_startblock + ltrec.rm_blockcount == bno && + ltrec.rm_offset + ltrec.rm_blockcount == offset) { + /* + * Left edge contiguous, merge into left record. + * + * ltbno ltlen + * orig: |ooooooooo| + * adding: |aaaaaaaaa| + * result: |rrrrrrrrrrrrrrrrrrr| + * bno len + */ + ltrec.rm_blockcount += len; + if (have_gt && + bno + len == gtrec.rm_startblock && + offset + len == gtrec.rm_offset) { + /* + * Right edge also contiguous, delete right record + * and merge into left record. + * + * ltbno ltlen gtbno gtlen + * orig: |ooooooooo| |ooooooooo| + * adding: |aaaaaaaaa| + * result: |rrrrrrrrrrrrrrrrrrrrrrrrrrrrr| + */ + ltrec.rm_blockcount += gtrec.rm_blockcount; + error = xfs_rmap_delete(cur, gtrec.rm_startblock, + gtrec.rm_blockcount, gtrec.rm_owner, + gtrec.rm_offset, gtrec.rm_flags); + if (error) + goto out_error; + } + + /* Point the cursor back to the left record and update. */ + error = xfs_rmap_lookup_eq(cur, ltrec.rm_startblock, + ltrec.rm_blockcount, ltrec.rm_owner, + ltrec.rm_offset, ltrec.rm_flags, &i); + if (error) + goto out_error; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out_error; + } + + error = xfs_rmap_update(cur, <rec); + if (error) + goto out_error; + } else if (have_gt && + bno + len == gtrec.rm_startblock && + offset + len == gtrec.rm_offset) { + /* + * Right edge contiguous, merge into right record. + * + * gtbno gtlen + * Orig: |ooooooooo| + * adding: |aaaaaaaaa| + * Result: |rrrrrrrrrrrrrrrrrrr| + * bno len + */ + /* Delete the old record. */ + error = xfs_rmap_delete(cur, gtrec.rm_startblock, + gtrec.rm_blockcount, gtrec.rm_owner, + gtrec.rm_offset, gtrec.rm_flags); + if (error) + goto out_error; + + /* Move the start and re-add it. */ + gtrec.rm_startblock = bno; + gtrec.rm_blockcount += len; + gtrec.rm_offset = offset; + error = xfs_rmap_insert(cur, gtrec.rm_startblock, + gtrec.rm_blockcount, gtrec.rm_owner, + gtrec.rm_offset, gtrec.rm_flags); + if (error) + goto out_error; + } else { + /* + * No contiguous edge with identical owner, insert + * new record at current cursor position. + */ + error = xfs_rmap_insert(cur, bno, len, owner, offset, flags); + if (error) + goto out_error; + } + + trace_xfs_rmap_map_done(cur, bno, len, unwritten, oinfo); +out_error: + if (error) + trace_xfs_rmap_map_error(cur, error, _RET_IP_); + return error; +} + +/* Insert a raw rmap into the rmapbt. */ +int +xfs_rmap_map_raw( + struct xfs_btree_cur *cur, + struct xfs_rmap_irec *rmap) +{ + struct xfs_owner_info oinfo; + + xfs_owner_info_pack(&oinfo, rmap->rm_owner, rmap->rm_offset, + rmap->rm_flags); + + if ((rmap->rm_flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK | + XFS_RMAP_UNWRITTEN)) || + XFS_RMAP_NON_INODE_OWNER(rmap->rm_owner)) + return xfs_rmap_map(cur, rmap->rm_startblock, + rmap->rm_blockcount, + rmap->rm_flags & XFS_RMAP_UNWRITTEN, + &oinfo); + + return xfs_rmap_map_shared(cur, rmap->rm_startblock, + rmap->rm_blockcount, + rmap->rm_flags & XFS_RMAP_UNWRITTEN, + &oinfo); +} + +struct xfs_rmap_query_range_info { + xfs_rmap_query_range_fn fn; + void *priv; +}; + +/* Format btree record and pass to our callback. */ +STATIC int +xfs_rmap_query_range_helper( + struct xfs_btree_cur *cur, + const union xfs_btree_rec *rec, + void *priv) +{ + struct xfs_rmap_query_range_info *query = priv; + struct xfs_rmap_irec irec; + xfs_failaddr_t fa; + + fa = xfs_rmap_btrec_to_irec(rec, &irec); + if (!fa) + fa = xfs_rmap_check_btrec(cur, &irec); + if (fa) + return xfs_rmap_complain_bad_rec(cur, fa, &irec); + + return query->fn(cur, &irec, query->priv); +} + +/* Find all rmaps between two keys. */ +int +xfs_rmap_query_range( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *low_rec, + const struct xfs_rmap_irec *high_rec, + xfs_rmap_query_range_fn fn, + void *priv) +{ + union xfs_btree_irec low_brec = { .r = *low_rec }; + union xfs_btree_irec high_brec = { .r = *high_rec }; + struct xfs_rmap_query_range_info query = { .priv = priv, .fn = fn }; + + return xfs_btree_query_range(cur, &low_brec, &high_brec, + xfs_rmap_query_range_helper, &query); +} + +/* Find all rmaps. */ +int +xfs_rmap_query_all( + struct xfs_btree_cur *cur, + xfs_rmap_query_range_fn fn, + void *priv) +{ + struct xfs_rmap_query_range_info query; + + query.priv = priv; + query.fn = fn; + return xfs_btree_query_all(cur, xfs_rmap_query_range_helper, &query); +} + +/* Commit an rmap operation into the ondisk tree. */ +int +__xfs_rmap_finish_intent( + struct xfs_btree_cur *rcur, + enum xfs_rmap_intent_type op, + xfs_agblock_t bno, + xfs_extlen_t len, + const struct xfs_owner_info *oinfo, + bool unwritten) +{ + switch (op) { + case XFS_RMAP_ALLOC: + case XFS_RMAP_MAP: + return xfs_rmap_map(rcur, bno, len, unwritten, oinfo); + case XFS_RMAP_MAP_SHARED: + return xfs_rmap_map_shared(rcur, bno, len, unwritten, oinfo); + case XFS_RMAP_FREE: + case XFS_RMAP_UNMAP: + return xfs_rmap_unmap(rcur, bno, len, unwritten, oinfo); + case XFS_RMAP_UNMAP_SHARED: + return xfs_rmap_unmap_shared(rcur, bno, len, unwritten, oinfo); + case XFS_RMAP_CONVERT: + return xfs_rmap_convert(rcur, bno, len, !unwritten, oinfo); + case XFS_RMAP_CONVERT_SHARED: + return xfs_rmap_convert_shared(rcur, bno, len, !unwritten, + oinfo); + default: + ASSERT(0); + return -EFSCORRUPTED; + } +} + +static int +xfs_rmap_finish_init_cursor( + struct xfs_trans *tp, + struct xfs_rmap_intent *ri, + struct xfs_btree_cur **pcur) +{ + struct xfs_perag *pag = to_perag(ri->ri_group); + struct xfs_buf *agbp = NULL; + int error; + + /* + * Refresh the freelist before we start changing the rmapbt, because a + * shape change could cause us to allocate blocks. + */ + error = xfs_free_extent_fix_freelist(tp, pag, &agbp); + if (error) { + xfs_ag_mark_sick(pag, XFS_SICK_AG_AGFL); + return error; + } + if (XFS_IS_CORRUPT(tp->t_mountp, !agbp)) { + xfs_ag_mark_sick(pag, XFS_SICK_AG_AGFL); + return -EFSCORRUPTED; + } + *pcur = xfs_rmapbt_init_cursor(tp->t_mountp, tp, agbp, pag); + return 0; +} + +static int +xfs_rtrmap_finish_init_cursor( + struct xfs_trans *tp, + struct xfs_rmap_intent *ri, + struct xfs_btree_cur **pcur) +{ + struct xfs_rtgroup *rtg = to_rtg(ri->ri_group); + + xfs_rtgroup_lock(rtg, XFS_RTGLOCK_RMAP); + xfs_rtgroup_trans_join(tp, rtg, XFS_RTGLOCK_RMAP); + *pcur = xfs_rtrmapbt_init_cursor(tp, rtg); + return 0; +} + +/* + * Process one of the deferred rmap operations. We pass back the + * btree cursor to maintain our lock on the rmapbt between calls. + * This saves time and eliminates a buffer deadlock between the + * superblock and the AGF because we'll always grab them in the same + * order. + */ +int +xfs_rmap_finish_one( + struct xfs_trans *tp, + struct xfs_rmap_intent *ri, + struct xfs_btree_cur **pcur) +{ + struct xfs_owner_info oinfo; + struct xfs_mount *mp = tp->t_mountp; + xfs_agblock_t bno; + bool unwritten; + int error = 0; + + trace_xfs_rmap_deferred(mp, ri); + + if (XFS_TEST_ERROR(mp, XFS_ERRTAG_RMAP_FINISH_ONE)) + return -EIO; + + /* + * If we haven't gotten a cursor or the cursor AG doesn't match + * the startblock, get one now. + */ + if (*pcur != NULL && (*pcur)->bc_group != ri->ri_group) { + xfs_btree_del_cursor(*pcur, 0); + *pcur = NULL; + } + if (*pcur == NULL) { + if (ri->ri_group->xg_type == XG_TYPE_RTG) + error = xfs_rtrmap_finish_init_cursor(tp, ri, pcur); + else + error = xfs_rmap_finish_init_cursor(tp, ri, pcur); + if (error) + return error; + } + + xfs_rmap_ino_owner(&oinfo, ri->ri_owner, ri->ri_whichfork, + ri->ri_bmap.br_startoff); + unwritten = ri->ri_bmap.br_state == XFS_EXT_UNWRITTEN; + + bno = xfs_fsb_to_gbno(mp, ri->ri_bmap.br_startblock, + ri->ri_group->xg_type); + error = __xfs_rmap_finish_intent(*pcur, ri->ri_type, bno, + ri->ri_bmap.br_blockcount, &oinfo, unwritten); + if (error) + return error; + + xfs_rmap_update_hook(tp, ri->ri_group, ri->ri_type, bno, + ri->ri_bmap.br_blockcount, unwritten, &oinfo); + return 0; +} + +/* + * Don't defer an rmap if we aren't an rmap filesystem. + */ +static bool +xfs_rmap_update_is_needed( + struct xfs_mount *mp, + int whichfork) +{ + return xfs_has_rmapbt(mp) && whichfork != XFS_COW_FORK; +} + +/* + * Record a rmap intent; the list is kept sorted first by AG and then by + * increasing age. + */ +static void +__xfs_rmap_add( + struct xfs_trans *tp, + enum xfs_rmap_intent_type type, + uint64_t owner, + bool isrt, + int whichfork, + struct xfs_bmbt_irec *bmap) +{ + struct xfs_rmap_intent *ri; + + ri = kmem_cache_alloc(xfs_rmap_intent_cache, GFP_KERNEL | __GFP_NOFAIL); + INIT_LIST_HEAD(&ri->ri_list); + ri->ri_type = type; + ri->ri_owner = owner; + ri->ri_whichfork = whichfork; + ri->ri_bmap = *bmap; + ri->ri_realtime = isrt; + + xfs_rmap_defer_add(tp, ri); +} + +/* Map an extent into a file. */ +void +xfs_rmap_map_extent( + struct xfs_trans *tp, + struct xfs_inode *ip, + int whichfork, + struct xfs_bmbt_irec *PREV) +{ + enum xfs_rmap_intent_type type = XFS_RMAP_MAP; + bool isrt = xfs_ifork_is_realtime(ip, whichfork); + + if (!xfs_rmap_update_is_needed(tp->t_mountp, whichfork)) + return; + + if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip)) + type = XFS_RMAP_MAP_SHARED; + + __xfs_rmap_add(tp, type, I_INO(ip), isrt, whichfork, PREV); +} + +/* Unmap an extent out of a file. */ +void +xfs_rmap_unmap_extent( + struct xfs_trans *tp, + struct xfs_inode *ip, + int whichfork, + struct xfs_bmbt_irec *PREV) +{ + enum xfs_rmap_intent_type type = XFS_RMAP_UNMAP; + bool isrt = xfs_ifork_is_realtime(ip, whichfork); + + if (!xfs_rmap_update_is_needed(tp->t_mountp, whichfork)) + return; + + if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip)) + type = XFS_RMAP_UNMAP_SHARED; + + __xfs_rmap_add(tp, type, I_INO(ip), isrt, whichfork, PREV); +} + +/* + * Convert a data fork extent from unwritten to real or vice versa. + * + * Note that tp can be NULL here as no transaction is used for COW fork + * unwritten conversion. + */ +void +xfs_rmap_convert_extent( + struct xfs_mount *mp, + struct xfs_trans *tp, + struct xfs_inode *ip, + int whichfork, + struct xfs_bmbt_irec *PREV) +{ + enum xfs_rmap_intent_type type = XFS_RMAP_CONVERT; + bool isrt = xfs_ifork_is_realtime(ip, whichfork); + + if (!xfs_rmap_update_is_needed(mp, whichfork)) + return; + + if (whichfork != XFS_ATTR_FORK && xfs_is_reflink_inode(ip)) + type = XFS_RMAP_CONVERT_SHARED; + + __xfs_rmap_add(tp, type, I_INO(ip), isrt, whichfork, PREV); +} + +/* Schedule the creation of an rmap for non-file data. */ +void +xfs_rmap_alloc_extent( + struct xfs_trans *tp, + bool isrt, + xfs_fsblock_t fsbno, + xfs_extlen_t len, + uint64_t owner) +{ + struct xfs_bmbt_irec bmap; + + if (!xfs_rmap_update_is_needed(tp->t_mountp, XFS_DATA_FORK)) + return; + + bmap.br_startblock = fsbno; + bmap.br_blockcount = len; + bmap.br_startoff = 0; + bmap.br_state = XFS_EXT_NORM; + + __xfs_rmap_add(tp, XFS_RMAP_ALLOC, owner, isrt, XFS_DATA_FORK, &bmap); +} + +/* Schedule the deletion of an rmap for non-file data. */ +void +xfs_rmap_free_extent( + struct xfs_trans *tp, + bool isrt, + xfs_fsblock_t fsbno, + xfs_extlen_t len, + uint64_t owner) +{ + struct xfs_bmbt_irec bmap; + + if (!xfs_rmap_update_is_needed(tp->t_mountp, XFS_DATA_FORK)) + return; + + bmap.br_startblock = fsbno; + bmap.br_blockcount = len; + bmap.br_startoff = 0; + bmap.br_state = XFS_EXT_NORM; + + __xfs_rmap_add(tp, XFS_RMAP_FREE, owner, isrt, XFS_DATA_FORK, &bmap); +} + +/* Compare rmap records. Returns -1 if a < b, 1 if a > b, and 0 if equal. */ +int +xfs_rmap_compare( + const struct xfs_rmap_irec *a, + const struct xfs_rmap_irec *b) +{ + __u64 oa; + __u64 ob; + + oa = xfs_rmap_irec_offset_pack(a); + ob = xfs_rmap_irec_offset_pack(b); + + if (a->rm_startblock < b->rm_startblock) + return -1; + else if (a->rm_startblock > b->rm_startblock) + return 1; + else if (a->rm_owner < b->rm_owner) + return -1; + else if (a->rm_owner > b->rm_owner) + return 1; + else if (oa < ob) + return -1; + else if (oa > ob) + return 1; + else + return 0; +} + +/* + * Scan the physical storage part of the keyspace of the reverse mapping index + * and tell us if the area has no records, is fully mapped by records, or is + * partially filled. + */ +int +xfs_rmap_has_records( + struct xfs_btree_cur *cur, + xfs_agblock_t bno, + xfs_extlen_t len, + enum xbtree_recpacking *outcome) +{ + union xfs_btree_key mask = { + .rmap.rm_startblock = cpu_to_be32(-1U), + }; + union xfs_btree_irec low; + union xfs_btree_irec high; + + memset(&low, 0, sizeof(low)); + low.r.rm_startblock = bno; + memset(&high, 0xFF, sizeof(high)); + high.r.rm_startblock = bno + len - 1; + + return xfs_btree_has_records(cur, &low, &high, &mask, outcome); +} + +struct xfs_rmap_ownercount { + /* Owner that we're looking for. */ + struct xfs_rmap_irec good; + + /* rmap search keys */ + struct xfs_rmap_irec low; + struct xfs_rmap_irec high; + + struct xfs_rmap_matches *results; + + /* Stop early if we find a nonmatch? */ + bool stop_on_nonmatch; +}; + +/* Does this rmap represent space that can have multiple owners? */ +static inline bool +xfs_rmap_shareable( + struct xfs_mount *mp, + const struct xfs_rmap_irec *rmap) +{ + if (!xfs_has_reflink(mp)) + return false; + if (XFS_RMAP_NON_INODE_OWNER(rmap->rm_owner)) + return false; + if (rmap->rm_flags & (XFS_RMAP_ATTR_FORK | + XFS_RMAP_BMBT_BLOCK)) + return false; + return true; +} + +static inline void +xfs_rmap_ownercount_init( + struct xfs_rmap_ownercount *roc, + xfs_agblock_t bno, + xfs_extlen_t len, + const struct xfs_owner_info *oinfo, + struct xfs_rmap_matches *results) +{ + memset(roc, 0, sizeof(*roc)); + roc->results = results; + + roc->low.rm_startblock = bno; + memset(&roc->high, 0xFF, sizeof(roc->high)); + roc->high.rm_startblock = bno + len - 1; + + memset(results, 0, sizeof(*results)); + roc->good.rm_startblock = bno; + roc->good.rm_blockcount = len; + roc->good.rm_owner = oinfo->oi_owner; + roc->good.rm_offset = oinfo->oi_offset; + if (oinfo->oi_flags & XFS_OWNER_INFO_ATTR_FORK) + roc->good.rm_flags |= XFS_RMAP_ATTR_FORK; + if (oinfo->oi_flags & XFS_OWNER_INFO_BMBT_BLOCK) + roc->good.rm_flags |= XFS_RMAP_BMBT_BLOCK; +} + +/* Figure out if this is a match for the owner. */ +STATIC int +xfs_rmap_count_owners_helper( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv) +{ + struct xfs_rmap_ownercount *roc = priv; + struct xfs_rmap_irec check = *rec; + unsigned int keyflags; + bool filedata; + int64_t delta; + + filedata = !XFS_RMAP_NON_INODE_OWNER(check.rm_owner) && + !(check.rm_flags & XFS_RMAP_BMBT_BLOCK); + + /* Trim the part of check that comes before the comparison range. */ + delta = (int64_t)roc->good.rm_startblock - check.rm_startblock; + if (delta > 0) { + check.rm_startblock += delta; + check.rm_blockcount -= delta; + if (filedata) + check.rm_offset += delta; + } + + /* Trim the part of check that comes after the comparison range. */ + delta = (check.rm_startblock + check.rm_blockcount) - + (roc->good.rm_startblock + roc->good.rm_blockcount); + if (delta > 0) + check.rm_blockcount -= delta; + + /* Don't care about unwritten status for establishing ownership. */ + keyflags = check.rm_flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK); + + if (check.rm_startblock == roc->good.rm_startblock && + check.rm_blockcount == roc->good.rm_blockcount && + check.rm_owner == roc->good.rm_owner && + check.rm_offset == roc->good.rm_offset && + keyflags == roc->good.rm_flags) { + roc->results->matches++; + } else { + roc->results->non_owner_matches++; + if (xfs_rmap_shareable(cur->bc_mp, &roc->good) ^ + xfs_rmap_shareable(cur->bc_mp, &check)) + roc->results->bad_non_owner_matches++; + } + + if (roc->results->non_owner_matches && roc->stop_on_nonmatch) + return -ECANCELED; + + return 0; +} + +/* Count the number of owners and non-owners of this range of blocks. */ +int +xfs_rmap_count_owners( + struct xfs_btree_cur *cur, + xfs_agblock_t bno, + xfs_extlen_t len, + const struct xfs_owner_info *oinfo, + struct xfs_rmap_matches *results) +{ + struct xfs_rmap_ownercount roc; + int error; + + xfs_rmap_ownercount_init(&roc, bno, len, oinfo, results); + error = xfs_rmap_query_range(cur, &roc.low, &roc.high, + xfs_rmap_count_owners_helper, &roc); + if (error) + return error; + + /* + * There can't be any non-owner rmaps that conflict with the given + * owner if we didn't find any rmaps matching the owner. + */ + if (!results->matches) + results->bad_non_owner_matches = 0; + + return 0; +} + +/* + * Given an extent and some owner info, can we find records overlapping + * the extent whose owner info does not match the given owner? + */ +int +xfs_rmap_has_other_keys( + struct xfs_btree_cur *cur, + xfs_agblock_t bno, + xfs_extlen_t len, + const struct xfs_owner_info *oinfo, + bool *has_other) +{ + struct xfs_rmap_matches res; + struct xfs_rmap_ownercount roc; + int error; + + xfs_rmap_ownercount_init(&roc, bno, len, oinfo, &res); + roc.stop_on_nonmatch = true; + + error = xfs_rmap_query_range(cur, &roc.low, &roc.high, + xfs_rmap_count_owners_helper, &roc); + if (error == -ECANCELED) { + *has_other = true; + return 0; + } + if (error) + return error; + + *has_other = false; + return 0; +} + +const struct xfs_owner_info XFS_RMAP_OINFO_SKIP_UPDATE = { + .oi_owner = XFS_RMAP_OWN_NULL, +}; +const struct xfs_owner_info XFS_RMAP_OINFO_ANY_OWNER = { + .oi_owner = XFS_RMAP_OWN_UNKNOWN, +}; +const struct xfs_owner_info XFS_RMAP_OINFO_FS = { + .oi_owner = XFS_RMAP_OWN_FS, +}; +const struct xfs_owner_info XFS_RMAP_OINFO_LOG = { + .oi_owner = XFS_RMAP_OWN_LOG, +}; +const struct xfs_owner_info XFS_RMAP_OINFO_AG = { + .oi_owner = XFS_RMAP_OWN_AG, +}; +const struct xfs_owner_info XFS_RMAP_OINFO_INOBT = { + .oi_owner = XFS_RMAP_OWN_INOBT, +}; +const struct xfs_owner_info XFS_RMAP_OINFO_INODES = { + .oi_owner = XFS_RMAP_OWN_INODES, +}; +const struct xfs_owner_info XFS_RMAP_OINFO_REFC = { + .oi_owner = XFS_RMAP_OWN_REFC, +}; +const struct xfs_owner_info XFS_RMAP_OINFO_COW = { + .oi_owner = XFS_RMAP_OWN_COW, +}; + +int __init +xfs_rmap_intent_init_cache(void) +{ + xfs_rmap_intent_cache = kmem_cache_create("xfs_rmap_intent", + sizeof(struct xfs_rmap_intent), + 0, 0, NULL); + + return xfs_rmap_intent_cache != NULL ? 0 : -ENOMEM; +} + +void +xfs_rmap_intent_destroy_cache(void) +{ + kmem_cache_destroy(xfs_rmap_intent_cache); + xfs_rmap_intent_cache = NULL; +} diff --git a/fs/xfs/libxfs/xfs_rmap.h b/fs/xfs/libxfs/xfs_rmap.h new file mode 100644 index 000000000..4afa33575 --- /dev/null +++ b/fs/xfs/libxfs/xfs_rmap.h @@ -0,0 +1,280 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2016 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_RMAP_H__ +#define __XFS_RMAP_H__ + +struct xfs_perag; +struct xfs_rtgroup; + +static inline void +xfs_rmap_ino_bmbt_owner( + struct xfs_owner_info *oi, + xfs_ino_t ino, + int whichfork) +{ + oi->oi_owner = ino; + oi->oi_offset = 0; + oi->oi_flags = XFS_OWNER_INFO_BMBT_BLOCK; + if (whichfork == XFS_ATTR_FORK) + oi->oi_flags |= XFS_OWNER_INFO_ATTR_FORK; +} +#define xfs_rmap_inode_bmbt_owner(oi, ip, whichfork) \ + xfs_rmap_ino_bmbt_owner(oi, I_INO(ip), whichfork) + +static inline void +xfs_rmap_ino_owner( + struct xfs_owner_info *oi, + xfs_ino_t ino, + int whichfork, + xfs_fileoff_t offset) +{ + oi->oi_owner = ino; + oi->oi_offset = offset; + oi->oi_flags = 0; + if (whichfork == XFS_ATTR_FORK) + oi->oi_flags |= XFS_OWNER_INFO_ATTR_FORK; +} +#define xfs_rmap_inode_owner(oi, ip, whichfork, offset) \ + xfs_rmap_ino_owner(oi, I_INO(ip), whichfork, offset) + +static inline bool +xfs_rmap_should_skip_owner_update( + const struct xfs_owner_info *oi) +{ + return oi->oi_owner == XFS_RMAP_OWN_NULL; +} + +/* Reverse mapping functions. */ + +struct xfs_buf; + +static inline __u64 +xfs_rmap_irec_offset_pack( + const struct xfs_rmap_irec *irec) +{ + __u64 x; + + x = XFS_RMAP_OFF(irec->rm_offset); + if (irec->rm_flags & XFS_RMAP_ATTR_FORK) + x |= XFS_RMAP_OFF_ATTR_FORK; + if (irec->rm_flags & XFS_RMAP_BMBT_BLOCK) + x |= XFS_RMAP_OFF_BMBT_BLOCK; + if (irec->rm_flags & XFS_RMAP_UNWRITTEN) + x |= XFS_RMAP_OFF_UNWRITTEN; + return x; +} + +static inline xfs_failaddr_t +xfs_rmap_irec_offset_unpack( + __u64 offset, + struct xfs_rmap_irec *irec) +{ + if (offset & ~(XFS_RMAP_OFF_MASK | XFS_RMAP_OFF_FLAGS)) + return __this_address; + + irec->rm_offset = XFS_RMAP_OFF(offset); + irec->rm_flags = 0; + if (offset & XFS_RMAP_OFF_ATTR_FORK) + irec->rm_flags |= XFS_RMAP_ATTR_FORK; + if (offset & XFS_RMAP_OFF_BMBT_BLOCK) + irec->rm_flags |= XFS_RMAP_BMBT_BLOCK; + if (offset & XFS_RMAP_OFF_UNWRITTEN) + irec->rm_flags |= XFS_RMAP_UNWRITTEN; + return NULL; +} + +static inline void +xfs_owner_info_unpack( + const struct xfs_owner_info *oinfo, + uint64_t *owner, + uint64_t *offset, + unsigned int *flags) +{ + unsigned int r = 0; + + *owner = oinfo->oi_owner; + *offset = oinfo->oi_offset; + if (oinfo->oi_flags & XFS_OWNER_INFO_ATTR_FORK) + r |= XFS_RMAP_ATTR_FORK; + if (oinfo->oi_flags & XFS_OWNER_INFO_BMBT_BLOCK) + r |= XFS_RMAP_BMBT_BLOCK; + *flags = r; +} + +static inline void +xfs_owner_info_pack( + struct xfs_owner_info *oinfo, + uint64_t owner, + uint64_t offset, + unsigned int flags) +{ + oinfo->oi_owner = owner; + oinfo->oi_offset = XFS_RMAP_OFF(offset); + oinfo->oi_flags = 0; + if (flags & XFS_RMAP_ATTR_FORK) + oinfo->oi_flags |= XFS_OWNER_INFO_ATTR_FORK; + if (flags & XFS_RMAP_BMBT_BLOCK) + oinfo->oi_flags |= XFS_OWNER_INFO_BMBT_BLOCK; +} + +int xfs_rmap_alloc(struct xfs_trans *tp, struct xfs_buf *agbp, + struct xfs_perag *pag, xfs_agblock_t bno, xfs_extlen_t len, + const struct xfs_owner_info *oinfo); +int xfs_rmap_free(struct xfs_trans *tp, struct xfs_buf *agbp, + struct xfs_perag *pag, xfs_agblock_t bno, xfs_extlen_t len, + const struct xfs_owner_info *oinfo); + +int xfs_rmap_lookup_le(struct xfs_btree_cur *cur, xfs_agblock_t bno, + uint64_t owner, uint64_t offset, unsigned int flags, + struct xfs_rmap_irec *irec, int *stat); +int xfs_rmap_lookup_eq(struct xfs_btree_cur *cur, xfs_agblock_t bno, + xfs_extlen_t len, uint64_t owner, uint64_t offset, + unsigned int flags, int *stat); +int xfs_rmap_insert(struct xfs_btree_cur *rcur, xfs_agblock_t agbno, + xfs_extlen_t len, uint64_t owner, uint64_t offset, + unsigned int flags); +int xfs_rmap_get_rec(struct xfs_btree_cur *cur, struct xfs_rmap_irec *irec, + int *stat); + +typedef int (*xfs_rmap_query_range_fn)( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv); + +int xfs_rmap_query_range(struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *low_rec, + const struct xfs_rmap_irec *high_rec, + xfs_rmap_query_range_fn fn, void *priv); +int xfs_rmap_query_all(struct xfs_btree_cur *cur, xfs_rmap_query_range_fn fn, + void *priv); + +enum xfs_rmap_intent_type { + XFS_RMAP_MAP, + XFS_RMAP_MAP_SHARED, + XFS_RMAP_UNMAP, + XFS_RMAP_UNMAP_SHARED, + XFS_RMAP_CONVERT, + XFS_RMAP_CONVERT_SHARED, + XFS_RMAP_ALLOC, + XFS_RMAP_FREE, +}; + +#define XFS_RMAP_INTENT_STRINGS \ + { XFS_RMAP_MAP, "map" }, \ + { XFS_RMAP_MAP_SHARED, "map_shared" }, \ + { XFS_RMAP_UNMAP, "unmap" }, \ + { XFS_RMAP_UNMAP_SHARED, "unmap_shared" }, \ + { XFS_RMAP_CONVERT, "cvt" }, \ + { XFS_RMAP_CONVERT_SHARED, "cvt_shared" }, \ + { XFS_RMAP_ALLOC, "alloc" }, \ + { XFS_RMAP_FREE, "free" } + +struct xfs_rmap_intent { + struct list_head ri_list; + enum xfs_rmap_intent_type ri_type; + int ri_whichfork; + uint64_t ri_owner; + struct xfs_bmbt_irec ri_bmap; + struct xfs_group *ri_group; + bool ri_realtime; +}; + +/* functions for updating the rmapbt based on bmbt map/unmap operations */ +void xfs_rmap_map_extent(struct xfs_trans *tp, struct xfs_inode *ip, + int whichfork, struct xfs_bmbt_irec *imap); +void xfs_rmap_unmap_extent(struct xfs_trans *tp, struct xfs_inode *ip, + int whichfork, struct xfs_bmbt_irec *imap); +void xfs_rmap_convert_extent(struct xfs_mount *mp, struct xfs_trans *tp, + struct xfs_inode *ip, int whichfork, + struct xfs_bmbt_irec *imap); +void xfs_rmap_alloc_extent(struct xfs_trans *tp, bool isrt, xfs_fsblock_t fsbno, + xfs_extlen_t len, uint64_t owner); +void xfs_rmap_free_extent(struct xfs_trans *tp, bool isrt, xfs_fsblock_t fsbno, + xfs_extlen_t len, uint64_t owner); + +int xfs_rmap_finish_one(struct xfs_trans *tp, struct xfs_rmap_intent *ri, + struct xfs_btree_cur **pcur); +int __xfs_rmap_finish_intent(struct xfs_btree_cur *rcur, + enum xfs_rmap_intent_type op, xfs_agblock_t bno, + xfs_extlen_t len, const struct xfs_owner_info *oinfo, + bool unwritten); + +int xfs_rmap_lookup_le_range(struct xfs_btree_cur *cur, xfs_agblock_t bno, + uint64_t owner, uint64_t offset, unsigned int flags, + struct xfs_rmap_irec *irec, int *stat); +int xfs_rmap_compare(const struct xfs_rmap_irec *a, + const struct xfs_rmap_irec *b); +union xfs_btree_rec; +xfs_failaddr_t xfs_rmap_btrec_to_irec(const union xfs_btree_rec *rec, + struct xfs_rmap_irec *irec); +xfs_failaddr_t xfs_rmap_check_irec(struct xfs_perag *pag, + const struct xfs_rmap_irec *irec); +xfs_failaddr_t xfs_rtrmap_check_irec(struct xfs_rtgroup *rtg, + const struct xfs_rmap_irec *irec); + +int xfs_rmap_has_records(struct xfs_btree_cur *cur, xfs_agblock_t bno, + xfs_extlen_t len, enum xbtree_recpacking *outcome); + +struct xfs_rmap_matches { + /* Number of owner matches. */ + unsigned long long matches; + + /* Number of non-owner matches. */ + unsigned long long non_owner_matches; + + /* Number of non-owner matches that conflict with the owner matches. */ + unsigned long long bad_non_owner_matches; +}; + +int xfs_rmap_count_owners(struct xfs_btree_cur *cur, xfs_agblock_t bno, + xfs_extlen_t len, const struct xfs_owner_info *oinfo, + struct xfs_rmap_matches *rmatch); +int xfs_rmap_has_other_keys(struct xfs_btree_cur *cur, xfs_agblock_t bno, + xfs_extlen_t len, const struct xfs_owner_info *oinfo, + bool *has_other); +int xfs_rmap_map_raw(struct xfs_btree_cur *cur, struct xfs_rmap_irec *rmap); + +extern const struct xfs_owner_info XFS_RMAP_OINFO_SKIP_UPDATE; +extern const struct xfs_owner_info XFS_RMAP_OINFO_ANY_OWNER; +extern const struct xfs_owner_info XFS_RMAP_OINFO_FS; +extern const struct xfs_owner_info XFS_RMAP_OINFO_LOG; +extern const struct xfs_owner_info XFS_RMAP_OINFO_AG; +extern const struct xfs_owner_info XFS_RMAP_OINFO_INOBT; +extern const struct xfs_owner_info XFS_RMAP_OINFO_INODES; +extern const struct xfs_owner_info XFS_RMAP_OINFO_REFC; +extern const struct xfs_owner_info XFS_RMAP_OINFO_COW; + +extern struct kmem_cache *xfs_rmap_intent_cache; + +int __init xfs_rmap_intent_init_cache(void); +void xfs_rmap_intent_destroy_cache(void); + +/* + * Parameters for tracking reverse mapping changes. The hook function arg + * parameter is enum xfs_rmap_intent_type, and the rest is below. + */ +struct xfs_rmap_update_params { + xfs_agblock_t startblock; + xfs_extlen_t blockcount; + struct xfs_owner_info oinfo; + bool unwritten; +}; + +#ifdef CONFIG_XFS_LIVE_HOOKS + +struct xfs_rmap_hook { + struct xfs_hook rmap_hook; +}; + +void xfs_rmap_hook_disable(void); +void xfs_rmap_hook_enable(void); + +int xfs_rmap_hook_add(struct xfs_group *xg, struct xfs_rmap_hook *hook); +void xfs_rmap_hook_del(struct xfs_group *xg, struct xfs_rmap_hook *hook); +void xfs_rmap_hook_setup(struct xfs_rmap_hook *hook, notifier_fn_t mod_fn); +#endif + +#endif /* __XFS_RMAP_H__ */ diff --git a/fs/xfs/libxfs/xfs_rmap_btree.c b/fs/xfs/libxfs/xfs_rmap_btree.c new file mode 100644 index 000000000..10b327223 --- /dev/null +++ b/fs/xfs/libxfs/xfs_rmap_btree.c @@ -0,0 +1,872 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2014 Red Hat, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_trans.h" +#include "xfs_alloc.h" +#include "xfs_btree.h" +#include "xfs_btree_staging.h" +#include "xfs_rmap.h" +#include "xfs_rmap_btree.h" +#include "xfs_health.h" +#include "xfs_trace.h" +#include "xfs_error.h" +#include "xfs_extent_busy.h" +#include "xfs_ag.h" +#include "xfs_ag_resv.h" +#include "xfs_buf_mem.h" +#include "xfs_btree_mem.h" + +static struct kmem_cache *xfs_rmapbt_cur_cache; + +/* + * Reverse map btree. + * + * This is a per-ag tree used to track the owner(s) of a given extent. With + * reflink it is possible for there to be multiple owners, which is a departure + * from classic XFS. Owner records for data extents are inserted when the + * extent is mapped and removed when an extent is unmapped. Owner records for + * all other block types (i.e. metadata) are inserted when an extent is + * allocated and removed when an extent is freed. There can only be one owner + * of a metadata extent, usually an inode or some other metadata structure like + * an AG btree. + * + * The rmap btree is part of the free space management, so blocks for the tree + * are sourced from the agfl. Hence we need transaction reservation support for + * this tree so that the freelist is always large enough. This also impacts on + * the minimum space we need to leave free in the AG. + * + * The tree is ordered by [ag block, owner, offset]. This is a large key size, + * but it is the only way to enforce unique keys when a block can be owned by + * multiple files at any offset. There's no need to order/search by extent + * size for online updating/management of the tree. It is intended that most + * reverse lookups will be to find the owner(s) of a particular block, or to + * try to recover tree and file data from corrupt primary metadata. + */ + +static struct xfs_btree_cur * +xfs_rmapbt_dup_cursor( + struct xfs_btree_cur *cur) +{ + return xfs_rmapbt_init_cursor(cur->bc_mp, cur->bc_tp, + cur->bc_ag.agbp, to_perag(cur->bc_group)); +} + +STATIC void +xfs_rmapbt_set_root( + struct xfs_btree_cur *cur, + const union xfs_btree_ptr *ptr, + int inc) +{ + struct xfs_buf *agbp = cur->bc_ag.agbp; + struct xfs_agf *agf = agbp->b_addr; + struct xfs_perag *pag = to_perag(cur->bc_group); + + ASSERT(ptr->s != 0); + + agf->agf_rmap_root = ptr->s; + be32_add_cpu(&agf->agf_rmap_level, inc); + pag->pagf_rmap_level += inc; + + xfs_alloc_log_agf(cur->bc_tp, agbp, XFS_AGF_ROOTS | XFS_AGF_LEVELS); +} + +STATIC int +xfs_rmapbt_alloc_block( + struct xfs_btree_cur *cur, + const union xfs_btree_ptr *start, + union xfs_btree_ptr *new, + int *stat) +{ + struct xfs_buf *agbp = cur->bc_ag.agbp; + struct xfs_agf *agf = agbp->b_addr; + struct xfs_perag *pag = to_perag(cur->bc_group); + struct xfs_alloc_arg args = { .len = 1 }; + int error; + xfs_agblock_t bno; + + /* Allocate the new block from the freelist. If we can't, give up. */ + error = xfs_alloc_get_freelist(pag, cur->bc_tp, cur->bc_ag.agbp, + &bno, 1); + if (error) + return error; + if (bno == NULLAGBLOCK) { + *stat = 0; + return 0; + } + + xfs_extent_busy_reuse(pag_group(pag), bno, 1, false); + + new->s = cpu_to_be32(bno); + be32_add_cpu(&agf->agf_rmap_blocks, 1); + xfs_alloc_log_agf(cur->bc_tp, agbp, XFS_AGF_RMAP_BLOCKS); + + /* + * Since rmapbt blocks are sourced from the AGFL, they are allocated one + * at a time and the reservation updates don't require a transaction. + */ + xfs_ag_resv_alloc_extent(pag, XFS_AG_RESV_RMAPBT, &args); + + *stat = 1; + return 0; +} + +STATIC int +xfs_rmapbt_free_block( + struct xfs_btree_cur *cur, + struct xfs_buf *bp) +{ + struct xfs_buf *agbp = cur->bc_ag.agbp; + struct xfs_agf *agf = agbp->b_addr; + struct xfs_perag *pag = to_perag(cur->bc_group); + xfs_agblock_t bno; + int error; + + bno = xfs_daddr_to_agbno(cur->bc_mp, xfs_buf_daddr(bp)); + be32_add_cpu(&agf->agf_rmap_blocks, -1); + xfs_alloc_log_agf(cur->bc_tp, agbp, XFS_AGF_RMAP_BLOCKS); + error = xfs_alloc_put_freelist(pag, cur->bc_tp, agbp, NULL, bno, 1); + if (error) + return error; + + xfs_extent_busy_insert(cur->bc_tp, pag_group(pag), bno, 1, + XFS_EXTENT_BUSY_SKIP_DISCARD); + + xfs_ag_resv_free_extent(pag, XFS_AG_RESV_RMAPBT, NULL, 1); + return 0; +} + +STATIC int +xfs_rmapbt_get_minrecs( + struct xfs_btree_cur *cur, + int level) +{ + return cur->bc_mp->m_rmap_mnr[level != 0]; +} + +STATIC int +xfs_rmapbt_get_maxrecs( + struct xfs_btree_cur *cur, + int level) +{ + return cur->bc_mp->m_rmap_mxr[level != 0]; +} + +/* + * Convert the ondisk record's offset field into the ondisk key's offset field. + * Fork and bmbt are significant parts of the rmap record key, but written + * status is merely a record attribute. + */ +static inline __be64 ondisk_rec_offset_to_key(const union xfs_btree_rec *rec) +{ + return rec->rmap.rm_offset & ~cpu_to_be64(XFS_RMAP_OFF_UNWRITTEN); +} + +STATIC void +xfs_rmapbt_init_key_from_rec( + union xfs_btree_key *key, + const union xfs_btree_rec *rec) +{ + key->rmap.rm_startblock = rec->rmap.rm_startblock; + key->rmap.rm_owner = rec->rmap.rm_owner; + key->rmap.rm_offset = ondisk_rec_offset_to_key(rec); +} + +/* + * The high key for a reverse mapping record can be computed by shifting + * the startblock and offset to the highest value that would still map + * to that record. In practice this means that we add blockcount-1 to + * the startblock for all records, and if the record is for a data/attr + * fork mapping, we add blockcount-1 to the offset too. + */ +STATIC void +xfs_rmapbt_init_high_key_from_rec( + union xfs_btree_key *key, + const union xfs_btree_rec *rec) +{ + uint64_t off; + int adj; + + adj = be32_to_cpu(rec->rmap.rm_blockcount) - 1; + + key->rmap.rm_startblock = rec->rmap.rm_startblock; + be32_add_cpu(&key->rmap.rm_startblock, adj); + key->rmap.rm_owner = rec->rmap.rm_owner; + key->rmap.rm_offset = ondisk_rec_offset_to_key(rec); + if (XFS_RMAP_NON_INODE_OWNER(be64_to_cpu(rec->rmap.rm_owner)) || + XFS_RMAP_IS_BMBT_BLOCK(be64_to_cpu(rec->rmap.rm_offset))) + return; + off = be64_to_cpu(key->rmap.rm_offset); + off = (XFS_RMAP_OFF(off) + adj) | (off & ~XFS_RMAP_OFF_MASK); + key->rmap.rm_offset = cpu_to_be64(off); +} + +STATIC void +xfs_rmapbt_init_rec_from_cur( + struct xfs_btree_cur *cur, + union xfs_btree_rec *rec) +{ + rec->rmap.rm_startblock = cpu_to_be32(cur->bc_rec.r.rm_startblock); + rec->rmap.rm_blockcount = cpu_to_be32(cur->bc_rec.r.rm_blockcount); + rec->rmap.rm_owner = cpu_to_be64(cur->bc_rec.r.rm_owner); + rec->rmap.rm_offset = cpu_to_be64( + xfs_rmap_irec_offset_pack(&cur->bc_rec.r)); +} + +STATIC void +xfs_rmapbt_init_ptr_from_cur( + struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr) +{ + struct xfs_agf *agf = cur->bc_ag.agbp->b_addr; + + ASSERT(cur->bc_group->xg_gno == be32_to_cpu(agf->agf_seqno)); + + ptr->s = agf->agf_rmap_root; +} + +/* + * Mask the appropriate parts of the ondisk key field for a key comparison. + * Fork and bmbt are significant parts of the rmap record key, but written + * status is merely a record attribute. + */ +static inline uint64_t offset_keymask(uint64_t offset) +{ + return offset & ~XFS_RMAP_OFF_UNWRITTEN; +} + +STATIC int +xfs_rmapbt_cmp_key_with_cur( + struct xfs_btree_cur *cur, + const union xfs_btree_key *key) +{ + struct xfs_rmap_irec *rec = &cur->bc_rec.r; + const struct xfs_rmap_key *kp = &key->rmap; + + return cmp_int(be32_to_cpu(kp->rm_startblock), rec->rm_startblock) ?: + cmp_int(be64_to_cpu(kp->rm_owner), rec->rm_owner) ?: + cmp_int(offset_keymask(be64_to_cpu(kp->rm_offset)), + offset_keymask(xfs_rmap_irec_offset_pack(rec))); +} + +STATIC int +xfs_rmapbt_cmp_two_keys( + struct xfs_btree_cur *cur, + const union xfs_btree_key *k1, + const union xfs_btree_key *k2, + const union xfs_btree_key *mask) +{ + const struct xfs_rmap_key *kp1 = &k1->rmap; + const struct xfs_rmap_key *kp2 = &k2->rmap; + int d; + + /* Doesn't make sense to mask off the physical space part */ + ASSERT(!mask || mask->rmap.rm_startblock); + + d = cmp_int(be32_to_cpu(kp1->rm_startblock), + be32_to_cpu(kp2->rm_startblock)); + if (d) + return d; + + if (!mask || mask->rmap.rm_owner) { + d = cmp_int(be64_to_cpu(kp1->rm_owner), + be64_to_cpu(kp2->rm_owner)); + if (d) + return d; + } + + if (!mask || mask->rmap.rm_offset) { + /* Doesn't make sense to allow offset but not owner */ + ASSERT(!mask || mask->rmap.rm_owner); + + d = cmp_int(offset_keymask(be64_to_cpu(kp1->rm_offset)), + offset_keymask(be64_to_cpu(kp2->rm_offset))); + if (d) + return d; + } + + return 0; +} + +static xfs_failaddr_t +xfs_rmapbt_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_btree_block *block = XFS_BUF_TO_BLOCK(bp); + struct xfs_perag *pag = bp->b_pag; + xfs_failaddr_t fa; + unsigned int level; + + /* + * magic number and level verification + * + * During growfs operations, we can't verify the exact level or owner as + * the perag is not fully initialised and hence not attached to the + * buffer. In this case, check against the maximum tree depth. + * + * Similarly, during log recovery we will have a perag structure + * attached, but the agf information will not yet have been initialised + * from the on disk AGF. Again, we can only check against maximum limits + * in this case. + */ + if (!xfs_verify_magic(bp, block->bb_magic)) + return __this_address; + + if (!xfs_has_rmapbt(mp)) + return __this_address; + fa = xfs_btree_agblock_v5hdr_verify(bp); + if (fa) + return fa; + + level = be16_to_cpu(block->bb_level); + if (pag && xfs_perag_initialised_agf(pag)) { + unsigned int maxlevel = pag->pagf_rmap_level; + +#ifdef CONFIG_XFS_ONLINE_REPAIR + /* + * Online repair could be rewriting the free space btrees, so + * we'll validate against the larger of either tree while this + * is going on. + */ + maxlevel = max_t(unsigned int, maxlevel, + pag->pagf_repair_rmap_level); +#endif + if (level >= maxlevel) + return __this_address; + } else if (level >= mp->m_rmap_maxlevels) + return __this_address; + + return xfs_btree_agblock_verify(bp, mp->m_rmap_mxr[level != 0]); +} + +static void +xfs_rmapbt_read_verify( + struct xfs_buf *bp) +{ + xfs_failaddr_t fa; + + if (!xfs_btree_agblock_verify_crc(bp)) + xfs_verifier_error(bp, -EFSBADCRC, __this_address); + else { + fa = xfs_rmapbt_verify(bp); + if (fa) + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + } + + if (bp->b_error) + trace_xfs_btree_corrupt(bp, _RET_IP_); +} + +static void +xfs_rmapbt_write_verify( + struct xfs_buf *bp) +{ + xfs_failaddr_t fa; + + fa = xfs_rmapbt_verify(bp); + if (fa) { + trace_xfs_btree_corrupt(bp, _RET_IP_); + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + return; + } + xfs_btree_agblock_calc_crc(bp); + +} + +const struct xfs_buf_ops xfs_rmapbt_buf_ops = { + .name = "xfs_rmapbt", + .magic = { 0, cpu_to_be32(XFS_RMAP_CRC_MAGIC) }, + .verify_read = xfs_rmapbt_read_verify, + .verify_write = xfs_rmapbt_write_verify, + .verify_struct = xfs_rmapbt_verify, +}; + +STATIC int +xfs_rmapbt_keys_inorder( + struct xfs_btree_cur *cur, + const union xfs_btree_key *k1, + const union xfs_btree_key *k2) +{ + uint32_t x; + uint32_t y; + uint64_t a; + uint64_t b; + + x = be32_to_cpu(k1->rmap.rm_startblock); + y = be32_to_cpu(k2->rmap.rm_startblock); + if (x < y) + return 1; + else if (x > y) + return 0; + a = be64_to_cpu(k1->rmap.rm_owner); + b = be64_to_cpu(k2->rmap.rm_owner); + if (a < b) + return 1; + else if (a > b) + return 0; + a = offset_keymask(be64_to_cpu(k1->rmap.rm_offset)); + b = offset_keymask(be64_to_cpu(k2->rmap.rm_offset)); + if (a <= b) + return 1; + return 0; +} + +STATIC int +xfs_rmapbt_recs_inorder( + struct xfs_btree_cur *cur, + const union xfs_btree_rec *r1, + const union xfs_btree_rec *r2) +{ + uint32_t x; + uint32_t y; + uint64_t a; + uint64_t b; + + x = be32_to_cpu(r1->rmap.rm_startblock); + y = be32_to_cpu(r2->rmap.rm_startblock); + if (x < y) + return 1; + else if (x > y) + return 0; + a = be64_to_cpu(r1->rmap.rm_owner); + b = be64_to_cpu(r2->rmap.rm_owner); + if (a < b) + return 1; + else if (a > b) + return 0; + a = offset_keymask(be64_to_cpu(r1->rmap.rm_offset)); + b = offset_keymask(be64_to_cpu(r2->rmap.rm_offset)); + if (a <= b) + return 1; + return 0; +} + +STATIC enum xbtree_key_contig +xfs_rmapbt_keys_contiguous( + struct xfs_btree_cur *cur, + const union xfs_btree_key *key1, + const union xfs_btree_key *key2, + const union xfs_btree_key *mask) +{ + ASSERT(!mask || mask->rmap.rm_startblock); + + /* + * We only support checking contiguity of the physical space component. + * If any callers ever need more specificity than that, they'll have to + * implement it here. + */ + ASSERT(!mask || (!mask->rmap.rm_owner && !mask->rmap.rm_offset)); + + return xbtree_key_contig(be32_to_cpu(key1->rmap.rm_startblock), + be32_to_cpu(key2->rmap.rm_startblock)); +} + +const struct xfs_btree_ops xfs_rmapbt_ops = { + .name = "rmap", + .type = XFS_BTREE_TYPE_AG, + .geom_flags = XFS_BTGEO_OVERLAPPING, + + .rec_len = sizeof(struct xfs_rmap_rec), + /* Overlapping btree; 2 keys per pointer. */ + .key_len = 2 * sizeof(struct xfs_rmap_key), + .ptr_len = XFS_BTREE_SHORT_PTR_LEN, + + .lru_refs = XFS_RMAP_BTREE_REF, + .statoff = XFS_STATS_CALC_INDEX(xs_rmap_2), + .sick_mask = XFS_SICK_AG_RMAPBT, + + .dup_cursor = xfs_rmapbt_dup_cursor, + .set_root = xfs_rmapbt_set_root, + .alloc_block = xfs_rmapbt_alloc_block, + .free_block = xfs_rmapbt_free_block, + .get_minrecs = xfs_rmapbt_get_minrecs, + .get_maxrecs = xfs_rmapbt_get_maxrecs, + .init_key_from_rec = xfs_rmapbt_init_key_from_rec, + .init_high_key_from_rec = xfs_rmapbt_init_high_key_from_rec, + .init_rec_from_cur = xfs_rmapbt_init_rec_from_cur, + .init_ptr_from_cur = xfs_rmapbt_init_ptr_from_cur, + .cmp_key_with_cur = xfs_rmapbt_cmp_key_with_cur, + .buf_ops = &xfs_rmapbt_buf_ops, + .cmp_two_keys = xfs_rmapbt_cmp_two_keys, + .keys_inorder = xfs_rmapbt_keys_inorder, + .recs_inorder = xfs_rmapbt_recs_inorder, + .keys_contiguous = xfs_rmapbt_keys_contiguous, +}; + +/* + * Create a new reverse mapping btree cursor. + * + * For staging cursors tp and agbp are NULL. + */ +struct xfs_btree_cur * +xfs_rmapbt_init_cursor( + struct xfs_mount *mp, + struct xfs_trans *tp, + struct xfs_buf *agbp, + struct xfs_perag *pag) +{ + struct xfs_btree_cur *cur; + + cur = xfs_btree_alloc_cursor(mp, tp, &xfs_rmapbt_ops, + mp->m_rmap_maxlevels, xfs_rmapbt_cur_cache); + cur->bc_group = xfs_group_hold(pag_group(pag)); + cur->bc_ag.agbp = agbp; + if (agbp) { + struct xfs_agf *agf = agbp->b_addr; + + cur->bc_nlevels = be32_to_cpu(agf->agf_rmap_level); + } + return cur; +} + +#ifdef CONFIG_XFS_BTREE_IN_MEM +static inline unsigned int +xfs_rmapbt_mem_block_maxrecs( + unsigned int blocklen, + bool leaf) +{ + if (leaf) + return blocklen / sizeof(struct xfs_rmap_rec); + return blocklen / + (2 * sizeof(struct xfs_rmap_key) + sizeof(__be64)); +} + +/* + * Validate an in-memory rmap btree block. Callers are allowed to generate an + * in-memory btree even if the ondisk feature is not enabled. + */ +static xfs_failaddr_t +xfs_rmapbt_mem_verify( + struct xfs_buf *bp) +{ + struct xfs_btree_block *block = XFS_BUF_TO_BLOCK(bp); + xfs_failaddr_t fa; + unsigned int level; + unsigned int maxrecs; + + if (!xfs_verify_magic(bp, block->bb_magic)) + return __this_address; + + fa = xfs_btree_fsblock_v5hdr_verify(bp, XFS_RMAP_OWN_UNKNOWN); + if (fa) + return fa; + + level = be16_to_cpu(block->bb_level); + if (level >= xfs_rmapbt_maxlevels_ondisk()) + return __this_address; + + maxrecs = xfs_rmapbt_mem_block_maxrecs( + XFBNO_BLOCKSIZE - XFS_BTREE_LBLOCK_CRC_LEN, level == 0); + return xfs_btree_memblock_verify(bp, maxrecs); +} + +static void +xfs_rmapbt_mem_rw_verify( + struct xfs_buf *bp) +{ + xfs_failaddr_t fa = xfs_rmapbt_mem_verify(bp); + + if (fa) + xfs_verifier_error(bp, -EFSCORRUPTED, fa); +} + +/* skip crc checks on in-memory btrees to save time */ +static const struct xfs_buf_ops xfs_rmapbt_mem_buf_ops = { + .name = "xfs_rmapbt_mem", + .magic = { 0, cpu_to_be32(XFS_RMAP_CRC_MAGIC) }, + .verify_read = xfs_rmapbt_mem_rw_verify, + .verify_write = xfs_rmapbt_mem_rw_verify, + .verify_struct = xfs_rmapbt_mem_verify, +}; + +const struct xfs_btree_ops xfs_rmapbt_mem_ops = { + .name = "mem_rmap", + .type = XFS_BTREE_TYPE_MEM, + .geom_flags = XFS_BTGEO_OVERLAPPING, + + .rec_len = sizeof(struct xfs_rmap_rec), + /* Overlapping btree; 2 keys per pointer. */ + .key_len = 2 * sizeof(struct xfs_rmap_key), + .ptr_len = XFS_BTREE_LONG_PTR_LEN, + + .lru_refs = XFS_RMAP_BTREE_REF, + .statoff = XFS_STATS_CALC_INDEX(xs_rmap_mem_2), + + .dup_cursor = xfbtree_dup_cursor, + .set_root = xfbtree_set_root, + .alloc_block = xfbtree_alloc_block, + .free_block = xfbtree_free_block, + .get_minrecs = xfbtree_get_minrecs, + .get_maxrecs = xfbtree_get_maxrecs, + .init_key_from_rec = xfs_rmapbt_init_key_from_rec, + .init_high_key_from_rec = xfs_rmapbt_init_high_key_from_rec, + .init_rec_from_cur = xfs_rmapbt_init_rec_from_cur, + .init_ptr_from_cur = xfbtree_init_ptr_from_cur, + .cmp_key_with_cur = xfs_rmapbt_cmp_key_with_cur, + .buf_ops = &xfs_rmapbt_mem_buf_ops, + .cmp_two_keys = xfs_rmapbt_cmp_two_keys, + .keys_inorder = xfs_rmapbt_keys_inorder, + .recs_inorder = xfs_rmapbt_recs_inorder, + .keys_contiguous = xfs_rmapbt_keys_contiguous, +}; + +/* Create a cursor for an in-memory btree. */ +struct xfs_btree_cur * +xfs_rmapbt_mem_cursor( + struct xfs_perag *pag, + struct xfs_trans *tp, + struct xfbtree *xfbt) +{ + struct xfs_btree_cur *cur; + + cur = xfs_btree_alloc_cursor(pag_mount(pag), tp, &xfs_rmapbt_mem_ops, + xfs_rmapbt_maxlevels_ondisk(), xfs_rmapbt_cur_cache); + cur->bc_mem.xfbtree = xfbt; + cur->bc_nlevels = xfbt->nlevels; + + cur->bc_group = xfs_group_hold(pag_group(pag)); + return cur; +} + +/* Create an in-memory rmap btree. */ +int +xfs_rmapbt_mem_init( + struct xfs_mount *mp, + struct xfbtree *xfbt, + struct xfs_buftarg *btp, + xfs_agnumber_t agno) +{ + xfbt->owner = agno; + return xfbtree_init(mp, xfbt, btp, &xfs_rmapbt_mem_ops); +} + +/* Compute the max possible height for reverse mapping btrees in memory. */ +static unsigned int +xfs_rmapbt_mem_maxlevels(void) +{ + unsigned int minrecs[2]; + unsigned int blocklen; + + blocklen = XFBNO_BLOCKSIZE - XFS_BTREE_LBLOCK_CRC_LEN; + + minrecs[0] = xfs_rmapbt_mem_block_maxrecs(blocklen, true) / 2; + minrecs[1] = xfs_rmapbt_mem_block_maxrecs(blocklen, false) / 2; + + /* + * How tall can an in-memory rmap btree become if we filled the entire + * AG with rmap records? + */ + return xfs_btree_compute_maxlevels(minrecs, + XFS_MAX_AG_BYTES / sizeof(struct xfs_rmap_rec)); +} +#else +# define xfs_rmapbt_mem_maxlevels() (0) +#endif /* CONFIG_XFS_BTREE_IN_MEM */ + +/* + * Install a new reverse mapping btree root. Caller is responsible for + * invalidating and freeing the old btree blocks. + */ +void +xfs_rmapbt_commit_staged_btree( + struct xfs_btree_cur *cur, + struct xfs_trans *tp, + struct xfs_buf *agbp) +{ + struct xfs_agf *agf = agbp->b_addr; + struct xbtree_afakeroot *afake = cur->bc_ag.afake; + + ASSERT(cur->bc_flags & XFS_BTREE_STAGING); + + agf->agf_rmap_root = cpu_to_be32(afake->af_root); + agf->agf_rmap_level = cpu_to_be32(afake->af_levels); + agf->agf_rmap_blocks = cpu_to_be32(afake->af_blocks); + xfs_alloc_log_agf(tp, agbp, XFS_AGF_ROOTS | XFS_AGF_LEVELS | + XFS_AGF_RMAP_BLOCKS); + xfs_btree_commit_afakeroot(cur, tp, agbp); +} + +/* Calculate number of records in a reverse mapping btree block. */ +static inline unsigned int +xfs_rmapbt_block_maxrecs( + unsigned int blocklen, + bool leaf) +{ + if (leaf) + return blocklen / sizeof(struct xfs_rmap_rec); + return blocklen / + (2 * sizeof(struct xfs_rmap_key) + sizeof(xfs_rmap_ptr_t)); +} + +/* + * Calculate number of records in an rmap btree block. + */ +unsigned int +xfs_rmapbt_maxrecs( + struct xfs_mount *mp, + unsigned int blocklen, + bool leaf) +{ + blocklen -= XFS_RMAP_BLOCK_LEN; + return xfs_rmapbt_block_maxrecs(blocklen, leaf); +} + +/* Compute the max possible height for reverse mapping btrees. */ +unsigned int +xfs_rmapbt_maxlevels_ondisk(void) +{ + unsigned int minrecs[2]; + unsigned int blocklen; + + blocklen = XFS_MIN_CRC_BLOCKSIZE - XFS_BTREE_SBLOCK_CRC_LEN; + + minrecs[0] = xfs_rmapbt_block_maxrecs(blocklen, true) / 2; + minrecs[1] = xfs_rmapbt_block_maxrecs(blocklen, false) / 2; + + /* + * Compute the asymptotic maxlevels for an rmapbt on any reflink fs. + * + * On a reflink filesystem, each AG block can have up to 2^32 (per the + * refcount record format) owners, which means that theoretically we + * could face up to 2^64 rmap records. However, we're likely to run + * out of blocks in the AG long before that happens, which means that + * we must compute the max height based on what the btree will look + * like if it consumes almost all the blocks in the AG due to maximal + * sharing factor. + */ + return max(xfs_btree_space_to_height(minrecs, XFS_MAX_CRC_AG_BLOCKS), + xfs_rmapbt_mem_maxlevels()); +} + +/* Compute the maximum height of an rmap btree. */ +void +xfs_rmapbt_compute_maxlevels( + struct xfs_mount *mp) +{ + if (!xfs_has_rmapbt(mp)) { + mp->m_rmap_maxlevels = 0; + return; + } + + if (xfs_has_reflink(mp)) { + /* + * Compute the asymptotic maxlevels for an rmap btree on a + * filesystem that supports reflink. + * + * On a reflink filesystem, each AG block can have up to 2^32 + * (per the refcount record format) owners, which means that + * theoretically we could face up to 2^64 rmap records. + * However, we're likely to run out of blocks in the AG long + * before that happens, which means that we must compute the + * max height based on what the btree will look like if it + * consumes almost all the blocks in the AG due to maximal + * sharing factor. + */ + mp->m_rmap_maxlevels = xfs_btree_space_to_height(mp->m_rmap_mnr, + mp->m_sb.sb_agblocks); + } else { + /* + * If there's no block sharing, compute the maximum rmapbt + * height assuming one rmap record per AG block. + */ + mp->m_rmap_maxlevels = xfs_btree_compute_maxlevels( + mp->m_rmap_mnr, mp->m_sb.sb_agblocks); + } + ASSERT(mp->m_rmap_maxlevels <= xfs_rmapbt_maxlevels_ondisk()); +} + +/* Calculate the refcount btree size for some records. */ +xfs_extlen_t +xfs_rmapbt_calc_size( + struct xfs_mount *mp, + unsigned long long len) +{ + return xfs_btree_calc_size(mp->m_rmap_mnr, len); +} + +/* + * Calculate the maximum refcount btree size. + */ +xfs_extlen_t +xfs_rmapbt_max_size( + struct xfs_mount *mp, + xfs_agblock_t agblocks) +{ + /* Bail out if we're uninitialized, which can happen in mkfs. */ + if (mp->m_rmap_mxr[0] == 0) + return 0; + + return xfs_rmapbt_calc_size(mp, agblocks); +} + +/* + * Figure out how many blocks to reserve and how many are used by this btree. + */ +int +xfs_rmapbt_calc_reserves( + struct xfs_mount *mp, + struct xfs_trans *tp, + struct xfs_perag *pag, + xfs_extlen_t *ask, + xfs_extlen_t *used) +{ + struct xfs_buf *agbp; + struct xfs_agf *agf; + xfs_agblock_t agblocks; + xfs_extlen_t tree_len; + int error; + + if (!xfs_has_rmapbt(mp)) + return 0; + + error = xfs_alloc_read_agf(pag, tp, 0, &agbp); + if (error) + return error; + + agf = agbp->b_addr; + agblocks = be32_to_cpu(agf->agf_length); + tree_len = be32_to_cpu(agf->agf_rmap_blocks); + xfs_trans_brelse(tp, agbp); + + /* + * The log is permanently allocated, so the space it occupies will + * never be available for the kinds of things that would require btree + * expansion. We therefore can pretend the space isn't there. + */ + if (xfs_ag_contains_log(mp, pag_agno(pag))) + agblocks -= mp->m_sb.sb_logblocks; + + /* Reserve 1% of the AG or enough for 1 block per record. */ + *ask += max(agblocks / 100, xfs_rmapbt_max_size(mp, agblocks)); + *used += tree_len; + + return error; +} + +int __init +xfs_rmapbt_init_cur_cache(void) +{ + xfs_rmapbt_cur_cache = kmem_cache_create("xfs_rmapbt_cur", + xfs_btree_cur_sizeof(xfs_rmapbt_maxlevels_ondisk()), + 0, 0, NULL); + + if (!xfs_rmapbt_cur_cache) + return -ENOMEM; + return 0; +} + +void +xfs_rmapbt_destroy_cur_cache(void) +{ + kmem_cache_destroy(xfs_rmapbt_cur_cache); + xfs_rmapbt_cur_cache = NULL; +} diff --git a/fs/xfs/libxfs/xfs_rmap_btree.h b/fs/xfs/libxfs/xfs_rmap_btree.h new file mode 100644 index 000000000..119b1567c --- /dev/null +++ b/fs/xfs/libxfs/xfs_rmap_btree.h @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2014 Red Hat, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_RMAP_BTREE_H__ +#define __XFS_RMAP_BTREE_H__ + +struct xfs_buf; +struct xfs_btree_cur; +struct xfs_mount; +struct xbtree_afakeroot; +struct xfbtree; + +/* rmaps only exist on crc enabled filesystems */ +#define XFS_RMAP_BLOCK_LEN XFS_BTREE_SBLOCK_CRC_LEN + +/* + * Record, key, and pointer address macros for btree blocks. + * + * (note that some of these may appear unused, but they are used in userspace) + */ +#define XFS_RMAP_REC_ADDR(block, index) \ + ((struct xfs_rmap_rec *) \ + ((char *)(block) + XFS_RMAP_BLOCK_LEN + \ + (((index) - 1) * sizeof(struct xfs_rmap_rec)))) + +#define XFS_RMAP_KEY_ADDR(block, index) \ + ((struct xfs_rmap_key *) \ + ((char *)(block) + XFS_RMAP_BLOCK_LEN + \ + ((index) - 1) * 2 * sizeof(struct xfs_rmap_key))) + +#define XFS_RMAP_HIGH_KEY_ADDR(block, index) \ + ((struct xfs_rmap_key *) \ + ((char *)(block) + XFS_RMAP_BLOCK_LEN + \ + sizeof(struct xfs_rmap_key) + \ + ((index) - 1) * 2 * sizeof(struct xfs_rmap_key))) + +#define XFS_RMAP_PTR_ADDR(block, index, maxrecs) \ + ((xfs_rmap_ptr_t *) \ + ((char *)(block) + XFS_RMAP_BLOCK_LEN + \ + (maxrecs) * 2 * sizeof(struct xfs_rmap_key) + \ + ((index) - 1) * sizeof(xfs_rmap_ptr_t))) + +struct xfs_btree_cur *xfs_rmapbt_init_cursor(struct xfs_mount *mp, + struct xfs_trans *tp, struct xfs_buf *bp, + struct xfs_perag *pag); +void xfs_rmapbt_commit_staged_btree(struct xfs_btree_cur *cur, + struct xfs_trans *tp, struct xfs_buf *agbp); +unsigned int xfs_rmapbt_maxrecs(struct xfs_mount *mp, unsigned int blocklen, + bool leaf); +extern void xfs_rmapbt_compute_maxlevels(struct xfs_mount *mp); + +extern xfs_extlen_t xfs_rmapbt_calc_size(struct xfs_mount *mp, + unsigned long long len); +extern xfs_extlen_t xfs_rmapbt_max_size(struct xfs_mount *mp, + xfs_agblock_t agblocks); + +extern int xfs_rmapbt_calc_reserves(struct xfs_mount *mp, struct xfs_trans *tp, + struct xfs_perag *pag, xfs_extlen_t *ask, xfs_extlen_t *used); + +unsigned int xfs_rmapbt_maxlevels_ondisk(void); + +int __init xfs_rmapbt_init_cur_cache(void); +void xfs_rmapbt_destroy_cur_cache(void); + +struct xfs_btree_cur *xfs_rmapbt_mem_cursor(struct xfs_perag *pag, + struct xfs_trans *tp, struct xfbtree *xfbtree); +int xfs_rmapbt_mem_init(struct xfs_mount *mp, struct xfbtree *xfbtree, + struct xfs_buftarg *btp, xfs_agnumber_t agno); + +#endif /* __XFS_RMAP_BTREE_H__ */ diff --git a/fs/xfs/libxfs/xfs_rtbitmap.c b/fs/xfs/libxfs/xfs_rtbitmap.c new file mode 100644 index 000000000..e62786e90 --- /dev/null +++ b/fs/xfs/libxfs/xfs_rtbitmap.c @@ -0,0 +1,1504 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_bmap.h" +#include "xfs_bmap_btree.h" +#include "xfs_trans_space.h" +#include "xfs_trans.h" +#include "xfs_rtalloc.h" +#include "xfs_error.h" +#include "xfs_rtbitmap.h" +#include "xfs_health.h" +#include "xfs_sb.h" +#include "xfs_errortag.h" +#include "xfs_log.h" +#include "xfs_buf_item.h" +#include "xfs_extent_busy.h" + +/* + * Realtime allocator bitmap functions shared with userspace. + */ + +static xfs_failaddr_t +xfs_rtbuf_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_rtbuf_blkinfo *hdr = bp->b_addr; + + if (!xfs_verify_magic(bp, hdr->rt_magic)) + return __this_address; + if (!xfs_has_rtgroups(mp)) + return __this_address; + if (!xfs_has_crc(mp)) + return __this_address; + if (!uuid_equal(&hdr->rt_uuid, &mp->m_sb.sb_meta_uuid)) + return __this_address; + if (hdr->rt_blkno != cpu_to_be64(xfs_buf_daddr(bp))) + return __this_address; + return NULL; +} + +static void +xfs_rtbuf_verify_read( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_rtbuf_blkinfo *hdr = bp->b_addr; + xfs_failaddr_t fa; + + if (!xfs_has_rtgroups(mp)) + return; + + if (!xfs_log_check_lsn(mp, be64_to_cpu(hdr->rt_lsn))) { + fa = __this_address; + goto fail; + } + + if (!xfs_buf_verify_cksum(bp, XFS_RTBUF_CRC_OFF)) { + fa = __this_address; + goto fail; + } + + fa = xfs_rtbuf_verify(bp); + if (fa) + goto fail; + + return; +fail: + xfs_verifier_error(bp, -EFSCORRUPTED, fa); +} + +static void +xfs_rtbuf_verify_write( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_rtbuf_blkinfo *hdr = bp->b_addr; + struct xfs_buf_log_item *bip = bp->b_log_item; + xfs_failaddr_t fa; + + if (!xfs_has_rtgroups(mp)) + return; + + fa = xfs_rtbuf_verify(bp); + if (fa) { + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + return; + } + + if (bip) + hdr->rt_lsn = cpu_to_be64(bip->bli_item.li_lsn); + xfs_buf_update_cksum(bp, XFS_RTBUF_CRC_OFF); +} + +const struct xfs_buf_ops xfs_rtbuf_ops = { + .name = "rtbuf", + .verify_read = xfs_rtbuf_verify_read, + .verify_write = xfs_rtbuf_verify_write, +}; + +const struct xfs_buf_ops xfs_rtbitmap_buf_ops = { + .name = "xfs_rtbitmap", + .magic = { 0, cpu_to_be32(XFS_RTBITMAP_MAGIC) }, + .verify_read = xfs_rtbuf_verify_read, + .verify_write = xfs_rtbuf_verify_write, + .verify_struct = xfs_rtbuf_verify, +}; + +const struct xfs_buf_ops xfs_rtsummary_buf_ops = { + .name = "xfs_rtsummary", + .magic = { 0, cpu_to_be32(XFS_RTSUMMARY_MAGIC) }, + .verify_read = xfs_rtbuf_verify_read, + .verify_write = xfs_rtbuf_verify_write, + .verify_struct = xfs_rtbuf_verify, +}; + +/* Release cached rt bitmap and summary buffers. */ +void +xfs_rtbuf_cache_relse( + struct xfs_rtalloc_args *args) +{ + if (args->rbmbp) { + xfs_trans_brelse(args->tp, args->rbmbp); + args->rbmbp = NULL; + args->rbmoff = NULLFILEOFF; + } + if (args->sumbp) { + xfs_trans_brelse(args->tp, args->sumbp); + args->sumbp = NULL; + args->sumoff = NULLFILEOFF; + } +} + +/* + * Get a buffer for the bitmap or summary file block specified. + * The buffer is returned read and locked. + */ +static int +xfs_rtbuf_get( + struct xfs_rtalloc_args *args, + xfs_fileoff_t block, /* block number in bitmap or summary */ + enum xfs_rtg_inodes type) +{ + struct xfs_inode *ip = args->rtg->rtg_inodes[type]; + struct xfs_mount *mp = args->mp; + struct xfs_buf **cbpp; /* cached block buffer */ + xfs_fileoff_t *coffp; /* cached block number */ + struct xfs_buf *bp; /* block buffer, result */ + struct xfs_bmbt_irec map; + enum xfs_blft buf_type; + int nmap = 1; + int error; + + switch (type) { + case XFS_RTGI_SUMMARY: + cbpp = &args->sumbp; + coffp = &args->sumoff; + buf_type = XFS_BLFT_RTSUMMARY_BUF; + break; + case XFS_RTGI_BITMAP: + cbpp = &args->rbmbp; + coffp = &args->rbmoff; + buf_type = XFS_BLFT_RTBITMAP_BUF; + break; + default: + return -EINVAL; + } + + /* + * If we have a cached buffer, and the block number matches, use that. + */ + if (*cbpp && *coffp == block) + return 0; + + /* + * Otherwise we have to have to get the buffer. If there was an old + * one, get rid of it first. + */ + if (*cbpp) { + xfs_trans_brelse(args->tp, *cbpp); + *cbpp = NULL; + } + + error = xfs_bmapi_read(ip, block, 1, &map, &nmap, 0); + if (error) + return error; + + if (XFS_IS_CORRUPT(mp, nmap == 0 || !xfs_bmap_is_written_extent(&map))) { + xfs_rtginode_mark_sick(args->rtg, type); + return -EFSCORRUPTED; + } + + ASSERT(map.br_startblock != NULLFSBLOCK); + error = xfs_trans_read_buf(mp, args->tp, mp->m_ddev_targp, + XFS_FSB_TO_DADDR(mp, map.br_startblock), + mp->m_bsize, 0, &bp, + xfs_rtblock_ops(mp, type)); + if (xfs_metadata_is_sick(error)) + xfs_rtginode_mark_sick(args->rtg, type); + if (error) + return error; + + if (xfs_has_rtgroups(mp)) { + struct xfs_rtbuf_blkinfo *hdr = bp->b_addr; + + if (hdr->rt_owner != cpu_to_be64(I_INO(ip))) { + xfs_buf_mark_corrupt(bp); + xfs_trans_brelse(args->tp, bp); + xfs_rtginode_mark_sick(args->rtg, type); + return -EFSCORRUPTED; + } + } + + xfs_trans_buf_set_type(args->tp, bp, buf_type); + *cbpp = bp; + *coffp = block; + return 0; +} + +int +xfs_rtbitmap_read_buf( + struct xfs_rtalloc_args *args, + xfs_fileoff_t block) +{ + struct xfs_mount *mp = args->mp; + + if (XFS_IS_CORRUPT(mp, block >= mp->m_sb.sb_rbmblocks)) { + xfs_rtginode_mark_sick(args->rtg, XFS_RTGI_BITMAP); + return -EFSCORRUPTED; + } + + return xfs_rtbuf_get(args, block, XFS_RTGI_BITMAP); +} + +int +xfs_rtsummary_read_buf( + struct xfs_rtalloc_args *args, + xfs_fileoff_t block) +{ + struct xfs_mount *mp = args->mp; + + if (XFS_IS_CORRUPT(mp, block >= mp->m_rsumblocks)) { + xfs_rtginode_mark_sick(args->rtg, XFS_RTGI_SUMMARY); + return -EFSCORRUPTED; + } + return xfs_rtbuf_get(args, block, XFS_RTGI_SUMMARY); +} + +/* + * Searching backward from start find the first block whose allocated/free state + * is different from start's. + */ +int +xfs_rtfind_back( + struct xfs_rtalloc_args *args, + xfs_rtxnum_t start, /* starting rtext to look at */ + xfs_rtxnum_t *rtx) /* out: start rtext found */ +{ + struct xfs_mount *mp = args->mp; + int bit; /* bit number in the word */ + xfs_fileoff_t block; /* bitmap block number */ + int error; /* error value */ + xfs_rtxnum_t firstbit; /* first useful bit in the word */ + xfs_rtxnum_t i; /* current bit number rel. to start */ + xfs_rtxnum_t len; /* length of inspected area */ + xfs_rtword_t mask; /* mask of relevant bits for value */ + xfs_rtword_t want; /* mask for "good" values */ + xfs_rtword_t wdiff; /* difference from wanted value */ + xfs_rtword_t incore; + unsigned int word; /* word number in the buffer */ + + /* + * Compute and read in starting bitmap block for starting block. + */ + block = xfs_rtx_to_rbmblock(mp, start); + error = xfs_rtbitmap_read_buf(args, block); + if (error) + return error; + + /* + * Get the first word's index & point to it. + */ + word = xfs_rtx_to_rbmword(mp, start); + bit = (int)(start & (XFS_NBWORD - 1)); + len = start + 1; + /* + * Compute match value, based on the bit at start: if 1 (free) + * then all-ones, else all-zeroes. + */ + incore = xfs_rtbitmap_getword(args, word); + want = (incore & ((xfs_rtword_t)1 << bit)) ? -1 : 0; + /* + * If the starting position is not word-aligned, deal with the + * partial word. + */ + if (bit < XFS_NBWORD - 1) { + /* + * Calculate first (leftmost) bit number to look at, + * and mask for all the relevant bits in this word. + */ + firstbit = max_t(xfs_srtblock_t, bit - len + 1, 0); + mask = (((xfs_rtword_t)1 << (bit - firstbit + 1)) - 1) << + firstbit; + /* + * Calculate the difference between the value there + * and what we're looking for. + */ + if ((wdiff = (incore ^ want) & mask)) { + /* + * Different. Mark where we are and return. + */ + i = bit - xfs_highbit32(wdiff); + *rtx = start - i + 1; + return 0; + } + i = bit - firstbit + 1; + /* + * Go on to previous block if that's where the previous word is + * and we need the previous word. + */ + if (--word == -1 && i < len) { + /* + * If done with this block, get the previous one. + */ + error = xfs_rtbitmap_read_buf(args, --block); + if (error) + return error; + + word = mp->m_blockwsize - 1; + } + } else { + /* + * Starting on a word boundary, no partial word. + */ + i = 0; + } + /* + * Loop over whole words in buffers. When we use up one buffer + * we move on to the previous one. + */ + while (len - i >= XFS_NBWORD) { + /* + * Compute difference between actual and desired value. + */ + incore = xfs_rtbitmap_getword(args, word); + if ((wdiff = incore ^ want)) { + /* + * Different, mark where we are and return. + */ + i += XFS_NBWORD - 1 - xfs_highbit32(wdiff); + *rtx = start - i + 1; + return 0; + } + i += XFS_NBWORD; + /* + * Go on to previous block if that's where the previous word is + * and we need the previous word. + */ + if (--word == -1 && i < len) { + /* + * If done with this block, get the previous one. + */ + error = xfs_rtbitmap_read_buf(args, --block); + if (error) + return error; + + word = mp->m_blockwsize - 1; + } + } + /* + * If not ending on a word boundary, deal with the last + * (partial) word. + */ + if (len - i) { + /* + * Calculate first (leftmost) bit number to look at, + * and mask for all the relevant bits in this word. + */ + firstbit = XFS_NBWORD - (len - i); + mask = (((xfs_rtword_t)1 << (len - i)) - 1) << firstbit; + /* + * Compute difference between actual and desired value. + */ + incore = xfs_rtbitmap_getword(args, word); + if ((wdiff = (incore ^ want) & mask)) { + /* + * Different, mark where we are and return. + */ + i += XFS_NBWORD - 1 - xfs_highbit32(wdiff); + *rtx = start - i + 1; + return 0; + } else + i = len; + } + /* + * No match, return that we scanned the whole area. + */ + *rtx = start - i + 1; + return 0; +} + +/* + * Searching forward from start to limit, find the first block whose + * allocated/free state is different from start's. + */ +int +xfs_rtfind_forw( + struct xfs_rtalloc_args *args, + xfs_rtxnum_t start, /* starting rtext to look at */ + xfs_rtxnum_t limit, /* last rtext to look at */ + xfs_rtxnum_t *rtx) /* out: start rtext found */ +{ + struct xfs_mount *mp = args->mp; + int bit; /* bit number in the word */ + xfs_fileoff_t block; /* bitmap block number */ + int error; + xfs_rtxnum_t i; /* current bit number rel. to start */ + xfs_rtxnum_t lastbit;/* last useful bit in the word */ + xfs_rtxnum_t len; /* length of inspected area */ + xfs_rtword_t mask; /* mask of relevant bits for value */ + xfs_rtword_t want; /* mask for "good" values */ + xfs_rtword_t wdiff; /* difference from wanted value */ + xfs_rtword_t incore; + unsigned int word; /* word number in the buffer */ + + ASSERT(start <= limit); + + /* + * Compute and read in starting bitmap block for starting block. + */ + block = xfs_rtx_to_rbmblock(mp, start); + error = xfs_rtbitmap_read_buf(args, block); + if (error) + return error; + + /* + * Get the first word's index & point to it. + */ + word = xfs_rtx_to_rbmword(mp, start); + bit = (int)(start & (XFS_NBWORD - 1)); + len = limit - start + 1; + /* + * Compute match value, based on the bit at start: if 1 (free) + * then all-ones, else all-zeroes. + */ + incore = xfs_rtbitmap_getword(args, word); + want = (incore & ((xfs_rtword_t)1 << bit)) ? -1 : 0; + /* + * If the starting position is not word-aligned, deal with the + * partial word. + */ + if (bit) { + /* + * Calculate last (rightmost) bit number to look at, + * and mask for all the relevant bits in this word. + */ + lastbit = min(bit + len, XFS_NBWORD); + mask = (((xfs_rtword_t)1 << (lastbit - bit)) - 1) << bit; + /* + * Calculate the difference between the value there + * and what we're looking for. + */ + if ((wdiff = (incore ^ want) & mask)) { + /* + * Different. Mark where we are and return. + */ + i = xfs_lowbit32(wdiff) - bit; + *rtx = start + i - 1; + return 0; + } + i = lastbit - bit; + /* + * Go on to next block if that's where the next word is + * and we need the next word. + */ + if (++word == mp->m_blockwsize && i < len) { + /* + * If done with this block, get the previous one. + */ + error = xfs_rtbitmap_read_buf(args, ++block); + if (error) + return error; + + word = 0; + } + } else { + /* + * Starting on a word boundary, no partial word. + */ + i = 0; + } + /* + * Loop over whole words in buffers. When we use up one buffer + * we move on to the next one. + */ + while (len - i >= XFS_NBWORD) { + /* + * Compute difference between actual and desired value. + */ + incore = xfs_rtbitmap_getword(args, word); + if ((wdiff = incore ^ want)) { + /* + * Different, mark where we are and return. + */ + i += xfs_lowbit32(wdiff); + *rtx = start + i - 1; + return 0; + } + i += XFS_NBWORD; + /* + * Go on to next block if that's where the next word is + * and we need the next word. + */ + if (++word == mp->m_blockwsize && i < len) { + /* + * If done with this block, get the next one. + */ + error = xfs_rtbitmap_read_buf(args, ++block); + if (error) + return error; + + word = 0; + } + } + /* + * If not ending on a word boundary, deal with the last + * (partial) word. + */ + if ((lastbit = len - i)) { + /* + * Calculate mask for all the relevant bits in this word. + */ + mask = ((xfs_rtword_t)1 << lastbit) - 1; + /* + * Compute difference between actual and desired value. + */ + incore = xfs_rtbitmap_getword(args, word); + if ((wdiff = (incore ^ want) & mask)) { + /* + * Different, mark where we are and return. + */ + i += xfs_lowbit32(wdiff); + *rtx = start + i - 1; + return 0; + } else + i = len; + } + /* + * No match, return that we scanned the whole area. + */ + *rtx = start + i - 1; + return 0; +} + +/* Log rtsummary counter at @infoword. */ +static inline void +xfs_trans_log_rtsummary( + struct xfs_rtalloc_args *args, + unsigned int infoword) +{ + struct xfs_buf *bp = args->sumbp; + size_t first, last; + + first = (void *)xfs_rsumblock_infoptr(args, infoword) - bp->b_addr; + last = first + sizeof(xfs_suminfo_t) - 1; + + xfs_trans_log_buf(args->tp, bp, first, last); +} + +/* + * Modify the summary information for a given extent size, bitmap block + * combination. + */ +int +xfs_rtmodify_summary( + struct xfs_rtalloc_args *args, + int log, /* log2 of extent size */ + xfs_fileoff_t bbno, /* bitmap block number */ + int delta) /* in/out: summary block number */ +{ + struct xfs_mount *mp = args->mp; + xfs_rtsumoff_t so = xfs_rtsumoffs(mp, log, bbno); + uint8_t *rsum_cache = args->rtg->rtg_rsum_cache; + unsigned int infoword; + xfs_suminfo_t val; + int error; + + error = xfs_rtsummary_read_buf(args, xfs_rtsumoffs_to_block(mp, so)); + if (error) + return error; + + infoword = xfs_rtsumoffs_to_infoword(mp, so); + val = xfs_suminfo_add(args, infoword, delta); + + if (rsum_cache) { + if (val == 0 && log + 1 == rsum_cache[bbno]) + rsum_cache[bbno] = log; + if (val != 0 && log >= rsum_cache[bbno]) + rsum_cache[bbno] = log + 1; + } + + xfs_trans_log_rtsummary(args, infoword); + return 0; +} + +/* + * Read and return the summary information for a given extent size, bitmap block + * combination. + */ +int +xfs_rtget_summary( + struct xfs_rtalloc_args *args, + int log, /* log2 of extent size */ + xfs_fileoff_t bbno, /* bitmap block number */ + xfs_suminfo_t *sum) /* out: summary info for this block */ +{ + struct xfs_mount *mp = args->mp; + xfs_rtsumoff_t so = xfs_rtsumoffs(mp, log, bbno); + int error; + + error = xfs_rtsummary_read_buf(args, xfs_rtsumoffs_to_block(mp, so)); + if (!error) + *sum = xfs_suminfo_get(args, xfs_rtsumoffs_to_infoword(mp, so)); + return error; +} + +/* Log rtbitmap block from the word @from to the byte before @next. */ +static inline void +xfs_trans_log_rtbitmap( + struct xfs_rtalloc_args *args, + unsigned int from, + unsigned int next) +{ + struct xfs_buf *bp = args->rbmbp; + size_t first, last; + + first = (void *)xfs_rbmblock_wordptr(args, from) - bp->b_addr; + last = ((void *)xfs_rbmblock_wordptr(args, next) - 1) - bp->b_addr; + + xfs_trans_log_buf(args->tp, bp, first, last); +} + +/* + * Set the given range of bitmap bits to the given value. + * Do whatever I/O and logging is required. + */ +int +xfs_rtmodify_range( + struct xfs_rtalloc_args *args, + xfs_rtxnum_t start, /* starting rtext to modify */ + xfs_rtxlen_t len, /* length of extent to modify */ + int val) /* 1 for free, 0 for allocated */ +{ + struct xfs_mount *mp = args->mp; + int bit; /* bit number in the word */ + xfs_fileoff_t block; /* bitmap block number */ + int error; + int i; /* current bit number rel. to start */ + int lastbit; /* last useful bit in word */ + xfs_rtword_t mask; /* mask of relevant bits for value */ + xfs_rtword_t incore; + unsigned int firstword; /* first word used in the buffer */ + unsigned int word; /* word number in the buffer */ + + /* + * Compute starting bitmap block number. + */ + block = xfs_rtx_to_rbmblock(mp, start); + /* + * Read the bitmap block, and point to its data. + */ + error = xfs_rtbitmap_read_buf(args, block); + if (error) + return error; + + /* + * Compute the starting word's address, and starting bit. + */ + firstword = word = xfs_rtx_to_rbmword(mp, start); + bit = (int)(start & (XFS_NBWORD - 1)); + /* + * 0 (allocated) => all zeroes; 1 (free) => all ones. + */ + val = -val; + /* + * If not starting on a word boundary, deal with the first + * (partial) word. + */ + if (bit) { + /* + * Compute first bit not changed and mask of relevant bits. + */ + lastbit = min(bit + len, XFS_NBWORD); + mask = (((xfs_rtword_t)1 << (lastbit - bit)) - 1) << bit; + /* + * Set/clear the active bits. + */ + incore = xfs_rtbitmap_getword(args, word); + if (val) + incore |= mask; + else + incore &= ~mask; + xfs_rtbitmap_setword(args, word, incore); + i = lastbit - bit; + /* + * Go on to the next block if that's where the next word is + * and we need the next word. + */ + if (++word == mp->m_blockwsize && i < len) { + /* + * Log the changed part of this block. + * Get the next one. + */ + xfs_trans_log_rtbitmap(args, firstword, word); + error = xfs_rtbitmap_read_buf(args, ++block); + if (error) + return error; + + firstword = word = 0; + } + } else { + /* + * Starting on a word boundary, no partial word. + */ + i = 0; + } + /* + * Loop over whole words in buffers. When we use up one buffer + * we move on to the next one. + */ + while (len - i >= XFS_NBWORD) { + /* + * Set the word value correctly. + */ + xfs_rtbitmap_setword(args, word, val); + i += XFS_NBWORD; + /* + * Go on to the next block if that's where the next word is + * and we need the next word. + */ + if (++word == mp->m_blockwsize && i < len) { + /* + * Log the changed part of this block. + * Get the next one. + */ + xfs_trans_log_rtbitmap(args, firstword, word); + error = xfs_rtbitmap_read_buf(args, ++block); + if (error) + return error; + + firstword = word = 0; + } + } + /* + * If not ending on a word boundary, deal with the last + * (partial) word. + */ + if ((lastbit = len - i)) { + /* + * Compute a mask of relevant bits. + */ + mask = ((xfs_rtword_t)1 << lastbit) - 1; + /* + * Set/clear the active bits. + */ + incore = xfs_rtbitmap_getword(args, word); + if (val) + incore |= mask; + else + incore &= ~mask; + xfs_rtbitmap_setword(args, word, incore); + word++; + } + /* + * Log any remaining changed bytes. + */ + if (word > firstword) + xfs_trans_log_rtbitmap(args, firstword, word); + return 0; +} + +/* + * Mark an extent specified by start and len freed. + * Updates all the summary information as well as the bitmap. + */ +int +xfs_rtfree_range( + struct xfs_rtalloc_args *args, + xfs_rtxnum_t start, /* starting rtext to free */ + xfs_rtxlen_t len) /* in/out: summary block number */ +{ + struct xfs_mount *mp = args->mp; + xfs_rtxnum_t end; /* end of the freed extent */ + int error; /* error value */ + xfs_rtxnum_t postblock; /* first rtext freed > end */ + xfs_rtxnum_t preblock; /* first rtext freed < start */ + + end = start + len - 1; + /* + * Modify the bitmap to mark this extent freed. + */ + error = xfs_rtmodify_range(args, start, len, 1); + if (error) { + return error; + } + /* + * Assume we're freeing out of the middle of an allocated extent. + * We need to find the beginning and end of the extent so we can + * properly update the summary. + */ + error = xfs_rtfind_back(args, start, &preblock); + if (error) { + return error; + } + /* + * Find the next allocated block (end of allocated extent). + */ + error = xfs_rtfind_forw(args, end, args->rtg->rtg_extents - 1, + &postblock); + if (error) + return error; + /* + * If there are blocks not being freed at the front of the + * old extent, add summary data for them to be allocated. + */ + if (preblock < start) { + error = xfs_rtmodify_summary(args, + xfs_highbit64(start - preblock), + xfs_rtx_to_rbmblock(mp, preblock), -1); + if (error) { + return error; + } + } + /* + * If there are blocks not being freed at the end of the + * old extent, add summary data for them to be allocated. + */ + if (postblock > end) { + error = xfs_rtmodify_summary(args, + xfs_highbit64(postblock - end), + xfs_rtx_to_rbmblock(mp, end + 1), -1); + if (error) { + return error; + } + } + /* + * Increment the summary information corresponding to the entire + * (new) free extent. + */ + return xfs_rtmodify_summary(args, + xfs_highbit64(postblock + 1 - preblock), + xfs_rtx_to_rbmblock(mp, preblock), 1); +} + +/* + * Check that the given range is either all allocated (val = 0) or + * all free (val = 1). + */ +int +xfs_rtcheck_range( + struct xfs_rtalloc_args *args, + xfs_rtxnum_t start, /* starting rtext number of extent */ + xfs_rtxlen_t len, /* length of extent */ + int val, /* 1 for free, 0 for allocated */ + xfs_rtxnum_t *new, /* out: first rtext not matching */ + int *stat) /* out: 1 for matches, 0 for not */ +{ + struct xfs_mount *mp = args->mp; + int bit; /* bit number in the word */ + xfs_fileoff_t block; /* bitmap block number */ + int error; + xfs_rtxnum_t i; /* current bit number rel. to start */ + xfs_rtxnum_t lastbit; /* last useful bit in word */ + xfs_rtword_t mask; /* mask of relevant bits for value */ + xfs_rtword_t wdiff; /* difference from wanted value */ + xfs_rtword_t incore; + unsigned int word; /* word number in the buffer */ + + /* + * Compute starting bitmap block number + */ + block = xfs_rtx_to_rbmblock(mp, start); + /* + * Read the bitmap block. + */ + error = xfs_rtbitmap_read_buf(args, block); + if (error) + return error; + + /* + * Compute the starting word's address, and starting bit. + */ + word = xfs_rtx_to_rbmword(mp, start); + bit = (int)(start & (XFS_NBWORD - 1)); + /* + * 0 (allocated) => all zero's; 1 (free) => all one's. + */ + val = -val; + /* + * If not starting on a word boundary, deal with the first + * (partial) word. + */ + if (bit) { + /* + * Compute first bit not examined. + */ + lastbit = min(bit + len, XFS_NBWORD); + /* + * Mask of relevant bits. + */ + mask = (((xfs_rtword_t)1 << (lastbit - bit)) - 1) << bit; + /* + * Compute difference between actual and desired value. + */ + incore = xfs_rtbitmap_getword(args, word); + if ((wdiff = (incore ^ val) & mask)) { + /* + * Different, compute first wrong bit and return. + */ + i = xfs_lowbit32(wdiff) - bit; + *new = start + i; + *stat = 0; + return 0; + } + i = lastbit - bit; + /* + * Go on to next block if that's where the next word is + * and we need the next word. + */ + if (++word == mp->m_blockwsize && i < len) { + /* + * If done with this block, get the next one. + */ + error = xfs_rtbitmap_read_buf(args, ++block); + if (error) + return error; + + word = 0; + } + } else { + /* + * Starting on a word boundary, no partial word. + */ + i = 0; + } + /* + * Loop over whole words in buffers. When we use up one buffer + * we move on to the next one. + */ + while (len - i >= XFS_NBWORD) { + /* + * Compute difference between actual and desired value. + */ + incore = xfs_rtbitmap_getword(args, word); + if ((wdiff = incore ^ val)) { + /* + * Different, compute first wrong bit and return. + */ + i += xfs_lowbit32(wdiff); + *new = start + i; + *stat = 0; + return 0; + } + i += XFS_NBWORD; + /* + * Go on to next block if that's where the next word is + * and we need the next word. + */ + if (++word == mp->m_blockwsize && i < len) { + /* + * If done with this block, get the next one. + */ + error = xfs_rtbitmap_read_buf(args, ++block); + if (error) + return error; + + word = 0; + } + } + /* + * If not ending on a word boundary, deal with the last + * (partial) word. + */ + if ((lastbit = len - i)) { + /* + * Mask of relevant bits. + */ + mask = ((xfs_rtword_t)1 << lastbit) - 1; + /* + * Compute difference between actual and desired value. + */ + incore = xfs_rtbitmap_getword(args, word); + if ((wdiff = (incore ^ val) & mask)) { + /* + * Different, compute first wrong bit and return. + */ + i += xfs_lowbit32(wdiff); + *new = start + i; + *stat = 0; + return 0; + } else + i = len; + } + /* + * Successful, return. + */ + *new = start + i; + *stat = 1; + return 0; +} + +#ifdef DEBUG +/* + * Check that the given extent (block range) is allocated already. + */ +STATIC int +xfs_rtcheck_alloc_range( + struct xfs_rtalloc_args *args, + xfs_rtxnum_t start, /* starting rtext number of extent */ + xfs_rtxlen_t len) /* length of extent */ +{ + xfs_rtxnum_t new; /* dummy for xfs_rtcheck_range */ + int stat; + int error; + + error = xfs_rtcheck_range(args, start, len, 0, &new, &stat); + if (error) + return error; + ASSERT(stat); + return 0; +} +#else +#define xfs_rtcheck_alloc_range(a,b,l) (0) +#endif +/* + * Free an extent in the realtime subvolume. Length is expressed in + * realtime extents, as is the block number. + */ +int +xfs_rtfree_extent( + struct xfs_trans *tp, /* transaction pointer */ + struct xfs_rtgroup *rtg, + xfs_rtxnum_t start, /* starting rtext number to free */ + xfs_rtxlen_t len) /* length of extent freed */ +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_inode *rbmip = rtg_bitmap(rtg); + struct xfs_rtalloc_args args = { + .mp = mp, + .tp = tp, + .rtg = rtg, + }; + int error; + struct timespec64 atime; + + ASSERT(rbmip->i_itemp != NULL); + xfs_assert_ilocked(rbmip, XFS_ILOCK_EXCL); + + if (XFS_TEST_ERROR(mp, XFS_ERRTAG_FREE_EXTENT)) + return -EIO; + + error = xfs_rtcheck_alloc_range(&args, start, len); + if (error) + return error; + + /* + * Free the range of realtime blocks. + */ + error = xfs_rtfree_range(&args, start, len); + if (error) + goto out; + + /* + * Mark more blocks free in the superblock. + */ + xfs_trans_mod_sb(tp, XFS_TRANS_SB_FREXTENTS, (long)len); + + /* + * If we've now freed all the blocks, reset the file sequence + * number to 0 for pre-RTG file systems. + */ + if (!xfs_has_rtgroups(mp) && + tp->t_frextents_delta + mp->m_sb.sb_frextents == + mp->m_sb.sb_rextents) { + if (!(rbmip->i_diflags & XFS_DIFLAG_NEWRTBM)) + rbmip->i_diflags |= XFS_DIFLAG_NEWRTBM; + + atime = inode_get_atime(VFS_I(rbmip)); + atime.tv_sec = 0; + inode_set_atime_to_ts(VFS_I(rbmip), atime); + xfs_trans_log_inode(tp, rbmip, XFS_ILOG_CORE); + } + error = 0; +out: + xfs_rtbuf_cache_relse(&args); + return error; +} + +/* + * Free some blocks in the realtime subvolume. rtbno and rtlen are in units of + * rt blocks, not rt extents; must be aligned to the rt extent size; and rtlen + * cannot exceed XFS_MAX_BMBT_EXTLEN. + */ +int +xfs_rtfree_blocks( + struct xfs_trans *tp, + struct xfs_rtgroup *rtg, + xfs_fsblock_t rtbno, + xfs_filblks_t rtlen) +{ + struct xfs_mount *mp = tp->t_mountp; + xfs_extlen_t mod; + int error; + + ASSERT(!xfs_has_zoned(mp)); + ASSERT(rtlen <= XFS_MAX_BMBT_EXTLEN); + + mod = xfs_blen_to_rtxoff(mp, rtlen); + if (mod) { + ASSERT(mod == 0); + return -EIO; + } + + mod = xfs_rtb_to_rtxoff(mp, rtbno); + if (mod) { + ASSERT(mod == 0); + return -EIO; + } + + error = xfs_rtfree_extent(tp, rtg, xfs_rtb_to_rtx(mp, rtbno), + xfs_extlen_to_rtxlen(mp, rtlen)); + if (error) + return error; + + if (xfs_has_rtgroups(mp)) + xfs_extent_busy_insert(tp, rtg_group(rtg), + xfs_rtb_to_rgbno(mp, rtbno), rtlen, 0); + + return 0; +} + +/* Find all the free records within a given range. */ +int +xfs_rtalloc_query_range( + struct xfs_rtgroup *rtg, + struct xfs_trans *tp, + xfs_rtxnum_t start, + xfs_rtxnum_t end, + xfs_rtalloc_query_range_fn fn, + void *priv) +{ + struct xfs_mount *mp = rtg_mount(rtg); + struct xfs_rtalloc_args args = { + .rtg = rtg, + .mp = mp, + .tp = tp, + }; + int error = 0; + + if (start > end) + return -EINVAL; + if (start == end || start >= rtg->rtg_extents) + return 0; + + end = min(end, rtg->rtg_extents - 1); + + if (xfs_has_zoned(mp)) + return -EINVAL; + + /* Iterate the bitmap, looking for discrepancies. */ + while (start <= end) { + struct xfs_rtalloc_rec rec; + int is_free; + xfs_rtxnum_t rtend; + + /* Is the first block free? */ + error = xfs_rtcheck_range(&args, start, 1, 1, &rtend, + &is_free); + if (error) + break; + + /* How long does the extent go for? */ + error = xfs_rtfind_forw(&args, start, end, &rtend); + if (error) + break; + + if (is_free) { + rec.ar_startext = start; + rec.ar_extcount = rtend - start + 1; + + error = fn(rtg, tp, &rec, priv); + if (error) + break; + } + + start = rtend + 1; + } + + xfs_rtbuf_cache_relse(&args); + return error; +} + +/* Find all the free records. */ +int +xfs_rtalloc_query_all( + struct xfs_rtgroup *rtg, + struct xfs_trans *tp, + xfs_rtalloc_query_range_fn fn, + void *priv) +{ + return xfs_rtalloc_query_range(rtg, tp, 0, rtg->rtg_extents - 1, fn, + priv); +} + +/* Is the given extent all free? */ +int +xfs_rtalloc_extent_is_free( + struct xfs_rtgroup *rtg, + struct xfs_trans *tp, + xfs_rtxnum_t start, + xfs_rtxlen_t len, + bool *is_free) +{ + struct xfs_rtalloc_args args = { + .mp = rtg_mount(rtg), + .rtg = rtg, + .tp = tp, + }; + xfs_rtxnum_t end; + int matches; + int error; + + error = xfs_rtcheck_range(&args, start, len, 1, &end, &matches); + xfs_rtbuf_cache_relse(&args); + if (error) + return error; + + *is_free = matches; + return 0; +} + +/* Compute the number of rt extents tracked by a single bitmap block. */ +xfs_rtxnum_t +xfs_rtbitmap_rtx_per_rbmblock( + struct xfs_mount *mp) +{ + unsigned int rbmblock_bytes = mp->m_sb.sb_blocksize; + + if (xfs_has_rtgroups(mp)) + rbmblock_bytes -= sizeof(struct xfs_rtbuf_blkinfo); + + return rbmblock_bytes * NBBY; +} + +/* + * Compute the number of rtbitmap blocks needed to track the given number of rt + * extents. + */ +xfs_filblks_t +xfs_rtbitmap_blockcount_len( + struct xfs_mount *mp, + xfs_rtbxlen_t rtextents) +{ + if (xfs_has_zoned(mp)) + return 0; + return howmany_64(rtextents, xfs_rtbitmap_rtx_per_rbmblock(mp)); +} + +/* How many rt extents does each rtbitmap file track? */ +static inline xfs_rtbxlen_t +xfs_rtbitmap_bitcount( + struct xfs_mount *mp) +{ + if (!mp->m_sb.sb_rextents) + return 0; + + /* rtgroup size can be nonzero even if rextents is zero */ + if (xfs_has_rtgroups(mp)) + return mp->m_sb.sb_rgextents; + + return mp->m_sb.sb_rextents; +} + +/* + * Compute the number of rtbitmap blocks used for a given file system. + */ +xfs_filblks_t +xfs_rtbitmap_blockcount( + struct xfs_mount *mp) +{ + return xfs_rtbitmap_blockcount_len(mp, xfs_rtbitmap_bitcount(mp)); +} + +/* + * Compute the geometry of the rtsummary file needed to track the given rt + * space. + */ +xfs_filblks_t +xfs_rtsummary_blockcount( + struct xfs_mount *mp, + unsigned int *rsumlevels) +{ + xfs_rtbxlen_t rextents = xfs_rtbitmap_bitcount(mp); + unsigned long long rsumwords; + + if (xfs_has_zoned(mp)) { + *rsumlevels = 0; + return 0; + } + + *rsumlevels = xfs_compute_rextslog(rextents) + 1; + rsumwords = xfs_rtbitmap_blockcount_len(mp, rextents) * (*rsumlevels); + return howmany_64(rsumwords, mp->m_blockwsize); +} + +static int +xfs_rtfile_alloc_blocks( + struct xfs_inode *ip, + xfs_fileoff_t offset_fsb, + xfs_filblks_t count_fsb, + struct xfs_bmbt_irec *map) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_trans *tp; + int nmap = 1; + int error; + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_growrtalloc, + XFS_GROWFSRT_SPACE_RES(mp, count_fsb), 0, 0, &tp); + if (error) + return error; + + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL); + + error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, + XFS_IEXT_ADD_NOSPLIT_CNT); + if (error) + goto out_trans_cancel; + + error = xfs_bmapi_write(tp, ip, offset_fsb, count_fsb, + XFS_BMAPI_METADATA, 0, map, &nmap); + if (error) + goto out_trans_cancel; + + return xfs_trans_commit(tp); + +out_trans_cancel: + xfs_trans_cancel(tp); + return error; +} + +/* Get a buffer for the block. */ +static int +xfs_rtfile_initialize_block( + struct xfs_rtgroup *rtg, + enum xfs_rtg_inodes type, + xfs_fsblock_t fsbno, + void *data) +{ + struct xfs_mount *mp = rtg_mount(rtg); + struct xfs_inode *ip = rtg->rtg_inodes[type]; + struct xfs_trans *tp; + struct xfs_buf *bp; + void *bufdata; + const size_t copylen = mp->m_blockwsize << XFS_WORDLOG; + enum xfs_blft buf_type; + int error; + + if (type == XFS_RTGI_BITMAP) + buf_type = XFS_BLFT_RTBITMAP_BUF; + else if (type == XFS_RTGI_SUMMARY) + buf_type = XFS_BLFT_RTSUMMARY_BUF; + else + return -EINVAL; + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_growrtzero, 0, 0, 0, &tp); + if (error) + return error; + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL); + + error = xfs_trans_get_buf(tp, mp->m_ddev_targp, + XFS_FSB_TO_DADDR(mp, fsbno), mp->m_bsize, 0, &bp); + if (error) { + xfs_trans_cancel(tp); + return error; + } + bufdata = bp->b_addr; + + xfs_trans_buf_set_type(tp, bp, buf_type); + bp->b_ops = xfs_rtblock_ops(mp, type); + + if (xfs_has_rtgroups(mp)) { + struct xfs_rtbuf_blkinfo *hdr = bp->b_addr; + + if (type == XFS_RTGI_BITMAP) + hdr->rt_magic = cpu_to_be32(XFS_RTBITMAP_MAGIC); + else + hdr->rt_magic = cpu_to_be32(XFS_RTSUMMARY_MAGIC); + hdr->rt_owner = cpu_to_be64(I_INO(ip)); + hdr->rt_blkno = cpu_to_be64(XFS_FSB_TO_DADDR(mp, fsbno)); + hdr->rt_lsn = 0; + uuid_copy(&hdr->rt_uuid, &mp->m_sb.sb_meta_uuid); + + bufdata += sizeof(*hdr); + } + + if (data) + memcpy(bufdata, data, copylen); + else + memset(bufdata, 0, copylen); + xfs_trans_log_buf(tp, bp, 0, mp->m_sb.sb_blocksize - 1); + return xfs_trans_commit(tp); +} + +/* + * Allocate space to the bitmap or summary file, and zero it, for growfs. + * @data must be a contiguous buffer large enough to fill all blocks in the + * file; or NULL to initialize the contents to zeroes. + */ +int +xfs_rtfile_initialize_blocks( + struct xfs_rtgroup *rtg, + enum xfs_rtg_inodes type, + xfs_fileoff_t offset_fsb, /* offset to start from */ + xfs_fileoff_t end_fsb, /* offset to allocate to */ + void *data) /* data to fill the blocks */ +{ + struct xfs_mount *mp = rtg_mount(rtg); + const size_t copylen = mp->m_blockwsize << XFS_WORDLOG; + + while (offset_fsb < end_fsb) { + struct xfs_bmbt_irec map; + xfs_filblks_t i; + int error; + + error = xfs_rtfile_alloc_blocks(rtg->rtg_inodes[type], + offset_fsb, end_fsb - offset_fsb, &map); + if (error) + return error; + + /* + * Now we need to clear the allocated blocks. + * + * Do this one block per transaction, to keep it simple. + */ + for (i = 0; i < map.br_blockcount; i++) { + error = xfs_rtfile_initialize_block(rtg, type, + map.br_startblock + i, data); + if (error) + return error; + if (data) + data += copylen; + } + + offset_fsb = map.br_startoff + map.br_blockcount; + } + + return 0; +} + +int +xfs_rtbitmap_create( + struct xfs_rtgroup *rtg, + struct xfs_inode *ip, + struct xfs_trans *tp, + bool init) +{ + struct xfs_mount *mp = rtg_mount(rtg); + + ip->i_disk_size = mp->m_sb.sb_rbmblocks * mp->m_sb.sb_blocksize; + if (init && !xfs_has_rtgroups(mp)) { + ip->i_diflags |= XFS_DIFLAG_NEWRTBM; + inode_set_atime(VFS_I(ip), 0, 0); + } + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + return 0; +} + +int +xfs_rtsummary_create( + struct xfs_rtgroup *rtg, + struct xfs_inode *ip, + struct xfs_trans *tp, + bool init) +{ + struct xfs_mount *mp = rtg_mount(rtg); + + ip->i_disk_size = mp->m_rsumblocks * mp->m_sb.sb_blocksize; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + return 0; +} diff --git a/fs/xfs/libxfs/xfs_rtbitmap.h b/fs/xfs/libxfs/xfs_rtbitmap.h new file mode 100644 index 000000000..22e5d9cd9 --- /dev/null +++ b/fs/xfs/libxfs/xfs_rtbitmap.h @@ -0,0 +1,453 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2003,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_RTBITMAP_H__ +#define __XFS_RTBITMAP_H__ + +#include "xfs_rtgroup.h" + +struct xfs_rtalloc_args { + struct xfs_rtgroup *rtg; + struct xfs_mount *mp; + struct xfs_trans *tp; + + struct xfs_buf *rbmbp; /* bitmap block buffer */ + struct xfs_buf *sumbp; /* summary block buffer */ + + xfs_fileoff_t rbmoff; /* bitmap block number */ + xfs_fileoff_t sumoff; /* summary block number */ +}; + +static inline xfs_rtblock_t +xfs_rtx_to_rtb( + struct xfs_rtgroup *rtg, + xfs_rtxnum_t rtx) +{ + struct xfs_mount *mp = rtg_mount(rtg); + xfs_rtblock_t start = xfs_group_start_fsb(rtg_group(rtg)); + + if (mp->m_rtxblklog >= 0) + return start + (rtx << mp->m_rtxblklog); + return start + (rtx * mp->m_sb.sb_rextsize); +} + +/* Convert an rgbno into an rt extent number. */ +static inline xfs_rtxnum_t +xfs_rgbno_to_rtx( + struct xfs_mount *mp, + xfs_rgblock_t rgbno) +{ + if (likely(mp->m_rtxblklog >= 0)) + return rgbno >> mp->m_rtxblklog; + return rgbno / mp->m_sb.sb_rextsize; +} + +static inline uint64_t +xfs_rtbxlen_to_blen( + struct xfs_mount *mp, + xfs_rtbxlen_t rtbxlen) +{ + if (mp->m_rtxblklog >= 0) + return rtbxlen << mp->m_rtxblklog; + + return rtbxlen * mp->m_sb.sb_rextsize; +} + +static inline xfs_extlen_t +xfs_rtxlen_to_extlen( + struct xfs_mount *mp, + xfs_rtxlen_t rtxlen) +{ + if (mp->m_rtxblklog >= 0) + return rtxlen << mp->m_rtxblklog; + + return rtxlen * mp->m_sb.sb_rextsize; +} + +/* Compute the misalignment between an extent length and a realtime extent .*/ +static inline unsigned int +xfs_extlen_to_rtxmod( + struct xfs_mount *mp, + xfs_extlen_t len) +{ + if (mp->m_rtxblklog >= 0) + return len & mp->m_rtxblkmask; + + return len % mp->m_sb.sb_rextsize; +} + +static inline xfs_rtxlen_t +xfs_extlen_to_rtxlen( + struct xfs_mount *mp, + xfs_extlen_t len) +{ + if (mp->m_rtxblklog >= 0) + return len >> mp->m_rtxblklog; + + return len / mp->m_sb.sb_rextsize; +} + +/* Convert an rt block count into an rt extent count. */ +static inline xfs_rtbxlen_t +xfs_blen_to_rtbxlen( + struct xfs_mount *mp, + uint64_t blen) +{ + if (likely(mp->m_rtxblklog >= 0)) + return blen >> mp->m_rtxblklog; + + return div_u64(blen, mp->m_sb.sb_rextsize); +} + +/* Return the offset of a file block length within an rt extent. */ +static inline xfs_extlen_t +xfs_blen_to_rtxoff( + struct xfs_mount *mp, + xfs_filblks_t blen) +{ + if (likely(mp->m_rtxblklog >= 0)) + return blen & mp->m_rtxblkmask; + + return do_div(blen, mp->m_sb.sb_rextsize); +} + +/* Round this block count up to the nearest rt extent size. */ +static inline xfs_filblks_t +xfs_blen_roundup_rtx( + struct xfs_mount *mp, + xfs_filblks_t blen) +{ + return roundup_64(blen, mp->m_sb.sb_rextsize); +} + +/* Convert an rt block number into an rt extent number. */ +static inline xfs_rtxnum_t +xfs_rtb_to_rtx( + struct xfs_mount *mp, + xfs_rtblock_t rtbno) +{ + /* open-coded 64-bit masking operation */ + rtbno &= mp->m_groups[XG_TYPE_RTG].blkmask; + if (likely(mp->m_rtxblklog >= 0)) + return rtbno >> mp->m_rtxblklog; + return div_u64(rtbno, mp->m_sb.sb_rextsize); +} + +/* Return the offset of a rtgroup block number within an rt extent. */ +static inline xfs_extlen_t +xfs_rgbno_to_rtxoff( + struct xfs_mount *mp, + xfs_rgblock_t rgbno) +{ + return rgbno % mp->m_sb.sb_rextsize; +} + +/* Return the offset of an rt block number within an rt extent. */ +static inline xfs_extlen_t +xfs_rtb_to_rtxoff( + struct xfs_mount *mp, + xfs_rtblock_t rtbno) +{ + /* open-coded 64-bit masking operation */ + rtbno &= mp->m_groups[XG_TYPE_RTG].blkmask; + if (likely(mp->m_rtxblklog >= 0)) + return rtbno & mp->m_rtxblkmask; + return do_div(rtbno, mp->m_sb.sb_rextsize); +} + +/* Round this file block offset up to the nearest rt extent size. */ +static inline xfs_rtblock_t +xfs_fileoff_roundup_rtx( + struct xfs_mount *mp, + xfs_fileoff_t off) +{ + return roundup_64(off, mp->m_sb.sb_rextsize); +} + +/* Round this file block offset down to the nearest rt extent size. */ +static inline xfs_rtblock_t +xfs_fileoff_rounddown_rtx( + struct xfs_mount *mp, + xfs_fileoff_t off) +{ + return rounddown_64(off, mp->m_sb.sb_rextsize); +} + +/* Convert an rt extent number to a file block offset in the rt bitmap file. */ +static inline xfs_fileoff_t +xfs_rtx_to_rbmblock( + struct xfs_mount *mp, + xfs_rtxnum_t rtx) +{ + if (xfs_has_rtgroups(mp)) + return div_u64(rtx, mp->m_rtx_per_rbmblock); + + return rtx >> mp->m_blkbit_log; +} + +/* Convert an rt extent number to a word offset within an rt bitmap block. */ +static inline unsigned int +xfs_rtx_to_rbmword( + struct xfs_mount *mp, + xfs_rtxnum_t rtx) +{ + if (xfs_has_rtgroups(mp)) { + unsigned int mod; + + div_u64_rem(rtx >> XFS_NBWORDLOG, mp->m_blockwsize, &mod); + return mod; + } + + return (rtx >> XFS_NBWORDLOG) & (mp->m_blockwsize - 1); +} + +/* Convert a file block offset in the rt bitmap file to an rt extent number. */ +static inline xfs_rtxnum_t +xfs_rbmblock_to_rtx( + struct xfs_mount *mp, + xfs_fileoff_t rbmoff) +{ + if (xfs_has_rtgroups(mp)) + return rbmoff * mp->m_rtx_per_rbmblock; + + return rbmoff << mp->m_blkbit_log; +} + +/* Return a pointer to a bitmap word within a rt bitmap block. */ +static inline union xfs_rtword_raw * +xfs_rbmblock_wordptr( + struct xfs_rtalloc_args *args, + unsigned int index) +{ + struct xfs_mount *mp = args->mp; + union xfs_rtword_raw *words; + struct xfs_rtbuf_blkinfo *hdr = args->rbmbp->b_addr; + + if (xfs_has_rtgroups(mp)) + words = (union xfs_rtword_raw *)(hdr + 1); + else + words = args->rbmbp->b_addr; + + return words + index; +} + +/* Convert an ondisk bitmap word to its incore representation. */ +static inline xfs_rtword_t +xfs_rtbitmap_getword( + struct xfs_rtalloc_args *args, + unsigned int index) +{ + union xfs_rtword_raw *word = xfs_rbmblock_wordptr(args, index); + + if (xfs_has_rtgroups(args->mp)) + return be32_to_cpu(word->rtg); + return word->old; +} + +/* Set an ondisk bitmap word from an incore representation. */ +static inline void +xfs_rtbitmap_setword( + struct xfs_rtalloc_args *args, + unsigned int index, + xfs_rtword_t value) +{ + union xfs_rtword_raw *word = xfs_rbmblock_wordptr(args, index); + + if (xfs_has_rtgroups(args->mp)) + word->rtg = cpu_to_be32(value); + else + word->old = value; +} + +/* + * Convert a rt extent length and rt bitmap block number to a xfs_suminfo_t + * offset within the rt summary file. + */ +static inline xfs_rtsumoff_t +xfs_rtsumoffs( + struct xfs_mount *mp, + int log2_len, + xfs_fileoff_t rbmoff) +{ + return log2_len * mp->m_sb.sb_rbmblocks + rbmoff; +} + +/* + * Convert an xfs_suminfo_t offset to a file block offset within the rt summary + * file. + */ +static inline xfs_fileoff_t +xfs_rtsumoffs_to_block( + struct xfs_mount *mp, + xfs_rtsumoff_t rsumoff) +{ + if (xfs_has_rtgroups(mp)) + return rsumoff / mp->m_blockwsize; + + return XFS_B_TO_FSBT(mp, rsumoff * sizeof(xfs_suminfo_t)); +} + +/* + * Convert an xfs_suminfo_t offset to an info word offset within an rt summary + * block. + */ +static inline unsigned int +xfs_rtsumoffs_to_infoword( + struct xfs_mount *mp, + xfs_rtsumoff_t rsumoff) +{ + unsigned int mask = mp->m_blockmask >> XFS_SUMINFOLOG; + + if (xfs_has_rtgroups(mp)) + return rsumoff % mp->m_blockwsize; + + return rsumoff & mask; +} + +/* Return a pointer to a summary info word within a rt summary block. */ +static inline union xfs_suminfo_raw * +xfs_rsumblock_infoptr( + struct xfs_rtalloc_args *args, + unsigned int index) +{ + union xfs_suminfo_raw *info; + struct xfs_rtbuf_blkinfo *hdr = args->sumbp->b_addr; + + if (xfs_has_rtgroups(args->mp)) + info = (union xfs_suminfo_raw *)(hdr + 1); + else + info = args->sumbp->b_addr; + + return info + index; +} + +/* Get the current value of a summary counter. */ +static inline xfs_suminfo_t +xfs_suminfo_get( + struct xfs_rtalloc_args *args, + unsigned int index) +{ + union xfs_suminfo_raw *info = xfs_rsumblock_infoptr(args, index); + + if (xfs_has_rtgroups(args->mp)) + return be32_to_cpu(info->rtg); + return info->old; +} + +/* Add to the current value of a summary counter and return the new value. */ +static inline xfs_suminfo_t +xfs_suminfo_add( + struct xfs_rtalloc_args *args, + unsigned int index, + int delta) +{ + union xfs_suminfo_raw *info = xfs_rsumblock_infoptr(args, index); + + if (xfs_has_rtgroups(args->mp)) { + be32_add_cpu(&info->rtg, delta); + return be32_to_cpu(info->rtg); + } + + info->old += delta; + return info->old; +} + +static inline const struct xfs_buf_ops * +xfs_rtblock_ops( + struct xfs_mount *mp, + enum xfs_rtg_inodes type) +{ + if (xfs_has_rtgroups(mp)) { + if (type == XFS_RTGI_SUMMARY) + return &xfs_rtsummary_buf_ops; + return &xfs_rtbitmap_buf_ops; + } + return &xfs_rtbuf_ops; +} + +/* + * Functions for walking free space rtextents in the realtime bitmap. + */ +struct xfs_rtalloc_rec { + xfs_rtxnum_t ar_startext; + xfs_rtbxlen_t ar_extcount; +}; + +typedef int (*xfs_rtalloc_query_range_fn)( + struct xfs_rtgroup *rtg, + struct xfs_trans *tp, + const struct xfs_rtalloc_rec *rec, + void *priv); + +#ifdef CONFIG_XFS_RT +void xfs_rtbuf_cache_relse(struct xfs_rtalloc_args *args); +int xfs_rtbitmap_read_buf(struct xfs_rtalloc_args *args, xfs_fileoff_t block); +int xfs_rtsummary_read_buf(struct xfs_rtalloc_args *args, xfs_fileoff_t block); +int xfs_rtcheck_range(struct xfs_rtalloc_args *args, xfs_rtxnum_t start, + xfs_rtxlen_t len, int val, xfs_rtxnum_t *new, int *stat); +int xfs_rtfind_back(struct xfs_rtalloc_args *args, xfs_rtxnum_t start, + xfs_rtxnum_t *rtblock); +int xfs_rtfind_forw(struct xfs_rtalloc_args *args, xfs_rtxnum_t start, + xfs_rtxnum_t limit, xfs_rtxnum_t *rtblock); +int xfs_rtmodify_range(struct xfs_rtalloc_args *args, xfs_rtxnum_t start, + xfs_rtxlen_t len, int val); +int xfs_rtget_summary(struct xfs_rtalloc_args *args, int log, + xfs_fileoff_t bbno, xfs_suminfo_t *sum); +int xfs_rtmodify_summary(struct xfs_rtalloc_args *args, int log, + xfs_fileoff_t bbno, int delta); +int xfs_rtfree_range(struct xfs_rtalloc_args *args, xfs_rtxnum_t start, + xfs_rtxlen_t len); +int xfs_rtalloc_query_range(struct xfs_rtgroup *rtg, struct xfs_trans *tp, + xfs_rtxnum_t start, xfs_rtxnum_t end, + xfs_rtalloc_query_range_fn fn, void *priv); +int xfs_rtalloc_query_all(struct xfs_rtgroup *rtg, struct xfs_trans *tp, + xfs_rtalloc_query_range_fn fn, void *priv); +int xfs_rtalloc_extent_is_free(struct xfs_rtgroup *rtg, struct xfs_trans *tp, + xfs_rtxnum_t start, xfs_rtxlen_t len, bool *is_free); +int xfs_rtfree_extent(struct xfs_trans *tp, struct xfs_rtgroup *rtg, + xfs_rtxnum_t start, xfs_rtxlen_t len); +/* Same as above, but in units of rt blocks. */ +int xfs_rtfree_blocks(struct xfs_trans *tp, struct xfs_rtgroup *rtg, + xfs_fsblock_t rtbno, xfs_filblks_t rtlen); + +xfs_rtxnum_t xfs_rtbitmap_rtx_per_rbmblock(struct xfs_mount *mp); +xfs_filblks_t xfs_rtbitmap_blockcount(struct xfs_mount *mp); +xfs_filblks_t xfs_rtbitmap_blockcount_len(struct xfs_mount *mp, + xfs_rtbxlen_t rtextents); +xfs_filblks_t xfs_rtsummary_blockcount(struct xfs_mount *mp, + unsigned int *rsumlevels); + +int xfs_rtfile_initialize_blocks(struct xfs_rtgroup *rtg, + enum xfs_rtg_inodes type, xfs_fileoff_t offset_fsb, + xfs_fileoff_t end_fsb, void *data); +int xfs_rtbitmap_create(struct xfs_rtgroup *rtg, struct xfs_inode *ip, + struct xfs_trans *tp, bool init); +int xfs_rtsummary_create(struct xfs_rtgroup *rtg, struct xfs_inode *ip, + struct xfs_trans *tp, bool init); + +#else /* CONFIG_XFS_RT */ +# define xfs_rtfree_extent(t,b,l) (-ENOSYS) + +static inline int xfs_rtfree_blocks(struct xfs_trans *tp, + struct xfs_rtgroup *rtg, xfs_fsblock_t rtbno, + xfs_filblks_t rtlen) +{ + return -ENOSYS; +} +# define xfs_rtalloc_query_range(m,t,l,h,f,p) (-ENOSYS) +# define xfs_rtalloc_query_all(m,t,f,p) (-ENOSYS) +# define xfs_rtbitmap_read_buf(a,b) (-ENOSYS) +# define xfs_rtsummary_read_buf(a,b) (-ENOSYS) +# define xfs_rtbuf_cache_relse(a) (0) +# define xfs_rtalloc_extent_is_free(m,t,s,l,i) (-ENOSYS) +static inline xfs_filblks_t +xfs_rtbitmap_blockcount_len(struct xfs_mount *mp, xfs_rtbxlen_t rtextents) +{ + /* shut up gcc */ + return 0; +} +#endif /* CONFIG_XFS_RT */ + +#endif /* __XFS_RTBITMAP_H__ */ diff --git a/fs/xfs/libxfs/xfs_rtgroup.c b/fs/xfs/libxfs/xfs_rtgroup.c new file mode 100644 index 000000000..fe7222bbe --- /dev/null +++ b/fs/xfs/libxfs/xfs_rtgroup.c @@ -0,0 +1,745 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2022-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_sb.h" +#include "xfs_mount.h" +#include "xfs_btree.h" +#include "xfs_alloc_btree.h" +#include "xfs_rmap_btree.h" +#include "xfs_alloc.h" +#include "xfs_ialloc.h" +#include "xfs_rmap.h" +#include "xfs_ag.h" +#include "xfs_ag_resv.h" +#include "xfs_health.h" +#include "xfs_error.h" +#include "xfs_bmap.h" +#include "xfs_defer.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_trace.h" +#include "xfs_inode.h" +#include "xfs_icache.h" +#include "xfs_buf_item.h" +#include "xfs_rtgroup.h" +#include "xfs_rtbitmap.h" +#include "xfs_metafile.h" +#include "xfs_metadir.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_rtrefcount_btree.h" + +/* Find the first usable fsblock in this rtgroup. */ +static inline uint32_t +xfs_rtgroup_min_block( + struct xfs_mount *mp, + xfs_rgnumber_t rgno) +{ + if (xfs_has_rtsb(mp) && rgno == 0) + return mp->m_sb.sb_rextsize; + + return 0; +} + +/* Compute the number of rt extents in this realtime group. */ +static xfs_rtxnum_t +__xfs_rtgroup_extents( + struct xfs_mount *mp, + xfs_rgnumber_t rgno, + xfs_rgnumber_t rgcount, + xfs_rtbxlen_t rextents) +{ + ASSERT(rgno < rgcount); + if (rgno == rgcount - 1) + return rextents - ((xfs_rtxnum_t)rgno * mp->m_sb.sb_rgextents); + + ASSERT(xfs_has_rtgroups(mp)); + return mp->m_sb.sb_rgextents; +} + +xfs_rtxnum_t +xfs_rtgroup_extents( + struct xfs_mount *mp, + xfs_rgnumber_t rgno) +{ + return __xfs_rtgroup_extents(mp, rgno, mp->m_sb.sb_rgcount, + mp->m_sb.sb_rextents); +} + +/* Precompute this group's geometry */ +void +xfs_rtgroup_calc_geometry( + struct xfs_mount *mp, + struct xfs_rtgroup *rtg, + xfs_rgnumber_t rgno, + xfs_rgnumber_t rgcount, + xfs_rtbxlen_t rextents) +{ + rtg->rtg_extents = __xfs_rtgroup_extents(mp, rgno, rgcount, rextents); + rtg_group(rtg)->xg_block_count = + rtg->rtg_extents * mp->m_sb.sb_rextsize; + rtg_group(rtg)->xg_min_gbno = xfs_rtgroup_min_block(mp, rgno); +} + +int +xfs_rtgroup_alloc( + struct xfs_mount *mp, + xfs_rgnumber_t rgno, + xfs_rgnumber_t rgcount, + xfs_rtbxlen_t rextents) +{ + struct xfs_rtgroup *rtg; + int error; + + rtg = kzalloc_obj(struct xfs_rtgroup); + if (!rtg) + return -ENOMEM; + + xfs_rtgroup_calc_geometry(mp, rtg, rgno, rgcount, rextents); + + error = xfs_group_insert(mp, rtg_group(rtg), rgno, XG_TYPE_RTG); + if (error) + goto out_free_rtg; + return 0; + +out_free_rtg: + kfree(rtg); + return error; +} + +void +xfs_rtgroup_free( + struct xfs_mount *mp, + xfs_rgnumber_t rgno) +{ + xfs_group_free(mp, rgno, XG_TYPE_RTG, NULL); +} + +/* Free a range of incore rtgroup objects. */ +void +xfs_free_rtgroups( + struct xfs_mount *mp, + xfs_rgnumber_t first_rgno, + xfs_rgnumber_t end_rgno) +{ + xfs_rgnumber_t rgno; + + for (rgno = first_rgno; rgno < end_rgno; rgno++) + xfs_rtgroup_free(mp, rgno); +} + +/* Initialize some range of incore rtgroup objects. */ +int +xfs_initialize_rtgroups( + struct xfs_mount *mp, + xfs_rgnumber_t first_rgno, + xfs_rgnumber_t end_rgno, + xfs_rtbxlen_t rextents) +{ + xfs_rgnumber_t index; + int error; + + if (first_rgno >= end_rgno) + return 0; + + for (index = first_rgno; index < end_rgno; index++) { + error = xfs_rtgroup_alloc(mp, index, end_rgno, rextents); + if (error) + goto out_unwind_new_rtgs; + } + + return 0; + +out_unwind_new_rtgs: + xfs_free_rtgroups(mp, first_rgno, index); + return error; +} + +/* + * Update the rt extent count of the previous tail rtgroup if it changed during + * recovery (i.e. recovery of a growfs). + */ +int +xfs_update_last_rtgroup_size( + struct xfs_mount *mp, + xfs_rgnumber_t prev_rgcount) +{ + struct xfs_rtgroup *rtg; + + ASSERT(prev_rgcount > 0); + + rtg = xfs_rtgroup_grab(mp, prev_rgcount - 1); + if (!rtg) + return -EFSCORRUPTED; + rtg->rtg_extents = __xfs_rtgroup_extents(mp, prev_rgcount - 1, + mp->m_sb.sb_rgcount, mp->m_sb.sb_rextents); + rtg_group(rtg)->xg_block_count = rtg->rtg_extents * mp->m_sb.sb_rextsize; + xfs_rtgroup_rele(rtg); + return 0; +} + +/* Lock metadata inodes associated with this rt group. */ +void +xfs_rtgroup_lock( + struct xfs_rtgroup *rtg, + unsigned int rtglock_flags) +{ + ASSERT(!(rtglock_flags & ~XFS_RTGLOCK_ALL_FLAGS)); + ASSERT(!(rtglock_flags & XFS_RTGLOCK_BITMAP_SHARED) || + !(rtglock_flags & XFS_RTGLOCK_BITMAP)); + + if (!xfs_has_zoned(rtg_mount(rtg))) { + if (rtglock_flags & XFS_RTGLOCK_BITMAP) { + /* + * Lock both realtime free space metadata inodes for a + * freespace update. + */ + xfs_ilock(rtg_bitmap(rtg), XFS_ILOCK_EXCL); + xfs_ilock(rtg_summary(rtg), XFS_ILOCK_EXCL); + } else if (rtglock_flags & XFS_RTGLOCK_BITMAP_SHARED) { + xfs_ilock(rtg_bitmap(rtg), XFS_ILOCK_SHARED); + } + } + + if ((rtglock_flags & XFS_RTGLOCK_RMAP) && rtg_rmap(rtg)) + xfs_ilock(rtg_rmap(rtg), XFS_ILOCK_EXCL); + + if ((rtglock_flags & XFS_RTGLOCK_REFCOUNT) && rtg_refcount(rtg)) + xfs_ilock(rtg_refcount(rtg), XFS_ILOCK_EXCL); +} + +/* Unlock metadata inodes associated with this rt group. */ +void +xfs_rtgroup_unlock( + struct xfs_rtgroup *rtg, + unsigned int rtglock_flags) +{ + ASSERT(!(rtglock_flags & ~XFS_RTGLOCK_ALL_FLAGS)); + ASSERT(!(rtglock_flags & XFS_RTGLOCK_BITMAP_SHARED) || + !(rtglock_flags & XFS_RTGLOCK_BITMAP)); + + if ((rtglock_flags & XFS_RTGLOCK_REFCOUNT) && rtg_refcount(rtg)) + xfs_iunlock(rtg_refcount(rtg), XFS_ILOCK_EXCL); + + if ((rtglock_flags & XFS_RTGLOCK_RMAP) && rtg_rmap(rtg)) + xfs_iunlock(rtg_rmap(rtg), XFS_ILOCK_EXCL); + + if (!xfs_has_zoned(rtg_mount(rtg))) { + if (rtglock_flags & XFS_RTGLOCK_BITMAP) { + xfs_iunlock(rtg_summary(rtg), XFS_ILOCK_EXCL); + xfs_iunlock(rtg_bitmap(rtg), XFS_ILOCK_EXCL); + } else if (rtglock_flags & XFS_RTGLOCK_BITMAP_SHARED) { + xfs_iunlock(rtg_bitmap(rtg), XFS_ILOCK_SHARED); + } + } +} + +/* + * Join realtime group metadata inodes to the transaction. The ILOCKs will be + * released on transaction commit. + */ +void +xfs_rtgroup_trans_join( + struct xfs_trans *tp, + struct xfs_rtgroup *rtg, + unsigned int rtglock_flags) +{ + ASSERT(!(rtglock_flags & ~XFS_RTGLOCK_ALL_FLAGS)); + ASSERT(!(rtglock_flags & XFS_RTGLOCK_BITMAP_SHARED)); + + if (!xfs_has_zoned(rtg_mount(rtg)) && + (rtglock_flags & XFS_RTGLOCK_BITMAP)) { + xfs_trans_ijoin(tp, rtg_bitmap(rtg), XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, rtg_summary(rtg), XFS_ILOCK_EXCL); + } + + if ((rtglock_flags & XFS_RTGLOCK_RMAP) && rtg_rmap(rtg)) + xfs_trans_ijoin(tp, rtg_rmap(rtg), XFS_ILOCK_EXCL); + + if ((rtglock_flags & XFS_RTGLOCK_REFCOUNT) && rtg_refcount(rtg)) + xfs_trans_ijoin(tp, rtg_refcount(rtg), XFS_ILOCK_EXCL); +} + +/* Retrieve rt group geometry. */ +int +xfs_rtgroup_get_geometry( + struct xfs_rtgroup *rtg, + struct xfs_rtgroup_geometry *rgeo) +{ + /* Fill out form. */ + memset(rgeo, 0, sizeof(*rgeo)); + rgeo->rg_number = rtg_rgno(rtg); + rgeo->rg_length = rtg_blocks(rtg); + xfs_rtgroup_geom_health(rtg, rgeo); + return 0; +} + +#ifdef CONFIG_PROVE_LOCKING +static struct lock_class_key xfs_rtginode_lock_class; + +static int +xfs_rtginode_ilock_cmp_fn( + const struct lockdep_map *m1, + const struct lockdep_map *m2) +{ + const struct xfs_inode *ip1 = + container_of(m1, struct xfs_inode, i_lock.dep_map); + const struct xfs_inode *ip2 = + container_of(m2, struct xfs_inode, i_lock.dep_map); + + if (ip1->i_projid < ip2->i_projid) + return -1; + if (ip1->i_projid > ip2->i_projid) + return 1; + return 0; +} + +static inline void +xfs_rtginode_ilock_print_fn( + const struct lockdep_map *m) +{ + const struct xfs_inode *ip = + container_of(m, struct xfs_inode, i_lock.dep_map); + + printk(KERN_CONT " rgno=%u metatype=%s", ip->i_projid, + xfs_metafile_type_str(ip->i_metatype)); +} + +/* + * Most of the time each of the RTG inode locks are only taken one at a time. + * But when committing deferred ops, more than one of a kind can be taken. + * However, deferred rt ops will be committed in rgno order so there is no + * potential for deadlocks. The code here is needed to tell lockdep about this + * order. + */ +static inline void +xfs_rtginode_lockdep_setup( + struct xfs_inode *ip, + xfs_rgnumber_t rgno, + enum xfs_rtg_inodes type) +{ + lockdep_set_class_and_subclass(&ip->i_lock, &xfs_rtginode_lock_class, + type); + lock_set_cmp_fn(&ip->i_lock, xfs_rtginode_ilock_cmp_fn, + xfs_rtginode_ilock_print_fn); +} +#else +#define xfs_rtginode_lockdep_setup(ip, rgno, type) do { } while (0) +#endif /* CONFIG_PROVE_LOCKING */ + +struct xfs_rtginode_ops { + const char *name; /* short name */ + + enum xfs_metafile_type metafile_type; + + unsigned int sick; /* rtgroup sickness flag */ + + unsigned int fmt_mask; /* all valid data fork formats */ + + /* Does the fs have this feature? */ + bool (*enabled)(const struct xfs_mount *mp); + + /* Create this rtgroup metadata inode and initialize it. */ + int (*create)(struct xfs_rtgroup *rtg, + struct xfs_inode *ip, + struct xfs_trans *tp, + bool init); +}; + +static const struct xfs_rtginode_ops xfs_rtginode_ops[XFS_RTGI_MAX] = { + [XFS_RTGI_BITMAP] = { + .name = "bitmap", + .metafile_type = XFS_METAFILE_RTBITMAP, + .sick = XFS_SICK_RG_BITMAP, + .fmt_mask = (1U << XFS_DINODE_FMT_EXTENTS) | + (1U << XFS_DINODE_FMT_BTREE), + .enabled = xfs_has_nonzoned, + .create = xfs_rtbitmap_create, + }, + [XFS_RTGI_SUMMARY] = { + .name = "summary", + .metafile_type = XFS_METAFILE_RTSUMMARY, + .sick = XFS_SICK_RG_SUMMARY, + .fmt_mask = (1U << XFS_DINODE_FMT_EXTENTS) | + (1U << XFS_DINODE_FMT_BTREE), + .enabled = xfs_has_nonzoned, + .create = xfs_rtsummary_create, + }, + [XFS_RTGI_RMAP] = { + .name = "rmap", + .metafile_type = XFS_METAFILE_RTRMAP, + .sick = XFS_SICK_RG_RMAPBT, + .fmt_mask = 1U << XFS_DINODE_FMT_META_BTREE, + /* + * growfs must create the rtrmap inodes before adding a + * realtime volume to the filesystem, so we cannot use the + * rtrmapbt predicate here. + */ + .enabled = xfs_has_rmapbt, + .create = xfs_rtrmapbt_create, + }, + [XFS_RTGI_REFCOUNT] = { + .name = "refcount", + .metafile_type = XFS_METAFILE_RTREFCOUNT, + .sick = XFS_SICK_RG_REFCNTBT, + .fmt_mask = 1U << XFS_DINODE_FMT_META_BTREE, + /* same comment about growfs and rmap inodes applies here */ + .enabled = xfs_has_reflink, + .create = xfs_rtrefcountbt_create, + }, +}; + +/* Return the shortname of this rtgroup inode. */ +const char * +xfs_rtginode_name( + enum xfs_rtg_inodes type) +{ + return xfs_rtginode_ops[type].name; +} + +/* Return the metafile type of this rtgroup inode. */ +enum xfs_metafile_type +xfs_rtginode_metafile_type( + enum xfs_rtg_inodes type) +{ + return xfs_rtginode_ops[type].metafile_type; +} + +/* Should this rtgroup inode be present? */ +bool +xfs_rtginode_enabled( + struct xfs_rtgroup *rtg, + enum xfs_rtg_inodes type) +{ + const struct xfs_rtginode_ops *ops = &xfs_rtginode_ops[type]; + + if (!ops->enabled) + return true; + return ops->enabled(rtg_mount(rtg)); +} + +/* Mark an rtgroup inode sick */ +void +xfs_rtginode_mark_sick( + struct xfs_rtgroup *rtg, + enum xfs_rtg_inodes type) +{ + const struct xfs_rtginode_ops *ops = &xfs_rtginode_ops[type]; + + xfs_group_mark_sick(rtg_group(rtg), ops->sick); +} + +/* Load and existing rtgroup inode into the rtgroup structure. */ +int +xfs_rtginode_load( + struct xfs_rtgroup *rtg, + enum xfs_rtg_inodes type, + struct xfs_trans *tp) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_inode *ip; + const struct xfs_rtginode_ops *ops = &xfs_rtginode_ops[type]; + int error; + + if (!xfs_rtginode_enabled(rtg, type)) + return 0; + + if (!xfs_has_rtgroups(mp)) { + xfs_ino_t ino; + + switch (type) { + case XFS_RTGI_BITMAP: + ino = mp->m_sb.sb_rbmino; + break; + case XFS_RTGI_SUMMARY: + ino = mp->m_sb.sb_rsumino; + break; + default: + /* None of the other types exist on !rtgroups */ + return 0; + } + + error = xfs_trans_metafile_iget(tp, ino, ops->metafile_type, + &ip); + } else { + const char *path; + + if (!mp->m_rtdirip) { + xfs_fs_mark_sick(mp, XFS_SICK_FS_METADIR); + return -EFSCORRUPTED; + } + + path = xfs_rtginode_path(rtg_rgno(rtg), type); + if (!path) + return -ENOMEM; + error = xfs_metadir_load(tp, mp->m_rtdirip, path, + ops->metafile_type, &ip); + kfree(path); + } + + if (error) { + if (xfs_metadata_is_sick(error)) + xfs_rtginode_mark_sick(rtg, type); + return error; + } + + if (XFS_IS_CORRUPT(mp, !((1U << ip->i_df.if_format) & ops->fmt_mask))) { + xfs_irele(ip); + xfs_rtginode_mark_sick(rtg, type); + return -EFSCORRUPTED; + } + + if (XFS_IS_CORRUPT(mp, ip->i_projid != rtg_rgno(rtg))) { + xfs_irele(ip); + xfs_rtginode_mark_sick(rtg, type); + return -EFSCORRUPTED; + } + + xfs_rtginode_lockdep_setup(ip, rtg_rgno(rtg), type); + rtg->rtg_inodes[type] = ip; + return 0; +} + +/* Release an rtgroup metadata inode. */ +void +xfs_rtginode_irele( + struct xfs_inode **ipp) +{ + if (*ipp) + xfs_irele(*ipp); + *ipp = NULL; +} + +struct xfs_rtginode_create { + struct xfs_rtgroup *rtg; + enum xfs_rtg_inodes type; + bool init; +}; + +static int +xfs_rtginode_init( + struct xfs_metadir_update *upd, + void *priv) +{ + struct xfs_rtginode_create *rc = priv; + const struct xfs_rtginode_ops *ops = &xfs_rtginode_ops[rc->type]; + + xfs_rtginode_lockdep_setup(upd->ip, rtg_rgno(rc->rtg), rc->type); + upd->ip->i_projid = rtg_rgno(rc->rtg); + return ops->create(rc->rtg, upd->ip, upd->tp, rc->init); +} + +/* Add a metadata inode for a realtime rmap btree. */ +int +xfs_rtginode_create( + struct xfs_rtgroup *rtg, + enum xfs_rtg_inodes type, + bool init) +{ + const struct xfs_rtginode_ops *ops = &xfs_rtginode_ops[type]; + struct xfs_mount *mp = rtg_mount(rtg); + struct xfs_rtginode_create rc = { + .rtg = rtg, + .type = type, + .init = init, + }; + struct xfs_metadir_update upd = { + .dp = mp->m_rtdirip, + .metafile_type = ops->metafile_type, + }; + int error; + + if (!xfs_rtginode_enabled(rtg, type)) + return 0; + + if (!mp->m_rtdirip) { + xfs_fs_mark_sick(mp, XFS_SICK_FS_METADIR); + return -EFSCORRUPTED; + } + + upd.path = xfs_rtginode_path(rtg_rgno(rtg), type); + if (!upd.path) + return -ENOMEM; + + error = xfs_metadir_create_file(&upd, S_IFREG, xfs_rtginode_init, &rc, + &rtg->rtg_inodes[type]); + kfree(upd.path); + return error; +} + +/* Create the parent directory for all rtgroup inodes and load it. */ +int +xfs_rtginode_mkdir_parent( + struct xfs_mount *mp) +{ + if (!mp->m_metadirip) { + xfs_fs_mark_sick(mp, XFS_SICK_FS_METADIR); + return -EFSCORRUPTED; + } + + return xfs_metadir_mkdir(mp->m_metadirip, "rtgroups", &mp->m_rtdirip); +} + +/* Load the parent directory of all rtgroup inodes. */ +int +xfs_rtginode_load_parent( + struct xfs_trans *tp) +{ + struct xfs_mount *mp = tp->t_mountp; + + if (!mp->m_metadirip) { + xfs_fs_mark_sick(mp, XFS_SICK_FS_METADIR); + return -EFSCORRUPTED; + } + + return xfs_metadir_load(tp, mp->m_metadirip, "rtgroups", + XFS_METAFILE_DIR, &mp->m_rtdirip); +} + +/* Check superblock fields for a read or a write. */ +static xfs_failaddr_t +xfs_rtsb_verify_common( + struct xfs_buf *bp) +{ + struct xfs_rtsb *rsb = bp->b_addr; + + if (!xfs_verify_magic(bp, rsb->rsb_magicnum)) + return __this_address; + if (rsb->rsb_pad) + return __this_address; + + /* Everything to the end of the fs block must be zero */ + if (memchr_inv(rsb + 1, 0, BBTOB(bp->b_length) - sizeof(*rsb))) + return __this_address; + + return NULL; +} + +/* Check superblock fields for a read or revalidation. */ +static inline xfs_failaddr_t +xfs_rtsb_verify_all( + struct xfs_buf *bp) +{ + struct xfs_rtsb *rsb = bp->b_addr; + struct xfs_mount *mp = bp->b_mount; + xfs_failaddr_t fa; + + fa = xfs_rtsb_verify_common(bp); + if (fa) + return fa; + + if (memcmp(&rsb->rsb_fname, &mp->m_sb.sb_fname, XFSLABEL_MAX)) + return __this_address; + if (!uuid_equal(&rsb->rsb_uuid, &mp->m_sb.sb_uuid)) + return __this_address; + if (!uuid_equal(&rsb->rsb_meta_uuid, &mp->m_sb.sb_meta_uuid)) + return __this_address; + + return NULL; +} + +static void +xfs_rtsb_read_verify( + struct xfs_buf *bp) +{ + xfs_failaddr_t fa; + + if (!xfs_buf_verify_cksum(bp, XFS_RTSB_CRC_OFF)) { + xfs_verifier_error(bp, -EFSBADCRC, __this_address); + return; + } + + fa = xfs_rtsb_verify_all(bp); + if (fa) + xfs_verifier_error(bp, -EFSCORRUPTED, fa); +} + +static void +xfs_rtsb_write_verify( + struct xfs_buf *bp) +{ + xfs_failaddr_t fa; + + fa = xfs_rtsb_verify_common(bp); + if (fa) { + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + return; + } + + xfs_buf_update_cksum(bp, XFS_RTSB_CRC_OFF); +} + +const struct xfs_buf_ops xfs_rtsb_buf_ops = { + .name = "xfs_rtsb", + .magic = { 0, cpu_to_be32(XFS_RTSB_MAGIC) }, + .verify_read = xfs_rtsb_read_verify, + .verify_write = xfs_rtsb_write_verify, + .verify_struct = xfs_rtsb_verify_all, +}; + +/* Update a realtime superblock from the primary fs super */ +void +xfs_update_rtsb( + struct xfs_buf *rtsb_bp, + const struct xfs_buf *sb_bp) +{ + const struct xfs_dsb *dsb = sb_bp->b_addr; + struct xfs_rtsb *rsb = rtsb_bp->b_addr; + const uuid_t *meta_uuid; + + rsb->rsb_magicnum = cpu_to_be32(XFS_RTSB_MAGIC); + + rsb->rsb_pad = 0; + memcpy(&rsb->rsb_fname, &dsb->sb_fname, XFSLABEL_MAX); + + memcpy(&rsb->rsb_uuid, &dsb->sb_uuid, sizeof(rsb->rsb_uuid)); + + /* + * The metadata uuid is the fs uuid if the metauuid feature is not + * enabled. + */ + if (dsb->sb_features_incompat & + cpu_to_be32(XFS_SB_FEAT_INCOMPAT_META_UUID)) + meta_uuid = &dsb->sb_meta_uuid; + else + meta_uuid = &dsb->sb_uuid; + memcpy(&rsb->rsb_meta_uuid, meta_uuid, sizeof(rsb->rsb_meta_uuid)); +} + +/* + * Update the realtime superblock from a filesystem superblock and log it to + * the given transaction. + */ +struct xfs_buf * +xfs_log_rtsb( + struct xfs_trans *tp, + const struct xfs_buf *sb_bp) +{ + struct xfs_buf *rtsb_bp; + + if (!xfs_has_rtsb(tp->t_mountp)) + return NULL; + + rtsb_bp = xfs_trans_getrtsb(tp); + if (!rtsb_bp) { + /* + * It's possible for the rtgroups feature to be enabled but + * there is no incore rt superblock buffer if the rt geometry + * was specified at mkfs time but the rt section has not yet + * been attached. In this case, rblocks must be zero. + */ + ASSERT(tp->t_mountp->m_sb.sb_rblocks == 0); + return NULL; + } + + xfs_update_rtsb(rtsb_bp, sb_bp); + xfs_trans_ordered_buf(tp, rtsb_bp); + return rtsb_bp; +} diff --git a/fs/xfs/libxfs/xfs_rtgroup.h b/fs/xfs/libxfs/xfs_rtgroup.h new file mode 100644 index 000000000..c0b9f9f2c --- /dev/null +++ b/fs/xfs/libxfs/xfs_rtgroup.h @@ -0,0 +1,389 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2022-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __LIBXFS_RTGROUP_H +#define __LIBXFS_RTGROUP_H 1 + +#include "xfs_group.h" + +struct xfs_mount; +struct xfs_trans; + +enum xfs_rtg_inodes { + XFS_RTGI_BITMAP, /* allocation bitmap */ + XFS_RTGI_SUMMARY, /* allocation summary */ + XFS_RTGI_RMAP, /* rmap btree inode */ + XFS_RTGI_REFCOUNT, /* refcount btree inode */ + + XFS_RTGI_MAX, +}; + +#ifdef MAX_LOCKDEP_SUBCLASSES +static_assert(XFS_RTGI_MAX <= MAX_LOCKDEP_SUBCLASSES); +#endif + +/* + * Realtime group incore structure, similar to the per-AG structure. + */ +struct xfs_rtgroup { + struct xfs_group rtg_group; + + /* per-rtgroup metadata inodes */ + struct xfs_inode *rtg_inodes[XFS_RTGI_MAX]; + + /* Number of blocks in this group */ + xfs_rtxnum_t rtg_extents; + + /* + * For bitmap based RT devices this points to a cache of rt summary + * level per bitmap block with the invariant that rtg_rsum_cache[bbno] + * > the maximum i for which rsum[i][bbno] != 0, or 0 if + * rsum[i][bbno] == 0 for all i. + * Reads and writes are serialized by the rsumip inode lock. + * + * For zoned RT devices this points to the open zone structure for + * a group that is open for writers, or is NULL. + */ + union { + uint8_t *rtg_rsum_cache; + struct xfs_open_zone *rtg_open_zone; + }; + + /* + * Count of outstanding GC operations for zoned XFS. Any RTG with a + * non-zero rtg_gccount will not be picked as new GC victim. + */ + atomic_t rtg_gccount; +}; + +/* + * For zoned RT devices this is set on groups that have no written blocks + * and can be picked by the allocator for opening. + */ +#define XFS_RTG_FREE XA_MARK_0 + +static inline struct xfs_rtgroup *to_rtg(struct xfs_group *xg) +{ + return container_of(xg, struct xfs_rtgroup, rtg_group); +} + +static inline struct xfs_group *rtg_group(struct xfs_rtgroup *rtg) +{ + return &rtg->rtg_group; +} + +static inline struct xfs_mount *rtg_mount(const struct xfs_rtgroup *rtg) +{ + return rtg->rtg_group.xg_mount; +} + +static inline xfs_rgnumber_t rtg_rgno(const struct xfs_rtgroup *rtg) +{ + return rtg->rtg_group.xg_gno; +} + +static inline xfs_rgblock_t rtg_blocks(const struct xfs_rtgroup *rtg) +{ + return rtg->rtg_group.xg_block_count; +} + +static inline struct xfs_inode *rtg_bitmap(const struct xfs_rtgroup *rtg) +{ + return rtg->rtg_inodes[XFS_RTGI_BITMAP]; +} + +static inline struct xfs_inode *rtg_summary(const struct xfs_rtgroup *rtg) +{ + return rtg->rtg_inodes[XFS_RTGI_SUMMARY]; +} + +static inline struct xfs_inode *rtg_rmap(const struct xfs_rtgroup *rtg) +{ + return rtg->rtg_inodes[XFS_RTGI_RMAP]; +} + +static inline struct xfs_inode *rtg_refcount(const struct xfs_rtgroup *rtg) +{ + return rtg->rtg_inodes[XFS_RTGI_REFCOUNT]; +} + +/* Passive rtgroup references */ +static inline struct xfs_rtgroup * +xfs_rtgroup_get( + struct xfs_mount *mp, + xfs_rgnumber_t rgno) +{ + return to_rtg(xfs_group_get(mp, rgno, XG_TYPE_RTG)); +} + +static inline struct xfs_rtgroup * +xfs_rtgroup_hold( + struct xfs_rtgroup *rtg) +{ + return to_rtg(xfs_group_hold(rtg_group(rtg))); +} + +static inline void +xfs_rtgroup_put( + struct xfs_rtgroup *rtg) +{ + xfs_group_put(rtg_group(rtg)); +} + +/* Active rtgroup references */ +static inline struct xfs_rtgroup * +xfs_rtgroup_grab( + struct xfs_mount *mp, + xfs_rgnumber_t rgno) +{ + return to_rtg(xfs_group_grab(mp, rgno, XG_TYPE_RTG)); +} + +static inline void +xfs_rtgroup_rele( + struct xfs_rtgroup *rtg) +{ + xfs_group_rele(rtg_group(rtg)); +} + +static inline struct xfs_rtgroup * +xfs_rtgroup_next_range( + struct xfs_mount *mp, + struct xfs_rtgroup *rtg, + xfs_rgnumber_t start_rgno, + xfs_rgnumber_t end_rgno) +{ + return to_rtg(xfs_group_next_range(mp, rtg ? rtg_group(rtg) : NULL, + start_rgno, end_rgno, XG_TYPE_RTG)); +} + +static inline struct xfs_rtgroup * +xfs_rtgroup_next( + struct xfs_mount *mp, + struct xfs_rtgroup *rtg) +{ + return xfs_rtgroup_next_range(mp, rtg, 0, mp->m_sb.sb_rgcount - 1); +} + +static inline bool +xfs_verify_rgbno( + struct xfs_rtgroup *rtg, + xfs_rgblock_t rgbno) +{ + ASSERT(xfs_has_rtgroups(rtg_mount(rtg))); + + return xfs_verify_gbno(rtg_group(rtg), rgbno); +} + +/* + * Check that [@rgbno,@len] is a valid extent range in @rtg. + * + * Must only be used for RTG-enabled file systems. + */ +static inline bool +xfs_verify_rgbext( + struct xfs_rtgroup *rtg, + xfs_rgblock_t rgbno, + xfs_extlen_t len) +{ + ASSERT(xfs_has_rtgroups(rtg_mount(rtg))); + + return xfs_verify_gbext(rtg_group(rtg), rgbno, len); +} + +static inline xfs_rtblock_t +xfs_rgbno_to_rtb( + struct xfs_rtgroup *rtg, + xfs_rgblock_t rgbno) +{ + return xfs_gbno_to_fsb(rtg_group(rtg), rgbno); +} + +static inline xfs_rgnumber_t +xfs_rtb_to_rgno( + struct xfs_mount *mp, + xfs_rtblock_t rtbno) +{ + return xfs_fsb_to_gno(mp, rtbno, XG_TYPE_RTG); +} + +static inline xfs_rgblock_t +xfs_rtb_to_rgbno( + struct xfs_mount *mp, + xfs_rtblock_t rtbno) +{ + return xfs_fsb_to_gbno(mp, rtbno, XG_TYPE_RTG); +} + +/* Is rtbno the start of a RT group? */ +static inline bool +xfs_rtbno_is_group_start( + struct xfs_mount *mp, + xfs_rtblock_t rtbno) +{ + return (rtbno & mp->m_groups[XG_TYPE_RTG].blkmask) == 0; +} + +/* Convert an rtgroups rt extent number into an rgbno. */ +static inline xfs_rgblock_t +xfs_rtx_to_rgbno( + struct xfs_rtgroup *rtg, + xfs_rtxnum_t rtx) +{ + struct xfs_mount *mp = rtg_mount(rtg); + + if (likely(mp->m_rtxblklog >= 0)) + return rtx << mp->m_rtxblklog; + return rtx * mp->m_sb.sb_rextsize; +} + +static inline xfs_daddr_t +xfs_rtb_to_daddr( + struct xfs_mount *mp, + xfs_rtblock_t rtbno) +{ + struct xfs_groups *g = &mp->m_groups[XG_TYPE_RTG]; + + if (xfs_has_rtgroups(mp) && !g->has_daddr_gaps) { + xfs_rgnumber_t rgno = xfs_rtb_to_rgno(mp, rtbno); + + rtbno = (xfs_rtblock_t)rgno * g->blocks + (rtbno & g->blkmask); + } + + return XFS_FSB_TO_BB(mp, g->start_fsb + rtbno); +} + +static inline xfs_rtblock_t +xfs_daddr_to_rtb( + struct xfs_mount *mp, + xfs_daddr_t daddr) +{ + struct xfs_groups *g = &mp->m_groups[XG_TYPE_RTG]; + xfs_rfsblock_t bno; + + bno = XFS_BB_TO_FSBT(mp, daddr) - g->start_fsb; + if (xfs_has_rtgroups(mp) && !g->has_daddr_gaps) { + xfs_rgnumber_t rgno; + uint32_t rgbno; + + rgno = div_u64_rem(bno, g->blocks, &rgbno); + return ((xfs_rtblock_t)rgno << g->blklog) + rgbno; + } + + return bno; +} + +#ifdef CONFIG_XFS_RT +int xfs_rtgroup_alloc(struct xfs_mount *mp, xfs_rgnumber_t rgno, + xfs_rgnumber_t rgcount, xfs_rtbxlen_t rextents); +void xfs_rtgroup_free(struct xfs_mount *mp, xfs_rgnumber_t rgno); + +void xfs_free_rtgroups(struct xfs_mount *mp, xfs_rgnumber_t first_rgno, + xfs_rgnumber_t end_rgno); +int xfs_initialize_rtgroups(struct xfs_mount *mp, xfs_rgnumber_t first_rgno, + xfs_rgnumber_t end_rgno, xfs_rtbxlen_t rextents); + +xfs_rtxnum_t xfs_rtgroup_extents(struct xfs_mount *mp, xfs_rgnumber_t rgno); +void xfs_rtgroup_calc_geometry(struct xfs_mount *mp, struct xfs_rtgroup *rtg, + xfs_rgnumber_t rgno, xfs_rgnumber_t rgcount, + xfs_rtbxlen_t rextents); + +int xfs_update_last_rtgroup_size(struct xfs_mount *mp, + xfs_rgnumber_t prev_rgcount); + +/* Lock the rt bitmap inode in exclusive mode */ +#define XFS_RTGLOCK_BITMAP (1U << 0) +/* Lock the rt bitmap inode in shared mode */ +#define XFS_RTGLOCK_BITMAP_SHARED (1U << 1) +/* Lock the rt rmap inode in exclusive mode */ +#define XFS_RTGLOCK_RMAP (1U << 2) +/* Lock the rt refcount inode in exclusive mode */ +#define XFS_RTGLOCK_REFCOUNT (1U << 3) + +#define XFS_RTGLOCK_ALL_FLAGS (XFS_RTGLOCK_BITMAP | \ + XFS_RTGLOCK_BITMAP_SHARED | \ + XFS_RTGLOCK_RMAP | \ + XFS_RTGLOCK_REFCOUNT) + +void xfs_rtgroup_lock(struct xfs_rtgroup *rtg, unsigned int rtglock_flags); +void xfs_rtgroup_unlock(struct xfs_rtgroup *rtg, unsigned int rtglock_flags); +void xfs_rtgroup_trans_join(struct xfs_trans *tp, struct xfs_rtgroup *rtg, + unsigned int rtglock_flags); + +int xfs_rtgroup_get_geometry(struct xfs_rtgroup *rtg, + struct xfs_rtgroup_geometry *rgeo); + +int xfs_rtginode_mkdir_parent(struct xfs_mount *mp); +int xfs_rtginode_load_parent(struct xfs_trans *tp); + +const char *xfs_rtginode_name(enum xfs_rtg_inodes type); +enum xfs_metafile_type xfs_rtginode_metafile_type(enum xfs_rtg_inodes type); +bool xfs_rtginode_enabled(struct xfs_rtgroup *rtg, enum xfs_rtg_inodes type); +void xfs_rtginode_mark_sick(struct xfs_rtgroup *rtg, enum xfs_rtg_inodes type); +int xfs_rtginode_load(struct xfs_rtgroup *rtg, enum xfs_rtg_inodes type, + struct xfs_trans *tp); +int xfs_rtginode_create(struct xfs_rtgroup *rtg, enum xfs_rtg_inodes type, + bool init); +void xfs_rtginode_irele(struct xfs_inode **ipp); + +void xfs_rtginode_irele(struct xfs_inode **ipp); + +static inline const char *xfs_rtginode_path(xfs_rgnumber_t rgno, + enum xfs_rtg_inodes type) +{ + return kasprintf(GFP_KERNEL, "%u.%s", rgno, xfs_rtginode_name(type)); +} + +void xfs_update_rtsb(struct xfs_buf *rtsb_bp, + const struct xfs_buf *sb_bp); +struct xfs_buf *xfs_log_rtsb(struct xfs_trans *tp, + const struct xfs_buf *sb_bp); +#else +static inline void xfs_free_rtgroups(struct xfs_mount *mp, + xfs_rgnumber_t first_rgno, xfs_rgnumber_t end_rgno) +{ +} + +static inline int xfs_initialize_rtgroups(struct xfs_mount *mp, + xfs_rgnumber_t first_rgno, xfs_rgnumber_t end_rgno, + xfs_rtbxlen_t rextents) +{ + return 0; +} + +# define xfs_rtgroup_extents(mp, rgno) (0) +# define xfs_update_last_rtgroup_size(mp, rgno) (0) +# define xfs_rtgroup_lock(rtg, gf) ((void)0) +# define xfs_rtgroup_unlock(rtg, gf) ((void)0) +# define xfs_rtgroup_trans_join(tp, rtg, gf) ((void)0) +# define xfs_update_rtsb(bp, sb_bp) ((void)0) +# define xfs_log_rtsb(tp, sb_bp) (NULL) +# define xfs_rtgroup_get_geometry(rtg, rgeo) (-EOPNOTSUPP) +#endif /* CONFIG_XFS_RT */ + +static inline xfs_rfsblock_t +xfs_rtgs_to_rfsbs( + struct xfs_mount *mp, + uint32_t nr_groups) +{ + return xfs_groups_to_rfsbs(mp, nr_groups, XG_TYPE_RTG); +} + +/* + * Return the "raw" size of a group on the hardware device. This includes the + * daddr gaps present for XFS_SB_FEAT_INCOMPAT_ZONE_GAPS file systems. + */ +static inline xfs_rgblock_t +xfs_rtgroup_raw_size( + struct xfs_mount *mp) +{ + struct xfs_groups *g = &mp->m_groups[XG_TYPE_RTG]; + + if (g->has_daddr_gaps) + return 1U << g->blklog; + return g->blocks; +} + +#endif /* __LIBXFS_RTGROUP_H */ diff --git a/fs/xfs/libxfs/xfs_rtrefcount_btree.c b/fs/xfs/libxfs/xfs_rtrefcount_btree.c new file mode 100644 index 000000000..22acc1411 --- /dev/null +++ b/fs/xfs/libxfs/xfs_rtrefcount_btree.c @@ -0,0 +1,757 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2021-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_sb.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_alloc.h" +#include "xfs_btree.h" +#include "xfs_btree_staging.h" +#include "xfs_rtrefcount_btree.h" +#include "xfs_refcount.h" +#include "xfs_trace.h" +#include "xfs_cksum.h" +#include "xfs_error.h" +#include "xfs_extent_busy.h" +#include "xfs_rtgroup.h" +#include "xfs_rtbitmap.h" +#include "xfs_metafile.h" +#include "xfs_health.h" + +static struct kmem_cache *xfs_rtrefcountbt_cur_cache; + +/* + * Realtime Reference Count btree. + * + * This is a btree used to track the owner(s) of a given extent in the realtime + * device. See the comments in xfs_refcount_btree.c for more information. + * + * This tree is basically the same as the regular refcount btree except that + * it's rooted in an inode. + */ + +static struct xfs_btree_cur * +xfs_rtrefcountbt_dup_cursor( + struct xfs_btree_cur *cur) +{ + return xfs_rtrefcountbt_init_cursor(cur->bc_tp, to_rtg(cur->bc_group)); +} + +STATIC int +xfs_rtrefcountbt_get_minrecs( + struct xfs_btree_cur *cur, + int level) +{ + if (level == cur->bc_nlevels - 1) { + struct xfs_ifork *ifp = xfs_btree_ifork_ptr(cur); + + return xfs_rtrefcountbt_maxrecs(cur->bc_mp, ifp->if_broot_bytes, + level == 0) / 2; + } + + return cur->bc_mp->m_rtrefc_mnr[level != 0]; +} + +STATIC int +xfs_rtrefcountbt_get_maxrecs( + struct xfs_btree_cur *cur, + int level) +{ + if (level == cur->bc_nlevels - 1) { + struct xfs_ifork *ifp = xfs_btree_ifork_ptr(cur); + + return xfs_rtrefcountbt_maxrecs(cur->bc_mp, ifp->if_broot_bytes, + level == 0); + } + + return cur->bc_mp->m_rtrefc_mxr[level != 0]; +} + +/* + * Calculate number of records in a realtime refcount btree inode root. + */ +unsigned int +xfs_rtrefcountbt_droot_maxrecs( + unsigned int blocklen, + bool leaf) +{ + blocklen -= sizeof(struct xfs_rtrefcount_root); + + if (leaf) + return blocklen / sizeof(struct xfs_refcount_rec); + return blocklen / (2 * sizeof(struct xfs_refcount_key) + + sizeof(xfs_rtrefcount_ptr_t)); +} + +/* + * Get the maximum records we could store in the on-disk format. + * + * For non-root nodes this is equivalent to xfs_rtrefcountbt_get_maxrecs, but + * for the root node this checks the available space in the dinode fork so that + * we can resize the in-memory buffer to match it. After a resize to the + * maximum size this function returns the same value as + * xfs_rtrefcountbt_get_maxrecs for the root node, too. + */ +STATIC int +xfs_rtrefcountbt_get_dmaxrecs( + struct xfs_btree_cur *cur, + int level) +{ + if (level != cur->bc_nlevels - 1) + return cur->bc_mp->m_rtrefc_mxr[level != 0]; + return xfs_rtrefcountbt_droot_maxrecs(cur->bc_ino.forksize, level == 0); +} + +STATIC void +xfs_rtrefcountbt_init_key_from_rec( + union xfs_btree_key *key, + const union xfs_btree_rec *rec) +{ + key->refc.rc_startblock = rec->refc.rc_startblock; +} + +STATIC void +xfs_rtrefcountbt_init_high_key_from_rec( + union xfs_btree_key *key, + const union xfs_btree_rec *rec) +{ + __u32 x; + + x = be32_to_cpu(rec->refc.rc_startblock); + x += be32_to_cpu(rec->refc.rc_blockcount) - 1; + key->refc.rc_startblock = cpu_to_be32(x); +} + +STATIC void +xfs_rtrefcountbt_init_rec_from_cur( + struct xfs_btree_cur *cur, + union xfs_btree_rec *rec) +{ + const struct xfs_refcount_irec *irec = &cur->bc_rec.rc; + uint32_t start; + + start = xfs_refcount_encode_startblock(irec->rc_startblock, + irec->rc_domain); + rec->refc.rc_startblock = cpu_to_be32(start); + rec->refc.rc_blockcount = cpu_to_be32(cur->bc_rec.rc.rc_blockcount); + rec->refc.rc_refcount = cpu_to_be32(cur->bc_rec.rc.rc_refcount); +} + +STATIC void +xfs_rtrefcountbt_init_ptr_from_cur( + struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr) +{ + ptr->l = 0; +} + +STATIC int +xfs_rtrefcountbt_cmp_key_with_cur( + struct xfs_btree_cur *cur, + const union xfs_btree_key *key) +{ + const struct xfs_refcount_key *kp = &key->refc; + const struct xfs_refcount_irec *irec = &cur->bc_rec.rc; + uint32_t start; + + start = xfs_refcount_encode_startblock(irec->rc_startblock, + irec->rc_domain); + return cmp_int(be32_to_cpu(kp->rc_startblock), start); +} + +STATIC int +xfs_rtrefcountbt_cmp_two_keys( + struct xfs_btree_cur *cur, + const union xfs_btree_key *k1, + const union xfs_btree_key *k2, + const union xfs_btree_key *mask) +{ + ASSERT(!mask || mask->refc.rc_startblock); + + return cmp_int(be32_to_cpu(k1->refc.rc_startblock), + be32_to_cpu(k2->refc.rc_startblock)); +} + +static xfs_failaddr_t +xfs_rtrefcountbt_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_target->bt_mount; + struct xfs_btree_block *block = XFS_BUF_TO_BLOCK(bp); + xfs_failaddr_t fa; + int level; + + if (!xfs_verify_magic(bp, block->bb_magic)) + return __this_address; + + if (!xfs_has_reflink(mp)) + return __this_address; + fa = xfs_btree_fsblock_v5hdr_verify(bp, XFS_RMAP_OWN_UNKNOWN); + if (fa) + return fa; + level = be16_to_cpu(block->bb_level); + if (level >= mp->m_rtrefc_maxlevels) + return __this_address; + + return xfs_btree_fsblock_verify(bp, mp->m_rtrefc_mxr[level != 0]); +} + +static void +xfs_rtrefcountbt_read_verify( + struct xfs_buf *bp) +{ + xfs_failaddr_t fa; + + if (!xfs_btree_fsblock_verify_crc(bp)) + xfs_verifier_error(bp, -EFSBADCRC, __this_address); + else { + fa = xfs_rtrefcountbt_verify(bp); + if (fa) + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + } + + if (bp->b_error) + trace_xfs_btree_corrupt(bp, _RET_IP_); +} + +static void +xfs_rtrefcountbt_write_verify( + struct xfs_buf *bp) +{ + xfs_failaddr_t fa; + + fa = xfs_rtrefcountbt_verify(bp); + if (fa) { + trace_xfs_btree_corrupt(bp, _RET_IP_); + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + return; + } + xfs_btree_fsblock_calc_crc(bp); + +} + +const struct xfs_buf_ops xfs_rtrefcountbt_buf_ops = { + .name = "xfs_rtrefcountbt", + .magic = { 0, cpu_to_be32(XFS_RTREFC_CRC_MAGIC) }, + .verify_read = xfs_rtrefcountbt_read_verify, + .verify_write = xfs_rtrefcountbt_write_verify, + .verify_struct = xfs_rtrefcountbt_verify, +}; + +STATIC int +xfs_rtrefcountbt_keys_inorder( + struct xfs_btree_cur *cur, + const union xfs_btree_key *k1, + const union xfs_btree_key *k2) +{ + return be32_to_cpu(k1->refc.rc_startblock) < + be32_to_cpu(k2->refc.rc_startblock); +} + +STATIC int +xfs_rtrefcountbt_recs_inorder( + struct xfs_btree_cur *cur, + const union xfs_btree_rec *r1, + const union xfs_btree_rec *r2) +{ + return be32_to_cpu(r1->refc.rc_startblock) + + be32_to_cpu(r1->refc.rc_blockcount) <= + be32_to_cpu(r2->refc.rc_startblock); +} + +STATIC enum xbtree_key_contig +xfs_rtrefcountbt_keys_contiguous( + struct xfs_btree_cur *cur, + const union xfs_btree_key *key1, + const union xfs_btree_key *key2, + const union xfs_btree_key *mask) +{ + ASSERT(!mask || mask->refc.rc_startblock); + + return xbtree_key_contig(be32_to_cpu(key1->refc.rc_startblock), + be32_to_cpu(key2->refc.rc_startblock)); +} + +static inline void +xfs_rtrefcountbt_move_ptrs( + struct xfs_mount *mp, + struct xfs_btree_block *broot, + short old_size, + size_t new_size, + unsigned int numrecs) +{ + void *dptr; + void *sptr; + + sptr = xfs_rtrefcount_broot_ptr_addr(mp, broot, 1, old_size); + dptr = xfs_rtrefcount_broot_ptr_addr(mp, broot, 1, new_size); + memmove(dptr, sptr, numrecs * sizeof(xfs_rtrefcount_ptr_t)); +} + +static struct xfs_btree_block * +xfs_rtrefcountbt_broot_realloc( + struct xfs_btree_cur *cur, + unsigned int new_numrecs) +{ + struct xfs_mount *mp = cur->bc_mp; + struct xfs_ifork *ifp = xfs_btree_ifork_ptr(cur); + struct xfs_btree_block *broot; + unsigned int new_size; + unsigned int old_size = ifp->if_broot_bytes; + const unsigned int level = cur->bc_nlevels - 1; + + new_size = xfs_rtrefcount_broot_space_calc(mp, level, new_numrecs); + + /* Handle the nop case quietly. */ + if (new_size == old_size) + return ifp->if_broot; + + if (new_size > old_size) { + unsigned int old_numrecs; + + /* + * If there wasn't any memory allocated before, just allocate + * it now and get out. + */ + if (old_size == 0) + return xfs_broot_realloc(ifp, new_size); + + /* + * If there is already an existing if_broot, then we need to + * realloc it and possibly move the node block pointers because + * those are not butted up against the btree block header. + */ + old_numrecs = xfs_rtrefcountbt_maxrecs(mp, old_size, level); + broot = xfs_broot_realloc(ifp, new_size); + if (level > 0) + xfs_rtrefcountbt_move_ptrs(mp, broot, old_size, + new_size, old_numrecs); + goto out_broot; + } + + /* + * We're reducing numrecs. If we're going all the way to zero, just + * free the block. + */ + ASSERT(ifp->if_broot != NULL && old_size > 0); + if (new_size == 0) + return xfs_broot_realloc(ifp, 0); + + /* + * Shrink the btree root by possibly moving the rtrmapbt pointers, + * since they are not butted up against the btree block header. Then + * reallocate broot. + */ + if (level > 0) + xfs_rtrefcountbt_move_ptrs(mp, ifp->if_broot, old_size, + new_size, new_numrecs); + broot = xfs_broot_realloc(ifp, new_size); + +out_broot: + ASSERT(xfs_rtrefcount_droot_space(broot) <= + xfs_inode_fork_size(cur->bc_ino.ip, cur->bc_ino.whichfork)); + return broot; +} + +const struct xfs_btree_ops xfs_rtrefcountbt_ops = { + .name = "rtrefcount", + .type = XFS_BTREE_TYPE_INODE, + .geom_flags = XFS_BTGEO_IROOT_RECORDS, + + .rec_len = sizeof(struct xfs_refcount_rec), + .key_len = sizeof(struct xfs_refcount_key), + .ptr_len = XFS_BTREE_LONG_PTR_LEN, + + .lru_refs = XFS_REFC_BTREE_REF, + .statoff = XFS_STATS_CALC_INDEX(xs_rtrefcbt_2), + .sick_mask = XFS_SICK_RG_REFCNTBT, + + .dup_cursor = xfs_rtrefcountbt_dup_cursor, + .alloc_block = xfs_btree_alloc_metafile_block, + .free_block = xfs_btree_free_metafile_block, + .get_minrecs = xfs_rtrefcountbt_get_minrecs, + .get_maxrecs = xfs_rtrefcountbt_get_maxrecs, + .get_dmaxrecs = xfs_rtrefcountbt_get_dmaxrecs, + .init_key_from_rec = xfs_rtrefcountbt_init_key_from_rec, + .init_high_key_from_rec = xfs_rtrefcountbt_init_high_key_from_rec, + .init_rec_from_cur = xfs_rtrefcountbt_init_rec_from_cur, + .init_ptr_from_cur = xfs_rtrefcountbt_init_ptr_from_cur, + .cmp_key_with_cur = xfs_rtrefcountbt_cmp_key_with_cur, + .buf_ops = &xfs_rtrefcountbt_buf_ops, + .cmp_two_keys = xfs_rtrefcountbt_cmp_two_keys, + .keys_inorder = xfs_rtrefcountbt_keys_inorder, + .recs_inorder = xfs_rtrefcountbt_recs_inorder, + .keys_contiguous = xfs_rtrefcountbt_keys_contiguous, + .broot_realloc = xfs_rtrefcountbt_broot_realloc, +}; + +/* Allocate a new rt refcount btree cursor. */ +struct xfs_btree_cur * +xfs_rtrefcountbt_init_cursor( + struct xfs_trans *tp, + struct xfs_rtgroup *rtg) +{ + struct xfs_inode *ip = rtg_refcount(rtg); + struct xfs_mount *mp = rtg_mount(rtg); + struct xfs_btree_cur *cur; + + xfs_assert_ilocked(ip, XFS_ILOCK_SHARED | XFS_ILOCK_EXCL); + + cur = xfs_btree_alloc_cursor(mp, tp, &xfs_rtrefcountbt_ops, + mp->m_rtrefc_maxlevels, xfs_rtrefcountbt_cur_cache); + + cur->bc_ino.ip = ip; + cur->bc_refc.nr_ops = 0; + cur->bc_refc.shape_changes = 0; + cur->bc_group = xfs_group_hold(rtg_group(rtg)); + cur->bc_nlevels = be16_to_cpu(ip->i_df.if_broot->bb_level) + 1; + cur->bc_ino.forksize = xfs_inode_fork_size(ip, XFS_DATA_FORK); + cur->bc_ino.whichfork = XFS_DATA_FORK; + return cur; +} + +/* + * Install a new rt reverse mapping btree root. Caller is responsible for + * invalidating and freeing the old btree blocks. + */ +void +xfs_rtrefcountbt_commit_staged_btree( + struct xfs_btree_cur *cur, + struct xfs_trans *tp) +{ + struct xbtree_ifakeroot *ifake = cur->bc_ino.ifake; + struct xfs_ifork *ifp; + int flags = XFS_ILOG_CORE | XFS_ILOG_DBROOT; + + ASSERT(cur->bc_flags & XFS_BTREE_STAGING); + ASSERT(ifake->if_fork->if_format == XFS_DINODE_FMT_META_BTREE); + + /* + * Free any resources hanging off the real fork, then shallow-copy the + * staging fork's contents into the real fork to transfer everything + * we just built. + */ + ifp = xfs_ifork_ptr(cur->bc_ino.ip, XFS_DATA_FORK); + xfs_idestroy_fork(ifp); + memcpy(ifp, ifake->if_fork, sizeof(struct xfs_ifork)); + + cur->bc_ino.ip->i_projid = cur->bc_group->xg_gno; + xfs_trans_log_inode(tp, cur->bc_ino.ip, flags); + xfs_btree_commit_ifakeroot(cur, tp, XFS_DATA_FORK); +} + +/* Calculate number of records in a realtime refcount btree block. */ +static inline unsigned int +xfs_rtrefcountbt_block_maxrecs( + unsigned int blocklen, + bool leaf) +{ + + if (leaf) + return blocklen / sizeof(struct xfs_refcount_rec); + return blocklen / (sizeof(struct xfs_refcount_key) + + sizeof(xfs_rtrefcount_ptr_t)); +} + +/* + * Calculate number of records in an refcount btree block. + */ +unsigned int +xfs_rtrefcountbt_maxrecs( + struct xfs_mount *mp, + unsigned int blocklen, + bool leaf) +{ + blocklen -= XFS_RTREFCOUNT_BLOCK_LEN; + return xfs_rtrefcountbt_block_maxrecs(blocklen, leaf); +} + +/* Compute the max possible height for realtime refcount btrees. */ +unsigned int +xfs_rtrefcountbt_maxlevels_ondisk(void) +{ + unsigned int minrecs[2]; + unsigned int blocklen; + + blocklen = XFS_MIN_CRC_BLOCKSIZE - XFS_BTREE_LBLOCK_CRC_LEN; + + minrecs[0] = xfs_rtrefcountbt_block_maxrecs(blocklen, true) / 2; + minrecs[1] = xfs_rtrefcountbt_block_maxrecs(blocklen, false) / 2; + + /* We need at most one record for every block in an rt group. */ + return xfs_btree_compute_maxlevels(minrecs, XFS_MAX_RGBLOCKS); +} + +int __init +xfs_rtrefcountbt_init_cur_cache(void) +{ + xfs_rtrefcountbt_cur_cache = kmem_cache_create("xfs_rtrefcountbt_cur", + xfs_btree_cur_sizeof( + xfs_rtrefcountbt_maxlevels_ondisk()), + 0, 0, NULL); + + if (!xfs_rtrefcountbt_cur_cache) + return -ENOMEM; + return 0; +} + +void +xfs_rtrefcountbt_destroy_cur_cache(void) +{ + kmem_cache_destroy(xfs_rtrefcountbt_cur_cache); + xfs_rtrefcountbt_cur_cache = NULL; +} + +/* Compute the maximum height of a realtime refcount btree. */ +void +xfs_rtrefcountbt_compute_maxlevels( + struct xfs_mount *mp) +{ + unsigned int d_maxlevels, r_maxlevels; + + if (!xfs_has_rtreflink(mp)) { + mp->m_rtrefc_maxlevels = 0; + return; + } + + /* + * The realtime refcountbt lives on the data device, which means that + * its maximum height is constrained by the size of the data device and + * the height required to store one refcount record for each rtextent + * in an rt group. + */ + d_maxlevels = xfs_btree_space_to_height(mp->m_rtrefc_mnr, + mp->m_sb.sb_dblocks); + r_maxlevels = xfs_btree_compute_maxlevels(mp->m_rtrefc_mnr, + mp->m_sb.sb_rgextents); + + /* Add one level to handle the inode root level. */ + mp->m_rtrefc_maxlevels = min(d_maxlevels, r_maxlevels) + 1; +} + +/* Calculate the rtrefcount btree size for some records. */ +unsigned long long +xfs_rtrefcountbt_calc_size( + struct xfs_mount *mp, + unsigned long long len) +{ + return xfs_btree_calc_size(mp->m_rtrefc_mnr, len); +} + +/* + * Calculate the maximum refcount btree size. + */ +static unsigned long long +xfs_rtrefcountbt_max_size( + struct xfs_mount *mp, + xfs_rtblock_t rtblocks) +{ + /* Bail out if we're uninitialized, which can happen in mkfs. */ + if (mp->m_rtrefc_mxr[0] == 0) + return 0; + + return xfs_rtrefcountbt_calc_size(mp, rtblocks); +} + +/* + * Figure out how many blocks to reserve and how many are used by this btree. + * We need enough space to hold one record for every rt extent in the rtgroup. + */ +xfs_filblks_t +xfs_rtrefcountbt_calc_reserves( + struct xfs_mount *mp) +{ + if (!xfs_has_rtreflink(mp)) + return 0; + + return xfs_rtrefcountbt_max_size(mp, mp->m_sb.sb_rgextents); +} + +/* + * Convert on-disk form of btree root to in-memory form. + */ +STATIC void +xfs_rtrefcountbt_from_disk( + struct xfs_inode *ip, + struct xfs_rtrefcount_root *dblock, + int dblocklen, + struct xfs_btree_block *rblock) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_refcount_key *fkp; + __be64 *fpp; + struct xfs_refcount_key *tkp; + __be64 *tpp; + struct xfs_refcount_rec *frp; + struct xfs_refcount_rec *trp; + unsigned int numrecs; + unsigned int maxrecs; + unsigned int rblocklen; + + rblocklen = xfs_rtrefcount_broot_space(mp, dblock); + + xfs_btree_init_block(mp, rblock, &xfs_rtrefcountbt_ops, 0, 0, + I_INO(ip)); + + rblock->bb_level = dblock->bb_level; + rblock->bb_numrecs = dblock->bb_numrecs; + + if (be16_to_cpu(rblock->bb_level) > 0) { + maxrecs = xfs_rtrefcountbt_droot_maxrecs(dblocklen, false); + fkp = xfs_rtrefcount_droot_key_addr(dblock, 1); + tkp = xfs_rtrefcount_key_addr(rblock, 1); + fpp = xfs_rtrefcount_droot_ptr_addr(dblock, 1, maxrecs); + tpp = xfs_rtrefcount_broot_ptr_addr(mp, rblock, 1, rblocklen); + numrecs = be16_to_cpu(dblock->bb_numrecs); + memcpy(tkp, fkp, 2 * sizeof(*fkp) * numrecs); + memcpy(tpp, fpp, sizeof(*fpp) * numrecs); + } else { + frp = xfs_rtrefcount_droot_rec_addr(dblock, 1); + trp = xfs_rtrefcount_rec_addr(rblock, 1); + numrecs = be16_to_cpu(dblock->bb_numrecs); + memcpy(trp, frp, sizeof(*frp) * numrecs); + } +} + +/* Load a realtime reference count btree root in from disk. */ +int +xfs_iformat_rtrefcount( + struct xfs_inode *ip, + struct xfs_dinode *dip) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_rtrefcount_root *dfp = XFS_DFORK_PTR(dip, XFS_DATA_FORK); + struct xfs_btree_block *broot; + unsigned int numrecs; + unsigned int level; + int dsize; + + /* + * growfs must create the rtrefcount inodes before adding a realtime + * volume to the filesystem, so we cannot use the rtrefcount predicate + * here. + */ + if (!xfs_has_reflink(ip->i_mount)) { + xfs_inode_mark_sick(ip, XFS_SICK_INO_CORE); + return -EFSCORRUPTED; + } + + dsize = XFS_DFORK_SIZE(dip, mp, XFS_DATA_FORK); + numrecs = be16_to_cpu(dfp->bb_numrecs); + level = be16_to_cpu(dfp->bb_level); + + if (level >= mp->m_rtrefc_maxlevels || + xfs_rtrefcount_droot_space_calc(level, numrecs) > dsize) { + xfs_inode_mark_sick(ip, XFS_SICK_INO_CORE); + return -EFSCORRUPTED; + } + + broot = xfs_broot_alloc(xfs_ifork_ptr(ip, XFS_DATA_FORK), + xfs_rtrefcount_broot_space_calc(mp, level, numrecs)); + if (broot) + xfs_rtrefcountbt_from_disk(ip, dfp, dsize, broot); + return 0; +} + +/* + * Convert in-memory form of btree root to on-disk form. + */ +void +xfs_rtrefcountbt_to_disk( + struct xfs_mount *mp, + struct xfs_btree_block *rblock, + int rblocklen, + struct xfs_rtrefcount_root *dblock, + int dblocklen) +{ + struct xfs_refcount_key *fkp; + __be64 *fpp; + struct xfs_refcount_key *tkp; + __be64 *tpp; + struct xfs_refcount_rec *frp; + struct xfs_refcount_rec *trp; + unsigned int maxrecs; + unsigned int numrecs; + + ASSERT(rblock->bb_magic == cpu_to_be32(XFS_RTREFC_CRC_MAGIC)); + ASSERT(uuid_equal(&rblock->bb_u.l.bb_uuid, &mp->m_sb.sb_meta_uuid)); + ASSERT(rblock->bb_u.l.bb_blkno == cpu_to_be64(XFS_BUF_DADDR_NULL)); + ASSERT(rblock->bb_u.l.bb_leftsib == cpu_to_be64(NULLFSBLOCK)); + ASSERT(rblock->bb_u.l.bb_rightsib == cpu_to_be64(NULLFSBLOCK)); + + dblock->bb_level = rblock->bb_level; + dblock->bb_numrecs = rblock->bb_numrecs; + + if (be16_to_cpu(rblock->bb_level) > 0) { + maxrecs = xfs_rtrefcountbt_droot_maxrecs(dblocklen, false); + fkp = xfs_rtrefcount_key_addr(rblock, 1); + tkp = xfs_rtrefcount_droot_key_addr(dblock, 1); + fpp = xfs_rtrefcount_broot_ptr_addr(mp, rblock, 1, rblocklen); + tpp = xfs_rtrefcount_droot_ptr_addr(dblock, 1, maxrecs); + numrecs = be16_to_cpu(rblock->bb_numrecs); + memcpy(tkp, fkp, 2 * sizeof(*fkp) * numrecs); + memcpy(tpp, fpp, sizeof(*fpp) * numrecs); + } else { + frp = xfs_rtrefcount_rec_addr(rblock, 1); + trp = xfs_rtrefcount_droot_rec_addr(dblock, 1); + numrecs = be16_to_cpu(rblock->bb_numrecs); + memcpy(trp, frp, sizeof(*frp) * numrecs); + } +} + +/* Flush a realtime reference count btree root out to disk. */ +void +xfs_iflush_rtrefcount( + struct xfs_inode *ip, + struct xfs_dinode *dip) +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK); + struct xfs_rtrefcount_root *dfp = XFS_DFORK_PTR(dip, XFS_DATA_FORK); + + ASSERT(ifp->if_broot != NULL); + ASSERT(ifp->if_broot_bytes > 0); + ASSERT(xfs_rtrefcount_droot_space(ifp->if_broot) <= + xfs_inode_fork_size(ip, XFS_DATA_FORK)); + xfs_rtrefcountbt_to_disk(ip->i_mount, ifp->if_broot, + ifp->if_broot_bytes, dfp, + XFS_DFORK_SIZE(dip, ip->i_mount, XFS_DATA_FORK)); +} + +/* + * Create a realtime refcount btree inode. + */ +int +xfs_rtrefcountbt_create( + struct xfs_rtgroup *rtg, + struct xfs_inode *ip, + struct xfs_trans *tp, + bool init) +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK); + struct xfs_mount *mp = ip->i_mount; + struct xfs_btree_block *broot; + + ifp->if_format = XFS_DINODE_FMT_META_BTREE; + ASSERT(ifp->if_broot_bytes == 0); + ASSERT(ifp->if_bytes == 0); + + /* Initialize the empty incore btree root. */ + broot = xfs_broot_realloc(ifp, + xfs_rtrefcount_broot_space_calc(mp, 0, 0)); + if (broot) + xfs_btree_init_block(mp, broot, &xfs_rtrefcountbt_ops, 0, 0, + I_INO(ip)); + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE | XFS_ILOG_DBROOT); + return 0; +} diff --git a/fs/xfs/libxfs/xfs_rtrefcount_btree.h b/fs/xfs/libxfs/xfs_rtrefcount_btree.h new file mode 100644 index 000000000..a99b7a8ae --- /dev/null +++ b/fs/xfs/libxfs/xfs_rtrefcount_btree.h @@ -0,0 +1,189 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2021-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_RTREFCOUNT_BTREE_H__ +#define __XFS_RTREFCOUNT_BTREE_H__ + +struct xfs_buf; +struct xfs_btree_cur; +struct xfs_mount; +struct xbtree_ifakeroot; +struct xfs_rtgroup; + +/* refcounts only exist on crc enabled filesystems */ +#define XFS_RTREFCOUNT_BLOCK_LEN XFS_BTREE_LBLOCK_CRC_LEN + +struct xfs_btree_cur *xfs_rtrefcountbt_init_cursor(struct xfs_trans *tp, + struct xfs_rtgroup *rtg); +struct xfs_btree_cur *xfs_rtrefcountbt_stage_cursor(struct xfs_mount *mp, + struct xfs_rtgroup *rtg, struct xfs_inode *ip, + struct xbtree_ifakeroot *ifake); +void xfs_rtrefcountbt_commit_staged_btree(struct xfs_btree_cur *cur, + struct xfs_trans *tp); +unsigned int xfs_rtrefcountbt_maxrecs(struct xfs_mount *mp, + unsigned int blocklen, bool leaf); +void xfs_rtrefcountbt_compute_maxlevels(struct xfs_mount *mp); +unsigned int xfs_rtrefcountbt_droot_maxrecs(unsigned int blocklen, bool leaf); + +/* + * Addresses of records, keys, and pointers within an incore rtrefcountbt block. + * + * (note that some of these may appear unused, but they are used in userspace) + */ +static inline struct xfs_refcount_rec * +xfs_rtrefcount_rec_addr( + struct xfs_btree_block *block, + unsigned int index) +{ + return (struct xfs_refcount_rec *) + ((char *)block + XFS_RTREFCOUNT_BLOCK_LEN + + (index - 1) * sizeof(struct xfs_refcount_rec)); +} + +static inline struct xfs_refcount_key * +xfs_rtrefcount_key_addr( + struct xfs_btree_block *block, + unsigned int index) +{ + return (struct xfs_refcount_key *) + ((char *)block + XFS_RTREFCOUNT_BLOCK_LEN + + (index - 1) * sizeof(struct xfs_refcount_key)); +} + +static inline xfs_rtrefcount_ptr_t * +xfs_rtrefcount_ptr_addr( + struct xfs_btree_block *block, + unsigned int index, + unsigned int maxrecs) +{ + return (xfs_rtrefcount_ptr_t *) + ((char *)block + XFS_RTREFCOUNT_BLOCK_LEN + + maxrecs * sizeof(struct xfs_refcount_key) + + (index - 1) * sizeof(xfs_rtrefcount_ptr_t)); +} + +unsigned int xfs_rtrefcountbt_maxlevels_ondisk(void); +int __init xfs_rtrefcountbt_init_cur_cache(void); +void xfs_rtrefcountbt_destroy_cur_cache(void); + +xfs_filblks_t xfs_rtrefcountbt_calc_reserves(struct xfs_mount *mp); +unsigned long long xfs_rtrefcountbt_calc_size(struct xfs_mount *mp, + unsigned long long len); + +/* Addresses of key, pointers, and records within an ondisk rtrefcount block. */ + +static inline struct xfs_refcount_rec * +xfs_rtrefcount_droot_rec_addr( + struct xfs_rtrefcount_root *block, + unsigned int index) +{ + return (struct xfs_refcount_rec *) + ((char *)(block + 1) + + (index - 1) * sizeof(struct xfs_refcount_rec)); +} + +static inline struct xfs_refcount_key * +xfs_rtrefcount_droot_key_addr( + struct xfs_rtrefcount_root *block, + unsigned int index) +{ + return (struct xfs_refcount_key *) + ((char *)(block + 1) + + (index - 1) * sizeof(struct xfs_refcount_key)); +} + +static inline xfs_rtrefcount_ptr_t * +xfs_rtrefcount_droot_ptr_addr( + struct xfs_rtrefcount_root *block, + unsigned int index, + unsigned int maxrecs) +{ + return (xfs_rtrefcount_ptr_t *) + ((char *)(block + 1) + + maxrecs * sizeof(struct xfs_refcount_key) + + (index - 1) * sizeof(xfs_rtrefcount_ptr_t)); +} + +/* + * Address of pointers within the incore btree root. + * + * These are to be used when we know the size of the block and + * we don't have a cursor. + */ +static inline xfs_rtrefcount_ptr_t * +xfs_rtrefcount_broot_ptr_addr( + struct xfs_mount *mp, + struct xfs_btree_block *bb, + unsigned int index, + unsigned int block_size) +{ + return xfs_rtrefcount_ptr_addr(bb, index, + xfs_rtrefcountbt_maxrecs(mp, block_size, false)); +} + +/* + * Compute the space required for the incore btree root containing the given + * number of records. + */ +static inline size_t +xfs_rtrefcount_broot_space_calc( + struct xfs_mount *mp, + unsigned int level, + unsigned int nrecs) +{ + size_t sz = XFS_RTREFCOUNT_BLOCK_LEN; + + if (level > 0) + return sz + nrecs * (sizeof(struct xfs_refcount_key) + + sizeof(xfs_rtrefcount_ptr_t)); + return sz + nrecs * sizeof(struct xfs_refcount_rec); +} + +/* + * Compute the space required for the incore btree root given the ondisk + * btree root block. + */ +static inline size_t +xfs_rtrefcount_broot_space(struct xfs_mount *mp, struct xfs_rtrefcount_root *bb) +{ + return xfs_rtrefcount_broot_space_calc(mp, be16_to_cpu(bb->bb_level), + be16_to_cpu(bb->bb_numrecs)); +} + +/* Compute the space required for the ondisk root block. */ +static inline size_t +xfs_rtrefcount_droot_space_calc( + unsigned int level, + unsigned int nrecs) +{ + size_t sz = sizeof(struct xfs_rtrefcount_root); + + if (level > 0) + return sz + nrecs * (sizeof(struct xfs_refcount_key) + + sizeof(xfs_rtrefcount_ptr_t)); + return sz + nrecs * sizeof(struct xfs_refcount_rec); +} + +/* + * Compute the space required for the ondisk root block given an incore root + * block. + */ +static inline size_t +xfs_rtrefcount_droot_space(struct xfs_btree_block *bb) +{ + return xfs_rtrefcount_droot_space_calc(be16_to_cpu(bb->bb_level), + be16_to_cpu(bb->bb_numrecs)); +} + +int xfs_iformat_rtrefcount(struct xfs_inode *ip, struct xfs_dinode *dip); +void xfs_rtrefcountbt_to_disk(struct xfs_mount *mp, + struct xfs_btree_block *rblock, int rblocklen, + struct xfs_rtrefcount_root *dblock, int dblocklen); +void xfs_iflush_rtrefcount(struct xfs_inode *ip, struct xfs_dinode *dip); + +int xfs_rtrefcountbt_create(struct xfs_rtgroup *rtg, struct xfs_inode *ip, + struct xfs_trans *tp, bool init); + +#endif /* __XFS_RTREFCOUNT_BTREE_H__ */ diff --git a/fs/xfs/libxfs/xfs_rtrmap_btree.c b/fs/xfs/libxfs/xfs_rtrmap_btree.c new file mode 100644 index 000000000..c264bc565 --- /dev/null +++ b/fs/xfs/libxfs/xfs_rtrmap_btree.c @@ -0,0 +1,1033 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2018-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_sb.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_alloc.h" +#include "xfs_btree.h" +#include "xfs_btree_staging.h" +#include "xfs_metafile.h" +#include "xfs_rmap.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_trace.h" +#include "xfs_cksum.h" +#include "xfs_error.h" +#include "xfs_extent_busy.h" +#include "xfs_rtgroup.h" +#include "xfs_bmap.h" +#include "xfs_health.h" +#include "xfs_buf_mem.h" +#include "xfs_btree_mem.h" + +static struct kmem_cache *xfs_rtrmapbt_cur_cache; + +/* + * Realtime Reverse Map btree. + * + * This is a btree used to track the owner(s) of a given extent in the realtime + * device. See the comments in xfs_rmap_btree.c for more information. + * + * This tree is basically the same as the regular rmap btree except that it + * is rooted in an inode and does not live in free space. + */ + +static struct xfs_btree_cur * +xfs_rtrmapbt_dup_cursor( + struct xfs_btree_cur *cur) +{ + return xfs_rtrmapbt_init_cursor(cur->bc_tp, to_rtg(cur->bc_group)); +} + +STATIC int +xfs_rtrmapbt_get_minrecs( + struct xfs_btree_cur *cur, + int level) +{ + if (level == cur->bc_nlevels - 1) { + struct xfs_ifork *ifp = xfs_btree_ifork_ptr(cur); + + return xfs_rtrmapbt_maxrecs(cur->bc_mp, ifp->if_broot_bytes, + level == 0) / 2; + } + + return cur->bc_mp->m_rtrmap_mnr[level != 0]; +} + +STATIC int +xfs_rtrmapbt_get_maxrecs( + struct xfs_btree_cur *cur, + int level) +{ + if (level == cur->bc_nlevels - 1) { + struct xfs_ifork *ifp = xfs_btree_ifork_ptr(cur); + + return xfs_rtrmapbt_maxrecs(cur->bc_mp, ifp->if_broot_bytes, + level == 0); + } + + return cur->bc_mp->m_rtrmap_mxr[level != 0]; +} + +/* Calculate number of records in the ondisk realtime rmap btree inode root. */ +unsigned int +xfs_rtrmapbt_droot_maxrecs( + unsigned int blocklen, + bool leaf) +{ + blocklen -= sizeof(struct xfs_rtrmap_root); + + if (leaf) + return blocklen / sizeof(struct xfs_rmap_rec); + return blocklen / (2 * sizeof(struct xfs_rmap_key) + + sizeof(xfs_rtrmap_ptr_t)); +} + +/* + * Get the maximum records we could store in the on-disk format. + * + * For non-root nodes this is equivalent to xfs_rtrmapbt_get_maxrecs, but + * for the root node this checks the available space in the dinode fork + * so that we can resize the in-memory buffer to match it. After a + * resize to the maximum size this function returns the same value + * as xfs_rtrmapbt_get_maxrecs for the root node, too. + */ +STATIC int +xfs_rtrmapbt_get_dmaxrecs( + struct xfs_btree_cur *cur, + int level) +{ + if (level != cur->bc_nlevels - 1) + return cur->bc_mp->m_rtrmap_mxr[level != 0]; + return xfs_rtrmapbt_droot_maxrecs(cur->bc_ino.forksize, level == 0); +} + +/* + * Convert the ondisk record's offset field into the ondisk key's offset field. + * Fork and bmbt are significant parts of the rmap record key, but written + * status is merely a record attribute. + */ +static inline __be64 ondisk_rec_offset_to_key(const union xfs_btree_rec *rec) +{ + return rec->rmap.rm_offset & ~cpu_to_be64(XFS_RMAP_OFF_UNWRITTEN); +} + +STATIC void +xfs_rtrmapbt_init_key_from_rec( + union xfs_btree_key *key, + const union xfs_btree_rec *rec) +{ + key->rmap.rm_startblock = rec->rmap.rm_startblock; + key->rmap.rm_owner = rec->rmap.rm_owner; + key->rmap.rm_offset = ondisk_rec_offset_to_key(rec); +} + +STATIC void +xfs_rtrmapbt_init_high_key_from_rec( + union xfs_btree_key *key, + const union xfs_btree_rec *rec) +{ + uint64_t off; + int adj; + + adj = be32_to_cpu(rec->rmap.rm_blockcount) - 1; + + key->rmap.rm_startblock = rec->rmap.rm_startblock; + be32_add_cpu(&key->rmap.rm_startblock, adj); + key->rmap.rm_owner = rec->rmap.rm_owner; + key->rmap.rm_offset = ondisk_rec_offset_to_key(rec); + if (XFS_RMAP_NON_INODE_OWNER(be64_to_cpu(rec->rmap.rm_owner)) || + XFS_RMAP_IS_BMBT_BLOCK(be64_to_cpu(rec->rmap.rm_offset))) + return; + off = be64_to_cpu(key->rmap.rm_offset); + off = (XFS_RMAP_OFF(off) + adj) | (off & ~XFS_RMAP_OFF_MASK); + key->rmap.rm_offset = cpu_to_be64(off); +} + +STATIC void +xfs_rtrmapbt_init_rec_from_cur( + struct xfs_btree_cur *cur, + union xfs_btree_rec *rec) +{ + rec->rmap.rm_startblock = cpu_to_be32(cur->bc_rec.r.rm_startblock); + rec->rmap.rm_blockcount = cpu_to_be32(cur->bc_rec.r.rm_blockcount); + rec->rmap.rm_owner = cpu_to_be64(cur->bc_rec.r.rm_owner); + rec->rmap.rm_offset = cpu_to_be64( + xfs_rmap_irec_offset_pack(&cur->bc_rec.r)); +} + +STATIC void +xfs_rtrmapbt_init_ptr_from_cur( + struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr) +{ + ptr->l = 0; +} + +/* + * Mask the appropriate parts of the ondisk key field for a key comparison. + * Fork and bmbt are significant parts of the rmap record key, but written + * status is merely a record attribute. + */ +static inline uint64_t offset_keymask(uint64_t offset) +{ + return offset & ~XFS_RMAP_OFF_UNWRITTEN; +} + +STATIC int +xfs_rtrmapbt_cmp_key_with_cur( + struct xfs_btree_cur *cur, + const union xfs_btree_key *key) +{ + struct xfs_rmap_irec *rec = &cur->bc_rec.r; + const struct xfs_rmap_key *kp = &key->rmap; + + return cmp_int(be32_to_cpu(kp->rm_startblock), rec->rm_startblock) ?: + cmp_int(be64_to_cpu(kp->rm_owner), rec->rm_owner) ?: + cmp_int(offset_keymask(be64_to_cpu(kp->rm_offset)), + offset_keymask(xfs_rmap_irec_offset_pack(rec))); +} + +STATIC int +xfs_rtrmapbt_cmp_two_keys( + struct xfs_btree_cur *cur, + const union xfs_btree_key *k1, + const union xfs_btree_key *k2, + const union xfs_btree_key *mask) +{ + const struct xfs_rmap_key *kp1 = &k1->rmap; + const struct xfs_rmap_key *kp2 = &k2->rmap; + int d; + + /* Doesn't make sense to mask off the physical space part */ + ASSERT(!mask || mask->rmap.rm_startblock); + + d = cmp_int(be32_to_cpu(kp1->rm_startblock), + be32_to_cpu(kp2->rm_startblock)); + if (d) + return d; + + if (!mask || mask->rmap.rm_owner) { + d = cmp_int(be64_to_cpu(kp1->rm_owner), + be64_to_cpu(kp2->rm_owner)); + if (d) + return d; + } + + if (!mask || mask->rmap.rm_offset) { + /* Doesn't make sense to allow offset but not owner */ + ASSERT(!mask || mask->rmap.rm_owner); + + d = cmp_int(offset_keymask(be64_to_cpu(kp1->rm_offset)), + offset_keymask(be64_to_cpu(kp2->rm_offset))); + if (d) + return d; + } + + return 0; +} + +static xfs_failaddr_t +xfs_rtrmapbt_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_target->bt_mount; + struct xfs_btree_block *block = XFS_BUF_TO_BLOCK(bp); + xfs_failaddr_t fa; + int level; + + if (!xfs_verify_magic(bp, block->bb_magic)) + return __this_address; + + if (!xfs_has_rmapbt(mp)) + return __this_address; + fa = xfs_btree_fsblock_v5hdr_verify(bp, XFS_RMAP_OWN_UNKNOWN); + if (fa) + return fa; + level = be16_to_cpu(block->bb_level); + if (level > mp->m_rtrmap_maxlevels) + return __this_address; + + return xfs_btree_fsblock_verify(bp, mp->m_rtrmap_mxr[level != 0]); +} + +static void +xfs_rtrmapbt_read_verify( + struct xfs_buf *bp) +{ + xfs_failaddr_t fa; + + if (!xfs_btree_fsblock_verify_crc(bp)) + xfs_verifier_error(bp, -EFSBADCRC, __this_address); + else { + fa = xfs_rtrmapbt_verify(bp); + if (fa) + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + } + + if (bp->b_error) + trace_xfs_btree_corrupt(bp, _RET_IP_); +} + +static void +xfs_rtrmapbt_write_verify( + struct xfs_buf *bp) +{ + xfs_failaddr_t fa; + + fa = xfs_rtrmapbt_verify(bp); + if (fa) { + trace_xfs_btree_corrupt(bp, _RET_IP_); + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + return; + } + xfs_btree_fsblock_calc_crc(bp); + +} + +const struct xfs_buf_ops xfs_rtrmapbt_buf_ops = { + .name = "xfs_rtrmapbt", + .magic = { 0, cpu_to_be32(XFS_RTRMAP_CRC_MAGIC) }, + .verify_read = xfs_rtrmapbt_read_verify, + .verify_write = xfs_rtrmapbt_write_verify, + .verify_struct = xfs_rtrmapbt_verify, +}; + +STATIC int +xfs_rtrmapbt_keys_inorder( + struct xfs_btree_cur *cur, + const union xfs_btree_key *k1, + const union xfs_btree_key *k2) +{ + uint32_t x; + uint32_t y; + uint64_t a; + uint64_t b; + + x = be32_to_cpu(k1->rmap.rm_startblock); + y = be32_to_cpu(k2->rmap.rm_startblock); + if (x < y) + return 1; + else if (x > y) + return 0; + a = be64_to_cpu(k1->rmap.rm_owner); + b = be64_to_cpu(k2->rmap.rm_owner); + if (a < b) + return 1; + else if (a > b) + return 0; + a = offset_keymask(be64_to_cpu(k1->rmap.rm_offset)); + b = offset_keymask(be64_to_cpu(k2->rmap.rm_offset)); + if (a <= b) + return 1; + return 0; +} + +STATIC int +xfs_rtrmapbt_recs_inorder( + struct xfs_btree_cur *cur, + const union xfs_btree_rec *r1, + const union xfs_btree_rec *r2) +{ + uint32_t x; + uint32_t y; + uint64_t a; + uint64_t b; + + x = be32_to_cpu(r1->rmap.rm_startblock); + y = be32_to_cpu(r2->rmap.rm_startblock); + if (x < y) + return 1; + else if (x > y) + return 0; + a = be64_to_cpu(r1->rmap.rm_owner); + b = be64_to_cpu(r2->rmap.rm_owner); + if (a < b) + return 1; + else if (a > b) + return 0; + a = offset_keymask(be64_to_cpu(r1->rmap.rm_offset)); + b = offset_keymask(be64_to_cpu(r2->rmap.rm_offset)); + if (a <= b) + return 1; + return 0; +} + +STATIC enum xbtree_key_contig +xfs_rtrmapbt_keys_contiguous( + struct xfs_btree_cur *cur, + const union xfs_btree_key *key1, + const union xfs_btree_key *key2, + const union xfs_btree_key *mask) +{ + ASSERT(!mask || mask->rmap.rm_startblock); + + /* + * We only support checking contiguity of the physical space component. + * If any callers ever need more specificity than that, they'll have to + * implement it here. + */ + ASSERT(!mask || (!mask->rmap.rm_owner && !mask->rmap.rm_offset)); + + return xbtree_key_contig(be32_to_cpu(key1->rmap.rm_startblock), + be32_to_cpu(key2->rmap.rm_startblock)); +} + +static inline void +xfs_rtrmapbt_move_ptrs( + struct xfs_mount *mp, + struct xfs_btree_block *broot, + short old_size, + size_t new_size, + unsigned int numrecs) +{ + void *dptr; + void *sptr; + + sptr = xfs_rtrmap_broot_ptr_addr(mp, broot, 1, old_size); + dptr = xfs_rtrmap_broot_ptr_addr(mp, broot, 1, new_size); + memmove(dptr, sptr, numrecs * sizeof(xfs_rtrmap_ptr_t)); +} + +static struct xfs_btree_block * +xfs_rtrmapbt_broot_realloc( + struct xfs_btree_cur *cur, + unsigned int new_numrecs) +{ + struct xfs_mount *mp = cur->bc_mp; + struct xfs_ifork *ifp = xfs_btree_ifork_ptr(cur); + struct xfs_btree_block *broot; + unsigned int new_size; + unsigned int old_size = ifp->if_broot_bytes; + const unsigned int level = cur->bc_nlevels - 1; + + new_size = xfs_rtrmap_broot_space_calc(mp, level, new_numrecs); + + /* Handle the nop case quietly. */ + if (new_size == old_size) + return ifp->if_broot; + + if (new_size > old_size) { + unsigned int old_numrecs; + + /* + * If there wasn't any memory allocated before, just allocate + * it now and get out. + */ + if (old_size == 0) + return xfs_broot_realloc(ifp, new_size); + + /* + * If there is already an existing if_broot, then we need to + * realloc it and possibly move the node block pointers because + * those are not butted up against the btree block header. + */ + old_numrecs = xfs_rtrmapbt_maxrecs(mp, old_size, level == 0); + broot = xfs_broot_realloc(ifp, new_size); + if (level > 0) + xfs_rtrmapbt_move_ptrs(mp, broot, old_size, new_size, + old_numrecs); + goto out_broot; + } + + /* + * We're reducing numrecs. If we're going all the way to zero, just + * free the block. + */ + ASSERT(ifp->if_broot != NULL && old_size > 0); + if (new_size == 0) + return xfs_broot_realloc(ifp, 0); + + /* + * Shrink the btree root by possibly moving the rtrmapbt pointers, + * since they are not butted up against the btree block header. Then + * reallocate broot. + */ + if (level > 0) + xfs_rtrmapbt_move_ptrs(mp, ifp->if_broot, old_size, new_size, + new_numrecs); + broot = xfs_broot_realloc(ifp, new_size); + +out_broot: + ASSERT(xfs_rtrmap_droot_space(broot) <= + xfs_inode_fork_size(cur->bc_ino.ip, cur->bc_ino.whichfork)); + return broot; +} + +const struct xfs_btree_ops xfs_rtrmapbt_ops = { + .name = "rtrmap", + .type = XFS_BTREE_TYPE_INODE, + .geom_flags = XFS_BTGEO_OVERLAPPING | + XFS_BTGEO_IROOT_RECORDS, + + .rec_len = sizeof(struct xfs_rmap_rec), + /* Overlapping btree; 2 keys per pointer. */ + .key_len = 2 * sizeof(struct xfs_rmap_key), + .ptr_len = XFS_BTREE_LONG_PTR_LEN, + + .lru_refs = XFS_RMAP_BTREE_REF, + .statoff = XFS_STATS_CALC_INDEX(xs_rtrmap_2), + .sick_mask = XFS_SICK_RG_RMAPBT, + + .dup_cursor = xfs_rtrmapbt_dup_cursor, + .alloc_block = xfs_btree_alloc_metafile_block, + .free_block = xfs_btree_free_metafile_block, + .get_minrecs = xfs_rtrmapbt_get_minrecs, + .get_maxrecs = xfs_rtrmapbt_get_maxrecs, + .get_dmaxrecs = xfs_rtrmapbt_get_dmaxrecs, + .init_key_from_rec = xfs_rtrmapbt_init_key_from_rec, + .init_high_key_from_rec = xfs_rtrmapbt_init_high_key_from_rec, + .init_rec_from_cur = xfs_rtrmapbt_init_rec_from_cur, + .init_ptr_from_cur = xfs_rtrmapbt_init_ptr_from_cur, + .cmp_key_with_cur = xfs_rtrmapbt_cmp_key_with_cur, + .buf_ops = &xfs_rtrmapbt_buf_ops, + .cmp_two_keys = xfs_rtrmapbt_cmp_two_keys, + .keys_inorder = xfs_rtrmapbt_keys_inorder, + .recs_inorder = xfs_rtrmapbt_recs_inorder, + .keys_contiguous = xfs_rtrmapbt_keys_contiguous, + .broot_realloc = xfs_rtrmapbt_broot_realloc, +}; + +/* Allocate a new rt rmap btree cursor. */ +struct xfs_btree_cur * +xfs_rtrmapbt_init_cursor( + struct xfs_trans *tp, + struct xfs_rtgroup *rtg) +{ + struct xfs_inode *ip = rtg_rmap(rtg); + struct xfs_mount *mp = rtg_mount(rtg); + struct xfs_btree_cur *cur; + + xfs_assert_ilocked(ip, XFS_ILOCK_SHARED | XFS_ILOCK_EXCL); + + cur = xfs_btree_alloc_cursor(mp, tp, &xfs_rtrmapbt_ops, + mp->m_rtrmap_maxlevels, xfs_rtrmapbt_cur_cache); + + cur->bc_ino.ip = ip; + cur->bc_group = xfs_group_hold(rtg_group(rtg)); + cur->bc_ino.whichfork = XFS_DATA_FORK; + cur->bc_nlevels = be16_to_cpu(ip->i_df.if_broot->bb_level) + 1; + cur->bc_ino.forksize = xfs_inode_fork_size(ip, XFS_DATA_FORK); + + return cur; +} + +#ifdef CONFIG_XFS_BTREE_IN_MEM +/* + * Validate an in-memory realtime rmap btree block. Callers are allowed to + * generate an in-memory btree even if the ondisk feature is not enabled. + */ +static xfs_failaddr_t +xfs_rtrmapbt_mem_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_btree_block *block = XFS_BUF_TO_BLOCK(bp); + xfs_failaddr_t fa; + unsigned int level; + unsigned int maxrecs; + + if (!xfs_verify_magic(bp, block->bb_magic)) + return __this_address; + + fa = xfs_btree_fsblock_v5hdr_verify(bp, XFS_RMAP_OWN_UNKNOWN); + if (fa) + return fa; + + level = be16_to_cpu(block->bb_level); + if (xfs_has_rmapbt(mp)) { + if (level >= mp->m_rtrmap_maxlevels) + return __this_address; + } else { + if (level >= xfs_rtrmapbt_maxlevels_ondisk()) + return __this_address; + } + + maxrecs = xfs_rtrmapbt_maxrecs(mp, XFBNO_BLOCKSIZE, level == 0); + return xfs_btree_memblock_verify(bp, maxrecs); +} + +static void +xfs_rtrmapbt_mem_rw_verify( + struct xfs_buf *bp) +{ + xfs_failaddr_t fa = xfs_rtrmapbt_mem_verify(bp); + + if (fa) + xfs_verifier_error(bp, -EFSCORRUPTED, fa); +} + +/* skip crc checks on in-memory btrees to save time */ +static const struct xfs_buf_ops xfs_rtrmapbt_mem_buf_ops = { + .name = "xfs_rtrmapbt_mem", + .magic = { 0, cpu_to_be32(XFS_RTRMAP_CRC_MAGIC) }, + .verify_read = xfs_rtrmapbt_mem_rw_verify, + .verify_write = xfs_rtrmapbt_mem_rw_verify, + .verify_struct = xfs_rtrmapbt_mem_verify, +}; + +const struct xfs_btree_ops xfs_rtrmapbt_mem_ops = { + .type = XFS_BTREE_TYPE_MEM, + .geom_flags = XFS_BTGEO_OVERLAPPING, + + .rec_len = sizeof(struct xfs_rmap_rec), + /* Overlapping btree; 2 keys per pointer. */ + .key_len = 2 * sizeof(struct xfs_rmap_key), + .ptr_len = XFS_BTREE_LONG_PTR_LEN, + + .lru_refs = XFS_RMAP_BTREE_REF, + .statoff = XFS_STATS_CALC_INDEX(xs_rtrmap_mem_2), + + .dup_cursor = xfbtree_dup_cursor, + .set_root = xfbtree_set_root, + .alloc_block = xfbtree_alloc_block, + .free_block = xfbtree_free_block, + .get_minrecs = xfbtree_get_minrecs, + .get_maxrecs = xfbtree_get_maxrecs, + .init_key_from_rec = xfs_rtrmapbt_init_key_from_rec, + .init_high_key_from_rec = xfs_rtrmapbt_init_high_key_from_rec, + .init_rec_from_cur = xfs_rtrmapbt_init_rec_from_cur, + .init_ptr_from_cur = xfbtree_init_ptr_from_cur, + .cmp_key_with_cur = xfs_rtrmapbt_cmp_key_with_cur, + .buf_ops = &xfs_rtrmapbt_mem_buf_ops, + .cmp_two_keys = xfs_rtrmapbt_cmp_two_keys, + .keys_inorder = xfs_rtrmapbt_keys_inorder, + .recs_inorder = xfs_rtrmapbt_recs_inorder, + .keys_contiguous = xfs_rtrmapbt_keys_contiguous, +}; + +/* Create a cursor for an in-memory btree. */ +struct xfs_btree_cur * +xfs_rtrmapbt_mem_cursor( + struct xfs_rtgroup *rtg, + struct xfs_trans *tp, + struct xfbtree *xfbt) +{ + struct xfs_mount *mp = rtg_mount(rtg); + struct xfs_btree_cur *cur; + + cur = xfs_btree_alloc_cursor(mp, tp, &xfs_rtrmapbt_mem_ops, + mp->m_rtrmap_maxlevels, xfs_rtrmapbt_cur_cache); + cur->bc_mem.xfbtree = xfbt; + cur->bc_nlevels = xfbt->nlevels; + cur->bc_group = xfs_group_hold(rtg_group(rtg)); + return cur; +} + +/* Create an in-memory realtime rmap btree. */ +int +xfs_rtrmapbt_mem_init( + struct xfs_mount *mp, + struct xfbtree *xfbt, + struct xfs_buftarg *btp, + xfs_rgnumber_t rgno) +{ + xfbt->owner = rgno; + return xfbtree_init(mp, xfbt, btp, &xfs_rtrmapbt_mem_ops); +} +#endif /* CONFIG_XFS_BTREE_IN_MEM */ + +/* + * Install a new rt reverse mapping btree root. Caller is responsible for + * invalidating and freeing the old btree blocks. + */ +void +xfs_rtrmapbt_commit_staged_btree( + struct xfs_btree_cur *cur, + struct xfs_trans *tp) +{ + struct xbtree_ifakeroot *ifake = cur->bc_ino.ifake; + struct xfs_ifork *ifp; + int flags = XFS_ILOG_CORE | XFS_ILOG_DBROOT; + + ASSERT(cur->bc_flags & XFS_BTREE_STAGING); + ASSERT(ifake->if_fork->if_format == XFS_DINODE_FMT_META_BTREE); + + /* + * Free any resources hanging off the real fork, then shallow-copy the + * staging fork's contents into the real fork to transfer everything + * we just built. + */ + ifp = xfs_ifork_ptr(cur->bc_ino.ip, XFS_DATA_FORK); + xfs_idestroy_fork(ifp); + memcpy(ifp, ifake->if_fork, sizeof(struct xfs_ifork)); + + cur->bc_ino.ip->i_projid = cur->bc_group->xg_gno; + xfs_trans_log_inode(tp, cur->bc_ino.ip, flags); + xfs_btree_commit_ifakeroot(cur, tp, XFS_DATA_FORK); +} + +/* Calculate number of records in a rt reverse mapping btree block. */ +static inline unsigned int +xfs_rtrmapbt_block_maxrecs( + unsigned int blocklen, + bool leaf) +{ + if (leaf) + return blocklen / sizeof(struct xfs_rmap_rec); + return blocklen / + (2 * sizeof(struct xfs_rmap_key) + sizeof(xfs_rtrmap_ptr_t)); +} + +/* + * Calculate number of records in an rt reverse mapping btree block. + */ +unsigned int +xfs_rtrmapbt_maxrecs( + struct xfs_mount *mp, + unsigned int blocklen, + bool leaf) +{ + blocklen -= XFS_RTRMAP_BLOCK_LEN; + return xfs_rtrmapbt_block_maxrecs(blocklen, leaf); +} + +/* Compute the max possible height for realtime reverse mapping btrees. */ +unsigned int +xfs_rtrmapbt_maxlevels_ondisk(void) +{ + unsigned long long max_dblocks; + unsigned int minrecs[2]; + unsigned int blocklen; + + blocklen = XFS_MIN_CRC_BLOCKSIZE - XFS_BTREE_LBLOCK_CRC_LEN; + + minrecs[0] = xfs_rtrmapbt_block_maxrecs(blocklen, true) / 2; + minrecs[1] = xfs_rtrmapbt_block_maxrecs(blocklen, false) / 2; + + /* + * Compute the asymptotic maxlevels for an rtrmapbt on any rtreflink fs. + * + * On a reflink filesystem, each block in an rtgroup can have up to + * 2^32 (per the refcount record format) owners, which means that + * theoretically we could face up to 2^64 rmap records. However, we're + * likely to run out of blocks in the data device long before that + * happens, which means that we must compute the max height based on + * what the btree will look like if it consumes almost all the blocks + * in the data device due to maximal sharing factor. + */ + max_dblocks = -1U; /* max ag count */ + max_dblocks *= XFS_MAX_CRC_AG_BLOCKS; + return xfs_btree_space_to_height(minrecs, max_dblocks); +} + +int __init +xfs_rtrmapbt_init_cur_cache(void) +{ + xfs_rtrmapbt_cur_cache = kmem_cache_create("xfs_rtrmapbt_cur", + xfs_btree_cur_sizeof(xfs_rtrmapbt_maxlevels_ondisk()), + 0, 0, NULL); + + if (!xfs_rtrmapbt_cur_cache) + return -ENOMEM; + return 0; +} + +void +xfs_rtrmapbt_destroy_cur_cache(void) +{ + kmem_cache_destroy(xfs_rtrmapbt_cur_cache); + xfs_rtrmapbt_cur_cache = NULL; +} + +/* Compute the maximum height of an rt reverse mapping btree. */ +void +xfs_rtrmapbt_compute_maxlevels( + struct xfs_mount *mp) +{ + unsigned int d_maxlevels, r_maxlevels; + + if (!xfs_has_rtrmapbt(mp)) { + mp->m_rtrmap_maxlevels = 0; + return; + } + + /* + * The realtime rmapbt lives on the data device, which means that its + * maximum height is constrained by the size of the data device and + * the height required to store one rmap record for each block in an + * rt group. + * + * On a reflink filesystem, each rt block can have up to 2^32 (per the + * refcount record format) owners, which means that theoretically we + * could face up to 2^64 rmap records. This makes the computation of + * maxlevels based on record count meaningless, so we only consider the + * size of the data device. + */ + d_maxlevels = xfs_btree_space_to_height(mp->m_rtrmap_mnr, + mp->m_sb.sb_dblocks); + if (xfs_has_rtreflink(mp)) { + mp->m_rtrmap_maxlevels = d_maxlevels + 1; + return; + } + + r_maxlevels = xfs_btree_compute_maxlevels(mp->m_rtrmap_mnr, + mp->m_groups[XG_TYPE_RTG].blocks); + + /* Add one level to handle the inode root level. */ + mp->m_rtrmap_maxlevels = min(d_maxlevels, r_maxlevels) + 1; +} + +/* Calculate the rtrmap btree size for some records. */ +unsigned long long +xfs_rtrmapbt_calc_size( + struct xfs_mount *mp, + unsigned long long len) +{ + return xfs_btree_calc_size(mp->m_rtrmap_mnr, len); +} + +/* + * Calculate the maximum rmap btree size. + */ +static unsigned long long +xfs_rtrmapbt_max_size( + struct xfs_mount *mp, + xfs_rtblock_t rtblocks) +{ + /* Bail out if we're uninitialized, which can happen in mkfs. */ + if (mp->m_rtrmap_mxr[0] == 0) + return 0; + + return xfs_rtrmapbt_calc_size(mp, rtblocks); +} + +/* + * Figure out how many blocks to reserve and how many are used by this btree. + */ +xfs_filblks_t +xfs_rtrmapbt_calc_reserves( + struct xfs_mount *mp) +{ + uint32_t blocks = mp->m_groups[XG_TYPE_RTG].blocks; + + if (!xfs_has_rtrmapbt(mp)) + return 0; + + /* Reserve 1% of the rtgroup or enough for 1 block per record. */ + return max_t(xfs_filblks_t, blocks / 100, + xfs_rtrmapbt_max_size(mp, blocks)); +} + +/* Convert on-disk form of btree root to in-memory form. */ +STATIC void +xfs_rtrmapbt_from_disk( + struct xfs_inode *ip, + struct xfs_rtrmap_root *dblock, + unsigned int dblocklen, + struct xfs_btree_block *rblock) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_rmap_key *fkp; + __be64 *fpp; + struct xfs_rmap_key *tkp; + __be64 *tpp; + struct xfs_rmap_rec *frp; + struct xfs_rmap_rec *trp; + unsigned int rblocklen = xfs_rtrmap_broot_space(mp, dblock); + unsigned int numrecs; + unsigned int maxrecs; + + xfs_btree_init_block(mp, rblock, &xfs_rtrmapbt_ops, 0, 0, I_INO(ip)); + + rblock->bb_level = dblock->bb_level; + rblock->bb_numrecs = dblock->bb_numrecs; + numrecs = be16_to_cpu(dblock->bb_numrecs); + + if (be16_to_cpu(rblock->bb_level) > 0) { + maxrecs = xfs_rtrmapbt_droot_maxrecs(dblocklen, false); + fkp = xfs_rtrmap_droot_key_addr(dblock, 1); + tkp = xfs_rtrmap_key_addr(rblock, 1); + fpp = xfs_rtrmap_droot_ptr_addr(dblock, 1, maxrecs); + tpp = xfs_rtrmap_broot_ptr_addr(mp, rblock, 1, rblocklen); + memcpy(tkp, fkp, 2 * sizeof(*fkp) * numrecs); + memcpy(tpp, fpp, sizeof(*fpp) * numrecs); + } else { + frp = xfs_rtrmap_droot_rec_addr(dblock, 1); + trp = xfs_rtrmap_rec_addr(rblock, 1); + memcpy(trp, frp, sizeof(*frp) * numrecs); + } +} + +/* Load a realtime reverse mapping btree root in from disk. */ +int +xfs_iformat_rtrmap( + struct xfs_inode *ip, + struct xfs_dinode *dip) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_rtrmap_root *dfp = XFS_DFORK_PTR(dip, XFS_DATA_FORK); + struct xfs_btree_block *broot; + unsigned int numrecs; + unsigned int level; + int dsize; + + /* + * growfs must create the rtrmap inodes before adding a realtime volume + * to the filesystem, so we cannot use the rtrmapbt predicate here. + */ + if (!xfs_has_rmapbt(ip->i_mount)) { + xfs_inode_mark_sick(ip, XFS_SICK_INO_CORE); + return -EFSCORRUPTED; + } + + dsize = XFS_DFORK_SIZE(dip, mp, XFS_DATA_FORK); + numrecs = be16_to_cpu(dfp->bb_numrecs); + level = be16_to_cpu(dfp->bb_level); + + if (level > mp->m_rtrmap_maxlevels || + xfs_rtrmap_droot_space_calc(level, numrecs) > dsize) { + xfs_inode_mark_sick(ip, XFS_SICK_INO_CORE); + return -EFSCORRUPTED; + } + + broot = xfs_broot_alloc(xfs_ifork_ptr(ip, XFS_DATA_FORK), + xfs_rtrmap_broot_space_calc(mp, level, numrecs)); + if (broot) + xfs_rtrmapbt_from_disk(ip, dfp, dsize, broot); + return 0; +} + +/* Convert in-memory form of btree root to on-disk form. */ +void +xfs_rtrmapbt_to_disk( + struct xfs_mount *mp, + struct xfs_btree_block *rblock, + unsigned int rblocklen, + struct xfs_rtrmap_root *dblock, + unsigned int dblocklen) +{ + struct xfs_rmap_key *fkp; + __be64 *fpp; + struct xfs_rmap_key *tkp; + __be64 *tpp; + struct xfs_rmap_rec *frp; + struct xfs_rmap_rec *trp; + unsigned int numrecs; + unsigned int maxrecs; + + ASSERT(rblock->bb_magic == cpu_to_be32(XFS_RTRMAP_CRC_MAGIC)); + ASSERT(uuid_equal(&rblock->bb_u.l.bb_uuid, &mp->m_sb.sb_meta_uuid)); + ASSERT(rblock->bb_u.l.bb_blkno == cpu_to_be64(XFS_BUF_DADDR_NULL)); + ASSERT(rblock->bb_u.l.bb_leftsib == cpu_to_be64(NULLFSBLOCK)); + ASSERT(rblock->bb_u.l.bb_rightsib == cpu_to_be64(NULLFSBLOCK)); + + dblock->bb_level = rblock->bb_level; + dblock->bb_numrecs = rblock->bb_numrecs; + numrecs = be16_to_cpu(rblock->bb_numrecs); + + if (be16_to_cpu(rblock->bb_level) > 0) { + maxrecs = xfs_rtrmapbt_droot_maxrecs(dblocklen, false); + fkp = xfs_rtrmap_key_addr(rblock, 1); + tkp = xfs_rtrmap_droot_key_addr(dblock, 1); + fpp = xfs_rtrmap_broot_ptr_addr(mp, rblock, 1, rblocklen); + tpp = xfs_rtrmap_droot_ptr_addr(dblock, 1, maxrecs); + memcpy(tkp, fkp, 2 * sizeof(*fkp) * numrecs); + memcpy(tpp, fpp, sizeof(*fpp) * numrecs); + } else { + frp = xfs_rtrmap_rec_addr(rblock, 1); + trp = xfs_rtrmap_droot_rec_addr(dblock, 1); + memcpy(trp, frp, sizeof(*frp) * numrecs); + } +} + +/* Flush a realtime reverse mapping btree root out to disk. */ +void +xfs_iflush_rtrmap( + struct xfs_inode *ip, + struct xfs_dinode *dip) +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK); + struct xfs_rtrmap_root *dfp = XFS_DFORK_PTR(dip, XFS_DATA_FORK); + + ASSERT(ifp->if_broot != NULL); + ASSERT(ifp->if_broot_bytes > 0); + ASSERT(xfs_rtrmap_droot_space(ifp->if_broot) <= + xfs_inode_fork_size(ip, XFS_DATA_FORK)); + xfs_rtrmapbt_to_disk(ip->i_mount, ifp->if_broot, ifp->if_broot_bytes, + dfp, XFS_DFORK_SIZE(dip, ip->i_mount, XFS_DATA_FORK)); +} + +/* + * Create a realtime rmap btree inode. + */ +int +xfs_rtrmapbt_create( + struct xfs_rtgroup *rtg, + struct xfs_inode *ip, + struct xfs_trans *tp, + bool init) +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK); + struct xfs_mount *mp = ip->i_mount; + struct xfs_btree_block *broot; + + ifp->if_format = XFS_DINODE_FMT_META_BTREE; + ASSERT(ifp->if_broot_bytes == 0); + ASSERT(ifp->if_bytes == 0); + + /* Initialize the empty incore btree root. */ + broot = xfs_broot_realloc(ifp, xfs_rtrmap_broot_space_calc(mp, 0, 0)); + if (broot) + xfs_btree_init_block(mp, broot, &xfs_rtrmapbt_ops, 0, 0, + I_INO(ip)); + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE | XFS_ILOG_DBROOT); + + return 0; +} + +/* + * Initialize an rmap for a realtime superblock using the potentially updated + * rt geometry in the provided @mp. + */ +int +xfs_rtrmapbt_init_rtsb( + struct xfs_mount *mp, + struct xfs_rtgroup *rtg, + struct xfs_trans *tp) +{ + struct xfs_rmap_irec rmap = { + .rm_blockcount = mp->m_sb.sb_rextsize, + .rm_owner = XFS_RMAP_OWN_FS, + }; + struct xfs_btree_cur *cur; + int error; + + ASSERT(xfs_has_rtsb(mp)); + ASSERT(rtg_rgno(rtg) == 0); + + cur = xfs_rtrmapbt_init_cursor(tp, rtg); + error = xfs_rmap_map_raw(cur, &rmap); + xfs_btree_del_cursor(cur, error); + return error; +} + +/* + * Return the highest rgbno currently tracked by the rmap for this rtg. + */ +xfs_rgblock_t +xfs_rtrmap_highest_rgbno( + struct xfs_rtgroup *rtg) +{ + struct xfs_btree_block *block = rtg_rmap(rtg)->i_df.if_broot; + union xfs_btree_key key = {}; + struct xfs_btree_cur *cur; + + if (block->bb_numrecs == 0) + return NULLRGBLOCK; + cur = xfs_rtrmapbt_init_cursor(NULL, rtg); + xfs_btree_get_keys(cur, block, &key); + xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR); + return be32_to_cpu(key.__rmap_bigkey[1].rm_startblock); +} diff --git a/fs/xfs/libxfs/xfs_rtrmap_btree.h b/fs/xfs/libxfs/xfs_rtrmap_btree.h new file mode 100644 index 000000000..e328fd62a --- /dev/null +++ b/fs/xfs/libxfs/xfs_rtrmap_btree.h @@ -0,0 +1,212 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2018-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_RTRMAP_BTREE_H__ +#define __XFS_RTRMAP_BTREE_H__ + +struct xfs_buf; +struct xfs_btree_cur; +struct xfs_mount; +struct xbtree_ifakeroot; +struct xfs_rtgroup; +struct xfbtree; + +/* rmaps only exist on crc enabled filesystems */ +#define XFS_RTRMAP_BLOCK_LEN XFS_BTREE_LBLOCK_CRC_LEN + +struct xfs_btree_cur *xfs_rtrmapbt_init_cursor(struct xfs_trans *tp, + struct xfs_rtgroup *rtg); +struct xfs_btree_cur *xfs_rtrmapbt_stage_cursor(struct xfs_mount *mp, + struct xfs_rtgroup *rtg, struct xfs_inode *ip, + struct xbtree_ifakeroot *ifake); +void xfs_rtrmapbt_commit_staged_btree(struct xfs_btree_cur *cur, + struct xfs_trans *tp); +unsigned int xfs_rtrmapbt_maxrecs(struct xfs_mount *mp, unsigned int blocklen, + bool leaf); +void xfs_rtrmapbt_compute_maxlevels(struct xfs_mount *mp); +unsigned int xfs_rtrmapbt_droot_maxrecs(unsigned int blocklen, bool leaf); + +/* + * Addresses of records, keys, and pointers within an incore rtrmapbt block. + * + * (note that some of these may appear unused, but they are used in userspace) + */ +static inline struct xfs_rmap_rec * +xfs_rtrmap_rec_addr( + struct xfs_btree_block *block, + unsigned int index) +{ + return (struct xfs_rmap_rec *) + ((char *)block + XFS_RTRMAP_BLOCK_LEN + + (index - 1) * sizeof(struct xfs_rmap_rec)); +} + +static inline struct xfs_rmap_key * +xfs_rtrmap_key_addr( + struct xfs_btree_block *block, + unsigned int index) +{ + return (struct xfs_rmap_key *) + ((char *)block + XFS_RTRMAP_BLOCK_LEN + + (index - 1) * 2 * sizeof(struct xfs_rmap_key)); +} + +static inline struct xfs_rmap_key * +xfs_rtrmap_high_key_addr( + struct xfs_btree_block *block, + unsigned int index) +{ + return (struct xfs_rmap_key *) + ((char *)block + XFS_RTRMAP_BLOCK_LEN + + sizeof(struct xfs_rmap_key) + + (index - 1) * 2 * sizeof(struct xfs_rmap_key)); +} + +static inline xfs_rtrmap_ptr_t * +xfs_rtrmap_ptr_addr( + struct xfs_btree_block *block, + unsigned int index, + unsigned int maxrecs) +{ + return (xfs_rtrmap_ptr_t *) + ((char *)block + XFS_RTRMAP_BLOCK_LEN + + maxrecs * 2 * sizeof(struct xfs_rmap_key) + + (index - 1) * sizeof(xfs_rtrmap_ptr_t)); +} + +unsigned int xfs_rtrmapbt_maxlevels_ondisk(void); + +int __init xfs_rtrmapbt_init_cur_cache(void); +void xfs_rtrmapbt_destroy_cur_cache(void); + +xfs_filblks_t xfs_rtrmapbt_calc_reserves(struct xfs_mount *mp); + +/* Addresses of key, pointers, and records within an ondisk rtrmapbt block. */ + +static inline struct xfs_rmap_rec * +xfs_rtrmap_droot_rec_addr( + struct xfs_rtrmap_root *block, + unsigned int index) +{ + return (struct xfs_rmap_rec *) + ((char *)(block + 1) + + (index - 1) * sizeof(struct xfs_rmap_rec)); +} + +static inline struct xfs_rmap_key * +xfs_rtrmap_droot_key_addr( + struct xfs_rtrmap_root *block, + unsigned int index) +{ + return (struct xfs_rmap_key *) + ((char *)(block + 1) + + (index - 1) * 2 * sizeof(struct xfs_rmap_key)); +} + +static inline xfs_rtrmap_ptr_t * +xfs_rtrmap_droot_ptr_addr( + struct xfs_rtrmap_root *block, + unsigned int index, + unsigned int maxrecs) +{ + return (xfs_rtrmap_ptr_t *) + ((char *)(block + 1) + + maxrecs * 2 * sizeof(struct xfs_rmap_key) + + (index - 1) * sizeof(xfs_rtrmap_ptr_t)); +} + +/* + * Address of pointers within the incore btree root. + * + * These are to be used when we know the size of the block and + * we don't have a cursor. + */ +static inline xfs_rtrmap_ptr_t * +xfs_rtrmap_broot_ptr_addr( + struct xfs_mount *mp, + struct xfs_btree_block *bb, + unsigned int index, + unsigned int block_size) +{ + return xfs_rtrmap_ptr_addr(bb, index, + xfs_rtrmapbt_maxrecs(mp, block_size, false)); +} + +/* + * Compute the space required for the incore btree root containing the given + * number of records. + */ +static inline size_t +xfs_rtrmap_broot_space_calc( + struct xfs_mount *mp, + unsigned int level, + unsigned int nrecs) +{ + size_t sz = XFS_RTRMAP_BLOCK_LEN; + + if (level > 0) + return sz + nrecs * (2 * sizeof(struct xfs_rmap_key) + + sizeof(xfs_rtrmap_ptr_t)); + return sz + nrecs * sizeof(struct xfs_rmap_rec); +} + +/* + * Compute the space required for the incore btree root given the ondisk + * btree root block. + */ +static inline size_t +xfs_rtrmap_broot_space(struct xfs_mount *mp, struct xfs_rtrmap_root *bb) +{ + return xfs_rtrmap_broot_space_calc(mp, be16_to_cpu(bb->bb_level), + be16_to_cpu(bb->bb_numrecs)); +} + +/* Compute the space required for the ondisk root block. */ +static inline size_t +xfs_rtrmap_droot_space_calc( + unsigned int level, + unsigned int nrecs) +{ + size_t sz = sizeof(struct xfs_rtrmap_root); + + if (level > 0) + return sz + nrecs * (2 * sizeof(struct xfs_rmap_key) + + sizeof(xfs_rtrmap_ptr_t)); + return sz + nrecs * sizeof(struct xfs_rmap_rec); +} + +/* + * Compute the space required for the ondisk root block given an incore root + * block. + */ +static inline size_t +xfs_rtrmap_droot_space(struct xfs_btree_block *bb) +{ + return xfs_rtrmap_droot_space_calc(be16_to_cpu(bb->bb_level), + be16_to_cpu(bb->bb_numrecs)); +} + +int xfs_iformat_rtrmap(struct xfs_inode *ip, struct xfs_dinode *dip); +void xfs_rtrmapbt_to_disk(struct xfs_mount *mp, struct xfs_btree_block *rblock, + unsigned int rblocklen, struct xfs_rtrmap_root *dblock, + unsigned int dblocklen); +void xfs_iflush_rtrmap(struct xfs_inode *ip, struct xfs_dinode *dip); + +int xfs_rtrmapbt_create(struct xfs_rtgroup *rtg, struct xfs_inode *ip, + struct xfs_trans *tp, bool init); +int xfs_rtrmapbt_init_rtsb(struct xfs_mount *mp, struct xfs_rtgroup *rtg, + struct xfs_trans *tp); + +unsigned long long xfs_rtrmapbt_calc_size(struct xfs_mount *mp, + unsigned long long len); + +struct xfs_btree_cur *xfs_rtrmapbt_mem_cursor(struct xfs_rtgroup *rtg, + struct xfs_trans *tp, struct xfbtree *xfbtree); +int xfs_rtrmapbt_mem_init(struct xfs_mount *mp, struct xfbtree *xfbtree, + struct xfs_buftarg *btp, xfs_rgnumber_t rgno); + +xfs_rgblock_t xfs_rtrmap_highest_rgbno(struct xfs_rtgroup *rtg); + +#endif /* __XFS_RTRMAP_BTREE_H__ */ diff --git a/fs/xfs/libxfs/xfs_sb.c b/fs/xfs/libxfs/xfs_sb.c new file mode 100644 index 000000000..75f2a021e --- /dev/null +++ b/fs/xfs/libxfs/xfs_sb.c @@ -0,0 +1,1755 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_sb.h" +#include "xfs_mount.h" +#include "xfs_ialloc.h" +#include "xfs_alloc.h" +#include "xfs_error.h" +#include "xfs_trans.h" +#include "xfs_buf_item.h" +#include "xfs_bmap_btree.h" +#include "xfs_alloc_btree.h" +#include "xfs_log.h" +#include "xfs_rmap_btree.h" +#include "xfs_refcount_btree.h" +#include "xfs_da_format.h" +#include "xfs_health.h" +#include "xfs_ag.h" +#include "xfs_rtbitmap.h" +#include "xfs_exchrange.h" +#include "xfs_rtgroup.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_rtrefcount_btree.h" + +/* + * Physical superblock buffer manipulations. Shared with libxfs in userspace. + */ + +/* + * Check that all the V4 feature bits that the V5 filesystem format requires are + * correctly set. + */ +static bool +xfs_sb_validate_v5_features( + struct xfs_sb *sbp) +{ + /* We must not have any unknown V4 feature bits set */ + if (sbp->sb_versionnum & ~XFS_SB_VERSION_OKBITS) + return false; + + /* + * The CRC bit is considered an invalid V4 flag, so we have to add it + * manually to the OKBITS mask. + */ + if (sbp->sb_features2 & ~(XFS_SB_VERSION2_OKBITS | + XFS_SB_VERSION2_CRCBIT)) + return false; + + /* Now check all the required V4 feature flags are set. */ + +#define V5_VERS_FLAGS (XFS_SB_VERSION_NLINKBIT | \ + XFS_SB_VERSION_ALIGNBIT | \ + XFS_SB_VERSION_LOGV2BIT | \ + XFS_SB_VERSION_EXTFLGBIT | \ + XFS_SB_VERSION_DIRV2BIT | \ + XFS_SB_VERSION_MOREBITSBIT) + +#define V5_FEAT_FLAGS (XFS_SB_VERSION2_LAZYSBCOUNTBIT | \ + XFS_SB_VERSION2_ATTR2BIT | \ + XFS_SB_VERSION2_PROJID32BIT | \ + XFS_SB_VERSION2_CRCBIT) + + if ((sbp->sb_versionnum & V5_VERS_FLAGS) != V5_VERS_FLAGS) + return false; + if ((sbp->sb_features2 & V5_FEAT_FLAGS) != V5_FEAT_FLAGS) + return false; + return true; +} + +/* + * We current support XFS v5 formats with known features and v4 superblocks with + * at least V2 directories. + */ +bool +xfs_sb_good_version( + struct xfs_sb *sbp) +{ + /* + * All v5 filesystems are supported, but we must check that all the + * required v4 feature flags are enabled correctly as the code checks + * those flags and not for v5 support. + */ + if (xfs_sb_is_v5(sbp)) + return xfs_sb_validate_v5_features(sbp); + + /* versions prior to v4 are not supported */ + if (XFS_SB_VERSION_NUM(sbp) != XFS_SB_VERSION_4) + return false; + + /* We must not have any unknown v4 feature bits set */ + if ((sbp->sb_versionnum & ~XFS_SB_VERSION_OKBITS) || + ((sbp->sb_versionnum & XFS_SB_VERSION_MOREBITSBIT) && + (sbp->sb_features2 & ~XFS_SB_VERSION2_OKBITS))) + return false; + + /* V4 filesystems need v2 directories and unwritten extents */ + if (!(sbp->sb_versionnum & XFS_SB_VERSION_DIRV2BIT)) + return false; + if (!(sbp->sb_versionnum & XFS_SB_VERSION_EXTFLGBIT)) + return false; + + /* It's a supported v4 filesystem */ + return true; +} + +uint64_t +xfs_sb_version_to_features( + struct xfs_sb *sbp) +{ + uint64_t features = 0; + + /* optional V4 features */ + if (sbp->sb_rblocks > 0) + features |= XFS_FEAT_REALTIME; + if (sbp->sb_versionnum & XFS_SB_VERSION_NLINKBIT) + features |= XFS_FEAT_NLINK; + if (sbp->sb_versionnum & XFS_SB_VERSION_ATTRBIT) + features |= XFS_FEAT_ATTR; + if (sbp->sb_versionnum & XFS_SB_VERSION_QUOTABIT) + features |= XFS_FEAT_QUOTA; + if (sbp->sb_versionnum & XFS_SB_VERSION_ALIGNBIT) + features |= XFS_FEAT_ALIGN; + if (sbp->sb_versionnum & XFS_SB_VERSION_LOGV2BIT) + features |= XFS_FEAT_LOGV2; + if (sbp->sb_versionnum & XFS_SB_VERSION_DALIGNBIT) + features |= XFS_FEAT_DALIGN; + if (sbp->sb_versionnum & XFS_SB_VERSION_EXTFLGBIT) + features |= XFS_FEAT_EXTFLG; + if (sbp->sb_versionnum & XFS_SB_VERSION_SECTORBIT) + features |= XFS_FEAT_SECTOR; + if (sbp->sb_versionnum & XFS_SB_VERSION_BORGBIT) + features |= XFS_FEAT_ASCIICI; + if (sbp->sb_versionnum & XFS_SB_VERSION_MOREBITSBIT) { + if (sbp->sb_features2 & XFS_SB_VERSION2_LAZYSBCOUNTBIT) + features |= XFS_FEAT_LAZYSBCOUNT; + if (sbp->sb_features2 & XFS_SB_VERSION2_PROJID32BIT) + features |= XFS_FEAT_PROJID32; + if (sbp->sb_features2 & XFS_SB_VERSION2_FTYPE) + features |= XFS_FEAT_FTYPE; + } + + if (!xfs_sb_is_v5(sbp)) + return features; + + /* Always on V5 features */ + features |= XFS_FEAT_ALIGN | XFS_FEAT_LOGV2 | XFS_FEAT_EXTFLG | + XFS_FEAT_LAZYSBCOUNT | XFS_FEAT_PROJID32 | + XFS_FEAT_V3INODES | XFS_FEAT_CRC | XFS_FEAT_PQUOTINO; + + /* Optional V5 features */ + if (sbp->sb_features_ro_compat & XFS_SB_FEAT_RO_COMPAT_FINOBT) + features |= XFS_FEAT_FINOBT; + if (sbp->sb_features_ro_compat & XFS_SB_FEAT_RO_COMPAT_RMAPBT) + features |= XFS_FEAT_RMAPBT; + if (sbp->sb_features_ro_compat & XFS_SB_FEAT_RO_COMPAT_REFLINK) + features |= XFS_FEAT_REFLINK; + if (sbp->sb_features_ro_compat & XFS_SB_FEAT_RO_COMPAT_INOBTCNT) + features |= XFS_FEAT_INOBTCNT; + if (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_FTYPE) + features |= XFS_FEAT_FTYPE; + if (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_SPINODES) + features |= XFS_FEAT_SPINODES; + if (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_META_UUID) + features |= XFS_FEAT_META_UUID; + if (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_BIGTIME) + features |= XFS_FEAT_BIGTIME; + if (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_NEEDSREPAIR) + features |= XFS_FEAT_NEEDSREPAIR; + if (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_NREXT64) + features |= XFS_FEAT_NREXT64; + if (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_EXCHRANGE) + features |= XFS_FEAT_EXCHANGE_RANGE; + if (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_PARENT) + features |= XFS_FEAT_PARENT; + if (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_METADIR) + features |= XFS_FEAT_METADIR; + if (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_ZONED) + features |= XFS_FEAT_ZONED; + + return features; +} + +/* Check all the superblock fields we care about when reading one in. */ +STATIC int +xfs_validate_sb_read( + struct xfs_mount *mp, + struct xfs_sb *sbp) +{ + if (!xfs_sb_is_v5(sbp)) + return 0; + + /* + * Version 5 superblock feature mask validation. Reject combinations + * the kernel cannot support up front before checking anything else. + */ + if (xfs_sb_has_compat_feature(sbp, XFS_SB_FEAT_COMPAT_UNKNOWN)) { + xfs_warn(mp, +"Superblock has unknown compatible features (0x%x) enabled.", + (sbp->sb_features_compat & XFS_SB_FEAT_COMPAT_UNKNOWN)); + xfs_warn(mp, +"Using a more recent kernel is recommended."); + } + + if (xfs_sb_has_ro_compat_feature(sbp, XFS_SB_FEAT_RO_COMPAT_UNKNOWN)) { + xfs_alert(mp, +"Superblock has unknown read-only compatible features (0x%x) enabled.", + (sbp->sb_features_ro_compat & + XFS_SB_FEAT_RO_COMPAT_UNKNOWN)); + if (!xfs_is_readonly(mp)) { + xfs_warn(mp, +"Attempted to mount read-only compatible filesystem read-write."); + xfs_warn(mp, +"Filesystem can only be safely mounted read only."); + + return -EINVAL; + } + } + if (xfs_sb_has_incompat_feature(sbp, XFS_SB_FEAT_INCOMPAT_UNKNOWN)) { + xfs_warn(mp, +"Superblock has unknown incompatible features (0x%x) enabled.", + (sbp->sb_features_incompat & + XFS_SB_FEAT_INCOMPAT_UNKNOWN)); + xfs_warn(mp, +"Filesystem cannot be safely mounted by this kernel."); + return -EINVAL; + } + + return 0; +} + +/* Return the number of extents covered by a single rt bitmap file */ +static xfs_rtbxlen_t +xfs_extents_per_rbm( + struct xfs_sb *sbp) +{ + if (xfs_sb_is_v5(sbp) && + (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_METADIR)) + return sbp->sb_rgextents; + return sbp->sb_rextents; +} + +/* + * Return the payload size of a single rt bitmap block (without the metadata + * header if any). + */ +static inline unsigned int +xfs_rtbmblock_size( + struct xfs_sb *sbp) +{ + if (xfs_sb_is_v5(sbp) && + (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_METADIR)) + return sbp->sb_blocksize - sizeof(struct xfs_rtbuf_blkinfo); + return sbp->sb_blocksize; +} + +static uint64_t +xfs_expected_rbmblocks( + struct xfs_sb *sbp) +{ + if (xfs_sb_is_v5(sbp) && + (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_ZONED)) + return 0; + return howmany_64(xfs_extents_per_rbm(sbp), + NBBY * xfs_rtbmblock_size(sbp)); +} + +/* Validate the realtime geometry */ +bool +xfs_validate_rt_geometry( + struct xfs_sb *sbp) +{ + if (xfs_sb_is_v5(sbp) && + (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_ZONED)) { + if (sbp->sb_rextsize != 1) + return false; + } else { + if (sbp->sb_rextsize * sbp->sb_blocksize > XFS_MAX_RTEXTSIZE || + sbp->sb_rextsize * sbp->sb_blocksize < XFS_MIN_RTEXTSIZE) + return false; + } + + if (sbp->sb_rblocks == 0) { + if (sbp->sb_rextents != 0 || sbp->sb_rbmblocks != 0 || + sbp->sb_rextslog != 0 || sbp->sb_frextents != 0) + return false; + return true; + } + + if (sbp->sb_rextents == 0 || + sbp->sb_rextents != div_u64(sbp->sb_rblocks, sbp->sb_rextsize) || + sbp->sb_rextslog != xfs_compute_rextslog(sbp->sb_rextents) || + sbp->sb_rbmblocks != xfs_expected_rbmblocks(sbp)) + return false; + + if (xfs_sb_is_v5(sbp) && + (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_ZONED)) { + uint32_t mod; + + /* + * Zoned RT devices must be aligned to the RT group size, + * because garbage collection assumes that all zones have the + * same size to avoid insane complexity if that weren't the + * case. + */ + div_u64_rem(sbp->sb_rextents, sbp->sb_rgextents, &mod); + if (mod) + return false; + } + + return true; +} + +/* Check all the superblock fields we care about when writing one out. */ +STATIC int +xfs_validate_sb_write( + struct xfs_mount *mp, + struct xfs_buf *bp, + struct xfs_sb *sbp) +{ + /* + * Carry out additional sb summary counter sanity checks when we write + * the superblock. We skip this in the read validator because there + * could be newer superblocks in the log and if the values are garbage + * even after replay we'll recalculate them at the end of log mount. + * + * mkfs has traditionally written zeroed counters to inprogress and + * secondary superblocks, so allow this usage to continue because + * we never read counters from such superblocks. + */ + if (xfs_buf_daddr(bp) == XFS_SB_DADDR && !sbp->sb_inprogress && + (sbp->sb_fdblocks > sbp->sb_dblocks || + !xfs_verify_icount(mp, sbp->sb_icount) || + sbp->sb_ifree > sbp->sb_icount)) { + xfs_warn(mp, "SB summary counter sanity check failed"); + return -EFSCORRUPTED; + } + + if (!xfs_sb_is_v5(sbp)) + return 0; + + /* + * Version 5 superblock feature mask validation. Reject combinations + * the kernel cannot support since we checked for unsupported bits in + * the read verifier, which means that memory is corrupt. + */ + if (!xfs_is_readonly(mp) && + xfs_sb_has_ro_compat_feature(sbp, XFS_SB_FEAT_RO_COMPAT_UNKNOWN)) { + xfs_alert(mp, +"Corruption detected in superblock read-only compatible features (0x%x)!", + (sbp->sb_features_ro_compat & + XFS_SB_FEAT_RO_COMPAT_UNKNOWN)); + return -EFSCORRUPTED; + } + if (xfs_sb_has_incompat_feature(sbp, XFS_SB_FEAT_INCOMPAT_UNKNOWN)) { + xfs_warn(mp, +"Corruption detected in superblock incompatible features (0x%x)!", + (sbp->sb_features_incompat & + XFS_SB_FEAT_INCOMPAT_UNKNOWN)); + return -EFSCORRUPTED; + } + if (xfs_sb_has_incompat_log_feature(sbp, + XFS_SB_FEAT_INCOMPAT_LOG_UNKNOWN)) { + xfs_warn(mp, +"Corruption detected in superblock incompatible log features (0x%x)!", + (sbp->sb_features_log_incompat & + XFS_SB_FEAT_INCOMPAT_LOG_UNKNOWN)); + return -EFSCORRUPTED; + } + + /* + * We can't read verify the sb LSN because the read verifier is called + * before the log is allocated and processed. We know the log is set up + * before write verifier calls, so check it here. + */ + if (!xfs_log_check_lsn(mp, sbp->sb_lsn)) + return -EFSCORRUPTED; + + return 0; +} + +int +xfs_compute_rgblklog( + xfs_rtxlen_t rgextents, + xfs_rgblock_t rextsize) +{ + uint64_t rgblocks = (uint64_t)rgextents * rextsize; + + return xfs_highbit64(rgblocks - 1) + 1; +} + +static int +xfs_validate_sb_rtgroups( + struct xfs_mount *mp, + struct xfs_sb *sbp) +{ + uint64_t groups; + int rgblklog; + + if (sbp->sb_rextsize == 0) { + xfs_warn(mp, +"Realtime extent size must not be zero."); + return -EINVAL; + } + + if (sbp->sb_rgextents > XFS_MAX_RGBLOCKS / sbp->sb_rextsize) { + xfs_warn(mp, +"Realtime group size (%u) must be less than %u rt extents.", + sbp->sb_rgextents, + XFS_MAX_RGBLOCKS / sbp->sb_rextsize); + return -EINVAL; + } + + if (sbp->sb_rgextents < XFS_MIN_RGEXTENTS) { + xfs_warn(mp, +"Realtime group size (%u) must be at least %u rt extents.", + sbp->sb_rgextents, XFS_MIN_RGEXTENTS); + return -EINVAL; + } + + if (sbp->sb_rgcount > XFS_MAX_RGNUMBER) { + xfs_warn(mp, +"Realtime groups (%u) must be less than %u.", + sbp->sb_rgcount, XFS_MAX_RGNUMBER); + return -EINVAL; + } + + groups = howmany_64(sbp->sb_rextents, sbp->sb_rgextents); + if (groups != sbp->sb_rgcount) { + xfs_warn(mp, +"Realtime groups (%u) do not cover the entire rt section; need (%llu) groups.", + sbp->sb_rgcount, groups); + return -EINVAL; + } + + /* Exchange-range is required for fsr to work on realtime files */ + if (!(sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_EXCHRANGE)) { + xfs_warn(mp, +"Realtime groups feature requires exchange-range support."); + return -EINVAL; + } + + rgblklog = xfs_compute_rgblklog(sbp->sb_rgextents, sbp->sb_rextsize); + if (sbp->sb_rgblklog != rgblklog) { + xfs_warn(mp, +"Realtime group log (%d) does not match expected value (%d).", + sbp->sb_rgblklog, rgblklog); + return -EINVAL; + } + + return 0; +} + +static int +xfs_validate_sb_zoned( + struct xfs_mount *mp, + struct xfs_sb *sbp) +{ + if (sbp->sb_frextents != 0) { + xfs_warn(mp, +"sb_frextents must be zero for zoned file systems."); + return -EINVAL; + } + + if (sbp->sb_rtstart && sbp->sb_rtstart < sbp->sb_dblocks) { + xfs_warn(mp, +"sb_rtstart (%lld) overlaps sb_dblocks (%lld).", + sbp->sb_rtstart, sbp->sb_dblocks); + return -EINVAL; + } + + if (sbp->sb_rtreserved && sbp->sb_rtreserved >= sbp->sb_rblocks) { + xfs_warn(mp, +"sb_rtreserved (%lld) larger than sb_rblocks (%lld).", + sbp->sb_rtreserved, sbp->sb_rblocks); + return -EINVAL; + } + + return 0; +} + +/* Check the validity of the SB. */ +STATIC int +xfs_validate_sb_common( + struct xfs_mount *mp, + struct xfs_buf *bp, + struct xfs_sb *sbp) +{ + struct xfs_dsb *dsb = bp->b_addr; + uint32_t agcount = 0; + uint32_t rem; + bool has_dalign; + int error; + + if (!xfs_verify_magic(bp, dsb->sb_magicnum)) { + xfs_warn(mp, +"Superblock has bad magic number 0x%x. Not an XFS filesystem?", + be32_to_cpu(dsb->sb_magicnum)); + return -EWRONGFS; + } + + if (!xfs_sb_good_version(sbp)) { + xfs_warn(mp, +"Superblock has unknown features enabled or corrupted feature masks."); + return -EWRONGFS; + } + + /* + * Validate feature flags and state + */ + if (xfs_sb_is_v5(sbp)) { + if (sbp->sb_blocksize < XFS_MIN_CRC_BLOCKSIZE) { + xfs_notice(mp, +"Block size (%u bytes) too small for Version 5 superblock (minimum %d bytes)", + sbp->sb_blocksize, XFS_MIN_CRC_BLOCKSIZE); + return -EFSCORRUPTED; + } + + /* V5 has a separate project quota inode */ + if (sbp->sb_qflags & (XFS_OQUOTA_ENFD | XFS_OQUOTA_CHKD)) { + xfs_notice(mp, + "Version 5 of Super block has XFS_OQUOTA bits."); + return -EFSCORRUPTED; + } + + /* + * Full inode chunks must be aligned to inode chunk size when + * sparse inodes are enabled to support the sparse chunk + * allocation algorithm and prevent overlapping inode records. + */ + if (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_SPINODES) { + uint32_t align; + + align = XFS_INODES_PER_CHUNK * sbp->sb_inodesize + >> sbp->sb_blocklog; + if (sbp->sb_inoalignmt != align) { + xfs_warn(mp, +"Inode block alignment (%u) must match chunk size (%u) for sparse inodes.", + sbp->sb_inoalignmt, align); + return -EINVAL; + } + + if (sbp->sb_spino_align && + (sbp->sb_spino_align > sbp->sb_inoalignmt || + (sbp->sb_inoalignmt % sbp->sb_spino_align) != 0)) { + xfs_warn(mp, +"Sparse inode alignment (%u) is invalid, must be integer factor of (%u).", + sbp->sb_spino_align, + sbp->sb_inoalignmt); + return -EINVAL; + } + } else if (sbp->sb_spino_align) { + xfs_warn(mp, + "Sparse inode alignment (%u) should be zero.", + sbp->sb_spino_align); + return -EINVAL; + } + + if (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_METADIR) { + if (memchr_inv(sbp->sb_pad, 0, sizeof(sbp->sb_pad))) { + xfs_warn(mp, +"Metadir superblock padding fields must be zero."); + return -EINVAL; + } + + error = xfs_validate_sb_rtgroups(mp, sbp); + if (error) + return error; + } + if (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_ZONED) { + error = xfs_validate_sb_zoned(mp, sbp); + if (error) + return error; + } + } else if (sbp->sb_qflags & (XFS_PQUOTA_ENFD | XFS_GQUOTA_ENFD | + XFS_PQUOTA_CHKD | XFS_GQUOTA_CHKD)) { + xfs_notice(mp, +"Superblock earlier than Version 5 has XFS_{P|G}QUOTA_{ENFD|CHKD} bits."); + return -EFSCORRUPTED; + } + + if (unlikely( + sbp->sb_logstart == 0 && mp->m_logdev_targp == mp->m_ddev_targp)) { + xfs_warn(mp, + "filesystem is marked as having an external log; " + "specify logdev on the mount command line."); + return -EINVAL; + } + + if (unlikely( + sbp->sb_logstart != 0 && mp->m_logdev_targp != mp->m_ddev_targp)) { + xfs_warn(mp, + "filesystem is marked as having an internal log; " + "do not specify logdev on the mount command line."); + return -EINVAL; + } + + /* Compute agcount for this number of dblocks and agblocks */ + if (sbp->sb_agblocks) { + agcount = div_u64_rem(sbp->sb_dblocks, sbp->sb_agblocks, &rem); + if (rem) + agcount++; + } + + /* + * More sanity checking. Most of these were stolen directly from + * xfs_repair. + */ + if (unlikely( + sbp->sb_agcount <= 0 || + sbp->sb_sectsize < XFS_MIN_SECTORSIZE || + sbp->sb_sectsize > XFS_MAX_SECTORSIZE || + sbp->sb_sectlog < XFS_MIN_SECTORSIZE_LOG || + sbp->sb_sectlog > XFS_MAX_SECTORSIZE_LOG || + sbp->sb_sectsize != (1 << sbp->sb_sectlog) || + sbp->sb_blocksize < XFS_MIN_BLOCKSIZE || + sbp->sb_blocksize > XFS_MAX_BLOCKSIZE || + sbp->sb_blocklog < XFS_MIN_BLOCKSIZE_LOG || + sbp->sb_blocklog > XFS_MAX_BLOCKSIZE_LOG || + sbp->sb_blocksize != (1 << sbp->sb_blocklog) || + sbp->sb_dirblklog + sbp->sb_blocklog > XFS_MAX_BLOCKSIZE_LOG || + sbp->sb_inodesize < XFS_DINODE_MIN_SIZE || + sbp->sb_inodesize > XFS_DINODE_MAX_SIZE || + sbp->sb_inodelog < XFS_DINODE_MIN_LOG || + sbp->sb_inodelog > XFS_DINODE_MAX_LOG || + sbp->sb_inodesize != (1 << sbp->sb_inodelog) || + sbp->sb_inopblock != howmany(sbp->sb_blocksize,sbp->sb_inodesize) || + XFS_FSB_TO_B(mp, sbp->sb_agblocks) < XFS_MIN_AG_BYTES || + XFS_FSB_TO_B(mp, sbp->sb_agblocks) > XFS_MAX_AG_BYTES || + sbp->sb_agblklog != xfs_highbit32(sbp->sb_agblocks - 1) + 1 || + agcount == 0 || agcount != sbp->sb_agcount || + (sbp->sb_blocklog - sbp->sb_inodelog != sbp->sb_inopblog) || + (sbp->sb_rextsize * sbp->sb_blocksize > XFS_MAX_RTEXTSIZE) || + (sbp->sb_rextsize * sbp->sb_blocksize < XFS_MIN_RTEXTSIZE) || + (sbp->sb_imax_pct > 100 /* zero sb_imax_pct is valid */) || + sbp->sb_dblocks == 0 || + sbp->sb_dblocks > XFS_MAX_DBLOCKS(sbp) || + sbp->sb_dblocks < XFS_MIN_DBLOCKS(sbp) || + sbp->sb_shared_vn != 0)) { + xfs_notice(mp, "SB sanity check failed"); + return -EFSCORRUPTED; + } + + /* + * Logs that are too large are not supported at all. Reject them + * outright. Logs that are too small are tolerated on v4 filesystems, + * but we can only check that when mounting the log. Hence we skip + * those checks here. + */ + if (sbp->sb_logblocks > XFS_MAX_LOG_BLOCKS) { + xfs_notice(mp, + "Log size 0x%x blocks too large, maximum size is 0x%llx blocks", + sbp->sb_logblocks, XFS_MAX_LOG_BLOCKS); + return -EFSCORRUPTED; + } + + if (XFS_FSB_TO_B(mp, sbp->sb_logblocks) > XFS_MAX_LOG_BYTES) { + xfs_warn(mp, + "log size 0x%llx bytes too large, maximum size is 0x%llx bytes", + XFS_FSB_TO_B(mp, sbp->sb_logblocks), + XFS_MAX_LOG_BYTES); + return -EFSCORRUPTED; + } + + /* + * Do not allow filesystems with corrupted log sector or stripe units to + * be mounted. We cannot safely size the iclogs or write to the log if + * the log stripe unit is not valid. + */ + if (sbp->sb_versionnum & XFS_SB_VERSION_SECTORBIT) { + if (sbp->sb_logsectsize != (1U << sbp->sb_logsectlog)) { + xfs_notice(mp, + "log sector size in bytes/log2 (0x%x/0x%x) must match", + sbp->sb_logsectsize, 1U << sbp->sb_logsectlog); + return -EFSCORRUPTED; + } + } else if (sbp->sb_logsectsize || sbp->sb_logsectlog) { + xfs_notice(mp, + "log sector size in bytes/log2 (0x%x/0x%x) are not zero", + sbp->sb_logsectsize, sbp->sb_logsectlog); + return -EFSCORRUPTED; + } + + if (sbp->sb_logsunit > 1) { + if (sbp->sb_logsunit % sbp->sb_blocksize) { + xfs_notice(mp, + "log stripe unit 0x%x bytes must be a multiple of block size", + sbp->sb_logsunit); + return -EFSCORRUPTED; + } + if (sbp->sb_logsunit > XLOG_MAX_RECORD_BSIZE) { + xfs_notice(mp, + "log stripe unit 0x%x bytes over maximum size (0x%x bytes)", + sbp->sb_logsunit, XLOG_MAX_RECORD_BSIZE); + return -EFSCORRUPTED; + } + } + + if (!xfs_validate_rt_geometry(sbp)) { + xfs_notice(mp, + "realtime %sgeometry check failed", + sbp->sb_rblocks ? "" : "zeroed "); + return -EFSCORRUPTED; + } + + /* + * Either (sb_unit and !hasdalign) or (!sb_unit and hasdalign) + * would imply the image is corrupted. + */ + has_dalign = sbp->sb_versionnum & XFS_SB_VERSION_DALIGNBIT; + if (!!sbp->sb_unit ^ has_dalign) { + xfs_notice(mp, "SB stripe alignment sanity check failed"); + return -EFSCORRUPTED; + } + + if (!xfs_validate_stripe_geometry(mp, XFS_FSB_TO_B(mp, sbp->sb_unit), + XFS_FSB_TO_B(mp, sbp->sb_width), 0, + xfs_buf_daddr(bp) == XFS_SB_DADDR, false)) + return -EFSCORRUPTED; + + /* + * Currently only very few inode sizes are supported. + */ + switch (sbp->sb_inodesize) { + case 256: + case 512: + case 1024: + case 2048: + break; + default: + xfs_warn(mp, "inode size of %d bytes not supported", + sbp->sb_inodesize); + return -ENOSYS; + } + + return 0; +} + +void +xfs_sb_quota_from_disk(struct xfs_sb *sbp) +{ + if (xfs_sb_is_v5(sbp) && + (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_METADIR)) { + sbp->sb_uquotino = NULLFSINO; + sbp->sb_gquotino = NULLFSINO; + sbp->sb_pquotino = NULLFSINO; + return; + } + + /* + * older mkfs doesn't initialize quota inodes to NULLFSINO. This + * leads to in-core values having two different values for a quota + * inode to be invalid: 0 and NULLFSINO. Change it to a single value + * NULLFSINO. + * + * Note that this change affect only the in-core values. These + * values are not written back to disk unless any quota information + * is written to the disk. Even in that case, sb_pquotino field is + * not written to disk unless the superblock supports pquotino. + */ + if (sbp->sb_uquotino == 0) + sbp->sb_uquotino = NULLFSINO; + if (sbp->sb_gquotino == 0) + sbp->sb_gquotino = NULLFSINO; + if (sbp->sb_pquotino == 0) + sbp->sb_pquotino = NULLFSINO; + + /* + * We need to do these manipilations only if we are working + * with an older version of on-disk superblock. + */ + if (xfs_sb_is_v5(sbp)) + return; + + if (sbp->sb_qflags & XFS_OQUOTA_ENFD) + sbp->sb_qflags |= (sbp->sb_qflags & XFS_PQUOTA_ACCT) ? + XFS_PQUOTA_ENFD : XFS_GQUOTA_ENFD; + if (sbp->sb_qflags & XFS_OQUOTA_CHKD) + sbp->sb_qflags |= (sbp->sb_qflags & XFS_PQUOTA_ACCT) ? + XFS_PQUOTA_CHKD : XFS_GQUOTA_CHKD; + sbp->sb_qflags &= ~(XFS_OQUOTA_ENFD | XFS_OQUOTA_CHKD); + + if (sbp->sb_qflags & XFS_PQUOTA_ACCT && + sbp->sb_gquotino != NULLFSINO) { + /* + * In older version of superblock, on-disk superblock only + * has sb_gquotino, and in-core superblock has both sb_gquotino + * and sb_pquotino. But, only one of them is supported at any + * point of time. So, if PQUOTA is set in disk superblock, + * copy over sb_gquotino to sb_pquotino. The NULLFSINO test + * above is to make sure we don't do this twice and wipe them + * both out! + */ + sbp->sb_pquotino = sbp->sb_gquotino; + sbp->sb_gquotino = NULLFSINO; + } +} + +static void +__xfs_sb_from_disk( + struct xfs_sb *to, + struct xfs_dsb *from, + bool convert_xquota) +{ + to->sb_magicnum = be32_to_cpu(from->sb_magicnum); + to->sb_blocksize = be32_to_cpu(from->sb_blocksize); + to->sb_dblocks = be64_to_cpu(from->sb_dblocks); + to->sb_rblocks = be64_to_cpu(from->sb_rblocks); + to->sb_rextents = be64_to_cpu(from->sb_rextents); + memcpy(&to->sb_uuid, &from->sb_uuid, sizeof(to->sb_uuid)); + to->sb_logstart = be64_to_cpu(from->sb_logstart); + to->sb_rootino = be64_to_cpu(from->sb_rootino); + to->sb_rbmino = be64_to_cpu(from->sb_rbmino); + to->sb_rsumino = be64_to_cpu(from->sb_rsumino); + to->sb_rextsize = be32_to_cpu(from->sb_rextsize); + to->sb_agblocks = be32_to_cpu(from->sb_agblocks); + to->sb_agcount = be32_to_cpu(from->sb_agcount); + to->sb_rbmblocks = be32_to_cpu(from->sb_rbmblocks); + to->sb_logblocks = be32_to_cpu(from->sb_logblocks); + to->sb_versionnum = be16_to_cpu(from->sb_versionnum); + to->sb_sectsize = be16_to_cpu(from->sb_sectsize); + to->sb_inodesize = be16_to_cpu(from->sb_inodesize); + to->sb_inopblock = be16_to_cpu(from->sb_inopblock); + memcpy(&to->sb_fname, &from->sb_fname, sizeof(to->sb_fname)); + to->sb_blocklog = from->sb_blocklog; + to->sb_sectlog = from->sb_sectlog; + to->sb_inodelog = from->sb_inodelog; + to->sb_inopblog = from->sb_inopblog; + to->sb_agblklog = from->sb_agblklog; + to->sb_rextslog = from->sb_rextslog; + to->sb_inprogress = from->sb_inprogress; + to->sb_imax_pct = from->sb_imax_pct; + to->sb_icount = be64_to_cpu(from->sb_icount); + to->sb_ifree = be64_to_cpu(from->sb_ifree); + to->sb_fdblocks = be64_to_cpu(from->sb_fdblocks); + to->sb_frextents = be64_to_cpu(from->sb_frextents); + to->sb_uquotino = be64_to_cpu(from->sb_uquotino); + to->sb_gquotino = be64_to_cpu(from->sb_gquotino); + to->sb_qflags = be16_to_cpu(from->sb_qflags); + to->sb_flags = from->sb_flags; + to->sb_shared_vn = from->sb_shared_vn; + to->sb_inoalignmt = be32_to_cpu(from->sb_inoalignmt); + to->sb_unit = be32_to_cpu(from->sb_unit); + to->sb_width = be32_to_cpu(from->sb_width); + to->sb_dirblklog = from->sb_dirblklog; + to->sb_logsectlog = from->sb_logsectlog; + to->sb_logsectsize = be16_to_cpu(from->sb_logsectsize); + to->sb_logsunit = be32_to_cpu(from->sb_logsunit); + to->sb_features2 = be32_to_cpu(from->sb_features2); + to->sb_bad_features2 = be32_to_cpu(from->sb_bad_features2); + to->sb_features_compat = be32_to_cpu(from->sb_features_compat); + to->sb_features_ro_compat = be32_to_cpu(from->sb_features_ro_compat); + to->sb_features_incompat = be32_to_cpu(from->sb_features_incompat); + to->sb_features_log_incompat = + be32_to_cpu(from->sb_features_log_incompat); + /* crc is only used on disk, not in memory; just init to 0 here. */ + to->sb_crc = 0; + to->sb_spino_align = be32_to_cpu(from->sb_spino_align); + to->sb_pquotino = be64_to_cpu(from->sb_pquotino); + to->sb_lsn = be64_to_cpu(from->sb_lsn); + /* + * sb_meta_uuid is only on disk if it differs from sb_uuid and the + * feature flag is set; if not set we keep it only in memory. + */ + if (xfs_sb_is_v5(to) && + (to->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_META_UUID)) + uuid_copy(&to->sb_meta_uuid, &from->sb_meta_uuid); + else + uuid_copy(&to->sb_meta_uuid, &from->sb_uuid); + /* Convert on-disk flags to in-memory flags? */ + if (convert_xquota) + xfs_sb_quota_from_disk(to); + + if (to->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_METADIR) { + to->sb_metadirino = be64_to_cpu(from->sb_metadirino); + to->sb_rgblklog = from->sb_rgblklog; + memcpy(to->sb_pad, from->sb_pad, sizeof(to->sb_pad)); + to->sb_rgcount = be32_to_cpu(from->sb_rgcount); + to->sb_rgextents = be32_to_cpu(from->sb_rgextents); + to->sb_rbmino = NULLFSINO; + to->sb_rsumino = NULLFSINO; + } else { + to->sb_metadirino = NULLFSINO; + to->sb_rgcount = 1; + to->sb_rgextents = 0; + } + + if (to->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_ZONED) { + to->sb_rtstart = be64_to_cpu(from->sb_rtstart); + to->sb_rtreserved = be64_to_cpu(from->sb_rtreserved); + } else { + to->sb_rtstart = 0; + to->sb_rtreserved = 0; + } +} + +void +xfs_sb_from_disk( + struct xfs_sb *to, + struct xfs_dsb *from) +{ + __xfs_sb_from_disk(to, from, true); +} + +static void +xfs_sb_quota_to_disk( + struct xfs_dsb *to, + struct xfs_sb *from) +{ + uint16_t qflags = from->sb_qflags; + + if (xfs_sb_is_v5(from) && + (from->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_METADIR)) { + to->sb_qflags = cpu_to_be16(from->sb_qflags); + to->sb_uquotino = cpu_to_be64(0); + to->sb_gquotino = cpu_to_be64(0); + to->sb_pquotino = cpu_to_be64(0); + return; + } + + to->sb_uquotino = cpu_to_be64(from->sb_uquotino); + + /* + * The in-memory superblock quota state matches the v5 on-disk format so + * just write them out and return + */ + if (xfs_sb_is_v5(from)) { + to->sb_qflags = cpu_to_be16(from->sb_qflags); + to->sb_gquotino = cpu_to_be64(from->sb_gquotino); + to->sb_pquotino = cpu_to_be64(from->sb_pquotino); + return; + } + + /* + * For older superblocks (v4), the in-core version of sb_qflags do not + * have XFS_OQUOTA_* flags, whereas the on-disk version does. So, + * convert incore XFS_{PG}QUOTA_* flags to on-disk XFS_OQUOTA_* flags. + */ + qflags &= ~(XFS_PQUOTA_ENFD | XFS_PQUOTA_CHKD | + XFS_GQUOTA_ENFD | XFS_GQUOTA_CHKD); + + if (from->sb_qflags & + (XFS_PQUOTA_ENFD | XFS_GQUOTA_ENFD)) + qflags |= XFS_OQUOTA_ENFD; + if (from->sb_qflags & + (XFS_PQUOTA_CHKD | XFS_GQUOTA_CHKD)) + qflags |= XFS_OQUOTA_CHKD; + to->sb_qflags = cpu_to_be16(qflags); + + /* + * GQUOTINO and PQUOTINO cannot be used together in versions + * of superblock that do not have pquotino. from->sb_flags + * tells us which quota is active and should be copied to + * disk. If neither are active, we should NULL the inode. + * + * In all cases, the separate pquotino must remain 0 because it + * is beyond the "end" of the valid non-pquotino superblock. + */ + if (from->sb_qflags & XFS_GQUOTA_ACCT) + to->sb_gquotino = cpu_to_be64(from->sb_gquotino); + else if (from->sb_qflags & XFS_PQUOTA_ACCT) + to->sb_gquotino = cpu_to_be64(from->sb_pquotino); + else { + /* + * We can't rely on just the fields being logged to tell us + * that it is safe to write NULLFSINO - we should only do that + * if quotas are not actually enabled. Hence only write + * NULLFSINO if both in-core quota inodes are NULL. + */ + if (from->sb_gquotino == NULLFSINO && + from->sb_pquotino == NULLFSINO) + to->sb_gquotino = cpu_to_be64(NULLFSINO); + } + + to->sb_pquotino = 0; +} + +void +xfs_sb_to_disk( + struct xfs_dsb *to, + struct xfs_sb *from) +{ + xfs_sb_quota_to_disk(to, from); + + to->sb_magicnum = cpu_to_be32(from->sb_magicnum); + to->sb_blocksize = cpu_to_be32(from->sb_blocksize); + to->sb_dblocks = cpu_to_be64(from->sb_dblocks); + to->sb_rblocks = cpu_to_be64(from->sb_rblocks); + to->sb_rextents = cpu_to_be64(from->sb_rextents); + memcpy(&to->sb_uuid, &from->sb_uuid, sizeof(to->sb_uuid)); + to->sb_logstart = cpu_to_be64(from->sb_logstart); + to->sb_rootino = cpu_to_be64(from->sb_rootino); + to->sb_rbmino = cpu_to_be64(from->sb_rbmino); + to->sb_rsumino = cpu_to_be64(from->sb_rsumino); + to->sb_rextsize = cpu_to_be32(from->sb_rextsize); + to->sb_agblocks = cpu_to_be32(from->sb_agblocks); + to->sb_agcount = cpu_to_be32(from->sb_agcount); + to->sb_rbmblocks = cpu_to_be32(from->sb_rbmblocks); + to->sb_logblocks = cpu_to_be32(from->sb_logblocks); + to->sb_versionnum = cpu_to_be16(from->sb_versionnum); + to->sb_sectsize = cpu_to_be16(from->sb_sectsize); + to->sb_inodesize = cpu_to_be16(from->sb_inodesize); + to->sb_inopblock = cpu_to_be16(from->sb_inopblock); + memcpy(&to->sb_fname, &from->sb_fname, sizeof(to->sb_fname)); + to->sb_blocklog = from->sb_blocklog; + to->sb_sectlog = from->sb_sectlog; + to->sb_inodelog = from->sb_inodelog; + to->sb_inopblog = from->sb_inopblog; + to->sb_agblklog = from->sb_agblklog; + to->sb_rextslog = from->sb_rextslog; + to->sb_inprogress = from->sb_inprogress; + to->sb_imax_pct = from->sb_imax_pct; + to->sb_icount = cpu_to_be64(from->sb_icount); + to->sb_ifree = cpu_to_be64(from->sb_ifree); + to->sb_fdblocks = cpu_to_be64(from->sb_fdblocks); + to->sb_frextents = cpu_to_be64(from->sb_frextents); + + to->sb_flags = from->sb_flags; + to->sb_shared_vn = from->sb_shared_vn; + to->sb_inoalignmt = cpu_to_be32(from->sb_inoalignmt); + to->sb_unit = cpu_to_be32(from->sb_unit); + to->sb_width = cpu_to_be32(from->sb_width); + to->sb_dirblklog = from->sb_dirblklog; + to->sb_logsectlog = from->sb_logsectlog; + to->sb_logsectsize = cpu_to_be16(from->sb_logsectsize); + to->sb_logsunit = cpu_to_be32(from->sb_logsunit); + + /* + * We need to ensure that bad_features2 always matches features2. + * Hence we enforce that here rather than having to remember to do it + * everywhere else that updates features2. + */ + from->sb_bad_features2 = from->sb_features2; + to->sb_features2 = cpu_to_be32(from->sb_features2); + to->sb_bad_features2 = cpu_to_be32(from->sb_bad_features2); + + if (!xfs_sb_is_v5(from)) + return; + + to->sb_features_compat = cpu_to_be32(from->sb_features_compat); + to->sb_features_ro_compat = + cpu_to_be32(from->sb_features_ro_compat); + to->sb_features_incompat = + cpu_to_be32(from->sb_features_incompat); + to->sb_features_log_incompat = + cpu_to_be32(from->sb_features_log_incompat); + to->sb_spino_align = cpu_to_be32(from->sb_spino_align); + to->sb_lsn = cpu_to_be64(from->sb_lsn); + if (from->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_META_UUID) + uuid_copy(&to->sb_meta_uuid, &from->sb_meta_uuid); + + if (from->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_METADIR) { + to->sb_metadirino = cpu_to_be64(from->sb_metadirino); + to->sb_rgblklog = from->sb_rgblklog; + memset(to->sb_pad, 0, sizeof(to->sb_pad)); + to->sb_rgcount = cpu_to_be32(from->sb_rgcount); + to->sb_rgextents = cpu_to_be32(from->sb_rgextents); + to->sb_rbmino = cpu_to_be64(0); + to->sb_rsumino = cpu_to_be64(0); + } + + if (from->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_ZONED) { + to->sb_rtstart = cpu_to_be64(from->sb_rtstart); + to->sb_rtreserved = cpu_to_be64(from->sb_rtreserved); + } +} + +/* + * If the superblock has the CRC feature bit set or the CRC field is non-null, + * check that the CRC is valid. We check the CRC field is non-null because a + * single bit error could clear the feature bit and unused parts of the + * superblock are supposed to be zero. Hence a non-null crc field indicates that + * we've potentially lost a feature bit and we should check it anyway. + * + * However, past bugs (i.e. in growfs) left non-zeroed regions beyond the + * last field in V4 secondary superblocks. So for secondary superblocks, + * we are more forgiving, and ignore CRC failures if the primary doesn't + * indicate that the fs version is V5. + */ +static void +xfs_sb_read_verify( + struct xfs_buf *bp) +{ + struct xfs_sb sb; + struct xfs_mount *mp = bp->b_mount; + struct xfs_dsb *dsb = bp->b_addr; + int error; + + /* + * open code the version check to avoid needing to convert the entire + * superblock from disk order just to check the version number + */ + if (dsb->sb_magicnum == cpu_to_be32(XFS_SB_MAGIC) && + (((be16_to_cpu(dsb->sb_versionnum) & XFS_SB_VERSION_NUMBITS) == + XFS_SB_VERSION_5) || + dsb->sb_crc != 0)) { + + if (!xfs_buf_verify_cksum(bp, XFS_SB_CRC_OFF)) { + /* Only fail bad secondaries on a known V5 filesystem */ + if (xfs_buf_daddr(bp) == XFS_SB_DADDR || + xfs_has_crc(mp)) { + error = -EFSBADCRC; + goto out_error; + } + } + } + + /* + * Check all the superblock fields. Don't byteswap the xquota flags + * because _verify_common checks the on-disk values. + */ + __xfs_sb_from_disk(&sb, dsb, false); + error = xfs_validate_sb_read(mp, &sb); + if (error) + goto out_error; + error = xfs_validate_sb_common(mp, bp, &sb); + +out_error: + if (error == -EFSCORRUPTED || error == -EFSBADCRC) + xfs_verifier_error(bp, error, __this_address); + else if (error) + xfs_buf_ioerror(bp, error); +} + +/* + * We may be probed for a filesystem match, so we may not want to emit + * messages when the superblock buffer is not actually an XFS superblock. + * If we find an XFS superblock, then run a normal, noisy mount because we are + * really going to mount it and want to know about errors. + */ +static void +xfs_sb_quiet_read_verify( + struct xfs_buf *bp) +{ + struct xfs_dsb *dsb = bp->b_addr; + + if (dsb->sb_magicnum == cpu_to_be32(XFS_SB_MAGIC)) { + /* XFS filesystem, verify noisily! */ + xfs_sb_read_verify(bp); + return; + } + /* quietly fail */ + xfs_buf_ioerror(bp, -EWRONGFS); +} + +static void +xfs_sb_write_verify( + struct xfs_buf *bp) +{ + struct xfs_sb sb; + struct xfs_mount *mp = bp->b_mount; + struct xfs_buf_log_item *bip = bp->b_log_item; + struct xfs_dsb *dsb = bp->b_addr; + int error; + + /* + * Check all the superblock fields. Don't byteswap the xquota flags + * because _verify_common checks the on-disk values. + */ + __xfs_sb_from_disk(&sb, dsb, false); + error = xfs_validate_sb_common(mp, bp, &sb); + if (error) + goto out_error; + error = xfs_validate_sb_write(mp, bp, &sb); + if (error) + goto out_error; + + if (!xfs_sb_is_v5(&sb)) + return; + + if (bip) + dsb->sb_lsn = cpu_to_be64(bip->bli_item.li_lsn); + + xfs_buf_update_cksum(bp, XFS_SB_CRC_OFF); + return; + +out_error: + xfs_verifier_error(bp, error, __this_address); +} + +const struct xfs_buf_ops xfs_sb_buf_ops = { + .name = "xfs_sb", + .magic = { cpu_to_be32(XFS_SB_MAGIC), cpu_to_be32(XFS_SB_MAGIC) }, + .verify_read = xfs_sb_read_verify, + .verify_write = xfs_sb_write_verify, +}; + +const struct xfs_buf_ops xfs_sb_quiet_buf_ops = { + .name = "xfs_sb_quiet", + .magic = { cpu_to_be32(XFS_SB_MAGIC), cpu_to_be32(XFS_SB_MAGIC) }, + .verify_read = xfs_sb_quiet_read_verify, + .verify_write = xfs_sb_write_verify, +}; + +/* Compute cached rt geometry from the incore sb. */ +void +xfs_sb_mount_rextsize( + struct xfs_mount *mp, + struct xfs_sb *sbp) +{ + struct xfs_groups *rgs = &mp->m_groups[XG_TYPE_RTG]; + + mp->m_rtxblklog = log2_if_power2(sbp->sb_rextsize); + mp->m_rtxblkmask = mask64_if_power2(sbp->sb_rextsize); + + if (xfs_sb_is_v5(sbp) && + (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_METADIR)) { + rgs->blocks = sbp->sb_rgextents * sbp->sb_rextsize; + rgs->blklog = mp->m_sb.sb_rgblklog; + rgs->blkmask = xfs_mask32lo(mp->m_sb.sb_rgblklog); + rgs->start_fsb = mp->m_sb.sb_rtstart; + if (xfs_sb_has_incompat_feature(sbp, + XFS_SB_FEAT_INCOMPAT_ZONE_GAPS)) + rgs->has_daddr_gaps = true; + } else { + rgs->blocks = 0; + rgs->blklog = 0; + rgs->blkmask = (uint64_t)-1; + } +} + +/* Update incore sb rt extent size, then recompute the cached rt geometry. */ +void +xfs_mount_sb_set_rextsize( + struct xfs_mount *mp, + struct xfs_sb *sbp, + xfs_agblock_t rextsize) +{ + sbp->sb_rextsize = rextsize; + if (xfs_sb_is_v5(sbp) && + (sbp->sb_features_incompat & XFS_SB_FEAT_INCOMPAT_METADIR)) + sbp->sb_rgblklog = xfs_compute_rgblklog(sbp->sb_rgextents, + rextsize); + + xfs_sb_mount_rextsize(mp, sbp); +} + +/* + * xfs_mount_common + * + * Mount initialization code establishing various mount + * fields from the superblock associated with the given + * mount structure. + * + * Inode geometry are calculated in xfs_ialloc_setup_geometry. + */ +void +xfs_sb_mount_common( + struct xfs_mount *mp, + struct xfs_sb *sbp) +{ + struct xfs_groups *ags = &mp->m_groups[XG_TYPE_AG]; + + mp->m_agfrotor = 0; + atomic_set(&mp->m_agirotor, 0); + mp->m_maxagi = mp->m_sb.sb_agcount; + mp->m_blkbit_log = sbp->sb_blocklog + XFS_NBBYLOG; + mp->m_blkbb_log = sbp->sb_blocklog - BBSHIFT; + mp->m_sectbb_log = sbp->sb_sectlog - BBSHIFT; + mp->m_agno_log = xfs_highbit32(sbp->sb_agcount - 1) + 1; + mp->m_blockmask = sbp->sb_blocksize - 1; + mp->m_blockwsize = xfs_rtbmblock_size(sbp) >> XFS_WORDLOG; + mp->m_rtx_per_rbmblock = mp->m_blockwsize << XFS_NBWORDLOG; + + ags->blocks = mp->m_sb.sb_agblocks; + ags->blklog = mp->m_sb.sb_agblklog; + ags->blkmask = xfs_mask32lo(mp->m_sb.sb_agblklog); + + xfs_sb_mount_rextsize(mp, sbp); + + mp->m_alloc_mxr[0] = xfs_allocbt_maxrecs(mp, sbp->sb_blocksize, true); + mp->m_alloc_mxr[1] = xfs_allocbt_maxrecs(mp, sbp->sb_blocksize, false); + mp->m_alloc_mnr[0] = mp->m_alloc_mxr[0] / 2; + mp->m_alloc_mnr[1] = mp->m_alloc_mxr[1] / 2; + + mp->m_bmap_dmxr[0] = xfs_bmbt_maxrecs(mp, sbp->sb_blocksize, true); + mp->m_bmap_dmxr[1] = xfs_bmbt_maxrecs(mp, sbp->sb_blocksize, false); + mp->m_bmap_dmnr[0] = mp->m_bmap_dmxr[0] / 2; + mp->m_bmap_dmnr[1] = mp->m_bmap_dmxr[1] / 2; + + mp->m_rmap_mxr[0] = xfs_rmapbt_maxrecs(mp, sbp->sb_blocksize, true); + mp->m_rmap_mxr[1] = xfs_rmapbt_maxrecs(mp, sbp->sb_blocksize, false); + mp->m_rmap_mnr[0] = mp->m_rmap_mxr[0] / 2; + mp->m_rmap_mnr[1] = mp->m_rmap_mxr[1] / 2; + + mp->m_rtrmap_mxr[0] = xfs_rtrmapbt_maxrecs(mp, sbp->sb_blocksize, true); + mp->m_rtrmap_mxr[1] = xfs_rtrmapbt_maxrecs(mp, sbp->sb_blocksize, false); + mp->m_rtrmap_mnr[0] = mp->m_rtrmap_mxr[0] / 2; + mp->m_rtrmap_mnr[1] = mp->m_rtrmap_mxr[1] / 2; + + mp->m_refc_mxr[0] = xfs_refcountbt_maxrecs(mp, sbp->sb_blocksize, true); + mp->m_refc_mxr[1] = xfs_refcountbt_maxrecs(mp, sbp->sb_blocksize, false); + mp->m_refc_mnr[0] = mp->m_refc_mxr[0] / 2; + mp->m_refc_mnr[1] = mp->m_refc_mxr[1] / 2; + + mp->m_rtrefc_mxr[0] = xfs_rtrefcountbt_maxrecs(mp, sbp->sb_blocksize, + true); + mp->m_rtrefc_mxr[1] = xfs_rtrefcountbt_maxrecs(mp, sbp->sb_blocksize, + false); + mp->m_rtrefc_mnr[0] = mp->m_rtrefc_mxr[0] / 2; + mp->m_rtrefc_mnr[1] = mp->m_rtrefc_mxr[1] / 2; + + mp->m_bsize = XFS_FSB_TO_BB(mp, 1); + mp->m_alloc_set_aside = xfs_alloc_set_aside(mp); + mp->m_ag_max_usable = xfs_alloc_ag_max_usable(mp); +} + +/* + * xfs_log_sb() can be used to copy arbitrary changes to the in-core superblock + * into the superblock buffer to be logged. It does not provide the higher + * level of locking that is needed to protect the in-core superblock from + * concurrent access. + */ +void +xfs_log_sb( + struct xfs_trans *tp) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_buf *bp = xfs_trans_getsb(tp); + + /* + * Lazy sb counters don't update the in-core superblock so do that now. + * If this is at unmount, the counters will be exactly correct, but at + * any other time they will only be ballpark correct because of + * reservations that have been taken out percpu counters. If we have an + * unclean shutdown, this will be corrected by log recovery rebuilding + * the counters from the AGF block counts. + */ + if (xfs_has_lazysbcount(mp)) { + mp->m_sb.sb_icount = percpu_counter_sum_positive(&mp->m_icount); + mp->m_sb.sb_ifree = min_t(uint64_t, + percpu_counter_sum_positive(&mp->m_ifree), + mp->m_sb.sb_icount); + mp->m_sb.sb_fdblocks = xfs_sum_freecounter(mp, XC_FREE_BLOCKS); + } + + /* + * sb_frextents was added to the lazy sb counters when the rt groups + * feature was introduced. This counter can go negative due to the way + * we handle nearly-lockless reservations, so we must use the _positive + * variant here to avoid writing out nonsense frextents. + * + * RT groups are only supported on v5 file systems, which always + * have lazy SB counters. + */ + if (xfs_has_rtgroups(mp) && !xfs_has_zoned(mp)) { + mp->m_sb.sb_frextents = + xfs_sum_freecounter(mp, XC_FREE_RTEXTENTS); + } + + xfs_sb_to_disk(bp->b_addr, &mp->m_sb); + xfs_trans_buf_set_type(tp, bp, XFS_BLFT_SB_BUF); + xfs_trans_log_buf(tp, bp, 0, sizeof(struct xfs_dsb) - 1); +} + +/* + * xfs_sync_sb + * + * Sync the superblock to disk. + * + * Note that the caller is responsible for checking the frozen state of the + * filesystem. This procedure uses the non-blocking transaction allocator and + * thus will allow modifications to a frozen fs. This is required because this + * code can be called during the process of freezing where use of the high-level + * allocator would deadlock. + */ +int +xfs_sync_sb( + struct xfs_mount *mp, + bool wait) +{ + struct xfs_trans *tp; + int error; + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_sb, 0, 0, + XFS_TRANS_NO_WRITECOUNT, &tp); + if (error) + return error; + + xfs_log_sb(tp); + if (wait) + xfs_trans_set_sync(tp); + return xfs_trans_commit(tp); +} + +/* + * Update all the secondary superblocks to match the new state of the primary. + * Because we are completely overwriting all the existing fields in the + * secondary superblock buffers, there is no need to read them in from disk. + * Just get a new buffer, stamp it and write it. + * + * The sb buffers need to be cached here so that we serialise against other + * operations that access the secondary superblocks, but we don't want to keep + * them in memory once it is written so we mark it as a one-shot buffer. + */ +int +xfs_update_secondary_sbs( + struct xfs_mount *mp) +{ + struct xfs_perag *pag = NULL; + int saved_error = 0; + int error = 0; + LIST_HEAD (buffer_list); + + /* update secondary superblocks. */ + while ((pag = xfs_perag_next_from(mp, pag, 1))) { + struct xfs_buf *bp; + + error = xfs_buf_get(mp->m_ddev_targp, + XFS_AG_DADDR(mp, pag_agno(pag), XFS_SB_DADDR), + XFS_FSS_TO_BB(mp, 1), &bp); + /* + * If we get an error reading or writing alternate superblocks, + * continue. xfs_repair chooses the "best" superblock based + * on most matches; if we break early, we'll leave more + * superblocks un-updated than updated, and xfs_repair may + * pick them over the properly-updated primary. + */ + if (error) { + xfs_warn(mp, + "error allocating secondary superblock for ag %d", + pag_agno(pag)); + if (!saved_error) + saved_error = error; + continue; + } + + bp->b_ops = &xfs_sb_buf_ops; + xfs_buf_oneshot(bp); + xfs_buf_zero(bp, 0, BBTOB(bp->b_length)); + xfs_sb_to_disk(bp->b_addr, &mp->m_sb); + xfs_buf_delwri_queue(bp, &buffer_list); + xfs_buf_relse(bp); + + /* don't hold too many buffers at once */ + if (pag_agno(pag) % 16) + continue; + + error = xfs_buf_delwri_submit(&buffer_list); + if (error) { + xfs_warn(mp, + "write error %d updating a secondary superblock near ag %d", + error, pag_agno(pag)); + if (!saved_error) + saved_error = error; + continue; + } + } + error = xfs_buf_delwri_submit(&buffer_list); + if (error) + xfs_warn(mp, "error %d writing secondary superblocks", error); + return saved_error ? saved_error : error; +} + +/* + * Same behavior as xfs_sync_sb, except that it is always synchronous and it + * also writes the superblock buffer to disk sector 0 immediately. + */ +int +xfs_sync_sb_buf( + struct xfs_mount *mp, + bool update_rtsb) +{ + struct xfs_trans *tp; + struct xfs_buf *bp; + struct xfs_buf *rtsb_bp = NULL; + int error; + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_sb, 0, 0, 0, &tp); + if (error) + return error; + + bp = xfs_trans_getsb(tp); + xfs_log_sb(tp); + xfs_trans_bhold(tp, bp); + if (update_rtsb) { + rtsb_bp = xfs_log_rtsb(tp, bp); + if (rtsb_bp) + xfs_trans_bhold(tp, rtsb_bp); + } + xfs_trans_set_sync(tp); + error = xfs_trans_commit(tp); + if (error) + goto out; + /* + * write out the sb buffer to get the changes to disk + */ + error = xfs_bwrite(bp); + if (!error && rtsb_bp) + error = xfs_bwrite(rtsb_bp); +out: + if (rtsb_bp) + xfs_buf_relse(rtsb_bp); + xfs_buf_relse(bp); + return error; +} + +void +xfs_fs_geometry( + struct xfs_mount *mp, + struct xfs_fsop_geom *geo, + int struct_version) +{ + struct xfs_sb *sbp = &mp->m_sb; + + memset(geo, 0, sizeof(struct xfs_fsop_geom)); + + geo->blocksize = sbp->sb_blocksize; + geo->rtextsize = sbp->sb_rextsize; + geo->agblocks = sbp->sb_agblocks; + geo->agcount = sbp->sb_agcount; + geo->logblocks = sbp->sb_logblocks; + geo->sectsize = sbp->sb_sectsize; + geo->inodesize = sbp->sb_inodesize; + geo->imaxpct = sbp->sb_imax_pct; + geo->datablocks = sbp->sb_dblocks; + geo->rtblocks = sbp->sb_rblocks; + geo->rtextents = sbp->sb_rextents; + geo->logstart = sbp->sb_logstart; + BUILD_BUG_ON(sizeof(geo->uuid) != sizeof(sbp->sb_uuid)); + memcpy(geo->uuid, &sbp->sb_uuid, sizeof(sbp->sb_uuid)); + + if (struct_version < 2) + return; + + geo->sunit = sbp->sb_unit; + geo->swidth = sbp->sb_width; + + if (struct_version < 3) + return; + + geo->version = XFS_FSOP_GEOM_VERSION; + geo->flags = XFS_FSOP_GEOM_FLAGS_NLINK | + XFS_FSOP_GEOM_FLAGS_DIRV2 | + XFS_FSOP_GEOM_FLAGS_EXTFLG | + XFS_FSOP_GEOM_FLAGS_ATTR2; + if (xfs_has_attr(mp)) + geo->flags |= XFS_FSOP_GEOM_FLAGS_ATTR; + if (xfs_has_quota(mp)) + geo->flags |= XFS_FSOP_GEOM_FLAGS_QUOTA; + if (xfs_has_align(mp)) + geo->flags |= XFS_FSOP_GEOM_FLAGS_IALIGN; + if (xfs_has_dalign(mp)) + geo->flags |= XFS_FSOP_GEOM_FLAGS_DALIGN; + if (xfs_has_asciici(mp)) + geo->flags |= XFS_FSOP_GEOM_FLAGS_DIRV2CI; + if (xfs_has_lazysbcount(mp)) + geo->flags |= XFS_FSOP_GEOM_FLAGS_LAZYSB; + if (xfs_has_projid32(mp)) + geo->flags |= XFS_FSOP_GEOM_FLAGS_PROJID32; + if (xfs_has_crc(mp)) + geo->flags |= XFS_FSOP_GEOM_FLAGS_V5SB; + if (xfs_has_ftype(mp)) + geo->flags |= XFS_FSOP_GEOM_FLAGS_FTYPE; + if (xfs_has_finobt(mp)) + geo->flags |= XFS_FSOP_GEOM_FLAGS_FINOBT; + if (xfs_has_sparseinodes(mp)) + geo->flags |= XFS_FSOP_GEOM_FLAGS_SPINODES; + if (xfs_has_rmapbt(mp)) + geo->flags |= XFS_FSOP_GEOM_FLAGS_RMAPBT; + if (xfs_has_reflink(mp)) + geo->flags |= XFS_FSOP_GEOM_FLAGS_REFLINK; + if (xfs_has_bigtime(mp)) + geo->flags |= XFS_FSOP_GEOM_FLAGS_BIGTIME; + if (xfs_has_inobtcounts(mp)) + geo->flags |= XFS_FSOP_GEOM_FLAGS_INOBTCNT; + if (xfs_has_parent(mp)) + geo->flags |= XFS_FSOP_GEOM_FLAGS_PARENT; + if (xfs_has_sector(mp)) { + geo->flags |= XFS_FSOP_GEOM_FLAGS_SECTOR; + geo->logsectsize = sbp->sb_logsectsize; + } else { + geo->logsectsize = BBSIZE; + } + if (xfs_has_large_extent_counts(mp)) + geo->flags |= XFS_FSOP_GEOM_FLAGS_NREXT64; + if (xfs_has_exchange_range(mp)) + geo->flags |= XFS_FSOP_GEOM_FLAGS_EXCHANGE_RANGE; + if (xfs_has_metadir(mp)) + geo->flags |= XFS_FSOP_GEOM_FLAGS_METADIR; + if (xfs_has_zoned(mp)) + geo->flags |= XFS_FSOP_GEOM_FLAGS_ZONED; + geo->rtsectsize = sbp->sb_blocksize; + geo->dirblocksize = xfs_dir2_dirblock_bytes(sbp); + + if (struct_version < 4) + return; + + if (xfs_has_logv2(mp)) + geo->flags |= XFS_FSOP_GEOM_FLAGS_LOGV2; + + geo->logsunit = sbp->sb_logsunit; + + if (struct_version < 5) + return; + + geo->version = XFS_FSOP_GEOM_VERSION_V5; + + if (xfs_has_rtgroups(mp)) { + geo->rgcount = sbp->sb_rgcount; + geo->rgextents = sbp->sb_rgextents; + } + if (xfs_has_zoned(mp)) { + geo->rtstart = sbp->sb_rtstart; + geo->rtreserved = sbp->sb_rtreserved; + } +} + +/* Read a secondary superblock. */ +int +xfs_sb_read_secondary( + struct xfs_mount *mp, + struct xfs_trans *tp, + xfs_agnumber_t agno, + struct xfs_buf **bpp) +{ + struct xfs_buf *bp; + int error; + + ASSERT(agno != 0 && agno != NULLAGNUMBER); + error = xfs_trans_read_buf(mp, tp, mp->m_ddev_targp, + XFS_AG_DADDR(mp, agno, XFS_SB_BLOCK(mp)), + XFS_FSS_TO_BB(mp, 1), 0, &bp, &xfs_sb_buf_ops); + if (xfs_metadata_is_sick(error)) + xfs_agno_mark_sick(mp, agno, XFS_SICK_AG_SB); + if (error) + return error; + xfs_buf_set_ref(bp, XFS_SSB_REF); + *bpp = bp; + return 0; +} + +/* Get an uninitialised secondary superblock buffer. */ +int +xfs_sb_get_secondary( + struct xfs_mount *mp, + struct xfs_trans *tp, + xfs_agnumber_t agno, + struct xfs_buf **bpp) +{ + struct xfs_buf *bp; + int error; + + ASSERT(agno != 0 && agno != NULLAGNUMBER); + error = xfs_trans_get_buf(tp, mp->m_ddev_targp, + XFS_AG_DADDR(mp, agno, XFS_SB_BLOCK(mp)), + XFS_FSS_TO_BB(mp, 1), 0, &bp); + if (error) + return error; + bp->b_ops = &xfs_sb_buf_ops; + xfs_buf_oneshot(bp); + *bpp = bp; + return 0; +} + +/* + * sunit, swidth, sectorsize(optional with 0) should be all in bytes, so users + * won't be confused by values in error messages. This function returns false + * if the stripe geometry is invalid and the caller is unable to repair the + * stripe configuration later in the mount process. + */ +bool +xfs_validate_stripe_geometry( + struct xfs_mount *mp, + __s64 sunit, + __s64 swidth, + int sectorsize, + bool may_repair, + bool silent) +{ + if (swidth > INT_MAX) { + if (!silent) + xfs_notice(mp, +"stripe width (%lld) is too large", swidth); + goto check_override; + } + + if (sunit > swidth) { + if (!silent) + xfs_notice(mp, +"stripe unit (%lld) is larger than the stripe width (%lld)", sunit, swidth); + goto check_override; + } + + if (sectorsize && (int)sunit % sectorsize) { + if (!silent) + xfs_notice(mp, +"stripe unit (%lld) must be a multiple of the sector size (%d)", + sunit, sectorsize); + goto check_override; + } + + if (sunit && !swidth) { + if (!silent) + xfs_notice(mp, +"invalid stripe unit (%lld) and stripe width of 0", sunit); + goto check_override; + } + + if (!sunit && swidth) { + if (!silent) + xfs_notice(mp, +"invalid stripe width (%lld) and stripe unit of 0", swidth); + goto check_override; + } + + if (sunit && (int)swidth % (int)sunit) { + if (!silent) + xfs_notice(mp, +"stripe width (%lld) must be a multiple of the stripe unit (%lld)", + swidth, sunit); + goto check_override; + } + return true; + +check_override: + if (!may_repair) + return false; + /* + * During mount, mp->m_dalign will not be set unless the sunit mount + * option was set. If it was set, ignore the bad stripe alignment values + * and allow the validation and overwrite later in the mount process to + * attempt to overwrite the bad stripe alignment values with the values + * supplied by mount options. + */ + if (!mp->m_dalign) + return false; + if (!silent) + xfs_notice(mp, +"Will try to correct with specified mount options sunit (%d) and swidth (%d)", + BBTOB(mp->m_dalign), BBTOB(mp->m_swidth)); + return true; +} + +/* + * Compute the maximum level number of the realtime summary file, as defined by + * mkfs. The historic use of highbit32 on a 64-bit quantity prohibited correct + * use of rt volumes with more than 2^32 extents. + */ +uint8_t +xfs_compute_rextslog( + xfs_rtbxlen_t rtextents) +{ + if (!rtextents) + return 0; + return xfs_highbit64(rtextents); +} diff --git a/fs/xfs/libxfs/xfs_sb.h b/fs/xfs/libxfs/xfs_sb.h new file mode 100644 index 000000000..34d0dd374 --- /dev/null +++ b/fs/xfs/libxfs/xfs_sb.h @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_SB_H__ +#define __XFS_SB_H__ + +struct xfs_mount; +struct xfs_sb; +struct xfs_dsb; +struct xfs_trans; +struct xfs_fsop_geom; +struct xfs_perag; + +extern void xfs_log_sb(struct xfs_trans *tp); +extern int xfs_sync_sb(struct xfs_mount *mp, bool wait); +extern int xfs_sync_sb_buf(struct xfs_mount *mp, bool update_rtsb); +extern void xfs_sb_mount_common(struct xfs_mount *mp, struct xfs_sb *sbp); +void xfs_sb_mount_rextsize(struct xfs_mount *mp, struct xfs_sb *sbp); +void xfs_mount_sb_set_rextsize(struct xfs_mount *mp, + struct xfs_sb *sbp, xfs_agblock_t rextsize); +extern void xfs_sb_from_disk(struct xfs_sb *to, struct xfs_dsb *from); +extern void xfs_sb_to_disk(struct xfs_dsb *to, struct xfs_sb *from); +extern void xfs_sb_quota_from_disk(struct xfs_sb *sbp); +extern bool xfs_sb_good_version(struct xfs_sb *sbp); +extern uint64_t xfs_sb_version_to_features(struct xfs_sb *sbp); + +extern int xfs_update_secondary_sbs(struct xfs_mount *mp); + +#define XFS_FS_GEOM_MAX_STRUCT_VER (5) +extern void xfs_fs_geometry(struct xfs_mount *mp, struct xfs_fsop_geom *geo, + int struct_version); +extern int xfs_sb_read_secondary(struct xfs_mount *mp, + struct xfs_trans *tp, xfs_agnumber_t agno, + struct xfs_buf **bpp); +extern int xfs_sb_get_secondary(struct xfs_mount *mp, + struct xfs_trans *tp, xfs_agnumber_t agno, + struct xfs_buf **bpp); + +bool xfs_validate_stripe_geometry(struct xfs_mount *mp, + __s64 sunit, __s64 swidth, int sectorsize, bool may_repair, + bool silent); +bool xfs_validate_rt_geometry(struct xfs_sb *sbp); + +uint8_t xfs_compute_rextslog(xfs_rtbxlen_t rtextents); +int xfs_compute_rgblklog(xfs_rtxlen_t rgextents, xfs_rgblock_t rextsize); + +#endif /* __XFS_SB_H__ */ diff --git a/fs/xfs/libxfs/xfs_shared.h b/fs/xfs/libxfs/xfs_shared.h new file mode 100644 index 000000000..b1e0d9bc1 --- /dev/null +++ b/fs/xfs/libxfs/xfs_shared.h @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * Copyright (c) 2013 Red Hat, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_SHARED_H__ +#define __XFS_SHARED_H__ + +/* + * Definitions shared between kernel and userspace that don't fit into any other + * header file that is shared with userspace. + */ +struct xfs_ifork; +struct xfs_buf; +struct xfs_buf_ops; +struct xfs_mount; +struct xfs_trans; +struct xfs_inode; + +/* + * Buffer verifier operations are widely used, including userspace tools + */ +extern const struct xfs_buf_ops xfs_agf_buf_ops; +extern const struct xfs_buf_ops xfs_agfl_buf_ops; +extern const struct xfs_buf_ops xfs_agi_buf_ops; +extern const struct xfs_buf_ops xfs_attr3_leaf_buf_ops; +extern const struct xfs_buf_ops xfs_attr3_rmt_buf_ops; +extern const struct xfs_buf_ops xfs_bmbt_buf_ops; +extern const struct xfs_buf_ops xfs_bnobt_buf_ops; +extern const struct xfs_buf_ops xfs_cntbt_buf_ops; +extern const struct xfs_buf_ops xfs_da3_node_buf_ops; +extern const struct xfs_buf_ops xfs_dquot_buf_ops; +extern const struct xfs_buf_ops xfs_dquot_buf_ra_ops; +extern const struct xfs_buf_ops xfs_finobt_buf_ops; +extern const struct xfs_buf_ops xfs_inobt_buf_ops; +extern const struct xfs_buf_ops xfs_inode_buf_ops; +extern const struct xfs_buf_ops xfs_inode_buf_ra_ops; +extern const struct xfs_buf_ops xfs_refcountbt_buf_ops; +extern const struct xfs_buf_ops xfs_rmapbt_buf_ops; +extern const struct xfs_buf_ops xfs_rtbitmap_buf_ops; +extern const struct xfs_buf_ops xfs_rtsummary_buf_ops; +extern const struct xfs_buf_ops xfs_rtbuf_ops; +extern const struct xfs_buf_ops xfs_rtsb_buf_ops; +extern const struct xfs_buf_ops xfs_rtrefcountbt_buf_ops; +extern const struct xfs_buf_ops xfs_rtrmapbt_buf_ops; +extern const struct xfs_buf_ops xfs_sb_buf_ops; +extern const struct xfs_buf_ops xfs_sb_quiet_buf_ops; +extern const struct xfs_buf_ops xfs_symlink_buf_ops; + +/* btree ops */ +extern const struct xfs_btree_ops xfs_bnobt_ops; +extern const struct xfs_btree_ops xfs_cntbt_ops; +extern const struct xfs_btree_ops xfs_inobt_ops; +extern const struct xfs_btree_ops xfs_finobt_ops; +extern const struct xfs_btree_ops xfs_bmbt_ops; +extern const struct xfs_btree_ops xfs_refcountbt_ops; +extern const struct xfs_btree_ops xfs_rmapbt_ops; +extern const struct xfs_btree_ops xfs_rmapbt_mem_ops; +extern const struct xfs_btree_ops xfs_rtrmapbt_ops; +extern const struct xfs_btree_ops xfs_rtrmapbt_mem_ops; +extern const struct xfs_btree_ops xfs_rtrefcountbt_ops; + +static inline bool xfs_btree_is_bno(const struct xfs_btree_ops *ops) +{ + return ops == &xfs_bnobt_ops; +} + +static inline bool xfs_btree_is_cnt(const struct xfs_btree_ops *ops) +{ + return ops == &xfs_cntbt_ops; +} + +static inline bool xfs_btree_is_bmap(const struct xfs_btree_ops *ops) +{ + return ops == &xfs_bmbt_ops; +} + +static inline bool xfs_btree_is_ino(const struct xfs_btree_ops *ops) +{ + return ops == &xfs_inobt_ops; +} + +static inline bool xfs_btree_is_fino(const struct xfs_btree_ops *ops) +{ + return ops == &xfs_finobt_ops; +} + +static inline bool xfs_btree_is_refcount(const struct xfs_btree_ops *ops) +{ + return ops == &xfs_refcountbt_ops; +} + +static inline bool xfs_btree_is_rmap(const struct xfs_btree_ops *ops) +{ + return ops == &xfs_rmapbt_ops; +} + +#ifdef CONFIG_XFS_BTREE_IN_MEM +static inline bool xfs_btree_is_mem_rmap(const struct xfs_btree_ops *ops) +{ + return ops == &xfs_rmapbt_mem_ops; +} + +static inline bool xfs_btree_is_mem_rtrmap(const struct xfs_btree_ops *ops) +{ + return ops == &xfs_rtrmapbt_mem_ops; +} +#else +# define xfs_btree_is_mem_rmap(...) (false) +# define xfs_btree_is_mem_rtrmap(...) (false) +#endif + +static inline bool xfs_btree_is_rtrmap(const struct xfs_btree_ops *ops) +{ + return ops == &xfs_rtrmapbt_ops; +} + +static inline bool xfs_btree_is_rtrefcount(const struct xfs_btree_ops *ops) +{ + return ops == &xfs_rtrefcountbt_ops; +} + +/* log size calculation functions */ +int xfs_log_calc_unit_res(struct xfs_mount *mp, int unit_bytes); +int xfs_log_calc_minimum_size(struct xfs_mount *); + +struct xfs_trans_res; +void xfs_log_get_max_trans_res(struct xfs_mount *mp, + struct xfs_trans_res *max_resp); + +/* + * Values for t_flags. + */ +/* Transaction needs to be logged */ +#define XFS_TRANS_DIRTY (1u << 0) +/* Superblock is dirty and needs to be logged */ +#define XFS_TRANS_SB_DIRTY (1u << 1) +/* Transaction took a permanent log reservation */ +#define XFS_TRANS_PERM_LOG_RES (1u << 2) +/* Synchronous transaction commit needed */ +#define XFS_TRANS_SYNC (1u << 3) +/* Transaction can use reserve block pool */ +#define XFS_TRANS_RESERVE (1u << 4) +/* Transaction should avoid VFS level superblock write accounting */ +#define XFS_TRANS_NO_WRITECOUNT (1u << 5) +/* Transaction has freed blocks returned to it's reservation */ +#define XFS_TRANS_RES_FDBLKS (1u << 6) +/* Transaction contains an intent done log item */ +#define XFS_TRANS_HAS_INTENT_DONE (1u << 7) +/* + * LOWMODE is used by the allocator to activate the lowspace algorithm - when + * free space is running low the extent allocator may choose to allocate an + * extent from an AG without leaving sufficient space for a btree split when + * inserting the new extent. In this case the allocator will enable the + * lowspace algorithm which is supposed to allow further allocations (such as + * btree splits and newroots) to allocate from sequential AGs. In order to + * avoid locking AGs out of order the lowspace algorithm will start searching + * for free space from AG 0. If the correct transaction reservations have been + * made then this algorithm will eventually find all the space it needs. + */ +#define XFS_TRANS_LOWMODE (1u << 8) + +/* Transaction has locked the rtbitmap and rtsum inodes */ +#define XFS_TRANS_RTBITMAP_LOCKED (1u << 9) + +/* + * Field values for xfs_trans_mod_sb. + */ +#define XFS_TRANS_SB_ICOUNT 0x00000001 +#define XFS_TRANS_SB_IFREE 0x00000002 +#define XFS_TRANS_SB_FDBLOCKS 0x00000004 +#define XFS_TRANS_SB_RES_FDBLOCKS 0x00000008 +#define XFS_TRANS_SB_FREXTENTS 0x00000010 +#define XFS_TRANS_SB_RES_FREXTENTS 0x00000020 +#define XFS_TRANS_SB_DBLOCKS 0x00000040 +#define XFS_TRANS_SB_AGCOUNT 0x00000080 +#define XFS_TRANS_SB_IMAXPCT 0x00000100 +#define XFS_TRANS_SB_REXTSIZE 0x00000200 +#define XFS_TRANS_SB_RBMBLOCKS 0x00000400 +#define XFS_TRANS_SB_RBLOCKS 0x00000800 +#define XFS_TRANS_SB_REXTENTS 0x00001000 +#define XFS_TRANS_SB_REXTSLOG 0x00002000 +#define XFS_TRANS_SB_RGCOUNT 0x00004000 + +/* + * Here we centralize the specification of XFS meta-data buffer reference count + * values. This determines how hard the buffer cache tries to hold onto the + * buffer. + */ +#define XFS_AGF_REF 4 +#define XFS_AGI_REF 4 +#define XFS_AGFL_REF 3 +#define XFS_INO_BTREE_REF 3 +#define XFS_ALLOC_BTREE_REF 2 +#define XFS_BMAP_BTREE_REF 2 +#define XFS_RMAP_BTREE_REF 2 +#define XFS_DIR_BTREE_REF 2 +#define XFS_INO_REF 2 +#define XFS_ATTR_BTREE_REF 1 +#define XFS_DQUOT_REF 1 +#define XFS_REFC_BTREE_REF 1 +#define XFS_SSB_REF 0 + +/* Computed inode geometry for the filesystem. */ +struct xfs_ino_geometry { + /* Maximum inode count in this filesystem. */ + uint64_t maxicount; + + /* Actual inode cluster buffer size, in bytes. */ + unsigned int inode_cluster_size; + + /* + * Desired inode cluster buffer size, in bytes. This value is not + * rounded up to at least one filesystem block, which is necessary for + * the sole purpose of validating sb_spino_align. Runtime code must + * only ever use inode_cluster_size. + */ + unsigned int inode_cluster_size_raw; + + /* Inode cluster sizes, adjusted to be at least 1 fsb. */ + unsigned int inodes_per_cluster; + unsigned int blocks_per_cluster; + + /* Inode cluster alignment. */ + unsigned int cluster_align; + unsigned int cluster_align_inodes; + unsigned int inoalign_mask; /* mask sb_inoalignmt if used */ + + unsigned int inobt_mxr[2]; /* max inobt btree records */ + unsigned int inobt_mnr[2]; /* min inobt btree records */ + unsigned int inobt_maxlevels; /* max inobt btree levels. */ + + /* Size of inode allocations under normal operation. */ + unsigned int ialloc_inos; + unsigned int ialloc_blks; + + /* Minimum inode blocks for a sparse allocation. */ + unsigned int ialloc_min_blks; + + /* stripe unit inode alignment */ + unsigned int ialloc_align; + + unsigned int agino_log; /* #bits for agino in inum */ + + /* precomputed default inode attribute fork offset */ + unsigned int attr_fork_offset; + + /* precomputed value for di_flags2 */ + uint64_t new_diflags2; + + /* minimum folio order of a page cache allocation */ + unsigned int min_folio_order; + +}; + +#endif /* __XFS_SHARED_H__ */ diff --git a/fs/xfs/libxfs/xfs_symlink_remote.c b/fs/xfs/libxfs/xfs_symlink_remote.c new file mode 100644 index 000000000..b0dc3888b --- /dev/null +++ b/fs/xfs/libxfs/xfs_symlink_remote.c @@ -0,0 +1,432 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2006 Silicon Graphics, Inc. + * Copyright (c) 2012-2013 Red Hat, Inc. + * All rights reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_shared.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_error.h" +#include "xfs_trans.h" +#include "xfs_buf_item.h" +#include "xfs_log.h" +#include "xfs_symlink_remote.h" +#include "xfs_bit.h" +#include "xfs_bmap.h" +#include "xfs_health.h" + +/* + * Each contiguous block has a header, so it is not just a simple pathlen + * to FSB conversion. + */ +int +xfs_symlink_blocks( + struct xfs_mount *mp, + int pathlen) +{ + int buflen = XFS_SYMLINK_BUF_SPACE(mp, mp->m_sb.sb_blocksize); + + return (pathlen + buflen - 1) / buflen; +} + +int +xfs_symlink_hdr_set( + struct xfs_mount *mp, + xfs_ino_t ino, + uint32_t offset, + uint32_t size, + struct xfs_buf *bp) +{ + struct xfs_dsymlink_hdr *dsl = bp->b_addr; + + if (!xfs_has_crc(mp)) + return 0; + + memset(dsl, 0, sizeof(struct xfs_dsymlink_hdr)); + dsl->sl_magic = cpu_to_be32(XFS_SYMLINK_MAGIC); + dsl->sl_offset = cpu_to_be32(offset); + dsl->sl_bytes = cpu_to_be32(size); + uuid_copy(&dsl->sl_uuid, &mp->m_sb.sb_meta_uuid); + dsl->sl_owner = cpu_to_be64(ino); + dsl->sl_blkno = cpu_to_be64(xfs_buf_daddr(bp)); + bp->b_ops = &xfs_symlink_buf_ops; + + return sizeof(struct xfs_dsymlink_hdr); +} + +/* + * Checking of the symlink header is split into two parts. the verifier does + * CRC, location and bounds checking, the unpacking function checks the path + * parameters and owner. + */ +bool +xfs_symlink_hdr_ok( + xfs_ino_t ino, + uint32_t offset, + uint32_t size, + struct xfs_buf *bp) +{ + struct xfs_dsymlink_hdr *dsl = bp->b_addr; + + if (offset != be32_to_cpu(dsl->sl_offset)) + return false; + if (size != be32_to_cpu(dsl->sl_bytes)) + return false; + if (ino != be64_to_cpu(dsl->sl_owner)) + return false; + + /* ok */ + return true; +} + +static xfs_failaddr_t +xfs_symlink_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_dsymlink_hdr *dsl = bp->b_addr; + + /* no verification of non-crc buffers */ + if (!xfs_has_crc(mp)) + return NULL; + + if (!xfs_verify_magic(bp, dsl->sl_magic)) + return __this_address; + if (!uuid_equal(&dsl->sl_uuid, &mp->m_sb.sb_meta_uuid)) + return __this_address; + if (xfs_buf_daddr(bp) != be64_to_cpu(dsl->sl_blkno)) + return __this_address; + if (be32_to_cpu(dsl->sl_offset) + + be32_to_cpu(dsl->sl_bytes) >= XFS_SYMLINK_MAXLEN) + return __this_address; + if (dsl->sl_owner == 0) + return __this_address; + if (!xfs_log_check_lsn(mp, be64_to_cpu(dsl->sl_lsn))) + return __this_address; + + return NULL; +} + +static void +xfs_symlink_read_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + xfs_failaddr_t fa; + + /* no verification of non-crc buffers */ + if (!xfs_has_crc(mp)) + return; + + if (!xfs_buf_verify_cksum(bp, XFS_SYMLINK_CRC_OFF)) + xfs_verifier_error(bp, -EFSBADCRC, __this_address); + else { + fa = xfs_symlink_verify(bp); + if (fa) + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + } +} + +static void +xfs_symlink_write_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_buf_log_item *bip = bp->b_log_item; + xfs_failaddr_t fa; + + /* no verification of non-crc buffers */ + if (!xfs_has_crc(mp)) + return; + + fa = xfs_symlink_verify(bp); + if (fa) { + xfs_verifier_error(bp, -EFSCORRUPTED, fa); + return; + } + + if (bip) { + struct xfs_dsymlink_hdr *dsl = bp->b_addr; + dsl->sl_lsn = cpu_to_be64(bip->bli_item.li_lsn); + } + xfs_buf_update_cksum(bp, XFS_SYMLINK_CRC_OFF); +} + +const struct xfs_buf_ops xfs_symlink_buf_ops = { + .name = "xfs_symlink", + .magic = { 0, cpu_to_be32(XFS_SYMLINK_MAGIC) }, + .verify_read = xfs_symlink_read_verify, + .verify_write = xfs_symlink_write_verify, + .verify_struct = xfs_symlink_verify, +}; + +void +xfs_symlink_local_to_remote( + struct xfs_trans *tp, + struct xfs_buf *bp, + struct xfs_inode *ip, + struct xfs_ifork *ifp, + void *priv) +{ + struct xfs_mount *mp = ip->i_mount; + char *buf; + + xfs_trans_buf_set_type(tp, bp, XFS_BLFT_SYMLINK_BUF); + + if (!xfs_has_crc(mp)) { + bp->b_ops = NULL; + memcpy(bp->b_addr, ifp->if_data, ifp->if_bytes); + xfs_trans_log_buf(tp, bp, 0, ifp->if_bytes - 1); + return; + } + + /* + * As this symlink fits in an inode literal area, it must also fit in + * the smallest buffer the filesystem supports. + */ + ASSERT(BBTOB(bp->b_length) >= + ifp->if_bytes + sizeof(struct xfs_dsymlink_hdr)); + + bp->b_ops = &xfs_symlink_buf_ops; + + buf = bp->b_addr; + buf += xfs_symlink_hdr_set(mp, I_INO(ip), 0, ifp->if_bytes, bp); + memcpy(buf, ifp->if_data, ifp->if_bytes); + xfs_trans_log_buf(tp, bp, 0, sizeof(struct xfs_dsymlink_hdr) + + ifp->if_bytes - 1); +} + +/* + * Verify the in-memory consistency of an inline symlink data fork. This + * does not do on-disk format checks. + */ +xfs_failaddr_t +xfs_symlink_shortform_verify( + void *sfp, + int64_t size) +{ + char *endp = sfp + size; + + /* + * Zero length symlinks should never occur in memory as they are + * never allowed to exist on disk. + */ + if (!size) + return __this_address; + + /* No negative sizes or overly long symlink targets. */ + if (size < 0 || size > XFS_SYMLINK_MAXLEN) + return __this_address; + + /* No NULLs in the target either. */ + if (memchr(sfp, 0, size - 1)) + return __this_address; + + /* We /did/ null-terminate the buffer, right? */ + if (*endp != 0) + return __this_address; + return NULL; +} + +/* Read a remote symlink target into the buffer. */ +int +xfs_symlink_remote_read( + struct xfs_inode *ip, + char *link) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_bmbt_irec mval[XFS_SYMLINK_MAPS]; + struct xfs_buf *bp; + xfs_daddr_t d; + char *cur_chunk; + int pathlen = ip->i_disk_size; + int nmaps = XFS_SYMLINK_MAPS; + int byte_cnt; + int n; + int error = 0; + int fsblocks = 0; + int offset; + + xfs_assert_ilocked(ip, XFS_ILOCK_SHARED | XFS_ILOCK_EXCL); + + fsblocks = xfs_symlink_blocks(mp, pathlen); + error = xfs_bmapi_read(ip, 0, fsblocks, mval, &nmaps, 0); + if (error) + goto out; + + offset = 0; + for (n = 0; n < nmaps; n++) { + d = XFS_FSB_TO_DADDR(mp, mval[n].br_startblock); + byte_cnt = XFS_FSB_TO_B(mp, mval[n].br_blockcount); + + error = xfs_buf_read(mp->m_ddev_targp, d, BTOBB(byte_cnt), 0, + &bp, &xfs_symlink_buf_ops); + if (xfs_metadata_is_sick(error)) + xfs_inode_mark_sick(ip, XFS_SICK_INO_SYMLINK); + if (error) + return error; + byte_cnt = XFS_SYMLINK_BUF_SPACE(mp, byte_cnt); + if (pathlen < byte_cnt) + byte_cnt = pathlen; + + cur_chunk = bp->b_addr; + if (xfs_has_crc(mp)) { + if (!xfs_symlink_hdr_ok(I_INO(ip), offset, + byte_cnt, bp)) { + xfs_inode_mark_sick(ip, XFS_SICK_INO_SYMLINK); + error = -EFSCORRUPTED; + xfs_alert(mp, +"symlink header does not match required off/len/owner (0x%x/0x%x,0x%llx)", + offset, byte_cnt, I_INO(ip)); + xfs_buf_relse(bp); + goto out; + + } + + cur_chunk += sizeof(struct xfs_dsymlink_hdr); + } + + memcpy(link + offset, cur_chunk, byte_cnt); + + pathlen -= byte_cnt; + offset += byte_cnt; + + xfs_buf_relse(bp); + } + ASSERT(pathlen == 0); + + link[ip->i_disk_size] = '\0'; + error = 0; + + out: + return error; +} + +/* Write the symlink target into the inode. */ +int +xfs_symlink_write_target( + struct xfs_trans *tp, + struct xfs_inode *ip, + xfs_ino_t owner, + const char *target_path, + int pathlen, + xfs_fsblock_t fs_blocks, + uint resblks) +{ + struct xfs_bmbt_irec mval[XFS_SYMLINK_MAPS]; + struct xfs_mount *mp = tp->t_mountp; + const char *cur_chunk; + struct xfs_buf *bp; + xfs_daddr_t d; + int byte_cnt; + int nmaps; + int offset = 0; + int n; + int error; + + /* + * If the symlink will fit into the inode, write it inline. + */ + if (pathlen <= xfs_inode_data_fork_size(ip)) { + xfs_init_local_fork(ip, XFS_DATA_FORK, target_path, pathlen); + + ip->i_disk_size = pathlen; + ip->i_df.if_format = XFS_DINODE_FMT_LOCAL; + xfs_trans_log_inode(tp, ip, XFS_ILOG_DDATA | XFS_ILOG_CORE); + return 0; + } + + nmaps = XFS_SYMLINK_MAPS; + error = xfs_bmapi_write(tp, ip, 0, fs_blocks, XFS_BMAPI_METADATA, + resblks, mval, &nmaps); + if (error) + return error; + + ip->i_disk_size = pathlen; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + + cur_chunk = target_path; + offset = 0; + for (n = 0; n < nmaps; n++) { + char *buf; + + d = XFS_FSB_TO_DADDR(mp, mval[n].br_startblock); + byte_cnt = XFS_FSB_TO_B(mp, mval[n].br_blockcount); + error = xfs_trans_get_buf(tp, mp->m_ddev_targp, d, + BTOBB(byte_cnt), 0, &bp); + if (error) + return error; + bp->b_ops = &xfs_symlink_buf_ops; + + byte_cnt = XFS_SYMLINK_BUF_SPACE(mp, byte_cnt); + byte_cnt = min(byte_cnt, pathlen); + + buf = bp->b_addr; + buf += xfs_symlink_hdr_set(mp, owner, offset, byte_cnt, bp); + + memcpy(buf, cur_chunk, byte_cnt); + + cur_chunk += byte_cnt; + pathlen -= byte_cnt; + offset += byte_cnt; + + xfs_trans_buf_set_type(tp, bp, XFS_BLFT_SYMLINK_BUF); + xfs_trans_log_buf(tp, bp, 0, (buf + byte_cnt - 1) - + (char *)bp->b_addr); + } + ASSERT(pathlen == 0); + return 0; +} + +/* Remove all the blocks from a symlink and invalidate buffers. */ +int +xfs_symlink_remote_truncate( + struct xfs_trans *tp, + struct xfs_inode *ip) +{ + struct xfs_bmbt_irec mval[XFS_SYMLINK_MAPS]; + struct xfs_mount *mp = tp->t_mountp; + struct xfs_buf *bp; + int nmaps = XFS_SYMLINK_MAPS; + int done = 0; + int i; + int error; + + /* Read mappings and invalidate buffers. */ + error = xfs_bmapi_read(ip, 0, XFS_MAX_FILEOFF, mval, &nmaps, 0); + if (error) + return error; + + for (i = 0; i < nmaps; i++) { + if (!xfs_bmap_is_real_extent(&mval[i])) + break; + + error = xfs_trans_get_buf(tp, mp->m_ddev_targp, + XFS_FSB_TO_DADDR(mp, mval[i].br_startblock), + XFS_FSB_TO_BB(mp, mval[i].br_blockcount), 0, + &bp); + if (error) + return error; + + xfs_trans_binval(tp, bp); + } + + /* Unmap the remote blocks. */ + error = xfs_bunmapi(tp, ip, 0, XFS_MAX_FILEOFF, 0, nmaps, &done); + if (error) + return error; + if (!done) { + ASSERT(done); + xfs_inode_mark_sick(ip, XFS_SICK_INO_SYMLINK); + return -EFSCORRUPTED; + } + + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + return 0; +} diff --git a/fs/xfs/libxfs/xfs_symlink_remote.h b/fs/xfs/libxfs/xfs_symlink_remote.h new file mode 100644 index 000000000..c1672fe1f --- /dev/null +++ b/fs/xfs/libxfs/xfs_symlink_remote.h @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * Copyright (c) 2013 Red Hat, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_SYMLINK_REMOTE_H +#define __XFS_SYMLINK_REMOTE_H + +/* + * Symlink decoding/encoding functions + */ +int xfs_symlink_blocks(struct xfs_mount *mp, int pathlen); +int xfs_symlink_hdr_set(struct xfs_mount *mp, xfs_ino_t ino, uint32_t offset, + uint32_t size, struct xfs_buf *bp); +bool xfs_symlink_hdr_ok(xfs_ino_t ino, uint32_t offset, + uint32_t size, struct xfs_buf *bp); +void xfs_symlink_local_to_remote(struct xfs_trans *tp, struct xfs_buf *bp, + struct xfs_inode *ip, struct xfs_ifork *ifp, + void *priv); +xfs_failaddr_t xfs_symlink_shortform_verify(void *sfp, int64_t size); +int xfs_symlink_remote_read(struct xfs_inode *ip, char *link); +int xfs_symlink_write_target(struct xfs_trans *tp, struct xfs_inode *ip, + xfs_ino_t owner, const char *target_path, int pathlen, + xfs_fsblock_t fs_blocks, uint resblks); +int xfs_symlink_remote_truncate(struct xfs_trans *tp, struct xfs_inode *ip); + +#endif /* __XFS_SYMLINK_REMOTE_H */ diff --git a/fs/xfs/libxfs/xfs_trans_inode.c b/fs/xfs/libxfs/xfs_trans_inode.c new file mode 100644 index 000000000..1a0fdcbf3 --- /dev/null +++ b/fs/xfs/libxfs/xfs_trans_inode.c @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_trans_priv.h" +#include "xfs_inode_item.h" + +#include + +/* + * Add a locked inode to the transaction. + * + * The inode must be locked, and it cannot be associated with any transaction. + * If lock_flags is non-zero the inode will be unlocked on transaction commit. + */ +void +xfs_trans_ijoin( + struct xfs_trans *tp, + struct xfs_inode *ip, + uint lock_flags) +{ + struct xfs_inode_log_item *iip; + + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + if (ip->i_itemp == NULL) + xfs_inode_item_init(ip, ip->i_mount); + iip = ip->i_itemp; + + ASSERT(iip->ili_lock_flags == 0); + iip->ili_lock_flags = lock_flags; + ASSERT(!xfs_iflags_test(ip, XFS_ISTALE)); + + /* Reset the per-tx dirty context and add the item to the tx. */ + iip->ili_dirty_flags = 0; + xfs_trans_add_item(tp, &iip->ili_item); +} + +/* + * Transactional inode timestamp update. Requires the inode to be locked and + * joined to the transaction supplied. Relies on the transaction subsystem to + * track dirty state and update/writeback the inode accordingly. + */ +void +xfs_trans_ichgtime( + struct xfs_trans *tp, + struct xfs_inode *ip, + int flags) +{ + struct inode *inode = VFS_I(ip); + struct timespec64 tv; + + ASSERT(tp); + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + + /* If the mtime changes, then ctime must also change */ + ASSERT(flags & XFS_ICHGTIME_CHG); + + tv = inode_set_ctime_current(inode); + if (flags & XFS_ICHGTIME_MOD) + inode_set_mtime_to_ts(inode, tv); + if (flags & XFS_ICHGTIME_ACCESS) + inode_set_atime_to_ts(inode, tv); + if (flags & XFS_ICHGTIME_CREATE) + ip->i_crtime = tv; +} + +/* + * This is called to mark the fields indicated in fieldmask as needing to be + * logged when the transaction is committed. The inode must already be + * associated with the given transaction. All we do here is record where the + * inode was dirtied and mark the transaction and inode log item dirty; + * everything else is done in the ->precommit log item operation after the + * changes in the transaction have been completed. + */ +void +xfs_trans_log_inode( + struct xfs_trans *tp, + struct xfs_inode *ip, + uint flags) +{ + struct xfs_inode_log_item *iip = ip->i_itemp; + struct inode *inode = VFS_I(ip); + + ASSERT(iip); + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + ASSERT(!xfs_iflags_test(ip, XFS_ISTALE)); + + tp->t_flags |= XFS_TRANS_DIRTY; + + /* + * First time we log the inode in a transaction, bump the inode change + * counter if it is configured for this to occur. While we have the + * inode locked exclusively for metadata modification, we can usually + * avoid setting XFS_ILOG_CORE if no one has queried the value since + * the last time it was incremented. If we have XFS_ILOG_CORE already + * set however, then go ahead and bump the i_version counter + * unconditionally. + */ + if (!test_and_set_bit(XFS_LI_DIRTY, &iip->ili_item.li_flags)) { + if (IS_I_VERSION(inode) && + inode_maybe_inc_iversion(inode, flags & XFS_ILOG_CORE)) + flags |= XFS_ILOG_IVERSION; + } + + iip->ili_dirty_flags |= flags; +} + +int +xfs_trans_roll_inode( + struct xfs_trans **tpp, + struct xfs_inode *ip) +{ + int error; + + xfs_trans_log_inode(*tpp, ip, XFS_ILOG_CORE); + error = xfs_trans_roll(tpp); + if (!error) + xfs_trans_ijoin(*tpp, ip, 0); + return error; +} diff --git a/fs/xfs/libxfs/xfs_trans_resv.c b/fs/xfs/libxfs/xfs_trans_resv.c new file mode 100644 index 000000000..3151e97ca --- /dev/null +++ b/fs/xfs/libxfs/xfs_trans_resv.c @@ -0,0 +1,1559 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2003,2005 Silicon Graphics, Inc. + * Copyright (C) 2010 Red Hat, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_inode.h" +#include "xfs_bmap_btree.h" +#include "xfs_quota.h" +#include "xfs_trans.h" +#include "xfs_qm.h" +#include "xfs_trans_space.h" +#include "xfs_rtbitmap.h" +#include "xfs_attr_item.h" +#include "xfs_log.h" +#include "xfs_defer.h" +#include "xfs_bmap_item.h" +#include "xfs_extfree_item.h" +#include "xfs_rmap_item.h" +#include "xfs_refcount_item.h" +#include "xfs_trace.h" + +#define _ALLOC true +#define _FREE false + +/* + * A buffer has a format structure overhead in the log in addition + * to the data, so we need to take this into account when reserving + * space in a transaction for a buffer. Round the space required up + * to a multiple of 128 bytes so that we don't change the historical + * reservation that has been used for this overhead. + */ +STATIC uint +xfs_buf_log_overhead(void) +{ + return round_up(sizeof(struct xlog_op_header) + + sizeof(struct xfs_buf_log_format), 128); +} + +/* + * Calculate out transaction log reservation per item in bytes. + * + * The nbufs argument is used to indicate the number of items that + * will be changed in a transaction. size is used to tell how many + * bytes should be reserved per item. + */ +STATIC uint +xfs_calc_buf_res( + uint nbufs, + uint size) +{ + return nbufs * (size + xfs_buf_log_overhead()); +} + +/* + * Per-extent log reservation for the btree changes involved in freeing or + * allocating an extent. In classic XFS there were two trees that will be + * modified (bnobt + cntbt). With rmap enabled, there are three trees + * (rmapbt). The number of blocks reserved is based on the formula: + * + * num trees * ((2 blocks/level * max depth) - 1) + * + * Keep in mind that max depth is calculated separately for each type of tree. + */ +uint +xfs_allocfree_block_count( + struct xfs_mount *mp, + uint num_ops) +{ + uint blocks; + + blocks = num_ops * 2 * (2 * mp->m_alloc_maxlevels - 1); + if (xfs_has_rmapbt(mp)) + blocks += num_ops * (2 * mp->m_rmap_maxlevels - 1); + + return blocks; +} + +/* + * Per-extent log reservation for refcount btree changes. These are never done + * in the same transaction as an allocation or a free, so we compute them + * separately. + */ +static unsigned int +xfs_refcountbt_block_count( + struct xfs_mount *mp, + unsigned int num_ops) +{ + return num_ops * (2 * mp->m_refc_maxlevels - 1); +} + +static unsigned int +xfs_rtrefcountbt_block_count( + struct xfs_mount *mp, + unsigned int num_ops) +{ + return num_ops * (2 * mp->m_rtrefc_maxlevels - 1); +} + +/* + * Logging inodes is really tricksy. They are logged in memory format, + * which means that what we write into the log doesn't directly translate into + * the amount of space they use on disk. + * + * Case in point - btree format forks in memory format use more space than the + * on-disk format. In memory, the buffer contains a normal btree block header so + * the btree code can treat it as though it is just another generic buffer. + * However, when we write it to the inode fork, we don't write all of this + * header as it isn't needed. e.g. the root is only ever in the inode, so + * there's no need for sibling pointers which would waste 16 bytes of space. + * + * Hence when we have an inode with a maximally sized btree format fork, then + * amount of information we actually log is greater than the size of the inode + * on disk. Hence we need an inode reservation function that calculates all this + * correctly. So, we log: + * + * - 4 log op headers for object + * - for the ilf, the inode core and 2 forks + * - inode log format object + * - the inode core + * - two inode forks containing bmap btree root blocks. + * - the btree data contained by both forks will fit into the inode size, + * hence when combined with the inode core above, we have a total of the + * actual inode size. + * - the BMBT headers need to be accounted separately, as they are + * additional to the records and pointers that fit inside the inode + * forks. + */ +STATIC uint +xfs_calc_inode_res( + struct xfs_mount *mp, + uint ninodes) +{ + return ninodes * + (4 * sizeof(struct xlog_op_header) + + sizeof(struct xfs_inode_log_format) + + mp->m_sb.sb_inodesize + + 2 * xfs_bmbt_block_len(mp)); +} + +/* + * Inode btree record insertion/removal modifies the inode btree and free space + * btrees (since the inobt does not use the agfl). This requires the following + * reservation: + * + * the inode btree: max depth * blocksize + * the allocation btrees: 2 trees * (max depth - 1) * block size + * + * The caller must account for SB and AG header modifications, etc. + */ +STATIC uint +xfs_calc_inobt_res( + struct xfs_mount *mp) +{ + return xfs_calc_buf_res(M_IGEO(mp)->inobt_maxlevels, + XFS_FSB_TO_B(mp, 1)) + + xfs_calc_buf_res(xfs_allocfree_block_count(mp, 1), + XFS_FSB_TO_B(mp, 1)); +} + +/* + * The free inode btree is a conditional feature. The behavior differs slightly + * from that of the traditional inode btree in that the finobt tracks records + * for inode chunks with at least one free inode. A record can be removed from + * the tree during individual inode allocation. Therefore the finobt + * reservation is unconditional for both the inode chunk allocation and + * individual inode allocation (modify) cases. + * + * Behavior aside, the reservation for finobt modification is equivalent to the + * traditional inobt: cover a full finobt shape change plus block allocation. + */ +STATIC uint +xfs_calc_finobt_res( + struct xfs_mount *mp) +{ + if (!xfs_has_finobt(mp)) + return 0; + + return xfs_calc_inobt_res(mp); +} + +/* + * Calculate the reservation required to allocate or free an inode chunk. This + * includes: + * + * the allocation btrees: 2 trees * (max depth - 1) * block size + * the inode chunk: m_ino_geo.ialloc_blks * N + * + * The size N of the inode chunk reservation depends on whether it is for + * allocation or free and which type of create transaction is in use. An inode + * chunk free always invalidates the buffers and only requires reservation for + * headers (N == 0). An inode chunk allocation requires a chunk sized + * reservation on v4 and older superblocks to initialize the chunk. No chunk + * reservation is required for allocation on v5 supers, which use ordered + * buffers to initialize. + */ +STATIC uint +xfs_calc_inode_chunk_res( + struct xfs_mount *mp, + bool alloc) +{ + uint res, size = 0; + + res = xfs_calc_buf_res(xfs_allocfree_block_count(mp, 1), + XFS_FSB_TO_B(mp, 1)); + if (alloc) { + /* icreate tx uses ordered buffers */ + if (xfs_has_v3inodes(mp)) + return res; + size = XFS_FSB_TO_B(mp, 1); + } + + res += xfs_calc_buf_res(M_IGEO(mp)->ialloc_blks, size); + return res; +} + +/* + * Per-extent log reservation for the btree changes involved in freeing or + * allocating a realtime extent. We have to be able to log as many rtbitmap + * blocks as needed to mark inuse XFS_BMBT_MAX_EXTLEN blocks' worth of realtime + * extents, as well as the realtime summary block (t1). Realtime rmap btree + * operations happen in a second transaction, so factor in a couple of rtrmapbt + * splits (t2). + */ +static unsigned int +xfs_rtalloc_block_count( + struct xfs_mount *mp, + unsigned int num_ops) +{ + unsigned int rtbmp_blocks; + xfs_rtxlen_t rtxlen; + unsigned int t1, t2 = 0; + + rtxlen = xfs_extlen_to_rtxlen(mp, XFS_MAX_BMBT_EXTLEN); + rtbmp_blocks = xfs_rtbitmap_blockcount_len(mp, rtxlen); + t1 = (rtbmp_blocks + 1) * num_ops; + + if (xfs_has_rmapbt(mp)) + t2 = num_ops * (2 * mp->m_rtrmap_maxlevels - 1); + + return max(t1, t2); +} + +/* + * Various log reservation values. + * + * These are based on the size of the file system block because that is what + * most transactions manipulate. Each adds in an additional 128 bytes per + * item logged to try to account for the overhead of the transaction mechanism. + * + * Note: Most of the reservations underestimate the number of allocation + * groups into which they could free extents in the xfs_defer_finish() call. + * This is because the number in the worst case is quite high and quite + * unusual. In order to fix this we need to change xfs_defer_finish() to free + * extents in only a single AG at a time. This will require changes to the + * EFI code as well, however, so that the EFI for the extents not freed is + * logged again in each transaction. See SGI PV #261917. + * + * Reservation functions here avoid a huge stack in xfs_trans_init due to + * register overflow from temporaries in the calculations. + */ + +/* + * Finishing a data device refcount updates (t1): + * the agfs of the ags containing the blocks: nr_ops * sector size + * the refcount btrees: nr_ops * 1 trees * (2 * max depth - 1) * block size + */ +inline unsigned int +xfs_calc_finish_cui_reservation( + struct xfs_mount *mp, + unsigned int nr_ops) +{ + if (!xfs_has_reflink(mp)) + return 0; + + return xfs_calc_buf_res(nr_ops, mp->m_sb.sb_sectsize) + + xfs_calc_buf_res(xfs_refcountbt_block_count(mp, nr_ops), + mp->m_sb.sb_blocksize); +} + +/* + * Realtime refcount updates (t2); + * the rt refcount inode + * the rtrefcount btrees: nr_ops * 1 trees * (2 * max depth - 1) * block size + */ +inline unsigned int +xfs_calc_finish_rt_cui_reservation( + struct xfs_mount *mp, + unsigned int nr_ops) +{ + if (!xfs_has_rtreflink(mp)) + return 0; + + return xfs_calc_inode_res(mp, 1) + + xfs_calc_buf_res(xfs_rtrefcountbt_block_count(mp, nr_ops), + mp->m_sb.sb_blocksize); +} + +/* + * Compute the log reservation required to handle the refcount update + * transaction. Refcount updates are always done via deferred log items. + * + * This is calculated as the max of: + * Data device refcount updates (t1): + * the agfs of the ags containing the blocks: nr_ops * sector size + * the refcount btrees: nr_ops * 1 trees * (2 * max depth - 1) * block size + * Realtime refcount updates (t2); + * the rt refcount inode + * the rtrefcount btrees: nr_ops * 1 trees * (2 * max depth - 1) * block size + */ +static unsigned int +xfs_calc_refcountbt_reservation( + struct xfs_mount *mp, + unsigned int nr_ops) +{ + unsigned int t1, t2; + + t1 = xfs_calc_finish_cui_reservation(mp, nr_ops); + t2 = xfs_calc_finish_rt_cui_reservation(mp, nr_ops); + + return max(t1, t2); +} + +/* + * In a write transaction we can allocate a maximum of 2 + * extents. This gives (t1): + * the inode getting the new extents: inode size + * the inode's bmap btree: max depth * block size + * the agfs of the ags from which the extents are allocated: 2 * sector + * the superblock free block counter: sector size + * the allocation btrees: 2 exts * 2 trees * (2 * max depth - 1) * block size + * Or, if we're writing to a realtime file (t2): + * the inode getting the new extents: inode size + * the inode's bmap btree: max depth * block size + * the agfs of the ags from which the extents are allocated: 2 * sector + * the superblock free block counter: sector size + * the realtime bitmap: ((XFS_BMBT_MAX_EXTLEN / rtextsize) / NBBY) bytes + * the realtime summary: 1 block + * the allocation btrees: 2 trees * (2 * max depth - 1) * block size + * And the bmap_finish transaction can free bmap blocks in a join (t3): + * the agfs of the ags containing the blocks: 2 * sector size + * the agfls of the ags containing the blocks: 2 * sector size + * the super block free block counter: sector size + * the allocation btrees: 2 exts * 2 trees * (2 * max depth - 1) * block size + * And any refcount updates that happen in a separate transaction (t4). + */ +STATIC uint +xfs_calc_write_reservation( + struct xfs_mount *mp, + bool for_minlogsize) +{ + unsigned int t1, t2, t3, t4; + unsigned int blksz = XFS_FSB_TO_B(mp, 1); + + t1 = xfs_calc_inode_res(mp, 1) + + xfs_calc_buf_res(XFS_BM_MAXLEVELS(mp, XFS_DATA_FORK), blksz) + + xfs_calc_buf_res(3, mp->m_sb.sb_sectsize) + + xfs_calc_buf_res(xfs_allocfree_block_count(mp, 2), blksz); + + if (xfs_has_realtime(mp)) { + t2 = xfs_calc_inode_res(mp, 1) + + xfs_calc_buf_res(XFS_BM_MAXLEVELS(mp, XFS_DATA_FORK), + blksz) + + xfs_calc_buf_res(3, mp->m_sb.sb_sectsize) + + xfs_calc_buf_res(xfs_rtalloc_block_count(mp, 1), blksz) + + xfs_calc_buf_res(xfs_allocfree_block_count(mp, 1), blksz); + } else { + t2 = 0; + } + + t3 = xfs_calc_buf_res(5, mp->m_sb.sb_sectsize) + + xfs_calc_buf_res(xfs_allocfree_block_count(mp, 2), blksz); + + /* + * In the early days of reflink, we included enough reservation to log + * two refcountbt splits for each transaction. The codebase runs + * refcountbt updates in separate transactions now, so to compute the + * minimum log size, add the refcountbtree splits back to t1 and t3 and + * do not account them separately as t4. Reflink did not support + * realtime when the reservations were established, so no adjustment to + * t2 is needed. + */ + if (for_minlogsize) { + unsigned int adj = 0; + + if (xfs_has_reflink(mp)) + adj = xfs_calc_buf_res( + xfs_refcountbt_block_count(mp, 2), + blksz); + t1 += adj; + t3 += adj; + return XFS_DQUOT_LOGRES + max3(t1, t2, t3); + } + + t4 = xfs_calc_refcountbt_reservation(mp, 1); + return XFS_DQUOT_LOGRES + max(t4, max3(t1, t2, t3)); +} + +unsigned int +xfs_calc_write_reservation_minlogsize( + struct xfs_mount *mp) +{ + return xfs_calc_write_reservation(mp, true); +} + +/* + * Finishing an EFI can free the blocks and bmap blocks (t2): + * the agf for each of the ags: nr * sector size + * the agfl for each of the ags: nr * sector size + * the super block to reflect the freed blocks: sector size + * worst case split in allocation btrees per extent assuming nr extents: + * nr exts * 2 trees * (2 * max depth - 1) * block size + */ +inline unsigned int +xfs_calc_finish_efi_reservation( + struct xfs_mount *mp, + unsigned int nr) +{ + return xfs_calc_buf_res((2 * nr) + 1, mp->m_sb.sb_sectsize) + + xfs_calc_buf_res(xfs_allocfree_block_count(mp, nr), + mp->m_sb.sb_blocksize); +} + +/* + * Or, if it's a realtime file (t3): + * the agf for each of the ags: 2 * sector size + * the agfl for each of the ags: 2 * sector size + * the super block to reflect the freed blocks: sector size + * the realtime bitmap: + * 2 exts * ((XFS_BMBT_MAX_EXTLEN / rtextsize) / NBBY) bytes + * the realtime summary: 2 exts * 1 block + * worst case split in allocation btrees per extent assuming 2 extents: + * 2 exts * 2 trees * (2 * max depth - 1) * block size + */ +inline unsigned int +xfs_calc_finish_rt_efi_reservation( + struct xfs_mount *mp, + unsigned int nr) +{ + if (!xfs_has_realtime(mp)) + return 0; + + return xfs_calc_buf_res((2 * nr) + 1, mp->m_sb.sb_sectsize) + + xfs_calc_buf_res(xfs_rtalloc_block_count(mp, nr), + mp->m_sb.sb_blocksize) + + xfs_calc_buf_res(xfs_allocfree_block_count(mp, nr), + mp->m_sb.sb_blocksize); +} + +/* + * Finishing an RUI is the same as an EFI. We can split the rmap btree twice + * on each end of the record, and that can cause the AGFL to be refilled or + * emptied out. + */ +inline unsigned int +xfs_calc_finish_rui_reservation( + struct xfs_mount *mp, + unsigned int nr) +{ + if (!xfs_has_rmapbt(mp)) + return 0; + return xfs_calc_finish_efi_reservation(mp, nr); +} + +/* + * Finishing an RUI is the same as an EFI. We can split the rmap btree twice + * on each end of the record, and that can cause the AGFL to be refilled or + * emptied out. + */ +inline unsigned int +xfs_calc_finish_rt_rui_reservation( + struct xfs_mount *mp, + unsigned int nr) +{ + if (!xfs_has_rtrmapbt(mp)) + return 0; + return xfs_calc_finish_rt_efi_reservation(mp, nr); +} + +/* + * In finishing a BUI, we can modify: + * the inode being truncated: inode size + * dquots + * the inode's bmap btree: (max depth + 1) * block size + */ +inline unsigned int +xfs_calc_finish_bui_reservation( + struct xfs_mount *mp, + unsigned int nr) +{ + return xfs_calc_inode_res(mp, 1) + XFS_DQUOT_LOGRES + + xfs_calc_buf_res(XFS_BM_MAXLEVELS(mp, XFS_DATA_FORK) + 1, + mp->m_sb.sb_blocksize); +} + +/* + * In truncating a file we free up to two extents at once. We can modify (t1): + * the inode being truncated: inode size + * the inode's bmap btree: (max depth + 1) * block size + * And the bmap_finish transaction can free the blocks and bmap blocks (t2): + * the agf for each of the ags: 4 * sector size + * the agfl for each of the ags: 4 * sector size + * the super block to reflect the freed blocks: sector size + * worst case split in allocation btrees per extent assuming 4 extents: + * 4 exts * 2 trees * (2 * max depth - 1) * block size + * Or, if it's a realtime file (t3): + * the agf for each of the ags: 2 * sector size + * the agfl for each of the ags: 2 * sector size + * the super block to reflect the freed blocks: sector size + * the realtime bitmap: + * 2 exts * ((XFS_BMBT_MAX_EXTLEN / rtextsize) / NBBY) bytes + * the realtime summary: 2 exts * 1 block + * worst case split in allocation btrees per extent assuming 2 extents: + * 2 exts * 2 trees * (2 * max depth - 1) * block size + * And any refcount updates that happen in a separate transaction (t4). + */ +STATIC uint +xfs_calc_itruncate_reservation( + struct xfs_mount *mp, + bool for_minlogsize) +{ + unsigned int t1, t2, t3, t4; + unsigned int blksz = XFS_FSB_TO_B(mp, 1); + + t1 = xfs_calc_inode_res(mp, 1) + + xfs_calc_buf_res(XFS_BM_MAXLEVELS(mp, XFS_DATA_FORK) + 1, blksz); + + t2 = xfs_calc_finish_efi_reservation(mp, 4); + t3 = xfs_calc_finish_rt_efi_reservation(mp, 2); + + /* + * In the early days of reflink, we included enough reservation to log + * four refcountbt splits in the same transaction as bnobt/cntbt + * updates. The codebase runs refcountbt updates in separate + * transactions now, so to compute the minimum log size, add the + * refcount btree splits back here and do not compute them separately + * as t4. Reflink did not support realtime when the reservations were + * established, so do not adjust t3. + */ + if (for_minlogsize) { + if (xfs_has_reflink(mp)) + t2 += xfs_calc_buf_res( + xfs_refcountbt_block_count(mp, 4), + blksz); + + return XFS_DQUOT_LOGRES + max3(t1, t2, t3); + } + + t4 = xfs_calc_refcountbt_reservation(mp, 2); + return XFS_DQUOT_LOGRES + max(t4, max3(t1, t2, t3)); +} + +unsigned int +xfs_calc_itruncate_reservation_minlogsize( + struct xfs_mount *mp) +{ + return xfs_calc_itruncate_reservation(mp, true); +} + +static inline unsigned int xfs_calc_pptr_link_overhead(void) +{ + return sizeof(struct xfs_attri_log_format) + + xlog_calc_iovec_len(sizeof(struct xfs_parent_rec)) + + xlog_calc_iovec_len(MAXNAMELEN - 1); +} +static inline unsigned int xfs_calc_pptr_unlink_overhead(void) +{ + return sizeof(struct xfs_attri_log_format) + + xlog_calc_iovec_len(sizeof(struct xfs_parent_rec)) + + xlog_calc_iovec_len(MAXNAMELEN - 1); +} +static inline unsigned int xfs_calc_pptr_replace_overhead(void) +{ + return sizeof(struct xfs_attri_log_format) + + xlog_calc_iovec_len(sizeof(struct xfs_parent_rec)) + + xlog_calc_iovec_len(MAXNAMELEN - 1) + + xlog_calc_iovec_len(sizeof(struct xfs_parent_rec)) + + xlog_calc_iovec_len(MAXNAMELEN - 1); +} + +/* + * In renaming a files we can modify: + * the five inodes involved: 5 * inode size + * the two directory btrees: 2 * (max depth + v2) * dir block size + * the two directory bmap btrees: 2 * max depth * block size + * And the bmap_finish transaction can free dir and bmap blocks (two sets + * of bmap blocks) giving (t2): + * the agf for the ags in which the blocks live: 3 * sector size + * the agfl for the ags in which the blocks live: 3 * sector size + * the superblock for the free block count: sector size + * the allocation btrees: 3 exts * 2 trees * (2 * max depth - 1) * block size + * If parent pointers are enabled (t3), then each transaction in the chain + * must be capable of setting or removing the extended attribute + * containing the parent information. It must also be able to handle + * the three xattr intent items that track the progress of the parent + * pointer update. + */ +STATIC uint +xfs_calc_rename_reservation( + struct xfs_mount *mp) +{ + unsigned int overhead = XFS_DQUOT_LOGRES; + struct xfs_trans_resv *resp = M_RES(mp); + unsigned int t1, t2, t3 = 0; + + t1 = xfs_calc_inode_res(mp, 5) + + xfs_calc_buf_res(2 * XFS_DIROP_LOG_COUNT(mp), + XFS_FSB_TO_B(mp, 1)); + + t2 = xfs_calc_finish_efi_reservation(mp, 3); + + if (xfs_has_parent(mp)) { + unsigned int rename_overhead, exchange_overhead; + + t3 = max(resp->tr_attrsetm.tr_logres, + resp->tr_attrrm.tr_logres); + + /* + * For a standard rename, the three xattr intent log items + * are (1) replacing the pptr for the source file; (2) + * removing the pptr on the dest file; and (3) adding a + * pptr for the whiteout file in the src dir. + * + * For an RENAME_EXCHANGE, there are two xattr intent + * items to replace the pptr for both src and dest + * files. Link counts don't change and there is no + * whiteout. + * + * In the worst case we can end up relogging all log + * intent items to allow the log tail to move ahead, so + * they become overhead added to each transaction in a + * processing chain. + */ + rename_overhead = xfs_calc_pptr_replace_overhead() + + xfs_calc_pptr_unlink_overhead() + + xfs_calc_pptr_link_overhead(); + exchange_overhead = 2 * xfs_calc_pptr_replace_overhead(); + + overhead += max(rename_overhead, exchange_overhead); + } + + return overhead + max3(t1, t2, t3); +} + +static inline unsigned int +xfs_rename_log_count( + struct xfs_mount *mp, + struct xfs_trans_resv *resp) +{ + /* One for the rename, one more for freeing blocks */ + unsigned int ret = XFS_RENAME_LOG_COUNT; + + /* + * Pre-reserve enough log reservation to handle the transaction + * rolling needed to remove or add one parent pointer. + */ + if (xfs_has_parent(mp)) + ret += max(resp->tr_attrsetm.tr_logcount, + resp->tr_attrrm.tr_logcount); + + return ret; +} + +/* + * For removing an inode from unlinked list at first, we can modify: + * the agi hash list and counters: sector size + * the on disk inode before ours in the agi hash list: inode cluster size + * the on disk inode in the agi hash list: inode cluster size + */ +STATIC uint +xfs_calc_iunlink_remove_reservation( + struct xfs_mount *mp) +{ + return xfs_calc_buf_res(1, mp->m_sb.sb_sectsize) + + 2 * M_IGEO(mp)->inode_cluster_size; +} + +static inline unsigned int +xfs_link_log_count( + struct xfs_mount *mp, + struct xfs_trans_resv *resp) +{ + unsigned int ret = XFS_LINK_LOG_COUNT; + + /* + * Pre-reserve enough log reservation to handle the transaction + * rolling needed to add one parent pointer. + */ + if (xfs_has_parent(mp)) + ret += resp->tr_attrsetm.tr_logcount; + + return ret; +} + +/* + * For creating a link to an inode: + * the parent directory inode: inode size + * the linked inode: inode size + * the directory btree could split: (max depth + v2) * dir block size + * the directory bmap btree could join or split: (max depth + v2) * blocksize + * And the bmap_finish transaction can free some bmap blocks giving: + * the agf for the ag in which the blocks live: sector size + * the agfl for the ag in which the blocks live: sector size + * the superblock for the free block count: sector size + * the allocation btrees: 2 trees * (2 * max depth - 1) * block size + */ +STATIC uint +xfs_calc_link_reservation( + struct xfs_mount *mp) +{ + unsigned int overhead = XFS_DQUOT_LOGRES; + struct xfs_trans_resv *resp = M_RES(mp); + unsigned int t1, t2, t3 = 0; + + overhead += xfs_calc_iunlink_remove_reservation(mp); + t1 = xfs_calc_inode_res(mp, 2) + + xfs_calc_buf_res(XFS_DIROP_LOG_COUNT(mp), XFS_FSB_TO_B(mp, 1)); + t2 = xfs_calc_finish_efi_reservation(mp, 1); + + if (xfs_has_parent(mp)) { + t3 = resp->tr_attrsetm.tr_logres; + overhead += xfs_calc_pptr_link_overhead(); + } + + return overhead + max3(t1, t2, t3); +} + +/* + * For adding an inode to unlinked list we can modify: + * the agi hash list: sector size + * the on disk inode: inode cluster size + */ +STATIC uint +xfs_calc_iunlink_add_reservation(xfs_mount_t *mp) +{ + return xfs_calc_buf_res(1, mp->m_sb.sb_sectsize) + + M_IGEO(mp)->inode_cluster_size; +} + +static inline unsigned int +xfs_remove_log_count( + struct xfs_mount *mp, + struct xfs_trans_resv *resp) +{ + unsigned int ret = XFS_REMOVE_LOG_COUNT; + + /* + * Pre-reserve enough log reservation to handle the transaction + * rolling needed to add one parent pointer. + */ + if (xfs_has_parent(mp)) + ret += resp->tr_attrrm.tr_logcount; + + return ret; +} + +/* + * For removing a directory entry we can modify: + * the parent directory inode: inode size + * the removed inode: inode size + * the directory btree could join: (max depth + v2) * dir block size + * the directory bmap btree could join or split: (max depth + v2) * blocksize + * And the bmap_finish transaction can free the dir and bmap blocks giving: + * the agf for the ag in which the blocks live: 2 * sector size + * the agfl for the ag in which the blocks live: 2 * sector size + * the superblock for the free block count: sector size + * the allocation btrees: 2 exts * 2 trees * (2 * max depth - 1) * block size + */ +STATIC uint +xfs_calc_remove_reservation( + struct xfs_mount *mp) +{ + unsigned int overhead = XFS_DQUOT_LOGRES; + struct xfs_trans_resv *resp = M_RES(mp); + unsigned int t1, t2, t3 = 0; + + overhead += xfs_calc_iunlink_add_reservation(mp); + + t1 = xfs_calc_inode_res(mp, 2) + + xfs_calc_buf_res(XFS_DIROP_LOG_COUNT(mp), XFS_FSB_TO_B(mp, 1)); + t2 = xfs_calc_finish_efi_reservation(mp, 2); + + if (xfs_has_parent(mp)) { + t3 = resp->tr_attrrm.tr_logres; + overhead += xfs_calc_pptr_unlink_overhead(); + } + + return overhead + max3(t1, t2, t3); +} + +/* + * For create, break it in to the two cases that the transaction + * covers. We start with the modify case - allocation done by modification + * of the state of existing inodes - and the allocation case. + */ + +/* + * For create we can modify: + * the parent directory inode: inode size + * the new inode: inode size + * the inode btree entry: block size + * the superblock for the nlink flag: sector size + * the directory btree: (max depth + v2) * dir block size + * the directory inode's bmap btree: (max depth + v2) * block size + * the finobt (record modification and allocation btrees) + */ +STATIC uint +xfs_calc_create_resv_modify( + struct xfs_mount *mp) +{ + return xfs_calc_inode_res(mp, 2) + + xfs_calc_buf_res(1, mp->m_sb.sb_sectsize) + + (uint)XFS_FSB_TO_B(mp, 1) + + xfs_calc_buf_res(XFS_DIROP_LOG_COUNT(mp), XFS_FSB_TO_B(mp, 1)) + + xfs_calc_finobt_res(mp); +} + +/* + * For icreate we can allocate some inodes giving: + * the agi and agf of the ag getting the new inodes: 2 * sectorsize + * the superblock for the nlink flag: sector size + * the inode chunk (allocation, optional init) + * the inobt (record insertion) + * the finobt (optional, record insertion) + */ +STATIC uint +xfs_calc_icreate_resv_alloc( + struct xfs_mount *mp) +{ + return xfs_calc_buf_res(2, mp->m_sb.sb_sectsize) + + mp->m_sb.sb_sectsize + + xfs_calc_inode_chunk_res(mp, _ALLOC) + + xfs_calc_inobt_res(mp) + + xfs_calc_finobt_res(mp); +} + +static inline unsigned int +xfs_icreate_log_count( + struct xfs_mount *mp, + struct xfs_trans_resv *resp) +{ + unsigned int ret = XFS_CREATE_LOG_COUNT; + + /* + * Pre-reserve enough log reservation to handle the transaction + * rolling needed to add one parent pointer. + */ + if (xfs_has_parent(mp)) + ret += resp->tr_attrsetm.tr_logcount; + + return ret; +} + +STATIC uint +xfs_calc_icreate_reservation( + struct xfs_mount *mp) +{ + struct xfs_trans_resv *resp = M_RES(mp); + unsigned int overhead = XFS_DQUOT_LOGRES; + unsigned int t1, t2, t3 = 0; + + t1 = xfs_calc_icreate_resv_alloc(mp); + t2 = xfs_calc_create_resv_modify(mp); + + if (xfs_has_parent(mp)) { + t3 = resp->tr_attrsetm.tr_logres; + overhead += xfs_calc_pptr_link_overhead(); + } + + return overhead + max3(t1, t2, t3); +} + +STATIC uint +xfs_calc_create_tmpfile_reservation( + struct xfs_mount *mp) +{ + uint res = XFS_DQUOT_LOGRES; + + res += xfs_calc_icreate_resv_alloc(mp); + return res + xfs_calc_iunlink_add_reservation(mp); +} + +static inline unsigned int +xfs_mkdir_log_count( + struct xfs_mount *mp, + struct xfs_trans_resv *resp) +{ + unsigned int ret = XFS_MKDIR_LOG_COUNT; + + /* + * Pre-reserve enough log reservation to handle the transaction + * rolling needed to add one parent pointer. + */ + if (xfs_has_parent(mp)) + ret += resp->tr_attrsetm.tr_logcount; + + return ret; +} + +/* + * Making a new directory is the same as creating a new file. + */ +STATIC uint +xfs_calc_mkdir_reservation( + struct xfs_mount *mp) +{ + return xfs_calc_icreate_reservation(mp); +} + +static inline unsigned int +xfs_symlink_log_count( + struct xfs_mount *mp, + struct xfs_trans_resv *resp) +{ + unsigned int ret = XFS_SYMLINK_LOG_COUNT; + + /* + * Pre-reserve enough log reservation to handle the transaction + * rolling needed to add one parent pointer. + */ + if (xfs_has_parent(mp)) + ret += resp->tr_attrsetm.tr_logcount; + + return ret; +} + +/* + * Making a new symplink is the same as creating a new file, but + * with the added blocks for remote symlink data which can be up to 1kB in + * length (XFS_SYMLINK_MAXLEN). + */ +STATIC uint +xfs_calc_symlink_reservation( + struct xfs_mount *mp) +{ + return xfs_calc_icreate_reservation(mp) + + xfs_calc_buf_res(1, XFS_SYMLINK_MAXLEN); +} + +/* + * In freeing an inode we can modify: + * the inode being freed: inode size + * the super block free inode counter, AGF and AGFL: sector size + * the on disk inode (agi unlinked list removal) + * the inode chunk (invalidated, headers only) + * the inode btree + * the finobt (record insertion, removal or modification) + * + * Note that the inode chunk res. includes an allocfree res. for freeing of the + * inode chunk. This is technically extraneous because the inode chunk free is + * deferred (it occurs after a transaction roll). Include the extra reservation + * anyways since we've had reports of ifree transaction overruns due to too many + * agfl fixups during inode chunk frees. + */ +STATIC uint +xfs_calc_ifree_reservation( + struct xfs_mount *mp) +{ + return XFS_DQUOT_LOGRES + + xfs_calc_inode_res(mp, 1) + + xfs_calc_buf_res(3, mp->m_sb.sb_sectsize) + + xfs_calc_iunlink_remove_reservation(mp) + + xfs_calc_inode_chunk_res(mp, _FREE) + + xfs_calc_inobt_res(mp) + + xfs_calc_finobt_res(mp); +} + +/* + * When only changing the inode we log the inode and possibly the superblock + * We also add a bit of slop for the transaction stuff. + */ +STATIC uint +xfs_calc_ichange_reservation( + struct xfs_mount *mp) +{ + return XFS_DQUOT_LOGRES + + xfs_calc_inode_res(mp, 1) + + xfs_calc_buf_res(1, mp->m_sb.sb_sectsize); + +} + +/* + * Growing the data section of the filesystem. + * superblock + * agi and agf + * allocation btrees + */ +STATIC uint +xfs_calc_growdata_reservation( + struct xfs_mount *mp) +{ + return xfs_calc_buf_res(3, mp->m_sb.sb_sectsize) + + xfs_calc_buf_res(xfs_allocfree_block_count(mp, 1), + XFS_FSB_TO_B(mp, 1)); +} + +/* + * Growing the rt section of the filesystem. + * In the first set of transactions (ALLOC) we allocate space to the + * bitmap or summary files. + * superblock: sector size + * agf of the ag from which the extent is allocated: sector size + * bmap btree for bitmap/summary inode: max depth * blocksize + * bitmap/summary inode: inode size + * allocation btrees for 1 block alloc: 2 * (2 * maxdepth - 1) * blocksize + */ +STATIC uint +xfs_calc_growrtalloc_reservation( + struct xfs_mount *mp) +{ + return xfs_calc_buf_res(2, mp->m_sb.sb_sectsize) + + xfs_calc_buf_res(XFS_BM_MAXLEVELS(mp, XFS_DATA_FORK), + XFS_FSB_TO_B(mp, 1)) + + xfs_calc_inode_res(mp, 1) + + xfs_calc_buf_res(xfs_allocfree_block_count(mp, 1), + XFS_FSB_TO_B(mp, 1)); +} + +/* + * Growing the rt section of the filesystem. + * In the second set of transactions (ZERO) we zero the new metadata blocks. + * one bitmap/summary block: blocksize + */ +STATIC uint +xfs_calc_growrtzero_reservation( + struct xfs_mount *mp) +{ + return xfs_calc_buf_res(1, mp->m_sb.sb_blocksize); +} + +/* + * Growing the rt section of the filesystem. + * In the third set of transactions (FREE) we update metadata without + * allocating any new blocks. + * superblock: sector size + * bitmap inode: inode size + * summary inode: inode size + * one bitmap block: blocksize + * summary blocks: new summary size + */ +STATIC uint +xfs_calc_growrtfree_reservation( + struct xfs_mount *mp) +{ + return xfs_calc_buf_res(1, mp->m_sb.sb_sectsize) + + xfs_calc_inode_res(mp, 2) + + xfs_calc_buf_res(1, mp->m_sb.sb_blocksize) + + xfs_calc_buf_res(1, XFS_FSB_TO_B(mp, mp->m_rsumblocks)); +} + +/* + * Logging the inode modification timestamp on a synchronous write. + * inode + */ +STATIC uint +xfs_calc_swrite_reservation( + struct xfs_mount *mp) +{ + return xfs_calc_inode_res(mp, 1); +} + +/* + * Logging the inode mode bits when writing a setuid/setgid file + * inode + */ +STATIC uint +xfs_calc_writeid_reservation( + struct xfs_mount *mp) +{ + return xfs_calc_inode_res(mp, 1); +} + +/* + * Converting the inode from non-attributed to attributed. + * the inode being converted: inode size + * agf block and superblock (for block allocation) + * the new block (directory sized) + * bmap blocks for the new directory block + * allocation btrees + */ +STATIC uint +xfs_calc_addafork_reservation( + struct xfs_mount *mp) +{ + return XFS_DQUOT_LOGRES + + xfs_calc_inode_res(mp, 1) + + xfs_calc_buf_res(2, mp->m_sb.sb_sectsize) + + xfs_calc_buf_res(1, mp->m_dir_geo->blksize) + + xfs_calc_buf_res(XFS_DAENTER_BMAP1B(mp, XFS_DATA_FORK) + 1, + XFS_FSB_TO_B(mp, 1)) + + xfs_calc_buf_res(xfs_allocfree_block_count(mp, 1), + XFS_FSB_TO_B(mp, 1)); +} + +/* + * Removing the attribute fork of a file + * the inode being truncated: inode size + * the inode's bmap btree: max depth * block size + * And the bmap_finish transaction can free the blocks and bmap blocks: + * the agf for each of the ags: 4 * sector size + * the agfl for each of the ags: 4 * sector size + * the super block to reflect the freed blocks: sector size + * worst case split in allocation btrees per extent assuming 4 extents: + * 4 exts * 2 trees * (2 * max depth - 1) * block size + */ +STATIC uint +xfs_calc_attrinval_reservation( + struct xfs_mount *mp) +{ + return max((xfs_calc_inode_res(mp, 1) + + xfs_calc_buf_res(XFS_BM_MAXLEVELS(mp, XFS_ATTR_FORK), + XFS_FSB_TO_B(mp, 1))), + (xfs_calc_buf_res(9, mp->m_sb.sb_sectsize) + + xfs_calc_buf_res(xfs_allocfree_block_count(mp, 4), + XFS_FSB_TO_B(mp, 1)))); +} + +/* + * Setting an attribute at mount time. + * the inode getting the attribute + * the superblock for allocations + * the agfs extents are allocated from + * the attribute btree * max depth + * the inode allocation btree + * Since attribute transaction space is dependent on the size of the attribute, + * the calculation is done partially at mount time and partially at runtime(see + * below). + */ +STATIC uint +xfs_calc_attrsetm_reservation( + struct xfs_mount *mp) +{ + return XFS_DQUOT_LOGRES + + xfs_calc_inode_res(mp, 1) + + xfs_calc_buf_res(1, mp->m_sb.sb_sectsize) + + xfs_calc_buf_res(XFS_DA_NODE_MAXDEPTH, XFS_FSB_TO_B(mp, 1)); +} + +/* + * Setting an attribute at runtime, transaction space unit per block. + * the superblock for allocations: sector size + * the inode bmap btree could join or split: max depth * block size + * Since the runtime attribute transaction space is dependent on the total + * blocks needed for the 1st bmap, here we calculate out the space unit for + * one block so that the caller could figure out the total space according + * to the attibute extent length in blocks by: + * ext * M_RES(mp)->tr_attrsetrt.tr_logres + */ +STATIC uint +xfs_calc_attrsetrt_reservation( + struct xfs_mount *mp) +{ + return xfs_calc_buf_res(1, mp->m_sb.sb_sectsize) + + xfs_calc_buf_res(XFS_BM_MAXLEVELS(mp, XFS_ATTR_FORK), + XFS_FSB_TO_B(mp, 1)); +} + +/* + * Removing an attribute. + * the inode: inode size + * the attribute btree could join: max depth * block size + * the inode bmap btree could join or split: max depth * block size + * And the bmap_finish transaction can free the attr blocks freed giving: + * the agf for the ag in which the blocks live: 2 * sector size + * the agfl for the ag in which the blocks live: 2 * sector size + * the superblock for the free block count: sector size + * the allocation btrees: 2 exts * 2 trees * (2 * max depth - 1) * block size + */ +STATIC uint +xfs_calc_attrrm_reservation( + struct xfs_mount *mp) +{ + return XFS_DQUOT_LOGRES + + max((xfs_calc_inode_res(mp, 1) + + xfs_calc_buf_res(XFS_DA_NODE_MAXDEPTH, + XFS_FSB_TO_B(mp, 1)) + + (uint)XFS_FSB_TO_B(mp, + XFS_BM_MAXLEVELS(mp, XFS_ATTR_FORK)) + + xfs_calc_buf_res(XFS_BM_MAXLEVELS(mp, XFS_DATA_FORK), 0)), + (xfs_calc_buf_res(5, mp->m_sb.sb_sectsize) + + xfs_calc_buf_res(xfs_allocfree_block_count(mp, 2), + XFS_FSB_TO_B(mp, 1)))); +} + +/* + * Clearing a bad agino number in an agi hash bucket. + */ +STATIC uint +xfs_calc_clear_agi_bucket_reservation( + struct xfs_mount *mp) +{ + return xfs_calc_buf_res(1, mp->m_sb.sb_sectsize); +} + +/* + * Adjusting quota limits. + * the disk quota buffer: sizeof(struct xfs_disk_dquot) + */ +STATIC uint +xfs_calc_qm_setqlim_reservation(void) +{ + return xfs_calc_buf_res(1, sizeof(struct xfs_disk_dquot)); +} + +/* + * Allocating quota on disk if needed. + * the write transaction log space for quota file extent allocation + * the unit of quota allocation: one system block size + */ +STATIC uint +xfs_calc_qm_dqalloc_reservation( + struct xfs_mount *mp, + bool for_minlogsize) +{ + return xfs_calc_write_reservation(mp, for_minlogsize) + + xfs_calc_buf_res(1, + XFS_FSB_TO_B(mp, XFS_DQUOT_CLUSTER_SIZE_FSB) - 1); +} + +unsigned int +xfs_calc_qm_dqalloc_reservation_minlogsize( + struct xfs_mount *mp) +{ + return xfs_calc_qm_dqalloc_reservation(mp, true); +} + +/* + * Syncing the incore super block changes to disk. + * the super block to reflect the changes: sector size + */ +STATIC uint +xfs_calc_sb_reservation( + struct xfs_mount *mp) +{ + return xfs_calc_buf_res(1, mp->m_sb.sb_sectsize); +} + +/* + * Namespace reservations. + * + * These get tricky when parent pointers are enabled as we have attribute + * modifications occurring from within these transactions. Rather than confuse + * each of these reservation calculations with the conditional attribute + * reservations, add them here in a clear and concise manner. This requires that + * the attribute reservations have already been calculated. + * + * Note that we only include the static attribute reservation here; the runtime + * reservation will have to be modified by the size of the attributes being + * added/removed/modified. See the comments on the attribute reservation + * calculations for more details. + */ +STATIC void +xfs_calc_namespace_reservations( + struct xfs_mount *mp, + struct xfs_trans_resv *resp) +{ + ASSERT(resp->tr_attrsetm.tr_logres > 0); + + resp->tr_rename.tr_logres = xfs_calc_rename_reservation(mp); + resp->tr_rename.tr_logcount = xfs_rename_log_count(mp, resp); + resp->tr_rename.tr_logflags |= XFS_TRANS_PERM_LOG_RES; + + resp->tr_link.tr_logres = xfs_calc_link_reservation(mp); + resp->tr_link.tr_logcount = xfs_link_log_count(mp, resp); + resp->tr_link.tr_logflags |= XFS_TRANS_PERM_LOG_RES; + + resp->tr_remove.tr_logres = xfs_calc_remove_reservation(mp); + resp->tr_remove.tr_logcount = xfs_remove_log_count(mp, resp); + resp->tr_remove.tr_logflags |= XFS_TRANS_PERM_LOG_RES; + + resp->tr_symlink.tr_logres = xfs_calc_symlink_reservation(mp); + resp->tr_symlink.tr_logcount = xfs_symlink_log_count(mp, resp); + resp->tr_symlink.tr_logflags |= XFS_TRANS_PERM_LOG_RES; + + resp->tr_create.tr_logres = xfs_calc_icreate_reservation(mp); + resp->tr_create.tr_logcount = xfs_icreate_log_count(mp, resp); + resp->tr_create.tr_logflags |= XFS_TRANS_PERM_LOG_RES; + + resp->tr_mkdir.tr_logres = xfs_calc_mkdir_reservation(mp); + resp->tr_mkdir.tr_logcount = xfs_mkdir_log_count(mp, resp); + resp->tr_mkdir.tr_logflags |= XFS_TRANS_PERM_LOG_RES; +} + +STATIC void +xfs_calc_default_atomic_ioend_reservation( + struct xfs_mount *mp, + struct xfs_trans_resv *resp) +{ + /* Pick a default that will scale reasonably for the log size. */ + resp->tr_atomic_ioend = resp->tr_itruncate; +} + +void +xfs_trans_resv_calc( + struct xfs_mount *mp, + struct xfs_trans_resv *resp) +{ + int logcount_adj = 0; + + /* + * The following transactions are logged in physical format and + * require a permanent reservation on space. + */ + resp->tr_write.tr_logres = xfs_calc_write_reservation(mp, false); + resp->tr_write.tr_logcount = XFS_WRITE_LOG_COUNT; + resp->tr_write.tr_logflags |= XFS_TRANS_PERM_LOG_RES; + + resp->tr_itruncate.tr_logres = xfs_calc_itruncate_reservation(mp, false); + resp->tr_itruncate.tr_logcount = XFS_ITRUNCATE_LOG_COUNT; + resp->tr_itruncate.tr_logflags |= XFS_TRANS_PERM_LOG_RES; + + resp->tr_create_tmpfile.tr_logres = + xfs_calc_create_tmpfile_reservation(mp); + resp->tr_create_tmpfile.tr_logcount = XFS_CREATE_TMPFILE_LOG_COUNT; + resp->tr_create_tmpfile.tr_logflags |= XFS_TRANS_PERM_LOG_RES; + + resp->tr_ifree.tr_logres = xfs_calc_ifree_reservation(mp); + resp->tr_ifree.tr_logcount = XFS_INACTIVE_LOG_COUNT; + resp->tr_ifree.tr_logflags |= XFS_TRANS_PERM_LOG_RES; + + resp->tr_addafork.tr_logres = xfs_calc_addafork_reservation(mp); + resp->tr_addafork.tr_logcount = XFS_ADDAFORK_LOG_COUNT; + resp->tr_addafork.tr_logflags |= XFS_TRANS_PERM_LOG_RES; + + resp->tr_attrinval.tr_logres = xfs_calc_attrinval_reservation(mp); + resp->tr_attrinval.tr_logcount = XFS_ATTRINVAL_LOG_COUNT; + resp->tr_attrinval.tr_logflags |= XFS_TRANS_PERM_LOG_RES; + + resp->tr_attrsetm.tr_logres = xfs_calc_attrsetm_reservation(mp); + resp->tr_attrsetm.tr_logcount = XFS_ATTRSET_LOG_COUNT; + resp->tr_attrsetm.tr_logflags |= XFS_TRANS_PERM_LOG_RES; + + resp->tr_attrrm.tr_logres = xfs_calc_attrrm_reservation(mp); + resp->tr_attrrm.tr_logcount = XFS_ATTRRM_LOG_COUNT; + resp->tr_attrrm.tr_logflags |= XFS_TRANS_PERM_LOG_RES; + + resp->tr_growrtalloc.tr_logres = xfs_calc_growrtalloc_reservation(mp); + resp->tr_growrtalloc.tr_logcount = XFS_DEFAULT_PERM_LOG_COUNT; + resp->tr_growrtalloc.tr_logflags |= XFS_TRANS_PERM_LOG_RES; + + resp->tr_qm_dqalloc.tr_logres = xfs_calc_qm_dqalloc_reservation(mp, + false); + resp->tr_qm_dqalloc.tr_logcount = XFS_WRITE_LOG_COUNT; + resp->tr_qm_dqalloc.tr_logflags |= XFS_TRANS_PERM_LOG_RES; + + xfs_calc_namespace_reservations(mp, resp); + + /* + * The following transactions are logged in logical format with + * a default log count. + */ + resp->tr_qm_setqlim.tr_logres = xfs_calc_qm_setqlim_reservation(); + resp->tr_qm_setqlim.tr_logcount = XFS_DEFAULT_LOG_COUNT; + + resp->tr_sb.tr_logres = xfs_calc_sb_reservation(mp); + resp->tr_sb.tr_logcount = XFS_DEFAULT_LOG_COUNT; + + /* growdata requires permanent res; it can free space to the last AG */ + resp->tr_growdata.tr_logres = xfs_calc_growdata_reservation(mp); + resp->tr_growdata.tr_logcount = XFS_DEFAULT_PERM_LOG_COUNT; + resp->tr_growdata.tr_logflags |= XFS_TRANS_PERM_LOG_RES; + + /* The following transaction are logged in logical format */ + resp->tr_ichange.tr_logres = xfs_calc_ichange_reservation(mp); + resp->tr_fsyncts.tr_logres = xfs_calc_swrite_reservation(mp); + resp->tr_writeid.tr_logres = xfs_calc_writeid_reservation(mp); + resp->tr_attrsetrt.tr_logres = xfs_calc_attrsetrt_reservation(mp); + resp->tr_clearagi.tr_logres = xfs_calc_clear_agi_bucket_reservation(mp); + resp->tr_growrtzero.tr_logres = xfs_calc_growrtzero_reservation(mp); + resp->tr_growrtfree.tr_logres = xfs_calc_growrtfree_reservation(mp); + + /* + * Add one logcount for BUI items that appear with rmap or reflink, + * one logcount for refcount intent items, and one logcount for rmap + * intent items. + */ + if (xfs_has_reflink(mp) || xfs_has_rmapbt(mp)) + logcount_adj++; + if (xfs_has_reflink(mp)) + logcount_adj++; + if (xfs_has_rmapbt(mp)) + logcount_adj++; + + resp->tr_itruncate.tr_logcount += logcount_adj; + resp->tr_write.tr_logcount += logcount_adj; + resp->tr_qm_dqalloc.tr_logcount += logcount_adj; + + /* + * Now that we've finished computing the static reservations, we can + * compute the dynamic reservation for atomic writes. + */ + xfs_calc_default_atomic_ioend_reservation(mp, resp); +} + +/* + * Return the per-extent and fixed transaction reservation sizes needed to + * complete an atomic write. + */ +STATIC unsigned int +xfs_calc_atomic_write_ioend_geometry( + struct xfs_mount *mp, + unsigned int *step_size) +{ + const unsigned int efi = xfs_efi_log_space(1); + const unsigned int efd = xfs_efd_log_space(1); + const unsigned int rui = xfs_rui_log_space(1); + const unsigned int rud = xfs_rud_log_space(); + const unsigned int cui = xfs_cui_log_space(1); + const unsigned int cud = xfs_cud_log_space(); + const unsigned int bui = xfs_bui_log_space(1); + const unsigned int bud = xfs_bud_log_space(); + + /* + * Maximum overhead to complete an atomic write ioend in software: + * remove data fork extent + remove cow fork extent + map extent into + * data fork. + * + * tx0: Creates a BUI and a CUI and that's all it needs. + * + * tx1: Roll to finish the BUI. Need space for the BUD, an RUI, and + * enough space to relog the CUI (== CUI + CUD). + * + * tx2: Roll again to finish the RUI. Need space for the RUD and space + * to relog the CUI. + * + * tx3: Roll again, need space for the CUD and possibly a new EFI. + * + * tx4: Roll again, need space for an EFD. + * + * If the extent referenced by the pair of BUI/CUI items is not the one + * being currently processed, then we need to reserve space to relog + * both items. + */ + const unsigned int tx0 = bui + cui; + const unsigned int tx1 = bud + rui + cui + cud; + const unsigned int tx2 = rud + cui + cud; + const unsigned int tx3 = cud + efi; + const unsigned int tx4 = efd; + const unsigned int relog = bui + bud + cui + cud; + + const unsigned int per_intent = max(max3(tx0, tx1, tx2), + max3(tx3, tx4, relog)); + + /* Overhead to finish one step of each intent item type */ + const unsigned int f1 = xfs_calc_finish_efi_reservation(mp, 1); + const unsigned int f2 = xfs_calc_finish_rui_reservation(mp, 1); + const unsigned int f3 = xfs_calc_finish_cui_reservation(mp, 1); + const unsigned int f4 = xfs_calc_finish_bui_reservation(mp, 1); + + /* We only finish one item per transaction in a chain */ + *step_size = max(f4, max3(f1, f2, f3)); + + return per_intent; +} + +/* + * Compute the maximum size (in fsblocks) of atomic writes that we can complete + * given the existing log reservations. + */ +xfs_extlen_t +xfs_calc_max_atomic_write_fsblocks( + struct xfs_mount *mp) +{ + const struct xfs_trans_res *resv = &M_RES(mp)->tr_atomic_ioend; + unsigned int per_intent = 0; + unsigned int step_size = 0; + unsigned int ret = 0; + + if (resv->tr_logres > 0) { + per_intent = xfs_calc_atomic_write_ioend_geometry(mp, + &step_size); + + if (resv->tr_logres >= step_size) + ret = (resv->tr_logres - step_size) / per_intent; + } + + trace_xfs_calc_max_atomic_write_fsblocks(mp, per_intent, step_size, + resv->tr_logres, ret); + + return ret; +} + +/* + * Compute the log blocks and transaction reservation needed to complete an + * atomic write of a given number of blocks. Worst case, each block requires + * separate handling. A return value of 0 means something went wrong. + */ +xfs_extlen_t +xfs_calc_atomic_write_log_geometry( + struct xfs_mount *mp, + xfs_extlen_t blockcount, + unsigned int *new_logres) +{ + struct xfs_trans_res *curr_res = &M_RES(mp)->tr_atomic_ioend; + uint old_logres = curr_res->tr_logres; + unsigned int per_intent, step_size; + unsigned int logres; + xfs_extlen_t min_logblocks; + + ASSERT(blockcount > 0); + + xfs_calc_default_atomic_ioend_reservation(mp, M_RES(mp)); + + per_intent = xfs_calc_atomic_write_ioend_geometry(mp, &step_size); + + /* Check for overflows */ + if (check_mul_overflow(blockcount, per_intent, &logres) || + check_add_overflow(logres, step_size, &logres)) + return 0; + + curr_res->tr_logres = logres; + min_logblocks = xfs_log_calc_minimum_size(mp); + curr_res->tr_logres = old_logres; + + trace_xfs_calc_max_atomic_write_log_geometry(mp, per_intent, step_size, + blockcount, min_logblocks, logres); + + *new_logres = logres; + return min_logblocks; +} + +/* + * Compute the transaction reservation needed to complete an out of place + * atomic write of a given number of blocks. + */ +int +xfs_calc_atomic_write_reservation( + struct xfs_mount *mp, + xfs_extlen_t blockcount) +{ + unsigned int new_logres; + xfs_extlen_t min_logblocks; + + /* + * If the caller doesn't ask for a specific atomic write size, then + * use the defaults. + */ + if (blockcount == 0) { + xfs_calc_default_atomic_ioend_reservation(mp, M_RES(mp)); + return 0; + } + + min_logblocks = xfs_calc_atomic_write_log_geometry(mp, blockcount, + &new_logres); + if (!min_logblocks || min_logblocks > mp->m_sb.sb_logblocks) + return -EINVAL; + + M_RES(mp)->tr_atomic_ioend.tr_logres = new_logres; + return 0; +} diff --git a/fs/xfs/libxfs/xfs_trans_resv.h b/fs/xfs/libxfs/xfs_trans_resv.h new file mode 100644 index 000000000..336279e0f --- /dev/null +++ b/fs/xfs/libxfs/xfs_trans_resv.h @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_TRANS_RESV_H__ +#define __XFS_TRANS_RESV_H__ + +struct xfs_mount; + +/* + * structure for maintaining pre-calculated transaction reservations. + */ +struct xfs_trans_res { + uint tr_logres; /* log space unit in bytes per log ticket */ + int tr_logcount; /* number of log operations per log ticket */ + int tr_logflags; /* log flags, currently only used for indicating + * a reservation request is permanent or not */ +}; + +struct xfs_trans_resv { + struct xfs_trans_res tr_write; /* extent alloc trans */ + struct xfs_trans_res tr_itruncate; /* truncate trans */ + struct xfs_trans_res tr_rename; /* rename trans */ + struct xfs_trans_res tr_link; /* link trans */ + struct xfs_trans_res tr_remove; /* unlink trans */ + struct xfs_trans_res tr_symlink; /* symlink trans */ + struct xfs_trans_res tr_create; /* create trans */ + struct xfs_trans_res tr_create_tmpfile; /* create O_TMPFILE trans */ + struct xfs_trans_res tr_mkdir; /* mkdir trans */ + struct xfs_trans_res tr_ifree; /* inode free trans */ + struct xfs_trans_res tr_ichange; /* inode update trans */ + struct xfs_trans_res tr_growdata; /* fs data section grow trans */ + struct xfs_trans_res tr_addafork; /* add inode attr fork trans */ + struct xfs_trans_res tr_writeid; /* write setuid/setgid file */ + struct xfs_trans_res tr_attrinval; /* attr fork buffer + * invalidation */ + struct xfs_trans_res tr_attrsetm; /* set/create an attribute at + * mount time */ + struct xfs_trans_res tr_attrsetrt; /* set/create an attribute at + * runtime */ + struct xfs_trans_res tr_attrrm; /* remove an attribute */ + struct xfs_trans_res tr_clearagi; /* clear agi unlinked bucket */ + struct xfs_trans_res tr_growrtalloc; /* grow realtime allocations */ + struct xfs_trans_res tr_growrtzero; /* grow realtime zeroing */ + struct xfs_trans_res tr_growrtfree; /* grow realtime freeing */ + struct xfs_trans_res tr_qm_setqlim; /* adjust quota limits */ + struct xfs_trans_res tr_qm_dqalloc; /* allocate quota on disk */ + struct xfs_trans_res tr_sb; /* modify superblock */ + struct xfs_trans_res tr_fsyncts; /* update timestamps on fsync */ + struct xfs_trans_res tr_atomic_ioend; /* untorn write completion */ +}; + +/* shorthand way of accessing reservation structure */ +#define M_RES(mp) (&(mp)->m_resv) + +/* + * Per-directory log reservation for any directory change. + * dir blocks: (1 btree block per level + data block + free block) * dblock size + * bmap btree: (levels + 2) * max depth * block size + * v2 directory blocks can be fragmented below the dirblksize down to the fsb + * size, so account for that in the DAENTER macros. + */ +#define XFS_DIROP_LOG_RES(mp) \ + (XFS_FSB_TO_B(mp, XFS_DAENTER_BLOCKS(mp, XFS_DATA_FORK)) + \ + (XFS_FSB_TO_B(mp, XFS_DAENTER_BMAPS(mp, XFS_DATA_FORK) + 1))) +#define XFS_DIROP_LOG_COUNT(mp) \ + (XFS_DAENTER_BLOCKS(mp, XFS_DATA_FORK) + \ + XFS_DAENTER_BMAPS(mp, XFS_DATA_FORK) + 1) + +/* + * Various log count values. + */ +#define XFS_DEFAULT_LOG_COUNT 1 +#define XFS_DEFAULT_PERM_LOG_COUNT 2 +#define XFS_ITRUNCATE_LOG_COUNT 2 +#define XFS_INACTIVE_LOG_COUNT 2 +#define XFS_CREATE_LOG_COUNT 2 +#define XFS_CREATE_TMPFILE_LOG_COUNT 2 +#define XFS_MKDIR_LOG_COUNT 3 +#define XFS_SYMLINK_LOG_COUNT 3 +#define XFS_REMOVE_LOG_COUNT 2 +#define XFS_LINK_LOG_COUNT 2 +#define XFS_RENAME_LOG_COUNT 2 +#define XFS_WRITE_LOG_COUNT 2 +#define XFS_ADDAFORK_LOG_COUNT 2 +#define XFS_ATTRINVAL_LOG_COUNT 1 +#define XFS_ATTRSET_LOG_COUNT 3 +#define XFS_ATTRRM_LOG_COUNT 3 + +/* + * Original log operation counts were overestimated in the early days of + * reflink. These are retained here purely for minimum log size calculations + * and must not be used for runtime reservations. + */ +#define XFS_ITRUNCATE_LOG_COUNT_REFLINK 8 +#define XFS_WRITE_LOG_COUNT_REFLINK 8 + +void xfs_trans_resv_calc(struct xfs_mount *mp, struct xfs_trans_resv *resp); +uint xfs_allocfree_block_count(struct xfs_mount *mp, uint num_ops); + +unsigned int xfs_calc_finish_bui_reservation(struct xfs_mount *mp, + unsigned int nr_ops); + +unsigned int xfs_calc_finish_efi_reservation(struct xfs_mount *mp, + unsigned int nr_ops); +unsigned int xfs_calc_finish_rt_efi_reservation(struct xfs_mount *mp, + unsigned int nr_ops); + +unsigned int xfs_calc_finish_rui_reservation(struct xfs_mount *mp, + unsigned int nr_ops); +unsigned int xfs_calc_finish_rt_rui_reservation(struct xfs_mount *mp, + unsigned int nr_ops); + +unsigned int xfs_calc_finish_cui_reservation(struct xfs_mount *mp, + unsigned int nr_ops); +unsigned int xfs_calc_finish_rt_cui_reservation(struct xfs_mount *mp, + unsigned int nr_ops); + +unsigned int xfs_calc_itruncate_reservation_minlogsize(struct xfs_mount *mp); +unsigned int xfs_calc_write_reservation_minlogsize(struct xfs_mount *mp); +unsigned int xfs_calc_qm_dqalloc_reservation_minlogsize(struct xfs_mount *mp); + +xfs_extlen_t xfs_calc_max_atomic_write_fsblocks(struct xfs_mount *mp); +xfs_extlen_t xfs_calc_atomic_write_log_geometry(struct xfs_mount *mp, + xfs_extlen_t blockcount, unsigned int *new_logres); +int xfs_calc_atomic_write_reservation(struct xfs_mount *mp, + xfs_extlen_t blockcount); + +#endif /* __XFS_TRANS_RESV_H__ */ diff --git a/fs/xfs/libxfs/xfs_trans_space.c b/fs/xfs/libxfs/xfs_trans_space.c new file mode 100644 index 000000000..9b8f495c9 --- /dev/null +++ b/fs/xfs/libxfs/xfs_trans_space.c @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_da_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_da_btree.h" +#include "xfs_bmap_btree.h" +#include "xfs_trans_space.h" + +/* Calculate the disk space required to add a parent pointer. */ +unsigned int +xfs_parent_calc_space_res( + struct xfs_mount *mp, + unsigned int namelen) +{ + /* + * Parent pointers are always the first attr in an attr tree, and never + * larger than a block + */ + return XFS_DAENTER_SPACE_RES(mp, XFS_ATTR_FORK) + + XFS_NEXTENTADD_SPACE_RES(mp, namelen, XFS_ATTR_FORK); +} + +unsigned int +xfs_create_space_res( + struct xfs_mount *mp, + unsigned int namelen) +{ + unsigned int ret; + + ret = XFS_IALLOC_SPACE_RES(mp) + XFS_DIRENTER_SPACE_RES(mp, namelen); + if (xfs_has_parent(mp)) + ret += xfs_parent_calc_space_res(mp, namelen); + + return ret; +} + +unsigned int +xfs_mkdir_space_res( + struct xfs_mount *mp, + unsigned int namelen) +{ + return xfs_create_space_res(mp, namelen); +} + +unsigned int +xfs_link_space_res( + struct xfs_mount *mp, + unsigned int namelen) +{ + unsigned int ret; + + ret = XFS_DIRENTER_SPACE_RES(mp, namelen); + if (xfs_has_parent(mp)) + ret += xfs_parent_calc_space_res(mp, namelen); + + return ret; +} + +unsigned int +xfs_symlink_space_res( + struct xfs_mount *mp, + unsigned int namelen, + unsigned int fsblocks) +{ + unsigned int ret; + + ret = XFS_IALLOC_SPACE_RES(mp) + XFS_DIRENTER_SPACE_RES(mp, namelen) + + fsblocks; + + if (xfs_has_parent(mp)) + ret += xfs_parent_calc_space_res(mp, namelen); + + return ret; +} + +unsigned int +xfs_remove_space_res( + struct xfs_mount *mp, + unsigned int namelen) +{ + unsigned int ret = XFS_DIRREMOVE_SPACE_RES(mp); + + if (xfs_has_parent(mp)) + ret += xfs_parent_calc_space_res(mp, namelen); + + return ret; +} + +unsigned int +xfs_rename_space_res( + struct xfs_mount *mp, + unsigned int src_namelen, + bool target_exists, + unsigned int target_namelen, + bool has_whiteout) +{ + unsigned int ret; + + ret = XFS_DIRREMOVE_SPACE_RES(mp) + + XFS_DIRENTER_SPACE_RES(mp, target_namelen); + + if (xfs_has_parent(mp)) { + if (has_whiteout) + ret += xfs_parent_calc_space_res(mp, src_namelen); + ret += 2 * xfs_parent_calc_space_res(mp, target_namelen); + } + + if (target_exists) + ret += xfs_parent_calc_space_res(mp, target_namelen); + + return ret; +} diff --git a/fs/xfs/libxfs/xfs_trans_space.h b/fs/xfs/libxfs/xfs_trans_space.h new file mode 100644 index 000000000..d89b570aa --- /dev/null +++ b/fs/xfs/libxfs/xfs_trans_space.h @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_TRANS_SPACE_H__ +#define __XFS_TRANS_SPACE_H__ + +/* + * Components of space reservations. + */ + +/* Worst case number of bmaps that can be held in a block. */ +#define XFS_MAX_CONTIG_BMAPS_PER_BLOCK(mp) \ + (((mp)->m_bmap_dmxr[0]) - ((mp)->m_bmap_dmnr[0])) + +/* Worst case number of realtime rmaps that can be held in a block. */ +#define XFS_MAX_CONTIG_RTRMAPS_PER_BLOCK(mp) \ + (((mp)->m_rtrmap_mxr[0]) - ((mp)->m_rtrmap_mnr[0])) + +/* Adding one realtime rmap could split every level to the top of the tree. */ +#define XFS_RTRMAPADD_SPACE_RES(mp) ((mp)->m_rtrmap_maxlevels) + +/* Blocks we might need to add "b" realtime rmaps to a tree. */ +#define XFS_NRTRMAPADD_SPACE_RES(mp, b) \ + ((((b) + XFS_MAX_CONTIG_RTRMAPS_PER_BLOCK(mp) - 1) / \ + XFS_MAX_CONTIG_RTRMAPS_PER_BLOCK(mp)) * \ + XFS_RTRMAPADD_SPACE_RES(mp)) + +/* Worst case number of rmaps that can be held in a block. */ +#define XFS_MAX_CONTIG_RMAPS_PER_BLOCK(mp) \ + (((mp)->m_rmap_mxr[0]) - ((mp)->m_rmap_mnr[0])) + +/* Adding one rmap could split every level up to the top of the tree. */ +#define XFS_RMAPADD_SPACE_RES(mp) ((mp)->m_rmap_maxlevels) + +/* + * Note that we historically set m_rmap_maxlevels to 9 when reflink is enabled, + * so we must preserve this behavior to avoid changing the transaction space + * reservations and minimum log size calculations for existing filesystems. + */ +#define XFS_OLD_REFLINK_RMAP_MAXLEVELS 9 + +/* Blocks we might need to add "b" rmaps to a tree. */ +#define XFS_NRMAPADD_SPACE_RES(mp, b)\ + (((b + XFS_MAX_CONTIG_RMAPS_PER_BLOCK(mp) - 1) / \ + XFS_MAX_CONTIG_RMAPS_PER_BLOCK(mp)) * \ + XFS_RMAPADD_SPACE_RES(mp)) + +#define XFS_MAX_CONTIG_EXTENTS_PER_BLOCK(mp) \ + (((mp)->m_alloc_mxr[0]) - ((mp)->m_alloc_mnr[0])) +#define XFS_EXTENTADD_SPACE_RES(mp,w) (XFS_BM_MAXLEVELS(mp,w) - 1) +#define XFS_NEXTENTADD_SPACE_RES(mp,b,w)\ + (((b + XFS_MAX_CONTIG_EXTENTS_PER_BLOCK(mp) - 1) / \ + XFS_MAX_CONTIG_EXTENTS_PER_BLOCK(mp)) * \ + XFS_EXTENTADD_SPACE_RES(mp,w)) + +/* Blocks we might need to add "b" mappings & rmappings to a file. */ +#define XFS_SWAP_RMAP_SPACE_RES(mp,b,w)\ + (XFS_NEXTENTADD_SPACE_RES((mp), (b), (w)) + \ + XFS_NRMAPADD_SPACE_RES((mp), (b))) + +#define XFS_DAENTER_1B(mp,w) \ + ((w) == XFS_DATA_FORK ? (mp)->m_dir_geo->fsbcount : 1) +#define XFS_DAENTER_DBS(mp,w) \ + (XFS_DA_NODE_MAXDEPTH + (((w) == XFS_DATA_FORK) ? 2 : 0)) +#define XFS_DAENTER_BLOCKS(mp,w) \ + (XFS_DAENTER_1B(mp,w) * XFS_DAENTER_DBS(mp,w)) +#define XFS_DAENTER_BMAP1B(mp,w) \ + XFS_NEXTENTADD_SPACE_RES(mp, XFS_DAENTER_1B(mp, w), w) +#define XFS_DAENTER_BMAPS(mp,w) \ + (XFS_DAENTER_DBS(mp,w) * XFS_DAENTER_BMAP1B(mp,w)) +#define XFS_DAENTER_SPACE_RES(mp,w) \ + (XFS_DAENTER_BLOCKS(mp,w) + XFS_DAENTER_BMAPS(mp,w)) +#define XFS_DAREMOVE_SPACE_RES(mp,w) XFS_DAENTER_BMAPS(mp,w) +#define XFS_DIRENTER_MAX_SPLIT(mp,nl) 1 +#define XFS_DIRENTER_SPACE_RES(mp,nl) \ + (XFS_DAENTER_SPACE_RES(mp, XFS_DATA_FORK) * \ + XFS_DIRENTER_MAX_SPLIT(mp,nl)) +#define XFS_DIRREMOVE_SPACE_RES(mp) \ + XFS_DAREMOVE_SPACE_RES(mp, XFS_DATA_FORK) +#define XFS_IALLOC_SPACE_RES(mp) \ + (M_IGEO(mp)->ialloc_blks + \ + ((xfs_has_finobt(mp) ? 2 : 1) * M_IGEO(mp)->inobt_maxlevels)) + +/* + * Space reservation values for various transactions. + */ +#define XFS_ADDAFORK_SPACE_RES(mp) \ + ((mp)->m_dir_geo->fsbcount + XFS_DAENTER_BMAP1B(mp, XFS_DATA_FORK)) +#define XFS_ATTRRM_SPACE_RES(mp) \ + XFS_DAREMOVE_SPACE_RES(mp, XFS_ATTR_FORK) +/* This macro is not used - see inline code in xfs_attr_set */ +#define XFS_ATTRSET_SPACE_RES(mp, v) \ + (XFS_DAENTER_SPACE_RES(mp, XFS_ATTR_FORK) + XFS_B_TO_FSB(mp, v)) +#define XFS_DIOSTRAT_SPACE_RES(mp, v) \ + (XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK) + (v)) +#define XFS_GROWFS_SPACE_RES(mp) \ + (2 * (mp)->m_alloc_maxlevels) +#define XFS_GROWFSRT_SPACE_RES(mp,b) \ + ((b) + XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK)) +#define XFS_QM_DQALLOC_SPACE_RES(mp) \ + (XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK) + \ + XFS_DQUOT_CLUSTER_SIZE_FSB) +#define XFS_QM_QINOCREATE_SPACE_RES(mp) \ + XFS_IALLOC_SPACE_RES(mp) +#define XFS_IFREE_SPACE_RES(mp) \ + (xfs_has_finobt(mp) ? M_IGEO(mp)->inobt_maxlevels : 0) + +unsigned int xfs_parent_calc_space_res(struct xfs_mount *mp, + unsigned int namelen); + +unsigned int xfs_create_space_res(struct xfs_mount *mp, unsigned int namelen); +unsigned int xfs_mkdir_space_res(struct xfs_mount *mp, unsigned int namelen); +unsigned int xfs_link_space_res(struct xfs_mount *mp, unsigned int namelen); +unsigned int xfs_symlink_space_res(struct xfs_mount *mp, unsigned int namelen, + unsigned int fsblocks); +unsigned int xfs_remove_space_res(struct xfs_mount *mp, unsigned int namelen); + +unsigned int xfs_rename_space_res(struct xfs_mount *mp, + unsigned int src_namelen, bool target_exists, + unsigned int target_namelen, bool has_whiteout); + +#endif /* __XFS_TRANS_SPACE_H__ */ diff --git a/fs/xfs/libxfs/xfs_types.c b/fs/xfs/libxfs/xfs_types.c new file mode 100644 index 000000000..67c947a47 --- /dev/null +++ b/fs/xfs/libxfs/xfs_types.c @@ -0,0 +1,256 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc. + * Copyright (C) 2017 Oracle. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_format.h" +#include "xfs_shared.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_mount.h" +#include "xfs_ag.h" +#include "xfs_rtbitmap.h" +#include "xfs_rtgroup.h" + + +/* + * Verify that an AG block number pointer neither points outside the AG + * nor points at static metadata. + */ +static inline bool +xfs_verify_agno_agbno( + struct xfs_mount *mp, + xfs_agnumber_t agno, + xfs_agblock_t agbno) +{ + xfs_agblock_t eoag; + + eoag = xfs_ag_block_count(mp, agno); + if (agbno >= eoag) + return false; + if (agbno <= XFS_AGFL_BLOCK(mp)) + return false; + return true; +} + +/* + * Verify that an FS block number pointer neither points outside the + * filesystem nor points at static AG metadata. + */ +inline bool +xfs_verify_fsbno( + struct xfs_mount *mp, + xfs_fsblock_t fsbno) +{ + xfs_agnumber_t agno = XFS_FSB_TO_AGNO(mp, fsbno); + + if (agno >= mp->m_sb.sb_agcount) + return false; + return xfs_verify_agno_agbno(mp, agno, XFS_FSB_TO_AGBNO(mp, fsbno)); +} + +/* + * Verify that a data device extent is fully contained inside the filesystem, + * does not cross an AG boundary, and does not point at static metadata. + */ +bool +xfs_verify_fsbext( + struct xfs_mount *mp, + xfs_fsblock_t fsbno, + xfs_fsblock_t len) +{ + if (fsbno + len <= fsbno) + return false; + + if (!xfs_verify_fsbno(mp, fsbno)) + return false; + + if (!xfs_verify_fsbno(mp, fsbno + len - 1)) + return false; + + return XFS_FSB_TO_AGNO(mp, fsbno) == + XFS_FSB_TO_AGNO(mp, fsbno + len - 1); +} + +/* + * Verify that an AG inode number pointer neither points outside the AG + * nor points at static metadata. + */ +static inline bool +xfs_verify_agno_agino( + struct xfs_mount *mp, + xfs_agnumber_t agno, + xfs_agino_t agino) +{ + xfs_agino_t first; + xfs_agino_t last; + + xfs_agino_range(mp, agno, &first, &last); + return agino >= first && agino <= last; +} + +/* + * Verify that an FS inode number pointer neither points outside the + * filesystem nor points at static AG metadata. + */ +inline bool +xfs_verify_ino( + struct xfs_mount *mp, + xfs_ino_t ino) +{ + xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, ino); + xfs_agino_t agino = XFS_INO_TO_AGINO(mp, ino); + + if (agno >= mp->m_sb.sb_agcount) + return false; + if (XFS_AGINO_TO_INO(mp, agno, agino) != ino) + return false; + return xfs_verify_agno_agino(mp, agno, agino); +} + +/* Is this an internal inode number? */ +inline bool +xfs_is_sb_inum( + struct xfs_mount *mp, + xfs_ino_t ino) +{ + return ino == mp->m_sb.sb_rbmino || ino == mp->m_sb.sb_rsumino || + (xfs_has_quota(mp) && + xfs_is_quota_inode(&mp->m_sb, ino)); +} + +/* + * Verify that a directory entry's inode number doesn't point at an internal + * inode, empty space, or static AG metadata. + */ +bool +xfs_verify_dir_ino( + struct xfs_mount *mp, + xfs_ino_t ino) +{ + if (xfs_is_sb_inum(mp, ino)) + return false; + return xfs_verify_ino(mp, ino); +} + +/* + * Verify that a realtime block number pointer neither points outside the + * allocatable areas of the rtgroup nor off the end of the realtime + * device. + */ +inline bool +xfs_verify_rtbno( + struct xfs_mount *mp, + xfs_rtblock_t rtbno) +{ + if (xfs_has_rtgroups(mp)) { + xfs_rgnumber_t rgno = xfs_rtb_to_rgno(mp, rtbno); + xfs_rtxnum_t rtx = xfs_rtb_to_rtx(mp, rtbno); + + if (rgno >= mp->m_sb.sb_rgcount) + return false; + if (rtx >= xfs_rtgroup_extents(mp, rgno)) + return false; + if (xfs_has_rtsb(mp) && rgno == 0 && rtx == 0) + return false; + return true; + } + + return rtbno < mp->m_sb.sb_rblocks; +} + +/* + * Verify that an allocated realtime device extent neither points outside + * allocatable areas of the rtgroup, across an rtgroup boundary, nor off the + * end of the realtime device. + */ +bool +xfs_verify_rtbext( + struct xfs_mount *mp, + xfs_rtblock_t rtbno, + xfs_filblks_t len) +{ + if (rtbno + len <= rtbno) + return false; + + if (!xfs_verify_rtbno(mp, rtbno)) + return false; + + if (!xfs_verify_rtbno(mp, rtbno + len - 1)) + return false; + + if (xfs_has_rtgroups(mp) && + xfs_rtb_to_rgno(mp, rtbno) != xfs_rtb_to_rgno(mp, rtbno + len - 1)) + return false; + + return true; +} + +/* Calculate the range of valid icount values. */ +inline void +xfs_icount_range( + struct xfs_mount *mp, + unsigned long long *min, + unsigned long long *max) +{ + unsigned long long nr_inos = 0; + struct xfs_perag *pag = NULL; + + /* root, rtbitmap, rtsum all live in the first chunk */ + *min = XFS_INODES_PER_CHUNK; + + while ((pag = xfs_perag_next(mp, pag))) + nr_inos += pag->agino_max - pag->agino_min + 1; + *max = nr_inos; +} + +/* Sanity-checking of inode counts. */ +bool +xfs_verify_icount( + struct xfs_mount *mp, + unsigned long long icount) +{ + unsigned long long min, max; + + xfs_icount_range(mp, &min, &max); + return icount >= min && icount <= max; +} + +/* Sanity-checking of dir/attr block offsets. */ +bool +xfs_verify_dablk( + struct xfs_mount *mp, + xfs_fileoff_t dabno) +{ + xfs_dablk_t max_dablk = -1U; + + return dabno <= max_dablk; +} + +/* Check that a file block offset does not exceed the maximum. */ +bool +xfs_verify_fileoff( + struct xfs_mount *mp, + xfs_fileoff_t off) +{ + return off <= XFS_MAX_FILEOFF; +} + +/* Check that a range of file block offsets do not exceed the maximum. */ +bool +xfs_verify_fileext( + struct xfs_mount *mp, + xfs_fileoff_t off, + xfs_fileoff_t len) +{ + if (off + len <= off) + return false; + + if (!xfs_verify_fileoff(mp, off)) + return false; + + return xfs_verify_fileoff(mp, off + len - 1); +} diff --git a/fs/xfs/libxfs/xfs_types.h b/fs/xfs/libxfs/xfs_types.h new file mode 100644 index 000000000..f6f4f2d4b --- /dev/null +++ b/fs/xfs/libxfs/xfs_types.h @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_TYPES_H__ +#define __XFS_TYPES_H__ + +typedef uint32_t prid_t; /* project ID */ + +typedef uint32_t xfs_agblock_t; /* blockno in alloc. group */ +typedef uint32_t xfs_rgblock_t; /* blockno in realtime group */ +typedef uint32_t xfs_agino_t; /* inode # within allocation grp */ +typedef uint32_t xfs_extlen_t; /* extent length in blocks */ +typedef uint32_t xfs_rtxlen_t; /* file extent length in rtextents */ +typedef uint32_t xfs_agnumber_t; /* allocation group number */ +typedef uint32_t xfs_rgnumber_t; /* realtime group number */ +typedef uint64_t xfs_extnum_t; /* # of extents in a file */ +typedef uint32_t xfs_aextnum_t; /* # extents in an attribute fork */ +typedef int64_t xfs_fsize_t; /* bytes in a file */ +typedef uint64_t xfs_ufsize_t; /* unsigned bytes in a file */ + +typedef int32_t xfs_suminfo_t; /* type of bitmap summary info */ +typedef uint32_t xfs_rtsumoff_t; /* offset of an rtsummary info word */ +typedef uint32_t xfs_rtword_t; /* word type for bitmap manipulations */ + +typedef int64_t xfs_lsn_t; /* log sequence number */ +typedef int64_t xfs_csn_t; /* CIL sequence number */ + +typedef uint32_t xfs_dablk_t; /* dir/attr block number (in file) */ +typedef uint32_t xfs_dahash_t; /* dir/attr hash value */ + +typedef uint64_t xfs_fsblock_t; /* blockno in filesystem (agno|agbno) */ +typedef uint64_t xfs_rfsblock_t; /* blockno in filesystem (raw) */ +typedef uint64_t xfs_rtblock_t; /* extent (block) in realtime area */ +typedef uint64_t xfs_fileoff_t; /* block number in a file */ +typedef uint64_t xfs_filblks_t; /* number of blocks in a file */ +typedef uint64_t xfs_rtxnum_t; /* rtextent number */ +typedef uint64_t xfs_rtbxlen_t; /* rtbitmap extent length in rtextents */ + +typedef int64_t xfs_srtblock_t; /* signed version of xfs_rtblock_t */ + +/* + * New verifiers will return the instruction address of the failing check. + * NULL means everything is ok. + */ +typedef void * xfs_failaddr_t; + +/* + * Null values for the types. + */ +#define NULLFSBLOCK ((xfs_fsblock_t)-1) +#define NULLRFSBLOCK ((xfs_rfsblock_t)-1) +#define NULLRTBLOCK ((xfs_rtblock_t)-1) +#define NULLFILEOFF ((xfs_fileoff_t)-1) + +#define NULLAGBLOCK ((xfs_agblock_t)-1) +#define NULLRGBLOCK ((xfs_rgblock_t)-1) +#define NULLAGNUMBER ((xfs_agnumber_t)-1) +#define NULLRGNUMBER ((xfs_rgnumber_t)-1) + +#define NULLCOMMITLSN ((xfs_lsn_t)-1) + +#define NULLFSINO ((xfs_ino_t)-1) +#define NULLAGINO ((xfs_agino_t)-1) + +/* + * Minimum and maximum blocksize and sectorsize. + * The blocksize upper limit is pretty much arbitrary. + * The sectorsize upper limit is due to sizeof(sb_sectsize). + * CRC enable filesystems use 512 byte inodes, meaning 512 byte block sizes + * cannot be used. + */ +#define XFS_MIN_BLOCKSIZE_LOG 9 /* i.e. 512 bytes */ +#define XFS_MAX_BLOCKSIZE_LOG 16 /* i.e. 65536 bytes */ +#define XFS_MIN_BLOCKSIZE (1 << XFS_MIN_BLOCKSIZE_LOG) +#define XFS_MAX_BLOCKSIZE (1 << XFS_MAX_BLOCKSIZE_LOG) +#define XFS_MIN_CRC_BLOCKSIZE (1 << (XFS_MIN_BLOCKSIZE_LOG + 1)) +#define XFS_MIN_SECTORSIZE_LOG 9 /* i.e. 512 bytes */ +#define XFS_MAX_SECTORSIZE_LOG 15 /* i.e. 32768 bytes */ +#define XFS_MIN_SECTORSIZE (1 << XFS_MIN_SECTORSIZE_LOG) +#define XFS_MAX_SECTORSIZE (1 << XFS_MAX_SECTORSIZE_LOG) + +/* + * Inode fork identifiers. + */ +#define XFS_STAGING_FORK (-1) /* fake fork for staging a btree */ +#define XFS_DATA_FORK (0) +#define XFS_ATTR_FORK (1) +#define XFS_COW_FORK (2) + +#define XFS_WHICHFORK_STRINGS \ + { XFS_STAGING_FORK, "staging" }, \ + { XFS_DATA_FORK, "data" }, \ + { XFS_ATTR_FORK, "attr" }, \ + { XFS_COW_FORK, "cow" } + +/* + * Min numbers of data/attr fork btree root pointers. + */ +#define MINDBTPTRS 3 +#define MINABTPTRS 2 + +/* + * MAXNAMELEN is the length (including the terminating null) of + * the longest permissible file (component) name. + */ +#define MAXNAMELEN 256 + +/* + * This enum is used in string mapping in xfs_trace.h; please keep the + * TRACE_DEFINE_ENUMs for it up to date. + */ +typedef enum { + XFS_LOOKUP_EQi, XFS_LOOKUP_LEi, XFS_LOOKUP_GEi +} xfs_lookup_t; + +#define XFS_AG_BTREE_CMP_FORMAT_STR \ + { XFS_LOOKUP_EQi, "eq" }, \ + { XFS_LOOKUP_LEi, "le" }, \ + { XFS_LOOKUP_GEi, "ge" } + +struct xfs_name { + const unsigned char *name; + int len; + int type; +}; + +/* + * uid_t and gid_t are hard-coded to 32 bits in the inode. + * Hence, an 'id' in a dquot is 32 bits.. + */ +typedef uint32_t xfs_dqid_t; + +/* + * Constants for bit manipulations. + */ +#define XFS_NBBYLOG 3 /* log2(NBBY) */ +#define XFS_WORDLOG 2 /* log2(sizeof(xfs_rtword_t)) */ +#define XFS_SUMINFOLOG 2 /* log2(sizeof(xfs_suminfo_t)) */ +#define XFS_NBWORDLOG (XFS_NBBYLOG + XFS_WORDLOG) +#define XFS_NBWORD (1 << XFS_NBWORDLOG) +#define XFS_WORDMASK ((1 << XFS_WORDLOG) - 1) + +struct xfs_iext_cursor { + struct xfs_iext_leaf *leaf; + int pos; +}; + +typedef enum { + XFS_EXT_NORM, XFS_EXT_UNWRITTEN, +} xfs_exntst_t; + +typedef struct xfs_bmbt_irec +{ + xfs_fileoff_t br_startoff; /* starting file offset */ + xfs_fsblock_t br_startblock; /* starting block number */ + xfs_filblks_t br_blockcount; /* number of blocks */ + xfs_exntst_t br_state; /* extent state */ +} xfs_bmbt_irec_t; + +enum xfs_refc_domain { + XFS_REFC_DOMAIN_SHARED = 0, + XFS_REFC_DOMAIN_COW, +}; + +#define XFS_REFC_DOMAIN_STRINGS \ + { XFS_REFC_DOMAIN_SHARED, "shared" }, \ + { XFS_REFC_DOMAIN_COW, "cow" } + +struct xfs_refcount_irec { + xfs_agblock_t rc_startblock; /* starting block number */ + xfs_extlen_t rc_blockcount; /* count of free blocks */ + xfs_nlink_t rc_refcount; /* number of inodes linked here */ + enum xfs_refc_domain rc_domain; /* shared or cow staging extent? */ +}; + +#define XFS_RMAP_ATTR_FORK (1 << 0) +#define XFS_RMAP_BMBT_BLOCK (1 << 1) +#define XFS_RMAP_UNWRITTEN (1 << 2) +#define XFS_RMAP_KEY_FLAGS (XFS_RMAP_ATTR_FORK | \ + XFS_RMAP_BMBT_BLOCK) +#define XFS_RMAP_REC_FLAGS (XFS_RMAP_UNWRITTEN) +struct xfs_rmap_irec { + xfs_agblock_t rm_startblock; /* extent start block */ + xfs_extlen_t rm_blockcount; /* extent length */ + uint64_t rm_owner; /* extent owner */ + uint64_t rm_offset; /* offset within the owner */ + unsigned int rm_flags; /* state flags */ +}; + +/* per-AG block reservation types */ +enum xfs_ag_resv_type { + XFS_AG_RESV_NONE = 0, + XFS_AG_RESV_AGFL, + XFS_AG_RESV_METADATA, + XFS_AG_RESV_RMAPBT, + + /* + * Don't increase fdblocks when freeing extent. This is a pony for + * the bnobt repair functions to re-free the free space without + * altering fdblocks. If you think you need this you're wrong. + */ + XFS_AG_RESV_IGNORE, + + /* + * This allocation activity is being done on behalf of a metadata file. + * These files maintain their own permanent space reservations and are + * required to adjust fdblocks using the xfs_metafile_resv_* helpers. + */ + XFS_AG_RESV_METAFILE, +}; + +/* Results of scanning a btree keyspace to check occupancy. */ +enum xbtree_recpacking { + /* None of the keyspace maps to records. */ + XBTREE_RECPACKING_EMPTY = 0, + + /* Some, but not all, of the keyspace maps to records. */ + XBTREE_RECPACKING_SPARSE, + + /* The entire keyspace maps to records. */ + XBTREE_RECPACKING_FULL, +}; + +enum xfs_group_type { + XG_TYPE_AG, + XG_TYPE_RTG, + XG_TYPE_MAX, +} __packed; + +#define XG_TYPE_STRINGS \ + { XG_TYPE_AG, "ag" }, \ + { XG_TYPE_RTG, "rtg" } + +enum xfs_free_counter { + /* + * Number of free blocks on the data device. + */ + XC_FREE_BLOCKS, + + /* + * Number of free RT extents on the RT device. + */ + XC_FREE_RTEXTENTS, + + /* + * Number of available for use RT extents. + * + * This counter only exists for zoned RT device and indicates the number + * of RT extents that can be directly used by writes. XC_FREE_RTEXTENTS + * also includes blocks that have been written previously and freed, but + * sit in a rtgroup that still needs a zone reset. + */ + XC_FREE_RTAVAILABLE, + XC_FREE_NR, +}; + +#define XFS_FREECOUNTER_STR \ + { XC_FREE_BLOCKS, "blocks" }, \ + { XC_FREE_RTEXTENTS, "rtextents" }, \ + { XC_FREE_RTAVAILABLE, "rtavailable" } + +/* + * Type verifier functions + */ +struct xfs_mount; + +bool xfs_verify_fsbno(struct xfs_mount *mp, xfs_fsblock_t fsbno); +bool xfs_verify_fsbext(struct xfs_mount *mp, xfs_fsblock_t fsbno, + xfs_fsblock_t len); + +bool xfs_verify_ino(struct xfs_mount *mp, xfs_ino_t ino); +bool xfs_is_sb_inum(struct xfs_mount *mp, xfs_ino_t ino); +bool xfs_verify_dir_ino(struct xfs_mount *mp, xfs_ino_t ino); +bool xfs_verify_rtbno(struct xfs_mount *mp, xfs_rtblock_t rtbno); +bool xfs_verify_rtbext(struct xfs_mount *mp, xfs_rtblock_t rtbno, + xfs_filblks_t len); +bool xfs_verify_icount(struct xfs_mount *mp, unsigned long long icount); +bool xfs_verify_dablk(struct xfs_mount *mp, xfs_fileoff_t off); +void xfs_icount_range(struct xfs_mount *mp, unsigned long long *min, + unsigned long long *max); +bool xfs_verify_fileoff(struct xfs_mount *mp, xfs_fileoff_t off); +bool xfs_verify_fileext(struct xfs_mount *mp, xfs_fileoff_t off, + xfs_fileoff_t len); + +#endif /* __XFS_TYPES_H__ */ diff --git a/fs/xfs/libxfs/xfs_zones.c b/fs/xfs/libxfs/xfs_zones.c new file mode 100644 index 000000000..24e350c31 --- /dev/null +++ b/fs/xfs/libxfs/xfs_zones.c @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2023-2025 Christoph Hellwig. + * Copyright (c) 2024-2025, Western Digital Corporation or its affiliates. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_rtgroup.h" +#include "xfs_zones.h" + +static bool +xfs_validate_blk_zone_seq( + struct xfs_mount *mp, + struct blk_zone *zone, + unsigned int zone_no, + xfs_rgblock_t *write_pointer) +{ + switch (zone->cond) { + case BLK_ZONE_COND_EMPTY: + *write_pointer = 0; + return true; + case BLK_ZONE_COND_IMP_OPEN: + case BLK_ZONE_COND_EXP_OPEN: + case BLK_ZONE_COND_CLOSED: + case BLK_ZONE_COND_ACTIVE: + if (zone->wp < zone->start || + zone->wp >= zone->start + zone->capacity) { + xfs_warn(mp, + "zone %u write pointer (%llu) outside of zone.", + zone_no, zone->wp); + return false; + } + + *write_pointer = XFS_BB_TO_FSB(mp, zone->wp - zone->start); + return true; + case BLK_ZONE_COND_FULL: + *write_pointer = XFS_BB_TO_FSB(mp, zone->capacity); + return true; + case BLK_ZONE_COND_NOT_WP: + case BLK_ZONE_COND_OFFLINE: + case BLK_ZONE_COND_READONLY: + xfs_warn(mp, "zone %u has unsupported zone condition 0x%x.", + zone_no, zone->cond); + return false; + default: + xfs_warn(mp, "zone %u has unknown zone condition 0x%x.", + zone_no, zone->cond); + return false; + } +} + +static bool +xfs_validate_blk_zone_conv( + struct xfs_mount *mp, + struct blk_zone *zone, + unsigned int zone_no) +{ + switch (zone->cond) { + case BLK_ZONE_COND_NOT_WP: + return true; + default: + xfs_warn(mp, +"conventional zone %u has unsupported zone condition 0x%x.", + zone_no, zone->cond); + return false; + } +} + +bool +xfs_validate_blk_zone( + struct xfs_mount *mp, + struct blk_zone *zone, + unsigned int zone_no, + uint32_t expected_size, + uint32_t expected_capacity, + xfs_rgblock_t *write_pointer) +{ + /* + * Check that the zone capacity matches the rtgroup size stored in the + * superblock. Note that all zones including the last one must have a + * uniform capacity. + */ + if (XFS_BB_TO_FSB(mp, zone->capacity) != expected_capacity) { + xfs_warn(mp, +"zone %u capacity (%llu) does not match RT group size (%u).", + zone_no, XFS_BB_TO_FSB(mp, zone->capacity), + expected_capacity); + return false; + } + + if (XFS_BB_TO_FSB(mp, zone->len) != expected_size) { + xfs_warn(mp, +"zone %u length (%llu) does not match geometry (%u).", + zone_no, XFS_BB_TO_FSB(mp, zone->len), + expected_size); + return false; + } + + switch (zone->type) { + case BLK_ZONE_TYPE_CONVENTIONAL: + return xfs_validate_blk_zone_conv(mp, zone, zone_no); + case BLK_ZONE_TYPE_SEQWRITE_REQ: + return xfs_validate_blk_zone_seq(mp, zone, zone_no, + write_pointer); + default: + xfs_warn(mp, "zoned %u has unsupported type 0x%x.", + zone_no, zone->type); + return false; + } +} diff --git a/fs/xfs/libxfs/xfs_zones.h b/fs/xfs/libxfs/xfs_zones.h new file mode 100644 index 000000000..c16089c9a --- /dev/null +++ b/fs/xfs/libxfs/xfs_zones.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LIBXFS_ZONES_H +#define _LIBXFS_ZONES_H + +struct xfs_rtgroup; +struct blk_zone; + +/* + * In order to guarantee forward progress for GC we need to reserve at least + * two zones: one that will be used for moving data into and one spare zone + * making sure that we have enough space to relocate a nearly-full zone. + * To allow for slightly sloppy accounting for when we need to reserve the + * second zone, we actually reserve three as that is easier than doing fully + * accurate bookkeeping. + */ +#define XFS_GC_ZONES 3U + +/* + * In addition we need two zones for user writes, one open zone for writing + * and one to still have available blocks without resetting the open zone + * when data in the open zone has been freed. + */ +#define XFS_RESERVED_ZONES (XFS_GC_ZONES + 1) +#define XFS_MIN_ZONES (XFS_RESERVED_ZONES + 1) + +/* + * Always keep one zone out of the general open zone pool to allow for GC to + * happen while other writers are waiting for free space. + */ +#define XFS_OPEN_GC_ZONES 1U +#define XFS_MIN_OPEN_ZONES (XFS_OPEN_GC_ZONES + 1U) + +/* + * For zoned devices that do not have a limit on the number of open zones, and + * for regular devices using the zoned allocator, use the most common SMR disks + * limit (128) as the default limit on the number of open zones. + */ +#define XFS_DEFAULT_MAX_OPEN_ZONES 128 + +bool xfs_validate_blk_zone(struct xfs_mount *mp, struct blk_zone *zone, + unsigned int zone_no, uint32_t expected_size, + uint32_t expected_capacity, xfs_rgblock_t *write_pointer); + +#endif /* _LIBXFS_ZONES_H */ diff --git a/fs/xfs/scrub/agb_bitmap.c b/fs/xfs/scrub/agb_bitmap.c new file mode 100644 index 000000000..0194e3aaa --- /dev/null +++ b/fs/xfs/scrub/agb_bitmap.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_shared.h" +#include "xfs_bit.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_btree.h" +#include "bitmap.h" +#include "scrub/agb_bitmap.h" + +/* + * Record all btree blocks seen while iterating all records of a btree. + * + * We know that the btree query_all function starts at the left edge and walks + * towards the right edge of the tree. Therefore, we know that we can walk up + * the btree cursor towards the root; if the pointer for a given level points + * to the first record/key in that block, we haven't seen this block before; + * and therefore we need to remember that we saw this block in the btree. + * + * So if our btree is: + * + * 4 + * / | \ + * 1 2 3 + * + * Pretend for this example that each leaf block has 100 btree records. For + * the first btree record, we'll observe that bc_levels[0].ptr == 1, so we + * record that we saw block 1. Then we observe that bc_levels[1].ptr == 1, so + * we record block 4. The list is [1, 4]. + * + * For the second btree record, we see that bc_levels[0].ptr == 2, so we exit + * the loop. The list remains [1, 4]. + * + * For the 101st btree record, we've moved onto leaf block 2. Now + * bc_levels[0].ptr == 1 again, so we record that we saw block 2. We see that + * bc_levels[1].ptr == 2, so we exit the loop. The list is now [1, 4, 2]. + * + * For the 102nd record, bc_levels[0].ptr == 2, so we continue. + * + * For the 201st record, we've moved on to leaf block 3. + * bc_levels[0].ptr == 1, so we add 3 to the list. Now it is [1, 4, 2, 3]. + * + * For the 300th record we just exit, with the list being [1, 4, 2, 3]. + */ + +/* Mark a btree block to the agblock bitmap. */ +STATIC int +xagb_bitmap_visit_btblock( + struct xfs_btree_cur *cur, + int level, + void *priv) +{ + struct xagb_bitmap *bitmap = priv; + struct xfs_buf *bp; + xfs_fsblock_t fsbno; + xfs_agblock_t agbno; + + xfs_btree_get_block(cur, level, &bp); + if (!bp) + return 0; + + fsbno = XFS_DADDR_TO_FSB(cur->bc_mp, xfs_buf_daddr(bp)); + agbno = XFS_FSB_TO_AGBNO(cur->bc_mp, fsbno); + + return xagb_bitmap_set(bitmap, agbno, 1); +} + +/* Mark all (per-AG) btree blocks in the agblock bitmap. */ +int +xagb_bitmap_set_btblocks( + struct xagb_bitmap *bitmap, + struct xfs_btree_cur *cur) +{ + return xfs_btree_visit_blocks(cur, xagb_bitmap_visit_btblock, + XFS_BTREE_VISIT_ALL, bitmap); +} + +/* + * Record all the buffers pointed to by the btree cursor. Callers already + * engaged in a btree walk should call this function to capture the list of + * blocks going from the leaf towards the root. + */ +int +xagb_bitmap_set_btcur_path( + struct xagb_bitmap *bitmap, + struct xfs_btree_cur *cur) +{ + int i; + int error; + + for (i = 0; i < cur->bc_nlevels && cur->bc_levels[i].ptr == 1; i++) { + error = xagb_bitmap_visit_btblock(cur, i, bitmap); + if (error) + return error; + } + + return 0; +} diff --git a/fs/xfs/scrub/agb_bitmap.h b/fs/xfs/scrub/agb_bitmap.h new file mode 100644 index 000000000..e488e1f4f --- /dev/null +++ b/fs/xfs/scrub/agb_bitmap.h @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_AGB_BITMAP_H__ +#define __XFS_SCRUB_AGB_BITMAP_H__ + +/* Bitmaps, but for type-checked for xfs_agblock_t */ + +struct xagb_bitmap { + struct xbitmap32 agbitmap; +}; + +static inline void xagb_bitmap_init(struct xagb_bitmap *bitmap) +{ + xbitmap32_init(&bitmap->agbitmap); +} + +static inline void xagb_bitmap_destroy(struct xagb_bitmap *bitmap) +{ + xbitmap32_destroy(&bitmap->agbitmap); +} + +static inline int xagb_bitmap_clear(struct xagb_bitmap *bitmap, + xfs_agblock_t start, xfs_extlen_t len) +{ + return xbitmap32_clear(&bitmap->agbitmap, start, len); +} +static inline int xagb_bitmap_set(struct xagb_bitmap *bitmap, + xfs_agblock_t start, xfs_extlen_t len) +{ + return xbitmap32_set(&bitmap->agbitmap, start, len); +} + +static inline bool xagb_bitmap_test(struct xagb_bitmap *bitmap, + xfs_agblock_t start, xfs_extlen_t *len) +{ + return xbitmap32_test(&bitmap->agbitmap, start, len); +} + +static inline int xagb_bitmap_disunion(struct xagb_bitmap *bitmap, + struct xagb_bitmap *sub) +{ + return xbitmap32_disunion(&bitmap->agbitmap, &sub->agbitmap); +} + +static inline uint32_t xagb_bitmap_hweight(struct xagb_bitmap *bitmap) +{ + return xbitmap32_hweight(&bitmap->agbitmap); +} +static inline bool xagb_bitmap_empty(struct xagb_bitmap *bitmap) +{ + return xbitmap32_empty(&bitmap->agbitmap); +} + +static inline int xagb_bitmap_walk(struct xagb_bitmap *bitmap, + xbitmap32_walk_fn fn, void *priv) +{ + return xbitmap32_walk(&bitmap->agbitmap, fn, priv); +} + +int xagb_bitmap_set_btblocks(struct xagb_bitmap *bitmap, + struct xfs_btree_cur *cur); +int xagb_bitmap_set_btcur_path(struct xagb_bitmap *bitmap, + struct xfs_btree_cur *cur); + +static inline uint32_t xagb_bitmap_count_set_regions(struct xagb_bitmap *b) +{ + return xbitmap32_count_set_regions(&b->agbitmap); +} + +#endif /* __XFS_SCRUB_AGB_BITMAP_H__ */ diff --git a/fs/xfs/scrub/agheader.c b/fs/xfs/scrub/agheader.c new file mode 100644 index 000000000..1fa66aa68 --- /dev/null +++ b/fs/xfs/scrub/agheader.c @@ -0,0 +1,1109 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_btree.h" +#include "xfs_sb.h" +#include "xfs_alloc.h" +#include "xfs_ialloc.h" +#include "xfs_rmap.h" +#include "xfs_ag.h" +#include "xfs_inode.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/bitmap.h" +#include "scrub/agino_bitmap.h" + +int +xchk_setup_agheader( + struct xfs_scrub *sc) +{ + if (xchk_need_intent_drain(sc)) + xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN); + return xchk_setup_fs(sc); +} + +/* Superblock */ + +/* Cross-reference with the other btrees. */ +STATIC void +xchk_superblock_xref( + struct xfs_scrub *sc, + struct xfs_buf *bp) +{ + struct xfs_mount *mp = sc->mp; + xfs_agnumber_t agno = sc->sm->sm_agno; + xfs_agblock_t agbno; + int error; + + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return; + + agbno = XFS_SB_BLOCK(mp); + + error = xchk_ag_init_existing(sc, agno, &sc->sa); + if (!xchk_xref_process_error(sc, agno, agbno, &error)) + return; + + xchk_xref_is_used_space(sc, agbno, 1); + xchk_xref_is_not_inode_chunk(sc, agbno, 1); + xchk_xref_is_only_owned_by(sc, agbno, 1, &XFS_RMAP_OINFO_FS); + xchk_xref_is_not_shared(sc, agbno, 1); + xchk_xref_is_not_cow_staging(sc, agbno, 1); + + /* scrub teardown will take care of sc->sa for us */ +} + +/* + * Calculate the ondisk superblock size in bytes given the feature set of the + * mounted filesystem (aka the primary sb). This is subtlely different from + * the logic in xfs_repair, which computes the size of a secondary sb given the + * featureset listed in the secondary sb. + */ +STATIC size_t +xchk_superblock_ondisk_size( + struct xfs_mount *mp) +{ + if (xfs_has_zoned(mp)) + return offsetofend(struct xfs_dsb, sb_rtreserved); + if (xfs_has_metadir(mp)) + return offsetofend(struct xfs_dsb, sb_pad); + if (xfs_has_metauuid(mp)) + return offsetofend(struct xfs_dsb, sb_meta_uuid); + if (xfs_has_crc(mp)) + return offsetofend(struct xfs_dsb, sb_lsn); + if (xfs_sb_version_hasmorebits(&mp->m_sb)) + return offsetofend(struct xfs_dsb, sb_bad_features2); + if (xfs_has_logv2(mp)) + return offsetofend(struct xfs_dsb, sb_logsunit); + if (xfs_has_sector(mp)) + return offsetofend(struct xfs_dsb, sb_logsectsize); + /* only support dirv2 or more recent */ + return offsetofend(struct xfs_dsb, sb_dirblklog); +} + +/* + * Scrub the filesystem superblock. + * + * Note: We do /not/ attempt to check AG 0's superblock. Mount is + * responsible for validating all the geometry information in sb 0, so + * if the filesystem is capable of initiating online scrub, then clearly + * sb 0 is ok and we can use its information to check everything else. + */ +int +xchk_superblock( + struct xfs_scrub *sc) +{ + struct xfs_mount *mp = sc->mp; + struct xfs_buf *bp; + struct xfs_dsb *sb; + struct xfs_perag *pag; + size_t sblen; + xfs_agnumber_t agno; + uint32_t v2_ok; + __be32 features_mask; + int error; + __be16 vernum_mask; + + agno = sc->sm->sm_agno; + if (agno == 0) + return 0; + + /* + * Grab an active reference to the perag structure. If we can't get + * it, we're racing with something that's tearing down the AG, so + * signal that the AG no longer exists. + */ + pag = xfs_perag_get(mp, agno); + if (!pag) + return -ENOENT; + + error = xfs_sb_read_secondary(mp, sc->tp, agno, &bp); + /* + * The superblock verifier can return several different error codes + * if it thinks the superblock doesn't look right. For a mount these + * would all get bounced back to userspace, but if we're here then the + * fs mounted successfully, which means that this secondary superblock + * is simply incorrect. Treat all these codes the same way we treat + * any corruption. + */ + switch (error) { + case -EINVAL: /* also -EWRONGFS */ + case -ENOSYS: + case -EFBIG: + error = -EFSCORRUPTED; + fallthrough; + default: + break; + } + if (!xchk_process_error(sc, agno, XFS_SB_BLOCK(mp), &error)) + goto out_pag; + + sb = bp->b_addr; + + /* + * Verify the geometries match. Fields that are permanently + * set by mkfs are checked; fields that can be updated later + * (and are not propagated to backup superblocks) are preen + * checked. + */ + if (sb->sb_blocksize != cpu_to_be32(mp->m_sb.sb_blocksize)) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_dblocks != cpu_to_be64(mp->m_sb.sb_dblocks)) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_rblocks != cpu_to_be64(mp->m_sb.sb_rblocks)) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_rextents != cpu_to_be64(mp->m_sb.sb_rextents)) + xchk_block_set_corrupt(sc, bp); + + if (!uuid_equal(&sb->sb_uuid, &mp->m_sb.sb_uuid)) + xchk_block_set_preen(sc, bp); + + if (sb->sb_logstart != cpu_to_be64(mp->m_sb.sb_logstart)) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_rootino != cpu_to_be64(mp->m_sb.sb_rootino)) + xchk_block_set_preen(sc, bp); + + if (xfs_has_metadir(sc->mp)) { + if (sb->sb_rbmino != cpu_to_be64(0)) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_rsumino != cpu_to_be64(0)) + xchk_block_set_corrupt(sc, bp); + } else { + if (sb->sb_rbmino != cpu_to_be64(mp->m_sb.sb_rbmino)) + xchk_block_set_preen(sc, bp); + + if (sb->sb_rsumino != cpu_to_be64(mp->m_sb.sb_rsumino)) + xchk_block_set_preen(sc, bp); + } + + if (sb->sb_rextsize != cpu_to_be32(mp->m_sb.sb_rextsize)) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_agblocks != cpu_to_be32(mp->m_sb.sb_agblocks)) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_agcount != cpu_to_be32(mp->m_sb.sb_agcount)) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_rbmblocks != cpu_to_be32(mp->m_sb.sb_rbmblocks)) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_logblocks != cpu_to_be32(mp->m_sb.sb_logblocks)) + xchk_block_set_corrupt(sc, bp); + + /* Check sb_versionnum bits that are set at mkfs time. */ + vernum_mask = cpu_to_be16(XFS_SB_VERSION_NUMBITS | + XFS_SB_VERSION_ALIGNBIT | + XFS_SB_VERSION_DALIGNBIT | + XFS_SB_VERSION_SHAREDBIT | + XFS_SB_VERSION_LOGV2BIT | + XFS_SB_VERSION_SECTORBIT | + XFS_SB_VERSION_EXTFLGBIT | + XFS_SB_VERSION_DIRV2BIT); + if ((sb->sb_versionnum & vernum_mask) != + (cpu_to_be16(mp->m_sb.sb_versionnum) & vernum_mask)) + xchk_block_set_corrupt(sc, bp); + + /* Check sb_versionnum bits that can be set after mkfs time. */ + vernum_mask = cpu_to_be16(XFS_SB_VERSION_ATTRBIT | + XFS_SB_VERSION_NLINKBIT | + XFS_SB_VERSION_QUOTABIT); + if ((sb->sb_versionnum & vernum_mask) != + (cpu_to_be16(mp->m_sb.sb_versionnum) & vernum_mask)) + xchk_block_set_preen(sc, bp); + + if (sb->sb_sectsize != cpu_to_be16(mp->m_sb.sb_sectsize)) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_inodesize != cpu_to_be16(mp->m_sb.sb_inodesize)) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_inopblock != cpu_to_be16(mp->m_sb.sb_inopblock)) + xchk_block_set_corrupt(sc, bp); + + if (memcmp(sb->sb_fname, mp->m_sb.sb_fname, sizeof(sb->sb_fname))) + xchk_block_set_preen(sc, bp); + + if (sb->sb_blocklog != mp->m_sb.sb_blocklog) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_sectlog != mp->m_sb.sb_sectlog) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_inodelog != mp->m_sb.sb_inodelog) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_inopblog != mp->m_sb.sb_inopblog) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_agblklog != mp->m_sb.sb_agblklog) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_rextslog != mp->m_sb.sb_rextslog) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_imax_pct != mp->m_sb.sb_imax_pct) + xchk_block_set_preen(sc, bp); + + /* + * Skip the summary counters since we track them in memory anyway. + * sb_icount, sb_ifree, sb_fdblocks, sb_frexents + */ + + if (xfs_has_metadir(mp)) { + if (sb->sb_uquotino != cpu_to_be64(0)) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_gquotino != cpu_to_be64(0)) + xchk_block_set_corrupt(sc, bp); + } else { + if (sb->sb_uquotino != cpu_to_be64(mp->m_sb.sb_uquotino)) + xchk_block_set_preen(sc, bp); + + if (sb->sb_gquotino != cpu_to_be64(mp->m_sb.sb_gquotino)) + xchk_block_set_preen(sc, bp); + } + + /* + * Skip the quota flags since repair will force quotacheck. + * sb_qflags + */ + + if (sb->sb_flags != mp->m_sb.sb_flags) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_shared_vn != mp->m_sb.sb_shared_vn) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_inoalignmt != cpu_to_be32(mp->m_sb.sb_inoalignmt)) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_unit != cpu_to_be32(mp->m_sb.sb_unit)) + xchk_block_set_preen(sc, bp); + + if (sb->sb_width != cpu_to_be32(mp->m_sb.sb_width)) + xchk_block_set_preen(sc, bp); + + if (sb->sb_dirblklog != mp->m_sb.sb_dirblklog) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_logsectlog != mp->m_sb.sb_logsectlog) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_logsectsize != cpu_to_be16(mp->m_sb.sb_logsectsize)) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_logsunit != cpu_to_be32(mp->m_sb.sb_logsunit)) + xchk_block_set_corrupt(sc, bp); + + /* Do we see any invalid bits in sb_features2? */ + if (!xfs_sb_version_hasmorebits(&mp->m_sb)) { + if (sb->sb_features2 != 0) + xchk_block_set_corrupt(sc, bp); + } else { + v2_ok = XFS_SB_VERSION2_OKBITS; + if (xfs_sb_is_v5(&mp->m_sb)) + v2_ok |= XFS_SB_VERSION2_CRCBIT; + + if (!!(sb->sb_features2 & cpu_to_be32(~v2_ok))) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_features2 != sb->sb_bad_features2) + xchk_block_set_preen(sc, bp); + } + + /* Check sb_features2 flags that are set at mkfs time. */ + features_mask = cpu_to_be32(XFS_SB_VERSION2_LAZYSBCOUNTBIT | + XFS_SB_VERSION2_PROJID32BIT | + XFS_SB_VERSION2_CRCBIT | + XFS_SB_VERSION2_FTYPE); + if ((sb->sb_features2 & features_mask) != + (cpu_to_be32(mp->m_sb.sb_features2) & features_mask)) + xchk_block_set_corrupt(sc, bp); + + /* Check sb_features2 flags that can be set after mkfs time. */ + features_mask = cpu_to_be32(XFS_SB_VERSION2_ATTR2BIT); + if ((sb->sb_features2 & features_mask) != + (cpu_to_be32(mp->m_sb.sb_features2) & features_mask)) + xchk_block_set_preen(sc, bp); + + if (!xfs_has_crc(mp)) { + /* all v5 fields must be zero */ + if (memchr_inv(&sb->sb_features_compat, 0, + sizeof(struct xfs_dsb) - + offsetof(struct xfs_dsb, sb_features_compat))) + xchk_block_set_corrupt(sc, bp); + } else { + /* compat features must match */ + if (sb->sb_features_compat != + cpu_to_be32(mp->m_sb.sb_features_compat)) + xchk_block_set_corrupt(sc, bp); + + /* ro compat features must match */ + if (sb->sb_features_ro_compat != + cpu_to_be32(mp->m_sb.sb_features_ro_compat)) + xchk_block_set_corrupt(sc, bp); + + /* + * NEEDSREPAIR is ignored on a secondary super, so we should + * clear it when we find it, though it's not a corruption. + */ + features_mask = cpu_to_be32(XFS_SB_FEAT_INCOMPAT_NEEDSREPAIR); + if ((cpu_to_be32(mp->m_sb.sb_features_incompat) ^ + sb->sb_features_incompat) & features_mask) + xchk_block_set_preen(sc, bp); + + /* all other incompat features must match */ + if ((cpu_to_be32(mp->m_sb.sb_features_incompat) ^ + sb->sb_features_incompat) & ~features_mask) + xchk_block_set_corrupt(sc, bp); + + /* + * log incompat features protect newer log record types from + * older log recovery code. Log recovery doesn't check the + * secondary supers, so we can clear these if needed. + */ + if (sb->sb_features_log_incompat) + xchk_block_set_preen(sc, bp); + + /* Don't care about sb_crc */ + + if (sb->sb_spino_align != cpu_to_be32(mp->m_sb.sb_spino_align)) + xchk_block_set_corrupt(sc, bp); + + if (xfs_has_metadir(mp)) { + if (sb->sb_pquotino != cpu_to_be64(0)) + xchk_block_set_corrupt(sc, bp); + } else { + if (sb->sb_pquotino != cpu_to_be64(mp->m_sb.sb_pquotino)) + xchk_block_set_preen(sc, bp); + } + + /* Don't care about sb_lsn */ + } + + if (xfs_has_metauuid(mp)) { + /* The metadata UUID must be the same for all supers */ + if (!uuid_equal(&sb->sb_meta_uuid, &mp->m_sb.sb_meta_uuid)) + xchk_block_set_corrupt(sc, bp); + } + + if (xfs_has_metadir(mp)) { + if (sb->sb_metadirino != cpu_to_be64(mp->m_sb.sb_metadirino)) + xchk_block_set_preen(sc, bp); + + if (sb->sb_rgcount != cpu_to_be32(mp->m_sb.sb_rgcount)) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_rgextents != cpu_to_be32(mp->m_sb.sb_rgextents)) + xchk_block_set_corrupt(sc, bp); + + if (sb->sb_rgblklog != mp->m_sb.sb_rgblklog) + xchk_block_set_corrupt(sc, bp); + + if (memchr_inv(sb->sb_pad, 0, sizeof(sb->sb_pad))) + xchk_block_set_corrupt(sc, bp); + } + + /* Everything else must be zero. */ + sblen = xchk_superblock_ondisk_size(mp); + if (memchr_inv((char *)sb + sblen, 0, BBTOB(bp->b_length) - sblen)) + xchk_block_set_corrupt(sc, bp); + + xchk_superblock_xref(sc, bp); +out_pag: + xfs_perag_put(pag); + return error; +} + +/* AGF */ + +/* Tally freespace record lengths. */ +STATIC int +xchk_agf_record_bno_lengths( + struct xfs_btree_cur *cur, + const struct xfs_alloc_rec_incore *rec, + void *priv) +{ + xfs_extlen_t *blocks = priv; + + (*blocks) += rec->ar_blockcount; + return 0; +} + +/* Check agf_freeblks */ +static inline void +xchk_agf_xref_freeblks( + struct xfs_scrub *sc) +{ + struct xfs_agf *agf = sc->sa.agf_bp->b_addr; + xfs_extlen_t blocks = 0; + int error; + + if (!sc->sa.bno_cur) + return; + + error = xfs_alloc_query_all(sc->sa.bno_cur, + xchk_agf_record_bno_lengths, &blocks); + if (!xchk_should_check_xref(sc, &error, &sc->sa.bno_cur)) + return; + if (blocks != be32_to_cpu(agf->agf_freeblks)) + xchk_block_xref_set_corrupt(sc, sc->sa.agf_bp); +} + +/* Cross reference the AGF with the cntbt (freespace by length btree) */ +static inline void +xchk_agf_xref_cntbt( + struct xfs_scrub *sc) +{ + struct xfs_agf *agf = sc->sa.agf_bp->b_addr; + xfs_agblock_t agbno; + xfs_extlen_t blocks; + int have; + int error; + + if (!sc->sa.cnt_cur) + return; + + /* Any freespace at all? */ + error = xfs_alloc_lookup_le(sc->sa.cnt_cur, 0, -1U, &have); + if (!xchk_should_check_xref(sc, &error, &sc->sa.cnt_cur)) + return; + if (!have) { + if (agf->agf_freeblks != cpu_to_be32(0)) + xchk_block_xref_set_corrupt(sc, sc->sa.agf_bp); + return; + } + + /* Check agf_longest */ + error = xfs_alloc_get_rec(sc->sa.cnt_cur, &agbno, &blocks, &have); + if (!xchk_should_check_xref(sc, &error, &sc->sa.cnt_cur)) + return; + if (!have || blocks != be32_to_cpu(agf->agf_longest)) + xchk_block_xref_set_corrupt(sc, sc->sa.agf_bp); +} + +/* Check the btree block counts in the AGF against the btrees. */ +STATIC void +xchk_agf_xref_btreeblks( + struct xfs_scrub *sc) +{ + struct xfs_agf *agf = sc->sa.agf_bp->b_addr; + struct xfs_mount *mp = sc->mp; + xfs_filblks_t blocks; + xfs_agblock_t btreeblks; + int error; + + /* agf_btreeblks didn't exist before lazysbcount */ + if (!xfs_has_lazysbcount(sc->mp)) + return; + + /* Check agf_rmap_blocks; set up for agf_btreeblks check */ + if (sc->sa.rmap_cur) { + error = xfs_btree_count_blocks(sc->sa.rmap_cur, &blocks); + if (!xchk_should_check_xref(sc, &error, &sc->sa.rmap_cur)) + return; + btreeblks = blocks - 1; + if (blocks != be32_to_cpu(agf->agf_rmap_blocks)) + xchk_block_xref_set_corrupt(sc, sc->sa.agf_bp); + } else { + btreeblks = 0; + } + + /* + * No rmap cursor; we can't xref if we have the rmapbt feature. + * We also can't do it if we're missing the free space btree cursors. + */ + if ((xfs_has_rmapbt(mp) && !sc->sa.rmap_cur) || + !sc->sa.bno_cur || !sc->sa.cnt_cur) + return; + + /* Check agf_btreeblks */ + error = xfs_btree_count_blocks(sc->sa.bno_cur, &blocks); + if (!xchk_should_check_xref(sc, &error, &sc->sa.bno_cur)) + return; + btreeblks += blocks - 1; + + error = xfs_btree_count_blocks(sc->sa.cnt_cur, &blocks); + if (!xchk_should_check_xref(sc, &error, &sc->sa.cnt_cur)) + return; + btreeblks += blocks - 1; + + if (btreeblks != be32_to_cpu(agf->agf_btreeblks)) + xchk_block_xref_set_corrupt(sc, sc->sa.agf_bp); +} + +/* Check agf_refcount_blocks against tree size */ +static inline void +xchk_agf_xref_refcblks( + struct xfs_scrub *sc) +{ + struct xfs_agf *agf = sc->sa.agf_bp->b_addr; + xfs_filblks_t blocks; + int error; + + if (!sc->sa.refc_cur) + return; + + error = xfs_btree_count_blocks(sc->sa.refc_cur, &blocks); + if (!xchk_should_check_xref(sc, &error, &sc->sa.refc_cur)) + return; + if (blocks != be32_to_cpu(agf->agf_refcount_blocks)) + xchk_block_xref_set_corrupt(sc, sc->sa.agf_bp); +} + +/* Cross-reference with the other btrees. */ +STATIC void +xchk_agf_xref( + struct xfs_scrub *sc) +{ + struct xfs_mount *mp = sc->mp; + xfs_agblock_t agbno; + + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return; + + agbno = XFS_AGF_BLOCK(mp); + + xchk_ag_btcur_init(sc, &sc->sa); + + xchk_xref_is_used_space(sc, agbno, 1); + xchk_agf_xref_freeblks(sc); + xchk_agf_xref_cntbt(sc); + xchk_xref_is_not_inode_chunk(sc, agbno, 1); + xchk_xref_is_only_owned_by(sc, agbno, 1, &XFS_RMAP_OINFO_FS); + xchk_agf_xref_btreeblks(sc); + xchk_xref_is_not_shared(sc, agbno, 1); + xchk_xref_is_not_cow_staging(sc, agbno, 1); + xchk_agf_xref_refcblks(sc); + + /* scrub teardown will take care of sc->sa for us */ +} + +/* Scrub the AGF. */ +int +xchk_agf( + struct xfs_scrub *sc) +{ + struct xfs_mount *mp = sc->mp; + struct xfs_agf *agf; + struct xfs_perag *pag; + xfs_agnumber_t agno = sc->sm->sm_agno; + xfs_agblock_t agbno; + xfs_agblock_t eoag; + xfs_agblock_t agfl_first; + xfs_agblock_t agfl_last; + xfs_agblock_t agfl_count; + xfs_agblock_t fl_count; + int level; + int error = 0; + + error = xchk_ag_read_headers(sc, agno, &sc->sa); + if (!xchk_process_error(sc, agno, XFS_AGF_BLOCK(sc->mp), &error)) + goto out; + xchk_buffer_recheck(sc, sc->sa.agf_bp); + + agf = sc->sa.agf_bp->b_addr; + pag = sc->sa.pag; + + /* Check the AG length */ + eoag = be32_to_cpu(agf->agf_length); + if (eoag != pag_group(pag)->xg_block_count) + xchk_block_set_corrupt(sc, sc->sa.agf_bp); + + /* Check the AGF btree roots and levels */ + agbno = be32_to_cpu(agf->agf_bno_root); + if (!xfs_verify_agbno(pag, agbno)) + xchk_block_set_corrupt(sc, sc->sa.agf_bp); + + agbno = be32_to_cpu(agf->agf_cnt_root); + if (!xfs_verify_agbno(pag, agbno)) + xchk_block_set_corrupt(sc, sc->sa.agf_bp); + + level = be32_to_cpu(agf->agf_bno_level); + if (level <= 0 || level > mp->m_alloc_maxlevels) + xchk_block_set_corrupt(sc, sc->sa.agf_bp); + + level = be32_to_cpu(agf->agf_cnt_level); + if (level <= 0 || level > mp->m_alloc_maxlevels) + xchk_block_set_corrupt(sc, sc->sa.agf_bp); + + if (xfs_has_rmapbt(mp)) { + agbno = be32_to_cpu(agf->agf_rmap_root); + if (!xfs_verify_agbno(pag, agbno)) + xchk_block_set_corrupt(sc, sc->sa.agf_bp); + + level = be32_to_cpu(agf->agf_rmap_level); + if (level <= 0 || level > mp->m_rmap_maxlevels) + xchk_block_set_corrupt(sc, sc->sa.agf_bp); + } + + if (xfs_has_reflink(mp)) { + agbno = be32_to_cpu(agf->agf_refcount_root); + if (!xfs_verify_agbno(pag, agbno)) + xchk_block_set_corrupt(sc, sc->sa.agf_bp); + + level = be32_to_cpu(agf->agf_refcount_level); + if (level <= 0 || level > mp->m_refc_maxlevels) + xchk_block_set_corrupt(sc, sc->sa.agf_bp); + } + + /* Check the AGFL counters */ + agfl_first = be32_to_cpu(agf->agf_flfirst); + agfl_last = be32_to_cpu(agf->agf_fllast); + agfl_count = be32_to_cpu(agf->agf_flcount); + if (agfl_last > agfl_first) + fl_count = agfl_last - agfl_first + 1; + else + fl_count = xfs_agfl_size(mp) - agfl_first + agfl_last + 1; + if (agfl_count != 0 && fl_count != agfl_count) + xchk_block_set_corrupt(sc, sc->sa.agf_bp); + + /* Do the incore counters match? */ + if (pag->pagf_freeblks != be32_to_cpu(agf->agf_freeblks)) + xchk_block_set_corrupt(sc, sc->sa.agf_bp); + if (pag->pagf_flcount != be32_to_cpu(agf->agf_flcount)) + xchk_block_set_corrupt(sc, sc->sa.agf_bp); + if (xfs_has_lazysbcount(sc->mp) && + pag->pagf_btreeblks != be32_to_cpu(agf->agf_btreeblks)) + xchk_block_set_corrupt(sc, sc->sa.agf_bp); + + xchk_agf_xref(sc); +out: + return error; +} + +/* AGFL */ + +struct xchk_agfl_info { + /* Number of AGFL entries that the AGF claims are in use. */ + unsigned int agflcount; + + /* Number of AGFL entries that we found. */ + unsigned int nr_entries; + + /* Buffer to hold AGFL entries for extent checking. */ + xfs_agblock_t *entries; + + struct xfs_buf *agfl_bp; + struct xfs_scrub *sc; +}; + +/* Cross-reference with the other btrees. */ +STATIC void +xchk_agfl_block_xref( + struct xfs_scrub *sc, + xfs_agblock_t agbno) +{ + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return; + + xchk_xref_is_used_space(sc, agbno, 1); + xchk_xref_is_not_inode_chunk(sc, agbno, 1); + xchk_xref_is_only_owned_by(sc, agbno, 1, &XFS_RMAP_OINFO_AG); + xchk_xref_is_not_shared(sc, agbno, 1); + xchk_xref_is_not_cow_staging(sc, agbno, 1); +} + +/* Scrub an AGFL block. */ +STATIC int +xchk_agfl_block( + struct xfs_mount *mp, + xfs_agblock_t agbno, + void *priv) +{ + struct xchk_agfl_info *sai = priv; + struct xfs_scrub *sc = sai->sc; + + if (xfs_verify_agbno(sc->sa.pag, agbno) && + sai->nr_entries < sai->agflcount) + sai->entries[sai->nr_entries++] = agbno; + else + xchk_block_set_corrupt(sc, sai->agfl_bp); + + xchk_agfl_block_xref(sc, agbno); + + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return -ECANCELED; + + return 0; +} + +static int +xchk_agblock_cmp( + const void *pa, + const void *pb) +{ + const xfs_agblock_t *a = pa; + const xfs_agblock_t *b = pb; + + return (int)*a - (int)*b; +} + +/* Cross-reference with the other btrees. */ +STATIC void +xchk_agfl_xref( + struct xfs_scrub *sc) +{ + struct xfs_mount *mp = sc->mp; + xfs_agblock_t agbno; + + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return; + + agbno = XFS_AGFL_BLOCK(mp); + + xchk_ag_btcur_init(sc, &sc->sa); + + xchk_xref_is_used_space(sc, agbno, 1); + xchk_xref_is_not_inode_chunk(sc, agbno, 1); + xchk_xref_is_only_owned_by(sc, agbno, 1, &XFS_RMAP_OINFO_FS); + xchk_xref_is_not_shared(sc, agbno, 1); + xchk_xref_is_not_cow_staging(sc, agbno, 1); + + /* + * Scrub teardown will take care of sc->sa for us. Leave sc->sa + * active so that the agfl block xref can use it too. + */ +} + +/* Scrub the AGFL. */ +int +xchk_agfl( + struct xfs_scrub *sc) +{ + struct xchk_agfl_info sai = { + .sc = sc, + }; + struct xfs_agf *agf; + xfs_agnumber_t agno = sc->sm->sm_agno; + unsigned int i; + int error; + + /* Lock the AGF and AGI so that nobody can touch this AG. */ + error = xchk_ag_read_headers(sc, agno, &sc->sa); + if (!xchk_process_error(sc, agno, XFS_AGFL_BLOCK(sc->mp), &error)) + return error; + if (!sc->sa.agf_bp) + return -EFSCORRUPTED; + + /* Try to read the AGFL, and verify its structure if we get it. */ + error = xfs_alloc_read_agfl(sc->sa.pag, sc->tp, &sai.agfl_bp); + if (!xchk_process_error(sc, agno, XFS_AGFL_BLOCK(sc->mp), &error)) + return error; + xchk_buffer_recheck(sc, sai.agfl_bp); + + xchk_agfl_xref(sc); + + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out; + + /* Allocate buffer to ensure uniqueness of AGFL entries. */ + agf = sc->sa.agf_bp->b_addr; + sai.agflcount = be32_to_cpu(agf->agf_flcount); + if (sai.agflcount > xfs_agfl_size(sc->mp)) { + xchk_block_set_corrupt(sc, sc->sa.agf_bp); + goto out; + } + sai.entries = kvzalloc_objs(xfs_agblock_t, sai.agflcount, + XCHK_GFP_FLAGS); + if (!sai.entries) { + error = -ENOMEM; + goto out; + } + + /* Check the blocks in the AGFL. */ + error = xfs_agfl_walk(sc->mp, sc->sa.agf_bp->b_addr, sai.agfl_bp, + xchk_agfl_block, &sai); + if (error == -ECANCELED) { + error = 0; + goto out_free; + } + if (error) + goto out_free; + + if (sai.agflcount != sai.nr_entries) { + xchk_block_set_corrupt(sc, sc->sa.agf_bp); + goto out_free; + } + + /* Sort entries, check for duplicates. */ + sort(sai.entries, sai.nr_entries, sizeof(sai.entries[0]), + xchk_agblock_cmp, NULL); + for (i = 1; i < sai.nr_entries; i++) { + if (sai.entries[i] == sai.entries[i - 1]) { + xchk_block_set_corrupt(sc, sc->sa.agf_bp); + break; + } + } + +out_free: + kvfree(sai.entries); +out: + return error; +} + +/* AGI */ + +/* Check agi_count/agi_freecount */ +static inline void +xchk_agi_xref_icounts( + struct xfs_scrub *sc) +{ + struct xfs_agi *agi = sc->sa.agi_bp->b_addr; + xfs_agino_t icount; + xfs_agino_t freecount; + int error; + + if (!sc->sa.ino_cur) + return; + + error = xfs_ialloc_count_inodes(sc->sa.ino_cur, &icount, &freecount); + if (!xchk_should_check_xref(sc, &error, &sc->sa.ino_cur)) + return; + if (be32_to_cpu(agi->agi_count) != icount || + be32_to_cpu(agi->agi_freecount) != freecount) + xchk_block_xref_set_corrupt(sc, sc->sa.agi_bp); +} + +/* Check agi_[fi]blocks against tree size */ +static inline void +xchk_agi_xref_fiblocks( + struct xfs_scrub *sc) +{ + struct xfs_agi *agi = sc->sa.agi_bp->b_addr; + xfs_filblks_t blocks; + int error = 0; + + if (!xfs_has_inobtcounts(sc->mp)) + return; + + if (sc->sa.ino_cur) { + error = xfs_btree_count_blocks(sc->sa.ino_cur, &blocks); + if (!xchk_should_check_xref(sc, &error, &sc->sa.ino_cur)) + return; + if (blocks != be32_to_cpu(agi->agi_iblocks)) + xchk_block_xref_set_corrupt(sc, sc->sa.agi_bp); + } + + if (sc->sa.fino_cur) { + error = xfs_btree_count_blocks(sc->sa.fino_cur, &blocks); + if (!xchk_should_check_xref(sc, &error, &sc->sa.fino_cur)) + return; + if (blocks != be32_to_cpu(agi->agi_fblocks)) + xchk_block_xref_set_corrupt(sc, sc->sa.agi_bp); + } +} + +/* Cross-reference with the other btrees. */ +STATIC void +xchk_agi_xref( + struct xfs_scrub *sc) +{ + struct xfs_mount *mp = sc->mp; + xfs_agblock_t agbno; + + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return; + + agbno = XFS_AGI_BLOCK(mp); + + xchk_ag_btcur_init(sc, &sc->sa); + + xchk_xref_is_used_space(sc, agbno, 1); + xchk_xref_is_not_inode_chunk(sc, agbno, 1); + xchk_agi_xref_icounts(sc); + xchk_xref_is_only_owned_by(sc, agbno, 1, &XFS_RMAP_OINFO_FS); + xchk_xref_is_not_shared(sc, agbno, 1); + xchk_xref_is_not_cow_staging(sc, agbno, 1); + xchk_agi_xref_fiblocks(sc); + + /* scrub teardown will take care of sc->sa for us */ +} + +/* + * Walk the incore unlinked list for a particular AGI bucket to construct + * the unlinked inode bitmap for later reconstruction of the unlinked list. + * Returns 1 if we should keep checking, 0 to stop checking, or a negative + * errno. + */ +static int +xchk_iunlink_bucket( + struct xfs_scrub *sc, + unsigned int bucket, + xfs_agino_t agino) +{ + struct xagino_bitmap seen; + int ret; + + xagino_bitmap_init(&seen); + + while (agino != NULLAGINO) { + struct xfs_inode *ip; + unsigned int len = 1; + + if (agino % XFS_AGI_UNLINKED_BUCKETS != bucket) { + xchk_block_set_corrupt(sc, sc->sa.agi_bp); + goto bad; + } + + if (xagino_bitmap_test(&seen, agino, &len)) { + xchk_block_set_corrupt(sc, sc->sa.agi_bp); + goto bad; + } + + ip = xfs_iunlink_lookup(sc->sa.pag, agino); + if (!ip) { + xchk_block_set_corrupt(sc, sc->sa.agi_bp); + goto bad; + } + + if (!xfs_inode_on_unlinked_list(ip)) { + xchk_block_set_corrupt(sc, sc->sa.agi_bp); + goto bad; + } + + ret = xagino_bitmap_set(&seen, agino, 1); + if (ret) + goto out_bitmap; + + agino = ip->i_next_unlinked; + } + ret = 1; + +out_bitmap: + xagino_bitmap_destroy(&seen); + return ret; +bad: + ret = 0; + goto out_bitmap; +} + +/* + * Check the unlinked buckets for links to bad inodes. We hold the AGI, so + * there cannot be any threads updating unlinked list pointers in this AG. + */ +STATIC int +xchk_iunlink( + struct xfs_scrub *sc, + struct xfs_agi *agi) +{ + unsigned int i; + + for (i = 0; i < XFS_AGI_UNLINKED_BUCKETS; i++) { + int ret; + + ret = xchk_iunlink_bucket(sc, i, + be32_to_cpu(agi->agi_unlinked[i])); + if (ret < 1) + return ret; + } + + return 0; +} + +/* Scrub the AGI. */ +int +xchk_agi( + struct xfs_scrub *sc) +{ + struct xfs_mount *mp = sc->mp; + struct xfs_agi *agi; + struct xfs_perag *pag; + struct xfs_ino_geometry *igeo = M_IGEO(sc->mp); + xfs_agnumber_t agno = sc->sm->sm_agno; + xfs_agblock_t agbno; + xfs_agblock_t eoag; + xfs_agino_t agino; + xfs_agino_t first_agino; + xfs_agino_t last_agino; + xfs_agino_t icount; + int i; + int level; + int error = 0; + + error = xchk_ag_read_headers(sc, agno, &sc->sa); + if (!xchk_process_error(sc, agno, XFS_AGI_BLOCK(sc->mp), &error)) + goto out; + xchk_buffer_recheck(sc, sc->sa.agi_bp); + + agi = sc->sa.agi_bp->b_addr; + pag = sc->sa.pag; + + /* Check the AG length */ + eoag = be32_to_cpu(agi->agi_length); + if (eoag != pag_group(pag)->xg_block_count) + xchk_block_set_corrupt(sc, sc->sa.agi_bp); + + /* Check btree roots and levels */ + agbno = be32_to_cpu(agi->agi_root); + if (!xfs_verify_agbno(pag, agbno)) + xchk_block_set_corrupt(sc, sc->sa.agi_bp); + + level = be32_to_cpu(agi->agi_level); + if (level <= 0 || level > igeo->inobt_maxlevels) + xchk_block_set_corrupt(sc, sc->sa.agi_bp); + + if (xfs_has_finobt(mp)) { + agbno = be32_to_cpu(agi->agi_free_root); + if (!xfs_verify_agbno(pag, agbno)) + xchk_block_set_corrupt(sc, sc->sa.agi_bp); + + level = be32_to_cpu(agi->agi_free_level); + if (level <= 0 || level > igeo->inobt_maxlevels) + xchk_block_set_corrupt(sc, sc->sa.agi_bp); + } + + /* Check inode counters */ + xfs_agino_range(mp, agno, &first_agino, &last_agino); + icount = be32_to_cpu(agi->agi_count); + if (icount > last_agino - first_agino + 1 || + icount < be32_to_cpu(agi->agi_freecount)) + xchk_block_set_corrupt(sc, sc->sa.agi_bp); + + /* Check inode pointers */ + agino = be32_to_cpu(agi->agi_newino); + if (!xfs_verify_agino_or_null(pag, agino)) + xchk_block_set_corrupt(sc, sc->sa.agi_bp); + + agino = be32_to_cpu(agi->agi_dirino); + if (!xfs_verify_agino_or_null(pag, agino)) + xchk_block_set_corrupt(sc, sc->sa.agi_bp); + + /* Check unlinked inode buckets */ + for (i = 0; i < XFS_AGI_UNLINKED_BUCKETS; i++) { + agino = be32_to_cpu(agi->agi_unlinked[i]); + if (!xfs_verify_agino_or_null(pag, agino)) + xchk_block_set_corrupt(sc, sc->sa.agi_bp); + } + + if (agi->agi_pad32 != cpu_to_be32(0)) + xchk_block_set_corrupt(sc, sc->sa.agi_bp); + + /* Do the incore counters match? */ + if (pag->pagi_count != be32_to_cpu(agi->agi_count)) + xchk_block_set_corrupt(sc, sc->sa.agi_bp); + if (pag->pagi_freecount != be32_to_cpu(agi->agi_freecount)) + xchk_block_set_corrupt(sc, sc->sa.agi_bp); + + error = xchk_iunlink(sc, agi); + if (error) + goto out; + + xchk_agi_xref(sc); +out: + return error; +} diff --git a/fs/xfs/scrub/agheader_repair.c b/fs/xfs/scrub/agheader_repair.c new file mode 100644 index 000000000..2104512f1 --- /dev/null +++ b/fs/xfs/scrub/agheader_repair.c @@ -0,0 +1,1906 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_btree.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_sb.h" +#include "xfs_alloc.h" +#include "xfs_alloc_btree.h" +#include "xfs_ialloc.h" +#include "xfs_ialloc_btree.h" +#include "xfs_rmap.h" +#include "xfs_rmap_btree.h" +#include "xfs_refcount_btree.h" +#include "xfs_ag.h" +#include "xfs_inode.h" +#include "xfs_iunlink_item.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/bitmap.h" +#include "scrub/agb_bitmap.h" +#include "scrub/agino_bitmap.h" +#include "scrub/reap.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" + +/* Superblock */ + +/* Repair the superblock. */ +int +xrep_superblock( + struct xfs_scrub *sc) +{ + struct xfs_mount *mp = sc->mp; + struct xfs_buf *bp; + xfs_agnumber_t agno; + int error; + + /* Don't try to repair AG 0's sb; let xfs_repair deal with it. */ + agno = sc->sm->sm_agno; + if (agno == 0) + return -EOPNOTSUPP; + + error = xfs_sb_get_secondary(mp, sc->tp, agno, &bp); + if (error) + return error; + + /* Last chance to abort before we start committing fixes. */ + if (xchk_should_terminate(sc, &error)) + return error; + + /* Copy AG 0's superblock to this one. */ + xfs_buf_zero(bp, 0, BBTOB(bp->b_length)); + xfs_sb_to_disk(bp->b_addr, &mp->m_sb); + + /* + * Don't write out a secondary super with NEEDSREPAIR or log incompat + * features set, since both are ignored when set on a secondary. + */ + if (xfs_has_crc(mp)) { + struct xfs_dsb *sb = bp->b_addr; + + sb->sb_features_incompat &= + ~cpu_to_be32(XFS_SB_FEAT_INCOMPAT_NEEDSREPAIR); + sb->sb_features_log_incompat = 0; + } + + /* Write this to disk. */ + xfs_trans_buf_set_type(sc->tp, bp, XFS_BLFT_SB_BUF); + xfs_trans_log_buf(sc->tp, bp, 0, BBTOB(bp->b_length) - 1); + return 0; +} + +/* AGF */ + +struct xrep_agf_allocbt { + struct xfs_scrub *sc; + xfs_agblock_t freeblks; + xfs_agblock_t longest; +}; + +/* Record free space shape information. */ +STATIC int +xrep_agf_walk_allocbt( + struct xfs_btree_cur *cur, + const struct xfs_alloc_rec_incore *rec, + void *priv) +{ + struct xrep_agf_allocbt *raa = priv; + int error = 0; + + if (xchk_should_terminate(raa->sc, &error)) + return error; + + raa->freeblks += rec->ar_blockcount; + if (rec->ar_blockcount > raa->longest) + raa->longest = rec->ar_blockcount; + return error; +} + +/* Does this AGFL block look sane? */ +STATIC int +xrep_agf_check_agfl_block( + struct xfs_mount *mp, + xfs_agblock_t agbno, + void *priv) +{ + struct xfs_scrub *sc = priv; + + if (!xfs_verify_agbno(sc->sa.pag, agbno)) + return -EFSCORRUPTED; + return 0; +} + +/* + * Offset within the xrep_find_ag_btree array for each btree type. Avoid the + * XFS_BTNUM_ names here to avoid creating a sparse array. + */ +enum { + XREP_AGF_BNOBT = 0, + XREP_AGF_CNTBT, + XREP_AGF_RMAPBT, + XREP_AGF_REFCOUNTBT, + XREP_AGF_END, + XREP_AGF_MAX +}; + +/* Check a btree root candidate. */ +static inline bool +xrep_check_btree_root( + struct xfs_scrub *sc, + struct xrep_find_ag_btree *fab) +{ + return xfs_verify_agbno(sc->sa.pag, fab->root) && + fab->height <= fab->maxlevels; +} + +/* + * Given the btree roots described by *fab, find the roots, check them for + * sanity, and pass the root data back out via *fab. + * + * This is /also/ a chicken and egg problem because we have to use the rmapbt + * (rooted in the AGF) to find the btrees rooted in the AGF. We also have no + * idea if the btrees make any sense. If we hit obvious corruptions in those + * btrees we'll bail out. + */ +STATIC int +xrep_agf_find_btrees( + struct xfs_scrub *sc, + struct xfs_buf *agf_bp, + struct xrep_find_ag_btree *fab, + struct xfs_buf *agfl_bp) +{ + struct xfs_agf *old_agf = agf_bp->b_addr; + int error; + + /* Go find the root data. */ + error = xrep_find_ag_btree_roots(sc, agf_bp, fab, agfl_bp); + if (error) + return error; + + /* We must find the bnobt, cntbt, and rmapbt roots. */ + if (!xrep_check_btree_root(sc, &fab[XREP_AGF_BNOBT]) || + !xrep_check_btree_root(sc, &fab[XREP_AGF_CNTBT]) || + !xrep_check_btree_root(sc, &fab[XREP_AGF_RMAPBT])) + return -EFSCORRUPTED; + + /* + * We relied on the rmapbt to reconstruct the AGF. If we get a + * different root then something's seriously wrong. + */ + if (fab[XREP_AGF_RMAPBT].root != be32_to_cpu(old_agf->agf_rmap_root)) + return -EFSCORRUPTED; + + /* We must find the refcountbt root if that feature is enabled. */ + if (xfs_has_reflink(sc->mp) && + !xrep_check_btree_root(sc, &fab[XREP_AGF_REFCOUNTBT])) + return -EFSCORRUPTED; + + return 0; +} + +/* + * Reinitialize the AGF header, making an in-core copy of the old contents so + * that we know which in-core state needs to be reinitialized. + */ +STATIC void +xrep_agf_init_header( + struct xfs_scrub *sc, + struct xfs_buf *agf_bp, + struct xfs_agf *old_agf) +{ + struct xfs_mount *mp = sc->mp; + struct xfs_perag *pag = sc->sa.pag; + struct xfs_agf *agf = agf_bp->b_addr; + + memcpy(old_agf, agf, sizeof(*old_agf)); + memset(agf, 0, BBTOB(agf_bp->b_length)); + agf->agf_magicnum = cpu_to_be32(XFS_AGF_MAGIC); + agf->agf_versionnum = cpu_to_be32(XFS_AGF_VERSION); + agf->agf_seqno = cpu_to_be32(pag_agno(pag)); + agf->agf_length = cpu_to_be32(pag_group(pag)->xg_block_count); + agf->agf_flfirst = old_agf->agf_flfirst; + agf->agf_fllast = old_agf->agf_fllast; + agf->agf_flcount = old_agf->agf_flcount; + if (xfs_has_crc(mp)) + uuid_copy(&agf->agf_uuid, &mp->m_sb.sb_meta_uuid); + + /* Mark the incore AGF data stale until we're done fixing things. */ + ASSERT(xfs_perag_initialised_agf(pag)); + clear_bit(XFS_AGSTATE_AGF_INIT, &pag->pag_opstate); +} + +/* Set btree root information in an AGF. */ +STATIC void +xrep_agf_set_roots( + struct xfs_scrub *sc, + struct xfs_agf *agf, + struct xrep_find_ag_btree *fab) +{ + agf->agf_bno_root = cpu_to_be32(fab[XREP_AGF_BNOBT].root); + agf->agf_bno_level = cpu_to_be32(fab[XREP_AGF_BNOBT].height); + + agf->agf_cnt_root = cpu_to_be32(fab[XREP_AGF_CNTBT].root); + agf->agf_cnt_level = cpu_to_be32(fab[XREP_AGF_CNTBT].height); + + agf->agf_rmap_root = cpu_to_be32(fab[XREP_AGF_RMAPBT].root); + agf->agf_rmap_level = cpu_to_be32(fab[XREP_AGF_RMAPBT].height); + + if (xfs_has_reflink(sc->mp)) { + agf->agf_refcount_root = + cpu_to_be32(fab[XREP_AGF_REFCOUNTBT].root); + agf->agf_refcount_level = + cpu_to_be32(fab[XREP_AGF_REFCOUNTBT].height); + } +} + +/* Update all AGF fields which derive from btree contents. */ +STATIC int +xrep_agf_calc_from_btrees( + struct xfs_scrub *sc, + struct xfs_buf *agf_bp) +{ + struct xrep_agf_allocbt raa = { .sc = sc }; + struct xfs_btree_cur *cur = NULL; + struct xfs_agf *agf = agf_bp->b_addr; + struct xfs_mount *mp = sc->mp; + xfs_agblock_t btreeblks; + xfs_filblks_t blocks; + int error; + + /* Update the AGF counters from the bnobt. */ + cur = xfs_bnobt_init_cursor(mp, sc->tp, agf_bp, sc->sa.pag); + error = xfs_alloc_query_all(cur, xrep_agf_walk_allocbt, &raa); + if (error) + goto err; + error = xfs_btree_count_blocks(cur, &blocks); + if (error) + goto err; + xfs_btree_del_cursor(cur, error); + btreeblks = blocks - 1; + agf->agf_freeblks = cpu_to_be32(raa.freeblks); + agf->agf_longest = cpu_to_be32(raa.longest); + + /* Update the AGF counters from the cntbt. */ + cur = xfs_cntbt_init_cursor(mp, sc->tp, agf_bp, sc->sa.pag); + error = xfs_btree_count_blocks(cur, &blocks); + if (error) + goto err; + xfs_btree_del_cursor(cur, error); + btreeblks += blocks - 1; + + /* Update the AGF counters from the rmapbt. */ + cur = xfs_rmapbt_init_cursor(mp, sc->tp, agf_bp, sc->sa.pag); + error = xfs_btree_count_blocks(cur, &blocks); + if (error) + goto err; + xfs_btree_del_cursor(cur, error); + agf->agf_rmap_blocks = cpu_to_be32(blocks); + btreeblks += blocks - 1; + + agf->agf_btreeblks = cpu_to_be32(btreeblks); + + /* Update the AGF counters from the refcountbt. */ + if (xfs_has_reflink(mp)) { + cur = xfs_refcountbt_init_cursor(mp, sc->tp, agf_bp, + sc->sa.pag); + error = xfs_btree_count_blocks(cur, &blocks); + if (error) + goto err; + xfs_btree_del_cursor(cur, error); + agf->agf_refcount_blocks = cpu_to_be32(blocks); + } + + return 0; +err: + xfs_btree_del_cursor(cur, error); + return error; +} + +/* Commit the new AGF and reinitialize the incore state. */ +STATIC int +xrep_agf_commit_new( + struct xfs_scrub *sc, + struct xfs_buf *agf_bp) +{ + struct xfs_perag *pag; + struct xfs_agf *agf = agf_bp->b_addr; + + /* Trigger fdblocks recalculation */ + xfs_force_summary_recalc(sc->mp); + + /* Write this to disk. */ + xfs_trans_buf_set_type(sc->tp, agf_bp, XFS_BLFT_AGF_BUF); + xfs_trans_log_buf(sc->tp, agf_bp, 0, BBTOB(agf_bp->b_length) - 1); + + /* Now reinitialize the in-core counters we changed. */ + pag = sc->sa.pag; + pag->pagf_btreeblks = be32_to_cpu(agf->agf_btreeblks); + pag->pagf_freeblks = be32_to_cpu(agf->agf_freeblks); + pag->pagf_longest = be32_to_cpu(agf->agf_longest); + pag->pagf_bno_level = be32_to_cpu(agf->agf_bno_level); + pag->pagf_cnt_level = be32_to_cpu(agf->agf_cnt_level); + pag->pagf_rmap_level = be32_to_cpu(agf->agf_rmap_level); + pag->pagf_refcount_level = be32_to_cpu(agf->agf_refcount_level); + set_bit(XFS_AGSTATE_AGF_INIT, &pag->pag_opstate); + + return xrep_roll_ag_trans(sc); +} + +/* Repair the AGF. v5 filesystems only. */ +int +xrep_agf( + struct xfs_scrub *sc) +{ + struct xrep_find_ag_btree fab[XREP_AGF_MAX] = { + [XREP_AGF_BNOBT] = { + .rmap_owner = XFS_RMAP_OWN_AG, + .buf_ops = &xfs_bnobt_buf_ops, + .maxlevels = sc->mp->m_alloc_maxlevels, + }, + [XREP_AGF_CNTBT] = { + .rmap_owner = XFS_RMAP_OWN_AG, + .buf_ops = &xfs_cntbt_buf_ops, + .maxlevels = sc->mp->m_alloc_maxlevels, + }, + [XREP_AGF_RMAPBT] = { + .rmap_owner = XFS_RMAP_OWN_AG, + .buf_ops = &xfs_rmapbt_buf_ops, + .maxlevels = sc->mp->m_rmap_maxlevels, + }, + [XREP_AGF_REFCOUNTBT] = { + .rmap_owner = XFS_RMAP_OWN_REFC, + .buf_ops = &xfs_refcountbt_buf_ops, + .maxlevels = sc->mp->m_refc_maxlevels, + }, + [XREP_AGF_END] = { + .buf_ops = NULL, + }, + }; + struct xfs_agf old_agf; + struct xfs_mount *mp = sc->mp; + struct xfs_buf *agf_bp; + struct xfs_buf *agfl_bp; + struct xfs_agf *agf; + int error; + + /* We require the rmapbt to rebuild anything. */ + if (!xfs_has_rmapbt(mp)) + return -EOPNOTSUPP; + + /* + * Make sure we have the AGF buffer, as scrub might have decided it + * was corrupt after xfs_alloc_read_agf failed with -EFSCORRUPTED. + */ + error = xfs_trans_read_buf(mp, sc->tp, mp->m_ddev_targp, + XFS_AG_DADDR(mp, pag_agno(sc->sa.pag), + XFS_AGF_DADDR(mp)), + XFS_FSS_TO_BB(mp, 1), 0, &agf_bp, NULL); + if (error) + return error; + agf_bp->b_ops = &xfs_agf_buf_ops; + agf = agf_bp->b_addr; + + /* + * Load the AGFL so that we can screen out OWN_AG blocks that are on + * the AGFL now; these blocks might have once been part of the + * bno/cnt/rmap btrees but are not now. This is a chicken and egg + * problem: the AGF is corrupt, so we have to trust the AGFL contents + * because we can't do any serious cross-referencing with any of the + * btrees rooted in the AGF. If the AGFL contents are obviously bad + * then we'll bail out. + */ + error = xfs_alloc_read_agfl(sc->sa.pag, sc->tp, &agfl_bp); + if (error) + return error; + + /* + * Spot-check the AGFL blocks; if they're obviously corrupt then + * there's nothing we can do but bail out. + */ + error = xfs_agfl_walk(sc->mp, agf_bp->b_addr, agfl_bp, + xrep_agf_check_agfl_block, sc); + if (error) + return error; + + /* + * Find the AGF btree roots. This is also a chicken-and-egg situation; + * see the function for more details. + */ + error = xrep_agf_find_btrees(sc, agf_bp, fab, agfl_bp); + if (error) + return error; + + /* Last chance to abort before we start committing fixes. */ + if (xchk_should_terminate(sc, &error)) + return error; + + /* Start rewriting the header and implant the btrees we found. */ + xrep_agf_init_header(sc, agf_bp, &old_agf); + xrep_agf_set_roots(sc, agf, fab); + error = xrep_agf_calc_from_btrees(sc, agf_bp); + if (error) + goto out_revert; + + /* Commit the changes and reinitialize incore state. */ + return xrep_agf_commit_new(sc, agf_bp); + +out_revert: + /* Mark the incore AGF state stale and revert the AGF. */ + clear_bit(XFS_AGSTATE_AGF_INIT, &sc->sa.pag->pag_opstate); + memcpy(agf, &old_agf, sizeof(old_agf)); + return error; +} + +/* AGFL */ + +struct xrep_agfl { + /* Bitmap of alleged AGFL blocks that we're not going to add. */ + struct xagb_bitmap crossed; + + /* Bitmap of other OWN_AG metadata blocks. */ + struct xagb_bitmap agmetablocks; + + /* Bitmap of free space. */ + struct xagb_bitmap *freesp; + + /* rmapbt cursor for finding crosslinked blocks */ + struct xfs_btree_cur *rmap_cur; + + struct xfs_scrub *sc; +}; + +/* Record all OWN_AG (free space btree) information from the rmap data. */ +STATIC int +xrep_agfl_walk_rmap( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv) +{ + struct xrep_agfl *ra = priv; + int error = 0; + + if (xchk_should_terminate(ra->sc, &error)) + return error; + + /* Record all the OWN_AG blocks. */ + if (rec->rm_owner == XFS_RMAP_OWN_AG) { + error = xagb_bitmap_set(ra->freesp, rec->rm_startblock, + rec->rm_blockcount); + if (error) + return error; + } + + return xagb_bitmap_set_btcur_path(&ra->agmetablocks, cur); +} + +/* Strike out the blocks that are cross-linked according to the rmapbt. */ +STATIC int +xrep_agfl_check_extent( + uint32_t agbno, + uint32_t len, + void *priv) +{ + struct xrep_agfl *ra = priv; + xfs_agblock_t last_agbno = agbno + len - 1; + int error; + + while (agbno <= last_agbno) { + bool other_owners; + + error = xfs_rmap_has_other_keys(ra->rmap_cur, agbno, 1, + &XFS_RMAP_OINFO_AG, &other_owners); + if (error) + return error; + + if (other_owners) { + error = xagb_bitmap_set(&ra->crossed, agbno, 1); + if (error) + return error; + } + + if (xchk_should_terminate(ra->sc, &error)) + return error; + agbno++; + } + + return 0; +} + +/* + * Map out all the non-AGFL OWN_AG space in this AG so that we can deduce + * which blocks belong to the AGFL. + * + * Compute the set of old AGFL blocks by subtracting from the list of OWN_AG + * blocks the list of blocks owned by all other OWN_AG metadata (bnobt, cntbt, + * rmapbt). These are the old AGFL blocks, so return that list and the number + * of blocks we're actually going to put back on the AGFL. + */ +STATIC int +xrep_agfl_collect_blocks( + struct xfs_scrub *sc, + struct xfs_buf *agf_bp, + struct xagb_bitmap *agfl_extents, + xfs_agblock_t *flcount) +{ + struct xrep_agfl ra; + struct xfs_mount *mp = sc->mp; + struct xfs_btree_cur *cur; + int error; + + ra.sc = sc; + ra.freesp = agfl_extents; + xagb_bitmap_init(&ra.agmetablocks); + xagb_bitmap_init(&ra.crossed); + + /* Find all space used by the free space btrees & rmapbt. */ + cur = xfs_rmapbt_init_cursor(mp, sc->tp, agf_bp, sc->sa.pag); + error = xfs_rmap_query_all(cur, xrep_agfl_walk_rmap, &ra); + xfs_btree_del_cursor(cur, error); + if (error) + goto out_bmp; + + /* Find all blocks currently being used by the bnobt. */ + cur = xfs_bnobt_init_cursor(mp, sc->tp, agf_bp, sc->sa.pag); + error = xagb_bitmap_set_btblocks(&ra.agmetablocks, cur); + xfs_btree_del_cursor(cur, error); + if (error) + goto out_bmp; + + /* Find all blocks currently being used by the cntbt. */ + cur = xfs_cntbt_init_cursor(mp, sc->tp, agf_bp, sc->sa.pag); + error = xagb_bitmap_set_btblocks(&ra.agmetablocks, cur); + xfs_btree_del_cursor(cur, error); + if (error) + goto out_bmp; + + /* + * Drop the freesp meta blocks that are in use by btrees. + * The remaining blocks /should/ be AGFL blocks. + */ + error = xagb_bitmap_disunion(agfl_extents, &ra.agmetablocks); + if (error) + goto out_bmp; + + /* Strike out the blocks that are cross-linked. */ + ra.rmap_cur = xfs_rmapbt_init_cursor(mp, sc->tp, agf_bp, sc->sa.pag); + error = xagb_bitmap_walk(agfl_extents, xrep_agfl_check_extent, &ra); + xfs_btree_del_cursor(ra.rmap_cur, error); + if (error) + goto out_bmp; + error = xagb_bitmap_disunion(agfl_extents, &ra.crossed); + if (error) + goto out_bmp; + + /* + * Calculate the new AGFL size. If we found more blocks than fit in + * the AGFL we'll free them later. + */ + *flcount = min_t(uint64_t, xagb_bitmap_hweight(agfl_extents), + xfs_agfl_size(mp)); + +out_bmp: + xagb_bitmap_destroy(&ra.crossed); + xagb_bitmap_destroy(&ra.agmetablocks); + return error; +} + +/* Update the AGF and reset the in-core state. */ +STATIC void +xrep_agfl_update_agf( + struct xfs_scrub *sc, + struct xfs_buf *agf_bp, + xfs_agblock_t flcount) +{ + struct xfs_agf *agf = agf_bp->b_addr; + + ASSERT(flcount <= xfs_agfl_size(sc->mp)); + + /* Trigger fdblocks recalculation */ + xfs_force_summary_recalc(sc->mp); + + /* Update the AGF counters. */ + if (xfs_perag_initialised_agf(sc->sa.pag)) { + sc->sa.pag->pagf_flcount = flcount; + clear_bit(XFS_AGSTATE_AGFL_NEEDS_RESET, + &sc->sa.pag->pag_opstate); + } + agf->agf_flfirst = cpu_to_be32(0); + agf->agf_flcount = cpu_to_be32(flcount); + if (flcount) + agf->agf_fllast = cpu_to_be32(flcount - 1); + else + agf->agf_fllast = cpu_to_be32(xfs_agfl_size(sc->mp) - 1); + + xfs_alloc_log_agf(sc->tp, agf_bp, + XFS_AGF_FLFIRST | XFS_AGF_FLLAST | XFS_AGF_FLCOUNT); +} + +struct xrep_agfl_fill { + struct xagb_bitmap used_extents; + struct xfs_scrub *sc; + __be32 *agfl_bno; + xfs_agblock_t flcount; + unsigned int fl_off; +}; + +/* Fill the AGFL with whatever blocks are in this extent. */ +static int +xrep_agfl_fill( + uint32_t start, + uint32_t len, + void *priv) +{ + struct xrep_agfl_fill *af = priv; + struct xfs_scrub *sc = af->sc; + xfs_agblock_t agbno = start; + int error; + + trace_xrep_agfl_insert(pag_group(sc->sa.pag), agbno, len); + + while (agbno < start + len && af->fl_off < af->flcount) + af->agfl_bno[af->fl_off++] = cpu_to_be32(agbno++); + + error = xagb_bitmap_set(&af->used_extents, start, agbno - start); + if (error) + return error; + + if (af->fl_off == af->flcount) + return -ECANCELED; + + return 0; +} + +/* Write out a totally new AGFL. */ +STATIC int +xrep_agfl_init_header( + struct xfs_scrub *sc, + struct xfs_buf *agfl_bp, + struct xagb_bitmap *agfl_extents, + xfs_agblock_t flcount) +{ + struct xrep_agfl_fill af = { + .sc = sc, + .flcount = flcount, + }; + struct xfs_mount *mp = sc->mp; + struct xfs_agfl *agfl; + int error; + + ASSERT(flcount <= xfs_agfl_size(mp)); + + /* + * Start rewriting the header by setting the bno[] array to + * NULLAGBLOCK, then setting AGFL header fields. + */ + agfl = XFS_BUF_TO_AGFL(agfl_bp); + memset(agfl, 0xFF, BBTOB(agfl_bp->b_length)); + agfl->agfl_magicnum = cpu_to_be32(XFS_AGFL_MAGIC); + agfl->agfl_seqno = cpu_to_be32(pag_agno(sc->sa.pag)); + uuid_copy(&agfl->agfl_uuid, &mp->m_sb.sb_meta_uuid); + + /* + * Fill the AGFL with the remaining blocks. If agfl_extents has more + * blocks than fit in the AGFL, they will be freed in a subsequent + * step. + */ + xagb_bitmap_init(&af.used_extents); + af.agfl_bno = xfs_buf_to_agfl_bno(agfl_bp); + xagb_bitmap_walk(agfl_extents, xrep_agfl_fill, &af); + error = xagb_bitmap_disunion(agfl_extents, &af.used_extents); + if (error) + return error; + + /* Write new AGFL to disk. */ + xfs_trans_buf_set_type(sc->tp, agfl_bp, XFS_BLFT_AGFL_BUF); + xfs_trans_log_buf(sc->tp, agfl_bp, 0, BBTOB(agfl_bp->b_length) - 1); + xagb_bitmap_destroy(&af.used_extents); + return 0; +} + +/* Repair the AGFL. */ +int +xrep_agfl( + struct xfs_scrub *sc) +{ + struct xagb_bitmap agfl_extents; + struct xfs_mount *mp = sc->mp; + struct xfs_buf *agf_bp; + struct xfs_buf *agfl_bp; + xfs_agblock_t flcount; + int error; + + /* We require the rmapbt to rebuild anything. */ + if (!xfs_has_rmapbt(mp)) + return -EOPNOTSUPP; + + xagb_bitmap_init(&agfl_extents); + + /* + * Read the AGF so that we can query the rmapbt. We hope that there's + * nothing wrong with the AGF, but all the AG header repair functions + * have this chicken-and-egg problem. + */ + error = xfs_alloc_read_agf(sc->sa.pag, sc->tp, 0, &agf_bp); + if (error) + return error; + + /* + * Make sure we have the AGFL buffer, as scrub might have decided it + * was corrupt after xfs_alloc_read_agfl failed with -EFSCORRUPTED. + */ + error = xfs_trans_read_buf(mp, sc->tp, mp->m_ddev_targp, + XFS_AG_DADDR(mp, pag_agno(sc->sa.pag), + XFS_AGFL_DADDR(mp)), + XFS_FSS_TO_BB(mp, 1), 0, &agfl_bp, NULL); + if (error) + return error; + agfl_bp->b_ops = &xfs_agfl_buf_ops; + + /* Gather all the extents we're going to put on the new AGFL. */ + error = xrep_agfl_collect_blocks(sc, agf_bp, &agfl_extents, &flcount); + if (error) + goto err; + + /* Last chance to abort before we start committing fixes. */ + if (xchk_should_terminate(sc, &error)) + goto err; + + /* + * Update AGF and AGFL. We reset the global free block counter when + * we adjust the AGF flcount (which can fail) so avoid updating any + * buffers until we know that part works. + */ + xrep_agfl_update_agf(sc, agf_bp, flcount); + error = xrep_agfl_init_header(sc, agfl_bp, &agfl_extents, flcount); + if (error) + goto err; + + /* + * Ok, the AGFL should be ready to go now. Roll the transaction to + * make the new AGFL permanent before we start using it to return + * freespace overflow to the freespace btrees. + */ + sc->sa.agf_bp = agf_bp; + error = xrep_roll_ag_trans(sc); + if (error) + goto err; + + /* Dump any AGFL overflow. */ + error = xrep_reap_agblocks(sc, &agfl_extents, &XFS_RMAP_OINFO_AG, + XFS_AG_RESV_AGFL); + if (error) + goto err; + +err: + xagb_bitmap_destroy(&agfl_extents); + return error; +} + +/* AGI */ + +/* + * Offset within the xrep_find_ag_btree array for each btree type. Avoid the + * XFS_BTNUM_ names here to avoid creating a sparse array. + */ +enum { + XREP_AGI_INOBT = 0, + XREP_AGI_FINOBT, + XREP_AGI_END, + XREP_AGI_MAX +}; + +#define XREP_AGI_LOOKUP_BATCH 32 + +struct xrep_agi { + struct xfs_scrub *sc; + + /* AGI buffer, tracked separately */ + struct xfs_buf *agi_bp; + + /* context for finding btree roots */ + struct xrep_find_ag_btree fab[XREP_AGI_MAX]; + + /* old AGI contents in case we have to revert */ + struct xfs_agi old_agi; + + /* bitmap of which inodes are unlinked */ + struct xagino_bitmap iunlink_bmp; + + /* heads of the unlinked inode bucket lists */ + xfs_agino_t iunlink_heads[XFS_AGI_UNLINKED_BUCKETS]; + + /* scratchpad for batched lookups of the radix tree */ + struct xfs_inode *lookup_batch[XREP_AGI_LOOKUP_BATCH]; + + /* Map of ino -> next_ino for unlinked inode processing. */ + struct xfarray *iunlink_next; + + /* Map of ino -> prev_ino for unlinked inode processing. */ + struct xfarray *iunlink_prev; +}; + +static void +xrep_agi_buf_cleanup( + void *buf) +{ + struct xrep_agi *ragi = buf; + + if (ragi->iunlink_prev) + xfarray_destroy(ragi->iunlink_prev); + ragi->iunlink_prev = NULL; + if (ragi->iunlink_next) + xfarray_destroy(ragi->iunlink_next); + ragi->iunlink_next = NULL; + xagino_bitmap_destroy(&ragi->iunlink_bmp); +} + +/* + * Given the inode btree roots described by *fab, find the roots, check them + * for sanity, and pass the root data back out via *fab. + */ +STATIC int +xrep_agi_find_btrees( + struct xrep_agi *ragi) +{ + struct xfs_scrub *sc = ragi->sc; + struct xrep_find_ag_btree *fab = ragi->fab; + struct xfs_buf *agf_bp; + struct xfs_mount *mp = sc->mp; + int error; + + /* Read the AGF. */ + error = xfs_alloc_read_agf(sc->sa.pag, sc->tp, 0, &agf_bp); + if (error) + return error; + + /* Find the btree roots. */ + error = xrep_find_ag_btree_roots(sc, agf_bp, fab, NULL); + if (error) + return error; + + /* We must find the inobt root. */ + if (!xrep_check_btree_root(sc, &fab[XREP_AGI_INOBT])) + return -EFSCORRUPTED; + + /* We must find the finobt root if that feature is enabled. */ + if (xfs_has_finobt(mp) && + !xrep_check_btree_root(sc, &fab[XREP_AGI_FINOBT])) + return -EFSCORRUPTED; + + return 0; +} + +/* + * Reinitialize the AGI header, making an in-core copy of the old contents so + * that we know which in-core state needs to be reinitialized. + */ +STATIC void +xrep_agi_init_header( + struct xrep_agi *ragi) +{ + struct xfs_scrub *sc = ragi->sc; + struct xfs_buf *agi_bp = ragi->agi_bp; + struct xfs_agi *old_agi = &ragi->old_agi; + struct xfs_agi *agi = agi_bp->b_addr; + struct xfs_perag *pag = sc->sa.pag; + struct xfs_mount *mp = sc->mp; + + memcpy(old_agi, agi, sizeof(*old_agi)); + memset(agi, 0, BBTOB(agi_bp->b_length)); + agi->agi_magicnum = cpu_to_be32(XFS_AGI_MAGIC); + agi->agi_versionnum = cpu_to_be32(XFS_AGI_VERSION); + agi->agi_seqno = cpu_to_be32(pag_agno(pag)); + agi->agi_length = cpu_to_be32(pag_group(pag)->xg_block_count); + agi->agi_newino = cpu_to_be32(NULLAGINO); + agi->agi_dirino = cpu_to_be32(NULLAGINO); + if (xfs_has_crc(mp)) + uuid_copy(&agi->agi_uuid, &mp->m_sb.sb_meta_uuid); + + /* Mark the incore AGF data stale until we're done fixing things. */ + ASSERT(xfs_perag_initialised_agi(pag)); + clear_bit(XFS_AGSTATE_AGI_INIT, &pag->pag_opstate); +} + +/* Set btree root information in an AGI. */ +STATIC void +xrep_agi_set_roots( + struct xrep_agi *ragi) +{ + struct xfs_scrub *sc = ragi->sc; + struct xfs_agi *agi = ragi->agi_bp->b_addr; + struct xrep_find_ag_btree *fab = ragi->fab; + + agi->agi_root = cpu_to_be32(fab[XREP_AGI_INOBT].root); + agi->agi_level = cpu_to_be32(fab[XREP_AGI_INOBT].height); + + if (xfs_has_finobt(sc->mp)) { + agi->agi_free_root = cpu_to_be32(fab[XREP_AGI_FINOBT].root); + agi->agi_free_level = cpu_to_be32(fab[XREP_AGI_FINOBT].height); + } +} + +/* Update the AGI counters. */ +STATIC int +xrep_agi_calc_from_btrees( + struct xrep_agi *ragi) +{ + struct xfs_scrub *sc = ragi->sc; + struct xfs_buf *agi_bp = ragi->agi_bp; + struct xfs_btree_cur *cur; + struct xfs_agi *agi = agi_bp->b_addr; + struct xfs_mount *mp = sc->mp; + xfs_agino_t count; + xfs_agino_t freecount; + int error; + + cur = xfs_inobt_init_cursor(sc->sa.pag, sc->tp, agi_bp); + error = xfs_ialloc_count_inodes(cur, &count, &freecount); + if (error) + goto err; + if (xfs_has_inobtcounts(mp)) { + xfs_filblks_t blocks; + + error = xfs_btree_count_blocks(cur, &blocks); + if (error) + goto err; + agi->agi_iblocks = cpu_to_be32(blocks); + } + xfs_btree_del_cursor(cur, error); + + agi->agi_count = cpu_to_be32(count); + agi->agi_freecount = cpu_to_be32(freecount); + + if (xfs_has_finobt(mp) && xfs_has_inobtcounts(mp)) { + xfs_filblks_t blocks; + + cur = xfs_finobt_init_cursor(sc->sa.pag, sc->tp, agi_bp); + error = xfs_btree_count_blocks(cur, &blocks); + if (error) + goto err; + xfs_btree_del_cursor(cur, error); + agi->agi_fblocks = cpu_to_be32(blocks); + } + + return 0; +err: + xfs_btree_del_cursor(cur, error); + return error; +} + +/* + * Magic value that means "not unlinked" because xfarrays don't support storing + * totally zeroed elements. There can't be a cluster that starts in daddr 0 so + * there can't be an inode #1 either. + */ +#define LINKED_AGINO (0x1) + +/* + * Record a forwards unlinked chain pointer from agino -> next_agino in our + * staging information. + */ +static inline int +xrep_iunlink_store_next( + struct xrep_agi *ragi, + xfs_agino_t agino, + xfs_agino_t next_agino) +{ + ASSERT(next_agino != 0); + + return xfarray_store(ragi->iunlink_next, agino, &next_agino); +} + +/* + * Record a backwards unlinked chain pointer from prev_ino <- agino in our + * staging information. + */ +static inline int +xrep_iunlink_store_prev( + struct xrep_agi *ragi, + xfs_agino_t agino, + xfs_agino_t prev_agino) +{ + ASSERT(prev_agino != 0); + + return xfarray_store(ragi->iunlink_prev, agino, &prev_agino); +} + +/* + * Given an @agino, look up the next inode in the iunlink bucket. Returns + * NULLAGINO if we're at the end of the chain, 0 if @agino is not in memory + * like it should be, or a per-AG inode number. + */ +static inline xfs_agino_t +xrep_iunlink_next( + struct xfs_scrub *sc, + xfs_agino_t agino) +{ + struct xfs_inode *ip; + + ip = xfs_iunlink_lookup(sc->sa.pag, agino); + if (!ip) + return 0; + + return ip->i_next_unlinked; +} + +/* + * Load the inode @agino into memory, set its i_prev_unlinked, and drop the + * inode so it can be inactivated. Returns NULLAGINO if we're at the end of + * the chain or if we should stop walking the chain due to corruption; or a + * per-AG inode number. + */ +STATIC int +xrep_iunlink_reload_next( + struct xrep_agi *ragi, + xfs_agino_t prev_agino, + xfs_agino_t agino, + xfs_agino_t *next_agino) +{ + struct xfs_scrub *sc = ragi->sc; + struct xfs_inode *ip; + int error; + + *next_agino = NULLAGINO; + + error = xchk_iget(ragi->sc, xfs_agino_to_ino(sc->sa.pag, agino), &ip); + if (error) + return 0; + + trace_xrep_iunlink_reload_next(ip, prev_agino); + + /* If this is a linked inode, stop processing the chain. */ + if (VFS_I(ip)->i_nlink != 0) { + error = xrep_iunlink_store_next(ragi, agino, NULLAGINO); + if (error) + return error; + + error = xrep_iunlink_store_prev(ragi, agino, LINKED_AGINO); + if (error) + return error; + + goto rele; + } + + ip->i_prev_unlinked = prev_agino; + *next_agino = ip->i_next_unlinked; + + /* + * Drop the inode reference that we just took. We hold the AGI, so + * this inode cannot move off the unlinked list and hence cannot be + * reclaimed. + */ +rele: + xchk_irele(sc, ip); + return 0; +} + +/* + * Walk an AGI unlinked bucket's list to load incore any unlinked inodes that + * still existed at mount time. This can happen if iunlink processing fails + * during log recovery. + */ +STATIC int +xrep_iunlink_walk_ondisk_bucket( + struct xrep_agi *ragi, + unsigned int bucket) +{ + struct xagino_bitmap seen; + struct xfs_scrub *sc = ragi->sc; + struct xfs_agi *agi = ragi->agi_bp->b_addr; + xfs_agino_t prev_agino = NULLAGINO; + xfs_agino_t next_agino; + int error = 0; + + xagino_bitmap_init(&seen); + + next_agino = be32_to_cpu(agi->agi_unlinked[bucket]); + while (next_agino != NULLAGINO) { + xfs_agino_t agino = next_agino; + unsigned int len = 1; + + if (xchk_should_terminate(ragi->sc, &error)) + goto out_bitmap; + + trace_xrep_iunlink_walk_ondisk_bucket(sc->sa.pag, bucket, + prev_agino, agino); + + if (bucket != agino % XFS_AGI_UNLINKED_BUCKETS) + break; + + if (xagino_bitmap_test(&seen, agino, &len)) + break; + + next_agino = xrep_iunlink_next(sc, agino); + if (!next_agino) { + error = xrep_iunlink_reload_next(ragi, prev_agino, + agino, &next_agino); + if (error) + break; + } + + error = xagino_bitmap_set(&seen, agino, 1); + if (error) + goto out_bitmap; + + prev_agino = agino; + } + +out_bitmap: + xagino_bitmap_destroy(&seen); + return error; +} + +/* Decide if this is an unlinked inode in this AG. */ +STATIC bool +xrep_iunlink_igrab( + struct xfs_perag *pag, + struct xfs_inode *ip) +{ + if (XFS_INODE_TO_AGNO(ip) != pag_agno(pag)) + return false; + + if (!xfs_inode_on_unlinked_list(ip)) + return false; + + return true; +} + +/* + * Mark the given inode in the lookup batch in our unlinked inode bitmap, and + * remember if this inode is the start of the unlinked chain. + */ +STATIC int +xrep_iunlink_visit( + struct xrep_agi *ragi, + unsigned int batch_idx) +{ + struct xfs_inode *ip = ragi->lookup_batch[batch_idx]; + xfs_agino_t agino = XFS_INODE_TO_AGINO(ip); + unsigned int bucket = agino % XFS_AGI_UNLINKED_BUCKETS; + int error; + + ASSERT(XFS_INODE_TO_AGNO(ip) == pag_agno(ragi->sc->sa.pag)); + ASSERT(xfs_inode_on_unlinked_list(ip)); + + trace_xrep_iunlink_visit(ragi->sc->sa.pag, bucket, + ragi->iunlink_heads[bucket], ip); + + error = xagino_bitmap_set(&ragi->iunlink_bmp, agino, 1); + if (error) + return error; + + if (ip->i_prev_unlinked == NULLAGINO) { + if (ragi->iunlink_heads[bucket] == NULLAGINO) + ragi->iunlink_heads[bucket] = agino; + } + + return 0; +} + +/* + * Find all incore unlinked inodes so that we can rebuild the unlinked buckets. + * We hold the AGI so there should not be any modifications to the unlinked + * list. + */ +STATIC int +xrep_iunlink_mark_incore( + struct xrep_agi *ragi) +{ + struct xfs_perag *pag = ragi->sc->sa.pag; + struct xfs_mount *mp = pag_mount(pag); + uint32_t first_index = 0; + bool done = false; + unsigned int nr_found = 0; + + do { + unsigned int i; + int error = 0; + + if (xchk_should_terminate(ragi->sc, &error)) + return error; + + rcu_read_lock(); + + nr_found = radix_tree_gang_lookup(&pag->pag_ici_root, + (void **)&ragi->lookup_batch, first_index, + XREP_AGI_LOOKUP_BATCH); + if (!nr_found) { + rcu_read_unlock(); + return 0; + } + + for (i = 0; i < nr_found; i++) { + struct xfs_inode *ip = ragi->lookup_batch[i]; + + if (done || !xrep_iunlink_igrab(pag, ip)) + ragi->lookup_batch[i] = NULL; + + /* + * Update the index for the next lookup. Catch + * overflows into the next AG range which can occur if + * we have inodes in the last block of the AG and we + * are currently pointing to the last inode. + * + * Because we may see inodes that are from the wrong AG + * due to RCU freeing and reallocation, only update the + * index if it lies in this AG. It was a race that lead + * us to see this inode, so another lookup from the + * same index will not find it again. + */ + if (XFS_INODE_TO_AGNO(ip) != pag_agno(pag)) + continue; + first_index = XFS_INO_TO_AGINO(mp, I_INO(ip) + 1); + if (first_index < XFS_INODE_TO_AGINO(ip)) + done = true; + } + + /* unlock now we've grabbed the inodes. */ + rcu_read_unlock(); + + for (i = 0; i < nr_found; i++) { + if (!ragi->lookup_batch[i]) + continue; + error = xrep_iunlink_visit(ragi, i); + if (error) + return error; + } + } while (!done); + + return 0; +} + +/* Mark all the unlinked ondisk inodes in this inobt record in iunlink_bmp. */ +STATIC int +xrep_iunlink_mark_ondisk_rec( + struct xfs_btree_cur *cur, + const union xfs_btree_rec *rec, + void *priv) +{ + struct xfs_inobt_rec_incore irec; + struct xrep_agi *ragi = priv; + struct xfs_scrub *sc = ragi->sc; + struct xfs_mount *mp = cur->bc_mp; + xfs_agino_t agino; + unsigned int i; + int error = 0; + + xfs_inobt_btrec_to_irec(mp, rec, &irec); + + for (i = 0, agino = irec.ir_startino; + i < XFS_INODES_PER_CHUNK; + i++, agino++) { + struct xfs_inode *ip; + unsigned int len = 1; + + /* Skip free inodes */ + if (XFS_INOBT_MASK(i) & irec.ir_free) + continue; + /* Skip inodes we've seen before */ + if (xagino_bitmap_test(&ragi->iunlink_bmp, agino, &len)) + continue; + + /* + * Skip incore inodes; these were already picked up by + * the _mark_incore step. + */ + rcu_read_lock(); + ip = radix_tree_lookup(&sc->sa.pag->pag_ici_root, agino); + rcu_read_unlock(); + if (ip) + continue; + + /* + * Try to look up this inode. If we can't get it, just move + * on because we haven't actually scrubbed the inobt or the + * inodes yet. + */ + error = xchk_iget(ragi->sc, xfs_agino_to_ino(sc->sa.pag, agino), + &ip); + if (error) + continue; + + trace_xrep_iunlink_reload_ondisk(ip); + + if (VFS_I(ip)->i_nlink == 0) + error = xagino_bitmap_set(&ragi->iunlink_bmp, agino, 1); + xchk_irele(sc, ip); + if (error) + break; + } + + return error; +} + +/* + * Find ondisk inodes that are unlinked and not in cache, and mark them in + * iunlink_bmp. We haven't checked the inobt yet, so we don't error out if + * the btree is corrupt. + */ +STATIC int +xrep_iunlink_mark_ondisk( + struct xrep_agi *ragi) +{ + struct xfs_scrub *sc = ragi->sc; + struct xfs_buf *agi_bp = ragi->agi_bp; + struct xfs_btree_cur *cur; + int error; + + cur = xfs_inobt_init_cursor(sc->sa.pag, sc->tp, agi_bp); + error = xfs_btree_query_all(cur, xrep_iunlink_mark_ondisk_rec, ragi); + xfs_btree_del_cursor(cur, error); + + /* + * Don't proceed if we couldn't set a bit in the bitmap. All other + * errors we ignore because we haven't actually checked the inobt yet. + */ + if (error == -ENOMEM) + return -ENOMEM; + return 0; +} + +/* + * Walk an iunlink bucket's inode list. For each inode that should be on this + * chain, clear its entry in in iunlink_bmp because it's ok and we don't need + * to touch it further. + */ +STATIC int +xrep_iunlink_resolve_bucket( + struct xrep_agi *ragi, + unsigned int bucket) +{ + struct xagino_bitmap seen; + struct xfs_scrub *sc = ragi->sc; + struct xfs_inode *ip; + xfs_agino_t prev_agino = NULLAGINO; + xfs_agino_t next_agino = ragi->iunlink_heads[bucket]; + int error = 0; + + xagino_bitmap_init(&seen); + + while (next_agino != NULLAGINO) { + unsigned int len = 1; + + if (xchk_should_terminate(ragi->sc, &error)) + goto out_bitmap; + + /* Inode already seen? We're stuck in a loop */ + if (xagino_bitmap_test(&seen, next_agino, &len)) { + trace_xrep_iunlink_resolve_infinite_loop(sc->sa.pag, + bucket, prev_agino, next_agino); + next_agino = NULLAGINO; + break; + } + + error = xagino_bitmap_set(&seen, next_agino, 1); + if (error) + goto out_bitmap; + + /* Find the next inode in the chain. */ + ip = xfs_iunlink_lookup(sc->sa.pag, next_agino); + if (!ip) { + /* Inode not incore? Terminate the chain. */ + trace_xrep_iunlink_resolve_uncached(sc->sa.pag, + bucket, prev_agino, next_agino); + + next_agino = NULLAGINO; + break; + } + + if (VFS_I(ip)->i_nlink != 0) { + /* + * Inode is linked somewhere! Blow out both unlinked + * list pointers, advance the list, and pretend we + * didn't see this inode. Clear it from iunlink_bmp + * because it's linked. + */ + trace_xrep_iunlink_resolve_allocated(sc->sa.pag, + bucket, prev_agino, next_agino); + + error = xrep_iunlink_store_next(ragi, next_agino, + NULLAGINO); + if (error) + goto out_bitmap; + + error = xrep_iunlink_store_prev(ragi, next_agino, + LINKED_AGINO); + if (error) + goto out_bitmap; + + error = xagino_bitmap_clear(&ragi->iunlink_bmp, + next_agino, 1); + if (error) + goto out_bitmap; + + next_agino = ip->i_next_unlinked; + continue; + } + + if (next_agino % XFS_AGI_UNLINKED_BUCKETS != bucket) { + /* + * Inode is in the wrong bucket. Advance the list, + * but pretend we didn't see this inode. + */ + trace_xrep_iunlink_resolve_wronglist(sc->sa.pag, + bucket, prev_agino, next_agino); + + next_agino = ip->i_next_unlinked; + continue; + } + + if (!xfs_inode_on_unlinked_list(ip)) { + /* + * Incore inode doesn't think this inode is on an + * unlinked list. This is probably because we reloaded + * it from disk. Advance the list, but pretend we + * didn't see this inode; we'll fix that later. + */ + trace_xrep_iunlink_resolve_nolist(sc->sa.pag, + bucket, prev_agino, next_agino); + next_agino = ip->i_next_unlinked; + continue; + } + + trace_xrep_iunlink_resolve_ok(sc->sa.pag, bucket, prev_agino, + next_agino); + + /* + * Otherwise, this inode's unlinked pointers are ok. Clear it + * from the unlinked bitmap since we're done with it, and make + * sure the chain is still correct. + */ + error = xagino_bitmap_clear(&ragi->iunlink_bmp, next_agino, 1); + if (error) + goto out_bitmap; + + /* Remember the previous inode's next pointer. */ + if (prev_agino != NULLAGINO) { + error = xrep_iunlink_store_next(ragi, prev_agino, + next_agino); + if (error) + goto out_bitmap; + } + + /* Remember this inode's previous pointer. */ + error = xrep_iunlink_store_prev(ragi, next_agino, prev_agino); + if (error) + goto out_bitmap; + + /* Advance the list and remember this inode. */ + prev_agino = next_agino; + next_agino = ip->i_next_unlinked; + } + + /* Update the previous inode's next pointer. */ + if (prev_agino != NULLAGINO) { + error = xrep_iunlink_store_next(ragi, prev_agino, next_agino); + if (error) + goto out_bitmap; + } + +out_bitmap: + xagino_bitmap_destroy(&seen); + return error; +} + +/* Reinsert this unlinked inode into the head of the staged bucket list. */ +STATIC int +xrep_iunlink_add_to_bucket( + struct xrep_agi *ragi, + xfs_agino_t agino) +{ + xfs_agino_t current_head; + unsigned int bucket; + int error; + + bucket = agino % XFS_AGI_UNLINKED_BUCKETS; + + /* Point this inode at the current head of the bucket list. */ + current_head = ragi->iunlink_heads[bucket]; + + trace_xrep_iunlink_add_to_bucket(ragi->sc->sa.pag, bucket, agino, + current_head); + + error = xrep_iunlink_store_next(ragi, agino, current_head); + if (error) + return error; + + error = xrep_iunlink_store_prev(ragi, agino, NULLAGINO); + if (error) + return error; + + /* Remember the head inode's previous pointer. */ + if (current_head != NULLAGINO) { + error = xrep_iunlink_store_prev(ragi, current_head, agino); + if (error) + return error; + } + + ragi->iunlink_heads[bucket] = agino; + return 0; +} + +/* Reinsert unlinked inodes into the staged iunlink buckets. */ +STATIC int +xrep_iunlink_add_lost_inodes( + uint32_t start, + uint32_t len, + void *priv) +{ + struct xrep_agi *ragi = priv; + int error; + + for (; len > 0; start++, len--) { + error = xrep_iunlink_add_to_bucket(ragi, start); + if (error) + return error; + } + + return 0; +} + +/* + * Figure out the iunlink bucket values and find inodes that need to be + * reinserted into the list. + */ +STATIC int +xrep_iunlink_rebuild_buckets( + struct xrep_agi *ragi) +{ + unsigned int i; + int error; + + /* + * Walk the ondisk AGI unlinked list to find inodes that are on the + * list but aren't in memory. This can happen if a past log recovery + * tried to clear the iunlinked list but failed. Our scan rebuilds the + * unlinked list using incore inodes, so we must load and link them + * properly. + */ + for (i = 0; i < XFS_AGI_UNLINKED_BUCKETS; i++) { + error = xrep_iunlink_walk_ondisk_bucket(ragi, i); + if (error) + return error; + } + + /* + * Record all the incore unlinked inodes in iunlink_bmp that we didn't + * find by walking the ondisk iunlink buckets. This shouldn't happen, + * but we can't risk forgetting an inode somewhere. + */ + error = xrep_iunlink_mark_incore(ragi); + if (error) + return error; + + /* + * If there are ondisk inodes that are unlinked and are not been loaded + * into cache, record them in iunlink_bmp. + */ + error = xrep_iunlink_mark_ondisk(ragi); + if (error) + return error; + + /* + * Walk each iunlink bucket to (re)construct as much of the incore list + * as would be correct. For each inode that survives this step, mark + * it clear in iunlink_bmp; we're done with those inodes. + */ + for (i = 0; i < XFS_AGI_UNLINKED_BUCKETS; i++) { + error = xrep_iunlink_resolve_bucket(ragi, i); + if (error) + return error; + } + + /* + * Any unlinked inodes that we didn't find through the bucket list + * walk (or was ignored by the walk) must be inserted into the bucket + * list. Stage this in memory for now. + */ + return xagino_bitmap_walk(&ragi->iunlink_bmp, + xrep_iunlink_add_lost_inodes, ragi); +} + +static inline void +set_inode_prev_unlinked( + struct xfs_inode *ip, + xfs_agino_t prev_agino) +{ + /* + * Magic value that means "not unlinked" because xfarrays don't support + * storing totally zeroed elements. + */ + if (prev_agino == LINKED_AGINO) + prev_agino = 0; + + if (ip->i_prev_unlinked != prev_agino) { + trace_xrep_iunlink_relink_prev(ip, prev_agino); + ip->i_prev_unlinked = prev_agino; + } +} + +/* Update i_next_iunlinked for the inode @agino. */ +STATIC int +xrep_iunlink_relink_next( + struct xrep_agi *ragi, + xfarray_idx_t idx, + xfs_agino_t next_agino) +{ + struct xfs_scrub *sc = ragi->sc; + struct xfs_perag *pag = sc->sa.pag; + struct xfs_inode *ip; + xfarray_idx_t agino = idx - 1; + bool want_rele = false; + int error = 0; + + ip = xfs_iunlink_lookup(pag, agino); + if (!ip) { + xfs_agino_t prev_agino; + + /* + * No inode exists in cache. Load it off the disk so that we + * can reinsert it into the incore unlinked list. + */ + error = xchk_iget(sc, xfs_agino_to_ino(pag, agino), &ip); + if (error) + return -EFSCORRUPTED; + + want_rele = true; + + /* Set the backward pointer since this just came off disk. */ + error = xfarray_load(ragi->iunlink_prev, agino, &prev_agino); + if (error) + goto out_rele; + + set_inode_prev_unlinked(ip, prev_agino); + } + + /* Update the forward pointer. */ + if (ip->i_next_unlinked != next_agino) { + error = xfs_iunlink_log_inode(sc->tp, ip, pag, next_agino); + if (error) + goto out_rele; + + trace_xrep_iunlink_relink_next(ip, next_agino); + ip->i_next_unlinked = next_agino; + } + +out_rele: + /* + * The iunlink lookup doesn't igrab because we hold the AGI buffer lock + * and the inode cannot be reclaimed. However, if we used iget to load + * a missing inode, we must irele it here. + */ + if (want_rele) + xchk_irele(sc, ip); + return error; +} + +/* Update i_prev_iunlinked for the inode @agino. */ +STATIC int +xrep_iunlink_relink_prev( + struct xrep_agi *ragi, + xfarray_idx_t idx, + xfs_agino_t prev_agino) +{ + struct xfs_scrub *sc = ragi->sc; + struct xfs_perag *pag = sc->sa.pag; + struct xfs_inode *ip; + xfarray_idx_t agino = idx - 1; + bool want_rele = false; + int error = 0; + + ASSERT(prev_agino != 0); + + ip = xfs_iunlink_lookup(pag, agino); + if (!ip) { + xfs_agino_t next_agino; + + /* + * No inode exists in cache. Load it off the disk so that we + * can reinsert it into the incore unlinked list. + */ + error = xchk_iget(sc, xfs_agino_to_ino(pag, agino), &ip); + if (error) + return -EFSCORRUPTED; + + want_rele = true; + + /* Set the forward pointer since this just came off disk. */ + error = xfarray_load(ragi->iunlink_next, agino, &next_agino); + if (error) + goto out_rele; + + error = xfs_iunlink_log_inode(sc->tp, ip, pag, next_agino); + if (error) + goto out_rele; + + trace_xrep_iunlink_relink_next(ip, next_agino); + ip->i_next_unlinked = next_agino; + } + + set_inode_prev_unlinked(ip, prev_agino); + +out_rele: + /* + * The iunlink lookup doesn't igrab because we hold the AGI buffer lock + * and the inode cannot be reclaimed. However, if we used iget to load + * a missing inode, we must irele it here. + */ + if (want_rele) + xchk_irele(sc, ip); + return error; +} + +/* Log all the iunlink updates we need to finish regenerating the AGI. */ +STATIC int +xrep_iunlink_commit( + struct xrep_agi *ragi) +{ + struct xfs_agi *agi = ragi->agi_bp->b_addr; + xfarray_idx_t idx = XFARRAY_CURSOR_INIT; + xfs_agino_t agino; + unsigned int i; + int error; + + /* Fix all the forward links */ + while ((error = xfarray_iter(ragi->iunlink_next, &idx, &agino)) == 1) { + error = xrep_iunlink_relink_next(ragi, idx, agino); + if (error) + return error; + } + if (error < 0) + return error; + + /* Fix all the back links */ + idx = XFARRAY_CURSOR_INIT; + while ((error = xfarray_iter(ragi->iunlink_prev, &idx, &agino)) == 1) { + error = xrep_iunlink_relink_prev(ragi, idx, agino); + if (error) + return error; + } + if (error < 0) + return error; + + /* Copy the staged iunlink buckets to the new AGI. */ + for (i = 0; i < XFS_AGI_UNLINKED_BUCKETS; i++) { + trace_xrep_iunlink_commit_bucket(ragi->sc->sa.pag, i, + be32_to_cpu(ragi->old_agi.agi_unlinked[i]), + ragi->iunlink_heads[i]); + + agi->agi_unlinked[i] = cpu_to_be32(ragi->iunlink_heads[i]); + } + + return 0; +} + +/* Trigger reinitialization of the in-core data. */ +STATIC int +xrep_agi_commit_new( + struct xrep_agi *ragi) +{ + struct xfs_scrub *sc = ragi->sc; + struct xfs_buf *agi_bp = ragi->agi_bp; + struct xfs_perag *pag; + struct xfs_agi *agi = agi_bp->b_addr; + + /* Trigger inode count recalculation */ + xfs_force_summary_recalc(sc->mp); + + /* Write this to disk. */ + xfs_trans_buf_set_type(sc->tp, agi_bp, XFS_BLFT_AGI_BUF); + xfs_trans_log_buf(sc->tp, agi_bp, 0, BBTOB(agi_bp->b_length) - 1); + + /* Now reinitialize the in-core counters if necessary. */ + pag = sc->sa.pag; + pag->pagi_count = be32_to_cpu(agi->agi_count); + pag->pagi_freecount = be32_to_cpu(agi->agi_freecount); + set_bit(XFS_AGSTATE_AGI_INIT, &pag->pag_opstate); + + return xrep_roll_ag_trans(sc); +} + +/* Repair the AGI. */ +int +xrep_agi( + struct xfs_scrub *sc) +{ + struct xrep_agi *ragi; + struct xfs_mount *mp = sc->mp; + unsigned int i; + int error; + + /* We require the rmapbt to rebuild anything. */ + if (!xfs_has_rmapbt(mp)) + return -EOPNOTSUPP; + + sc->buf = kzalloc_obj(struct xrep_agi, XCHK_GFP_FLAGS); + if (!sc->buf) + return -ENOMEM; + ragi = sc->buf; + ragi->sc = sc; + + ragi->fab[XREP_AGI_INOBT] = (struct xrep_find_ag_btree){ + .rmap_owner = XFS_RMAP_OWN_INOBT, + .buf_ops = &xfs_inobt_buf_ops, + .maxlevels = M_IGEO(sc->mp)->inobt_maxlevels, + }; + ragi->fab[XREP_AGI_FINOBT] = (struct xrep_find_ag_btree){ + .rmap_owner = XFS_RMAP_OWN_INOBT, + .buf_ops = &xfs_finobt_buf_ops, + .maxlevels = M_IGEO(sc->mp)->inobt_maxlevels, + }; + ragi->fab[XREP_AGI_END] = (struct xrep_find_ag_btree){ + .buf_ops = NULL, + }; + + for (i = 0; i < XFS_AGI_UNLINKED_BUCKETS; i++) + ragi->iunlink_heads[i] = NULLAGINO; + + xagino_bitmap_init(&ragi->iunlink_bmp); + sc->buf_cleanup = xrep_agi_buf_cleanup; + + error = xfarray_create("iunlinked next pointers", 0, + sizeof(xfs_agino_t), &ragi->iunlink_next); + if (error) + return error; + + error = xfarray_create("iunlinked prev pointers", 0, + sizeof(xfs_agino_t), &ragi->iunlink_prev); + if (error) + return error; + + /* + * Make sure we have the AGI buffer, as scrub might have decided it + * was corrupt after xfs_ialloc_read_agi failed with -EFSCORRUPTED. + */ + error = xfs_trans_read_buf(mp, sc->tp, mp->m_ddev_targp, + XFS_AG_DADDR(mp, pag_agno(sc->sa.pag), + XFS_AGI_DADDR(mp)), + XFS_FSS_TO_BB(mp, 1), 0, &ragi->agi_bp, NULL); + if (error) + return error; + ragi->agi_bp->b_ops = &xfs_agi_buf_ops; + + /* Find the AGI btree roots. */ + error = xrep_agi_find_btrees(ragi); + if (error) + return error; + + error = xrep_iunlink_rebuild_buckets(ragi); + if (error) + return error; + + /* Last chance to abort before we start committing fixes. */ + if (xchk_should_terminate(sc, &error)) + return error; + + /* Start rewriting the header and implant the btrees we found. */ + xrep_agi_init_header(ragi); + xrep_agi_set_roots(ragi); + error = xrep_agi_calc_from_btrees(ragi); + if (error) + goto out_revert; + error = xrep_iunlink_commit(ragi); + if (error) + goto out_revert; + + /* Reinitialize in-core state. */ + return xrep_agi_commit_new(ragi); + +out_revert: + /* Mark the incore AGI state stale and revert the AGI. */ + clear_bit(XFS_AGSTATE_AGI_INIT, &sc->sa.pag->pag_opstate); + memcpy(ragi->agi_bp->b_addr, &ragi->old_agi, sizeof(struct xfs_agi)); + return error; +} diff --git a/fs/xfs/scrub/agino_bitmap.h b/fs/xfs/scrub/agino_bitmap.h new file mode 100644 index 000000000..56d7db5f1 --- /dev/null +++ b/fs/xfs/scrub/agino_bitmap.h @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2018-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_AGINO_BITMAP_H__ +#define __XFS_SCRUB_AGINO_BITMAP_H__ + +/* Bitmaps, but for type-checked for xfs_agino_t */ + +struct xagino_bitmap { + struct xbitmap32 aginobitmap; +}; + +static inline void xagino_bitmap_init(struct xagino_bitmap *bitmap) +{ + xbitmap32_init(&bitmap->aginobitmap); +} + +static inline void xagino_bitmap_destroy(struct xagino_bitmap *bitmap) +{ + xbitmap32_destroy(&bitmap->aginobitmap); +} + +static inline int xagino_bitmap_clear(struct xagino_bitmap *bitmap, + xfs_agino_t agino, unsigned int len) +{ + return xbitmap32_clear(&bitmap->aginobitmap, agino, len); +} + +static inline int xagino_bitmap_set(struct xagino_bitmap *bitmap, + xfs_agino_t agino, unsigned int len) +{ + return xbitmap32_set(&bitmap->aginobitmap, agino, len); +} + +static inline bool xagino_bitmap_test(struct xagino_bitmap *bitmap, + xfs_agino_t agino, unsigned int *len) +{ + return xbitmap32_test(&bitmap->aginobitmap, agino, len); +} + +static inline int xagino_bitmap_walk(struct xagino_bitmap *bitmap, + xbitmap32_walk_fn fn, void *priv) +{ + return xbitmap32_walk(&bitmap->aginobitmap, fn, priv); +} + +#endif /* __XFS_SCRUB_AGINO_BITMAP_H__ */ diff --git a/fs/xfs/scrub/alloc.c b/fs/xfs/scrub/alloc.c new file mode 100644 index 000000000..c666be69f --- /dev/null +++ b/fs/xfs/scrub/alloc.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_btree.h" +#include "xfs_alloc.h" +#include "xfs_rmap.h" +#include "xfs_ag.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/btree.h" +#include "scrub/repair.h" + +/* + * Set us up to scrub free space btrees. + */ +int +xchk_setup_ag_allocbt( + struct xfs_scrub *sc) +{ + int error; + + if (xchk_need_intent_drain(sc)) + xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN); + + error = xchk_setup_ag_btree(sc, false); + if (error) + return error; + + if (xchk_could_repair(sc)) + return xrep_setup_ag_allocbt(sc); + + return 0; +} + +/* Free space btree scrubber. */ + +struct xchk_alloc { + /* Previous free space extent. */ + struct xfs_alloc_rec_incore prev; +}; + +/* + * Ensure there's a corresponding cntbt/bnobt record matching this + * bnobt/cntbt record, respectively. + */ +STATIC void +xchk_allocbt_xref_other( + struct xfs_scrub *sc, + xfs_agblock_t agbno, + xfs_extlen_t len) +{ + struct xfs_btree_cur **pcur; + xfs_agblock_t fbno; + xfs_extlen_t flen; + int has_otherrec; + int error; + + if (sc->sm->sm_type == XFS_SCRUB_TYPE_BNOBT) + pcur = &sc->sa.cnt_cur; + else + pcur = &sc->sa.bno_cur; + if (!*pcur || xchk_skip_xref(sc->sm)) + return; + + error = xfs_alloc_lookup_le(*pcur, agbno, len, &has_otherrec); + if (!xchk_should_check_xref(sc, &error, pcur)) + return; + if (!has_otherrec) { + xchk_btree_xref_set_corrupt(sc, *pcur, 0); + return; + } + + error = xfs_alloc_get_rec(*pcur, &fbno, &flen, &has_otherrec); + if (!xchk_should_check_xref(sc, &error, pcur)) + return; + if (!has_otherrec) { + xchk_btree_xref_set_corrupt(sc, *pcur, 0); + return; + } + + if (fbno != agbno || flen != len) + xchk_btree_xref_set_corrupt(sc, *pcur, 0); +} + +/* Cross-reference with the other btrees. */ +STATIC void +xchk_allocbt_xref( + struct xfs_scrub *sc, + const struct xfs_alloc_rec_incore *irec) +{ + xfs_agblock_t agbno = irec->ar_startblock; + xfs_extlen_t len = irec->ar_blockcount; + + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return; + + xchk_allocbt_xref_other(sc, agbno, len); + xchk_xref_is_not_inode_chunk(sc, agbno, len); + xchk_xref_has_no_owner(sc, agbno, len); + xchk_xref_is_not_shared(sc, agbno, len); + xchk_xref_is_not_cow_staging(sc, agbno, len); +} + +/* Flag failures for records that could be merged. */ +STATIC void +xchk_allocbt_mergeable( + struct xchk_btree *bs, + struct xchk_alloc *ca, + const struct xfs_alloc_rec_incore *irec) +{ + if (bs->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return; + + if (ca->prev.ar_blockcount > 0 && + ca->prev.ar_startblock + ca->prev.ar_blockcount == irec->ar_startblock && + ca->prev.ar_blockcount + irec->ar_blockcount < (uint32_t)~0U) + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + + memcpy(&ca->prev, irec, sizeof(*irec)); +} + +/* Scrub a bnobt/cntbt record. */ +STATIC int +xchk_allocbt_rec( + struct xchk_btree *bs, + const union xfs_btree_rec *rec) +{ + struct xfs_alloc_rec_incore irec; + struct xchk_alloc *ca = bs->private; + + xfs_alloc_btrec_to_irec(rec, &irec); + if (xfs_alloc_check_irec(to_perag(bs->cur->bc_group), &irec) != NULL) { + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + return 0; + } + + if (bs->sc->sm->sm_type == XFS_SCRUB_TYPE_BNOBT) + xchk_allocbt_mergeable(bs, ca, &irec); + xchk_allocbt_xref(bs->sc, &irec); + + return 0; +} + +/* Scrub one of the freespace btrees for some AG. */ +int +xchk_allocbt( + struct xfs_scrub *sc) +{ + struct xchk_alloc ca = { }; + struct xfs_btree_cur *cur; + + switch (sc->sm->sm_type) { + case XFS_SCRUB_TYPE_BNOBT: + cur = sc->sa.bno_cur; + break; + case XFS_SCRUB_TYPE_CNTBT: + cur = sc->sa.cnt_cur; + break; + default: + ASSERT(0); + return -EIO; + } + + return xchk_btree(sc, cur, xchk_allocbt_rec, &XFS_RMAP_OINFO_AG, &ca); +} + +/* xref check that the extent is not free */ +void +xchk_xref_is_used_space( + struct xfs_scrub *sc, + xfs_agblock_t agbno, + xfs_extlen_t len) +{ + enum xbtree_recpacking outcome; + int error; + + if (!sc->sa.bno_cur || xchk_skip_xref(sc->sm)) + return; + + error = xfs_alloc_has_records(sc->sa.bno_cur, agbno, len, &outcome); + if (!xchk_should_check_xref(sc, &error, &sc->sa.bno_cur)) + return; + if (outcome != XBTREE_RECPACKING_EMPTY) + xchk_btree_xref_set_corrupt(sc, sc->sa.bno_cur, 0); +} diff --git a/fs/xfs/scrub/alloc_repair.c b/fs/xfs/scrub/alloc_repair.c new file mode 100644 index 000000000..dce6ab042 --- /dev/null +++ b/fs/xfs/scrub/alloc_repair.c @@ -0,0 +1,946 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_btree.h" +#include "xfs_btree_staging.h" +#include "xfs_bit.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_sb.h" +#include "xfs_alloc.h" +#include "xfs_alloc_btree.h" +#include "xfs_rmap.h" +#include "xfs_rmap_btree.h" +#include "xfs_inode.h" +#include "xfs_refcount.h" +#include "xfs_extent_busy.h" +#include "xfs_health.h" +#include "xfs_bmap.h" +#include "xfs_ialloc.h" +#include "xfs_ag.h" +#include "scrub/xfs_scrub.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/btree.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/bitmap.h" +#include "scrub/agb_bitmap.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/newbt.h" +#include "scrub/reap.h" + +/* + * Free Space Btree Repair + * ======================= + * + * The reverse mappings are supposed to record all space usage for the entire + * AG. Therefore, we can recreate the free extent records in an AG by looking + * for gaps in the physical extents recorded in the rmapbt. These records are + * staged in @free_records. Identifying the gaps is more difficult on a + * reflink filesystem because rmap records are allowed to overlap. + * + * Because the final step of building a new index is to free the space used by + * the old index, repair needs to find that space. Unfortunately, all + * structures that live in the free space (bnobt, cntbt, rmapbt, agfl) share + * the same rmapbt owner code (OWN_AG), so this is not straightforward. + * + * The scan of the reverse mapping information records the space used by OWN_AG + * in @old_allocbt_blocks, which (at this stage) is somewhat misnamed. While + * walking the rmapbt records, we create a second bitmap @not_allocbt_blocks to + * record all visited rmap btree blocks and all blocks owned by the AGFL. + * + * After that is where the definitions of old_allocbt_blocks shifts. This + * expression identifies possible former bnobt/cntbt blocks: + * + * (OWN_AG blocks) & ~(rmapbt blocks | agfl blocks); + * + * Substituting from above definitions, that becomes: + * + * old_allocbt_blocks & ~not_allocbt_blocks + * + * The OWN_AG bitmap itself isn't needed after this point, so what we really do + * instead is: + * + * old_allocbt_blocks &= ~not_allocbt_blocks; + * + * After this point, @old_allocbt_blocks is a bitmap of alleged former + * bnobt/cntbt blocks. The xagb_bitmap_disunion operation modifies its first + * parameter in place to avoid copying records around. + * + * Next, some of the space described by @free_records are diverted to the newbt + * reservation and used to format new btree blocks. The remaining records are + * written to the new btree indices. We reconstruct both bnobt and cntbt at + * the same time since we've already done all the work. + * + * We use the prefix 'xrep_abt' here because we regenerate both free space + * allocation btrees at the same time. + */ + +struct xrep_abt { + /* Blocks owned by the rmapbt or the agfl. */ + struct xagb_bitmap not_allocbt_blocks; + + /* All OWN_AG blocks. */ + struct xagb_bitmap old_allocbt_blocks; + + /* + * New bnobt information. All btree block reservations are added to + * the reservation list in new_bnobt. + */ + struct xrep_newbt new_bnobt; + + /* new cntbt information */ + struct xrep_newbt new_cntbt; + + /* Free space extents. */ + struct xfarray *free_records; + + struct xfs_scrub *sc; + + /* Number of non-null records in @free_records. */ + uint64_t nr_real_records; + + /* get_records()'s position in the free space record array. */ + xfarray_idx_t array_cur; + + /* + * Next block we anticipate seeing in the rmap records. If the next + * rmap record is greater than next_agbno, we have found unused space. + */ + xfs_agblock_t next_agbno; + + /* Number of free blocks in this AG. */ + xfs_agblock_t nr_blocks; + + /* Longest free extent we found in the AG. */ + xfs_agblock_t longest; +}; + +/* Set up to repair AG free space btrees. */ +int +xrep_setup_ag_allocbt( + struct xfs_scrub *sc) +{ + struct xfs_group *xg = pag_group(sc->sa.pag); + unsigned int busy_gen; + + /* + * Make sure the busy extent list is clear because we can't put extents + * on there twice. + */ + if (xfs_extent_busy_list_empty(xg, &busy_gen)) + return 0; + return xfs_extent_busy_flush(sc->tp, xg, busy_gen, 0); +} + +/* Check for any obvious conflicts in the free extent. */ +STATIC int +xrep_abt_check_free_ext( + struct xfs_scrub *sc, + const struct xfs_alloc_rec_incore *rec) +{ + enum xbtree_recpacking outcome; + int error; + + if (xfs_alloc_check_irec(sc->sa.pag, rec) != NULL) + return -EFSCORRUPTED; + + /* Must not be an inode chunk. */ + error = xfs_ialloc_has_inodes_at_extent(sc->sa.ino_cur, + rec->ar_startblock, rec->ar_blockcount, &outcome); + if (error) + return error; + if (outcome != XBTREE_RECPACKING_EMPTY) + return -EFSCORRUPTED; + + /* Must not be shared or CoW staging. */ + if (sc->sa.refc_cur) { + error = xfs_refcount_has_records(sc->sa.refc_cur, + XFS_REFC_DOMAIN_SHARED, rec->ar_startblock, + rec->ar_blockcount, &outcome); + if (error) + return error; + if (outcome != XBTREE_RECPACKING_EMPTY) + return -EFSCORRUPTED; + + error = xfs_refcount_has_records(sc->sa.refc_cur, + XFS_REFC_DOMAIN_COW, rec->ar_startblock, + rec->ar_blockcount, &outcome); + if (error) + return error; + if (outcome != XBTREE_RECPACKING_EMPTY) + return -EFSCORRUPTED; + } + + return 0; +} + +/* + * Stash a free space record for all the space since the last bno we found + * all the way up to @end. + */ +static int +xrep_abt_stash( + struct xrep_abt *ra, + xfs_agblock_t end) +{ + struct xfs_alloc_rec_incore arec = { + .ar_startblock = ra->next_agbno, + .ar_blockcount = end - ra->next_agbno, + }; + struct xfs_scrub *sc = ra->sc; + int error = 0; + + if (xchk_should_terminate(sc, &error)) + return error; + + error = xrep_abt_check_free_ext(ra->sc, &arec); + if (error) + return error; + + trace_xrep_abt_found(sc->sa.pag, &arec); + + error = xfarray_append(ra->free_records, &arec); + if (error) + return error; + + ra->nr_blocks += arec.ar_blockcount; + return 0; +} + +/* Record extents that aren't in use from gaps in the rmap records. */ +STATIC int +xrep_abt_walk_rmap( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv) +{ + struct xrep_abt *ra = priv; + int error; + + /* Record all the OWN_AG blocks... */ + if (rec->rm_owner == XFS_RMAP_OWN_AG) { + error = xagb_bitmap_set(&ra->old_allocbt_blocks, + rec->rm_startblock, rec->rm_blockcount); + if (error) + return error; + } + + /* ...and all the rmapbt blocks... */ + error = xagb_bitmap_set_btcur_path(&ra->not_allocbt_blocks, cur); + if (error) + return error; + + /* ...and all the free space. */ + if (rec->rm_startblock > ra->next_agbno) { + error = xrep_abt_stash(ra, rec->rm_startblock); + if (error) + return error; + } + + /* + * rmap records can overlap on reflink filesystems, so project + * next_agbno as far out into the AG space as we currently know about. + */ + ra->next_agbno = max_t(xfs_agblock_t, ra->next_agbno, + rec->rm_startblock + rec->rm_blockcount); + return 0; +} + +/* Collect an AGFL block for the not-to-release list. */ +static int +xrep_abt_walk_agfl( + struct xfs_mount *mp, + xfs_agblock_t agbno, + void *priv) +{ + struct xrep_abt *ra = priv; + + return xagb_bitmap_set(&ra->not_allocbt_blocks, agbno, 1); +} + +/* + * Compare two free space extents by block number. We want to sort in order of + * increasing block number. + */ +static int +xrep_bnobt_extent_cmp( + const void *a, + const void *b) +{ + const struct xfs_alloc_rec_incore *ap = a; + const struct xfs_alloc_rec_incore *bp = b; + + if (ap->ar_startblock > bp->ar_startblock) + return 1; + else if (ap->ar_startblock < bp->ar_startblock) + return -1; + return 0; +} + +/* + * Re-sort the free extents by block number so that we can put the records into + * the bnobt in the correct order. Make sure the records do not overlap in + * physical space. + */ +STATIC int +xrep_bnobt_sort_records( + struct xrep_abt *ra) +{ + struct xfs_alloc_rec_incore arec; + xfarray_idx_t cur = XFARRAY_CURSOR_INIT; + xfs_agblock_t next_agbno = 0; + int error; + + error = xfarray_sort(ra->free_records, xrep_bnobt_extent_cmp, 0); + if (error) + return error; + + while ((error = xfarray_iter(ra->free_records, &cur, &arec)) == 1) { + if (arec.ar_startblock < next_agbno) + return -EFSCORRUPTED; + + next_agbno = arec.ar_startblock + arec.ar_blockcount; + } + + return error; +} + +/* + * Compare two free space extents by length and then block number. We want + * to sort first in order of increasing length and then in order of increasing + * block number. + */ +static int +xrep_cntbt_extent_cmp( + const void *a, + const void *b) +{ + const struct xfs_alloc_rec_incore *ap = a; + const struct xfs_alloc_rec_incore *bp = b; + + if (ap->ar_blockcount > bp->ar_blockcount) + return 1; + else if (ap->ar_blockcount < bp->ar_blockcount) + return -1; + return xrep_bnobt_extent_cmp(a, b); +} + +/* + * Sort the free extents by length so so that we can put the records into the + * cntbt in the correct order. Don't let userspace kill us if we're resorting + * after allocating btree blocks. + */ +STATIC int +xrep_cntbt_sort_records( + struct xrep_abt *ra, + bool is_resort) +{ + return xfarray_sort(ra->free_records, xrep_cntbt_extent_cmp, + is_resort ? 0 : XFARRAY_SORT_KILLABLE); +} + +/* + * Iterate all reverse mappings to find (1) the gaps between rmap records (all + * unowned space), (2) the OWN_AG extents (which encompass the free space + * btrees, the rmapbt, and the agfl), (3) the rmapbt blocks, and (4) the AGFL + * blocks. The free space is (1) + (2) - (3) - (4). + */ +STATIC int +xrep_abt_find_freespace( + struct xrep_abt *ra) +{ + struct xfs_scrub *sc = ra->sc; + struct xfs_mount *mp = sc->mp; + struct xfs_agf *agf = sc->sa.agf_bp->b_addr; + struct xfs_buf *agfl_bp; + xfs_agblock_t agend; + int error; + + xagb_bitmap_init(&ra->not_allocbt_blocks); + + xrep_ag_btcur_init(sc, &sc->sa); + + /* + * Iterate all the reverse mappings to find gaps in the physical + * mappings, all the OWN_AG blocks, and all the rmapbt extents. + */ + error = xfs_rmap_query_all(sc->sa.rmap_cur, xrep_abt_walk_rmap, ra); + if (error) + goto err; + + /* Insert a record for space between the last rmap and EOAG. */ + agend = be32_to_cpu(agf->agf_length); + if (ra->next_agbno < agend) { + error = xrep_abt_stash(ra, agend); + if (error) + goto err; + } + + /* Collect all the AGFL blocks. */ + error = xfs_alloc_read_agfl(sc->sa.pag, sc->tp, &agfl_bp); + if (error) + goto err; + + error = xfs_agfl_walk(mp, agf, agfl_bp, xrep_abt_walk_agfl, ra); + if (error) + goto err_agfl; + + /* Compute the old bnobt/cntbt blocks. */ + error = xagb_bitmap_disunion(&ra->old_allocbt_blocks, + &ra->not_allocbt_blocks); + if (error) + goto err_agfl; + + ra->nr_real_records = xfarray_length(ra->free_records); +err_agfl: + xfs_trans_brelse(sc->tp, agfl_bp); +err: + xchk_ag_btcur_free(&sc->sa); + xagb_bitmap_destroy(&ra->not_allocbt_blocks); + return error; +} + +/* + * We're going to use the observed free space records to reserve blocks for the + * new free space btrees, so we play an iterative game where we try to converge + * on the number of blocks we need: + * + * 1. Estimate how many blocks we'll need to store the records. + * 2. If the first free record has more blocks than we need, we're done. + * We will have to re-sort the records prior to building the cntbt. + * 3. If that record has exactly the number of blocks we need, null out the + * record. We're done. + * 4. Otherwise, we still need more blocks. Null out the record, subtract its + * length from the number of blocks we need, and go back to step 1. + * + * Fortunately, we don't have to do any transaction work to play this game, so + * we don't have to tear down the staging cursors. + */ +STATIC int +xrep_abt_reserve_space( + struct xrep_abt *ra, + struct xfs_btree_cur *bno_cur, + struct xfs_btree_cur *cnt_cur, + bool *needs_resort) +{ + struct xfs_scrub *sc = ra->sc; + xfarray_idx_t record_nr; + unsigned int allocated = 0; + int error = 0; + + record_nr = xfarray_length(ra->free_records) - 1; + do { + struct xfs_alloc_rec_incore arec; + uint64_t required; + unsigned int desired; + unsigned int len; + + /* Compute how many blocks we'll need. */ + error = xfs_btree_bload_compute_geometry(cnt_cur, + &ra->new_cntbt.bload, ra->nr_real_records); + if (error) + break; + + error = xfs_btree_bload_compute_geometry(bno_cur, + &ra->new_bnobt.bload, ra->nr_real_records); + if (error) + break; + + /* How many btree blocks do we need to store all records? */ + required = ra->new_bnobt.bload.nr_blocks + + ra->new_cntbt.bload.nr_blocks; + ASSERT(required < INT_MAX); + + /* If we've reserved enough blocks, we're done. */ + if (allocated >= required) + break; + + desired = required - allocated; + + /* We need space but there's none left; bye! */ + if (ra->nr_real_records == 0) { + error = -ENOSPC; + break; + } + + /* Grab the first record from the list. */ + error = xfarray_load(ra->free_records, record_nr, &arec); + if (error) + break; + + ASSERT(arec.ar_blockcount <= UINT_MAX); + len = min_t(unsigned int, arec.ar_blockcount, desired); + + trace_xrep_newbt_alloc_ag_blocks(sc->sa.pag, arec.ar_startblock, + len, XFS_RMAP_OWN_AG); + + error = xrep_newbt_add_extent(&ra->new_bnobt, sc->sa.pag, + arec.ar_startblock, len); + if (error) + break; + allocated += len; + ra->nr_blocks -= len; + + if (arec.ar_blockcount > desired) { + /* + * Record has more space than we need. The number of + * free records doesn't change, so shrink the free + * record, inform the caller that the records are no + * longer sorted by length, and exit. + */ + arec.ar_startblock += desired; + arec.ar_blockcount -= desired; + error = xfarray_store(ra->free_records, record_nr, + &arec); + if (error) + break; + + *needs_resort = true; + return 0; + } + + /* + * We're going to use up the entire record, so unset it and + * move on to the next one. This changes the number of free + * records (but doesn't break the sorting order), so we must + * go around the loop once more to re-run _bload_init. + */ + error = xfarray_unset(ra->free_records, record_nr); + if (error) + break; + ra->nr_real_records--; + record_nr--; + } while (1); + + return error; +} + +STATIC int +xrep_abt_dispose_one( + struct xrep_abt *ra, + struct xrep_newbt_resv *resv) +{ + struct xfs_scrub *sc = ra->sc; + struct xfs_perag *pag = sc->sa.pag; + xfs_agblock_t free_agbno = resv->agbno + resv->used; + xfs_extlen_t free_aglen = resv->len - resv->used; + int error; + + ASSERT(pag == resv->pag); + + /* Add a deferred rmap for each extent we used. */ + if (resv->used > 0) + xfs_rmap_alloc_extent(sc->tp, false, + xfs_agbno_to_fsb(pag, resv->agbno), resv->used, + XFS_RMAP_OWN_AG); + + /* + * For each reserved btree block we didn't use, add it to the free + * space btree. We didn't touch fdblocks when we reserved them, so + * we don't touch it now. + */ + if (free_aglen == 0) + return 0; + + trace_xrep_newbt_free_blocks(resv->pag, free_agbno, free_aglen, + ra->new_bnobt.oinfo.oi_owner); + + error = __xfs_free_extent(sc->tp, resv->pag, free_agbno, free_aglen, + &ra->new_bnobt.oinfo, XFS_AG_RESV_IGNORE, true); + if (error) + return error; + + return xrep_defer_finish(sc); +} + +/* + * Deal with all the space we reserved. Blocks that were allocated for the + * free space btrees need to have a (deferred) rmap added for the OWN_AG + * allocation, and blocks that didn't get used can be freed via the usual + * (deferred) means. + */ +STATIC void +xrep_abt_dispose_reservations( + struct xrep_abt *ra, + int error) +{ + struct xrep_newbt_resv *resv, *n; + + if (error) + goto junkit; + + list_for_each_entry_safe(resv, n, &ra->new_bnobt.resv_list, list) { + error = xrep_abt_dispose_one(ra, resv); + if (error) + goto junkit; + } + +junkit: + list_for_each_entry_safe(resv, n, &ra->new_bnobt.resv_list, list) { + xfs_perag_put(resv->pag); + list_del(&resv->list); + kfree(resv); + } + + xrep_newbt_cancel(&ra->new_bnobt); + xrep_newbt_cancel(&ra->new_cntbt); +} + +/* Retrieve free space data for bulk load. */ +STATIC int +xrep_abt_get_records( + struct xfs_btree_cur *cur, + unsigned int idx, + struct xfs_btree_block *block, + unsigned int nr_wanted, + void *priv) +{ + struct xfs_alloc_rec_incore *arec = &cur->bc_rec.a; + struct xrep_abt *ra = priv; + union xfs_btree_rec *block_rec; + unsigned int loaded; + int error; + + for (loaded = 0; loaded < nr_wanted; loaded++, idx++) { + error = xfarray_load_next(ra->free_records, &ra->array_cur, + arec); + if (error) + return error; + + ra->longest = max(ra->longest, arec->ar_blockcount); + + block_rec = xfs_btree_rec_addr(cur, idx, block); + cur->bc_ops->init_rec_from_cur(cur, block_rec); + } + + return loaded; +} + +/* Feed one of the new btree blocks to the bulk loader. */ +STATIC int +xrep_abt_claim_block( + struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr, + void *priv) +{ + struct xrep_abt *ra = priv; + + return xrep_newbt_claim_block(cur, &ra->new_bnobt, ptr); +} + +/* + * Reset the AGF counters to reflect the free space btrees that we just + * rebuilt, then reinitialize the per-AG data. + */ +STATIC int +xrep_abt_reset_counters( + struct xrep_abt *ra) +{ + struct xfs_scrub *sc = ra->sc; + struct xfs_perag *pag = sc->sa.pag; + struct xfs_agf *agf = sc->sa.agf_bp->b_addr; + unsigned int freesp_btreeblks = 0; + + /* + * Compute the contribution to agf_btreeblks for the new free space + * btrees. This is the computed btree size minus anything we didn't + * use. + */ + freesp_btreeblks += ra->new_bnobt.bload.nr_blocks - 1; + freesp_btreeblks += ra->new_cntbt.bload.nr_blocks - 1; + + freesp_btreeblks -= xrep_newbt_unused_blocks(&ra->new_bnobt); + freesp_btreeblks -= xrep_newbt_unused_blocks(&ra->new_cntbt); + + /* + * The AGF header contains extra information related to the free space + * btrees, so we must update those fields here. + */ + agf->agf_btreeblks = cpu_to_be32(freesp_btreeblks + + (be32_to_cpu(agf->agf_rmap_blocks) - 1)); + agf->agf_freeblks = cpu_to_be32(ra->nr_blocks); + agf->agf_longest = cpu_to_be32(ra->longest); + xfs_alloc_log_agf(sc->tp, sc->sa.agf_bp, XFS_AGF_BTREEBLKS | + XFS_AGF_LONGEST | + XFS_AGF_FREEBLKS); + + /* + * After we commit the new btree to disk, it is possible that the + * process to reap the old btree blocks will race with the AIL trying + * to checkpoint the old btree blocks into the filesystem. If the new + * tree is shorter than the old one, the allocbt write verifier will + * fail and the AIL will shut down the filesystem. + * + * To avoid this, save the old incore btree height values as the alt + * height values before re-initializing the perag info from the updated + * AGF to capture all the new values. + */ + pag->pagf_repair_bno_level = pag->pagf_bno_level; + pag->pagf_repair_cnt_level = pag->pagf_cnt_level; + + /* Reinitialize with the values we just logged. */ + return xrep_reinit_pagf(sc); +} + +/* + * Use the collected free space information to stage new free space btrees. + * If this is successful we'll return with the new btree root + * information logged to the repair transaction but not yet committed. + */ +STATIC int +xrep_abt_build_new_trees( + struct xrep_abt *ra) +{ + struct xfs_scrub *sc = ra->sc; + struct xfs_btree_cur *bno_cur; + struct xfs_btree_cur *cnt_cur; + struct xfs_perag *pag = sc->sa.pag; + bool needs_resort = false; + int error; + + /* + * Sort the free extents by length so that we can set up the free space + * btrees in as few extents as possible. This reduces the amount of + * deferred rmap / free work we have to do at the end. + */ + error = xrep_cntbt_sort_records(ra, false); + if (error) + return error; + + /* + * Prepare to construct the new btree by reserving disk space for the + * new btree and setting up all the accounting information we'll need + * to root the new btree while it's under construction and before we + * attach it to the AG header. + */ + xrep_newbt_init_bare(&ra->new_bnobt, sc); + xrep_newbt_init_bare(&ra->new_cntbt, sc); + + ra->new_bnobt.bload.get_records = xrep_abt_get_records; + ra->new_cntbt.bload.get_records = xrep_abt_get_records; + + ra->new_bnobt.bload.claim_block = xrep_abt_claim_block; + ra->new_cntbt.bload.claim_block = xrep_abt_claim_block; + + /* Allocate cursors for the staged btrees. */ + bno_cur = xfs_bnobt_init_cursor(sc->mp, NULL, NULL, pag); + xfs_btree_stage_afakeroot(bno_cur, &ra->new_bnobt.afake); + + cnt_cur = xfs_cntbt_init_cursor(sc->mp, NULL, NULL, pag); + xfs_btree_stage_afakeroot(cnt_cur, &ra->new_cntbt.afake); + + /* Last chance to abort before we start committing fixes. */ + if (xchk_should_terminate(sc, &error)) + goto err_cur; + + /* Reserve the space we'll need for the new btrees. */ + error = xrep_abt_reserve_space(ra, bno_cur, cnt_cur, &needs_resort); + if (error) + goto err_cur; + + /* + * If we need to re-sort the free extents by length, do so so that we + * can put the records into the cntbt in the correct order. + */ + if (needs_resort) { + error = xrep_cntbt_sort_records(ra, needs_resort); + if (error) + goto err_cur; + } + + /* + * Due to btree slack factors, it's possible for a new btree to be one + * level taller than the old btree. Update the alternate incore btree + * height so that we don't trip the verifiers when writing the new + * btree blocks to disk. + */ + pag->pagf_repair_bno_level = ra->new_bnobt.bload.btree_height; + pag->pagf_repair_cnt_level = ra->new_cntbt.bload.btree_height; + + /* Load the free space by length tree. */ + ra->array_cur = XFARRAY_CURSOR_INIT; + ra->longest = 0; + error = xfs_btree_bload(cnt_cur, &ra->new_cntbt.bload, ra); + if (error) + goto err_levels; + + error = xrep_bnobt_sort_records(ra); + if (error) + goto err_levels; + + /* Load the free space by block number tree. */ + ra->array_cur = XFARRAY_CURSOR_INIT; + error = xfs_btree_bload(bno_cur, &ra->new_bnobt.bload, ra); + if (error) + goto err_levels; + + /* + * Install the new btrees in the AG header. After this point the old + * btrees are no longer accessible and the new trees are live. + */ + xfs_allocbt_commit_staged_btree(bno_cur, sc->tp, sc->sa.agf_bp); + xfs_btree_del_cursor(bno_cur, 0); + xfs_allocbt_commit_staged_btree(cnt_cur, sc->tp, sc->sa.agf_bp); + xfs_btree_del_cursor(cnt_cur, 0); + + /* Reset the AGF counters now that we've changed the btree shape. */ + error = xrep_abt_reset_counters(ra); + if (error) + goto err_newbt; + + /* Dispose of any unused blocks and the accounting information. */ + xrep_abt_dispose_reservations(ra, error); + + return xrep_roll_ag_trans(sc); + +err_levels: + pag->pagf_repair_bno_level = 0; + pag->pagf_repair_cnt_level = 0; +err_cur: + xfs_btree_del_cursor(cnt_cur, error); + xfs_btree_del_cursor(bno_cur, error); +err_newbt: + xrep_abt_dispose_reservations(ra, error); + return error; +} + +/* + * Now that we've logged the roots of the new btrees, invalidate all of the + * old blocks and free them. + */ +STATIC int +xrep_abt_remove_old_trees( + struct xrep_abt *ra) +{ + struct xfs_perag *pag = ra->sc->sa.pag; + int error; + + /* Free the old btree blocks if they're not in use. */ + error = xrep_reap_agblocks(ra->sc, &ra->old_allocbt_blocks, + &XFS_RMAP_OINFO_AG, XFS_AG_RESV_IGNORE); + if (error) + return error; + + /* + * Now that we've zapped all the old allocbt blocks we can turn off + * the alternate height mechanism. + */ + pag->pagf_repair_bno_level = 0; + pag->pagf_repair_cnt_level = 0; + return 0; +} + +/* Repair the freespace btrees for some AG. */ +int +xrep_allocbt( + struct xfs_scrub *sc) +{ + struct xrep_abt *ra; + struct xfs_mount *mp = sc->mp; + unsigned int busy_gen; + int error; + + /* We require the rmapbt to rebuild anything. */ + if (!xfs_has_rmapbt(mp)) + return -EOPNOTSUPP; + + ra = kzalloc_obj(struct xrep_abt, XCHK_GFP_FLAGS); + if (!ra) + return -ENOMEM; + ra->sc = sc; + + /* We rebuild both data structures. */ + sc->sick_mask = XFS_SICK_AG_BNOBT | XFS_SICK_AG_CNTBT; + + /* + * Make sure the busy extent list is clear because we can't put extents + * on there twice. In theory we cleared this before we started, but + * let's not risk the filesystem. + */ + if (!xfs_extent_busy_list_empty(pag_group(sc->sa.pag), &busy_gen)) { + error = -EDEADLOCK; + goto out_ra; + } + + /* Set up enough storage to handle maximally fragmented free space. */ + error = xfarray_create("free space records", mp->m_sb.sb_agblocks / 2, + sizeof(struct xfs_alloc_rec_incore), + &ra->free_records); + if (error) + goto out_ra; + + /* Collect the free space data and find the old btree blocks. */ + xagb_bitmap_init(&ra->old_allocbt_blocks); + error = xrep_abt_find_freespace(ra); + if (error) + goto out_bitmap; + + /* Rebuild the free space information. */ + error = xrep_abt_build_new_trees(ra); + if (error) + goto out_bitmap; + + /* Kill the old trees. */ + error = xrep_abt_remove_old_trees(ra); + if (error) + goto out_bitmap; + +out_bitmap: + xagb_bitmap_destroy(&ra->old_allocbt_blocks); + xfarray_destroy(ra->free_records); +out_ra: + kfree(ra); + return error; +} + +/* Make sure both btrees are ok after we've rebuilt them. */ +int +xrep_revalidate_allocbt( + struct xfs_scrub *sc) +{ + __u32 old_type = sc->sm->sm_type; + int error; + + /* + * We must update sm_type temporarily so that the tree-to-tree cross + * reference checks will work in the correct direction, and also so + * that tracing will report correctly if there are more errors. + */ + sc->sm->sm_type = XFS_SCRUB_TYPE_BNOBT; + error = xchk_allocbt(sc); + if (error) + goto out; + + /* + * If the bnobt is still corrupt, we've failed to repair the filesystem + * and should just bail out. + * + * If the bnobt fails cross-examination with the cntbt, the scan will + * free the cntbt cursor, so we need to mark the repair incomplete + * and avoid walking off the end of the NULL cntbt cursor. + */ + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out; + + sc->sm->sm_type = XFS_SCRUB_TYPE_CNTBT; + if (!sc->sa.cnt_cur) { + xchk_set_incomplete(sc); + goto out; + } + error = xchk_allocbt(sc); +out: + sc->sm->sm_type = old_type; + return error; +} diff --git a/fs/xfs/scrub/attr.c b/fs/xfs/scrub/attr.c new file mode 100644 index 000000000..d5226939c --- /dev/null +++ b/fs/xfs/scrub/attr.c @@ -0,0 +1,686 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_inode.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_attr.h" +#include "xfs_attr_leaf.h" +#include "xfs_attr_sf.h" +#include "xfs_parent.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/dabtree.h" +#include "scrub/attr.h" +#include "scrub/listxattr.h" +#include "scrub/repair.h" + +/* Free the buffers linked from the xattr buffer. */ +static void +xchk_xattr_buf_cleanup( + void *priv) +{ + struct xchk_xattr_buf *ab = priv; + + kvfree(ab->freemap); + ab->freemap = NULL; + kvfree(ab->usedmap); + ab->usedmap = NULL; + kvfree(ab->value); + ab->value = NULL; + ab->value_sz = 0; + kvfree(ab->name); + ab->name = NULL; +} + +/* + * Allocate the free space bitmap if we're trying harder; there are leaf blocks + * in the attr fork; or we can't tell if there are leaf blocks. + */ +static inline bool +xchk_xattr_want_freemap( + struct xfs_scrub *sc) +{ + struct xfs_ifork *ifp; + + if (sc->flags & XCHK_TRY_HARDER) + return true; + + if (!sc->ip) + return true; + + ifp = xfs_ifork_ptr(sc->ip, XFS_ATTR_FORK); + if (!ifp) + return false; + + return xfs_ifork_has_extents(ifp); +} + +/* + * Allocate enough memory to hold an attr value and attr block bitmaps, + * reallocating the buffer if necessary. Buffer contents are not preserved + * across a reallocation. + */ +int +xchk_setup_xattr_buf( + struct xfs_scrub *sc, + size_t value_size) +{ + size_t bmp_sz; + struct xchk_xattr_buf *ab = sc->buf; + void *new_val; + + bmp_sz = sizeof(long) * BITS_TO_LONGS(sc->mp->m_attr_geo->blksize); + + if (ab) + goto resize_value; + + ab = kvzalloc_obj(struct xchk_xattr_buf, XCHK_GFP_FLAGS); + if (!ab) + return -ENOMEM; + sc->buf = ab; + sc->buf_cleanup = xchk_xattr_buf_cleanup; + + ab->usedmap = kvmalloc(bmp_sz, XCHK_GFP_FLAGS); + if (!ab->usedmap) + return -ENOMEM; + + if (xchk_xattr_want_freemap(sc)) { + ab->freemap = kvmalloc(bmp_sz, XCHK_GFP_FLAGS); + if (!ab->freemap) + return -ENOMEM; + } + + if (xchk_could_repair(sc)) { + ab->name = kvmalloc(XATTR_NAME_MAX + 1, XCHK_GFP_FLAGS); + if (!ab->name) + return -ENOMEM; + } + +resize_value: + if (ab->value_sz >= value_size) + return 0; + + if (ab->value) { + kvfree(ab->value); + ab->value = NULL; + ab->value_sz = 0; + } + + new_val = kvmalloc(value_size, XCHK_GFP_FLAGS); + if (!new_val) + return -ENOMEM; + + ab->value = new_val; + ab->value_sz = value_size; + return 0; +} + +/* Set us up to scrub an inode's extended attributes. */ +int +xchk_setup_xattr( + struct xfs_scrub *sc) +{ + int error; + + if (xchk_could_repair(sc)) { + error = xrep_setup_xattr(sc); + if (error) + return error; + } + + /* + * We failed to get memory while checking attrs, so this time try to + * get all the memory we're ever going to need. Allocate the buffer + * without the inode lock held, which means we can sleep. + */ + if (sc->flags & XCHK_TRY_HARDER) { + error = xchk_setup_xattr_buf(sc, XATTR_SIZE_MAX); + if (error) + return error; + } + + return xchk_setup_inode_contents(sc, 0); +} + +/* Extended Attributes */ + +/* + * Check that an extended attribute key can be looked up by hash. + * + * We use the extended attribute walk helper to call this function for every + * attribute key in an inode. Once we're here, we load the attribute value to + * see if any errors happen, or if we get more or less data than we expected. + */ +static int +xchk_xattr_actor( + struct xfs_scrub *sc, + struct xfs_inode *ip, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + const void *value, + unsigned int valuelen, + void *priv) +{ + struct xfs_da_args args = { + .attr_filter = attr_flags & XFS_ATTR_NSP_ONDISK_MASK, + .geo = sc->mp->m_attr_geo, + .whichfork = XFS_ATTR_FORK, + .dp = ip, + .name = name, + .namelen = namelen, + .trans = sc->tp, + .valuelen = valuelen, + .owner = I_INO(ip), + }; + struct xchk_xattr_buf *ab; + int error = 0; + + ab = sc->buf; + + if (xchk_should_terminate(sc, &error)) + return error; + + if (attr_flags & ~XFS_ATTR_ONDISK_MASK) { + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, args.blkno); + return -ECANCELED; + } + + if (attr_flags & XFS_ATTR_INCOMPLETE) { + /* Incomplete attr key, just mark the inode for preening. */ + xchk_ino_set_preen(sc, I_INO(ip)); + return 0; + } + + /* Does this name make sense? */ + if (!xfs_attr_namecheck(attr_flags, name, namelen)) { + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, args.blkno); + return -ECANCELED; + } + + /* Check parent pointer record. */ + if ((attr_flags & XFS_ATTR_PARENT) && + !xfs_parent_valuecheck(sc->mp, value, valuelen)) { + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, args.blkno); + return -ECANCELED; + } + + /* + * Try to allocate enough memory to extract the attr value. If that + * doesn't work, return -EDEADLOCK as a signal to try again with a + * maximally sized buffer. + */ + error = xchk_setup_xattr_buf(sc, valuelen); + if (error == -ENOMEM) + error = -EDEADLOCK; + if (error) + return error; + + /* + * Parent pointers are matched on attr name and value, so we must + * supply the xfs_parent_rec here when confirming that the dabtree + * indexing works correctly. + */ + if (attr_flags & XFS_ATTR_PARENT) + memcpy(ab->value, value, valuelen); + + args.value = ab->value; + + /* + * Get the attr value to ensure that lookup can find this attribute + * through the dabtree indexing and that remote value retrieval also + * works correctly. + */ + xfs_attr_sethash(&args); + error = xfs_attr_get_ilocked(&args); + /* ENODATA means the hash lookup failed and the attr is bad */ + if (error == -ENODATA) + error = -EFSCORRUPTED; + if (!xchk_fblock_process_error(sc, XFS_ATTR_FORK, args.blkno, + &error)) + return error; + if (args.valuelen != valuelen) + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, args.blkno); + + return 0; +} + +/* + * Mark a range [start, start+len) in this map. Returns true if the + * region was free, and false if there's a conflict or a problem. + * + * Within a char, the lowest bit of the char represents the byte with + * the smallest address + */ +bool +xchk_xattr_set_map( + struct xfs_scrub *sc, + unsigned long *map, + unsigned int start, + unsigned int len) +{ + unsigned int mapsize = sc->mp->m_attr_geo->blksize; + bool ret = true; + + if (start >= mapsize) + return false; + if (start + len > mapsize) { + len = mapsize - start; + ret = false; + } + + if (find_next_bit(map, mapsize, start) < start + len) + ret = false; + bitmap_set(map, start, len); + + return ret; +} + +/* + * Check this leaf entry's relations to everything else. + * Returns the number of bytes used for the name/value data. + */ +STATIC void +xchk_xattr_entry( + struct xchk_da_btree *ds, + int level, + char *buf_end, + struct xfs_attr_leafblock *leaf, + struct xfs_attr3_icleaf_hdr *leafhdr, + struct xfs_attr_leaf_entry *ent, + int idx, + unsigned int *usedbytes, + __u32 *last_hashval) +{ + struct xfs_mount *mp = ds->state->mp; + struct xchk_xattr_buf *ab = ds->sc->buf; + char *name_end; + struct xfs_attr_leaf_name_local *lentry; + struct xfs_attr_leaf_name_remote *rentry; + unsigned int nameidx; + unsigned int namesize; + + if (ent->pad2 != 0) + xchk_da_set_corrupt(ds, level); + + /* Hash values in order? */ + if (be32_to_cpu(ent->hashval) < *last_hashval) + xchk_da_set_corrupt(ds, level); + *last_hashval = be32_to_cpu(ent->hashval); + + nameidx = be16_to_cpu(ent->nameidx); + if (nameidx < leafhdr->firstused || + nameidx >= mp->m_attr_geo->blksize) { + xchk_da_set_corrupt(ds, level); + return; + } + + /* Check the name information. */ + if (ent->flags & XFS_ATTR_LOCAL) { + lentry = xfs_attr3_leaf_name_local(leaf, idx); + namesize = xfs_attr_leaf_entsize_local(lentry->namelen, + be16_to_cpu(lentry->valuelen)); + name_end = (char *)lentry + namesize; + if (lentry->namelen == 0) + xchk_da_set_corrupt(ds, level); + } else { + rentry = xfs_attr3_leaf_name_remote(leaf, idx); + namesize = xfs_attr_leaf_entsize_remote(rentry->namelen); + name_end = (char *)rentry + namesize; + if (rentry->namelen == 0) + xchk_da_set_corrupt(ds, level); + if (rentry->valueblk == 0 && + !(ent->flags & XFS_ATTR_INCOMPLETE)) + xchk_da_set_corrupt(ds, level); + } + if (name_end > buf_end) + xchk_da_set_corrupt(ds, level); + + if (!xchk_xattr_set_map(ds->sc, ab->usedmap, nameidx, namesize)) + xchk_da_set_corrupt(ds, level); + if (!(ds->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) + *usedbytes += namesize; +} + +/* Scrub an attribute leaf. */ +STATIC int +xchk_xattr_block( + struct xchk_da_btree *ds, + int level) +{ + struct xfs_attr3_icleaf_hdr leafhdr; + struct xfs_mount *mp = ds->state->mp; + struct xfs_da_state_blk *blk = &ds->state->path.blk[level]; + struct xfs_buf *bp = blk->bp; + xfs_dablk_t *last_checked = ds->private; + struct xfs_attr_leafblock *leaf = bp->b_addr; + struct xfs_attr_leaf_entry *ent; + struct xfs_attr_leaf_entry *entries; + struct xchk_xattr_buf *ab = ds->sc->buf; + char *buf_end; + size_t off; + __u32 last_hashval = 0; + unsigned int usedbytes = 0; + unsigned int hdrsize; + int i; + + if (*last_checked == blk->blkno) + return 0; + + *last_checked = blk->blkno; + bitmap_zero(ab->usedmap, mp->m_attr_geo->blksize); + bitmap_zero(ab->freemap, mp->m_attr_geo->blksize); + + /* Check all the padding. */ + if (xfs_has_crc(ds->sc->mp)) { + struct xfs_attr3_leafblock *leaf3 = bp->b_addr; + + if (leaf3->hdr.pad1 != 0 || leaf3->hdr.pad2 != 0 || + leaf3->hdr.info.hdr.pad != 0) + xchk_da_set_corrupt(ds, level); + } else { + if (leaf->hdr.pad1 != 0 || leaf->hdr.info.pad != 0) + xchk_da_set_corrupt(ds, level); + } + + /* Check the leaf header */ + xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, &leafhdr, leaf); + hdrsize = xfs_attr3_leaf_hdr_size(leaf); + + /* + * Empty xattr leaf blocks mapped at block 0 are probably a byproduct + * of a race between setxattr and a log shutdown. Anywhere else in the + * attr fork is a corruption. + */ + if (leafhdr.count == 0) { + if (blk->blkno == 0) + xchk_da_set_preen(ds, level); + else + xchk_da_set_corrupt(ds, level); + } + if (leafhdr.usedbytes > mp->m_attr_geo->blksize) + xchk_da_set_corrupt(ds, level); + if (leafhdr.firstused > mp->m_attr_geo->blksize) + xchk_da_set_corrupt(ds, level); + if (leafhdr.firstused < hdrsize) + xchk_da_set_corrupt(ds, level); + if (!xchk_xattr_set_map(ds->sc, ab->usedmap, 0, hdrsize)) + xchk_da_set_corrupt(ds, level); + if (leafhdr.holes) + xchk_da_set_preen(ds, level); + + if (ds->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out; + + entries = xfs_attr3_leaf_entryp(leaf); + if ((char *)&entries[leafhdr.count] > (char *)leaf + leafhdr.firstused) + xchk_da_set_corrupt(ds, level); + + if (ds->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out; + + buf_end = (char *)bp->b_addr + mp->m_attr_geo->blksize; + for (i = 0, ent = entries; i < leafhdr.count; ent++, i++) { + /* Mark the leaf entry itself. */ + off = (char *)ent - (char *)leaf; + if (!xchk_xattr_set_map(ds->sc, ab->usedmap, off, + sizeof(xfs_attr_leaf_entry_t))) { + xchk_da_set_corrupt(ds, level); + goto out; + } + + /* Check the entry and nameval. */ + xchk_xattr_entry(ds, level, buf_end, leaf, &leafhdr, + ent, i, &usedbytes, &last_hashval); + + if (ds->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out; + } + + /* Construct bitmap of freemap contents. */ + for (i = 0; i < XFS_ATTR_LEAF_MAPSIZE; i++) { + if (!xchk_xattr_set_map(ds->sc, ab->freemap, + leafhdr.freemap[i].base, + leafhdr.freemap[i].size)) + xchk_da_set_corrupt(ds, level); + + /* + * freemap entries with zero length and nonzero base can cause + * problems with older kernels, so we mark these for preening + * even though there's no inconsistency. + */ + if (leafhdr.freemap[i].size == 0 && + leafhdr.freemap[i].base > 0) + xchk_da_set_preen(ds, level); + + if (ds->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out; + } + + /* Look for bits that are set in freemap and are marked in use. */ + if (bitmap_intersects(ab->freemap, ab->usedmap, + mp->m_attr_geo->blksize)) + xchk_da_set_corrupt(ds, level); + + if (leafhdr.usedbytes != usedbytes) + xchk_da_set_corrupt(ds, level); + +out: + return 0; +} + +/* Scrub a attribute btree record. */ +STATIC int +xchk_xattr_rec( + struct xchk_da_btree *ds, + int level) +{ + struct xfs_mount *mp = ds->state->mp; + struct xfs_da_state_blk *blk = &ds->state->path.blk[level]; + struct xfs_attr_leaf_name_local *lentry; + struct xfs_attr_leaf_name_remote *rentry; + struct xfs_buf *bp; + struct xfs_attr_leaf_entry *ent; + xfs_dahash_t calc_hash; + xfs_dahash_t hash; + int nameidx; + int hdrsize; + int error; + + ASSERT(blk->magic == XFS_ATTR_LEAF_MAGIC); + + ent = xfs_attr3_leaf_entryp(blk->bp->b_addr) + blk->index; + + /* Check the whole block, if necessary. */ + error = xchk_xattr_block(ds, level); + if (error) + goto out; + if (ds->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out; + + /* Check the hash of the entry. */ + error = xchk_da_btree_hash(ds, level, &ent->hashval); + if (error) + goto out; + + /* Find the attr entry's location. */ + bp = blk->bp; + hdrsize = xfs_attr3_leaf_hdr_size(bp->b_addr); + nameidx = be16_to_cpu(ent->nameidx); + if (nameidx < hdrsize || nameidx >= mp->m_attr_geo->blksize) { + xchk_da_set_corrupt(ds, level); + goto out; + } + + /* Retrieve the entry and check it. */ + hash = be32_to_cpu(ent->hashval); + if (ent->flags & ~XFS_ATTR_ONDISK_MASK) { + xchk_da_set_corrupt(ds, level); + return 0; + } + if (!xfs_attr_check_namespace(ent->flags)) { + xchk_da_set_corrupt(ds, level); + return 0; + } + + if (ent->flags & XFS_ATTR_LOCAL) { + lentry = (struct xfs_attr_leaf_name_local *) + (((char *)bp->b_addr) + nameidx); + if (lentry->namelen <= 0) { + xchk_da_set_corrupt(ds, level); + goto out; + } + calc_hash = xfs_attr_hashval(mp, ent->flags, lentry->nameval, + lentry->namelen, + lentry->nameval + lentry->namelen, + be16_to_cpu(lentry->valuelen)); + } else { + rentry = (struct xfs_attr_leaf_name_remote *) + (((char *)bp->b_addr) + nameidx); + if (rentry->namelen <= 0) { + xchk_da_set_corrupt(ds, level); + goto out; + } + if (ent->flags & XFS_ATTR_PARENT) { + xchk_da_set_corrupt(ds, level); + goto out; + } + calc_hash = xfs_attr_hashval(mp, ent->flags, rentry->name, + rentry->namelen, NULL, + be32_to_cpu(rentry->valuelen)); + } + if (calc_hash != hash) + xchk_da_set_corrupt(ds, level); + +out: + return error; +} + +/* Check space usage of shortform attrs. */ +STATIC int +xchk_xattr_check_sf( + struct xfs_scrub *sc) +{ + struct xchk_xattr_buf *ab = sc->buf; + struct xfs_ifork *ifp = &sc->ip->i_af; + struct xfs_attr_sf_hdr *sf = ifp->if_data; + struct xfs_attr_sf_entry *sfe = xfs_attr_sf_firstentry(sf); + struct xfs_attr_sf_entry *next; + unsigned char *end = ifp->if_data + ifp->if_bytes; + int i; + int error = 0; + + bitmap_zero(ab->usedmap, ifp->if_bytes); + xchk_xattr_set_map(sc, ab->usedmap, 0, sizeof(*sf)); + + if ((unsigned char *)sfe > end) { + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, 0); + return 0; + } + + for (i = 0; i < sf->count; i++) { + unsigned char *name = sfe->nameval; + unsigned char *value = &sfe->nameval[sfe->namelen]; + + if (xchk_should_terminate(sc, &error)) + return error; + + next = xfs_attr_sf_nextentry(sfe); + if ((unsigned char *)next > end) { + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, 0); + break; + } + + /* + * Shortform entries do not set LOCAL or INCOMPLETE, so the + * only valid flag bits here are for namespaces. + */ + if (sfe->flags & ~XFS_ATTR_NSP_ONDISK_MASK) { + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, 0); + break; + } + + if (!xchk_xattr_set_map(sc, ab->usedmap, + (char *)sfe - (char *)sf, + sizeof(struct xfs_attr_sf_entry))) { + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, 0); + break; + } + + if (!xchk_xattr_set_map(sc, ab->usedmap, + (char *)name - (char *)sf, + sfe->namelen)) { + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, 0); + break; + } + + if (!xchk_xattr_set_map(sc, ab->usedmap, + (char *)value - (char *)sf, + sfe->valuelen)) { + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, 0); + break; + } + + sfe = next; + } + + return 0; +} + +/* Scrub the extended attribute metadata. */ +int +xchk_xattr( + struct xfs_scrub *sc) +{ + xfs_dablk_t last_checked = -1U; + int error = 0; + + if (!xfs_inode_hasattr(sc->ip)) + return -ENOENT; + + /* Allocate memory for xattr checking. */ + error = xchk_setup_xattr_buf(sc, 0); + if (error == -ENOMEM) + return -EDEADLOCK; + if (error) + return error; + + /* Check the physical structure of the xattr. */ + if (sc->ip->i_af.if_format == XFS_DINODE_FMT_LOCAL) + error = xchk_xattr_check_sf(sc); + else + error = xchk_da_btree(sc, XFS_ATTR_FORK, xchk_xattr_rec, + &last_checked); + if (error) + return error; + + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return 0; + + /* + * Look up every xattr in this file by name and hash. + * + * The VFS only locks i_rwsem when modifying attrs, so keep all + * three locks held because that's the only way to ensure we're + * the only thread poking into the da btree. We traverse the da + * btree while holding a leaf buffer locked for the xattr name + * iteration, which doesn't really follow the usual buffer + * locking order. + */ + error = xchk_xattr_walk(sc, sc->ip, xchk_xattr_actor, NULL, NULL); + if (!xchk_fblock_process_error(sc, XFS_ATTR_FORK, 0, &error)) + return error; + + return 0; +} diff --git a/fs/xfs/scrub/attr.h b/fs/xfs/scrub/attr.h new file mode 100644 index 000000000..7db58af56 --- /dev/null +++ b/fs/xfs/scrub/attr.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2019-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_ATTR_H__ +#define __XFS_SCRUB_ATTR_H__ + +/* + * Temporary storage for online scrub and repair of extended attributes. + */ +struct xchk_xattr_buf { + /* Bitmap of used space in xattr leaf blocks and shortform forks. */ + unsigned long *usedmap; + + /* Bitmap of free space in xattr leaf blocks. */ + unsigned long *freemap; + + /* Memory buffer used to hold salvaged xattr names. */ + unsigned char *name; + + /* Memory buffer used to extract xattr values. */ + void *value; + size_t value_sz; +}; + +bool xchk_xattr_set_map(struct xfs_scrub *sc, unsigned long *map, + unsigned int start, unsigned int len); +int xchk_setup_xattr_buf(struct xfs_scrub *sc, size_t value_size); + +#endif /* __XFS_SCRUB_ATTR_H__ */ diff --git a/fs/xfs/scrub/attr_repair.c b/fs/xfs/scrub/attr_repair.c new file mode 100644 index 000000000..6e6af142f --- /dev/null +++ b/fs/xfs/scrub/attr_repair.c @@ -0,0 +1,1656 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2018-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_btree.h" +#include "xfs_bit.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_sb.h" +#include "xfs_inode.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_dir2.h" +#include "xfs_attr.h" +#include "xfs_attr_leaf.h" +#include "xfs_attr_sf.h" +#include "xfs_attr_remote.h" +#include "xfs_bmap.h" +#include "xfs_bmap_util.h" +#include "xfs_exchmaps.h" +#include "xfs_exchrange.h" +#include "xfs_acl.h" +#include "xfs_parent.h" +#include "scrub/xfs_scrub.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/tempfile.h" +#include "scrub/tempexch.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/xfblob.h" +#include "scrub/attr.h" +#include "scrub/reap.h" +#include "scrub/attr_repair.h" + +/* + * Extended Attribute Repair + * ========================= + * + * We repair extended attributes by reading the attr leaf blocks looking for + * attributes entries that look salvageable (name passes verifiers, value can + * be retrieved, etc). Each extended attribute worth salvaging is stashed in + * memory, and the stashed entries are periodically replayed into a temporary + * file to constrain memory use. Batching the construction of the temporary + * extended attribute structure in this fashion reduces lock cycling of the + * file being repaired and the temporary file. + * + * When salvaging completes, the remaining stashed attributes are replayed to + * the temporary file. An atomic file contents exchange is used to commit the + * new xattr blocks to the file being repaired. This will disrupt attrmulti + * cursors. + */ + +struct xrep_xattr_key { + /* Cookie for retrieval of the xattr name. */ + xfblob_cookie name_cookie; + + /* Cookie for retrieval of the xattr value. */ + xfblob_cookie value_cookie; + + /* XFS_ATTR_* flags */ + int flags; + + /* Length of the value and name. */ + uint32_t valuelen; + uint16_t namelen; +}; + +/* + * Stash up to 8 pages of attrs in xattr_records/xattr_blobs before we write + * them to the temp file. + */ +#define XREP_XATTR_MAX_STASH_BYTES (PAGE_SIZE * 8) + +struct xrep_xattr { + struct xfs_scrub *sc; + + /* Information for exchanging attr fork mappings at the end. */ + struct xrep_tempexch tx; + + /* xattr keys */ + struct xfarray *xattr_records; + + /* xattr values */ + struct xfblob *xattr_blobs; + + /* Number of attributes that we are salvaging. */ + unsigned long long attrs_found; + + /* Can we flush stashed attrs to the tempfile? */ + bool can_flush; + + /* Did the live update fail, and hence the repair is now out of date? */ + bool live_update_aborted; + + /* Lock protecting parent pointer updates */ + struct mutex lock; + + /* Fixed-size array of xrep_xattr_pptr structures. */ + struct xfarray *pptr_recs; + + /* Blobs containing parent pointer names. */ + struct xfblob *pptr_names; + + /* Hook to capture parent pointer updates. */ + struct xfs_dir_hook dhook; + + /* Scratch buffer for capturing parent pointers. */ + struct xfs_da_args pptr_args; + + /* Name buffer */ + struct xfs_name xname; + char namebuf[MAXNAMELEN]; +}; + +/* Create a parent pointer in the tempfile. */ +#define XREP_XATTR_PPTR_ADD (1) + +/* Remove a parent pointer from the tempfile. */ +#define XREP_XATTR_PPTR_REMOVE (2) + +/* A stashed parent pointer update. */ +struct xrep_xattr_pptr { + /* Cookie for retrieval of the pptr name. */ + xfblob_cookie name_cookie; + + /* Parent pointer record. */ + struct xfs_parent_rec pptr_rec; + + /* Length of the pptr name. */ + uint8_t namelen; + + /* XREP_XATTR_PPTR_{ADD,REMOVE} */ + uint8_t action; +}; + +/* Set up to recreate the extended attributes. */ +int +xrep_setup_xattr( + struct xfs_scrub *sc) +{ + if (xfs_has_parent(sc->mp)) + xchk_fsgates_enable(sc, XCHK_FSGATES_DIRENTS); + + return xrep_tempfile_create(sc, S_IFREG); +} + +/* + * Decide if we want to salvage this attribute. We don't bother with + * incomplete or oversized keys or values. The @value parameter can be null + * for remote attrs. + */ +STATIC int +xrep_xattr_want_salvage( + struct xrep_xattr *rx, + unsigned int attr_flags, + const void *name, + int namelen, + const void *value, + int valuelen) +{ + if (attr_flags & XFS_ATTR_INCOMPLETE) + return false; + if (namelen > XATTR_NAME_MAX || namelen <= 0) + return false; + if (!xfs_attr_namecheck(attr_flags, name, namelen)) + return false; + if (valuelen > XATTR_SIZE_MAX || valuelen < 0) + return false; + if (attr_flags & XFS_ATTR_PARENT) + return xfs_parent_valuecheck(rx->sc->mp, value, valuelen); + + return true; +} + +/* Allocate an in-core record to hold xattrs while we rebuild the xattr data. */ +STATIC int +xrep_xattr_salvage_key( + struct xrep_xattr *rx, + int flags, + unsigned char *name, + int namelen, + unsigned char *value, + int valuelen) +{ + struct xrep_xattr_key key = { + .valuelen = valuelen, + .flags = flags & XFS_ATTR_NSP_ONDISK_MASK, + }; + unsigned int i = 0; + int error = 0; + + if (xchk_should_terminate(rx->sc, &error)) + return error; + + /* + * Truncate the name to the first character that would trip namecheck. + * If we no longer have a name after that, ignore this attribute. + */ + if (flags & XFS_ATTR_PARENT) { + key.namelen = namelen; + + trace_xrep_xattr_salvage_pptr(rx->sc->ip, flags, name, + key.namelen, value, valuelen); + } else { + while (i < namelen && name[i] != 0) + i++; + if (i == 0) + return 0; + key.namelen = i; + + trace_xrep_xattr_salvage_rec(rx->sc->ip, flags, name, + key.namelen, valuelen); + } + + error = xfblob_store(rx->xattr_blobs, &key.name_cookie, name, + key.namelen); + if (error) + return error; + + error = xfblob_store(rx->xattr_blobs, &key.value_cookie, value, + key.valuelen); + if (error) + return error; + + error = xfarray_append(rx->xattr_records, &key); + if (error) + return error; + + rx->attrs_found++; + return 0; +} + +/* + * Record a shortform extended attribute key & value for later reinsertion + * into the inode. + */ +STATIC int +xrep_xattr_salvage_sf_attr( + struct xrep_xattr *rx, + struct xfs_attr_sf_hdr *hdr, + struct xfs_attr_sf_entry *sfe) +{ + struct xfs_scrub *sc = rx->sc; + struct xchk_xattr_buf *ab = sc->buf; + unsigned char *name = sfe->nameval; + unsigned char *value = &sfe->nameval[sfe->namelen]; + + if (!xchk_xattr_set_map(sc, ab->usedmap, (char *)name - (char *)hdr, + sfe->namelen)) + return 0; + + if (!xchk_xattr_set_map(sc, ab->usedmap, (char *)value - (char *)hdr, + sfe->valuelen)) + return 0; + + if (!xrep_xattr_want_salvage(rx, sfe->flags, sfe->nameval, + sfe->namelen, value, sfe->valuelen)) + return 0; + + return xrep_xattr_salvage_key(rx, sfe->flags, sfe->nameval, + sfe->namelen, value, sfe->valuelen); +} + +/* + * Record a local format extended attribute key & value for later reinsertion + * into the inode. + */ +STATIC int +xrep_xattr_salvage_local_attr( + struct xrep_xattr *rx, + struct xfs_attr_leaf_entry *ent, + unsigned int nameidx, + const char *buf_end, + struct xfs_attr_leaf_name_local *lentry) +{ + struct xchk_xattr_buf *ab = rx->sc->buf; + unsigned char *value; + unsigned int valuelen; + unsigned int namesize; + + /* + * Decode the leaf local entry format. If something seems wrong, we + * junk the attribute. + */ + value = &lentry->nameval[lentry->namelen]; + valuelen = be16_to_cpu(lentry->valuelen); + namesize = xfs_attr_leaf_entsize_local(lentry->namelen, valuelen); + if ((char *)lentry + namesize > buf_end) + return 0; + if (!xrep_xattr_want_salvage(rx, ent->flags, lentry->nameval, + lentry->namelen, value, valuelen)) + return 0; + if (!xchk_xattr_set_map(rx->sc, ab->usedmap, nameidx, namesize)) + return 0; + + /* Try to save this attribute. */ + return xrep_xattr_salvage_key(rx, ent->flags, lentry->nameval, + lentry->namelen, value, valuelen); +} + +/* + * Record a remote format extended attribute key & value for later reinsertion + * into the inode. + */ +STATIC int +xrep_xattr_salvage_remote_attr( + struct xrep_xattr *rx, + struct xfs_attr_leaf_entry *ent, + unsigned int nameidx, + const char *buf_end, + struct xfs_attr_leaf_name_remote *rentry, + unsigned int ent_idx, + struct xfs_buf *leaf_bp) +{ + struct xchk_xattr_buf *ab = rx->sc->buf; + struct xfs_da_args args = { + .trans = rx->sc->tp, + .dp = rx->sc->ip, + .index = ent_idx, + .geo = rx->sc->mp->m_attr_geo, + .owner = I_INO(rx->sc->ip), + .attr_filter = ent->flags & XFS_ATTR_NSP_ONDISK_MASK, + .namelen = rentry->namelen, + .name = rentry->name, + .valuelen = be32_to_cpu(rentry->valuelen), + }; + unsigned int namesize; + int error; + + /* + * Decode the leaf remote entry format. If something seems wrong, we + * junk the attribute. Note that we should never find a zero-length + * remote attribute value. + */ + namesize = xfs_attr_leaf_entsize_remote(rentry->namelen); + if ((char *)rentry + namesize > buf_end) + return 0; + if (args.valuelen == 0 || + !xrep_xattr_want_salvage(rx, ent->flags, rentry->name, + rentry->namelen, NULL, args.valuelen)) + return 0; + if (!xchk_xattr_set_map(rx->sc, ab->usedmap, nameidx, namesize)) + return 0; + + /* + * Enlarge the buffer (if needed) to hold the value that we're trying + * to salvage from the old extended attribute data. + */ + error = xchk_setup_xattr_buf(rx->sc, args.valuelen); + if (error == -ENOMEM) + error = -EDEADLOCK; + if (error) + return error; + args.value = ab->value; + + /* Look up the remote value and stash it for reconstruction. */ + error = xfs_attr3_leaf_getvalue(leaf_bp, &args); + if (error || args.rmtblkno == 0) + goto err_free; + + error = xfs_attr_rmtval_get(&args); + if (error) + goto err_free; + + /* Try to save this attribute. */ + error = xrep_xattr_salvage_key(rx, ent->flags, rentry->name, + rentry->namelen, ab->value, args.valuelen); +err_free: + /* remote value was garbage, junk it */ + if (error == -EFSBADCRC || error == -EFSCORRUPTED) + error = 0; + return error; +} + +/* Extract every xattr key that we can from this attr fork block. */ +STATIC int +xrep_xattr_recover_leaf( + struct xrep_xattr *rx, + struct xfs_buf *bp) +{ + struct xfs_attr3_icleaf_hdr leafhdr; + struct xfs_scrub *sc = rx->sc; + struct xfs_mount *mp = sc->mp; + struct xfs_attr_leafblock *leaf; + struct xfs_attr_leaf_name_local *lentry; + struct xfs_attr_leaf_name_remote *rentry; + struct xfs_attr_leaf_entry *ent; + struct xfs_attr_leaf_entry *entries; + struct xchk_xattr_buf *ab = rx->sc->buf; + char *buf_end; + size_t off; + unsigned int nameidx; + unsigned int hdrsize; + int i; + int error = 0; + + bitmap_zero(ab->usedmap, mp->m_attr_geo->blksize); + + /* Check the leaf header */ + leaf = bp->b_addr; + xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, &leafhdr, leaf); + hdrsize = xfs_attr3_leaf_hdr_size(leaf); + xchk_xattr_set_map(sc, ab->usedmap, 0, hdrsize); + entries = xfs_attr3_leaf_entryp(leaf); + + buf_end = (char *)bp->b_addr + mp->m_attr_geo->blksize; + for (i = 0, ent = entries; i < leafhdr.count; ent++, i++) { + if (xchk_should_terminate(sc, &error)) + return error; + + /* Skip key if it conflicts with something else? */ + off = (char *)ent - (char *)leaf; + if (!xchk_xattr_set_map(sc, ab->usedmap, off, + sizeof(xfs_attr_leaf_entry_t))) + continue; + + /* Check the name information. */ + nameidx = be16_to_cpu(ent->nameidx); + if (nameidx < leafhdr.firstused || + nameidx >= mp->m_attr_geo->blksize) + continue; + + if (ent->flags & XFS_ATTR_LOCAL) { + lentry = xfs_attr3_leaf_name_local(leaf, i); + error = xrep_xattr_salvage_local_attr(rx, ent, nameidx, + buf_end, lentry); + } else { + rentry = xfs_attr3_leaf_name_remote(leaf, i); + error = xrep_xattr_salvage_remote_attr(rx, ent, nameidx, + buf_end, rentry, i, bp); + } + if (error) + return error; + } + + return 0; +} + +/* Try to recover shortform attrs. */ +STATIC int +xrep_xattr_recover_sf( + struct xrep_xattr *rx) +{ + struct xfs_scrub *sc = rx->sc; + struct xchk_xattr_buf *ab = sc->buf; + struct xfs_attr_sf_hdr *hdr; + struct xfs_attr_sf_entry *sfe; + struct xfs_attr_sf_entry *next; + struct xfs_ifork *ifp; + unsigned char *end; + int i; + int error = 0; + + ifp = xfs_ifork_ptr(rx->sc->ip, XFS_ATTR_FORK); + hdr = ifp->if_data; + + bitmap_zero(ab->usedmap, ifp->if_bytes); + end = (unsigned char *)ifp->if_data + ifp->if_bytes; + xchk_xattr_set_map(sc, ab->usedmap, 0, sizeof(*hdr)); + + sfe = xfs_attr_sf_firstentry(hdr); + if ((unsigned char *)sfe > end) + return 0; + + for (i = 0; i < hdr->count; i++) { + if (xchk_should_terminate(sc, &error)) + return error; + + next = xfs_attr_sf_nextentry(sfe); + if ((unsigned char *)next > end) + break; + + if (xchk_xattr_set_map(sc, ab->usedmap, + (char *)sfe - (char *)hdr, + sizeof(struct xfs_attr_sf_entry))) { + /* + * No conflicts with the sf entry; let's save this + * attribute. + */ + error = xrep_xattr_salvage_sf_attr(rx, hdr, sfe); + if (error) + return error; + } + + sfe = next; + } + + return 0; +} + +/* + * Try to return a buffer of xattr data for a given physical extent. + * + * Because the buffer cache get function complains if it finds a buffer + * matching the block number but not matching the length, we must be careful to + * look for incore buffers (up to the maximum length of a remote value) that + * could be hiding anywhere in the physical range. If we find an incore + * buffer, we can pass that to the caller. Optionally, read a single block and + * pass that back. + * + * Note the subtlety that remote attr value blocks for which there is no incore + * buffer will be passed to the callback one block at a time. These buffers + * will not have any ops attached and must be staled to prevent aliasing with + * multiblock buffers once we drop the ILOCK. + */ +STATIC int +xrep_xattr_find_buf( + struct xfs_mount *mp, + xfs_fsblock_t fsbno, + xfs_extlen_t max_len, + bool can_read, + struct xfs_buf **bpp) +{ + struct xrep_bufscan scan = { + .daddr = XFS_FSB_TO_DADDR(mp, fsbno), + .max_sectors = xrep_bufscan_max_sectors(mp, max_len), + .daddr_step = XFS_FSB_TO_BB(mp, 1), + }; + struct xfs_buf *bp; + + while ((bp = xrep_bufscan_advance(mp, &scan)) != NULL) { + *bpp = bp; + return 0; + } + + if (!can_read) { + *bpp = NULL; + return 0; + } + + return xfs_buf_read(mp->m_ddev_targp, scan.daddr, XFS_FSB_TO_BB(mp, 1), + XBF_TRYLOCK, bpp, NULL); +} + +/* + * Deal with a buffer that we found during our walk of the attr fork. + * + * Attribute leaf and node blocks are simple -- they're a single block, so we + * can walk them one at a time and we never have to worry about discontiguous + * multiblock buffers like we do for directories. + * + * Unfortunately, remote attr blocks add a lot of complexity here. Each disk + * block is totally self contained, in the sense that the v5 header provides no + * indication that there could be more data in the next block. The incore + * buffers can span multiple blocks, though they never cross extent records. + * However, they don't necessarily start or end on an extent record boundary. + * Therefore, we need a special buffer find function to walk the buffer cache + * for us. + * + * The caller must hold the ILOCK on the file being repaired. We use + * XBF_TRYLOCK here to skip any locked buffer on the assumption that we don't + * own the block and don't want to hang the system on a potentially garbage + * buffer. + */ +STATIC int +xrep_xattr_recover_block( + struct xrep_xattr *rx, + xfs_dablk_t dabno, + xfs_fsblock_t fsbno, + xfs_extlen_t max_len, + xfs_extlen_t *actual_len) +{ + struct xfs_da_blkinfo *info; + struct xfs_buf *bp; + int error; + + error = xrep_xattr_find_buf(rx->sc->mp, fsbno, max_len, true, &bp); + if (error) + return error; + info = bp->b_addr; + *actual_len = XFS_BB_TO_FSB(rx->sc->mp, bp->b_length); + + trace_xrep_xattr_recover_leafblock(rx->sc->ip, dabno, + be16_to_cpu(info->magic)); + + /* + * If the buffer has the right magic number for an attr leaf block and + * passes a structure check (we don't care about checksums), salvage + * as much as we can from the block. */ + if (info->magic == cpu_to_be16(XFS_ATTR3_LEAF_MAGIC) && + xrep_buf_verify_struct(bp, &xfs_attr3_leaf_buf_ops) && + xfs_attr3_leaf_header_check(bp, I_INO(rx->sc->ip)) == NULL) + error = xrep_xattr_recover_leaf(rx, bp); + + /* + * If the buffer didn't already have buffer ops set, it was read in by + * the _find_buf function and could very well be /part/ of a multiblock + * remote block. Mark it stale so that it doesn't hang around in + * memory to cause problems. + */ + if (bp->b_ops == NULL) + xfs_buf_stale(bp); + + xfs_buf_relse(bp); + return error; +} + +/* Insert one xattr key/value. */ +STATIC int +xrep_xattr_insert_rec( + struct xrep_xattr *rx, + const struct xrep_xattr_key *key) +{ + struct xfs_da_args args = { + .dp = rx->sc->tempip, + .attr_filter = key->flags, + .namelen = key->namelen, + .valuelen = key->valuelen, + .owner = I_INO(rx->sc->ip), + .geo = rx->sc->mp->m_attr_geo, + .whichfork = XFS_ATTR_FORK, + .op_flags = XFS_DA_OP_OKNOENT, + }; + struct xchk_xattr_buf *ab = rx->sc->buf; + int error; + + /* + * Grab pointers to the scrub buffer so that we can use them to insert + * attrs into the temp file. + */ + args.name = ab->name; + args.value = ab->value; + + /* + * The attribute name is stored near the end of the in-core buffer, + * though we reserve one more byte to ensure null termination. + */ + ab->name[XATTR_NAME_MAX] = 0; + + error = xfblob_load(rx->xattr_blobs, key->name_cookie, ab->name, + key->namelen); + if (error) + return error; + + error = xfblob_free(rx->xattr_blobs, key->name_cookie); + if (error) + return error; + + error = xfblob_load(rx->xattr_blobs, key->value_cookie, args.value, + key->valuelen); + if (error) + return error; + + error = xfblob_free(rx->xattr_blobs, key->value_cookie); + if (error) + return error; + + ab->name[key->namelen] = 0; + + if (key->flags & XFS_ATTR_PARENT) { + trace_xrep_xattr_insert_pptr(rx->sc->tempip, key->flags, + ab->name, key->namelen, ab->value, + key->valuelen); + args.op_flags |= XFS_DA_OP_LOGGED; + } else { + trace_xrep_xattr_insert_rec(rx->sc->tempip, key->flags, + ab->name, key->namelen, key->valuelen); + } + + /* + * xfs_attr_set creates and commits its own transaction. If the attr + * already exists, we'll just drop it during the rebuild. + */ + xfs_attr_sethash(&args); + error = xfs_attr_set(&args, XFS_ATTRUPDATE_CREATE, false); + if (error == -EEXIST) + error = 0; + + return error; +} + +/* + * Periodically flush salvaged attributes to the temporary file. This is done + * to reduce the memory requirements of the xattr rebuild because files can + * contain millions of attributes. + */ +STATIC int +xrep_xattr_flush_stashed( + struct xrep_xattr *rx) +{ + xfarray_idx_t array_cur; + int error; + + /* + * Entering this function, the scrub context has a reference to the + * inode being repaired, the temporary file, and a scrub transaction + * that we use during xattr salvaging to avoid livelocking if there + * are cycles in the xattr structures. We hold ILOCK_EXCL on both + * the inode being repaired, though it is not ijoined to the scrub + * transaction. + * + * To constrain kernel memory use, we occasionally flush salvaged + * xattrs from the xfarray and xfblob structures into the temporary + * file in preparation for exchanging the xattr structures at the end. + * Updating the temporary file requires a transaction, so we commit the + * scrub transaction and drop the two ILOCKs so that xfs_attr_set can + * allocate whatever transaction it wants. + * + * We still hold IOLOCK_EXCL on the inode being repaired, which + * prevents anyone from modifying the damaged xattr data while we + * repair it. + */ + error = xrep_trans_commit(rx->sc); + if (error) + return error; + xchk_iunlock(rx->sc, XFS_ILOCK_EXCL); + + /* + * Take the IOLOCK of the temporary file while we modify xattrs. This + * isn't strictly required because the temporary file is never revealed + * to userspace, but we follow the same locking rules. We still hold + * sc->ip's IOLOCK. + */ + error = xrep_tempfile_iolock_polled(rx->sc); + if (error) + return error; + + /* Add all the salvaged attrs to the temporary file. */ + foreach_xfarray_idx(rx->xattr_records, array_cur) { + struct xrep_xattr_key key; + + error = xfarray_load(rx->xattr_records, array_cur, &key); + if (error) + return error; + + error = xrep_xattr_insert_rec(rx, &key); + if (error) + return error; + } + + /* Empty out both arrays now that we've added the entries. */ + xfarray_truncate(rx->xattr_records); + xfblob_truncate(rx->xattr_blobs); + + xrep_tempfile_iounlock(rx->sc); + + /* Recreate the salvage transaction and relock the inode. */ + error = xchk_trans_alloc(rx->sc, 0); + if (error) + return error; + xchk_ilock(rx->sc, XFS_ILOCK_EXCL); + return 0; +} + +/* Decide if we've stashed too much xattr data in memory. */ +static inline bool +xrep_xattr_want_flush_stashed( + struct xrep_xattr *rx) +{ + unsigned long long bytes; + + if (!rx->can_flush) + return false; + + bytes = xfarray_bytes(rx->xattr_records) + + xfblob_bytes(rx->xattr_blobs); + return bytes > XREP_XATTR_MAX_STASH_BYTES; +} + +/* + * Did we observe rename changing parent pointer xattrs while we were flushing + * salvaged attrs? + */ +static inline bool +xrep_xattr_saw_pptr_conflict( + struct xrep_xattr *rx) +{ + bool ret; + + ASSERT(rx->can_flush); + + if (!xfs_has_parent(rx->sc->mp)) + return false; + + xfs_assert_ilocked(rx->sc->ip, XFS_ILOCK_EXCL); + + mutex_lock(&rx->lock); + ret = xfarray_bytes(rx->pptr_recs) > 0; + mutex_unlock(&rx->lock); + + return ret; +} + +/* + * Reset the entire repair state back to initial conditions, now that we've + * detected a parent pointer update to the attr structure while we were + * flushing salvaged attrs. See the locking notes in dir_repair.c for more + * information on why this is all necessary. + */ +STATIC int +xrep_xattr_full_reset( + struct xrep_xattr *rx) +{ + struct xfs_scrub *sc = rx->sc; + struct xfs_attr_sf_hdr *hdr; + struct xfs_ifork *ifp = &sc->tempip->i_af; + int error; + + trace_xrep_xattr_full_reset(sc->ip, sc->tempip); + + /* The temporary file's data fork had better not be in btree format. */ + if (sc->tempip->i_df.if_format == XFS_DINODE_FMT_BTREE) { + ASSERT(0); + return -EIO; + } + + /* + * We begin in transaction context with sc->ip ILOCKed but not joined + * to the transaction. To reset to the initial state, we must hold + * sc->ip's ILOCK to prevent rename from updating parent pointer + * information and the tempfile's ILOCK to clear its contents. + */ + xchk_iunlock(rx->sc, XFS_ILOCK_EXCL); + xrep_tempfile_ilock_both(sc); + xfs_trans_ijoin(sc->tp, sc->ip, 0); + xfs_trans_ijoin(sc->tp, sc->tempip, 0); + + /* + * Free all the blocks of the attr fork of the temp file, and reset + * it back to local format. + */ + if (xfs_ifork_has_extents(&sc->tempip->i_af)) { + error = xrep_reap_ifork(sc, sc->tempip, XFS_ATTR_FORK); + if (error) + return error; + + ASSERT(ifp->if_bytes == 0); + ifp->if_format = XFS_DINODE_FMT_LOCAL; + xfs_idata_realloc(sc->tempip, sizeof(*hdr), XFS_ATTR_FORK); + } + + /* Reinitialize the attr fork to an empty shortform structure. */ + hdr = ifp->if_data; + memset(hdr, 0, sizeof(*hdr)); + hdr->totsize = cpu_to_be16(sizeof(*hdr)); + xfs_trans_log_inode(sc->tp, sc->tempip, XFS_ILOG_CORE | XFS_ILOG_ADATA); + + /* + * Roll this transaction to commit our reset ondisk. The tempfile + * should no longer be joined to the transaction, so we drop its ILOCK. + * This should leave us in transaction context with sc->ip ILOCKed but + * not joined to the transaction. + */ + error = xrep_roll_trans(sc); + if (error) + return error; + xrep_tempfile_iunlock(sc); + + /* + * Erase any accumulated parent pointer updates now that we've erased + * the tempfile's attr fork. We're resetting the entire repair state + * back to where we were initially, except now we won't flush salvaged + * xattrs until the very end. + */ + mutex_lock(&rx->lock); + xfarray_truncate(rx->pptr_recs); + xfblob_truncate(rx->pptr_names); + mutex_unlock(&rx->lock); + + rx->can_flush = false; + rx->attrs_found = 0; + + ASSERT(xfarray_bytes(rx->xattr_records) == 0); + ASSERT(xfblob_bytes(rx->xattr_blobs) == 0); + return 0; +} + +/* Extract as many attribute keys and values as we can. */ +STATIC int +xrep_xattr_recover( + struct xrep_xattr *rx) +{ + struct xfs_bmbt_irec got; + struct xfs_scrub *sc = rx->sc; + struct xfs_da_geometry *geo = sc->mp->m_attr_geo; + xfs_fileoff_t offset; + xfs_extlen_t len; + xfs_dablk_t dabno; + int nmap; + int error; + +restart: + /* + * Iterate each xattr leaf block in the attr fork to scan them for any + * attributes that we might salvage. + */ + for (offset = 0; + offset < XFS_MAX_FILEOFF; + offset = got.br_startoff + got.br_blockcount) { + nmap = 1; + error = xfs_bmapi_read(sc->ip, offset, XFS_MAX_FILEOFF - offset, + &got, &nmap, XFS_BMAPI_ATTRFORK); + if (error) + return error; + if (nmap != 1) + return -EFSCORRUPTED; + if (!xfs_bmap_is_written_extent(&got)) + continue; + + for (dabno = round_up(got.br_startoff, geo->fsbcount); + dabno < got.br_startoff + got.br_blockcount; + dabno += len) { + xfs_fileoff_t curr_offset = dabno - got.br_startoff; + xfs_extlen_t maxlen; + + if (xchk_should_terminate(rx->sc, &error)) + return error; + + maxlen = min_t(xfs_filblks_t, INT_MAX, + got.br_blockcount - curr_offset); + error = xrep_xattr_recover_block(rx, dabno, + curr_offset + got.br_startblock, + maxlen, &len); + if (error) + return error; + + if (xrep_xattr_want_flush_stashed(rx)) { + error = xrep_xattr_flush_stashed(rx); + if (error) + return error; + + if (xrep_xattr_saw_pptr_conflict(rx)) { + error = xrep_xattr_full_reset(rx); + if (error) + return error; + + goto restart; + } + } + } + } + + return 0; +} + +/* + * Reset the extended attribute fork to a state where we can start re-adding + * the salvaged attributes. + */ +STATIC int +xrep_xattr_fork_remove( + struct xfs_scrub *sc, + struct xfs_inode *ip) +{ + struct xfs_attr_sf_hdr *hdr; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_ATTR_FORK); + + /* + * If the data fork is in btree format, we can't change di_forkoff + * because we could run afoul of the rule that the data fork isn't + * supposed to be in btree format if there's enough space in the fork + * that it could have used extents format. Instead, reinitialize the + * attr fork to have a shortform structure with zero attributes. + */ + if (ip->i_df.if_format == XFS_DINODE_FMT_BTREE) { + ifp->if_format = XFS_DINODE_FMT_LOCAL; + hdr = xfs_idata_realloc(ip, (int)sizeof(*hdr) - ifp->if_bytes, + XFS_ATTR_FORK); + hdr->count = 0; + hdr->totsize = cpu_to_be16(sizeof(*hdr)); + xfs_trans_log_inode(sc->tp, ip, + XFS_ILOG_CORE | XFS_ILOG_ADATA); + return 0; + } + + /* If we still have attr fork extents, something's wrong. */ + if (ifp->if_nextents != 0) { + struct xfs_iext_cursor icur; + struct xfs_bmbt_irec irec; + unsigned int i = 0; + + xfs_emerg(sc->mp, + "inode 0x%llx attr fork still has %llu attr extents, format %d?!", + I_INO(ip), ifp->if_nextents, ifp->if_format); + for_each_xfs_iext(ifp, &icur, &irec) { + xfs_err(sc->mp, + "[%u]: startoff %llu startblock %llu blockcount %llu state %u", + i++, irec.br_startoff, + irec.br_startblock, irec.br_blockcount, + irec.br_state); + } + ASSERT(0); + return -EFSCORRUPTED; + } + + xfs_attr_fork_remove(ip, sc->tp); + return 0; +} + +/* + * Free all the attribute fork blocks of the file being repaired and delete the + * fork. The caller must ILOCK the scrub file and join it to the transaction. + * This function returns with the inode joined to a clean transaction. + */ +int +xrep_xattr_reset_fork( + struct xfs_scrub *sc) +{ + int error; + + trace_xrep_xattr_reset_fork(sc->ip, sc->ip); + + /* Unmap all the attr blocks. */ + if (xfs_ifork_has_extents(&sc->ip->i_af)) { + error = xrep_reap_ifork(sc, sc->ip, XFS_ATTR_FORK); + if (error) + return error; + } + + error = xrep_xattr_fork_remove(sc, sc->ip); + if (error) + return error; + + return xfs_trans_roll_inode(&sc->tp, sc->ip); +} + +/* + * Free all the attribute fork blocks of the temporary file and delete the attr + * fork. The caller must ILOCK the tempfile and join it to the transaction. + * This function returns with the inode joined to a clean scrub transaction. + */ +int +xrep_xattr_reset_tempfile_fork( + struct xfs_scrub *sc) +{ + int error; + + trace_xrep_xattr_reset_fork(sc->ip, sc->tempip); + + /* + * Wipe out the attr fork of the temp file so that regular inode + * inactivation won't trip over the corrupt attr fork. + */ + if (xfs_ifork_has_extents(&sc->tempip->i_af)) { + error = xrep_reap_ifork(sc, sc->tempip, XFS_ATTR_FORK); + if (error) + return error; + } + + return xrep_xattr_fork_remove(sc, sc->tempip); +} + +/* + * Find all the extended attributes for this inode by scraping them out of the + * attribute key blocks by hand, and flushing them into the temp file. + * When we're done, free the staging memory before exchanging the xattr + * structures to reduce memory usage. + */ +STATIC int +xrep_xattr_salvage_attributes( + struct xrep_xattr *rx) +{ + struct xfs_inode *ip = rx->sc->ip; + int error; + + /* Short format xattrs are easy! */ + if (rx->sc->ip->i_af.if_format == XFS_DINODE_FMT_LOCAL) { + error = xrep_xattr_recover_sf(rx); + if (error) + return error; + + return xrep_xattr_flush_stashed(rx); + } + + /* + * For non-inline xattr structures, the salvage function scans the + * buffer cache looking for potential attr leaf blocks. The scan + * requires the ability to lock any buffer found and runs independently + * of any transaction <-> buffer item <-> buffer linkage. Therefore, + * roll the transaction to ensure there are no buffers joined. We hold + * the ILOCK independently of the transaction. + */ + error = xfs_trans_roll(&rx->sc->tp); + if (error) + return error; + + error = xfs_iread_extents(rx->sc->tp, ip, XFS_ATTR_FORK); + if (error) + return error; + + error = xrep_xattr_recover(rx); + if (error) + return error; + + return xrep_xattr_flush_stashed(rx); +} + +/* + * Add this stashed incore parent pointer to the temporary file. The caller + * must hold the tempdir's IOLOCK, must not hold any ILOCKs, and must not be in + * transaction context. + */ +STATIC int +xrep_xattr_replay_pptr_update( + struct xrep_xattr *rx, + const struct xfs_name *xname, + struct xrep_xattr_pptr *pptr) +{ + struct xfs_scrub *sc = rx->sc; + int error; + + switch (pptr->action) { + case XREP_XATTR_PPTR_ADD: + /* Create parent pointer. */ + trace_xrep_xattr_replay_parentadd(sc->tempip, xname, + &pptr->pptr_rec); + + error = xfs_parent_set(sc->tempip, I_INO(sc->ip), xname, + &pptr->pptr_rec, &rx->pptr_args); + ASSERT(error != -EEXIST); + return error; + case XREP_XATTR_PPTR_REMOVE: + /* Remove parent pointer. */ + trace_xrep_xattr_replay_parentremove(sc->tempip, xname, + &pptr->pptr_rec); + + error = xfs_parent_unset(sc->tempip, I_INO(sc->ip), xname, + &pptr->pptr_rec, &rx->pptr_args); + ASSERT(error != -ENOATTR); + return error; + } + + ASSERT(0); + return -EIO; +} + +/* + * Flush stashed parent pointer updates that have been recorded by the scanner. + * This is done to reduce the memory requirements of the xattr rebuild, since + * files can have a lot of hardlinks and the fs can be busy. + * + * Caller must not hold transactions or ILOCKs. Caller must hold the tempfile + * IOLOCK. + */ +STATIC int +xrep_xattr_replay_pptr_updates( + struct xrep_xattr *rx) +{ + xfarray_idx_t array_cur; + int error; + + mutex_lock(&rx->lock); + foreach_xfarray_idx(rx->pptr_recs, array_cur) { + struct xrep_xattr_pptr pptr; + + error = xfarray_load(rx->pptr_recs, array_cur, &pptr); + if (error) + goto out_unlock; + + error = xfblob_loadname(rx->pptr_names, pptr.name_cookie, + &rx->xname, pptr.namelen); + if (error) + goto out_unlock; + mutex_unlock(&rx->lock); + + error = xrep_xattr_replay_pptr_update(rx, &rx->xname, &pptr); + if (error) + return error; + + mutex_lock(&rx->lock); + } + + /* Empty out both arrays now that we've added the entries. */ + xfarray_truncate(rx->pptr_recs); + xfblob_truncate(rx->pptr_names); + mutex_unlock(&rx->lock); + return 0; +out_unlock: + mutex_unlock(&rx->lock); + return error; +} + +/* + * Remember that we want to create a parent pointer in the tempfile. These + * stashed actions will be replayed later. + */ +STATIC int +xrep_xattr_stash_parentadd( + struct xrep_xattr *rx, + const struct xfs_name *name, + const struct xfs_inode *dp) +{ + struct xrep_xattr_pptr pptr = { + .action = XREP_XATTR_PPTR_ADD, + .namelen = name->len, + }; + int error; + + trace_xrep_xattr_stash_parentadd(rx->sc->tempip, dp, name); + + xfs_inode_to_parent_rec(&pptr.pptr_rec, dp); + error = xfblob_storename(rx->pptr_names, &pptr.name_cookie, name); + if (error) + return error; + + return xfarray_append(rx->pptr_recs, &pptr); +} + +/* + * Remember that we want to remove a parent pointer from the tempfile. These + * stashed actions will be replayed later. + */ +STATIC int +xrep_xattr_stash_parentremove( + struct xrep_xattr *rx, + const struct xfs_name *name, + const struct xfs_inode *dp) +{ + struct xrep_xattr_pptr pptr = { + .action = XREP_XATTR_PPTR_REMOVE, + .namelen = name->len, + }; + int error; + + trace_xrep_xattr_stash_parentremove(rx->sc->tempip, dp, name); + + xfs_inode_to_parent_rec(&pptr.pptr_rec, dp); + error = xfblob_storename(rx->pptr_names, &pptr.name_cookie, name); + if (error) + return error; + + return xfarray_append(rx->pptr_recs, &pptr); +} + +/* + * Capture dirent updates being made by other threads. We will have to replay + * the parent pointer updates before exchanging attr forks. + */ +STATIC int +xrep_xattr_live_dirent_update( + struct notifier_block *nb, + unsigned long action, + void *data) +{ + struct xfs_dir_update_params *p = data; + struct xrep_xattr *rx; + struct xfs_scrub *sc; + int error; + + rx = container_of(nb, struct xrep_xattr, dhook.dirent_hook.nb); + sc = rx->sc; + + /* + * This thread updated a dirent that points to the file that we're + * repairing, so stash the update for replay against the temporary + * file. + */ + if (I_INO(p->ip) != I_INO(sc->ip)) + return NOTIFY_DONE; + + mutex_lock(&rx->lock); + if (p->delta > 0) + error = xrep_xattr_stash_parentadd(rx, p->name, p->dp); + else + error = xrep_xattr_stash_parentremove(rx, p->name, p->dp); + if (error) + rx->live_update_aborted = true; + mutex_unlock(&rx->lock); + return NOTIFY_DONE; +} + +/* + * Prepare both inodes' attribute forks for an exchange. Promote the tempfile + * from short format to leaf format, and if the file being repaired has a short + * format attr fork, turn it into an empty extent list. + */ +STATIC int +xrep_xattr_swap_prep( + struct xfs_scrub *sc, + bool temp_local, + bool ip_local) +{ + int error; + + /* + * If the tempfile's attributes are in shortform format, convert that + * to a single leaf extent so that we can use the atomic mapping + * exchange. + */ + if (temp_local) { + struct xfs_da_args args = { + .dp = sc->tempip, + .geo = sc->mp->m_attr_geo, + .whichfork = XFS_ATTR_FORK, + .trans = sc->tp, + .total = 1, + .owner = I_INO(sc->ip), + }; + + error = xfs_attr_shortform_to_leaf(&args); + if (error) + return error; + + /* + * Roll the deferred log items to get us back to a clean + * transaction. + */ + error = xfs_defer_finish(&sc->tp); + if (error) + return error; + } + + /* + * If the file being repaired had a shortform attribute fork, convert + * that to an empty extent list in preparation for the atomic mapping + * exchange. + */ + if (ip_local) { + struct xfs_ifork *ifp; + + ifp = xfs_ifork_ptr(sc->ip, XFS_ATTR_FORK); + + xfs_idestroy_fork(ifp); + ifp->if_format = XFS_DINODE_FMT_EXTENTS; + ifp->if_nextents = 0; + ifp->if_bytes = 0; + ifp->if_data = NULL; + ifp->if_height = 0; + + xfs_trans_log_inode(sc->tp, sc->ip, + XFS_ILOG_CORE | XFS_ILOG_ADATA); + } + + return 0; +} + +/* Exchange the temporary file's attribute fork with the one being repaired. */ +int +xrep_xattr_swap( + struct xfs_scrub *sc, + struct xrep_tempexch *tx) +{ + bool ip_local, temp_local; + int error = 0; + + ip_local = sc->ip->i_af.if_format == XFS_DINODE_FMT_LOCAL; + temp_local = sc->tempip->i_af.if_format == XFS_DINODE_FMT_LOCAL; + + /* + * If the both files have a local format attr fork and the rebuilt + * xattr data would fit in the repaired file's attr fork, just copy + * the contents from the tempfile and declare ourselves done. + */ + if (ip_local && temp_local) { + int forkoff; + int newsize; + + newsize = xfs_attr_sf_totsize(sc->tempip); + forkoff = xfs_attr_shortform_bytesfit(sc->ip, newsize); + if (forkoff > 0) { + sc->ip->i_forkoff = forkoff; + xrep_tempfile_copyout_local(sc, XFS_ATTR_FORK); + return 0; + } + } + + /* Otherwise, make sure both attr forks are in block-mapping mode. */ + error = xrep_xattr_swap_prep(sc, temp_local, ip_local); + if (error) + return error; + + return xrep_tempexch_contents(sc, tx); +} + +/* + * Finish replaying stashed parent pointer updates, allocate a transaction for + * exchanging extent mappings, and take the ILOCKs of both files before we + * commit the new extended attribute structure. + */ +STATIC int +xrep_xattr_finalize_tempfile( + struct xrep_xattr *rx) +{ + struct xfs_scrub *sc = rx->sc; + int error; + + if (!xfs_has_parent(sc->mp)) + return xrep_tempexch_trans_alloc(sc, XFS_ATTR_FORK, &rx->tx); + + /* + * Repair relies on the ILOCK to quiesce all possible xattr updates. + * Replay all queued parent pointer updates into the tempfile before + * exchanging the contents, even if that means dropping the ILOCKs and + * the transaction. + */ + do { + error = xrep_xattr_replay_pptr_updates(rx); + if (error) + return error; + + error = xrep_tempexch_trans_alloc(sc, XFS_ATTR_FORK, &rx->tx); + if (error) + return error; + + if (xfarray_length(rx->pptr_recs) == 0) + break; + + xchk_trans_cancel(sc); + xrep_tempfile_iunlock_both(sc); + } while (!xchk_should_terminate(sc, &error)); + return error; +} + +/* + * Exchange the new extended attribute data (which we created in the tempfile) + * with the file being repaired. + */ +STATIC int +xrep_xattr_rebuild_tree( + struct xrep_xattr *rx) +{ + struct xfs_scrub *sc = rx->sc; + int error; + + /* + * If we didn't find any attributes to salvage, repair the file by + * zapping its attr fork. + */ + if (rx->attrs_found == 0 && + (!xfs_has_parent(sc->mp) || xfarray_length(rx->pptr_recs) == 0)) { + xfs_trans_ijoin(sc->tp, sc->ip, 0); + error = xrep_xattr_reset_fork(sc); + if (error) + return error; + + goto forget_acls; + } + + trace_xrep_xattr_rebuild_tree(sc->ip, sc->tempip); + + /* + * Commit the repair transaction and drop the ILOCKs so that we can use + * the atomic file content exchange helper functions to compute the + * correct resource reservations. + * + * We still hold IOLOCK_EXCL (aka i_rwsem) which will prevent xattr + * modifications, but there's nothing to prevent userspace from reading + * the attributes until we're ready for the exchange operation. Reads + * will return -EIO without shutting down the fs, so we're ok with + * that. + */ + error = xrep_trans_commit(sc); + if (error) + return error; + + xchk_iunlock(sc, XFS_ILOCK_EXCL); + + /* + * Take the IOLOCK on the temporary file so that we can run xattr + * operations with the same locks held as we would for a normal file. + * We still hold sc->ip's IOLOCK. + */ + error = xrep_tempfile_iolock_polled(rx->sc); + if (error) + return error; + + /* + * Allocate transaction, lock inodes, and make sure that we've replayed + * all the stashed parent pointer updates to the temp file. After this + * point, we're ready to exchange attr fork mappings. + */ + error = xrep_xattr_finalize_tempfile(rx); + if (error) + return error; + + /* + * Exchange the blocks mapped by the tempfile's attr fork with the file + * being repaired. The old attr blocks will then be attached to the + * tempfile, so reap its attr fork. + */ + error = xrep_xattr_swap(sc, &rx->tx); + if (error) + return error; + + error = xrep_xattr_reset_tempfile_fork(sc); + if (error) + return error; + + /* + * Roll to get a transaction without any inodes joined to it. Then we + * can drop the tempfile's ILOCK and IOLOCK before doing more work on + * the scrub target file. + */ + error = xfs_trans_roll(&sc->tp); + if (error) + return error; + + xrep_tempfile_iunlock(sc); + xrep_tempfile_iounlock(sc); + +forget_acls: + /* Invalidate cached ACLs now that we've reloaded all the xattrs. */ + xfs_forget_acl(VFS_I(sc->ip), SGI_ACL_FILE); + xfs_forget_acl(VFS_I(sc->ip), SGI_ACL_DEFAULT); + return 0; +} + +/* Tear down all the incore scan stuff we created. */ +STATIC void +xrep_xattr_teardown( + struct xrep_xattr *rx) +{ + if (xfs_has_parent(rx->sc->mp)) + xfs_dir_hook_del(rx->sc->mp, &rx->dhook); + if (rx->pptr_names) + xfblob_destroy(rx->pptr_names); + if (rx->pptr_recs) + xfarray_destroy(rx->pptr_recs); + if (rx->xattr_blobs) + xfblob_destroy(rx->xattr_blobs); + if (rx->xattr_records) + xfarray_destroy(rx->xattr_records); + mutex_destroy(&rx->lock); + kfree(rx); +} + +/* Set up the filesystem scan so we can regenerate extended attributes. */ +STATIC int +xrep_xattr_setup_scan( + struct xfs_scrub *sc, + struct xrep_xattr **rxp) +{ + struct xrep_xattr *rx; + int max_len; + int error; + + rx = kzalloc_obj(struct xrep_xattr, XCHK_GFP_FLAGS); + if (!rx) + return -ENOMEM; + rx->sc = sc; + rx->can_flush = true; + rx->xname.name = rx->namebuf; + + mutex_init(&rx->lock); + + /* + * Allocate enough memory to handle loading local attr values from the + * xfblob data while flushing stashed attrs to the temporary file. + * We only realloc the buffer when salvaging remote attr values. + */ + max_len = xfs_attr_leaf_entsize_local_max(sc->mp->m_attr_geo->blksize); + error = xchk_setup_xattr_buf(rx->sc, max_len); + if (error == -ENOMEM) + error = -EDEADLOCK; + if (error) + goto out_rx; + + /* Set up some staging for salvaged attribute keys and values */ + error = xfarray_create("xattr keys", 0, sizeof(struct xrep_xattr_key), + &rx->xattr_records); + if (error) + goto out_rx; + + error = xfblob_create("xattr names", &rx->xattr_blobs); + if (error) + goto out_keys; + + if (xfs_has_parent(sc->mp)) { + ASSERT(sc->flags & XCHK_FSGATES_DIRENTS); + + error = xfarray_create("xattr parent pointer entries", 0, + sizeof(struct xrep_xattr_pptr), + &rx->pptr_recs); + if (error) + goto out_values; + + error = xfblob_create("xattr parent pointer names", + &rx->pptr_names); + if (error) + goto out_pprecs; + + xfs_dir_hook_setup(&rx->dhook, xrep_xattr_live_dirent_update); + error = xfs_dir_hook_add(sc->mp, &rx->dhook); + if (error) + goto out_ppnames; + } + + *rxp = rx; + return 0; +out_ppnames: + xfblob_destroy(rx->pptr_names); +out_pprecs: + xfarray_destroy(rx->pptr_recs); +out_values: + xfblob_destroy(rx->xattr_blobs); +out_keys: + xfarray_destroy(rx->xattr_records); +out_rx: + mutex_destroy(&rx->lock); + kfree(rx); + return error; +} + +/* + * Repair the extended attribute metadata. + * + * XXX: Remote attribute value buffers encompass the entire (up to 64k) buffer. + * The buffer cache in XFS can't handle aliased multiblock buffers, so this + * might misbehave if the attr fork is crosslinked with other filesystem + * metadata. + */ +int +xrep_xattr( + struct xfs_scrub *sc) +{ + struct xrep_xattr *rx = NULL; + int error; + + if (!xfs_inode_hasattr(sc->ip)) + return -ENOENT; + + /* The rmapbt is required to reap the old attr fork. */ + if (!xfs_has_rmapbt(sc->mp)) + return -EOPNOTSUPP; + /* We require atomic file exchange range to rebuild anything. */ + if (!xfs_has_exchange_range(sc->mp)) + return -EOPNOTSUPP; + + error = xrep_xattr_setup_scan(sc, &rx); + if (error) + return error; + + ASSERT(sc->ilock_flags & XFS_ILOCK_EXCL); + + error = xrep_xattr_salvage_attributes(rx); + if (error) + goto out_scan; + + if (rx->live_update_aborted) { + error = -EIO; + goto out_scan; + } + + /* Last chance to abort before we start committing fixes. */ + if (xchk_should_terminate(sc, &error)) + goto out_scan; + + error = xrep_xattr_rebuild_tree(rx); + if (error) + goto out_scan; + +out_scan: + xrep_xattr_teardown(rx); + return error; +} diff --git a/fs/xfs/scrub/attr_repair.h b/fs/xfs/scrub/attr_repair.h new file mode 100644 index 000000000..979729bd4 --- /dev/null +++ b/fs/xfs/scrub/attr_repair.h @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2018-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_ATTR_REPAIR_H__ +#define __XFS_SCRUB_ATTR_REPAIR_H__ + +struct xrep_tempexch; + +int xrep_xattr_swap(struct xfs_scrub *sc, struct xrep_tempexch *tx); +int xrep_xattr_reset_fork(struct xfs_scrub *sc); +int xrep_xattr_reset_tempfile_fork(struct xfs_scrub *sc); + +#endif /* __XFS_SCRUB_ATTR_REPAIR_H__ */ diff --git a/fs/xfs/scrub/bitmap.c b/fs/xfs/scrub/bitmap.c new file mode 100644 index 000000000..c7fa908d9 --- /dev/null +++ b/fs/xfs/scrub/bitmap.c @@ -0,0 +1,584 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_bit.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_btree.h" +#include "scrub/scrub.h" +#include "scrub/bitmap.h" + +#include + +/* u64 bitmap */ + +struct xbitmap64_node { + struct rb_node bn_rbnode; + + /* First set bit of this interval and subtree. */ + uint64_t bn_start; + + /* Last set bit of this interval. */ + uint64_t bn_last; + + /* Last set bit of this subtree. Do not touch this. */ + uint64_t __bn_subtree_last; +}; + +/* Define our own interval tree type with uint64_t parameters. */ + +#define START(node) ((node)->bn_start) +#define LAST(node) ((node)->bn_last) + +/* + * These functions are defined by the INTERVAL_TREE_DEFINE macro, but we'll + * forward-declare them anyway for clarity. + */ +static inline __maybe_unused void +xbitmap64_tree_insert(struct xbitmap64_node *node, struct rb_root_cached *root); + +static inline __maybe_unused void +xbitmap64_tree_remove(struct xbitmap64_node *node, struct rb_root_cached *root); + +static inline __maybe_unused struct xbitmap64_node * +xbitmap64_tree_iter_first(struct rb_root_cached *root, uint64_t start, + uint64_t last); + +static inline __maybe_unused struct xbitmap64_node * +xbitmap64_tree_iter_next(struct xbitmap64_node *node, uint64_t start, + uint64_t last); + +INTERVAL_TREE_DEFINE(struct xbitmap64_node, bn_rbnode, uint64_t, + __bn_subtree_last, START, LAST, static inline __maybe_unused, + xbitmap64_tree) + +/* Iterate each interval of a bitmap. Do not change the bitmap. */ +#define for_each_xbitmap64_extent(bn, bitmap) \ + for ((bn) = rb_entry_safe(rb_first(&(bitmap)->xb_root.rb_root), \ + struct xbitmap64_node, bn_rbnode); \ + (bn) != NULL; \ + (bn) = rb_entry_safe(rb_next(&(bn)->bn_rbnode), \ + struct xbitmap64_node, bn_rbnode)) + +/* Clear a range of this bitmap. */ +int +xbitmap64_clear( + struct xbitmap64 *bitmap, + uint64_t start, + uint64_t len) +{ + struct xbitmap64_node *bn; + struct xbitmap64_node *new_bn; + uint64_t last = start + len - 1; + + while ((bn = xbitmap64_tree_iter_first(&bitmap->xb_root, start, last))) { + if (bn->bn_start < start && bn->bn_last > last) { + uint64_t old_last = bn->bn_last; + + /* overlaps with the entire clearing range */ + xbitmap64_tree_remove(bn, &bitmap->xb_root); + bn->bn_last = start - 1; + xbitmap64_tree_insert(bn, &bitmap->xb_root); + + /* add an extent */ + new_bn = kmalloc_obj(struct xbitmap64_node, + XCHK_GFP_FLAGS); + if (!new_bn) + return -ENOMEM; + new_bn->bn_start = last + 1; + new_bn->bn_last = old_last; + xbitmap64_tree_insert(new_bn, &bitmap->xb_root); + } else if (bn->bn_start < start) { + /* overlaps with the left side of the clearing range */ + xbitmap64_tree_remove(bn, &bitmap->xb_root); + bn->bn_last = start - 1; + xbitmap64_tree_insert(bn, &bitmap->xb_root); + } else if (bn->bn_last > last) { + /* overlaps with the right side of the clearing range */ + xbitmap64_tree_remove(bn, &bitmap->xb_root); + bn->bn_start = last + 1; + xbitmap64_tree_insert(bn, &bitmap->xb_root); + break; + } else { + /* in the middle of the clearing range */ + xbitmap64_tree_remove(bn, &bitmap->xb_root); + kfree(bn); + } + } + + return 0; +} + +/* Set a range of this bitmap. */ +int +xbitmap64_set( + struct xbitmap64 *bitmap, + uint64_t start, + uint64_t len) +{ + struct xbitmap64_node *left; + struct xbitmap64_node *right; + uint64_t last = start + len - 1; + int error; + + /* Is this whole range already set? */ + left = xbitmap64_tree_iter_first(&bitmap->xb_root, start, last); + if (left && left->bn_start <= start && left->bn_last >= last) + return 0; + + /* Clear out everything in the range we want to set. */ + error = xbitmap64_clear(bitmap, start, len); + if (error) + return error; + + /* Do we have a left-adjacent extent? */ + left = xbitmap64_tree_iter_first(&bitmap->xb_root, start - 1, start - 1); + ASSERT(!left || left->bn_last + 1 == start); + + /* Do we have a right-adjacent extent? */ + right = xbitmap64_tree_iter_first(&bitmap->xb_root, last + 1, last + 1); + ASSERT(!right || right->bn_start == last + 1); + + if (left && right) { + /* combine left and right adjacent extent */ + xbitmap64_tree_remove(left, &bitmap->xb_root); + xbitmap64_tree_remove(right, &bitmap->xb_root); + left->bn_last = right->bn_last; + xbitmap64_tree_insert(left, &bitmap->xb_root); + kfree(right); + } else if (left) { + /* combine with left extent */ + xbitmap64_tree_remove(left, &bitmap->xb_root); + left->bn_last = last; + xbitmap64_tree_insert(left, &bitmap->xb_root); + } else if (right) { + /* combine with right extent */ + xbitmap64_tree_remove(right, &bitmap->xb_root); + right->bn_start = start; + xbitmap64_tree_insert(right, &bitmap->xb_root); + } else { + /* add an extent */ + left = kmalloc_obj(struct xbitmap64_node, XCHK_GFP_FLAGS); + if (!left) + return -ENOMEM; + left->bn_start = start; + left->bn_last = last; + xbitmap64_tree_insert(left, &bitmap->xb_root); + } + + return 0; +} + +/* Free everything related to this bitmap. */ +void +xbitmap64_destroy( + struct xbitmap64 *bitmap) +{ + struct xbitmap64_node *bn; + + while ((bn = xbitmap64_tree_iter_first(&bitmap->xb_root, 0, -1ULL))) { + xbitmap64_tree_remove(bn, &bitmap->xb_root); + kfree(bn); + } +} + +/* Set up a per-AG block bitmap. */ +void +xbitmap64_init( + struct xbitmap64 *bitmap) +{ + bitmap->xb_root = RB_ROOT_CACHED; +} + +/* + * Remove all the blocks mentioned in @sub from the extents in @bitmap. + * + * The intent is that callers will iterate the rmapbt for all of its records + * for a given owner to generate @bitmap; and iterate all the blocks of the + * metadata structures that are not being rebuilt and have the same rmapbt + * owner to generate @sub. This routine subtracts all the extents + * mentioned in sub from all the extents linked in @bitmap, which leaves + * @bitmap as the list of blocks that are not accounted for, which we assume + * are the dead blocks of the old metadata structure. The blocks mentioned in + * @bitmap can be reaped. + * + * This is the logical equivalent of bitmap &= ~sub. + */ +int +xbitmap64_disunion( + struct xbitmap64 *bitmap, + struct xbitmap64 *sub) +{ + struct xbitmap64_node *bn; + int error; + + if (xbitmap64_empty(bitmap) || xbitmap64_empty(sub)) + return 0; + + for_each_xbitmap64_extent(bn, sub) { + error = xbitmap64_clear(bitmap, bn->bn_start, + bn->bn_last - bn->bn_start + 1); + if (error) + return error; + } + + return 0; +} + +/* How many bits are set in this bitmap? */ +uint64_t +xbitmap64_hweight( + struct xbitmap64 *bitmap) +{ + struct xbitmap64_node *bn; + uint64_t ret = 0; + + for_each_xbitmap64_extent(bn, bitmap) + ret += bn->bn_last - bn->bn_start + 1; + + return ret; +} + +/* Call a function for every run of set bits in this bitmap. */ +int +xbitmap64_walk( + struct xbitmap64 *bitmap, + xbitmap64_walk_fn fn, + void *priv) +{ + struct xbitmap64_node *bn; + int error = 0; + + for_each_xbitmap64_extent(bn, bitmap) { + error = fn(bn->bn_start, bn->bn_last - bn->bn_start + 1, priv); + if (error) + break; + } + + return error; +} + +/* Does this bitmap have no bits set at all? */ +bool +xbitmap64_empty( + struct xbitmap64 *bitmap) +{ + return bitmap->xb_root.rb_root.rb_node == NULL; +} + +/* Is the start of the range set or clear? And for how long? */ +bool +xbitmap64_test( + struct xbitmap64 *bitmap, + uint64_t start, + uint64_t *len) +{ + struct xbitmap64_node *bn; + uint64_t last = start + *len - 1; + + bn = xbitmap64_tree_iter_first(&bitmap->xb_root, start, last); + if (!bn) + return false; + if (bn->bn_start <= start) { + if (bn->bn_last < last) + *len = bn->bn_last - start + 1; + return true; + } + *len = bn->bn_start - start; + return false; +} + +/* u32 bitmap */ + +struct xbitmap32_node { + struct rb_node bn_rbnode; + + /* First set bit of this interval and subtree. */ + uint32_t bn_start; + + /* Last set bit of this interval. */ + uint32_t bn_last; + + /* Last set bit of this subtree. Do not touch this. */ + uint32_t __bn_subtree_last; +}; + +/* Define our own interval tree type with uint32_t parameters. */ + +/* + * These functions are defined by the INTERVAL_TREE_DEFINE macro, but we'll + * forward-declare them anyway for clarity. + */ +static inline __maybe_unused void +xbitmap32_tree_insert(struct xbitmap32_node *node, struct rb_root_cached *root); + +static inline __maybe_unused void +xbitmap32_tree_remove(struct xbitmap32_node *node, struct rb_root_cached *root); + +static inline __maybe_unused struct xbitmap32_node * +xbitmap32_tree_iter_first(struct rb_root_cached *root, uint32_t start, + uint32_t last); + +static inline __maybe_unused struct xbitmap32_node * +xbitmap32_tree_iter_next(struct xbitmap32_node *node, uint32_t start, + uint32_t last); + +INTERVAL_TREE_DEFINE(struct xbitmap32_node, bn_rbnode, uint32_t, + __bn_subtree_last, START, LAST, static inline __maybe_unused, + xbitmap32_tree) + +/* Iterate each interval of a bitmap. Do not change the bitmap. */ +#define for_each_xbitmap32_extent(bn, bitmap) \ + for ((bn) = rb_entry_safe(rb_first(&(bitmap)->xb_root.rb_root), \ + struct xbitmap32_node, bn_rbnode); \ + (bn) != NULL; \ + (bn) = rb_entry_safe(rb_next(&(bn)->bn_rbnode), \ + struct xbitmap32_node, bn_rbnode)) + +/* Clear a range of this bitmap. */ +int +xbitmap32_clear( + struct xbitmap32 *bitmap, + uint32_t start, + uint32_t len) +{ + struct xbitmap32_node *bn; + struct xbitmap32_node *new_bn; + uint32_t last = start + len - 1; + + while ((bn = xbitmap32_tree_iter_first(&bitmap->xb_root, start, last))) { + if (bn->bn_start < start && bn->bn_last > last) { + uint32_t old_last = bn->bn_last; + + /* overlaps with the entire clearing range */ + xbitmap32_tree_remove(bn, &bitmap->xb_root); + bn->bn_last = start - 1; + xbitmap32_tree_insert(bn, &bitmap->xb_root); + + /* add an extent */ + new_bn = kmalloc_obj(struct xbitmap32_node, + XCHK_GFP_FLAGS); + if (!new_bn) + return -ENOMEM; + new_bn->bn_start = last + 1; + new_bn->bn_last = old_last; + xbitmap32_tree_insert(new_bn, &bitmap->xb_root); + } else if (bn->bn_start < start) { + /* overlaps with the left side of the clearing range */ + xbitmap32_tree_remove(bn, &bitmap->xb_root); + bn->bn_last = start - 1; + xbitmap32_tree_insert(bn, &bitmap->xb_root); + } else if (bn->bn_last > last) { + /* overlaps with the right side of the clearing range */ + xbitmap32_tree_remove(bn, &bitmap->xb_root); + bn->bn_start = last + 1; + xbitmap32_tree_insert(bn, &bitmap->xb_root); + break; + } else { + /* in the middle of the clearing range */ + xbitmap32_tree_remove(bn, &bitmap->xb_root); + kfree(bn); + } + } + + return 0; +} + +/* Set a range of this bitmap. */ +int +xbitmap32_set( + struct xbitmap32 *bitmap, + uint32_t start, + uint32_t len) +{ + struct xbitmap32_node *left; + struct xbitmap32_node *right; + uint32_t last = start + len - 1; + int error; + + /* Is this whole range already set? */ + left = xbitmap32_tree_iter_first(&bitmap->xb_root, start, last); + if (left && left->bn_start <= start && left->bn_last >= last) + return 0; + + /* Clear out everything in the range we want to set. */ + error = xbitmap32_clear(bitmap, start, len); + if (error) + return error; + + /* Do we have a left-adjacent extent? */ + left = xbitmap32_tree_iter_first(&bitmap->xb_root, start - 1, start - 1); + ASSERT(!left || left->bn_last + 1 == start); + + /* Do we have a right-adjacent extent? */ + right = xbitmap32_tree_iter_first(&bitmap->xb_root, last + 1, last + 1); + ASSERT(!right || right->bn_start == last + 1); + + if (left && right) { + /* combine left and right adjacent extent */ + xbitmap32_tree_remove(left, &bitmap->xb_root); + xbitmap32_tree_remove(right, &bitmap->xb_root); + left->bn_last = right->bn_last; + xbitmap32_tree_insert(left, &bitmap->xb_root); + kfree(right); + } else if (left) { + /* combine with left extent */ + xbitmap32_tree_remove(left, &bitmap->xb_root); + left->bn_last = last; + xbitmap32_tree_insert(left, &bitmap->xb_root); + } else if (right) { + /* combine with right extent */ + xbitmap32_tree_remove(right, &bitmap->xb_root); + right->bn_start = start; + xbitmap32_tree_insert(right, &bitmap->xb_root); + } else { + /* add an extent */ + left = kmalloc_obj(struct xbitmap32_node, XCHK_GFP_FLAGS); + if (!left) + return -ENOMEM; + left->bn_start = start; + left->bn_last = last; + xbitmap32_tree_insert(left, &bitmap->xb_root); + } + + return 0; +} + +/* Free everything related to this bitmap. */ +void +xbitmap32_destroy( + struct xbitmap32 *bitmap) +{ + struct xbitmap32_node *bn; + + while ((bn = xbitmap32_tree_iter_first(&bitmap->xb_root, 0, -1U))) { + xbitmap32_tree_remove(bn, &bitmap->xb_root); + kfree(bn); + } +} + +/* Set up a per-AG block bitmap. */ +void +xbitmap32_init( + struct xbitmap32 *bitmap) +{ + bitmap->xb_root = RB_ROOT_CACHED; +} + +/* + * Remove all the blocks mentioned in @sub from the extents in @bitmap. + * + * The intent is that callers will iterate the rmapbt for all of its records + * for a given owner to generate @bitmap; and iterate all the blocks of the + * metadata structures that are not being rebuilt and have the same rmapbt + * owner to generate @sub. This routine subtracts all the extents + * mentioned in sub from all the extents linked in @bitmap, which leaves + * @bitmap as the list of blocks that are not accounted for, which we assume + * are the dead blocks of the old metadata structure. The blocks mentioned in + * @bitmap can be reaped. + * + * This is the logical equivalent of bitmap &= ~sub. + */ +int +xbitmap32_disunion( + struct xbitmap32 *bitmap, + struct xbitmap32 *sub) +{ + struct xbitmap32_node *bn; + int error; + + if (xbitmap32_empty(bitmap) || xbitmap32_empty(sub)) + return 0; + + for_each_xbitmap32_extent(bn, sub) { + error = xbitmap32_clear(bitmap, bn->bn_start, + bn->bn_last - bn->bn_start + 1); + if (error) + return error; + } + + return 0; +} + +/* How many bits are set in this bitmap? */ +uint32_t +xbitmap32_hweight( + struct xbitmap32 *bitmap) +{ + struct xbitmap32_node *bn; + uint32_t ret = 0; + + for_each_xbitmap32_extent(bn, bitmap) + ret += bn->bn_last - bn->bn_start + 1; + + return ret; +} + +/* Call a function for every run of set bits in this bitmap. */ +int +xbitmap32_walk( + struct xbitmap32 *bitmap, + xbitmap32_walk_fn fn, + void *priv) +{ + struct xbitmap32_node *bn; + int error = 0; + + for_each_xbitmap32_extent(bn, bitmap) { + error = fn(bn->bn_start, bn->bn_last - bn->bn_start + 1, priv); + if (error) + break; + } + + return error; +} + +/* Does this bitmap have no bits set at all? */ +bool +xbitmap32_empty( + struct xbitmap32 *bitmap) +{ + return bitmap->xb_root.rb_root.rb_node == NULL; +} + +/* Is the start of the range set or clear? And for how long? */ +bool +xbitmap32_test( + struct xbitmap32 *bitmap, + uint32_t start, + uint32_t *len) +{ + struct xbitmap32_node *bn; + uint32_t last = start + *len - 1; + + bn = xbitmap32_tree_iter_first(&bitmap->xb_root, start, last); + if (!bn) + return false; + if (bn->bn_start <= start) { + if (bn->bn_last < last) + *len = bn->bn_last - start + 1; + return true; + } + *len = bn->bn_start - start; + return false; +} + +/* Count the number of set regions in this bitmap. */ +uint32_t +xbitmap32_count_set_regions( + struct xbitmap32 *bitmap) +{ + struct xbitmap32_node *bn; + uint32_t nr = 0; + + for_each_xbitmap32_extent(bn, bitmap) + nr++; + + return nr; +} diff --git a/fs/xfs/scrub/bitmap.h b/fs/xfs/scrub/bitmap.h new file mode 100644 index 000000000..710c1ac5e --- /dev/null +++ b/fs/xfs/scrub/bitmap.h @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_BITMAP_H__ +#define __XFS_SCRUB_BITMAP_H__ + +/* u64 bitmap */ + +struct xbitmap64 { + struct rb_root_cached xb_root; +}; + +void xbitmap64_init(struct xbitmap64 *bitmap); +void xbitmap64_destroy(struct xbitmap64 *bitmap); + +int xbitmap64_clear(struct xbitmap64 *bitmap, uint64_t start, uint64_t len); +int xbitmap64_set(struct xbitmap64 *bitmap, uint64_t start, uint64_t len); +int xbitmap64_disunion(struct xbitmap64 *bitmap, struct xbitmap64 *sub); +uint64_t xbitmap64_hweight(struct xbitmap64 *bitmap); + +/* + * Return codes for the bitmap iterator functions are 0 to continue iterating, + * and non-zero to stop iterating. Any non-zero value will be passed up to the + * iteration caller. The special value -ECANCELED can be used to stop + * iteration, because neither bitmap iterator ever generates that error code on + * its own. Callers must not modify the bitmap while walking it. + */ +typedef int (*xbitmap64_walk_fn)(uint64_t start, uint64_t len, void *priv); +int xbitmap64_walk(struct xbitmap64 *bitmap, xbitmap64_walk_fn fn, + void *priv); + +bool xbitmap64_empty(struct xbitmap64 *bitmap); +bool xbitmap64_test(struct xbitmap64 *bitmap, uint64_t start, uint64_t *len); + +/* u32 bitmap */ + +struct xbitmap32 { + struct rb_root_cached xb_root; +}; + +void xbitmap32_init(struct xbitmap32 *bitmap); +void xbitmap32_destroy(struct xbitmap32 *bitmap); + +int xbitmap32_clear(struct xbitmap32 *bitmap, uint32_t start, uint32_t len); +int xbitmap32_set(struct xbitmap32 *bitmap, uint32_t start, uint32_t len); +int xbitmap32_disunion(struct xbitmap32 *bitmap, struct xbitmap32 *sub); +uint32_t xbitmap32_hweight(struct xbitmap32 *bitmap); + +/* + * Return codes for the bitmap iterator functions are 0 to continue iterating, + * and non-zero to stop iterating. Any non-zero value will be passed up to the + * iteration caller. The special value -ECANCELED can be used to stop + * iteration, because neither bitmap iterator ever generates that error code on + * its own. Callers must not modify the bitmap while walking it. + */ +typedef int (*xbitmap32_walk_fn)(uint32_t start, uint32_t len, void *priv); +int xbitmap32_walk(struct xbitmap32 *bitmap, xbitmap32_walk_fn fn, + void *priv); + +bool xbitmap32_empty(struct xbitmap32 *bitmap); +bool xbitmap32_test(struct xbitmap32 *bitmap, uint32_t start, uint32_t *len); + +uint32_t xbitmap32_count_set_regions(struct xbitmap32 *bitmap); + +#endif /* __XFS_SCRUB_BITMAP_H__ */ diff --git a/fs/xfs/scrub/bmap.c b/fs/xfs/scrub/bmap.c new file mode 100644 index 000000000..401c27872 --- /dev/null +++ b/fs/xfs/scrub/bmap.c @@ -0,0 +1,1192 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_btree.h" +#include "xfs_bit.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_inode.h" +#include "xfs_alloc.h" +#include "xfs_bmap.h" +#include "xfs_bmap_btree.h" +#include "xfs_rmap.h" +#include "xfs_rmap_btree.h" +#include "xfs_rtgroup.h" +#include "xfs_health.h" +#include "xfs_rtalloc.h" +#include "xfs_rtrmap_btree.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/btree.h" +#include "scrub/health.h" +#include "xfs_ag.h" + +/* Set us up with an inode's bmap. */ +int +xchk_setup_inode_bmap( + struct xfs_scrub *sc) +{ + int error; + + if (xchk_need_intent_drain(sc)) + xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN); + + error = xchk_iget_for_scrubbing(sc); + if (error) + goto out; + + xchk_ilock(sc, XFS_IOLOCK_EXCL); + + /* + * We don't want any ephemeral data/cow fork updates sitting around + * while we inspect block mappings, so wait for directio to finish + * and flush dirty data if we have delalloc reservations. + */ + if (S_ISREG(VFS_I(sc->ip)->i_mode) && + sc->sm->sm_type != XFS_SCRUB_TYPE_BMBTA) { + struct address_space *mapping = VFS_I(sc->ip)->i_mapping; + bool is_repair = xchk_could_repair(sc); + + xchk_ilock(sc, XFS_MMAPLOCK_EXCL); + + /* Break all our leases, we're going to mess with things. */ + if (is_repair) { + error = xfs_break_layouts(VFS_I(sc->ip), + &sc->ilock_flags, BREAK_WRITE); + if (error) + goto out; + } + + inode_dio_wait(VFS_I(sc->ip)); + + /* + * Try to flush all incore state to disk before we examine the + * space mappings for the data fork. Leave accumulated errors + * in the mapping for the writer threads to consume. + * + * On ENOSPC or EIO writeback errors, we continue into the + * extent mapping checks because write failures do not + * necessarily imply anything about the correctness of the file + * metadata. The metadata and the file data could be on + * completely separate devices; a media failure might only + * affect a subset of the disk, etc. We can handle delalloc + * extents in the scrubber, so leaving them in memory is fine. + */ + error = filemap_fdatawrite(mapping); + if (!error) + error = filemap_fdatawait_keep_errors(mapping); + if (error && (error != -ENOSPC && error != -EIO)) + goto out; + + /* Drop the page cache if we're repairing block mappings. */ + if (is_repair) { + error = invalidate_inode_pages2( + VFS_I(sc->ip)->i_mapping); + if (error) + goto out; + } + + } + + /* Got the inode, lock it and we're ready to go. */ + error = xchk_trans_alloc(sc, 0); + if (error) + goto out; + + error = xchk_ino_dqattach(sc); + if (error) + goto out; + + xchk_ilock(sc, XFS_ILOCK_EXCL); +out: + /* scrub teardown will unlock and release the inode */ + return error; +} + +/* + * Inode fork block mapping (BMBT) scrubber. + * More complex than the others because we have to scrub + * all the extents regardless of whether or not the fork + * is in btree format. + */ + +struct xchk_bmap_info { + struct xfs_scrub *sc; + + /* Incore extent tree cursor */ + struct xfs_iext_cursor icur; + + /* Previous fork mapping that we examined */ + struct xfs_bmbt_irec prev_rec; + + /* Is this a realtime fork? */ + bool is_rt; + + /* May mappings point to shared space? */ + bool is_shared; + + /* Was the incore extent tree loaded? */ + bool was_loaded; + + /* Which inode fork are we checking? */ + int whichfork; +}; + +/* Look for a corresponding rmap for this irec. */ +static inline bool +xchk_bmap_get_rmap( + struct xchk_bmap_info *info, + struct xfs_bmbt_irec *irec, + xfs_agblock_t bno, + uint64_t owner, + struct xfs_rmap_irec *rmap) +{ + struct xfs_btree_cur **curp = &info->sc->sa.rmap_cur; + xfs_fileoff_t offset; + unsigned int rflags = 0; + int has_rmap; + int error; + + if (xfs_ifork_is_realtime(info->sc->ip, info->whichfork)) + curp = &info->sc->sr.rmap_cur; + + if (*curp == NULL) + return false; + + if (info->whichfork == XFS_ATTR_FORK) + rflags |= XFS_RMAP_ATTR_FORK; + if (irec->br_state == XFS_EXT_UNWRITTEN) + rflags |= XFS_RMAP_UNWRITTEN; + + /* + * CoW staging extents are owned (on disk) by the refcountbt, so + * their rmaps do not have offsets. + */ + if (info->whichfork == XFS_COW_FORK) + offset = 0; + else + offset = irec->br_startoff; + + /* + * If the caller thinks this could be a shared bmbt extent (IOWs, + * any data fork extent of a reflink inode) then we have to use the + * range rmap lookup to make sure we get the correct owner/offset. + */ + if (info->is_shared) { + error = xfs_rmap_lookup_le_range(*curp, bno, owner, offset, + rflags, rmap, &has_rmap); + } else { + error = xfs_rmap_lookup_le(*curp, bno, owner, offset, + rflags, rmap, &has_rmap); + } + if (!xchk_should_check_xref(info->sc, &error, curp)) + return false; + + if (!has_rmap) + xchk_fblock_xref_set_corrupt(info->sc, info->whichfork, + irec->br_startoff); + return has_rmap; +} + +/* Make sure that we have rmapbt records for this data/attr fork extent. */ +STATIC void +xchk_bmap_xref_rmap( + struct xchk_bmap_info *info, + struct xfs_bmbt_irec *irec, + xfs_agblock_t bno) +{ + struct xfs_rmap_irec rmap; + unsigned long long rmap_end; + uint64_t owner = I_INO(info->sc->ip); + + if (xchk_skip_xref(info->sc->sm)) + return; + + /* Find the rmap record for this irec. */ + if (!xchk_bmap_get_rmap(info, irec, bno, owner, &rmap)) + return; + + /* + * The rmap must be an exact match for this incore file mapping record, + * which may have arisen from multiple ondisk records. + */ + if (rmap.rm_startblock != bno) + xchk_fblock_xref_set_corrupt(info->sc, info->whichfork, + irec->br_startoff); + + rmap_end = (unsigned long long)rmap.rm_startblock + rmap.rm_blockcount; + if (rmap_end != bno + irec->br_blockcount) + xchk_fblock_xref_set_corrupt(info->sc, info->whichfork, + irec->br_startoff); + + /* Check the logical offsets. */ + if (rmap.rm_offset != irec->br_startoff) + xchk_fblock_xref_set_corrupt(info->sc, info->whichfork, + irec->br_startoff); + + rmap_end = (unsigned long long)rmap.rm_offset + rmap.rm_blockcount; + if (rmap_end != irec->br_startoff + irec->br_blockcount) + xchk_fblock_xref_set_corrupt(info->sc, info->whichfork, + irec->br_startoff); + + /* Check the owner */ + if (rmap.rm_owner != owner) + xchk_fblock_xref_set_corrupt(info->sc, info->whichfork, + irec->br_startoff); + + /* + * Check for discrepancies between the unwritten flag in the irec and + * the rmap. Note that the (in-memory) CoW fork distinguishes between + * unwritten and written extents, but we don't track that in the rmap + * records because the blocks are owned (on-disk) by the refcountbt, + * which doesn't track unwritten state. + */ + if (!!(irec->br_state == XFS_EXT_UNWRITTEN) != + !!(rmap.rm_flags & XFS_RMAP_UNWRITTEN)) + xchk_fblock_xref_set_corrupt(info->sc, info->whichfork, + irec->br_startoff); + + if (!!(info->whichfork == XFS_ATTR_FORK) != + !!(rmap.rm_flags & XFS_RMAP_ATTR_FORK)) + xchk_fblock_xref_set_corrupt(info->sc, info->whichfork, + irec->br_startoff); + if (rmap.rm_flags & XFS_RMAP_BMBT_BLOCK) + xchk_fblock_xref_set_corrupt(info->sc, info->whichfork, + irec->br_startoff); +} + +/* Make sure that we have rmapbt records for this COW fork extent. */ +STATIC void +xchk_bmap_xref_rmap_cow( + struct xchk_bmap_info *info, + struct xfs_bmbt_irec *irec, + xfs_agblock_t bno) +{ + struct xfs_rmap_irec rmap; + unsigned long long rmap_end; + uint64_t owner = XFS_RMAP_OWN_COW; + + if (!info->sc->sa.rmap_cur || xchk_skip_xref(info->sc->sm)) + return; + + /* Find the rmap record for this irec. */ + if (!xchk_bmap_get_rmap(info, irec, bno, owner, &rmap)) + return; + + /* + * CoW staging extents are owned by the refcount btree, so the rmap + * can start before and end after the physical space allocated to this + * mapping. There are no offsets to check. + */ + if (rmap.rm_startblock > bno) + xchk_fblock_xref_set_corrupt(info->sc, info->whichfork, + irec->br_startoff); + + rmap_end = (unsigned long long)rmap.rm_startblock + rmap.rm_blockcount; + if (rmap_end < bno + irec->br_blockcount) + xchk_fblock_xref_set_corrupt(info->sc, info->whichfork, + irec->br_startoff); + + /* Check the owner */ + if (rmap.rm_owner != owner) + xchk_fblock_xref_set_corrupt(info->sc, info->whichfork, + irec->br_startoff); + + /* + * No flags allowed. Note that the (in-memory) CoW fork distinguishes + * between unwritten and written extents, but we don't track that in + * the rmap records because the blocks are owned (on-disk) by the + * refcountbt, which doesn't track unwritten state. + */ + if (rmap.rm_flags & XFS_RMAP_ATTR_FORK) + xchk_fblock_xref_set_corrupt(info->sc, info->whichfork, + irec->br_startoff); + if (rmap.rm_flags & XFS_RMAP_BMBT_BLOCK) + xchk_fblock_xref_set_corrupt(info->sc, info->whichfork, + irec->br_startoff); + if (rmap.rm_flags & XFS_RMAP_UNWRITTEN) + xchk_fblock_xref_set_corrupt(info->sc, info->whichfork, + irec->br_startoff); +} + +/* Cross-reference a single rtdev extent record. */ +STATIC void +xchk_bmap_rt_iextent_xref( + struct xfs_inode *ip, + struct xchk_bmap_info *info, + struct xfs_bmbt_irec *irec) +{ + struct xfs_owner_info oinfo; + xfs_rgblock_t rgbno; + int error; + + error = xchk_rtgroup_init_existing(info->sc, + xfs_rtb_to_rgno(ip->i_mount, irec->br_startblock), + &info->sc->sr); + if (!xchk_fblock_process_error(info->sc, info->whichfork, + irec->br_startoff, &error)) + return; + + error = xchk_rtgroup_lock(info->sc, &info->sc->sr, XCHK_RTGLOCK_ALL); + if (!xchk_fblock_process_error(info->sc, info->whichfork, + irec->br_startoff, &error)) + goto out_free; + + xchk_xref_is_used_rt_space(info->sc, irec->br_startblock, + irec->br_blockcount); + + if (!xfs_has_rtrmapbt(info->sc->mp)) + goto out_cur; + + rgbno = xfs_rtb_to_rgbno(info->sc->mp, irec->br_startblock); + + switch (info->whichfork) { + case XFS_DATA_FORK: + xchk_bmap_xref_rmap(info, irec, rgbno); + if (!xfs_is_reflink_inode(info->sc->ip)) { + xfs_rmap_inode_owner(&oinfo, info->sc->ip, + info->whichfork, irec->br_startoff); + xchk_xref_is_only_rt_owned_by(info->sc, rgbno, + irec->br_blockcount, &oinfo); + xchk_xref_is_not_rt_shared(info->sc, rgbno, + irec->br_blockcount); + } + xchk_xref_is_not_rt_cow_staging(info->sc, rgbno, + irec->br_blockcount); + break; + case XFS_COW_FORK: + xchk_bmap_xref_rmap_cow(info, irec, rgbno); + xchk_xref_is_only_rt_owned_by(info->sc, rgbno, + irec->br_blockcount, &XFS_RMAP_OINFO_COW); + xchk_xref_is_rt_cow_staging(info->sc, rgbno, + irec->br_blockcount); + xchk_xref_is_not_rt_shared(info->sc, rgbno, + irec->br_blockcount); + break; + } +out_cur: + xchk_rtgroup_btcur_free(&info->sc->sr); +out_free: + xchk_rtgroup_free(info->sc, &info->sc->sr); +} + +/* Cross-reference a single datadev extent record. */ +STATIC void +xchk_bmap_iextent_xref( + struct xfs_inode *ip, + struct xchk_bmap_info *info, + struct xfs_bmbt_irec *irec) +{ + struct xfs_owner_info oinfo; + struct xfs_mount *mp = info->sc->mp; + xfs_agnumber_t agno; + xfs_agblock_t agbno; + xfs_extlen_t len; + int error; + + agno = XFS_FSB_TO_AGNO(mp, irec->br_startblock); + agbno = XFS_FSB_TO_AGBNO(mp, irec->br_startblock); + len = irec->br_blockcount; + + error = xchk_ag_init_existing(info->sc, agno, &info->sc->sa); + if (!xchk_fblock_process_error(info->sc, info->whichfork, + irec->br_startoff, &error)) + goto out_free; + + xchk_xref_is_used_space(info->sc, agbno, len); + xchk_xref_is_not_inode_chunk(info->sc, agbno, len); + switch (info->whichfork) { + case XFS_DATA_FORK: + xchk_bmap_xref_rmap(info, irec, agbno); + if (!xfs_is_reflink_inode(info->sc->ip)) { + xfs_rmap_inode_owner(&oinfo, info->sc->ip, + info->whichfork, irec->br_startoff); + xchk_xref_is_only_owned_by(info->sc, agbno, + irec->br_blockcount, &oinfo); + xchk_xref_is_not_shared(info->sc, agbno, + irec->br_blockcount); + } + xchk_xref_is_not_cow_staging(info->sc, agbno, + irec->br_blockcount); + break; + case XFS_ATTR_FORK: + xchk_bmap_xref_rmap(info, irec, agbno); + xfs_rmap_inode_owner(&oinfo, info->sc->ip, + info->whichfork, irec->br_startoff); + xchk_xref_is_only_owned_by(info->sc, agbno, irec->br_blockcount, + &oinfo); + xchk_xref_is_not_shared(info->sc, agbno, + irec->br_blockcount); + xchk_xref_is_not_cow_staging(info->sc, agbno, + irec->br_blockcount); + break; + case XFS_COW_FORK: + xchk_bmap_xref_rmap_cow(info, irec, agbno); + xchk_xref_is_only_owned_by(info->sc, agbno, irec->br_blockcount, + &XFS_RMAP_OINFO_COW); + xchk_xref_is_cow_staging(info->sc, agbno, + irec->br_blockcount); + xchk_xref_is_not_shared(info->sc, agbno, + irec->br_blockcount); + break; + } + +out_free: + xchk_ag_free(info->sc, &info->sc->sa); +} + +/* + * Directories and attr forks should never have blocks that can't be addressed + * by a xfs_dablk_t. + */ +STATIC void +xchk_bmap_dirattr_extent( + struct xfs_inode *ip, + struct xchk_bmap_info *info, + struct xfs_bmbt_irec *irec) +{ + struct xfs_mount *mp = ip->i_mount; + xfs_fileoff_t off; + + if (!S_ISDIR(VFS_I(ip)->i_mode) && info->whichfork != XFS_ATTR_FORK) + return; + + if (!xfs_verify_dablk(mp, irec->br_startoff)) + xchk_fblock_set_corrupt(info->sc, info->whichfork, + irec->br_startoff); + + off = irec->br_startoff + irec->br_blockcount - 1; + if (!xfs_verify_dablk(mp, off)) + xchk_fblock_set_corrupt(info->sc, info->whichfork, off); +} + +/* Scrub a single extent record. */ +STATIC void +xchk_bmap_iextent( + struct xfs_inode *ip, + struct xchk_bmap_info *info, + struct xfs_bmbt_irec *irec) +{ + struct xfs_mount *mp = info->sc->mp; + + /* + * Check for out-of-order extents. This record could have come + * from the incore list, for which there is no ordering check. + */ + if (irec->br_startoff < info->prev_rec.br_startoff + + info->prev_rec.br_blockcount) + xchk_fblock_set_corrupt(info->sc, info->whichfork, + irec->br_startoff); + + if (!xfs_verify_fileext(mp, irec->br_startoff, irec->br_blockcount)) + xchk_fblock_set_corrupt(info->sc, info->whichfork, + irec->br_startoff); + + xchk_bmap_dirattr_extent(ip, info, irec); + + /* Make sure the extent points to a valid place. */ + if (info->is_rt && + !xfs_verify_rtbext(mp, irec->br_startblock, irec->br_blockcount)) + xchk_fblock_set_corrupt(info->sc, info->whichfork, + irec->br_startoff); + if (!info->is_rt && + !xfs_verify_fsbext(mp, irec->br_startblock, irec->br_blockcount)) + xchk_fblock_set_corrupt(info->sc, info->whichfork, + irec->br_startoff); + + /* We don't allow unwritten extents on attr forks. */ + if (irec->br_state == XFS_EXT_UNWRITTEN && + info->whichfork == XFS_ATTR_FORK) + xchk_fblock_set_corrupt(info->sc, info->whichfork, + irec->br_startoff); + + if (info->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return; + + if (info->is_rt) + xchk_bmap_rt_iextent_xref(ip, info, irec); + else + xchk_bmap_iextent_xref(ip, info, irec); +} + +/* Scrub a bmbt record. */ +STATIC int +xchk_bmapbt_rec( + struct xchk_btree *bs, + const union xfs_btree_rec *rec) +{ + struct xfs_bmbt_irec irec; + struct xfs_bmbt_irec iext_irec; + struct xfs_iext_cursor icur; + struct xchk_bmap_info *info = bs->private; + struct xfs_inode *ip = bs->cur->bc_ino.ip; + struct xfs_buf *bp = NULL; + struct xfs_btree_block *block; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, info->whichfork); + uint64_t owner; + int i; + + /* + * Check the owners of the btree blocks up to the level below + * the root since the verifiers don't do that. + */ + if (xfs_has_crc(bs->cur->bc_mp) && + bs->cur->bc_levels[0].ptr == 1) { + for (i = 0; i < bs->cur->bc_nlevels - 1; i++) { + block = xfs_btree_get_block(bs->cur, i, &bp); + owner = be64_to_cpu(block->bb_u.l.bb_owner); + if (owner != I_INO(ip)) + xchk_fblock_set_corrupt(bs->sc, + info->whichfork, 0); + } + } + + /* + * Check that the incore extent tree contains an extent that matches + * this one exactly. We validate those cached bmaps later, so we don't + * need to check them here. If the incore extent tree was just loaded + * from disk by the scrubber, we assume that its contents match what's + * on disk (we still hold the ILOCK) and skip the equivalence check. + */ + if (!info->was_loaded) + return 0; + + xfs_bmbt_disk_get_all(&rec->bmbt, &irec); + if (xfs_bmap_validate_extent(ip, info->whichfork, &irec) != NULL) { + xchk_fblock_set_corrupt(bs->sc, info->whichfork, + irec.br_startoff); + return 0; + } + + if (!xfs_iext_lookup_extent(ip, ifp, irec.br_startoff, &icur, + &iext_irec) || + irec.br_startoff != iext_irec.br_startoff || + irec.br_startblock != iext_irec.br_startblock || + irec.br_blockcount != iext_irec.br_blockcount || + irec.br_state != iext_irec.br_state) + xchk_fblock_set_corrupt(bs->sc, info->whichfork, + irec.br_startoff); + return 0; +} + +/* Scan the btree records. */ +STATIC int +xchk_bmap_btree( + struct xfs_scrub *sc, + int whichfork, + struct xchk_bmap_info *info) +{ + struct xfs_owner_info oinfo; + struct xfs_ifork *ifp = xfs_ifork_ptr(sc->ip, whichfork); + struct xfs_mount *mp = sc->mp; + struct xfs_inode *ip = sc->ip; + struct xfs_btree_cur *cur; + int error; + + /* Load the incore bmap cache if it's not loaded. */ + info->was_loaded = !xfs_need_iread_extents(ifp); + + error = xfs_iread_extents(sc->tp, ip, whichfork); + if (!xchk_fblock_process_error(sc, whichfork, 0, &error)) + goto out; + + /* Check the btree structure. */ + cur = xfs_bmbt_init_cursor(mp, sc->tp, ip, whichfork); + xfs_rmap_inode_bmbt_owner(&oinfo, ip, whichfork); + error = xchk_btree(sc, cur, xchk_bmapbt_rec, &oinfo, info); + xfs_btree_del_cursor(cur, error); +out: + return error; +} + +struct xchk_bmap_check_rmap_info { + struct xfs_scrub *sc; + int whichfork; + struct xfs_iext_cursor icur; +}; + +/* Can we find bmaps that fit this rmap? */ +STATIC int +xchk_bmap_check_rmap( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv) +{ + struct xfs_bmbt_irec irec; + struct xfs_rmap_irec check_rec; + struct xchk_bmap_check_rmap_info *sbcri = priv; + struct xfs_ifork *ifp; + struct xfs_scrub *sc = sbcri->sc; + bool have_map; + + /* Is this even the right fork? */ + if (rec->rm_owner != I_INO(sc->ip)) + return 0; + if ((sbcri->whichfork == XFS_ATTR_FORK) ^ + !!(rec->rm_flags & XFS_RMAP_ATTR_FORK)) + return 0; + if (rec->rm_flags & XFS_RMAP_BMBT_BLOCK) + return 0; + + /* Now look up the bmbt record. */ + ifp = xfs_ifork_ptr(sc->ip, sbcri->whichfork); + if (!ifp) { + xchk_fblock_set_corrupt(sc, sbcri->whichfork, + rec->rm_offset); + goto out; + } + have_map = xfs_iext_lookup_extent(sc->ip, ifp, rec->rm_offset, + &sbcri->icur, &irec); + if (!have_map) + xchk_fblock_set_corrupt(sc, sbcri->whichfork, + rec->rm_offset); + /* + * bmap extent record lengths are constrained to 2^21 blocks in length + * because of space constraints in the on-disk metadata structure. + * However, rmap extent record lengths are constrained only by AG + * length, so we have to loop through the bmbt to make sure that the + * entire rmap is covered by bmbt records. + */ + check_rec = *rec; + while (have_map) { + if (irec.br_startoff != check_rec.rm_offset) + xchk_fblock_set_corrupt(sc, sbcri->whichfork, + check_rec.rm_offset); + if (irec.br_startblock != + xfs_gbno_to_fsb(cur->bc_group, check_rec.rm_startblock)) + xchk_fblock_set_corrupt(sc, sbcri->whichfork, + check_rec.rm_offset); + if (irec.br_blockcount > check_rec.rm_blockcount) + xchk_fblock_set_corrupt(sc, sbcri->whichfork, + check_rec.rm_offset); + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + break; + check_rec.rm_startblock += irec.br_blockcount; + check_rec.rm_offset += irec.br_blockcount; + check_rec.rm_blockcount -= irec.br_blockcount; + if (check_rec.rm_blockcount == 0) + break; + have_map = xfs_iext_next_extent(ifp, &sbcri->icur, &irec); + if (!have_map) + xchk_fblock_set_corrupt(sc, sbcri->whichfork, + check_rec.rm_offset); + } + +out: + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return -ECANCELED; + return 0; +} + +/* Make sure each rmap has a corresponding bmbt entry. */ +STATIC int +xchk_bmap_check_ag_rmaps( + struct xfs_scrub *sc, + int whichfork, + struct xfs_perag *pag) +{ + struct xchk_bmap_check_rmap_info sbcri; + struct xfs_btree_cur *cur; + struct xfs_buf *agf; + int error; + + error = xfs_alloc_read_agf(pag, sc->tp, 0, &agf); + if (error) + return error; + + cur = xfs_rmapbt_init_cursor(sc->mp, sc->tp, agf, pag); + + sbcri.sc = sc; + sbcri.whichfork = whichfork; + error = xfs_rmap_query_all(cur, xchk_bmap_check_rmap, &sbcri); + if (error == -ECANCELED) + error = 0; + + xfs_btree_del_cursor(cur, error); + xfs_trans_brelse(sc->tp, agf); + return error; +} + +/* Make sure each rt rmap has a corresponding bmbt entry. */ +STATIC int +xchk_bmap_check_rt_rmaps( + struct xfs_scrub *sc, + struct xfs_rtgroup *rtg) +{ + struct xchk_bmap_check_rmap_info sbcri; + struct xfs_btree_cur *cur; + int error; + + xfs_rtgroup_lock(rtg, XFS_RTGLOCK_RMAP); + cur = xfs_rtrmapbt_init_cursor(sc->tp, rtg); + + sbcri.sc = sc; + sbcri.whichfork = XFS_DATA_FORK; + error = xfs_rmap_query_all(cur, xchk_bmap_check_rmap, &sbcri); + if (error == -ECANCELED) + error = 0; + + xfs_btree_del_cursor(cur, error); + xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_RMAP); + return error; +} + +/* + * Decide if we want to scan the reverse mappings to determine if the attr + * fork /really/ has zero space mappings. + */ +STATIC bool +xchk_bmap_check_empty_attrfork( + struct xfs_inode *ip) +{ + struct xfs_ifork *ifp = &ip->i_af; + + /* + * If the dinode repair found a bad attr fork, it will reset the fork + * to extents format with zero records and wait for the this scrubber + * to reconstruct the block mappings. If the fork is not in this + * state, then the fork cannot have been zapped. + */ + if (ifp->if_format != XFS_DINODE_FMT_EXTENTS || ifp->if_nextents != 0) + return false; + + /* + * Files can have an attr fork in EXTENTS format with zero records for + * several reasons: + * + * a) an attr set created a fork but ran out of space + * b) attr replace deleted an old attr but failed during the set step + * c) the data fork was in btree format when all attrs were deleted, so + * the fork was left in place + * d) the inode repair code zapped the fork + * + * Only in case (d) do we want to scan the rmapbt to see if we need to + * rebuild the attr fork. The fork zap code clears all DAC permission + * bits and zeroes the uid and gid, so avoid the scan if any of those + * three conditions are not met. + */ + if ((VFS_I(ip)->i_mode & 0777) != 0) + return false; + if (!uid_eq(VFS_I(ip)->i_uid, GLOBAL_ROOT_UID)) + return false; + if (!gid_eq(VFS_I(ip)->i_gid, GLOBAL_ROOT_GID)) + return false; + + return true; +} + +/* + * Decide if we want to scan the reverse mappings to determine if the data + * fork /really/ has zero space mappings. + */ +STATIC bool +xchk_bmap_check_empty_datafork( + struct xfs_inode *ip) +{ + struct xfs_ifork *ifp = &ip->i_df; + + /* + * If the dinode repair found a bad data fork, it will reset the fork + * to extents format with zero records and wait for the this scrubber + * to reconstruct the block mappings. If the fork is not in this + * state, then the fork cannot have been zapped. + */ + if (ifp->if_format != XFS_DINODE_FMT_EXTENTS || ifp->if_nextents != 0) + return false; + + /* + * If we encounter an empty data fork along with evidence that the fork + * might not really be empty, we need to scan the reverse mappings to + * decide if we're going to rebuild the fork. Data forks with nonzero + * file size are scanned. + */ + return i_size_read(VFS_I(ip)) != 0; +} + +/* + * Decide if we want to walk every rmap btree in the fs to make sure that each + * rmap for this file fork has corresponding bmbt entries. + */ +static bool +xchk_bmap_want_check_rmaps( + struct xchk_bmap_info *info) +{ + struct xfs_scrub *sc = info->sc; + + if (!xfs_has_rmapbt(sc->mp)) + return false; + if (info->whichfork == XFS_COW_FORK) + return false; + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return false; + + if (info->whichfork == XFS_ATTR_FORK) + return xchk_bmap_check_empty_attrfork(sc->ip); + + return xchk_bmap_check_empty_datafork(sc->ip); +} + +/* Make sure each rmap has a corresponding bmbt entry. */ +STATIC int +xchk_bmap_check_rmaps( + struct xfs_scrub *sc, + int whichfork) +{ + struct xfs_perag *pag = NULL; + int error; + + if (xfs_ifork_is_realtime(sc->ip, whichfork)) { + struct xfs_rtgroup *rtg = NULL; + + while ((rtg = xfs_rtgroup_next(sc->mp, rtg))) { + error = xchk_bmap_check_rt_rmaps(sc, rtg); + if (error || + (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) { + xfs_rtgroup_rele(rtg); + return error; + } + } + + return 0; + } + + while ((pag = xfs_perag_next(sc->mp, pag))) { + error = xchk_bmap_check_ag_rmaps(sc, whichfork, pag); + if (error || + (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) { + xfs_perag_rele(pag); + return error; + } + } + + return 0; +} + +/* Scrub a delalloc reservation from the incore extent map tree. */ +STATIC void +xchk_bmap_iextent_delalloc( + struct xfs_inode *ip, + struct xchk_bmap_info *info, + struct xfs_bmbt_irec *irec) +{ + struct xfs_mount *mp = info->sc->mp; + + /* + * Check for out-of-order extents. This record could have come + * from the incore list, for which there is no ordering check. + */ + if (irec->br_startoff < info->prev_rec.br_startoff + + info->prev_rec.br_blockcount) + xchk_fblock_set_corrupt(info->sc, info->whichfork, + irec->br_startoff); + + if (!xfs_verify_fileext(mp, irec->br_startoff, irec->br_blockcount)) + xchk_fblock_set_corrupt(info->sc, info->whichfork, + irec->br_startoff); + + /* Make sure the extent points to a valid place. */ + if (irec->br_blockcount > XFS_MAX_BMBT_EXTLEN) + xchk_fblock_set_corrupt(info->sc, info->whichfork, + irec->br_startoff); +} + +/* Decide if this individual fork mapping is ok. */ +static bool +xchk_bmap_iext_mapping( + struct xchk_bmap_info *info, + const struct xfs_bmbt_irec *irec) +{ + /* There should never be a "hole" extent in either extent list. */ + if (irec->br_startblock == HOLESTARTBLOCK) + return false; + if (irec->br_blockcount > XFS_MAX_BMBT_EXTLEN) + return false; + return true; +} + +/* Are these two mappings contiguous with each other? */ +static inline bool +xchk_are_bmaps_contiguous( + const struct xchk_bmap_info *info, + const struct xfs_bmbt_irec *b1, + const struct xfs_bmbt_irec *b2) +{ + struct xfs_mount *mp = info->sc->mp; + + /* Don't try to combine unallocated mappings. */ + if (!xfs_bmap_is_real_extent(b1)) + return false; + if (!xfs_bmap_is_real_extent(b2)) + return false; + + /* Does b2 come right after b1 in the logical and physical range? */ + if (b1->br_startoff + b1->br_blockcount != b2->br_startoff) + return false; + if (b1->br_startblock + b1->br_blockcount != b2->br_startblock) + return false; + if (b1->br_state != b2->br_state) + return false; + + /* + * Don't combine bmaps that would cross rtgroup boundaries. This is a + * valid state, but if combined they will fail rtb extent checks. + */ + if (info->is_rt && xfs_has_rtgroups(mp)) { + if (xfs_rtb_to_rgno(mp, b1->br_startblock) != + xfs_rtb_to_rgno(mp, b2->br_startblock)) + return false; + } + + return true; +} + +/* + * Walk the incore extent records, accumulating consecutive contiguous records + * into a single incore mapping. Returns true if @irec has been set to a + * mapping or false if there are no more mappings. Caller must ensure that + * @info.icur is zeroed before the first call. + */ +static bool +xchk_bmap_iext_iter( + struct xchk_bmap_info *info, + struct xfs_bmbt_irec *irec) +{ + struct xfs_bmbt_irec got; + struct xfs_ifork *ifp; + unsigned int nr = 0; + + ifp = xfs_ifork_ptr(info->sc->ip, info->whichfork); + + /* Advance to the next iextent record and check the mapping. */ + xfs_iext_next(ifp, &info->icur); + if (!xfs_iext_get_extent(ifp, &info->icur, irec)) + return false; + + if (!xchk_bmap_iext_mapping(info, irec)) { + xchk_fblock_set_corrupt(info->sc, info->whichfork, + irec->br_startoff); + return false; + } + nr++; + + /* + * Iterate subsequent iextent records and merge them with the one + * that we just read, if possible. + */ + while (xfs_iext_peek_next_extent(ifp, &info->icur, &got)) { + if (!xchk_are_bmaps_contiguous(info, irec, &got)) + break; + + if (!xchk_bmap_iext_mapping(info, &got)) { + xchk_fblock_set_corrupt(info->sc, info->whichfork, + got.br_startoff); + return false; + } + nr++; + + irec->br_blockcount += got.br_blockcount; + xfs_iext_next(ifp, &info->icur); + } + + /* + * If the merged mapping could be expressed with fewer bmbt records + * than we actually found, notify the user that this fork could be + * optimized. CoW forks only exist in memory so we ignore them. + */ + if (nr > 1 && info->whichfork != XFS_COW_FORK && + howmany_64(irec->br_blockcount, XFS_MAX_BMBT_EXTLEN) < nr) + xchk_ino_set_preen(info->sc, I_INO(info->sc->ip)); + + return true; +} + +/* + * Scrub an inode fork's block mappings. + * + * First we scan every record in every btree block, if applicable. + * Then we unconditionally scan the incore extent cache. + */ +STATIC int +xchk_bmap( + struct xfs_scrub *sc, + int whichfork) +{ + struct xfs_bmbt_irec irec; + struct xchk_bmap_info info = { NULL }; + struct xfs_mount *mp = sc->mp; + struct xfs_inode *ip = sc->ip; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + xfs_fileoff_t endoff; + int error = 0; + + /* Non-existent forks can be ignored. */ + if (!ifp) + return -ENOENT; + + info.is_rt = xfs_ifork_is_realtime(ip, whichfork); + info.whichfork = whichfork; + info.is_shared = whichfork == XFS_DATA_FORK && xfs_is_reflink_inode(ip); + info.sc = sc; + + switch (whichfork) { + case XFS_COW_FORK: + /* No CoW forks filesystem doesn't support out of place writes */ + if (!xfs_has_reflink(mp) && !xfs_has_zoned(mp)) { + xchk_ip_set_corrupt(sc, sc->ip); + return 0; + } + break; + case XFS_ATTR_FORK: + /* + * "attr" means that an attr fork was created at some point in + * the life of this filesystem. "attr2" means that inodes have + * variable-sized data/attr fork areas. Hence we only check + * attr here. + */ + if (!xfs_has_attr(mp)) + xchk_ip_set_corrupt(sc, sc->ip); + break; + default: + ASSERT(whichfork == XFS_DATA_FORK); + break; + } + + /* Check the fork values */ + switch (ifp->if_format) { + case XFS_DINODE_FMT_UUID: + case XFS_DINODE_FMT_DEV: + case XFS_DINODE_FMT_LOCAL: + case XFS_DINODE_FMT_META_BTREE: + /* No mappings to check. */ + if (whichfork == XFS_COW_FORK) + xchk_fblock_set_corrupt(sc, whichfork, 0); + return 0; + case XFS_DINODE_FMT_EXTENTS: + break; + case XFS_DINODE_FMT_BTREE: + if (whichfork == XFS_COW_FORK) { + xchk_fblock_set_corrupt(sc, whichfork, 0); + return 0; + } + + error = xchk_bmap_btree(sc, whichfork, &info); + if (error) + return error; + break; + default: + xchk_fblock_set_corrupt(sc, whichfork, 0); + return 0; + } + + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return 0; + + /* Find the offset of the last extent in the mapping. */ + error = xfs_bmap_last_offset(ip, &endoff, whichfork); + if (!xchk_fblock_process_error(sc, whichfork, 0, &error)) + return error; + + /* + * Scrub extent records. We use a special iterator function here that + * combines adjacent mappings if they are logically and physically + * contiguous. For large allocations that require multiple bmbt + * records, this reduces the number of cross-referencing calls, which + * reduces runtime. Cross referencing with the rmap is simpler because + * the rmap must match the combined mapping exactly. + */ + while (xchk_bmap_iext_iter(&info, &irec)) { + if (xchk_should_terminate(sc, &error) || + (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) + return 0; + + if (irec.br_startoff >= endoff) { + xchk_fblock_set_corrupt(sc, whichfork, + irec.br_startoff); + return 0; + } + + if (isnullstartblock(irec.br_startblock)) + xchk_bmap_iextent_delalloc(ip, &info, &irec); + else + xchk_bmap_iextent(ip, &info, &irec); + memcpy(&info.prev_rec, &irec, sizeof(struct xfs_bmbt_irec)); + } + + if (xchk_bmap_want_check_rmaps(&info)) { + error = xchk_bmap_check_rmaps(sc, whichfork); + if (!xchk_fblock_xref_process_error(sc, whichfork, 0, &error)) + return error; + } + + return 0; +} + +/* Scrub an inode's data fork. */ +int +xchk_bmap_data( + struct xfs_scrub *sc) +{ + int error; + + if (xchk_file_looks_zapped(sc, XFS_SICK_INO_BMBTD_ZAPPED)) { + xchk_ip_set_corrupt(sc, sc->ip); + return 0; + } + + error = xchk_bmap(sc, XFS_DATA_FORK); + if (error) + return error; + + /* If the data fork is clean, it is clearly not zapped. */ + xchk_mark_healthy_if_clean(sc, XFS_SICK_INO_BMBTD_ZAPPED); + return 0; +} + +/* Scrub an inode's attr fork. */ +int +xchk_bmap_attr( + struct xfs_scrub *sc) +{ + int error; + + /* + * If the attr fork has been zapped, it's possible that forkoff was + * reset to zero and hence sc->ip->i_afp is NULL. We don't want the + * NULL ifp check in xchk_bmap to conclude that the attr fork is ok, + * so short circuit that logic by setting the corruption flag and + * returning immediately. + */ + if (xchk_file_looks_zapped(sc, XFS_SICK_INO_BMBTA_ZAPPED)) { + xchk_ip_set_corrupt(sc, sc->ip); + return 0; + } + + error = xchk_bmap(sc, XFS_ATTR_FORK); + /* A repaired, empty attr fork no longer has mappings to check. */ + if (error == -ENOENT && (sc->flags & XREP_ALREADY_FIXED)) { + xchk_mark_healthy_if_clean(sc, XFS_SICK_INO_BMBTA_ZAPPED); + return 0; + } + if (error) + return error; + + /* If the attr fork is clean, it is clearly not zapped. */ + xchk_mark_healthy_if_clean(sc, XFS_SICK_INO_BMBTA_ZAPPED); + return 0; +} + +/* Scrub an inode's CoW fork. */ +int +xchk_bmap_cow( + struct xfs_scrub *sc) +{ + return xchk_bmap(sc, XFS_COW_FORK); +} diff --git a/fs/xfs/scrub/bmap_repair.c b/fs/xfs/scrub/bmap_repair.c new file mode 100644 index 000000000..eabffba47 --- /dev/null +++ b/fs/xfs/scrub/bmap_repair.c @@ -0,0 +1,992 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_btree.h" +#include "xfs_btree_staging.h" +#include "xfs_bit.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_sb.h" +#include "xfs_inode.h" +#include "xfs_inode_fork.h" +#include "xfs_alloc.h" +#include "xfs_rtalloc.h" +#include "xfs_bmap.h" +#include "xfs_bmap_util.h" +#include "xfs_bmap_btree.h" +#include "xfs_rmap.h" +#include "xfs_rmap_btree.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_refcount.h" +#include "xfs_quota.h" +#include "xfs_ialloc.h" +#include "xfs_ag.h" +#include "xfs_reflink.h" +#include "xfs_rtgroup.h" +#include "scrub/xfs_scrub.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/btree.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/bitmap.h" +#include "scrub/fsb_bitmap.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/newbt.h" +#include "scrub/reap.h" + +/* + * Inode Fork Block Mapping (BMBT) Repair + * ====================================== + * + * Gather all the rmap records for the inode and fork we're fixing, reset the + * incore fork, then recreate the btree. + */ + +enum reflink_scan_state { + RLS_IRRELEVANT = -1, /* not applicable to this file */ + RLS_UNKNOWN, /* shared extent scans required */ + RLS_SET_IFLAG, /* iflag must be set */ +}; + +struct xrep_bmap { + /* Old bmbt blocks */ + struct xfsb_bitmap old_bmbt_blocks; + + /* New fork. */ + struct xrep_newbt new_bmapbt; + + /* List of new bmap records. */ + struct xfarray *bmap_records; + + struct xfs_scrub *sc; + + /* How many blocks did we find allocated to this file? */ + xfs_rfsblock_t nblocks; + + /* How many bmbt blocks did we find for this fork? */ + xfs_rfsblock_t old_bmbt_block_count; + + /* get_records()'s position in the free space record array. */ + xfarray_idx_t array_cur; + + /* How many real (non-hole, non-delalloc) mappings do we have? */ + uint64_t real_mappings; + + /* Which fork are we fixing? */ + int whichfork; + + /* What d the REFLINK flag be set when the repair is over? */ + enum reflink_scan_state reflink_scan; + + /* Do we allow unwritten extents? */ + bool allow_unwritten; +}; + +/* Is this space extent shared? Flag the inode if it is. */ +STATIC int +xrep_bmap_discover_shared( + struct xrep_bmap *rb, + xfs_fsblock_t startblock, + xfs_filblks_t blockcount) +{ + struct xfs_scrub *sc = rb->sc; + struct xfs_btree_cur *cur; + xfs_agblock_t agbno; + xfs_agblock_t fbno; + xfs_extlen_t flen; + int error; + + if (XFS_IS_REALTIME_INODE(sc->ip)) { + agbno = xfs_rtb_to_rgbno(sc->mp, startblock); + cur = sc->sr.refc_cur; + } else { + agbno = XFS_FSB_TO_AGBNO(sc->mp, startblock); + cur = sc->sa.refc_cur; + } + error = xfs_refcount_find_shared(cur, agbno, blockcount, &fbno, &flen, + false); + if (error) + return error; + + if (fbno != NULLAGBLOCK) + rb->reflink_scan = RLS_SET_IFLAG; + + return 0; +} + +/* Remember this reverse-mapping as a series of bmap records. */ +STATIC int +xrep_bmap_from_rmap( + struct xrep_bmap *rb, + xfs_fileoff_t startoff, + xfs_fsblock_t startblock, + xfs_filblks_t blockcount, + bool unwritten) +{ + struct xfs_bmbt_irec irec = { + .br_startoff = startoff, + .br_startblock = startblock, + .br_state = unwritten ? XFS_EXT_UNWRITTEN : XFS_EXT_NORM, + }; + struct xfs_bmbt_rec rbe; + struct xfs_scrub *sc = rb->sc; + int error = 0; + + /* + * If we're repairing the data fork of a non-reflinked regular file on + * a reflink filesystem, we need to figure out if this space extent is + * shared. + */ + if (rb->reflink_scan == RLS_UNKNOWN && !unwritten) { + error = xrep_bmap_discover_shared(rb, startblock, blockcount); + if (error) + return error; + } + + do { + xfs_failaddr_t fa; + + irec.br_blockcount = min_t(xfs_filblks_t, blockcount, + XFS_MAX_BMBT_EXTLEN); + + fa = xfs_bmap_validate_extent(sc->ip, rb->whichfork, &irec); + if (fa) + return -EFSCORRUPTED; + + xfs_bmbt_disk_set_all(&rbe, &irec); + + trace_xrep_bmap_found(sc->ip, rb->whichfork, &irec); + + if (xchk_should_terminate(sc, &error)) + return error; + + error = xfarray_append(rb->bmap_records, &rbe); + if (error) + return error; + + rb->real_mappings++; + + irec.br_startblock += irec.br_blockcount; + irec.br_startoff += irec.br_blockcount; + blockcount -= irec.br_blockcount; + } while (blockcount > 0); + + return 0; +} + +/* Check for any obvious errors or conflicts in the file mapping. */ +STATIC int +xrep_bmap_check_fork_rmap( + struct xrep_bmap *rb, + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec) +{ + struct xfs_scrub *sc = rb->sc; + enum xbtree_recpacking outcome; + int error; + + /* + * Data extents for rt files are never stored on the data device, but + * everything else (xattrs, bmbt blocks) can be. + */ + if (XFS_IS_REALTIME_INODE(sc->ip) && + !(rec->rm_flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK))) + return -EFSCORRUPTED; + + /* Check that this is within the AG. */ + if (!xfs_verify_agbext(to_perag(cur->bc_group), rec->rm_startblock, + rec->rm_blockcount)) + return -EFSCORRUPTED; + + /* Check the file offset range. */ + if (!(rec->rm_flags & XFS_RMAP_BMBT_BLOCK) && + !xfs_verify_fileext(sc->mp, rec->rm_offset, rec->rm_blockcount)) + return -EFSCORRUPTED; + + /* No contradictory flags. */ + if ((rec->rm_flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK)) && + (rec->rm_flags & XFS_RMAP_UNWRITTEN)) + return -EFSCORRUPTED; + + /* Make sure this isn't free space. */ + error = xfs_alloc_has_records(sc->sa.bno_cur, rec->rm_startblock, + rec->rm_blockcount, &outcome); + if (error) + return error; + if (outcome != XBTREE_RECPACKING_EMPTY) + return -EFSCORRUPTED; + + /* Must not be an inode chunk. */ + error = xfs_ialloc_has_inodes_at_extent(sc->sa.ino_cur, + rec->rm_startblock, rec->rm_blockcount, &outcome); + if (error) + return error; + if (outcome != XBTREE_RECPACKING_EMPTY) + return -EFSCORRUPTED; + + return 0; +} + +/* Record extents that belong to this inode's fork. */ +STATIC int +xrep_bmap_walk_rmap( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv) +{ + struct xrep_bmap *rb = priv; + xfs_fsblock_t fsbno; + int error = 0; + + if (xchk_should_terminate(rb->sc, &error)) + return error; + + if (rec->rm_owner != I_INO(rb->sc->ip)) + return 0; + + error = xrep_bmap_check_fork_rmap(rb, cur, rec); + if (error) + return error; + + /* + * Record all blocks allocated to this file even if the extent isn't + * for the fork we're rebuilding so that we can reset di_nblocks later. + */ + rb->nblocks += rec->rm_blockcount; + + /* If this rmap isn't for the fork we want, we're done. */ + if (rb->whichfork == XFS_DATA_FORK && + (rec->rm_flags & XFS_RMAP_ATTR_FORK)) + return 0; + if (rb->whichfork == XFS_ATTR_FORK && + !(rec->rm_flags & XFS_RMAP_ATTR_FORK)) + return 0; + + /* Reject unwritten extents if we don't allow those. */ + if ((rec->rm_flags & XFS_RMAP_UNWRITTEN) && !rb->allow_unwritten) + return -EFSCORRUPTED; + + fsbno = xfs_agbno_to_fsb(to_perag(cur->bc_group), rec->rm_startblock); + + if (rec->rm_flags & XFS_RMAP_BMBT_BLOCK) { + rb->old_bmbt_block_count += rec->rm_blockcount; + return xfsb_bitmap_set(&rb->old_bmbt_blocks, fsbno, + rec->rm_blockcount); + } + + return xrep_bmap_from_rmap(rb, rec->rm_offset, fsbno, + rec->rm_blockcount, + rec->rm_flags & XFS_RMAP_UNWRITTEN); +} + +/* + * Compare two block mapping records. We want to sort in order of increasing + * file offset. + */ +static int +xrep_bmap_extent_cmp( + const void *a, + const void *b) +{ + const struct xfs_bmbt_rec *ba = a; + const struct xfs_bmbt_rec *bb = b; + xfs_fileoff_t ao = xfs_bmbt_disk_get_startoff(ba); + xfs_fileoff_t bo = xfs_bmbt_disk_get_startoff(bb); + + if (ao > bo) + return 1; + else if (ao < bo) + return -1; + return 0; +} + +/* + * Sort the bmap extents by fork offset or else the records will be in the + * wrong order. Ensure there are no overlaps in the file offset ranges. + */ +STATIC int +xrep_bmap_sort_records( + struct xrep_bmap *rb) +{ + struct xfs_bmbt_irec irec; + xfs_fileoff_t next_off = 0; + xfarray_idx_t array_cur; + int error; + + error = xfarray_sort(rb->bmap_records, xrep_bmap_extent_cmp, + XFARRAY_SORT_KILLABLE); + if (error) + return error; + + foreach_xfarray_idx(rb->bmap_records, array_cur) { + struct xfs_bmbt_rec rec; + + if (xchk_should_terminate(rb->sc, &error)) + return error; + + error = xfarray_load(rb->bmap_records, array_cur, &rec); + if (error) + return error; + + xfs_bmbt_disk_get_all(&rec, &irec); + + if (irec.br_startoff < next_off) + return -EFSCORRUPTED; + + next_off = irec.br_startoff + irec.br_blockcount; + } + + return 0; +} + +/* Scan one AG for reverse mappings that we can turn into extent maps. */ +STATIC int +xrep_bmap_scan_ag( + struct xrep_bmap *rb, + struct xfs_perag *pag) +{ + struct xfs_scrub *sc = rb->sc; + int error; + + error = xrep_ag_init(sc, pag, &sc->sa); + if (error) + return error; + + error = xfs_rmap_query_all(sc->sa.rmap_cur, xrep_bmap_walk_rmap, rb); + xchk_ag_free(sc, &sc->sa); + return error; +} + +#ifdef CONFIG_XFS_RT +/* Check for any obvious errors or conflicts in the file mapping. */ +STATIC int +xrep_bmap_check_rtfork_rmap( + struct xfs_scrub *sc, + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec) +{ + /* xattr extents are never stored on realtime devices */ + if (rec->rm_flags & XFS_RMAP_ATTR_FORK) + return -EFSCORRUPTED; + + /* bmbt blocks are never stored on realtime devices */ + if (rec->rm_flags & XFS_RMAP_BMBT_BLOCK) + return -EFSCORRUPTED; + + /* Data extents for non-rt files are never stored on the rt device. */ + if (!XFS_IS_REALTIME_INODE(sc->ip)) + return -EFSCORRUPTED; + + /* Check the file offsets and physical extents. */ + if (!xfs_verify_fileext(sc->mp, rec->rm_offset, rec->rm_blockcount)) + return -EFSCORRUPTED; + + /* Check that this is within the rtgroup. */ + if (!xfs_verify_rgbext(to_rtg(cur->bc_group), rec->rm_startblock, + rec->rm_blockcount)) + return -EFSCORRUPTED; + + /* Make sure this isn't free space. */ + return xrep_require_rtext_inuse(sc, rec->rm_startblock, + rec->rm_blockcount); +} + +/* Record realtime extents that belong to this inode's fork. */ +STATIC int +xrep_bmap_walk_rtrmap( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv) +{ + struct xrep_bmap *rb = priv; + int error = 0; + + if (xchk_should_terminate(rb->sc, &error)) + return error; + + /* Skip extents which are not owned by this inode and fork. */ + if (rec->rm_owner != I_INO(rb->sc->ip)) + return 0; + + error = xrep_bmap_check_rtfork_rmap(rb->sc, cur, rec); + if (error) + return error; + + /* + * Record all blocks allocated to this file even if the extent isn't + * for the fork we're rebuilding so that we can reset di_nblocks later. + */ + rb->nblocks += rec->rm_blockcount; + + /* If this rmap isn't for the fork we want, we're done. */ + if (rb->whichfork == XFS_DATA_FORK && + (rec->rm_flags & XFS_RMAP_ATTR_FORK)) + return 0; + if (rb->whichfork == XFS_ATTR_FORK && + !(rec->rm_flags & XFS_RMAP_ATTR_FORK)) + return 0; + + return xrep_bmap_from_rmap(rb, rec->rm_offset, + xfs_rgbno_to_rtb(to_rtg(cur->bc_group), + rec->rm_startblock), + rec->rm_blockcount, + rec->rm_flags & XFS_RMAP_UNWRITTEN); +} + +/* Scan the realtime reverse mappings to build the new extent map. */ +STATIC int +xrep_bmap_scan_rtgroup( + struct xrep_bmap *rb, + struct xfs_rtgroup *rtg) +{ + struct xfs_scrub *sc = rb->sc; + int error; + + if (!xfs_has_rtrmapbt(sc->mp)) + return 0; + + error = xrep_rtgroup_init(sc, rtg, &sc->sr, + XFS_RTGLOCK_RMAP | + XFS_RTGLOCK_REFCOUNT | + XFS_RTGLOCK_BITMAP_SHARED); + if (error) + return error; + + error = xfs_rmap_query_all(sc->sr.rmap_cur, xrep_bmap_walk_rtrmap, rb); + xchk_rtgroup_btcur_free(&sc->sr); + xchk_rtgroup_free(sc, &sc->sr); + return error; +} +#else +static inline int +xrep_bmap_scan_rtgroup(struct xrep_bmap *rb, struct xfs_rtgroup *rtg) +{ + return -EFSCORRUPTED; +} +#endif + +/* Find the delalloc extents from the old incore extent tree. */ +STATIC int +xrep_bmap_find_delalloc( + struct xrep_bmap *rb) +{ + struct xfs_bmbt_irec irec; + struct xfs_iext_cursor icur; + struct xfs_bmbt_rec rbe; + struct xfs_inode *ip = rb->sc->ip; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, rb->whichfork); + int error = 0; + + /* + * Skip this scan if we don't expect to find delayed allocation + * reservations in this fork. + */ + if (rb->whichfork == XFS_ATTR_FORK || ip->i_delayed_blks == 0) + return 0; + + for_each_xfs_iext(ifp, &icur, &irec) { + if (!isnullstartblock(irec.br_startblock)) + continue; + + xfs_bmbt_disk_set_all(&rbe, &irec); + + trace_xrep_bmap_found(ip, rb->whichfork, &irec); + + if (xchk_should_terminate(rb->sc, &error)) + return error; + + error = xfarray_append(rb->bmap_records, &rbe); + if (error) + return error; + } + + return 0; +} + +/* + * Collect block mappings for this fork of this inode and decide if we have + * enough space to rebuild. Caller is responsible for cleaning up the list if + * anything goes wrong. + */ +STATIC int +xrep_bmap_find_mappings( + struct xrep_bmap *rb) +{ + struct xfs_scrub *sc = rb->sc; + struct xfs_perag *pag = NULL; + int error = 0; + + /* + * Iterate the rtrmaps for extents. Metadata files never have content + * on the realtime device, so there's no need to scan them. + */ + if (!xfs_is_metadir_inode(sc->ip)) { + struct xfs_rtgroup *rtg = NULL; + + while ((rtg = xfs_rtgroup_next(sc->mp, rtg))) { + error = xrep_bmap_scan_rtgroup(rb, rtg); + if (error) { + xfs_rtgroup_rele(rtg); + return error; + } + } + } + + /* Iterate the rmaps for extents. */ + while ((pag = xfs_perag_next(sc->mp, pag))) { + error = xrep_bmap_scan_ag(rb, pag); + if (error) { + xfs_perag_rele(pag); + return error; + } + } + + return xrep_bmap_find_delalloc(rb); +} + +/* Retrieve real extent mappings for bulk loading the bmap btree. */ +STATIC int +xrep_bmap_get_records( + struct xfs_btree_cur *cur, + unsigned int idx, + struct xfs_btree_block *block, + unsigned int nr_wanted, + void *priv) +{ + struct xfs_bmbt_rec rec; + struct xfs_bmbt_irec *irec = &cur->bc_rec.b; + struct xrep_bmap *rb = priv; + union xfs_btree_rec *block_rec; + unsigned int loaded; + int error; + + for (loaded = 0; loaded < nr_wanted; loaded++, idx++) { + do { + error = xfarray_load(rb->bmap_records, rb->array_cur++, + &rec); + if (error) + return error; + + xfs_bmbt_disk_get_all(&rec, irec); + } while (isnullstartblock(irec->br_startblock)); + + block_rec = xfs_btree_rec_addr(cur, idx, block); + cur->bc_ops->init_rec_from_cur(cur, block_rec); + } + + return loaded; +} + +/* Feed one of the new btree blocks to the bulk loader. */ +STATIC int +xrep_bmap_claim_block( + struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr, + void *priv) +{ + struct xrep_bmap *rb = priv; + + return xrep_newbt_claim_block(cur, &rb->new_bmapbt, ptr); +} + +/* Figure out how much space we need to create the incore btree root block. */ +STATIC size_t +xrep_bmap_iroot_size( + struct xfs_btree_cur *cur, + unsigned int level, + unsigned int nr_this_level, + void *priv) +{ + ASSERT(level > 0); + + return xfs_bmap_broot_space_calc(cur->bc_mp, nr_this_level); +} + +/* Update the inode counters. */ +STATIC int +xrep_bmap_reset_counters( + struct xrep_bmap *rb) +{ + struct xfs_scrub *sc = rb->sc; + struct xbtree_ifakeroot *ifake = &rb->new_bmapbt.ifake; + int64_t delta; + + if (rb->reflink_scan == RLS_SET_IFLAG) + sc->ip->i_diflags2 |= XFS_DIFLAG2_REFLINK; + + /* + * Update the inode block counts to reflect the extents we found in the + * rmapbt. + */ + delta = ifake->if_blocks - rb->old_bmbt_block_count; + sc->ip->i_nblocks = rb->nblocks + delta; + xfs_trans_log_inode(sc->tp, sc->ip, XFS_ILOG_CORE); + + /* + * Adjust the quota counts by the difference in size between the old + * and new bmbt. + */ + xfs_trans_mod_dquot_byino(sc->tp, sc->ip, XFS_TRANS_DQ_BCOUNT, delta); + return 0; +} + +/* + * Create a new iext tree and load it with block mappings. If the inode is + * in extents format, that's all we need to do to commit the new mappings. + * If it is in btree format, this takes care of preloading the incore tree. + */ +STATIC int +xrep_bmap_extents_load( + struct xrep_bmap *rb) +{ + struct xfs_iext_cursor icur; + struct xfs_bmbt_irec irec; + struct xfs_ifork *ifp = rb->new_bmapbt.ifake.if_fork; + xfarray_idx_t array_cur; + int error; + + ASSERT(ifp->if_bytes == 0); + + /* Add all the mappings (incl. delalloc) to the incore extent tree. */ + xfs_iext_first(ifp, &icur); + foreach_xfarray_idx(rb->bmap_records, array_cur) { + struct xfs_bmbt_rec rec; + + error = xfarray_load(rb->bmap_records, array_cur, &rec); + if (error) + return error; + + xfs_bmbt_disk_get_all(&rec, &irec); + + xfs_iext_insert_raw(ifp, &icur, &irec); + if (!isnullstartblock(irec.br_startblock)) + ifp->if_nextents++; + + xfs_iext_next(ifp, &icur); + } + + return xrep_ino_ensure_extent_count(rb->sc, rb->whichfork, + ifp->if_nextents); +} + +/* + * Reserve new btree blocks, bulk load the bmap records into the ondisk btree, + * and load the incore extent tree. + */ +STATIC int +xrep_bmap_btree_load( + struct xrep_bmap *rb, + struct xfs_btree_cur *bmap_cur) +{ + struct xfs_scrub *sc = rb->sc; + int error; + + /* Compute how many blocks we'll need. */ + error = xfs_btree_bload_compute_geometry(bmap_cur, + &rb->new_bmapbt.bload, rb->real_mappings); + if (error) + return error; + + /* Last chance to abort before we start committing fixes. */ + if (xchk_should_terminate(sc, &error)) + return error; + + /* + * Guess how many blocks we're going to need to rebuild an entire bmap + * from the number of extents we found, and pump up our transaction to + * have sufficient block reservation. We're allowed to exceed file + * quota to repair inconsistent metadata. + */ + error = xfs_trans_reserve_more_inode(sc->tp, sc->ip, + rb->new_bmapbt.bload.nr_blocks, 0, true); + if (error) + return error; + + /* Reserve the space we'll need for the new btree. */ + error = xrep_newbt_alloc_blocks(&rb->new_bmapbt, + rb->new_bmapbt.bload.nr_blocks); + if (error) + return error; + + /* Add all observed bmap records. */ + rb->array_cur = XFARRAY_CURSOR_INIT; + error = xfs_btree_bload(bmap_cur, &rb->new_bmapbt.bload, rb); + if (error) + return error; + + /* + * Load the new bmap records into the new incore extent tree to + * preserve delalloc reservations for regular files. The directory + * code loads the extent tree during xfs_dir_open and assumes + * thereafter that it remains loaded, so we must not violate that + * assumption. + */ + return xrep_bmap_extents_load(rb); +} + +/* + * Use the collected bmap information to stage a new bmap fork. If this is + * successful we'll return with the new fork information logged to the repair + * transaction but not yet committed. The caller must ensure that the inode + * is joined to the transaction; the inode will be joined to a clean + * transaction when the function returns. + */ +STATIC int +xrep_bmap_build_new_fork( + struct xrep_bmap *rb) +{ + struct xfs_owner_info oinfo; + struct xfs_scrub *sc = rb->sc; + struct xfs_btree_cur *bmap_cur; + struct xbtree_ifakeroot *ifake = &rb->new_bmapbt.ifake; + int error; + + error = xrep_bmap_sort_records(rb); + if (error) + return error; + + /* + * Prepare to construct the new fork by initializing the new btree + * structure and creating a fake ifork in the ifakeroot structure. + */ + xfs_rmap_inode_bmbt_owner(&oinfo, sc->ip, rb->whichfork); + error = xrep_newbt_init_inode(&rb->new_bmapbt, sc, rb->whichfork, + &oinfo); + if (error) + return error; + + rb->new_bmapbt.bload.get_records = xrep_bmap_get_records; + rb->new_bmapbt.bload.claim_block = xrep_bmap_claim_block; + rb->new_bmapbt.bload.iroot_size = xrep_bmap_iroot_size; + + /* + * Allocate a new bmap btree cursor for reloading an inode block mapping + * data structure. + */ + bmap_cur = xfs_bmbt_init_cursor(sc->mp, NULL, sc->ip, XFS_STAGING_FORK); + xfs_btree_stage_ifakeroot(bmap_cur, ifake); + + /* + * Figure out the size and format of the new fork, then fill it with + * all the bmap records we've found. Join the inode to the transaction + * so that we can roll the transaction while holding the inode locked. + */ + if (rb->real_mappings <= XFS_IFORK_MAXEXT(sc->ip, rb->whichfork)) { + ifake->if_fork->if_format = XFS_DINODE_FMT_EXTENTS; + error = xrep_bmap_extents_load(rb); + } else { + ifake->if_fork->if_format = XFS_DINODE_FMT_BTREE; + error = xrep_bmap_btree_load(rb, bmap_cur); + } + if (error) + goto err_cur; + + /* + * Install the new fork in the inode. After this point the old mapping + * data are no longer accessible and the new tree is live. We delete + * the cursor immediately after committing the staged root because the + * staged fork might be in extents format. + */ + xfs_bmbt_commit_staged_btree(bmap_cur, sc->tp, rb->whichfork); + xfs_btree_del_cursor(bmap_cur, 0); + + /* Reset the inode counters now that we've changed the fork. */ + error = xrep_bmap_reset_counters(rb); + if (error) + goto err_newbt; + + /* Dispose of any unused blocks and the accounting information. */ + error = xrep_newbt_commit(&rb->new_bmapbt); + if (error) + return error; + + return xrep_roll_trans(sc); + +err_cur: + if (bmap_cur) + xfs_btree_del_cursor(bmap_cur, error); +err_newbt: + xrep_newbt_cancel(&rb->new_bmapbt); + return error; +} + +/* + * Now that we've logged the new inode btree, invalidate all of the old blocks + * and free them, if there were any. + */ +STATIC int +xrep_bmap_remove_old_tree( + struct xrep_bmap *rb) +{ + struct xfs_scrub *sc = rb->sc; + struct xfs_owner_info oinfo; + + /* Free the old bmbt blocks if they're not in use. */ + xfs_rmap_inode_bmbt_owner(&oinfo, sc->ip, rb->whichfork); + return xrep_reap_fsblocks(sc, &rb->old_bmbt_blocks, &oinfo); +} + +/* Check for garbage inputs. Returns -ECANCELED if there's nothing to do. */ +STATIC int +xrep_bmap_check_inputs( + struct xfs_scrub *sc, + int whichfork) +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(sc->ip, whichfork); + + ASSERT(whichfork == XFS_DATA_FORK || whichfork == XFS_ATTR_FORK); + + if (!xfs_has_rmapbt(sc->mp)) + return -EOPNOTSUPP; + + /* No fork means nothing to rebuild. */ + if (!ifp) + return -ECANCELED; + + /* + * We only know how to repair extent mappings, which is to say that we + * only support extents and btree fork format. Repairs to a local + * format fork require a higher level repair function, so we do not + * have any work to do here. + */ + switch (ifp->if_format) { + case XFS_DINODE_FMT_DEV: + case XFS_DINODE_FMT_LOCAL: + case XFS_DINODE_FMT_UUID: + case XFS_DINODE_FMT_META_BTREE: + return -ECANCELED; + case XFS_DINODE_FMT_EXTENTS: + case XFS_DINODE_FMT_BTREE: + break; + default: + return -EFSCORRUPTED; + } + + if (whichfork == XFS_ATTR_FORK) + return 0; + + /* Only files, symlinks, and directories get to have data forks. */ + switch (VFS_I(sc->ip)->i_mode & S_IFMT) { + case S_IFREG: + case S_IFDIR: + case S_IFLNK: + /* ok */ + break; + default: + return -EINVAL; + } + + return 0; +} + +/* Set up the initial state of the reflink scan. */ +static inline enum reflink_scan_state +xrep_bmap_init_reflink_scan( + struct xfs_scrub *sc, + int whichfork) +{ + /* cannot share on non-reflink filesystem */ + if (!xfs_has_reflink(sc->mp)) + return RLS_IRRELEVANT; + + /* preserve flag if it's already set */ + if (xfs_is_reflink_inode(sc->ip)) + return RLS_SET_IFLAG; + + /* can only share regular files */ + if (!S_ISREG(VFS_I(sc->ip)->i_mode)) + return RLS_IRRELEVANT; + + /* cannot share attr fork extents */ + if (whichfork != XFS_DATA_FORK) + return RLS_IRRELEVANT; + + return RLS_UNKNOWN; +} + +/* Repair an inode fork. */ +int +xrep_bmap( + struct xfs_scrub *sc, + int whichfork, + bool allow_unwritten) +{ + struct xrep_bmap *rb; + xfs_extnum_t max_bmbt_recs; + bool large_extcount; + int error = 0; + + error = xrep_bmap_check_inputs(sc, whichfork); + if (error == -ECANCELED) + return 0; + if (error) + return error; + + rb = kzalloc_obj(struct xrep_bmap, XCHK_GFP_FLAGS); + if (!rb) + return -ENOMEM; + rb->sc = sc; + rb->whichfork = whichfork; + rb->reflink_scan = xrep_bmap_init_reflink_scan(sc, whichfork); + rb->allow_unwritten = allow_unwritten; + + /* Set up enough storage to handle the max records for this fork. */ + large_extcount = xfs_has_large_extent_counts(sc->mp); + max_bmbt_recs = xfs_iext_max_nextents(large_extcount, whichfork); + error = xfarray_create("fork mapping records", max_bmbt_recs, + sizeof(struct xfs_bmbt_rec), &rb->bmap_records); + if (error) + goto out_rb; + + /* Collect all reverse mappings for this fork's extents. */ + xfsb_bitmap_init(&rb->old_bmbt_blocks); + error = xrep_bmap_find_mappings(rb); + if (error) + goto out_bitmap; + + xfs_trans_ijoin(sc->tp, sc->ip, 0); + + /* Rebuild the bmap information. */ + error = xrep_bmap_build_new_fork(rb); + if (error) + goto out_bitmap; + + /* Kill the old tree. */ + error = xrep_bmap_remove_old_tree(rb); + if (error) + goto out_bitmap; + +out_bitmap: + xfsb_bitmap_destroy(&rb->old_bmbt_blocks); + xfarray_destroy(rb->bmap_records); +out_rb: + kfree(rb); + return error; +} + +/* Repair an inode's data fork. */ +int +xrep_bmap_data( + struct xfs_scrub *sc) +{ + return xrep_bmap(sc, XFS_DATA_FORK, true); +} + +/* Repair an inode's attr fork. */ +int +xrep_bmap_attr( + struct xfs_scrub *sc) +{ + return xrep_bmap(sc, XFS_ATTR_FORK, false); +} diff --git a/fs/xfs/scrub/btree.c b/fs/xfs/scrub/btree.c new file mode 100644 index 000000000..786b1e7c4 --- /dev/null +++ b/fs/xfs/scrub/btree.c @@ -0,0 +1,807 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_btree.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/btree.h" +#include "scrub/trace.h" + +/* btree scrubbing */ + +/* + * Check for btree operation errors. See the section about handling + * operational errors in common.c. + */ +static bool +__xchk_btree_process_error( + struct xfs_scrub *sc, + struct xfs_btree_cur *cur, + int level, + int *error, + __u32 errflag, + void *ret_ip) +{ + if (*error == 0) + return true; + + switch (*error) { + case -EDEADLOCK: + case -ECHRNG: + /* Used to restart an op with deadlock avoidance. */ + trace_xchk_deadlock_retry(sc->ip, sc->sm, *error); + break; + case -EFSBADCRC: + case -EFSCORRUPTED: + case -EIO: + case -ENODATA: + /* Note the badness but don't abort. */ + sc->sm->sm_flags |= errflag; + *error = 0; + fallthrough; + default: + if (cur->bc_ops->type == XFS_BTREE_TYPE_INODE) + trace_xchk_ifork_btree_op_error(sc, cur, level, + *error, ret_ip); + else + trace_xchk_btree_op_error(sc, cur, level, + *error, ret_ip); + break; + } + return false; +} + +bool +xchk_btree_process_error( + struct xfs_scrub *sc, + struct xfs_btree_cur *cur, + int level, + int *error) +{ + return __xchk_btree_process_error(sc, cur, level, error, + XFS_SCRUB_OFLAG_CORRUPT, __return_address); +} + +bool +xchk_btree_xref_process_error( + struct xfs_scrub *sc, + struct xfs_btree_cur *cur, + int level, + int *error) +{ + return __xchk_btree_process_error(sc, cur, level, error, + XFS_SCRUB_OFLAG_XFAIL, __return_address); +} + +/* Record btree block corruption. */ +static void +__xchk_btree_set_corrupt( + struct xfs_scrub *sc, + struct xfs_btree_cur *cur, + int level, + __u32 errflag, + void *ret_ip) +{ + sc->sm->sm_flags |= errflag; + + if (cur->bc_ops->type == XFS_BTREE_TYPE_INODE) + trace_xchk_ifork_btree_error(sc, cur, level, + ret_ip); + else + trace_xchk_btree_error(sc, cur, level, + ret_ip); +} + +void +xchk_btree_set_corrupt( + struct xfs_scrub *sc, + struct xfs_btree_cur *cur, + int level) +{ + __xchk_btree_set_corrupt(sc, cur, level, XFS_SCRUB_OFLAG_CORRUPT, + __return_address); +} + +void +xchk_btree_xref_set_corrupt( + struct xfs_scrub *sc, + struct xfs_btree_cur *cur, + int level) +{ + __xchk_btree_set_corrupt(sc, cur, level, XFS_SCRUB_OFLAG_XCORRUPT, + __return_address); +} + +void +xchk_btree_set_preen( + struct xfs_scrub *sc, + struct xfs_btree_cur *cur, + int level) +{ + __xchk_btree_set_corrupt(sc, cur, level, XFS_SCRUB_OFLAG_PREEN, + __return_address); +} + +/* + * Make sure this record is in order and doesn't stray outside of the parent + * keys. + */ +STATIC void +xchk_btree_rec( + struct xchk_btree *bs) +{ + struct xfs_btree_cur *cur = bs->cur; + union xfs_btree_rec *rec; + union xfs_btree_key key; + union xfs_btree_key hkey; + union xfs_btree_key *keyp; + struct xfs_btree_block *block; + struct xfs_btree_block *keyblock; + struct xfs_buf *bp; + + block = xfs_btree_get_block(cur, 0, &bp); + rec = xfs_btree_rec_addr(cur, cur->bc_levels[0].ptr, block); + + trace_xchk_btree_rec(bs->sc, cur, 0); + + /* Are all records across all record blocks in order? */ + if (bs->lastrec_valid && + !cur->bc_ops->recs_inorder(cur, &bs->lastrec, rec)) + xchk_btree_set_corrupt(bs->sc, cur, 0); + memcpy(&bs->lastrec, rec, cur->bc_ops->rec_len); + bs->lastrec_valid = true; + + if (cur->bc_nlevels == 1) + return; + + /* Is low_key(rec) at least as large as the parent low key? */ + cur->bc_ops->init_key_from_rec(&key, rec); + keyblock = xfs_btree_get_block(cur, 1, &bp); + keyp = xfs_btree_key_addr(cur, cur->bc_levels[1].ptr, keyblock); + if (xfs_btree_keycmp_lt(cur, &key, keyp)) + xchk_btree_set_corrupt(bs->sc, cur, 1); + + if (!(cur->bc_ops->geom_flags & XFS_BTGEO_OVERLAPPING)) + return; + + /* Is high_key(rec) no larger than the parent high key? */ + cur->bc_ops->init_high_key_from_rec(&hkey, rec); + keyp = xfs_btree_high_key_addr(cur, cur->bc_levels[1].ptr, keyblock); + if (xfs_btree_keycmp_lt(cur, keyp, &hkey)) + xchk_btree_set_corrupt(bs->sc, cur, 1); +} + +/* + * Make sure this key is in order and doesn't stray outside of the parent + * keys. + */ +STATIC void +xchk_btree_key( + struct xchk_btree *bs, + int level) +{ + struct xfs_btree_cur *cur = bs->cur; + union xfs_btree_key *key; + union xfs_btree_key *keyp; + struct xfs_btree_block *block; + struct xfs_btree_block *keyblock; + struct xfs_buf *bp; + + block = xfs_btree_get_block(cur, level, &bp); + key = xfs_btree_key_addr(cur, cur->bc_levels[level].ptr, block); + + trace_xchk_btree_key(bs->sc, cur, level); + + /* Are all low keys across all node blocks in order? */ + if (bs->lastkey[level - 1].valid && + !cur->bc_ops->keys_inorder(cur, &bs->lastkey[level - 1].key, key)) + xchk_btree_set_corrupt(bs->sc, cur, level); + memcpy(&bs->lastkey[level - 1].key, key, cur->bc_ops->key_len); + bs->lastkey[level - 1].valid = true; + + if (level + 1 >= cur->bc_nlevels) + return; + + /* Is this block's low key at least as large as the parent low key? */ + keyblock = xfs_btree_get_block(cur, level + 1, &bp); + keyp = xfs_btree_key_addr(cur, cur->bc_levels[level + 1].ptr, keyblock); + if (xfs_btree_keycmp_lt(cur, key, keyp)) + xchk_btree_set_corrupt(bs->sc, cur, level); + + if (!(cur->bc_ops->geom_flags & XFS_BTGEO_OVERLAPPING)) + return; + + /* Is this block's high key no larger than the parent high key? */ + key = xfs_btree_high_key_addr(cur, cur->bc_levels[level].ptr, block); + keyp = xfs_btree_high_key_addr(cur, cur->bc_levels[level + 1].ptr, + keyblock); + if (xfs_btree_keycmp_lt(cur, keyp, key)) + xchk_btree_set_corrupt(bs->sc, cur, level); +} + +/* + * Check a btree pointer. Returns true if it's ok to use this pointer. + * Callers do not need to set the corrupt flag. + */ +static bool +xchk_btree_ptr_ok( + struct xchk_btree *bs, + int level, + union xfs_btree_ptr *ptr) +{ + /* A btree rooted in an inode has no block pointer to the root. */ + if (bs->cur->bc_ops->type == XFS_BTREE_TYPE_INODE && + level == bs->cur->bc_nlevels) + return true; + + /* Otherwise, check the pointers. */ + if (__xfs_btree_check_ptr(bs->cur, ptr, 0, level)) { + xchk_btree_set_corrupt(bs->sc, bs->cur, level); + return false; + } + + return true; +} + +/* Check that a btree block's sibling matches what we expect it. */ +STATIC int +xchk_btree_block_check_sibling( + struct xchk_btree *bs, + int level, + int direction, + union xfs_btree_ptr *sibling) +{ + struct xfs_btree_cur *cur = bs->cur; + struct xfs_btree_block *pblock; + struct xfs_buf *pbp; + struct xfs_btree_cur *ncur = NULL; + union xfs_btree_ptr *pp; + int success; + int error; + + error = xfs_btree_dup_cursor(cur, &ncur); + if (!xchk_btree_process_error(bs->sc, cur, level + 1, &error) || + !ncur) + return error; + + /* + * If the pointer is null, we shouldn't be able to move the upper + * level pointer anywhere. + */ + if (xfs_btree_ptr_is_null(cur, sibling)) { + if (direction > 0) + error = xfs_btree_increment(ncur, level + 1, &success); + else + error = xfs_btree_decrement(ncur, level + 1, &success); + if (error == 0 && success) + xchk_btree_set_corrupt(bs->sc, cur, level); + error = 0; + goto out; + } + + /* Increment upper level pointer. */ + if (direction > 0) + error = xfs_btree_increment(ncur, level + 1, &success); + else + error = xfs_btree_decrement(ncur, level + 1, &success); + if (!xchk_btree_process_error(bs->sc, cur, level + 1, &error)) + goto out; + if (!success) { + xchk_btree_set_corrupt(bs->sc, cur, level + 1); + goto out; + } + + /* Compare upper level pointer to sibling pointer. */ + pblock = xfs_btree_get_block(ncur, level + 1, &pbp); + pp = xfs_btree_ptr_addr(ncur, ncur->bc_levels[level + 1].ptr, pblock); + if (!xchk_btree_ptr_ok(bs, level + 1, pp)) + goto out; + if (pbp) + xchk_buffer_recheck(bs->sc, pbp); + + if (xfs_btree_cmp_two_ptrs(cur, pp, sibling)) + xchk_btree_set_corrupt(bs->sc, cur, level); +out: + xfs_btree_del_cursor(ncur, XFS_BTREE_ERROR); + return error; +} + +/* Check the siblings of a btree block. */ +STATIC int +xchk_btree_block_check_siblings( + struct xchk_btree *bs, + struct xfs_btree_block *block) +{ + struct xfs_btree_cur *cur = bs->cur; + union xfs_btree_ptr leftsib; + union xfs_btree_ptr rightsib; + int level; + int error = 0; + + xfs_btree_get_sibling(cur, block, &leftsib, XFS_BB_LEFTSIB); + xfs_btree_get_sibling(cur, block, &rightsib, XFS_BB_RIGHTSIB); + level = xfs_btree_get_level(block); + + /* Root block should never have siblings. */ + if (level == cur->bc_nlevels - 1) { + if (!xfs_btree_ptr_is_null(cur, &leftsib) || + !xfs_btree_ptr_is_null(cur, &rightsib)) + xchk_btree_set_corrupt(bs->sc, cur, level); + goto out; + } + + /* + * Does the left & right sibling pointers match the adjacent + * parent level pointers? + * (These function absorbs error codes for us.) + */ + error = xchk_btree_block_check_sibling(bs, level, -1, &leftsib); + if (error) + return error; + error = xchk_btree_block_check_sibling(bs, level, 1, &rightsib); + if (error) + return error; +out: + return error; +} + +struct check_owner { + struct list_head list; + xfs_daddr_t daddr; + int level; +}; + +/* + * Make sure this btree block isn't in the free list and that there's + * an rmap record for it. + */ +STATIC int +xchk_btree_check_block_owner( + struct xchk_btree *bs, + int level, + xfs_daddr_t daddr) +{ + xfs_agnumber_t agno; + xfs_agblock_t agbno; + bool is_bnobt, is_rmapbt; + bool init_sa; + int error = 0; + + if (!bs->cur) + return 0; + + is_bnobt = xfs_btree_is_bno(bs->cur->bc_ops); + is_rmapbt = xfs_btree_is_rmap(bs->cur->bc_ops); + agno = xfs_daddr_to_agno(bs->cur->bc_mp, daddr); + agbno = xfs_daddr_to_agbno(bs->cur->bc_mp, daddr); + + /* + * If the btree being examined is not itself a per-AG btree, initialize + * sc->sa so that we can check for the presence of an ownership record + * in the rmap btree for the AG containing the block. + */ + init_sa = bs->cur->bc_ops->type != XFS_BTREE_TYPE_AG; + if (init_sa) { + error = xchk_ag_init_existing(bs->sc, agno, &bs->sc->sa); + if (!xchk_btree_xref_process_error(bs->sc, bs->cur, + level, &error)) + goto out_free; + } + + xchk_xref_is_used_space(bs->sc, agbno, 1); + /* + * The bnobt scrubber aliases bs->cur to bs->sc->sa.bno_cur, so we + * have to nullify it (to shut down further block owner checks) if + * self-xref encounters problems. + */ + if (!bs->sc->sa.bno_cur && is_bnobt) + bs->cur = NULL; + + xchk_xref_is_only_owned_by(bs->sc, agbno, 1, bs->oinfo); + if (!bs->sc->sa.rmap_cur && is_rmapbt) + bs->cur = NULL; + +out_free: + if (init_sa) + xchk_ag_free(bs->sc, &bs->sc->sa); + + return error; +} + +/* Check the owner of a btree block. */ +STATIC int +xchk_btree_check_owner( + struct xchk_btree *bs, + int level, + struct xfs_buf *bp) +{ + struct xfs_btree_cur *cur = bs->cur; + + /* + * In theory, xfs_btree_get_block should only give us a null buffer + * pointer for the root of a root-in-inode btree type, but we need + * to check defensively here in case the cursor state is also screwed + * up. + */ + if (bp == NULL) { + if (cur->bc_ops->type != XFS_BTREE_TYPE_INODE) + xchk_btree_set_corrupt(bs->sc, bs->cur, level); + return 0; + } + + /* + * We want to cross-reference each btree block with the bnobt + * and the rmapbt. We cannot cross-reference the bnobt or + * rmapbt while scanning the bnobt or rmapbt, respectively, + * because we cannot alter the cursor and we'd prefer not to + * duplicate cursors. Therefore, save the buffer daddr for + * later scanning. + */ + if (xfs_btree_is_bno(cur->bc_ops) || xfs_btree_is_rmap(cur->bc_ops)) { + struct check_owner *co; + + co = kmalloc_obj(struct check_owner, XCHK_GFP_FLAGS); + if (!co) + return -ENOMEM; + + INIT_LIST_HEAD(&co->list); + co->level = level; + co->daddr = xfs_buf_daddr(bp); + list_add_tail(&co->list, &bs->to_check); + return 0; + } + + return xchk_btree_check_block_owner(bs, level, xfs_buf_daddr(bp)); +} + +/* Decide if we want to check minrecs of a btree block in the inode root. */ +static inline bool +xchk_btree_check_iroot_minrecs( + struct xchk_btree *bs) +{ + /* + * xfs_bmap_add_attrfork_btree had an implementation bug wherein it + * would miscalculate the space required for the data fork bmbt root + * when adding an attr fork, and promote the iroot contents to an + * external block unnecessarily. This went unnoticed for many years + * until scrub found filesystems in this state. Inode rooted btrees are + * not supposed to have immediate child blocks that are small enough + * that the contents could fit in the inode root, but we can't fail + * existing filesystems, so instead we disable the check for data fork + * bmap btrees when there's an attr fork. + */ + if (xfs_btree_is_bmap(bs->cur->bc_ops) && + bs->cur->bc_ino.whichfork == XFS_DATA_FORK && + xfs_inode_has_attr_fork(bs->sc->ip)) + return false; + + return true; +} + +/* + * Check that this btree block has at least minrecs records or is one of the + * special blocks that don't require that. + */ +STATIC void +xchk_btree_check_minrecs( + struct xchk_btree *bs, + int level, + struct xfs_btree_block *block) +{ + struct xfs_btree_cur *cur = bs->cur; + unsigned int root_level = cur->bc_nlevels - 1; + unsigned int numrecs = be16_to_cpu(block->bb_numrecs); + + /* More records than minrecs means the block is ok. */ + if (numrecs >= cur->bc_ops->get_minrecs(cur, level)) + return; + + /* + * For btrees rooted in the inode, it's possible that the root block + * contents spilled into a regular ondisk block because there wasn't + * enough space in the inode root. The number of records in that + * child block might be less than the standard minrecs, but that's ok + * provided that there's only one direct child of the root. + */ + if (cur->bc_ops->type == XFS_BTREE_TYPE_INODE && + level == cur->bc_nlevels - 2) { + struct xfs_btree_block *root_block; + struct xfs_buf *root_bp; + int root_maxrecs; + + root_block = xfs_btree_get_block(cur, root_level, &root_bp); + root_maxrecs = cur->bc_ops->get_dmaxrecs(cur, root_level); + if (xchk_btree_check_iroot_minrecs(bs) && + (be16_to_cpu(root_block->bb_numrecs) != 1 || + numrecs <= root_maxrecs)) + xchk_btree_set_corrupt(bs->sc, cur, level); + return; + } + + /* + * Otherwise, only the root level is allowed to have fewer than minrecs + * records or keyptrs. + */ + if (level < root_level) + xchk_btree_set_corrupt(bs->sc, cur, level); +} + +/* + * If this btree block has a parent, make sure that the parent's keys capture + * the keyspace contained in this block. + */ +STATIC void +xchk_btree_block_check_keys( + struct xchk_btree *bs, + int level, + struct xfs_btree_block *block) +{ + union xfs_btree_key block_key; + union xfs_btree_key *block_high_key; + union xfs_btree_key *parent_low_key, *parent_high_key; + struct xfs_btree_cur *cur = bs->cur; + struct xfs_btree_block *parent_block; + struct xfs_buf *bp; + + if (level == cur->bc_nlevels - 1) + return; + + xfs_btree_get_keys(cur, block, &block_key); + + /* Make sure the low key of this block matches the parent. */ + parent_block = xfs_btree_get_block(cur, level + 1, &bp); + parent_low_key = xfs_btree_key_addr(cur, cur->bc_levels[level + 1].ptr, + parent_block); + if (xfs_btree_keycmp_ne(cur, &block_key, parent_low_key)) { + xchk_btree_set_corrupt(bs->sc, bs->cur, level); + return; + } + + if (!(cur->bc_ops->geom_flags & XFS_BTGEO_OVERLAPPING)) + return; + + /* Make sure the high key of this block matches the parent. */ + parent_high_key = xfs_btree_high_key_addr(cur, + cur->bc_levels[level + 1].ptr, parent_block); + block_high_key = xfs_btree_high_key_from_key(cur, &block_key); + if (xfs_btree_keycmp_ne(cur, block_high_key, parent_high_key)) + xchk_btree_set_corrupt(bs->sc, bs->cur, level); +} + +/* + * Grab and scrub a btree block given a btree pointer. Returns block + * and buffer pointers (if applicable) if they're ok to use. + */ +STATIC int +xchk_btree_get_block( + struct xchk_btree *bs, + int level, + union xfs_btree_ptr *pp, + struct xfs_btree_block **pblock, + struct xfs_buf **pbp) +{ + int error; + + *pblock = NULL; + *pbp = NULL; + + error = xfs_btree_lookup_get_block(bs->cur, level, pp, pblock); + if (!xchk_btree_process_error(bs->sc, bs->cur, level, &error) || + !*pblock) + return error; + + xfs_btree_get_block(bs->cur, level, pbp); + if (__xfs_btree_check_block(bs->cur, *pblock, level, *pbp)) { + xchk_btree_set_corrupt(bs->sc, bs->cur, level); + return 0; + } + if (*pbp) + xchk_buffer_recheck(bs->sc, *pbp); + + xchk_btree_check_minrecs(bs, level, *pblock); + + /* + * Check the block's owner; this function absorbs error codes + * for us. + */ + error = xchk_btree_check_owner(bs, level, *pbp); + if (error) + return error; + + /* + * Check the block's siblings; this function absorbs error codes + * for us. + */ + error = xchk_btree_block_check_siblings(bs, *pblock); + if (error) + return error; + + xchk_btree_block_check_keys(bs, level, *pblock); + return 0; +} + +/* + * Check that the low and high keys of this block match the keys stored + * in the parent block. + */ +STATIC void +xchk_btree_block_keys( + struct xchk_btree *bs, + int level, + struct xfs_btree_block *block) +{ + union xfs_btree_key block_keys; + struct xfs_btree_cur *cur = bs->cur; + union xfs_btree_key *high_bk; + union xfs_btree_key *parent_keys; + union xfs_btree_key *high_pk; + struct xfs_btree_block *parent_block; + struct xfs_buf *bp; + + if (level >= cur->bc_nlevels - 1) + return; + + /* Calculate the keys for this block. */ + xfs_btree_get_keys(cur, block, &block_keys); + + /* Obtain the parent's copy of the keys for this block. */ + parent_block = xfs_btree_get_block(cur, level + 1, &bp); + parent_keys = xfs_btree_key_addr(cur, cur->bc_levels[level + 1].ptr, + parent_block); + + if (xfs_btree_keycmp_ne(cur, &block_keys, parent_keys)) + xchk_btree_set_corrupt(bs->sc, cur, 1); + + if (!(cur->bc_ops->geom_flags & XFS_BTGEO_OVERLAPPING)) + return; + + /* Get high keys */ + high_bk = xfs_btree_high_key_from_key(cur, &block_keys); + high_pk = xfs_btree_high_key_addr(cur, cur->bc_levels[level + 1].ptr, + parent_block); + + if (xfs_btree_keycmp_ne(cur, high_bk, high_pk)) + xchk_btree_set_corrupt(bs->sc, cur, 1); +} + +/* + * Visit all nodes and leaves of a btree. Check that all pointers and + * records are in order, that the keys reflect the records, and use a callback + * so that the caller can verify individual records. + */ +int +xchk_btree( + struct xfs_scrub *sc, + struct xfs_btree_cur *cur, + xchk_btree_rec_fn scrub_fn, + const struct xfs_owner_info *oinfo, + void *private) +{ + union xfs_btree_ptr ptr; + struct xchk_btree *bs; + union xfs_btree_ptr *pp; + union xfs_btree_rec *recp; + struct xfs_btree_block *block; + struct xfs_buf *bp; + struct check_owner *co; + struct check_owner *n; + size_t cur_sz; + int level; + int error = 0; + + /* + * Allocate the btree scrub context from the heap, because this + * structure can get rather large. Don't let a caller feed us a + * totally absurd size. + */ + cur_sz = xchk_btree_sizeof(cur->bc_nlevels); + if (cur_sz > PAGE_SIZE) { + xchk_btree_set_corrupt(sc, cur, 0); + return 0; + } + bs = kzalloc(cur_sz, XCHK_GFP_FLAGS); + if (!bs) + return -ENOMEM; + bs->cur = cur; + bs->scrub_rec = scrub_fn; + bs->oinfo = oinfo; + bs->private = private; + bs->sc = sc; + + /* Initialize scrub state */ + INIT_LIST_HEAD(&bs->to_check); + + /* + * Load the root of the btree. The helper function absorbs + * error codes for us. + */ + level = cur->bc_nlevels - 1; + xfs_btree_init_ptr_from_cur(cur, &ptr); + if (!xchk_btree_ptr_ok(bs, cur->bc_nlevels, &ptr)) + goto out; + error = xchk_btree_get_block(bs, level, &ptr, &block, &bp); + if (error || !block) + goto out; + + cur->bc_levels[level].ptr = 1; + + while (level < cur->bc_nlevels) { + block = xfs_btree_get_block(cur, level, &bp); + + if (level == 0) { + /* End of leaf, pop back towards the root. */ + if (cur->bc_levels[level].ptr > + be16_to_cpu(block->bb_numrecs)) { + xchk_btree_block_keys(bs, level, block); + if (level < cur->bc_nlevels - 1) + cur->bc_levels[level + 1].ptr++; + level++; + continue; + } + + /* Records in order for scrub? */ + xchk_btree_rec(bs); + + /* Call out to the record checker. */ + recp = xfs_btree_rec_addr(cur, cur->bc_levels[0].ptr, + block); + error = bs->scrub_rec(bs, recp); + if (error) + break; + if (xchk_should_terminate(sc, &error) || + (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) + break; + + cur->bc_levels[level].ptr++; + continue; + } + + /* End of node, pop back towards the root. */ + if (cur->bc_levels[level].ptr > + be16_to_cpu(block->bb_numrecs)) { + xchk_btree_block_keys(bs, level, block); + if (level < cur->bc_nlevels - 1) + cur->bc_levels[level + 1].ptr++; + level++; + continue; + } + + /* Keys in order for scrub? */ + xchk_btree_key(bs, level); + + /* Drill another level deeper. */ + pp = xfs_btree_ptr_addr(cur, cur->bc_levels[level].ptr, block); + if (!xchk_btree_ptr_ok(bs, level, pp)) { + cur->bc_levels[level].ptr++; + continue; + } + level--; + error = xchk_btree_get_block(bs, level, pp, &block, &bp); + if (error || !block) + goto out; + + cur->bc_levels[level].ptr = 1; + } + +out: + /* Process deferred owner checks on btree blocks. */ + list_for_each_entry_safe(co, n, &bs->to_check, list) { + if (!error && bs->cur) + error = xchk_btree_check_block_owner(bs, co->level, + co->daddr); + list_del(&co->list); + kfree(co); + } + kfree(bs); + + return error; +} diff --git a/fs/xfs/scrub/btree.h b/fs/xfs/scrub/btree.h new file mode 100644 index 000000000..c32b5fad6 --- /dev/null +++ b/fs/xfs/scrub/btree.h @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_BTREE_H__ +#define __XFS_SCRUB_BTREE_H__ + +/* btree scrub */ + +/* Check for btree operation errors. */ +bool xchk_btree_process_error(struct xfs_scrub *sc, + struct xfs_btree_cur *cur, int level, int *error); + +/* Check for btree xref operation errors. */ +bool xchk_btree_xref_process_error(struct xfs_scrub *sc, + struct xfs_btree_cur *cur, int level, int *error); + +/* Check for btree corruption. */ +void xchk_btree_set_corrupt(struct xfs_scrub *sc, + struct xfs_btree_cur *cur, int level); +void xchk_btree_set_preen(struct xfs_scrub *sc, struct xfs_btree_cur *cur, + int level); + +/* Check for btree xref discrepancies. */ +void xchk_btree_xref_set_corrupt(struct xfs_scrub *sc, + struct xfs_btree_cur *cur, int level); + +struct xchk_btree; +typedef int (*xchk_btree_rec_fn)( + struct xchk_btree *bs, + const union xfs_btree_rec *rec); + +struct xchk_btree_key { + union xfs_btree_key key; + bool valid; +}; + +struct xchk_btree { + /* caller-provided scrub state */ + struct xfs_scrub *sc; + struct xfs_btree_cur *cur; + xchk_btree_rec_fn scrub_rec; + const struct xfs_owner_info *oinfo; + void *private; + + /* internal scrub state */ + bool lastrec_valid; + union xfs_btree_rec lastrec; + struct list_head to_check; + + /* this element must come last! */ + struct xchk_btree_key lastkey[]; +}; + +/* + * Calculate the size of a xchk_btree structure. There are nlevels-1 slots for + * keys because we track leaf records separately in lastrec. + */ +static inline size_t +xchk_btree_sizeof(unsigned int nlevels) +{ + return struct_size_t(struct xchk_btree, lastkey, nlevels - 1); +} + +int xchk_btree(struct xfs_scrub *sc, struct xfs_btree_cur *cur, + xchk_btree_rec_fn scrub_fn, const struct xfs_owner_info *oinfo, + void *private); + +#endif /* __XFS_SCRUB_BTREE_H__ */ diff --git a/fs/xfs/scrub/common.c b/fs/xfs/scrub/common.c new file mode 100644 index 000000000..e5233e31a --- /dev/null +++ b/fs/xfs/scrub/common.c @@ -0,0 +1,1756 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_btree.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_inode.h" +#include "xfs_icache.h" +#include "xfs_alloc.h" +#include "xfs_alloc_btree.h" +#include "xfs_ialloc.h" +#include "xfs_ialloc_btree.h" +#include "xfs_refcount_btree.h" +#include "xfs_rmap.h" +#include "xfs_rmap_btree.h" +#include "xfs_log.h" +#include "xfs_trans_priv.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_dir2_priv.h" +#include "xfs_dir2.h" +#include "xfs_attr.h" +#include "xfs_reflink.h" +#include "xfs_ag.h" +#include "xfs_error.h" +#include "xfs_quota.h" +#include "xfs_exchmaps.h" +#include "xfs_rtbitmap.h" +#include "xfs_rtgroup.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_bmap_util.h" +#include "xfs_rtrefcount_btree.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/health.h" +#include "scrub/tempfile.h" + +/* Common code for the metadata scrubbers. */ + +/* + * Handling operational errors. + * + * The *_process_error() family of functions are used to process error return + * codes from functions called as part of a scrub operation. + * + * If there's no error, we return true to tell the caller that it's ok + * to move on to the next check in its list. + * + * For non-verifier errors (e.g. ENOMEM) we return false to tell the + * caller that something bad happened, and we preserve *error so that + * the caller can return the *error up the stack to userspace. + * + * Verifier errors (EFSBADCRC/EFSCORRUPTED) are recorded by setting + * OFLAG_CORRUPT in sm_flags and the *error is cleared. In other words, + * we track verifier errors (and failed scrub checks) via OFLAG_CORRUPT, + * not via return codes. We return false to tell the caller that + * something bad happened. Since the error has been cleared, the caller + * will (presumably) return that zero and scrubbing will move on to + * whatever's next. + * + * ftrace can be used to record the precise metadata location and the + * approximate code location of the failed operation. + */ + +/* Check for operational errors. */ +static bool +__xchk_process_error( + struct xfs_scrub *sc, + xfs_agnumber_t agno, + xfs_agblock_t bno, + int *error, + __u32 errflag, + void *ret_ip) +{ + switch (*error) { + case 0: + return true; + case -EDEADLOCK: + case -ECHRNG: + /* Used to restart an op with deadlock avoidance. */ + trace_xchk_deadlock_retry( + sc->ip ? sc->ip : XFS_I(file_inode(sc->file)), + sc->sm, *error); + break; + case -ECANCELED: + /* + * ECANCELED here means that the caller set one of the scrub + * outcome flags (corrupt, xfail, xcorrupt) and wants to exit + * quickly. Set error to zero and do not continue. + */ + trace_xchk_op_error(sc, agno, bno, *error, ret_ip); + *error = 0; + break; + case -EFSBADCRC: + case -EFSCORRUPTED: + case -EIO: + case -ENODATA: + /* Note the badness but don't abort. */ + sc->sm->sm_flags |= errflag; + *error = 0; + fallthrough; + default: + trace_xchk_op_error(sc, agno, bno, *error, ret_ip); + break; + } + return false; +} + +bool +xchk_process_error( + struct xfs_scrub *sc, + xfs_agnumber_t agno, + xfs_agblock_t bno, + int *error) +{ + return __xchk_process_error(sc, agno, bno, error, + XFS_SCRUB_OFLAG_CORRUPT, __return_address); +} + +bool +xchk_process_rt_error( + struct xfs_scrub *sc, + xfs_rgnumber_t rgno, + xfs_rgblock_t rgbno, + int *error) +{ + return __xchk_process_error(sc, rgno, rgbno, error, + XFS_SCRUB_OFLAG_CORRUPT, __return_address); +} + +bool +xchk_xref_process_error( + struct xfs_scrub *sc, + xfs_agnumber_t agno, + xfs_agblock_t bno, + int *error) +{ + return __xchk_process_error(sc, agno, bno, error, + XFS_SCRUB_OFLAG_XFAIL, __return_address); +} + +/* Check for operational errors for a file offset. */ +static bool +__xchk_fblock_process_error( + struct xfs_scrub *sc, + int whichfork, + xfs_fileoff_t offset, + int *error, + __u32 errflag, + void *ret_ip) +{ + switch (*error) { + case 0: + return true; + case -EDEADLOCK: + case -ECHRNG: + /* Used to restart an op with deadlock avoidance. */ + trace_xchk_deadlock_retry(sc->ip, sc->sm, *error); + break; + case -ECANCELED: + /* + * ECANCELED here means that the caller set one of the scrub + * outcome flags (corrupt, xfail, xcorrupt) and wants to exit + * quickly. Set error to zero and do not continue. + */ + trace_xchk_file_op_error(sc, whichfork, offset, *error, + ret_ip); + *error = 0; + break; + case -EFSBADCRC: + case -EFSCORRUPTED: + case -EIO: + case -ENODATA: + /* Note the badness but don't abort. */ + sc->sm->sm_flags |= errflag; + *error = 0; + fallthrough; + default: + trace_xchk_file_op_error(sc, whichfork, offset, *error, + ret_ip); + break; + } + return false; +} + +bool +xchk_fblock_process_error( + struct xfs_scrub *sc, + int whichfork, + xfs_fileoff_t offset, + int *error) +{ + return __xchk_fblock_process_error(sc, whichfork, offset, error, + XFS_SCRUB_OFLAG_CORRUPT, __return_address); +} + +bool +xchk_fblock_xref_process_error( + struct xfs_scrub *sc, + int whichfork, + xfs_fileoff_t offset, + int *error) +{ + return __xchk_fblock_process_error(sc, whichfork, offset, error, + XFS_SCRUB_OFLAG_XFAIL, __return_address); +} + +/* + * Handling scrub corruption/optimization/warning checks. + * + * The *_set_{corrupt,preen,warning}() family of functions are used to + * record the presence of metadata that is incorrect (corrupt), could be + * optimized somehow (preen), or should be flagged for administrative + * review but is not incorrect (warn). + * + * ftrace can be used to record the precise metadata location and + * approximate code location of the failed check. + */ + +/* Record a block which could be optimized. */ +void +xchk_block_set_preen( + struct xfs_scrub *sc, + struct xfs_buf *bp) +{ + sc->sm->sm_flags |= XFS_SCRUB_OFLAG_PREEN; + trace_xchk_block_preen(sc, xfs_buf_daddr(bp), __return_address); +} + +/* + * Record an inode which could be optimized. The trace data will + * include the block given by bp if bp is given; otherwise it will use + * the block location of the inode record itself. + */ +void +xchk_ino_set_preen( + struct xfs_scrub *sc, + xfs_ino_t ino) +{ + sc->sm->sm_flags |= XFS_SCRUB_OFLAG_PREEN; + trace_xchk_ino_preen(sc, ino, __return_address); +} + +/* Record a block indexed by a file fork that could be optimized. */ +void +xchk_fblock_set_preen( + struct xfs_scrub *sc, + int whichfork, + xfs_fileoff_t offset) +{ + sc->sm->sm_flags |= XFS_SCRUB_OFLAG_PREEN; + trace_xchk_fblock_preen(sc, whichfork, offset, __return_address); +} + +/* Record something being wrong with the filesystem primary superblock. */ +void +xchk_set_corrupt( + struct xfs_scrub *sc) +{ + sc->sm->sm_flags |= XFS_SCRUB_OFLAG_CORRUPT; + trace_xchk_fs_error(sc, 0, __return_address); +} + +/* Record a corrupt block. */ +void +xchk_block_set_corrupt( + struct xfs_scrub *sc, + struct xfs_buf *bp) +{ + sc->sm->sm_flags |= XFS_SCRUB_OFLAG_CORRUPT; + trace_xchk_block_error(sc, xfs_buf_daddr(bp), __return_address); +} + +#ifdef CONFIG_XFS_QUOTA +/* Record a corrupt quota counter. */ +void +xchk_qcheck_set_corrupt( + struct xfs_scrub *sc, + unsigned int dqtype, + xfs_dqid_t id) +{ + sc->sm->sm_flags |= XFS_SCRUB_OFLAG_CORRUPT; + trace_xchk_qcheck_error(sc, dqtype, id, __return_address); +} +#endif + +/* Record a corruption while cross-referencing. */ +void +xchk_block_xref_set_corrupt( + struct xfs_scrub *sc, + struct xfs_buf *bp) +{ + sc->sm->sm_flags |= XFS_SCRUB_OFLAG_XCORRUPT; + trace_xchk_block_error(sc, xfs_buf_daddr(bp), __return_address); +} + +/* + * Record a corrupt inode. The trace data will include the block given + * by bp if bp is given; otherwise it will use the block location of the + * inode record itself. + */ +void +xchk_ino_set_corrupt( + struct xfs_scrub *sc, + xfs_ino_t ino) +{ + sc->sm->sm_flags |= XFS_SCRUB_OFLAG_CORRUPT; + trace_xchk_ino_error(sc, ino, __return_address); +} + +/* Record a corruption while cross-referencing with an inode. */ +void +xchk_ip_xref_set_corrupt( + struct xfs_scrub *sc, + struct xfs_inode *ip) +{ + sc->sm->sm_flags |= XFS_SCRUB_OFLAG_XCORRUPT; + trace_xchk_ino_error(sc, I_INO(ip), __return_address); +} + +/* Record corruption in a block indexed by a file fork. */ +void +xchk_fblock_set_corrupt( + struct xfs_scrub *sc, + int whichfork, + xfs_fileoff_t offset) +{ + sc->sm->sm_flags |= XFS_SCRUB_OFLAG_CORRUPT; + trace_xchk_fblock_error(sc, whichfork, offset, __return_address); +} + +/* Record a corruption while cross-referencing a fork block. */ +void +xchk_fblock_xref_set_corrupt( + struct xfs_scrub *sc, + int whichfork, + xfs_fileoff_t offset) +{ + sc->sm->sm_flags |= XFS_SCRUB_OFLAG_XCORRUPT; + trace_xchk_fblock_error(sc, whichfork, offset, __return_address); +} + +/* + * Warn about inodes that need administrative review but is not + * incorrect. + */ +void +xchk_ino_set_warning( + struct xfs_scrub *sc, + xfs_ino_t ino) +{ + sc->sm->sm_flags |= XFS_SCRUB_OFLAG_WARNING; + trace_xchk_ino_warning(sc, ino, __return_address); +} + +/* Warn about a block indexed by a file fork that needs review. */ +void +xchk_fblock_set_warning( + struct xfs_scrub *sc, + int whichfork, + xfs_fileoff_t offset) +{ + sc->sm->sm_flags |= XFS_SCRUB_OFLAG_WARNING; + trace_xchk_fblock_warning(sc, whichfork, offset, __return_address); +} + +/* Signal an incomplete scrub. */ +void +xchk_set_incomplete( + struct xfs_scrub *sc) +{ + sc->sm->sm_flags |= XFS_SCRUB_OFLAG_INCOMPLETE; + trace_xchk_incomplete(sc, __return_address); +} + +/* + * rmap scrubbing -- compute the number of blocks with a given owner, + * at least according to the reverse mapping data. + */ + +struct xchk_rmap_ownedby_info { + const struct xfs_owner_info *oinfo; + xfs_filblks_t *blocks; +}; + +STATIC int +xchk_count_rmap_ownedby_irec( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv) +{ + struct xchk_rmap_ownedby_info *sroi = priv; + bool irec_attr; + bool oinfo_attr; + + irec_attr = rec->rm_flags & XFS_RMAP_ATTR_FORK; + oinfo_attr = sroi->oinfo->oi_flags & XFS_OWNER_INFO_ATTR_FORK; + + if (rec->rm_owner != sroi->oinfo->oi_owner) + return 0; + + if (XFS_RMAP_NON_INODE_OWNER(rec->rm_owner) || irec_attr == oinfo_attr) + (*sroi->blocks) += rec->rm_blockcount; + + return 0; +} + +/* + * Calculate the number of blocks the rmap thinks are owned by something. + * The caller should pass us an rmapbt cursor. + */ +int +xchk_count_rmap_ownedby_ag( + struct xfs_scrub *sc, + struct xfs_btree_cur *cur, + const struct xfs_owner_info *oinfo, + xfs_filblks_t *blocks) +{ + struct xchk_rmap_ownedby_info sroi = { + .oinfo = oinfo, + .blocks = blocks, + }; + + *blocks = 0; + return xfs_rmap_query_all(cur, xchk_count_rmap_ownedby_irec, + &sroi); +} + +/* + * AG scrubbing + * + * These helpers facilitate locking an allocation group's header + * buffers, setting up cursors for all btrees that are present, and + * cleaning everything up once we're through. + */ + +/* Decide if we want to return an AG header read failure. */ +static inline bool +want_ag_read_header_failure( + struct xfs_scrub *sc, + unsigned int type) +{ + /* Return all AG header read failures when scanning btrees. */ + if (sc->sm->sm_type != XFS_SCRUB_TYPE_AGF && + sc->sm->sm_type != XFS_SCRUB_TYPE_AGFL && + sc->sm->sm_type != XFS_SCRUB_TYPE_AGI) + return true; + /* + * If we're scanning a given type of AG header, we only want to + * see read failures from that specific header. We'd like the + * other headers to cross-check them, but this isn't required. + */ + if (sc->sm->sm_type == type) + return true; + return false; +} + +/* + * Grab the AG header buffers for the attached perag structure. + * + * The headers should be released by xchk_ag_free, but as a fail safe we attach + * all the buffers we grab to the scrub transaction so they'll all be freed + * when we cancel it. + */ +static inline int +xchk_perag_read_headers( + struct xfs_scrub *sc, + struct xchk_ag *sa) +{ + int error; + + error = xfs_ialloc_read_agi(sa->pag, sc->tp, 0, &sa->agi_bp); + if (error && want_ag_read_header_failure(sc, XFS_SCRUB_TYPE_AGI)) + return error; + + error = xfs_alloc_read_agf(sa->pag, sc->tp, 0, &sa->agf_bp); + if (error && want_ag_read_header_failure(sc, XFS_SCRUB_TYPE_AGF)) + return error; + + return 0; +} + +/* + * Grab the AG headers for the attached perag structure and wait for pending + * intents to drain. + */ +int +xchk_perag_drain_and_lock( + struct xfs_scrub *sc) +{ + struct xchk_ag *sa = &sc->sa; + int error = 0; + + ASSERT(sa->pag != NULL); + ASSERT(sa->agi_bp == NULL); + ASSERT(sa->agf_bp == NULL); + + do { + if (xchk_should_terminate(sc, &error)) + return error; + + error = xchk_perag_read_headers(sc, sa); + if (error) + return error; + + /* + * If we've grabbed an inode for scrubbing then we assume that + * holding its ILOCK will suffice to coordinate with any intent + * chains involving this inode. + */ + if (sc->ip) + return 0; + + /* + * Decide if this AG is quiet enough for all metadata to be + * consistent with each other. XFS allows the AG header buffer + * locks to cycle across transaction rolls while processing + * chains of deferred ops, which means that there could be + * other threads in the middle of processing a chain of + * deferred ops. For regular operations we are careful about + * ordering operations to prevent collisions between threads + * (which is why we don't need a per-AG lock), but scrub and + * repair have to serialize against chained operations. + * + * We just locked all the AG headers buffers; now take a look + * to see if there are any intents in progress. If there are, + * drop the AG headers and wait for the intents to drain. + * Since we hold all the AG header locks for the duration of + * the scrub, this is the only time we have to sample the + * intents counter; any threads increasing it after this point + * can't possibly be in the middle of a chain of AG metadata + * updates. + * + * Obviously, this should be slanted against scrub and in favor + * of runtime threads. + */ + if (!xfs_group_intent_busy(pag_group(sa->pag))) + return 0; + + if (sa->agf_bp) { + xfs_trans_brelse(sc->tp, sa->agf_bp); + sa->agf_bp = NULL; + } + + if (sa->agi_bp) { + xfs_trans_brelse(sc->tp, sa->agi_bp); + sa->agi_bp = NULL; + } + + if (!(sc->flags & XCHK_FSGATES_DRAIN)) + return -ECHRNG; + error = xfs_group_intent_drain(pag_group(sa->pag)); + if (error == -ERESTARTSYS) + error = -EINTR; + } while (!error); + + return error; +} + +/* + * Grab the per-AG structure, grab all AG header buffers, and wait until there + * aren't any pending intents. Returns -ENOENT if we can't grab the perag + * structure. + */ +int +xchk_ag_read_headers( + struct xfs_scrub *sc, + xfs_agnumber_t agno, + struct xchk_ag *sa) +{ + struct xfs_mount *mp = sc->mp; + + ASSERT(!sa->pag); + sa->pag = xfs_perag_get(mp, agno); + if (!sa->pag) + return -ENOENT; + + return xchk_perag_drain_and_lock(sc); +} + +/* Release all the AG btree cursors. */ +void +xchk_ag_btcur_free( + struct xchk_ag *sa) +{ + if (sa->refc_cur) + xfs_btree_del_cursor(sa->refc_cur, XFS_BTREE_ERROR); + if (sa->rmap_cur) + xfs_btree_del_cursor(sa->rmap_cur, XFS_BTREE_ERROR); + if (sa->fino_cur) + xfs_btree_del_cursor(sa->fino_cur, XFS_BTREE_ERROR); + if (sa->ino_cur) + xfs_btree_del_cursor(sa->ino_cur, XFS_BTREE_ERROR); + if (sa->cnt_cur) + xfs_btree_del_cursor(sa->cnt_cur, XFS_BTREE_ERROR); + if (sa->bno_cur) + xfs_btree_del_cursor(sa->bno_cur, XFS_BTREE_ERROR); + + sa->refc_cur = NULL; + sa->rmap_cur = NULL; + sa->fino_cur = NULL; + sa->ino_cur = NULL; + sa->bno_cur = NULL; + sa->cnt_cur = NULL; +} + +/* Initialize all the btree cursors for an AG. */ +void +xchk_ag_btcur_init( + struct xfs_scrub *sc, + struct xchk_ag *sa) +{ + struct xfs_mount *mp = sc->mp; + + if (sa->agf_bp) { + /* Set up a bnobt cursor for cross-referencing. */ + sa->bno_cur = xfs_bnobt_init_cursor(mp, sc->tp, sa->agf_bp, + sa->pag); + xchk_ag_btree_del_cursor_if_sick(sc, &sa->bno_cur, + XFS_SCRUB_TYPE_BNOBT); + + /* Set up a cntbt cursor for cross-referencing. */ + sa->cnt_cur = xfs_cntbt_init_cursor(mp, sc->tp, sa->agf_bp, + sa->pag); + xchk_ag_btree_del_cursor_if_sick(sc, &sa->cnt_cur, + XFS_SCRUB_TYPE_CNTBT); + + /* Set up a rmapbt cursor for cross-referencing. */ + if (xfs_has_rmapbt(mp)) { + sa->rmap_cur = xfs_rmapbt_init_cursor(mp, sc->tp, + sa->agf_bp, sa->pag); + xchk_ag_btree_del_cursor_if_sick(sc, &sa->rmap_cur, + XFS_SCRUB_TYPE_RMAPBT); + } + + /* Set up a refcountbt cursor for cross-referencing. */ + if (xfs_has_reflink(mp)) { + sa->refc_cur = xfs_refcountbt_init_cursor(mp, sc->tp, + sa->agf_bp, sa->pag); + xchk_ag_btree_del_cursor_if_sick(sc, &sa->refc_cur, + XFS_SCRUB_TYPE_REFCNTBT); + } + } + + if (sa->agi_bp) { + /* Set up a inobt cursor for cross-referencing. */ + sa->ino_cur = xfs_inobt_init_cursor(sa->pag, sc->tp, + sa->agi_bp); + xchk_ag_btree_del_cursor_if_sick(sc, &sa->ino_cur, + XFS_SCRUB_TYPE_INOBT); + + /* Set up a finobt cursor for cross-referencing. */ + if (xfs_has_finobt(mp)) { + sa->fino_cur = xfs_finobt_init_cursor(sa->pag, sc->tp, + sa->agi_bp); + xchk_ag_btree_del_cursor_if_sick(sc, &sa->fino_cur, + XFS_SCRUB_TYPE_FINOBT); + } + } +} + +/* Release the AG header context and btree cursors. */ +void +xchk_ag_free( + struct xfs_scrub *sc, + struct xchk_ag *sa) +{ + xchk_ag_btcur_free(sa); + xrep_reset_perag_resv(sc); + if (sa->agf_bp) { + xfs_trans_brelse(sc->tp, sa->agf_bp); + sa->agf_bp = NULL; + } + if (sa->agi_bp) { + xfs_trans_brelse(sc->tp, sa->agi_bp); + sa->agi_bp = NULL; + } + if (sa->pag) { + xfs_perag_put(sa->pag); + sa->pag = NULL; + } +} + +/* + * For scrub, grab the perag structure, the AGI, and the AGF headers, in that + * order. Locking order requires us to get the AGI before the AGF. We use the + * transaction to avoid deadlocking on crosslinked metadata buffers; either the + * caller passes one in (bmap scrub) or we have to create a transaction + * ourselves. Returns ENOENT if the perag struct cannot be grabbed. + */ +int +xchk_ag_init( + struct xfs_scrub *sc, + xfs_agnumber_t agno, + struct xchk_ag *sa) +{ + int error; + + error = xchk_ag_read_headers(sc, agno, sa); + if (error) + return error; + + xchk_ag_btcur_init(sc, sa); + return 0; +} + +#ifdef CONFIG_XFS_RT +/* + * For scrubbing a realtime group, grab all the in-core resources we'll need to + * check the metadata, which means taking the ILOCK of the realtime group's + * metadata inodes. Callers must not join these inodes to the transaction with + * non-zero lockflags or concurrency problems will result. The @rtglock_flags + * argument takes XFS_RTGLOCK_* flags. + */ +int +xchk_rtgroup_init( + struct xfs_scrub *sc, + xfs_rgnumber_t rgno, + struct xchk_rt *sr) +{ + ASSERT(sr->rtg == NULL); + ASSERT(sr->rtlock_flags == 0); + + sr->rtg = xfs_rtgroup_get(sc->mp, rgno); + if (!sr->rtg) + return -ENOENT; + return 0; +} + +/* Lock all the rt group metadata inode ILOCKs and wait for intents. */ +int +xchk_rtgroup_lock( + struct xfs_scrub *sc, + struct xchk_rt *sr, + unsigned int rtglock_flags) +{ + int error = 0; + + ASSERT(sr->rtg != NULL); + + /* + * If we're /only/ locking the rtbitmap in shared mode, then we're + * obviously not trying to compare records in two metadata inodes. + * There's no need to drain intents here because the caller (most + * likely the rgsuper scanner) doesn't need that level of consistency. + */ + if (rtglock_flags == XFS_RTGLOCK_BITMAP_SHARED) { + xfs_rtgroup_lock(sr->rtg, rtglock_flags); + sr->rtlock_flags = rtglock_flags; + return 0; + } + + do { + if (xchk_should_terminate(sc, &error)) + return error; + + xfs_rtgroup_lock(sr->rtg, rtglock_flags); + + /* + * If we've grabbed a non-metadata file for scrubbing, we + * assume that holding its ILOCK will suffice to coordinate + * with any rt intent chains involving this inode. + */ + if (sc->ip && !xfs_is_internal_inode(sc->ip)) + break; + + /* + * Decide if the rt group is quiet enough for all metadata to + * be consistent with each other. Regular file IO doesn't get + * to lock all the rt inodes at the same time, which means that + * there could be other threads in the middle of processing a + * chain of deferred ops. + * + * We just locked all the metadata inodes for this rt group; + * now take a look to see if there are any intents in progress. + * If there are, drop the rt group inode locks and wait for the + * intents to drain. Since we hold the rt group inode locks + * for the duration of the scrub, this is the only time we have + * to sample the intents counter; any threads increasing it + * after this point can't possibly be in the middle of a chain + * of rt metadata updates. + * + * Obviously, this should be slanted against scrub and in favor + * of runtime threads. + */ + if (!xfs_group_intent_busy(rtg_group(sr->rtg))) + break; + + xfs_rtgroup_unlock(sr->rtg, rtglock_flags); + + if (!(sc->flags & XCHK_FSGATES_DRAIN)) + return -ECHRNG; + error = xfs_group_intent_drain(rtg_group(sr->rtg)); + if (error) { + if (error == -ERESTARTSYS) + error = -EINTR; + return error; + } + } while (1); + + sr->rtlock_flags = rtglock_flags; + + if (xfs_has_rtrmapbt(sc->mp) && (rtglock_flags & XFS_RTGLOCK_RMAP)) + sr->rmap_cur = xfs_rtrmapbt_init_cursor(sc->tp, sr->rtg); + + if (xfs_has_rtreflink(sc->mp) && (rtglock_flags & XFS_RTGLOCK_REFCOUNT)) + sr->refc_cur = xfs_rtrefcountbt_init_cursor(sc->tp, sr->rtg); + + return 0; +} + +/* + * Free all the btree cursors and other incore data relating to the realtime + * group. This has to be done /before/ committing (or cancelling) the scrub + * transaction. + */ +void +xchk_rtgroup_btcur_free( + struct xchk_rt *sr) +{ + if (sr->rmap_cur) + xfs_btree_del_cursor(sr->rmap_cur, XFS_BTREE_ERROR); + if (sr->refc_cur) + xfs_btree_del_cursor(sr->refc_cur, XFS_BTREE_ERROR); + + sr->refc_cur = NULL; + sr->rmap_cur = NULL; +} + +/* + * Unlock the realtime group. This must be done /after/ committing (or + * cancelling) the scrub transaction. + */ +void +xchk_rtgroup_unlock( + struct xchk_rt *sr) +{ + ASSERT(sr->rtg != NULL); + + if (sr->rtlock_flags) { + xfs_rtgroup_unlock(sr->rtg, sr->rtlock_flags); + sr->rtlock_flags = 0; + } +} + +/* + * Unlock the realtime group and release its resources. This must be done + * /after/ committing (or cancelling) the scrub transaction. + */ +void +xchk_rtgroup_free( + struct xfs_scrub *sc, + struct xchk_rt *sr) +{ + ASSERT(sr->rtg != NULL); + + xchk_rtgroup_unlock(sr); + + xfs_rtgroup_put(sr->rtg); + sr->rtg = NULL; +} +#endif /* CONFIG_XFS_RT */ + +/* Per-scrubber setup functions */ + +void +xchk_trans_cancel( + struct xfs_scrub *sc) +{ + xfs_trans_cancel(sc->tp); + sc->tp = NULL; +} + +void +xchk_trans_alloc_empty( + struct xfs_scrub *sc) +{ + sc->tp = xfs_trans_alloc_empty(sc->mp); +} + +/* + * Grab an empty transaction so that we can re-grab locked buffers if + * one of our btrees turns out to be cyclic. + * + * If we're going to repair something, we need to ask for the largest possible + * log reservation so that we can handle the worst case scenario for metadata + * updates while rebuilding a metadata item. We also need to reserve as many + * blocks in the head transaction as we think we're going to need to rebuild + * the metadata object. + */ +int +xchk_trans_alloc( + struct xfs_scrub *sc, + uint resblks) +{ + if (sc->sm->sm_flags & XFS_SCRUB_IFLAG_REPAIR) + return xfs_trans_alloc(sc->mp, &M_RES(sc->mp)->tr_itruncate, + resblks, 0, 0, &sc->tp); + + xchk_trans_alloc_empty(sc); + return 0; +} + +/* Set us up with a transaction and an empty context. */ +int +xchk_setup_fs( + struct xfs_scrub *sc) +{ + uint resblks; + + resblks = xrep_calc_ag_resblks(sc); + return xchk_trans_alloc(sc, resblks); +} + +/* Set us up with a transaction and an empty context to repair rt metadata. */ +int +xchk_setup_rt( + struct xfs_scrub *sc) +{ + return xchk_trans_alloc(sc, xrep_calc_rtgroup_resblks(sc)); +} + +/* Set us up with AG headers and btree cursors. */ +int +xchk_setup_ag_btree( + struct xfs_scrub *sc, + bool force_log) +{ + struct xfs_mount *mp = sc->mp; + int error; + + /* + * If the caller asks us to checkpont the log, do so. This + * expensive operation should be performed infrequently and only + * as a last resort. Any caller that sets force_log should + * document why they need to do so. + */ + if (force_log) { + error = xchk_checkpoint_log(mp); + if (error) + return error; + } + + error = xchk_setup_fs(sc); + if (error) + return error; + + return xchk_ag_init(sc, sc->sm->sm_agno, &sc->sa); +} + +/* Push everything out of the log onto disk. */ +int +xchk_checkpoint_log( + struct xfs_mount *mp) +{ + int error; + + error = xfs_log_force(mp, XFS_LOG_SYNC); + if (error) + return error; + xfs_ail_push_all_sync(mp->m_ail); + return 0; +} + +/* Verify that an inode is allocated ondisk, then return its cached inode. */ +int +xchk_iget( + struct xfs_scrub *sc, + xfs_ino_t inum, + struct xfs_inode **ipp) +{ + ASSERT(sc->tp != NULL); + + return xfs_iget(sc->mp, sc->tp, inum, XCHK_IGET_FLAGS, 0, ipp); +} + +/* + * Try to grab an inode in a manner that avoids races with physical inode + * allocation. If we can't, return the locked AGI buffer so that the caller + * can single-step the loading process to see where things went wrong. + * Callers must have a valid scrub transaction. + * + * If the iget succeeds, return 0, a NULL AGI, and the inode. + * + * If the iget fails, return the error, the locked AGI, and a NULL inode. This + * can include -EINVAL and -ENOENT for invalid inode numbers or inodes that are + * no longer allocated; or any other corruption or runtime error. + * + * If the AGI read fails, return the error, a NULL AGI, and NULL inode. + * + * If a fatal signal is pending, return -EINTR, a NULL AGI, and a NULL inode. + */ +int +xchk_iget_agi( + struct xfs_scrub *sc, + xfs_ino_t inum, + struct xfs_buf **agi_bpp, + struct xfs_inode **ipp) +{ + struct xfs_mount *mp = sc->mp; + struct xfs_trans *tp = sc->tp; + struct xfs_perag *pag; + int error; + + ASSERT(sc->tp != NULL); + +again: + *agi_bpp = NULL; + *ipp = NULL; + error = 0; + + if (xchk_should_terminate(sc, &error)) + return error; + + /* + * Attach the AGI buffer to the scrub transaction to avoid deadlocks + * in the iget cache miss path. + */ + pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, inum)); + error = xfs_ialloc_read_agi(pag, tp, 0, agi_bpp); + xfs_perag_put(pag); + if (error) + return error; + + error = xfs_iget(mp, tp, inum, XFS_IGET_NORETRY | XCHK_IGET_FLAGS, 0, + ipp); + if (error == -EAGAIN) { + /* + * The inode may be in core but temporarily unavailable and may + * require the AGI buffer before it can be returned. Drop the + * AGI buffer and retry the lookup. + * + * Incore lookup will fail with EAGAIN on a cache hit if the + * inode is queued to the inactivation list. The inactivation + * worker may remove the inode from the unlinked list and hence + * needs the AGI. + * + * Hence xchk_iget_agi() needs to drop the AGI lock on EAGAIN + * to allow inodegc to make progress and move the inode to + * IRECLAIMABLE state where xfs_iget will be able to return it + * again if it can lock the inode. + */ + xfs_trans_brelse(tp, *agi_bpp); + delay(1); + goto again; + } + if (error) + return error; + + /* We got the inode, so we can release the AGI. */ + ASSERT(*ipp != NULL); + xfs_trans_brelse(tp, *agi_bpp); + *agi_bpp = NULL; + return 0; +} + +#ifdef CONFIG_XFS_QUOTA +/* + * Try to attach dquots to this inode if we think we might want to repair it. + * Callers must not hold any ILOCKs. If the dquots are broken and cannot be + * attached, a quotacheck will be scheduled. + */ +int +xchk_ino_dqattach( + struct xfs_scrub *sc) +{ + ASSERT(sc->tp != NULL); + ASSERT(sc->ip != NULL); + + if (!xchk_could_repair(sc)) + return 0; + + return xrep_ino_dqattach(sc); +} +#endif + +/* Install an inode that we opened by handle for scrubbing. */ +int +xchk_install_handle_inode( + struct xfs_scrub *sc, + struct xfs_inode *ip) +{ + if (VFS_I(ip)->i_generation != sc->sm->sm_gen) { + xchk_irele(sc, ip); + return -ENOENT; + } + + sc->ip = ip; + return 0; +} + +/* + * Install an already-referenced inode for scrubbing. Get our own reference to + * the inode to make disposal simpler. The inode must not be in I_FREEING or + * I_WILL_FREE state! + */ +int +xchk_install_live_inode( + struct xfs_scrub *sc, + struct xfs_inode *ip) +{ + if (!igrab(VFS_I(ip))) { + xchk_ip_set_corrupt(sc, ip); + return -EFSCORRUPTED; + } + + sc->ip = ip; + return 0; +} + +/* + * In preparation to scrub metadata structures that hang off of an inode, + * grab either the inode referenced in the scrub control structure or the + * inode passed in. If the inumber does not reference an allocated inode + * record, the function returns ENOENT to end the scrub early. The inode + * is not locked. + */ +int +xchk_iget_for_scrubbing( + struct xfs_scrub *sc) +{ + struct xfs_imap imap; + struct xfs_mount *mp = sc->mp; + struct xfs_perag *pag; + struct xfs_buf *agi_bp; + struct xfs_inode *ip_in = XFS_I(file_inode(sc->file)); + struct xfs_inode *ip = NULL; + xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, sc->sm->sm_ino); + int error; + + ASSERT(sc->tp == NULL); + + /* We want to scan the inode we already had opened. */ + if (sc->sm->sm_ino == 0 || sc->sm->sm_ino == I_INO(ip_in)) + return xchk_install_live_inode(sc, ip_in); + + /* + * On pre-metadir filesystems, reject internal metadata files. For + * metadir filesystems, limited scrubbing of any file in the metadata + * directory tree by handle is allowed, because that is the only way to + * validate the lack of parent pointers in the sb-root metadata inodes. + */ + if (!xfs_has_metadir(mp) && xfs_is_sb_inum(mp, sc->sm->sm_ino)) + return -ENOENT; + /* Reject obviously bad inode numbers. */ + if (!xfs_verify_ino(sc->mp, sc->sm->sm_ino)) + return -ENOENT; + + /* Try a safe untrusted iget. */ + error = xchk_iget_safe(sc, sc->sm->sm_ino, &ip); + if (!error) + return xchk_install_handle_inode(sc, ip); + if (error == -ENOENT) + return error; + if (error != -EINVAL) + goto out_error; + + /* + * EINVAL with IGET_UNTRUSTED probably means one of several things: + * userspace gave us an inode number that doesn't correspond to fs + * space; the inode btree lacks a record for this inode; or there is a + * record, and it says this inode is free. + * + * We want to look up this inode in the inobt to distinguish two + * scenarios: (1) the inobt says the inode is free, in which case + * there's nothing to do; and (2) the inobt says the inode is + * allocated, but loading it failed due to corruption. + * + * Allocate a transaction and grab the AGI to prevent inobt activity + * in this AG. Retry the iget in case someone allocated a new inode + * after the first iget failed. + */ + error = xchk_trans_alloc(sc, 0); + if (error) + goto out_error; + + error = xchk_iget_agi(sc, sc->sm->sm_ino, &agi_bp, &ip); + if (error == 0) { + /* Actually got the inode, so install it. */ + xchk_trans_cancel(sc); + return xchk_install_handle_inode(sc, ip); + } + if (error == -ENOENT) + goto out_gone; + if (error != -EINVAL) + goto out_cancel; + + /* Ensure that we have protected against inode allocation/freeing. */ + if (agi_bp == NULL) { + ASSERT(agi_bp != NULL); + error = -ECANCELED; + goto out_cancel; + } + + /* + * Untrusted iget failed a second time. Let's try an inobt lookup. + * If the inobt thinks this the inode neither can exist inside the + * filesystem nor is allocated, return ENOENT to signal that the check + * can be skipped. + * + * If the lookup returns corruption, we'll mark this inode corrupt and + * exit to userspace. There's little chance of fixing anything until + * the inobt is straightened out, but there's nothing we can do here. + * + * If the lookup encounters any other error, exit to userspace. + * + * If the lookup succeeds, something else must be very wrong in the fs + * such that setting up the incore inode failed in some strange way. + * Treat those as corruptions. + */ + pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, sc->sm->sm_ino)); + if (!pag) { + error = -EFSCORRUPTED; + goto out_cancel; + } + + error = xfs_imap(pag, sc->tp, sc->sm->sm_ino, &imap, + XFS_IGET_UNTRUSTED); + xfs_perag_put(pag); + if (error == -EINVAL || error == -ENOENT) + goto out_gone; + if (!error) + error = -EFSCORRUPTED; + +out_cancel: + xchk_trans_cancel(sc); +out_error: + trace_xchk_op_error(sc, agno, XFS_INO_TO_AGBNO(mp, sc->sm->sm_ino), + error, __return_address); + return error; +out_gone: + /* The file is gone, so there's nothing to check. */ + xchk_trans_cancel(sc); + return -ENOENT; +} + +/* Release an inode, possibly dropping it in the process. */ +void +xchk_irele( + struct xfs_scrub *sc, + struct xfs_inode *ip) +{ + if (sc->tp) { + /* + * If we are in a transaction, we /cannot/ drop the inode + * ourselves, because the VFS will trigger writeback, which + * can require a transaction. Clear DONTCACHE to force the + * inode to the LRU, where someone else can take care of + * dropping it. + * + * Note that when we grabbed our reference to the inode, it + * could have had an active ref and DONTCACHE set if a sysadmin + * is trying to coerce a change in file access mode. icache + * hits do not clear DONTCACHE, so we must do it here. + */ + spin_lock(&VFS_I(ip)->i_lock); + inode_state_clear(VFS_I(ip), I_DONTCACHE); + spin_unlock(&VFS_I(ip)->i_lock); + } + + xfs_irele(ip); +} + +/* + * Set us up to scrub metadata mapped by a file's fork. Callers must not use + * this to operate on user-accessible regular file data because the MMAPLOCK is + * not taken. + */ +int +xchk_setup_inode_contents( + struct xfs_scrub *sc, + unsigned int resblks) +{ + int error; + + error = xchk_iget_for_scrubbing(sc); + if (error) + return error; + + error = xrep_tempfile_adjust_directory_tree(sc); + if (error) + return error; + + /* Lock the inode so the VFS cannot touch this file. */ + xchk_ilock(sc, XFS_IOLOCK_EXCL); + + error = xchk_trans_alloc(sc, resblks); + if (error) + goto out; + + error = xchk_ino_dqattach(sc); + if (error) + goto out; + + xchk_ilock(sc, XFS_ILOCK_EXCL); +out: + /* scrub teardown will unlock and release the inode for us */ + return error; +} + +void +xchk_ilock( + struct xfs_scrub *sc, + unsigned int ilock_flags) +{ + xfs_ilock(sc->ip, ilock_flags); + sc->ilock_flags |= ilock_flags; +} + +bool +xchk_ilock_nowait( + struct xfs_scrub *sc, + unsigned int ilock_flags) +{ + if (xfs_ilock_nowait(sc->ip, ilock_flags)) { + sc->ilock_flags |= ilock_flags; + return true; + } + + return false; +} + +void +xchk_iunlock( + struct xfs_scrub *sc, + unsigned int ilock_flags) +{ + sc->ilock_flags &= ~ilock_flags; + xfs_iunlock(sc->ip, ilock_flags); +} + +/* + * Predicate that decides if we need to evaluate the cross-reference check. + * If there was an error accessing the cross-reference btree, just delete + * the cursor and skip the check. + */ +bool +xchk_should_check_xref( + struct xfs_scrub *sc, + int *error, + struct xfs_btree_cur **curpp) +{ + /* No point in xref if we already know we're corrupt. */ + if (xchk_skip_xref(sc->sm)) + return false; + + if (*error == 0) + return true; + + if (curpp) { + /* If we've already given up on xref, just bail out. */ + if (!*curpp) + return false; + + /* xref error, delete cursor and bail out. */ + xfs_btree_del_cursor(*curpp, XFS_BTREE_ERROR); + *curpp = NULL; + } + + sc->sm->sm_flags |= XFS_SCRUB_OFLAG_XFAIL; + trace_xchk_xref_error(sc, *error, __return_address); + + /* + * Errors encountered during cross-referencing with another + * data structure should not cause this scrubber to abort. + */ + *error = 0; + return false; +} + +/* Run the structure verifiers on in-memory buffers to detect bad memory. */ +void +xchk_buffer_recheck( + struct xfs_scrub *sc, + struct xfs_buf *bp) +{ + xfs_failaddr_t fa; + + if (bp->b_ops == NULL) { + xchk_block_set_corrupt(sc, bp); + return; + } + if (bp->b_ops->verify_struct == NULL) { + xchk_set_incomplete(sc); + return; + } + fa = bp->b_ops->verify_struct(bp); + if (!fa) + return; + sc->sm->sm_flags |= XFS_SCRUB_OFLAG_CORRUPT; + trace_xchk_block_error(sc, xfs_buf_daddr(bp), fa); +} + +static inline int +xchk_metadata_inode_subtype( + struct xfs_scrub *sc, + unsigned int scrub_type) +{ + struct xfs_scrub_subord *sub; + int error; + + sub = xchk_scrub_create_subord(sc, scrub_type); + if (!sub) + return -ENOMEM; + + error = sub->sc.ops->scrub(&sub->sc); + xchk_scrub_free_subord(sub); + return error; +} + +/* + * Scrub the attr/data forks of a metadata inode. The metadata inode must be + * pointed to by sc->ip and the ILOCK must be held. + */ +int +xchk_metadata_inode_forks( + struct xfs_scrub *sc) +{ + bool shared; + int error; + + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return 0; + + /* Check the inode record. */ + error = xchk_metadata_inode_subtype(sc, XFS_SCRUB_TYPE_INODE); + if (error || (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) + return error; + + /* Metadata inodes don't live on the rt device. */ + if (sc->ip->i_diflags & XFS_DIFLAG_REALTIME) { + xchk_ip_set_corrupt(sc, sc->ip); + return 0; + } + + /* They should never participate in reflink. */ + if (xfs_is_reflink_inode(sc->ip)) { + xchk_ip_set_corrupt(sc, sc->ip); + return 0; + } + + /* Invoke the data fork scrubber. */ + error = xchk_metadata_inode_subtype(sc, XFS_SCRUB_TYPE_BMBTD); + if (error || (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) + return error; + + /* Look for incorrect shared blocks. */ + if (xfs_has_reflink(sc->mp)) { + error = xfs_reflink_inode_has_shared_extents(sc->tp, sc->ip, + &shared); + if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, 0, + &error)) + return error; + if (shared) + xchk_ip_set_corrupt(sc, sc->ip); + } + + /* + * Metadata files can only have extended attributes on metadir + * filesystems, either for parent pointers or for actual xattr data. + */ + if (xfs_inode_hasattr(sc->ip)) { + if (!xfs_has_metadir(sc->mp)) { + xchk_ip_set_corrupt(sc, sc->ip); + return 0; + } + + error = xchk_metadata_inode_subtype(sc, XFS_SCRUB_TYPE_BMBTA); + if (error || (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) + return error; + } + + return 0; +} + +/* + * Enable filesystem hooks (i.e. runtime code patching) before starting a scrub + * operation. Callers must not hold any locks that intersect with the CPU + * hotplug lock (e.g. writeback locks) because code patching must halt the CPUs + * to change kernel code. + */ +void +xchk_fsgates_enable( + struct xfs_scrub *sc, + unsigned int scrub_fsgates) +{ + ASSERT(!(scrub_fsgates & ~XCHK_FSGATES_ALL)); + ASSERT(!(sc->flags & scrub_fsgates)); + + trace_xchk_fsgates_enable(sc, scrub_fsgates); + + if (scrub_fsgates & XCHK_FSGATES_DRAIN) + xfs_defer_drain_wait_enable(); + + if (scrub_fsgates & XCHK_FSGATES_QUOTA) + xfs_dqtrx_hook_enable(); + + if (scrub_fsgates & XCHK_FSGATES_DIRENTS) + xfs_dir_hook_enable(); + + if (scrub_fsgates & XCHK_FSGATES_RMAP) + xfs_rmap_hook_enable(); + + sc->flags |= scrub_fsgates; +} + +/* + * Decide if this is this a cached inode that's also allocated. The caller + * must hold a reference to an AG and the AGI buffer lock to prevent inodes + * from being allocated or freed. + * + * Look up an inode by number in the given file system. If the inode number + * is invalid, return -EINVAL. If the inode is not in cache, return -ENODATA. + * If the inode is being reclaimed, return -ENODATA because we know the inode + * cache cannot be updating the ondisk metadata. + * + * Otherwise, the incore inode is the one we want, and it is either live, + * somewhere in the inactivation machinery, or reclaimable. The inode is + * allocated if i_mode is nonzero. In all three cases, the cached inode will + * be more up to date than the ondisk inode buffer, so we must use the incore + * i_mode. + */ +int +xchk_inode_is_allocated( + struct xfs_scrub *sc, + xfs_agino_t agino, + bool *inuse) +{ + struct xfs_mount *mp = sc->mp; + struct xfs_perag *pag = sc->sa.pag; + xfs_ino_t ino; + struct xfs_inode *ip; + int error; + + /* caller must hold perag reference */ + if (pag == NULL) { + ASSERT(pag != NULL); + return -EINVAL; + } + + /* caller must have AGI buffer */ + if (sc->sa.agi_bp == NULL) { + ASSERT(sc->sa.agi_bp != NULL); + return -EINVAL; + } + + /* reject inode numbers outside existing AGs */ + ino = xfs_agino_to_ino(pag, agino); + if (!xfs_verify_ino(mp, ino)) + return -EINVAL; + + error = -ENODATA; + rcu_read_lock(); + ip = radix_tree_lookup(&pag->pag_ici_root, agino); + if (!ip) { + /* cache miss */ + goto out_rcu; + } + + /* + * If the inode number doesn't match, the incore inode got reused + * during an RCU grace period and the radix tree hasn't been updated. + * This isn't the inode we want. + */ + spin_lock(&ip->i_flags_lock); + if (I_INO(ip) != ino) + goto out_skip; + + trace_xchk_inode_is_allocated(ip); + + /* + * We have an incore inode that matches the inode we want, and the + * caller holds the perag structure and the AGI buffer. Let's check + * our assumptions below: + */ + +#ifdef DEBUG + /* + * (1) If the incore inode is live (i.e. referenced from the dcache), + * it will not be INEW, nor will it be in the inactivation or reclaim + * machinery. The ondisk inode had better be allocated. This is the + * most trivial case. + */ + if (!(ip->i_flags & (XFS_NEED_INACTIVE | XFS_INEW | XFS_IRECLAIMABLE | + XFS_INACTIVATING))) { + /* live inode */ + ASSERT(VFS_I(ip)->i_mode != 0); + } + + /* + * If the incore inode is INEW, there are several possibilities: + * + * (2) For a file that is being created, note that we allocate the + * ondisk inode before allocating, initializing, and adding the incore + * inode to the radix tree. + * + * (3) If the incore inode is being recycled, the inode has to be + * allocated because we don't allow freed inodes to be recycled. + * Recycling doesn't touch i_mode. + */ + if (ip->i_flags & XFS_INEW) { + /* created on disk already or recycling */ + ASSERT(VFS_I(ip)->i_mode != 0); + } + + /* + * (4) If the inode is queued for inactivation (NEED_INACTIVE) but + * inactivation has not started (!INACTIVATING), it is still allocated. + */ + if ((ip->i_flags & XFS_NEED_INACTIVE) && + !(ip->i_flags & XFS_INACTIVATING)) { + /* definitely before difree */ + ASSERT(VFS_I(ip)->i_mode != 0); + } +#endif + + /* + * If the incore inode is undergoing inactivation (INACTIVATING), there + * are two possibilities: + * + * (5) It is before the point where it would get freed ondisk, in which + * case i_mode is still nonzero. + * + * (6) It has already been freed, in which case i_mode is zero. + * + * We don't take the ILOCK here, but difree and dialloc update the AGI, + * and we've taken the AGI buffer lock, which prevents that from + * happening. + */ + + /* + * (7) Inodes undergoing inactivation (INACTIVATING) or queued for + * reclaim (IRECLAIMABLE) could be allocated or free. i_mode still + * reflects the ondisk state. + */ + + /* + * (8) If the inode is in IFLUSHING, it's safe to query i_mode because + * the flush code uses i_mode to format the ondisk inode. + */ + + /* + * (9) If the inode is in IRECLAIM and was reachable via the radix + * tree, it still has the same i_mode as it did before it entered + * reclaim. The inode object is still alive because we hold the RCU + * read lock. + */ + + *inuse = VFS_I(ip)->i_mode != 0; + error = 0; + +out_skip: + spin_unlock(&ip->i_flags_lock); +out_rcu: + rcu_read_unlock(); + return error; +} + +/* Is this inode a root directory for either tree? */ +bool +xchk_inode_is_dirtree_root(const struct xfs_inode *ip) +{ + struct xfs_mount *mp = ip->i_mount; + + return ip == mp->m_rootip || + (xfs_has_metadir(mp) && ip == mp->m_metadirip); +} + +/* Does the superblock point down to this inode? */ +bool +xchk_inode_is_sb_rooted(const struct xfs_inode *ip) +{ + return xchk_inode_is_dirtree_root(ip) || + xfs_is_sb_inum(ip->i_mount, I_INO(ip)); +} + +/* What is the root directory inumber for this inode? */ +xfs_ino_t +xchk_inode_rootdir_inum(const struct xfs_inode *ip) +{ + struct xfs_mount *mp = ip->i_mount; + + if (xfs_is_metadir_inode(ip)) + return I_INO(mp->m_metadirip); + return I_INO(mp->m_rootip); +} + +static int +xchk_meta_btree_count_blocks( + struct xfs_scrub *sc, + xfs_extnum_t *nextents, + xfs_filblks_t *count) +{ + struct xfs_btree_cur *cur; + int error; + + if (!sc->sr.rtg) { + ASSERT(0); + return -EFSCORRUPTED; + } + + switch (sc->ip->i_metatype) { + case XFS_METAFILE_RTRMAP: + cur = xfs_rtrmapbt_init_cursor(sc->tp, sc->sr.rtg); + break; + case XFS_METAFILE_RTREFCOUNT: + cur = xfs_rtrefcountbt_init_cursor(sc->tp, sc->sr.rtg); + break; + default: + ASSERT(0); + return -EFSCORRUPTED; + } + + error = xfs_btree_count_blocks(cur, count); + xfs_btree_del_cursor(cur, error); + if (!error) { + *nextents = 0; + (*count)--; /* don't count the btree iroot */ + } + return error; +} + +/* Count the blocks used by a file, even if it's a metadata inode. */ +int +xchk_inode_count_blocks( + struct xfs_scrub *sc, + int whichfork, + xfs_extnum_t *nextents, + xfs_filblks_t *count) +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(sc->ip, whichfork); + + if (!ifp) { + *nextents = 0; + *count = 0; + return 0; + } + + if (ifp->if_format == XFS_DINODE_FMT_META_BTREE) { + ASSERT(whichfork == XFS_DATA_FORK); + return xchk_meta_btree_count_blocks(sc, nextents, count); + } + + return xfs_bmap_count_blocks(sc->tp, sc->ip, whichfork, nextents, + count); +} diff --git a/fs/xfs/scrub/common.h b/fs/xfs/scrub/common.h new file mode 100644 index 000000000..9d627fd50 --- /dev/null +++ b/fs/xfs/scrub/common.h @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_COMMON_H__ +#define __XFS_SCRUB_COMMON_H__ + +int xchk_trans_alloc(struct xfs_scrub *sc, uint resblks); +void xchk_trans_alloc_empty(struct xfs_scrub *sc); +void xchk_trans_cancel(struct xfs_scrub *sc); + +bool xchk_process_error(struct xfs_scrub *sc, xfs_agnumber_t agno, + xfs_agblock_t bno, int *error); +bool xchk_process_rt_error(struct xfs_scrub *sc, xfs_rgnumber_t rgno, + xfs_rgblock_t rgbno, int *error); +bool xchk_fblock_process_error(struct xfs_scrub *sc, int whichfork, + xfs_fileoff_t offset, int *error); + +bool xchk_xref_process_error(struct xfs_scrub *sc, + xfs_agnumber_t agno, xfs_agblock_t bno, int *error); +bool xchk_fblock_xref_process_error(struct xfs_scrub *sc, + int whichfork, xfs_fileoff_t offset, int *error); + +void xchk_block_set_preen(struct xfs_scrub *sc, + struct xfs_buf *bp); +void xchk_ino_set_preen(struct xfs_scrub *sc, xfs_ino_t ino); +void xchk_fblock_set_preen(struct xfs_scrub *sc, + int whichfork, xfs_fileoff_t offset); + +void xchk_set_corrupt(struct xfs_scrub *sc); +void xchk_block_set_corrupt(struct xfs_scrub *sc, + struct xfs_buf *bp); +void xchk_ino_set_corrupt(struct xfs_scrub *sc, xfs_ino_t ino); +#define xchk_ip_set_corrupt(_sc, _ip) \ + xchk_ino_set_corrupt((_sc), I_INO(_ip)) +void xchk_fblock_set_corrupt(struct xfs_scrub *sc, int whichfork, + xfs_fileoff_t offset); +#ifdef CONFIG_XFS_QUOTA +void xchk_qcheck_set_corrupt(struct xfs_scrub *sc, unsigned int dqtype, + xfs_dqid_t id); +#endif + +void xchk_block_xref_set_corrupt(struct xfs_scrub *sc, + struct xfs_buf *bp); +void xchk_ip_xref_set_corrupt(struct xfs_scrub *sc, + struct xfs_inode *ip); +void xchk_fblock_xref_set_corrupt(struct xfs_scrub *sc, + int whichfork, xfs_fileoff_t offset); + +void xchk_ino_set_warning(struct xfs_scrub *sc, xfs_ino_t ino); +void xchk_fblock_set_warning(struct xfs_scrub *sc, int whichfork, + xfs_fileoff_t offset); + +void xchk_set_incomplete(struct xfs_scrub *sc); +int xchk_checkpoint_log(struct xfs_mount *mp); + +/* Are we set up for a cross-referencing check? */ +bool xchk_should_check_xref(struct xfs_scrub *sc, int *error, + struct xfs_btree_cur **curpp); + +static inline int xchk_setup_nothing(struct xfs_scrub *sc) +{ + return -ENOENT; +} + +/* Setup functions */ +int xchk_setup_agheader(struct xfs_scrub *sc); +int xchk_setup_fs(struct xfs_scrub *sc); +int xchk_setup_rt(struct xfs_scrub *sc); +int xchk_setup_ag_allocbt(struct xfs_scrub *sc); +int xchk_setup_ag_iallocbt(struct xfs_scrub *sc); +int xchk_setup_ag_rmapbt(struct xfs_scrub *sc); +int xchk_setup_ag_refcountbt(struct xfs_scrub *sc); +int xchk_setup_inode(struct xfs_scrub *sc); +int xchk_setup_inode_bmap(struct xfs_scrub *sc); +int xchk_setup_inode_bmap_data(struct xfs_scrub *sc); +int xchk_setup_directory(struct xfs_scrub *sc); +int xchk_setup_xattr(struct xfs_scrub *sc); +int xchk_setup_symlink(struct xfs_scrub *sc); +int xchk_setup_parent(struct xfs_scrub *sc); +int xchk_setup_dirtree(struct xfs_scrub *sc); +int xchk_setup_metapath(struct xfs_scrub *sc); +#ifdef CONFIG_XFS_RT +int xchk_setup_rtbitmap(struct xfs_scrub *sc); +int xchk_setup_rtsummary(struct xfs_scrub *sc); +int xchk_setup_rgsuperblock(struct xfs_scrub *sc); +int xchk_setup_rtrmapbt(struct xfs_scrub *sc); +int xchk_setup_rtrefcountbt(struct xfs_scrub *sc); +#else +# define xchk_setup_rtbitmap xchk_setup_nothing +# define xchk_setup_rtsummary xchk_setup_nothing +# define xchk_setup_rgsuperblock xchk_setup_nothing +# define xchk_setup_rtrmapbt xchk_setup_nothing +# define xchk_setup_rtrefcountbt xchk_setup_nothing +#endif +#ifdef CONFIG_XFS_QUOTA +int xchk_ino_dqattach(struct xfs_scrub *sc); +int xchk_setup_quota(struct xfs_scrub *sc); +int xchk_setup_quotacheck(struct xfs_scrub *sc); +#else +static inline int +xchk_ino_dqattach(struct xfs_scrub *sc) +{ + return 0; +} +# define xchk_setup_quota xchk_setup_nothing +# define xchk_setup_quotacheck xchk_setup_nothing +#endif +int xchk_setup_fscounters(struct xfs_scrub *sc); +int xchk_setup_nlinks(struct xfs_scrub *sc); + +void xchk_ag_free(struct xfs_scrub *sc, struct xchk_ag *sa); +int xchk_ag_init(struct xfs_scrub *sc, xfs_agnumber_t agno, + struct xchk_ag *sa); +int xchk_perag_drain_and_lock(struct xfs_scrub *sc); + +/* + * Grab all AG resources, treating the inability to grab the perag structure as + * a fs corruption. This is intended for callers checking an ondisk reference + * to a given AG, which means that the AG must still exist. + */ +static inline int +xchk_ag_init_existing( + struct xfs_scrub *sc, + xfs_agnumber_t agno, + struct xchk_ag *sa) +{ + int error = xchk_ag_init(sc, agno, sa); + + return error == -ENOENT ? -EFSCORRUPTED : error; +} + +#ifdef CONFIG_XFS_RT + +/* All the locks we need to check an rtgroup. */ +#define XCHK_RTGLOCK_ALL (XFS_RTGLOCK_BITMAP | \ + XFS_RTGLOCK_RMAP | \ + XFS_RTGLOCK_REFCOUNT) + +int xchk_rtgroup_init(struct xfs_scrub *sc, xfs_rgnumber_t rgno, + struct xchk_rt *sr); + +static inline int +xchk_rtgroup_init_existing( + struct xfs_scrub *sc, + xfs_rgnumber_t rgno, + struct xchk_rt *sr) +{ + int error = xchk_rtgroup_init(sc, rgno, sr); + + return error == -ENOENT ? -EFSCORRUPTED : error; +} + +int xchk_rtgroup_lock(struct xfs_scrub *sc, struct xchk_rt *sr, + unsigned int rtglock_flags); +void xchk_rtgroup_unlock(struct xchk_rt *sr); +void xchk_rtgroup_btcur_free(struct xchk_rt *sr); +void xchk_rtgroup_free(struct xfs_scrub *sc, struct xchk_rt *sr); +#else +# define xchk_rtgroup_init(sc, rgno, sr) (-EFSCORRUPTED) +# define xchk_rtgroup_init_existing(sc, rgno, sr) (-EFSCORRUPTED) +# define xchk_rtgroup_lock(sc, sr, lockflags) (-EFSCORRUPTED) +# define xchk_rtgroup_unlock(sr) do { } while (0) +# define xchk_rtgroup_btcur_free(sr) do { } while (0) +# define xchk_rtgroup_free(sc, sr) do { } while (0) +#endif /* CONFIG_XFS_RT */ + +int xchk_ag_read_headers(struct xfs_scrub *sc, xfs_agnumber_t agno, + struct xchk_ag *sa); +void xchk_ag_btcur_free(struct xchk_ag *sa); +void xchk_ag_btcur_init(struct xfs_scrub *sc, struct xchk_ag *sa); +int xchk_count_rmap_ownedby_ag(struct xfs_scrub *sc, struct xfs_btree_cur *cur, + const struct xfs_owner_info *oinfo, xfs_filblks_t *blocks); + +int xchk_setup_ag_btree(struct xfs_scrub *sc, bool force_log); +int xchk_iget_for_scrubbing(struct xfs_scrub *sc); +int xchk_setup_inode_contents(struct xfs_scrub *sc, unsigned int resblks); +int xchk_install_live_inode(struct xfs_scrub *sc, struct xfs_inode *ip); + +void xchk_ilock(struct xfs_scrub *sc, unsigned int ilock_flags); +bool xchk_ilock_nowait(struct xfs_scrub *sc, unsigned int ilock_flags); +void xchk_iunlock(struct xfs_scrub *sc, unsigned int ilock_flags); + +void xchk_buffer_recheck(struct xfs_scrub *sc, struct xfs_buf *bp); + +/* + * Grab the inode at @inum. The caller must have created a scrub transaction + * so that we can confirm the inumber by walking the inobt and not deadlock on + * a loop in the inobt. + */ +int xchk_iget(struct xfs_scrub *sc, xfs_ino_t inum, struct xfs_inode **ipp); +int xchk_iget_agi(struct xfs_scrub *sc, xfs_ino_t inum, + struct xfs_buf **agi_bpp, struct xfs_inode **ipp); +void xchk_irele(struct xfs_scrub *sc, struct xfs_inode *ip); +int xchk_install_handle_inode(struct xfs_scrub *sc, struct xfs_inode *ip); + +/* + * Safe version of (untrusted) xchk_iget that uses an empty transaction to + * avoid deadlocking on loops in the inobt. This should only be used in a + * scrub or repair setup routine, and only prior to grabbing a transaction. + */ +static inline int +xchk_iget_safe(struct xfs_scrub *sc, xfs_ino_t inum, struct xfs_inode **ipp) +{ + int error; + + ASSERT(sc->tp == NULL); + + error = xchk_trans_alloc(sc, 0); + if (error) + return error; + error = xchk_iget(sc, inum, ipp); + xchk_trans_cancel(sc); + return error; +} + +/* + * Don't bother cross-referencing if we already found corruption or cross + * referencing discrepancies. + */ +static inline bool xchk_skip_xref(struct xfs_scrub_metadata *sm) +{ + return sm->sm_flags & (XFS_SCRUB_OFLAG_CORRUPT | + XFS_SCRUB_OFLAG_XCORRUPT); +} + +bool xchk_dir_looks_zapped(struct xfs_inode *dp); +bool xchk_pptr_looks_zapped(struct xfs_inode *ip); + +/* Decide if a repair is required. */ +static inline bool xchk_needs_repair(const struct xfs_scrub_metadata *sm) +{ + return sm->sm_flags & (XFS_SCRUB_OFLAG_CORRUPT | + XFS_SCRUB_OFLAG_XCORRUPT | + XFS_SCRUB_OFLAG_PREEN); +} + +/* + * "Should we prepare for a repair?" + * + * Return true if the caller permits us to repair metadata and we're not + * setting up for a post-repair evaluation. + */ +static inline bool xchk_could_repair(const struct xfs_scrub *sc) +{ + return (sc->sm->sm_flags & XFS_SCRUB_IFLAG_REPAIR) && + !(sc->flags & XREP_ALREADY_FIXED); +} + +int xchk_metadata_inode_forks(struct xfs_scrub *sc); + +/* + * Setting up a hook to wait for intents to drain is costly -- we have to take + * the CPU hotplug lock and force an i-cache flush on all CPUs once to set it + * up, and again to tear it down. These costs add up quickly, so we only want + * to enable the drain waiter if the drain actually detected a conflict with + * running intent chains. + */ +static inline bool xchk_need_intent_drain(struct xfs_scrub *sc) +{ + return sc->flags & XCHK_NEED_DRAIN; +} + +void xchk_fsgates_enable(struct xfs_scrub *sc, unsigned int scrub_fshooks); + +int xchk_inode_is_allocated(struct xfs_scrub *sc, xfs_agino_t agino, + bool *inuse); +int xchk_inode_count_blocks(struct xfs_scrub *sc, int whichfork, + xfs_extnum_t *nextents, xfs_filblks_t *count); + +bool xchk_inode_is_dirtree_root(const struct xfs_inode *ip); +bool xchk_inode_is_sb_rooted(const struct xfs_inode *ip); +xfs_ino_t xchk_inode_rootdir_inum(const struct xfs_inode *ip); + +#endif /* __XFS_SCRUB_COMMON_H__ */ diff --git a/fs/xfs/scrub/cow_repair.c b/fs/xfs/scrub/cow_repair.c new file mode 100644 index 000000000..8dd9c0266 --- /dev/null +++ b/fs/xfs/scrub/cow_repair.c @@ -0,0 +1,743 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2022-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_btree.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_inode.h" +#include "xfs_inode_fork.h" +#include "xfs_alloc.h" +#include "xfs_bmap.h" +#include "xfs_rmap.h" +#include "xfs_refcount.h" +#include "xfs_quota.h" +#include "xfs_ialloc.h" +#include "xfs_ag.h" +#include "xfs_error.h" +#include "xfs_errortag.h" +#include "xfs_icache.h" +#include "xfs_refcount_btree.h" +#include "xfs_rtalloc.h" +#include "xfs_rtbitmap.h" +#include "xfs_rtgroup.h" +#include "xfs_bmap_util.h" +#include "scrub/xfs_scrub.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/bitmap.h" +#include "scrub/off_bitmap.h" +#include "scrub/fsb_bitmap.h" +#include "scrub/rtb_bitmap.h" +#include "scrub/reap.h" + +/* + * CoW Fork Mapping Repair + * ======================= + * + * Although CoW staging extents are owned by incore CoW inode forks, on disk + * they are owned by the refcount btree. The ondisk metadata does not record + * any ownership information, which limits what we can do to repair the + * mappings in the CoW fork. At most, we can replace ifork mappings that lack + * an entry in the refcount btree or are described by a reverse mapping record + * whose owner is not OWN_COW. + * + * Replacing extents is also tricky -- we can't touch written CoW fork extents + * since they are undergoing writeback, and delalloc extents do not require + * repair since they only exist incore. Hence the most we can do is find the + * bad parts of unwritten mappings, allocate a replacement set of blocks, and + * replace the incore mapping. We use the regular reaping process to unmap + * or free the discarded blocks, as appropriate. + */ +struct xrep_cow { + struct xfs_scrub *sc; + + /* Bitmap of file offset ranges that need replacing. */ + struct xoff_bitmap bad_fileoffs; + + /* Bitmap of fsblocks that were removed from the CoW fork. */ + union { + struct xfsb_bitmap old_cowfork_fsblocks; + struct xrtb_bitmap old_cowfork_rtblocks; + }; + + /* CoW fork mappings used to scan for bad CoW staging extents. */ + struct xfs_bmbt_irec irec; + + /* refcount btree block number of irec.br_startblock */ + unsigned int irec_startbno; + + /* refcount btree block number of the next refcount record we expect */ + unsigned int next_bno; +}; + +/* + * Mark the part of the file range that corresponds to the given physical + * space. Caller must ensure that the physical range is within xc->irec. + */ +STATIC int +xrep_cow_mark_file_range( + struct xrep_cow *xc, + xfs_fsblock_t startblock, + xfs_filblks_t blockcount) +{ + xfs_fileoff_t startoff; + + startoff = xc->irec.br_startoff + + (startblock - xc->irec.br_startblock); + + trace_xrep_cow_mark_file_range(xc->sc->ip, startblock, startoff, + blockcount); + + return xoff_bitmap_set(&xc->bad_fileoffs, startoff, blockcount); +} + +/* + * Trim @src to fit within the CoW fork mapping being examined, and put the + * result in @dst. + */ +static inline void +xrep_cow_trim_refcount( + struct xrep_cow *xc, + struct xfs_refcount_irec *dst, + const struct xfs_refcount_irec *src) +{ + unsigned int adj; + + memcpy(dst, src, sizeof(*dst)); + + if (dst->rc_startblock < xc->irec_startbno) { + adj = xc->irec_startbno - dst->rc_startblock; + dst->rc_blockcount -= adj; + dst->rc_startblock += adj; + } + + if (dst->rc_startblock + dst->rc_blockcount > + xc->irec_startbno + xc->irec.br_blockcount) { + adj = (dst->rc_startblock + dst->rc_blockcount) - + (xc->irec_startbno + xc->irec.br_blockcount); + dst->rc_blockcount -= adj; + } +} + +/* Mark any shared CoW staging extents. */ +STATIC int +xrep_cow_mark_shared_staging( + struct xfs_btree_cur *cur, + const struct xfs_refcount_irec *rec, + void *priv) +{ + struct xrep_cow *xc = priv; + struct xfs_refcount_irec rrec; + + if (!xfs_refcount_check_domain(rec) || + rec->rc_domain != XFS_REFC_DOMAIN_SHARED) + return -EFSCORRUPTED; + + xrep_cow_trim_refcount(xc, &rrec, rec); + + return xrep_cow_mark_file_range(xc, + xfs_gbno_to_fsb(cur->bc_group, rrec.rc_startblock), + rrec.rc_blockcount); +} + +/* + * Mark any portion of the CoW fork file offset range where there is not a CoW + * staging extent record in the refcountbt, and keep a record of where we did + * find correct refcountbt records. Staging records are always cleaned out at + * mount time, so any two inodes trying to map the same staging area would have + * already taken the fs down due to refcount btree verifier errors. Hence this + * inode should be the sole creator of the staging extent records ondisk. + */ +STATIC int +xrep_cow_mark_missing_staging( + struct xfs_btree_cur *cur, + const struct xfs_refcount_irec *rec, + void *priv) +{ + struct xrep_cow *xc = priv; + struct xfs_refcount_irec rrec; + int error; + + if (!xfs_refcount_check_domain(rec) || + rec->rc_domain != XFS_REFC_DOMAIN_COW) + return -EFSCORRUPTED; + + xrep_cow_trim_refcount(xc, &rrec, rec); + + if (xc->next_bno >= rrec.rc_startblock) + goto next; + + error = xrep_cow_mark_file_range(xc, + xfs_gbno_to_fsb(cur->bc_group, xc->next_bno), + rrec.rc_startblock - xc->next_bno); + if (error) + return error; + +next: + xc->next_bno = rrec.rc_startblock + rrec.rc_blockcount; + return 0; +} + +/* + * Mark any area that does not correspond to a CoW staging rmap. These are + * cross-linked areas that must be avoided. + */ +STATIC int +xrep_cow_mark_missing_staging_rmap( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv) +{ + struct xrep_cow *xc = priv; + xfs_agblock_t rec_bno; + xfs_extlen_t rec_len; + unsigned int adj; + + if (rec->rm_owner == XFS_RMAP_OWN_COW) + return 0; + + rec_bno = rec->rm_startblock; + rec_len = rec->rm_blockcount; + if (rec_bno < xc->irec_startbno) { + adj = xc->irec_startbno - rec_bno; + rec_len -= adj; + rec_bno += adj; + } + + if (rec_bno + rec_len > xc->irec_startbno + xc->irec.br_blockcount) { + adj = (rec_bno + rec_len) - + (xc->irec_startbno + xc->irec.br_blockcount); + rec_len -= adj; + } + + return xrep_cow_mark_file_range(xc, + xfs_gbno_to_fsb(cur->bc_group, rec_bno), rec_len); +} + +/* + * Trim the start and end of the current mapping by up to 1/4 of the length + * and mark that as "bad" to test the cow fork repair mechanism. + */ +static inline int +xrep_cow_debug_replacement( + struct xrep_cow *xc) +{ + xfs_fsblock_t fsbno = xc->irec.br_startblock; + xfs_extlen_t len = xc->irec.br_blockcount; + uint32_t trim; + + /* get_random_u32_below requires a nonzero argument */ + trim = len > 4 ? get_random_u32_below(len / 4) : 0; + len -= trim; + + trim = len > 4 ? get_random_u32_below(len / 4) : 0; + fsbno += trim; + len -= trim; + + return xrep_cow_mark_file_range(xc, fsbno, len); +} + +/* + * Find any part of the CoW fork mapping that isn't a single-owner CoW staging + * extent and mark the corresponding part of the file range in the bitmap. + */ +STATIC int +xrep_cow_find_bad( + struct xrep_cow *xc) +{ + struct xfs_refcount_irec rc_low = { 0 }; + struct xfs_refcount_irec rc_high = { 0 }; + struct xfs_rmap_irec rm_low = { 0 }; + struct xfs_rmap_irec rm_high = { 0 }; + struct xfs_perag *pag; + struct xfs_scrub *sc = xc->sc; + xfs_agnumber_t agno; + int error; + + agno = XFS_FSB_TO_AGNO(sc->mp, xc->irec.br_startblock); + xc->irec_startbno = XFS_FSB_TO_AGBNO(sc->mp, xc->irec.br_startblock); + + pag = xfs_perag_get(sc->mp, agno); + if (!pag) + return -EFSCORRUPTED; + + error = xrep_ag_init(sc, pag, &sc->sa); + if (error) + goto out_pag; + + /* Mark any CoW fork extents that are shared. */ + rc_low.rc_startblock = xc->irec_startbno; + rc_high.rc_startblock = xc->irec_startbno + xc->irec.br_blockcount - 1; + rc_low.rc_domain = rc_high.rc_domain = XFS_REFC_DOMAIN_SHARED; + error = xfs_refcount_query_range(sc->sa.refc_cur, &rc_low, &rc_high, + xrep_cow_mark_shared_staging, xc); + if (error) + goto out_sa; + + /* Make sure there are CoW staging extents for the whole mapping. */ + rc_low.rc_startblock = xc->irec_startbno; + rc_high.rc_startblock = xc->irec_startbno + xc->irec.br_blockcount - 1; + rc_low.rc_domain = rc_high.rc_domain = XFS_REFC_DOMAIN_COW; + xc->next_bno = xc->irec_startbno; + error = xfs_refcount_query_range(sc->sa.refc_cur, &rc_low, &rc_high, + xrep_cow_mark_missing_staging, xc); + if (error) + goto out_sa; + + if (xc->next_bno < xc->irec_startbno + xc->irec.br_blockcount) { + error = xrep_cow_mark_file_range(xc, + xfs_agbno_to_fsb(pag, xc->next_bno), + xc->irec_startbno + xc->irec.br_blockcount - + xc->next_bno); + if (error) + goto out_sa; + } + + /* Mark any area has an rmap that isn't a COW staging extent. */ + rm_low.rm_startblock = xc->irec_startbno; + memset(&rm_high, 0xFF, sizeof(rm_high)); + rm_high.rm_startblock = xc->irec_startbno + xc->irec.br_blockcount - 1; + error = xfs_rmap_query_range(sc->sa.rmap_cur, &rm_low, &rm_high, + xrep_cow_mark_missing_staging_rmap, xc); + if (error) + goto out_sa; + + /* + * If userspace is forcing us to rebuild the CoW fork or someone turned + * on the debugging knob, replace everything in the CoW fork. + */ + if (XFS_TEST_ERROR(sc->mp, XFS_ERRTAG_FORCE_SCRUB_REPAIR)) + error = xrep_cow_debug_replacement(xc); + else if (sc->sm->sm_flags & XFS_SCRUB_IFLAG_FORCE_REBUILD) + error = xrep_cow_mark_file_range(xc, xc->irec.br_startblock, + xc->irec.br_blockcount); + +out_sa: + xchk_ag_free(sc, &sc->sa); +out_pag: + xfs_perag_put(pag); + return error; +} + +/* + * Find any part of the CoW fork mapping that isn't a single-owner CoW staging + * extent and mark the corresponding part of the file range in the bitmap. + */ +STATIC int +xrep_cow_find_bad_rt( + struct xrep_cow *xc) +{ + struct xfs_refcount_irec rc_low = { 0 }; + struct xfs_refcount_irec rc_high = { 0 }; + struct xfs_rmap_irec rm_low = { 0 }; + struct xfs_rmap_irec rm_high = { 0 }; + struct xfs_scrub *sc = xc->sc; + struct xfs_rtgroup *rtg; + int error = 0; + + xc->irec_startbno = xfs_rtb_to_rgbno(sc->mp, xc->irec.br_startblock); + + rtg = xfs_rtgroup_get(sc->mp, + xfs_rtb_to_rgno(sc->mp, xc->irec.br_startblock)); + if (!rtg) + return -EFSCORRUPTED; + + error = xrep_rtgroup_init(sc, rtg, &sc->sr, + XFS_RTGLOCK_RMAP | XFS_RTGLOCK_REFCOUNT); + if (error) + goto out_rtg; + + /* Mark any CoW fork extents that are shared. */ + rc_low.rc_startblock = xc->irec_startbno; + rc_high.rc_startblock = xc->irec_startbno + xc->irec.br_blockcount - 1; + rc_low.rc_domain = rc_high.rc_domain = XFS_REFC_DOMAIN_SHARED; + error = xfs_refcount_query_range(sc->sr.refc_cur, &rc_low, &rc_high, + xrep_cow_mark_shared_staging, xc); + if (error) + goto out_sr; + + /* Make sure there are CoW staging extents for the whole mapping. */ + rc_low.rc_startblock = xc->irec_startbno; + rc_high.rc_startblock = xc->irec_startbno + xc->irec.br_blockcount - 1; + rc_low.rc_domain = rc_high.rc_domain = XFS_REFC_DOMAIN_COW; + xc->next_bno = xc->irec_startbno; + error = xfs_refcount_query_range(sc->sr.refc_cur, &rc_low, &rc_high, + xrep_cow_mark_missing_staging, xc); + if (error) + goto out_sr; + + if (xc->next_bno < xc->irec_startbno + xc->irec.br_blockcount) { + error = xrep_cow_mark_file_range(xc, + xfs_rgbno_to_rtb(rtg, xc->next_bno), + xc->irec_startbno + xc->irec.br_blockcount - + xc->next_bno); + if (error) + goto out_sr; + } + + /* Mark any area has an rmap that isn't a COW staging extent. */ + rm_low.rm_startblock = xc->irec_startbno; + memset(&rm_high, 0xFF, sizeof(rm_high)); + rm_high.rm_startblock = xc->irec_startbno + xc->irec.br_blockcount - 1; + error = xfs_rmap_query_range(sc->sr.rmap_cur, &rm_low, &rm_high, + xrep_cow_mark_missing_staging_rmap, xc); + if (error) + goto out_sr; + + /* + * If userspace is forcing us to rebuild the CoW fork or someone + * turned on the debugging knob, replace everything in the + * CoW fork and then scan for staging extents in the refcountbt. + */ + if (XFS_TEST_ERROR(sc->mp, XFS_ERRTAG_FORCE_SCRUB_REPAIR)) + error = xrep_cow_debug_replacement(xc); + else if (sc->sm->sm_flags & XFS_SCRUB_IFLAG_FORCE_REBUILD) + error = xrep_cow_mark_file_range(xc, xc->irec.br_startblock, + xc->irec.br_blockcount); + +out_sr: + xchk_rtgroup_btcur_free(&sc->sr); + xchk_rtgroup_free(sc, &sc->sr); +out_rtg: + xfs_rtgroup_put(rtg); + return error; +} + +/* + * Allocate a replacement CoW staging extent of up to the given number of + * blocks, and fill out the mapping. + */ +STATIC int +xrep_cow_alloc( + struct xfs_scrub *sc, + struct xfs_bmbt_irec *del) +{ + struct xfs_alloc_arg args = { + .tp = sc->tp, + .mp = sc->mp, + .oinfo = XFS_RMAP_OINFO_SKIP_UPDATE, + .minlen = 1, + .maxlen = del->br_blockcount, + .prod = 1, + .resv = XFS_AG_RESV_NONE, + .datatype = XFS_ALLOC_USERDATA, + }; + int error; + + error = xfs_trans_reserve_more(sc->tp, del->br_blockcount, 0); + if (error) + return error; + + error = xfs_alloc_vextent_start_ag(&args, XFS_INODE_TO_FSB(sc->ip)); + if (error) + return error; + if (args.fsbno == NULLFSBLOCK) + return -ENOSPC; + + xfs_refcount_alloc_cow_extent(sc->tp, false, args.fsbno, args.len); + + del->br_startblock = args.fsbno; + del->br_blockcount = args.len; + return 0; +} + +/* + * Allocate a replacement rt CoW staging extent of up to the given number of + * blocks, and fill out the mapping. + */ +STATIC int +xrep_cow_alloc_rt( + struct xfs_scrub *sc, + struct xfs_bmbt_irec *del) +{ + xfs_fsblock_t fsbno; + xfs_rtxlen_t maxrtx = + min(U32_MAX, xfs_blen_to_rtbxlen(sc->mp, del->br_blockcount)); + xfs_extlen_t len; + int error; + + error = xfs_trans_reserve_more(sc->tp, 0, maxrtx); + if (error) + return error; + + error = xfs_rtallocate_rtgs(sc->tp, NULLRTBLOCK, 1, maxrtx, 1, false, + false, &fsbno, &len); + if (error) + return error; + + xfs_refcount_alloc_cow_extent(sc->tp, true, fsbno, len); + + del->br_startblock = fsbno; + del->br_blockcount = len; + return 0; +} + +/* + * Look up the current CoW fork mapping so that we only allocate enough to + * replace a single mapping. If we don't find a mapping that covers the start + * of the file range, or we find a delalloc or written extent, something is + * seriously wrong, since we didn't drop the ILOCK. + */ +static inline int +xrep_cow_find_mapping( + struct xrep_cow *xc, + struct xfs_iext_cursor *icur, + xfs_fileoff_t badoff, + xfs_extlen_t badlen, + struct xfs_bmbt_irec *got, + struct xfs_bmbt_irec *rep) +{ + struct xfs_inode *ip = xc->sc->ip; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_COW_FORK); + + if (!xfs_iext_lookup_extent(ip, ifp, badoff, icur, got)) + goto bad; + memcpy(rep, got, sizeof(*rep)); + + if (got->br_startoff > badoff) + goto bad; + + if (isnullstartblock(got->br_startblock)) + goto bad; + + if (xfs_bmap_is_written_extent(got)) + goto bad; + + if (got->br_startoff < badoff) { + const int64_t delta = badoff - got->br_startoff; + + rep->br_blockcount -= delta; + rep->br_startoff += delta; + rep->br_startblock += delta; + } + + if (got->br_startoff + got->br_blockcount > badoff + badlen) { + const int64_t delta = (got->br_startoff + got->br_blockcount) - + (badoff + badlen); + + rep->br_blockcount -= delta; + } + + if (got->br_blockcount == 0) + goto bad; + + return 0; +bad: + ASSERT(0); + return -EFSCORRUPTED; +} + +/* + * Replace the unwritten CoW staging extent backing the given file range with a + * new space extent that isn't as problematic. + */ +STATIC int +xrep_cow_replace_range( + struct xrep_cow *xc, + xfs_fileoff_t startoff, + xfs_extlen_t *blockcount) +{ + struct xfs_iext_cursor icur; + struct xfs_bmbt_irec got, rep; + struct xfs_scrub *sc = xc->sc; + xfs_fsblock_t old_fsbno; + int error; + + /* + * Put the existing CoW fork mapping in @got, and put in @rep the + * contents of @got trimmed to @startoff/@blockcount. We only want + * to replace the bad region, and only one mapping at a time. + */ + error = xrep_cow_find_mapping(xc, &icur, startoff, *blockcount, &got, + &rep); + if (error) + return error; + old_fsbno = rep.br_startblock; + + /* + * Allocate a replacement extent. If we don't fill all the blocks, + * shorten the quantity that will be deleted in this step. + */ + if (XFS_IS_REALTIME_INODE(sc->ip)) + error = xrep_cow_alloc_rt(sc, &rep); + else + error = xrep_cow_alloc(sc, &rep); + if (error) + return error; + + /* + * Replace the old mapping with the new one, and commit the metadata + * changes made so far. + */ + xfs_bmap_replace_cow_mapping(sc->ip, &icur, &got, &rep); + + xfs_inode_set_cowblocks_tag(sc->ip); + error = xfs_defer_finish(&sc->tp); + if (error) + return error; + + /* Note the old CoW staging extents; we'll reap them all later. */ + if (XFS_IS_REALTIME_INODE(sc->ip)) + error = xrtb_bitmap_set(&xc->old_cowfork_rtblocks, old_fsbno, + rep.br_blockcount); + else + error = xfsb_bitmap_set(&xc->old_cowfork_fsblocks, old_fsbno, + rep.br_blockcount); + if (error) + return error; + + *blockcount = rep.br_blockcount; + return 0; +} + +/* + * Replace a bad part of an unwritten CoW staging extent with a fresh delalloc + * reservation. + */ +STATIC int +xrep_cow_replace( + uint64_t startoff, + uint64_t blockcount, + void *priv) +{ + struct xrep_cow *xc = priv; + int error = 0; + + while (blockcount > 0) { + xfs_extlen_t len = min_t(xfs_filblks_t, blockcount, + XFS_MAX_BMBT_EXTLEN); + + error = xrep_cow_replace_range(xc, startoff, &len); + if (error) + break; + + blockcount -= len; + startoff += len; + } + + return error; +} + +/* + * Repair an inode's CoW fork. The CoW fork is an in-core structure, so + * there's no btree to rebuid. Instead, we replace any mappings that are + * cross-linked or lack ondisk CoW fork records in the refcount btree. + */ +int +xrep_bmap_cow( + struct xfs_scrub *sc) +{ + struct xrep_cow *xc; + struct xfs_iext_cursor icur; + struct xfs_ifork *ifp = xfs_ifork_ptr(sc->ip, XFS_COW_FORK); + int error; + + if (!xfs_has_rmapbt(sc->mp) || !xfs_has_reflink(sc->mp)) + return -EOPNOTSUPP; + + if (!ifp) + return 0; + + /* + * Realtime files with large extent sizes are not supported because + * we could encounter an CoW mapping that has been partially written + * out *and* requires replacement, and there's no solution to that. + */ + if (xfs_inode_has_bigrtalloc(sc->ip)) + return -EOPNOTSUPP; + + /* Metadata inodes aren't supposed to have data on the rt volume. */ + if (xfs_is_metadir_inode(sc->ip) && XFS_IS_REALTIME_INODE(sc->ip)) + return -EOPNOTSUPP; + + /* + * If we're somehow not in extents format, then reinitialize it to + * an empty extent mapping fork and exit. + */ + if (ifp->if_format != XFS_DINODE_FMT_EXTENTS) { + ifp->if_format = XFS_DINODE_FMT_EXTENTS; + ifp->if_nextents = 0; + return 0; + } + + xc = kzalloc_obj(struct xrep_cow, XCHK_GFP_FLAGS); + if (!xc) + return -ENOMEM; + + xfs_trans_ijoin(sc->tp, sc->ip, 0); + + xc->sc = sc; + xoff_bitmap_init(&xc->bad_fileoffs); + if (XFS_IS_REALTIME_INODE(sc->ip)) + xrtb_bitmap_init(&xc->old_cowfork_rtblocks); + else + xfsb_bitmap_init(&xc->old_cowfork_fsblocks); + + for_each_xfs_iext(ifp, &icur, &xc->irec) { + if (xchk_should_terminate(sc, &error)) + goto out_bitmap; + + /* + * delalloc reservations only exist incore, so there is no + * ondisk metadata that we can examine. Hence we leave them + * alone. + */ + if (isnullstartblock(xc->irec.br_startblock)) + continue; + + /* + * COW fork extents are only in the written state if writeback + * is actively writing to disk. We cannot restart the write + * at a different disk address since we've already issued the + * IO, so we leave these alone and hope for the best. + */ + if (xfs_bmap_is_written_extent(&xc->irec)) + continue; + + if (XFS_IS_REALTIME_INODE(sc->ip)) + error = xrep_cow_find_bad_rt(xc); + else + error = xrep_cow_find_bad(xc); + if (error) + goto out_bitmap; + } + + /* Replace any bad unwritten mappings with fresh reservations. */ + error = xoff_bitmap_walk(&xc->bad_fileoffs, xrep_cow_replace, xc); + if (error) + goto out_bitmap; + + /* + * Reap as many of the old CoW blocks as we can. They are owned ondisk + * by the refcount btree, not the inode, so it is correct to treat them + * like inode metadata. + */ + if (XFS_IS_REALTIME_INODE(sc->ip)) + error = xrep_reap_rtblocks(sc, &xc->old_cowfork_rtblocks, + &XFS_RMAP_OINFO_COW); + else + error = xrep_reap_fsblocks(sc, &xc->old_cowfork_fsblocks, + &XFS_RMAP_OINFO_COW); + if (error) + goto out_bitmap; + +out_bitmap: + if (XFS_IS_REALTIME_INODE(sc->ip)) + xrtb_bitmap_destroy(&xc->old_cowfork_rtblocks); + else + xfsb_bitmap_destroy(&xc->old_cowfork_fsblocks); + xoff_bitmap_destroy(&xc->bad_fileoffs); + kfree(xc); + return error; +} diff --git a/fs/xfs/scrub/dab_bitmap.h b/fs/xfs/scrub/dab_bitmap.h new file mode 100644 index 000000000..0c6e3aad4 --- /dev/null +++ b/fs/xfs/scrub/dab_bitmap.h @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2022-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_DAB_BITMAP_H__ +#define __XFS_SCRUB_DAB_BITMAP_H__ + +/* Bitmaps, but for type-checked for xfs_dablk_t */ + +struct xdab_bitmap { + struct xbitmap32 dabitmap; +}; + +static inline void xdab_bitmap_init(struct xdab_bitmap *bitmap) +{ + xbitmap32_init(&bitmap->dabitmap); +} + +static inline void xdab_bitmap_destroy(struct xdab_bitmap *bitmap) +{ + xbitmap32_destroy(&bitmap->dabitmap); +} + +static inline int xdab_bitmap_set(struct xdab_bitmap *bitmap, + xfs_dablk_t dabno, xfs_extlen_t len) +{ + return xbitmap32_set(&bitmap->dabitmap, dabno, len); +} + +static inline bool xdab_bitmap_test(struct xdab_bitmap *bitmap, + xfs_dablk_t dabno, xfs_extlen_t *len) +{ + return xbitmap32_test(&bitmap->dabitmap, dabno, len); +} + +#endif /* __XFS_SCRUB_DAB_BITMAP_H__ */ diff --git a/fs/xfs/scrub/dabtree.c b/fs/xfs/scrub/dabtree.c new file mode 100644 index 000000000..d4e5afbde --- /dev/null +++ b/fs/xfs/scrub/dabtree.c @@ -0,0 +1,628 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_inode.h" +#include "xfs_dir2.h" +#include "xfs_dir2_priv.h" +#include "xfs_attr_leaf.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/trace.h" +#include "scrub/dabtree.h" + +/* Directory/Attribute Btree */ + +/* + * Check for da btree operation errors. See the section about handling + * operational errors in common.c. + */ +bool +xchk_da_process_error( + struct xchk_da_btree *ds, + int level, + int *error) +{ + struct xfs_scrub *sc = ds->sc; + + if (*error == 0) + return true; + + switch (*error) { + case -EDEADLOCK: + case -ECHRNG: + /* Used to restart an op with deadlock avoidance. */ + trace_xchk_deadlock_retry(sc->ip, sc->sm, *error); + break; + case -EFSBADCRC: + case -EFSCORRUPTED: + case -EIO: + case -ENODATA: + /* Note the badness but don't abort. */ + sc->sm->sm_flags |= XFS_SCRUB_OFLAG_CORRUPT; + *error = 0; + fallthrough; + default: + trace_xchk_file_op_error(sc, ds->dargs.whichfork, + xfs_dir2_da_to_db(ds->dargs.geo, + ds->state->path.blk[level].blkno), + *error, __return_address); + break; + } + return false; +} + +/* + * Check for da btree corruption. See the section about handling + * operational errors in common.c. + */ +void +xchk_da_set_corrupt( + struct xchk_da_btree *ds, + int level) +{ + struct xfs_scrub *sc = ds->sc; + + sc->sm->sm_flags |= XFS_SCRUB_OFLAG_CORRUPT; + + trace_xchk_fblock_error(sc, ds->dargs.whichfork, + xfs_dir2_da_to_db(ds->dargs.geo, + ds->state->path.blk[level].blkno), + __return_address); +} + +/* Flag a da btree node in need of optimization. */ +void +xchk_da_set_preen( + struct xchk_da_btree *ds, + int level) +{ + struct xfs_scrub *sc = ds->sc; + + sc->sm->sm_flags |= XFS_SCRUB_OFLAG_PREEN; + trace_xchk_fblock_preen(sc, ds->dargs.whichfork, + xfs_dir2_da_to_db(ds->dargs.geo, + ds->state->path.blk[level].blkno), + __return_address); +} + +/* Find an entry at a certain level in a da btree. */ +static struct xfs_da_node_entry * +xchk_da_btree_node_entry( + struct xchk_da_btree *ds, + int level) +{ + struct xfs_da_state_blk *blk = &ds->state->path.blk[level]; + struct xfs_da3_icnode_hdr hdr; + + ASSERT(blk->magic == XFS_DA_NODE_MAGIC); + + xfs_da3_node_hdr_from_disk(ds->sc->mp, &hdr, blk->bp->b_addr); + return hdr.btree + blk->index; +} + +/* Scrub a da btree hash (key). */ +int +xchk_da_btree_hash( + struct xchk_da_btree *ds, + int level, + __be32 *hashp) +{ + struct xfs_da_node_entry *entry; + xfs_dahash_t hash; + xfs_dahash_t parent_hash; + + /* Is this hash in order? */ + hash = be32_to_cpu(*hashp); + if (hash < ds->hashes[level]) + xchk_da_set_corrupt(ds, level); + ds->hashes[level] = hash; + + if (level == 0) + return 0; + + /* Is this hash no larger than the parent hash? */ + entry = xchk_da_btree_node_entry(ds, level - 1); + parent_hash = be32_to_cpu(entry->hashval); + if (parent_hash < hash) + xchk_da_set_corrupt(ds, level); + + return 0; +} + +/* + * Check a da btree pointer. Returns true if it's ok to use this + * pointer. + */ +STATIC bool +xchk_da_btree_ptr_ok( + struct xchk_da_btree *ds, + int level, + xfs_dablk_t blkno) +{ + if (blkno < ds->lowest || (ds->highest != 0 && blkno >= ds->highest)) { + xchk_da_set_corrupt(ds, level); + return false; + } + + return true; +} + +/* + * The da btree scrubber can handle leaf1 blocks as a degenerate + * form of leafn blocks. Since the regular da code doesn't handle + * leaf1, we must multiplex the verifiers. + */ +static void +xchk_da_btree_read_verify( + struct xfs_buf *bp) +{ + struct xfs_da_blkinfo *info = bp->b_addr; + + switch (be16_to_cpu(info->magic)) { + case XFS_DIR2_LEAF1_MAGIC: + case XFS_DIR3_LEAF1_MAGIC: + bp->b_ops = &xfs_dir3_leaf1_buf_ops; + bp->b_ops->verify_read(bp); + return; + default: + /* + * xfs_da3_node_buf_ops already know how to handle + * DA*_NODE, ATTR*_LEAF, and DIR*_LEAFN blocks. + */ + bp->b_ops = &xfs_da3_node_buf_ops; + bp->b_ops->verify_read(bp); + return; + } +} +static void +xchk_da_btree_write_verify( + struct xfs_buf *bp) +{ + struct xfs_da_blkinfo *info = bp->b_addr; + + switch (be16_to_cpu(info->magic)) { + case XFS_DIR2_LEAF1_MAGIC: + case XFS_DIR3_LEAF1_MAGIC: + bp->b_ops = &xfs_dir3_leaf1_buf_ops; + bp->b_ops->verify_write(bp); + return; + default: + /* + * xfs_da3_node_buf_ops already know how to handle + * DA*_NODE, ATTR*_LEAF, and DIR*_LEAFN blocks. + */ + bp->b_ops = &xfs_da3_node_buf_ops; + bp->b_ops->verify_write(bp); + return; + } +} +static void * +xchk_da_btree_verify( + struct xfs_buf *bp) +{ + struct xfs_da_blkinfo *info = bp->b_addr; + + switch (be16_to_cpu(info->magic)) { + case XFS_DIR2_LEAF1_MAGIC: + case XFS_DIR3_LEAF1_MAGIC: + bp->b_ops = &xfs_dir3_leaf1_buf_ops; + return bp->b_ops->verify_struct(bp); + default: + bp->b_ops = &xfs_da3_node_buf_ops; + return bp->b_ops->verify_struct(bp); + } +} + +static const struct xfs_buf_ops xchk_da_btree_buf_ops = { + .name = "xchk_da_btree", + .verify_read = xchk_da_btree_read_verify, + .verify_write = xchk_da_btree_write_verify, + .verify_struct = xchk_da_btree_verify, +}; + +/* Check a block's sibling. */ +STATIC int +xchk_da_btree_block_check_sibling( + struct xchk_da_btree *ds, + int level, + int direction, + xfs_dablk_t sibling) +{ + struct xfs_da_state_path *path = &ds->state->path; + struct xfs_da_state_path *altpath = &ds->state->altpath; + int retval; + int plevel; + int error; + + memcpy(altpath, path, sizeof(ds->state->altpath)); + + /* + * If the pointer is null, we shouldn't be able to move the upper + * level pointer anywhere. + */ + if (sibling == 0) { + error = xfs_da3_path_shift(ds->state, altpath, direction, + false, &retval); + if (error == 0 && retval == 0) + xchk_da_set_corrupt(ds, level); + error = 0; + goto out; + } + + /* Move the alternate cursor one block in the direction given. */ + error = xfs_da3_path_shift(ds->state, altpath, direction, false, + &retval); + if (!xchk_da_process_error(ds, level, &error)) + goto out; + if (retval) { + xchk_da_set_corrupt(ds, level); + goto out; + } + if (altpath->blk[level].bp) + xchk_buffer_recheck(ds->sc, altpath->blk[level].bp); + + /* Compare upper level pointer to sibling pointer. */ + if (altpath->blk[level].blkno != sibling) + xchk_da_set_corrupt(ds, level); + +out: + /* Free all buffers in the altpath that aren't referenced from path. */ + for (plevel = 0; plevel < altpath->active; plevel++) { + if (altpath->blk[plevel].bp == NULL || + (plevel < path->active && + altpath->blk[plevel].bp == path->blk[plevel].bp)) + continue; + + xfs_trans_brelse(ds->dargs.trans, altpath->blk[plevel].bp); + altpath->blk[plevel].bp = NULL; + } + + return error; +} + +/* Check a block's sibling pointers. */ +STATIC int +xchk_da_btree_block_check_siblings( + struct xchk_da_btree *ds, + int level, + struct xfs_da_blkinfo *hdr) +{ + xfs_dablk_t forw; + xfs_dablk_t back; + int error = 0; + + forw = be32_to_cpu(hdr->forw); + back = be32_to_cpu(hdr->back); + + /* Top level blocks should not have sibling pointers. */ + if (level == 0) { + if (forw != 0 || back != 0) + xchk_da_set_corrupt(ds, level); + return 0; + } + + /* + * Check back (left) and forw (right) pointers. These functions + * absorb error codes for us. + */ + error = xchk_da_btree_block_check_sibling(ds, level, 0, back); + if (error) + goto out; + error = xchk_da_btree_block_check_sibling(ds, level, 1, forw); + +out: + memset(&ds->state->altpath, 0, sizeof(ds->state->altpath)); + return error; +} + +/* Load a dir/attribute block from a btree. */ +STATIC int +xchk_da_btree_block( + struct xchk_da_btree *ds, + int level, + xfs_dablk_t blkno) +{ + struct xfs_da_state_blk *blk; + struct xfs_da_intnode *node; + struct xfs_da_node_entry *btree; + struct xfs_da3_blkinfo *hdr3; + struct xfs_da_args *dargs = &ds->dargs; + struct xfs_inode *ip = ds->dargs.dp; + xfs_failaddr_t fa; + xfs_ino_t owner; + int *pmaxrecs; + struct xfs_da3_icnode_hdr nodehdr; + int error = 0; + + blk = &ds->state->path.blk[level]; + ds->state->path.active = level + 1; + + /* Release old block. */ + if (blk->bp) { + xfs_trans_brelse(dargs->trans, blk->bp); + blk->bp = NULL; + } + + /* Check the pointer. */ + blk->blkno = blkno; + if (!xchk_da_btree_ptr_ok(ds, level, blkno)) + goto out_nobuf; + + /* Read the buffer. */ + error = xfs_da_read_buf(dargs->trans, dargs->dp, blk->blkno, + XFS_DABUF_MAP_HOLE_OK, &blk->bp, dargs->whichfork, + &xchk_da_btree_buf_ops); + if (!xchk_da_process_error(ds, level, &error)) + goto out_nobuf; + if (blk->bp) + xchk_buffer_recheck(ds->sc, blk->bp); + + /* + * We didn't find a dir btree root block, which means that + * there's no LEAF1/LEAFN tree (at least not where it's supposed + * to be), so jump out now. + */ + if (ds->dargs.whichfork == XFS_DATA_FORK && level == 0 && + blk->bp == NULL) + goto out_nobuf; + + /* It's /not/ ok for attr trees not to have a da btree. */ + if (blk->bp == NULL) { + xchk_da_set_corrupt(ds, level); + goto out_nobuf; + } + + hdr3 = blk->bp->b_addr; + blk->magic = be16_to_cpu(hdr3->hdr.magic); + pmaxrecs = &ds->maxrecs[level]; + + /* We only started zeroing the header on v5 filesystems. */ + if (xfs_has_crc(ds->sc->mp) && hdr3->hdr.pad) + xchk_da_set_corrupt(ds, level); + + /* Check the owner. */ + if (xfs_has_crc(ip->i_mount)) { + owner = be64_to_cpu(hdr3->owner); + if (owner != I_INO(ip)) + xchk_da_set_corrupt(ds, level); + } + + /* Check the siblings. */ + error = xchk_da_btree_block_check_siblings(ds, level, &hdr3->hdr); + if (error) + goto out; + + /* Interpret the buffer. */ + switch (blk->magic) { + case XFS_ATTR_LEAF_MAGIC: + case XFS_ATTR3_LEAF_MAGIC: + xfs_trans_buf_set_type(dargs->trans, blk->bp, + XFS_BLFT_ATTR_LEAF_BUF); + blk->magic = XFS_ATTR_LEAF_MAGIC; + blk->hashval = xfs_attr_leaf_lasthash(blk->bp, pmaxrecs); + if (ds->tree_level != 0) + xchk_da_set_corrupt(ds, level); + break; + case XFS_DIR2_LEAFN_MAGIC: + case XFS_DIR3_LEAFN_MAGIC: + xfs_trans_buf_set_type(dargs->trans, blk->bp, + XFS_BLFT_DIR_LEAFN_BUF); + blk->magic = XFS_DIR2_LEAFN_MAGIC; + blk->hashval = xfs_dir2_leaf_lasthash(ip, blk->bp, pmaxrecs); + if (ds->tree_level != 0) + xchk_da_set_corrupt(ds, level); + break; + case XFS_DIR2_LEAF1_MAGIC: + case XFS_DIR3_LEAF1_MAGIC: + xfs_trans_buf_set_type(dargs->trans, blk->bp, + XFS_BLFT_DIR_LEAF1_BUF); + blk->magic = XFS_DIR2_LEAF1_MAGIC; + blk->hashval = xfs_dir2_leaf_lasthash(ip, blk->bp, pmaxrecs); + if (ds->tree_level != 0) + xchk_da_set_corrupt(ds, level); + break; + case XFS_DA_NODE_MAGIC: + case XFS_DA3_NODE_MAGIC: + xfs_trans_buf_set_type(dargs->trans, blk->bp, + XFS_BLFT_DA_NODE_BUF); + blk->magic = XFS_DA_NODE_MAGIC; + node = blk->bp->b_addr; + xfs_da3_node_hdr_from_disk(ip->i_mount, &nodehdr, node); + btree = nodehdr.btree; + *pmaxrecs = nodehdr.count; + blk->hashval = be32_to_cpu(btree[*pmaxrecs - 1].hashval); + if (level == 0) { + if (nodehdr.level >= XFS_DA_NODE_MAXDEPTH) { + xchk_da_set_corrupt(ds, level); + goto out_freebp; + } + ds->tree_level = nodehdr.level; + } else { + if (ds->tree_level != nodehdr.level) { + xchk_da_set_corrupt(ds, level); + goto out_freebp; + } + } + + if (xfs_has_crc(ip->i_mount)) { + struct xfs_da3_node_hdr *nodehdr3 = blk->bp->b_addr; + + if (nodehdr3->__pad32) + xchk_da_set_preen(ds, level); + } + break; + default: + xchk_da_set_corrupt(ds, level); + goto out_freebp; + } + + fa = xfs_da3_header_check(blk->bp, dargs->owner); + if (fa) { + xchk_da_set_corrupt(ds, level); + goto out_freebp; + } + + /* + * If we've been handed a block that is below the dabtree root, does + * its hashval match what the parent block expected to see? + */ + if (level > 0) { + struct xfs_da_node_entry *key; + + key = xchk_da_btree_node_entry(ds, level - 1); + if (be32_to_cpu(key->hashval) != blk->hashval) { + xchk_da_set_corrupt(ds, level); + goto out_freebp; + } + } + +out: + return error; +out_freebp: + xfs_trans_brelse(dargs->trans, blk->bp); + blk->bp = NULL; +out_nobuf: + blk->blkno = 0; + return error; +} + +/* Visit all nodes and leaves of a da btree. */ +int +xchk_da_btree( + struct xfs_scrub *sc, + int whichfork, + xchk_da_btree_rec_fn scrub_fn, + void *private) +{ + struct xchk_da_btree *ds; + struct xfs_mount *mp = sc->mp; + struct xfs_da_state_blk *blks; + struct xfs_da_node_entry *key; + xfs_dablk_t blkno; + int level; + int error; + + /* Skip short format data structures; no btree to scan. */ + if (!xfs_ifork_has_extents(xfs_ifork_ptr(sc->ip, whichfork))) + return 0; + + /* Set up initial da state. */ + ds = kzalloc_obj(struct xchk_da_btree, XCHK_GFP_FLAGS); + if (!ds) + return -ENOMEM; + ds->dargs.dp = sc->ip; + ds->dargs.whichfork = whichfork; + ds->dargs.trans = sc->tp; + ds->dargs.op_flags = XFS_DA_OP_OKNOENT; + ds->dargs.owner = I_INO(sc->ip); + ds->state = xfs_da_state_alloc(&ds->dargs); + ds->sc = sc; + ds->private = private; + if (whichfork == XFS_ATTR_FORK) { + ds->dargs.geo = mp->m_attr_geo; + ds->lowest = 0; + ds->highest = 0; + } else { + ds->dargs.geo = mp->m_dir_geo; + ds->lowest = ds->dargs.geo->leafblk; + ds->highest = ds->dargs.geo->freeblk; + } + blkno = ds->lowest; + level = 0; + + /* Find the root of the da tree, if present. */ + blks = ds->state->path.blk; + error = xchk_da_btree_block(ds, level, blkno); + if (error) + goto out_state; + /* + * We didn't find a block at ds->lowest, which means that there's + * no LEAF1/LEAFN tree (at least not where it's supposed to be), + * so jump out now. + */ + if (blks[level].bp == NULL) + goto out_state; + + blks[level].index = 0; + while (level >= 0 && level < XFS_DA_NODE_MAXDEPTH) { + /* Handle leaf block. */ + if (blks[level].magic != XFS_DA_NODE_MAGIC) { + /* End of leaf, pop back towards the root. */ + if (blks[level].index >= ds->maxrecs[level]) { + if (level > 0) + blks[level - 1].index++; + ds->tree_level++; + level--; + continue; + } + + /* Dispatch record scrubbing. */ + error = scrub_fn(ds, level); + if (error) + break; + if (xchk_should_terminate(sc, &error) || + (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) + break; + + blks[level].index++; + continue; + } + + + /* End of node, pop back towards the root. */ + if (blks[level].index >= ds->maxrecs[level]) { + if (level > 0) + blks[level - 1].index++; + ds->tree_level++; + level--; + continue; + } + + /* Hashes in order for scrub? */ + key = xchk_da_btree_node_entry(ds, level); + error = xchk_da_btree_hash(ds, level, &key->hashval); + if (error) + goto out; + + /* Drill another level deeper. */ + blkno = be32_to_cpu(key->before); + level++; + if (level >= XFS_DA_NODE_MAXDEPTH) { + /* Too deep! */ + xchk_da_set_corrupt(ds, level - 1); + break; + } + ds->tree_level--; + error = xchk_da_btree_block(ds, level, blkno); + if (error) + goto out; + if (blks[level].bp == NULL) + goto out; + + blks[level].index = 0; + } + +out: + /* Release all the buffers we're tracking. */ + for (level = 0; level < XFS_DA_NODE_MAXDEPTH; level++) { + if (blks[level].bp == NULL) + continue; + xfs_trans_brelse(sc->tp, blks[level].bp); + blks[level].bp = NULL; + } + +out_state: + xfs_da_state_free(ds->state); + kfree(ds); + return error; +} diff --git a/fs/xfs/scrub/dabtree.h b/fs/xfs/scrub/dabtree.h new file mode 100644 index 000000000..de291e3b7 --- /dev/null +++ b/fs/xfs/scrub/dabtree.h @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_DABTREE_H__ +#define __XFS_SCRUB_DABTREE_H__ + +/* dir/attr btree */ + +struct xchk_da_btree { + struct xfs_da_args dargs; + xfs_dahash_t hashes[XFS_DA_NODE_MAXDEPTH]; + int maxrecs[XFS_DA_NODE_MAXDEPTH]; + struct xfs_da_state *state; + struct xfs_scrub *sc; + void *private; + + /* + * Lowest and highest directory block address in which we expect + * to find dir/attr btree node blocks. For a directory this + * (presumably) means between LEAF_OFFSET and FREE_OFFSET; for + * attributes there is no limit. + */ + xfs_dablk_t lowest; + xfs_dablk_t highest; + + int tree_level; +}; + +typedef int (*xchk_da_btree_rec_fn)(struct xchk_da_btree *ds, int level); + +/* Check for da btree operation errors. */ +bool xchk_da_process_error(struct xchk_da_btree *ds, int level, int *error); + +/* Check for da btree corruption. */ +void xchk_da_set_corrupt(struct xchk_da_btree *ds, int level); +void xchk_da_set_preen(struct xchk_da_btree *ds, int level); + +void xchk_da_set_preen(struct xchk_da_btree *ds, int level); + +int xchk_da_btree_hash(struct xchk_da_btree *ds, int level, __be32 *hashp); +int xchk_da_btree(struct xfs_scrub *sc, int whichfork, + xchk_da_btree_rec_fn scrub_fn, void *private); + +#endif /* __XFS_SCRUB_DABTREE_H__ */ diff --git a/fs/xfs/scrub/dir.c b/fs/xfs/scrub/dir.c new file mode 100644 index 000000000..2a037aae9 --- /dev/null +++ b/fs/xfs/scrub/dir.c @@ -0,0 +1,1183 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_inode.h" +#include "xfs_icache.h" +#include "xfs_dir2.h" +#include "xfs_dir2_priv.h" +#include "xfs_health.h" +#include "xfs_attr.h" +#include "xfs_parent.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/dabtree.h" +#include "scrub/readdir.h" +#include "scrub/health.h" +#include "scrub/repair.h" +#include "scrub/trace.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/xfblob.h" + +/* Set us up to scrub directories. */ +int +xchk_setup_directory( + struct xfs_scrub *sc) +{ + int error; + + if (xchk_could_repair(sc)) { + error = xrep_setup_directory(sc); + if (error) + return error; + } + + return xchk_setup_inode_contents(sc, 0); +} + +/* Directories */ + +/* Deferred directory entry that we saved for later. */ +struct xchk_dirent { + /* Cookie for retrieval of the dirent name. */ + xfblob_cookie name_cookie; + + /* Child inode number. */ + xfs_ino_t ino; + + /* Length of the pptr name. */ + uint8_t namelen; +}; + +struct xchk_dir { + struct xfs_scrub *sc; + + /* information for parent pointer validation. */ + struct xfs_parent_rec pptr_rec; + struct xfs_da_args pptr_args; + + /* Fixed-size array of xchk_dirent structures. */ + struct xfarray *dir_entries; + + /* Blobs containing dirent names. */ + struct xfblob *dir_names; + + /* If we've cycled the ILOCK, we must revalidate deferred dirents. */ + bool need_revalidate; + + /* Name buffer for dirent revalidation. */ + struct xfs_name xname; + uint8_t namebuf[MAXNAMELEN]; +}; + +/* Scrub a directory entry. */ + +/* Check that an inode's mode matches a given XFS_DIR3_FT_* type. */ +STATIC void +xchk_dir_check_ftype( + struct xfs_scrub *sc, + xfs_fileoff_t offset, + struct xfs_inode *ip, + int ftype) +{ + struct xfs_mount *mp = sc->mp; + + if (!xfs_has_ftype(mp)) { + if (ftype != XFS_DIR3_FT_UNKNOWN && ftype != XFS_DIR3_FT_DIR) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset); + return; + } + + if (xfs_mode_to_ftype(VFS_I(ip)->i_mode) != ftype) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset); + + /* + * Metadata and regular inodes cannot cross trees. This property + * cannot change without a full inode free and realloc cycle, so it's + * safe to check this without holding locks. + */ + if (xfs_is_metadir_inode(ip) != xfs_is_metadir_inode(sc->ip)) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset); +} + +/* + * Try to lock a child file for checking parent pointers. Returns the inode + * flags for the locks we now hold, or zero if we failed. + */ +STATIC unsigned int +xchk_dir_lock_child( + struct xfs_scrub *sc, + struct xfs_inode *ip) +{ + if (!xfs_ilock_nowait(ip, XFS_IOLOCK_SHARED)) + return 0; + + if (!xfs_ilock_nowait(ip, XFS_ILOCK_SHARED)) { + xfs_iunlock(ip, XFS_IOLOCK_SHARED); + return 0; + } + + if (!xfs_inode_has_attr_fork(ip) || !xfs_need_iread_extents(&ip->i_af)) + return XFS_IOLOCK_SHARED | XFS_ILOCK_SHARED; + + xfs_iunlock(ip, XFS_ILOCK_SHARED); + + if (!xfs_ilock_nowait(ip, XFS_ILOCK_EXCL)) { + xfs_iunlock(ip, XFS_IOLOCK_SHARED); + return 0; + } + + return XFS_IOLOCK_SHARED | XFS_ILOCK_EXCL; +} + +/* Check the backwards link (parent pointer) associated with this dirent. */ +STATIC int +xchk_dir_parent_pointer( + struct xchk_dir *sd, + const struct xfs_name *name, + struct xfs_inode *ip) +{ + struct xfs_scrub *sc = sd->sc; + int error; + + xfs_inode_to_parent_rec(&sd->pptr_rec, sc->ip); + error = xfs_parent_lookup(sc->tp, ip, name, &sd->pptr_rec, + &sd->pptr_args); + if (error == -ENOATTR) + xchk_fblock_xref_set_corrupt(sc, XFS_DATA_FORK, 0); + + return 0; +} + +/* Look for a parent pointer matching this dirent, if the child isn't busy. */ +STATIC int +xchk_dir_check_pptr_fast( + struct xchk_dir *sd, + xfs_dir2_dataptr_t dapos, + const struct xfs_name *name, + struct xfs_inode *ip) +{ + struct xfs_scrub *sc = sd->sc; + unsigned int lockmode; + int error; + + /* dot and dotdot entries do not have parent pointers */ + if (xfs_dir2_samename(name, &xfs_name_dot) || + xfs_dir2_samename(name, &xfs_name_dotdot)) + return 0; + + /* No self-referential non-dot or dotdot dirents. */ + if (ip == sc->ip) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); + return -ECANCELED; + } + + /* Try to lock the inode. */ + lockmode = xchk_dir_lock_child(sc, ip); + if (!lockmode) { + struct xchk_dirent save_de = { + .namelen = name->len, + .ino = I_INO(ip), + }; + + /* Couldn't lock the inode, so save the dirent for later. */ + trace_xchk_dir_defer(sc->ip, name, I_INO(ip)); + + error = xfblob_storename(sd->dir_names, &save_de.name_cookie, + name); + if (!xchk_fblock_xref_process_error(sc, XFS_DATA_FORK, 0, + &error)) + return error; + + error = xfarray_append(sd->dir_entries, &save_de); + if (!xchk_fblock_xref_process_error(sc, XFS_DATA_FORK, 0, + &error)) + return error; + + return 0; + } + + error = xchk_dir_parent_pointer(sd, name, ip); + xfs_iunlock(ip, lockmode); + return error; +} + +/* + * Scrub a single directory entry. + * + * Check the inode number to make sure it's sane, then we check that we can + * look up this filename. Finally, we check the ftype. + */ +STATIC int +xchk_dir_actor( + struct xfs_scrub *sc, + struct xfs_inode *dp, + xfs_dir2_dataptr_t dapos, + const struct xfs_name *name, + xfs_ino_t ino, + void *priv) +{ + struct xfs_mount *mp = dp->i_mount; + struct xfs_inode *ip; + struct xchk_dir *sd = priv; + xfs_ino_t lookup_ino; + xfs_dablk_t offset; + int error = 0; + + offset = xfs_dir2_db_to_da(mp->m_dir_geo, + xfs_dir2_dataptr_to_db(mp->m_dir_geo, dapos)); + + if (xchk_should_terminate(sc, &error)) + return error; + + /* Does this inode number make sense? */ + if (!xfs_verify_dir_ino(mp, ino)) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset); + return -ECANCELED; + } + + /* Does this name make sense? */ + if (!xfs_dir2_namecheck(name->name, name->len)) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset); + return -ECANCELED; + } + + if (xfs_dir2_samename(name, &xfs_name_dot)) { + /* If this is "." then check that the inum matches the dir. */ + if (ino != I_INO(dp)) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset); + } else if (xfs_dir2_samename(name, &xfs_name_dotdot)) { + /* + * If this is ".." in the root inode, check that the inum + * matches this dir. + */ + if (xchk_inode_is_dirtree_root(dp) && ino != I_INO(dp)) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset); + } + + /* Verify that we can look up this name by hash. */ + error = xchk_dir_lookup(sc, dp, name, &lookup_ino); + /* ENOENT means the hash lookup failed and the dir is corrupt */ + if (error == -ENOENT) + error = -EFSCORRUPTED; + if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, offset, &error)) + goto out; + if (lookup_ino != ino) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset); + return -ECANCELED; + } + + /* + * Grab the inode pointed to by the dirent. We release the inode + * before we cancel the scrub transaction. + * + * If _iget returns -EINVAL or -ENOENT then the child inode number is + * garbage and the directory is corrupt. If the _iget returns + * -EFSCORRUPTED or -EFSBADCRC then the child is corrupt which is a + * cross referencing error. Any other error is an operational error. + */ + error = xchk_iget(sc, ino, &ip); + if (error == -EINVAL || error == -ENOENT) { + error = -EFSCORRUPTED; + xchk_fblock_process_error(sc, XFS_DATA_FORK, 0, &error); + goto out; + } + if (!xchk_fblock_xref_process_error(sc, XFS_DATA_FORK, offset, &error)) + goto out; + + xchk_dir_check_ftype(sc, offset, ip, name->type); + + if (xfs_has_parent(mp)) { + error = xchk_dir_check_pptr_fast(sd, dapos, name, ip); + if (error) + goto out_rele; + } + +out_rele: + xchk_irele(sc, ip); +out: + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return -ECANCELED; + return error; +} + +/* Scrub a directory btree record. */ +STATIC int +xchk_dir_rec( + struct xchk_da_btree *ds, + int level) +{ + struct xfs_name dname = { }; + struct xfs_da_state_blk *blk = &ds->state->path.blk[level]; + struct xfs_mount *mp = ds->state->mp; + struct xfs_inode *dp = ds->dargs.dp; + struct xfs_da_geometry *geo = mp->m_dir_geo; + struct xfs_dir2_data_entry *dent; + struct xfs_buf *bp; + struct xfs_dir2_leaf_entry *ent; + unsigned int end; + unsigned int iter_off; + xfs_ino_t ino; + xfs_dablk_t rec_bno; + xfs_dir2_db_t db; + xfs_dir2_data_aoff_t off; + xfs_dir2_dataptr_t ptr; + xfs_dahash_t calc_hash; + xfs_dahash_t hash; + struct xfs_dir3_icleaf_hdr hdr; + unsigned int tag; + int error; + + ASSERT(blk->magic == XFS_DIR2_LEAF1_MAGIC || + blk->magic == XFS_DIR2_LEAFN_MAGIC); + + xfs_dir2_leaf_hdr_from_disk(mp, &hdr, blk->bp->b_addr); + ent = hdr.ents + blk->index; + + /* Check the hash of the entry. */ + error = xchk_da_btree_hash(ds, level, &ent->hashval); + if (error) + goto out; + + /* Valid hash pointer? */ + ptr = be32_to_cpu(ent->address); + if (ptr == 0) + return 0; + + /* Find the directory entry's location. */ + db = xfs_dir2_dataptr_to_db(geo, ptr); + off = xfs_dir2_dataptr_to_off(geo, ptr); + rec_bno = xfs_dir2_db_to_da(geo, db); + + if (rec_bno >= geo->leafblk) { + xchk_da_set_corrupt(ds, level); + goto out; + } + error = xfs_dir3_data_read(ds->dargs.trans, dp, ds->dargs.owner, + rec_bno, XFS_DABUF_MAP_HOLE_OK, &bp); + if (!xchk_fblock_process_error(ds->sc, XFS_DATA_FORK, rec_bno, + &error)) + goto out; + if (!bp) { + xchk_fblock_set_corrupt(ds->sc, XFS_DATA_FORK, rec_bno); + goto out; + } + xchk_buffer_recheck(ds->sc, bp); + + if (ds->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out_relse; + + dent = bp->b_addr + off; + + /* Make sure we got a real directory entry. */ + iter_off = geo->data_entry_offset; + end = xfs_dir3_data_end_offset(geo, bp->b_addr); + if (!end) { + xchk_fblock_set_corrupt(ds->sc, XFS_DATA_FORK, rec_bno); + goto out_relse; + } + for (;;) { + struct xfs_dir2_data_entry *dep = bp->b_addr + iter_off; + struct xfs_dir2_data_unused *dup = bp->b_addr + iter_off; + + if (iter_off >= end) { + xchk_fblock_set_corrupt(ds->sc, XFS_DATA_FORK, rec_bno); + goto out_relse; + } + + if (be16_to_cpu(dup->freetag) == XFS_DIR2_DATA_FREE_TAG) { + iter_off += be16_to_cpu(dup->length); + continue; + } + if (dep == dent) + break; + iter_off += xfs_dir2_data_entsize(mp, dep->namelen); + } + + /* Retrieve the entry, sanity check it, and compare hashes. */ + ino = be64_to_cpu(dent->inumber); + hash = be32_to_cpu(ent->hashval); + tag = be16_to_cpup(xfs_dir2_data_entry_tag_p(mp, dent)); + if (!xfs_verify_dir_ino(mp, ino) || tag != off) + xchk_fblock_set_corrupt(ds->sc, XFS_DATA_FORK, rec_bno); + if (dent->namelen == 0) { + xchk_fblock_set_corrupt(ds->sc, XFS_DATA_FORK, rec_bno); + goto out_relse; + } + + /* Does the directory hash match? */ + dname.name = dent->name; + dname.len = dent->namelen; + calc_hash = xfs_dir2_hashname(mp, &dname); + if (calc_hash != hash) + xchk_fblock_set_corrupt(ds->sc, XFS_DATA_FORK, rec_bno); + +out_relse: + xfs_trans_brelse(ds->dargs.trans, bp); +out: + return error; +} + +/* + * Is this unused entry either in the bestfree or smaller than all of + * them? We've already checked that the bestfrees are sorted longest to + * shortest, and that there aren't any bogus entries. + */ +STATIC void +xchk_directory_check_free_entry( + struct xfs_scrub *sc, + xfs_dablk_t lblk, + struct xfs_dir2_data_free *bf, + struct xfs_dir2_data_unused *dup) +{ + struct xfs_dir2_data_free *dfp; + unsigned int dup_length; + + dup_length = be16_to_cpu(dup->length); + + /* Unused entry is shorter than any of the bestfrees */ + if (dup_length < be16_to_cpu(bf[XFS_DIR2_DATA_FD_COUNT - 1].length)) + return; + + for (dfp = &bf[XFS_DIR2_DATA_FD_COUNT - 1]; dfp >= bf; dfp--) + if (dup_length == be16_to_cpu(dfp->length)) + return; + + /* Unused entry should be in the bestfrees but wasn't found. */ + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); +} + +/* Check free space info in a directory data block. */ +STATIC int +xchk_directory_data_bestfree( + struct xfs_scrub *sc, + xfs_dablk_t lblk, + bool is_block) +{ + struct xfs_dir2_data_unused *dup; + struct xfs_dir2_data_free *dfp; + struct xfs_buf *bp; + struct xfs_dir2_data_free *bf; + struct xfs_mount *mp = sc->mp; + u16 tag; + unsigned int nr_bestfrees = 0; + unsigned int nr_frees = 0; + unsigned int smallest_bestfree; + int newlen; + unsigned int offset; + unsigned int end; + int error; + + if (is_block) { + /* dir block format */ + if (lblk != XFS_B_TO_FSBT(mp, XFS_DIR2_DATA_OFFSET)) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); + error = xfs_dir3_block_read(sc->tp, sc->ip, I_INO(sc->ip), &bp); + } else { + /* dir data format */ + error = xfs_dir3_data_read(sc->tp, sc->ip, I_INO(sc->ip), lblk, + 0, &bp); + } + if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, lblk, &error)) + goto out; + xchk_buffer_recheck(sc, bp); + + if (xfs_has_crc(sc->mp)) { + struct xfs_dir3_data_hdr *hdr3 = bp->b_addr; + + if (hdr3->pad) + xchk_fblock_set_preen(sc, XFS_DATA_FORK, lblk); + } + + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out_buf; + + /* Do the bestfrees correspond to actual free space? */ + bf = xfs_dir2_data_bestfree_p(mp, bp->b_addr); + smallest_bestfree = UINT_MAX; + for (dfp = &bf[0]; dfp < &bf[XFS_DIR2_DATA_FD_COUNT]; dfp++) { + offset = be16_to_cpu(dfp->offset); + if (offset == 0) + continue; + if (offset >= mp->m_dir_geo->blksize) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); + goto out_buf; + } + dup = bp->b_addr + offset; + tag = be16_to_cpu(*xfs_dir2_data_unused_tag_p(dup)); + + /* bestfree doesn't match the entry it points at? */ + if (dup->freetag != cpu_to_be16(XFS_DIR2_DATA_FREE_TAG) || + be16_to_cpu(dup->length) != be16_to_cpu(dfp->length) || + tag != offset) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); + goto out_buf; + } + + /* bestfree records should be ordered largest to smallest */ + if (smallest_bestfree < be16_to_cpu(dfp->length)) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); + goto out_buf; + } + + smallest_bestfree = be16_to_cpu(dfp->length); + nr_bestfrees++; + } + + /* Make sure the bestfrees are actually the best free spaces. */ + offset = mp->m_dir_geo->data_entry_offset; + end = xfs_dir3_data_end_offset(mp->m_dir_geo, bp->b_addr); + + /* Iterate the entries, stopping when we hit or go past the end. */ + while (offset < end) { + dup = bp->b_addr + offset; + + /* Skip real entries */ + if (dup->freetag != cpu_to_be16(XFS_DIR2_DATA_FREE_TAG)) { + struct xfs_dir2_data_entry *dep = bp->b_addr + offset; + + newlen = xfs_dir2_data_entsize(mp, dep->namelen); + if (newlen <= 0) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, + lblk); + goto out_buf; + } + offset += newlen; + continue; + } + + /* Spot check this free entry */ + tag = be16_to_cpu(*xfs_dir2_data_unused_tag_p(dup)); + if (tag != offset) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); + goto out_buf; + } + + /* + * Either this entry is a bestfree or it's smaller than + * any of the bestfrees. + */ + xchk_directory_check_free_entry(sc, lblk, bf, dup); + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out_buf; + + /* Move on. */ + newlen = be16_to_cpu(dup->length); + if (newlen <= 0) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); + goto out_buf; + } + offset += newlen; + if (offset <= end) + nr_frees++; + } + + /* We're required to fill all the space. */ + if (offset != end) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); + + /* Did we see at least as many free slots as there are bestfrees? */ + if (nr_frees < nr_bestfrees) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); +out_buf: + xfs_trans_brelse(sc->tp, bp); +out: + return error; +} + +/* + * Does the free space length in the free space index block ($len) match + * the longest length in the directory data block's bestfree array? + * Assume that we've already checked that the data block's bestfree + * array is in order. + */ +STATIC void +xchk_directory_check_freesp( + struct xfs_scrub *sc, + xfs_dablk_t lblk, + struct xfs_buf *dbp, + unsigned int len) +{ + struct xfs_dir2_data_free *dfp; + + dfp = xfs_dir2_data_bestfree_p(sc->mp, dbp->b_addr); + + if (len != be16_to_cpu(dfp->length)) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); + + if (len > 0 && be16_to_cpu(dfp->offset) == 0) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); +} + +/* Check free space info in a directory leaf1 block. */ +STATIC int +xchk_directory_leaf1_bestfree( + struct xfs_scrub *sc, + struct xfs_da_args *args, + xfs_dir2_db_t last_data_db, + xfs_dablk_t lblk) +{ + struct xfs_dir3_icleaf_hdr leafhdr; + struct xfs_dir2_leaf_tail *ltp; + struct xfs_dir2_leaf *leaf; + struct xfs_buf *dbp; + struct xfs_buf *bp; + struct xfs_da_geometry *geo = sc->mp->m_dir_geo; + __be16 *bestp; + __u16 best; + __u32 hash; + __u32 lasthash = 0; + __u32 bestcount; + unsigned int stale = 0; + int i; + int error; + + /* Read the free space block. */ + error = xfs_dir3_leaf_read(sc->tp, sc->ip, I_INO(sc->ip), lblk, &bp); + if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, lblk, &error)) + return error; + xchk_buffer_recheck(sc, bp); + + leaf = bp->b_addr; + xfs_dir2_leaf_hdr_from_disk(sc->ip->i_mount, &leafhdr, leaf); + ltp = xfs_dir2_leaf_tail_p(geo, leaf); + bestcount = be32_to_cpu(ltp->bestcount); + bestp = xfs_dir2_leaf_bests_p(ltp); + + if (xfs_has_crc(sc->mp)) { + struct xfs_dir3_leaf_hdr *hdr3 = bp->b_addr; + + if (hdr3->pad != cpu_to_be32(0)) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); + } + + /* + * There must be enough bestfree slots to cover all the directory data + * blocks that we scanned. It is possible for there to be a hole + * between the last data block and i_disk_size. This seems like an + * oversight to the scrub author, but as we have been writing out + * directories like this (and xfs_repair doesn't mind them) for years, + * that's what we have to check. + */ + if (bestcount != last_data_db + 1) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); + goto out; + } + + /* Is the leaf count even remotely sane? */ + if (leafhdr.count > geo->leaf_max_ents) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); + goto out; + } + + /* Leaves and bests don't overlap in leaf format. */ + if ((char *)&leafhdr.ents[leafhdr.count] > (char *)bestp) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); + goto out; + } + + /* Check hash value order, count stale entries. */ + for (i = 0; i < leafhdr.count; i++) { + hash = be32_to_cpu(leafhdr.ents[i].hashval); + if (i > 0 && lasthash > hash) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); + lasthash = hash; + if (leafhdr.ents[i].address == + cpu_to_be32(XFS_DIR2_NULL_DATAPTR)) + stale++; + } + if (leafhdr.stale != stale) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out; + + /* Check all the bestfree entries. */ + for (i = 0; i < bestcount; i++, bestp++) { + best = be16_to_cpu(*bestp); + error = xfs_dir3_data_read(sc->tp, sc->ip, args->owner, + xfs_dir2_db_to_da(args->geo, i), + XFS_DABUF_MAP_HOLE_OK, &dbp); + if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, lblk, + &error)) + break; + + if (!dbp) { + if (best != NULLDATAOFF) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, + lblk); + break; + } + continue; + } + + if (best == NULLDATAOFF) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); + else + xchk_directory_check_freesp(sc, lblk, dbp, best); + xfs_trans_brelse(sc->tp, dbp); + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + break; + } +out: + xfs_trans_brelse(sc->tp, bp); + return error; +} + +/* Check free space info in a directory freespace block. */ +STATIC int +xchk_directory_free_bestfree( + struct xfs_scrub *sc, + struct xfs_da_args *args, + xfs_dablk_t lblk) +{ + struct xfs_dir3_icfree_hdr freehdr; + struct xfs_buf *dbp; + struct xfs_buf *bp; + __u16 best; + unsigned int stale = 0; + int i; + int error; + + /* Read the free space block */ + error = xfs_dir2_free_read(sc->tp, sc->ip, I_INO(sc->ip), lblk, &bp); + if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, lblk, &error)) + return error; + xchk_buffer_recheck(sc, bp); + + if (xfs_has_crc(sc->mp)) { + struct xfs_dir3_free_hdr *hdr3 = bp->b_addr; + + if (hdr3->pad != cpu_to_be32(0)) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); + } + + /* Check all the entries. */ + xfs_dir2_free_hdr_from_disk(sc->ip->i_mount, &freehdr, bp->b_addr); + for (i = 0; i < freehdr.nvalid; i++) { + best = be16_to_cpu(freehdr.bests[i]); + if (best == NULLDATAOFF) { + stale++; + continue; + } + error = xfs_dir3_data_read(sc->tp, sc->ip, args->owner, + (freehdr.firstdb + i) * args->geo->fsbcount, + 0, &dbp); + if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, lblk, + &error)) + goto out; + xchk_directory_check_freesp(sc, lblk, dbp, best); + xfs_trans_brelse(sc->tp, dbp); + } + + if (freehdr.nused + stale != freehdr.nvalid) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); +out: + xfs_trans_brelse(sc->tp, bp); + return error; +} + +/* Check free space information in directories. */ +STATIC int +xchk_directory_blocks( + struct xfs_scrub *sc) +{ + struct xfs_bmbt_irec got; + struct xfs_da_args args = { + .dp = sc->ip, + .whichfork = XFS_DATA_FORK, + .geo = sc->mp->m_dir_geo, + .trans = sc->tp, + .owner = I_INO(sc->ip), + }; + struct xfs_ifork *ifp = xfs_ifork_ptr(sc->ip, XFS_DATA_FORK); + struct xfs_mount *mp = sc->mp; + xfs_fileoff_t leaf_lblk; + xfs_fileoff_t free_lblk; + xfs_fileoff_t lblk; + struct xfs_iext_cursor icur; + xfs_dablk_t dabno; + xfs_dir2_db_t last_data_db = 0; + bool found; + bool is_block = false; + int error; + + /* Ignore local format directories. */ + if (ifp->if_format != XFS_DINODE_FMT_EXTENTS && + ifp->if_format != XFS_DINODE_FMT_BTREE) + return 0; + + lblk = XFS_B_TO_FSB(mp, XFS_DIR2_DATA_OFFSET); + leaf_lblk = XFS_B_TO_FSB(mp, XFS_DIR2_LEAF_OFFSET); + free_lblk = XFS_B_TO_FSB(mp, XFS_DIR2_FREE_OFFSET); + + /* Is this a block dir? */ + if (xfs_dir2_format(&args, &error) == XFS_DIR2_FMT_BLOCK) + is_block = true; + if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, lblk, &error)) + goto out; + + /* Iterate all the data extents in the directory... */ + found = xfs_iext_lookup_extent(sc->ip, ifp, lblk, &icur, &got); + while (found && !(sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) { + /* No more data blocks... */ + if (got.br_startoff >= leaf_lblk) + break; + + /* + * Check each data block's bestfree data. + * + * Iterate all the fsbcount-aligned block offsets in + * this directory. The directory block reading code is + * smart enough to do its own bmap lookups to handle + * discontiguous directory blocks. When we're done + * with the extent record, re-query the bmap at the + * next fsbcount-aligned offset to avoid redundant + * block checks. + */ + for (lblk = roundup((xfs_dablk_t)got.br_startoff, + args.geo->fsbcount); + lblk < got.br_startoff + got.br_blockcount; + lblk += args.geo->fsbcount) { + last_data_db = xfs_dir2_da_to_db(args.geo, lblk); + error = xchk_directory_data_bestfree(sc, lblk, + is_block); + if (error) + goto out; + } + dabno = got.br_startoff + got.br_blockcount; + lblk = roundup(dabno, args.geo->fsbcount); + found = xfs_iext_lookup_extent(sc->ip, ifp, lblk, &icur, &got); + } + + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out; + + /* Look for a leaf1 block, which has free info. */ + if (xfs_iext_lookup_extent(sc->ip, ifp, leaf_lblk, &icur, &got) && + got.br_startoff == leaf_lblk && + got.br_blockcount == args.geo->fsbcount && + !xfs_iext_next_extent(ifp, &icur, &got)) { + if (is_block) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); + goto out; + } + error = xchk_directory_leaf1_bestfree(sc, &args, last_data_db, + leaf_lblk); + if (error) + goto out; + } + + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out; + + /* Scan for free blocks */ + lblk = free_lblk; + found = xfs_iext_lookup_extent(sc->ip, ifp, lblk, &icur, &got); + while (found && !(sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) { + /* + * Dirs can't have blocks mapped above 2^32. + * Single-block dirs shouldn't even be here. + */ + lblk = got.br_startoff; + if (lblk & ~0xFFFFFFFFULL) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); + goto out; + } + if (is_block) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, lblk); + goto out; + } + + /* + * Check each dir free block's bestfree data. + * + * Iterate all the fsbcount-aligned block offsets in + * this directory. The directory block reading code is + * smart enough to do its own bmap lookups to handle + * discontiguous directory blocks. When we're done + * with the extent record, re-query the bmap at the + * next fsbcount-aligned offset to avoid redundant + * block checks. + */ + for (lblk = roundup((xfs_dablk_t)got.br_startoff, + args.geo->fsbcount); + lblk < got.br_startoff + got.br_blockcount; + lblk += args.geo->fsbcount) { + error = xchk_directory_free_bestfree(sc, &args, + lblk); + if (error) + goto out; + } + dabno = got.br_startoff + got.br_blockcount; + lblk = roundup(dabno, args.geo->fsbcount); + found = xfs_iext_lookup_extent(sc->ip, ifp, lblk, &icur, &got); + } +out: + return error; +} + +/* + * Revalidate a dirent that we collected in the past but couldn't check because + * of lock contention. Returns 0 if the dirent is still valid, -ENOENT if it + * has gone away on us, or a negative errno. + */ +STATIC int +xchk_dir_revalidate_dirent( + struct xchk_dir *sd, + const struct xfs_name *xname, + xfs_ino_t ino) +{ + struct xfs_scrub *sc = sd->sc; + xfs_ino_t child_ino; + int error; + + /* + * Look up the directory entry. If we get -ENOENT, the directory entry + * went away and there's nothing to revalidate. Return any other + * error. + */ + error = xchk_dir_lookup(sc, sc->ip, xname, &child_ino); + if (error) + return error; + + /* The inode number changed, nothing to revalidate. */ + if (ino != child_ino) + return -ENOENT; + + return 0; +} + +/* + * Check a directory entry's parent pointers the slow way, which means we cycle + * locks a bunch and put up with revalidation until we get it done. + */ +STATIC int +xchk_dir_slow_dirent( + struct xchk_dir *sd, + struct xchk_dirent *dirent, + const struct xfs_name *xname) +{ + struct xfs_scrub *sc = sd->sc; + struct xfs_inode *ip; + unsigned int lockmode; + int error; + + /* Check that the deferred dirent still exists. */ + if (sd->need_revalidate) { + error = xchk_dir_revalidate_dirent(sd, xname, dirent->ino); + if (error == -ENOENT) + return 0; + if (!xchk_fblock_xref_process_error(sc, XFS_DATA_FORK, 0, + &error)) + return error; + } + + error = xchk_iget(sc, dirent->ino, &ip); + if (error == -EINVAL || error == -ENOENT) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); + return 0; + } + if (!xchk_fblock_xref_process_error(sc, XFS_DATA_FORK, 0, &error)) + return error; + + /* + * If we can grab both IOLOCK and ILOCK of the alleged child, we can + * proceed with the validation. + */ + lockmode = xchk_dir_lock_child(sc, ip); + if (lockmode) { + trace_xchk_dir_slowpath(sc->ip, xname, I_INO(ip)); + goto check_pptr; + } + + /* + * We couldn't lock the child file. Drop all the locks and try to + * get them again, one at a time. + */ + xchk_iunlock(sc, sc->ilock_flags); + sd->need_revalidate = true; + + trace_xchk_dir_ultraslowpath(sc->ip, xname, I_INO(ip)); + + error = xchk_dir_trylock_for_pptrs(sc, ip, &lockmode); + if (error) + goto out_rele; + + /* Revalidate, since we just cycled the locks. */ + error = xchk_dir_revalidate_dirent(sd, xname, dirent->ino); + if (error == -ENOENT) { + error = 0; + goto out_unlock; + } + if (!xchk_fblock_xref_process_error(sc, XFS_DATA_FORK, 0, &error)) + goto out_unlock; + +check_pptr: + error = xchk_dir_parent_pointer(sd, xname, ip); +out_unlock: + xfs_iunlock(ip, lockmode); +out_rele: + xchk_irele(sc, ip); + return error; +} + +/* Check all the dirents that we deferred the first time around. */ +STATIC int +xchk_dir_finish_slow_dirents( + struct xchk_dir *sd) +{ + xfarray_idx_t array_cur; + int error; + + foreach_xfarray_idx(sd->dir_entries, array_cur) { + struct xchk_dirent dirent; + + if (sd->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return 0; + + error = xfarray_load(sd->dir_entries, array_cur, &dirent); + if (error) + return error; + + error = xfblob_loadname(sd->dir_names, dirent.name_cookie, + &sd->xname, dirent.namelen); + if (error) + return error; + + error = xchk_dir_slow_dirent(sd, &dirent, &sd->xname); + if (error) + return error; + } + + return 0; +} + +/* Scrub a whole directory. */ +int +xchk_directory( + struct xfs_scrub *sc) +{ + struct xchk_dir *sd; + int error; + + if (!S_ISDIR(VFS_I(sc->ip)->i_mode)) + return -ENOENT; + + if (xchk_file_looks_zapped(sc, XFS_SICK_INO_DIR_ZAPPED)) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); + return 0; + } + + /* Plausible size? */ + if (sc->ip->i_disk_size < xfs_dir2_sf_hdr_size(0)) { + xchk_ip_set_corrupt(sc, sc->ip); + return 0; + } + + /* Check directory tree structure */ + error = xchk_da_btree(sc, XFS_DATA_FORK, xchk_dir_rec, NULL); + if (error) + return error; + + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return 0; + + /* Check the freespace. */ + error = xchk_directory_blocks(sc); + if (error) + return error; + + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return 0; + + sd = kvzalloc_obj(struct xchk_dir, XCHK_GFP_FLAGS); + if (!sd) + return -ENOMEM; + sd->sc = sc; + sd->xname.name = sd->namebuf; + + if (xfs_has_parent(sc->mp)) { + /* + * Set up some staging memory for dirents that we can't check + * due to locking contention. + */ + error = xfarray_create("slow directory entries", 0, + sizeof(struct xchk_dirent), &sd->dir_entries); + if (error) + goto out_sd; + + error = xfblob_create("slow directory entry names", + &sd->dir_names); + if (error) + goto out_entries; + } + + /* Look up every name in this directory by hash. */ + error = xchk_dir_walk(sc, sc->ip, xchk_dir_actor, sd); + if (error == -ECANCELED) + error = 0; + if (error) + goto out_names; + + if (xfs_has_parent(sc->mp)) { + error = xchk_dir_finish_slow_dirents(sd); + if (error == -ETIMEDOUT) { + /* Couldn't grab a lock, scrub was marked incomplete */ + error = 0; + goto out_names; + } + if (error) + goto out_names; + } + +out_names: + if (sd->dir_names) + xfblob_destroy(sd->dir_names); +out_entries: + if (sd->dir_entries) + xfarray_destroy(sd->dir_entries); +out_sd: + kvfree(sd); + if (error) + return error; + + /* If the dir is clean, it is clearly not zapped. */ + xchk_mark_healthy_if_clean(sc, XFS_SICK_INO_DIR_ZAPPED); + return 0; +} + +/* + * Decide if this directory has been zapped to satisfy the inode and ifork + * verifiers. Checking and repairing should be postponed until the directory + * is fixed. + */ +bool +xchk_dir_looks_zapped( + struct xfs_inode *dp) +{ + /* Repair zapped this dir's data fork a short time ago */ + if (xfs_ifork_zapped(dp, XFS_DATA_FORK)) + return true; + + /* + * If the dinode repair found a bad data fork, it will reset the fork + * to extents format with zero records and wait for the bmapbtd + * scrubber to reconstruct the block mappings. Directories always + * contain some content, so this is a clear sign of a zapped directory. + * The state checked by xfs_ifork_zapped is not persisted, so this is + * the secondary strategy if repairs are interrupted by a crash or an + * unmount. + */ + return dp->i_df.if_format == XFS_DINODE_FMT_EXTENTS && + dp->i_df.if_nextents == 0; +} diff --git a/fs/xfs/scrub/dir_repair.c b/fs/xfs/scrub/dir_repair.c new file mode 100644 index 000000000..1c088cfba --- /dev/null +++ b/fs/xfs/scrub/dir_repair.c @@ -0,0 +1,1963 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_bit.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_sb.h" +#include "xfs_inode.h" +#include "xfs_icache.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_dir2.h" +#include "xfs_dir2_priv.h" +#include "xfs_bmap.h" +#include "xfs_quota.h" +#include "xfs_bmap_btree.h" +#include "xfs_trans_space.h" +#include "xfs_bmap_util.h" +#include "xfs_exchmaps.h" +#include "xfs_exchrange.h" +#include "xfs_ag.h" +#include "xfs_parent.h" +#include "scrub/xfs_scrub.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/tempfile.h" +#include "scrub/tempexch.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/xfblob.h" +#include "scrub/iscan.h" +#include "scrub/readdir.h" +#include "scrub/reap.h" +#include "scrub/findparent.h" +#include "scrub/orphanage.h" +#include "scrub/listxattr.h" + +/* + * Directory Repair + * ================ + * + * We repair directories by reading the directory data blocks looking for + * directory entries that look salvageable (name passes verifiers, entry points + * to a valid allocated inode, etc). Each entry worth salvaging is stashed in + * memory, and the stashed entries are periodically replayed into a temporary + * directory to constrain memory use. Batching the construction of the + * temporary directory in this fashion reduces lock cycling of the directory + * being repaired and the temporary directory, and will later become important + * for parent pointer scanning. + * + * If parent pointers are enabled on this filesystem, we instead reconstruct + * the directory by visiting each parent pointer of each file in the filesystem + * and translating the relevant parent pointer records into dirents. In this + * case, it is advantageous to stash all directory entries created from parent + * pointers for a single child file before replaying them into the temporary + * directory. To save memory, the live filesystem scan reuses the findparent + * fields. Directory repair chooses either parent pointer scanning or + * directory entry salvaging, but not both. + * + * Directory entries added to the temporary directory do not elevate the link + * counts of the inodes found. When salvaging completes, the remaining stashed + * entries are replayed to the temporary directory. An atomic mapping exchange + * is used to commit the new directory blocks to the directory being repaired. + * This will disrupt readdir cursors. + * + * Locking Issues + * -------------- + * + * If /a, /a/b, and /c are all directories, the VFS does not take i_rwsem on + * /a/b for a "mv /a/b /c/" operation. This means that only b's ILOCK protects + * b's dotdot update. This is in contrast to every other dotdot update (link, + * remove, mkdir). If the repair code drops the ILOCK, it must either + * revalidate the dotdot entry or use dirent hooks to capture updates from + * other threads. + */ + +/* Create a dirent in the tempdir. */ +#define XREP_DIRENT_ADD (1) + +/* Remove a dirent from the tempdir. */ +#define XREP_DIRENT_REMOVE (2) + +/* Directory entry to be restored in the new directory. */ +struct xrep_dirent { + /* Cookie for retrieval of the dirent name. */ + xfblob_cookie name_cookie; + + /* Target inode number. */ + xfs_ino_t ino; + + /* Length of the dirent name. */ + uint8_t namelen; + + /* File type of the dirent. */ + uint8_t ftype; + + /* XREP_DIRENT_{ADD,REMOVE} */ + uint8_t action; +}; + +/* + * Stash up to 8 pages of recovered dirent data in dir_entries and dir_names + * before we write them to the temp dir. + */ +#define XREP_DIR_MAX_STASH_BYTES (PAGE_SIZE * 8) + +struct xrep_dir { + struct xfs_scrub *sc; + + /* Fixed-size array of xrep_dirent structures. */ + struct xfarray *dir_entries; + + /* Blobs containing directory entry names. */ + struct xfblob *dir_names; + + /* Information for exchanging data forks at the end. */ + struct xrep_tempexch tx; + + /* Preallocated args struct for performing dir operations */ + struct xfs_da_args args; + + /* + * Information used to scan the filesystem to find the inumber of the + * dotdot entry for this directory. For directory salvaging when + * parent pointers are not enabled, we use the findparent_* functions + * on this object and access only the parent_ino field directly. + * + * When parent pointers are enabled, however, the pptr scanner uses the + * iscan, hooks, lock, and parent_ino fields of this object directly. + * @pscan.lock coordinates access to dir_entries, dir_names, + * parent_ino, subdirs, dirents, and args. This reduces the memory + * requirements of this structure. + */ + struct xrep_parent_scan_info pscan; + + /* + * Context information for attaching this directory to the lost+found + * if this directory does not have a parent. + */ + struct xrep_adoption adoption; + + /* How many subdirectories did we find? */ + uint64_t subdirs; + + /* How many dirents did we find? */ + unsigned int dirents; + + /* Should we move this directory to the orphanage? */ + bool needs_adoption; + + /* Directory entry name, plus the trailing null. */ + struct xfs_name xname; + unsigned char namebuf[MAXNAMELEN]; +}; + +/* Tear down all the incore stuff we created. */ +static void +xrep_dir_teardown( + struct xfs_scrub *sc) +{ + struct xrep_dir *rd = sc->buf; + + xrep_findparent_scan_teardown(&rd->pscan); + if (rd->dir_names) + xfblob_destroy(rd->dir_names); + rd->dir_names = NULL; + if (rd->dir_entries) + xfarray_destroy(rd->dir_entries); + rd->dir_entries = NULL; +} + +/* Set up for a directory repair. */ +int +xrep_setup_directory( + struct xfs_scrub *sc) +{ + struct xrep_dir *rd; + int error; + + xchk_fsgates_enable(sc, XCHK_FSGATES_DIRENTS); + + error = xrep_orphanage_try_create(sc); + if (error) + return error; + + error = xrep_tempfile_create(sc, S_IFDIR); + if (error) + return error; + + rd = kvzalloc_obj(struct xrep_dir, XCHK_GFP_FLAGS); + if (!rd) + return -ENOMEM; + rd->sc = sc; + rd->xname.name = rd->namebuf; + sc->buf = rd; + + return 0; +} + +/* + * Look up the dotdot entry and confirm that it's really the parent. + * Returns NULLFSINO if we don't know what to do. + */ +static inline xfs_ino_t +xrep_dir_lookup_parent( + struct xrep_dir *rd) +{ + struct xfs_scrub *sc = rd->sc; + xfs_ino_t ino; + int error; + + error = xfs_dir_lookup(sc->tp, sc->ip, &xfs_name_dotdot, &ino, NULL); + if (error) + return NULLFSINO; + if (!xfs_verify_dir_ino(sc->mp, ino)) + return NULLFSINO; + + error = xrep_findparent_confirm(sc, &ino); + if (error) + return NULLFSINO; + + return ino; +} + +/* + * Look up '..' in the dentry cache and confirm that it's really the parent. + * Returns NULLFSINO if the dcache misses or if the hit is implausible. + */ +static inline xfs_ino_t +xrep_dir_dcache_parent( + struct xrep_dir *rd) +{ + struct xfs_scrub *sc = rd->sc; + xfs_ino_t parent_ino; + int error; + + parent_ino = xrep_findparent_from_dcache(sc); + if (parent_ino == NULLFSINO) + return parent_ino; + + error = xrep_findparent_confirm(sc, &parent_ino); + if (error) + return NULLFSINO; + + return parent_ino; +} + +/* Try to find the parent of the directory being repaired. */ +STATIC int +xrep_dir_find_parent( + struct xrep_dir *rd) +{ + xfs_ino_t ino; + + ino = xrep_findparent_self_reference(rd->sc); + if (ino != NULLFSINO) { + xrep_findparent_scan_finish_early(&rd->pscan, ino); + return 0; + } + + ino = xrep_dir_dcache_parent(rd); + if (ino != NULLFSINO) { + xrep_findparent_scan_finish_early(&rd->pscan, ino); + return 0; + } + + ino = xrep_dir_lookup_parent(rd); + if (ino != NULLFSINO) { + xrep_findparent_scan_finish_early(&rd->pscan, ino); + return 0; + } + + /* + * A full filesystem scan is the last resort. On a busy filesystem, + * the scan can fail with -EBUSY if we cannot grab IOLOCKs. That means + * that we don't know what who the parent is, so we should return to + * userspace. + */ + return xrep_findparent_scan(&rd->pscan); +} + +/* + * Decide if we want to salvage this entry. We don't bother with oversized + * names or the dot entry. + */ +STATIC int +xrep_dir_want_salvage( + struct xrep_dir *rd, + const char *name, + int namelen, + xfs_ino_t ino) +{ + struct xfs_mount *mp = rd->sc->mp; + + /* No pointers to ourselves or to garbage. */ + if (ino == I_INO(rd->sc->ip)) + return false; + if (!xfs_verify_dir_ino(mp, ino)) + return false; + + /* No weird looking names or dot entries. */ + if (namelen >= MAXNAMELEN || namelen <= 0) + return false; + if (namelen == 1 && name[0] == '.') + return false; + if (!xfs_dir2_namecheck(name, namelen)) + return false; + + return true; +} + +/* + * Remember that we want to create a dirent in the tempdir. These stashed + * actions will be replayed later. + */ +STATIC int +xrep_dir_stash_createname( + struct xrep_dir *rd, + const struct xfs_name *name, + xfs_ino_t ino) +{ + struct xrep_dirent dirent = { + .action = XREP_DIRENT_ADD, + .ino = ino, + .namelen = name->len, + .ftype = name->type, + }; + int error; + + trace_xrep_dir_stash_createname(rd->sc->tempip, name, ino); + + error = xfblob_storename(rd->dir_names, &dirent.name_cookie, name); + if (error) + return error; + + return xfarray_append(rd->dir_entries, &dirent); +} + +/* + * Remember that we want to remove a dirent from the tempdir. These stashed + * actions will be replayed later. + */ +STATIC int +xrep_dir_stash_removename( + struct xrep_dir *rd, + const struct xfs_name *name, + xfs_ino_t ino) +{ + struct xrep_dirent dirent = { + .action = XREP_DIRENT_REMOVE, + .ino = ino, + .namelen = name->len, + .ftype = name->type, + }; + int error; + + trace_xrep_dir_stash_removename(rd->sc->tempip, name, ino); + + error = xfblob_storename(rd->dir_names, &dirent.name_cookie, name); + if (error) + return error; + + return xfarray_append(rd->dir_entries, &dirent); +} + +/* Allocate an in-core record to hold entries while we rebuild the dir data. */ +STATIC int +xrep_dir_salvage_entry( + struct xrep_dir *rd, + unsigned char *name, + unsigned int namelen, + xfs_ino_t ino) +{ + struct xfs_name xname = { + .name = name, + }; + struct xfs_scrub *sc = rd->sc; + struct xfs_inode *ip; + unsigned int i = 0; + int error = 0; + + if (xchk_should_terminate(sc, &error)) + return error; + + /* + * Truncate the name to the first character that would trip namecheck. + * If we no longer have a name after that, ignore this entry. + */ + while (i < namelen && name[i] != 0 && name[i] != '/') + i++; + if (i == 0) + return 0; + xname.len = i; + + /* Ignore '..' entries; we already picked the new parent. */ + if (xname.len == 2 && name[0] == '.' && name[1] == '.') { + trace_xrep_dir_salvaged_parent(sc->ip, ino); + return 0; + } + + trace_xrep_dir_salvage_entry(sc->ip, &xname, ino); + + /* + * Compute the ftype or dump the entry if we can't. We don't lock the + * inode because inodes can't change type while we have a reference. + */ + error = xchk_iget(sc, ino, &ip); + if (error) + return 0; + + /* Don't mix metadata and regular directory trees. */ + if (xfs_is_metadir_inode(ip) != xfs_is_metadir_inode(rd->sc->ip)) { + xchk_irele(sc, ip); + return 0; + } + + xname.type = xfs_mode_to_ftype(VFS_I(ip)->i_mode); + xchk_irele(sc, ip); + + return xrep_dir_stash_createname(rd, &xname, ino); +} + +/* Record a shortform directory entry for later reinsertion. */ +STATIC int +xrep_dir_salvage_sf_entry( + struct xrep_dir *rd, + struct xfs_dir2_sf_hdr *sfp, + struct xfs_dir2_sf_entry *sfep) +{ + xfs_ino_t ino; + + ino = xfs_dir2_sf_get_ino(rd->sc->mp, sfp, sfep); + if (!xrep_dir_want_salvage(rd, sfep->name, sfep->namelen, ino)) + return 0; + + return xrep_dir_salvage_entry(rd, sfep->name, sfep->namelen, ino); +} + +/* Record a regular directory entry for later reinsertion. */ +STATIC int +xrep_dir_salvage_data_entry( + struct xrep_dir *rd, + struct xfs_dir2_data_entry *dep) +{ + xfs_ino_t ino; + + ino = be64_to_cpu(dep->inumber); + if (!xrep_dir_want_salvage(rd, dep->name, dep->namelen, ino)) + return 0; + + return xrep_dir_salvage_entry(rd, dep->name, dep->namelen, ino); +} + +/* Try to recover block/data format directory entries. */ +STATIC int +xrep_dir_recover_data( + struct xrep_dir *rd, + struct xfs_buf *bp) +{ + struct xfs_da_geometry *geo = rd->sc->mp->m_dir_geo; + unsigned int offset; + unsigned int end; + int error = 0; + + /* + * Loop over the data portion of the block. + * Each object is a real entry (dep) or an unused one (dup). + */ + offset = geo->data_entry_offset; + end = min_t(unsigned int, BBTOB(bp->b_length), + xfs_dir3_data_end_offset(geo, bp->b_addr)); + + while (offset < end) { + struct xfs_dir2_data_unused *dup = bp->b_addr + offset; + struct xfs_dir2_data_entry *dep = bp->b_addr + offset; + + if (xchk_should_terminate(rd->sc, &error)) + return error; + + /* Skip unused entries. */ + if (be16_to_cpu(dup->freetag) == XFS_DIR2_DATA_FREE_TAG) { + offset += be16_to_cpu(dup->length); + continue; + } + + /* Don't walk off the end of the block. */ + offset += xfs_dir2_data_entsize(rd->sc->mp, dep->namelen); + if (offset > end) + break; + + /* Ok, let's save this entry. */ + error = xrep_dir_salvage_data_entry(rd, dep); + if (error) + return error; + + } + + return 0; +} + +/* Try to recover shortform directory entries. */ +STATIC int +xrep_dir_recover_sf( + struct xrep_dir *rd) +{ + struct xfs_dir2_sf_hdr *hdr; + struct xfs_dir2_sf_entry *sfep; + struct xfs_dir2_sf_entry *next; + struct xfs_ifork *ifp; + xfs_ino_t ino; + unsigned char *end; + int error = 0; + + ifp = xfs_ifork_ptr(rd->sc->ip, XFS_DATA_FORK); + hdr = ifp->if_data; + end = (unsigned char *)ifp->if_data + ifp->if_bytes; + + ino = xfs_dir2_sf_get_parent_ino(hdr); + trace_xrep_dir_salvaged_parent(rd->sc->ip, ino); + + sfep = xfs_dir2_sf_firstentry(hdr); + while ((unsigned char *)sfep < end) { + if (xchk_should_terminate(rd->sc, &error)) + return error; + + next = xfs_dir2_sf_nextentry(rd->sc->mp, hdr, sfep); + if ((unsigned char *)next > end) + break; + + /* Ok, let's save this entry. */ + error = xrep_dir_salvage_sf_entry(rd, hdr, sfep); + if (error) + return error; + + sfep = next; + } + + return 0; +} + +/* + * Try to figure out the format of this directory from the data fork mappings + * and the directory size. If we can be reasonably sure of format, we can be + * more aggressive in salvaging directory entries. On return, @magic_guess + * will be set to DIR3_BLOCK_MAGIC if we think this is a "block format" + * directory; DIR3_DATA_MAGIC if we think this is a "data format" directory, + * and 0 if we can't tell. + */ +STATIC void +xrep_dir_guess_format( + struct xrep_dir *rd, + __be32 *magic_guess) +{ + struct xfs_inode *dp = rd->sc->ip; + struct xfs_mount *mp = rd->sc->mp; + struct xfs_da_geometry *geo = mp->m_dir_geo; + xfs_fileoff_t last; + int error; + + ASSERT(xfs_has_crc(mp)); + + *magic_guess = 0; + + /* + * If there's a single directory block and the directory size is + * exactly one block, this has to be a single block format directory. + */ + error = xfs_bmap_last_offset(dp, &last, XFS_DATA_FORK); + if (!error && XFS_FSB_TO_B(mp, last) == geo->blksize && + dp->i_disk_size == geo->blksize) { + *magic_guess = cpu_to_be32(XFS_DIR3_BLOCK_MAGIC); + return; + } + + /* + * If the last extent before the leaf offset matches the directory + * size and the directory size is larger than 1 block, this is a + * data format directory. + */ + last = geo->leafblk; + error = xfs_bmap_last_before(rd->sc->tp, dp, &last, XFS_DATA_FORK); + if (!error && + XFS_FSB_TO_B(mp, last) > geo->blksize && + XFS_FSB_TO_B(mp, last) == dp->i_disk_size) { + *magic_guess = cpu_to_be32(XFS_DIR3_DATA_MAGIC); + return; + } +} + +/* Recover directory entries from a specific directory block. */ +STATIC int +xrep_dir_recover_dirblock( + struct xrep_dir *rd, + __be32 magic_guess, + xfs_dablk_t dabno) +{ + struct xfs_dir2_data_hdr *hdr; + struct xfs_buf *bp; + __be32 oldmagic; + int error; + + /* + * Try to read buffer. We invalidate them in the next step so we don't + * bother to set a buffer type or ops. + */ + error = xfs_da_read_buf(rd->sc->tp, rd->sc->ip, dabno, + XFS_DABUF_MAP_HOLE_OK, &bp, XFS_DATA_FORK, NULL); + if (error || !bp) + return error; + + hdr = bp->b_addr; + oldmagic = hdr->magic; + + trace_xrep_dir_recover_dirblock(rd->sc->ip, dabno, + be32_to_cpu(hdr->magic), be32_to_cpu(magic_guess)); + + /* + * If we're sure of the block's format, proceed with the salvage + * operation using the specified magic number. + */ + if (magic_guess) { + hdr->magic = magic_guess; + goto recover; + } + + /* + * If we couldn't guess what type of directory this is, then we will + * only salvage entries from directory blocks that match the magic + * number and pass verifiers. + */ + switch (hdr->magic) { + case cpu_to_be32(XFS_DIR2_BLOCK_MAGIC): + case cpu_to_be32(XFS_DIR3_BLOCK_MAGIC): + if (!xrep_buf_verify_struct(bp, &xfs_dir3_block_buf_ops)) + goto out; + if (xfs_dir3_block_header_check(bp, I_INO(rd->sc->ip)) != NULL) + goto out; + break; + case cpu_to_be32(XFS_DIR2_DATA_MAGIC): + case cpu_to_be32(XFS_DIR3_DATA_MAGIC): + if (!xrep_buf_verify_struct(bp, &xfs_dir3_data_buf_ops)) + goto out; + if (xfs_dir3_data_header_check(bp, I_INO(rd->sc->ip)) != NULL) + goto out; + break; + default: + goto out; + } + +recover: + error = xrep_dir_recover_data(rd, bp); + +out: + hdr->magic = oldmagic; + xfs_trans_brelse(rd->sc->tp, bp); + return error; +} + +static inline void +xrep_dir_init_args( + struct xrep_dir *rd, + struct xfs_inode *dp, + const struct xfs_name *name) +{ + memset(&rd->args, 0, sizeof(struct xfs_da_args)); + rd->args.geo = rd->sc->mp->m_dir_geo; + rd->args.whichfork = XFS_DATA_FORK; + rd->args.owner = I_INO(rd->sc->ip); + rd->args.trans = rd->sc->tp; + rd->args.dp = dp; + if (!name) + return; + rd->args.name = name->name; + rd->args.namelen = name->len; + rd->args.filetype = name->type; + rd->args.hashval = xfs_dir2_hashname(rd->sc->mp, name); +} + +/* Replay a stashed createname into the temporary directory. */ +STATIC int +xrep_dir_replay_createname( + struct xrep_dir *rd, + const struct xfs_name *name, + xfs_ino_t inum, + xfs_extlen_t total) +{ + struct xfs_scrub *sc = rd->sc; + struct xfs_inode *dp = rd->sc->tempip; + int error; + + ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); + + error = xfs_dir_ino_validate(sc->mp, inum); + if (error) + return error; + + trace_xrep_dir_replay_createname(dp, name, inum); + + xrep_dir_init_args(rd, dp, name); + rd->args.inumber = inum; + rd->args.total = total; + rd->args.op_flags = XFS_DA_OP_ADDNAME | XFS_DA_OP_OKNOENT; + return xfs_dir_createname_args(&rd->args); +} + +/* Replay a stashed removename onto the temporary directory. */ +STATIC int +xrep_dir_replay_removename( + struct xrep_dir *rd, + const struct xfs_name *name, + xfs_extlen_t total) +{ + struct xfs_inode *dp = rd->args.dp; + + ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); + + xrep_dir_init_args(rd, dp, name); + rd->args.op_flags = 0; + rd->args.total = total; + + trace_xrep_dir_replay_removename(dp, name, 0); + return xfs_dir_removename_args(&rd->args); +} + +/* + * Add this stashed incore directory entry to the temporary directory. + * The caller must hold the tempdir's IOLOCK, must not hold any ILOCKs, and + * must not be in transaction context. + */ +STATIC int +xrep_dir_replay_update( + struct xrep_dir *rd, + const struct xfs_name *xname, + const struct xrep_dirent *dirent) +{ + struct xfs_mount *mp = rd->sc->mp; +#ifdef DEBUG + xfs_ino_t ino; +#endif + uint resblks; + int error; + + resblks = xfs_link_space_res(mp, xname->len); + error = xchk_trans_alloc(rd->sc, resblks); + if (error) + return error; + + /* Lock the temporary directory and join it to the transaction */ + xrep_tempfile_ilock(rd->sc); + xfs_trans_ijoin(rd->sc->tp, rd->sc->tempip, 0); + + switch (dirent->action) { + case XREP_DIRENT_ADD: + /* + * Create a replacement dirent in the temporary directory. + * Note that _createname doesn't check for existing entries. + * There shouldn't be any in the temporary dir, but we'll + * verify this in debug mode. + */ +#ifdef DEBUG + error = xchk_dir_lookup(rd->sc, rd->sc->tempip, xname, &ino); + if (error != -ENOENT) { + ASSERT(error != -ENOENT); + goto out_cancel; + } +#endif + + error = xrep_dir_replay_createname(rd, xname, dirent->ino, + resblks); + if (error) + goto out_cancel; + + if (xname->type == XFS_DIR3_FT_DIR) + rd->subdirs++; + rd->dirents++; + break; + case XREP_DIRENT_REMOVE: + /* + * Remove a dirent from the temporary directory. Note that + * _removename doesn't check the inode target of the exist + * entry. There should be a perfect match in the temporary + * dir, but we'll verify this in debug mode. + */ +#ifdef DEBUG + error = xchk_dir_lookup(rd->sc, rd->sc->tempip, xname, &ino); + if (error) { + ASSERT(error != 0); + goto out_cancel; + } + if (ino != dirent->ino) { + ASSERT(ino == dirent->ino); + error = -EIO; + goto out_cancel; + } +#endif + + error = xrep_dir_replay_removename(rd, xname, resblks); + if (error) + goto out_cancel; + + if (xname->type == XFS_DIR3_FT_DIR) + rd->subdirs--; + rd->dirents--; + break; + default: + ASSERT(0); + error = -EIO; + goto out_cancel; + } + + /* Commit and unlock. */ + error = xrep_trans_commit(rd->sc); + if (error) + return error; + + xrep_tempfile_iunlock(rd->sc); + return 0; +out_cancel: + xchk_trans_cancel(rd->sc); + xrep_tempfile_iunlock(rd->sc); + return error; +} + +/* + * Flush stashed incore dirent updates that have been recorded by the scanner. + * This is done to reduce the memory requirements of the directory rebuild, + * since directories can contain up to 32GB of directory data. + * + * Caller must not hold transactions or ILOCKs. Caller must hold the tempdir + * IOLOCK. + */ +STATIC int +xrep_dir_replay_updates( + struct xrep_dir *rd) +{ + xfarray_idx_t array_cur; + int error; + + /* Add all the salvaged dirents to the temporary directory. */ + mutex_lock(&rd->pscan.lock); + foreach_xfarray_idx(rd->dir_entries, array_cur) { + struct xrep_dirent dirent; + + error = xfarray_load(rd->dir_entries, array_cur, &dirent); + if (error) + goto out_unlock; + + error = xfblob_loadname(rd->dir_names, dirent.name_cookie, + &rd->xname, dirent.namelen); + if (error) + goto out_unlock; + rd->xname.type = dirent.ftype; + mutex_unlock(&rd->pscan.lock); + + error = xrep_dir_replay_update(rd, &rd->xname, &dirent); + if (error) + return error; + mutex_lock(&rd->pscan.lock); + } + + /* Empty out both arrays now that we've added the entries. */ + xfarray_truncate(rd->dir_entries); + xfblob_truncate(rd->dir_names); + mutex_unlock(&rd->pscan.lock); + return 0; +out_unlock: + mutex_unlock(&rd->pscan.lock); + return error; +} + +/* + * Periodically flush stashed directory entries to the temporary dir. This + * is done to reduce the memory requirements of the directory rebuild, since + * directories can contain up to 32GB of directory data. + */ +STATIC int +xrep_dir_flush_stashed( + struct xrep_dir *rd) +{ + int error; + + /* + * Entering this function, the scrub context has a reference to the + * inode being repaired, the temporary file, and a scrub transaction + * that we use during dirent salvaging to avoid livelocking if there + * are cycles in the directory structures. We hold ILOCK_EXCL on both + * the inode being repaired and the temporary file, though they are + * not ijoined to the scrub transaction. + * + * To constrain kernel memory use, we occasionally write salvaged + * dirents from the xfarray and xfblob structures into the temporary + * directory in preparation for exchanging the directory structures at + * the end. Updating the temporary file requires a transaction, so we + * commit the scrub transaction and drop the two ILOCKs so that + * we can allocate whatever transaction we want. + * + * We still hold IOLOCK_EXCL on the inode being repaired, which + * prevents anyone from accessing the damaged directory data while we + * repair it. + */ + error = xrep_trans_commit(rd->sc); + if (error) + return error; + xchk_iunlock(rd->sc, XFS_ILOCK_EXCL); + + /* + * Take the IOLOCK of the temporary file while we modify dirents. This + * isn't strictly required because the temporary file is never revealed + * to userspace, but we follow the same locking rules. We still hold + * sc->ip's IOLOCK. + */ + error = xrep_tempfile_iolock_polled(rd->sc); + if (error) + return error; + + /* Write to the tempdir all the updates that we've stashed. */ + error = xrep_dir_replay_updates(rd); + xrep_tempfile_iounlock(rd->sc); + if (error) + return error; + + /* + * Recreate the salvage transaction and relock the dir we're salvaging. + */ + error = xchk_trans_alloc(rd->sc, 0); + if (error) + return error; + xchk_ilock(rd->sc, XFS_ILOCK_EXCL); + return 0; +} + +/* Decide if we've stashed too much dirent data in memory. */ +static inline bool +xrep_dir_want_flush_stashed( + struct xrep_dir *rd) +{ + unsigned long long bytes; + + bytes = xfarray_bytes(rd->dir_entries) + xfblob_bytes(rd->dir_names); + return bytes > XREP_DIR_MAX_STASH_BYTES; +} + +/* Extract as many directory entries as we can. */ +STATIC int +xrep_dir_recover( + struct xrep_dir *rd) +{ + struct xfs_bmbt_irec got; + struct xfs_scrub *sc = rd->sc; + struct xfs_da_geometry *geo = sc->mp->m_dir_geo; + xfs_fileoff_t offset; + xfs_dablk_t dabno; + __be32 magic_guess; + int nmap; + int error; + + xrep_dir_guess_format(rd, &magic_guess); + + /* Iterate each directory data block in the data fork. */ + for (offset = 0; + offset < geo->leafblk; + offset = got.br_startoff + got.br_blockcount) { + nmap = 1; + error = xfs_bmapi_read(sc->ip, offset, geo->leafblk - offset, + &got, &nmap, 0); + if (error) + return error; + if (nmap != 1) + return -EFSCORRUPTED; + if (!xfs_bmap_is_written_extent(&got)) + continue; + + for (dabno = round_up(got.br_startoff, geo->fsbcount); + dabno < got.br_startoff + got.br_blockcount; + dabno += geo->fsbcount) { + if (xchk_should_terminate(rd->sc, &error)) + return error; + + error = xrep_dir_recover_dirblock(rd, + magic_guess, dabno); + if (error) + return error; + + /* Flush dirents to constrain memory usage. */ + if (xrep_dir_want_flush_stashed(rd)) { + error = xrep_dir_flush_stashed(rd); + if (error) + return error; + } + } + } + + return 0; +} + +/* + * Find all the directory entries for this inode by scraping them out of the + * directory leaf blocks by hand, and flushing them into the temp dir. + */ +STATIC int +xrep_dir_find_entries( + struct xrep_dir *rd) +{ + struct xfs_inode *dp = rd->sc->ip; + int error; + + /* + * Salvage directory entries from the old directory, and write them to + * the temporary directory. + */ + if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) { + error = xrep_dir_recover_sf(rd); + } else { + error = xfs_iread_extents(rd->sc->tp, dp, XFS_DATA_FORK); + if (error) + return error; + + error = xrep_dir_recover(rd); + } + if (error) + return error; + + return xrep_dir_flush_stashed(rd); +} + +/* Scan all files in the filesystem for dirents. */ +STATIC int +xrep_dir_salvage_entries( + struct xrep_dir *rd) +{ + struct xfs_scrub *sc = rd->sc; + int error; + + /* + * Drop the ILOCK on this directory so that we can scan for this + * directory's parent. Figure out who is going to be the parent of + * this directory, then retake the ILOCK so that we can salvage + * directory entries. + */ + xchk_iunlock(sc, XFS_ILOCK_EXCL); + error = xrep_dir_find_parent(rd); + xchk_ilock(sc, XFS_ILOCK_EXCL); + if (error) + return error; + + /* + * Collect directory entries by parsing raw leaf blocks to salvage + * whatever we can. When we're done, free the staging memory before + * exchanging the directories to reduce memory usage. + */ + error = xrep_dir_find_entries(rd); + if (error) + return error; + + /* + * Cancel the repair transaction and drop the ILOCK so that we can + * (later) use the atomic mapping exchange functions to compute the + * correct block reservations and re-lock the inodes. + * + * We still hold IOLOCK_EXCL (aka i_rwsem) which will prevent directory + * modifications, but there's nothing to prevent userspace from reading + * the directory until we're ready for the exchange operation. Reads + * will return -EIO without shutting down the fs, so we're ok with + * that. + * + * The VFS can change dotdot on us, but the findparent scan will keep + * our incore parent inode up to date. See the note on locking issues + * for more details. + */ + error = xrep_trans_commit(sc); + if (error) + return error; + + xchk_iunlock(sc, XFS_ILOCK_EXCL); + return 0; +} + + +/* + * Examine a parent pointer of a file. If it leads us back to the directory + * that we're rebuilding, create an incore dirent from the parent pointer and + * stash it. + */ +STATIC int +xrep_dir_scan_pptr( + struct xfs_scrub *sc, + struct xfs_inode *ip, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + const void *value, + unsigned int valuelen, + void *priv) +{ + struct xfs_name xname = { + .name = name, + .len = namelen, + .type = xfs_mode_to_ftype(VFS_I(ip)->i_mode), + }; + xfs_ino_t parent_ino; + uint32_t parent_gen; + struct xrep_dir *rd = priv; + int error; + + if (!(attr_flags & XFS_ATTR_PARENT)) + return 0; + + /* + * Ignore parent pointers that point back to a different dir, list the + * wrong generation number, or are invalid. + */ + error = xfs_parent_from_attr(sc->mp, attr_flags, name, namelen, value, + valuelen, &parent_ino, &parent_gen); + if (error) + return error; + + if (parent_ino != I_INO(sc->ip) || + parent_gen != VFS_I(sc->ip)->i_generation) + return 0; + + mutex_lock(&rd->pscan.lock); + error = xrep_dir_stash_createname(rd, &xname, I_INO(ip)); + mutex_unlock(&rd->pscan.lock); + return error; +} + +/* + * If this child dirent points to the directory being repaired, remember that + * fact so that we can reset the dotdot entry if necessary. + */ +STATIC int +xrep_dir_scan_dirent( + struct xfs_scrub *sc, + struct xfs_inode *dp, + xfs_dir2_dataptr_t dapos, + const struct xfs_name *name, + xfs_ino_t ino, + void *priv) +{ + struct xrep_dir *rd = priv; + + /* Dirent doesn't point to this directory. */ + if (ino != I_INO(rd->sc->ip)) + return 0; + + /* Ignore garbage inum. */ + if (!xfs_verify_dir_ino(rd->sc->mp, ino)) + return 0; + + /* No weird looking names. */ + if (name->len >= MAXNAMELEN || name->len <= 0) + return 0; + + /* Don't pick up dot or dotdot entries; we only want child dirents. */ + if (xfs_dir2_samename(name, &xfs_name_dotdot) || + xfs_dir2_samename(name, &xfs_name_dot)) + return 0; + + trace_xrep_dir_stash_createname(sc->tempip, &xfs_name_dotdot, + I_INO(dp)); + + xrep_findparent_scan_found(&rd->pscan, I_INO(dp)); + return 0; +} + +/* + * Decide if we want to look for child dirents or parent pointers in this file. + * Skip the dir being repaired and any files being used to stage repairs. + */ +static inline bool +xrep_dir_want_scan( + struct xrep_dir *rd, + const struct xfs_inode *ip) +{ + return ip != rd->sc->ip && !xrep_is_tempfile(ip); +} + +/* + * Take ILOCK on a file that we want to scan. + * + * Select ILOCK_EXCL if the file is a directory with an unloaded data bmbt or + * has an unloaded attr bmbt. Otherwise, take ILOCK_SHARED. + */ +static inline unsigned int +xrep_dir_scan_ilock( + struct xrep_dir *rd, + struct xfs_inode *ip) +{ + uint lock_mode = XFS_ILOCK_SHARED; + + /* Need to take the shared ILOCK to advance the iscan cursor. */ + if (!xrep_dir_want_scan(rd, ip)) + goto lock; + + if (S_ISDIR(VFS_I(ip)->i_mode) && xfs_need_iread_extents(&ip->i_df)) { + lock_mode = XFS_ILOCK_EXCL; + goto lock; + } + + if (xfs_inode_has_attr_fork(ip) && xfs_need_iread_extents(&ip->i_af)) + lock_mode = XFS_ILOCK_EXCL; + +lock: + xfs_ilock(ip, lock_mode); + return lock_mode; +} + +/* + * Scan this file for relevant child dirents or parent pointers that point to + * the directory we're rebuilding. + */ +STATIC int +xrep_dir_scan_file( + struct xrep_dir *rd, + struct xfs_inode *ip) +{ + unsigned int lock_mode; + int error = 0; + + lock_mode = xrep_dir_scan_ilock(rd, ip); + + if (!xrep_dir_want_scan(rd, ip)) + goto scan_done; + + /* + * If the extended attributes look as though they has been zapped by + * the inode record repair code, we cannot scan for parent pointers. + */ + if (xchk_pptr_looks_zapped(ip)) { + error = -EBUSY; + goto scan_done; + } + + error = xchk_xattr_walk(rd->sc, ip, xrep_dir_scan_pptr, NULL, rd); + if (error) + goto scan_done; + + if (S_ISDIR(VFS_I(ip)->i_mode)) { + /* + * If the directory looks as though it has been zapped by the + * inode record repair code, we cannot scan for child dirents. + */ + if (xchk_dir_looks_zapped(ip)) { + error = -EBUSY; + goto scan_done; + } + + error = xchk_dir_walk(rd->sc, ip, xrep_dir_scan_dirent, rd); + if (error) + goto scan_done; + } + +scan_done: + xchk_iscan_mark_visited(&rd->pscan.iscan, ip); + xfs_iunlock(ip, lock_mode); + return error; +} + +/* + * Scan all files in the filesystem for parent pointers that we can turn into + * replacement dirents, and a dirent that we can use to set the dotdot pointer. + */ +STATIC int +xrep_dir_scan_dirtree( + struct xrep_dir *rd) +{ + struct xfs_scrub *sc = rd->sc; + struct xfs_inode *ip; + int error; + + /* Roots of directory trees are their own parents. */ + if (xchk_inode_is_dirtree_root(sc->ip)) + xrep_findparent_scan_found(&rd->pscan, I_INO(sc->ip)); + + /* + * Filesystem scans are time consuming. Drop the directory ILOCK and + * all other resources for the duration of the scan and hope for the + * best. The live update hooks will keep our scan information up to + * date even though we've dropped the locks. + */ + xchk_trans_cancel(sc); + if (sc->ilock_flags & (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)) + xchk_iunlock(sc, sc->ilock_flags & (XFS_ILOCK_SHARED | + XFS_ILOCK_EXCL)); + xchk_trans_alloc_empty(sc); + + while ((error = xchk_iscan_iter(&rd->pscan.iscan, &ip)) == 1) { + bool flush; + + error = xrep_dir_scan_file(rd, ip); + xchk_irele(sc, ip); + if (error) + break; + + /* Flush stashed dirent updates to constrain memory usage. */ + mutex_lock(&rd->pscan.lock); + flush = xrep_dir_want_flush_stashed(rd); + mutex_unlock(&rd->pscan.lock); + if (flush) { + xchk_trans_cancel(sc); + + error = xrep_tempfile_iolock_polled(sc); + if (error) + break; + + error = xrep_dir_replay_updates(rd); + xrep_tempfile_iounlock(sc); + if (error) + break; + + xchk_trans_alloc_empty(sc); + } + + if (xchk_should_terminate(sc, &error)) + break; + } + xchk_iscan_iter_finish(&rd->pscan.iscan); + if (error) { + /* + * If we couldn't grab an inode that was busy with a state + * change, change the error code so that we exit to userspace + * as quickly as possible. + */ + if (error == -EBUSY) + return -ECANCELED; + return error; + } + + /* + * Cancel the empty transaction so that we can (later) use the atomic + * file mapping exchange functions to lock files and commit the new + * directory. + */ + xchk_trans_cancel(rd->sc); + return 0; +} + +/* + * Capture dirent updates being made by other threads which are relevant to the + * directory being repaired. + */ +STATIC int +xrep_dir_live_update( + struct notifier_block *nb, + unsigned long action, + void *data) +{ + struct xfs_dir_update_params *p = data; + struct xrep_dir *rd; + struct xfs_scrub *sc; + int error = 0; + + rd = container_of(nb, struct xrep_dir, pscan.dhook.dirent_hook.nb); + sc = rd->sc; + + /* + * This thread updated a child dirent in the directory that we're + * rebuilding. Stash the update for replay against the temporary + * directory. + */ + if (I_INO(p->dp) == I_INO(sc->ip) && + xchk_iscan_want_live_update(&rd->pscan.iscan, I_INO(p->ip))) { + mutex_lock(&rd->pscan.lock); + if (p->delta > 0) + error = xrep_dir_stash_createname(rd, p->name, + I_INO(p->ip)); + else + error = xrep_dir_stash_removename(rd, p->name, + I_INO(p->ip)); + mutex_unlock(&rd->pscan.lock); + if (error) + goto out_abort; + } + + /* + * This thread updated another directory's child dirent that points to + * the directory that we're rebuilding, so remember the new dotdot + * target. + */ + if (I_INO(p->ip) == I_INO(sc->ip) && + xchk_iscan_want_live_update(&rd->pscan.iscan, I_INO(p->dp))) { + if (p->delta > 0) { + trace_xrep_dir_stash_createname(sc->tempip, + &xfs_name_dotdot, + I_INO(p->dp)); + + xrep_findparent_scan_found(&rd->pscan, I_INO(p->dp)); + } else { + trace_xrep_dir_stash_removename(sc->tempip, + &xfs_name_dotdot, + rd->pscan.parent_ino); + + xrep_findparent_scan_found(&rd->pscan, NULLFSINO); + } + } + + return NOTIFY_DONE; +out_abort: + xchk_iscan_abort(&rd->pscan.iscan); + return NOTIFY_DONE; +} + +/* + * Free all the directory blocks and reset the data fork. The caller must + * join the inode to the transaction. This function returns with the inode + * joined to a clean scrub transaction. + */ +STATIC int +xrep_dir_reset_fork( + struct xrep_dir *rd, + xfs_ino_t parent_ino) +{ + struct xfs_scrub *sc = rd->sc; + struct xfs_ifork *ifp = xfs_ifork_ptr(sc->tempip, XFS_DATA_FORK); + int error; + + /* Unmap all the directory buffers. */ + if (xfs_ifork_has_extents(ifp)) { + error = xrep_reap_ifork(sc, sc->tempip, XFS_DATA_FORK); + if (error) + return error; + } + + trace_xrep_dir_reset_fork(sc->tempip, parent_ino); + + /* Reset the data fork to an empty data fork. */ + xfs_idestroy_fork(ifp); + ifp->if_bytes = 0; + sc->tempip->i_disk_size = 0; + + /* Reinitialize the short form directory. */ + xrep_dir_init_args(rd, sc->tempip, NULL); + return xfs_dir2_sf_create(&rd->args, parent_ino); +} + +/* + * Prepare both inodes' directory forks for exchanging mappings. Promote the + * tempfile from short format to leaf format, and if the file being repaired + * has a short format data fork, turn it into an empty extent list. + */ +STATIC int +xrep_dir_swap_prep( + struct xfs_scrub *sc, + bool temp_local, + bool ip_local) +{ + int error; + + /* + * If the tempfile's directory is in shortform format, convert that to + * a single leaf extent so that we can use the atomic mapping exchange. + */ + if (temp_local) { + struct xfs_da_args args = { + .dp = sc->tempip, + .geo = sc->mp->m_dir_geo, + .whichfork = XFS_DATA_FORK, + .trans = sc->tp, + .total = 1, + .owner = I_INO(sc->ip), + }; + + error = xfs_dir2_sf_to_block(&args); + if (error) + return error; + + /* + * Roll the deferred log items to get us back to a clean + * transaction. + */ + error = xfs_defer_finish(&sc->tp); + if (error) + return error; + } + + /* + * If the file being repaired had a shortform data fork, convert that + * to an empty extent list in preparation for the atomic mapping + * exchange. + */ + if (ip_local) { + struct xfs_ifork *ifp; + + ifp = xfs_ifork_ptr(sc->ip, XFS_DATA_FORK); + xfs_idestroy_fork(ifp); + ifp->if_format = XFS_DINODE_FMT_EXTENTS; + ifp->if_nextents = 0; + ifp->if_bytes = 0; + ifp->if_data = NULL; + ifp->if_height = 0; + + xfs_trans_log_inode(sc->tp, sc->ip, + XFS_ILOG_CORE | XFS_ILOG_DDATA); + } + + return 0; +} + +/* + * Replace the inode number of a directory entry. + */ +static int +xrep_dir_replace( + struct xrep_dir *rd, + struct xfs_inode *dp, + const struct xfs_name *name, + xfs_ino_t inum, + xfs_extlen_t total) +{ + struct xfs_scrub *sc = rd->sc; + int error; + + ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); + + error = xfs_dir_ino_validate(sc->mp, inum); + if (error) + return error; + + xrep_dir_init_args(rd, dp, name); + rd->args.inumber = inum; + rd->args.total = total; + return xfs_dir_replace_args(&rd->args); +} + +/* + * Reset the link count of this directory and adjust the unlinked list pointers + * as needed. + */ +STATIC int +xrep_dir_set_nlink( + struct xrep_dir *rd) +{ + struct xfs_scrub *sc = rd->sc; + struct xfs_inode *dp = sc->ip; + struct xfs_perag *pag; + unsigned int new_nlink = min_t(unsigned long long, + rd->subdirs + 2, + XFS_NLINK_PINNED); + int error; + + /* + * The directory is not on the incore unlinked list, which means that + * it needs to be reachable via the directory tree. Update the nlink + * with our observed link count. If the directory has no parent, it + * will be moved to the orphanage. + */ + if (!xfs_inode_on_unlinked_list(dp)) + goto reset_nlink; + + /* + * The directory is on the unlinked list and we did not find any + * dirents. Set the link count to zero and let the directory + * inactivate when the last reference drops. + */ + if (rd->dirents == 0) { + rd->needs_adoption = false; + new_nlink = 0; + goto reset_nlink; + } + + /* + * The directory is on the unlinked list and we found dirents. This + * directory needs to be reachable via the directory tree. Remove the + * dir from the unlinked list and update nlink with the observed link + * count. If the directory has no parent, it will be moved to the + * orphanage. + */ + pag = xfs_perag_get(sc->mp, XFS_INODE_TO_AGNO(dp)); + if (!pag) { + ASSERT(0); + return -EFSCORRUPTED; + } + + error = xfs_iunlink_remove(sc->tp, pag, dp); + xfs_perag_put(pag); + if (error) + return error; + +reset_nlink: + if (VFS_I(dp)->i_nlink != new_nlink) + set_nlink(VFS_I(dp), new_nlink); + return 0; +} + +/* + * Finish replaying stashed dirent updates, allocate a transaction for + * exchanging data fork mappings, and take the ILOCKs of both directories + * before we commit the new directory structure. + */ +STATIC int +xrep_dir_finalize_tempdir( + struct xrep_dir *rd) +{ + struct xfs_scrub *sc = rd->sc; + int error; + + if (!xfs_has_parent(sc->mp)) + return xrep_tempexch_trans_alloc(sc, XFS_DATA_FORK, &rd->tx); + + /* + * Repair relies on the ILOCK to quiesce all possible dirent updates. + * Replay all queued dirent updates into the tempdir before exchanging + * the contents, even if that means dropping the ILOCKs and the + * transaction. + */ + do { + error = xrep_dir_replay_updates(rd); + if (error) + return error; + + error = xrep_tempexch_trans_alloc(sc, XFS_DATA_FORK, &rd->tx); + if (error) + return error; + + if (xfarray_length(rd->dir_entries) == 0) + break; + + xchk_trans_cancel(sc); + xrep_tempfile_iunlock_both(sc); + } while (!xchk_should_terminate(sc, &error)); + return error; +} + +/* Exchange the temporary directory's data fork with the one being repaired. */ +STATIC int +xrep_dir_swap( + struct xrep_dir *rd) +{ + struct xfs_scrub *sc = rd->sc; + xfs_ino_t ino; + bool ip_local, temp_local; + int error = 0; + + /* + * If we never found the parent for this directory, temporarily assign + * the root dir as the parent; we'll move this to the orphanage after + * exchanging the dir contents. We hold the ILOCK of the dir being + * repaired, so we're not worried about racy updates of dotdot. + */ + ASSERT(sc->ilock_flags & XFS_ILOCK_EXCL); + if (rd->pscan.parent_ino == NULLFSINO) { + rd->needs_adoption = true; + rd->pscan.parent_ino = rd->sc->mp->m_sb.sb_rootino; + } + + /* + * Reset the temporary directory's '..' entry to point to the parent + * that we found. The dirent replace code asserts if the dirent + * already points at the new inumber, so we look it up here. + * + * It's also possible that this replacement could also expand a sf + * tempdir into block format. + */ + error = xchk_dir_lookup(sc, rd->sc->tempip, &xfs_name_dotdot, &ino); + if (error) + return error; + + if (rd->pscan.parent_ino != ino) { + error = xrep_dir_replace(rd, rd->sc->tempip, &xfs_name_dotdot, + rd->pscan.parent_ino, rd->tx.req.resblks); + if (error) + return error; + } + + /* + * Changing the dot and dotdot entries could have changed the shape of + * the directory, so we recompute these. + */ + ip_local = sc->ip->i_df.if_format == XFS_DINODE_FMT_LOCAL; + temp_local = sc->tempip->i_df.if_format == XFS_DINODE_FMT_LOCAL; + + /* + * If the both files have a local format data fork and the rebuilt + * directory data would fit in the repaired file's data fork, copy + * the contents from the tempfile and update the directory link count. + * We're done now. + */ + if (ip_local && temp_local && + sc->tempip->i_disk_size <= xfs_inode_data_fork_size(sc->ip)) { + xrep_tempfile_copyout_local(sc, XFS_DATA_FORK); + return xrep_dir_set_nlink(rd); + } + + /* + * Clean the transaction before we start working on exchanging + * directory contents. + */ + error = xrep_tempfile_roll_trans(rd->sc); + if (error) + return error; + + /* Otherwise, make sure both data forks are in block-mapping mode. */ + error = xrep_dir_swap_prep(sc, temp_local, ip_local); + if (error) + return error; + + /* + * Set nlink of the directory in the same transaction sequence that + * (atomically) commits the new directory data. + */ + error = xrep_dir_set_nlink(rd); + if (error) + return error; + + return xrep_tempexch_contents(sc, &rd->tx); +} + +/* + * Exchange the new directory contents (which we created in the tempfile) with + * the directory being repaired. + */ +STATIC int +xrep_dir_rebuild_tree( + struct xrep_dir *rd) +{ + struct xfs_scrub *sc = rd->sc; + int error; + + trace_xrep_dir_rebuild_tree(sc->ip, rd->pscan.parent_ino); + + /* + * Take the IOLOCK on the temporary file so that we can run dir + * operations with the same locks held as we would for a normal file. + * We still hold sc->ip's IOLOCK. + */ + error = xrep_tempfile_iolock_polled(rd->sc); + if (error) + return error; + + /* + * Allocate transaction, lock inodes, and make sure that we've replayed + * all the stashed dirent updates to the tempdir. After this point, + * we're ready to exchange data fork mappings. + */ + error = xrep_dir_finalize_tempdir(rd); + if (error) + return error; + + if (xchk_iscan_aborted(&rd->pscan.iscan)) + return -ECANCELED; + + /* + * Exchange the tempdir's data fork with the file being repaired. This + * recreates the transaction and re-takes the ILOCK in the scrub + * context. + */ + error = xrep_dir_swap(rd); + if (error) + return error; + + /* + * Release the old directory blocks and reset the data fork of the temp + * directory to an empty shortform directory because inactivation does + * nothing for directories. + */ + error = xrep_dir_reset_fork(rd, I_INO(sc->mp->m_rootip)); + if (error) + return error; + + /* + * Roll to get a transaction without any inodes joined to it. Then we + * can drop the tempfile's ILOCK and IOLOCK before doing more work on + * the scrub target directory. + */ + error = xfs_trans_roll(&sc->tp); + if (error) + return error; + + xrep_tempfile_iunlock(sc); + xrep_tempfile_iounlock(sc); + return 0; +} + +/* Set up the filesystem scan so we can regenerate directory entries. */ +STATIC int +xrep_dir_setup_scan( + struct xrep_dir *rd) +{ + struct xfs_scrub *sc = rd->sc; + int error; + + /* Set up some staging memory for salvaging dirents. */ + error = xfarray_create("directory entries", 0, + sizeof(struct xrep_dirent), &rd->dir_entries); + if (error) + return error; + + error = xfblob_create("directory entry names", &rd->dir_names); + if (error) + goto out_xfarray; + + if (xfs_has_parent(sc->mp)) + error = __xrep_findparent_scan_start(sc, &rd->pscan, + xrep_dir_live_update); + else + error = xrep_findparent_scan_start(sc, &rd->pscan); + if (error) + goto out_xfblob; + + return 0; + +out_xfblob: + xfblob_destroy(rd->dir_names); + rd->dir_names = NULL; +out_xfarray: + xfarray_destroy(rd->dir_entries); + rd->dir_entries = NULL; + return error; +} + +/* + * Move the current file to the orphanage. + * + * Caller must hold IOLOCK_EXCL on @sc->ip, and no other inode locks. Upon + * successful return, the scrub transaction will have enough extra reservation + * to make the move; it will hold IOLOCK_EXCL and ILOCK_EXCL of @sc->ip and the + * orphanage; and both inodes will be ijoined. + */ +STATIC int +xrep_dir_move_to_orphanage( + struct xrep_dir *rd) +{ + struct xfs_scrub *sc = rd->sc; + xfs_ino_t orig_parent, new_parent; + int error; + + /* + * We are about to drop the ILOCK on sc->ip to lock the orphanage and + * prepare for the adoption. Therefore, look up the old dotdot entry + * for sc->ip so that we can compare it after we re-lock sc->ip. + */ + error = xchk_dir_lookup(sc, sc->ip, &xfs_name_dotdot, &orig_parent); + if (error) + return error; + + /* + * Drop the ILOCK on the scrub target and commit the transaction. + * Adoption computes its own resource requirements and gathers the + * necessary components. + */ + error = xrep_trans_commit(sc); + if (error) + return error; + xchk_iunlock(sc, XFS_ILOCK_EXCL); + + /* If we can take the orphanage's iolock then we're ready to move. */ + if (!xrep_orphanage_ilock_nowait(sc, XFS_IOLOCK_EXCL)) { + xchk_iunlock(sc, sc->ilock_flags); + error = xrep_orphanage_iolock_two(sc); + if (error) + return error; + } + + /* Grab transaction and ILOCK the two files. */ + error = xrep_adoption_trans_alloc(sc, &rd->adoption); + if (error) + return error; + + error = xrep_adoption_compute_name(&rd->adoption, &rd->xname); + if (error) + return error; + + /* + * Now that we've reacquired the ILOCK on sc->ip, look up the dotdot + * entry again. If the parent changed or the child was unlinked while + * the child directory was unlocked, we don't need to move the child to + * the orphanage after all. + */ + error = xchk_dir_lookup(sc, sc->ip, &xfs_name_dotdot, &new_parent); + if (error) + return error; + + /* + * Attach to the orphanage if we still have a linked directory and it + * hasn't been moved. + */ + if (orig_parent == new_parent && VFS_I(sc->ip)->i_nlink > 0) { + error = xrep_adoption_move(&rd->adoption); + if (error) + return error; + } + + /* + * Launder the scrub transaction so we can drop the orphanage ILOCK + * and IOLOCK. Return holding the scrub target's ILOCK and IOLOCK. + */ + error = xrep_adoption_trans_roll(&rd->adoption); + if (error) + return error; + + xrep_orphanage_iunlock(sc, XFS_ILOCK_EXCL); + xrep_orphanage_iunlock(sc, XFS_IOLOCK_EXCL); + return 0; +} + +/* + * Repair the directory metadata. + * + * XXX: Directory entry buffers can be multiple fsblocks in size. The buffer + * cache in XFS can't handle aliased multiblock buffers, so this might + * misbehave if the directory blocks are crosslinked with other filesystem + * metadata. + * + * XXX: Is it necessary to check the dcache for this directory to make sure + * that we always recreate every cached entry? + */ +int +xrep_directory( + struct xfs_scrub *sc) +{ + struct xrep_dir *rd = sc->buf; + int error; + + /* The rmapbt is required to reap the old data fork. */ + if (!xfs_has_rmapbt(sc->mp)) + return -EOPNOTSUPP; + /* We require atomic file exchange range to rebuild anything. */ + if (!xfs_has_exchange_range(sc->mp)) + return -EOPNOTSUPP; + + error = xrep_dir_setup_scan(rd); + if (error) + return error; + + if (xfs_has_parent(sc->mp)) + error = xrep_dir_scan_dirtree(rd); + else + error = xrep_dir_salvage_entries(rd); + if (error) + goto out_teardown; + + /* Last chance to abort before we start committing fixes. */ + if (xchk_should_terminate(sc, &error)) + goto out_teardown; + + error = xrep_dir_rebuild_tree(rd); + if (error) + goto out_teardown; + + if (rd->needs_adoption) { + if (!xrep_orphanage_can_adopt(rd->sc)) + error = -EFSCORRUPTED; + else + error = xrep_dir_move_to_orphanage(rd); + if (error) + goto out_teardown; + } + +out_teardown: + xrep_dir_teardown(sc); + return error; +} diff --git a/fs/xfs/scrub/dirtree.c b/fs/xfs/scrub/dirtree.c new file mode 100644 index 000000000..b2cf6e543 --- /dev/null +++ b/fs/xfs/scrub/dirtree.c @@ -0,0 +1,1008 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2023-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_inode.h" +#include "xfs_icache.h" +#include "xfs_dir2.h" +#include "xfs_dir2_priv.h" +#include "xfs_attr.h" +#include "xfs_parent.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/bitmap.h" +#include "scrub/ino_bitmap.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/xfblob.h" +#include "scrub/listxattr.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/orphanage.h" +#include "scrub/dirtree.h" + +/* + * Directory Tree Structure Validation + * =================================== + * + * Validating the tree qualities of the directory tree structure can be + * difficult. If the tree is frozen, running a depth (or breadth) first search + * and marking a bitmap suffices to determine if there is a cycle. XORing the + * mark bitmap with the inode bitmap afterwards tells us if there are + * disconnected cycles. If the tree is not frozen, directory updates can move + * subtrees across the scanner wavefront, which complicates the design greatly. + * + * Directory parent pointers change that by enabling an incremental approach to + * validation of the tree structure. Instead of using one thread to scan the + * entire filesystem, we instead can have multiple threads walking individual + * subdirectories upwards to the root. In a perfect world, the IOLOCK would + * suffice to stabilize two directories in a parent -> child relationship. + * Unfortunately, the VFS does not take the IOLOCK when moving a child + * subdirectory, so we instead synchronize on ILOCK and use dirent update hooks + * to detect a race. If a race occurs in a path, we restart the scan. + * + * If the walk terminates without reaching the root, we know the path is + * disconnected and ought to be attached to the lost and found. If on the walk + * we find the same subdir that we're scanning, we know this is a cycle and + * should delete an incoming edge. If we find multiple paths to the root, we + * know to delete an incoming edge. + * + * There are two big hitches with this approach: first, all file link counts + * must be correct to prevent other writers from doing the wrong thing with the + * directory tree structure. Second, because we're walking upwards in a tree + * of arbitrary depth, we cannot hold all the ILOCKs. Instead, we will use a + * directory update hook to invalidate the scan results if one of the paths + * we've scanned has changed. + */ + +/* Clean up the dirtree checking resources. */ +STATIC void +xchk_dirtree_buf_cleanup( + void *buf) +{ + struct xchk_dirtree *dl = buf; + struct xchk_dirpath *path, *n; + + if (dl->scan_ino != NULLFSINO) + xfs_dir_hook_del(dl->sc->mp, &dl->dhook); + + xchk_dirtree_for_each_path_safe(dl, path, n) { + list_del_init(&path->list); + xino_bitmap_destroy(&path->seen_inodes); + kfree(path); + } + + if (dl->path_names) + xfblob_destroy(dl->path_names); + dl->path_names = NULL; + if (dl->path_steps) + xfarray_destroy(dl->path_steps); + dl->path_steps = NULL; + mutex_destroy(&dl->lock); +} + +/* Set us up to look for directory loops. */ +int +xchk_setup_dirtree( + struct xfs_scrub *sc) +{ + struct xchk_dirtree *dl; + int error; + + xchk_fsgates_enable(sc, XCHK_FSGATES_DIRENTS); + + if (xchk_could_repair(sc)) { + error = xrep_setup_dirtree(sc); + if (error) + return error; + } + + dl = kvzalloc_obj(struct xchk_dirtree, XCHK_GFP_FLAGS); + if (!dl) + return -ENOMEM; + dl->sc = sc; + dl->xname.name = dl->namebuf; + dl->hook_xname.name = dl->hook_namebuf; + INIT_LIST_HEAD(&dl->path_list); + dl->root_ino = NULLFSINO; + dl->scan_ino = NULLFSINO; + dl->parent_ino = NULLFSINO; + + mutex_init(&dl->lock); + + error = xfarray_create("dirtree path steps", 0, + sizeof(struct xchk_dirpath_step), &dl->path_steps); + if (error) + goto out_dl; + + error = xfblob_create("dirtree path names", &dl->path_names); + if (error) + goto out_steps; + + error = xchk_setup_inode_contents(sc, 0); + if (error) + goto out_names; + + sc->buf = dl; + sc->buf_cleanup = xchk_dirtree_buf_cleanup; + return 0; + +out_names: + xfblob_destroy(dl->path_names); +out_steps: + xfarray_destroy(dl->path_steps); +out_dl: + mutex_destroy(&dl->lock); + kvfree(dl); + return error; +} + +/* + * Add the parent pointer described by @dl->pptr to the given path as a new + * step. Returns -ELNRNG if the path is too deep. + */ +int +xchk_dirpath_append( + struct xchk_dirtree *dl, + struct xfs_inode *ip, + struct xchk_dirpath *path, + const struct xfs_name *name, + const struct xfs_parent_rec *pptr) +{ + struct xchk_dirpath_step step = { + .pptr_rec = *pptr, /* struct copy */ + .name_len = name->len, + }; + int error; + + /* + * If this path is more than 2 billion steps long, this directory tree + * is too far gone to fix. + */ + if (path->nr_steps >= XFS_MAXLINK) + return -ELNRNG; + + error = xfblob_storename(dl->path_names, &step.name_cookie, name); + if (error) + return error; + + error = xino_bitmap_set(&path->seen_inodes, I_INO(ip)); + if (error) + return error; + + error = xfarray_append(dl->path_steps, &step); + if (error) + return error; + + path->nr_steps++; + return 0; +} + +/* + * Create an xchk_path for each parent pointer of the directory that we're + * scanning. For each path created, we will eventually try to walk towards the + * root with the goal of deleting all parents except for one that leads to the + * root. + * + * Returns -EFSCORRUPTED to signal that the inode being scanned has a corrupt + * parent pointer and hence there's no point in continuing; or -ENOSR if there + * are too many parent pointers for this directory. + */ +STATIC int +xchk_dirtree_create_path( + struct xfs_scrub *sc, + struct xfs_inode *ip, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + const void *value, + unsigned int valuelen, + void *priv) +{ + struct xfs_name xname = { + .name = name, + .len = namelen, + }; + struct xchk_dirtree *dl = priv; + struct xchk_dirpath *path; + const struct xfs_parent_rec *rec = value; + int error; + + if (!(attr_flags & XFS_ATTR_PARENT)) + return 0; + + error = xfs_parent_from_attr(sc->mp, attr_flags, name, namelen, value, + valuelen, NULL, NULL); + if (error) + return error; + + /* + * If there are more than 2 billion actual parent pointers for this + * subdirectory, this fs is too far gone to fix. + */ + if (dl->nr_paths >= XFS_MAXLINK) + return -ENOSR; + + trace_xchk_dirtree_create_path(sc, ip, dl->nr_paths, &xname, rec); + + /* + * Create a new xchk_path structure to remember this parent pointer + * and record the first name step. + */ + path = kmalloc_obj(struct xchk_dirpath, XCHK_GFP_FLAGS); + if (!path) + return -ENOMEM; + + INIT_LIST_HEAD(&path->list); + xino_bitmap_init(&path->seen_inodes); + path->nr_steps = 0; + path->outcome = XCHK_DIRPATH_SCANNING; + + error = xchk_dirpath_append(dl, sc->ip, path, &xname, rec); + if (error) + goto out_path; + + path->first_step = xfarray_length(dl->path_steps) - 1; + path->second_step = XFARRAY_NULLIDX; + path->path_nr = dl->nr_paths; + + list_add_tail(&path->list, &dl->path_list); + dl->nr_paths++; + return 0; +out_path: + kfree(path); + return error; +} + +/* + * Validate that the first step of this path still has a corresponding + * parent pointer in @sc->ip. We probably dropped @sc->ip's ILOCK while + * walking towards the roots, which is why this is necessary. + * + * This function has a side effect of loading the first parent pointer of this + * path into the parent pointer scratch pad. This prepares us to walk up the + * directory tree towards the root. Returns -ESTALE if the scan data is now + * out of date. + */ +STATIC int +xchk_dirpath_revalidate( + struct xchk_dirtree *dl, + struct xchk_dirpath *path) +{ + struct xfs_scrub *sc = dl->sc; + int error; + + /* + * Look up the parent pointer that corresponds to the start of this + * path. If the parent pointer has disappeared on us, dump all the + * scan results and try again. + */ + error = xfs_parent_lookup(sc->tp, sc->ip, &dl->xname, &dl->pptr_rec, + &dl->pptr_args); + if (error == -ENOATTR) { + trace_xchk_dirpath_disappeared(dl->sc, sc->ip, path->path_nr, + path->first_step, &dl->xname, &dl->pptr_rec); + dl->stale = true; + return -ESTALE; + } + + return error; +} + +/* + * Walk the parent pointers of a directory at the end of a path and record + * the parent that we find in @dl->xname/pptr_rec. + */ +STATIC int +xchk_dirpath_find_next_step( + struct xfs_scrub *sc, + struct xfs_inode *ip, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + const void *value, + unsigned int valuelen, + void *priv) +{ + struct xchk_dirtree *dl = priv; + const struct xfs_parent_rec *rec = value; + int error; + + if (!(attr_flags & XFS_ATTR_PARENT)) + return 0; + + error = xfs_parent_from_attr(sc->mp, attr_flags, name, namelen, value, + valuelen, NULL, NULL); + if (error) + return error; + + /* + * If we've already set @dl->pptr_rec, then this directory has multiple + * parents. Signal this back to the caller via -EMLINK. + */ + if (dl->parents_found > 0) + return -EMLINK; + + dl->parents_found++; + memcpy(dl->namebuf, name, namelen); + dl->xname.len = namelen; + dl->pptr_rec = *rec; /* struct copy */ + return 0; +} + +/* Set and log the outcome of a path walk. */ +static inline void +xchk_dirpath_set_outcome( + struct xchk_dirtree *dl, + struct xchk_dirpath *path, + enum xchk_dirpath_outcome outcome) +{ + trace_xchk_dirpath_set_outcome(dl->sc, path->path_nr, path->nr_steps, + outcome); + + path->outcome = outcome; +} + +/* + * Scan the directory at the end of this path for its parent directory link. + * If we find one, extend the path. Returns -ESTALE if the scan data out of + * date. Returns -EFSCORRUPTED if the parent pointer is bad; or -ELNRNG if + * the path got too deep. + */ +STATIC int +xchk_dirpath_step_up( + struct xchk_dirtree *dl, + struct xchk_dirpath *path, + bool is_metadir) +{ + struct xfs_scrub *sc = dl->sc; + struct xfs_inode *dp; + xfs_ino_t parent_ino = be64_to_cpu(dl->pptr_rec.p_ino); + unsigned int lock_mode; + int error; + + /* Grab and lock the parent directory. */ + error = xchk_iget(sc, parent_ino, &dp); + if (error) + return error; + + lock_mode = xfs_ilock_attr_map_shared(dp); + mutex_lock(&dl->lock); + + if (dl->stale) { + error = -ESTALE; + goto out_scanlock; + } + + /* The handle encoded in the parent pointer must match. */ + if (VFS_I(dp)->i_generation != be32_to_cpu(dl->pptr_rec.p_gen)) { + trace_xchk_dirpath_badgen(dl->sc, dp, path->path_nr, + path->nr_steps, &dl->xname, &dl->pptr_rec); + error = -EFSCORRUPTED; + goto out_scanlock; + } + + /* We've reached the root directory; the path is ok. */ + if (parent_ino == dl->root_ino) { + xchk_dirpath_set_outcome(dl, path, XCHK_DIRPATH_OK); + error = 0; + goto out_scanlock; + } + + /* + * The inode being scanned is its own distant ancestor! Get rid of + * this path. + */ + if (parent_ino == I_INO(sc->ip)) { + xchk_dirpath_set_outcome(dl, path, XCHK_DIRPATH_DELETE); + error = 0; + goto out_scanlock; + } + + /* + * We've seen this inode before during the path walk. There's a loop + * above us in the directory tree. This probably means that we cannot + * continue, but let's keep walking paths to get a full picture. + */ + if (xino_bitmap_test(&path->seen_inodes, parent_ino)) { + xchk_dirpath_set_outcome(dl, path, XCHK_DIRPATH_LOOP); + error = 0; + goto out_scanlock; + } + + /* Parent pointer must point up to a directory. */ + if (!S_ISDIR(VFS_I(dp)->i_mode)) { + trace_xchk_dirpath_nondir_parent(dl->sc, dp, path->path_nr, + path->nr_steps, &dl->xname, &dl->pptr_rec); + error = -EFSCORRUPTED; + goto out_scanlock; + } + + /* Parent cannot be an unlinked directory. */ + if (VFS_I(dp)->i_nlink == 0) { + trace_xchk_dirpath_unlinked_parent(dl->sc, dp, path->path_nr, + path->nr_steps, &dl->xname, &dl->pptr_rec); + error = -EFSCORRUPTED; + goto out_scanlock; + } + + /* Parent must be in the same directory tree. */ + if (is_metadir != xfs_is_metadir_inode(dp)) { + trace_xchk_dirpath_crosses_tree(dl->sc, dp, path->path_nr, + path->nr_steps, &dl->xname, &dl->pptr_rec); + error = -EFSCORRUPTED; + goto out_scanlock; + } + + /* + * If the extended attributes look as though they has been zapped by + * the inode record repair code, we cannot scan for parent pointers. + */ + if (xchk_pptr_looks_zapped(dp)) { + error = -EBUSY; + xchk_set_incomplete(sc); + goto out_scanlock; + } + + /* + * Walk the parent pointers of @dp to find the parent of this directory + * to find the next step in our walk. If we find that @dp has exactly + * one parent, the parent pointer information will be stored in + * @dl->pptr_rec. This prepares us for the next step of the walk. + */ + mutex_unlock(&dl->lock); + dl->parents_found = 0; + error = xchk_xattr_walk(sc, dp, xchk_dirpath_find_next_step, NULL, dl); + mutex_lock(&dl->lock); + if (error == -EFSCORRUPTED || error == -EMLINK || + (!error && dl->parents_found == 0)) { + /* + * Further up the directory tree from @sc->ip, we found a + * corrupt parent pointer, multiple parent pointers while + * finding this directory's parent, or zero parents despite + * having a nonzero link count. Keep looking for other paths. + */ + xchk_dirpath_set_outcome(dl, path, XCHK_DIRPATH_CORRUPT); + error = 0; + goto out_scanlock; + } + if (error) + goto out_scanlock; + + if (dl->stale) { + error = -ESTALE; + goto out_scanlock; + } + + trace_xchk_dirpath_found_next_step(sc, dp, path->path_nr, + path->nr_steps, &dl->xname, &dl->pptr_rec); + + /* Append to the path steps */ + error = xchk_dirpath_append(dl, dp, path, &dl->xname, &dl->pptr_rec); + if (error) + goto out_scanlock; + + if (path->second_step == XFARRAY_NULLIDX) + path->second_step = xfarray_length(dl->path_steps) - 1; + +out_scanlock: + mutex_unlock(&dl->lock); + xfs_iunlock(dp, lock_mode); + xchk_irele(sc, dp); + return error; +} + +/* + * Walk the directory tree upwards towards what is hopefully the root + * directory, recording path steps as we go. The current path components are + * stored in dl->pptr_rec and dl->xname. + * + * Returns -ESTALE if the scan data are out of date. Returns -EFSCORRUPTED + * only if the direct parent pointer of @sc->ip associated with this path is + * corrupt. + */ +STATIC int +xchk_dirpath_walk_upwards( + struct xchk_dirtree *dl, + struct xchk_dirpath *path) +{ + struct xfs_scrub *sc = dl->sc; + bool is_metadir; + int error; + + ASSERT(sc->ilock_flags & XFS_ILOCK_EXCL); + + /* Reload the start of this path and make sure it's still there. */ + error = xchk_dirpath_revalidate(dl, path); + if (error) + return error; + + trace_xchk_dirpath_walk_upwards(sc, sc->ip, path->path_nr, &dl->xname, + &dl->pptr_rec); + + /* + * The inode being scanned is its own direct ancestor! + * Get rid of this path. + */ + if (be64_to_cpu(dl->pptr_rec.p_ino) == I_INO(sc->ip)) { + xchk_dirpath_set_outcome(dl, path, XCHK_DIRPATH_DELETE); + return 0; + } + + /* + * Drop ILOCK_EXCL on the inode being scanned. We still hold + * IOLOCK_EXCL on it, so it cannot move around or be renamed. + * + * Beyond this point we're walking up the directory tree, which means + * that we can acquire and drop the ILOCK on an alias of sc->ip. The + * ILOCK state is no longer tracked in the scrub context. Hence we + * must drop @sc->ip's ILOCK during the walk. + */ + is_metadir = xfs_is_metadir_inode(sc->ip); + mutex_unlock(&dl->lock); + xchk_iunlock(sc, XFS_ILOCK_EXCL); + + /* + * Take the first step in the walk towards the root by checking the + * start of this path, which is a direct parent pointer of @sc->ip. + * If we see any kind of error here (including corruptions), the parent + * pointer of @sc->ip is corrupt. Stop the whole scan. + */ + error = xchk_dirpath_step_up(dl, path, is_metadir); + if (error) { + xchk_ilock(sc, XFS_ILOCK_EXCL); + mutex_lock(&dl->lock); + return error; + } + + /* + * Take steps upward from the second step in this path towards the + * root. If we hit corruption errors here, there's a problem + * *somewhere* in the path, but we don't need to stop scanning. + */ + while (!error && path->outcome == XCHK_DIRPATH_SCANNING) + error = xchk_dirpath_step_up(dl, path, is_metadir); + + /* Retake the locks we had, mark paths, etc. */ + xchk_ilock(sc, XFS_ILOCK_EXCL); + mutex_lock(&dl->lock); + if (error == -EFSCORRUPTED) { + xchk_dirpath_set_outcome(dl, path, XCHK_DIRPATH_CORRUPT); + error = 0; + } + if (!error && dl->stale) + return -ESTALE; + return error; +} + +/* + * Decide if this path step has been touched by this live update. Returns + * 1 for yes, 0 for no, or a negative errno. + */ +STATIC int +xchk_dirpath_step_is_stale( + struct xchk_dirtree *dl, + struct xchk_dirpath *path, + unsigned int step_nr, + xfarray_idx_t step_idx, + struct xfs_dir_update_params *p, + xfs_ino_t *cursor) +{ + struct xchk_dirpath_step step; + xfs_ino_t child_ino = *cursor; + int error; + + error = xfarray_load(dl->path_steps, step_idx, &step); + if (error) + return error; + *cursor = be64_to_cpu(step.pptr_rec.p_ino); + + /* + * If the parent and child being updated are not the ones mentioned in + * this path step, the scan data is still ok. + */ + if (I_INO(p->ip) != child_ino || I_INO(p->dp) != *cursor) + return 0; + + /* + * If the dirent name lengths or byte sequences are different, the scan + * data is still ok. + */ + if (p->name->len != step.name_len) + return 0; + + error = xfblob_loadname(dl->path_names, step.name_cookie, + &dl->hook_xname, step.name_len); + if (error) + return error; + + if (memcmp(dl->hook_xname.name, p->name->name, p->name->len) != 0) + return 0; + + /* + * If the update comes from the repair code itself, walk the state + * machine forward. + */ + if (I_INO(p->ip) == dl->scan_ino && + path->outcome == XREP_DIRPATH_ADOPTING) { + xchk_dirpath_set_outcome(dl, path, XREP_DIRPATH_ADOPTED); + return 0; + } + + if (I_INO(p->ip) == dl->scan_ino && + path->outcome == XREP_DIRPATH_DELETING) { + xchk_dirpath_set_outcome(dl, path, XREP_DIRPATH_DELETED); + return 0; + } + + /* Exact match, scan data is out of date. */ + trace_xchk_dirpath_changed(dl->sc, path->path_nr, step_nr, p->dp, + p->ip, p->name); + return 1; +} + +/* + * Decide if this path has been touched by this live update. Returns 1 for + * yes, 0 for no, or a negative errno. + */ +STATIC int +xchk_dirpath_is_stale( + struct xchk_dirtree *dl, + struct xchk_dirpath *path, + struct xfs_dir_update_params *p) +{ + xfs_ino_t cursor = dl->scan_ino; + xfarray_idx_t idx = path->first_step; + unsigned int i; + int ret; + + /* + * The child being updated has not been seen by this path at all; this + * path cannot be stale. + */ + if (!xino_bitmap_test(&path->seen_inodes, I_INO(p->ip))) + return 0; + + ret = xchk_dirpath_step_is_stale(dl, path, 0, idx, p, &cursor); + if (ret != 0) + return ret; + + for (i = 1, idx = path->second_step; i < path->nr_steps; i++, idx++) { + ret = xchk_dirpath_step_is_stale(dl, path, i, idx, p, &cursor); + if (ret != 0) + return ret; + } + + return 0; +} + +/* + * Decide if a directory update from the regular filesystem touches any of the + * paths we've scanned, and invalidate the scan data if true. + */ +STATIC int +xchk_dirtree_live_update( + struct notifier_block *nb, + unsigned long action, + void *data) +{ + struct xfs_dir_update_params *p = data; + struct xchk_dirtree *dl; + struct xchk_dirpath *path; + int ret; + + dl = container_of(nb, struct xchk_dirtree, dhook.dirent_hook.nb); + + trace_xchk_dirtree_live_update(dl->sc, p->dp, action, p->ip, p->delta, + p->name); + + mutex_lock(&dl->lock); + + if (dl->stale || dl->aborted) + goto out_unlock; + + xchk_dirtree_for_each_path(dl, path) { + ret = xchk_dirpath_is_stale(dl, path, p); + if (ret < 0) { + dl->aborted = true; + break; + } + if (ret == 1) { + dl->stale = true; + break; + } + } + +out_unlock: + mutex_unlock(&dl->lock); + return NOTIFY_DONE; +} + +/* Delete all the collected path information. */ +STATIC void +xchk_dirtree_reset( + void *buf) +{ + struct xchk_dirtree *dl = buf; + struct xchk_dirpath *path, *n; + + ASSERT(dl->sc->ilock_flags & XFS_ILOCK_EXCL); + + xchk_dirtree_for_each_path_safe(dl, path, n) { + list_del_init(&path->list); + xino_bitmap_destroy(&path->seen_inodes); + kfree(path); + } + dl->nr_paths = 0; + + xfarray_truncate(dl->path_steps); + xfblob_truncate(dl->path_names); + + dl->stale = false; +} + +/* + * Load the name/pptr from the first step in this path into @dl->pptr_rec and + * @dl->xname. + */ +STATIC int +xchk_dirtree_load_path( + struct xchk_dirtree *dl, + struct xchk_dirpath *path) +{ + struct xchk_dirpath_step step; + int error; + + error = xfarray_load(dl->path_steps, path->first_step, &step); + if (error) + return error; + + error = xfblob_loadname(dl->path_names, step.name_cookie, &dl->xname, + step.name_len); + if (error) + return error; + + dl->pptr_rec = step.pptr_rec; /* struct copy */ + return 0; +} + +/* + * For each parent pointer of this subdir, trace a path upwards towards the + * root directory and record what we find. Returns 0 for success; + * -EFSCORRUPTED if walking the parent pointers of @sc->ip failed, -ELNRNG if a + * path was too deep; -ENOSR if there were too many parent pointers; or + * a negative errno. + */ +int +xchk_dirtree_find_paths_to_root( + struct xchk_dirtree *dl) +{ + struct xfs_scrub *sc = dl->sc; + struct xchk_dirpath *path; + int error = 0; + + do { + if (xchk_should_terminate(sc, &error)) + return error; + + xchk_dirtree_reset(dl); + + /* + * If the extended attributes look as though they has been + * zapped by the inode record repair code, we cannot scan for + * parent pointers. + */ + if (xchk_pptr_looks_zapped(sc->ip)) { + xchk_set_incomplete(sc); + return -EBUSY; + } + + /* + * Create path walk contexts for each parent of the directory + * that is being scanned. Directories are supposed to have + * only one parent, but this is how we detect multiple parents. + */ + error = xchk_xattr_walk(sc, sc->ip, xchk_dirtree_create_path, + NULL, dl); + if (error) + return error; + + xchk_dirtree_for_each_path(dl, path) { + /* Load path components into dl->pptr/xname */ + error = xchk_dirtree_load_path(dl, path); + if (error) + return error; + + /* + * Try to walk up each path to the root. This enables + * us to find directory loops in ancestors, and the + * like. + */ + error = xchk_dirpath_walk_upwards(dl, path); + if (error == -EFSCORRUPTED) { + /* + * A parent pointer of @sc->ip is bad, don't + * bother continuing. + */ + break; + } + if (error == -ESTALE) { + /* This had better be an invalidation. */ + ASSERT(dl->stale); + break; + } + if (error) + return error; + if (dl->aborted) + return 0; + } + } while (dl->stale); + + return error; +} + +/* + * Figure out what to do with the paths we tried to find. Do not call this + * if the scan results are stale. + */ +void +xchk_dirtree_evaluate( + struct xchk_dirtree *dl, + struct xchk_dirtree_outcomes *oc) +{ + struct xchk_dirpath *path; + + ASSERT(!dl->stale); + + /* Scan the paths we have to decide what to do. */ + memset(oc, 0, sizeof(struct xchk_dirtree_outcomes)); + xchk_dirtree_for_each_path(dl, path) { + trace_xchk_dirpath_evaluate_path(dl->sc, path->path_nr, + path->nr_steps, path->outcome); + + switch (path->outcome) { + case XCHK_DIRPATH_SCANNING: + /* shouldn't get here */ + ASSERT(0); + break; + case XCHK_DIRPATH_DELETE: + /* This one is already going away. */ + oc->bad++; + break; + case XCHK_DIRPATH_CORRUPT: + case XCHK_DIRPATH_LOOP: + /* Couldn't find the end of this path. */ + oc->suspect++; + break; + case XCHK_DIRPATH_STALE: + /* shouldn't get here either */ + ASSERT(0); + break; + case XCHK_DIRPATH_OK: + /* This path got all the way to the root. */ + oc->good++; + break; + case XREP_DIRPATH_DELETING: + case XREP_DIRPATH_DELETED: + case XREP_DIRPATH_ADOPTING: + case XREP_DIRPATH_ADOPTED: + /* These should not be in progress! */ + ASSERT(0); + break; + } + } + + trace_xchk_dirtree_evaluate(dl, oc); +} + +/* Look for directory loops. */ +int +xchk_dirtree( + struct xfs_scrub *sc) +{ + struct xchk_dirtree_outcomes oc; + struct xchk_dirtree *dl = sc->buf; + int error; + + /* + * Nondirectories do not point downwards to other files, so they cannot + * cause a cycle in the directory tree. + */ + if (!S_ISDIR(VFS_I(sc->ip)->i_mode)) + return -ENOENT; + + ASSERT(xfs_has_parent(sc->mp)); + + /* + * Find the root of the directory tree. Remember which directory to + * scan, because the hook doesn't detach until after sc->ip gets + * released during teardown. + */ + dl->root_ino = xchk_inode_rootdir_inum(sc->ip); + dl->scan_ino = I_INO(sc->ip); + + trace_xchk_dirtree_start(sc->ip, sc->sm, 0); + + /* + * Hook into the directory entry code so that we can capture updates to + * paths that we have already scanned. The scanner thread takes each + * directory's ILOCK, which means that any in-progress directory update + * will finish before we can scan the directory. + */ + ASSERT(sc->flags & XCHK_FSGATES_DIRENTS); + xfs_dir_hook_setup(&dl->dhook, xchk_dirtree_live_update); + error = xfs_dir_hook_add(sc->mp, &dl->dhook); + if (error) + goto out; + + mutex_lock(&dl->lock); + + /* Trace each parent pointer's path to the root. */ + error = xchk_dirtree_find_paths_to_root(dl); + if (error == -EFSCORRUPTED || error == -ELNRNG || error == -ENOSR) { + /* + * Don't bother walking the paths if the xattr structure or the + * parent pointers are corrupt; this scan cannot be completed + * without full information. + */ + xchk_ip_xref_set_corrupt(sc, sc->ip); + error = 0; + goto out_scanlock; + } + if (error == -EBUSY) { + /* + * We couldn't scan some directory's parent pointers because + * the attr fork looked like it had been zapped. The + * scan was marked incomplete, so no further error code + * is necessary. + */ + error = 0; + goto out_scanlock; + } + if (error) + goto out_scanlock; + if (dl->aborted) { + xchk_set_incomplete(sc); + goto out_scanlock; + } + + /* Assess what we found in our path evaluation. */ + xchk_dirtree_evaluate(dl, &oc); + if (xchk_dirtree_parentless(dl)) { + if (oc.good || oc.bad || oc.suspect) + xchk_ip_set_corrupt(sc, sc->ip); + } else { + if (oc.bad || oc.good + oc.suspect != 1) + xchk_ip_set_corrupt(sc, sc->ip); + if (oc.suspect) + xchk_ip_xref_set_corrupt(sc, sc->ip); + } + +out_scanlock: + mutex_unlock(&dl->lock); +out: + trace_xchk_dirtree_done(sc->ip, sc->sm, error); + return error; +} + +/* Does the directory targetted by this scrub have no parents? */ +bool +xchk_dirtree_parentless(const struct xchk_dirtree *dl) +{ + struct xfs_scrub *sc = dl->sc; + + if (xchk_inode_is_dirtree_root(sc->ip)) + return true; + if (VFS_I(sc->ip)->i_nlink == 0) + return true; + return false; +} diff --git a/fs/xfs/scrub/dirtree.h b/fs/xfs/scrub/dirtree.h new file mode 100644 index 000000000..9e5d95492 --- /dev/null +++ b/fs/xfs/scrub/dirtree.h @@ -0,0 +1,168 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2023-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_DIRTREE_H__ +#define __XFS_SCRUB_DIRTREE_H__ + +/* + * Each of these represents one parent pointer path step in a chain going + * up towards the directory tree root. These are stored inside an xfarray. + */ +struct xchk_dirpath_step { + /* Directory entry name associated with this parent link. */ + xfblob_cookie name_cookie; + unsigned int name_len; + + /* Handle of the parent directory. */ + struct xfs_parent_rec pptr_rec; +}; + +enum xchk_dirpath_outcome { + XCHK_DIRPATH_SCANNING = 0, /* still being put together */ + XCHK_DIRPATH_DELETE, /* delete this path */ + XCHK_DIRPATH_CORRUPT, /* corruption detected in path */ + XCHK_DIRPATH_LOOP, /* cycle detected further up */ + XCHK_DIRPATH_STALE, /* path is stale */ + XCHK_DIRPATH_OK, /* path reaches the root */ + + XREP_DIRPATH_DELETING, /* path is being deleted */ + XREP_DIRPATH_DELETED, /* path has been deleted */ + XREP_DIRPATH_ADOPTING, /* path is being adopted */ + XREP_DIRPATH_ADOPTED, /* path has been adopted */ +}; + +/* + * Each of these represents one parent pointer path out of the directory being + * scanned. These exist in-core, and hopefully there aren't more than a + * handful of them. + */ +struct xchk_dirpath { + struct list_head list; + + /* Index of the first step in this path. */ + xfarray_idx_t first_step; + + /* Index of the second step in this path. */ + xfarray_idx_t second_step; + + /* Inodes seen while walking this path. */ + struct xino_bitmap seen_inodes; + + /* Number of steps in this path. */ + unsigned int nr_steps; + + /* Which path is this? */ + unsigned int path_nr; + + /* What did we conclude from following this path? */ + enum xchk_dirpath_outcome outcome; +}; + +struct xchk_dirtree_outcomes { + /* Number of XCHK_DIRPATH_DELETE */ + unsigned int bad; + + /* Number of XCHK_DIRPATH_CORRUPT or XCHK_DIRPATH_LOOP */ + unsigned int suspect; + + /* Number of XCHK_DIRPATH_OK */ + unsigned int good; + + /* Directory needs to be added to lost+found */ + bool needs_adoption; +}; + +struct xchk_dirtree { + struct xfs_scrub *sc; + + /* Root inode that we're looking for. */ + xfs_ino_t root_ino; + + /* + * This is the inode that we're scanning. The live update hook can + * continue to be called after xchk_teardown drops sc->ip but before + * it calls buf_cleanup, so we keep a copy. + */ + xfs_ino_t scan_ino; + + /* + * If we start deleting redundant paths to this subdirectory, this is + * the inode number of the surviving parent and the dotdot entry will + * be set to this value. If the value is NULLFSINO, then use @root_ino + * as a stand-in until the orphanage can adopt the subdirectory. + */ + xfs_ino_t parent_ino; + + /* Scratch buffer for scanning pptr xattrs */ + struct xfs_parent_rec pptr_rec; + struct xfs_da_args pptr_args; + + /* Name buffer */ + struct xfs_name xname; + char namebuf[MAXNAMELEN]; + + /* Information for reparenting this directory. */ + struct xrep_adoption adoption; + + /* + * Hook into directory updates so that we can receive live updates + * from other writer threads. + */ + struct xfs_dir_hook dhook; + + /* Parent pointer update arguments. */ + struct xfs_parent_args ppargs; + + /* lock for everything below here */ + struct mutex lock; + + /* buffer for the live update functions to use for dirent names */ + struct xfs_name hook_xname; + unsigned char hook_namebuf[MAXNAMELEN]; + + /* + * All path steps observed during this scan. Each of the path + * steps for a particular pathwalk are recorded in sequential + * order in the xfarray. A pathwalk ends either with a step + * pointing to the root directory (success) or pointing to NULLFSINO + * (loop detected, empty dir detected, etc). + */ + struct xfarray *path_steps; + + /* All names observed during this scan. */ + struct xfblob *path_names; + + /* All paths being tracked by this scanner. */ + struct list_head path_list; + + /* Number of paths in path_list. */ + unsigned int nr_paths; + + /* Number of parents found by a pptr scan. */ + unsigned int parents_found; + + /* Have the path data been invalidated by a concurrent update? */ + bool stale:1; + + /* Has the scan been aborted? */ + bool aborted:1; +}; + +#define xchk_dirtree_for_each_path_safe(dl, path, n) \ + list_for_each_entry_safe((path), (n), &(dl)->path_list, list) + +#define xchk_dirtree_for_each_path(dl, path) \ + list_for_each_entry((path), &(dl)->path_list, list) + +bool xchk_dirtree_parentless(const struct xchk_dirtree *dl); + +int xchk_dirtree_find_paths_to_root(struct xchk_dirtree *dl); +int xchk_dirpath_append(struct xchk_dirtree *dl, struct xfs_inode *ip, + struct xchk_dirpath *path, const struct xfs_name *name, + const struct xfs_parent_rec *pptr); +void xchk_dirtree_evaluate(struct xchk_dirtree *dl, + struct xchk_dirtree_outcomes *oc); + +#endif /* __XFS_SCRUB_DIRTREE_H__ */ diff --git a/fs/xfs/scrub/dirtree_repair.c b/fs/xfs/scrub/dirtree_repair.c new file mode 100644 index 000000000..bbf6acf6f --- /dev/null +++ b/fs/xfs/scrub/dirtree_repair.c @@ -0,0 +1,835 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2023-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_trans_space.h" +#include "xfs_mount.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_inode.h" +#include "xfs_icache.h" +#include "xfs_dir2.h" +#include "xfs_dir2_priv.h" +#include "xfs_attr.h" +#include "xfs_parent.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/bitmap.h" +#include "scrub/ino_bitmap.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/xfblob.h" +#include "scrub/listxattr.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/orphanage.h" +#include "scrub/dirtree.h" +#include "scrub/readdir.h" + +/* + * Directory Tree Structure Repairs + * ================================ + * + * If we decide that the directory being scanned is participating in a + * directory loop, the only change we can make is to remove directory entries + * pointing down to @sc->ip. If that leaves it with no parents, the directory + * should be adopted by the orphanage. + */ + +/* Set up to repair directory loops. */ +int +xrep_setup_dirtree( + struct xfs_scrub *sc) +{ + return xrep_orphanage_try_create(sc); +} + +/* Change the outcome of this path. */ +static inline void +xrep_dirpath_set_outcome( + struct xchk_dirtree *dl, + struct xchk_dirpath *path, + enum xchk_dirpath_outcome outcome) +{ + trace_xrep_dirpath_set_outcome(dl->sc, path->path_nr, path->nr_steps, + outcome); + + path->outcome = outcome; +} + +/* Delete all paths. */ +STATIC void +xrep_dirtree_delete_all_paths( + struct xchk_dirtree *dl, + struct xchk_dirtree_outcomes *oc) +{ + struct xchk_dirpath *path; + + xchk_dirtree_for_each_path(dl, path) { + switch (path->outcome) { + case XCHK_DIRPATH_CORRUPT: + case XCHK_DIRPATH_LOOP: + oc->suspect--; + oc->bad++; + xrep_dirpath_set_outcome(dl, path, XCHK_DIRPATH_DELETE); + break; + case XCHK_DIRPATH_OK: + oc->good--; + oc->bad++; + xrep_dirpath_set_outcome(dl, path, XCHK_DIRPATH_DELETE); + break; + default: + break; + } + } + + ASSERT(oc->suspect == 0); + ASSERT(oc->good == 0); +} + +/* Since this is the surviving path, set the dotdot entry to this value. */ +STATIC void +xrep_dirpath_retain_parent( + struct xchk_dirtree *dl, + struct xchk_dirpath *path) +{ + struct xchk_dirpath_step step; + int error; + + error = xfarray_load(dl->path_steps, path->first_step, &step); + if (error) + return; + + dl->parent_ino = be64_to_cpu(step.pptr_rec.p_ino); +} + +/* Find the one surviving path so we know how to set dotdot. */ +STATIC void +xrep_dirtree_find_surviving_path( + struct xchk_dirtree *dl, + struct xchk_dirtree_outcomes *oc) +{ + struct xchk_dirpath *path; + bool foundit = false; + + xchk_dirtree_for_each_path(dl, path) { + switch (path->outcome) { + case XCHK_DIRPATH_CORRUPT: + case XCHK_DIRPATH_LOOP: + case XCHK_DIRPATH_OK: + if (!foundit) { + xrep_dirpath_retain_parent(dl, path); + foundit = true; + continue; + } + ASSERT(foundit == false); + break; + default: + break; + } + } + + ASSERT(oc->suspect + oc->good == 1); +} + +/* Delete all paths except for the one good one. */ +STATIC void +xrep_dirtree_keep_one_good_path( + struct xchk_dirtree *dl, + struct xchk_dirtree_outcomes *oc) +{ + struct xchk_dirpath *path; + bool foundit = false; + + xchk_dirtree_for_each_path(dl, path) { + switch (path->outcome) { + case XCHK_DIRPATH_CORRUPT: + case XCHK_DIRPATH_LOOP: + oc->suspect--; + oc->bad++; + xrep_dirpath_set_outcome(dl, path, XCHK_DIRPATH_DELETE); + break; + case XCHK_DIRPATH_OK: + if (!foundit) { + xrep_dirpath_retain_parent(dl, path); + foundit = true; + continue; + } + oc->good--; + oc->bad++; + xrep_dirpath_set_outcome(dl, path, XCHK_DIRPATH_DELETE); + break; + default: + break; + } + } + + ASSERT(oc->suspect == 0); + ASSERT(oc->good < 2); +} + +/* Delete all paths except for one suspect one. */ +STATIC void +xrep_dirtree_keep_one_suspect_path( + struct xchk_dirtree *dl, + struct xchk_dirtree_outcomes *oc) +{ + struct xchk_dirpath *path; + bool foundit = false; + + xchk_dirtree_for_each_path(dl, path) { + switch (path->outcome) { + case XCHK_DIRPATH_CORRUPT: + case XCHK_DIRPATH_LOOP: + if (!foundit) { + xrep_dirpath_retain_parent(dl, path); + foundit = true; + continue; + } + oc->suspect--; + oc->bad++; + xrep_dirpath_set_outcome(dl, path, XCHK_DIRPATH_DELETE); + break; + case XCHK_DIRPATH_OK: + ASSERT(0); + break; + default: + break; + } + } + + ASSERT(oc->suspect == 1); + ASSERT(oc->good == 0); +} + +/* + * Figure out what to do with the paths we tried to find. Returns -EDEADLOCK + * if the scan results have become stale. + */ +STATIC void +xrep_dirtree_decide_fate( + struct xchk_dirtree *dl, + struct xchk_dirtree_outcomes *oc) +{ + xchk_dirtree_evaluate(dl, oc); + + /* Parentless directories should not have any paths at all. */ + if (xchk_dirtree_parentless(dl)) { + xrep_dirtree_delete_all_paths(dl, oc); + return; + } + + /* One path is exactly the number of paths we want. */ + if (oc->good + oc->suspect == 1) { + xrep_dirtree_find_surviving_path(dl, oc); + return; + } + + /* Zero paths means we should reattach the subdir to the orphanage. */ + if (oc->good + oc->suspect == 0) { + if (dl->sc->orphanage) + oc->needs_adoption = true; + return; + } + + /* + * Otherwise, this subdirectory has too many parents. If there's at + * least one good path, keep it and delete the others. + */ + if (oc->good > 0) { + xrep_dirtree_keep_one_good_path(dl, oc); + return; + } + + /* + * There are no good paths and there are too many suspect paths. + * Keep the first suspect path and delete the rest. + */ + xrep_dirtree_keep_one_suspect_path(dl, oc); +} + +/* + * Load the first step of this path into @step and @dl->xname/pptr + * for later repair work. + */ +STATIC int +xrep_dirtree_prep_path( + struct xchk_dirtree *dl, + struct xchk_dirpath *path, + struct xchk_dirpath_step *step) +{ + int error; + + error = xfarray_load(dl->path_steps, path->first_step, step); + if (error) + return error; + + error = xfblob_loadname(dl->path_names, step->name_cookie, &dl->xname, + step->name_len); + if (error) + return error; + + dl->pptr_rec = step->pptr_rec; /* struct copy */ + return 0; +} + +/* Delete the VFS dentry for a removed child. */ +STATIC int +xrep_dirtree_purge_dentry( + struct xchk_dirtree *dl, + struct xfs_inode *dp, + const struct xfs_name *name) +{ + struct qstr qname = QSTR_INIT(name->name, name->len); + struct dentry *parent_dentry, *child_dentry; + int error = 0; + + /* + * Find the dentry for the parent directory. If there isn't one, we're + * done. Caller already holds i_rwsem for parent and child. + */ + parent_dentry = d_find_alias(VFS_I(dp)); + if (!parent_dentry) + return 0; + + /* The VFS thinks the parent is a directory, right? */ + if (!d_is_dir(parent_dentry)) { + ASSERT(d_is_dir(parent_dentry)); + error = -EFSCORRUPTED; + goto out_dput_parent; + } + + /* + * Try to find the dirent pointing to the child. If there isn't one, + * we're done. + */ + qname.hash = full_name_hash(parent_dentry, name->name, name->len); + child_dentry = d_lookup(parent_dentry, &qname); + if (!child_dentry) { + error = 0; + goto out_dput_parent; + } + + trace_xrep_dirtree_delete_child(dp->i_mount, child_dentry); + + /* Child is not a directory? We're screwed. */ + if (!d_is_dir(child_dentry)) { + ASSERT(d_is_dir(child_dentry)); + error = -EFSCORRUPTED; + goto out_dput_child; + } + + /* Replace the child dentry with a negative one. */ + d_delete(child_dentry); + +out_dput_child: + dput(child_dentry); +out_dput_parent: + dput(parent_dentry); + return error; +} + +/* + * Prepare to delete a link by taking the IOLOCK of the parent and the child + * (scrub target). Caller must hold IOLOCK_EXCL on @sc->ip. Returns 0 if we + * took both locks, or a negative errno if we couldn't lock the parent in time. + */ +static inline int +xrep_dirtree_unlink_iolock( + struct xfs_scrub *sc, + struct xfs_inode *dp) +{ + int error; + + ASSERT(sc->ilock_flags & XFS_IOLOCK_EXCL); + + if (sc->ip == dp) + return 0; + if (xfs_ilock_nowait(dp, XFS_IOLOCK_EXCL)) + return 0; + + xchk_iunlock(sc, XFS_IOLOCK_EXCL); + do { + xfs_ilock(dp, XFS_IOLOCK_EXCL); + if (xchk_ilock_nowait(sc, XFS_IOLOCK_EXCL)) + break; + xfs_iunlock(dp, XFS_IOLOCK_EXCL); + + if (xchk_should_terminate(sc, &error)) { + xchk_ilock(sc, XFS_IOLOCK_EXCL); + return error; + } + + delay(1); + } while (1); + + return 0; +} + +/* + * Remove a link from the directory tree and update the dcache. Returns + * -ESTALE if the scan data are now out of date. + */ +STATIC int +xrep_dirtree_unlink( + struct xchk_dirtree *dl, + struct xfs_inode *dp, + struct xchk_dirpath *path, + struct xchk_dirpath_step *step) +{ + struct xfs_scrub *sc = dl->sc; + struct xfs_mount *mp = sc->mp; + xfs_ino_t dotdot_ino; + xfs_ino_t parent_ino = dl->parent_ino; + unsigned int resblks; + int dontcare; + int error; + + /* Take IOLOCK_EXCL of the parent and child. */ + error = xrep_dirtree_unlink_iolock(sc, dp); + if (error) + return error; + + /* + * Create the transaction that we need to sever the path. Ignore + * EDQUOT and ENOSPC being returned via nospace_error because the + * directory code can handle a reservationless update. + */ + resblks = xfs_remove_space_res(mp, step->name_len); + if (sc->ip == dp) { +again: + error = xfs_trans_alloc_inode(dp, &M_RES(mp)->tr_remove, + resblks, 0, false, &sc->tp); + if ((error == -ENOSPC || error == -EDQUOT) && resblks > 0) { + resblks = 0; + goto again; + } + } else { + error = xfs_trans_alloc_dir(dp, &M_RES(mp)->tr_remove, sc->ip, + &resblks, &sc->tp, &dontcare); + } + if (error) + goto out_iolock; + + /* + * Cancel if someone invalidate the paths while we were trying to get + * the ILOCK. + */ + mutex_lock(&dl->lock); + if (dl->stale) { + mutex_unlock(&dl->lock); + error = -ESTALE; + goto out_trans_cancel; + } + xrep_dirpath_set_outcome(dl, path, XREP_DIRPATH_DELETING); + mutex_unlock(&dl->lock); + + trace_xrep_dirtree_delete_path(dl->sc, sc->ip, path->path_nr, + &dl->xname, &dl->pptr_rec); + + /* + * Decide if we need to reset the dotdot entry. Rules: + * + * - If there's a surviving parent, we want dotdot to point there. + * - If we don't have any surviving parents, then point dotdot at the + * root dir. + * - If dotdot is already set to the value we want, pass in NULLFSINO + * for no change necessary. + * + * Do this /before/ we dirty anything, in case the dotdot lookup + * fails. + */ + error = xchk_dir_lookup(sc, sc->ip, &xfs_name_dotdot, &dotdot_ino); + if (error) + goto out_trans_cancel; + if (parent_ino == NULLFSINO) + parent_ino = dl->root_ino; + if (dotdot_ino == parent_ino) + parent_ino = NULLFSINO; + + /* Drop the link from sc->ip's dotdot entry. */ + error = xfs_droplink(sc->tp, dp); + if (error) + goto out_trans_cancel; + + /* Reset the dotdot entry to a surviving parent. */ + if (parent_ino != NULLFSINO) { + error = xfs_dir_replace(sc->tp, sc->ip, &xfs_name_dotdot, + parent_ino, 0); + if (error) + goto out_trans_cancel; + } + + /* Drop the link from dp to sc->ip. */ + error = xfs_droplink(sc->tp, sc->ip); + if (error) + goto out_trans_cancel; + + error = xfs_dir_removename(sc->tp, dp, &dl->xname, I_INO(sc->ip), + resblks); + if (error) { + ASSERT(error != -ENOENT); + goto out_trans_cancel; + } + + if (xfs_has_parent(sc->mp)) { + error = xfs_parent_removename(sc->tp, &dl->ppargs, dp, + &dl->xname, sc->ip); + if (error) + goto out_trans_cancel; + } + + /* + * Notify dirent hooks that we removed the bad link, invalidate the + * dcache, and commit the repair. + */ + xfs_dir_update_hook(dp, sc->ip, -1, &dl->xname); + error = xrep_dirtree_purge_dentry(dl, dp, &dl->xname); + if (error) + goto out_trans_cancel; + + error = xrep_trans_commit(sc); + goto out_ilock; + +out_trans_cancel: + xchk_trans_cancel(sc); +out_ilock: + xfs_iunlock(sc->ip, XFS_ILOCK_EXCL); + if (dp != sc->ip) + xfs_iunlock(dp, XFS_ILOCK_EXCL); +out_iolock: + if (dp != sc->ip) + xfs_iunlock(dp, XFS_IOLOCK_EXCL); + return error; +} + +/* + * Delete a directory entry that points to this directory. Returns -ESTALE + * if the scan data are now out of date. + */ +STATIC int +xrep_dirtree_delete_path( + struct xchk_dirtree *dl, + struct xchk_dirpath *path) +{ + struct xchk_dirpath_step step; + struct xfs_scrub *sc = dl->sc; + struct xfs_inode *dp; + int error; + + /* + * Load the parent pointer and directory inode for this path, then + * drop the scan lock, the ILOCK, and the transaction so that + * _delete_path can reserve the proper transaction. This sets up + * @dl->xname for the deletion. + */ + error = xrep_dirtree_prep_path(dl, path, &step); + if (error) + return error; + + error = xchk_iget(sc, be64_to_cpu(step.pptr_rec.p_ino), &dp); + if (error) + return error; + + mutex_unlock(&dl->lock); + xchk_trans_cancel(sc); + xchk_iunlock(sc, XFS_ILOCK_EXCL); + + /* Delete the directory link and release the parent. */ + error = xrep_dirtree_unlink(dl, dp, path, &step); + xchk_irele(sc, dp); + + /* + * Retake all the resources we had at the beginning even if the repair + * failed or the scan data are now stale. This keeps things simple for + * the caller. + */ + xchk_trans_alloc_empty(sc); + xchk_ilock(sc, XFS_ILOCK_EXCL); + mutex_lock(&dl->lock); + + if (!error && dl->stale) + error = -ESTALE; + return error; +} + +/* Add a new path to represent our in-progress adoption. */ +STATIC int +xrep_dirtree_create_adoption_path( + struct xchk_dirtree *dl) +{ + struct xfs_scrub *sc = dl->sc; + struct xchk_dirpath *path; + int error; + + /* + * We should have capped the number of paths at XFS_MAXLINK-1 in the + * scanner. + */ + if (dl->nr_paths > XFS_MAXLINK) { + ASSERT(dl->nr_paths <= XFS_MAXLINK); + return -EFSCORRUPTED; + } + + /* + * Create a new xchk_path structure to remember this parent pointer + * and record the first name step. + */ + path = kmalloc_obj(struct xchk_dirpath, XCHK_GFP_FLAGS); + if (!path) + return -ENOMEM; + + INIT_LIST_HEAD(&path->list); + xino_bitmap_init(&path->seen_inodes); + path->nr_steps = 0; + path->outcome = XREP_DIRPATH_ADOPTING; + + /* + * Record the new link that we just created in the orphanage. Because + * adoption is the last repair that we perform, we don't bother filling + * in the path all the way back to the root. + */ + xfs_inode_to_parent_rec(&dl->pptr_rec, sc->orphanage); + + error = xino_bitmap_set(&path->seen_inodes, I_INO(sc->orphanage)); + if (error) + goto out_path; + + trace_xrep_dirtree_create_adoption(sc, sc->ip, dl->nr_paths, + &dl->xname, &dl->pptr_rec); + + error = xchk_dirpath_append(dl, sc->ip, path, &dl->xname, + &dl->pptr_rec); + if (error) + goto out_path; + + path->first_step = xfarray_length(dl->path_steps) - 1; + path->second_step = XFARRAY_NULLIDX; + path->path_nr = dl->nr_paths; + + list_add_tail(&path->list, &dl->path_list); + dl->nr_paths++; + return 0; + +out_path: + kfree(path); + return error; +} + +/* + * Prepare to move a file to the orphanage by taking the IOLOCK of the + * orphanage and the child (scrub target). Caller must hold IOLOCK_EXCL on + * @sc->ip. Returns 0 if we took both locks, or a negative errno if we + * couldn't lock the orphanage in time. + */ +static inline int +xrep_dirtree_adopt_iolock( + struct xfs_scrub *sc) +{ + int error; + + ASSERT(sc->ilock_flags & XFS_IOLOCK_EXCL); + + if (xrep_orphanage_ilock_nowait(sc, XFS_IOLOCK_EXCL)) + return 0; + + xchk_iunlock(sc, XFS_IOLOCK_EXCL); + do { + xrep_orphanage_ilock(sc, XFS_IOLOCK_EXCL); + if (xchk_ilock_nowait(sc, XFS_IOLOCK_EXCL)) + break; + xrep_orphanage_iunlock(sc, XFS_IOLOCK_EXCL); + + if (xchk_should_terminate(sc, &error)) { + xchk_ilock(sc, XFS_IOLOCK_EXCL); + return error; + } + + delay(1); + } while (1); + + return 0; +} + +/* + * Reattach this orphaned directory to the orphanage. Do not call this with + * any resources held. Returns -ESTALE if the scan data have become out of + * date. + */ +STATIC int +xrep_dirtree_adopt( + struct xchk_dirtree *dl) +{ + struct xfs_scrub *sc = dl->sc; + int error; + + /* Take the IOLOCK of the orphanage and the scrub target. */ + error = xrep_dirtree_adopt_iolock(sc); + if (error) + return error; + + /* + * Set up for an adoption. The directory tree fixer runs after the + * link counts have been corrected. Therefore, we must bump the + * child's link count since there will be no further opportunity to fix + * errors. + */ + error = xrep_adoption_trans_alloc(sc, &dl->adoption); + if (error) + goto out_iolock; + dl->adoption.bump_child_nlink = true; + + /* Figure out what name we're going to use here. */ + error = xrep_adoption_compute_name(&dl->adoption, &dl->xname); + if (error) + goto out_trans; + + /* + * Now that we have a proposed name for the orphanage entry, create + * a faux path so that the live update hook will see it. + */ + mutex_lock(&dl->lock); + if (dl->stale) { + mutex_unlock(&dl->lock); + error = -ESTALE; + goto out_trans; + } + error = xrep_dirtree_create_adoption_path(dl); + mutex_unlock(&dl->lock); + if (error) + goto out_trans; + + /* Reparent the directory. */ + error = xrep_adoption_move(&dl->adoption); + if (error) + goto out_trans; + + /* + * Commit the name and release all inode locks except for the scrub + * target's IOLOCK. + */ + error = xrep_trans_commit(sc); + goto out_ilock; + +out_trans: + xchk_trans_cancel(sc); +out_ilock: + xchk_iunlock(sc, XFS_ILOCK_EXCL); + xrep_orphanage_iunlock(sc, XFS_ILOCK_EXCL); +out_iolock: + xrep_orphanage_iunlock(sc, XFS_IOLOCK_EXCL); + return error; +} + +/* + * This newly orphaned directory needs to be adopted by the orphanage. + * Make this happen. + */ +STATIC int +xrep_dirtree_move_to_orphanage( + struct xchk_dirtree *dl) +{ + struct xfs_scrub *sc = dl->sc; + int error; + + /* + * Start by dropping all the resources that we hold so that we can grab + * all the resources that we need for the adoption. + */ + mutex_unlock(&dl->lock); + xchk_trans_cancel(sc); + xchk_iunlock(sc, XFS_ILOCK_EXCL); + + /* Perform the adoption. */ + error = xrep_dirtree_adopt(dl); + + /* + * Retake all the resources we had at the beginning even if the repair + * failed or the scan data are now stale. This keeps things simple for + * the caller. + */ + xchk_trans_alloc_empty(sc); + xchk_ilock(sc, XFS_ILOCK_EXCL); + mutex_lock(&dl->lock); + + if (!error && dl->stale) + error = -ESTALE; + return error; +} + +/* + * Try to fix all the problems. Returns -ESTALE if the scan data have become + * out of date. + */ +STATIC int +xrep_dirtree_fix_problems( + struct xchk_dirtree *dl, + struct xchk_dirtree_outcomes *oc) +{ + struct xchk_dirpath *path; + int error; + + /* Delete all the paths we don't want. */ + xchk_dirtree_for_each_path(dl, path) { + if (path->outcome != XCHK_DIRPATH_DELETE) + continue; + + error = xrep_dirtree_delete_path(dl, path); + if (error) + return error; + } + + /* Reparent this directory to the orphanage. */ + if (oc->needs_adoption) { + if (xrep_orphanage_can_adopt(dl->sc)) + return xrep_dirtree_move_to_orphanage(dl); + return -EFSCORRUPTED; + } + + return 0; +} + +/* Fix directory loops involving this directory. */ +int +xrep_dirtree( + struct xfs_scrub *sc) +{ + struct xchk_dirtree *dl = sc->buf; + struct xchk_dirtree_outcomes oc; + int error; + + /* + * Prepare to fix the directory tree by retaking the scan lock. The + * order of resource acquisition is still IOLOCK -> transaction -> + * ILOCK -> scan lock. + */ + mutex_lock(&dl->lock); + do { + /* + * Decide what we're going to do, then do it. An -ESTALE + * return here means the scan results are invalid and we have + * to walk again. + */ + if (!dl->stale) { + xrep_dirtree_decide_fate(dl, &oc); + + trace_xrep_dirtree_decided_fate(dl, &oc); + + error = xrep_dirtree_fix_problems(dl, &oc); + if (!error || error != -ESTALE) + break; + } + error = xchk_dirtree_find_paths_to_root(dl); + if (error == -ELNRNG || error == -ENOSR) + error = -EFSCORRUPTED; + } while (!error); + mutex_unlock(&dl->lock); + + return error; +} diff --git a/fs/xfs/scrub/dqiterate.c b/fs/xfs/scrub/dqiterate.c new file mode 100644 index 000000000..079dc4e69 --- /dev/null +++ b/fs/xfs/scrub/dqiterate.c @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_bit.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_inode.h" +#include "xfs_quota.h" +#include "xfs_qm.h" +#include "xfs_bmap.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/quota.h" +#include "scrub/trace.h" + +/* Initialize a dquot iteration cursor. */ +void +xchk_dqiter_init( + struct xchk_dqiter *cursor, + struct xfs_scrub *sc, + xfs_dqtype_t dqtype) +{ + cursor->sc = sc; + cursor->bmap.br_startoff = NULLFILEOFF; + cursor->dqtype = dqtype & XFS_DQTYPE_REC_MASK; + cursor->quota_ip = xfs_quota_inode(sc->mp, cursor->dqtype); + cursor->id = 0; +} + +/* + * Ensure that the cached data fork mapping for the dqiter cursor is fresh and + * covers the dquot pointed to by the scan cursor. + */ +STATIC int +xchk_dquot_iter_revalidate_bmap( + struct xchk_dqiter *cursor) +{ + struct xfs_quotainfo *qi = cursor->sc->mp->m_quotainfo; + struct xfs_ifork *ifp = xfs_ifork_ptr(cursor->quota_ip, + XFS_DATA_FORK); + xfs_fileoff_t fileoff; + xfs_dqid_t this_id = cursor->id; + int nmaps = 1; + int error; + + fileoff = this_id / qi->qi_dqperchunk; + + /* + * If we have a mapping for cursor->id and it's still fresh, there's + * no need to reread the bmbt. + */ + if (cursor->bmap.br_startoff != NULLFILEOFF && + cursor->if_seq == ifp->if_seq && + cursor->bmap.br_startoff + cursor->bmap.br_blockcount > fileoff) + return 0; + + /* Look up the data fork mapping for the dquot id of interest. */ + error = xfs_bmapi_read(cursor->quota_ip, fileoff, + XFS_MAX_FILEOFF - fileoff, &cursor->bmap, &nmaps, 0); + if (error) + return error; + if (!nmaps) { + ASSERT(nmaps > 0); + return -EFSCORRUPTED; + } + if (cursor->bmap.br_startoff > fileoff) { + ASSERT(cursor->bmap.br_startoff == fileoff); + return -EFSCORRUPTED; + } + + cursor->if_seq = ifp->if_seq; + trace_xchk_dquot_iter_revalidate_bmap(cursor, cursor->id); + return 0; +} + +/* Advance the dqiter cursor to the next non-sparse region of the quota file. */ +STATIC int +xchk_dquot_iter_advance_bmap( + struct xchk_dqiter *cursor, + uint64_t *next_ondisk_id) +{ + struct xfs_quotainfo *qi = cursor->sc->mp->m_quotainfo; + struct xfs_ifork *ifp = xfs_ifork_ptr(cursor->quota_ip, + XFS_DATA_FORK); + xfs_fileoff_t fileoff; + uint64_t next_id; + int nmaps = 1; + int error; + + /* Find the dquot id for the next non-hole mapping. */ + do { + fileoff = cursor->bmap.br_startoff + cursor->bmap.br_blockcount; + if (fileoff > XFS_DQ_ID_MAX / qi->qi_dqperchunk) { + /* The hole goes beyond the max dquot id, we're done */ + *next_ondisk_id = -1ULL; + return 0; + } + + error = xfs_bmapi_read(cursor->quota_ip, fileoff, + XFS_MAX_FILEOFF - fileoff, &cursor->bmap, + &nmaps, 0); + if (error) + return error; + if (!nmaps) { + /* Must have reached the end of the mappings. */ + *next_ondisk_id = -1ULL; + return 0; + } + if (cursor->bmap.br_startoff > fileoff) { + ASSERT(cursor->bmap.br_startoff == fileoff); + return -EFSCORRUPTED; + } + } while (!xfs_bmap_is_real_extent(&cursor->bmap)); + + next_id = cursor->bmap.br_startoff * qi->qi_dqperchunk; + if (next_id > XFS_DQ_ID_MAX) { + /* The hole goes beyond the max dquot id, we're done */ + *next_ondisk_id = -1ULL; + return 0; + } + + /* Propose jumping forward to the dquot in the next allocated block. */ + *next_ondisk_id = next_id; + cursor->if_seq = ifp->if_seq; + trace_xchk_dquot_iter_advance_bmap(cursor, *next_ondisk_id); + return 0; +} + +/* + * Find the id of the next highest incore dquot. Normally this will correspond + * exactly with the quota file block mappings, but repair might have erased a + * mapping because it was crosslinked; in that case, we need to re-allocate the + * space so that we can reset q_blkno. + */ +STATIC void +xchk_dquot_iter_advance_incore( + struct xchk_dqiter *cursor, + uint64_t *next_incore_id) +{ + struct xfs_quotainfo *qi = cursor->sc->mp->m_quotainfo; + struct radix_tree_root *tree = xfs_dquot_tree(qi, cursor->dqtype); + struct xfs_dquot *dq; + unsigned int nr_found; + + *next_incore_id = -1ULL; + + mutex_lock(&qi->qi_tree_lock); + nr_found = radix_tree_gang_lookup(tree, (void **)&dq, cursor->id, 1); + if (nr_found) + *next_incore_id = dq->q_id; + mutex_unlock(&qi->qi_tree_lock); + + trace_xchk_dquot_iter_advance_incore(cursor, *next_incore_id); +} + +/* + * Walk all incore dquots of this filesystem. Caller must set *@cursorp to + * zero before the first call, and must not hold the quota file ILOCK. + * Returns 1 and a valid *@dqpp; 0 and *@dqpp == NULL when there are no more + * dquots to iterate; or a negative errno. + */ +int +xchk_dquot_iter( + struct xchk_dqiter *cursor, + struct xfs_dquot **dqpp) +{ + struct xfs_mount *mp = cursor->sc->mp; + struct xfs_dquot *dq = NULL; + uint64_t next_ondisk, next_incore = -1ULL; + unsigned int lock_mode; + int error = 0; + + if (cursor->id > XFS_DQ_ID_MAX) + return 0; + next_ondisk = cursor->id; + + /* Revalidate and/or advance the cursor. */ + lock_mode = xfs_ilock_data_map_shared(cursor->quota_ip); + error = xchk_dquot_iter_revalidate_bmap(cursor); + if (!error && !xfs_bmap_is_real_extent(&cursor->bmap)) + error = xchk_dquot_iter_advance_bmap(cursor, &next_ondisk); + xfs_iunlock(cursor->quota_ip, lock_mode); + if (error) + return error; + + if (next_ondisk > cursor->id) + xchk_dquot_iter_advance_incore(cursor, &next_incore); + + /* Pick the next dquot in the sequence and return it. */ + cursor->id = min(next_ondisk, next_incore); + if (cursor->id > XFS_DQ_ID_MAX) + return 0; + + trace_xchk_dquot_iter(cursor, cursor->id); + + error = xfs_qm_dqget(mp, cursor->id, cursor->dqtype, false, &dq); + if (error) + return error; + + cursor->id = (uint64_t)dq->q_id + 1; + *dqpp = dq; + return 1; +} diff --git a/fs/xfs/scrub/findparent.c b/fs/xfs/scrub/findparent.c new file mode 100644 index 000000000..04b6b96b0 --- /dev/null +++ b/fs/xfs/scrub/findparent.c @@ -0,0 +1,469 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_bit.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_sb.h" +#include "xfs_inode.h" +#include "xfs_icache.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_dir2.h" +#include "xfs_bmap_btree.h" +#include "xfs_dir2_priv.h" +#include "xfs_trans_space.h" +#include "xfs_health.h" +#include "xfs_exchmaps.h" +#include "xfs_parent.h" +#include "scrub/xfs_scrub.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/iscan.h" +#include "scrub/findparent.h" +#include "scrub/readdir.h" +#include "scrub/tempfile.h" +#include "scrub/listxattr.h" + +/* + * Finding the Parent of a Directory + * ================================= + * + * Directories have parent pointers, in the sense that each directory contains + * a dotdot entry that points to the single allowed parent. The brute force + * way to find the parent of a given directory is to scan every directory in + * the filesystem looking for a child dirent that references this directory. + * + * This module wraps the process of scanning the directory tree. It requires + * that @sc->ip is the directory whose parent we want to find, and that the + * caller hold only the IOLOCK on that directory. The scan itself needs to + * take the ILOCK of each directory visited. + * + * Because we cannot hold @sc->ip's ILOCK during a scan of the whole fs, it is + * necessary to use dirent hook to update the parent scan results. Callers + * must not read the scan results without re-taking @sc->ip's ILOCK. + * + * There are a few shortcuts that we can take to avoid scanning the entire + * filesystem, such as noticing directory tree roots and querying the dentry + * cache for parent information. + */ + +struct xrep_findparent_info { + /* The directory currently being scanned. */ + struct xfs_inode *dp; + + /* + * Scrub context. We're looking for a @dp containing a directory + * entry pointing to I_INO(sc->ip). + */ + struct xfs_scrub *sc; + + /* Optional scan information for a xrep_findparent_scan call. */ + struct xrep_parent_scan_info *parent_scan; + + /* + * Parent that we've found for sc->ip. If we're scanning the entire + * directory tree, we need this to ensure that we only find /one/ + * parent directory. + */ + xfs_ino_t found_parent; + + /* + * This is set to true if @found_parent was not observed directly from + * the directory scan but by noticing a change in dotdot entries after + * cycling the sc->ip IOLOCK. + */ + bool parent_tentative; +}; + +/* + * If this directory entry points to the scrub target inode, then the directory + * we're scanning is the parent of the scrub target inode. + */ +STATIC int +xrep_findparent_dirent( + struct xfs_scrub *sc, + struct xfs_inode *dp, + xfs_dir2_dataptr_t dapos, + const struct xfs_name *name, + xfs_ino_t ino, + void *priv) +{ + struct xrep_findparent_info *fpi = priv; + int error = 0; + + if (xchk_should_terminate(fpi->sc, &error)) + return error; + + if (ino != I_INO(fpi->sc->ip)) + return 0; + + /* Ignore garbage directory entry names. */ + if (name->len == 0 || !xfs_dir2_namecheck(name->name, name->len)) + return -EFSCORRUPTED; + + /* + * Ignore dotdot and dot entries -- we're looking for parent -> child + * links only. + */ + if (name->name[0] == '.' && (name->len == 1 || + (name->len == 2 && name->name[1] == '.'))) + return 0; + + /* Uhoh, more than one parent for a dir? */ + if (fpi->found_parent != NULLFSINO && + !(fpi->parent_tentative && fpi->found_parent == I_INO(fpi->dp))) { + trace_xrep_findparent_dirent(fpi->sc->ip, 0); + return -EFSCORRUPTED; + } + + /* We found a potential parent; remember this. */ + trace_xrep_findparent_dirent(fpi->sc->ip, I_INO(fpi->dp)); + fpi->found_parent = I_INO(fpi->dp); + fpi->parent_tentative = false; + + if (fpi->parent_scan) + xrep_findparent_scan_found(fpi->parent_scan, I_INO(fpi->dp)); + + return 0; +} + +/* + * If this is a directory, walk the dirents looking for any that point to the + * scrub target inode. + */ +STATIC int +xrep_findparent_walk_directory( + struct xrep_findparent_info *fpi) +{ + struct xfs_scrub *sc = fpi->sc; + struct xfs_inode *dp = fpi->dp; + unsigned int lock_mode; + int error = 0; + + /* + * The inode being scanned cannot be its own parent, nor can any + * temporary directory we created to stage this repair. + */ + if (dp == sc->ip || dp == sc->tempip) + return 0; + + /* + * Similarly, temporary files created to stage a repair cannot be the + * parent of this inode. + */ + if (xrep_is_tempfile(dp)) + return 0; + + /* + * Scan the directory to see if there it contains an entry pointing to + * the directory that we are repairing. + */ + lock_mode = xfs_ilock_data_map_shared(dp); + + /* Don't mix metadata and regular directory trees. */ + if (xfs_is_metadir_inode(dp) != xfs_is_metadir_inode(sc->ip)) + goto out_unlock; + + /* + * If this directory is known to be sick, we cannot scan it reliably + * and must abort. + */ + if (xfs_inode_has_sickness(dp, XFS_SICK_INO_CORE | + XFS_SICK_INO_BMBTD | + XFS_SICK_INO_DIR)) { + error = -EFSCORRUPTED; + goto out_unlock; + } + + /* + * We cannot complete our parent pointer scan if a directory looks as + * though it has been zapped by the inode record repair code. + */ + if (xchk_dir_looks_zapped(dp)) { + error = -EBUSY; + goto out_unlock; + } + + error = xchk_dir_walk(sc, dp, xrep_findparent_dirent, fpi); + if (error) + goto out_unlock; + +out_unlock: + xfs_iunlock(dp, lock_mode); + return error; +} + +/* + * Update this directory's dotdot pointer based on ongoing dirent updates. + */ +STATIC int +xrep_findparent_live_update( + struct notifier_block *nb, + unsigned long action, + void *data) +{ + struct xfs_dir_update_params *p = data; + struct xrep_parent_scan_info *pscan; + struct xfs_scrub *sc; + + pscan = container_of(nb, struct xrep_parent_scan_info, + dhook.dirent_hook.nb); + sc = pscan->sc; + + /* + * If @p->ip is the subdirectory that we're interested in and we've + * already scanned @p->dp, update the dotdot target inumber to the + * parent inode. + */ + if (I_INO(p->ip) == I_INO(sc->ip) && + xchk_iscan_want_live_update(&pscan->iscan, I_INO(p->dp))) { + if (p->delta > 0) + xrep_findparent_scan_found(pscan, I_INO(p->dp)); + else + xrep_findparent_scan_found(pscan, NULLFSINO); + } + + return NOTIFY_DONE; +} + +/* + * Set up a scan to find the parent of a directory. The provided dirent hook + * will be called when there is a dotdot update for the inode being repaired. + */ +int +__xrep_findparent_scan_start( + struct xfs_scrub *sc, + struct xrep_parent_scan_info *pscan, + notifier_fn_t custom_fn) +{ + int error; + + if (!(sc->flags & XCHK_FSGATES_DIRENTS)) { + ASSERT(sc->flags & XCHK_FSGATES_DIRENTS); + return -EINVAL; + } + + pscan->sc = sc; + pscan->parent_ino = NULLFSINO; + + mutex_init(&pscan->lock); + + xchk_iscan_start(sc, 30000, 100, &pscan->iscan); + + /* + * Hook into the dirent update code. The hook only operates on inodes + * that were already scanned, and the scanner thread takes each inode's + * ILOCK, which means that any in-progress inode updates will finish + * before we can scan the inode. + */ + if (custom_fn) + xfs_dir_hook_setup(&pscan->dhook, custom_fn); + else + xfs_dir_hook_setup(&pscan->dhook, xrep_findparent_live_update); + error = xfs_dir_hook_add(sc->mp, &pscan->dhook); + if (error) + goto out_iscan; + + return 0; +out_iscan: + xchk_iscan_teardown(&pscan->iscan); + mutex_destroy(&pscan->lock); + return error; +} + +/* + * Scan the entire filesystem looking for a parent inode for the inode being + * scrubbed. @sc->ip must not be the root of a directory tree. Callers must + * not hold a dirty transaction or any lock that would interfere with taking + * an ILOCK. + * + * Returns 0 with @pscan->parent_ino set to the parent that we found. + * Returns 0 with @pscan->parent_ino set to NULLFSINO if we found no parents. + * Returns the usual negative errno if something else happened. + */ +int +xrep_findparent_scan( + struct xrep_parent_scan_info *pscan) +{ + struct xrep_findparent_info fpi = { + .sc = pscan->sc, + .found_parent = NULLFSINO, + .parent_scan = pscan, + }; + struct xfs_scrub *sc = pscan->sc; + int ret; + + ASSERT(S_ISDIR(VFS_IC(sc->ip)->i_mode)); + + while ((ret = xchk_iscan_iter(&pscan->iscan, &fpi.dp)) == 1) { + if (S_ISDIR(VFS_I(fpi.dp)->i_mode)) + ret = xrep_findparent_walk_directory(&fpi); + else + ret = 0; + xchk_iscan_mark_visited(&pscan->iscan, fpi.dp); + xchk_irele(sc, fpi.dp); + if (ret) + break; + + if (xchk_should_terminate(sc, &ret)) + break; + } + xchk_iscan_iter_finish(&pscan->iscan); + + return ret; +} + +/* Tear down a parent scan. */ +void +xrep_findparent_scan_teardown( + struct xrep_parent_scan_info *pscan) +{ + xfs_dir_hook_del(pscan->sc->mp, &pscan->dhook); + xchk_iscan_teardown(&pscan->iscan); + mutex_destroy(&pscan->lock); +} + +/* Finish a parent scan early. */ +void +xrep_findparent_scan_finish_early( + struct xrep_parent_scan_info *pscan, + xfs_ino_t ino) +{ + xrep_findparent_scan_found(pscan, ino); + xchk_iscan_finish_early(&pscan->iscan); +} + +/* + * Confirm that the directory @parent_ino actually contains a directory entry + * pointing to the child @sc->ip->ino. This function returns one of several + * ways: + * + * Returns 0 with @parent_ino unchanged if the parent was confirmed. + * Returns 0 with @parent_ino set to NULLFSINO if the parent was not valid. + * Returns the usual negative errno if something else happened. + */ +int +xrep_findparent_confirm( + struct xfs_scrub *sc, + xfs_ino_t *parent_ino) +{ + struct xrep_findparent_info fpi = { + .sc = sc, + .found_parent = NULLFSINO, + }; + int error; + + /* The root directory always points to itself. */ + if (sc->ip == sc->mp->m_rootip) { + *parent_ino = sc->mp->m_sb.sb_rootino; + return 0; + } + + /* The metadata root directory always points to itself. */ + if (sc->ip == sc->mp->m_metadirip) { + *parent_ino = sc->mp->m_sb.sb_metadirino; + return 0; + } + + /* Unlinked dirs can point anywhere; point them up to the root dir. */ + if (VFS_I(sc->ip)->i_nlink == 0) { + *parent_ino = xchk_inode_rootdir_inum(sc->ip); + return 0; + } + + /* Reject garbage parent inode numbers and self-referential parents. */ + if (*parent_ino == NULLFSINO) + return 0; + if (!xfs_verify_dir_ino(sc->mp, *parent_ino) || + *parent_ino == I_INO(sc->ip)) { + *parent_ino = NULLFSINO; + return 0; + } + + error = xchk_iget(sc, *parent_ino, &fpi.dp); + if (error) + return error; + + if (!S_ISDIR(VFS_I(fpi.dp)->i_mode)) { + *parent_ino = NULLFSINO; + goto out_rele; + } + + error = xrep_findparent_walk_directory(&fpi); + if (error) + goto out_rele; + + *parent_ino = fpi.found_parent; +out_rele: + xchk_irele(sc, fpi.dp); + return error; +} + +/* + * If we're the root of a directory tree, we are our own parent. If we're an + * unlinked directory, the parent /won't/ have a link to us. Set the parent + * directory to the root for both cases. Returns NULLFSINO if we don't know + * what to do. + */ +xfs_ino_t +xrep_findparent_self_reference( + struct xfs_scrub *sc) +{ + if (I_INO(sc->ip) == sc->mp->m_sb.sb_rootino) + return sc->mp->m_sb.sb_rootino; + + if (I_INO(sc->ip) == sc->mp->m_sb.sb_metadirino) + return sc->mp->m_sb.sb_metadirino; + + if (VFS_I(sc->ip)->i_nlink == 0) + return xchk_inode_rootdir_inum(sc->ip); + + return NULLFSINO; +} + +/* Check the dentry cache to see if knows of a parent for the scrub target. */ +xfs_ino_t +xrep_findparent_from_dcache( + struct xfs_scrub *sc) +{ + struct inode *pip = NULL; + struct dentry *dentry, *parent; + xfs_ino_t ret = NULLFSINO; + + dentry = d_find_alias(VFS_I(sc->ip)); + if (!dentry) + goto out; + + parent = dget_parent(dentry); + if (!parent) + goto out_dput; + + ASSERT(parent->d_sb == sc->ip->i_mount->m_super); + + pip = igrab(d_inode(parent)); + dput(parent); + + if (S_ISDIR(pip->i_mode)) { + ret = pip->i_ino; + trace_xrep_findparent_from_dcache(sc->ip, ret); + } + + xchk_irele(sc, XFS_I(pip)); + +out_dput: + dput(dentry); +out: + return ret; +} diff --git a/fs/xfs/scrub/findparent.h b/fs/xfs/scrub/findparent.h new file mode 100644 index 000000000..d998c7a88 --- /dev/null +++ b/fs/xfs/scrub/findparent.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_FINDPARENT_H__ +#define __XFS_SCRUB_FINDPARENT_H__ + +struct xrep_parent_scan_info { + struct xfs_scrub *sc; + + /* Inode scan cursor. */ + struct xchk_iscan iscan; + + /* Hook to capture directory entry updates. */ + struct xfs_dir_hook dhook; + + /* Lock protecting parent_ino. */ + struct mutex lock; + + /* Parent inode that we've found. */ + xfs_ino_t parent_ino; + + bool lookup_parent; +}; + +int __xrep_findparent_scan_start(struct xfs_scrub *sc, + struct xrep_parent_scan_info *pscan, + notifier_fn_t custom_fn); +static inline int xrep_findparent_scan_start(struct xfs_scrub *sc, + struct xrep_parent_scan_info *pscan) +{ + return __xrep_findparent_scan_start(sc, pscan, NULL); +} +int xrep_findparent_scan(struct xrep_parent_scan_info *pscan); +void xrep_findparent_scan_teardown(struct xrep_parent_scan_info *pscan); + +static inline void +xrep_findparent_scan_found( + struct xrep_parent_scan_info *pscan, + xfs_ino_t ino) +{ + mutex_lock(&pscan->lock); + pscan->parent_ino = ino; + mutex_unlock(&pscan->lock); +} + +void xrep_findparent_scan_finish_early(struct xrep_parent_scan_info *pscan, + xfs_ino_t ino); + +int xrep_findparent_confirm(struct xfs_scrub *sc, xfs_ino_t *parent_ino); + +xfs_ino_t xrep_findparent_self_reference(struct xfs_scrub *sc); +xfs_ino_t xrep_findparent_from_dcache(struct xfs_scrub *sc); + +#endif /* __XFS_SCRUB_FINDPARENT_H__ */ diff --git a/fs/xfs/scrub/fsb_bitmap.h b/fs/xfs/scrub/fsb_bitmap.h new file mode 100644 index 000000000..40b462c1d --- /dev/null +++ b/fs/xfs/scrub/fsb_bitmap.h @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_FSB_BITMAP_H__ +#define __XFS_SCRUB_FSB_BITMAP_H__ + +/* Bitmaps, but for type-checked for xfs_fsblock_t */ + +struct xfsb_bitmap { + struct xbitmap64 fsbitmap; +}; + +static inline void xfsb_bitmap_init(struct xfsb_bitmap *bitmap) +{ + xbitmap64_init(&bitmap->fsbitmap); +} + +static inline void xfsb_bitmap_destroy(struct xfsb_bitmap *bitmap) +{ + xbitmap64_destroy(&bitmap->fsbitmap); +} + +static inline int xfsb_bitmap_set(struct xfsb_bitmap *bitmap, + xfs_fsblock_t start, xfs_filblks_t len) +{ + return xbitmap64_set(&bitmap->fsbitmap, start, len); +} + +static inline int xfsb_bitmap_walk(struct xfsb_bitmap *bitmap, + xbitmap64_walk_fn fn, void *priv) +{ + return xbitmap64_walk(&bitmap->fsbitmap, fn, priv); +} + +#endif /* __XFS_SCRUB_FSB_BITMAP_H__ */ diff --git a/fs/xfs/scrub/fscounters.c b/fs/xfs/scrub/fscounters.c new file mode 100644 index 000000000..916699f3d --- /dev/null +++ b/fs/xfs/scrub/fscounters.c @@ -0,0 +1,621 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2019-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_mount.h" +#include "xfs_alloc.h" +#include "xfs_ialloc.h" +#include "xfs_health.h" +#include "xfs_btree.h" +#include "xfs_ag.h" +#include "xfs_rtbitmap.h" +#include "xfs_inode.h" +#include "xfs_icache.h" +#include "xfs_rtgroup.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/trace.h" +#include "scrub/fscounters.h" + +/* + * FS Summary Counters + * =================== + * + * The basics of filesystem summary counter checking are that we iterate the + * AGs counting the number of free blocks, free space btree blocks, per-AG + * reservations, inodes, delayed allocation reservations, and free inodes. + * Then we compare what we computed against the in-core counters. + * + * However, the reality is that summary counters are a tricky beast to check. + * While we /could/ freeze the filesystem and scramble around the AGs counting + * the free blocks, in practice we prefer not do that for a scan because + * freezing is costly. To get around this, we added a per-cpu counter of the + * delalloc reservations so that we can rotor around the AGs relatively + * quickly, and we allow the counts to be slightly off because we're not taking + * any locks while we do this. + * + * So the first thing we do is warm up the buffer cache in the setup routine by + * walking all the AGs to make sure the incore per-AG structure has been + * initialized. The expected value calculation then iterates the incore per-AG + * structures as quickly as it can. We snapshot the percpu counters before and + * after this operation and use the difference in counter values to guess at + * our tolerance for mismatch between expected and actual counter values. + */ + +/* + * Since the expected value computation is lockless but only browses incore + * values, the percpu counters should be fairly close to each other. However, + * we'll allow ourselves to be off by at least this (arbitrary) amount. + */ +#define XCHK_FSCOUNT_MIN_VARIANCE (512) + +/* + * Make sure the per-AG structure has been initialized from the on-disk header + * contents and trust that the incore counters match the ondisk counters. (The + * AGF and AGI scrubbers check them, and a normal xfs_scrub run checks the + * summary counters after checking all AG headers). Do this from the setup + * function so that the inner AG aggregation loop runs as quickly as possible. + * + * This function runs during the setup phase /before/ we start checking any + * metadata. + */ +STATIC int +xchk_fscount_warmup( + struct xfs_scrub *sc) +{ + struct xfs_mount *mp = sc->mp; + struct xfs_buf *agi_bp = NULL; + struct xfs_buf *agf_bp = NULL; + struct xfs_perag *pag = NULL; + int error = 0; + + while ((pag = xfs_perag_next(mp, pag))) { + if (xchk_should_terminate(sc, &error)) + break; + if (xfs_perag_initialised_agi(pag) && + xfs_perag_initialised_agf(pag)) + continue; + + /* Lock both AG headers. */ + error = xfs_ialloc_read_agi(pag, sc->tp, 0, &agi_bp); + if (error) + break; + error = xfs_alloc_read_agf(pag, sc->tp, 0, &agf_bp); + if (error) + break; + + /* + * These are supposed to be initialized by the header read + * function. + */ + if (!xfs_perag_initialised_agi(pag) || + !xfs_perag_initialised_agf(pag)) { + error = -EFSCORRUPTED; + break; + } + + xfs_buf_relse(agf_bp); + agf_bp = NULL; + xfs_buf_relse(agi_bp); + agi_bp = NULL; + } + + if (agf_bp) + xfs_buf_relse(agf_bp); + if (agi_bp) + xfs_buf_relse(agi_bp); + if (pag) + xfs_perag_rele(pag); + return error; +} + +static inline int +xchk_fsfreeze( + struct xfs_scrub *sc) +{ + int error; + + error = freeze_super(sc->mp->m_super, FREEZE_HOLDER_KERNEL, NULL); + trace_xchk_fsfreeze(sc, error); + return error; +} + +static inline int +xchk_fsthaw( + struct xfs_scrub *sc) +{ + int error; + + /* This should always succeed, we have a kernel freeze */ + error = thaw_super(sc->mp->m_super, FREEZE_HOLDER_KERNEL, NULL); + trace_xchk_fsthaw(sc, error); + return error; +} + +/* + * We couldn't stabilize the filesystem long enough to sample all the variables + * that comprise the summary counters and compare them to the percpu counters. + * We need to disable all writer threads, which means taking the first two + * freeze levels to put userspace to sleep, and the third freeze level to + * prevent background threads from starting new transactions. Take one level + * more to prevent other callers from unfreezing the filesystem while we run. + */ +STATIC int +xchk_fscounters_freeze( + struct xfs_scrub *sc) +{ + struct xchk_fscounters *fsc = sc->buf; + int error = 0; + + if (sc->flags & XCHK_HAVE_FREEZE_PROT) { + sc->flags &= ~XCHK_HAVE_FREEZE_PROT; + mnt_drop_write_file(sc->file); + } + + /* Try to grab a kernel freeze. */ + while ((error = xchk_fsfreeze(sc)) == -EBUSY) { + if (xchk_should_terminate(sc, &error)) + return error; + + delay(HZ / 10); + } + if (error) + return error; + + fsc->frozen = true; + return 0; +} + +/* Thaw the filesystem after checking or repairing fscounters. */ +STATIC void +xchk_fscounters_cleanup( + void *buf) +{ + struct xchk_fscounters *fsc = buf; + struct xfs_scrub *sc = fsc->sc; + int error; + + if (!fsc->frozen) + return; + + error = xchk_fsthaw(sc); + if (error) + xfs_emerg(sc->mp, "still frozen after scrub, err=%d", error); + else + fsc->frozen = false; +} + +int +xchk_setup_fscounters( + struct xfs_scrub *sc) +{ + struct xchk_fscounters *fsc; + int error; + + /* + * If the AGF doesn't track btreeblks, we have to lock the AGF to count + * btree block usage by walking the actual btrees. + */ + if (!xfs_has_lazysbcount(sc->mp)) + xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN); + + sc->buf = kzalloc_obj(struct xchk_fscounters, XCHK_GFP_FLAGS); + if (!sc->buf) + return -ENOMEM; + sc->buf_cleanup = xchk_fscounters_cleanup; + fsc = sc->buf; + fsc->sc = sc; + + xfs_icount_range(sc->mp, &fsc->icount_min, &fsc->icount_max); + + /* We must get the incore counters set up before we can proceed. */ + error = xchk_fscount_warmup(sc); + if (error) + return error; + + /* + * Pause all writer activity in the filesystem while we're scrubbing to + * reduce the likelihood of background perturbations to the counters + * throwing off our calculations. + * + * If we're repairing, we need to prevent any other thread from + * changing the global fs summary counters while we're repairing them. + * This requires the fs to be frozen, which will disable background + * reclaim and purge all inactive inodes. + */ + if ((sc->flags & XCHK_TRY_HARDER) || xchk_could_repair(sc)) { + error = xchk_fscounters_freeze(sc); + if (error) + return error; + } + + xchk_trans_alloc_empty(sc); + return 0; +} + +/* + * Part 1: Collecting filesystem summary counts. For each AG, we add its + * summary counts (total inodes, free inodes, free data blocks) to an incore + * copy of the overall filesystem summary counts. + * + * To avoid false corruption reports in part 2, any failure in this part must + * set the INCOMPLETE flag even when a negative errno is returned. This care + * must be taken with certain errno values (i.e. EFSBADCRC, EFSCORRUPTED, + * ECANCELED) that are absorbed into a scrub state flag update by + * xchk_*_process_error. Scrub and repair share the same incore data + * structures, so the INCOMPLETE flag is critical to prevent a repair based on + * insufficient information. + */ + +/* Count free space btree blocks manually for pre-lazysbcount filesystems. */ +static int +xchk_fscount_btreeblks( + struct xfs_scrub *sc, + struct xchk_fscounters *fsc, + xfs_agnumber_t agno) +{ + xfs_filblks_t blocks; + int error; + + error = xchk_ag_init_existing(sc, agno, &sc->sa); + if (error) + goto out_free; + + error = xfs_btree_count_blocks(sc->sa.bno_cur, &blocks); + if (error) + goto out_free; + fsc->fdblocks += blocks - 1; + + error = xfs_btree_count_blocks(sc->sa.cnt_cur, &blocks); + if (error) + goto out_free; + fsc->fdblocks += blocks - 1; + +out_free: + xchk_ag_free(sc, &sc->sa); + return error; +} + +/* + * Calculate what the global in-core counters ought to be from the incore + * per-AG structure. Callers can compare this to the actual in-core counters + * to estimate by how much both in-core and on-disk counters need to be + * adjusted. + */ +STATIC int +xchk_fscount_aggregate_agcounts( + struct xfs_scrub *sc, + struct xchk_fscounters *fsc) +{ + struct xfs_mount *mp = sc->mp; + struct xfs_perag *pag = NULL; + uint64_t delayed; + int tries = 8; + int error = 0; + +retry: + fsc->icount = 0; + fsc->ifree = 0; + fsc->fdblocks = 0; + + while ((pag = xfs_perag_next(mp, pag))) { + if (xchk_should_terminate(sc, &error)) + break; + + /* This somehow got unset since the warmup? */ + if (!xfs_perag_initialised_agi(pag) || + !xfs_perag_initialised_agf(pag)) { + error = -EFSCORRUPTED; + break; + } + + /* Count all the inodes */ + fsc->icount += pag->pagi_count; + fsc->ifree += pag->pagi_freecount; + + /* Add up the free/freelist/bnobt/cntbt blocks */ + fsc->fdblocks += pag->pagf_freeblks; + fsc->fdblocks += pag->pagf_flcount; + if (xfs_has_lazysbcount(sc->mp)) { + fsc->fdblocks += pag->pagf_btreeblks; + } else { + error = xchk_fscount_btreeblks(sc, fsc, pag_agno(pag)); + if (error) + break; + } + + /* + * Per-AG reservations are taken out of the incore counters, + * so they must be left out of the free blocks computation. + */ + fsc->fdblocks -= pag->pag_meta_resv.ar_reserved; + fsc->fdblocks -= pag->pag_rmapbt_resv.ar_orig_reserved; + + } + if (pag) + xfs_perag_rele(pag); + if (error) { + xchk_set_incomplete(sc); + return error; + } + + /* + * The global incore space reservation is taken from the incore + * counters, so leave that out of the computation. + */ + fsc->fdblocks -= mp->m_free[XC_FREE_BLOCKS].res_avail; + + /* + * Delayed allocation reservations are taken out of the incore counters + * but not recorded on disk, so leave them and their indlen blocks out + * of the computation. + */ + delayed = percpu_counter_sum(&mp->m_delalloc_blks); + fsc->fdblocks -= delayed; + + trace_xchk_fscounters_calc(mp, fsc->icount, fsc->ifree, fsc->fdblocks, + delayed); + + + /* Bail out if the values we compute are totally nonsense. */ + if (fsc->icount < fsc->icount_min || fsc->icount > fsc->icount_max || + fsc->fdblocks > mp->m_sb.sb_dblocks || + fsc->ifree > fsc->icount_max) + return -EFSCORRUPTED; + + /* + * If ifree > icount then we probably had some perturbation in the + * counters while we were calculating things. We'll try a few times + * to maintain ifree <= icount before giving up. + */ + if (fsc->ifree > fsc->icount) { + if (tries--) + goto retry; + return -EDEADLOCK; + } + + return 0; +} + +#ifdef CONFIG_XFS_RT +STATIC int +xchk_fscount_add_frextent( + struct xfs_rtgroup *rtg, + struct xfs_trans *tp, + const struct xfs_rtalloc_rec *rec, + void *priv) +{ + struct xchk_fscounters *fsc = priv; + int error = 0; + + fsc->frextents += rec->ar_extcount; + + xchk_should_terminate(fsc->sc, &error); + return error; +} + +/* Calculate the number of free realtime extents from the realtime bitmap. */ +STATIC int +xchk_fscount_count_frextents( + struct xfs_scrub *sc, + struct xchk_fscounters *fsc) +{ + struct xfs_mount *mp = sc->mp; + struct xfs_rtgroup *rtg = NULL; + int error; + + fsc->frextents = 0; + fsc->frextents_delayed = 0; + + /* + * Don't bother verifying and repairing the fs counters for zoned file + * systems as they don't track an on-disk frextents count, and the + * in-memory percpu counter also includes reservations. + */ + if (!xfs_has_realtime(mp) || xfs_has_zoned(mp)) + return 0; + + while ((rtg = xfs_rtgroup_next(mp, rtg))) { + xfs_rtgroup_lock(rtg, XFS_RTGLOCK_BITMAP_SHARED); + error = xfs_rtalloc_query_all(rtg, sc->tp, + xchk_fscount_add_frextent, fsc); + xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_BITMAP_SHARED); + if (error) { + xchk_set_incomplete(sc); + xfs_rtgroup_rele(rtg); + return error; + } + } + + fsc->frextents_delayed = percpu_counter_sum(&mp->m_delalloc_rtextents); + return 0; +} +#else +STATIC int +xchk_fscount_count_frextents( + struct xfs_scrub *sc, + struct xchk_fscounters *fsc) +{ + fsc->frextents = 0; + fsc->frextents_delayed = 0; + return 0; +} +#endif /* CONFIG_XFS_RT */ + +/* + * Part 2: Comparing filesystem summary counters. All we have to do here is + * sum the percpu counters and compare them to what we've observed. + */ + +/* + * Is the @counter reasonably close to the @expected value? + * + * We neither locked nor froze anything in the filesystem while aggregating the + * per-AG data to compute the @expected value, which means that the counter + * could have changed. We know the @old_value of the summation of the counter + * before the aggregation, and we re-sum the counter now. If the expected + * value falls between the two summations, we're ok. + * + * Otherwise, we /might/ have a problem. If the change in the summations is + * more than we want to tolerate, the filesystem is probably busy and we should + * just send back INCOMPLETE and see if userspace will try again. + * + * If we're repairing then we require an exact match. + */ +static inline bool +xchk_fscount_within_range( + struct xfs_scrub *sc, + const int64_t old_value, + struct percpu_counter *counter, + uint64_t expected) +{ + int64_t min_value, max_value; + int64_t curr_value = percpu_counter_sum(counter); + + trace_xchk_fscounters_within_range(sc->mp, expected, curr_value, + old_value); + + /* Negative values are always wrong. */ + if (curr_value < 0) + return false; + + /* Exact matches are always ok. */ + if (curr_value == expected) + return true; + + /* We require exact matches when repair is running. */ + if (sc->sm->sm_flags & XFS_SCRUB_IFLAG_REPAIR) + return false; + + min_value = min(old_value, curr_value); + max_value = max(old_value, curr_value); + + /* Within the before-and-after range is ok. */ + if (expected >= min_value && expected <= max_value) + return true; + + /* Everything else is bad. */ + return false; +} + +/* Check the superblock counters. */ +int +xchk_fscounters( + struct xfs_scrub *sc) +{ + struct xfs_mount *mp = sc->mp; + struct xchk_fscounters *fsc = sc->buf; + int64_t icount, ifree, fdblocks, frextents; + bool try_again = false; + int error; + + /* Snapshot the percpu counters. */ + icount = percpu_counter_sum(&mp->m_icount); + ifree = percpu_counter_sum(&mp->m_ifree); + fdblocks = xfs_sum_freecounter_raw(mp, XC_FREE_BLOCKS); + frextents = xfs_sum_freecounter_raw(mp, XC_FREE_RTEXTENTS); + + /* No negative values, please! */ + if (icount < 0 || ifree < 0) + xchk_set_corrupt(sc); + + /* + * If the filesystem is not frozen, the counter summation calls above + * can race with xfs_dec_freecounter, which subtracts a requested space + * reservation from the counter and undoes the subtraction if that made + * the counter go negative. Therefore, it's possible to see negative + * values here, and we should only flag that as a corruption if we + * froze the fs. This is much more likely to happen with frextents + * since there are no reserved pools. + */ + if (fdblocks < 0 || frextents < 0) { + if (!fsc->frozen) + return -EDEADLOCK; + + xchk_set_corrupt(sc); + return 0; + } + + /* See if icount is obviously wrong. */ + if (icount < fsc->icount_min || icount > fsc->icount_max) + xchk_set_corrupt(sc); + + /* See if fdblocks is obviously wrong. */ + if (fdblocks > mp->m_sb.sb_dblocks) + xchk_set_corrupt(sc); + + /* See if frextents is obviously wrong. */ + if (frextents > mp->m_sb.sb_rextents) + xchk_set_corrupt(sc); + + /* + * If ifree exceeds icount by more than the minimum variance then + * something's probably wrong with the counters. + */ + if (ifree > icount && ifree - icount > XCHK_FSCOUNT_MIN_VARIANCE) + xchk_set_corrupt(sc); + + /* Walk the incore AG headers to calculate the expected counters. */ + error = xchk_fscount_aggregate_agcounts(sc, fsc); + if (!xchk_process_error(sc, 0, XFS_SB_BLOCK(mp), &error)) + return error; + + /* Count the free extents counter for rt volumes. */ + error = xchk_fscount_count_frextents(sc, fsc); + if (!xchk_process_error(sc, 0, XFS_SB_BLOCK(mp), &error)) + return error; + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_INCOMPLETE) + return 0; + + /* + * Compare the in-core counters with whatever we counted. If the fs is + * frozen, we treat the discrepancy as a corruption because the freeze + * should have stabilized the counter values. Otherwise, we need + * userspace to call us back having granted us freeze permission. + */ + if (!xchk_fscount_within_range(sc, icount, &mp->m_icount, + fsc->icount)) { + if (fsc->frozen) + xchk_set_corrupt(sc); + else + try_again = true; + } + + if (!xchk_fscount_within_range(sc, ifree, &mp->m_ifree, fsc->ifree)) { + if (fsc->frozen) + xchk_set_corrupt(sc); + else + try_again = true; + } + + if (!xchk_fscount_within_range(sc, fdblocks, + &mp->m_free[XC_FREE_BLOCKS].count, fsc->fdblocks)) { + if (fsc->frozen) + xchk_set_corrupt(sc); + else + try_again = true; + } + + if (!xfs_has_zoned(mp) && + !xchk_fscount_within_range(sc, frextents, + &mp->m_free[XC_FREE_RTEXTENTS].count, + fsc->frextents - fsc->frextents_delayed)) { + if (fsc->frozen) + xchk_set_corrupt(sc); + else + try_again = true; + } + + if (try_again) + return -EDEADLOCK; + + return 0; +} diff --git a/fs/xfs/scrub/fscounters.h b/fs/xfs/scrub/fscounters.h new file mode 100644 index 000000000..bcf56e1c3 --- /dev/null +++ b/fs/xfs/scrub/fscounters.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2021-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_FSCOUNTERS_H__ +#define __XFS_SCRUB_FSCOUNTERS_H__ + +struct xchk_fscounters { + struct xfs_scrub *sc; + uint64_t icount; + uint64_t ifree; + uint64_t fdblocks; + uint64_t frextents; + uint64_t frextents_delayed; + unsigned long long icount_min; + unsigned long long icount_max; + bool frozen; +}; + +#endif /* __XFS_SCRUB_FSCOUNTERS_H__ */ diff --git a/fs/xfs/scrub/fscounters_repair.c b/fs/xfs/scrub/fscounters_repair.c new file mode 100644 index 000000000..783e409f8 --- /dev/null +++ b/fs/xfs/scrub/fscounters_repair.c @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2018-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_btree.h" +#include "xfs_bit.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_sb.h" +#include "xfs_inode.h" +#include "xfs_alloc.h" +#include "xfs_ialloc.h" +#include "xfs_rmap.h" +#include "xfs_health.h" +#include "scrub/xfs_scrub.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/fscounters.h" + +/* + * FS Summary Counters + * =================== + * + * We correct errors in the filesystem summary counters by setting them to the + * values computed during the obligatory scrub phase. However, we must be + * careful not to allow any other thread to change the counters while we're + * computing and setting new values. To achieve this, we freeze the + * filesystem for the whole operation if the REPAIR flag is set. The checking + * function is stricter when we've frozen the fs. + */ + +/* + * Reset the superblock counters. Caller is responsible for freezing the + * filesystem during the calculation and reset phases. + */ +int +xrep_fscounters( + struct xfs_scrub *sc) +{ + struct xfs_mount *mp = sc->mp; + struct xchk_fscounters *fsc = sc->buf; + + /* + * Reinitialize the in-core counters from what we computed. We froze + * the filesystem, so there shouldn't be anyone else trying to modify + * these counters. + */ + if (!fsc->frozen) { + ASSERT(fsc->frozen); + return -EFSCORRUPTED; + } + + trace_xrep_reset_counters(mp, fsc); + + percpu_counter_set(&mp->m_icount, fsc->icount); + percpu_counter_set(&mp->m_ifree, fsc->ifree); + xfs_set_freecounter(mp, XC_FREE_BLOCKS, fsc->fdblocks); + + /* + * Online repair is only supported on v5 file systems, which require + * lazy sb counters and thus no update of sb_fdblocks here. But + * sb_frextents only uses a lazy counter with rtgroups, and thus needs + * to be updated directly here otherwise. And for that we need to keep + * track of the delalloc reservations separately, as they are are + * subtracted from m_frextents, but not included in sb_frextents. + */ + if (!xfs_has_zoned(mp)) { + xfs_set_freecounter(mp, XC_FREE_RTEXTENTS, + fsc->frextents - fsc->frextents_delayed); + if (!xfs_has_rtgroups(mp)) + mp->m_sb.sb_frextents = fsc->frextents; + } + + return 0; +} diff --git a/fs/xfs/scrub/health.c b/fs/xfs/scrub/health.c new file mode 100644 index 000000000..2171bcf0f --- /dev/null +++ b/fs/xfs/scrub/health.c @@ -0,0 +1,337 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2019-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_btree.h" +#include "xfs_ag.h" +#include "xfs_health.h" +#include "xfs_rtgroup.h" +#include "scrub/scrub.h" +#include "scrub/health.h" +#include "scrub/common.h" + +/* + * Scrub and In-Core Filesystem Health Assessments + * =============================================== + * + * Online scrub and repair have the time and the ability to perform stronger + * checks than we can do from the metadata verifiers, because they can + * cross-reference records between data structures. Therefore, scrub is in a + * good position to update the online filesystem health assessments to reflect + * the good/bad state of the data structure. + * + * We therefore extend scrub in the following ways to achieve this: + * + * 1. Create a "sick_mask" field in the scrub context. When we're setting up a + * scrub call, set this to the default XFS_SICK_* flag(s) for the selected + * scrub type (call it A). Scrub and repair functions can override the default + * sick_mask value if they choose. + * + * 2. If the scrubber returns a runtime error code, we exit making no changes + * to the incore sick state. + * + * 3. If the scrubber finds that A is clean, use sick_mask to clear the incore + * sick flags before exiting. + * + * 4. If the scrubber finds that A is corrupt, use sick_mask to set the incore + * sick flags. If the user didn't want to repair then we exit, leaving the + * metadata structure unfixed and the sick flag set. + * + * 5. Now we know that A is corrupt and the user wants to repair, so run the + * repairer. If the repairer returns an error code, we exit with that error + * code, having made no further changes to the incore sick state. + * + * 6. If repair rebuilds A correctly and the subsequent re-scrub of A is clean, + * use sick_mask to clear the incore sick flags. This should have the effect + * that A is no longer marked sick. + * + * 7. If repair rebuilds A incorrectly, the re-scrub will find it corrupt and + * use sick_mask to set the incore sick flags. This should have no externally + * visible effect since we already set them in step (4). + * + * There are some complications to this story, however. For certain types of + * complementary metadata indices (e.g. inobt/finobt), it is easier to rebuild + * both structures at the same time. The following principles apply to this + * type of repair strategy: + * + * 8. Any repair function that rebuilds multiple structures should update + * sick_mask_visible to reflect whatever other structures are rebuilt, and + * verify that all the rebuilt structures can pass a scrub check. The outcomes + * of 5-7 still apply, but with a sick_mask that covers everything being + * rebuilt. + */ + +/* Map our scrub type to a sick mask and a set of health update functions. */ + +enum xchk_health_group { + XHG_NONE = 1, + XHG_FS, + XHG_AG, + XHG_INO, + XHG_RTGROUP, +}; + +struct xchk_health_map { + enum xchk_health_group group; + unsigned int sick_mask; +}; + +static const struct xchk_health_map type_to_health_flag[XFS_SCRUB_TYPE_NR] = { + [XFS_SCRUB_TYPE_PROBE] = { XHG_NONE, 0 }, + [XFS_SCRUB_TYPE_SB] = { XHG_AG, XFS_SICK_AG_SB }, + [XFS_SCRUB_TYPE_AGF] = { XHG_AG, XFS_SICK_AG_AGF }, + [XFS_SCRUB_TYPE_AGFL] = { XHG_AG, XFS_SICK_AG_AGFL }, + [XFS_SCRUB_TYPE_AGI] = { XHG_AG, XFS_SICK_AG_AGI }, + [XFS_SCRUB_TYPE_BNOBT] = { XHG_AG, XFS_SICK_AG_BNOBT }, + [XFS_SCRUB_TYPE_CNTBT] = { XHG_AG, XFS_SICK_AG_CNTBT }, + [XFS_SCRUB_TYPE_INOBT] = { XHG_AG, XFS_SICK_AG_INOBT }, + [XFS_SCRUB_TYPE_FINOBT] = { XHG_AG, XFS_SICK_AG_FINOBT }, + [XFS_SCRUB_TYPE_RMAPBT] = { XHG_AG, XFS_SICK_AG_RMAPBT }, + [XFS_SCRUB_TYPE_REFCNTBT] = { XHG_AG, XFS_SICK_AG_REFCNTBT }, + [XFS_SCRUB_TYPE_INODE] = { XHG_INO, XFS_SICK_INO_CORE }, + [XFS_SCRUB_TYPE_BMBTD] = { XHG_INO, XFS_SICK_INO_BMBTD }, + [XFS_SCRUB_TYPE_BMBTA] = { XHG_INO, XFS_SICK_INO_BMBTA }, + [XFS_SCRUB_TYPE_BMBTC] = { XHG_INO, XFS_SICK_INO_BMBTC }, + [XFS_SCRUB_TYPE_DIR] = { XHG_INO, XFS_SICK_INO_DIR }, + [XFS_SCRUB_TYPE_XATTR] = { XHG_INO, XFS_SICK_INO_XATTR }, + [XFS_SCRUB_TYPE_SYMLINK] = { XHG_INO, XFS_SICK_INO_SYMLINK }, + [XFS_SCRUB_TYPE_PARENT] = { XHG_INO, XFS_SICK_INO_PARENT }, + [XFS_SCRUB_TYPE_RTBITMAP] = { XHG_RTGROUP, XFS_SICK_RG_BITMAP }, + [XFS_SCRUB_TYPE_RTSUM] = { XHG_RTGROUP, XFS_SICK_RG_SUMMARY }, + [XFS_SCRUB_TYPE_UQUOTA] = { XHG_FS, XFS_SICK_FS_UQUOTA }, + [XFS_SCRUB_TYPE_GQUOTA] = { XHG_FS, XFS_SICK_FS_GQUOTA }, + [XFS_SCRUB_TYPE_PQUOTA] = { XHG_FS, XFS_SICK_FS_PQUOTA }, + [XFS_SCRUB_TYPE_FSCOUNTERS] = { XHG_FS, XFS_SICK_FS_COUNTERS }, + [XFS_SCRUB_TYPE_QUOTACHECK] = { XHG_FS, XFS_SICK_FS_QUOTACHECK }, + [XFS_SCRUB_TYPE_NLINKS] = { XHG_FS, XFS_SICK_FS_NLINKS }, + [XFS_SCRUB_TYPE_DIRTREE] = { XHG_INO, XFS_SICK_INO_DIRTREE }, + [XFS_SCRUB_TYPE_METAPATH] = { XHG_FS, XFS_SICK_FS_METAPATH }, + [XFS_SCRUB_TYPE_RGSUPER] = { XHG_RTGROUP, XFS_SICK_RG_SUPER }, + [XFS_SCRUB_TYPE_RTRMAPBT] = { XHG_RTGROUP, XFS_SICK_RG_RMAPBT }, + [XFS_SCRUB_TYPE_RTREFCBT] = { XHG_RTGROUP, XFS_SICK_RG_REFCNTBT }, +}; + +/* Return the health status mask for this scrub type. */ +unsigned int +xchk_health_mask_for_scrub_type( + __u32 scrub_type) +{ + return type_to_health_flag[scrub_type].sick_mask; +} + +/* + * If the scrub state is clean, add @mask to the scrub sick mask to clear + * additional sick flags from the metadata object's sick state. + */ +void +xchk_mark_healthy_if_clean( + struct xfs_scrub *sc, + unsigned int mask) +{ + if (!(sc->sm->sm_flags & (XFS_SCRUB_OFLAG_CORRUPT | + XFS_SCRUB_OFLAG_XCORRUPT))) + sc->healthy_mask |= mask; +} + +/* + * If we're scrubbing a piece of file metadata for the first time, does it look + * like it has been zapped? Skip the check if we just repaired the metadata + * and are revalidating it. + */ +bool +xchk_file_looks_zapped( + struct xfs_scrub *sc, + unsigned int mask) +{ + ASSERT((mask & ~XFS_SICK_INO_ZAPPED) == 0); + + if (sc->flags & XREP_ALREADY_FIXED) + return false; + + return xfs_inode_has_sickness(sc->ip, mask); +} + +/* + * Scrub gave the filesystem a clean bill of health, so clear all the indirect + * markers of past problems (at least for the fs and ags) so that we can be + * healthy again. + */ +STATIC void +xchk_mark_all_healthy( + struct xfs_mount *mp) +{ + struct xfs_perag *pag = NULL; + struct xfs_rtgroup *rtg = NULL; + + xfs_fs_mark_healthy(mp, XFS_SICK_FS_INDIRECT); + while ((pag = xfs_perag_next(mp, pag))) + xfs_group_mark_healthy(pag_group(pag), XFS_SICK_AG_INDIRECT); + while ((rtg = xfs_rtgroup_next(mp, rtg))) + xfs_group_mark_healthy(rtg_group(rtg), XFS_SICK_RG_INDIRECT); +} + +/* + * Update filesystem health assessments based on what we found and did. + * + * If the scrubber finds errors, we mark sick whatever's mentioned in + * sick_mask, no matter whether this is a first scan or an + * evaluation of repair effectiveness. + * + * Otherwise, no direct corruption was found, so mark whatever's in + * sick_mask as healthy. + */ +void +xchk_update_health( + struct xfs_scrub *sc) +{ + struct xfs_perag *pag; + struct xfs_rtgroup *rtg; + unsigned int mask = sc->sick_mask; + bool bad; + + /* + * The HEALTHY scrub type is a request from userspace to clear all the + * indirect flags after a clean scan of the entire filesystem. As such + * there's no sick flag defined for it, so we branch here ahead of the + * mask check. + */ + if (sc->sm->sm_type == XFS_SCRUB_TYPE_HEALTHY && + !(sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) { + xchk_mark_all_healthy(sc->mp); + return; + } + + bad = (sc->sm->sm_flags & (XFS_SCRUB_OFLAG_CORRUPT | + XFS_SCRUB_OFLAG_XCORRUPT)); + if (!bad) + mask |= sc->healthy_mask; + switch (type_to_health_flag[sc->sm->sm_type].group) { + case XHG_NONE: + break; + case XHG_AG: + if (!mask) + return; + pag = xfs_perag_get(sc->mp, sc->sm->sm_agno); + if (bad) + xfs_group_mark_corrupt(pag_group(pag), mask); + else + xfs_group_mark_healthy(pag_group(pag), mask); + xfs_perag_put(pag); + break; + case XHG_INO: + if (!sc->ip) + return; + /* + * If we're coming in for repairs then we don't want sickness + * flags to propagate to the incore health status if the inode + * gets inactivated before we can fix it. + */ + if (sc->sm->sm_flags & XFS_SCRUB_IFLAG_REPAIR) + mask |= XFS_SICK_INO_FORGET; + if (!mask) + return; + if (bad) + xfs_inode_mark_corrupt(sc->ip, mask); + else + xfs_inode_mark_healthy(sc->ip, mask); + break; + case XHG_FS: + if (!mask) + return; + if (bad) + xfs_fs_mark_corrupt(sc->mp, mask); + else + xfs_fs_mark_healthy(sc->mp, mask); + break; + case XHG_RTGROUP: + if (!mask) + return; + rtg = xfs_rtgroup_get(sc->mp, sc->sm->sm_agno); + if (bad) + xfs_group_mark_corrupt(rtg_group(rtg), mask); + else + xfs_group_mark_healthy(rtg_group(rtg), mask); + xfs_rtgroup_put(rtg); + break; + default: + ASSERT(0); + break; + } +} + +/* Is the given per-AG btree healthy enough for scanning? */ +void +xchk_ag_btree_del_cursor_if_sick( + struct xfs_scrub *sc, + struct xfs_btree_cur **curp, + unsigned int sm_type) +{ + unsigned int mask = (*curp)->bc_ops->sick_mask; + + /* + * We always want the cursor if it's the same type as whatever we're + * scrubbing, even if we already know the structure is corrupt. + * + * Otherwise, we're only interested in the btree for cross-referencing. + * If we know the btree is bad then don't bother, just set XFAIL. + */ + if (sc->sm->sm_type == sm_type) + return; + + /* + * If we just repaired some AG metadata, sc->sick_mask will reflect all + * the per-AG metadata types that were repaired. Exclude these from + * the filesystem health query because we have not yet updated the + * health status and we want everything to be scanned. + */ + if ((sc->flags & XREP_ALREADY_FIXED) && + type_to_health_flag[sc->sm->sm_type].group == XHG_AG) + mask &= ~sc->sick_mask; + + if (xfs_group_has_sickness((*curp)->bc_group, mask)) { + sc->sm->sm_flags |= XFS_SCRUB_OFLAG_XFAIL; + xfs_btree_del_cursor(*curp, XFS_BTREE_NOERROR); + *curp = NULL; + } +} + +/* + * Quick scan to double-check that there isn't any evidence of lingering + * primary health problems. If we're still clear, then the health update will + * take care of clearing the indirect evidence. + */ +int +xchk_health_record( + struct xfs_scrub *sc) +{ + struct xfs_mount *mp = sc->mp; + struct xfs_perag *pag = NULL; + struct xfs_rtgroup *rtg = NULL; + unsigned int sick; + unsigned int checked; + + xfs_fs_measure_sickness(mp, &sick, &checked); + if (sick & XFS_SICK_FS_PRIMARY) + xchk_set_corrupt(sc); + + while ((pag = xfs_perag_next(mp, pag))) { + xfs_group_measure_sickness(pag_group(pag), &sick, &checked); + if (sick & XFS_SICK_AG_PRIMARY) + xchk_set_corrupt(sc); + } + + while ((rtg = xfs_rtgroup_next(mp, rtg))) { + xfs_group_measure_sickness(rtg_group(rtg), &sick, &checked); + if (sick & XFS_SICK_RG_PRIMARY) + xchk_set_corrupt(sc); + } + + return 0; +} diff --git a/fs/xfs/scrub/health.h b/fs/xfs/scrub/health.h new file mode 100644 index 000000000..63fc426eb --- /dev/null +++ b/fs/xfs/scrub/health.h @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2019-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_HEALTH_H__ +#define __XFS_SCRUB_HEALTH_H__ + +unsigned int xchk_health_mask_for_scrub_type(__u32 scrub_type); +void xchk_update_health(struct xfs_scrub *sc); +void xchk_ag_btree_del_cursor_if_sick(struct xfs_scrub *sc, + struct xfs_btree_cur **curp, unsigned int sm_type); +void xchk_mark_healthy_if_clean(struct xfs_scrub *sc, unsigned int mask); +bool xchk_file_looks_zapped(struct xfs_scrub *sc, unsigned int mask); +int xchk_health_record(struct xfs_scrub *sc); + +#endif /* __XFS_SCRUB_HEALTH_H__ */ diff --git a/fs/xfs/scrub/ialloc.c b/fs/xfs/scrub/ialloc.c new file mode 100644 index 000000000..19c0b1b2a --- /dev/null +++ b/fs/xfs/scrub/ialloc.c @@ -0,0 +1,796 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_btree.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_inode.h" +#include "xfs_ialloc.h" +#include "xfs_ialloc_btree.h" +#include "xfs_icache.h" +#include "xfs_rmap.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/btree.h" +#include "scrub/trace.h" +#include "xfs_ag.h" + +/* + * Set us up to scrub inode btrees. + * If we detect a discrepancy between the inobt and the inode, + * try again after forcing logged inode cores out to disk. + */ +int +xchk_setup_ag_iallocbt( + struct xfs_scrub *sc) +{ + if (xchk_need_intent_drain(sc)) + xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN); + return xchk_setup_ag_btree(sc, sc->flags & XCHK_TRY_HARDER); +} + +/* Inode btree scrubber. */ + +struct xchk_iallocbt { + /* Number of inodes we see while scanning inobt. */ + unsigned long long inodes; + + /* Expected next startino, for big block filesystems. */ + xfs_agino_t next_startino; + + /* Expected end of the current inode cluster. */ + xfs_agino_t next_cluster_ino; +}; + +/* + * Does the finobt have a record for this inode with the same hole/free state? + * This is a bit complicated because of the following: + * + * - The finobt need not have a record if all inodes in the inobt record are + * allocated. + * - The finobt need not have a record if all inodes in the inobt record are + * free. + * - The finobt need not have a record if the inobt record says this is a hole. + * This likely doesn't happen in practice. + */ +STATIC int +xchk_inobt_xref_finobt( + struct xfs_scrub *sc, + struct xfs_inobt_rec_incore *irec, + xfs_agino_t agino, + bool free, + bool hole) +{ + struct xfs_inobt_rec_incore frec; + struct xfs_btree_cur *cur = sc->sa.fino_cur; + bool ffree, fhole; + unsigned int frec_idx, fhole_idx; + int has_record; + int error; + + ASSERT(xfs_btree_is_fino(cur->bc_ops)); + + error = xfs_inobt_lookup(cur, agino, XFS_LOOKUP_LE, &has_record); + if (error) + return error; + if (!has_record) + goto no_record; + + error = xfs_inobt_get_rec(cur, &frec, &has_record); + if (!has_record) + return -EFSCORRUPTED; + + if (frec.ir_startino + XFS_INODES_PER_CHUNK <= agino) + goto no_record; + + /* There's a finobt record; free and hole status must match. */ + frec_idx = agino - frec.ir_startino; + ffree = frec.ir_free & (1ULL << frec_idx); + fhole_idx = frec_idx / XFS_INODES_PER_HOLEMASK_BIT; + fhole = frec.ir_holemask & (1U << fhole_idx); + + if (ffree != free) + xchk_btree_xref_set_corrupt(sc, cur, 0); + if (fhole != hole) + xchk_btree_xref_set_corrupt(sc, cur, 0); + return 0; + +no_record: + /* inobt record is fully allocated */ + if (irec->ir_free == 0) + return 0; + + /* inobt record is totally unallocated */ + if (irec->ir_free == XFS_INOBT_ALL_FREE) + return 0; + + /* inobt record says this is a hole */ + if (hole) + return 0; + + /* finobt doesn't care about allocated inodes */ + if (!free) + return 0; + + xchk_btree_xref_set_corrupt(sc, cur, 0); + return 0; +} + +/* + * Make sure that each inode of this part of an inobt record has the same + * sparse and free status as the finobt. + */ +STATIC void +xchk_inobt_chunk_xref_finobt( + struct xfs_scrub *sc, + struct xfs_inobt_rec_incore *irec, + xfs_agino_t agino, + unsigned int nr_inodes) +{ + xfs_agino_t i; + unsigned int rec_idx; + int error; + + ASSERT(sc->sm->sm_type == XFS_SCRUB_TYPE_INOBT); + + if (!sc->sa.fino_cur || xchk_skip_xref(sc->sm)) + return; + + for (i = agino, rec_idx = agino - irec->ir_startino; + i < agino + nr_inodes; + i++, rec_idx++) { + bool free, hole; + unsigned int hole_idx; + + free = irec->ir_free & (1ULL << rec_idx); + hole_idx = rec_idx / XFS_INODES_PER_HOLEMASK_BIT; + hole = irec->ir_holemask & (1U << hole_idx); + + error = xchk_inobt_xref_finobt(sc, irec, i, free, hole); + if (!xchk_should_check_xref(sc, &error, &sc->sa.fino_cur)) + return; + } +} + +/* + * Does the inobt have a record for this inode with the same hole/free state? + * The inobt must always have a record if there's a finobt record. + */ +STATIC int +xchk_finobt_xref_inobt( + struct xfs_scrub *sc, + struct xfs_inobt_rec_incore *frec, + xfs_agino_t agino, + bool ffree, + bool fhole) +{ + struct xfs_inobt_rec_incore irec; + struct xfs_btree_cur *cur = sc->sa.ino_cur; + bool free, hole; + unsigned int rec_idx, hole_idx; + int has_record; + int error; + + ASSERT(xfs_btree_is_ino(cur->bc_ops)); + + error = xfs_inobt_lookup(cur, agino, XFS_LOOKUP_LE, &has_record); + if (error) + return error; + if (!has_record) + goto no_record; + + error = xfs_inobt_get_rec(cur, &irec, &has_record); + if (!has_record) + return -EFSCORRUPTED; + + if (irec.ir_startino + XFS_INODES_PER_CHUNK <= agino) + goto no_record; + + /* There's an inobt record; free and hole status must match. */ + rec_idx = agino - irec.ir_startino; + free = irec.ir_free & (1ULL << rec_idx); + hole_idx = rec_idx / XFS_INODES_PER_HOLEMASK_BIT; + hole = irec.ir_holemask & (1U << hole_idx); + + if (ffree != free) + xchk_btree_xref_set_corrupt(sc, cur, 0); + if (fhole != hole) + xchk_btree_xref_set_corrupt(sc, cur, 0); + return 0; + +no_record: + /* finobt should never have a record for which the inobt does not */ + xchk_btree_xref_set_corrupt(sc, cur, 0); + return 0; +} + +/* + * Make sure that each inode of this part of an finobt record has the same + * sparse and free status as the inobt. + */ +STATIC void +xchk_finobt_chunk_xref_inobt( + struct xfs_scrub *sc, + struct xfs_inobt_rec_incore *frec, + xfs_agino_t agino, + unsigned int nr_inodes) +{ + xfs_agino_t i; + unsigned int rec_idx; + int error; + + ASSERT(sc->sm->sm_type == XFS_SCRUB_TYPE_FINOBT); + + if (!sc->sa.ino_cur || xchk_skip_xref(sc->sm)) + return; + + for (i = agino, rec_idx = agino - frec->ir_startino; + i < agino + nr_inodes; + i++, rec_idx++) { + bool ffree, fhole; + unsigned int hole_idx; + + ffree = frec->ir_free & (1ULL << rec_idx); + hole_idx = rec_idx / XFS_INODES_PER_HOLEMASK_BIT; + fhole = frec->ir_holemask & (1U << hole_idx); + + error = xchk_finobt_xref_inobt(sc, frec, i, ffree, fhole); + if (!xchk_should_check_xref(sc, &error, &sc->sa.ino_cur)) + return; + } +} + +/* Is this chunk worth checking and cross-referencing? */ +STATIC bool +xchk_iallocbt_chunk( + struct xchk_btree *bs, + struct xfs_inobt_rec_incore *irec, + xfs_agino_t agino, + unsigned int nr_inodes) +{ + struct xfs_scrub *sc = bs->sc; + struct xfs_mount *mp = bs->cur->bc_mp; + struct xfs_perag *pag = to_perag(bs->cur->bc_group); + xfs_agblock_t agbno; + xfs_extlen_t len; + + agbno = XFS_AGINO_TO_AGBNO(mp, agino); + len = XFS_B_TO_FSB(mp, nr_inodes * mp->m_sb.sb_inodesize); + + if (!xfs_verify_agbext(pag, agbno, len)) + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + + if (bs->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return false; + + xchk_xref_is_used_space(sc, agbno, len); + if (sc->sm->sm_type == XFS_SCRUB_TYPE_INOBT) + xchk_inobt_chunk_xref_finobt(sc, irec, agino, nr_inodes); + else + xchk_finobt_chunk_xref_inobt(sc, irec, agino, nr_inodes); + xchk_xref_is_only_owned_by(sc, agbno, len, &XFS_RMAP_OINFO_INODES); + xchk_xref_is_not_shared(sc, agbno, len); + xchk_xref_is_not_cow_staging(sc, agbno, len); + return true; +} + +/* + * Check that an inode's allocation status matches ir_free in the inobt + * record. First we try querying the in-core inode state, and if the inode + * isn't loaded we examine the on-disk inode directly. + * + * Since there can be 1:M and M:1 mappings between inobt records and inode + * clusters, we pass in the inode location information as an inobt record; + * the index of an inode cluster within the inobt record (as well as the + * cluster buffer itself); and the index of the inode within the cluster. + * + * @irec is the inobt record. + * @irec_ino is the inode offset from the start of the record. + * @dip is the on-disk inode. + */ +STATIC int +xchk_iallocbt_check_cluster_ifree( + struct xchk_btree *bs, + struct xfs_inobt_rec_incore *irec, + unsigned int irec_ino, + struct xfs_dinode *dip) +{ + xfs_ino_t fsino; + xfs_agino_t agino; + bool irec_free; + bool ino_inuse; + bool freemask_ok; + int error = 0; + + if (xchk_should_terminate(bs->sc, &error)) + return error; + + /* + * Given an inobt record and the offset of an inode from the start of + * the record, compute which fs inode we're talking about. + */ + agino = irec->ir_startino + irec_ino; + fsino = xfs_agino_to_ino(to_perag(bs->cur->bc_group), agino); + irec_free = (irec->ir_free & XFS_INOBT_MASK(irec_ino)); + + if (be16_to_cpu(dip->di_magic) != XFS_DINODE_MAGIC || + (dip->di_version >= 3 && be64_to_cpu(dip->di_ino) != fsino)) { + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + goto out; + } + + error = xchk_inode_is_allocated(bs->sc, agino, &ino_inuse); + if (error == -ENODATA) { + /* Not cached, just read the disk buffer */ + freemask_ok = irec_free ^ !!(dip->di_mode); + if (!(bs->sc->flags & XCHK_TRY_HARDER) && !freemask_ok) + return -EDEADLOCK; + } else if (error < 0) { + /* + * Inode is only half assembled, or there was an IO error, + * or the verifier failed, so don't bother trying to check. + * The inode scrubber can deal with this. + */ + goto out; + } else { + /* Inode is all there. */ + freemask_ok = irec_free ^ ino_inuse; + } + if (!freemask_ok) + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); +out: + return 0; +} + +/* + * Check that the holemask and freemask of a hypothetical inode cluster match + * what's actually on disk. If sparse inodes are enabled, the cluster does + * not actually have to map to inodes if the corresponding holemask bit is set. + * + * @cluster_base is the first inode in the cluster within the @irec. + */ +STATIC int +xchk_iallocbt_check_cluster( + struct xchk_btree *bs, + struct xfs_inobt_rec_incore *irec, + unsigned int cluster_base) +{ + struct xfs_perag *pag = to_perag(bs->cur->bc_group); + struct xfs_imap imap; + struct xfs_mount *mp = bs->cur->bc_mp; + struct xfs_buf *cluster_bp; + unsigned int nr_inodes; + xfs_agblock_t agbno; + unsigned int cluster_index; + uint16_t cluster_mask = 0; + uint16_t ir_holemask; + int error = 0; + + nr_inodes = min_t(unsigned int, XFS_INODES_PER_CHUNK, + M_IGEO(mp)->inodes_per_cluster); + + /* Map this inode cluster */ + agbno = XFS_AGINO_TO_AGBNO(mp, irec->ir_startino + cluster_base); + + /* Compute a bitmask for this cluster that can be used for holemask. */ + for (cluster_index = 0; + cluster_index < nr_inodes; + cluster_index += XFS_INODES_PER_HOLEMASK_BIT) + cluster_mask |= XFS_INOBT_MASK((cluster_base + cluster_index) / + XFS_INODES_PER_HOLEMASK_BIT); + + /* + * Map the first inode of this cluster to a buffer and offset. + * Be careful about inobt records that don't align with the start of + * the inode buffer when block sizes are large enough to hold multiple + * inode chunks. When this happens, cluster_base will be zero but + * ir_startino can be large enough to make im_boffset nonzero. + */ + ir_holemask = (irec->ir_holemask & cluster_mask); + imap.im_agbno = agbno; + imap.im_boffset = XFS_INO_TO_OFFSET(mp, irec->ir_startino) << + mp->m_sb.sb_inodelog; + + if (imap.im_boffset != 0 && cluster_base != 0) { + ASSERT(imap.im_boffset == 0 || cluster_base == 0); + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + return 0; + } + + trace_xchk_iallocbt_check_cluster(pag, irec->ir_startino, imap.im_agbno, + XFS_FSB_TO_BB(mp, M_IGEO(mp)->blocks_per_cluster), + cluster_base, nr_inodes, cluster_mask, ir_holemask, + XFS_INO_TO_OFFSET(mp, irec->ir_startino + + cluster_base)); + + /* The whole cluster must be a hole or not a hole. */ + if (ir_holemask != cluster_mask && ir_holemask != 0) { + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + return 0; + } + + /* If any part of this is a hole, skip it. */ + if (ir_holemask) { + xchk_xref_is_not_owned_by(bs->sc, agbno, + M_IGEO(mp)->blocks_per_cluster, + &XFS_RMAP_OINFO_INODES); + return 0; + } + + xchk_xref_is_only_owned_by(bs->sc, agbno, M_IGEO(mp)->blocks_per_cluster, + &XFS_RMAP_OINFO_INODES); + + /* Grab the inode cluster buffer. */ + error = xfs_read_icluster(pag, bs->cur->bc_tp, imap.im_agbno, + &cluster_bp); + if (!xchk_btree_xref_process_error(bs->sc, bs->cur, 0, &error)) + return error; + + /* Check free status of each inode within this cluster. */ + for (cluster_index = 0; cluster_index < nr_inodes; cluster_index++) { + struct xfs_dinode *dip; + + if (imap.im_boffset >= BBTOB(cluster_bp->b_length)) { + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + break; + } + + dip = xfs_buf_offset(cluster_bp, imap.im_boffset); + error = xchk_iallocbt_check_cluster_ifree(bs, irec, + cluster_base + cluster_index, dip); + if (error) + break; + imap.im_boffset += mp->m_sb.sb_inodesize; + } + + xfs_trans_brelse(bs->cur->bc_tp, cluster_bp); + return error; +} + +/* + * For all the inode clusters that could map to this inobt record, make sure + * that the holemask makes sense and that the allocation status of each inode + * matches the freemask. + */ +STATIC int +xchk_iallocbt_check_clusters( + struct xchk_btree *bs, + struct xfs_inobt_rec_incore *irec) +{ + unsigned int cluster_base; + int error = 0; + + /* + * For the common case where this inobt record maps to multiple inode + * clusters this will call _check_cluster for each cluster. + * + * For the case that multiple inobt records map to a single cluster, + * this will call _check_cluster once. + */ + for (cluster_base = 0; + cluster_base < XFS_INODES_PER_CHUNK; + cluster_base += M_IGEO(bs->sc->mp)->inodes_per_cluster) { + error = xchk_iallocbt_check_cluster(bs, irec, cluster_base); + if (error) + break; + } + + return error; +} + +/* + * Make sure this inode btree record is aligned properly. Because a fs block + * contains multiple inodes, we check that the inobt record is aligned to the + * correct inode, not just the correct block on disk. This results in a finer + * grained corruption check. + */ +STATIC void +xchk_iallocbt_rec_alignment( + struct xchk_btree *bs, + struct xfs_inobt_rec_incore *irec) +{ + struct xfs_mount *mp = bs->sc->mp; + struct xchk_iallocbt *iabt = bs->private; + struct xfs_ino_geometry *igeo = M_IGEO(mp); + + /* + * finobt records have different positioning requirements than inobt + * records: each finobt record must have a corresponding inobt record. + * That is checked in the xref function, so for now we only catch the + * obvious case where the record isn't at all aligned properly. + * + * Note that if a fs block contains more than a single chunk of inodes, + * we will have finobt records only for those chunks containing free + * inodes, and therefore expect chunk alignment of finobt records. + * Otherwise, we expect that the finobt record is aligned to the + * cluster alignment as told by the superblock. + */ + if (xfs_btree_is_fino(bs->cur->bc_ops)) { + unsigned int imask; + + imask = min_t(unsigned int, XFS_INODES_PER_CHUNK, + igeo->cluster_align_inodes) - 1; + if (irec->ir_startino & imask) + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + return; + } + + if (iabt->next_startino != NULLAGINO) { + /* + * We're midway through a cluster of inodes that is mapped by + * multiple inobt records. Did we get the record for the next + * irec in the sequence? + */ + if (irec->ir_startino != iabt->next_startino) { + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + return; + } + + iabt->next_startino += XFS_INODES_PER_CHUNK; + + /* Are we done with the cluster? */ + if (iabt->next_startino >= iabt->next_cluster_ino) { + iabt->next_startino = NULLAGINO; + iabt->next_cluster_ino = NULLAGINO; + } + return; + } + + /* inobt records must be aligned to cluster and inoalignmnt size. */ + if (irec->ir_startino & (igeo->cluster_align_inodes - 1)) { + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + return; + } + + if (irec->ir_startino & (igeo->inodes_per_cluster - 1)) { + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + return; + } + + if (igeo->inodes_per_cluster <= XFS_INODES_PER_CHUNK) + return; + + /* + * If this is the start of an inode cluster that can be mapped by + * multiple inobt records, the next inobt record must follow exactly + * after this one. + */ + iabt->next_startino = irec->ir_startino + XFS_INODES_PER_CHUNK; + iabt->next_cluster_ino = irec->ir_startino + igeo->inodes_per_cluster; +} + +/* Scrub an inobt/finobt record. */ +STATIC int +xchk_iallocbt_rec( + struct xchk_btree *bs, + const union xfs_btree_rec *rec) +{ + struct xfs_mount *mp = bs->cur->bc_mp; + struct xchk_iallocbt *iabt = bs->private; + struct xfs_inobt_rec_incore irec; + uint64_t holes; + xfs_agino_t agino; + int holecount; + int i; + int error = 0; + uint16_t holemask; + + xfs_inobt_btrec_to_irec(mp, rec, &irec); + if (xfs_inobt_check_irec(to_perag(bs->cur->bc_group), &irec) != NULL) { + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + return 0; + } + + agino = irec.ir_startino; + + xchk_iallocbt_rec_alignment(bs, &irec); + if (bs->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out; + + iabt->inodes += irec.ir_count; + + /* Handle non-sparse inodes */ + if (!xfs_inobt_issparse(irec.ir_holemask)) { + if (irec.ir_count != XFS_INODES_PER_CHUNK) + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + + if (!xchk_iallocbt_chunk(bs, &irec, agino, + XFS_INODES_PER_CHUNK)) + goto out; + goto check_clusters; + } + + /* Check each chunk of a sparse inode cluster. */ + holemask = irec.ir_holemask; + holecount = 0; + holes = ~xfs_inobt_irec_to_allocmask(&irec); + if ((holes & irec.ir_free) != holes || + irec.ir_freecount > irec.ir_count) + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + + for (i = 0; i < XFS_INOBT_HOLEMASK_BITS; i++) { + if (holemask & 1) + holecount += XFS_INODES_PER_HOLEMASK_BIT; + else if (!xchk_iallocbt_chunk(bs, &irec, agino, + XFS_INODES_PER_HOLEMASK_BIT)) + goto out; + holemask >>= 1; + agino += XFS_INODES_PER_HOLEMASK_BIT; + } + + if (holecount > XFS_INODES_PER_CHUNK || + holecount + irec.ir_count != XFS_INODES_PER_CHUNK) + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + +check_clusters: + if (bs->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out; + + error = xchk_iallocbt_check_clusters(bs, &irec); + if (error) + goto out; + +out: + return error; +} + +/* + * Make sure the inode btrees are as large as the rmap thinks they are. + * Don't bother if we're missing btree cursors, as we're already corrupt. + */ +STATIC void +xchk_iallocbt_xref_rmap_btreeblks( + struct xfs_scrub *sc) +{ + xfs_filblks_t blocks; + xfs_filblks_t inobt_blocks = 0; + xfs_filblks_t finobt_blocks = 0; + int error; + + if (!sc->sa.ino_cur || !sc->sa.rmap_cur || + (xfs_has_finobt(sc->mp) && !sc->sa.fino_cur) || + xchk_skip_xref(sc->sm)) + return; + + /* Check that we saw as many inobt blocks as the rmap says. */ + error = xfs_btree_count_blocks(sc->sa.ino_cur, &inobt_blocks); + if (!xchk_process_error(sc, 0, 0, &error)) + return; + + if (sc->sa.fino_cur) { + error = xfs_btree_count_blocks(sc->sa.fino_cur, &finobt_blocks); + if (!xchk_process_error(sc, 0, 0, &error)) + return; + } + + error = xchk_count_rmap_ownedby_ag(sc, sc->sa.rmap_cur, + &XFS_RMAP_OINFO_INOBT, &blocks); + if (!xchk_should_check_xref(sc, &error, &sc->sa.rmap_cur)) + return; + if (blocks != inobt_blocks + finobt_blocks) + xchk_btree_set_corrupt(sc, sc->sa.ino_cur, 0); +} + +/* + * Make sure that the inobt records point to the same number of blocks as + * the rmap says are owned by inodes. + */ +STATIC void +xchk_iallocbt_xref_rmap_inodes( + struct xfs_scrub *sc, + unsigned long long inodes) +{ + xfs_filblks_t blocks; + xfs_filblks_t inode_blocks; + int error; + + if (!sc->sa.rmap_cur || xchk_skip_xref(sc->sm)) + return; + + /* Check that we saw as many inode blocks as the rmap knows about. */ + error = xchk_count_rmap_ownedby_ag(sc, sc->sa.rmap_cur, + &XFS_RMAP_OINFO_INODES, &blocks); + if (!xchk_should_check_xref(sc, &error, &sc->sa.rmap_cur)) + return; + inode_blocks = XFS_B_TO_FSB(sc->mp, inodes * sc->mp->m_sb.sb_inodesize); + if (blocks != inode_blocks) + xchk_btree_xref_set_corrupt(sc, sc->sa.rmap_cur, 0); +} + +/* Scrub one of the inode btrees for some AG. */ +int +xchk_iallocbt( + struct xfs_scrub *sc) +{ + struct xfs_btree_cur *cur; + struct xchk_iallocbt iabt = { + .inodes = 0, + .next_startino = NULLAGINO, + .next_cluster_ino = NULLAGINO, + }; + int error; + + switch (sc->sm->sm_type) { + case XFS_SCRUB_TYPE_INOBT: + cur = sc->sa.ino_cur; + break; + case XFS_SCRUB_TYPE_FINOBT: + cur = sc->sa.fino_cur; + break; + default: + ASSERT(0); + return -EIO; + } + + error = xchk_btree(sc, cur, xchk_iallocbt_rec, &XFS_RMAP_OINFO_INOBT, + &iabt); + if (error) + return error; + + xchk_iallocbt_xref_rmap_btreeblks(sc); + + /* + * If we're scrubbing the inode btree, inode_blocks is the number of + * blocks pointed to by all the inode chunk records. Therefore, we + * should compare to the number of inode chunk blocks that the rmap + * knows about. We can't do this for the finobt since it only points + * to inode chunks with free inodes. + */ + if (sc->sm->sm_type == XFS_SCRUB_TYPE_INOBT) + xchk_iallocbt_xref_rmap_inodes(sc, iabt.inodes); + return error; +} + +/* See if an inode btree has (or doesn't have) an inode chunk record. */ +static inline void +xchk_xref_inode_check( + struct xfs_scrub *sc, + xfs_agblock_t agbno, + xfs_extlen_t len, + struct xfs_btree_cur **icur, + enum xbtree_recpacking expected) +{ + enum xbtree_recpacking outcome; + int error; + + if (!(*icur) || xchk_skip_xref(sc->sm)) + return; + + error = xfs_ialloc_has_inodes_at_extent(*icur, agbno, len, &outcome); + if (!xchk_should_check_xref(sc, &error, icur)) + return; + if (outcome != expected) + xchk_btree_xref_set_corrupt(sc, *icur, 0); +} + +/* xref check that the extent is not covered by inodes */ +void +xchk_xref_is_not_inode_chunk( + struct xfs_scrub *sc, + xfs_agblock_t agbno, + xfs_extlen_t len) +{ + xchk_xref_inode_check(sc, agbno, len, &sc->sa.ino_cur, + XBTREE_RECPACKING_EMPTY); + xchk_xref_inode_check(sc, agbno, len, &sc->sa.fino_cur, + XBTREE_RECPACKING_EMPTY); +} + +/* xref check that the extent is covered by inodes */ +void +xchk_xref_is_inode_chunk( + struct xfs_scrub *sc, + xfs_agblock_t agbno, + xfs_extlen_t len) +{ + xchk_xref_inode_check(sc, agbno, len, &sc->sa.ino_cur, + XBTREE_RECPACKING_FULL); +} diff --git a/fs/xfs/scrub/ialloc_repair.c b/fs/xfs/scrub/ialloc_repair.c new file mode 100644 index 000000000..46b1c8ac5 --- /dev/null +++ b/fs/xfs/scrub/ialloc_repair.c @@ -0,0 +1,885 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_btree.h" +#include "xfs_btree_staging.h" +#include "xfs_bit.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_sb.h" +#include "xfs_inode.h" +#include "xfs_alloc.h" +#include "xfs_ialloc.h" +#include "xfs_ialloc_btree.h" +#include "xfs_icache.h" +#include "xfs_rmap.h" +#include "xfs_rmap_btree.h" +#include "xfs_log.h" +#include "xfs_trans_priv.h" +#include "xfs_error.h" +#include "xfs_health.h" +#include "xfs_ag.h" +#include "scrub/xfs_scrub.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/btree.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/bitmap.h" +#include "scrub/agb_bitmap.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/newbt.h" +#include "scrub/reap.h" + +/* + * Inode Btree Repair + * ================== + * + * A quick refresher of inode btrees on a v5 filesystem: + * + * - Inode records are read into memory in units of 'inode clusters'. However + * many inodes fit in a cluster buffer is the smallest number of inodes that + * can be allocated or freed. Clusters are never smaller than one fs block + * though they can span multiple blocks. The size (in fs blocks) is + * computed with xfs_icluster_size_fsb(). The fs block alignment of a + * cluster is computed with xfs_ialloc_cluster_alignment(). + * + * - Each inode btree record can describe a single 'inode chunk'. The chunk + * size is defined to be 64 inodes. If sparse inodes are enabled, every + * inobt record must be aligned to the chunk size; if not, every record must + * be aligned to the start of a cluster. It is possible to construct an XFS + * geometry where one inobt record maps to multiple inode clusters; it is + * also possible to construct a geometry where multiple inobt records map to + * different parts of one inode cluster. + * + * - If sparse inodes are not enabled, the smallest unit of allocation for + * inode records is enough to contain one inode chunk's worth of inodes. + * + * - If sparse inodes are enabled, the holemask field will be active. Each + * bit of the holemask represents 4 potential inodes; if set, the + * corresponding space does *not* contain inodes and must be left alone. + * Clusters cannot be smaller than 4 inodes. The smallest unit of allocation + * of inode records is one inode cluster. + * + * So what's the rebuild algorithm? + * + * Iterate the reverse mapping records looking for OWN_INODES and OWN_INOBT + * records. The OWN_INOBT records are the old inode btree blocks and will be + * cleared out after we've rebuilt the tree. Each possible inode cluster + * within an OWN_INODES record will be read in; for each possible inobt record + * associated with that cluster, compute the freemask calculated from the + * i_mode data in the inode chunk. For sparse inodes the holemask will be + * calculated by creating the properly aligned inobt record and punching out + * any chunk that's missing. Inode allocations and frees grab the AGI first, + * so repair protects itself from concurrent access by locking the AGI. + * + * Once we've reconstructed all the inode records, we can create new inode + * btree roots and reload the btrees. We rebuild both inode trees at the same + * time because they have the same rmap owner and it would be more complex to + * figure out if the other tree isn't in need of a rebuild and which OWN_INOBT + * blocks it owns. We have all the data we need to build both, so dump + * everything and start over. + * + * We use the prefix 'xrep_ibt' because we rebuild both inode btrees at once. + */ + +struct xrep_ibt { + /* Record under construction. */ + struct xfs_inobt_rec_incore rie; + + /* new inobt information */ + struct xrep_newbt new_inobt; + + /* new finobt information */ + struct xrep_newbt new_finobt; + + /* Old inode btree blocks we found in the rmap. */ + struct xagb_bitmap old_iallocbt_blocks; + + /* Reconstructed inode records. */ + struct xfarray *inode_records; + + struct xfs_scrub *sc; + + /* Number of inodes assigned disk space. */ + unsigned int icount; + + /* Number of inodes in use. */ + unsigned int iused; + + /* Number of finobt records needed. */ + unsigned int finobt_recs; + + /* get_records()'s position in the inode record array. */ + xfarray_idx_t array_cur; +}; + +/* + * Is this inode in use? If the inode is in memory we can tell from i_mode, + * otherwise we have to check di_mode in the on-disk buffer. We only care + * that the high (i.e. non-permission) bits of _mode are zero. This should be + * safe because repair keeps all AG headers locked until the end, and process + * trying to perform an inode allocation/free must lock the AGI. + * + * @cluster_ag_base is the inode offset of the cluster within the AG. + * @cluster_bp is the cluster buffer. + * @cluster_index is the inode offset within the inode cluster. + */ +STATIC int +xrep_ibt_check_ifree( + struct xrep_ibt *ri, + xfs_agino_t cluster_ag_base, + struct xfs_buf *cluster_bp, + unsigned int cluster_index, + bool *inuse) +{ + struct xfs_scrub *sc = ri->sc; + struct xfs_mount *mp = sc->mp; + struct xfs_dinode *dip; + xfs_agino_t agino; + unsigned int cluster_buf_base; + unsigned int offset; + int error; + + agino = cluster_ag_base + cluster_index; + + /* Inode uncached or half assembled, read disk buffer */ + cluster_buf_base = XFS_INO_TO_OFFSET(mp, cluster_ag_base); + offset = (cluster_buf_base + cluster_index) * mp->m_sb.sb_inodesize; + if (offset >= BBTOB(cluster_bp->b_length)) + return -EFSCORRUPTED; + dip = xfs_buf_offset(cluster_bp, offset); + if (be16_to_cpu(dip->di_magic) != XFS_DINODE_MAGIC) + return -EFSCORRUPTED; + + if (dip->di_version >= 3 && + be64_to_cpu(dip->di_ino) != xfs_agino_to_ino(ri->sc->sa.pag, agino)) + return -EFSCORRUPTED; + + /* Will the in-core inode tell us if it's in use? */ + error = xchk_inode_is_allocated(sc, agino, inuse); + if (!error) + return 0; + + *inuse = dip->di_mode != 0; + return 0; +} + +/* Stash the accumulated inobt record for rebuilding. */ +STATIC int +xrep_ibt_stash( + struct xrep_ibt *ri) +{ + int error = 0; + + if (xchk_should_terminate(ri->sc, &error)) + return error; + + ri->rie.ir_freecount = xfs_inobt_rec_freecount(&ri->rie); + if (xfs_inobt_check_irec(ri->sc->sa.pag, &ri->rie) != NULL) + return -EFSCORRUPTED; + + if (ri->rie.ir_freecount > 0) + ri->finobt_recs++; + + trace_xrep_ibt_found(ri->sc->sa.pag, &ri->rie); + + error = xfarray_append(ri->inode_records, &ri->rie); + if (error) + return error; + + ri->rie.ir_startino = NULLAGINO; + return 0; +} + +/* + * Given an extent of inodes and an inode cluster buffer, calculate the + * location of the corresponding inobt record (creating it if necessary), + * then update the parts of the holemask and freemask of that record that + * correspond to the inode extent we were given. + * + * @cluster_ir_startino is the AG inode number of an inobt record that we're + * proposing to create for this inode cluster. If sparse inodes are enabled, + * we must round down to a chunk boundary to find the actual sparse record. + * @cluster_bp is the buffer of the inode cluster. + * @nr_inodes is the number of inodes to check from the cluster. + */ +STATIC int +xrep_ibt_cluster_record( + struct xrep_ibt *ri, + xfs_agino_t cluster_ir_startino, + struct xfs_buf *cluster_bp, + unsigned int nr_inodes) +{ + struct xfs_scrub *sc = ri->sc; + struct xfs_mount *mp = sc->mp; + xfs_agino_t ir_startino; + unsigned int cluster_base; + unsigned int cluster_index; + int error = 0; + + ir_startino = cluster_ir_startino; + if (xfs_has_sparseinodes(mp)) + ir_startino = rounddown(ir_startino, XFS_INODES_PER_CHUNK); + cluster_base = cluster_ir_startino - ir_startino; + + /* + * If the accumulated inobt record doesn't map this cluster, add it to + * the list and reset it. + */ + if (ri->rie.ir_startino != NULLAGINO && + ri->rie.ir_startino + XFS_INODES_PER_CHUNK <= ir_startino) { + error = xrep_ibt_stash(ri); + if (error) + return error; + } + + if (ri->rie.ir_startino == NULLAGINO) { + ri->rie.ir_startino = ir_startino; + ri->rie.ir_free = XFS_INOBT_ALL_FREE; + ri->rie.ir_holemask = 0xFFFF; + ri->rie.ir_count = 0; + } + + /* Record the whole cluster. */ + ri->icount += nr_inodes; + ri->rie.ir_count += nr_inodes; + ri->rie.ir_holemask &= ~xfs_inobt_maskn( + cluster_base / XFS_INODES_PER_HOLEMASK_BIT, + nr_inodes / XFS_INODES_PER_HOLEMASK_BIT); + + /* Which inodes within this cluster are free? */ + for (cluster_index = 0; cluster_index < nr_inodes; cluster_index++) { + bool inuse = false; + + error = xrep_ibt_check_ifree(ri, cluster_ir_startino, + cluster_bp, cluster_index, &inuse); + if (error) + return error; + if (!inuse) + continue; + ri->iused++; + ri->rie.ir_free &= ~XFS_INOBT_MASK(cluster_base + + cluster_index); + } + return 0; +} + +/* + * For each inode cluster covering the physical extent recorded by the rmapbt, + * we must calculate the properly aligned startino of that cluster, then + * iterate each cluster to fill in used and filled masks appropriately. We + * then use the (startino, used, filled) information to construct the + * appropriate inode records. + */ +STATIC int +xrep_ibt_process_cluster( + struct xrep_ibt *ri, + xfs_agblock_t cluster_bno) +{ + struct xfs_buf *cluster_bp; + struct xfs_scrub *sc = ri->sc; + struct xfs_mount *mp = sc->mp; + struct xfs_ino_geometry *igeo = M_IGEO(mp); + xfs_agino_t cluster_ag_base; + xfs_agino_t irec_index; + unsigned int nr_inodes; + int error; + + nr_inodes = min_t(unsigned int, igeo->inodes_per_cluster, + XFS_INODES_PER_CHUNK); + + /* + * Grab the inode cluster buffer. This is safe to do with a broken + * inobt because imap_to_bp directly maps the buffer without touching + * either inode btree. + */ + error = xfs_read_icluster(sc->sa.pag, sc->tp, cluster_bno, + &cluster_bp); + if (error) + return error; + + /* + * Record the contents of each possible inobt record mapping this + * cluster. + */ + cluster_ag_base = XFS_AGB_TO_AGINO(mp, cluster_bno); + for (irec_index = 0; + irec_index < igeo->inodes_per_cluster; + irec_index += XFS_INODES_PER_CHUNK) { + error = xrep_ibt_cluster_record(ri, + cluster_ag_base + irec_index, cluster_bp, + nr_inodes); + if (error) + break; + + } + + xfs_trans_brelse(sc->tp, cluster_bp); + return error; +} + +/* Check for any obvious conflicts in the inode chunk extent. */ +STATIC int +xrep_ibt_check_inode_ext( + struct xfs_scrub *sc, + xfs_agblock_t agbno, + xfs_extlen_t len) +{ + struct xfs_mount *mp = sc->mp; + struct xfs_ino_geometry *igeo = M_IGEO(mp); + xfs_agino_t agino; + enum xbtree_recpacking outcome; + int error; + + /* Inode records must be within the AG. */ + if (!xfs_verify_agbext(sc->sa.pag, agbno, len)) + return -EFSCORRUPTED; + + /* The entire record must align to the inode cluster size. */ + if (!IS_ALIGNED(agbno, igeo->blocks_per_cluster) || + !IS_ALIGNED(agbno + len, igeo->blocks_per_cluster)) + return -EFSCORRUPTED; + + /* + * The entire record must also adhere to the inode cluster alignment + * size if sparse inodes are not enabled. + */ + if (!xfs_has_sparseinodes(mp) && + (!IS_ALIGNED(agbno, igeo->cluster_align) || + !IS_ALIGNED(agbno + len, igeo->cluster_align))) + return -EFSCORRUPTED; + + /* + * On a sparse inode fs, this cluster could be part of a sparse chunk. + * Sparse clusters must be aligned to sparse chunk alignment. + */ + if (xfs_has_sparseinodes(mp) && mp->m_sb.sb_spino_align && + (!IS_ALIGNED(agbno, mp->m_sb.sb_spino_align) || + !IS_ALIGNED(agbno + len, mp->m_sb.sb_spino_align))) + return -EFSCORRUPTED; + + /* Make sure the entire range of blocks are valid AG inodes. */ + agino = XFS_AGB_TO_AGINO(mp, agbno); + if (!xfs_verify_agino(sc->sa.pag, agino)) + return -EFSCORRUPTED; + + agino = XFS_AGB_TO_AGINO(mp, agbno + len) - 1; + if (!xfs_verify_agino(sc->sa.pag, agino)) + return -EFSCORRUPTED; + + /* Make sure this isn't free space. */ + error = xfs_alloc_has_records(sc->sa.bno_cur, agbno, len, &outcome); + if (error) + return error; + if (outcome != XBTREE_RECPACKING_EMPTY) + return -EFSCORRUPTED; + + return 0; +} + +/* Found a fragment of the old inode btrees; dispose of them later. */ +STATIC int +xrep_ibt_record_old_btree_blocks( + struct xrep_ibt *ri, + const struct xfs_rmap_irec *rec) +{ + if (!xfs_verify_agbext(ri->sc->sa.pag, rec->rm_startblock, + rec->rm_blockcount)) + return -EFSCORRUPTED; + + return xagb_bitmap_set(&ri->old_iallocbt_blocks, rec->rm_startblock, + rec->rm_blockcount); +} + +/* Record extents that belong to inode cluster blocks. */ +STATIC int +xrep_ibt_record_inode_blocks( + struct xrep_ibt *ri, + const struct xfs_rmap_irec *rec) +{ + struct xfs_mount *mp = ri->sc->mp; + struct xfs_ino_geometry *igeo = M_IGEO(mp); + xfs_agblock_t cluster_base; + int error; + + error = xrep_ibt_check_inode_ext(ri->sc, rec->rm_startblock, + rec->rm_blockcount); + if (error) + return error; + + trace_xrep_ibt_walk_rmap(ri->sc->sa.pag, rec); + + /* + * Record the free/hole masks for each inode cluster that could be + * mapped by this rmap record. + */ + for (cluster_base = 0; + cluster_base < rec->rm_blockcount; + cluster_base += igeo->blocks_per_cluster) { + error = xrep_ibt_process_cluster(ri, + rec->rm_startblock + cluster_base); + if (error) + return error; + } + + return 0; +} + +STATIC int +xrep_ibt_walk_rmap( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv) +{ + struct xrep_ibt *ri = priv; + int error = 0; + + if (xchk_should_terminate(ri->sc, &error)) + return error; + + switch (rec->rm_owner) { + case XFS_RMAP_OWN_INOBT: + return xrep_ibt_record_old_btree_blocks(ri, rec); + case XFS_RMAP_OWN_INODES: + return xrep_ibt_record_inode_blocks(ri, rec); + } + return 0; +} + +/* + * Iterate all reverse mappings to find the inodes (OWN_INODES) and the inode + * btrees (OWN_INOBT). Figure out if we have enough free space to reconstruct + * the inode btrees. The caller must clean up the lists if anything goes + * wrong. + */ +STATIC int +xrep_ibt_find_inodes( + struct xrep_ibt *ri) +{ + struct xfs_scrub *sc = ri->sc; + int error; + + ri->rie.ir_startino = NULLAGINO; + + /* Collect all reverse mappings for inode blocks. */ + xrep_ag_btcur_init(sc, &sc->sa); + error = xfs_rmap_query_all(sc->sa.rmap_cur, xrep_ibt_walk_rmap, ri); + xchk_ag_btcur_free(&sc->sa); + if (error) + return error; + + /* If we have a record ready to go, add it to the array. */ + if (ri->rie.ir_startino != NULLAGINO) + return xrep_ibt_stash(ri); + + return 0; +} + +/* Update the AGI counters. */ +STATIC int +xrep_ibt_reset_counters( + struct xrep_ibt *ri) +{ + struct xfs_scrub *sc = ri->sc; + struct xfs_agi *agi = sc->sa.agi_bp->b_addr; + unsigned int freecount = ri->icount - ri->iused; + + /* Trigger inode count recalculation */ + xfs_force_summary_recalc(sc->mp); + + /* + * The AGI header contains extra information related to the inode + * btrees, so we must update those fields here. + */ + agi->agi_count = cpu_to_be32(ri->icount); + agi->agi_freecount = cpu_to_be32(freecount); + xfs_ialloc_log_agi(sc->tp, sc->sa.agi_bp, + XFS_AGI_COUNT | XFS_AGI_FREECOUNT); + + /* Reinitialize with the values we just logged. */ + return xrep_reinit_pagi(sc); +} + +/* Retrieve finobt data for bulk load. */ +STATIC int +xrep_fibt_get_records( + struct xfs_btree_cur *cur, + unsigned int idx, + struct xfs_btree_block *block, + unsigned int nr_wanted, + void *priv) +{ + struct xfs_inobt_rec_incore *irec = &cur->bc_rec.i; + struct xrep_ibt *ri = priv; + union xfs_btree_rec *block_rec; + unsigned int loaded; + int error; + + for (loaded = 0; loaded < nr_wanted; loaded++, idx++) { + do { + error = xfarray_load(ri->inode_records, + ri->array_cur++, irec); + } while (error == 0 && xfs_inobt_rec_freecount(irec) == 0); + if (error) + return error; + + block_rec = xfs_btree_rec_addr(cur, idx, block); + cur->bc_ops->init_rec_from_cur(cur, block_rec); + } + + return loaded; +} + +/* Retrieve inobt data for bulk load. */ +STATIC int +xrep_ibt_get_records( + struct xfs_btree_cur *cur, + unsigned int idx, + struct xfs_btree_block *block, + unsigned int nr_wanted, + void *priv) +{ + struct xfs_inobt_rec_incore *irec = &cur->bc_rec.i; + struct xrep_ibt *ri = priv; + union xfs_btree_rec *block_rec; + unsigned int loaded; + int error; + + for (loaded = 0; loaded < nr_wanted; loaded++, idx++) { + error = xfarray_load(ri->inode_records, ri->array_cur++, irec); + if (error) + return error; + + block_rec = xfs_btree_rec_addr(cur, idx, block); + cur->bc_ops->init_rec_from_cur(cur, block_rec); + } + + return loaded; +} + +/* Feed one of the new inobt blocks to the bulk loader. */ +STATIC int +xrep_ibt_claim_block( + struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr, + void *priv) +{ + struct xrep_ibt *ri = priv; + + return xrep_newbt_claim_block(cur, &ri->new_inobt, ptr); +} + +/* Feed one of the new finobt blocks to the bulk loader. */ +STATIC int +xrep_fibt_claim_block( + struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr, + void *priv) +{ + struct xrep_ibt *ri = priv; + + return xrep_newbt_claim_block(cur, &ri->new_finobt, ptr); +} + +/* Make sure the records do not overlap in inumber address space. */ +STATIC int +xrep_ibt_check_overlap( + struct xrep_ibt *ri) +{ + struct xfs_inobt_rec_incore irec; + xfarray_idx_t cur; + xfs_agino_t next_agino = 0; + int error = 0; + + foreach_xfarray_idx(ri->inode_records, cur) { + if (xchk_should_terminate(ri->sc, &error)) + return error; + + error = xfarray_load(ri->inode_records, cur, &irec); + if (error) + return error; + + if (irec.ir_startino < next_agino) + return -EFSCORRUPTED; + + next_agino = irec.ir_startino + XFS_INODES_PER_CHUNK; + } + + return error; +} + +/* Build new inode btrees and dispose of the old one. */ +STATIC int +xrep_ibt_build_new_trees( + struct xrep_ibt *ri) +{ + struct xfs_scrub *sc = ri->sc; + struct xfs_btree_cur *ino_cur; + struct xfs_btree_cur *fino_cur = NULL; + bool need_finobt; + int error; + + need_finobt = xfs_has_finobt(sc->mp); + + /* + * Create new btrees for staging all the inobt records we collected + * earlier. The records were collected in order of increasing agino, + * so we do not have to sort them. Ensure there are no overlapping + * records. + */ + error = xrep_ibt_check_overlap(ri); + if (error) + return error; + + /* + * The new inode btrees will not be rooted in the AGI until we've + * successfully rebuilt the tree. + * + * Start by setting up the inobt staging cursor. + */ + xrep_newbt_init_ag(&ri->new_inobt, sc, &XFS_RMAP_OINFO_INOBT, + xfs_agbno_to_fsb(sc->sa.pag, XFS_IBT_BLOCK(sc->mp)), + XFS_AG_RESV_NONE); + ri->new_inobt.bload.claim_block = xrep_ibt_claim_block; + ri->new_inobt.bload.get_records = xrep_ibt_get_records; + + ino_cur = xfs_inobt_init_cursor(sc->sa.pag, NULL, NULL); + xfs_btree_stage_afakeroot(ino_cur, &ri->new_inobt.afake); + error = xfs_btree_bload_compute_geometry(ino_cur, &ri->new_inobt.bload, + xfarray_length(ri->inode_records)); + if (error) + goto err_inocur; + + /* Set up finobt staging cursor. */ + if (need_finobt) { + enum xfs_ag_resv_type resv = XFS_AG_RESV_METADATA; + + if (sc->mp->m_finobt_nores) + resv = XFS_AG_RESV_NONE; + + xrep_newbt_init_ag(&ri->new_finobt, sc, &XFS_RMAP_OINFO_INOBT, + xfs_agbno_to_fsb(sc->sa.pag, XFS_FIBT_BLOCK(sc->mp)), + resv); + ri->new_finobt.bload.claim_block = xrep_fibt_claim_block; + ri->new_finobt.bload.get_records = xrep_fibt_get_records; + + fino_cur = xfs_finobt_init_cursor(sc->sa.pag, NULL, NULL); + xfs_btree_stage_afakeroot(fino_cur, &ri->new_finobt.afake); + error = xfs_btree_bload_compute_geometry(fino_cur, + &ri->new_finobt.bload, ri->finobt_recs); + if (error) + goto err_finocur; + } + + /* Last chance to abort before we start committing fixes. */ + if (xchk_should_terminate(sc, &error)) + goto err_finocur; + + /* Reserve all the space we need to build the new btrees. */ + error = xrep_newbt_alloc_blocks(&ri->new_inobt, + ri->new_inobt.bload.nr_blocks); + if (error) + goto err_finocur; + + if (need_finobt) { + error = xrep_newbt_alloc_blocks(&ri->new_finobt, + ri->new_finobt.bload.nr_blocks); + if (error) + goto err_finocur; + } + + /* Add all inobt records. */ + ri->array_cur = XFARRAY_CURSOR_INIT; + error = xfs_btree_bload(ino_cur, &ri->new_inobt.bload, ri); + if (error) + goto err_finocur; + + /* Add all finobt records. */ + if (need_finobt) { + ri->array_cur = XFARRAY_CURSOR_INIT; + error = xfs_btree_bload(fino_cur, &ri->new_finobt.bload, ri); + if (error) + goto err_finocur; + } + + /* + * Install the new btrees in the AG header. After this point the old + * btrees are no longer accessible and the new trees are live. + */ + xfs_inobt_commit_staged_btree(ino_cur, sc->tp, sc->sa.agi_bp); + xfs_btree_del_cursor(ino_cur, 0); + + if (fino_cur) { + xfs_inobt_commit_staged_btree(fino_cur, sc->tp, sc->sa.agi_bp); + xfs_btree_del_cursor(fino_cur, 0); + } + + /* Reset the AGI counters now that we've changed the inode roots. */ + error = xrep_ibt_reset_counters(ri); + if (error) + goto err_finobt; + + /* Free unused blocks and bitmap. */ + if (need_finobt) { + error = xrep_newbt_commit(&ri->new_finobt); + if (error) + goto err_inobt; + } + error = xrep_newbt_commit(&ri->new_inobt); + if (error) + return error; + + return xrep_roll_ag_trans(sc); + +err_finocur: + if (need_finobt) + xfs_btree_del_cursor(fino_cur, error); +err_inocur: + xfs_btree_del_cursor(ino_cur, error); +err_finobt: + if (need_finobt) + xrep_newbt_cancel(&ri->new_finobt); +err_inobt: + xrep_newbt_cancel(&ri->new_inobt); + return error; +} + +/* + * Now that we've logged the roots of the new btrees, invalidate all of the + * old blocks and free them. + */ +STATIC int +xrep_ibt_remove_old_trees( + struct xrep_ibt *ri) +{ + struct xfs_scrub *sc = ri->sc; + int error; + + /* + * Free the old inode btree blocks if they're not in use. It's ok to + * reap with XFS_AG_RESV_NONE even if the finobt had a per-AG + * reservation because we reset the reservation before releasing the + * AGI and AGF header buffer locks. + */ + error = xrep_reap_agblocks(sc, &ri->old_iallocbt_blocks, + &XFS_RMAP_OINFO_INOBT, XFS_AG_RESV_NONE); + if (error) + return error; + + /* + * If the finobt is enabled and has a per-AG reservation, make sure we + * reinitialize the per-AG reservations. + */ + if (xfs_has_finobt(sc->mp) && !sc->mp->m_finobt_nores) + sc->flags |= XREP_RESET_PERAG_RESV; + + return 0; +} + +/* Repair both inode btrees. */ +int +xrep_iallocbt( + struct xfs_scrub *sc) +{ + struct xrep_ibt *ri; + struct xfs_mount *mp = sc->mp; + xfs_agino_t first_agino, last_agino; + int error = 0; + + /* We require the rmapbt to rebuild anything. */ + if (!xfs_has_rmapbt(mp)) + return -EOPNOTSUPP; + + ri = kzalloc_obj(struct xrep_ibt, XCHK_GFP_FLAGS); + if (!ri) + return -ENOMEM; + ri->sc = sc; + + /* We rebuild both inode btrees. */ + sc->sick_mask = XFS_SICK_AG_INOBT | XFS_SICK_AG_FINOBT; + + /* Set up enough storage to handle an AG with nothing but inodes. */ + xfs_agino_range(mp, pag_agno(sc->sa.pag), &first_agino, &last_agino); + last_agino /= XFS_INODES_PER_CHUNK; + error = xfarray_create("inode index records", last_agino, + sizeof(struct xfs_inobt_rec_incore), + &ri->inode_records); + if (error) + goto out_ri; + + /* Collect the inode data and find the old btree blocks. */ + xagb_bitmap_init(&ri->old_iallocbt_blocks); + error = xrep_ibt_find_inodes(ri); + if (error) + goto out_bitmap; + + /* Rebuild the inode indexes. */ + error = xrep_ibt_build_new_trees(ri); + if (error) + goto out_bitmap; + + /* Kill the old tree. */ + error = xrep_ibt_remove_old_trees(ri); + if (error) + goto out_bitmap; + +out_bitmap: + xagb_bitmap_destroy(&ri->old_iallocbt_blocks); + xfarray_destroy(ri->inode_records); +out_ri: + kfree(ri); + return error; +} + +/* Make sure both btrees are ok after we've rebuilt them. */ +int +xrep_revalidate_iallocbt( + struct xfs_scrub *sc) +{ + __u32 old_type = sc->sm->sm_type; + int error; + + /* + * We must update sm_type temporarily so that the tree-to-tree cross + * reference checks will work in the correct direction, and also so + * that tracing will report correctly if there are more errors. + */ + sc->sm->sm_type = XFS_SCRUB_TYPE_INOBT; + error = xchk_iallocbt(sc); + if (error) + goto out; + + /* + * If the inobt is still corrupt, we've failed to repair the filesystem + * and should just bail out. + * + * If the inobt fails cross-examination with the finobt, the scan will + * free the finobt cursor, so we need to mark the repair incomplete + * and avoid walking off the end of the NULL finobt cursor. + */ + if (!xfs_has_finobt(sc->mp) || + (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) + goto out; + + sc->sm->sm_type = XFS_SCRUB_TYPE_FINOBT; + if (!sc->sa.fino_cur) { + xchk_set_incomplete(sc); + goto out; + } + error = xchk_iallocbt(sc); + +out: + sc->sm->sm_type = old_type; + return error; +} diff --git a/fs/xfs/scrub/ino_bitmap.h b/fs/xfs/scrub/ino_bitmap.h new file mode 100644 index 000000000..130083367 --- /dev/null +++ b/fs/xfs/scrub/ino_bitmap.h @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2023-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_INO_BITMAP_H__ +#define __XFS_SCRUB_INO_BITMAP_H__ + +/* Bitmaps, but for type-checked for xfs_ino_t */ + +struct xino_bitmap { + struct xbitmap64 inobitmap; +}; + +static inline void xino_bitmap_init(struct xino_bitmap *bitmap) +{ + xbitmap64_init(&bitmap->inobitmap); +} + +static inline void xino_bitmap_destroy(struct xino_bitmap *bitmap) +{ + xbitmap64_destroy(&bitmap->inobitmap); +} + +static inline int xino_bitmap_set(struct xino_bitmap *bitmap, xfs_ino_t ino) +{ + return xbitmap64_set(&bitmap->inobitmap, ino, 1); +} + +static inline int xino_bitmap_test(struct xino_bitmap *bitmap, xfs_ino_t ino) +{ + uint64_t len = 1; + + return xbitmap64_test(&bitmap->inobitmap, ino, &len); +} + +#endif /* __XFS_SCRUB_INO_BITMAP_H__ */ diff --git a/fs/xfs/scrub/inode.c b/fs/xfs/scrub/inode.c new file mode 100644 index 000000000..65b13e311 --- /dev/null +++ b/fs/xfs/scrub/inode.c @@ -0,0 +1,841 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_btree.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_ag.h" +#include "xfs_inode.h" +#include "xfs_ialloc.h" +#include "xfs_icache.h" +#include "xfs_da_format.h" +#include "xfs_reflink.h" +#include "xfs_rmap.h" +#include "xfs_bmap_util.h" +#include "xfs_rtbitmap.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/btree.h" +#include "scrub/trace.h" +#include "scrub/repair.h" + +/* Prepare the attached inode for scrubbing. */ +static inline int +xchk_prepare_iscrub( + struct xfs_scrub *sc) +{ + int error; + + xchk_ilock(sc, XFS_IOLOCK_EXCL); + + error = xchk_trans_alloc(sc, 0); + if (error) + return error; + + error = xchk_ino_dqattach(sc); + if (error) + return error; + + xchk_ilock(sc, XFS_ILOCK_EXCL); + return 0; +} + +/* Install this scrub-by-handle inode and prepare it for scrubbing. */ +static inline int +xchk_install_handle_iscrub( + struct xfs_scrub *sc, + struct xfs_inode *ip) +{ + int error; + + error = xchk_install_handle_inode(sc, ip); + if (error) + return error; + + /* + * Don't allow scrubbing by handle of any non-directory inode records + * in the metadata directory tree. We don't know if any of the scans + * launched by this scrubber will end up indirectly trying to lock this + * file. + * + * Scrubbers of inode-rooted metadata files (e.g. quota files) will + * attach all the resources needed to scrub the inode and call + * xchk_inode directly. Userspace cannot call this directly. + */ + if (xfs_is_metadir_inode(ip) && !S_ISDIR(VFS_I(ip)->i_mode)) { + xchk_irele(sc, ip); + sc->ip = NULL; + return -ENOENT; + } + + return xchk_prepare_iscrub(sc); +} + +/* + * Grab total control of the inode metadata. In the best case, we grab the + * incore inode and take all locks on it. If the incore inode cannot be + * constructed due to corruption problems, lock the AGI so that we can single + * step the loading process to fix everything that can go wrong. + */ +int +xchk_setup_inode( + struct xfs_scrub *sc) +{ + struct xfs_imap imap; + struct xfs_inode *ip; + struct xfs_mount *mp = sc->mp; + struct xfs_inode *ip_in = XFS_I(file_inode(sc->file)); + struct xfs_buf *agi_bp; + struct xfs_perag *pag; + xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, sc->sm->sm_ino); + int error; + + if (xchk_need_intent_drain(sc)) + xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN); + + /* We want to scan the opened inode, so lock it and exit. */ + if (sc->sm->sm_ino == 0 || sc->sm->sm_ino == I_INO(ip_in)) { + error = xchk_install_live_inode(sc, ip_in); + if (error) + return error; + + return xchk_prepare_iscrub(sc); + } + + /* + * On pre-metadir filesystems, reject internal metadata files. For + * metadir filesystems, limited scrubbing of any file in the metadata + * directory tree by handle is allowed, because that is the only way to + * validate the lack of parent pointers in the sb-root metadata inodes. + */ + if (!xfs_has_metadir(mp) && xfs_is_sb_inum(mp, sc->sm->sm_ino)) + return -ENOENT; + /* Reject obviously bad inode numbers. */ + if (!xfs_verify_ino(sc->mp, sc->sm->sm_ino)) + return -ENOENT; + + /* Try a safe untrusted iget. */ + error = xchk_iget_safe(sc, sc->sm->sm_ino, &ip); + if (!error) + return xchk_install_handle_iscrub(sc, ip); + if (error == -ENOENT) + return error; + if (error != -EFSCORRUPTED && error != -EFSBADCRC && error != -EINVAL) + goto out_error; + + /* + * EINVAL with IGET_UNTRUSTED probably means one of several things: + * userspace gave us an inode number that doesn't correspond to fs + * space; the inode btree lacks a record for this inode; or there is + * a record, and it says this inode is free. + * + * EFSCORRUPTED/EFSBADCRC could mean that the inode was mappable, but + * some other metadata corruption (e.g. inode forks) prevented + * instantiation of the incore inode. Or it could mean the inobt is + * corrupt. + * + * We want to look up this inode in the inobt directly to distinguish + * three different scenarios: (1) the inobt says the inode is free, + * in which case there's nothing to do; (2) the inobt is corrupt so we + * should flag the corruption and exit to userspace to let it fix the + * inobt; and (3) the inobt says the inode is allocated, but loading it + * failed due to corruption. + * + * Allocate a transaction and grab the AGI to prevent inobt activity in + * this AG. Retry the iget in case someone allocated a new inode after + * the first iget failed. + */ + error = xchk_trans_alloc(sc, 0); + if (error) + goto out_error; + + error = xchk_iget_agi(sc, sc->sm->sm_ino, &agi_bp, &ip); + if (error == 0) { + /* Actually got the incore inode, so install it and proceed. */ + xchk_trans_cancel(sc); + return xchk_install_handle_iscrub(sc, ip); + } + if (error == -ENOENT) + goto out_gone; + if (error != -EFSCORRUPTED && error != -EFSBADCRC && error != -EINVAL) + goto out_cancel; + + /* Ensure that we have protected against inode allocation/freeing. */ + if (agi_bp == NULL) { + ASSERT(agi_bp != NULL); + error = -ECANCELED; + goto out_cancel; + } + + /* + * Untrusted iget failed a second time. Let's try an inobt lookup. + * If the inobt doesn't think this is an allocated inode then we'll + * return ENOENT to signal that the check can be skipped. + * + * If the lookup signals corruption, we'll mark this inode corrupt and + * exit to userspace. There's little chance of fixing anything until + * the inobt is straightened out, but there's nothing we can do here. + * + * If the lookup encounters a runtime error, exit to userspace. + */ + pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, sc->sm->sm_ino)); + if (!pag) { + error = -EFSCORRUPTED; + goto out_cancel; + } + + error = xfs_imap(pag, sc->tp, sc->sm->sm_ino, &imap, + XFS_IGET_UNTRUSTED); + xfs_perag_put(pag); + if (error == -EINVAL || error == -ENOENT) + goto out_gone; + if (error) + goto out_cancel; + + /* + * The lookup succeeded. Chances are the ondisk inode is corrupt and + * preventing iget from reading it. Retain the scrub transaction and + * the AGI buffer to prevent anyone from allocating or freeing inodes. + * This ensures that we preserve the inconsistency between the inobt + * saying the inode is allocated and the icache being unable to load + * the inode until we can flag the corruption in xchk_inode. The + * scrub function has to note the corruption, since we're not really + * supposed to do that from the setup function. Save the mapping to + * make repairs to the ondisk inode buffer. + */ + if (xchk_could_repair(sc)) + xrep_setup_inode(sc, &imap); + return 0; + +out_cancel: + xchk_trans_cancel(sc); +out_error: + trace_xchk_op_error(sc, agno, XFS_INO_TO_AGBNO(mp, sc->sm->sm_ino), + error, __return_address); + return error; +out_gone: + /* The file is gone, so there's nothing to check. */ + xchk_trans_cancel(sc); + return -ENOENT; +} + +/* Inode core */ + +/* Validate di_extsize hint. */ +STATIC void +xchk_inode_extsize( + struct xfs_scrub *sc, + struct xfs_dinode *dip, + xfs_ino_t ino, + uint16_t mode, + uint16_t flags) +{ + xfs_failaddr_t fa; + uint32_t value = be32_to_cpu(dip->di_extsize); + + fa = xfs_inode_validate_extsize(sc->mp, value, mode, flags); + if (fa) + xchk_ino_set_corrupt(sc, ino); + + /* + * XFS allows a sysadmin to change the rt extent size when adding a rt + * section to a filesystem after formatting. If there are any + * directories with extszinherit and rtinherit set, the hint could + * become misaligned with the new rextsize. The verifier doesn't check + * this, because we allow rtinherit directories even without an rt + * device. Flag this as an administrative warning since we will clean + * this up eventually. + */ + if ((flags & XFS_DIFLAG_RTINHERIT) && + (flags & XFS_DIFLAG_EXTSZINHERIT) && + xfs_extlen_to_rtxmod(sc->mp, value) > 0) + xchk_ino_set_warning(sc, ino); +} + +/* Validate di_cowextsize hint. */ +STATIC void +xchk_inode_cowextsize( + struct xfs_scrub *sc, + struct xfs_dinode *dip, + xfs_ino_t ino, + uint16_t mode, + uint16_t flags, + uint64_t flags2) +{ + xfs_failaddr_t fa; + uint32_t value = be32_to_cpu(dip->di_cowextsize); + + /* + * The used block counter for rtrmap is checked and repaired elsewhere. + */ + if (xfs_has_zoned(sc->mp) && + dip->di_metatype == cpu_to_be16(XFS_METAFILE_RTRMAP)) + return; + + fa = xfs_inode_validate_cowextsize(sc->mp, value, mode, flags, flags2); + if (fa) + xchk_ino_set_corrupt(sc, ino); + + /* + * XFS allows a sysadmin to change the rt extent size when adding a rt + * section to a filesystem after formatting. If there are any + * directories with cowextsize and rtinherit set, the hint could become + * misaligned with the new rextsize. The verifier doesn't check this, + * because we allow rtinherit directories even without an rt device. + * Flag this as an administrative warning since we will clean this up + * eventually. + */ + if ((flags & XFS_DIFLAG_RTINHERIT) && + (flags2 & XFS_DIFLAG2_COWEXTSIZE) && + value % sc->mp->m_sb.sb_rextsize > 0) + xchk_ino_set_warning(sc, ino); +} + +/* Make sure the di_flags make sense for the inode. */ +STATIC void +xchk_inode_flags( + struct xfs_scrub *sc, + struct xfs_dinode *dip, + xfs_ino_t ino, + uint16_t mode, + uint16_t flags) +{ + struct xfs_mount *mp = sc->mp; + + /* di_flags are all taken, last bit cannot be used */ + if (flags & ~XFS_DIFLAG_ANY) + goto bad; + + /* rt flags require rt device */ + if ((flags & XFS_DIFLAG_REALTIME) && !mp->m_rtdev_targp) + goto bad; + + /* new rt bitmap flag only valid for rbmino */ + if ((flags & XFS_DIFLAG_NEWRTBM) && ino != mp->m_sb.sb_rbmino) + goto bad; + + /* directory-only flags */ + if ((flags & (XFS_DIFLAG_RTINHERIT | + XFS_DIFLAG_EXTSZINHERIT | + XFS_DIFLAG_PROJINHERIT | + XFS_DIFLAG_NOSYMLINKS)) && + !S_ISDIR(mode)) + goto bad; + + /* file-only flags */ + if ((flags & (XFS_DIFLAG_REALTIME | FS_XFLAG_EXTSIZE)) && + !S_ISREG(mode)) + goto bad; + + /* filestreams and rt make no sense */ + if ((flags & XFS_DIFLAG_FILESTREAM) && (flags & XFS_DIFLAG_REALTIME)) + goto bad; + + return; +bad: + xchk_ino_set_corrupt(sc, ino); +} + +/* Make sure the di_flags2 make sense for the inode. */ +STATIC void +xchk_inode_flags2( + struct xfs_scrub *sc, + struct xfs_dinode *dip, + xfs_ino_t ino, + uint16_t mode, + uint16_t flags, + uint64_t flags2) +{ + struct xfs_mount *mp = sc->mp; + + /* Unknown di_flags2 could be from a future kernel */ + if (flags2 & ~XFS_DIFLAG2_ANY) + xchk_ino_set_warning(sc, ino); + + /* reflink flag requires reflink feature */ + if ((flags2 & XFS_DIFLAG2_REFLINK) && + !xfs_has_reflink(mp)) + goto bad; + + /* cowextsize flag is checked w.r.t. mode separately */ + + /* file/dir-only flags */ + if ((flags2 & XFS_DIFLAG2_DAX) && !(S_ISREG(mode) || S_ISDIR(mode))) + goto bad; + + /* file-only flags */ + if ((flags2 & XFS_DIFLAG2_REFLINK) && !S_ISREG(mode)) + goto bad; + + /* realtime and reflink don't always go together */ + if ((flags & XFS_DIFLAG_REALTIME) && (flags2 & XFS_DIFLAG2_REFLINK) && + !xfs_has_rtreflink(mp)) + goto bad; + + /* no bigtime iflag without the bigtime feature */ + if (xfs_dinode_has_bigtime(dip) && !xfs_has_bigtime(mp)) + goto bad; + + /* no large extent counts without the filesystem feature */ + if ((flags2 & XFS_DIFLAG2_NREXT64) && !xfs_has_large_extent_counts(mp)) + goto bad; + + return; +bad: + xchk_ino_set_corrupt(sc, ino); +} + +static inline void +xchk_dinode_nsec( + struct xfs_scrub *sc, + xfs_ino_t ino, + struct xfs_dinode *dip, + const xfs_timestamp_t ts) +{ + struct timespec64 tv; + + tv = xfs_inode_from_disk_ts(dip, ts); + if (tv.tv_nsec < 0 || tv.tv_nsec >= NSEC_PER_SEC) + xchk_ino_set_corrupt(sc, ino); +} + +/* Scrub all the ondisk inode fields. */ +STATIC void +xchk_dinode( + struct xfs_scrub *sc, + struct xfs_dinode *dip, + xfs_ino_t ino) +{ + struct xfs_mount *mp = sc->mp; + size_t fork_recs; + unsigned long long isize; + uint64_t flags2; + xfs_extnum_t nextents; + xfs_extnum_t naextents; + prid_t prid; + uint16_t flags; + uint16_t mode; + + flags = be16_to_cpu(dip->di_flags); + if (dip->di_version >= 3) + flags2 = be64_to_cpu(dip->di_flags2); + else + flags2 = 0; + + /* di_mode */ + mode = be16_to_cpu(dip->di_mode); + switch (mode & S_IFMT) { + case S_IFLNK: + case S_IFREG: + case S_IFDIR: + case S_IFCHR: + case S_IFBLK: + case S_IFIFO: + case S_IFSOCK: + /* mode is recognized */ + break; + default: + xchk_ino_set_corrupt(sc, ino); + break; + } + + /* v1/v2 fields */ + switch (dip->di_version) { + case 1: + /* + * We autoconvert v1 inodes into v2 inodes on writeout, + * so just mark this inode for preening. + */ + xchk_ino_set_preen(sc, ino); + prid = 0; + break; + case 2: + case 3: + if (xfs_dinode_is_metadir(dip)) { + if (be16_to_cpu(dip->di_metatype) >= XFS_METAFILE_MAX) + xchk_ino_set_corrupt(sc, ino); + } else { + if (dip->di_metatype != 0) + xchk_ino_set_corrupt(sc, ino); + } + + if (dip->di_mode == 0 && sc->ip) + xchk_ino_set_corrupt(sc, ino); + + if (dip->di_projid_hi != 0 && + !xfs_has_projid32(mp)) + xchk_ino_set_corrupt(sc, ino); + + prid = be16_to_cpu(dip->di_projid_lo); + break; + default: + xchk_ino_set_corrupt(sc, ino); + return; + } + + if (xfs_has_projid32(mp)) + prid |= (prid_t)be16_to_cpu(dip->di_projid_hi) << 16; + + /* + * di_uid/di_gid -- -1 isn't invalid, but there's no way that + * userspace could have created that. + */ + if (dip->di_uid == cpu_to_be32(-1U) || + dip->di_gid == cpu_to_be32(-1U)) + xchk_ino_set_warning(sc, ino); + + /* + * project id of -1 isn't supposed to be valid, but the kernel didn't + * always validate that. + */ + if (prid == -1U) + xchk_ino_set_warning(sc, ino); + + /* di_format */ + switch (dip->di_format) { + case XFS_DINODE_FMT_DEV: + if (!S_ISCHR(mode) && !S_ISBLK(mode) && + !S_ISFIFO(mode) && !S_ISSOCK(mode)) + xchk_ino_set_corrupt(sc, ino); + break; + case XFS_DINODE_FMT_LOCAL: + if (!S_ISDIR(mode) && !S_ISLNK(mode)) + xchk_ino_set_corrupt(sc, ino); + break; + case XFS_DINODE_FMT_EXTENTS: + if (!S_ISREG(mode) && !S_ISDIR(mode) && !S_ISLNK(mode)) + xchk_ino_set_corrupt(sc, ino); + break; + case XFS_DINODE_FMT_BTREE: + if (!S_ISREG(mode) && !S_ISDIR(mode)) + xchk_ino_set_corrupt(sc, ino); + break; + case XFS_DINODE_FMT_META_BTREE: + if (!S_ISREG(mode)) + xchk_ino_set_corrupt(sc, ino); + break; + case XFS_DINODE_FMT_UUID: + default: + xchk_ino_set_corrupt(sc, ino); + break; + } + + /* di_[amc]time.nsec */ + xchk_dinode_nsec(sc, ino, dip, dip->di_atime); + xchk_dinode_nsec(sc, ino, dip, dip->di_mtime); + xchk_dinode_nsec(sc, ino, dip, dip->di_ctime); + + /* + * di_size. xfs_dinode_verify checks for things that screw up + * the VFS such as the upper bit being set and zero-length + * symlinks/directories, but we can do more here. + */ + isize = be64_to_cpu(dip->di_size); + if (isize & (1ULL << 63)) + xchk_ino_set_corrupt(sc, ino); + + /* Devices, fifos, and sockets must have zero size */ + if (!S_ISDIR(mode) && !S_ISREG(mode) && !S_ISLNK(mode) && isize != 0) + xchk_ino_set_corrupt(sc, ino); + + /* Directories can't be larger than the data section size (32G) */ + if (S_ISDIR(mode) && (isize == 0 || isize >= XFS_DIR2_SPACE_SIZE)) + xchk_ino_set_corrupt(sc, ino); + + /* Symlinks can't be larger than SYMLINK_MAXLEN */ + if (S_ISLNK(mode) && (isize == 0 || isize >= XFS_SYMLINK_MAXLEN)) + xchk_ino_set_corrupt(sc, ino); + + /* + * Warn if the running kernel can't handle the kinds of offsets + * needed to deal with the file size. In other words, if the + * pagecache can't cache all the blocks in this file due to + * overly large offsets, flag the inode for admin review. + */ + if (isize > mp->m_super->s_maxbytes) + xchk_ino_set_warning(sc, ino); + + /* di_nblocks */ + if (flags2 & XFS_DIFLAG2_REFLINK) { + ; /* nblocks can exceed dblocks */ + } else if (flags & XFS_DIFLAG_REALTIME) { + /* + * nblocks is the sum of data extents (in the rtdev), + * attr extents (in the datadev), and both forks' bmbt + * blocks (in the datadev). This clumsy check is the + * best we can do without cross-referencing with the + * inode forks. + */ + if (be64_to_cpu(dip->di_nblocks) >= + mp->m_sb.sb_dblocks + mp->m_sb.sb_rblocks) + xchk_ino_set_corrupt(sc, ino); + } else { + if (be64_to_cpu(dip->di_nblocks) >= mp->m_sb.sb_dblocks) + xchk_ino_set_corrupt(sc, ino); + } + + xchk_inode_flags(sc, dip, ino, mode, flags); + + xchk_inode_extsize(sc, dip, ino, mode, flags); + + nextents = xfs_dfork_data_extents(dip); + naextents = xfs_dfork_attr_extents(dip); + + /* di_nextents */ + fork_recs = XFS_DFORK_DSIZE(dip, mp) / sizeof(struct xfs_bmbt_rec); + switch (dip->di_format) { + case XFS_DINODE_FMT_EXTENTS: + if (nextents > fork_recs) + xchk_ino_set_corrupt(sc, ino); + break; + case XFS_DINODE_FMT_BTREE: + if (nextents <= fork_recs) + xchk_ino_set_corrupt(sc, ino); + break; + default: + if (nextents != 0) + xchk_ino_set_corrupt(sc, ino); + break; + } + + /* di_forkoff */ + if (XFS_DFORK_BOFF(dip) >= mp->m_sb.sb_inodesize) + xchk_ino_set_corrupt(sc, ino); + if (naextents != 0 && dip->di_forkoff == 0) + xchk_ino_set_corrupt(sc, ino); + if (dip->di_forkoff == 0 && dip->di_aformat != XFS_DINODE_FMT_EXTENTS) + xchk_ino_set_corrupt(sc, ino); + + /* di_aformat */ + if (dip->di_aformat != XFS_DINODE_FMT_LOCAL && + dip->di_aformat != XFS_DINODE_FMT_EXTENTS && + dip->di_aformat != XFS_DINODE_FMT_BTREE) + xchk_ino_set_corrupt(sc, ino); + + /* di_anextents */ + fork_recs = XFS_DFORK_ASIZE(dip, mp) / sizeof(struct xfs_bmbt_rec); + switch (dip->di_aformat) { + case XFS_DINODE_FMT_EXTENTS: + if (naextents > fork_recs) + xchk_ino_set_corrupt(sc, ino); + break; + case XFS_DINODE_FMT_BTREE: + if (naextents <= fork_recs) + xchk_ino_set_corrupt(sc, ino); + break; + default: + if (naextents != 0) + xchk_ino_set_corrupt(sc, ino); + } + + if (dip->di_version >= 3) { + xchk_dinode_nsec(sc, ino, dip, dip->di_crtime); + xchk_inode_flags2(sc, dip, ino, mode, flags, flags2); + xchk_inode_cowextsize(sc, dip, ino, mode, flags, + flags2); + } +} + +/* + * Make sure the finobt doesn't think this inode is free. + * We don't have to check the inobt ourselves because we got the inode via + * IGET_UNTRUSTED, which checks the inobt for us. + */ +static void +xchk_inode_xref_finobt( + struct xfs_scrub *sc, + xfs_ino_t ino) +{ + struct xfs_inobt_rec_incore rec; + xfs_agino_t agino; + int has_record; + int error; + + if (!sc->sa.fino_cur || xchk_skip_xref(sc->sm)) + return; + + agino = XFS_INO_TO_AGINO(sc->mp, ino); + + /* + * Try to get the finobt record. If we can't get it, then we're + * in good shape. + */ + error = xfs_inobt_lookup(sc->sa.fino_cur, agino, XFS_LOOKUP_LE, + &has_record); + if (!xchk_should_check_xref(sc, &error, &sc->sa.fino_cur) || + !has_record) + return; + + error = xfs_inobt_get_rec(sc->sa.fino_cur, &rec, &has_record); + if (!xchk_should_check_xref(sc, &error, &sc->sa.fino_cur) || + !has_record) + return; + + /* + * Otherwise, make sure this record either doesn't cover this inode, + * or that it does but it's marked present. + */ + if (rec.ir_startino > agino || + rec.ir_startino + XFS_INODES_PER_CHUNK <= agino) + return; + + if (rec.ir_free & XFS_INOBT_MASK(agino - rec.ir_startino)) + xchk_btree_xref_set_corrupt(sc, sc->sa.fino_cur, 0); +} + +/* Cross reference the inode fields with the forks. */ +STATIC void +xchk_inode_xref_bmap( + struct xfs_scrub *sc, + struct xfs_dinode *dip) +{ + xfs_extnum_t nextents; + xfs_filblks_t count; + xfs_filblks_t acount; + int error; + + if (xchk_skip_xref(sc->sm)) + return; + + /* Walk all the extents to check nextents/naextents/nblocks. */ + error = xchk_inode_count_blocks(sc, XFS_DATA_FORK, &nextents, &count); + if (!xchk_should_check_xref(sc, &error, NULL)) + return; + if (nextents < xfs_dfork_data_extents(dip)) + xchk_ip_xref_set_corrupt(sc, sc->ip); + + error = xchk_inode_count_blocks(sc, XFS_ATTR_FORK, &nextents, &acount); + if (!xchk_should_check_xref(sc, &error, NULL)) + return; + if (nextents != xfs_dfork_attr_extents(dip)) + xchk_ip_xref_set_corrupt(sc, sc->ip); + + /* Check nblocks against the inode. */ + if (count + acount != be64_to_cpu(dip->di_nblocks)) + xchk_ip_xref_set_corrupt(sc, sc->ip); +} + +/* Cross-reference with the other btrees. */ +STATIC void +xchk_inode_xref( + struct xfs_scrub *sc, + xfs_ino_t ino, + struct xfs_dinode *dip) +{ + xfs_agnumber_t agno; + xfs_agblock_t agbno; + int error; + + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return; + + agno = XFS_INO_TO_AGNO(sc->mp, ino); + agbno = XFS_INO_TO_AGBNO(sc->mp, ino); + + error = xchk_ag_init_existing(sc, agno, &sc->sa); + if (!xchk_xref_process_error(sc, agno, agbno, &error)) + goto out_free; + + xchk_xref_is_used_space(sc, agbno, 1); + xchk_inode_xref_finobt(sc, ino); + xchk_xref_is_only_owned_by(sc, agbno, 1, &XFS_RMAP_OINFO_INODES); + xchk_xref_is_not_shared(sc, agbno, 1); + xchk_xref_is_not_cow_staging(sc, agbno, 1); + xchk_inode_xref_bmap(sc, dip); + +out_free: + xchk_ag_free(sc, &sc->sa); +} + +/* + * If the reflink iflag disagrees with a scan for shared data fork extents, + * either flag an error (shared extents w/ no flag) or a preen (flag set w/o + * any shared extents). We already checked for reflink iflag set on a non + * reflink filesystem. + */ +static void +xchk_inode_check_reflink_iflag( + struct xfs_scrub *sc, + xfs_ino_t ino) +{ + struct xfs_mount *mp = sc->mp; + bool has_shared; + int error; + + if (!xfs_has_reflink(mp)) + return; + + error = xfs_reflink_inode_has_shared_extents(sc->tp, sc->ip, + &has_shared); + if (!xchk_xref_process_error(sc, XFS_INO_TO_AGNO(mp, ino), + XFS_INO_TO_AGBNO(mp, ino), &error)) + return; + if (xfs_is_reflink_inode(sc->ip) && !has_shared) + xchk_ino_set_preen(sc, ino); + else if (!xfs_is_reflink_inode(sc->ip) && has_shared) + xchk_ino_set_corrupt(sc, ino); +} + +/* + * If this inode has zero link count, it must be on the unlinked list. If + * it has nonzero link count, it must not be on the unlinked list. + */ +STATIC void +xchk_inode_check_unlinked( + struct xfs_scrub *sc) +{ + if (VFS_I(sc->ip)->i_nlink == 0) { + if (!xfs_inode_on_unlinked_list(sc->ip)) + xchk_ino_set_corrupt(sc, I_INO(sc->ip)); + } else { + if (xfs_inode_on_unlinked_list(sc->ip)) + xchk_ino_set_corrupt(sc, I_INO(sc->ip)); + } +} + +/* Scrub an inode. */ +int +xchk_inode( + struct xfs_scrub *sc) +{ + struct xfs_dinode di; + int error = 0; + + /* + * If sc->ip is NULL, that means that the setup function called + * xfs_iget to look up the inode. xfs_iget returned a EFSCORRUPTED + * and a NULL inode, so flag the corruption error and return. + */ + if (!sc->ip) { + xchk_ino_set_corrupt(sc, sc->sm->sm_ino); + return 0; + } + + /* Scrub the inode core. */ + xfs_inode_to_disk(sc->ip, &di, 0); + xchk_dinode(sc, &di, I_INO(sc->ip)); + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out; + + /* + * Look for discrepancies between file's data blocks and the reflink + * iflag. We already checked the iflag against the file mode when + * we scrubbed the dinode. + */ + if (S_ISREG(VFS_I(sc->ip)->i_mode)) + xchk_inode_check_reflink_iflag(sc, I_INO(sc->ip)); + + xchk_inode_check_unlinked(sc); + + xchk_inode_xref(sc, I_INO(sc->ip), &di); +out: + return error; +} diff --git a/fs/xfs/scrub/inode_repair.c b/fs/xfs/scrub/inode_repair.c new file mode 100644 index 000000000..8bc508336 --- /dev/null +++ b/fs/xfs/scrub/inode_repair.c @@ -0,0 +1,2093 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_btree.h" +#include "xfs_bit.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_sb.h" +#include "xfs_inode.h" +#include "xfs_icache.h" +#include "xfs_inode_buf.h" +#include "xfs_inode_fork.h" +#include "xfs_ialloc.h" +#include "xfs_da_format.h" +#include "xfs_reflink.h" +#include "xfs_alloc.h" +#include "xfs_rmap.h" +#include "xfs_rmap_btree.h" +#include "xfs_bmap.h" +#include "xfs_bmap_btree.h" +#include "xfs_bmap_util.h" +#include "xfs_dir2.h" +#include "xfs_dir2_priv.h" +#include "xfs_quota_defs.h" +#include "xfs_quota.h" +#include "xfs_ag.h" +#include "xfs_rtbitmap.h" +#include "xfs_attr_leaf.h" +#include "xfs_log_priv.h" +#include "xfs_health.h" +#include "xfs_symlink_remote.h" +#include "xfs_rtgroup.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_rtrefcount_btree.h" +#include "scrub/xfs_scrub.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/btree.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/iscan.h" +#include "scrub/readdir.h" +#include "scrub/tempfile.h" + +/* + * Inode Record Repair + * =================== + * + * Roughly speaking, inode problems can be classified based on whether or not + * they trip the dinode verifiers. If those trip, then we won't be able to + * xfs_iget ourselves the inode. + * + * Therefore, the xrep_dinode_* functions fix anything that will cause the + * inode buffer verifier or the dinode verifier. The xrep_inode_* functions + * fix things on live incore inodes. The inode repair functions make decisions + * with security and usability implications when reviving a file: + * + * - Files with zero di_mode or a garbage di_mode are converted to regular file + * that only root can read. This file may not actually contain user data, + * if the file was not previously a regular file. Setuid and setgid bits + * are cleared. + * + * - Zero-size directories can be truncated to look empty. It is necessary to + * run the bmapbtd and directory repair functions to fully rebuild the + * directory. + * + * - Zero-size symbolic link targets can be truncated to '?'. It is necessary + * to run the bmapbtd and symlink repair functions to salvage the symlink. + * + * - Invalid extent size hints will be removed. + * + * - Quotacheck will be scheduled if we repaired an inode that was so badly + * damaged that the ondisk inode had to be rebuilt. + * + * - Invalid user, group, or project IDs (aka -1U) will be reset to zero. + * Setuid and setgid bits are cleared. + * + * - Data and attr forks are reset to extents format with zero extents if the + * fork data is inconsistent. It is necessary to run the bmapbtd or bmapbta + * repair functions to recover the space mapping. + * + * - ACLs will not be recovered if the attr fork is zapped or the extended + * attribute structure itself requires salvaging. + * + * - If the attr fork is zapped, the user and group ids are reset to root and + * the setuid and setgid bits are removed. + */ + +/* + * All the information we need to repair the ondisk inode if we can't iget the + * incore inode. We don't allocate this buffer unless we're going to perform + * a repair to the ondisk inode cluster buffer. + */ +struct xrep_inode { + /* Inode mapping that we saved from the initial lookup attempt. */ + struct xfs_imap imap; + + struct xfs_scrub *sc; + + /* Blocks in use on the data device by data extents or bmbt blocks. */ + xfs_rfsblock_t data_blocks; + + /* Blocks in use on the rt device. */ + xfs_rfsblock_t rt_blocks; + + /* Blocks in use by the attr fork. */ + xfs_rfsblock_t attr_blocks; + + /* Number of data device extents for the data fork. */ + xfs_extnum_t data_extents; + + /* + * Number of realtime device extents for the data fork. If + * data_extents and rt_extents indicate that the data fork has extents + * on both devices, we'll just back away slowly. + */ + xfs_extnum_t rt_extents; + + /* Number of (data device) extents for the attr fork. */ + xfs_aextnum_t attr_extents; + + /* Sick state to set after zapping parts of the inode. */ + unsigned int ino_sick_mask; + + /* Must we remove all access from this file? */ + bool zap_acls; + + /* Inode scanner to see if we can find the ftype from dirents */ + struct xchk_iscan ftype_iscan; + uint8_t alleged_ftype; +}; + +/* + * Setup function for inode repair. @imap contains the ondisk inode mapping + * information so that we can correct the ondisk inode cluster buffer if + * necessary to make iget work. + */ +int +xrep_setup_inode( + struct xfs_scrub *sc, + const struct xfs_imap *imap) +{ + struct xrep_inode *ri; + + sc->buf = kzalloc_obj(struct xrep_inode, XCHK_GFP_FLAGS); + if (!sc->buf) + return -ENOMEM; + + ri = sc->buf; + memcpy(&ri->imap, imap, sizeof(struct xfs_imap)); + ri->sc = sc; + return 0; +} + +/* + * Make sure this ondisk inode can pass the inode buffer verifier. This is + * not the same as the dinode verifier. + */ +STATIC void +xrep_dinode_buf_core( + struct xfs_scrub *sc, + struct xfs_buf *bp, + unsigned int ioffset) +{ + struct xfs_dinode *dip = xfs_buf_offset(bp, ioffset); + struct xfs_trans *tp = sc->tp; + struct xfs_mount *mp = sc->mp; + xfs_agino_t agino; + bool crc_ok = false; + bool magic_ok = false; + bool unlinked_ok = false; + + agino = be32_to_cpu(dip->di_next_unlinked); + + if (xfs_verify_agino_or_null(bp->b_pag, agino)) + unlinked_ok = true; + + if (dip->di_magic == cpu_to_be16(XFS_DINODE_MAGIC) && + xfs_dinode_good_version(mp, dip->di_version)) + magic_ok = true; + + if (xfs_verify_cksum((char *)dip, mp->m_sb.sb_inodesize, + XFS_DINODE_CRC_OFF)) + crc_ok = true; + + if (magic_ok && unlinked_ok && crc_ok) + return; + + if (!magic_ok) { + dip->di_magic = cpu_to_be16(XFS_DINODE_MAGIC); + dip->di_version = 3; + } + if (!unlinked_ok) + dip->di_next_unlinked = cpu_to_be32(NULLAGINO); + xfs_dinode_calc_crc(mp, dip); + xfs_trans_buf_set_type(tp, bp, XFS_BLFT_DINO_BUF); + xfs_trans_log_buf(tp, bp, ioffset, + ioffset + sizeof(struct xfs_dinode) - 1); +} + +/* Make sure this inode cluster buffer can pass the inode buffer verifier. */ +STATIC void +xrep_dinode_buf( + struct xfs_scrub *sc, + struct xfs_buf *bp) +{ + struct xfs_mount *mp = sc->mp; + int i; + int ni; + + ni = XFS_BB_TO_FSB(mp, bp->b_length) * mp->m_sb.sb_inopblock; + for (i = 0; i < ni; i++) + xrep_dinode_buf_core(sc, bp, i << mp->m_sb.sb_inodelog); +} + +/* Reinitialize things that never change in an inode. */ +STATIC void +xrep_dinode_header( + struct xfs_scrub *sc, + struct xfs_dinode *dip) +{ + trace_xrep_dinode_header(sc, dip); + + dip->di_magic = cpu_to_be16(XFS_DINODE_MAGIC); + if (!xfs_dinode_good_version(sc->mp, dip->di_version)) + dip->di_version = 3; + dip->di_ino = cpu_to_be64(sc->sm->sm_ino); + uuid_copy(&dip->di_uuid, &sc->mp->m_sb.sb_meta_uuid); + dip->di_gen = cpu_to_be32(sc->sm->sm_gen); +} + +/* + * If this directory entry points to the scrub target inode, then the directory + * we're scanning is the parent of the scrub target inode. + */ +STATIC int +xrep_dinode_findmode_dirent( + struct xfs_scrub *sc, + struct xfs_inode *dp, + xfs_dir2_dataptr_t dapos, + const struct xfs_name *name, + xfs_ino_t ino, + void *priv) +{ + struct xrep_inode *ri = priv; + int error = 0; + + if (xchk_should_terminate(ri->sc, &error)) + return error; + + if (ino != sc->sm->sm_ino) + return 0; + + /* Ignore garbage directory entry names. */ + if (name->len == 0 || !xfs_dir2_namecheck(name->name, name->len)) + return -EFSCORRUPTED; + + /* Don't pick up dot or dotdot entries; we only want child dirents. */ + if (xfs_dir2_samename(name, &xfs_name_dotdot) || + xfs_dir2_samename(name, &xfs_name_dot)) + return 0; + + /* + * Uhoh, more than one parent for this inode and they don't agree on + * the file type? + */ + if (ri->alleged_ftype != XFS_DIR3_FT_UNKNOWN && + ri->alleged_ftype != name->type) { + trace_xrep_dinode_findmode_dirent_inval(ri->sc, dp, name->type, + ri->alleged_ftype); + return -EFSCORRUPTED; + } + + /* We found a potential parent; remember the ftype. */ + trace_xrep_dinode_findmode_dirent(ri->sc, dp, name->type); + ri->alleged_ftype = name->type; + return 0; +} + +/* Try to lock a directory, or wait a jiffy. */ +static inline int +xrep_dinode_ilock_nowait( + struct xfs_inode *dp, + unsigned int lock_mode) +{ + if (xfs_ilock_nowait(dp, lock_mode)) + return true; + + schedule_timeout_killable(1); + return false; +} + +/* + * Try to lock a directory to look for ftype hints. Since we already hold the + * AGI buffer, we cannot block waiting for the ILOCK because rename can take + * the ILOCK and then try to lock AGIs. + */ +STATIC int +xrep_dinode_trylock_directory( + struct xrep_inode *ri, + struct xfs_inode *dp, + unsigned int *lock_modep) +{ + unsigned long deadline = jiffies + msecs_to_jiffies(30000); + unsigned int lock_mode; + int error = 0; + + do { + if (xchk_should_terminate(ri->sc, &error)) + return error; + + if (xfs_need_iread_extents(&dp->i_df)) + lock_mode = XFS_ILOCK_EXCL; + else + lock_mode = XFS_ILOCK_SHARED; + + if (xrep_dinode_ilock_nowait(dp, lock_mode)) { + *lock_modep = lock_mode; + return 0; + } + } while (!time_is_before_jiffies(deadline)); + return -EBUSY; +} + +/* + * If this is a directory, walk the dirents looking for any that point to the + * scrub target inode. + */ +STATIC int +xrep_dinode_findmode_walk_directory( + struct xrep_inode *ri, + struct xfs_inode *dp) +{ + struct xfs_scrub *sc = ri->sc; + unsigned int lock_mode; + int error = 0; + + /* Ignore temporary repair directories. */ + if (xrep_is_tempfile(dp)) + return 0; + + /* + * Scan the directory to see if there it contains an entry pointing to + * the directory that we are repairing. + */ + error = xrep_dinode_trylock_directory(ri, dp, &lock_mode); + if (error) + return error; + + /* + * If this directory is known to be sick, we cannot scan it reliably + * and must abort. + */ + if (xfs_inode_has_sickness(dp, XFS_SICK_INO_CORE | + XFS_SICK_INO_BMBTD | + XFS_SICK_INO_DIR)) { + error = -EFSCORRUPTED; + goto out_unlock; + } + + /* + * We cannot complete our parent pointer scan if a directory looks as + * though it has been zapped by the inode record repair code. + */ + if (xchk_dir_looks_zapped(dp)) { + error = -EBUSY; + goto out_unlock; + } + + error = xchk_dir_walk(sc, dp, xrep_dinode_findmode_dirent, ri); + if (error) + goto out_unlock; + +out_unlock: + xfs_iunlock(dp, lock_mode); + return error; +} + +/* + * Try to find the mode of the inode being repaired by looking for directories + * that point down to this file. + */ +STATIC int +xrep_dinode_find_mode( + struct xrep_inode *ri, + uint16_t *mode) +{ + struct xfs_scrub *sc = ri->sc; + struct xfs_inode *dp; + int error; + + /* No ftype means we have no other metadata to consult. */ + if (!xfs_has_ftype(sc->mp)) { + *mode = S_IFREG; + return 0; + } + + /* + * Scan all directories for parents that might point down to this + * inode. Skip the inode being repaired during the scan since it + * cannot be its own parent. Note that we still hold the AGI locked + * so there's a real possibility that _iscan_iter can return EBUSY. + */ + xchk_iscan_start(sc, 5000, 100, &ri->ftype_iscan); + xchk_iscan_set_agi_trylock(&ri->ftype_iscan); + ri->ftype_iscan.skip_ino = sc->sm->sm_ino; + ri->alleged_ftype = XFS_DIR3_FT_UNKNOWN; + while ((error = xchk_iscan_iter(&ri->ftype_iscan, &dp)) == 1) { + if (S_ISDIR(VFS_I(dp)->i_mode)) + error = xrep_dinode_findmode_walk_directory(ri, dp); + xchk_iscan_mark_visited(&ri->ftype_iscan, dp); + xchk_irele(sc, dp); + if (error < 0) + break; + if (xchk_should_terminate(sc, &error)) + break; + } + xchk_iscan_iter_finish(&ri->ftype_iscan); + xchk_iscan_teardown(&ri->ftype_iscan); + + if (error == -EBUSY) { + if (ri->alleged_ftype != XFS_DIR3_FT_UNKNOWN) { + /* + * If we got an EBUSY after finding at least one + * dirent, that means the scan found an inode on the + * inactivation list and could not open it. Accept the + * alleged ftype and install a new mode below. + */ + error = 0; + } else if (!(sc->flags & XCHK_TRY_HARDER)) { + /* + * Otherwise, retry the operation one time to see if + * the reason for the delay is an inode from the same + * cluster buffer waiting on the inactivation list. + */ + error = -EDEADLOCK; + } + } + if (error) + return error; + + /* + * Convert the discovered ftype into the file mode. If all else fails, + * return S_IFREG. + */ + switch (ri->alleged_ftype) { + case XFS_DIR3_FT_DIR: + *mode = S_IFDIR; + break; + case XFS_DIR3_FT_WHT: + case XFS_DIR3_FT_CHRDEV: + *mode = S_IFCHR; + break; + case XFS_DIR3_FT_BLKDEV: + *mode = S_IFBLK; + break; + case XFS_DIR3_FT_FIFO: + *mode = S_IFIFO; + break; + case XFS_DIR3_FT_SOCK: + *mode = S_IFSOCK; + break; + case XFS_DIR3_FT_SYMLINK: + *mode = S_IFLNK; + break; + default: + *mode = S_IFREG; + break; + } + return 0; +} + +/* Turn di_mode into /something/ recognizable. Returns true if we succeed. */ +STATIC int +xrep_dinode_mode( + struct xrep_inode *ri, + struct xfs_dinode *dip) +{ + struct xfs_scrub *sc = ri->sc; + uint16_t mode = be16_to_cpu(dip->di_mode); + int error; + + trace_xrep_dinode_mode(sc, dip); + + if (mode == 0 || xfs_mode_to_ftype(mode) != XFS_DIR3_FT_UNKNOWN) + return 0; + + /* Try to fix the mode. If we cannot, then leave everything alone. */ + error = xrep_dinode_find_mode(ri, &mode); + switch (error) { + case -EINTR: + case -EBUSY: + case -EDEADLOCK: + /* temporary failure or fatal signal */ + return error; + case 0: + /* found mode */ + break; + default: + /* some other error, assume S_IFREG */ + mode = S_IFREG; + break; + } + + /* bad mode, so we set it to a file that only root can read */ + dip->di_mode = cpu_to_be16(mode); + dip->di_uid = 0; + dip->di_gid = 0; + ri->zap_acls = true; + return 0; +} + +/* Fix unused link count fields having nonzero values. */ +STATIC void +xrep_dinode_nlinks( + struct xfs_dinode *dip) +{ + if (dip->di_version < 2) { + dip->di_nlink = 0; + return; + } + + if (xfs_dinode_is_metadir(dip)) { + if (be16_to_cpu(dip->di_metatype) >= XFS_METAFILE_MAX) + dip->di_metatype = cpu_to_be16(XFS_METAFILE_UNKNOWN); + } else { + dip->di_metatype = 0; + } +} + +/* Fix any conflicting flags that the verifiers complain about. */ +STATIC void +xrep_dinode_flags( + struct xfs_scrub *sc, + struct xfs_dinode *dip, + bool isrt) +{ + struct xfs_mount *mp = sc->mp; + uint64_t flags2 = be64_to_cpu(dip->di_flags2); + uint16_t flags = be16_to_cpu(dip->di_flags); + uint16_t mode = be16_to_cpu(dip->di_mode); + + trace_xrep_dinode_flags(sc, dip); + + if (isrt) + flags |= XFS_DIFLAG_REALTIME; + else + flags &= ~XFS_DIFLAG_REALTIME; + + /* + * For regular files on a reflink filesystem, set the REFLINK flag to + * protect shared extents. A later stage will actually check those + * extents and clear the flag if possible. + */ + if (xfs_has_reflink(mp) && S_ISREG(mode)) + flags2 |= XFS_DIFLAG2_REFLINK; + else + flags2 &= ~(XFS_DIFLAG2_REFLINK | XFS_DIFLAG2_COWEXTSIZE); + if (!xfs_has_bigtime(mp)) + flags2 &= ~XFS_DIFLAG2_BIGTIME; + if (!xfs_has_large_extent_counts(mp)) + flags2 &= ~XFS_DIFLAG2_NREXT64; + if (flags2 & XFS_DIFLAG2_NREXT64) + dip->di_nrext64_pad = 0; + else if (dip->di_version >= 3) + dip->di_v3_pad = 0; + + if (flags2 & XFS_DIFLAG2_METADATA) { + xfs_failaddr_t fa; + + fa = xfs_dinode_verify_metadir(sc->mp, dip, mode, flags, + flags2); + if (fa) + flags2 &= ~XFS_DIFLAG2_METADATA; + } + + dip->di_flags = cpu_to_be16(flags); + dip->di_flags2 = cpu_to_be64(flags2); +} + +/* + * Blow out symlink; now it points nowhere. We don't have to worry about + * incore state because this inode is failing the verifiers. + */ +STATIC void +xrep_dinode_zap_symlink( + struct xrep_inode *ri, + struct xfs_dinode *dip) +{ + struct xfs_scrub *sc = ri->sc; + char *p; + + trace_xrep_dinode_zap_symlink(sc, dip); + + dip->di_format = XFS_DINODE_FMT_LOCAL; + dip->di_size = cpu_to_be64(1); + p = XFS_DFORK_PTR(dip, XFS_DATA_FORK); + *p = '?'; + ri->ino_sick_mask |= XFS_SICK_INO_SYMLINK_ZAPPED; +} + +/* + * Blow out dir, make the parent point to the root. In the future repair will + * reconstruct this directory for us. Note that there's no in-core directory + * inode because the sf verifier tripped, so we don't have to worry about the + * dentry cache. + */ +STATIC void +xrep_dinode_zap_dir( + struct xrep_inode *ri, + struct xfs_dinode *dip) +{ + struct xfs_scrub *sc = ri->sc; + struct xfs_mount *mp = sc->mp; + struct xfs_dir2_sf_hdr *sfp; + int i8count; + + trace_xrep_dinode_zap_dir(sc, dip); + + dip->di_format = XFS_DINODE_FMT_LOCAL; + i8count = mp->m_sb.sb_rootino > XFS_DIR2_MAX_SHORT_INUM; + sfp = XFS_DFORK_PTR(dip, XFS_DATA_FORK); + sfp->count = 0; + sfp->i8count = i8count; + xfs_dir2_sf_put_parent_ino(sfp, mp->m_sb.sb_rootino); + dip->di_size = cpu_to_be64(xfs_dir2_sf_hdr_size(i8count)); + ri->ino_sick_mask |= XFS_SICK_INO_DIR_ZAPPED; +} + +/* Make sure we don't have a garbage file size. */ +STATIC void +xrep_dinode_size( + struct xrep_inode *ri, + struct xfs_dinode *dip) +{ + struct xfs_scrub *sc = ri->sc; + uint64_t size = be64_to_cpu(dip->di_size); + uint16_t mode = be16_to_cpu(dip->di_mode); + + trace_xrep_dinode_size(sc, dip); + + switch (mode & S_IFMT) { + case S_IFIFO: + case S_IFCHR: + case S_IFBLK: + case S_IFSOCK: + /* di_size can't be nonzero for special files */ + dip->di_size = 0; + break; + case S_IFREG: + /* Regular files can't be larger than 2^63-1 bytes. */ + dip->di_size = cpu_to_be64(size & ~(1ULL << 63)); + break; + case S_IFLNK: + /* + * Truncate ridiculously oversized symlinks. If the size is + * zero, reset it to point to the current directory. Both of + * these conditions trigger dinode verifier errors, so there + * is no in-core state to reset. + */ + if (size > XFS_SYMLINK_MAXLEN) + dip->di_size = cpu_to_be64(XFS_SYMLINK_MAXLEN); + else if (size == 0) + xrep_dinode_zap_symlink(ri, dip); + break; + case S_IFDIR: + /* + * Directories can't have a size larger than 32G. If the size + * is zero, reset it to an empty directory. Both of these + * conditions trigger dinode verifier errors, so there is no + * in-core state to reset. + */ + if (size > XFS_DIR2_SPACE_SIZE) + dip->di_size = cpu_to_be64(XFS_DIR2_SPACE_SIZE); + else if (size == 0) + xrep_dinode_zap_dir(ri, dip); + break; + } +} + +/* Fix extent size hints. */ +STATIC void +xrep_dinode_extsize_hints( + struct xfs_scrub *sc, + struct xfs_dinode *dip) +{ + struct xfs_mount *mp = sc->mp; + uint64_t flags2 = be64_to_cpu(dip->di_flags2); + uint16_t flags = be16_to_cpu(dip->di_flags); + uint16_t mode = be16_to_cpu(dip->di_mode); + + xfs_failaddr_t fa; + + trace_xrep_dinode_extsize_hints(sc, dip); + + fa = xfs_inode_validate_extsize(mp, be32_to_cpu(dip->di_extsize), + mode, flags); + if (fa) { + dip->di_extsize = 0; + dip->di_flags &= ~cpu_to_be16(XFS_DIFLAG_EXTSIZE | + XFS_DIFLAG_EXTSZINHERIT); + } + + if (dip->di_version < 3 || + (xfs_has_zoned(sc->mp) && + dip->di_metatype == cpu_to_be16(XFS_METAFILE_RTRMAP))) + return; + + fa = xfs_inode_validate_cowextsize(mp, be32_to_cpu(dip->di_cowextsize), + mode, flags, flags2); + if (fa) { + dip->di_cowextsize = 0; + dip->di_flags2 &= ~cpu_to_be64(XFS_DIFLAG2_COWEXTSIZE); + } +} + +/* Count extents and blocks for an inode given an rmap. */ +STATIC int +xrep_dinode_walk_rmap( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv) +{ + struct xrep_inode *ri = priv; + int error = 0; + + if (xchk_should_terminate(ri->sc, &error)) + return error; + + /* We only care about this inode. */ + if (rec->rm_owner != ri->sc->sm->sm_ino) + return 0; + + if (rec->rm_flags & XFS_RMAP_ATTR_FORK) { + ri->attr_blocks += rec->rm_blockcount; + if (!(rec->rm_flags & XFS_RMAP_BMBT_BLOCK)) + ri->attr_extents++; + + return 0; + } + + ri->data_blocks += rec->rm_blockcount; + if (!(rec->rm_flags & XFS_RMAP_BMBT_BLOCK)) + ri->data_extents++; + + return 0; +} + +/* Count extents and blocks for an inode from all AG rmap data. */ +STATIC int +xrep_dinode_count_ag_rmaps( + struct xrep_inode *ri, + struct xfs_perag *pag) +{ + struct xfs_btree_cur *cur; + struct xfs_buf *agf; + int error; + + error = xfs_alloc_read_agf(pag, ri->sc->tp, 0, &agf); + if (error) + return error; + + cur = xfs_rmapbt_init_cursor(ri->sc->mp, ri->sc->tp, agf, pag); + error = xfs_rmap_query_all(cur, xrep_dinode_walk_rmap, ri); + xfs_btree_del_cursor(cur, error); + xfs_trans_brelse(ri->sc->tp, agf); + return error; +} + +/* Count extents and blocks for an inode given an rt rmap. */ +STATIC int +xrep_dinode_walk_rtrmap( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv) +{ + struct xrep_inode *ri = priv; + int error = 0; + + if (xchk_should_terminate(ri->sc, &error)) + return error; + + /* We only care about this inode. */ + if (rec->rm_owner != ri->sc->sm->sm_ino) + return 0; + + if (rec->rm_flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK)) + return -EFSCORRUPTED; + + ri->rt_blocks += rec->rm_blockcount; + ri->rt_extents++; + return 0; +} + +/* Count extents and blocks for an inode from all realtime rmap data. */ +STATIC int +xrep_dinode_count_rtgroup_rmaps( + struct xrep_inode *ri, + struct xfs_rtgroup *rtg) +{ + struct xfs_scrub *sc = ri->sc; + int error; + + error = xrep_rtgroup_init(sc, rtg, &sc->sr, XFS_RTGLOCK_RMAP); + if (error) + return error; + + error = xfs_rmap_query_all(sc->sr.rmap_cur, xrep_dinode_walk_rtrmap, + ri); + xchk_rtgroup_btcur_free(&sc->sr); + xchk_rtgroup_free(sc, &sc->sr); + return error; +} + +/* Count extents and blocks for a given inode from all rmap data. */ +STATIC int +xrep_dinode_count_rmaps( + struct xrep_inode *ri) +{ + struct xfs_perag *pag = NULL; + struct xfs_rtgroup *rtg = NULL; + int error; + + if (!xfs_has_rmapbt(ri->sc->mp)) + return -EOPNOTSUPP; + + while ((rtg = xfs_rtgroup_next(ri->sc->mp, rtg))) { + error = xrep_dinode_count_rtgroup_rmaps(ri, rtg); + if (error) { + xfs_rtgroup_rele(rtg); + return error; + } + } + + while ((pag = xfs_perag_next(ri->sc->mp, pag))) { + error = xrep_dinode_count_ag_rmaps(ri, pag); + if (error) { + xfs_perag_rele(pag); + return error; + } + } + + /* Can't have extents on both the rt and the data device. */ + if (ri->data_extents && ri->rt_extents) + return -EFSCORRUPTED; + + trace_xrep_dinode_count_rmaps(ri->sc, + ri->data_blocks, ri->rt_blocks, ri->attr_blocks, + ri->data_extents, ri->rt_extents, ri->attr_extents); + return 0; +} + +/* Return true if this extents-format ifork looks like garbage. */ +STATIC bool +xrep_dinode_bad_extents_fork( + struct xfs_scrub *sc, + struct xfs_dinode *dip, + unsigned int dfork_size, + int whichfork) +{ + struct xfs_bmbt_irec new; + struct xfs_bmbt_rec *dp; + xfs_extnum_t nex; + bool isrt; + unsigned int i; + + nex = xfs_dfork_nextents(dip, whichfork); + if (nex > dfork_size / sizeof(struct xfs_bmbt_rec)) + return true; + + dp = XFS_DFORK_PTR(dip, whichfork); + + isrt = dip->di_flags & cpu_to_be16(XFS_DIFLAG_REALTIME); + for (i = 0; i < nex; i++, dp++) { + xfs_failaddr_t fa; + + xfs_bmbt_disk_get_all(dp, &new); + fa = xfs_bmap_validate_extent_raw(sc->mp, isrt, whichfork, + &new); + if (fa) + return true; + } + + return false; +} + +/* Return true if this btree-format ifork looks like garbage. */ +STATIC bool +xrep_dinode_bad_bmbt_fork( + struct xfs_scrub *sc, + struct xfs_dinode *dip, + unsigned int dfork_size, + int whichfork) +{ + struct xfs_bmdr_block *dfp; + xfs_extnum_t nex; + unsigned int i; + unsigned int dmxr; + unsigned int nrecs; + unsigned int level; + + nex = xfs_dfork_nextents(dip, whichfork); + if (nex <= dfork_size / sizeof(struct xfs_bmbt_rec)) + return true; + + if (dfork_size < sizeof(struct xfs_bmdr_block)) + return true; + + dfp = XFS_DFORK_PTR(dip, whichfork); + nrecs = be16_to_cpu(dfp->bb_numrecs); + level = be16_to_cpu(dfp->bb_level); + + if (nrecs == 0 || xfs_bmdr_space_calc(nrecs) > dfork_size) + return true; + if (level == 0 || level > XFS_BM_MAXLEVELS(sc->mp, whichfork)) + return true; + + dmxr = xfs_bmdr_maxrecs(dfork_size, 0); + for (i = 1; i <= nrecs; i++) { + struct xfs_bmbt_key *fkp; + xfs_bmbt_ptr_t *fpp; + xfs_fileoff_t fileoff; + xfs_fsblock_t fsbno; + + fkp = xfs_bmdr_key_addr(dfp, i); + fileoff = be64_to_cpu(fkp->br_startoff); + if (!xfs_verify_fileoff(sc->mp, fileoff)) + return true; + + fpp = xfs_bmdr_ptr_addr(dfp, i, dmxr); + fsbno = be64_to_cpu(*fpp); + if (!xfs_verify_fsbno(sc->mp, fsbno)) + return true; + } + + return false; +} + +/* Return true if this rmap-format ifork looks like garbage. */ +STATIC bool +xrep_dinode_bad_rtrmapbt_fork( + struct xfs_scrub *sc, + struct xfs_dinode *dip, + unsigned int dfork_size) +{ + struct xfs_rtrmap_root *dfp; + unsigned int nrecs; + unsigned int level; + + if (dfork_size < sizeof(struct xfs_rtrmap_root)) + return true; + + dfp = XFS_DFORK_PTR(dip, XFS_DATA_FORK); + nrecs = be16_to_cpu(dfp->bb_numrecs); + level = be16_to_cpu(dfp->bb_level); + + if (level > sc->mp->m_rtrmap_maxlevels) + return true; + if (xfs_rtrmap_droot_space_calc(level, nrecs) > dfork_size) + return true; + if (level > 0 && nrecs == 0) + return true; + + return false; +} + +/* Return true if this refcount-format ifork looks like garbage. */ +STATIC bool +xrep_dinode_bad_rtrefcountbt_fork( + struct xfs_scrub *sc, + struct xfs_dinode *dip, + unsigned int dfork_size) +{ + struct xfs_rtrefcount_root *dfp; + unsigned int nrecs; + unsigned int level; + + if (dfork_size < sizeof(struct xfs_rtrefcount_root)) + return true; + + dfp = XFS_DFORK_PTR(dip, XFS_DATA_FORK); + nrecs = be16_to_cpu(dfp->bb_numrecs); + level = be16_to_cpu(dfp->bb_level); + + if (level > sc->mp->m_rtrefc_maxlevels) + return true; + if (xfs_rtrefcount_droot_space_calc(level, nrecs) > dfork_size) + return true; + if (level > 0 && nrecs == 0) + return true; + + return false; +} + +/* Check a metadata-btree fork. */ +STATIC bool +xrep_dinode_bad_metabt_fork( + struct xfs_scrub *sc, + struct xfs_dinode *dip, + unsigned int dfork_size, + int whichfork) +{ + if (whichfork != XFS_DATA_FORK) + return true; + + switch (be16_to_cpu(dip->di_metatype)) { + case XFS_METAFILE_RTRMAP: + return xrep_dinode_bad_rtrmapbt_fork(sc, dip, dfork_size); + case XFS_METAFILE_RTREFCOUNT: + return xrep_dinode_bad_rtrefcountbt_fork(sc, dip, dfork_size); + default: + return true; + } + + return false; +} + +/* + * Check the data fork for things that will fail the ifork verifiers or the + * ifork formatters. + */ +STATIC bool +xrep_dinode_check_dfork( + struct xfs_scrub *sc, + struct xfs_dinode *dip, + uint16_t mode) +{ + void *dfork_ptr; + int64_t data_size; + unsigned int fmt; + unsigned int dfork_size; + + /* + * Verifier functions take signed int64_t, so check for bogus negative + * values first. + */ + data_size = be64_to_cpu(dip->di_size); + if (data_size < 0) + return true; + + fmt = XFS_DFORK_FORMAT(dip, XFS_DATA_FORK); + switch (mode & S_IFMT) { + case S_IFIFO: + case S_IFCHR: + case S_IFBLK: + case S_IFSOCK: + if (fmt != XFS_DINODE_FMT_DEV) + return true; + break; + case S_IFREG: + switch (fmt) { + case XFS_DINODE_FMT_LOCAL: + return true; + case XFS_DINODE_FMT_EXTENTS: + case XFS_DINODE_FMT_BTREE: + case XFS_DINODE_FMT_META_BTREE: + break; + default: + return true; + } + break; + case S_IFLNK: + case S_IFDIR: + switch (fmt) { + case XFS_DINODE_FMT_LOCAL: + case XFS_DINODE_FMT_EXTENTS: + case XFS_DINODE_FMT_BTREE: + break; + default: + return true; + } + break; + default: + return true; + } + + dfork_size = XFS_DFORK_SIZE(dip, sc->mp, XFS_DATA_FORK); + dfork_ptr = XFS_DFORK_PTR(dip, XFS_DATA_FORK); + + switch (fmt) { + case XFS_DINODE_FMT_DEV: + break; + case XFS_DINODE_FMT_LOCAL: + /* dir/symlink structure cannot be larger than the fork */ + if (data_size > dfork_size) + return true; + /* directory structure must pass verification. */ + if (S_ISDIR(mode) && + xfs_dir2_sf_verify(sc->mp, dfork_ptr, data_size) != NULL) + return true; + /* symlink structure must pass verification. */ + if (S_ISLNK(mode) && + xfs_symlink_shortform_verify(dfork_ptr, data_size) != NULL) + return true; + break; + case XFS_DINODE_FMT_EXTENTS: + if (xrep_dinode_bad_extents_fork(sc, dip, dfork_size, + XFS_DATA_FORK)) + return true; + break; + case XFS_DINODE_FMT_BTREE: + if (xrep_dinode_bad_bmbt_fork(sc, dip, dfork_size, + XFS_DATA_FORK)) + return true; + break; + case XFS_DINODE_FMT_META_BTREE: + if (xrep_dinode_bad_metabt_fork(sc, dip, dfork_size, + XFS_DATA_FORK)) + return true; + break; + default: + return true; + } + + return false; +} + +static void +xrep_dinode_set_data_nextents( + struct xfs_dinode *dip, + xfs_extnum_t nextents) +{ + if (xfs_dinode_has_large_extent_counts(dip)) + dip->di_big_nextents = cpu_to_be64(nextents); + else + dip->di_nextents = cpu_to_be32(nextents); +} + +static void +xrep_dinode_set_attr_nextents( + struct xfs_dinode *dip, + xfs_extnum_t nextents) +{ + if (xfs_dinode_has_large_extent_counts(dip)) + dip->di_big_anextents = cpu_to_be32(nextents); + else + dip->di_anextents = cpu_to_be16(nextents); +} + +/* Reset the data fork to something sane. */ +STATIC void +xrep_dinode_zap_dfork( + struct xrep_inode *ri, + struct xfs_dinode *dip, + uint16_t mode) +{ + struct xfs_scrub *sc = ri->sc; + + trace_xrep_dinode_zap_dfork(sc, dip); + + ri->ino_sick_mask |= XFS_SICK_INO_BMBTD_ZAPPED; + + xrep_dinode_set_data_nextents(dip, 0); + ri->data_blocks = 0; + ri->rt_blocks = 0; + + /* Special files always get reset to DEV */ + switch (mode & S_IFMT) { + case S_IFIFO: + case S_IFCHR: + case S_IFBLK: + case S_IFSOCK: + dip->di_format = XFS_DINODE_FMT_DEV; + dip->di_size = 0; + return; + } + + /* + * If we have data extents, reset to an empty map and hope the user + * will run the bmapbtd checker next. + */ + if (ri->data_extents || ri->rt_extents || S_ISREG(mode)) { + dip->di_format = XFS_DINODE_FMT_EXTENTS; + return; + } + + /* Otherwise, reset the local format to the minimum. */ + switch (mode & S_IFMT) { + case S_IFLNK: + xrep_dinode_zap_symlink(ri, dip); + break; + case S_IFDIR: + xrep_dinode_zap_dir(ri, dip); + break; + } +} + +/* + * Check the attr fork for things that will fail the ifork verifiers or the + * ifork formatters. + */ +STATIC bool +xrep_dinode_check_afork( + struct xfs_scrub *sc, + struct xfs_dinode *dip) +{ + struct xfs_attr_sf_hdr *afork_ptr; + size_t attr_size; + unsigned int afork_size; + + if (XFS_DFORK_BOFF(dip) == 0) + return dip->di_aformat != XFS_DINODE_FMT_EXTENTS || + xfs_dfork_attr_extents(dip) != 0; + + afork_size = XFS_DFORK_SIZE(dip, sc->mp, XFS_ATTR_FORK); + afork_ptr = XFS_DFORK_PTR(dip, XFS_ATTR_FORK); + + switch (XFS_DFORK_FORMAT(dip, XFS_ATTR_FORK)) { + case XFS_DINODE_FMT_LOCAL: + /* Fork has to be large enough to extract the xattr size. */ + if (afork_size < sizeof(struct xfs_attr_sf_hdr)) + return true; + + /* xattr structure cannot be larger than the fork */ + attr_size = be16_to_cpu(afork_ptr->totsize); + if (attr_size > afork_size) + return true; + + /* xattr structure must pass verification. */ + return xfs_attr_shortform_verify(afork_ptr, attr_size) != NULL; + case XFS_DINODE_FMT_EXTENTS: + if (xrep_dinode_bad_extents_fork(sc, dip, afork_size, + XFS_ATTR_FORK)) + return true; + break; + case XFS_DINODE_FMT_BTREE: + if (xrep_dinode_bad_bmbt_fork(sc, dip, afork_size, + XFS_ATTR_FORK)) + return true; + break; + case XFS_DINODE_FMT_META_BTREE: + if (xrep_dinode_bad_metabt_fork(sc, dip, afork_size, + XFS_ATTR_FORK)) + return true; + break; + default: + return true; + } + + return false; +} + +/* + * Reset the attr fork to empty. Since the attr fork could have contained + * ACLs, make the file readable only by root. + */ +STATIC void +xrep_dinode_zap_afork( + struct xrep_inode *ri, + struct xfs_dinode *dip, + uint16_t mode) +{ + struct xfs_scrub *sc = ri->sc; + + trace_xrep_dinode_zap_afork(sc, dip); + + ri->ino_sick_mask |= XFS_SICK_INO_BMBTA_ZAPPED; + + dip->di_aformat = XFS_DINODE_FMT_EXTENTS; + xrep_dinode_set_attr_nextents(dip, 0); + ri->attr_blocks = 0; + + /* + * If the data fork is in btree format, removing the attr fork entirely + * might cause verifier failures if the next level down in the bmbt + * could now fit in the data fork area. + */ + if (dip->di_format != XFS_DINODE_FMT_BTREE) + dip->di_forkoff = 0; + dip->di_mode = cpu_to_be16(mode & ~0777); + dip->di_uid = 0; + dip->di_gid = 0; +} + +/* Make sure the fork offset is a sensible value. */ +STATIC void +xrep_dinode_ensure_forkoff( + struct xrep_inode *ri, + struct xfs_dinode *dip, + uint16_t mode) +{ + struct xfs_bmdr_block *bmdr; + struct xfs_rtrmap_root *rmdr; + struct xfs_rtrefcount_root *rcdr; + struct xfs_scrub *sc = ri->sc; + xfs_extnum_t attr_extents, data_extents; + size_t bmdr_minsz = xfs_bmdr_space_calc(1); + unsigned int lit_sz = XFS_LITINO(sc->mp); + unsigned int afork_min, dfork_min; + + trace_xrep_dinode_ensure_forkoff(sc, dip); + + /* + * Before calling this function, xrep_dinode_core ensured that both + * forks actually fit inside their respective literal areas. If this + * was not the case, the fork was reset to FMT_EXTENTS with zero + * records. If the rmapbt scan found attr or data fork blocks, this + * will be noted in the dinode_stats, and we must leave enough room + * for the bmap repair code to reconstruct the mapping structure. + * + * First, compute the minimum space required for the attr fork. + */ + switch (dip->di_aformat) { + case XFS_DINODE_FMT_LOCAL: + /* + * If we still have a shortform xattr structure at all, that + * means the attr fork area was exactly large enough to fit + * the sf structure. + */ + afork_min = XFS_DFORK_SIZE(dip, sc->mp, XFS_ATTR_FORK); + break; + case XFS_DINODE_FMT_EXTENTS: + attr_extents = xfs_dfork_attr_extents(dip); + if (attr_extents) { + /* + * We must maintain sufficient space to hold the entire + * extent map array in the data fork. Note that we + * previously zapped the fork if it had no chance of + * fitting in the inode. + */ + afork_min = sizeof(struct xfs_bmbt_rec) * attr_extents; + } else if (ri->attr_extents > 0) { + /* + * The attr fork thinks it has zero extents, but we + * found some xattr extents. We need to leave enough + * empty space here so that the incore attr fork will + * get created (and hence trigger the attr fork bmap + * repairer). + */ + afork_min = bmdr_minsz; + } else { + /* No extents on disk or found in rmapbt. */ + afork_min = 0; + } + break; + case XFS_DINODE_FMT_BTREE: + /* Must have space for btree header and key/pointers. */ + bmdr = XFS_DFORK_PTR(dip, XFS_ATTR_FORK); + afork_min = xfs_bmap_broot_space(sc->mp, bmdr); + break; + default: + /* We should never see any other formats. */ + afork_min = 0; + break; + } + + /* Compute the minimum space required for the data fork. */ + switch (dip->di_format) { + case XFS_DINODE_FMT_DEV: + dfork_min = sizeof(__be32); + break; + case XFS_DINODE_FMT_UUID: + dfork_min = sizeof(uuid_t); + break; + case XFS_DINODE_FMT_LOCAL: + /* + * If we still have a shortform data fork at all, that means + * the data fork area was large enough to fit whatever was in + * there. + */ + dfork_min = be64_to_cpu(dip->di_size); + break; + case XFS_DINODE_FMT_EXTENTS: + data_extents = xfs_dfork_data_extents(dip); + if (data_extents) { + /* + * We must maintain sufficient space to hold the entire + * extent map array in the data fork. Note that we + * previously zapped the fork if it had no chance of + * fitting in the inode. + */ + dfork_min = sizeof(struct xfs_bmbt_rec) * data_extents; + } else if (ri->data_extents > 0 || ri->rt_extents > 0) { + /* + * The data fork thinks it has zero extents, but we + * found some data extents. We need to leave enough + * empty space here so that the data fork bmap repair + * will recover the mappings. + */ + dfork_min = bmdr_minsz; + } else { + /* No extents on disk or found in rmapbt. */ + dfork_min = 0; + } + break; + case XFS_DINODE_FMT_BTREE: + /* Must have space for btree header and key/pointers. */ + bmdr = XFS_DFORK_PTR(dip, XFS_DATA_FORK); + dfork_min = xfs_bmap_broot_space(sc->mp, bmdr); + break; + case XFS_DINODE_FMT_META_BTREE: + switch (be16_to_cpu(dip->di_metatype)) { + case XFS_METAFILE_RTRMAP: + rmdr = XFS_DFORK_PTR(dip, XFS_DATA_FORK); + dfork_min = xfs_rtrmap_broot_space(sc->mp, rmdr); + break; + case XFS_METAFILE_RTREFCOUNT: + rcdr = XFS_DFORK_PTR(dip, XFS_DATA_FORK); + dfork_min = xfs_rtrefcount_broot_space(sc->mp, rcdr); + break; + default: + dfork_min = 0; + break; + } + break; + default: + dfork_min = 0; + break; + } + + /* + * Round all values up to the nearest 8 bytes, because that is the + * precision of di_forkoff. + */ + afork_min = roundup(afork_min, 8); + dfork_min = roundup(dfork_min, 8); + bmdr_minsz = roundup(bmdr_minsz, 8); + + ASSERT(dfork_min <= lit_sz); + ASSERT(afork_min <= lit_sz); + + /* + * If the data fork was zapped and we don't have enough space for the + * recovery fork, move the attr fork up. + */ + if (dip->di_format == XFS_DINODE_FMT_EXTENTS && + xfs_dfork_data_extents(dip) == 0 && + (ri->data_extents > 0 || ri->rt_extents > 0) && + bmdr_minsz > XFS_DFORK_DSIZE(dip, sc->mp)) { + if (bmdr_minsz + afork_min > lit_sz) { + /* + * The attr for and the stub fork we need to recover + * the data fork won't both fit. Zap the attr fork. + */ + xrep_dinode_zap_afork(ri, dip, mode); + afork_min = bmdr_minsz; + } else { + void *before, *after; + + /* Otherwise, just slide the attr fork up. */ + before = XFS_DFORK_APTR(dip); + dip->di_forkoff = bmdr_minsz >> 3; + after = XFS_DFORK_APTR(dip); + memmove(after, before, XFS_DFORK_ASIZE(dip, sc->mp)); + } + } + + /* + * If the attr fork was zapped and we don't have enough space for the + * recovery fork, move the attr fork down. + */ + if (dip->di_aformat == XFS_DINODE_FMT_EXTENTS && + xfs_dfork_attr_extents(dip) == 0 && + ri->attr_extents > 0 && + bmdr_minsz > XFS_DFORK_ASIZE(dip, sc->mp)) { + if (dip->di_format == XFS_DINODE_FMT_BTREE) { + /* + * If the data fork is in btree format then we can't + * adjust forkoff because that runs the risk of + * violating the extents/btree format transition rules. + */ + } else if (bmdr_minsz + dfork_min > lit_sz) { + /* + * If we can't move the attr fork, too bad, we lose the + * attr fork and leak its blocks. + */ + xrep_dinode_zap_afork(ri, dip, mode); + } else { + /* + * Otherwise, just slide the attr fork down. The attr + * fork is empty, so we don't have any old contents to + * move here. + */ + dip->di_forkoff = (lit_sz - bmdr_minsz) >> 3; + } + } +} + +/* + * Zap the data/attr forks if we spot anything that isn't going to pass the + * ifork verifiers or the ifork formatters, because we need to get the inode + * into good enough shape that the higher level repair functions can run. + */ +STATIC void +xrep_dinode_zap_forks( + struct xrep_inode *ri, + struct xfs_dinode *dip) +{ + struct xfs_scrub *sc = ri->sc; + xfs_extnum_t data_extents; + xfs_extnum_t attr_extents; + xfs_filblks_t nblocks; + uint16_t mode; + bool zap_datafork = false; + bool zap_attrfork = ri->zap_acls; + + trace_xrep_dinode_zap_forks(sc, dip); + + mode = be16_to_cpu(dip->di_mode); + + data_extents = xfs_dfork_data_extents(dip); + attr_extents = xfs_dfork_attr_extents(dip); + nblocks = be64_to_cpu(dip->di_nblocks); + + /* Inode counters don't make sense? */ + if (data_extents > nblocks) + zap_datafork = true; + if (attr_extents > nblocks) + zap_attrfork = true; + if (data_extents + attr_extents > nblocks) + zap_datafork = zap_attrfork = true; + + if (!zap_datafork) + zap_datafork = xrep_dinode_check_dfork(sc, dip, mode); + if (!zap_attrfork) + zap_attrfork = xrep_dinode_check_afork(sc, dip); + + /* Zap whatever's bad. */ + if (zap_attrfork) + xrep_dinode_zap_afork(ri, dip, mode); + if (zap_datafork) + xrep_dinode_zap_dfork(ri, dip, mode); + xrep_dinode_ensure_forkoff(ri, dip, mode); + + /* + * Zero di_nblocks if we don't have any extents at all to satisfy the + * buffer verifier. + */ + data_extents = xfs_dfork_data_extents(dip); + attr_extents = xfs_dfork_attr_extents(dip); + if (data_extents + attr_extents == 0) + dip->di_nblocks = 0; +} + +/* Inode didn't pass dinode verifiers, so fix the raw buffer and retry iget. */ +STATIC int +xrep_dinode_core( + struct xrep_inode *ri) +{ + struct xfs_scrub *sc = ri->sc; + struct xfs_mount *mp = sc->mp; + struct xfs_buf *bp; + struct xfs_dinode *dip; + xfs_ino_t ino = sc->sm->sm_ino; + int error; + int iget_error; + + /* Figure out what this inode had mapped in both forks. */ + error = xrep_dinode_count_rmaps(ri); + if (error) + return error; + + /* Read the inode cluster buffer. */ + error = xfs_trans_read_buf(mp, sc->tp, mp->m_ddev_targp, + XFS_AGB_TO_DADDR(mp, XFS_INO_TO_AGNO(mp, ino), + ri->imap.im_agbno), + XFS_FSB_TO_BB(mp, M_IGEO(mp)->blocks_per_cluster), + 0, &bp, NULL); + if (error) + return error; + + /* Make sure we can pass the inode buffer verifier. */ + xrep_dinode_buf(sc, bp); + bp->b_ops = &xfs_inode_buf_ops; + + /* Fix everything the verifier will complain about. */ + dip = xfs_buf_offset(bp, ri->imap.im_boffset); + xrep_dinode_header(sc, dip); + iget_error = xrep_dinode_mode(ri, dip); + if (iget_error) + goto write; + xrep_dinode_nlinks(dip); + xrep_dinode_flags(sc, dip, ri->rt_extents > 0); + xrep_dinode_size(ri, dip); + xrep_dinode_extsize_hints(sc, dip); + xrep_dinode_zap_forks(ri, dip); + +write: + /* Write out the inode. */ + trace_xrep_dinode_fixed(sc, dip); + xfs_dinode_calc_crc(mp, dip); + xfs_trans_buf_set_type(sc->tp, bp, XFS_BLFT_DINO_BUF); + xfs_trans_log_buf(sc->tp, bp, ri->imap.im_boffset, + ri->imap.im_boffset + mp->m_sb.sb_inodesize - 1); + + /* + * In theory, we've fixed the ondisk inode record enough that we should + * be able to load the inode into the cache. Try to iget that inode + * now while we hold the AGI and the inode cluster buffer and take the + * IOLOCK so that we can continue with repairs without anyone else + * accessing the inode. If iget fails, we still need to commit the + * changes. + */ + if (!iget_error) + iget_error = xchk_iget(sc, ino, &sc->ip); + if (!iget_error) + xchk_ilock(sc, XFS_IOLOCK_EXCL); + + /* + * Commit the inode cluster buffer updates and drop the AGI buffer that + * we've been holding since scrub setup. From here on out, repairs + * deal only with the cached inode. + */ + error = xrep_trans_commit(sc); + if (error) + return error; + + if (iget_error) + return iget_error; + + error = xchk_trans_alloc(sc, 0); + if (error) + return error; + + error = xrep_ino_dqattach(sc); + if (error) + return error; + + xchk_ilock(sc, XFS_ILOCK_EXCL); + if (ri->ino_sick_mask) + xfs_inode_mark_sick(sc->ip, ri->ino_sick_mask); + return 0; +} + +/* Fix everything xfs_dinode_verify cares about. */ +STATIC int +xrep_dinode_problems( + struct xrep_inode *ri) +{ + struct xfs_scrub *sc = ri->sc; + int error; + + error = xrep_dinode_core(ri); + if (error) + return error; + + /* We had to fix a totally busted inode, schedule quotacheck. */ + if (XFS_IS_UQUOTA_ON(sc->mp)) + xrep_force_quotacheck(sc, XFS_DQTYPE_USER); + if (XFS_IS_GQUOTA_ON(sc->mp)) + xrep_force_quotacheck(sc, XFS_DQTYPE_GROUP); + if (XFS_IS_PQUOTA_ON(sc->mp)) + xrep_force_quotacheck(sc, XFS_DQTYPE_PROJ); + + return 0; +} + +/* + * Fix problems that the verifiers don't care about. In general these are + * errors that don't cause problems elsewhere in the kernel that we can easily + * detect, so we don't check them all that rigorously. + */ + +/* Make sure block and extent counts are ok. */ +STATIC int +xrep_inode_blockcounts( + struct xfs_scrub *sc) +{ + struct xfs_ifork *ifp; + xfs_filblks_t count; + xfs_filblks_t acount; + xfs_extnum_t nextents; + int error; + + trace_xrep_inode_blockcounts(sc); + + /* Set data fork counters from the data fork mappings. */ + error = xchk_inode_count_blocks(sc, XFS_DATA_FORK, &nextents, &count); + if (error) + return error; + if (xfs_is_reflink_inode(sc->ip)) { + /* + * data fork blockcount can exceed physical storage if a user + * reflinks the same block over and over again. + */ + ; + } else if (XFS_IS_REALTIME_INODE(sc->ip)) { + if (count >= sc->mp->m_sb.sb_rblocks) + return -EFSCORRUPTED; + } else { + if (count >= sc->mp->m_sb.sb_dblocks) + return -EFSCORRUPTED; + } + error = xrep_ino_ensure_extent_count(sc, XFS_DATA_FORK, nextents); + if (error) + return error; + sc->ip->i_df.if_nextents = nextents; + + /* Set attr fork counters from the attr fork mappings. */ + ifp = xfs_ifork_ptr(sc->ip, XFS_ATTR_FORK); + if (ifp) { + error = xchk_inode_count_blocks(sc, XFS_ATTR_FORK, &nextents, + &acount); + if (error) + return error; + if (count >= sc->mp->m_sb.sb_dblocks) + return -EFSCORRUPTED; + error = xrep_ino_ensure_extent_count(sc, XFS_ATTR_FORK, + nextents); + if (error) + return error; + ifp->if_nextents = nextents; + } else { + acount = 0; + } + + sc->ip->i_nblocks = count + acount; + return 0; +} + +/* Check for invalid uid/gid/prid. */ +STATIC void +xrep_inode_ids( + struct xfs_scrub *sc) +{ + bool dirty = false; + + trace_xrep_inode_ids(sc); + + if (!uid_valid(VFS_I(sc->ip)->i_uid)) { + i_uid_write(VFS_I(sc->ip), 0); + dirty = true; + if (XFS_IS_UQUOTA_ON(sc->mp)) + xrep_force_quotacheck(sc, XFS_DQTYPE_USER); + } + + if (!gid_valid(VFS_I(sc->ip)->i_gid)) { + i_gid_write(VFS_I(sc->ip), 0); + dirty = true; + if (XFS_IS_GQUOTA_ON(sc->mp)) + xrep_force_quotacheck(sc, XFS_DQTYPE_GROUP); + } + + if (sc->ip->i_projid == -1U) { + sc->ip->i_projid = 0; + dirty = true; + if (XFS_IS_PQUOTA_ON(sc->mp)) + xrep_force_quotacheck(sc, XFS_DQTYPE_PROJ); + } + + /* strip setuid/setgid if we touched any of the ids */ + if (dirty) + VFS_I(sc->ip)->i_mode &= ~(S_ISUID | S_ISGID); +} + +static inline void +xrep_clamp_timestamp( + struct xfs_inode *ip, + struct timespec64 *ts) +{ + ts->tv_nsec = clamp_t(long, ts->tv_nsec, 0, NSEC_PER_SEC - 1); + *ts = timestamp_truncate(*ts, VFS_I(ip)); +} + +/* Nanosecond counters can't have more than 1 billion. */ +STATIC void +xrep_inode_timestamps( + struct xfs_inode *ip) +{ + struct timespec64 tstamp; + struct inode *inode = VFS_I(ip); + + tstamp = inode_get_atime(inode); + xrep_clamp_timestamp(ip, &tstamp); + inode_set_atime_to_ts(inode, tstamp); + + tstamp = inode_get_mtime(inode); + xrep_clamp_timestamp(ip, &tstamp); + inode_set_mtime_to_ts(inode, tstamp); + + tstamp = inode_get_ctime(inode); + xrep_clamp_timestamp(ip, &tstamp); + inode_set_ctime_to_ts(inode, tstamp); + + xrep_clamp_timestamp(ip, &ip->i_crtime); +} + +/* Fix inode flags that don't make sense together. */ +STATIC void +xrep_inode_flags( + struct xfs_scrub *sc) +{ + uint16_t mode; + + trace_xrep_inode_flags(sc); + + mode = VFS_I(sc->ip)->i_mode; + + /* Clear junk flags */ + if (sc->ip->i_diflags & ~XFS_DIFLAG_ANY) + sc->ip->i_diflags &= XFS_DIFLAG_ANY; + + /* NEWRTBM only applies to realtime bitmaps */ + if (I_INO(sc->ip) == sc->mp->m_sb.sb_rbmino) + sc->ip->i_diflags |= XFS_DIFLAG_NEWRTBM; + else + sc->ip->i_diflags &= ~XFS_DIFLAG_NEWRTBM; + + /* These only make sense for directories. */ + if (!S_ISDIR(mode)) + sc->ip->i_diflags &= ~(XFS_DIFLAG_RTINHERIT | + XFS_DIFLAG_EXTSZINHERIT | + XFS_DIFLAG_PROJINHERIT | + XFS_DIFLAG_NOSYMLINKS); + + /* These only make sense for files. */ + if (!S_ISREG(mode)) + sc->ip->i_diflags &= ~(XFS_DIFLAG_REALTIME | + XFS_DIFLAG_EXTSIZE); + + /* These only make sense for non-rt files. */ + if (sc->ip->i_diflags & XFS_DIFLAG_REALTIME) + sc->ip->i_diflags &= ~XFS_DIFLAG_FILESTREAM; + + /* Immutable and append only? Drop the append. */ + if ((sc->ip->i_diflags & XFS_DIFLAG_IMMUTABLE) && + (sc->ip->i_diflags & XFS_DIFLAG_APPEND)) + sc->ip->i_diflags &= ~XFS_DIFLAG_APPEND; + + /* Clear junk flags. */ + if (sc->ip->i_diflags2 & ~XFS_DIFLAG2_ANY) + sc->ip->i_diflags2 &= XFS_DIFLAG2_ANY; + + /* No reflink flag unless we support it and it's a file. */ + if (!xfs_has_reflink(sc->mp) || !S_ISREG(mode)) + sc->ip->i_diflags2 &= ~XFS_DIFLAG2_REFLINK; + + /* DAX only applies to files and dirs. */ + if (!(S_ISREG(mode) || S_ISDIR(mode))) + sc->ip->i_diflags2 &= ~XFS_DIFLAG2_DAX; +} + +/* + * Fix size problems with block/node format directories. If we fail to find + * the extent list, just bail out and let the bmapbtd repair functions clean + * up that mess. + */ +STATIC void +xrep_inode_blockdir_size( + struct xfs_scrub *sc) +{ + struct xfs_iext_cursor icur; + struct xfs_bmbt_irec got; + struct xfs_ifork *ifp; + xfs_fileoff_t off; + int error; + + trace_xrep_inode_blockdir_size(sc); + + error = xfs_iread_extents(sc->tp, sc->ip, XFS_DATA_FORK); + if (error) + return; + + /* Find the last block before 32G; this is the dir size. */ + ifp = xfs_ifork_ptr(sc->ip, XFS_DATA_FORK); + off = XFS_B_TO_FSB(sc->mp, XFS_DIR2_SPACE_SIZE); + if (!xfs_iext_lookup_extent_before(sc->ip, ifp, &off, &icur, &got)) { + /* zero-extents directory? */ + return; + } + + off = got.br_startoff + got.br_blockcount; + sc->ip->i_disk_size = min_t(loff_t, XFS_DIR2_SPACE_SIZE, + XFS_FSB_TO_B(sc->mp, off)); +} + +/* Fix size problems with short format directories. */ +STATIC void +xrep_inode_sfdir_size( + struct xfs_scrub *sc) +{ + struct xfs_ifork *ifp; + + trace_xrep_inode_sfdir_size(sc); + + ifp = xfs_ifork_ptr(sc->ip, XFS_DATA_FORK); + sc->ip->i_disk_size = ifp->if_bytes; +} + +/* + * Fix any irregularities in a directory inode's size now that we can iterate + * extent maps and access other regular inode data. + */ +STATIC void +xrep_inode_dir_size( + struct xfs_scrub *sc) +{ + trace_xrep_inode_dir_size(sc); + + switch (sc->ip->i_df.if_format) { + case XFS_DINODE_FMT_EXTENTS: + case XFS_DINODE_FMT_BTREE: + xrep_inode_blockdir_size(sc); + break; + case XFS_DINODE_FMT_LOCAL: + xrep_inode_sfdir_size(sc); + break; + } +} + +/* Fix extent size hint problems. */ +STATIC void +xrep_inode_extsize( + struct xfs_scrub *sc) +{ + /* Fix misaligned extent size hints on a directory. */ + if ((sc->ip->i_diflags & XFS_DIFLAG_RTINHERIT) && + (sc->ip->i_diflags & XFS_DIFLAG_EXTSZINHERIT) && + xfs_extlen_to_rtxmod(sc->mp, sc->ip->i_extsize) > 0) { + sc->ip->i_extsize = 0; + sc->ip->i_diflags &= ~XFS_DIFLAG_EXTSZINHERIT; + } +} + +/* Ensure this file has an attr fork if it needs to hold a parent pointer. */ +STATIC int +xrep_inode_pptr( + struct xfs_scrub *sc) +{ + struct xfs_mount *mp = sc->mp; + struct xfs_inode *ip = sc->ip; + struct inode *inode = VFS_I(ip); + + if (!xfs_has_parent(mp)) + return 0; + + /* + * Unlinked inodes that cannot be added to the directory tree will not + * have a parent pointer. + */ + if (inode->i_nlink == 0 && !(inode_state_read_once(inode) & I_LINKABLE)) + return 0; + + /* Children of the superblock do not have parent pointers. */ + if (xchk_inode_is_sb_rooted(ip)) + return 0; + + /* Inode already has an attr fork; no further work possible here. */ + if (xfs_inode_has_attr_fork(ip)) + return 0; + + return xfs_bmap_add_attrfork(sc->tp, ip, + sizeof(struct xfs_attr_sf_hdr), true); +} + +/* Fix COW extent size hint problems. */ +STATIC void +xrep_inode_cowextsize( + struct xfs_scrub *sc) +{ + /* Fix misaligned CoW extent size hints on a directory. */ + if ((sc->ip->i_diflags & XFS_DIFLAG_RTINHERIT) && + (sc->ip->i_diflags2 & XFS_DIFLAG2_COWEXTSIZE) && + xfs_extlen_to_rtxmod(sc->mp, sc->ip->i_cowextsize) > 0) { + sc->ip->i_cowextsize = 0; + sc->ip->i_diflags2 &= ~XFS_DIFLAG2_COWEXTSIZE; + } +} + +/* Fix any irregularities in an inode that the verifiers don't catch. */ +STATIC int +xrep_inode_problems( + struct xfs_scrub *sc) +{ + int error; + + error = xrep_inode_blockcounts(sc); + if (error) + return error; + error = xrep_inode_pptr(sc); + if (error) + return error; + xrep_inode_timestamps(sc->ip); + xrep_inode_flags(sc); + xrep_inode_ids(sc); + /* + * We can now do a better job fixing the size of a directory now that + * we can scan the data fork extents than we could in xrep_dinode_size. + */ + if (S_ISDIR(VFS_I(sc->ip)->i_mode)) + xrep_inode_dir_size(sc); + xrep_inode_extsize(sc); + xrep_inode_cowextsize(sc); + + trace_xrep_inode_fixed(sc); + xfs_trans_log_inode(sc->tp, sc->ip, XFS_ILOG_CORE); + return xrep_roll_trans(sc); +} + +/* + * Make sure this inode's unlinked list pointers are consistent with its + * link count. + */ +STATIC int +xrep_inode_unlinked( + struct xfs_scrub *sc) +{ + unsigned int nlink = VFS_I(sc->ip)->i_nlink; + int error; + + /* + * If this inode is linked from the directory tree and on the unlinked + * list, remove it from the unlinked list. + */ + if (nlink > 0 && xfs_inode_on_unlinked_list(sc->ip)) { + struct xfs_perag *pag; + int error; + + pag = xfs_perag_get(sc->mp, XFS_INODE_TO_AGNO(sc->ip)); + error = xfs_iunlink_remove(sc->tp, pag, sc->ip); + xfs_perag_put(pag); + if (error) + return error; + } + + /* + * If this inode is not linked from the directory tree yet not on the + * unlinked list, put it on the unlinked list. + */ + if (nlink == 0 && !xfs_inode_on_unlinked_list(sc->ip)) { + error = xfs_iunlink(sc->tp, sc->ip); + if (error) + return error; + } + + return 0; +} + +/* Repair an inode's fields. */ +int +xrep_inode( + struct xfs_scrub *sc) +{ + int error = 0; + + /* + * No inode? That means we failed the _iget verifiers. Repair all + * the things that the inode verifiers care about, then retry _iget. + */ + if (!sc->ip) { + struct xrep_inode *ri = sc->buf; + + ASSERT(ri != NULL); + + error = xrep_dinode_problems(ri); + if (error == -EBUSY) { + /* + * Directory scan to recover inode mode encountered a + * busy inode, so we did not continue repairing things. + */ + return 0; + } + if (error) + return error; + + /* By this point we had better have a working incore inode. */ + if (!sc->ip) + return -EFSCORRUPTED; + } + + xfs_trans_ijoin(sc->tp, sc->ip, 0); + + /* If we found corruption of any kind, try to fix it. */ + if ((sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) || + (sc->sm->sm_flags & XFS_SCRUB_OFLAG_XCORRUPT)) { + error = xrep_inode_problems(sc); + if (error) + return error; + } + + /* See if we can clear the reflink flag. */ + if (xfs_is_reflink_inode(sc->ip)) { + error = xfs_reflink_clear_inode_flag(sc->ip, &sc->tp); + if (error) + return error; + } + + /* Reconnect incore unlinked list */ + error = xrep_inode_unlinked(sc); + if (error) + return error; + + return xrep_defer_finish(sc); +} diff --git a/fs/xfs/scrub/iscan.c b/fs/xfs/scrub/iscan.c new file mode 100644 index 000000000..03f72ba9e --- /dev/null +++ b/fs/xfs/scrub/iscan.c @@ -0,0 +1,826 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2021-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_inode.h" +#include "xfs_btree.h" +#include "xfs_ialloc.h" +#include "xfs_ialloc_btree.h" +#include "xfs_ag.h" +#include "xfs_error.h" +#include "xfs_bit.h" +#include "xfs_icache.h" +#include "scrub/scrub.h" +#include "scrub/iscan.h" +#include "scrub/common.h" +#include "scrub/trace.h" + +/* + * Live File Scan + * ============== + * + * Live file scans walk every inode in a live filesystem. This is more or + * less like a regular iwalk, except that when we're advancing the scan cursor, + * we must ensure that inodes cannot be added or deleted anywhere between the + * old cursor value and the new cursor value. If we're advancing the cursor + * by one inode, the caller must hold that inode; if we're finding the next + * inode to scan, we must grab the AGI and hold it until we've updated the + * scan cursor. + * + * Callers are expected to use this code to scan all files in the filesystem to + * construct a new metadata index of some kind. The scan races against other + * live updates, which means there must be a provision to update the new index + * when updates are made to inodes that already been scanned. The iscan lock + * can be used in live update hook code to stop the scan and protect this data + * structure. + * + * To keep the new index up to date with other metadata updates being made to + * the live filesystem, it is assumed that the caller will add hooks as needed + * to be notified when a metadata update occurs. The inode scanner must tell + * the hook code when an inode has been visited with xchk_iscan_mark_visit. + * Hook functions can use xchk_iscan_want_live_update to decide if the + * scanner's observations must be updated. + */ + +/* + * If the inobt record @rec covers @iscan->skip_ino, mark the inode free so + * that the scan ignores that inode. + */ +STATIC void +xchk_iscan_mask_skipino( + struct xchk_iscan *iscan, + struct xfs_perag *pag, + struct xfs_inobt_rec_incore *rec, + xfs_agino_t lastrecino) +{ + struct xfs_scrub *sc = iscan->sc; + struct xfs_mount *mp = sc->mp; + xfs_agnumber_t skip_agno = XFS_INO_TO_AGNO(mp, iscan->skip_ino); + xfs_agnumber_t skip_agino = XFS_INO_TO_AGINO(mp, iscan->skip_ino); + + if (pag_agno(pag) != skip_agno) + return; + if (skip_agino < rec->ir_startino) + return; + if (skip_agino > lastrecino) + return; + + rec->ir_free |= xfs_inobt_maskn(skip_agino - rec->ir_startino, 1); +} + +/* + * Set *cursor to the next allocated inode after whatever it's set to now. + * If there are no more inodes in this AG, cursor is set to NULLAGINO. + */ +STATIC int +xchk_iscan_find_next( + struct xchk_iscan *iscan, + struct xfs_buf *agi_bp, + struct xfs_perag *pag, + xfs_inofree_t *allocmaskp, + xfs_agino_t *cursor, + uint8_t *nr_inodesp) +{ + struct xfs_scrub *sc = iscan->sc; + struct xfs_inobt_rec_incore rec; + struct xfs_btree_cur *cur; + struct xfs_mount *mp = sc->mp; + struct xfs_trans *tp = sc->tp; + xfs_agnumber_t agno = pag_agno(pag); + xfs_agino_t lastino = NULLAGINO; + xfs_agino_t first, last; + xfs_agino_t agino = *cursor; + int has_rec; + int error; + + /* If the cursor is beyond the end of this AG, move to the next one. */ + xfs_agino_range(mp, agno, &first, &last); + if (agino > last) { + *cursor = NULLAGINO; + return 0; + } + + /* + * Look up the inode chunk for the current cursor position. If there + * is no chunk here, we want the next one. + */ + cur = xfs_inobt_init_cursor(pag, tp, agi_bp); + error = xfs_inobt_lookup(cur, agino, XFS_LOOKUP_LE, &has_rec); + if (!error && !has_rec) + error = xfs_btree_increment(cur, 0, &has_rec); + for (; !error; error = xfs_btree_increment(cur, 0, &has_rec)) { + xfs_inofree_t allocmask; + + /* + * If we've run out of inobt records in this AG, move the + * cursor on to the next AG and exit. The caller can try + * again with the next AG. + */ + if (!has_rec) { + *cursor = NULLAGINO; + break; + } + + error = xfs_inobt_get_rec(cur, &rec, &has_rec); + if (error) + break; + if (!has_rec) { + error = -EFSCORRUPTED; + break; + } + + /* Make sure that we always move forward. */ + if (lastino != NULLAGINO && + XFS_IS_CORRUPT(mp, lastino >= rec.ir_startino)) { + error = -EFSCORRUPTED; + break; + } + lastino = rec.ir_startino + XFS_INODES_PER_CHUNK - 1; + + /* + * If this record only covers inodes that come before the + * cursor, advance to the next record. + */ + if (rec.ir_startino + XFS_INODES_PER_CHUNK <= agino) + continue; + + if (iscan->skip_ino) + xchk_iscan_mask_skipino(iscan, pag, &rec, lastino); + + /* + * If the incoming lookup put us in the middle of an inobt + * record, mark it and the previous inodes "free" so that the + * search for allocated inodes will start at the cursor. + * We don't care about ir_freecount here. + */ + if (agino >= rec.ir_startino) + rec.ir_free |= xfs_inobt_maskn(0, + agino + 1 - rec.ir_startino); + + /* + * If there are allocated inodes in this chunk, find them + * and update the scan cursor. + */ + allocmask = ~rec.ir_free; + if (hweight64(allocmask) > 0) { + int next = xfs_lowbit64(allocmask); + + ASSERT(next >= 0); + *cursor = rec.ir_startino + next; + *allocmaskp = allocmask >> next; + *nr_inodesp = XFS_INODES_PER_CHUNK - next; + break; + } + } + + xfs_btree_del_cursor(cur, error); + return error; +} + +/* + * Advance both the scan and the visited cursors. + * + * The inumber address space for a given filesystem is sparse, which means that + * the scan cursor can jump a long ways in a single iter() call. There are no + * inodes in these sparse areas, so we must move the visited cursor forward at + * the same time so that the scan user can receive live updates for inodes that + * may get created once we release the AGI buffer. + */ +static inline void +xchk_iscan_move_cursor( + struct xchk_iscan *iscan, + xfs_agnumber_t agno, + xfs_agino_t agino) +{ + struct xfs_scrub *sc = iscan->sc; + struct xfs_mount *mp = sc->mp; + xfs_ino_t cursor, visited; + + BUILD_BUG_ON(XFS_MAXINUMBER == NULLFSINO); + + /* + * Special-case ino == 0 here so that we never set visited_ino to + * NULLFSINO when wrapping around EOFS, for that will let through all + * live updates. + */ + cursor = XFS_AGINO_TO_INO(mp, agno, agino); + if (cursor == 0) + visited = XFS_MAXINUMBER; + else + visited = cursor - 1; + + mutex_lock(&iscan->lock); + iscan->cursor_ino = cursor; + iscan->__visited_ino = visited; + trace_xchk_iscan_move_cursor(iscan); + mutex_unlock(&iscan->lock); +} + +/* + * Prepare to return agno/agino to the iscan caller by moving the lastino + * cursor to the previous inode. Do this while we still hold the AGI so that + * no other threads can create or delete inodes in this AG. + */ +static inline void +xchk_iscan_finish( + struct xchk_iscan *iscan) +{ + mutex_lock(&iscan->lock); + iscan->cursor_ino = NULLFSINO; + + /* All live updates will be applied from now on */ + iscan->__visited_ino = NULLFSINO; + + mutex_unlock(&iscan->lock); +} + +/* Mark an inode scan finished before we actually scan anything. */ +void +xchk_iscan_finish_early( + struct xchk_iscan *iscan) +{ + ASSERT(iscan->cursor_ino == iscan->scan_start_ino); + ASSERT(iscan->__visited_ino == iscan->scan_start_ino); + + xchk_iscan_finish(iscan); +} + +/* + * Grab the AGI to advance the inode scan. Returns 0 if *agi_bpp is now set, + * -ECANCELED if the live scan aborted, -EBUSY if the AGI could not be grabbed, + * or the usual negative errno. + */ +STATIC int +xchk_iscan_read_agi( + struct xchk_iscan *iscan, + struct xfs_perag *pag, + struct xfs_buf **agi_bpp) +{ + struct xfs_scrub *sc = iscan->sc; + unsigned long relax; + int ret; + + if (!xchk_iscan_agi_needs_trylock(iscan)) + return xfs_ialloc_read_agi(pag, sc->tp, 0, agi_bpp); + + relax = msecs_to_jiffies(iscan->iget_retry_delay); + do { + ret = xfs_ialloc_read_agi(pag, sc->tp, XFS_IALLOC_FLAG_TRYLOCK, + agi_bpp); + if (ret != -EAGAIN) + return ret; + if (!iscan->iget_timeout || + time_is_before_jiffies(iscan->__iget_deadline)) + return -EBUSY; + + trace_xchk_iscan_agi_retry_wait(iscan); + } while (!schedule_timeout_killable(relax) && + !xchk_iscan_aborted(iscan)); + return -ECANCELED; +} + +/* + * Advance ino to the next inode that the inobt thinks is allocated, being + * careful to jump to the next AG if we've reached the right end of this AG's + * inode btree. Advancing ino effectively means that we've pushed the inode + * scan forward, so set the iscan cursor to (ino - 1) so that our live update + * predicates will track inode allocations in that part of the inode number + * key space once we release the AGI buffer. + * + * Returns 1 if there's a new inode to examine, 0 if we've run out of inodes, + * -ECANCELED if the live scan aborted, or the usual negative errno. + */ +STATIC int +xchk_iscan_advance( + struct xchk_iscan *iscan, + struct xfs_perag **pagp, + struct xfs_buf **agi_bpp, + xfs_inofree_t *allocmaskp, + uint8_t *nr_inodesp) +{ + struct xfs_scrub *sc = iscan->sc; + struct xfs_mount *mp = sc->mp; + struct xfs_buf *agi_bp; + struct xfs_perag *pag; + xfs_agnumber_t agno; + xfs_agino_t agino; + int ret; + + ASSERT(iscan->cursor_ino >= iscan->__visited_ino); + + do { + if (xchk_iscan_aborted(iscan)) + return -ECANCELED; + + agno = XFS_INO_TO_AGNO(mp, iscan->cursor_ino); + pag = xfs_perag_get(mp, agno); + if (!pag) + return -ECANCELED; + + ret = xchk_iscan_read_agi(iscan, pag, &agi_bp); + if (ret) + goto out_pag; + + agino = XFS_INO_TO_AGINO(mp, iscan->cursor_ino); + ret = xchk_iscan_find_next(iscan, agi_bp, pag, allocmaskp, + &agino, nr_inodesp); + if (ret) + goto out_buf; + + if (agino != NULLAGINO) { + /* + * Found the next inode in this AG, so return it along + * with the AGI buffer and the perag structure to + * ensure it cannot go away. + */ + xchk_iscan_move_cursor(iscan, agno, agino); + *agi_bpp = agi_bp; + *pagp = pag; + return 1; + } + + /* + * Did not find any more inodes in this AG, move on to the next + * AG. + */ + agno = (agno + 1) % mp->m_sb.sb_agcount; + xchk_iscan_move_cursor(iscan, agno, 0); + xfs_trans_brelse(sc->tp, agi_bp); + xfs_perag_put(pag); + + trace_xchk_iscan_advance_ag(iscan); + } while (iscan->cursor_ino != iscan->scan_start_ino); + + xchk_iscan_finish(iscan); + return 0; + +out_buf: + xfs_trans_brelse(sc->tp, agi_bp); +out_pag: + xfs_perag_put(pag); + return ret; +} + +/* + * Grabbing the inode failed, so we need to back up the scan and ask the caller + * to try to _advance the scan again. Returns -EBUSY if we've run out of retry + * opportunities, -ECANCELED if the process has a fatal signal pending, or + * -EAGAIN if we should try again. + */ +STATIC int +xchk_iscan_iget_retry( + struct xchk_iscan *iscan, + bool wait) +{ + ASSERT(iscan->cursor_ino == iscan->__visited_ino + 1); + + if (!iscan->iget_timeout || + time_is_before_jiffies(iscan->__iget_deadline)) + return -EBUSY; + + if (wait) { + unsigned long relax; + + /* + * Sleep for a period of time to let the rest of the system + * catch up. If we return early, someone sent a kill signal to + * the calling process. + */ + relax = msecs_to_jiffies(iscan->iget_retry_delay); + trace_xchk_iscan_iget_retry_wait(iscan); + + if (schedule_timeout_killable(relax) || + xchk_iscan_aborted(iscan)) + return -ECANCELED; + } + + iscan->cursor_ino--; + return -EAGAIN; +} + +/* + * For an inode scan, we hold the AGI and want to try to grab a batch of + * inodes. Holding the AGI prevents inodegc from clearing freed inodes, + * so we must use noretry here. For every inode after the first one in the + * batch, we don't want to wait, so we use retry there too. Finally, use + * dontcache to avoid polluting the cache. + */ +#define ISCAN_IGET_FLAGS (XFS_IGET_NORETRY | XFS_IGET_DONTCACHE) + +/* + * Grab an inode as part of an inode scan. While scanning this inode, the + * caller must ensure that no other threads can modify the inode until a call + * to xchk_iscan_visit succeeds. + * + * Returns the number of incore inodes grabbed; -EAGAIN if the caller should + * call again xchk_iscan_advance; -EBUSY if we couldn't grab an inode; + * -ECANCELED if there's a fatal signal pending; or some other negative errno. + */ +STATIC int +xchk_iscan_iget( + struct xchk_iscan *iscan, + struct xfs_perag *pag, + struct xfs_buf *agi_bp, + xfs_inofree_t allocmask, + uint8_t nr_inodes) +{ + struct xfs_scrub *sc = iscan->sc; + struct xfs_mount *mp = sc->mp; + xfs_ino_t ino = iscan->cursor_ino; + unsigned int idx = 0; + unsigned int i; + int error; + + ASSERT(iscan->__inodes[0] == NULL); + + /* Fill the first slot in the inode array. */ + error = xfs_iget(sc->mp, sc->tp, ino, ISCAN_IGET_FLAGS, 0, + &iscan->__inodes[idx]); + + trace_xchk_iscan_iget(iscan, error); + + if (error == -ENOENT || error == -EAGAIN) { + xfs_trans_brelse(sc->tp, agi_bp); + xfs_perag_put(pag); + + /* + * It's possible that this inode has lost all of its links but + * hasn't yet been inactivated. If we don't have a transaction + * or it's not writable, flush the inodegc workers and wait. + * If we have a non-empty transaction, we must not block on + * inodegc, which allocates its own transactions. + */ + if (sc->tp && !(sc->tp->t_flags & XFS_TRANS_NO_WRITECOUNT)) + xfs_inodegc_push(mp); + else + xfs_inodegc_flush(mp); + return xchk_iscan_iget_retry(iscan, true); + } + + if (error == -EINVAL) { + xfs_trans_brelse(sc->tp, agi_bp); + xfs_perag_put(pag); + + /* + * We thought the inode was allocated, but the inode btree + * lookup failed, which means that it was freed since the last + * time we advanced the cursor. Back up and try again. This + * should never happen since still hold the AGI buffer from the + * inobt check, but we need to be careful about infinite loops. + */ + return xchk_iscan_iget_retry(iscan, false); + } + + if (error) { + xfs_trans_brelse(sc->tp, agi_bp); + xfs_perag_put(pag); + return error; + } + idx++; + ino++; + allocmask >>= 1; + + /* + * Now that we've filled the first slot in __inodes, try to fill the + * rest of the batch with consecutively ordered inodes. to reduce the + * number of _iter calls. Make a bitmap of unallocated inodes from the + * zeroes in the inuse bitmap; these inodes will not be scanned, but + * the _want_live_update predicate will pass through all live updates. + * + * If we can't iget an allocated inode, stop and return what we have. + */ + mutex_lock(&iscan->lock); + iscan->__batch_ino = ino - 1; + iscan->__skipped_inomask = 0; + mutex_unlock(&iscan->lock); + + for (i = 1; i < nr_inodes; i++, ino++, allocmask >>= 1) { + if (!(allocmask & 1)) { + ASSERT(!(iscan->__skipped_inomask & (1ULL << i))); + + mutex_lock(&iscan->lock); + iscan->cursor_ino = ino; + iscan->__skipped_inomask |= (1ULL << i); + mutex_unlock(&iscan->lock); + continue; + } + + ASSERT(iscan->__inodes[idx] == NULL); + + error = xfs_iget(sc->mp, sc->tp, ino, ISCAN_IGET_FLAGS, 0, + &iscan->__inodes[idx]); + if (error) + break; + + mutex_lock(&iscan->lock); + iscan->cursor_ino = ino; + mutex_unlock(&iscan->lock); + idx++; + } + + trace_xchk_iscan_iget_batch(sc->mp, iscan, nr_inodes, idx); + xfs_trans_brelse(sc->tp, agi_bp); + xfs_perag_put(pag); + return idx; +} + +/* + * Advance the visit cursor to reflect skipped inodes beyond whatever we + * scanned. + */ +STATIC void +xchk_iscan_finish_batch( + struct xchk_iscan *iscan) +{ + xfs_ino_t highest_skipped; + + mutex_lock(&iscan->lock); + + if (iscan->__batch_ino != NULLFSINO) { + highest_skipped = iscan->__batch_ino + + xfs_highbit64(iscan->__skipped_inomask); + iscan->__visited_ino = max(iscan->__visited_ino, + highest_skipped); + + trace_xchk_iscan_skip(iscan); + } + + iscan->__batch_ino = NULLFSINO; + iscan->__skipped_inomask = 0; + + mutex_unlock(&iscan->lock); +} + +/* + * Advance the inode scan cursor to the next allocated inode and return up to + * 64 consecutive allocated inodes starting with the cursor position. + */ +STATIC int +xchk_iscan_iter_batch( + struct xchk_iscan *iscan) +{ + struct xfs_scrub *sc = iscan->sc; + int ret; + + xchk_iscan_finish_batch(iscan); + + if (iscan->iget_timeout) + iscan->__iget_deadline = jiffies + + msecs_to_jiffies(iscan->iget_timeout); + + do { + struct xfs_buf *agi_bp = NULL; + struct xfs_perag *pag = NULL; + xfs_inofree_t allocmask = 0; + uint8_t nr_inodes = 0; + + ret = xchk_iscan_advance(iscan, &pag, &agi_bp, &allocmask, + &nr_inodes); + if (ret != 1) + return ret; + + if (xchk_iscan_aborted(iscan)) { + xfs_trans_brelse(sc->tp, agi_bp); + xfs_perag_put(pag); + ret = -ECANCELED; + break; + } + + ret = xchk_iscan_iget(iscan, pag, agi_bp, allocmask, nr_inodes); + } while (ret == -EAGAIN); + + return ret; +} + +/* + * Advance the inode scan cursor to the next allocated inode and return the + * incore inode structure associated with it. + * + * Returns 1 if there's a new inode to examine, 0 if we've run out of inodes, + * -ECANCELED if the live scan aborted, -EBUSY if the incore inode could not be + * grabbed, or the usual negative errno. + * + * If the function returns -EBUSY and the caller can handle skipping an inode, + * it may call this function again to continue the scan with the next allocated + * inode. + */ +int +xchk_iscan_iter( + struct xchk_iscan *iscan, + struct xfs_inode **ipp) +{ + unsigned int i; + int error; + + /* Find a cached inode, or go get another batch. */ + for (i = 0; i < XFS_INODES_PER_CHUNK; i++) { + if (iscan->__inodes[i]) + goto foundit; + } + + error = xchk_iscan_iter_batch(iscan); + if (error <= 0) + return error; + + ASSERT(iscan->__inodes[0] != NULL); + i = 0; + +foundit: + /* Give the caller our reference. */ + *ipp = iscan->__inodes[i]; + iscan->__inodes[i] = NULL; + return 1; +} + +/* Clean up an xfs_iscan_iter call by dropping any inodes that we still hold. */ +void +xchk_iscan_iter_finish( + struct xchk_iscan *iscan) +{ + struct xfs_scrub *sc = iscan->sc; + unsigned int i; + + for (i = 0; i < XFS_INODES_PER_CHUNK; i++) { + if (iscan->__inodes[i]) { + xchk_irele(sc, iscan->__inodes[i]); + iscan->__inodes[i] = NULL; + } + } +} + +/* Mark this inode scan finished and release resources. */ +void +xchk_iscan_teardown( + struct xchk_iscan *iscan) +{ + xchk_iscan_iter_finish(iscan); + xchk_iscan_finish(iscan); + mutex_destroy(&iscan->lock); +} + +/* Pick an AG from which to start a scan. */ +static inline xfs_ino_t +xchk_iscan_rotor( + struct xfs_mount *mp) +{ + static atomic_t agi_rotor; + unsigned int r = atomic_inc_return(&agi_rotor) - 1; + + /* + * Rotoring *backwards* through the AGs, so we add one here before + * subtracting from the agcount to arrive at an AG number. + */ + r = (r % mp->m_sb.sb_agcount) + 1; + + return XFS_AGINO_TO_INO(mp, mp->m_sb.sb_agcount - r, 0); +} + +/* + * Set ourselves up to start an inode scan. If the @iget_timeout and + * @iget_retry_delay parameters are set, the scan will try to iget each inode + * for @iget_timeout milliseconds. If an iget call indicates that the inode is + * waiting to be inactivated, the CPU will relax for @iget_retry_delay + * milliseconds after pushing the inactivation workers. + */ +void +xchk_iscan_start( + struct xfs_scrub *sc, + unsigned int iget_timeout, + unsigned int iget_retry_delay, + struct xchk_iscan *iscan) +{ + xfs_ino_t start_ino; + + start_ino = xchk_iscan_rotor(sc->mp); + + iscan->__batch_ino = NULLFSINO; + iscan->__skipped_inomask = 0; + + iscan->sc = sc; + clear_bit(XCHK_ISCAN_OPSTATE_ABORTED, &iscan->__opstate); + iscan->iget_timeout = iget_timeout; + iscan->iget_retry_delay = iget_retry_delay; + iscan->__visited_ino = start_ino; + iscan->cursor_ino = start_ino; + iscan->scan_start_ino = start_ino; + mutex_init(&iscan->lock); + memset(iscan->__inodes, 0, sizeof(iscan->__inodes)); + + trace_xchk_iscan_start(iscan, start_ino); +} + +/* + * Mark this inode as having been visited. Callers must hold a sufficiently + * exclusive lock on the inode to prevent concurrent modifications. + */ +void +xchk_iscan_mark_visited( + struct xchk_iscan *iscan, + struct xfs_inode *ip) +{ + mutex_lock(&iscan->lock); + iscan->__visited_ino = I_INO(ip); + trace_xchk_iscan_visit(iscan); + mutex_unlock(&iscan->lock); +} + +/* + * Did we skip this inode because it wasn't allocated when we loaded the batch? + * If so, it is newly allocated and will not be scanned. All live updates to + * this inode must be passed to the caller to maintain scan correctness. + */ +static inline bool +xchk_iscan_skipped( + const struct xchk_iscan *iscan, + xfs_ino_t ino) +{ + if (iscan->__batch_ino == NULLFSINO) + return false; + if (ino < iscan->__batch_ino) + return false; + if (ino >= iscan->__batch_ino + XFS_INODES_PER_CHUNK) + return false; + + return iscan->__skipped_inomask & (1ULL << (ino - iscan->__batch_ino)); +} + +/* + * Do we need a live update for this inode? This is true if the scanner thread + * has visited this inode and the scan hasn't been aborted due to errors. + * Callers must hold a sufficiently exclusive lock on the inode to prevent + * scanners from reading any inode metadata. + */ +bool +xchk_iscan_want_live_update( + struct xchk_iscan *iscan, + xfs_ino_t ino) +{ + bool ret = false; + + if (xchk_iscan_aborted(iscan)) + return false; + + mutex_lock(&iscan->lock); + + trace_xchk_iscan_want_live_update(iscan, ino); + + /* Scan is finished, caller should receive all updates. */ + if (iscan->__visited_ino == NULLFSINO) { + ret = true; + goto unlock; + } + + /* + * No inodes have been visited yet, so the visited cursor points at the + * start of the scan range. The caller should not receive any updates. + */ + if (iscan->scan_start_ino == iscan->__visited_ino) { + ret = false; + goto unlock; + } + + /* + * This inode was not allocated at the time of the iscan batch. + * The caller should receive all updates. + */ + if (xchk_iscan_skipped(iscan, ino)) { + ret = true; + goto unlock; + } + + /* + * The visited cursor hasn't yet wrapped around the end of the FS. If + * @ino is inside the starred range, the caller should receive updates: + * + * 0 ------------ S ************ V ------------ EOFS + */ + if (iscan->scan_start_ino <= iscan->__visited_ino) { + if (ino >= iscan->scan_start_ino && + ino <= iscan->__visited_ino) + ret = true; + + goto unlock; + } + + /* + * The visited cursor wrapped around the end of the FS. If @ino is + * inside the starred range, the caller should receive updates: + * + * 0 ************ V ------------ S ************ EOFS + */ + if (ino >= iscan->scan_start_ino || ino <= iscan->__visited_ino) + ret = true; + +unlock: + mutex_unlock(&iscan->lock); + return ret; +} diff --git a/fs/xfs/scrub/iscan.h b/fs/xfs/scrub/iscan.h new file mode 100644 index 000000000..f9f47fa01 --- /dev/null +++ b/fs/xfs/scrub/iscan.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2021-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_ISCAN_H__ +#define __XFS_SCRUB_ISCAN_H__ + +struct xchk_iscan { + struct xfs_scrub *sc; + + /* Lock to protect the scan cursor. */ + struct mutex lock; + + /* + * This is the first inode in the inumber address space that we + * examined. When the scan wraps around back to here, the scan is + * finished. + */ + xfs_ino_t scan_start_ino; + + /* This is the inode that will be examined next. */ + xfs_ino_t cursor_ino; + + /* If nonzero and non-NULL, skip this inode when scanning. */ + xfs_ino_t skip_ino; + + /* + * This is the last inode that we've successfully scanned, either + * because the caller scanned it, or we moved the cursor past an empty + * part of the inode address space. Scan callers should only use the + * xchk_iscan_visit function to modify this. + */ + xfs_ino_t __visited_ino; + + /* Operational state of the livescan. */ + unsigned long __opstate; + + /* Give up on iterating @cursor_ino if we can't iget it by this time. */ + unsigned long __iget_deadline; + + /* Amount of time (in ms) that we will try to iget an inode. */ + unsigned int iget_timeout; + + /* Wait this many ms to retry an iget. */ + unsigned int iget_retry_delay; + + /* + * The scan grabs batches of inodes and stashes them here before + * handing them out with _iter. Unallocated inodes are set in the + * mask so that all updates to that inode are selected for live + * update propagation. + */ + xfs_ino_t __batch_ino; + xfs_inofree_t __skipped_inomask; + struct xfs_inode *__inodes[XFS_INODES_PER_CHUNK]; +}; + +/* Set if the scan has been aborted due to some event in the fs. */ +#define XCHK_ISCAN_OPSTATE_ABORTED (1) + +/* Use trylock to acquire the AGI */ +#define XCHK_ISCAN_OPSTATE_TRYLOCK_AGI (2) + +static inline bool +xchk_iscan_aborted(const struct xchk_iscan *iscan) +{ + return test_bit(XCHK_ISCAN_OPSTATE_ABORTED, &iscan->__opstate); +} + +static inline void +xchk_iscan_abort(struct xchk_iscan *iscan) +{ + set_bit(XCHK_ISCAN_OPSTATE_ABORTED, &iscan->__opstate); +} + +static inline bool +xchk_iscan_agi_needs_trylock(const struct xchk_iscan *iscan) +{ + return test_bit(XCHK_ISCAN_OPSTATE_TRYLOCK_AGI, &iscan->__opstate); +} + +static inline void +xchk_iscan_set_agi_trylock(struct xchk_iscan *iscan) +{ + set_bit(XCHK_ISCAN_OPSTATE_TRYLOCK_AGI, &iscan->__opstate); +} + +void xchk_iscan_start(struct xfs_scrub *sc, unsigned int iget_timeout, + unsigned int iget_retry_delay, struct xchk_iscan *iscan); +void xchk_iscan_finish_early(struct xchk_iscan *iscan); +void xchk_iscan_teardown(struct xchk_iscan *iscan); + +int xchk_iscan_iter(struct xchk_iscan *iscan, struct xfs_inode **ipp); +void xchk_iscan_iter_finish(struct xchk_iscan *iscan); + +void xchk_iscan_mark_visited(struct xchk_iscan *iscan, struct xfs_inode *ip); +bool xchk_iscan_want_live_update(struct xchk_iscan *iscan, xfs_ino_t ino); + +#endif /* __XFS_SCRUB_ISCAN_H__ */ diff --git a/fs/xfs/scrub/listxattr.c b/fs/xfs/scrub/listxattr.c new file mode 100644 index 000000000..9b4cf3937 --- /dev/null +++ b/fs/xfs/scrub/listxattr.c @@ -0,0 +1,320 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2022-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_attr.h" +#include "xfs_attr_leaf.h" +#include "xfs_attr_sf.h" +#include "xfs_trans.h" +#include "scrub/scrub.h" +#include "scrub/bitmap.h" +#include "scrub/dab_bitmap.h" +#include "scrub/listxattr.h" + +/* Call a function for every entry in a shortform xattr structure. */ +STATIC int +xchk_xattr_walk_sf( + struct xfs_scrub *sc, + struct xfs_inode *ip, + xchk_xattr_fn attr_fn, + void *priv) +{ + struct xfs_attr_sf_hdr *hdr = ip->i_af.if_data; + struct xfs_attr_sf_entry *sfe; + unsigned int i; + int error; + + sfe = xfs_attr_sf_firstentry(hdr); + for (i = 0; i < hdr->count; i++) { + error = attr_fn(sc, ip, sfe->flags, sfe->nameval, sfe->namelen, + &sfe->nameval[sfe->namelen], sfe->valuelen, + priv); + if (error) + return error; + + sfe = xfs_attr_sf_nextentry(sfe); + } + + return 0; +} + +/* Call a function for every entry in this xattr leaf block. */ +STATIC int +xchk_xattr_walk_leaf_entries( + struct xfs_scrub *sc, + struct xfs_inode *ip, + xchk_xattr_fn attr_fn, + struct xfs_buf *bp, + void *priv) +{ + struct xfs_attr3_icleaf_hdr ichdr; + struct xfs_mount *mp = sc->mp; + struct xfs_attr_leafblock *leaf = bp->b_addr; + struct xfs_attr_leaf_entry *entry; + unsigned int i; + int error; + + xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, &ichdr, leaf); + entry = xfs_attr3_leaf_entryp(leaf); + + for (i = 0; i < ichdr.count; entry++, i++) { + void *value; + unsigned char *name; + unsigned int namelen, valuelen; + + if (entry->flags & XFS_ATTR_LOCAL) { + struct xfs_attr_leaf_name_local *name_loc; + + name_loc = xfs_attr3_leaf_name_local(leaf, i); + name = name_loc->nameval; + namelen = name_loc->namelen; + value = &name_loc->nameval[name_loc->namelen]; + valuelen = be16_to_cpu(name_loc->valuelen); + } else { + struct xfs_attr_leaf_name_remote *name_rmt; + + name_rmt = xfs_attr3_leaf_name_remote(leaf, i); + name = name_rmt->name; + namelen = name_rmt->namelen; + value = NULL; + valuelen = be32_to_cpu(name_rmt->valuelen); + } + + error = attr_fn(sc, ip, entry->flags, name, namelen, value, + valuelen, priv); + if (error) + return error; + + } + + return 0; +} + +/* + * Call a function for every entry in a leaf-format xattr structure. Avoid + * memory allocations for the loop detector since there's only one block. + */ +STATIC int +xchk_xattr_walk_leaf( + struct xfs_scrub *sc, + struct xfs_inode *ip, + xchk_xattr_fn attr_fn, + void *priv) +{ + struct xfs_buf *leaf_bp; + int error; + + error = xfs_attr3_leaf_read(sc->tp, ip, I_INO(ip), 0, &leaf_bp); + if (error) + return error; + + error = xchk_xattr_walk_leaf_entries(sc, ip, attr_fn, leaf_bp, priv); + xfs_trans_brelse(sc->tp, leaf_bp); + return error; +} + +/* Find the leftmost leaf in the xattr dabtree. */ +STATIC int +xchk_xattr_find_leftmost_leaf( + struct xfs_scrub *sc, + struct xfs_inode *ip, + struct xdab_bitmap *seen_dablks, + struct xfs_buf **leaf_bpp) +{ + struct xfs_da3_icnode_hdr nodehdr; + struct xfs_mount *mp = sc->mp; + struct xfs_trans *tp = sc->tp; + struct xfs_da_intnode *node; + struct xfs_da_node_entry *btree; + struct xfs_buf *bp; + xfs_failaddr_t fa; + xfs_dablk_t blkno = 0; + unsigned int expected_level = 0; + int error; + + for (;;) { + xfs_extlen_t len = 1; + uint16_t magic; + + /* Make sure we haven't seen this new block already. */ + if (xdab_bitmap_test(seen_dablks, blkno, &len)) + return -EFSCORRUPTED; + + error = xfs_da3_node_read(tp, ip, blkno, &bp, XFS_ATTR_FORK); + if (error) + return error; + + node = bp->b_addr; + magic = be16_to_cpu(node->hdr.info.magic); + if (magic == XFS_ATTR_LEAF_MAGIC || + magic == XFS_ATTR3_LEAF_MAGIC) + break; + + error = -EFSCORRUPTED; + if (magic != XFS_DA_NODE_MAGIC && + magic != XFS_DA3_NODE_MAGIC) + goto out_buf; + + fa = xfs_da3_node_header_check(bp, I_INO(ip)); + if (fa) + goto out_buf; + + xfs_da3_node_hdr_from_disk(mp, &nodehdr, node); + + if (nodehdr.count == 0 || nodehdr.level >= XFS_DA_NODE_MAXDEPTH) + goto out_buf; + + /* Check the level from the root node. */ + if (blkno == 0) + expected_level = nodehdr.level - 1; + else if (expected_level != nodehdr.level) + goto out_buf; + else + expected_level--; + + /* Remember that we've seen this node. */ + error = xdab_bitmap_set(seen_dablks, blkno, 1); + if (error) + goto out_buf; + + /* Find the next level towards the leaves of the dabtree. */ + btree = nodehdr.btree; + blkno = be32_to_cpu(btree->before); + xfs_trans_brelse(tp, bp); + } + + error = -EFSCORRUPTED; + fa = xfs_attr3_leaf_header_check(bp, I_INO(ip)); + if (fa) + goto out_buf; + + if (expected_level != 0) + goto out_buf; + + /* Remember that we've seen this leaf. */ + error = xdab_bitmap_set(seen_dablks, blkno, 1); + if (error) + goto out_buf; + + *leaf_bpp = bp; + return 0; + +out_buf: + xfs_trans_brelse(tp, bp); + return error; +} + +/* Call a function for every entry in a node-format xattr structure. */ +STATIC int +xchk_xattr_walk_node( + struct xfs_scrub *sc, + struct xfs_inode *ip, + xchk_xattr_fn attr_fn, + xchk_xattrleaf_fn leaf_fn, + void *priv) +{ + struct xfs_attr3_icleaf_hdr leafhdr; + struct xdab_bitmap seen_dablks; + struct xfs_mount *mp = sc->mp; + struct xfs_attr_leafblock *leaf; + struct xfs_buf *leaf_bp; + int error; + + xdab_bitmap_init(&seen_dablks); + + error = xchk_xattr_find_leftmost_leaf(sc, ip, &seen_dablks, &leaf_bp); + if (error) + goto out_bitmap; + + for (;;) { + xfs_extlen_t len; + + error = xchk_xattr_walk_leaf_entries(sc, ip, attr_fn, leaf_bp, + priv); + if (error) + goto out_leaf; + + /* Find the right sibling of this leaf block. */ + leaf = leaf_bp->b_addr; + xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, &leafhdr, leaf); + if (leafhdr.forw == 0) + goto out_leaf; + + xfs_trans_brelse(sc->tp, leaf_bp); + + if (leaf_fn) { + error = leaf_fn(sc, priv); + if (error) + goto out_bitmap; + } + + /* Make sure we haven't seen this new leaf already. */ + len = 1; + if (xdab_bitmap_test(&seen_dablks, leafhdr.forw, &len)) { + error = -EFSCORRUPTED; + goto out_bitmap; + } + + error = xfs_attr3_leaf_read(sc->tp, ip, I_INO(ip), leafhdr.forw, + &leaf_bp); + if (error) + goto out_bitmap; + + /* Remember that we've seen this new leaf. */ + error = xdab_bitmap_set(&seen_dablks, leafhdr.forw, 1); + if (error) + goto out_leaf; + } + +out_leaf: + xfs_trans_brelse(sc->tp, leaf_bp); +out_bitmap: + xdab_bitmap_destroy(&seen_dablks); + return error; +} + +/* + * Call a function for every extended attribute in a file. + * + * Callers must hold the ILOCK. No validation or cursor restarts allowed. + * Returns -EFSCORRUPTED on any problem, including loops in the dabtree. + */ +int +xchk_xattr_walk( + struct xfs_scrub *sc, + struct xfs_inode *ip, + xchk_xattr_fn attr_fn, + xchk_xattrleaf_fn leaf_fn, + void *priv) +{ + int error; + + xfs_assert_ilocked(ip, XFS_ILOCK_SHARED | XFS_ILOCK_EXCL); + + if (!xfs_inode_hasattr(ip)) + return 0; + + if (ip->i_af.if_format == XFS_DINODE_FMT_LOCAL) + return xchk_xattr_walk_sf(sc, ip, attr_fn, priv); + + /* attr functions require that the attr fork is loaded */ + error = xfs_iread_extents(sc->tp, ip, XFS_ATTR_FORK); + if (error) + return error; + + if (xfs_attr_is_leaf(ip)) + return xchk_xattr_walk_leaf(sc, ip, attr_fn, priv); + + return xchk_xattr_walk_node(sc, ip, attr_fn, leaf_fn, priv); +} diff --git a/fs/xfs/scrub/listxattr.h b/fs/xfs/scrub/listxattr.h new file mode 100644 index 000000000..703cfb7b1 --- /dev/null +++ b/fs/xfs/scrub/listxattr.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2022-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_LISTXATTR_H__ +#define __XFS_SCRUB_LISTXATTR_H__ + +typedef int (*xchk_xattr_fn)(struct xfs_scrub *sc, struct xfs_inode *ip, + unsigned int attr_flags, const unsigned char *name, + unsigned int namelen, const void *value, unsigned int valuelen, + void *priv); + +typedef int (*xchk_xattrleaf_fn)(struct xfs_scrub *sc, void *priv); + +int xchk_xattr_walk(struct xfs_scrub *sc, struct xfs_inode *ip, + xchk_xattr_fn attr_fn, xchk_xattrleaf_fn leaf_fn, void *priv); + +#endif /* __XFS_SCRUB_LISTXATTR_H__ */ diff --git a/fs/xfs/scrub/metapath.c b/fs/xfs/scrub/metapath.c new file mode 100644 index 000000000..ff1ff762b --- /dev/null +++ b/fs/xfs/scrub/metapath.c @@ -0,0 +1,677 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2023-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_inode.h" +#include "xfs_metafile.h" +#include "xfs_quota.h" +#include "xfs_qm.h" +#include "xfs_dir2.h" +#include "xfs_parent.h" +#include "xfs_bmap_btree.h" +#include "xfs_trans_space.h" +#include "xfs_attr.h" +#include "xfs_rtgroup.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_rtrefcount_btree.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/trace.h" +#include "scrub/readdir.h" +#include "scrub/repair.h" + +/* + * Metadata Directory Tree Paths + * ============================= + * + * A filesystem with metadir enabled expects to find metadata structures + * attached to files that are accessible by walking a path down the metadata + * directory tree. Given the metadir path and the incore inode storing the + * metadata, this scrubber ensures that the ondisk metadir path points to the + * ondisk inode represented by the incore inode. + */ + +struct xchk_metapath { + struct xfs_scrub *sc; + + /* Name for lookup */ + struct xfs_name xname; + + /* Directory update for repairs */ + struct xfs_dir_update du; + + /* Path down to this metadata file from the parent directory */ + const char *path; + + /* Directory parent of the metadata file. */ + struct xfs_inode *dp; + + /* Locks held on dp */ + unsigned int dp_ilock_flags; + + /* Transaction block reservations */ + unsigned int link_resblks; + unsigned int unlink_resblks; + + /* Parent pointer updates */ + struct xfs_parent_args link_ppargs; + struct xfs_parent_args unlink_ppargs; + + /* Scratchpads for removing links */ + struct xfs_da_args pptr_args; +}; + +/* Release resources tracked in the buffer. */ +static inline void +xchk_metapath_cleanup( + void *buf) +{ + struct xchk_metapath *mpath = buf; + + if (mpath->dp_ilock_flags) + xfs_iunlock(mpath->dp, mpath->dp_ilock_flags); + kfree_const(mpath->path); +} + +/* Set up a metadir path scan. @path must be dynamically allocated. */ +static inline int +xchk_setup_metapath_scan( + struct xfs_scrub *sc, + struct xfs_inode *dp, + const char *path, + struct xfs_inode *ip) +{ + struct xchk_metapath *mpath; + int error; + + if (!path) + return -ENOMEM; + + error = xchk_install_live_inode(sc, ip); + if (error) { + kfree_const(path); + return error; + } + + mpath = kzalloc_obj(struct xchk_metapath, XCHK_GFP_FLAGS); + if (!mpath) { + kfree_const(path); + return -ENOMEM; + } + + mpath->sc = sc; + sc->buf = mpath; + sc->buf_cleanup = xchk_metapath_cleanup; + + mpath->dp = dp; + mpath->path = path; /* path is now owned by mpath */ + + mpath->xname.name = mpath->path; + mpath->xname.len = strlen(mpath->path); + mpath->xname.type = xfs_mode_to_ftype(VFS_I(ip)->i_mode); + + return 0; +} + +#ifdef CONFIG_XFS_RT +/* Scan the /rtgroups directory itself. */ +static int +xchk_setup_metapath_rtdir( + struct xfs_scrub *sc) +{ + if (!sc->mp->m_rtdirip) + return -ENOENT; + + return xchk_setup_metapath_scan(sc, sc->mp->m_metadirip, + kstrdup_const("rtgroups", GFP_KERNEL), sc->mp->m_rtdirip); +} + +/* Scan a rtgroup inode under the /rtgroups directory. */ +static int +xchk_setup_metapath_rtginode( + struct xfs_scrub *sc, + enum xfs_rtg_inodes type) +{ + struct xfs_rtgroup *rtg; + struct xfs_inode *ip; + int error; + + rtg = xfs_rtgroup_get(sc->mp, sc->sm->sm_agno); + if (!rtg) + return -ENOENT; + + ip = rtg->rtg_inodes[type]; + if (!ip) { + error = -ENOENT; + goto out_put_rtg; + } + + error = xchk_setup_metapath_scan(sc, sc->mp->m_rtdirip, + xfs_rtginode_path(rtg_rgno(rtg), type), ip); + +out_put_rtg: + xfs_rtgroup_put(rtg); + return error; +} +#else +# define xchk_setup_metapath_rtdir(...) (-ENOENT) +# define xchk_setup_metapath_rtginode(...) (-ENOENT) +#endif /* CONFIG_XFS_RT */ + +#ifdef CONFIG_XFS_QUOTA +/* Scan the /quota directory itself. */ +static int +xchk_setup_metapath_quotadir( + struct xfs_scrub *sc) +{ + struct xfs_quotainfo *qi = sc->mp->m_quotainfo; + + if (!qi || !qi->qi_dirip) + return -ENOENT; + + return xchk_setup_metapath_scan(sc, sc->mp->m_metadirip, + kstrdup_const("quota", GFP_KERNEL), qi->qi_dirip); +} + +/* Scan a quota inode under the /quota directory. */ +static int +xchk_setup_metapath_dqinode( + struct xfs_scrub *sc, + xfs_dqtype_t type) +{ + struct xfs_quotainfo *qi = sc->mp->m_quotainfo; + struct xfs_inode *ip = NULL; + + if (!qi) + return -ENOENT; + + switch (type) { + case XFS_DQTYPE_USER: + ip = qi->qi_uquotaip; + break; + case XFS_DQTYPE_GROUP: + ip = qi->qi_gquotaip; + break; + case XFS_DQTYPE_PROJ: + ip = qi->qi_pquotaip; + break; + default: + ASSERT(0); + return -EINVAL; + } + if (!ip) + return -ENOENT; + + return xchk_setup_metapath_scan(sc, qi->qi_dirip, + kstrdup_const(xfs_dqinode_path(type), GFP_KERNEL), ip); +} +#else +# define xchk_setup_metapath_quotadir(...) (-ENOENT) +# define xchk_setup_metapath_dqinode(...) (-ENOENT) +#endif /* CONFIG_XFS_QUOTA */ + +int +xchk_setup_metapath( + struct xfs_scrub *sc) +{ + if (!xfs_has_metadir(sc->mp)) + return -ENOENT; + if (sc->sm->sm_gen) + return -EINVAL; + + switch (sc->sm->sm_ino) { + case XFS_SCRUB_METAPATH_PROBE: + /* Just probing, nothing else to do. */ + if (sc->sm->sm_agno) + return -EINVAL; + return 0; + case XFS_SCRUB_METAPATH_RTDIR: + return xchk_setup_metapath_rtdir(sc); + case XFS_SCRUB_METAPATH_RTBITMAP: + return xchk_setup_metapath_rtginode(sc, XFS_RTGI_BITMAP); + case XFS_SCRUB_METAPATH_RTSUMMARY: + return xchk_setup_metapath_rtginode(sc, XFS_RTGI_SUMMARY); + case XFS_SCRUB_METAPATH_QUOTADIR: + return xchk_setup_metapath_quotadir(sc); + case XFS_SCRUB_METAPATH_USRQUOTA: + return xchk_setup_metapath_dqinode(sc, XFS_DQTYPE_USER); + case XFS_SCRUB_METAPATH_GRPQUOTA: + return xchk_setup_metapath_dqinode(sc, XFS_DQTYPE_GROUP); + case XFS_SCRUB_METAPATH_PRJQUOTA: + return xchk_setup_metapath_dqinode(sc, XFS_DQTYPE_PROJ); + case XFS_SCRUB_METAPATH_RTRMAPBT: + return xchk_setup_metapath_rtginode(sc, XFS_RTGI_RMAP); + case XFS_SCRUB_METAPATH_RTREFCOUNTBT: + return xchk_setup_metapath_rtginode(sc, XFS_RTGI_REFCOUNT); + default: + return -ENOENT; + } +} + +/* + * Take the ILOCK on the metadata directory parent and child. We do not know + * that the metadata directory is not corrupt, so we lock the parent and try + * to lock the child. Returns 0 if successful, or -EINTR to abort the scrub. + */ +STATIC int +xchk_metapath_ilock_both( + struct xchk_metapath *mpath) +{ + struct xfs_scrub *sc = mpath->sc; + int error = 0; + + while (true) { + xfs_ilock(mpath->dp, XFS_ILOCK_EXCL); + if (xchk_ilock_nowait(sc, XFS_ILOCK_EXCL)) { + mpath->dp_ilock_flags |= XFS_ILOCK_EXCL; + return 0; + } + xfs_iunlock(mpath->dp, XFS_ILOCK_EXCL); + + if (xchk_should_terminate(sc, &error)) + return error; + + delay(1); + } + + ASSERT(0); + return -EINTR; +} + +/* Unlock parent and child inodes. */ +static inline void +xchk_metapath_iunlock( + struct xchk_metapath *mpath) +{ + struct xfs_scrub *sc = mpath->sc; + + xchk_iunlock(sc, XFS_ILOCK_EXCL); + + mpath->dp_ilock_flags &= ~XFS_ILOCK_EXCL; + xfs_iunlock(mpath->dp, XFS_ILOCK_EXCL); +} + +int +xchk_metapath( + struct xfs_scrub *sc) +{ + struct xchk_metapath *mpath = sc->buf; + xfs_ino_t ino = NULLFSINO; + int error; + + /* Just probing, nothing else to do. */ + if (sc->sm->sm_ino == XFS_SCRUB_METAPATH_PROBE) + return 0; + + /* Parent required to do anything else. */ + if (mpath->dp == NULL) { + xchk_ip_set_corrupt(sc, sc->ip); + return 0; + } + + xchk_trans_alloc_empty(sc); + + error = xchk_metapath_ilock_both(mpath); + if (error) + goto out_cancel; + + /* Make sure the parent dir has a dirent pointing to this file. */ + error = xchk_dir_lookup(sc, mpath->dp, &mpath->xname, &ino); + trace_xchk_metapath_lookup(sc, mpath->path, mpath->dp, ino); + if (error == -ENOENT) { + /* No directory entry at all */ + xchk_ip_set_corrupt(sc, sc->ip); + error = 0; + goto out_ilock; + } + if (!xchk_fblock_xref_process_error(sc, XFS_DATA_FORK, 0, &error)) + goto out_ilock; + if (ino != I_INO(sc->ip)) { + /* Pointing to wrong inode */ + xchk_ip_set_corrupt(sc, sc->ip); + } + +out_ilock: + xchk_metapath_iunlock(mpath); +out_cancel: + xchk_trans_cancel(sc); + return error; +} + +#ifdef CONFIG_XFS_ONLINE_REPAIR +/* Create the dirent represented by the final component of the path. */ +STATIC int +xrep_metapath_link( + struct xchk_metapath *mpath) +{ + struct xfs_scrub *sc = mpath->sc; + + mpath->du.dp = mpath->dp; + mpath->du.name = &mpath->xname; + mpath->du.ip = sc->ip; + + if (xfs_has_parent(sc->mp)) + mpath->du.ppargs = &mpath->link_ppargs; + else + mpath->du.ppargs = NULL; + + trace_xrep_metapath_link(sc, mpath->path, mpath->dp, I_INO(sc->ip)); + + return xfs_dir_add_child(sc->tp, mpath->link_resblks, &mpath->du); +} + +/* Remove the dirent at the final component of the path. */ +STATIC int +xrep_metapath_unlink( + struct xchk_metapath *mpath, + xfs_ino_t ino, + struct xfs_inode *ip) +{ + struct xfs_parent_rec rec; + struct xfs_scrub *sc = mpath->sc; + struct xfs_mount *mp = sc->mp; + int error; + + trace_xrep_metapath_unlink(sc, mpath->path, mpath->dp, ino); + + if (!ip) { + /* The child inode isn't allocated. Junk the dirent. */ + xfs_trans_log_inode(sc->tp, mpath->dp, XFS_ILOG_CORE); + return xfs_dir_removename(sc->tp, mpath->dp, &mpath->xname, + ino, mpath->unlink_resblks); + } + + mpath->du.dp = mpath->dp; + mpath->du.name = &mpath->xname; + mpath->du.ip = ip; + mpath->du.ppargs = NULL; + + /* Figure out if we're removing a parent pointer too. */ + if (xfs_has_parent(mp)) { + xfs_inode_to_parent_rec(&rec, ip); + error = xfs_parent_lookup(sc->tp, ip, &mpath->xname, &rec, + &mpath->pptr_args); + switch (error) { + case -ENOATTR: + break; + case 0: + mpath->du.ppargs = &mpath->unlink_ppargs; + break; + default: + return error; + } + } + + return xfs_dir_remove_child(sc->tp, mpath->unlink_resblks, &mpath->du); +} + +/* + * Try to create a dirent in @mpath->dp with the name @mpath->xname that points + * to @sc->ip. Returns: + * + * -EEXIST and an @alleged_child if the dirent that points to the wrong inode; + * 0 if there is now a dirent pointing to @sc->ip; or + * A negative errno on error. + */ +STATIC int +xrep_metapath_try_link( + struct xchk_metapath *mpath, + xfs_ino_t *alleged_child) +{ + struct xfs_scrub *sc = mpath->sc; + xfs_ino_t ino; + int error; + + /* Allocate transaction, lock inodes, join to transaction. */ + error = xchk_trans_alloc(sc, mpath->link_resblks); + if (error) + return error; + + error = xchk_metapath_ilock_both(mpath); + if (error) { + xchk_trans_cancel(sc); + return error; + } + xfs_trans_ijoin(sc->tp, mpath->dp, 0); + xfs_trans_ijoin(sc->tp, sc->ip, 0); + + error = xchk_dir_lookup(sc, mpath->dp, &mpath->xname, &ino); + trace_xrep_metapath_lookup(sc, mpath->path, mpath->dp, ino); + if (error == -ENOENT) { + /* + * There is no dirent in the directory. Create an entry + * pointing to @sc->ip. + */ + error = xrep_metapath_link(mpath); + if (error) + goto out_cancel; + + error = xrep_trans_commit(sc); + xchk_metapath_iunlock(mpath); + return error; + } + if (error) + goto out_cancel; + + if (ino == I_INO(sc->ip)) { + /* The dirent already points to @sc->ip; we're done. */ + error = 0; + goto out_cancel; + } + + /* + * The dirent points elsewhere; pass that back so that the caller + * can try to remove the dirent. + */ + *alleged_child = ino; + error = -EEXIST; + +out_cancel: + xchk_trans_cancel(sc); + xchk_metapath_iunlock(mpath); + return error; +} + +/* + * Take the ILOCK on the metadata directory parent and a bad child, if one is + * supplied. We do not know that the metadata directory is not corrupt, so we + * lock the parent and try to lock the child. Returns 0 if successful, or + * -EINTR to abort the repair. The lock state of @dp is not recorded in @mpath. + */ +STATIC int +xchk_metapath_ilock_parent_and_child( + struct xchk_metapath *mpath, + struct xfs_inode *ip) +{ + struct xfs_scrub *sc = mpath->sc; + int error = 0; + + while (true) { + xfs_ilock(mpath->dp, XFS_ILOCK_EXCL); + if (!ip || xfs_ilock_nowait(ip, XFS_ILOCK_EXCL)) + return 0; + xfs_iunlock(mpath->dp, XFS_ILOCK_EXCL); + + if (xchk_should_terminate(sc, &error)) + return error; + + delay(1); + } + + ASSERT(0); + return -EINTR; +} + +/* + * Try to remove a dirent in @mpath->dp with the name @mpath->xname that points + * to @alleged_child. Returns: + * + * 0 if there is no longer a dirent; + * -EEXIST if the dirent points to @sc->ip; + * -EAGAIN and an updated @alleged_child if the dirent points elsewhere; or + * A negative errno for any other error. + */ +STATIC int +xrep_metapath_try_unlink( + struct xchk_metapath *mpath, + xfs_ino_t *alleged_child) +{ + struct xfs_scrub *sc = mpath->sc; + struct xfs_inode *ip = NULL; + xfs_ino_t ino; + int error; + + ASSERT(*alleged_child != I_INO(sc->ip)); + + trace_xrep_metapath_try_unlink(sc, mpath->path, mpath->dp, + *alleged_child); + + /* + * Allocate transaction, grab the alleged child inode, lock inodes, + * join to transaction. + */ + error = xchk_trans_alloc(sc, mpath->unlink_resblks); + if (error) + return error; + + error = xchk_iget(sc, *alleged_child, &ip); + if (error == -EINVAL || error == -ENOENT) { + /* inode number is bogus, junk the dirent */ + error = 0; + } + if (error) { + xchk_trans_cancel(sc); + return error; + } + + error = xchk_metapath_ilock_parent_and_child(mpath, ip); + if (error) { + xchk_trans_cancel(sc); + return error; + } + xfs_trans_ijoin(sc->tp, mpath->dp, 0); + if (ip) + xfs_trans_ijoin(sc->tp, ip, 0); + + error = xchk_dir_lookup(sc, mpath->dp, &mpath->xname, &ino); + trace_xrep_metapath_lookup(sc, mpath->path, mpath->dp, ino); + if (error == -ENOENT) { + /* + * There is no dirent in the directory anymore. We're ready to + * try the link operation again. + */ + error = 0; + goto out_cancel; + } + if (error) + goto out_cancel; + + if (ino == I_INO(sc->ip)) { + /* The dirent already points to @sc->ip; we're done. */ + error = -EEXIST; + goto out_cancel; + } + + /* + * The dirent does not point to the alleged child. Update the caller + * and signal that we want to be called again. + */ + if (ino != *alleged_child) { + *alleged_child = ino; + error = -EAGAIN; + goto out_cancel; + } + + /* Remove the link to the child. */ + error = xrep_metapath_unlink(mpath, ino, ip); + if (error) + goto out_cancel; + + error = xrep_trans_commit(sc); + goto out_unlock; + +out_cancel: + xchk_trans_cancel(sc); +out_unlock: + xfs_iunlock(mpath->dp, XFS_ILOCK_EXCL); + if (ip) { + xfs_iunlock(ip, XFS_ILOCK_EXCL); + xchk_irele(sc, ip); + } + return error; +} + +/* + * Make sure the metadata directory path points to the child being examined. + * + * Repair needs to be able to create a directory structure, create its own + * transactions, and take ILOCKs. This function /must/ be called after all + * other repairs have completed. + */ +int +xrep_metapath( + struct xfs_scrub *sc) +{ + struct xchk_metapath *mpath = sc->buf; + struct xfs_mount *mp = sc->mp; + int error = 0; + + /* Just probing, nothing to repair. */ + if (sc->sm->sm_ino == XFS_SCRUB_METAPATH_PROBE) + return 0; + + /* Parent required to do anything else. */ + if (mpath->dp == NULL) + return -EFSCORRUPTED; + + /* + * Make sure the child file actually has an attr fork to receive a new + * parent pointer if the fs has parent pointers. + */ + if (xfs_has_parent(mp)) { + error = xfs_attr_add_fork(sc->ip, + sizeof(struct xfs_attr_sf_hdr), 1); + if (error) + return error; + } + + /* Compute block reservation required to unlink and link a file. */ + mpath->unlink_resblks = xfs_remove_space_res(mp, MAXNAMELEN); + mpath->link_resblks = xfs_link_space_res(mp, MAXNAMELEN); + + do { + xfs_ino_t alleged_child; + + /* Re-establish the link, or tell us which inode to remove. */ + error = xrep_metapath_try_link(mpath, &alleged_child); + if (!error) + return 0; + if (error != -EEXIST) + return error; + + /* + * Remove an incorrect link to an alleged child, or tell us + * which inode to remove. + */ + do { + error = xrep_metapath_try_unlink(mpath, &alleged_child); + } while (error == -EAGAIN); + if (error == -EEXIST) { + /* Link established; we're done. */ + error = 0; + break; + } + } while (!error); + + return error; +} +#endif /* CONFIG_XFS_ONLINE_REPAIR */ diff --git a/fs/xfs/scrub/newbt.c b/fs/xfs/scrub/newbt.c new file mode 100644 index 000000000..c82f4631f --- /dev/null +++ b/fs/xfs/scrub/newbt.c @@ -0,0 +1,611 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2022-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_btree.h" +#include "xfs_btree_staging.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_sb.h" +#include "xfs_inode.h" +#include "xfs_alloc.h" +#include "xfs_rmap.h" +#include "xfs_ag.h" +#include "xfs_defer.h" +#include "xfs_metafile.h" +#include "xfs_quota.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/newbt.h" + +/* + * This is the maximum number of deferred extent freeing item extents (EFIs) + * that we'll attach to a transaction without rolling the transaction to avoid + * overrunning a tr_itruncate reservation. The newbt code should reserve + * exactly the correct number of blocks to rebuild the btree, so there should + * not be any excess blocks to free when committing a new btree. + */ +#define XREP_MAX_ITRUNCATE_EFIS (128) + +/* + * Estimate proper slack values for a btree that's being reloaded. + * + * Under most circumstances, we'll take whatever default loading value the + * btree bulk loading code calculates for us. However, there are some + * exceptions to this rule: + * + * (0) If someone turned one of the debug knobs. + * (1) If this is a per-AG btree and the AG has less than 10% space free. + * (2) If this is an inode btree and the FS has less than 10% space free. + + * In either case, format the new btree blocks almost completely full to + * minimize space usage. + */ +static void +xrep_newbt_estimate_slack( + struct xrep_newbt *xnr) +{ + struct xfs_scrub *sc = xnr->sc; + struct xfs_btree_bload *bload = &xnr->bload; + uint64_t free; + uint64_t sz; + + /* + * The xfs_globals values are set to -1 (i.e. take the bload defaults) + * unless someone has set them otherwise, so we just pull the values + * here. + */ + bload->leaf_slack = xfs_globals.bload_leaf_slack; + bload->node_slack = xfs_globals.bload_node_slack; + + if (sc->ops->type == ST_PERAG) { + free = sc->sa.pag->pagf_freeblks; + sz = xfs_ag_block_count(sc->mp, pag_agno(sc->sa.pag)); + } else { + free = xfs_sum_freecounter_raw(sc->mp, XC_FREE_BLOCKS); + sz = sc->mp->m_sb.sb_dblocks; + } + + /* No further changes if there's more than 10% free space left. */ + if (free >= div_u64(sz, 10)) + return; + + /* + * We're low on space; load the btrees as tightly as possible. Leave + * a couple of open slots in each btree block so that we don't end up + * splitting the btrees like crazy after a mount. + */ + if (bload->leaf_slack < 0) + bload->leaf_slack = 2; + if (bload->node_slack < 0) + bload->node_slack = 2; +} + +/* Initialize accounting resources for staging a new AG btree. */ +void +xrep_newbt_init_ag( + struct xrep_newbt *xnr, + struct xfs_scrub *sc, + const struct xfs_owner_info *oinfo, + xfs_fsblock_t alloc_hint, + enum xfs_ag_resv_type resv) +{ + memset(xnr, 0, sizeof(struct xrep_newbt)); + xnr->sc = sc; + xnr->oinfo = *oinfo; /* structure copy */ + xnr->alloc_hint = alloc_hint; + xnr->resv = resv; + INIT_LIST_HEAD(&xnr->resv_list); + xnr->bload.max_dirty = XFS_B_TO_FSBT(sc->mp, 256U << 10); /* 256K */ + xrep_newbt_estimate_slack(xnr); +} + +/* Initialize accounting resources for staging a new inode fork btree. */ +int +xrep_newbt_init_inode( + struct xrep_newbt *xnr, + struct xfs_scrub *sc, + int whichfork, + const struct xfs_owner_info *oinfo) +{ + struct xfs_ifork *ifp; + + ifp = kmem_cache_zalloc(xfs_ifork_cache, XCHK_GFP_FLAGS); + if (!ifp) + return -ENOMEM; + + xrep_newbt_init_ag(xnr, sc, oinfo, XFS_INODE_TO_FSB(sc->ip), + XFS_AG_RESV_NONE); + xnr->ifake.if_fork = ifp; + xnr->ifake.if_fork_size = xfs_inode_fork_size(sc->ip, whichfork); + return 0; +} + +/* + * Initialize accounting resources for staging a new metadata inode btree. + * If the metadata file has a space reservation, the caller must adjust that + * reservation when committing the new ondisk btree. + */ +int +xrep_newbt_init_metadir_inode( + struct xrep_newbt *xnr, + struct xfs_scrub *sc) +{ + struct xfs_owner_info oinfo; + struct xfs_ifork *ifp; + + ASSERT(xfs_is_metadir_inode(sc->ip)); + + xfs_rmap_inode_bmbt_owner(&oinfo, sc->ip, XFS_DATA_FORK); + + ifp = kmem_cache_zalloc(xfs_ifork_cache, XCHK_GFP_FLAGS); + if (!ifp) + return -ENOMEM; + + /* + * Allocate new metadir btree blocks with XFS_AG_RESV_NONE because the + * inode metadata space reservations can only account allocated space + * to the i_nblocks. We do not want to change the inode core fields + * until we're ready to commit the new tree, so we allocate the blocks + * as if they were regular file blocks. This exposes us to a higher + * risk of the repair being cancelled due to ENOSPC. + */ + xrep_newbt_init_ag(xnr, sc, &oinfo, XFS_INODE_TO_FSB(sc->ip), + XFS_AG_RESV_NONE); + xnr->ifake.if_fork = ifp; + xnr->ifake.if_fork_size = xfs_inode_fork_size(sc->ip, XFS_DATA_FORK); + return 0; +} + +/* + * Initialize accounting resources for staging a new btree. Callers are + * expected to add their own reservations (and clean them up) manually. + */ +void +xrep_newbt_init_bare( + struct xrep_newbt *xnr, + struct xfs_scrub *sc) +{ + xrep_newbt_init_ag(xnr, sc, &XFS_RMAP_OINFO_ANY_OWNER, NULLFSBLOCK, + XFS_AG_RESV_NONE); +} + +/* + * Designate specific blocks to be used to build our new btree. @pag must be + * a passive reference. + */ +STATIC int +xrep_newbt_add_blocks( + struct xrep_newbt *xnr, + struct xfs_perag *pag, + const struct xfs_alloc_arg *args) +{ + struct xfs_mount *mp = xnr->sc->mp; + struct xrep_newbt_resv *resv; + int error; + + resv = kmalloc_obj(struct xrep_newbt_resv, XCHK_GFP_FLAGS); + if (!resv) + return -ENOMEM; + + INIT_LIST_HEAD(&resv->list); + resv->agbno = XFS_FSB_TO_AGBNO(mp, args->fsbno); + resv->len = args->len; + resv->used = 0; + resv->pag = xfs_perag_hold(pag); + + if (args->tp) { + ASSERT(xnr->oinfo.oi_offset == 0); + + error = xfs_alloc_schedule_autoreap(args, + XFS_FREE_EXTENT_SKIP_DISCARD, &resv->autoreap); + if (error) + goto out_pag; + } + + list_add_tail(&resv->list, &xnr->resv_list); + return 0; +out_pag: + xfs_perag_put(resv->pag); + kfree(resv); + return error; +} + +/* + * Add an extent to the new btree reservation pool. Callers are required to + * reap this reservation manually if the repair is cancelled. @pag must be a + * passive reference. + */ +int +xrep_newbt_add_extent( + struct xrep_newbt *xnr, + struct xfs_perag *pag, + xfs_agblock_t agbno, + xfs_extlen_t len) +{ + struct xfs_alloc_arg args = { + .tp = NULL, /* no autoreap */ + .oinfo = xnr->oinfo, + .fsbno = xfs_agbno_to_fsb(pag, agbno), + .len = len, + .resv = xnr->resv, + }; + + return xrep_newbt_add_blocks(xnr, pag, &args); +} + +/* Don't let our allocation hint take us beyond this AG */ +static inline void +xrep_newbt_validate_ag_alloc_hint( + struct xrep_newbt *xnr) +{ + struct xfs_scrub *sc = xnr->sc; + xfs_agnumber_t agno = XFS_FSB_TO_AGNO(sc->mp, xnr->alloc_hint); + + if (agno == pag_agno(sc->sa.pag) && + xfs_verify_fsbno(sc->mp, xnr->alloc_hint)) + return; + + xnr->alloc_hint = + xfs_agbno_to_fsb(sc->sa.pag, XFS_AGFL_BLOCK(sc->mp) + 1); +} + +/* Allocate disk space for a new per-AG btree. */ +STATIC int +xrep_newbt_alloc_ag_blocks( + struct xrep_newbt *xnr, + uint64_t nr_blocks) +{ + struct xfs_scrub *sc = xnr->sc; + struct xfs_mount *mp = sc->mp; + int error = 0; + + ASSERT(sc->sa.pag != NULL); + ASSERT(xnr->resv != XFS_AG_RESV_METAFILE); + + while (nr_blocks > 0) { + struct xfs_alloc_arg args = { + .tp = sc->tp, + .mp = mp, + .oinfo = xnr->oinfo, + .minlen = 1, + .maxlen = nr_blocks, + .prod = 1, + .resv = xnr->resv, + }; + xfs_agnumber_t agno; + + xrep_newbt_validate_ag_alloc_hint(xnr); + + if (xnr->alloc_vextent) + error = xnr->alloc_vextent(sc, &args, xnr->alloc_hint); + else + error = xfs_alloc_vextent_near_bno(&args, + xnr->alloc_hint); + if (error) + return error; + if (args.fsbno == NULLFSBLOCK) + return -ENOSPC; + + agno = XFS_FSB_TO_AGNO(mp, args.fsbno); + if (agno != pag_agno(sc->sa.pag)) { + ASSERT(agno == pag_agno(sc->sa.pag)); + return -EFSCORRUPTED; + } + + trace_xrep_newbt_alloc_ag_blocks(sc->sa.pag, + XFS_FSB_TO_AGBNO(mp, args.fsbno), args.len, + xnr->oinfo.oi_owner); + + error = xrep_newbt_add_blocks(xnr, sc->sa.pag, &args); + if (error) + return error; + + nr_blocks -= args.len; + xnr->alloc_hint = args.fsbno + args.len; + + error = xrep_defer_finish(sc); + if (error) + return error; + } + + return 0; +} + +/* Don't let our allocation hint take us beyond EOFS */ +static inline void +xrep_newbt_validate_file_alloc_hint( + struct xrep_newbt *xnr) +{ + struct xfs_scrub *sc = xnr->sc; + + if (xfs_verify_fsbno(sc->mp, xnr->alloc_hint)) + return; + + xnr->alloc_hint = XFS_AGB_TO_FSB(sc->mp, 0, XFS_AGFL_BLOCK(sc->mp) + 1); +} + +/* Allocate disk space for our new file-based btree. */ +STATIC int +xrep_newbt_alloc_file_blocks( + struct xrep_newbt *xnr, + uint64_t nr_blocks) +{ + struct xfs_scrub *sc = xnr->sc; + struct xfs_mount *mp = sc->mp; + int error = 0; + + ASSERT(xnr->resv != XFS_AG_RESV_METAFILE); + + while (nr_blocks > 0) { + struct xfs_alloc_arg args = { + .tp = sc->tp, + .mp = mp, + .oinfo = xnr->oinfo, + .minlen = 1, + .maxlen = nr_blocks, + .prod = 1, + .resv = xnr->resv, + }; + struct xfs_perag *pag; + xfs_agnumber_t agno; + + xrep_newbt_validate_file_alloc_hint(xnr); + + if (xnr->alloc_vextent) + error = xnr->alloc_vextent(sc, &args, xnr->alloc_hint); + else + error = xfs_alloc_vextent_start_ag(&args, + xnr->alloc_hint); + if (error) + return error; + if (args.fsbno == NULLFSBLOCK) + return -ENOSPC; + + agno = XFS_FSB_TO_AGNO(mp, args.fsbno); + + pag = xfs_perag_get(mp, agno); + if (!pag) { + ASSERT(0); + return -EFSCORRUPTED; + } + + trace_xrep_newbt_alloc_file_blocks(pag, + XFS_FSB_TO_AGBNO(mp, args.fsbno), args.len, + xnr->oinfo.oi_owner); + + error = xrep_newbt_add_blocks(xnr, pag, &args); + xfs_perag_put(pag); + if (error) + return error; + + nr_blocks -= args.len; + xnr->alloc_hint = args.fsbno + args.len; + + error = xrep_defer_finish(sc); + if (error) + return error; + } + + return 0; +} + +/* Allocate disk space for our new btree. */ +int +xrep_newbt_alloc_blocks( + struct xrep_newbt *xnr, + uint64_t nr_blocks) +{ + if (xnr->sc->ip) + return xrep_newbt_alloc_file_blocks(xnr, nr_blocks); + return xrep_newbt_alloc_ag_blocks(xnr, nr_blocks); +} + +/* + * Free the unused part of a space extent that was reserved for a new ondisk + * structure. Returns the number of EFIs logged or a negative errno. + */ +STATIC int +xrep_newbt_free_extent( + struct xrep_newbt *xnr, + struct xrep_newbt_resv *resv, + bool btree_committed) +{ + struct xfs_scrub *sc = xnr->sc; + xfs_agblock_t free_agbno = resv->agbno; + xfs_extlen_t free_aglen = resv->len; + int error; + + if (!btree_committed || resv->used == 0) { + /* + * If we're not committing a new btree or we didn't use the + * space reservation, let the existing EFI free the entire + * space extent. + */ + trace_xrep_newbt_free_blocks(resv->pag, free_agbno, free_aglen, + xnr->oinfo.oi_owner); + xfs_alloc_commit_autoreap(sc->tp, &resv->autoreap); + return 1; + } + + /* + * We used space and committed the btree. Cancel the autoreap, remove + * the written blocks from the reservation, and possibly log a new EFI + * to free any unused reservation space. + */ + xfs_alloc_cancel_autoreap(sc->tp, &resv->autoreap); + free_agbno += resv->used; + free_aglen -= resv->used; + + if (free_aglen == 0) + return 0; + + trace_xrep_newbt_free_blocks(resv->pag, free_agbno, free_aglen, + xnr->oinfo.oi_owner); + + ASSERT(xnr->resv != XFS_AG_RESV_AGFL); + ASSERT(xnr->resv != XFS_AG_RESV_IGNORE); + + /* + * Use EFIs to free the reservations. This reduces the chance + * that we leak blocks if the system goes down. + */ + error = xfs_free_extent_later(sc->tp, + xfs_agbno_to_fsb(resv->pag, free_agbno), free_aglen, + &xnr->oinfo, xnr->resv, XFS_FREE_EXTENT_SKIP_DISCARD); + if (error) + return error; + + return 1; +} + +/* Free all the accounting info and disk space we reserved for a new btree. */ +STATIC int +xrep_newbt_free( + struct xrep_newbt *xnr, + bool btree_committed) +{ + struct xfs_scrub *sc = xnr->sc; + struct xrep_newbt_resv *resv, *n; + unsigned int freed = 0; + int error = 0; + + /* + * If the filesystem already went down, we can't free the blocks. Skip + * ahead to freeing the incore metadata because we can't fix anything. + */ + if (xfs_is_shutdown(sc->mp)) + goto junkit; + + list_for_each_entry_safe(resv, n, &xnr->resv_list, list) { + int ret; + + ret = xrep_newbt_free_extent(xnr, resv, btree_committed); + list_del(&resv->list); + xfs_perag_put(resv->pag); + kfree(resv); + if (ret < 0) { + error = ret; + goto junkit; + } + + freed += ret; + if (freed >= XREP_MAX_ITRUNCATE_EFIS) { + error = xrep_defer_finish(sc); + if (error) + goto junkit; + freed = 0; + } + } + + if (freed) + error = xrep_defer_finish(sc); + +junkit: + /* + * If we still have reservations attached to @newbt, cleanup must have + * failed and the filesystem is about to go down. Clean up the incore + * reservations and try to commit to freeing the space we used. + */ + list_for_each_entry_safe(resv, n, &xnr->resv_list, list) { + xfs_alloc_commit_autoreap(sc->tp, &resv->autoreap); + list_del(&resv->list); + xfs_perag_put(resv->pag); + kfree(resv); + } + + if (sc->ip) { + kmem_cache_free(xfs_ifork_cache, xnr->ifake.if_fork); + xnr->ifake.if_fork = NULL; + } + + return error; +} + +/* + * Free all the accounting info and unused disk space allocations after + * committing a new btree. + */ +int +xrep_newbt_commit( + struct xrep_newbt *xnr) +{ + return xrep_newbt_free(xnr, true); +} + +/* + * Free all the accounting info and all of the disk space we reserved for a new + * btree that we're not going to commit. We want to try to roll things back + * cleanly for things like ENOSPC midway through allocation. + */ +void +xrep_newbt_cancel( + struct xrep_newbt *xnr) +{ + xrep_newbt_free(xnr, false); +} + +/* Feed one of the reserved btree blocks to the bulk loader. */ +int +xrep_newbt_claim_block( + struct xfs_btree_cur *cur, + struct xrep_newbt *xnr, + union xfs_btree_ptr *ptr) +{ + struct xrep_newbt_resv *resv; + xfs_agblock_t agbno; + + /* + * The first item in the list should always have a free block unless + * we're completely out. + */ + resv = list_first_entry(&xnr->resv_list, struct xrep_newbt_resv, list); + if (resv->used == resv->len) + return -ENOSPC; + + /* + * Peel off a block from the start of the reservation. We allocate + * blocks in order to place blocks on disk in increasing record or key + * order. The block reservations tend to end up on the list in + * decreasing order, which hopefully results in leaf blocks ending up + * together. + */ + agbno = resv->agbno + resv->used; + resv->used++; + + /* If we used all the blocks in this reservation, move it to the end. */ + if (resv->used == resv->len) + list_move_tail(&resv->list, &xnr->resv_list); + + trace_xrep_newbt_claim_block(resv->pag, agbno, 1, xnr->oinfo.oi_owner); + + if (cur->bc_ops->ptr_len == XFS_BTREE_LONG_PTR_LEN) + ptr->l = cpu_to_be64(xfs_agbno_to_fsb(resv->pag, agbno)); + else + ptr->s = cpu_to_be32(agbno); + + /* Relog all the EFIs. */ + return xrep_defer_finish(xnr->sc); +} + +/* How many reserved blocks are unused? */ +unsigned int +xrep_newbt_unused_blocks( + struct xrep_newbt *xnr) +{ + struct xrep_newbt_resv *resv; + unsigned int unused = 0; + + list_for_each_entry(resv, &xnr->resv_list, list) + unused += resv->len - resv->used; + return unused; +} diff --git a/fs/xfs/scrub/newbt.h b/fs/xfs/scrub/newbt.h new file mode 100644 index 000000000..5ce785599 --- /dev/null +++ b/fs/xfs/scrub/newbt.h @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2022-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_NEWBT_H__ +#define __XFS_SCRUB_NEWBT_H__ + +struct xfs_alloc_arg; + +struct xrep_newbt_resv { + /* Link to list of extents that we've reserved. */ + struct list_head list; + + struct xfs_perag *pag; + + /* Auto-freeing this reservation if we don't commit. */ + struct xfs_alloc_autoreap autoreap; + + /* AG block of the extent we reserved. */ + xfs_agblock_t agbno; + + /* Length of the reservation. */ + xfs_extlen_t len; + + /* How much of this reservation has been used. */ + xfs_extlen_t used; +}; + +struct xrep_newbt { + struct xfs_scrub *sc; + + /* Custom allocation function, or NULL for xfs_alloc_vextent */ + int (*alloc_vextent)(struct xfs_scrub *sc, + struct xfs_alloc_arg *args, + xfs_fsblock_t alloc_hint); + + /* List of extents that we've reserved. */ + struct list_head resv_list; + + /* Fake root for new btree. */ + union { + struct xbtree_afakeroot afake; + struct xbtree_ifakeroot ifake; + }; + + /* rmap owner of these blocks */ + struct xfs_owner_info oinfo; + + /* btree geometry for the bulk loader */ + struct xfs_btree_bload bload; + + /* Allocation hint */ + xfs_fsblock_t alloc_hint; + + /* per-ag reservation type */ + enum xfs_ag_resv_type resv; +}; + +void xrep_newbt_init_bare(struct xrep_newbt *xnr, struct xfs_scrub *sc); +void xrep_newbt_init_ag(struct xrep_newbt *xnr, struct xfs_scrub *sc, + const struct xfs_owner_info *oinfo, xfs_fsblock_t alloc_hint, + enum xfs_ag_resv_type resv); +int xrep_newbt_init_inode(struct xrep_newbt *xnr, struct xfs_scrub *sc, + int whichfork, const struct xfs_owner_info *oinfo); +int xrep_newbt_init_metadir_inode(struct xrep_newbt *xnr, struct xfs_scrub *sc); +int xrep_newbt_alloc_blocks(struct xrep_newbt *xnr, uint64_t nr_blocks); +int xrep_newbt_add_extent(struct xrep_newbt *xnr, struct xfs_perag *pag, + xfs_agblock_t agbno, xfs_extlen_t len); +void xrep_newbt_cancel(struct xrep_newbt *xnr); +int xrep_newbt_commit(struct xrep_newbt *xnr); +int xrep_newbt_claim_block(struct xfs_btree_cur *cur, struct xrep_newbt *xnr, + union xfs_btree_ptr *ptr); +unsigned int xrep_newbt_unused_blocks(struct xrep_newbt *xnr); + +#endif /* __XFS_SCRUB_NEWBT_H__ */ diff --git a/fs/xfs/scrub/nlinks.c b/fs/xfs/scrub/nlinks.c new file mode 100644 index 000000000..bcedb8c3e --- /dev/null +++ b/fs/xfs/scrub/nlinks.c @@ -0,0 +1,1073 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2021-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_inode.h" +#include "xfs_icache.h" +#include "xfs_iwalk.h" +#include "xfs_ialloc.h" +#include "xfs_dir2.h" +#include "xfs_dir2_priv.h" +#include "xfs_ag.h" +#include "xfs_parent.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/repair.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/iscan.h" +#include "scrub/orphanage.h" +#include "scrub/nlinks.h" +#include "scrub/trace.h" +#include "scrub/readdir.h" +#include "scrub/tempfile.h" +#include "scrub/listxattr.h" + +/* + * Live Inode Link Count Checking + * ============================== + * + * Inode link counts are "summary" metadata, in the sense that they are + * computed as the number of directory entries referencing each file on the + * filesystem. Therefore, we compute the correct link counts by creating a + * shadow link count structure and walking every inode. + */ + +/* Set us up to scrub inode link counts. */ +int +xchk_setup_nlinks( + struct xfs_scrub *sc) +{ + struct xchk_nlink_ctrs *xnc; + int error; + + xchk_fsgates_enable(sc, XCHK_FSGATES_DIRENTS); + + if (xchk_could_repair(sc)) { + error = xrep_setup_nlinks(sc); + if (error) + return error; + } + + xnc = kvzalloc_obj(struct xchk_nlink_ctrs, XCHK_GFP_FLAGS); + if (!xnc) + return -ENOMEM; + xnc->xname.name = xnc->namebuf; + xnc->sc = sc; + sc->buf = xnc; + + return xchk_setup_fs(sc); +} + +/* + * Part 1: Collecting file link counts. For each file, we create a shadow link + * counting structure, then walk the entire directory tree, incrementing parent + * and child link counts for each directory entry seen. + * + * To avoid false corruption reports in part 2, any failure in this part must + * set the INCOMPLETE flag even when a negative errno is returned. This care + * must be taken with certain errno values (i.e. EFSBADCRC, EFSCORRUPTED, + * ECANCELED) that are absorbed into a scrub state flag update by + * xchk_*_process_error. Scrub and repair share the same incore data + * structures, so the INCOMPLETE flag is critical to prevent a repair based on + * insufficient information. + * + * Because we are scanning a live filesystem, it's possible that another thread + * will try to update the link counts for an inode that we've already scanned. + * This will cause our counts to be incorrect. Therefore, we hook all + * directory entry updates because that is when link count updates occur. By + * shadowing transaction updates in this manner, live nlink check can ensure by + * locking the inode and the shadow structure that its own copies are not out + * of date. Because the hook code runs in a different process context from the + * scrub code and the scrub state flags are not accessed atomically, failures + * in the hook code must abort the iscan and the scrubber must notice the + * aborted scan and set the incomplete flag. + * + * Note that we use jump labels and srcu notifier hooks to minimize the + * overhead when live nlinks is /not/ running. Locking order for nlink + * observations is inode ILOCK -> iscan_lock/xchk_nlink_ctrs lock. + */ + +/* + * Add a delta to an nlink counter, clamping the value to U32_MAX. Because + * XFS_MAXLINK < U32_MAX, the checking code will produce the correct results + * even if we lose some precision. + */ +static inline void +careful_add( + xfs_nlink_t *nlinkp, + int delta) +{ + uint64_t new_value = (uint64_t)(*nlinkp) + delta; + + BUILD_BUG_ON(XFS_MAXLINK > U32_MAX); + *nlinkp = min_t(uint64_t, new_value, U32_MAX); +} + +/* Update incore link count information. Caller must hold the nlinks lock. */ +STATIC int +xchk_nlinks_update_incore( + struct xchk_nlink_ctrs *xnc, + xfs_ino_t ino, + int parents_delta, + int backrefs_delta, + int children_delta) +{ + struct xchk_nlink nl; + int error; + + if (!xnc->nlinks) + return 0; + + error = xfarray_load_sparse(xnc->nlinks, ino, &nl); + if (error) + return error; + + trace_xchk_nlinks_update_incore(xnc->sc->mp, ino, &nl, parents_delta, + backrefs_delta, children_delta); + + careful_add(&nl.parents, parents_delta); + careful_add(&nl.backrefs, backrefs_delta); + careful_add(&nl.children, children_delta); + + nl.flags |= XCHK_NLINK_WRITTEN; + error = xfarray_store(xnc->nlinks, ino, &nl); + if (error == -EFBIG) { + /* + * EFBIG means we tried to store data at too high a byte offset + * in the sparse array. IOWs, we cannot complete the check and + * must notify userspace that the check was incomplete. + */ + error = -ECANCELED; + } + return error; +} + +/* + * Apply a link count change from the regular filesystem into our shadow link + * count structure based on a directory update in progress. + */ +STATIC int +xchk_nlinks_live_update( + struct notifier_block *nb, + unsigned long action, + void *data) +{ + struct xfs_dir_update_params *p = data; + struct xchk_nlink_ctrs *xnc; + int error; + + xnc = container_of(nb, struct xchk_nlink_ctrs, dhook.dirent_hook.nb); + + /* + * Ignore temporary directories being used to stage dir repairs, since + * we don't bump the link counts of the children. + */ + if (xrep_is_tempfile(p->dp)) + return NOTIFY_DONE; + + trace_xchk_nlinks_live_update(xnc->sc->mp, p->dp, action, I_INO(p->ip), + p->delta, p->name->name, p->name->len); + + /* + * If we've already scanned @dp, update the number of parents that link + * to @ip. If @ip is a subdirectory, update the number of child links + * going out of @dp. + */ + if (xchk_iscan_want_live_update(&xnc->collect_iscan, I_INO(p->dp))) { + mutex_lock(&xnc->lock); + error = xchk_nlinks_update_incore(xnc, I_INO(p->ip), p->delta, + 0, 0); + if (!error && S_ISDIR(VFS_IC(p->ip)->i_mode)) + error = xchk_nlinks_update_incore(xnc, I_INO(p->dp), 0, + 0, p->delta); + mutex_unlock(&xnc->lock); + if (error) + goto out_abort; + } + + /* + * If @ip is a subdirectory and we've already scanned it, update the + * number of backrefs pointing to @dp. + */ + if (S_ISDIR(VFS_IC(p->ip)->i_mode) && + xchk_iscan_want_live_update(&xnc->collect_iscan, I_INO(p->ip))) { + mutex_lock(&xnc->lock); + error = xchk_nlinks_update_incore(xnc, I_INO(p->dp), 0, + p->delta, 0); + mutex_unlock(&xnc->lock); + if (error) + goto out_abort; + } + + return NOTIFY_DONE; + +out_abort: + xchk_iscan_abort(&xnc->collect_iscan); + return NOTIFY_DONE; +} + +/* Bump the observed link count for the inode referenced by this entry. */ +STATIC int +xchk_nlinks_collect_dirent( + struct xfs_scrub *sc, + struct xfs_inode *dp, + xfs_dir2_dataptr_t dapos, + const struct xfs_name *name, + xfs_ino_t ino, + void *priv) +{ + struct xchk_nlink_ctrs *xnc = priv; + bool dot = false, dotdot = false; + int error; + + /* Does this name make sense? */ + if (name->len == 0 || !xfs_dir2_namecheck(name->name, name->len)) { + error = -ECANCELED; + goto out_abort; + } + + if (name->len == 1 && name->name[0] == '.') + dot = true; + else if (name->len == 2 && name->name[0] == '.' && + name->name[1] == '.') + dotdot = true; + + /* Don't accept a '.' entry that points somewhere else. */ + if (dot && ino != I_INO(dp)) { + error = -ECANCELED; + goto out_abort; + } + + /* Don't accept an invalid inode number. */ + if (!xfs_verify_dir_ino(sc->mp, ino)) { + error = -ECANCELED; + goto out_abort; + } + + /* Update the shadow link counts if we haven't already failed. */ + + if (xchk_iscan_aborted(&xnc->collect_iscan)) { + error = -ECANCELED; + goto out_incomplete; + } + + trace_xchk_nlinks_collect_dirent(sc->mp, dp, ino, name); + + mutex_lock(&xnc->lock); + + /* + * If this is a dotdot entry, it is a back link from dp to ino. How + * we handle this depends on whether or not dp is the root directory. + * + * The root directory is its own parent, so we pretend the dotdot entry + * establishes the "parent" of the root directory. Increment the + * number of parents of the root directory. + * + * Otherwise, increment the number of backrefs pointing back to ino. + * + * If the filesystem has parent pointers, we walk the pptrs to + * determine the backref count. + */ + if (dotdot) { + if (xchk_inode_is_dirtree_root(dp)) + error = xchk_nlinks_update_incore(xnc, ino, 1, 0, 0); + else if (!xfs_has_parent(sc->mp)) + error = xchk_nlinks_update_incore(xnc, ino, 0, 1, 0); + else + error = 0; + if (error) + goto out_unlock; + } + + /* + * If this dirent is a forward link from dp to ino, increment the + * number of parents linking into ino. + */ + if (!dot && !dotdot) { + error = xchk_nlinks_update_incore(xnc, ino, 1, 0, 0); + if (error) + goto out_unlock; + } + + /* + * If this dirent is a forward link to a subdirectory, increment the + * number of child links of dp. + */ + if (!dot && !dotdot && name->type == XFS_DIR3_FT_DIR) { + error = xchk_nlinks_update_incore(xnc, I_INO(dp), 0, 0, 1); + if (error) + goto out_unlock; + } + + mutex_unlock(&xnc->lock); + return 0; + +out_unlock: + mutex_unlock(&xnc->lock); +out_abort: + xchk_iscan_abort(&xnc->collect_iscan); +out_incomplete: + xchk_set_incomplete(sc); + return error; +} + +/* Bump the backref count for the inode referenced by this parent pointer. */ +STATIC int +xchk_nlinks_collect_pptr( + struct xfs_scrub *sc, + struct xfs_inode *ip, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + const void *value, + unsigned int valuelen, + void *priv) +{ + struct xfs_name xname = { + .name = name, + .len = namelen, + }; + struct xchk_nlink_ctrs *xnc = priv; + const struct xfs_parent_rec *pptr_rec = value; + xfs_ino_t parent_ino; + int error; + + /* Update the shadow link counts if we haven't already failed. */ + + if (xchk_iscan_aborted(&xnc->collect_iscan)) { + error = -ECANCELED; + goto out_incomplete; + } + + if (!(attr_flags & XFS_ATTR_PARENT)) + return 0; + + error = xfs_parent_from_attr(sc->mp, attr_flags, name, namelen, value, + valuelen, &parent_ino, NULL); + if (error) + return error; + + trace_xchk_nlinks_collect_pptr(sc->mp, ip, &xname, pptr_rec); + + mutex_lock(&xnc->lock); + + error = xchk_nlinks_update_incore(xnc, parent_ino, 0, 1, 0); + if (error) + goto out_unlock; + + mutex_unlock(&xnc->lock); + return 0; + +out_unlock: + mutex_unlock(&xnc->lock); + xchk_iscan_abort(&xnc->collect_iscan); +out_incomplete: + xchk_set_incomplete(sc); + return error; +} + +static uint +xchk_nlinks_ilock_dir( + struct xfs_inode *ip) +{ + uint lock_mode = XFS_ILOCK_SHARED; + + /* + * Take the IOLOCK so that other threads cannot start a directory + * update while we're scanning. + */ + xfs_ilock(ip, XFS_IOLOCK_SHARED); + + /* + * We're going to scan the directory entries, so we must be ready to + * pull the data fork mappings into memory if they aren't already. + */ + if (xfs_need_iread_extents(&ip->i_df)) + lock_mode = XFS_ILOCK_EXCL; + + /* + * We're going to scan the parent pointers, so we must be ready to + * pull the attr fork mappings into memory if they aren't already. + */ + if (xfs_has_parent(ip->i_mount) && xfs_inode_has_attr_fork(ip) && + xfs_need_iread_extents(&ip->i_af)) + lock_mode = XFS_ILOCK_EXCL; + + xfs_ilock(ip, lock_mode); + return lock_mode | XFS_IOLOCK_SHARED; +} + +/* Walk a directory to bump the observed link counts of the children. */ +STATIC int +xchk_nlinks_collect_dir( + struct xchk_nlink_ctrs *xnc, + struct xfs_inode *dp) +{ + struct xfs_scrub *sc = xnc->sc; + unsigned int lock_mode; + int error = 0; + + /* + * Ignore temporary directories being used to stage dir repairs, since + * we don't bump the link counts of the children. + */ + if (xrep_is_tempfile(dp)) + return 0; + + /* Prevent anyone from changing this directory while we walk it. */ + lock_mode = xchk_nlinks_ilock_dir(dp); + + /* + * The dotdot entry of an unlinked directory still points to the last + * parent, but the parent no longer links to this directory. Skip the + * directory to avoid overcounting. + */ + if (VFS_I(dp)->i_nlink == 0) + goto out_unlock; + + /* + * We cannot count file links if the directory looks as though it has + * been zapped by the inode record repair code. + */ + if (xchk_dir_looks_zapped(dp)) { + error = -EBUSY; + goto out_abort; + } + + error = xchk_dir_walk(sc, dp, xchk_nlinks_collect_dirent, xnc); + if (error == -ECANCELED) { + error = 0; + goto out_unlock; + } + if (error) + goto out_abort; + + /* Walk the parent pointers to get real backref counts. */ + if (xfs_has_parent(sc->mp)) { + /* + * If the extended attributes look as though they has been + * zapped by the inode record repair code, we cannot scan for + * parent pointers. + */ + if (xchk_pptr_looks_zapped(dp)) { + error = -EBUSY; + goto out_unlock; + } + + error = xchk_xattr_walk(sc, dp, xchk_nlinks_collect_pptr, NULL, + xnc); + if (error == -ECANCELED) { + error = 0; + goto out_unlock; + } + if (error) + goto out_abort; + } + + xchk_iscan_mark_visited(&xnc->collect_iscan, dp); + goto out_unlock; + +out_abort: + xchk_set_incomplete(sc); + xchk_iscan_abort(&xnc->collect_iscan); +out_unlock: + xfs_iunlock(dp, lock_mode); + return error; +} + +/* If this looks like a valid pointer, count it. */ +static inline int +xchk_nlinks_collect_metafile( + struct xchk_nlink_ctrs *xnc, + xfs_ino_t ino) +{ + if (!xfs_verify_ino(xnc->sc->mp, ino)) + return 0; + + trace_xchk_nlinks_collect_metafile(xnc->sc->mp, ino); + return xchk_nlinks_update_incore(xnc, ino, 1, 0, 0); +} + +/* Bump the link counts of metadata files rooted in the superblock. */ +STATIC int +xchk_nlinks_collect_metafiles( + struct xchk_nlink_ctrs *xnc) +{ + struct xfs_mount *mp = xnc->sc->mp; + int error = -ECANCELED; + + + if (xchk_iscan_aborted(&xnc->collect_iscan)) + goto out_incomplete; + + mutex_lock(&xnc->lock); + error = xchk_nlinks_collect_metafile(xnc, mp->m_sb.sb_rbmino); + if (error) + goto out_abort; + + error = xchk_nlinks_collect_metafile(xnc, mp->m_sb.sb_rsumino); + if (error) + goto out_abort; + + error = xchk_nlinks_collect_metafile(xnc, mp->m_sb.sb_uquotino); + if (error) + goto out_abort; + + error = xchk_nlinks_collect_metafile(xnc, mp->m_sb.sb_gquotino); + if (error) + goto out_abort; + + error = xchk_nlinks_collect_metafile(xnc, mp->m_sb.sb_pquotino); + if (error) + goto out_abort; + mutex_unlock(&xnc->lock); + + return 0; + +out_abort: + mutex_unlock(&xnc->lock); + xchk_iscan_abort(&xnc->collect_iscan); +out_incomplete: + xchk_set_incomplete(xnc->sc); + return error; +} + +/* Advance the collection scan cursor for this non-directory file. */ +static inline int +xchk_nlinks_collect_file( + struct xchk_nlink_ctrs *xnc, + struct xfs_inode *ip) +{ + xfs_ilock(ip, XFS_IOLOCK_SHARED); + xchk_iscan_mark_visited(&xnc->collect_iscan, ip); + xfs_iunlock(ip, XFS_IOLOCK_SHARED); + return 0; +} + +/* Walk all directories and count inode links. */ +STATIC int +xchk_nlinks_collect( + struct xchk_nlink_ctrs *xnc) +{ + struct xfs_scrub *sc = xnc->sc; + struct xfs_inode *ip; + int error; + + /* Count the rt and quota files that are rooted in the superblock. */ + error = xchk_nlinks_collect_metafiles(xnc); + if (error) + return error; + + /* + * Set up for a potentially lengthy filesystem scan by reducing our + * transaction resource usage for the duration. Specifically: + * + * Cancel the transaction to release the log grant space while we scan + * the filesystem. + * + * Create a new empty transaction to eliminate the possibility of the + * inode scan deadlocking on cyclical metadata. + * + * We pass the empty transaction to the file scanning function to avoid + * repeatedly cycling empty transactions. This can be done even though + * we take the IOLOCK to quiesce the file because empty transactions + * do not take sb_internal. + */ + xchk_trans_cancel(sc); + xchk_trans_alloc_empty(sc); + + while ((error = xchk_iscan_iter(&xnc->collect_iscan, &ip)) == 1) { + if (S_ISDIR(VFS_I(ip)->i_mode)) + error = xchk_nlinks_collect_dir(xnc, ip); + else + error = xchk_nlinks_collect_file(xnc, ip); + xchk_irele(sc, ip); + if (error) + break; + + if (xchk_should_terminate(sc, &error)) + break; + } + xchk_iscan_iter_finish(&xnc->collect_iscan); + if (error) { + xchk_set_incomplete(sc); + /* + * If we couldn't grab an inode that was busy with a state + * change, change the error code so that we exit to userspace + * as quickly as possible. + */ + if (error == -EBUSY) + return -ECANCELED; + return error; + } + + /* + * Switch out for a real transaction in preparation for building a new + * tree. + */ + xchk_trans_cancel(sc); + return xchk_setup_fs(sc); +} + +/* + * Part 2: Comparing file link counters. Walk each inode and compare the link + * counts against our shadow information; and then walk each shadow link count + * structure (that wasn't covered in the first part), comparing it against the + * file. + */ + +/* Read the observed link count for comparison with the actual inode. */ +STATIC int +xchk_nlinks_comparison_read( + struct xchk_nlink_ctrs *xnc, + xfs_ino_t ino, + struct xchk_nlink *obs) +{ + struct xchk_nlink nl; + int error; + + error = xfarray_load_sparse(xnc->nlinks, ino, &nl); + if (error) + return error; + + nl.flags |= (XCHK_NLINK_COMPARE_SCANNED | XCHK_NLINK_WRITTEN); + + error = xfarray_store(xnc->nlinks, ino, &nl); + if (error == -EFBIG) { + /* + * EFBIG means we tried to store data at too high a byte offset + * in the sparse array. IOWs, we cannot complete the check and + * must notify userspace that the check was incomplete. This + * shouldn't really happen outside of the collection phase. + */ + xchk_set_incomplete(xnc->sc); + return -ECANCELED; + } + if (error) + return error; + + /* Copy the counters, but do not expose the internal state. */ + obs->parents = nl.parents; + obs->backrefs = nl.backrefs; + obs->children = nl.children; + obs->flags = 0; + return 0; +} + +/* Check our link count against an inode. */ +STATIC int +xchk_nlinks_compare_inode( + struct xchk_nlink_ctrs *xnc, + struct xfs_inode *ip) +{ + struct xchk_nlink obs; + struct xfs_scrub *sc = xnc->sc; + uint64_t total_links; + unsigned int actual_nlink; + int error; + + /* + * Ignore temporary files being used to stage repairs, since we assume + * they're correct for non-directories, and the directory repair code + * doesn't bump the link counts for the children. + */ + if (xrep_is_tempfile(ip)) + return 0; + + xfs_ilock(ip, XFS_ILOCK_SHARED); + mutex_lock(&xnc->lock); + + if (xchk_iscan_aborted(&xnc->collect_iscan)) { + xchk_set_incomplete(xnc->sc); + error = -ECANCELED; + goto out_scanlock; + } + + error = xchk_nlinks_comparison_read(xnc, I_INO(ip), &obs); + if (error) + goto out_scanlock; + + /* + * If we don't have ftype to get an accurate count of the subdirectory + * entries in this directory, take advantage of the fact that on a + * consistent ftype=0 filesystem, the number of subdirectory + * backreferences (dotdot entries) pointing towards this directory + * should be equal to the number of subdirectory entries in the + * directory. + */ + if (!xfs_has_ftype(sc->mp) && S_ISDIR(VFS_I(ip)->i_mode)) + obs.children = obs.backrefs; + + total_links = xchk_nlink_total(ip, &obs); + actual_nlink = VFS_I(ip)->i_nlink; + + trace_xchk_nlinks_compare_inode(sc->mp, ip, &obs); + + /* + * If we found so many parents that we'd overflow i_nlink, we must flag + * this as a corruption. The VFS won't let users increase the link + * count, but it will let them decrease it. + */ + if (total_links > XFS_NLINK_PINNED) { + xchk_ip_set_corrupt(sc, ip); + goto out_corrupt; + } else if (total_links > XFS_MAXLINK) { + xchk_ino_set_warning(sc, I_INO(ip)); + } + + /* Link counts should match. */ + if (total_links != actual_nlink) { + xchk_ip_set_corrupt(sc, ip); + goto out_corrupt; + } + + if (S_ISDIR(VFS_I(ip)->i_mode) && actual_nlink > 0) { + /* + * The collection phase ignores directories with zero link + * count, so we ignore them here too. + * + * The number of subdirectory backreferences (dotdot entries) + * pointing towards this directory should be equal to the + * number of subdirectory entries in the directory. + */ + if (obs.children != obs.backrefs) + xchk_ip_xref_set_corrupt(sc, ip); + } else { + /* + * Non-directories and unlinked directories should not have + * back references. + */ + if (obs.backrefs != 0) { + xchk_ip_set_corrupt(sc, ip); + goto out_corrupt; + } + + /* + * Non-directories and unlinked directories should not have + * children. + */ + if (obs.children != 0) { + xchk_ip_set_corrupt(sc, ip); + goto out_corrupt; + } + } + + if (xchk_inode_is_dirtree_root(ip)) { + /* + * For the root of a directory tree, both the '.' and '..' + * entries should point to the root directory. The dotdot + * entry is counted as a parent of the root /and/ a backref of + * the root directory. + */ + if (obs.parents != 1) { + xchk_ip_set_corrupt(sc, ip); + goto out_corrupt; + } + } else if (actual_nlink > 0) { + /* + * Linked files that are not the root directory should have at + * least one parent. + */ + if (obs.parents == 0) { + xchk_ip_set_corrupt(sc, ip); + goto out_corrupt; + } + } + +out_corrupt: + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + error = -ECANCELED; +out_scanlock: + mutex_unlock(&xnc->lock); + xfs_iunlock(ip, XFS_ILOCK_SHARED); + return error; +} + +/* + * Check our link count against an inode that wasn't checked previously. This + * is intended to catch directories with dangling links, though we could be + * racing with inode allocation in other threads. + */ +STATIC int +xchk_nlinks_compare_inum( + struct xchk_nlink_ctrs *xnc, + xfs_ino_t ino) +{ + struct xchk_nlink obs; + struct xfs_mount *mp = xnc->sc->mp; + struct xfs_trans *tp = xnc->sc->tp; + struct xfs_buf *agi_bp; + struct xfs_inode *ip; + int error; + + /* + * The first iget failed, so try again with the variant that returns + * either an incore inode or the AGI buffer. If the function returns + * EINVAL/ENOENT, it should have passed us the AGI buffer so that we + * can guarantee that the inode won't be allocated while we check for + * a zero link count in the observed link count data. + */ + error = xchk_iget_agi(xnc->sc, ino, &agi_bp, &ip); + if (!error) { + /* Actually got an inode, so use the inode compare. */ + error = xchk_nlinks_compare_inode(xnc, ip); + xchk_irele(xnc->sc, ip); + return error; + } + if (error == -ENOENT || error == -EINVAL) { + /* No inode was found. Check for zero link count below. */ + error = 0; + } + if (error) + goto out_agi; + + /* Ensure that we have protected against inode allocation/freeing. */ + if (agi_bp == NULL) { + ASSERT(agi_bp != NULL); + xchk_set_incomplete(xnc->sc); + return -ECANCELED; + } + + if (xchk_iscan_aborted(&xnc->collect_iscan)) { + xchk_set_incomplete(xnc->sc); + error = -ECANCELED; + goto out_agi; + } + + mutex_lock(&xnc->lock); + error = xchk_nlinks_comparison_read(xnc, ino, &obs); + if (error) + goto out_scanlock; + + trace_xchk_nlinks_check_zero(mp, ino, &obs); + + /* + * If we can't grab the inode, the link count had better be zero. We + * still hold the AGI to prevent inode allocation/freeing. + */ + if (xchk_nlink_total(NULL, &obs) != 0) { + xchk_ino_set_corrupt(xnc->sc, ino); + error = -ECANCELED; + } + +out_scanlock: + mutex_unlock(&xnc->lock); +out_agi: + if (agi_bp) + xfs_trans_brelse(tp, agi_bp); + return error; +} + +/* + * Try to visit every inode in the filesystem to compare the link count. Move + * on if we can't grab an inode, since we'll revisit unchecked nlink records in + * the second part. + */ +static int +xchk_nlinks_compare_iter( + struct xchk_nlink_ctrs *xnc, + struct xfs_inode **ipp) +{ + int error; + + do { + error = xchk_iscan_iter(&xnc->compare_iscan, ipp); + } while (error == -EBUSY); + + return error; +} + +/* Compare the link counts we observed against the live information. */ +STATIC int +xchk_nlinks_compare( + struct xchk_nlink_ctrs *xnc) +{ + struct xchk_nlink nl; + struct xfs_scrub *sc = xnc->sc; + struct xfs_inode *ip; + xfarray_idx_t cur = XFARRAY_CURSOR_INIT; + int error; + + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return 0; + + /* + * Create a new empty transaction so that we can advance the iscan + * cursor without deadlocking if the inobt has a cycle and push on the + * inactivation workqueue. + */ + xchk_trans_cancel(sc); + xchk_trans_alloc_empty(sc); + + /* + * Use the inobt to walk all allocated inodes to compare the link + * counts. Inodes skipped by _compare_iter will be tried again in the + * next phase of the scan. + */ + xchk_iscan_start(sc, 0, 0, &xnc->compare_iscan); + while ((error = xchk_nlinks_compare_iter(xnc, &ip)) == 1) { + error = xchk_nlinks_compare_inode(xnc, ip); + xchk_iscan_mark_visited(&xnc->compare_iscan, ip); + xchk_irele(sc, ip); + if (error) + break; + + if (xchk_should_terminate(sc, &error)) + break; + } + xchk_iscan_iter_finish(&xnc->compare_iscan); + xchk_iscan_teardown(&xnc->compare_iscan); + if (error) + return error; + + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return 0; + + /* + * Walk all the non-null nlink observations that weren't checked in the + * previous step. + */ + mutex_lock(&xnc->lock); + while ((error = xfarray_iter(xnc->nlinks, &cur, &nl)) == 1) { + xfs_ino_t ino = cur - 1; + + if (nl.flags & XCHK_NLINK_COMPARE_SCANNED) + continue; + + mutex_unlock(&xnc->lock); + + error = xchk_nlinks_compare_inum(xnc, ino); + if (error) + return error; + + if (xchk_should_terminate(xnc->sc, &error)) + return error; + + mutex_lock(&xnc->lock); + } + mutex_unlock(&xnc->lock); + + return error; +} + +/* Tear down everything associated with a nlinks check. */ +static void +xchk_nlinks_teardown_scan( + void *priv) +{ + struct xchk_nlink_ctrs *xnc = priv; + + /* Discourage any hook functions that might be running. */ + xchk_iscan_abort(&xnc->collect_iscan); + + xfs_dir_hook_del(xnc->sc->mp, &xnc->dhook); + + if (xnc->nlinks) + xfarray_destroy(xnc->nlinks); + xnc->nlinks = NULL; + + xchk_iscan_teardown(&xnc->collect_iscan); + mutex_destroy(&xnc->lock); + xnc->sc = NULL; +} + +/* + * Scan all inodes in the entire filesystem to generate link count data. If + * the scan is successful, the counts will be left alive for a repair. If any + * error occurs, we'll tear everything down. + */ +STATIC int +xchk_nlinks_setup_scan( + struct xfs_scrub *sc, + struct xchk_nlink_ctrs *xnc) +{ + struct xfs_mount *mp = sc->mp; + unsigned long long max_inos; + xfs_agnumber_t last_agno = mp->m_sb.sb_agcount - 1; + xfs_agino_t first_agino, last_agino; + int error; + + mutex_init(&xnc->lock); + + /* Retry iget every tenth of a second for up to 30 seconds. */ + xchk_iscan_start(sc, 30000, 100, &xnc->collect_iscan); + + /* + * Set up enough space to store an nlink record for the highest + * possible inode number in this system. + */ + xfs_agino_range(mp, last_agno, &first_agino, &last_agino); + max_inos = XFS_AGINO_TO_INO(mp, last_agno, last_agino) + 1; + error = xfarray_create("file link counts", + min(XFS_MAXINUMBER + 1, max_inos), + sizeof(struct xchk_nlink), &xnc->nlinks); + if (error) + goto out_teardown; + + /* + * Hook into the directory entry code so that we can capture updates to + * file link counts. The hook only triggers for inodes that were + * already scanned, and the scanner thread takes each inode's ILOCK, + * which means that any in-progress inode updates will finish before we + * can scan the inode. + */ + ASSERT(sc->flags & XCHK_FSGATES_DIRENTS); + xfs_dir_hook_setup(&xnc->dhook, xchk_nlinks_live_update); + error = xfs_dir_hook_add(mp, &xnc->dhook); + if (error) + goto out_teardown; + + /* Use deferred cleanup to pass the inode link count data to repair. */ + sc->buf_cleanup = xchk_nlinks_teardown_scan; + return 0; + +out_teardown: + xchk_nlinks_teardown_scan(xnc); + return error; +} + +/* Scrub the link count of all inodes on the filesystem. */ +int +xchk_nlinks( + struct xfs_scrub *sc) +{ + struct xchk_nlink_ctrs *xnc = sc->buf; + int error = 0; + + /* Set ourselves up to check link counts on the live filesystem. */ + error = xchk_nlinks_setup_scan(sc, xnc); + if (error) + return error; + + /* Walk all inodes, picking up link count information. */ + error = xchk_nlinks_collect(xnc); + if (!xchk_xref_process_error(sc, 0, 0, &error)) + return error; + + /* Fail fast if we're not playing with a full dataset. */ + if (xchk_iscan_aborted(&xnc->collect_iscan)) + xchk_set_incomplete(sc); + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_INCOMPLETE) + return 0; + + /* Compare link counts. */ + error = xchk_nlinks_compare(xnc); + if (!xchk_xref_process_error(sc, 0, 0, &error)) + return error; + + /* Check one last time for an incomplete dataset. */ + if (xchk_iscan_aborted(&xnc->collect_iscan)) + xchk_set_incomplete(sc); + + return 0; +} diff --git a/fs/xfs/scrub/nlinks.h b/fs/xfs/scrub/nlinks.h new file mode 100644 index 000000000..b820712bf --- /dev/null +++ b/fs/xfs/scrub/nlinks.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2021-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_NLINKS_H__ +#define __XFS_SCRUB_NLINKS_H__ + +/* Live link count control structure. */ +struct xchk_nlink_ctrs { + struct xfs_scrub *sc; + + /* Shadow link count data and its mutex. */ + struct xfarray *nlinks; + struct mutex lock; + + /* + * The collection step uses a separate iscan context from the compare + * step because the collection iscan coordinates live updates to the + * observation data while this scanner is running. The compare iscan + * is secondary and can be reinitialized as needed. + */ + struct xchk_iscan collect_iscan; + struct xchk_iscan compare_iscan; + + /* + * Hook into directory updates so that we can receive live updates + * from other writer threads. + */ + struct xfs_dir_hook dhook; + + /* Orphanage reparenting request. */ + struct xrep_adoption adoption; + + /* Directory entry name, plus the trailing null. */ + struct xfs_name xname; + char namebuf[MAXNAMELEN]; +}; + +/* + * In-core link counts for a given inode in the filesystem. + * + * For an empty rootdir, the directory entries and the field to which they are + * accounted are as follows: + * + * Root directory: + * + * . points to self (root.child) + * .. points to self (root.parent) + * f1 points to a child file (f1.parent) + * d1 points to a child dir (d1.parent, root.child) + * + * Subdirectory d1: + * + * . points to self (d1.child) + * .. points to root dir (root.backref) + * f2 points to child file (f2.parent) + * f3 points to root.f1 (f1.parent) + * + * root.nlink == 3 (root.dot, root.dotdot, root.d1) + * d1.nlink == 2 (root.d1, d1.dot) + * f1.nlink == 2 (root.f1, d1.f3) + * f2.nlink == 1 (d1.f2) + */ +struct xchk_nlink { + /* Count of forward links from parent directories to this file. */ + xfs_nlink_t parents; + + /* + * Count of back links to this parent directory from child + * subdirectories. + */ + xfs_nlink_t backrefs; + + /* + * Count of forward links from this directory to all child files and + * the number of dot entries. Should be zero for non-directories. + */ + xfs_nlink_t children; + + /* Record state flags */ + unsigned int flags; +}; + +/* + * This incore link count has been written at least once. We never want to + * store an xchk_nlink that looks uninitialized. + */ +#define XCHK_NLINK_WRITTEN (1U << 0) + +/* Already checked this link count record. */ +#define XCHK_NLINK_COMPARE_SCANNED (1U << 1) + +/* Already made a repair with this link count record. */ +#define XREP_NLINK_DIRTY (1U << 2) + +/* Compute total link count, using large enough variables to detect overflow. */ +static inline uint64_t +xchk_nlink_total(struct xfs_inode *ip, const struct xchk_nlink *live) +{ + uint64_t ret = live->parents; + + /* Add one link count for the dot entry of any linked directory. */ + if (ip && S_ISDIR(VFS_I(ip)->i_mode) && VFS_I(ip)->i_nlink) + ret++; + return ret + live->children; +} + +#endif /* __XFS_SCRUB_NLINKS_H__ */ diff --git a/fs/xfs/scrub/nlinks_repair.c b/fs/xfs/scrub/nlinks_repair.c new file mode 100644 index 000000000..09e097e16 --- /dev/null +++ b/fs/xfs/scrub/nlinks_repair.c @@ -0,0 +1,354 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2021-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_inode.h" +#include "xfs_icache.h" +#include "xfs_bmap_util.h" +#include "xfs_iwalk.h" +#include "xfs_ialloc.h" +#include "xfs_sb.h" +#include "xfs_ag.h" +#include "xfs_dir2.h" +#include "xfs_parent.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/repair.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/iscan.h" +#include "scrub/orphanage.h" +#include "scrub/nlinks.h" +#include "scrub/trace.h" +#include "scrub/tempfile.h" + +/* + * Live Inode Link Count Repair + * ============================ + * + * Use the live inode link count information that we collected to replace the + * nlink values of the incore inodes. A scrub->repair cycle should have left + * the live data and hooks active, so this is safe so long as we make sure the + * inode is locked. + */ + +/* Set up to repair inode link counts. */ +int +xrep_setup_nlinks( + struct xfs_scrub *sc) +{ + return xrep_orphanage_try_create(sc); +} + +/* + * Inodes that aren't the root directory or the orphanage, have a nonzero link + * count, and no observed parents should be moved to the orphanage. + */ +static inline bool +xrep_nlinks_is_orphaned( + struct xfs_scrub *sc, + struct xfs_inode *ip, + unsigned int actual_nlink, + const struct xchk_nlink *obs) +{ + if (obs->parents != 0) + return false; + if (xchk_inode_is_dirtree_root(ip) || ip == sc->orphanage) + return false; + return actual_nlink != 0; +} + +/* Remove an inode from the unlinked list. */ +STATIC int +xrep_nlinks_iunlink_remove( + struct xfs_scrub *sc) +{ + struct xfs_perag *pag; + int error; + + pag = xfs_perag_get(sc->mp, XFS_INODE_TO_AGNO(sc->ip)); + error = xfs_iunlink_remove(sc->tp, pag, sc->ip); + xfs_perag_put(pag); + return error; +} + +/* + * Correct the link count of the given inode. Because we have to grab locks + * and resources in a certain order, it's possible that this will be a no-op. + */ +STATIC int +xrep_nlinks_repair_inode( + struct xchk_nlink_ctrs *xnc) +{ + struct xchk_nlink obs; + struct xfs_scrub *sc = xnc->sc; + struct xfs_mount *mp = sc->mp; + struct xfs_inode *ip = sc->ip; + uint64_t total_links; + uint64_t actual_nlink; + bool orphanage_available = false; + bool dirty = false; + int error; + + /* + * Ignore temporary files being used to stage repairs, since we assume + * they're correct for non-directories, and the directory repair code + * doesn't bump the link counts for the children. + */ + if (xrep_is_tempfile(ip)) + return 0; + + /* + * If the filesystem has an orphanage attached to the scrub context, + * prepare for a link count repair that could involve @ip being adopted + * by the lost+found. + */ + if (xrep_orphanage_can_adopt(sc)) { + error = xrep_orphanage_iolock_two(sc); + if (error) + return error; + + error = xrep_adoption_trans_alloc(sc, &xnc->adoption); + if (error) { + xchk_iunlock(sc, XFS_IOLOCK_EXCL); + xrep_orphanage_iunlock(sc, XFS_IOLOCK_EXCL); + } else { + orphanage_available = true; + } + } + + /* + * Either there is no orphanage or we couldn't allocate resources for + * that kind of update. Let's try again with only the resources we + * need for a simple link count update, since that's much more common. + */ + if (!orphanage_available) { + xchk_ilock(sc, XFS_IOLOCK_EXCL); + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_link, 0, 0, 0, + &sc->tp); + if (error) { + xchk_iunlock(sc, XFS_IOLOCK_EXCL); + return error; + } + + xchk_ilock(sc, XFS_ILOCK_EXCL); + xfs_trans_ijoin(sc->tp, ip, 0); + } + + mutex_lock(&xnc->lock); + + if (xchk_iscan_aborted(&xnc->collect_iscan)) { + error = -ECANCELED; + goto out_scanlock; + } + + error = xfarray_load_sparse(xnc->nlinks, I_INO(ip), &obs); + if (error) + goto out_scanlock; + + /* + * We're done accessing the shared scan data, so we can drop the lock. + * We still hold @ip's ILOCK, so its link count cannot change. + */ + mutex_unlock(&xnc->lock); + + total_links = xchk_nlink_total(ip, &obs); + actual_nlink = VFS_I(ip)->i_nlink; + + /* + * Non-directories cannot have directories pointing up to them. + * + * We previously set error to zero, but set it again because one static + * checker author fears that programmers will fail to maintain this + * invariant and built their tool to flag this as a security risk. A + * different tool author made their bot complain about the redundant + * store. This is a never-ending and stupid battle; both tools missed + * *actual bugs* elsewhere; and I no longer care. + */ + if (!S_ISDIR(VFS_I(ip)->i_mode) && obs.children != 0) { + trace_xrep_nlinks_unfixable_inode(mp, ip, &obs); + error = 0; + goto out_trans; + } + + /* + * Decide if we're going to move this file to the orphanage, and fix + * up the incore link counts if we are. + */ + if (orphanage_available && + xrep_nlinks_is_orphaned(sc, ip, actual_nlink, &obs)) { + /* Figure out what name we're going to use here. */ + error = xrep_adoption_compute_name(&xnc->adoption, &xnc->xname); + if (error) + goto out_trans; + + /* + * Reattach this file to the directory tree by moving it to + * the orphanage per the adoption parameters that we already + * computed. + */ + error = xrep_adoption_move(&xnc->adoption); + if (error) + goto out_trans; + + /* + * Re-read the link counts since the reparenting will have + * updated our scan info. + */ + mutex_lock(&xnc->lock); + error = xfarray_load_sparse(xnc->nlinks, I_INO(ip), &obs); + mutex_unlock(&xnc->lock); + if (error) + goto out_trans; + + total_links = xchk_nlink_total(ip, &obs); + actual_nlink = VFS_I(ip)->i_nlink; + dirty = true; + } + + /* + * If this inode is linked from the directory tree and on the unlinked + * list, remove it from the unlinked list. + */ + if (total_links > 0 && xfs_inode_on_unlinked_list(ip)) { + error = xrep_nlinks_iunlink_remove(sc); + if (error) + goto out_trans; + dirty = true; + } + + /* + * If this inode is not linked from the directory tree yet not on the + * unlinked list, put it on the unlinked list. + */ + if (total_links == 0 && !xfs_inode_on_unlinked_list(ip)) { + if (actual_nlink) + clear_nlink(VFS_I(ip)); + error = xfs_iunlink(sc->tp, ip); + if (error) { + if (actual_nlink) + set_nlink(VFS_I(ip), actual_nlink); + goto out_trans; + } + dirty = true; + } + + /* Commit the new link count if it changed. */ + if (total_links != actual_nlink) { + trace_xrep_nlinks_update_inode(mp, ip, &obs); + + set_nlink(VFS_I(ip), min_t(unsigned long long, total_links, + XFS_NLINK_PINNED)); + dirty = true; + } + + if (!dirty) { + error = 0; + goto out_trans; + } + + xfs_trans_log_inode(sc->tp, ip, XFS_ILOG_CORE); + + error = xrep_trans_commit(sc); + goto out_unlock; + +out_scanlock: + mutex_unlock(&xnc->lock); +out_trans: + xchk_trans_cancel(sc); +out_unlock: + xchk_iunlock(sc, XFS_ILOCK_EXCL); + if (orphanage_available) { + xrep_orphanage_iunlock(sc, XFS_ILOCK_EXCL); + xrep_orphanage_iunlock(sc, XFS_IOLOCK_EXCL); + } + xchk_iunlock(sc, XFS_IOLOCK_EXCL); + return error; +} + +/* + * Try to visit every inode in the filesystem for repairs. Move on if we can't + * grab an inode, since we're still making forward progress. + */ +static int +xrep_nlinks_iter( + struct xchk_nlink_ctrs *xnc, + struct xfs_inode **ipp) +{ + int error; + + do { + error = xchk_iscan_iter(&xnc->compare_iscan, ipp); + } while (error == -EBUSY); + + return error; +} + +/* Commit the new inode link counters. */ +int +xrep_nlinks( + struct xfs_scrub *sc) +{ + struct xchk_nlink_ctrs *xnc = sc->buf; + int error; + + /* + * We need ftype for an accurate count of the number of child + * subdirectory links. Child subdirectories with a back link (dotdot + * entry) but no forward link are moved to the orphanage, so we cannot + * repair the link count of the parent directory based on the back link + * count alone. Filesystems without ftype support are rare (old V4) so + * we just skip out here. + */ + if (!xfs_has_ftype(sc->mp)) + return -EOPNOTSUPP; + + /* + * Use the inobt to walk all allocated inodes to compare and fix the + * link counts. Retry iget every tenth of a second for up to 30 + * seconds -- even if repair misses a few inodes, we still try to fix + * as many of them as we can. + */ + xchk_iscan_start(sc, 30000, 100, &xnc->compare_iscan); + ASSERT(sc->ip == NULL); + + while ((error = xrep_nlinks_iter(xnc, &sc->ip)) == 1) { + /* + * Commit the scrub transaction so that we can create repair + * transactions with the correct reservations. + */ + xchk_trans_cancel(sc); + + error = xrep_nlinks_repair_inode(xnc); + xchk_iscan_mark_visited(&xnc->compare_iscan, sc->ip); + xchk_irele(sc, sc->ip); + sc->ip = NULL; + if (error) + break; + + if (xchk_should_terminate(sc, &error)) + break; + + /* + * Create a new empty transaction so that we can advance the + * iscan cursor without deadlocking if the inobt has a cycle. + * We can only push the inactivation workqueues with an empty + * transaction. + */ + xchk_trans_alloc_empty(sc); + } + xchk_iscan_iter_finish(&xnc->compare_iscan); + xchk_iscan_teardown(&xnc->compare_iscan); + + return error; +} diff --git a/fs/xfs/scrub/off_bitmap.h b/fs/xfs/scrub/off_bitmap.h new file mode 100644 index 000000000..0d3f9e6c1 --- /dev/null +++ b/fs/xfs/scrub/off_bitmap.h @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2022-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_OFF_BITMAP_H__ +#define __XFS_SCRUB_OFF_BITMAP_H__ + +/* Bitmaps, but for type-checked for xfs_fileoff_t */ + +struct xoff_bitmap { + struct xbitmap64 offbitmap; +}; + +static inline void xoff_bitmap_init(struct xoff_bitmap *bitmap) +{ + xbitmap64_init(&bitmap->offbitmap); +} + +static inline void xoff_bitmap_destroy(struct xoff_bitmap *bitmap) +{ + xbitmap64_destroy(&bitmap->offbitmap); +} + +static inline int xoff_bitmap_set(struct xoff_bitmap *bitmap, + xfs_fileoff_t off, xfs_filblks_t len) +{ + return xbitmap64_set(&bitmap->offbitmap, off, len); +} + +static inline int xoff_bitmap_walk(struct xoff_bitmap *bitmap, + xbitmap64_walk_fn fn, void *priv) +{ + return xbitmap64_walk(&bitmap->offbitmap, fn, priv); +} + +#endif /* __XFS_SCRUB_OFF_BITMAP_H__ */ diff --git a/fs/xfs/scrub/orphanage.c b/fs/xfs/scrub/orphanage.c new file mode 100644 index 000000000..3aca66869 --- /dev/null +++ b/fs/xfs/scrub/orphanage.c @@ -0,0 +1,631 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2021-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_inode.h" +#include "xfs_ialloc.h" +#include "xfs_quota.h" +#include "xfs_trans_space.h" +#include "xfs_dir2.h" +#include "xfs_icache.h" +#include "xfs_bmap.h" +#include "xfs_bmap_btree.h" +#include "xfs_parent.h" +#include "xfs_attr_sf.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/repair.h" +#include "scrub/trace.h" +#include "scrub/orphanage.h" +#include "scrub/readdir.h" + +#include + +/* + * The Orphanage + * ============= + * + * If the directory tree is damaged, children of that directory become + * inaccessible via that file path. If a child has no other parents, the file + * is said to be orphaned. xfs_repair fixes this situation by creating a + * orphanage directory (specifically, /lost+found) and creating a directory + * entry pointing to the orphaned file. + * + * Online repair follows this tactic by creating a root-owned /lost+found + * directory if one does not exist. If an orphan is found, it will move that + * files into orphanage. + */ + +/* Make the orphanage owned by root. */ +STATIC int +xrep_chown_orphanage( + struct xfs_scrub *sc, + struct xfs_inode *dp) +{ + struct xfs_trans *tp; + struct xfs_mount *mp = sc->mp; + struct xfs_dquot *udqp = NULL, *gdqp = NULL, *pdqp = NULL; + struct xfs_dquot *oldu = NULL, *oldg = NULL, *oldp = NULL; + struct inode *inode = VFS_I(dp); + int error; + + error = xfs_qm_vop_dqalloc(dp, GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, 0, + XFS_QMOPT_QUOTALL, &udqp, &gdqp, &pdqp); + if (error) + return error; + + error = xfs_trans_alloc_ichange(dp, udqp, gdqp, pdqp, true, &tp); + if (error) + goto out_dqrele; + + /* + * Always clear setuid/setgid/sticky on the orphanage since we don't + * normally want that functionality on this directory and xfs_repair + * doesn't create it this way either. Leave the other access bits + * unchanged. + */ + inode->i_mode &= ~(S_ISUID | S_ISGID | S_ISVTX); + + /* + * Change the ownerships and register quota modifications + * in the transaction. + */ + if (!uid_eq(inode->i_uid, GLOBAL_ROOT_UID)) { + if (XFS_IS_UQUOTA_ON(mp)) + oldu = xfs_qm_vop_chown(tp, dp, &dp->i_udquot, udqp); + inode->i_uid = GLOBAL_ROOT_UID; + } + if (!gid_eq(inode->i_gid, GLOBAL_ROOT_GID)) { + if (XFS_IS_GQUOTA_ON(mp)) + oldg = xfs_qm_vop_chown(tp, dp, &dp->i_gdquot, gdqp); + inode->i_gid = GLOBAL_ROOT_GID; + } + if (dp->i_projid != 0) { + if (XFS_IS_PQUOTA_ON(mp)) + oldp = xfs_qm_vop_chown(tp, dp, &dp->i_pdquot, pdqp); + dp->i_projid = 0; + } + + dp->i_diflags &= ~(XFS_DIFLAG_REALTIME | XFS_DIFLAG_RTINHERIT); + xfs_trans_log_inode(tp, dp, XFS_ILOG_CORE); + + XFS_STATS_INC(mp, xs_ig_attrchg); + + if (xfs_has_wsync(mp)) + xfs_trans_set_sync(tp); + error = xfs_trans_commit(tp); + + xfs_qm_dqrele(oldu); + xfs_qm_dqrele(oldg); + xfs_qm_dqrele(oldp); + +out_dqrele: + xfs_qm_dqrele(udqp); + xfs_qm_dqrele(gdqp); + xfs_qm_dqrele(pdqp); + return error; +} + +#define ORPHANAGE "lost+found" + +/* Create the orphanage directory, and set sc->orphanage to it. */ +int +xrep_orphanage_create( + struct xfs_scrub *sc) +{ + struct xfs_mount *mp = sc->mp; + struct dentry *root_dentry, *orphanage_dentry; + struct inode *root_inode = VFS_I(sc->mp->m_rootip); + struct inode *orphanage_inode; + int error; + + if (xfs_is_shutdown(mp)) + return -EIO; + if (xfs_is_readonly(mp)) { + sc->orphanage = NULL; + return 0; + } + + ASSERT(sc->tp == NULL); + ASSERT(sc->orphanage == NULL); + + /* Find the dentry for the root directory... */ + root_dentry = d_find_alias(root_inode); + if (!root_dentry) { + error = -EFSCORRUPTED; + goto out; + } + + /* ...which is a directory, right? */ + if (!d_is_dir(root_dentry)) { + error = -EFSCORRUPTED; + goto out_dput_root; + } + + /* Try to find the orphanage directory. */ + orphanage_dentry = start_creating_noperm(root_dentry, &QSTR(ORPHANAGE)); + if (IS_ERR(orphanage_dentry)) { + error = PTR_ERR(orphanage_dentry); + goto out_dput_root; + } + + /* + * Nothing found? Call mkdir to create the orphanage. Create the + * directory without other-user access because we're live and someone + * could have been relying partly on minimal access to a parent + * directory to control access to a file we put in here. + */ + if (d_really_is_negative(orphanage_dentry)) { + orphanage_dentry = vfs_mkdir(&nop_mnt_idmap, root_inode, + orphanage_dentry, 0750, NULL); + error = PTR_ERR(orphanage_dentry); + if (IS_ERR(orphanage_dentry)) + goto out_dput_orphanage; + } + + /* Not a directory? Bail out. */ + if (!d_is_dir(orphanage_dentry)) { + error = -ENOTDIR; + goto out_dput_orphanage; + } + + /* + * Grab a reference to the orphanage. This /should/ succeed since + * we hold the root directory locked and therefore nobody can delete + * the orphanage. + */ + orphanage_inode = igrab(d_inode(orphanage_dentry)); + if (!orphanage_inode) { + error = -ENOENT; + goto out_dput_orphanage; + } + + /* Make sure the orphanage is owned by root. */ + error = xrep_chown_orphanage(sc, XFS_I(orphanage_inode)); + if (error) + goto out_dput_orphanage; + + /* Stash the reference for later and bail out. */ + sc->orphanage = XFS_I(orphanage_inode); + sc->orphanage_ilock_flags = 0; + +out_dput_orphanage: + end_creating(orphanage_dentry); +out_dput_root: + dput(root_dentry); +out: + return error; +} + +void +xrep_orphanage_ilock( + struct xfs_scrub *sc, + unsigned int ilock_flags) +{ + sc->orphanage_ilock_flags |= ilock_flags; + xfs_ilock(sc->orphanage, ilock_flags); +} + +bool +xrep_orphanage_ilock_nowait( + struct xfs_scrub *sc, + unsigned int ilock_flags) +{ + if (xfs_ilock_nowait(sc->orphanage, ilock_flags)) { + sc->orphanage_ilock_flags |= ilock_flags; + return true; + } + + return false; +} + +void +xrep_orphanage_iunlock( + struct xfs_scrub *sc, + unsigned int ilock_flags) +{ + xfs_iunlock(sc->orphanage, ilock_flags); + sc->orphanage_ilock_flags &= ~ilock_flags; +} + +/* Grab the IOLOCK of the orphanage and sc->ip. */ +int +xrep_orphanage_iolock_two( + struct xfs_scrub *sc) +{ + int error = 0; + + while (true) { + if (xchk_should_terminate(sc, &error)) + return error; + + /* + * Normal XFS takes the IOLOCK before grabbing a transaction. + * Scrub holds a transaction, which means that we can't block + * on either IOLOCK. + */ + if (xrep_orphanage_ilock_nowait(sc, XFS_IOLOCK_EXCL)) { + if (xchk_ilock_nowait(sc, XFS_IOLOCK_EXCL)) + break; + xrep_orphanage_iunlock(sc, XFS_IOLOCK_EXCL); + } + delay(1); + } + + return 0; +} + +/* Release the orphanage. */ +void +xrep_orphanage_rele( + struct xfs_scrub *sc) +{ + if (!sc->orphanage) + return; + + if (sc->orphanage_ilock_flags) + xfs_iunlock(sc->orphanage, sc->orphanage_ilock_flags); + + xchk_irele(sc, sc->orphanage); + sc->orphanage = NULL; +} + +/* Adoption moves a file into /lost+found */ + +/* Can the orphanage adopt @sc->ip? */ +bool +xrep_orphanage_can_adopt( + struct xfs_scrub *sc) +{ + ASSERT(sc->ip != NULL); + + if (!sc->orphanage) + return false; + if (sc->ip == sc->orphanage) + return false; + if (xchk_inode_is_sb_rooted(sc->ip)) + return false; + if (xfs_is_internal_inode(sc->ip)) + return false; + return true; +} + +/* + * Create a new transaction to send a child to the orphanage. + * + * Allocate a new transaction with sufficient disk space to handle the + * adoption, take ILOCK_EXCL of the orphanage and sc->ip, joins them to the + * transaction, and reserve quota to reparent the latter. Caller must hold the + * IOLOCK of the orphanage and sc->ip. + */ +int +xrep_adoption_trans_alloc( + struct xfs_scrub *sc, + struct xrep_adoption *adopt) +{ + struct xfs_mount *mp = sc->mp; + unsigned int child_blkres = 0; + int error; + + ASSERT(sc->tp == NULL); + ASSERT(sc->ip != NULL); + ASSERT(sc->orphanage != NULL); + ASSERT(sc->ilock_flags & XFS_IOLOCK_EXCL); + ASSERT(sc->orphanage_ilock_flags & XFS_IOLOCK_EXCL); + ASSERT(!(sc->ilock_flags & (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL))); + ASSERT(!(sc->orphanage_ilock_flags & + (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL))); + + /* Compute the worst case space reservation that we need. */ + adopt->sc = sc; + adopt->orphanage_blkres = xfs_link_space_res(mp, MAXNAMELEN); + if (S_ISDIR(VFS_I(sc->ip)->i_mode)) + child_blkres = xfs_rename_space_res(mp, 0, false, + xfs_name_dotdot.len, false); + if (xfs_has_parent(mp)) + child_blkres += XFS_ADDAFORK_SPACE_RES(mp); + adopt->child_blkres = child_blkres; + + /* + * Allocate a transaction to link the child into the parent, along with + * enough disk space to handle expansion of both the orphanage and the + * dotdot entry of a child directory. + */ + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_link, + adopt->orphanage_blkres + adopt->child_blkres, 0, 0, + &sc->tp); + if (error) + return error; + + xfs_lock_two_inodes(sc->orphanage, XFS_ILOCK_EXCL, + sc->ip, XFS_ILOCK_EXCL); + sc->ilock_flags |= XFS_ILOCK_EXCL; + sc->orphanage_ilock_flags |= XFS_ILOCK_EXCL; + + xfs_trans_ijoin(sc->tp, sc->orphanage, 0); + xfs_trans_ijoin(sc->tp, sc->ip, 0); + + /* + * Reserve enough quota in the orphan directory to add the new name. + * Normally the orphanage should have user/group/project ids of zero + * and hence is not subject to quota enforcement, but we're allowed to + * exceed quota to reattach disconnected parts of the directory tree. + */ + error = xfs_trans_reserve_quota_nblks(sc->tp, sc->orphanage, + adopt->orphanage_blkres, 0, true); + if (error) + goto out_cancel; + + /* + * Reserve enough quota in the child directory to change dotdot. + * Here we're also allowed to exceed file quota to repair inconsistent + * metadata. + */ + if (adopt->child_blkres) { + error = xfs_trans_reserve_quota_nblks(sc->tp, sc->ip, + adopt->child_blkres, 0, true); + if (error) + goto out_cancel; + } + + return 0; +out_cancel: + xchk_trans_cancel(sc); + xrep_orphanage_iunlock(sc, XFS_ILOCK_EXCL); + xchk_iunlock(sc, XFS_ILOCK_EXCL); + return error; +} + +/* + * Compute the xfs_name for the directory entry that we're adding to the + * orphanage. Caller must hold ILOCKs of sc->ip and the orphanage and must not + * reuse namebuf until the adoption completes or is dissolved. + */ +int +xrep_adoption_compute_name( + struct xrep_adoption *adopt, + struct xfs_name *xname) +{ + struct xfs_scrub *sc = adopt->sc; + char *namebuf = (void *)xname->name; + xfs_ino_t ino; + unsigned int incr = 0; + int error = 0; + + adopt->xname = xname; + xname->len = snprintf(namebuf, MAXNAMELEN, "%llu", I_INO(sc->ip)); + xname->type = xfs_mode_to_ftype(VFS_I(sc->ip)->i_mode); + + /* Make sure the filename is unique in the lost+found. */ + error = xchk_dir_lookup(sc, sc->orphanage, xname, &ino); + while (error == 0 && incr < 10000) { + xname->len = snprintf(namebuf, MAXNAMELEN, "%llu.%u", + I_INO(sc->ip), ++incr); + error = xchk_dir_lookup(sc, sc->orphanage, xname, &ino); + } + if (error == 0) { + /* We already have 10,000 entries in the orphanage? */ + return -EFSCORRUPTED; + } + + if (error != -ENOENT) + return error; + return 0; +} + +/* + * Make sure the dcache does not have a positive dentry for the name we've + * chosen. The caller should have checked with the ondisk directory, so any + * discrepancy is a sign that something is seriously wrong. + */ +static int +xrep_adoption_check_dcache( + struct xrep_adoption *adopt) +{ + struct qstr qname = QSTR_INIT(adopt->xname->name, + adopt->xname->len); + struct xfs_scrub *sc = adopt->sc; + struct dentry *d_orphanage, *d_child; + int error = 0; + + d_orphanage = d_find_alias(VFS_I(sc->orphanage)); + if (!d_orphanage) + return 0; + + d_child = try_lookup_noperm(&qname, d_orphanage); + if (IS_ERR(d_child)) { + dput(d_orphanage); + return PTR_ERR(d_child); + } + + if (d_child) { + trace_xrep_adoption_check_child(sc->mp, d_child); + + if (d_is_positive(d_child)) { + ASSERT(d_is_negative(d_child)); + error = -EFSCORRUPTED; + } + + dput(d_child); + } + + dput(d_orphanage); + return error; +} + +/* + * Invalidate all dentries for the name that was added to the orphanage + * directory, and all dentries pointing to the child inode that was moved. + * + * There should not be any positive entries for the name, since we've + * maintained our lock on the orphanage directory. + */ +static void +xrep_adoption_zap_dcache( + struct xrep_adoption *adopt) +{ + struct qstr qname = QSTR_INIT(adopt->xname->name, + adopt->xname->len); + struct xfs_scrub *sc = adopt->sc; + struct dentry *d_orphanage, *d_child; + + /* Invalidate all dentries for the adoption name */ + d_orphanage = d_find_alias(VFS_I(sc->orphanage)); + if (!d_orphanage) + return; + + d_child = try_lookup_noperm(&qname, d_orphanage); + while (!IS_ERR_OR_NULL(d_child)) { + trace_xrep_adoption_invalidate_child(sc->mp, d_child); + + ASSERT(d_is_negative(d_child)); + d_invalidate(d_child); + dput(d_child); + d_child = d_lookup(d_orphanage, &qname); + } + + dput(d_orphanage); + + /* Invalidate all the dentries pointing down to this file. */ + while ((d_child = d_find_alias(VFS_I(sc->ip))) != NULL) { + trace_xrep_adoption_invalidate_child(sc->mp, d_child); + + d_invalidate(d_child); + dput(d_child); + } +} + +/* + * If we have to add an attr fork ahead of a parent pointer update, how much + * space should we ask for? + */ +static inline int +xrep_adoption_attr_sizeof( + const struct xrep_adoption *adopt) +{ + return sizeof(struct xfs_attr_sf_hdr) + + xfs_attr_sf_entsize_byname(sizeof(struct xfs_parent_rec), + adopt->xname->len); +} + +/* + * Move the current file to the orphanage under the computed name. + * + * Returns with a dirty transaction so that the caller can handle any other + * work, such as fixing up unlinked lists or resetting link counts. + */ +int +xrep_adoption_move( + struct xrep_adoption *adopt) +{ + struct xfs_scrub *sc = adopt->sc; + bool isdir = S_ISDIR(VFS_I(sc->ip)->i_mode); + int error; + + trace_xrep_adoption_reparent(sc->orphanage, adopt->xname, + I_INO(sc->ip)); + + error = xrep_adoption_check_dcache(adopt); + if (error) + return error; + + /* + * If this filesystem has parent pointers, ensure that the file being + * moved to the orphanage has an attribute fork. This is required + * because the parent pointer code does not itself add attr forks. + */ + if (!xfs_inode_has_attr_fork(sc->ip) && xfs_has_parent(sc->mp)) { + int sf_size = xrep_adoption_attr_sizeof(adopt); + + error = xfs_bmap_add_attrfork(sc->tp, sc->ip, sf_size, true); + if (error) + return error; + } + + /* Create the new name in the orphanage. */ + error = xfs_dir_createname(sc->tp, sc->orphanage, adopt->xname, + I_INO(sc->ip), adopt->orphanage_blkres); + if (error) + return error; + + /* + * Bump the link count of the orphanage if we just added a + * subdirectory, and update its timestamps. + */ + xfs_trans_ichgtime(sc->tp, sc->orphanage, + XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG); + if (isdir) + xfs_bumplink(sc->tp, sc->orphanage); + xfs_trans_log_inode(sc->tp, sc->orphanage, XFS_ILOG_CORE); + + /* Bump the link count of the child. */ + if (adopt->bump_child_nlink) { + xfs_bumplink(sc->tp, sc->ip); + xfs_trans_log_inode(sc->tp, sc->ip, XFS_ILOG_CORE); + } + + /* Replace the dotdot entry if the child is a subdirectory. */ + if (isdir) { + error = xfs_dir_replace(sc->tp, sc->ip, &xfs_name_dotdot, + I_INO(sc->orphanage), adopt->child_blkres); + if (error) + return error; + } + + /* Add a parent pointer from the file back to the lost+found. */ + if (xfs_has_parent(sc->mp)) { + error = xfs_parent_addname(sc->tp, &adopt->ppargs, + sc->orphanage, adopt->xname, sc->ip); + if (error) + return error; + } + + /* + * Notify dirent hooks that we moved the file to /lost+found, and + * finish all the deferred work so that we know the adoption is fully + * recorded in the log. + */ + xfs_dir_update_hook(sc->orphanage, sc->ip, 1, adopt->xname); + + /* Remove negative dentries from the lost+found's dcache */ + xrep_adoption_zap_dcache(adopt); + return 0; +} + +/* + * Roll to a clean scrub transaction so that we can release the orphanage, + * even if xrep_adoption_move was not called. + * + * Commits all the work and deferred ops attached to an adoption request and + * rolls to a clean scrub transaction. On success, returns 0 with the scrub + * context holding a clean transaction with no inodes joined. On failure, + * returns negative errno with no scrub transaction. All inode locks are + * still held after this function returns. + */ +int +xrep_adoption_trans_roll( + struct xrep_adoption *adopt) +{ + struct xfs_scrub *sc = adopt->sc; + int error; + + trace_xrep_adoption_trans_roll(sc->orphanage, sc->ip, + !!(sc->tp->t_flags & XFS_TRANS_DIRTY)); + + /* Finish all the deferred ops to commit all repairs. */ + error = xrep_defer_finish(sc); + if (error) + return error; + + /* Roll the transaction once more to detach the inodes. */ + return xfs_trans_roll(&sc->tp); +} diff --git a/fs/xfs/scrub/orphanage.h b/fs/xfs/scrub/orphanage.h new file mode 100644 index 000000000..7c7a2e7d8 --- /dev/null +++ b/fs/xfs/scrub/orphanage.h @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2021-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_ORPHANAGE_H__ +#define __XFS_SCRUB_ORPHANAGE_H__ + +#ifdef CONFIG_XFS_ONLINE_REPAIR +int xrep_orphanage_create(struct xfs_scrub *sc); + +/* + * If we're doing a repair, ensure that the orphanage exists and attach it to + * the scrub context. + */ +static inline int +xrep_orphanage_try_create( + struct xfs_scrub *sc) +{ + int error; + + ASSERT(sc->sm->sm_flags & XFS_SCRUB_IFLAG_REPAIR); + + error = xrep_orphanage_create(sc); + switch (error) { + case 0: + case -ENOENT: + case -ENOTDIR: + case -ENOSPC: + /* + * If the orphanage can't be found or isn't a directory, we'll + * keep going, but we won't be able to attach the file to the + * orphanage if we can't find the parent. + */ + return 0; + } + + return error; +} + +int xrep_orphanage_iolock_two(struct xfs_scrub *sc); + +void xrep_orphanage_ilock(struct xfs_scrub *sc, unsigned int ilock_flags); +bool xrep_orphanage_ilock_nowait(struct xfs_scrub *sc, + unsigned int ilock_flags); +void xrep_orphanage_iunlock(struct xfs_scrub *sc, unsigned int ilock_flags); + +void xrep_orphanage_rele(struct xfs_scrub *sc); + +/* Information about a request to add a file to the orphanage. */ +struct xrep_adoption { + struct xfs_scrub *sc; + + /* Name used for the adoption. */ + struct xfs_name *xname; + + /* Parent pointer context tracking */ + struct xfs_parent_args ppargs; + + /* Block reservations for orphanage and child (if directory). */ + unsigned int orphanage_blkres; + unsigned int child_blkres; + + /* + * Does the caller want us to bump the child link count? This is not + * needed when reattaching files that have become disconnected but have + * nlink > 1. It is necessary when changing the directory tree + * structure. + */ + bool bump_child_nlink:1; +}; + +bool xrep_orphanage_can_adopt(struct xfs_scrub *sc); + +int xrep_adoption_trans_alloc(struct xfs_scrub *sc, + struct xrep_adoption *adopt); +int xrep_adoption_compute_name(struct xrep_adoption *adopt, + struct xfs_name *xname); +int xrep_adoption_move(struct xrep_adoption *adopt); +int xrep_adoption_trans_roll(struct xrep_adoption *adopt); +#else +struct xrep_adoption { /* empty */ }; +# define xrep_orphanage_rele(sc) ((void)0) +#endif /* CONFIG_XFS_ONLINE_REPAIR */ + +#endif /* __XFS_SCRUB_ORPHANAGE_H__ */ diff --git a/fs/xfs/scrub/parent.c b/fs/xfs/scrub/parent.c new file mode 100644 index 000000000..99b60773b --- /dev/null +++ b/fs/xfs/scrub/parent.c @@ -0,0 +1,952 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_inode.h" +#include "xfs_icache.h" +#include "xfs_dir2.h" +#include "xfs_dir2_priv.h" +#include "xfs_attr.h" +#include "xfs_parent.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/readdir.h" +#include "scrub/tempfile.h" +#include "scrub/repair.h" +#include "scrub/listxattr.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/xfblob.h" +#include "scrub/trace.h" + +/* Set us up to scrub parents. */ +int +xchk_setup_parent( + struct xfs_scrub *sc) +{ + int error; + + if (xchk_could_repair(sc)) { + error = xrep_setup_parent(sc); + if (error) + return error; + } + + return xchk_setup_inode_contents(sc, 0); +} + +/* Parent pointers */ + +/* Look for an entry in a parent pointing to this inode. */ + +struct xchk_parent_ctx { + struct xfs_scrub *sc; + xfs_nlink_t nlink; +}; + +/* Look for a single entry in a directory pointing to an inode. */ +STATIC int +xchk_parent_actor( + struct xfs_scrub *sc, + struct xfs_inode *dp, + xfs_dir2_dataptr_t dapos, + const struct xfs_name *name, + xfs_ino_t ino, + void *priv) +{ + struct xchk_parent_ctx *spc = priv; + int error = 0; + + /* Does this name make sense? */ + if (!xfs_dir2_namecheck(name->name, name->len)) + error = -EFSCORRUPTED; + if (!xchk_fblock_xref_process_error(sc, XFS_DATA_FORK, 0, &error)) + return error; + + if (I_INO(sc->ip) == ino) + spc->nlink++; + + if (xchk_should_terminate(spc->sc, &error)) + return error; + + return 0; +} + +/* + * Try to lock a parent directory for checking dirents. Returns the inode + * flags for the locks we now hold, or zero if we failed. + */ +STATIC unsigned int +xchk_parent_ilock_dir( + struct xfs_inode *dp) +{ + if (!xfs_ilock_nowait(dp, XFS_ILOCK_SHARED)) + return 0; + + if (!xfs_need_iread_extents(&dp->i_df)) + return XFS_ILOCK_SHARED; + + xfs_iunlock(dp, XFS_ILOCK_SHARED); + + if (!xfs_ilock_nowait(dp, XFS_ILOCK_EXCL)) + return 0; + + return XFS_ILOCK_EXCL; +} + +/* + * Given the inode number of the alleged parent of the inode being scrubbed, + * try to validate that the parent has exactly one directory entry pointing + * back to the inode being scrubbed. Returns -EAGAIN if we need to revalidate + * the dotdot entry. + */ +STATIC int +xchk_parent_validate( + struct xfs_scrub *sc, + xfs_ino_t parent_ino) +{ + struct xchk_parent_ctx spc = { + .sc = sc, + .nlink = 0, + }; + struct xfs_mount *mp = sc->mp; + xfs_ino_t ino = I_INO(sc->ip); + struct xfs_inode *dp = NULL; + xfs_nlink_t expected_nlink; + unsigned int lock_mode; + int error = 0; + + /* Is this the root dir? Then '..' must point to itself. */ + if (sc->ip == mp->m_rootip) { + if (ino != mp->m_sb.sb_rootino || ino != parent_ino) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); + return 0; + } + + /* Is this the metadata root dir? Then '..' must point to itself. */ + if (sc->ip == mp->m_metadirip) { + if (ino != mp->m_sb.sb_metadirino || ino != parent_ino) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); + return 0; + } + + /* '..' must not point to ourselves. */ + if (ino == parent_ino) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); + return 0; + } + + /* + * If we're an unlinked directory, the parent /won't/ have a link + * to us. Otherwise, it should have one link. + */ + expected_nlink = VFS_I(sc->ip)->i_nlink == 0 ? 0 : 1; + + /* + * Grab the parent directory inode. This must be released before we + * cancel the scrub transaction. + * + * If _iget returns -EINVAL or -ENOENT then the parent inode number is + * garbage and the directory is corrupt. If the _iget returns + * -EFSCORRUPTED or -EFSBADCRC then the parent is corrupt which is a + * cross referencing error. Any other error is an operational error. + */ + error = xchk_iget(sc, parent_ino, &dp); + if (error == -EINVAL || error == -ENOENT) { + error = -EFSCORRUPTED; + xchk_fblock_process_error(sc, XFS_DATA_FORK, 0, &error); + return error; + } + if (!xchk_fblock_xref_process_error(sc, XFS_DATA_FORK, 0, &error)) + return error; + if (dp == sc->ip || xrep_is_tempfile(dp) || + !S_ISDIR(VFS_I(dp)->i_mode)) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); + goto out_rele; + } + + lock_mode = xchk_parent_ilock_dir(dp); + if (!lock_mode) { + xchk_iunlock(sc, XFS_ILOCK_EXCL); + xchk_ilock(sc, XFS_ILOCK_EXCL); + error = -EAGAIN; + goto out_rele; + } + + /* + * We cannot yet validate this parent pointer if the directory looks as + * though it has been zapped by the inode record repair code. + */ + if (xchk_dir_looks_zapped(dp)) { + error = -EBUSY; + xchk_set_incomplete(sc); + goto out_unlock; + } + + /* Metadata and regular inodes cannot cross trees. */ + if (xfs_is_metadir_inode(dp) != xfs_is_metadir_inode(sc->ip)) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); + goto out_unlock; + } + + /* Look for a directory entry in the parent pointing to the child. */ + error = xchk_dir_walk(sc, dp, xchk_parent_actor, &spc); + if (!xchk_fblock_xref_process_error(sc, XFS_DATA_FORK, 0, &error)) + goto out_unlock; + + /* + * Ensure that the parent has as many links to the child as the child + * thinks it has to the parent. + */ + if (spc.nlink != expected_nlink) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); + +out_unlock: + xfs_iunlock(dp, lock_mode); +out_rele: + xchk_irele(sc, dp); + return error; +} + +/* + * Checking of Parent Pointers + * =========================== + * + * On filesystems with directory parent pointers, we check the referential + * integrity by visiting each parent pointer of a child file and checking that + * the directory referenced by the pointer actually has a dirent pointing + * forward to the child file. + */ + +/* Deferred parent pointer entry that we saved for later. */ +struct xchk_pptr { + /* Cookie for retrieval of the pptr name. */ + xfblob_cookie name_cookie; + + /* Parent pointer record. */ + struct xfs_parent_rec pptr_rec; + + /* Length of the pptr name. */ + uint8_t namelen; +}; + +struct xchk_pptrs { + struct xfs_scrub *sc; + + /* How many parent pointers did we find at the end? */ + unsigned long long pptrs_found; + + /* Parent of this directory. */ + xfs_ino_t parent_ino; + + /* Fixed-size array of xchk_pptr structures. */ + struct xfarray *pptr_entries; + + /* Blobs containing parent pointer names. */ + struct xfblob *pptr_names; + + /* Scratch buffer for scanning pptr xattrs */ + struct xfs_da_args pptr_args; + + /* If we've cycled the ILOCK, we must revalidate all deferred pptrs. */ + bool need_revalidate; + + /* Name buffer */ + struct xfs_name xname; + char namebuf[MAXNAMELEN]; +}; + +/* Does this parent pointer match the dotdot entry? */ +STATIC int +xchk_parent_scan_dotdot( + struct xfs_scrub *sc, + struct xfs_inode *ip, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + const void *value, + unsigned int valuelen, + void *priv) +{ + struct xchk_pptrs *pp = priv; + xfs_ino_t parent_ino; + int error; + + if (!(attr_flags & XFS_ATTR_PARENT)) + return 0; + + error = xfs_parent_from_attr(sc->mp, attr_flags, name, namelen, value, + valuelen, &parent_ino, NULL); + if (error) + return error; + + if (pp->parent_ino == parent_ino) + return -ECANCELED; + + return 0; +} + +/* Look up the dotdot entry so that we can check it as we walk the pptrs. */ +STATIC int +xchk_parent_pptr_and_dotdot( + struct xchk_pptrs *pp) +{ + struct xfs_scrub *sc = pp->sc; + int error; + + /* Look up '..' */ + error = xchk_dir_lookup(sc, sc->ip, &xfs_name_dotdot, &pp->parent_ino); + if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, 0, &error)) + return error; + if (!xfs_verify_dir_ino(sc->mp, pp->parent_ino)) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); + return 0; + } + + /* Is this the root dir? Then '..' must point to itself. */ + if (xchk_inode_is_dirtree_root(sc->ip)) { + if (I_INO(sc->ip) != pp->parent_ino) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); + return 0; + } + + /* + * If this is now an unlinked directory, the dotdot value is + * meaningless as long as it points to a valid inode. + */ + if (VFS_I(sc->ip)->i_nlink == 0) + return 0; + + if (pp->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return 0; + + /* Otherwise, walk the pptrs again, and check. */ + error = xchk_xattr_walk(sc, sc->ip, xchk_parent_scan_dotdot, NULL, pp); + if (error == -ECANCELED) { + /* Found a parent pointer that matches dotdot. */ + return 0; + } + if (!error || error == -EFSCORRUPTED) { + /* Found a broken parent pointer or no match. */ + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, 0); + return 0; + } + return error; +} + +/* + * Try to lock a parent directory for checking dirents. Returns the inode + * flags for the locks we now hold, or zero if we failed. + */ +STATIC unsigned int +xchk_parent_lock_dir( + struct xfs_scrub *sc, + struct xfs_inode *dp) +{ + if (!xfs_ilock_nowait(dp, XFS_IOLOCK_SHARED)) + return 0; + + if (!xfs_ilock_nowait(dp, XFS_ILOCK_SHARED)) { + xfs_iunlock(dp, XFS_IOLOCK_SHARED); + return 0; + } + + if (!xfs_need_iread_extents(&dp->i_df)) + return XFS_IOLOCK_SHARED | XFS_ILOCK_SHARED; + + xfs_iunlock(dp, XFS_ILOCK_SHARED); + + if (!xfs_ilock_nowait(dp, XFS_ILOCK_EXCL)) { + xfs_iunlock(dp, XFS_IOLOCK_SHARED); + return 0; + } + + return XFS_IOLOCK_SHARED | XFS_ILOCK_EXCL; +} + +/* Check the forward link (dirent) associated with this parent pointer. */ +STATIC int +xchk_parent_dirent( + struct xchk_pptrs *pp, + const struct xfs_name *xname, + struct xfs_inode *dp) +{ + struct xfs_scrub *sc = pp->sc; + xfs_ino_t child_ino; + int error; + + /* + * Use the name attached to this parent pointer to look up the + * directory entry in the alleged parent. + */ + error = xchk_dir_lookup(sc, dp, xname, &child_ino); + if (error == -ENOENT) { + xchk_fblock_xref_set_corrupt(sc, XFS_ATTR_FORK, 0); + return 0; + } + if (!xchk_fblock_xref_process_error(sc, XFS_ATTR_FORK, 0, &error)) + return error; + + /* Does the inode number match? */ + if (child_ino != I_INO(sc->ip)) { + xchk_fblock_xref_set_corrupt(sc, XFS_ATTR_FORK, 0); + return 0; + } + + return 0; +} + +/* Try to grab a parent directory. */ +STATIC int +xchk_parent_iget( + struct xchk_pptrs *pp, + const struct xfs_parent_rec *pptr, + struct xfs_inode **dpp) +{ + struct xfs_scrub *sc = pp->sc; + struct xfs_inode *ip; + xfs_ino_t parent_ino = be64_to_cpu(pptr->p_ino); + int error; + + /* Validate inode number. */ + error = xfs_dir_ino_validate(sc->mp, parent_ino); + if (error) { + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, 0); + return -ECANCELED; + } + + error = xchk_iget(sc, parent_ino, &ip); + if (error == -EINVAL || error == -ENOENT) { + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, 0); + return -ECANCELED; + } + if (!xchk_fblock_xref_process_error(sc, XFS_ATTR_FORK, 0, &error)) + return error; + + /* The parent must be a directory. */ + if (!S_ISDIR(VFS_I(ip)->i_mode)) { + xchk_fblock_xref_set_corrupt(sc, XFS_ATTR_FORK, 0); + goto out_rele; + } + + /* Validate generation number. */ + if (VFS_I(ip)->i_generation != be32_to_cpu(pptr->p_gen)) { + xchk_fblock_xref_set_corrupt(sc, XFS_ATTR_FORK, 0); + goto out_rele; + } + + *dpp = ip; + return 0; +out_rele: + xchk_irele(sc, ip); + return 0; +} + +/* + * Walk an xattr of a file. If this xattr is a parent pointer, follow it up + * to a parent directory and check that the parent has a dirent pointing back + * to us. + */ +STATIC int +xchk_parent_scan_attr( + struct xfs_scrub *sc, + struct xfs_inode *ip, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + const void *value, + unsigned int valuelen, + void *priv) +{ + struct xfs_name xname = { + .name = name, + .len = namelen, + }; + struct xchk_pptrs *pp = priv; + struct xfs_inode *dp = NULL; + const struct xfs_parent_rec *pptr_rec = value; + xfs_ino_t parent_ino; + unsigned int lockmode; + int error; + + if (!(attr_flags & XFS_ATTR_PARENT)) + return 0; + + error = xfs_parent_from_attr(sc->mp, attr_flags, name, namelen, value, + valuelen, &parent_ino, NULL); + if (error) { + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, 0); + return -ECANCELED; + } + + /* No self-referential parent pointers. */ + if (parent_ino == I_INO(sc->ip)) { + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, 0); + return -ECANCELED; + } + + pp->pptrs_found++; + + error = xchk_parent_iget(pp, pptr_rec, &dp); + if (error) + return error; + if (!dp) + return 0; + + /* Try to lock the inode. */ + lockmode = xchk_parent_lock_dir(sc, dp); + if (!lockmode) { + struct xchk_pptr save_pp = { + .pptr_rec = *pptr_rec, /* struct copy */ + .namelen = namelen, + }; + + /* Couldn't lock the inode, so save the pptr for later. */ + trace_xchk_parent_defer(sc->ip, &xname, I_INO(dp)); + + error = xfblob_storename(pp->pptr_names, &save_pp.name_cookie, + &xname); + if (!xchk_fblock_xref_process_error(sc, XFS_ATTR_FORK, 0, + &error)) + goto out_rele; + + error = xfarray_append(pp->pptr_entries, &save_pp); + if (!xchk_fblock_xref_process_error(sc, XFS_ATTR_FORK, 0, + &error)) + goto out_rele; + + goto out_rele; + } + + error = xchk_parent_dirent(pp, &xname, dp); + if (error) + goto out_unlock; + +out_unlock: + xfs_iunlock(dp, lockmode); +out_rele: + xchk_irele(sc, dp); + return error; +} + +/* + * Revalidate a parent pointer that we collected in the past but couldn't check + * because of lock contention. Returns 0 if the parent pointer is still valid, + * -ENOENT if it has gone away on us, or a negative errno. + */ +STATIC int +xchk_parent_revalidate_pptr( + struct xchk_pptrs *pp, + const struct xfs_name *xname, + struct xfs_parent_rec *pptr) +{ + struct xfs_scrub *sc = pp->sc; + int error; + + error = xfs_parent_lookup(sc->tp, sc->ip, xname, pptr, &pp->pptr_args); + if (error == -ENOATTR) { + /* Parent pointer went away, nothing to revalidate. */ + return -ENOENT; + } + + return error; +} + +/* + * Check a parent pointer the slow way, which means we cycle locks a bunch + * and put up with revalidation until we get it done. + */ +STATIC int +xchk_parent_slow_pptr( + struct xchk_pptrs *pp, + const struct xfs_name *xname, + struct xfs_parent_rec *pptr) +{ + struct xfs_scrub *sc = pp->sc; + struct xfs_inode *dp = NULL; + unsigned int lockmode; + int error; + + /* Check that the deferred parent pointer still exists. */ + if (pp->need_revalidate) { + error = xchk_parent_revalidate_pptr(pp, xname, pptr); + if (error == -ENOENT) + return 0; + if (!xchk_fblock_xref_process_error(sc, XFS_ATTR_FORK, 0, + &error)) + return error; + } + + error = xchk_parent_iget(pp, pptr, &dp); + if (error) + return error; + if (!dp) + return 0; + + /* + * If we can grab both IOLOCK and ILOCK of the alleged parent, we + * can proceed with the validation. + */ + lockmode = xchk_parent_lock_dir(sc, dp); + if (lockmode) { + trace_xchk_parent_slowpath(sc->ip, xname, I_INO(dp)); + goto check_dirent; + } + + /* + * We couldn't lock the parent dir. Drop all the locks and try to + * get them again, one at a time. + */ + xchk_iunlock(sc, sc->ilock_flags); + pp->need_revalidate = true; + + trace_xchk_parent_ultraslowpath(sc->ip, xname, I_INO(dp)); + + error = xchk_dir_trylock_for_pptrs(sc, dp, &lockmode); + if (error) + goto out_rele; + + /* Revalidate the parent pointer now that we cycled locks. */ + error = xchk_parent_revalidate_pptr(pp, xname, pptr); + if (error == -ENOENT) { + error = 0; + goto out_unlock; + } + if (!xchk_fblock_xref_process_error(sc, XFS_ATTR_FORK, 0, &error)) + goto out_unlock; + +check_dirent: + error = xchk_parent_dirent(pp, xname, dp); +out_unlock: + xfs_iunlock(dp, lockmode); +out_rele: + xchk_irele(sc, dp); + return error; +} + +/* Check all the parent pointers that we deferred the first time around. */ +STATIC int +xchk_parent_finish_slow_pptrs( + struct xchk_pptrs *pp) +{ + xfarray_idx_t array_cur; + int error; + + foreach_xfarray_idx(pp->pptr_entries, array_cur) { + struct xchk_pptr pptr; + + if (pp->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return 0; + + error = xfarray_load(pp->pptr_entries, array_cur, &pptr); + if (error) + return error; + + error = xfblob_loadname(pp->pptr_names, pptr.name_cookie, + &pp->xname, pptr.namelen); + if (error) + return error; + + error = xchk_parent_slow_pptr(pp, &pp->xname, &pptr.pptr_rec); + if (error) + return error; + } + + /* Empty out both xfiles now that we've checked everything. */ + xfarray_truncate(pp->pptr_entries); + xfblob_truncate(pp->pptr_names); + return 0; +} + +/* Count the number of parent pointers. */ +STATIC int +xchk_parent_count_pptr( + struct xfs_scrub *sc, + struct xfs_inode *ip, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + const void *value, + unsigned int valuelen, + void *priv) +{ + struct xchk_pptrs *pp = priv; + int error; + + if (!(attr_flags & XFS_ATTR_PARENT)) + return 0; + + error = xfs_parent_from_attr(sc->mp, attr_flags, name, namelen, value, + valuelen, NULL, NULL); + if (error) + return error; + + pp->pptrs_found++; + return 0; +} + +/* + * Compare the number of parent pointers to the link count. For + * non-directories these should be the same. For unlinked directories the + * count should be zero; for linked directories, it should be nonzero. + */ +STATIC int +xchk_parent_count_pptrs( + struct xchk_pptrs *pp) +{ + struct xfs_scrub *sc = pp->sc; + int error; + + /* + * If we cycled the ILOCK while cross-checking parent pointers with + * dirents, then we need to recalculate the number of parent pointers. + */ + if (pp->need_revalidate) { + pp->pptrs_found = 0; + error = xchk_xattr_walk(sc, sc->ip, xchk_parent_count_pptr, + NULL, pp); + if (error == -EFSCORRUPTED) { + /* Found a bad parent pointer */ + xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, 0); + return 0; + } + if (error) + return error; + } + + if (S_ISDIR(VFS_I(sc->ip)->i_mode)) { + if (xchk_inode_is_dirtree_root(sc->ip)) + pp->pptrs_found++; + + if (VFS_I(sc->ip)->i_nlink == 0 && pp->pptrs_found > 0) + xchk_ip_set_corrupt(sc, sc->ip); + else if (VFS_I(sc->ip)->i_nlink > 0 && + pp->pptrs_found == 0) + xchk_ip_set_corrupt(sc, sc->ip); + } else { + /* + * Starting with metadir, we allow checking of parent pointers + * of non-directory files that are children of the superblock. + * Pretend that we found a parent pointer attr. + */ + if (xfs_has_metadir(sc->mp) && xchk_inode_is_sb_rooted(sc->ip)) + pp->pptrs_found++; + + if (VFS_I(sc->ip)->i_nlink != pp->pptrs_found) + xchk_ip_set_corrupt(sc, sc->ip); + } + + return 0; +} + +/* Check parent pointers of a file. */ +STATIC int +xchk_parent_pptr( + struct xfs_scrub *sc) +{ + struct xchk_pptrs *pp; + int error; + + pp = kvzalloc_obj(struct xchk_pptrs, XCHK_GFP_FLAGS); + if (!pp) + return -ENOMEM; + pp->sc = sc; + pp->xname.name = pp->namebuf; + + /* + * Set up some staging memory for parent pointers that we can't check + * due to locking contention. + */ + error = xfarray_create("slow parent pointer entries", 0, + sizeof(struct xchk_pptr), &pp->pptr_entries); + if (error) + goto out_pp; + + error = xfblob_create("slow parent pointer names", &pp->pptr_names); + if (error) + goto out_entries; + + error = xchk_xattr_walk(sc, sc->ip, xchk_parent_scan_attr, NULL, pp); + if (error == -ECANCELED) { + error = 0; + goto out_names; + } + if (error) + goto out_names; + + error = xchk_parent_finish_slow_pptrs(pp); + if (error == -ETIMEDOUT) { + /* Couldn't grab a lock, scrub was marked incomplete */ + error = 0; + goto out_names; + } + if (error) + goto out_names; + + if (pp->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out_names; + + /* + * For subdirectories, make sure the dotdot entry references the same + * inode as the parent pointers. + * + * If we're scanning a /consistent/ directory, there should only be + * one parent pointer, and it should point to the same directory as + * the dotdot entry. + * + * However, a corrupt directory tree might feature a subdirectory with + * multiple parents. The directory loop scanner is responsible for + * correcting that kind of problem, so for now we only validate that + * the dotdot entry matches /one/ of the parents. + */ + if (S_ISDIR(VFS_I(sc->ip)->i_mode)) { + error = xchk_parent_pptr_and_dotdot(pp); + if (error) + goto out_names; + } + + if (pp->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out_names; + + /* + * Complain if the number of parent pointers doesn't match the link + * count. This could be a sign of missing parent pointers (or an + * incorrect link count). + */ + error = xchk_parent_count_pptrs(pp); + if (error) + goto out_names; + +out_names: + xfblob_destroy(pp->pptr_names); +out_entries: + xfarray_destroy(pp->pptr_entries); +out_pp: + kvfree(pp); + return error; +} + +/* Scrub a parent pointer. */ +int +xchk_parent( + struct xfs_scrub *sc) +{ + struct xfs_mount *mp = sc->mp; + xfs_ino_t parent_ino; + int error = 0; + + if (xfs_has_parent(mp)) + return xchk_parent_pptr(sc); + + /* + * If we're a directory, check that the '..' link points up to + * a directory that has one entry pointing to us. + */ + if (!S_ISDIR(VFS_I(sc->ip)->i_mode)) + return -ENOENT; + + /* We're not a special inode, are we? */ + if (!xfs_verify_dir_ino(mp, I_INO(sc->ip))) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); + return 0; + } + + do { + if (xchk_should_terminate(sc, &error)) + break; + + /* Look up '..' */ + error = xchk_dir_lookup(sc, sc->ip, &xfs_name_dotdot, + &parent_ino); + if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, 0, &error)) + return error; + if (!xfs_verify_dir_ino(mp, parent_ino)) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); + return 0; + } + + /* + * Check that the dotdot entry points to a parent directory + * containing a dirent pointing to this subdirectory. + */ + error = xchk_parent_validate(sc, parent_ino); + } while (error == -EAGAIN); + if (error == -EBUSY) { + /* + * We could not scan a directory, so we marked the check + * incomplete. No further error return is necessary. + */ + return 0; + } + + return error; +} + +/* + * Decide if this file's extended attributes (and therefore its parent + * pointers) have been zapped to satisfy the inode and ifork verifiers. + * Checking and repairing should be postponed until the extended attribute + * structure is fixed. + */ +bool +xchk_pptr_looks_zapped( + struct xfs_inode *ip) +{ + struct inode *inode = VFS_I(ip); + + ASSERT(xfs_has_parent(ip->i_mount)); + + /* + * Temporary files that cannot be linked into the directory tree do not + * have attr forks because they cannot ever have parents. + */ + if (inode->i_nlink == 0 && !(inode_state_read_once(inode) & I_LINKABLE)) + return false; + + /* + * Directory tree roots do not have parents, so the expected outcome + * of a parent pointer scan is always the empty set. It's safe to scan + * them even if the attr fork was zapped. + */ + if (xchk_inode_is_dirtree_root(ip)) + return false; + + /* + * Metadata inodes that are rooted in the superblock do not have any + * parents. Hence the attr fork will not be initialized, but there are + * no parent pointers that might have been zapped. + */ + if (xchk_inode_is_sb_rooted(ip)) + return false; + + /* + * Linked and linkable non-rootdir files should always have an + * attribute fork because that is where parent pointers are + * stored. If the fork is absent, something is amiss. + */ + if (!xfs_inode_has_attr_fork(ip)) + return true; + + /* Repair zapped this file's attr fork a short time ago */ + if (xfs_ifork_zapped(ip, XFS_ATTR_FORK)) + return true; + + /* + * If the dinode repair found a bad attr fork, it will reset the fork + * to extents format with zero records and wait for the bmapbta + * scrubber to reconstruct the block mappings. The extended attribute + * structure always contain some content when parent pointers are + * enabled, so this is a clear sign of a zapped attr fork. + */ + return ip->i_af.if_format == XFS_DINODE_FMT_EXTENTS && + ip->i_af.if_nextents == 0; +} diff --git a/fs/xfs/scrub/parent_repair.c b/fs/xfs/scrub/parent_repair.c new file mode 100644 index 000000000..f64372491 --- /dev/null +++ b/fs/xfs/scrub/parent_repair.c @@ -0,0 +1,1622 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_bit.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_sb.h" +#include "xfs_inode.h" +#include "xfs_icache.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_dir2.h" +#include "xfs_bmap_btree.h" +#include "xfs_dir2_priv.h" +#include "xfs_trans_space.h" +#include "xfs_health.h" +#include "xfs_exchmaps.h" +#include "xfs_parent.h" +#include "xfs_attr.h" +#include "xfs_bmap.h" +#include "xfs_ag.h" +#include "scrub/xfs_scrub.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/iscan.h" +#include "scrub/findparent.h" +#include "scrub/readdir.h" +#include "scrub/tempfile.h" +#include "scrub/tempexch.h" +#include "scrub/orphanage.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/xfblob.h" +#include "scrub/attr_repair.h" +#include "scrub/listxattr.h" + +/* + * Repairing The Directory Parent Pointer + * ====================================== + * + * Currently, only directories support parent pointers (in the form of '..' + * entries), so we simply scan the filesystem and update the '..' entry. + * + * Note that because the only parent pointer is the dotdot entry, we won't + * touch an unhealthy directory, since the directory repair code is perfectly + * capable of rebuilding a directory with the proper parent inode. + * + * See the section on locking issues in dir_repair.c for more information about + * conflicts with the VFS. The findparent code wll keep our incore parent + * inode up to date. + * + * If parent pointers are enabled, we instead reconstruct the parent pointer + * information by visiting every directory entry of every directory in the + * system and translating the relevant dirents into parent pointers. In this + * case, it is advantageous to stash all parent pointers created from dirents + * from a single parent file before replaying them into the temporary file. To + * save memory, the live filesystem scan reuses the findparent object. Parent + * pointer repair chooses either directory scanning or findparent, but not + * both. + * + * When salvaging completes, the remaining stashed entries are replayed to the + * temporary file. All non-parent pointer extended attributes are copied to + * the temporary file's extended attributes. An atomic file mapping exchange + * is used to commit the new xattr blocks to the file being repaired. This + * will disrupt attrmulti cursors. + */ + +/* Create a parent pointer in the tempfile. */ +#define XREP_PPTR_ADD (1) + +/* Remove a parent pointer from the tempfile. */ +#define XREP_PPTR_REMOVE (2) + +/* A stashed parent pointer update. */ +struct xrep_pptr { + /* Cookie for retrieval of the pptr name. */ + xfblob_cookie name_cookie; + + /* Parent pointer record. */ + struct xfs_parent_rec pptr_rec; + + /* Length of the pptr name. */ + uint8_t namelen; + + /* XREP_PPTR_{ADD,REMOVE} */ + uint8_t action; +}; + +/* + * Stash up to 8 pages of recovered parent pointers in pptr_recs and + * pptr_names before we write them to the temp file. + */ +#define XREP_PARENT_MAX_STASH_BYTES (PAGE_SIZE * 8) + +struct xrep_parent { + struct xfs_scrub *sc; + + /* Fixed-size array of xrep_pptr structures. */ + struct xfarray *pptr_recs; + + /* Blobs containing parent pointer names. */ + struct xfblob *pptr_names; + + /* xattr keys */ + struct xfarray *xattr_records; + + /* xattr values */ + struct xfblob *xattr_blobs; + + /* Scratch buffers for saving extended attributes */ + unsigned char *xattr_name; + void *xattr_value; + unsigned int xattr_value_sz; + + /* + * Information used to exchange the attr fork mappings, if the fs + * supports parent pointers. + */ + struct xrep_tempexch tx; + + /* + * Information used to scan the filesystem to find the inumber of the + * dotdot entry for this directory. On filesystems without parent + * pointers, we use the findparent_* functions on this object and + * access only the parent_ino field directly. + * + * When parent pointers are enabled, the directory entry scanner uses + * the iscan, hooks, and lock fields of this object directly. + * @pscan.lock coordinates access to pptr_recs, pptr_names, pptr, and + * pptr_scratch. This reduces the memory requirements of this + * structure. + * + * The lock also controls access to xattr_records and xattr_blobs(?) + */ + struct xrep_parent_scan_info pscan; + + /* Orphanage reparenting request. */ + struct xrep_adoption adoption; + + /* Directory entry name, plus the trailing null. */ + struct xfs_name xname; + unsigned char namebuf[MAXNAMELEN]; + + /* Scratch buffer for scanning pptr xattrs */ + struct xfs_da_args pptr_args; + + /* Have we seen any live updates of parent pointers recently? */ + bool saw_pptr_updates; + + /* Number of parents we found after all other repairs */ + unsigned long long parents; +}; + +struct xrep_parent_xattr { + /* Cookie for retrieval of the xattr name. */ + xfblob_cookie name_cookie; + + /* Cookie for retrieval of the xattr value. */ + xfblob_cookie value_cookie; + + /* XFS_ATTR_* flags */ + int flags; + + /* Length of the value and name. */ + uint32_t valuelen; + uint16_t namelen; +}; + +/* + * Stash up to 8 pages of attrs in xattr_records/xattr_blobs before we write + * them to the temp file. + */ +#define XREP_PARENT_XATTR_MAX_STASH_BYTES (PAGE_SIZE * 8) + +/* Tear down all the incore stuff we created. */ +static void +xrep_parent_teardown( + struct xrep_parent *rp) +{ + xrep_findparent_scan_teardown(&rp->pscan); + kvfree(rp->xattr_name); + rp->xattr_name = NULL; + kvfree(rp->xattr_value); + rp->xattr_value = NULL; + if (rp->xattr_blobs) + xfblob_destroy(rp->xattr_blobs); + rp->xattr_blobs = NULL; + if (rp->xattr_records) + xfarray_destroy(rp->xattr_records); + rp->xattr_records = NULL; + if (rp->pptr_names) + xfblob_destroy(rp->pptr_names); + rp->pptr_names = NULL; + if (rp->pptr_recs) + xfarray_destroy(rp->pptr_recs); + rp->pptr_recs = NULL; +} + +/* Set up for a parent repair. */ +int +xrep_setup_parent( + struct xfs_scrub *sc) +{ + struct xrep_parent *rp; + int error; + + xchk_fsgates_enable(sc, XCHK_FSGATES_DIRENTS); + + rp = kvzalloc_obj(struct xrep_parent, XCHK_GFP_FLAGS); + if (!rp) + return -ENOMEM; + rp->sc = sc; + rp->xname.name = rp->namebuf; + sc->buf = rp; + + error = xrep_tempfile_create(sc, S_IFREG); + if (error) + return error; + + return xrep_orphanage_try_create(sc); +} + +/* + * Scan all files in the filesystem for a child dirent that we can turn into + * the dotdot entry for this directory. + */ +STATIC int +xrep_parent_find_dotdot( + struct xrep_parent *rp) +{ + struct xfs_scrub *sc = rp->sc; + xfs_ino_t ino; + unsigned int sick, checked; + int error; + + /* + * Avoid sick directories. There shouldn't be anyone else clearing the + * directory's sick status. + */ + xfs_inode_measure_sickness(sc->ip, &sick, &checked); + if (sick & XFS_SICK_INO_DIR) + return -EFSCORRUPTED; + + ino = xrep_findparent_self_reference(sc); + if (ino != NULLFSINO) { + xrep_findparent_scan_finish_early(&rp->pscan, ino); + return 0; + } + + /* + * Drop the ILOCK on this directory so that we can scan for the dotdot + * entry. Figure out who is going to be the parent of this directory, + * then retake the ILOCK so that we can salvage directory entries. + */ + xchk_iunlock(sc, XFS_ILOCK_EXCL); + + /* Does the VFS dcache have an answer for us? */ + ino = xrep_findparent_from_dcache(sc); + if (ino != NULLFSINO) { + error = xrep_findparent_confirm(sc, &ino); + if (!error && ino != NULLFSINO) { + xrep_findparent_scan_finish_early(&rp->pscan, ino); + goto out_relock; + } + } + + /* Scan the entire filesystem for a parent. */ + error = xrep_findparent_scan(&rp->pscan); +out_relock: + xchk_ilock(sc, XFS_ILOCK_EXCL); + + return error; +} + +/* + * Add this stashed incore parent pointer to the temporary file. + * The caller must hold the tempdir's IOLOCK, must not hold any ILOCKs, and + * must not be in transaction context. + */ +STATIC int +xrep_parent_replay_update( + struct xrep_parent *rp, + const struct xfs_name *xname, + struct xrep_pptr *pptr) +{ + struct xfs_scrub *sc = rp->sc; + + switch (pptr->action) { + case XREP_PPTR_ADD: + /* Create parent pointer. */ + trace_xrep_parent_replay_parentadd(sc->tempip, xname, + &pptr->pptr_rec); + + return xfs_parent_set(sc->tempip, I_INO(sc->ip), xname, + &pptr->pptr_rec, &rp->pptr_args); + case XREP_PPTR_REMOVE: + /* Remove parent pointer. */ + trace_xrep_parent_replay_parentremove(sc->tempip, xname, + &pptr->pptr_rec); + + return xfs_parent_unset(sc->tempip, I_INO(sc->ip), xname, + &pptr->pptr_rec, &rp->pptr_args); + } + + ASSERT(0); + return -EIO; +} + +/* + * Flush stashed parent pointer updates that have been recorded by the scanner. + * This is done to reduce the memory requirements of the parent pointer + * rebuild, since files can have a lot of hardlinks and the fs can be busy. + * + * Caller must not hold transactions or ILOCKs. Caller must hold the tempfile + * IOLOCK. + */ +STATIC int +xrep_parent_replay_updates( + struct xrep_parent *rp) +{ + xfarray_idx_t array_cur; + int error; + + mutex_lock(&rp->pscan.lock); + foreach_xfarray_idx(rp->pptr_recs, array_cur) { + struct xrep_pptr pptr; + + error = xfarray_load(rp->pptr_recs, array_cur, &pptr); + if (error) + goto out_unlock; + + error = xfblob_loadname(rp->pptr_names, pptr.name_cookie, + &rp->xname, pptr.namelen); + if (error) + goto out_unlock; + rp->xname.len = pptr.namelen; + mutex_unlock(&rp->pscan.lock); + + error = xrep_parent_replay_update(rp, &rp->xname, &pptr); + if (error) + return error; + + mutex_lock(&rp->pscan.lock); + } + + /* Empty out both arrays now that we've added the entries. */ + xfarray_truncate(rp->pptr_recs); + xfblob_truncate(rp->pptr_names); + mutex_unlock(&rp->pscan.lock); + return 0; +out_unlock: + mutex_unlock(&rp->pscan.lock); + return error; +} + +/* + * Remember that we want to create a parent pointer in the tempfile. These + * stashed actions will be replayed later. + */ +STATIC int +xrep_parent_stash_parentadd( + struct xrep_parent *rp, + const struct xfs_name *name, + const struct xfs_inode *dp) +{ + struct xrep_pptr pptr = { + .action = XREP_PPTR_ADD, + .namelen = name->len, + }; + int error; + + trace_xrep_parent_stash_parentadd(rp->sc->tempip, dp, name); + + xfs_inode_to_parent_rec(&pptr.pptr_rec, dp); + error = xfblob_storename(rp->pptr_names, &pptr.name_cookie, name); + if (error) + return error; + + return xfarray_append(rp->pptr_recs, &pptr); +} + +/* + * Remember that we want to remove a parent pointer from the tempfile. These + * stashed actions will be replayed later. + */ +STATIC int +xrep_parent_stash_parentremove( + struct xrep_parent *rp, + const struct xfs_name *name, + const struct xfs_inode *dp) +{ + struct xrep_pptr pptr = { + .action = XREP_PPTR_REMOVE, + .namelen = name->len, + }; + int error; + + trace_xrep_parent_stash_parentremove(rp->sc->tempip, dp, name); + + xfs_inode_to_parent_rec(&pptr.pptr_rec, dp); + error = xfblob_storename(rp->pptr_names, &pptr.name_cookie, name); + if (error) + return error; + + return xfarray_append(rp->pptr_recs, &pptr); +} + +/* + * Examine an entry of a directory. If this dirent leads us back to the file + * whose parent pointers we're rebuilding, add a pptr to the temporary + * directory. + */ +STATIC int +xrep_parent_scan_dirent( + struct xfs_scrub *sc, + struct xfs_inode *dp, + xfs_dir2_dataptr_t dapos, + const struct xfs_name *name, + xfs_ino_t ino, + void *priv) +{ + struct xrep_parent *rp = priv; + int error; + + /* Dirent doesn't point to this directory. */ + if (ino != I_INO(rp->sc->ip)) + return 0; + + /* No weird looking names. */ + if (name->len == 0 || !xfs_dir2_namecheck(name->name, name->len)) + return -EFSCORRUPTED; + + /* No mismatching ftypes. */ + if (name->type != xfs_mode_to_ftype(VFS_I(sc->ip)->i_mode)) + return -EFSCORRUPTED; + + /* Don't pick up dot or dotdot entries; we only want child dirents. */ + if (xfs_dir2_samename(name, &xfs_name_dotdot) || + xfs_dir2_samename(name, &xfs_name_dot)) + return 0; + + /* + * Transform this dirent into a parent pointer and queue it for later + * addition to the temporary file. + */ + mutex_lock(&rp->pscan.lock); + error = xrep_parent_stash_parentadd(rp, name, dp); + mutex_unlock(&rp->pscan.lock); + return error; +} + +/* + * Decide if we want to look for dirents in this directory. Skip the file + * being repaired and any files being used to stage repairs. + */ +static inline bool +xrep_parent_want_scan( + struct xrep_parent *rp, + const struct xfs_inode *ip) +{ + return ip != rp->sc->ip && !xrep_is_tempfile(ip); +} + +/* + * Take ILOCK on a file that we want to scan. + * + * Select ILOCK_EXCL if the file is a directory with an unloaded data bmbt. + * Otherwise, take ILOCK_SHARED. + */ +static inline unsigned int +xrep_parent_scan_ilock( + struct xrep_parent *rp, + struct xfs_inode *ip) +{ + uint lock_mode = XFS_ILOCK_SHARED; + + /* Still need to take the shared ILOCK to advance the iscan cursor. */ + if (!xrep_parent_want_scan(rp, ip)) + goto lock; + + if (S_ISDIR(VFS_I(ip)->i_mode) && xfs_need_iread_extents(&ip->i_df)) { + lock_mode = XFS_ILOCK_EXCL; + goto lock; + } + +lock: + xfs_ilock(ip, lock_mode); + return lock_mode; +} + +/* + * Scan this file for relevant child dirents that point to the file whose + * parent pointers we're rebuilding. + */ +STATIC int +xrep_parent_scan_file( + struct xrep_parent *rp, + struct xfs_inode *ip) +{ + unsigned int lock_mode; + int error = 0; + + lock_mode = xrep_parent_scan_ilock(rp, ip); + + if (!xrep_parent_want_scan(rp, ip)) + goto scan_done; + + if (S_ISDIR(VFS_I(ip)->i_mode)) { + /* + * If the directory looks as though it has been zapped by the + * inode record repair code, we cannot scan for child dirents. + */ + if (xchk_dir_looks_zapped(ip)) { + error = -EBUSY; + goto scan_done; + } + + error = xchk_dir_walk(rp->sc, ip, xrep_parent_scan_dirent, rp); + if (error) + goto scan_done; + } + +scan_done: + xchk_iscan_mark_visited(&rp->pscan.iscan, ip); + xfs_iunlock(ip, lock_mode); + return error; +} + +/* Decide if we've stashed too much pptr data in memory. */ +static inline bool +xrep_parent_want_flush_stashed( + struct xrep_parent *rp) +{ + unsigned long long bytes; + + bytes = xfarray_bytes(rp->pptr_recs) + xfblob_bytes(rp->pptr_names); + return bytes > XREP_PARENT_MAX_STASH_BYTES; +} + +/* + * Scan all directories in the filesystem to look for dirents that we can turn + * into parent pointers. + */ +STATIC int +xrep_parent_scan_dirtree( + struct xrep_parent *rp) +{ + struct xfs_scrub *sc = rp->sc; + struct xfs_inode *ip; + int error; + + /* + * Filesystem scans are time consuming. Drop the file ILOCK and all + * other resources for the duration of the scan and hope for the best. + * The live update hooks will keep our scan information up to date. + */ + xchk_trans_cancel(sc); + if (sc->ilock_flags & (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)) + xchk_iunlock(sc, sc->ilock_flags & (XFS_ILOCK_SHARED | + XFS_ILOCK_EXCL)); + xchk_trans_alloc_empty(sc); + + while ((error = xchk_iscan_iter(&rp->pscan.iscan, &ip)) == 1) { + bool flush; + + error = xrep_parent_scan_file(rp, ip); + xchk_irele(sc, ip); + if (error) + break; + + /* Flush stashed pptr updates to constrain memory usage. */ + mutex_lock(&rp->pscan.lock); + flush = xrep_parent_want_flush_stashed(rp); + mutex_unlock(&rp->pscan.lock); + if (flush) { + xchk_trans_cancel(sc); + + error = xrep_tempfile_iolock_polled(sc); + if (error) + break; + + error = xrep_parent_replay_updates(rp); + xrep_tempfile_iounlock(sc); + if (error) + break; + + xchk_trans_alloc_empty(sc); + } + + if (xchk_should_terminate(sc, &error)) + break; + } + xchk_iscan_iter_finish(&rp->pscan.iscan); + if (error) { + /* + * If we couldn't grab an inode that was busy with a state + * change, change the error code so that we exit to userspace + * as quickly as possible. + */ + if (error == -EBUSY) + return -ECANCELED; + return error; + } + + /* + * Retake sc->ip's ILOCK now that we're done flushing stashed parent + * pointers. We end this function with an empty transaction and the + * ILOCK. + */ + xchk_ilock(rp->sc, XFS_ILOCK_EXCL); + return 0; +} + +/* + * Capture dirent updates being made by other threads which are relevant to the + * file being repaired. + */ +STATIC int +xrep_parent_live_update( + struct notifier_block *nb, + unsigned long action, + void *data) +{ + struct xfs_dir_update_params *p = data; + struct xrep_parent *rp; + struct xfs_scrub *sc; + int error; + + rp = container_of(nb, struct xrep_parent, pscan.dhook.dirent_hook.nb); + sc = rp->sc; + + /* + * This thread updated a dirent that points to the file that we're + * repairing, so stash the update for replay against the temporary + * file. + */ + if (I_INO(p->ip) == I_INO(sc->ip) && + xchk_iscan_want_live_update(&rp->pscan.iscan, I_INO(p->dp))) { + mutex_lock(&rp->pscan.lock); + if (p->delta > 0) + error = xrep_parent_stash_parentadd(rp, p->name, p->dp); + else + error = xrep_parent_stash_parentremove(rp, p->name, + p->dp); + if (!error) + rp->saw_pptr_updates = true; + mutex_unlock(&rp->pscan.lock); + if (error) + goto out_abort; + } + + return NOTIFY_DONE; +out_abort: + xchk_iscan_abort(&rp->pscan.iscan); + return NOTIFY_DONE; +} + +/* Reset a directory's dotdot entry, if needed. */ +STATIC int +xrep_parent_reset_dotdot( + struct xrep_parent *rp) +{ + struct xfs_scrub *sc = rp->sc; + xfs_ino_t ino; + unsigned int spaceres; + int error = 0; + + ASSERT(sc->ilock_flags & XFS_ILOCK_EXCL); + + error = xchk_dir_lookup(sc, sc->ip, &xfs_name_dotdot, &ino); + if (error || ino == rp->pscan.parent_ino) + return error; + + xfs_trans_ijoin(sc->tp, sc->ip, 0); + + trace_xrep_parent_reset_dotdot(sc->ip, rp->pscan.parent_ino); + + /* + * Reserve more space just in case we have to expand the dir. We're + * allowed to exceed quota to repair inconsistent metadata. + */ + spaceres = xfs_rename_space_res(sc->mp, 0, false, xfs_name_dotdot.len, + false); + error = xfs_trans_reserve_more_inode(sc->tp, sc->ip, spaceres, 0, + true); + if (error) + return error; + + error = xfs_dir_replace(sc->tp, sc->ip, &xfs_name_dotdot, + rp->pscan.parent_ino, spaceres); + if (error) + return error; + + /* + * Roll transaction to detach the inode from the transaction but retain + * ILOCK_EXCL. + */ + return xfs_trans_roll(&sc->tp); +} + +/* Pass back the parent inumber if this a parent pointer */ +STATIC int +xrep_parent_lookup_pptr( + struct xfs_scrub *sc, + struct xfs_inode *ip, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + const void *value, + unsigned int valuelen, + void *priv) +{ + xfs_ino_t *inop = priv; + xfs_ino_t parent_ino; + int error; + + if (!(attr_flags & XFS_ATTR_PARENT)) + return 0; + + error = xfs_parent_from_attr(sc->mp, attr_flags, name, namelen, value, + valuelen, &parent_ino, NULL); + if (error) + return error; + + *inop = parent_ino; + return -ECANCELED; +} + +/* + * Find the first parent of the scrub target by walking parent pointers for + * the purpose of deciding if we're going to move it to the orphanage. + * We don't care if the attr fork is zapped. + */ +STATIC int +xrep_parent_lookup_pptrs( + struct xfs_scrub *sc, + xfs_ino_t *inop) +{ + int error; + + *inop = NULLFSINO; + + error = xchk_xattr_walk(sc, sc->ip, xrep_parent_lookup_pptr, NULL, + inop); + if (error && error != -ECANCELED) + return error; + return 0; +} + +/* + * Move the current file to the orphanage. + * + * Caller must hold IOLOCK_EXCL on @sc->ip, and no other inode locks. Upon + * successful return, the scrub transaction will have enough extra reservation + * to make the move; it will hold IOLOCK_EXCL and ILOCK_EXCL of @sc->ip and the + * orphanage; and both inodes will be ijoined. + */ +STATIC int +xrep_parent_move_to_orphanage( + struct xrep_parent *rp) +{ + struct xfs_scrub *sc = rp->sc; + xfs_ino_t orig_parent, new_parent; + int error; + + if (S_ISDIR(VFS_I(sc->ip)->i_mode)) { + /* + * We are about to drop the ILOCK on sc->ip to lock the + * orphanage and prepare for the adoption. Therefore, look up + * the old dotdot entry for sc->ip so that we can compare it + * after we re-lock sc->ip. + */ + error = xchk_dir_lookup(sc, sc->ip, &xfs_name_dotdot, + &orig_parent); + if (error) + return error; + } else { + /* + * We haven't dropped the ILOCK since we committed the new + * xattr structure (and hence the new parent pointer records), + * which means that the file cannot have been moved in the + * directory tree, and there are no parents. + */ + orig_parent = NULLFSINO; + } + + /* + * Drop the ILOCK on the scrub target and commit the transaction. + * Adoption computes its own resource requirements and gathers the + * necessary components. + */ + error = xrep_trans_commit(sc); + if (error) + return error; + xchk_iunlock(sc, XFS_ILOCK_EXCL); + + /* If we can take the orphanage's iolock then we're ready to move. */ + if (!xrep_orphanage_ilock_nowait(sc, XFS_IOLOCK_EXCL)) { + xchk_iunlock(sc, sc->ilock_flags); + error = xrep_orphanage_iolock_two(sc); + if (error) + return error; + } + + /* Grab transaction and ILOCK the two files. */ + error = xrep_adoption_trans_alloc(sc, &rp->adoption); + if (error) + return error; + + error = xrep_adoption_compute_name(&rp->adoption, &rp->xname); + if (error) + return error; + + /* + * Now that we've reacquired the ILOCK on sc->ip, look up the dotdot + * entry again. If the parent changed or the child was unlinked while + * the child directory was unlocked, we don't need to move the child to + * the orphanage after all. For a non-directory, we have to scan for + * the first parent pointer to see if one has been added. + */ + if (S_ISDIR(VFS_I(sc->ip)->i_mode)) + error = xchk_dir_lookup(sc, sc->ip, &xfs_name_dotdot, + &new_parent); + else + error = xrep_parent_lookup_pptrs(sc, &new_parent); + if (error) + return error; + + /* + * Attach to the orphanage if we still have a linked directory and it + * hasn't been moved. + */ + if (orig_parent == new_parent && VFS_I(sc->ip)->i_nlink > 0) { + error = xrep_adoption_move(&rp->adoption); + if (error) + return error; + } + + /* + * Launder the scrub transaction so we can drop the orphanage ILOCK + * and IOLOCK. Return holding the scrub target's ILOCK and IOLOCK. + */ + error = xrep_adoption_trans_roll(&rp->adoption); + if (error) + return error; + + xrep_orphanage_iunlock(sc, XFS_ILOCK_EXCL); + xrep_orphanage_iunlock(sc, XFS_IOLOCK_EXCL); + return 0; +} + +/* Ensure that the xattr value buffer is large enough. */ +STATIC int +xrep_parent_alloc_xattr_value( + struct xrep_parent *rp, + size_t bufsize) +{ + void *new_val; + + if (rp->xattr_value_sz >= bufsize) + return 0; + + if (rp->xattr_value) { + kvfree(rp->xattr_value); + rp->xattr_value = NULL; + rp->xattr_value_sz = 0; + } + + new_val = kvmalloc(bufsize, XCHK_GFP_FLAGS); + if (!new_val) + return -ENOMEM; + + rp->xattr_value = new_val; + rp->xattr_value_sz = bufsize; + return 0; +} + +/* Retrieve the (remote) value of a non-pptr xattr. */ +STATIC int +xrep_parent_fetch_xattr_remote( + struct xrep_parent *rp, + struct xfs_inode *ip, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + unsigned int valuelen) +{ + struct xfs_scrub *sc = rp->sc; + struct xfs_da_args args = { + .attr_filter = attr_flags & XFS_ATTR_NSP_ONDISK_MASK, + .geo = sc->mp->m_attr_geo, + .whichfork = XFS_ATTR_FORK, + .dp = ip, + .name = name, + .namelen = namelen, + .trans = sc->tp, + .valuelen = valuelen, + .owner = I_INO(ip), + }; + int error; + + /* + * If we need a larger value buffer, try to allocate one. If that + * fails, return with -EDEADLOCK to try harder. + */ + error = xrep_parent_alloc_xattr_value(rp, valuelen); + if (error == -ENOMEM) + return -EDEADLOCK; + if (error) + return error; + + args.value = rp->xattr_value; + xfs_attr_sethash(&args); + return xfs_attr_get_ilocked(&args); +} + +/* Stash non-pptr attributes for later replay into the temporary file. */ +STATIC int +xrep_parent_stash_xattr( + struct xfs_scrub *sc, + struct xfs_inode *ip, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + const void *value, + unsigned int valuelen, + void *priv) +{ + struct xrep_parent_xattr key = { + .valuelen = valuelen, + .namelen = namelen, + .flags = attr_flags & XFS_ATTR_NSP_ONDISK_MASK, + }; + struct xrep_parent *rp = priv; + int error; + + if (attr_flags & (XFS_ATTR_INCOMPLETE | XFS_ATTR_PARENT)) + return 0; + + if (!value) { + error = xrep_parent_fetch_xattr_remote(rp, ip, attr_flags, + name, namelen, valuelen); + if (error) + return error; + + value = rp->xattr_value; + } + + trace_xrep_parent_stash_xattr(rp->sc->tempip, key.flags, (void *)name, + key.namelen, key.valuelen); + + error = xfblob_store(rp->xattr_blobs, &key.name_cookie, name, + key.namelen); + if (error) + return error; + + error = xfblob_store(rp->xattr_blobs, &key.value_cookie, value, + key.valuelen); + if (error) + return error; + + return xfarray_append(rp->xattr_records, &key); +} + +/* Insert one xattr key/value. */ +STATIC int +xrep_parent_insert_xattr( + struct xrep_parent *rp, + const struct xrep_parent_xattr *key) +{ + struct xfs_da_args args = { + .dp = rp->sc->tempip, + .attr_filter = key->flags, + .namelen = key->namelen, + .valuelen = key->valuelen, + .owner = I_INO(rp->sc->ip), + .geo = rp->sc->mp->m_attr_geo, + .whichfork = XFS_ATTR_FORK, + .op_flags = XFS_DA_OP_OKNOENT, + }; + int error; + + ASSERT(!(key->flags & XFS_ATTR_PARENT)); + + /* + * Grab pointers to the scrub buffer so that we can use them to insert + * attrs into the temp file. + */ + args.name = rp->xattr_name; + args.value = rp->xattr_value; + + /* + * The attribute name is stored near the end of the in-core buffer, + * though we reserve one more byte to ensure null termination. + */ + rp->xattr_name[XATTR_NAME_MAX] = 0; + + error = xfblob_load(rp->xattr_blobs, key->name_cookie, rp->xattr_name, + key->namelen); + if (error) + return error; + + error = xfblob_free(rp->xattr_blobs, key->name_cookie); + if (error) + return error; + + error = xfblob_load(rp->xattr_blobs, key->value_cookie, args.value, + key->valuelen); + if (error) + return error; + + error = xfblob_free(rp->xattr_blobs, key->value_cookie); + if (error) + return error; + + rp->xattr_name[key->namelen] = 0; + + trace_xrep_parent_insert_xattr(rp->sc->tempip, key->flags, + rp->xattr_name, key->namelen, key->valuelen); + + xfs_attr_sethash(&args); + return xfs_attr_set(&args, XFS_ATTRUPDATE_UPSERT, false); +} + +/* + * Periodically flush salvaged attributes to the temporary file. This is done + * to reduce the memory requirements of the xattr rebuild because files can + * contain millions of attributes. + */ +STATIC int +xrep_parent_flush_xattrs( + struct xrep_parent *rp) +{ + xfarray_idx_t array_cur; + int error; + + /* + * Entering this function, the scrub context has a reference to the + * inode being repaired, the temporary file, and the empty scrub + * transaction that we created for the xattr scan. We hold ILOCK_EXCL + * on the inode being repaired. + * + * To constrain kernel memory use, we occasionally flush salvaged + * xattrs from the xfarray and xfblob structures into the temporary + * file in preparation for exchanging the xattr structures at the end. + * Updating the temporary file requires a transaction, so we commit the + * scrub transaction and drop the ILOCK so that xfs_attr_set can + * allocate whatever transaction it wants. + * + * We still hold IOLOCK_EXCL on the inode being repaired, which + * prevents anyone from adding xattrs (or parent pointers) while we're + * flushing. + */ + xchk_trans_cancel(rp->sc); + xchk_iunlock(rp->sc, XFS_ILOCK_EXCL); + + /* + * Take the IOLOCK of the temporary file while we modify xattrs. This + * isn't strictly required because the temporary file is never revealed + * to userspace, but we follow the same locking rules. We still hold + * sc->ip's IOLOCK. + */ + error = xrep_tempfile_iolock_polled(rp->sc); + if (error) + return error; + + /* Add all the salvaged attrs to the temporary file. */ + foreach_xfarray_idx(rp->xattr_records, array_cur) { + struct xrep_parent_xattr key; + + error = xfarray_load(rp->xattr_records, array_cur, &key); + if (error) + return error; + + error = xrep_parent_insert_xattr(rp, &key); + if (error) + return error; + } + + /* Empty out both arrays now that we've added the entries. */ + xfarray_truncate(rp->xattr_records); + xfblob_truncate(rp->xattr_blobs); + + xrep_tempfile_iounlock(rp->sc); + + /* Recreate the empty transaction and relock the inode. */ + xchk_trans_alloc_empty(rp->sc); + xchk_ilock(rp->sc, XFS_ILOCK_EXCL); + return 0; +} + +/* Decide if we've stashed too much xattr data in memory. */ +static inline bool +xrep_parent_want_flush_xattrs( + struct xrep_parent *rp) +{ + unsigned long long bytes; + + bytes = xfarray_bytes(rp->xattr_records) + + xfblob_bytes(rp->xattr_blobs); + return bytes > XREP_PARENT_XATTR_MAX_STASH_BYTES; +} + +/* Flush staged attributes to the temporary file if we're over the limit. */ +STATIC int +xrep_parent_try_flush_xattrs( + struct xfs_scrub *sc, + void *priv) +{ + struct xrep_parent *rp = priv; + int error; + + if (!xrep_parent_want_flush_xattrs(rp)) + return 0; + + error = xrep_parent_flush_xattrs(rp); + if (error) + return error; + + /* + * If there were any parent pointer updates to the xattr structure + * while we dropped the ILOCK, the xattr structure is now stale. + * Signal to the attr copy process that we need to start over, but + * this time without opportunistic attr flushing. + * + * This is unlikely to happen, so we're ok with restarting the copy. + */ + mutex_lock(&rp->pscan.lock); + if (rp->saw_pptr_updates) + error = -ESTALE; + mutex_unlock(&rp->pscan.lock); + return error; +} + +/* Copy all the non-pptr extended attributes into the temporary file. */ +STATIC int +xrep_parent_copy_xattrs( + struct xrep_parent *rp) +{ + struct xfs_scrub *sc = rp->sc; + int error; + + /* + * Clear the pptr updates flag. We hold sc->ip ILOCKed, so there + * can't be any parent pointer updates in progress. + */ + mutex_lock(&rp->pscan.lock); + rp->saw_pptr_updates = false; + mutex_unlock(&rp->pscan.lock); + + /* Copy xattrs, stopping periodically to flush the incore buffers. */ + error = xchk_xattr_walk(sc, sc->ip, xrep_parent_stash_xattr, + xrep_parent_try_flush_xattrs, rp); + if (error && error != -ESTALE) + return error; + + if (error == -ESTALE) { + /* + * The xattr copy collided with a parent pointer update. + * Restart the copy, but this time hold the ILOCK all the way + * to the end to lock out any directory parent pointer updates. + */ + error = xchk_xattr_walk(sc, sc->ip, xrep_parent_stash_xattr, + NULL, rp); + if (error) + return error; + } + + /* Flush any remaining stashed xattrs to the temporary file. */ + if (xfarray_bytes(rp->xattr_records) == 0) + return 0; + + return xrep_parent_flush_xattrs(rp); +} + +/* + * Ensure that @sc->ip and @sc->tempip both have attribute forks before we head + * into the attr fork exchange transaction. All files on a filesystem with + * parent pointers must have an attr fork because the parent pointer code does + * not itself add attribute forks. + * + * Note: Unlinkable unlinked files don't need one, but the overhead of having + * an unnecessary attr fork is not justified by the additional code complexity + * that would be needed to track that state correctly. + */ +STATIC int +xrep_parent_ensure_attr_fork( + struct xrep_parent *rp) +{ + struct xfs_scrub *sc = rp->sc; + int error; + + error = xfs_attr_add_fork(sc->tempip, + sizeof(struct xfs_attr_sf_hdr), 1); + if (error) + return error; + return xfs_attr_add_fork(sc->ip, sizeof(struct xfs_attr_sf_hdr), 1); +} + +/* + * Finish replaying stashed parent pointer updates, allocate a transaction for + * exchanging extent mappings, and take the ILOCKs of both files before we + * commit the new attribute structure. + */ +STATIC int +xrep_parent_finalize_tempfile( + struct xrep_parent *rp) +{ + struct xfs_scrub *sc = rp->sc; + int error; + + /* + * Repair relies on the ILOCK to quiesce all possible xattr updates. + * Replay all queued parent pointer updates into the tempfile before + * exchanging the contents, even if that means dropping the ILOCKs and + * the transaction. + */ + do { + error = xrep_parent_replay_updates(rp); + if (error) + return error; + + error = xrep_parent_ensure_attr_fork(rp); + if (error) + return error; + + error = xrep_tempexch_trans_alloc(sc, XFS_ATTR_FORK, &rp->tx); + if (error) + return error; + + if (xfarray_length(rp->pptr_recs) == 0) + break; + + xchk_trans_cancel(sc); + xrep_tempfile_iunlock_both(sc); + } while (!xchk_should_terminate(sc, &error)); + return error; +} + +/* + * Replay all the stashed parent pointers into the temporary file, copy all + * the non-pptr xattrs from the file being repaired into the temporary file, + * and exchange the attr fork contents atomically. + */ +STATIC int +xrep_parent_rebuild_pptrs( + struct xrep_parent *rp) +{ + struct xfs_scrub *sc = rp->sc; + xfs_ino_t parent_ino = NULLFSINO; + int error; + + /* + * Copy non-ppttr xattrs from the file being repaired into the + * temporary file's xattr structure. We hold sc->ip's IOLOCK, which + * prevents setxattr/removexattr calls from occurring, but renames + * update the parent pointers without holding IOLOCK. If we detect + * stale attr structures, we restart the scan but only flush at the + * end. + */ + error = xrep_parent_copy_xattrs(rp); + if (error) + return error; + + /* + * Cancel the empty transaction that we used to walk and copy attrs, + * and drop the ILOCK so that we can take the IOLOCK on the temporary + * file. We still hold sc->ip's IOLOCK. + */ + xchk_trans_cancel(sc); + xchk_iunlock(sc, XFS_ILOCK_EXCL); + + error = xrep_tempfile_iolock_polled(sc); + if (error) + return error; + + /* + * Allocate transaction, lock inodes, and make sure that we've replayed + * all the stashed pptr updates to the tempdir. After this point, + * we're ready to exchange the attr fork mappings. + */ + error = xrep_parent_finalize_tempfile(rp); + if (error) + return error; + + /* Last chance to abort before we start committing pptr fixes. */ + if (xchk_should_terminate(sc, &error)) + return error; + + if (xchk_iscan_aborted(&rp->pscan.iscan)) + return -ECANCELED; + + /* + * Exchange the attr fork contents and junk the old attr fork contents, + * which are now in the tempfile. + */ + error = xrep_xattr_swap(sc, &rp->tx); + if (error) + return error; + error = xrep_xattr_reset_tempfile_fork(sc); + if (error) + return error; + + /* + * Roll to get a transaction without any inodes joined to it. Then we + * can drop the tempfile's ILOCK and IOLOCK before doing more work on + * the scrub target file. + */ + error = xfs_trans_roll(&sc->tp); + if (error) + return error; + xrep_tempfile_iunlock(sc); + xrep_tempfile_iounlock(sc); + + /* + * We've committed the new parent pointers. Find at least one parent + * so that we can decide if we're moving this file to the orphanage. + * For this purpose, root directories are their own parents. + */ + if (xchk_inode_is_dirtree_root(sc->ip)) { + xrep_findparent_scan_found(&rp->pscan, I_INO(sc->ip)); + } else { + error = xrep_parent_lookup_pptrs(sc, &parent_ino); + if (error) + return error; + if (parent_ino != NULLFSINO) + xrep_findparent_scan_found(&rp->pscan, parent_ino); + } + return 0; +} + +/* + * Commit the new parent pointer structure (currently only the dotdot entry) to + * the file that we're repairing. + */ +STATIC int +xrep_parent_rebuild_tree( + struct xrep_parent *rp) +{ + struct xfs_scrub *sc = rp->sc; + bool try_adoption; + int error; + + if (xfs_has_parent(sc->mp)) { + error = xrep_parent_rebuild_pptrs(rp); + if (error) + return error; + } + + /* + * Any file with no parent could be adopted. This check happens after + * rebuilding the parent pointer structure because we might have cycled + * the ILOCK during that process. + */ + try_adoption = rp->pscan.parent_ino == NULLFSINO; + + /* + * Starting with metadir, we allow checking of parent pointers + * of non-directory files that are children of the superblock. + * Lack of parent is ok here. + */ + if (try_adoption && xfs_has_metadir(sc->mp) && + xchk_inode_is_sb_rooted(sc->ip)) + try_adoption = false; + + if (try_adoption) { + if (xrep_orphanage_can_adopt(sc)) + return xrep_parent_move_to_orphanage(rp); + return -EFSCORRUPTED; + + } + + if (S_ISDIR(VFS_I(sc->ip)->i_mode)) + return xrep_parent_reset_dotdot(rp); + + return 0; +} + +/* Count the number of parent pointers. */ +STATIC int +xrep_parent_count_pptr( + struct xfs_scrub *sc, + struct xfs_inode *ip, + unsigned int attr_flags, + const unsigned char *name, + unsigned int namelen, + const void *value, + unsigned int valuelen, + void *priv) +{ + struct xrep_parent *rp = priv; + int error; + + if (!(attr_flags & XFS_ATTR_PARENT)) + return 0; + + error = xfs_parent_from_attr(sc->mp, attr_flags, name, namelen, value, + valuelen, NULL, NULL); + if (error) + return error; + + rp->parents++; + return 0; +} + +/* + * After all parent pointer rebuilding and adoption activity completes, reset + * the link count of this nondirectory, having scanned the fs to rebuild all + * parent pointers. + */ +STATIC int +xrep_parent_set_nondir_nlink( + struct xrep_parent *rp) +{ + struct xfs_scrub *sc = rp->sc; + struct xfs_inode *ip = sc->ip; + struct xfs_perag *pag; + bool joined = false; + int error; + + /* Count parent pointers so we can reset the file link count. */ + rp->parents = 0; + error = xchk_xattr_walk(sc, ip, xrep_parent_count_pptr, NULL, rp); + if (error) + return error; + + /* + * Starting with metadir, we allow checking of parent pointers of + * non-directory files that are children of the superblock. Pretend + * that we found a parent pointer attr. + */ + if (xfs_has_metadir(sc->mp) && xchk_inode_is_sb_rooted(sc->ip)) + rp->parents++; + + if (rp->parents > 0 && xfs_inode_on_unlinked_list(ip)) { + xfs_trans_ijoin(sc->tp, sc->ip, 0); + joined = true; + + /* + * The file is on the unlinked list but we found parents. + * Remove the file from the unlinked list. + */ + pag = xfs_perag_get(sc->mp, XFS_INODE_TO_AGNO(ip)); + if (!pag) { + ASSERT(0); + return -EFSCORRUPTED; + } + + error = xfs_iunlink_remove(sc->tp, pag, ip); + xfs_perag_put(pag); + if (error) + return error; + } else if (rp->parents == 0 && !xfs_inode_on_unlinked_list(ip)) { + xfs_trans_ijoin(sc->tp, sc->ip, 0); + joined = true; + + /* + * The file is not on the unlinked list but we found no + * parents. Add the file to the unlinked list. + */ + error = xfs_iunlink(sc->tp, ip); + if (error) + return error; + } + + /* Set the correct link count. */ + if (VFS_I(ip)->i_nlink != rp->parents) { + if (!joined) { + xfs_trans_ijoin(sc->tp, sc->ip, 0); + joined = true; + } + + set_nlink(VFS_I(ip), min_t(unsigned long long, rp->parents, + XFS_NLINK_PINNED)); + } + + /* Log the inode to keep it moving forward if we dirtied anything. */ + if (joined) + xfs_trans_log_inode(sc->tp, ip, XFS_ILOG_CORE); + return 0; +} + +/* Set up the filesystem scan so we can look for parents. */ +STATIC int +xrep_parent_setup_scan( + struct xrep_parent *rp) +{ + struct xfs_scrub *sc = rp->sc; + struct xfs_da_geometry *geo = sc->mp->m_attr_geo; + int max_len; + int error; + + if (!xfs_has_parent(sc->mp)) + return xrep_findparent_scan_start(sc, &rp->pscan); + + /* Buffers for copying non-pptr attrs to the tempfile */ + rp->xattr_name = kvmalloc(XATTR_NAME_MAX + 1, XCHK_GFP_FLAGS); + if (!rp->xattr_name) + return -ENOMEM; + + /* + * Allocate enough memory to handle loading local attr values from the + * xfblob data while flushing stashed attrs to the temporary file. + * We only realloc the buffer when salvaging remote attr values, so + * TRY_HARDER means we allocate the maximal attr value size. + */ + if (sc->flags & XCHK_TRY_HARDER) + max_len = XATTR_SIZE_MAX; + else + max_len = xfs_attr_leaf_entsize_local_max(geo->blksize); + error = xrep_parent_alloc_xattr_value(rp, max_len); + if (error) + goto out_xattr_name; + + /* Set up some staging memory for logging parent pointer updates. */ + error = xfarray_create("parent pointer entries", 0, + sizeof(struct xrep_pptr), &rp->pptr_recs); + if (error) + goto out_xattr_value; + + error = xfblob_create("parent pointer names", &rp->pptr_names); + if (error) + goto out_recs; + + /* Set up some storage for copying attrs before the mapping exchange */ + error = xfarray_create("parent pointer xattr entries", 0, + sizeof(struct xrep_parent_xattr), &rp->xattr_records); + if (error) + goto out_names; + + error = xfblob_create("parent pointer xattr values", &rp->xattr_blobs); + if (error) + goto out_attr_keys; + + error = __xrep_findparent_scan_start(sc, &rp->pscan, + xrep_parent_live_update); + if (error) + goto out_attr_values; + + return 0; + +out_attr_values: + xfblob_destroy(rp->xattr_blobs); + rp->xattr_blobs = NULL; +out_attr_keys: + xfarray_destroy(rp->xattr_records); + rp->xattr_records = NULL; +out_names: + xfblob_destroy(rp->pptr_names); + rp->pptr_names = NULL; +out_recs: + xfarray_destroy(rp->pptr_recs); + rp->pptr_recs = NULL; +out_xattr_value: + kvfree(rp->xattr_value); + rp->xattr_value = NULL; +out_xattr_name: + kvfree(rp->xattr_name); + rp->xattr_name = NULL; + return error; +} + +int +xrep_parent( + struct xfs_scrub *sc) +{ + struct xrep_parent *rp = sc->buf; + int error; + + /* + * When the parent pointers feature is enabled, repairs are committed + * by atomically committing a new xattr structure and reaping the old + * attr fork. Reaping requires rmap and exchange-range to be enabled. + */ + if (xfs_has_parent(sc->mp)) { + if (!xfs_has_rmapbt(sc->mp)) + return -EOPNOTSUPP; + if (!xfs_has_exchange_range(sc->mp)) + return -EOPNOTSUPP; + } + + error = xrep_parent_setup_scan(rp); + if (error) + return error; + + if (xfs_has_parent(sc->mp)) + error = xrep_parent_scan_dirtree(rp); + else + error = xrep_parent_find_dotdot(rp); + if (error) + goto out_teardown; + + /* Last chance to abort before we start committing dotdot fixes. */ + if (xchk_should_terminate(sc, &error)) + goto out_teardown; + + error = xrep_parent_rebuild_tree(rp); + if (error) + goto out_teardown; + if (xfs_has_parent(sc->mp) && !S_ISDIR(VFS_I(sc->ip)->i_mode)) { + error = xrep_parent_set_nondir_nlink(rp); + if (error) + goto out_teardown; + } + + error = xrep_defer_finish(sc); + +out_teardown: + xrep_parent_teardown(rp); + return error; +} diff --git a/fs/xfs/scrub/quota.c b/fs/xfs/scrub/quota.c new file mode 100644 index 000000000..222812fe2 --- /dev/null +++ b/fs/xfs/scrub/quota.c @@ -0,0 +1,345 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_bit.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_inode.h" +#include "xfs_quota.h" +#include "xfs_qm.h" +#include "xfs_bmap.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/quota.h" + +/* Convert a scrub type code to a DQ flag, or return 0 if error. */ +xfs_dqtype_t +xchk_quota_to_dqtype( + struct xfs_scrub *sc) +{ + switch (sc->sm->sm_type) { + case XFS_SCRUB_TYPE_UQUOTA: + return XFS_DQTYPE_USER; + case XFS_SCRUB_TYPE_GQUOTA: + return XFS_DQTYPE_GROUP; + case XFS_SCRUB_TYPE_PQUOTA: + return XFS_DQTYPE_PROJ; + default: + return 0; + } +} + +/* Set us up to scrub a quota. */ +int +xchk_setup_quota( + struct xfs_scrub *sc) +{ + xfs_dqtype_t dqtype; + int error; + + if (!XFS_IS_QUOTA_ON(sc->mp)) + return -ENOENT; + + dqtype = xchk_quota_to_dqtype(sc); + if (dqtype == 0) + return -EINVAL; + + if (!xfs_this_quota_on(sc->mp, dqtype)) + return -ENOENT; + + if (xchk_need_intent_drain(sc)) + xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN); + + error = xchk_setup_fs(sc); + if (error) + return error; + + error = xchk_install_live_inode(sc, xfs_quota_inode(sc->mp, dqtype)); + if (error) + return error; + + xchk_ilock(sc, XFS_ILOCK_EXCL); + return 0; +} + +/* Quotas. */ + +struct xchk_quota_info { + struct xfs_scrub *sc; + xfs_dqid_t last_id; +}; + +/* There's a written block backing this dquot, right? */ +STATIC int +xchk_quota_item_bmap( + struct xfs_scrub *sc, + struct xfs_dquot *dq, + xfs_fileoff_t offset) +{ + struct xfs_bmbt_irec irec; + struct xfs_mount *mp = sc->mp; + int nmaps = 1; + int error; + + if (!xfs_verify_fileoff(mp, offset)) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset); + return 0; + } + + if (dq->q_fileoffset != offset) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset); + return 0; + } + + error = xfs_bmapi_read(sc->ip, offset, 1, &irec, &nmaps, 0); + if (error) + return error; + + if (nmaps != 1) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset); + return 0; + } + + if (!xfs_verify_fsbno(mp, irec.br_startblock)) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset); + if (XFS_FSB_TO_DADDR(mp, irec.br_startblock) != dq->q_blkno) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset); + if (!xfs_bmap_is_written_extent(&irec)) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset); + + return 0; +} + +/* Complain if a quota timer is incorrectly set. */ +static inline void +xchk_quota_item_timer( + struct xfs_scrub *sc, + xfs_fileoff_t offset, + const struct xfs_dquot_res *res) +{ + if ((res->softlimit && res->count > res->softlimit) || + (res->hardlimit && res->count > res->hardlimit)) { + if (!res->timer) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset); + } else { + if (res->timer) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset); + } +} + +/* Scrub the fields in an individual quota item. */ +STATIC int +xchk_quota_item( + struct xchk_quota_info *sqi, + struct xfs_dquot *dq) +{ + struct xfs_scrub *sc = sqi->sc; + struct xfs_mount *mp = sc->mp; + struct xfs_quotainfo *qi = mp->m_quotainfo; + xfs_fileoff_t offset; + xfs_ino_t fs_icount; + int error = 0; + + if (xchk_should_terminate(sc, &error)) + return error; + + /* + * We want to validate the bmap record for the storage backing this + * dquot, so we need to lock the dquot and the quota file. For quota + * operations, the locking order is first the ILOCK and then the dquot. + */ + xchk_ilock(sc, XFS_ILOCK_SHARED); + mutex_lock(&dq->q_qlock); + + /* + * Except for the root dquot, the actual dquot we got must either have + * the same or higher id as we saw before. + */ + offset = dq->q_id / qi->qi_dqperchunk; + if (dq->q_id && dq->q_id <= sqi->last_id) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset); + + sqi->last_id = dq->q_id; + + error = xchk_quota_item_bmap(sc, dq, offset); + xchk_iunlock(sc, XFS_ILOCK_SHARED); + if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, offset, &error)) { + mutex_unlock(&dq->q_qlock); + return error; + } + + /* + * Warn if the hard limits are larger than the fs. + * Administrators can do this, though in production this seems + * suspect, which is why we flag it for review. + * + * Complain about corruption if the soft limit is greater than + * the hard limit. + */ + if (dq->q_blk.hardlimit > mp->m_sb.sb_dblocks) + xchk_fblock_set_warning(sc, XFS_DATA_FORK, offset); + if (dq->q_blk.softlimit > dq->q_blk.hardlimit) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset); + + if (dq->q_ino.hardlimit > M_IGEO(mp)->maxicount) + xchk_fblock_set_warning(sc, XFS_DATA_FORK, offset); + if (dq->q_ino.softlimit > dq->q_ino.hardlimit) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset); + + if (dq->q_rtb.hardlimit > mp->m_sb.sb_rblocks) + xchk_fblock_set_warning(sc, XFS_DATA_FORK, offset); + if (dq->q_rtb.softlimit > dq->q_rtb.hardlimit) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset); + + /* Check the resource counts. */ + fs_icount = percpu_counter_sum(&mp->m_icount); + + /* + * Check that usage doesn't exceed physical limits. However, on + * a reflink filesystem we're allowed to exceed physical space + * if there are no quota limits. + */ + if (xfs_has_reflink(mp)) { + if (mp->m_sb.sb_dblocks < dq->q_blk.count) + xchk_fblock_set_warning(sc, XFS_DATA_FORK, + offset); + if (mp->m_sb.sb_rblocks < dq->q_rtb.count) + xchk_fblock_set_warning(sc, XFS_DATA_FORK, + offset); + } else { + if (mp->m_sb.sb_dblocks < dq->q_blk.count) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, + offset); + if (mp->m_sb.sb_rblocks < dq->q_rtb.count) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, + offset); + } + if (dq->q_ino.count > fs_icount) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, offset); + + /* + * We can violate the hard limits if the admin suddenly sets a + * lower limit than the actual usage. However, we flag it for + * admin review. + */ + if (dq->q_id == 0) + goto out; + + if (dq->q_blk.hardlimit != 0 && + dq->q_blk.count > dq->q_blk.hardlimit) + xchk_fblock_set_warning(sc, XFS_DATA_FORK, offset); + + if (dq->q_ino.hardlimit != 0 && + dq->q_ino.count > dq->q_ino.hardlimit) + xchk_fblock_set_warning(sc, XFS_DATA_FORK, offset); + + if (dq->q_rtb.hardlimit != 0 && + dq->q_rtb.count > dq->q_rtb.hardlimit) + xchk_fblock_set_warning(sc, XFS_DATA_FORK, offset); + + xchk_quota_item_timer(sc, offset, &dq->q_blk); + xchk_quota_item_timer(sc, offset, &dq->q_ino); + xchk_quota_item_timer(sc, offset, &dq->q_rtb); + +out: + mutex_unlock(&dq->q_qlock); + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return -ECANCELED; + + return 0; +} + +/* Check the quota's data fork. */ +STATIC int +xchk_quota_data_fork( + struct xfs_scrub *sc) +{ + struct xfs_bmbt_irec irec = { 0 }; + struct xfs_iext_cursor icur; + struct xfs_quotainfo *qi = sc->mp->m_quotainfo; + struct xfs_ifork *ifp; + xfs_fileoff_t max_dqid_off; + int error = 0; + + /* Invoke the fork scrubber. */ + error = xchk_metadata_inode_forks(sc); + if (error || (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) + return error; + + /* Check for data fork problems that apply only to quota files. */ + max_dqid_off = XFS_DQ_ID_MAX / qi->qi_dqperchunk; + ifp = xfs_ifork_ptr(sc->ip, XFS_DATA_FORK); + for_each_xfs_iext(ifp, &icur, &irec) { + if (xchk_should_terminate(sc, &error)) + break; + + /* + * delalloc/unwritten extents or blocks mapped above the highest + * quota id shouldn't happen. + */ + if (!xfs_bmap_is_written_extent(&irec) || + irec.br_startoff > max_dqid_off || + irec.br_startoff + irec.br_blockcount - 1 > max_dqid_off) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, + irec.br_startoff); + break; + } + } + + return error; +} + +/* Scrub all of a quota type's items. */ +int +xchk_quota( + struct xfs_scrub *sc) +{ + struct xchk_dqiter cursor = { }; + struct xchk_quota_info sqi = { .sc = sc }; + struct xfs_mount *mp = sc->mp; + struct xfs_quotainfo *qi = mp->m_quotainfo; + struct xfs_dquot *dq; + xfs_dqtype_t dqtype; + int error = 0; + + dqtype = xchk_quota_to_dqtype(sc); + + /* Look for problem extents. */ + error = xchk_quota_data_fork(sc); + if (error) + goto out; + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + goto out; + + /* + * Check all the quota items. Now that we've checked the quota inode + * data fork we have to drop ILOCK_EXCL to use the regular dquot + * functions. + */ + xchk_iunlock(sc, sc->ilock_flags); + + /* Now look for things that the quota verifiers won't complain about. */ + xchk_dqiter_init(&cursor, sc, dqtype); + while ((error = xchk_dquot_iter(&cursor, &dq)) == 1) { + error = xchk_quota_item(&sqi, dq); + xfs_qm_dqrele(dq); + if (error) + break; + } + if (error == -ECANCELED) + error = 0; + if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, + sqi.last_id * qi->qi_dqperchunk, &error)) + goto out; + +out: + return error; +} diff --git a/fs/xfs/scrub/quota.h b/fs/xfs/scrub/quota.h new file mode 100644 index 000000000..6c7134ce2 --- /dev/null +++ b/fs/xfs/scrub/quota.h @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_QUOTA_H__ +#define __XFS_SCRUB_QUOTA_H__ + +xfs_dqtype_t xchk_quota_to_dqtype(struct xfs_scrub *sc); + +/* dquot iteration code */ + +struct xchk_dqiter { + struct xfs_scrub *sc; + + /* Quota file that we're walking. */ + struct xfs_inode *quota_ip; + + /* Cached data fork mapping for the dquot. */ + struct xfs_bmbt_irec bmap; + + /* The next dquot to scan. */ + uint64_t id; + + /* Quota type (user/group/project). */ + xfs_dqtype_t dqtype; + + /* Data fork sequence number to detect stale mappings. */ + unsigned int if_seq; +}; + +void xchk_dqiter_init(struct xchk_dqiter *cursor, struct xfs_scrub *sc, + xfs_dqtype_t dqtype); +int xchk_dquot_iter(struct xchk_dqiter *cursor, struct xfs_dquot **dqpp); + +#endif /* __XFS_SCRUB_QUOTA_H__ */ diff --git a/fs/xfs/scrub/quota_repair.c b/fs/xfs/scrub/quota_repair.c new file mode 100644 index 000000000..487bd4f68 --- /dev/null +++ b/fs/xfs/scrub/quota_repair.c @@ -0,0 +1,566 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_btree.h" +#include "xfs_bit.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_sb.h" +#include "xfs_inode.h" +#include "xfs_inode_fork.h" +#include "xfs_alloc.h" +#include "xfs_bmap.h" +#include "xfs_quota.h" +#include "xfs_qm.h" +#include "xfs_dquot.h" +#include "xfs_dquot_item.h" +#include "xfs_reflink.h" +#include "xfs_bmap_btree.h" +#include "xfs_trans_space.h" +#include "scrub/xfs_scrub.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/quota.h" +#include "scrub/trace.h" +#include "scrub/repair.h" + +/* + * Quota Repair + * ============ + * + * Quota repairs are fairly simplistic; we fix everything that the dquot + * verifiers complain about, cap any counters or limits that make no sense, + * and schedule a quotacheck if we had to fix anything. We also repair any + * data fork extent records that don't apply to metadata files. + */ + +struct xrep_quota_info { + struct xfs_scrub *sc; + bool need_quotacheck; +}; + +/* + * Allocate a new block into a sparse hole in the quota file backing this + * dquot, initialize the block, and commit the whole mess. + */ +STATIC int +xrep_quota_item_fill_bmap_hole( + struct xfs_scrub *sc, + struct xfs_dquot *dq, + struct xfs_bmbt_irec *irec) +{ + struct xfs_buf *bp; + struct xfs_mount *mp = sc->mp; + int nmaps = 1; + int error; + + xfs_trans_ijoin(sc->tp, sc->ip, 0); + + /* Map a block into the file. */ + error = xfs_trans_reserve_more(sc->tp, XFS_QM_DQALLOC_SPACE_RES(mp), + 0); + if (error) + return error; + + error = xfs_bmapi_write(sc->tp, sc->ip, dq->q_fileoffset, + XFS_DQUOT_CLUSTER_SIZE_FSB, XFS_BMAPI_METADATA, 0, + irec, &nmaps); + if (error) + return error; + + dq->q_blkno = XFS_FSB_TO_DADDR(mp, irec->br_startblock); + + trace_xrep_dquot_item_fill_bmap_hole(sc->mp, dq->q_type, dq->q_id); + + /* Initialize the new block. */ + error = xfs_trans_get_buf(sc->tp, mp->m_ddev_targp, dq->q_blkno, + mp->m_quotainfo->qi_dqchunklen, 0, &bp); + if (error) + return error; + bp->b_ops = &xfs_dquot_buf_ops; + + xfs_qm_init_dquot_blk(sc->tp, dq->q_id, dq->q_type, bp); + xfs_buf_set_ref(bp, XFS_DQUOT_REF); + + /* + * Finish the mapping transactions and roll one more time to + * disconnect sc->ip from sc->tp. + */ + error = xrep_defer_finish(sc); + if (error) + return error; + return xfs_trans_roll(&sc->tp); +} + +/* Make sure there's a written block backing this dquot */ +STATIC int +xrep_quota_item_bmap( + struct xfs_scrub *sc, + struct xfs_dquot *dq, + bool *dirty) +{ + struct xfs_bmbt_irec irec; + struct xfs_mount *mp = sc->mp; + struct xfs_quotainfo *qi = mp->m_quotainfo; + xfs_fileoff_t offset = dq->q_id / qi->qi_dqperchunk; + int nmaps = 1; + int error; + + /* The computed file offset should always be valid. */ + if (!xfs_verify_fileoff(mp, offset)) { + ASSERT(xfs_verify_fileoff(mp, offset)); + return -EFSCORRUPTED; + } + dq->q_fileoffset = offset; + + error = xfs_bmapi_read(sc->ip, offset, 1, &irec, &nmaps, 0); + if (error) + return error; + + if (nmaps < 1 || !xfs_bmap_is_real_extent(&irec)) { + /* Hole/delalloc extent; allocate a real block. */ + error = xrep_quota_item_fill_bmap_hole(sc, dq, &irec); + if (error) + return error; + } else if (irec.br_state != XFS_EXT_NORM) { + /* Unwritten extent, which we already took care of? */ + ASSERT(irec.br_state == XFS_EXT_NORM); + return -EFSCORRUPTED; + } else if (dq->q_blkno != XFS_FSB_TO_DADDR(mp, irec.br_startblock)) { + /* + * If the cached daddr is incorrect, repair probably punched a + * hole out of the quota file and filled it back in with a new + * block. Update the block mapping in the dquot. + */ + dq->q_blkno = XFS_FSB_TO_DADDR(mp, irec.br_startblock); + } + + *dirty = true; + return 0; +} + +/* Reset quota timers if incorrectly set. */ +static inline void +xrep_quota_item_timer( + struct xfs_scrub *sc, + const struct xfs_dquot_res *res, + bool *dirty) +{ + if ((res->softlimit && res->count > res->softlimit) || + (res->hardlimit && res->count > res->hardlimit)) { + if (!res->timer) + *dirty = true; + } else { + if (res->timer) + *dirty = true; + } +} + +/* Scrub the fields in an individual quota item. */ +STATIC int +xrep_quota_item( + struct xrep_quota_info *rqi, + struct xfs_dquot *dq) +{ + struct xfs_scrub *sc = rqi->sc; + struct xfs_mount *mp = sc->mp; + xfs_ino_t fs_icount; + bool dirty = false; + int error = 0; + + /* Last chance to abort before we start committing fixes. */ + if (xchk_should_terminate(sc, &error)) + return error; + + /* + * We might need to fix holes in the bmap record for the storage + * backing this dquot, so we need to lock the dquot and the quota file. + */ + xchk_ilock(sc, XFS_ILOCK_EXCL); + mutex_lock(&dq->q_qlock); + error = xrep_quota_item_bmap(sc, dq, &dirty); + xchk_iunlock(sc, XFS_ILOCK_EXCL); + if (error) + goto out_unlock_dquot; + + /* Check the limits. */ + if (dq->q_blk.softlimit > dq->q_blk.hardlimit) { + dq->q_blk.softlimit = dq->q_blk.hardlimit; + dirty = true; + } + + if (dq->q_ino.softlimit > dq->q_ino.hardlimit) { + dq->q_ino.softlimit = dq->q_ino.hardlimit; + dirty = true; + } + + if (dq->q_rtb.softlimit > dq->q_rtb.hardlimit) { + dq->q_rtb.softlimit = dq->q_rtb.hardlimit; + dirty = true; + } + + /* + * Check that usage doesn't exceed physical limits. However, on + * a reflink filesystem we're allowed to exceed physical space + * if there are no quota limits. We don't know what the real number + * is, but we can make quotacheck find out for us. + */ + if (!xfs_has_reflink(mp) && dq->q_blk.count > mp->m_sb.sb_dblocks) { + dq->q_blk.reserved -= dq->q_blk.count; + dq->q_blk.reserved += mp->m_sb.sb_dblocks; + dq->q_blk.count = mp->m_sb.sb_dblocks; + rqi->need_quotacheck = true; + dirty = true; + } + fs_icount = percpu_counter_sum(&mp->m_icount); + if (dq->q_ino.count > fs_icount) { + dq->q_ino.reserved -= dq->q_ino.count; + dq->q_ino.reserved += fs_icount; + dq->q_ino.count = fs_icount; + rqi->need_quotacheck = true; + dirty = true; + } + if (!xfs_has_reflink(mp) && dq->q_rtb.count > mp->m_sb.sb_rblocks) { + dq->q_rtb.reserved -= dq->q_rtb.count; + dq->q_rtb.reserved += mp->m_sb.sb_rblocks; + dq->q_rtb.count = mp->m_sb.sb_rblocks; + rqi->need_quotacheck = true; + dirty = true; + } + + xrep_quota_item_timer(sc, &dq->q_blk, &dirty); + xrep_quota_item_timer(sc, &dq->q_ino, &dirty); + xrep_quota_item_timer(sc, &dq->q_rtb, &dirty); + + if (!dirty) + goto out_unlock_dquot; + + trace_xrep_dquot_item(sc->mp, dq->q_type, dq->q_id); + + dq->q_flags |= XFS_DQFLAG_DIRTY; + xfs_trans_dqjoin(sc->tp, dq); + if (dq->q_id) { + xfs_qm_adjust_dqlimits(dq); + xfs_qm_adjust_dqtimers(dq); + } + xfs_trans_log_dquot(sc->tp, dq); + return xfs_trans_roll(&sc->tp); + +out_unlock_dquot: + mutex_unlock(&dq->q_qlock); + return error; +} + +/* Fix a quota timer so that we can pass the verifier. */ +STATIC void +xrep_quota_fix_timer( + struct xfs_mount *mp, + const struct xfs_disk_dquot *ddq, + __be64 softlimit, + __be64 countnow, + __be32 *timer, + time64_t timelimit) +{ + uint64_t soft = be64_to_cpu(softlimit); + uint64_t count = be64_to_cpu(countnow); + time64_t new_timer; + uint32_t t; + + if (!soft || count <= soft || *timer != 0) + return; + + new_timer = xfs_dquot_set_timeout(mp, + ktime_get_real_seconds() + timelimit); + if (ddq->d_type & XFS_DQTYPE_BIGTIME) + t = xfs_dq_unix_to_bigtime(new_timer); + else + t = new_timer; + + *timer = cpu_to_be32(t); +} + +/* Fix anything the verifiers complain about. */ +STATIC int +xrep_quota_block( + struct xfs_scrub *sc, + xfs_daddr_t daddr, + xfs_dqtype_t dqtype, + xfs_dqid_t id) +{ + struct xfs_dqblk *dqblk; + struct xfs_disk_dquot *ddq; + struct xfs_quotainfo *qi = sc->mp->m_quotainfo; + struct xfs_def_quota *defq = xfs_get_defquota(qi, dqtype); + struct xfs_buf *bp = NULL; + enum xfs_blft buftype = 0; + int i; + int error; + + error = xfs_trans_read_buf(sc->mp, sc->tp, sc->mp->m_ddev_targp, daddr, + qi->qi_dqchunklen, 0, &bp, &xfs_dquot_buf_ops); + switch (error) { + case -EFSBADCRC: + case -EFSCORRUPTED: + /* Failed verifier, retry read with no ops. */ + error = xfs_trans_read_buf(sc->mp, sc->tp, + sc->mp->m_ddev_targp, daddr, qi->qi_dqchunklen, + 0, &bp, NULL); + if (error) + return error; + break; + case 0: + dqblk = bp->b_addr; + ddq = &dqblk[0].dd_diskdq; + + /* + * If there's nothing that would impede a dqiterate, we're + * done. + */ + if ((ddq->d_type & XFS_DQTYPE_REC_MASK) != dqtype || + id == be32_to_cpu(ddq->d_id)) { + xfs_trans_brelse(sc->tp, bp); + return 0; + } + break; + default: + return error; + } + + /* Something's wrong with the block, fix the whole thing. */ + dqblk = bp->b_addr; + bp->b_ops = &xfs_dquot_buf_ops; + for (i = 0; i < qi->qi_dqperchunk; i++, dqblk++) { + ddq = &dqblk->dd_diskdq; + + trace_xrep_disk_dquot(sc->mp, dqtype, id + i); + + ddq->d_magic = cpu_to_be16(XFS_DQUOT_MAGIC); + ddq->d_version = XFS_DQUOT_VERSION; + ddq->d_type = dqtype; + ddq->d_id = cpu_to_be32(id + i); + + if (xfs_has_bigtime(sc->mp) && ddq->d_id) + ddq->d_type |= XFS_DQTYPE_BIGTIME; + + xrep_quota_fix_timer(sc->mp, ddq, ddq->d_blk_softlimit, + ddq->d_bcount, &ddq->d_btimer, + defq->blk.time); + + xrep_quota_fix_timer(sc->mp, ddq, ddq->d_ino_softlimit, + ddq->d_icount, &ddq->d_itimer, + defq->ino.time); + + xrep_quota_fix_timer(sc->mp, ddq, ddq->d_rtb_softlimit, + ddq->d_rtbcount, &ddq->d_rtbtimer, + defq->rtb.time); + + /* We only support v5 filesystems so always set these. */ + uuid_copy(&dqblk->dd_uuid, &sc->mp->m_sb.sb_meta_uuid); + xfs_update_cksum((char *)dqblk, sizeof(struct xfs_dqblk), + XFS_DQUOT_CRC_OFF); + dqblk->dd_lsn = 0; + } + switch (dqtype) { + case XFS_DQTYPE_USER: + buftype = XFS_BLFT_UDQUOT_BUF; + break; + case XFS_DQTYPE_GROUP: + buftype = XFS_BLFT_GDQUOT_BUF; + break; + case XFS_DQTYPE_PROJ: + buftype = XFS_BLFT_PDQUOT_BUF; + break; + } + xfs_trans_buf_set_type(sc->tp, bp, buftype); + xfs_trans_log_buf(sc->tp, bp, 0, BBTOB(bp->b_length) - 1); + return xrep_roll_trans(sc); +} + +/* + * Repair a quota file's data fork. The function returns with the inode + * joined. + */ +STATIC int +xrep_quota_data_fork( + struct xfs_scrub *sc, + xfs_dqtype_t dqtype) +{ + struct xfs_bmbt_irec irec = { 0 }; + struct xfs_iext_cursor icur; + struct xfs_quotainfo *qi = sc->mp->m_quotainfo; + struct xfs_ifork *ifp; + xfs_fileoff_t max_dqid_off; + xfs_fileoff_t off; + xfs_fsblock_t fsbno; + bool truncate = false; + bool joined = false; + int error = 0; + + error = xrep_metadata_inode_forks(sc); + if (error) + goto out; + + /* Check for data fork problems that apply only to quota files. */ + max_dqid_off = XFS_DQ_ID_MAX / qi->qi_dqperchunk; + ifp = xfs_ifork_ptr(sc->ip, XFS_DATA_FORK); + for_each_xfs_iext(ifp, &icur, &irec) { + if (isnullstartblock(irec.br_startblock)) { + error = -EFSCORRUPTED; + goto out; + } + + if (irec.br_startoff > max_dqid_off || + irec.br_startoff + irec.br_blockcount - 1 > max_dqid_off) { + truncate = true; + break; + } + + /* Convert unwritten extents to real ones. */ + if (irec.br_state == XFS_EXT_UNWRITTEN) { + struct xfs_bmbt_irec nrec; + int nmap = 1; + + if (!joined) { + xfs_trans_ijoin(sc->tp, sc->ip, 0); + joined = true; + } + + error = xfs_bmapi_write(sc->tp, sc->ip, + irec.br_startoff, irec.br_blockcount, + XFS_BMAPI_CONVERT, 0, &nrec, &nmap); + if (error) + goto out; + ASSERT(nrec.br_startoff == irec.br_startoff); + ASSERT(nrec.br_blockcount == irec.br_blockcount); + + error = xfs_defer_finish(&sc->tp); + if (error) + goto out; + } + } + + if (!joined) { + xfs_trans_ijoin(sc->tp, sc->ip, 0); + joined = true; + } + + if (truncate) { + /* Erase everything after the block containing the max dquot */ + error = xfs_bunmapi_range(&sc->tp, sc->ip, 0, + max_dqid_off * sc->mp->m_sb.sb_blocksize, + XFS_MAX_FILEOFF); + if (error) + goto out; + + /* Remove all CoW reservations. */ + error = xfs_reflink_cancel_cow_blocks(sc->ip, &sc->tp, 0, + XFS_MAX_FILEOFF, true); + if (error) + goto out; + sc->ip->i_diflags2 &= ~XFS_DIFLAG2_REFLINK; + + /* + * Always re-log the inode so that our permanent transaction + * can keep on rolling it forward in the log. + */ + xfs_trans_log_inode(sc->tp, sc->ip, XFS_ILOG_CORE); + } + + /* Now go fix anything that fails the verifiers. */ + for_each_xfs_iext(ifp, &icur, &irec) { + for (fsbno = irec.br_startblock, off = irec.br_startoff; + fsbno < irec.br_startblock + irec.br_blockcount; + fsbno += XFS_DQUOT_CLUSTER_SIZE_FSB, + off += XFS_DQUOT_CLUSTER_SIZE_FSB) { + error = xrep_quota_block(sc, + XFS_FSB_TO_DADDR(sc->mp, fsbno), + dqtype, off * qi->qi_dqperchunk); + if (error) + goto out; + } + } + +out: + return error; +} + +/* + * Go fix anything in the quota items that we could have been mad about. Now + * that we've checked the quota inode data fork we have to drop ILOCK_EXCL to + * use the regular dquot functions. + */ +STATIC int +xrep_quota_problems( + struct xfs_scrub *sc, + xfs_dqtype_t dqtype) +{ + struct xchk_dqiter cursor = { }; + struct xrep_quota_info rqi = { .sc = sc }; + struct xfs_dquot *dq; + int error; + + xchk_dqiter_init(&cursor, sc, dqtype); + while ((error = xchk_dquot_iter(&cursor, &dq)) == 1) { + error = xrep_quota_item(&rqi, dq); + xfs_qm_dqrele(dq); + if (error) + break; + } + if (error) + return error; + + /* Make a quotacheck happen. */ + if (rqi.need_quotacheck) + xrep_force_quotacheck(sc, dqtype); + return 0; +} + +/* Repair all of a quota type's items. */ +int +xrep_quota( + struct xfs_scrub *sc) +{ + xfs_dqtype_t dqtype; + int error; + + dqtype = xchk_quota_to_dqtype(sc); + + /* + * Re-take the ILOCK so that we can fix any problems that we found + * with the data fork mappings, or with the dquot bufs themselves. + */ + if (!(sc->ilock_flags & XFS_ILOCK_EXCL)) + xchk_ilock(sc, XFS_ILOCK_EXCL); + error = xrep_quota_data_fork(sc, dqtype); + if (error) + return error; + + /* + * Finish deferred items and roll the transaction to unjoin the quota + * inode from transaction so that we can unlock the quota inode; we + * play only with dquots from now on. + */ + error = xrep_defer_finish(sc); + if (error) + return error; + error = xfs_trans_roll(&sc->tp); + if (error) + return error; + xchk_iunlock(sc, sc->ilock_flags); + + /* Fix anything the dquot verifiers don't complain about. */ + error = xrep_quota_problems(sc, dqtype); + if (error) + return error; + + return xrep_trans_commit(sc); +} diff --git a/fs/xfs/scrub/quotacheck.c b/fs/xfs/scrub/quotacheck.c new file mode 100644 index 000000000..c199d1285 --- /dev/null +++ b/fs/xfs/scrub/quotacheck.c @@ -0,0 +1,860 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_inode.h" +#include "xfs_quota.h" +#include "xfs_qm.h" +#include "xfs_icache.h" +#include "xfs_bmap_util.h" +#include "xfs_ialloc.h" +#include "xfs_ag.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/repair.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/iscan.h" +#include "scrub/quota.h" +#include "scrub/quotacheck.h" +#include "scrub/trace.h" + +/* + * Live Quotacheck + * =============== + * + * Quota counters are "summary" metadata, in the sense that they are computed + * as the summation of the block usage counts for every file on the filesystem. + * Therefore, we compute the correct icount, bcount, and rtbcount values by + * creating a shadow quota counter structure and walking every inode. + */ + +/* Track the quota deltas for a dquot in a transaction. */ +struct xqcheck_dqtrx { + xfs_dqtype_t q_type; + xfs_dqid_t q_id; + + int64_t icount_delta; + + int64_t bcount_delta; + int64_t delbcnt_delta; + + int64_t rtbcount_delta; + int64_t delrtb_delta; +}; + +#define XQCHECK_MAX_NR_DQTRXS (XFS_QM_TRANS_DQTYPES * XFS_QM_TRANS_MAXDQS) + +/* + * Track the quota deltas for all dquots attached to a transaction if the + * quota deltas are being applied to an inode that we already scanned. + */ +struct xqcheck_dqacct { + struct rhash_head hash; + uintptr_t tx_id; + struct xqcheck_dqtrx dqtrx[XQCHECK_MAX_NR_DQTRXS]; + unsigned int refcount; +}; + +/* Free a shadow dquot accounting structure. */ +static void +xqcheck_dqacct_free( + void *ptr, + void *arg) +{ + struct xqcheck_dqacct *dqa = ptr; + + kfree(dqa); +} + +/* Set us up to scrub quota counters. */ +int +xchk_setup_quotacheck( + struct xfs_scrub *sc) +{ + if (!XFS_IS_QUOTA_ON(sc->mp)) + return -ENOENT; + + xchk_fsgates_enable(sc, XCHK_FSGATES_QUOTA); + + sc->buf = kzalloc_obj(struct xqcheck, XCHK_GFP_FLAGS); + if (!sc->buf) + return -ENOMEM; + + return xchk_setup_fs(sc); +} + +/* + * Part 1: Collecting dquot resource usage counts. For each xfs_dquot attached + * to each inode, we create a shadow dquot, and compute the inode count and add + * the data/rt block usage from what we see. + * + * To avoid false corruption reports in part 2, any failure in this part must + * set the INCOMPLETE flag even when a negative errno is returned. This care + * must be taken with certain errno values (i.e. EFSBADCRC, EFSCORRUPTED, + * ECANCELED) that are absorbed into a scrub state flag update by + * xchk_*_process_error. Scrub and repair share the same incore data + * structures, so the INCOMPLETE flag is critical to prevent a repair based on + * insufficient information. + * + * Because we are scanning a live filesystem, it's possible that another thread + * will try to update the quota counters for an inode that we've already + * scanned. This will cause our counts to be incorrect. Therefore, we hook + * the live transaction code in two places: (1) when the callers update the + * per-transaction dqtrx structure to log quota counter updates; and (2) when + * transaction commit actually logs those updates to the incore dquot. By + * shadowing transaction updates in this manner, live quotacheck can ensure + * by locking the dquot and the shadow structure that its own copies are not + * out of date. Because the hook code runs in a different process context from + * the scrub code and the scrub state flags are not accessed atomically, + * failures in the hook code must abort the iscan and the scrubber must notice + * the aborted scan and set the incomplete flag. + * + * Note that we use srcu notifier hooks to minimize the overhead when live + * quotacheck is /not/ running. + */ + +/* Update an incore dquot counter information from a live update. */ +static int +xqcheck_update_incore_counts( + struct xqcheck *xqc, + struct xfarray *counts, + xfs_dqid_t id, + int64_t inodes, + int64_t nblks, + int64_t rtblks) +{ + struct xqcheck_dquot xcdq; + int error; + + error = xfarray_load_sparse(counts, id, &xcdq); + if (error) + return error; + + xcdq.flags |= XQCHECK_DQUOT_WRITTEN; + xcdq.icount += inodes; + xcdq.bcount += nblks; + xcdq.rtbcount += rtblks; + + error = xfarray_store(counts, id, &xcdq); + if (error == -EFBIG) { + /* + * EFBIG means we tried to store data at too high a byte offset + * in the sparse array. IOWs, we cannot complete the check and + * must notify userspace that the check was incomplete. + */ + error = -ECANCELED; + } + return error; +} + +/* Decide if this is the shadow dquot accounting structure for a transaction. */ +static int +xqcheck_dqacct_obj_cmpfn( + struct rhashtable_compare_arg *arg, + const void *obj) +{ + const uintptr_t *tx_idp = arg->key; + const struct xqcheck_dqacct *dqa = obj; + + if (dqa->tx_id != *tx_idp) + return 1; + return 0; +} + +static const struct rhashtable_params xqcheck_dqacct_hash_params = { + .min_size = 32, + .key_len = sizeof(uintptr_t), + .key_offset = offsetof(struct xqcheck_dqacct, tx_id), + .head_offset = offsetof(struct xqcheck_dqacct, hash), + .automatic_shrinking = true, + .obj_cmpfn = xqcheck_dqacct_obj_cmpfn, +}; + +/* Find a shadow dqtrx slot for the given dquot. */ +STATIC struct xqcheck_dqtrx * +xqcheck_get_dqtrx( + struct xqcheck_dqacct *dqa, + xfs_dqtype_t q_type, + xfs_dqid_t q_id) +{ + int i; + + for (i = 0; i < XQCHECK_MAX_NR_DQTRXS; i++) { + if (dqa->dqtrx[i].q_type == 0 || + (dqa->dqtrx[i].q_type == q_type && + dqa->dqtrx[i].q_id == q_id)) + return &dqa->dqtrx[i]; + } + + return NULL; +} + +/* + * Create and fill out a quota delta tracking structure to shadow the updates + * going on in the regular quota code. + */ +static int +xqcheck_mod_live_ino_dqtrx( + struct notifier_block *nb, + unsigned long action, + void *data) +{ + struct xfs_mod_ino_dqtrx_params *p = data; + struct xqcheck *xqc; + struct xqcheck_dqacct *dqa; + struct xqcheck_dqtrx *dqtrx; + int error; + + xqc = container_of(nb, struct xqcheck, qhook.mod_hook.nb); + + /* Skip quota reservation fields. */ + switch (action) { + case XFS_TRANS_DQ_BCOUNT: + case XFS_TRANS_DQ_DELBCOUNT: + case XFS_TRANS_DQ_ICOUNT: + case XFS_TRANS_DQ_RTBCOUNT: + case XFS_TRANS_DQ_DELRTBCOUNT: + break; + default: + return NOTIFY_DONE; + } + + /* Ignore dqtrx updates for quota types we don't care about. */ + switch (p->q_type) { + case XFS_DQTYPE_USER: + if (!xqc->ucounts) + return NOTIFY_DONE; + break; + case XFS_DQTYPE_GROUP: + if (!xqc->gcounts) + return NOTIFY_DONE; + break; + case XFS_DQTYPE_PROJ: + if (!xqc->pcounts) + return NOTIFY_DONE; + break; + default: + return NOTIFY_DONE; + } + + /* Skip inodes that haven't been scanned yet. */ + if (!xchk_iscan_want_live_update(&xqc->iscan, p->ino)) + return NOTIFY_DONE; + + /* Make a shadow quota accounting tracker for this transaction. */ + mutex_lock(&xqc->lock); + dqa = rhashtable_lookup_fast(&xqc->shadow_dquot_acct, &p->tx_id, + xqcheck_dqacct_hash_params); + if (!dqa) { + dqa = kzalloc_obj(struct xqcheck_dqacct, XCHK_GFP_FLAGS); + if (!dqa) + goto out_abort; + + dqa->tx_id = p->tx_id; + error = rhashtable_insert_fast(&xqc->shadow_dquot_acct, + &dqa->hash, xqcheck_dqacct_hash_params); + if (error) + goto out_abort; + } + + /* Find the shadow dqtrx (or an empty slot) here. */ + dqtrx = xqcheck_get_dqtrx(dqa, p->q_type, p->q_id); + if (!dqtrx) + goto out_abort; + if (dqtrx->q_type == 0) { + dqtrx->q_type = p->q_type; + dqtrx->q_id = p->q_id; + dqa->refcount++; + } + + /* Update counter */ + switch (action) { + case XFS_TRANS_DQ_BCOUNT: + dqtrx->bcount_delta += p->delta; + break; + case XFS_TRANS_DQ_DELBCOUNT: + dqtrx->delbcnt_delta += p->delta; + break; + case XFS_TRANS_DQ_ICOUNT: + dqtrx->icount_delta += p->delta; + break; + case XFS_TRANS_DQ_RTBCOUNT: + dqtrx->rtbcount_delta += p->delta; + break; + case XFS_TRANS_DQ_DELRTBCOUNT: + dqtrx->delrtb_delta += p->delta; + break; + } + + mutex_unlock(&xqc->lock); + return NOTIFY_DONE; + +out_abort: + xchk_iscan_abort(&xqc->iscan); + mutex_unlock(&xqc->lock); + return NOTIFY_DONE; +} + +/* + * Apply the transaction quota deltas to our shadow quota accounting info when + * the regular quota code are doing the same. + */ +static int +xqcheck_apply_live_dqtrx( + struct notifier_block *nb, + unsigned long action, + void *data) +{ + struct xfs_apply_dqtrx_params *p = data; + struct xqcheck *xqc; + struct xqcheck_dqacct *dqa; + struct xqcheck_dqtrx *dqtrx; + struct xfarray *counts; + int error; + + xqc = container_of(nb, struct xqcheck, qhook.apply_hook.nb); + + /* Map the dquot type to an incore counter object. */ + switch (p->q_type) { + case XFS_DQTYPE_USER: + counts = xqc->ucounts; + break; + case XFS_DQTYPE_GROUP: + counts = xqc->gcounts; + break; + case XFS_DQTYPE_PROJ: + counts = xqc->pcounts; + break; + default: + return NOTIFY_DONE; + } + + if (xchk_iscan_aborted(&xqc->iscan) || counts == NULL) + return NOTIFY_DONE; + + /* + * Find the shadow dqtrx for this transaction and dquot, if any deltas + * need to be applied here. If not, we're finished early. + */ + mutex_lock(&xqc->lock); + dqa = rhashtable_lookup_fast(&xqc->shadow_dquot_acct, &p->tx_id, + xqcheck_dqacct_hash_params); + if (!dqa) + goto out_unlock; + dqtrx = xqcheck_get_dqtrx(dqa, p->q_type, p->q_id); + if (!dqtrx || dqtrx->q_type == 0) + goto out_unlock; + + /* Update our shadow dquot if we're committing. */ + if (action == XFS_APPLY_DQTRX_COMMIT) { + error = xqcheck_update_incore_counts(xqc, counts, p->q_id, + dqtrx->icount_delta, + dqtrx->bcount_delta + dqtrx->delbcnt_delta, + dqtrx->rtbcount_delta + dqtrx->delrtb_delta); + if (error) + goto out_abort; + } + + /* Free the shadow accounting structure if that was the last user. */ + dqa->refcount--; + if (dqa->refcount == 0) { + error = rhashtable_remove_fast(&xqc->shadow_dquot_acct, + &dqa->hash, xqcheck_dqacct_hash_params); + if (error) + goto out_abort; + xqcheck_dqacct_free(dqa, NULL); + } + + mutex_unlock(&xqc->lock); + return NOTIFY_DONE; + +out_abort: + xchk_iscan_abort(&xqc->iscan); +out_unlock: + mutex_unlock(&xqc->lock); + return NOTIFY_DONE; +} + +/* Record this inode's quota usage in our shadow quota counter data. */ +STATIC int +xqcheck_collect_inode( + struct xqcheck *xqc, + struct xfs_inode *ip) +{ + struct xfs_trans *tp = xqc->sc->tp; + xfs_filblks_t nblks, rtblks; + uint ilock_flags = 0; + xfs_dqid_t id; + bool isreg = S_ISREG(VFS_I(ip)->i_mode); + int error = 0; + + if (xfs_is_metadir_inode(ip) || + xfs_is_quota_inode(&tp->t_mountp->m_sb, I_INO(ip))) { + /* + * Quota files are never counted towards quota, so we do not + * need to take the lock. Files do not switch between the + * metadata and regular directory trees without a reallocation, + * so we do not need to ILOCK them either. + */ + xchk_iscan_mark_visited(&xqc->iscan, ip); + return 0; + } + + /* Figure out the data / rt device block counts. */ + xfs_ilock(ip, XFS_IOLOCK_SHARED); + if (isreg) + xfs_ilock(ip, XFS_MMAPLOCK_SHARED); + if (XFS_IS_REALTIME_INODE(ip)) { + /* + * Read in the data fork for rt files so that _count_blocks + * can count the number of blocks allocated from the rt volume. + * Inodes do not track that separately. + */ + ilock_flags = xfs_ilock_data_map_shared(ip); + error = xfs_iread_extents(tp, ip, XFS_DATA_FORK); + if (error) + goto out_abort; + } else { + ilock_flags = XFS_ILOCK_SHARED; + xfs_ilock(ip, XFS_ILOCK_SHARED); + } + xfs_inode_count_blocks(tp, ip, &nblks, &rtblks); + + if (xchk_iscan_aborted(&xqc->iscan)) { + error = -ECANCELED; + goto out_incomplete; + } + + /* Update the shadow dquot counters. */ + mutex_lock(&xqc->lock); + if (xqc->ucounts) { + id = xfs_qm_id_for_quotatype(ip, XFS_DQTYPE_USER); + error = xqcheck_update_incore_counts(xqc, xqc->ucounts, id, 1, + nblks, rtblks); + if (error) + goto out_mutex; + } + + if (xqc->gcounts) { + id = xfs_qm_id_for_quotatype(ip, XFS_DQTYPE_GROUP); + error = xqcheck_update_incore_counts(xqc, xqc->gcounts, id, 1, + nblks, rtblks); + if (error) + goto out_mutex; + } + + if (xqc->pcounts) { + id = xfs_qm_id_for_quotatype(ip, XFS_DQTYPE_PROJ); + error = xqcheck_update_incore_counts(xqc, xqc->pcounts, id, 1, + nblks, rtblks); + if (error) + goto out_mutex; + } + mutex_unlock(&xqc->lock); + + xchk_iscan_mark_visited(&xqc->iscan, ip); + goto out_ilock; + +out_mutex: + mutex_unlock(&xqc->lock); +out_abort: + xchk_iscan_abort(&xqc->iscan); +out_incomplete: + xchk_set_incomplete(xqc->sc); +out_ilock: + xfs_iunlock(ip, ilock_flags); + if (isreg) + xfs_iunlock(ip, XFS_MMAPLOCK_SHARED); + xfs_iunlock(ip, XFS_IOLOCK_SHARED); + return error; +} + +/* Walk all the allocated inodes and run a quota scan on them. */ +STATIC int +xqcheck_collect_counts( + struct xqcheck *xqc) +{ + struct xfs_scrub *sc = xqc->sc; + struct xfs_inode *ip; + int error; + + /* + * Set up for a potentially lengthy filesystem scan by reducing our + * transaction resource usage for the duration. Specifically: + * + * Cancel the transaction to release the log grant space while we scan + * the filesystem. + * + * Create a new empty transaction to eliminate the possibility of the + * inode scan deadlocking on cyclical metadata. + * + * We pass the empty transaction to the file scanning function to avoid + * repeatedly cycling empty transactions. This can be done without + * risk of deadlock between sb_internal and the IOLOCK (we take the + * IOLOCK to quiesce the file before scanning) because empty + * transactions do not take sb_internal. + */ + xchk_trans_cancel(sc); + xchk_trans_alloc_empty(sc); + + while ((error = xchk_iscan_iter(&xqc->iscan, &ip)) == 1) { + error = xqcheck_collect_inode(xqc, ip); + xchk_irele(sc, ip); + if (error) + break; + + if (xchk_should_terminate(sc, &error)) + break; + } + xchk_iscan_iter_finish(&xqc->iscan); + if (error) { + xchk_set_incomplete(sc); + /* + * If we couldn't grab an inode that was busy with a state + * change, change the error code so that we exit to userspace + * as quickly as possible. + */ + if (error == -EBUSY) + return -ECANCELED; + return error; + } + + /* + * Switch out for a real transaction in preparation for building a new + * tree. + */ + xchk_trans_cancel(sc); + return xchk_setup_fs(sc); +} + +/* + * Part 2: Comparing dquot resource counters. Walk each xfs_dquot, comparing + * the resource usage counters against our shadow dquots; and then walk each + * shadow dquot (that wasn't covered in the first part), comparing it against + * the xfs_dquot. + */ + +/* + * Check the dquot data against what we observed. Caller must hold the dquot + * lock. + */ +STATIC int +xqcheck_compare_dquot( + struct xqcheck *xqc, + xfs_dqtype_t dqtype, + struct xfs_dquot *dq) +{ + struct xqcheck_dquot xcdq; + struct xfarray *counts = xqcheck_counters_for(xqc, dqtype); + int error; + + if (xchk_iscan_aborted(&xqc->iscan)) { + xchk_set_incomplete(xqc->sc); + return -ECANCELED; + } + + mutex_lock(&dq->q_qlock); + mutex_lock(&xqc->lock); + error = xfarray_load_sparse(counts, dq->q_id, &xcdq); + if (error) + goto out_unlock; + + if (xcdq.icount != dq->q_ino.count) + xchk_qcheck_set_corrupt(xqc->sc, dqtype, dq->q_id); + + if (xcdq.bcount != dq->q_blk.count) + xchk_qcheck_set_corrupt(xqc->sc, dqtype, dq->q_id); + + if (xcdq.rtbcount != dq->q_rtb.count) + xchk_qcheck_set_corrupt(xqc->sc, dqtype, dq->q_id); + + xcdq.flags |= (XQCHECK_DQUOT_COMPARE_SCANNED | XQCHECK_DQUOT_WRITTEN); + error = xfarray_store(counts, dq->q_id, &xcdq); + if (error == -EFBIG) { + /* + * EFBIG means we tried to store data at too high a byte offset + * in the sparse array. IOWs, we cannot complete the check and + * must notify userspace that the check was incomplete. This + * should never happen outside of the collection phase. + */ + xchk_set_incomplete(xqc->sc); + error = -ECANCELED; + } +out_unlock: + mutex_unlock(&xqc->lock); + mutex_unlock(&dq->q_qlock); + if (error) + return error; + + if (xqc->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return -ECANCELED; + + return 0; +} + +/* + * Walk all the observed dquots, and make sure there's a matching incore + * dquot and that its counts match ours. + */ +STATIC int +xqcheck_walk_observations( + struct xqcheck *xqc, + xfs_dqtype_t dqtype) +{ + struct xqcheck_dquot xcdq; + struct xfs_dquot *dq; + struct xfarray *counts = xqcheck_counters_for(xqc, dqtype); + xfarray_idx_t cur = XFARRAY_CURSOR_INIT; + int error; + + mutex_lock(&xqc->lock); + while ((error = xfarray_iter(counts, &cur, &xcdq)) == 1) { + xfs_dqid_t id = cur - 1; + + if (xcdq.flags & XQCHECK_DQUOT_COMPARE_SCANNED) + continue; + + mutex_unlock(&xqc->lock); + + error = xfs_qm_dqget(xqc->sc->mp, id, dqtype, false, &dq); + if (error == -ENOENT) { + xchk_qcheck_set_corrupt(xqc->sc, dqtype, id); + return 0; + } + if (error) + return error; + + error = xqcheck_compare_dquot(xqc, dqtype, dq); + xfs_qm_dqrele(dq); + if (error) + return error; + + if (xchk_should_terminate(xqc->sc, &error)) + return error; + + mutex_lock(&xqc->lock); + } + mutex_unlock(&xqc->lock); + + return error; +} + +/* Compare the quota counters we observed against the live dquots. */ +STATIC int +xqcheck_compare_dqtype( + struct xqcheck *xqc, + xfs_dqtype_t dqtype) +{ + struct xchk_dqiter cursor = { }; + struct xfs_scrub *sc = xqc->sc; + struct xfs_dquot *dq; + int error; + + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return 0; + + /* If the quota CHKD flag is cleared, we need to repair this quota. */ + if (!(xfs_quota_chkd_flag(dqtype) & sc->mp->m_qflags)) { + xchk_qcheck_set_corrupt(xqc->sc, dqtype, 0); + return 0; + } + + /* Compare what we observed against the actual dquots. */ + xchk_dqiter_init(&cursor, sc, dqtype); + while ((error = xchk_dquot_iter(&cursor, &dq)) == 1) { + error = xqcheck_compare_dquot(xqc, dqtype, dq); + xfs_qm_dqrele(dq); + if (error) + break; + } + if (error) + return error; + + /* Walk all the observed dquots and compare to the incore ones. */ + return xqcheck_walk_observations(xqc, dqtype); +} + +/* Tear down everything associated with a quotacheck. */ +static void +xqcheck_teardown_scan( + void *priv) +{ + struct xqcheck *xqc = priv; + struct xfs_quotainfo *qi = xqc->sc->mp->m_quotainfo; + + /* Discourage any hook functions that might be running. */ + xchk_iscan_abort(&xqc->iscan); + + /* + * As noted above, the apply hook is responsible for cleaning up the + * shadow dquot accounting data when a transaction completes. The mod + * hook must be removed before the apply hook so that we don't + * mistakenly leave an active shadow account for the mod hook to get + * its hands on. No hooks should be running after these functions + * return. + */ + xfs_dqtrx_hook_del(qi, &xqc->qhook); + + if (xqc->shadow_dquot_acct.key_len) { + rhashtable_free_and_destroy(&xqc->shadow_dquot_acct, + xqcheck_dqacct_free, NULL); + xqc->shadow_dquot_acct.key_len = 0; + } + + if (xqc->pcounts) { + xfarray_destroy(xqc->pcounts); + xqc->pcounts = NULL; + } + + if (xqc->gcounts) { + xfarray_destroy(xqc->gcounts); + xqc->gcounts = NULL; + } + + if (xqc->ucounts) { + xfarray_destroy(xqc->ucounts); + xqc->ucounts = NULL; + } + + xchk_iscan_teardown(&xqc->iscan); + mutex_destroy(&xqc->lock); + xqc->sc = NULL; +} + +/* + * Scan all inodes in the entire filesystem to generate quota counter data. + * If the scan is successful, the quota data will be left alive for a repair. + * If any error occurs, we'll tear everything down. + */ +STATIC int +xqcheck_setup_scan( + struct xfs_scrub *sc, + struct xqcheck *xqc) +{ + struct xfs_quotainfo *qi = sc->mp->m_quotainfo; + unsigned long long max_dquots = XFS_DQ_ID_MAX + 1ULL; + int error; + + ASSERT(xqc->sc == NULL); + xqc->sc = sc; + + mutex_init(&xqc->lock); + + /* Retry iget every tenth of a second for up to 30 seconds. */ + xchk_iscan_start(sc, 30000, 100, &xqc->iscan); + + error = -ENOMEM; + if (xfs_this_quota_on(sc->mp, XFS_DQTYPE_USER)) { + error = xfarray_create("user dquot records", max_dquots, + sizeof(struct xqcheck_dquot), &xqc->ucounts); + if (error) + goto out_teardown; + } + + if (xfs_this_quota_on(sc->mp, XFS_DQTYPE_GROUP)) { + error = xfarray_create("group dquot records", max_dquots, + sizeof(struct xqcheck_dquot), &xqc->gcounts); + if (error) + goto out_teardown; + } + + if (xfs_this_quota_on(sc->mp, XFS_DQTYPE_PROJ)) { + error = xfarray_create("project dquot records", max_dquots, + sizeof(struct xqcheck_dquot), &xqc->pcounts); + if (error) + goto out_teardown; + } + + /* + * Set up hash table to map transactions to our internal shadow dqtrx + * structures. + */ + error = rhashtable_init(&xqc->shadow_dquot_acct, + &xqcheck_dqacct_hash_params); + if (error) + goto out_teardown; + + /* + * Hook into the quota code. The hook only triggers for inodes that + * were already scanned, and the scanner thread takes each inode's + * ILOCK, which means that any in-progress inode updates will finish + * before we can scan the inode. + * + * The apply hook (which removes the shadow dquot accounting struct) + * must be installed before the mod hook so that we never fail to catch + * the end of a quota update sequence and leave stale shadow data. + */ + ASSERT(sc->flags & XCHK_FSGATES_QUOTA); + xfs_dqtrx_hook_setup(&xqc->qhook, xqcheck_mod_live_ino_dqtrx, + xqcheck_apply_live_dqtrx); + + error = xfs_dqtrx_hook_add(qi, &xqc->qhook); + if (error) + goto out_teardown; + + /* Use deferred cleanup to pass the quota count data to repair. */ + sc->buf_cleanup = xqcheck_teardown_scan; + return 0; + +out_teardown: + xqcheck_teardown_scan(xqc); + return error; +} + +/* Scrub all counters for a given quota type. */ +int +xchk_quotacheck( + struct xfs_scrub *sc) +{ + struct xqcheck *xqc = sc->buf; + int error = 0; + + /* Check quota counters on the live filesystem. */ + error = xqcheck_setup_scan(sc, xqc); + if (error) + return error; + + /* Walk all inodes, picking up quota information. */ + error = xqcheck_collect_counts(xqc); + if (!xchk_xref_process_error(sc, 0, 0, &error)) + return error; + + /* Fail fast if we're not playing with a full dataset. */ + if (xchk_iscan_aborted(&xqc->iscan)) + xchk_set_incomplete(sc); + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_INCOMPLETE) + return 0; + + /* Compare quota counters. */ + if (xqc->ucounts) { + error = xqcheck_compare_dqtype(xqc, XFS_DQTYPE_USER); + if (!xchk_xref_process_error(sc, 0, 0, &error)) + return error; + } + if (xqc->gcounts) { + error = xqcheck_compare_dqtype(xqc, XFS_DQTYPE_GROUP); + if (!xchk_xref_process_error(sc, 0, 0, &error)) + return error; + } + if (xqc->pcounts) { + error = xqcheck_compare_dqtype(xqc, XFS_DQTYPE_PROJ); + if (!xchk_xref_process_error(sc, 0, 0, &error)) + return error; + } + + /* Check one last time for an incomplete dataset. */ + if (xchk_iscan_aborted(&xqc->iscan)) + xchk_set_incomplete(sc); + + return 0; +} diff --git a/fs/xfs/scrub/quotacheck.h b/fs/xfs/scrub/quotacheck.h new file mode 100644 index 000000000..4ea5f249c --- /dev/null +++ b/fs/xfs/scrub/quotacheck.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_QUOTACHECK_H__ +#define __XFS_SCRUB_QUOTACHECK_H__ + +/* Quota counters for live quotacheck. */ +struct xqcheck_dquot { + /* block usage count */ + int64_t bcount; + + /* inode usage count */ + int64_t icount; + + /* realtime block usage count */ + int64_t rtbcount; + + /* Record state */ + unsigned int flags; +}; + +/* + * This incore dquot record has been written at least once. We never want to + * store an xqcheck_dquot that looks uninitialized. + */ +#define XQCHECK_DQUOT_WRITTEN (1U << 0) + +/* Already checked this dquot. */ +#define XQCHECK_DQUOT_COMPARE_SCANNED (1U << 1) + +/* Already repaired this dquot. */ +#define XQCHECK_DQUOT_REPAIR_SCANNED (1U << 2) + +/* Live quotacheck control structure. */ +struct xqcheck { + struct xfs_scrub *sc; + + /* Shadow dquot counter data. */ + struct xfarray *ucounts; + struct xfarray *gcounts; + struct xfarray *pcounts; + + /* Lock protecting quotacheck count observations */ + struct mutex lock; + + struct xchk_iscan iscan; + + /* Hooks into the quota code. */ + struct xfs_dqtrx_hook qhook; + + /* Shadow quota delta tracking structure. */ + struct rhashtable shadow_dquot_acct; +}; + +/* Return the incore counter array for a given quota type. */ +static inline struct xfarray * +xqcheck_counters_for( + struct xqcheck *xqc, + xfs_dqtype_t dqtype) +{ + switch (dqtype) { + case XFS_DQTYPE_USER: + return xqc->ucounts; + case XFS_DQTYPE_GROUP: + return xqc->gcounts; + case XFS_DQTYPE_PROJ: + return xqc->pcounts; + } + + ASSERT(0); + return NULL; +} + +#endif /* __XFS_SCRUB_QUOTACHECK_H__ */ diff --git a/fs/xfs/scrub/quotacheck_repair.c b/fs/xfs/scrub/quotacheck_repair.c new file mode 100644 index 000000000..dbb522e15 --- /dev/null +++ b/fs/xfs/scrub/quotacheck_repair.c @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_inode.h" +#include "xfs_quota.h" +#include "xfs_qm.h" +#include "xfs_icache.h" +#include "xfs_bmap_util.h" +#include "xfs_iwalk.h" +#include "xfs_ialloc.h" +#include "xfs_sb.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/repair.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/iscan.h" +#include "scrub/quota.h" +#include "scrub/quotacheck.h" +#include "scrub/trace.h" + +/* + * Live Quotacheck Repair + * ====================== + * + * Use the live quota counter information that we collected to replace the + * counter values in the incore dquots. A scrub->repair cycle should have left + * the live data and hooks active, so this is safe so long as we make sure the + * dquot is locked. + */ + +/* Commit new counters to a dquot. */ +static int +xqcheck_commit_dquot( + struct xqcheck *xqc, + xfs_dqtype_t dqtype, + struct xfs_dquot *dq) +{ + struct xqcheck_dquot xcdq; + struct xfarray *counts = xqcheck_counters_for(xqc, dqtype); + int64_t delta; + bool dirty = false; + int error = 0; + + error = xchk_trans_alloc(xqc->sc, 0); + if (error) + return error; + + mutex_lock(&dq->q_qlock); + xfs_trans_dqjoin(xqc->sc->tp, dq); + + if (xchk_iscan_aborted(&xqc->iscan)) { + error = -ECANCELED; + goto out_cancel; + } + + mutex_lock(&xqc->lock); + error = xfarray_load_sparse(counts, dq->q_id, &xcdq); + if (error) + goto out_unlock; + + /* Adjust counters as needed. */ + delta = (int64_t)xcdq.icount - dq->q_ino.count; + if (delta) { + dq->q_ino.reserved += delta; + dq->q_ino.count += delta; + dirty = true; + } + + delta = (int64_t)xcdq.bcount - dq->q_blk.count; + if (delta) { + dq->q_blk.reserved += delta; + dq->q_blk.count += delta; + dirty = true; + } + + delta = (int64_t)xcdq.rtbcount - dq->q_rtb.count; + if (delta) { + dq->q_rtb.reserved += delta; + dq->q_rtb.count += delta; + dirty = true; + } + + xcdq.flags |= (XQCHECK_DQUOT_REPAIR_SCANNED | XQCHECK_DQUOT_WRITTEN); + error = xfarray_store(counts, dq->q_id, &xcdq); + if (error == -EFBIG) { + /* + * EFBIG means we tried to store data at too high a byte offset + * in the sparse array. IOWs, we cannot complete the repair + * and must cancel the whole operation. This should never + * happen, but we need to catch it anyway. + */ + error = -ECANCELED; + } + mutex_unlock(&xqc->lock); + if (error || !dirty) + goto out_cancel; + + trace_xrep_quotacheck_dquot(xqc->sc->mp, dq->q_type, dq->q_id); + + /* Commit the dirty dquot to disk. */ + dq->q_flags |= XFS_DQFLAG_DIRTY; + if (dq->q_id) + xfs_qm_adjust_dqtimers(dq); + xfs_trans_log_dquot(xqc->sc->tp, dq); + return xrep_trans_commit(xqc->sc); + +out_unlock: + mutex_unlock(&xqc->lock); +out_cancel: + xchk_trans_cancel(xqc->sc); + return error; +} + +/* Commit new quota counters for a particular quota type. */ +STATIC int +xqcheck_commit_dqtype( + struct xqcheck *xqc, + unsigned int dqtype) +{ + struct xchk_dqiter cursor = { }; + struct xqcheck_dquot xcdq; + struct xfs_scrub *sc = xqc->sc; + struct xfs_mount *mp = sc->mp; + struct xfarray *counts = xqcheck_counters_for(xqc, dqtype); + struct xfs_dquot *dq; + xfarray_idx_t cur = XFARRAY_CURSOR_INIT; + int error; + + /* + * Update the counters of every dquot that the quota file knows about. + */ + xchk_dqiter_init(&cursor, sc, dqtype); + while ((error = xchk_dquot_iter(&cursor, &dq)) == 1) { + error = xqcheck_commit_dquot(xqc, dqtype, dq); + xfs_qm_dqrele(dq); + if (error) + break; + } + if (error) + return error; + + /* + * Make a second pass to deal with the dquots that we know about but + * the quota file previously did not know about. + */ + mutex_lock(&xqc->lock); + while ((error = xfarray_iter(counts, &cur, &xcdq)) == 1) { + xfs_dqid_t id = cur - 1; + + if (xcdq.flags & XQCHECK_DQUOT_REPAIR_SCANNED) + continue; + + mutex_unlock(&xqc->lock); + + /* + * Grab the dquot, allowing for dquot block allocation in a + * separate transaction. We committed the scrub transaction + * in a previous step, so we will not be creating nested + * transactions here. + */ + error = xfs_qm_dqget(mp, id, dqtype, true, &dq); + if (error) + return error; + + error = xqcheck_commit_dquot(xqc, dqtype, dq); + xfs_qm_dqrele(dq); + if (error) + return error; + + mutex_lock(&xqc->lock); + } + mutex_unlock(&xqc->lock); + + return error; +} + +/* Figure out quota CHKD flags for the running quota types. */ +static inline unsigned int +xqcheck_chkd_flags( + struct xfs_mount *mp) +{ + unsigned int ret = 0; + + if (XFS_IS_UQUOTA_ON(mp)) + ret |= XFS_UQUOTA_CHKD; + if (XFS_IS_GQUOTA_ON(mp)) + ret |= XFS_GQUOTA_CHKD; + if (XFS_IS_PQUOTA_ON(mp)) + ret |= XFS_PQUOTA_CHKD; + return ret; +} + +/* Commit the new dquot counters. */ +int +xrep_quotacheck( + struct xfs_scrub *sc) +{ + struct xqcheck *xqc = sc->buf; + unsigned int qflags = xqcheck_chkd_flags(sc->mp); + int error; + + /* + * Clear the CHKD flag for the running quota types and commit the scrub + * transaction so that we can allocate new quota block mappings if we + * have to. If we crash after this point, the sb still has the CHKD + * flags cleared, so mount quotacheck will fix all of this up. + */ + xrep_update_qflags(sc, qflags, 0); + error = xrep_trans_commit(sc); + if (error) + return error; + + /* Commit the new counters to the dquots. */ + if (xqc->ucounts) { + error = xqcheck_commit_dqtype(xqc, XFS_DQTYPE_USER); + if (error) + return error; + } + if (xqc->gcounts) { + error = xqcheck_commit_dqtype(xqc, XFS_DQTYPE_GROUP); + if (error) + return error; + } + if (xqc->pcounts) { + error = xqcheck_commit_dqtype(xqc, XFS_DQTYPE_PROJ); + if (error) + return error; + } + + /* Set the CHKD flags now that we've fixed quota counts. */ + error = xchk_trans_alloc(sc, 0); + if (error) + return error; + + xrep_update_qflags(sc, 0, qflags); + return xrep_trans_commit(sc); +} diff --git a/fs/xfs/scrub/rcbag.c b/fs/xfs/scrub/rcbag.c new file mode 100644 index 000000000..8f3a10f7c --- /dev/null +++ b/fs/xfs/scrub/rcbag.c @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2022-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_btree.h" +#include "xfs_buf_mem.h" +#include "xfs_btree_mem.h" +#include "xfs_error.h" +#include "scrub/scrub.h" +#include "scrub/rcbag_btree.h" +#include "scrub/rcbag.h" +#include "scrub/trace.h" + +struct rcbag { + struct xfs_mount *mp; + struct xfbtree xfbtree; + uint64_t nr_items; +}; + +int +rcbag_init( + struct xfs_mount *mp, + struct xfs_buftarg *btp, + struct rcbag **bagp) +{ + struct rcbag *bag; + int error; + + bag = kzalloc_obj(struct rcbag, XCHK_GFP_FLAGS); + if (!bag) + return -ENOMEM; + + bag->nr_items = 0; + bag->mp = mp; + + error = rcbagbt_mem_init(mp, &bag->xfbtree, btp); + if (error) + goto out_bag; + + *bagp = bag; + return 0; + +out_bag: + kfree(bag); + return error; +} + +void +rcbag_free( + struct rcbag **bagp) +{ + struct rcbag *bag = *bagp; + + xfbtree_destroy(&bag->xfbtree); + kfree(bag); + *bagp = NULL; +} + +/* Track an rmap in the refcount bag. */ +int +rcbag_add( + struct rcbag *bag, + struct xfs_trans *tp, + const struct xfs_rmap_irec *rmap) +{ + struct rcbag_rec bagrec; + struct xfs_mount *mp = bag->mp; + struct xfs_btree_cur *cur; + int has; + int error; + + cur = rcbagbt_mem_cursor(mp, tp, &bag->xfbtree); + error = rcbagbt_lookup_eq(cur, rmap, &has); + if (error) + goto out_cur; + + if (has) { + error = rcbagbt_get_rec(cur, &bagrec, &has); + if (error) + goto out_cur; + if (!has) { + error = -EFSCORRUPTED; + goto out_cur; + } + + bagrec.rbg_refcount++; + error = rcbagbt_update(cur, &bagrec); + if (error) + goto out_cur; + } else { + bagrec.rbg_startblock = rmap->rm_startblock; + bagrec.rbg_blockcount = rmap->rm_blockcount; + bagrec.rbg_refcount = 1; + + error = rcbagbt_insert(cur, &bagrec, &has); + if (error) + goto out_cur; + if (!has) { + error = -EFSCORRUPTED; + goto out_cur; + } + } + + xfs_btree_del_cursor(cur, 0); + + error = xfbtree_trans_commit(&bag->xfbtree, tp); + if (error) + return error; + + bag->nr_items++; + return 0; + +out_cur: + xfs_btree_del_cursor(cur, error); + xfbtree_trans_cancel(&bag->xfbtree, tp); + return error; +} + +/* Return the number of records in the bag. */ +uint64_t +rcbag_count( + const struct rcbag *rcbag) +{ + return rcbag->nr_items; +} + +static inline uint32_t rcbag_rec_next_bno(const struct rcbag_rec *r) +{ + return r->rbg_startblock + r->rbg_blockcount; +} + +/* + * Find the next block where the refcount changes, given the next rmap we + * looked at and the ones we're already tracking. + */ +int +rcbag_next_edge( + struct rcbag *bag, + struct xfs_trans *tp, + const struct xfs_rmap_irec *next_rmap, + bool next_valid, + uint32_t *next_bnop) +{ + struct rcbag_rec bagrec; + struct xfs_mount *mp = bag->mp; + struct xfs_btree_cur *cur; + uint32_t next_bno = NULLAGBLOCK; + int has; + int error; + + if (next_valid) + next_bno = next_rmap->rm_startblock; + + cur = rcbagbt_mem_cursor(mp, tp, &bag->xfbtree); + error = xfs_btree_goto_left_edge(cur); + if (error) + goto out_cur; + + while (true) { + error = xfs_btree_increment(cur, 0, &has); + if (error) + goto out_cur; + if (!has) + break; + + error = rcbagbt_get_rec(cur, &bagrec, &has); + if (error) + goto out_cur; + if (!has) { + error = -EFSCORRUPTED; + goto out_cur; + } + + next_bno = min(next_bno, rcbag_rec_next_bno(&bagrec)); + } + + /* + * We should have found /something/ because either next_rrm is the next + * interesting rmap to look at after emitting this refcount extent, or + * there are other rmaps in rmap_bag contributing to the current + * sharing count. But if something is seriously wrong, bail out. + */ + if (next_bno == NULLAGBLOCK) { + error = -EFSCORRUPTED; + goto out_cur; + } + + xfs_btree_del_cursor(cur, 0); + + *next_bnop = next_bno; + return 0; + +out_cur: + xfs_btree_del_cursor(cur, error); + return error; +} + +/* Pop all refcount bag records that end at next_bno */ +int +rcbag_remove_ending_at( + struct rcbag *bag, + struct xfs_trans *tp, + uint32_t next_bno) +{ + struct rcbag_rec bagrec; + struct xfs_mount *mp = bag->mp; + struct xfs_btree_cur *cur; + int has; + int error; + + /* go to the right edge of the tree */ + cur = rcbagbt_mem_cursor(mp, tp, &bag->xfbtree); + memset(&cur->bc_rec, 0xFF, sizeof(cur->bc_rec)); + error = xfs_btree_lookup(cur, XFS_LOOKUP_GE, &has); + if (error) + goto out_cur; + + while (true) { + error = xfs_btree_decrement(cur, 0, &has); + if (error) + goto out_cur; + if (!has) + break; + + error = rcbagbt_get_rec(cur, &bagrec, &has); + if (error) + goto out_cur; + if (!has) { + error = -EFSCORRUPTED; + goto out_cur; + } + + if (rcbag_rec_next_bno(&bagrec) != next_bno) + continue; + + error = xfs_btree_delete(cur, &has); + if (error) + goto out_cur; + if (!has) { + error = -EFSCORRUPTED; + goto out_cur; + } + + bag->nr_items -= bagrec.rbg_refcount; + } + + xfs_btree_del_cursor(cur, 0); + return xfbtree_trans_commit(&bag->xfbtree, tp); +out_cur: + xfs_btree_del_cursor(cur, error); + xfbtree_trans_cancel(&bag->xfbtree, tp); + return error; +} + +/* Dump the rcbag. */ +void +rcbag_dump( + struct rcbag *bag, + struct xfs_trans *tp) +{ + struct rcbag_rec bagrec; + struct xfs_mount *mp = bag->mp; + struct xfs_btree_cur *cur; + unsigned long long nr = 0; + int has; + int error; + + cur = rcbagbt_mem_cursor(mp, tp, &bag->xfbtree); + error = xfs_btree_goto_left_edge(cur); + if (error) + goto out_cur; + + while (true) { + error = xfs_btree_increment(cur, 0, &has); + if (error) + goto out_cur; + if (!has) + break; + + error = rcbagbt_get_rec(cur, &bagrec, &has); + if (error) + goto out_cur; + if (!has) { + error = -EFSCORRUPTED; + goto out_cur; + } + + xfs_err(bag->mp, "[%llu]: bno 0x%x fsbcount 0x%x refcount 0x%llx\n", + nr++, + (unsigned int)bagrec.rbg_startblock, + (unsigned int)bagrec.rbg_blockcount, + (unsigned long long)bagrec.rbg_refcount); + } + +out_cur: + xfs_btree_del_cursor(cur, error); +} diff --git a/fs/xfs/scrub/rcbag.h b/fs/xfs/scrub/rcbag.h new file mode 100644 index 000000000..e29ef788b --- /dev/null +++ b/fs/xfs/scrub/rcbag.h @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2022-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_RCBAG_H__ +#define __XFS_SCRUB_RCBAG_H__ + +struct xfs_mount; +struct rcbag; +struct xfs_buftarg; + +int rcbag_init(struct xfs_mount *mp, struct xfs_buftarg *btp, + struct rcbag **bagp); +void rcbag_free(struct rcbag **bagp); +int rcbag_add(struct rcbag *bag, struct xfs_trans *tp, + const struct xfs_rmap_irec *rmap); +uint64_t rcbag_count(const struct rcbag *bag); + +int rcbag_next_edge(struct rcbag *bag, struct xfs_trans *tp, + const struct xfs_rmap_irec *next_rmap, bool next_valid, + uint32_t *next_bnop); +int rcbag_remove_ending_at(struct rcbag *bag, struct xfs_trans *tp, + uint32_t next_bno); + +void rcbag_dump(struct rcbag *bag, struct xfs_trans *tp); + +#endif /* __XFS_SCRUB_RCBAG_H__ */ diff --git a/fs/xfs/scrub/rcbag_btree.c b/fs/xfs/scrub/rcbag_btree.c new file mode 100644 index 000000000..367f8ccf5 --- /dev/null +++ b/fs/xfs/scrub/rcbag_btree.c @@ -0,0 +1,352 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2022-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_btree.h" +#include "xfs_buf_mem.h" +#include "xfs_btree_mem.h" +#include "xfs_error.h" +#include "scrub/rcbag_btree.h" +#include "scrub/trace.h" + +static struct kmem_cache *rcbagbt_cur_cache; + +STATIC void +rcbagbt_init_key_from_rec( + union xfs_btree_key *key, + const union xfs_btree_rec *rec) +{ + struct rcbag_key *bag_key = (struct rcbag_key *)key; + const struct rcbag_rec *bag_rec = (const struct rcbag_rec *)rec; + + BUILD_BUG_ON(sizeof(struct rcbag_key) > sizeof(union xfs_btree_key)); + BUILD_BUG_ON(sizeof(struct rcbag_rec) > sizeof(union xfs_btree_rec)); + + bag_key->rbg_startblock = bag_rec->rbg_startblock; + bag_key->rbg_blockcount = bag_rec->rbg_blockcount; +} + +STATIC void +rcbagbt_init_rec_from_cur( + struct xfs_btree_cur *cur, + union xfs_btree_rec *rec) +{ + struct rcbag_rec *bag_rec = (struct rcbag_rec *)rec; + struct rcbag_rec *bag_irec = (struct rcbag_rec *)&cur->bc_rec; + + bag_rec->rbg_startblock = bag_irec->rbg_startblock; + bag_rec->rbg_blockcount = bag_irec->rbg_blockcount; + bag_rec->rbg_refcount = bag_irec->rbg_refcount; +} + +STATIC int +rcbagbt_cmp_key_with_cur( + struct xfs_btree_cur *cur, + const union xfs_btree_key *key) +{ + struct rcbag_rec *rec = (struct rcbag_rec *)&cur->bc_rec; + const struct rcbag_key *kp = (const struct rcbag_key *)key; + + return cmp_int(kp->rbg_startblock, rec->rbg_startblock) ?: + cmp_int(kp->rbg_blockcount, rec->rbg_blockcount); +} + +STATIC int +rcbagbt_cmp_two_keys( + struct xfs_btree_cur *cur, + const union xfs_btree_key *k1, + const union xfs_btree_key *k2, + const union xfs_btree_key *mask) +{ + const struct rcbag_key *kp1 = (const struct rcbag_key *)k1; + const struct rcbag_key *kp2 = (const struct rcbag_key *)k2; + + ASSERT(mask == NULL); + + return cmp_int(kp1->rbg_startblock, kp2->rbg_startblock) ?: + cmp_int(kp1->rbg_blockcount, kp2->rbg_blockcount); +} + +STATIC int +rcbagbt_keys_inorder( + struct xfs_btree_cur *cur, + const union xfs_btree_key *k1, + const union xfs_btree_key *k2) +{ + const struct rcbag_key *kp1 = (const struct rcbag_key *)k1; + const struct rcbag_key *kp2 = (const struct rcbag_key *)k2; + + if (kp1->rbg_startblock > kp2->rbg_startblock) + return 0; + if (kp1->rbg_startblock < kp2->rbg_startblock) + return 1; + + if (kp1->rbg_blockcount > kp2->rbg_blockcount) + return 0; + if (kp1->rbg_blockcount < kp2->rbg_blockcount) + return 1; + + return 0; +} + +STATIC int +rcbagbt_recs_inorder( + struct xfs_btree_cur *cur, + const union xfs_btree_rec *r1, + const union xfs_btree_rec *r2) +{ + const struct rcbag_rec *rp1 = (const struct rcbag_rec *)r1; + const struct rcbag_rec *rp2 = (const struct rcbag_rec *)r2; + + if (rp1->rbg_startblock > rp2->rbg_startblock) + return 0; + if (rp1->rbg_startblock < rp2->rbg_startblock) + return 1; + + if (rp1->rbg_blockcount > rp2->rbg_blockcount) + return 0; + if (rp1->rbg_blockcount < rp2->rbg_blockcount) + return 1; + + return 0; +} + +static xfs_failaddr_t +rcbagbt_verify( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_btree_block *block = XFS_BUF_TO_BLOCK(bp); + xfs_failaddr_t fa; + unsigned int level; + unsigned int maxrecs; + + if (!xfs_verify_magic(bp, block->bb_magic)) + return __this_address; + + fa = xfs_btree_fsblock_v5hdr_verify(bp, XFS_RMAP_OWN_UNKNOWN); + if (fa) + return fa; + + level = be16_to_cpu(block->bb_level); + if (level >= rcbagbt_maxlevels_possible()) + return __this_address; + + maxrecs = rcbagbt_maxrecs(mp, XFBNO_BLOCKSIZE, level == 0); + return xfs_btree_memblock_verify(bp, maxrecs); +} + +static void +rcbagbt_rw_verify( + struct xfs_buf *bp) +{ + xfs_failaddr_t fa = rcbagbt_verify(bp); + + if (fa) + xfs_verifier_error(bp, -EFSCORRUPTED, fa); +} + +/* skip crc checks on in-memory btrees to save time */ +static const struct xfs_buf_ops rcbagbt_mem_buf_ops = { + .name = "rcbagbt_mem", + .magic = { 0, cpu_to_be32(RCBAG_MAGIC) }, + .verify_read = rcbagbt_rw_verify, + .verify_write = rcbagbt_rw_verify, + .verify_struct = rcbagbt_verify, +}; + +static const struct xfs_btree_ops rcbagbt_mem_ops = { + .name = "rcbag", + .type = XFS_BTREE_TYPE_MEM, + + .rec_len = sizeof(struct rcbag_rec), + .key_len = sizeof(struct rcbag_key), + .ptr_len = XFS_BTREE_LONG_PTR_LEN, + + .lru_refs = 1, + .statoff = XFS_STATS_CALC_INDEX(xs_rcbag_2), + + .dup_cursor = xfbtree_dup_cursor, + .set_root = xfbtree_set_root, + .alloc_block = xfbtree_alloc_block, + .free_block = xfbtree_free_block, + .get_minrecs = xfbtree_get_minrecs, + .get_maxrecs = xfbtree_get_maxrecs, + .init_key_from_rec = rcbagbt_init_key_from_rec, + .init_rec_from_cur = rcbagbt_init_rec_from_cur, + .init_ptr_from_cur = xfbtree_init_ptr_from_cur, + .cmp_key_with_cur = rcbagbt_cmp_key_with_cur, + .buf_ops = &rcbagbt_mem_buf_ops, + .cmp_two_keys = rcbagbt_cmp_two_keys, + .keys_inorder = rcbagbt_keys_inorder, + .recs_inorder = rcbagbt_recs_inorder, +}; + +/* Create a cursor for an in-memory btree. */ +struct xfs_btree_cur * +rcbagbt_mem_cursor( + struct xfs_mount *mp, + struct xfs_trans *tp, + struct xfbtree *xfbtree) +{ + struct xfs_btree_cur *cur; + + cur = xfs_btree_alloc_cursor(mp, tp, &rcbagbt_mem_ops, + rcbagbt_maxlevels_possible(), rcbagbt_cur_cache); + + cur->bc_mem.xfbtree = xfbtree; + cur->bc_nlevels = xfbtree->nlevels; + return cur; +} + +/* Create an in-memory refcount bag btree. */ +int +rcbagbt_mem_init( + struct xfs_mount *mp, + struct xfbtree *xfbt, + struct xfs_buftarg *btp) +{ + xfbt->owner = 0; + return xfbtree_init(mp, xfbt, btp, &rcbagbt_mem_ops); +} + +/* Calculate number of records in a refcount bag btree block. */ +static inline unsigned int +rcbagbt_block_maxrecs( + unsigned int blocklen, + bool leaf) +{ + if (leaf) + return blocklen / sizeof(struct rcbag_rec); + return blocklen / + (sizeof(struct rcbag_key) + sizeof(rcbag_ptr_t)); +} + +/* + * Calculate number of records in an refcount bag btree block. + */ +unsigned int +rcbagbt_maxrecs( + struct xfs_mount *mp, + unsigned int blocklen, + bool leaf) +{ + blocklen -= RCBAG_BLOCK_LEN; + return rcbagbt_block_maxrecs(blocklen, leaf); +} + +/* Compute the max possible height for refcount bag btrees. */ +unsigned int +rcbagbt_maxlevels_possible(void) +{ + unsigned int minrecs[2]; + unsigned int blocklen; + + blocklen = XFBNO_BLOCKSIZE - XFS_BTREE_LBLOCK_CRC_LEN; + + minrecs[0] = rcbagbt_block_maxrecs(blocklen, true) / 2; + minrecs[1] = rcbagbt_block_maxrecs(blocklen, false) / 2; + + return xfs_btree_space_to_height(minrecs, ULLONG_MAX); +} + +/* Calculate the refcount bag btree size for some records. */ +unsigned long long +rcbagbt_calc_size( + unsigned long long nr_records) +{ + unsigned int minrecs[2]; + unsigned int blocklen; + + blocklen = XFBNO_BLOCKSIZE - XFS_BTREE_LBLOCK_CRC_LEN; + + minrecs[0] = rcbagbt_block_maxrecs(blocklen, true) / 2; + minrecs[1] = rcbagbt_block_maxrecs(blocklen, false) / 2; + + return xfs_btree_calc_size(minrecs, nr_records); +} + +int __init +rcbagbt_init_cur_cache(void) +{ + rcbagbt_cur_cache = kmem_cache_create("xfs_rcbagbt_cur", + xfs_btree_cur_sizeof(rcbagbt_maxlevels_possible()), + 0, 0, NULL); + + if (!rcbagbt_cur_cache) + return -ENOMEM; + return 0; +} + +void +rcbagbt_destroy_cur_cache(void) +{ + kmem_cache_destroy(rcbagbt_cur_cache); + rcbagbt_cur_cache = NULL; +} + +/* Look up the refcount bag record corresponding to this reverse mapping. */ +int +rcbagbt_lookup_eq( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rmap, + int *success) +{ + struct rcbag_rec *rec = (struct rcbag_rec *)&cur->bc_rec; + + rec->rbg_startblock = rmap->rm_startblock; + rec->rbg_blockcount = rmap->rm_blockcount; + + return xfs_btree_lookup(cur, XFS_LOOKUP_EQ, success); +} + +/* Get the data from the pointed-to record. */ +int +rcbagbt_get_rec( + struct xfs_btree_cur *cur, + struct rcbag_rec *rec, + int *has) +{ + union xfs_btree_rec *btrec; + int error; + + error = xfs_btree_get_rec(cur, &btrec, has); + if (error || !(*has)) + return error; + + memcpy(rec, btrec, sizeof(struct rcbag_rec)); + return 0; +} + +/* Update the record referred to by cur to the value given. */ +int +rcbagbt_update( + struct xfs_btree_cur *cur, + const struct rcbag_rec *rec) +{ + union xfs_btree_rec btrec; + + memcpy(&btrec, rec, sizeof(struct rcbag_rec)); + return xfs_btree_update(cur, &btrec); +} + +/* Update the record referred to by cur to the value given. */ +int +rcbagbt_insert( + struct xfs_btree_cur *cur, + const struct rcbag_rec *rec, + int *success) +{ + struct rcbag_rec *btrec = (struct rcbag_rec *)&cur->bc_rec; + + memcpy(btrec, rec, sizeof(struct rcbag_rec)); + return xfs_btree_insert(cur, success); +} diff --git a/fs/xfs/scrub/rcbag_btree.h b/fs/xfs/scrub/rcbag_btree.h new file mode 100644 index 000000000..03cadb032 --- /dev/null +++ b/fs/xfs/scrub/rcbag_btree.h @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2022-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_RCBAG_BTREE_H__ +#define __XFS_SCRUB_RCBAG_BTREE_H__ + +#ifdef CONFIG_XFS_BTREE_IN_MEM + +struct xfs_buf; +struct xfs_btree_cur; +struct xfs_mount; + +#define RCBAG_MAGIC 0x74826671 /* 'JRBG' */ + +struct rcbag_key { + uint32_t rbg_startblock; + uint32_t rbg_blockcount; +}; + +struct rcbag_rec { + uint32_t rbg_startblock; + uint32_t rbg_blockcount; + uint64_t rbg_refcount; +}; + +typedef __be64 rcbag_ptr_t; + +/* reflinks only exist on crc enabled filesystems */ +#define RCBAG_BLOCK_LEN XFS_BTREE_LBLOCK_CRC_LEN + +/* + * Record, key, and pointer address macros for btree blocks. + * + * (note that some of these may appear unused, but they are used in userspace) + */ +#define RCBAG_REC_ADDR(block, index) \ + ((struct rcbag_rec *) \ + ((char *)(block) + RCBAG_BLOCK_LEN + \ + (((index) - 1) * sizeof(struct rcbag_rec)))) + +#define RCBAG_KEY_ADDR(block, index) \ + ((struct rcbag_key *) \ + ((char *)(block) + RCBAG_BLOCK_LEN + \ + ((index) - 1) * sizeof(struct rcbag_key))) + +#define RCBAG_PTR_ADDR(block, index, maxrecs) \ + ((rcbag_ptr_t *) \ + ((char *)(block) + RCBAG_BLOCK_LEN + \ + (maxrecs) * sizeof(struct rcbag_key) + \ + ((index) - 1) * sizeof(rcbag_ptr_t))) + +unsigned int rcbagbt_maxrecs(struct xfs_mount *mp, unsigned int blocklen, + bool leaf); + +unsigned long long rcbagbt_calc_size(unsigned long long nr_records); + +unsigned int rcbagbt_maxlevels_possible(void); + +int __init rcbagbt_init_cur_cache(void); +void rcbagbt_destroy_cur_cache(void); + +struct xfs_btree_cur *rcbagbt_mem_cursor(struct xfs_mount *mp, + struct xfs_trans *tp, struct xfbtree *xfbtree); +int rcbagbt_mem_init(struct xfs_mount *mp, struct xfbtree *xfbtree, + struct xfs_buftarg *btp); + +int rcbagbt_lookup_eq(struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rmap, int *success); +int rcbagbt_get_rec(struct xfs_btree_cur *cur, struct rcbag_rec *rec, int *has); +int rcbagbt_update(struct xfs_btree_cur *cur, const struct rcbag_rec *rec); +int rcbagbt_insert(struct xfs_btree_cur *cur, const struct rcbag_rec *rec, + int *success); + +#else +# define rcbagbt_init_cur_cache() 0 +# define rcbagbt_destroy_cur_cache() ((void)0) +#endif /* CONFIG_XFS_BTREE_IN_MEM */ + +#endif /* __XFS_SCRUB_RCBAG_BTREE_H__ */ diff --git a/fs/xfs/scrub/readdir.c b/fs/xfs/scrub/readdir.c new file mode 100644 index 000000000..d68bf20b4 --- /dev/null +++ b/fs/xfs/scrub/readdir.c @@ -0,0 +1,421 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2022-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_dir2.h" +#include "xfs_dir2_priv.h" +#include "xfs_trace.h" +#include "xfs_bmap.h" +#include "xfs_trans.h" +#include "xfs_error.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/readdir.h" + +/* Call a function for every entry in a shortform directory. */ +STATIC int +xchk_dir_walk_sf( + struct xfs_scrub *sc, + struct xfs_inode *dp, + xchk_dirent_fn dirent_fn, + void *priv) +{ + struct xfs_name name = { + .name = ".", + .len = 1, + .type = XFS_DIR3_FT_DIR, + }; + struct xfs_mount *mp = dp->i_mount; + struct xfs_da_geometry *geo = mp->m_dir_geo; + struct xfs_dir2_sf_entry *sfep; + struct xfs_dir2_sf_hdr *sfp = dp->i_df.if_data; + xfs_ino_t ino; + xfs_dir2_dataptr_t dapos; + unsigned int i; + int error; + + ASSERT(dp->i_df.if_bytes == dp->i_disk_size); + ASSERT(sfp != NULL); + + /* dot entry */ + dapos = xfs_dir2_db_off_to_dataptr(geo, geo->datablk, + geo->data_entry_offset); + + error = dirent_fn(sc, dp, dapos, &name, I_INO(dp), priv); + if (error) + return error; + + /* dotdot entry */ + dapos = xfs_dir2_db_off_to_dataptr(geo, geo->datablk, + geo->data_entry_offset + + xfs_dir2_data_entsize(mp, sizeof(".") - 1)); + ino = xfs_dir2_sf_get_parent_ino(sfp); + name.name = ".."; + name.len = 2; + + error = dirent_fn(sc, dp, dapos, &name, ino, priv); + if (error) + return error; + + /* iterate everything else */ + sfep = xfs_dir2_sf_firstentry(sfp); + for (i = 0; i < sfp->count; i++) { + dapos = xfs_dir2_db_off_to_dataptr(geo, geo->datablk, + xfs_dir2_sf_get_offset(sfep)); + ino = xfs_dir2_sf_get_ino(mp, sfp, sfep); + name.name = sfep->name; + name.len = sfep->namelen; + name.type = xfs_dir2_sf_get_ftype(mp, sfep); + + error = dirent_fn(sc, dp, dapos, &name, ino, priv); + if (error) + return error; + + sfep = xfs_dir2_sf_nextentry(mp, sfp, sfep); + } + + return 0; +} + +/* Call a function for every entry in a block directory. */ +STATIC int +xchk_dir_walk_block( + struct xfs_scrub *sc, + struct xfs_inode *dp, + xchk_dirent_fn dirent_fn, + void *priv) +{ + struct xfs_mount *mp = dp->i_mount; + struct xfs_da_geometry *geo = mp->m_dir_geo; + struct xfs_buf *bp; + unsigned int off, next_off, end; + int error; + + error = xfs_dir3_block_read(sc->tp, dp, I_INO(dp), &bp); + if (error) + return error; + + /* Walk each directory entry. */ + end = xfs_dir3_data_end_offset(geo, bp->b_addr); + for (off = geo->data_entry_offset; off < end; off = next_off) { + struct xfs_name name = { }; + struct xfs_dir2_data_unused *dup = bp->b_addr + off; + struct xfs_dir2_data_entry *dep = bp->b_addr + off; + xfs_ino_t ino; + xfs_dir2_dataptr_t dapos; + + /* Skip an empty entry. */ + if (be16_to_cpu(dup->freetag) == XFS_DIR2_DATA_FREE_TAG) { + next_off = off + be16_to_cpu(dup->length); + continue; + } + + /* Otherwise, find the next entry and report it. */ + next_off = off + xfs_dir2_data_entsize(mp, dep->namelen); + if (next_off > end) + break; + + dapos = xfs_dir2_db_off_to_dataptr(geo, geo->datablk, off); + ino = be64_to_cpu(dep->inumber); + name.name = dep->name; + name.len = dep->namelen; + name.type = xfs_dir2_data_get_ftype(mp, dep); + + error = dirent_fn(sc, dp, dapos, &name, ino, priv); + if (error) + break; + } + + xfs_trans_brelse(sc->tp, bp); + return error; +} + +/* Read a leaf-format directory buffer. */ +STATIC int +xchk_read_leaf_dir_buf( + struct xfs_trans *tp, + struct xfs_inode *dp, + struct xfs_da_geometry *geo, + xfs_dir2_off_t *curoff, + struct xfs_buf **bpp) +{ + struct xfs_iext_cursor icur; + struct xfs_bmbt_irec map; + struct xfs_ifork *ifp = xfs_ifork_ptr(dp, XFS_DATA_FORK); + xfs_dablk_t last_da; + xfs_dablk_t map_off; + xfs_dir2_off_t new_off; + + *bpp = NULL; + + /* + * Look for mapped directory blocks at or above the current offset. + * Truncate down to the nearest directory block to start the scanning + * operation. + */ + last_da = xfs_dir2_byte_to_da(geo, XFS_DIR2_LEAF_OFFSET); + map_off = xfs_dir2_db_to_da(geo, xfs_dir2_byte_to_db(geo, *curoff)); + + if (!xfs_iext_lookup_extent(dp, ifp, map_off, &icur, &map)) + return 0; + if (map.br_startoff >= last_da) + return 0; + xfs_trim_extent(&map, map_off, last_da - map_off); + + /* Read the directory block of that first mapping. */ + new_off = xfs_dir2_da_to_byte(geo, map.br_startoff); + if (new_off > *curoff) + *curoff = new_off; + + return xfs_dir3_data_read(tp, dp, I_INO(dp), map.br_startoff, 0, bpp); +} + +/* Call a function for every entry in a leaf directory. */ +STATIC int +xchk_dir_walk_leaf( + struct xfs_scrub *sc, + struct xfs_inode *dp, + xchk_dirent_fn dirent_fn, + void *priv) +{ + struct xfs_mount *mp = dp->i_mount; + struct xfs_da_geometry *geo = mp->m_dir_geo; + struct xfs_buf *bp = NULL; + xfs_dir2_off_t curoff = 0; + unsigned int offset = 0; + int error; + + /* Iterate every directory offset in this directory. */ + while (curoff < XFS_DIR2_LEAF_OFFSET) { + struct xfs_name name = { }; + struct xfs_dir2_data_unused *dup; + struct xfs_dir2_data_entry *dep; + xfs_ino_t ino; + unsigned int length; + xfs_dir2_dataptr_t dapos; + + /* + * If we have no buffer, or we're off the end of the + * current buffer, need to get another one. + */ + if (!bp || offset >= geo->blksize) { + if (bp) { + xfs_trans_brelse(sc->tp, bp); + bp = NULL; + } + + error = xchk_read_leaf_dir_buf(sc->tp, dp, geo, &curoff, + &bp); + if (error || !bp) + break; + + /* + * Find our position in the block. + */ + offset = geo->data_entry_offset; + curoff += geo->data_entry_offset; + } + + /* Skip an empty entry. */ + dup = bp->b_addr + offset; + if (be16_to_cpu(dup->freetag) == XFS_DIR2_DATA_FREE_TAG) { + length = be16_to_cpu(dup->length); + offset += length; + curoff += length; + continue; + } + + /* Otherwise, find the next entry and report it. */ + dep = bp->b_addr + offset; + length = xfs_dir2_data_entsize(mp, dep->namelen); + + dapos = xfs_dir2_byte_to_dataptr(curoff) & 0x7fffffff; + ino = be64_to_cpu(dep->inumber); + name.name = dep->name; + name.len = dep->namelen; + name.type = xfs_dir2_data_get_ftype(mp, dep); + + error = dirent_fn(sc, dp, dapos, &name, ino, priv); + if (error) + break; + + /* Advance to the next entry. */ + offset += length; + curoff += length; + } + + if (bp) + xfs_trans_brelse(sc->tp, bp); + return error; +} + +/* + * Call a function for every entry in a directory. + * + * Callers must hold the ILOCK. File types are XFS_DIR3_FT_*. + */ +int +xchk_dir_walk( + struct xfs_scrub *sc, + struct xfs_inode *dp, + xchk_dirent_fn dirent_fn, + void *priv) +{ + struct xfs_da_args args = { + .dp = dp, + .geo = dp->i_mount->m_dir_geo, + .trans = sc->tp, + .owner = I_INO(dp), + }; + int error; + + if (xfs_is_shutdown(dp->i_mount)) + return -EIO; + + ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); + xfs_assert_ilocked(dp, XFS_ILOCK_SHARED | XFS_ILOCK_EXCL); + + switch (xfs_dir2_format(&args, &error)) { + case XFS_DIR2_FMT_SF: + return xchk_dir_walk_sf(sc, dp, dirent_fn, priv); + case XFS_DIR2_FMT_BLOCK: + return xchk_dir_walk_block(sc, dp, dirent_fn, priv); + case XFS_DIR2_FMT_LEAF: + case XFS_DIR2_FMT_NODE: + return xchk_dir_walk_leaf(sc, dp, dirent_fn, priv); + default: + return error; + } +} + +/* + * Look up the inode number for an exact name in a directory. + * + * Callers must hold the ILOCK. File types are XFS_DIR3_FT_*. Names are not + * checked for correctness. + */ +int +xchk_dir_lookup( + struct xfs_scrub *sc, + struct xfs_inode *dp, + const struct xfs_name *name, + xfs_ino_t *ino) +{ + struct xfs_da_args args = { + .dp = dp, + .geo = dp->i_mount->m_dir_geo, + .trans = sc->tp, + .name = name->name, + .namelen = name->len, + .filetype = name->type, + .hashval = xfs_dir2_hashname(dp->i_mount, name), + .whichfork = XFS_DATA_FORK, + .op_flags = XFS_DA_OP_OKNOENT, + .owner = I_INO(dp), + }; + int error; + + if (xfs_is_shutdown(dp->i_mount)) + return -EIO; + + /* + * A temporary directory's block headers are written with the owner + * set to sc->ip, so we must switch the owner here for the lookup. + */ + if (dp == sc->tempip) + args.owner = I_INO(sc->ip); + + ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); + xfs_assert_ilocked(dp, XFS_ILOCK_SHARED | XFS_ILOCK_EXCL); + + error = xfs_dir_lookup_args(&args); + if (!error) + *ino = args.inumber; + return error; +} + +/* + * Try to grab the IOLOCK and ILOCK of sc->ip and ip, returning @ip's lock + * state. The caller may have a transaction, so we must use trylock for both + * IOLOCKs. + */ +static inline unsigned int +xchk_dir_trylock_both( + struct xfs_scrub *sc, + struct xfs_inode *ip) +{ + if (!xchk_ilock_nowait(sc, XFS_IOLOCK_EXCL)) + return 0; + + if (!xfs_ilock_nowait(ip, XFS_IOLOCK_SHARED)) + goto parent_iolock; + + xchk_ilock(sc, XFS_ILOCK_EXCL); + if (!xfs_ilock_nowait(ip, XFS_ILOCK_EXCL)) + goto parent_ilock; + + return XFS_IOLOCK_SHARED | XFS_ILOCK_EXCL; + +parent_ilock: + xchk_iunlock(sc, XFS_ILOCK_EXCL); + xfs_iunlock(ip, XFS_IOLOCK_SHARED); +parent_iolock: + xchk_iunlock(sc, XFS_IOLOCK_EXCL); + return 0; +} + +/* + * Try for a limited time to grab the IOLOCK and ILOCK of both the scrub target + * (@sc->ip) and the inode at the other end (@ip) of a directory or parent + * pointer link so that we can check that link. + * + * We do not know ahead of time that the directory tree is /not/ corrupt, so we + * cannot use the "lock two inode" functions because we do not know that there + * is not a racing thread trying to take the locks in opposite order. First + * take IOLOCK_EXCL of the scrub target, and then try to take IOLOCK_SHARED + * of @ip to synchronize with the VFS. Next, take ILOCK_EXCL of the scrub + * target and @ip to synchronize with XFS. + * + * If the trylocks succeed, *lockmode will be set to the locks held for @ip; + * @sc->ilock_flags will be set for the locks held for @sc->ip; and zero will + * be returned. If not, returns -EDEADLOCK to try again; or -ETIMEDOUT if + * XCHK_TRY_HARDER was set. Returns -EINTR if the process has been killed. + */ +int +xchk_dir_trylock_for_pptrs( + struct xfs_scrub *sc, + struct xfs_inode *ip, + unsigned int *lockmode) +{ + unsigned int nr; + int error = 0; + + ASSERT(sc->ilock_flags == 0); + + for (nr = 0; nr < HZ; nr++) { + *lockmode = xchk_dir_trylock_both(sc, ip); + if (*lockmode) + return 0; + + if (xchk_should_terminate(sc, &error)) + return error; + + delay(1); + } + + if (sc->flags & XCHK_TRY_HARDER) { + xchk_set_incomplete(sc); + return -ETIMEDOUT; + } + + return -EDEADLOCK; +} diff --git a/fs/xfs/scrub/readdir.h b/fs/xfs/scrub/readdir.h new file mode 100644 index 000000000..da501877a --- /dev/null +++ b/fs/xfs/scrub/readdir.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2022-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_READDIR_H__ +#define __XFS_SCRUB_READDIR_H__ + +typedef int (*xchk_dirent_fn)(struct xfs_scrub *sc, struct xfs_inode *dp, + xfs_dir2_dataptr_t dapos, const struct xfs_name *name, + xfs_ino_t ino, void *priv); + +int xchk_dir_walk(struct xfs_scrub *sc, struct xfs_inode *dp, + xchk_dirent_fn dirent_fn, void *priv); + +int xchk_dir_lookup(struct xfs_scrub *sc, struct xfs_inode *dp, + const struct xfs_name *name, xfs_ino_t *ino); + +int xchk_dir_trylock_for_pptrs(struct xfs_scrub *sc, struct xfs_inode *ip, + unsigned int *lockmode); + +#endif /* __XFS_SCRUB_READDIR_H__ */ diff --git a/fs/xfs/scrub/reap.c b/fs/xfs/scrub/reap.c new file mode 100644 index 000000000..fcd14c170 --- /dev/null +++ b/fs/xfs/scrub/reap.c @@ -0,0 +1,1694 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2022-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_btree.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_sb.h" +#include "xfs_inode.h" +#include "xfs_alloc.h" +#include "xfs_alloc_btree.h" +#include "xfs_ialloc.h" +#include "xfs_ialloc_btree.h" +#include "xfs_rmap.h" +#include "xfs_rmap_btree.h" +#include "xfs_refcount.h" +#include "xfs_refcount_btree.h" +#include "xfs_extent_busy.h" +#include "xfs_ag.h" +#include "xfs_ag_resv.h" +#include "xfs_quota.h" +#include "xfs_qm.h" +#include "xfs_bmap.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_attr.h" +#include "xfs_attr_remote.h" +#include "xfs_defer.h" +#include "xfs_metafile.h" +#include "xfs_rtgroup.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_extfree_item.h" +#include "xfs_rmap_item.h" +#include "xfs_refcount_item.h" +#include "xfs_buf_item.h" +#include "xfs_bmap_item.h" +#include "xfs_bmap_btree.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/bitmap.h" +#include "scrub/agb_bitmap.h" +#include "scrub/fsb_bitmap.h" +#include "scrub/rtb_bitmap.h" +#include "scrub/reap.h" + +/* + * Disposal of Blocks from Old Metadata + * + * Now that we've constructed a new btree to replace the damaged one, we want + * to dispose of the blocks that (we think) the old btree was using. + * Previously, we used the rmapbt to collect the extents (bitmap) with the + * rmap owner corresponding to the tree we rebuilt, collected extents for any + * blocks with the same rmap owner that are owned by another data structure + * (sublist), and subtracted sublist from bitmap. In theory the extents + * remaining in bitmap are the old btree's blocks. + * + * Unfortunately, it's possible that the btree was crosslinked with other + * blocks on disk. The rmap data can tell us if there are multiple owners, so + * if the rmapbt says there is an owner of this block other than @oinfo, then + * the block is crosslinked. Remove the reverse mapping and continue. + * + * If there is one rmap record, we can free the block, which removes the + * reverse mapping but doesn't add the block to the free space. Our repair + * strategy is to hope the other metadata objects crosslinked on this block + * will be rebuilt (atop different blocks), thereby removing all the cross + * links. + * + * If there are no rmap records at all, we also free the block. If the btree + * being rebuilt lives in the free space (bnobt/cntbt/rmapbt) then there isn't + * supposed to be a rmap record and everything is ok. For other btrees there + * had to have been an rmap entry for the block to have ended up on @bitmap, + * so if it's gone now there's something wrong and the fs will shut down. + * + * Note: If there are multiple rmap records with only the same rmap owner as + * the btree we're trying to rebuild and the block is indeed owned by another + * data structure with the same rmap owner, then the block will be in sublist + * and therefore doesn't need disposal. If there are multiple rmap records + * with only the same rmap owner but the block is not owned by something with + * the same rmap owner, the block will be freed. + * + * The caller is responsible for locking the AG headers/inode for the entire + * rebuild operation so that nothing else can sneak in and change the incore + * state while we're not looking. We must also invalidate any buffers + * associated with @bitmap. + */ + +/* Information about reaping extents after a repair. */ +struct xreap_state { + struct xfs_scrub *sc; + + union { + struct { + /* + * For AG blocks, this is reverse mapping owner and + * metadata reservation type. + */ + const struct xfs_owner_info *oinfo; + enum xfs_ag_resv_type resv; + }; + struct { + /* For file blocks, this is the inode and fork. */ + struct xfs_inode *ip; + int whichfork; + }; + }; + + /* Number of invalidated buffers logged to the current transaction. */ + unsigned int nr_binval; + + /* Maximum number of buffers we can invalidate in a single tx. */ + unsigned int max_binval; + + /* Number of deferred reaps attached to the current transaction. */ + unsigned int nr_deferred; + + /* Maximum number of intents we can reap in a single transaction. */ + unsigned int max_deferred; +}; + +/* Put a block back on the AGFL. */ +STATIC int +xreap_put_freelist( + struct xfs_scrub *sc, + xfs_agblock_t agbno) +{ + struct xfs_buf *agfl_bp; + int error; + + /* Make sure there's space on the freelist. */ + error = xrep_fix_freelist(sc, 0); + if (error) + return error; + + /* + * Since we're "freeing" a lost block onto the AGFL, we have to + * create an rmap for the block prior to merging it or else other + * parts will break. + */ + error = xfs_rmap_alloc(sc->tp, sc->sa.agf_bp, sc->sa.pag, agbno, 1, + &XFS_RMAP_OINFO_AG); + if (error) + return error; + + /* Put the block on the AGFL. */ + error = xfs_alloc_read_agfl(sc->sa.pag, sc->tp, &agfl_bp); + if (error) + return error; + + error = xfs_alloc_put_freelist(sc->sa.pag, sc->tp, sc->sa.agf_bp, + agfl_bp, agbno, 0); + if (error) + return error; + xfs_extent_busy_insert(sc->tp, pag_group(sc->sa.pag), agbno, 1, + XFS_EXTENT_BUSY_SKIP_DISCARD); + + return 0; +} + +/* Are there any uncommitted reap operations? */ +static inline bool xreap_is_dirty(const struct xreap_state *rs) +{ + return rs->nr_binval > 0 || rs->nr_deferred > 0; +} + +/* + * Decide if we need to roll the transaction to clear out the the log + * reservation that we allocated to buffer invalidations. + */ +static inline bool xreap_want_binval_roll(const struct xreap_state *rs) +{ + return rs->nr_binval >= rs->max_binval; +} + +/* Reset the buffer invalidation count after rolling. */ +static inline void xreap_binval_reset(struct xreap_state *rs) +{ + rs->nr_binval = 0; +} + +/* + * Bump the number of invalidated buffers, and return true if we can continue, + * or false if we need to roll the transaction. + */ +static inline bool xreap_inc_binval(struct xreap_state *rs) +{ + rs->nr_binval++; + return rs->nr_binval < rs->max_binval; +} + +/* + * Decide if we want to finish the deferred ops that are attached to the scrub + * transaction. We don't want to queue huge chains of deferred ops because + * that can consume a lot of log space and kernel memory. Hence we trigger a + * xfs_defer_finish if there are too many deferred reap operations or we've run + * out of space for invalidations. + */ +static inline bool xreap_want_defer_finish(const struct xreap_state *rs) +{ + return rs->nr_deferred >= rs->max_deferred; +} + +/* + * Reset the defer chain length and buffer invalidation count after finishing + * items. + */ +static inline void xreap_defer_finish_reset(struct xreap_state *rs) +{ + rs->nr_deferred = 0; + rs->nr_binval = 0; +} + +/* + * Bump the number of deferred extent reaps. + */ +static inline void xreap_inc_defer(struct xreap_state *rs) +{ + rs->nr_deferred++; +} + +/* Force the caller to finish a deferred item chain. */ +static inline void xreap_force_defer_finish(struct xreap_state *rs) +{ + rs->nr_deferred = rs->max_deferred; +} + +/* Maximum number of fsblocks that we might find in a buffer to invalidate. */ +static inline unsigned int +xrep_binval_max_fsblocks( + struct xfs_mount *mp) +{ + /* Remote xattr values are the largest buffers that we support. */ + return xfs_attr3_max_rmt_blocks(mp); +} + +/* + * Compute the maximum length of a buffer cache scan (in units of sectors), + * given a quantity of fs blocks. + */ +xfs_daddr_t +xrep_bufscan_max_sectors( + struct xfs_mount *mp, + xfs_extlen_t fsblocks) +{ + return XFS_FSB_TO_BB(mp, min_t(xfs_extlen_t, fsblocks, + xrep_binval_max_fsblocks(mp))); +} + +/* + * Return an incore buffer from a sector scan, or NULL if there are no buffers + * left to return. + */ +struct xfs_buf * +xrep_bufscan_advance( + struct xfs_mount *mp, + struct xrep_bufscan *scan) +{ + scan->__sector_count += scan->daddr_step; + while (scan->__sector_count <= scan->max_sectors) { + struct xfs_buf *bp = NULL; + int error; + + error = xfs_buf_incore(mp->m_ddev_targp, scan->daddr, + scan->__sector_count, XBF_LIVESCAN, &bp); + if (!error) + return bp; + + scan->__sector_count += scan->daddr_step; + } + + return NULL; +} + +/* Try to invalidate the incore buffers for an extent that we're freeing. */ +STATIC void +xreap_agextent_binval( + struct xreap_state *rs, + xfs_agblock_t agbno, + xfs_extlen_t *aglenp) +{ + struct xfs_scrub *sc = rs->sc; + struct xfs_perag *pag = sc->sa.pag; + struct xfs_mount *mp = sc->mp; + xfs_agblock_t agbno_next = agbno + *aglenp; + xfs_agblock_t bno = agbno; + + /* + * Avoid invalidating AG headers and post-EOFS blocks because we never + * own those. + */ + if (!xfs_verify_agbno(pag, agbno) || + !xfs_verify_agbno(pag, agbno_next - 1)) + return; + + /* + * If there are incore buffers for these blocks, invalidate them. We + * assume that the lack of any other known owners means that the buffer + * can be locked without risk of deadlocking. The buffer cache cannot + * detect aliasing, so employ nested loops to scan for incore buffers + * of any plausible size. + */ + while (bno < agbno_next) { + struct xrep_bufscan scan = { + .daddr = xfs_agbno_to_daddr(pag, bno), + .max_sectors = xrep_bufscan_max_sectors(mp, + agbno_next - bno), + .daddr_step = XFS_FSB_TO_BB(mp, 1), + }; + struct xfs_buf *bp; + + while ((bp = xrep_bufscan_advance(mp, &scan)) != NULL) { + xfs_trans_bjoin(sc->tp, bp); + xfs_trans_binval(sc->tp, bp); + + /* + * Stop invalidating if we've hit the limit; we should + * still have enough reservation left to free however + * far we've gotten. + */ + if (!xreap_inc_binval(rs)) { + *aglenp -= agbno_next - bno; + goto out; + } + } + + bno++; + } + +out: + trace_xreap_agextent_binval(pag_group(sc->sa.pag), agbno, *aglenp); +} + +/* + * Figure out the longest run of blocks that we can dispose of with a single + * call. Cross-linked blocks should have their reverse mappings removed, but + * single-owner extents can be freed. AGFL blocks can only be put back one at + * a time. + */ +STATIC int +xreap_agextent_select( + struct xreap_state *rs, + xfs_agblock_t agbno, + xfs_agblock_t agbno_next, + bool *crosslinked, + xfs_extlen_t *aglenp) +{ + struct xfs_scrub *sc = rs->sc; + struct xfs_btree_cur *cur; + xfs_agblock_t bno = agbno + 1; + xfs_extlen_t len = 1; + int error; + + /* + * Determine if there are any other rmap records covering the first + * block of this extent. If so, the block is crosslinked. + */ + cur = xfs_rmapbt_init_cursor(sc->mp, sc->tp, sc->sa.agf_bp, + sc->sa.pag); + error = xfs_rmap_has_other_keys(cur, agbno, 1, rs->oinfo, + crosslinked); + if (error) + goto out_cur; + + /* AGFL blocks can only be deal with one at a time. */ + if (rs->resv == XFS_AG_RESV_AGFL) + goto out_found; + + /* + * Figure out how many of the subsequent blocks have the same crosslink + * status. + */ + while (bno < agbno_next) { + bool also_crosslinked; + + error = xfs_rmap_has_other_keys(cur, bno, 1, rs->oinfo, + &also_crosslinked); + if (error) + goto out_cur; + + if (*crosslinked != also_crosslinked) + break; + + len++; + bno++; + } + +out_found: + *aglenp = len; + trace_xreap_agextent_select(pag_group(sc->sa.pag), agbno, len, + *crosslinked); +out_cur: + xfs_btree_del_cursor(cur, error); + return error; +} + +/* + * Dispose of as much of the beginning of this AG extent as possible. The + * number of blocks disposed of will be returned in @aglenp. + */ +STATIC int +xreap_agextent_iter( + struct xreap_state *rs, + xfs_agblock_t agbno, + xfs_extlen_t *aglenp, + bool crosslinked) +{ + struct xfs_scrub *sc = rs->sc; + xfs_fsblock_t fsbno; + int error = 0; + + ASSERT(rs->resv != XFS_AG_RESV_METAFILE); + + fsbno = xfs_agbno_to_fsb(sc->sa.pag, agbno); + + /* + * If there are other rmappings, this block is cross linked and must + * not be freed. Remove the reverse mapping and move on. Otherwise, + * we were the only owner of the block, so free the extent, which will + * also remove the rmap. + * + * XXX: XFS doesn't support detecting the case where a single block + * metadata structure is crosslinked with a multi-block structure + * because the buffer cache doesn't detect aliasing problems, so we + * can't fix 100% of crosslinking problems (yet). The verifiers will + * blow on writeout, the filesystem will shut down, and the admin gets + * to run xfs_repair. + */ + if (crosslinked) { + trace_xreap_dispose_unmap_extent(pag_group(sc->sa.pag), agbno, + *aglenp); + + if (rs->oinfo == &XFS_RMAP_OINFO_COW) { + /* + * t0: Unmapping CoW staging extents, remove the + * records from the refcountbt, which will remove the + * rmap record as well. + */ + xfs_refcount_free_cow_extent(sc->tp, false, fsbno, + *aglenp); + xreap_inc_defer(rs); + return 0; + } + + /* t1: unmap crosslinked metadata blocks */ + xfs_rmap_free_extent(sc->tp, false, fsbno, *aglenp, + rs->oinfo->oi_owner); + xreap_inc_defer(rs); + return 0; + } + + trace_xreap_dispose_free_extent(pag_group(sc->sa.pag), agbno, *aglenp); + + /* + * Invalidate as many buffers as we can, starting at agbno. If this + * function sets *aglenp to zero, the transaction is full of logged + * buffer invalidations, so we need to return early so that we can + * roll and retry. + */ + xreap_agextent_binval(rs, agbno, aglenp); + if (*aglenp == 0) { + ASSERT(xreap_want_binval_roll(rs)); + return 0; + } + + /* + * t2: To get rid of CoW staging extents, use deferred work items + * to remove the refcountbt records (which removes the rmap records) + * and free the extent. We're not worried about the system going down + * here because log recovery walks the refcount btree to clean out the + * CoW staging extents. + */ + if (rs->oinfo == &XFS_RMAP_OINFO_COW) { + ASSERT(rs->resv == XFS_AG_RESV_NONE); + + xfs_refcount_free_cow_extent(sc->tp, false, fsbno, *aglenp); + error = xfs_free_extent_later(sc->tp, fsbno, *aglenp, NULL, + rs->resv, XFS_FREE_EXTENT_SKIP_DISCARD); + if (error) + return error; + + xreap_inc_defer(rs); + return 0; + } + + /* t3: Put blocks back on the AGFL one at a time. */ + if (rs->resv == XFS_AG_RESV_AGFL) { + ASSERT(*aglenp == 1); + error = xreap_put_freelist(sc, agbno); + if (error) + return error; + + xreap_force_defer_finish(rs); + return 0; + } + + /* + * t4: Use deferred frees to get rid of the old btree blocks to try to + * minimize the window in which we could crash and lose the old blocks. + * Add a defer ops barrier every other extent to avoid stressing the + * system with large EFIs. + */ + error = xfs_free_extent_later(sc->tp, fsbno, *aglenp, rs->oinfo, + rs->resv, XFS_FREE_EXTENT_SKIP_DISCARD); + if (error) + return error; + + xreap_inc_defer(rs); + if (rs->nr_deferred % 2 == 0) + xfs_defer_add_barrier(sc->tp); + return 0; +} + +/* Configure the deferral and invalidation limits */ +static inline void +xreap_configure_limits( + struct xreap_state *rs, + unsigned int fixed_overhead, + unsigned int variable_overhead, + unsigned int per_intent, + unsigned int per_binval) +{ + struct xfs_scrub *sc = rs->sc; + unsigned int res = sc->tp->t_log_res - fixed_overhead; + + /* Don't underflow the reservation */ + if (sc->tp->t_log_res < (fixed_overhead + variable_overhead)) { + ASSERT(sc->tp->t_log_res >= + (fixed_overhead + variable_overhead)); + xfs_force_shutdown(sc->mp, SHUTDOWN_CORRUPT_INCORE); + return; + } + + rs->max_deferred = per_intent ? res / variable_overhead : 0; + res -= rs->max_deferred * per_intent; + rs->max_binval = per_binval ? res / per_binval : 0; +} + +/* + * Compute the maximum number of intent items that reaping can attach to the + * scrub transaction given the worst case log overhead of the intent items + * needed to reap a single per-AG space extent. This is not for freeing CoW + * staging extents. + */ +STATIC void +xreap_configure_agextent_limits( + struct xreap_state *rs) +{ + struct xfs_scrub *sc = rs->sc; + struct xfs_mount *mp = sc->mp; + + /* + * In the worst case, relogging an intent item causes both an intent + * item and a done item to be attached to a transaction for each extent + * that we'd like to process. + */ + const unsigned int efi = xfs_efi_log_space(1) + + xfs_efd_log_space(1); + const unsigned int rui = xfs_rui_log_space(1) + + xfs_rud_log_space(); + + /* + * Various things can happen when reaping non-CoW metadata blocks: + * + * t1: Unmapping crosslinked metadata blocks: deferred removal of rmap + * record. + * + * t3: Freeing to AGFL: roll and finish deferred items for every block. + * Limits here do not matter. + * + * t4: Freeing metadata blocks: deferred freeing of the space, which + * also removes the rmap record. + * + * For simplicity, we'll use the worst-case intents size to determine + * the maximum number of deferred extents before we have to finish the + * whole chain. If we're trying to reap a btree larger than this size, + * a crash midway through reaping can result in leaked blocks. + */ + const unsigned int t1 = rui; + const unsigned int t4 = rui + efi; + const unsigned int per_intent = max(t1, t4); + + /* + * For each transaction in a reap chain, we must be able to take one + * step in the defer item chain, which should only consist of EFI or + * RUI items. + */ + const unsigned int f1 = xfs_calc_finish_efi_reservation(mp, 1); + const unsigned int f2 = xfs_calc_finish_rui_reservation(mp, 1); + const unsigned int step_size = max(f1, f2); + + /* Largest buffer size (in fsblocks) that can be invalidated. */ + const unsigned int max_binval = xrep_binval_max_fsblocks(mp); + + /* Maximum overhead of invalidating one buffer. */ + const unsigned int per_binval = + xfs_buf_inval_log_space(1, XFS_B_TO_FSBT(mp, max_binval)); + + /* + * For each transaction in a reap chain, we can delete some number of + * extents and invalidate some number of blocks. We assume that btree + * blocks aren't usually contiguous; and that scrub likely pulled all + * the buffers into memory. From these assumptions, set the maximum + * number of deferrals we can queue before flushing the defer chain, + * and the number of invalidations we can queue before rolling to a + * clean transaction (and possibly relogging some of the deferrals) to + * the same quantity. + */ + const unsigned int variable_overhead = per_intent + per_binval; + + xreap_configure_limits(rs, step_size, variable_overhead, per_intent, + per_binval); + + trace_xreap_agextent_limits(sc->tp, per_binval, rs->max_binval, + step_size, per_intent, rs->max_deferred); +} + +/* + * Compute the maximum number of intent items that reaping can attach to the + * scrub transaction given the worst case log overhead of the intent items + * needed to reap a single CoW staging extent. This is not for freeing + * metadata blocks. + */ +STATIC void +xreap_configure_agcow_limits( + struct xreap_state *rs) +{ + struct xfs_scrub *sc = rs->sc; + struct xfs_mount *mp = sc->mp; + + /* + * In the worst case, relogging an intent item causes both an intent + * item and a done item to be attached to a transaction for each extent + * that we'd like to process. + */ + const unsigned int efi = xfs_efi_log_space(1) + + xfs_efd_log_space(1); + const unsigned int rui = xfs_rui_log_space(1) + + xfs_rud_log_space(); + const unsigned int cui = xfs_cui_log_space(1) + + xfs_cud_log_space(); + + /* + * Various things can happen when reaping non-CoW metadata blocks: + * + * t0: Unmapping crosslinked CoW blocks: deferred removal of refcount + * record, which defers removal of rmap record + * + * t2: Freeing CoW blocks: deferred removal of refcount record, which + * defers removal of rmap record; and deferred removal of the space + * + * For simplicity, we'll use the worst-case intents size to determine + * the maximum number of deferred extents before we have to finish the + * whole chain. If we're trying to reap a btree larger than this size, + * a crash midway through reaping can result in leaked blocks. + */ + const unsigned int t0 = cui + rui; + const unsigned int t2 = cui + rui + efi; + const unsigned int per_intent = max(t0, t2); + + /* + * For each transaction in a reap chain, we must be able to take one + * step in the defer item chain, which should only consist of CUI, EFI, + * or RUI items. + */ + const unsigned int f1 = xfs_calc_finish_efi_reservation(mp, 1); + const unsigned int f2 = xfs_calc_finish_rui_reservation(mp, 1); + const unsigned int f3 = xfs_calc_finish_cui_reservation(mp, 1); + const unsigned int step_size = max3(f1, f2, f3); + + /* Largest buffer size (in fsblocks) that can be invalidated. */ + const unsigned int max_binval = xrep_binval_max_fsblocks(mp); + + /* Overhead of invalidating one buffer */ + const unsigned int per_binval = + xfs_buf_inval_log_space(1, XFS_B_TO_FSBT(mp, max_binval)); + + /* + * For each transaction in a reap chain, we can delete some number of + * extents and invalidate some number of blocks. We assume that CoW + * staging extents are usually more than 1 fsblock, and that there + * shouldn't be any buffers for those blocks. From the assumptions, + * set the number of deferrals to use as much of the reservation as + * it can, but leave space to invalidate 1/8th that number of buffers. + */ + const unsigned int variable_overhead = per_intent + + (per_binval / 8); + + xreap_configure_limits(rs, step_size, variable_overhead, per_intent, + per_binval); + + trace_xreap_agcow_limits(sc->tp, per_binval, rs->max_binval, step_size, + per_intent, rs->max_deferred); +} + +/* + * Break an AG metadata extent into sub-extents by fate (crosslinked, not + * crosslinked), and dispose of each sub-extent separately. + */ +STATIC int +xreap_agmeta_extent( + uint32_t agbno, + uint32_t len, + void *priv) +{ + struct xreap_state *rs = priv; + struct xfs_scrub *sc = rs->sc; + xfs_agblock_t agbno_next = agbno + len; + int error = 0; + + ASSERT(len <= XFS_MAX_BMBT_EXTLEN); + ASSERT(sc->ip == NULL); + + while (agbno < agbno_next) { + xfs_extlen_t aglen; + bool crosslinked; + + error = xreap_agextent_select(rs, agbno, agbno_next, + &crosslinked, &aglen); + if (error) + return error; + + error = xreap_agextent_iter(rs, agbno, &aglen, crosslinked); + if (error) + return error; + + if (xreap_want_defer_finish(rs)) { + error = xrep_defer_finish(sc); + if (error) + return error; + xreap_defer_finish_reset(rs); + } else if (xreap_want_binval_roll(rs)) { + error = xrep_roll_ag_trans(sc); + if (error) + return error; + xreap_binval_reset(rs); + } + + agbno += aglen; + } + + return 0; +} + +/* Dispose of every block of every AG metadata extent in the bitmap. */ +int +xrep_reap_agblocks( + struct xfs_scrub *sc, + struct xagb_bitmap *bitmap, + const struct xfs_owner_info *oinfo, + enum xfs_ag_resv_type type) +{ + struct xreap_state rs = { + .sc = sc, + .oinfo = oinfo, + .resv = type, + }; + int error; + + ASSERT(xfs_has_rmapbt(sc->mp)); + ASSERT(sc->ip == NULL); + + xreap_configure_agextent_limits(&rs); + error = xagb_bitmap_walk(bitmap, xreap_agmeta_extent, &rs); + if (error) + return error; + + if (xreap_is_dirty(&rs)) + return xrep_defer_finish(sc); + + return 0; +} + +/* + * Break a file metadata extent into sub-extents by fate (crosslinked, not + * crosslinked), and dispose of each sub-extent separately. The extent must + * not cross an AG boundary. + */ +STATIC int +xreap_fsmeta_extent( + uint64_t fsbno, + uint64_t len, + void *priv) +{ + struct xreap_state *rs = priv; + struct xfs_scrub *sc = rs->sc; + xfs_agnumber_t agno = XFS_FSB_TO_AGNO(sc->mp, fsbno); + xfs_agblock_t agbno = XFS_FSB_TO_AGBNO(sc->mp, fsbno); + xfs_agblock_t agbno_next = agbno + len; + int error = 0; + + ASSERT(len <= XFS_MAX_BMBT_EXTLEN); + ASSERT(sc->ip != NULL); + ASSERT(!sc->sa.pag); + + /* + * We're reaping blocks after repairing file metadata, which means that + * we have to init the xchk_ag structure ourselves. + */ + sc->sa.pag = xfs_perag_get(sc->mp, agno); + if (!sc->sa.pag) + return -EFSCORRUPTED; + + error = xfs_alloc_read_agf(sc->sa.pag, sc->tp, 0, &sc->sa.agf_bp); + if (error) + goto out_pag; + + while (agbno < agbno_next) { + xfs_extlen_t aglen; + bool crosslinked; + + error = xreap_agextent_select(rs, agbno, agbno_next, + &crosslinked, &aglen); + if (error) + goto out_agf; + + error = xreap_agextent_iter(rs, agbno, &aglen, crosslinked); + if (error) + goto out_agf; + + if (xreap_want_defer_finish(rs)) { + /* + * Holds the AGF buffer across the deferred chain + * processing. + */ + error = xrep_defer_finish(sc); + if (error) + goto out_agf; + xreap_defer_finish_reset(rs); + } else if (xreap_want_binval_roll(rs)) { + /* + * Hold the AGF buffer across the transaction roll so + * that we don't have to reattach it to the scrub + * context. + */ + xfs_trans_bhold(sc->tp, sc->sa.agf_bp); + error = xfs_trans_roll_inode(&sc->tp, sc->ip); + xfs_trans_bjoin(sc->tp, sc->sa.agf_bp); + if (error) + goto out_agf; + xreap_binval_reset(rs); + } + + agbno += aglen; + } + +out_agf: + xfs_trans_brelse(sc->tp, sc->sa.agf_bp); + sc->sa.agf_bp = NULL; +out_pag: + xfs_perag_put(sc->sa.pag); + sc->sa.pag = NULL; + return error; +} + +/* + * Dispose of every block of every fs metadata extent in the bitmap. + * Do not use this to dispose of the mappings in an ondisk inode fork. + */ +int +xrep_reap_fsblocks( + struct xfs_scrub *sc, + struct xfsb_bitmap *bitmap, + const struct xfs_owner_info *oinfo) +{ + struct xreap_state rs = { + .sc = sc, + .oinfo = oinfo, + .resv = XFS_AG_RESV_NONE, + }; + int error; + + ASSERT(xfs_has_rmapbt(sc->mp)); + ASSERT(sc->ip != NULL); + + if (oinfo == &XFS_RMAP_OINFO_COW) + xreap_configure_agcow_limits(&rs); + else + xreap_configure_agextent_limits(&rs); + error = xfsb_bitmap_walk(bitmap, xreap_fsmeta_extent, &rs); + if (error) + return error; + + if (xreap_is_dirty(&rs)) + return xrep_defer_finish(sc); + + return 0; +} + +#ifdef CONFIG_XFS_RT +/* + * Figure out the longest run of blocks that we can dispose of with a single + * call. Cross-linked blocks should have their reverse mappings removed, but + * single-owner extents can be freed. Units are rt blocks, not rt extents. + */ +STATIC int +xreap_rgextent_select( + struct xreap_state *rs, + xfs_rgblock_t rgbno, + xfs_rgblock_t rgbno_next, + bool *crosslinked, + xfs_extlen_t *rglenp) +{ + struct xfs_scrub *sc = rs->sc; + struct xfs_btree_cur *cur; + xfs_rgblock_t bno = rgbno + 1; + xfs_extlen_t len = 1; + int error; + + /* + * Determine if there are any other rmap records covering the first + * block of this extent. If so, the block is crosslinked. + */ + cur = xfs_rtrmapbt_init_cursor(sc->tp, sc->sr.rtg); + error = xfs_rmap_has_other_keys(cur, rgbno, 1, rs->oinfo, + crosslinked); + if (error) + goto out_cur; + + /* + * Figure out how many of the subsequent blocks have the same crosslink + * status. + */ + while (bno < rgbno_next) { + bool also_crosslinked; + + error = xfs_rmap_has_other_keys(cur, bno, 1, rs->oinfo, + &also_crosslinked); + if (error) + goto out_cur; + + if (*crosslinked != also_crosslinked) + break; + + len++; + bno++; + } + + *rglenp = len; + trace_xreap_agextent_select(rtg_group(sc->sr.rtg), rgbno, len, + *crosslinked); +out_cur: + xfs_btree_del_cursor(cur, error); + return error; +} + +/* + * Dispose of as much of the beginning of this rtgroup extent as possible. + * The number of blocks disposed of will be returned in @rglenp. + */ +STATIC int +xreap_rgextent_iter( + struct xreap_state *rs, + xfs_rgblock_t rgbno, + xfs_extlen_t *rglenp, + bool crosslinked) +{ + struct xfs_scrub *sc = rs->sc; + xfs_rtblock_t rtbno; + int error; + + /* + * The only caller so far is CoW fork repair, so we only know how to + * unlink or free CoW staging extents. Here we don't have to worry + * about invalidating buffers! + */ + if (rs->oinfo != &XFS_RMAP_OINFO_COW) { + ASSERT(rs->oinfo == &XFS_RMAP_OINFO_COW); + return -EFSCORRUPTED; + } + ASSERT(rs->resv == XFS_AG_RESV_NONE); + + rtbno = xfs_rgbno_to_rtb(sc->sr.rtg, rgbno); + + /* + * t1: There are other rmappings; this block is cross linked and must + * not be freed. Remove the forward and reverse mapping and move on. + */ + if (crosslinked) { + trace_xreap_dispose_unmap_extent(rtg_group(sc->sr.rtg), rgbno, + *rglenp); + + xfs_refcount_free_cow_extent(sc->tp, true, rtbno, *rglenp); + xreap_inc_defer(rs); + return 0; + } + + trace_xreap_dispose_free_extent(rtg_group(sc->sr.rtg), rgbno, *rglenp); + + /* + * t2: The CoW staging extent is not crosslinked. Use deferred work + * to remove the refcountbt records (which removes the rmap records) + * and free the extent. We're not worried about the system going down + * here because log recovery walks the refcount btree to clean out the + * CoW staging extents. + */ + xfs_refcount_free_cow_extent(sc->tp, true, rtbno, *rglenp); + error = xfs_free_extent_later(sc->tp, rtbno, *rglenp, NULL, + rs->resv, + XFS_FREE_EXTENT_REALTIME | + XFS_FREE_EXTENT_SKIP_DISCARD); + if (error) + return error; + + xreap_inc_defer(rs); + return 0; +} + +/* + * Compute the maximum number of intent items that reaping can attach to the + * scrub transaction given the worst case log overhead of the intent items + * needed to reap a single CoW staging extent. This is not for freeing + * metadata blocks. + */ +STATIC void +xreap_configure_rgcow_limits( + struct xreap_state *rs) +{ + struct xfs_scrub *sc = rs->sc; + struct xfs_mount *mp = sc->mp; + + /* + * In the worst case, relogging an intent item causes both an intent + * item and a done item to be attached to a transaction for each extent + * that we'd like to process. + */ + const unsigned int efi = xfs_efi_log_space(1) + + xfs_efd_log_space(1); + const unsigned int rui = xfs_rui_log_space(1) + + xfs_rud_log_space(); + const unsigned int cui = xfs_cui_log_space(1) + + xfs_cud_log_space(); + + /* + * Various things can happen when reaping non-CoW metadata blocks: + * + * t1: Unmapping crosslinked CoW blocks: deferred removal of refcount + * record, which defers removal of rmap record + * + * t2: Freeing CoW blocks: deferred removal of refcount record, which + * defers removal of rmap record; and deferred removal of the space + * + * For simplicity, we'll use the worst-case intents size to determine + * the maximum number of deferred extents before we have to finish the + * whole chain. If we're trying to reap a btree larger than this size, + * a crash midway through reaping can result in leaked blocks. + */ + const unsigned int t1 = cui + rui; + const unsigned int t2 = cui + rui + efi; + const unsigned int per_intent = max(t1, t2); + + /* + * For each transaction in a reap chain, we must be able to take one + * step in the defer item chain, which should only consist of CUI, EFI, + * or RUI items. + */ + const unsigned int f1 = xfs_calc_finish_rt_efi_reservation(mp, 1); + const unsigned int f2 = xfs_calc_finish_rt_rui_reservation(mp, 1); + const unsigned int f3 = xfs_calc_finish_rt_cui_reservation(mp, 1); + const unsigned int step_size = max3(f1, f2, f3); + + /* + * The only buffer for the rt device is the rtgroup super, so we don't + * need to save space for buffer invalidations. + */ + xreap_configure_limits(rs, step_size, per_intent, per_intent, 0); + + trace_xreap_rgcow_limits(sc->tp, 0, 0, step_size, per_intent, + rs->max_deferred); +} + +#define XREAP_RTGLOCK_ALL (XFS_RTGLOCK_BITMAP | \ + XFS_RTGLOCK_RMAP | \ + XFS_RTGLOCK_REFCOUNT) + +/* + * Break a rt file metadata extent into sub-extents by fate (crosslinked, not + * crosslinked), and dispose of each sub-extent separately. The extent must + * be aligned to a realtime extent. + */ +STATIC int +xreap_rtmeta_extent( + uint64_t rtbno, + uint64_t len, + void *priv) +{ + struct xreap_state *rs = priv; + struct xfs_scrub *sc = rs->sc; + xfs_rgblock_t rgbno = xfs_rtb_to_rgbno(sc->mp, rtbno); + xfs_rgblock_t rgbno_next = rgbno + len; + int error = 0; + + ASSERT(sc->ip != NULL); + ASSERT(!sc->sr.rtg); + + /* + * We're reaping blocks after repairing file metadata, which means that + * we have to init the xchk_ag structure ourselves. + */ + sc->sr.rtg = xfs_rtgroup_get(sc->mp, xfs_rtb_to_rgno(sc->mp, rtbno)); + if (!sc->sr.rtg) + return -EFSCORRUPTED; + + xfs_rtgroup_lock(sc->sr.rtg, XREAP_RTGLOCK_ALL); + + while (rgbno < rgbno_next) { + xfs_extlen_t rglen; + bool crosslinked; + + error = xreap_rgextent_select(rs, rgbno, rgbno_next, + &crosslinked, &rglen); + if (error) + goto out_unlock; + + error = xreap_rgextent_iter(rs, rgbno, &rglen, crosslinked); + if (error) + goto out_unlock; + + if (xreap_want_defer_finish(rs)) { + error = xfs_defer_finish(&sc->tp); + if (error) + goto out_unlock; + xreap_defer_finish_reset(rs); + } else if (xreap_want_binval_roll(rs)) { + error = xfs_trans_roll_inode(&sc->tp, sc->ip); + if (error) + goto out_unlock; + xreap_binval_reset(rs); + } + + rgbno += rglen; + } + +out_unlock: + xfs_rtgroup_unlock(sc->sr.rtg, XREAP_RTGLOCK_ALL); + xfs_rtgroup_put(sc->sr.rtg); + sc->sr.rtg = NULL; + return error; +} + +/* + * Dispose of every block of every rt metadata extent in the bitmap. + * Do not use this to dispose of the mappings in an ondisk inode fork. + */ +int +xrep_reap_rtblocks( + struct xfs_scrub *sc, + struct xrtb_bitmap *bitmap, + const struct xfs_owner_info *oinfo) +{ + struct xreap_state rs = { + .sc = sc, + .oinfo = oinfo, + .resv = XFS_AG_RESV_NONE, + }; + int error; + + ASSERT(xfs_has_rmapbt(sc->mp)); + ASSERT(sc->ip != NULL); + ASSERT(oinfo == &XFS_RMAP_OINFO_COW); + + xreap_configure_rgcow_limits(&rs); + error = xrtb_bitmap_walk(bitmap, xreap_rtmeta_extent, &rs); + if (error) + return error; + + if (xreap_is_dirty(&rs)) + return xrep_defer_finish(sc); + + return 0; +} +#endif /* CONFIG_XFS_RT */ + +/* + * Dispose of every block of an old metadata btree that used to be rooted in a + * metadata directory file. + */ +int +xrep_reap_metadir_fsblocks( + struct xfs_scrub *sc, + struct xfsb_bitmap *bitmap) +{ + /* + * Reap old metadir btree blocks with XFS_AG_RESV_NONE because the old + * blocks are no longer mapped by the inode, and inode metadata space + * reservations can only account freed space to the i_nblocks. + */ + struct xfs_owner_info oinfo; + struct xreap_state rs = { + .sc = sc, + .oinfo = &oinfo, + .resv = XFS_AG_RESV_NONE, + }; + int error; + + ASSERT(xfs_has_rmapbt(sc->mp)); + ASSERT(sc->ip != NULL); + ASSERT(xfs_is_metadir_inode(sc->ip)); + + xreap_configure_agextent_limits(&rs); + xfs_rmap_inode_bmbt_owner(&oinfo, sc->ip, XFS_DATA_FORK); + error = xfsb_bitmap_walk(bitmap, xreap_fsmeta_extent, &rs); + if (error) + return error; + + if (xreap_is_dirty(&rs)) { + error = xrep_defer_finish(sc); + if (error) + return error; + } + + return xrep_reset_metafile_resv(sc); +} + +/* + * Metadata files are not supposed to share blocks with anything else. + * If blocks are shared, we remove the reverse mapping (thus reducing the + * crosslink factor); if blocks are not shared, we also need to free them. + * + * This first step determines the longest subset of the passed-in imap + * (starting at its beginning) that is either crosslinked or not crosslinked. + * The blockcount will be adjust down as needed. + */ +STATIC int +xreap_bmapi_select( + struct xreap_state *rs, + struct xfs_bmbt_irec *imap, + bool *crosslinked) +{ + struct xfs_owner_info oinfo; + struct xfs_scrub *sc = rs->sc; + struct xfs_btree_cur *cur; + xfs_filblks_t len = 1; + xfs_agblock_t bno; + xfs_agblock_t agbno; + xfs_agblock_t agbno_next; + int error; + + agbno = XFS_FSB_TO_AGBNO(sc->mp, imap->br_startblock); + agbno_next = agbno + imap->br_blockcount; + + cur = xfs_rmapbt_init_cursor(sc->mp, sc->tp, sc->sa.agf_bp, + sc->sa.pag); + + xfs_rmap_inode_owner(&oinfo, rs->ip, rs->whichfork, imap->br_startoff); + error = xfs_rmap_has_other_keys(cur, agbno, 1, &oinfo, crosslinked); + if (error) + goto out_cur; + + bno = agbno + 1; + while (bno < agbno_next) { + bool also_crosslinked; + + oinfo.oi_offset++; + error = xfs_rmap_has_other_keys(cur, bno, 1, &oinfo, + &also_crosslinked); + if (error) + goto out_cur; + + if (also_crosslinked != *crosslinked) + break; + + len++; + bno++; + } + + imap->br_blockcount = len; + trace_xreap_bmapi_select(pag_group(sc->sa.pag), agbno, len, + *crosslinked); +out_cur: + xfs_btree_del_cursor(cur, error); + return error; +} + +/* + * Decide if this buffer can be joined to a transaction. This is true for most + * buffers, but there are two cases that we want to catch: large remote xattr + * value buffers are not logged and can overflow the buffer log item dirty + * bitmap size; and oversized cached buffers if things have really gone + * haywire. + */ +static inline bool +xreap_buf_loggable( + const struct xfs_buf *bp) +{ + int i; + + for (i = 0; i < bp->b_map_count; i++) { + int chunks; + int map_size; + + chunks = DIV_ROUND_UP(BBTOB(bp->b_maps[i].bm_len), + XFS_BLF_CHUNK); + map_size = DIV_ROUND_UP(chunks, NBWORD); + if (map_size > XFS_BLF_DATAMAP_SIZE) + return false; + } + + return true; +} + +/* + * Invalidate any buffers for this file mapping. The @imap blockcount may be + * adjusted downward if we need to roll the transaction. + */ +STATIC int +xreap_bmapi_binval( + struct xreap_state *rs, + struct xfs_bmbt_irec *imap) +{ + struct xfs_scrub *sc = rs->sc; + struct xfs_mount *mp = sc->mp; + struct xfs_perag *pag = sc->sa.pag; + int bmap_flags = xfs_bmapi_aflag(rs->whichfork); + xfs_fileoff_t off; + xfs_fileoff_t max_off; + xfs_extlen_t scan_blocks; + xfs_agblock_t bno; + xfs_agblock_t agbno; + xfs_agblock_t agbno_next; + int error; + + /* + * Avoid invalidating AG headers and post-EOFS blocks because we never + * own those. + */ + agbno = bno = XFS_FSB_TO_AGBNO(sc->mp, imap->br_startblock); + agbno_next = agbno + imap->br_blockcount; + if (!xfs_verify_agbno(pag, agbno) || + !xfs_verify_agbno(pag, agbno_next - 1)) + return 0; + + /* + * Buffers for file blocks can span multiple contiguous mappings. This + * means that for each block in the mapping, there could exist an + * xfs_buf indexed by that block with any length up to the maximum + * buffer size (remote xattr values) or to the next hole in the fork. + * To set up our binval scan, first we need to figure out the location + * of the next hole. + */ + off = imap->br_startoff + imap->br_blockcount; + max_off = off + xfs_attr3_max_rmt_blocks(mp); + while (off < max_off) { + struct xfs_bmbt_irec hmap; + int nhmaps = 1; + + error = xfs_bmapi_read(rs->ip, off, max_off - off, &hmap, + &nhmaps, bmap_flags); + if (error) + return error; + if (nhmaps != 1 || hmap.br_startblock == DELAYSTARTBLOCK) { + ASSERT(0); + return -EFSCORRUPTED; + } + + if (!xfs_bmap_is_real_extent(&hmap)) + break; + + off = hmap.br_startoff + hmap.br_blockcount; + } + scan_blocks = off - imap->br_startoff; + + trace_xreap_bmapi_binval_scan(sc, imap, scan_blocks); + + /* + * If there are incore buffers for these blocks, invalidate them. If + * we can't (try)lock the buffer we assume it's owned by someone else + * and leave it alone. The buffer cache cannot detect aliasing, so + * employ nested loops to detect incore buffers of any plausible size. + */ + while (bno < agbno_next) { + struct xrep_bufscan scan = { + .daddr = xfs_agbno_to_daddr(pag, bno), + .max_sectors = xrep_bufscan_max_sectors(mp, + scan_blocks), + .daddr_step = XFS_FSB_TO_BB(mp, 1), + }; + struct xfs_buf *bp; + + while ((bp = xrep_bufscan_advance(mp, &scan)) != NULL) { + if (xreap_buf_loggable(bp)) { + xfs_trans_bjoin(sc->tp, bp); + xfs_trans_binval(sc->tp, bp); + } else { + xfs_buf_stale(bp); + xfs_buf_relse(bp); + } + + /* + * Stop invalidating if we've hit the limit; we should + * still have enough reservation left to free however + * far we've gotten. + */ + if (!xreap_inc_binval(rs)) { + imap->br_blockcount = agbno_next - bno; + goto out; + } + } + + bno++; + scan_blocks--; + } + +out: + trace_xreap_bmapi_binval(pag_group(sc->sa.pag), agbno, + imap->br_blockcount); + return 0; +} + +/* + * Dispose of as much of the beginning of this file fork mapping as possible. + * The number of blocks disposed of is returned in @imap->br_blockcount. + */ +STATIC int +xrep_reap_bmapi_iter( + struct xreap_state *rs, + struct xfs_bmbt_irec *imap, + bool crosslinked) +{ + struct xfs_scrub *sc = rs->sc; + int error; + + if (crosslinked) { + /* + * If there are other rmappings, this block is cross linked and + * must not be freed. Remove the reverse mapping, leave the + * buffer cache in its possibly confused state, and move on. + * We don't want to risk discarding valid data buffers from + * anybody else who thinks they own the block, even though that + * runs the risk of stale buffer warnings in the future. + */ + trace_xreap_dispose_unmap_extent(pag_group(sc->sa.pag), + XFS_FSB_TO_AGBNO(sc->mp, imap->br_startblock), + imap->br_blockcount); + + /* + * t0: Schedule removal of the mapping from the fork. We use + * deferred log intents in this function to control the exact + * sequence of metadata updates. + */ + xfs_bmap_unmap_extent(sc->tp, rs->ip, rs->whichfork, imap); + xfs_trans_mod_dquot_byino(sc->tp, rs->ip, XFS_TRANS_DQ_BCOUNT, + -(int64_t)imap->br_blockcount); + xfs_rmap_unmap_extent(sc->tp, rs->ip, rs->whichfork, imap); + return 0; + } + + /* + * If the block is not crosslinked, we can invalidate all the incore + * buffers for the extent, and then free the extent. This is a bit of + * a mess since we don't detect discontiguous buffers that are indexed + * by a block starting before the first block of the extent but overlap + * anyway. + */ + trace_xreap_dispose_free_extent(pag_group(sc->sa.pag), + XFS_FSB_TO_AGBNO(sc->mp, imap->br_startblock), + imap->br_blockcount); + + /* + * Invalidate as many buffers as we can, starting at the beginning of + * this mapping. If this function sets blockcount to zero, the + * transaction is full of logged buffer invalidations, so we need to + * return early so that we can roll and retry. + */ + error = xreap_bmapi_binval(rs, imap); + if (error || imap->br_blockcount == 0) + return error; + + /* + * t1: Schedule removal of the mapping from the fork. We use deferred + * work in this function to control the exact sequence of metadata + * updates. + */ + xfs_bmap_unmap_extent(sc->tp, rs->ip, rs->whichfork, imap); + xfs_trans_mod_dquot_byino(sc->tp, rs->ip, XFS_TRANS_DQ_BCOUNT, + -(int64_t)imap->br_blockcount); + return xfs_free_extent_later(sc->tp, imap->br_startblock, + imap->br_blockcount, NULL, XFS_AG_RESV_NONE, + XFS_FREE_EXTENT_SKIP_DISCARD); +} + +/* Compute the maximum mapcount of a file buffer. */ +static unsigned int +xreap_bmapi_binval_mapcount( + struct xfs_scrub *sc) +{ + /* directory blocks can span multiple fsblocks and be discontiguous */ + if (sc->sm->sm_type == XFS_SCRUB_TYPE_DIR) + return sc->mp->m_dir_geo->fsbcount; + + /* all other file xattr/symlink blocks must be contiguous */ + return 1; +} + +/* Compute the maximum block size of a file buffer. */ +static unsigned int +xreap_bmapi_binval_blocksize( + struct xfs_scrub *sc) +{ + switch (sc->sm->sm_type) { + case XFS_SCRUB_TYPE_DIR: + return sc->mp->m_dir_geo->blksize; + case XFS_SCRUB_TYPE_XATTR: + case XFS_SCRUB_TYPE_PARENT: + /* + * The xattr structure itself consists of single fsblocks, but + * there could be remote xattr blocks to invalidate. + */ + return XFS_XATTR_SIZE_MAX; + } + + /* everything else is a single block */ + return sc->mp->m_sb.sb_blocksize; +} + +/* + * Compute the maximum number of buffer invalidations that we can do while + * reaping a single extent from a file fork. + */ +STATIC void +xreap_configure_bmapi_limits( + struct xreap_state *rs) +{ + struct xfs_scrub *sc = rs->sc; + struct xfs_mount *mp = sc->mp; + + /* overhead of invalidating a buffer */ + const unsigned int per_binval = + xfs_buf_inval_log_space(xreap_bmapi_binval_mapcount(sc), + xreap_bmapi_binval_blocksize(sc)); + + /* + * In the worst case, relogging an intent item causes both an intent + * item and a done item to be attached to a transaction for each extent + * that we'd like to process. + */ + const unsigned int efi = xfs_efi_log_space(1) + + xfs_efd_log_space(1); + const unsigned int rui = xfs_rui_log_space(1) + + xfs_rud_log_space(); + const unsigned int bui = xfs_bui_log_space(1) + + xfs_bud_log_space(); + + /* + * t1: Unmapping crosslinked file data blocks: one bmap deletion, + * possibly an EFI for underfilled bmbt blocks, and an rmap deletion. + * + * t2: Freeing freeing file data blocks: one bmap deletion, possibly an + * EFI for underfilled bmbt blocks, and another EFI for the space + * itself. + */ + const unsigned int t1 = (bui + efi) + rui; + const unsigned int t2 = (bui + efi) + efi; + const unsigned int per_intent = max(t1, t2); + + /* + * For each transaction in a reap chain, we must be able to take one + * step in the defer item chain, which should only consist of CUI, EFI, + * or RUI items. + */ + const unsigned int f1 = xfs_calc_finish_efi_reservation(mp, 1); + const unsigned int f2 = xfs_calc_finish_rui_reservation(mp, 1); + const unsigned int f3 = xfs_calc_finish_bui_reservation(mp, 1); + const unsigned int step_size = max3(f1, f2, f3); + + /* + * Each call to xreap_ifork_extent starts with a clean transaction and + * operates on a single mapping by creating a chain of log intent items + * for that mapping. We need to leave enough reservation in the + * transaction to log btree buffer and inode updates for each step in + * the chain, and to relog the log intents. + */ + const unsigned int per_extent_res = per_intent + step_size; + + xreap_configure_limits(rs, per_extent_res, per_binval, 0, per_binval); + + trace_xreap_bmapi_limits(sc->tp, per_binval, rs->max_binval, + step_size, per_intent, 1); +} + +/* + * Dispose of as much of this file extent as we can. Upon successful return, + * the imap will reflect the mapping that was removed from the fork. + */ +STATIC int +xreap_ifork_extent( + struct xreap_state *rs, + struct xfs_bmbt_irec *imap) +{ + struct xfs_scrub *sc = rs->sc; + xfs_agnumber_t agno; + bool crosslinked; + int error; + + ASSERT(sc->sa.pag == NULL); + + trace_xreap_ifork_extent(sc, rs->ip, rs->whichfork, imap); + + agno = XFS_FSB_TO_AGNO(sc->mp, imap->br_startblock); + sc->sa.pag = xfs_perag_get(sc->mp, agno); + if (!sc->sa.pag) + return -EFSCORRUPTED; + + error = xfs_alloc_read_agf(sc->sa.pag, sc->tp, 0, &sc->sa.agf_bp); + if (error) + goto out_pag; + + /* + * Decide the fate of the blocks at the beginning of the mapping, then + * update the mapping to use it with the unmap calls. + */ + error = xreap_bmapi_select(rs, imap, &crosslinked); + if (error) + goto out_agf; + + error = xrep_reap_bmapi_iter(rs, imap, crosslinked); + if (error) + goto out_agf; + +out_agf: + xfs_trans_brelse(sc->tp, sc->sa.agf_bp); + sc->sa.agf_bp = NULL; +out_pag: + xfs_perag_put(sc->sa.pag); + sc->sa.pag = NULL; + return error; +} + +/* + * Dispose of each block mapped to the given fork of the given file. Callers + * must hold ILOCK_EXCL, and ip can only be sc->ip or sc->tempip. The fork + * must not have any delalloc reservations. + */ +int +xrep_reap_ifork( + struct xfs_scrub *sc, + struct xfs_inode *ip, + int whichfork) +{ + struct xreap_state rs = { + .sc = sc, + .ip = ip, + .whichfork = whichfork, + }; + xfs_fileoff_t off = 0; + int bmap_flags = xfs_bmapi_aflag(whichfork); + int error; + + ASSERT(xfs_has_rmapbt(sc->mp)); + ASSERT(ip == sc->ip || ip == sc->tempip); + ASSERT(whichfork == XFS_ATTR_FORK || !XFS_IS_REALTIME_INODE(ip)); + + xreap_configure_bmapi_limits(&rs); + while (off < XFS_MAX_FILEOFF) { + struct xfs_bmbt_irec imap; + int nimaps = 1; + + /* Read the next extent, skip past holes and delalloc. */ + error = xfs_bmapi_read(ip, off, XFS_MAX_FILEOFF - off, &imap, + &nimaps, bmap_flags); + if (error) + return error; + if (nimaps != 1 || imap.br_startblock == DELAYSTARTBLOCK) { + ASSERT(0); + return -EFSCORRUPTED; + } + + /* + * If this is a real space mapping, reap as much of it as we + * can in a single transaction. + */ + if (xfs_bmap_is_real_extent(&imap)) { + error = xreap_ifork_extent(&rs, &imap); + if (error) + return error; + + error = xfs_defer_finish(&sc->tp); + if (error) + return error; + xreap_defer_finish_reset(&rs); + } + + off = imap.br_startoff + imap.br_blockcount; + } + + return 0; +} diff --git a/fs/xfs/scrub/reap.h b/fs/xfs/scrub/reap.h new file mode 100644 index 000000000..4c8f62701 --- /dev/null +++ b/fs/xfs/scrub/reap.h @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2022-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_REAP_H__ +#define __XFS_SCRUB_REAP_H__ + +struct xagb_bitmap; +struct xfsb_bitmap; + +int xrep_reap_agblocks(struct xfs_scrub *sc, struct xagb_bitmap *bitmap, + const struct xfs_owner_info *oinfo, enum xfs_ag_resv_type type); +int xrep_reap_fsblocks(struct xfs_scrub *sc, struct xfsb_bitmap *bitmap, + const struct xfs_owner_info *oinfo); +int xrep_reap_ifork(struct xfs_scrub *sc, struct xfs_inode *ip, int whichfork); +int xrep_reap_metadir_fsblocks(struct xfs_scrub *sc, + struct xfsb_bitmap *bitmap); + +#ifdef CONFIG_XFS_RT +int xrep_reap_rtblocks(struct xfs_scrub *sc, struct xrtb_bitmap *bitmap, + const struct xfs_owner_info *oinfo); +#else +# define xrep_reap_rtblocks(...) (-EOPNOTSUPP) +#endif /* CONFIG_XFS_RT */ + +/* Buffer cache scan context. */ +struct xrep_bufscan { + /* Disk address for the buffers we want to scan. */ + xfs_daddr_t daddr; + + /* Maximum number of sectors to scan. */ + xfs_daddr_t max_sectors; + + /* Each round, increment the search length by this number of sectors. */ + xfs_daddr_t daddr_step; + + /* Internal scan state; initialize to zero. */ + xfs_daddr_t __sector_count; +}; + +xfs_daddr_t xrep_bufscan_max_sectors(struct xfs_mount *mp, + xfs_extlen_t fsblocks); +struct xfs_buf *xrep_bufscan_advance(struct xfs_mount *mp, + struct xrep_bufscan *scan); + +#endif /* __XFS_SCRUB_REAP_H__ */ diff --git a/fs/xfs/scrub/refcount.c b/fs/xfs/scrub/refcount.c new file mode 100644 index 000000000..4e1bf23e5 --- /dev/null +++ b/fs/xfs/scrub/refcount.c @@ -0,0 +1,632 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_trans.h" +#include "xfs_ag.h" +#include "xfs_btree.h" +#include "xfs_rmap.h" +#include "xfs_refcount.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/btree.h" +#include "scrub/trace.h" +#include "scrub/repair.h" + +/* + * Set us up to scrub reference count btrees. + */ +int +xchk_setup_ag_refcountbt( + struct xfs_scrub *sc) +{ + if (xchk_need_intent_drain(sc)) + xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN); + + if (xchk_could_repair(sc)) { + int error; + + error = xrep_setup_ag_refcountbt(sc); + if (error) + return error; + } + + return xchk_setup_ag_btree(sc, false); +} + +/* Reference count btree scrubber. */ + +/* + * Confirming Reference Counts via Reverse Mappings + * + * We want to count the reverse mappings overlapping a refcount record + * (bno, len, refcount), allowing for the possibility that some of the + * overlap may come from smaller adjoining reverse mappings, while some + * comes from single extents which overlap the range entirely. The + * outer loop is as follows: + * + * 1. For all reverse mappings overlapping the refcount extent, + * a. If a given rmap completely overlaps, mark it as seen. + * b. Otherwise, record the fragment (in agbno order) for later + * processing. + * + * Once we've seen all the rmaps, we know that for all blocks in the + * refcount record we want to find $refcount owners and we've already + * visited $seen extents that overlap all the blocks. Therefore, we + * need to find ($refcount - $seen) owners for every block in the + * extent; call that quantity $target_nr. Proceed as follows: + * + * 2. Pull the first $target_nr fragments from the list; all of them + * should start at or before the start of the extent. + * Call this subset of fragments the working set. + * 3. Until there are no more unprocessed fragments, + * a. Find the shortest fragments in the set and remove them. + * b. Note the block number of the end of these fragments. + * c. Pull the same number of fragments from the list. All of these + * fragments should start at the block number recorded in the + * previous step. + * d. Put those fragments in the set. + * 4. Check that there are $target_nr fragments remaining in the list, + * and that they all end at or beyond the end of the refcount extent. + * + * If the refcount is correct, all the check conditions in the algorithm + * should always hold true. If not, the refcount is incorrect. + */ +struct xchk_refcnt_frag { + struct list_head list; + struct xfs_rmap_irec rm; +}; + +struct xchk_refcnt_check { + struct xfs_scrub *sc; + struct list_head fragments; + + /* refcount extent we're examining */ + xfs_agblock_t bno; + xfs_extlen_t len; + xfs_nlink_t refcount; + + /* number of owners seen */ + xfs_nlink_t seen; +}; + +/* + * Decide if the given rmap is large enough that we can redeem it + * towards refcount verification now, or if it's a fragment, in + * which case we'll hang onto it in the hopes that we'll later + * discover that we've collected exactly the correct number of + * fragments as the refcountbt says we should have. + */ +STATIC int +xchk_refcountbt_rmap_check( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv) +{ + struct xchk_refcnt_check *refchk = priv; + struct xchk_refcnt_frag *frag; + xfs_agblock_t rm_last; + xfs_agblock_t rc_last; + int error = 0; + + if (xchk_should_terminate(refchk->sc, &error)) + return error; + + rm_last = rec->rm_startblock + rec->rm_blockcount - 1; + rc_last = refchk->bno + refchk->len - 1; + + /* Confirm that a single-owner refc extent is a CoW stage. */ + if (refchk->refcount == 1 && rec->rm_owner != XFS_RMAP_OWN_COW) { + xchk_btree_xref_set_corrupt(refchk->sc, cur, 0); + return 0; + } + + if (rec->rm_startblock <= refchk->bno && rm_last >= rc_last) { + /* + * The rmap overlaps the refcount record, so we can confirm + * one refcount owner seen. + */ + refchk->seen++; + } else { + /* + * This rmap covers only part of the refcount record, so + * save the fragment for later processing. If the rmapbt + * is healthy each rmap_irec we see will be in agbno order + * so we don't need insertion sort here. + */ + frag = kmalloc_obj(struct xchk_refcnt_frag, XCHK_GFP_FLAGS); + if (!frag) + return -ENOMEM; + memcpy(&frag->rm, rec, sizeof(frag->rm)); + list_add_tail(&frag->list, &refchk->fragments); + } + + return 0; +} + +/* + * Given a bunch of rmap fragments, iterate through them, keeping + * a running tally of the refcount. If this ever deviates from + * what we expect (which is the refcountbt's refcount minus the + * number of extents that totally covered the refcountbt extent), + * we have a refcountbt error. + */ +STATIC void +xchk_refcountbt_process_rmap_fragments( + struct xchk_refcnt_check *refchk) +{ + struct list_head worklist; + struct xchk_refcnt_frag *frag; + struct xchk_refcnt_frag *n; + xfs_agblock_t bno; + xfs_agblock_t rbno; + xfs_agblock_t next_rbno; + xfs_nlink_t nr; + xfs_nlink_t target_nr; + + target_nr = refchk->refcount - refchk->seen; + if (target_nr == 0) + return; + + /* + * There are (refchk->rc.rc_refcount - refchk->nr refcount) + * references we haven't found yet. Pull that many off the + * fragment list and figure out where the smallest rmap ends + * (and therefore the next rmap should start). All the rmaps + * we pull off should start at or before the beginning of the + * refcount record's range. + */ + INIT_LIST_HEAD(&worklist); + rbno = NULLAGBLOCK; + + /* Make sure the fragments actually /are/ in agbno order. */ + bno = 0; + list_for_each_entry(frag, &refchk->fragments, list) { + if (frag->rm.rm_startblock < bno) + goto done; + bno = frag->rm.rm_startblock; + } + + /* + * Find all the rmaps that start at or before the refc extent, + * and put them on the worklist. + */ + nr = 0; + list_for_each_entry_safe(frag, n, &refchk->fragments, list) { + if (frag->rm.rm_startblock > refchk->bno || nr > target_nr) + break; + bno = frag->rm.rm_startblock + frag->rm.rm_blockcount; + if (bno < rbno) + rbno = bno; + list_move_tail(&frag->list, &worklist); + nr++; + } + + /* + * We should have found exactly $target_nr rmap fragments starting + * at or before the refcount extent. + */ + if (nr != target_nr) + goto done; + + while (!list_empty(&refchk->fragments)) { + /* Discard any fragments ending at rbno from the worklist. */ + nr = 0; + next_rbno = NULLAGBLOCK; + list_for_each_entry_safe(frag, n, &worklist, list) { + bno = frag->rm.rm_startblock + frag->rm.rm_blockcount; + if (bno != rbno) { + if (bno < next_rbno) + next_rbno = bno; + continue; + } + list_del(&frag->list); + kfree(frag); + nr++; + } + + /* Try to add nr rmaps starting at rbno to the worklist. */ + list_for_each_entry_safe(frag, n, &refchk->fragments, list) { + bno = frag->rm.rm_startblock + frag->rm.rm_blockcount; + if (frag->rm.rm_startblock != rbno) + goto done; + list_move_tail(&frag->list, &worklist); + if (next_rbno > bno) + next_rbno = bno; + nr--; + if (nr == 0) + break; + } + + /* + * If we get here and nr > 0, this means that we added fewer + * items to the worklist than we discarded because the fragment + * list ran out of items. Therefore, we cannot maintain the + * required refcount. Something is wrong, so we're done. + */ + if (nr) + goto done; + + rbno = next_rbno; + } + + /* + * Make sure the last extent we processed ends at or beyond + * the end of the refcount extent. + */ + if (rbno < refchk->bno + refchk->len) + goto done; + + /* Actually record us having seen the remaining refcount. */ + refchk->seen = refchk->refcount; +done: + /* Delete fragments and work list. */ + list_for_each_entry_safe(frag, n, &worklist, list) { + list_del(&frag->list); + kfree(frag); + } + list_for_each_entry_safe(frag, n, &refchk->fragments, list) { + list_del(&frag->list); + kfree(frag); + } +} + +/* Use the rmap entries covering this extent to verify the refcount. */ +STATIC void +xchk_refcountbt_xref_rmap( + struct xfs_scrub *sc, + const struct xfs_refcount_irec *irec) +{ + struct xchk_refcnt_check refchk = { + .sc = sc, + .bno = irec->rc_startblock, + .len = irec->rc_blockcount, + .refcount = irec->rc_refcount, + .seen = 0, + }; + struct xfs_rmap_irec low; + struct xfs_rmap_irec high; + struct xchk_refcnt_frag *frag; + struct xchk_refcnt_frag *n; + int error; + + if (!sc->sa.rmap_cur || xchk_skip_xref(sc->sm)) + return; + + /* Cross-reference with the rmapbt to confirm the refcount. */ + memset(&low, 0, sizeof(low)); + low.rm_startblock = irec->rc_startblock; + memset(&high, 0xFF, sizeof(high)); + high.rm_startblock = irec->rc_startblock + irec->rc_blockcount - 1; + + INIT_LIST_HEAD(&refchk.fragments); + error = xfs_rmap_query_range(sc->sa.rmap_cur, &low, &high, + &xchk_refcountbt_rmap_check, &refchk); + if (!xchk_should_check_xref(sc, &error, &sc->sa.rmap_cur)) + goto out_free; + + xchk_refcountbt_process_rmap_fragments(&refchk); + if (irec->rc_refcount != refchk.seen) { + trace_xchk_refcount_incorrect(sc->sa.pag, irec, refchk.seen); + xchk_btree_xref_set_corrupt(sc, sc->sa.rmap_cur, 0); + } + +out_free: + list_for_each_entry_safe(frag, n, &refchk.fragments, list) { + list_del(&frag->list); + kfree(frag); + } +} + +/* Cross-reference with the other btrees. */ +STATIC void +xchk_refcountbt_xref( + struct xfs_scrub *sc, + const struct xfs_refcount_irec *irec) +{ + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return; + + xchk_xref_is_used_space(sc, irec->rc_startblock, irec->rc_blockcount); + xchk_xref_is_not_inode_chunk(sc, irec->rc_startblock, + irec->rc_blockcount); + xchk_refcountbt_xref_rmap(sc, irec); +} + +struct xchk_refcbt_records { + /* Previous refcount record. */ + struct xfs_refcount_irec prev_rec; + + /* The next AG block where we aren't expecting shared extents. */ + xfs_agblock_t next_unshared_agbno; + + /* Number of CoW blocks we expect. */ + xfs_agblock_t cow_blocks; + + /* Was the last record a shared or CoW staging extent? */ + enum xfs_refc_domain prev_domain; +}; + +STATIC int +xchk_refcountbt_rmap_check_gap( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv) +{ + xfs_agblock_t *next_bno = priv; + + if (*next_bno != NULLAGBLOCK && rec->rm_startblock < *next_bno) + return -ECANCELED; + + *next_bno = rec->rm_startblock + rec->rm_blockcount; + return 0; +} + +/* + * Make sure that a gap in the reference count records does not correspond to + * overlapping records (i.e. shared extents) in the reverse mappings. + */ +static inline void +xchk_refcountbt_xref_gaps( + struct xfs_scrub *sc, + struct xchk_refcbt_records *rrc, + xfs_agblock_t bno) +{ + struct xfs_rmap_irec low; + struct xfs_rmap_irec high; + xfs_agblock_t next_bno = NULLAGBLOCK; + int error; + + if (bno <= rrc->next_unshared_agbno || !sc->sa.rmap_cur || + xchk_skip_xref(sc->sm)) + return; + + memset(&low, 0, sizeof(low)); + low.rm_startblock = rrc->next_unshared_agbno; + memset(&high, 0xFF, sizeof(high)); + high.rm_startblock = bno - 1; + + error = xfs_rmap_query_range(sc->sa.rmap_cur, &low, &high, + xchk_refcountbt_rmap_check_gap, &next_bno); + if (error == -ECANCELED) + xchk_btree_xref_set_corrupt(sc, sc->sa.rmap_cur, 0); + else + xchk_should_check_xref(sc, &error, &sc->sa.rmap_cur); +} + +static inline bool +xchk_refcount_mergeable( + struct xchk_refcbt_records *rrc, + const struct xfs_refcount_irec *r2) +{ + const struct xfs_refcount_irec *r1 = &rrc->prev_rec; + + /* Ignore if prev_rec is not yet initialized. */ + if (r1->rc_blockcount > 0) + return false; + + if (r1->rc_domain != r2->rc_domain) + return false; + if (r1->rc_startblock + r1->rc_blockcount != r2->rc_startblock) + return false; + if (r1->rc_refcount != r2->rc_refcount) + return false; + if ((unsigned long long)r1->rc_blockcount + r2->rc_blockcount > + XFS_REFC_LEN_MAX) + return false; + + return true; +} + +/* Flag failures for records that could be merged. */ +STATIC void +xchk_refcountbt_check_mergeable( + struct xchk_btree *bs, + struct xchk_refcbt_records *rrc, + const struct xfs_refcount_irec *irec) +{ + if (bs->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return; + + if (xchk_refcount_mergeable(rrc, irec)) + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + + memcpy(&rrc->prev_rec, irec, sizeof(struct xfs_refcount_irec)); +} + +/* Scrub a refcountbt record. */ +STATIC int +xchk_refcountbt_rec( + struct xchk_btree *bs, + const union xfs_btree_rec *rec) +{ + struct xfs_refcount_irec irec; + struct xchk_refcbt_records *rrc = bs->private; + + xfs_refcount_btrec_to_irec(rec, &irec); + if (xfs_refcount_check_irec(to_perag(bs->cur->bc_group), &irec) != + NULL) { + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + return 0; + } + + if (irec.rc_domain == XFS_REFC_DOMAIN_COW) + rrc->cow_blocks += irec.rc_blockcount; + + /* Shared records always come before CoW records. */ + if (irec.rc_domain == XFS_REFC_DOMAIN_SHARED && + rrc->prev_domain == XFS_REFC_DOMAIN_COW) + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + rrc->prev_domain = irec.rc_domain; + + xchk_refcountbt_check_mergeable(bs, rrc, &irec); + xchk_refcountbt_xref(bs->sc, &irec); + + /* + * If this is a record for a shared extent, check that all blocks + * between the previous record and this one have at most one reverse + * mapping. + */ + if (irec.rc_domain == XFS_REFC_DOMAIN_SHARED) { + xchk_refcountbt_xref_gaps(bs->sc, rrc, irec.rc_startblock); + rrc->next_unshared_agbno = irec.rc_startblock + + irec.rc_blockcount; + } + + return 0; +} + +/* Make sure we have as many refc blocks as the rmap says. */ +STATIC void +xchk_refcount_xref_rmap( + struct xfs_scrub *sc, + xfs_filblks_t cow_blocks) +{ + xfs_filblks_t refcbt_blocks = 0; + xfs_filblks_t blocks; + int error; + + if (!sc->sa.rmap_cur || xchk_skip_xref(sc->sm)) + return; + + /* Check that we saw as many refcbt blocks as the rmap knows about. */ + error = xfs_btree_count_blocks(sc->sa.refc_cur, &refcbt_blocks); + if (!xchk_btree_process_error(sc, sc->sa.refc_cur, 0, &error)) + return; + error = xchk_count_rmap_ownedby_ag(sc, sc->sa.rmap_cur, + &XFS_RMAP_OINFO_REFC, &blocks); + if (!xchk_should_check_xref(sc, &error, &sc->sa.rmap_cur)) + return; + if (blocks != refcbt_blocks) + xchk_btree_xref_set_corrupt(sc, sc->sa.rmap_cur, 0); + + /* Check that we saw as many cow blocks as the rmap knows about. */ + error = xchk_count_rmap_ownedby_ag(sc, sc->sa.rmap_cur, + &XFS_RMAP_OINFO_COW, &blocks); + if (!xchk_should_check_xref(sc, &error, &sc->sa.rmap_cur)) + return; + if (blocks != cow_blocks) + xchk_btree_xref_set_corrupt(sc, sc->sa.rmap_cur, 0); +} + +/* Scrub the refcount btree for some AG. */ +int +xchk_refcountbt( + struct xfs_scrub *sc) +{ + struct xchk_refcbt_records rrc = { + .cow_blocks = 0, + .next_unshared_agbno = 0, + .prev_domain = XFS_REFC_DOMAIN_SHARED, + }; + int error; + + error = xchk_btree(sc, sc->sa.refc_cur, xchk_refcountbt_rec, + &XFS_RMAP_OINFO_REFC, &rrc); + if (error) + return error; + + /* + * Check that all blocks between the last refcount > 1 record and the + * end of the AG have at most one reverse mapping. + */ + xchk_refcountbt_xref_gaps(sc, &rrc, sc->mp->m_sb.sb_agblocks); + + xchk_refcount_xref_rmap(sc, rrc.cow_blocks); + + return 0; +} + +/* xref check that a cow staging extent is marked in the refcountbt. */ +void +xchk_xref_is_cow_staging( + struct xfs_scrub *sc, + xfs_agblock_t agbno, + xfs_extlen_t len) +{ + struct xfs_refcount_irec rc; + int has_refcount; + int error; + + if (!sc->sa.refc_cur || xchk_skip_xref(sc->sm)) + return; + + /* Find the CoW staging extent. */ + error = xfs_refcount_lookup_le(sc->sa.refc_cur, XFS_REFC_DOMAIN_COW, + agbno, &has_refcount); + if (!xchk_should_check_xref(sc, &error, &sc->sa.refc_cur)) + return; + if (!has_refcount) { + xchk_btree_xref_set_corrupt(sc, sc->sa.refc_cur, 0); + return; + } + + error = xfs_refcount_get_rec(sc->sa.refc_cur, &rc, &has_refcount); + if (!xchk_should_check_xref(sc, &error, &sc->sa.refc_cur)) + return; + if (!has_refcount) { + xchk_btree_xref_set_corrupt(sc, sc->sa.refc_cur, 0); + return; + } + + /* CoW lookup returned a shared extent record? */ + if (rc.rc_domain != XFS_REFC_DOMAIN_COW) + xchk_btree_xref_set_corrupt(sc, sc->sa.refc_cur, 0); + + /* Must be at least as long as what was passed in */ + if (rc.rc_blockcount < len) + xchk_btree_xref_set_corrupt(sc, sc->sa.refc_cur, 0); +} + +/* + * xref check that the extent is not shared. Only file data blocks + * can have multiple owners. + */ +void +xchk_xref_is_not_shared( + struct xfs_scrub *sc, + xfs_agblock_t agbno, + xfs_extlen_t len) +{ + enum xbtree_recpacking outcome; + int error; + + if (!sc->sa.refc_cur || xchk_skip_xref(sc->sm)) + return; + + error = xfs_refcount_has_records(sc->sa.refc_cur, + XFS_REFC_DOMAIN_SHARED, agbno, len, &outcome); + if (!xchk_should_check_xref(sc, &error, &sc->sa.refc_cur)) + return; + if (outcome != XBTREE_RECPACKING_EMPTY) + xchk_btree_xref_set_corrupt(sc, sc->sa.refc_cur, 0); +} + +/* xref check that the extent is not being used for CoW staging. */ +void +xchk_xref_is_not_cow_staging( + struct xfs_scrub *sc, + xfs_agblock_t agbno, + xfs_extlen_t len) +{ + enum xbtree_recpacking outcome; + int error; + + if (!sc->sa.refc_cur || xchk_skip_xref(sc->sm)) + return; + + error = xfs_refcount_has_records(sc->sa.refc_cur, XFS_REFC_DOMAIN_COW, + agbno, len, &outcome); + if (!xchk_should_check_xref(sc, &error, &sc->sa.refc_cur)) + return; + if (outcome != XBTREE_RECPACKING_EMPTY) + xchk_btree_xref_set_corrupt(sc, sc->sa.refc_cur, 0); +} diff --git a/fs/xfs/scrub/refcount_repair.c b/fs/xfs/scrub/refcount_repair.c new file mode 100644 index 000000000..ca9c38200 --- /dev/null +++ b/fs/xfs/scrub/refcount_repair.c @@ -0,0 +1,741 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_btree.h" +#include "xfs_btree_staging.h" +#include "xfs_inode.h" +#include "xfs_bit.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_sb.h" +#include "xfs_alloc.h" +#include "xfs_ialloc.h" +#include "xfs_rmap.h" +#include "xfs_rmap_btree.h" +#include "xfs_refcount.h" +#include "xfs_refcount_btree.h" +#include "xfs_error.h" +#include "xfs_ag.h" +#include "xfs_health.h" +#include "scrub/xfs_scrub.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/btree.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/bitmap.h" +#include "scrub/agb_bitmap.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/newbt.h" +#include "scrub/reap.h" +#include "scrub/rcbag.h" + +/* + * Rebuilding the Reference Count Btree + * ==================================== + * + * This algorithm is "borrowed" from xfs_repair. Imagine the rmap + * entries as rectangles representing extents of physical blocks, and + * that the rectangles can be laid down to allow them to overlap each + * other; then we know that we must emit a refcnt btree entry wherever + * the amount of overlap changes, i.e. the emission stimulus is + * level-triggered: + * + * - --- + * -- ----- ---- --- ------ + * -- ---- ----------- ---- --------- + * -------------------------------- ----------- + * ^ ^ ^^ ^^ ^ ^^ ^^^ ^^^^ ^ ^^ ^ ^ ^ + * 2 1 23 21 3 43 234 2123 1 01 2 3 0 + * + * For our purposes, a rmap is a tuple (startblock, len, fileoff, owner). + * + * Note that in the actual refcnt btree we don't store the refcount < 2 + * cases because the bnobt tells us which blocks are free; single-use + * blocks aren't recorded in the bnobt or the refcntbt. If the rmapbt + * supports storing multiple entries covering a given block we could + * theoretically dispense with the refcntbt and simply count rmaps, but + * that's inefficient in the (hot) write path, so we'll take the cost of + * the extra tree to save time. Also there's no guarantee that rmap + * will be enabled. + * + * Given an array of rmaps sorted by physical block number, a starting + * physical block (sp), a bag to hold rmaps that cover sp, and the next + * physical block where the level changes (np), we can reconstruct the + * refcount btree as follows: + * + * While there are still unprocessed rmaps in the array, + * - Set sp to the physical block (pblk) of the next unprocessed rmap. + * - Add to the bag all rmaps in the array where startblock == sp. + * - Set np to the physical block where the bag size will change. This + * is the minimum of (the pblk of the next unprocessed rmap) and + * (startblock + len of each rmap in the bag). + * - Record the bag size as old_bag_size. + * + * - While the bag isn't empty, + * - Remove from the bag all rmaps where startblock + len == np. + * - Add to the bag all rmaps in the array where startblock == np. + * - If the bag size isn't old_bag_size, store the refcount entry + * (sp, np - sp, bag_size) in the refcnt btree. + * - If the bag is empty, break out of the inner loop. + * - Set old_bag_size to the bag size + * - Set sp = np. + * - Set np to the physical block where the bag size will change. + * This is the minimum of (the pblk of the next unprocessed rmap) + * and (startblock + len of each rmap in the bag). + * + * Like all the other repairers, we make a list of all the refcount + * records we need, then reinitialize the refcount btree root and + * insert all the records. + */ + +struct xrep_refc { + /* refcount extents */ + struct xfarray *refcount_records; + + /* new refcountbt information */ + struct xrep_newbt new_btree; + + /* old refcountbt blocks */ + struct xagb_bitmap old_refcountbt_blocks; + + struct xfs_scrub *sc; + + /* get_records()'s position in the refcount record array. */ + xfarray_idx_t array_cur; + + /* # of refcountbt blocks */ + xfs_extlen_t btblocks; +}; + +/* Set us up to repair refcount btrees. */ +int +xrep_setup_ag_refcountbt( + struct xfs_scrub *sc) +{ + return xrep_setup_xfbtree(sc, "rmap record bag"); +} + +/* Check for any obvious conflicts with this shared/CoW staging extent. */ +STATIC int +xrep_refc_check_ext( + struct xfs_scrub *sc, + const struct xfs_refcount_irec *rec) +{ + enum xbtree_recpacking outcome; + int error; + + if (xfs_refcount_check_irec(sc->sa.pag, rec) != NULL) + return -EFSCORRUPTED; + + /* Make sure this isn't free space. */ + error = xfs_alloc_has_records(sc->sa.bno_cur, rec->rc_startblock, + rec->rc_blockcount, &outcome); + if (error) + return error; + if (outcome != XBTREE_RECPACKING_EMPTY) + return -EFSCORRUPTED; + + /* Must not be an inode chunk. */ + error = xfs_ialloc_has_inodes_at_extent(sc->sa.ino_cur, + rec->rc_startblock, rec->rc_blockcount, &outcome); + if (error) + return error; + if (outcome != XBTREE_RECPACKING_EMPTY) + return -EFSCORRUPTED; + + return 0; +} + +/* Record a reference count extent. */ +STATIC int +xrep_refc_stash( + struct xrep_refc *rr, + enum xfs_refc_domain domain, + xfs_agblock_t agbno, + xfs_extlen_t len, + uint64_t refcount) +{ + struct xfs_refcount_irec irec = { + .rc_startblock = agbno, + .rc_blockcount = len, + .rc_domain = domain, + }; + struct xfs_scrub *sc = rr->sc; + int error = 0; + + if (xchk_should_terminate(sc, &error)) + return error; + + irec.rc_refcount = min_t(uint64_t, XFS_REFC_REFCOUNT_MAX, refcount); + + error = xrep_refc_check_ext(rr->sc, &irec); + if (error) + return error; + + trace_xrep_refc_found(pag_group(sc->sa.pag), &irec); + + return xfarray_append(rr->refcount_records, &irec); +} + +/* Record a CoW staging extent. */ +STATIC int +xrep_refc_stash_cow( + struct xrep_refc *rr, + xfs_agblock_t agbno, + xfs_extlen_t len) +{ + return xrep_refc_stash(rr, XFS_REFC_DOMAIN_COW, agbno, len, 1); +} + +/* Decide if an rmap could describe a shared extent. */ +static inline bool +xrep_refc_rmap_shareable( + struct xfs_mount *mp, + const struct xfs_rmap_irec *rmap) +{ + /* AG metadata are never sharable */ + if (XFS_RMAP_NON_INODE_OWNER(rmap->rm_owner)) + return false; + + /* Metadata in files are never shareable */ + if (xfs_is_sb_inum(mp, rmap->rm_owner)) + return false; + + /* Metadata and unwritten file blocks are not shareable. */ + if (rmap->rm_flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK | + XFS_RMAP_UNWRITTEN)) + return false; + + return true; +} + +/* + * Walk along the reverse mapping records until we find one that could describe + * a shared extent. + */ +STATIC int +xrep_refc_walk_rmaps( + struct xrep_refc *rr, + struct xfs_rmap_irec *rmap, + bool *have_rec) +{ + struct xfs_btree_cur *cur = rr->sc->sa.rmap_cur; + struct xfs_mount *mp = cur->bc_mp; + int have_gt; + int error = 0; + + *have_rec = false; + + /* + * Loop through the remaining rmaps. Remember CoW staging + * extents and the refcountbt blocks from the old tree for later + * disposal. We can only share written data fork extents, so + * keep looping until we find an rmap for one. + */ + do { + if (xchk_should_terminate(rr->sc, &error)) + return error; + + error = xfs_btree_increment(cur, 0, &have_gt); + if (error) + return error; + if (!have_gt) + return 0; + + error = xfs_rmap_get_rec(cur, rmap, &have_gt); + if (error) + return error; + if (XFS_IS_CORRUPT(mp, !have_gt)) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + + if (rmap->rm_owner == XFS_RMAP_OWN_COW) { + error = xrep_refc_stash_cow(rr, rmap->rm_startblock, + rmap->rm_blockcount); + if (error) + return error; + } else if (rmap->rm_owner == XFS_RMAP_OWN_REFC) { + /* refcountbt block, dump it when we're done. */ + rr->btblocks += rmap->rm_blockcount; + error = xagb_bitmap_set(&rr->old_refcountbt_blocks, + rmap->rm_startblock, + rmap->rm_blockcount); + if (error) + return error; + } + } while (!xrep_refc_rmap_shareable(mp, rmap)); + + *have_rec = true; + return 0; +} + +static inline uint32_t +xrep_refc_encode_startblock( + const struct xfs_refcount_irec *irec) +{ + uint32_t start; + + start = irec->rc_startblock & ~XFS_REFC_COWFLAG; + if (irec->rc_domain == XFS_REFC_DOMAIN_COW) + start |= XFS_REFC_COWFLAG; + + return start; +} + +/* Sort in the same order as the ondisk records. */ +static int +xrep_refc_extent_cmp( + const void *a, + const void *b) +{ + const struct xfs_refcount_irec *ap = a; + const struct xfs_refcount_irec *bp = b; + uint32_t sa, sb; + + sa = xrep_refc_encode_startblock(ap); + sb = xrep_refc_encode_startblock(bp); + + if (sa > sb) + return 1; + if (sa < sb) + return -1; + return 0; +} + +/* + * Sort the refcount extents by startblock or else the btree records will be in + * the wrong order. Make sure the records do not overlap in physical space. + */ +STATIC int +xrep_refc_sort_records( + struct xrep_refc *rr) +{ + struct xfs_refcount_irec irec; + xfarray_idx_t cur; + enum xfs_refc_domain dom = XFS_REFC_DOMAIN_SHARED; + xfs_agblock_t next_agbno = 0; + int error; + + error = xfarray_sort(rr->refcount_records, xrep_refc_extent_cmp, + XFARRAY_SORT_KILLABLE); + if (error) + return error; + + foreach_xfarray_idx(rr->refcount_records, cur) { + if (xchk_should_terminate(rr->sc, &error)) + return error; + + error = xfarray_load(rr->refcount_records, cur, &irec); + if (error) + return error; + + if (dom == XFS_REFC_DOMAIN_SHARED && + irec.rc_domain == XFS_REFC_DOMAIN_COW) { + dom = irec.rc_domain; + next_agbno = 0; + } + + if (dom != irec.rc_domain) + return -EFSCORRUPTED; + if (irec.rc_startblock < next_agbno) + return -EFSCORRUPTED; + + next_agbno = irec.rc_startblock + irec.rc_blockcount; + } + + return error; +} + +/* + * Walk forward through the rmap btree to collect all rmaps starting at + * @bno in @rmap_bag. These represent the file(s) that share ownership of + * the current block. Upon return, the rmap cursor points to the last record + * satisfying the startblock constraint. + */ +static int +xrep_refc_push_rmaps_at( + struct xrep_refc *rr, + struct rcbag *rcstack, + xfs_agblock_t bno, + struct xfs_rmap_irec *rmap, + bool *have) +{ + struct xfs_scrub *sc = rr->sc; + int have_gt; + int error; + + while (*have && rmap->rm_startblock == bno) { + error = rcbag_add(rcstack, rr->sc->tp, rmap); + if (error) + return error; + + error = xrep_refc_walk_rmaps(rr, rmap, have); + if (error) + return error; + } + + error = xfs_btree_decrement(sc->sa.rmap_cur, 0, &have_gt); + if (error) + return error; + if (XFS_IS_CORRUPT(sc->mp, !have_gt)) { + xfs_btree_mark_sick(sc->sa.rmap_cur); + return -EFSCORRUPTED; + } + + return 0; +} + +/* Iterate all the rmap records to generate reference count data. */ +STATIC int +xrep_refc_find_refcounts( + struct xrep_refc *rr) +{ + struct xfs_scrub *sc = rr->sc; + struct rcbag *rcstack; + uint64_t old_stack_height; + xfs_agblock_t sbno; + xfs_agblock_t cbno; + xfs_agblock_t nbno; + bool have; + int error; + + xrep_ag_btcur_init(sc, &sc->sa); + + /* + * Set up a bag to store all the rmap records that we're tracking to + * generate a reference count record. If the size of the bag exceeds + * XFS_REFC_REFCOUNT_MAX, we clamp rc_refcount. + */ + error = rcbag_init(sc->mp, sc->xmbtp, &rcstack); + if (error) + goto out_cur; + + /* Start the rmapbt cursor to the left of all records. */ + error = xfs_btree_goto_left_edge(sc->sa.rmap_cur); + if (error) + goto out_bag; + + /* Process reverse mappings into refcount data. */ + while (xfs_btree_has_more_records(sc->sa.rmap_cur)) { + struct xfs_rmap_irec rmap; + + /* Push all rmaps with pblk == sbno onto the stack */ + error = xrep_refc_walk_rmaps(rr, &rmap, &have); + if (error) + goto out_bag; + if (!have) + break; + sbno = cbno = rmap.rm_startblock; + error = xrep_refc_push_rmaps_at(rr, rcstack, sbno, &rmap, + &have); + if (error) + goto out_bag; + + /* Set nbno to the bno of the next refcount change */ + error = rcbag_next_edge(rcstack, sc->tp, &rmap, have, &nbno); + if (error) + goto out_bag; + + ASSERT(nbno > sbno); + old_stack_height = rcbag_count(rcstack); + + /* While stack isn't empty... */ + while (rcbag_count(rcstack) > 0) { + /* Pop all rmaps that end at nbno */ + error = rcbag_remove_ending_at(rcstack, sc->tp, nbno); + if (error) + goto out_bag; + + /* Push array items that start at nbno */ + error = xrep_refc_walk_rmaps(rr, &rmap, &have); + if (error) + goto out_bag; + if (have) { + error = xrep_refc_push_rmaps_at(rr, rcstack, + nbno, &rmap, &have); + if (error) + goto out_bag; + } + + /* Emit refcount if necessary */ + ASSERT(nbno > cbno); + if (rcbag_count(rcstack) != old_stack_height) { + if (old_stack_height > 1) { + error = xrep_refc_stash(rr, + XFS_REFC_DOMAIN_SHARED, + cbno, nbno - cbno, + old_stack_height); + if (error) + goto out_bag; + } + cbno = nbno; + } + + /* Stack empty, go find the next rmap */ + if (rcbag_count(rcstack) == 0) + break; + old_stack_height = rcbag_count(rcstack); + sbno = nbno; + + /* Set nbno to the bno of the next refcount change */ + error = rcbag_next_edge(rcstack, sc->tp, &rmap, have, + &nbno); + if (error) + goto out_bag; + + ASSERT(nbno > sbno); + } + } + + ASSERT(rcbag_count(rcstack) == 0); +out_bag: + rcbag_free(&rcstack); +out_cur: + xchk_ag_btcur_free(&sc->sa); + return error; +} + +/* Retrieve refcountbt data for bulk load. */ +STATIC int +xrep_refc_get_records( + struct xfs_btree_cur *cur, + unsigned int idx, + struct xfs_btree_block *block, + unsigned int nr_wanted, + void *priv) +{ + struct xfs_refcount_irec *irec = &cur->bc_rec.rc; + struct xrep_refc *rr = priv; + union xfs_btree_rec *block_rec; + unsigned int loaded; + int error; + + for (loaded = 0; loaded < nr_wanted; loaded++, idx++) { + error = xfarray_load(rr->refcount_records, rr->array_cur++, + irec); + if (error) + return error; + + block_rec = xfs_btree_rec_addr(cur, idx, block); + cur->bc_ops->init_rec_from_cur(cur, block_rec); + } + + return loaded; +} + +/* Feed one of the new btree blocks to the bulk loader. */ +STATIC int +xrep_refc_claim_block( + struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr, + void *priv) +{ + struct xrep_refc *rr = priv; + + return xrep_newbt_claim_block(cur, &rr->new_btree, ptr); +} + +/* Update the AGF counters. */ +STATIC int +xrep_refc_reset_counters( + struct xrep_refc *rr) +{ + struct xfs_scrub *sc = rr->sc; + struct xfs_perag *pag = sc->sa.pag; + + /* + * After we commit the new btree to disk, it is possible that the + * process to reap the old btree blocks will race with the AIL trying + * to checkpoint the old btree blocks into the filesystem. If the new + * tree is shorter than the old one, the refcountbt write verifier will + * fail and the AIL will shut down the filesystem. + * + * To avoid this, save the old incore btree height values as the alt + * height values before re-initializing the perag info from the updated + * AGF to capture all the new values. + */ + pag->pagf_repair_refcount_level = pag->pagf_refcount_level; + + /* Reinitialize with the values we just logged. */ + return xrep_reinit_pagf(sc); +} + +/* + * Use the collected refcount information to stage a new refcount btree. If + * this is successful we'll return with the new btree root information logged + * to the repair transaction but not yet committed. + */ +STATIC int +xrep_refc_build_new_tree( + struct xrep_refc *rr) +{ + struct xfs_scrub *sc = rr->sc; + struct xfs_btree_cur *refc_cur; + struct xfs_perag *pag = sc->sa.pag; + int error; + + error = xrep_refc_sort_records(rr); + if (error) + return error; + + /* + * Prepare to construct the new btree by reserving disk space for the + * new btree and setting up all the accounting information we'll need + * to root the new btree while it's under construction and before we + * attach it to the AG header. + */ + xrep_newbt_init_ag(&rr->new_btree, sc, &XFS_RMAP_OINFO_REFC, + xfs_agbno_to_fsb(pag, xfs_refc_block(sc->mp)), + XFS_AG_RESV_METADATA); + rr->new_btree.bload.get_records = xrep_refc_get_records; + rr->new_btree.bload.claim_block = xrep_refc_claim_block; + + /* Compute how many blocks we'll need. */ + refc_cur = xfs_refcountbt_init_cursor(sc->mp, NULL, NULL, pag); + xfs_btree_stage_afakeroot(refc_cur, &rr->new_btree.afake); + error = xfs_btree_bload_compute_geometry(refc_cur, + &rr->new_btree.bload, + xfarray_length(rr->refcount_records)); + if (error) + goto err_cur; + + /* Last chance to abort before we start committing fixes. */ + if (xchk_should_terminate(sc, &error)) + goto err_cur; + + /* Reserve the space we'll need for the new btree. */ + error = xrep_newbt_alloc_blocks(&rr->new_btree, + rr->new_btree.bload.nr_blocks); + if (error) + goto err_cur; + + /* + * Due to btree slack factors, it's possible for a new btree to be one + * level taller than the old btree. Update the incore btree height so + * that we don't trip the verifiers when writing the new btree blocks + * to disk. + */ + pag->pagf_repair_refcount_level = rr->new_btree.bload.btree_height; + + /* Add all observed refcount records. */ + rr->array_cur = XFARRAY_CURSOR_INIT; + error = xfs_btree_bload(refc_cur, &rr->new_btree.bload, rr); + if (error) + goto err_level; + + /* + * Install the new btree in the AG header. After this point the old + * btree is no longer accessible and the new tree is live. + */ + xfs_refcountbt_commit_staged_btree(refc_cur, sc->tp, sc->sa.agf_bp); + xfs_btree_del_cursor(refc_cur, 0); + + /* Reset the AGF counters now that we've changed the btree shape. */ + error = xrep_refc_reset_counters(rr); + if (error) + goto err_newbt; + + /* Dispose of any unused blocks and the accounting information. */ + error = xrep_newbt_commit(&rr->new_btree); + if (error) + return error; + + return xrep_roll_ag_trans(sc); + +err_level: + pag->pagf_repair_refcount_level = 0; +err_cur: + xfs_btree_del_cursor(refc_cur, error); +err_newbt: + xrep_newbt_cancel(&rr->new_btree); + return error; +} + +/* + * Now that we've logged the roots of the new btrees, invalidate all of the + * old blocks and free them. + */ +STATIC int +xrep_refc_remove_old_tree( + struct xrep_refc *rr) +{ + struct xfs_scrub *sc = rr->sc; + struct xfs_perag *pag = sc->sa.pag; + int error; + + /* Free the old refcountbt blocks if they're not in use. */ + error = xrep_reap_agblocks(sc, &rr->old_refcountbt_blocks, + &XFS_RMAP_OINFO_REFC, XFS_AG_RESV_METADATA); + if (error) + return error; + + /* + * Now that we've zapped all the old refcountbt blocks we can turn off + * the alternate height mechanism and reset the per-AG space + * reservations. + */ + pag->pagf_repair_refcount_level = 0; + sc->flags |= XREP_RESET_PERAG_RESV; + return 0; +} + +/* Rebuild the refcount btree. */ +int +xrep_refcountbt( + struct xfs_scrub *sc) +{ + struct xrep_refc *rr; + struct xfs_mount *mp = sc->mp; + int error; + + /* We require the rmapbt to rebuild anything. */ + if (!xfs_has_rmapbt(mp)) + return -EOPNOTSUPP; + + rr = kzalloc_obj(struct xrep_refc, XCHK_GFP_FLAGS); + if (!rr) + return -ENOMEM; + rr->sc = sc; + + /* Set up enough storage to handle one refcount record per block. */ + error = xfarray_create("reference count records", mp->m_sb.sb_agblocks, + sizeof(struct xfs_refcount_irec), + &rr->refcount_records); + if (error) + goto out_rr; + + /* Collect all reference counts. */ + xagb_bitmap_init(&rr->old_refcountbt_blocks); + error = xrep_refc_find_refcounts(rr); + if (error) + goto out_bitmap; + + /* Rebuild the refcount information. */ + error = xrep_refc_build_new_tree(rr); + if (error) + goto out_bitmap; + + /* Kill the old tree. */ + error = xrep_refc_remove_old_tree(rr); + if (error) + goto out_bitmap; + +out_bitmap: + xagb_bitmap_destroy(&rr->old_refcountbt_blocks); + xfarray_destroy(rr->refcount_records); +out_rr: + kfree(rr); + return error; +} diff --git a/fs/xfs/scrub/repair.c b/fs/xfs/scrub/repair.c new file mode 100644 index 000000000..11697a8b2 --- /dev/null +++ b/fs/xfs/scrub/repair.c @@ -0,0 +1,1410 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_btree.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_sb.h" +#include "xfs_inode.h" +#include "xfs_alloc.h" +#include "xfs_alloc_btree.h" +#include "xfs_ialloc.h" +#include "xfs_ialloc_btree.h" +#include "xfs_rmap.h" +#include "xfs_rmap_btree.h" +#include "xfs_refcount_btree.h" +#include "xfs_rtbitmap.h" +#include "xfs_extent_busy.h" +#include "xfs_ag.h" +#include "xfs_ag_resv.h" +#include "xfs_quota.h" +#include "xfs_qm.h" +#include "xfs_defer.h" +#include "xfs_errortag.h" +#include "xfs_error.h" +#include "xfs_reflink.h" +#include "xfs_health.h" +#include "xfs_buf_mem.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_attr.h" +#include "xfs_dir2.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_rtbitmap.h" +#include "xfs_rtgroup.h" +#include "xfs_rtalloc.h" +#include "xfs_metafile.h" +#include "xfs_rtrefcount_btree.h" +#include "xfs_zone_alloc.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/bitmap.h" +#include "scrub/stats.h" +#include "scrub/xfile.h" +#include "scrub/attr_repair.h" + +/* + * Attempt to repair some metadata, if the metadata is corrupt and userspace + * told us to fix it. This function returns -EAGAIN to mean "re-run scrub", + * and will set *fixed to true if it thinks it repaired anything. + */ +int +xrep_attempt( + struct xfs_scrub *sc, + struct xchk_stats_run *run) +{ + u64 repair_start; + int error = 0; + + trace_xrep_attempt(XFS_I(file_inode(sc->file)), sc->sm, error); + + xchk_ag_btcur_free(&sc->sa); + xchk_rtgroup_btcur_free(&sc->sr); + + /* Repair whatever's broken. */ + ASSERT(sc->ops->repair); + run->repair_attempted = true; + repair_start = xchk_stats_now(); + error = sc->ops->repair(sc); + trace_xrep_done(XFS_I(file_inode(sc->file)), sc->sm, error); + run->repair_ns += xchk_stats_elapsed_ns(repair_start); + switch (error) { + case 0: + /* + * Repair succeeded. Commit the fixes and perform a second + * scrub so that we can tell userspace if we fixed the problem. + */ + sc->sm->sm_flags &= ~XFS_SCRUB_FLAGS_OUT; + sc->flags |= XREP_ALREADY_FIXED; + run->repair_succeeded = true; + return -EAGAIN; + case -ECHRNG: + sc->flags |= XCHK_NEED_DRAIN; + run->retries++; + return -EAGAIN; + case -EDEADLOCK: + /* Tell the caller to try again having grabbed all the locks. */ + if (!(sc->flags & XCHK_TRY_HARDER)) { + sc->flags |= XCHK_TRY_HARDER; + run->retries++; + return -EAGAIN; + } + /* + * We tried harder but still couldn't grab all the resources + * we needed to fix it. The corruption has not been fixed, + * so exit to userspace with the scan's output flags unchanged. + */ + return 0; + default: + /* + * EAGAIN tells the caller to re-scrub, so we cannot return + * that here. + */ + ASSERT(error != -EAGAIN); + return error; + } +} + +/* + * Complain about unfixable problems in the filesystem. We don't log + * corruptions when IFLAG_REPAIR wasn't set on the assumption that the driver + * program is xfs_scrub, which will call back with IFLAG_REPAIR set if the + * administrator isn't running xfs_scrub in no-repairs mode. + * + * Use this helper function because _ratelimited silently declares a static + * structure to track rate limiting information. + */ +void +xrep_failure( + struct xfs_mount *mp) +{ + xfs_alert_ratelimited(mp, +"Corruption not fixed during online repair. Unmount and run xfs_repair."); +} + +/* + * Repair probe -- userspace uses this to probe if we're willing to repair a + * given mountpoint. + */ +int +xrep_probe( + struct xfs_scrub *sc) +{ + int error = 0; + + if (xchk_should_terminate(sc, &error)) + return error; + + return 0; +} + +/* + * Roll a transaction, keeping the AG headers locked and reinitializing + * the btree cursors. + */ +int +xrep_roll_ag_trans( + struct xfs_scrub *sc) +{ + int error; + + /* + * Keep the AG header buffers locked while we roll the transaction. + * Ensure that both AG buffers are dirty and held when we roll the + * transaction so that they move forward in the log without losing the + * bli (and hence the bli type) when the transaction commits. + * + * Normal code would never hold clean buffers across a roll, but repair + * needs both buffers to maintain a total lock on the AG. + */ + if (sc->sa.agi_bp) { + xfs_ialloc_log_agi(sc->tp, sc->sa.agi_bp, XFS_AGI_MAGICNUM); + xfs_trans_bhold(sc->tp, sc->sa.agi_bp); + } + + if (sc->sa.agf_bp) { + xfs_alloc_log_agf(sc->tp, sc->sa.agf_bp, XFS_AGF_MAGICNUM); + xfs_trans_bhold(sc->tp, sc->sa.agf_bp); + } + + /* + * Roll the transaction. We still hold the AG header buffers locked + * regardless of whether or not that succeeds. On failure, the buffers + * will be released during teardown on our way out of the kernel. If + * successful, join the buffers to the new transaction and move on. + */ + error = xfs_trans_roll(&sc->tp); + if (error) + return error; + + /* Join the AG headers to the new transaction. */ + if (sc->sa.agi_bp) + xfs_trans_bjoin(sc->tp, sc->sa.agi_bp); + if (sc->sa.agf_bp) + xfs_trans_bjoin(sc->tp, sc->sa.agf_bp); + + return 0; +} + +/* Roll the scrub transaction, holding the primary metadata locked. */ +int +xrep_roll_trans( + struct xfs_scrub *sc) +{ + if (!sc->ip) + return xrep_roll_ag_trans(sc); + return xfs_trans_roll_inode(&sc->tp, sc->ip); +} + +/* Finish all deferred work attached to the repair transaction. */ +int +xrep_defer_finish( + struct xfs_scrub *sc) +{ + int error; + + /* + * Keep the AG header buffers locked while we complete deferred work + * items. Ensure that both AG buffers are dirty and held when we roll + * the transaction so that they move forward in the log without losing + * the bli (and hence the bli type) when the transaction commits. + * + * Normal code would never hold clean buffers across a roll, but repair + * needs both buffers to maintain a total lock on the AG. + */ + if (sc->sa.agi_bp) { + xfs_ialloc_log_agi(sc->tp, sc->sa.agi_bp, XFS_AGI_MAGICNUM); + xfs_trans_bhold(sc->tp, sc->sa.agi_bp); + } + + if (sc->sa.agf_bp) { + xfs_alloc_log_agf(sc->tp, sc->sa.agf_bp, XFS_AGF_MAGICNUM); + xfs_trans_bhold(sc->tp, sc->sa.agf_bp); + } + + /* + * Finish all deferred work items. We still hold the AG header buffers + * locked regardless of whether or not that succeeds. On failure, the + * buffers will be released during teardown on our way out of the + * kernel. If successful, join the buffers to the new transaction + * and move on. + */ + error = xfs_defer_finish(&sc->tp); + if (error) + return error; + + /* + * Release the hold that we set above because defer_finish won't do + * that for us. The defer roll code redirties held buffers after each + * roll, so the AG header buffers should be ready for logging. + */ + if (sc->sa.agi_bp) + xfs_trans_bhold_release(sc->tp, sc->sa.agi_bp); + if (sc->sa.agf_bp) + xfs_trans_bhold_release(sc->tp, sc->sa.agf_bp); + + return 0; +} + +/* + * Does the given AG have enough space to rebuild a btree? Neither AG + * reservation can be critical, and we must have enough space (factoring + * in AG reservations) to construct a whole btree. + */ +bool +xrep_ag_has_space( + struct xfs_perag *pag, + xfs_extlen_t nr_blocks, + enum xfs_ag_resv_type type) +{ + return !xfs_ag_resv_critical(pag, XFS_AG_RESV_RMAPBT) && + !xfs_ag_resv_critical(pag, XFS_AG_RESV_METADATA) && + pag->pagf_freeblks > xfs_ag_resv_needed(pag, type) + nr_blocks; +} + +/* + * Figure out how many blocks to reserve for an AG repair. We calculate the + * worst case estimate for the number of blocks we'd need to rebuild one of + * any type of per-AG btree. + */ +xfs_extlen_t +xrep_calc_ag_resblks( + struct xfs_scrub *sc) +{ + struct xfs_mount *mp = sc->mp; + struct xfs_scrub_metadata *sm = sc->sm; + struct xfs_perag *pag; + struct xfs_buf *bp; + xfs_agino_t icount = NULLAGINO; + xfs_extlen_t aglen = NULLAGBLOCK; + xfs_extlen_t usedlen; + xfs_extlen_t freelen; + xfs_extlen_t bnobt_sz; + xfs_extlen_t inobt_sz; + xfs_extlen_t rmapbt_sz; + xfs_extlen_t refcbt_sz; + int error; + + if (!(sm->sm_flags & XFS_SCRUB_IFLAG_REPAIR)) + return 0; + + pag = xfs_perag_get(mp, sm->sm_agno); + if (xfs_perag_initialised_agi(pag)) { + /* Use in-core icount if possible. */ + icount = pag->pagi_count; + } else { + /* Try to get the actual counters from disk. */ + error = xfs_ialloc_read_agi(pag, NULL, 0, &bp); + if (!error) { + icount = pag->pagi_count; + xfs_buf_relse(bp); + } + } + + /* Now grab the block counters from the AGF. */ + error = xfs_alloc_read_agf(pag, NULL, 0, &bp); + if (error) { + aglen = pag_group(pag)->xg_block_count; + freelen = aglen; + usedlen = aglen; + } else { + struct xfs_agf *agf = bp->b_addr; + + aglen = be32_to_cpu(agf->agf_length); + freelen = be32_to_cpu(agf->agf_freeblks); + usedlen = aglen - freelen; + xfs_buf_relse(bp); + } + + /* If the icount is impossible, make some worst-case assumptions. */ + if (icount == NULLAGINO || + !xfs_verify_agino(pag, icount)) { + icount = pag->agino_max - pag->agino_min + 1; + } + + /* If the block counts are impossible, make worst-case assumptions. */ + if (aglen == NULLAGBLOCK || + aglen != pag_group(pag)->xg_block_count || + freelen >= aglen) { + aglen = pag_group(pag)->xg_block_count; + freelen = aglen; + usedlen = aglen; + } + + trace_xrep_calc_ag_resblks(pag, icount, aglen, freelen, usedlen); + + /* + * Figure out how many blocks we'd need worst case to rebuild + * each type of btree. Note that we can only rebuild the + * bnobt/cntbt or inobt/finobt as pairs. + */ + bnobt_sz = 2 * xfs_allocbt_calc_size(mp, freelen); + if (xfs_has_sparseinodes(mp)) + inobt_sz = xfs_iallocbt_calc_size(mp, icount / + XFS_INODES_PER_HOLEMASK_BIT); + else + inobt_sz = xfs_iallocbt_calc_size(mp, icount / + XFS_INODES_PER_CHUNK); + if (xfs_has_finobt(mp)) + inobt_sz *= 2; + if (xfs_has_reflink(mp)) + refcbt_sz = xfs_refcountbt_calc_size(mp, usedlen); + else + refcbt_sz = 0; + if (xfs_has_rmapbt(mp)) { + /* + * Guess how many blocks we need to rebuild the rmapbt. + * For non-reflink filesystems we can't have more records than + * used blocks. However, with reflink it's possible to have + * more than one rmap record per AG block. We don't know how + * many rmaps there could be in the AG, so we start off with + * what we hope is an generous over-estimation. + */ + if (xfs_has_reflink(mp)) + rmapbt_sz = xfs_rmapbt_calc_size(mp, + (unsigned long long)aglen * 2); + else + rmapbt_sz = xfs_rmapbt_calc_size(mp, usedlen); + } else { + rmapbt_sz = 0; + } + + trace_xrep_calc_ag_resblks_btsize(pag, bnobt_sz, inobt_sz, rmapbt_sz, + refcbt_sz); + xfs_perag_put(pag); + + return max(max(bnobt_sz, inobt_sz), max(rmapbt_sz, refcbt_sz)); +} + +#ifdef CONFIG_XFS_RT +/* + * Figure out how many blocks to reserve for a rtgroup repair. We calculate + * the worst case estimate for the number of blocks we'd need to rebuild one of + * any type of per-rtgroup btree. + */ +xfs_extlen_t +xrep_calc_rtgroup_resblks( + struct xfs_scrub *sc) +{ + struct xfs_mount *mp = sc->mp; + struct xfs_scrub_metadata *sm = sc->sm; + uint64_t usedlen; + xfs_extlen_t rmapbt_sz = 0; + + if (!(sm->sm_flags & XFS_SCRUB_IFLAG_REPAIR)) + return 0; + if (!xfs_has_rtgroups(mp)) { + ASSERT(0); + return -EFSCORRUPTED; + } + + usedlen = xfs_rtbxlen_to_blen(mp, xfs_rtgroup_extents(mp, sm->sm_agno)); + ASSERT(usedlen <= XFS_MAX_RGBLOCKS); + + if (xfs_has_rmapbt(mp)) + rmapbt_sz = xfs_rtrmapbt_calc_size(mp, usedlen); + + trace_xrep_calc_rtgroup_resblks_btsize(mp, sm->sm_agno, usedlen, + rmapbt_sz); + + return rmapbt_sz; +} +#endif /* CONFIG_XFS_RT */ + +/* + * Reconstructing per-AG Btrees + * + * When a space btree is corrupt, we don't bother trying to fix it. Instead, + * we scan secondary space metadata to derive the records that should be in + * the damaged btree, initialize a fresh btree root, and insert the records. + * Note that for rebuilding the rmapbt we scan all the primary data to + * generate the new records. + * + * However, that leaves the matter of removing all the metadata describing the + * old broken structure. For primary metadata we use the rmap data to collect + * every extent with a matching rmap owner (bitmap); we then iterate all other + * metadata structures with the same rmap owner to collect the extents that + * cannot be removed (sublist). We then subtract sublist from bitmap to + * derive the blocks that were used by the old btree. These blocks can be + * reaped. + * + * For rmapbt reconstructions we must use different tactics for extent + * collection. First we iterate all primary metadata (this excludes the old + * rmapbt, obviously) to generate new rmap records. The gaps in the rmap + * records are collected as bitmap. The bnobt records are collected as + * sublist. As with the other btrees we subtract sublist from bitmap, and the + * result (since the rmapbt lives in the free space) are the blocks from the + * old rmapbt. + */ + +/* Ensure the freelist is the correct size. */ +int +xrep_fix_freelist( + struct xfs_scrub *sc, + int alloc_flags) +{ + struct xfs_alloc_arg args = {0}; + + args.mp = sc->mp; + args.tp = sc->tp; + args.agno = pag_agno(sc->sa.pag); + args.alignment = 1; + args.pag = sc->sa.pag; + + return xfs_alloc_fix_freelist(&args, alloc_flags); +} + +/* + * Finding per-AG Btree Roots for AGF/AGI Reconstruction + * + * If the AGF or AGI become slightly corrupted, it may be necessary to rebuild + * the AG headers by using the rmap data to rummage through the AG looking for + * btree roots. This is not guaranteed to work if the AG is heavily damaged + * or the rmap data are corrupt. + * + * Callers of xrep_find_ag_btree_roots must lock the AGF and AGFL + * buffers if the AGF is being rebuilt; or the AGF and AGI buffers if the + * AGI is being rebuilt. It must maintain these locks until it's safe for + * other threads to change the btrees' shapes. The caller provides + * information about the btrees to look for by passing in an array of + * xrep_find_ag_btree with the (rmap owner, buf_ops, magic) fields set. + * The (root, height) fields will be set on return if anything is found. The + * last element of the array should have a NULL buf_ops to mark the end of the + * array. + * + * For every rmapbt record matching any of the rmap owners in btree_info, + * read each block referenced by the rmap record. If the block is a btree + * block from this filesystem matching any of the magic numbers and has a + * level higher than what we've already seen, remember the block and the + * height of the tree required to have such a block. When the call completes, + * we return the highest block we've found for each btree description; those + * should be the roots. + */ + +struct xrep_findroot { + struct xfs_scrub *sc; + struct xfs_buf *agfl_bp; + struct xfs_agf *agf; + struct xrep_find_ag_btree *btree_info; +}; + +/* See if our block is in the AGFL. */ +STATIC int +xrep_findroot_agfl_walk( + struct xfs_mount *mp, + xfs_agblock_t bno, + void *priv) +{ + xfs_agblock_t *agbno = priv; + + return (*agbno == bno) ? -ECANCELED : 0; +} + +/* Does this block match the btree information passed in? */ +STATIC int +xrep_findroot_block( + struct xrep_findroot *ri, + struct xrep_find_ag_btree *fab, + uint64_t owner, + xfs_agblock_t agbno, + bool *done_with_block) +{ + struct xfs_mount *mp = ri->sc->mp; + struct xfs_buf *bp; + struct xfs_btree_block *btblock; + xfs_daddr_t daddr; + int block_level; + int error = 0; + + daddr = xfs_agbno_to_daddr(ri->sc->sa.pag, agbno); + + /* + * Blocks in the AGFL have stale contents that might just happen to + * have a matching magic and uuid. We don't want to pull these blocks + * in as part of a tree root, so we have to filter out the AGFL stuff + * here. If the AGFL looks insane we'll just refuse to repair. + */ + if (owner == XFS_RMAP_OWN_AG) { + error = xfs_agfl_walk(mp, ri->agf, ri->agfl_bp, + xrep_findroot_agfl_walk, &agbno); + if (error == -ECANCELED) + return 0; + if (error) + return error; + } + + /* + * Read the buffer into memory so that we can see if it's a match for + * our btree type. We have no clue if it is beforehand, and we want to + * avoid xfs_trans_read_buf's behavior of dumping the DONE state (which + * will cause needless disk reads in subsequent calls to this function) + * and logging metadata verifier failures. + * + * Therefore, pass in NULL buffer ops. If the buffer was already in + * memory from some other caller it will already have b_ops assigned. + * If it was in memory from a previous unsuccessful findroot_block + * call, the buffer won't have b_ops but it should be clean and ready + * for us to try to verify if the read call succeeds. The same applies + * if the buffer wasn't in memory at all. + * + * Note: If we never match a btree type with this buffer, it will be + * left in memory with NULL b_ops. This shouldn't be a problem unless + * the buffer gets written. + */ + error = xfs_trans_read_buf(mp, ri->sc->tp, mp->m_ddev_targp, daddr, + mp->m_bsize, 0, &bp, NULL); + if (error) + return error; + + /* Ensure the block magic matches the btree type we're looking for. */ + btblock = XFS_BUF_TO_BLOCK(bp); + ASSERT(fab->buf_ops->magic[1] != 0); + if (btblock->bb_magic != fab->buf_ops->magic[1]) + goto out; + + /* + * If the buffer already has ops applied and they're not the ones for + * this btree type, we know this block doesn't match the btree and we + * can bail out. + * + * If the buffer ops match ours, someone else has already validated + * the block for us, so we can move on to checking if this is a root + * block candidate. + * + * If the buffer does not have ops, nobody has successfully validated + * the contents and the buffer cannot be dirty. If the magic, uuid, + * and structure match this btree type then we'll move on to checking + * if it's a root block candidate. If there is no match, bail out. + */ + if (bp->b_ops) { + if (bp->b_ops != fab->buf_ops) + goto out; + } else { + ASSERT(!xfs_trans_buf_is_dirty(bp)); + if (!uuid_equal(&btblock->bb_u.s.bb_uuid, + &mp->m_sb.sb_meta_uuid)) + goto out; + /* + * Read verifiers can reference b_ops, so we set the pointer + * here. If the verifier fails we'll reset the buffer state + * to what it was before we touched the buffer. + */ + bp->b_ops = fab->buf_ops; + fab->buf_ops->verify_read(bp); + if (bp->b_error) { + bp->b_ops = NULL; + bp->b_error = 0; + goto out; + } + + /* + * Some read verifiers will (re)set b_ops, so we must be + * careful not to change b_ops after running the verifier. + */ + } + + /* + * This block passes the magic/uuid and verifier tests for this btree + * type. We don't need the caller to try the other tree types. + */ + *done_with_block = true; + + /* + * Compare this btree block's level to the height of the current + * candidate root block. + * + * If the level matches the root we found previously, throw away both + * blocks because there can't be two candidate roots. + * + * If level is lower in the tree than the root we found previously, + * ignore this block. + */ + block_level = xfs_btree_get_level(btblock); + if (block_level + 1 == fab->height) { + fab->root = NULLAGBLOCK; + goto out; + } else if (block_level < fab->height) { + goto out; + } + + /* + * This is the highest block in the tree that we've found so far. + * Update the btree height to reflect what we've learned from this + * block. + */ + fab->height = block_level + 1; + + /* + * If this block doesn't have sibling pointers, then it's the new root + * block candidate. Otherwise, the root will be found farther up the + * tree. + */ + if (btblock->bb_u.s.bb_leftsib == cpu_to_be32(NULLAGBLOCK) && + btblock->bb_u.s.bb_rightsib == cpu_to_be32(NULLAGBLOCK)) + fab->root = agbno; + else + fab->root = NULLAGBLOCK; + + trace_xrep_findroot_block(ri->sc->sa.pag, agbno, + be32_to_cpu(btblock->bb_magic), fab->height - 1); +out: + xfs_trans_brelse(ri->sc->tp, bp); + return error; +} + +/* + * Do any of the blocks in this rmap record match one of the btrees we're + * looking for? + */ +STATIC int +xrep_findroot_rmap( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv) +{ + struct xrep_findroot *ri = priv; + struct xrep_find_ag_btree *fab; + xfs_agblock_t b; + bool done; + int error = 0; + + /* Ignore anything that isn't AG metadata. */ + if (!XFS_RMAP_NON_INODE_OWNER(rec->rm_owner)) + return 0; + + /* Otherwise scan each block + btree type. */ + for (b = 0; b < rec->rm_blockcount; b++) { + done = false; + for (fab = ri->btree_info; fab->buf_ops; fab++) { + if (rec->rm_owner != fab->rmap_owner) + continue; + error = xrep_findroot_block(ri, fab, + rec->rm_owner, rec->rm_startblock + b, + &done); + if (error) + return error; + if (done) + break; + } + } + + return 0; +} + +/* Find the roots of the per-AG btrees described in btree_info. */ +int +xrep_find_ag_btree_roots( + struct xfs_scrub *sc, + struct xfs_buf *agf_bp, + struct xrep_find_ag_btree *btree_info, + struct xfs_buf *agfl_bp) +{ + struct xfs_mount *mp = sc->mp; + struct xrep_findroot ri; + struct xrep_find_ag_btree *fab; + struct xfs_btree_cur *cur; + int error; + + ASSERT(xfs_buf_islocked(agf_bp)); + ASSERT(agfl_bp == NULL || xfs_buf_islocked(agfl_bp)); + + ri.sc = sc; + ri.btree_info = btree_info; + ri.agf = agf_bp->b_addr; + ri.agfl_bp = agfl_bp; + for (fab = btree_info; fab->buf_ops; fab++) { + ASSERT(agfl_bp || fab->rmap_owner != XFS_RMAP_OWN_AG); + ASSERT(XFS_RMAP_NON_INODE_OWNER(fab->rmap_owner)); + fab->root = NULLAGBLOCK; + fab->height = 0; + } + + cur = xfs_rmapbt_init_cursor(mp, sc->tp, agf_bp, sc->sa.pag); + error = xfs_rmap_query_all(cur, xrep_findroot_rmap, &ri); + xfs_btree_del_cursor(cur, error); + + return error; +} + +#ifdef CONFIG_XFS_QUOTA +/* Update some quota flags in the superblock. */ +void +xrep_update_qflags( + struct xfs_scrub *sc, + unsigned int clear_flags, + unsigned int set_flags) +{ + struct xfs_mount *mp = sc->mp; + struct xfs_buf *bp; + + mutex_lock(&mp->m_quotainfo->qi_quotaofflock); + if ((mp->m_qflags & clear_flags) == 0 && + (mp->m_qflags & set_flags) == set_flags) + goto no_update; + + mp->m_qflags &= ~clear_flags; + mp->m_qflags |= set_flags; + + spin_lock(&mp->m_sb_lock); + mp->m_sb.sb_qflags &= ~clear_flags; + mp->m_sb.sb_qflags |= set_flags; + spin_unlock(&mp->m_sb_lock); + + /* + * Update the quota flags in the ondisk superblock without touching + * the summary counters. We have not quiesced inode chunk allocation, + * so we cannot coordinate with updates to the icount and ifree percpu + * counters. + */ + bp = xfs_trans_getsb(sc->tp); + xfs_sb_to_disk(bp->b_addr, &mp->m_sb); + xfs_trans_buf_set_type(sc->tp, bp, XFS_BLFT_SB_BUF); + xfs_trans_log_buf(sc->tp, bp, 0, sizeof(struct xfs_dsb) - 1); + +no_update: + mutex_unlock(&mp->m_quotainfo->qi_quotaofflock); +} + +/* Force a quotacheck the next time we mount. */ +void +xrep_force_quotacheck( + struct xfs_scrub *sc, + xfs_dqtype_t type) +{ + uint flag; + + flag = xfs_quota_chkd_flag(type); + if (!(flag & sc->mp->m_qflags)) + return; + + xrep_update_qflags(sc, flag, 0); +} + +/* + * Attach dquots to this inode, or schedule quotacheck to fix them. + * + * This function ensures that the appropriate dquots are attached to an inode. + * We cannot allow the dquot code to allocate an on-disk dquot block here + * because we're already in transaction context. The on-disk dquot should + * already exist anyway. If the quota code signals corruption or missing quota + * information, schedule quotacheck, which will repair corruptions in the quota + * metadata. + */ +int +xrep_ino_dqattach( + struct xfs_scrub *sc) +{ + int error; + + ASSERT(sc->tp != NULL); + ASSERT(sc->ip != NULL); + + error = xfs_qm_dqattach(sc->ip); + switch (error) { + case -EFSBADCRC: + case -EFSCORRUPTED: + case -ENOENT: + xfs_err_ratelimited(sc->mp, +"inode %llu repair encountered quota error %d, quotacheck forced.", + (unsigned long long)I_INO(sc->ip), error); + if (XFS_IS_UQUOTA_ON(sc->mp) && !sc->ip->i_udquot) + xrep_force_quotacheck(sc, XFS_DQTYPE_USER); + if (XFS_IS_GQUOTA_ON(sc->mp) && !sc->ip->i_gdquot) + xrep_force_quotacheck(sc, XFS_DQTYPE_GROUP); + if (XFS_IS_PQUOTA_ON(sc->mp) && !sc->ip->i_pdquot) + xrep_force_quotacheck(sc, XFS_DQTYPE_PROJ); + fallthrough; + case -ESRCH: + error = 0; + break; + default: + break; + } + + return error; +} +#endif /* CONFIG_XFS_QUOTA */ + +/* + * Ensure that the inode being repaired is ready to handle a certain number of + * extents, or return EFSCORRUPTED. Caller must hold the ILOCK of the inode + * being repaired and have joined it to the scrub transaction. + */ +int +xrep_ino_ensure_extent_count( + struct xfs_scrub *sc, + int whichfork, + xfs_extnum_t nextents) +{ + xfs_extnum_t max_extents; + bool inode_has_nrext64; + + inode_has_nrext64 = xfs_inode_has_large_extent_counts(sc->ip); + max_extents = xfs_iext_max_nextents(inode_has_nrext64, whichfork); + if (nextents <= max_extents) + return 0; + if (inode_has_nrext64) + return -EFSCORRUPTED; + if (!xfs_has_large_extent_counts(sc->mp)) + return -EFSCORRUPTED; + + max_extents = xfs_iext_max_nextents(true, whichfork); + if (nextents > max_extents) + return -EFSCORRUPTED; + + sc->ip->i_diflags2 |= XFS_DIFLAG2_NREXT64; + xfs_trans_log_inode(sc->tp, sc->ip, XFS_ILOG_CORE); + return 0; +} + +/* + * Initialize all the btree cursors for an AG repair except for the btree that + * we're rebuilding. + */ +void +xrep_ag_btcur_init( + struct xfs_scrub *sc, + struct xchk_ag *sa) +{ + struct xfs_mount *mp = sc->mp; + + /* Set up a bnobt cursor for cross-referencing. */ + if (sc->sm->sm_type != XFS_SCRUB_TYPE_BNOBT && + sc->sm->sm_type != XFS_SCRUB_TYPE_CNTBT) { + sa->bno_cur = xfs_bnobt_init_cursor(mp, sc->tp, sa->agf_bp, + sc->sa.pag); + sa->cnt_cur = xfs_cntbt_init_cursor(mp, sc->tp, sa->agf_bp, + sc->sa.pag); + } + + /* Set up a inobt cursor for cross-referencing. */ + if (sc->sm->sm_type != XFS_SCRUB_TYPE_INOBT && + sc->sm->sm_type != XFS_SCRUB_TYPE_FINOBT) { + sa->ino_cur = xfs_inobt_init_cursor(sc->sa.pag, sc->tp, + sa->agi_bp); + if (xfs_has_finobt(mp)) + sa->fino_cur = xfs_finobt_init_cursor(sc->sa.pag, + sc->tp, sa->agi_bp); + } + + /* Set up a rmapbt cursor for cross-referencing. */ + if (sc->sm->sm_type != XFS_SCRUB_TYPE_RMAPBT && + xfs_has_rmapbt(mp)) + sa->rmap_cur = xfs_rmapbt_init_cursor(mp, sc->tp, sa->agf_bp, + sc->sa.pag); + + /* Set up a refcountbt cursor for cross-referencing. */ + if (sc->sm->sm_type != XFS_SCRUB_TYPE_REFCNTBT && + xfs_has_reflink(mp)) + sa->refc_cur = xfs_refcountbt_init_cursor(mp, sc->tp, + sa->agf_bp, sc->sa.pag); +} + +/* + * Reinitialize the in-core AG state after a repair by rereading the AGF + * buffer. We had better get the same AGF buffer as the one that's attached + * to the scrub context. + */ +int +xrep_reinit_pagf( + struct xfs_scrub *sc) +{ + struct xfs_perag *pag = sc->sa.pag; + struct xfs_buf *bp; + int error; + + ASSERT(pag); + ASSERT(xfs_perag_initialised_agf(pag)); + + clear_bit(XFS_AGSTATE_AGF_INIT, &pag->pag_opstate); + error = xfs_alloc_read_agf(pag, sc->tp, 0, &bp); + if (error) + return error; + + if (bp != sc->sa.agf_bp) { + ASSERT(bp == sc->sa.agf_bp); + return -EFSCORRUPTED; + } + + return 0; +} + +/* + * Reinitialize the in-core AG state after a repair by rereading the AGI + * buffer. We had better get the same AGI buffer as the one that's attached + * to the scrub context. + */ +int +xrep_reinit_pagi( + struct xfs_scrub *sc) +{ + struct xfs_perag *pag = sc->sa.pag; + struct xfs_buf *bp; + int error; + + ASSERT(pag); + ASSERT(xfs_perag_initialised_agi(pag)); + + clear_bit(XFS_AGSTATE_AGI_INIT, &pag->pag_opstate); + error = xfs_ialloc_read_agi(pag, sc->tp, 0, &bp); + if (error) + return error; + + if (bp != sc->sa.agi_bp) { + ASSERT(bp == sc->sa.agi_bp); + return -EFSCORRUPTED; + } + + return 0; +} + +/* + * Given an active reference to a perag structure, load AG headers and cursors. + * This should only be called to scan an AG while repairing file-based metadata. + */ +int +xrep_ag_init( + struct xfs_scrub *sc, + struct xfs_perag *pag, + struct xchk_ag *sa) +{ + int error; + + ASSERT(!sa->pag); + + error = xfs_ialloc_read_agi(pag, sc->tp, 0, &sa->agi_bp); + if (error) + return error; + + error = xfs_alloc_read_agf(pag, sc->tp, 0, &sa->agf_bp); + if (error) + return error; + + /* Grab our own passive reference from the caller's ref. */ + sa->pag = xfs_perag_hold(pag); + xrep_ag_btcur_init(sc, sa); + return 0; +} + +#ifdef CONFIG_XFS_RT +/* Initialize all the btree cursors for a RT repair. */ +void +xrep_rtgroup_btcur_init( + struct xfs_scrub *sc, + struct xchk_rt *sr) +{ + struct xfs_mount *mp = sc->mp; + + ASSERT(sr->rtg != NULL); + + if (sc->sm->sm_type != XFS_SCRUB_TYPE_RTRMAPBT && + (sr->rtlock_flags & XFS_RTGLOCK_RMAP) && + xfs_has_rtrmapbt(mp)) + sr->rmap_cur = xfs_rtrmapbt_init_cursor(sc->tp, sr->rtg); + + if (sc->sm->sm_type != XFS_SCRUB_TYPE_RTREFCBT && + (sr->rtlock_flags & XFS_RTGLOCK_REFCOUNT) && + xfs_has_rtreflink(mp)) + sr->refc_cur = xfs_rtrefcountbt_init_cursor(sc->tp, sr->rtg); +} + +/* + * Given a reference to a rtgroup structure, lock rtgroup btree inodes and + * create btree cursors. Must only be called to repair a regular rt file. + */ +int +xrep_rtgroup_init( + struct xfs_scrub *sc, + struct xfs_rtgroup *rtg, + struct xchk_rt *sr, + unsigned int rtglock_flags) +{ + ASSERT(sr->rtg == NULL); + + xfs_rtgroup_lock(rtg, rtglock_flags); + sr->rtlock_flags = rtglock_flags; + + /* Grab our own passive reference from the caller's ref. */ + sr->rtg = xfs_rtgroup_hold(rtg); + xrep_rtgroup_btcur_init(sc, sr); + return 0; +} + +/* Ensure that all rt blocks in the given range are not marked free. */ +int +xrep_require_rtext_inuse( + struct xfs_scrub *sc, + xfs_rgblock_t rgbno, + xfs_filblks_t len) +{ + struct xfs_mount *mp = sc->mp; + xfs_rtxnum_t startrtx; + xfs_rtxnum_t endrtx; + bool is_free = false; + int error = 0; + + if (xfs_has_zoned(mp)) { + if (!xfs_zone_rgbno_is_valid(sc->sr.rtg, rgbno + len - 1)) + return -EFSCORRUPTED; + return 0; + } + + startrtx = xfs_rgbno_to_rtx(mp, rgbno); + endrtx = xfs_rgbno_to_rtx(mp, rgbno + len - 1); + + error = xfs_rtalloc_extent_is_free(sc->sr.rtg, sc->tp, startrtx, + endrtx - startrtx + 1, &is_free); + if (error) + return error; + if (is_free) + return -EFSCORRUPTED; + + return 0; +} +#endif /* CONFIG_XFS_RT */ + +/* Reinitialize the per-AG block reservation for the AG we just fixed. */ +int +xrep_reset_perag_resv( + struct xfs_scrub *sc) +{ + int error; + + if (!(sc->flags & XREP_RESET_PERAG_RESV)) + return 0; + + ASSERT(sc->sa.pag != NULL); + ASSERT(sc->ops->type == ST_PERAG); + ASSERT(sc->tp); + + sc->flags &= ~XREP_RESET_PERAG_RESV; + xfs_ag_resv_free(sc->sa.pag); + error = xfs_ag_resv_init(sc->sa.pag, sc->tp); + if (error == -ENOSPC) { + xfs_err(sc->mp, +"Insufficient free space to reset per-AG reservation for AG %u after repair.", + pag_agno(sc->sa.pag)); + error = 0; + } + + return error; +} + +/* Decide if we are going to call the repair function for a scrub type. */ +bool +xrep_will_attempt( + struct xfs_scrub *sc) +{ + /* Userspace asked us to rebuild the structure regardless. */ + if (sc->sm->sm_flags & XFS_SCRUB_IFLAG_FORCE_REBUILD) + return true; + + /* Let debug users force us into the repair routines. */ + if (XFS_TEST_ERROR(sc->mp, XFS_ERRTAG_FORCE_SCRUB_REPAIR)) + return true; + + /* Metadata is corrupt or failed cross-referencing. */ + if (xchk_needs_repair(sc->sm)) + return true; + + return false; +} + +/* Try to fix some part of a metadata inode by calling another scrubber. */ +STATIC int +xrep_metadata_inode_subtype( + struct xfs_scrub *sc, + unsigned int scrub_type) +{ + struct xfs_scrub_subord *sub; + int error; + + /* + * Let's see if the inode needs repair. Use a subordinate scrub context + * to call the scrub and repair functions so that we can hang on to the + * resources that we already acquired instead of using the standard + * setup/teardown routines. + */ + sub = xchk_scrub_create_subord(sc, scrub_type); + if (!sub) + return -ENOMEM; + + error = sub->sc.ops->scrub(&sub->sc); + if (error) + goto out; + if (!xrep_will_attempt(&sub->sc)) + goto out; + + /* + * Repair some part of the inode. This will potentially join the inode + * to the transaction. + */ + error = sub->sc.ops->repair(&sub->sc); + if (error) + goto out; + + /* + * Finish all deferred intent items and then roll the transaction so + * that the inode will not be joined to the transaction when we exit + * the function. + */ + error = xfs_defer_finish(&sub->sc.tp); + if (error) + goto out; + error = xfs_trans_roll(&sub->sc.tp); + if (error) + goto out; + + /* + * Clear the corruption flags and re-check the metadata that we just + * repaired. + */ + sub->sc.sm->sm_flags &= ~XFS_SCRUB_FLAGS_OUT; + error = sub->sc.ops->scrub(&sub->sc); + if (error) + goto out; + + /* If corruption persists, the repair has failed. */ + if (xchk_needs_repair(sub->sc.sm)) { + error = -EFSCORRUPTED; + goto out; + } +out: + xchk_scrub_free_subord(sub); + return error; +} + +/* + * Repair the ondisk forks of a metadata inode. The caller must ensure that + * sc->ip points to the metadata inode and the ILOCK is held on that inode. + * The inode must not be joined to the transaction before the call, and will + * not be afterwards. + */ +int +xrep_metadata_inode_forks( + struct xfs_scrub *sc) +{ + bool dirty = false; + int error; + + /* Repair the inode record and the data fork. */ + error = xrep_metadata_inode_subtype(sc, XFS_SCRUB_TYPE_INODE); + if (error) + return error; + + error = xrep_metadata_inode_subtype(sc, XFS_SCRUB_TYPE_BMBTD); + if (error) + return error; + + /* + * Metadata files can only have extended attributes on metadir + * filesystems, either for parent pointers or for actual xattr data. + * For a non-metadir filesystem, make sure the attr fork looks ok + * before we delete it. + */ + if (xfs_inode_hasattr(sc->ip)) { + error = xrep_metadata_inode_subtype(sc, XFS_SCRUB_TYPE_BMBTA); + if (error) + return error; + } + + /* Clear the reflink flag since metadata never shares. */ + if (xfs_is_reflink_inode(sc->ip)) { + dirty = true; + xfs_trans_ijoin(sc->tp, sc->ip, 0); + error = xfs_reflink_clear_inode_flag(sc->ip, &sc->tp); + if (error) + return error; + } + + /* + * Metadata files on non-metadir filesystems cannot have attr forks, + * so clear them now. + */ + if (xfs_inode_hasattr(sc->ip) && !xfs_has_metadir(sc->mp)) { + if (!dirty) { + dirty = true; + xfs_trans_ijoin(sc->tp, sc->ip, 0); + } + error = xrep_xattr_reset_fork(sc); + if (error) + return error; + } + + /* + * If we modified the inode, roll the transaction but don't rejoin the + * inode to the new transaction because xrep_bmap_data can do that. + */ + if (dirty) { + error = xfs_trans_roll(&sc->tp); + if (error) + return error; + dirty = false; + } + + return 0; +} + +/* + * Set up an in-memory buffer cache so that we can use the xfbtree. Allocating + * a shmem file might take loks, so we cannot be in transaction context. Park + * our resources in the scrub context and let the teardown function take care + * of them at the right time. + */ +int +xrep_setup_xfbtree( + struct xfs_scrub *sc, + const char *descr) +{ + ASSERT(sc->tp == NULL); + + return xmbuf_alloc(sc->mp, descr, &sc->xmbtp); +} + +/* + * See if this buffer can pass the given ->verify_struct() function. + * + * If the buffer already has ops attached and they're not the ones that were + * passed in, we reject the buffer. Otherwise, we perform the structure test + * (note that we do not check CRCs) and return the outcome of the test. The + * buffer ops and error state are left unchanged. + */ +bool +xrep_buf_verify_struct( + struct xfs_buf *bp, + const struct xfs_buf_ops *ops) +{ + const struct xfs_buf_ops *old_ops = bp->b_ops; + xfs_failaddr_t fa; + int old_error; + + if (old_ops) { + if (old_ops != ops) + return false; + } + + old_error = bp->b_error; + bp->b_ops = ops; + fa = bp->b_ops->verify_struct(bp); + bp->b_ops = old_ops; + bp->b_error = old_error; + + return fa == NULL; +} + +/* Check the sanity of a rmap record for a metadata btree inode. */ +int +xrep_check_ino_btree_mapping( + struct xfs_scrub *sc, + const struct xfs_rmap_irec *rec) +{ + enum xbtree_recpacking outcome; + int error; + + /* + * Metadata btree inodes never have extended attributes, and all blocks + * should have the bmbt block flag set. + */ + if ((rec->rm_flags & XFS_RMAP_ATTR_FORK) || + !(rec->rm_flags & XFS_RMAP_BMBT_BLOCK)) + return -EFSCORRUPTED; + + /* Make sure the block is within the AG. */ + if (!xfs_verify_agbext(sc->sa.pag, rec->rm_startblock, + rec->rm_blockcount)) + return -EFSCORRUPTED; + + /* Make sure this isn't free space. */ + error = xfs_alloc_has_records(sc->sa.bno_cur, rec->rm_startblock, + rec->rm_blockcount, &outcome); + if (error) + return error; + if (outcome != XBTREE_RECPACKING_EMPTY) + return -EFSCORRUPTED; + + return 0; +} + +/* + * Reset the block count of the inode being repaired, and adjust the dquot + * block usage to match. The inode must not have an xattr fork. + */ +void +xrep_inode_set_nblocks( + struct xfs_scrub *sc, + int64_t new_blocks) +{ + int64_t delta = + new_blocks - sc->ip->i_nblocks; + + sc->ip->i_nblocks = new_blocks; + + xfs_trans_log_inode(sc->tp, sc->ip, XFS_ILOG_CORE); + if (delta != 0) + xfs_trans_mod_dquot_byino(sc->tp, sc->ip, XFS_TRANS_DQ_BCOUNT, + delta); +} + +/* Reset the block reservation for a metadata inode. */ +int +xrep_reset_metafile_resv( + struct xfs_scrub *sc) +{ + struct xfs_mount *mp = sc->mp; + int64_t delta; + int error; + + delta = mp->m_metafile_resv_used + mp->m_metafile_resv_avail - + mp->m_metafile_resv_target; + if (delta == 0) + return 0; + + /* + * Too many blocks have been reserved, transfer some from the incore + * reservation back to the filesystem. + */ + if (delta > 0) { + int64_t give_back; + + give_back = min_t(uint64_t, delta, mp->m_metafile_resv_avail); + if (give_back > 0) { + xfs_mod_sb_delalloc(mp, -give_back); + xfs_add_fdblocks(mp, give_back); + mp->m_metafile_resv_avail -= give_back; + } + + return 0; + } + + /* + * Not enough reservation; try to take some blocks from the filesystem + * to the metabtree reservation. + */ + delta = -delta; /* delta is negative here, so invert the sign. */ + error = xfs_dec_fdblocks(mp, delta, true); + while (error == -ENOSPC) { + delta--; + if (delta == 0) { + xfs_warn(sc->mp, +"Insufficient free space to reset metabtree reservation after repair."); + return 0; + } + error = xfs_dec_fdblocks(mp, delta, true); + } + if (error) + return error; + + xfs_mod_sb_delalloc(mp, delta); + mp->m_metafile_resv_avail += delta; + return 0; +} diff --git a/fs/xfs/scrub/repair.h b/fs/xfs/scrub/repair.h new file mode 100644 index 000000000..2bb125c4f --- /dev/null +++ b/fs/xfs/scrub/repair.h @@ -0,0 +1,283 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_REPAIR_H__ +#define __XFS_SCRUB_REPAIR_H__ + +#include "xfs_quota_defs.h" + +struct xfs_rtgroup; +struct xchk_stats_run; + +static inline int xrep_notsupported(struct xfs_scrub *sc) +{ + return -EOPNOTSUPP; +} + +#ifdef CONFIG_XFS_ONLINE_REPAIR + +/* Repair helpers */ + +int xrep_attempt(struct xfs_scrub *sc, struct xchk_stats_run *run); +bool xrep_will_attempt(struct xfs_scrub *sc); +void xrep_failure(struct xfs_mount *mp); +int xrep_roll_ag_trans(struct xfs_scrub *sc); +int xrep_roll_trans(struct xfs_scrub *sc); +int xrep_defer_finish(struct xfs_scrub *sc); +bool xrep_ag_has_space(struct xfs_perag *pag, xfs_extlen_t nr_blocks, + enum xfs_ag_resv_type type); +xfs_extlen_t xrep_calc_ag_resblks(struct xfs_scrub *sc); + +static inline int +xrep_trans_commit( + struct xfs_scrub *sc) +{ + int error = xfs_trans_commit(sc->tp); + + sc->tp = NULL; + return error; +} + +struct xbitmap; +struct xagb_bitmap; +struct xrgb_bitmap; +struct xfsb_bitmap; +struct xrtb_bitmap; + +int xrep_fix_freelist(struct xfs_scrub *sc, int alloc_flags); + +struct xrep_find_ag_btree { + /* in: rmap owner of the btree we're looking for */ + uint64_t rmap_owner; + + /* in: buffer ops */ + const struct xfs_buf_ops *buf_ops; + + /* in: maximum btree height */ + unsigned int maxlevels; + + /* out: the highest btree block found and the tree height */ + xfs_agblock_t root; + unsigned int height; +}; + +int xrep_find_ag_btree_roots(struct xfs_scrub *sc, struct xfs_buf *agf_bp, + struct xrep_find_ag_btree *btree_info, struct xfs_buf *agfl_bp); + +#ifdef CONFIG_XFS_QUOTA +void xrep_update_qflags(struct xfs_scrub *sc, unsigned int clear_flags, + unsigned int set_flags); +void xrep_force_quotacheck(struct xfs_scrub *sc, xfs_dqtype_t type); +int xrep_ino_dqattach(struct xfs_scrub *sc); +#else +# define xrep_force_quotacheck(sc, type) ((void)0) +# define xrep_ino_dqattach(sc) (0) +#endif /* CONFIG_XFS_QUOTA */ + +int xrep_setup_xfbtree(struct xfs_scrub *sc, const char *descr); + +int xrep_ino_ensure_extent_count(struct xfs_scrub *sc, int whichfork, + xfs_extnum_t nextents); +int xrep_reset_perag_resv(struct xfs_scrub *sc); +int xrep_bmap(struct xfs_scrub *sc, int whichfork, bool allow_unwritten); +int xrep_metadata_inode_forks(struct xfs_scrub *sc); +int xrep_setup_ag_rmapbt(struct xfs_scrub *sc); +int xrep_setup_ag_refcountbt(struct xfs_scrub *sc); +int xrep_setup_xattr(struct xfs_scrub *sc); +int xrep_setup_directory(struct xfs_scrub *sc); +int xrep_setup_parent(struct xfs_scrub *sc); +int xrep_setup_nlinks(struct xfs_scrub *sc); +int xrep_setup_symlink(struct xfs_scrub *sc, unsigned int *resblks); +int xrep_setup_dirtree(struct xfs_scrub *sc); +int xrep_setup_rtrmapbt(struct xfs_scrub *sc); +int xrep_setup_rtrefcountbt(struct xfs_scrub *sc); + +/* Repair setup functions */ +int xrep_setup_ag_allocbt(struct xfs_scrub *sc); + +struct xfs_imap; +int xrep_setup_inode(struct xfs_scrub *sc, const struct xfs_imap *imap); + +void xrep_ag_btcur_init(struct xfs_scrub *sc, struct xchk_ag *sa); +int xrep_ag_init(struct xfs_scrub *sc, struct xfs_perag *pag, + struct xchk_ag *sa); +#ifdef CONFIG_XFS_RT +int xrep_rtgroup_init(struct xfs_scrub *sc, struct xfs_rtgroup *rtg, + struct xchk_rt *sr, unsigned int rtglock_flags); +void xrep_rtgroup_btcur_init(struct xfs_scrub *sc, struct xchk_rt *sr); +int xrep_require_rtext_inuse(struct xfs_scrub *sc, xfs_rgblock_t rgbno, + xfs_filblks_t len); +xfs_extlen_t xrep_calc_rtgroup_resblks(struct xfs_scrub *sc); +#else +# define xrep_rtgroup_init(sc, rtg, sr, lockflags) (-ENOSYS) +# define xrep_calc_rtgroup_resblks(sc) (0) +#endif /* CONFIG_XFS_RT */ + +int xrep_check_ino_btree_mapping(struct xfs_scrub *sc, + const struct xfs_rmap_irec *rec); + +/* Metadata revalidators */ + +int xrep_revalidate_allocbt(struct xfs_scrub *sc); +int xrep_revalidate_iallocbt(struct xfs_scrub *sc); + +/* Metadata repairers */ + +int xrep_probe(struct xfs_scrub *sc); +int xrep_superblock(struct xfs_scrub *sc); +int xrep_agf(struct xfs_scrub *sc); +int xrep_agfl(struct xfs_scrub *sc); +int xrep_agi(struct xfs_scrub *sc); +int xrep_allocbt(struct xfs_scrub *sc); +int xrep_iallocbt(struct xfs_scrub *sc); +int xrep_rmapbt(struct xfs_scrub *sc); +int xrep_refcountbt(struct xfs_scrub *sc); +int xrep_inode(struct xfs_scrub *sc); +int xrep_bmap_data(struct xfs_scrub *sc); +int xrep_bmap_attr(struct xfs_scrub *sc); +int xrep_bmap_cow(struct xfs_scrub *sc); +int xrep_nlinks(struct xfs_scrub *sc); +int xrep_fscounters(struct xfs_scrub *sc); +int xrep_xattr(struct xfs_scrub *sc); +int xrep_directory(struct xfs_scrub *sc); +int xrep_parent(struct xfs_scrub *sc); +int xrep_symlink(struct xfs_scrub *sc); +int xrep_dirtree(struct xfs_scrub *sc); +int xrep_metapath(struct xfs_scrub *sc); + +#ifdef CONFIG_XFS_RT +int xrep_rtbitmap(struct xfs_scrub *sc); +int xrep_rtsummary(struct xfs_scrub *sc); +int xrep_rgsuperblock(struct xfs_scrub *sc); +int xrep_rtrmapbt(struct xfs_scrub *sc); +int xrep_rtrefcountbt(struct xfs_scrub *sc); +#else +# define xrep_rtbitmap xrep_notsupported +# define xrep_rtsummary xrep_notsupported +# define xrep_rgsuperblock xrep_notsupported +# define xrep_rtrmapbt xrep_notsupported +# define xrep_rtrefcountbt xrep_notsupported +#endif /* CONFIG_XFS_RT */ + +#ifdef CONFIG_XFS_QUOTA +int xrep_quota(struct xfs_scrub *sc); +int xrep_quotacheck(struct xfs_scrub *sc); +#else +# define xrep_quota xrep_notsupported +# define xrep_quotacheck xrep_notsupported +#endif /* CONFIG_XFS_QUOTA */ + +int xrep_reinit_pagf(struct xfs_scrub *sc); +int xrep_reinit_pagi(struct xfs_scrub *sc); + +bool xrep_buf_verify_struct(struct xfs_buf *bp, const struct xfs_buf_ops *ops); +void xrep_inode_set_nblocks(struct xfs_scrub *sc, int64_t new_blocks); +int xrep_reset_metafile_resv(struct xfs_scrub *sc); + +#else + +#define xrep_ino_dqattach(sc) (0) + +/* + * When online repair is not built into the kernel, we still want to attempt + * the repair so that the stub xrep_attempt below will return EOPNOTSUPP. + */ +static inline bool xrep_will_attempt(const struct xfs_scrub *sc) +{ + return (sc->sm->sm_flags & XFS_SCRUB_IFLAG_FORCE_REBUILD) || + xchk_needs_repair(sc->sm); +} + +static inline int +xrep_attempt( + struct xfs_scrub *sc, + struct xchk_stats_run *run) +{ + return -EOPNOTSUPP; +} + +static inline void xrep_failure(struct xfs_mount *mp) {} + +static inline xfs_extlen_t +xrep_calc_ag_resblks( + struct xfs_scrub *sc) +{ + return 0; +} + +#define xrep_calc_rtgroup_resblks xrep_calc_ag_resblks + +static inline int +xrep_reset_perag_resv( + struct xfs_scrub *sc) +{ + if (!(sc->flags & XREP_RESET_PERAG_RESV)) + return 0; + + ASSERT(0); + return -EOPNOTSUPP; +} + +/* repair setup functions for no-repair */ +static inline int +xrep_setup_nothing( + struct xfs_scrub *sc) +{ + return 0; +} +#define xrep_setup_ag_allocbt xrep_setup_nothing +#define xrep_setup_ag_rmapbt xrep_setup_nothing +#define xrep_setup_ag_refcountbt xrep_setup_nothing +#define xrep_setup_xattr xrep_setup_nothing +#define xrep_setup_directory xrep_setup_nothing +#define xrep_setup_parent xrep_setup_nothing +#define xrep_setup_nlinks xrep_setup_nothing +#define xrep_setup_dirtree xrep_setup_nothing +#define xrep_setup_metapath xrep_setup_nothing +#define xrep_setup_rtrmapbt xrep_setup_nothing +#define xrep_setup_rtrefcountbt xrep_setup_nothing + +#define xrep_setup_inode(sc, imap) ((void)0) + +static inline int xrep_setup_symlink(struct xfs_scrub *sc, unsigned int *x) +{ + return 0; +} + +#define xrep_revalidate_allocbt (NULL) +#define xrep_revalidate_iallocbt (NULL) + +#define xrep_probe xrep_notsupported +#define xrep_superblock xrep_notsupported +#define xrep_agf xrep_notsupported +#define xrep_agfl xrep_notsupported +#define xrep_agi xrep_notsupported +#define xrep_allocbt xrep_notsupported +#define xrep_iallocbt xrep_notsupported +#define xrep_rmapbt xrep_notsupported +#define xrep_refcountbt xrep_notsupported +#define xrep_inode xrep_notsupported +#define xrep_bmap_data xrep_notsupported +#define xrep_bmap_attr xrep_notsupported +#define xrep_bmap_cow xrep_notsupported +#define xrep_rtbitmap xrep_notsupported +#define xrep_quota xrep_notsupported +#define xrep_quotacheck xrep_notsupported +#define xrep_nlinks xrep_notsupported +#define xrep_fscounters xrep_notsupported +#define xrep_rtsummary xrep_notsupported +#define xrep_xattr xrep_notsupported +#define xrep_directory xrep_notsupported +#define xrep_parent xrep_notsupported +#define xrep_symlink xrep_notsupported +#define xrep_dirtree xrep_notsupported +#define xrep_metapath xrep_notsupported +#define xrep_rgsuperblock xrep_notsupported +#define xrep_rtrmapbt xrep_notsupported +#define xrep_rtrefcountbt xrep_notsupported + +#endif /* CONFIG_XFS_ONLINE_REPAIR */ + +#endif /* __XFS_SCRUB_REPAIR_H__ */ diff --git a/fs/xfs/scrub/rgb_bitmap.h b/fs/xfs/scrub/rgb_bitmap.h new file mode 100644 index 000000000..4c3126b66 --- /dev/null +++ b/fs/xfs/scrub/rgb_bitmap.h @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_RGB_BITMAP_H__ +#define __XFS_SCRUB_RGB_BITMAP_H__ + +/* Bitmaps, but for type-checked for xfs_rgblock_t */ + +struct xrgb_bitmap { + struct xbitmap32 rgbitmap; +}; + +static inline void xrgb_bitmap_init(struct xrgb_bitmap *bitmap) +{ + xbitmap32_init(&bitmap->rgbitmap); +} + +static inline void xrgb_bitmap_destroy(struct xrgb_bitmap *bitmap) +{ + xbitmap32_destroy(&bitmap->rgbitmap); +} + +static inline int xrgb_bitmap_set(struct xrgb_bitmap *bitmap, + xfs_rgblock_t start, xfs_extlen_t len) +{ + return xbitmap32_set(&bitmap->rgbitmap, start, len); +} + +static inline int xrgb_bitmap_walk(struct xrgb_bitmap *bitmap, + xbitmap32_walk_fn fn, void *priv) +{ + return xbitmap32_walk(&bitmap->rgbitmap, fn, priv); +} + +#endif /* __XFS_SCRUB_RGB_BITMAP_H__ */ diff --git a/fs/xfs/scrub/rgsuper.c b/fs/xfs/scrub/rgsuper.c new file mode 100644 index 000000000..2bd2c0351 --- /dev/null +++ b/fs/xfs/scrub/rgsuper.c @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2022-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_rtgroup.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_sb.h" +#include "xfs_rmap.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/repair.h" + +/* Set us up with a transaction and an empty context. */ +int +xchk_setup_rgsuperblock( + struct xfs_scrub *sc) +{ + if (xchk_need_intent_drain(sc)) + xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN); + return xchk_trans_alloc(sc, 0); +} + +/* Cross-reference with the other rt metadata. */ +STATIC void +xchk_rgsuperblock_xref( + struct xfs_scrub *sc) +{ + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return; + + xchk_xref_is_used_rt_space(sc, xfs_rgbno_to_rtb(sc->sr.rtg, 0), 1); + xchk_xref_is_only_rt_owned_by(sc, 0, 1, &XFS_RMAP_OINFO_FS); +} + +int +xchk_rgsuperblock( + struct xfs_scrub *sc) +{ + xfs_rgnumber_t rgno = sc->sm->sm_agno; + unsigned int flags; + int error; + + /* + * Only rtgroup 0 has a superblock. We may someday want to use higher + * rgno for other functions, similar to what we do with the primary + * super scrub function. + */ + if (rgno != 0) + return -ENOENT; + + /* + * Grab an active reference to the rtgroup structure. If we can't get + * it, we're racing with something that's tearing down the group, so + * signal that the group no longer exists. Take the rtbitmap in shared + * mode so that the group can't change while we're doing things. + */ + error = xchk_rtgroup_init_existing(sc, rgno, &sc->sr); + if (!xchk_xref_process_error(sc, 0, 0, &error)) + return error; + + if (xfs_has_rtrmapbt(sc->mp)) + flags = XFS_RTGLOCK_BITMAP | XFS_RTGLOCK_RMAP; + else + flags = XFS_RTGLOCK_BITMAP_SHARED; + + error = xchk_rtgroup_lock(sc, &sc->sr, flags); + if (error) + return error; + + /* + * Since we already validated the rt superblock at mount time, we don't + * need to check its contents again. All we need is to cross-reference. + */ + xchk_rgsuperblock_xref(sc); + return 0; +} + +#ifdef CONFIG_XFS_ONLINE_REPAIR +int +xrep_rgsuperblock( + struct xfs_scrub *sc) +{ + struct xfs_buf *sb_bp; + + ASSERT(rtg_rgno(sc->sr.rtg) == 0); + + sb_bp = xfs_trans_getsb(sc->tp); + xfs_log_sb(sc->tp); + xfs_log_rtsb(sc->tp, sb_bp); + return 0; +} +#endif /* CONFIG_XFS_ONLINE_REPAIR */ diff --git a/fs/xfs/scrub/rmap.c b/fs/xfs/scrub/rmap.c new file mode 100644 index 000000000..0cd3eecd2 --- /dev/null +++ b/fs/xfs/scrub/rmap.c @@ -0,0 +1,648 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_trans.h" +#include "xfs_btree.h" +#include "xfs_rmap.h" +#include "xfs_refcount.h" +#include "xfs_ag.h" +#include "xfs_bit.h" +#include "xfs_alloc.h" +#include "xfs_alloc_btree.h" +#include "xfs_ialloc_btree.h" +#include "xfs_refcount_btree.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/btree.h" +#include "scrub/bitmap.h" +#include "scrub/agb_bitmap.h" +#include "scrub/repair.h" + +/* + * Set us up to scrub reverse mapping btrees. + */ +int +xchk_setup_ag_rmapbt( + struct xfs_scrub *sc) +{ + if (xchk_need_intent_drain(sc)) + xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN); + + if (xchk_could_repair(sc)) { + int error; + + error = xrep_setup_ag_rmapbt(sc); + if (error) + return error; + } + + return xchk_setup_ag_btree(sc, false); +} + +/* Reverse-mapping scrubber. */ + +struct xchk_rmap { + /* + * The furthest-reaching of the rmapbt records that we've already + * processed. This enables us to detect overlapping records for space + * allocations that cannot be shared. + */ + struct xfs_rmap_irec overlap_rec; + + /* + * The previous rmapbt record, so that we can check for two records + * that could be one. + */ + struct xfs_rmap_irec prev_rec; + + /* Bitmaps containing all blocks for each type of AG metadata. */ + struct xagb_bitmap fs_owned; + struct xagb_bitmap log_owned; + struct xagb_bitmap ag_owned; + struct xagb_bitmap inobt_owned; + struct xagb_bitmap refcbt_owned; + + /* Did we complete the AG space metadata bitmaps? */ + bool bitmaps_complete; +}; + +/* Cross-reference a rmap against the refcount btree. */ +STATIC void +xchk_rmapbt_xref_refc( + struct xfs_scrub *sc, + struct xfs_rmap_irec *irec) +{ + xfs_agblock_t fbno; + xfs_extlen_t flen; + bool non_inode; + bool is_bmbt; + bool is_attr; + bool is_unwritten; + int error; + + if (!sc->sa.refc_cur || xchk_skip_xref(sc->sm)) + return; + + non_inode = XFS_RMAP_NON_INODE_OWNER(irec->rm_owner); + is_bmbt = irec->rm_flags & XFS_RMAP_BMBT_BLOCK; + is_attr = irec->rm_flags & XFS_RMAP_ATTR_FORK; + is_unwritten = irec->rm_flags & XFS_RMAP_UNWRITTEN; + + /* If this is shared, must be a data fork extent. */ + error = xfs_refcount_find_shared(sc->sa.refc_cur, irec->rm_startblock, + irec->rm_blockcount, &fbno, &flen, false); + if (!xchk_should_check_xref(sc, &error, &sc->sa.refc_cur)) + return; + if (flen != 0 && (non_inode || is_attr || is_bmbt || is_unwritten)) + xchk_btree_xref_set_corrupt(sc, sc->sa.refc_cur, 0); +} + +/* Cross-reference with the other btrees. */ +STATIC void +xchk_rmapbt_xref( + struct xfs_scrub *sc, + struct xfs_rmap_irec *irec) +{ + xfs_agblock_t agbno = irec->rm_startblock; + xfs_extlen_t len = irec->rm_blockcount; + + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return; + + xchk_xref_is_used_space(sc, agbno, len); + if (irec->rm_owner == XFS_RMAP_OWN_INODES) + xchk_xref_is_inode_chunk(sc, agbno, len); + else + xchk_xref_is_not_inode_chunk(sc, agbno, len); + if (irec->rm_owner == XFS_RMAP_OWN_COW) + xchk_xref_is_cow_staging(sc, irec->rm_startblock, + irec->rm_blockcount); + else + xchk_rmapbt_xref_refc(sc, irec); +} + +/* + * Check for bogus UNWRITTEN flags in the rmapbt node block keys. + * + * In reverse mapping records, the file mapping extent state + * (XFS_RMAP_OFF_UNWRITTEN) is a record attribute, not a key field. It is not + * involved in lookups in any way. In older kernels, the functions that + * convert rmapbt records to keys forgot to filter out the extent state bit, + * even though the key comparison functions have filtered the flag correctly. + * If we spot an rmap key with the unwritten bit set in rm_offset, we should + * mark the btree as needing optimization to rebuild the btree without those + * flags. + */ +STATIC void +xchk_rmapbt_check_unwritten_in_keyflags( + struct xchk_btree *bs) +{ + struct xfs_scrub *sc = bs->sc; + struct xfs_btree_cur *cur = bs->cur; + struct xfs_btree_block *keyblock; + union xfs_btree_key *lkey, *hkey; + __be64 badflag = cpu_to_be64(XFS_RMAP_OFF_UNWRITTEN); + unsigned int level; + + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_PREEN) + return; + + for (level = 1; level < cur->bc_nlevels; level++) { + struct xfs_buf *bp; + unsigned int ptr; + + /* Only check the first time we've seen this node block. */ + if (cur->bc_levels[level].ptr > 1) + continue; + + keyblock = xfs_btree_get_block(cur, level, &bp); + for (ptr = 1; ptr <= be16_to_cpu(keyblock->bb_numrecs); ptr++) { + lkey = xfs_btree_key_addr(cur, ptr, keyblock); + + if (lkey->rmap.rm_offset & badflag) { + xchk_btree_set_preen(sc, cur, level); + break; + } + + hkey = xfs_btree_high_key_addr(cur, ptr, keyblock); + if (hkey->rmap.rm_offset & badflag) { + xchk_btree_set_preen(sc, cur, level); + break; + } + } + } +} + +static inline bool +xchk_rmapbt_is_shareable( + struct xfs_scrub *sc, + const struct xfs_rmap_irec *irec) +{ + if (!xfs_has_reflink(sc->mp)) + return false; + if (XFS_RMAP_NON_INODE_OWNER(irec->rm_owner)) + return false; + if (irec->rm_flags & (XFS_RMAP_BMBT_BLOCK | XFS_RMAP_ATTR_FORK | + XFS_RMAP_UNWRITTEN)) + return false; + return true; +} + +/* Flag failures for records that overlap but cannot. */ +STATIC void +xchk_rmapbt_check_overlapping( + struct xchk_btree *bs, + struct xchk_rmap *cr, + const struct xfs_rmap_irec *irec) +{ + xfs_agblock_t pnext, inext; + + if (bs->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return; + + /* No previous record? */ + if (cr->overlap_rec.rm_blockcount == 0) + goto set_prev; + + /* Do overlap_rec and irec overlap? */ + pnext = cr->overlap_rec.rm_startblock + cr->overlap_rec.rm_blockcount; + if (pnext <= irec->rm_startblock) + goto set_prev; + + /* Overlap is only allowed if both records are data fork mappings. */ + if (!xchk_rmapbt_is_shareable(bs->sc, &cr->overlap_rec) || + !xchk_rmapbt_is_shareable(bs->sc, irec)) + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + + /* Save whichever rmap record extends furthest. */ + inext = irec->rm_startblock + irec->rm_blockcount; + if (pnext > inext) + return; + +set_prev: + memcpy(&cr->overlap_rec, irec, sizeof(struct xfs_rmap_irec)); +} + +/* Decide if two reverse-mapping records can be merged. */ +static inline bool +xchk_rmap_mergeable( + struct xchk_rmap *cr, + const struct xfs_rmap_irec *r2) +{ + const struct xfs_rmap_irec *r1 = &cr->prev_rec; + + /* Ignore if prev_rec is not yet initialized. */ + if (cr->prev_rec.rm_blockcount == 0) + return false; + + if (r1->rm_owner != r2->rm_owner) + return false; + if (r1->rm_startblock + r1->rm_blockcount != r2->rm_startblock) + return false; + if ((unsigned long long)r1->rm_blockcount + r2->rm_blockcount > + XFS_RMAP_LEN_MAX) + return false; + if (XFS_RMAP_NON_INODE_OWNER(r2->rm_owner)) + return true; + /* must be an inode owner below here */ + if (r1->rm_flags != r2->rm_flags) + return false; + if (r1->rm_flags & XFS_RMAP_BMBT_BLOCK) + return true; + return r1->rm_offset + r1->rm_blockcount == r2->rm_offset; +} + +/* Flag failures for records that could be merged. */ +STATIC void +xchk_rmapbt_check_mergeable( + struct xchk_btree *bs, + struct xchk_rmap *cr, + const struct xfs_rmap_irec *irec) +{ + if (bs->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return; + + if (xchk_rmap_mergeable(cr, irec)) + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + + memcpy(&cr->prev_rec, irec, sizeof(struct xfs_rmap_irec)); +} + +/* Compare an rmap for AG metadata against the metadata walk. */ +STATIC int +xchk_rmapbt_mark_bitmap( + struct xchk_btree *bs, + struct xchk_rmap *cr, + const struct xfs_rmap_irec *irec) +{ + struct xfs_scrub *sc = bs->sc; + struct xagb_bitmap *bmp = NULL; + xfs_extlen_t fsbcount = irec->rm_blockcount; + + /* + * Skip corrupt records. It is essential that we detect records in the + * btree that cannot overlap but do, flag those as CORRUPT, and skip + * the bitmap comparison to avoid generating false XCORRUPT reports. + */ + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return 0; + + /* + * If the AG metadata walk didn't complete, there's no point in + * comparing against partial results. + */ + if (!cr->bitmaps_complete) + return 0; + + switch (irec->rm_owner) { + case XFS_RMAP_OWN_FS: + bmp = &cr->fs_owned; + break; + case XFS_RMAP_OWN_LOG: + bmp = &cr->log_owned; + break; + case XFS_RMAP_OWN_AG: + bmp = &cr->ag_owned; + break; + case XFS_RMAP_OWN_INOBT: + bmp = &cr->inobt_owned; + break; + case XFS_RMAP_OWN_REFC: + bmp = &cr->refcbt_owned; + break; + } + + if (!bmp) + return 0; + + if (xagb_bitmap_test(bmp, irec->rm_startblock, &fsbcount)) { + /* + * The start of this reverse mapping corresponds to a set + * region in the bitmap. If the mapping covers more area than + * the set region, then it covers space that wasn't found by + * the AG metadata walk. + */ + if (fsbcount < irec->rm_blockcount) + xchk_btree_xref_set_corrupt(bs->sc, + bs->sc->sa.rmap_cur, 0); + } else { + /* + * The start of this reverse mapping does not correspond to a + * completely set region in the bitmap. The region wasn't + * fully set by walking the AG metadata, so this is a + * cross-referencing corruption. + */ + xchk_btree_xref_set_corrupt(bs->sc, bs->sc->sa.rmap_cur, 0); + } + + /* Unset the region so that we can detect missing rmap records. */ + return xagb_bitmap_clear(bmp, irec->rm_startblock, irec->rm_blockcount); +} + +/* Scrub an rmapbt record. */ +STATIC int +xchk_rmapbt_rec( + struct xchk_btree *bs, + const union xfs_btree_rec *rec) +{ + struct xchk_rmap *cr = bs->private; + struct xfs_rmap_irec irec; + + if (xfs_rmap_btrec_to_irec(rec, &irec) != NULL || + xfs_rmap_check_irec(to_perag(bs->cur->bc_group), &irec) != NULL) { + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + return 0; + } + + xchk_rmapbt_check_unwritten_in_keyflags(bs); + xchk_rmapbt_check_mergeable(bs, cr, &irec); + xchk_rmapbt_check_overlapping(bs, cr, &irec); + xchk_rmapbt_xref(bs->sc, &irec); + + return xchk_rmapbt_mark_bitmap(bs, cr, &irec); +} + +/* Add an AGFL block to the rmap list. */ +STATIC int +xchk_rmapbt_walk_agfl( + struct xfs_mount *mp, + xfs_agblock_t agbno, + void *priv) +{ + struct xagb_bitmap *bitmap = priv; + + return xagb_bitmap_set(bitmap, agbno, 1); +} + +/* + * Set up bitmaps mapping all the AG metadata to compare with the rmapbt + * records. + * + * Grab our own btree cursors here if the scrub setup function didn't give us a + * btree cursor due to reports of poor health. We need to find out if the + * rmapbt disagrees with primary metadata btrees to tag the rmapbt as being + * XCORRUPT. + */ +STATIC int +xchk_rmapbt_walk_ag_metadata( + struct xfs_scrub *sc, + struct xchk_rmap *cr) +{ + struct xfs_mount *mp = sc->mp; + struct xfs_buf *agfl_bp; + struct xfs_agf *agf = sc->sa.agf_bp->b_addr; + struct xfs_btree_cur *cur; + int error; + + /* OWN_FS: AG headers */ + error = xagb_bitmap_set(&cr->fs_owned, XFS_SB_BLOCK(mp), + XFS_AGFL_BLOCK(mp) - XFS_SB_BLOCK(mp) + 1); + if (error) + goto out; + + /* OWN_LOG: Internal log */ + if (xfs_ag_contains_log(mp, pag_agno(sc->sa.pag))) { + error = xagb_bitmap_set(&cr->log_owned, + XFS_FSB_TO_AGBNO(mp, mp->m_sb.sb_logstart), + mp->m_sb.sb_logblocks); + if (error) + goto out; + } + + /* OWN_AG: bnobt, cntbt, rmapbt, and AGFL */ + cur = sc->sa.bno_cur; + if (!cur) + cur = xfs_bnobt_init_cursor(sc->mp, sc->tp, sc->sa.agf_bp, + sc->sa.pag); + error = xagb_bitmap_set_btblocks(&cr->ag_owned, cur); + if (cur != sc->sa.bno_cur) + xfs_btree_del_cursor(cur, error); + if (error) + goto out; + + cur = sc->sa.cnt_cur; + if (!cur) + cur = xfs_cntbt_init_cursor(sc->mp, sc->tp, sc->sa.agf_bp, + sc->sa.pag); + error = xagb_bitmap_set_btblocks(&cr->ag_owned, cur); + if (cur != sc->sa.cnt_cur) + xfs_btree_del_cursor(cur, error); + if (error) + goto out; + + error = xagb_bitmap_set_btblocks(&cr->ag_owned, sc->sa.rmap_cur); + if (error) + goto out; + + error = xfs_alloc_read_agfl(sc->sa.pag, sc->tp, &agfl_bp); + if (error) + goto out; + + error = xfs_agfl_walk(sc->mp, agf, agfl_bp, xchk_rmapbt_walk_agfl, + &cr->ag_owned); + xfs_trans_brelse(sc->tp, agfl_bp); + if (error) + goto out; + + /* OWN_INOBT: inobt, finobt */ + cur = sc->sa.ino_cur; + if (!cur) + cur = xfs_inobt_init_cursor(sc->sa.pag, sc->tp, sc->sa.agi_bp); + error = xagb_bitmap_set_btblocks(&cr->inobt_owned, cur); + if (cur != sc->sa.ino_cur) + xfs_btree_del_cursor(cur, error); + if (error) + goto out; + + if (xfs_has_finobt(sc->mp)) { + cur = sc->sa.fino_cur; + if (!cur) + cur = xfs_finobt_init_cursor(sc->sa.pag, sc->tp, + sc->sa.agi_bp); + error = xagb_bitmap_set_btblocks(&cr->inobt_owned, cur); + if (cur != sc->sa.fino_cur) + xfs_btree_del_cursor(cur, error); + if (error) + goto out; + } + + /* OWN_REFC: refcountbt */ + if (xfs_has_reflink(sc->mp)) { + cur = sc->sa.refc_cur; + if (!cur) + cur = xfs_refcountbt_init_cursor(sc->mp, sc->tp, + sc->sa.agf_bp, sc->sa.pag); + error = xagb_bitmap_set_btblocks(&cr->refcbt_owned, cur); + if (cur != sc->sa.refc_cur) + xfs_btree_del_cursor(cur, error); + if (error) + goto out; + } + +out: + /* + * If there's an error, set XFAIL and disable the bitmap + * cross-referencing checks, but proceed with the scrub anyway. + */ + if (error) + xchk_btree_xref_process_error(sc, sc->sa.rmap_cur, + sc->sa.rmap_cur->bc_nlevels - 1, &error); + else + cr->bitmaps_complete = true; + return 0; +} + +/* + * Check for set regions in the bitmaps; if there are any, the rmap records do + * not describe all the AG metadata. + */ +STATIC void +xchk_rmapbt_check_bitmaps( + struct xfs_scrub *sc, + struct xchk_rmap *cr) +{ + struct xfs_btree_cur *cur = sc->sa.rmap_cur; + unsigned int level; + + if (sc->sm->sm_flags & (XFS_SCRUB_OFLAG_CORRUPT | + XFS_SCRUB_OFLAG_XFAIL)) + return; + if (!cur) + return; + level = cur->bc_nlevels - 1; + + /* + * Any bitmap with bits still set indicates that the reverse mapping + * doesn't cover the entire primary structure. + */ + if (xagb_bitmap_hweight(&cr->fs_owned) != 0) + xchk_btree_xref_set_corrupt(sc, cur, level); + + if (xagb_bitmap_hweight(&cr->log_owned) != 0) + xchk_btree_xref_set_corrupt(sc, cur, level); + + if (xagb_bitmap_hweight(&cr->ag_owned) != 0) + xchk_btree_xref_set_corrupt(sc, cur, level); + + if (xagb_bitmap_hweight(&cr->inobt_owned) != 0) + xchk_btree_xref_set_corrupt(sc, cur, level); + + if (xagb_bitmap_hweight(&cr->refcbt_owned) != 0) + xchk_btree_xref_set_corrupt(sc, cur, level); +} + +/* Scrub the rmap btree for some AG. */ +int +xchk_rmapbt( + struct xfs_scrub *sc) +{ + struct xchk_rmap *cr; + int error; + + cr = kzalloc_obj(struct xchk_rmap, XCHK_GFP_FLAGS); + if (!cr) + return -ENOMEM; + + xagb_bitmap_init(&cr->fs_owned); + xagb_bitmap_init(&cr->log_owned); + xagb_bitmap_init(&cr->ag_owned); + xagb_bitmap_init(&cr->inobt_owned); + xagb_bitmap_init(&cr->refcbt_owned); + + error = xchk_rmapbt_walk_ag_metadata(sc, cr); + if (error) + goto out; + + error = xchk_btree(sc, sc->sa.rmap_cur, xchk_rmapbt_rec, + &XFS_RMAP_OINFO_AG, cr); + if (error) + goto out; + + xchk_rmapbt_check_bitmaps(sc, cr); + +out: + xagb_bitmap_destroy(&cr->refcbt_owned); + xagb_bitmap_destroy(&cr->inobt_owned); + xagb_bitmap_destroy(&cr->ag_owned); + xagb_bitmap_destroy(&cr->log_owned); + xagb_bitmap_destroy(&cr->fs_owned); + kfree(cr); + return error; +} + +/* xref check that the extent is owned only by a given owner */ +void +xchk_xref_is_only_owned_by( + struct xfs_scrub *sc, + xfs_agblock_t bno, + xfs_extlen_t len, + const struct xfs_owner_info *oinfo) +{ + struct xfs_rmap_matches res; + int error; + + if (!sc->sa.rmap_cur || xchk_skip_xref(sc->sm)) + return; + + error = xfs_rmap_count_owners(sc->sa.rmap_cur, bno, len, oinfo, &res); + if (!xchk_should_check_xref(sc, &error, &sc->sa.rmap_cur)) + return; + if (res.matches != 1) + xchk_btree_xref_set_corrupt(sc, sc->sa.rmap_cur, 0); + if (res.bad_non_owner_matches) + xchk_btree_xref_set_corrupt(sc, sc->sa.rmap_cur, 0); + if (res.non_owner_matches) + xchk_btree_xref_set_corrupt(sc, sc->sa.rmap_cur, 0); +} + +/* xref check that the extent is not owned by a given owner */ +void +xchk_xref_is_not_owned_by( + struct xfs_scrub *sc, + xfs_agblock_t bno, + xfs_extlen_t len, + const struct xfs_owner_info *oinfo) +{ + struct xfs_rmap_matches res; + int error; + + if (!sc->sa.rmap_cur || xchk_skip_xref(sc->sm)) + return; + + error = xfs_rmap_count_owners(sc->sa.rmap_cur, bno, len, oinfo, &res); + if (!xchk_should_check_xref(sc, &error, &sc->sa.rmap_cur)) + return; + if (res.matches != 0) + xchk_btree_xref_set_corrupt(sc, sc->sa.rmap_cur, 0); + if (res.bad_non_owner_matches) + xchk_btree_xref_set_corrupt(sc, sc->sa.rmap_cur, 0); +} + +/* xref check that the extent has no reverse mapping at all */ +void +xchk_xref_has_no_owner( + struct xfs_scrub *sc, + xfs_agblock_t bno, + xfs_extlen_t len) +{ + enum xbtree_recpacking outcome; + int error; + + if (!sc->sa.rmap_cur || xchk_skip_xref(sc->sm)) + return; + + error = xfs_rmap_has_records(sc->sa.rmap_cur, bno, len, &outcome); + if (!xchk_should_check_xref(sc, &error, &sc->sa.rmap_cur)) + return; + if (outcome != XBTREE_RECPACKING_EMPTY) + xchk_btree_xref_set_corrupt(sc, sc->sa.rmap_cur, 0); +} diff --git a/fs/xfs/scrub/rmap_repair.c b/fs/xfs/scrub/rmap_repair.c new file mode 100644 index 000000000..590f9f418 --- /dev/null +++ b/fs/xfs/scrub/rmap_repair.c @@ -0,0 +1,1734 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2018-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_btree.h" +#include "xfs_btree_staging.h" +#include "xfs_buf_mem.h" +#include "xfs_btree_mem.h" +#include "xfs_bit.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_sb.h" +#include "xfs_alloc.h" +#include "xfs_alloc_btree.h" +#include "xfs_ialloc.h" +#include "xfs_ialloc_btree.h" +#include "xfs_rmap.h" +#include "xfs_rmap_btree.h" +#include "xfs_inode.h" +#include "xfs_icache.h" +#include "xfs_bmap.h" +#include "xfs_bmap_btree.h" +#include "xfs_refcount.h" +#include "xfs_refcount_btree.h" +#include "xfs_ag.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_rtgroup.h" +#include "xfs_rtrefcount_btree.h" +#include "scrub/xfs_scrub.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/btree.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/bitmap.h" +#include "scrub/agb_bitmap.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/iscan.h" +#include "scrub/newbt.h" +#include "scrub/reap.h" + +/* + * Reverse Mapping Btree Repair + * ============================ + * + * This is the most involved of all the AG space btree rebuilds. Everywhere + * else in XFS we lock inodes and then AG data structures, but generating the + * list of rmap records requires that we be able to scan both block mapping + * btrees of every inode in the filesystem to see if it owns any extents in + * this AG. We can't tolerate any inode updates while we do this, so we + * freeze the filesystem to lock everyone else out, and grant ourselves + * special privileges to run transactions with regular background reclamation + * turned off. + * + * We also have to be very careful not to allow inode reclaim to start a + * transaction because all transactions (other than our own) will block. + * Deferred inode inactivation helps us out there. + * + * I) Reverse mappings for all non-space metadata and file data are collected + * according to the following algorithm: + * + * 1. For each fork of each inode: + * 1.1. Create a bitmap BMBIT to track bmbt blocks if necessary. + * 1.2. If the incore extent map isn't loaded, walk the bmbt to accumulate + * bmaps into rmap records (see 1.1.4). Set bits in BMBIT for each btree + * block. + * 1.3. If the incore extent map is loaded but the fork is in btree format, + * just visit the bmbt blocks to set the corresponding BMBIT areas. + * 1.4. From the incore extent map, accumulate each bmap that falls into our + * target AG. Remember, multiple bmap records can map to a single rmap + * record, so we cannot simply emit rmap records 1:1. + * 1.5. Emit rmap records for each extent in BMBIT and free it. + * 2. Create bitmaps INOBIT and ICHUNKBIT. + * 3. For each record in the inobt, set the corresponding areas in ICHUNKBIT, + * and set bits in INOBIT for each btree block. If the inobt has no records + * at all, we must be careful to record its root in INOBIT. + * 4. For each block in the finobt, set the corresponding INOBIT area. + * 5. Emit rmap records for each extent in INOBIT and ICHUNKBIT and free them. + * 6. Create bitmaps REFCBIT and COWBIT. + * 7. For each CoW staging extent in the refcountbt, set the corresponding + * areas in COWBIT. + * 8. For each block in the refcountbt, set the corresponding REFCBIT area. + * 9. Emit rmap records for each extent in REFCBIT and COWBIT and free them. + * A. Emit rmap for the AG headers. + * B. Emit rmap for the log, if there is one. + * + * II) The rmapbt shape and space metadata rmaps are computed as follows: + * + * 1. Count the rmaps collected in the previous step. (= NR) + * 2. Estimate the number of rmapbt blocks needed to store NR records. (= RMB) + * 3. Reserve RMB blocks through the newbt using the allocator in normap mode. + * 4. Create bitmap AGBIT. + * 5. For each reservation in the newbt, set the corresponding areas in AGBIT. + * 6. For each block in the AGFL, bnobt, and cntbt, set the bits in AGBIT. + * 7. Count the extents in AGBIT. (= AGNR) + * 8. Estimate the number of rmapbt blocks needed for NR + AGNR rmaps. (= RMB') + * 9. If RMB' >= RMB, reserve RMB' - RMB more newbt blocks, set RMB = RMB', + * and clear AGBIT. Go to step 5. + * A. Emit rmaps for each extent in AGBIT. + * + * III) The rmapbt is constructed and set in place as follows: + * + * 1. Sort the rmap records. + * 2. Bulk load the rmaps. + * + * IV) Reap the old btree blocks. + * + * 1. Create a bitmap OLDRMBIT. + * 2. For each gap in the new rmapbt, set the corresponding areas of OLDRMBIT. + * 3. For each extent in the bnobt, clear the corresponding parts of OLDRMBIT. + * 4. Reap the extents corresponding to the set areas in OLDRMBIT. These are + * the parts of the AG that the rmap didn't find during its scan of the + * primary metadata and aren't known to be in the free space, which implies + * that they were the old rmapbt blocks. + * 5. Commit. + * + * We use the 'xrep_rmap' prefix for all the rmap functions. + */ + +/* Context for collecting rmaps */ +struct xrep_rmap { + /* new rmapbt information */ + struct xrep_newbt new_btree; + + /* lock for the xfbtree and xfile */ + struct mutex lock; + + /* rmap records generated from primary metadata */ + struct xfbtree rmap_btree; + + struct xfs_scrub *sc; + + /* in-memory btree cursor for the xfs_btree_bload iteration */ + struct xfs_btree_cur *mcur; + + /* Hooks into rmap update code. */ + struct xfs_rmap_hook rhook; + + /* inode scan cursor */ + struct xchk_iscan iscan; + + /* Number of non-freespace records found. */ + unsigned long long nr_records; + + /* bnobt/cntbt contribution to btreeblks */ + xfs_agblock_t freesp_btblocks; + + /* old agf_rmap_blocks counter */ + unsigned int old_rmapbt_fsbcount; +}; + +/* Set us up to repair reverse mapping btrees. */ +int +xrep_setup_ag_rmapbt( + struct xfs_scrub *sc) +{ + struct xrep_rmap *rr; + int error; + + xchk_fsgates_enable(sc, XCHK_FSGATES_RMAP); + + error = xrep_setup_xfbtree(sc, "reverse mapping records"); + if (error) + return error; + + rr = kzalloc_obj(struct xrep_rmap, XCHK_GFP_FLAGS); + if (!rr) + return -ENOMEM; + + rr->sc = sc; + sc->buf = rr; + return 0; +} + +/* Make sure there's nothing funny about this mapping. */ +STATIC int +xrep_rmap_check_mapping( + struct xfs_scrub *sc, + const struct xfs_rmap_irec *rec) +{ + enum xbtree_recpacking outcome; + int error; + + if (xfs_rmap_check_irec(sc->sa.pag, rec) != NULL) + return -EFSCORRUPTED; + + /* Make sure this isn't free space. */ + error = xfs_alloc_has_records(sc->sa.bno_cur, rec->rm_startblock, + rec->rm_blockcount, &outcome); + if (error) + return error; + if (outcome != XBTREE_RECPACKING_EMPTY) + return -EFSCORRUPTED; + + return 0; +} + +/* Store a reverse-mapping record. */ +static inline int +xrep_rmap_stash( + struct xrep_rmap *rr, + xfs_agblock_t startblock, + xfs_extlen_t blockcount, + uint64_t owner, + uint64_t offset, + unsigned int flags) +{ + struct xfs_rmap_irec rmap = { + .rm_startblock = startblock, + .rm_blockcount = blockcount, + .rm_owner = owner, + .rm_offset = offset, + .rm_flags = flags, + }; + struct xfs_scrub *sc = rr->sc; + struct xfs_btree_cur *mcur; + int error = 0; + + if (xchk_should_terminate(sc, &error)) + return error; + + if (xchk_iscan_aborted(&rr->iscan)) + return -EFSCORRUPTED; + + trace_xrep_rmap_found(sc->sa.pag, &rmap); + + mutex_lock(&rr->lock); + mcur = xfs_rmapbt_mem_cursor(sc->sa.pag, sc->tp, &rr->rmap_btree); + error = xfs_rmap_map_raw(mcur, &rmap); + xfs_btree_del_cursor(mcur, error); + if (error) + goto out_cancel; + + error = xfbtree_trans_commit(&rr->rmap_btree, sc->tp); + if (error) + goto out_abort; + + mutex_unlock(&rr->lock); + return 0; + +out_cancel: + xfbtree_trans_cancel(&rr->rmap_btree, sc->tp); +out_abort: + xchk_iscan_abort(&rr->iscan); + mutex_unlock(&rr->lock); + return error; +} + +struct xrep_rmap_stash_run { + struct xrep_rmap *rr; + uint64_t owner; + unsigned int rmap_flags; +}; + +static int +xrep_rmap_stash_run( + uint32_t start, + uint32_t len, + void *priv) +{ + struct xrep_rmap_stash_run *rsr = priv; + struct xrep_rmap *rr = rsr->rr; + + return xrep_rmap_stash(rr, start, len, rsr->owner, 0, rsr->rmap_flags); +} + +/* + * Emit rmaps for every extent of bits set in the bitmap. Caller must ensure + * that the ranges are in units of FS blocks. + */ +STATIC int +xrep_rmap_stash_bitmap( + struct xrep_rmap *rr, + struct xagb_bitmap *bitmap, + const struct xfs_owner_info *oinfo) +{ + struct xrep_rmap_stash_run rsr = { + .rr = rr, + .owner = oinfo->oi_owner, + .rmap_flags = 0, + }; + + if (oinfo->oi_flags & XFS_OWNER_INFO_ATTR_FORK) + rsr.rmap_flags |= XFS_RMAP_ATTR_FORK; + if (oinfo->oi_flags & XFS_OWNER_INFO_BMBT_BLOCK) + rsr.rmap_flags |= XFS_RMAP_BMBT_BLOCK; + + return xagb_bitmap_walk(bitmap, xrep_rmap_stash_run, &rsr); +} + +/* Section (I): Finding all file and bmbt extents. */ + +/* Context for accumulating rmaps for an inode fork. */ +struct xrep_rmap_ifork { + /* + * Accumulate rmap data here to turn multiple adjacent bmaps into a + * single rmap. + */ + struct xfs_rmap_irec accum; + + /* Bitmap of bmbt blocks in this AG. */ + struct xagb_bitmap bmbt_blocks; + + struct xrep_rmap *rr; + + /* Which inode fork? */ + int whichfork; +}; + +/* Stash an rmap that we accumulated while walking an inode fork. */ +STATIC int +xrep_rmap_stash_accumulated( + struct xrep_rmap_ifork *rf) +{ + if (rf->accum.rm_blockcount == 0) + return 0; + + return xrep_rmap_stash(rf->rr, rf->accum.rm_startblock, + rf->accum.rm_blockcount, rf->accum.rm_owner, + rf->accum.rm_offset, rf->accum.rm_flags); +} + +/* Accumulate a bmbt record. */ +STATIC int +xrep_rmap_visit_bmbt( + struct xfs_btree_cur *cur, + struct xfs_bmbt_irec *rec, + void *priv) +{ + struct xrep_rmap_ifork *rf = priv; + struct xfs_mount *mp = rf->rr->sc->mp; + struct xfs_rmap_irec *accum = &rf->accum; + xfs_agblock_t agbno; + unsigned int rmap_flags = 0; + int error; + + if (XFS_FSB_TO_AGNO(mp, rec->br_startblock) != + pag_agno(rf->rr->sc->sa.pag)) + return 0; + + agbno = XFS_FSB_TO_AGBNO(mp, rec->br_startblock); + if (rf->whichfork == XFS_ATTR_FORK) + rmap_flags |= XFS_RMAP_ATTR_FORK; + if (rec->br_state == XFS_EXT_UNWRITTEN) + rmap_flags |= XFS_RMAP_UNWRITTEN; + + /* If this bmap is adjacent to the previous one, just add it. */ + if (accum->rm_blockcount > 0 && + rec->br_startoff == accum->rm_offset + accum->rm_blockcount && + agbno == accum->rm_startblock + accum->rm_blockcount && + rmap_flags == accum->rm_flags) { + accum->rm_blockcount += rec->br_blockcount; + return 0; + } + + /* Otherwise stash the old rmap and start accumulating a new one. */ + error = xrep_rmap_stash_accumulated(rf); + if (error) + return error; + + accum->rm_startblock = agbno; + accum->rm_blockcount = rec->br_blockcount; + accum->rm_offset = rec->br_startoff; + accum->rm_flags = rmap_flags; + return 0; +} + +/* Add a btree block to the bitmap. */ +STATIC int +xrep_rmap_visit_iroot_btree_block( + struct xfs_btree_cur *cur, + int level, + void *priv) +{ + struct xrep_rmap_ifork *rf = priv; + struct xfs_buf *bp; + xfs_fsblock_t fsbno; + xfs_agblock_t agbno; + + xfs_btree_get_block(cur, level, &bp); + if (!bp) + return 0; + + fsbno = XFS_DADDR_TO_FSB(cur->bc_mp, xfs_buf_daddr(bp)); + if (XFS_FSB_TO_AGNO(cur->bc_mp, fsbno) != pag_agno(rf->rr->sc->sa.pag)) + return 0; + + agbno = XFS_FSB_TO_AGBNO(cur->bc_mp, fsbno); + return xagb_bitmap_set(&rf->bmbt_blocks, agbno, 1); +} + +/* + * Iterate a metadata btree rooted in an inode to collect rmap records for + * anything in this fork that matches the AG. + */ +STATIC int +xrep_rmap_scan_iroot_btree( + struct xrep_rmap_ifork *rf, + struct xfs_btree_cur *cur) +{ + struct xfs_owner_info oinfo; + struct xrep_rmap *rr = rf->rr; + int error; + + xagb_bitmap_init(&rf->bmbt_blocks); + + /* Record all the blocks in the btree itself. */ + error = xfs_btree_visit_blocks(cur, xrep_rmap_visit_iroot_btree_block, + XFS_BTREE_VISIT_ALL, rf); + if (error) + goto out; + + /* Emit rmaps for the btree blocks. */ + xfs_rmap_ino_bmbt_owner(&oinfo, rf->accum.rm_owner, rf->whichfork); + error = xrep_rmap_stash_bitmap(rr, &rf->bmbt_blocks, &oinfo); + if (error) + goto out; + + /* Stash any remaining accumulated rmaps. */ + error = xrep_rmap_stash_accumulated(rf); +out: + xagb_bitmap_destroy(&rf->bmbt_blocks); + return error; +} + +/* + * Iterate the block mapping btree to collect rmap records for anything in this + * fork that matches the AG. Sets @mappings_done to true if we've scanned the + * block mappings in this fork. + */ +STATIC int +xrep_rmap_scan_bmbt( + struct xrep_rmap_ifork *rf, + struct xfs_inode *ip, + bool *mappings_done) +{ + struct xrep_rmap *rr = rf->rr; + struct xfs_btree_cur *cur; + struct xfs_ifork *ifp; + int error; + + *mappings_done = false; + ifp = xfs_ifork_ptr(ip, rf->whichfork); + cur = xfs_bmbt_init_cursor(rr->sc->mp, rr->sc->tp, ip, rf->whichfork); + + if (!xfs_ifork_is_realtime(ip, rf->whichfork) && + xfs_need_iread_extents(ifp)) { + /* + * If the incore extent cache isn't loaded, scan the bmbt for + * mapping records. This avoids loading the incore extent + * tree, which will increase memory pressure at a time when + * we're trying to run as quickly as we possibly can. Ignore + * realtime extents. + */ + error = xfs_bmap_query_all(cur, xrep_rmap_visit_bmbt, rf); + if (error) + goto out_cur; + + *mappings_done = true; + } + + /* Scan for the bmbt blocks, which always live on the data device. */ + error = xrep_rmap_scan_iroot_btree(rf, cur); +out_cur: + xfs_btree_del_cursor(cur, error); + return error; +} + +/* + * Iterate the in-core extent cache to collect rmap records for anything in + * this fork that matches the AG. + */ +STATIC int +xrep_rmap_scan_iext( + struct xrep_rmap_ifork *rf, + struct xfs_ifork *ifp) +{ + struct xfs_bmbt_irec rec; + struct xfs_iext_cursor icur; + int error; + + for_each_xfs_iext(ifp, &icur, &rec) { + if (isnullstartblock(rec.br_startblock)) + continue; + error = xrep_rmap_visit_bmbt(NULL, &rec, rf); + if (error) + return error; + } + + return xrep_rmap_stash_accumulated(rf); +} + +static int +xrep_rmap_scan_meta_btree( + struct xrep_rmap_ifork *rf, + struct xfs_inode *ip) +{ + struct xfs_scrub *sc = rf->rr->sc; + struct xfs_rtgroup *rtg = NULL; + struct xfs_btree_cur *cur = NULL; + enum xfs_rtg_inodes type; + int error; + + if (rf->whichfork != XFS_DATA_FORK) + return -EFSCORRUPTED; + + switch (ip->i_metatype) { + case XFS_METAFILE_RTRMAP: + type = XFS_RTGI_RMAP; + break; + case XFS_METAFILE_RTREFCOUNT: + type = XFS_RTGI_REFCOUNT; + break; + default: + ASSERT(0); + return -EFSCORRUPTED; + } + + while ((rtg = xfs_rtgroup_next(sc->mp, rtg))) { + if (ip == rtg->rtg_inodes[type]) + goto found; + } + + /* + * We should never find an rt metadata btree inode that isn't + * associated with an rtgroup yet has ondisk blocks allocated to it. + */ + if (ip->i_nblocks) { + ASSERT(0); + return -EFSCORRUPTED; + } + + return 0; + +found: + switch (ip->i_metatype) { + case XFS_METAFILE_RTRMAP: + cur = xfs_rtrmapbt_init_cursor(sc->tp, rtg); + break; + case XFS_METAFILE_RTREFCOUNT: + cur = xfs_rtrefcountbt_init_cursor(sc->tp, rtg); + break; + default: + ASSERT(0); + error = -EFSCORRUPTED; + goto out_rtg; + } + + error = xrep_rmap_scan_iroot_btree(rf, cur); + xfs_btree_del_cursor(cur, error); +out_rtg: + xfs_rtgroup_rele(rtg); + return error; +} + +/* Find all the extents from a given AG in an inode fork. */ +STATIC int +xrep_rmap_scan_ifork( + struct xrep_rmap *rr, + struct xfs_inode *ip, + int whichfork) +{ + struct xrep_rmap_ifork rf = { + .accum = { .rm_owner = I_INO(ip), }, + .rr = rr, + .whichfork = whichfork, + }; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + bool mappings_done; + int error = 0; + + if (!ifp) + return 0; + + switch (ifp->if_format) { + case XFS_DINODE_FMT_BTREE: + /* + * Scan the bmap btree for data device mappings. This includes + * the btree blocks themselves, even if this is a realtime + * file. + */ + error = xrep_rmap_scan_bmbt(&rf, ip, &mappings_done); + if (error || mappings_done) + return error; + fallthrough; + case XFS_DINODE_FMT_EXTENTS: + /* Scan incore extent cache if this isn't a realtime file. */ + if (xfs_ifork_is_realtime(ip, whichfork)) + return 0; + + return xrep_rmap_scan_iext(&rf, ifp); + case XFS_DINODE_FMT_META_BTREE: + return xrep_rmap_scan_meta_btree(&rf, ip); + } + + return 0; +} + +/* + * Take ILOCK on a file that we want to scan. + * + * Select ILOCK_EXCL if the file has an unloaded data bmbt or has an unloaded + * attr bmbt. Otherwise, take ILOCK_SHARED. + */ +static inline unsigned int +xrep_rmap_scan_ilock( + struct xfs_inode *ip) +{ + uint lock_mode = XFS_ILOCK_SHARED; + + if (xfs_need_iread_extents(&ip->i_df)) { + lock_mode = XFS_ILOCK_EXCL; + goto lock; + } + + if (xfs_inode_has_attr_fork(ip) && xfs_need_iread_extents(&ip->i_af)) + lock_mode = XFS_ILOCK_EXCL; + +lock: + xfs_ilock(ip, lock_mode); + return lock_mode; +} + +/* Record reverse mappings for a file. */ +STATIC int +xrep_rmap_scan_inode( + struct xrep_rmap *rr, + struct xfs_inode *ip) +{ + unsigned int lock_mode = xrep_rmap_scan_ilock(ip); + int error; + + /* Check the data fork. */ + error = xrep_rmap_scan_ifork(rr, ip, XFS_DATA_FORK); + if (error) + goto out_unlock; + + /* Check the attr fork. */ + error = xrep_rmap_scan_ifork(rr, ip, XFS_ATTR_FORK); + if (error) + goto out_unlock; + + /* COW fork extents are "owned" by the refcount btree. */ + + xchk_iscan_mark_visited(&rr->iscan, ip); +out_unlock: + xfs_iunlock(ip, lock_mode); + return error; +} + +/* Section (I): Find all AG metadata extents except for free space metadata. */ + +struct xrep_rmap_inodes { + struct xrep_rmap *rr; + struct xagb_bitmap inobt_blocks; /* INOBIT */ + struct xagb_bitmap ichunk_blocks; /* ICHUNKBIT */ +}; + +/* Record inode btree rmaps. */ +STATIC int +xrep_rmap_walk_inobt( + struct xfs_btree_cur *cur, + const union xfs_btree_rec *rec, + void *priv) +{ + struct xfs_inobt_rec_incore irec; + struct xrep_rmap_inodes *ri = priv; + struct xfs_mount *mp = cur->bc_mp; + xfs_agblock_t agbno; + xfs_extlen_t aglen; + xfs_agino_t agino; + xfs_agino_t iperhole; + unsigned int i; + int error; + + /* Record the inobt blocks. */ + error = xagb_bitmap_set_btcur_path(&ri->inobt_blocks, cur); + if (error) + return error; + + xfs_inobt_btrec_to_irec(mp, rec, &irec); + if (xfs_inobt_check_irec(to_perag(cur->bc_group), &irec) != NULL) + return -EFSCORRUPTED; + + agino = irec.ir_startino; + + /* Record a non-sparse inode chunk. */ + if (!xfs_inobt_issparse(irec.ir_holemask)) { + agbno = XFS_AGINO_TO_AGBNO(mp, agino); + aglen = max_t(xfs_extlen_t, 1, + XFS_INODES_PER_CHUNK / mp->m_sb.sb_inopblock); + + return xagb_bitmap_set(&ri->ichunk_blocks, agbno, aglen); + } + + /* Iterate each chunk. */ + iperhole = max_t(xfs_agino_t, mp->m_sb.sb_inopblock, + XFS_INODES_PER_HOLEMASK_BIT); + aglen = iperhole / mp->m_sb.sb_inopblock; + for (i = 0, agino = irec.ir_startino; + i < XFS_INOBT_HOLEMASK_BITS; + i += iperhole / XFS_INODES_PER_HOLEMASK_BIT, agino += iperhole) { + /* Skip holes. */ + if (irec.ir_holemask & (1 << i)) + continue; + + /* Record the inode chunk otherwise. */ + agbno = XFS_AGINO_TO_AGBNO(mp, agino); + error = xagb_bitmap_set(&ri->ichunk_blocks, agbno, aglen); + if (error) + return error; + } + + return 0; +} + +/* Collect rmaps for the blocks containing inode btrees and the inode chunks. */ +STATIC int +xrep_rmap_find_inode_rmaps( + struct xrep_rmap *rr) +{ + struct xrep_rmap_inodes ri = { + .rr = rr, + }; + struct xfs_scrub *sc = rr->sc; + int error; + + xagb_bitmap_init(&ri.inobt_blocks); + xagb_bitmap_init(&ri.ichunk_blocks); + + /* + * Iterate every record in the inobt so we can capture all the inode + * chunks and the blocks in the inobt itself. + */ + error = xfs_btree_query_all(sc->sa.ino_cur, xrep_rmap_walk_inobt, &ri); + if (error) + goto out_bitmap; + + /* + * Note that if there are zero records in the inobt then query_all does + * nothing and we have to account the empty inobt root manually. + */ + if (xagb_bitmap_empty(&ri.ichunk_blocks)) { + struct xfs_agi *agi = sc->sa.agi_bp->b_addr; + + error = xagb_bitmap_set(&ri.inobt_blocks, + be32_to_cpu(agi->agi_root), 1); + if (error) + goto out_bitmap; + } + + /* Scan the finobt too. */ + if (xfs_has_finobt(sc->mp)) { + error = xagb_bitmap_set_btblocks(&ri.inobt_blocks, + sc->sa.fino_cur); + if (error) + goto out_bitmap; + } + + /* Generate rmaps for everything. */ + error = xrep_rmap_stash_bitmap(rr, &ri.inobt_blocks, + &XFS_RMAP_OINFO_INOBT); + if (error) + goto out_bitmap; + error = xrep_rmap_stash_bitmap(rr, &ri.ichunk_blocks, + &XFS_RMAP_OINFO_INODES); + +out_bitmap: + xagb_bitmap_destroy(&ri.inobt_blocks); + xagb_bitmap_destroy(&ri.ichunk_blocks); + return error; +} + +/* Record a CoW staging extent. */ +STATIC int +xrep_rmap_walk_cowblocks( + struct xfs_btree_cur *cur, + const struct xfs_refcount_irec *irec, + void *priv) +{ + struct xagb_bitmap *bitmap = priv; + + if (!xfs_refcount_check_domain(irec) || + irec->rc_domain != XFS_REFC_DOMAIN_COW) + return -EFSCORRUPTED; + + return xagb_bitmap_set(bitmap, irec->rc_startblock, irec->rc_blockcount); +} + +/* + * Collect rmaps for the blocks containing the refcount btree, and all CoW + * staging extents. + */ +STATIC int +xrep_rmap_find_refcount_rmaps( + struct xrep_rmap *rr) +{ + struct xagb_bitmap refcountbt_blocks; /* REFCBIT */ + struct xagb_bitmap cow_blocks; /* COWBIT */ + struct xfs_refcount_irec low = { + .rc_startblock = 0, + .rc_domain = XFS_REFC_DOMAIN_COW, + }; + struct xfs_refcount_irec high = { + .rc_startblock = -1U, + .rc_domain = XFS_REFC_DOMAIN_COW, + }; + struct xfs_scrub *sc = rr->sc; + int error; + + if (!xfs_has_reflink(sc->mp)) + return 0; + + xagb_bitmap_init(&refcountbt_blocks); + xagb_bitmap_init(&cow_blocks); + + /* refcountbt */ + error = xagb_bitmap_set_btblocks(&refcountbt_blocks, sc->sa.refc_cur); + if (error) + goto out_bitmap; + + /* Collect rmaps for CoW staging extents. */ + error = xfs_refcount_query_range(sc->sa.refc_cur, &low, &high, + xrep_rmap_walk_cowblocks, &cow_blocks); + if (error) + goto out_bitmap; + + /* Generate rmaps for everything. */ + error = xrep_rmap_stash_bitmap(rr, &cow_blocks, &XFS_RMAP_OINFO_COW); + if (error) + goto out_bitmap; + error = xrep_rmap_stash_bitmap(rr, &refcountbt_blocks, + &XFS_RMAP_OINFO_REFC); + +out_bitmap: + xagb_bitmap_destroy(&cow_blocks); + xagb_bitmap_destroy(&refcountbt_blocks); + return error; +} + +/* Generate rmaps for the AG headers (AGI/AGF/AGFL) */ +STATIC int +xrep_rmap_find_agheader_rmaps( + struct xrep_rmap *rr) +{ + struct xfs_scrub *sc = rr->sc; + + /* Create a record for the AG sb->agfl. */ + return xrep_rmap_stash(rr, XFS_SB_BLOCK(sc->mp), + XFS_AGFL_BLOCK(sc->mp) - XFS_SB_BLOCK(sc->mp) + 1, + XFS_RMAP_OWN_FS, 0, 0); +} + +/* Generate rmaps for the log, if it's in this AG. */ +STATIC int +xrep_rmap_find_log_rmaps( + struct xrep_rmap *rr) +{ + struct xfs_scrub *sc = rr->sc; + + if (!xfs_ag_contains_log(sc->mp, pag_agno(sc->sa.pag))) + return 0; + + return xrep_rmap_stash(rr, + XFS_FSB_TO_AGBNO(sc->mp, sc->mp->m_sb.sb_logstart), + sc->mp->m_sb.sb_logblocks, XFS_RMAP_OWN_LOG, 0, 0); +} + +/* Check and count all the records that we gathered. */ +STATIC int +xrep_rmap_check_record( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv) +{ + struct xrep_rmap *rr = priv; + int error; + + error = xrep_rmap_check_mapping(rr->sc, rec); + if (error) + return error; + + rr->nr_records++; + return 0; +} + +/* + * Generate all the reverse-mappings for this AG, a list of the old rmapbt + * blocks, and the new btreeblks count. Figure out if we have enough free + * space to reconstruct the inode btrees. The caller must clean up the lists + * if anything goes wrong. This implements section (I) above. + */ +STATIC int +xrep_rmap_find_rmaps( + struct xrep_rmap *rr) +{ + struct xfs_scrub *sc = rr->sc; + struct xchk_ag *sa = &sc->sa; + struct xfs_inode *ip; + struct xfs_btree_cur *mcur; + int error; + + /* Find all the per-AG metadata. */ + xrep_ag_btcur_init(sc, &sc->sa); + + error = xrep_rmap_find_inode_rmaps(rr); + if (error) + goto end_agscan; + + error = xrep_rmap_find_refcount_rmaps(rr); + if (error) + goto end_agscan; + + error = xrep_rmap_find_agheader_rmaps(rr); + if (error) + goto end_agscan; + + error = xrep_rmap_find_log_rmaps(rr); +end_agscan: + xchk_ag_btcur_free(&sc->sa); + if (error) + return error; + + /* + * Set up for a potentially lengthy filesystem scan by reducing our + * transaction resource usage for the duration. Specifically: + * + * Unlock the AG header buffers and cancel the transaction to release + * the log grant space while we scan the filesystem. + * + * Create a new empty transaction to eliminate the possibility of the + * inode scan deadlocking on cyclical metadata. + * + * We pass the empty transaction to the file scanning function to avoid + * repeatedly cycling empty transactions. This can be done even though + * we take the IOLOCK to quiesce the file because empty transactions + * do not take sb_internal. + */ + sa->agf_bp = NULL; + sa->agi_bp = NULL; + xchk_trans_cancel(sc); + xchk_trans_alloc_empty(sc); + + /* Iterate all AGs for inodes rmaps. */ + while ((error = xchk_iscan_iter(&rr->iscan, &ip)) == 1) { + error = xrep_rmap_scan_inode(rr, ip); + xchk_irele(sc, ip); + if (error) + break; + + if (xchk_should_terminate(sc, &error)) + break; + } + xchk_iscan_iter_finish(&rr->iscan); + if (error) + return error; + + /* + * Switch out for a real transaction and lock the AG headers in + * preparation for building a new tree. + */ + xchk_trans_cancel(sc); + error = xchk_setup_fs(sc); + if (error) + return error; + error = xchk_perag_drain_and_lock(sc); + if (error) + return error; + + /* + * If a hook failed to update the in-memory btree, we lack the data to + * continue the repair. + */ + if (xchk_iscan_aborted(&rr->iscan)) + return -EFSCORRUPTED; + + /* + * Now that we have everything locked again, we need to count the + * number of rmap records stashed in the btree. This should reflect + * all actively-owned space in the filesystem. At the same time, check + * all our records before we start building a new btree, which requires + * a bnobt cursor. + */ + mcur = xfs_rmapbt_mem_cursor(rr->sc->sa.pag, NULL, &rr->rmap_btree); + sc->sa.bno_cur = xfs_bnobt_init_cursor(sc->mp, sc->tp, sc->sa.agf_bp, + sc->sa.pag); + + rr->nr_records = 0; + error = xfs_rmap_query_all(mcur, xrep_rmap_check_record, rr); + + xfs_btree_del_cursor(sc->sa.bno_cur, error); + sc->sa.bno_cur = NULL; + xfs_btree_del_cursor(mcur, error); + + return error; +} + +/* Section (II): Reserving space for new rmapbt and setting free space bitmap */ + +struct xrep_rmap_agfl { + struct xagb_bitmap *bitmap; + xfs_agnumber_t agno; +}; + +/* Add an AGFL block to the rmap list. */ +STATIC int +xrep_rmap_walk_agfl( + struct xfs_mount *mp, + xfs_agblock_t agbno, + void *priv) +{ + struct xrep_rmap_agfl *ra = priv; + + return xagb_bitmap_set(ra->bitmap, agbno, 1); +} + +/* + * Run one round of reserving space for the new rmapbt and recomputing the + * number of blocks needed to store the previously observed rmapbt records and + * the ones we'll create for the free space metadata. When we don't need more + * blocks, return a bitmap of OWN_AG extents in @freesp_blocks and set @done to + * true. + */ +STATIC int +xrep_rmap_try_reserve( + struct xrep_rmap *rr, + struct xfs_btree_cur *rmap_cur, + struct xagb_bitmap *freesp_blocks, + uint64_t *blocks_reserved, + bool *done) +{ + struct xrep_rmap_agfl ra = { + .bitmap = freesp_blocks, + .agno = pag_agno(rr->sc->sa.pag), + }; + struct xfs_scrub *sc = rr->sc; + struct xrep_newbt_resv *resv, *n; + struct xfs_agf *agf = sc->sa.agf_bp->b_addr; + struct xfs_buf *agfl_bp; + uint64_t nr_blocks; /* RMB */ + uint64_t freesp_records; + int error; + + /* + * We're going to recompute new_btree.bload.nr_blocks at the end of + * this function to reflect however many btree blocks we need to store + * all the rmap records (including the ones that reflect the changes we + * made to support the new rmapbt blocks), so we save the old value + * here so we can decide if we've reserved enough blocks. + */ + nr_blocks = rr->new_btree.bload.nr_blocks; + + /* + * Make sure we've reserved enough space for the new btree. This can + * change the shape of the free space btrees, which can cause secondary + * interactions with the rmap records because all three space btrees + * have the same rmap owner. We'll account for all that below. + */ + error = xrep_newbt_alloc_blocks(&rr->new_btree, + nr_blocks - *blocks_reserved); + if (error) + return error; + + *blocks_reserved = rr->new_btree.bload.nr_blocks; + + /* Clear everything in the bitmap. */ + xagb_bitmap_destroy(freesp_blocks); + + /* Set all the bnobt blocks in the bitmap. */ + sc->sa.bno_cur = xfs_bnobt_init_cursor(sc->mp, sc->tp, sc->sa.agf_bp, + sc->sa.pag); + error = xagb_bitmap_set_btblocks(freesp_blocks, sc->sa.bno_cur); + xfs_btree_del_cursor(sc->sa.bno_cur, error); + sc->sa.bno_cur = NULL; + if (error) + return error; + + /* Set all the cntbt blocks in the bitmap. */ + sc->sa.cnt_cur = xfs_cntbt_init_cursor(sc->mp, sc->tp, sc->sa.agf_bp, + sc->sa.pag); + error = xagb_bitmap_set_btblocks(freesp_blocks, sc->sa.cnt_cur); + xfs_btree_del_cursor(sc->sa.cnt_cur, error); + sc->sa.cnt_cur = NULL; + if (error) + return error; + + /* Record our new btreeblks value. */ + rr->freesp_btblocks = xagb_bitmap_hweight(freesp_blocks) - 2; + + /* Set all the new rmapbt blocks in the bitmap. */ + list_for_each_entry_safe(resv, n, &rr->new_btree.resv_list, list) { + error = xagb_bitmap_set(freesp_blocks, resv->agbno, resv->len); + if (error) + return error; + } + + /* Set all the AGFL blocks in the bitmap. */ + error = xfs_alloc_read_agfl(sc->sa.pag, sc->tp, &agfl_bp); + if (error) + return error; + + error = xfs_agfl_walk(sc->mp, agf, agfl_bp, xrep_rmap_walk_agfl, &ra); + if (error) + return error; + + /* Count the extents in the bitmap. */ + freesp_records = xagb_bitmap_count_set_regions(freesp_blocks); + + /* Compute how many blocks we'll need for all the rmaps. */ + error = xfs_btree_bload_compute_geometry(rmap_cur, + &rr->new_btree.bload, rr->nr_records + freesp_records); + if (error) + return error; + + /* We're done when we don't need more blocks. */ + *done = nr_blocks >= rr->new_btree.bload.nr_blocks; + return 0; +} + +/* + * Iteratively reserve space for rmap btree while recording OWN_AG rmaps for + * the free space metadata. This implements section (II) above. + */ +STATIC int +xrep_rmap_reserve_space( + struct xrep_rmap *rr, + struct xfs_btree_cur *rmap_cur) +{ + struct xagb_bitmap freesp_blocks; /* AGBIT */ + uint64_t blocks_reserved = 0; + bool done = false; + int error; + + /* Compute how many blocks we'll need for the rmaps collected so far. */ + error = xfs_btree_bload_compute_geometry(rmap_cur, + &rr->new_btree.bload, rr->nr_records); + if (error) + return error; + + /* Last chance to abort before we start committing fixes. */ + if (xchk_should_terminate(rr->sc, &error)) + return error; + + xagb_bitmap_init(&freesp_blocks); + + /* + * Iteratively reserve space for the new rmapbt and recompute the + * number of blocks needed to store the previously observed rmapbt + * records and the ones we'll create for the free space metadata. + * Finish when we don't need more blocks. + */ + do { + error = xrep_rmap_try_reserve(rr, rmap_cur, &freesp_blocks, + &blocks_reserved, &done); + if (error) + goto out_bitmap; + } while (!done); + + /* Emit rmaps for everything in the free space bitmap. */ + xrep_ag_btcur_init(rr->sc, &rr->sc->sa); + error = xrep_rmap_stash_bitmap(rr, &freesp_blocks, &XFS_RMAP_OINFO_AG); + xchk_ag_btcur_free(&rr->sc->sa); + +out_bitmap: + xagb_bitmap_destroy(&freesp_blocks); + return error; +} + +/* Section (III): Building the new rmap btree. */ + +/* Update the AGF counters. */ +STATIC int +xrep_rmap_reset_counters( + struct xrep_rmap *rr) +{ + struct xfs_scrub *sc = rr->sc; + struct xfs_perag *pag = sc->sa.pag; + struct xfs_agf *agf = sc->sa.agf_bp->b_addr; + xfs_agblock_t rmap_btblocks; + + /* + * The AGF header contains extra information related to the reverse + * mapping btree, so we must update those fields here. + */ + rmap_btblocks = rr->new_btree.afake.af_blocks - 1; + agf->agf_btreeblks = cpu_to_be32(rr->freesp_btblocks + rmap_btblocks); + xfs_alloc_log_agf(sc->tp, sc->sa.agf_bp, XFS_AGF_BTREEBLKS); + + /* + * After we commit the new btree to disk, it is possible that the + * process to reap the old btree blocks will race with the AIL trying + * to checkpoint the old btree blocks into the filesystem. If the new + * tree is shorter than the old one, the rmapbt write verifier will + * fail and the AIL will shut down the filesystem. + * + * To avoid this, save the old incore btree height values as the alt + * height values before re-initializing the perag info from the updated + * AGF to capture all the new values. + */ + pag->pagf_repair_rmap_level = pag->pagf_rmap_level; + + /* Reinitialize with the values we just logged. */ + return xrep_reinit_pagf(sc); +} + +/* Retrieve rmapbt data for bulk load. */ +STATIC int +xrep_rmap_get_records( + struct xfs_btree_cur *cur, + unsigned int idx, + struct xfs_btree_block *block, + unsigned int nr_wanted, + void *priv) +{ + struct xrep_rmap *rr = priv; + union xfs_btree_rec *block_rec; + unsigned int loaded; + int error; + + for (loaded = 0; loaded < nr_wanted; loaded++, idx++) { + int stat = 0; + + error = xfs_btree_increment(rr->mcur, 0, &stat); + if (error) + return error; + if (!stat) + return -EFSCORRUPTED; + + error = xfs_rmap_get_rec(rr->mcur, &cur->bc_rec.r, &stat); + if (error) + return error; + if (!stat) + return -EFSCORRUPTED; + + block_rec = xfs_btree_rec_addr(cur, idx, block); + cur->bc_ops->init_rec_from_cur(cur, block_rec); + } + + return loaded; +} + +/* Feed one of the new btree blocks to the bulk loader. */ +STATIC int +xrep_rmap_claim_block( + struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr, + void *priv) +{ + struct xrep_rmap *rr = priv; + + return xrep_newbt_claim_block(cur, &rr->new_btree, ptr); +} + +/* Custom allocation function for new rmap btrees. */ +STATIC int +xrep_rmap_alloc_vextent( + struct xfs_scrub *sc, + struct xfs_alloc_arg *args, + xfs_fsblock_t alloc_hint) +{ + int error; + + /* + * We don't want an rmap update on the allocation, since we iteratively + * compute the OWN_AG records /after/ allocating blocks for the records + * that we already know we need to store. Therefore, fix the freelist + * with the NORMAP flag set so that we don't also try to create an rmap + * for new AGFL blocks. + */ + error = xrep_fix_freelist(sc, XFS_ALLOC_FLAG_NORMAP); + if (error) + return error; + + /* + * If xrep_fix_freelist fixed the freelist by moving blocks from the + * free space btrees or by removing blocks from the AGFL and queueing + * an EFI to free the block, the transaction will be dirty. This + * second case is of interest to us. + * + * Later on, we will need to compare gaps in the new recordset against + * the block usage of all OWN_AG owners in order to free the old + * btree's blocks, which means that we can't have EFIs for former AGFL + * blocks attached to the repair transaction when we commit the new + * btree. + * + * xrep_newbt_alloc_blocks guarantees this for us by calling + * xrep_defer_finish to commit anything that fix_freelist may have + * added to the transaction. + */ + return xfs_alloc_vextent_near_bno(args, alloc_hint); +} + + +/* Count the records in this btree. */ +STATIC int +xrep_rmap_count_records( + struct xfs_btree_cur *cur, + unsigned long long *nr) +{ + int running = 1; + int error; + + *nr = 0; + + error = xfs_btree_goto_left_edge(cur); + if (error) + return error; + + while (running && !(error = xfs_btree_increment(cur, 0, &running))) { + if (running) + (*nr)++; + } + + return error; +} +/* + * Use the collected rmap information to stage a new rmap btree. If this is + * successful we'll return with the new btree root information logged to the + * repair transaction but not yet committed. This implements section (III) + * above. + */ +STATIC int +xrep_rmap_build_new_tree( + struct xrep_rmap *rr) +{ + struct xfs_scrub *sc = rr->sc; + struct xfs_perag *pag = sc->sa.pag; + struct xfs_agf *agf = sc->sa.agf_bp->b_addr; + struct xfs_btree_cur *rmap_cur; + int error; + + /* + * Preserve the old rmapbt block count so that we can adjust the + * per-AG rmapbt reservation after we commit the new btree root and + * want to dispose of the old btree blocks. + */ + rr->old_rmapbt_fsbcount = be32_to_cpu(agf->agf_rmap_blocks); + + /* + * Prepare to construct the new btree by reserving disk space for the + * new btree and setting up all the accounting information we'll need + * to root the new btree while it's under construction and before we + * attach it to the AG header. The new blocks are accounted to the + * rmapbt per-AG reservation, which we will adjust further after + * committing the new btree. + */ + xrep_newbt_init_ag(&rr->new_btree, sc, &XFS_RMAP_OINFO_SKIP_UPDATE, + xfs_agbno_to_fsb(pag, XFS_RMAP_BLOCK(sc->mp)), + XFS_AG_RESV_RMAPBT); + rr->new_btree.bload.get_records = xrep_rmap_get_records; + rr->new_btree.bload.claim_block = xrep_rmap_claim_block; + rr->new_btree.alloc_vextent = xrep_rmap_alloc_vextent; + rmap_cur = xfs_rmapbt_init_cursor(sc->mp, NULL, NULL, pag); + xfs_btree_stage_afakeroot(rmap_cur, &rr->new_btree.afake); + + /* + * Initialize @rr->new_btree, reserve space for the new rmapbt, + * and compute OWN_AG rmaps. + */ + error = xrep_rmap_reserve_space(rr, rmap_cur); + if (error) + goto err_cur; + + /* + * Count the rmapbt records again, because the space reservation + * for the rmapbt itself probably added more records to the btree. + */ + rr->mcur = xfs_rmapbt_mem_cursor(rr->sc->sa.pag, NULL, + &rr->rmap_btree); + + error = xrep_rmap_count_records(rr->mcur, &rr->nr_records); + if (error) + goto err_mcur; + + /* + * Due to btree slack factors, it's possible for a new btree to be one + * level taller than the old btree. Update the incore btree height so + * that we don't trip the verifiers when writing the new btree blocks + * to disk. + */ + pag->pagf_repair_rmap_level = rr->new_btree.bload.btree_height; + + /* + * Move the cursor to the left edge of the tree so that the first + * increment in ->get_records positions us at the first record. + */ + error = xfs_btree_goto_left_edge(rr->mcur); + if (error) + goto err_level; + + /* Add all observed rmap records. */ + error = xfs_btree_bload(rmap_cur, &rr->new_btree.bload, rr); + if (error) + goto err_level; + + /* + * Install the new btree in the AG header. After this point the old + * btree is no longer accessible and the new tree is live. + */ + xfs_rmapbt_commit_staged_btree(rmap_cur, sc->tp, sc->sa.agf_bp); + xfs_btree_del_cursor(rmap_cur, 0); + xfs_btree_del_cursor(rr->mcur, 0); + rr->mcur = NULL; + + /* + * Now that we've written the new btree to disk, we don't need to keep + * updating the in-memory btree. Abort the scan to stop live updates. + */ + xchk_iscan_abort(&rr->iscan); + + /* + * The newly committed rmap recordset includes mappings for the blocks + * that we reserved to build the new btree. If there is excess space + * reservation to be freed, the corresponding rmap records must also be + * removed. + */ + rr->new_btree.oinfo = XFS_RMAP_OINFO_AG; + + /* Reset the AGF counters now that we've changed the btree shape. */ + error = xrep_rmap_reset_counters(rr); + if (error) + goto err_newbt; + + /* Dispose of any unused blocks and the accounting information. */ + error = xrep_newbt_commit(&rr->new_btree); + if (error) + return error; + + return xrep_roll_ag_trans(sc); + +err_level: + pag->pagf_repair_rmap_level = 0; +err_mcur: + xfs_btree_del_cursor(rr->mcur, error); +err_cur: + xfs_btree_del_cursor(rmap_cur, error); +err_newbt: + xrep_newbt_cancel(&rr->new_btree); + return error; +} + +/* Section (IV): Reaping the old btree. */ + +struct xrep_rmap_find_gaps { + struct xagb_bitmap rmap_gaps; + xfs_agblock_t next_agbno; +}; + +/* Subtract each free extent in the bnobt from the rmap gaps. */ +STATIC int +xrep_rmap_find_freesp( + struct xfs_btree_cur *cur, + const struct xfs_alloc_rec_incore *rec, + void *priv) +{ + struct xrep_rmap_find_gaps *rfg = priv; + + return xagb_bitmap_clear(&rfg->rmap_gaps, rec->ar_startblock, + rec->ar_blockcount); +} + +/* Record the free space we find, as part of cleaning out the btree. */ +STATIC int +xrep_rmap_find_gaps( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv) +{ + struct xrep_rmap_find_gaps *rfg = priv; + int error; + + if (rec->rm_startblock > rfg->next_agbno) { + error = xagb_bitmap_set(&rfg->rmap_gaps, rfg->next_agbno, + rec->rm_startblock - rfg->next_agbno); + if (error) + return error; + } + + rfg->next_agbno = max_t(xfs_agblock_t, rfg->next_agbno, + rec->rm_startblock + rec->rm_blockcount); + return 0; +} + +/* + * Reap the old rmapbt blocks. Now that the rmapbt is fully rebuilt, we make + * a list of gaps in the rmap records and a list of the extents mentioned in + * the bnobt. Any block that's in the new rmapbt gap list but not mentioned + * in the bnobt is a block from the old rmapbt and can be removed. + */ +STATIC int +xrep_rmap_remove_old_tree( + struct xrep_rmap *rr) +{ + struct xrep_rmap_find_gaps rfg = { + .next_agbno = 0, + }; + struct xfs_scrub *sc = rr->sc; + struct xfs_agf *agf = sc->sa.agf_bp->b_addr; + struct xfs_perag *pag = sc->sa.pag; + struct xfs_btree_cur *mcur; + xfs_agblock_t agend; + int error; + + xagb_bitmap_init(&rfg.rmap_gaps); + + /* Compute free space from the new rmapbt. */ + mcur = xfs_rmapbt_mem_cursor(rr->sc->sa.pag, NULL, &rr->rmap_btree); + + error = xfs_rmap_query_all(mcur, xrep_rmap_find_gaps, &rfg); + xfs_btree_del_cursor(mcur, error); + if (error) + goto out_bitmap; + + /* Insert a record for space between the last rmap and EOAG. */ + agend = be32_to_cpu(agf->agf_length); + if (rfg.next_agbno < agend) { + error = xagb_bitmap_set(&rfg.rmap_gaps, rfg.next_agbno, + agend - rfg.next_agbno); + if (error) + goto out_bitmap; + } + + /* Compute free space from the existing bnobt. */ + sc->sa.bno_cur = xfs_bnobt_init_cursor(sc->mp, sc->tp, sc->sa.agf_bp, + sc->sa.pag); + error = xfs_alloc_query_all(sc->sa.bno_cur, xrep_rmap_find_freesp, + &rfg); + xfs_btree_del_cursor(sc->sa.bno_cur, error); + sc->sa.bno_cur = NULL; + if (error) + goto out_bitmap; + + /* + * Free the "free" blocks that the new rmapbt knows about but the bnobt + * doesn't--these are the old rmapbt blocks. Credit the old rmapbt + * block usage count back to the per-AG rmapbt reservation (and not + * fdblocks, since the rmap btree lives in free space) to keep the + * reservation and free space accounting correct. + */ + error = xrep_reap_agblocks(sc, &rfg.rmap_gaps, + &XFS_RMAP_OINFO_ANY_OWNER, XFS_AG_RESV_RMAPBT); + if (error) + goto out_bitmap; + + /* + * Now that we've zapped all the old rmapbt blocks we can turn off + * the alternate height mechanism and reset the per-AG space + * reservation. + */ + pag->pagf_repair_rmap_level = 0; + sc->flags |= XREP_RESET_PERAG_RESV; +out_bitmap: + xagb_bitmap_destroy(&rfg.rmap_gaps); + return error; +} + +static inline bool +xrep_rmapbt_want_live_update( + struct xchk_iscan *iscan, + const struct xfs_owner_info *oi) +{ + if (xchk_iscan_aborted(iscan)) + return false; + + /* + * Before unlocking the AG header to perform the inode scan, we + * recorded reverse mappings for all AG metadata except for the OWN_AG + * metadata. IOWs, the in-memory btree knows about the AG headers, the + * two inode btrees, the CoW staging extents, and the refcount btrees. + * For these types of metadata, we need to record the live updates in + * the in-memory rmap btree. + * + * However, we do not scan the free space btrees or the AGFL until we + * have re-locked the AGF and are ready to reserve space for the new + * rmap btree, so we do not want live updates for OWN_AG metadata. + */ + if (XFS_RMAP_NON_INODE_OWNER(oi->oi_owner)) + return oi->oi_owner != XFS_RMAP_OWN_AG; + + /* Ignore updates to files that the scanner hasn't visited yet. */ + return xchk_iscan_want_live_update(iscan, oi->oi_owner); +} + +/* + * Apply a rmapbt update from the regular filesystem into our shadow btree. + * We're running from the thread that owns the AGF buffer and is generating + * the update, so we must be careful about which parts of the struct xrep_rmap + * that we change. + */ +static int +xrep_rmapbt_live_update( + struct notifier_block *nb, + unsigned long action, + void *data) +{ + struct xfs_rmap_update_params *p = data; + struct xrep_rmap *rr; + struct xfs_mount *mp; + struct xfs_btree_cur *mcur; + struct xfs_trans *tp; + int error; + + rr = container_of(nb, struct xrep_rmap, rhook.rmap_hook.nb); + mp = rr->sc->mp; + + if (!xrep_rmapbt_want_live_update(&rr->iscan, &p->oinfo)) + goto out_unlock; + + trace_xrep_rmap_live_update(pag_group(rr->sc->sa.pag), action, p); + + tp = xfs_trans_alloc_empty(mp); + + mutex_lock(&rr->lock); + mcur = xfs_rmapbt_mem_cursor(rr->sc->sa.pag, tp, &rr->rmap_btree); + error = __xfs_rmap_finish_intent(mcur, action, p->startblock, + p->blockcount, &p->oinfo, p->unwritten); + xfs_btree_del_cursor(mcur, error); + if (error) + goto out_cancel; + + error = xfbtree_trans_commit(&rr->rmap_btree, tp); + if (error) + goto out_cancel; + + xfs_trans_cancel(tp); + mutex_unlock(&rr->lock); + return NOTIFY_DONE; + +out_cancel: + xfbtree_trans_cancel(&rr->rmap_btree, tp); + xfs_trans_cancel(tp); + mutex_unlock(&rr->lock); + xchk_iscan_abort(&rr->iscan); +out_unlock: + return NOTIFY_DONE; +} + +/* Set up the filesystem scan components. */ +STATIC int +xrep_rmap_setup_scan( + struct xrep_rmap *rr) +{ + struct xfs_scrub *sc = rr->sc; + int error; + + mutex_init(&rr->lock); + + /* Set up in-memory rmap btree */ + error = xfs_rmapbt_mem_init(sc->mp, &rr->rmap_btree, sc->xmbtp, + pag_agno(sc->sa.pag)); + if (error) + goto out_mutex; + + /* Retry iget every tenth of a second for up to 30 seconds. */ + xchk_iscan_start(sc, 30000, 100, &rr->iscan); + + /* + * Hook into live rmap operations so that we can update our in-memory + * btree to reflect live changes on the filesystem. Since we drop the + * AGF buffer to scan all the inodes, we need this piece to avoid + * installing a stale btree. + */ + ASSERT(sc->flags & XCHK_FSGATES_RMAP); + xfs_rmap_hook_setup(&rr->rhook, xrep_rmapbt_live_update); + error = xfs_rmap_hook_add(pag_group(sc->sa.pag), &rr->rhook); + if (error) + goto out_iscan; + return 0; + +out_iscan: + xchk_iscan_teardown(&rr->iscan); + xfbtree_destroy(&rr->rmap_btree); +out_mutex: + mutex_destroy(&rr->lock); + return error; +} + +/* Tear down scan components. */ +STATIC void +xrep_rmap_teardown( + struct xrep_rmap *rr) +{ + struct xfs_scrub *sc = rr->sc; + + xchk_iscan_abort(&rr->iscan); + xfs_rmap_hook_del(pag_group(sc->sa.pag), &rr->rhook); + xchk_iscan_teardown(&rr->iscan); + xfbtree_destroy(&rr->rmap_btree); + mutex_destroy(&rr->lock); +} + +/* Repair the rmap btree for some AG. */ +int +xrep_rmapbt( + struct xfs_scrub *sc) +{ + struct xrep_rmap *rr = sc->buf; + int error; + + error = xrep_rmap_setup_scan(rr); + if (error) + return error; + + /* + * Collect rmaps for everything in this AG that isn't space metadata. + * These rmaps won't change even as we try to allocate blocks. + */ + error = xrep_rmap_find_rmaps(rr); + if (error) + goto out_records; + + /* Rebuild the rmap information. */ + error = xrep_rmap_build_new_tree(rr); + if (error) + goto out_records; + + /* Kill the old tree. */ + error = xrep_rmap_remove_old_tree(rr); + if (error) + goto out_records; + +out_records: + xrep_rmap_teardown(rr); + return error; +} diff --git a/fs/xfs/scrub/rtb_bitmap.h b/fs/xfs/scrub/rtb_bitmap.h new file mode 100644 index 000000000..1313ef605 --- /dev/null +++ b/fs/xfs/scrub/rtb_bitmap.h @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2022-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_RTB_BITMAP_H__ +#define __XFS_SCRUB_RTB_BITMAP_H__ + +/* Bitmaps, but for type-checked for xfs_rtblock_t */ + +struct xrtb_bitmap { + struct xbitmap64 rtbitmap; +}; + +static inline void xrtb_bitmap_init(struct xrtb_bitmap *bitmap) +{ + xbitmap64_init(&bitmap->rtbitmap); +} + +static inline void xrtb_bitmap_destroy(struct xrtb_bitmap *bitmap) +{ + xbitmap64_destroy(&bitmap->rtbitmap); +} + +static inline int xrtb_bitmap_set(struct xrtb_bitmap *bitmap, + xfs_rtblock_t start, xfs_filblks_t len) +{ + return xbitmap64_set(&bitmap->rtbitmap, start, len); +} + +static inline int xrtb_bitmap_walk(struct xrtb_bitmap *bitmap, + xbitmap64_walk_fn fn, void *priv) +{ + return xbitmap64_walk(&bitmap->rtbitmap, fn, priv); +} + +#endif /* __XFS_SCRUB_RTB_BITMAP_H__ */ diff --git a/fs/xfs/scrub/rtbitmap.c b/fs/xfs/scrub/rtbitmap.c new file mode 100644 index 000000000..52c24d3d4 --- /dev/null +++ b/fs/xfs/scrub/rtbitmap.c @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_btree.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_rtbitmap.h" +#include "xfs_inode.h" +#include "xfs_bmap.h" +#include "xfs_bit.h" +#include "xfs_rtgroup.h" +#include "xfs_sb.h" +#include "xfs_rmap.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_exchmaps.h" +#include "xfs_zone_alloc.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/repair.h" +#include "scrub/tempexch.h" +#include "scrub/rtbitmap.h" +#include "scrub/btree.h" + +/* Set us up with the realtime metadata locked. */ +int +xchk_setup_rtbitmap( + struct xfs_scrub *sc) +{ + struct xfs_mount *mp = sc->mp; + struct xchk_rtbitmap *rtb; + int error; + + if (xchk_need_intent_drain(sc)) + xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN); + + rtb = kzalloc_flex(*rtb, words, xchk_rtbitmap_wordcnt(sc), + XCHK_GFP_FLAGS); + if (!rtb) + return -ENOMEM; + sc->buf = rtb; + rtb->sc = sc; + + error = xchk_rtgroup_init(sc, sc->sm->sm_agno, &sc->sr); + if (error) + return error; + + if (xchk_could_repair(sc)) { + error = xrep_setup_rtbitmap(sc, rtb); + if (error) + return error; + } + + error = xchk_trans_alloc(sc, rtb->resblks); + if (error) + return error; + + error = xchk_install_live_inode(sc, rtg_bitmap(sc->sr.rtg)); + if (error) + return error; + + error = xchk_ino_dqattach(sc); + if (error) + return error; + + error = xchk_rtgroup_lock(sc, &sc->sr, XCHK_RTGLOCK_ALL); + if (error) + return error; + + /* + * Now that we've locked the rtbitmap, we can't race with growfsrt + * trying to expand the bitmap or change the size of the rt volume. + * Hence it is safe to compute and check the geometry values. + */ + if (mp->m_sb.sb_rblocks) { + rtb->rextents = xfs_blen_to_rtbxlen(mp, mp->m_sb.sb_rblocks); + rtb->rextslog = xfs_compute_rextslog(rtb->rextents); + rtb->rbmblocks = xfs_rtbitmap_blockcount(mp); + } + + return 0; +} + +/* Per-rtgroup bitmap contents. */ + +/* Cross-reference rtbitmap entries with other metadata. */ +STATIC void +xchk_rtbitmap_xref( + struct xchk_rtbitmap *rtb, + xfs_rtblock_t startblock, + xfs_rtblock_t blockcount) +{ + struct xfs_scrub *sc = rtb->sc; + xfs_rgblock_t rgbno = xfs_rtb_to_rgbno(sc->mp, startblock); + + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return; + if (!sc->sr.rmap_cur) + return; + + xchk_xref_has_no_rt_owner(sc, rgbno, blockcount); + xchk_xref_is_not_rt_shared(sc, rgbno, blockcount); + xchk_xref_is_not_rt_cow_staging(sc, rgbno, blockcount); + + if (rtb->next_free_rgbno < rgbno) + xchk_xref_has_rt_owner(sc, rtb->next_free_rgbno, + rgbno - rtb->next_free_rgbno); + rtb->next_free_rgbno = rgbno + blockcount; +} + +/* Scrub a free extent record from the realtime bitmap. */ +STATIC int +xchk_rtbitmap_rec( + struct xfs_rtgroup *rtg, + struct xfs_trans *tp, + const struct xfs_rtalloc_rec *rec, + void *priv) +{ + struct xchk_rtbitmap *rtb = priv; + struct xfs_scrub *sc = rtb->sc; + xfs_rtblock_t startblock; + xfs_filblks_t blockcount; + + startblock = xfs_rtx_to_rtb(rtg, rec->ar_startext); + blockcount = xfs_rtxlen_to_extlen(rtg_mount(rtg), rec->ar_extcount); + + if (!xfs_verify_rtbext(rtg_mount(rtg), startblock, blockcount)) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); + + xchk_rtbitmap_xref(rtb, startblock, blockcount); + + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return -ECANCELED; + + return 0; +} + +/* Make sure the entire rtbitmap file is mapped with written extents. */ +STATIC int +xchk_rtbitmap_check_extents( + struct xfs_scrub *sc) +{ + struct xfs_bmbt_irec map; + struct xfs_iext_cursor icur; + struct xfs_mount *mp = sc->mp; + struct xfs_inode *ip = sc->ip; + xfs_fileoff_t off = 0; + xfs_fileoff_t endoff; + int error = 0; + + /* Mappings may not cross or lie beyond EOF. */ + endoff = XFS_B_TO_FSB(mp, ip->i_disk_size); + if (xfs_iext_lookup_extent(ip, &ip->i_df, endoff, &icur, &map)) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, endoff); + return 0; + } + + while (off < endoff) { + int nmap = 1; + + if (xchk_should_terminate(sc, &error) || + (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) + break; + + /* Make sure we have a written extent. */ + error = xfs_bmapi_read(ip, off, endoff - off, &map, &nmap, + XFS_DATA_FORK); + if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, off, &error)) + break; + + if (nmap != 1 || !xfs_bmap_is_written_extent(&map)) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, off); + break; + } + + off += map.br_blockcount; + } + + return error; +} + +/* Scrub this group's realtime bitmap. */ +int +xchk_rtbitmap( + struct xfs_scrub *sc) +{ + struct xfs_mount *mp = sc->mp; + struct xfs_rtgroup *rtg = sc->sr.rtg; + struct xfs_inode *rbmip = rtg_bitmap(rtg); + struct xchk_rtbitmap *rtb = sc->buf; + xfs_rgblock_t last_rgbno; + int error; + + /* Is sb_rextents correct? */ + if (mp->m_sb.sb_rextents != rtb->rextents) { + xchk_ip_set_corrupt(sc, rbmip); + return 0; + } + + /* Is sb_rextslog correct? */ + if (mp->m_sb.sb_rextslog != rtb->rextslog) { + xchk_ip_set_corrupt(sc, rbmip); + return 0; + } + + /* + * Is sb_rbmblocks large enough to handle the current rt volume? In no + * case can we exceed 4bn bitmap blocks since the super field is a u32. + */ + if (rtb->rbmblocks > U32_MAX) { + xchk_ip_set_corrupt(sc, rbmip); + return 0; + } + if (mp->m_sb.sb_rbmblocks != rtb->rbmblocks) { + xchk_ip_set_corrupt(sc, rbmip); + return 0; + } + + /* The bitmap file length must be aligned to an fsblock. */ + if (rbmip->i_disk_size & mp->m_blockmask) { + xchk_ip_set_corrupt(sc, rbmip); + return 0; + } + + /* + * Is the bitmap file itself large enough to handle the rt volume? + * growfsrt expands the bitmap file before updating sb_rextents, so the + * file can be larger than sb_rbmblocks. + */ + if (rbmip->i_disk_size < XFS_FSB_TO_B(mp, rtb->rbmblocks)) { + xchk_ip_set_corrupt(sc, rbmip); + return 0; + } + + /* Invoke the fork scrubber. */ + error = xchk_metadata_inode_forks(sc); + if (error || (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) + return error; + + error = xchk_rtbitmap_check_extents(sc); + if (error || (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) + return error; + + rtb->next_free_rgbno = 0; + error = xfs_rtalloc_query_all(rtg, sc->tp, xchk_rtbitmap_rec, rtb); + if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, 0, &error)) + return error; + + /* + * Check that the are rmappings for all rt extents between the end of + * the last free extent we saw and the last possible extent in the rt + * group. + */ + last_rgbno = rtg->rtg_extents * mp->m_sb.sb_rextsize; + if (rtb->next_free_rgbno < last_rgbno) + xchk_xref_has_rt_owner(sc, rtb->next_free_rgbno, + last_rgbno - rtb->next_free_rgbno); + return 0; +} + +/* xref check that the extent is not free in the rtbitmap */ +void +xchk_xref_is_used_rt_space( + struct xfs_scrub *sc, + xfs_rtblock_t rtbno, + xfs_extlen_t len) +{ + struct xfs_rtgroup *rtg = sc->sr.rtg; + xfs_rtxnum_t startext; + xfs_rtxnum_t endext; + bool is_free; + int error; + + if (xchk_skip_xref(sc->sm)) + return; + + if (xfs_has_zoned(sc->mp)) { + if (!xfs_zone_rgbno_is_valid(rtg, + xfs_rtb_to_rgbno(sc->mp, rtbno) + len - 1)) + xchk_ip_xref_set_corrupt(sc, rtg_rmap(rtg)); + return; + } + + startext = xfs_rtb_to_rtx(sc->mp, rtbno); + endext = xfs_rtb_to_rtx(sc->mp, rtbno + len - 1); + error = xfs_rtalloc_extent_is_free(rtg, sc->tp, startext, + endext - startext + 1, &is_free); + if (!xchk_should_check_xref(sc, &error, NULL)) + return; + if (is_free) + xchk_ip_xref_set_corrupt(sc, rtg_bitmap(rtg)); +} diff --git a/fs/xfs/scrub/rtbitmap.h b/fs/xfs/scrub/rtbitmap.h new file mode 100644 index 000000000..fe52b8772 --- /dev/null +++ b/fs/xfs/scrub/rtbitmap.h @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_RTBITMAP_H__ +#define __XFS_SCRUB_RTBITMAP_H__ + +/* + * We use an xfile to construct new bitmap blocks for the portion of the + * rtbitmap file that we're replacing. Whereas the ondisk bitmap must be + * accessed through the buffer cache, the xfile bitmap supports direct + * word-level accesses. Therefore, we create a small abstraction for linear + * access. + */ +typedef unsigned long long xrep_wordoff_t; +typedef unsigned int xrep_wordcnt_t; + +/* Mask to round an rtx down to the nearest bitmap word. */ +#define XREP_RTBMP_WORDMASK ((1ULL << XFS_NBWORDLOG) - 1) + + +struct xchk_rtbitmap { + struct xfs_scrub *sc; + + uint64_t rextents; + uint64_t rbmblocks; + unsigned int rextslog; + unsigned int resblks; + + /* The next free rt group block number that we expect to see. */ + xfs_rgblock_t next_free_rgbno; + +#ifdef CONFIG_XFS_ONLINE_REPAIR + /* stuff for staging a new bitmap */ + struct xfs_rtalloc_args args; + struct xrep_tempexch tempexch; +#endif + + /* The next rtgroup block we expect to see during our rtrmapbt walk. */ + xfs_rgblock_t next_rgbno; + + /* rtgroup lock flags */ + unsigned int rtglock_flags; + + /* rtword position of xfile as we write buffers to disk. */ + xrep_wordoff_t prep_wordoff; + + /* In-Memory rtbitmap for repair. */ + union xfs_rtword_raw words[]; +}; + +#ifdef CONFIG_XFS_ONLINE_REPAIR +int xrep_setup_rtbitmap(struct xfs_scrub *sc, struct xchk_rtbitmap *rtb); + +/* + * How big should the words[] buffer be? + * + * For repairs, we want a full fsblock worth of space so that we can memcpy a + * buffer full of 1s into the xfile bitmap. The xfile bitmap doesn't have + * rtbitmap block headers, so we don't use blockwsize. Scrub doesn't use the + * words buffer at all. + */ +static inline unsigned int +xchk_rtbitmap_wordcnt( + struct xfs_scrub *sc) +{ + if (xchk_could_repair(sc)) + return sc->mp->m_sb.sb_blocksize >> XFS_WORDLOG; + return 0; +} +#else +# define xrep_setup_rtbitmap(sc, rtb) (0) +# define xchk_rtbitmap_wordcnt(sc) (0) +#endif /* CONFIG_XFS_ONLINE_REPAIR */ + +#endif /* __XFS_SCRUB_RTBITMAP_H__ */ diff --git a/fs/xfs/scrub/rtbitmap_repair.c b/fs/xfs/scrub/rtbitmap_repair.c new file mode 100644 index 000000000..442a17bf9 --- /dev/null +++ b/fs/xfs/scrub/rtbitmap_repair.c @@ -0,0 +1,643 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2020-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_btree.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_rtalloc.h" +#include "xfs_inode.h" +#include "xfs_bit.h" +#include "xfs_bmap.h" +#include "xfs_bmap_btree.h" +#include "xfs_rmap.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_exchmaps.h" +#include "xfs_rtbitmap.h" +#include "xfs_rtgroup.h" +#include "xfs_extent_busy.h" +#include "xfs_refcount.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/xfile.h" +#include "scrub/tempfile.h" +#include "scrub/tempexch.h" +#include "scrub/reap.h" +#include "scrub/rtbitmap.h" + +/* rt bitmap content repairs */ + +/* + * Reserve enough blocks to write out a completely new bitmap file, plus twice + * as many blocks as we would need if we can only allocate one block per data + * fork mapping. This should cover the preallocation of the temporary file and + * exchanging the extent mappings. + * + * We cannot use xfs_exchmaps_estimate because we have not yet constructed the + * replacement bitmap and therefore do not know how many extents it will use. + * By the time we do, we will have a dirty transaction (which we cannot drop + * because we cannot drop the rtbitmap ILOCK) and cannot ask for more + * reservation. + */ +static inline unsigned long long +xrep_rtbitmap_calc_blocks(struct xfs_mount *mp, unsigned long long blocks) +{ + return blocks + (xfs_bmbt_calc_size(mp, blocks) * 2); +} + +/* Set up to repair the realtime bitmap for this group. */ +int +xrep_setup_rtbitmap( + struct xfs_scrub *sc, + struct xchk_rtbitmap *rtb) +{ + struct xfs_mount *mp = sc->mp; + unsigned long long blocks = mp->m_sb.sb_rbmblocks; + int error; + + error = xrep_tempfile_create(sc, S_IFREG); + if (error) + return error; + + /* Create an xfile to hold our reconstructed bitmap. */ + error = xfile_create("realtime bitmap file", + blocks * mp->m_sb.sb_blocksize, &sc->xfile); + if (error) + return error; + + blocks = xrep_rtbitmap_calc_blocks(mp, mp->m_sb.sb_rbmblocks); + if (blocks > UINT_MAX) + return -EOPNOTSUPP; + + rtb->resblks += blocks; + return 0; +} + +static inline xrep_wordoff_t +rtx_to_wordoff( + struct xfs_mount *mp, + xfs_rtxnum_t rtx) +{ + return rtx >> XFS_NBWORDLOG; +} + +static inline xrep_wordcnt_t +rtxlen_to_wordcnt( + xfs_rtxlen_t rtxlen) +{ + return rtxlen >> XFS_NBWORDLOG; +} + +/* Helper functions to record rtwords in an xfile. */ + +static inline int +xfbmp_load( + struct xchk_rtbitmap *rtb, + xrep_wordoff_t wordoff, + xfs_rtword_t *word) +{ + union xfs_rtword_raw urk; + int error; + + ASSERT(xfs_has_rtgroups(rtb->sc->mp)); + + error = xfile_load(rtb->sc->xfile, &urk, + sizeof(union xfs_rtword_raw), + wordoff << XFS_WORDLOG); + if (error) + return error; + + *word = be32_to_cpu(urk.rtg); + return 0; +} + +static inline int +xfbmp_store( + struct xchk_rtbitmap *rtb, + xrep_wordoff_t wordoff, + const xfs_rtword_t word) +{ + union xfs_rtword_raw urk; + + ASSERT(xfs_has_rtgroups(rtb->sc->mp)); + + urk.rtg = cpu_to_be32(word); + return xfile_store(rtb->sc->xfile, &urk, + sizeof(union xfs_rtword_raw), + wordoff << XFS_WORDLOG); +} + +static inline int +xfbmp_copyin( + struct xchk_rtbitmap *rtb, + xrep_wordoff_t wordoff, + const union xfs_rtword_raw *word, + xrep_wordcnt_t nr_words) +{ + return xfile_store(rtb->sc->xfile, word, nr_words << XFS_WORDLOG, + wordoff << XFS_WORDLOG); +} + +static inline int +xfbmp_copyout( + struct xchk_rtbitmap *rtb, + xrep_wordoff_t wordoff, + union xfs_rtword_raw *word, + xrep_wordcnt_t nr_words) +{ + return xfile_load(rtb->sc->xfile, word, nr_words << XFS_WORDLOG, + wordoff << XFS_WORDLOG); +} + +/* Perform a logical OR operation on an rtword in the incore bitmap. */ +static int +xrep_rtbitmap_or( + struct xchk_rtbitmap *rtb, + xrep_wordoff_t wordoff, + xfs_rtword_t mask) +{ + xfs_rtword_t word; + int error; + + error = xfbmp_load(rtb, wordoff, &word); + if (error) + return error; + + trace_xrep_rtbitmap_or(rtb->sc->mp, wordoff, mask, word); + + return xfbmp_store(rtb, wordoff, word | mask); +} + +/* + * Mark as free every rt extent between the next rt block we expected to see + * in the rtrmap records and the given rt block. + */ +STATIC int +xrep_rtbitmap_mark_free( + struct xchk_rtbitmap *rtb, + xfs_rgblock_t rgbno) +{ + struct xfs_mount *mp = rtb->sc->mp; + struct xchk_rt *sr = &rtb->sc->sr; + struct xfs_rtgroup *rtg = sr->rtg; + xfs_rtxnum_t startrtx; + xfs_rtxnum_t nextrtx; + xrep_wordoff_t wordoff, nextwordoff; + unsigned int bit; + unsigned int bufwsize; + xfs_extlen_t mod; + xfs_rtword_t mask; + enum xbtree_recpacking outcome; + int error; + + if (!xfs_verify_rgbext(rtg, rtb->next_rgbno, rgbno - rtb->next_rgbno)) + return -EFSCORRUPTED; + + /* + * Convert rt blocks to rt extents The block range we find must be + * aligned to an rtextent boundary on both ends. + */ + startrtx = xfs_rgbno_to_rtx(mp, rtb->next_rgbno); + mod = xfs_rgbno_to_rtxoff(mp, rtb->next_rgbno); + if (mod) + return -EFSCORRUPTED; + + nextrtx = xfs_rgbno_to_rtx(mp, rgbno - 1) + 1; + mod = xfs_rgbno_to_rtxoff(mp, rgbno - 1); + if (mod != mp->m_sb.sb_rextsize - 1) + return -EFSCORRUPTED; + + /* Must not be shared or CoW staging. */ + if (sr->refc_cur) { + error = xfs_refcount_has_records(sr->refc_cur, + XFS_REFC_DOMAIN_SHARED, rtb->next_rgbno, + rgbno - rtb->next_rgbno, &outcome); + if (error) + return error; + if (outcome != XBTREE_RECPACKING_EMPTY) + return -EFSCORRUPTED; + + error = xfs_refcount_has_records(sr->refc_cur, + XFS_REFC_DOMAIN_COW, rtb->next_rgbno, + rgbno - rtb->next_rgbno, &outcome); + if (error) + return error; + if (outcome != XBTREE_RECPACKING_EMPTY) + return -EFSCORRUPTED; + } + + trace_xrep_rtbitmap_record_free(mp, startrtx, nextrtx - 1); + + /* Set bits as needed to round startrtx up to the nearest word. */ + bit = startrtx & XREP_RTBMP_WORDMASK; + if (bit) { + xfs_rtblock_t len = nextrtx - startrtx; + unsigned int lastbit; + + lastbit = min(bit + len, XFS_NBWORD); + mask = (((xfs_rtword_t)1 << (lastbit - bit)) - 1) << bit; + + error = xrep_rtbitmap_or(rtb, rtx_to_wordoff(mp, startrtx), + mask); + if (error || lastbit - bit == len) + return error; + startrtx += XFS_NBWORD - bit; + } + + /* Set bits as needed to round nextrtx down to the nearest word. */ + bit = nextrtx & XREP_RTBMP_WORDMASK; + if (bit) { + mask = ((xfs_rtword_t)1 << bit) - 1; + + error = xrep_rtbitmap_or(rtb, rtx_to_wordoff(mp, nextrtx), + mask); + if (error || startrtx + bit == nextrtx) + return error; + nextrtx -= bit; + } + + trace_xrep_rtbitmap_record_free_bulk(mp, startrtx, nextrtx - 1); + + /* Set all the words in between, up to a whole fs block at once. */ + wordoff = rtx_to_wordoff(mp, startrtx); + nextwordoff = rtx_to_wordoff(mp, nextrtx); + bufwsize = mp->m_sb.sb_blocksize >> XFS_WORDLOG; + + while (wordoff < nextwordoff) { + xrep_wordoff_t rem; + xrep_wordcnt_t wordcnt; + + wordcnt = min_t(xrep_wordcnt_t, nextwordoff - wordoff, + bufwsize); + + /* + * Try to keep us aligned to the rtwords buffer to reduce the + * number of xfile writes. + */ + rem = wordoff & (bufwsize - 1); + if (rem) + wordcnt = min_t(xrep_wordcnt_t, wordcnt, + bufwsize - rem); + + error = xfbmp_copyin(rtb, wordoff, rtb->words, wordcnt); + if (error) + return error; + + wordoff += wordcnt; + } + + return 0; +} + +/* Set free space in the rtbitmap based on rtrmapbt records. */ +STATIC int +xrep_rtbitmap_walk_rtrmap( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv) +{ + struct xchk_rtbitmap *rtb = priv; + int error = 0; + + if (xchk_should_terminate(rtb->sc, &error)) + return error; + + if (rtb->next_rgbno < rec->rm_startblock) { + error = xrep_rtbitmap_mark_free(rtb, rec->rm_startblock); + if (error) + return error; + } + + rtb->next_rgbno = max(rtb->next_rgbno, + rec->rm_startblock + rec->rm_blockcount); + return 0; +} + +/* + * Walk the rtrmapbt to find all the gaps between records, and mark the gaps + * in the realtime bitmap that we're computing. + */ +STATIC int +xrep_rtbitmap_find_freespace( + struct xchk_rtbitmap *rtb) +{ + struct xfs_scrub *sc = rtb->sc; + struct xfs_mount *mp = sc->mp; + struct xfs_rtgroup *rtg = sc->sr.rtg; + uint64_t blockcount; + int error; + + /* Prepare a buffer of ones so that we can accelerate bulk setting. */ + memset(rtb->words, 0xFF, mp->m_sb.sb_blocksize); + + xrep_rtgroup_btcur_init(sc, &sc->sr); + error = xfs_rmap_query_all(sc->sr.rmap_cur, xrep_rtbitmap_walk_rtrmap, + rtb); + if (error) + goto out; + + /* + * Mark as free every possible rt extent from the last one we saw to + * the end of the rt group. + */ + blockcount = rtg->rtg_extents * mp->m_sb.sb_rextsize; + if (rtb->next_rgbno < blockcount) { + error = xrep_rtbitmap_mark_free(rtb, blockcount); + if (error) + goto out; + } + +out: + xchk_rtgroup_btcur_free(&sc->sr); + return error; +} + +static int +xrep_rtbitmap_prep_buf( + struct xfs_scrub *sc, + struct xfs_buf *bp, + void *data) +{ + struct xchk_rtbitmap *rtb = data; + struct xfs_mount *mp = sc->mp; + union xfs_rtword_raw *ondisk; + int error; + + rtb->args.mp = sc->mp; + rtb->args.tp = sc->tp; + rtb->args.rbmbp = bp; + ondisk = xfs_rbmblock_wordptr(&rtb->args, 0); + rtb->args.rbmbp = NULL; + + error = xfbmp_copyout(rtb, rtb->prep_wordoff, ondisk, + mp->m_blockwsize); + if (error) + return error; + + if (xfs_has_rtgroups(sc->mp)) { + struct xfs_rtbuf_blkinfo *hdr = bp->b_addr; + + hdr->rt_magic = cpu_to_be32(XFS_RTBITMAP_MAGIC); + hdr->rt_owner = cpu_to_be64(I_INO(sc->ip)); + hdr->rt_blkno = cpu_to_be64(xfs_buf_daddr(bp)); + hdr->rt_lsn = 0; + uuid_copy(&hdr->rt_uuid, &sc->mp->m_sb.sb_meta_uuid); + bp->b_ops = &xfs_rtbitmap_buf_ops; + } else { + bp->b_ops = &xfs_rtbuf_ops; + } + + rtb->prep_wordoff += mp->m_blockwsize; + xfs_trans_buf_set_type(sc->tp, bp, XFS_BLFT_RTBITMAP_BUF); + return 0; +} + +/* + * Make sure that the given range of the data fork of the realtime file is + * mapped to written blocks. The caller must ensure that the inode is joined + * to the transaction. + */ +STATIC int +xrep_rtbitmap_data_mappings( + struct xfs_scrub *sc, + xfs_filblks_t len) +{ + struct xfs_bmbt_irec map; + xfs_fileoff_t off = 0; + int error; + + ASSERT(sc->ip != NULL); + + while (off < len) { + int nmaps = 1; + + /* + * If we have a real extent mapping this block then we're + * in ok shape. + */ + error = xfs_bmapi_read(sc->ip, off, len - off, &map, &nmaps, + XFS_DATA_FORK); + if (error) + return error; + if (nmaps == 0) { + ASSERT(nmaps != 0); + return -EFSCORRUPTED; + } + + /* + * Written extents are ok. Holes are not filled because we + * do not know the freespace information. + */ + if (xfs_bmap_is_written_extent(&map) || + map.br_startblock == HOLESTARTBLOCK) { + off = map.br_startoff + map.br_blockcount; + continue; + } + + /* + * If we find a delalloc reservation then something is very + * very wrong. Bail out. + */ + if (map.br_startblock == DELAYSTARTBLOCK) + return -EFSCORRUPTED; + + /* Make sure we're really converting an unwritten extent. */ + if (map.br_state != XFS_EXT_UNWRITTEN) { + ASSERT(map.br_state == XFS_EXT_UNWRITTEN); + return -EFSCORRUPTED; + } + + /* Make sure this block has a real zeroed extent mapped. */ + nmaps = 1; + error = xfs_bmapi_write(sc->tp, sc->ip, map.br_startoff, + map.br_blockcount, + XFS_BMAPI_CONVERT | XFS_BMAPI_ZERO, + 0, &map, &nmaps); + if (error) + return error; + + /* Commit new extent and all deferred work. */ + error = xrep_defer_finish(sc); + if (error) + return error; + + off = map.br_startoff + map.br_blockcount; + } + + return 0; +} + +/* Fix broken rt volume geometry. */ +STATIC int +xrep_rtbitmap_geometry( + struct xfs_scrub *sc, + struct xchk_rtbitmap *rtb) +{ + struct xfs_mount *mp = sc->mp; + struct xfs_trans *tp = sc->tp; + + /* Superblock fields */ + if (mp->m_sb.sb_rextents != rtb->rextents) + xfs_trans_mod_sb(sc->tp, XFS_TRANS_SB_REXTENTS, + rtb->rextents - mp->m_sb.sb_rextents); + + if (mp->m_sb.sb_rbmblocks != rtb->rbmblocks) + xfs_trans_mod_sb(tp, XFS_TRANS_SB_RBMBLOCKS, + rtb->rbmblocks - mp->m_sb.sb_rbmblocks); + + if (mp->m_sb.sb_rextslog != rtb->rextslog) + xfs_trans_mod_sb(tp, XFS_TRANS_SB_REXTSLOG, + rtb->rextslog - mp->m_sb.sb_rextslog); + + /* Fix broken isize */ + sc->ip->i_disk_size = roundup_64(sc->ip->i_disk_size, + mp->m_sb.sb_blocksize); + + if (sc->ip->i_disk_size < XFS_FSB_TO_B(mp, rtb->rbmblocks)) + sc->ip->i_disk_size = XFS_FSB_TO_B(mp, rtb->rbmblocks); + + xfs_trans_log_inode(sc->tp, sc->ip, XFS_ILOG_CORE); + return xrep_roll_trans(sc); +} + +/* Repair the realtime bitmap file metadata. */ +int +xrep_rtbitmap( + struct xfs_scrub *sc) +{ + struct xchk_rtbitmap *rtb = sc->buf; + struct xfs_mount *mp = sc->mp; + struct xfs_group *xg = rtg_group(sc->sr.rtg); + unsigned long long blocks; + unsigned int busy_gen; + int error; + + /* We require the realtime rmapbt to rebuild anything. */ + if (!xfs_has_rtrmapbt(sc->mp)) + return -EOPNOTSUPP; + /* We require atomic file exchange range to rebuild anything. */ + if (!xfs_has_exchange_range(sc->mp)) + return -EOPNOTSUPP; + + /* Impossibly large rtbitmap means we can't touch the filesystem. */ + if (rtb->rbmblocks > U32_MAX) + return 0; + + /* + * If the size of the rt bitmap file is larger than what we reserved, + * figure out if we need to adjust the block reservation in the + * transaction. + */ + blocks = xrep_rtbitmap_calc_blocks(mp, rtb->rbmblocks); + if (blocks > UINT_MAX) + return -EOPNOTSUPP; + if (blocks > rtb->resblks) { + uint64_t delta = blocks - rtb->resblks; + + if (delta > UINT_MAX) + return -EOPNOTSUPP; + + error = xfs_trans_reserve_more(sc->tp, delta, 0); + if (error) + return error; + + rtb->resblks += delta; + } + + /* Fix inode core and forks. */ + error = xrep_metadata_inode_forks(sc); + if (error) + return error; + + xfs_trans_ijoin(sc->tp, sc->ip, 0); + + /* Ensure no unwritten extents. */ + error = xrep_rtbitmap_data_mappings(sc, rtb->rbmblocks); + if (error) + return error; + + /* + * Fix inconsistent bitmap geometry. This function returns with a + * clean scrub transaction. + */ + error = xrep_rtbitmap_geometry(sc, rtb); + if (error) + return error; + + /* + * Make sure the busy extent list is clear because we can't put extents + * on there twice. + */ + if (!xfs_extent_busy_list_empty(xg, &busy_gen)) { + error = xfs_extent_busy_flush(sc->tp, xg, busy_gen, 0); + if (error) + return error; + } + + /* + * Generate the new rtbitmap data. We don't need the rtbmp information + * once this call is finished. + */ + error = xrep_rtbitmap_find_freespace(rtb); + if (error) + return error; + + /* + * Try to take ILOCK_EXCL of the temporary file. We had better be the + * only ones holding onto this inode, but we can't block while holding + * the rtbitmap file's ILOCK_EXCL. + */ + while (!xrep_tempfile_ilock_nowait(sc)) { + if (xchk_should_terminate(sc, &error)) + return error; + delay(1); + } + + /* + * Make sure we have space allocated for the part of the bitmap + * file that corresponds to this group. We already joined sc->ip. + */ + xfs_trans_ijoin(sc->tp, sc->tempip, 0); + error = xrep_tempfile_prealloc(sc, 0, rtb->rbmblocks); + if (error) + return error; + + /* Last chance to abort before we start committing fixes. */ + if (xchk_should_terminate(sc, &error)) + return error; + + /* Copy the bitmap file that we generated. */ + error = xrep_tempfile_copyin(sc, 0, rtb->rbmblocks, + xrep_rtbitmap_prep_buf, rtb); + if (error) + return error; + error = xrep_tempfile_set_isize(sc, + XFS_FSB_TO_B(sc->mp, sc->mp->m_sb.sb_rbmblocks)); + if (error) + return error; + + /* + * Now exchange the data fork contents. We're done with the temporary + * buffer, so we can reuse it for the tempfile exchmaps information. + */ + error = xrep_tempexch_trans_reserve(sc, XFS_DATA_FORK, 0, + rtb->rbmblocks, &rtb->tempexch); + if (error) + return error; + + error = xrep_tempexch_contents(sc, &rtb->tempexch); + if (error) + return error; + + /* Free the old rtbitmap blocks if they're not in use. */ + return xrep_reap_ifork(sc, sc->tempip, XFS_DATA_FORK); +} diff --git a/fs/xfs/scrub/rtrefcount.c b/fs/xfs/scrub/rtrefcount.c new file mode 100644 index 000000000..4e7c540c8 --- /dev/null +++ b/fs/xfs/scrub/rtrefcount.c @@ -0,0 +1,660 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2021-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_trans.h" +#include "xfs_btree.h" +#include "xfs_rmap.h" +#include "xfs_refcount.h" +#include "xfs_inode.h" +#include "xfs_rtbitmap.h" +#include "xfs_rtgroup.h" +#include "xfs_metafile.h" +#include "xfs_rtrefcount_btree.h" +#include "xfs_rtalloc.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/btree.h" +#include "scrub/repair.h" + +/* Set us up with the realtime refcount metadata locked. */ +int +xchk_setup_rtrefcountbt( + struct xfs_scrub *sc) +{ + int error; + + if (xchk_need_intent_drain(sc)) + xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN); + + if (xchk_could_repair(sc)) { + error = xrep_setup_rtrefcountbt(sc); + if (error) + return error; + } + + error = xchk_rtgroup_init(sc, sc->sm->sm_agno, &sc->sr); + if (error) + return error; + + error = xchk_setup_rt(sc); + if (error) + return error; + + error = xchk_install_live_inode(sc, rtg_refcount(sc->sr.rtg)); + if (error) + return error; + + return xchk_rtgroup_lock(sc, &sc->sr, XCHK_RTGLOCK_ALL); +} + +/* Realtime Reference count btree scrubber. */ + +/* + * Confirming Reference Counts via Reverse Mappings + * + * We want to count the reverse mappings overlapping a refcount record + * (bno, len, refcount), allowing for the possibility that some of the + * overlap may come from smaller adjoining reverse mappings, while some + * comes from single extents which overlap the range entirely. The + * outer loop is as follows: + * + * 1. For all reverse mappings overlapping the refcount extent, + * a. If a given rmap completely overlaps, mark it as seen. + * b. Otherwise, record the fragment (in agbno order) for later + * processing. + * + * Once we've seen all the rmaps, we know that for all blocks in the + * refcount record we want to find $refcount owners and we've already + * visited $seen extents that overlap all the blocks. Therefore, we + * need to find ($refcount - $seen) owners for every block in the + * extent; call that quantity $target_nr. Proceed as follows: + * + * 2. Pull the first $target_nr fragments from the list; all of them + * should start at or before the start of the extent. + * Call this subset of fragments the working set. + * 3. Until there are no more unprocessed fragments, + * a. Find the shortest fragments in the set and remove them. + * b. Note the block number of the end of these fragments. + * c. Pull the same number of fragments from the list. All of these + * fragments should start at the block number recorded in the + * previous step. + * d. Put those fragments in the set. + * 4. Check that there are $target_nr fragments remaining in the list, + * and that they all end at or beyond the end of the refcount extent. + * + * If the refcount is correct, all the check conditions in the algorithm + * should always hold true. If not, the refcount is incorrect. + */ +struct xchk_rtrefcnt_frag { + struct list_head list; + struct xfs_rmap_irec rm; +}; + +struct xchk_rtrefcnt_check { + struct xfs_scrub *sc; + struct list_head fragments; + + /* refcount extent we're examining */ + xfs_rgblock_t bno; + xfs_extlen_t len; + xfs_nlink_t refcount; + + /* number of owners seen */ + xfs_nlink_t seen; +}; + +/* + * Decide if the given rmap is large enough that we can redeem it + * towards refcount verification now, or if it's a fragment, in + * which case we'll hang onto it in the hopes that we'll later + * discover that we've collected exactly the correct number of + * fragments as the rtrefcountbt says we should have. + */ +STATIC int +xchk_rtrefcountbt_rmap_check( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv) +{ + struct xchk_rtrefcnt_check *refchk = priv; + struct xchk_rtrefcnt_frag *frag; + xfs_rgblock_t rm_last; + xfs_rgblock_t rc_last; + int error = 0; + + if (xchk_should_terminate(refchk->sc, &error)) + return error; + + rm_last = rec->rm_startblock + rec->rm_blockcount - 1; + rc_last = refchk->bno + refchk->len - 1; + + /* Confirm that a single-owner refc extent is a CoW stage. */ + if (refchk->refcount == 1 && rec->rm_owner != XFS_RMAP_OWN_COW) { + xchk_btree_xref_set_corrupt(refchk->sc, cur, 0); + return 0; + } + + if (rec->rm_startblock <= refchk->bno && rm_last >= rc_last) { + /* + * The rmap overlaps the refcount record, so we can confirm + * one refcount owner seen. + */ + refchk->seen++; + } else { + /* + * This rmap covers only part of the refcount record, so + * save the fragment for later processing. If the rmapbt + * is healthy each rmap_irec we see will be in agbno order + * so we don't need insertion sort here. + */ + frag = kmalloc_obj(struct xchk_rtrefcnt_frag, XCHK_GFP_FLAGS); + if (!frag) + return -ENOMEM; + memcpy(&frag->rm, rec, sizeof(frag->rm)); + list_add_tail(&frag->list, &refchk->fragments); + } + + return 0; +} + +/* + * Given a bunch of rmap fragments, iterate through them, keeping + * a running tally of the refcount. If this ever deviates from + * what we expect (which is the rtrefcountbt's refcount minus the + * number of extents that totally covered the rtrefcountbt extent), + * we have a rtrefcountbt error. + */ +STATIC void +xchk_rtrefcountbt_process_rmap_fragments( + struct xchk_rtrefcnt_check *refchk) +{ + struct list_head worklist; + struct xchk_rtrefcnt_frag *frag; + struct xchk_rtrefcnt_frag *n; + xfs_rgblock_t bno; + xfs_rgblock_t rbno; + xfs_rgblock_t next_rbno; + xfs_nlink_t nr; + xfs_nlink_t target_nr; + + target_nr = refchk->refcount - refchk->seen; + if (target_nr == 0) + return; + + /* + * There are (refchk->rc.rc_refcount - refchk->nr refcount) + * references we haven't found yet. Pull that many off the + * fragment list and figure out where the smallest rmap ends + * (and therefore the next rmap should start). All the rmaps + * we pull off should start at or before the beginning of the + * refcount record's range. + */ + INIT_LIST_HEAD(&worklist); + rbno = NULLRGBLOCK; + + /* Make sure the fragments actually /are/ in bno order. */ + bno = 0; + list_for_each_entry(frag, &refchk->fragments, list) { + if (frag->rm.rm_startblock < bno) + goto done; + bno = frag->rm.rm_startblock; + } + + /* + * Find all the rmaps that start at or before the refc extent, + * and put them on the worklist. + */ + nr = 0; + list_for_each_entry_safe(frag, n, &refchk->fragments, list) { + if (frag->rm.rm_startblock > refchk->bno || nr > target_nr) + break; + bno = frag->rm.rm_startblock + frag->rm.rm_blockcount; + if (bno < rbno) + rbno = bno; + list_move_tail(&frag->list, &worklist); + nr++; + } + + /* + * We should have found exactly $target_nr rmap fragments starting + * at or before the refcount extent. + */ + if (nr != target_nr) + goto done; + + while (!list_empty(&refchk->fragments)) { + /* Discard any fragments ending at rbno from the worklist. */ + nr = 0; + next_rbno = NULLRGBLOCK; + list_for_each_entry_safe(frag, n, &worklist, list) { + bno = frag->rm.rm_startblock + frag->rm.rm_blockcount; + if (bno != rbno) { + if (bno < next_rbno) + next_rbno = bno; + continue; + } + list_del(&frag->list); + kfree(frag); + nr++; + } + + /* Try to add nr rmaps starting at rbno to the worklist. */ + list_for_each_entry_safe(frag, n, &refchk->fragments, list) { + bno = frag->rm.rm_startblock + frag->rm.rm_blockcount; + if (frag->rm.rm_startblock != rbno) + goto done; + list_move_tail(&frag->list, &worklist); + if (next_rbno > bno) + next_rbno = bno; + nr--; + if (nr == 0) + break; + } + + /* + * If we get here and nr > 0, this means that we added fewer + * items to the worklist than we discarded because the fragment + * list ran out of items. Therefore, we cannot maintain the + * required refcount. Something is wrong, so we're done. + */ + if (nr) + goto done; + + rbno = next_rbno; + } + + /* + * Make sure the last extent we processed ends at or beyond + * the end of the refcount extent. + */ + if (rbno < refchk->bno + refchk->len) + goto done; + + /* Actually record us having seen the remaining refcount. */ + refchk->seen = refchk->refcount; +done: + /* Delete fragments and work list. */ + list_for_each_entry_safe(frag, n, &worklist, list) { + list_del(&frag->list); + kfree(frag); + } + list_for_each_entry_safe(frag, n, &refchk->fragments, list) { + list_del(&frag->list); + kfree(frag); + } +} + +/* Use the rmap entries covering this extent to verify the refcount. */ +STATIC void +xchk_rtrefcountbt_xref_rmap( + struct xfs_scrub *sc, + const struct xfs_refcount_irec *irec) +{ + struct xchk_rtrefcnt_check refchk = { + .sc = sc, + .bno = irec->rc_startblock, + .len = irec->rc_blockcount, + .refcount = irec->rc_refcount, + .seen = 0, + }; + struct xfs_rmap_irec low; + struct xfs_rmap_irec high; + struct xchk_rtrefcnt_frag *frag; + struct xchk_rtrefcnt_frag *n; + int error; + + if (!sc->sr.rmap_cur || xchk_skip_xref(sc->sm)) + return; + + /* Cross-reference with the rmapbt to confirm the refcount. */ + memset(&low, 0, sizeof(low)); + low.rm_startblock = irec->rc_startblock; + memset(&high, 0xFF, sizeof(high)); + high.rm_startblock = irec->rc_startblock + irec->rc_blockcount - 1; + + INIT_LIST_HEAD(&refchk.fragments); + error = xfs_rmap_query_range(sc->sr.rmap_cur, &low, &high, + xchk_rtrefcountbt_rmap_check, &refchk); + if (!xchk_should_check_xref(sc, &error, &sc->sr.rmap_cur)) + goto out_free; + + xchk_rtrefcountbt_process_rmap_fragments(&refchk); + if (irec->rc_refcount != refchk.seen) + xchk_btree_xref_set_corrupt(sc, sc->sr.rmap_cur, 0); + +out_free: + list_for_each_entry_safe(frag, n, &refchk.fragments, list) { + list_del(&frag->list); + kfree(frag); + } +} + +/* Cross-reference with the other btrees. */ +STATIC void +xchk_rtrefcountbt_xref( + struct xfs_scrub *sc, + const struct xfs_refcount_irec *irec) +{ + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return; + + xchk_xref_is_used_rt_space(sc, + xfs_rgbno_to_rtb(sc->sr.rtg, irec->rc_startblock), + irec->rc_blockcount); + xchk_rtrefcountbt_xref_rmap(sc, irec); +} + +struct xchk_rtrefcbt_records { + /* Previous refcount record. */ + struct xfs_refcount_irec prev_rec; + + /* The next rtgroup block where we aren't expecting shared extents. */ + xfs_rgblock_t next_unshared_rgbno; + + /* Number of CoW blocks we expect. */ + xfs_extlen_t cow_blocks; + + /* Was the last record a shared or CoW staging extent? */ + enum xfs_refc_domain prev_domain; +}; + +static inline bool +xchk_rtrefcount_mergeable( + struct xchk_rtrefcbt_records *rrc, + const struct xfs_refcount_irec *r2) +{ + const struct xfs_refcount_irec *r1 = &rrc->prev_rec; + + /* Ignore if prev_rec is not yet initialized. */ + if (r1->rc_blockcount > 0) + return false; + + if (r1->rc_startblock + r1->rc_blockcount != r2->rc_startblock) + return false; + if (r1->rc_refcount != r2->rc_refcount) + return false; + if ((unsigned long long)r1->rc_blockcount + r2->rc_blockcount > + XFS_REFC_LEN_MAX) + return false; + + return true; +} + +/* Flag failures for records that could be merged. */ +STATIC void +xchk_rtrefcountbt_check_mergeable( + struct xchk_btree *bs, + struct xchk_rtrefcbt_records *rrc, + const struct xfs_refcount_irec *irec) +{ + if (bs->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return; + + if (xchk_rtrefcount_mergeable(rrc, irec)) + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + + memcpy(&rrc->prev_rec, irec, sizeof(struct xfs_refcount_irec)); +} + +STATIC int +xchk_rtrefcountbt_rmap_check_gap( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv) +{ + xfs_rgblock_t *next_bno = priv; + + if (*next_bno != NULLRGBLOCK && rec->rm_startblock < *next_bno) + return -ECANCELED; + + *next_bno = rec->rm_startblock + rec->rm_blockcount; + return 0; +} + +/* + * Make sure that a gap in the reference count records does not correspond to + * overlapping records (i.e. shared extents) in the reverse mappings. + */ +static inline void +xchk_rtrefcountbt_xref_gaps( + struct xfs_scrub *sc, + struct xchk_rtrefcbt_records *rrc, + xfs_rtblock_t bno) +{ + struct xfs_rmap_irec low; + struct xfs_rmap_irec high; + xfs_rgblock_t next_bno = NULLRGBLOCK; + int error; + + if (bno <= rrc->next_unshared_rgbno || !sc->sr.rmap_cur || + xchk_skip_xref(sc->sm)) + return; + + memset(&low, 0, sizeof(low)); + low.rm_startblock = rrc->next_unshared_rgbno; + memset(&high, 0xFF, sizeof(high)); + high.rm_startblock = bno - 1; + + error = xfs_rmap_query_range(sc->sr.rmap_cur, &low, &high, + xchk_rtrefcountbt_rmap_check_gap, &next_bno); + if (error == -ECANCELED) + xchk_btree_xref_set_corrupt(sc, sc->sr.rmap_cur, 0); + else + xchk_should_check_xref(sc, &error, &sc->sr.rmap_cur); +} + +/* Scrub a rtrefcountbt record. */ +STATIC int +xchk_rtrefcountbt_rec( + struct xchk_btree *bs, + const union xfs_btree_rec *rec) +{ + struct xfs_mount *mp = bs->cur->bc_mp; + struct xchk_rtrefcbt_records *rrc = bs->private; + struct xfs_refcount_irec irec; + u32 mod; + + xfs_refcount_btrec_to_irec(rec, &irec); + if (xfs_rtrefcount_check_irec(to_rtg(bs->cur->bc_group), &irec) != + NULL) { + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + return 0; + } + + /* We can only share full rt extents. */ + mod = xfs_rgbno_to_rtxoff(mp, irec.rc_startblock); + if (mod) + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + mod = xfs_extlen_to_rtxmod(mp, irec.rc_blockcount); + if (mod) + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + + if (irec.rc_domain == XFS_REFC_DOMAIN_COW) + rrc->cow_blocks += irec.rc_blockcount; + + /* Shared records always come before CoW records. */ + if (irec.rc_domain == XFS_REFC_DOMAIN_SHARED && + rrc->prev_domain == XFS_REFC_DOMAIN_COW) + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + rrc->prev_domain = irec.rc_domain; + + xchk_rtrefcountbt_check_mergeable(bs, rrc, &irec); + xchk_rtrefcountbt_xref(bs->sc, &irec); + + /* + * If this is a record for a shared extent, check that all blocks + * between the previous record and this one have at most one reverse + * mapping. + */ + if (irec.rc_domain == XFS_REFC_DOMAIN_SHARED) { + xchk_rtrefcountbt_xref_gaps(bs->sc, rrc, irec.rc_startblock); + rrc->next_unshared_rgbno = irec.rc_startblock + + irec.rc_blockcount; + } + + return 0; +} + +/* Make sure we have as many refc blocks as the rmap says. */ +STATIC void +xchk_refcount_xref_rmap( + struct xfs_scrub *sc, + const struct xfs_owner_info *btree_oinfo, + xfs_extlen_t cow_blocks) +{ + xfs_filblks_t refcbt_blocks = 0; + xfs_filblks_t blocks; + int error; + + if (!sc->sr.rmap_cur || !sc->sa.rmap_cur || xchk_skip_xref(sc->sm)) + return; + + /* Check that we saw as many refcbt blocks as the rmap knows about. */ + error = xfs_btree_count_blocks(sc->sr.refc_cur, &refcbt_blocks); + if (!xchk_btree_process_error(sc, sc->sr.refc_cur, 0, &error)) + return; + error = xchk_count_rmap_ownedby_ag(sc, sc->sa.rmap_cur, btree_oinfo, + &blocks); + if (!xchk_should_check_xref(sc, &error, &sc->sa.rmap_cur)) + return; + if (blocks != refcbt_blocks) + xchk_btree_xref_set_corrupt(sc, sc->sa.rmap_cur, 0); + + /* Check that we saw as many cow blocks as the rmap knows about. */ + error = xchk_count_rmap_ownedby_ag(sc, sc->sr.rmap_cur, + &XFS_RMAP_OINFO_COW, &blocks); + if (!xchk_should_check_xref(sc, &error, &sc->sr.rmap_cur)) + return; + if (blocks != cow_blocks) + xchk_btree_xref_set_corrupt(sc, sc->sr.rmap_cur, 0); +} + +/* Scrub the refcount btree for some AG. */ +int +xchk_rtrefcountbt( + struct xfs_scrub *sc) +{ + struct xfs_owner_info btree_oinfo; + struct xchk_rtrefcbt_records rrc = { + .cow_blocks = 0, + .next_unshared_rgbno = 0, + .prev_domain = XFS_REFC_DOMAIN_SHARED, + }; + int error; + + error = xchk_metadata_inode_forks(sc); + if (error || (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) + return error; + + xfs_rmap_inode_bmbt_owner(&btree_oinfo, rtg_refcount(sc->sr.rtg), + XFS_DATA_FORK); + error = xchk_btree(sc, sc->sr.refc_cur, xchk_rtrefcountbt_rec, + &btree_oinfo, &rrc); + if (error || (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) + return error; + + /* + * Check that all blocks between the last refcount > 1 record and the + * end of the rt volume have at most one reverse mapping. + */ + xchk_rtrefcountbt_xref_gaps(sc, &rrc, sc->mp->m_sb.sb_rblocks); + + xchk_refcount_xref_rmap(sc, &btree_oinfo, rrc.cow_blocks); + + return 0; +} + +/* xref check that a cow staging extent is marked in the rtrefcountbt. */ +void +xchk_xref_is_rt_cow_staging( + struct xfs_scrub *sc, + xfs_rgblock_t bno, + xfs_extlen_t len) +{ + struct xfs_refcount_irec rc; + int has_refcount; + int error; + + if (!sc->sr.refc_cur || xchk_skip_xref(sc->sm)) + return; + + /* Find the CoW staging extent. */ + error = xfs_refcount_lookup_le(sc->sr.refc_cur, XFS_REFC_DOMAIN_COW, + bno, &has_refcount); + if (!xchk_should_check_xref(sc, &error, &sc->sr.refc_cur)) + return; + if (!has_refcount) { + xchk_btree_xref_set_corrupt(sc, sc->sr.refc_cur, 0); + return; + } + + error = xfs_refcount_get_rec(sc->sr.refc_cur, &rc, &has_refcount); + if (!xchk_should_check_xref(sc, &error, &sc->sr.refc_cur)) + return; + if (!has_refcount) { + xchk_btree_xref_set_corrupt(sc, sc->sr.refc_cur, 0); + return; + } + + /* CoW lookup returned a shared extent record? */ + if (rc.rc_domain != XFS_REFC_DOMAIN_COW) + xchk_btree_xref_set_corrupt(sc, sc->sr.refc_cur, 0); + + /* Must be at least as long as what was passed in */ + if (rc.rc_blockcount < len) + xchk_btree_xref_set_corrupt(sc, sc->sr.refc_cur, 0); +} + +/* + * xref check that the extent is not shared. Only file data blocks + * can have multiple owners. + */ +void +xchk_xref_is_not_rt_shared( + struct xfs_scrub *sc, + xfs_rgblock_t bno, + xfs_extlen_t len) +{ + enum xbtree_recpacking outcome; + int error; + + if (!sc->sr.refc_cur || xchk_skip_xref(sc->sm)) + return; + + error = xfs_refcount_has_records(sc->sr.refc_cur, + XFS_REFC_DOMAIN_SHARED, bno, len, &outcome); + if (!xchk_should_check_xref(sc, &error, &sc->sr.refc_cur)) + return; + if (outcome != XBTREE_RECPACKING_EMPTY) + xchk_btree_xref_set_corrupt(sc, sc->sr.refc_cur, 0); +} + +/* xref check that the extent is not being used for CoW staging. */ +void +xchk_xref_is_not_rt_cow_staging( + struct xfs_scrub *sc, + xfs_rgblock_t bno, + xfs_extlen_t len) +{ + enum xbtree_recpacking outcome; + int error; + + if (!sc->sr.refc_cur || xchk_skip_xref(sc->sm)) + return; + + error = xfs_refcount_has_records(sc->sr.refc_cur, XFS_REFC_DOMAIN_COW, + bno, len, &outcome); + if (!xchk_should_check_xref(sc, &error, &sc->sr.refc_cur)) + return; + if (outcome != XBTREE_RECPACKING_EMPTY) + xchk_btree_xref_set_corrupt(sc, sc->sr.refc_cur, 0); +} diff --git a/fs/xfs/scrub/rtrefcount_repair.c b/fs/xfs/scrub/rtrefcount_repair.c new file mode 100644 index 000000000..2b939960c --- /dev/null +++ b/fs/xfs/scrub/rtrefcount_repair.c @@ -0,0 +1,752 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2021-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_btree.h" +#include "xfs_btree_staging.h" +#include "xfs_bit.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_sb.h" +#include "xfs_alloc.h" +#include "xfs_ialloc.h" +#include "xfs_rmap.h" +#include "xfs_rmap_btree.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_refcount.h" +#include "xfs_rtrefcount_btree.h" +#include "xfs_error.h" +#include "xfs_health.h" +#include "xfs_inode.h" +#include "xfs_quota.h" +#include "xfs_rtalloc.h" +#include "xfs_ag.h" +#include "xfs_rtgroup.h" +#include "xfs_rtbitmap.h" +#include "scrub/xfs_scrub.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/btree.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/bitmap.h" +#include "scrub/fsb_bitmap.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/newbt.h" +#include "scrub/reap.h" +#include "scrub/rcbag.h" + +/* + * Rebuilding the Reference Count Btree + * ==================================== + * + * This algorithm is "borrowed" from xfs_repair. Imagine the rmap + * entries as rectangles representing extents of physical blocks, and + * that the rectangles can be laid down to allow them to overlap each + * other; then we know that we must emit a refcnt btree entry wherever + * the amount of overlap changes, i.e. the emission stimulus is + * level-triggered: + * + * - --- + * -- ----- ---- --- ------ + * -- ---- ----------- ---- --------- + * -------------------------------- ----------- + * ^ ^ ^^ ^^ ^ ^^ ^^^ ^^^^ ^ ^^ ^ ^ ^ + * 2 1 23 21 3 43 234 2123 1 01 2 3 0 + * + * For our purposes, a rmap is a tuple (startblock, len, fileoff, owner). + * + * Note that in the actual refcnt btree we don't store the refcount < 2 + * cases because the bnobt tells us which blocks are free; single-use + * blocks aren't recorded in the bnobt or the refcntbt. If the rmapbt + * supports storing multiple entries covering a given block we could + * theoretically dispense with the refcntbt and simply count rmaps, but + * that's inefficient in the (hot) write path, so we'll take the cost of + * the extra tree to save time. Also there's no guarantee that rmap + * will be enabled. + * + * Given an array of rmaps sorted by physical block number, a starting + * physical block (sp), a bag to hold rmaps that cover sp, and the next + * physical block where the level changes (np), we can reconstruct the + * rt refcount btree as follows: + * + * While there are still unprocessed rmaps in the array, + * - Set sp to the physical block (pblk) of the next unprocessed rmap. + * - Add to the bag all rmaps in the array where startblock == sp. + * - Set np to the physical block where the bag size will change. This + * is the minimum of (the pblk of the next unprocessed rmap) and + * (startblock + len of each rmap in the bag). + * - Record the bag size as old_bag_size. + * + * - While the bag isn't empty, + * - Remove from the bag all rmaps where startblock + len == np. + * - Add to the bag all rmaps in the array where startblock == np. + * - If the bag size isn't old_bag_size, store the refcount entry + * (sp, np - sp, bag_size) in the refcnt btree. + * - If the bag is empty, break out of the inner loop. + * - Set old_bag_size to the bag size + * - Set sp = np. + * - Set np to the physical block where the bag size will change. + * This is the minimum of (the pblk of the next unprocessed rmap) + * and (startblock + len of each rmap in the bag). + * + * Like all the other repairers, we make a list of all the refcount + * records we need, then reinitialize the rt refcount btree root and + * insert all the records. + */ + +struct xrep_rtrefc { + /* refcount extents */ + struct xfarray *refcount_records; + + /* new refcountbt information */ + struct xrep_newbt new_btree; + + /* old refcountbt blocks */ + struct xfsb_bitmap old_rtrefcountbt_blocks; + + struct xfs_scrub *sc; + + /* get_records()'s position in the rt refcount record array. */ + xfarray_idx_t array_cur; + + /* # of refcountbt blocks */ + xfs_filblks_t btblocks; +}; + +/* Set us up to repair refcount btrees. */ +int +xrep_setup_rtrefcountbt( + struct xfs_scrub *sc) +{ + return xrep_setup_xfbtree(sc, "realtime rmap record bag"); +} + +/* Check for any obvious conflicts with this shared/CoW staging extent. */ +STATIC int +xrep_rtrefc_check_ext( + struct xfs_scrub *sc, + const struct xfs_refcount_irec *rec) +{ + xfs_rgblock_t last; + + if (xfs_rtrefcount_check_irec(sc->sr.rtg, rec) != NULL) + return -EFSCORRUPTED; + + if (xfs_rgbno_to_rtxoff(sc->mp, rec->rc_startblock) != 0) + return -EFSCORRUPTED; + + last = rec->rc_startblock + rec->rc_blockcount - 1; + if (xfs_rgbno_to_rtxoff(sc->mp, last) != sc->mp->m_sb.sb_rextsize - 1) + return -EFSCORRUPTED; + + /* Make sure this isn't free space or misaligned. */ + return xrep_require_rtext_inuse(sc, rec->rc_startblock, + rec->rc_blockcount); +} + +/* Record a reference count extent. */ +STATIC int +xrep_rtrefc_stash( + struct xrep_rtrefc *rr, + enum xfs_refc_domain domain, + xfs_rgblock_t bno, + xfs_extlen_t len, + uint64_t refcount) +{ + struct xfs_refcount_irec irec = { + .rc_startblock = bno, + .rc_blockcount = len, + .rc_refcount = refcount, + .rc_domain = domain, + }; + int error = 0; + + if (xchk_should_terminate(rr->sc, &error)) + return error; + + irec.rc_refcount = min_t(uint64_t, XFS_REFC_REFCOUNT_MAX, refcount); + + error = xrep_rtrefc_check_ext(rr->sc, &irec); + if (error) + return error; + + trace_xrep_refc_found(rtg_group(rr->sc->sr.rtg), &irec); + + return xfarray_append(rr->refcount_records, &irec); +} + +/* Record a CoW staging extent. */ +STATIC int +xrep_rtrefc_stash_cow( + struct xrep_rtrefc *rr, + xfs_rgblock_t bno, + xfs_extlen_t len) +{ + return xrep_rtrefc_stash(rr, XFS_REFC_DOMAIN_COW, bno, len, 1); +} + +/* Decide if an rmap could describe a shared extent. */ +static inline bool +xrep_rtrefc_rmap_shareable( + const struct xfs_rmap_irec *rmap) +{ + /* rt metadata are never sharable */ + if (XFS_RMAP_NON_INODE_OWNER(rmap->rm_owner)) + return false; + + /* Unwritten file blocks are not shareable. */ + if (rmap->rm_flags & XFS_RMAP_UNWRITTEN) + return false; + + return true; +} + +/* Grab the next (abbreviated) rmap record from the rmapbt. */ +STATIC int +xrep_rtrefc_walk_rmaps( + struct xrep_rtrefc *rr, + struct xfs_rmap_irec *rmap, + bool *have_rec) +{ + struct xfs_btree_cur *cur = rr->sc->sr.rmap_cur; + struct xfs_mount *mp = cur->bc_mp; + int have_gt; + int error = 0; + + *have_rec = false; + + /* + * Loop through the remaining rmaps. Remember CoW staging + * extents and the refcountbt blocks from the old tree for later + * disposal. We can only share written data fork extents, so + * keep looping until we find an rmap for one. + */ + do { + if (xchk_should_terminate(rr->sc, &error)) + return error; + + error = xfs_btree_increment(cur, 0, &have_gt); + if (error) + return error; + if (!have_gt) + return 0; + + error = xfs_rmap_get_rec(cur, rmap, &have_gt); + if (error) + return error; + if (XFS_IS_CORRUPT(mp, !have_gt)) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + + if (rmap->rm_owner == XFS_RMAP_OWN_COW) { + error = xrep_rtrefc_stash_cow(rr, rmap->rm_startblock, + rmap->rm_blockcount); + if (error) + return error; + } else if (xfs_is_sb_inum(mp, rmap->rm_owner) || + (rmap->rm_flags & (XFS_RMAP_ATTR_FORK | + XFS_RMAP_BMBT_BLOCK))) { + xfs_btree_mark_sick(cur); + return -EFSCORRUPTED; + } + } while (!xrep_rtrefc_rmap_shareable(rmap)); + + *have_rec = true; + return 0; +} + +static inline uint32_t +xrep_rtrefc_encode_startblock( + const struct xfs_refcount_irec *irec) +{ + uint32_t start; + + start = irec->rc_startblock & ~XFS_REFC_COWFLAG; + if (irec->rc_domain == XFS_REFC_DOMAIN_COW) + start |= XFS_REFC_COWFLAG; + + return start; +} + +/* + * Compare two refcount records. We want to sort in order of increasing block + * number. + */ +static int +xrep_rtrefc_extent_cmp( + const void *a, + const void *b) +{ + const struct xfs_refcount_irec *ap = a; + const struct xfs_refcount_irec *bp = b; + uint32_t sa, sb; + + sa = xrep_rtrefc_encode_startblock(ap); + sb = xrep_rtrefc_encode_startblock(bp); + + if (sa > sb) + return 1; + if (sa < sb) + return -1; + return 0; +} + +/* + * Sort the refcount extents by startblock or else the btree records will be in + * the wrong order. Make sure the records do not overlap in physical space. + */ +STATIC int +xrep_rtrefc_sort_records( + struct xrep_rtrefc *rr) +{ + struct xfs_refcount_irec irec; + xfarray_idx_t cur; + enum xfs_refc_domain dom = XFS_REFC_DOMAIN_SHARED; + xfs_rgblock_t next_rgbno = 0; + int error; + + error = xfarray_sort(rr->refcount_records, xrep_rtrefc_extent_cmp, + XFARRAY_SORT_KILLABLE); + if (error) + return error; + + foreach_xfarray_idx(rr->refcount_records, cur) { + if (xchk_should_terminate(rr->sc, &error)) + return error; + + error = xfarray_load(rr->refcount_records, cur, &irec); + if (error) + return error; + + if (dom == XFS_REFC_DOMAIN_SHARED && + irec.rc_domain == XFS_REFC_DOMAIN_COW) { + dom = irec.rc_domain; + next_rgbno = 0; + } + + if (dom != irec.rc_domain) + return -EFSCORRUPTED; + if (irec.rc_startblock < next_rgbno) + return -EFSCORRUPTED; + + next_rgbno = irec.rc_startblock + irec.rc_blockcount; + } + + return error; +} + +/* Record extents that belong to the realtime refcount inode. */ +STATIC int +xrep_rtrefc_walk_rmap( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv) +{ + struct xrep_rtrefc *rr = priv; + int error = 0; + + if (xchk_should_terminate(rr->sc, &error)) + return error; + + /* Skip extents which are not owned by this inode and fork. */ + if (rec->rm_owner != I_INO(rr->sc->ip)) + return 0; + + error = xrep_check_ino_btree_mapping(rr->sc, rec); + if (error) + return error; + + return xfsb_bitmap_set(&rr->old_rtrefcountbt_blocks, + xfs_gbno_to_fsb(cur->bc_group, rec->rm_startblock), + rec->rm_blockcount); +} + +/* + * Walk forward through the rmap btree to collect all rmaps starting at + * @bno in @rmap_bag. These represent the file(s) that share ownership of + * the current block. Upon return, the rmap cursor points to the last record + * satisfying the startblock constraint. + */ +static int +xrep_rtrefc_push_rmaps_at( + struct xrep_rtrefc *rr, + struct rcbag *rcstack, + xfs_rgblock_t bno, + struct xfs_rmap_irec *rmap, + bool *have) +{ + struct xfs_scrub *sc = rr->sc; + int have_gt; + int error; + + while (*have && rmap->rm_startblock == bno) { + error = rcbag_add(rcstack, rr->sc->tp, rmap); + if (error) + return error; + + error = xrep_rtrefc_walk_rmaps(rr, rmap, have); + if (error) + return error; + } + + error = xfs_btree_decrement(sc->sr.rmap_cur, 0, &have_gt); + if (error) + return error; + if (XFS_IS_CORRUPT(sc->mp, !have_gt)) { + xfs_btree_mark_sick(sc->sr.rmap_cur); + return -EFSCORRUPTED; + } + + return 0; +} + +/* Scan one AG for reverse mappings for the realtime refcount btree. */ +STATIC int +xrep_rtrefc_scan_ag( + struct xrep_rtrefc *rr, + struct xfs_perag *pag) +{ + struct xfs_scrub *sc = rr->sc; + int error; + + error = xrep_ag_init(sc, pag, &sc->sa); + if (error) + return error; + + error = xfs_rmap_query_all(sc->sa.rmap_cur, xrep_rtrefc_walk_rmap, rr); + xchk_ag_free(sc, &sc->sa); + return error; +} + +/* Iterate all the rmap records to generate reference count data. */ +STATIC int +xrep_rtrefc_find_refcounts( + struct xrep_rtrefc *rr) +{ + struct xfs_scrub *sc = rr->sc; + struct rcbag *rcstack; + struct xfs_perag *pag = NULL; + uint64_t old_stack_height; + xfs_rgblock_t sbno; + xfs_rgblock_t cbno; + xfs_rgblock_t nbno; + bool have; + int error; + + /* Scan for old rtrefc btree blocks. */ + while ((pag = xfs_perag_next(sc->mp, pag))) { + error = xrep_rtrefc_scan_ag(rr, pag); + if (error) { + xfs_perag_rele(pag); + return error; + } + } + + xrep_rtgroup_btcur_init(sc, &sc->sr); + + /* + * Set up a bag to store all the rmap records that we're tracking to + * generate a reference count record. If this exceeds + * XFS_REFC_REFCOUNT_MAX, we clamp rc_refcount. + */ + error = rcbag_init(sc->mp, sc->xmbtp, &rcstack); + if (error) + goto out_cur; + + /* Start the rtrmapbt cursor to the left of all records. */ + error = xfs_btree_goto_left_edge(sc->sr.rmap_cur); + if (error) + goto out_bag; + + /* Process reverse mappings into refcount data. */ + while (xfs_btree_has_more_records(sc->sr.rmap_cur)) { + struct xfs_rmap_irec rmap; + + /* Push all rmaps with pblk == sbno onto the stack */ + error = xrep_rtrefc_walk_rmaps(rr, &rmap, &have); + if (error) + goto out_bag; + if (!have) + break; + sbno = cbno = rmap.rm_startblock; + error = xrep_rtrefc_push_rmaps_at(rr, rcstack, sbno, &rmap, + &have); + if (error) + goto out_bag; + + /* Set nbno to the bno of the next refcount change */ + error = rcbag_next_edge(rcstack, sc->tp, &rmap, have, &nbno); + if (error) + goto out_bag; + + ASSERT(nbno > sbno); + old_stack_height = rcbag_count(rcstack); + + /* While stack isn't empty... */ + while (rcbag_count(rcstack) > 0) { + /* Pop all rmaps that end at nbno */ + error = rcbag_remove_ending_at(rcstack, sc->tp, nbno); + if (error) + goto out_bag; + + /* Push array items that start at nbno */ + error = xrep_rtrefc_walk_rmaps(rr, &rmap, &have); + if (error) + goto out_bag; + if (have) { + error = xrep_rtrefc_push_rmaps_at(rr, rcstack, + nbno, &rmap, &have); + if (error) + goto out_bag; + } + + /* Emit refcount if necessary */ + ASSERT(nbno > cbno); + if (rcbag_count(rcstack) != old_stack_height) { + if (old_stack_height > 1) { + error = xrep_rtrefc_stash(rr, + XFS_REFC_DOMAIN_SHARED, + cbno, nbno - cbno, + old_stack_height); + if (error) + goto out_bag; + } + cbno = nbno; + } + + /* Stack empty, go find the next rmap */ + if (rcbag_count(rcstack) == 0) + break; + old_stack_height = rcbag_count(rcstack); + sbno = nbno; + + /* Set nbno to the bno of the next refcount change */ + error = rcbag_next_edge(rcstack, sc->tp, &rmap, have, + &nbno); + if (error) + goto out_bag; + + ASSERT(nbno > sbno); + } + } + + ASSERT(rcbag_count(rcstack) == 0); +out_bag: + rcbag_free(&rcstack); +out_cur: + xchk_rtgroup_btcur_free(&sc->sr); + return error; +} + +/* Retrieve refcountbt data for bulk load. */ +STATIC int +xrep_rtrefc_get_records( + struct xfs_btree_cur *cur, + unsigned int idx, + struct xfs_btree_block *block, + unsigned int nr_wanted, + void *priv) +{ + struct xrep_rtrefc *rr = priv; + union xfs_btree_rec *block_rec; + unsigned int loaded; + int error; + + for (loaded = 0; loaded < nr_wanted; loaded++, idx++) { + error = xfarray_load(rr->refcount_records, rr->array_cur++, + &cur->bc_rec.rc); + if (error) + return error; + + block_rec = xfs_btree_rec_addr(cur, idx, block); + cur->bc_ops->init_rec_from_cur(cur, block_rec); + } + + return loaded; +} + +/* Feed one of the new btree blocks to the bulk loader. */ +STATIC int +xrep_rtrefc_claim_block( + struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr, + void *priv) +{ + struct xrep_rtrefc *rr = priv; + + return xrep_newbt_claim_block(cur, &rr->new_btree, ptr); +} + +/* Figure out how much space we need to create the incore btree root block. */ +STATIC size_t +xrep_rtrefc_iroot_size( + struct xfs_btree_cur *cur, + unsigned int level, + unsigned int nr_this_level, + void *priv) +{ + return xfs_rtrefcount_broot_space_calc(cur->bc_mp, level, + nr_this_level); +} + +/* + * Use the collected refcount information to stage a new rt refcount btree. If + * this is successful we'll return with the new btree root information logged + * to the repair transaction but not yet committed. + */ +STATIC int +xrep_rtrefc_build_new_tree( + struct xrep_rtrefc *rr) +{ + struct xfs_scrub *sc = rr->sc; + struct xfs_rtgroup *rtg = sc->sr.rtg; + struct xfs_btree_cur *refc_cur; + int error; + + error = xrep_rtrefc_sort_records(rr); + if (error) + return error; + + /* + * Prepare to construct the new btree by reserving disk space for the + * new btree and setting up all the accounting information we'll need + * to root the new btree while it's under construction and before we + * attach it to the realtime refcount inode. + */ + error = xrep_newbt_init_metadir_inode(&rr->new_btree, sc); + if (error) + return error; + + rr->new_btree.bload.get_records = xrep_rtrefc_get_records; + rr->new_btree.bload.claim_block = xrep_rtrefc_claim_block; + rr->new_btree.bload.iroot_size = xrep_rtrefc_iroot_size; + + refc_cur = xfs_rtrefcountbt_init_cursor(NULL, rtg); + xfs_btree_stage_ifakeroot(refc_cur, &rr->new_btree.ifake); + + /* Compute how many blocks we'll need. */ + error = xfs_btree_bload_compute_geometry(refc_cur, &rr->new_btree.bload, + xfarray_length(rr->refcount_records)); + if (error) + goto err_cur; + + /* Last chance to abort before we start committing fixes. */ + if (xchk_should_terminate(sc, &error)) + goto err_cur; + + /* + * Guess how many blocks we're going to need to rebuild an entire + * rtrefcountbt from the number of extents we found, and pump up our + * transaction to have sufficient block reservation. We're allowed + * to exceed quota to repair inconsistent metadata, though this is + * unlikely. + */ + error = xfs_trans_reserve_more_inode(sc->tp, rtg_refcount(rtg), + rr->new_btree.bload.nr_blocks, 0, true); + if (error) + goto err_cur; + + /* Reserve the space we'll need for the new btree. */ + error = xrep_newbt_alloc_blocks(&rr->new_btree, + rr->new_btree.bload.nr_blocks); + if (error) + goto err_cur; + + /* Add all observed refcount records. */ + rr->new_btree.ifake.if_fork->if_format = XFS_DINODE_FMT_META_BTREE; + rr->array_cur = XFARRAY_CURSOR_INIT; + error = xfs_btree_bload(refc_cur, &rr->new_btree.bload, rr); + if (error) + goto err_cur; + + /* + * Install the new rtrefc btree in the inode. After this point the old + * btree is no longer accessible, the new tree is live, and we can + * delete the cursor. + */ + xfs_rtrefcountbt_commit_staged_btree(refc_cur, sc->tp); + xrep_inode_set_nblocks(rr->sc, rr->new_btree.ifake.if_blocks); + xfs_btree_del_cursor(refc_cur, 0); + + /* Dispose of any unused blocks and the accounting information. */ + error = xrep_newbt_commit(&rr->new_btree); + if (error) + return error; + + return xrep_roll_trans(sc); +err_cur: + xfs_btree_del_cursor(refc_cur, error); + xrep_newbt_cancel(&rr->new_btree); + return error; +} + +/* Rebuild the rt refcount btree. */ +int +xrep_rtrefcountbt( + struct xfs_scrub *sc) +{ + struct xrep_rtrefc *rr; + struct xfs_mount *mp = sc->mp; + int error; + + /* We require the rmapbt to rebuild anything. */ + if (!xfs_has_rtrmapbt(mp)) + return -EOPNOTSUPP; + + /* Make sure any problems with the fork are fixed. */ + error = xrep_metadata_inode_forks(sc); + if (error) + return error; + + rr = kzalloc_obj(struct xrep_rtrefc, XCHK_GFP_FLAGS); + if (!rr) + return -ENOMEM; + rr->sc = sc; + + /* Set up enough storage to handle one refcount record per rt extent. */ + error = xfarray_create("realtime reference count records", + mp->m_sb.sb_rextents, sizeof(struct xfs_refcount_irec), + &rr->refcount_records); + if (error) + goto out_rr; + + /* Collect all reference counts. */ + xfsb_bitmap_init(&rr->old_rtrefcountbt_blocks); + error = xrep_rtrefc_find_refcounts(rr); + if (error) + goto out_bitmap; + + xfs_trans_ijoin(sc->tp, sc->ip, 0); + + /* Rebuild the refcount information. */ + error = xrep_rtrefc_build_new_tree(rr); + if (error) + goto out_bitmap; + + /* + * Free all the extents that were allocated to the former rtrefcountbt + * and aren't cross-linked with something else. + */ + error = xrep_reap_metadir_fsblocks(rr->sc, + &rr->old_rtrefcountbt_blocks); + if (error) + goto out_bitmap; + +out_bitmap: + xfsb_bitmap_destroy(&rr->old_rtrefcountbt_blocks); + xfarray_destroy(rr->refcount_records); +out_rr: + kfree(rr); + return error; +} diff --git a/fs/xfs/scrub/rtrmap.c b/fs/xfs/scrub/rtrmap.c new file mode 100644 index 000000000..564d19a97 --- /dev/null +++ b/fs/xfs/scrub/rtrmap.c @@ -0,0 +1,328 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2018-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_btree.h" +#include "xfs_bit.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_sb.h" +#include "xfs_rmap.h" +#include "xfs_rmap_btree.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_inode.h" +#include "xfs_rtalloc.h" +#include "xfs_rtgroup.h" +#include "xfs_metafile.h" +#include "xfs_refcount.h" +#include "scrub/xfs_scrub.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/btree.h" +#include "scrub/trace.h" +#include "scrub/repair.h" + +/* Set us up with the realtime metadata locked. */ +int +xchk_setup_rtrmapbt( + struct xfs_scrub *sc) +{ + int error; + + if (xchk_need_intent_drain(sc)) + xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN); + + if (xchk_could_repair(sc)) { + error = xrep_setup_rtrmapbt(sc); + if (error) + return error; + } + + error = xchk_rtgroup_init(sc, sc->sm->sm_agno, &sc->sr); + if (error) + return error; + + error = xchk_setup_rt(sc); + if (error) + return error; + + error = xchk_install_live_inode(sc, rtg_rmap(sc->sr.rtg)); + if (error) + return error; + + return xchk_rtgroup_lock(sc, &sc->sr, XCHK_RTGLOCK_ALL); +} + +/* Realtime reverse mapping. */ + +struct xchk_rtrmap { + /* + * The furthest-reaching of the rmapbt records that we've already + * processed. This enables us to detect overlapping records for space + * allocations that cannot be shared. + */ + struct xfs_rmap_irec overlap_rec; + + /* + * The previous rmapbt record, so that we can check for two records + * that could be one. + */ + struct xfs_rmap_irec prev_rec; +}; + +static inline bool +xchk_rtrmapbt_is_shareable( + struct xfs_scrub *sc, + const struct xfs_rmap_irec *irec) +{ + if (!xfs_has_rtreflink(sc->mp)) + return false; + if (irec->rm_flags & XFS_RMAP_UNWRITTEN) + return false; + if (irec->rm_owner == XFS_RMAP_OWN_COW || + irec->rm_owner == XFS_RMAP_OWN_FS) + return false; + return true; +} + +/* Flag failures for records that overlap but cannot. */ +STATIC void +xchk_rtrmapbt_check_overlapping( + struct xchk_btree *bs, + struct xchk_rtrmap *cr, + const struct xfs_rmap_irec *irec) +{ + xfs_rtblock_t pnext, inext; + + if (bs->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return; + + /* No previous record? */ + if (cr->overlap_rec.rm_blockcount == 0) + goto set_prev; + + /* Do overlap_rec and irec overlap? */ + pnext = cr->overlap_rec.rm_startblock + cr->overlap_rec.rm_blockcount; + if (pnext <= irec->rm_startblock) + goto set_prev; + + /* Overlap is only allowed if both records are data fork mappings. */ + if (!xchk_rtrmapbt_is_shareable(bs->sc, &cr->overlap_rec) || + !xchk_rtrmapbt_is_shareable(bs->sc, irec)) + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + + /* Save whichever rmap record extends furthest. */ + inext = irec->rm_startblock + irec->rm_blockcount; + if (pnext > inext) + return; + +set_prev: + memcpy(&cr->overlap_rec, irec, sizeof(struct xfs_rmap_irec)); +} + +/* Decide if two reverse-mapping records can be merged. */ +static inline bool +xchk_rtrmap_mergeable( + struct xchk_rtrmap *cr, + const struct xfs_rmap_irec *r2) +{ + const struct xfs_rmap_irec *r1 = &cr->prev_rec; + + /* Ignore if prev_rec is not yet initialized. */ + if (cr->prev_rec.rm_blockcount == 0) + return false; + + if (r1->rm_owner != r2->rm_owner) + return false; + if (r1->rm_startblock + r1->rm_blockcount != r2->rm_startblock) + return false; + if ((unsigned long long)r1->rm_blockcount + r2->rm_blockcount > + XFS_RMAP_LEN_MAX) + return false; + if (r1->rm_flags != r2->rm_flags) + return false; + if (r1->rm_owner == XFS_RMAP_OWN_COW || + r1->rm_owner == XFS_RMAP_OWN_FS) + return true; + return r1->rm_offset + r1->rm_blockcount == r2->rm_offset; +} + +/* Flag failures for records that could be merged. */ +STATIC void +xchk_rtrmapbt_check_mergeable( + struct xchk_btree *bs, + struct xchk_rtrmap *cr, + const struct xfs_rmap_irec *irec) +{ + if (bs->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return; + + if (xchk_rtrmap_mergeable(cr, irec)) + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + + memcpy(&cr->prev_rec, irec, sizeof(struct xfs_rmap_irec)); +} + +/* Cross-reference a rmap against the refcount btree. */ +STATIC void +xchk_rtrmapbt_xref_rtrefc( + struct xfs_scrub *sc, + struct xfs_rmap_irec *irec) +{ + xfs_rgblock_t fbno; + xfs_extlen_t flen; + bool is_inode; + bool is_bmbt; + bool is_attr; + bool is_unwritten; + int error; + + if (!sc->sr.refc_cur || xchk_skip_xref(sc->sm)) + return; + + is_inode = !XFS_RMAP_NON_INODE_OWNER(irec->rm_owner); + is_bmbt = irec->rm_flags & XFS_RMAP_BMBT_BLOCK; + is_attr = irec->rm_flags & XFS_RMAP_ATTR_FORK; + is_unwritten = irec->rm_flags & XFS_RMAP_UNWRITTEN; + + /* If this is shared, must be a data fork extent. */ + error = xfs_refcount_find_shared(sc->sr.refc_cur, irec->rm_startblock, + irec->rm_blockcount, &fbno, &flen, false); + if (!xchk_should_check_xref(sc, &error, &sc->sr.refc_cur)) + return; + if (flen != 0 && (!is_inode || is_attr || is_bmbt || is_unwritten)) + xchk_btree_xref_set_corrupt(sc, sc->sr.refc_cur, 0); +} + +/* Cross-reference with other metadata. */ +STATIC void +xchk_rtrmapbt_xref( + struct xfs_scrub *sc, + struct xfs_rmap_irec *irec) +{ + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return; + + xchk_xref_is_used_rt_space(sc, + xfs_rgbno_to_rtb(sc->sr.rtg, irec->rm_startblock), + irec->rm_blockcount); + if (irec->rm_owner == XFS_RMAP_OWN_COW) + xchk_xref_is_rt_cow_staging(sc, irec->rm_startblock, + irec->rm_blockcount); + else + xchk_rtrmapbt_xref_rtrefc(sc, irec); +} + +/* Scrub a realtime rmapbt record. */ +STATIC int +xchk_rtrmapbt_rec( + struct xchk_btree *bs, + const union xfs_btree_rec *rec) +{ + struct xchk_rtrmap *cr = bs->private; + struct xfs_rmap_irec irec; + + if (xfs_rmap_btrec_to_irec(rec, &irec) != NULL || + xfs_rtrmap_check_irec(to_rtg(bs->cur->bc_group), &irec) != NULL) { + xchk_btree_set_corrupt(bs->sc, bs->cur, 0); + return 0; + } + + if (bs->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return 0; + + xchk_rtrmapbt_check_mergeable(bs, cr, &irec); + xchk_rtrmapbt_check_overlapping(bs, cr, &irec); + xchk_rtrmapbt_xref(bs->sc, &irec); + return 0; +} + +/* Scrub the realtime rmap btree. */ +int +xchk_rtrmapbt( + struct xfs_scrub *sc) +{ + struct xfs_owner_info oinfo; + struct xchk_rtrmap cr = { }; + int error; + + error = xchk_metadata_inode_forks(sc); + if (error || (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) + return error; + + xfs_rmap_inode_bmbt_owner(&oinfo, rtg_rmap(sc->sr.rtg), XFS_DATA_FORK); + return xchk_btree(sc, sc->sr.rmap_cur, xchk_rtrmapbt_rec, &oinfo, &cr); +} + +/* xref check that the extent has no realtime reverse mapping at all */ +void +xchk_xref_has_no_rt_owner( + struct xfs_scrub *sc, + xfs_rgblock_t bno, + xfs_extlen_t len) +{ + enum xbtree_recpacking outcome; + int error; + + if (!sc->sr.rmap_cur || xchk_skip_xref(sc->sm)) + return; + + error = xfs_rmap_has_records(sc->sr.rmap_cur, bno, len, &outcome); + if (!xchk_should_check_xref(sc, &error, &sc->sr.rmap_cur)) + return; + if (outcome != XBTREE_RECPACKING_EMPTY) + xchk_btree_xref_set_corrupt(sc, sc->sr.rmap_cur, 0); +} + +/* xref check that the extent is completely mapped */ +void +xchk_xref_has_rt_owner( + struct xfs_scrub *sc, + xfs_rgblock_t bno, + xfs_extlen_t len) +{ + enum xbtree_recpacking outcome; + int error; + + if (!sc->sr.rmap_cur || xchk_skip_xref(sc->sm)) + return; + + error = xfs_rmap_has_records(sc->sr.rmap_cur, bno, len, &outcome); + if (!xchk_should_check_xref(sc, &error, &sc->sr.rmap_cur)) + return; + if (outcome != XBTREE_RECPACKING_FULL) + xchk_btree_xref_set_corrupt(sc, sc->sr.rmap_cur, 0); +} + +/* xref check that the extent is only owned by a given owner */ +void +xchk_xref_is_only_rt_owned_by( + struct xfs_scrub *sc, + xfs_agblock_t bno, + xfs_extlen_t len, + const struct xfs_owner_info *oinfo) +{ + struct xfs_rmap_matches res; + int error; + + if (!sc->sr.rmap_cur || xchk_skip_xref(sc->sm)) + return; + + error = xfs_rmap_count_owners(sc->sr.rmap_cur, bno, len, oinfo, &res); + if (!xchk_should_check_xref(sc, &error, &sc->sr.rmap_cur)) + return; + if (res.matches != 1) + xchk_btree_xref_set_corrupt(sc, sc->sr.rmap_cur, 0); + if (res.bad_non_owner_matches) + xchk_btree_xref_set_corrupt(sc, sc->sr.rmap_cur, 0); + if (res.non_owner_matches) + xchk_btree_xref_set_corrupt(sc, sc->sr.rmap_cur, 0); +} diff --git a/fs/xfs/scrub/rtrmap_repair.c b/fs/xfs/scrub/rtrmap_repair.c new file mode 100644 index 000000000..a2b72e61e --- /dev/null +++ b/fs/xfs/scrub/rtrmap_repair.c @@ -0,0 +1,978 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_btree.h" +#include "xfs_btree_staging.h" +#include "xfs_buf_mem.h" +#include "xfs_btree_mem.h" +#include "xfs_bit.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_sb.h" +#include "xfs_alloc.h" +#include "xfs_rmap.h" +#include "xfs_rmap_btree.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_inode.h" +#include "xfs_icache.h" +#include "xfs_bmap.h" +#include "xfs_bmap_btree.h" +#include "xfs_quota.h" +#include "xfs_rtalloc.h" +#include "xfs_ag.h" +#include "xfs_rtgroup.h" +#include "xfs_refcount.h" +#include "scrub/xfs_scrub.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/btree.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/bitmap.h" +#include "scrub/fsb_bitmap.h" +#include "scrub/rgb_bitmap.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/iscan.h" +#include "scrub/newbt.h" +#include "scrub/reap.h" + +/* + * Realtime Reverse Mapping Btree Repair + * ===================================== + * + * This isn't quite as difficult as repairing the rmap btree on the data + * device, since we only store the data fork extents of realtime files on the + * realtime device. We still have to freeze the filesystem and stop the + * background threads like we do for the rmap repair, but we only have to scan + * realtime inodes. + * + * Collecting entries for the new realtime rmap btree is easy -- all we have + * to do is generate rtrmap entries from the data fork mappings of all realtime + * files in the filesystem. We then scan the rmap btrees of the data device + * looking for extents belonging to the old btree and note them in a bitmap. + * + * To rebuild the realtime rmap btree, we bulk-load the collected mappings into + * a new btree cursor and atomically swap that into the realtime inode. Then + * we can free the blocks from the old btree. + * + * We use the 'xrep_rtrmap' prefix for all the rmap functions. + */ + +/* Context for collecting rmaps */ +struct xrep_rtrmap { + /* new rtrmapbt information */ + struct xrep_newbt new_btree; + + /* lock for the xfbtree and xfile */ + struct mutex lock; + + /* rmap records generated from primary metadata */ + struct xfbtree rtrmap_btree; + + struct xfs_scrub *sc; + + /* bitmap of old rtrmapbt blocks */ + struct xfsb_bitmap old_rtrmapbt_blocks; + + /* Hooks into rtrmap update code. */ + struct xfs_rmap_hook rhook; + + /* inode scan cursor */ + struct xchk_iscan iscan; + + /* in-memory btree cursor for the ->get_blocks walk */ + struct xfs_btree_cur *mcur; + + /* Number of records we're staging in the new btree. */ + uint64_t nr_records; +}; + +/* Set us up to repair rt reverse mapping btrees. */ +int +xrep_setup_rtrmapbt( + struct xfs_scrub *sc) +{ + struct xrep_rtrmap *rr; + int error; + + xchk_fsgates_enable(sc, XCHK_FSGATES_RMAP); + + error = xrep_setup_xfbtree(sc, "realtime reverse mapping records"); + if (error) + return error; + + rr = kzalloc_obj(struct xrep_rtrmap, XCHK_GFP_FLAGS); + if (!rr) + return -ENOMEM; + + rr->sc = sc; + sc->buf = rr; + return 0; +} + +/* Make sure there's nothing funny about this mapping. */ +STATIC int +xrep_rtrmap_check_mapping( + struct xfs_scrub *sc, + const struct xfs_rmap_irec *rec) +{ + if (xfs_rtrmap_check_irec(sc->sr.rtg, rec) != NULL) + return -EFSCORRUPTED; + + /* Make sure this isn't free space. */ + return xrep_require_rtext_inuse(sc, rec->rm_startblock, + rec->rm_blockcount); +} + +/* Store a reverse-mapping record. */ +static inline int +xrep_rtrmap_stash( + struct xrep_rtrmap *rr, + xfs_rgblock_t startblock, + xfs_extlen_t blockcount, + uint64_t owner, + uint64_t offset, + unsigned int flags) +{ + struct xfs_rmap_irec rmap = { + .rm_startblock = startblock, + .rm_blockcount = blockcount, + .rm_owner = owner, + .rm_offset = offset, + .rm_flags = flags, + }; + struct xfs_scrub *sc = rr->sc; + struct xfs_btree_cur *mcur; + int error = 0; + + if (xchk_should_terminate(sc, &error)) + return error; + + if (xchk_iscan_aborted(&rr->iscan)) + return -EFSCORRUPTED; + + trace_xrep_rtrmap_found(sc->mp, &rmap); + + /* Add entry to in-memory btree. */ + mutex_lock(&rr->lock); + mcur = xfs_rtrmapbt_mem_cursor(sc->sr.rtg, sc->tp, &rr->rtrmap_btree); + error = xfs_rmap_map_raw(mcur, &rmap); + xfs_btree_del_cursor(mcur, error); + if (error) + goto out_cancel; + + error = xfbtree_trans_commit(&rr->rtrmap_btree, sc->tp); + if (error) + goto out_abort; + + mutex_unlock(&rr->lock); + return 0; + +out_cancel: + xfbtree_trans_cancel(&rr->rtrmap_btree, sc->tp); +out_abort: + xchk_iscan_abort(&rr->iscan); + mutex_unlock(&rr->lock); + return error; +} + +/* Finding all file and bmbt extents. */ + +/* Context for accumulating rmaps for an inode fork. */ +struct xrep_rtrmap_ifork { + /* + * Accumulate rmap data here to turn multiple adjacent bmaps into a + * single rmap. + */ + struct xfs_rmap_irec accum; + + struct xrep_rtrmap *rr; +}; + +/* Stash an rmap that we accumulated while walking an inode fork. */ +STATIC int +xrep_rtrmap_stash_accumulated( + struct xrep_rtrmap_ifork *rf) +{ + if (rf->accum.rm_blockcount == 0) + return 0; + + return xrep_rtrmap_stash(rf->rr, rf->accum.rm_startblock, + rf->accum.rm_blockcount, rf->accum.rm_owner, + rf->accum.rm_offset, rf->accum.rm_flags); +} + +/* Accumulate a bmbt record. */ +STATIC int +xrep_rtrmap_visit_bmbt( + struct xfs_btree_cur *cur, + struct xfs_bmbt_irec *rec, + void *priv) +{ + struct xrep_rtrmap_ifork *rf = priv; + struct xfs_rmap_irec *accum = &rf->accum; + struct xfs_mount *mp = rf->rr->sc->mp; + xfs_rgblock_t rgbno; + unsigned int rmap_flags = 0; + int error; + + if (xfs_rtb_to_rgno(mp, rec->br_startblock) != + rtg_rgno(rf->rr->sc->sr.rtg)) + return 0; + + if (rec->br_state == XFS_EXT_UNWRITTEN) + rmap_flags |= XFS_RMAP_UNWRITTEN; + + /* If this bmap is adjacent to the previous one, just add it. */ + rgbno = xfs_rtb_to_rgbno(mp, rec->br_startblock); + if (accum->rm_blockcount > 0 && + rec->br_startoff == accum->rm_offset + accum->rm_blockcount && + rgbno == accum->rm_startblock + accum->rm_blockcount && + rmap_flags == accum->rm_flags) { + accum->rm_blockcount += rec->br_blockcount; + return 0; + } + + /* Otherwise stash the old rmap and start accumulating a new one. */ + error = xrep_rtrmap_stash_accumulated(rf); + if (error) + return error; + + accum->rm_startblock = rgbno; + accum->rm_blockcount = rec->br_blockcount; + accum->rm_offset = rec->br_startoff; + accum->rm_flags = rmap_flags; + return 0; +} + +/* + * Iterate the block mapping btree to collect rmap records for anything in this + * fork that maps to the rt volume. Sets @mappings_done to true if we've + * scanned the block mappings in this fork. + */ +STATIC int +xrep_rtrmap_scan_bmbt( + struct xrep_rtrmap_ifork *rf, + struct xfs_inode *ip, + bool *mappings_done) +{ + struct xrep_rtrmap *rr = rf->rr; + struct xfs_btree_cur *cur; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK); + int error = 0; + + *mappings_done = false; + + /* + * If the incore extent cache is already loaded, we'll just use the + * incore extent scanner to record mappings. Don't bother walking the + * ondisk extent tree. + */ + if (!xfs_need_iread_extents(ifp)) + return 0; + + /* Accumulate all the mappings in the bmap btree. */ + cur = xfs_bmbt_init_cursor(rr->sc->mp, rr->sc->tp, ip, XFS_DATA_FORK); + error = xfs_bmap_query_all(cur, xrep_rtrmap_visit_bmbt, rf); + xfs_btree_del_cursor(cur, error); + if (error) + return error; + + /* Stash any remaining accumulated rmaps and exit. */ + *mappings_done = true; + return xrep_rtrmap_stash_accumulated(rf); +} + +/* + * Iterate the in-core extent cache to collect rmap records for anything in + * this fork that matches the AG. + */ +STATIC int +xrep_rtrmap_scan_iext( + struct xrep_rtrmap_ifork *rf, + struct xfs_ifork *ifp) +{ + struct xfs_bmbt_irec rec; + struct xfs_iext_cursor icur; + int error; + + for_each_xfs_iext(ifp, &icur, &rec) { + if (isnullstartblock(rec.br_startblock)) + continue; + error = xrep_rtrmap_visit_bmbt(NULL, &rec, rf); + if (error) + return error; + } + + return xrep_rtrmap_stash_accumulated(rf); +} + +/* Find all the extents on the realtime device mapped by an inode fork. */ +STATIC int +xrep_rtrmap_scan_dfork( + struct xrep_rtrmap *rr, + struct xfs_inode *ip) +{ + struct xrep_rtrmap_ifork rf = { + .accum = { .rm_owner = I_INO(ip), }, + .rr = rr, + }; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK); + int error = 0; + + if (ifp->if_format == XFS_DINODE_FMT_BTREE) { + bool mappings_done; + + /* + * Scan the bmbt for mappings. If the incore extent tree is + * loaded, we want to scan the cached mappings since that's + * faster when the extent counts are very high. + */ + error = xrep_rtrmap_scan_bmbt(&rf, ip, &mappings_done); + if (error || mappings_done) + return error; + } else if (ifp->if_format != XFS_DINODE_FMT_EXTENTS) { + /* realtime data forks should only be extents or btree */ + return -EFSCORRUPTED; + } + + /* Scan incore extent cache. */ + return xrep_rtrmap_scan_iext(&rf, ifp); +} + +/* Record reverse mappings for a file. */ +STATIC int +xrep_rtrmap_scan_inode( + struct xrep_rtrmap *rr, + struct xfs_inode *ip) +{ + unsigned int lock_mode; + int error = 0; + + /* Skip the rt rmap btree inode. */ + if (rr->sc->ip == ip) + return 0; + + lock_mode = xfs_ilock_data_map_shared(ip); + + /* Check the data fork if it's on the realtime device. */ + if (XFS_IS_REALTIME_INODE(ip)) { + error = xrep_rtrmap_scan_dfork(rr, ip); + if (error) + goto out_unlock; + } + + xchk_iscan_mark_visited(&rr->iscan, ip); +out_unlock: + xfs_iunlock(ip, lock_mode); + return error; +} + +/* Record extents that belong to the realtime rmap inode. */ +STATIC int +xrep_rtrmap_walk_rmap( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv) +{ + struct xrep_rtrmap *rr = priv; + int error = 0; + + if (xchk_should_terminate(rr->sc, &error)) + return error; + + /* Skip extents which are not owned by this inode and fork. */ + if (rec->rm_owner != I_INO(rr->sc->ip)) + return 0; + + error = xrep_check_ino_btree_mapping(rr->sc, rec); + if (error) + return error; + + return xfsb_bitmap_set(&rr->old_rtrmapbt_blocks, + xfs_gbno_to_fsb(cur->bc_group, rec->rm_startblock), + rec->rm_blockcount); +} + +/* Scan one AG for reverse mappings for the realtime rmap btree. */ +STATIC int +xrep_rtrmap_scan_ag( + struct xrep_rtrmap *rr, + struct xfs_perag *pag) +{ + struct xfs_scrub *sc = rr->sc; + int error; + + error = xrep_ag_init(sc, pag, &sc->sa); + if (error) + return error; + + error = xfs_rmap_query_all(sc->sa.rmap_cur, xrep_rtrmap_walk_rmap, rr); + xchk_ag_free(sc, &sc->sa); + return error; +} + +struct xrep_rtrmap_stash_run { + struct xrep_rtrmap *rr; + uint64_t owner; +}; + +static int +xrep_rtrmap_stash_run( + uint32_t start, + uint32_t len, + void *priv) +{ + struct xrep_rtrmap_stash_run *rsr = priv; + struct xrep_rtrmap *rr = rsr->rr; + xfs_rgblock_t rgbno = start; + + return xrep_rtrmap_stash(rr, rgbno, len, rsr->owner, 0, 0); +} + +/* + * Emit rmaps for every extent of bits set in the bitmap. Caller must ensure + * that the ranges are in units of FS blocks. + */ +STATIC int +xrep_rtrmap_stash_bitmap( + struct xrep_rtrmap *rr, + struct xrgb_bitmap *bitmap, + const struct xfs_owner_info *oinfo) +{ + struct xrep_rtrmap_stash_run rsr = { + .rr = rr, + .owner = oinfo->oi_owner, + }; + + return xrgb_bitmap_walk(bitmap, xrep_rtrmap_stash_run, &rsr); +} + +/* Record a CoW staging extent. */ +STATIC int +xrep_rtrmap_walk_cowblocks( + struct xfs_btree_cur *cur, + const struct xfs_refcount_irec *irec, + void *priv) +{ + struct xrgb_bitmap *bitmap = priv; + + if (!xfs_refcount_check_domain(irec) || + irec->rc_domain != XFS_REFC_DOMAIN_COW) + return -EFSCORRUPTED; + + return xrgb_bitmap_set(bitmap, irec->rc_startblock, + irec->rc_blockcount); +} + +/* + * Collect rmaps for the blocks containing the refcount btree, and all CoW + * staging extents. + */ +STATIC int +xrep_rtrmap_find_refcount_rmaps( + struct xrep_rtrmap *rr) +{ + struct xrgb_bitmap cow_blocks; /* COWBIT */ + struct xfs_refcount_irec low = { + .rc_startblock = 0, + .rc_domain = XFS_REFC_DOMAIN_COW, + }; + struct xfs_refcount_irec high = { + .rc_startblock = -1U, + .rc_domain = XFS_REFC_DOMAIN_COW, + }; + struct xfs_scrub *sc = rr->sc; + int error; + + if (!xfs_has_rtreflink(sc->mp)) + return 0; + + xrgb_bitmap_init(&cow_blocks); + + /* Collect rmaps for CoW staging extents. */ + error = xfs_refcount_query_range(sc->sr.refc_cur, &low, &high, + xrep_rtrmap_walk_cowblocks, &cow_blocks); + if (error) + goto out_bitmap; + + /* Generate rmaps for everything. */ + error = xrep_rtrmap_stash_bitmap(rr, &cow_blocks, &XFS_RMAP_OINFO_COW); + if (error) + goto out_bitmap; + +out_bitmap: + xrgb_bitmap_destroy(&cow_blocks); + return error; +} + +/* Count and check all collected records. */ +STATIC int +xrep_rtrmap_check_record( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv) +{ + struct xrep_rtrmap *rr = priv; + int error; + + error = xrep_rtrmap_check_mapping(rr->sc, rec); + if (error) + return error; + + rr->nr_records++; + return 0; +} + +/* Generate all the reverse-mappings for the realtime device. */ +STATIC int +xrep_rtrmap_find_rmaps( + struct xrep_rtrmap *rr) +{ + struct xfs_scrub *sc = rr->sc; + struct xfs_perag *pag = NULL; + struct xfs_inode *ip; + struct xfs_btree_cur *mcur; + int error; + + /* Generate rmaps for the realtime superblock */ + if (xfs_has_rtsb(sc->mp) && rtg_rgno(rr->sc->sr.rtg) == 0) { + error = xrep_rtrmap_stash(rr, 0, sc->mp->m_sb.sb_rextsize, + XFS_RMAP_OWN_FS, 0, 0); + if (error) + return error; + } + + /* Find CoW staging extents. */ + xrep_rtgroup_btcur_init(sc, &sc->sr); + error = xrep_rtrmap_find_refcount_rmaps(rr); + xchk_rtgroup_btcur_free(&sc->sr); + if (error) + return error; + + /* + * Set up for a potentially lengthy filesystem scan by reducing our + * transaction resource usage for the duration. Specifically: + * + * Unlock the realtime metadata inodes and cancel the transaction to + * release the log grant space while we scan the filesystem. + * + * Create a new empty transaction to eliminate the possibility of the + * inode scan deadlocking on cyclical metadata. + * + * We pass the empty transaction to the file scanning function to avoid + * repeatedly cycling empty transactions. This can be done even though + * we take the IOLOCK to quiesce the file because empty transactions + * do not take sb_internal. + */ + xchk_trans_cancel(sc); + xchk_rtgroup_unlock(&sc->sr); + xchk_trans_alloc_empty(sc); + + while ((error = xchk_iscan_iter(&rr->iscan, &ip)) == 1) { + error = xrep_rtrmap_scan_inode(rr, ip); + xchk_irele(sc, ip); + if (error) + break; + + if (xchk_should_terminate(sc, &error)) + break; + } + xchk_iscan_iter_finish(&rr->iscan); + if (error) + return error; + + /* + * Switch out for a real transaction and lock the RT metadata in + * preparation for building a new tree. + */ + xchk_trans_cancel(sc); + error = xchk_setup_rt(sc); + if (error) + return error; + error = xchk_rtgroup_lock(sc, &sc->sr, XCHK_RTGLOCK_ALL); + if (error) + return error; + + /* + * If a hook failed to update the in-memory btree, we lack the data to + * continue the repair. + */ + if (xchk_iscan_aborted(&rr->iscan)) + return -EFSCORRUPTED; + + /* Scan for old rtrmap blocks. */ + while ((pag = xfs_perag_next(sc->mp, pag))) { + error = xrep_rtrmap_scan_ag(rr, pag); + if (error) { + xfs_perag_rele(pag); + return error; + } + } + + /* + * Now that we have everything locked again, we need to count the + * number of rmap records stashed in the btree. This should reflect + * all actively-owned rt files in the filesystem. At the same time, + * check all our records before we start building a new btree, which + * requires the rtbitmap lock. + */ + mcur = xfs_rtrmapbt_mem_cursor(rr->sc->sr.rtg, NULL, &rr->rtrmap_btree); + rr->nr_records = 0; + error = xfs_rmap_query_all(mcur, xrep_rtrmap_check_record, rr); + xfs_btree_del_cursor(mcur, error); + + return error; +} + +/* Building the new rtrmap btree. */ + +/* Retrieve rtrmapbt data for bulk load. */ +STATIC int +xrep_rtrmap_get_records( + struct xfs_btree_cur *cur, + unsigned int idx, + struct xfs_btree_block *block, + unsigned int nr_wanted, + void *priv) +{ + struct xrep_rtrmap *rr = priv; + union xfs_btree_rec *block_rec; + unsigned int loaded; + int error; + + for (loaded = 0; loaded < nr_wanted; loaded++, idx++) { + int stat = 0; + + error = xfs_btree_increment(rr->mcur, 0, &stat); + if (error) + return error; + if (!stat) + return -EFSCORRUPTED; + + error = xfs_rmap_get_rec(rr->mcur, &cur->bc_rec.r, &stat); + if (error) + return error; + if (!stat) + return -EFSCORRUPTED; + + block_rec = xfs_btree_rec_addr(cur, idx, block); + cur->bc_ops->init_rec_from_cur(cur, block_rec); + } + + return loaded; +} + +/* Feed one of the new btree blocks to the bulk loader. */ +STATIC int +xrep_rtrmap_claim_block( + struct xfs_btree_cur *cur, + union xfs_btree_ptr *ptr, + void *priv) +{ + struct xrep_rtrmap *rr = priv; + + return xrep_newbt_claim_block(cur, &rr->new_btree, ptr); +} + +/* Figure out how much space we need to create the incore btree root block. */ +STATIC size_t +xrep_rtrmap_iroot_size( + struct xfs_btree_cur *cur, + unsigned int level, + unsigned int nr_this_level, + void *priv) +{ + return xfs_rtrmap_broot_space_calc(cur->bc_mp, level, nr_this_level); +} + +/* + * Use the collected rmap information to stage a new rmap btree. If this is + * successful we'll return with the new btree root information logged to the + * repair transaction but not yet committed. This implements section (III) + * above. + */ +STATIC int +xrep_rtrmap_build_new_tree( + struct xrep_rtrmap *rr) +{ + struct xfs_scrub *sc = rr->sc; + struct xfs_rtgroup *rtg = sc->sr.rtg; + struct xfs_btree_cur *rmap_cur; + int error; + + /* + * Prepare to construct the new btree by reserving disk space for the + * new btree and setting up all the accounting information we'll need + * to root the new btree while it's under construction and before we + * attach it to the realtime rmapbt inode. + */ + error = xrep_newbt_init_metadir_inode(&rr->new_btree, sc); + if (error) + return error; + + rr->new_btree.bload.get_records = xrep_rtrmap_get_records; + rr->new_btree.bload.claim_block = xrep_rtrmap_claim_block; + rr->new_btree.bload.iroot_size = xrep_rtrmap_iroot_size; + + rmap_cur = xfs_rtrmapbt_init_cursor(NULL, rtg); + xfs_btree_stage_ifakeroot(rmap_cur, &rr->new_btree.ifake); + + /* Compute how many blocks we'll need for the rmaps collected. */ + error = xfs_btree_bload_compute_geometry(rmap_cur, + &rr->new_btree.bload, rr->nr_records); + if (error) + goto err_cur; + + /* Last chance to abort before we start committing fixes. */ + if (xchk_should_terminate(sc, &error)) + goto err_cur; + + /* + * Guess how many blocks we're going to need to rebuild an entire + * rtrmapbt from the number of extents we found, and pump up our + * transaction to have sufficient block reservation. We're allowed + * to exceed quota to repair inconsistent metadata, though this is + * unlikely. + */ + error = xfs_trans_reserve_more_inode(sc->tp, rtg_rmap(rtg), + rr->new_btree.bload.nr_blocks, 0, true); + if (error) + goto err_cur; + + /* Reserve the space we'll need for the new btree. */ + error = xrep_newbt_alloc_blocks(&rr->new_btree, + rr->new_btree.bload.nr_blocks); + if (error) + goto err_cur; + + /* + * Create a cursor to the in-memory btree so that we can bulk load the + * new btree. + */ + rr->mcur = xfs_rtrmapbt_mem_cursor(sc->sr.rtg, NULL, &rr->rtrmap_btree); + error = xfs_btree_goto_left_edge(rr->mcur); + if (error) + goto err_mcur; + + /* Add all observed rmap records. */ + rr->new_btree.ifake.if_fork->if_format = XFS_DINODE_FMT_META_BTREE; + error = xfs_btree_bload(rmap_cur, &rr->new_btree.bload, rr); + if (error) + goto err_mcur; + + /* + * Install the new rtrmap btree in the inode. After this point the old + * btree is no longer accessible, the new tree is live, and we can + * delete the cursor. + */ + xfs_rtrmapbt_commit_staged_btree(rmap_cur, sc->tp); + xrep_inode_set_nblocks(rr->sc, rr->new_btree.ifake.if_blocks); + xfs_btree_del_cursor(rmap_cur, 0); + xfs_btree_del_cursor(rr->mcur, 0); + rr->mcur = NULL; + + /* + * Now that we've written the new btree to disk, we don't need to keep + * updating the in-memory btree. Abort the scan to stop live updates. + */ + xchk_iscan_abort(&rr->iscan); + + /* Dispose of any unused blocks and the accounting information. */ + error = xrep_newbt_commit(&rr->new_btree); + if (error) + return error; + + return xrep_roll_trans(sc); + +err_mcur: + xfs_btree_del_cursor(rr->mcur, error); +err_cur: + xfs_btree_del_cursor(rmap_cur, error); + xrep_newbt_cancel(&rr->new_btree); + return error; +} + +/* Reaping the old btree. */ + +static inline bool +xrep_rtrmapbt_want_live_update( + struct xchk_iscan *iscan, + const struct xfs_owner_info *oi) +{ + if (xchk_iscan_aborted(iscan)) + return false; + + /* + * We scanned the CoW staging extents before we started the iscan, so + * we need all the updates. + */ + if (XFS_RMAP_NON_INODE_OWNER(oi->oi_owner)) + return true; + + /* Ignore updates to files that the scanner hasn't visited yet. */ + return xchk_iscan_want_live_update(iscan, oi->oi_owner); +} + +/* + * Apply a rtrmapbt update from the regular filesystem into our shadow btree. + * We're running from the thread that owns the rtrmap ILOCK and is generating + * the update, so we must be careful about which parts of the struct + * xrep_rtrmap that we change. + */ +static int +xrep_rtrmapbt_live_update( + struct notifier_block *nb, + unsigned long action, + void *data) +{ + struct xfs_rmap_update_params *p = data; + struct xrep_rtrmap *rr; + struct xfs_mount *mp; + struct xfs_btree_cur *mcur; + struct xfs_trans *tp; + int error; + + rr = container_of(nb, struct xrep_rtrmap, rhook.rmap_hook.nb); + mp = rr->sc->mp; + + if (!xrep_rtrmapbt_want_live_update(&rr->iscan, &p->oinfo)) + goto out_unlock; + + trace_xrep_rmap_live_update(rtg_group(rr->sc->sr.rtg), action, p); + + tp = xfs_trans_alloc_empty(mp); + + mutex_lock(&rr->lock); + mcur = xfs_rtrmapbt_mem_cursor(rr->sc->sr.rtg, tp, &rr->rtrmap_btree); + error = __xfs_rmap_finish_intent(mcur, action, p->startblock, + p->blockcount, &p->oinfo, p->unwritten); + xfs_btree_del_cursor(mcur, error); + if (error) + goto out_cancel; + + error = xfbtree_trans_commit(&rr->rtrmap_btree, tp); + if (error) + goto out_cancel; + + xfs_trans_cancel(tp); + mutex_unlock(&rr->lock); + return NOTIFY_DONE; + +out_cancel: + xfbtree_trans_cancel(&rr->rtrmap_btree, tp); + xfs_trans_cancel(tp); + xchk_iscan_abort(&rr->iscan); + mutex_unlock(&rr->lock); +out_unlock: + return NOTIFY_DONE; +} + +/* Set up the filesystem scan components. */ +STATIC int +xrep_rtrmap_setup_scan( + struct xrep_rtrmap *rr) +{ + struct xfs_scrub *sc = rr->sc; + int error; + + mutex_init(&rr->lock); + xfsb_bitmap_init(&rr->old_rtrmapbt_blocks); + + /* Set up some storage */ + error = xfs_rtrmapbt_mem_init(sc->mp, &rr->rtrmap_btree, sc->xmbtp, + rtg_rgno(sc->sr.rtg)); + if (error) + goto out_bitmap; + + /* Retry iget every tenth of a second for up to 30 seconds. */ + xchk_iscan_start(sc, 30000, 100, &rr->iscan); + + /* + * Hook into live rtrmap operations so that we can update our in-memory + * btree to reflect live changes on the filesystem. Since we drop the + * rtrmap ILOCK to scan all the inodes, we need this piece to avoid + * installing a stale btree. + */ + ASSERT(sc->flags & XCHK_FSGATES_RMAP); + xfs_rmap_hook_setup(&rr->rhook, xrep_rtrmapbt_live_update); + error = xfs_rmap_hook_add(rtg_group(sc->sr.rtg), &rr->rhook); + if (error) + goto out_iscan; + return 0; + +out_iscan: + xchk_iscan_teardown(&rr->iscan); + xfbtree_destroy(&rr->rtrmap_btree); +out_bitmap: + xfsb_bitmap_destroy(&rr->old_rtrmapbt_blocks); + mutex_destroy(&rr->lock); + return error; +} + +/* Tear down scan components. */ +STATIC void +xrep_rtrmap_teardown( + struct xrep_rtrmap *rr) +{ + struct xfs_scrub *sc = rr->sc; + + xchk_iscan_abort(&rr->iscan); + xfs_rmap_hook_del(rtg_group(sc->sr.rtg), &rr->rhook); + xchk_iscan_teardown(&rr->iscan); + xfbtree_destroy(&rr->rtrmap_btree); + xfsb_bitmap_destroy(&rr->old_rtrmapbt_blocks); + mutex_destroy(&rr->lock); +} + +/* Repair the realtime rmap btree. */ +int +xrep_rtrmapbt( + struct xfs_scrub *sc) +{ + struct xrep_rtrmap *rr = sc->buf; + int error; + + /* Make sure any problems with the fork are fixed. */ + error = xrep_metadata_inode_forks(sc); + if (error) + return error; + + error = xrep_rtrmap_setup_scan(rr); + if (error) + return error; + + /* Collect rmaps for realtime files. */ + error = xrep_rtrmap_find_rmaps(rr); + if (error) + goto out_records; + + xfs_trans_ijoin(sc->tp, sc->ip, 0); + + /* Rebuild the rtrmap information. */ + error = xrep_rtrmap_build_new_tree(rr); + if (error) + goto out_records; + + /* + * Free all the extents that were allocated to the former rtrmapbt and + * aren't cross-linked with something else. + */ + error = xrep_reap_metadir_fsblocks(rr->sc, &rr->old_rtrmapbt_blocks); + if (error) + goto out_records; + +out_records: + xrep_rtrmap_teardown(rr); + return error; +} diff --git a/fs/xfs/scrub/rtsummary.c b/fs/xfs/scrub/rtsummary.c new file mode 100644 index 000000000..546b335ad --- /dev/null +++ b/fs/xfs/scrub/rtsummary.c @@ -0,0 +1,369 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_btree.h" +#include "xfs_inode.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_rtbitmap.h" +#include "xfs_bit.h" +#include "xfs_bmap.h" +#include "xfs_sb.h" +#include "xfs_exchmaps.h" +#include "xfs_rtgroup.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/trace.h" +#include "scrub/xfile.h" +#include "scrub/repair.h" +#include "scrub/tempexch.h" +#include "scrub/rtsummary.h" + +/* + * Realtime Summary + * ================ + * + * We check the realtime summary by scanning the realtime bitmap file to create + * a new summary file incore, and then we compare the computed version against + * the ondisk version. We use the 'xfile' functionality to store this + * (potentially large) amount of data in pageable memory. + */ + +/* Set us up to check the rtsummary file. */ +int +xchk_setup_rtsummary( + struct xfs_scrub *sc) +{ + struct xfs_mount *mp = sc->mp; + struct xchk_rtsummary *rts; + int error; + + if (xchk_need_intent_drain(sc)) + xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN); + + rts = kvzalloc_flex(*rts, words, mp->m_blockwsize, XCHK_GFP_FLAGS); + if (!rts) + return -ENOMEM; + sc->buf = rts; + + error = xchk_rtgroup_init(sc, sc->sm->sm_agno, &sc->sr); + if (error) + return error; + + if (xchk_could_repair(sc)) { + error = xrep_setup_rtsummary(sc, rts); + if (error) + return error; + } + + /* + * Create an xfile to construct a new rtsummary file. The xfile allows + * us to avoid pinning kernel memory for this purpose. + */ + error = xfile_create("realtime summary file", + XFS_FSB_TO_B(mp, mp->m_rsumblocks), &sc->xfile); + if (error) + return error; + + error = xchk_trans_alloc(sc, rts->resblks); + if (error) + return error; + + error = xchk_install_live_inode(sc, rtg_summary(sc->sr.rtg)); + if (error) + return error; + + error = xchk_ino_dqattach(sc); + if (error) + return error; + + error = xchk_rtgroup_lock(sc, &sc->sr, XFS_RTGLOCK_BITMAP); + if (error) + return error; + + /* + * Now that we've locked the rtbitmap and rtsummary, we can't race with + * growfsrt trying to expand the summary or change the size of the rt + * volume. Hence it is safe to compute and check the geometry values. + * + * Note that there is no strict requirement for an exclusive lock on the + * summary here, but to keep the locking APIs simple we lock both inodes + * exclusively here. If we ever start caring about running concurrent + * fsmap with scrub this could be changed. + */ + if (mp->m_sb.sb_rblocks) { + rts->rextents = xfs_blen_to_rtbxlen(mp, mp->m_sb.sb_rblocks); + rts->rbmblocks = xfs_rtbitmap_blockcount(mp); + rts->rsumblocks = + xfs_rtsummary_blockcount(mp, &rts->rsumlevels); + } + + return 0; +} + +/* Helper functions to record suminfo words in an xfile. */ + +static inline int +xfsum_load( + struct xfs_scrub *sc, + xfs_rtsumoff_t sumoff, + union xfs_suminfo_raw *rawinfo) +{ + return xfile_load(sc->xfile, rawinfo, + sizeof(union xfs_suminfo_raw), + sumoff << XFS_WORDLOG); +} + +static inline int +xfsum_store( + struct xfs_scrub *sc, + xfs_rtsumoff_t sumoff, + const union xfs_suminfo_raw rawinfo) +{ + return xfile_store(sc->xfile, &rawinfo, + sizeof(union xfs_suminfo_raw), + sumoff << XFS_WORDLOG); +} + +inline int +xfsum_copyout( + struct xfs_scrub *sc, + xfs_rtsumoff_t sumoff, + union xfs_suminfo_raw *rawinfo, + unsigned int nr_words) +{ + return xfile_load(sc->xfile, rawinfo, nr_words << XFS_WORDLOG, + sumoff << XFS_WORDLOG); +} + +static inline xfs_suminfo_t +xchk_rtsum_inc( + struct xfs_mount *mp, + union xfs_suminfo_raw *v) +{ + if (xfs_has_rtgroups(mp)) { + be32_add_cpu(&v->rtg, 1); + return be32_to_cpu(v->rtg); + } + + v->old += 1; + return v->old; +} + +/* Update the summary file to reflect the free extent that we've accumulated. */ +STATIC int +xchk_rtsum_record_free( + struct xfs_rtgroup *rtg, + struct xfs_trans *tp, + const struct xfs_rtalloc_rec *rec, + void *priv) +{ + struct xfs_mount *mp = rtg_mount(rtg); + struct xfs_scrub *sc = priv; + xfs_fileoff_t rbmoff; + xfs_rtblock_t rtbno; + xfs_filblks_t rtlen; + xfs_rtsumoff_t offs; + unsigned int lenlog; + union xfs_suminfo_raw v; + xfs_suminfo_t value; + int error = 0; + + if (xchk_should_terminate(sc, &error)) + return error; + + /* Compute the relevant location in the rtsum file. */ + rbmoff = xfs_rtx_to_rbmblock(mp, rec->ar_startext); + lenlog = xfs_highbit64(rec->ar_extcount); + offs = xfs_rtsumoffs(mp, lenlog, rbmoff); + + rtbno = xfs_rtx_to_rtb(rtg, rec->ar_startext); + rtlen = xfs_rtxlen_to_extlen(mp, rec->ar_extcount); + + if (!xfs_verify_rtbext(mp, rtbno, rtlen)) { + xchk_ip_xref_set_corrupt(sc, rtg_bitmap(rtg)); + return -EFSCORRUPTED; + } + + /* Bump the summary count. */ + error = xfsum_load(sc, offs, &v); + if (error) + return error; + + value = xchk_rtsum_inc(sc->mp, &v); + trace_xchk_rtsum_record_free(mp, rec->ar_startext, rec->ar_extcount, + lenlog, offs, value); + + return xfsum_store(sc, offs, v); +} + +/* Compute the realtime summary from the realtime bitmap. */ +STATIC int +xchk_rtsum_compute( + struct xfs_scrub *sc) +{ + struct xfs_mount *mp = sc->mp; + struct xfs_rtgroup *rtg = sc->sr.rtg; + + /* If the bitmap size doesn't match the computed size, bail. */ + if (XFS_FSB_TO_B(mp, xfs_rtbitmap_blockcount(mp)) != + rtg_bitmap(rtg)->i_disk_size) + return -EFSCORRUPTED; + + return xfs_rtalloc_query_all(rtg, sc->tp, xchk_rtsum_record_free, sc); +} + +/* Compare the rtsummary file against the one we computed. */ +STATIC int +xchk_rtsum_compare( + struct xfs_scrub *sc) +{ + struct xfs_bmbt_irec map; + struct xfs_iext_cursor icur; + + struct xfs_mount *mp = sc->mp; + struct xfs_inode *ip = sc->ip; + struct xchk_rtsummary *rts = sc->buf; + xfs_fileoff_t off = 0; + xfs_fileoff_t endoff; + xfs_rtsumoff_t sumoff = 0; + int error = 0; + + rts->args.mp = mp; + rts->args.tp = sc->tp; + rts->args.rtg = sc->sr.rtg; + + /* Mappings may not cross or lie beyond EOF. */ + endoff = XFS_B_TO_FSB(mp, ip->i_disk_size); + if (xfs_iext_lookup_extent(ip, &ip->i_df, endoff, &icur, &map)) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, endoff); + return 0; + } + + while (off < endoff) { + int nmap = 1; + + if (xchk_should_terminate(sc, &error)) + return error; + if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + return 0; + + /* Make sure we have a written extent. */ + error = xfs_bmapi_read(ip, off, endoff - off, &map, &nmap, + XFS_DATA_FORK); + if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, off, &error)) + return error; + + if (nmap != 1 || !xfs_bmap_is_written_extent(&map)) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, off); + return 0; + } + + off += map.br_blockcount; + } + + for (off = 0; off < endoff; off++) { + union xfs_suminfo_raw *ondisk_info; + + /* Read a block's worth of ondisk rtsummary file. */ + error = xfs_rtsummary_read_buf(&rts->args, off); + if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, off, &error)) + return error; + + /* Read a block's worth of computed rtsummary file. */ + error = xfsum_copyout(sc, sumoff, rts->words, mp->m_blockwsize); + if (error) { + xfs_rtbuf_cache_relse(&rts->args); + return error; + } + + ondisk_info = xfs_rsumblock_infoptr(&rts->args, 0); + if (memcmp(ondisk_info, rts->words, + mp->m_blockwsize << XFS_WORDLOG) != 0) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, off); + xfs_rtbuf_cache_relse(&rts->args); + return error; + } + + xfs_rtbuf_cache_relse(&rts->args); + sumoff += mp->m_blockwsize; + } + + return 0; +} + +/* Scrub the realtime summary. */ +int +xchk_rtsummary( + struct xfs_scrub *sc) +{ + struct xfs_mount *mp = sc->mp; + struct xfs_rtgroup *rtg = sc->sr.rtg; + struct xfs_inode *rbmip = rtg_bitmap(rtg); + struct xfs_inode *rsumip = rtg_summary(rtg); + struct xchk_rtsummary *rts = sc->buf; + int error; + + /* Is sb_rextents correct? */ + if (mp->m_sb.sb_rextents != rts->rextents) { + xchk_ip_set_corrupt(sc, rbmip); + return 0; + } + + /* Is m_rsumlevels correct? */ + if (mp->m_rsumlevels != rts->rsumlevels) { + xchk_ip_set_corrupt(sc, rsumip); + return 0; + } + + /* Is m_rsumsize correct? */ + if (mp->m_rsumblocks != rts->rsumblocks) { + xchk_ip_set_corrupt(sc, rsumip); + return 0; + } + + /* The summary file length must be aligned to an fsblock. */ + if (rsumip->i_disk_size & mp->m_blockmask) { + xchk_ip_set_corrupt(sc, rsumip); + return 0; + } + + /* + * Is the summary file itself large enough to handle the rt volume? + * growfsrt expands the summary file before updating sb_rextents, so + * the file can be larger than rsumsize. + */ + if (rsumip->i_disk_size < XFS_FSB_TO_B(mp, rts->rsumblocks)) { + xchk_ip_set_corrupt(sc, rsumip); + return 0; + } + + /* Invoke the fork scrubber. */ + error = xchk_metadata_inode_forks(sc); + if (error || (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) + return error; + + /* Construct the new summary file from the rtbitmap. */ + error = xchk_rtsum_compute(sc); + if (error == -EFSCORRUPTED) { + /* + * EFSCORRUPTED means the rtbitmap is corrupt, which is an xref + * error since we're checking the summary file. + */ + xchk_ip_xref_set_corrupt(sc, rbmip); + return 0; + } + if (error) + return error; + + /* Does the computed summary file match the actual rtsummary file? */ + return xchk_rtsum_compare(sc); +} diff --git a/fs/xfs/scrub/rtsummary.h b/fs/xfs/scrub/rtsummary.h new file mode 100644 index 000000000..e44b04cb6 --- /dev/null +++ b/fs/xfs/scrub/rtsummary.h @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_RTSUMMARY_H__ +#define __XFS_SCRUB_RTSUMMARY_H__ + +struct xchk_rtsummary { +#ifdef CONFIG_XFS_ONLINE_REPAIR + struct xrep_tempexch tempexch; +#endif + struct xfs_rtalloc_args args; + + uint64_t rextents; + uint64_t rbmblocks; + xfs_filblks_t rsumblocks; + unsigned int rsumlevels; + unsigned int resblks; + + /* suminfo position of xfile as we write buffers to disk. */ + xfs_rtsumoff_t prep_wordoff; + + /* Memory buffer for the summary comparison. */ + union xfs_suminfo_raw words[]; +}; + +int xfsum_copyout(struct xfs_scrub *sc, xfs_rtsumoff_t sumoff, + union xfs_suminfo_raw *rawinfo, unsigned int nr_words); + +#ifdef CONFIG_XFS_ONLINE_REPAIR +int xrep_setup_rtsummary(struct xfs_scrub *sc, struct xchk_rtsummary *rts); +#else +# define xrep_setup_rtsummary(sc, rts) (0) +#endif /* CONFIG_XFS_ONLINE_REPAIR */ + +#endif /* __XFS_SCRUB_RTSUMMARY_H__ */ diff --git a/fs/xfs/scrub/rtsummary_repair.c b/fs/xfs/scrub/rtsummary_repair.c new file mode 100644 index 000000000..f065c3e51 --- /dev/null +++ b/fs/xfs/scrub/rtsummary_repair.c @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_btree.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_rtalloc.h" +#include "xfs_inode.h" +#include "xfs_bit.h" +#include "xfs_bmap.h" +#include "xfs_bmap_btree.h" +#include "xfs_exchmaps.h" +#include "xfs_rtbitmap.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/tempfile.h" +#include "scrub/tempexch.h" +#include "scrub/reap.h" +#include "scrub/xfile.h" +#include "scrub/rtsummary.h" + +/* Set us up to repair the rtsummary file. */ +int +xrep_setup_rtsummary( + struct xfs_scrub *sc, + struct xchk_rtsummary *rts) +{ + struct xfs_mount *mp = sc->mp; + unsigned long long blocks; + int error; + + error = xrep_tempfile_create(sc, S_IFREG); + if (error) + return error; + + /* + * If we're doing a repair, we reserve enough blocks to write out a + * completely new summary file, plus twice as many blocks as we would + * need if we can only allocate one block per data fork mapping. This + * should cover the preallocation of the temporary file and exchanging + * the extent mappings. + * + * We cannot use xfs_exchmaps_estimate because we have not yet + * constructed the replacement rtsummary and therefore do not know how + * many extents it will use. By the time we do, we will have a dirty + * transaction (which we cannot drop because we cannot drop the + * rtsummary ILOCK) and cannot ask for more reservation. + */ + blocks = mp->m_rsumblocks; + blocks += xfs_bmbt_calc_size(mp, blocks) * 2; + if (blocks > UINT_MAX) + return -EOPNOTSUPP; + + rts->resblks += blocks; + return 0; +} + +static int +xrep_rtsummary_prep_buf( + struct xfs_scrub *sc, + struct xfs_buf *bp, + void *data) +{ + struct xchk_rtsummary *rts = data; + struct xfs_mount *mp = sc->mp; + union xfs_suminfo_raw *ondisk; + int error; + + rts->args.mp = mp; + rts->args.tp = sc->tp; + rts->args.rtg = sc->sr.rtg; + rts->args.sumbp = bp; + ondisk = xfs_rsumblock_infoptr(&rts->args, 0); + rts->args.sumbp = NULL; + + error = xfsum_copyout(sc, rts->prep_wordoff, ondisk, mp->m_blockwsize); + if (error) + return error; + + if (xfs_has_rtgroups(sc->mp)) { + struct xfs_rtbuf_blkinfo *hdr = bp->b_addr; + + hdr->rt_magic = cpu_to_be32(XFS_RTSUMMARY_MAGIC); + hdr->rt_owner = cpu_to_be64(I_INO(sc->ip)); + hdr->rt_blkno = cpu_to_be64(xfs_buf_daddr(bp)); + hdr->rt_lsn = 0; + uuid_copy(&hdr->rt_uuid, &sc->mp->m_sb.sb_meta_uuid); + bp->b_ops = &xfs_rtsummary_buf_ops; + } else { + bp->b_ops = &xfs_rtbuf_ops; + } + + rts->prep_wordoff += mp->m_blockwsize; + xfs_trans_buf_set_type(sc->tp, bp, XFS_BLFT_RTSUMMARY_BUF); + return 0; +} + +/* Repair the realtime summary. */ +int +xrep_rtsummary( + struct xfs_scrub *sc) +{ + struct xchk_rtsummary *rts = sc->buf; + struct xfs_mount *mp = sc->mp; + int error; + + /* We require the rmapbt to rebuild anything. */ + if (!xfs_has_rmapbt(mp)) + return -EOPNOTSUPP; + /* We require atomic file exchange range to rebuild anything. */ + if (!xfs_has_exchange_range(mp)) + return -EOPNOTSUPP; + + /* Walk away if we disagree on the size of the rt bitmap. */ + if (rts->rbmblocks != mp->m_sb.sb_rbmblocks) + return 0; + + /* Make sure any problems with the fork are fixed. */ + error = xrep_metadata_inode_forks(sc); + if (error) + return error; + + /* + * Try to take ILOCK_EXCL of the temporary file. We had better be the + * only ones holding onto this inode, but we can't block while holding + * the rtsummary file's ILOCK_EXCL. + */ + while (!xrep_tempfile_ilock_nowait(sc)) { + if (xchk_should_terminate(sc, &error)) + return error; + delay(1); + } + + /* Make sure we have space allocated for the entire summary file. */ + xfs_trans_ijoin(sc->tp, sc->ip, 0); + xfs_trans_ijoin(sc->tp, sc->tempip, 0); + error = xrep_tempfile_prealloc(sc, 0, rts->rsumblocks); + if (error) + return error; + + /* Last chance to abort before we start committing fixes. */ + if (xchk_should_terminate(sc, &error)) + return error; + + /* Copy the rtsummary file that we generated. */ + error = xrep_tempfile_copyin(sc, 0, rts->rsumblocks, + xrep_rtsummary_prep_buf, rts); + if (error) + return error; + error = xrep_tempfile_set_isize(sc, XFS_FSB_TO_B(mp, rts->rsumblocks)); + if (error) + return error; + + /* + * Now exchange the contents. Nothing in repair uses the temporary + * buffer, so we can reuse it for the tempfile exchrange information. + */ + error = xrep_tempexch_trans_reserve(sc, XFS_DATA_FORK, 0, + rts->rsumblocks, &rts->tempexch); + if (error) + return error; + + error = xrep_tempexch_contents(sc, &rts->tempexch); + if (error) + return error; + + /* Reset incore state and blow out the summary cache. */ + if (sc->sr.rtg->rtg_rsum_cache) + memset(sc->sr.rtg->rtg_rsum_cache, 0xFF, mp->m_sb.sb_rbmblocks); + + mp->m_rsumlevels = rts->rsumlevels; + mp->m_rsumblocks = rts->rsumblocks; + + /* Free the old rtsummary blocks if they're not in use. */ + return xrep_reap_ifork(sc, sc->tempip, XFS_DATA_FORK); +} diff --git a/fs/xfs/scrub/scrub.c b/fs/xfs/scrub/scrub.c new file mode 100644 index 000000000..8742445c8 --- /dev/null +++ b/fs/xfs/scrub/scrub.c @@ -0,0 +1,1012 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_inode.h" +#include "xfs_quota.h" +#include "xfs_qm.h" +#include "xfs_scrub.h" +#include "xfs_buf_mem.h" +#include "xfs_rmap.h" +#include "xfs_exchrange.h" +#include "xfs_exchmaps.h" +#include "xfs_dir2.h" +#include "xfs_parent.h" +#include "xfs_icache.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/health.h" +#include "scrub/stats.h" +#include "scrub/xfile.h" +#include "scrub/tempfile.h" +#include "scrub/orphanage.h" + +/* + * Online Scrub and Repair + * + * Traditionally, XFS (the kernel driver) did not know how to check or + * repair on-disk data structures. That task was left to the xfs_check + * and xfs_repair tools, both of which require taking the filesystem + * offline for a thorough but time consuming examination. Online + * scrub & repair, on the other hand, enables us to check the metadata + * for obvious errors while carefully stepping around the filesystem's + * ongoing operations, locking rules, etc. + * + * Given that most XFS metadata consist of records stored in a btree, + * most of the checking functions iterate the btree blocks themselves + * looking for irregularities. When a record block is encountered, each + * record can be checked for obviously bad values. Record values can + * also be cross-referenced against other btrees to look for potential + * misunderstandings between pieces of metadata. + * + * It is expected that the checkers responsible for per-AG metadata + * structures will lock the AG headers (AGI, AGF, AGFL), iterate the + * metadata structure, and perform any relevant cross-referencing before + * unlocking the AG and returning the results to userspace. These + * scrubbers must not keep an AG locked for too long to avoid tying up + * the block and inode allocators. + * + * Block maps and b-trees rooted in an inode present a special challenge + * because they can involve extents from any AG. The general scrubber + * structure of lock -> check -> xref -> unlock still holds, but AG + * locking order rules /must/ be obeyed to avoid deadlocks. The + * ordering rule, of course, is that we must lock in increasing AG + * order. Helper functions are provided to track which AG headers we've + * already locked. If we detect an imminent locking order violation, we + * can signal a potential deadlock, in which case the scrubber can jump + * out to the top level, lock all the AGs in order, and retry the scrub. + * + * For file data (directories, extended attributes, symlinks) scrub, we + * can simply lock the inode and walk the data. For btree data + * (directories and attributes) we follow the same btree-scrubbing + * strategy outlined previously to check the records. + * + * We use a bit of trickery with transactions to avoid buffer deadlocks + * if there is a cycle in the metadata. The basic problem is that + * travelling down a btree involves locking the current buffer at each + * tree level. If a pointer should somehow point back to a buffer that + * we've already examined, we will deadlock due to the second buffer + * locking attempt. Note however that grabbing a buffer in transaction + * context links the locked buffer to the transaction. If we try to + * re-grab the buffer in the context of the same transaction, we avoid + * the second lock attempt and continue. Between the verifier and the + * scrubber, something will notice that something is amiss and report + * the corruption. Therefore, each scrubber will allocate an empty + * transaction, attach buffers to it, and cancel the transaction at the + * end of the scrub run. Cancelling a non-dirty transaction simply + * unlocks the buffers. + * + * There are four pieces of data that scrub can communicate to + * userspace. The first is the error code (errno), which can be used to + * communicate operational errors in performing the scrub. There are + * also three flags that can be set in the scrub context. If the data + * structure itself is corrupt, the CORRUPT flag will be set. If + * the metadata is correct but otherwise suboptimal, the PREEN flag + * will be set. + * + * We perform secondary validation of filesystem metadata by + * cross-referencing every record with all other available metadata. + * For example, for block mapping extents, we verify that there are no + * records in the free space and inode btrees corresponding to that + * space extent and that there is a corresponding entry in the reverse + * mapping btree. Inconsistent metadata is noted by setting the + * XCORRUPT flag; btree query function errors are noted by setting the + * XFAIL flag and deleting the cursor to prevent further attempts to + * cross-reference with a defective btree. + * + * If a piece of metadata proves corrupt or suboptimal, the userspace + * program can ask the kernel to apply some tender loving care (TLC) to + * the metadata object by setting the REPAIR flag and re-calling the + * scrub ioctl. "Corruption" is defined by metadata violating the + * on-disk specification; operations cannot continue if the violation is + * left untreated. It is possible for XFS to continue if an object is + * "suboptimal", however performance may be degraded. Repairs are + * usually performed by rebuilding the metadata entirely out of + * redundant metadata. Optimizing, on the other hand, can sometimes be + * done without rebuilding entire structures. + * + * Generally speaking, the repair code has the following code structure: + * Lock -> scrub -> repair -> commit -> re-lock -> re-scrub -> unlock. + * The first check helps us figure out if we need to rebuild or simply + * optimize the structure so that the rebuild knows what to do. The + * second check evaluates the completeness of the repair; that is what + * is reported to userspace. + * + * A quick note on symbol prefixes: + * - "xfs_" are general XFS symbols. + * - "xchk_" are symbols related to metadata checking. + * - "xrep_" are symbols related to metadata repair. + * - "xfs_scrub_" are symbols that tie online fsck to the rest of XFS. + */ + +/* + * Scrub probe -- userspace uses this to probe if we're willing to scrub + * or repair a given mountpoint. This will be used by xfs_scrub to + * probe the kernel's abilities to scrub (and repair) the metadata. We + * do this by validating the ioctl inputs from userspace, preparing the + * filesystem for a scrub (or a repair) operation, and immediately + * returning to userspace. Userspace can use the returned errno and + * structure state to decide (in broad terms) if scrub/repair are + * supported by the running kernel. + */ +static int +xchk_probe( + struct xfs_scrub *sc) +{ + int error = 0; + + if (xchk_should_terminate(sc, &error)) + return error; + + /* + * If the caller is probing to see if repair works but repair isn't + * built into the kernel, return EOPNOTSUPP because that's the signal + * that userspace expects. If online repair is built in, set the + * CORRUPT flag (without any of the usual tracing/logging) to force us + * into xrep_probe. + */ + if (xchk_could_repair(sc)) { + if (!IS_ENABLED(CONFIG_XFS_ONLINE_REPAIR)) + return -EOPNOTSUPP; + sc->sm->sm_flags |= XFS_SCRUB_OFLAG_CORRUPT; + } + return 0; +} + +/* Scrub setup and teardown */ + +static inline void +xchk_fsgates_disable( + struct xfs_scrub *sc) +{ + if (!(sc->flags & XCHK_FSGATES_ALL)) + return; + + trace_xchk_fsgates_disable(sc, sc->flags & XCHK_FSGATES_ALL); + + if (sc->flags & XCHK_FSGATES_DRAIN) + xfs_defer_drain_wait_disable(); + + if (sc->flags & XCHK_FSGATES_QUOTA) + xfs_dqtrx_hook_disable(); + + if (sc->flags & XCHK_FSGATES_DIRENTS) + xfs_dir_hook_disable(); + + if (sc->flags & XCHK_FSGATES_RMAP) + xfs_rmap_hook_disable(); + + sc->flags &= ~XCHK_FSGATES_ALL; +} + +/* Free the resources associated with a scrub subtype. */ +void +xchk_scrub_free_subord( + struct xfs_scrub_subord *sub) +{ + struct xfs_scrub *sc = sub->parent_sc; + + ASSERT(sc->ip == sub->sc.ip); + ASSERT(sc->orphanage == sub->sc.orphanage); + ASSERT(sc->tempip == sub->sc.tempip); + + sc->sm->sm_type = sub->old_smtype; + sc->sm->sm_flags = sub->old_smflags | + (sc->sm->sm_flags & XFS_SCRUB_FLAGS_OUT); + sc->tp = sub->sc.tp; + + if (sub->sc.buf) { + if (sub->sc.buf_cleanup) + sub->sc.buf_cleanup(sub->sc.buf); + kvfree(sub->sc.buf); + } + if (sub->sc.xmbtp) + xmbuf_free(sub->sc.xmbtp); + if (sub->sc.xfile) + xfile_destroy(sub->sc.xfile); + + sc->ilock_flags = sub->sc.ilock_flags; + sc->orphanage_ilock_flags = sub->sc.orphanage_ilock_flags; + sc->temp_ilock_flags = sub->sc.temp_ilock_flags; + + kfree(sub); +} + +/* Free all the resources and finish the transactions. */ +STATIC int +xchk_teardown( + struct xfs_scrub *sc, + int error) +{ + xchk_ag_free(sc, &sc->sa); + xchk_rtgroup_btcur_free(&sc->sr); + + if (sc->tp) { + if (error == 0 && (sc->sm->sm_flags & XFS_SCRUB_IFLAG_REPAIR)) + error = xfs_trans_commit(sc->tp); + else + xfs_trans_cancel(sc->tp); + sc->tp = NULL; + } + if (sc->sr.rtg) + xchk_rtgroup_free(sc, &sc->sr); + if (sc->ip) { + if (sc->ilock_flags) + xchk_iunlock(sc, sc->ilock_flags); + xchk_irele(sc, sc->ip); + sc->ip = NULL; + } + if (sc->flags & XCHK_HAVE_FREEZE_PROT) { + sc->flags &= ~XCHK_HAVE_FREEZE_PROT; + mnt_drop_write_file(sc->file); + } + if (sc->xmbtp) { + xmbuf_free(sc->xmbtp); + sc->xmbtp = NULL; + } + if (sc->xfile) { + xfile_destroy(sc->xfile); + sc->xfile = NULL; + } + if (sc->buf) { + if (sc->buf_cleanup) + sc->buf_cleanup(sc->buf); + kvfree(sc->buf); + sc->buf_cleanup = NULL; + sc->buf = NULL; + } + + xrep_tempfile_rele(sc); + xrep_orphanage_rele(sc); + xchk_fsgates_disable(sc); + return error; +} + +/* Scrubbing dispatch. */ + +static const struct xchk_meta_ops meta_scrub_ops[] = { + [XFS_SCRUB_TYPE_PROBE] = { /* ioctl presence test */ + .type = ST_NONE, + .setup = xchk_setup_fs, + .scrub = xchk_probe, + .repair = xrep_probe, + }, + [XFS_SCRUB_TYPE_SB] = { /* superblock */ + .type = ST_PERAG, + .setup = xchk_setup_agheader, + .scrub = xchk_superblock, + .repair = xrep_superblock, + }, + [XFS_SCRUB_TYPE_AGF] = { /* agf */ + .type = ST_PERAG, + .setup = xchk_setup_agheader, + .scrub = xchk_agf, + .repair = xrep_agf, + }, + [XFS_SCRUB_TYPE_AGFL]= { /* agfl */ + .type = ST_PERAG, + .setup = xchk_setup_agheader, + .scrub = xchk_agfl, + .repair = xrep_agfl, + }, + [XFS_SCRUB_TYPE_AGI] = { /* agi */ + .type = ST_PERAG, + .setup = xchk_setup_agheader, + .scrub = xchk_agi, + .repair = xrep_agi, + }, + [XFS_SCRUB_TYPE_BNOBT] = { /* bnobt */ + .type = ST_PERAG, + .setup = xchk_setup_ag_allocbt, + .scrub = xchk_allocbt, + .repair = xrep_allocbt, + .repair_eval = xrep_revalidate_allocbt, + }, + [XFS_SCRUB_TYPE_CNTBT] = { /* cntbt */ + .type = ST_PERAG, + .setup = xchk_setup_ag_allocbt, + .scrub = xchk_allocbt, + .repair = xrep_allocbt, + .repair_eval = xrep_revalidate_allocbt, + }, + [XFS_SCRUB_TYPE_INOBT] = { /* inobt */ + .type = ST_PERAG, + .setup = xchk_setup_ag_iallocbt, + .scrub = xchk_iallocbt, + .repair = xrep_iallocbt, + .repair_eval = xrep_revalidate_iallocbt, + }, + [XFS_SCRUB_TYPE_FINOBT] = { /* finobt */ + .type = ST_PERAG, + .setup = xchk_setup_ag_iallocbt, + .scrub = xchk_iallocbt, + .has = xfs_has_finobt, + .repair = xrep_iallocbt, + .repair_eval = xrep_revalidate_iallocbt, + }, + [XFS_SCRUB_TYPE_RMAPBT] = { /* rmapbt */ + .type = ST_PERAG, + .setup = xchk_setup_ag_rmapbt, + .scrub = xchk_rmapbt, + .has = xfs_has_rmapbt, + .repair = xrep_rmapbt, + }, + [XFS_SCRUB_TYPE_REFCNTBT] = { /* refcountbt */ + .type = ST_PERAG, + .setup = xchk_setup_ag_refcountbt, + .scrub = xchk_refcountbt, + .has = xfs_has_reflink, + .repair = xrep_refcountbt, + }, + [XFS_SCRUB_TYPE_INODE] = { /* inode record */ + .type = ST_INODE, + .setup = xchk_setup_inode, + .scrub = xchk_inode, + .repair = xrep_inode, + }, + [XFS_SCRUB_TYPE_BMBTD] = { /* inode data fork */ + .type = ST_INODE, + .setup = xchk_setup_inode_bmap, + .scrub = xchk_bmap_data, + .repair = xrep_bmap_data, + }, + [XFS_SCRUB_TYPE_BMBTA] = { /* inode attr fork */ + .type = ST_INODE, + .setup = xchk_setup_inode_bmap, + .scrub = xchk_bmap_attr, + .repair = xrep_bmap_attr, + }, + [XFS_SCRUB_TYPE_BMBTC] = { /* inode CoW fork */ + .type = ST_INODE, + .setup = xchk_setup_inode_bmap, + .scrub = xchk_bmap_cow, + .repair = xrep_bmap_cow, + }, + [XFS_SCRUB_TYPE_DIR] = { /* directory */ + .type = ST_INODE, + .setup = xchk_setup_directory, + .scrub = xchk_directory, + .repair = xrep_directory, + }, + [XFS_SCRUB_TYPE_XATTR] = { /* extended attributes */ + .type = ST_INODE, + .setup = xchk_setup_xattr, + .scrub = xchk_xattr, + .repair = xrep_xattr, + }, + [XFS_SCRUB_TYPE_SYMLINK] = { /* symbolic link */ + .type = ST_INODE, + .setup = xchk_setup_symlink, + .scrub = xchk_symlink, + .repair = xrep_symlink, + }, + [XFS_SCRUB_TYPE_PARENT] = { /* parent pointers */ + .type = ST_INODE, + .setup = xchk_setup_parent, + .scrub = xchk_parent, + .repair = xrep_parent, + }, + [XFS_SCRUB_TYPE_RTBITMAP] = { /* realtime bitmap */ + .type = ST_RTGROUP, + .has = xfs_has_nonzoned, + .setup = xchk_setup_rtbitmap, + .scrub = xchk_rtbitmap, + .repair = xrep_rtbitmap, + }, + [XFS_SCRUB_TYPE_RTSUM] = { /* realtime summary */ + .type = ST_RTGROUP, + .has = xfs_has_nonzoned, + .setup = xchk_setup_rtsummary, + .scrub = xchk_rtsummary, + .repair = xrep_rtsummary, + }, + [XFS_SCRUB_TYPE_UQUOTA] = { /* user quota */ + .type = ST_FS, + .setup = xchk_setup_quota, + .scrub = xchk_quota, + .repair = xrep_quota, + }, + [XFS_SCRUB_TYPE_GQUOTA] = { /* group quota */ + .type = ST_FS, + .setup = xchk_setup_quota, + .scrub = xchk_quota, + .repair = xrep_quota, + }, + [XFS_SCRUB_TYPE_PQUOTA] = { /* project quota */ + .type = ST_FS, + .setup = xchk_setup_quota, + .scrub = xchk_quota, + .repair = xrep_quota, + }, + [XFS_SCRUB_TYPE_FSCOUNTERS] = { /* fs summary counters */ + .type = ST_FS, + .setup = xchk_setup_fscounters, + .scrub = xchk_fscounters, + .repair = xrep_fscounters, + }, + [XFS_SCRUB_TYPE_QUOTACHECK] = { /* quota counters */ + .type = ST_FS, + .setup = xchk_setup_quotacheck, + .scrub = xchk_quotacheck, + .repair = xrep_quotacheck, + }, + [XFS_SCRUB_TYPE_NLINKS] = { /* inode link counts */ + .type = ST_FS, + .setup = xchk_setup_nlinks, + .scrub = xchk_nlinks, + .repair = xrep_nlinks, + }, + [XFS_SCRUB_TYPE_HEALTHY] = { /* fs healthy; clean all reminders */ + .type = ST_FS, + .setup = xchk_setup_fs, + .scrub = xchk_health_record, + .repair = xrep_notsupported, + }, + [XFS_SCRUB_TYPE_DIRTREE] = { /* directory tree structure */ + .type = ST_INODE, + .setup = xchk_setup_dirtree, + .scrub = xchk_dirtree, + .has = xfs_has_parent, + .repair = xrep_dirtree, + }, + [XFS_SCRUB_TYPE_METAPATH] = { /* metadata directory tree path */ + .type = ST_GENERIC, + .setup = xchk_setup_metapath, + .scrub = xchk_metapath, + .has = xfs_has_metadir, + .repair = xrep_metapath, + }, + [XFS_SCRUB_TYPE_RGSUPER] = { /* realtime group superblock */ + .type = ST_RTGROUP, + .setup = xchk_setup_rgsuperblock, + .scrub = xchk_rgsuperblock, + .has = xfs_has_rtsb, + .repair = xrep_rgsuperblock, + }, + [XFS_SCRUB_TYPE_RTRMAPBT] = { /* realtime group rmapbt */ + .type = ST_RTGROUP, + .setup = xchk_setup_rtrmapbt, + .scrub = xchk_rtrmapbt, + .has = xfs_has_rtrmapbt, + .repair = xrep_rtrmapbt, + }, + [XFS_SCRUB_TYPE_RTREFCBT] = { /* realtime refcountbt */ + .type = ST_RTGROUP, + .setup = xchk_setup_rtrefcountbt, + .scrub = xchk_rtrefcountbt, + .has = xfs_has_rtreflink, + .repair = xrep_rtrefcountbt, + }, +}; + +static int +xchk_validate_inputs( + struct xfs_mount *mp, + struct xfs_scrub_metadata *sm) +{ + int error; + const struct xchk_meta_ops *ops; + + error = -EINVAL; + /* Check our inputs. */ + sm->sm_flags &= ~XFS_SCRUB_FLAGS_OUT; + if (sm->sm_flags & ~XFS_SCRUB_FLAGS_IN) + goto out; + /* sm_reserved[] must be zero */ + if (memchr_inv(sm->sm_reserved, 0, sizeof(sm->sm_reserved))) + goto out; + + error = -ENOENT; + /* Do we know about this type of metadata? */ + if (sm->sm_type >= XFS_SCRUB_TYPE_NR) + goto out; + ops = &meta_scrub_ops[sm->sm_type]; + if (ops->setup == NULL || ops->scrub == NULL) + goto out; + /* Does this fs even support this type of metadata? */ + if (ops->has && !ops->has(mp)) + goto out; + + error = -EINVAL; + /* restricting fields must be appropriate for type */ + switch (ops->type) { + case ST_NONE: + case ST_FS: + if (sm->sm_ino || sm->sm_gen || sm->sm_agno) + goto out; + break; + case ST_PERAG: + if (sm->sm_ino || sm->sm_gen || + sm->sm_agno >= mp->m_sb.sb_agcount) + goto out; + break; + case ST_INODE: + if (sm->sm_agno || (sm->sm_gen && !sm->sm_ino)) + goto out; + break; + case ST_GENERIC: + break; + case ST_RTGROUP: + if (sm->sm_ino || sm->sm_gen) + goto out; + if (xfs_has_rtgroups(mp)) { + /* + * On a rtgroups filesystem, there won't be an rtbitmap + * or rtsummary file for group 0 unless there's + * actually a realtime volume attached. However, older + * xfs_scrub always calls the rtbitmap/rtsummary + * scrubbers with sm_agno==0 so transform the error + * code to ENOENT. + */ + if (sm->sm_agno >= mp->m_sb.sb_rgcount) { + if (sm->sm_agno == 0) + error = -ENOENT; + goto out; + } + } else { + /* + * Prior to rtgroups, the rtbitmap/rtsummary scrubbers + * accepted sm_agno==0, so we still accept that for + * scrubbing pre-rtgroups filesystems. + */ + if (sm->sm_agno != 0) + goto out; + } + break; + default: + goto out; + } + + /* No rebuild without repair. */ + if ((sm->sm_flags & XFS_SCRUB_IFLAG_FORCE_REBUILD) && + !(sm->sm_flags & XFS_SCRUB_IFLAG_REPAIR)) + return -EINVAL; + + /* + * We only want to repair read-write v5+ filesystems. Defer the check + * for ops->repair until after our scrub confirms that we need to + * perform repairs so that we avoid failing due to not supporting + * repairing an object that doesn't need repairs. + */ + if (sm->sm_flags & XFS_SCRUB_IFLAG_REPAIR) { + error = -EOPNOTSUPP; + if (!xfs_has_crc(mp)) + goto out; + + error = -EROFS; + if (xfs_is_readonly(mp)) + goto out; + } + + error = 0; +out: + return error; +} + +#ifdef CONFIG_XFS_ONLINE_REPAIR +static inline void xchk_postmortem(struct xfs_scrub *sc) +{ + /* + * Userspace asked us to repair something, we repaired it, rescanned + * it, and the rescan says it's still broken. Scream about this in + * the system logs. + */ + if ((sc->sm->sm_flags & XFS_SCRUB_IFLAG_REPAIR) && + (sc->sm->sm_flags & (XFS_SCRUB_OFLAG_CORRUPT | + XFS_SCRUB_OFLAG_XCORRUPT))) + xrep_failure(sc->mp); +} +#else +static inline void xchk_postmortem(struct xfs_scrub *sc) +{ + /* + * Userspace asked us to scrub something, it's broken, and we have no + * way of fixing it. Scream in the logs. + */ + if (sc->sm->sm_flags & (XFS_SCRUB_OFLAG_CORRUPT | + XFS_SCRUB_OFLAG_XCORRUPT)) + xfs_alert_ratelimited(sc->mp, + "Corruption detected during scrub."); +} +#endif /* CONFIG_XFS_ONLINE_REPAIR */ + +/* + * Create a new scrub context from an existing one, but with a different scrub + * type. + */ +struct xfs_scrub_subord * +xchk_scrub_create_subord( + struct xfs_scrub *sc, + unsigned int subtype) +{ + struct xfs_scrub_subord *sub; + + sub = kzalloc_obj(*sub, XCHK_GFP_FLAGS); + if (!sub) + return NULL; + + sub->old_smtype = sc->sm->sm_type; + sub->old_smflags = sc->sm->sm_flags; + sub->parent_sc = sc; + memcpy(&sub->sc, sc, sizeof(struct xfs_scrub)); + sub->sc.ops = &meta_scrub_ops[subtype]; + sub->sc.sm->sm_type = subtype; + sub->sc.sm->sm_flags &= ~XFS_SCRUB_FLAGS_OUT; + sub->sc.buf = NULL; + sub->sc.buf_cleanup = NULL; + sub->sc.xfile = NULL; + sub->sc.xmbtp = NULL; + + return sub; +} + +/* Dispatch metadata scrubbing. */ +STATIC int +xfs_scrub_metadata( + struct file *file, + struct xfs_scrub_metadata *sm) +{ + struct xchk_stats_run run = { }; + struct xfs_scrub *sc; + struct xfs_mount *mp = XFS_I(file_inode(file))->i_mount; + u64 check_start; + int error = 0; + + BUILD_BUG_ON(sizeof(meta_scrub_ops) != + (sizeof(struct xchk_meta_ops) * XFS_SCRUB_TYPE_NR)); + + trace_xchk_start(XFS_I(file_inode(file)), sm, error); + + /* Forbidden if we are shut down or mounted norecovery. */ + error = -ESHUTDOWN; + if (xfs_is_shutdown(mp)) + goto out; + error = -ENOTRECOVERABLE; + if (xfs_has_norecovery(mp)) + goto out; + + error = xchk_validate_inputs(mp, sm); + if (error) + goto out; + + sc = kzalloc_obj(struct xfs_scrub, XCHK_GFP_FLAGS); + if (!sc) { + error = -ENOMEM; + goto out; + } + + sc->mp = mp; + sc->file = file; + sc->sm = sm; + sc->ops = &meta_scrub_ops[sm->sm_type]; + sc->sick_mask = xchk_health_mask_for_scrub_type(sm->sm_type); + sc->relax = INIT_XCHK_RELAX; +retry_op: + /* + * When repairs are allowed, prevent freezing or readonly remount while + * scrub is running with a real transaction. + */ + if (sm->sm_flags & XFS_SCRUB_IFLAG_REPAIR) { + error = mnt_want_write_file(sc->file); + if (error) + goto out_sc; + + sc->flags |= XCHK_HAVE_FREEZE_PROT; + } + + /* Set up for the operation. */ + error = sc->ops->setup(sc); + if (error == -EDEADLOCK && !(sc->flags & XCHK_TRY_HARDER)) + goto try_harder; + if (error == -ECHRNG && !(sc->flags & XCHK_NEED_DRAIN)) + goto need_drain; + if (error) + goto out_teardown; + + /* Scrub for errors. */ + check_start = xchk_stats_now(); + if ((sc->flags & XREP_ALREADY_FIXED) && sc->ops->repair_eval != NULL) + error = sc->ops->repair_eval(sc); + else + error = sc->ops->scrub(sc); + run.scrub_ns += xchk_stats_elapsed_ns(check_start); + if (error == -EDEADLOCK && !(sc->flags & XCHK_TRY_HARDER)) + goto try_harder; + if (error == -ECHRNG && !(sc->flags & XCHK_NEED_DRAIN)) + goto need_drain; + if (error || (sm->sm_flags & XFS_SCRUB_OFLAG_INCOMPLETE)) + goto out_teardown; + + xchk_update_health(sc); + + if (xchk_could_repair(sc)) { + /* + * If userspace asked for a repair but it wasn't necessary, + * report that back to userspace. + */ + if (!xrep_will_attempt(sc)) { + sc->sm->sm_flags |= XFS_SCRUB_OFLAG_NO_REPAIR_NEEDED; + goto out_nofix; + } + + /* + * If it's broken, userspace wants us to fix it, and we haven't + * already tried to fix it, then attempt a repair. + */ + error = xrep_attempt(sc, &run); + if (error == -EAGAIN) { + /* + * Either the repair function succeeded or it couldn't + * get all the resources it needs; either way, we go + * back to the beginning and call the scrub function. + */ + error = xchk_teardown(sc, 0); + if (error) { + xrep_failure(mp); + goto out_sc; + } + goto retry_op; + } + } + +out_nofix: + xchk_postmortem(sc); +out_teardown: + error = xchk_teardown(sc, error); +out_sc: + if (error != -ENOENT) + xchk_stats_merge(mp, sm, &run); + kfree(sc); +out: + trace_xchk_done(XFS_I(file_inode(file)), sm, error); + if (error == -EFSCORRUPTED || error == -EFSBADCRC) { + sm->sm_flags |= XFS_SCRUB_OFLAG_CORRUPT; + error = 0; + } + return error; +need_drain: + error = xchk_teardown(sc, 0); + if (error) + goto out_sc; + sc->flags |= XCHK_NEED_DRAIN; + run.retries++; + goto retry_op; +try_harder: + /* + * Scrubbers return -EDEADLOCK to mean 'try harder'. Tear down + * everything we hold, then set up again with preparation for + * worst-case scenarios. + */ + error = xchk_teardown(sc, 0); + if (error) + goto out_sc; + sc->flags |= XCHK_TRY_HARDER; + run.retries++; + goto retry_op; +} + +/* Scrub one aspect of one piece of metadata. */ +int +xfs_ioc_scrub_metadata( + struct file *file, + void __user *arg) +{ + struct xfs_scrub_metadata scrub; + int error; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (copy_from_user(&scrub, arg, sizeof(scrub))) + return -EFAULT; + + error = xfs_scrub_metadata(file, &scrub); + if (error) + return error; + + if (copy_to_user(arg, &scrub, sizeof(scrub))) + return -EFAULT; + + return 0; +} + +/* Decide if there have been any scrub failures up to this point. */ +static inline int +xfs_scrubv_check_barrier( + struct xfs_mount *mp, + const struct xfs_scrub_vec *vectors, + const struct xfs_scrub_vec *stop_vec) +{ + const struct xfs_scrub_vec *v; + __u32 failmask; + + failmask = stop_vec->sv_flags & XFS_SCRUB_FLAGS_OUT; + + for (v = vectors; v < stop_vec; v++) { + if (v->sv_type == XFS_SCRUB_TYPE_BARRIER) + continue; + + /* + * Runtime errors count as a previous failure, except the ones + * used to ask userspace to retry. + */ + switch (v->sv_ret) { + case -EBUSY: + case -ENOENT: + case -EUSERS: + case 0: + break; + default: + return -ECANCELED; + } + + /* + * If any of the out-flags on the scrub vector match the mask + * that was set on the barrier vector, that's a previous fail. + */ + if (v->sv_flags & failmask) + return -ECANCELED; + } + + return 0; +} + +/* + * If the caller provided us with a nonzero inode number that isn't the ioctl + * file, try to grab a reference to it to eliminate all further untrusted inode + * lookups. If we can't get the inode, let each scrub function try again. + */ +STATIC struct xfs_inode * +xchk_scrubv_open_by_handle( + struct xfs_mount *mp, + const struct xfs_scrub_vec_head *head) +{ + struct xfs_trans *tp; + struct xfs_inode *ip; + int error; + + tp = xfs_trans_alloc_empty(mp); + error = xfs_iget(mp, tp, head->svh_ino, XCHK_IGET_FLAGS, 0, &ip); + xfs_trans_cancel(tp); + if (error) + return NULL; + + if (VFS_I(ip)->i_generation != head->svh_gen) { + xfs_irele(ip); + return NULL; + } + + return ip; +} + +/* Vectored scrub implementation to reduce ioctl calls. */ +int +xfs_ioc_scrubv_metadata( + struct file *file, + void __user *arg) +{ + struct xfs_scrub_vec_head head; + struct xfs_scrub_vec_head __user *uhead = arg; + struct xfs_scrub_vec *vectors; + struct xfs_scrub_vec __user *uvectors; + struct xfs_inode *ip_in = XFS_I(file_inode(file)); + struct xfs_mount *mp = ip_in->i_mount; + struct xfs_inode *handle_ip = NULL; + struct xfs_scrub_vec *v; + size_t vec_bytes; + unsigned int i; + int error = 0; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (copy_from_user(&head, uhead, sizeof(head))) + return -EFAULT; + + if (head.svh_reserved) + return -EINVAL; + if (head.svh_flags & ~XFS_SCRUB_VEC_FLAGS_ALL) + return -EINVAL; + if (head.svh_nr == 0) + return 0; + + vec_bytes = array_size(head.svh_nr, sizeof(struct xfs_scrub_vec)); + if (vec_bytes > PAGE_SIZE) + return -ENOMEM; + + uvectors = u64_to_user_ptr(head.svh_vectors); + vectors = memdup_user(uvectors, vec_bytes); + if (IS_ERR(vectors)) + return PTR_ERR(vectors); + + trace_xchk_scrubv_start(ip_in, &head); + + for (i = 0, v = vectors; i < head.svh_nr; i++, v++) { + if (v->sv_reserved) { + error = -EINVAL; + goto out_free; + } + + if (v->sv_type == XFS_SCRUB_TYPE_BARRIER && + (v->sv_flags & ~XFS_SCRUB_FLAGS_OUT)) { + error = -EINVAL; + goto out_free; + } + + trace_xchk_scrubv_item(mp, &head, i, v); + } + + /* + * If the caller wants us to do a scrub-by-handle and the file used to + * call the ioctl is not the same file, load the incore inode and pin + * it across all the scrubv actions to avoid repeated UNTRUSTED + * lookups. The reference is not passed to deeper layers of scrub + * because each scrubber gets to decide its own strategy and return + * values for getting an inode. + */ + if (head.svh_ino && head.svh_ino != I_INO(ip_in)) + handle_ip = xchk_scrubv_open_by_handle(mp, &head); + + /* Run all the scrubbers. */ + for (i = 0, v = vectors; i < head.svh_nr; i++, v++) { + struct xfs_scrub_metadata sm = { + .sm_type = v->sv_type, + .sm_flags = v->sv_flags, + .sm_ino = head.svh_ino, + .sm_gen = head.svh_gen, + .sm_agno = head.svh_agno, + }; + + if (v->sv_type == XFS_SCRUB_TYPE_BARRIER) { + v->sv_ret = xfs_scrubv_check_barrier(mp, vectors, v); + if (v->sv_ret) { + trace_xchk_scrubv_barrier_fail(mp, &head, i, v); + break; + } + + continue; + } + + v->sv_ret = xfs_scrub_metadata(file, &sm); + v->sv_flags = sm.sm_flags; + + trace_xchk_scrubv_outcome(mp, &head, i, v); + + if (head.svh_rest_us) { + ktime_t expires; + + expires = ktime_add_ns(ktime_get(), + head.svh_rest_us * 1000); + set_current_state(TASK_KILLABLE); + schedule_hrtimeout(&expires, HRTIMER_MODE_ABS); + } + + if (fatal_signal_pending(current)) { + error = -EINTR; + goto out_free; + } + } + + if (copy_to_user(uvectors, vectors, vec_bytes) || + copy_to_user(uhead, &head, sizeof(head))) { + error = -EFAULT; + goto out_free; + } + +out_free: + if (handle_ip) + xfs_irele(handle_ip); + kfree(vectors); + return error; +} diff --git a/fs/xfs/scrub/scrub.h b/fs/xfs/scrub/scrub.h new file mode 100644 index 000000000..6d7d3523b --- /dev/null +++ b/fs/xfs/scrub/scrub.h @@ -0,0 +1,350 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_SCRUB_H__ +#define __XFS_SCRUB_SCRUB_H__ + +struct xfs_scrub; + +struct xchk_relax { + unsigned long next_resched; + unsigned int resched_nr; + bool killable; +}; + +/* Yield to the scheduler at most 10x per second. */ +#define XCHK_RELAX_NEXT (jiffies + (HZ / 10)) + +#define INIT_XCHK_RELAX \ + (struct xchk_relax){ \ + .next_resched = XCHK_RELAX_NEXT, \ + .resched_nr = 0, \ + .killable = true, \ + } + +/* + * Relax during a scrub operation and exit if there's a fatal signal pending. + * + * If preemption is disabled, we need to yield to the scheduler every now and + * then so that we don't run afoul of the soft lockup watchdog or RCU stall + * detector. cond_resched calls are somewhat expensive (~5ns) so we want to + * ratelimit this to 10x per second. Amortize the cost of the other checks by + * only doing it once every 100 calls. + */ +static inline int xchk_maybe_relax(struct xchk_relax *widget) +{ + /* Amortize the cost of scheduling and checking signals. */ + if (likely(++widget->resched_nr < 100)) + return 0; + widget->resched_nr = 0; + + if (unlikely(widget->next_resched <= jiffies)) { + cond_resched(); + widget->next_resched = XCHK_RELAX_NEXT; + } + + if (widget->killable && fatal_signal_pending(current)) + return -EINTR; + + return 0; +} + +/* + * Standard flags for allocating memory within scrub. NOFS context is + * configured by the process allocation scope. Scrub and repair must be able + * to back out gracefully if there isn't enough memory. Force-cast to avoid + * complaints from static checkers. + */ +#define XCHK_GFP_FLAGS ((__force gfp_t)(GFP_KERNEL | __GFP_NOWARN | \ + __GFP_RETRY_MAYFAIL)) + +/* + * For opening files by handle for fsck operations, we don't trust the inumber + * or the allocation state; therefore, perform an untrusted lookup. We don't + * want these inodes to pollute the cache, so mark them for immediate removal. + */ +#define XCHK_IGET_FLAGS (XFS_IGET_UNTRUSTED | XFS_IGET_DONTCACHE) + +/* Type info and names for the scrub types. */ +enum xchk_type { + ST_NONE = 1, /* disabled */ + ST_PERAG, /* per-AG metadata */ + ST_FS, /* per-FS metadata */ + ST_INODE, /* per-inode metadata */ + ST_GENERIC, /* determined by the scrubber */ + ST_RTGROUP, /* rtgroup metadata */ +}; + +struct xchk_meta_ops { + /* Acquire whatever resources are needed for the operation. */ + int (*setup)(struct xfs_scrub *sc); + + /* Examine metadata for errors. */ + int (*scrub)(struct xfs_scrub *); + + /* Repair or optimize the metadata. */ + int (*repair)(struct xfs_scrub *); + + /* + * Re-scrub the metadata we repaired, in case there's extra work that + * we need to do to check our repair work. If this is NULL, we'll use + * the ->scrub function pointer, assuming that the regular scrub is + * sufficient. + */ + int (*repair_eval)(struct xfs_scrub *sc); + + /* Decide if we even have this piece of metadata. */ + bool (*has)(const struct xfs_mount *); + + /* type describing required/allowed inputs */ + enum xchk_type type; +}; + +/* Buffer pointers and btree cursors for an entire AG. */ +struct xchk_ag { + struct xfs_perag *pag; + + /* AG btree roots */ + struct xfs_buf *agf_bp; + struct xfs_buf *agi_bp; + + /* AG btrees */ + struct xfs_btree_cur *bno_cur; + struct xfs_btree_cur *cnt_cur; + struct xfs_btree_cur *ino_cur; + struct xfs_btree_cur *fino_cur; + struct xfs_btree_cur *rmap_cur; + struct xfs_btree_cur *refc_cur; +}; + +/* Inode lock state for the RT volume. */ +struct xchk_rt { + /* incore rtgroup, if applicable */ + struct xfs_rtgroup *rtg; + + /* XFS_RTGLOCK_* lock state if locked */ + unsigned int rtlock_flags; + + /* rtgroup btrees */ + struct xfs_btree_cur *rmap_cur; + struct xfs_btree_cur *refc_cur; +}; + +struct xfs_scrub { + /* General scrub state. */ + struct xfs_mount *mp; + struct xfs_scrub_metadata *sm; + const struct xchk_meta_ops *ops; + struct xfs_trans *tp; + + /* File that scrub was called with. */ + struct file *file; + + /* + * File that is undergoing the scrub operation. This can differ from + * the file that scrub was called with if we're checking file-based fs + * metadata (e.g. rt bitmaps) or if we're doing a scrub-by-handle for + * something that can't be opened directly (e.g. symlinks). + */ + struct xfs_inode *ip; + + /* Kernel memory buffer used by scrubbers; freed at teardown. */ + void *buf; + + /* + * Clean up resources owned by whatever is in the buffer. Cleanup can + * be deferred with this hook as a means for scrub functions to pass + * data to repair functions. This function must not free the buffer + * itself. + */ + void (*buf_cleanup)(void *buf); + + /* xfile used by the scrubbers; freed at teardown. */ + struct xfile *xfile; + + /* buffer target for in-memory btrees; also freed at teardown. */ + struct xfs_buftarg *xmbtp; + + /* Lock flags for @ip. */ + uint ilock_flags; + + /* The orphanage, for stashing files that have lost their parent. */ + uint orphanage_ilock_flags; + struct xfs_inode *orphanage; + + /* A temporary file on this filesystem, for staging new metadata. */ + struct xfs_inode *tempip; + uint temp_ilock_flags; + + /* See the XCHK/XREP state flags below. */ + unsigned int flags; + + /* + * The XFS_SICK_* flags that correspond to the metadata being scrubbed + * or repaired. We will use this mask to update the in-core fs health + * status with whatever we find. + */ + unsigned int sick_mask; + + /* + * Clear these XFS_SICK_* flags but only if the scan is ok. Useful for + * removing ZAPPED flags after a repair. + */ + unsigned int healthy_mask; + + /* next time we want to cond_resched() */ + struct xchk_relax relax; + + /* State tracking for single-AG operations. */ + struct xchk_ag sa; + + /* State tracking for realtime operations. */ + struct xchk_rt sr; +}; + +/* XCHK state flags grow up from zero, XREP state flags grown down from 2^31 */ +#define XCHK_TRY_HARDER (1U << 0) /* can't get resources, try again */ +#define XCHK_HAVE_FREEZE_PROT (1U << 1) /* do we have freeze protection? */ +#define XCHK_FSGATES_DRAIN (1U << 2) /* defer ops draining enabled */ +#define XCHK_NEED_DRAIN (1U << 3) /* scrub needs to drain defer ops */ +#define XCHK_FSGATES_QUOTA (1U << 4) /* quota live update enabled */ +#define XCHK_FSGATES_DIRENTS (1U << 5) /* directory live update enabled */ +#define XCHK_FSGATES_RMAP (1U << 6) /* rmapbt live update enabled */ +#define XREP_RESET_PERAG_RESV (1U << 30) /* must reset AG space reservation */ +#define XREP_ALREADY_FIXED (1U << 31) /* checking our repair work */ + +/* + * The XCHK_FSGATES* flags reflect functionality in the main filesystem that + * are only enabled for this particular online fsck. When not in use, the + * features are gated off via dynamic code patching, which is why the state + * must be enabled during scrub setup and can only be torn down afterwards. + */ +#define XCHK_FSGATES_ALL (XCHK_FSGATES_DRAIN | \ + XCHK_FSGATES_QUOTA | \ + XCHK_FSGATES_DIRENTS | \ + XCHK_FSGATES_RMAP) + +struct xfs_scrub_subord { + struct xfs_scrub sc; + struct xfs_scrub *parent_sc; + unsigned int old_smtype; + unsigned int old_smflags; +}; + +struct xfs_scrub_subord *xchk_scrub_create_subord(struct xfs_scrub *sc, + unsigned int subtype); +void xchk_scrub_free_subord(struct xfs_scrub_subord *sub); + +/* + * We /could/ terminate a scrub/repair operation early. If we're not + * in a good place to continue (fatal signal, etc.) then bail out. + * Note that we're careful not to make any judgements about *error. + */ +static inline bool +xchk_should_terminate( + struct xfs_scrub *sc, + int *error) +{ + if (xchk_maybe_relax(&sc->relax)) { + if (*error == 0) + *error = -EINTR; + return true; + } + return false; +} + +static inline int xchk_nothing(struct xfs_scrub *sc) +{ + return -ENOENT; +} + +/* Metadata scrubbers */ +int xchk_tester(struct xfs_scrub *sc); +int xchk_superblock(struct xfs_scrub *sc); +int xchk_agf(struct xfs_scrub *sc); +int xchk_agfl(struct xfs_scrub *sc); +int xchk_agi(struct xfs_scrub *sc); +int xchk_allocbt(struct xfs_scrub *sc); +int xchk_iallocbt(struct xfs_scrub *sc); +int xchk_rmapbt(struct xfs_scrub *sc); +int xchk_refcountbt(struct xfs_scrub *sc); +int xchk_inode(struct xfs_scrub *sc); +int xchk_bmap_data(struct xfs_scrub *sc); +int xchk_bmap_attr(struct xfs_scrub *sc); +int xchk_bmap_cow(struct xfs_scrub *sc); +int xchk_directory(struct xfs_scrub *sc); +int xchk_xattr(struct xfs_scrub *sc); +int xchk_symlink(struct xfs_scrub *sc); +int xchk_parent(struct xfs_scrub *sc); +int xchk_dirtree(struct xfs_scrub *sc); +int xchk_metapath(struct xfs_scrub *sc); +#ifdef CONFIG_XFS_RT +int xchk_rtbitmap(struct xfs_scrub *sc); +int xchk_rtsummary(struct xfs_scrub *sc); +int xchk_rgsuperblock(struct xfs_scrub *sc); +int xchk_rtrmapbt(struct xfs_scrub *sc); +int xchk_rtrefcountbt(struct xfs_scrub *sc); +#else +# define xchk_rtbitmap xchk_nothing +# define xchk_rtsummary xchk_nothing +# define xchk_rgsuperblock xchk_nothing +# define xchk_rtrmapbt xchk_nothing +# define xchk_rtrefcountbt xchk_nothing +#endif +#ifdef CONFIG_XFS_QUOTA +int xchk_quota(struct xfs_scrub *sc); +int xchk_quotacheck(struct xfs_scrub *sc); +#else +# define xchk_quota xchk_nothing +# define xchk_quotacheck xchk_nothing +#endif +int xchk_fscounters(struct xfs_scrub *sc); +int xchk_nlinks(struct xfs_scrub *sc); + +/* cross-referencing helpers */ +void xchk_xref_is_used_space(struct xfs_scrub *sc, xfs_agblock_t agbno, + xfs_extlen_t len); +void xchk_xref_is_not_inode_chunk(struct xfs_scrub *sc, xfs_agblock_t agbno, + xfs_extlen_t len); +void xchk_xref_is_inode_chunk(struct xfs_scrub *sc, xfs_agblock_t agbno, + xfs_extlen_t len); +void xchk_xref_is_only_owned_by(struct xfs_scrub *sc, xfs_agblock_t agbno, + xfs_extlen_t len, const struct xfs_owner_info *oinfo); +void xchk_xref_is_not_owned_by(struct xfs_scrub *sc, xfs_agblock_t agbno, + xfs_extlen_t len, const struct xfs_owner_info *oinfo); +void xchk_xref_has_no_owner(struct xfs_scrub *sc, xfs_agblock_t agbno, + xfs_extlen_t len); +void xchk_xref_is_cow_staging(struct xfs_scrub *sc, xfs_agblock_t bno, + xfs_extlen_t len); +void xchk_xref_is_not_shared(struct xfs_scrub *sc, xfs_agblock_t bno, + xfs_extlen_t len); +void xchk_xref_is_not_cow_staging(struct xfs_scrub *sc, xfs_agblock_t bno, + xfs_extlen_t len); +#ifdef CONFIG_XFS_RT +void xchk_xref_is_used_rt_space(struct xfs_scrub *sc, xfs_rtblock_t rtbno, + xfs_extlen_t len); +void xchk_xref_has_no_rt_owner(struct xfs_scrub *sc, xfs_rgblock_t rgbno, + xfs_extlen_t len); +void xchk_xref_has_rt_owner(struct xfs_scrub *sc, xfs_rgblock_t rgbno, + xfs_extlen_t len); +void xchk_xref_is_only_rt_owned_by(struct xfs_scrub *sc, xfs_rgblock_t rgbno, + xfs_extlen_t len, const struct xfs_owner_info *oinfo); +void xchk_xref_is_rt_cow_staging(struct xfs_scrub *sc, xfs_rgblock_t rgbno, + xfs_extlen_t len); +void xchk_xref_is_not_rt_shared(struct xfs_scrub *sc, xfs_rgblock_t rgbno, + xfs_extlen_t len); +void xchk_xref_is_not_rt_cow_staging(struct xfs_scrub *sc, xfs_rgblock_t rgbno, + xfs_extlen_t len); +#else +# define xchk_xref_is_used_rt_space(sc, rtbno, len) do { } while (0) +# define xchk_xref_has_no_rt_owner(sc, rtbno, len) do { } while (0) +# define xchk_xref_has_rt_owner(sc, rtbno, len) do { } while (0) +# define xchk_xref_is_only_rt_owned_by(sc, bno, len, oinfo) do { } while (0) +# define xchk_xref_is_rt_cow_staging(sc, bno, len) do { } while (0) +# define xchk_xref_is_not_rt_shared(sc, bno, len) do { } while (0) +# define xchk_xref_is_not_rt_cow_staging(sc, bno, len) do { } while (0) +#endif + +#endif /* __XFS_SCRUB_SCRUB_H__ */ diff --git a/fs/xfs/scrub/stats.c b/fs/xfs/scrub/stats.c new file mode 100644 index 000000000..ef3f6abdb --- /dev/null +++ b/fs/xfs/scrub/stats.c @@ -0,0 +1,415 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_sysfs.h" +#include "xfs_btree.h" +#include "xfs_super.h" +#include "scrub/scrub.h" +#include "scrub/stats.h" +#include "scrub/trace.h" + +struct xchk_scrub_stats { + /* all 32-bit counters here */ + + /* checking stats */ + uint32_t invocations; + uint32_t clean; + uint32_t corrupt; + uint32_t preen; + uint32_t xfail; + uint32_t xcorrupt; + uint32_t incomplete; + uint32_t warning; + uint32_t retries; + + /* repair stats */ + uint32_t repair_invocations; + uint32_t repair_success; + + /* all 64-bit items here */ + + /* runtimes */ + uint64_t checktime_us; + uint64_t repairtime_us; + + /* non-counter state must go at the end for clearall */ + spinlock_t css_lock; +}; + +struct xchk_stats { + struct dentry *cs_debugfs; + struct xchk_scrub_stats cs_stats[XFS_SCRUB_TYPE_NR]; +}; + + +static struct xchk_stats global_stats; + +static const char *name_map[XFS_SCRUB_TYPE_NR] = { + [XFS_SCRUB_TYPE_SB] = "sb", + [XFS_SCRUB_TYPE_AGF] = "agf", + [XFS_SCRUB_TYPE_AGFL] = "agfl", + [XFS_SCRUB_TYPE_AGI] = "agi", + [XFS_SCRUB_TYPE_BNOBT] = "bnobt", + [XFS_SCRUB_TYPE_CNTBT] = "cntbt", + [XFS_SCRUB_TYPE_INOBT] = "inobt", + [XFS_SCRUB_TYPE_FINOBT] = "finobt", + [XFS_SCRUB_TYPE_RMAPBT] = "rmapbt", + [XFS_SCRUB_TYPE_REFCNTBT] = "refcountbt", + [XFS_SCRUB_TYPE_INODE] = "inode", + [XFS_SCRUB_TYPE_BMBTD] = "bmapbtd", + [XFS_SCRUB_TYPE_BMBTA] = "bmapbta", + [XFS_SCRUB_TYPE_BMBTC] = "bmapbtc", + [XFS_SCRUB_TYPE_DIR] = "directory", + [XFS_SCRUB_TYPE_XATTR] = "xattr", + [XFS_SCRUB_TYPE_SYMLINK] = "symlink", + [XFS_SCRUB_TYPE_PARENT] = "parent", + [XFS_SCRUB_TYPE_RTBITMAP] = "rtbitmap", + [XFS_SCRUB_TYPE_RTSUM] = "rtsummary", + [XFS_SCRUB_TYPE_UQUOTA] = "usrquota", + [XFS_SCRUB_TYPE_GQUOTA] = "grpquota", + [XFS_SCRUB_TYPE_PQUOTA] = "prjquota", + [XFS_SCRUB_TYPE_FSCOUNTERS] = "fscounters", + [XFS_SCRUB_TYPE_QUOTACHECK] = "quotacheck", + [XFS_SCRUB_TYPE_NLINKS] = "nlinks", + [XFS_SCRUB_TYPE_DIRTREE] = "dirtree", + [XFS_SCRUB_TYPE_METAPATH] = "metapath", + [XFS_SCRUB_TYPE_RGSUPER] = "rgsuper", + [XFS_SCRUB_TYPE_RTRMAPBT] = "rtrmapbt", + [XFS_SCRUB_TYPE_RTREFCBT] = "rtrefcountbt", +}; + +/* Format the scrub stats into a text buffer, similar to pcp style. */ +STATIC ssize_t +xchk_stats_format( + struct xchk_stats *cs, + char *buf, + size_t remaining) +{ + struct xchk_scrub_stats *css = &cs->cs_stats[0]; + unsigned int i; + ssize_t copied = 0; + int ret = 0; + + for (i = 0; i < XFS_SCRUB_TYPE_NR; i++, css++) { + if (!name_map[i]) + continue; + + ret = scnprintf(buf, remaining, + "%s %u %u %u %u %u %u %u %u %u %llu %u %u %llu\n", + name_map[i], + (unsigned int)css->invocations, + (unsigned int)css->clean, + (unsigned int)css->corrupt, + (unsigned int)css->preen, + (unsigned int)css->xfail, + (unsigned int)css->xcorrupt, + (unsigned int)css->incomplete, + (unsigned int)css->warning, + (unsigned int)css->retries, + (unsigned long long)css->checktime_us, + (unsigned int)css->repair_invocations, + (unsigned int)css->repair_success, + (unsigned long long)css->repairtime_us); + if (ret <= 0) + break; + + remaining -= ret; + copied += ret; + buf += ret; + } + + return copied > 0 ? copied : ret; +} + +/* Estimate the worst case buffer size required to hold the whole report. */ +STATIC size_t +xchk_stats_estimate_bufsize( + struct xchk_stats *cs) +{ + struct xchk_scrub_stats *css = &cs->cs_stats[0]; + unsigned int i; + size_t field_width; + size_t ret = 0; + + /* 4294967296 plus one space for each u32 field */ + field_width = 11 * (offsetof(struct xchk_scrub_stats, checktime_us) / + sizeof(uint32_t)); + + /* 18446744073709551615 plus one space for each u64 field */ + field_width += 21 * ((offsetof(struct xchk_scrub_stats, css_lock) - + offsetof(struct xchk_scrub_stats, checktime_us)) / + sizeof(uint64_t)); + + for (i = 0; i < XFS_SCRUB_TYPE_NR; i++, css++) { + if (!name_map[i]) + continue; + + /* name plus one space */ + ret += 1 + strlen(name_map[i]); + + /* all fields, plus newline */ + ret += field_width + 1; + } + + return ret; +} + +/* Clear all counters. */ +STATIC void +xchk_stats_clearall( + struct xchk_stats *cs) +{ + struct xchk_scrub_stats *css = &cs->cs_stats[0]; + unsigned int i; + + for (i = 0; i < XFS_SCRUB_TYPE_NR; i++, css++) { + spin_lock(&css->css_lock); + memset(css, 0, offsetof(struct xchk_scrub_stats, css_lock)); + spin_unlock(&css->css_lock); + } +} + +#define XFS_SCRUB_OFLAG_UNCLEAN (XFS_SCRUB_OFLAG_CORRUPT | \ + XFS_SCRUB_OFLAG_PREEN | \ + XFS_SCRUB_OFLAG_XFAIL | \ + XFS_SCRUB_OFLAG_XCORRUPT | \ + XFS_SCRUB_OFLAG_INCOMPLETE | \ + XFS_SCRUB_OFLAG_WARNING) + +STATIC void +xchk_stats_merge_one( + struct xchk_stats *cs, + const struct xfs_scrub_metadata *sm, + const struct xchk_stats_run *run) +{ + struct xchk_scrub_stats *css; + + if (sm->sm_type >= XFS_SCRUB_TYPE_NR) { + ASSERT(sm->sm_type < XFS_SCRUB_TYPE_NR); + return; + } + + css = &cs->cs_stats[sm->sm_type]; + spin_lock(&css->css_lock); + css->invocations++; + if (!(sm->sm_flags & XFS_SCRUB_OFLAG_UNCLEAN)) + css->clean++; + if (sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + css->corrupt++; + if (sm->sm_flags & XFS_SCRUB_OFLAG_PREEN) + css->preen++; + if (sm->sm_flags & XFS_SCRUB_OFLAG_XFAIL) + css->xfail++; + if (sm->sm_flags & XFS_SCRUB_OFLAG_XCORRUPT) + css->xcorrupt++; + if (sm->sm_flags & XFS_SCRUB_OFLAG_INCOMPLETE) + css->incomplete++; + if (sm->sm_flags & XFS_SCRUB_OFLAG_WARNING) + css->warning++; + css->retries += run->retries; + css->checktime_us += howmany_64(run->scrub_ns, NSEC_PER_USEC); + + if (run->repair_attempted) + css->repair_invocations++; + if (run->repair_succeeded) + css->repair_success++; + css->repairtime_us += howmany_64(run->repair_ns, NSEC_PER_USEC); + spin_unlock(&css->css_lock); +} + +/* Merge these scrub-run stats into the global and mount stat data. */ +void +xchk_stats_merge( + struct xfs_mount *mp, + const struct xfs_scrub_metadata *sm, + const struct xchk_stats_run *run) +{ + xchk_stats_merge_one(&global_stats, sm, run); + xchk_stats_merge_one(mp->m_scrub_stats, sm, run); +} + +/* debugfs boilerplate */ + +static ssize_t +xchk_scrub_stats_read( + struct file *file, + char __user *ubuf, + size_t count, + loff_t *ppos) +{ + struct xchk_stats *cs = file->private_data; + char *buf; + size_t bufsize; + ssize_t avail, ret; + + /* + * This generates stringly snapshot of all the scrub counters, so we + * do not want userspace to receive garbled text from multiple calls. + * If the file position is greater than 0, return a short read. + */ + if (*ppos > 0) + return 0; + + bufsize = xchk_stats_estimate_bufsize(cs); + + buf = kvmalloc(bufsize, XCHK_GFP_FLAGS); + if (!buf) + return -ENOMEM; + + avail = xchk_stats_format(cs, buf, bufsize); + if (avail < 0) { + ret = avail; + goto out; + } + + ret = simple_read_from_buffer(ubuf, count, ppos, buf, avail); +out: + kvfree(buf); + return ret; +} + +static const struct file_operations scrub_stats_fops = { + .open = simple_open, + .read = xchk_scrub_stats_read, +}; + +static ssize_t +xchk_clear_scrub_stats_write( + struct file *file, + const char __user *ubuf, + size_t count, + loff_t *ppos) +{ + struct xchk_stats *cs = file->private_data; + unsigned int val; + int ret; + + ret = kstrtouint_from_user(ubuf, count, 0, &val); + if (ret) + return ret; + + if (val != 1) + return -EINVAL; + + xchk_stats_clearall(cs); + return count; +} + +static const struct file_operations clear_scrub_stats_fops = { + .open = simple_open, + .write = xchk_clear_scrub_stats_write, +}; + +/* Initialize the stats object. */ +STATIC int +xchk_stats_init( + struct xchk_stats *cs, + struct xfs_mount *mp) +{ + struct xchk_scrub_stats *css = &cs->cs_stats[0]; + unsigned int i; + + for (i = 0; i < XFS_SCRUB_TYPE_NR; i++, css++) + spin_lock_init(&css->css_lock); + + return 0; +} + +/* Connect the stats object to debugfs. */ +void +xchk_stats_register( + struct xchk_stats *cs, + struct dentry *parent) +{ + if (!parent) + return; + + cs->cs_debugfs = xfs_debugfs_mkdir("scrub", parent); + if (!cs->cs_debugfs) + return; + + debugfs_create_file("stats", 0444, cs->cs_debugfs, cs, + &scrub_stats_fops); + debugfs_create_file("clear_stats", 0200, cs->cs_debugfs, cs, + &clear_scrub_stats_fops); +} + +/* Free all resources related to the stats object. */ +STATIC int +xchk_stats_teardown( + struct xchk_stats *cs) +{ + return 0; +} + +/* Disconnect the stats object from debugfs. */ +void +xchk_stats_unregister( + struct xchk_stats *cs) +{ + debugfs_remove(cs->cs_debugfs); +} + +/* Initialize global stats and register them */ +int __init +xchk_global_stats_setup( + struct dentry *parent) +{ + int error; + + error = xchk_stats_init(&global_stats, NULL); + if (error) + return error; + + xchk_stats_register(&global_stats, parent); + return 0; +} + +/* Unregister global stats and tear them down */ +void +xchk_global_stats_teardown(void) +{ + xchk_stats_unregister(&global_stats); + xchk_stats_teardown(&global_stats); +} + +/* Allocate per-mount stats */ +int +xchk_mount_stats_alloc( + struct xfs_mount *mp) +{ + struct xchk_stats *cs; + int error; + + cs = kvzalloc_obj(struct xchk_stats); + if (!cs) + return -ENOMEM; + + error = xchk_stats_init(cs, mp); + if (error) + goto out_free; + + mp->m_scrub_stats = cs; + return 0; +out_free: + kvfree(cs); + return error; +} + +/* Free per-mount stats */ +void +xchk_mount_stats_free( + struct xfs_mount *mp) +{ + xchk_stats_teardown(mp->m_scrub_stats); + kvfree(mp->m_scrub_stats); + mp->m_scrub_stats = NULL; +} diff --git a/fs/xfs/scrub/stats.h b/fs/xfs/scrub/stats.h new file mode 100644 index 000000000..b358ad8d8 --- /dev/null +++ b/fs/xfs/scrub/stats.h @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_STATS_H__ +#define __XFS_SCRUB_STATS_H__ + +struct xchk_stats_run { + u64 scrub_ns; + u64 repair_ns; + unsigned int retries; + bool repair_attempted; + bool repair_succeeded; +}; + +#ifdef CONFIG_XFS_ONLINE_SCRUB_STATS +struct xchk_stats; + +int __init xchk_global_stats_setup(struct dentry *parent); +void xchk_global_stats_teardown(void); + +int xchk_mount_stats_alloc(struct xfs_mount *mp); +void xchk_mount_stats_free(struct xfs_mount *mp); + +void xchk_stats_register(struct xchk_stats *cs, struct dentry *parent); +void xchk_stats_unregister(struct xchk_stats *cs); + +void xchk_stats_merge(struct xfs_mount *mp, const struct xfs_scrub_metadata *sm, + const struct xchk_stats_run *run); + +static inline u64 xchk_stats_now(void) { return ktime_get_ns(); } +static inline u64 xchk_stats_elapsed_ns(u64 since) +{ + u64 now = xchk_stats_now(); + + /* + * If the system doesn't have a high enough resolution clock, charge at + * least one nanosecond so that our stats don't report instantaneous + * runtimes. + */ + if (now == since) + return 1; + + return now - since; +} +#else +# define xchk_global_stats_setup(parent) (0) +# define xchk_global_stats_teardown() ((void)0) +# define xchk_mount_stats_alloc(mp) (0) +# define xchk_mount_stats_free(mp) ((void)0) +# define xchk_stats_register(cs, parent) ((void)0) +# define xchk_stats_unregister(cs) ((void)0) +# define xchk_stats_now() (0) +# define xchk_stats_elapsed_ns(x) (0 * (x)) +# define xchk_stats_merge(mp, sm, run) ((void)0) +#endif /* CONFIG_XFS_ONLINE_SCRUB_STATS */ + +#endif /* __XFS_SCRUB_STATS_H__ */ diff --git a/fs/xfs/scrub/symlink.c b/fs/xfs/scrub/symlink.c new file mode 100644 index 000000000..91d40b9fb --- /dev/null +++ b/fs/xfs/scrub/symlink.c @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_inode.h" +#include "xfs_symlink.h" +#include "xfs_health.h" +#include "xfs_symlink_remote.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/health.h" +#include "scrub/repair.h" + +/* Set us up to scrub a symbolic link. */ +int +xchk_setup_symlink( + struct xfs_scrub *sc) +{ + unsigned int resblks = 0; + int error; + + /* Allocate the buffer without the inode lock held. */ + sc->buf = kvzalloc(XFS_SYMLINK_MAXLEN + 1, XCHK_GFP_FLAGS); + if (!sc->buf) + return -ENOMEM; + + if (xchk_could_repair(sc)) { + error = xrep_setup_symlink(sc, &resblks); + if (error) + return error; + } + + return xchk_setup_inode_contents(sc, resblks); +} + +/* Symbolic links. */ + +int +xchk_symlink( + struct xfs_scrub *sc) +{ + struct xfs_inode *ip = sc->ip; + struct xfs_ifork *ifp; + loff_t len; + int error = 0; + + if (!S_ISLNK(VFS_I(ip)->i_mode)) + return -ENOENT; + + if (xchk_file_looks_zapped(sc, XFS_SICK_INO_SYMLINK_ZAPPED)) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); + return 0; + } + + ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK); + len = ip->i_disk_size; + + /* Plausible size? */ + if (len > XFS_SYMLINK_MAXLEN || len <= 0) { + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); + return 0; + } + + /* Inline symlink? */ + if (ifp->if_format == XFS_DINODE_FMT_LOCAL) { + if (len > xfs_inode_data_fork_size(ip) || + len > strnlen(ifp->if_data, xfs_inode_data_fork_size(ip))) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); + return 0; + } + + /* Remote symlink; must read the contents. */ + error = xfs_symlink_remote_read(sc->ip, sc->buf); + if (!xchk_fblock_process_error(sc, XFS_DATA_FORK, 0, &error)) + return error; + if (strnlen(sc->buf, XFS_SYMLINK_MAXLEN) < len) + xchk_fblock_set_corrupt(sc, XFS_DATA_FORK, 0); + + /* If a remote symlink is clean, it is clearly not zapped. */ + xchk_mark_healthy_if_clean(sc, XFS_SICK_INO_SYMLINK_ZAPPED); + return 0; +} diff --git a/fs/xfs/scrub/symlink_repair.c b/fs/xfs/scrub/symlink_repair.c new file mode 100644 index 000000000..91c86ea0e --- /dev/null +++ b/fs/xfs/scrub/symlink_repair.c @@ -0,0 +1,510 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2018-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_btree.h" +#include "xfs_bit.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_sb.h" +#include "xfs_inode.h" +#include "xfs_inode_fork.h" +#include "xfs_symlink.h" +#include "xfs_bmap.h" +#include "xfs_quota.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_bmap_btree.h" +#include "xfs_trans_space.h" +#include "xfs_symlink_remote.h" +#include "xfs_exchmaps.h" +#include "xfs_exchrange.h" +#include "xfs_health.h" +#include "scrub/xfs_scrub.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/trace.h" +#include "scrub/repair.h" +#include "scrub/tempfile.h" +#include "scrub/tempexch.h" +#include "scrub/reap.h" +#include "scrub/health.h" + +/* + * Symbolic Link Repair + * ==================== + * + * We repair symbolic links by reading whatever target data we can find, up to + * the first NULL byte. If the recovered target strlen matches i_size, then + * we rewrite the target. In all other cases, we replace the target with an + * overly long string that cannot possibly resolve. The new target is written + * into a private hidden temporary file, and then a file contents exchange + * commits the new symlink target to the file being repaired. + */ + +/* Set us up to repair the symlink file. */ +int +xrep_setup_symlink( + struct xfs_scrub *sc, + unsigned int *resblks) +{ + struct xfs_mount *mp = sc->mp; + unsigned long long blocks; + int error; + + error = xrep_tempfile_create(sc, S_IFLNK); + if (error) + return error; + + /* + * If we're doing a repair, we reserve enough blocks to write out a + * completely new symlink file, plus twice as many blocks as we would + * need if we can only allocate one block per data fork mapping. This + * should cover the preallocation of the temporary file and exchanging + * the extent mappings. + * + * We cannot use xfs_exchmaps_estimate because we have not yet + * constructed the replacement symlink and therefore do not know how + * many extents it will use. By the time we do, we will have a dirty + * transaction (which we cannot drop because we cannot drop the + * symlink ILOCK) and cannot ask for more reservation. + */ + blocks = xfs_symlink_blocks(sc->mp, XFS_SYMLINK_MAXLEN); + blocks += xfs_bmbt_calc_size(mp, blocks) * 2; + if (blocks > UINT_MAX) + return -EOPNOTSUPP; + + *resblks += blocks; + return 0; +} + +/* + * Try to salvage the pathname from remote blocks. Returns the number of bytes + * salvaged or a negative errno. + */ +STATIC ssize_t +xrep_symlink_salvage_remote( + struct xfs_scrub *sc) +{ + struct xfs_bmbt_irec mval[XFS_SYMLINK_MAPS]; + struct xfs_inode *ip = sc->ip; + struct xfs_buf *bp; + char *target_buf = sc->buf; + xfs_failaddr_t fa; + xfs_filblks_t fsblocks; + xfs_daddr_t d; + loff_t len; + loff_t offset = 0; + unsigned int byte_cnt; + bool magic_ok; + bool hdr_ok; + int n; + int nmaps = XFS_SYMLINK_MAPS; + int error; + + /* We'll only read until the buffer is full. */ + len = min_t(loff_t, ip->i_disk_size, XFS_SYMLINK_MAXLEN); + fsblocks = xfs_symlink_blocks(sc->mp, len); + error = xfs_bmapi_read(ip, 0, fsblocks, mval, &nmaps, 0); + if (error) + return error; + + for (n = 0; n < nmaps; n++) { + struct xfs_dsymlink_hdr *dsl; + + d = XFS_FSB_TO_DADDR(sc->mp, mval[n].br_startblock); + + /* Read the rmt block. We'll run the verifiers manually. */ + error = xfs_trans_read_buf(sc->mp, sc->tp, sc->mp->m_ddev_targp, + d, XFS_FSB_TO_BB(sc->mp, mval[n].br_blockcount), + 0, &bp, NULL); + if (error) + return error; + bp->b_ops = &xfs_symlink_buf_ops; + + /* How many bytes do we expect to get out of this buffer? */ + byte_cnt = XFS_FSB_TO_B(sc->mp, mval[n].br_blockcount); + byte_cnt = XFS_SYMLINK_BUF_SPACE(sc->mp, byte_cnt); + byte_cnt = min_t(unsigned int, byte_cnt, len); + + /* + * See if the verifiers accept this block. We're willing to + * salvage if the if the offset/byte/ino are ok and either the + * verifier passed or the magic is ok. Anything else and we + * stop dead in our tracks. + */ + fa = bp->b_ops->verify_struct(bp); + dsl = bp->b_addr; + magic_ok = dsl->sl_magic == cpu_to_be32(XFS_SYMLINK_MAGIC); + hdr_ok = xfs_symlink_hdr_ok(I_INO(ip), offset, byte_cnt, bp); + if (!hdr_ok || (fa != NULL && !magic_ok)) + break; + + memcpy(target_buf + offset, dsl + 1, byte_cnt); + + len -= byte_cnt; + offset += byte_cnt; + } + return offset; +} + +/* + * Try to salvage an inline symlink's contents. Returns the number of bytes + * salvaged or a negative errno. + */ +STATIC ssize_t +xrep_symlink_salvage_inline( + struct xfs_scrub *sc) +{ + struct xfs_inode *ip = sc->ip; + char *target_buf = sc->buf; + char *old_target; + struct xfs_ifork *ifp; + unsigned int nr; + + ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK); + if (!ifp->if_data) + return 0; + + /* + * If inode repair zapped the link target, pretend that we didn't find + * any bytes at all so that we can replace the (now totally lost) link + * target with a warning message. + */ + old_target = ifp->if_data; + if (xfs_inode_has_sickness(sc->ip, XFS_SICK_INO_SYMLINK_ZAPPED) && + sc->ip->i_disk_size == 1 && old_target[0] == '?') + return 0; + + nr = min(XFS_SYMLINK_MAXLEN, ifp->if_bytes); + memcpy(target_buf, ifp->if_data, nr); + return nr; +} + +#define DUMMY_TARGET \ + "The target of this symbolic link could not be recovered at all and " \ + "has been replaced with this explanatory message. To avoid " \ + "accidentally pointing to an existing file path, this message is " \ + "longer than the maximum supported file name length. That is an " \ + "acceptable length for a symlink target on XFS but will produce " \ + "File Name Too Long errors if resolved." + +/* Salvage whatever we can of the target. */ +STATIC int +xrep_symlink_salvage( + struct xfs_scrub *sc) +{ + char *target_buf = sc->buf; + ssize_t buflen = 0; + + BUILD_BUG_ON(sizeof(DUMMY_TARGET) - 1 <= NAME_MAX); + + /* + * Salvage the target if there weren't any corruption problems observed + * while scanning it. + */ + if (!(sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)) { + if (sc->ip->i_df.if_format == XFS_DINODE_FMT_LOCAL) + buflen = xrep_symlink_salvage_inline(sc); + else + buflen = xrep_symlink_salvage_remote(sc); + if (buflen < 0) + return buflen; + + /* + * NULL-terminate the buffer because the ondisk target does not + * do that for us. If salvage didn't find the exact amount of + * data that we expected to find, don't salvage anything. + */ + target_buf[buflen] = 0; + if (strlen(target_buf) != sc->ip->i_disk_size) + buflen = 0; + } + + /* + * Change an empty target into a dummy target and clear the symlink + * target zapped flag. + */ + if (buflen == 0) { + xchk_mark_healthy_if_clean(sc, XFS_SICK_INO_SYMLINK_ZAPPED); + sprintf(target_buf, DUMMY_TARGET); + } + + trace_xrep_symlink_salvage_target(sc->ip, target_buf, + strlen(target_buf)); + return 0; +} + +STATIC void +xrep_symlink_local_to_remote( + struct xfs_trans *tp, + struct xfs_buf *bp, + struct xfs_inode *ip, + struct xfs_ifork *ifp, + void *priv) +{ + struct xfs_scrub *sc = priv; + struct xfs_dsymlink_hdr *dsl = bp->b_addr; + + xfs_symlink_local_to_remote(tp, bp, ip, ifp, NULL); + + if (!xfs_has_crc(sc->mp)) + return; + + dsl->sl_owner = cpu_to_be64(I_INO(sc->ip)); + xfs_trans_log_buf(tp, bp, 0, + sizeof(struct xfs_dsymlink_hdr) + ifp->if_bytes - 1); +} + +/* + * Prepare both links' data forks for an exchange. Promote the tempfile from + * local format to extents format, and if the file being repaired has a short + * format data fork, turn it into an empty extent list. + */ +STATIC int +xrep_symlink_swap_prep( + struct xfs_scrub *sc, + bool temp_local, + bool ip_local) +{ + int error; + + /* + * If the temp link is in shortform format, convert that to a remote + * target so that we can use the atomic mapping exchange. + */ + if (temp_local) { + int logflags = XFS_ILOG_CORE; + + error = xfs_bmap_local_to_extents(sc->tp, sc->tempip, 1, + &logflags, XFS_DATA_FORK, + xrep_symlink_local_to_remote, + sc); + if (error) + return error; + + xfs_trans_log_inode(sc->tp, sc->ip, 0); + + error = xfs_defer_finish(&sc->tp); + if (error) + return error; + } + + /* + * If the file being repaired had a shortform data fork, convert that + * to an empty extent list in preparation for the atomic mapping + * exchange. + */ + if (ip_local) { + struct xfs_ifork *ifp; + + ifp = xfs_ifork_ptr(sc->ip, XFS_DATA_FORK); + xfs_idestroy_fork(ifp); + ifp->if_format = XFS_DINODE_FMT_EXTENTS; + ifp->if_nextents = 0; + ifp->if_bytes = 0; + ifp->if_data = NULL; + ifp->if_height = 0; + + xfs_trans_log_inode(sc->tp, sc->ip, + XFS_ILOG_CORE | XFS_ILOG_DDATA); + } + + return 0; +} + +/* Exchange the temporary symlink's data fork with the one being repaired. */ +STATIC int +xrep_symlink_swap( + struct xfs_scrub *sc) +{ + struct xrep_tempexch *tx = sc->buf; + bool ip_local, temp_local; + int error; + + ip_local = sc->ip->i_df.if_format == XFS_DINODE_FMT_LOCAL; + temp_local = sc->tempip->i_df.if_format == XFS_DINODE_FMT_LOCAL; + + /* + * If the both links have a local format data fork and the rebuilt + * remote data would fit in the repaired file's data fork, copy the + * contents from the tempfile and declare ourselves done. + */ + if (ip_local && temp_local && + sc->tempip->i_disk_size <= xfs_inode_data_fork_size(sc->ip)) { + xrep_tempfile_copyout_local(sc, XFS_DATA_FORK); + return 0; + } + + /* Otherwise, make sure both data forks are in block-mapping mode. */ + error = xrep_symlink_swap_prep(sc, temp_local, ip_local); + if (error) + return error; + + return xrep_tempexch_contents(sc, tx); +} + +/* + * Free all the remote blocks and reset the data fork. The caller must join + * the inode to the transaction. This function returns with the inode joined + * to a clean scrub transaction. + */ +STATIC int +xrep_symlink_reset_fork( + struct xfs_scrub *sc) +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(sc->tempip, XFS_DATA_FORK); + int error; + + /* Unmap all the remote target buffers. */ + if (xfs_ifork_has_extents(ifp)) { + error = xrep_reap_ifork(sc, sc->tempip, XFS_DATA_FORK); + if (error) + return error; + } + + trace_xrep_symlink_reset_fork(sc->tempip); + + /* Reset the temp symlink target to dummy content. */ + xfs_idestroy_fork(ifp); + return xfs_symlink_write_target(sc->tp, sc->tempip, I_INO(sc->tempip), + "?", 1, 0, 0); +} + +/* + * Reinitialize a link target. Caller must ensure the inode is joined to + * the transaction. + */ +STATIC int +xrep_symlink_rebuild( + struct xfs_scrub *sc) +{ + struct xrep_tempexch *tx; + char *target_buf = sc->buf; + xfs_fsblock_t fs_blocks; + unsigned int target_len; + unsigned int resblks; + int error; + + /* How many blocks do we need? */ + target_len = strlen(target_buf); + ASSERT(target_len != 0); + if (target_len == 0 || target_len > XFS_SYMLINK_MAXLEN) + return -EFSCORRUPTED; + + trace_xrep_symlink_rebuild(sc->ip); + + /* + * In preparation to write the new symlink target to the temporary + * file, drop the ILOCK of the file being repaired (it shouldn't be + * joined) and take the ILOCK of the temporary file. + * + * The VFS does not take the IOLOCK while reading a symlink (and new + * symlinks are hidden with INEW until they've been written) so it's + * possible that a readlink() could see the old corrupted contents + * while we're doing this. + */ + xchk_iunlock(sc, XFS_ILOCK_EXCL); + xrep_tempfile_ilock(sc); + xfs_trans_ijoin(sc->tp, sc->tempip, 0); + + /* + * Reserve resources to reinitialize the target. We're allowed to + * exceed file quota to repair inconsistent metadata, though this is + * unlikely. + */ + fs_blocks = xfs_symlink_blocks(sc->mp, target_len); + resblks = xfs_symlink_space_res(sc->mp, target_len, fs_blocks); + error = xfs_trans_reserve_quota_nblks(sc->tp, sc->tempip, resblks, 0, + true); + if (error) + return error; + + /* Erase the dummy target set up by the tempfile initialization. */ + xfs_idestroy_fork(&sc->tempip->i_df); + sc->tempip->i_df.if_bytes = 0; + sc->tempip->i_df.if_format = XFS_DINODE_FMT_EXTENTS; + + /* Write the salvaged target to the temporary link. */ + error = xfs_symlink_write_target(sc->tp, sc->tempip, I_INO(sc->ip), + target_buf, target_len, fs_blocks, resblks); + if (error) + return error; + + /* + * Commit the repair transaction so that we can use the atomic mapping + * exchange functions to compute the correct block reservations and + * re-lock the inodes. + */ + target_buf = NULL; + error = xrep_trans_commit(sc); + if (error) + return error; + + /* Last chance to abort before we start committing fixes. */ + if (xchk_should_terminate(sc, &error)) + return error; + + xrep_tempfile_iunlock(sc); + + /* + * We're done with the temporary buffer, so we can reuse it for the + * tempfile contents exchange information. + */ + tx = sc->buf; + error = xrep_tempexch_trans_alloc(sc, XFS_DATA_FORK, tx); + if (error) + return error; + + /* + * Exchange the temp link's data fork with the file being repaired. + * This recreates the transaction and takes the ILOCKs of the file + * being repaired and the temporary file. + */ + error = xrep_symlink_swap(sc); + if (error) + return error; + + /* + * Release the old symlink blocks and reset the data fork of the temp + * link to an empty shortform link. This is the last repair action we + * perform on the symlink, so we don't need to clean the transaction. + */ + return xrep_symlink_reset_fork(sc); +} + +/* Repair a symbolic link. */ +int +xrep_symlink( + struct xfs_scrub *sc) +{ + int error; + + /* The rmapbt is required to reap the old data fork. */ + if (!xfs_has_rmapbt(sc->mp)) + return -EOPNOTSUPP; + /* We require atomic file exchange range to rebuild anything. */ + if (!xfs_has_exchange_range(sc->mp)) + return -EOPNOTSUPP; + + ASSERT(sc->ilock_flags & XFS_ILOCK_EXCL); + + error = xrep_symlink_salvage(sc); + if (error) + return error; + + /* Now reset the target. */ + error = xrep_symlink_rebuild(sc); + if (error) + return error; + + return xrep_trans_commit(sc); +} diff --git a/fs/xfs/scrub/tempexch.h b/fs/xfs/scrub/tempexch.h new file mode 100644 index 000000000..eccda720c --- /dev/null +++ b/fs/xfs/scrub/tempexch.h @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2022-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_TEMPEXCH_H__ +#define __XFS_SCRUB_TEMPEXCH_H__ + +#ifdef CONFIG_XFS_ONLINE_REPAIR +struct xrep_tempexch { + struct xfs_exchmaps_req req; +}; + +int xrep_tempexch_trans_reserve(struct xfs_scrub *sc, int whichfork, + xfs_fileoff_t off, xfs_filblks_t len, struct xrep_tempexch *ti); +int xrep_tempexch_trans_alloc(struct xfs_scrub *sc, int whichfork, + struct xrep_tempexch *ti); + +int xrep_tempexch_contents(struct xfs_scrub *sc, struct xrep_tempexch *ti); +#endif /* CONFIG_XFS_ONLINE_REPAIR */ + +#endif /* __XFS_SCRUB_TEMPEXCH_H__ */ diff --git a/fs/xfs/scrub/tempfile.c b/fs/xfs/scrub/tempfile.c new file mode 100644 index 000000000..98820003b --- /dev/null +++ b/fs/xfs/scrub/tempfile.c @@ -0,0 +1,981 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2021-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_inode.h" +#include "xfs_ialloc.h" +#include "xfs_quota.h" +#include "xfs_bmap.h" +#include "xfs_bmap_btree.h" +#include "xfs_trans_space.h" +#include "xfs_dir2.h" +#include "xfs_exchrange.h" +#include "xfs_exchmaps.h" +#include "xfs_defer.h" +#include "xfs_symlink_remote.h" +#include "xfs_metafile.h" +#include "scrub/scrub.h" +#include "scrub/common.h" +#include "scrub/repair.h" +#include "scrub/trace.h" +#include "scrub/tempfile.h" +#include "scrub/tempexch.h" +#include "scrub/xfile.h" + +/* + * Create a temporary file for reconstructing metadata, with the intention of + * atomically exchanging the temporary file's contents with the file that's + * being repaired. + */ +int +xrep_tempfile_create( + struct xfs_scrub *sc, + uint16_t mode) +{ + struct xfs_icreate_args args = { + .pip = sc->mp->m_rootip, + .mode = mode, + .flags = XFS_ICREATE_TMPFILE | XFS_ICREATE_UNLINKABLE, + }; + struct xfs_mount *mp = sc->mp; + struct xfs_trans *tp = NULL; + struct xfs_dquot *udqp; + struct xfs_dquot *gdqp; + struct xfs_dquot *pdqp; + struct xfs_trans_res *tres; + struct xfs_inode *dp = mp->m_rootip; + xfs_ino_t ino; + unsigned int resblks; + bool is_dir = S_ISDIR(mode); + int error; + + if (xfs_is_shutdown(mp)) + return -EIO; + if (xfs_is_readonly(mp)) + return -EROFS; + + ASSERT(sc->tp == NULL); + ASSERT(sc->tempip == NULL); + + /* + * Make sure that we have allocated dquot(s) on disk. The temporary + * inode should be completely root owned so that we don't fail due to + * quota limits. + */ + error = xfs_icreate_dqalloc(&args, &udqp, &gdqp, &pdqp); + if (error) + return error; + + if (is_dir) { + resblks = xfs_mkdir_space_res(mp, 0); + tres = &M_RES(mp)->tr_mkdir; + } else { + resblks = XFS_IALLOC_SPACE_RES(mp); + tres = &M_RES(mp)->tr_create_tmpfile; + } + + error = xfs_trans_alloc_icreate(mp, tres, udqp, gdqp, pdqp, resblks, + &tp); + if (error) + goto out_release_dquots; + + /* Allocate inode, set up directory. */ + error = xfs_dialloc(&tp, &args, &ino); + if (error) + goto out_trans_cancel; + error = xfs_icreate(tp, ino, &args, &sc->tempip); + if (error) + goto out_trans_cancel; + + /* We don't touch file data, so drop the realtime flags. */ + sc->tempip->i_diflags &= ~(XFS_DIFLAG_REALTIME | XFS_DIFLAG_RTINHERIT); + xfs_trans_log_inode(tp, sc->tempip, XFS_ILOG_CORE); + + /* + * Mark our temporary file as private so that LSMs and the ACL code + * don't try to add their own metadata or reason about these files. + * The file should never be exposed to userspace. + */ + VFS_I(sc->tempip)->i_flags |= S_PRIVATE; + VFS_I(sc->tempip)->i_opflags &= ~IOP_XATTR; + + if (is_dir) { + error = xfs_dir_init(tp, sc->tempip, dp); + if (error) + goto out_trans_cancel; + } else if (S_ISLNK(VFS_I(sc->tempip)->i_mode)) { + /* + * Initialize the temporary symlink with a meaningless target + * that won't trip the verifiers. Repair must rewrite the + * target with meaningful content before swapping with the file + * being repaired. A single-byte target will not write a + * remote target block, so the owner is irrelevant. + */ + error = xfs_symlink_write_target(tp, sc->tempip, + I_INO(sc->tempip), ".", 1, 0, 0); + if (error) + goto out_trans_cancel; + } + + /* + * Attach the dquot(s) to the inodes and modify them incore. + * These ids of the inode couldn't have changed since the new + * inode has been locked ever since it was created. + */ + xfs_qm_vop_create_dqattach(tp, sc->tempip, udqp, gdqp, pdqp); + + /* + * Put our temp file on the unlinked list so it's purged automatically. + * All file-based metadata being reconstructed using this file must be + * atomically exchanged with the original file because the contents + * here will be purged when the inode is dropped or log recovery cleans + * out the unlinked list. + */ + error = xfs_iunlink(tp, sc->tempip); + if (error) + goto out_trans_cancel; + + error = xfs_trans_commit(tp); + if (error) + goto out_release_inode; + + trace_xrep_tempfile_create(sc); + + xfs_qm_dqrele(udqp); + xfs_qm_dqrele(gdqp); + xfs_qm_dqrele(pdqp); + + /* Finish setting up the incore / vfs context. */ + xfs_iunlock(sc->tempip, XFS_ILOCK_EXCL); + xfs_setup_iops(sc->tempip); + xfs_finish_inode_setup(sc->tempip); + + sc->temp_ilock_flags = 0; + return error; + +out_trans_cancel: + xfs_trans_cancel(tp); +out_release_inode: + /* + * Wait until after the current transaction is aborted to finish the + * setup of the inode and release the inode. This prevents recursive + * transactions and deadlocks from xfs_inactive. + */ + if (sc->tempip) { + xfs_iunlock(sc->tempip, XFS_ILOCK_EXCL); + xfs_finish_inode_setup(sc->tempip); + xchk_irele(sc, sc->tempip); + sc->tempip = NULL; + } +out_release_dquots: + xfs_qm_dqrele(udqp); + xfs_qm_dqrele(gdqp); + xfs_qm_dqrele(pdqp); + + return error; +} + +/* + * Move sc->tempip from the regular directory tree to the metadata directory + * tree if sc->ip is part of the metadata directory tree and tempip has an + * eligible file mode. + * + * Temporary files have to be created before we even know which inode we're + * going to scrub, so we assume that they will be part of the regular directory + * tree. If it turns out that we're actually scrubbing a file from the + * metadata directory tree, we have to subtract the temp file from the root + * dquots and detach the dquots prior to setting the METADATA iflag. However, + * the scrub setup functions grab sc->ip and create sc->tempip before we + * actually get around to checking if the file mode is the right type for the + * scrubber. + */ +int +xrep_tempfile_adjust_directory_tree( + struct xfs_scrub *sc) +{ + int error; + + if (!sc->tempip) + return 0; + + ASSERT(sc->tp == NULL); + ASSERT(!xfs_is_metadir_inode(sc->tempip)); + + if (!sc->ip || !xfs_is_metadir_inode(sc->ip)) + return 0; + if (!S_ISDIR(VFS_I(sc->tempip)->i_mode) && + !S_ISREG(VFS_I(sc->tempip)->i_mode)) + return 0; + + xfs_ilock(sc->tempip, XFS_IOLOCK_EXCL); + sc->temp_ilock_flags |= XFS_IOLOCK_EXCL; + + error = xchk_trans_alloc(sc, 0); + if (error) + goto out_iolock; + + xrep_tempfile_ilock(sc); + xfs_trans_ijoin(sc->tp, sc->tempip, 0); + + /* Metadir files are not accounted in quota, so drop icount */ + xfs_trans_mod_dquot_byino(sc->tp, sc->tempip, XFS_TRANS_DQ_ICOUNT, -1L); + xfs_metafile_set_iflag(sc->tp, sc->tempip, XFS_METAFILE_UNKNOWN); + + error = xrep_trans_commit(sc); + if (error) + goto out_ilock; + + xfs_iflags_set(sc->tempip, XFS_IRECOVERY); + xfs_qm_dqdetach(sc->tempip); +out_ilock: + xrep_tempfile_iunlock(sc); +out_iolock: + xrep_tempfile_iounlock(sc); + return error; +} + +/* + * Remove this temporary file from the metadata directory tree so that it can + * be inactivated the normal way. + */ +STATIC int +xrep_tempfile_remove_metadir( + struct xfs_scrub *sc) +{ + int error; + + if (!sc->tempip || !xfs_is_metadir_inode(sc->tempip)) + return 0; + + ASSERT(sc->tp == NULL); + + xfs_iflags_clear(sc->tempip, XFS_IRECOVERY); + + xfs_ilock(sc->tempip, XFS_IOLOCK_EXCL); + sc->temp_ilock_flags |= XFS_IOLOCK_EXCL; + + error = xchk_trans_alloc(sc, 0); + if (error) + goto out_iolock; + + xrep_tempfile_ilock(sc); + xfs_trans_ijoin(sc->tp, sc->tempip, 0); + + xfs_metafile_clear_iflag(sc->tp, sc->tempip); + + /* Non-metadir files are accounted in quota, so bump bcount/icount */ + error = xfs_qm_dqattach_locked(sc->tempip, false); + if (error) + goto out_cancel; + + xfs_trans_mod_dquot_byino(sc->tp, sc->tempip, XFS_TRANS_DQ_ICOUNT, 1L); + xfs_trans_mod_dquot_byino(sc->tp, sc->tempip, XFS_TRANS_DQ_BCOUNT, + sc->tempip->i_nblocks); + error = xrep_trans_commit(sc); + goto out_ilock; + +out_cancel: + xchk_trans_cancel(sc); +out_ilock: + xrep_tempfile_iunlock(sc); +out_iolock: + xrep_tempfile_iounlock(sc); + return error; +} + +/* Take IOLOCK_EXCL on the temporary file, maybe. */ +bool +xrep_tempfile_iolock_nowait( + struct xfs_scrub *sc) +{ + if (xfs_ilock_nowait(sc->tempip, XFS_IOLOCK_EXCL)) { + sc->temp_ilock_flags |= XFS_IOLOCK_EXCL; + return true; + } + + return false; +} + +/* + * Take the temporary file's IOLOCK while holding a different inode's IOLOCK. + * In theory nobody else should hold the tempfile's IOLOCK, but we use trylock + * to avoid deadlocks and lockdep complaints. + */ +int +xrep_tempfile_iolock_polled( + struct xfs_scrub *sc) +{ + int error = 0; + + while (!xrep_tempfile_iolock_nowait(sc)) { + if (xchk_should_terminate(sc, &error)) + return error; + delay(1); + } + + return 0; +} + +/* Release IOLOCK_EXCL on the temporary file. */ +void +xrep_tempfile_iounlock( + struct xfs_scrub *sc) +{ + xfs_iunlock(sc->tempip, XFS_IOLOCK_EXCL); + sc->temp_ilock_flags &= ~XFS_IOLOCK_EXCL; +} + +/* Prepare the temporary file for metadata updates by grabbing ILOCK_EXCL. */ +void +xrep_tempfile_ilock( + struct xfs_scrub *sc) +{ + sc->temp_ilock_flags |= XFS_ILOCK_EXCL; + xfs_ilock(sc->tempip, XFS_ILOCK_EXCL); +} + +/* Try to grab ILOCK_EXCL on the temporary file. */ +bool +xrep_tempfile_ilock_nowait( + struct xfs_scrub *sc) +{ + if (xfs_ilock_nowait(sc->tempip, XFS_ILOCK_EXCL)) { + sc->temp_ilock_flags |= XFS_ILOCK_EXCL; + return true; + } + + return false; +} + +/* Unlock ILOCK_EXCL on the temporary file after an update. */ +void +xrep_tempfile_iunlock( + struct xfs_scrub *sc) +{ + xfs_iunlock(sc->tempip, XFS_ILOCK_EXCL); + sc->temp_ilock_flags &= ~XFS_ILOCK_EXCL; +} + +/* + * Begin the process of making changes to both the file being scrubbed and + * the temporary file by taking ILOCK_EXCL on both. + */ +void +xrep_tempfile_ilock_both( + struct xfs_scrub *sc) +{ + xfs_lock_two_inodes(sc->ip, XFS_ILOCK_EXCL, sc->tempip, XFS_ILOCK_EXCL); + sc->ilock_flags |= XFS_ILOCK_EXCL; + sc->temp_ilock_flags |= XFS_ILOCK_EXCL; +} + +/* Unlock ILOCK_EXCL on both files. */ +void +xrep_tempfile_iunlock_both( + struct xfs_scrub *sc) +{ + xrep_tempfile_iunlock(sc); + xchk_iunlock(sc, XFS_ILOCK_EXCL); +} + +/* Release the temporary file. */ +void +xrep_tempfile_rele( + struct xfs_scrub *sc) +{ + if (!sc->tempip) + return; + + if (sc->temp_ilock_flags) { + xfs_iunlock(sc->tempip, sc->temp_ilock_flags); + sc->temp_ilock_flags = 0; + } + + xrep_tempfile_remove_metadir(sc); + xchk_irele(sc, sc->tempip); + sc->tempip = NULL; +} + +/* + * Make sure that the given range of the data fork of the temporary file is + * mapped to written blocks. The caller must ensure that both inodes are + * joined to the transaction. + */ +int +xrep_tempfile_prealloc( + struct xfs_scrub *sc, + xfs_fileoff_t off, + xfs_filblks_t len) +{ + struct xfs_bmbt_irec map; + xfs_fileoff_t end = off + len; + int error; + + ASSERT(sc->tempip != NULL); + ASSERT(!XFS_NOT_DQATTACHED(sc->mp, sc->tempip)); + + for (; off < end; off = map.br_startoff + map.br_blockcount) { + int nmaps = 1; + + /* + * If we have a real extent mapping this block then we're + * in ok shape. + */ + error = xfs_bmapi_read(sc->tempip, off, end - off, &map, &nmaps, + XFS_DATA_FORK); + if (error) + return error; + if (nmaps == 0) { + ASSERT(nmaps != 0); + return -EFSCORRUPTED; + } + + if (xfs_bmap_is_written_extent(&map)) + continue; + + /* + * If we find a delalloc reservation then something is very + * very wrong. Bail out. + */ + if (map.br_startblock == DELAYSTARTBLOCK) + return -EFSCORRUPTED; + + /* + * Make sure this block has a real zeroed extent allocated to + * it. + */ + nmaps = 1; + error = xfs_bmapi_write(sc->tp, sc->tempip, off, end - off, + XFS_BMAPI_CONVERT | XFS_BMAPI_ZERO, 0, &map, + &nmaps); + if (error) + return error; + if (nmaps != 1) + return -EFSCORRUPTED; + + trace_xrep_tempfile_prealloc(sc, XFS_DATA_FORK, &map); + + /* Commit new extent and all deferred work. */ + error = xfs_defer_finish(&sc->tp); + if (error) + return error; + } + + return 0; +} + +/* + * Write data to each block of a file. The given range of the tempfile's data + * fork must already be populated with written extents. + */ +int +xrep_tempfile_copyin( + struct xfs_scrub *sc, + xfs_fileoff_t off, + xfs_filblks_t len, + xrep_tempfile_copyin_fn prep_fn, + void *data) +{ + LIST_HEAD(buffers_list); + struct xfs_mount *mp = sc->mp; + struct xfs_buf *bp; + xfs_fileoff_t flush_mask; + xfs_fileoff_t end = off + len; + loff_t pos = XFS_FSB_TO_B(mp, off); + int error = 0; + + ASSERT(S_ISREG(VFS_I(sc->tempip)->i_mode)); + + /* Flush buffers to disk every 512K */ + flush_mask = XFS_B_TO_FSBT(mp, (1U << 19)) - 1; + + for (; off < end; off++, pos += mp->m_sb.sb_blocksize) { + struct xfs_bmbt_irec map; + int nmaps = 1; + + /* Read block mapping for this file block. */ + error = xfs_bmapi_read(sc->tempip, off, 1, &map, &nmaps, 0); + if (error) + goto out_err; + if (nmaps == 0 || !xfs_bmap_is_written_extent(&map)) { + error = -EFSCORRUPTED; + goto out_err; + } + + /* Get the metadata buffer for this offset in the file. */ + error = xfs_trans_get_buf(sc->tp, mp->m_ddev_targp, + XFS_FSB_TO_DADDR(mp, map.br_startblock), + mp->m_bsize, 0, &bp); + if (error) + goto out_err; + + trace_xrep_tempfile_copyin(sc, XFS_DATA_FORK, &map); + + /* Read in a block's worth of data from the xfile. */ + error = prep_fn(sc, bp, data); + if (error) { + xfs_trans_brelse(sc->tp, bp); + goto out_err; + } + + /* Queue buffer, and flush if we have too much dirty data. */ + xfs_buf_delwri_queue_here(bp, &buffers_list); + xfs_trans_brelse(sc->tp, bp); + + if (!(off & flush_mask)) { + error = xfs_buf_delwri_submit(&buffers_list); + if (error) + goto out_err; + } + } + + /* + * Write the new blocks to disk. If the ordered list isn't empty after + * that, then something went wrong and we have to fail. This should + * never happen, but we'll check anyway. + */ + error = xfs_buf_delwri_submit(&buffers_list); + if (error) + goto out_err; + + if (!list_empty(&buffers_list)) { + ASSERT(list_empty(&buffers_list)); + error = -EIO; + goto out_err; + } + + return 0; + +out_err: + xfs_buf_delwri_cancel(&buffers_list); + return error; +} + +/* + * Set the temporary file's size. Caller must join the tempfile to the scrub + * transaction and is responsible for adjusting block mappings as needed. + */ +int +xrep_tempfile_set_isize( + struct xfs_scrub *sc, + unsigned long long isize) +{ + if (sc->tempip->i_disk_size == isize) + return 0; + + sc->tempip->i_disk_size = isize; + i_size_write(VFS_I(sc->tempip), isize); + return xrep_tempfile_roll_trans(sc); +} + +/* + * Roll a repair transaction involving the temporary file. Caller must join + * both the temporary file and the file being scrubbed to the transaction. + * This function return with both inodes joined to a new scrub transaction, + * or the usual negative errno. + */ +int +xrep_tempfile_roll_trans( + struct xfs_scrub *sc) +{ + int error; + + xfs_trans_log_inode(sc->tp, sc->tempip, XFS_ILOG_CORE); + error = xrep_roll_trans(sc); + if (error) + return error; + + xfs_trans_ijoin(sc->tp, sc->tempip, 0); + return 0; +} + +/* + * Fill out the mapping exchange request in preparation for atomically + * committing the contents of a metadata file that we've rebuilt in the temp + * file. + */ +STATIC int +xrep_tempexch_prep_request( + struct xfs_scrub *sc, + int whichfork, + xfs_fileoff_t off, + xfs_filblks_t len, + struct xrep_tempexch *tx) +{ + struct xfs_exchmaps_req *req = &tx->req; + + memset(tx, 0, sizeof(struct xrep_tempexch)); + + /* COW forks don't exist on disk. */ + if (whichfork == XFS_COW_FORK) { + ASSERT(0); + return -EINVAL; + } + + /* Both files should have the relevant forks. */ + if (!xfs_ifork_ptr(sc->ip, whichfork) || + !xfs_ifork_ptr(sc->tempip, whichfork)) { + ASSERT(xfs_ifork_ptr(sc->ip, whichfork) != NULL); + ASSERT(xfs_ifork_ptr(sc->tempip, whichfork) != NULL); + return -EINVAL; + } + + /* Exchange all mappings in both forks. */ + req->ip1 = sc->tempip; + req->ip2 = sc->ip; + req->startoff1 = off; + req->startoff2 = off; + switch (whichfork) { + case XFS_ATTR_FORK: + req->flags |= XFS_EXCHMAPS_ATTR_FORK; + break; + case XFS_DATA_FORK: + /* Exchange sizes when exchanging all data fork mappings. */ + if (off == 0 && len == XFS_MAX_FILEOFF) + req->flags |= XFS_EXCHMAPS_SET_SIZES; + break; + } + req->blockcount = len; + + return 0; +} + +/* + * Fill out the mapping exchange resource estimation structures in preparation + * for exchanging the contents of a metadata file that we've rebuilt in the + * temp file. Caller must hold IOLOCK_EXCL but not ILOCK_EXCL on both files. + */ +STATIC int +xrep_tempexch_estimate( + struct xfs_scrub *sc, + struct xrep_tempexch *tx) +{ + struct xfs_exchmaps_req *req = &tx->req; + struct xfs_ifork *ifp; + struct xfs_ifork *tifp; + int whichfork = xfs_exchmaps_reqfork(req); + int state = 0; + + /* + * The exchmaps code only knows how to exchange file fork space + * mappings. Any fork data in local format must be promoted to a + * single block before the exchange can take place. + */ + ifp = xfs_ifork_ptr(sc->ip, whichfork); + if (ifp->if_format == XFS_DINODE_FMT_LOCAL) + state |= 1; + + tifp = xfs_ifork_ptr(sc->tempip, whichfork); + if (tifp->if_format == XFS_DINODE_FMT_LOCAL) + state |= 2; + + switch (state) { + case 0: + /* Both files have mapped extents; use the regular estimate. */ + return xfs_exchrange_estimate(req); + case 1: + /* + * The file being repaired is in local format, but the temp + * file has mapped extents. To perform the exchange, the file + * being repaired must have its shorform data converted to an + * ondisk block so that the forks will be in extents format. + * We need one resblk for the conversion; the number of + * exchanges is (worst case) the temporary file's extent count + * plus the block we converted. + */ + req->ip1_bcount = sc->tempip->i_nblocks; + req->ip2_bcount = 1; + req->nr_exchanges = 1 + tifp->if_nextents; + req->resblks = 1; + break; + case 2: + /* + * The temporary file is in local format, but the file being + * repaired has mapped extents. To perform the exchange, the + * temp file must have its shortform data converted to an + * ondisk block, and the fork changed to extents format. We + * need one resblk for the conversion; the number of exchanges + * is (worst case) the extent count of the file being repaired + * plus the block we converted. + */ + req->ip1_bcount = 1; + req->ip2_bcount = sc->ip->i_nblocks; + req->nr_exchanges = 1 + ifp->if_nextents; + req->resblks = 1; + break; + case 3: + /* + * Both forks are in local format. To perform the exchange, + * both files must have their shortform data converted to + * fsblocks, and both forks must be converted to extents + * format. We need two resblks for the two conversions, and + * the number of exchanges is 1 since there's only one block at + * fileoff 0. Presumably, the caller could not exchange the + * two inode fork areas directly. + */ + req->ip1_bcount = 1; + req->ip2_bcount = 1; + req->nr_exchanges = 1; + req->resblks = 2; + break; + } + + return xfs_exchmaps_estimate_overhead(req); +} + +/* + * Obtain a quota reservation to make sure we don't hit EDQUOT. We can skip + * this if quota enforcement is disabled or if both inodes' dquots are the + * same. The qretry structure must be initialized to zeroes before the first + * call to this function. + */ +STATIC int +xrep_tempexch_reserve_quota( + struct xfs_scrub *sc, + const struct xrep_tempexch *tx) +{ + struct xfs_trans *tp = sc->tp; + const struct xfs_exchmaps_req *req = &tx->req; + int64_t ddelta, rdelta; + int error; + + /* + * Don't bother with a quota reservation if we're not enforcing them + * or the two inodes have the same dquots. + */ + if (!XFS_IS_QUOTA_ON(tp->t_mountp) || req->ip1 == req->ip2 || + xfs_is_metadir_inode(req->ip1) || + (req->ip1->i_udquot == req->ip2->i_udquot && + req->ip1->i_gdquot == req->ip2->i_gdquot && + req->ip1->i_pdquot == req->ip2->i_pdquot)) + return 0; + + /* + * Quota reservation for each file comes from two sources. First, we + * need to account for any net gain in mapped blocks during the + * exchange. Second, we need reservation for the gross gain in mapped + * blocks so that we don't trip over any quota block reservation + * assertions. We must reserve the gross gain because the quota code + * subtracts from bcount the number of blocks that we unmap; it does + * not add that quantity back to the quota block reservation. + */ + ddelta = max_t(int64_t, 0, req->ip2_bcount - req->ip1_bcount); + rdelta = max_t(int64_t, 0, req->ip2_rtbcount - req->ip1_rtbcount); + error = xfs_trans_reserve_quota_nblks(tp, req->ip1, + ddelta + req->ip1_bcount, rdelta + req->ip1_rtbcount, + true); + if (error) + return error; + + ddelta = max_t(int64_t, 0, req->ip1_bcount - req->ip2_bcount); + rdelta = max_t(int64_t, 0, req->ip1_rtbcount - req->ip2_rtbcount); + return xfs_trans_reserve_quota_nblks(tp, req->ip2, + ddelta + req->ip2_bcount, rdelta + req->ip2_rtbcount, + true); +} + +/* + * Prepare an existing transaction for an atomic file contents exchange. + * + * This function fills out the mapping exchange request and resource estimation + * structures in preparation for exchanging the contents of a metadata file + * that has been rebuilt in the temp file. Next, it reserves space and quota + * for the transaction. + * + * The caller must hold ILOCK_EXCL of the scrub target file and the temporary + * file. The caller must join both inodes to the transaction with no unlock + * flags, and is responsible for dropping both ILOCKs when appropriate. Only + * use this when those ILOCKs cannot be dropped. + */ +int +xrep_tempexch_trans_reserve( + struct xfs_scrub *sc, + int whichfork, + xfs_fileoff_t off, + xfs_filblks_t len, + struct xrep_tempexch *tx) +{ + int error; + + ASSERT(sc->tp != NULL); + xfs_assert_ilocked(sc->ip, XFS_ILOCK_EXCL); + xfs_assert_ilocked(sc->tempip, XFS_ILOCK_EXCL); + + error = xrep_tempexch_prep_request(sc, whichfork, off, len, tx); + if (error) + return error; + + error = xfs_exchmaps_estimate(&tx->req); + if (error) + return error; + + error = xfs_trans_reserve_more(sc->tp, tx->req.resblks, 0); + if (error) + return error; + + return xrep_tempexch_reserve_quota(sc, tx); +} + +/* + * Create a new transaction for a file contents exchange. + * + * This function fills out the mapping excahange request and resource + * estimation structures in preparation for exchanging the contents of a + * metadata file that has been rebuilt in the temp file. Next, it reserves + * space, takes ILOCK_EXCL of both inodes, joins them to the transaction and + * reserves quota for the transaction. + * + * The caller is responsible for dropping both ILOCKs when appropriate. + */ +int +xrep_tempexch_trans_alloc( + struct xfs_scrub *sc, + int whichfork, + struct xrep_tempexch *tx) +{ + unsigned int flags = 0; + int error; + + ASSERT(sc->tp == NULL); + ASSERT(xfs_has_exchange_range(sc->mp)); + + error = xrep_tempexch_prep_request(sc, whichfork, 0, XFS_MAX_FILEOFF, + tx); + if (error) + return error; + + error = xrep_tempexch_estimate(sc, tx); + if (error) + return error; + + if (xfs_has_lazysbcount(sc->mp)) + flags |= XFS_TRANS_RES_FDBLKS; + + error = xfs_trans_alloc(sc->mp, &M_RES(sc->mp)->tr_itruncate, + tx->req.resblks, 0, flags, &sc->tp); + if (error) + return error; + + sc->temp_ilock_flags |= XFS_ILOCK_EXCL; + sc->ilock_flags |= XFS_ILOCK_EXCL; + xfs_exchrange_ilock(sc->tp, sc->ip, sc->tempip); + + return xrep_tempexch_reserve_quota(sc, tx); +} + +/* + * Exchange file mappings (and hence file contents) between the file being + * repaired and the temporary file. Returns with both inodes locked and joined + * to a clean scrub transaction. + */ +int +xrep_tempexch_contents( + struct xfs_scrub *sc, + struct xrep_tempexch *tx) +{ + int error; + + ASSERT(xfs_has_exchange_range(sc->mp)); + + xfs_exchange_mappings(sc->tp, &tx->req); + error = xfs_defer_finish(&sc->tp); + if (error) + return error; + + /* + * If we exchanged the ondisk sizes of two metadata files, we must + * exchanged the incore sizes as well. + */ + if (tx->req.flags & XFS_EXCHMAPS_SET_SIZES) { + loff_t temp; + + temp = i_size_read(VFS_I(sc->ip)); + i_size_write(VFS_I(sc->ip), i_size_read(VFS_I(sc->tempip))); + i_size_write(VFS_I(sc->tempip), temp); + } + + return 0; +} + +/* + * Write local format data from one of the temporary file's forks into the same + * fork of file being repaired, and exchange the file sizes, if appropriate. + * Caller must ensure that the file being repaired has enough fork space to + * hold all the bytes. + */ +void +xrep_tempfile_copyout_local( + struct xfs_scrub *sc, + int whichfork) +{ + struct xfs_ifork *temp_ifp; + struct xfs_ifork *ifp; + unsigned int ilog_flags = XFS_ILOG_CORE; + + temp_ifp = xfs_ifork_ptr(sc->tempip, whichfork); + ifp = xfs_ifork_ptr(sc->ip, whichfork); + + ASSERT(temp_ifp != NULL); + ASSERT(ifp != NULL); + ASSERT(temp_ifp->if_format == XFS_DINODE_FMT_LOCAL); + ASSERT(ifp->if_format == XFS_DINODE_FMT_LOCAL); + + switch (whichfork) { + case XFS_DATA_FORK: + ASSERT(sc->tempip->i_disk_size <= + xfs_inode_data_fork_size(sc->ip)); + break; + case XFS_ATTR_FORK: + ASSERT(sc->tempip->i_forkoff >= sc->ip->i_forkoff); + break; + default: + ASSERT(0); + return; + } + + /* Recreate @sc->ip's incore fork (ifp) with data from temp_ifp. */ + xfs_idestroy_fork(ifp); + xfs_init_local_fork(sc->ip, whichfork, temp_ifp->if_data, + temp_ifp->if_bytes); + + if (whichfork == XFS_DATA_FORK) { + i_size_write(VFS_I(sc->ip), i_size_read(VFS_I(sc->tempip))); + sc->ip->i_disk_size = sc->tempip->i_disk_size; + } + + ilog_flags |= xfs_ilog_fdata(whichfork); + xfs_trans_log_inode(sc->tp, sc->ip, ilog_flags); +} + +/* Decide if a given XFS inode is a temporary file for a repair. */ +bool +xrep_is_tempfile( + const struct xfs_inode *ip) +{ + const struct inode *inode = &ip->i_vnode; + struct xfs_mount *mp = ip->i_mount; + + /* + * Files in the metadata directory tree also have S_PRIVATE set and + * IOP_XATTR unset, so we must distinguish them separately. We (ab)use + * the IRECOVERY flag to mark temporary metadir inodes knowing that the + * end of log recovery clears IRECOVERY, so the only ones that can + * exist during online repair are the ones we create. + */ + if (xfs_has_metadir(mp) && (ip->i_diflags2 & XFS_DIFLAG2_METADATA)) + return __xfs_iflags_test(ip, XFS_IRECOVERY); + + if (IS_PRIVATE(inode) && !(inode->i_opflags & IOP_XATTR)) + return true; + + return false; +} diff --git a/fs/xfs/scrub/tempfile.h b/fs/xfs/scrub/tempfile.h new file mode 100644 index 000000000..71c1b5459 --- /dev/null +++ b/fs/xfs/scrub/tempfile.h @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2021-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_TEMPFILE_H__ +#define __XFS_SCRUB_TEMPFILE_H__ + +#ifdef CONFIG_XFS_ONLINE_REPAIR +int xrep_tempfile_create(struct xfs_scrub *sc, uint16_t mode); +void xrep_tempfile_rele(struct xfs_scrub *sc); + +int xrep_tempfile_adjust_directory_tree(struct xfs_scrub *sc); + +bool xrep_tempfile_iolock_nowait(struct xfs_scrub *sc); +int xrep_tempfile_iolock_polled(struct xfs_scrub *sc); +void xrep_tempfile_iounlock(struct xfs_scrub *sc); + +void xrep_tempfile_ilock(struct xfs_scrub *sc); +bool xrep_tempfile_ilock_nowait(struct xfs_scrub *sc); +void xrep_tempfile_iunlock(struct xfs_scrub *sc); +void xrep_tempfile_iunlock_both(struct xfs_scrub *sc); +void xrep_tempfile_ilock_both(struct xfs_scrub *sc); + +int xrep_tempfile_prealloc(struct xfs_scrub *sc, xfs_fileoff_t off, + xfs_filblks_t len); + +enum xfs_blft; + +typedef int (*xrep_tempfile_copyin_fn)(struct xfs_scrub *sc, + struct xfs_buf *bp, void *data); + +int xrep_tempfile_copyin(struct xfs_scrub *sc, xfs_fileoff_t off, + xfs_filblks_t len, xrep_tempfile_copyin_fn fn, void *data); + +int xrep_tempfile_set_isize(struct xfs_scrub *sc, unsigned long long isize); + +int xrep_tempfile_roll_trans(struct xfs_scrub *sc); +void xrep_tempfile_copyout_local(struct xfs_scrub *sc, int whichfork); +bool xrep_is_tempfile(const struct xfs_inode *ip); +#else +static inline void xrep_tempfile_iolock_both(struct xfs_scrub *sc) +{ + xchk_ilock(sc, XFS_IOLOCK_EXCL); +} +# define xrep_is_tempfile(ip) (false) +# define xrep_tempfile_adjust_directory_tree(sc) (0) +# define xrep_tempfile_rele(sc) +#endif /* CONFIG_XFS_ONLINE_REPAIR */ + +#endif /* __XFS_SCRUB_TEMPFILE_H__ */ diff --git a/fs/xfs/scrub/trace.c b/fs/xfs/scrub/trace.c new file mode 100644 index 000000000..e5efc6806 --- /dev/null +++ b/fs/xfs/scrub/trace.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_btree.h" +#include "xfs_ag.h" +#include "xfs_rtbitmap.h" +#include "xfs_quota.h" +#include "xfs_quota_defs.h" +#include "xfs_da_format.h" +#include "xfs_dir2.h" +#include "xfs_rmap.h" +#include "xfs_parent.h" +#include "xfs_metafile.h" +#include "xfs_rtgroup.h" +#include "xfs_trans.h" +#include "scrub/scrub.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/quota.h" +#include "scrub/iscan.h" +#include "scrub/orphanage.h" +#include "scrub/nlinks.h" +#include "scrub/fscounters.h" +#include "scrub/bitmap.h" +#include "scrub/ino_bitmap.h" +#include "scrub/xfblob.h" +#include "scrub/dirtree.h" + +/* Figure out which block the btree cursor was pointing to. */ +static inline xfs_fsblock_t +xchk_btree_cur_fsbno( + struct xfs_btree_cur *cur, + int level) +{ + if (level < cur->bc_nlevels && cur->bc_levels[level].bp) + return XFS_DADDR_TO_FSB(cur->bc_mp, + xfs_buf_daddr(cur->bc_levels[level].bp)); + + if (level == cur->bc_nlevels - 1 && + cur->bc_ops->type == XFS_BTREE_TYPE_INODE) + return XFS_INODE_TO_FSB(cur->bc_ino.ip); + + return NULLFSBLOCK; +} + +/* + * We include this last to have the helpers above available for the trace + * event implementations. + */ +#define CREATE_TRACE_POINTS +#include "scrub/trace.h" diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h new file mode 100644 index 000000000..14aa0ec1f --- /dev/null +++ b/fs/xfs/scrub/trace.h @@ -0,0 +1,3849 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + * + * NOTE: none of these tracepoints shall be considered a stable kernel ABI + * as they can change at any time. See xfs_trace.h for documentation of + * specific units found in tracepoint output. + */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM xfs_scrub + +#if !defined(_TRACE_XFS_SCRUB_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_XFS_SCRUB_TRACE_H + +#include +#include "xfs_bit.h" +#include "xfs_quota_defs.h" + +struct xfs_rtgroup; +struct xfs_scrub; +struct xfile; +struct xfarray; +struct xfarray_sortinfo; +struct xchk_dqiter; +struct xchk_iscan; +struct xchk_nlink; +struct xchk_fscounters; +struct xfs_rmap_update_params; +struct xfs_parent_rec; +enum xchk_dirpath_outcome; +struct xchk_dirtree; +struct xchk_dirtree_outcomes; + +/* + * ftrace's __print_symbolic requires that all enum values be wrapped in the + * TRACE_DEFINE_ENUM macro so that the enum value can be encoded in the ftrace + * ring buffer. Somehow this was only worth mentioning in the ftrace sample + * code. + */ +TRACE_DEFINE_ENUM(XFS_REFC_DOMAIN_SHARED); +TRACE_DEFINE_ENUM(XFS_REFC_DOMAIN_COW); + +TRACE_DEFINE_ENUM(XG_TYPE_AG); +TRACE_DEFINE_ENUM(XG_TYPE_RTG); + +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_PROBE); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_SB); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_AGF); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_AGFL); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_AGI); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_BNOBT); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_CNTBT); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_INOBT); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_FINOBT); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_RMAPBT); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_REFCNTBT); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_INODE); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_BMBTD); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_BMBTA); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_BMBTC); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_DIR); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_XATTR); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_SYMLINK); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_PARENT); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_RTBITMAP); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_RTSUM); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_UQUOTA); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_GQUOTA); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_PQUOTA); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_FSCOUNTERS); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_QUOTACHECK); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_NLINKS); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_HEALTHY); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_DIRTREE); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_BARRIER); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_METAPATH); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_RGSUPER); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_RTRMAPBT); +TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_RTREFCBT); + +#define XFS_SCRUB_TYPE_STRINGS \ + { XFS_SCRUB_TYPE_PROBE, "probe" }, \ + { XFS_SCRUB_TYPE_SB, "sb" }, \ + { XFS_SCRUB_TYPE_AGF, "agf" }, \ + { XFS_SCRUB_TYPE_AGFL, "agfl" }, \ + { XFS_SCRUB_TYPE_AGI, "agi" }, \ + { XFS_SCRUB_TYPE_BNOBT, "bnobt" }, \ + { XFS_SCRUB_TYPE_CNTBT, "cntbt" }, \ + { XFS_SCRUB_TYPE_INOBT, "inobt" }, \ + { XFS_SCRUB_TYPE_FINOBT, "finobt" }, \ + { XFS_SCRUB_TYPE_RMAPBT, "rmapbt" }, \ + { XFS_SCRUB_TYPE_REFCNTBT, "refcountbt" }, \ + { XFS_SCRUB_TYPE_INODE, "inode" }, \ + { XFS_SCRUB_TYPE_BMBTD, "bmapbtd" }, \ + { XFS_SCRUB_TYPE_BMBTA, "bmapbta" }, \ + { XFS_SCRUB_TYPE_BMBTC, "bmapbtc" }, \ + { XFS_SCRUB_TYPE_DIR, "directory" }, \ + { XFS_SCRUB_TYPE_XATTR, "xattr" }, \ + { XFS_SCRUB_TYPE_SYMLINK, "symlink" }, \ + { XFS_SCRUB_TYPE_PARENT, "parent" }, \ + { XFS_SCRUB_TYPE_RTBITMAP, "rtbitmap" }, \ + { XFS_SCRUB_TYPE_RTSUM, "rtsummary" }, \ + { XFS_SCRUB_TYPE_UQUOTA, "usrquota" }, \ + { XFS_SCRUB_TYPE_GQUOTA, "grpquota" }, \ + { XFS_SCRUB_TYPE_PQUOTA, "prjquota" }, \ + { XFS_SCRUB_TYPE_FSCOUNTERS, "fscounters" }, \ + { XFS_SCRUB_TYPE_QUOTACHECK, "quotacheck" }, \ + { XFS_SCRUB_TYPE_NLINKS, "nlinks" }, \ + { XFS_SCRUB_TYPE_HEALTHY, "healthy" }, \ + { XFS_SCRUB_TYPE_DIRTREE, "dirtree" }, \ + { XFS_SCRUB_TYPE_BARRIER, "barrier" }, \ + { XFS_SCRUB_TYPE_METAPATH, "metapath" }, \ + { XFS_SCRUB_TYPE_RGSUPER, "rgsuper" }, \ + { XFS_SCRUB_TYPE_RTRMAPBT, "rtrmapbt" }, \ + { XFS_SCRUB_TYPE_RTREFCBT, "rtrefcountbt" } + +#define XFS_SCRUB_FLAG_STRINGS \ + { XFS_SCRUB_IFLAG_REPAIR, "repair" }, \ + { XFS_SCRUB_OFLAG_CORRUPT, "corrupt" }, \ + { XFS_SCRUB_OFLAG_PREEN, "preen" }, \ + { XFS_SCRUB_OFLAG_XFAIL, "xfail" }, \ + { XFS_SCRUB_OFLAG_XCORRUPT, "xcorrupt" }, \ + { XFS_SCRUB_OFLAG_INCOMPLETE, "incomplete" }, \ + { XFS_SCRUB_OFLAG_WARNING, "warning" }, \ + { XFS_SCRUB_OFLAG_NO_REPAIR_NEEDED, "norepair" }, \ + { XFS_SCRUB_IFLAG_FORCE_REBUILD, "rebuild" } + +#define XFS_SCRUB_STATE_STRINGS \ + { XCHK_TRY_HARDER, "try_harder" }, \ + { XCHK_HAVE_FREEZE_PROT, "nofreeze" }, \ + { XCHK_FSGATES_DRAIN, "fsgates_drain" }, \ + { XCHK_NEED_DRAIN, "need_drain" }, \ + { XCHK_FSGATES_QUOTA, "fsgates_quota" }, \ + { XCHK_FSGATES_DIRENTS, "fsgates_dirents" }, \ + { XCHK_FSGATES_RMAP, "fsgates_rmap" }, \ + { XREP_RESET_PERAG_RESV, "reset_perag_resv" }, \ + { XREP_ALREADY_FIXED, "already_fixed" } + +TRACE_DEFINE_ENUM(XFS_RMAP_MAP); +TRACE_DEFINE_ENUM(XFS_RMAP_MAP_SHARED); +TRACE_DEFINE_ENUM(XFS_RMAP_UNMAP); +TRACE_DEFINE_ENUM(XFS_RMAP_UNMAP_SHARED); +TRACE_DEFINE_ENUM(XFS_RMAP_CONVERT); +TRACE_DEFINE_ENUM(XFS_RMAP_CONVERT_SHARED); +TRACE_DEFINE_ENUM(XFS_RMAP_ALLOC); +TRACE_DEFINE_ENUM(XFS_RMAP_FREE); + +DECLARE_EVENT_CLASS(xchk_class, + TP_PROTO(struct xfs_inode *ip, struct xfs_scrub_metadata *sm, + int error), + TP_ARGS(ip, sm, error), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(unsigned int, type) + __field(xfs_agnumber_t, agno) + __field(xfs_ino_t, inum) + __field(unsigned int, gen) + __field(unsigned int, flags) + __field(int, error) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = I_INO(ip); + __entry->type = sm->sm_type; + __entry->agno = sm->sm_agno; + __entry->inum = sm->sm_ino; + __entry->gen = sm->sm_gen; + __entry->flags = sm->sm_flags; + __entry->error = error; + ), + TP_printk("dev %d:%d ino 0x%llx type %s agno 0x%x inum 0x%llx gen 0x%x flags (%s) error %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __print_symbolic(__entry->type, XFS_SCRUB_TYPE_STRINGS), + __entry->agno, + __entry->inum, + __entry->gen, + __print_flags(__entry->flags, "|", XFS_SCRUB_FLAG_STRINGS), + __entry->error) +) +#define DEFINE_SCRUB_EVENT(name) \ +DEFINE_EVENT(xchk_class, name, \ + TP_PROTO(struct xfs_inode *ip, struct xfs_scrub_metadata *sm, \ + int error), \ + TP_ARGS(ip, sm, error)) + +DEFINE_SCRUB_EVENT(xchk_start); +DEFINE_SCRUB_EVENT(xchk_done); +DEFINE_SCRUB_EVENT(xchk_deadlock_retry); +DEFINE_SCRUB_EVENT(xchk_dirtree_start); +DEFINE_SCRUB_EVENT(xchk_dirtree_done); +DEFINE_SCRUB_EVENT(xrep_attempt); +DEFINE_SCRUB_EVENT(xrep_done); + +DECLARE_EVENT_CLASS(xchk_fsgate_class, + TP_PROTO(struct xfs_scrub *sc, unsigned int fsgate_flags), + TP_ARGS(sc, fsgate_flags), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, type) + __field(unsigned int, fsgate_flags) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->type = sc->sm->sm_type; + __entry->fsgate_flags = fsgate_flags; + ), + TP_printk("dev %d:%d type %s fsgates '%s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XFS_SCRUB_TYPE_STRINGS), + __print_flags(__entry->fsgate_flags, "|", XFS_SCRUB_STATE_STRINGS)) +) + +#define DEFINE_SCRUB_FSHOOK_EVENT(name) \ +DEFINE_EVENT(xchk_fsgate_class, name, \ + TP_PROTO(struct xfs_scrub *sc, unsigned int fsgates_flags), \ + TP_ARGS(sc, fsgates_flags)) + +DEFINE_SCRUB_FSHOOK_EVENT(xchk_fsgates_enable); +DEFINE_SCRUB_FSHOOK_EVENT(xchk_fsgates_disable); + +DECLARE_EVENT_CLASS(xchk_vector_head_class, + TP_PROTO(struct xfs_inode *ip, struct xfs_scrub_vec_head *vhead), + TP_ARGS(ip, vhead), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_agnumber_t, agno) + __field(xfs_ino_t, inum) + __field(unsigned int, gen) + __field(unsigned int, flags) + __field(unsigned short, rest_us) + __field(unsigned short, nr_vecs) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = I_INO(ip); + __entry->agno = vhead->svh_agno; + __entry->inum = vhead->svh_ino; + __entry->gen = vhead->svh_gen; + __entry->flags = vhead->svh_flags; + __entry->rest_us = vhead->svh_rest_us; + __entry->nr_vecs = vhead->svh_nr; + ), + TP_printk("dev %d:%d ino 0x%llx agno 0x%x inum 0x%llx gen 0x%x flags 0x%x rest_us %u nr_vecs %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->agno, + __entry->inum, + __entry->gen, + __entry->flags, + __entry->rest_us, + __entry->nr_vecs) +) +#define DEFINE_SCRUBV_HEAD_EVENT(name) \ +DEFINE_EVENT(xchk_vector_head_class, name, \ + TP_PROTO(struct xfs_inode *ip, struct xfs_scrub_vec_head *vhead), \ + TP_ARGS(ip, vhead)) + +DEFINE_SCRUBV_HEAD_EVENT(xchk_scrubv_start); + +DECLARE_EVENT_CLASS(xchk_vector_class, + TP_PROTO(struct xfs_mount *mp, struct xfs_scrub_vec_head *vhead, + unsigned int vec_nr, struct xfs_scrub_vec *v), + TP_ARGS(mp, vhead, vec_nr, v), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, vec_nr) + __field(unsigned int, vec_type) + __field(unsigned int, vec_flags) + __field(int, vec_ret) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->vec_nr = vec_nr; + __entry->vec_type = v->sv_type; + __entry->vec_flags = v->sv_flags; + __entry->vec_ret = v->sv_ret; + ), + TP_printk("dev %d:%d vec[%u] type %s flags %s ret %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->vec_nr, + __print_symbolic(__entry->vec_type, XFS_SCRUB_TYPE_STRINGS), + __print_flags(__entry->vec_flags, "|", XFS_SCRUB_FLAG_STRINGS), + __entry->vec_ret) +) +#define DEFINE_SCRUBV_EVENT(name) \ +DEFINE_EVENT(xchk_vector_class, name, \ + TP_PROTO(struct xfs_mount *mp, struct xfs_scrub_vec_head *vhead, \ + unsigned int vec_nr, struct xfs_scrub_vec *v), \ + TP_ARGS(mp, vhead, vec_nr, v)) + +DEFINE_SCRUBV_EVENT(xchk_scrubv_barrier_fail); +DEFINE_SCRUBV_EVENT(xchk_scrubv_item); +DEFINE_SCRUBV_EVENT(xchk_scrubv_outcome); + +TRACE_EVENT(xchk_op_error, + TP_PROTO(struct xfs_scrub *sc, xfs_agnumber_t agno, + xfs_agblock_t bno, int error, void *ret_ip), + TP_ARGS(sc, agno, bno, error, ret_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, type) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, bno) + __field(int, error) + __field(void *, ret_ip) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->type = sc->sm->sm_type; + __entry->agno = agno; + __entry->bno = bno; + __entry->error = error; + __entry->ret_ip = ret_ip; + ), + TP_printk("dev %d:%d type %s agno 0x%x agbno 0x%x error %d ret_ip %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XFS_SCRUB_TYPE_STRINGS), + __entry->agno, + __entry->bno, + __entry->error, + __entry->ret_ip) +); + +TRACE_EVENT(xchk_file_op_error, + TP_PROTO(struct xfs_scrub *sc, int whichfork, + xfs_fileoff_t offset, int error, void *ret_ip), + TP_ARGS(sc, whichfork, offset, error, ret_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(int, whichfork) + __field(unsigned int, type) + __field(xfs_fileoff_t, offset) + __field(int, error) + __field(void *, ret_ip) + ), + TP_fast_assign( + __entry->dev = sc->ip->i_mount->m_super->s_dev; + __entry->ino = I_INO(sc->ip); + __entry->whichfork = whichfork; + __entry->type = sc->sm->sm_type; + __entry->offset = offset; + __entry->error = error; + __entry->ret_ip = ret_ip; + ), + TP_printk("dev %d:%d ino 0x%llx fork %s type %s fileoff 0x%llx error %d ret_ip %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __print_symbolic(__entry->whichfork, XFS_WHICHFORK_STRINGS), + __print_symbolic(__entry->type, XFS_SCRUB_TYPE_STRINGS), + __entry->offset, + __entry->error, + __entry->ret_ip) +); + +DECLARE_EVENT_CLASS(xchk_block_error_class, + TP_PROTO(struct xfs_scrub *sc, xfs_daddr_t daddr, void *ret_ip), + TP_ARGS(sc, daddr, ret_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, type) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, agbno) + __field(void *, ret_ip) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->type = sc->sm->sm_type; + __entry->agno = xfs_daddr_to_agno(sc->mp, daddr); + __entry->agbno = xfs_daddr_to_agbno(sc->mp, daddr); + __entry->ret_ip = ret_ip; + ), + TP_printk("dev %d:%d type %s agno 0x%x agbno 0x%x ret_ip %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XFS_SCRUB_TYPE_STRINGS), + __entry->agno, + __entry->agbno, + __entry->ret_ip) +) + +#define DEFINE_SCRUB_BLOCK_ERROR_EVENT(name) \ +DEFINE_EVENT(xchk_block_error_class, name, \ + TP_PROTO(struct xfs_scrub *sc, xfs_daddr_t daddr, \ + void *ret_ip), \ + TP_ARGS(sc, daddr, ret_ip)) + +DEFINE_SCRUB_BLOCK_ERROR_EVENT(xchk_fs_error); +DEFINE_SCRUB_BLOCK_ERROR_EVENT(xchk_block_error); +DEFINE_SCRUB_BLOCK_ERROR_EVENT(xchk_block_preen); + +DECLARE_EVENT_CLASS(xchk_ino_error_class, + TP_PROTO(struct xfs_scrub *sc, xfs_ino_t ino, void *ret_ip), + TP_ARGS(sc, ino, ret_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(unsigned int, type) + __field(void *, ret_ip) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->ino = ino; + __entry->type = sc->sm->sm_type; + __entry->ret_ip = ret_ip; + ), + TP_printk("dev %d:%d ino 0x%llx type %s ret_ip %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __print_symbolic(__entry->type, XFS_SCRUB_TYPE_STRINGS), + __entry->ret_ip) +) + +#define DEFINE_SCRUB_INO_ERROR_EVENT(name) \ +DEFINE_EVENT(xchk_ino_error_class, name, \ + TP_PROTO(struct xfs_scrub *sc, xfs_ino_t ino, \ + void *ret_ip), \ + TP_ARGS(sc, ino, ret_ip)) + +DEFINE_SCRUB_INO_ERROR_EVENT(xchk_ino_error); +DEFINE_SCRUB_INO_ERROR_EVENT(xchk_ino_preen); +DEFINE_SCRUB_INO_ERROR_EVENT(xchk_ino_warning); + +DECLARE_EVENT_CLASS(xchk_fblock_error_class, + TP_PROTO(struct xfs_scrub *sc, int whichfork, + xfs_fileoff_t offset, void *ret_ip), + TP_ARGS(sc, whichfork, offset, ret_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(int, whichfork) + __field(unsigned int, type) + __field(xfs_fileoff_t, offset) + __field(void *, ret_ip) + ), + TP_fast_assign( + __entry->dev = sc->ip->i_mount->m_super->s_dev; + __entry->ino = I_INO(sc->ip); + __entry->whichfork = whichfork; + __entry->type = sc->sm->sm_type; + __entry->offset = offset; + __entry->ret_ip = ret_ip; + ), + TP_printk("dev %d:%d ino 0x%llx fork %s type %s fileoff 0x%llx ret_ip %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __print_symbolic(__entry->whichfork, XFS_WHICHFORK_STRINGS), + __print_symbolic(__entry->type, XFS_SCRUB_TYPE_STRINGS), + __entry->offset, + __entry->ret_ip) +); + +#define DEFINE_SCRUB_FBLOCK_ERROR_EVENT(name) \ +DEFINE_EVENT(xchk_fblock_error_class, name, \ + TP_PROTO(struct xfs_scrub *sc, int whichfork, \ + xfs_fileoff_t offset, void *ret_ip), \ + TP_ARGS(sc, whichfork, offset, ret_ip)) + +DEFINE_SCRUB_FBLOCK_ERROR_EVENT(xchk_fblock_error); +DEFINE_SCRUB_FBLOCK_ERROR_EVENT(xchk_fblock_warning); +DEFINE_SCRUB_FBLOCK_ERROR_EVENT(xchk_fblock_preen); + +#ifdef CONFIG_XFS_QUOTA +DECLARE_EVENT_CLASS(xchk_dqiter_class, + TP_PROTO(struct xchk_dqiter *cursor, uint64_t id), + TP_ARGS(cursor, id), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_dqtype_t, dqtype) + __field(xfs_ino_t, ino) + __field(unsigned long long, cur_id) + __field(unsigned long long, id) + __field(xfs_fileoff_t, startoff) + __field(xfs_fsblock_t, startblock) + __field(xfs_filblks_t, blockcount) + __field(xfs_exntst_t, state) + ), + TP_fast_assign( + __entry->dev = cursor->sc->mp->m_super->s_dev; + __entry->dqtype = cursor->dqtype; + __entry->ino = I_INO(cursor->quota_ip); + __entry->cur_id = cursor->id; + __entry->startoff = cursor->bmap.br_startoff; + __entry->startblock = cursor->bmap.br_startblock; + __entry->blockcount = cursor->bmap.br_blockcount; + __entry->state = cursor->bmap.br_state; + __entry->id = id; + ), + TP_printk("dev %d:%d dquot type %s ino 0x%llx cursor_id 0x%llx startoff 0x%llx startblock 0x%llx blockcount 0x%llx state %u id 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->dqtype, XFS_DQTYPE_STRINGS), + __entry->ino, + __entry->cur_id, + __entry->startoff, + __entry->startblock, + __entry->blockcount, + __entry->state, + __entry->id) +); + +#define DEFINE_SCRUB_DQITER_EVENT(name) \ +DEFINE_EVENT(xchk_dqiter_class, name, \ + TP_PROTO(struct xchk_dqiter *cursor, uint64_t id), \ + TP_ARGS(cursor, id)) +DEFINE_SCRUB_DQITER_EVENT(xchk_dquot_iter_revalidate_bmap); +DEFINE_SCRUB_DQITER_EVENT(xchk_dquot_iter_advance_bmap); +DEFINE_SCRUB_DQITER_EVENT(xchk_dquot_iter_advance_incore); +DEFINE_SCRUB_DQITER_EVENT(xchk_dquot_iter); + +TRACE_EVENT(xchk_qcheck_error, + TP_PROTO(struct xfs_scrub *sc, xfs_dqtype_t dqtype, xfs_dqid_t id, + void *ret_ip), + TP_ARGS(sc, dqtype, id, ret_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_dqtype_t, dqtype) + __field(xfs_dqid_t, id) + __field(void *, ret_ip) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->dqtype = dqtype; + __entry->id = id; + __entry->ret_ip = ret_ip; + ), + TP_printk("dev %d:%d dquot type %s id 0x%x ret_ip %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->dqtype, XFS_DQTYPE_STRINGS), + __entry->id, + __entry->ret_ip) +); +#endif /* CONFIG_XFS_QUOTA */ + +TRACE_EVENT(xchk_incomplete, + TP_PROTO(struct xfs_scrub *sc, void *ret_ip), + TP_ARGS(sc, ret_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, type) + __field(void *, ret_ip) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->type = sc->sm->sm_type; + __entry->ret_ip = ret_ip; + ), + TP_printk("dev %d:%d type %s ret_ip %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XFS_SCRUB_TYPE_STRINGS), + __entry->ret_ip) +); + +TRACE_EVENT(xchk_btree_op_error, + TP_PROTO(struct xfs_scrub *sc, struct xfs_btree_cur *cur, + int level, int error, void *ret_ip), + TP_ARGS(sc, cur, level, error, ret_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, type) + __string(name, cur->bc_ops->name) + __field(int, level) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, bno) + __field(int, ptr) + __field(int, error) + __field(void *, ret_ip) + ), + TP_fast_assign( + xfs_fsblock_t fsbno = xchk_btree_cur_fsbno(cur, level); + + __entry->dev = sc->mp->m_super->s_dev; + __entry->type = sc->sm->sm_type; + __assign_str(name); + __entry->level = level; + __entry->agno = XFS_FSB_TO_AGNO(cur->bc_mp, fsbno); + __entry->bno = XFS_FSB_TO_AGBNO(cur->bc_mp, fsbno); + __entry->ptr = cur->bc_levels[level].ptr; + __entry->error = error; + __entry->ret_ip = ret_ip; + ), + TP_printk("dev %d:%d type %s %sbt level %d ptr %d agno 0x%x agbno 0x%x error %d ret_ip %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XFS_SCRUB_TYPE_STRINGS), + __get_str(name), + __entry->level, + __entry->ptr, + __entry->agno, + __entry->bno, + __entry->error, + __entry->ret_ip) +); + +TRACE_EVENT(xchk_ifork_btree_op_error, + TP_PROTO(struct xfs_scrub *sc, struct xfs_btree_cur *cur, + int level, int error, void *ret_ip), + TP_ARGS(sc, cur, level, error, ret_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(int, whichfork) + __field(unsigned int, type) + __string(name, cur->bc_ops->name) + __field(int, level) + __field(int, ptr) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, bno) + __field(int, error) + __field(void *, ret_ip) + ), + TP_fast_assign( + xfs_fsblock_t fsbno = xchk_btree_cur_fsbno(cur, level); + __entry->dev = sc->mp->m_super->s_dev; + __entry->ino = I_INO(cur->bc_ino.ip); + __entry->whichfork = cur->bc_ino.whichfork; + __entry->type = sc->sm->sm_type; + __assign_str(name); + __entry->level = level; + __entry->ptr = cur->bc_levels[level].ptr; + __entry->agno = XFS_FSB_TO_AGNO(cur->bc_mp, fsbno); + __entry->bno = XFS_FSB_TO_AGBNO(cur->bc_mp, fsbno); + __entry->error = error; + __entry->ret_ip = ret_ip; + ), + TP_printk("dev %d:%d ino 0x%llx fork %s type %s %sbt level %d ptr %d agno 0x%x agbno 0x%x error %d ret_ip %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __print_symbolic(__entry->whichfork, XFS_WHICHFORK_STRINGS), + __print_symbolic(__entry->type, XFS_SCRUB_TYPE_STRINGS), + __get_str(name), + __entry->level, + __entry->ptr, + __entry->agno, + __entry->bno, + __entry->error, + __entry->ret_ip) +); + +TRACE_EVENT(xchk_btree_error, + TP_PROTO(struct xfs_scrub *sc, struct xfs_btree_cur *cur, + int level, void *ret_ip), + TP_ARGS(sc, cur, level, ret_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, type) + __string(name, cur->bc_ops->name) + __field(int, level) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, bno) + __field(int, ptr) + __field(void *, ret_ip) + ), + TP_fast_assign( + xfs_fsblock_t fsbno = xchk_btree_cur_fsbno(cur, level); + __entry->dev = sc->mp->m_super->s_dev; + __entry->type = sc->sm->sm_type; + __assign_str(name); + __entry->level = level; + __entry->agno = XFS_FSB_TO_AGNO(cur->bc_mp, fsbno); + __entry->bno = XFS_FSB_TO_AGBNO(cur->bc_mp, fsbno); + __entry->ptr = cur->bc_levels[level].ptr; + __entry->ret_ip = ret_ip; + ), + TP_printk("dev %d:%d type %s %sbt level %d ptr %d agno 0x%x agbno 0x%x ret_ip %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XFS_SCRUB_TYPE_STRINGS), + __get_str(name), + __entry->level, + __entry->ptr, + __entry->agno, + __entry->bno, + __entry->ret_ip) +); + +TRACE_EVENT(xchk_ifork_btree_error, + TP_PROTO(struct xfs_scrub *sc, struct xfs_btree_cur *cur, + int level, void *ret_ip), + TP_ARGS(sc, cur, level, ret_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(int, whichfork) + __field(unsigned int, type) + __string(name, cur->bc_ops->name) + __field(int, level) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, bno) + __field(int, ptr) + __field(void *, ret_ip) + ), + TP_fast_assign( + xfs_fsblock_t fsbno = xchk_btree_cur_fsbno(cur, level); + __entry->dev = sc->mp->m_super->s_dev; + __entry->ino = I_INO(sc->ip); + __entry->whichfork = cur->bc_ino.whichfork; + __entry->type = sc->sm->sm_type; + __assign_str(name); + __entry->level = level; + __entry->agno = XFS_FSB_TO_AGNO(cur->bc_mp, fsbno); + __entry->bno = XFS_FSB_TO_AGBNO(cur->bc_mp, fsbno); + __entry->ptr = cur->bc_levels[level].ptr; + __entry->ret_ip = ret_ip; + ), + TP_printk("dev %d:%d ino 0x%llx fork %s type %s %sbt level %d ptr %d agno 0x%x agbno 0x%x ret_ip %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __print_symbolic(__entry->whichfork, XFS_WHICHFORK_STRINGS), + __print_symbolic(__entry->type, XFS_SCRUB_TYPE_STRINGS), + __get_str(name), + __entry->level, + __entry->ptr, + __entry->agno, + __entry->bno, + __entry->ret_ip) +); + +DECLARE_EVENT_CLASS(xchk_sbtree_class, + TP_PROTO(struct xfs_scrub *sc, struct xfs_btree_cur *cur, + int level), + TP_ARGS(sc, cur, level), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(int, type) + __string(name, cur->bc_ops->name) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, bno) + __field(int, level) + __field(int, nlevels) + __field(int, ptr) + ), + TP_fast_assign( + xfs_fsblock_t fsbno = xchk_btree_cur_fsbno(cur, level); + + __entry->dev = sc->mp->m_super->s_dev; + __entry->type = sc->sm->sm_type; + __assign_str(name); + __entry->agno = XFS_FSB_TO_AGNO(cur->bc_mp, fsbno); + __entry->bno = XFS_FSB_TO_AGBNO(cur->bc_mp, fsbno); + __entry->level = level; + __entry->nlevels = cur->bc_nlevels; + __entry->ptr = cur->bc_levels[level].ptr; + ), + TP_printk("dev %d:%d type %s %sbt agno 0x%x agbno 0x%x level %d nlevels %d ptr %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XFS_SCRUB_TYPE_STRINGS), + __get_str(name), + __entry->agno, + __entry->bno, + __entry->level, + __entry->nlevels, + __entry->ptr) +) +#define DEFINE_SCRUB_SBTREE_EVENT(name) \ +DEFINE_EVENT(xchk_sbtree_class, name, \ + TP_PROTO(struct xfs_scrub *sc, struct xfs_btree_cur *cur, \ + int level), \ + TP_ARGS(sc, cur, level)) + +DEFINE_SCRUB_SBTREE_EVENT(xchk_btree_rec); +DEFINE_SCRUB_SBTREE_EVENT(xchk_btree_key); + +TRACE_EVENT(xchk_xref_error, + TP_PROTO(struct xfs_scrub *sc, int error, void *ret_ip), + TP_ARGS(sc, error, ret_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(int, type) + __field(int, error) + __field(void *, ret_ip) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->type = sc->sm->sm_type; + __entry->error = error; + __entry->ret_ip = ret_ip; + ), + TP_printk("dev %d:%d type %s xref error %d ret_ip %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XFS_SCRUB_TYPE_STRINGS), + __entry->error, + __entry->ret_ip) +); + +TRACE_EVENT(xchk_iallocbt_check_cluster, + TP_PROTO(const struct xfs_perag *pag, xfs_agino_t startino, + xfs_agblock_t map_agblock, unsigned short map_len, + unsigned int chunk_ino, unsigned int nr_inodes, + uint16_t cluster_mask, uint16_t holemask, + unsigned int cluster_ino), + TP_ARGS(pag, startino, map_agblock, map_len, chunk_ino, nr_inodes, + cluster_mask, holemask, cluster_ino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agino_t, startino) + __field(xfs_agblock_t, map_agblock) + __field(unsigned short, map_len) + __field(unsigned int, chunk_ino) + __field(unsigned int, nr_inodes) + __field(unsigned int, cluster_ino) + __field(uint16_t, cluster_mask) + __field(uint16_t, holemask) + ), + TP_fast_assign( + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->agno = pag_agno(pag); + __entry->startino = startino; + __entry->map_agblock = map_agblock; + __entry->map_len = map_len; + __entry->chunk_ino = chunk_ino; + __entry->nr_inodes = nr_inodes; + __entry->cluster_mask = cluster_mask; + __entry->holemask = holemask; + __entry->cluster_ino = cluster_ino; + ), + TP_printk("dev %d:%d agno 0x%x startino 0x%x agblock 0x%x bbcount 0x%x chunkino 0x%x nr_inodes %u cluster_mask 0x%x holemask 0x%x cluster_ino 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->startino, + __entry->map_agblock, + __entry->map_len, + __entry->chunk_ino, + __entry->nr_inodes, + __entry->cluster_mask, + __entry->holemask, + __entry->cluster_ino) +) + +TRACE_EVENT(xchk_inode_is_allocated, + TP_PROTO(struct xfs_inode *ip), + TP_ARGS(ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(unsigned long, iflags) + __field(umode_t, mode) + ), + TP_fast_assign( + __entry->dev = VFS_I(ip)->i_sb->s_dev; + __entry->ino = I_INO(ip); + __entry->iflags = ip->i_flags; + __entry->mode = VFS_I(ip)->i_mode; + ), + TP_printk("dev %d:%d ino 0x%llx iflags 0x%lx mode 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->iflags, + __entry->mode) +); + +TRACE_EVENT(xchk_fscounters_calc, + TP_PROTO(struct xfs_mount *mp, uint64_t icount, uint64_t ifree, + uint64_t fdblocks, uint64_t delalloc), + TP_ARGS(mp, icount, ifree, fdblocks, delalloc), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(int64_t, icount_sb) + __field(uint64_t, icount_calculated) + __field(int64_t, ifree_sb) + __field(uint64_t, ifree_calculated) + __field(int64_t, fdblocks_sb) + __field(uint64_t, fdblocks_calculated) + __field(uint64_t, delalloc) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->icount_sb = mp->m_sb.sb_icount; + __entry->icount_calculated = icount; + __entry->ifree_sb = mp->m_sb.sb_ifree; + __entry->ifree_calculated = ifree; + __entry->fdblocks_sb = mp->m_sb.sb_fdblocks; + __entry->fdblocks_calculated = fdblocks; + __entry->delalloc = delalloc; + ), + TP_printk("dev %d:%d icount %lld:%llu ifree %lld::%llu fdblocks %lld::%llu delalloc %llu", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->icount_sb, + __entry->icount_calculated, + __entry->ifree_sb, + __entry->ifree_calculated, + __entry->fdblocks_sb, + __entry->fdblocks_calculated, + __entry->delalloc) +) + +TRACE_EVENT(xchk_fscounters_within_range, + TP_PROTO(struct xfs_mount *mp, uint64_t expected, int64_t curr_value, + int64_t old_value), + TP_ARGS(mp, expected, curr_value, old_value), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(uint64_t, expected) + __field(int64_t, curr_value) + __field(int64_t, old_value) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->expected = expected; + __entry->curr_value = curr_value; + __entry->old_value = old_value; + ), + TP_printk("dev %d:%d expected %llu curr_value %lld old_value %lld", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->expected, + __entry->curr_value, + __entry->old_value) +) + +DECLARE_EVENT_CLASS(xchk_fsfreeze_class, + TP_PROTO(struct xfs_scrub *sc, int error), + TP_ARGS(sc, error), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, type) + __field(int, error) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->type = sc->sm->sm_type; + __entry->error = error; + ), + TP_printk("dev %d:%d type %s error %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XFS_SCRUB_TYPE_STRINGS), + __entry->error) +); +#define DEFINE_XCHK_FSFREEZE_EVENT(name) \ +DEFINE_EVENT(xchk_fsfreeze_class, name, \ + TP_PROTO(struct xfs_scrub *sc, int error), \ + TP_ARGS(sc, error)) +DEFINE_XCHK_FSFREEZE_EVENT(xchk_fsfreeze); +DEFINE_XCHK_FSFREEZE_EVENT(xchk_fsthaw); + +TRACE_EVENT(xchk_refcount_incorrect, + TP_PROTO(const struct xfs_perag *pag, + const struct xfs_refcount_irec *irec, + xfs_nlink_t seen), + TP_ARGS(pag, irec, seen), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(enum xfs_refc_domain, domain) + __field(xfs_agblock_t, startblock) + __field(xfs_extlen_t, blockcount) + __field(xfs_nlink_t, refcount) + __field(xfs_nlink_t, seen) + ), + TP_fast_assign( + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->agno = pag_agno(pag); + __entry->domain = irec->rc_domain; + __entry->startblock = irec->rc_startblock; + __entry->blockcount = irec->rc_blockcount; + __entry->refcount = irec->rc_refcount; + __entry->seen = seen; + ), + TP_printk("dev %d:%d agno 0x%x dom %s agbno 0x%x fsbcount 0x%x refcount %u seen %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __print_symbolic(__entry->domain, XFS_REFC_DOMAIN_STRINGS), + __entry->startblock, + __entry->blockcount, + __entry->refcount, + __entry->seen) +) + +TRACE_EVENT(xfile_create, + TP_PROTO(struct xfile *xf), + TP_ARGS(xf), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned long, ino) + ), + TP_fast_assign( + __entry->ino = file_inode(xf->file)->i_ino; + ), + TP_printk("xfino 0x%lx", + __entry->ino) +); + +TRACE_EVENT(xfile_destroy, + TP_PROTO(struct xfile *xf), + TP_ARGS(xf), + TP_STRUCT__entry( + __field(unsigned long, ino) + __field(unsigned long long, bytes) + __field(loff_t, size) + ), + TP_fast_assign( + struct inode *inode = file_inode(xf->file); + + __entry->ino = inode->i_ino; + __entry->bytes = inode->i_blocks << SECTOR_SHIFT; + __entry->size = i_size_read(inode); + ), + TP_printk("xfino 0x%lx mem_bytes 0x%llx isize 0x%llx", + __entry->ino, + __entry->bytes, + __entry->size) +); + +DECLARE_EVENT_CLASS(xfile_class, + TP_PROTO(struct xfile *xf, loff_t pos, unsigned long long bytecount), + TP_ARGS(xf, pos, bytecount), + TP_STRUCT__entry( + __field(unsigned long, ino) + __field(unsigned long long, bytes_used) + __field(loff_t, pos) + __field(loff_t, size) + __field(unsigned long long, bytecount) + ), + TP_fast_assign( + struct inode *inode = file_inode(xf->file); + + __entry->ino = inode->i_ino; + __entry->bytes_used = inode->i_blocks << SECTOR_SHIFT; + __entry->pos = pos; + __entry->size = i_size_read(inode); + __entry->bytecount = bytecount; + ), + TP_printk("xfino 0x%lx mem_bytes 0x%llx pos 0x%llx bytecount 0x%llx isize 0x%llx", + __entry->ino, + __entry->bytes_used, + __entry->pos, + __entry->bytecount, + __entry->size) +); +#define DEFINE_XFILE_EVENT(name) \ +DEFINE_EVENT(xfile_class, name, \ + TP_PROTO(struct xfile *xf, loff_t pos, unsigned long long bytecount), \ + TP_ARGS(xf, pos, bytecount)) +DEFINE_XFILE_EVENT(xfile_load); +DEFINE_XFILE_EVENT(xfile_store); +DEFINE_XFILE_EVENT(xfile_seek_data); +DEFINE_XFILE_EVENT(xfile_get_folio); +DEFINE_XFILE_EVENT(xfile_put_folio); +DEFINE_XFILE_EVENT(xfile_discard); + +TRACE_EVENT(xfarray_create, + TP_PROTO(struct xfarray *xfa, unsigned long long required_capacity), + TP_ARGS(xfa, required_capacity), + TP_STRUCT__entry( + __field(unsigned long, ino) + __field(uint64_t, max_nr) + __field(size_t, obj_size) + __field(int, obj_size_log) + __field(unsigned long long, required_capacity) + ), + TP_fast_assign( + __entry->max_nr = xfa->max_nr; + __entry->obj_size = xfa->obj_size; + __entry->obj_size_log = xfa->obj_size_log; + __entry->ino = file_inode(xfa->xfile->file)->i_ino; + __entry->required_capacity = required_capacity; + ), + TP_printk("xfino 0x%lx max_nr %llu reqd_nr %llu objsz %zu objszlog %d", + __entry->ino, + __entry->max_nr, + __entry->required_capacity, + __entry->obj_size, + __entry->obj_size_log) +); + +TRACE_EVENT(xfarray_isort, + TP_PROTO(struct xfarray_sortinfo *si, uint64_t lo, uint64_t hi), + TP_ARGS(si, lo, hi), + TP_STRUCT__entry( + __field(unsigned long, ino) + __field(unsigned long long, lo) + __field(unsigned long long, hi) + ), + TP_fast_assign( + __entry->ino = file_inode(si->array->xfile->file)->i_ino; + __entry->lo = lo; + __entry->hi = hi; + ), + TP_printk("xfino 0x%lx lo %llu hi %llu elts %llu", + __entry->ino, + __entry->lo, + __entry->hi, + __entry->hi - __entry->lo) +); + +TRACE_EVENT(xfarray_foliosort, + TP_PROTO(struct xfarray_sortinfo *si, uint64_t lo, uint64_t hi), + TP_ARGS(si, lo, hi), + TP_STRUCT__entry( + __field(unsigned long, ino) + __field(unsigned long long, lo) + __field(unsigned long long, hi) + ), + TP_fast_assign( + __entry->ino = file_inode(si->array->xfile->file)->i_ino; + __entry->lo = lo; + __entry->hi = hi; + ), + TP_printk("xfino 0x%lx lo %llu hi %llu elts %llu", + __entry->ino, + __entry->lo, + __entry->hi, + __entry->hi - __entry->lo) +); + +TRACE_EVENT(xfarray_qsort, + TP_PROTO(struct xfarray_sortinfo *si, uint64_t lo, uint64_t hi), + TP_ARGS(si, lo, hi), + TP_STRUCT__entry( + __field(unsigned long, ino) + __field(unsigned long long, lo) + __field(unsigned long long, hi) + __field(int, stack_depth) + __field(int, max_stack_depth) + ), + TP_fast_assign( + __entry->ino = file_inode(si->array->xfile->file)->i_ino; + __entry->lo = lo; + __entry->hi = hi; + __entry->stack_depth = si->stack_depth; + __entry->max_stack_depth = si->max_stack_depth; + ), + TP_printk("xfino 0x%lx lo %llu hi %llu elts %llu stack %d/%d", + __entry->ino, + __entry->lo, + __entry->hi, + __entry->hi - __entry->lo, + __entry->stack_depth, + __entry->max_stack_depth) +); + +TRACE_EVENT(xfarray_sort, + TP_PROTO(struct xfarray_sortinfo *si, size_t bytes), + TP_ARGS(si, bytes), + TP_STRUCT__entry( + __field(unsigned long, ino) + __field(unsigned long long, nr) + __field(size_t, obj_size) + __field(size_t, bytes) + __field(unsigned int, max_stack_depth) + ), + TP_fast_assign( + __entry->nr = si->array->nr; + __entry->obj_size = si->array->obj_size; + __entry->ino = file_inode(si->array->xfile->file)->i_ino; + __entry->bytes = bytes; + __entry->max_stack_depth = si->max_stack_depth; + ), + TP_printk("xfino 0x%lx nr %llu objsz %zu stack %u bytes %zu", + __entry->ino, + __entry->nr, + __entry->obj_size, + __entry->max_stack_depth, + __entry->bytes) +); + +TRACE_EVENT(xfarray_sort_scan, + TP_PROTO(struct xfarray_sortinfo *si, unsigned long long idx), + TP_ARGS(si, idx), + TP_STRUCT__entry( + __field(unsigned long, ino) + __field(unsigned long long, nr) + __field(size_t, obj_size) + __field(unsigned long long, idx) + __field(unsigned long long, folio_pos) + __field(unsigned long, folio_bytes) + __field(unsigned long long, first_idx) + __field(unsigned long long, last_idx) + ), + TP_fast_assign( + __entry->nr = si->array->nr; + __entry->obj_size = si->array->obj_size; + __entry->ino = file_inode(si->array->xfile->file)->i_ino; + __entry->idx = idx; + if (si->folio) { + __entry->folio_pos = folio_pos(si->folio); + __entry->folio_bytes = folio_size(si->folio); + __entry->first_idx = si->first_folio_idx; + __entry->last_idx = si->last_folio_idx; + } else { + __entry->folio_pos = 0; + __entry->folio_bytes = 0; + __entry->first_idx = 0; + __entry->last_idx = 0; + } + ), + TP_printk("xfino 0x%lx nr %llu objsz %zu idx %llu folio_pos 0x%llx folio_bytes 0x%lx first_idx %llu last_idx %llu", + __entry->ino, + __entry->nr, + __entry->obj_size, + __entry->idx, + __entry->folio_pos, + __entry->folio_bytes, + __entry->first_idx, + __entry->last_idx) +); + +TRACE_EVENT(xfarray_sort_stats, + TP_PROTO(struct xfarray_sortinfo *si, int error), + TP_ARGS(si, error), + TP_STRUCT__entry( + __field(unsigned long, ino) +#ifdef DEBUG + __field(unsigned long long, loads) + __field(unsigned long long, stores) + __field(unsigned long long, compares) + __field(unsigned long long, heapsorts) +#endif + __field(unsigned int, max_stack_depth) + __field(unsigned int, max_stack_used) + __field(int, error) + ), + TP_fast_assign( + __entry->ino = file_inode(si->array->xfile->file)->i_ino; +#ifdef DEBUG + __entry->loads = si->loads; + __entry->stores = si->stores; + __entry->compares = si->compares; + __entry->heapsorts = si->heapsorts; +#endif + __entry->max_stack_depth = si->max_stack_depth; + __entry->max_stack_used = si->max_stack_used; + __entry->error = error; + ), + TP_printk( +#ifdef DEBUG + "xfino 0x%lx loads %llu stores %llu compares %llu heapsorts %llu stack_depth %u/%u error %d", +#else + "xfino 0x%lx stack_depth %u/%u error %d", +#endif + __entry->ino, +#ifdef DEBUG + __entry->loads, + __entry->stores, + __entry->compares, + __entry->heapsorts, +#endif + __entry->max_stack_used, + __entry->max_stack_depth, + __entry->error) +); + +#ifdef CONFIG_XFS_RT +TRACE_EVENT(xchk_rtsum_record_free, + TP_PROTO(struct xfs_mount *mp, xfs_rtxnum_t start, + xfs_rtbxlen_t len, unsigned int log, loff_t pos, + xfs_suminfo_t value), + TP_ARGS(mp, start, len, log, pos, value), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(dev_t, rtdev) + __field(xfs_rtxnum_t, start) + __field(unsigned long long, len) + __field(unsigned int, log) + __field(loff_t, pos) + __field(xfs_suminfo_t, value) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->rtdev = mp->m_rtdev_targp->bt_dev; + __entry->start = start; + __entry->len = len; + __entry->log = log; + __entry->pos = pos; + __entry->value = value; + ), + TP_printk("dev %d:%d rtdev %d:%d rtx 0x%llx rtxcount 0x%llx log %u rsumpos 0x%llx sumcount %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + MAJOR(__entry->rtdev), MINOR(__entry->rtdev), + __entry->start, + __entry->len, + __entry->log, + __entry->pos, + __entry->value) +); +#endif /* CONFIG_XFS_RT */ + +DECLARE_EVENT_CLASS(xchk_iscan_class, + TP_PROTO(struct xchk_iscan *iscan), + TP_ARGS(iscan), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, cursor) + __field(xfs_ino_t, visited) + ), + TP_fast_assign( + __entry->dev = iscan->sc->mp->m_super->s_dev; + __entry->cursor = iscan->cursor_ino; + __entry->visited = iscan->__visited_ino; + ), + TP_printk("dev %d:%d iscan cursor 0x%llx visited 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->cursor, + __entry->visited) +) +#define DEFINE_ISCAN_EVENT(name) \ +DEFINE_EVENT(xchk_iscan_class, name, \ + TP_PROTO(struct xchk_iscan *iscan), \ + TP_ARGS(iscan)) +DEFINE_ISCAN_EVENT(xchk_iscan_move_cursor); +DEFINE_ISCAN_EVENT(xchk_iscan_visit); +DEFINE_ISCAN_EVENT(xchk_iscan_skip); +DEFINE_ISCAN_EVENT(xchk_iscan_advance_ag); + +DECLARE_EVENT_CLASS(xchk_iscan_ino_class, + TP_PROTO(struct xchk_iscan *iscan, xfs_ino_t ino), + TP_ARGS(iscan, ino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, startino) + __field(xfs_ino_t, cursor) + __field(xfs_ino_t, visited) + __field(xfs_ino_t, ino) + ), + TP_fast_assign( + __entry->dev = iscan->sc->mp->m_super->s_dev; + __entry->startino = iscan->scan_start_ino; + __entry->cursor = iscan->cursor_ino; + __entry->visited = iscan->__visited_ino; + __entry->ino = ino; + ), + TP_printk("dev %d:%d iscan start 0x%llx cursor 0x%llx visited 0x%llx ino 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->startino, + __entry->cursor, + __entry->visited, + __entry->ino) +) +#define DEFINE_ISCAN_INO_EVENT(name) \ +DEFINE_EVENT(xchk_iscan_ino_class, name, \ + TP_PROTO(struct xchk_iscan *iscan, xfs_ino_t ino), \ + TP_ARGS(iscan, ino)) +DEFINE_ISCAN_INO_EVENT(xchk_iscan_want_live_update); +DEFINE_ISCAN_INO_EVENT(xchk_iscan_start); + +TRACE_EVENT(xchk_iscan_iget, + TP_PROTO(struct xchk_iscan *iscan, int error), + TP_ARGS(iscan, error), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, cursor) + __field(xfs_ino_t, visited) + __field(int, error) + ), + TP_fast_assign( + __entry->dev = iscan->sc->mp->m_super->s_dev; + __entry->cursor = iscan->cursor_ino; + __entry->visited = iscan->__visited_ino; + __entry->error = error; + ), + TP_printk("dev %d:%d iscan cursor 0x%llx visited 0x%llx error %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->cursor, + __entry->visited, + __entry->error) +); + +TRACE_EVENT(xchk_iscan_iget_batch, + TP_PROTO(struct xfs_mount *mp, struct xchk_iscan *iscan, + unsigned int nr, unsigned int avail), + TP_ARGS(mp, iscan, nr, avail), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, cursor) + __field(xfs_ino_t, visited) + __field(unsigned int, nr) + __field(unsigned int, avail) + __field(unsigned int, unavail) + __field(xfs_ino_t, batch_ino) + __field(unsigned long long, skipmask) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->cursor = iscan->cursor_ino; + __entry->visited = iscan->__visited_ino; + __entry->nr = nr; + __entry->avail = avail; + __entry->unavail = hweight64(iscan->__skipped_inomask); + __entry->batch_ino = iscan->__batch_ino; + __entry->skipmask = iscan->__skipped_inomask; + ), + TP_printk("dev %d:%d iscan cursor 0x%llx visited 0x%llx batchino 0x%llx skipmask 0x%llx nr %u avail %u unavail %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->cursor, + __entry->visited, + __entry->batch_ino, + __entry->skipmask, + __entry->nr, + __entry->avail, + __entry->unavail) +); + +DECLARE_EVENT_CLASS(xchk_iscan_retry_wait_class, + TP_PROTO(struct xchk_iscan *iscan), + TP_ARGS(iscan), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, cursor) + __field(xfs_ino_t, visited) + __field(unsigned int, retry_delay) + __field(unsigned long, remaining) + __field(unsigned int, iget_timeout) + ), + TP_fast_assign( + __entry->dev = iscan->sc->mp->m_super->s_dev; + __entry->cursor = iscan->cursor_ino; + __entry->visited = iscan->__visited_ino; + __entry->retry_delay = iscan->iget_retry_delay; + __entry->remaining = jiffies_to_msecs(iscan->__iget_deadline - jiffies); + __entry->iget_timeout = iscan->iget_timeout; + ), + TP_printk("dev %d:%d iscan cursor 0x%llx visited 0x%llx remaining %lu timeout %u delay %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->cursor, + __entry->visited, + __entry->remaining, + __entry->iget_timeout, + __entry->retry_delay) +) +#define DEFINE_ISCAN_RETRY_WAIT_EVENT(name) \ +DEFINE_EVENT(xchk_iscan_retry_wait_class, name, \ + TP_PROTO(struct xchk_iscan *iscan), \ + TP_ARGS(iscan)) +DEFINE_ISCAN_RETRY_WAIT_EVENT(xchk_iscan_iget_retry_wait); +DEFINE_ISCAN_RETRY_WAIT_EVENT(xchk_iscan_agi_retry_wait); + +TRACE_EVENT(xchk_nlinks_collect_dirent, + TP_PROTO(struct xfs_mount *mp, struct xfs_inode *dp, + xfs_ino_t ino, const struct xfs_name *name), + TP_ARGS(mp, dp, ino, name), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, dir) + __field(xfs_ino_t, ino) + __field(unsigned int, namelen) + __dynamic_array(char, name, name->len) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->dir = I_INO(dp); + __entry->ino = ino; + __entry->namelen = name->len; + memcpy(__get_str(name), name->name, name->len); + ), + TP_printk("dev %d:%d dir 0x%llx -> ino 0x%llx name '%.*s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->dir, + __entry->ino, + __entry->namelen, + __get_str(name)) +); + +TRACE_EVENT(xchk_nlinks_collect_pptr, + TP_PROTO(struct xfs_mount *mp, struct xfs_inode *dp, + const struct xfs_name *name, + const struct xfs_parent_rec *pptr), + TP_ARGS(mp, dp, name, pptr), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, dir) + __field(xfs_ino_t, ino) + __field(unsigned int, namelen) + __dynamic_array(char, name, name->len) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->dir = I_INO(dp); + __entry->ino = be64_to_cpu(pptr->p_ino); + __entry->namelen = name->len; + memcpy(__get_str(name), name->name, name->len); + ), + TP_printk("dev %d:%d dir 0x%llx -> ino 0x%llx name '%.*s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->dir, + __entry->ino, + __entry->namelen, + __get_str(name)) +); + +TRACE_EVENT(xchk_nlinks_collect_metafile, + TP_PROTO(struct xfs_mount *mp, xfs_ino_t ino), + TP_ARGS(mp, ino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->ino = ino; + ), + TP_printk("dev %d:%d ino 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino) +); + +TRACE_EVENT(xchk_nlinks_live_update, + TP_PROTO(struct xfs_mount *mp, const struct xfs_inode *dp, + int action, xfs_ino_t ino, int delta, + const char *name, unsigned int namelen), + TP_ARGS(mp, dp, action, ino, delta, name, namelen), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, dir) + __field(int, action) + __field(xfs_ino_t, ino) + __field(int, delta) + __field(unsigned int, namelen) + __dynamic_array(char, name, namelen) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->dir = dp ? I_INO(dp) : NULLFSINO; + __entry->action = action; + __entry->ino = ino; + __entry->delta = delta; + __entry->namelen = namelen; + memcpy(__get_str(name), name, namelen); + ), + TP_printk("dev %d:%d dir 0x%llx ino 0x%llx nlink_delta %d name '%.*s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->dir, + __entry->ino, + __entry->delta, + __entry->namelen, + __get_str(name)) +); + +TRACE_EVENT(xchk_nlinks_check_zero, + TP_PROTO(struct xfs_mount *mp, xfs_ino_t ino, + const struct xchk_nlink *live), + TP_ARGS(mp, ino, live), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_nlink_t, parents) + __field(xfs_nlink_t, backrefs) + __field(xfs_nlink_t, children) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->ino = ino; + __entry->parents = live->parents; + __entry->backrefs = live->backrefs; + __entry->children = live->children; + ), + TP_printk("dev %d:%d ino 0x%llx parents %u backrefs %u children %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->parents, + __entry->backrefs, + __entry->children) +); + +TRACE_EVENT(xchk_nlinks_update_incore, + TP_PROTO(struct xfs_mount *mp, xfs_ino_t ino, + const struct xchk_nlink *live, int parents_delta, + int backrefs_delta, int children_delta), + TP_ARGS(mp, ino, live, parents_delta, backrefs_delta, children_delta), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_nlink_t, parents) + __field(xfs_nlink_t, backrefs) + __field(xfs_nlink_t, children) + __field(int, parents_delta) + __field(int, backrefs_delta) + __field(int, children_delta) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->ino = ino; + __entry->parents = live->parents; + __entry->backrefs = live->backrefs; + __entry->children = live->children; + __entry->parents_delta = parents_delta; + __entry->backrefs_delta = backrefs_delta; + __entry->children_delta = children_delta; + ), + TP_printk("dev %d:%d ino 0x%llx parents %d:%u backrefs %d:%u children %d:%u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->parents_delta, + __entry->parents, + __entry->backrefs_delta, + __entry->backrefs, + __entry->children_delta, + __entry->children) +); + +DECLARE_EVENT_CLASS(xchk_nlinks_diff_class, + TP_PROTO(struct xfs_mount *mp, struct xfs_inode *ip, + const struct xchk_nlink *live), + TP_ARGS(mp, ip, live), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(uint8_t, ftype) + __field(xfs_nlink_t, nlink) + __field(xfs_nlink_t, parents) + __field(xfs_nlink_t, backrefs) + __field(xfs_nlink_t, children) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->ino = I_INO(ip); + __entry->ftype = xfs_mode_to_ftype(VFS_I(ip)->i_mode); + __entry->nlink = VFS_I(ip)->i_nlink; + __entry->parents = live->parents; + __entry->backrefs = live->backrefs; + __entry->children = live->children; + ), + TP_printk("dev %d:%d ino 0x%llx ftype %s nlink %u parents %u backrefs %u children %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __print_symbolic(__entry->ftype, XFS_DIR3_FTYPE_STR), + __entry->nlink, + __entry->parents, + __entry->backrefs, + __entry->children) +); +#define DEFINE_SCRUB_NLINKS_DIFF_EVENT(name) \ +DEFINE_EVENT(xchk_nlinks_diff_class, name, \ + TP_PROTO(struct xfs_mount *mp, struct xfs_inode *ip, \ + const struct xchk_nlink *live), \ + TP_ARGS(mp, ip, live)) +DEFINE_SCRUB_NLINKS_DIFF_EVENT(xchk_nlinks_compare_inode); + +DECLARE_EVENT_CLASS(xchk_pptr_class, + TP_PROTO(struct xfs_inode *ip, const struct xfs_name *name, + xfs_ino_t far_ino), + TP_ARGS(ip, name, far_ino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(unsigned int, namelen) + __dynamic_array(char, name, name->len) + __field(xfs_ino_t, far_ino) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = I_INO(ip); + __entry->namelen = name->len; + memcpy(__get_str(name), name, name->len); + __entry->far_ino = far_ino; + ), + TP_printk("dev %d:%d ino 0x%llx name '%.*s' far_ino 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->namelen, + __get_str(name), + __entry->far_ino) +) +#define DEFINE_XCHK_PPTR_EVENT(name) \ +DEFINE_EVENT(xchk_pptr_class, name, \ + TP_PROTO(struct xfs_inode *ip, const struct xfs_name *name, \ + xfs_ino_t far_ino), \ + TP_ARGS(ip, name, far_ino)) +DEFINE_XCHK_PPTR_EVENT(xchk_dir_defer); +DEFINE_XCHK_PPTR_EVENT(xchk_dir_slowpath); +DEFINE_XCHK_PPTR_EVENT(xchk_dir_ultraslowpath); +DEFINE_XCHK_PPTR_EVENT(xchk_parent_defer); +DEFINE_XCHK_PPTR_EVENT(xchk_parent_slowpath); +DEFINE_XCHK_PPTR_EVENT(xchk_parent_ultraslowpath); + +DECLARE_EVENT_CLASS(xchk_dirtree_class, + TP_PROTO(struct xfs_scrub *sc, struct xfs_inode *ip, + unsigned int path_nr, const struct xfs_name *name, + const struct xfs_parent_rec *pptr), + TP_ARGS(sc, ip, path_nr, name, pptr), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, path_nr) + __field(xfs_ino_t, child_ino) + __field(unsigned int, child_gen) + __field(xfs_ino_t, parent_ino) + __field(unsigned int, parent_gen) + __field(unsigned int, namelen) + __dynamic_array(char, name, name->len) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->path_nr = path_nr; + __entry->child_ino = I_INO(ip); + __entry->child_gen = VFS_I(ip)->i_generation; + __entry->parent_ino = be64_to_cpu(pptr->p_ino); + __entry->parent_gen = be32_to_cpu(pptr->p_gen); + __entry->namelen = name->len; + memcpy(__get_str(name), name->name, name->len); + ), + TP_printk("dev %d:%d path %u child_ino 0x%llx child_gen 0x%x parent_ino 0x%llx parent_gen 0x%x name '%.*s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->path_nr, + __entry->child_ino, + __entry->child_gen, + __entry->parent_ino, + __entry->parent_gen, + __entry->namelen, + __get_str(name)) +); +#define DEFINE_XCHK_DIRTREE_EVENT(name) \ +DEFINE_EVENT(xchk_dirtree_class, name, \ + TP_PROTO(struct xfs_scrub *sc, struct xfs_inode *ip, \ + unsigned int path_nr, const struct xfs_name *name, \ + const struct xfs_parent_rec *pptr), \ + TP_ARGS(sc, ip, path_nr, name, pptr)) +DEFINE_XCHK_DIRTREE_EVENT(xchk_dirtree_create_path); +DEFINE_XCHK_DIRTREE_EVENT(xchk_dirpath_walk_upwards); + +DECLARE_EVENT_CLASS(xchk_dirpath_class, + TP_PROTO(struct xfs_scrub *sc, struct xfs_inode *ip, + unsigned int path_nr, unsigned int step_nr, + const struct xfs_name *name, + const struct xfs_parent_rec *pptr), + TP_ARGS(sc, ip, path_nr, step_nr, name, pptr), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, path_nr) + __field(unsigned int, step_nr) + __field(xfs_ino_t, child_ino) + __field(unsigned int, child_gen) + __field(xfs_ino_t, parent_ino) + __field(unsigned int, parent_gen) + __field(unsigned int, namelen) + __dynamic_array(char, name, name->len) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->path_nr = path_nr; + __entry->step_nr = step_nr; + __entry->child_ino = I_INO(ip); + __entry->child_gen = VFS_I(ip)->i_generation; + __entry->parent_ino = be64_to_cpu(pptr->p_ino); + __entry->parent_gen = be32_to_cpu(pptr->p_gen); + __entry->namelen = name->len; + memcpy(__get_str(name), name->name, name->len); + ), + TP_printk("dev %d:%d path %u step %u child_ino 0x%llx child_gen 0x%x parent_ino 0x%llx parent_gen 0x%x name '%.*s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->path_nr, + __entry->step_nr, + __entry->child_ino, + __entry->child_gen, + __entry->parent_ino, + __entry->parent_gen, + __entry->namelen, + __get_str(name)) +); +#define DEFINE_XCHK_DIRPATH_EVENT(name) \ +DEFINE_EVENT(xchk_dirpath_class, name, \ + TP_PROTO(struct xfs_scrub *sc, struct xfs_inode *ip, \ + unsigned int path_nr, unsigned int step_nr, \ + const struct xfs_name *name, \ + const struct xfs_parent_rec *pptr), \ + TP_ARGS(sc, ip, path_nr, step_nr, name, pptr)) +DEFINE_XCHK_DIRPATH_EVENT(xchk_dirpath_disappeared); +DEFINE_XCHK_DIRPATH_EVENT(xchk_dirpath_badgen); +DEFINE_XCHK_DIRPATH_EVENT(xchk_dirpath_nondir_parent); +DEFINE_XCHK_DIRPATH_EVENT(xchk_dirpath_unlinked_parent); +DEFINE_XCHK_DIRPATH_EVENT(xchk_dirpath_found_next_step); +DEFINE_XCHK_DIRPATH_EVENT(xchk_dirpath_crosses_tree); + +TRACE_DEFINE_ENUM(XCHK_DIRPATH_SCANNING); +TRACE_DEFINE_ENUM(XCHK_DIRPATH_DELETE); +TRACE_DEFINE_ENUM(XCHK_DIRPATH_CORRUPT); +TRACE_DEFINE_ENUM(XCHK_DIRPATH_LOOP); +TRACE_DEFINE_ENUM(XCHK_DIRPATH_STALE); +TRACE_DEFINE_ENUM(XCHK_DIRPATH_OK); +TRACE_DEFINE_ENUM(XREP_DIRPATH_DELETING); +TRACE_DEFINE_ENUM(XREP_DIRPATH_DELETED); +TRACE_DEFINE_ENUM(XREP_DIRPATH_ADOPTING); +TRACE_DEFINE_ENUM(XREP_DIRPATH_ADOPTED); + +#define XCHK_DIRPATH_OUTCOME_STRINGS \ + { XCHK_DIRPATH_SCANNING, "scanning" }, \ + { XCHK_DIRPATH_DELETE, "delete" }, \ + { XCHK_DIRPATH_CORRUPT, "corrupt" }, \ + { XCHK_DIRPATH_LOOP, "loop" }, \ + { XCHK_DIRPATH_STALE, "stale" }, \ + { XCHK_DIRPATH_OK, "ok" }, \ + { XREP_DIRPATH_DELETING, "deleting" }, \ + { XREP_DIRPATH_DELETED, "deleted" }, \ + { XREP_DIRPATH_ADOPTING, "adopting" }, \ + { XREP_DIRPATH_ADOPTED, "adopted" } + +DECLARE_EVENT_CLASS(xchk_dirpath_outcome_class, + TP_PROTO(struct xfs_scrub *sc, unsigned long long path_nr, + unsigned int nr_steps, \ + unsigned int outcome), + TP_ARGS(sc, path_nr, nr_steps, outcome), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned long long, path_nr) + __field(unsigned int, nr_steps) + __field(unsigned int, outcome) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->path_nr = path_nr; + __entry->nr_steps = nr_steps; + __entry->outcome = outcome; + ), + TP_printk("dev %d:%d path %llu steps %u outcome %s", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->path_nr, + __entry->nr_steps, + __print_symbolic(__entry->outcome, XCHK_DIRPATH_OUTCOME_STRINGS)) +); +#define DEFINE_XCHK_DIRPATH_OUTCOME_EVENT(name) \ +DEFINE_EVENT(xchk_dirpath_outcome_class, name, \ + TP_PROTO(struct xfs_scrub *sc, unsigned long long path_nr, \ + unsigned int nr_steps, \ + unsigned int outcome), \ + TP_ARGS(sc, path_nr, nr_steps, outcome)) +DEFINE_XCHK_DIRPATH_OUTCOME_EVENT(xchk_dirpath_set_outcome); +DEFINE_XCHK_DIRPATH_OUTCOME_EVENT(xchk_dirpath_evaluate_path); + +DECLARE_EVENT_CLASS(xchk_dirtree_evaluate_class, + TP_PROTO(const struct xchk_dirtree *dl, + const struct xchk_dirtree_outcomes *oc), + TP_ARGS(dl, oc), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_ino_t, rootino) + __field(unsigned int, nr_paths) + __field(unsigned int, bad) + __field(unsigned int, suspect) + __field(unsigned int, good) + __field(bool, needs_adoption) + ), + TP_fast_assign( + __entry->dev = dl->sc->mp->m_super->s_dev; + __entry->ino = I_INO(dl->sc->ip); + __entry->rootino = dl->root_ino; + __entry->nr_paths = dl->nr_paths; + __entry->bad = oc->bad; + __entry->suspect = oc->suspect; + __entry->good = oc->good; + __entry->needs_adoption = oc->needs_adoption ? 1 : 0; + ), + TP_printk("dev %d:%d ino 0x%llx rootino 0x%llx nr_paths %u bad %u suspect %u good %u adopt? %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->rootino, + __entry->nr_paths, + __entry->bad, + __entry->suspect, + __entry->good, + __entry->needs_adoption) +); +#define DEFINE_XCHK_DIRTREE_EVALUATE_EVENT(name) \ +DEFINE_EVENT(xchk_dirtree_evaluate_class, name, \ + TP_PROTO(const struct xchk_dirtree *dl, \ + const struct xchk_dirtree_outcomes *oc), \ + TP_ARGS(dl, oc)) +DEFINE_XCHK_DIRTREE_EVALUATE_EVENT(xchk_dirtree_evaluate); + +TRACE_EVENT(xchk_dirpath_changed, + TP_PROTO(struct xfs_scrub *sc, unsigned int path_nr, + unsigned int step_nr, const struct xfs_inode *dp, + const struct xfs_inode *ip, const struct xfs_name *xname), + TP_ARGS(sc, path_nr, step_nr, dp, ip, xname), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, path_nr) + __field(unsigned int, step_nr) + __field(xfs_ino_t, child_ino) + __field(xfs_ino_t, parent_ino) + __field(unsigned int, namelen) + __dynamic_array(char, name, xname->len) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->path_nr = path_nr; + __entry->step_nr = step_nr; + __entry->child_ino = I_INO(ip); + __entry->parent_ino = I_INO(dp); + __entry->namelen = xname->len; + memcpy(__get_str(name), xname->name, xname->len); + ), + TP_printk("dev %d:%d path %u step %u child_ino 0x%llx parent_ino 0x%llx name '%.*s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->path_nr, + __entry->step_nr, + __entry->child_ino, + __entry->parent_ino, + __entry->namelen, + __get_str(name)) +); + +TRACE_EVENT(xchk_dirtree_live_update, + TP_PROTO(struct xfs_scrub *sc, const struct xfs_inode *dp, + int action, const struct xfs_inode *ip, int delta, + const struct xfs_name *xname), + TP_ARGS(sc, dp, action, ip, delta, xname), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, parent_ino) + __field(int, action) + __field(xfs_ino_t, child_ino) + __field(int, delta) + __field(unsigned int, namelen) + __dynamic_array(char, name, xname->len) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->parent_ino = I_INO(dp); + __entry->action = action; + __entry->child_ino = I_INO(ip); + __entry->delta = delta; + __entry->namelen = xname->len; + memcpy(__get_str(name), xname->name, xname->len); + ), + TP_printk("dev %d:%d parent_ino 0x%llx child_ino 0x%llx nlink_delta %d name '%.*s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->parent_ino, + __entry->child_ino, + __entry->delta, + __entry->namelen, + __get_str(name)) +); + +DECLARE_EVENT_CLASS(xchk_metapath_class, + TP_PROTO(struct xfs_scrub *sc, const char *path, + struct xfs_inode *dp, xfs_ino_t ino), + TP_ARGS(sc, path, dp, ino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, scrub_ino) + __field(xfs_ino_t, parent_ino) + __field(xfs_ino_t, ino) + __string(name, path) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->scrub_ino = sc->ip ? I_INO(sc->ip) : NULLFSINO; + __entry->parent_ino = dp ? I_INO(dp) : NULLFSINO; + __entry->ino = ino; + __assign_str(name); + ), + TP_printk("dev %d:%d ino 0x%llx parent_ino 0x%llx name '%s' ino 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->scrub_ino, + __entry->parent_ino, + __get_str(name), + __entry->ino) +); +#define DEFINE_XCHK_METAPATH_EVENT(name) \ +DEFINE_EVENT(xchk_metapath_class, name, \ + TP_PROTO(struct xfs_scrub *sc, const char *path, \ + struct xfs_inode *dp, xfs_ino_t ino), \ + TP_ARGS(sc, path, dp, ino)) +DEFINE_XCHK_METAPATH_EVENT(xchk_metapath_lookup); + +/* repair tracepoints */ +#if IS_ENABLED(CONFIG_XFS_ONLINE_REPAIR) + +DECLARE_EVENT_CLASS(xrep_extent_class, + TP_PROTO(const struct xfs_group *xg, xfs_agblock_t agbno, + xfs_extlen_t len), + TP_ARGS(xg, agbno, len), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(enum xfs_group_type, type) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, agbno) + __field(xfs_extlen_t, len) + ), + TP_fast_assign( + __entry->dev = xg->xg_mount->m_super->s_dev; + __entry->type = xg->xg_type; + __entry->agno = xg->xg_gno; + __entry->agbno = agbno; + __entry->len = len; + ), + TP_printk("dev %d:%d %sno 0x%x %sbno 0x%x fsbcount 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agno, + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agbno, + __entry->len) +); +#define DEFINE_REPAIR_EXTENT_EVENT(name) \ +DEFINE_EVENT(xrep_extent_class, name, \ + TP_PROTO(const struct xfs_group *xg, xfs_agblock_t agbno, \ + xfs_extlen_t len), \ + TP_ARGS(xg, agbno, len)) +DEFINE_REPAIR_EXTENT_EVENT(xreap_dispose_unmap_extent); +DEFINE_REPAIR_EXTENT_EVENT(xreap_dispose_free_extent); +DEFINE_REPAIR_EXTENT_EVENT(xreap_agextent_binval); +DEFINE_REPAIR_EXTENT_EVENT(xreap_bmapi_binval); +DEFINE_REPAIR_EXTENT_EVENT(xrep_agfl_insert); + +DECLARE_EVENT_CLASS(xrep_reap_limits_class, + TP_PROTO(const struct xfs_trans *tp, unsigned int per_binval, + unsigned int max_binval, unsigned int step_size, + unsigned int per_intent, + unsigned int max_deferred), + TP_ARGS(tp, per_binval, max_binval, step_size, per_intent, max_deferred), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, log_res) + __field(unsigned int, per_binval) + __field(unsigned int, max_binval) + __field(unsigned int, step_size) + __field(unsigned int, per_intent) + __field(unsigned int, max_deferred) + ), + TP_fast_assign( + __entry->dev = tp->t_mountp->m_super->s_dev; + __entry->log_res = tp->t_log_res; + __entry->per_binval = per_binval; + __entry->max_binval = max_binval; + __entry->step_size = step_size; + __entry->per_intent = per_intent; + __entry->max_deferred = max_deferred; + ), + TP_printk("dev %d:%d logres %u per_binval %u max_binval %u step_size %u per_intent %u max_deferred %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->log_res, + __entry->per_binval, + __entry->max_binval, + __entry->step_size, + __entry->per_intent, + __entry->max_deferred) +); +#define DEFINE_REPAIR_REAP_LIMITS_EVENT(name) \ +DEFINE_EVENT(xrep_reap_limits_class, name, \ + TP_PROTO(const struct xfs_trans *tp, unsigned int per_binval, \ + unsigned int max_binval, unsigned int step_size, \ + unsigned int per_intent, \ + unsigned int max_deferred), \ + TP_ARGS(tp, per_binval, max_binval, step_size, per_intent, max_deferred)) +DEFINE_REPAIR_REAP_LIMITS_EVENT(xreap_agextent_limits); +DEFINE_REPAIR_REAP_LIMITS_EVENT(xreap_agcow_limits); +DEFINE_REPAIR_REAP_LIMITS_EVENT(xreap_rgcow_limits); +DEFINE_REPAIR_REAP_LIMITS_EVENT(xreap_bmapi_limits); + +DECLARE_EVENT_CLASS(xrep_reap_find_class, + TP_PROTO(const struct xfs_group *xg, xfs_agblock_t agbno, + xfs_extlen_t len, bool crosslinked), + TP_ARGS(xg, agbno, len, crosslinked), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(enum xfs_group_type, type) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, agbno) + __field(xfs_extlen_t, len) + __field(bool, crosslinked) + ), + TP_fast_assign( + __entry->dev = xg->xg_mount->m_super->s_dev; + __entry->type = xg->xg_type; + __entry->agno = xg->xg_gno; + __entry->agbno = agbno; + __entry->len = len; + __entry->crosslinked = crosslinked; + ), + TP_printk("dev %d:%d %sno 0x%x %sbno 0x%x fsbcount 0x%x crosslinked %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agno, + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agbno, + __entry->len, + __entry->crosslinked ? 1 : 0) +); +#define DEFINE_REPAIR_REAP_FIND_EVENT(name) \ +DEFINE_EVENT(xrep_reap_find_class, name, \ + TP_PROTO(const struct xfs_group *xg, xfs_agblock_t agbno, \ + xfs_extlen_t len, bool crosslinked), \ + TP_ARGS(xg, agbno, len, crosslinked)) +DEFINE_REPAIR_REAP_FIND_EVENT(xreap_agextent_select); +DEFINE_REPAIR_REAP_FIND_EVENT(xreap_bmapi_select); + +TRACE_EVENT(xrep_ibt_walk_rmap, + TP_PROTO(const struct xfs_perag *pag, const struct xfs_rmap_irec *rec), + TP_ARGS(pag, rec), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, agbno) + __field(xfs_extlen_t, len) + __field(uint64_t, owner) + __field(uint64_t, offset) + __field(unsigned int, flags) + ), + TP_fast_assign( + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->agno = pag_agno(pag); + __entry->agbno = rec->rm_startblock; + __entry->len = rec->rm_blockcount; + __entry->owner = rec->rm_owner; + __entry->offset = rec->rm_offset; + __entry->flags = rec->rm_flags; + ), + TP_printk("dev %d:%d agno 0x%x agbno 0x%x fsbcount 0x%x owner 0x%llx fileoff 0x%llx flags 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->agbno, + __entry->len, + __entry->owner, + __entry->offset, + __entry->flags) +); + +TRACE_EVENT(xrep_abt_found, + TP_PROTO(const struct xfs_perag *pag, + const struct xfs_alloc_rec_incore *rec), + TP_ARGS(pag, rec), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, startblock) + __field(xfs_extlen_t, blockcount) + ), + TP_fast_assign( + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->agno = pag_agno(pag); + __entry->startblock = rec->ar_startblock; + __entry->blockcount = rec->ar_blockcount; + ), + TP_printk("dev %d:%d agno 0x%x agbno 0x%x fsbcount 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->startblock, + __entry->blockcount) +) + +TRACE_EVENT(xrep_ibt_found, + TP_PROTO(const struct xfs_perag *pag, + const struct xfs_inobt_rec_incore *rec), + TP_ARGS(pag, rec), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agino_t, startino) + __field(uint16_t, holemask) + __field(uint8_t, count) + __field(uint8_t, freecount) + __field(uint64_t, freemask) + ), + TP_fast_assign( + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->agno = pag_agno(pag); + __entry->startino = rec->ir_startino; + __entry->holemask = rec->ir_holemask; + __entry->count = rec->ir_count; + __entry->freecount = rec->ir_freecount; + __entry->freemask = rec->ir_free; + ), + TP_printk("dev %d:%d agno 0x%x agino 0x%x holemask 0x%x count 0x%x freecount 0x%x freemask 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->startino, + __entry->holemask, + __entry->count, + __entry->freecount, + __entry->freemask) +) + +TRACE_EVENT(xrep_refc_found, + TP_PROTO(const struct xfs_group *xg, + const struct xfs_refcount_irec *rec), + TP_ARGS(xg, rec), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(enum xfs_refc_domain, domain) + __field(enum xfs_group_type, type) + __field(xfs_agblock_t, startblock) + __field(xfs_extlen_t, blockcount) + __field(xfs_nlink_t, refcount) + ), + TP_fast_assign( + __entry->dev = xg->xg_mount->m_super->s_dev; + __entry->agno = xg->xg_gno; + __entry->type = xg->xg_type; + __entry->domain = rec->rc_domain; + __entry->startblock = rec->rc_startblock; + __entry->blockcount = rec->rc_blockcount; + __entry->refcount = rec->rc_refcount; + ), + TP_printk("dev %d:%d %sno 0x%x dom %s %sbno 0x%x fsbcount 0x%x refcount %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agno, + __print_symbolic(__entry->domain, XFS_REFC_DOMAIN_STRINGS), + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->startblock, + __entry->blockcount, + __entry->refcount) +) + +TRACE_EVENT(xrep_bmap_found, + TP_PROTO(struct xfs_inode *ip, int whichfork, + struct xfs_bmbt_irec *irec), + TP_ARGS(ip, whichfork, irec), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(int, whichfork) + __field(xfs_fileoff_t, lblk) + __field(xfs_filblks_t, len) + __field(xfs_fsblock_t, pblk) + __field(int, state) + ), + TP_fast_assign( + __entry->dev = VFS_I(ip)->i_sb->s_dev; + __entry->ino = I_INO(ip); + __entry->whichfork = whichfork; + __entry->lblk = irec->br_startoff; + __entry->len = irec->br_blockcount; + __entry->pblk = irec->br_startblock; + __entry->state = irec->br_state; + ), + TP_printk("dev %d:%d ino 0x%llx whichfork %s fileoff 0x%llx fsbcount 0x%llx startblock 0x%llx state %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __print_symbolic(__entry->whichfork, XFS_WHICHFORK_STRINGS), + __entry->lblk, + __entry->len, + __entry->pblk, + __entry->state) +); + +TRACE_EVENT(xrep_rmap_found, + TP_PROTO(const struct xfs_perag *pag, const struct xfs_rmap_irec *rec), + TP_ARGS(pag, rec), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, agbno) + __field(xfs_extlen_t, len) + __field(uint64_t, owner) + __field(uint64_t, offset) + __field(unsigned int, flags) + ), + TP_fast_assign( + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->agno = pag_agno(pag); + __entry->agbno = rec->rm_startblock; + __entry->len = rec->rm_blockcount; + __entry->owner = rec->rm_owner; + __entry->offset = rec->rm_offset; + __entry->flags = rec->rm_flags; + ), + TP_printk("dev %d:%d agno 0x%x agbno 0x%x fsbcount 0x%x owner 0x%llx fileoff 0x%llx flags 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->agbno, + __entry->len, + __entry->owner, + __entry->offset, + __entry->flags) +); + +TRACE_EVENT(xrep_findroot_block, + TP_PROTO(const struct xfs_perag *pag, xfs_agblock_t agbno, + uint32_t magic, uint16_t level), + TP_ARGS(pag, agbno, magic, level), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, agbno) + __field(uint32_t, magic) + __field(uint16_t, level) + ), + TP_fast_assign( + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->agno = pag_agno(pag); + __entry->agbno = agbno; + __entry->magic = magic; + __entry->level = level; + ), + TP_printk("dev %d:%d agno 0x%x agbno 0x%x magic 0x%x level %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->agbno, + __entry->magic, + __entry->level) +) +TRACE_EVENT(xrep_calc_ag_resblks, + TP_PROTO(const struct xfs_perag *pag, xfs_agino_t icount, + xfs_agblock_t aglen, xfs_agblock_t freelen, + xfs_agblock_t usedlen), + TP_ARGS(pag, icount, aglen, freelen, usedlen), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agino_t, icount) + __field(xfs_agblock_t, aglen) + __field(xfs_agblock_t, freelen) + __field(xfs_agblock_t, usedlen) + ), + TP_fast_assign( + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->agno = pag_agno(pag); + __entry->icount = icount; + __entry->aglen = aglen; + __entry->freelen = freelen; + __entry->usedlen = usedlen; + ), + TP_printk("dev %d:%d agno 0x%x icount %u aglen %u freelen %u usedlen %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->icount, + __entry->aglen, + __entry->freelen, + __entry->usedlen) +) +TRACE_EVENT(xrep_calc_ag_resblks_btsize, + TP_PROTO(const struct xfs_perag *pag, xfs_agblock_t bnobt_sz, + xfs_agblock_t inobt_sz, xfs_agblock_t rmapbt_sz, + xfs_agblock_t refcbt_sz), + TP_ARGS(pag, bnobt_sz, inobt_sz, rmapbt_sz, refcbt_sz), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, bnobt_sz) + __field(xfs_agblock_t, inobt_sz) + __field(xfs_agblock_t, rmapbt_sz) + __field(xfs_agblock_t, refcbt_sz) + ), + TP_fast_assign( + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->agno = pag_agno(pag); + __entry->bnobt_sz = bnobt_sz; + __entry->inobt_sz = inobt_sz; + __entry->rmapbt_sz = rmapbt_sz; + __entry->refcbt_sz = refcbt_sz; + ), + TP_printk("dev %d:%d agno 0x%x bnobt %u inobt %u rmapbt %u refcountbt %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->bnobt_sz, + __entry->inobt_sz, + __entry->rmapbt_sz, + __entry->refcbt_sz) +) + +#ifdef CONFIG_XFS_RT +TRACE_EVENT(xrep_calc_rtgroup_resblks_btsize, + TP_PROTO(struct xfs_mount *mp, xfs_rgnumber_t rgno, + xfs_rgblock_t usedlen, xfs_rgblock_t rmapbt_sz), + TP_ARGS(mp, rgno, usedlen, rmapbt_sz), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_rgnumber_t, rgno) + __field(xfs_rgblock_t, usedlen) + __field(xfs_rgblock_t, rmapbt_sz) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->rgno = rgno; + __entry->usedlen = usedlen; + __entry->rmapbt_sz = rmapbt_sz; + ), + TP_printk("dev %d:%d rgno 0x%x usedlen %u rmapbt %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->rgno, + __entry->usedlen, + __entry->rmapbt_sz) +); +#endif /* CONFIG_XFS_RT */ + +TRACE_EVENT(xrep_reset_counters, + TP_PROTO(struct xfs_mount *mp, struct xchk_fscounters *fsc), + TP_ARGS(mp, fsc), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(uint64_t, icount) + __field(uint64_t, ifree) + __field(uint64_t, fdblocks) + __field(uint64_t, frextents) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->icount = fsc->icount; + __entry->ifree = fsc->ifree; + __entry->fdblocks = fsc->fdblocks; + __entry->frextents = fsc->frextents; + ), + TP_printk("dev %d:%d icount %llu ifree %llu fdblocks %llu frextents %llu", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->icount, + __entry->ifree, + __entry->fdblocks, + __entry->frextents) +) + +DECLARE_EVENT_CLASS(xrep_newbt_extent_class, + TP_PROTO(const struct xfs_perag *pag, xfs_agblock_t agbno, + xfs_extlen_t len, int64_t owner), + TP_ARGS(pag, agbno, len, owner), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, agbno) + __field(xfs_extlen_t, len) + __field(int64_t, owner) + ), + TP_fast_assign( + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->agno = pag_agno(pag); + __entry->agbno = agbno; + __entry->len = len; + __entry->owner = owner; + ), + TP_printk("dev %d:%d agno 0x%x agbno 0x%x fsbcount 0x%x owner 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->agbno, + __entry->len, + __entry->owner) +); +#define DEFINE_NEWBT_EXTENT_EVENT(name) \ +DEFINE_EVENT(xrep_newbt_extent_class, name, \ + TP_PROTO(const struct xfs_perag *pag, xfs_agblock_t agbno, \ + xfs_extlen_t len, int64_t owner), \ + TP_ARGS(pag, agbno, len, owner)) +DEFINE_NEWBT_EXTENT_EVENT(xrep_newbt_alloc_ag_blocks); +DEFINE_NEWBT_EXTENT_EVENT(xrep_newbt_alloc_file_blocks); +DEFINE_NEWBT_EXTENT_EVENT(xrep_newbt_free_blocks); +DEFINE_NEWBT_EXTENT_EVENT(xrep_newbt_claim_block); + +DECLARE_EVENT_CLASS(xrep_dinode_class, + TP_PROTO(struct xfs_scrub *sc, struct xfs_dinode *dip), + TP_ARGS(sc, dip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(uint16_t, mode) + __field(uint8_t, version) + __field(uint8_t, format) + __field(uint32_t, uid) + __field(uint32_t, gid) + __field(uint64_t, size) + __field(uint64_t, nblocks) + __field(uint32_t, extsize) + __field(uint32_t, nextents) + __field(uint16_t, anextents) + __field(uint8_t, forkoff) + __field(uint8_t, aformat) + __field(uint16_t, flags) + __field(uint32_t, gen) + __field(uint64_t, flags2) + __field(uint32_t, cowextsize) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->ino = sc->sm->sm_ino; + __entry->mode = be16_to_cpu(dip->di_mode); + __entry->version = dip->di_version; + __entry->format = dip->di_format; + __entry->uid = be32_to_cpu(dip->di_uid); + __entry->gid = be32_to_cpu(dip->di_gid); + __entry->size = be64_to_cpu(dip->di_size); + __entry->nblocks = be64_to_cpu(dip->di_nblocks); + __entry->extsize = be32_to_cpu(dip->di_extsize); + __entry->nextents = be32_to_cpu(dip->di_nextents); + __entry->anextents = be16_to_cpu(dip->di_anextents); + __entry->forkoff = dip->di_forkoff; + __entry->aformat = dip->di_aformat; + __entry->flags = be16_to_cpu(dip->di_flags); + __entry->gen = be32_to_cpu(dip->di_gen); + __entry->flags2 = be64_to_cpu(dip->di_flags2); + __entry->cowextsize = be32_to_cpu(dip->di_cowextsize); + ), + TP_printk("dev %d:%d ino 0x%llx mode 0x%x version %u format %u uid %u gid %u disize 0x%llx nblocks 0x%llx extsize %u nextents %u anextents %u forkoff 0x%x aformat %u flags 0x%x gen 0x%x flags2 0x%llx cowextsize %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->mode, + __entry->version, + __entry->format, + __entry->uid, + __entry->gid, + __entry->size, + __entry->nblocks, + __entry->extsize, + __entry->nextents, + __entry->anextents, + __entry->forkoff, + __entry->aformat, + __entry->flags, + __entry->gen, + __entry->flags2, + __entry->cowextsize) +) + +#define DEFINE_REPAIR_DINODE_EVENT(name) \ +DEFINE_EVENT(xrep_dinode_class, name, \ + TP_PROTO(struct xfs_scrub *sc, struct xfs_dinode *dip), \ + TP_ARGS(sc, dip)) +DEFINE_REPAIR_DINODE_EVENT(xrep_dinode_header); +DEFINE_REPAIR_DINODE_EVENT(xrep_dinode_mode); +DEFINE_REPAIR_DINODE_EVENT(xrep_dinode_flags); +DEFINE_REPAIR_DINODE_EVENT(xrep_dinode_size); +DEFINE_REPAIR_DINODE_EVENT(xrep_dinode_extsize_hints); +DEFINE_REPAIR_DINODE_EVENT(xrep_dinode_zap_symlink); +DEFINE_REPAIR_DINODE_EVENT(xrep_dinode_zap_dir); +DEFINE_REPAIR_DINODE_EVENT(xrep_dinode_fixed); +DEFINE_REPAIR_DINODE_EVENT(xrep_dinode_zap_forks); +DEFINE_REPAIR_DINODE_EVENT(xrep_dinode_zap_dfork); +DEFINE_REPAIR_DINODE_EVENT(xrep_dinode_zap_afork); +DEFINE_REPAIR_DINODE_EVENT(xrep_dinode_ensure_forkoff); + +DECLARE_EVENT_CLASS(xrep_inode_class, + TP_PROTO(struct xfs_scrub *sc), + TP_ARGS(sc), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_fsize_t, size) + __field(xfs_rfsblock_t, nblocks) + __field(uint16_t, flags) + __field(uint64_t, flags2) + __field(uint32_t, nextents) + __field(uint8_t, format) + __field(uint32_t, anextents) + __field(uint8_t, aformat) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->ino = sc->sm->sm_ino; + __entry->size = sc->ip->i_disk_size; + __entry->nblocks = sc->ip->i_nblocks; + __entry->flags = sc->ip->i_diflags; + __entry->flags2 = sc->ip->i_diflags2; + __entry->nextents = sc->ip->i_df.if_nextents; + __entry->format = sc->ip->i_df.if_format; + __entry->anextents = sc->ip->i_af.if_nextents; + __entry->aformat = sc->ip->i_af.if_format; + ), + TP_printk("dev %d:%d ino 0x%llx disize 0x%llx nblocks 0x%llx flags 0x%x flags2 0x%llx nextents %u format %u anextents %u aformat %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->size, + __entry->nblocks, + __entry->flags, + __entry->flags2, + __entry->nextents, + __entry->format, + __entry->anextents, + __entry->aformat) +) + +#define DEFINE_REPAIR_INODE_EVENT(name) \ +DEFINE_EVENT(xrep_inode_class, name, \ + TP_PROTO(struct xfs_scrub *sc), \ + TP_ARGS(sc)) +DEFINE_REPAIR_INODE_EVENT(xrep_inode_blockcounts); +DEFINE_REPAIR_INODE_EVENT(xrep_inode_ids); +DEFINE_REPAIR_INODE_EVENT(xrep_inode_flags); +DEFINE_REPAIR_INODE_EVENT(xrep_inode_blockdir_size); +DEFINE_REPAIR_INODE_EVENT(xrep_inode_sfdir_size); +DEFINE_REPAIR_INODE_EVENT(xrep_inode_dir_size); +DEFINE_REPAIR_INODE_EVENT(xrep_inode_fixed); + +TRACE_EVENT(xrep_dinode_count_rmaps, + TP_PROTO(struct xfs_scrub *sc, xfs_rfsblock_t data_blocks, + xfs_rfsblock_t rt_blocks, xfs_rfsblock_t attr_blocks, + xfs_extnum_t data_extents, xfs_extnum_t rt_extents, + xfs_aextnum_t attr_extents), + TP_ARGS(sc, data_blocks, rt_blocks, attr_blocks, data_extents, + rt_extents, attr_extents), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_rfsblock_t, data_blocks) + __field(xfs_rfsblock_t, rt_blocks) + __field(xfs_rfsblock_t, attr_blocks) + __field(xfs_extnum_t, data_extents) + __field(xfs_extnum_t, rt_extents) + __field(xfs_aextnum_t, attr_extents) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->ino = sc->sm->sm_ino; + __entry->data_blocks = data_blocks; + __entry->rt_blocks = rt_blocks; + __entry->attr_blocks = attr_blocks; + __entry->data_extents = data_extents; + __entry->rt_extents = rt_extents; + __entry->attr_extents = attr_extents; + ), + TP_printk("dev %d:%d ino 0x%llx dblocks 0x%llx rtblocks 0x%llx ablocks 0x%llx dextents %llu rtextents %llu aextents %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->data_blocks, + __entry->rt_blocks, + __entry->attr_blocks, + __entry->data_extents, + __entry->rt_extents, + __entry->attr_extents) +); + +TRACE_EVENT(xrep_dinode_findmode_dirent, + TP_PROTO(struct xfs_scrub *sc, struct xfs_inode *dp, + unsigned int ftype), + TP_ARGS(sc, dp, ftype), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_ino_t, parent_ino) + __field(unsigned int, ftype) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->ino = sc->sm->sm_ino; + __entry->parent_ino = I_INO(dp); + __entry->ftype = ftype; + ), + TP_printk("dev %d:%d ino 0x%llx parent_ino 0x%llx ftype '%s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->parent_ino, + __print_symbolic(__entry->ftype, XFS_DIR3_FTYPE_STR)) +); + +TRACE_EVENT(xrep_dinode_findmode_dirent_inval, + TP_PROTO(struct xfs_scrub *sc, struct xfs_inode *dp, + unsigned int ftype, unsigned int found_ftype), + TP_ARGS(sc, dp, ftype, found_ftype), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_ino_t, parent_ino) + __field(unsigned int, ftype) + __field(unsigned int, found_ftype) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->ino = sc->sm->sm_ino; + __entry->parent_ino = I_INO(dp); + __entry->ftype = ftype; + __entry->found_ftype = found_ftype; + ), + TP_printk("dev %d:%d ino 0x%llx parent_ino 0x%llx ftype '%s' found_ftype '%s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->parent_ino, + __print_symbolic(__entry->ftype, XFS_DIR3_FTYPE_STR), + __print_symbolic(__entry->found_ftype, XFS_DIR3_FTYPE_STR)) +); + +TRACE_EVENT(xrep_cow_mark_file_range, + TP_PROTO(struct xfs_inode *ip, xfs_fsblock_t startblock, + xfs_fileoff_t startoff, xfs_filblks_t blockcount), + TP_ARGS(ip, startblock, startoff, blockcount), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_fsblock_t, startblock) + __field(xfs_fileoff_t, startoff) + __field(xfs_filblks_t, blockcount) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = I_INO(ip); + __entry->startoff = startoff; + __entry->startblock = startblock; + __entry->blockcount = blockcount; + ), + TP_printk("dev %d:%d ino 0x%llx fileoff 0x%llx startblock 0x%llx fsbcount 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->startoff, + __entry->startblock, + __entry->blockcount) +); + +TRACE_EVENT(xrep_cow_free_staging, + TP_PROTO(const struct xfs_perag *pag, xfs_agblock_t agbno, + xfs_extlen_t blockcount), + TP_ARGS(pag, agbno, blockcount), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, agbno) + __field(xfs_extlen_t, blockcount) + ), + TP_fast_assign( + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->agno = pag_agno(pag); + __entry->agbno = agbno; + __entry->blockcount = blockcount; + ), + TP_printk("dev %d:%d agno 0x%x agbno 0x%x fsbcount 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->agbno, + __entry->blockcount) +); + +#ifdef CONFIG_XFS_QUOTA +DECLARE_EVENT_CLASS(xrep_dquot_class, + TP_PROTO(struct xfs_mount *mp, uint8_t type, uint32_t id), + TP_ARGS(mp, type, id), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(uint8_t, type) + __field(uint32_t, id) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->id = id; + __entry->type = type; + ), + TP_printk("dev %d:%d type %s id 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_flags(__entry->type, "|", XFS_DQTYPE_STRINGS), + __entry->id) +); + +#define DEFINE_XREP_DQUOT_EVENT(name) \ +DEFINE_EVENT(xrep_dquot_class, name, \ + TP_PROTO(struct xfs_mount *mp, uint8_t type, uint32_t id), \ + TP_ARGS(mp, type, id)) +DEFINE_XREP_DQUOT_EVENT(xrep_dquot_item); +DEFINE_XREP_DQUOT_EVENT(xrep_disk_dquot); +DEFINE_XREP_DQUOT_EVENT(xrep_dquot_item_fill_bmap_hole); +DEFINE_XREP_DQUOT_EVENT(xrep_quotacheck_dquot); +#endif /* CONFIG_XFS_QUOTA */ + +DEFINE_SCRUB_NLINKS_DIFF_EVENT(xrep_nlinks_update_inode); +DEFINE_SCRUB_NLINKS_DIFF_EVENT(xrep_nlinks_unfixable_inode); + +TRACE_EVENT(xrep_rmap_live_update, + TP_PROTO(const struct xfs_group *xg, unsigned int op, + const struct xfs_rmap_update_params *p), + TP_ARGS(xg, op, p), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(enum xfs_group_type, type) + __field(xfs_agnumber_t, agno) + __field(unsigned int, op) + __field(xfs_agblock_t, agbno) + __field(xfs_extlen_t, len) + __field(uint64_t, owner) + __field(uint64_t, offset) + __field(unsigned int, flags) + ), + TP_fast_assign( + __entry->dev = xg->xg_mount->m_super->s_dev; + __entry->type = xg->xg_type; + __entry->agno = xg->xg_gno; + __entry->op = op; + __entry->agbno = p->startblock; + __entry->len = p->blockcount; + xfs_owner_info_unpack(&p->oinfo, &__entry->owner, + &__entry->offset, &__entry->flags); + if (p->unwritten) + __entry->flags |= XFS_RMAP_UNWRITTEN; + ), + TP_printk("dev %d:%d %sno 0x%x op %d %sbno 0x%x fsbcount 0x%x owner 0x%llx fileoff 0x%llx flags 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agno, + __entry->op, + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agbno, + __entry->len, + __entry->owner, + __entry->offset, + __entry->flags) +); + +TRACE_EVENT(xrep_tempfile_create, + TP_PROTO(struct xfs_scrub *sc), + TP_ARGS(sc), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(unsigned int, type) + __field(xfs_agnumber_t, agno) + __field(xfs_ino_t, inum) + __field(unsigned int, gen) + __field(unsigned int, flags) + __field(xfs_ino_t, temp_inum) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->ino = sc->file ? file_inode(sc->file)->i_ino : 0; + __entry->type = sc->sm->sm_type; + __entry->agno = sc->sm->sm_agno; + __entry->inum = sc->sm->sm_ino; + __entry->gen = sc->sm->sm_gen; + __entry->flags = sc->sm->sm_flags; + __entry->temp_inum = I_INO(sc->tempip); + ), + TP_printk("dev %d:%d ino 0x%llx type %s inum 0x%llx gen 0x%x flags 0x%x temp_inum 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __print_symbolic(__entry->type, XFS_SCRUB_TYPE_STRINGS), + __entry->inum, + __entry->gen, + __entry->flags, + __entry->temp_inum) +); + +DECLARE_EVENT_CLASS(xrep_tempfile_class, + TP_PROTO(struct xfs_scrub *sc, int whichfork, + struct xfs_bmbt_irec *irec), + TP_ARGS(sc, whichfork, irec), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(int, whichfork) + __field(xfs_fileoff_t, lblk) + __field(xfs_filblks_t, len) + __field(xfs_fsblock_t, pblk) + __field(int, state) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->ino = I_INO(sc->tempip); + __entry->whichfork = whichfork; + __entry->lblk = irec->br_startoff; + __entry->len = irec->br_blockcount; + __entry->pblk = irec->br_startblock; + __entry->state = irec->br_state; + ), + TP_printk("dev %d:%d ino 0x%llx whichfork %s fileoff 0x%llx fsbcount 0x%llx startblock 0x%llx state %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __print_symbolic(__entry->whichfork, XFS_WHICHFORK_STRINGS), + __entry->lblk, + __entry->len, + __entry->pblk, + __entry->state) +); +#define DEFINE_XREP_TEMPFILE_EVENT(name) \ +DEFINE_EVENT(xrep_tempfile_class, name, \ + TP_PROTO(struct xfs_scrub *sc, int whichfork, \ + struct xfs_bmbt_irec *irec), \ + TP_ARGS(sc, whichfork, irec)) +DEFINE_XREP_TEMPFILE_EVENT(xrep_tempfile_prealloc); +DEFINE_XREP_TEMPFILE_EVENT(xrep_tempfile_copyin); + +TRACE_EVENT(xreap_ifork_extent, + TP_PROTO(struct xfs_scrub *sc, struct xfs_inode *ip, int whichfork, + const struct xfs_bmbt_irec *irec), + TP_ARGS(sc, ip, whichfork, irec), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(int, whichfork) + __field(xfs_fileoff_t, fileoff) + __field(xfs_filblks_t, len) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, agbno) + __field(int, state) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->ino = I_INO(ip); + __entry->whichfork = whichfork; + __entry->fileoff = irec->br_startoff; + __entry->len = irec->br_blockcount; + __entry->agno = XFS_FSB_TO_AGNO(sc->mp, irec->br_startblock); + __entry->agbno = XFS_FSB_TO_AGBNO(sc->mp, irec->br_startblock); + __entry->state = irec->br_state; + ), + TP_printk("dev %d:%d ip 0x%llx whichfork %s agno 0x%x agbno 0x%x fileoff 0x%llx fsbcount 0x%llx state 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __print_symbolic(__entry->whichfork, XFS_WHICHFORK_STRINGS), + __entry->agno, + __entry->agbno, + __entry->fileoff, + __entry->len, + __entry->state) +); + +TRACE_EVENT(xreap_bmapi_binval_scan, + TP_PROTO(struct xfs_scrub *sc, const struct xfs_bmbt_irec *irec, + xfs_extlen_t scan_blocks), + TP_ARGS(sc, irec, scan_blocks), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_filblks_t, len) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, agbno) + __field(xfs_extlen_t, scan_blocks) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->len = irec->br_blockcount; + __entry->agno = XFS_FSB_TO_AGNO(sc->mp, irec->br_startblock); + __entry->agbno = XFS_FSB_TO_AGBNO(sc->mp, irec->br_startblock); + __entry->scan_blocks = scan_blocks; + ), + TP_printk("dev %d:%d agno 0x%x agbno 0x%x fsbcount 0x%llx scan_blocks 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->agbno, + __entry->len, + __entry->scan_blocks) +); + +TRACE_EVENT(xrep_xattr_recover_leafblock, + TP_PROTO(struct xfs_inode *ip, xfs_dablk_t dabno, uint16_t magic), + TP_ARGS(ip, dabno, magic), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_dablk_t, dabno) + __field(uint16_t, magic) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = I_INO(ip); + __entry->dabno = dabno; + __entry->magic = magic; + ), + TP_printk("dev %d:%d ino 0x%llx dablk 0x%x magic 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->dabno, + __entry->magic) +); + +DECLARE_EVENT_CLASS(xrep_xattr_salvage_class, + TP_PROTO(struct xfs_inode *ip, unsigned int flags, char *name, + unsigned int namelen, unsigned int valuelen), + TP_ARGS(ip, flags, name, namelen, valuelen), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(unsigned int, flags) + __field(unsigned int, namelen) + __dynamic_array(char, name, namelen) + __field(unsigned int, valuelen) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = I_INO(ip); + __entry->flags = flags; + __entry->namelen = namelen; + memcpy(__get_str(name), name, namelen); + __entry->valuelen = valuelen; + ), + TP_printk("dev %d:%d ino 0x%llx flags %s name '%.*s' valuelen 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __print_flags(__entry->flags, "|", XFS_ATTR_NAMESPACE_STR), + __entry->namelen, + __get_str(name), + __entry->valuelen) +); +#define DEFINE_XREP_XATTR_SALVAGE_EVENT(name) \ +DEFINE_EVENT(xrep_xattr_salvage_class, name, \ + TP_PROTO(struct xfs_inode *ip, unsigned int flags, char *name, \ + unsigned int namelen, unsigned int valuelen), \ + TP_ARGS(ip, flags, name, namelen, valuelen)) +DEFINE_XREP_XATTR_SALVAGE_EVENT(xrep_xattr_salvage_rec); +DEFINE_XREP_XATTR_SALVAGE_EVENT(xrep_xattr_insert_rec); +DEFINE_XREP_XATTR_SALVAGE_EVENT(xrep_parent_stash_xattr); +DEFINE_XREP_XATTR_SALVAGE_EVENT(xrep_parent_insert_xattr); + +DECLARE_EVENT_CLASS(xrep_pptr_salvage_class, + TP_PROTO(struct xfs_inode *ip, unsigned int flags, const void *name, + unsigned int namelen, const void *value, unsigned int valuelen), + TP_ARGS(ip, flags, name, namelen, value, valuelen), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_ino_t, parent_ino) + __field(unsigned int, parent_gen) + __field(unsigned int, namelen) + __dynamic_array(char, name, namelen) + ), + TP_fast_assign( + const struct xfs_parent_rec *rec = value; + + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = I_INO(ip); + __entry->parent_ino = be64_to_cpu(rec->p_ino); + __entry->parent_gen = be32_to_cpu(rec->p_gen); + __entry->namelen = namelen; + memcpy(__get_str(name), name, namelen); + ), + TP_printk("dev %d:%d ino 0x%llx parent_ino 0x%llx parent_gen 0x%x name '%.*s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->parent_ino, + __entry->parent_gen, + __entry->namelen, + __get_str(name)) +) +#define DEFINE_XREP_PPTR_SALVAGE_EVENT(name) \ +DEFINE_EVENT(xrep_pptr_salvage_class, name, \ + TP_PROTO(struct xfs_inode *ip, unsigned int flags, const void *name, \ + unsigned int namelen, const void *value, unsigned int valuelen), \ + TP_ARGS(ip, flags, name, namelen, value, valuelen)) +DEFINE_XREP_PPTR_SALVAGE_EVENT(xrep_xattr_salvage_pptr); +DEFINE_XREP_PPTR_SALVAGE_EVENT(xrep_xattr_insert_pptr); + +DECLARE_EVENT_CLASS(xrep_xattr_class, + TP_PROTO(struct xfs_inode *ip, struct xfs_inode *arg_ip), + TP_ARGS(ip, arg_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_ino_t, src_ino) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = I_INO(ip); + __entry->src_ino = I_INO(arg_ip); + ), + TP_printk("dev %d:%d ino 0x%llx src 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->src_ino) +) +#define DEFINE_XREP_XATTR_EVENT(name) \ +DEFINE_EVENT(xrep_xattr_class, name, \ + TP_PROTO(struct xfs_inode *ip, struct xfs_inode *arg_ip), \ + TP_ARGS(ip, arg_ip)) +DEFINE_XREP_XATTR_EVENT(xrep_xattr_rebuild_tree); +DEFINE_XREP_XATTR_EVENT(xrep_xattr_reset_fork); +DEFINE_XREP_XATTR_EVENT(xrep_xattr_full_reset); + +DECLARE_EVENT_CLASS(xrep_xattr_pptr_scan_class, + TP_PROTO(struct xfs_inode *ip, const struct xfs_inode *dp, + const struct xfs_name *name), + TP_ARGS(ip, dp, name), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_ino_t, parent_ino) + __field(unsigned int, parent_gen) + __field(unsigned int, namelen) + __dynamic_array(char, name, name->len) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = I_INO(ip); + __entry->parent_ino = I_INO(dp); + __entry->parent_gen = VFS_IC(dp)->i_generation; + __entry->namelen = name->len; + memcpy(__get_str(name), name->name, name->len); + ), + TP_printk("dev %d:%d ino 0x%llx parent_ino 0x%llx parent_gen 0x%x name '%.*s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->parent_ino, + __entry->parent_gen, + __entry->namelen, + __get_str(name)) +) +#define DEFINE_XREP_XATTR_PPTR_SCAN_EVENT(name) \ +DEFINE_EVENT(xrep_xattr_pptr_scan_class, name, \ + TP_PROTO(struct xfs_inode *ip, const struct xfs_inode *dp, \ + const struct xfs_name *name), \ + TP_ARGS(ip, dp, name)) +DEFINE_XREP_XATTR_PPTR_SCAN_EVENT(xrep_xattr_stash_parentadd); +DEFINE_XREP_XATTR_PPTR_SCAN_EVENT(xrep_xattr_stash_parentremove); + +TRACE_EVENT(xrep_dir_recover_dirblock, + TP_PROTO(struct xfs_inode *dp, xfs_dablk_t dabno, uint32_t magic, + uint32_t magic_guess), + TP_ARGS(dp, dabno, magic, magic_guess), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, dir_ino) + __field(xfs_dablk_t, dabno) + __field(uint32_t, magic) + __field(uint32_t, magic_guess) + ), + TP_fast_assign( + __entry->dev = dp->i_mount->m_super->s_dev; + __entry->dir_ino = I_INO(dp); + __entry->dabno = dabno; + __entry->magic = magic; + __entry->magic_guess = magic_guess; + ), + TP_printk("dev %d:%d dir 0x%llx dablk 0x%x magic 0x%x magic_guess 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->dir_ino, + __entry->dabno, + __entry->magic, + __entry->magic_guess) +); + +DECLARE_EVENT_CLASS(xrep_dir_class, + TP_PROTO(struct xfs_inode *dp, xfs_ino_t parent_ino), + TP_ARGS(dp, parent_ino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, dir_ino) + __field(xfs_ino_t, parent_ino) + ), + TP_fast_assign( + __entry->dev = dp->i_mount->m_super->s_dev; + __entry->dir_ino = I_INO(dp); + __entry->parent_ino = parent_ino; + ), + TP_printk("dev %d:%d dir 0x%llx parent 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->dir_ino, + __entry->parent_ino) +) +#define DEFINE_XREP_DIR_EVENT(name) \ +DEFINE_EVENT(xrep_dir_class, name, \ + TP_PROTO(struct xfs_inode *dp, xfs_ino_t parent_ino), \ + TP_ARGS(dp, parent_ino)) +DEFINE_XREP_DIR_EVENT(xrep_dir_rebuild_tree); +DEFINE_XREP_DIR_EVENT(xrep_dir_reset_fork); +DEFINE_XREP_DIR_EVENT(xrep_parent_reset_dotdot); + +DECLARE_EVENT_CLASS(xrep_dirent_class, + TP_PROTO(struct xfs_inode *dp, const struct xfs_name *name, + xfs_ino_t ino), + TP_ARGS(dp, name, ino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, dir_ino) + __field(unsigned int, namelen) + __dynamic_array(char, name, name->len) + __field(xfs_ino_t, ino) + __field(uint8_t, ftype) + ), + TP_fast_assign( + __entry->dev = dp->i_mount->m_super->s_dev; + __entry->dir_ino = I_INO(dp); + __entry->namelen = name->len; + memcpy(__get_str(name), name->name, name->len); + __entry->ino = ino; + __entry->ftype = name->type; + ), + TP_printk("dev %d:%d dir 0x%llx ftype %s name '%.*s' ino 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->dir_ino, + __print_symbolic(__entry->ftype, XFS_DIR3_FTYPE_STR), + __entry->namelen, + __get_str(name), + __entry->ino) +) +#define DEFINE_XREP_DIRENT_EVENT(name) \ +DEFINE_EVENT(xrep_dirent_class, name, \ + TP_PROTO(struct xfs_inode *dp, const struct xfs_name *name, \ + xfs_ino_t ino), \ + TP_ARGS(dp, name, ino)) +DEFINE_XREP_DIRENT_EVENT(xrep_dir_salvage_entry); +DEFINE_XREP_DIRENT_EVENT(xrep_dir_stash_createname); +DEFINE_XREP_DIRENT_EVENT(xrep_dir_replay_createname); +DEFINE_XREP_DIRENT_EVENT(xrep_adoption_reparent); +DEFINE_XREP_DIRENT_EVENT(xrep_dir_stash_removename); +DEFINE_XREP_DIRENT_EVENT(xrep_dir_replay_removename); + +DECLARE_EVENT_CLASS(xrep_adoption_class, + TP_PROTO(struct xfs_inode *dp, struct xfs_inode *ip, bool moved), + TP_ARGS(dp, ip, moved), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, dir_ino) + __field(xfs_ino_t, child_ino) + __field(bool, moved) + ), + TP_fast_assign( + __entry->dev = dp->i_mount->m_super->s_dev; + __entry->dir_ino = I_INO(dp); + __entry->child_ino = I_INO(ip); + __entry->moved = moved; + ), + TP_printk("dev %d:%d dir 0x%llx child 0x%llx moved? %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->dir_ino, + __entry->child_ino, + __entry->moved) +); +#define DEFINE_XREP_ADOPTION_EVENT(name) \ +DEFINE_EVENT(xrep_adoption_class, name, \ + TP_PROTO(struct xfs_inode *dp, struct xfs_inode *ip, bool moved), \ + TP_ARGS(dp, ip, moved)) +DEFINE_XREP_ADOPTION_EVENT(xrep_adoption_trans_roll); + +DECLARE_EVENT_CLASS(xrep_parent_salvage_class, + TP_PROTO(struct xfs_inode *dp, xfs_ino_t ino), + TP_ARGS(dp, ino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, dir_ino) + __field(xfs_ino_t, ino) + ), + TP_fast_assign( + __entry->dev = dp->i_mount->m_super->s_dev; + __entry->dir_ino = I_INO(dp); + __entry->ino = ino; + ), + TP_printk("dev %d:%d dir 0x%llx parent 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->dir_ino, + __entry->ino) +) +#define DEFINE_XREP_PARENT_SALVAGE_EVENT(name) \ +DEFINE_EVENT(xrep_parent_salvage_class, name, \ + TP_PROTO(struct xfs_inode *dp, xfs_ino_t ino), \ + TP_ARGS(dp, ino)) +DEFINE_XREP_PARENT_SALVAGE_EVENT(xrep_dir_salvaged_parent); +DEFINE_XREP_PARENT_SALVAGE_EVENT(xrep_findparent_dirent); +DEFINE_XREP_PARENT_SALVAGE_EVENT(xrep_findparent_from_dcache); + +DECLARE_EVENT_CLASS(xrep_pptr_class, + TP_PROTO(struct xfs_inode *ip, const struct xfs_name *name, + const struct xfs_parent_rec *pptr), + TP_ARGS(ip, name, pptr), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_ino_t, parent_ino) + __field(unsigned int, parent_gen) + __field(unsigned int, namelen) + __dynamic_array(char, name, name->len) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = I_INO(ip); + __entry->parent_ino = be64_to_cpu(pptr->p_ino); + __entry->parent_gen = be32_to_cpu(pptr->p_gen); + __entry->namelen = name->len; + memcpy(__get_str(name), name->name, name->len); + ), + TP_printk("dev %d:%d ino 0x%llx parent_ino 0x%llx parent_gen 0x%x name '%.*s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->parent_ino, + __entry->parent_gen, + __entry->namelen, + __get_str(name)) +) +#define DEFINE_XREP_PPTR_EVENT(name) \ +DEFINE_EVENT(xrep_pptr_class, name, \ + TP_PROTO(struct xfs_inode *ip, const struct xfs_name *name, \ + const struct xfs_parent_rec *pptr), \ + TP_ARGS(ip, name, pptr)) +DEFINE_XREP_PPTR_EVENT(xrep_xattr_replay_parentadd); +DEFINE_XREP_PPTR_EVENT(xrep_xattr_replay_parentremove); +DEFINE_XREP_PPTR_EVENT(xrep_parent_replay_parentadd); +DEFINE_XREP_PPTR_EVENT(xrep_parent_replay_parentremove); + +DECLARE_EVENT_CLASS(xrep_pptr_scan_class, + TP_PROTO(struct xfs_inode *ip, const struct xfs_inode *dp, + const struct xfs_name *name), + TP_ARGS(ip, dp, name), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_ino_t, parent_ino) + __field(unsigned int, parent_gen) + __field(unsigned int, namelen) + __dynamic_array(char, name, name->len) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = I_INO(ip); + __entry->parent_ino = I_INO(dp); + __entry->parent_gen = VFS_IC(dp)->i_generation; + __entry->namelen = name->len; + memcpy(__get_str(name), name->name, name->len); + ), + TP_printk("dev %d:%d ino 0x%llx parent_ino 0x%llx parent_gen 0x%x name '%.*s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->parent_ino, + __entry->parent_gen, + __entry->namelen, + __get_str(name)) +) +#define DEFINE_XREP_PPTR_SCAN_EVENT(name) \ +DEFINE_EVENT(xrep_pptr_scan_class, name, \ + TP_PROTO(struct xfs_inode *ip, const struct xfs_inode *dp, \ + const struct xfs_name *name), \ + TP_ARGS(ip, dp, name)) +DEFINE_XREP_PPTR_SCAN_EVENT(xrep_parent_stash_parentadd); +DEFINE_XREP_PPTR_SCAN_EVENT(xrep_parent_stash_parentremove); + +TRACE_EVENT(xrep_nlinks_set_record, + TP_PROTO(struct xfs_mount *mp, xfs_ino_t ino, + const struct xchk_nlink *obs), + TP_ARGS(mp, ino, obs), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_nlink_t, parents) + __field(xfs_nlink_t, backrefs) + __field(xfs_nlink_t, children) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->ino = ino; + __entry->parents = obs->parents; + __entry->backrefs = obs->backrefs; + __entry->children = obs->children; + ), + TP_printk("dev %d:%d ino 0x%llx parents %u backrefs %u children %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->parents, + __entry->backrefs, + __entry->children) +); + +DECLARE_EVENT_CLASS(xrep_dentry_class, + TP_PROTO(struct xfs_mount *mp, const struct dentry *dentry), + TP_ARGS(mp, dentry), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, flags) + __field(unsigned long, ino) + __field(bool, positive) + __field(unsigned long, parent_ino) + __field(unsigned int, namelen) + __dynamic_array(char, name, dentry->d_name.len) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->flags = dentry->d_flags; + __entry->positive = d_is_positive(dentry); + if (dentry->d_parent && d_inode(dentry->d_parent)) + __entry->parent_ino = d_inode(dentry->d_parent)->i_ino; + else + __entry->parent_ino = -1UL; + __entry->ino = d_inode(dentry) ? d_inode(dentry)->i_ino : 0; + __entry->namelen = dentry->d_name.len; + memcpy(__get_str(name), dentry->d_name.name, dentry->d_name.len); + ), + TP_printk("dev %d:%d flags 0x%x positive? %d parent_ino 0x%lx ino 0x%lx name '%.*s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->flags, + __entry->positive, + __entry->parent_ino, + __entry->ino, + __entry->namelen, + __get_str(name)) +); +#define DEFINE_REPAIR_DENTRY_EVENT(name) \ +DEFINE_EVENT(xrep_dentry_class, name, \ + TP_PROTO(struct xfs_mount *mp, const struct dentry *dentry), \ + TP_ARGS(mp, dentry)) +DEFINE_REPAIR_DENTRY_EVENT(xrep_adoption_check_child); +DEFINE_REPAIR_DENTRY_EVENT(xrep_adoption_invalidate_child); +DEFINE_REPAIR_DENTRY_EVENT(xrep_dirtree_delete_child); + +TRACE_EVENT(xrep_symlink_salvage_target, + TP_PROTO(struct xfs_inode *ip, char *target, unsigned int targetlen), + TP_ARGS(ip, target, targetlen), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(unsigned int, targetlen) + __dynamic_array(char, target, targetlen + 1) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = I_INO(ip); + __entry->targetlen = targetlen; + memcpy(__get_str(target), target, targetlen); + __get_str(target)[targetlen] = 0; + ), + TP_printk("dev %d:%d ip 0x%llx target '%.*s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->targetlen, + __get_str(target)) +); + +DECLARE_EVENT_CLASS(xrep_symlink_class, + TP_PROTO(struct xfs_inode *ip), + TP_ARGS(ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = I_INO(ip); + ), + TP_printk("dev %d:%d ip 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino) +); + +#define DEFINE_XREP_SYMLINK_EVENT(name) \ +DEFINE_EVENT(xrep_symlink_class, name, \ + TP_PROTO(struct xfs_inode *ip), \ + TP_ARGS(ip)) +DEFINE_XREP_SYMLINK_EVENT(xrep_symlink_rebuild); +DEFINE_XREP_SYMLINK_EVENT(xrep_symlink_reset_fork); + +TRACE_EVENT(xrep_iunlink_visit, + TP_PROTO(const struct xfs_perag *pag, unsigned int bucket, + xfs_agino_t bucket_agino, struct xfs_inode *ip), + TP_ARGS(pag, bucket, bucket_agino, ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agino_t, agino) + __field(unsigned int, bucket) + __field(xfs_agino_t, bucket_agino) + __field(xfs_agino_t, prev_agino) + __field(xfs_agino_t, next_agino) + ), + TP_fast_assign( + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->agno = pag_agno(pag); + __entry->agino = XFS_INODE_TO_AGINO(ip); + __entry->bucket = bucket; + __entry->bucket_agino = bucket_agino; + __entry->prev_agino = ip->i_prev_unlinked; + __entry->next_agino = ip->i_next_unlinked; + ), + TP_printk("dev %d:%d agno 0x%x bucket %u agino 0x%x bucket_agino 0x%x prev_agino 0x%x next_agino 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->bucket, + __entry->agino, + __entry->bucket_agino, + __entry->prev_agino, + __entry->next_agino) +); + +TRACE_EVENT(xrep_iunlink_reload_next, + TP_PROTO(struct xfs_inode *ip, xfs_agino_t prev_agino), + TP_ARGS(ip, prev_agino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agino_t, agino) + __field(xfs_agino_t, old_prev_agino) + __field(xfs_agino_t, prev_agino) + __field(xfs_agino_t, next_agino) + __field(unsigned int, nlink) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->agno = XFS_INODE_TO_AGNO(ip); + __entry->agino = XFS_INODE_TO_AGINO(ip); + __entry->old_prev_agino = ip->i_prev_unlinked; + __entry->prev_agino = prev_agino; + __entry->next_agino = ip->i_next_unlinked; + __entry->nlink = VFS_I(ip)->i_nlink; + ), + TP_printk("dev %d:%d agno 0x%x bucket %u agino 0x%x nlink %u old_prev_agino %u prev_agino 0x%x next_agino 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->agino % XFS_AGI_UNLINKED_BUCKETS, + __entry->agino, + __entry->nlink, + __entry->old_prev_agino, + __entry->prev_agino, + __entry->next_agino) +); + +TRACE_EVENT(xrep_iunlink_reload_ondisk, + TP_PROTO(struct xfs_inode *ip), + TP_ARGS(ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agino_t, agino) + __field(unsigned int, nlink) + __field(xfs_agino_t, next_agino) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->agno = XFS_INODE_TO_AGNO(ip); + __entry->agino = XFS_INODE_TO_AGINO(ip); + __entry->nlink = VFS_I(ip)->i_nlink; + __entry->next_agino = ip->i_next_unlinked; + ), + TP_printk("dev %d:%d agno 0x%x bucket %u agino 0x%x nlink %u next_agino 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->agino % XFS_AGI_UNLINKED_BUCKETS, + __entry->agino, + __entry->nlink, + __entry->next_agino) +); + +TRACE_EVENT(xrep_iunlink_walk_ondisk_bucket, + TP_PROTO(const struct xfs_perag *pag, unsigned int bucket, + xfs_agino_t prev_agino, xfs_agino_t next_agino), + TP_ARGS(pag, bucket, prev_agino, next_agino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(unsigned int, bucket) + __field(xfs_agino_t, prev_agino) + __field(xfs_agino_t, next_agino) + ), + TP_fast_assign( + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->agno = pag_agno(pag); + __entry->bucket = bucket; + __entry->prev_agino = prev_agino; + __entry->next_agino = next_agino; + ), + TP_printk("dev %d:%d agno 0x%x bucket %u prev_agino 0x%x next_agino 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->bucket, + __entry->prev_agino, + __entry->next_agino) +); + +DECLARE_EVENT_CLASS(xrep_iunlink_resolve_class, + TP_PROTO(const struct xfs_perag *pag, unsigned int bucket, + xfs_agino_t prev_agino, xfs_agino_t next_agino), + TP_ARGS(pag, bucket, prev_agino, next_agino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(unsigned int, bucket) + __field(xfs_agino_t, prev_agino) + __field(xfs_agino_t, next_agino) + ), + TP_fast_assign( + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->agno = pag_agno(pag); + __entry->bucket = bucket; + __entry->prev_agino = prev_agino; + __entry->next_agino = next_agino; + ), + TP_printk("dev %d:%d agno 0x%x bucket %u prev_agino 0x%x next_agino 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->bucket, + __entry->prev_agino, + __entry->next_agino) +); +#define DEFINE_REPAIR_IUNLINK_RESOLVE_EVENT(name) \ +DEFINE_EVENT(xrep_iunlink_resolve_class, name, \ + TP_PROTO(const struct xfs_perag *pag, unsigned int bucket, \ + xfs_agino_t prev_agino, xfs_agino_t next_agino), \ + TP_ARGS(pag, bucket, prev_agino, next_agino)) +DEFINE_REPAIR_IUNLINK_RESOLVE_EVENT(xrep_iunlink_resolve_infinite_loop); +DEFINE_REPAIR_IUNLINK_RESOLVE_EVENT(xrep_iunlink_resolve_uncached); +DEFINE_REPAIR_IUNLINK_RESOLVE_EVENT(xrep_iunlink_resolve_wronglist); +DEFINE_REPAIR_IUNLINK_RESOLVE_EVENT(xrep_iunlink_resolve_nolist); +DEFINE_REPAIR_IUNLINK_RESOLVE_EVENT(xrep_iunlink_resolve_ok); +DEFINE_REPAIR_IUNLINK_RESOLVE_EVENT(xrep_iunlink_resolve_allocated); + +TRACE_EVENT(xrep_iunlink_relink_next, + TP_PROTO(struct xfs_inode *ip, xfs_agino_t next_agino), + TP_ARGS(ip, next_agino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agino_t, agino) + __field(xfs_agino_t, next_agino) + __field(xfs_agino_t, new_next_agino) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->agno = XFS_INODE_TO_AGNO(ip); + __entry->agino = XFS_INODE_TO_AGINO(ip); + __entry->next_agino = ip->i_next_unlinked; + __entry->new_next_agino = next_agino; + ), + TP_printk("dev %d:%d agno 0x%x bucket %u agino 0x%x next_agino 0x%x -> 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->agino % XFS_AGI_UNLINKED_BUCKETS, + __entry->agino, + __entry->next_agino, + __entry->new_next_agino) +); + +TRACE_EVENT(xrep_iunlink_relink_prev, + TP_PROTO(struct xfs_inode *ip, xfs_agino_t prev_agino), + TP_ARGS(ip, prev_agino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agino_t, agino) + __field(xfs_agino_t, prev_agino) + __field(xfs_agino_t, new_prev_agino) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->agno = XFS_INODE_TO_AGNO(ip); + __entry->agino = XFS_INODE_TO_AGINO(ip); + __entry->prev_agino = ip->i_prev_unlinked; + __entry->new_prev_agino = prev_agino; + ), + TP_printk("dev %d:%d agno 0x%x bucket %u agino 0x%x prev_agino 0x%x -> 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->agino % XFS_AGI_UNLINKED_BUCKETS, + __entry->agino, + __entry->prev_agino, + __entry->new_prev_agino) +); + +TRACE_EVENT(xrep_iunlink_add_to_bucket, + TP_PROTO(const struct xfs_perag *pag, unsigned int bucket, + xfs_agino_t agino, xfs_agino_t curr_head), + TP_ARGS(pag, bucket, agino, curr_head), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(unsigned int, bucket) + __field(xfs_agino_t, agino) + __field(xfs_agino_t, next_agino) + ), + TP_fast_assign( + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->agno = pag_agno(pag); + __entry->bucket = bucket; + __entry->agino = agino; + __entry->next_agino = curr_head; + ), + TP_printk("dev %d:%d agno 0x%x bucket %u agino 0x%x next_agino 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->bucket, + __entry->agino, + __entry->next_agino) +); + +TRACE_EVENT(xrep_iunlink_commit_bucket, + TP_PROTO(const struct xfs_perag *pag, unsigned int bucket, + xfs_agino_t old_agino, xfs_agino_t agino), + TP_ARGS(pag, bucket, old_agino, agino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(unsigned int, bucket) + __field(xfs_agino_t, old_agino) + __field(xfs_agino_t, agino) + ), + TP_fast_assign( + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->agno = pag_agno(pag); + __entry->bucket = bucket; + __entry->old_agino = old_agino; + __entry->agino = agino; + ), + TP_printk("dev %d:%d agno 0x%x bucket %u agino 0x%x -> 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->bucket, + __entry->old_agino, + __entry->agino) +); + +DEFINE_XCHK_DIRPATH_OUTCOME_EVENT(xrep_dirpath_set_outcome); +DEFINE_XCHK_DIRTREE_EVENT(xrep_dirtree_delete_path); +DEFINE_XCHK_DIRTREE_EVENT(xrep_dirtree_create_adoption); +DEFINE_XCHK_DIRTREE_EVALUATE_EVENT(xrep_dirtree_decided_fate); + +DEFINE_XCHK_METAPATH_EVENT(xrep_metapath_lookup); +DEFINE_XCHK_METAPATH_EVENT(xrep_metapath_try_unlink); +DEFINE_XCHK_METAPATH_EVENT(xrep_metapath_unlink); +DEFINE_XCHK_METAPATH_EVENT(xrep_metapath_link); + +#ifdef CONFIG_XFS_RT +DECLARE_EVENT_CLASS(xrep_rtbitmap_class, + TP_PROTO(struct xfs_mount *mp, xfs_rtxnum_t start, xfs_rtxnum_t end), + TP_ARGS(mp, start, end), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(dev_t, rtdev) + __field(xfs_rtxnum_t, start) + __field(xfs_rtxnum_t, end) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->rtdev = mp->m_rtdev_targp->bt_dev; + __entry->start = start; + __entry->end = end; + ), + TP_printk("dev %d:%d rtdev %d:%d startrtx 0x%llx endrtx 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + MAJOR(__entry->rtdev), MINOR(__entry->rtdev), + __entry->start, + __entry->end) +); +#define DEFINE_REPAIR_RGBITMAP_EVENT(name) \ +DEFINE_EVENT(xrep_rtbitmap_class, name, \ + TP_PROTO(struct xfs_mount *mp, xfs_rtxnum_t start, \ + xfs_rtxnum_t end), \ + TP_ARGS(mp, start, end)) +DEFINE_REPAIR_RGBITMAP_EVENT(xrep_rtbitmap_record_free); +DEFINE_REPAIR_RGBITMAP_EVENT(xrep_rtbitmap_record_free_bulk); + +TRACE_EVENT(xrep_rtbitmap_or, + TP_PROTO(struct xfs_mount *mp, unsigned long long wordoff, + xfs_rtword_t mask, xfs_rtword_t word), + TP_ARGS(mp, wordoff, mask, word), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(dev_t, rtdev) + __field(unsigned long long, wordoff) + __field(unsigned int, mask) + __field(unsigned int, word) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->rtdev = mp->m_rtdev_targp->bt_dev; + __entry->wordoff = wordoff; + __entry->mask = mask; + __entry->word = word; + ), + TP_printk("dev %d:%d rtdev %d:%d wordoff 0x%llx mask 0x%x word 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + MAJOR(__entry->rtdev), MINOR(__entry->rtdev), + __entry->wordoff, + __entry->mask, + __entry->word) +); + +TRACE_EVENT(xrep_rtbitmap_load, + TP_PROTO(struct xfs_rtgroup *rtg, xfs_fileoff_t rbmoff, + xfs_rtxnum_t rtx, xfs_rtxnum_t len), + TP_ARGS(rtg, rbmoff, rtx, len), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(dev_t, rtdev) + __field(xfs_rgnumber_t, rgno) + __field(xfs_fileoff_t, rbmoff) + __field(xfs_rtxnum_t, rtx) + __field(xfs_rtxnum_t, len) + ), + TP_fast_assign( + __entry->dev = rtg_mount(rtg)->m_super->s_dev; + __entry->rtdev = rtg_mount(rtg)->m_rtdev_targp->bt_dev; + __entry->rgno = rtg_rgno(rtg); + __entry->rbmoff = rbmoff; + __entry->rtx = rtx; + __entry->len = len; + ), + TP_printk("dev %d:%d rtdev %d:%d rgno 0x%x rbmoff 0x%llx rtx 0x%llx rtxcount 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + MAJOR(__entry->rtdev), MINOR(__entry->rtdev), + __entry->rgno, + __entry->rbmoff, + __entry->rtx, + __entry->len) +); + +TRACE_EVENT(xrep_rtbitmap_load_words, + TP_PROTO(struct xfs_mount *mp, xfs_fileoff_t rbmoff, + unsigned long long wordoff, unsigned int wordcnt), + TP_ARGS(mp, rbmoff, wordoff, wordcnt), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(dev_t, rtdev) + __field(xfs_fileoff_t, rbmoff) + __field(unsigned long long, wordoff) + __field(unsigned int, wordcnt) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->rtdev = mp->m_rtdev_targp->bt_dev; + __entry->rbmoff = rbmoff; + __entry->wordoff = wordoff; + __entry->wordcnt = wordcnt; + ), + TP_printk("dev %d:%d rtdev %d:%d rbmoff 0x%llx wordoff 0x%llx wordcnt 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + MAJOR(__entry->rtdev), MINOR(__entry->rtdev), + __entry->rbmoff, + __entry->wordoff, + __entry->wordcnt) +); + +TRACE_EVENT(xrep_rtbitmap_load_word, + TP_PROTO(struct xfs_mount *mp, unsigned long long wordoff, + unsigned int bit, xfs_rtword_t ondisk_word, + xfs_rtword_t xfile_word, xfs_rtword_t word_mask), + TP_ARGS(mp, wordoff, bit, ondisk_word, xfile_word, word_mask), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(dev_t, rtdev) + __field(unsigned long long, wordoff) + __field(unsigned int, bit) + __field(xfs_rtword_t, ondisk_word) + __field(xfs_rtword_t, xfile_word) + __field(xfs_rtword_t, word_mask) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->rtdev = mp->m_rtdev_targp->bt_dev; + __entry->wordoff = wordoff; + __entry->bit = bit; + __entry->ondisk_word = ondisk_word; + __entry->xfile_word = xfile_word; + __entry->word_mask = word_mask; + ), + TP_printk("dev %d:%d rtdev %d:%d wordoff 0x%llx bit %u ondisk 0x%x(0x%x) inmem 0x%x(0x%x) result 0x%x mask 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + MAJOR(__entry->rtdev), MINOR(__entry->rtdev), + __entry->wordoff, + __entry->bit, + __entry->ondisk_word, + __entry->ondisk_word & __entry->word_mask, + __entry->xfile_word, + __entry->xfile_word & ~__entry->word_mask, + (__entry->xfile_word & ~__entry->word_mask) | + (__entry->ondisk_word & __entry->word_mask), + __entry->word_mask) +); + +TRACE_EVENT(xrep_rtrmap_found, + TP_PROTO(struct xfs_mount *mp, const struct xfs_rmap_irec *rec), + TP_ARGS(mp, rec), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(dev_t, rtdev) + __field(xfs_rgblock_t, rgbno) + __field(xfs_extlen_t, len) + __field(uint64_t, owner) + __field(uint64_t, offset) + __field(unsigned int, flags) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->rtdev = mp->m_rtdev_targp->bt_dev; + __entry->rgbno = rec->rm_startblock; + __entry->len = rec->rm_blockcount; + __entry->owner = rec->rm_owner; + __entry->offset = rec->rm_offset; + __entry->flags = rec->rm_flags; + ), + TP_printk("dev %d:%d rtdev %d:%d rgbno 0x%x fsbcount 0x%x owner 0x%llx fileoff 0x%llx flags 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + MAJOR(__entry->rtdev), MINOR(__entry->rtdev), + __entry->rgbno, + __entry->len, + __entry->owner, + __entry->offset, + __entry->flags) +); +#endif /* CONFIG_XFS_RT */ + +#endif /* IS_ENABLED(CONFIG_XFS_ONLINE_REPAIR) */ + +#endif /* _TRACE_XFS_SCRUB_TRACE_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE scrub/trace +#include diff --git a/fs/xfs/scrub/xfarray.c b/fs/xfs/scrub/xfarray.c new file mode 100644 index 000000000..2ce24bfe4 --- /dev/null +++ b/fs/xfs/scrub/xfarray.c @@ -0,0 +1,1072 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2021-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "scrub/scrub.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/trace.h" + +/* + * Large Arrays of Fixed-Size Records + * ================================== + * + * This memory array uses an xfile (which itself is a shmem file) to store + * large numbers of fixed-size records in memory that can be paged out. This + * puts less stress on the memory reclaim algorithms during an online repair + * because we don't have to pin so much memory. However, array access is less + * direct than would be in a regular memory array. Access to the array is + * performed via indexed load and store methods, and an append method is + * provided for convenience. Array elements can be unset, which sets them to + * all zeroes. Unset entries are skipped during iteration, though direct loads + * will return a zeroed buffer. Callers are responsible for concurrency + * control. + */ + +/* + * Pointer to scratch space. Because we can't access the xfile data directly, + * we allocate a small amount of memory on the end of the xfarray structure to + * buffer array items when we need space to store values temporarily. + */ +static inline void *xfarray_scratch(struct xfarray *array) +{ + return (array + 1); +} + +/* Compute array index given an xfile offset. */ +static xfarray_idx_t +xfarray_idx( + struct xfarray *array, + loff_t pos) +{ + if (array->obj_size_log >= 0) + return (xfarray_idx_t)pos >> array->obj_size_log; + + return div_u64((xfarray_idx_t)pos, array->obj_size); +} + +/* Compute xfile offset of array element. */ +static inline loff_t xfarray_pos(struct xfarray *array, xfarray_idx_t idx) +{ + if (array->obj_size_log >= 0) + return idx << array->obj_size_log; + + return idx * array->obj_size; +} + +/* + * Initialize a big memory array. Array records cannot be larger than a + * page, and the array cannot span more bytes than the page cache supports. + * If @required_capacity is nonzero, the maximum array size will be set to this + * quantity and the array creation will fail if the underlying storage cannot + * support that many records. + */ +int +xfarray_create( + const char *description, + unsigned long long required_capacity, + size_t obj_size, + struct xfarray **arrayp) +{ + struct xfarray *array; + struct xfile *xfile; + int error; + + ASSERT(obj_size < PAGE_SIZE); + + error = xfile_create(description, 0, &xfile); + if (error) + return error; + + error = -ENOMEM; + array = kzalloc(sizeof(struct xfarray) + obj_size, XCHK_GFP_FLAGS); + if (!array) + goto out_xfile; + + array->xfile = xfile; + array->obj_size = obj_size; + + if (is_power_of_2(obj_size)) + array->obj_size_log = ilog2(obj_size); + else + array->obj_size_log = -1; + + array->max_nr = xfarray_idx(array, MAX_LFS_FILESIZE); + trace_xfarray_create(array, required_capacity); + + if (required_capacity > 0) { + if (array->max_nr < required_capacity) { + error = -ENOMEM; + goto out_xfarray; + } + array->max_nr = required_capacity; + } + + *arrayp = array; + return 0; + +out_xfarray: + kfree(array); +out_xfile: + xfile_destroy(xfile); + return error; +} + +/* Destroy the array. */ +void +xfarray_destroy( + struct xfarray *array) +{ + xfile_destroy(array->xfile); + kfree(array); +} + +/* Load an element from the array. */ +int +xfarray_load( + struct xfarray *array, + xfarray_idx_t idx, + void *ptr) +{ + if (idx >= array->nr) + return -ENODATA; + + return xfile_load(array->xfile, ptr, array->obj_size, + xfarray_pos(array, idx)); +} + +/* Is this array element potentially unset? */ +static inline bool +xfarray_is_unset( + struct xfarray *array, + loff_t pos) +{ + void *temp = xfarray_scratch(array); + int error; + + if (array->unset_slots == 0) + return false; + + error = xfile_load(array->xfile, temp, array->obj_size, pos); + if (!error && xfarray_element_is_null(array, temp)) + return true; + + return false; +} + +/* + * Unset an array element. If @idx is the last element in the array, the + * array will be truncated. Otherwise, the entry will be zeroed. + */ +int +xfarray_unset( + struct xfarray *array, + xfarray_idx_t idx) +{ + void *temp = xfarray_scratch(array); + loff_t pos = xfarray_pos(array, idx); + int error; + + if (idx >= array->nr) + return -ENODATA; + + if (idx == array->nr - 1) { + array->nr--; + return 0; + } + + if (xfarray_is_unset(array, pos)) + return 0; + + memset(temp, 0, array->obj_size); + error = xfile_store(array->xfile, temp, array->obj_size, pos); + if (error) + return error; + + array->unset_slots++; + return 0; +} + +/* + * Store an element in the array. The element must not be completely zeroed, + * because those are considered unset sparse elements. + */ +int +xfarray_store( + struct xfarray *array, + xfarray_idx_t idx, + const void *ptr) +{ + int ret; + + if (idx >= array->max_nr) + return -EFBIG; + + ASSERT(!xfarray_element_is_null(array, ptr)); + + ret = xfile_store(array->xfile, ptr, array->obj_size, + xfarray_pos(array, idx)); + if (ret) + return ret; + + array->nr = max(array->nr, idx + 1); + return 0; +} + +/* Is this array element NULL? */ +bool +xfarray_element_is_null( + struct xfarray *array, + const void *ptr) +{ + return !memchr_inv(ptr, 0, array->obj_size); +} + +/* + * Store an element anywhere in the array that is unset. If there are no + * unset slots, append the element to the array. + */ +int +xfarray_store_anywhere( + struct xfarray *array, + const void *ptr) +{ + void *temp = xfarray_scratch(array); + loff_t endpos = xfarray_pos(array, array->nr); + loff_t pos; + int error; + + /* Find an unset slot to put it in. */ + for (pos = 0; + pos < endpos && array->unset_slots > 0; + pos += array->obj_size) { + error = xfile_load(array->xfile, temp, array->obj_size, + pos); + if (error || !xfarray_element_is_null(array, temp)) + continue; + + error = xfile_store(array->xfile, ptr, array->obj_size, + pos); + if (error) + return error; + + array->unset_slots--; + return 0; + } + + /* No unset slots found; attach it on the end. */ + array->unset_slots = 0; + return xfarray_append(array, ptr); +} + +/* Return length of array. */ +uint64_t +xfarray_length( + struct xfarray *array) +{ + return array->nr; +} + +/* + * Decide which array item we're going to read as part of an _iter_get. + * @cur is the array index, and @pos is the file offset of that array index in + * the backing xfile. Returns ENODATA if we reach the end of the records. + * + * Reading from a hole in a sparse xfile causes page instantiation, so for + * iterating a (possibly sparse) array we need to figure out if the cursor is + * pointing at a totally uninitialized hole and move the cursor up if + * necessary. + */ +static inline int +xfarray_find_data( + struct xfarray *array, + xfarray_idx_t *cur, + loff_t *pos) +{ + unsigned int pgoff = offset_in_page(*pos); + loff_t end_pos = *pos + array->obj_size - 1; + loff_t new_pos; + + /* + * If the current array record is not adjacent to a page boundary, we + * are in the middle of the page. We do not need to move the cursor. + */ + if (pgoff != 0 && pgoff + array->obj_size - 1 < PAGE_SIZE) + return 0; + + /* + * Call SEEK_DATA on the last byte in the record we're about to read. + * If the record ends at (or crosses) the end of a page then we know + * that the first byte of the record is backed by pages and don't need + * to query it. If instead the record begins at the start of the page + * then we know that querying the last byte is just as good as querying + * the first byte, since records cannot be larger than a page. + * + * If the call returns the same file offset, we know this record is + * backed by real pages. We do not need to move the cursor. + */ + new_pos = xfile_seek_data(array->xfile, end_pos); + if (new_pos == -ENXIO) + return -ENODATA; + if (new_pos < 0) + return new_pos; + if (new_pos == end_pos) + return 0; + + /* + * Otherwise, SEEK_DATA told us how far up to move the file pointer to + * find more data. Move the array index to the first record past the + * byte offset we were given. + */ + new_pos = roundup_64(new_pos, array->obj_size); + *cur = xfarray_idx(array, new_pos); + *pos = xfarray_pos(array, *cur); + return 0; +} + +/* + * Starting at *idx, fetch the next non-null array entry and advance the index + * to set up the next _load_next call. Returns ENODATA if we reach the end of + * the array. Callers must set @*idx to XFARRAY_CURSOR_INIT before the first + * call to this function. + */ +int +xfarray_load_next( + struct xfarray *array, + xfarray_idx_t *idx, + void *rec) +{ + xfarray_idx_t cur = *idx; + loff_t pos = xfarray_pos(array, cur); + int error; + + do { + if (cur >= array->nr) + return -ENODATA; + + /* + * Ask the backing store for the location of next possible + * written record, then retrieve that record. + */ + error = xfarray_find_data(array, &cur, &pos); + if (error) + return error; + error = xfarray_load(array, cur, rec); + if (error) + return error; + + cur++; + pos += array->obj_size; + } while (xfarray_element_is_null(array, rec)); + + *idx = cur; + return 0; +} + +/* Sorting functions */ + +#ifdef DEBUG +# define xfarray_sort_bump_loads(si) do { (si)->loads++; } while (0) +# define xfarray_sort_bump_stores(si) do { (si)->stores++; } while (0) +# define xfarray_sort_bump_compares(si) do { (si)->compares++; } while (0) +# define xfarray_sort_bump_heapsorts(si) do { (si)->heapsorts++; } while (0) +#else +# define xfarray_sort_bump_loads(si) +# define xfarray_sort_bump_stores(si) +# define xfarray_sort_bump_compares(si) +# define xfarray_sort_bump_heapsorts(si) +#endif /* DEBUG */ + +/* Load an array element for sorting. */ +static inline int +xfarray_sort_load( + struct xfarray_sortinfo *si, + xfarray_idx_t idx, + void *ptr) +{ + xfarray_sort_bump_loads(si); + return xfarray_load(si->array, idx, ptr); +} + +/* Store an array element for sorting. */ +static inline int +xfarray_sort_store( + struct xfarray_sortinfo *si, + xfarray_idx_t idx, + void *ptr) +{ + xfarray_sort_bump_stores(si); + return xfarray_store(si->array, idx, ptr); +} + +/* Compare an array element for sorting. */ +static inline int +xfarray_sort_cmp( + struct xfarray_sortinfo *si, + const void *a, + const void *b) +{ + xfarray_sort_bump_compares(si); + return si->cmp_fn(a, b); +} + +/* Return a pointer to the low index stack for quicksort partitioning. */ +static inline xfarray_idx_t *xfarray_sortinfo_lo(struct xfarray_sortinfo *si) +{ + return (xfarray_idx_t *)(si + 1); +} + +/* Return a pointer to the high index stack for quicksort partitioning. */ +static inline xfarray_idx_t *xfarray_sortinfo_hi(struct xfarray_sortinfo *si) +{ + return xfarray_sortinfo_lo(si) + si->max_stack_depth; +} + +/* Size of each element in the quicksort pivot array. */ +static inline size_t +xfarray_pivot_rec_sz( + struct xfarray *array) +{ + return round_up(array->obj_size, 8) + sizeof(xfarray_idx_t); +} + +/* Allocate memory to handle the sort. */ +static inline int +xfarray_sortinfo_alloc( + struct xfarray *array, + xfarray_cmp_fn cmp_fn, + unsigned int flags, + struct xfarray_sortinfo **infop) +{ + struct xfarray_sortinfo *si; + size_t nr_bytes = sizeof(struct xfarray_sortinfo); + size_t pivot_rec_sz = xfarray_pivot_rec_sz(array); + int max_stack_depth; + + /* + * The median-of-nine pivot algorithm doesn't work if a subset has + * fewer than 9 items. Make sure the in-memory sort will always take + * over for subsets where this wouldn't be the case. + */ + BUILD_BUG_ON(XFARRAY_QSORT_PIVOT_NR >= XFARRAY_ISORT_NR); + + /* + * Tail-call recursion during the partitioning phase means that + * quicksort will never recurse more than log2(nr) times. We need one + * extra level of stack to hold the initial parameters. In-memory + * sort will always take care of the last few levels of recursion for + * us, so we can reduce the stack depth by that much. + */ + max_stack_depth = ilog2(array->nr) + 1 - (XFARRAY_ISORT_SHIFT - 1); + if (max_stack_depth < 1) + max_stack_depth = 1; + + /* Each level of quicksort uses a lo and a hi index */ + nr_bytes += max_stack_depth * sizeof(xfarray_idx_t) * 2; + + /* Scratchpad for in-memory sort, or finding the pivot */ + nr_bytes += max_t(size_t, + (XFARRAY_QSORT_PIVOT_NR + 1) * pivot_rec_sz, + XFARRAY_ISORT_NR * array->obj_size); + + si = kvzalloc(nr_bytes, XCHK_GFP_FLAGS); + if (!si) + return -ENOMEM; + + si->array = array; + si->cmp_fn = cmp_fn; + si->flags = flags; + si->max_stack_depth = max_stack_depth; + si->max_stack_used = 1; + + xfarray_sortinfo_lo(si)[0] = 0; + xfarray_sortinfo_hi(si)[0] = array->nr - 1; + si->relax = INIT_XCHK_RELAX; + si->relax.killable = !!(flags & XFARRAY_SORT_KILLABLE); + + trace_xfarray_sort(si, nr_bytes); + *infop = si; + return 0; +} + +/* Should this sort be terminated by a fatal signal? */ +static inline bool +xfarray_sort_terminated( + struct xfarray_sortinfo *si, + int *error) +{ + /* + * If preemption is disabled, we need to yield to the scheduler every + * few seconds so that we don't run afoul of the soft lockup watchdog + * or RCU stall detector. + */ + if (xchk_maybe_relax(&si->relax)) { + if (*error == 0) + *error = -EINTR; + return true; + } + return false; +} + +/* Do we want an in-memory sort? */ +static inline bool +xfarray_want_isort( + struct xfarray_sortinfo *si, + xfarray_idx_t start, + xfarray_idx_t end) +{ + /* + * For array subsets that fit in the scratchpad, it's much faster to + * use the kernel's heapsort than quicksort's stack machine. + */ + return (end - start) < XFARRAY_ISORT_NR; +} + +/* Return the scratch space within the sortinfo structure. */ +static inline void *xfarray_sortinfo_isort_scratch(struct xfarray_sortinfo *si) +{ + return xfarray_sortinfo_hi(si) + si->max_stack_depth; +} + +/* + * Sort a small number of array records using scratchpad memory. The records + * need not be contiguous in the xfile's memory pages. + */ +STATIC int +xfarray_isort( + struct xfarray_sortinfo *si, + xfarray_idx_t lo, + xfarray_idx_t hi) +{ + void *scratch = xfarray_sortinfo_isort_scratch(si); + loff_t lo_pos = xfarray_pos(si->array, lo); + loff_t len = xfarray_pos(si->array, hi - lo + 1); + int error; + + trace_xfarray_isort(si, lo, hi); + + xfarray_sort_bump_loads(si); + error = xfile_load(si->array->xfile, scratch, len, lo_pos); + if (error) + return error; + + xfarray_sort_bump_heapsorts(si); + sort(scratch, hi - lo + 1, si->array->obj_size, si->cmp_fn, NULL); + + xfarray_sort_bump_stores(si); + return xfile_store(si->array->xfile, scratch, len, lo_pos); +} + +/* + * Sort the records from lo to hi (inclusive) if they are all backed by the + * same memory folio. Returns 1 if it sorted, 0 if it did not, or a negative + * errno. + */ +STATIC int +xfarray_foliosort( + struct xfarray_sortinfo *si, + xfarray_idx_t lo, + xfarray_idx_t hi) +{ + struct folio *folio; + void *startp; + loff_t lo_pos = xfarray_pos(si->array, lo); + uint64_t len = xfarray_pos(si->array, hi - lo + 1); + + /* No single folio could back this many records. */ + if (len > XFILE_MAX_FOLIO_SIZE) + return 0; + + xfarray_sort_bump_loads(si); + folio = xfile_get_folio(si->array->xfile, lo_pos, len, XFILE_ALLOC); + if (IS_ERR(folio)) + return PTR_ERR(folio); + if (!folio) + return 0; + + trace_xfarray_foliosort(si, lo, hi); + + xfarray_sort_bump_heapsorts(si); + startp = folio_address(folio) + offset_in_folio(folio, lo_pos); + sort(startp, hi - lo + 1, si->array->obj_size, si->cmp_fn, NULL); + + xfarray_sort_bump_stores(si); + xfile_put_folio(si->array->xfile, folio); + return 1; +} + +/* Return a pointer to the xfarray pivot record within the sortinfo struct. */ +static inline void *xfarray_sortinfo_pivot(struct xfarray_sortinfo *si) +{ + return xfarray_sortinfo_hi(si) + si->max_stack_depth; +} + +/* Return a pointer to the start of the pivot array. */ +static inline void * +xfarray_sortinfo_pivot_array( + struct xfarray_sortinfo *si) +{ + return xfarray_sortinfo_pivot(si) + si->array->obj_size; +} + +/* The xfarray record is stored at the start of each pivot array element. */ +static inline void * +xfarray_pivot_array_rec( + void *pa, + size_t pa_recsz, + unsigned int pa_idx) +{ + return pa + (pa_recsz * pa_idx); +} + +/* The xfarray index is stored at the end of each pivot array element. */ +static inline xfarray_idx_t * +xfarray_pivot_array_idx( + void *pa, + size_t pa_recsz, + unsigned int pa_idx) +{ + return xfarray_pivot_array_rec(pa, pa_recsz, pa_idx + 1) - + sizeof(xfarray_idx_t); +} + +/* + * Find a pivot value for quicksort partitioning, swap it with a[lo], and save + * the cached pivot record for the next step. + * + * Load evenly-spaced records within the given range into memory, sort them, + * and choose the pivot from the median record. Using multiple points will + * improve the quality of the pivot selection, and hopefully avoid the worst + * quicksort behavior, since our array values are nearly always evenly sorted. + */ +STATIC int +xfarray_qsort_pivot( + struct xfarray_sortinfo *si, + xfarray_idx_t lo, + xfarray_idx_t hi) +{ + void *pivot = xfarray_sortinfo_pivot(si); + void *parray = xfarray_sortinfo_pivot_array(si); + void *recp; + xfarray_idx_t *idxp; + xfarray_idx_t step = (hi - lo) / (XFARRAY_QSORT_PIVOT_NR - 1); + size_t pivot_rec_sz = xfarray_pivot_rec_sz(si->array); + int i, j; + int error; + + ASSERT(step > 0); + + /* + * Load the xfarray indexes of the records we intend to sample into the + * pivot array. + */ + idxp = xfarray_pivot_array_idx(parray, pivot_rec_sz, 0); + *idxp = lo; + for (i = 1; i < XFARRAY_QSORT_PIVOT_NR - 1; i++) { + idxp = xfarray_pivot_array_idx(parray, pivot_rec_sz, i); + *idxp = lo + (i * step); + } + idxp = xfarray_pivot_array_idx(parray, pivot_rec_sz, + XFARRAY_QSORT_PIVOT_NR - 1); + *idxp = hi; + + /* Load the selected xfarray records into the pivot array. */ + for (i = 0; i < XFARRAY_QSORT_PIVOT_NR; i++) { + xfarray_idx_t idx; + + recp = xfarray_pivot_array_rec(parray, pivot_rec_sz, i); + idxp = xfarray_pivot_array_idx(parray, pivot_rec_sz, i); + + /* No unset records; load directly into the array. */ + if (likely(si->array->unset_slots == 0)) { + error = xfarray_sort_load(si, *idxp, recp); + if (error) + return error; + continue; + } + + /* + * Load non-null records into the scratchpad without changing + * the xfarray_idx_t in the pivot array. + */ + idx = *idxp; + xfarray_sort_bump_loads(si); + error = xfarray_load_next(si->array, &idx, recp); + if (error) + return error; + } + + xfarray_sort_bump_heapsorts(si); + sort(parray, XFARRAY_QSORT_PIVOT_NR, pivot_rec_sz, si->cmp_fn, NULL); + + /* + * We sorted the pivot array records (which includes the xfarray + * indices) in xfarray record order. The median element of the pivot + * array contains the xfarray record that we will use as the pivot. + * Copy that xfarray record to the designated space. + */ + recp = xfarray_pivot_array_rec(parray, pivot_rec_sz, + XFARRAY_QSORT_PIVOT_NR / 2); + memcpy(pivot, recp, si->array->obj_size); + + /* If the pivot record we chose was already in a[lo] then we're done. */ + idxp = xfarray_pivot_array_idx(parray, pivot_rec_sz, + XFARRAY_QSORT_PIVOT_NR / 2); + if (*idxp == lo) + return 0; + + /* + * Find the cached copy of a[lo] in the pivot array so that we can swap + * a[lo] and a[pivot]. + */ + for (i = 0, j = -1; i < XFARRAY_QSORT_PIVOT_NR; i++) { + idxp = xfarray_pivot_array_idx(parray, pivot_rec_sz, i); + if (*idxp == lo) + j = i; + } + if (j < 0) { + ASSERT(j >= 0); + return -EFSCORRUPTED; + } + + /* Swap a[lo] and a[pivot]. */ + error = xfarray_sort_store(si, lo, pivot); + if (error) + return error; + + recp = xfarray_pivot_array_rec(parray, pivot_rec_sz, j); + idxp = xfarray_pivot_array_idx(parray, pivot_rec_sz, + XFARRAY_QSORT_PIVOT_NR / 2); + return xfarray_sort_store(si, *idxp, recp); +} + +/* + * Set up the pointers for the next iteration. We push onto the stack all of + * the unsorted values between a[lo + 1] and a[end[i]], and we tweak the + * current stack frame to point to the unsorted values between a[beg[i]] and + * a[lo] so that those values will be sorted when we pop the stack. + */ +static inline int +xfarray_qsort_push( + struct xfarray_sortinfo *si, + xfarray_idx_t *si_lo, + xfarray_idx_t *si_hi, + xfarray_idx_t lo, + xfarray_idx_t hi) +{ + /* Check for stack overflows */ + if (si->stack_depth >= si->max_stack_depth - 1) { + ASSERT(si->stack_depth < si->max_stack_depth - 1); + return -EFSCORRUPTED; + } + + si->max_stack_used = max_t(uint8_t, si->max_stack_used, + si->stack_depth + 2); + + si_lo[si->stack_depth + 1] = lo + 1; + si_hi[si->stack_depth + 1] = si_hi[si->stack_depth]; + si_hi[si->stack_depth++] = lo - 1; + + /* + * Always start with the smaller of the two partitions to keep the + * amount of recursion in check. + */ + if (si_hi[si->stack_depth] - si_lo[si->stack_depth] > + si_hi[si->stack_depth - 1] - si_lo[si->stack_depth - 1]) { + swap(si_lo[si->stack_depth], si_lo[si->stack_depth - 1]); + swap(si_hi[si->stack_depth], si_hi[si->stack_depth - 1]); + } + + return 0; +} + +static inline void +xfarray_sort_scan_done( + struct xfarray_sortinfo *si) +{ + if (si->folio) + xfile_put_folio(si->array->xfile, si->folio); + si->folio = NULL; +} + +/* + * Cache the folio backing the start of the given array element. If the array + * element is contained entirely within the folio, return a pointer to the + * cached folio. Otherwise, load the element into the scratchpad and return a + * pointer to the scratchpad. + */ +static inline int +xfarray_sort_scan( + struct xfarray_sortinfo *si, + xfarray_idx_t idx, + void **ptrp) +{ + loff_t idx_pos = xfarray_pos(si->array, idx); + int error = 0; + + if (xfarray_sort_terminated(si, &error)) + return error; + + trace_xfarray_sort_scan(si, idx); + + /* If the cached folio doesn't cover this index, release it. */ + if (si->folio && + (idx < si->first_folio_idx || idx > si->last_folio_idx)) + xfarray_sort_scan_done(si); + + /* Grab the first folio that backs this array element. */ + if (!si->folio) { + struct folio *folio; + loff_t next_pos; + + folio = xfile_get_folio(si->array->xfile, idx_pos, + si->array->obj_size, XFILE_ALLOC); + if (IS_ERR(folio)) + return PTR_ERR(folio); + si->folio = folio; + + si->first_folio_idx = xfarray_idx(si->array, + folio_pos(si->folio) + si->array->obj_size - 1); + + next_pos = folio_next_pos(si->folio); + si->last_folio_idx = xfarray_idx(si->array, next_pos - 1); + if (xfarray_pos(si->array, si->last_folio_idx + 1) > next_pos) + si->last_folio_idx--; + + trace_xfarray_sort_scan(si, idx); + } + + /* + * If this folio still doesn't cover the desired element, it must cross + * a folio boundary. Read into the scratchpad and we're done. + */ + if (idx < si->first_folio_idx || idx > si->last_folio_idx) { + void *temp = xfarray_scratch(si->array); + + error = xfile_load(si->array->xfile, temp, si->array->obj_size, + idx_pos); + if (error) + return error; + + *ptrp = temp; + return 0; + } + + /* Otherwise return a pointer to the array element in the folio. */ + *ptrp = folio_address(si->folio) + offset_in_folio(si->folio, idx_pos); + return 0; +} + +/* + * Sort the array elements via quicksort. This implementation incorporates + * four optimizations discussed in Sedgewick: + * + * 1. Use an explicit stack of array indices to store the next array partition + * to sort. This helps us to avoid recursion in the call stack, which is + * particularly expensive in the kernel. + * + * 2. For arrays with records in arbitrary or user-controlled order, choose the + * pivot element using a median-of-nine decision tree. This reduces the + * probability of selecting a bad pivot value which causes worst case + * behavior (i.e. partition sizes of 1). + * + * 3. The smaller of the two sub-partitions is pushed onto the stack to start + * the next level of recursion, and the larger sub-partition replaces the + * current stack frame. This guarantees that we won't need more than + * log2(nr) stack space. + * + * 4. For small sets, load the records into the scratchpad and run heapsort on + * them because that is very fast. In the author's experience, this yields + * a ~10% reduction in runtime. + * + * If a small set is contained entirely within a single xfile memory page, + * map the page directly and run heap sort directly on the xfile page + * instead of using the load/store interface. This halves the runtime. + * + * 5. This optimization is specific to the implementation. When converging lo + * and hi after selecting a pivot, we will try to retain the xfile memory + * page between load calls, which reduces run time by 50%. + */ + +/* + * Due to the use of signed indices, we can only support up to 2^63 records. + * Files can only grow to 2^63 bytes, so this is not much of a limitation. + */ +#define QSORT_MAX_RECS (1ULL << 63) + +int +xfarray_sort( + struct xfarray *array, + xfarray_cmp_fn cmp_fn, + unsigned int flags) +{ + struct xfarray_sortinfo *si; + xfarray_idx_t *si_lo, *si_hi; + void *pivot; + void *scratch = xfarray_scratch(array); + xfarray_idx_t lo, hi; + int error = 0; + + if (array->nr < 2) + return 0; + if (array->nr >= QSORT_MAX_RECS) + return -E2BIG; + + error = xfarray_sortinfo_alloc(array, cmp_fn, flags, &si); + if (error) + return error; + si_lo = xfarray_sortinfo_lo(si); + si_hi = xfarray_sortinfo_hi(si); + pivot = xfarray_sortinfo_pivot(si); + + while (si->stack_depth >= 0) { + int ret; + + lo = si_lo[si->stack_depth]; + hi = si_hi[si->stack_depth]; + + trace_xfarray_qsort(si, lo, hi); + + /* Nothing left in this partition to sort; pop stack. */ + if (lo >= hi) { + si->stack_depth--; + continue; + } + + /* + * If directly mapping the folio and sorting can solve our + * problems, we're done. + */ + ret = xfarray_foliosort(si, lo, hi); + if (ret < 0) + goto out_free; + if (ret == 1) { + si->stack_depth--; + continue; + } + + /* If insertion sort can solve our problems, we're done. */ + if (xfarray_want_isort(si, lo, hi)) { + error = xfarray_isort(si, lo, hi); + if (error) + goto out_free; + si->stack_depth--; + continue; + } + + /* Pick a pivot, move it to a[lo] and stash it. */ + error = xfarray_qsort_pivot(si, lo, hi); + if (error) + goto out_free; + + /* + * Rearrange a[lo..hi] such that everything smaller than the + * pivot is on the left side of the range and everything larger + * than the pivot is on the right side of the range. + */ + while (lo < hi) { + void *p; + + /* + * Decrement hi until it finds an a[hi] less than the + * pivot value. + */ + error = xfarray_sort_scan(si, hi, &p); + if (error) + goto out_free; + while (xfarray_sort_cmp(si, p, pivot) >= 0 && lo < hi) { + hi--; + error = xfarray_sort_scan(si, hi, &p); + if (error) + goto out_free; + } + if (p != scratch) + memcpy(scratch, p, si->array->obj_size); + xfarray_sort_scan_done(si); + if (xfarray_sort_terminated(si, &error)) + goto out_free; + + /* Copy that item (a[hi]) to a[lo]. */ + if (lo < hi) { + error = xfarray_sort_store(si, lo++, scratch); + if (error) + goto out_free; + } + + /* + * Increment lo until it finds an a[lo] greater than + * the pivot value. + */ + error = xfarray_sort_scan(si, lo, &p); + if (error) + goto out_free; + while (xfarray_sort_cmp(si, p, pivot) <= 0 && lo < hi) { + lo++; + error = xfarray_sort_scan(si, lo, &p); + if (error) + goto out_free; + } + if (p != scratch) + memcpy(scratch, p, si->array->obj_size); + xfarray_sort_scan_done(si); + if (xfarray_sort_terminated(si, &error)) + goto out_free; + + /* Copy that item (a[lo]) to a[hi]. */ + if (lo < hi) { + error = xfarray_sort_store(si, hi--, scratch); + if (error) + goto out_free; + } + + if (xfarray_sort_terminated(si, &error)) + goto out_free; + } + + /* + * Put our pivot value in the correct place at a[lo]. All + * values between a[beg[i]] and a[lo - 1] should be less than + * the pivot; and all values between a[lo + 1] and a[end[i]-1] + * should be greater than the pivot. + */ + error = xfarray_sort_store(si, lo, pivot); + if (error) + goto out_free; + + /* Set up the stack frame to process the two partitions. */ + error = xfarray_qsort_push(si, si_lo, si_hi, lo, hi); + if (error) + goto out_free; + + if (xfarray_sort_terminated(si, &error)) + goto out_free; + } + +out_free: + trace_xfarray_sort_stats(si, error); + xfarray_sort_scan_done(si); + kvfree(si); + return error; +} + +/* How many bytes is this array consuming? */ +unsigned long long +xfarray_bytes( + struct xfarray *array) +{ + return xfile_bytes(array->xfile); +} + +/* Empty the entire array. */ +void +xfarray_truncate( + struct xfarray *array) +{ + xfile_discard(array->xfile, 0, MAX_LFS_FILESIZE); + array->nr = 0; +} diff --git a/fs/xfs/scrub/xfarray.h b/fs/xfs/scrub/xfarray.h new file mode 100644 index 000000000..5eeeeed13 --- /dev/null +++ b/fs/xfs/scrub/xfarray.h @@ -0,0 +1,193 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2021-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_XFARRAY_H__ +#define __XFS_SCRUB_XFARRAY_H__ + +/* xfile array index type, along with cursor initialization */ +typedef uint64_t xfarray_idx_t; +#define XFARRAY_NULLIDX ((__force xfarray_idx_t)-1ULL) +#define XFARRAY_CURSOR_INIT ((__force xfarray_idx_t)0) + +/* Iterate each index of an xfile array. */ +#define foreach_xfarray_idx(array, idx) \ + for ((idx) = XFARRAY_CURSOR_INIT; \ + (idx) < xfarray_length(array); \ + (idx)++) + +struct xfarray { + /* Underlying file that backs the array. */ + struct xfile *xfile; + + /* Number of array elements. */ + xfarray_idx_t nr; + + /* Maximum possible array size. */ + xfarray_idx_t max_nr; + + /* Number of unset slots in the array below @nr. */ + uint64_t unset_slots; + + /* Size of an array element. */ + size_t obj_size; + + /* log2 of array element size, if possible. */ + int obj_size_log; +}; + +int xfarray_create(const char *descr, unsigned long long required_capacity, + size_t obj_size, struct xfarray **arrayp); +void xfarray_destroy(struct xfarray *array); +int xfarray_load(struct xfarray *array, xfarray_idx_t idx, void *ptr); +int xfarray_unset(struct xfarray *array, xfarray_idx_t idx); +int xfarray_store(struct xfarray *array, xfarray_idx_t idx, const void *ptr); +int xfarray_store_anywhere(struct xfarray *array, const void *ptr); +bool xfarray_element_is_null(struct xfarray *array, const void *ptr); +void xfarray_truncate(struct xfarray *array); +unsigned long long xfarray_bytes(struct xfarray *array); + +/* + * Load an array element, but zero the buffer if there's no data because we + * haven't stored to that array element yet. + */ +static inline int +xfarray_load_sparse( + struct xfarray *array, + uint64_t idx, + void *rec) +{ + int error = xfarray_load(array, idx, rec); + + if (error == -ENODATA) { + memset(rec, 0, array->obj_size); + return 0; + } + return error; +} + +/* Append an element to the array. */ +static inline int xfarray_append(struct xfarray *array, const void *ptr) +{ + return xfarray_store(array, array->nr, ptr); +} + +uint64_t xfarray_length(struct xfarray *array); +int xfarray_load_next(struct xfarray *array, xfarray_idx_t *idx, void *rec); + +/* + * Iterate the non-null elements in a sparse xfarray. Callers should + * initialize *idx to XFARRAY_CURSOR_INIT before the first call; on return, it + * will be set to one more than the index of the record that was retrieved. + * Returns 1 if a record was retrieved, 0 if there weren't any more records, or + * a negative errno. + */ +static inline int +xfarray_iter( + struct xfarray *array, + xfarray_idx_t *idx, + void *rec) +{ + int ret = xfarray_load_next(array, idx, rec); + + if (ret == -ENODATA) + return 0; + if (ret == 0) + return 1; + return ret; +} + +/* Declarations for xfile array sort functionality. */ + +typedef cmp_func_t xfarray_cmp_fn; + +/* Perform an in-memory heapsort for small subsets. */ +#define XFARRAY_ISORT_SHIFT (4) +#define XFARRAY_ISORT_NR (1U << XFARRAY_ISORT_SHIFT) + +/* Evalulate this many points to find the qsort pivot. */ +#define XFARRAY_QSORT_PIVOT_NR (9) + +struct xfarray_sortinfo { + struct xfarray *array; + + /* Comparison function for the sort. */ + xfarray_cmp_fn cmp_fn; + + /* Maximum height of the partition stack. */ + uint8_t max_stack_depth; + + /* Current height of the partition stack. */ + int8_t stack_depth; + + /* Maximum stack depth ever used. */ + uint8_t max_stack_used; + + /* XFARRAY_SORT_* flags; see below. */ + unsigned int flags; + + /* next time we want to cond_resched() */ + struct xchk_relax relax; + + /* Cache a folio here for faster scanning for pivots */ + struct folio *folio; + + /* First array index in folio that is completely readable */ + xfarray_idx_t first_folio_idx; + + /* Last array index in folio that is completely readable */ + xfarray_idx_t last_folio_idx; + +#ifdef DEBUG + /* Performance statistics. */ + uint64_t loads; + uint64_t stores; + uint64_t compares; + uint64_t heapsorts; +#endif + /* + * Extra bytes are allocated beyond the end of the structure to store + * quicksort information. C does not permit multiple VLAs per struct, + * so we document all of this in a comment. + * + * Pretend that we have a typedef for array records: + * + * typedef char[array->obj_size] xfarray_rec_t; + * + * First comes the quicksort partition stack: + * + * xfarray_idx_t lo[max_stack_depth]; + * xfarray_idx_t hi[max_stack_depth]; + * + * union { + * + * If for a given subset we decide to use an in-memory sort, we use a + * block of scratchpad records here to compare items: + * + * xfarray_rec_t scratch[ISORT_NR]; + * + * Otherwise, we want to partition the records to partition the array. + * We store the chosen pivot record at the start of the scratchpad area + * and use the rest to sample some records to estimate the median. + * The format of the qsort_pivot array enables us to use the kernel + * heapsort function to place the median value in the middle. + * + * struct { + * xfarray_rec_t pivot; + * struct { + * xfarray_rec_t rec; (rounded up to 8 bytes) + * xfarray_idx_t idx; + * } qsort_pivot[QSORT_PIVOT_NR]; + * }; + * } + */ +}; + +/* Sort can be interrupted by a fatal signal. */ +#define XFARRAY_SORT_KILLABLE (1U << 0) + +int xfarray_sort(struct xfarray *array, xfarray_cmp_fn cmp_fn, + unsigned int flags); + +#endif /* __XFS_SCRUB_XFARRAY_H__ */ diff --git a/fs/xfs/scrub/xfblob.c b/fs/xfs/scrub/xfblob.c new file mode 100644 index 000000000..84237fb41 --- /dev/null +++ b/fs/xfs/scrub/xfblob.c @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2021-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "scrub/scrub.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/xfblob.h" + +/* + * XFS Blob Storage + * ================ + * Stores and retrieves blobs using an xfile. Objects are appended to the file + * and the offset is returned as a magic cookie for retrieval. + */ + +#define XB_KEY_MAGIC 0xABAADDAD +struct xb_key { + uint32_t xb_magic; /* XB_KEY_MAGIC */ + uint32_t xb_size; /* size of the blob, in bytes */ + loff_t xb_offset; /* byte offset of this key */ + /* blob comes after here */ +} __packed; + +/* Initialize a blob storage object. */ +int +xfblob_create( + const char *description, + struct xfblob **blobp) +{ + struct xfblob *blob; + struct xfile *xfile; + int error; + + error = xfile_create(description, 0, &xfile); + if (error) + return error; + + blob = kmalloc_obj(struct xfblob, XCHK_GFP_FLAGS); + if (!blob) { + error = -ENOMEM; + goto out_xfile; + } + + blob->xfile = xfile; + blob->last_offset = PAGE_SIZE; + + *blobp = blob; + return 0; + +out_xfile: + xfile_destroy(xfile); + return error; +} + +/* Destroy a blob storage object. */ +void +xfblob_destroy( + struct xfblob *blob) +{ + xfile_destroy(blob->xfile); + kfree(blob); +} + +/* Retrieve a blob. */ +int +xfblob_load( + struct xfblob *blob, + xfblob_cookie cookie, + void *ptr, + uint32_t size) +{ + struct xb_key key; + int error; + + error = xfile_load(blob->xfile, &key, sizeof(key), cookie); + if (error) + return error; + + if (key.xb_magic != XB_KEY_MAGIC || key.xb_offset != cookie) { + ASSERT(0); + return -ENODATA; + } + if (size < key.xb_size) { + ASSERT(0); + return -EFBIG; + } + + return xfile_load(blob->xfile, ptr, key.xb_size, + cookie + sizeof(key)); +} + +/* Store a blob. */ +int +xfblob_store( + struct xfblob *blob, + xfblob_cookie *cookie, + const void *ptr, + uint32_t size) +{ + struct xb_key key = { + .xb_offset = blob->last_offset, + .xb_magic = XB_KEY_MAGIC, + .xb_size = size, + }; + loff_t pos = blob->last_offset; + int error; + + error = xfile_store(blob->xfile, &key, sizeof(key), pos); + if (error) + return error; + + pos += sizeof(key); + error = xfile_store(blob->xfile, ptr, size, pos); + if (error) + goto out_err; + + *cookie = blob->last_offset; + blob->last_offset += sizeof(key) + size; + return 0; +out_err: + xfile_discard(blob->xfile, blob->last_offset, sizeof(key)); + return error; +} + +/* Free a blob. */ +int +xfblob_free( + struct xfblob *blob, + xfblob_cookie cookie) +{ + struct xb_key key; + int error; + + error = xfile_load(blob->xfile, &key, sizeof(key), cookie); + if (error) + return error; + + if (key.xb_magic != XB_KEY_MAGIC || key.xb_offset != cookie) { + ASSERT(0); + return -ENODATA; + } + + xfile_discard(blob->xfile, cookie, sizeof(key) + key.xb_size); + return 0; +} + +/* How many bytes is this blob storage object consuming? */ +unsigned long long +xfblob_bytes( + struct xfblob *blob) +{ + return xfile_bytes(blob->xfile); +} + +/* Drop all the blobs. */ +void +xfblob_truncate( + struct xfblob *blob) +{ + xfile_discard(blob->xfile, PAGE_SIZE, MAX_LFS_FILESIZE - PAGE_SIZE); + blob->last_offset = PAGE_SIZE; +} diff --git a/fs/xfs/scrub/xfblob.h b/fs/xfs/scrub/xfblob.h new file mode 100644 index 000000000..ae7832261 --- /dev/null +++ b/fs/xfs/scrub/xfblob.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2021-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_XFBLOB_H__ +#define __XFS_SCRUB_XFBLOB_H__ + +struct xfblob { + struct xfile *xfile; + loff_t last_offset; +}; + +typedef loff_t xfblob_cookie; + +int xfblob_create(const char *descr, struct xfblob **blobp); +void xfblob_destroy(struct xfblob *blob); +int xfblob_load(struct xfblob *blob, xfblob_cookie cookie, void *ptr, + uint32_t size); +int xfblob_store(struct xfblob *blob, xfblob_cookie *cookie, const void *ptr, + uint32_t size); +int xfblob_free(struct xfblob *blob, xfblob_cookie cookie); +unsigned long long xfblob_bytes(struct xfblob *blob); +void xfblob_truncate(struct xfblob *blob); + +static inline int +xfblob_storename( + struct xfblob *blob, + xfblob_cookie *cookie, + const struct xfs_name *xname) +{ + return xfblob_store(blob, cookie, xname->name, xname->len); +} + +static inline int +xfblob_loadname( + struct xfblob *blob, + xfblob_cookie cookie, + struct xfs_name *xname, + uint32_t size) +{ + int ret = xfblob_load(blob, cookie, (void *)xname->name, size); + if (ret) + return ret; + + xname->len = size; + return 0; +} + +#endif /* __XFS_SCRUB_XFBLOB_H__ */ diff --git a/fs/xfs/scrub/xfile.c b/fs/xfs/scrub/xfile.c new file mode 100644 index 000000000..055815718 --- /dev/null +++ b/fs/xfs/scrub/xfile.c @@ -0,0 +1,325 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "scrub/scrub.h" +#include "scrub/xfile.h" +#include "scrub/xfarray.h" +#include "scrub/trace.h" +#include + +/* + * Swappable Temporary Memory + * ========================== + * + * Online checking sometimes needs to be able to stage a large amount of data + * in memory. This information might not fit in the available memory and it + * doesn't all need to be accessible at all times. In other words, we want an + * indexed data buffer to store data that can be paged out. + * + * When CONFIG_TMPFS=y, shmemfs is enough of a filesystem to meet those + * requirements. Therefore, the xfile mechanism uses an unlinked shmem file to + * store our staging data. This file is not installed in the file descriptor + * table so that user programs cannot access the data, which means that the + * xfile must be freed with xfile_destroy. + * + * xfiles assume that the caller will handle all required concurrency + * management; standard vfs locks (freezer and inode) are not taken. Reads + * and writes are satisfied directly from the page cache. + */ + +/* + * xfiles must not be exposed to userspace and require upper layers to + * coordinate access to the one handle returned by the constructor, so + * establish a separate lock class for xfiles to avoid confusing lockdep. + */ +static struct lock_class_key xfile_i_mutex_key; + +/* + * Create an xfile of the given size. The description will be used in the + * trace output. + */ +int +xfile_create( + const char *description, + loff_t isize, + struct xfile **xfilep) +{ + struct inode *inode; + struct xfile *xf; + int error; + + xf = kmalloc_obj(struct xfile, XCHK_GFP_FLAGS); + if (!xf) + return -ENOMEM; + + xf->file = shmem_kernel_file_setup(description, isize, + mk_vma_flags(VMA_NORESERVE_BIT)); + if (IS_ERR(xf->file)) { + error = PTR_ERR(xf->file); + goto out_xfile; + } + + inode = file_inode(xf->file); + lockdep_set_class(&inode->i_rwsem, &xfile_i_mutex_key); + + /* + * We don't want to bother with kmapping data during repair, so don't + * allow highmem pages to back this mapping. + */ + mapping_set_gfp_mask(inode->i_mapping, GFP_KERNEL); + + trace_xfile_create(xf); + + *xfilep = xf; + return 0; +out_xfile: + kfree(xf); + return error; +} + +/* Close the file and release all resources. */ +void +xfile_destroy( + struct xfile *xf) +{ + struct inode *inode = file_inode(xf->file); + + trace_xfile_destroy(xf); + + lockdep_set_class(&inode->i_rwsem, &inode->i_sb->s_type->i_mutex_key); + fput(xf->file); + kfree(xf); +} + +/* + * Load an object. Since we're treating this file as "memory", any error or + * short IO is treated as a failure to allocate memory. + */ +int +xfile_load( + struct xfile *xf, + void *buf, + size_t count, + loff_t pos) +{ + struct inode *inode = file_inode(xf->file); + unsigned int pflags; + + if (count > MAX_RW_COUNT) + return -ENOMEM; + if (inode->i_sb->s_maxbytes - pos < count) + return -ENOMEM; + + trace_xfile_load(xf, pos, count); + + pflags = memalloc_nofs_save(); + while (count > 0) { + struct folio *folio; + unsigned int len; + unsigned int offset; + + if (shmem_get_folio(inode, pos >> PAGE_SHIFT, 0, &folio, + SGP_READ) < 0) + break; + if (!folio) { + /* + * No data stored at this offset, just zero the output + * buffer until the next page boundary. + */ + len = min_t(ssize_t, count, + PAGE_SIZE - offset_in_page(pos)); + memset(buf, 0, len); + } else { + if (filemap_check_wb_err(inode->i_mapping, 0)) { + folio_unlock(folio); + folio_put(folio); + break; + } + + offset = offset_in_folio(folio, pos); + len = min_t(ssize_t, count, folio_size(folio) - offset); + memcpy(buf, folio_address(folio) + offset, len); + + folio_unlock(folio); + folio_put(folio); + } + count -= len; + pos += len; + buf += len; + } + memalloc_nofs_restore(pflags); + + if (count) + return -ENOMEM; + return 0; +} + +/* + * Store an object. Since we're treating this file as "memory", any error or + * short IO is treated as a failure to allocate memory. + */ +int +xfile_store( + struct xfile *xf, + const void *buf, + size_t count, + loff_t pos) +{ + struct inode *inode = file_inode(xf->file); + unsigned int pflags; + + if (count > MAX_RW_COUNT) + return -ENOMEM; + if (inode->i_sb->s_maxbytes - pos < count) + return -ENOMEM; + + trace_xfile_store(xf, pos, count); + + /* + * Increase the file size first so that shmem_get_folio(..., SGP_CACHE), + * actually allocates a folio instead of erroring out. + */ + if (pos + count > i_size_read(inode)) + i_size_write(inode, pos + count); + + pflags = memalloc_nofs_save(); + while (count > 0) { + struct folio *folio; + unsigned int len; + unsigned int offset; + + if (shmem_get_folio(inode, pos >> PAGE_SHIFT, 0, &folio, + SGP_CACHE) < 0) + break; + if (filemap_check_wb_err(inode->i_mapping, 0)) { + folio_unlock(folio); + folio_put(folio); + break; + } + + offset = offset_in_folio(folio, pos); + len = min_t(ssize_t, count, folio_size(folio) - offset); + memcpy(folio_address(folio) + offset, buf, len); + + folio_mark_dirty(folio); + folio_unlock(folio); + folio_put(folio); + + count -= len; + pos += len; + buf += len; + } + memalloc_nofs_restore(pflags); + + if (count) + return -ENOMEM; + return 0; +} + +/* Find the next written area in the xfile data for a given offset. */ +loff_t +xfile_seek_data( + struct xfile *xf, + loff_t pos) +{ + loff_t ret; + + ret = vfs_llseek(xf->file, pos, SEEK_DATA); + trace_xfile_seek_data(xf, pos, ret); + return ret; +} + +/* + * Grab the (locked) folio for a memory object. The object cannot span a folio + * boundary. Returns the locked folio if successful, NULL if there was no + * folio or it didn't cover the range requested, or an ERR_PTR on failure. + */ +struct folio * +xfile_get_folio( + struct xfile *xf, + loff_t pos, + size_t len, + unsigned int flags) +{ + struct inode *inode = file_inode(xf->file); + struct folio *folio = NULL; + unsigned int pflags; + int error; + + if (inode->i_sb->s_maxbytes - pos < len) + return ERR_PTR(-ENOMEM); + + trace_xfile_get_folio(xf, pos, len); + + /* + * Increase the file size first so that shmem_get_folio(..., SGP_CACHE), + * actually allocates a folio instead of erroring out. + */ + if ((flags & XFILE_ALLOC) && pos + len > i_size_read(inode)) + i_size_write(inode, pos + len); + + pflags = memalloc_nofs_save(); + error = shmem_get_folio(inode, pos >> PAGE_SHIFT, 0, &folio, + (flags & XFILE_ALLOC) ? SGP_CACHE : SGP_READ); + memalloc_nofs_restore(pflags); + if (error) + return ERR_PTR(error); + + if (!folio) + return NULL; + + if (len > folio_size(folio) - offset_in_folio(folio, pos)) { + folio_unlock(folio); + folio_put(folio); + return NULL; + } + + if (filemap_check_wb_err(inode->i_mapping, 0)) { + folio_unlock(folio); + folio_put(folio); + return ERR_PTR(-EIO); + } + + /* + * Mark the folio dirty so that it won't be reclaimed once we drop the + * (potentially last) reference in xfile_put_folio. + */ + if (flags & XFILE_ALLOC) + folio_mark_dirty(folio); + return folio; +} + +/* + * Release the (locked) folio for a memory object. + */ +void +xfile_put_folio( + struct xfile *xf, + struct folio *folio) +{ + trace_xfile_put_folio(xf, folio_pos(folio), folio_size(folio)); + + folio_unlock(folio); + folio_put(folio); +} + +/* Discard the page cache that's backing a range of the xfile. */ +void +xfile_discard( + struct xfile *xf, + loff_t pos, + u64 count) +{ + trace_xfile_discard(xf, pos, count); + + shmem_truncate_range(file_inode(xf->file), pos, pos + count - 1); +} diff --git a/fs/xfs/scrub/xfile.h b/fs/xfs/scrub/xfile.h new file mode 100644 index 000000000..cc2cc1714 --- /dev/null +++ b/fs/xfs/scrub/xfile.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2018-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_XFILE_H__ +#define __XFS_SCRUB_XFILE_H__ + +struct xfile { + struct file *file; +}; + +int xfile_create(const char *description, loff_t isize, struct xfile **xfilep); +void xfile_destroy(struct xfile *xf); + +int xfile_load(struct xfile *xf, void *buf, size_t count, loff_t pos); +int xfile_store(struct xfile *xf, const void *buf, size_t count, + loff_t pos); + +void xfile_discard(struct xfile *xf, loff_t pos, u64 count); +loff_t xfile_seek_data(struct xfile *xf, loff_t pos); + +#define XFILE_MAX_FOLIO_SIZE (PAGE_SIZE << MAX_PAGECACHE_ORDER) + +#define XFILE_ALLOC (1 << 0) /* allocate folio if not present */ +struct folio *xfile_get_folio(struct xfile *xf, loff_t offset, size_t len, + unsigned int flags); +void xfile_put_folio(struct xfile *xf, struct folio *folio); + +static inline unsigned long long xfile_bytes(struct xfile *xf) +{ + return file_inode(xf->file)->i_blocks << SECTOR_SHIFT; +} + +#endif /* __XFS_SCRUB_XFILE_H__ */ diff --git a/fs/xfs/scrub/xfs_scrub.h b/fs/xfs/scrub/xfs_scrub.h new file mode 100644 index 000000000..f17173b83 --- /dev/null +++ b/fs/xfs/scrub/xfs_scrub.h @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_SCRUB_H__ +#define __XFS_SCRUB_H__ + +#ifndef CONFIG_XFS_ONLINE_SCRUB +# define xfs_ioc_scrub_metadata(f, a) (-ENOTTY) +# define xfs_ioc_scrubv_metadata(f, a) (-ENOTTY) +#else +int xfs_ioc_scrub_metadata(struct file *file, void __user *arg); +int xfs_ioc_scrubv_metadata(struct file *file, void __user *arg); +#endif /* CONFIG_XFS_ONLINE_SCRUB */ + +#endif /* __XFS_SCRUB_H__ */ diff --git a/fs/xfs/xfs_acl.c b/fs/xfs/xfs_acl.c new file mode 100644 index 000000000..fdfca6fc7 --- /dev/null +++ b/fs/xfs/xfs_acl.c @@ -0,0 +1,294 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2008, Christoph Hellwig + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_attr.h" +#include "xfs_trace.h" +#include "xfs_error.h" +#include "xfs_acl.h" +#include "xfs_trans.h" +#include "xfs_xattr.h" + +#include + +/* + * Locking scheme: + * - all ACL updates are protected by inode->i_mutex, which is taken before + * calling into this file. + */ + +STATIC struct posix_acl * +xfs_acl_from_disk( + struct xfs_mount *mp, + const struct xfs_acl *aclp, + int len, + int max_entries) +{ + struct posix_acl_entry *acl_e; + struct posix_acl *acl; + const struct xfs_acl_entry *ace; + unsigned int count, i; + + if (len < sizeof(*aclp)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, aclp, + len); + return ERR_PTR(-EFSCORRUPTED); + } + + count = be32_to_cpu(aclp->acl_cnt); + if (count > max_entries || XFS_ACL_SIZE(count) != len) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, aclp, + len); + return ERR_PTR(-EFSCORRUPTED); + } + + acl = posix_acl_alloc(count, GFP_KERNEL); + if (!acl) + return ERR_PTR(-ENOMEM); + + for (i = 0; i < count; i++) { + acl_e = &acl->a_entries[i]; + ace = &aclp->acl_entry[i]; + + /* + * The tag is 32 bits on disk and 16 bits in core. + * + * Because every access to it goes through the core + * format first this is not a problem. + */ + acl_e->e_tag = be32_to_cpu(ace->ae_tag); + acl_e->e_perm = be16_to_cpu(ace->ae_perm); + + switch (acl_e->e_tag) { + case ACL_USER: + acl_e->e_uid = make_kuid(&init_user_ns, + be32_to_cpu(ace->ae_id)); + break; + case ACL_GROUP: + acl_e->e_gid = make_kgid(&init_user_ns, + be32_to_cpu(ace->ae_id)); + break; + case ACL_USER_OBJ: + case ACL_GROUP_OBJ: + case ACL_MASK: + case ACL_OTHER: + break; + default: + goto fail; + } + } + return acl; + +fail: + posix_acl_release(acl); + return ERR_PTR(-EINVAL); +} + +STATIC void +xfs_acl_to_disk(struct xfs_acl *aclp, const struct posix_acl *acl) +{ + const struct posix_acl_entry *acl_e; + struct xfs_acl_entry *ace; + int i; + + aclp->acl_cnt = cpu_to_be32(acl->a_count); + for (i = 0; i < acl->a_count; i++) { + ace = &aclp->acl_entry[i]; + acl_e = &acl->a_entries[i]; + + ace->ae_tag = cpu_to_be32(acl_e->e_tag); + switch (acl_e->e_tag) { + case ACL_USER: + ace->ae_id = cpu_to_be32( + from_kuid(&init_user_ns, acl_e->e_uid)); + break; + case ACL_GROUP: + ace->ae_id = cpu_to_be32( + from_kgid(&init_user_ns, acl_e->e_gid)); + break; + default: + ace->ae_id = cpu_to_be32(ACL_UNDEFINED_ID); + break; + } + + ace->ae_perm = cpu_to_be16(acl_e->e_perm); + } +} + +struct posix_acl * +xfs_get_acl(struct inode *inode, int type, bool rcu) +{ + struct xfs_inode *ip = XFS_I(inode); + struct xfs_mount *mp = ip->i_mount; + struct posix_acl *acl = NULL; + struct xfs_da_args args = { + .dp = ip, + .attr_filter = XFS_ATTR_ROOT, + .valuelen = XFS_ACL_MAX_SIZE(mp), + }; + int error; + + if (rcu) + return ERR_PTR(-ECHILD); + + trace_xfs_get_acl(ip); + + switch (type) { + case ACL_TYPE_ACCESS: + args.name = SGI_ACL_FILE; + break; + case ACL_TYPE_DEFAULT: + args.name = SGI_ACL_DEFAULT; + break; + default: + BUG(); + } + args.namelen = strlen(args.name); + + /* + * If the attribute doesn't exist make sure we have a negative cache + * entry, for any other error assume it is transient. + */ + error = xfs_attr_get(&args); + if (!error) { + acl = xfs_acl_from_disk(mp, args.value, args.valuelen, + XFS_ACL_MAX_ENTRIES(mp)); + } else if (error != -ENOATTR) { + acl = ERR_PTR(error); + } + + kvfree(args.value); + return acl; +} + +int +__xfs_set_acl(struct inode *inode, struct posix_acl *acl, int type) +{ + struct xfs_inode *ip = XFS_I(inode); + struct xfs_da_args args = { + .dp = ip, + .attr_filter = XFS_ATTR_ROOT, + }; + int error; + + switch (type) { + case ACL_TYPE_ACCESS: + args.name = SGI_ACL_FILE; + break; + case ACL_TYPE_DEFAULT: + if (!S_ISDIR(inode->i_mode)) + return acl ? -EACCES : 0; + args.name = SGI_ACL_DEFAULT; + break; + default: + return -EINVAL; + } + args.namelen = strlen(args.name); + + if (acl) { + args.valuelen = XFS_ACL_SIZE(acl->a_count); + args.value = kvzalloc(args.valuelen, GFP_KERNEL); + if (!args.value) + return -ENOMEM; + xfs_acl_to_disk(args.value, acl); + error = xfs_attr_change(&args, XFS_ATTRUPDATE_UPSERT); + kvfree(args.value); + } else { + error = xfs_attr_change(&args, XFS_ATTRUPDATE_REMOVE); + /* + * If the attribute didn't exist to start with that's fine. + */ + if (error == -ENOATTR) + error = 0; + } + + if (!error) + set_cached_acl(inode, type, acl); + return error; +} + +static int +xfs_acl_set_mode( + struct inode *inode, + umode_t mode) +{ + struct xfs_inode *ip = XFS_I(inode); + struct xfs_mount *mp = ip->i_mount; + struct xfs_trans *tp; + int error; + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_ichange, 0, 0, 0, &tp); + if (error) + return error; + + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL); + inode->i_mode = mode; + inode_set_ctime_current(inode); + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + + if (xfs_has_wsync(mp)) + xfs_trans_set_sync(tp); + return xfs_trans_commit(tp); +} + +int +xfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type) +{ + umode_t mode; + bool set_mode = false; + int error = 0; + struct inode *inode = d_inode(dentry); + + if (!acl) + goto set_acl; + + error = -E2BIG; + if (acl->a_count > XFS_ACL_MAX_ENTRIES(XFS_M(inode->i_sb))) + return error; + + if (type == ACL_TYPE_ACCESS) { + error = posix_acl_update_mode(idmap, inode, &mode, &acl); + if (error) + return error; + set_mode = true; + } + + set_acl: + /* + * We set the mode after successfully updating the ACL xattr because the + * xattr update can fail at ENOSPC and we don't want to change the mode + * if the ACL update hasn't been applied. + */ + error = __xfs_set_acl(inode, acl, type); + if (!error && set_mode && mode != inode->i_mode) + error = xfs_acl_set_mode(inode, mode); + return error; +} + +/* + * Invalidate any cached ACLs if the user has bypassed the ACL interface. + * We don't validate the content whatsoever so it is caller responsibility to + * provide data in valid format and ensure i_mode is consistent. + */ +void +xfs_forget_acl( + struct inode *inode, + const char *name) +{ + if (!strcmp(name, SGI_ACL_FILE)) + forget_cached_acl(inode, ACL_TYPE_ACCESS); + else if (!strcmp(name, SGI_ACL_DEFAULT)) + forget_cached_acl(inode, ACL_TYPE_DEFAULT); +} diff --git a/fs/xfs/xfs_acl.h b/fs/xfs/xfs_acl.h new file mode 100644 index 000000000..bf7f96099 --- /dev/null +++ b/fs/xfs/xfs_acl.h @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2001-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_ACL_H__ +#define __XFS_ACL_H__ + +struct inode; +struct posix_acl; + +#ifdef CONFIG_XFS_POSIX_ACL +extern struct posix_acl *xfs_get_acl(struct inode *inode, int type, bool rcu); +extern int xfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, + struct posix_acl *acl, int type); +extern int __xfs_set_acl(struct inode *inode, struct posix_acl *acl, int type); +void xfs_forget_acl(struct inode *inode, const char *name); +#else +#define xfs_get_acl NULL +#define xfs_set_acl NULL +static inline int __xfs_set_acl(struct inode *inode, struct posix_acl *acl, + int type) +{ + return 0; +} +static inline void xfs_forget_acl(struct inode *inode, const char *name) +{ +} +#endif /* CONFIG_XFS_POSIX_ACL */ + +#endif /* __XFS_ACL_H__ */ diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c new file mode 100644 index 000000000..74a6089ab --- /dev/null +++ b/fs/xfs/xfs_aops.c @@ -0,0 +1,880 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * Copyright (c) 2016-2025 Christoph Hellwig. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_iomap.h" +#include "xfs_trace.h" +#include "xfs_bmap.h" +#include "xfs_bmap_util.h" +#include "xfs_reflink.h" +#include "xfs_errortag.h" +#include "xfs_error.h" +#include "xfs_icache.h" +#include "xfs_zone_alloc.h" +#include "xfs_rtgroup.h" +#include + +struct xfs_writepage_ctx { + struct iomap_writepage_ctx ctx; + unsigned int data_seq; + unsigned int cow_seq; +}; + +static inline struct xfs_writepage_ctx * +XFS_WPC(struct iomap_writepage_ctx *ctx) +{ + return container_of(ctx, struct xfs_writepage_ctx, ctx); +} + +/* + * Fast and loose check if this write could update the on-disk inode size. + */ +static inline bool xfs_ioend_is_append(struct iomap_ioend *ioend) +{ + return ioend->io_offset + ioend->io_size > + XFS_I(ioend->io_inode)->i_disk_size; +} + +/* + * Update on-disk file size now that data has been written to disk. + */ +int +xfs_setfilesize( + struct xfs_inode *ip, + xfs_off_t offset, + size_t size) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_trans *tp; + xfs_fsize_t isize; + int error; + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_fsyncts, 0, 0, 0, &tp); + if (error) + return error; + + xfs_ilock(ip, XFS_ILOCK_EXCL); + isize = xfs_new_eof(ip, offset + size); + if (!isize) { + xfs_iunlock(ip, XFS_ILOCK_EXCL); + xfs_trans_cancel(tp); + return 0; + } + + trace_xfs_setfilesize(ip, offset, size); + + ip->i_disk_size = isize; + xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL); + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + + return xfs_trans_commit(tp); +} + +static void +xfs_ioend_put_open_zones( + struct iomap_ioend *ioend) +{ + struct iomap_ioend *tmp; + + /* + * Put the open zone for all ioends merged into this one (if any). + */ + list_for_each_entry(tmp, &ioend->io_list, io_list) + xfs_open_zone_put(tmp->io_private); + + /* + * The main ioend might not have an open zone if the submission failed + * before xfs_zone_alloc_and_submit got called. + */ + if (ioend->io_private) + xfs_open_zone_put(ioend->io_private); +} + +/* + * IO write completion. + */ +STATIC void +xfs_end_ioend_write( + struct iomap_ioend *ioend) +{ + struct xfs_inode *ip = XFS_I(ioend->io_inode); + struct xfs_mount *mp = ip->i_mount; + bool is_zoned = xfs_is_zoned_inode(ip); + xfs_off_t offset = ioend->io_offset; + size_t size = ioend->io_size; + unsigned int nofs_flag; + int error; + + /* + * We can allocate memory here while doing writeback on behalf of + * memory reclaim. To avoid memory allocation deadlocks set the + * task-wide nofs context for the following operations. + */ + nofs_flag = memalloc_nofs_save(); + + /* + * Just clean up the in-memory structures if the fs has been shut down. + */ + if (xfs_is_shutdown(mp)) { + error = -EIO; + goto done; + } + + /* + * Clean up all COW blocks and underlying data fork delalloc blocks on + * I/O error. The delalloc punch is required because this ioend was + * mapped to blocks in the COW fork and the associated pages are no + * longer dirty. If we don't remove delalloc blocks here, they become + * stale and can corrupt free space accounting on unmount. + */ + error = blk_status_to_errno(ioend->io_bio.bi_status); + if (unlikely(error)) { + /* + * Zoned writes update the in-core open zone accounting before + * I/O submission. A failed write leaves that state + * inconsistent, so shut down the filesystem instead of letting + * later writers wait forever for open zone space to become + * available. + */ + if (is_zoned) { + xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR); + goto done; + } + if (ioend->io_flags & IOMAP_IOEND_SHARED) { + ASSERT(!is_zoned); + xfs_reflink_cancel_cow_range(ip, offset, size, true); + xfs_bmap_punch_delalloc_range(ip, XFS_DATA_FORK, offset, + offset + size, NULL); + } + goto done; + } + + /* + * Success: commit the COW or unwritten blocks if needed. + */ + if (is_zoned) + error = xfs_zoned_end_io(ip, offset, size, ioend->io_sector, + ioend->io_private, NULLFSBLOCK); + else if (ioend->io_flags & IOMAP_IOEND_SHARED) + error = xfs_reflink_end_cow(ip, offset, size); + else if (ioend->io_flags & IOMAP_IOEND_UNWRITTEN) + error = xfs_iomap_write_unwritten(ip, offset, size, false); + + if (!error && + !(ioend->io_flags & IOMAP_IOEND_DIRECT) && + xfs_ioend_is_append(ioend)) + error = xfs_setfilesize(ip, offset, size); +done: + if (is_zoned) + xfs_ioend_put_open_zones(ioend); + iomap_finish_ioends(ioend, error); + memalloc_nofs_restore(nofs_flag); +} + +/* + * Finish all pending IO completions that require transactional modifications. + * + * We try to merge physical and logically contiguous ioends before completion to + * minimise the number of transactions we need to perform during IO completion. + * Both unwritten extent conversion and COW remapping need to iterate and modify + * one physical extent at a time, so we gain nothing by merging physically + * discontiguous extents here. + * + * The ioend chain length that we can be processing here is largely unbound in + * length and we may have to perform significant amounts of work on each ioend + * to complete it. Hence we have to be careful about holding the CPU for too + * long in this loop. + */ +void +xfs_end_io( + struct work_struct *work) +{ + struct xfs_inode *ip = + container_of(work, struct xfs_inode, i_ioend_work); + struct iomap_ioend *ioend; + struct list_head tmp; + unsigned long flags; + + spin_lock_irqsave(&ip->i_ioend_lock, flags); + list_replace_init(&ip->i_ioend_list, &tmp); + spin_unlock_irqrestore(&ip->i_ioend_lock, flags); + + iomap_sort_ioends(&tmp); + while ((ioend = list_first_entry_or_null(&tmp, struct iomap_ioend, + io_list))) { + list_del_init(&ioend->io_list); + iomap_ioend_try_merge(ioend, &tmp); + if (bio_op(&ioend->io_bio) == REQ_OP_READ) + iomap_finish_ioends(ioend, + blk_status_to_errno(ioend->io_bio.bi_status)); + else + xfs_end_ioend_write(ioend); + cond_resched(); + } +} + +void +xfs_end_bio( + struct bio *bio) +{ + struct iomap_ioend *ioend = iomap_ioend_from_bio(bio); + struct xfs_inode *ip = XFS_I(ioend->io_inode); + struct xfs_mount *mp = ip->i_mount; + unsigned long flags; + + /* + * For Appends record the actually written block number and set the + * boundary flag if needed. + */ + if (IS_ENABLED(CONFIG_XFS_RT) && bio_is_zone_append(bio)) { + ioend->io_sector = bio->bi_iter.bi_sector; + xfs_mark_rtg_boundary(ioend); + } + + spin_lock_irqsave(&ip->i_ioend_lock, flags); + if (list_empty(&ip->i_ioend_list)) + WARN_ON_ONCE(!queue_work(mp->m_unwritten_workqueue, + &ip->i_ioend_work)); + list_add_tail(&ioend->io_list, &ip->i_ioend_list); + spin_unlock_irqrestore(&ip->i_ioend_lock, flags); +} + +/* + * We cannot cancel the ioend directly on error. We may have already set other + * pages under writeback and hence we have to run I/O completion to mark the + * error state of the pages under writeback appropriately. + * + * If the folio has delalloc blocks on it, the caller is asking us to punch them + * out. If we don't, we can leave a stale delalloc mapping covered by a clean + * page that needs to be dirtied again before the delalloc mapping can be + * converted. This stale delalloc mapping can trip up a later direct I/O read + * operation on the same region. + * + * We prevent this by truncating away the delalloc regions on the folio. Because + * they are delalloc, we can do this without needing a transaction. Indeed - if + * we get ENOSPC errors, we have to be able to do this truncation without a + * transaction as there is no space left for block reservation (typically why + * we see a ENOSPC in writeback). + */ +static void +xfs_discard_folio( + struct folio *folio, + loff_t pos) +{ + struct xfs_inode *ip = XFS_I(folio->mapping->host); + struct xfs_mount *mp = ip->i_mount; + + if (xfs_is_shutdown(mp)) + return; + + xfs_alert_ratelimited(mp, + "page discard on page "PTR_FMT", inode 0x%llx, pos %llu.", + folio, I_INO(ip), pos); + + /* + * The end of the punch range is always the offset of the first + * byte of the next folio. Hence the end offset is only dependent on the + * folio itself and not the start offset that is passed in. + */ + xfs_bmap_punch_delalloc_range(ip, XFS_DATA_FORK, pos, + folio_next_pos(folio), NULL); +} + +/* + * Fast revalidation of the cached writeback mapping. Return true if the current + * mapping is valid, false otherwise. + */ +static bool +xfs_imap_valid( + struct iomap_writepage_ctx *wpc, + struct xfs_inode *ip, + loff_t offset) +{ + if (offset < wpc->iomap.offset || + offset >= wpc->iomap.offset + wpc->iomap.length) + return false; + /* + * If this is a COW mapping, it is sufficient to check that the mapping + * covers the offset. Be careful to check this first because the caller + * can revalidate a COW mapping without updating the data seqno. + */ + if (wpc->iomap.flags & IOMAP_F_SHARED) + return true; + + /* + * This is not a COW mapping. Check the sequence number of the data fork + * because concurrent changes could have invalidated the extent. Check + * the COW fork because concurrent changes since the last time we + * checked (and found nothing at this offset) could have added + * overlapping blocks. + */ + if (XFS_WPC(wpc)->data_seq != READ_ONCE(ip->i_df.if_seq)) { + trace_xfs_wb_data_iomap_invalid(ip, &wpc->iomap, + XFS_WPC(wpc)->data_seq, XFS_DATA_FORK); + return false; + } + if (xfs_inode_has_cow_data(ip) && + XFS_WPC(wpc)->cow_seq != READ_ONCE(ip->i_cowfp->if_seq)) { + trace_xfs_wb_cow_iomap_invalid(ip, &wpc->iomap, + XFS_WPC(wpc)->cow_seq, XFS_COW_FORK); + return false; + } + return true; +} + +static int +xfs_map_blocks( + struct iomap_writepage_ctx *wpc, + loff_t offset, + unsigned int len) +{ + struct xfs_inode *ip = XFS_I(wpc->inode); + struct xfs_mount *mp = ip->i_mount; + ssize_t count = i_blocksize(wpc->inode); + xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); + xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + count); + xfs_fileoff_t cow_fsb; + int whichfork; + struct xfs_bmbt_irec imap; + struct xfs_iext_cursor icur; + int retries = 0; + int error = 0; + unsigned int *seq; + + if (xfs_is_shutdown(mp)) + return -EIO; + + XFS_ERRORTAG_DELAY(mp, XFS_ERRTAG_WB_DELAY_MS); + + /* + * COW fork blocks can overlap data fork blocks even if the blocks + * aren't shared. COW I/O always takes precedent, so we must always + * check for overlap on reflink inodes unless the mapping is already a + * COW one, or the COW fork hasn't changed from the last time we looked + * at it. + * + * It's safe to check the COW fork if_seq here without the ILOCK because + * we've indirectly protected against concurrent updates: writeback has + * the page locked, which prevents concurrent invalidations by reflink + * and directio and prevents concurrent buffered writes to the same + * page. Changes to if_seq always happen under i_lock, which protects + * against concurrent updates and provides a memory barrier on the way + * out that ensures that we always see the current value. + */ + if (xfs_imap_valid(wpc, ip, offset)) + return 0; + + /* + * If we don't have a valid map, now it's time to get a new one for this + * offset. This will convert delayed allocations (including COW ones) + * into real extents. If we return without a valid map, it means we + * landed in a hole and we skip the block. + */ +retry: + cow_fsb = NULLFILEOFF; + whichfork = XFS_DATA_FORK; + xfs_ilock(ip, XFS_ILOCK_SHARED); + ASSERT(!xfs_need_iread_extents(&ip->i_df)); + + /* + * Check if this is offset is covered by a COW extents, and if yes use + * it directly instead of looking up anything in the data fork. + */ + if (xfs_inode_has_cow_data(ip) && + xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &imap)) + cow_fsb = imap.br_startoff; + if (cow_fsb != NULLFILEOFF && cow_fsb <= offset_fsb) { + XFS_WPC(wpc)->cow_seq = READ_ONCE(ip->i_cowfp->if_seq); + xfs_iunlock(ip, XFS_ILOCK_SHARED); + + whichfork = XFS_COW_FORK; + goto allocate_blocks; + } + + /* + * No COW extent overlap. Revalidate now that we may have updated + * ->cow_seq. If the data mapping is still valid, we're done. + */ + if (xfs_imap_valid(wpc, ip, offset)) { + xfs_iunlock(ip, XFS_ILOCK_SHARED); + return 0; + } + + /* + * If we don't have a valid map, now it's time to get a new one for this + * offset. This will convert delayed allocations (including COW ones) + * into real extents. + */ + if (!xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &icur, &imap)) + imap.br_startoff = end_fsb; /* fake a hole past EOF */ + XFS_WPC(wpc)->data_seq = READ_ONCE(ip->i_df.if_seq); + xfs_iunlock(ip, XFS_ILOCK_SHARED); + + /* landed in a hole or beyond EOF? */ + if (imap.br_startoff > offset_fsb) { + imap.br_blockcount = imap.br_startoff - offset_fsb; + imap.br_startoff = offset_fsb; + imap.br_startblock = HOLESTARTBLOCK; + imap.br_state = XFS_EXT_NORM; + } + + /* + * Truncate to the next COW extent if there is one. This is the only + * opportunity to do this because we can skip COW fork lookups for the + * subsequent blocks in the mapping; however, the requirement to treat + * the COW range separately remains. + */ + if (cow_fsb != NULLFILEOFF && + cow_fsb < imap.br_startoff + imap.br_blockcount) + imap.br_blockcount = cow_fsb - imap.br_startoff; + + /* got a delalloc extent? */ + if (imap.br_startblock != HOLESTARTBLOCK && + isnullstartblock(imap.br_startblock)) + goto allocate_blocks; + + xfs_bmbt_to_iomap(ip, &wpc->iomap, &imap, 0, 0, XFS_WPC(wpc)->data_seq); + trace_xfs_map_blocks_found(ip, offset, count, whichfork, &imap); + return 0; +allocate_blocks: + /* + * Convert a dellalloc extent to a real one. The current page is held + * locked so nothing could have removed the block backing offset_fsb, + * although it could have moved from the COW to the data fork by another + * thread. + */ + if (whichfork == XFS_COW_FORK) + seq = &XFS_WPC(wpc)->cow_seq; + else + seq = &XFS_WPC(wpc)->data_seq; + + error = xfs_bmapi_convert_delalloc(ip, whichfork, offset, + &wpc->iomap, seq); + if (error) { + /* + * If we failed to find the extent in the COW fork we might have + * raced with a COW to data fork conversion or truncate. + * Restart the lookup to catch the extent in the data fork for + * the former case, but prevent additional retries to avoid + * looping forever for the latter case. + */ + if (error == -EAGAIN && whichfork == XFS_COW_FORK && !retries++) + goto retry; + ASSERT(error != -EAGAIN); + return error; + } + + /* + * Due to merging the return real extent might be larger than the + * original delalloc one. Trim the return extent to the next COW + * boundary again to force a re-lookup. + */ + if (whichfork != XFS_COW_FORK && cow_fsb != NULLFILEOFF) { + loff_t cow_offset = XFS_FSB_TO_B(mp, cow_fsb); + + if (cow_offset < wpc->iomap.offset + wpc->iomap.length) + wpc->iomap.length = cow_offset - wpc->iomap.offset; + } + + ASSERT(wpc->iomap.offset <= offset); + ASSERT(wpc->iomap.offset + wpc->iomap.length > offset); + trace_xfs_map_blocks_alloc(ip, offset, count, whichfork, &imap); + return 0; +} + +static ssize_t +xfs_writeback_range( + struct iomap_writepage_ctx *wpc, + struct folio *folio, + u64 offset, + unsigned int len, + u64 end_pos) +{ + ssize_t ret; + + ret = xfs_map_blocks(wpc, offset, len); + if (!ret) + ret = iomap_add_to_ioend(wpc, folio, offset, end_pos, len); + if (ret < 0) + xfs_discard_folio(folio, offset); + return ret; +} + +static bool +xfs_ioend_needs_wq_completion( + struct iomap_ioend *ioend) +{ + /* Changing inode size requires a transaction. */ + if (xfs_ioend_is_append(ioend)) + return true; + + /* Extent manipulation requires a transaction. */ + if (ioend->io_flags & (IOMAP_IOEND_UNWRITTEN | IOMAP_IOEND_SHARED)) + return true; + + return false; +} + +static int +xfs_writeback_submit( + struct iomap_writepage_ctx *wpc, + int error) +{ + struct iomap_ioend *ioend = wpc->wb_ctx; + + /* + * Convert CoW extents to regular. + * + * We can allocate memory here while doing writeback on behalf of memory + * reclaim. To avoid memory allocation deadlocks, set the task-wide + * nofs context. + */ + if (!error && (ioend->io_flags & IOMAP_IOEND_SHARED)) { + unsigned int nofs_flag; + + nofs_flag = memalloc_nofs_save(); + error = xfs_reflink_convert_cow(XFS_I(ioend->io_inode), + ioend->io_offset, ioend->io_size); + memalloc_nofs_restore(nofs_flag); + } + + /* + * Send ioends that might require a transaction to the completion wq, + * and disable the block layer task completion for them as there is no + * need to defer twice. + */ + if (xfs_ioend_needs_wq_completion(ioend)) { + ioend->io_bio.bi_end_io = xfs_end_bio; + bio_clear_flag(&ioend->io_bio, BIO_COMPLETE_IN_TASK); + } + + return iomap_ioend_writeback_submit(wpc, error); +} + +static const struct iomap_writeback_ops xfs_writeback_ops = { + .writeback_range = xfs_writeback_range, + .writeback_submit = xfs_writeback_submit, +}; + +struct xfs_zoned_writepage_ctx { + struct iomap_writepage_ctx ctx; + struct xfs_open_zone *open_zone; +}; + +static inline struct xfs_zoned_writepage_ctx * +XFS_ZWPC(struct iomap_writepage_ctx *ctx) +{ + return container_of(ctx, struct xfs_zoned_writepage_ctx, ctx); +} + +static int +xfs_zoned_map_blocks( + struct iomap_writepage_ctx *wpc, + loff_t offset, + unsigned int len) +{ + struct xfs_inode *ip = XFS_I(wpc->inode); + struct xfs_mount *mp = ip->i_mount; + xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); + xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + len); + xfs_filblks_t count_fsb; + struct xfs_bmbt_irec imap, del; + struct xfs_iext_cursor icur; + + if (xfs_is_shutdown(mp)) + return -EIO; + + XFS_ERRORTAG_DELAY(mp, XFS_ERRTAG_WB_DELAY_MS); + + /* + * All dirty data must be covered by delalloc extents. But truncate can + * remove delalloc extents underneath us or reduce their size. + * Returning a hole tells iomap to not write back any data from this + * range, which is the right thing to do in that case. + * + * Otherwise just tell iomap to treat ranges previously covered by a + * delalloc extent as mapped. The actual block allocation will be done + * just before submitting the bio. + * + * This implies we never map outside folios that are locked or marked + * as under writeback, and thus there is no need check the fork sequence + * count here. + */ + xfs_ilock(ip, XFS_ILOCK_EXCL); + if (!xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &imap)) + imap.br_startoff = end_fsb; /* fake a hole past EOF */ + if (imap.br_startoff > offset_fsb) { + imap.br_blockcount = imap.br_startoff - offset_fsb; + imap.br_startoff = offset_fsb; + imap.br_startblock = HOLESTARTBLOCK; + imap.br_state = XFS_EXT_NORM; + xfs_iunlock(ip, XFS_ILOCK_EXCL); + xfs_bmbt_to_iomap(ip, &wpc->iomap, &imap, 0, 0, 0); + return 0; + } + end_fsb = min(end_fsb, imap.br_startoff + imap.br_blockcount); + count_fsb = end_fsb - offset_fsb; + + del = imap; + xfs_trim_extent(&del, offset_fsb, count_fsb); + xfs_bmap_del_extent_delay(ip, XFS_COW_FORK, &icur, &imap, &del, + XFS_BMAPI_REMAP); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + + wpc->iomap.type = IOMAP_MAPPED; + wpc->iomap.flags = IOMAP_F_DIRTY; + wpc->iomap.bdev = mp->m_rtdev_targp->bt_bdev; + wpc->iomap.offset = offset; + wpc->iomap.length = XFS_FSB_TO_B(mp, count_fsb); + wpc->iomap.flags = IOMAP_F_ANON_WRITE; + + trace_xfs_zoned_map_blocks(ip, offset, wpc->iomap.length); + return 0; +} + +static ssize_t +xfs_zoned_writeback_range( + struct iomap_writepage_ctx *wpc, + struct folio *folio, + u64 offset, + unsigned int len, + u64 end_pos) +{ + ssize_t ret; + + ret = xfs_zoned_map_blocks(wpc, offset, len); + if (!ret) + ret = iomap_add_to_ioend(wpc, folio, offset, end_pos, len); + if (ret < 0) + xfs_discard_folio(folio, offset); + return ret; +} + +static int +xfs_zoned_writeback_submit( + struct iomap_writepage_ctx *wpc, + int error) +{ + struct iomap_ioend *ioend = wpc->wb_ctx; + + /* + * Defer all completions to our workqueue as all zoned writes require a + * transaction to be persisted. This also means we never need the block + * layer in-task completion for a task context. + */ + ioend->io_bio.bi_end_io = xfs_end_bio; + bio_clear_flag(&ioend->io_bio, BIO_COMPLETE_IN_TASK); + + if (error) { + ioend->io_bio.bi_status = errno_to_blk_status(error); + bio_endio(&ioend->io_bio); + return error; + } + if (wpc->iomap.flags & IOMAP_F_INTEGRITY) + fs_bio_integrity_generate(&ioend->io_bio); + xfs_zone_alloc_and_submit(ioend, &XFS_ZWPC(wpc)->open_zone); + return 0; +} + +static const struct iomap_writeback_ops xfs_zoned_writeback_ops = { + .writeback_range = xfs_zoned_writeback_range, + .writeback_submit = xfs_zoned_writeback_submit, +}; + +STATIC int +xfs_vm_writepages( + struct address_space *mapping, + struct writeback_control *wbc) +{ + struct xfs_inode *ip = XFS_I(mapping->host); + + xfs_iflags_clear(ip, XFS_ITRUNCATED); + + if (xfs_is_zoned_inode(ip)) { + struct xfs_zoned_writepage_ctx xc = { + .ctx = { + .inode = mapping->host, + .wbc = wbc, + .ops = &xfs_zoned_writeback_ops + }, + }; + int error; + + error = iomap_writepages(&xc.ctx); + if (xc.open_zone) + xfs_open_zone_put(xc.open_zone); + return error; + } else { + struct xfs_writepage_ctx wpc = { + .ctx = { + .inode = mapping->host, + .wbc = wbc, + .ops = &xfs_writeback_ops + }, + }; + + return iomap_writepages(&wpc.ctx); + } +} + +STATIC int +xfs_dax_writepages( + struct address_space *mapping, + struct writeback_control *wbc) +{ + struct xfs_inode *ip = XFS_I(mapping->host); + + xfs_iflags_clear(ip, XFS_ITRUNCATED); + return dax_writeback_mapping_range(mapping, + xfs_inode_buftarg(ip)->bt_daxdev, wbc); +} + +STATIC sector_t +xfs_vm_bmap( + struct address_space *mapping, + sector_t block) +{ + struct xfs_inode *ip = XFS_I(mapping->host); + + trace_xfs_vm_bmap(ip); + + /* + * The swap code (ab-)uses ->bmap to get a block mapping and then + * bypasses the file system for actual I/O. We really can't allow + * that on reflinks inodes, so we have to skip out here. And yes, + * 0 is the magic code for a bmap error. + * + * Since we don't pass back blockdev info, we can't return bmap + * information for rt files either. + */ + if (xfs_is_cow_inode(ip) || XFS_IS_REALTIME_INODE(ip)) + return 0; + return iomap_bmap(mapping, block, &xfs_read_iomap_ops); +} + +static void +xfs_bio_submit_read( + const struct iomap_iter *iter, + struct iomap_read_folio_ctx *ctx) +{ + struct bio *bio = ctx->read_ctx; + + /* defer read completions to the ioend workqueue */ + iomap_init_ioend(iter->inode, bio, ctx->read_ctx_file_offset, 0); + iomap_bio_submit_read_endio(iter, ctx, xfs_end_bio); +} + +static const struct iomap_read_ops xfs_iomap_read_ops = { + .read_folio_range = iomap_bio_read_folio_range, + .submit_read = xfs_bio_submit_read, + .bio_set = &iomap_ioend_bioset, +}; + +static inline const struct iomap_read_ops * +xfs_get_iomap_read_ops( + const struct address_space *mapping) +{ + struct xfs_inode *ip = XFS_I(mapping->host); + + if (bdev_has_integrity_csum(xfs_inode_buftarg(ip)->bt_bdev)) + return &xfs_iomap_read_ops; + return &iomap_bio_read_ops; +} + +STATIC int +xfs_vm_read_folio( + struct file *file, + struct folio *folio) +{ + struct iomap_read_folio_ctx ctx = { .cur_folio = folio }; + + ctx.ops = xfs_get_iomap_read_ops(folio->mapping); + iomap_read_folio(&xfs_read_iomap_ops, &ctx, NULL); + return 0; +} + +STATIC void +xfs_vm_readahead( + struct readahead_control *rac) +{ + struct iomap_read_folio_ctx ctx = { .rac = rac }; + + ctx.ops = xfs_get_iomap_read_ops(rac->mapping), + iomap_readahead(&xfs_read_iomap_ops, &ctx, NULL); +} + +static int +xfs_vm_swap_activate( + struct swap_info_struct *sis, + struct file *swap_file, + sector_t *span) +{ + struct xfs_inode *ip = XFS_I(file_inode(swap_file)); + + if (xfs_is_zoned_inode(ip)) + return -EINVAL; + + /* + * Swap file activation can race against concurrent shared extent + * removal in files that have been cloned. If this happens, + * iomap_swapfile_iter() can fail because it encountered a shared + * extent even though an operation is in progress to remove those + * shared extents. + * + * This race becomes problematic when we defer extent removal + * operations beyond the end of a syscall (i.e. use async background + * processing algorithms). Users think the extents are no longer + * shared, but iomap_swapfile_iter() still sees them as shared + * because the refcountbt entries for the extents being removed have + * not yet been updated. Hence the swapon call fails unexpectedly. + * + * The race condition is currently most obvious from the unlink() + * operation as extent removal is deferred until after the last + * reference to the inode goes away. We then process the extent + * removal asynchronously, hence triggers the "syscall completed but + * work not done" condition mentioned above. To close this race + * window, we need to flush any pending inodegc operations to ensure + * they have updated the refcountbt records before we try to map the + * swapfile. + */ + xfs_inodegc_flush(ip->i_mount); + + /* + * Direct the swap code to the correct block device when this file + * sits on the RT device. + */ + sis->bdev = xfs_inode_buftarg(ip)->bt_bdev; + + return iomap_swapfile_activate(sis, swap_file, span, + &xfs_read_iomap_ops); +} + +const struct address_space_operations xfs_address_space_operations = { + .read_folio = xfs_vm_read_folio, + .readahead = xfs_vm_readahead, + .writepages = xfs_vm_writepages, + .dirty_folio = iomap_dirty_folio, + .release_folio = iomap_release_folio, + .invalidate_folio = iomap_invalidate_folio, + .bmap = xfs_vm_bmap, + .migrate_folio = filemap_migrate_folio, + .is_partially_uptodate = iomap_is_partially_uptodate, + .error_remove_folio = generic_error_remove_folio, + .swap_activate = xfs_vm_swap_activate, +}; + +const struct address_space_operations xfs_dax_aops = { + .writepages = xfs_dax_writepages, + .dirty_folio = noop_dirty_folio, + .swap_activate = xfs_vm_swap_activate, +}; diff --git a/fs/xfs/xfs_aops.h b/fs/xfs/xfs_aops.h new file mode 100644 index 000000000..5a7a0f1a0 --- /dev/null +++ b/fs/xfs/xfs_aops.h @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2005-2006 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_AOPS_H__ +#define __XFS_AOPS_H__ + +extern const struct address_space_operations xfs_address_space_operations; +extern const struct address_space_operations xfs_dax_aops; + +int xfs_setfilesize(struct xfs_inode *ip, xfs_off_t offset, size_t size); +void xfs_end_bio(struct bio *bio); + +#endif /* __XFS_AOPS_H__ */ diff --git a/fs/xfs/xfs_attr_inactive.c b/fs/xfs/xfs_attr_inactive.c new file mode 100644 index 000000000..a5b69c0fb --- /dev/null +++ b/fs/xfs/xfs_attr_inactive.c @@ -0,0 +1,409 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * Copyright (c) 2013 Red Hat, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_mount.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_inode.h" +#include "xfs_attr.h" +#include "xfs_attr_remote.h" +#include "xfs_trans.h" +#include "xfs_bmap.h" +#include "xfs_attr_leaf.h" +#include "xfs_quota.h" +#include "xfs_dir2.h" +#include "xfs_error.h" +#include "xfs_health.h" + +/* + * Invalidate any incore buffers associated with this remote attribute value + * extent. We never log remote attribute value buffers, which means that they + * won't be attached to a transaction and are therefore safe to mark stale. + * The actual bunmapi will be taken care of later. + */ +STATIC int +xfs_attr3_rmt_stale( + struct xfs_inode *dp, + xfs_dablk_t blkno, + int blkcnt) +{ + struct xfs_bmbt_irec map; + int nmap; + int error; + + /* + * Roll through the "value", invalidating the attribute value's + * blocks. + */ + while (blkcnt > 0) { + /* + * Try to remember where we decided to put the value. + */ + nmap = 1; + error = xfs_bmapi_read(dp, (xfs_fileoff_t)blkno, blkcnt, + &map, &nmap, XFS_BMAPI_ATTRFORK); + if (error) + return error; + if (XFS_IS_CORRUPT(dp->i_mount, nmap != 1)) + return -EFSCORRUPTED; + + /* + * Mark any incore buffers for the remote value as stale. We + * never log remote attr value buffers, so the buffer should be + * easy to kill. + */ + error = xfs_attr_rmtval_stale(dp, &map, 0); + if (error) + return error; + + blkno += map.br_blockcount; + blkcnt -= map.br_blockcount; + } + + return 0; +} + +/* + * Invalidate all of the "remote" value regions pointed to by a particular + * leaf block. + * Note that we must release the lock on the buffer so that we are not + * caught holding something that the logging code wants to flush to disk. + */ +STATIC int +xfs_attr3_leaf_inactive( + struct xfs_trans **trans, + struct xfs_inode *dp, + struct xfs_buf *bp) +{ + struct xfs_attr3_icleaf_hdr ichdr; + struct xfs_mount *mp = bp->b_mount; + struct xfs_attr_leafblock *leaf = bp->b_addr; + struct xfs_attr_leaf_entry *entry; + struct xfs_attr_leaf_name_remote *name_rmt; + int error = 0; + int i; + + xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, &ichdr, leaf); + + /* + * Find the remote value extents for this leaf and invalidate their + * incore buffers. + */ + entry = xfs_attr3_leaf_entryp(leaf); + for (i = 0; i < ichdr.count; entry++, i++) { + int blkcnt; + + if (!entry->nameidx || (entry->flags & XFS_ATTR_LOCAL)) + continue; + + name_rmt = xfs_attr3_leaf_name_remote(leaf, i); + if (!name_rmt->valueblk) + continue; + + blkcnt = xfs_attr3_rmt_blocks(dp->i_mount, + be32_to_cpu(name_rmt->valuelen)); + error = xfs_attr3_rmt_stale(dp, + be32_to_cpu(name_rmt->valueblk), blkcnt); + if (error) + goto err; + } + + xfs_trans_brelse(*trans, bp); +err: + return error; +} + +/* + * Recurse (gasp!) through the attribute nodes until we find leaves. + * We're doing a depth-first traversal in order to invalidate everything. + */ +STATIC int +xfs_attr3_node_inactive( + struct xfs_trans **trans, + struct xfs_inode *dp, + struct xfs_buf *bp, + int level) +{ + struct xfs_mount *mp = dp->i_mount; + struct xfs_da_blkinfo *info; + xfs_dablk_t child_fsb; + xfs_daddr_t parent_blkno, child_blkno; + struct xfs_buf *child_bp; + struct xfs_da3_icnode_hdr ichdr; + int error; + + /* + * Since this code is recursive (gasp!) we must protect ourselves. + */ + if (level > XFS_DA_NODE_MAXDEPTH) { + xfs_buf_mark_corrupt(bp); + xfs_trans_brelse(*trans, bp); /* no locks for later trans */ + xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); + return -EFSCORRUPTED; + } + + xfs_da3_node_hdr_from_disk(mp, &ichdr, bp->b_addr); + parent_blkno = xfs_buf_daddr(bp); + if (!ichdr.count) { + xfs_trans_brelse(*trans, bp); + return 0; + } + child_fsb = be32_to_cpu(ichdr.btree[0].before); + xfs_trans_brelse(*trans, bp); /* no locks for later trans */ + bp = NULL; + + /* + * If this is the node level just above the leaves, simply loop + * over the leaves removing all of them. If this is higher up + * in the tree, recurse downward. + */ + while (ichdr.count > 0) { + /* + * Read the subsidiary block to see what we have to work with. + * Don't do this in a transaction. This is a depth-first + * traversal of the tree so we may deal with many blocks + * before we come back to this one. + */ + error = xfs_da3_node_read(*trans, dp, child_fsb, &child_bp, + XFS_ATTR_FORK); + if (error) + return error; + + /* save for re-read later */ + child_blkno = xfs_buf_daddr(child_bp); + + /* + * Invalidate the subtree, however we have to. + */ + info = child_bp->b_addr; + switch (info->magic) { + case cpu_to_be16(XFS_DA_NODE_MAGIC): + case cpu_to_be16(XFS_DA3_NODE_MAGIC): + error = xfs_attr3_node_inactive(trans, dp, child_bp, + level + 1); + break; + case cpu_to_be16(XFS_ATTR_LEAF_MAGIC): + case cpu_to_be16(XFS_ATTR3_LEAF_MAGIC): + error = xfs_attr3_leaf_inactive(trans, dp, child_bp); + break; + default: + xfs_buf_mark_corrupt(child_bp); + xfs_trans_brelse(*trans, child_bp); + xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); + error = -EFSCORRUPTED; + break; + } + if (error) + return error; + + /* + * Remove the subsidiary block from the cache and from the log. + */ + error = xfs_trans_get_buf(*trans, mp->m_ddev_targp, + child_blkno, + XFS_FSB_TO_BB(mp, mp->m_attr_geo->fsbcount), 0, + &child_bp); + if (error) + return error; + xfs_trans_binval(*trans, child_bp); + child_bp = NULL; + + error = xfs_da3_node_read_mapped(*trans, dp, + parent_blkno, &bp, XFS_ATTR_FORK); + if (error) + return error; + + /* + * Remove entry from parent node, prevents being indexed to. + */ + xfs_attr3_node_entry_remove(*trans, dp, bp, 0); + + xfs_da3_node_hdr_from_disk(mp, &ichdr, bp->b_addr); + bp = NULL; + + if (ichdr.count > 0) { + /* + * If we're not done, get the next child block number. + */ + child_fsb = be32_to_cpu(ichdr.btree[0].before); + + /* + * Atomically commit the whole invalidate stuff. + */ + error = xfs_trans_roll_inode(trans, dp); + if (error) + return error; + } + } + + return 0; +} + +/* + * Indiscriminately delete the entire attribute fork + * + * Recurse (gasp!) through the attribute nodes until we find leaves. + * We're doing a depth-first traversal in order to invalidate everything. + */ +static int +xfs_attr3_root_inactive( + struct xfs_trans **trans, + struct xfs_inode *dp) +{ + struct xfs_da_blkinfo *info; + struct xfs_buf *bp; + int error; + + /* + * Read block 0 to see what we have to work with. + * We only get here if we have extents, since we remove + * the extents in reverse order the extent containing + * block 0 must still be there. + */ + error = xfs_da3_node_read(*trans, dp, 0, &bp, XFS_ATTR_FORK); + if (error) + return error; + + /* + * Invalidate the tree, even if the "tree" is only a single leaf block. + * This is a depth-first traversal! + */ + info = bp->b_addr; + switch (info->magic) { + case cpu_to_be16(XFS_DA_NODE_MAGIC): + case cpu_to_be16(XFS_DA3_NODE_MAGIC): + error = xfs_attr3_node_inactive(trans, dp, bp, 1); + /* + * Empty root node block are not allowed, convert it to leaf. + */ + if (!error) + error = xfs_attr3_leaf_init(*trans, dp, 0); + if (!error) + error = xfs_trans_roll_inode(trans, dp); + break; + case cpu_to_be16(XFS_ATTR_LEAF_MAGIC): + case cpu_to_be16(XFS_ATTR3_LEAF_MAGIC): + error = xfs_attr3_leaf_inactive(trans, dp, bp); + /* + * Reinit the leaf before truncating extents so that a crash + * mid-truncation leaves an empty leaf rather than one with + * entries that may reference freed remote value blocks. + */ + if (!error) + error = xfs_attr3_leaf_init(*trans, dp, 0); + if (!error) + error = xfs_trans_roll_inode(trans, dp); + break; + default: + xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); + error = -EFSCORRUPTED; + xfs_buf_mark_corrupt(bp); + xfs_trans_brelse(*trans, bp); + break; + } + + return error; +} + +/* + * xfs_attr_inactive kills all traces of an attribute fork on an inode. It + * removes both the on-disk and in-memory inode fork. Note that this also has to + * handle the condition of inodes without attributes but with an attribute fork + * configured, so we can't use xfs_inode_hasattr() here. + * + * The in-memory attribute fork is removed even on error. + */ +int +xfs_attr_inactive( + struct xfs_inode *dp) +{ + struct xfs_trans *trans; + struct xfs_mount *mp; + struct xfs_buf *bp; + int lock_mode = XFS_ILOCK_SHARED; + int error = 0; + + mp = dp->i_mount; + + xfs_ilock(dp, lock_mode); + if (!xfs_inode_has_attr_fork(dp)) + goto out_destroy_fork; + xfs_iunlock(dp, lock_mode); + + lock_mode = 0; + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_attrinval, 0, 0, 0, &trans); + if (error) + goto out_destroy_fork; + + lock_mode = XFS_ILOCK_EXCL; + xfs_ilock(dp, lock_mode); + + if (!xfs_inode_has_attr_fork(dp)) + goto out_cancel; + + /* + * No need to make quota reservations here. We expect to release some + * blocks, not allocate, in the common case. + */ + xfs_trans_ijoin(trans, dp, 0); + + /* + * Invalidate and truncate the attribute fork extents. Make sure the + * fork actually has xattr blocks as otherwise the invalidation has no + * blocks to read and returns an error. In this case, just do the fork + * removal below. + */ + if (dp->i_af.if_nextents > 0) { + /* + * Invalidate and truncate all blocks but leave the root block. + */ + error = xfs_attr3_root_inactive(&trans, dp); + if (error) + goto out_cancel; + + error = xfs_itruncate_extents(&trans, dp, XFS_ATTR_FORK, + XFS_FSB_TO_B(mp, mp->m_attr_geo->fsbcount)); + if (error) + goto out_cancel; + + /* + * Invalidate and truncate the root block and ensure that the + * operation is completed within a single transaction. + */ + error = xfs_da_get_buf(trans, dp, 0, &bp, XFS_ATTR_FORK); + if (error) + goto out_cancel; + + xfs_trans_binval(trans, bp); + error = xfs_itruncate_extents(&trans, dp, XFS_ATTR_FORK, 0); + if (error) + goto out_cancel; + } + + /* Reset the attribute fork - this also destroys the in-core fork */ + xfs_attr_fork_remove(dp, trans); + + error = xfs_trans_commit(trans); + xfs_iunlock(dp, lock_mode); + return error; + +out_cancel: + xfs_trans_cancel(trans); +out_destroy_fork: + /* kill the in-core attr fork before we drop the inode lock */ + xfs_ifork_zap_attr(dp); + if (lock_mode) + xfs_iunlock(dp, lock_mode); + return error; +} diff --git a/fs/xfs/xfs_attr_item.c b/fs/xfs/xfs_attr_item.c new file mode 100644 index 000000000..a53ee2b0c --- /dev/null +++ b/fs/xfs/xfs_attr_item.c @@ -0,0 +1,1207 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2022 Oracle. All Rights Reserved. + * Author: Allison Henderson + */ + +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_shared.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_bmap_btree.h" +#include "xfs_trans_priv.h" +#include "xfs_log.h" +#include "xfs_inode.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_attr.h" +#include "xfs_attr_item.h" +#include "xfs_trace.h" +#include "xfs_trans_space.h" +#include "xfs_errortag.h" +#include "xfs_error.h" +#include "xfs_log_priv.h" +#include "xfs_log_recover.h" +#include "xfs_parent.h" + +struct kmem_cache *xfs_attri_cache; +struct kmem_cache *xfs_attrd_cache; + +static const struct xfs_item_ops xfs_attri_item_ops; +static const struct xfs_item_ops xfs_attrd_item_ops; + +static inline struct xfs_attri_log_item *ATTRI_ITEM(struct xfs_log_item *lip) +{ + return container_of(lip, struct xfs_attri_log_item, attri_item); +} + +/* + * Shared xattr name/value buffers for logged extended attribute operations + * + * When logging updates to extended attributes, we can create quite a few + * attribute log intent items for a single xattr update. To avoid cycling the + * memory allocator and memcpy overhead, the name (and value, for setxattr) + * are kept in a refcounted object that is shared across all related log items + * and the upper-level deferred work state structure. The shared buffer has + * a control structure, followed by the name, and then the value. + */ + +static inline struct xfs_attri_log_nameval * +xfs_attri_log_nameval_get( + struct xfs_attri_log_nameval *nv) +{ + if (!refcount_inc_not_zero(&nv->refcount)) + return NULL; + return nv; +} + +static inline void +xfs_attri_log_nameval_put( + struct xfs_attri_log_nameval *nv) +{ + if (!nv) + return; + if (refcount_dec_and_test(&nv->refcount)) + kvfree(nv); +} + +static inline struct xfs_attri_log_nameval * +xfs_attri_log_nameval_alloc( + const void *name, + unsigned int name_len, + const void *new_name, + unsigned int new_name_len, + const void *value, + unsigned int value_len, + const void *new_value, + unsigned int new_value_len) +{ + struct xfs_attri_log_nameval *nv; + + /* + * This could be over 64kB in length, so we have to use kvmalloc() for + * this. But kvmalloc() utterly sucks, so we use our own version. + */ + nv = xlog_kvmalloc(sizeof(struct xfs_attri_log_nameval) + + name_len + new_name_len + value_len + + new_value_len); + + nv->name.iov_base = nv + 1; + nv->name.iov_len = name_len; + memcpy(nv->name.iov_base, name, name_len); + + if (new_name_len) { + nv->new_name.iov_base = nv->name.iov_base + name_len; + nv->new_name.iov_len = new_name_len; + memcpy(nv->new_name.iov_base, new_name, new_name_len); + } else { + nv->new_name.iov_base = NULL; + nv->new_name.iov_len = 0; + } + + if (value_len) { + nv->value.iov_base = nv->name.iov_base + name_len + new_name_len; + nv->value.iov_len = value_len; + memcpy(nv->value.iov_base, value, value_len); + } else { + nv->value.iov_base = NULL; + nv->value.iov_len = 0; + } + + if (new_value_len) { + nv->new_value.iov_base = nv->name.iov_base + name_len + + new_name_len + value_len; + nv->new_value.iov_len = new_value_len; + memcpy(nv->new_value.iov_base, new_value, new_value_len); + } else { + nv->new_value.iov_base = NULL; + nv->new_value.iov_len = 0; + } + + refcount_set(&nv->refcount, 1); + return nv; +} + +STATIC void +xfs_attri_item_free( + struct xfs_attri_log_item *attrip) +{ + kvfree(attrip->attri_item.li_lv_shadow); + xfs_attri_log_nameval_put(attrip->attri_nameval); + kmem_cache_free(xfs_attri_cache, attrip); +} + +/* + * Freeing the attrip requires that we remove it from the AIL if it has already + * been placed there. However, the ATTRI may not yet have been placed in the + * AIL when called by xfs_attri_release() from ATTRD processing due to the + * ordering of committed vs unpin operations in bulk insert operations. Hence + * the reference count to ensure only the last caller frees the ATTRI. + */ +STATIC void +xfs_attri_release( + struct xfs_attri_log_item *attrip) +{ + ASSERT(atomic_read(&attrip->attri_refcount) > 0); + if (!atomic_dec_and_test(&attrip->attri_refcount)) + return; + + xfs_trans_ail_delete(&attrip->attri_item, 0); + xfs_attri_item_free(attrip); +} + +STATIC void +xfs_attri_item_size( + struct xfs_log_item *lip, + int *nvecs, + int *nbytes) +{ + struct xfs_attri_log_item *attrip = ATTRI_ITEM(lip); + struct xfs_attri_log_nameval *nv = attrip->attri_nameval; + + *nvecs += 2; + *nbytes += sizeof(struct xfs_attri_log_format) + + xlog_calc_iovec_len(nv->name.iov_len); + + if (nv->new_name.iov_len) { + *nvecs += 1; + *nbytes += xlog_calc_iovec_len(nv->new_name.iov_len); + } + + if (nv->value.iov_len) { + *nvecs += 1; + *nbytes += xlog_calc_iovec_len(nv->value.iov_len); + } + + if (nv->new_value.iov_len) { + *nvecs += 1; + *nbytes += xlog_calc_iovec_len(nv->new_value.iov_len); + } +} + +/* + * This is called to fill in the log iovecs for the given attri log + * item. We use 1 iovec for the attri_format_item, 1 for the name, and + * another for the value if it is present + */ +STATIC void +xfs_attri_item_format( + struct xfs_log_item *lip, + struct xlog_format_buf *lfb) +{ + struct xfs_attri_log_item *attrip = ATTRI_ITEM(lip); + struct xfs_attri_log_nameval *nv = attrip->attri_nameval; + + attrip->attri_format.alfi_type = XFS_LI_ATTRI; + attrip->attri_format.alfi_size = 1; + + /* + * This size accounting must be done before copying the attrip into the + * iovec. If we do it after, the wrong size will be recorded to the log + * and we trip across assertion checks for bad region sizes later during + * the log recovery. + */ + + ASSERT(nv->name.iov_len > 0); + attrip->attri_format.alfi_size++; + + if (nv->new_name.iov_len > 0) + attrip->attri_format.alfi_size++; + + if (nv->value.iov_len > 0) + attrip->attri_format.alfi_size++; + + if (nv->new_value.iov_len > 0) + attrip->attri_format.alfi_size++; + + xlog_format_copy(lfb, XLOG_REG_TYPE_ATTRI_FORMAT, &attrip->attri_format, + sizeof(struct xfs_attri_log_format)); + + xlog_format_copy(lfb, XLOG_REG_TYPE_ATTR_NAME, nv->name.iov_base, + nv->name.iov_len); + + if (nv->new_name.iov_len > 0) + xlog_format_copy(lfb, XLOG_REG_TYPE_ATTR_NEWNAME, + nv->new_name.iov_base, nv->new_name.iov_len); + + if (nv->value.iov_len > 0) + xlog_format_copy(lfb, XLOG_REG_TYPE_ATTR_VALUE, + nv->value.iov_base, nv->value.iov_len); + + if (nv->new_value.iov_len > 0) + xlog_format_copy(lfb, XLOG_REG_TYPE_ATTR_NEWVALUE, + nv->new_value.iov_base, nv->new_value.iov_len); +} + +/* + * The unpin operation is the last place an ATTRI is manipulated in the log. It + * is either inserted in the AIL or aborted in the event of a log I/O error. In + * either case, the ATTRI transaction has been successfully committed to make + * it this far. Therefore, we expect whoever committed the ATTRI to either + * construct and commit the ATTRD or drop the ATTRD's reference in the event of + * error. Simply drop the log's ATTRI reference now that the log is done with + * it. + */ +STATIC void +xfs_attri_item_unpin( + struct xfs_log_item *lip, + int remove) +{ + xfs_attri_release(ATTRI_ITEM(lip)); +} + + +STATIC void +xfs_attri_item_release( + struct xfs_log_item *lip) +{ + xfs_attri_release(ATTRI_ITEM(lip)); +} + +/* + * Allocate and initialize an attri item. Caller may allocate an additional + * trailing buffer for name and value + */ +STATIC struct xfs_attri_log_item * +xfs_attri_init( + struct xfs_mount *mp, + struct xfs_attri_log_nameval *nv) +{ + struct xfs_attri_log_item *attrip; + + attrip = kmem_cache_zalloc(xfs_attri_cache, GFP_KERNEL | __GFP_NOFAIL); + + /* + * Grab an extra reference to the name/value buffer for this log item. + * The caller retains its own reference! + */ + attrip->attri_nameval = xfs_attri_log_nameval_get(nv); + ASSERT(attrip->attri_nameval); + + xfs_log_item_init(mp, &attrip->attri_item, XFS_LI_ATTRI, + &xfs_attri_item_ops); + attrip->attri_format.alfi_id = (uintptr_t)(void *)attrip; + atomic_set(&attrip->attri_refcount, 2); + + return attrip; +} + +static inline struct xfs_attrd_log_item *ATTRD_ITEM(struct xfs_log_item *lip) +{ + return container_of(lip, struct xfs_attrd_log_item, attrd_item); +} + +STATIC void +xfs_attrd_item_free(struct xfs_attrd_log_item *attrdp) +{ + kvfree(attrdp->attrd_item.li_lv_shadow); + kmem_cache_free(xfs_attrd_cache, attrdp); +} + +STATIC void +xfs_attrd_item_size( + struct xfs_log_item *lip, + int *nvecs, + int *nbytes) +{ + *nvecs += 1; + *nbytes += sizeof(struct xfs_attrd_log_format); +} + +/* + * This is called to fill in the log iovecs for the given attrd log item. We use + * only 1 iovec for the attrd_format, and we point that at the attr_log_format + * structure embedded in the attrd item. + */ +STATIC void +xfs_attrd_item_format( + struct xfs_log_item *lip, + struct xlog_format_buf *lfb) +{ + struct xfs_attrd_log_item *attrdp = ATTRD_ITEM(lip); + + attrdp->attrd_format.alfd_type = XFS_LI_ATTRD; + attrdp->attrd_format.alfd_size = 1; + + xlog_format_copy(lfb, XLOG_REG_TYPE_ATTRD_FORMAT, + &attrdp->attrd_format, + sizeof(struct xfs_attrd_log_format)); +} + +/* + * The ATTRD is either committed or aborted if the transaction is canceled. If + * the transaction is canceled, drop our reference to the ATTRI and free the + * ATTRD. + */ +STATIC void +xfs_attrd_item_release( + struct xfs_log_item *lip) +{ + struct xfs_attrd_log_item *attrdp = ATTRD_ITEM(lip); + + xfs_attri_release(attrdp->attrd_attrip); + xfs_attrd_item_free(attrdp); +} + +static struct xfs_log_item * +xfs_attrd_item_intent( + struct xfs_log_item *lip) +{ + return &ATTRD_ITEM(lip)->attrd_attrip->attri_item; +} + +static inline unsigned int +xfs_attr_log_item_op(const struct xfs_attri_log_format *attrp) +{ + return attrp->alfi_op_flags & XFS_ATTRI_OP_FLAGS_TYPE_MASK; +} + +/* Log an attr to the intent item. */ +STATIC void +xfs_attr_log_item( + struct xfs_trans *tp, + struct xfs_attri_log_item *attrip, + const struct xfs_attr_intent *attr) +{ + struct xfs_attri_log_format *attrp; + struct xfs_attri_log_nameval *nv = attr->xattri_nameval; + struct xfs_da_args *args = attr->xattri_da_args; + + /* + * At this point the xfs_attr_intent has been constructed, and we've + * created the log intent. Fill in the attri log item and log format + * structure with fields from this xfs_attr_intent + */ + attrp = &attrip->attri_format; + attrp->alfi_ino = I_INO(args->dp); + ASSERT(!(attr->xattri_op_flags & ~XFS_ATTRI_OP_FLAGS_TYPE_MASK)); + attrp->alfi_op_flags = attr->xattri_op_flags; + attrp->alfi_value_len = nv->value.iov_len; + + switch (xfs_attr_log_item_op(attrp)) { + case XFS_ATTRI_OP_FLAGS_PPTR_REPLACE: + ASSERT(nv->value.iov_len == nv->new_value.iov_len); + + attrp->alfi_igen = VFS_I(args->dp)->i_generation; + attrp->alfi_old_name_len = nv->name.iov_len; + attrp->alfi_new_name_len = nv->new_name.iov_len; + break; + case XFS_ATTRI_OP_FLAGS_PPTR_REMOVE: + case XFS_ATTRI_OP_FLAGS_PPTR_SET: + attrp->alfi_igen = VFS_I(args->dp)->i_generation; + fallthrough; + default: + attrp->alfi_name_len = nv->name.iov_len; + break; + } + + ASSERT(!(args->attr_filter & ~XFS_ATTRI_FILTER_MASK)); + attrp->alfi_attr_filter = args->attr_filter; +} + +/* Get an ATTRI. */ +static struct xfs_log_item * +xfs_attr_create_intent( + struct xfs_trans *tp, + struct list_head *items, + unsigned int count, + bool sort) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_attri_log_item *attrip; + struct xfs_attr_intent *attr; + struct xfs_da_args *args; + + ASSERT(count == 1); + + /* + * Each attr item only performs one attribute operation at a time, so + * this is a list of one + */ + attr = list_first_entry_or_null(items, struct xfs_attr_intent, + xattri_list); + args = attr->xattri_da_args; + + if (!(args->op_flags & XFS_DA_OP_LOGGED)) + return NULL; + + /* + * Create a buffer to store the attribute name and value. This buffer + * will be shared between the higher level deferred xattr work state + * and the lower level xattr log items. + */ + if (!attr->xattri_nameval) { + /* + * Transfer our reference to the name/value buffer to the + * deferred work state structure. + */ + attr->xattri_nameval = xfs_attri_log_nameval_alloc( + args->name, args->namelen, + args->new_name, args->new_namelen, + args->value, args->valuelen, + args->new_value, args->new_valuelen); + } + + attrip = xfs_attri_init(mp, attr->xattri_nameval); + xfs_attr_log_item(tp, attrip, attr); + + return &attrip->attri_item; +} + +static inline void +xfs_attr_free_item( + struct xfs_attr_intent *attr) +{ + if (attr->xattri_da_state) + xfs_da_state_free(attr->xattri_da_state); + xfs_attri_log_nameval_put(attr->xattri_nameval); + if (attr->xattri_da_args->op_flags & XFS_DA_OP_RECOVERY) + kfree(attr); + else + kmem_cache_free(xfs_attr_intent_cache, attr); +} + +static inline struct xfs_attr_intent *attri_entry(const struct list_head *e) +{ + return list_entry(e, struct xfs_attr_intent, xattri_list); +} + +/* Process an attr. */ +STATIC int +xfs_attr_finish_item( + struct xfs_trans *tp, + struct xfs_log_item *done, + struct list_head *item, + struct xfs_btree_cur **state) +{ + struct xfs_attr_intent *attr = attri_entry(item); + struct xfs_da_args *args; + int error; + + args = attr->xattri_da_args; + + /* Reset trans after EAGAIN cycle since the transaction is new */ + args->trans = tp; + + if (XFS_TEST_ERROR(args->dp->i_mount, XFS_ERRTAG_LARP)) { + error = -EIO; + goto out; + } + + /* If an attr removal is trivially complete, we're done. */ + if (attr->xattri_op_flags == XFS_ATTRI_OP_FLAGS_REMOVE && + !xfs_inode_hasattr(args->dp)) { + error = 0; + goto out; + } + + error = xfs_attr_set_iter(attr); + if (!error && attr->xattri_dela_state != XFS_DAS_DONE) + return -EAGAIN; + +out: + xfs_attr_free_item(attr); + return error; +} + +/* Abort all pending ATTRs. */ +STATIC void +xfs_attr_abort_intent( + struct xfs_log_item *intent) +{ + xfs_attri_release(ATTRI_ITEM(intent)); +} + +/* Cancel an attr */ +STATIC void +xfs_attr_cancel_item( + struct list_head *item) +{ + struct xfs_attr_intent *attr = attri_entry(item); + + xfs_attr_free_item(attr); +} + +STATIC bool +xfs_attri_item_match( + struct xfs_log_item *lip, + uint64_t intent_id) +{ + return ATTRI_ITEM(lip)->attri_format.alfi_id == intent_id; +} + +static inline bool +xfs_attri_validate_namelen(unsigned int namelen) +{ + return namelen > 0 && namelen <= XATTR_NAME_MAX; +} + +/* Is this recovered ATTRI format ok? */ +static inline bool +xfs_attri_validate( + struct xfs_mount *mp, + struct xfs_attri_log_format *attrp) +{ + unsigned int op = xfs_attr_log_item_op(attrp); + + if (attrp->alfi_op_flags & ~XFS_ATTRI_OP_FLAGS_TYPE_MASK) + return false; + + if (attrp->alfi_attr_filter & ~XFS_ATTRI_FILTER_MASK) + return false; + + if (!xfs_attr_check_namespace(attrp->alfi_attr_filter & + XFS_ATTR_NSP_ONDISK_MASK)) + return false; + + switch (op) { + case XFS_ATTRI_OP_FLAGS_PPTR_SET: + case XFS_ATTRI_OP_FLAGS_PPTR_REMOVE: + if (!xfs_has_parent(mp)) + return false; + if (attrp->alfi_value_len != sizeof(struct xfs_parent_rec)) + return false; + if (!xfs_attri_validate_namelen(attrp->alfi_name_len)) + return false; + if (!(attrp->alfi_attr_filter & XFS_ATTR_PARENT)) + return false; + break; + case XFS_ATTRI_OP_FLAGS_SET: + case XFS_ATTRI_OP_FLAGS_REPLACE: + if (!xfs_is_using_logged_xattrs(mp)) + return false; + if (attrp->alfi_value_len > XATTR_SIZE_MAX) + return false; + if (!xfs_attri_validate_namelen(attrp->alfi_name_len)) + return false; + break; + case XFS_ATTRI_OP_FLAGS_REMOVE: + if (!xfs_is_using_logged_xattrs(mp)) + return false; + if (attrp->alfi_value_len != 0) + return false; + if (!xfs_attri_validate_namelen(attrp->alfi_name_len)) + return false; + break; + case XFS_ATTRI_OP_FLAGS_PPTR_REPLACE: + if (!xfs_has_parent(mp)) + return false; + if (!xfs_attri_validate_namelen(attrp->alfi_old_name_len)) + return false; + if (!xfs_attri_validate_namelen(attrp->alfi_new_name_len)) + return false; + if (attrp->alfi_value_len != sizeof(struct xfs_parent_rec)) + return false; + if (!(attrp->alfi_attr_filter & XFS_ATTR_PARENT)) + return false; + break; + default: + return false; + } + + return xfs_verify_ino(mp, attrp->alfi_ino); +} + +static int +xfs_attri_iread_extents( + struct xfs_inode *ip) +{ + struct xfs_trans *tp; + int error; + + tp = xfs_trans_alloc_empty(ip->i_mount); + xfs_ilock(ip, XFS_ILOCK_EXCL); + error = xfs_iread_extents(tp, ip, XFS_ATTR_FORK); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + xfs_trans_cancel(tp); + + return error; +} + +static inline struct xfs_attr_intent * +xfs_attri_recover_work( + struct xfs_mount *mp, + struct xfs_defer_pending *dfp, + struct xfs_attri_log_format *attrp, + struct xfs_inode **ipp, + struct xfs_attri_log_nameval *nv) +{ + struct xfs_attr_intent *attr; + struct xfs_da_args *args; + struct xfs_inode *ip; + int local; + int error; + + /* + * Parent pointer attr items record the generation but regular logged + * xattrs do not; select the right iget function. + */ + switch (xfs_attr_log_item_op(attrp)) { + case XFS_ATTRI_OP_FLAGS_PPTR_SET: + case XFS_ATTRI_OP_FLAGS_PPTR_REPLACE: + case XFS_ATTRI_OP_FLAGS_PPTR_REMOVE: + error = xlog_recover_iget_handle(mp, attrp->alfi_ino, + attrp->alfi_igen, &ip); + break; + default: + error = xlog_recover_iget(mp, attrp->alfi_ino, &ip); + break; + } + if (error) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, attrp, + sizeof(*attrp)); + return ERR_PTR(-EFSCORRUPTED); + } + + if (xfs_inode_has_attr_fork(ip)) { + error = xfs_attri_iread_extents(ip); + if (error) { + xfs_irele(ip); + return ERR_PTR(error); + } + } + + attr = kzalloc(sizeof(struct xfs_attr_intent) + + sizeof(struct xfs_da_args), GFP_KERNEL | __GFP_NOFAIL); + args = (struct xfs_da_args *)(attr + 1); + + attr->xattri_da_args = args; + attr->xattri_op_flags = xfs_attr_log_item_op(attrp); + + /* + * We're reconstructing the deferred work state structure from the + * recovered log item. Grab a reference to the name/value buffer and + * attach it to the new work state. + */ + attr->xattri_nameval = xfs_attri_log_nameval_get(nv); + ASSERT(attr->xattri_nameval); + + args->dp = ip; + args->geo = mp->m_attr_geo; + args->whichfork = XFS_ATTR_FORK; + args->name = nv->name.iov_base; + args->namelen = nv->name.iov_len; + args->new_name = nv->new_name.iov_base; + args->new_namelen = nv->new_name.iov_len; + args->value = nv->value.iov_base; + args->valuelen = nv->value.iov_len; + args->new_value = nv->new_value.iov_base; + args->new_valuelen = nv->new_value.iov_len; + args->attr_filter = attrp->alfi_attr_filter & XFS_ATTRI_FILTER_MASK; + args->op_flags = XFS_DA_OP_RECOVERY | XFS_DA_OP_OKNOENT | + XFS_DA_OP_LOGGED; + args->owner = I_INO(args->dp); + xfs_attr_sethash(args); + + switch (xfs_attr_intent_op(attr)) { + case XFS_ATTRI_OP_FLAGS_PPTR_SET: + case XFS_ATTRI_OP_FLAGS_PPTR_REPLACE: + case XFS_ATTRI_OP_FLAGS_SET: + case XFS_ATTRI_OP_FLAGS_REPLACE: + args->total = xfs_attr_calc_size(args, &local); + if (xfs_inode_hasattr(args->dp)) + attr->xattri_dela_state = xfs_attr_init_replace_state(args); + else + attr->xattri_dela_state = xfs_attr_init_add_state(args); + break; + case XFS_ATTRI_OP_FLAGS_PPTR_REMOVE: + case XFS_ATTRI_OP_FLAGS_REMOVE: + attr->xattri_dela_state = xfs_attr_init_remove_state(args); + break; + } + + xfs_defer_add_item(dfp, &attr->xattri_list); + *ipp = ip; + return attr; +} + +/* + * Process an attr intent item that was recovered from the log. We need to + * delete the attr that it describes. + */ +STATIC int +xfs_attr_recover_work( + struct xfs_defer_pending *dfp, + struct list_head *capture_list) +{ + struct xfs_log_item *lip = dfp->dfp_intent; + struct xfs_attri_log_item *attrip = ATTRI_ITEM(lip); + struct xfs_attr_intent *attr; + struct xfs_mount *mp = lip->li_log->l_mp; + struct xfs_inode *ip = NULL; + struct xfs_da_args *args; + struct xfs_trans *tp; + struct xfs_trans_res resv; + struct xfs_attri_log_format *attrp; + struct xfs_attri_log_nameval *nv = attrip->attri_nameval; + int error; + unsigned int total = 0; + + /* + * First check the validity of the attr described by the ATTRI. If any + * are bad, then assume that all are bad and just toss the ATTRI. + */ + attrp = &attrip->attri_format; + if (!xfs_attri_validate(mp, attrp) || + !xfs_attr_namecheck(attrp->alfi_attr_filter, nv->name.iov_base, + nv->name.iov_len)) + return -EFSCORRUPTED; + + attr = xfs_attri_recover_work(mp, dfp, attrp, &ip, nv); + if (IS_ERR(attr)) + return PTR_ERR(attr); + args = attr->xattri_da_args; + + switch (xfs_attr_intent_op(attr)) { + case XFS_ATTRI_OP_FLAGS_PPTR_SET: + case XFS_ATTRI_OP_FLAGS_PPTR_REPLACE: + case XFS_ATTRI_OP_FLAGS_SET: + case XFS_ATTRI_OP_FLAGS_REPLACE: + resv = xfs_attr_set_resv(args); + total = args->total; + break; + case XFS_ATTRI_OP_FLAGS_PPTR_REMOVE: + case XFS_ATTRI_OP_FLAGS_REMOVE: + resv = M_RES(mp)->tr_attrrm; + total = XFS_ATTRRM_SPACE_RES(mp); + break; + } + resv = xlog_recover_resv(&resv); + error = xfs_trans_alloc(mp, &resv, total, 0, XFS_TRANS_RESERVE, &tp); + if (error) + return error; + args->trans = tp; + + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, 0); + + error = xlog_recover_finish_intent(tp, dfp); + if (error == -EFSCORRUPTED) + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + &attrip->attri_format, + sizeof(attrip->attri_format)); + if (error) + goto out_cancel; + + error = xfs_defer_ops_capture_and_commit(tp, capture_list); +out_unlock: + xfs_iunlock(ip, XFS_ILOCK_EXCL); + xfs_irele(ip); + return error; +out_cancel: + xfs_trans_cancel(tp); + goto out_unlock; +} + +/* Re-log an intent item to push the log tail forward. */ +static struct xfs_log_item * +xfs_attr_relog_intent( + struct xfs_trans *tp, + struct xfs_log_item *intent, + struct xfs_log_item *done_item) +{ + struct xfs_attri_log_item *old_attrip; + struct xfs_attri_log_item *new_attrip; + struct xfs_attri_log_format *new_attrp; + struct xfs_attri_log_format *old_attrp; + + old_attrip = ATTRI_ITEM(intent); + old_attrp = &old_attrip->attri_format; + + /* + * Create a new log item that shares the same name/value buffer as the + * old log item. + */ + new_attrip = xfs_attri_init(tp->t_mountp, old_attrip->attri_nameval); + new_attrp = &new_attrip->attri_format; + + new_attrp->alfi_ino = old_attrp->alfi_ino; + new_attrp->alfi_igen = old_attrp->alfi_igen; + new_attrp->alfi_op_flags = old_attrp->alfi_op_flags; + new_attrp->alfi_value_len = old_attrp->alfi_value_len; + + switch (xfs_attr_log_item_op(old_attrp)) { + case XFS_ATTRI_OP_FLAGS_PPTR_REPLACE: + new_attrp->alfi_new_name_len = old_attrp->alfi_new_name_len; + new_attrp->alfi_old_name_len = old_attrp->alfi_old_name_len; + break; + default: + new_attrp->alfi_name_len = old_attrp->alfi_name_len; + break; + } + + new_attrp->alfi_attr_filter = old_attrp->alfi_attr_filter; + + return &new_attrip->attri_item; +} + +/* Get an ATTRD so we can process all the attrs. */ +static struct xfs_log_item * +xfs_attr_create_done( + struct xfs_trans *tp, + struct xfs_log_item *intent, + unsigned int count) +{ + struct xfs_attri_log_item *attrip; + struct xfs_attrd_log_item *attrdp; + + attrip = ATTRI_ITEM(intent); + + attrdp = kmem_cache_zalloc(xfs_attrd_cache, GFP_KERNEL | __GFP_NOFAIL); + + xfs_log_item_init(tp->t_mountp, &attrdp->attrd_item, XFS_LI_ATTRD, + &xfs_attrd_item_ops); + attrdp->attrd_attrip = attrip; + attrdp->attrd_format.alfd_alf_id = attrip->attri_format.alfi_id; + + return &attrdp->attrd_item; +} + +void +xfs_attr_defer_add( + struct xfs_da_args *args, + enum xfs_attr_defer_op op) +{ + struct xfs_attr_intent *new; + unsigned int log_op = 0; + bool is_pptr = args->attr_filter & XFS_ATTR_PARENT; + + if (is_pptr) { + ASSERT(xfs_has_parent(args->dp->i_mount)); + ASSERT((args->attr_filter & ~XFS_ATTR_PARENT) == 0); + ASSERT(args->op_flags & XFS_DA_OP_LOGGED); + ASSERT(args->valuelen == sizeof(struct xfs_parent_rec)); + } + + new = kmem_cache_zalloc(xfs_attr_intent_cache, + GFP_NOFS | __GFP_NOFAIL); + new->xattri_da_args = args; + + /* Compute log operation from the higher level op and namespace. */ + switch (op) { + case XFS_ATTR_DEFER_SET: + if (is_pptr) + log_op = XFS_ATTRI_OP_FLAGS_PPTR_SET; + else + log_op = XFS_ATTRI_OP_FLAGS_SET; + break; + case XFS_ATTR_DEFER_REPLACE: + if (is_pptr) + log_op = XFS_ATTRI_OP_FLAGS_PPTR_REPLACE; + else + log_op = XFS_ATTRI_OP_FLAGS_REPLACE; + break; + case XFS_ATTR_DEFER_REMOVE: + if (is_pptr) + log_op = XFS_ATTRI_OP_FLAGS_PPTR_REMOVE; + else + log_op = XFS_ATTRI_OP_FLAGS_REMOVE; + break; + default: + ASSERT(0); + break; + } + new->xattri_op_flags = log_op; + + /* Set up initial attr operation state. */ + switch (log_op) { + case XFS_ATTRI_OP_FLAGS_PPTR_SET: + case XFS_ATTRI_OP_FLAGS_SET: + new->xattri_dela_state = xfs_attr_init_add_state(args); + break; + case XFS_ATTRI_OP_FLAGS_PPTR_REPLACE: + ASSERT(args->new_valuelen == args->valuelen); + new->xattri_dela_state = xfs_attr_init_replace_state(args); + break; + case XFS_ATTRI_OP_FLAGS_REPLACE: + new->xattri_dela_state = xfs_attr_init_replace_state(args); + break; + case XFS_ATTRI_OP_FLAGS_PPTR_REMOVE: + case XFS_ATTRI_OP_FLAGS_REMOVE: + new->xattri_dela_state = xfs_attr_init_remove_state(args); + break; + } + + xfs_defer_add(args->trans, &new->xattri_list, &xfs_attr_defer_type); + trace_xfs_attr_defer_add(new->xattri_dela_state, args->dp); +} + +const struct xfs_defer_op_type xfs_attr_defer_type = { + .name = "attr", + .max_items = 1, + .create_intent = xfs_attr_create_intent, + .abort_intent = xfs_attr_abort_intent, + .create_done = xfs_attr_create_done, + .finish_item = xfs_attr_finish_item, + .cancel_item = xfs_attr_cancel_item, + .recover_work = xfs_attr_recover_work, + .relog_intent = xfs_attr_relog_intent, +}; + +static inline void * +xfs_attri_validate_name_iovec( + struct xfs_mount *mp, + struct xfs_attri_log_format *attri_formatp, + const struct kvec *iovec, + unsigned int name_len) +{ + if (iovec->iov_len != xlog_calc_iovec_len(name_len)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + attri_formatp, sizeof(*attri_formatp)); + return NULL; + } + + if (!xfs_attr_namecheck(attri_formatp->alfi_attr_filter, iovec->iov_base, + name_len)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + attri_formatp, sizeof(*attri_formatp)); + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + iovec->iov_base, iovec->iov_len); + return NULL; + } + + return iovec->iov_base; +} + +static inline void * +xfs_attri_validate_value_iovec( + struct xfs_mount *mp, + struct xfs_attri_log_format *attri_formatp, + const struct kvec *iovec, + unsigned int value_len) +{ + if (iovec->iov_len != xlog_calc_iovec_len(value_len)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + attri_formatp, sizeof(*attri_formatp)); + return NULL; + } + + if ((attri_formatp->alfi_attr_filter & XFS_ATTR_PARENT) && + !xfs_parent_valuecheck(mp, iovec->iov_base, value_len)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + attri_formatp, sizeof(*attri_formatp)); + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + iovec->iov_base, iovec->iov_len); + return NULL; + } + + return iovec->iov_base; +} + +STATIC int +xlog_recover_attri_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_mount *mp = log->l_mp; + struct xfs_attri_log_item *attrip; + struct xfs_attri_log_format *attri_formatp; + struct xfs_attri_log_nameval *nv; + const void *attr_name; + const void *attr_value = NULL; + const void *attr_new_name = NULL; + const void *attr_new_value = NULL; + size_t len; + unsigned int name_len = 0; + unsigned int value_len = 0; + unsigned int new_name_len = 0; + unsigned int new_value_len = 0; + unsigned int op, i = 0; + + /* Validate xfs_attri_log_format before the large memory allocation */ + len = sizeof(struct xfs_attri_log_format); + if (item->ri_buf[i].iov_len != len) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[0].iov_base, item->ri_buf[0].iov_len); + return -EFSCORRUPTED; + } + + attri_formatp = item->ri_buf[i].iov_base; + if (!xfs_attri_validate(mp, attri_formatp)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + attri_formatp, len); + return -EFSCORRUPTED; + } + + /* Check the number of log iovecs makes sense for the op code. */ + op = xfs_attr_log_item_op(attri_formatp); + switch (op) { + case XFS_ATTRI_OP_FLAGS_PPTR_REMOVE: + case XFS_ATTRI_OP_FLAGS_PPTR_SET: + /* Log item, attr name, attr value */ + if (item->ri_total != 3) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + attri_formatp, len); + return -EFSCORRUPTED; + } + name_len = attri_formatp->alfi_name_len; + value_len = attri_formatp->alfi_value_len; + break; + case XFS_ATTRI_OP_FLAGS_SET: + case XFS_ATTRI_OP_FLAGS_REPLACE: + /* Log item, attr name, optional attr value */ + if (item->ri_total != 2 + !!attri_formatp->alfi_value_len) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + attri_formatp, len); + return -EFSCORRUPTED; + } + name_len = attri_formatp->alfi_name_len; + value_len = attri_formatp->alfi_value_len; + break; + case XFS_ATTRI_OP_FLAGS_REMOVE: + /* Log item, attr name */ + if (item->ri_total != 2) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + attri_formatp, len); + return -EFSCORRUPTED; + } + name_len = attri_formatp->alfi_name_len; + break; + case XFS_ATTRI_OP_FLAGS_PPTR_REPLACE: + /* + * Log item, attr name, new attr name, attr value, new attr + * value + */ + if (item->ri_total != 5) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + attri_formatp, len); + return -EFSCORRUPTED; + } + name_len = attri_formatp->alfi_old_name_len; + new_name_len = attri_formatp->alfi_new_name_len; + new_value_len = value_len = attri_formatp->alfi_value_len; + break; + default: + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + attri_formatp, len); + return -EFSCORRUPTED; + } + i++; + + /* Validate the attr name */ + attr_name = xfs_attri_validate_name_iovec(mp, attri_formatp, + &item->ri_buf[i], name_len); + if (!attr_name) + return -EFSCORRUPTED; + i++; + + /* Validate the new attr name */ + if (new_name_len > 0) { + attr_new_name = xfs_attri_validate_name_iovec(mp, + attri_formatp, &item->ri_buf[i], + new_name_len); + if (!attr_new_name) + return -EFSCORRUPTED; + i++; + } + + /* Validate the attr value, if present */ + if (value_len != 0) { + attr_value = xfs_attri_validate_value_iovec(mp, attri_formatp, + &item->ri_buf[i], value_len); + if (!attr_value) + return -EFSCORRUPTED; + i++; + } + + /* Validate the new attr value, if present */ + if (new_value_len != 0) { + attr_new_value = xfs_attri_validate_value_iovec(mp, + attri_formatp, &item->ri_buf[i], + new_value_len); + if (!attr_new_value) + return -EFSCORRUPTED; + i++; + } + + /* + * Make sure we got the correct number of buffers for the operation + * that we just loaded. + */ + if (i != item->ri_total) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + attri_formatp, len); + return -EFSCORRUPTED; + } + + /* + * Memory alloc failure will cause replay to abort. We attach the + * name/value buffer to the recovered incore log item and drop our + * reference. + */ + nv = xfs_attri_log_nameval_alloc(attr_name, name_len, + attr_new_name, new_name_len, + attr_value, value_len, + attr_new_value, new_value_len); + + attrip = xfs_attri_init(mp, nv); + memcpy(&attrip->attri_format, attri_formatp, len); + + xlog_recover_intent_item(log, &attrip->attri_item, lsn, + &xfs_attr_defer_type); + xfs_attri_log_nameval_put(nv); + return 0; +} + +/* + * This routine is called when an ATTRD format structure is found in a committed + * transaction in the log. Its purpose is to cancel the corresponding ATTRI if + * it was still in the log. To do this it searches the AIL for the ATTRI with + * an id equal to that in the ATTRD format structure. If we find it we drop + * the ATTRD reference, which removes the ATTRI from the AIL and frees it. + */ +STATIC int +xlog_recover_attrd_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_attrd_log_format *attrd_formatp; + + attrd_formatp = item->ri_buf[0].iov_base; + if (item->ri_buf[0].iov_len != sizeof(struct xfs_attrd_log_format)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, log->l_mp, + item->ri_buf[0].iov_base, item->ri_buf[0].iov_len); + return -EFSCORRUPTED; + } + + xlog_recover_release_intent(log, XFS_LI_ATTRI, + attrd_formatp->alfd_alf_id); + return 0; +} + +static const struct xfs_item_ops xfs_attri_item_ops = { + .flags = XFS_ITEM_INTENT, + .iop_size = xfs_attri_item_size, + .iop_format = xfs_attri_item_format, + .iop_unpin = xfs_attri_item_unpin, + .iop_release = xfs_attri_item_release, + .iop_match = xfs_attri_item_match, +}; + +const struct xlog_recover_item_ops xlog_attri_item_ops = { + .item_type = XFS_LI_ATTRI, + .commit_pass2 = xlog_recover_attri_commit_pass2, +}; + +static const struct xfs_item_ops xfs_attrd_item_ops = { + .flags = XFS_ITEM_RELEASE_WHEN_COMMITTED | + XFS_ITEM_INTENT_DONE, + .iop_size = xfs_attrd_item_size, + .iop_format = xfs_attrd_item_format, + .iop_release = xfs_attrd_item_release, + .iop_intent = xfs_attrd_item_intent, +}; + +const struct xlog_recover_item_ops xlog_attrd_item_ops = { + .item_type = XFS_LI_ATTRD, + .commit_pass2 = xlog_recover_attrd_commit_pass2, +}; diff --git a/fs/xfs/xfs_attr_item.h b/fs/xfs/xfs_attr_item.h new file mode 100644 index 000000000..d108a11b5 --- /dev/null +++ b/fs/xfs/xfs_attr_item.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later + * + * Copyright (C) 2022 Oracle. All Rights Reserved. + * Author: Allison Henderson + */ +#ifndef __XFS_ATTR_ITEM_H__ +#define __XFS_ATTR_ITEM_H__ + +/* kernel only ATTRI/ATTRD definitions */ + +struct xfs_mount; +struct kmem_zone; + +struct xfs_attri_log_nameval { + struct kvec name; + struct kvec new_name; /* PPTR_REPLACE only */ + struct kvec value; + struct kvec new_value; /* PPTR_REPLACE only */ + refcount_t refcount; + + /* name and value follow the end of this struct */ +}; + +/* + * This is the "attr intention" log item. It is used to log the fact that some + * extended attribute operations need to be processed. An operation is + * currently either a set or remove. Set or remove operations are described by + * the xfs_attr_intent which may be logged to this intent. + * + * During a normal attr operation, name and value point to the name and value + * fields of the caller's xfs_da_args structure. During a recovery, the name + * and value buffers are copied from the log, and stored in a trailing buffer + * attached to the xfs_attr_intent until they are committed. They are freed + * when the xfs_attr_intent itself is freed when the work is done. + */ +struct xfs_attri_log_item { + struct xfs_log_item attri_item; + atomic_t attri_refcount; + struct xfs_attri_log_nameval *attri_nameval; + struct xfs_attri_log_format attri_format; +}; + +/* + * This is the "attr done" log item. It is used to log the fact that some attrs + * earlier mentioned in an attri item have been freed. + */ +struct xfs_attrd_log_item { + struct xfs_log_item attrd_item; + struct xfs_attri_log_item *attrd_attrip; + struct xfs_attrd_log_format attrd_format; +}; + +extern struct kmem_cache *xfs_attri_cache; +extern struct kmem_cache *xfs_attrd_cache; + +enum xfs_attr_defer_op { + XFS_ATTR_DEFER_SET, + XFS_ATTR_DEFER_REMOVE, + XFS_ATTR_DEFER_REPLACE, +}; + +void xfs_attr_defer_add(struct xfs_da_args *args, enum xfs_attr_defer_op op); + +#endif /* __XFS_ATTR_ITEM_H__ */ diff --git a/fs/xfs/xfs_attr_list.c b/fs/xfs/xfs_attr_list.c new file mode 100644 index 000000000..f751edb93 --- /dev/null +++ b/fs/xfs/xfs_attr_list.c @@ -0,0 +1,599 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * Copyright (c) 2013 Red Hat, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_da_format.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_bmap.h" +#include "xfs_da_btree.h" +#include "xfs_attr.h" +#include "xfs_attr_sf.h" +#include "xfs_attr_leaf.h" +#include "xfs_error.h" +#include "xfs_trace.h" +#include "xfs_dir2.h" +#include "xfs_health.h" + +STATIC int +xfs_attr_shortform_compare(const void *a, const void *b) +{ + xfs_attr_sf_sort_t *sa, *sb; + + sa = (xfs_attr_sf_sort_t *)a; + sb = (xfs_attr_sf_sort_t *)b; + if (sa->hash < sb->hash) { + return -1; + } else if (sa->hash > sb->hash) { + return 1; + } else { + return sa->entno - sb->entno; + } +} + +#define XFS_ISRESET_CURSOR(cursor) \ + (!((cursor)->initted) && !((cursor)->hashval) && \ + !((cursor)->blkno) && !((cursor)->offset)) +/* + * Copy out entries of shortform attribute lists for attr_list(). + * Shortform attribute lists are not stored in hashval sorted order. + * If the output buffer is not large enough to hold them all, then + * we have to calculate each entries' hashvalue and sort them before + * we can begin returning them to the user. + */ +static int +xfs_attr_shortform_list( + struct xfs_attr_list_context *context) +{ + struct xfs_attrlist_cursor_kern *cursor = &context->cursor; + struct xfs_inode *dp = context->dp; + struct xfs_attr_sf_sort *sbuf, *sbp; + struct xfs_attr_sf_hdr *sf = dp->i_af.if_data; + struct xfs_attr_sf_entry *sfe; + int sbsize, nsbuf, count, i; + int error = 0; + + ASSERT(sf != NULL); + if (!sf->count) + return 0; + + trace_xfs_attr_list_sf(context); + + /* + * If the buffer is large enough and the cursor is at the start, + * do not bother with sorting since we will return everything in + * one buffer and another call using the cursor won't need to be + * made. + * Note the generous fudge factor of 16 overhead bytes per entry. + * If bufsize is zero then put_listent must be a search function + * and can just scan through what we have. + */ + if (context->bufsize == 0 || + (XFS_ISRESET_CURSOR(cursor) && + (dp->i_af.if_bytes + sf->count * 16) < context->bufsize)) { + for (i = 0, sfe = xfs_attr_sf_firstentry(sf); i < sf->count; i++) { + if (XFS_IS_CORRUPT(context->dp->i_mount, + !xfs_attr_namecheck(sfe->flags, + sfe->nameval, + sfe->namelen))) { + xfs_dirattr_mark_sick(context->dp, XFS_ATTR_FORK); + return -EFSCORRUPTED; + } + context->put_listent(context, + sfe->flags, + sfe->nameval, + (int)sfe->namelen, + &sfe->nameval[sfe->namelen], + (int)sfe->valuelen); + /* + * Either search callback finished early or + * didn't fit it all in the buffer after all. + */ + if (context->seen_enough) + break; + sfe = xfs_attr_sf_nextentry(sfe); + } + trace_xfs_attr_list_sf_all(context); + return 0; + } + + /* do no more for a search callback */ + if (context->bufsize == 0) + return 0; + + /* + * It didn't all fit, so we have to sort everything on hashval. + */ + sbsize = sf->count * sizeof(*sbuf); + sbp = sbuf = kmalloc(sbsize, + GFP_KERNEL | __GFP_NOLOCKDEP | __GFP_NOFAIL); + + /* + * Scan the attribute list for the rest of the entries, storing + * the relevant info from only those that match into a buffer. + */ + nsbuf = 0; + for (i = 0, sfe = xfs_attr_sf_firstentry(sf); i < sf->count; i++) { + if (unlikely( + ((char *)sfe < (char *)sf) || + ((char *)sfe >= ((char *)sf + dp->i_af.if_bytes)) || + !xfs_attr_check_namespace(sfe->flags))) { + XFS_CORRUPTION_ERROR("xfs_attr_shortform_list", + XFS_ERRLEVEL_LOW, + context->dp->i_mount, sfe, + sizeof(*sfe)); + kfree(sbuf); + xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); + return -EFSCORRUPTED; + } + + sbp->entno = i; + sbp->name = sfe->nameval; + sbp->namelen = sfe->namelen; + /* These are bytes, and both on-disk, don't endian-flip */ + sbp->value = &sfe->nameval[sfe->namelen]; + sbp->valuelen = sfe->valuelen; + sbp->flags = sfe->flags; + sbp->hash = xfs_attr_hashval(dp->i_mount, sfe->flags, + sfe->nameval, sfe->namelen, + sfe->nameval + sfe->namelen, + sfe->valuelen); + sfe = xfs_attr_sf_nextentry(sfe); + sbp++; + nsbuf++; + } + + /* + * Sort the entries on hash then entno. + */ + xfs_sort(sbuf, nsbuf, sizeof(*sbuf), xfs_attr_shortform_compare); + + /* + * Re-find our place IN THE SORTED LIST. + */ + count = 0; + cursor->initted = 1; + cursor->blkno = 0; + for (sbp = sbuf, i = 0; i < nsbuf; i++, sbp++) { + if (sbp->hash == cursor->hashval) { + if (cursor->offset == count) { + break; + } + count++; + } else if (sbp->hash > cursor->hashval) { + break; + } + } + if (i == nsbuf) + goto out; + + /* + * Loop putting entries into the user buffer. + */ + for ( ; i < nsbuf; i++, sbp++) { + if (cursor->hashval != sbp->hash) { + cursor->hashval = sbp->hash; + cursor->offset = 0; + } + if (XFS_IS_CORRUPT(context->dp->i_mount, + !xfs_attr_namecheck(sbp->flags, sbp->name, + sbp->namelen))) { + xfs_dirattr_mark_sick(context->dp, XFS_ATTR_FORK); + error = -EFSCORRUPTED; + goto out; + } + context->put_listent(context, + sbp->flags, + sbp->name, + sbp->namelen, + sbp->value, + sbp->valuelen); + if (context->seen_enough) + break; + cursor->offset++; + } +out: + kfree(sbuf); + return error; +} + +/* + * We didn't find the block & hash mentioned in the cursor state, so + * walk down the attr btree looking for the hash. + */ +STATIC int +xfs_attr_node_list_lookup( + struct xfs_attr_list_context *context, + struct xfs_attrlist_cursor_kern *cursor, + struct xfs_buf **pbp) +{ + struct xfs_da3_icnode_hdr nodehdr; + struct xfs_da_intnode *node; + struct xfs_da_node_entry *btree; + struct xfs_inode *dp = context->dp; + struct xfs_mount *mp = dp->i_mount; + struct xfs_trans *tp = context->tp; + struct xfs_buf *bp; + xfs_failaddr_t fa; + int i; + int error = 0; + unsigned int expected_level = 0; + uint16_t magic; + + ASSERT(*pbp == NULL); + cursor->blkno = 0; + for (;;) { + error = xfs_da3_node_read(tp, dp, cursor->blkno, &bp, + XFS_ATTR_FORK); + if (error) + return error; + node = bp->b_addr; + magic = be16_to_cpu(node->hdr.info.magic); + if (magic == XFS_ATTR_LEAF_MAGIC || + magic == XFS_ATTR3_LEAF_MAGIC) + break; + if (magic != XFS_DA_NODE_MAGIC && + magic != XFS_DA3_NODE_MAGIC) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + node, sizeof(*node)); + goto out_corruptbuf; + } + + fa = xfs_da3_node_header_check(bp, I_INO(dp)); + if (fa) + goto out_corruptbuf; + + xfs_da3_node_hdr_from_disk(mp, &nodehdr, node); + + /* Tree taller than we can handle; bail out! */ + if (nodehdr.level >= XFS_DA_NODE_MAXDEPTH) + goto out_corruptbuf; + + /* Check the level from the root node. */ + if (cursor->blkno == 0) + expected_level = nodehdr.level - 1; + else if (expected_level != nodehdr.level) + goto out_corruptbuf; + else + expected_level--; + + btree = nodehdr.btree; + for (i = 0; i < nodehdr.count; btree++, i++) { + if (cursor->hashval <= be32_to_cpu(btree->hashval)) { + cursor->blkno = be32_to_cpu(btree->before); + trace_xfs_attr_list_node_descend(context, + btree); + break; + } + } + xfs_trans_brelse(tp, bp); + + if (i == nodehdr.count) + return 0; + + /* We can't point back to the root. */ + if (XFS_IS_CORRUPT(mp, cursor->blkno == 0)) { + xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); + return -EFSCORRUPTED; + } + } + + fa = xfs_attr3_leaf_header_check(bp, I_INO(dp)); + if (fa) { + __xfs_buf_mark_corrupt(bp, fa); + goto out_releasebuf; + } + + if (expected_level != 0) + goto out_corruptbuf; + + *pbp = bp; + return 0; + +out_corruptbuf: + xfs_buf_mark_corrupt(bp); +out_releasebuf: + xfs_trans_brelse(tp, bp); + xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); + return -EFSCORRUPTED; +} + +STATIC int +xfs_attr_node_list( + struct xfs_attr_list_context *context) +{ + struct xfs_attrlist_cursor_kern *cursor = &context->cursor; + struct xfs_attr3_icleaf_hdr leafhdr; + struct xfs_attr_leafblock *leaf; + struct xfs_da_intnode *node; + struct xfs_buf *bp; + struct xfs_inode *dp = context->dp; + struct xfs_mount *mp = dp->i_mount; + xfs_failaddr_t fa; + int error = 0; + + trace_xfs_attr_node_list(context); + + cursor->initted = 1; + + /* + * Do all sorts of validation on the passed-in cursor structure. + * If anything is amiss, ignore the cursor and look up the hashval + * starting from the btree root. + */ + bp = NULL; + if (cursor->blkno > 0) { + struct xfs_attr_leaf_entry *entries; + + error = xfs_da3_node_read(context->tp, dp, cursor->blkno, &bp, + XFS_ATTR_FORK); + if (xfs_metadata_is_sick(error)) + xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); + if (error != 0 && error != -EFSCORRUPTED) + return error; + if (!bp) + goto need_lookup; + + node = bp->b_addr; + switch (be16_to_cpu(node->hdr.info.magic)) { + case XFS_DA_NODE_MAGIC: + case XFS_DA3_NODE_MAGIC: + trace_xfs_attr_list_wrong_blk(context); + fa = xfs_da3_node_header_check(bp, I_INO(dp)); + if (fa) { + __xfs_buf_mark_corrupt(bp, fa); + xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); + } + xfs_trans_brelse(context->tp, bp); + bp = NULL; + break; + case XFS_ATTR_LEAF_MAGIC: + case XFS_ATTR3_LEAF_MAGIC: + leaf = bp->b_addr; + fa = xfs_attr3_leaf_header_check(bp, I_INO(dp)); + if (fa) { + __xfs_buf_mark_corrupt(bp, fa); + xfs_trans_brelse(context->tp, bp); + xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); + bp = NULL; + break; + } + xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, + &leafhdr, leaf); + entries = xfs_attr3_leaf_entryp(leaf); + if (cursor->hashval > be32_to_cpu( + entries[leafhdr.count - 1].hashval)) { + trace_xfs_attr_list_wrong_blk(context); + xfs_trans_brelse(context->tp, bp); + bp = NULL; + } else if (cursor->hashval <= be32_to_cpu( + entries[0].hashval)) { + trace_xfs_attr_list_wrong_blk(context); + xfs_trans_brelse(context->tp, bp); + bp = NULL; + } + break; + default: + trace_xfs_attr_list_wrong_blk(context); + xfs_trans_brelse(context->tp, bp); + bp = NULL; + } + } + + /* + * We did not find what we expected given the cursor's contents, + * so we start from the top and work down based on the hash value. + * Note that start of node block is same as start of leaf block. + */ + if (bp == NULL) { +need_lookup: + error = xfs_attr_node_list_lookup(context, cursor, &bp); + if (error || !bp) + return error; + } + ASSERT(bp != NULL); + + /* + * Roll upward through the blocks, processing each leaf block in + * order. As long as there is space in the result buffer, keep + * adding the information. + */ + for (;;) { + leaf = bp->b_addr; + error = xfs_attr3_leaf_list_int(bp, context); + if (error) + break; + xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, &leafhdr, leaf); + if (context->seen_enough || leafhdr.forw == 0) + break; + cursor->blkno = leafhdr.forw; + xfs_trans_brelse(context->tp, bp); + error = xfs_attr3_leaf_read(context->tp, dp, I_INO(dp), + cursor->blkno, &bp); + if (error) + return error; + } + xfs_trans_brelse(context->tp, bp); + return error; +} + +/* + * Copy out attribute list entries for attr_list(), for leaf attribute lists. + */ +int +xfs_attr3_leaf_list_int( + struct xfs_buf *bp, + struct xfs_attr_list_context *context) +{ + struct xfs_attrlist_cursor_kern *cursor = &context->cursor; + struct xfs_attr_leafblock *leaf; + struct xfs_attr3_icleaf_hdr ichdr; + struct xfs_attr_leaf_entry *entries; + struct xfs_attr_leaf_entry *entry; + int i; + struct xfs_mount *mp = context->dp->i_mount; + + trace_xfs_attr_list_leaf(context); + + leaf = bp->b_addr; + xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, &ichdr, leaf); + entries = xfs_attr3_leaf_entryp(leaf); + + cursor->initted = 1; + + /* + * Re-find our place in the leaf block if this is a new syscall. + */ + if (context->resynch) { + entry = &entries[0]; + for (i = 0; i < ichdr.count; entry++, i++) { + if (be32_to_cpu(entry->hashval) == cursor->hashval) { + if (cursor->offset == context->dupcnt) { + context->dupcnt = 0; + break; + } + context->dupcnt++; + } else if (be32_to_cpu(entry->hashval) > + cursor->hashval) { + context->dupcnt = 0; + break; + } + } + if (i == ichdr.count) { + trace_xfs_attr_list_notfound(context); + return 0; + } + } else { + entry = &entries[0]; + i = 0; + } + context->resynch = 0; + + /* + * We have found our place, start copying out the new attributes. + */ + for (; i < ichdr.count; entry++, i++) { + char *name; + void *value; + int namelen, valuelen; + + if (be32_to_cpu(entry->hashval) != cursor->hashval) { + cursor->hashval = be32_to_cpu(entry->hashval); + cursor->offset = 0; + } + + if ((entry->flags & XFS_ATTR_INCOMPLETE) && + !context->allow_incomplete) + continue; + + if (entry->flags & XFS_ATTR_LOCAL) { + xfs_attr_leaf_name_local_t *name_loc; + + name_loc = xfs_attr3_leaf_name_local(leaf, i); + name = name_loc->nameval; + namelen = name_loc->namelen; + value = &name_loc->nameval[name_loc->namelen]; + valuelen = be16_to_cpu(name_loc->valuelen); + } else { + xfs_attr_leaf_name_remote_t *name_rmt; + + name_rmt = xfs_attr3_leaf_name_remote(leaf, i); + name = name_rmt->name; + namelen = name_rmt->namelen; + value = NULL; + valuelen = be32_to_cpu(name_rmt->valuelen); + } + + if (XFS_IS_CORRUPT(context->dp->i_mount, + !xfs_attr_namecheck(entry->flags, name, + namelen))) { + xfs_dirattr_mark_sick(context->dp, XFS_ATTR_FORK); + return -EFSCORRUPTED; + } + context->put_listent(context, entry->flags, + name, namelen, value, valuelen); + if (context->seen_enough) + break; + cursor->offset++; + } + trace_xfs_attr_list_leaf_end(context); + return 0; +} + +/* + * Copy out attribute entries for attr_list(), for leaf attribute lists. + */ +STATIC int +xfs_attr_leaf_list( + struct xfs_attr_list_context *context) +{ + struct xfs_buf *bp; + int error; + + trace_xfs_attr_leaf_list(context); + + context->cursor.blkno = 0; + error = xfs_attr3_leaf_read(context->tp, context->dp, + I_INO(context->dp), 0, &bp); + if (error) + return error; + + error = xfs_attr3_leaf_list_int(bp, context); + xfs_trans_brelse(context->tp, bp); + return error; +} + +int +xfs_attr_list_ilocked( + struct xfs_attr_list_context *context) +{ + struct xfs_inode *dp = context->dp; + int error; + + xfs_assert_ilocked(dp, XFS_ILOCK_SHARED | XFS_ILOCK_EXCL); + + /* + * Decide on what work routines to call based on the inode size. + */ + if (!xfs_inode_hasattr(dp)) + return 0; + if (dp->i_af.if_format == XFS_DINODE_FMT_LOCAL) + return xfs_attr_shortform_list(context); + + /* Prerequisite for xfs_attr_is_leaf */ + error = xfs_iread_extents(NULL, dp, XFS_ATTR_FORK); + if (error) + return error; + + if (xfs_attr_is_leaf(dp)) + return xfs_attr_leaf_list(context); + return xfs_attr_node_list(context); +} + +int +xfs_attr_list( + struct xfs_attr_list_context *context) +{ + struct xfs_inode *dp = context->dp; + uint lock_mode; + int error; + + XFS_STATS_INC(dp->i_mount, xs_attr_list); + + if (xfs_is_shutdown(dp->i_mount)) + return -EIO; + + lock_mode = xfs_ilock_attr_map_shared(dp); + error = xfs_attr_list_ilocked(context); + xfs_iunlock(dp, lock_mode); + return error; +} diff --git a/fs/xfs/xfs_bio_io.c b/fs/xfs/xfs_bio_io.c new file mode 100644 index 000000000..b87e7975b --- /dev/null +++ b/fs/xfs/xfs_bio_io.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2019 Christoph Hellwig. + */ +#include "xfs_platform.h" + +static inline unsigned int bio_max_vecs(unsigned int count) +{ + return bio_max_segs(howmany(count, PAGE_SIZE)); +} + +int +xfs_rw_bdev( + struct block_device *bdev, + sector_t sector, + unsigned int count, + char *data, + enum req_op op) + +{ + unsigned int done = 0, added; + int error; + struct bio *bio; + + op |= REQ_META | REQ_SYNC; + if (!is_vmalloc_addr(data)) + return bdev_rw_virt(bdev, sector, data, count, op); + + bio = bio_alloc(bdev, bio_max_vecs(count), op, GFP_KERNEL); + bio->bi_iter.bi_sector = sector; + + do { + added = bio_add_vmalloc_chunk(bio, data + done, count - done); + if (!added) { + struct bio *prev = bio; + + bio = bio_alloc(prev->bi_bdev, + bio_max_vecs(count - done), + prev->bi_opf, GFP_KERNEL); + bio->bi_iter.bi_sector = bio_end_sector(prev); + bio_chain(prev, bio); + submit_bio(prev); + } + done += added; + } while (done < count); + + error = submit_bio_wait(bio); + bio_put(bio); + + if (op == REQ_OP_READ) + invalidate_kernel_vmap_range(data, count); + return error; +} diff --git a/fs/xfs/xfs_bmap_item.c b/fs/xfs/xfs_bmap_item.c new file mode 100644 index 000000000..89f6e79a9 --- /dev/null +++ b/fs/xfs/xfs_bmap_item.c @@ -0,0 +1,720 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2016 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_shared.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_trans_priv.h" +#include "xfs_bmap_item.h" +#include "xfs_log.h" +#include "xfs_bmap.h" +#include "xfs_icache.h" +#include "xfs_bmap_btree.h" +#include "xfs_trans_space.h" +#include "xfs_error.h" +#include "xfs_log_priv.h" +#include "xfs_log_recover.h" +#include "xfs_ag.h" +#include "xfs_trace.h" + +struct kmem_cache *xfs_bui_cache; +struct kmem_cache *xfs_bud_cache; + +static const struct xfs_item_ops xfs_bui_item_ops; + +static inline struct xfs_bui_log_item *BUI_ITEM(struct xfs_log_item *lip) +{ + return container_of(lip, struct xfs_bui_log_item, bui_item); +} + +STATIC void +xfs_bui_item_free( + struct xfs_bui_log_item *buip) +{ + kvfree(buip->bui_item.li_lv_shadow); + kmem_cache_free(xfs_bui_cache, buip); +} + +/* + * Freeing the BUI requires that we remove it from the AIL if it has already + * been placed there. However, the BUI may not yet have been placed in the AIL + * when called by xfs_bui_release() from BUD processing due to the ordering of + * committed vs unpin operations in bulk insert operations. Hence the reference + * count to ensure only the last caller frees the BUI. + */ +STATIC void +xfs_bui_release( + struct xfs_bui_log_item *buip) +{ + ASSERT(atomic_read(&buip->bui_refcount) > 0); + if (!atomic_dec_and_test(&buip->bui_refcount)) + return; + + xfs_trans_ail_delete(&buip->bui_item, 0); + xfs_bui_item_free(buip); +} + + +STATIC void +xfs_bui_item_size( + struct xfs_log_item *lip, + int *nvecs, + int *nbytes) +{ + struct xfs_bui_log_item *buip = BUI_ITEM(lip); + + *nvecs += 1; + *nbytes += xfs_bui_log_format_sizeof(buip->bui_format.bui_nextents); +} + +unsigned int xfs_bui_log_space(unsigned int nr) +{ + return xlog_item_space(1, xfs_bui_log_format_sizeof(nr)); +} + +/* + * This is called to fill in the vector of log iovecs for the + * given bui log item. We use only 1 iovec, and we point that + * at the bui_log_format structure embedded in the bui item. + * It is at this point that we assert that all of the extent + * slots in the bui item have been filled. + */ +STATIC void +xfs_bui_item_format( + struct xfs_log_item *lip, + struct xlog_format_buf *lfb) +{ + struct xfs_bui_log_item *buip = BUI_ITEM(lip); + + ASSERT(atomic_read(&buip->bui_next_extent) == + buip->bui_format.bui_nextents); + + buip->bui_format.bui_type = XFS_LI_BUI; + buip->bui_format.bui_size = 1; + + xlog_format_copy(lfb, XLOG_REG_TYPE_BUI_FORMAT, &buip->bui_format, + xfs_bui_log_format_sizeof(buip->bui_format.bui_nextents)); +} + +/* + * The unpin operation is the last place an BUI is manipulated in the log. It is + * either inserted in the AIL or aborted in the event of a log I/O error. In + * either case, the BUI transaction has been successfully committed to make it + * this far. Therefore, we expect whoever committed the BUI to either construct + * and commit the BUD or drop the BUD's reference in the event of error. Simply + * drop the log's BUI reference now that the log is done with it. + */ +STATIC void +xfs_bui_item_unpin( + struct xfs_log_item *lip, + int remove) +{ + struct xfs_bui_log_item *buip = BUI_ITEM(lip); + + xfs_bui_release(buip); +} + +/* + * The BUI has been either committed or aborted if the transaction has been + * cancelled. If the transaction was cancelled, an BUD isn't going to be + * constructed and thus we free the BUI here directly. + */ +STATIC void +xfs_bui_item_release( + struct xfs_log_item *lip) +{ + xfs_bui_release(BUI_ITEM(lip)); +} + +/* + * Allocate and initialize an bui item with the given number of extents. + */ +STATIC struct xfs_bui_log_item * +xfs_bui_init( + struct xfs_mount *mp) + +{ + struct xfs_bui_log_item *buip; + + buip = kmem_cache_zalloc(xfs_bui_cache, GFP_KERNEL | __GFP_NOFAIL); + + xfs_log_item_init(mp, &buip->bui_item, XFS_LI_BUI, &xfs_bui_item_ops); + buip->bui_format.bui_nextents = XFS_BUI_MAX_FAST_EXTENTS; + buip->bui_format.bui_id = (uintptr_t)(void *)buip; + atomic_set(&buip->bui_next_extent, 0); + atomic_set(&buip->bui_refcount, 2); + + return buip; +} + +static inline struct xfs_bud_log_item *BUD_ITEM(struct xfs_log_item *lip) +{ + return container_of(lip, struct xfs_bud_log_item, bud_item); +} + +STATIC void +xfs_bud_item_size( + struct xfs_log_item *lip, + int *nvecs, + int *nbytes) +{ + *nvecs += 1; + *nbytes += sizeof(struct xfs_bud_log_format); +} + +unsigned int xfs_bud_log_space(void) +{ + return xlog_item_space(1, sizeof(struct xfs_bud_log_format)); +} + +/* + * This is called to fill in the vector of log iovecs for the + * given bud log item. We use only 1 iovec, and we point that + * at the bud_log_format structure embedded in the bud item. + * It is at this point that we assert that all of the extent + * slots in the bud item have been filled. + */ +STATIC void +xfs_bud_item_format( + struct xfs_log_item *lip, + struct xlog_format_buf *lfb) +{ + struct xfs_bud_log_item *budp = BUD_ITEM(lip); + + budp->bud_format.bud_type = XFS_LI_BUD; + budp->bud_format.bud_size = 1; + + xlog_format_copy(lfb, XLOG_REG_TYPE_BUD_FORMAT, &budp->bud_format, + sizeof(struct xfs_bud_log_format)); +} + +/* + * The BUD is either committed or aborted if the transaction is cancelled. If + * the transaction is cancelled, drop our reference to the BUI and free the + * BUD. + */ +STATIC void +xfs_bud_item_release( + struct xfs_log_item *lip) +{ + struct xfs_bud_log_item *budp = BUD_ITEM(lip); + + xfs_bui_release(budp->bud_buip); + kvfree(budp->bud_item.li_lv_shadow); + kmem_cache_free(xfs_bud_cache, budp); +} + +static struct xfs_log_item * +xfs_bud_item_intent( + struct xfs_log_item *lip) +{ + return &BUD_ITEM(lip)->bud_buip->bui_item; +} + +static const struct xfs_item_ops xfs_bud_item_ops = { + .flags = XFS_ITEM_RELEASE_WHEN_COMMITTED | + XFS_ITEM_INTENT_DONE, + .iop_size = xfs_bud_item_size, + .iop_format = xfs_bud_item_format, + .iop_release = xfs_bud_item_release, + .iop_intent = xfs_bud_item_intent, +}; + +static inline struct xfs_bmap_intent *bi_entry(const struct list_head *e) +{ + return list_entry(e, struct xfs_bmap_intent, bi_list); +} + +/* Sort bmap intents by inode. */ +static int +xfs_bmap_update_diff_items( + void *priv, + const struct list_head *a, + const struct list_head *b) +{ + struct xfs_bmap_intent *ba = bi_entry(a); + struct xfs_bmap_intent *bb = bi_entry(b); + + return cmp_int(I_INO(ba->bi_owner), I_INO(bb->bi_owner)); +} + +/* Log bmap updates in the intent item. */ +STATIC void +xfs_bmap_update_log_item( + struct xfs_trans *tp, + struct xfs_bui_log_item *buip, + struct xfs_bmap_intent *bi) +{ + uint next_extent; + struct xfs_map_extent *map; + + /* + * atomic_inc_return gives us the value after the increment; + * we want to use it as an array index so we need to subtract 1 from + * it. + */ + next_extent = atomic_inc_return(&buip->bui_next_extent) - 1; + ASSERT(next_extent < buip->bui_format.bui_nextents); + map = &buip->bui_format.bui_extents[next_extent]; + map->me_owner = I_INO(bi->bi_owner); + map->me_startblock = bi->bi_bmap.br_startblock; + map->me_startoff = bi->bi_bmap.br_startoff; + map->me_len = bi->bi_bmap.br_blockcount; + + switch (bi->bi_type) { + case XFS_BMAP_MAP: + case XFS_BMAP_UNMAP: + map->me_flags = bi->bi_type; + break; + default: + ASSERT(0); + } + if (bi->bi_bmap.br_state == XFS_EXT_UNWRITTEN) + map->me_flags |= XFS_BMAP_EXTENT_UNWRITTEN; + if (bi->bi_whichfork == XFS_ATTR_FORK) + map->me_flags |= XFS_BMAP_EXTENT_ATTR_FORK; + if (xfs_ifork_is_realtime(bi->bi_owner, bi->bi_whichfork)) + map->me_flags |= XFS_BMAP_EXTENT_REALTIME; +} + +static struct xfs_log_item * +xfs_bmap_update_create_intent( + struct xfs_trans *tp, + struct list_head *items, + unsigned int count, + bool sort) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_bui_log_item *buip = xfs_bui_init(mp); + struct xfs_bmap_intent *bi; + + ASSERT(count == XFS_BUI_MAX_FAST_EXTENTS); + + if (sort) + list_sort(mp, items, xfs_bmap_update_diff_items); + list_for_each_entry(bi, items, bi_list) + xfs_bmap_update_log_item(tp, buip, bi); + return &buip->bui_item; +} + +/* Get an BUD so we can process all the deferred bmap updates. */ +static struct xfs_log_item * +xfs_bmap_update_create_done( + struct xfs_trans *tp, + struct xfs_log_item *intent, + unsigned int count) +{ + struct xfs_bui_log_item *buip = BUI_ITEM(intent); + struct xfs_bud_log_item *budp; + + budp = kmem_cache_zalloc(xfs_bud_cache, GFP_KERNEL | __GFP_NOFAIL); + xfs_log_item_init(tp->t_mountp, &budp->bud_item, XFS_LI_BUD, + &xfs_bud_item_ops); + budp->bud_buip = buip; + budp->bud_format.bud_bui_id = buip->bui_format.bui_id; + + return &budp->bud_item; +} + +/* Take a passive ref to the group containing the space we're mapping. */ +static inline void +xfs_bmap_update_get_group( + struct xfs_mount *mp, + struct xfs_bmap_intent *bi) +{ + enum xfs_group_type type = XG_TYPE_AG; + + if (xfs_ifork_is_realtime(bi->bi_owner, bi->bi_whichfork)) + type = XG_TYPE_RTG; + + /* + * Bump the intent count on behalf of the deferred rmap and refcount + * intent items that that we can queue when we finish this bmap work. + * This new intent item will bump the intent count before the bmap + * intent drops the intent count, ensuring that the intent count + * remains nonzero across the transaction roll. + */ + bi->bi_group = xfs_group_intent_get(mp, bi->bi_bmap.br_startblock, + type); +} + +/* Add this deferred BUI to the transaction. */ +void +xfs_bmap_defer_add( + struct xfs_trans *tp, + struct xfs_bmap_intent *bi) +{ + xfs_bmap_update_get_group(tp->t_mountp, bi); + + /* + * Ensure the deferred mapping is pre-recorded in i_delayed_blks. + * + * Otherwise stat can report zero blocks for an inode that actually has + * data when the entire mapping is in the process of being overwritten + * using the out of place write path. This is undone in xfs_bmapi_remap + * after it has incremented di_nblocks for a successful operation. + */ + if (bi->bi_type == XFS_BMAP_MAP) + bi->bi_owner->i_delayed_blks += bi->bi_bmap.br_blockcount; + + trace_xfs_bmap_defer(bi); + xfs_defer_add(tp, &bi->bi_list, &xfs_bmap_update_defer_type); +} + +/* Cancel a deferred bmap update. */ +STATIC void +xfs_bmap_update_cancel_item( + struct list_head *item) +{ + struct xfs_bmap_intent *bi = bi_entry(item); + + if (bi->bi_type == XFS_BMAP_MAP) + bi->bi_owner->i_delayed_blks -= bi->bi_bmap.br_blockcount; + + xfs_group_intent_put(bi->bi_group); + kmem_cache_free(xfs_bmap_intent_cache, bi); +} + +/* Process a deferred bmap update. */ +STATIC int +xfs_bmap_update_finish_item( + struct xfs_trans *tp, + struct xfs_log_item *done, + struct list_head *item, + struct xfs_btree_cur **state) +{ + struct xfs_bmap_intent *bi = bi_entry(item); + int error; + + error = xfs_bmap_finish_one(tp, bi); + if (!error && bi->bi_bmap.br_blockcount > 0) { + ASSERT(bi->bi_type == XFS_BMAP_UNMAP); + return -EAGAIN; + } + + xfs_bmap_update_cancel_item(item); + return error; +} + +/* Abort all pending BUIs. */ +STATIC void +xfs_bmap_update_abort_intent( + struct xfs_log_item *intent) +{ + xfs_bui_release(BUI_ITEM(intent)); +} + +/* Is this recovered BUI ok? */ +static inline bool +xfs_bui_validate( + struct xfs_mount *mp, + struct xfs_bui_log_item *buip) +{ + struct xfs_map_extent *map; + + /* Only one mapping operation per BUI... */ + if (buip->bui_format.bui_nextents != XFS_BUI_MAX_FAST_EXTENTS) + return false; + + map = &buip->bui_format.bui_extents[0]; + + if (map->me_flags & ~XFS_BMAP_EXTENT_FLAGS) + return false; + + switch (map->me_flags & XFS_BMAP_EXTENT_TYPE_MASK) { + case XFS_BMAP_MAP: + case XFS_BMAP_UNMAP: + break; + default: + return false; + } + + if (!xfs_verify_ino(mp, map->me_owner)) + return false; + + if (!xfs_verify_fileext(mp, map->me_startoff, map->me_len)) + return false; + + if (map->me_flags & XFS_BMAP_EXTENT_REALTIME) + return xfs_verify_rtbext(mp, map->me_startblock, map->me_len); + + return xfs_verify_fsbext(mp, map->me_startblock, map->me_len); +} + +static inline struct xfs_bmap_intent * +xfs_bui_recover_work( + struct xfs_mount *mp, + struct xfs_defer_pending *dfp, + struct xfs_inode **ipp, + struct xfs_map_extent *map) +{ + struct xfs_bmap_intent *bi; + int error; + + error = xlog_recover_iget(mp, map->me_owner, ipp); + if (error) + return ERR_PTR(error); + + bi = kmem_cache_zalloc(xfs_bmap_intent_cache, + GFP_KERNEL | __GFP_NOFAIL); + bi->bi_whichfork = (map->me_flags & XFS_BMAP_EXTENT_ATTR_FORK) ? + XFS_ATTR_FORK : XFS_DATA_FORK; + bi->bi_type = map->me_flags & XFS_BMAP_EXTENT_TYPE_MASK; + bi->bi_bmap.br_startblock = map->me_startblock; + bi->bi_bmap.br_startoff = map->me_startoff; + bi->bi_bmap.br_blockcount = map->me_len; + bi->bi_bmap.br_state = (map->me_flags & XFS_BMAP_EXTENT_UNWRITTEN) ? + XFS_EXT_UNWRITTEN : XFS_EXT_NORM; + bi->bi_owner = *ipp; + xfs_bmap_update_get_group(mp, bi); + + /* see xfs_bmap_defer_add for details */ + if (bi->bi_type == XFS_BMAP_MAP) + bi->bi_owner->i_delayed_blks += bi->bi_bmap.br_blockcount; + xfs_defer_add_item(dfp, &bi->bi_list); + return bi; +} + +/* + * Process a bmap update intent item that was recovered from the log. + * We need to update some inode's bmbt. + */ +STATIC int +xfs_bmap_recover_work( + struct xfs_defer_pending *dfp, + struct list_head *capture_list) +{ + struct xfs_trans_res resv; + struct xfs_log_item *lip = dfp->dfp_intent; + struct xfs_bui_log_item *buip = BUI_ITEM(lip); + struct xfs_trans *tp; + struct xfs_inode *ip = NULL; + struct xfs_mount *mp = lip->li_log->l_mp; + struct xfs_map_extent *map; + struct xfs_bmap_intent *work; + int iext_delta; + int error = 0; + + if (!xfs_bui_validate(mp, buip)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + &buip->bui_format, sizeof(buip->bui_format)); + return -EFSCORRUPTED; + } + + map = &buip->bui_format.bui_extents[0]; + work = xfs_bui_recover_work(mp, dfp, &ip, map); + if (IS_ERR(work)) + return PTR_ERR(work); + + /* Allocate transaction and do the work. */ + resv = xlog_recover_resv(&M_RES(mp)->tr_itruncate); + error = xfs_trans_alloc(mp, &resv, + XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK), 0, 0, &tp); + if (error) + goto err_rele; + + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, 0); + + if (!!(map->me_flags & XFS_BMAP_EXTENT_REALTIME) != + xfs_ifork_is_realtime(ip, work->bi_whichfork)) { + error = -EFSCORRUPTED; + goto err_cancel; + } + + if (work->bi_type == XFS_BMAP_MAP) + iext_delta = XFS_IEXT_ADD_NOSPLIT_CNT; + else + iext_delta = XFS_IEXT_PUNCH_HOLE_CNT; + + error = xfs_iext_count_extend(tp, ip, work->bi_whichfork, iext_delta); + if (error) + goto err_cancel; + + error = xlog_recover_finish_intent(tp, dfp); + if (error == -EFSCORRUPTED) + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + &buip->bui_format, sizeof(buip->bui_format)); + if (error) + goto err_cancel; + + /* + * Commit transaction, which frees the transaction and saves the inode + * for later replay activities. + */ + error = xfs_defer_ops_capture_and_commit(tp, capture_list); + if (error) + goto err_unlock; + + xfs_iunlock(ip, XFS_ILOCK_EXCL); + xfs_irele(ip); + return 0; + +err_cancel: + xfs_trans_cancel(tp); +err_unlock: + xfs_iunlock(ip, XFS_ILOCK_EXCL); +err_rele: + xfs_irele(ip); + return error; +} + +/* Relog an intent item to push the log tail forward. */ +static struct xfs_log_item * +xfs_bmap_relog_intent( + struct xfs_trans *tp, + struct xfs_log_item *intent, + struct xfs_log_item *done_item) +{ + struct xfs_bui_log_item *buip; + struct xfs_map_extent *map; + unsigned int count; + + count = BUI_ITEM(intent)->bui_format.bui_nextents; + map = BUI_ITEM(intent)->bui_format.bui_extents; + + buip = xfs_bui_init(tp->t_mountp); + memcpy(buip->bui_format.bui_extents, map, count * sizeof(*map)); + atomic_set(&buip->bui_next_extent, count); + + return &buip->bui_item; +} + +const struct xfs_defer_op_type xfs_bmap_update_defer_type = { + .name = "bmap", + .max_items = XFS_BUI_MAX_FAST_EXTENTS, + .create_intent = xfs_bmap_update_create_intent, + .abort_intent = xfs_bmap_update_abort_intent, + .create_done = xfs_bmap_update_create_done, + .finish_item = xfs_bmap_update_finish_item, + .cancel_item = xfs_bmap_update_cancel_item, + .recover_work = xfs_bmap_recover_work, + .relog_intent = xfs_bmap_relog_intent, +}; + +STATIC bool +xfs_bui_item_match( + struct xfs_log_item *lip, + uint64_t intent_id) +{ + return BUI_ITEM(lip)->bui_format.bui_id == intent_id; +} + +static const struct xfs_item_ops xfs_bui_item_ops = { + .flags = XFS_ITEM_INTENT, + .iop_size = xfs_bui_item_size, + .iop_format = xfs_bui_item_format, + .iop_unpin = xfs_bui_item_unpin, + .iop_release = xfs_bui_item_release, + .iop_match = xfs_bui_item_match, +}; + +static inline void +xfs_bui_copy_format( + struct xfs_bui_log_format *dst, + const struct xfs_bui_log_format *src) +{ + unsigned int i; + + memcpy(dst, src, offsetof(struct xfs_bui_log_format, bui_extents)); + + for (i = 0; i < src->bui_nextents; i++) + memcpy(&dst->bui_extents[i], &src->bui_extents[i], + sizeof(struct xfs_map_extent)); +} + +/* + * This routine is called to create an in-core extent bmap update + * item from the bui format structure which was logged on disk. + * It allocates an in-core bui, copies the extents from the format + * structure into it, and adds the bui to the AIL with the given + * LSN. + */ +STATIC int +xlog_recover_bui_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_mount *mp = log->l_mp; + struct xfs_bui_log_item *buip; + struct xfs_bui_log_format *bui_formatp; + size_t len; + + bui_formatp = item->ri_buf[0].iov_base; + + if (item->ri_buf[0].iov_len < xfs_bui_log_format_sizeof(0)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[0].iov_base, item->ri_buf[0].iov_len); + return -EFSCORRUPTED; + } + + if (bui_formatp->bui_nextents != XFS_BUI_MAX_FAST_EXTENTS) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[0].iov_base, item->ri_buf[0].iov_len); + return -EFSCORRUPTED; + } + + len = xfs_bui_log_format_sizeof(bui_formatp->bui_nextents); + if (item->ri_buf[0].iov_len != len) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[0].iov_base, item->ri_buf[0].iov_len); + return -EFSCORRUPTED; + } + + buip = xfs_bui_init(mp); + xfs_bui_copy_format(&buip->bui_format, bui_formatp); + atomic_set(&buip->bui_next_extent, bui_formatp->bui_nextents); + + xlog_recover_intent_item(log, &buip->bui_item, lsn, + &xfs_bmap_update_defer_type); + return 0; +} + +const struct xlog_recover_item_ops xlog_bui_item_ops = { + .item_type = XFS_LI_BUI, + .commit_pass2 = xlog_recover_bui_commit_pass2, +}; + +/* + * This routine is called when an BUD format structure is found in a committed + * transaction in the log. Its purpose is to cancel the corresponding BUI if it + * was still in the log. To do this it searches the AIL for the BUI with an id + * equal to that in the BUD format structure. If we find it we drop the BUD + * reference, which removes the BUI from the AIL and frees it. + */ +STATIC int +xlog_recover_bud_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_bud_log_format *bud_formatp; + + bud_formatp = item->ri_buf[0].iov_base; + if (item->ri_buf[0].iov_len != sizeof(struct xfs_bud_log_format)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, log->l_mp, + item->ri_buf[0].iov_base, item->ri_buf[0].iov_len); + return -EFSCORRUPTED; + } + + xlog_recover_release_intent(log, XFS_LI_BUI, bud_formatp->bud_bui_id); + return 0; +} + +const struct xlog_recover_item_ops xlog_bud_item_ops = { + .item_type = XFS_LI_BUD, + .commit_pass2 = xlog_recover_bud_commit_pass2, +}; diff --git a/fs/xfs/xfs_bmap_item.h b/fs/xfs/xfs_bmap_item.h new file mode 100644 index 000000000..b42fee068 --- /dev/null +++ b/fs/xfs/xfs_bmap_item.h @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2016 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_BMAP_ITEM_H__ +#define __XFS_BMAP_ITEM_H__ + +/* + * There are (currently) two pairs of bmap btree redo item types: map & unmap. + * The common abbreviations for these are BUI (bmap update intent) and BUD + * (bmap update done). The redo item type is encoded in the flags field of + * each xfs_map_extent. + * + * *I items should be recorded in the *first* of a series of rolled + * transactions, and the *D items should be recorded in the same transaction + * that records the associated bmbt updates. + * + * Should the system crash after the commit of the first transaction but + * before the commit of the final transaction in a series, log recovery will + * use the redo information recorded by the intent items to replay the + * bmbt metadata updates in the non-first transaction. + */ + +/* kernel only BUI/BUD definitions */ + +struct xfs_mount; +struct kmem_cache; + +/* + * Max number of extents in fast allocation path. + */ +#define XFS_BUI_MAX_FAST_EXTENTS 1 + +/* + * This is the "bmap update intent" log item. It is used to log the fact that + * some reverse mappings need to change. It is used in conjunction with the + * "bmap update done" log item described below. + * + * These log items follow the same rules as struct xfs_efi_log_item; see the + * comments about that structure (in xfs_extfree_item.h) for more details. + */ +struct xfs_bui_log_item { + struct xfs_log_item bui_item; + atomic_t bui_refcount; + atomic_t bui_next_extent; + struct xfs_bui_log_format bui_format; +}; + +static inline size_t +xfs_bui_log_item_sizeof( + unsigned int nr) +{ + return offsetof(struct xfs_bui_log_item, bui_format) + + xfs_bui_log_format_sizeof(nr); +} + +/* + * This is the "bmap update done" log item. It is used to log the fact that + * some bmbt updates mentioned in an earlier bui item have been performed. + */ +struct xfs_bud_log_item { + struct xfs_log_item bud_item; + struct xfs_bui_log_item *bud_buip; + struct xfs_bud_log_format bud_format; +}; + +extern struct kmem_cache *xfs_bui_cache; +extern struct kmem_cache *xfs_bud_cache; + +struct xfs_bmap_intent; + +void xfs_bmap_defer_add(struct xfs_trans *tp, struct xfs_bmap_intent *bi); + +unsigned int xfs_bui_log_space(unsigned int nr); +unsigned int xfs_bud_log_space(void); + +#endif /* __XFS_BMAP_ITEM_H__ */ diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c new file mode 100644 index 000000000..268d15933 --- /dev/null +++ b/fs/xfs/xfs_bmap_util.c @@ -0,0 +1,1865 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2006 Silicon Graphics, Inc. + * Copyright (c) 2012 Red Hat, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_inode.h" +#include "xfs_btree.h" +#include "xfs_trans.h" +#include "xfs_alloc.h" +#include "xfs_bmap.h" +#include "xfs_bmap_util.h" +#include "xfs_bmap_btree.h" +#include "xfs_rtalloc.h" +#include "xfs_error.h" +#include "xfs_quota.h" +#include "xfs_trans_space.h" +#include "xfs_trace.h" +#include "xfs_icache.h" +#include "xfs_iomap.h" +#include "xfs_reflink.h" +#include "xfs_rtbitmap.h" +#include "xfs_rtgroup.h" +#include "xfs_zone_alloc.h" + +/* Kernel only BMAP related definitions and functions */ + +/* + * Convert the given file system block to a disk block. We have to treat it + * differently based on whether the file is a real time file or not, because the + * bmap code does. + */ +xfs_daddr_t +xfs_fsb_to_db(struct xfs_inode *ip, xfs_fsblock_t fsb) +{ + if (XFS_IS_REALTIME_INODE(ip)) + return xfs_rtb_to_daddr(ip->i_mount, fsb); + return XFS_FSB_TO_DADDR(ip->i_mount, fsb); +} + +/* + * Routine to zero an extent on disk allocated to the specific inode. + */ +int +xfs_zero_extent( + struct xfs_inode *ip, + xfs_fsblock_t start_fsb, + xfs_off_t count_fsb) +{ + return blkdev_issue_zeroout(xfs_inode_buftarg(ip)->bt_bdev, + xfs_fsb_to_db(ip, start_fsb), + XFS_FSB_TO_BB(ip->i_mount, count_fsb), + GFP_KERNEL, 0); +} + +/* + * Extent tree block counting routines. + */ + +/* + * Count leaf blocks given a range of extent records. Delayed allocation + * extents are not counted towards the totals. + */ +xfs_extnum_t +xfs_bmap_count_leaves( + struct xfs_ifork *ifp, + xfs_filblks_t *count) +{ + struct xfs_iext_cursor icur; + struct xfs_bmbt_irec got; + xfs_extnum_t numrecs = 0; + + for_each_xfs_iext(ifp, &icur, &got) { + if (!isnullstartblock(got.br_startblock)) { + *count += got.br_blockcount; + numrecs++; + } + } + + return numrecs; +} + +/* + * Count fsblocks of the given fork. Delayed allocation extents are + * not counted towards the totals. + */ +int +xfs_bmap_count_blocks( + struct xfs_trans *tp, + struct xfs_inode *ip, + int whichfork, + xfs_extnum_t *nextents, + xfs_filblks_t *count) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + struct xfs_btree_cur *cur; + xfs_filblks_t btblocks = 0; + int error; + + *nextents = 0; + *count = 0; + + if (!ifp) + return 0; + + switch (ifp->if_format) { + case XFS_DINODE_FMT_BTREE: + error = xfs_iread_extents(tp, ip, whichfork); + if (error) + return error; + + cur = xfs_bmbt_init_cursor(mp, tp, ip, whichfork); + error = xfs_btree_count_blocks(cur, &btblocks); + xfs_btree_del_cursor(cur, error); + if (error) + return error; + + /* + * xfs_btree_count_blocks includes the root block contained in + * the inode fork in @btblocks, so subtract one because we're + * only interested in allocated disk blocks. + */ + *count += btblocks - 1; + + fallthrough; + case XFS_DINODE_FMT_EXTENTS: + *nextents = xfs_bmap_count_leaves(ifp, count); + break; + } + + return 0; +} + +static int +xfs_getbmap_report_one( + struct xfs_inode *ip, + struct getbmapx *bmv, + struct kgetbmap *out, + int64_t bmv_end, + struct xfs_bmbt_irec *got) +{ + struct kgetbmap *p = out + bmv->bmv_entries; + bool shared = false; + int error; + + error = xfs_reflink_trim_around_shared(ip, got, &shared); + if (error) + return error; + + if (isnullstartblock(got->br_startblock) || + got->br_startblock == DELAYSTARTBLOCK) { + /* + * Take the flush completion as being a point-in-time snapshot + * where there are no delalloc extents, and if any new ones + * have been created racily, just skip them as being 'after' + * the flush and so don't get reported. + */ + if (!(bmv->bmv_iflags & BMV_IF_DELALLOC)) + return 0; + + p->bmv_oflags |= BMV_OF_DELALLOC; + p->bmv_block = -2; + } else { + p->bmv_block = xfs_fsb_to_db(ip, got->br_startblock); + } + + if (got->br_state == XFS_EXT_UNWRITTEN && + (bmv->bmv_iflags & BMV_IF_PREALLOC)) + p->bmv_oflags |= BMV_OF_PREALLOC; + + if (shared) + p->bmv_oflags |= BMV_OF_SHARED; + + p->bmv_offset = XFS_FSB_TO_BB(ip->i_mount, got->br_startoff); + p->bmv_length = XFS_FSB_TO_BB(ip->i_mount, got->br_blockcount); + + bmv->bmv_offset = p->bmv_offset + p->bmv_length; + bmv->bmv_length = max(0LL, bmv_end - bmv->bmv_offset); + bmv->bmv_entries++; + return 0; +} + +static void +xfs_getbmap_report_hole( + struct xfs_inode *ip, + struct getbmapx *bmv, + struct kgetbmap *out, + int64_t bmv_end, + xfs_fileoff_t bno, + xfs_fileoff_t end) +{ + struct kgetbmap *p = out + bmv->bmv_entries; + + if (bmv->bmv_iflags & BMV_IF_NO_HOLES) + return; + + p->bmv_block = -1; + p->bmv_offset = XFS_FSB_TO_BB(ip->i_mount, bno); + p->bmv_length = XFS_FSB_TO_BB(ip->i_mount, end - bno); + + bmv->bmv_offset = p->bmv_offset + p->bmv_length; + bmv->bmv_length = max(0LL, bmv_end - bmv->bmv_offset); + bmv->bmv_entries++; +} + +static inline bool +xfs_getbmap_full( + struct getbmapx *bmv) +{ + return bmv->bmv_length == 0 || bmv->bmv_entries >= bmv->bmv_count - 1; +} + +static bool +xfs_getbmap_next_rec( + struct xfs_bmbt_irec *rec, + xfs_fileoff_t total_end) +{ + xfs_fileoff_t end = rec->br_startoff + rec->br_blockcount; + + if (end == total_end) + return false; + + rec->br_startoff += rec->br_blockcount; + if (!isnullstartblock(rec->br_startblock) && + rec->br_startblock != DELAYSTARTBLOCK) + rec->br_startblock += rec->br_blockcount; + rec->br_blockcount = total_end - end; + return true; +} + +/* + * Get inode's extents as described in bmv, and format for output. + * Calls formatter to fill the user's buffer until all extents + * are mapped, until the passed-in bmv->bmv_count slots have + * been filled, or until the formatter short-circuits the loop, + * if it is tracking filled-in extents on its own. + */ +int /* error code */ +xfs_getbmap( + struct xfs_inode *ip, + struct getbmapx *bmv, /* user bmap structure */ + struct kgetbmap *out) +{ + struct xfs_mount *mp = ip->i_mount; + int iflags = bmv->bmv_iflags; + int whichfork, lock, error = 0; + int64_t bmv_end, max_len; + xfs_fileoff_t bno, first_bno; + struct xfs_ifork *ifp; + struct xfs_bmbt_irec got, rec; + xfs_filblks_t len; + struct xfs_iext_cursor icur; + + if (bmv->bmv_iflags & ~BMV_IF_VALID) + return -EINVAL; +#ifndef DEBUG + /* Only allow CoW fork queries if we're debugging. */ + if (iflags & BMV_IF_COWFORK) + return -EINVAL; +#endif + if ((iflags & BMV_IF_ATTRFORK) && (iflags & BMV_IF_COWFORK)) + return -EINVAL; + + if (bmv->bmv_length < -1) + return -EINVAL; + bmv->bmv_entries = 0; + if (bmv->bmv_length == 0) + return 0; + + if (iflags & BMV_IF_ATTRFORK) + whichfork = XFS_ATTR_FORK; + else if (iflags & BMV_IF_COWFORK) + whichfork = XFS_COW_FORK; + else + whichfork = XFS_DATA_FORK; + + xfs_ilock(ip, XFS_IOLOCK_SHARED); + switch (whichfork) { + case XFS_ATTR_FORK: + lock = xfs_ilock_attr_map_shared(ip); + if (!xfs_inode_has_attr_fork(ip)) + goto out_unlock_ilock; + + max_len = 1LL << 32; + break; + case XFS_COW_FORK: + lock = XFS_ILOCK_SHARED; + xfs_ilock(ip, lock); + + /* No CoW fork? Just return */ + if (!xfs_ifork_ptr(ip, whichfork)) + goto out_unlock_ilock; + + if (xfs_get_cowextsz_hint(ip)) + max_len = mp->m_super->s_maxbytes; + else + max_len = XFS_ISIZE(ip); + break; + case XFS_DATA_FORK: + if (!(iflags & BMV_IF_DELALLOC) && + (ip->i_delayed_blks || XFS_ISIZE(ip) > ip->i_disk_size)) { + error = filemap_write_and_wait(VFS_I(ip)->i_mapping); + if (error) + goto out_unlock_iolock; + + /* + * Even after flushing the inode, there can still be + * delalloc blocks on the inode beyond EOF due to + * speculative preallocation. These are not removed + * until the release function is called or the inode + * is inactivated. Hence we cannot assert here that + * ip->i_delayed_blks == 0. + */ + } + + if (xfs_get_extsz_hint(ip) || + (ip->i_diflags & XFS_DIFLAG_PREALLOC)) + max_len = mp->m_super->s_maxbytes; + else + max_len = XFS_ISIZE(ip); + + lock = xfs_ilock_data_map_shared(ip); + break; + } + + ifp = xfs_ifork_ptr(ip, whichfork); + + switch (ifp->if_format) { + case XFS_DINODE_FMT_EXTENTS: + case XFS_DINODE_FMT_BTREE: + break; + case XFS_DINODE_FMT_LOCAL: + /* Local format inode forks report no extents. */ + goto out_unlock_ilock; + default: + error = -EINVAL; + goto out_unlock_ilock; + } + + if (bmv->bmv_length == -1) { + max_len = XFS_FSB_TO_BB(mp, XFS_B_TO_FSB(mp, max_len)); + bmv->bmv_length = max(0LL, max_len - bmv->bmv_offset); + } + + bmv_end = bmv->bmv_offset + bmv->bmv_length; + + first_bno = bno = XFS_BB_TO_FSBT(mp, bmv->bmv_offset); + len = XFS_BB_TO_FSB(mp, bmv->bmv_length); + + error = xfs_iread_extents(NULL, ip, whichfork); + if (error) + goto out_unlock_ilock; + + if (!xfs_iext_lookup_extent(ip, ifp, bno, &icur, &got)) { + /* + * Report a whole-file hole if the delalloc flag is set to + * stay compatible with the old implementation. + */ + if (iflags & BMV_IF_DELALLOC) + xfs_getbmap_report_hole(ip, bmv, out, bmv_end, bno, + XFS_B_TO_FSB(mp, XFS_ISIZE(ip))); + goto out_unlock_ilock; + } + + while (!xfs_getbmap_full(bmv)) { + xfs_trim_extent(&got, first_bno, len); + + /* + * Report an entry for a hole if this extent doesn't directly + * follow the previous one. + */ + if (got.br_startoff > bno) { + xfs_getbmap_report_hole(ip, bmv, out, bmv_end, bno, + got.br_startoff); + if (xfs_getbmap_full(bmv)) + break; + } + + /* + * In order to report shared extents accurately, we report each + * distinct shared / unshared part of a single bmbt record with + * an individual getbmapx record. + */ + bno = got.br_startoff + got.br_blockcount; + rec = got; + do { + error = xfs_getbmap_report_one(ip, bmv, out, bmv_end, + &rec); + if (error || xfs_getbmap_full(bmv)) + goto out_unlock_ilock; + } while (xfs_getbmap_next_rec(&rec, bno)); + + if (!xfs_iext_next_extent(ifp, &icur, &got)) { + xfs_fileoff_t end = XFS_B_TO_FSB(mp, XFS_ISIZE(ip)); + + if (bmv->bmv_entries > 0) + out[bmv->bmv_entries - 1].bmv_oflags |= + BMV_OF_LAST; + + if (whichfork != XFS_ATTR_FORK && bno < end && + !xfs_getbmap_full(bmv)) { + xfs_getbmap_report_hole(ip, bmv, out, bmv_end, + bno, end); + } + break; + } + + if (bno >= first_bno + len) + break; + } + +out_unlock_ilock: + xfs_iunlock(ip, lock); +out_unlock_iolock: + xfs_iunlock(ip, XFS_IOLOCK_SHARED); + return error; +} + +/* + * Dead simple method of punching delalyed allocation blocks from a range in + * the inode. This will always punch out both the start and end blocks, even + * if the ranges only partially overlap them, so it is up to the caller to + * ensure that partial blocks are not passed in. + */ +void +xfs_bmap_punch_delalloc_range( + struct xfs_inode *ip, + int whichfork, + xfs_off_t start_byte, + xfs_off_t end_byte, + struct xfs_zone_alloc_ctx *ac) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + xfs_fileoff_t start_fsb = XFS_B_TO_FSBT(mp, start_byte); + xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, end_byte); + struct xfs_bmbt_irec got, del; + struct xfs_iext_cursor icur; + + ASSERT(!xfs_need_iread_extents(ifp)); + + xfs_ilock(ip, XFS_ILOCK_EXCL); + if (!xfs_iext_lookup_extent_before(ip, ifp, &end_fsb, &icur, &got)) + goto out_unlock; + + while (got.br_startoff + got.br_blockcount > start_fsb) { + del = got; + xfs_trim_extent(&del, start_fsb, end_fsb - start_fsb); + + /* + * A delete can push the cursor forward. Step back to the + * previous extent on non-delalloc or extents outside the + * target range. + */ + if (!del.br_blockcount || + !isnullstartblock(del.br_startblock)) { + if (!xfs_iext_prev_extent(ifp, &icur, &got)) + break; + continue; + } + + if (xfs_is_zoned_inode(ip) && ac) { + /* + * In a zoned buffered write context we need to return + * the punched delalloc allocations to the allocation + * context. This allows reusing them in the following + * iomap iterations. + */ + xfs_bmap_del_extent_delay(ip, whichfork, &icur, &got, + &del, XFS_BMAPI_REMAP); + ac->reserved_blocks += del.br_blockcount; + } else { + xfs_bmap_del_extent_delay(ip, whichfork, &icur, &got, + &del, 0); + } + + if (!xfs_iext_get_extent(ifp, &icur, &got)) + break; + } + + if (whichfork == XFS_COW_FORK && !ifp->if_bytes) + xfs_inode_clear_cowblocks_tag(ip); + +out_unlock: + xfs_iunlock(ip, XFS_ILOCK_EXCL); +} + +/* + * Test whether it is appropriate to check an inode for and free post EOF + * blocks. + */ +bool +xfs_can_free_eofblocks( + struct xfs_inode *ip) +{ + struct xfs_mount *mp = ip->i_mount; + bool found_blocks = false; + xfs_fileoff_t end_fsb; + xfs_fileoff_t last_fsb; + struct xfs_bmbt_irec imap; + struct xfs_iext_cursor icur; + + /* + * Caller must either hold the exclusive io lock; or be inactivating + * the inode, which guarantees there are no other users of the inode. + */ + if (!(inode_state_read_once(VFS_I(ip)) & I_FREEING)) + xfs_assert_ilocked(ip, XFS_IOLOCK_EXCL); + + /* prealloc/delalloc exists only on regular files */ + if (!S_ISREG(VFS_I(ip)->i_mode)) + return false; + + /* + * Zero sized files with no cached pages and delalloc blocks will not + * have speculative prealloc/delalloc blocks to remove. + */ + if (VFS_I(ip)->i_size == 0 && + VFS_I(ip)->i_mapping->nrpages == 0 && + ip->i_delayed_blks == 0) + return false; + + /* If we haven't read in the extent list, then don't do it now. */ + if (xfs_need_iread_extents(&ip->i_df)) + return false; + + /* + * Do not free real extents in preallocated files unless the file has + * delalloc blocks and we are forced to remove them. + */ + if ((ip->i_diflags & XFS_DIFLAG_PREALLOC) && !ip->i_delayed_blks) + return false; + + /* + * Do not try to free post-EOF blocks if EOF is beyond the end of the + * range supported by the page cache, because the truncation will loop + * forever. + */ + end_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)XFS_ISIZE(ip)); + if (xfs_inode_has_bigrtalloc(ip)) + end_fsb = xfs_fileoff_roundup_rtx(mp, end_fsb); + last_fsb = XFS_B_TO_FSB(mp, mp->m_super->s_maxbytes); + if (last_fsb <= end_fsb) + return false; + + /* + * Check if there is an post-EOF extent to free. If there are any + * delalloc blocks attached to the inode (data fork delalloc + * reservations or CoW extents of any kind), we need to free them so + * that inactivation doesn't fail to erase them. + */ + xfs_ilock(ip, XFS_ILOCK_SHARED); + if (ip->i_delayed_blks || + xfs_iext_lookup_extent(ip, &ip->i_df, end_fsb, &icur, &imap)) + found_blocks = true; + xfs_iunlock(ip, XFS_ILOCK_SHARED); + return found_blocks; +} + +/* + * This is called to free any blocks beyond eof. The caller must hold + * IOLOCK_EXCL unless we are in the inode reclaim path and have the only + * reference to the inode. + */ +int +xfs_free_eofblocks( + struct xfs_inode *ip) +{ + struct xfs_trans *tp; + struct xfs_mount *mp = ip->i_mount; + int error; + + /* Attach the dquots to the inode up front. */ + error = xfs_qm_dqattach(ip); + if (error) + return error; + + /* Wait on dio to ensure i_size has settled. */ + inode_dio_wait(VFS_I(ip)); + + /* + * For preallocated files only free delayed allocations. + * + * Note that this means we also leave speculative preallocations in + * place for preallocated files. + */ + if (ip->i_diflags & (XFS_DIFLAG_PREALLOC | XFS_DIFLAG_APPEND)) { + if (ip->i_delayed_blks) { + xfs_bmap_punch_delalloc_range(ip, XFS_DATA_FORK, + round_up(XFS_ISIZE(ip), mp->m_sb.sb_blocksize), + LLONG_MAX, NULL); + } + xfs_inode_clear_eofblocks_tag(ip); + return 0; + } + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_itruncate, 0, 0, 0, &tp); + if (error) { + ASSERT(xfs_is_shutdown(mp)); + return error; + } + + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, 0); + + /* + * Do not update the on-disk file size. If we update the on-disk file + * size and then the system crashes before the contents of the file are + * flushed to disk then the files may be full of holes (ie NULL files + * bug). + */ + error = xfs_itruncate_extents_flags(&tp, ip, XFS_DATA_FORK, + XFS_ISIZE(ip), XFS_BMAPI_NODISCARD); + if (error) + goto err_cancel; + + error = xfs_trans_commit(tp); + if (error) + goto out_unlock; + + xfs_inode_clear_eofblocks_tag(ip); + goto out_unlock; + +err_cancel: + /* + * If we get an error at this point we simply don't + * bother truncating the file. + */ + xfs_trans_cancel(tp); +out_unlock: + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; +} + +/* + * Allocate space or convert extents for a file according to @mode: + * + * XFS_ALLOC_FILE_SPACE_PREALLOC: + * Preallocate unwritten extents over holes across the range and mark the inode + * as preallocated. + * + * XFS_ALLOC_FILE_SPACE_WRITE_ZEROES: + * Allocate written extents over holes and convert unwritten extents in the + * range to written extents, initialising both to contain zeroes. + * + * This function does not update the file size; callers that extend the file + * are responsible for updating it once the extents are allocated. + */ +int +xfs_alloc_file_space( + struct xfs_inode *ip, + xfs_off_t offset, + xfs_off_t len, + enum xfs_alloc_file_space_mode mode) +{ + xfs_mount_t *mp = ip->i_mount; + xfs_off_t count; + xfs_filblks_t allocatesize_fsb; + xfs_extlen_t extsz, temp; + xfs_fileoff_t startoffset_fsb; + xfs_fileoff_t endoffset_fsb; + int rt; + xfs_trans_t *tp; + xfs_bmbt_irec_t imaps[1], *imapp; + uint32_t bmapi_flags, nr_exts; + int error; + + if (xfs_is_always_cow_inode(ip)) + return 0; + + trace_xfs_alloc_file_space(ip); + + if (xfs_is_shutdown(mp)) + return -EIO; + + error = xfs_qm_dqattach(ip); + if (error) + return error; + + if (len <= 0) + return -EINVAL; + + switch (mode) { + case XFS_ALLOC_FILE_SPACE_PREALLOC: + bmapi_flags = XFS_BMAPI_PREALLOC; + nr_exts = XFS_IEXT_ADD_NOSPLIT_CNT; + break; + case XFS_ALLOC_FILE_SPACE_WRITE_ZEROES: + bmapi_flags = XFS_BMAPI_CONVERT | XFS_BMAPI_ZERO; + nr_exts = XFS_IEXT_WRITE_UNWRITTEN_CNT; + break; + default: + return -EINVAL; + } + + rt = XFS_IS_REALTIME_INODE(ip); + extsz = xfs_get_extsz_hint(ip); + + count = len; + imapp = &imaps[0]; + startoffset_fsb = XFS_B_TO_FSBT(mp, offset); + endoffset_fsb = XFS_B_TO_FSB(mp, offset + count); + allocatesize_fsb = endoffset_fsb - startoffset_fsb; + + /* + * Allocate file space until done or until there is an error + */ + while (allocatesize_fsb && !error) { + xfs_fileoff_t s, e; + unsigned int dblocks, rblocks, resblks; + int nimaps = 1; + + /* + * Determine space reservations for data/realtime. + */ + if (unlikely(extsz)) { + s = startoffset_fsb; + do_div(s, extsz); + s *= extsz; + e = startoffset_fsb + allocatesize_fsb; + div_u64_rem(startoffset_fsb, extsz, &temp); + if (temp) + e += temp; + div_u64_rem(e, extsz, &temp); + if (temp) + e += extsz - temp; + } else { + s = 0; + e = allocatesize_fsb; + } + + /* + * The transaction reservation is limited to a 32-bit block + * count, hence we need to limit the number of blocks we are + * trying to reserve to avoid an overflow. We can't allocate + * more than @nimaps extents, and an extent is limited on disk + * to XFS_BMBT_MAX_EXTLEN (21 bits), so use that to enforce the + * limit. + */ + resblks = min_t(xfs_fileoff_t, (e - s), + (XFS_MAX_BMBT_EXTLEN * nimaps)); + if (unlikely(rt)) { + dblocks = XFS_DIOSTRAT_SPACE_RES(mp, 0); + rblocks = resblks; + } else { + dblocks = XFS_DIOSTRAT_SPACE_RES(mp, resblks); + rblocks = 0; + } + + error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, + dblocks, rblocks, false, &tp); + if (error) + break; + + error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, nr_exts); + if (error) + goto error; + + /* + * If the allocator cannot find a single free extent large + * enough to cover the start block of the requested range, + * xfs_bmapi_write will return -ENOSR. + * + * In that case we simply need to keep looping with the same + * startoffset_fsb so that one of the following allocations + * will eventually reach the requested range. + */ + error = xfs_bmapi_write(tp, ip, startoffset_fsb, + allocatesize_fsb, bmapi_flags, 0, imapp, + &nimaps); + if (error) { + if (error != -ENOSR) + goto error; + error = 0; + } else { + startoffset_fsb += imapp->br_blockcount; + allocatesize_fsb -= imapp->br_blockcount; + } + + if (mode == XFS_ALLOC_FILE_SPACE_PREALLOC) { + ip->i_diflags |= XFS_DIFLAG_PREALLOC; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + } + + error = xfs_trans_commit(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + } + + return error; + +error: + xfs_trans_cancel(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; +} + +static int +xfs_unmap_extent( + struct xfs_inode *ip, + xfs_fileoff_t startoffset_fsb, + xfs_filblks_t len_fsb, + int *done) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_trans *tp; + uint resblks = XFS_DIOSTRAT_SPACE_RES(mp, 0); + int error; + + error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, resblks, 0, + false, &tp); + if (error) + return error; + + error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, + XFS_IEXT_PUNCH_HOLE_CNT); + if (error) + goto out_trans_cancel; + + error = xfs_bunmapi(tp, ip, startoffset_fsb, len_fsb, 0, 2, done); + if (error) + goto out_trans_cancel; + + error = xfs_trans_commit(tp); +out_unlock: + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; + +out_trans_cancel: + xfs_trans_cancel(tp); + goto out_unlock; +} + +/* Caller must first wait for the completion of any pending DIOs if required. */ +int +xfs_flush_unmap_range( + struct xfs_inode *ip, + xfs_off_t offset, + xfs_off_t len) +{ + struct inode *inode = VFS_I(ip); + xfs_off_t rounding, start, end; + int error; + + /* + * Make sure we extend the flush out to extent alignment + * boundaries so any extent range overlapping the start/end + * of the modification we are about to do is clean and idle. + */ + rounding = max_t(xfs_off_t, xfs_inode_alloc_unitsize(ip), PAGE_SIZE); + start = rounddown_64(offset, rounding); + end = roundup_64(offset + len, rounding) - 1; + + error = filemap_write_and_wait_range(inode->i_mapping, start, end); + if (error) + return error; + truncate_pagecache_range(inode, start, end); + return 0; +} + +int +xfs_free_file_space( + struct xfs_inode *ip, + xfs_off_t offset, + xfs_off_t len, + struct xfs_zone_alloc_ctx *ac) +{ + struct xfs_mount *mp = ip->i_mount; + xfs_fileoff_t startoffset_fsb; + xfs_fileoff_t endoffset_fsb; + int done = 0, error; + + trace_xfs_free_file_space(ip); + + error = xfs_qm_dqattach(ip); + if (error) + return error; + + if (len <= 0) /* if nothing being freed */ + return 0; + + /* + * Now AIO and DIO has drained we flush and (if necessary) invalidate + * the cached range over the first operation we are about to run. + */ + error = xfs_flush_unmap_range(ip, offset, len); + if (error) + return error; + + startoffset_fsb = XFS_B_TO_FSB(mp, offset); + endoffset_fsb = XFS_B_TO_FSBT(mp, offset + len); + + /* We can only free complete realtime extents. */ + if (xfs_inode_has_bigrtalloc(ip)) { + startoffset_fsb = xfs_fileoff_roundup_rtx(mp, startoffset_fsb); + endoffset_fsb = xfs_fileoff_rounddown_rtx(mp, endoffset_fsb); + } + + /* + * Need to zero the stuff we're not freeing, on disk. + */ + if (endoffset_fsb > startoffset_fsb) { + while (!done) { + error = xfs_unmap_extent(ip, startoffset_fsb, + endoffset_fsb - startoffset_fsb, &done); + if (error) + return error; + } + } + + /* + * Now that we've unmap all full blocks we'll have to zero out any + * partial block at the beginning and/or end. xfs_zero_range is smart + * enough to skip any holes, including those we just created, but we + * must take care not to zero beyond EOF and enlarge i_size. + */ + if (offset >= XFS_ISIZE(ip)) + return 0; + if (offset + len > XFS_ISIZE(ip)) + len = XFS_ISIZE(ip) - offset; + error = xfs_zero_range(ip, offset, len, ac, NULL); + if (error) + return error; + + /* + * If we zeroed right up to EOF and EOF straddles a page boundary we + * must make sure that the post-EOF area is also zeroed because the + * page could be mmap'd and xfs_zero_range doesn't do that for us. + * Writeback of the eof page will do this, albeit clumsily. + */ + if (offset + len >= XFS_ISIZE(ip) && offset_in_page(offset + len) > 0) { + error = filemap_write_and_wait_range(VFS_I(ip)->i_mapping, + round_down(offset + len, PAGE_SIZE), LLONG_MAX); + } + + return error; +} + +static int +xfs_prepare_shift( + struct xfs_inode *ip, + loff_t offset) +{ + unsigned int rounding; + int error; + + /* + * Trim eofblocks to avoid shifting uninitialized post-eof preallocation + * into the accessible region of the file. + */ + if (xfs_can_free_eofblocks(ip)) { + error = xfs_free_eofblocks(ip); + if (error) + return error; + } + + /* + * Shift operations must stabilize the start block offset boundary along + * with the full range of the operation. If we don't, a COW writeback + * completion could race with an insert, front merge with the start + * extent (after split) during the shift and corrupt the file. Start + * with the allocation unit just prior to the start to stabilize the + * boundary. + */ + rounding = xfs_inode_alloc_unitsize(ip); + offset = rounddown_64(offset, rounding); + if (offset) + offset -= rounding; + + /* + * Writeback and invalidate cache for the remainder of the file as we're + * about to shift down every extent from offset to EOF. + */ + error = xfs_flush_unmap_range(ip, offset, XFS_ISIZE(ip)); + if (error) + return error; + + /* + * Clean out anything hanging around in the cow fork now that + * we've flushed all the dirty data out to disk to avoid having + * CoW extents at the wrong offsets. + */ + if (xfs_inode_has_cow_data(ip)) { + error = xfs_reflink_cancel_cow_range(ip, offset, NULLFILEOFF, + true); + if (error) + return error; + } + + return 0; +} + +/* + * xfs_collapse_file_space() + * This routine frees disk space and shift extent for the given file. + * The first thing we do is to free data blocks in the specified range + * by calling xfs_free_file_space(). It would also sync dirty data + * and invalidate page cache over the region on which collapse range + * is working. And Shift extent records to the left to cover a hole. + * RETURNS: + * 0 on success + * errno on error + * + */ +int +xfs_collapse_file_space( + struct xfs_inode *ip, + xfs_off_t offset, + xfs_off_t len, + struct xfs_zone_alloc_ctx *ac) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_trans *tp; + int error; + xfs_fileoff_t next_fsb = XFS_B_TO_FSB(mp, offset + len); + xfs_fileoff_t shift_fsb = XFS_B_TO_FSB(mp, len); + bool done = false; + + xfs_assert_ilocked(ip, XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL); + + trace_xfs_collapse_file_space(ip); + + error = xfs_free_file_space(ip, offset, len, ac); + if (error) + return error; + + error = xfs_prepare_shift(ip, offset); + if (error) + return error; + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, 0, 0, 0, &tp); + if (error) + return error; + + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, 0); + + while (!done) { + error = xfs_bmap_collapse_extents(tp, ip, &next_fsb, shift_fsb, + &done); + if (error) + goto out_trans_cancel; + if (done) + break; + + /* finish any deferred frees and roll the transaction */ + error = xfs_defer_finish(&tp); + if (error) + goto out_trans_cancel; + } + + error = xfs_trans_commit(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; + +out_trans_cancel: + xfs_trans_cancel(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; +} + +/* + * xfs_insert_file_space() + * This routine create hole space by shifting extents for the given file. + * The first thing we do is to sync dirty data and invalidate page cache + * over the region on which insert range is working. And split an extent + * to two extents at given offset by calling xfs_bmap_split_extent. + * And shift all extent records which are laying between [offset, + * last allocated extent] to the right to reserve hole range. + * RETURNS: + * 0 on success + * errno on error + */ +int +xfs_insert_file_space( + struct xfs_inode *ip, + loff_t offset, + loff_t len) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_trans *tp; + int error; + xfs_fileoff_t stop_fsb = XFS_B_TO_FSB(mp, offset); + xfs_fileoff_t next_fsb = NULLFSBLOCK; + xfs_fileoff_t shift_fsb = XFS_B_TO_FSB(mp, len); + bool done = false; + + xfs_assert_ilocked(ip, XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL); + + trace_xfs_insert_file_space(ip); + + error = xfs_bmap_can_insert_extents(ip, stop_fsb, shift_fsb); + if (error) + return error; + + error = xfs_prepare_shift(ip, offset); + if (error) + return error; + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, + XFS_DIOSTRAT_SPACE_RES(mp, 0), 0, 0, &tp); + if (error) + return error; + + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, 0); + + error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, + XFS_IEXT_PUNCH_HOLE_CNT); + if (error) + goto out_trans_cancel; + + /* + * The extent shifting code works on extent granularity. So, if stop_fsb + * is not the starting block of extent, we need to split the extent at + * stop_fsb. + */ + error = xfs_bmap_split_extent(tp, ip, stop_fsb); + if (error) + goto out_trans_cancel; + + do { + error = xfs_defer_finish(&tp); + if (error) + goto out_trans_cancel; + + error = xfs_bmap_insert_extents(tp, ip, &next_fsb, shift_fsb, + &done, stop_fsb); + if (error) + goto out_trans_cancel; + } while (!done); + + error = xfs_trans_commit(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; + +out_trans_cancel: + xfs_trans_cancel(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; +} + +/* + * We need to check that the format of the data fork in the temporary inode is + * valid for the target inode before doing the swap. This is not a problem with + * attr1 because of the fixed fork offset, but attr2 has a dynamically sized + * data fork depending on the space the attribute fork is taking so we can get + * invalid formats on the target inode. + * + * E.g. target has space for 7 extents in extent format, temp inode only has + * space for 6. If we defragment down to 7 extents, then the tmp format is a + * btree, but when swapped it needs to be in extent format. Hence we can't just + * blindly swap data forks on attr2 filesystems. + * + * Note that we check the swap in both directions so that we don't end up with + * a corrupt temporary inode, either. + * + * Note that fixing the way xfs_fsr sets up the attribute fork in the source + * inode will prevent this situation from occurring, so all we do here is + * reject and log the attempt. basically we are putting the responsibility on + * userspace to get this right. + */ +static int +xfs_swap_extents_check_format( + struct xfs_inode *ip, /* target inode */ + struct xfs_inode *tip) /* tmp inode */ +{ + struct xfs_ifork *ifp = &ip->i_df; + struct xfs_ifork *tifp = &tip->i_df; + + /* User/group/project quota ids must match if quotas are enforced. */ + if (XFS_IS_QUOTA_ON(ip->i_mount) && + (!uid_eq(VFS_I(ip)->i_uid, VFS_I(tip)->i_uid) || + !gid_eq(VFS_I(ip)->i_gid, VFS_I(tip)->i_gid) || + ip->i_projid != tip->i_projid)) + return -EINVAL; + + /* Should never get a local format */ + if (ifp->if_format == XFS_DINODE_FMT_LOCAL || + tifp->if_format == XFS_DINODE_FMT_LOCAL) + return -EINVAL; + + /* + * if the target inode has less extents that then temporary inode then + * why did userspace call us? + */ + if (ifp->if_nextents < tifp->if_nextents) + return -EINVAL; + + /* + * If we have to use the (expensive) rmap swap method, we can + * handle any number of extents and any format. + */ + if (xfs_has_rmapbt(ip->i_mount)) + return 0; + + /* + * if the target inode is in extent form and the temp inode is in btree + * form then we will end up with the target inode in the wrong format + * as we already know there are less extents in the temp inode. + */ + if (ifp->if_format == XFS_DINODE_FMT_EXTENTS && + tifp->if_format == XFS_DINODE_FMT_BTREE) + return -EINVAL; + + /* Check temp in extent form to max in target */ + if (tifp->if_format == XFS_DINODE_FMT_EXTENTS && + tifp->if_nextents > XFS_IFORK_MAXEXT(ip, XFS_DATA_FORK)) + return -EINVAL; + + /* Check target in extent form to max in temp */ + if (ifp->if_format == XFS_DINODE_FMT_EXTENTS && + ifp->if_nextents > XFS_IFORK_MAXEXT(tip, XFS_DATA_FORK)) + return -EINVAL; + + /* + * If we are in a btree format, check that the temp root block will fit + * in the target and that it has enough extents to be in btree format + * in the target. + * + * Note that we have to be careful to allow btree->extent conversions + * (a common defrag case) which will occur when the temp inode is in + * extent format... + */ + if (tifp->if_format == XFS_DINODE_FMT_BTREE) { + if (xfs_inode_has_attr_fork(ip) && + xfs_bmap_bmdr_space(tifp->if_broot) > xfs_inode_fork_boff(ip)) + return -EINVAL; + if (tifp->if_nextents <= XFS_IFORK_MAXEXT(ip, XFS_DATA_FORK)) + return -EINVAL; + } + + /* Reciprocal target->temp btree format checks */ + if (ifp->if_format == XFS_DINODE_FMT_BTREE) { + if (xfs_inode_has_attr_fork(tip) && + xfs_bmap_bmdr_space(ip->i_df.if_broot) > xfs_inode_fork_boff(tip)) + return -EINVAL; + if (ifp->if_nextents <= XFS_IFORK_MAXEXT(tip, XFS_DATA_FORK)) + return -EINVAL; + } + + return 0; +} + +static int +xfs_swap_extent_flush( + struct xfs_inode *ip) +{ + int error; + + error = filemap_write_and_wait(VFS_I(ip)->i_mapping); + if (error) + return error; + truncate_pagecache_range(VFS_I(ip), 0, -1); + + /* Verify O_DIRECT for ftmp */ + if (VFS_I(ip)->i_mapping->nrpages) + return -EINVAL; + return 0; +} + +/* + * Move extents from one file to another, when rmap is enabled. + */ +STATIC int +xfs_swap_extent_rmap( + struct xfs_trans **tpp, + struct xfs_inode *ip, + struct xfs_inode *tip) +{ + struct xfs_trans *tp = *tpp; + struct xfs_bmbt_irec irec; + struct xfs_bmbt_irec uirec; + struct xfs_bmbt_irec tirec; + xfs_fileoff_t offset_fsb; + xfs_fileoff_t end_fsb; + xfs_filblks_t count_fsb; + int error; + xfs_filblks_t ilen; + xfs_filblks_t rlen; + int nimaps; + uint64_t tip_flags2; + + /* + * If the source file has shared blocks, we must flag the donor + * file as having shared blocks so that we get the shared-block + * rmap functions when we go to fix up the rmaps. The flags + * will be switch for reals later. + */ + tip_flags2 = tip->i_diflags2; + if (ip->i_diflags2 & XFS_DIFLAG2_REFLINK) + tip->i_diflags2 |= XFS_DIFLAG2_REFLINK; + + offset_fsb = 0; + end_fsb = XFS_B_TO_FSB(ip->i_mount, i_size_read(VFS_I(ip))); + count_fsb = (xfs_filblks_t)(end_fsb - offset_fsb); + + while (count_fsb) { + /* Read extent from the donor file */ + nimaps = 1; + error = xfs_bmapi_read(tip, offset_fsb, count_fsb, &tirec, + &nimaps, 0); + if (error) + goto out; + ASSERT(nimaps == 1); + ASSERT(tirec.br_startblock != DELAYSTARTBLOCK); + + trace_xfs_swap_extent_rmap_remap(tip, &tirec); + ilen = tirec.br_blockcount; + + /* Unmap the old blocks in the source file. */ + while (tirec.br_blockcount) { + ASSERT(tp->t_highest_agno == NULLAGNUMBER); + trace_xfs_swap_extent_rmap_remap_piece(tip, &tirec); + + /* Read extent from the source file */ + nimaps = 1; + error = xfs_bmapi_read(ip, tirec.br_startoff, + tirec.br_blockcount, &irec, + &nimaps, 0); + if (error) + goto out; + ASSERT(nimaps == 1); + ASSERT(tirec.br_startoff == irec.br_startoff); + trace_xfs_swap_extent_rmap_remap_piece(ip, &irec); + + /* Trim the extent. */ + uirec = tirec; + uirec.br_blockcount = rlen = min_t(xfs_filblks_t, + tirec.br_blockcount, + irec.br_blockcount); + trace_xfs_swap_extent_rmap_remap_piece(tip, &uirec); + + if (xfs_bmap_is_real_extent(&uirec)) { + error = xfs_iext_count_extend(tp, ip, + XFS_DATA_FORK, + XFS_IEXT_SWAP_RMAP_CNT); + if (error) + goto out; + } + + if (xfs_bmap_is_real_extent(&irec)) { + error = xfs_iext_count_extend(tp, tip, + XFS_DATA_FORK, + XFS_IEXT_SWAP_RMAP_CNT); + if (error) + goto out; + } + + /* Remove the mapping from the donor file. */ + xfs_bmap_unmap_extent(tp, tip, XFS_DATA_FORK, &uirec); + + /* Remove the mapping from the source file. */ + xfs_bmap_unmap_extent(tp, ip, XFS_DATA_FORK, &irec); + + /* Map the donor file's blocks into the source file. */ + xfs_bmap_map_extent(tp, ip, XFS_DATA_FORK, &uirec); + + /* Map the source file's blocks into the donor file. */ + xfs_bmap_map_extent(tp, tip, XFS_DATA_FORK, &irec); + + error = xfs_defer_finish(tpp); + tp = *tpp; + if (error) + goto out; + + tirec.br_startoff += rlen; + if (tirec.br_startblock != HOLESTARTBLOCK && + tirec.br_startblock != DELAYSTARTBLOCK) + tirec.br_startblock += rlen; + tirec.br_blockcount -= rlen; + } + + /* Roll on... */ + count_fsb -= ilen; + offset_fsb += ilen; + } + + tip->i_diflags2 = tip_flags2; + return 0; + +out: + trace_xfs_swap_extent_rmap_error(ip, error, _RET_IP_); + tip->i_diflags2 = tip_flags2; + return error; +} + +/* Swap the extents of two files by swapping data forks. */ +STATIC int +xfs_swap_extent_forks( + struct xfs_trans *tp, + struct xfs_inode *ip, + struct xfs_inode *tip, + int *src_log_flags, + int *target_log_flags) +{ + xfs_filblks_t aforkblks = 0; + xfs_filblks_t taforkblks = 0; + xfs_extnum_t junk; + uint64_t tmp; + int error; + + /* + * Count the number of extended attribute blocks + */ + if (xfs_inode_has_attr_fork(ip) && ip->i_af.if_nextents > 0 && + ip->i_af.if_format != XFS_DINODE_FMT_LOCAL) { + error = xfs_bmap_count_blocks(tp, ip, XFS_ATTR_FORK, &junk, + &aforkblks); + if (error) + return error; + } + if (xfs_inode_has_attr_fork(tip) && tip->i_af.if_nextents > 0 && + tip->i_af.if_format != XFS_DINODE_FMT_LOCAL) { + error = xfs_bmap_count_blocks(tp, tip, XFS_ATTR_FORK, &junk, + &taforkblks); + if (error) + return error; + } + + /* + * Btree format (v3) inodes have the inode number stamped in the bmbt + * block headers. We can't start changing the bmbt blocks until the + * inode owner change is logged so recovery does the right thing in the + * event of a crash. Set the owner change log flags now and leave the + * bmbt scan as the last step. + */ + if (xfs_has_v3inodes(ip->i_mount)) { + if (ip->i_df.if_format == XFS_DINODE_FMT_BTREE) + (*target_log_flags) |= XFS_ILOG_DOWNER; + if (tip->i_df.if_format == XFS_DINODE_FMT_BTREE) + (*src_log_flags) |= XFS_ILOG_DOWNER; + } + + /* + * Swap the data forks of the inodes + */ + swap(ip->i_df, tip->i_df); + + /* + * Fix the on-disk inode values + */ + tmp = (uint64_t)ip->i_nblocks; + ip->i_nblocks = tip->i_nblocks - taforkblks + aforkblks; + tip->i_nblocks = tmp + taforkblks - aforkblks; + + /* + * The extents in the source inode could still contain speculative + * preallocation beyond EOF (e.g. the file is open but not modified + * while defrag is in progress). In that case, we need to copy over the + * number of delalloc blocks the data fork in the source inode is + * tracking beyond EOF so that when the fork is truncated away when the + * temporary inode is unlinked we don't underrun the i_delayed_blks + * counter on that inode. + */ + ASSERT(tip->i_delayed_blks == 0); + tip->i_delayed_blks = ip->i_delayed_blks; + ip->i_delayed_blks = 0; + + switch (ip->i_df.if_format) { + case XFS_DINODE_FMT_EXTENTS: + (*src_log_flags) |= XFS_ILOG_DEXT; + break; + case XFS_DINODE_FMT_BTREE: + ASSERT(!xfs_has_v3inodes(ip->i_mount) || + (*src_log_flags & XFS_ILOG_DOWNER)); + (*src_log_flags) |= XFS_ILOG_DBROOT; + break; + } + + switch (tip->i_df.if_format) { + case XFS_DINODE_FMT_EXTENTS: + (*target_log_flags) |= XFS_ILOG_DEXT; + break; + case XFS_DINODE_FMT_BTREE: + (*target_log_flags) |= XFS_ILOG_DBROOT; + ASSERT(!xfs_has_v3inodes(ip->i_mount) || + (*target_log_flags & XFS_ILOG_DOWNER)); + break; + } + + return 0; +} + +/* + * Fix up the owners of the bmbt blocks to refer to the current inode. The + * change owner scan attempts to order all modified buffers in the current + * transaction. In the event of ordered buffer failure, the offending buffer is + * physically logged as a fallback and the scan returns -EAGAIN. We must roll + * the transaction in this case to replenish the fallback log reservation and + * restart the scan. This process repeats until the scan completes. + */ +static int +xfs_swap_change_owner( + struct xfs_trans **tpp, + struct xfs_inode *ip, + struct xfs_inode *tmpip) +{ + int error; + struct xfs_trans *tp = *tpp; + + do { + error = xfs_bmbt_change_owner(tp, ip, XFS_DATA_FORK, I_INO(ip), + NULL); + /* success or fatal error */ + if (error != -EAGAIN) + break; + + error = xfs_trans_roll(tpp); + if (error) + break; + tp = *tpp; + + /* + * Redirty both inodes so they can relog and keep the log tail + * moving forward. + */ + xfs_trans_ijoin(tp, ip, 0); + xfs_trans_ijoin(tp, tmpip, 0); + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + xfs_trans_log_inode(tp, tmpip, XFS_ILOG_CORE); + } while (true); + + return error; +} + +int +xfs_swap_extents( + struct xfs_inode *ip, /* target inode */ + struct xfs_inode *tip, /* tmp inode */ + struct xfs_swapext *sxp) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_trans *tp; + struct xfs_bstat *sbp = &sxp->sx_stat; + int src_log_flags, target_log_flags; + int error = 0; + uint64_t f; + int resblks = 0; + unsigned int flags = 0; + struct timespec64 ctime, mtime; + + /* + * Lock the inodes against other IO, page faults and truncate to + * begin with. Then we can ensure the inodes are flushed and have no + * page cache safely. Once we have done this we can take the ilocks and + * do the rest of the checks. + */ + lock_two_nondirectories(VFS_I(ip), VFS_I(tip)); + filemap_invalidate_lock_two(VFS_I(ip)->i_mapping, + VFS_I(tip)->i_mapping); + + /* Verify that both files have the same format */ + if ((VFS_I(ip)->i_mode & S_IFMT) != (VFS_I(tip)->i_mode & S_IFMT)) { + error = -EINVAL; + goto out_unlock; + } + + /* Verify both files are either real-time or non-realtime */ + if (XFS_IS_REALTIME_INODE(ip) != XFS_IS_REALTIME_INODE(tip)) { + error = -EINVAL; + goto out_unlock; + } + + /* + * The rmapbt implementation is unable to resume a swapext operation + * after a crash if the allocation unit size is larger than a block. + * This (deprecated) interface will not be upgraded to handle this + * situation. Defragmentation must be performed with the commit range + * ioctl. + */ + if (XFS_IS_REALTIME_INODE(ip) && xfs_has_rtgroups(ip->i_mount)) { + error = -EOPNOTSUPP; + goto out_unlock; + } + + error = xfs_qm_dqattach(ip); + if (error) + goto out_unlock; + + error = xfs_qm_dqattach(tip); + if (error) + goto out_unlock; + + error = xfs_swap_extent_flush(ip); + if (error) + goto out_unlock; + error = xfs_swap_extent_flush(tip); + if (error) + goto out_unlock; + + if (xfs_inode_has_cow_data(tip)) { + error = xfs_reflink_cancel_cow_range(tip, 0, NULLFILEOFF, true); + if (error) + goto out_unlock; + } + + /* + * Extent "swapping" with rmap requires a permanent reservation and + * a block reservation because it's really just a remap operation + * performed with log redo items! + */ + if (xfs_has_rmapbt(mp)) { + int w = XFS_DATA_FORK; + uint32_t ipnext = ip->i_df.if_nextents; + uint32_t tipnext = tip->i_df.if_nextents; + + /* + * Conceptually this shouldn't affect the shape of either bmbt, + * but since we atomically move extents one by one, we reserve + * enough space to rebuild both trees. + */ + resblks = XFS_SWAP_RMAP_SPACE_RES(mp, ipnext, w); + resblks += XFS_SWAP_RMAP_SPACE_RES(mp, tipnext, w); + + /* + * If either inode straddles a bmapbt block allocation boundary, + * the rmapbt algorithm triggers repeated allocs and frees as + * extents are remapped. This can exhaust the block reservation + * prematurely and cause shutdown. Return freed blocks to the + * transaction reservation to counter this behavior. + */ + flags |= XFS_TRANS_RES_FDBLKS; + } + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0, flags, + &tp); + if (error) + goto out_unlock; + + /* + * Lock and join the inodes to the tansaction so that transaction commit + * or cancel will unlock the inodes from this point onwards. + */ + xfs_lock_two_inodes(ip, XFS_ILOCK_EXCL, tip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, 0); + xfs_trans_ijoin(tp, tip, 0); + + + /* Verify all data are being swapped */ + if (sxp->sx_offset != 0 || + sxp->sx_length != ip->i_disk_size || + sxp->sx_length != tip->i_disk_size) { + error = -EFAULT; + goto out_trans_cancel; + } + + trace_xfs_swap_extent_before(ip, 0); + trace_xfs_swap_extent_before(tip, 1); + + /* check inode formats now that data is flushed */ + error = xfs_swap_extents_check_format(ip, tip); + if (error) { + xfs_notice(mp, + "%s: inode 0x%llx format is incompatible for exchanging.", + __func__, I_INO(ip)); + goto out_trans_cancel; + } + + /* + * Compare the current change & modify times with that + * passed in. If they differ, we abort this swap. + * This is the mechanism used to ensure the calling + * process that the file was not changed out from + * under it. + */ + ctime = inode_get_ctime(VFS_I(ip)); + mtime = inode_get_mtime(VFS_I(ip)); + if ((sbp->bs_ctime.tv_sec != ctime.tv_sec) || + (sbp->bs_ctime.tv_nsec != ctime.tv_nsec) || + (sbp->bs_mtime.tv_sec != mtime.tv_sec) || + (sbp->bs_mtime.tv_nsec != mtime.tv_nsec)) { + error = -EBUSY; + goto out_trans_cancel; + } + + /* + * Note the trickiness in setting the log flags - we set the owner log + * flag on the opposite inode (i.e. the inode we are setting the new + * owner to be) because once we swap the forks and log that, log + * recovery is going to see the fork as owned by the swapped inode, + * not the pre-swapped inodes. + */ + src_log_flags = XFS_ILOG_CORE; + target_log_flags = XFS_ILOG_CORE; + + if (xfs_has_rmapbt(mp)) + error = xfs_swap_extent_rmap(&tp, ip, tip); + else + error = xfs_swap_extent_forks(tp, ip, tip, &src_log_flags, + &target_log_flags); + if (error) + goto out_trans_cancel; + + /* Do we have to swap reflink flags? */ + if ((ip->i_diflags2 & XFS_DIFLAG2_REFLINK) ^ + (tip->i_diflags2 & XFS_DIFLAG2_REFLINK)) { + f = ip->i_diflags2 & XFS_DIFLAG2_REFLINK; + ip->i_diflags2 &= ~XFS_DIFLAG2_REFLINK; + ip->i_diflags2 |= tip->i_diflags2 & XFS_DIFLAG2_REFLINK; + tip->i_diflags2 &= ~XFS_DIFLAG2_REFLINK; + tip->i_diflags2 |= f & XFS_DIFLAG2_REFLINK; + } + + /* Swap the cow forks. */ + if (xfs_has_reflink(mp)) { + ASSERT(!ip->i_cowfp || + ip->i_cowfp->if_format == XFS_DINODE_FMT_EXTENTS); + ASSERT(!tip->i_cowfp || + tip->i_cowfp->if_format == XFS_DINODE_FMT_EXTENTS); + + swap(ip->i_cowfp, tip->i_cowfp); + + if (ip->i_cowfp && ip->i_cowfp->if_bytes) + xfs_inode_set_cowblocks_tag(ip); + else + xfs_inode_clear_cowblocks_tag(ip); + if (tip->i_cowfp && tip->i_cowfp->if_bytes) + xfs_inode_set_cowblocks_tag(tip); + else + xfs_inode_clear_cowblocks_tag(tip); + } + + xfs_trans_log_inode(tp, ip, src_log_flags); + xfs_trans_log_inode(tp, tip, target_log_flags); + + /* + * The extent forks have been swapped, but crc=1,rmapbt=0 filesystems + * have inode number owner values in the bmbt blocks that still refer to + * the old inode. Scan each bmbt to fix up the owner values with the + * inode number of the current inode. + */ + if (src_log_flags & XFS_ILOG_DOWNER) { + error = xfs_swap_change_owner(&tp, ip, tip); + if (error) + goto out_trans_cancel; + } + if (target_log_flags & XFS_ILOG_DOWNER) { + error = xfs_swap_change_owner(&tp, tip, ip); + if (error) + goto out_trans_cancel; + } + + /* + * If this is a synchronous mount, make sure that the + * transaction goes to disk before returning to the user. + */ + if (xfs_has_wsync(mp)) + xfs_trans_set_sync(tp); + + error = xfs_trans_commit(tp); + + trace_xfs_swap_extent_after(ip, 0); + trace_xfs_swap_extent_after(tip, 1); + +out_unlock_ilock: + xfs_iunlock(ip, XFS_ILOCK_EXCL); + xfs_iunlock(tip, XFS_ILOCK_EXCL); +out_unlock: + filemap_invalidate_unlock_two(VFS_I(ip)->i_mapping, + VFS_I(tip)->i_mapping); + unlock_two_nondirectories(VFS_I(ip), VFS_I(tip)); + return error; + +out_trans_cancel: + xfs_trans_cancel(tp); + goto out_unlock_ilock; +} + +/* + * Given a CoW fork mapping @got and a replacement mapping @rep, map the space + * described by @rep into the cow fork, pushing aside @got as necessary. @icur + * must point to iext tree leaf containing @got. + */ +void +xfs_bmap_replace_cow_mapping( + struct xfs_inode *ip, + struct xfs_iext_cursor *icur, + struct xfs_bmbt_irec *got, + struct xfs_bmbt_irec *rep) +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_COW_FORK); + xfs_fileoff_t rep_endoff = + rep->br_startoff + rep->br_blockcount; + xfs_fileoff_t got_endoff = + got->br_startoff + got->br_blockcount; + uint32_t state = BMAP_COWFORK; + + ASSERT(rep->br_blockcount > 0); + ASSERT(!isnullstartblock(got->br_startblock)); + ASSERT(got->br_startoff <= rep->br_startoff); + ASSERT(got_endoff >= rep_endoff); + + trace_xfs_bmap_replace_cow_mapping(ip, got, rep); + + if (got->br_startoff == rep->br_startoff) + state |= BMAP_LEFT_FILLING; + if (got_endoff == rep_endoff) + state |= BMAP_RIGHT_FILLING; + + switch (state & (BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING)) { + case BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING: + /* + * Replacement matches the whole mapping, update the record. + */ + xfs_iext_update_extent(ip, state, icur, rep); + break; + case BMAP_LEFT_FILLING: + /* + * Replace the first part of the mapping: Update the cursor + * position with the new mapping, then add a record with the + * tail of the old mapping. + */ + got->br_startoff = rep_endoff; + got->br_blockcount -= rep->br_blockcount; + got->br_startblock += rep->br_blockcount; + + xfs_iext_update_extent(ip, state, icur, rep); + xfs_iext_next(ifp, icur); + xfs_iext_insert(ip, icur, got, state); + break; + case BMAP_RIGHT_FILLING: + /* + * Replacing the last part of the mapping. Shorten the current + * mapping then add a record with the new mapping. + */ + got->br_blockcount -= rep->br_blockcount; + + xfs_iext_update_extent(ip, state, icur, got); + xfs_iext_next(ifp, icur); + xfs_iext_insert(ip, icur, rep, state); + break; + case 0: + /* + * Replacing the middle of the extent. Shorten the current + * mapping, add a new record with the new mapping, and add a + * second new record with the tail of the old mapping. + */ + got->br_blockcount = rep->br_startoff - got->br_startoff; + + struct xfs_bmbt_irec new = { + .br_startoff = rep_endoff, + .br_blockcount = got_endoff - rep_endoff, + .br_state = got->br_state, + .br_startblock = got->br_startblock + + rep->br_blockcount + + got->br_blockcount, + }; + + xfs_iext_update_extent(ip, state, icur, got); + xfs_iext_next(ifp, icur); + xfs_iext_insert(ip, icur, rep, state); + xfs_iext_next(ifp, icur); + xfs_iext_insert(ip, icur, &new, state); + break; + } +} diff --git a/fs/xfs/xfs_bmap_util.h b/fs/xfs/xfs_bmap_util.h new file mode 100644 index 000000000..c7b48b260 --- /dev/null +++ b/fs/xfs/xfs_bmap_util.h @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2006 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_BMAP_UTIL_H__ +#define __XFS_BMAP_UTIL_H__ + +/* Kernel only BMAP related definitions and functions */ + +struct xfs_bmbt_irec; +struct xfs_extent_free_item; +struct xfs_ifork; +struct xfs_inode; +struct xfs_mount; +struct xfs_trans; +struct xfs_bmalloca; +struct xfs_zone_alloc_ctx; + +#ifdef CONFIG_XFS_RT +int xfs_bmap_rtalloc(struct xfs_bmalloca *ap); +#else /* !CONFIG_XFS_RT */ +/* + * Attempts to allocate RT extents when RT is disable indicates corruption and + * should trigger a shutdown. + */ +static inline int +xfs_bmap_rtalloc(struct xfs_bmalloca *ap) +{ + return -EFSCORRUPTED; +} +#endif /* CONFIG_XFS_RT */ + +void xfs_bmap_punch_delalloc_range(struct xfs_inode *ip, int whichfork, + xfs_off_t start_byte, xfs_off_t end_byte, + struct xfs_zone_alloc_ctx *ac); + +struct kgetbmap { + __s64 bmv_offset; /* file offset of segment in blocks */ + __s64 bmv_block; /* starting block (64-bit daddr_t) */ + __s64 bmv_length; /* length of segment, blocks */ + __s32 bmv_oflags; /* output flags */ +}; +int xfs_getbmap(struct xfs_inode *ip, struct getbmapx *bmv, + struct kgetbmap *out); + +/* functions in xfs_bmap.c that are only needed by xfs_bmap_util.c */ +int xfs_bmap_extsize_align(struct xfs_mount *mp, struct xfs_bmbt_irec *gotp, + struct xfs_bmbt_irec *prevp, xfs_extlen_t extsz, + int rt, int eof, int delay, int convert, + xfs_fileoff_t *offp, xfs_extlen_t *lenp); +bool xfs_bmap_adjacent(struct xfs_bmalloca *ap); +int xfs_bmap_last_extent(struct xfs_trans *tp, struct xfs_inode *ip, + int whichfork, struct xfs_bmbt_irec *rec, + int *is_empty); + +/* preallocation and hole punch interface */ +enum xfs_alloc_file_space_mode { + XFS_ALLOC_FILE_SPACE_PREALLOC, + XFS_ALLOC_FILE_SPACE_WRITE_ZEROES, +}; + +int xfs_alloc_file_space(struct xfs_inode *ip, xfs_off_t offset, + xfs_off_t len, enum xfs_alloc_file_space_mode mode); +int xfs_free_file_space(struct xfs_inode *ip, xfs_off_t offset, + xfs_off_t len, struct xfs_zone_alloc_ctx *ac); +int xfs_collapse_file_space(struct xfs_inode *, xfs_off_t offset, + xfs_off_t len, struct xfs_zone_alloc_ctx *ac); +int xfs_insert_file_space(struct xfs_inode *, xfs_off_t offset, + xfs_off_t len); + +/* EOF block manipulation functions */ +bool xfs_can_free_eofblocks(struct xfs_inode *ip); +int xfs_free_eofblocks(struct xfs_inode *ip); + +int xfs_swap_extents(struct xfs_inode *ip, struct xfs_inode *tip, + struct xfs_swapext *sx); + +xfs_daddr_t xfs_fsb_to_db(struct xfs_inode *ip, xfs_fsblock_t fsb); + +xfs_extnum_t xfs_bmap_count_leaves(struct xfs_ifork *ifp, xfs_filblks_t *count); +int xfs_bmap_count_blocks(struct xfs_trans *tp, struct xfs_inode *ip, + int whichfork, xfs_extnum_t *nextents, + xfs_filblks_t *count); + +int xfs_flush_unmap_range(struct xfs_inode *ip, xfs_off_t offset, + xfs_off_t len); + +void xfs_bmap_replace_cow_mapping(struct xfs_inode *ip, + struct xfs_iext_cursor *icur, struct xfs_bmbt_irec *got, + struct xfs_bmbt_irec *rep); + +#endif /* __XFS_BMAP_UTIL_H__ */ diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c new file mode 100644 index 000000000..ee7c2e9c0 --- /dev/null +++ b/fs/xfs/xfs_buf.c @@ -0,0 +1,2108 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2006 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include +#include + +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_trace.h" +#include "xfs_log.h" +#include "xfs_log_recover.h" +#include "xfs_log_priv.h" +#include "xfs_trans.h" +#include "xfs_buf_item.h" +#include "xfs_errortag.h" +#include "xfs_error.h" +#include "xfs_ag.h" +#include "xfs_buf_mem.h" +#include "xfs_notify_failure.h" + +struct kmem_cache *xfs_buf_cache; + +/* + * Locking orders + * + * xfs_buf_stale: + * b_sema (caller holds) + * b_lockref.lock + * lru_lock + * + * xfs_buf_rele: + * b_lockref.lock + * lru_lock + * + * xfs_buftarg_drain_rele + * lru_lock + * b_lockref.lock (trylock due to inversion) + * + * xfs_buftarg_isolate + * lru_lock + * b_lockref.lock (trylock due to inversion) + */ + +static void xfs_buf_submit(struct xfs_buf *bp); +static int xfs_buf_iowait(struct xfs_buf *bp); + +static inline bool xfs_buf_is_uncached(struct xfs_buf *bp) +{ + return bp->b_rhash_key == XFS_BUF_DADDR_NULL; +} + +static inline void +xfs_buf_set_flags( + struct xfs_buf *bp, + unsigned int flags) +{ + WRITE_ONCE(bp->b_flags, bp->b_flags | flags); +} + +static inline void +xfs_buf_clear_flags( + struct xfs_buf *bp, + unsigned int flags) +{ + WRITE_ONCE(bp->b_flags, bp->b_flags & ~flags); +} + +void +xfs_buf_set_uptodate( + struct xfs_buf *bp) +{ + xfs_buf_set_flags(bp, XBF_DONE); +} + +/* + * When we mark a buffer stale, we remove the buffer from the LRU and clear the + * b_lru_ref count so that the buffer is freed immediately when the buffer + * reference count falls to zero. If the buffer is already on the LRU, we need + * to remove the reference that LRU holds on the buffer. + * + * This prevents build-up of stale buffers on the LRU. + */ +void +xfs_buf_stale( + struct xfs_buf *bp) +{ + ASSERT(xfs_buf_islocked(bp)); + + xfs_buf_set_flags(bp, XBF_STALE); + + /* + * Clear the delwri status so that a delwri queue walker will not + * flush this buffer to disk now that it is stale. The delwri queue has + * a reference to the buffer, so this is safe to do. + */ + xfs_buf_clear_flags(bp, _XBF_DELWRI_Q); + + spin_lock(&bp->b_lockref.lock); + atomic_set(&bp->b_lru_ref, 0); + if (!lockref_is_dead(&bp->b_lockref)) + list_lru_del_obj(&bp->b_target->bt_lru, &bp->b_lru); + spin_unlock(&bp->b_lockref.lock); +} + +void +xfs_buf_clear_stale( + struct xfs_buf *bp) +{ + ASSERT(bp->b_flags & XBF_STALE); + xfs_buf_clear_flags(bp, XBF_STALE); +} + +static void +xfs_buf_free_callback( + struct callback_head *cb) +{ + struct xfs_buf *bp = container_of(cb, struct xfs_buf, b_rcu); + + if (bp->b_maps != &bp->__b_map) + kfree(bp->b_maps); + kmem_cache_free(xfs_buf_cache, bp); +} + +static void +xfs_buf_free( + struct xfs_buf *bp) +{ + unsigned int size = BBTOB(bp->b_length); + + might_sleep(); + trace_xfs_buf_free(bp, _RET_IP_); + + ASSERT(list_empty(&bp->b_lru)); + + if (!xfs_buftarg_is_mem(bp->b_target) && size >= PAGE_SIZE) + mm_account_reclaimed_pages(howmany(size, PAGE_SHIFT)); + + if (is_vmalloc_addr(bp->b_addr)) + vfree(bp->b_addr); + else if (bp->b_flags & _XBF_KMEM) + kfree(bp->b_addr); + else if (bp->b_addr) + folio_put(virt_to_folio(bp->b_addr)); + + call_rcu(&bp->b_rcu, xfs_buf_free_callback); +} + +static int +xfs_buf_alloc_folio( + struct xfs_buf *bp, + size_t size, + gfp_t gfp_mask) +{ + struct folio *folio; + + folio = folio_alloc(gfp_mask, get_order(size)); + if (!folio) + return -ENOMEM; + bp->b_addr = folio_address(folio); + trace_xfs_buf_backing_folio(bp, _RET_IP_); + return 0; +} + +static int +xfs_buf_alloc_kmem( + struct xfs_buf *bp, + size_t size, + gfp_t gfp_mask) +{ + ASSERT(is_power_of_2(size)); + ASSERT(size < PAGE_SIZE); + + bp->b_addr = kmalloc(size, gfp_mask); + if (!bp->b_addr) + return -ENOMEM; + + /* + * Slab guarantees that we get back naturally aligned allocations for + * power of two sizes. Keep this check as the canary in the coal mine + * if anything changes in slab. + */ + if (WARN_ON_ONCE(!IS_ALIGNED((unsigned long)bp->b_addr, size))) { + kfree(bp->b_addr); + bp->b_addr = NULL; + return -ENOMEM; + } + xfs_buf_set_flags(bp, _XBF_KMEM); + trace_xfs_buf_backing_kmem(bp, _RET_IP_); + return 0; +} + +static int +xfs_buf_alloc_vmalloc( + struct xfs_buf *bp, + size_t size, + gfp_t gfp_mask) +{ + for (;;) { + bp->b_addr = __vmalloc(size, gfp_mask); + if (bp->b_addr) + break; + if (gfp_mask & __GFP_NORETRY) + return -ENOMEM; + XFS_STATS_INC(bp->b_mount, xb_page_retries); + memalloc_retry_wait(gfp_mask); + } + + trace_xfs_buf_backing_vmalloc(bp, _RET_IP_); + return 0; +} + +/* + * Allocate backing memory for a buffer. + * + * For tmpfs-backed buffers used by in-memory btrees this directly maps the + * tmpfs page cache folios. + * + * For real file system buffers there are three different kinds backing memory: + * + * The first type backs the buffer by a kmalloc allocation. This is done for + * less than PAGE_SIZE allocations to avoid wasting memory. + * + * The second type is a single folio buffer - this may be a high order folio or + * just a single page sized folio, but either way they get treated the same way + * by the rest of the code - the buffer memory spans a single contiguous memory + * region that we don't have to map and unmap to access the data directly. + * + * The third type of buffer is the vmalloc()d buffer. This provides the buffer + * with the required contiguous memory region but backed by discontiguous + * physical pages. + */ +static int +xfs_buf_alloc_backing_mem( + struct xfs_buf *bp, + xfs_buf_flags_t flags) +{ + size_t size = BBTOB(bp->b_length); + gfp_t gfp_mask = GFP_KERNEL | __GFP_NOLOCKDEP | __GFP_NOWARN; + + if (xfs_buftarg_is_mem(bp->b_target)) + return xmbuf_map_backing_mem(bp); + + /* Assure zeroed buffer for non-read cases. */ + if (!(flags & XBF_READ)) + gfp_mask |= __GFP_ZERO; + + if (flags & XBF_READ_AHEAD) + gfp_mask |= __GFP_NORETRY; + + /* + * Optimistically attempt a single high order folio allocation for + * larger than PAGE_SIZE buffers. + * + * Allocating a high order folio makes the assumption that buffers are a + * power-of-2 size, matching the power-of-2 folios sizes available. + * + * The exception here are user xattr data buffers, which can be arbitrarily + * sized up to 64kB plus structure metadata, skip straight to the vmalloc + * path for them instead of wasting memory here. + */ + if (size > PAGE_SIZE) { + if (is_power_of_2(size)) { + gfp_t folio_gfp = gfp_mask; + + folio_gfp &= ~__GFP_DIRECT_RECLAIM; + folio_gfp |= __GFP_NORETRY; + if (xfs_buf_alloc_folio(bp, size, folio_gfp) == 0) + return 0; + trace_xfs_buf_backing_fallback(bp, _RET_IP_); + } + return xfs_buf_alloc_vmalloc(bp, size, gfp_mask); + } + + /* + * The slab allocator now guarantees aligned allocations for all power + * of two sizes. This covers most smaller XFS buffers, so just use + * kmalloc in this case. + * + * Don't bother with the vmalloc fallback for allocations of page size + * or less: vmalloc won't do any better. + */ + if (!(gfp_mask & __GFP_NORETRY)) + gfp_mask |= __GFP_NOFAIL; + if (size < PAGE_SIZE && is_power_of_2(size)) + return xfs_buf_alloc_kmem(bp, size, gfp_mask); + return xfs_buf_alloc_folio(bp, size, gfp_mask); +} + +static int +xfs_buf_alloc( + struct xfs_buftarg *target, + struct xfs_buf_map *map, + int nmaps, + xfs_buf_flags_t flags, + struct xfs_buf **bpp) +{ + struct xfs_buf *bp; + int error; + int i; + + *bpp = NULL; + bp = kmem_cache_zalloc(xfs_buf_cache, + GFP_KERNEL | __GFP_NOLOCKDEP | __GFP_NOFAIL); + + /* + * We don't want certain flags to appear in b_flags unless they are + * specifically set by later operations on the buffer. + */ + flags &= ~(XBF_TRYLOCK | XBF_ASYNC | XBF_READ_AHEAD); + lockref_init(&bp->b_lockref); + sema_init(&bp->b_sema, 1); /* unlocked */ + atomic_set(&bp->b_lru_ref, 1); + init_completion(&bp->b_iowait); + INIT_LIST_HEAD(&bp->b_lru); + INIT_LIST_HEAD(&bp->b_list); + INIT_LIST_HEAD(&bp->b_li_list); + bp->b_target = target; + bp->b_mount = target->bt_mount; + WRITE_ONCE(bp->b_flags, flags); + bp->b_rhash_key = map[0].bm_bn; + bp->b_length = 0; + bp->b_map_count = nmaps; + if (nmaps == 1) + bp->b_maps = &bp->__b_map; + else + bp->b_maps = kzalloc_objs(struct xfs_buf_map, nmaps, + GFP_KERNEL | __GFP_NOLOCKDEP | __GFP_NOFAIL); + for (i = 0; i < nmaps; i++) { + bp->b_maps[i].bm_bn = map[i].bm_bn; + bp->b_maps[i].bm_len = map[i].bm_len; + bp->b_length += map[i].bm_len; + } + + atomic_set(&bp->b_pin_count, 0); + init_waitqueue_head(&bp->b_waiters); + + XFS_STATS_INC(bp->b_mount, xb_create); + trace_xfs_buf_init(bp, _RET_IP_); + + error = xfs_buf_alloc_backing_mem(bp, flags); + if (error) { + xfs_buf_free(bp); + return error; + } + + *bpp = bp; + return 0; +} + +/* + * Finding and Reading Buffers + */ +static int +_xfs_buf_obj_cmp( + struct rhashtable_compare_arg *arg, + const void *obj) +{ + const struct xfs_buf_map *map = arg->key; + const struct xfs_buf *bp = obj; + + /* + * The key hashing in the lookup path depends on the key being the + * first element of the compare_arg, make sure to assert this. + */ + BUILD_BUG_ON(offsetof(struct xfs_buf_map, bm_bn) != 0); + + if (bp->b_rhash_key != map->bm_bn) + return 1; + + if (unlikely(bp->b_length != map->bm_len)) { + /* + * found a block number match. If the range doesn't + * match, the only way this is allowed is if the buffer + * in the cache is stale and the transaction that made + * it stale has not yet committed. i.e. we are + * reallocating a busy extent. Skip this buffer and + * continue searching for an exact match. + * + * Note: If we're scanning for incore buffers to stale, don't + * complain if we find non-stale buffers. + */ + if (!(map->bm_flags & XBM_LIVESCAN)) + ASSERT(bp->b_flags & XBF_STALE); + return 1; + } + return 0; +} + +static const struct rhashtable_params xfs_buf_hash_params = { + .min_size = 32, /* empty AGs have minimal footprint */ + .nelem_hint = 16, + .key_len = sizeof(xfs_daddr_t), + .key_offset = offsetof(struct xfs_buf, b_rhash_key), + .head_offset = offsetof(struct xfs_buf, b_rhash_head), + .automatic_shrinking = true, + .obj_cmpfn = _xfs_buf_obj_cmp, +}; + +static int +xfs_buf_map_verify( + struct xfs_buftarg *btp, + struct xfs_buf_map *map) +{ + /* Check for IOs smaller than the sector size / not sector aligned */ + ASSERT(!(BBTOB(map->bm_len) < btp->bt_meta_sectorsize)); + ASSERT(!(BBTOB(map->bm_bn) & (xfs_off_t)btp->bt_meta_sectormask)); + + /* + * Corrupted block numbers can get through to here, unfortunately, so we + * have to check that the buffer falls within the filesystem bounds. + */ + if (map->bm_bn < 0 || map->bm_bn >= btp->bt_nr_sectors) { + xfs_alert(btp->bt_mount, + "%s: daddr 0x%llx out of range, EOFS 0x%llx", + __func__, map->bm_bn, btp->bt_nr_sectors); + WARN_ON(1); + return -EFSCORRUPTED; + } + return 0; +} + +static int +xfs_buf_find_lock( + struct xfs_buf *bp, + xfs_buf_flags_t flags) +{ + if (flags & XBF_TRYLOCK) { + if (!xfs_buf_trylock(bp)) { + XFS_STATS_INC(bp->b_mount, xb_busy_locked); + return -EAGAIN; + } + } else { + xfs_buf_lock(bp); + XFS_STATS_INC(bp->b_mount, xb_get_locked_waited); + } + + /* + * if the buffer is stale, clear all the external state associated with + * it. We need to keep flags such as how we allocated the buffer memory + * intact here. + */ + if (bp->b_flags & XBF_STALE) { + if (flags & XBF_LIVESCAN) { + xfs_buf_unlock(bp); + return -ENOENT; + } + ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0); + xfs_buf_clear_flags(bp, ~_XBF_KMEM); + bp->b_ops = NULL; + } + return 0; +} + +static inline struct xfs_buf * +xfs_buf_lookup( + struct xfs_buftarg *btp, + struct xfs_buf_map *map) +{ + struct xfs_buf *bp; + + rcu_read_lock(); + bp = rhashtable_lookup(&btp->bt_hash, map, xfs_buf_hash_params); + if (!bp || !lockref_get_not_dead(&bp->b_lockref)) { + rcu_read_unlock(); + XFS_STATS_INC(btp->bt_mount, xb_miss_locked); + return NULL; + } + rcu_read_unlock(); + + trace_xfs_buf_find(bp, _RET_IP_); + XFS_STATS_INC(btp->bt_mount, xb_get_locked); + return bp; +} + +/* + * Insert the new_bp into the hash table. This consumes the perag reference + * taken for the lookup regardless of the result of the insert. + */ +static int +xfs_buf_find_insert( + struct xfs_buftarg *btp, + struct xfs_buf_map *cmap, + struct xfs_buf_map *map, + int nmaps, + xfs_buf_flags_t flags, + struct xfs_buf **bpp) +{ + struct xfs_buf *new_bp; + struct xfs_buf *bp; + int error; + + error = xfs_buf_alloc(btp, map, nmaps, flags, &new_bp); + if (error) + return error; + + /* The new buffer keeps the perag reference until it is freed. */ + if (!xfs_buftarg_is_mem(btp)) { + new_bp->b_pag = xfs_perag_get(btp->bt_mount, + xfs_daddr_to_agno(btp->bt_mount, cmap->bm_bn)); + } + +retry: + rcu_read_lock(); + bp = rhashtable_lookup_get_insert_fast(&btp->bt_hash, + &new_bp->b_rhash_head, xfs_buf_hash_params); + if (IS_ERR(bp)) { + rcu_read_unlock(); + error = PTR_ERR(bp); + goto out_free_buf; + } + if (bp) { + /* + * If there is an existing buffer with a dead lockref, retry + * until the new buffer is added, or a usable buffer is found. + */ + if (!lockref_get_not_dead(&bp->b_lockref)) { + rcu_read_unlock(); + cpu_relax(); + goto retry; + } + rcu_read_unlock(); + *bpp = bp; + goto out_free_buf; + } + rcu_read_unlock(); + + *bpp = new_bp; + return 0; + +out_free_buf: + if (new_bp->b_pag) + xfs_perag_put(new_bp->b_pag); + xfs_buf_free(new_bp); + return error; +} + +/* + * Assembles a buffer covering the specified range. The code is optimised for + * cache hits, as metadata intensive workloads will see 3 orders of magnitude + * more hits than misses. + */ +static int +xfs_find_get_buf( + struct xfs_buftarg *btp, + struct xfs_buf_map *map, + int nmaps, + xfs_buf_flags_t flags, + struct xfs_buf **bpp) +{ + struct xfs_buf *bp = NULL; + struct xfs_buf_map cmap = { .bm_bn = map[0].bm_bn }; + int error; + int i; + + if (flags & XBF_LIVESCAN) + cmap.bm_flags |= XBM_LIVESCAN; + for (i = 0; i < nmaps; i++) + cmap.bm_len += map[i].bm_len; + + error = xfs_buf_map_verify(btp, &cmap); + if (error) + return error; + + /* cache hits always outnumber misses by at least 10:1 */ + bp = xfs_buf_lookup(btp, &cmap); + if (unlikely(!bp)) { + if (flags & XBF_INCORE) + return -ENOENT; + error = xfs_buf_find_insert(btp, &cmap, map, nmaps, flags, &bp); + if (error) + return error; + } + + *bpp = bp; + return 0; +} + +int +xfs_buf_get_map( + struct xfs_buftarg *btp, + struct xfs_buf_map *map, + int nmaps, + xfs_buf_flags_t flags, + struct xfs_buf **bpp) +{ + int error; + + ASSERT(!(flags & ~(XBF_TRYLOCK | XBF_INCORE | XBF_LIVESCAN))); + ASSERT(!(flags & XBF_LIVESCAN) || (flags & XBF_INCORE)); + + /* + * Zero the buffer and clear b_error as xfs_buf_get_map callers don't + * expect valid data to be found in the buffer. + */ + error = xfs_find_get_buf(btp, map, nmaps, flags, bpp); + if (error) + return error; + + error = xfs_buf_find_lock(*bpp, flags); + if (error) { + xfs_buf_rele(*bpp); + return error; + } + XFS_STATS_INC(btp->bt_mount, xb_get); + trace_xfs_buf_get(*bpp, flags, _RET_IP_); + xfs_buf_ioerror(*bpp, 0); + return 0; +} + +int +_xfs_buf_read( + struct xfs_buf *bp) +{ + ASSERT(bp->b_maps[0].bm_bn != XFS_BUF_DADDR_NULL); + + xfs_buf_clear_flags(bp, XBF_WRITE | XBF_ASYNC | XBF_READ_AHEAD | + XBF_DONE); + xfs_buf_set_flags(bp, XBF_READ); + xfs_buf_submit(bp); + return xfs_buf_iowait(bp); +} + +int +xfs_buf_read_map( + struct xfs_buftarg *target, + struct xfs_buf_map *map, + int nmaps, + xfs_buf_flags_t flags, + struct xfs_buf **bpp, + const struct xfs_buf_ops *ops, + xfs_failaddr_t fa) +{ + struct xfs_buf *bp; + int error; + + ASSERT(!(flags & ~XBF_TRYLOCK)); + + flags |= XBF_READ; + *bpp = NULL; + + error = xfs_find_get_buf(target, map, nmaps, flags, &bp); + if (error) + return error; + error = xfs_buf_find_lock(bp, flags); + if (error) { + xfs_buf_rele(bp); + return error; + } + + trace_xfs_buf_read(bp, flags, _RET_IP_); + + if (bp->b_flags & XBF_DONE) { + ASSERT(bp->b_error == 0); + + /* + * If the caller passed an ops structure and the buffer doesn't + * have ops assigned yet, set the ops and use them to verify the + * buffer contents. + * + * Under normal operations, every in-core buffer is verified on + * read I/O completion, but there are two scenarios that can + * lead to in-core buffers without an assigned ->b_ops: + * + * 1) During log recovery of buffers on a V4 filesystem. + * These buffers are purged at the end of recovery, though. + * 2) Oonline repair intentionally reads with a NULL buffer + * ops to run several verifiers across an in-core buffer in + * order to establish buffer type. If repair can't + * establish that, the buffer will be left in memory with + * NULL buffer ops. + */ + if (ops && !bp->b_ops) { + bp->b_ops = ops; + bp->b_ops->verify_read(bp); + /* + * If verification failed, clear XBF_DONE as we assume + * that buffers have no recorded errors when in XBF_DONE + * state. + */ + error = bp->b_error; + if (error) + xfs_buf_clear_flags(bp, XBF_DONE); + } + + /* We do not want read in the flags */ + xfs_buf_clear_flags(bp, XBF_READ); + } else { + /* Initiate the buffer read and wait. */ + XFS_STATS_INC(target->bt_mount, xb_get_read); + bp->b_ops = ops; + error = _xfs_buf_read(bp); + } + + if (error) + goto out_ioerror; + + *bpp = bp; + return 0; + +out_ioerror: + /* + * Check against log shutdown for error reporting because metadata + * writeback may require a read first and we need to report errors in + * metadata writeback until the log is shut down. High level + * transaction read functions already check against mount shutdown, so + * we only need to be concerned about low level/ IO interactions here. + */ + if (!xlog_is_shutdown(target->bt_mount->m_log)) + xfs_buf_ioerror_alert(bp, fa); + + /* + * If we've had a read error, then the contents of the buffer are + * invalid and should not be used. To ensure that a followup read tries + * to pull the buffer from disk again, we clear the XBF_DONE flag and + * mark the buffer stale. This ensures that anyone who has a current + * reference to the buffer will interpret it's contents correctly and + * future cache lookups will also treat it as an empty, uninitialised + * buffer. + */ + xfs_buf_clear_flags(bp, XBF_DONE); + xfs_buf_stale(bp); + xfs_buf_relse(bp); + + /* bad CRC means corrupted metadata */ + if (error == -EFSBADCRC) + return -EFSCORRUPTED; + return error; +} + +/* + * If we are not low on memory then do the readahead in a deadlock + * safe manner. + */ +void +xfs_buf_readahead_map( + struct xfs_buftarg *target, + struct xfs_buf_map *map, + int nmaps, + const struct xfs_buf_ops *ops) +{ + const xfs_buf_flags_t flags = XBF_READ | XBF_ASYNC | XBF_READ_AHEAD; + struct xfs_buf *bp; + + /* + * Currently we don't have a good means or justification for performing + * xmbuf_map_page asynchronously, so we don't do readahead. + */ + if (xfs_buftarg_is_mem(target)) + return; + + if (xfs_find_get_buf(target, map, nmaps, flags, &bp)) + return; + + /* + * Do a lockless fast path check for a valid uptodate buffer and avoid + * locking entirely in this case. + */ + if ((READ_ONCE(bp->b_flags) & (XBF_DONE | XBF_STALE)) == XBF_DONE) + goto out_rele; + + /* Otherwise lock the buffer to stabilize the state */ + if (!xfs_buf_trylock(bp)) + goto out_rele; + + /* Let the actual reader deal with stale buffers. */ + if (bp->b_flags & (XBF_STALE | XBF_DONE)) + goto out_unlock; + + trace_xfs_buf_readahead(bp, 0, _RET_IP_); + XFS_STATS_INC(target->bt_mount, xb_get_read); + bp->b_ops = ops; + xfs_buf_clear_flags(bp, XBF_WRITE | XBF_DONE); + xfs_buf_set_flags(bp, flags); + percpu_counter_inc(&target->bt_readahead_count); + xfs_buf_submit(bp); + return; +out_unlock: + xfs_buf_unlock(bp); +out_rele: + xfs_buf_rele(bp); +} + +/* + * Read an uncached buffer from disk. Allocates and returns a locked + * buffer containing the disk contents or nothing. Uncached buffers always have + * a cache index of XFS_BUF_DADDR_NULL so we can easily determine if the buffer + * is cached or uncached during fault diagnosis. + */ +int +xfs_buf_read_uncached( + struct xfs_buftarg *target, + xfs_daddr_t daddr, + size_t numblks, + struct xfs_buf **bpp, + const struct xfs_buf_ops *ops) +{ + struct xfs_buf *bp; + int error; + + *bpp = NULL; + + error = xfs_buf_get_uncached(target, numblks, &bp); + if (error) + return error; + + /* set up the buffer for a read IO */ + ASSERT(bp->b_map_count == 1); + bp->b_rhash_key = XFS_BUF_DADDR_NULL; + bp->b_maps[0].bm_bn = daddr; + xfs_buf_set_flags(bp, XBF_READ); + bp->b_ops = ops; + + xfs_buf_submit(bp); + error = xfs_buf_iowait(bp); + if (error) { + xfs_buf_relse(bp); + return error; + } + + *bpp = bp; + return 0; +} + +int +xfs_buf_get_uncached( + struct xfs_buftarg *target, + size_t numblks, + struct xfs_buf **bpp) +{ + int error; + DEFINE_SINGLE_BUF_MAP(map, XFS_BUF_DADDR_NULL, numblks); + + error = xfs_buf_alloc(target, &map, 1, 0, bpp); + if (error) + return error; + xfs_buf_lock(*bpp); + trace_xfs_buf_get_uncached(*bpp, _RET_IP_); + return 0; +} + +/* + * Increment reference count on buffer, to hold the buffer concurrently + * with another thread which may release (free) the buffer asynchronously. + * Must hold the buffer already to call this function. + */ +void +xfs_buf_hold( + struct xfs_buf *bp) +{ + trace_xfs_buf_hold(bp, _RET_IP_); + + lockref_get(&bp->b_lockref); +} + +static void +xfs_buf_destroy( + struct xfs_buf *bp) +{ + ASSERT(lockref_is_dead(&bp->b_lockref)); + ASSERT(!(bp->b_flags & _XBF_DELWRI_Q)); + + if (bp->b_pag) + xfs_perag_put(bp->b_pag); + xfs_buf_free(bp); +} + +static inline void +xfs_buf_kill( + struct xfs_buf *bp) +{ + lockref_mark_dead(&bp->b_lockref); + if (!xfs_buf_is_uncached(bp)) { + rhashtable_remove_fast(&bp->b_target->bt_hash, + &bp->b_rhash_head, xfs_buf_hash_params); + } +} + +/* + * Release a hold on the specified buffer. + */ +void +xfs_buf_rele( + struct xfs_buf *bp) +{ + trace_xfs_buf_rele(bp, _RET_IP_); + + if (lockref_put_or_lock(&bp->b_lockref)) + return; + if (!--bp->b_lockref.count) { + if (xfs_buf_is_uncached(bp) || !atomic_read(&bp->b_lru_ref)) + goto kill; + list_lru_add_obj(&bp->b_target->bt_lru, &bp->b_lru); + } + spin_unlock(&bp->b_lockref.lock); + return; + +kill: + xfs_buf_kill(bp); + list_lru_del_obj(&bp->b_target->bt_lru, &bp->b_lru); + spin_unlock(&bp->b_lockref.lock); + + xfs_buf_destroy(bp); +} + +/* + * Lock a buffer object, if it is not already locked. + * + * If we come across a stale, pinned, locked buffer, we know that we are + * being asked to lock a buffer that has been reallocated. Because it is + * pinned, we know that the log has not been pushed to disk and hence it + * will still be locked. Rather than continuing to have trylock attempts + * fail until someone else pushes the log, push it ourselves before + * returning. This means that the xfsaild will not get stuck trying + * to push on stale inode buffers. + */ +int +xfs_buf_trylock( + struct xfs_buf *bp) +{ + int locked; + + locked = down_trylock(&bp->b_sema) == 0; + if (locked) + trace_xfs_buf_trylock(bp, _RET_IP_); + else + trace_xfs_buf_trylock_fail(bp, _RET_IP_); + return locked; +} + +/* + * Lock a buffer object. + * + * If we come across a stale, pinned, locked buffer, we know that we + * are being asked to lock a buffer that has been reallocated. Because + * it is pinned, we know that the log has not been pushed to disk and + * hence it will still be locked. Rather than sleeping until someone + * else pushes the log, push it ourselves before trying to get the lock. + */ +void +xfs_buf_lock( + struct xfs_buf *bp) +{ + trace_xfs_buf_lock(bp, _RET_IP_); + + if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE)) + xfs_log_force(bp->b_mount, 0); + down(&bp->b_sema); + + trace_xfs_buf_lock_done(bp, _RET_IP_); +} + +void +xfs_buf_unlock( + struct xfs_buf *bp) +{ + ASSERT(xfs_buf_islocked(bp)); + + up(&bp->b_sema); + trace_xfs_buf_unlock(bp, _RET_IP_); +} + +STATIC void +xfs_buf_wait_unpin( + struct xfs_buf *bp) +{ + DECLARE_WAITQUEUE (wait, current); + + if (atomic_read(&bp->b_pin_count) == 0) + return; + + add_wait_queue(&bp->b_waiters, &wait); + for (;;) { + set_current_state(TASK_UNINTERRUPTIBLE); + if (atomic_read(&bp->b_pin_count) == 0) + break; + io_schedule(); + } + remove_wait_queue(&bp->b_waiters, &wait); + set_current_state(TASK_RUNNING); +} + +static void +xfs_buf_ioerror_alert_ratelimited( + struct xfs_buf *bp) +{ + static unsigned long lasttime; + static struct xfs_buftarg *lasttarg; + + if (bp->b_target != lasttarg || + time_after(jiffies, (lasttime + 5*HZ))) { + lasttime = jiffies; + xfs_buf_ioerror_alert(bp, __this_address); + } + lasttarg = bp->b_target; +} + +/* + * Account for this latest trip around the retry handler, and decide if + * we've failed enough times to constitute a permanent failure. + */ +static bool +xfs_buf_ioerror_permanent( + struct xfs_buf *bp, + struct xfs_error_cfg *cfg) +{ + struct xfs_mount *mp = bp->b_mount; + + if (cfg->max_retries != XFS_ERR_RETRY_FOREVER && + ++bp->b_retries > cfg->max_retries) + return true; + if (cfg->retry_timeout != XFS_ERR_RETRY_FOREVER && + time_after(jiffies, cfg->retry_timeout + bp->b_first_retry_time)) + return true; + + /* At unmount we may treat errors differently */ + if (xfs_is_unmounting(mp) && mp->m_fail_unmount) + return true; + + return false; +} + +/* + * On a sync write or shutdown we just want to stale the buffer and let the + * caller handle the error in bp->b_error appropriately. + * + * If the write was asynchronous then no one will be looking for the error. If + * this is the first failure of this type, clear the error state and write the + * buffer out again. This means we always retry an async write failure at least + * once, but we also need to set the buffer up to behave correctly now for + * repeated failures. + * + * If we get repeated async write failures, then we take action according to the + * error configuration we have been set up to use. + * + * Returns true if this function took care of error handling and the caller must + * not touch the buffer again. Return false if the caller should proceed with + * normal I/O completion handling. + */ +static bool +xfs_buf_ioend_handle_error( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_error_cfg *cfg; + struct xfs_log_item *lip; + + /* + * If we've already shutdown the journal because of I/O errors, there's + * no point in giving this a retry. + */ + if (xlog_is_shutdown(mp->m_log)) + goto out_stale; + + xfs_buf_ioerror_alert_ratelimited(bp); + + /* + * We're not going to bother about retrying this during recovery. + * One strike! + */ + if (mp->m_log && xlog_in_recovery(mp->m_log)) { + xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR); + return false; + } + + /* + * Synchronous writes will have callers process the error. + */ + if (!(bp->b_flags & XBF_ASYNC)) + goto out_stale; + + trace_xfs_buf_iodone_async(bp, _RET_IP_); + + cfg = xfs_error_get_cfg(mp, XFS_ERR_METADATA, bp->b_error); + if (bp->b_last_error != bp->b_error || + !(bp->b_flags & (XBF_STALE | XBF_WRITE_FAIL))) { + bp->b_last_error = bp->b_error; + if (cfg->retry_timeout != XFS_ERR_RETRY_FOREVER && + !bp->b_first_retry_time) + bp->b_first_retry_time = jiffies; + goto resubmit; + } + + /* + * Permanent error - we need to trigger a shutdown if we haven't already + * to indicate that inconsistency will result from this action. + */ + if (xfs_buf_ioerror_permanent(bp, cfg)) { + xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR); + goto out_stale; + } + + /* Still considered a transient error. Caller will schedule retries. */ + list_for_each_entry(lip, &bp->b_li_list, li_bio_list) { + set_bit(XFS_LI_FAILED, &lip->li_flags); + clear_bit(XFS_LI_FLUSHING, &lip->li_flags); + } + + xfs_buf_ioerror(bp, 0); + xfs_buf_relse(bp); + return true; + +resubmit: + xfs_buf_ioerror(bp, 0); + xfs_buf_set_flags(bp, XBF_DONE | XBF_WRITE_FAIL); + reinit_completion(&bp->b_iowait); + xfs_buf_submit(bp); + return true; +out_stale: + xfs_buf_stale(bp); + xfs_buf_set_flags(bp, XBF_DONE); + xfs_buf_clear_flags(bp, XBF_WRITE); + trace_xfs_buf_error_relse(bp, _RET_IP_); + return false; +} + +/* + * Complete a buffer read or write. + * + * Releases the buffer if the I/O was asynchronous. + */ +static void +xfs_buf_ioend( + struct xfs_buf *bp) +{ + bool async = bp->b_flags & XBF_ASYNC; + + trace_xfs_buf_iodone(bp, _RET_IP_); + + if (bp->b_flags & XBF_READ) { + if (!bp->b_error && is_vmalloc_addr(bp->b_addr)) + invalidate_kernel_vmap_range(bp->b_addr, + roundup(BBTOB(bp->b_length), PAGE_SIZE)); + if (!bp->b_error && bp->b_ops) + bp->b_ops->verify_read(bp); + if (!bp->b_error) + xfs_buf_set_flags(bp, XBF_DONE); + if (bp->b_flags & XBF_READ_AHEAD) + percpu_counter_dec(&bp->b_target->bt_readahead_count); + } else { + if (unlikely(bp->b_error)) { + if (xfs_buf_ioend_handle_error(bp)) { + ASSERT(async); + return; + } + } else { + xfs_buf_clear_flags(bp, XBF_WRITE_FAIL); + xfs_buf_set_flags(bp, XBF_DONE); + } + + /* clear the retry state */ + bp->b_last_error = 0; + bp->b_retries = 0; + bp->b_first_retry_time = 0; + + /* + * Note that for things like remote attribute buffers, there may + * not be a buffer log item here, so processing the buffer log + * item must remain optional. + */ + if (bp->b_log_item) + xfs_buf_item_done(bp); + + if (bp->b_iodone) + bp->b_iodone(bp); + } + + xfs_buf_clear_flags(bp, XBF_READ | XBF_WRITE | XBF_READ_AHEAD); + if (async) + xfs_buf_relse(bp); +} + +static void +xfs_buf_ioend_work( + struct work_struct *work) +{ + xfs_buf_ioend(container_of(work, struct xfs_buf, b_ioend_work)); +} + +void +__xfs_buf_ioerror( + struct xfs_buf *bp, + int error, + xfs_failaddr_t failaddr) +{ + ASSERT(error <= 0 && error >= -1000); + bp->b_error = error; + trace_xfs_buf_ioerror(bp, error, failaddr); +} + +void +xfs_buf_ioerror_alert( + struct xfs_buf *bp, + xfs_failaddr_t func) +{ + xfs_buf_alert_ratelimited(bp, "XFS: metadata IO error", + "metadata I/O error in \"%pS\" at daddr 0x%llx len %d error %d", + func, (uint64_t)xfs_buf_daddr(bp), + bp->b_length, -bp->b_error); +} + +/* + * Fail a locked and referenced buffer outside the I/O path. + * + * The caller transfers a reference which will be released after processing the + * error. + */ +void +xfs_buf_fail( + struct xfs_buf *bp) +{ + ASSERT(xfs_buf_islocked(bp)); + + xfs_buf_set_flags(bp, XBF_ASYNC); + xfs_buf_clear_flags(bp, XBF_DONE); + xfs_buf_stale(bp); + xfs_buf_ioerror(bp, -EIO); + xfs_buf_ioend(bp); +} + +int +xfs_bwrite( + struct xfs_buf *bp) +{ + int error; + + ASSERT(xfs_buf_islocked(bp)); + + xfs_buf_set_flags(bp, XBF_WRITE); + xfs_buf_clear_flags(bp, XBF_ASYNC | XBF_READ | _XBF_DELWRI_Q | + XBF_DONE); + + xfs_buf_submit(bp); + error = xfs_buf_iowait(bp); + if (error) + xfs_force_shutdown(bp->b_mount, SHUTDOWN_META_IO_ERROR); + return error; +} + +static void +xfs_buf_bio_end_io( + struct bio *bio) +{ + struct xfs_buf *bp = bio->bi_private; + + if (bio->bi_status) + xfs_buf_ioerror(bp, blk_status_to_errno(bio->bi_status)); + else if ((bp->b_flags & XBF_WRITE) && (bp->b_flags & XBF_ASYNC) && + XFS_TEST_ERROR(bp->b_mount, XFS_ERRTAG_BUF_IOERROR)) + xfs_buf_ioerror(bp, -EIO); + + if (bp->b_flags & XBF_ASYNC) { + INIT_WORK(&bp->b_ioend_work, xfs_buf_ioend_work); + queue_work(bp->b_mount->m_buf_workqueue, &bp->b_ioend_work); + } else { + complete(&bp->b_iowait); + } + + bio_put(bio); +} + +static inline blk_opf_t +xfs_buf_bio_op( + struct xfs_buf *bp) +{ + blk_opf_t op; + + if (bp->b_flags & XBF_WRITE) { + op = REQ_OP_WRITE; + } else { + op = REQ_OP_READ; + if (bp->b_flags & XBF_READ_AHEAD) + op |= REQ_RAHEAD; + } + + return op | REQ_META; +} + +static void +xfs_buf_submit_bio( + struct xfs_buf *bp) +{ + unsigned int len = BBTOB(bp->b_length); + unsigned int nr_vecs = bio_add_max_vecs(bp->b_addr, len); + unsigned int map = 0; + struct blk_plug plug; + struct bio *bio; + + bio = bio_alloc(bp->b_target->bt_bdev, nr_vecs, xfs_buf_bio_op(bp), + GFP_NOIO); + if (is_vmalloc_addr(bp->b_addr)) + bio_add_vmalloc(bio, bp->b_addr, len); + else + bio_add_virt_nofail(bio, bp->b_addr, len); + bio->bi_private = bp; + bio->bi_end_io = xfs_buf_bio_end_io; + + /* + * If there is more than one map segment, split out a new bio for each + * map except of the last one. The last map is handled by the + * remainder of the original bio outside the loop. + */ + blk_start_plug(&plug); + for (map = 0; map < bp->b_map_count - 1; map++) { + struct bio *split; + + split = bio_split(bio, bp->b_maps[map].bm_len, GFP_NOFS, + &fs_bio_set); + split->bi_iter.bi_sector = bp->b_maps[map].bm_bn; + bio_chain(split, bio); + submit_bio(split); + } + bio->bi_iter.bi_sector = bp->b_maps[map].bm_bn; + submit_bio(bio); + blk_finish_plug(&plug); +} + +/* + * Wait for I/O completion of a sync buffer and return the I/O error code. + */ +static int +xfs_buf_iowait( + struct xfs_buf *bp) +{ + ASSERT(!(bp->b_flags & XBF_ASYNC)); + + trace_xfs_buf_iowait(bp, _RET_IP_); + wait_for_completion(&bp->b_iowait); + trace_xfs_buf_iowait_done(bp, _RET_IP_); + + xfs_buf_ioend(bp); + return bp->b_error; +} + +/* + * Run the write verifier callback function if it exists. If this fails, mark + * the buffer with an error and do not dispatch the I/O. + */ +static bool +xfs_buf_verify_write( + struct xfs_buf *bp) +{ + if (bp->b_ops) { + bp->b_ops->verify_write(bp); + if (bp->b_error) + return false; + } else if (bp->b_rhash_key != XFS_BUF_DADDR_NULL) { + /* + * Non-crc filesystems don't attach verifiers during log + * recovery, so don't warn for such filesystems. + */ + if (xfs_has_crc(bp->b_mount)) { + xfs_warn(bp->b_mount, + "%s: no buf ops on daddr 0x%llx len %d", + __func__, xfs_buf_daddr(bp), + bp->b_length); + xfs_hex_dump(bp->b_addr, XFS_CORRUPTION_DUMP_LEN); + dump_stack(); + } + } + + return true; +} + +/* + * Buffer I/O submission path, read or write. Asynchronous submission transfers + * the buffer lock ownership and the current reference to the IO. It is not + * safe to reference the buffer after a call to this function unless the caller + * holds an additional reference itself. + */ +static void +xfs_buf_submit( + struct xfs_buf *bp) +{ + trace_xfs_buf_submit(bp, _RET_IP_); + + ASSERT(!(bp->b_flags & _XBF_DELWRI_Q)); + + /* + * On log shutdown we stale and complete the buffer immediately. We can + * be called to read the superblock before the log has been set up, so + * be careful checking the log state. + * + * Checking the mount shutdown state here can result in the log tail + * moving inappropriately on disk as the log may not yet be shut down. + * i.e. failing this buffer on mount shutdown can remove it from the AIL + * and move the tail of the log forwards without having written this + * buffer to disk. This corrupts the log tail state in memory, and + * because the log may not be shut down yet, it can then be propagated + * to disk before the log is shutdown. Hence we check log shutdown + * state here rather than mount state to avoid corrupting the log tail + * on shutdown. + */ + if (bp->b_mount->m_log && xlog_is_shutdown(bp->b_mount->m_log)) { + xfs_buf_ioerror(bp, -EIO); + goto ioerror; + } + + if (bp->b_flags & XBF_WRITE) + xfs_buf_wait_unpin(bp); + + /* + * Make sure we capture only current IO errors rather than stale errors + * left over from previous use of the buffer (e.g. failed readahead). + */ + bp->b_error = 0; + + if ((bp->b_flags & XBF_WRITE) && !xfs_buf_verify_write(bp)) { + /* ->verify_write should have set b_error already */ + xfs_force_shutdown(bp->b_mount, SHUTDOWN_CORRUPT_INCORE); + goto ioerror; + } + + /* In-memory targets are directly mapped, no I/O required. */ + if (xfs_buftarg_is_mem(bp->b_target)) + goto end_io; + + xfs_buf_submit_bio(bp); + return; + +ioerror: + xfs_buf_clear_flags(bp, XBF_DONE); + xfs_buf_stale(bp); +end_io: + if (bp->b_flags & XBF_ASYNC) + xfs_buf_ioend(bp); + else + complete(&bp->b_iowait); +} + +/* + * Log a message about and stale a buffer that a caller has decided is corrupt. + * + * This function should be called for the kinds of metadata corruption that + * cannot be detect from a verifier, such as incorrect inter-block relationship + * data. Do /not/ call this function from a verifier function. + * + * The buffer must be XBF_DONE prior to the call. Afterwards, the buffer will + * be marked stale, but b_error will not be set. The caller is responsible for + * releasing the buffer or fixing it. + */ +void +__xfs_buf_mark_corrupt( + struct xfs_buf *bp, + xfs_failaddr_t fa) +{ + ASSERT(bp->b_flags & XBF_DONE); + + xfs_buf_corruption_error(bp, fa); + xfs_buf_stale(bp); +} + +/* + * Handling of buffer targets (buftargs). + */ + +/* + * Wait for any bufs with callbacks that have been submitted but have not yet + * returned. These buffers will have an elevated hold count, so wait on those + * while freeing all the buffers only held by the LRU. + */ +static enum lru_status +xfs_buftarg_drain_rele( + struct list_head *item, + struct list_lru_one *lru, + void *arg) + +{ + struct xfs_buf *bp = container_of(item, struct xfs_buf, b_lru); + struct list_head *dispose = arg; + + if (!spin_trylock(&bp->b_lockref.lock)) + return LRU_SKIP; + if (bp->b_lockref.count > 0) { + /* need to wait, so skip it this pass */ + spin_unlock(&bp->b_lockref.lock); + trace_xfs_buf_drain_buftarg(bp, _RET_IP_); + return LRU_SKIP; + } + + xfs_buf_kill(bp); + list_lru_isolate_move(lru, item, dispose); + spin_unlock(&bp->b_lockref.lock); + return LRU_REMOVED; +} + +/* + * Wait for outstanding I/O on the buftarg to complete. + */ +void +xfs_buftarg_wait( + struct xfs_buftarg *btp) +{ + /* + * First wait for all in-flight readahead buffers to be released. This is + * critical as new buffers do not make the LRU until they are released. + * + * Next, flush the buffer workqueue to ensure all completion processing + * has finished. Just waiting on buffer locks is not sufficient for + * async IO as the reference count held over IO is not released until + * after the buffer lock is dropped. Hence we need to ensure here that + * all reference counts have been dropped before we start walking the + * LRU list. + */ + while (percpu_counter_sum(&btp->bt_readahead_count)) + delay(100); + flush_workqueue(btp->bt_mount->m_buf_workqueue); +} + +void +xfs_buftarg_drain( + struct xfs_buftarg *btp) +{ + LIST_HEAD(dispose); + int loop = 0; + bool write_fail = false; + + xfs_buftarg_wait(btp); + + /* loop until there is nothing left on the lru list. */ + while (list_lru_count(&btp->bt_lru)) { + list_lru_walk(&btp->bt_lru, xfs_buftarg_drain_rele, + &dispose, LONG_MAX); + + while (!list_empty(&dispose)) { + struct xfs_buf *bp; + bp = list_first_entry(&dispose, struct xfs_buf, b_lru); + list_del_init(&bp->b_lru); + if (bp->b_flags & XBF_WRITE_FAIL) { + write_fail = true; + xfs_buf_alert_ratelimited(bp, + "XFS: Corruption Alert", +"Corruption Alert: Buffer at daddr 0x%llx had permanent write failures!", + (long long)xfs_buf_daddr(bp)); + } + xfs_buf_destroy(bp); + } + if (loop++ != 0) + delay(100); + } + + /* + * If one or more failed buffers were freed, that means dirty metadata + * was thrown away. This should only ever happen after I/O completion + * handling has elevated I/O error(s) to permanent failures and shuts + * down the journal. + */ + if (write_fail) { + ASSERT(xlog_is_shutdown(btp->bt_mount->m_log)); + xfs_alert(btp->bt_mount, + "Please run xfs_repair to determine the extent of the problem."); + } +} + +static enum lru_status +xfs_buftarg_isolate( + struct list_head *item, + struct list_lru_one *lru, + void *arg) +{ + struct xfs_buf *bp = container_of(item, struct xfs_buf, b_lru); + struct list_head *dispose = arg; + + /* + * We are inverting the lru lock vs bp->b_lockref.lock order here, so + * use a trylock. If we fail to get the lock, just skip the buffer. + */ + if (!spin_trylock(&bp->b_lockref.lock)) + return LRU_SKIP; + + /* + * If the buffer is in use, remove it from the LRU for now. We can't + * free it while someone is using it, and we should also not count + * eviction passed for it, just as if it hadn't been added to the LRU + * yet. + */ + if (bp->b_lockref.count > 0) { + list_lru_isolate(lru, &bp->b_lru); + spin_unlock(&bp->b_lockref.lock); + return LRU_REMOVED; + } + + /* + * Decrement the b_lru_ref count unless the value is already + * zero. If the value is already zero, we need to reclaim the + * buffer, otherwise it gets another trip through the LRU. + */ + if (atomic_add_unless(&bp->b_lru_ref, -1, 0)) { + spin_unlock(&bp->b_lockref.lock); + return LRU_ROTATE; + } + + xfs_buf_kill(bp); + list_lru_isolate_move(lru, item, dispose); + spin_unlock(&bp->b_lockref.lock); + return LRU_REMOVED; +} + +static unsigned long +xfs_buftarg_shrink_scan( + struct shrinker *shrink, + struct shrink_control *sc) +{ + struct xfs_buftarg *btp = shrink->private_data; + LIST_HEAD(dispose); + unsigned long freed; + + freed = list_lru_shrink_walk(&btp->bt_lru, sc, + xfs_buftarg_isolate, &dispose); + + while (!list_empty(&dispose)) { + struct xfs_buf *bp; + bp = list_first_entry(&dispose, struct xfs_buf, b_lru); + list_del_init(&bp->b_lru); + xfs_buf_destroy(bp); + } + + return freed; +} + +static unsigned long +xfs_buftarg_shrink_count( + struct shrinker *shrink, + struct shrink_control *sc) +{ + struct xfs_buftarg *btp = shrink->private_data; + return list_lru_shrink_count(&btp->bt_lru, sc); +} + +void +xfs_destroy_buftarg( + struct xfs_buftarg *btp) +{ + shrinker_free(btp->bt_shrinker); + ASSERT(percpu_counter_sum(&btp->bt_readahead_count) == 0); + percpu_counter_destroy(&btp->bt_readahead_count); + list_lru_destroy(&btp->bt_lru); + rhashtable_destroy(&btp->bt_hash); +} + +void +xfs_free_buftarg( + struct xfs_buftarg *btp) +{ + xfs_destroy_buftarg(btp); + fs_put_dax(btp->bt_daxdev, btp->bt_mount); + /* the main block device is closed by kill_block_super */ + if (btp->bt_bdev != btp->bt_mount->m_super->s_bdev) + fs_bdev_file_release(btp->bt_file, btp->bt_mount->m_super); + kfree(btp); +} + +/* + * Configure this buffer target for hardware-assisted atomic writes if the + * underlying block device supports is congruent with the filesystem geometry. + */ +static inline void +xfs_configure_buftarg_atomic_writes( + struct xfs_buftarg *btp) +{ + struct xfs_mount *mp = btp->bt_mount; + unsigned int min_bytes, max_bytes; + + min_bytes = bdev_atomic_write_unit_min_bytes(btp->bt_bdev); + max_bytes = bdev_atomic_write_unit_max_bytes(btp->bt_bdev); + + /* + * Ignore atomic write geometry that is nonsense or doesn't even cover + * a single fsblock. + */ + if (min_bytes > max_bytes || + min_bytes > mp->m_sb.sb_blocksize || + max_bytes < mp->m_sb.sb_blocksize) { + min_bytes = 0; + max_bytes = 0; + } + + btp->bt_awu_min = min_bytes; + btp->bt_awu_max = max_bytes; +} + +/* Configure a buffer target that abstracts a block device. */ +int +xfs_configure_buftarg( + struct xfs_buftarg *btp, + unsigned int sectorsize, + xfs_rfsblock_t nr_blocks) +{ + struct xfs_mount *mp = btp->bt_mount; + + if (btp->bt_bdev) { + int error; + + error = bdev_validate_blocksize(btp->bt_bdev, sectorsize); + if (error) { + xfs_warn(mp, + "Cannot use blocksize %u on device %pg, err %d", + sectorsize, btp->bt_bdev, error); + return -EINVAL; + } + + if (bdev_can_atomic_write(btp->bt_bdev)) + xfs_configure_buftarg_atomic_writes(btp); + } + + btp->bt_meta_sectorsize = sectorsize; + btp->bt_meta_sectormask = sectorsize - 1; + /* m_blkbb_log is not set up yet */ + btp->bt_nr_sectors = nr_blocks << (mp->m_sb.sb_blocklog - BBSHIFT); + return 0; +} + +int +xfs_init_buftarg( + struct xfs_buftarg *btp, + size_t logical_sectorsize, + const char *descr) +{ + /* The maximum size of the buftarg is only known once the sb is read. */ + btp->bt_nr_sectors = XFS_BUF_DADDR_MAX; + + /* Set up device logical sector size mask */ + btp->bt_logical_sectorsize = logical_sectorsize; + btp->bt_logical_sectormask = logical_sectorsize - 1; + + /* + * Buffer IO error rate limiting. Limit it to no more than 10 messages + * per 30 seconds so as to not spam logs too much on repeated errors. + */ + ratelimit_state_init(&btp->bt_ioerror_rl, 30 * HZ, + DEFAULT_RATELIMIT_BURST); + + if (rhashtable_init(&btp->bt_hash, &xfs_buf_hash_params)) + return -ENOMEM; + if (list_lru_init(&btp->bt_lru)) + goto out_destroy_hash; + if (percpu_counter_init(&btp->bt_readahead_count, 0, GFP_KERNEL)) + goto out_destroy_lru; + + btp->bt_shrinker = + shrinker_alloc(SHRINKER_NUMA_AWARE, "xfs-buf:%s", descr); + if (!btp->bt_shrinker) + goto out_destroy_io_count; + btp->bt_shrinker->count_objects = xfs_buftarg_shrink_count; + btp->bt_shrinker->scan_objects = xfs_buftarg_shrink_scan; + btp->bt_shrinker->private_data = btp; + shrinker_register(btp->bt_shrinker); + return 0; + +out_destroy_io_count: + percpu_counter_destroy(&btp->bt_readahead_count); +out_destroy_lru: + list_lru_destroy(&btp->bt_lru); +out_destroy_hash: + rhashtable_destroy(&btp->bt_hash); + return -ENOMEM; +} + +struct xfs_buftarg * +xfs_alloc_buftarg( + struct xfs_mount *mp, + struct file *bdev_file) +{ + struct xfs_buftarg *btp; + const struct dax_holder_operations *ops = NULL; + int error; + + +#if defined(CONFIG_FS_DAX) && defined(CONFIG_MEMORY_FAILURE) + ops = &xfs_dax_holder_operations; +#endif + btp = kzalloc_obj(*btp, GFP_KERNEL | __GFP_NOFAIL); + + btp->bt_mount = mp; + btp->bt_file = bdev_file; + btp->bt_bdev = file_bdev(bdev_file); + btp->bt_dev = btp->bt_bdev->bd_dev; + btp->bt_daxdev = fs_dax_get_by_bdev(btp->bt_bdev, &btp->bt_dax_part_off, + mp, ops); + + /* + * Flush and invalidate all devices' pagecaches before reading any + * metadata because XFS doesn't use the bdev pagecache. + */ + error = sync_blockdev(btp->bt_bdev); + if (error) + goto error_free; + + /* + * When allocating the buftargs we have not yet read the super block and + * thus don't know the file system sector size yet. + */ + btp->bt_meta_sectorsize = bdev_logical_block_size(btp->bt_bdev); + btp->bt_meta_sectormask = btp->bt_meta_sectorsize - 1; + + error = xfs_init_buftarg(btp, btp->bt_meta_sectorsize, + mp->m_super->s_id); + if (error) + goto error_free; + + return btp; + +error_free: + fs_put_dax(btp->bt_daxdev, mp); + kfree(btp); + return ERR_PTR(error); +} + +static inline void +xfs_buf_list_del( + struct xfs_buf *bp) +{ + list_del_init(&bp->b_list); + wake_up_var(&bp->b_list); +} + +/* + * Cancel a delayed write list. + * + * Remove each buffer from the list, clear the delwri queue flag and drop the + * associated buffer reference. + */ +void +xfs_buf_delwri_cancel( + struct list_head *list) +{ + struct xfs_buf *bp; + + while (!list_empty(list)) { + bp = list_first_entry(list, struct xfs_buf, b_list); + + xfs_buf_lock(bp); + xfs_buf_clear_flags(bp, _XBF_DELWRI_Q); + xfs_buf_list_del(bp); + xfs_buf_relse(bp); + } +} + +/* + * Add a buffer to the delayed write list. + * + * This queues a buffer for writeout if it hasn't already been. Note that + * neither this routine nor the buffer list submission functions perform + * any internal synchronization. It is expected that the lists are thread-local + * to the callers. + * + * Returns true if we queued up the buffer, or false if it already had + * been on the buffer list. + */ +bool +xfs_buf_delwri_queue( + struct xfs_buf *bp, + struct list_head *list) +{ + ASSERT(xfs_buf_islocked(bp)); + ASSERT(!(bp->b_flags & XBF_READ)); + + /* + * If the buffer is already marked delwri it already is queued up + * by someone else for imediate writeout. Just ignore it in that + * case. + */ + if (bp->b_flags & _XBF_DELWRI_Q) { + trace_xfs_buf_delwri_queued(bp, _RET_IP_); + return false; + } + + trace_xfs_buf_delwri_queue(bp, _RET_IP_); + + /* + * If a buffer gets written out synchronously or marked stale while it + * is on a delwri list we lazily remove it. To do this, the other party + * clears the _XBF_DELWRI_Q flag but otherwise leaves the buffer alone. + * It remains referenced and on the list. In a rare corner case it + * might get readded to a delwri list after the synchronous writeout, in + * which case we need just need to re-add the flag here. + */ + xfs_buf_set_flags(bp, _XBF_DELWRI_Q); + if (list_empty(&bp->b_list)) { + xfs_buf_hold(bp); + list_add_tail(&bp->b_list, list); + } + + return true; +} + +/* + * Queue a buffer to this delwri list as part of a data integrity operation. + * If the buffer is on any other delwri list, we'll wait for that to clear + * so that the caller can submit the buffer for IO and wait for the result. + * Callers must ensure the buffer is not already on the list. + */ +void +xfs_buf_delwri_queue_here( + struct xfs_buf *bp, + struct list_head *buffer_list) +{ + /* + * We need this buffer to end up on the /caller's/ delwri list, not any + * old list. This can happen if the buffer is marked stale (which + * clears DELWRI_Q) after the AIL queues the buffer to its list but + * before the AIL has a chance to submit the list. + */ + while (!list_empty(&bp->b_list)) { + xfs_buf_unlock(bp); + wait_var_event(&bp->b_list, list_empty(&bp->b_list)); + xfs_buf_lock(bp); + } + + ASSERT(!(bp->b_flags & _XBF_DELWRI_Q)); + + xfs_buf_delwri_queue(bp, buffer_list); +} + +/* + * Compare function is more complex than it needs to be because + * the return value is only 32 bits and we are doing comparisons + * on 64 bit values + */ +static int +xfs_buf_cmp( + void *priv, + const struct list_head *a, + const struct list_head *b) +{ + struct xfs_buf *ap = container_of(a, struct xfs_buf, b_list); + struct xfs_buf *bp = container_of(b, struct xfs_buf, b_list); + xfs_daddr_t diff; + + diff = ap->b_maps[0].bm_bn - bp->b_maps[0].bm_bn; + if (diff < 0) + return -1; + if (diff > 0) + return 1; + return 0; +} + +static bool +xfs_buf_delwri_submit_prep( + struct xfs_buf *bp) +{ + /* + * Someone else might have written the buffer synchronously or marked it + * stale in the meantime. In that case only the _XBF_DELWRI_Q flag got + * cleared, and we have to drop the reference and remove it from the + * list here. + */ + if (!(bp->b_flags & _XBF_DELWRI_Q)) { + xfs_buf_list_del(bp); + xfs_buf_relse(bp); + return false; + } + + trace_xfs_buf_delwri_split(bp, _RET_IP_); + xfs_buf_clear_flags(bp, _XBF_DELWRI_Q); + xfs_buf_set_flags(bp, XBF_WRITE); + return true; +} + +/* + * Write out a buffer list asynchronously. + * + * This will take the @buffer_list, write all non-locked and non-pinned buffers + * out and not wait for I/O completion on any of the buffers. This interface + * is only safely useable for callers that can track I/O completion by higher + * level means, e.g. AIL pushing as the @buffer_list is consumed in this + * function. + * + * Note: this function will skip buffers it would block on, and in doing so + * leaves them on @buffer_list so they can be retried on a later pass. As such, + * it is up to the caller to ensure that the buffer list is fully submitted or + * cancelled appropriately when they are finished with the list. Failure to + * cancel or resubmit the list until it is empty will result in leaked buffers + * at unmount time. + */ +int +xfs_buf_delwri_submit_nowait( + struct list_head *buffer_list) +{ + struct xfs_buf *bp, *n; + int pinned = 0; + struct blk_plug plug; + + list_sort(NULL, buffer_list, xfs_buf_cmp); + + blk_start_plug(&plug); + list_for_each_entry_safe(bp, n, buffer_list, b_list) { + if (!xfs_buf_trylock(bp)) + continue; + if (xfs_buf_ispinned(bp)) { + xfs_buf_unlock(bp); + pinned++; + continue; + } + if (!xfs_buf_delwri_submit_prep(bp)) + continue; + xfs_buf_set_flags(bp, XBF_ASYNC); + xfs_buf_list_del(bp); + xfs_buf_submit(bp); + } + blk_finish_plug(&plug); + + return pinned; +} + +/* + * Write out a buffer list synchronously. + * + * This will take the @buffer_list, write all buffers out and wait for I/O + * completion on all of the buffers. @buffer_list is consumed by the function, + * so callers must have some other way of tracking buffers if they require such + * functionality. + */ +int +xfs_buf_delwri_submit( + struct list_head *buffer_list) +{ + LIST_HEAD (wait_list); + int error = 0, error2; + struct xfs_buf *bp, *n; + struct blk_plug plug; + + list_sort(NULL, buffer_list, xfs_buf_cmp); + + blk_start_plug(&plug); + list_for_each_entry_safe(bp, n, buffer_list, b_list) { + xfs_buf_lock(bp); + if (!xfs_buf_delwri_submit_prep(bp)) + continue; + xfs_buf_clear_flags(bp, XBF_ASYNC); + list_move_tail(&bp->b_list, &wait_list); + xfs_buf_submit(bp); + } + blk_finish_plug(&plug); + + /* Wait for IO to complete. */ + while (!list_empty(&wait_list)) { + bp = list_first_entry(&wait_list, struct xfs_buf, b_list); + + xfs_buf_list_del(bp); + + /* + * Wait on the locked buffer, check for errors and unlock and + * release the delwri queue reference. + */ + error2 = xfs_buf_iowait(bp); + xfs_buf_relse(bp); + if (!error) + error = error2; + } + + return error; +} + +void xfs_buf_set_ref(struct xfs_buf *bp, int lru_ref) +{ + /* + * Set the lru reference count to 0 based on the error injection tag. + * This allows userspace to disrupt buffer caching for debug/testing + * purposes. + */ + if (XFS_TEST_ERROR(bp->b_mount, XFS_ERRTAG_BUF_LRU_REF)) + lru_ref = 0; + + atomic_set(&bp->b_lru_ref, lru_ref); +} + +/* + * Verify an on-disk magic value against the magic value specified in the + * verifier structure. The verifier magic is in disk byte order so the caller is + * expected to pass the value directly from disk. + */ +bool +xfs_verify_magic( + struct xfs_buf *bp, + __be32 dmagic) +{ + struct xfs_mount *mp = bp->b_mount; + int idx; + + idx = xfs_has_crc(mp); + if (WARN_ON(!bp->b_ops || !bp->b_ops->magic[idx])) + return false; + return dmagic == bp->b_ops->magic[idx]; +} +/* + * Verify an on-disk magic value against the magic value specified in the + * verifier structure. The verifier magic is in disk byte order so the caller is + * expected to pass the value directly from disk. + */ +bool +xfs_verify_magic16( + struct xfs_buf *bp, + __be16 dmagic) +{ + struct xfs_mount *mp = bp->b_mount; + int idx; + + idx = xfs_has_crc(mp); + if (WARN_ON(!bp->b_ops || !bp->b_ops->magic16[idx])) + return false; + return dmagic == bp->b_ops->magic16[idx]; +} diff --git a/fs/xfs/xfs_buf.h b/fs/xfs/xfs_buf.h new file mode 100644 index 000000000..a4729253b --- /dev/null +++ b/fs/xfs/xfs_buf.h @@ -0,0 +1,374 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_BUF_H__ +#define __XFS_BUF_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern struct kmem_cache *xfs_buf_cache; + +/* + * Base types + */ +struct xfs_buf; + +#define XFS_BUF_DADDR_MAX ((xfs_daddr_t) S64_MAX) +#define XFS_BUF_DADDR_NULL ((xfs_daddr_t) (-1LL)) + +#define XBF_READ (1u << 0) /* buffer intended for reading from device */ +#define XBF_WRITE (1u << 1) /* buffer intended for writing to device */ +#define XBF_READ_AHEAD (1u << 2) /* asynchronous read-ahead */ +#define XBF_ASYNC (1u << 4) /* initiator will not wait for completion */ +#define XBF_DONE (1u << 5) /* all pages in the buffer uptodate */ +#define XBF_STALE (1u << 6) /* buffer has been staled, do not find it */ +#define XBF_WRITE_FAIL (1u << 7) /* async writes have failed on this buffer */ + +/* flags used only internally */ +#define _XBF_KMEM (1u << 21)/* backed by heap memory */ +#define _XBF_DELWRI_Q (1u << 22)/* buffer on a delwri queue */ + +/* flags used only as arguments to access routines */ +/* + * Online fsck is scanning the buffer cache for live buffers. Do not warn + * about length mismatches during lookups and do not return stale buffers. + */ +#define XBF_LIVESCAN (1u << 28) +#define XBF_INCORE (1u << 29)/* lookup only, return if found in cache */ +#define XBF_TRYLOCK (1u << 30)/* lock requested, but do not wait */ + + +typedef unsigned int xfs_buf_flags_t; + +#define XFS_BUF_FLAGS \ + { XBF_READ, "READ" }, \ + { XBF_WRITE, "WRITE" }, \ + { XBF_READ_AHEAD, "READ_AHEAD" }, \ + { XBF_ASYNC, "ASYNC" }, \ + { XBF_DONE, "DONE" }, \ + { XBF_STALE, "STALE" }, \ + { XBF_WRITE_FAIL, "WRITE_FAIL" }, \ + { _XBF_KMEM, "KMEM" }, \ + { _XBF_DELWRI_Q, "DELWRI_Q" }, \ + /* The following interface flags should never be set */ \ + { XBF_LIVESCAN, "LIVESCAN" }, \ + { XBF_INCORE, "INCORE" }, \ + { XBF_TRYLOCK, "TRYLOCK" } + +/* + * The xfs_buftarg contains 2 notions of "sector size" - + * + * 1) The metadata sector size, which is the minimum unit and + * alignment of IO which will be performed by metadata operations. + * 2) The device logical sector size + * + * The first is specified at mkfs time, and is stored on-disk in the + * superblock's sb_sectsize. + * + * The latter is derived from the underlying device, and controls direct IO + * alignment constraints. + */ +struct xfs_buftarg { + dev_t bt_dev; + struct block_device *bt_bdev; + struct dax_device *bt_daxdev; + struct file *bt_file; + u64 bt_dax_part_off; + struct xfs_mount *bt_mount; + unsigned int bt_meta_sectorsize; + size_t bt_meta_sectormask; + size_t bt_logical_sectorsize; + size_t bt_logical_sectormask; + xfs_daddr_t bt_nr_sectors; + + /* LRU control structures */ + struct shrinker *bt_shrinker; + struct list_lru bt_lru; + + struct percpu_counter bt_readahead_count; + struct ratelimit_state bt_ioerror_rl; + + /* Hardware atomic write unit values, bytes */ + unsigned int bt_awu_min; + unsigned int bt_awu_max; + + struct rhashtable bt_hash; +}; + +struct xfs_buf_map { + xfs_daddr_t bm_bn; /* block number for I/O */ + int bm_len; /* size of I/O */ + unsigned int bm_flags; +}; + +/* + * Online fsck is scanning the buffer cache for live buffers. Do not warn + * about length mismatches during lookups and do not return stale buffers. + */ +#define XBM_LIVESCAN (1U << 0) + +#define DEFINE_SINGLE_BUF_MAP(map, blkno, numblk) \ + struct xfs_buf_map (map) = { .bm_bn = (blkno), .bm_len = (numblk) }; + +struct xfs_buf_ops { + char *name; + union { + __be32 magic[2]; /* v4 and v5 on disk magic values */ + __be16 magic16[2]; /* v4 and v5 on disk magic values */ + }; + void (*verify_read)(struct xfs_buf *); + void (*verify_write)(struct xfs_buf *); + xfs_failaddr_t (*verify_struct)(struct xfs_buf *bp); +}; + +struct xfs_buf { + /* + * first cacheline holds all the fields needed for an uncontended cache + * hit to be fully processed. The semaphore straddles the cacheline + * boundary, but the counter and lock sits on the first cacheline, + * which is the only bit that is touched if we hit the semaphore + * fast-path on locking. + */ + struct rhash_head b_rhash_head; /* pag buffer hash node */ + + xfs_daddr_t b_rhash_key; /* buffer cache index */ + int b_length; /* size of buffer in BBs */ + struct lockref b_lockref; /* refcount + lock */ + atomic_t b_lru_ref; /* lru reclaim ref count */ + xfs_buf_flags_t b_flags; /* status flags */ + struct semaphore b_sema; /* semaphore for lockables */ + + /* + * concurrent access to b_lru and b_lru_flags are protected by + * bt_lru_lock and not by b_sema + */ + struct list_head b_lru; /* lru list */ + wait_queue_head_t b_waiters; /* unpin waiters */ + struct list_head b_list; + struct xfs_perag *b_pag; + struct xfs_mount *b_mount; + struct xfs_buftarg *b_target; /* buffer target (device) */ + void *b_addr; /* virtual address of buffer */ + struct work_struct b_ioend_work; + struct completion b_iowait; /* queue for I/O waiters */ + struct xfs_buf_log_item *b_log_item; + struct list_head b_li_list; /* Log items list head */ + struct xfs_trans *b_transp; + struct xfs_buf_map *b_maps; /* compound buffer map */ + struct xfs_buf_map __b_map; /* inline compound buffer map */ + int b_map_count; + atomic_t b_pin_count; /* pin count */ + int b_error; /* error code on I/O */ + void (*b_iodone)(struct xfs_buf *bp); + + /* + * async write failure retry count. Initialised to zero on the first + * failure, then when it exceeds the maximum configured without a + * success the write is considered to be failed permanently and the + * iodone handler will take appropriate action. + * + * For retry timeouts, we record the jiffy of the first failure. This + * means that we can change the retry timeout for buffers already under + * I/O and thus avoid getting stuck in a retry loop with a long timeout. + * + * last_error is used to ensure that we are getting repeated errors, not + * different errors. e.g. a block device might change ENOSPC to EIO when + * a failure timeout occurs, so we want to re-initialise the error + * retry behaviour appropriately when that happens. + */ + int b_retries; + unsigned long b_first_retry_time; /* in jiffies */ + int b_last_error; + + const struct xfs_buf_ops *b_ops; + struct rcu_head b_rcu; +}; + +/* Finding and Reading Buffers */ +int xfs_buf_get_map(struct xfs_buftarg *target, struct xfs_buf_map *map, + int nmaps, xfs_buf_flags_t flags, struct xfs_buf **bpp); +int xfs_buf_read_map(struct xfs_buftarg *target, struct xfs_buf_map *map, + int nmaps, xfs_buf_flags_t flags, struct xfs_buf **bpp, + const struct xfs_buf_ops *ops, xfs_failaddr_t fa); +void xfs_buf_readahead_map(struct xfs_buftarg *target, + struct xfs_buf_map *map, int nmaps, + const struct xfs_buf_ops *ops); + +static inline int +xfs_buf_incore( + struct xfs_buftarg *target, + xfs_daddr_t blkno, + size_t numblks, + xfs_buf_flags_t flags, + struct xfs_buf **bpp) +{ + DEFINE_SINGLE_BUF_MAP(map, blkno, numblks); + + return xfs_buf_get_map(target, &map, 1, XBF_INCORE | flags, bpp); +} + +static inline int +xfs_buf_get( + struct xfs_buftarg *target, + xfs_daddr_t blkno, + size_t numblks, + struct xfs_buf **bpp) +{ + DEFINE_SINGLE_BUF_MAP(map, blkno, numblks); + + return xfs_buf_get_map(target, &map, 1, 0, bpp); +} + +static inline int +xfs_buf_read( + struct xfs_buftarg *target, + xfs_daddr_t blkno, + size_t numblks, + xfs_buf_flags_t flags, + struct xfs_buf **bpp, + const struct xfs_buf_ops *ops) +{ + DEFINE_SINGLE_BUF_MAP(map, blkno, numblks); + + return xfs_buf_read_map(target, &map, 1, flags, bpp, ops, + __builtin_return_address(0)); +} + +static inline void +xfs_buf_readahead( + struct xfs_buftarg *target, + xfs_daddr_t blkno, + size_t numblks, + const struct xfs_buf_ops *ops) +{ + DEFINE_SINGLE_BUF_MAP(map, blkno, numblks); + return xfs_buf_readahead_map(target, &map, 1, ops); +} + +int xfs_buf_get_uncached(struct xfs_buftarg *target, size_t numblks, + struct xfs_buf **bpp); +int xfs_buf_read_uncached(struct xfs_buftarg *target, xfs_daddr_t daddr, + size_t numblks, struct xfs_buf **bpp, + const struct xfs_buf_ops *ops); +int _xfs_buf_read(struct xfs_buf *bp); +void xfs_buf_hold(struct xfs_buf *bp); + +/* Releasing Buffers */ +extern void xfs_buf_rele(struct xfs_buf *); + +/* Locking and Unlocking Buffers */ +extern int xfs_buf_trylock(struct xfs_buf *); +extern void xfs_buf_lock(struct xfs_buf *); +extern void xfs_buf_unlock(struct xfs_buf *); +#define xfs_buf_islocked(bp) \ + ((bp)->b_sema.count <= 0) + +static inline void xfs_buf_relse(struct xfs_buf *bp) +{ + xfs_buf_unlock(bp); + xfs_buf_rele(bp); +} + +/* Buffer Read and Write Routines */ +extern int xfs_bwrite(struct xfs_buf *bp); + +extern void __xfs_buf_ioerror(struct xfs_buf *bp, int error, + xfs_failaddr_t failaddr); +#define xfs_buf_ioerror(bp, err) __xfs_buf_ioerror((bp), (err), __this_address) +extern void xfs_buf_ioerror_alert(struct xfs_buf *bp, xfs_failaddr_t fa); +void xfs_buf_fail(struct xfs_buf *bp); +void __xfs_buf_mark_corrupt(struct xfs_buf *bp, xfs_failaddr_t fa); +#define xfs_buf_mark_corrupt(bp) __xfs_buf_mark_corrupt((bp), __this_address) + +/* Buffer Utility Routines */ +static inline void *xfs_buf_offset(struct xfs_buf *bp, size_t offset) +{ + return bp->b_addr + offset; +} + +static inline void xfs_buf_zero(struct xfs_buf *bp, size_t boff, size_t bsize) +{ + memset(bp->b_addr + boff, 0, bsize); +} + +void xfs_buf_set_uptodate(struct xfs_buf *bp); +void xfs_buf_stale(struct xfs_buf *bp); +void xfs_buf_clear_stale(struct xfs_buf *bp); + +/* Delayed Write Buffer Routines */ +extern void xfs_buf_delwri_cancel(struct list_head *); +extern bool xfs_buf_delwri_queue(struct xfs_buf *, struct list_head *); +void xfs_buf_delwri_queue_here(struct xfs_buf *bp, struct list_head *bl); +extern int xfs_buf_delwri_submit(struct list_head *); +extern int xfs_buf_delwri_submit_nowait(struct list_head *); + +static inline xfs_daddr_t xfs_buf_daddr(struct xfs_buf *bp) +{ + return bp->b_maps[0].bm_bn; +} + +void xfs_buf_set_ref(struct xfs_buf *bp, int lru_ref); + +/* + * If the buffer is already on the LRU, do nothing. Otherwise set the buffer + * up with a reference count of 0 so it will be tossed from the cache when + * released. + */ +static inline void xfs_buf_oneshot(struct xfs_buf *bp) +{ + if (!list_empty(&bp->b_lru) || atomic_read(&bp->b_lru_ref) > 1) + return; + atomic_set(&bp->b_lru_ref, 0); +} + +static inline int xfs_buf_ispinned(struct xfs_buf *bp) +{ + return atomic_read(&bp->b_pin_count); +} + +static inline int +xfs_buf_verify_cksum(struct xfs_buf *bp, unsigned long cksum_offset) +{ + return xfs_verify_cksum(bp->b_addr, BBTOB(bp->b_length), + cksum_offset); +} + +static inline void +xfs_buf_update_cksum(struct xfs_buf *bp, unsigned long cksum_offset) +{ + xfs_update_cksum(bp->b_addr, BBTOB(bp->b_length), + cksum_offset); +} + +/* + * Handling of buftargs. + */ +struct xfs_buftarg *xfs_alloc_buftarg(struct xfs_mount *mp, + struct file *bdev_file); +extern void xfs_free_buftarg(struct xfs_buftarg *); +extern void xfs_buftarg_wait(struct xfs_buftarg *); +extern void xfs_buftarg_drain(struct xfs_buftarg *); +int xfs_configure_buftarg(struct xfs_buftarg *btp, unsigned int sectorsize, + xfs_fsblock_t nr_blocks); + +#define xfs_readonly_buftarg(buftarg) bdev_read_only((buftarg)->bt_bdev) + +bool xfs_verify_magic(struct xfs_buf *bp, __be32 dmagic); +bool xfs_verify_magic16(struct xfs_buf *bp, __be16 dmagic); + +/* for xfs_buf_mem.c only: */ +int xfs_init_buftarg(struct xfs_buftarg *btp, size_t logical_sectorsize, + const char *descr); +void xfs_destroy_buftarg(struct xfs_buftarg *btp); + +#endif /* __XFS_BUF_H__ */ diff --git a/fs/xfs/xfs_buf_item.c b/fs/xfs/xfs_buf_item.c new file mode 100644 index 000000000..1a4ef34af --- /dev/null +++ b/fs/xfs/xfs_buf_item.c @@ -0,0 +1,1087 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_mount.h" +#include "xfs_trans.h" +#include "xfs_trans_priv.h" +#include "xfs_buf_item.h" +#include "xfs_inode.h" +#include "xfs_inode_item.h" +#include "xfs_quota.h" +#include "xfs_dquot_item.h" +#include "xfs_dquot.h" +#include "xfs_trace.h" +#include "xfs_log.h" +#include "xfs_log_priv.h" +#include "xfs_error.h" + + +struct kmem_cache *xfs_buf_item_cache; + +static inline struct xfs_buf_log_item *BUF_ITEM(struct xfs_log_item *lip) +{ + return container_of(lip, struct xfs_buf_log_item, bli_item); +} + +static void +xfs_buf_item_get_format( + struct xfs_buf_log_item *bip, + int count) +{ + ASSERT(bip->bli_formats == NULL); + bip->bli_format_count = count; + + if (count == 1) { + bip->bli_formats = &bip->__bli_format; + return; + } + + bip->bli_formats = kzalloc(count * sizeof(struct xfs_buf_log_format), + GFP_KERNEL | __GFP_NOFAIL); +} + +static void +xfs_buf_item_free_format( + struct xfs_buf_log_item *bip) +{ + if (bip->bli_formats != &bip->__bli_format) { + kfree(bip->bli_formats); + bip->bli_formats = NULL; + } +} + +static void +xfs_buf_item_free( + struct xfs_buf_log_item *bip) +{ + xfs_buf_item_free_format(bip); + kvfree(bip->bli_item.li_lv_shadow); + kmem_cache_free(xfs_buf_item_cache, bip); +} + +/* + * xfs_buf_item_relse() is called when the buf log item is no longer needed. + */ +static void +xfs_buf_item_relse( + struct xfs_buf_log_item *bip) +{ + struct xfs_buf *bp = bip->bli_buf; + + trace_xfs_buf_item_relse(bp, _RET_IP_); + + ASSERT(!test_bit(XFS_LI_IN_AIL, &bip->bli_item.li_flags)); + ASSERT(atomic_read(&bip->bli_refcount) == 0); + + bp->b_log_item = NULL; + xfs_buf_rele(bp); + xfs_buf_item_free(bip); +} + +/* Is this log iovec plausibly large enough to contain the buffer log format? */ +bool +xfs_buf_log_check_iovec( + struct kvec *iovec) +{ + struct xfs_buf_log_format *blfp = iovec->iov_base; + char *bmp_end; + char *item_end; + + if (offsetof(struct xfs_buf_log_format, blf_data_map) > iovec->iov_len) + return false; + + item_end = (char *)iovec->iov_base + iovec->iov_len; + bmp_end = (char *)&blfp->blf_data_map[blfp->blf_map_size]; + return bmp_end <= item_end; +} + +static inline int +xfs_buf_log_format_size( + struct xfs_buf_log_format *blfp) +{ + return offsetof(struct xfs_buf_log_format, blf_data_map) + + (blfp->blf_map_size * sizeof(blfp->blf_data_map[0])); +} + +/* + * Return the number of log iovecs and space needed to log the given buf log + * item segment. + * + * It calculates this as 1 iovec for the buf log format structure and 1 for each + * stretch of non-contiguous chunks to be logged. Contiguous chunks are logged + * in a single iovec. + */ +STATIC void +xfs_buf_item_size_segment( + struct xfs_buf_log_item *bip, + struct xfs_buf_log_format *blfp, + uint offset, + int *nvecs, + int *nbytes) +{ + int first_bit; + int nbits; + + first_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size, 0); + if (first_bit == -1) + return; + + (*nvecs)++; + *nbytes += xfs_buf_log_format_size(blfp); + + do { + nbits = xfs_contig_bits(blfp->blf_data_map, + blfp->blf_map_size, first_bit); + ASSERT(nbits > 0); + (*nvecs)++; + *nbytes += nbits * XFS_BLF_CHUNK; + + /* + * This takes the bit number to start looking from and + * returns the next set bit from there. It returns -1 + * if there are no more bits set or the start bit is + * beyond the end of the bitmap. + */ + first_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size, + (uint)first_bit + nbits + 1); + } while (first_bit != -1); + + return; +} + +/* + * Compute the worst case log item overhead for an invalidated buffer with the + * given map count and block size. + */ +unsigned int +xfs_buf_inval_log_space( + unsigned int map_count, + unsigned int blocksize) +{ + unsigned int chunks = DIV_ROUND_UP(blocksize, XFS_BLF_CHUNK); + unsigned int bitmap_size = DIV_ROUND_UP(chunks, NBWORD); + unsigned int ret = + offsetof(struct xfs_buf_log_format, blf_data_map) + + (bitmap_size * sizeof_field(struct xfs_buf_log_format, + blf_data_map[0])); + + return ret * map_count; +} + +/* + * Return the number of log iovecs and space needed to log the given buf log + * item. + * + * Discontiguous buffers need a format structure per region that is being + * logged. This makes the changes in the buffer appear to log recovery as though + * they came from separate buffers, just like would occur if multiple buffers + * were used instead of a single discontiguous buffer. This enables + * discontiguous buffers to be in-memory constructs, completely transparent to + * what ends up on disk. + * + * If the XFS_BLI_STALE flag has been set, then log nothing but the buf log + * format structures. If the item has previously been logged and has dirty + * regions, we do not relog them in stale buffers. This has the effect of + * reducing the size of the relogged item by the amount of dirty data tracked + * by the log item. This can result in the committing transaction reducing the + * amount of space being consumed by the CIL. + */ +STATIC void +xfs_buf_item_size( + struct xfs_log_item *lip, + int *nvecs, + int *nbytes) +{ + struct xfs_buf_log_item *bip = BUF_ITEM(lip); + struct xfs_buf *bp = bip->bli_buf; + int i; + int bytes; + uint offset = 0; + + ASSERT(atomic_read(&bip->bli_refcount) > 0); + if (bip->bli_flags & XFS_BLI_STALE) { + /* + * The buffer is stale, so all we need to log is the buf log + * format structure with the cancel flag in it as we are never + * going to replay the changes tracked in the log item. + */ + trace_xfs_buf_item_size_stale(bip); + ASSERT(bip->__bli_format.blf_flags & XFS_BLF_CANCEL); + *nvecs += bip->bli_format_count; + for (i = 0; i < bip->bli_format_count; i++) { + *nbytes += xfs_buf_log_format_size(&bip->bli_formats[i]); + } + return; + } + + ASSERT(bip->bli_flags & XFS_BLI_LOGGED); + + if (bip->bli_flags & XFS_BLI_ORDERED) { + /* + * The buffer has been logged just to order it. It is not being + * included in the transaction commit, so no vectors are used at + * all. + */ + trace_xfs_buf_item_size_ordered(bip); + *nvecs = XFS_LOG_VEC_ORDERED; + return; + } + + /* + * The vector count is based on the number of buffer vectors we have + * dirty bits in. This will only be greater than one when we have a + * compound buffer with more than one segment dirty. Hence for compound + * buffers we need to track which segment the dirty bits correspond to, + * and when we move from one segment to the next increment the vector + * count for the extra buf log format structure that will need to be + * written. + */ + bytes = 0; + for (i = 0; i < bip->bli_format_count; i++) { + xfs_buf_item_size_segment(bip, &bip->bli_formats[i], offset, + nvecs, &bytes); + offset += BBTOB(bp->b_maps[i].bm_len); + } + + /* + * Round up the buffer size required to minimise the number of memory + * allocations that need to be done as this item grows when relogged by + * repeated modifications. + */ + *nbytes = round_up(bytes, 512); + trace_xfs_buf_item_size(bip); +} + +static inline void +xfs_buf_item_copy_iovec( + struct xlog_format_buf *lfb, + struct xfs_buf *bp, + uint offset, + int first_bit, + uint nbits) +{ + offset += first_bit * XFS_BLF_CHUNK; + xlog_format_copy(lfb, XLOG_REG_TYPE_BCHUNK, xfs_buf_offset(bp, offset), + nbits * XFS_BLF_CHUNK); +} + +static void +xfs_buf_item_format_segment( + struct xfs_buf_log_item *bip, + struct xlog_format_buf *lfb, + uint offset, + struct xfs_buf_log_format *blfp) +{ + struct xfs_buf *bp = bip->bli_buf; + uint base_size; + int first_bit; + uint nbits; + + /* copy the flags across from the base format item */ + blfp->blf_flags = bip->__bli_format.blf_flags; + + /* + * Base size is the actual size of the ondisk structure - it reflects + * the actual size of the dirty bitmap rather than the size of the in + * memory structure. + */ + base_size = xfs_buf_log_format_size(blfp); + + first_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size, 0); + if (!(bip->bli_flags & XFS_BLI_STALE) && first_bit == -1) { + /* + * If the map is not be dirty in the transaction, mark + * the size as zero and do not advance the vector pointer. + */ + return; + } + + blfp = xlog_format_copy(lfb, XLOG_REG_TYPE_BFORMAT, blfp, base_size); + blfp->blf_size = 1; + + if (bip->bli_flags & XFS_BLI_STALE) { + /* + * The buffer is stale, so all we need to log + * is the buf log format structure with the + * cancel flag in it. + */ + trace_xfs_buf_item_format_stale(bip); + ASSERT(blfp->blf_flags & XFS_BLF_CANCEL); + return; + } + + + /* + * Fill in an iovec for each set of contiguous chunks. + */ + do { + ASSERT(first_bit >= 0); + nbits = xfs_contig_bits(blfp->blf_data_map, + blfp->blf_map_size, first_bit); + ASSERT(nbits > 0); + xfs_buf_item_copy_iovec(lfb, bp, offset, first_bit, nbits); + blfp->blf_size++; + + /* + * This takes the bit number to start looking from and + * returns the next set bit from there. It returns -1 + * if there are no more bits set or the start bit is + * beyond the end of the bitmap. + */ + first_bit = xfs_next_bit(blfp->blf_data_map, blfp->blf_map_size, + (uint)first_bit + nbits + 1); + } while (first_bit != -1); + + return; +} + +/* + * This is called to fill in the vector of log iovecs for the + * given log buf item. It fills the first entry with a buf log + * format structure, and the rest point to contiguous chunks + * within the buffer. + */ +STATIC void +xfs_buf_item_format( + struct xfs_log_item *lip, + struct xlog_format_buf *lfb) +{ + struct xfs_buf_log_item *bip = BUF_ITEM(lip); + struct xfs_buf *bp = bip->bli_buf; + uint offset = 0; + int i; + + ASSERT(atomic_read(&bip->bli_refcount) > 0); + ASSERT((bip->bli_flags & XFS_BLI_LOGGED) || + (bip->bli_flags & XFS_BLI_STALE)); + ASSERT((bip->bli_flags & XFS_BLI_STALE) || + (xfs_blft_from_flags(&bip->__bli_format) > XFS_BLFT_UNKNOWN_BUF + && xfs_blft_from_flags(&bip->__bli_format) < XFS_BLFT_MAX_BUF)); + ASSERT(!(bip->bli_flags & XFS_BLI_ORDERED) || + (bip->bli_flags & XFS_BLI_STALE)); + + + /* + * If it is an inode buffer, transfer the in-memory state to the + * format flags and clear the in-memory state. + * + * For buffer based inode allocation, we do not transfer + * this state if the inode buffer allocation has not yet been committed + * to the log as setting the XFS_BLI_INODE_BUF flag will prevent + * correct replay of the inode allocation. + * + * For icreate item based inode allocation, the buffers aren't written + * to the journal during allocation, and hence we should always tag the + * buffer as an inode buffer so that the correct unlinked list replay + * occurs during recovery. + */ + if (bip->bli_flags & XFS_BLI_INODE_BUF) { + if (xfs_has_v3inodes(lip->li_log->l_mp) || + !((bip->bli_flags & XFS_BLI_INODE_ALLOC_BUF) && + xfs_log_item_in_current_chkpt(lip))) + bip->__bli_format.blf_flags |= XFS_BLF_INODE_BUF; + bip->bli_flags &= ~XFS_BLI_INODE_BUF; + } + + for (i = 0; i < bip->bli_format_count; i++) { + xfs_buf_item_format_segment(bip, lfb, offset, + &bip->bli_formats[i]); + offset += BBTOB(bp->b_maps[i].bm_len); + } + + /* + * Check to make sure everything is consistent. + */ + trace_xfs_buf_item_format(bip); +} + +/* + * This is called to pin the buffer associated with the buf log item in memory + * so it cannot be written out. + * + * We take a reference to the buffer log item here so that the BLI life cycle + * extends at least until the buffer is unpinned via xfs_buf_item_unpin() and + * inserted into the AIL. + * + * We also need to take a reference to the buffer itself as the BLI unpin + * processing requires accessing the buffer after the BLI has dropped the final + * BLI reference. See xfs_buf_item_unpin() for an explanation. + * If unpins race to drop the final BLI reference and only the + * BLI owns a reference to the buffer, then the loser of the race can have the + * buffer fgreed from under it (e.g. on shutdown). Taking a buffer reference per + * pin count ensures the life cycle of the buffer extends for as + * long as we hold the buffer pin reference in xfs_buf_item_unpin(). + */ +STATIC void +xfs_buf_item_pin( + struct xfs_log_item *lip) +{ + struct xfs_buf_log_item *bip = BUF_ITEM(lip); + + ASSERT(atomic_read(&bip->bli_refcount) > 0); + ASSERT((bip->bli_flags & XFS_BLI_LOGGED) || + (bip->bli_flags & XFS_BLI_ORDERED) || + (bip->bli_flags & XFS_BLI_STALE)); + + trace_xfs_buf_item_pin(bip); + + xfs_buf_hold(bip->bli_buf); + atomic_inc(&bip->bli_refcount); + atomic_inc(&bip->bli_buf->b_pin_count); +} + +/* + * For a stale BLI, process all the necessary completions that must be + * performed when the final BLI reference goes away. The buffer will be + * referenced and locked here - we return to the caller with the buffer still + * referenced and locked for them to finalise processing of the buffer. + */ +static void +xfs_buf_item_finish_stale( + struct xfs_buf_log_item *bip) +{ + struct xfs_buf *bp = bip->bli_buf; + struct xfs_log_item *lip = &bip->bli_item; + + ASSERT(bip->bli_flags & XFS_BLI_STALE); + ASSERT(xfs_buf_islocked(bp)); + ASSERT(bp->b_flags & XBF_STALE); + ASSERT(bip->__bli_format.blf_flags & XFS_BLF_CANCEL); + ASSERT(list_empty(&lip->li_trans)); + ASSERT(!bp->b_transp); + + if (bip->bli_flags & XFS_BLI_STALE_INODE) { + xfs_buf_item_done(bp); + xfs_buf_inode_iodone(bp); + ASSERT(list_empty(&bp->b_li_list)); + return; + } + + /* + * We may or may not be on the AIL here, xfs_trans_ail_delete() will do + * the right thing regardless of the situation in which we are called. + */ + xfs_trans_ail_delete(lip, SHUTDOWN_LOG_IO_ERROR); + xfs_buf_item_relse(bip); + ASSERT(bp->b_log_item == NULL); +} + +/* + * This is called to unpin the buffer associated with the buf log item which was + * previously pinned with a call to xfs_buf_item_pin(). We enter this function + * with a buffer pin count, a buffer reference and a BLI reference. + * + * We must drop the BLI reference before we unpin the buffer because the AIL + * doesn't acquire a BLI reference whenever it accesses it. Therefore if the + * refcount drops to zero, the bli could still be AIL resident and the buffer + * submitted for I/O at any point before we return. This can result in IO + * completion freeing the buffer while we are still trying to access it here. + * This race condition can also occur in shutdown situations where we abort and + * unpin buffers from contexts other that journal IO completion. + * + * Hence we have to hold a buffer reference per pin count to ensure that the + * buffer cannot be freed until we have finished processing the unpin operation. + * The reference is taken in xfs_buf_item_pin(), and we must hold it until we + * are done processing the buffer state. In the case of an abort (remove = + * true) then we re-use the current pin reference as the IO reference we hand + * off to IO failure handling. + */ +STATIC void +xfs_buf_item_unpin( + struct xfs_log_item *lip, + int remove) +{ + struct xfs_buf_log_item *bip = BUF_ITEM(lip); + struct xfs_buf *bp = bip->bli_buf; + int stale = bip->bli_flags & XFS_BLI_STALE; + int freed; + + ASSERT(bp->b_log_item == bip); + ASSERT(atomic_read(&bip->bli_refcount) > 0); + + trace_xfs_buf_item_unpin(bip); + + freed = atomic_dec_and_test(&bip->bli_refcount); + if (atomic_dec_and_test(&bp->b_pin_count)) + wake_up_all(&bp->b_waiters); + + /* + * Nothing to do but drop the buffer pin reference if the BLI is + * still active. + */ + if (!freed) { + xfs_buf_rele(bp); + return; + } + + if (stale) { + trace_xfs_buf_item_unpin_stale(bip); + + /* + * The buffer has been locked and referenced since it was marked + * stale so we own both lock and reference exclusively here. We + * do not need the pin reference any more, so drop it now so + * that we only have one reference to drop once item completion + * processing is complete. + */ + xfs_buf_rele(bp); + xfs_buf_item_finish_stale(bip); + xfs_buf_relse(bp); + return; + } + + if (remove) { + /* + * We need to simulate an async IO failures here to ensure that + * the correct error completion is run on this buffer. This + * requires a reference to the buffer and for the buffer to be + * locked. We can safely pass ownership of the pin reference to + * the IO to ensure that nothing can free the buffer while we + * wait for the lock and then run the IO failure completion. + */ + xfs_buf_lock(bp); + xfs_buf_fail(bp); + return; + } + + /* + * BLI has no more active references - it will be moved to the AIL to + * manage the remaining BLI/buffer life cycle. There is nothing left for + * us to do here so drop the pin reference to the buffer. + */ + xfs_buf_rele(bp); +} + +STATIC uint +xfs_buf_item_push( + struct xfs_log_item *lip, + struct list_head *buffer_list) +{ + struct xfs_buf_log_item *bip = BUF_ITEM(lip); + struct xfs_buf *bp = bip->bli_buf; + uint rval = XFS_ITEM_SUCCESS; + + if (xfs_buf_ispinned(bp)) + return XFS_ITEM_PINNED; + if (!xfs_buf_trylock(bp)) { + /* + * If we have just raced with a buffer being pinned and it has + * been marked stale, we could end up stalling until someone else + * issues a log force to unpin the stale buffer. Check for the + * race condition here so xfsaild recognizes the buffer is pinned + * and queues a log force to move it along. + */ + if (xfs_buf_ispinned(bp)) + return XFS_ITEM_PINNED; + return XFS_ITEM_LOCKED; + } + + ASSERT(!(bip->bli_flags & XFS_BLI_STALE)); + + trace_xfs_buf_item_push(bip); + + /* has a previous flush failed due to IO errors? */ + if (bp->b_flags & XBF_WRITE_FAIL) { + xfs_buf_alert_ratelimited(bp, "XFS: Failing async write", + "Failing async write on buffer block 0x%llx. Retrying async write.", + (long long)xfs_buf_daddr(bp)); + } + + if (!xfs_buf_delwri_queue(bp, buffer_list)) + rval = XFS_ITEM_FLUSHING; + xfs_buf_unlock(bp); + return rval; +} + +/* + * Drop the buffer log item refcount and take appropriate action. This helper + * determines whether the bli must be freed or not, since a decrement to zero + * does not necessarily mean the bli is unused. + */ +void +xfs_buf_item_put( + struct xfs_buf_log_item *bip) +{ + + ASSERT(xfs_buf_islocked(bip->bli_buf)); + + /* drop the bli ref and return if it wasn't the last one */ + if (!atomic_dec_and_test(&bip->bli_refcount)) + return; + + /* If the BLI is in the AIL, then it is still dirty and in use */ + if (test_bit(XFS_LI_IN_AIL, &bip->bli_item.li_flags)) { + ASSERT(bip->bli_flags & XFS_BLI_DIRTY); + return; + } + + /* + * In shutdown conditions, we can be asked to free a dirty BLI that + * isn't in the AIL. This can occur due to a checkpoint aborting a BLI + * instead of inserting it into the AIL at checkpoint IO completion. If + * there's another bli reference (e.g. a btree cursor holds a clean + * reference) and it is released via xfs_trans_brelse(), we can get here + * with that aborted, dirty BLI. In this case, it is safe to free the + * dirty BLI immediately, as it is not in the AIL and there are no + * other references to it. + * + * We should never get here with a stale BLI via that path as + * xfs_trans_brelse() specifically holds onto stale buffers rather than + * releasing them. + */ + ASSERT(!(bip->bli_flags & XFS_BLI_DIRTY) || + test_bit(XFS_LI_ABORTED, &bip->bli_item.li_flags)); + ASSERT(!(bip->bli_flags & XFS_BLI_STALE)); + xfs_buf_item_relse(bip); +} + +/* + * Release the buffer associated with the buf log item. If there is no dirty + * logged data associated with the buffer recorded in the buf log item, then + * free the buf log item and remove the reference to it in the buffer. + * + * This call ignores the recursion count. It is only called when the buffer + * should REALLY be unlocked, regardless of the recursion count. + * + * We unconditionally drop the transaction's reference to the log item. If the + * item was logged, then another reference was taken when it was pinned, so we + * can safely drop the transaction reference now. This also allows us to avoid + * potential races with the unpin code freeing the bli by not referencing the + * bli after we've dropped the reference count. + * + * If the XFS_BLI_HOLD flag is set in the buf log item, then free the log item + * if necessary but do not unlock the buffer. This is for support of + * xfs_trans_bhold(). Make sure the XFS_BLI_HOLD field is cleared if we don't + * free the item. + * + * If the XFS_BLI_STALE flag is set, the last reference to the BLI *must* + * perform a completion abort of any objects attached to the buffer for IO + * tracking purposes. This generally only happens in shutdown situations, + * normally xfs_buf_item_unpin() will drop the last BLI reference and perform + * completion processing. However, because transaction completion can race with + * checkpoint completion during a shutdown, this release context may end up + * being the last active reference to the BLI and so needs to perform this + * cleanup. + */ +STATIC void +xfs_buf_item_release( + struct xfs_log_item *lip) +{ + struct xfs_buf_log_item *bip = BUF_ITEM(lip); + struct xfs_buf *bp = bip->bli_buf; + bool hold = bip->bli_flags & XFS_BLI_HOLD; + bool stale = bip->bli_flags & XFS_BLI_STALE; + bool aborted = test_bit(XFS_LI_ABORTED, + &lip->li_flags); + bool dirty = bip->bli_flags & XFS_BLI_DIRTY; +#if defined(DEBUG) || defined(XFS_WARN) + bool ordered = bip->bli_flags & XFS_BLI_ORDERED; +#endif + + trace_xfs_buf_item_release(bip); + + ASSERT(xfs_buf_islocked(bp)); + + /* + * The bli dirty state should match whether the blf has logged segments + * except for ordered buffers, where only the bli should be dirty. + */ + ASSERT((!ordered && dirty == xfs_buf_item_dirty_format(bip)) || + (ordered && dirty && !xfs_buf_item_dirty_format(bip))); + ASSERT(!stale || (bip->__bli_format.blf_flags & XFS_BLF_CANCEL)); + + /* + * Clear the buffer's association with this transaction and + * per-transaction state from the bli, which has been copied above. + */ + bp->b_transp = NULL; + bip->bli_flags &= ~(XFS_BLI_LOGGED | XFS_BLI_HOLD | XFS_BLI_ORDERED); + + /* If there are other references, then we have nothing to do. */ + if (!atomic_dec_and_test(&bip->bli_refcount)) + goto out_release; + + /* + * Stale buffer completion frees the BLI, unlocks and releases the + * buffer. Neither the BLI or buffer are safe to reference after this + * call, so there's nothing more we need to do here. + * + * If we get here with a stale buffer and references to the BLI remain, + * we must not unlock the buffer as the last BLI reference owns lock + * context, not us. + */ + if (stale) { + xfs_buf_item_finish_stale(bip); + xfs_buf_relse(bp); + ASSERT(!hold); + return; + } + + /* + * Dirty or clean, aborted items are done and need to be removed from + * the AIL and released. This frees the BLI, but leaves the buffer + * locked and referenced. + */ + if (aborted || xlog_is_shutdown(lip->li_log)) { + ASSERT(list_empty(&bip->bli_buf->b_li_list)); + xfs_buf_item_done(bp); + goto out_release; + } + + /* + * Clean, unreferenced BLIs can be immediately freed, leaving the buffer + * locked and referenced. + * + * Dirty, unreferenced BLIs *must* be in the AIL awaiting writeback. + */ + if (!dirty) + xfs_buf_item_relse(bip); + else + ASSERT(test_bit(XFS_LI_IN_AIL, &lip->li_flags)); + + /* Not safe to reference the BLI from here */ +out_release: + /* + * If we get here with a stale buffer, we must not unlock the + * buffer as the last BLI reference owns lock context, not us. + */ + if (stale || hold) + return; + xfs_buf_relse(bp); +} + +STATIC void +xfs_buf_item_committing( + struct xfs_log_item *lip, + xfs_csn_t seq) +{ + return xfs_buf_item_release(lip); +} + +/* + * This is called to find out where the oldest active copy of the + * buf log item in the on disk log resides now that the last log + * write of it completed at the given lsn. + * We always re-log all the dirty data in a buffer, so usually the + * latest copy in the on disk log is the only one that matters. For + * those cases we simply return the given lsn. + * + * The one exception to this is for buffers full of newly allocated + * inodes. These buffers are only relogged with the XFS_BLI_INODE_BUF + * flag set, indicating that only the di_next_unlinked fields from the + * inodes in the buffers will be replayed during recovery. If the + * original newly allocated inode images have not yet been flushed + * when the buffer is so relogged, then we need to make sure that we + * keep the old images in the 'active' portion of the log. We do this + * by returning the original lsn of that transaction here rather than + * the current one. + */ +STATIC xfs_lsn_t +xfs_buf_item_committed( + struct xfs_log_item *lip, + xfs_lsn_t lsn) +{ + struct xfs_buf_log_item *bip = BUF_ITEM(lip); + + trace_xfs_buf_item_committed(bip); + + if ((bip->bli_flags & XFS_BLI_INODE_ALLOC_BUF) && lip->li_lsn != 0) + return lip->li_lsn; + return lsn; +} + +#ifdef DEBUG_EXPENSIVE +static int +xfs_buf_item_precommit( + struct xfs_trans *tp, + struct xfs_log_item *lip) +{ + struct xfs_buf_log_item *bip = BUF_ITEM(lip); + struct xfs_buf *bp = bip->bli_buf; + struct xfs_mount *mp = bp->b_mount; + xfs_failaddr_t fa; + + if (!bp->b_ops || !bp->b_ops->verify_struct) + return 0; + if (bip->bli_flags & XFS_BLI_STALE) + return 0; + + fa = bp->b_ops->verify_struct(bp); + if (fa) { + xfs_buf_verifier_error(bp, -EFSCORRUPTED, bp->b_ops->name, + bp->b_addr, BBTOB(bp->b_length), fa); + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); + ASSERT(fa == NULL); + } + + return 0; +} +#else +# define xfs_buf_item_precommit NULL +#endif + +static const struct xfs_item_ops xfs_buf_item_ops = { + .iop_size = xfs_buf_item_size, + .iop_precommit = xfs_buf_item_precommit, + .iop_format = xfs_buf_item_format, + .iop_pin = xfs_buf_item_pin, + .iop_unpin = xfs_buf_item_unpin, + .iop_release = xfs_buf_item_release, + .iop_committing = xfs_buf_item_committing, + .iop_committed = xfs_buf_item_committed, + .iop_push = xfs_buf_item_push, +}; + +/* + * Allocate a new buf log item to go with the given buffer. + * Set the buffer's b_log_item field to point to the new + * buf log item. + */ +int +xfs_buf_item_init( + struct xfs_buf *bp, + struct xfs_mount *mp) +{ + struct xfs_buf_log_item *bip = bp->b_log_item; + int chunks; + int map_size; + int i; + + /* + * Check to see if there is already a buf log item for + * this buffer. If we do already have one, there is + * nothing to do here so return. + */ + ASSERT(bp->b_mount == mp); + if (bip) { + ASSERT(bip->bli_item.li_type == XFS_LI_BUF); + ASSERT(!bp->b_transp); + ASSERT(bip->bli_buf == bp); + return 0; + } + + bip = kmem_cache_zalloc(xfs_buf_item_cache, GFP_KERNEL | __GFP_NOFAIL); + xfs_log_item_init(mp, &bip->bli_item, XFS_LI_BUF, &xfs_buf_item_ops); + bip->bli_buf = bp; + + /* + * chunks is the number of XFS_BLF_CHUNK size pieces the buffer + * can be divided into. Make sure not to truncate any pieces. + * map_size is the size of the bitmap needed to describe the + * chunks of the buffer. + * + * Discontiguous buffer support follows the layout of the underlying + * buffer. This makes the implementation as simple as possible. + */ + xfs_buf_item_get_format(bip, bp->b_map_count); + + for (i = 0; i < bip->bli_format_count; i++) { + chunks = DIV_ROUND_UP(BBTOB(bp->b_maps[i].bm_len), + XFS_BLF_CHUNK); + map_size = DIV_ROUND_UP(chunks, NBWORD); + + if (map_size > XFS_BLF_DATAMAP_SIZE) { + xfs_buf_item_free_format(bip); + kmem_cache_free(xfs_buf_item_cache, bip); + xfs_err(mp, + "buffer item dirty bitmap (%u uints) too small to reflect %u bytes!", + map_size, + BBTOB(bp->b_maps[i].bm_len)); + return -EFSCORRUPTED; + } + + bip->bli_formats[i].blf_type = XFS_LI_BUF; + bip->bli_formats[i].blf_blkno = bp->b_maps[i].bm_bn; + bip->bli_formats[i].blf_len = bp->b_maps[i].bm_len; + bip->bli_formats[i].blf_map_size = map_size; + } + + bp->b_log_item = bip; + xfs_buf_hold(bp); + return 0; +} + + +/* + * Mark bytes first through last inclusive as dirty in the buf + * item's bitmap. + */ +static void +xfs_buf_item_log_segment( + uint first, + uint last, + uint *map) +{ + uint first_bit; + uint last_bit; + uint bits_to_set; + uint bits_set; + uint word_num; + uint *wordp; + uint bit; + uint end_bit; + uint mask; + + ASSERT(first < XFS_BLF_DATAMAP_SIZE * XFS_BLF_CHUNK * NBWORD); + ASSERT(last < XFS_BLF_DATAMAP_SIZE * XFS_BLF_CHUNK * NBWORD); + + /* + * Convert byte offsets to bit numbers. + */ + first_bit = first >> XFS_BLF_SHIFT; + last_bit = last >> XFS_BLF_SHIFT; + + /* + * Calculate the total number of bits to be set. + */ + bits_to_set = last_bit - first_bit + 1; + + /* + * Get a pointer to the first word in the bitmap + * to set a bit in. + */ + word_num = first_bit >> BIT_TO_WORD_SHIFT; + wordp = &map[word_num]; + + /* + * Calculate the starting bit in the first word. + */ + bit = first_bit & (uint)(NBWORD - 1); + + /* + * First set any bits in the first word of our range. + * If it starts at bit 0 of the word, it will be + * set below rather than here. That is what the variable + * bit tells us. The variable bits_set tracks the number + * of bits that have been set so far. End_bit is the number + * of the last bit to be set in this word plus one. + */ + if (bit) { + end_bit = min(bit + bits_to_set, (uint)NBWORD); + mask = ((1U << (end_bit - bit)) - 1) << bit; + *wordp |= mask; + wordp++; + bits_set = end_bit - bit; + } else { + bits_set = 0; + } + + /* + * Now set bits a whole word at a time that are between + * first_bit and last_bit. + */ + while ((bits_to_set - bits_set) >= NBWORD) { + *wordp = 0xffffffff; + bits_set += NBWORD; + wordp++; + } + + /* + * Finally, set any bits left to be set in one last partial word. + */ + end_bit = bits_to_set - bits_set; + if (end_bit) { + mask = (1U << end_bit) - 1; + *wordp |= mask; + } +} + +/* + * Mark bytes first through last inclusive as dirty in the buf + * item's bitmap. + */ +void +xfs_buf_item_log( + struct xfs_buf_log_item *bip, + uint first, + uint last) +{ + int i; + uint start; + uint end; + struct xfs_buf *bp = bip->bli_buf; + + /* + * walk each buffer segment and mark them dirty appropriately. + */ + start = 0; + for (i = 0; i < bip->bli_format_count; i++) { + if (start > last) + break; + end = start + BBTOB(bp->b_maps[i].bm_len) - 1; + + /* skip to the map that includes the first byte to log */ + if (first > end) { + start += BBTOB(bp->b_maps[i].bm_len); + continue; + } + + /* + * Trim the range to this segment and mark it in the bitmap. + * Note that we must convert buffer offsets to segment relative + * offsets (e.g., the first byte of each segment is byte 0 of + * that segment). + */ + if (first < start) + first = start; + if (end > last) + end = last; + xfs_buf_item_log_segment(first - start, end - start, + &bip->bli_formats[i].blf_data_map[0]); + + start += BBTOB(bp->b_maps[i].bm_len); + } +} + + +/* + * Return true if the buffer has any ranges logged/dirtied by a transaction, + * false otherwise. + */ +bool +xfs_buf_item_dirty_format( + struct xfs_buf_log_item *bip) +{ + int i; + + for (i = 0; i < bip->bli_format_count; i++) { + if (!xfs_bitmap_empty(bip->bli_formats[i].blf_data_map, + bip->bli_formats[i].blf_map_size)) + return true; + } + + return false; +} + +void +xfs_buf_item_done( + struct xfs_buf *bp) +{ + struct xfs_buf_log_item *bip = bp->b_log_item; + + /* + * If we are forcibly shutting down, this may well be off the AIL + * already. That's because we simulate the log-committed callbacks to + * unpin these buffers. Or we may never have put this item on AIL + * because of the transaction was aborted forcibly. + * xfs_trans_ail_delete() takes care of these. + * + * Either way, AIL is useless if we're forcing a shutdown. + * + * Note that log recovery writes might have buffer items that are not on + * the AIL even when the file system is not shut down. + */ + xfs_trans_ail_delete(&bip->bli_item, + xlog_in_recovery(bip->bli_item.li_log) ? + 0 : SHUTDOWN_CORRUPT_INCORE); + xfs_buf_item_relse(bip); +} diff --git a/fs/xfs/xfs_buf_item.h b/fs/xfs/xfs_buf_item.h new file mode 100644 index 000000000..3159325dd --- /dev/null +++ b/fs/xfs/xfs_buf_item.h @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2001,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_BUF_ITEM_H__ +#define __XFS_BUF_ITEM_H__ + +/* kernel only definitions */ + +struct xfs_buf; +struct xfs_mount; + +/* buf log item flags */ +#define XFS_BLI_HOLD (1u << 0) +#define XFS_BLI_DIRTY (1u << 1) +#define XFS_BLI_STALE (1u << 2) +#define XFS_BLI_LOGGED (1u << 3) +#define XFS_BLI_INODE_ALLOC_BUF (1u << 4) +#define XFS_BLI_STALE_INODE (1u << 5) +#define XFS_BLI_INODE_BUF (1u << 6) +#define XFS_BLI_ORDERED (1u << 7) + +#define XFS_BLI_FLAGS \ + { XFS_BLI_HOLD, "HOLD" }, \ + { XFS_BLI_DIRTY, "DIRTY" }, \ + { XFS_BLI_STALE, "STALE" }, \ + { XFS_BLI_LOGGED, "LOGGED" }, \ + { XFS_BLI_INODE_ALLOC_BUF, "INODE_ALLOC" }, \ + { XFS_BLI_STALE_INODE, "STALE_INODE" }, \ + { XFS_BLI_INODE_BUF, "INODE_BUF" }, \ + { XFS_BLI_ORDERED, "ORDERED" } + +/* + * This is the in core log item structure used to track information + * needed to log buffers. It tracks how many times the lock has been + * locked, and which 128 byte chunks of the buffer are dirty. + */ +struct xfs_buf_log_item { + struct xfs_log_item bli_item; /* common item structure */ + struct xfs_buf *bli_buf; /* real buffer pointer */ + unsigned int bli_flags; /* misc flags */ + unsigned int bli_recur; /* lock recursion count */ + atomic_t bli_refcount; /* cnt of tp refs */ + int bli_format_count; /* count of headers */ + struct xfs_buf_log_format *bli_formats; /* array of in-log header ptrs */ + struct xfs_buf_log_format __bli_format; /* embedded in-log header */ +}; + +int xfs_buf_item_init(struct xfs_buf *, struct xfs_mount *); +void xfs_buf_item_done(struct xfs_buf *bp); +void xfs_buf_item_put(struct xfs_buf_log_item *bip); +void xfs_buf_item_log(struct xfs_buf_log_item *, uint, uint); +bool xfs_buf_item_dirty_format(struct xfs_buf_log_item *); +void xfs_buf_inode_iodone(struct xfs_buf *); +#ifdef CONFIG_XFS_QUOTA +void xfs_buf_dquot_iodone(struct xfs_buf *); +#else +static inline void xfs_buf_dquot_iodone(struct xfs_buf *bp) +{ +} +#endif /* CONFIG_XFS_QUOTA */ +void xfs_buf_iodone(struct xfs_buf *); +bool xfs_buf_log_check_iovec(struct kvec *iovec); + +unsigned int xfs_buf_inval_log_space(unsigned int map_count, + unsigned int blocksize); + +extern struct kmem_cache *xfs_buf_item_cache; + +#endif /* __XFS_BUF_ITEM_H__ */ diff --git a/fs/xfs/xfs_buf_item_recover.c b/fs/xfs/xfs_buf_item_recover.c new file mode 100644 index 000000000..57929f115 --- /dev/null +++ b/fs/xfs/xfs_buf_item_recover.c @@ -0,0 +1,1238 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2006 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_mount.h" +#include "xfs_trans.h" +#include "xfs_buf_item.h" +#include "xfs_trans_priv.h" +#include "xfs_trace.h" +#include "xfs_log.h" +#include "xfs_log_priv.h" +#include "xfs_log_recover.h" +#include "xfs_error.h" +#include "xfs_inode.h" +#include "xfs_dir2.h" +#include "xfs_quota.h" +#include "xfs_alloc.h" +#include "xfs_ag.h" +#include "xfs_sb.h" +#include "xfs_rtgroup.h" +#include "xfs_rtbitmap.h" + +/* + * This is the number of entries in the l_buf_cancel_table used during + * recovery. + */ +#define XLOG_BC_TABLE_SIZE 64 + +#define XLOG_BUF_CANCEL_BUCKET(log, blkno) \ + ((log)->l_buf_cancel_table + ((uint64_t)blkno % XLOG_BC_TABLE_SIZE)) + +/* + * This structure is used during recovery to record the buf log items which + * have been canceled and should not be replayed. + */ +struct xfs_buf_cancel { + xfs_daddr_t bc_blkno; + uint bc_len; + int bc_refcount; + struct list_head bc_list; +}; + +static struct xfs_buf_cancel * +xlog_find_buffer_cancelled( + struct xlog *log, + xfs_daddr_t blkno, + uint len) +{ + struct list_head *bucket; + struct xfs_buf_cancel *bcp; + + if (!log->l_buf_cancel_table) + return NULL; + + bucket = XLOG_BUF_CANCEL_BUCKET(log, blkno); + list_for_each_entry(bcp, bucket, bc_list) { + if (bcp->bc_blkno == blkno && bcp->bc_len == len) + return bcp; + } + + return NULL; +} + +static bool +xlog_add_buffer_cancelled( + struct xlog *log, + xfs_daddr_t blkno, + uint len) +{ + struct xfs_buf_cancel *bcp; + + /* + * If we find an existing cancel record, this indicates that the buffer + * was cancelled multiple times. To ensure that during pass 2 we keep + * the record in the table until we reach its last occurrence in the + * log, a reference count is kept to tell how many times we expect to + * see this record during the second pass. + */ + bcp = xlog_find_buffer_cancelled(log, blkno, len); + if (bcp) { + bcp->bc_refcount++; + return false; + } + + bcp = kmalloc_obj(struct xfs_buf_cancel, GFP_KERNEL | __GFP_NOFAIL); + bcp->bc_blkno = blkno; + bcp->bc_len = len; + bcp->bc_refcount = 1; + list_add_tail(&bcp->bc_list, XLOG_BUF_CANCEL_BUCKET(log, blkno)); + return true; +} + +/* + * Check if there is and entry for blkno, len in the buffer cancel record table. + */ +bool +xlog_is_buffer_cancelled( + struct xlog *log, + xfs_daddr_t blkno, + uint len) +{ + return xlog_find_buffer_cancelled(log, blkno, len) != NULL; +} + +/* + * Check if there is and entry for blkno, len in the buffer cancel record table, + * and decremented the reference count on it if there is one. + * + * Remove the cancel record once the refcount hits zero, so that if the same + * buffer is re-used again after its last cancellation we actually replay the + * changes made at that point. + */ +static bool +xlog_put_buffer_cancelled( + struct xlog *log, + xfs_daddr_t blkno, + uint len) +{ + struct xfs_buf_cancel *bcp; + + bcp = xlog_find_buffer_cancelled(log, blkno, len); + if (!bcp) { + ASSERT(0); + return false; + } + + if (--bcp->bc_refcount == 0) { + list_del(&bcp->bc_list); + kfree(bcp); + } + return true; +} + +/* log buffer item recovery */ + +/* + * Sort buffer items for log recovery. Most buffer items should end up on the + * buffer list and are recovered first, with the following exceptions: + * + * 1. XFS_BLF_CANCEL buffers must be processed last because some log items + * might depend on the incor ecancellation record, and replaying a cancelled + * buffer item can remove the incore record. + * + * 2. XFS_BLF_INODE_BUF buffers are handled after most regular items so that + * we replay di_next_unlinked only after flushing the inode 'free' state + * to the inode buffer. + * + * See xlog_recover_reorder_trans for more details. + */ +STATIC enum xlog_recover_reorder +xlog_recover_buf_reorder( + struct xlog_recover_item *item) +{ + struct xfs_buf_log_format *buf_f = item->ri_buf[0].iov_base; + + if (buf_f->blf_flags & XFS_BLF_CANCEL) + return XLOG_REORDER_CANCEL_LIST; + if (buf_f->blf_flags & XFS_BLF_INODE_BUF) + return XLOG_REORDER_INODE_BUFFER_LIST; + return XLOG_REORDER_BUFFER_LIST; +} + +STATIC void +xlog_recover_buf_ra_pass2( + struct xlog *log, + struct xlog_recover_item *item) +{ + struct xfs_buf_log_format *buf_f = item->ri_buf[0].iov_base; + + xlog_buf_readahead(log, buf_f->blf_blkno, buf_f->blf_len, NULL); +} + +/* + * Build up the table of buf cancel records so that we don't replay cancelled + * data in the second pass. + */ +static int +xlog_recover_buf_commit_pass1( + struct xlog *log, + struct xlog_recover_item *item) +{ + struct xfs_buf_log_format *bf = item->ri_buf[0].iov_base; + + if (!xfs_buf_log_check_iovec(&item->ri_buf[0])) { + xfs_err(log->l_mp, "bad buffer log item size (%zd)", + item->ri_buf[0].iov_len); + return -EFSCORRUPTED; + } + + if (!(bf->blf_flags & XFS_BLF_CANCEL)) + trace_xfs_log_recover_buf_not_cancel(log, bf); + else if (xlog_add_buffer_cancelled(log, bf->blf_blkno, bf->blf_len)) + trace_xfs_log_recover_buf_cancel_add(log, bf); + else + trace_xfs_log_recover_buf_cancel_ref_inc(log, bf); + return 0; +} + +/* + * Validate the recovered buffer is of the correct type and attach the + * appropriate buffer operations to them for writeback. Magic numbers are in a + * few places: + * the first 16 bits of the buffer (inode buffer, dquot buffer), + * the first 32 bits of the buffer (most blocks), + * inside a struct xfs_da_blkinfo at the start of the buffer. + */ +static void +xlog_recover_validate_buf_type( + struct xfs_mount *mp, + struct xfs_buf *bp, + struct xfs_buf_log_format *buf_f, + xfs_lsn_t current_lsn) +{ + struct xfs_da_blkinfo *info = bp->b_addr; + uint32_t magic32; + uint16_t magic16; + uint16_t magicda; + char *warnmsg = NULL; + + /* + * We can only do post recovery validation on items on CRC enabled + * fielsystems as we need to know when the buffer was written to be able + * to determine if we should have replayed the item. If we replay old + * metadata over a newer buffer, then it will enter a temporarily + * inconsistent state resulting in verification failures. Hence for now + * just avoid the verification stage for non-crc filesystems + */ + if (!xfs_has_crc(mp)) + return; + + magic32 = be32_to_cpu(*(__be32 *)bp->b_addr); + magic16 = be16_to_cpu(*(__be16*)bp->b_addr); + magicda = be16_to_cpu(info->magic); + switch (xfs_blft_from_flags(buf_f)) { + case XFS_BLFT_BTREE_BUF: + switch (magic32) { + case XFS_ABTB_CRC_MAGIC: + case XFS_ABTB_MAGIC: + bp->b_ops = &xfs_bnobt_buf_ops; + break; + case XFS_ABTC_CRC_MAGIC: + case XFS_ABTC_MAGIC: + bp->b_ops = &xfs_cntbt_buf_ops; + break; + case XFS_IBT_CRC_MAGIC: + case XFS_IBT_MAGIC: + bp->b_ops = &xfs_inobt_buf_ops; + break; + case XFS_FIBT_CRC_MAGIC: + case XFS_FIBT_MAGIC: + bp->b_ops = &xfs_finobt_buf_ops; + break; + case XFS_BMAP_CRC_MAGIC: + case XFS_BMAP_MAGIC: + bp->b_ops = &xfs_bmbt_buf_ops; + break; + case XFS_RTRMAP_CRC_MAGIC: + bp->b_ops = &xfs_rtrmapbt_buf_ops; + break; + case XFS_RMAP_CRC_MAGIC: + bp->b_ops = &xfs_rmapbt_buf_ops; + break; + case XFS_REFC_CRC_MAGIC: + bp->b_ops = &xfs_refcountbt_buf_ops; + break; + case XFS_RTREFC_CRC_MAGIC: + bp->b_ops = &xfs_rtrefcountbt_buf_ops; + break; + default: + warnmsg = "Bad btree block magic!"; + break; + } + break; + case XFS_BLFT_AGF_BUF: + if (magic32 != XFS_AGF_MAGIC) { + warnmsg = "Bad AGF block magic!"; + break; + } + bp->b_ops = &xfs_agf_buf_ops; + break; + case XFS_BLFT_AGFL_BUF: + if (magic32 != XFS_AGFL_MAGIC) { + warnmsg = "Bad AGFL block magic!"; + break; + } + bp->b_ops = &xfs_agfl_buf_ops; + break; + case XFS_BLFT_AGI_BUF: + if (magic32 != XFS_AGI_MAGIC) { + warnmsg = "Bad AGI block magic!"; + break; + } + bp->b_ops = &xfs_agi_buf_ops; + break; + case XFS_BLFT_UDQUOT_BUF: + case XFS_BLFT_PDQUOT_BUF: + case XFS_BLFT_GDQUOT_BUF: +#ifdef CONFIG_XFS_QUOTA + if (magic16 != XFS_DQUOT_MAGIC) { + warnmsg = "Bad DQUOT block magic!"; + break; + } + bp->b_ops = &xfs_dquot_buf_ops; +#else + xfs_alert(mp, + "Trying to recover dquots without QUOTA support built in!"); + ASSERT(0); +#endif + break; + case XFS_BLFT_DINO_BUF: + if (magic16 != XFS_DINODE_MAGIC) { + warnmsg = "Bad INODE block magic!"; + break; + } + bp->b_ops = &xfs_inode_buf_ops; + break; + case XFS_BLFT_SYMLINK_BUF: + if (magic32 != XFS_SYMLINK_MAGIC) { + warnmsg = "Bad symlink block magic!"; + break; + } + bp->b_ops = &xfs_symlink_buf_ops; + break; + case XFS_BLFT_DIR_BLOCK_BUF: + if (magic32 != XFS_DIR2_BLOCK_MAGIC && + magic32 != XFS_DIR3_BLOCK_MAGIC) { + warnmsg = "Bad dir block magic!"; + break; + } + bp->b_ops = &xfs_dir3_block_buf_ops; + break; + case XFS_BLFT_DIR_DATA_BUF: + if (magic32 != XFS_DIR2_DATA_MAGIC && + magic32 != XFS_DIR3_DATA_MAGIC) { + warnmsg = "Bad dir data magic!"; + break; + } + bp->b_ops = &xfs_dir3_data_buf_ops; + break; + case XFS_BLFT_DIR_FREE_BUF: + if (magic32 != XFS_DIR2_FREE_MAGIC && + magic32 != XFS_DIR3_FREE_MAGIC) { + warnmsg = "Bad dir3 free magic!"; + break; + } + bp->b_ops = &xfs_dir3_free_buf_ops; + break; + case XFS_BLFT_DIR_LEAF1_BUF: + if (magicda != XFS_DIR2_LEAF1_MAGIC && + magicda != XFS_DIR3_LEAF1_MAGIC) { + warnmsg = "Bad dir leaf1 magic!"; + break; + } + bp->b_ops = &xfs_dir3_leaf1_buf_ops; + break; + case XFS_BLFT_DIR_LEAFN_BUF: + if (magicda != XFS_DIR2_LEAFN_MAGIC && + magicda != XFS_DIR3_LEAFN_MAGIC) { + warnmsg = "Bad dir leafn magic!"; + break; + } + bp->b_ops = &xfs_dir3_leafn_buf_ops; + break; + case XFS_BLFT_DA_NODE_BUF: + if (magicda != XFS_DA_NODE_MAGIC && + magicda != XFS_DA3_NODE_MAGIC) { + warnmsg = "Bad da node magic!"; + break; + } + bp->b_ops = &xfs_da3_node_buf_ops; + break; + case XFS_BLFT_ATTR_LEAF_BUF: + if (magicda != XFS_ATTR_LEAF_MAGIC && + magicda != XFS_ATTR3_LEAF_MAGIC) { + warnmsg = "Bad attr leaf magic!"; + break; + } + bp->b_ops = &xfs_attr3_leaf_buf_ops; + break; + case XFS_BLFT_ATTR_RMT_BUF: + if (magic32 != XFS_ATTR3_RMT_MAGIC) { + warnmsg = "Bad attr remote magic!"; + break; + } + bp->b_ops = &xfs_attr3_rmt_buf_ops; + break; + case XFS_BLFT_SB_BUF: + if (magic32 != XFS_SB_MAGIC) { + warnmsg = "Bad SB block magic!"; + break; + } + bp->b_ops = &xfs_sb_buf_ops; + break; +#ifdef CONFIG_XFS_RT + case XFS_BLFT_RTBITMAP_BUF: + if (xfs_has_rtgroups(mp) && magic32 != XFS_RTBITMAP_MAGIC) { + warnmsg = "Bad rtbitmap magic!"; + break; + } + bp->b_ops = xfs_rtblock_ops(mp, XFS_RTGI_BITMAP); + break; + case XFS_BLFT_RTSUMMARY_BUF: + if (xfs_has_rtgroups(mp) && magic32 != XFS_RTSUMMARY_MAGIC) { + warnmsg = "Bad rtsummary magic!"; + break; + } + bp->b_ops = xfs_rtblock_ops(mp, XFS_RTGI_SUMMARY); + break; +#endif /* CONFIG_XFS_RT */ + default: + xfs_warn(mp, "Unknown buffer type %d!", + xfs_blft_from_flags(buf_f)); + break; + } + + /* + * Nothing else to do in the case of a NULL current LSN as this means + * the buffer is more recent than the change in the log and will be + * skipped. + */ + if (current_lsn == NULLCOMMITLSN) + return; + + if (warnmsg) { + xfs_warn(mp, warnmsg); + ASSERT(0); + } + + /* + * We must update the metadata LSN of the buffer as it is written out to + * ensure that older transactions never replay over this one and corrupt + * the buffer. This can occur if log recovery is interrupted at some + * point after the current transaction completes, at which point a + * subsequent mount starts recovery from the beginning. + * + * Write verifiers update the metadata LSN from log items attached to + * the buffer. Therefore, initialize a bli purely to carry the LSN to + * the verifier. + */ + if (bp->b_ops) { + struct xfs_buf_log_item *bip; + + xfs_buf_item_init(bp, mp); + bip = bp->b_log_item; + bip->bli_item.li_lsn = current_lsn; + } +} + +/* + * Perform a 'normal' buffer recovery. Each logged region of the + * buffer should be copied over the corresponding region in the + * given buffer. The bitmap in the buf log format structure indicates + * where to place the logged data. + */ +STATIC int +xlog_recover_do_reg_buffer( + struct xfs_mount *mp, + struct xlog_recover_item *item, + struct xfs_buf *bp, + struct xfs_buf_log_format *buf_f, + xfs_lsn_t current_lsn) +{ + int i; + int bit; + int nbits; + xfs_failaddr_t fa; + const size_t size_disk_dquot = sizeof(struct xfs_disk_dquot); + + trace_xfs_log_recover_buf_reg_buf(mp->m_log, buf_f); + + bit = 0; + i = 1; /* 0 is the buf format structure */ + while (1) { + bit = xfs_next_bit(buf_f->blf_data_map, + buf_f->blf_map_size, bit); + if (bit == -1) + break; + nbits = xfs_contig_bits(buf_f->blf_data_map, + buf_f->blf_map_size, bit); + ASSERT(nbits > 0); + ASSERT(item->ri_buf[i].iov_base != NULL); + ASSERT(item->ri_buf[i].iov_len % XFS_BLF_CHUNK == 0); + /* + * The bitmap is only trustworthy to the extent that it + * describes a region that actually fits inside the buffer we + * read in based on the (attacker-controlled) blf_len. Do not + * rely on an ASSERT() for this -- it compiles away entirely on + * non-DEBUG kernels, which is exactly where this matters, so + * validate it for real and abort recovery of this buffer rather + * than copying past the end of it. + */ + if (XFS_IS_CORRUPT(mp, BBTOB(bp->b_length) < + ((uint)bit << XFS_BLF_SHIFT) + + (nbits << XFS_BLF_SHIFT))) { + xfs_alert(mp, + "Bad buffer log item dirty bitmap (bit %d, nbits %d) for %d-byte buffer at daddr 0x%llx.", + bit, nbits, BBTOB(bp->b_length), + xfs_buf_daddr(bp)); + return -EFSCORRUPTED; + } + + /* + * The dirty regions logged in the buffer, even though + * contiguous, may span multiple chunks. This is because the + * dirty region may span a physical page boundary in a buffer + * and hence be split into two separate vectors for writing into + * the log. Hence we need to trim nbits back to the length of + * the current region being copied out of the log. + */ + if (item->ri_buf[i].iov_len < (nbits << XFS_BLF_SHIFT)) + nbits = item->ri_buf[i].iov_len >> XFS_BLF_SHIFT; + + /* + * Do a sanity check if this is a dquot buffer. Just checking + * the first dquot in the buffer should do. XXXThis is + * probably a good thing to do for other buf types also. + */ + fa = NULL; + if (buf_f->blf_flags & + (XFS_BLF_UDQUOT_BUF|XFS_BLF_PDQUOT_BUF|XFS_BLF_GDQUOT_BUF)) { + if (item->ri_buf[i].iov_base == NULL) { + xfs_alert(mp, + "XFS: NULL dquot in %s.", __func__); + goto next; + } + if (item->ri_buf[i].iov_len < size_disk_dquot) { + xfs_alert(mp, + "XFS: dquot too small (%zd) in %s.", + item->ri_buf[i].iov_len, __func__); + goto next; + } + fa = xfs_dquot_verify(mp, item->ri_buf[i].iov_base, -1); + if (fa) { + xfs_alert(mp, + "dquot corrupt at %pS trying to replay into block 0x%llx", + fa, xfs_buf_daddr(bp)); + goto next; + } + } + + memcpy(xfs_buf_offset(bp, + (uint)bit << XFS_BLF_SHIFT), /* dest */ + item->ri_buf[i].iov_base, /* source */ + nbits<ri_total); + + xlog_recover_validate_buf_type(mp, bp, buf_f, current_lsn); + return 0; +} + +/* + * Perform a dquot buffer recovery. + * Simple algorithm: if we have found a QUOTAOFF log item of the same type + * (ie. USR or GRP), then just toss this buffer away; don't recover it. + * Else, treat it as a regular buffer and do recovery. + * + * Return 0 if the buffer was not recovered (tossed), 1 if it was recovered and + * needs writing, or a negative errno if recovery of the buffer failed. + */ +STATIC int +xlog_recover_do_dquot_buffer( + struct xfs_mount *mp, + struct xlog *log, + struct xlog_recover_item *item, + struct xfs_buf *bp, + struct xfs_buf_log_format *buf_f) +{ + uint type; + int error; + + trace_xfs_log_recover_buf_dquot_buf(log, buf_f); + + /* + * Filesystems are required to send in quota flags at mount time. + */ + if (!mp->m_qflags) + return 0; + + type = 0; + if (buf_f->blf_flags & XFS_BLF_UDQUOT_BUF) + type |= XFS_DQTYPE_USER; + if (buf_f->blf_flags & XFS_BLF_PDQUOT_BUF) + type |= XFS_DQTYPE_PROJ; + if (buf_f->blf_flags & XFS_BLF_GDQUOT_BUF) + type |= XFS_DQTYPE_GROUP; + /* + * This type of quotas was turned off, so ignore this buffer + */ + if (log->l_quotaoffs_flag & type) + return 0; + + error = xlog_recover_do_reg_buffer(mp, item, bp, buf_f, NULLCOMMITLSN); + if (error) + return error; + return 1; +} + +/* + * Perform recovery for a buffer full of inodes. In these buffers, the only + * data which should be recovered is that which corresponds to the + * di_next_unlinked pointers in the on disk inode structures. The rest of the + * data for the inodes is always logged through the inodes themselves rather + * than the inode buffer and is recovered in xlog_recover_inode_pass2(). + * + * The only time when buffers full of inodes are fully recovered is when the + * buffer is full of newly allocated inodes. In this case the buffer will + * not be marked as an inode buffer and so will be sent to + * xlog_recover_do_reg_buffer() below during recovery. + */ +STATIC int +xlog_recover_do_inode_buffer( + struct xfs_mount *mp, + struct xlog_recover_item *item, + struct xfs_buf *bp, + struct xfs_buf_log_format *buf_f) +{ + int i; + int item_index = 0; + int bit = 0; + int nbits = 0; + int reg_buf_offset = 0; + int reg_buf_bytes = 0; + int next_unlinked_offset; + int inodes_per_buf; + xfs_agino_t *logged_nextp; + xfs_agino_t *buffer_nextp; + + trace_xfs_log_recover_buf_inode_buf(mp->m_log, buf_f); + + /* + * Post recovery validation only works properly on CRC enabled + * filesystems. + */ + if (xfs_has_crc(mp)) + bp->b_ops = &xfs_inode_buf_ops; + + inodes_per_buf = BBTOB(bp->b_length) >> mp->m_sb.sb_inodelog; + for (i = 0; i < inodes_per_buf; i++) { + next_unlinked_offset = (i * mp->m_sb.sb_inodesize) + + offsetof(struct xfs_dinode, di_next_unlinked); + + while (next_unlinked_offset >= + (reg_buf_offset + reg_buf_bytes)) { + /* + * The next di_next_unlinked field is beyond + * the current logged region. Find the next + * logged region that contains or is beyond + * the current di_next_unlinked field. + */ + bit += nbits; + bit = xfs_next_bit(buf_f->blf_data_map, + buf_f->blf_map_size, bit); + + /* + * If there are no more logged regions in the + * buffer, then we're done. + */ + if (bit == -1) + return 0; + + nbits = xfs_contig_bits(buf_f->blf_data_map, + buf_f->blf_map_size, bit); + ASSERT(nbits > 0); + reg_buf_offset = bit << XFS_BLF_SHIFT; + reg_buf_bytes = nbits << XFS_BLF_SHIFT; + item_index++; + } + + /* + * If the current logged region starts after the current + * di_next_unlinked field, then move on to the next + * di_next_unlinked field. + */ + if (next_unlinked_offset < reg_buf_offset) + continue; + + ASSERT(item->ri_buf[item_index].iov_base != NULL); + ASSERT((item->ri_buf[item_index].iov_len % XFS_BLF_CHUNK) == 0); + ASSERT((reg_buf_offset + reg_buf_bytes) <= BBTOB(bp->b_length)); + + /* + * The current logged region contains a copy of the + * current di_next_unlinked field. Extract its value + * and copy it to the buffer copy. + */ + logged_nextp = item->ri_buf[item_index].iov_base + + next_unlinked_offset - reg_buf_offset; + if (XFS_IS_CORRUPT(mp, *logged_nextp == 0)) { + xfs_alert(mp, + "Bad inode buffer log record (ptr = "PTR_FMT", bp = "PTR_FMT"). " + "Trying to replay bad (0) inode di_next_unlinked field.", + item, bp); + return -EFSCORRUPTED; + } + + buffer_nextp = xfs_buf_offset(bp, next_unlinked_offset); + *buffer_nextp = *logged_nextp; + + /* + * If necessary, recalculate the CRC in the on-disk inode. We + * have to leave the inode in a consistent state for whoever + * reads it next.... + */ + xfs_dinode_calc_crc(mp, + xfs_buf_offset(bp, i * mp->m_sb.sb_inodesize)); + + } + + return 0; +} + +/* + * Update the in-memory superblock and perag structures from the primary SB + * buffer. + * + * This is required because transactions running after growfs may require the + * updated values to be set in a previous fully commit transaction. + */ +static int +xlog_recover_do_primary_sb_buffer( + struct xfs_mount *mp, + struct xlog_recover_item *item, + struct xfs_buf *bp, + struct xfs_buf_log_format *buf_f, + xfs_lsn_t current_lsn) +{ + struct xfs_dsb *dsb = bp->b_addr; + xfs_agnumber_t orig_agcount = mp->m_sb.sb_agcount; + xfs_rgnumber_t orig_rgcount = mp->m_sb.sb_rgcount; + int error; + + error = xlog_recover_do_reg_buffer(mp, item, bp, buf_f, current_lsn); + if (error) + return error; + + if (orig_agcount == 0) { + xfs_alert(mp, "Trying to grow file system without AGs"); + return -EFSCORRUPTED; + } + + /* + * Update the in-core super block from the freshly recovered on-disk one. + */ + xfs_sb_from_disk(&mp->m_sb, dsb); + + /* + * Grow can change the device size. Mirror that into the buftarg. + */ + mp->m_ddev_targp->bt_nr_sectors = + XFS_FSB_TO_BB(mp, mp->m_sb.sb_dblocks); + if (mp->m_rtdev_targp && mp->m_rtdev_targp != mp->m_ddev_targp) { + mp->m_rtdev_targp->bt_nr_sectors = + XFS_FSB_TO_BB(mp, mp->m_sb.sb_rblocks); + } + + if (mp->m_sb.sb_agcount < orig_agcount) { + xfs_alert(mp, "Shrinking AG count in log recovery not supported"); + return -EFSCORRUPTED; + } + if (mp->m_sb.sb_rgcount < orig_rgcount) { + xfs_warn(mp, + "Shrinking rtgroup count in log recovery not supported"); + return -EFSCORRUPTED; + } + + /* + * If the last AG was grown or shrunk, we also need to update the + * length in the in-core perag structure and values depending on it. + */ + error = xfs_update_last_ag_size(mp, orig_agcount); + if (error) + return error; + + /* + * If the last rtgroup was grown or shrunk, we also need to update the + * length in the in-core rtgroup structure and values depending on it. + * Ignore this on any filesystem with zero rtgroups. + */ + if (orig_rgcount > 0) { + error = xfs_update_last_rtgroup_size(mp, orig_rgcount); + if (error) + return error; + } + + /* + * Initialize the new perags, and also update various block and inode + * allocator setting based off the number of AGs or total blocks. + * Because of the latter this also needs to happen if the agcount did + * not change. + */ + error = xfs_initialize_perag(mp, orig_agcount, mp->m_sb.sb_agcount, + mp->m_sb.sb_dblocks, &mp->m_maxagi); + if (error) { + xfs_warn(mp, "Failed recovery per-ag init: %d", error); + return error; + } + mp->m_alloc_set_aside = xfs_alloc_set_aside(mp); + + error = xfs_initialize_rtgroups(mp, orig_rgcount, mp->m_sb.sb_rgcount, + mp->m_sb.sb_rextents); + if (error) { + xfs_warn(mp, "Failed recovery rtgroup init: %d", error); + return error; + } + return 0; +} + +/* + * V5 filesystems know the age of the buffer on disk being recovered. We can + * have newer objects on disk than we are replaying, and so for these cases we + * don't want to replay the current change as that will make the buffer contents + * temporarily invalid on disk. + * + * The magic number might not match the buffer type we are going to recover + * (e.g. reallocated blocks), so we ignore the xfs_buf_log_format flags. Hence + * extract the LSN of the existing object in the buffer based on it's current + * magic number. If we don't recognise the magic number in the buffer, then + * return a LSN of -1 so that the caller knows it was an unrecognised block and + * so can recover the buffer. + * + * Note: we cannot rely solely on magic number matches to determine that the + * buffer has a valid LSN - we also need to verify that it belongs to this + * filesystem, so we need to extract the object's LSN and compare it to that + * which we read from the superblock. If the UUIDs don't match, then we've got a + * stale metadata block from an old filesystem instance that we need to recover + * over the top of. + */ +static xfs_lsn_t +xlog_recover_get_buf_lsn( + struct xfs_mount *mp, + struct xfs_buf *bp, + struct xfs_buf_log_format *buf_f) +{ + uint32_t magic32; + uint16_t magic16; + uint16_t magicda; + void *blk = bp->b_addr; + uuid_t *uuid; + xfs_lsn_t lsn = -1; + uint16_t blft; + + /* v4 filesystems always recover immediately */ + if (!xfs_has_crc(mp)) + goto recover_immediately; + + /* + * realtime bitmap and summary file blocks do not have magic numbers or + * UUIDs, so we must recover them immediately. + */ + blft = xfs_blft_from_flags(buf_f); + if (!xfs_has_rtgroups(mp) && (blft == XFS_BLFT_RTBITMAP_BUF || + blft == XFS_BLFT_RTSUMMARY_BUF)) + goto recover_immediately; + + magic32 = be32_to_cpu(*(__be32 *)blk); + switch (magic32) { + case XFS_RTSUMMARY_MAGIC: + case XFS_RTBITMAP_MAGIC: { + struct xfs_rtbuf_blkinfo *hdr = blk; + + lsn = be64_to_cpu(hdr->rt_lsn); + uuid = &hdr->rt_uuid; + break; + } + case XFS_ABTB_CRC_MAGIC: + case XFS_ABTC_CRC_MAGIC: + case XFS_ABTB_MAGIC: + case XFS_ABTC_MAGIC: + case XFS_RMAP_CRC_MAGIC: + case XFS_REFC_CRC_MAGIC: + case XFS_FIBT_CRC_MAGIC: + case XFS_FIBT_MAGIC: + case XFS_IBT_CRC_MAGIC: + case XFS_IBT_MAGIC: { + struct xfs_btree_block *btb = blk; + + lsn = be64_to_cpu(btb->bb_u.s.bb_lsn); + uuid = &btb->bb_u.s.bb_uuid; + break; + } + case XFS_RTRMAP_CRC_MAGIC: + case XFS_RTREFC_CRC_MAGIC: + case XFS_BMAP_CRC_MAGIC: + case XFS_BMAP_MAGIC: { + struct xfs_btree_block *btb = blk; + + lsn = be64_to_cpu(btb->bb_u.l.bb_lsn); + uuid = &btb->bb_u.l.bb_uuid; + break; + } + case XFS_AGF_MAGIC: + lsn = be64_to_cpu(((struct xfs_agf *)blk)->agf_lsn); + uuid = &((struct xfs_agf *)blk)->agf_uuid; + break; + case XFS_AGFL_MAGIC: + lsn = be64_to_cpu(((struct xfs_agfl *)blk)->agfl_lsn); + uuid = &((struct xfs_agfl *)blk)->agfl_uuid; + break; + case XFS_AGI_MAGIC: + lsn = be64_to_cpu(((struct xfs_agi *)blk)->agi_lsn); + uuid = &((struct xfs_agi *)blk)->agi_uuid; + break; + case XFS_SYMLINK_MAGIC: + lsn = be64_to_cpu(((struct xfs_dsymlink_hdr *)blk)->sl_lsn); + uuid = &((struct xfs_dsymlink_hdr *)blk)->sl_uuid; + break; + case XFS_DIR3_BLOCK_MAGIC: + case XFS_DIR3_DATA_MAGIC: + case XFS_DIR3_FREE_MAGIC: + lsn = be64_to_cpu(((struct xfs_dir3_blk_hdr *)blk)->lsn); + uuid = &((struct xfs_dir3_blk_hdr *)blk)->uuid; + break; + case XFS_ATTR3_RMT_MAGIC: + /* + * Remote attr blocks are written synchronously, rather than + * being logged. That means they do not contain a valid LSN + * (i.e. transactionally ordered) in them, and hence any time we + * see a buffer to replay over the top of a remote attribute + * block we should simply do so. + */ + goto recover_immediately; + case XFS_SB_MAGIC: + /* + * superblock uuids are magic. We may or may not have a + * sb_meta_uuid on disk, but it will be set in the in-core + * superblock. We set the uuid pointer for verification + * according to the superblock feature mask to ensure we check + * the relevant UUID in the superblock. + */ + lsn = be64_to_cpu(((struct xfs_dsb *)blk)->sb_lsn); + if (xfs_has_metauuid(mp)) + uuid = &((struct xfs_dsb *)blk)->sb_meta_uuid; + else + uuid = &((struct xfs_dsb *)blk)->sb_uuid; + break; + default: + break; + } + + if (lsn != (xfs_lsn_t)-1) { + if (!uuid_equal(&mp->m_sb.sb_meta_uuid, uuid)) + goto recover_immediately; + return lsn; + } + + magicda = be16_to_cpu(((struct xfs_da_blkinfo *)blk)->magic); + switch (magicda) { + case XFS_DIR3_LEAF1_MAGIC: + case XFS_DIR3_LEAFN_MAGIC: + case XFS_ATTR3_LEAF_MAGIC: + case XFS_DA3_NODE_MAGIC: + lsn = be64_to_cpu(((struct xfs_da3_blkinfo *)blk)->lsn); + uuid = &((struct xfs_da3_blkinfo *)blk)->uuid; + break; + default: + break; + } + + if (lsn != (xfs_lsn_t)-1) { + if (!uuid_equal(&mp->m_sb.sb_meta_uuid, uuid)) + goto recover_immediately; + return lsn; + } + + /* + * We do individual object checks on dquot and inode buffers as they + * have their own individual LSN records. Also, we could have a stale + * buffer here, so we have to at least recognise these buffer types. + * + * A notd complexity here is inode unlinked list processing - it logs + * the inode directly in the buffer, but we don't know which inodes have + * been modified, and there is no global buffer LSN. Hence we need to + * recover all inode buffer types immediately. This problem will be + * fixed by logical logging of the unlinked list modifications. + */ + magic16 = be16_to_cpu(*(__be16 *)blk); + switch (magic16) { + case XFS_DQUOT_MAGIC: + case XFS_DINODE_MAGIC: + goto recover_immediately; + default: + break; + } + + /* unknown buffer contents, recover immediately */ + +recover_immediately: + return (xfs_lsn_t)-1; + +} + +/* + * This routine replays a modification made to a buffer at runtime. + * There are actually two types of buffer, regular and inode, which + * are handled differently. Inode buffers are handled differently + * in that we only recover a specific set of data from them, namely + * the inode di_next_unlinked fields. This is because all other inode + * data is actually logged via inode records and any data we replay + * here which overlaps that may be stale. + * + * When meta-data buffers are freed at run time we log a buffer item + * with the XFS_BLF_CANCEL bit set to indicate that previous copies + * of the buffer in the log should not be replayed at recovery time. + * This is so that if the blocks covered by the buffer are reused for + * file data before we crash we don't end up replaying old, freed + * meta-data into a user's file. + * + * To handle the cancellation of buffer log items, we make two passes + * over the log during recovery. During the first we build a table of + * those buffers which have been cancelled, and during the second we + * only replay those buffers which do not have corresponding cancel + * records in the table. See xlog_recover_buf_pass[1,2] above + * for more details on the implementation of the table of cancel records. + */ +STATIC int +xlog_recover_buf_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t current_lsn) +{ + struct xfs_buf_log_format *buf_f = item->ri_buf[0].iov_base; + struct xfs_mount *mp = log->l_mp; + struct xfs_buf *bp; + int error; + xfs_lsn_t lsn; + + /* + * In this pass we only want to recover all the buffers which have + * not been cancelled and are not cancellation buffers themselves. + */ + if (buf_f->blf_flags & XFS_BLF_CANCEL) { + if (xlog_put_buffer_cancelled(log, buf_f->blf_blkno, + buf_f->blf_len)) + goto cancelled; + } else { + + if (xlog_is_buffer_cancelled(log, buf_f->blf_blkno, + buf_f->blf_len)) + goto cancelled; + } + + trace_xfs_log_recover_buf_recover(log, buf_f); + error = xfs_buf_read(mp->m_ddev_targp, buf_f->blf_blkno, buf_f->blf_len, + 0, &bp, NULL); + if (error) + return error; + + /* + * Recover the buffer only if we get an LSN from it and it's less than + * the lsn of the transaction we are replaying. + * + * Note that we have to be extremely careful of readahead here. + * Readahead does not attach verfiers to the buffers so if we don't + * actually do any replay after readahead because of the LSN we found + * in the buffer if more recent than that current transaction then we + * need to attach the verifier directly. Failure to do so can lead to + * future recovery actions (e.g. EFI and unlinked list recovery) can + * operate on the buffers and they won't get the verifier attached. This + * can lead to blocks on disk having the correct content but a stale + * CRC. + * + * It is safe to assume these clean buffers are currently up to date. + * If the buffer is dirtied by a later transaction being replayed, then + * the verifier will be reset to match whatever recover turns that + * buffer into. + */ + lsn = xlog_recover_get_buf_lsn(mp, bp, buf_f); + if (lsn && lsn != -1 && XFS_LSN_CMP(lsn, current_lsn) >= 0) { + trace_xfs_log_recover_buf_skip(log, buf_f); + xlog_recover_validate_buf_type(mp, bp, buf_f, NULLCOMMITLSN); + + /* + * We're skipping replay of this buffer log item due to the log + * item LSN being behind the ondisk buffer. Verify the buffer + * contents since we aren't going to run the write verifier. + */ + if (bp->b_ops) { + bp->b_ops->verify_read(bp); + error = bp->b_error; + } + goto out_release; + } + + if (buf_f->blf_flags & XFS_BLF_INODE_BUF) { + error = xlog_recover_do_inode_buffer(mp, item, bp, buf_f); + if (error) + goto out_release; + } else if (buf_f->blf_flags & + (XFS_BLF_UDQUOT_BUF|XFS_BLF_PDQUOT_BUF|XFS_BLF_GDQUOT_BUF)) { + error = xlog_recover_do_dquot_buffer(mp, log, item, bp, buf_f); + if (error <= 0) + goto out_release; + /* write dirty buffer */ + error = 0; + } else if ((xfs_blft_from_flags(buf_f) & XFS_BLFT_SB_BUF) && + xfs_buf_daddr(bp) == 0) { + error = xlog_recover_do_primary_sb_buffer(mp, item, bp, buf_f, + current_lsn); + if (error) + goto out_writebuf; + + /* Update the rt superblock if we have one. */ + if (xfs_has_rtsb(mp) && mp->m_rtsb_bp) { + struct xfs_buf *rtsb_bp = mp->m_rtsb_bp; + + xfs_buf_lock(rtsb_bp); + xfs_buf_hold(rtsb_bp); + xfs_update_rtsb(rtsb_bp, bp); + xfs_buf_delwri_queue(rtsb_bp, buffer_list); + xfs_buf_relse(rtsb_bp); + } + } else { + error = xlog_recover_do_reg_buffer(mp, item, bp, buf_f, + current_lsn); + if (error) + goto out_release; + } + + /* + * Buffer held by buf log item during 'normal' buffer recovery must + * be committed through buffer I/O submission path to ensure proper + * release. When error occurs during sb buffer recovery, log shutdown + * will be done before submitting buffer list so that buffers can be + * released correctly through ioend failure path. + */ +out_writebuf: + + /* + * Perform delayed write on the buffer. Asynchronous writes will be + * slower when taking into account all the buffers to be flushed. + * + * Also make sure that only inode buffers with good sizes stay in + * the buffer cache. The kernel moves inodes in buffers of 1 block + * or inode_cluster_size bytes, whichever is bigger. The inode + * buffers in the log can be a different size if the log was generated + * by an older kernel using unclustered inode buffers or a newer kernel + * running with a different inode cluster size. Regardless, if + * the inode buffer size isn't max(blocksize, inode_cluster_size) + * for *our* value of inode_cluster_size, then we need to keep + * the buffer out of the buffer cache so that the buffer won't + * overlap with future reads of those inodes. + */ + if (XFS_DINODE_MAGIC == + be16_to_cpu(*((__be16 *)xfs_buf_offset(bp, 0))) && + (BBTOB(bp->b_length) != M_IGEO(log->l_mp)->inode_cluster_size)) { + xfs_buf_stale(bp); + error = xfs_bwrite(bp); + } else { + ASSERT(bp->b_mount == mp); + xfs_buf_delwri_queue(bp, buffer_list); + } + +out_release: + xfs_buf_relse(bp); + return error; +cancelled: + trace_xfs_log_recover_buf_cancel(log, buf_f); + return 0; +} + +const struct xlog_recover_item_ops xlog_buf_item_ops = { + .item_type = XFS_LI_BUF, + .reorder = xlog_recover_buf_reorder, + .ra_pass2 = xlog_recover_buf_ra_pass2, + .commit_pass1 = xlog_recover_buf_commit_pass1, + .commit_pass2 = xlog_recover_buf_commit_pass2, +}; + +#ifdef DEBUG +void +xlog_check_buf_cancel_table( + struct xlog *log) +{ + int i; + + for (i = 0; i < XLOG_BC_TABLE_SIZE; i++) + ASSERT(list_empty(&log->l_buf_cancel_table[i])); +} +#endif + +int +xlog_alloc_buf_cancel_table( + struct xlog *log) +{ + void *p; + int i; + + ASSERT(log->l_buf_cancel_table == NULL); + + p = kmalloc_objs(struct list_head, XLOG_BC_TABLE_SIZE); + if (!p) + return -ENOMEM; + + log->l_buf_cancel_table = p; + for (i = 0; i < XLOG_BC_TABLE_SIZE; i++) + INIT_LIST_HEAD(&log->l_buf_cancel_table[i]); + + return 0; +} + +void +xlog_free_buf_cancel_table( + struct xlog *log) +{ + int i; + + if (!log->l_buf_cancel_table) + return; + + for (i = 0; i < XLOG_BC_TABLE_SIZE; i++) { + struct xfs_buf_cancel *bc; + + while ((bc = list_first_entry_or_null( + &log->l_buf_cancel_table[i], + struct xfs_buf_cancel, bc_list))) { + list_del(&bc->bc_list); + kfree(bc); + } + } + + kfree(log->l_buf_cancel_table); + log->l_buf_cancel_table = NULL; +} diff --git a/fs/xfs/xfs_buf_mem.c b/fs/xfs/xfs_buf_mem.c new file mode 100644 index 000000000..b2fd7276b --- /dev/null +++ b/fs/xfs/xfs_buf_mem.c @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2023-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_buf.h" +#include "xfs_buf_mem.h" +#include "xfs_trace.h" +#include +#include "xfs_log_format.h" +#include "xfs_trans.h" +#include "xfs_buf_item.h" +#include "xfs_error.h" + +/* + * Buffer Cache for In-Memory Files + * ================================ + * + * Online fsck wants to create ephemeral ordered recordsets. The existing + * btree infrastructure can do this, but we need the buffer cache to target + * memory instead of block devices. + * + * When CONFIG_TMPFS=y, shmemfs is enough of a filesystem to meet those + * requirements. Therefore, the xmbuf mechanism uses an unlinked shmem file to + * store our staging data. This file is not installed in the file descriptor + * table so that user programs cannot access the data, which means that the + * xmbuf must be freed with xmbuf_destroy. + * + * xmbufs assume that the caller will handle all required concurrency + * management; standard vfs locks (freezer and inode) are not taken. Reads + * and writes are satisfied directly from the page cache. + * + * The only supported block size is PAGE_SIZE, and we cannot use highmem. + */ + +/* + * shmem files used to back an in-memory buffer cache must not be exposed to + * userspace. Upper layers must coordinate access to the one handle returned + * by the constructor, so establish a separate lock class for xmbufs to avoid + * confusing lockdep. + */ +static struct lock_class_key xmbuf_i_mutex_key; + +/* + * Allocate a buffer cache target for a memory-backed file and set up the + * buffer target. + */ +int +xmbuf_alloc( + struct xfs_mount *mp, + const char *descr, + struct xfs_buftarg **btpp) +{ + struct file *file; + struct inode *inode; + struct xfs_buftarg *btp; + int error; + + btp = kzalloc_obj(*btp); + if (!btp) + return -ENOMEM; + + file = shmem_kernel_file_setup(descr, 0, EMPTY_VMA_FLAGS); + if (IS_ERR(file)) { + error = PTR_ERR(file); + goto out_free_btp; + } + inode = file_inode(file); + + /* private file, private locking */ + lockdep_set_class(&inode->i_rwsem, &xmbuf_i_mutex_key); + + /* + * We don't want to bother with kmapping data during repair, so don't + * allow highmem folios to back this mapping. + */ + mapping_set_gfp_mask(inode->i_mapping, GFP_KERNEL); + + /* ensure all writes are below EOF to avoid pagecache zeroing */ + i_size_write(inode, inode->i_sb->s_maxbytes); + + /* Initialize buffer target */ + btp->bt_mount = mp; + btp->bt_dev = (dev_t)-1U; + btp->bt_bdev = NULL; /* in-memory buftargs have no bdev */ + btp->bt_file = file; + btp->bt_meta_sectorsize = XMBUF_BLOCKSIZE; + btp->bt_meta_sectormask = XMBUF_BLOCKSIZE - 1; + + error = xfs_init_buftarg(btp, XMBUF_BLOCKSIZE, descr); + if (error) + goto out_file; + + trace_xmbuf_create(btp); + + *btpp = btp; + return 0; + +out_file: + fput(file); +out_free_btp: + kfree(btp); + return error; +} + +/* Free a buffer cache target for a memory-backed buffer cache. */ +void +xmbuf_free( + struct xfs_buftarg *btp) +{ + ASSERT(xfs_buftarg_is_mem(btp)); + ASSERT(percpu_counter_sum(&btp->bt_readahead_count) == 0); + + trace_xmbuf_free(btp); + + xfs_destroy_buftarg(btp); + fput(btp->bt_file); + kfree(btp); +} + +/* Directly map a shmem folio into the buffer cache. */ +int +xmbuf_map_backing_mem( + struct xfs_buf *bp) +{ + struct inode *inode = file_inode(bp->b_target->bt_file); + struct folio *folio = NULL; + loff_t pos = BBTOB(xfs_buf_daddr(bp)); + int error; + + ASSERT(xfs_buftarg_is_mem(bp->b_target)); + + if (bp->b_map_count != 1) + return -ENOMEM; + if (BBTOB(bp->b_length) != XMBUF_BLOCKSIZE) + return -ENOMEM; + if (offset_in_page(pos) != 0) { + ASSERT(offset_in_page(pos)); + return -ENOMEM; + } + + error = shmem_get_folio(inode, pos >> PAGE_SHIFT, 0, &folio, SGP_CACHE); + if (error) + return error; + + if (filemap_check_wb_err(inode->i_mapping, 0)) { + folio_unlock(folio); + folio_put(folio); + return -EIO; + } + + /* + * Mark the folio dirty so that it won't be reclaimed once we drop the + * (potentially last) reference in xfs_buf_free. + */ + folio_set_dirty(folio); + folio_unlock(folio); + + bp->b_addr = folio_address(folio) + offset_in_folio(folio, pos); + return 0; +} + +/* Is this a valid daddr within the buftarg? */ +bool +xmbuf_verify_daddr( + struct xfs_buftarg *btp, + xfs_daddr_t daddr) +{ + struct inode *inode = file_inode(btp->bt_file); + + ASSERT(xfs_buftarg_is_mem(btp)); + + return daddr < (inode->i_sb->s_maxbytes >> BBSHIFT); +} + +/* Discard the folio backing this buffer. */ +static void +xmbuf_stale( + struct xfs_buf *bp) +{ + struct inode *inode = file_inode(bp->b_target->bt_file); + loff_t pos; + + ASSERT(xfs_buftarg_is_mem(bp->b_target)); + + pos = BBTOB(xfs_buf_daddr(bp)); + shmem_truncate_range(inode, pos, pos + BBTOB(bp->b_length) - 1); +} + +/* + * Finalize a buffer -- discard the backing folio if it's stale, or run the + * write verifier to detect problems. + */ +int +xmbuf_finalize( + struct xfs_buf *bp) +{ + xfs_failaddr_t fa; + int error = 0; + + if (bp->b_flags & XBF_STALE) { + xmbuf_stale(bp); + return 0; + } + + /* + * Although this btree is ephemeral, validate the buffer structure so + * that we can detect memory corruption errors and software bugs. + */ + fa = bp->b_ops->verify_struct(bp); + if (fa) { + error = -EFSCORRUPTED; + xfs_verifier_error(bp, error, fa); + } + + return error; +} + +/* + * Detach this xmbuf buffer from the transaction by any means necessary. + * All buffers are direct-mapped, so they do not need bwrite. + */ +void +xmbuf_trans_bdetach( + struct xfs_trans *tp, + struct xfs_buf *bp) +{ + struct xfs_buf_log_item *bli = bp->b_log_item; + + ASSERT(bli != NULL); + + bli->bli_flags &= ~(XFS_BLI_DIRTY | XFS_BLI_ORDERED | + XFS_BLI_LOGGED | XFS_BLI_STALE); + clear_bit(XFS_LI_DIRTY, &bli->bli_item.li_flags); + + while (bp->b_log_item != NULL) + xfs_trans_bdetach(tp, bp); +} diff --git a/fs/xfs/xfs_buf_mem.h b/fs/xfs/xfs_buf_mem.h new file mode 100644 index 000000000..67d525cc1 --- /dev/null +++ b/fs/xfs/xfs_buf_mem.h @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2023-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_BUF_MEM_H__ +#define __XFS_BUF_MEM_H__ + +#define XMBUF_BLOCKSIZE (PAGE_SIZE) +#define XMBUF_BLOCKSHIFT (PAGE_SHIFT) + +#ifdef CONFIG_XFS_MEMORY_BUFS +static inline bool xfs_buftarg_is_mem(const struct xfs_buftarg *btp) +{ + return btp->bt_bdev == NULL; +} + +int xmbuf_alloc(struct xfs_mount *mp, const char *descr, + struct xfs_buftarg **btpp); +void xmbuf_free(struct xfs_buftarg *btp); + +bool xmbuf_verify_daddr(struct xfs_buftarg *btp, xfs_daddr_t daddr); +void xmbuf_trans_bdetach(struct xfs_trans *tp, struct xfs_buf *bp); +int xmbuf_finalize(struct xfs_buf *bp); +#else +# define xfs_buftarg_is_mem(...) (false) +# define xmbuf_verify_daddr(...) (false) +#endif /* CONFIG_XFS_MEMORY_BUFS */ + +int xmbuf_map_backing_mem(struct xfs_buf *bp); + +#endif /* __XFS_BUF_MEM_H__ */ diff --git a/fs/xfs/xfs_dahash_test.c b/fs/xfs/xfs_dahash_test.c new file mode 100644 index 000000000..f1ee2643b --- /dev/null +++ b/fs/xfs/xfs_dahash_test.c @@ -0,0 +1,673 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_dir2_priv.h" +#include "xfs_dahash_test.h" + +/* 4096 random bytes */ +static uint8_t __initdata __attribute__((__aligned__(8))) test_buf[] = +{ + 0x5b, 0x85, 0x21, 0xcb, 0x09, 0x68, 0x7d, 0x30, + 0xc7, 0x69, 0xd7, 0x30, 0x92, 0xde, 0x59, 0xe4, + 0xc9, 0x6e, 0x8b, 0xdb, 0x98, 0x6b, 0xaa, 0x60, + 0xa8, 0xb5, 0xbc, 0x6c, 0xa9, 0xb1, 0x5b, 0x2c, + 0xea, 0xb4, 0x92, 0x6a, 0x3f, 0x79, 0x91, 0xe4, + 0xe9, 0x70, 0x51, 0x8c, 0x7f, 0x95, 0x6f, 0x1a, + 0x56, 0xa1, 0x5c, 0x27, 0x03, 0x67, 0x9f, 0x3a, + 0xe2, 0x31, 0x11, 0x29, 0x6b, 0x98, 0xfc, 0xc4, + 0x53, 0x24, 0xc5, 0x8b, 0xce, 0x47, 0xb2, 0xb9, + 0x32, 0xcb, 0xc1, 0xd0, 0x03, 0x57, 0x4e, 0xd4, + 0xe9, 0x3c, 0xa1, 0x63, 0xcf, 0x12, 0x0e, 0xca, + 0xe1, 0x13, 0xd1, 0x93, 0xa6, 0x88, 0x5c, 0x61, + 0x5b, 0xbb, 0xf0, 0x19, 0x46, 0xb4, 0xcf, 0x9e, + 0xb6, 0x6b, 0x4c, 0x3a, 0xcf, 0x60, 0xf9, 0x7a, + 0x8d, 0x07, 0x63, 0xdb, 0x40, 0xe9, 0x0b, 0x6f, + 0xad, 0x97, 0xf1, 0xed, 0xd0, 0x1e, 0x26, 0xfd, + 0xbf, 0xb7, 0xc8, 0x04, 0x94, 0xf8, 0x8b, 0x8c, + 0xf1, 0xab, 0x7a, 0xd4, 0xdd, 0xf3, 0xe8, 0x88, + 0xc3, 0xed, 0x17, 0x8a, 0x9b, 0x40, 0x0d, 0x53, + 0x62, 0x12, 0x03, 0x5f, 0x1b, 0x35, 0x32, 0x1f, + 0xb4, 0x7b, 0x93, 0x78, 0x0d, 0xdb, 0xce, 0xa4, + 0xc0, 0x47, 0xd5, 0xbf, 0x68, 0xe8, 0x5d, 0x74, + 0x8f, 0x8e, 0x75, 0x1c, 0xb2, 0x4f, 0x9a, 0x60, + 0xd1, 0xbe, 0x10, 0xf4, 0x5c, 0xa1, 0x53, 0x09, + 0xa5, 0xe0, 0x09, 0x54, 0x85, 0x5c, 0xdc, 0x07, + 0xe7, 0x21, 0x69, 0x7b, 0x8a, 0xfd, 0x90, 0xf1, + 0x22, 0xd0, 0xb4, 0x36, 0x28, 0xe6, 0xb8, 0x0f, + 0x39, 0xde, 0xc8, 0xf3, 0x86, 0x60, 0x34, 0xd2, + 0x5e, 0xdf, 0xfd, 0xcf, 0x0f, 0xa9, 0x65, 0xf0, + 0xd5, 0x4d, 0x96, 0x40, 0xe3, 0xdf, 0x3f, 0x95, + 0x5a, 0x39, 0x19, 0x93, 0xf4, 0x75, 0xce, 0x22, + 0x00, 0x1c, 0x93, 0xe2, 0x03, 0x66, 0xf4, 0x93, + 0x73, 0x86, 0x81, 0x8e, 0x29, 0x44, 0x48, 0x86, + 0x61, 0x7c, 0x48, 0xa3, 0x43, 0xd2, 0x9c, 0x8d, + 0xd4, 0x95, 0xdd, 0xe1, 0x22, 0x89, 0x3a, 0x40, + 0x4c, 0x1b, 0x8a, 0x04, 0xa8, 0x09, 0x69, 0x8b, + 0xea, 0xc6, 0x55, 0x8e, 0x57, 0xe6, 0x64, 0x35, + 0xf0, 0xc7, 0x16, 0x9f, 0x5d, 0x5e, 0x86, 0x40, + 0x46, 0xbb, 0xe5, 0x45, 0x88, 0xfe, 0xc9, 0x63, + 0x15, 0xfb, 0xf5, 0xbd, 0x71, 0x61, 0xeb, 0x7b, + 0x78, 0x70, 0x07, 0x31, 0x03, 0x9f, 0xb2, 0xc8, + 0xa7, 0xab, 0x47, 0xfd, 0xdf, 0xa0, 0x78, 0x72, + 0xa4, 0x2a, 0xe4, 0xb6, 0xba, 0xc0, 0x1e, 0x86, + 0x71, 0xe6, 0x3d, 0x18, 0x37, 0x70, 0xe6, 0xff, + 0xe0, 0xbc, 0x0b, 0x22, 0xa0, 0x1f, 0xd3, 0xed, + 0xa2, 0x55, 0x39, 0xab, 0xa8, 0x13, 0x73, 0x7c, + 0x3f, 0xb2, 0xd6, 0x19, 0xac, 0xff, 0x99, 0xed, + 0xe8, 0xe6, 0xa6, 0x22, 0xe3, 0x9c, 0xf1, 0x30, + 0xdc, 0x01, 0x0a, 0x56, 0xfa, 0xe4, 0xc9, 0x99, + 0xdd, 0xa8, 0xd8, 0xda, 0x35, 0x51, 0x73, 0xb4, + 0x40, 0x86, 0x85, 0xdb, 0x5c, 0xd5, 0x85, 0x80, + 0x14, 0x9c, 0xfd, 0x98, 0xa9, 0x82, 0xc5, 0x37, + 0xff, 0x32, 0x5d, 0xd0, 0x0b, 0xfa, 0xdc, 0x04, + 0x5e, 0x09, 0xd2, 0xca, 0x17, 0x4b, 0x1a, 0x8e, + 0x15, 0xe1, 0xcc, 0x4e, 0x52, 0x88, 0x35, 0xbd, + 0x48, 0xfe, 0x15, 0xa0, 0x91, 0xfd, 0x7e, 0x6c, + 0x0e, 0x5d, 0x79, 0x1b, 0x81, 0x79, 0xd2, 0x09, + 0x34, 0x70, 0x3d, 0x81, 0xec, 0xf6, 0x24, 0xbb, + 0xfb, 0xf1, 0x7b, 0xdf, 0x54, 0xea, 0x80, 0x9b, + 0xc7, 0x99, 0x9e, 0xbd, 0x16, 0x78, 0x12, 0x53, + 0x5e, 0x01, 0xa7, 0x4e, 0xbd, 0x67, 0xe1, 0x9b, + 0x4c, 0x0e, 0x61, 0x45, 0x97, 0xd2, 0xf0, 0x0f, + 0xfe, 0x15, 0x08, 0xb7, 0x11, 0x4c, 0xe7, 0xff, + 0x81, 0x53, 0xff, 0x91, 0x25, 0x38, 0x7e, 0x40, + 0x94, 0xe5, 0xe0, 0xad, 0xe6, 0xd9, 0x79, 0xb6, + 0x92, 0xc9, 0xfc, 0xde, 0xc3, 0x1a, 0x23, 0xbb, + 0xdd, 0xc8, 0x51, 0x0c, 0x3a, 0x72, 0xfa, 0x73, + 0x6f, 0xb7, 0xee, 0x61, 0x39, 0x03, 0x01, 0x3f, + 0x7f, 0x94, 0x2e, 0x2e, 0xba, 0x3a, 0xbb, 0xb4, + 0xfa, 0x6a, 0x17, 0xfe, 0xea, 0xef, 0x5e, 0x66, + 0x97, 0x3f, 0x32, 0x3d, 0xd7, 0x3e, 0xb1, 0xf1, + 0x6c, 0x14, 0x4c, 0xfd, 0x37, 0xd3, 0x38, 0x80, + 0xfb, 0xde, 0xa6, 0x24, 0x1e, 0xc8, 0xca, 0x7f, + 0x3a, 0x93, 0xd8, 0x8b, 0x18, 0x13, 0xb2, 0xe5, + 0xe4, 0x93, 0x05, 0x53, 0x4f, 0x84, 0x66, 0xa7, + 0x58, 0x5c, 0x7b, 0x86, 0x52, 0x6d, 0x0d, 0xce, + 0xa4, 0x30, 0x7d, 0xb6, 0x18, 0x9f, 0xeb, 0xff, + 0x22, 0xbb, 0x72, 0x29, 0xb9, 0x44, 0x0b, 0x48, + 0x1e, 0x84, 0x71, 0x81, 0xe3, 0x6d, 0x73, 0x26, + 0x92, 0xb4, 0x4d, 0x2a, 0x29, 0xb8, 0x1f, 0x72, + 0xed, 0xd0, 0xe1, 0x64, 0x77, 0xea, 0x8e, 0x88, + 0x0f, 0xef, 0x3f, 0xb1, 0x3b, 0xad, 0xf9, 0xc9, + 0x8b, 0xd0, 0xac, 0xc6, 0xcc, 0xa9, 0x40, 0xcc, + 0x76, 0xf6, 0x3b, 0x53, 0xb5, 0x88, 0xcb, 0xc8, + 0x37, 0xf1, 0xa2, 0xba, 0x23, 0x15, 0x99, 0x09, + 0xcc, 0xe7, 0x7a, 0x3b, 0x37, 0xf7, 0x58, 0xc8, + 0x46, 0x8c, 0x2b, 0x2f, 0x4e, 0x0e, 0xa6, 0x5c, + 0xea, 0x85, 0x55, 0xba, 0x02, 0x0e, 0x0e, 0x48, + 0xbc, 0xe1, 0xb1, 0x01, 0x35, 0x79, 0x13, 0x3d, + 0x1b, 0xc0, 0x53, 0x68, 0x11, 0xe7, 0x95, 0x0f, + 0x9d, 0x3f, 0x4c, 0x47, 0x7b, 0x4d, 0x1c, 0xae, + 0x50, 0x9b, 0xcb, 0xdd, 0x05, 0x8d, 0x9a, 0x97, + 0xfd, 0x8c, 0xef, 0x0c, 0x1d, 0x67, 0x73, 0xa8, + 0x28, 0x36, 0xd5, 0xb6, 0x92, 0x33, 0x40, 0x75, + 0x0b, 0x51, 0xc3, 0x64, 0xba, 0x1d, 0xc2, 0xcc, + 0xee, 0x7d, 0x54, 0x0f, 0x27, 0x69, 0xa7, 0x27, + 0x63, 0x30, 0x29, 0xd9, 0xc8, 0x84, 0xd8, 0xdf, + 0x9f, 0x68, 0x8d, 0x04, 0xca, 0xa6, 0xc5, 0xc7, + 0x7a, 0x5c, 0xc8, 0xd1, 0xcb, 0x4a, 0xec, 0xd0, + 0xd8, 0x20, 0x69, 0xc5, 0x17, 0xcd, 0x78, 0xc8, + 0x75, 0x23, 0x30, 0x69, 0xc9, 0xd4, 0xea, 0x5c, + 0x4f, 0x6b, 0x86, 0x3f, 0x8b, 0xfe, 0xee, 0x44, + 0xc9, 0x7c, 0xb7, 0xdd, 0x3e, 0xe5, 0xec, 0x54, + 0x03, 0x3e, 0xaa, 0x82, 0xc6, 0xdf, 0xb2, 0x38, + 0x0e, 0x5d, 0xb3, 0x88, 0xd9, 0xd3, 0x69, 0x5f, + 0x8f, 0x70, 0x8a, 0x7e, 0x11, 0xd9, 0x1e, 0x7b, + 0x38, 0xf1, 0x42, 0x1a, 0xc0, 0x35, 0xf5, 0xc7, + 0x36, 0x85, 0xf5, 0xf7, 0xb8, 0x7e, 0xc7, 0xef, + 0x18, 0xf1, 0x63, 0xd6, 0x7a, 0xc6, 0xc9, 0x0e, + 0x4d, 0x69, 0x4f, 0x84, 0xef, 0x26, 0x41, 0x0c, + 0xec, 0xc7, 0xe0, 0x7e, 0x3c, 0x67, 0x01, 0x4c, + 0x62, 0x1a, 0x20, 0x6f, 0xee, 0x47, 0x4d, 0xc0, + 0x99, 0x13, 0x8d, 0x91, 0x4a, 0x26, 0xd4, 0x37, + 0x28, 0x90, 0x58, 0x75, 0x66, 0x2b, 0x0a, 0xdf, + 0xda, 0xee, 0x92, 0x25, 0x90, 0x62, 0x39, 0x9e, + 0x44, 0x98, 0xad, 0xc1, 0x88, 0xed, 0xe4, 0xb4, + 0xaf, 0xf5, 0x8c, 0x9b, 0x48, 0x4d, 0x56, 0x60, + 0x97, 0x0f, 0x61, 0x59, 0x9e, 0xa6, 0x27, 0xfe, + 0xc1, 0x91, 0x15, 0x38, 0xb8, 0x0f, 0xae, 0x61, + 0x7d, 0x26, 0x13, 0x5a, 0x73, 0xff, 0x1c, 0xa3, + 0x61, 0x04, 0x58, 0x48, 0x55, 0x44, 0x11, 0xfe, + 0x15, 0xca, 0xc3, 0xbd, 0xca, 0xc5, 0xb4, 0x40, + 0x5d, 0x1b, 0x7f, 0x39, 0xb5, 0x9c, 0x35, 0xec, + 0x61, 0x15, 0x32, 0x32, 0xb8, 0x4e, 0x40, 0x9f, + 0x17, 0x1f, 0x0a, 0x4d, 0xa9, 0x91, 0xef, 0xb7, + 0xb0, 0xeb, 0xc2, 0x83, 0x9a, 0x6c, 0xd2, 0x79, + 0x43, 0x78, 0x5e, 0x2f, 0xe5, 0xdd, 0x1a, 0x3c, + 0x45, 0xab, 0x29, 0x40, 0x3a, 0x37, 0x5b, 0x6f, + 0xd7, 0xfc, 0x48, 0x64, 0x3c, 0x49, 0xfb, 0x21, + 0xbe, 0xc3, 0xff, 0x07, 0xfb, 0x17, 0xe9, 0xc9, + 0x0c, 0x4c, 0x5c, 0x15, 0x9e, 0x8e, 0x22, 0x30, + 0x0a, 0xde, 0x48, 0x7f, 0xdb, 0x0d, 0xd1, 0x2b, + 0x87, 0x38, 0x9e, 0xcc, 0x5a, 0x01, 0x16, 0xee, + 0x75, 0x49, 0x0d, 0x30, 0x01, 0x34, 0x6a, 0xb6, + 0x9a, 0x5a, 0x2a, 0xec, 0xbb, 0x48, 0xac, 0xd3, + 0x77, 0x83, 0xd8, 0x08, 0x86, 0x4f, 0x48, 0x09, + 0x29, 0x41, 0x79, 0xa1, 0x03, 0x12, 0xc4, 0xcd, + 0x90, 0x55, 0x47, 0x66, 0x74, 0x9a, 0xcc, 0x4f, + 0x35, 0x8c, 0xd6, 0x98, 0xef, 0xeb, 0x45, 0xb9, + 0x9a, 0x26, 0x2f, 0x39, 0xa5, 0x70, 0x6d, 0xfc, + 0xb4, 0x51, 0xee, 0xf4, 0x9c, 0xe7, 0x38, 0x59, + 0xad, 0xf4, 0xbc, 0x46, 0xff, 0x46, 0x8e, 0x60, + 0x9c, 0xa3, 0x60, 0x1d, 0xf8, 0x26, 0x72, 0xf5, + 0x72, 0x9d, 0x68, 0x80, 0x04, 0xf6, 0x0b, 0xa1, + 0x0a, 0xd5, 0xa7, 0x82, 0x3a, 0x3e, 0x47, 0xa8, + 0x5a, 0xde, 0x59, 0x4f, 0x7b, 0x07, 0xb3, 0xe9, + 0x24, 0x19, 0x3d, 0x34, 0x05, 0xec, 0xf1, 0xab, + 0x6e, 0x64, 0x8f, 0xd3, 0xe6, 0x41, 0x86, 0x80, + 0x70, 0xe3, 0x8d, 0x60, 0x9c, 0x34, 0x25, 0x01, + 0x07, 0x4d, 0x19, 0x41, 0x4e, 0x3d, 0x5c, 0x7e, + 0xa8, 0xf5, 0xcc, 0xd5, 0x7b, 0xe2, 0x7d, 0x3d, + 0x49, 0x86, 0x7d, 0x07, 0xb7, 0x10, 0xe3, 0x35, + 0xb8, 0x84, 0x6d, 0x76, 0xab, 0x17, 0xc6, 0x38, + 0xb4, 0xd3, 0x28, 0x57, 0xad, 0xd3, 0x88, 0x5a, + 0xda, 0xea, 0xc8, 0x94, 0xcc, 0x37, 0x19, 0xac, + 0x9c, 0x9f, 0x4b, 0x00, 0x15, 0xc0, 0xc8, 0xca, + 0x1f, 0x15, 0xaa, 0xe0, 0xdb, 0xf9, 0x2f, 0x57, + 0x1b, 0x24, 0xc7, 0x6f, 0x76, 0x29, 0xfb, 0xed, + 0x25, 0x0d, 0xc0, 0xfe, 0xbd, 0x5a, 0xbf, 0x20, + 0x08, 0x51, 0x05, 0xec, 0x71, 0xa3, 0xbf, 0xef, + 0x5e, 0x99, 0x75, 0xdb, 0x3c, 0x5f, 0x9a, 0x8c, + 0xbb, 0x19, 0x5c, 0x0e, 0x93, 0x19, 0xf8, 0x6a, + 0xbc, 0xf2, 0x12, 0x54, 0x2f, 0xcb, 0x28, 0x64, + 0x88, 0xb3, 0x92, 0x0d, 0x96, 0xd1, 0xa6, 0xe4, + 0x1f, 0xf1, 0x4d, 0xa4, 0xab, 0x1c, 0xee, 0x54, + 0xf2, 0xad, 0x29, 0x6d, 0x32, 0x37, 0xb2, 0x16, + 0x77, 0x5c, 0xdc, 0x2e, 0x54, 0xec, 0x75, 0x26, + 0xc6, 0x36, 0xd9, 0x17, 0x2c, 0xf1, 0x7a, 0xdc, + 0x4b, 0xf1, 0xe2, 0xd9, 0x95, 0xba, 0xac, 0x87, + 0xc1, 0xf3, 0x8e, 0x58, 0x08, 0xd8, 0x87, 0x60, + 0xc9, 0xee, 0x6a, 0xde, 0xa4, 0xd2, 0xfc, 0x0d, + 0xe5, 0x36, 0xc4, 0x5c, 0x52, 0xb3, 0x07, 0x54, + 0x65, 0x24, 0xc1, 0xb1, 0xd1, 0xb1, 0x53, 0x13, + 0x31, 0x79, 0x7f, 0x05, 0x76, 0xeb, 0x37, 0x59, + 0x15, 0x2b, 0xd1, 0x3f, 0xac, 0x08, 0x97, 0xeb, + 0x91, 0x98, 0xdf, 0x6c, 0x09, 0x0d, 0x04, 0x9f, + 0xdc, 0x3b, 0x0e, 0x60, 0x68, 0x47, 0x23, 0x15, + 0x16, 0xc6, 0x0b, 0x35, 0xf8, 0x77, 0xa2, 0x78, + 0x50, 0xd4, 0x64, 0x22, 0x33, 0xff, 0xfb, 0x93, + 0x71, 0x46, 0x50, 0x39, 0x1b, 0x9c, 0xea, 0x4e, + 0x8d, 0x0c, 0x37, 0xe5, 0x5c, 0x51, 0x3a, 0x31, + 0xb2, 0x85, 0x84, 0x3f, 0x41, 0xee, 0xa2, 0xc1, + 0xc6, 0x13, 0x3b, 0x54, 0x28, 0xd2, 0x18, 0x37, + 0xcc, 0x46, 0x9f, 0x6a, 0x91, 0x3d, 0x5a, 0x15, + 0x3c, 0x89, 0xa3, 0x61, 0x06, 0x7d, 0x2e, 0x78, + 0xbe, 0x7d, 0x40, 0xba, 0x2f, 0x95, 0xb1, 0x2f, + 0x87, 0x3b, 0x8a, 0xbe, 0x6a, 0xf4, 0xc2, 0x31, + 0x74, 0xee, 0x91, 0xe0, 0x23, 0xaa, 0x5d, 0x7f, + 0xdd, 0xf0, 0x44, 0x8c, 0x0b, 0x59, 0x2b, 0xfc, + 0x48, 0x3a, 0xdf, 0x07, 0x05, 0x38, 0x6c, 0xc9, + 0xeb, 0x18, 0x24, 0x68, 0x8d, 0x58, 0x98, 0xd3, + 0x31, 0xa3, 0xe4, 0x70, 0x59, 0xb1, 0x21, 0xbe, + 0x7e, 0x65, 0x7d, 0xb8, 0x04, 0xab, 0xf6, 0xe4, + 0xd7, 0xda, 0xec, 0x09, 0x8f, 0xda, 0x6d, 0x24, + 0x07, 0xcc, 0x29, 0x17, 0x05, 0x78, 0x1a, 0xc1, + 0xb1, 0xce, 0xfc, 0xaa, 0x2d, 0xe7, 0xcc, 0x85, + 0x84, 0x84, 0x03, 0x2a, 0x0c, 0x3f, 0xa9, 0xf8, + 0xfd, 0x84, 0x53, 0x59, 0x5c, 0xf0, 0xd4, 0x09, + 0xf0, 0xd2, 0x6c, 0x32, 0x03, 0xb0, 0xa0, 0x8c, + 0x52, 0xeb, 0x23, 0x91, 0x88, 0x43, 0x13, 0x46, + 0xf6, 0x1e, 0xb4, 0x1b, 0xf5, 0x8e, 0x3a, 0xb5, + 0x3d, 0x00, 0xf6, 0xe5, 0x08, 0x3d, 0x5f, 0x39, + 0xd3, 0x21, 0x69, 0xbc, 0x03, 0x22, 0x3a, 0xd2, + 0x5c, 0x84, 0xf8, 0x15, 0xc4, 0x80, 0x0b, 0xbc, + 0x29, 0x3c, 0xf3, 0x95, 0x98, 0xcd, 0x8f, 0x35, + 0xbc, 0xa5, 0x3e, 0xfc, 0xd4, 0x13, 0x9e, 0xde, + 0x4f, 0xce, 0x71, 0x9d, 0x09, 0xad, 0xf2, 0x80, + 0x6b, 0x65, 0x7f, 0x03, 0x00, 0x14, 0x7c, 0x15, + 0x85, 0x40, 0x6d, 0x70, 0xea, 0xdc, 0xb3, 0x63, + 0x35, 0x4f, 0x4d, 0xe0, 0xd9, 0xd5, 0x3c, 0x58, + 0x56, 0x23, 0x80, 0xe2, 0x36, 0xdd, 0x75, 0x1d, + 0x94, 0x11, 0x41, 0x8e, 0xe0, 0x81, 0x8e, 0xcf, + 0xe0, 0xe5, 0xf6, 0xde, 0xd1, 0xe7, 0x04, 0x12, + 0x79, 0x92, 0x2b, 0x71, 0x2a, 0x79, 0x8b, 0x7c, + 0x44, 0x79, 0x16, 0x30, 0x4e, 0xf4, 0xf6, 0x9b, + 0xb7, 0x40, 0xa3, 0x5a, 0xa7, 0x69, 0x3e, 0xc1, + 0x3a, 0x04, 0xd0, 0x88, 0xa0, 0x3b, 0xdd, 0xc6, + 0x9e, 0x7e, 0x1e, 0x1e, 0x8f, 0x44, 0xf7, 0x73, + 0x67, 0x1e, 0x1a, 0x78, 0xfa, 0x62, 0xf4, 0xa9, + 0xa8, 0xc6, 0x5b, 0xb8, 0xfa, 0x06, 0x7d, 0x5e, + 0x38, 0x1c, 0x9a, 0x39, 0xe9, 0x39, 0x98, 0x22, + 0x0b, 0xa7, 0xac, 0x0b, 0xf3, 0xbc, 0xf1, 0xeb, + 0x8c, 0x81, 0xe3, 0x48, 0x8a, 0xed, 0x42, 0xc2, + 0x38, 0xcf, 0x3e, 0xda, 0xd2, 0x89, 0x8d, 0x9c, + 0x53, 0xb5, 0x2f, 0x41, 0x01, 0x26, 0x84, 0x9c, + 0xa3, 0x56, 0xf6, 0x49, 0xc7, 0xd4, 0x9f, 0x93, + 0x1b, 0x96, 0x49, 0x5e, 0xad, 0xb3, 0x84, 0x1f, + 0x3c, 0xa4, 0xe0, 0x9b, 0xd1, 0x90, 0xbc, 0x38, + 0x6c, 0xdd, 0x95, 0x4d, 0x9d, 0xb1, 0x71, 0x57, + 0x2d, 0x34, 0xe8, 0xb8, 0x42, 0xc7, 0x99, 0x03, + 0xc7, 0x07, 0x30, 0x65, 0x91, 0x55, 0xd5, 0x90, + 0x70, 0x97, 0x37, 0x68, 0xd4, 0x11, 0xf9, 0xe8, + 0xce, 0xec, 0xdc, 0x34, 0xd5, 0xd3, 0xb7, 0xc4, + 0xb8, 0x97, 0x05, 0x92, 0xad, 0xf8, 0xe2, 0x36, + 0x64, 0x41, 0xc9, 0xc5, 0x41, 0x77, 0x52, 0xd7, + 0x2c, 0xa5, 0x24, 0x2f, 0xd9, 0x34, 0x0b, 0x47, + 0x35, 0xa7, 0x28, 0x8b, 0xc5, 0xcd, 0xe9, 0x46, + 0xac, 0x39, 0x94, 0x3c, 0x10, 0xc6, 0x29, 0x73, + 0x0e, 0x0e, 0x5d, 0xe0, 0x71, 0x03, 0x8a, 0x72, + 0x0e, 0x26, 0xb0, 0x7d, 0x84, 0xed, 0x95, 0x23, + 0x49, 0x5a, 0x45, 0x83, 0x45, 0x60, 0x11, 0x4a, + 0x46, 0x31, 0xd4, 0xd8, 0x16, 0x54, 0x98, 0x58, + 0xed, 0x6d, 0xcc, 0x5d, 0xd6, 0x50, 0x61, 0x9f, + 0x9d, 0xc5, 0x3e, 0x9d, 0x32, 0x47, 0xde, 0x96, + 0xe1, 0x5d, 0xd8, 0xf8, 0xb4, 0x69, 0x6f, 0xb9, + 0x15, 0x90, 0x57, 0x7a, 0xf6, 0xad, 0xb0, 0x5b, + 0xf5, 0xa6, 0x36, 0x94, 0xfd, 0x84, 0xce, 0x1c, + 0x0f, 0x4b, 0xd0, 0xc2, 0x5b, 0x6b, 0x56, 0xef, + 0x73, 0x93, 0x0b, 0xc3, 0xee, 0xd9, 0xcf, 0xd3, + 0xa4, 0x22, 0x58, 0xcd, 0x50, 0x6e, 0x65, 0xf4, + 0xe9, 0xb7, 0x71, 0xaf, 0x4b, 0xb3, 0xb6, 0x2f, + 0x0f, 0x0e, 0x3b, 0xc9, 0x85, 0x14, 0xf5, 0x17, + 0xe8, 0x7a, 0x3a, 0xbf, 0x5f, 0x5e, 0xf8, 0x18, + 0x48, 0xa6, 0x72, 0xab, 0x06, 0x95, 0xe9, 0xc8, + 0xa7, 0xf4, 0x32, 0x44, 0x04, 0x0c, 0x84, 0x98, + 0x73, 0xe3, 0x89, 0x8d, 0x5f, 0x7e, 0x4a, 0x42, + 0x8f, 0xc5, 0x28, 0xb1, 0x82, 0xef, 0x1c, 0x97, + 0x31, 0x3b, 0x4d, 0xe0, 0x0e, 0x10, 0x10, 0x97, + 0x93, 0x49, 0x78, 0x2f, 0x0d, 0x86, 0x8b, 0xa1, + 0x53, 0xa9, 0x81, 0x20, 0x79, 0xe7, 0x07, 0x77, + 0xb6, 0xac, 0x5e, 0xd2, 0x05, 0xcd, 0xe9, 0xdb, + 0x8a, 0x94, 0x82, 0x8a, 0x23, 0xb9, 0x3d, 0x1c, + 0xa9, 0x7d, 0x72, 0x4a, 0xed, 0x33, 0xa3, 0xdb, + 0x21, 0xa7, 0x86, 0x33, 0x45, 0xa5, 0xaa, 0x56, + 0x45, 0xb5, 0x83, 0x29, 0x40, 0x47, 0x79, 0x04, + 0x6e, 0xb9, 0x95, 0xd0, 0x81, 0x77, 0x2d, 0x48, + 0x1e, 0xfe, 0xc3, 0xc2, 0x1e, 0xe5, 0xf2, 0xbe, + 0xfd, 0x3b, 0x94, 0x9f, 0xc4, 0xc4, 0x26, 0x9d, + 0xe4, 0x66, 0x1e, 0x19, 0xee, 0x6c, 0x79, 0x97, + 0x11, 0x31, 0x4b, 0x0d, 0x01, 0xcb, 0xde, 0xa8, + 0xf6, 0x6d, 0x7c, 0x39, 0x46, 0x4e, 0x7e, 0x3f, + 0x94, 0x17, 0xdf, 0xa1, 0x7d, 0xd9, 0x1c, 0x8e, + 0xbc, 0x7d, 0x33, 0x7d, 0xe3, 0x12, 0x40, 0xca, + 0xab, 0x37, 0x11, 0x46, 0xd4, 0xae, 0xef, 0x44, + 0xa2, 0xb3, 0x6a, 0x66, 0x0e, 0x0c, 0x90, 0x7f, + 0xdf, 0x5c, 0x66, 0x5f, 0xf2, 0x94, 0x9f, 0xa6, + 0x73, 0x4f, 0xeb, 0x0d, 0xad, 0xbf, 0xc0, 0x63, + 0x5c, 0xdc, 0x46, 0x51, 0xe8, 0x8e, 0x90, 0x19, + 0xa8, 0xa4, 0x3c, 0x91, 0x79, 0xfa, 0x7e, 0x58, + 0x85, 0x13, 0x55, 0xc5, 0x19, 0x82, 0x37, 0x1b, + 0x0a, 0x02, 0x1f, 0x99, 0x6b, 0x18, 0xf1, 0x28, + 0x08, 0xa2, 0x73, 0xb8, 0x0f, 0x2e, 0xcd, 0xbf, + 0xf3, 0x86, 0x7f, 0xea, 0xef, 0xd0, 0xbb, 0xa6, + 0x21, 0xdf, 0x49, 0x73, 0x51, 0xcc, 0x36, 0xd3, + 0x3e, 0xa0, 0xf8, 0x44, 0xdf, 0xd3, 0xa6, 0xbe, + 0x8a, 0xd4, 0x57, 0xdd, 0x72, 0x94, 0x61, 0x0f, + 0x82, 0xd1, 0x07, 0xb8, 0x7c, 0x18, 0x83, 0xdf, + 0x3a, 0xe5, 0x50, 0x6a, 0x82, 0x20, 0xac, 0xa9, + 0xa8, 0xff, 0xd9, 0xf3, 0x77, 0x33, 0x5a, 0x9e, + 0x7f, 0x6d, 0xfe, 0x5d, 0x33, 0x41, 0x42, 0xe7, + 0x6c, 0x19, 0xe0, 0x44, 0x8a, 0x15, 0xf6, 0x70, + 0x98, 0xb7, 0x68, 0x4d, 0xfa, 0x97, 0x39, 0xb0, + 0x8e, 0xe8, 0x84, 0x8b, 0x75, 0x30, 0xb7, 0x7d, + 0x92, 0x69, 0x20, 0x9c, 0x81, 0xfb, 0x4b, 0xf4, + 0x01, 0x50, 0xeb, 0xce, 0x0c, 0x1c, 0x6c, 0xb5, + 0x4a, 0xd7, 0x27, 0x0c, 0xce, 0xbb, 0xe5, 0x85, + 0xf0, 0xb6, 0xee, 0xd5, 0x70, 0xdd, 0x3b, 0xfc, + 0xd4, 0x99, 0xf1, 0x33, 0xdd, 0x8b, 0xc4, 0x2f, + 0xae, 0xab, 0x74, 0x96, 0x32, 0xc7, 0x4c, 0x56, + 0x3c, 0x89, 0x0f, 0x96, 0x0b, 0x42, 0xc0, 0xcb, + 0xee, 0x0f, 0x0b, 0x8c, 0xfb, 0x7e, 0x47, 0x7b, + 0x64, 0x48, 0xfd, 0xb2, 0x00, 0x80, 0x89, 0xa5, + 0x13, 0x55, 0x62, 0xfc, 0x8f, 0xe2, 0x42, 0x03, + 0xb7, 0x4e, 0x2a, 0x79, 0xb4, 0x82, 0xea, 0x23, + 0x49, 0xda, 0xaf, 0x52, 0x63, 0x1e, 0x60, 0x03, + 0x89, 0x06, 0x44, 0x46, 0x08, 0xc3, 0xc4, 0x87, + 0x70, 0x2e, 0xda, 0x94, 0xad, 0x6b, 0xe0, 0xe4, + 0xd1, 0x8a, 0x06, 0xc2, 0xa8, 0xc0, 0xa7, 0x43, + 0x3c, 0x47, 0x52, 0x0e, 0xc3, 0x77, 0x81, 0x11, + 0x67, 0x0e, 0xa0, 0x70, 0x04, 0x47, 0x29, 0x40, + 0x86, 0x0d, 0x34, 0x56, 0xa7, 0xc9, 0x35, 0x59, + 0x68, 0xdc, 0x93, 0x81, 0x70, 0xee, 0x86, 0xd9, + 0x80, 0x06, 0x40, 0x4f, 0x1a, 0x0d, 0x40, 0x30, + 0x0b, 0xcb, 0x96, 0x47, 0xc1, 0xb7, 0x52, 0xfd, + 0x56, 0xe0, 0x72, 0x4b, 0xfb, 0xbd, 0x92, 0x45, + 0x61, 0x71, 0xc2, 0x33, 0x11, 0xbf, 0x52, 0x83, + 0x79, 0x26, 0xe0, 0x49, 0x6b, 0xb7, 0x05, 0x8b, + 0xe8, 0x0e, 0x87, 0x31, 0xd7, 0x9d, 0x8a, 0xf5, + 0xc0, 0x5f, 0x2e, 0x58, 0x4a, 0xdb, 0x11, 0xb3, + 0x6c, 0x30, 0x2a, 0x46, 0x19, 0xe3, 0x27, 0x84, + 0x1f, 0x63, 0x6e, 0xf6, 0x57, 0xc7, 0xc9, 0xd8, + 0x5e, 0xba, 0xb3, 0x87, 0xd5, 0x83, 0x26, 0x34, + 0x21, 0x9e, 0x65, 0xde, 0x42, 0xd3, 0xbe, 0x7b, + 0xbc, 0x91, 0x71, 0x44, 0x4d, 0x99, 0x3b, 0x31, + 0xe5, 0x3f, 0x11, 0x4e, 0x7f, 0x13, 0x51, 0x3b, + 0xae, 0x79, 0xc9, 0xd3, 0x81, 0x8e, 0x25, 0x40, + 0x10, 0xfc, 0x07, 0x1e, 0xf9, 0x7b, 0x9a, 0x4b, + 0x6c, 0xe3, 0xb3, 0xad, 0x1a, 0x0a, 0xdd, 0x9e, + 0x59, 0x0c, 0xa2, 0xcd, 0xae, 0x48, 0x4a, 0x38, + 0x5b, 0x47, 0x41, 0x94, 0x65, 0x6b, 0xbb, 0xeb, + 0x5b, 0xe3, 0xaf, 0x07, 0x5b, 0xd4, 0x4a, 0xa2, + 0xc9, 0x5d, 0x2f, 0x64, 0x03, 0xd7, 0x3a, 0x2c, + 0x6e, 0xce, 0x76, 0x95, 0xb4, 0xb3, 0xc0, 0xf1, + 0xe2, 0x45, 0x73, 0x7a, 0x5c, 0xab, 0xc1, 0xfc, + 0x02, 0x8d, 0x81, 0x29, 0xb3, 0xac, 0x07, 0xec, + 0x40, 0x7d, 0x45, 0xd9, 0x7a, 0x59, 0xee, 0x34, + 0xf0, 0xe9, 0xd5, 0x7b, 0x96, 0xb1, 0x3d, 0x95, + 0xcc, 0x86, 0xb5, 0xb6, 0x04, 0x2d, 0xb5, 0x92, + 0x7e, 0x76, 0xf4, 0x06, 0xa9, 0xa3, 0x12, 0x0f, + 0xb1, 0xaf, 0x26, 0xba, 0x7c, 0xfc, 0x7e, 0x1c, + 0xbc, 0x2c, 0x49, 0x97, 0x53, 0x60, 0x13, 0x0b, + 0xa6, 0x61, 0x83, 0x89, 0x42, 0xd4, 0x17, 0x0c, + 0x6c, 0x26, 0x52, 0xc3, 0xb3, 0xd4, 0x67, 0xf5, + 0xe3, 0x04, 0xb7, 0xf4, 0xcb, 0x80, 0xb8, 0xcb, + 0x77, 0x56, 0x3e, 0xaa, 0x57, 0x54, 0xee, 0xb4, + 0x2c, 0x67, 0xcf, 0xf2, 0xdc, 0xbe, 0x55, 0xf9, + 0x43, 0x1f, 0x6e, 0x22, 0x97, 0x67, 0x7f, 0xc4, + 0xef, 0xb1, 0x26, 0x31, 0x1e, 0x27, 0xdf, 0x41, + 0x80, 0x47, 0x6c, 0xe2, 0xfa, 0xa9, 0x8c, 0x2a, + 0xf6, 0xf2, 0xab, 0xf0, 0x15, 0xda, 0x6c, 0xc8, + 0xfe, 0xb5, 0x23, 0xde, 0xa9, 0x05, 0x3f, 0x06, + 0x54, 0x4c, 0xcd, 0xe1, 0xab, 0xfc, 0x0e, 0x62, + 0x33, 0x31, 0x73, 0x2c, 0x76, 0xcb, 0xb4, 0x47, + 0x1e, 0x20, 0xad, 0xd8, 0xf2, 0x31, 0xdd, 0xc4, + 0x8b, 0x0c, 0x77, 0xbe, 0xe1, 0x8b, 0x26, 0x00, + 0x02, 0x58, 0xd6, 0x8d, 0xef, 0xad, 0x74, 0x67, + 0xab, 0x3f, 0xef, 0xcb, 0x6f, 0xb0, 0xcc, 0x81, + 0x44, 0x4c, 0xaf, 0xe9, 0x49, 0x4f, 0xdb, 0xa0, + 0x25, 0xa4, 0xf0, 0x89, 0xf1, 0xbe, 0xd8, 0x10, + 0xff, 0xb1, 0x3b, 0x4b, 0xfa, 0x98, 0xf5, 0x79, + 0x6d, 0x1e, 0x69, 0x4d, 0x57, 0xb1, 0xc8, 0x19, + 0x1b, 0xbd, 0x1e, 0x8c, 0x84, 0xb7, 0x7b, 0xe8, + 0xd2, 0x2d, 0x09, 0x41, 0x41, 0x37, 0x3d, 0xb1, + 0x6f, 0x26, 0x5d, 0x71, 0x16, 0x3d, 0xb7, 0x83, + 0x27, 0x2c, 0xa7, 0xb6, 0x50, 0xbd, 0x91, 0x86, + 0xab, 0x24, 0xa1, 0x38, 0xfd, 0xea, 0x71, 0x55, + 0x7e, 0x9a, 0x07, 0x77, 0x4b, 0xfa, 0x61, 0x66, + 0x20, 0x1e, 0x28, 0x95, 0x18, 0x1b, 0xa4, 0xa0, + 0xfd, 0xc0, 0x89, 0x72, 0x43, 0xd9, 0x3b, 0x49, + 0x5a, 0x3f, 0x9d, 0xbf, 0xdb, 0xb4, 0x46, 0xea, + 0x42, 0x01, 0x77, 0x23, 0x68, 0x95, 0xb6, 0x24, + 0xb3, 0xa8, 0x6c, 0x28, 0x3b, 0x11, 0x40, 0x7e, + 0x18, 0x65, 0x6d, 0xd8, 0x24, 0x42, 0x7d, 0x88, + 0xc0, 0x52, 0xd9, 0x05, 0xe4, 0x95, 0x90, 0x87, + 0x8c, 0xf4, 0xd0, 0x6b, 0xb9, 0x83, 0x99, 0x34, + 0x6d, 0xfe, 0x54, 0x40, 0x94, 0x52, 0x21, 0x4f, + 0x14, 0x25, 0xc5, 0xd6, 0x5e, 0x95, 0xdc, 0x0a, + 0x2b, 0x89, 0x20, 0x11, 0x84, 0x48, 0xd6, 0x3a, + 0xcd, 0x5c, 0x24, 0xad, 0x62, 0xe3, 0xb1, 0x93, + 0x25, 0x8d, 0xcd, 0x7e, 0xfc, 0x27, 0xa3, 0x37, + 0xfd, 0x84, 0xfc, 0x1b, 0xb2, 0xf1, 0x27, 0x38, + 0x5a, 0xb7, 0xfc, 0xf2, 0xfa, 0x95, 0x66, 0xd4, + 0xfb, 0xba, 0xa7, 0xd7, 0xa3, 0x72, 0x69, 0x48, + 0x48, 0x8c, 0xeb, 0x28, 0x89, 0xfe, 0x33, 0x65, + 0x5a, 0x36, 0x01, 0x7e, 0x06, 0x79, 0x0a, 0x09, + 0x3b, 0x74, 0x11, 0x9a, 0x6e, 0xbf, 0xd4, 0x9e, + 0x58, 0x90, 0x49, 0x4f, 0x4d, 0x08, 0xd4, 0xe5, + 0x4a, 0x09, 0x21, 0xef, 0x8b, 0xb8, 0x74, 0x3b, + 0x91, 0xdd, 0x36, 0x85, 0x60, 0x2d, 0xfa, 0xd4, + 0x45, 0x7b, 0x45, 0x53, 0xf5, 0x47, 0x87, 0x7e, + 0xa6, 0x37, 0xc8, 0x78, 0x7a, 0x68, 0x9d, 0x8d, + 0x65, 0x2c, 0x0e, 0x91, 0x5c, 0xa2, 0x60, 0xf0, + 0x8e, 0x3f, 0xe9, 0x1a, 0xcd, 0xaa, 0xe7, 0xd5, + 0x77, 0x18, 0xaf, 0xc9, 0xbc, 0x18, 0xea, 0x48, + 0x1b, 0xfb, 0x22, 0x48, 0x70, 0x16, 0x29, 0x9e, + 0x5b, 0xc1, 0x2c, 0x66, 0x23, 0xbc, 0xf0, 0x1f, + 0xef, 0xaf, 0xe4, 0xd6, 0x04, 0x19, 0x82, 0x7a, + 0x0b, 0xba, 0x4b, 0x46, 0xb1, 0x6a, 0x85, 0x5d, + 0xb4, 0x73, 0xd6, 0x21, 0xa1, 0x71, 0x60, 0x14, + 0xee, 0x0a, 0x77, 0xc4, 0x66, 0x2e, 0xf9, 0x69, + 0x30, 0xaf, 0x41, 0x0b, 0xc8, 0x83, 0x3c, 0x53, + 0x99, 0x19, 0x27, 0x46, 0xf7, 0x41, 0x6e, 0x56, + 0xdc, 0x94, 0x28, 0x67, 0x4e, 0xb7, 0x25, 0x48, + 0x8a, 0xc2, 0xe0, 0x60, 0x96, 0xcc, 0x18, 0xf4, + 0x84, 0xdd, 0xa7, 0x5e, 0x3e, 0x05, 0x0b, 0x26, + 0x26, 0xb2, 0x5c, 0x1f, 0x57, 0x1a, 0x04, 0x7e, + 0x6a, 0xe3, 0x2f, 0xb4, 0x35, 0xb6, 0x38, 0x40, + 0x40, 0xcd, 0x6f, 0x87, 0x2e, 0xef, 0xa3, 0xd7, + 0xa9, 0xc2, 0xe8, 0x0d, 0x27, 0xdf, 0x44, 0x62, + 0x99, 0xa0, 0xfc, 0xcf, 0x81, 0x78, 0xcb, 0xfe, + 0xe5, 0xa0, 0x03, 0x4e, 0x6c, 0xd7, 0xf4, 0xaf, + 0x7a, 0xbb, 0x61, 0x82, 0xfe, 0x71, 0x89, 0xb2, + 0x22, 0x7c, 0x8e, 0x83, 0x04, 0xce, 0xf6, 0x5d, + 0x84, 0x8f, 0x95, 0x6a, 0x7f, 0xad, 0xfd, 0x32, + 0x9c, 0x5e, 0xe4, 0x9c, 0x89, 0x60, 0x54, 0xaa, + 0x96, 0x72, 0xd2, 0xd7, 0x36, 0x85, 0xa9, 0x45, + 0xd2, 0x2a, 0xa1, 0x81, 0x49, 0x6f, 0x7e, 0x04, + 0xfa, 0xe2, 0xfe, 0x90, 0x26, 0x77, 0x5a, 0x33, + 0xb8, 0x04, 0x9a, 0x7a, 0xe6, 0x4c, 0x4f, 0xad, + 0x72, 0x96, 0x08, 0x28, 0x58, 0x13, 0xf8, 0xc4, + 0x1c, 0xf0, 0xc3, 0x45, 0x95, 0x49, 0x20, 0x8c, + 0x9f, 0x39, 0x70, 0xe1, 0x77, 0xfe, 0xd5, 0x4b, + 0xaf, 0x86, 0xda, 0xef, 0x22, 0x06, 0x83, 0x36, + 0x29, 0x12, 0x11, 0x40, 0xbc, 0x3b, 0x86, 0xaa, + 0xaa, 0x65, 0x60, 0xc3, 0x80, 0xca, 0xed, 0xa9, + 0xf3, 0xb0, 0x79, 0x96, 0xa2, 0x55, 0x27, 0x28, + 0x55, 0x73, 0x26, 0xa5, 0x50, 0xea, 0x92, 0x4b, + 0x3c, 0x5c, 0x82, 0x33, 0xf0, 0x01, 0x3f, 0x03, + 0xc1, 0x08, 0x05, 0xbf, 0x98, 0xf4, 0x9b, 0x6d, + 0xa5, 0xa8, 0xb4, 0x82, 0x0c, 0x06, 0xfa, 0xff, + 0x2d, 0x08, 0xf3, 0x05, 0x4f, 0x57, 0x2a, 0x39, + 0xd4, 0x83, 0x0d, 0x75, 0x51, 0xd8, 0x5b, 0x1b, + 0xd3, 0x51, 0x5a, 0x32, 0x2a, 0x9b, 0x32, 0xb2, + 0xf2, 0xa4, 0x96, 0x12, 0xf2, 0xae, 0x40, 0x34, + 0x67, 0xa8, 0xf5, 0x44, 0xd5, 0x35, 0x53, 0xfe, + 0xa3, 0x60, 0x96, 0x63, 0x0f, 0x1f, 0x6e, 0xb0, + 0x5a, 0x42, 0xa6, 0xfc, 0x51, 0x0b, 0x60, 0x27, + 0xbc, 0x06, 0x71, 0xed, 0x65, 0x5b, 0x23, 0x86, + 0x4a, 0x07, 0x3b, 0x22, 0x07, 0x46, 0xe6, 0x90, + 0x3e, 0xf3, 0x25, 0x50, 0x1b, 0x4c, 0x7f, 0x03, + 0x08, 0xa8, 0x36, 0x6b, 0x87, 0xe5, 0xe3, 0xdb, + 0x9a, 0x38, 0x83, 0xff, 0x9f, 0x1a, 0x9f, 0x57, + 0xa4, 0x2a, 0xf6, 0x37, 0xbc, 0x1a, 0xff, 0xc9, + 0x1e, 0x35, 0x0c, 0xc3, 0x7c, 0xa3, 0xb2, 0xe5, + 0xd2, 0xc6, 0xb4, 0x57, 0x47, 0xe4, 0x32, 0x16, + 0x6d, 0xa9, 0xae, 0x64, 0xe6, 0x2d, 0x8d, 0xc5, + 0x8d, 0x50, 0x8e, 0xe8, 0x1a, 0x22, 0x34, 0x2a, + 0xd9, 0xeb, 0x51, 0x90, 0x4a, 0xb1, 0x41, 0x7d, + 0x64, 0xf9, 0xb9, 0x0d, 0xf6, 0x23, 0x33, 0xb0, + 0x33, 0xf4, 0xf7, 0x3f, 0x27, 0x84, 0xc6, 0x0f, + 0x54, 0xa5, 0xc0, 0x2e, 0xec, 0x0b, 0x3a, 0x48, + 0x6e, 0x80, 0x35, 0x81, 0x43, 0x9b, 0x90, 0xb1, + 0xd0, 0x2b, 0xea, 0x21, 0xdc, 0xda, 0x5b, 0x09, + 0xf4, 0xcc, 0x10, 0xb4, 0xc7, 0xfe, 0x79, 0x51, + 0xc3, 0xc5, 0xac, 0x88, 0x74, 0x84, 0x0b, 0x4b, + 0xca, 0x79, 0x16, 0x29, 0xfb, 0x69, 0x54, 0xdf, + 0x41, 0x7e, 0xe9, 0xc7, 0x8e, 0xea, 0xa5, 0xfe, + 0xfc, 0x76, 0x0e, 0x90, 0xc4, 0x92, 0x38, 0xad, + 0x7b, 0x48, 0xe6, 0x6e, 0xf7, 0x21, 0xfd, 0x4e, + 0x93, 0x0a, 0x7b, 0x41, 0x83, 0x68, 0xfb, 0x57, + 0x51, 0x76, 0x34, 0xa9, 0x6c, 0x00, 0xaa, 0x4f, + 0x66, 0x65, 0x98, 0x4a, 0x4f, 0xa3, 0xa0, 0xef, + 0x69, 0x3f, 0xe3, 0x1c, 0x92, 0x8c, 0xfd, 0xd8, + 0xe8, 0xde, 0x7c, 0x7f, 0x3e, 0x84, 0x8e, 0x69, + 0x3c, 0xf1, 0xf2, 0x05, 0x46, 0xdc, 0x2f, 0x9d, + 0x5e, 0x6e, 0x4c, 0xfb, 0xb5, 0x99, 0x2a, 0x59, + 0x63, 0xc1, 0x34, 0xbc, 0x57, 0xc0, 0x0d, 0xb9, + 0x61, 0x25, 0xf3, 0x33, 0x23, 0x51, 0xb6, 0x0d, + 0x07, 0xa6, 0xab, 0x94, 0x4a, 0xb7, 0x2a, 0xea, + 0xee, 0xac, 0xa3, 0xc3, 0x04, 0x8b, 0x0e, 0x56, + 0xfe, 0x44, 0xa7, 0x39, 0xe2, 0xed, 0xed, 0xb4, + 0x22, 0x2b, 0xac, 0x12, 0x32, 0x28, 0x91, 0xd8, + 0xa5, 0xab, 0xff, 0x5f, 0xe0, 0x4b, 0xda, 0x78, + 0x17, 0xda, 0xf1, 0x01, 0x5b, 0xcd, 0xe2, 0x5f, + 0x50, 0x45, 0x73, 0x2b, 0xe4, 0x76, 0x77, 0xf4, + 0x64, 0x1d, 0x43, 0xfb, 0x84, 0x7a, 0xea, 0x91, + 0xae, 0xf9, 0x9e, 0xb7, 0xb4, 0xb0, 0x91, 0x5f, + 0x16, 0x35, 0x9a, 0x11, 0xb8, 0xc7, 0xc1, 0x8c, + 0xc6, 0x10, 0x8d, 0x2f, 0x63, 0x4a, 0xa7, 0x57, + 0x3a, 0x51, 0xd6, 0x32, 0x2d, 0x64, 0x72, 0xd4, + 0x66, 0xdc, 0x10, 0xa6, 0x67, 0xd6, 0x04, 0x23, + 0x9d, 0x0a, 0x11, 0x77, 0xdd, 0x37, 0x94, 0x17, + 0x3c, 0xbf, 0x8b, 0x65, 0xb0, 0x2e, 0x5e, 0x66, + 0x47, 0x64, 0xac, 0xdd, 0xf0, 0x84, 0xfd, 0x39, + 0xfa, 0x15, 0x5d, 0xef, 0xae, 0xca, 0xc1, 0x36, + 0xa7, 0x5c, 0xbf, 0xc7, 0x08, 0xc2, 0x66, 0x00, + 0x74, 0x74, 0x4e, 0x27, 0x3f, 0x55, 0x8a, 0xb7, + 0x38, 0x66, 0x83, 0x6d, 0xcf, 0x99, 0x9e, 0x60, + 0x8f, 0xdd, 0x2e, 0x62, 0x22, 0x0e, 0xef, 0x0c, + 0x98, 0xa7, 0x85, 0x74, 0x3b, 0x9d, 0xec, 0x9e, + 0xa9, 0x19, 0x72, 0xa5, 0x7f, 0x2c, 0x39, 0xb7, + 0x7d, 0xb7, 0xf1, 0x12, 0x65, 0x27, 0x4b, 0x5a, + 0xde, 0x17, 0xfe, 0xad, 0x44, 0xf3, 0x20, 0x4d, + 0xfd, 0xe4, 0x1f, 0xb5, 0x81, 0xb0, 0x36, 0x37, + 0x08, 0x6f, 0xc3, 0x0c, 0xe9, 0x85, 0x98, 0x82, + 0xa9, 0x62, 0x0c, 0xc4, 0x97, 0xc0, 0x50, 0xc8, + 0xa7, 0x3c, 0x50, 0x9f, 0x43, 0xb9, 0xcd, 0x5e, + 0x4d, 0xfa, 0x1c, 0x4b, 0x0b, 0xa9, 0x98, 0x85, + 0x38, 0x92, 0xac, 0x8d, 0xe4, 0xad, 0x9b, 0x98, + 0xab, 0xd9, 0x38, 0xac, 0x62, 0x52, 0xa3, 0x22, + 0x63, 0x0f, 0xbf, 0x95, 0x48, 0xdf, 0x69, 0xe7, + 0x8b, 0x33, 0xd5, 0xb2, 0xbd, 0x05, 0x49, 0x49, + 0x9d, 0x57, 0x73, 0x19, 0x33, 0xae, 0xfa, 0x33, + 0xf1, 0x19, 0xa8, 0x80, 0xce, 0x04, 0x9f, 0xbc, + 0x1d, 0x65, 0x82, 0x1b, 0xe5, 0x3a, 0x51, 0xc8, + 0x1c, 0x21, 0xe3, 0x5d, 0xf3, 0x7d, 0x9b, 0x2f, + 0x2c, 0x1d, 0x4a, 0x7f, 0x9b, 0x68, 0x35, 0xa3, + 0xb2, 0x50, 0xf7, 0x62, 0x79, 0xcd, 0xf4, 0x98, + 0x4f, 0xe5, 0x63, 0x7c, 0x3e, 0x45, 0x31, 0x8c, + 0x16, 0xa0, 0x12, 0xc8, 0x58, 0xce, 0x39, 0xa6, + 0xbc, 0x54, 0xdb, 0xc5, 0xe0, 0xd5, 0xba, 0xbc, + 0xb9, 0x04, 0xf4, 0x8d, 0xe8, 0x2f, 0x15, 0x9d, +}; + +/* 100 test cases */ +static struct dahash_test { + uint16_t start; /* random 12 bit offset in buf */ + uint16_t length; /* random 8 bit length of test */ + xfs_dahash_t dahash; /* expected dahash result */ + xfs_dahash_t ascii_ci_dahash; /* expected ascii-ci dahash result */ +} test[] __initdata = +{ + {0x0567, 0x0097, 0x96951389, 0xc153aa0d}, + {0x0869, 0x0055, 0x6455ab4f, 0xd07f69bf}, + {0x0c51, 0x00be, 0x8663afde, 0xf9add90c}, + {0x044a, 0x00fc, 0x98fbe432, 0xbf2abb76}, + {0x0f29, 0x0079, 0x42371997, 0x282588b3}, + {0x08ba, 0x0052, 0x942be4f7, 0x2e023547}, + {0x01f2, 0x0013, 0x5262687e, 0x5266287e}, + {0x09e3, 0x00e2, 0x8ffb0908, 0x1da892f3}, + {0x007c, 0x0051, 0xb3158491, 0xb67f9e63}, + {0x0854, 0x001f, 0x83bb20d9, 0x22bb21db}, + {0x031b, 0x0008, 0x98970bdf, 0x9cd70adf}, + {0x0de7, 0x0027, 0xbfbf6f6c, 0xae3f296c}, + {0x0f76, 0x0005, 0x906a7105, 0x906a7105}, + {0x092e, 0x00d0, 0x86631850, 0xa3f6ac04}, + {0x0233, 0x0082, 0xdbdd914e, 0x5d8c7aac}, + {0x04c9, 0x0075, 0x5a400a9e, 0x12f60711}, + {0x0b66, 0x0099, 0xae128b45, 0x7551310d}, + {0x000d, 0x00ed, 0xe61c216a, 0xc22d3c4c}, + {0x0a31, 0x003d, 0xf69663b9, 0x51960bf8}, + {0x00a3, 0x0052, 0x643c39ae, 0xa93c73a8}, + {0x0125, 0x00d5, 0x7c310b0d, 0xf221cbb3}, + {0x0105, 0x004a, 0x06a77e74, 0xa4ef4561}, + {0x0858, 0x008e, 0x265bc739, 0xd6c36d9b}, + {0x045e, 0x0095, 0x13d6b192, 0x5f5c1d62}, + {0x0dab, 0x003c, 0xc4498704, 0x10414654}, + {0x00cd, 0x00b5, 0x802a4e2d, 0xfbd17c9d}, + {0x069b, 0x008c, 0x5df60f71, 0x91ddca5f}, + {0x0454, 0x006c, 0x5f03d8bb, 0x5c59fce0}, + {0x040e, 0x0032, 0x0ce513b5, 0xa8cd99b1}, + {0x0874, 0x00e2, 0x6a811fb3, 0xca028316}, + {0x0521, 0x00b4, 0x93296833, 0x2c4d4880}, + {0x0ddc, 0x00cf, 0xf9305338, 0x2c94210d}, + {0x0a70, 0x0023, 0x239549ea, 0x22b561aa}, + {0x083e, 0x0027, 0x2d88ba97, 0x5cd8bb9d}, + {0x0241, 0x00a7, 0xfe0b32e1, 0x17b506b8}, + {0x0dfc, 0x0096, 0x1a11e815, 0xee4141bd}, + {0x023e, 0x001e, 0xebc9a1f3, 0x5689a1f3}, + {0x067e, 0x0066, 0xb1067f81, 0xd9952571}, + {0x09ea, 0x000e, 0x46fd7247, 0x42b57245}, + {0x036b, 0x008c, 0x1a39acdf, 0x58bf1586}, + {0x078f, 0x0030, 0x964042ab, 0xb04218b9}, + {0x085c, 0x008f, 0x1829edab, 0x9ceca89c}, + {0x02ec, 0x009f, 0x6aefa72d, 0x634cc2a7}, + {0x043b, 0x00ce, 0x65642ff5, 0x6c8a584e}, + {0x0a32, 0x00b8, 0xbd82759e, 0x0f96a34f}, + {0x0d3c, 0x0087, 0xf4d66d54, 0xb71ba5f4}, + {0x09ec, 0x008a, 0x06bfa1ff, 0x576ca80f}, + {0x0902, 0x0015, 0x755025d2, 0x517225c2}, + {0x08fe, 0x000e, 0xf690ce2d, 0xf690cf3d}, + {0x00fb, 0x00dc, 0xe55f1528, 0x707d7d92}, + {0x0eaa, 0x003a, 0x0fe0a8d7, 0x87638cc5}, + {0x05fb, 0x0006, 0x86281cfb, 0x86281cf9}, + {0x0dd1, 0x00a7, 0x60ab51b4, 0xe28ef00c}, + {0x0005, 0x001b, 0xf51d969b, 0xe71dd6d3}, + {0x077c, 0x00dd, 0xc2fed268, 0xdc30c555}, + {0x0575, 0x00f5, 0x432c0b1a, 0x81dd7d16}, + {0x05be, 0x0088, 0x78baa04b, 0xd69b433e}, + {0x0c89, 0x0068, 0xeda9e428, 0xe9b4fa0a}, + {0x0f5c, 0x0068, 0xec143c76, 0x9947067a}, + {0x06a8, 0x0009, 0xd72651ce, 0xd72651ee}, + {0x060f, 0x008e, 0x765426cd, 0x2099626f}, + {0x07b1, 0x0047, 0x2cfcfa0c, 0x1a4baa07}, + {0x04f1, 0x0041, 0x55b172f9, 0x15331a79}, + {0x0e05, 0x00ac, 0x61efde93, 0x320568cc}, + {0x0bf7, 0x0097, 0x05b83eee, 0xc72fb7a3}, + {0x04e9, 0x00f3, 0x9928223a, 0xe8c77de2}, + {0x023a, 0x0005, 0xdfada9bc, 0xdfadb9be}, + {0x0acb, 0x000e, 0x2217cecd, 0x0017d6cd}, + {0x0148, 0x0060, 0xbc3f7405, 0xf5fd6615}, + {0x0764, 0x0059, 0xcbc201b1, 0xbb089bf4}, + {0x021f, 0x0059, 0x5d6b2256, 0xa16a0a59}, + {0x0f1e, 0x006c, 0xdefeeb45, 0xfc34f9d6}, + {0x071c, 0x00b9, 0xb9b59309, 0xb645eae2}, + {0x0564, 0x0063, 0xae064271, 0x954dc6d1}, + {0x0b14, 0x0044, 0xdb867d9b, 0xdf432309}, + {0x0e5a, 0x0055, 0xff06b685, 0xa65ff257}, + {0x015e, 0x00ba, 0x1115ccbc, 0x11c365f4}, + {0x0379, 0x00e6, 0x5f4e58dd, 0x2d176d31}, + {0x013b, 0x0067, 0x4897427e, 0xc40532fe}, + {0x0e64, 0x0071, 0x7af2b7a4, 0x1fb7bf43}, + {0x0a11, 0x0050, 0x92105726, 0xb1185e51}, + {0x0109, 0x0055, 0xd0d000f9, 0x60a60bfd}, + {0x00aa, 0x0022, 0x815d229d, 0x215d379c}, + {0x09ac, 0x004f, 0x02f9d985, 0x10b90b20}, + {0x0e1b, 0x00ce, 0x5cf92ab4, 0x6a477573}, + {0x08af, 0x00d8, 0x17ca72d1, 0x385af156}, + {0x0e33, 0x000a, 0xda2dba6b, 0xda2dbb69}, + {0x0ee3, 0x006a, 0xb00048e5, 0xa9a2decc}, + {0x0648, 0x001a, 0x2364b8cb, 0x3364b1cb}, + {0x0315, 0x0085, 0x0596fd0d, 0xa651740f}, + {0x0fbb, 0x003e, 0x298230ca, 0x7fc617c7}, + {0x0422, 0x006a, 0x78ada4ab, 0xc576ae2a}, + {0x04ba, 0x0073, 0xced1fbc2, 0xaac8455b}, + {0x007d, 0x0061, 0x4b7ff236, 0x347d5739}, + {0x070b, 0x00d0, 0x261cf0ae, 0xc7fb1c10}, + {0x0c1a, 0x0035, 0x8be92ee2, 0x8be9b4e1}, + {0x0af8, 0x0063, 0x824dcf03, 0x53010388}, + {0x08f8, 0x006d, 0xd289710c, 0x30418edd}, + {0x021b, 0x00ee, 0x6ac1c41d, 0x2557e9a3}, + {0x05b5, 0x00da, 0x8e52f0e2, 0x98531012}, +}; + +int __init +xfs_dahash_test(void) +{ + unsigned int i; + unsigned int errors = 0; + + for (i = 0; i < ARRAY_SIZE(test); i++) { + struct xfs_name xname = { }; + xfs_dahash_t hash; + + hash = xfs_da_hashname(test_buf + test[i].start, + test[i].length); + if (hash != test[i].dahash) + errors++; + + xname.name = test_buf + test[i].start; + xname.len = test[i].length; + hash = xfs_ascii_ci_hashname(&xname); + if (hash != test[i].ascii_ci_dahash) + errors++; + } + + if (errors) { + printk(KERN_ERR "xfs dir/attr hash test failed %u times!", + errors); + return -ERANGE; + } + + return 0; +} diff --git a/fs/xfs/xfs_dahash_test.h b/fs/xfs/xfs_dahash_test.h new file mode 100644 index 000000000..1a05bf4bd --- /dev/null +++ b/fs/xfs/xfs_dahash_test.h @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_DAHASH_TEST_H__ +#define __XFS_DAHASH_TEST_H__ + +int xfs_dahash_test(void); + +#endif /* __XFS_DAHASH_TEST_H__ */ + diff --git a/fs/xfs/xfs_dir2_readdir.c b/fs/xfs/xfs_dir2_readdir.c new file mode 100644 index 000000000..577f7a581 --- /dev/null +++ b/fs/xfs/xfs_dir2_readdir.c @@ -0,0 +1,556 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * Copyright (c) 2013 Red Hat, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_dir2.h" +#include "xfs_dir2_priv.h" +#include "xfs_trace.h" +#include "xfs_bmap.h" +#include "xfs_trans.h" +#include "xfs_error.h" +#include "xfs_health.h" + +/* + * Directory file type support functions + */ +static unsigned char xfs_dir3_filetype_table[] = { + DT_UNKNOWN, DT_REG, DT_DIR, DT_CHR, DT_BLK, + DT_FIFO, DT_SOCK, DT_LNK, DT_WHT, +}; + +unsigned char +xfs_dir3_get_dtype( + struct xfs_mount *mp, + uint8_t filetype) +{ + if (!xfs_has_ftype(mp)) + return DT_UNKNOWN; + + if (filetype >= XFS_DIR3_FT_MAX) + return DT_UNKNOWN; + + return xfs_dir3_filetype_table[filetype]; +} + +STATIC int +xfs_dir2_sf_getdents( + struct xfs_da_args *args, + struct dir_context *ctx) +{ + int i; /* shortform entry number */ + struct xfs_inode *dp = args->dp; /* incore directory inode */ + struct xfs_mount *mp = dp->i_mount; + xfs_dir2_dataptr_t off; /* current entry's offset */ + xfs_dir2_sf_entry_t *sfep; /* shortform directory entry */ + struct xfs_dir2_sf_hdr *sfp = dp->i_df.if_data; + xfs_dir2_dataptr_t dot_offset; + xfs_dir2_dataptr_t dotdot_offset; + xfs_ino_t ino; + struct xfs_da_geometry *geo = args->geo; + + ASSERT(dp->i_df.if_format == XFS_DINODE_FMT_LOCAL); + ASSERT(dp->i_df.if_bytes == dp->i_disk_size); + ASSERT(sfp != NULL); + + /* + * If the block number in the offset is out of range, we're done. + */ + if (xfs_dir2_dataptr_to_db(geo, ctx->pos) > geo->datablk) + return 0; + + /* + * Precalculate offsets for "." and ".." as we will always need them. + * This relies on the fact that directories always start with the + * entries for "." and "..". + */ + dot_offset = xfs_dir2_db_off_to_dataptr(geo, geo->datablk, + geo->data_entry_offset); + dotdot_offset = xfs_dir2_db_off_to_dataptr(geo, geo->datablk, + geo->data_entry_offset + + xfs_dir2_data_entsize(mp, sizeof(".") - 1)); + + /* + * Put . entry unless we're starting past it. + */ + if (ctx->pos <= dot_offset) { + ctx->pos = dot_offset & 0x7fffffff; + if (!dir_emit(ctx, ".", 1, I_INO(dp), DT_DIR)) + return 0; + } + + /* + * Put .. entry unless we're starting past it. + */ + if (ctx->pos <= dotdot_offset) { + ino = xfs_dir2_sf_get_parent_ino(sfp); + ctx->pos = dotdot_offset & 0x7fffffff; + if (!dir_emit(ctx, "..", 2, ino, DT_DIR)) + return 0; + } + + /* + * Loop while there are more entries and put'ing works. + */ + sfep = xfs_dir2_sf_firstentry(sfp); + for (i = 0; i < sfp->count; i++) { + uint8_t filetype; + + off = xfs_dir2_db_off_to_dataptr(geo, geo->datablk, + xfs_dir2_sf_get_offset(sfep)); + + if (ctx->pos > off) { + sfep = xfs_dir2_sf_nextentry(mp, sfp, sfep); + continue; + } + + ino = xfs_dir2_sf_get_ino(mp, sfp, sfep); + filetype = xfs_dir2_sf_get_ftype(mp, sfep); + ctx->pos = off & 0x7fffffff; + if (XFS_IS_CORRUPT(dp->i_mount, + !xfs_dir2_namecheck(sfep->name, + sfep->namelen))) { + xfs_dirattr_mark_sick(dp, XFS_DATA_FORK); + return -EFSCORRUPTED; + } + if (!dir_emit(ctx, (char *)sfep->name, sfep->namelen, ino, + xfs_dir3_get_dtype(mp, filetype))) + return 0; + sfep = xfs_dir2_sf_nextentry(mp, sfp, sfep); + } + + ctx->pos = xfs_dir2_db_off_to_dataptr(geo, geo->datablk + 1, 0) & + 0x7fffffff; + return 0; +} + +/* + * Readdir for block directories. + */ +STATIC int +xfs_dir2_block_getdents( + struct xfs_da_args *args, + struct dir_context *ctx, + unsigned int *lock_mode) +{ + struct xfs_inode *dp = args->dp; /* incore directory inode */ + struct xfs_buf *bp; /* buffer for block */ + int error; /* error return value */ + int wantoff; /* starting block offset */ + xfs_off_t cook; + struct xfs_da_geometry *geo = args->geo; + unsigned int offset, next_offset; + unsigned int end; + + /* + * If the block number in the offset is out of range, we're done. + */ + if (xfs_dir2_dataptr_to_db(geo, ctx->pos) > geo->datablk) + return 0; + + error = xfs_dir3_block_read(args->trans, dp, args->owner, &bp); + if (error) + return error; + + xfs_iunlock(dp, *lock_mode); + *lock_mode = 0; + + /* + * Extract the byte offset we start at from the seek pointer. + * We'll skip entries before this. + */ + wantoff = xfs_dir2_dataptr_to_off(geo, ctx->pos); + xfs_dir3_data_check(dp, bp); + + /* + * Loop over the data portion of the block. + * Each object is a real entry (dep) or an unused one (dup). + */ + end = xfs_dir3_data_end_offset(geo, bp->b_addr); + for (offset = geo->data_entry_offset; + offset < end; + offset = next_offset) { + struct xfs_dir2_data_unused *dup = bp->b_addr + offset; + struct xfs_dir2_data_entry *dep = bp->b_addr + offset; + uint8_t filetype; + + /* + * Unused, skip it. + */ + if (be16_to_cpu(dup->freetag) == XFS_DIR2_DATA_FREE_TAG) { + next_offset = offset + be16_to_cpu(dup->length); + continue; + } + + /* + * Bump pointer for the next iteration. + */ + next_offset = offset + + xfs_dir2_data_entsize(dp->i_mount, dep->namelen); + + /* + * The entry is before the desired starting point, skip it. + */ + if (offset < wantoff) + continue; + + cook = xfs_dir2_db_off_to_dataptr(geo, geo->datablk, offset); + + ctx->pos = cook & 0x7fffffff; + filetype = xfs_dir2_data_get_ftype(dp->i_mount, dep); + /* + * If it didn't fit, set the final offset to here & return. + */ + if (XFS_IS_CORRUPT(dp->i_mount, + !xfs_dir2_namecheck(dep->name, + dep->namelen))) { + xfs_dirattr_mark_sick(dp, XFS_DATA_FORK); + error = -EFSCORRUPTED; + goto out_rele; + } + if (!dir_emit(ctx, (char *)dep->name, dep->namelen, + be64_to_cpu(dep->inumber), + xfs_dir3_get_dtype(dp->i_mount, filetype))) + goto out_rele; + } + + /* + * Reached the end of the block. + * Set the offset to a non-existent block 1 and return. + */ + ctx->pos = xfs_dir2_db_off_to_dataptr(geo, geo->datablk + 1, 0) & + 0x7fffffff; +out_rele: + xfs_trans_brelse(args->trans, bp); + return error; +} + +/* + * Read a directory block and initiate readahead for blocks beyond that. + * We maintain a sliding readahead window of the remaining space in the + * buffer rounded up to the nearest block. + */ +STATIC int +xfs_dir2_leaf_readbuf( + struct xfs_da_args *args, + size_t bufsize, + xfs_dir2_off_t *cur_off, + xfs_dablk_t *ra_blk, + struct xfs_buf **bpp) +{ + struct xfs_inode *dp = args->dp; + struct xfs_buf *bp = NULL; + struct xfs_da_geometry *geo = args->geo; + struct xfs_ifork *ifp = xfs_ifork_ptr(dp, XFS_DATA_FORK); + struct xfs_bmbt_irec map; + struct blk_plug plug; + xfs_dir2_off_t new_off; + xfs_dablk_t next_ra; + xfs_dablk_t map_off; + xfs_dablk_t last_da; + struct xfs_iext_cursor icur; + int ra_want; + int error = 0; + + error = xfs_iread_extents(args->trans, dp, XFS_DATA_FORK); + if (error) + goto out; + + /* + * Look for mapped directory blocks at or above the current offset. + * Truncate down to the nearest directory block to start the scanning + * operation. + */ + last_da = xfs_dir2_byte_to_da(geo, XFS_DIR2_LEAF_OFFSET); + map_off = xfs_dir2_db_to_da(geo, xfs_dir2_byte_to_db(geo, *cur_off)); + if (!xfs_iext_lookup_extent(dp, ifp, map_off, &icur, &map)) + goto out; + if (map.br_startoff >= last_da) + goto out; + xfs_trim_extent(&map, map_off, last_da - map_off); + + /* Read the directory block of that first mapping. */ + new_off = xfs_dir2_da_to_byte(geo, map.br_startoff); + if (new_off > *cur_off) + *cur_off = new_off; + error = xfs_dir3_data_read(args->trans, dp, args->owner, + map.br_startoff, 0, &bp); + if (error) + goto out; + + /* + * Start readahead for the next bufsize's worth of dir data blocks. + * We may have already issued readahead for some of that range; + * ra_blk tracks the last block we tried to read(ahead). + */ + ra_want = howmany(bufsize + geo->blksize, (1 << geo->fsblog)); + if (*ra_blk >= last_da) + goto out; + else if (*ra_blk == 0) + *ra_blk = map.br_startoff; + next_ra = map.br_startoff + geo->fsbcount; + if (next_ra >= last_da) + goto out_no_ra; + if (map.br_blockcount < geo->fsbcount && + !xfs_iext_next_extent(ifp, &icur, &map)) + goto out_no_ra; + if (map.br_startoff >= last_da) + goto out_no_ra; + xfs_trim_extent(&map, next_ra, last_da - next_ra); + + /* Start ra for each dir (not fs) block that has a mapping. */ + blk_start_plug(&plug); + while (ra_want > 0) { + next_ra = roundup((xfs_dablk_t)map.br_startoff, geo->fsbcount); + while (ra_want > 0 && + next_ra < map.br_startoff + map.br_blockcount) { + if (next_ra >= last_da) { + *ra_blk = last_da; + break; + } + if (next_ra > *ra_blk) { + xfs_dir3_data_readahead(dp, next_ra, + XFS_DABUF_MAP_HOLE_OK); + *ra_blk = next_ra; + } + ra_want -= geo->fsbcount; + next_ra += geo->fsbcount; + } + if (!xfs_iext_next_extent(ifp, &icur, &map)) { + *ra_blk = last_da; + break; + } + } + blk_finish_plug(&plug); + +out: + *bpp = bp; + return error; +out_no_ra: + *ra_blk = last_da; + goto out; +} + +/* + * Getdents (readdir) for leaf and node directories. + * This reads the data blocks only, so is the same for both forms. + */ +STATIC int +xfs_dir2_leaf_getdents( + struct xfs_da_args *args, + struct dir_context *ctx, + size_t bufsize, + unsigned int *lock_mode) +{ + struct xfs_inode *dp = args->dp; + struct xfs_mount *mp = dp->i_mount; + struct xfs_buf *bp = NULL; /* data block buffer */ + xfs_dir2_data_entry_t *dep; /* data entry */ + xfs_dir2_data_unused_t *dup; /* unused entry */ + struct xfs_da_geometry *geo = args->geo; + xfs_dablk_t rablk = 0; /* current readahead block */ + xfs_dir2_off_t curoff; /* current overall offset */ + int length; /* temporary length value */ + int byteoff; /* offset in current block */ + unsigned int offset = 0; + int error = 0; /* error return value */ + + /* + * If the offset is at or past the largest allowed value, + * give up right away. + */ + if (ctx->pos >= XFS_DIR2_MAX_DATAPTR) + return 0; + + /* + * Inside the loop we keep the main offset value as a byte offset + * in the directory file. + */ + curoff = xfs_dir2_dataptr_to_byte(ctx->pos); + + /* + * Loop over directory entries until we reach the end offset. + * Get more blocks and readahead as necessary. + */ + while (curoff < XFS_DIR2_LEAF_OFFSET) { + uint8_t filetype; + + /* + * If we have no buffer, or we're off the end of the + * current buffer, need to get another one. + */ + if (!bp || offset >= geo->blksize) { + if (bp) { + xfs_trans_brelse(args->trans, bp); + bp = NULL; + } + + if (*lock_mode == 0) + *lock_mode = xfs_ilock_data_map_shared(dp); + error = xfs_dir2_leaf_readbuf(args, bufsize, &curoff, + &rablk, &bp); + if (error || !bp) + break; + + xfs_iunlock(dp, *lock_mode); + *lock_mode = 0; + + xfs_dir3_data_check(dp, bp); + /* + * Find our position in the block. + */ + offset = geo->data_entry_offset; + byteoff = xfs_dir2_byte_to_off(geo, curoff); + /* + * Skip past the header. + */ + if (byteoff == 0) + curoff += geo->data_entry_offset; + /* + * Skip past entries until we reach our offset. + */ + else { + while (offset < byteoff) { + dup = bp->b_addr + offset; + + if (be16_to_cpu(dup->freetag) + == XFS_DIR2_DATA_FREE_TAG) { + + length = be16_to_cpu(dup->length); + offset += length; + continue; + } + dep = bp->b_addr + offset; + length = xfs_dir2_data_entsize(mp, + dep->namelen); + offset += length; + } + /* + * Now set our real offset. + */ + curoff = + xfs_dir2_db_off_to_byte(geo, + xfs_dir2_byte_to_db(geo, curoff), + offset); + if (offset >= geo->blksize) + continue; + } + } + + /* + * We have a pointer to an entry. Is it a live one? + */ + dup = bp->b_addr + offset; + + /* + * No, it's unused, skip over it. + */ + if (be16_to_cpu(dup->freetag) == XFS_DIR2_DATA_FREE_TAG) { + length = be16_to_cpu(dup->length); + offset += length; + curoff += length; + continue; + } + + dep = bp->b_addr + offset; + length = xfs_dir2_data_entsize(mp, dep->namelen); + filetype = xfs_dir2_data_get_ftype(mp, dep); + + ctx->pos = xfs_dir2_byte_to_dataptr(curoff) & 0x7fffffff; + if (XFS_IS_CORRUPT(dp->i_mount, + !xfs_dir2_namecheck(dep->name, + dep->namelen))) { + xfs_dirattr_mark_sick(dp, XFS_DATA_FORK); + error = -EFSCORRUPTED; + break; + } + if (!dir_emit(ctx, (char *)dep->name, dep->namelen, + be64_to_cpu(dep->inumber), + xfs_dir3_get_dtype(dp->i_mount, filetype))) + break; + + /* + * Advance to next entry in the block. + */ + offset += length; + curoff += length; + /* bufsize may have just been a guess; don't go negative */ + bufsize = bufsize > length ? bufsize - length : 0; + } + + /* + * All done. Set output offset value to current offset. + */ + if (curoff > xfs_dir2_dataptr_to_byte(XFS_DIR2_MAX_DATAPTR)) + ctx->pos = XFS_DIR2_MAX_DATAPTR & 0x7fffffff; + else + ctx->pos = xfs_dir2_byte_to_dataptr(curoff) & 0x7fffffff; + if (bp) + xfs_trans_brelse(args->trans, bp); + return error; +} + +/* + * Read a directory. + * + * If supplied, the transaction collects locked dir buffers to avoid + * nested buffer deadlocks. This function does not dirty the + * transaction. The caller must hold the IOLOCK (shared or exclusive) + * before calling this function. + */ +int +xfs_readdir( + struct xfs_trans *tp, + struct xfs_inode *dp, + struct dir_context *ctx, + size_t bufsize) +{ + struct xfs_da_args args = { NULL }; + unsigned int lock_mode; + int error; + + trace_xfs_readdir(dp); + + if (xfs_is_shutdown(dp->i_mount)) + return -EIO; + if (xfs_ifork_zapped(dp, XFS_DATA_FORK)) + return -EIO; + + ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); + xfs_assert_ilocked(dp, XFS_IOLOCK_SHARED | XFS_IOLOCK_EXCL); + XFS_STATS_INC(dp->i_mount, xs_dir_getdents); + + args.dp = dp; + args.geo = dp->i_mount->m_dir_geo; + args.trans = tp; + args.owner = I_INO(dp); + + if (dp->i_df.if_format == XFS_DINODE_FMT_LOCAL) + return xfs_dir2_sf_getdents(&args, ctx); + + lock_mode = xfs_ilock_data_map_shared(dp); + switch (xfs_dir2_format(&args, &error)) { + case XFS_DIR2_FMT_BLOCK: + error = xfs_dir2_block_getdents(&args, ctx, &lock_mode); + break; + case XFS_DIR2_FMT_LEAF: + case XFS_DIR2_FMT_NODE: + error = xfs_dir2_leaf_getdents(&args, ctx, bufsize, &lock_mode); + break; + default: + break; + } + + if (lock_mode) + xfs_iunlock(dp, lock_mode); + return error; +} diff --git a/fs/xfs/xfs_discard.c b/fs/xfs/xfs_discard.c new file mode 100644 index 000000000..906221ffe --- /dev/null +++ b/fs/xfs/xfs_discard.c @@ -0,0 +1,886 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2010, 2023 Red Hat, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_trans.h" +#include "xfs_mount.h" +#include "xfs_btree.h" +#include "xfs_alloc_btree.h" +#include "xfs_alloc.h" +#include "xfs_discard.h" +#include "xfs_error.h" +#include "xfs_extent_busy.h" +#include "xfs_trace.h" +#include "xfs_log.h" +#include "xfs_ag.h" +#include "xfs_health.h" +#include "xfs_rtbitmap.h" +#include "xfs_rtgroup.h" + +/* + * Notes on an efficient, low latency fstrim algorithm + * + * We need to walk the filesystem free space and issue discards on the free + * space that meet the search criteria (size and location). We cannot issue + * discards on extents that might be in use, or are so recently in use they are + * still marked as busy. To serialise against extent state changes whilst we are + * gathering extents to trim, we must hold the AGF lock to lock out other + * allocations and extent free operations that might change extent state. + * + * However, we cannot just hold the AGF for the entire AG free space walk whilst + * we issue discards on each free space that is found. Storage devices can have + * extremely slow discard implementations (e.g. ceph RBD) and so walking a + * couple of million free extents and issuing synchronous discards on each + * extent can take a *long* time. Whilst we are doing this walk, nothing else + * can access the AGF, and we can stall transactions and hence the log whilst + * modifications wait for the AGF lock to be released. This can lead hung tasks + * kicking the hung task timer and rebooting the system. This is bad. + * + * Hence we need to take a leaf from the bulkstat playbook. It takes the AGI + * lock, gathers a range of inode cluster buffers that are allocated, drops the + * AGI lock and then reads all the inode cluster buffers and processes them. It + * loops doing this, using a cursor to keep track of where it is up to in the AG + * for each iteration to restart the INOBT lookup from. + * + * We can't do this exactly with free space - once we drop the AGF lock, the + * state of the free extent is out of our control and we cannot run a discard + * safely on it in this situation. Unless, of course, we've marked the free + * extent as busy and undergoing a discard operation whilst we held the AGF + * locked. + * + * This is exactly how online discard works - free extents are marked busy when + * they are freed, and once the extent free has been committed to the journal, + * the busy extent record is marked as "undergoing discard" and the discard is + * then issued on the free extent. Once the discard completes, the busy extent + * record is removed and the extent is able to be allocated again. + * + * In the context of fstrim, if we find a free extent we need to discard, we + * don't have to discard it immediately. All we need to do it record that free + * extent as being busy and under discard, and all the allocation routines will + * now avoid trying to allocate it. Hence if we mark the extent as busy under + * the AGF lock, we can safely discard it without holding the AGF lock because + * nothing will attempt to allocate that free space until the discard completes. + * + * This also allows us to issue discards asynchronously like we do with online + * discard, and so for fast devices fstrim will run much faster as we can have + * multiple discard operations in flight at once, as well as pipeline the free + * extent search so that it overlaps in flight discard IO. + */ + +#define XFS_DISCARD_MAX_EXAMINE (100) + +struct workqueue_struct *xfs_discard_wq; + +static void +xfs_discard_endio_work( + struct work_struct *work) +{ + struct xfs_busy_extents *extents = + container_of(work, struct xfs_busy_extents, endio_work); + + xfs_extent_busy_clear(&extents->extent_list, false); + kfree(extents->owner); +} + +/* + * Queue up the actual completion to a thread to avoid IRQ-safe locking for + * eb_lock. + */ +static void +xfs_discard_endio( + struct bio *bio) +{ + struct xfs_busy_extents *extents = bio->bi_private; + + INIT_WORK(&extents->endio_work, xfs_discard_endio_work); + queue_work(xfs_discard_wq, &extents->endio_work); + bio_put(bio); +} + +/* + * Walk the discard list and issue discards on all the busy extents in the + * list. We plug and chain the bios so that we only need a single completion + * call to clear all the busy extents once the discards are complete. + */ +void +xfs_discard_extents( + struct xfs_mount *mp, + struct xfs_busy_extents *extents) +{ + struct xfs_extent_busy *busyp; + struct bio *bio = NULL; + struct blk_plug plug; + + blk_start_plug(&plug); + list_for_each_entry(busyp, &extents->extent_list, list) { + struct xfs_group *xg = busyp->group; + struct xfs_buftarg *btp = + xfs_group_type_buftarg(xg->xg_mount, xg->xg_type); + + trace_xfs_discard_extent(xg, busyp->bno, busyp->length); + + __blkdev_issue_discard(btp->bt_bdev, + xfs_gbno_to_daddr(xg, busyp->bno), + XFS_FSB_TO_BB(mp, busyp->length), + GFP_KERNEL, &bio); + } + + if (bio) { + bio->bi_private = extents; + bio->bi_end_io = xfs_discard_endio; + submit_bio(bio); + } else { + xfs_discard_endio_work(&extents->endio_work); + } + blk_finish_plug(&plug); +} + +/* + * Care must be taken setting up the trim cursor as the perags may not have been + * initialised when the cursor is initialised. e.g. a clean mount which hasn't + * read in AGFs and the first operation run on the mounted fs is a trim. This + * can result in perag fields that aren't initialised until + * xfs_trim_gather_extents() calls xfs_alloc_read_agf() to lock down the AG for + * the free space search. + */ +struct xfs_trim_cur { + xfs_agblock_t start; + xfs_extlen_t count; + xfs_agblock_t end; + xfs_extlen_t minlen; + bool by_bno; +}; + +static int +xfs_trim_gather_extents( + struct xfs_perag *pag, + struct xfs_trim_cur *tcur, + struct xfs_busy_extents *extents) +{ + struct xfs_mount *mp = pag_mount(pag); + struct xfs_trans *tp; + struct xfs_btree_cur *cur; + struct xfs_buf *agbp; + int error; + int i; + int batch = XFS_DISCARD_MAX_EXAMINE; + + /* + * Force out the log. This means any transactions that might have freed + * space before we take the AGF buffer lock are now on disk, and the + * volatile disk cache is flushed. + */ + xfs_log_force(mp, XFS_LOG_SYNC); + + tp = xfs_trans_alloc_empty(mp); + + error = xfs_alloc_read_agf(pag, tp, 0, &agbp); + if (error) + goto out_trans_cancel; + + /* + * First time through tcur->count will not have been initialised as + * pag->pagf_longest is not guaranteed to be valid before we read + * the AGF buffer above. + */ + if (!tcur->count) + tcur->count = pag->pagf_longest; + + if (tcur->by_bno) { + /* sub-AG discard request always starts at tcur->start */ + cur = xfs_bnobt_init_cursor(mp, tp, agbp, pag); + error = xfs_alloc_lookup_le(cur, tcur->start, 0, &i); + if (!error && !i) + error = xfs_alloc_lookup_ge(cur, tcur->start, 0, &i); + } else if (tcur->start == 0) { + /* first time through a by-len starts with max length */ + cur = xfs_cntbt_init_cursor(mp, tp, agbp, pag); + error = xfs_alloc_lookup_ge(cur, 0, tcur->count, &i); + } else { + /* nth time through a by-len starts where we left off */ + cur = xfs_cntbt_init_cursor(mp, tp, agbp, pag); + error = xfs_alloc_lookup_le(cur, tcur->start, tcur->count, &i); + } + if (error) + goto out_del_cursor; + if (i == 0) { + /* nothing of that length left in the AG, we are done */ + tcur->count = 0; + goto out_del_cursor; + } + + /* + * Loop until we are done with all extents that are large + * enough to be worth discarding or we hit batch limits. + */ + while (i) { + xfs_agblock_t fbno; + xfs_extlen_t flen; + + error = xfs_alloc_get_rec(cur, &fbno, &flen, &i); + if (error) + break; + if (XFS_IS_CORRUPT(mp, i != 1)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + break; + } + + if (--batch <= 0) { + /* + * Update the cursor to point at this extent so we + * restart the next batch from this extent. + */ + tcur->start = fbno; + tcur->count = flen; + break; + } + + /* + * If the extent is entirely outside of the range we are + * supposed to skip it. Do not bother to trim down partially + * overlapping ranges for now. + */ + if (fbno + flen < tcur->start) { + trace_xfs_discard_exclude(pag_group(pag), fbno, flen); + goto next_extent; + } + if (fbno > tcur->end) { + trace_xfs_discard_exclude(pag_group(pag), fbno, flen); + if (tcur->by_bno) { + tcur->count = 0; + break; + } + goto next_extent; + } + + /* Trim the extent returned to the range we want. */ + if (fbno < tcur->start) { + flen -= tcur->start - fbno; + fbno = tcur->start; + } + if (fbno + flen > tcur->end + 1) + flen = tcur->end - fbno + 1; + + /* Too small? Give up. */ + if (flen < tcur->minlen) { + trace_xfs_discard_toosmall(pag_group(pag), fbno, flen); + if (tcur->by_bno) + goto next_extent; + tcur->count = 0; + break; + } + + /* + * If any blocks in the range are still busy, skip the + * discard and try again the next time. + */ + if (xfs_extent_busy_search(pag_group(pag), fbno, flen)) { + trace_xfs_discard_busy(pag_group(pag), fbno, flen); + goto next_extent; + } + + xfs_extent_busy_insert_discard(pag_group(pag), fbno, flen, + &extents->extent_list); +next_extent: + if (tcur->by_bno) + error = xfs_btree_increment(cur, 0, &i); + else + error = xfs_btree_decrement(cur, 0, &i); + if (error) + break; + + /* + * If there's no more records in the tree, we are done. Set the + * cursor block count to 0 to indicate to the caller that there + * is no more extents to search. + */ + if (i == 0) + tcur->count = 0; + } + + /* + * If there was an error, release all the gathered busy extents because + * we aren't going to issue a discard on them any more. + */ + if (error) + xfs_extent_busy_clear(&extents->extent_list, false); +out_del_cursor: + xfs_btree_del_cursor(cur, error); +out_trans_cancel: + xfs_trans_cancel(tp); + return error; +} + +static bool +xfs_trim_should_stop(void) +{ + return fatal_signal_pending(current) || freezing(current); +} + +/* + * Iterate the free list gathering extents and discarding them. We need a cursor + * for the repeated iteration of gather/discard loop, so use the longest extent + * we found in the last batch as the key to start the next. + */ +static int +xfs_trim_perag_extents( + struct xfs_perag *pag, + xfs_agblock_t start, + xfs_agblock_t end, + xfs_extlen_t minlen) +{ + struct xfs_trim_cur tcur = { + .start = start, + .end = end, + .minlen = minlen, + }; + int error = 0; + + if (start != 0 || end != pag_group(pag)->xg_block_count) + tcur.by_bno = true; + + do { + struct xfs_busy_extents *extents; + + extents = kzalloc_obj(*extents); + if (!extents) { + error = -ENOMEM; + break; + } + + extents->owner = extents; + INIT_LIST_HEAD(&extents->extent_list); + + error = xfs_trim_gather_extents(pag, &tcur, extents); + if (error) { + kfree(extents); + break; + } + + /* + * We hand the extent list to the discard function here so the + * discarded extents can be removed from the busy extent list. + * This allows the discards to run asynchronously with gathering + * the next round of extents to discard. + * + * However, we must ensure that we do not reference the extent + * list after this function call, as it may have been freed by + * the time control returns to us. + */ + xfs_discard_extents(pag_mount(pag), extents); + + if (xfs_trim_should_stop()) + break; + + } while (tcur.count != 0); + + return error; + +} + +static int +xfs_trim_datadev_extents( + struct xfs_mount *mp, + xfs_daddr_t start, + xfs_daddr_t end, + xfs_extlen_t minlen) +{ + xfs_agnumber_t start_agno, end_agno; + xfs_agblock_t start_agbno, end_agbno; + struct xfs_perag *pag = NULL; + xfs_daddr_t ddev_end; + int last_error = 0, error; + + ddev_end = min_t(xfs_daddr_t, end, + XFS_FSB_TO_BB(mp, mp->m_sb.sb_dblocks) - 1); + + start_agno = xfs_daddr_to_agno(mp, start); + start_agbno = xfs_daddr_to_agbno(mp, start); + end_agno = xfs_daddr_to_agno(mp, ddev_end); + end_agbno = xfs_daddr_to_agbno(mp, ddev_end); + + while ((pag = xfs_perag_next_range(mp, pag, start_agno, end_agno))) { + xfs_agblock_t agend = pag_group(pag)->xg_block_count; + + if (pag_agno(pag) == end_agno) + agend = end_agbno; + error = xfs_trim_perag_extents(pag, start_agbno, agend, minlen); + if (error) + last_error = error; + + if (xfs_trim_should_stop()) { + xfs_perag_rele(pag); + break; + } + start_agbno = 0; + } + + return last_error; +} + +#ifdef CONFIG_XFS_RT +struct xfs_trim_rtdev { + /* list of rt extents to free */ + struct list_head extent_list; + + /* minimum length that caller allows us to trim */ + xfs_rtblock_t minlen_fsb; + + /* restart point for the rtbitmap walk */ + xfs_rtxnum_t restart_rtx; + + /* stopping point for the current rtbitmap walk */ + xfs_rtxnum_t stop_rtx; +}; + +struct xfs_rtx_busy { + struct list_head list; + xfs_rtblock_t bno; + xfs_rtblock_t length; +}; + +static void +xfs_discard_free_rtdev_extents( + struct xfs_trim_rtdev *tr) +{ + struct xfs_rtx_busy *busyp, *n; + + list_for_each_entry_safe(busyp, n, &tr->extent_list, list) { + list_del_init(&busyp->list); + kfree(busyp); + } +} + +/* + * Walk the discard list and issue discards on all the busy extents in the + * list. We plug and chain the bios so that we only need a single completion + * call to clear all the busy extents once the discards are complete. + */ +static int +xfs_discard_rtdev_extents( + struct xfs_mount *mp, + struct xfs_trim_rtdev *tr) +{ + struct block_device *bdev = mp->m_rtdev_targp->bt_bdev; + struct xfs_rtx_busy *busyp; + struct bio *bio = NULL; + struct blk_plug plug; + xfs_rtblock_t start = NULLRTBLOCK, length = 0; + int error = 0; + + blk_start_plug(&plug); + list_for_each_entry(busyp, &tr->extent_list, list) { + if (start == NULLRTBLOCK) + start = busyp->bno; + length += busyp->length; + + trace_xfs_discard_rtextent(mp, busyp->bno, busyp->length); + + __blkdev_issue_discard(bdev, + xfs_rtb_to_daddr(mp, busyp->bno), + XFS_FSB_TO_BB(mp, busyp->length), + GFP_NOFS, &bio); + } + xfs_discard_free_rtdev_extents(tr); + + if (bio) { + error = submit_bio_wait(bio); + if (error == -EOPNOTSUPP) + error = 0; + if (error) + xfs_info(mp, + "discard failed for rtextent [0x%llx,%llu], error %d", + (unsigned long long)start, + (unsigned long long)length, + error); + bio_put(bio); + } + blk_finish_plug(&plug); + + return error; +} + +static int +xfs_trim_gather_rtextent( + struct xfs_rtgroup *rtg, + struct xfs_trans *tp, + const struct xfs_rtalloc_rec *rec, + void *priv) +{ + struct xfs_trim_rtdev *tr = priv; + struct xfs_rtx_busy *busyp; + xfs_rtblock_t rbno, rlen; + + if (rec->ar_startext > tr->stop_rtx) { + /* + * If we've scanned a large number of rtbitmap blocks, update + * the cursor to point at this extent so we restart the next + * batch from this extent. + */ + tr->restart_rtx = rec->ar_startext; + return -ECANCELED; + } + + rbno = xfs_rtx_to_rtb(rtg, rec->ar_startext); + rlen = xfs_rtbxlen_to_blen(rtg_mount(rtg), rec->ar_extcount); + + /* Ignore too small. */ + if (rlen < tr->minlen_fsb) { + trace_xfs_discard_rttoosmall(rtg_mount(rtg), rbno, rlen); + return 0; + } + + busyp = kzalloc_obj(struct xfs_rtx_busy); + if (!busyp) + return -ENOMEM; + + busyp->bno = rbno; + busyp->length = rlen; + INIT_LIST_HEAD(&busyp->list); + list_add_tail(&busyp->list, &tr->extent_list); + + tr->restart_rtx = rec->ar_startext + rec->ar_extcount; + return 0; +} + +/* Trim extents on an !rtgroups realtime device */ +static int +xfs_trim_rtextents( + struct xfs_rtgroup *rtg, + xfs_rtxnum_t low, + xfs_rtxnum_t high, + xfs_daddr_t minlen) +{ + struct xfs_mount *mp = rtg_mount(rtg); + struct xfs_trim_rtdev tr = { + .minlen_fsb = XFS_BB_TO_FSB(mp, minlen), + .extent_list = LIST_HEAD_INIT(tr.extent_list), + }; + struct xfs_trans *tp; + int error; + + tp = xfs_trans_alloc_empty(mp); + + /* + * Walk the free ranges between low and high. The query_range function + * trims the extents returned. + */ + do { + tr.stop_rtx = low + xfs_rtbitmap_rtx_per_rbmblock(mp); + xfs_rtgroup_lock(rtg, XFS_RTGLOCK_BITMAP_SHARED); + error = xfs_rtalloc_query_range(rtg, tp, low, high, + xfs_trim_gather_rtextent, &tr); + + if (error == -ECANCELED) + error = 0; + if (error) { + xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_BITMAP_SHARED); + xfs_discard_free_rtdev_extents(&tr); + break; + } + + if (list_empty(&tr.extent_list)) { + xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_BITMAP_SHARED); + break; + } + + error = xfs_discard_rtdev_extents(mp, &tr); + xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_BITMAP_SHARED); + if (error) + break; + + low = tr.restart_rtx; + } while (!xfs_trim_should_stop() && low <= high); + + xfs_trans_cancel(tp); + return error; +} + +struct xfs_trim_rtgroup { + /* list of rtgroup extents to free */ + struct xfs_busy_extents *extents; + + /* minimum length that caller allows us to trim */ + xfs_rtblock_t minlen_fsb; + + /* restart point for the rtbitmap walk */ + xfs_rtxnum_t restart_rtx; + + /* number of extents to examine before stopping to issue discard ios */ + int batch; + + /* number of extents queued for discard */ + int queued; +}; + +static int +xfs_trim_gather_rtgroup_extent( + struct xfs_rtgroup *rtg, + struct xfs_trans *tp, + const struct xfs_rtalloc_rec *rec, + void *priv) +{ + struct xfs_trim_rtgroup *tr = priv; + xfs_rgblock_t rgbno; + xfs_extlen_t len; + + if (--tr->batch <= 0) { + /* + * If we've checked a large number of extents, update the + * cursor to point at this extent so we restart the next batch + * from this extent. + */ + tr->restart_rtx = rec->ar_startext; + return -ECANCELED; + } + + rgbno = xfs_rtx_to_rgbno(rtg, rec->ar_startext); + len = xfs_rtxlen_to_extlen(rtg_mount(rtg), rec->ar_extcount); + + /* Ignore too small. */ + if (len < tr->minlen_fsb) { + trace_xfs_discard_toosmall(rtg_group(rtg), rgbno, len); + return 0; + } + + /* + * If any blocks in the range are still busy, skip the discard and try + * again the next time. + */ + if (xfs_extent_busy_search(rtg_group(rtg), rgbno, len)) { + trace_xfs_discard_busy(rtg_group(rtg), rgbno, len); + return 0; + } + + xfs_extent_busy_insert_discard(rtg_group(rtg), rgbno, len, + &tr->extents->extent_list); + + tr->queued++; + tr->restart_rtx = rec->ar_startext + rec->ar_extcount; + return 0; +} + +/* Trim extents in this rtgroup using the busy extent machinery. */ +static int +xfs_trim_rtgroup_extents( + struct xfs_rtgroup *rtg, + xfs_rtxnum_t low, + xfs_rtxnum_t high, + xfs_daddr_t minlen) +{ + struct xfs_mount *mp = rtg_mount(rtg); + struct xfs_trim_rtgroup tr = { + .minlen_fsb = XFS_BB_TO_FSB(mp, minlen), + }; + struct xfs_trans *tp; + int error; + + tp = xfs_trans_alloc_empty(mp); + + /* + * Walk the free ranges between low and high. The query_range function + * trims the extents returned. + */ + do { + tr.extents = kzalloc_obj(*tr.extents); + if (!tr.extents) { + error = -ENOMEM; + break; + } + + tr.queued = 0; + tr.batch = XFS_DISCARD_MAX_EXAMINE; + tr.extents->owner = tr.extents; + INIT_LIST_HEAD(&tr.extents->extent_list); + + xfs_rtgroup_lock(rtg, XFS_RTGLOCK_BITMAP_SHARED); + error = xfs_rtalloc_query_range(rtg, tp, low, high, + xfs_trim_gather_rtgroup_extent, &tr); + xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_BITMAP_SHARED); + if (error == -ECANCELED) + error = 0; + if (error) { + kfree(tr.extents); + break; + } + + if (!tr.queued) { + kfree(tr.extents); + break; + } + + /* + * We hand the extent list to the discard function here so the + * discarded extents can be removed from the busy extent list. + * This allows the discards to run asynchronously with + * gathering the next round of extents to discard. + * + * However, we must ensure that we do not reference the extent + * list after this function call, as it may have been freed by + * the time control returns to us. + */ + xfs_discard_extents(rtg_mount(rtg), tr.extents); + + low = tr.restart_rtx; + } while (!xfs_trim_should_stop() && low <= high); + + xfs_trans_cancel(tp); + return error; +} + +static int +xfs_trim_rtdev_extents( + struct xfs_mount *mp, + xfs_daddr_t start, + xfs_daddr_t end, + xfs_daddr_t minlen) +{ + xfs_rtblock_t start_rtbno, end_rtbno; + xfs_rtxnum_t start_rtx, end_rtx; + xfs_rgnumber_t start_rgno, end_rgno; + xfs_daddr_t daddr_offset; + int last_error = 0, error; + struct xfs_rtgroup *rtg = NULL; + + /* Shift the start and end downwards to match the rt device. */ + daddr_offset = XFS_FSB_TO_BB(mp, mp->m_sb.sb_dblocks); + if (start > daddr_offset) + start -= daddr_offset; + else + start = 0; + start_rtbno = xfs_daddr_to_rtb(mp, start); + start_rtx = xfs_rtb_to_rtx(mp, start_rtbno); + start_rgno = xfs_rtb_to_rgno(mp, start_rtbno); + + if (end <= daddr_offset) + return 0; + else + end -= daddr_offset; + end_rtbno = xfs_daddr_to_rtb(mp, end); + end_rtx = xfs_rtb_to_rtx(mp, end_rtbno + mp->m_sb.sb_rextsize - 1); + end_rgno = xfs_rtb_to_rgno(mp, end_rtbno); + + while ((rtg = xfs_rtgroup_next_range(mp, rtg, start_rgno, end_rgno))) { + xfs_rtxnum_t rtg_end = rtg->rtg_extents; + + if (rtg_rgno(rtg) == end_rgno) + rtg_end = min(rtg_end, end_rtx); + + if (xfs_has_rtgroups(mp)) + error = xfs_trim_rtgroup_extents(rtg, start_rtx, + rtg_end, minlen); + else + error = xfs_trim_rtextents(rtg, start_rtx, rtg_end, + minlen); + if (error) + last_error = error; + + if (xfs_trim_should_stop()) { + xfs_rtgroup_rele(rtg); + break; + } + start_rtx = 0; + } + + return last_error; +} +#else +# define xfs_trim_rtdev_extents(...) (-EOPNOTSUPP) +#endif /* CONFIG_XFS_RT */ + +/* + * trim a range of the filesystem. + * + * Note: the parameters passed from userspace are byte ranges into the + * filesystem which does not match to the format we use for filesystem block + * addressing. FSB addressing is sparse (AGNO|AGBNO), while the incoming format + * is a linear address range. Hence we need to use DADDR based conversions and + * comparisons for determining the correct offset and regions to trim. + * + * The realtime device is mapped into the FITRIM "address space" immediately + * after the data device. + */ +int +xfs_ioc_trim( + struct xfs_mount *mp, + struct fstrim_range __user *urange) +{ + unsigned int granularity = + bdev_discard_granularity(mp->m_ddev_targp->bt_bdev); + struct block_device *rt_bdev = NULL; + struct fstrim_range range; + xfs_daddr_t start, end; + xfs_extlen_t minlen; + xfs_rfsblock_t max_blocks; + int error, last_error = 0; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (mp->m_rtdev_targp && !xfs_has_zoned(mp) && + bdev_max_discard_sectors(mp->m_rtdev_targp->bt_bdev)) + rt_bdev = mp->m_rtdev_targp->bt_bdev; + if (!bdev_max_discard_sectors(mp->m_ddev_targp->bt_bdev) && !rt_bdev) + return -EOPNOTSUPP; + + if (rt_bdev) + granularity = max(granularity, + bdev_discard_granularity(rt_bdev)); + + /* + * We haven't recovered the log, so we cannot use our bnobt-guided + * storage zapping commands. + */ + if (xfs_has_norecovery(mp)) + return -EROFS; + + if (copy_from_user(&range, urange, sizeof(range))) + return -EFAULT; + + range.minlen = max_t(u64, granularity, range.minlen); + minlen = XFS_B_TO_FSB(mp, range.minlen); + + /* + * Truncating down the len isn't actually quite correct, but using + * BBTOB would mean we trivially get overflows for values + * of ULLONG_MAX or slightly lower. And ULLONG_MAX is the default + * used by the fstrim application. In the end it really doesn't + * matter as trimming blocks is an advisory interface. + */ + max_blocks = mp->m_sb.sb_dblocks + mp->m_sb.sb_rblocks; + if (range.start >= XFS_FSB_TO_B(mp, max_blocks) || + range.minlen > XFS_FSB_TO_B(mp, mp->m_ag_max_usable) || + range.len < mp->m_sb.sb_blocksize) + return -EINVAL; + + start = BTOBB(range.start); + end = start + BTOBBT(range.len) - 1; + + if (bdev_max_discard_sectors(mp->m_ddev_targp->bt_bdev)) { + error = xfs_trim_datadev_extents(mp, start, end, minlen); + if (error) + last_error = error; + } + + if (rt_bdev && !xfs_trim_should_stop()) { + error = xfs_trim_rtdev_extents(mp, start, end, minlen); + if (error) + last_error = error; + } + + if (last_error) + return last_error; + + range.len = min_t(unsigned long long, range.len, + XFS_FSB_TO_B(mp, max_blocks) - range.start); + if (copy_to_user(urange, &range, sizeof(range))) + return -EFAULT; + return 0; +} diff --git a/fs/xfs/xfs_discard.h b/fs/xfs/xfs_discard.h new file mode 100644 index 000000000..8c5cc4af6 --- /dev/null +++ b/fs/xfs/xfs_discard.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef XFS_DISCARD_H +#define XFS_DISCARD_H 1 + +struct fstrim_range; +struct xfs_mount; +struct xfs_busy_extents; + +void xfs_discard_extents(struct xfs_mount *mp, struct xfs_busy_extents *busy); +int xfs_ioc_trim(struct xfs_mount *mp, struct fstrim_range __user *fstrim); + +#endif /* XFS_DISCARD_H */ diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c new file mode 100644 index 000000000..b4f6c5948 --- /dev/null +++ b/fs/xfs/xfs_dquot.c @@ -0,0 +1,1552 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2003 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_shared.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_inode.h" +#include "xfs_bmap.h" +#include "xfs_quota.h" +#include "xfs_trans.h" +#include "xfs_buf_item.h" +#include "xfs_trans_space.h" +#include "xfs_trans_priv.h" +#include "xfs_qm.h" +#include "xfs_trace.h" +#include "xfs_log.h" +#include "xfs_bmap_btree.h" +#include "xfs_error.h" +#include "xfs_health.h" + +/* + * Lock order: + * + * ip->i_lock + * qi->qi_tree_lock + * dquot->q_qlock + * dquot->q_flush (xfs_dqflock() and friends) + * qi->qi_lru_lock + * + * If two dquots need to be locked the order is user before group/project, + * otherwise by the lowest id first, see xfs_dqlock2. + */ + +struct kmem_cache *xfs_dqtrx_cache; +static struct kmem_cache *xfs_dquot_cache; + +static struct lock_class_key xfs_dquot_group_class; +static struct lock_class_key xfs_dquot_project_class; + +/* Record observations of quota corruption with the health tracking system. */ +static void +xfs_dquot_mark_sick( + struct xfs_dquot *dqp) +{ + struct xfs_mount *mp = dqp->q_mount; + + switch (dqp->q_type) { + case XFS_DQTYPE_USER: + xfs_fs_mark_sick(mp, XFS_SICK_FS_UQUOTA); + break; + case XFS_DQTYPE_GROUP: + xfs_fs_mark_sick(mp, XFS_SICK_FS_GQUOTA); + break; + case XFS_DQTYPE_PROJ: + xfs_fs_mark_sick(mp, XFS_SICK_FS_PQUOTA); + break; + default: + ASSERT(0); + break; + } +} + +/* + * Detach the dquot buffer if it's still attached, because we can get called + * through dqpurge after a log shutdown. Caller must hold the dqflock or have + * otherwise isolated the dquot. + */ +void +xfs_dquot_detach_buf( + struct xfs_dquot *dqp) +{ + struct xfs_dq_logitem *qlip = &dqp->q_logitem; + struct xfs_buf *bp = NULL; + + spin_lock(&qlip->qli_lock); + if (qlip->qli_item.li_buf) { + bp = qlip->qli_item.li_buf; + qlip->qli_item.li_buf = NULL; + } + spin_unlock(&qlip->qli_lock); + if (bp) { + xfs_buf_lock(bp); + list_del_init(&qlip->qli_item.li_bio_list); + xfs_buf_relse(bp); + } +} + +/* + * This is called to free all the memory associated with a dquot + */ +void +xfs_qm_dqdestroy( + struct xfs_dquot *dqp) +{ + ASSERT(list_empty(&dqp->q_lru)); + ASSERT(dqp->q_logitem.qli_item.li_buf == NULL); + + kvfree(dqp->q_logitem.qli_item.li_lv_shadow); + mutex_destroy(&dqp->q_qlock); + + XFS_STATS_DEC(dqp->q_mount, xs_qm_dquot); + kmem_cache_free(xfs_dquot_cache, dqp); +} + +/* + * If default limits are in force, push them into the dquot now. + * We overwrite the dquot limits only if they are zero and this + * is not the root dquot. + */ +void +xfs_qm_adjust_dqlimits( + struct xfs_dquot *dq) +{ + struct xfs_mount *mp = dq->q_mount; + struct xfs_quotainfo *q = mp->m_quotainfo; + struct xfs_def_quota *defq; + int prealloc = 0; + + ASSERT(dq->q_id); + defq = xfs_get_defquota(q, xfs_dquot_type(dq)); + + if (!dq->q_blk.softlimit) { + dq->q_blk.softlimit = defq->blk.soft; + prealloc = 1; + } + if (!dq->q_blk.hardlimit) { + dq->q_blk.hardlimit = defq->blk.hard; + prealloc = 1; + } + if (!dq->q_ino.softlimit) + dq->q_ino.softlimit = defq->ino.soft; + if (!dq->q_ino.hardlimit) + dq->q_ino.hardlimit = defq->ino.hard; + if (!dq->q_rtb.softlimit) + dq->q_rtb.softlimit = defq->rtb.soft; + if (!dq->q_rtb.hardlimit) + dq->q_rtb.hardlimit = defq->rtb.hard; + + if (prealloc) + xfs_dquot_set_prealloc_limits(dq); +} + +/* Set the expiration time of a quota's grace period. */ +time64_t +xfs_dquot_set_timeout( + struct xfs_mount *mp, + time64_t timeout) +{ + struct xfs_quotainfo *qi = mp->m_quotainfo; + + return clamp_t(time64_t, timeout, qi->qi_expiry_min, + qi->qi_expiry_max); +} + +/* Set the length of the default grace period. */ +time64_t +xfs_dquot_set_grace_period( + time64_t grace) +{ + return clamp_t(time64_t, grace, XFS_DQ_GRACE_MIN, XFS_DQ_GRACE_MAX); +} + +/* + * Determine if this quota counter is over either limit and set the quota + * timers as appropriate. + */ +static inline void +xfs_qm_adjust_res_timer( + struct xfs_mount *mp, + struct xfs_dquot_res *res, + struct xfs_quota_limits *qlim) +{ + ASSERT(res->hardlimit == 0 || res->softlimit <= res->hardlimit); + + if ((res->softlimit && res->count > res->softlimit) || + (res->hardlimit && res->count > res->hardlimit)) { + if (res->timer == 0) + res->timer = xfs_dquot_set_timeout(mp, + ktime_get_real_seconds() + qlim->time); + } else { + res->timer = 0; + } +} + +/* + * Check the limits and timers of a dquot and start or reset timers + * if necessary. + * This gets called even when quota enforcement is OFF, which makes our + * life a little less complicated. (We just don't reject any quota + * reservations in that case, when enforcement is off). + * We also return 0 as the values of the timers in Q_GETQUOTA calls, when + * enforcement's off. + * In contrast, warnings are a little different in that they don't + * 'automatically' get started when limits get exceeded. They do + * get reset to zero, however, when we find the count to be under + * the soft limit (they are only ever set non-zero via userspace). + */ +void +xfs_qm_adjust_dqtimers( + struct xfs_dquot *dq) +{ + struct xfs_mount *mp = dq->q_mount; + struct xfs_quotainfo *qi = mp->m_quotainfo; + struct xfs_def_quota *defq; + + ASSERT(dq->q_id); + defq = xfs_get_defquota(qi, xfs_dquot_type(dq)); + + xfs_qm_adjust_res_timer(dq->q_mount, &dq->q_blk, &defq->blk); + xfs_qm_adjust_res_timer(dq->q_mount, &dq->q_ino, &defq->ino); + xfs_qm_adjust_res_timer(dq->q_mount, &dq->q_rtb, &defq->rtb); +} + +/* + * initialize a buffer full of dquots and log the whole thing + */ +void +xfs_qm_init_dquot_blk( + struct xfs_trans *tp, + xfs_dqid_t id, + xfs_dqtype_t type, + struct xfs_buf *bp) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_quotainfo *q = mp->m_quotainfo; + struct xfs_dqblk *d; + xfs_dqid_t curid; + unsigned int qflag; + unsigned int blftype; + int i; + + ASSERT(tp); + ASSERT(xfs_buf_islocked(bp)); + + switch (type) { + case XFS_DQTYPE_USER: + qflag = XFS_UQUOTA_CHKD; + blftype = XFS_BLF_UDQUOT_BUF; + break; + case XFS_DQTYPE_PROJ: + qflag = XFS_PQUOTA_CHKD; + blftype = XFS_BLF_PDQUOT_BUF; + break; + case XFS_DQTYPE_GROUP: + qflag = XFS_GQUOTA_CHKD; + blftype = XFS_BLF_GDQUOT_BUF; + break; + default: + ASSERT(0); + return; + } + + d = bp->b_addr; + + /* + * ID of the first dquot in the block - id's are zero based. + */ + curid = id - (id % q->qi_dqperchunk); + memset(d, 0, BBTOB(q->qi_dqchunklen)); + for (i = 0; i < q->qi_dqperchunk; i++, d++, curid++) { + d->dd_diskdq.d_magic = cpu_to_be16(XFS_DQUOT_MAGIC); + d->dd_diskdq.d_version = XFS_DQUOT_VERSION; + d->dd_diskdq.d_id = cpu_to_be32(curid); + d->dd_diskdq.d_type = type; + if (curid > 0 && xfs_has_bigtime(mp)) + d->dd_diskdq.d_type |= XFS_DQTYPE_BIGTIME; + if (xfs_has_crc(mp)) { + uuid_copy(&d->dd_uuid, &mp->m_sb.sb_meta_uuid); + xfs_update_cksum((char *)d, sizeof(struct xfs_dqblk), + XFS_DQUOT_CRC_OFF); + } + } + + xfs_trans_dquot_buf(tp, bp, blftype); + + /* + * quotacheck uses delayed writes to update all the dquots on disk in an + * efficient manner instead of logging the individual dquot changes as + * they are made. However if we log the buffer allocated here and crash + * after quotacheck while the logged initialisation is still in the + * active region of the log, log recovery can replay the dquot buffer + * initialisation over the top of the checked dquots and corrupt quota + * accounting. + * + * To avoid this problem, quotacheck cannot log the initialised buffer. + * We must still dirty the buffer and write it back before the + * allocation transaction clears the log. Therefore, mark the buffer as + * ordered instead of logging it directly. This is safe for quotacheck + * because it detects and repairs allocated but initialized dquot blocks + * in the quota inodes. + */ + if (!(mp->m_qflags & qflag)) + xfs_trans_ordered_buf(tp, bp); + else + xfs_trans_log_buf(tp, bp, 0, BBTOB(q->qi_dqchunklen) - 1); +} + +static void +xfs_dquot_set_prealloc( + struct xfs_dquot_pre *pre, + const struct xfs_dquot_res *res) +{ + xfs_qcnt_t space; + + pre->q_prealloc_hi_wmark = res->hardlimit; + pre->q_prealloc_lo_wmark = res->softlimit; + + space = div_u64(pre->q_prealloc_hi_wmark, 100); + if (!pre->q_prealloc_lo_wmark) + pre->q_prealloc_lo_wmark = space * 95; + + pre->q_low_space[XFS_QLOWSP_1_PCNT] = space; + pre->q_low_space[XFS_QLOWSP_3_PCNT] = space * 3; + pre->q_low_space[XFS_QLOWSP_5_PCNT] = space * 5; +} + +/* + * Initialize the dynamic speculative preallocation thresholds. The lo/hi + * watermarks correspond to the soft and hard limits by default. If a soft limit + * is not specified, we use 95% of the hard limit. + */ +void +xfs_dquot_set_prealloc_limits(struct xfs_dquot *dqp) +{ + xfs_dquot_set_prealloc(&dqp->q_blk_prealloc, &dqp->q_blk); + xfs_dquot_set_prealloc(&dqp->q_rtb_prealloc, &dqp->q_rtb); +} + +/* + * Ensure that the given in-core dquot has a buffer on disk backing it, and + * return the buffer locked and held. This is called when the bmapi finds a + * hole. + */ +STATIC int +xfs_dquot_disk_alloc( + struct xfs_dquot *dqp, + struct xfs_buf **bpp) +{ + struct xfs_bmbt_irec map; + struct xfs_trans *tp; + struct xfs_mount *mp = dqp->q_mount; + struct xfs_buf *bp; + xfs_dqtype_t qtype = xfs_dquot_type(dqp); + struct xfs_inode *quotip = xfs_quota_inode(mp, qtype); + int nmaps = 1; + int error; + + trace_xfs_dqalloc(dqp); + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_qm_dqalloc, + XFS_QM_DQALLOC_SPACE_RES(mp), 0, 0, &tp); + if (error) + return error; + + xfs_ilock(quotip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, quotip, 0); + + if (!xfs_this_quota_on(dqp->q_mount, qtype)) { + /* + * Return if this type of quotas is turned off while we didn't + * have an inode lock + */ + error = -ESRCH; + goto err_cancel; + } + + error = xfs_iext_count_extend(tp, quotip, XFS_DATA_FORK, + XFS_IEXT_ADD_NOSPLIT_CNT); + if (error) + goto err_cancel; + + /* Create the block mapping. */ + error = xfs_bmapi_write(tp, quotip, dqp->q_fileoffset, + XFS_DQUOT_CLUSTER_SIZE_FSB, XFS_BMAPI_METADATA, 0, &map, + &nmaps); + if (error) + goto err_cancel; + + ASSERT(map.br_blockcount == XFS_DQUOT_CLUSTER_SIZE_FSB); + ASSERT((map.br_startblock != DELAYSTARTBLOCK) && + (map.br_startblock != HOLESTARTBLOCK)); + + /* + * Keep track of the blkno to save a lookup later + */ + dqp->q_blkno = XFS_FSB_TO_DADDR(mp, map.br_startblock); + + /* now we can just get the buffer (there's nothing to read yet) */ + error = xfs_trans_get_buf(tp, mp->m_ddev_targp, dqp->q_blkno, + mp->m_quotainfo->qi_dqchunklen, 0, &bp); + if (error) + goto err_cancel; + bp->b_ops = &xfs_dquot_buf_ops; + + /* + * Make a chunk of dquots out of this buffer and log + * the entire thing. + */ + xfs_qm_init_dquot_blk(tp, dqp->q_id, qtype, bp); + xfs_buf_set_ref(bp, XFS_DQUOT_REF); + + /* + * Hold the buffer and join it to the dfops so that we'll still own + * the buffer when we return to the caller. The buffer disposal on + * error must be paid attention to very carefully, as it has been + * broken since commit efa092f3d4c6 "[XFS] Fixes a bug in the quota + * code when allocating a new dquot record" in 2005, and the later + * conversion to xfs_defer_ops in commit 310a75a3c6c747 failed to keep + * the buffer locked across the _defer_finish call. We can now do + * this correctly with xfs_defer_bjoin. + * + * Above, we allocated a disk block for the dquot information and used + * get_buf to initialize the dquot. If the _defer_finish fails, the old + * transaction is gone but the new buffer is not joined or held to any + * transaction, so we must _buf_relse it. + * + * If everything succeeds, the caller of this function is returned a + * buffer that is locked and held to the transaction. The caller + * is responsible for unlocking any buffer passed back, either + * manually or by committing the transaction. On error, the buffer is + * released and not passed back. + * + * Keep the quota inode ILOCKed until after the transaction commit to + * maintain the atomicity of bmap/rmap updates. + */ + xfs_trans_bhold(tp, bp); + error = xfs_trans_commit(tp); + xfs_iunlock(quotip, XFS_ILOCK_EXCL); + if (error) { + xfs_buf_relse(bp); + return error; + } + + *bpp = bp; + return 0; + +err_cancel: + xfs_trans_cancel(tp); + xfs_iunlock(quotip, XFS_ILOCK_EXCL); + return error; +} + +/* + * Read in the in-core dquot's on-disk metadata and return the buffer. + * Returns ENOENT to signal a hole. + */ +STATIC int +xfs_dquot_disk_read( + struct xfs_mount *mp, + struct xfs_dquot *dqp, + struct xfs_buf **bpp) +{ + struct xfs_bmbt_irec map; + struct xfs_buf *bp; + xfs_dqtype_t qtype = xfs_dquot_type(dqp); + struct xfs_inode *quotip = xfs_quota_inode(mp, qtype); + uint lock_mode; + int nmaps = 1; + int error; + + lock_mode = xfs_ilock_data_map_shared(quotip); + if (!xfs_this_quota_on(mp, qtype)) { + /* + * Return if this type of quotas is turned off while we + * didn't have the quota inode lock. + */ + xfs_iunlock(quotip, lock_mode); + return -ESRCH; + } + + /* + * Find the block map; no allocations yet + */ + error = xfs_bmapi_read(quotip, dqp->q_fileoffset, + XFS_DQUOT_CLUSTER_SIZE_FSB, &map, &nmaps, 0); + xfs_iunlock(quotip, lock_mode); + if (error) + return error; + + ASSERT(nmaps == 1); + ASSERT(map.br_blockcount >= 1); + ASSERT(map.br_startblock != DELAYSTARTBLOCK); + if (map.br_startblock == HOLESTARTBLOCK) + return -ENOENT; + + trace_xfs_dqtobp_read(dqp); + + /* + * store the blkno etc so that we don't have to do the + * mapping all the time + */ + dqp->q_blkno = XFS_FSB_TO_DADDR(mp, map.br_startblock); + + error = xfs_trans_read_buf(mp, NULL, mp->m_ddev_targp, dqp->q_blkno, + mp->m_quotainfo->qi_dqchunklen, 0, &bp, + &xfs_dquot_buf_ops); + if (xfs_metadata_is_sick(error)) + xfs_dquot_mark_sick(dqp); + if (error) { + ASSERT(bp == NULL); + return error; + } + + ASSERT(xfs_buf_islocked(bp)); + xfs_buf_set_ref(bp, XFS_DQUOT_REF); + *bpp = bp; + + return 0; +} + +/* Allocate and initialize everything we need for an incore dquot. */ +STATIC struct xfs_dquot * +xfs_dquot_alloc( + struct xfs_mount *mp, + xfs_dqid_t id, + xfs_dqtype_t type) +{ + struct xfs_dquot *dqp; + + dqp = kmem_cache_zalloc(xfs_dquot_cache, GFP_KERNEL | __GFP_NOFAIL); + + dqp->q_type = type; + dqp->q_id = id; + dqp->q_mount = mp; + INIT_LIST_HEAD(&dqp->q_lru); + mutex_init(&dqp->q_qlock); + init_waitqueue_head(&dqp->q_pinwait); + dqp->q_fileoffset = (xfs_fileoff_t)id / mp->m_quotainfo->qi_dqperchunk; + /* + * Offset of dquot in the (fixed sized) dquot chunk. + */ + dqp->q_bufoffset = (id % mp->m_quotainfo->qi_dqperchunk) * + sizeof(struct xfs_dqblk); + + /* + * Because we want to use a counting completion, complete + * the flush completion once to allow a single access to + * the flush completion without blocking. + */ + init_completion(&dqp->q_flush); + complete(&dqp->q_flush); + + /* + * Make sure group quotas have a different lock class than user + * quotas. + */ + switch (type) { + case XFS_DQTYPE_USER: + /* uses the default lock class */ + break; + case XFS_DQTYPE_GROUP: + lockdep_set_class(&dqp->q_qlock, &xfs_dquot_group_class); + break; + case XFS_DQTYPE_PROJ: + lockdep_set_class(&dqp->q_qlock, &xfs_dquot_project_class); + break; + default: + ASSERT(0); + break; + } + + xfs_qm_dquot_logitem_init(dqp); + + XFS_STATS_INC(mp, xs_qm_dquot); + return dqp; +} + +/* Check the ondisk dquot's id and type match what the incore dquot expects. */ +static bool +xfs_dquot_check_type( + struct xfs_dquot *dqp, + struct xfs_disk_dquot *ddqp) +{ + uint8_t ddqp_type; + uint8_t dqp_type; + + ddqp_type = ddqp->d_type & XFS_DQTYPE_REC_MASK; + dqp_type = xfs_dquot_type(dqp); + + if (be32_to_cpu(ddqp->d_id) != dqp->q_id) + return false; + + /* + * V5 filesystems always expect an exact type match. V4 filesystems + * expect an exact match for user dquots and for non-root group and + * project dquots. + */ + if (xfs_has_crc(dqp->q_mount) || + dqp_type == XFS_DQTYPE_USER || dqp->q_id != 0) + return ddqp_type == dqp_type; + + /* + * V4 filesystems support either group or project quotas, but not both + * at the same time. The non-user quota file can be switched between + * group and project quota uses depending on the mount options, which + * means that we can encounter the other type when we try to load quota + * defaults. Quotacheck will soon reset the entire quota file + * (including the root dquot) anyway, but don't log scary corruption + * reports to dmesg. + */ + return ddqp_type == XFS_DQTYPE_GROUP || ddqp_type == XFS_DQTYPE_PROJ; +} + +/* Copy the in-core quota fields in from the on-disk buffer. */ +STATIC int +xfs_dquot_from_disk( + struct xfs_dquot *dqp, + struct xfs_buf *bp) +{ + struct xfs_dqblk *dqb = xfs_buf_offset(bp, dqp->q_bufoffset); + struct xfs_disk_dquot *ddqp = &dqb->dd_diskdq; + + /* + * Ensure that we got the type and ID we were looking for. + * Everything else was checked by the dquot buffer verifier. + */ + if (!xfs_dquot_check_type(dqp, ddqp)) { + xfs_alert_tag(bp->b_mount, XFS_PTAG_VERIFIER_ERROR, + "Metadata corruption detected at %pS, quota %u", + __this_address, dqp->q_id); + xfs_alert(bp->b_mount, "Unmount and run xfs_repair"); + xfs_dquot_mark_sick(dqp); + return -EFSCORRUPTED; + } + + /* copy everything from disk dquot to the incore dquot */ + dqp->q_type = ddqp->d_type; + dqp->q_blk.hardlimit = be64_to_cpu(ddqp->d_blk_hardlimit); + dqp->q_blk.softlimit = be64_to_cpu(ddqp->d_blk_softlimit); + dqp->q_ino.hardlimit = be64_to_cpu(ddqp->d_ino_hardlimit); + dqp->q_ino.softlimit = be64_to_cpu(ddqp->d_ino_softlimit); + dqp->q_rtb.hardlimit = be64_to_cpu(ddqp->d_rtb_hardlimit); + dqp->q_rtb.softlimit = be64_to_cpu(ddqp->d_rtb_softlimit); + + dqp->q_blk.count = be64_to_cpu(ddqp->d_bcount); + dqp->q_ino.count = be64_to_cpu(ddqp->d_icount); + dqp->q_rtb.count = be64_to_cpu(ddqp->d_rtbcount); + + dqp->q_blk.timer = xfs_dquot_from_disk_ts(ddqp, ddqp->d_btimer); + dqp->q_ino.timer = xfs_dquot_from_disk_ts(ddqp, ddqp->d_itimer); + dqp->q_rtb.timer = xfs_dquot_from_disk_ts(ddqp, ddqp->d_rtbtimer); + + /* + * Reservation counters are defined as reservation plus current usage + * to avoid having to add every time. + */ + dqp->q_blk.reserved = dqp->q_blk.count; + dqp->q_ino.reserved = dqp->q_ino.count; + dqp->q_rtb.reserved = dqp->q_rtb.count; + + /* initialize the dquot speculative prealloc thresholds */ + xfs_dquot_set_prealloc_limits(dqp); + return 0; +} + +/* Copy the in-core quota fields into the on-disk buffer. */ +void +xfs_dquot_to_disk( + struct xfs_disk_dquot *ddqp, + struct xfs_dquot *dqp) +{ + ddqp->d_magic = cpu_to_be16(XFS_DQUOT_MAGIC); + ddqp->d_version = XFS_DQUOT_VERSION; + ddqp->d_type = dqp->q_type; + ddqp->d_id = cpu_to_be32(dqp->q_id); + ddqp->d_pad0 = 0; + ddqp->d_pad = 0; + + ddqp->d_blk_hardlimit = cpu_to_be64(dqp->q_blk.hardlimit); + ddqp->d_blk_softlimit = cpu_to_be64(dqp->q_blk.softlimit); + ddqp->d_ino_hardlimit = cpu_to_be64(dqp->q_ino.hardlimit); + ddqp->d_ino_softlimit = cpu_to_be64(dqp->q_ino.softlimit); + ddqp->d_rtb_hardlimit = cpu_to_be64(dqp->q_rtb.hardlimit); + ddqp->d_rtb_softlimit = cpu_to_be64(dqp->q_rtb.softlimit); + + ddqp->d_bcount = cpu_to_be64(dqp->q_blk.count); + ddqp->d_icount = cpu_to_be64(dqp->q_ino.count); + ddqp->d_rtbcount = cpu_to_be64(dqp->q_rtb.count); + + ddqp->d_bwarns = 0; + ddqp->d_iwarns = 0; + ddqp->d_rtbwarns = 0; + + ddqp->d_btimer = xfs_dquot_to_disk_ts(dqp, dqp->q_blk.timer); + ddqp->d_itimer = xfs_dquot_to_disk_ts(dqp, dqp->q_ino.timer); + ddqp->d_rtbtimer = xfs_dquot_to_disk_ts(dqp, dqp->q_rtb.timer); +} + +/* + * Read in the ondisk dquot using dqtobp() then copy it to an incore version, + * and release the buffer immediately. If @can_alloc is true, fill any + * holes in the on-disk metadata. + */ +static int +xfs_qm_dqread( + struct xfs_mount *mp, + xfs_dqid_t id, + xfs_dqtype_t type, + bool can_alloc, + struct xfs_dquot **dqpp) +{ + struct xfs_dquot *dqp; + struct xfs_buf *bp; + int error; + + dqp = xfs_dquot_alloc(mp, id, type); + trace_xfs_dqread(dqp); + + /* Try to read the buffer, allocating if necessary. */ + error = xfs_dquot_disk_read(mp, dqp, &bp); + if (error == -ENOENT && can_alloc) + error = xfs_dquot_disk_alloc(dqp, &bp); + if (error) + goto err; + + /* + * At this point we should have a clean locked buffer. Copy the data + * to the incore dquot and release the buffer since the incore dquot + * has its own locking protocol so we needn't tie up the buffer any + * further. + */ + ASSERT(xfs_buf_islocked(bp)); + error = xfs_dquot_from_disk(dqp, bp); + xfs_buf_relse(bp); + if (error) + goto err; + + *dqpp = dqp; + return error; + +err: + trace_xfs_dqread_fail(dqp); + xfs_qm_dqdestroy(dqp); + *dqpp = NULL; + return error; +} + +/* + * Advance to the next id in the current chunk, or if at the + * end of the chunk, skip ahead to first id in next allocated chunk + * using the SEEK_DATA interface. + */ +static int +xfs_dq_get_next_id( + struct xfs_mount *mp, + xfs_dqtype_t type, + xfs_dqid_t *id) +{ + struct xfs_inode *quotip = xfs_quota_inode(mp, type); + xfs_dqid_t next_id = *id + 1; /* simple advance */ + uint lock_flags; + struct xfs_bmbt_irec got; + struct xfs_iext_cursor cur; + xfs_fsblock_t start; + int error = 0; + + /* If we'd wrap past the max ID, stop */ + if (next_id < *id) + return -ENOENT; + + /* If new ID is within the current chunk, advancing it sufficed */ + if (next_id % mp->m_quotainfo->qi_dqperchunk) { + *id = next_id; + return 0; + } + + /* Nope, next_id is now past the current chunk, so find the next one */ + start = (xfs_fsblock_t)next_id / mp->m_quotainfo->qi_dqperchunk; + + lock_flags = xfs_ilock_data_map_shared(quotip); + error = xfs_iread_extents(NULL, quotip, XFS_DATA_FORK); + if (error) + goto out_unlock; + + if (xfs_iext_lookup_extent(quotip, "ip->i_df, start, &cur, &got)) { + /* contiguous chunk, bump startoff for the id calculation */ + if (got.br_startoff < start) + got.br_startoff = start; + *id = got.br_startoff * mp->m_quotainfo->qi_dqperchunk; + } else { + error = -ENOENT; + } + +out_unlock: + xfs_iunlock(quotip, lock_flags); + + return error; +} + +/* + * Look up the dquot in the in-core cache. If found, the dquot is returned + * locked and ready to go. + */ +static struct xfs_dquot * +xfs_qm_dqget_cache_lookup( + struct xfs_mount *mp, + xfs_dqid_t id, + xfs_dqtype_t type) +{ + struct xfs_quotainfo *qi = mp->m_quotainfo; + struct radix_tree_root *tree = xfs_dquot_tree(qi, type); + struct xfs_dquot *dqp; + +restart: + mutex_lock(&qi->qi_tree_lock); + dqp = radix_tree_lookup(tree, id); + if (!dqp) { + mutex_unlock(&qi->qi_tree_lock); + XFS_STATS_INC(mp, xs_qm_dqcachemisses); + return NULL; + } + + if (!lockref_get_not_dead(&dqp->q_lockref)) { + mutex_unlock(&qi->qi_tree_lock); + trace_xfs_dqget_freeing(dqp); + delay(1); + goto restart; + } + mutex_unlock(&qi->qi_tree_lock); + + trace_xfs_dqget_hit(dqp); + XFS_STATS_INC(mp, xs_qm_dqcachehits); + return dqp; +} + +/* + * Try to insert a new dquot into the in-core cache. If an error occurs the + * caller should throw away the dquot and start over. Otherwise, the dquot + * is returned (and held by the cache) as if there had been a cache hit. + * + * The insert needs to be done under memalloc_nofs context because the radix + * tree can do memory allocation during insert. The qi->qi_tree_lock is taken in + * memory reclaim when freeing unused dquots, so we cannot have the radix tree + * node allocation recursing into filesystem reclaim whilst we hold the + * qi_tree_lock. + */ +static int +xfs_qm_dqget_cache_insert( + struct xfs_mount *mp, + xfs_dqid_t id, + xfs_dqtype_t type, + struct xfs_dquot *dqp) +{ + struct xfs_quotainfo *qi = mp->m_quotainfo; + struct radix_tree_root *tree = xfs_dquot_tree(qi, type); + unsigned int nofs_flags; + int error; + + nofs_flags = memalloc_nofs_save(); + mutex_lock(&qi->qi_tree_lock); + error = radix_tree_insert(tree, id, dqp); + if (unlikely(error)) { + trace_xfs_dqget_dup(dqp); + goto out_unlock; + } + + lockref_init(&dqp->q_lockref); + qi->qi_dquots++; + +out_unlock: + mutex_unlock(&qi->qi_tree_lock); + memalloc_nofs_restore(nofs_flags); + return error; +} + +/* Check our input parameters. */ +static int +xfs_qm_dqget_checks( + struct xfs_mount *mp, + xfs_dqtype_t type) +{ + switch (type) { + case XFS_DQTYPE_USER: + if (!XFS_IS_UQUOTA_ON(mp)) + return -ESRCH; + return 0; + case XFS_DQTYPE_GROUP: + if (!XFS_IS_GQUOTA_ON(mp)) + return -ESRCH; + return 0; + case XFS_DQTYPE_PROJ: + if (!XFS_IS_PQUOTA_ON(mp)) + return -ESRCH; + return 0; + default: + WARN_ON_ONCE(0); + return -EINVAL; + } +} + +/* + * Given the file system, id, and type (UDQUOT/GDQUOT/PDQUOT), return a + * dquot, doing an allocation (if requested) as needed. + */ +int +xfs_qm_dqget( + struct xfs_mount *mp, + xfs_dqid_t id, + xfs_dqtype_t type, + bool can_alloc, + struct xfs_dquot **O_dqpp) +{ + struct xfs_dquot *dqp; + int error; + + error = xfs_qm_dqget_checks(mp, type); + if (error) + return error; + +restart: + dqp = xfs_qm_dqget_cache_lookup(mp, id, type); + if (dqp) + goto found; + + error = xfs_qm_dqread(mp, id, type, can_alloc, &dqp); + if (error) + return error; + + error = xfs_qm_dqget_cache_insert(mp, id, type, dqp); + if (error) { + xfs_qm_dqdestroy(dqp); + if (error == -EEXIST) { + /* + * Duplicate found. Just throw away the new dquot and + * start over. + */ + XFS_STATS_INC(mp, xs_qm_dquot_dups); + goto restart; + } + return error; + } + + trace_xfs_dqget_miss(dqp); +found: + *O_dqpp = dqp; + return 0; +} + +/* + * Given a dquot id and type, read and initialize a dquot from the on-disk + * metadata. This function is only for use during quota initialization so + * it ignores the dquot cache assuming that the dquot shrinker isn't set up. + * The caller is responsible for _qm_dqdestroy'ing the returned dquot. + */ +int +xfs_qm_dqget_uncached( + struct xfs_mount *mp, + xfs_dqid_t id, + xfs_dqtype_t type, + struct xfs_dquot **dqpp) +{ + int error; + + error = xfs_qm_dqget_checks(mp, type); + if (error) + return error; + + return xfs_qm_dqread(mp, id, type, 0, dqpp); +} + +/* Return the quota id for a given inode and type. */ +xfs_dqid_t +xfs_qm_id_for_quotatype( + struct xfs_inode *ip, + xfs_dqtype_t type) +{ + switch (type) { + case XFS_DQTYPE_USER: + return i_uid_read(VFS_I(ip)); + case XFS_DQTYPE_GROUP: + return i_gid_read(VFS_I(ip)); + case XFS_DQTYPE_PROJ: + return ip->i_projid; + } + ASSERT(0); + return 0; +} + +/* + * Return the dquot for a given inode and type. If @can_alloc is true, then + * allocate blocks if needed. The inode's ILOCK must be held and it must not + * have already had an inode attached. + */ +int +xfs_qm_dqget_inode( + struct xfs_inode *ip, + xfs_dqtype_t type, + bool can_alloc, + struct xfs_dquot **dqpp) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_dquot *dqp; + xfs_dqid_t id; + int error; + + ASSERT(!*dqpp); + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + + error = xfs_qm_dqget_checks(mp, type); + if (error) + return error; + + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + ASSERT(xfs_inode_dquot(ip, type) == NULL); + ASSERT(!xfs_is_metadir_inode(ip)); + + id = xfs_qm_id_for_quotatype(ip, type); + +restart: + dqp = xfs_qm_dqget_cache_lookup(mp, id, type); + if (dqp) + goto found; + + /* + * Dquot cache miss. We don't want to keep the inode lock across + * a (potential) disk read. Also we don't want to deal with the lock + * ordering between quotainode and this inode. OTOH, dropping the inode + * lock here means dealing with a chown that can happen before + * we re-acquire the lock. + */ + xfs_iunlock(ip, XFS_ILOCK_EXCL); + error = xfs_qm_dqread(mp, id, type, can_alloc, &dqp); + xfs_ilock(ip, XFS_ILOCK_EXCL); + if (error) + return error; + + /* + * A dquot could be attached to this inode by now, since we had + * dropped the ilock. + */ + if (xfs_this_quota_on(mp, type)) { + struct xfs_dquot *dqp1; + + dqp1 = xfs_inode_dquot(ip, type); + if (dqp1) { + xfs_qm_dqdestroy(dqp); + dqp = dqp1; + goto dqret; + } + } else { + /* inode stays locked on return */ + xfs_qm_dqdestroy(dqp); + return -ESRCH; + } + + error = xfs_qm_dqget_cache_insert(mp, id, type, dqp); + if (error) { + xfs_qm_dqdestroy(dqp); + if (error == -EEXIST) { + /* + * Duplicate found. Just throw away the new dquot and + * start over. + */ + XFS_STATS_INC(mp, xs_qm_dquot_dups); + goto restart; + } + return error; + } + +dqret: + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + trace_xfs_dqget_miss(dqp); +found: + trace_xfs_dqattach_get(dqp); + *dqpp = dqp; + return 0; +} + +/* + * Starting at @id and progressing upwards, look for an initialized incore + * dquot, lock it, and return it. + */ +int +xfs_qm_dqget_next( + struct xfs_mount *mp, + xfs_dqid_t id, + xfs_dqtype_t type, + struct xfs_dquot **dqpp) +{ + struct xfs_dquot *dqp; + int error = 0; + + *dqpp = NULL; + for (; !error; error = xfs_dq_get_next_id(mp, type, &id)) { + error = xfs_qm_dqget(mp, id, type, false, &dqp); + if (error == -ENOENT) + continue; + else if (error != 0) + break; + + mutex_lock(&dqp->q_qlock); + if (!XFS_IS_DQUOT_UNINITIALIZED(dqp)) { + *dqpp = dqp; + return 0; + } + + mutex_unlock(&dqp->q_qlock); + xfs_qm_dqrele(dqp); + } + + return error; +} + +/* + * Release a reference to the dquot. + */ +void +xfs_qm_dqrele( + struct xfs_dquot *dqp) +{ + if (!dqp) + return; + + trace_xfs_dqrele(dqp); + + if (lockref_put_or_lock(&dqp->q_lockref)) + return; + if (!--dqp->q_lockref.count) { + struct xfs_quotainfo *qi = dqp->q_mount->m_quotainfo; + + trace_xfs_dqrele_free(dqp); + if (list_lru_add_obj(&qi->qi_lru, &dqp->q_lru)) + XFS_STATS_INC(dqp->q_mount, xs_qm_dquot_unused); + } + spin_unlock(&dqp->q_lockref.lock); +} + +/* + * This is the dquot flushing I/O completion routine. It is called + * from interrupt level when the buffer containing the dquot is + * flushed to disk. It is responsible for removing the dquot logitem + * from the AIL if it has not been re-logged, and unlocking the dquot's + * flush lock. This behavior is very similar to that of inodes.. + */ +static void +xfs_qm_dqflush_done( + struct xfs_log_item *lip) +{ + struct xfs_dq_logitem *qlip = + container_of(lip, struct xfs_dq_logitem, qli_item); + struct xfs_dquot *dqp = qlip->qli_dquot; + struct xfs_ail *ailp = lip->li_ailp; + struct xfs_buf *bp = NULL; + xfs_lsn_t tail_lsn; + + /* + * We only want to pull the item from the AIL if its + * location in the log has not changed since we started the flush. + * Thus, we only bother if the dquot's lsn has + * not changed. First we check the lsn outside the lock + * since it's cheaper, and then we recheck while + * holding the lock before removing the dquot from the AIL. + */ + if (test_bit(XFS_LI_IN_AIL, &lip->li_flags) && + (lip->li_lsn == qlip->qli_flush_lsn || + test_bit(XFS_LI_FAILED, &lip->li_flags))) { + spin_lock(&ailp->ail_lock); + clear_bit(XFS_LI_FAILED, &lip->li_flags); + if (lip->li_lsn == qlip->qli_flush_lsn) { + /* xfs_ail_update_finish() drops the AIL lock */ + tail_lsn = xfs_ail_delete_one(ailp, lip); + xfs_ail_update_finish(ailp, tail_lsn); + } else { + spin_unlock(&ailp->ail_lock); + } + } + + /* + * If this dquot hasn't been dirtied since initiating the last dqflush, + * release the buffer reference. We already unlinked this dquot item + * from the buffer. + */ + spin_lock(&qlip->qli_lock); + if (!qlip->qli_dirty) { + bp = lip->li_buf; + lip->li_buf = NULL; + } + spin_unlock(&qlip->qli_lock); + if (bp) + xfs_buf_rele(bp); + + /* + * Release the dq's flush lock since we're done with it. + */ + xfs_dqfunlock(dqp); +} + +void +xfs_buf_dquot_iodone( + struct xfs_buf *bp) +{ + struct xfs_log_item *lip, *n; + + list_for_each_entry_safe(lip, n, &bp->b_li_list, li_bio_list) { + list_del_init(&lip->li_bio_list); + xfs_qm_dqflush_done(lip); + } +} + +/* Check incore dquot for errors before we flush. */ +static xfs_failaddr_t +xfs_qm_dqflush_check( + struct xfs_dquot *dqp) +{ + xfs_dqtype_t type = xfs_dquot_type(dqp); + + if (type != XFS_DQTYPE_USER && + type != XFS_DQTYPE_GROUP && + type != XFS_DQTYPE_PROJ) + return __this_address; + + /* bigtime flag should never be set on root dquots */ + if (dqp->q_type & XFS_DQTYPE_BIGTIME) { + if (!xfs_has_bigtime(dqp->q_mount)) + return __this_address; + if (dqp->q_id == 0) + return __this_address; + } + + if (dqp->q_id == 0) + return NULL; + + if (dqp->q_blk.softlimit && dqp->q_blk.count > dqp->q_blk.softlimit && + !dqp->q_blk.timer) + return __this_address; + + if (dqp->q_ino.softlimit && dqp->q_ino.count > dqp->q_ino.softlimit && + !dqp->q_ino.timer) + return __this_address; + + if (dqp->q_rtb.softlimit && dqp->q_rtb.count > dqp->q_rtb.softlimit && + !dqp->q_rtb.timer) + return __this_address; + + return NULL; +} + +/* + * Get the buffer containing the on-disk dquot. + * + * Requires dquot flush lock, will clear the dirty flag, delete the quota log + * item from the AIL, and shut down the system if something goes wrong. + */ +static int +xfs_dquot_read_buf( + struct xfs_trans *tp, + struct xfs_dquot *dqp, + struct xfs_buf **bpp) +{ + struct xfs_mount *mp = dqp->q_mount; + struct xfs_buf *bp = NULL; + int error; + + error = xfs_trans_read_buf(mp, tp, mp->m_ddev_targp, dqp->q_blkno, + mp->m_quotainfo->qi_dqchunklen, 0, + &bp, &xfs_dquot_buf_ops); + if (xfs_metadata_is_sick(error)) + xfs_dquot_mark_sick(dqp); + if (error) + goto out_abort; + + *bpp = bp; + return 0; + +out_abort: + dqp->q_flags &= ~XFS_DQFLAG_DIRTY; + xfs_trans_ail_delete(&dqp->q_logitem.qli_item, 0); + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); + return error; +} + +/* + * Attach a dquot buffer to this dquot to avoid allocating a buffer during a + * dqflush, since dqflush can be called from reclaim context. Caller must hold + * the dqlock. + */ +int +xfs_dquot_attach_buf( + struct xfs_trans *tp, + struct xfs_dquot *dqp) +{ + struct xfs_dq_logitem *qlip = &dqp->q_logitem; + struct xfs_log_item *lip = &qlip->qli_item; + int error; + + spin_lock(&qlip->qli_lock); + if (!lip->li_buf) { + struct xfs_buf *bp = NULL; + + spin_unlock(&qlip->qli_lock); + error = xfs_dquot_read_buf(tp, dqp, &bp); + if (error) + return error; + + /* + * Hold the dquot buffer so that we retain our ref to it after + * detaching it from the transaction, then give that ref to the + * dquot log item so that the AIL does not have to read the + * dquot buffer to push this item. + */ + xfs_buf_hold(bp); + xfs_trans_brelse(tp, bp); + + spin_lock(&qlip->qli_lock); + lip->li_buf = bp; + } + qlip->qli_dirty = true; + spin_unlock(&qlip->qli_lock); + + return 0; +} + +/* + * Get a new reference the dquot buffer attached to this dquot for a dqflush + * operation. + * + * Returns 0 and a NULL bp if none was attached to the dquot; 0 and a locked + * bp; or -EAGAIN if the buffer could not be locked. + */ +int +xfs_dquot_use_attached_buf( + struct xfs_dquot *dqp, + struct xfs_buf **bpp) +{ + struct xfs_buf *bp = dqp->q_logitem.qli_item.li_buf; + + /* + * A NULL buffer can happen if the dquot dirty flag was set but the + * filesystem shut down before transaction commit happened. In that + * case we're not going to flush anyway. + */ + if (!bp) { + ASSERT(xfs_is_shutdown(dqp->q_mount)); + + *bpp = NULL; + return 0; + } + + if (!xfs_buf_trylock(bp)) + return -EAGAIN; + + xfs_buf_hold(bp); + *bpp = bp; + return 0; +} + +/* + * Write a modified dquot to disk. + * The dquot must be locked and the flush lock too taken by caller. + * The flush lock will not be unlocked until the dquot reaches the disk, + * but the dquot is free to be unlocked and modified by the caller + * in the interim. Dquot is still locked on return. This behavior is + * identical to that of inodes. + */ +int +xfs_qm_dqflush( + struct xfs_dquot *dqp, + struct xfs_buf *bp) +{ + struct xfs_mount *mp = dqp->q_mount; + struct xfs_dq_logitem *qlip = &dqp->q_logitem; + struct xfs_log_item *lip = &qlip->qli_item; + struct xfs_dqblk *dqblk; + xfs_failaddr_t fa; + int error; + + ASSERT(XFS_DQ_IS_LOCKED(dqp)); + ASSERT(!completion_done(&dqp->q_flush)); + ASSERT(atomic_read(&dqp->q_pincount) == 0); + + trace_xfs_dqflush(dqp); + fa = xfs_qm_dqflush_check(dqp); + if (fa) { + xfs_alert(mp, "corrupt dquot ID 0x%x in memory at %pS", + dqp->q_id, fa); + xfs_dquot_mark_sick(dqp); + error = -EFSCORRUPTED; + goto out_abort; + } + + /* Flush the incore dquot to the ondisk buffer. */ + dqblk = xfs_buf_offset(bp, dqp->q_bufoffset); + xfs_dquot_to_disk(&dqblk->dd_diskdq, dqp); + + /* + * Clear the dirty field and remember the flush lsn for later use. + */ + dqp->q_flags &= ~XFS_DQFLAG_DIRTY; + + /* + * We hold the dquot lock, so nobody can dirty it while we're + * scheduling the write out. Clear the dirty-since-flush flag. + */ + spin_lock(&qlip->qli_lock); + qlip->qli_dirty = false; + spin_unlock(&qlip->qli_lock); + + xfs_trans_ail_copy_lsn(mp->m_ail, &qlip->qli_flush_lsn, &lip->li_lsn); + + /* + * copy the lsn into the on-disk dquot now while we have the in memory + * dquot here. This can't be done later in the write verifier as we + * can't get access to the log item at that point in time. + * + * We also calculate the CRC here so that the on-disk dquot in the + * buffer always has a valid CRC. This ensures there is no possibility + * of a dquot without an up-to-date CRC getting to disk. + */ + if (xfs_has_crc(mp)) { + dqblk->dd_lsn = cpu_to_be64(lip->li_lsn); + xfs_update_cksum((char *)dqblk, sizeof(struct xfs_dqblk), + XFS_DQUOT_CRC_OFF); + } + + /* + * Attach the dquot to the buffer so that we can remove this dquot from + * the AIL and release the flush lock once the dquot is synced to disk. + */ + bp->b_iodone = xfs_buf_dquot_iodone; + list_add_tail(&lip->li_bio_list, &bp->b_li_list); + + /* + * If the buffer is pinned then push on the log so we won't + * get stuck waiting in the write for too long. + */ + if (xfs_buf_ispinned(bp)) { + trace_xfs_dqflush_force(dqp); + xfs_log_force(mp, 0); + } + + trace_xfs_dqflush_done(dqp); + return 0; + +out_abort: + /* + * Shut down the log before removing the dquot item from the AIL. + * Otherwise, the log tail may advance past this item's LSN while + * log writes are still in progress, making these unflushed changes + * unrecoverable on the next mount. + */ + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); + dqp->q_flags &= ~XFS_DQFLAG_DIRTY; + xfs_trans_ail_delete(lip, 0); + xfs_dqfunlock(dqp); + return error; +} + +/* + * Lock two xfs_dquot structures. + * + * To avoid deadlocks we always lock the quota structure with + * the lowerd id first. + */ +void +xfs_dqlock2( + struct xfs_dquot *d1, + struct xfs_dquot *d2) +{ + if (d1 && d2) { + ASSERT(d1 != d2); + if (d1->q_id > d2->q_id) { + mutex_lock(&d2->q_qlock); + mutex_lock_nested(&d1->q_qlock, XFS_QLOCK_NESTED); + } else { + mutex_lock(&d1->q_qlock); + mutex_lock_nested(&d2->q_qlock, XFS_QLOCK_NESTED); + } + } else if (d1) { + mutex_lock(&d1->q_qlock); + } else if (d2) { + mutex_lock(&d2->q_qlock); + } +} + +static int +xfs_dqtrx_cmp( + const void *a, + const void *b) +{ + const struct xfs_dqtrx *qa = a; + const struct xfs_dqtrx *qb = b; + + if (qa->qt_dquot->q_id > qb->qt_dquot->q_id) + return 1; + if (qa->qt_dquot->q_id < qb->qt_dquot->q_id) + return -1; + return 0; +} + +void +xfs_dqlockn( + struct xfs_dqtrx *q) +{ + unsigned int i; + + BUILD_BUG_ON(XFS_QM_TRANS_MAXDQS > MAX_LOCKDEP_SUBCLASSES); + + /* Sort in order of dquot id, do not allow duplicates */ + for (i = 0; i < XFS_QM_TRANS_MAXDQS && q[i].qt_dquot != NULL; i++) { + unsigned int j; + + for (j = 0; j < i; j++) + ASSERT(q[i].qt_dquot != q[j].qt_dquot); + } + if (i == 0) + return; + + sort(q, i, sizeof(struct xfs_dqtrx), xfs_dqtrx_cmp, NULL); + + mutex_lock(&q[0].qt_dquot->q_qlock); + for (i = 1; i < XFS_QM_TRANS_MAXDQS && q[i].qt_dquot != NULL; i++) + mutex_lock_nested(&q[i].qt_dquot->q_qlock, + XFS_QLOCK_NESTED + i - 1); +} + +int __init +xfs_qm_init(void) +{ + xfs_dquot_cache = kmem_cache_create("xfs_dquot", + sizeof(struct xfs_dquot), + 0, 0, NULL); + if (!xfs_dquot_cache) + goto out; + + xfs_dqtrx_cache = kmem_cache_create("xfs_dqtrx", + sizeof(struct xfs_dquot_acct), + 0, 0, NULL); + if (!xfs_dqtrx_cache) + goto out_free_dquot_cache; + + return 0; + +out_free_dquot_cache: + kmem_cache_destroy(xfs_dquot_cache); +out: + return -ENOMEM; +} + +void +xfs_qm_exit(void) +{ + kmem_cache_destroy(xfs_dqtrx_cache); + kmem_cache_destroy(xfs_dquot_cache); +} diff --git a/fs/xfs/xfs_dquot.h b/fs/xfs/xfs_dquot.h new file mode 100644 index 000000000..bbb824adc --- /dev/null +++ b/fs/xfs/xfs_dquot.h @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_DQUOT_H__ +#define __XFS_DQUOT_H__ + +/* + * Dquots are structures that hold quota information about a user or a group, + * much like inodes are for files. In fact, dquots share many characteristics + * with inodes. However, dquots can also be a centralized resource, relative + * to a collection of inodes. In this respect, dquots share some characteristics + * of the superblock. + * XFS dquots exploit both those in its algorithms. They make every attempt + * to not be a bottleneck when quotas are on and have minimal impact, if any, + * when quotas are off. + */ + +struct xfs_mount; +struct xfs_trans; + +enum { + XFS_QLOWSP_1_PCNT = 0, + XFS_QLOWSP_3_PCNT, + XFS_QLOWSP_5_PCNT, + XFS_QLOWSP_MAX +}; + +struct xfs_dquot_res { + /* Total resources allocated and reserved. */ + xfs_qcnt_t reserved; + + /* Total resources allocated. */ + xfs_qcnt_t count; + + /* Absolute and preferred limits. */ + xfs_qcnt_t hardlimit; + xfs_qcnt_t softlimit; + + /* + * For root dquots, this is the default grace period, in seconds. + * Otherwise, this is when the quota grace period expires, + * in seconds since the Unix epoch. + */ + time64_t timer; +}; + +static inline bool +xfs_dquot_res_over_limits( + const struct xfs_dquot_res *qres) +{ + if ((qres->softlimit && qres->softlimit < qres->reserved) || + (qres->hardlimit && qres->hardlimit < qres->reserved)) + return true; + return false; +} + +struct xfs_dquot_pre { + xfs_qcnt_t q_prealloc_lo_wmark; + xfs_qcnt_t q_prealloc_hi_wmark; + int64_t q_low_space[XFS_QLOWSP_MAX]; +}; + +/* + * The incore dquot structure + */ +struct xfs_dquot { + struct list_head q_lru; + struct xfs_mount *q_mount; + xfs_dqtype_t q_type; + uint16_t q_flags; + xfs_dqid_t q_id; + struct lockref q_lockref; + int q_bufoffset; + xfs_daddr_t q_blkno; + xfs_fileoff_t q_fileoffset; + + struct xfs_dquot_res q_blk; /* regular blocks */ + struct xfs_dquot_res q_ino; /* inodes */ + struct xfs_dquot_res q_rtb; /* realtime blocks */ + + struct xfs_dq_logitem q_logitem; + + struct xfs_dquot_pre q_blk_prealloc; + struct xfs_dquot_pre q_rtb_prealloc; + + struct mutex q_qlock; + struct completion q_flush; + atomic_t q_pincount; + struct wait_queue_head q_pinwait; +}; + +/* + * Lock hierarchy for q_qlock: + * XFS_QLOCK_NORMAL is the implicit default, + * XFS_QLOCK_NESTED is the dquot with the higher id in xfs_dqlock2 + */ +enum { + XFS_QLOCK_NORMAL = 0, + XFS_QLOCK_NESTED, +}; + +/* + * Manage the q_flush completion queue embedded in the dquot. This completion + * queue synchronizes processes attempting to flush the in-core dquot back to + * disk. + */ +static inline void xfs_dqflock(struct xfs_dquot *dqp) +{ + wait_for_completion(&dqp->q_flush); +} + +static inline bool xfs_dqflock_nowait(struct xfs_dquot *dqp) +{ + return try_wait_for_completion(&dqp->q_flush); +} + +static inline void xfs_dqfunlock(struct xfs_dquot *dqp) +{ + complete(&dqp->q_flush); +} + +static inline int +xfs_dquot_type(const struct xfs_dquot *dqp) +{ + return dqp->q_type & XFS_DQTYPE_REC_MASK; +} + +static inline int xfs_this_quota_on(struct xfs_mount *mp, xfs_dqtype_t type) +{ + switch (type) { + case XFS_DQTYPE_USER: + return XFS_IS_UQUOTA_ON(mp); + case XFS_DQTYPE_GROUP: + return XFS_IS_GQUOTA_ON(mp); + case XFS_DQTYPE_PROJ: + return XFS_IS_PQUOTA_ON(mp); + default: + return 0; + } +} + +static inline struct xfs_dquot *xfs_inode_dquot( + struct xfs_inode *ip, + xfs_dqtype_t type) +{ + if (xfs_is_metadir_inode(ip)) + return NULL; + + switch (type) { + case XFS_DQTYPE_USER: + return ip->i_udquot; + case XFS_DQTYPE_GROUP: + return ip->i_gdquot; + case XFS_DQTYPE_PROJ: + return ip->i_pdquot; + default: + return NULL; + } +} + +/* Decide if the dquot's limits are actually being enforced. */ +static inline bool +xfs_dquot_is_enforced( + const struct xfs_dquot *dqp) +{ + switch (xfs_dquot_type(dqp)) { + case XFS_DQTYPE_USER: + return XFS_IS_UQUOTA_ENFORCED(dqp->q_mount); + case XFS_DQTYPE_GROUP: + return XFS_IS_GQUOTA_ENFORCED(dqp->q_mount); + case XFS_DQTYPE_PROJ: + return XFS_IS_PQUOTA_ENFORCED(dqp->q_mount); + } + ASSERT(0); + return false; +} + +/* + * Check whether a dquot is under low free space conditions. We assume the quota + * is enabled and enforced. + */ +static inline bool xfs_dquot_lowsp(struct xfs_dquot *dqp) +{ + int64_t freesp; + + freesp = dqp->q_blk.hardlimit - dqp->q_blk.reserved; + if (freesp < dqp->q_blk_prealloc.q_low_space[XFS_QLOWSP_1_PCNT]) + return true; + + freesp = dqp->q_rtb.hardlimit - dqp->q_rtb.reserved; + if (freesp < dqp->q_rtb_prealloc.q_low_space[XFS_QLOWSP_1_PCNT]) + return true; + + return false; +} + +void xfs_dquot_to_disk(struct xfs_disk_dquot *ddqp, struct xfs_dquot *dqp); + +#define XFS_DQ_IS_LOCKED(dqp) (mutex_is_locked(&((dqp)->q_qlock))) +#define XFS_DQ_IS_DIRTY(dqp) ((dqp)->q_flags & XFS_DQFLAG_DIRTY) + +void xfs_qm_dqdestroy(struct xfs_dquot *dqp); +int xfs_qm_dqflush(struct xfs_dquot *dqp, struct xfs_buf *bp); +void xfs_qm_dqunpin_wait(struct xfs_dquot *dqp); +void xfs_qm_adjust_dqtimers(struct xfs_dquot *d); +void xfs_qm_adjust_dqlimits(struct xfs_dquot *d); +xfs_dqid_t xfs_qm_id_for_quotatype(struct xfs_inode *ip, + xfs_dqtype_t type); +int xfs_qm_dqget(struct xfs_mount *mp, xfs_dqid_t id, + xfs_dqtype_t type, bool can_alloc, + struct xfs_dquot **dqpp); +int xfs_qm_dqget_inode(struct xfs_inode *ip, xfs_dqtype_t type, + bool can_alloc, struct xfs_dquot **dqpp); +int xfs_qm_dqget_next(struct xfs_mount *mp, xfs_dqid_t id, + xfs_dqtype_t type, struct xfs_dquot **dqpp); +int xfs_qm_dqget_uncached(struct xfs_mount *mp, + xfs_dqid_t id, xfs_dqtype_t type, + struct xfs_dquot **dqpp); + +void xfs_dqlock2(struct xfs_dquot *, struct xfs_dquot *); +void xfs_dqlockn(struct xfs_dqtrx *q); + +void xfs_dquot_set_prealloc_limits(struct xfs_dquot *); + +int xfs_dquot_attach_buf(struct xfs_trans *tp, struct xfs_dquot *dqp); +int xfs_dquot_use_attached_buf(struct xfs_dquot *dqp, struct xfs_buf **bpp); +void xfs_dquot_detach_buf(struct xfs_dquot *dqp); + +static inline struct xfs_dquot *xfs_qm_dqhold(struct xfs_dquot *dqp) +{ + lockref_get(&dqp->q_lockref); + return dqp; +} + +time64_t xfs_dquot_set_timeout(struct xfs_mount *mp, time64_t timeout); +time64_t xfs_dquot_set_grace_period(time64_t grace); + +void xfs_qm_init_dquot_blk(struct xfs_trans *tp, xfs_dqid_t id, xfs_dqtype_t + type, struct xfs_buf *bp); + +#endif /* __XFS_DQUOT_H__ */ diff --git a/fs/xfs/xfs_dquot_item.c b/fs/xfs/xfs_dquot_item.c new file mode 100644 index 000000000..65a0e69c3 --- /dev/null +++ b/fs/xfs/xfs_dquot_item.c @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2003 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_quota.h" +#include "xfs_trans.h" +#include "xfs_buf_item.h" +#include "xfs_trans_priv.h" +#include "xfs_qm.h" +#include "xfs_log.h" +#include "xfs_error.h" + +static inline struct xfs_dq_logitem *DQUOT_ITEM(struct xfs_log_item *lip) +{ + return container_of(lip, struct xfs_dq_logitem, qli_item); +} + +/* + * returns the number of iovecs needed to log the given dquot item. + */ +STATIC void +xfs_qm_dquot_logitem_size( + struct xfs_log_item *lip, + int *nvecs, + int *nbytes) +{ + *nvecs += 2; + *nbytes += sizeof(struct xfs_dq_logformat) + + sizeof(struct xfs_disk_dquot); +} + +/* + * fills in the vector of log iovecs for the given dquot log item. + */ +STATIC void +xfs_qm_dquot_logitem_format( + struct xfs_log_item *lip, + struct xlog_format_buf *lfb) +{ + struct xfs_disk_dquot ddq; + struct xfs_dq_logitem *qlip = DQUOT_ITEM(lip); + struct xfs_dq_logformat *qlf; + + qlf = xlog_format_start(lfb, XLOG_REG_TYPE_QFORMAT); + qlf->qlf_type = XFS_LI_DQUOT; + qlf->qlf_size = 2; + qlf->qlf_id = qlip->qli_dquot->q_id; + qlf->qlf_blkno = qlip->qli_dquot->q_blkno; + qlf->qlf_len = 1; + qlf->qlf_boffset = qlip->qli_dquot->q_bufoffset; + xlog_format_commit(lfb, sizeof(struct xfs_dq_logformat)); + + xfs_dquot_to_disk(&ddq, qlip->qli_dquot); + + xlog_format_copy(lfb, XLOG_REG_TYPE_DQUOT, &ddq, + sizeof(struct xfs_disk_dquot)); +} + +/* + * Increment the pin count of the given dquot. + */ +STATIC void +xfs_qm_dquot_logitem_pin( + struct xfs_log_item *lip) +{ + struct xfs_dquot *dqp = DQUOT_ITEM(lip)->qli_dquot; + + ASSERT(XFS_DQ_IS_LOCKED(dqp)); + atomic_inc(&dqp->q_pincount); +} + +/* + * Decrement the pin count of the given dquot, and wake up + * anyone in xfs_dqwait_unpin() if the count goes to 0. The + * dquot must have been previously pinned with a call to + * xfs_qm_dquot_logitem_pin(). + */ +STATIC void +xfs_qm_dquot_logitem_unpin( + struct xfs_log_item *lip, + int remove) +{ + struct xfs_dquot *dqp = DQUOT_ITEM(lip)->qli_dquot; + + ASSERT(atomic_read(&dqp->q_pincount) > 0); + if (atomic_dec_and_test(&dqp->q_pincount)) + wake_up(&dqp->q_pinwait); +} + +/* + * This is called to wait for the given dquot to be unpinned. + * Most of these pin/unpin routines are plagiarized from inode code. + */ +void +xfs_qm_dqunpin_wait( + struct xfs_dquot *dqp) +{ + ASSERT(XFS_DQ_IS_LOCKED(dqp)); + if (atomic_read(&dqp->q_pincount) == 0) + return; + + /* + * Give the log a push so we don't wait here too long. + */ + xfs_log_force(dqp->q_mount, 0); + wait_event(dqp->q_pinwait, (atomic_read(&dqp->q_pincount) == 0)); +} + +STATIC uint +xfs_qm_dquot_logitem_push( + struct xfs_log_item *lip, + struct list_head *buffer_list) + __releases(&lip->li_ailp->ail_lock) + __acquires(&lip->li_ailp->ail_lock) +{ + struct xfs_dq_logitem *qlip = DQUOT_ITEM(lip); + struct xfs_dquot *dqp = qlip->qli_dquot; + struct xfs_buf *bp; + struct xfs_ail *ailp = lip->li_ailp; + uint rval = XFS_ITEM_SUCCESS; + int error; + + if (atomic_read(&dqp->q_pincount) > 0) + return XFS_ITEM_PINNED; + + if (!mutex_trylock(&dqp->q_qlock)) + return XFS_ITEM_LOCKED; + + /* + * Re-check the pincount now that we stabilized the value by + * taking the quota lock. + */ + if (atomic_read(&dqp->q_pincount) > 0) { + rval = XFS_ITEM_PINNED; + goto out_unlock; + } + + /* + * Someone else is already flushing the dquot. Nothing we can do + * here but wait for the flush to finish and remove the item from + * the AIL. + */ + if (!xfs_dqflock_nowait(dqp)) { + rval = XFS_ITEM_FLUSHING; + goto out_unlock; + } + + spin_unlock(&ailp->ail_lock); + + error = xfs_dquot_use_attached_buf(dqp, &bp); + if (error == -EAGAIN) { + xfs_dqfunlock(dqp); + rval = XFS_ITEM_LOCKED; + goto out_relock_ail; + } + + /* + * dqflush completes dqflock on error, and the delwri ioend does it on + * success. + */ + error = xfs_qm_dqflush(dqp, bp); + if (!error) { + if (!xfs_buf_delwri_queue(bp, buffer_list)) + rval = XFS_ITEM_FLUSHING; + } + xfs_buf_relse(bp); + /* + * The buffer no longer protects the log item from reclaim, so + * do not reference lip after this point. + */ + +out_relock_ail: + spin_lock(&ailp->ail_lock); +out_unlock: + mutex_unlock(&dqp->q_qlock); + return rval; +} + +STATIC void +xfs_qm_dquot_logitem_release( + struct xfs_log_item *lip) +{ + struct xfs_dquot *dqp = DQUOT_ITEM(lip)->qli_dquot; + + ASSERT(XFS_DQ_IS_LOCKED(dqp)); + + /* + * dquots are never 'held' from getting unlocked at the end of + * a transaction. Their locking and unlocking is hidden inside the + * transaction layer, within trans_commit. Hence, no LI_HOLD flag + * for the logitem. + */ + mutex_unlock(&dqp->q_qlock); +} + +STATIC void +xfs_qm_dquot_logitem_committing( + struct xfs_log_item *lip, + xfs_csn_t seq) +{ + return xfs_qm_dquot_logitem_release(lip); +} + +#ifdef DEBUG_EXPENSIVE +static void +xfs_qm_dquot_logitem_precommit_check( + struct xfs_dquot *dqp) +{ + struct xfs_mount *mp = dqp->q_mount; + struct xfs_disk_dquot ddq = { }; + xfs_failaddr_t fa; + + xfs_dquot_to_disk(&ddq, dqp); + fa = xfs_dquot_verify(mp, &ddq, dqp->q_id); + if (fa) { + XFS_CORRUPTION_ERROR("Bad dquot during logging", + XFS_ERRLEVEL_LOW, mp, &ddq, sizeof(ddq)); + xfs_alert(mp, + "Metadata corruption detected at %pS, dquot 0x%x", + fa, dqp->q_id); + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); + ASSERT(fa == NULL); + } +} +#else +# define xfs_qm_dquot_logitem_precommit_check(...) ((void)0) +#endif + +static int +xfs_qm_dquot_logitem_precommit( + struct xfs_trans *tp, + struct xfs_log_item *lip) +{ + struct xfs_dq_logitem *qlip = DQUOT_ITEM(lip); + struct xfs_dquot *dqp = qlip->qli_dquot; + + xfs_qm_dquot_logitem_precommit_check(dqp); + + return xfs_dquot_attach_buf(tp, dqp); +} + +static const struct xfs_item_ops xfs_dquot_item_ops = { + .iop_size = xfs_qm_dquot_logitem_size, + .iop_precommit = xfs_qm_dquot_logitem_precommit, + .iop_format = xfs_qm_dquot_logitem_format, + .iop_pin = xfs_qm_dquot_logitem_pin, + .iop_unpin = xfs_qm_dquot_logitem_unpin, + .iop_release = xfs_qm_dquot_logitem_release, + .iop_committing = xfs_qm_dquot_logitem_committing, + .iop_push = xfs_qm_dquot_logitem_push, +}; + +/* + * Initialize the dquot log item for a newly allocated dquot. + * The dquot isn't locked at this point, but it isn't on any of the lists + * either, so we don't care. + */ +void +xfs_qm_dquot_logitem_init( + struct xfs_dquot *dqp) +{ + struct xfs_dq_logitem *lp = &dqp->q_logitem; + + xfs_log_item_init(dqp->q_mount, &lp->qli_item, XFS_LI_DQUOT, + &xfs_dquot_item_ops); + spin_lock_init(&lp->qli_lock); + lp->qli_dquot = dqp; + lp->qli_dirty = false; +} diff --git a/fs/xfs/xfs_dquot_item.h b/fs/xfs/xfs_dquot_item.h new file mode 100644 index 000000000..d66e52807 --- /dev/null +++ b/fs/xfs/xfs_dquot_item.h @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2003 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_DQUOT_ITEM_H__ +#define __XFS_DQUOT_ITEM_H__ + +struct xfs_dquot; +struct xfs_trans; +struct xfs_mount; + +struct xfs_dq_logitem { + struct xfs_log_item qli_item; /* common portion */ + struct xfs_dquot *qli_dquot; /* dquot ptr */ + xfs_lsn_t qli_flush_lsn; /* lsn at last flush */ + + /* + * We use this spinlock to coordinate access to the li_buf pointer in + * the log item and the qli_dirty flag. + */ + spinlock_t qli_lock; + bool qli_dirty; /* dirtied since last flush? */ +}; + +void xfs_qm_dquot_logitem_init(struct xfs_dquot *dqp); + +#endif /* __XFS_DQUOT_ITEM_H__ */ diff --git a/fs/xfs/xfs_dquot_item_recover.c b/fs/xfs/xfs_dquot_item_recover.c new file mode 100644 index 000000000..b8854316a --- /dev/null +++ b/fs/xfs/xfs_dquot_item_recover.c @@ -0,0 +1,215 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2006 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_quota.h" +#include "xfs_trans.h" +#include "xfs_buf_item.h" +#include "xfs_trans_priv.h" +#include "xfs_qm.h" +#include "xfs_log.h" +#include "xfs_log_priv.h" +#include "xfs_log_recover.h" +#include "xfs_error.h" + +STATIC void +xlog_recover_dquot_ra_pass2( + struct xlog *log, + struct xlog_recover_item *item) +{ + struct xfs_mount *mp = log->l_mp; + struct xfs_disk_dquot *recddq; + struct xfs_dq_logformat *dq_f; + uint type; + + if (mp->m_qflags == 0) + return; + + recddq = item->ri_buf[1].iov_base; + if (recddq == NULL) + return; + if (item->ri_buf[1].iov_len < sizeof(struct xfs_disk_dquot)) + return; + + type = recddq->d_type & XFS_DQTYPE_REC_MASK; + ASSERT(type); + if (log->l_quotaoffs_flag & type) + return; + + dq_f = item->ri_buf[0].iov_base; + ASSERT(dq_f); + ASSERT(dq_f->qlf_len == 1); + + xlog_buf_readahead(log, dq_f->qlf_blkno, + XFS_FSB_TO_BB(mp, dq_f->qlf_len), + &xfs_dquot_buf_ra_ops); +} + +/* + * Recover a dquot record + */ +STATIC int +xlog_recover_dquot_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t current_lsn) +{ + struct xfs_mount *mp = log->l_mp; + struct xfs_buf *bp; + struct xfs_dqblk *dqb; + struct xfs_disk_dquot *ddq, *recddq; + struct xfs_dq_logformat *dq_f; + xfs_failaddr_t fa; + int error; + uint type; + + /* + * Filesystems are required to send in quota flags at mount time. + */ + if (mp->m_qflags == 0) + return 0; + + recddq = item->ri_buf[1].iov_base; + if (recddq == NULL) { + xfs_alert(log->l_mp, "NULL dquot in %s.", __func__); + return -EFSCORRUPTED; + } + if (item->ri_buf[1].iov_len < sizeof(struct xfs_disk_dquot)) { + xfs_alert(log->l_mp, "dquot too small (%zd) in %s.", + item->ri_buf[1].iov_len, __func__); + return -EFSCORRUPTED; + } + + /* + * This type of quotas was turned off, so ignore this record. + */ + type = recddq->d_type & XFS_DQTYPE_REC_MASK; + ASSERT(type); + if (log->l_quotaoffs_flag & type) + return 0; + + /* + * At this point we know that quota was _not_ turned off. + * Since the mount flags are not indicating to us otherwise, this + * must mean that quota is on, and the dquot needs to be replayed. + * Remember that we may not have fully recovered the superblock yet, + * so we can't do the usual trick of looking at the SB quota bits. + * + * The other possibility, of course, is that the quota subsystem was + * removed since the last mount - ENOSYS. + */ + dq_f = item->ri_buf[0].iov_base; + ASSERT(dq_f); + fa = xfs_dquot_verify(mp, recddq, dq_f->qlf_id); + if (fa) { + xfs_alert(mp, "corrupt dquot ID 0x%x in log at %pS", + dq_f->qlf_id, fa); + return -EFSCORRUPTED; + } + ASSERT(dq_f->qlf_len == 1); + + /* + * At this point we are assuming that the dquots have been allocated + * and hence the buffer has valid dquots stamped in it. It should, + * therefore, pass verifier validation. If the dquot is bad, then the + * we'll return an error here, so we don't need to specifically check + * the dquot in the buffer after the verifier has run. + */ + error = xfs_trans_read_buf(mp, NULL, mp->m_ddev_targp, dq_f->qlf_blkno, + XFS_FSB_TO_BB(mp, dq_f->qlf_len), 0, &bp, + &xfs_dquot_buf_ops); + if (error) + return error; + + ASSERT(bp); + dqb = xfs_buf_offset(bp, dq_f->qlf_boffset); + ddq = &dqb->dd_diskdq; + + /* + * If the dquot has an LSN in it, recover the dquot only if it's less + * than the lsn of the transaction we are replaying. + */ + if (xfs_has_crc(mp)) { + xfs_lsn_t lsn = be64_to_cpu(dqb->dd_lsn); + + if (lsn && lsn != -1 && XFS_LSN_CMP(lsn, current_lsn) >= 0) { + goto out_release; + } + } + + memcpy(ddq, recddq, item->ri_buf[1].iov_len); + if (xfs_has_crc(mp)) { + xfs_update_cksum((char *)dqb, sizeof(struct xfs_dqblk), + XFS_DQUOT_CRC_OFF); + } + + /* Validate the recovered dquot. */ + fa = xfs_dqblk_verify(log->l_mp, dqb, dq_f->qlf_id); + if (fa) { + XFS_CORRUPTION_ERROR("Bad dquot after recovery", + XFS_ERRLEVEL_LOW, mp, dqb, + sizeof(struct xfs_dqblk)); + xfs_alert(mp, + "Metadata corruption detected at %pS, dquot 0x%x", + fa, dq_f->qlf_id); + error = -EFSCORRUPTED; + goto out_release; + } + + ASSERT(dq_f->qlf_size == 2); + ASSERT(bp->b_mount == mp); + xfs_buf_delwri_queue(bp, buffer_list); + +out_release: + xfs_buf_relse(bp); + return error; +} + +const struct xlog_recover_item_ops xlog_dquot_item_ops = { + .item_type = XFS_LI_DQUOT, + .ra_pass2 = xlog_recover_dquot_ra_pass2, + .commit_pass2 = xlog_recover_dquot_commit_pass2, +}; + +/* + * Recover QUOTAOFF records. We simply make a note of it in the xlog + * structure, so that we know not to do any dquot item or dquot buffer recovery, + * of that type. + */ +STATIC int +xlog_recover_quotaoff_commit_pass1( + struct xlog *log, + struct xlog_recover_item *item) +{ + struct xfs_qoff_logformat *qoff_f = item->ri_buf[0].iov_base; + ASSERT(qoff_f); + + /* + * The logitem format's flag tells us if this was user quotaoff, + * group/project quotaoff or both. + */ + if (qoff_f->qf_flags & XFS_UQUOTA_ACCT) + log->l_quotaoffs_flag |= XFS_DQTYPE_USER; + if (qoff_f->qf_flags & XFS_PQUOTA_ACCT) + log->l_quotaoffs_flag |= XFS_DQTYPE_PROJ; + if (qoff_f->qf_flags & XFS_GQUOTA_ACCT) + log->l_quotaoffs_flag |= XFS_DQTYPE_GROUP; + + return 0; +} + +const struct xlog_recover_item_ops xlog_quotaoff_item_ops = { + .item_type = XFS_LI_QUOTAOFF, + .commit_pass1 = xlog_recover_quotaoff_commit_pass1, + /* nothing to commit in pass2 */ +}; diff --git a/fs/xfs/xfs_drain.c b/fs/xfs/xfs_drain.c new file mode 100644 index 000000000..1ad67f6c1 --- /dev/null +++ b/fs/xfs/xfs_drain.c @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2022-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_ag.h" +#include "xfs_trace.h" + +/* + * Use a static key here to reduce the overhead of xfs_defer_drain_rele. If + * the compiler supports jump labels, the static branch will be replaced by a + * nop sled when there are no xfs_defer_drain_wait callers. Online fsck is + * currently the only caller, so this is a reasonable tradeoff. + * + * Note: Patching the kernel code requires taking the cpu hotplug lock. Other + * parts of the kernel allocate memory with that lock held, which means that + * XFS callers cannot hold any locks that might be used by memory reclaim or + * writeback when calling the static_branch_{inc,dec} functions. + */ +static DEFINE_STATIC_KEY_FALSE(xfs_defer_drain_waiter_gate); + +void +xfs_defer_drain_wait_disable(void) +{ + static_branch_dec(&xfs_defer_drain_waiter_gate); +} + +void +xfs_defer_drain_wait_enable(void) +{ + static_branch_inc(&xfs_defer_drain_waiter_gate); +} + +void +xfs_defer_drain_init( + struct xfs_defer_drain *dr) +{ + atomic_set(&dr->dr_count, 0); + init_waitqueue_head(&dr->dr_waiters); +} + +void +xfs_defer_drain_free(struct xfs_defer_drain *dr) +{ + ASSERT(atomic_read(&dr->dr_count) == 0); +} + +/* Increase the pending intent count. */ +static inline void xfs_defer_drain_grab(struct xfs_defer_drain *dr) +{ + atomic_inc(&dr->dr_count); +} + +static inline bool has_waiters(struct wait_queue_head *wq_head) +{ + /* + * This memory barrier is paired with the one in set_current_state on + * the waiting side. + */ + smp_mb__after_atomic(); + return waitqueue_active(wq_head); +} + +/* Decrease the pending intent count, and wake any waiters, if appropriate. */ +static inline void xfs_defer_drain_rele(struct xfs_defer_drain *dr) +{ + if (atomic_dec_and_test(&dr->dr_count) && + static_branch_unlikely(&xfs_defer_drain_waiter_gate) && + has_waiters(&dr->dr_waiters)) + wake_up(&dr->dr_waiters); +} + +/* Are there intents pending? */ +static inline bool xfs_defer_drain_busy(struct xfs_defer_drain *dr) +{ + return atomic_read(&dr->dr_count) > 0; +} + +/* + * Wait for the pending intent count for a drain to hit zero. + * + * Callers must not hold any locks that would prevent intents from being + * finished. + */ +static inline int xfs_defer_drain_wait(struct xfs_defer_drain *dr) +{ + return wait_event_killable(dr->dr_waiters, !xfs_defer_drain_busy(dr)); +} + +/* + * Get a passive reference to the group that contains a fsbno and declare an + * intent to update its metadata. + * + * Other threads that need exclusive access can decide to back off if they see + * declared intentions. + */ +struct xfs_group * +xfs_group_intent_get( + struct xfs_mount *mp, + xfs_fsblock_t fsbno, + enum xfs_group_type type) +{ + struct xfs_group *xg; + + xg = xfs_group_get_by_fsb(mp, fsbno, type); + if (!xg) + return NULL; + trace_xfs_group_intent_hold(xg, __return_address); + xfs_defer_drain_grab(&xg->xg_intents_drain); + return xg; +} + +/* + * Release our intent to update this groups metadata, and then release our + * passive ref to it. + */ +void +xfs_group_intent_put( + struct xfs_group *xg) +{ + trace_xfs_group_intent_rele(xg, __return_address); + xfs_defer_drain_rele(&xg->xg_intents_drain); + xfs_group_put(xg); +} + +/* + * Wait for the intent update count for this AG to hit zero. + * Callers must not hold any AG header buffers. + */ +int +xfs_group_intent_drain( + struct xfs_group *xg) +{ + trace_xfs_group_wait_intents(xg, __return_address); + return xfs_defer_drain_wait(&xg->xg_intents_drain); +} + +/* + * Has anyone declared an intent to update this group? + */ +bool +xfs_group_intent_busy( + struct xfs_group *xg) +{ + return xfs_defer_drain_busy(&xg->xg_intents_drain); +} diff --git a/fs/xfs/xfs_drain.h b/fs/xfs/xfs_drain.h new file mode 100644 index 000000000..4d446dbf6 --- /dev/null +++ b/fs/xfs/xfs_drain.h @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2022-2023 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef XFS_DRAIN_H_ +#define XFS_DRAIN_H_ + +struct xfs_group; +struct xfs_perag; + +#ifdef CONFIG_XFS_DRAIN_INTENTS +/* + * Passive drain mechanism. This data structure tracks a count of some items + * and contains a waitqueue for callers who would like to wake up when the + * count hits zero. + */ +struct xfs_defer_drain { + /* Number of items pending in some part of the filesystem. */ + atomic_t dr_count; + + /* Queue to wait for dri_count to go to zero */ + struct wait_queue_head dr_waiters; +}; + +void xfs_defer_drain_init(struct xfs_defer_drain *dr); +void xfs_defer_drain_free(struct xfs_defer_drain *dr); + +void xfs_defer_drain_wait_disable(void); +void xfs_defer_drain_wait_enable(void); + +/* + * Deferred Work Intent Drains + * =========================== + * + * When a writer thread executes a chain of log intent items, the AG header + * buffer locks will cycle during a transaction roll to get from one intent + * item to the next in a chain. Although scrub takes all AG header buffer + * locks, this isn't sufficient to guard against scrub checking an AG while + * that writer thread is in the middle of finishing a chain because there's no + * higher level locking primitive guarding allocation groups. + * + * When there's a collision, cross-referencing between data structures (e.g. + * rmapbt and refcountbt) yields false corruption events; if repair is running, + * this results in incorrect repairs, which is catastrophic. + * + * The solution is to the perag structure the count of active intents and make + * scrub wait until it has both AG header buffer locks and the intent counter + * reaches zero. It is therefore critical that deferred work threads hold the + * AGI or AGF buffers when decrementing the intent counter. + * + * Given a list of deferred work items, the deferred work manager will complete + * a work item and all the sub-items that the parent item creates before moving + * on to the next work item in the list. This is also true for all levels of + * sub-items. Writer threads are permitted to queue multiple work items + * targetting the same AG, so a deferred work item (such as a BUI) that creates + * sub-items (such as RUIs) must bump the intent counter and maintain it until + * the sub-items can themselves bump the intent counter. + * + * Therefore, the intent count tracks entire lifetimes of deferred work items. + * All functions that create work items must increment the intent counter as + * soon as the item is added to the transaction and cannot drop the counter + * until the item is finished or cancelled. + * + * The same principles apply to realtime groups because the rt metadata inode + * ILOCKs are not held across transaction rolls. + */ +struct xfs_group *xfs_group_intent_get(struct xfs_mount *mp, + xfs_fsblock_t fsbno, enum xfs_group_type type); +void xfs_group_intent_put(struct xfs_group *rtg); + +int xfs_group_intent_drain(struct xfs_group *xg); +bool xfs_group_intent_busy(struct xfs_group *xg); + +#else +struct xfs_defer_drain { /* empty */ }; + +#define xfs_defer_drain_free(dr) ((void)0) +#define xfs_defer_drain_init(dr) ((void)0) + +#define xfs_group_intent_get(_mp, _fsbno, _type) \ + xfs_group_get_by_fsb((_mp), (_fsbno), (_type)) +#define xfs_group_intent_put(xg) xfs_group_put(xg) + +#endif /* CONFIG_XFS_DRAIN_INTENTS */ + +#endif /* XFS_DRAIN_H_ */ diff --git a/fs/xfs/xfs_error.c b/fs/xfs/xfs_error.c new file mode 100644 index 000000000..9ece70fd9 --- /dev/null +++ b/fs/xfs/xfs_error.c @@ -0,0 +1,385 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2001,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_fs.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_error.h" +#include "xfs_sysfs.h" +#include "xfs_inode.h" + +#ifdef DEBUG + +#define XFS_ERRTAG(_tag, _name, _default) \ + [XFS_ERRTAG_##_tag] = (_default), +#include "xfs_errortag.h" +static const unsigned int xfs_errortag_random_default[] = { XFS_ERRTAGS }; +#undef XFS_ERRTAG + +#define XFS_ERRTAG(_tag, _name, _default) \ + [XFS_ERRTAG_##_tag] = __stringify(_name), +#include "xfs_errortag.h" +static const char *xfs_errortag_names[] = { XFS_ERRTAGS }; +#undef XFS_ERRTAG + +struct xfs_errortag_attr { + struct attribute attr; + unsigned int tag; +}; + +static inline struct xfs_errortag_attr * +to_attr(struct attribute *attr) +{ + return container_of(attr, struct xfs_errortag_attr, attr); +} + +static inline struct xfs_mount * +to_mp(struct kobject *kobject) +{ + struct xfs_kobj *kobj = to_kobj(kobject); + + return container_of(kobj, struct xfs_mount, m_errortag_kobj); +} + +STATIC ssize_t +xfs_errortag_attr_store( + struct kobject *kobject, + struct attribute *attr, + const char *buf, + size_t count) +{ + struct xfs_mount *mp = to_mp(kobject); + unsigned int error_tag = to_attr(attr)->tag; + unsigned int val; + int ret; + + if (strcmp(buf, "default") == 0) { + val = xfs_errortag_random_default[error_tag]; + } else { + ret = kstrtouint(buf, 0, &val); + if (ret) + return ret; + } + + WRITE_ONCE(mp->m_errortag[error_tag], val); + return count; +} + +STATIC ssize_t +xfs_errortag_attr_show( + struct kobject *kobject, + struct attribute *attr, + char *buf) +{ + struct xfs_mount *mp = to_mp(kobject); + + return snprintf(buf, PAGE_SIZE, "%u\n", + READ_ONCE(mp->m_errortag[to_attr(attr)->tag])); +} + +static const struct sysfs_ops xfs_errortag_sysfs_ops = { + .show = xfs_errortag_attr_show, + .store = xfs_errortag_attr_store, +}; + +#define XFS_ERRTAG(_tag, _name, _default) \ +static struct xfs_errortag_attr xfs_errortag_attr_##_name = { \ + .attr = {.name = __stringify(_name), \ + .mode = VERIFY_OCTAL_PERMISSIONS(S_IWUSR | S_IRUGO) }, \ + .tag = XFS_ERRTAG_##_tag, \ +}; +#include "xfs_errortag.h" +XFS_ERRTAGS +#undef XFS_ERRTAG + +#define XFS_ERRTAG(_tag, _name, _default) \ + &xfs_errortag_attr_##_name.attr, +#include "xfs_errortag.h" +static struct attribute *xfs_errortag_attrs[] = { + XFS_ERRTAGS + NULL +}; +ATTRIBUTE_GROUPS(xfs_errortag); +#undef XFS_ERRTAG + +/* -1 because XFS_ERRTAG_DROP_WRITES got removed, + 1 for NULL termination */ +static_assert(ARRAY_SIZE(xfs_errortag_attrs) == XFS_ERRTAG_MAX); + +static const struct kobj_type xfs_errortag_ktype = { + .release = xfs_sysfs_release, + .sysfs_ops = &xfs_errortag_sysfs_ops, + .default_groups = xfs_errortag_groups, +}; + +int +xfs_errortag_init( + struct xfs_mount *mp) +{ + return xfs_sysfs_init(&mp->m_errortag_kobj, &xfs_errortag_ktype, + &mp->m_kobj, "errortag"); +} + +void +xfs_errortag_del( + struct xfs_mount *mp) +{ + xfs_sysfs_del(&mp->m_errortag_kobj); +} + +bool +xfs_errortag_test( + struct xfs_mount *mp, + const char *file, + int line, + unsigned int error_tag) +{ + unsigned int randfactor; + + randfactor = READ_ONCE(mp->m_errortag[error_tag]); + if (!randfactor || get_random_u32_below(randfactor)) + return false; + + xfs_warn_ratelimited(mp, +"Injecting error at file %s, line %d, on filesystem \"%s\"", + file, line, mp->m_super->s_id); + return true; +} + +void +xfs_errortag_delay( + struct xfs_mount *mp, + const char *file, + int line, + unsigned int error_tag) +{ + unsigned int delay = READ_ONCE(mp->m_errortag[error_tag]); + + might_sleep(); + + if (!delay) + return; + + xfs_warn_ratelimited(mp, +"Injecting %ums delay at file %s, line %d, on filesystem \"%s\"", + delay, file, line, + mp->m_super->s_id); + mdelay(delay); +} + +int +xfs_errortag_add( + struct xfs_mount *mp, + unsigned int error_tag) +{ + BUILD_BUG_ON(ARRAY_SIZE(xfs_errortag_random_default) != XFS_ERRTAG_MAX); + + if (error_tag >= XFS_ERRTAG_MAX) + return -EINVAL; + + /* Error out removed injection types */ + switch (error_tag) { + case XFS_ERRTAG_DROP_WRITES: + return -EINVAL; + default: + break; + } + + WRITE_ONCE(mp->m_errortag[error_tag], + xfs_errortag_random_default[error_tag]); + return 0; +} + +int +xfs_errortag_add_name( + struct xfs_mount *mp, + const char *tag_name) +{ + unsigned int i; + + for (i = 0; i < XFS_ERRTAG_MAX; i++) { + if (xfs_errortag_names[i] && + !strcmp(xfs_errortag_names[i], tag_name)) + return xfs_errortag_add(mp, i); + } + + return -EINVAL; +} + +void +xfs_errortag_copy( + struct xfs_mount *dst_mp, + struct xfs_mount *src_mp) +{ + unsigned int val, i; + + for (i = 0; i < XFS_ERRTAG_MAX; i++) { + val = READ_ONCE(src_mp->m_errortag[i]); + if (val) + WRITE_ONCE(dst_mp->m_errortag[i], val); + } +} + +int +xfs_errortag_clearall( + struct xfs_mount *mp) +{ + unsigned int i; + + for (i = 0; i < XFS_ERRTAG_MAX; i++) + WRITE_ONCE(mp->m_errortag[i], 0); + return 0; +} +#endif /* DEBUG */ + +void +xfs_error_report( + const char *tag, + int level, + struct xfs_mount *mp, + const char *filename, + int linenum, + xfs_failaddr_t failaddr) +{ + if (level <= xfs_error_level) { + xfs_alert_tag(mp, XFS_PTAG_ERROR_REPORT, + "Internal error %s at line %d of file %s. Caller %pS", + tag, linenum, filename, failaddr); + + xfs_stack_trace(); + } +} + +void +xfs_corruption_error( + const char *tag, + int level, + struct xfs_mount *mp, + const void *buf, + size_t bufsize, + const char *filename, + int linenum, + xfs_failaddr_t failaddr) +{ + if (buf && level <= xfs_error_level) + xfs_hex_dump(buf, bufsize); + xfs_error_report(tag, level, mp, filename, linenum, failaddr); + xfs_alert(mp, "Corruption detected. Unmount and run xfs_repair"); +} + +/* + * Complain about the kinds of metadata corruption that we can't detect from a + * verifier, such as incorrect inter-block relationship data. Does not set + * bp->b_error. + * + * Call xfs_buf_mark_corrupt, not this function. + */ +void +xfs_buf_corruption_error( + struct xfs_buf *bp, + xfs_failaddr_t fa) +{ + struct xfs_mount *mp = bp->b_mount; + + xfs_alert_tag(mp, XFS_PTAG_VERIFIER_ERROR, + "Metadata corruption detected at %pS, %s block 0x%llx", + fa, bp->b_ops->name, xfs_buf_daddr(bp)); + + xfs_alert(mp, "Unmount and run xfs_repair"); + + if (xfs_error_level >= XFS_ERRLEVEL_HIGH) + xfs_stack_trace(); +} + +/* + * Warnings specifically for verifier errors. Differentiate CRC vs. invalid + * values, and omit the stack trace unless the error level is tuned high. + */ +void +xfs_buf_verifier_error( + struct xfs_buf *bp, + int error, + const char *name, + const void *buf, + size_t bufsz, + xfs_failaddr_t failaddr) +{ + struct xfs_mount *mp = bp->b_mount; + xfs_failaddr_t fa; + int sz; + + fa = failaddr ? failaddr : __return_address; + __xfs_buf_ioerror(bp, error, fa); + + xfs_alert_tag(mp, XFS_PTAG_VERIFIER_ERROR, + "Metadata %s detected at %pS, %s block 0x%llx %s", + bp->b_error == -EFSBADCRC ? "CRC error" : "corruption", + fa, bp->b_ops->name, xfs_buf_daddr(bp), name); + + xfs_alert(mp, "Unmount and run xfs_repair"); + + if (xfs_error_level >= XFS_ERRLEVEL_LOW) { + sz = min_t(size_t, XFS_CORRUPTION_DUMP_LEN, bufsz); + xfs_alert(mp, "First %d bytes of corrupted metadata buffer:", + sz); + xfs_hex_dump(buf, sz); + } + + if (xfs_error_level >= XFS_ERRLEVEL_HIGH) + xfs_stack_trace(); +} + +/* + * Warnings specifically for verifier errors. Differentiate CRC vs. invalid + * values, and omit the stack trace unless the error level is tuned high. + */ +void +xfs_verifier_error( + struct xfs_buf *bp, + int error, + xfs_failaddr_t failaddr) +{ + return xfs_buf_verifier_error(bp, error, "", xfs_buf_offset(bp, 0), + XFS_CORRUPTION_DUMP_LEN, failaddr); +} + +/* + * Warnings for inode corruption problems. Don't bother with the stack + * trace unless the error level is turned up high. + */ +void +xfs_inode_verifier_error( + struct xfs_inode *ip, + int error, + const char *name, + const void *buf, + size_t bufsz, + xfs_failaddr_t failaddr) +{ + struct xfs_mount *mp = ip->i_mount; + xfs_failaddr_t fa; + int sz; + + fa = failaddr ? failaddr : __return_address; + + xfs_alert(mp, "Metadata %s detected at %pS, inode 0x%llx %s", + error == -EFSBADCRC ? "CRC error" : "corruption", + fa, I_INO(ip), name); + + xfs_alert(mp, "Unmount and run xfs_repair"); + + if (buf && xfs_error_level >= XFS_ERRLEVEL_LOW) { + sz = min_t(size_t, XFS_CORRUPTION_DUMP_LEN, bufsz); + xfs_alert(mp, "First %d bytes of corrupted metadata buffer:", + sz); + xfs_hex_dump(buf, sz); + } + + if (xfs_error_level >= XFS_ERRLEVEL_HIGH) + xfs_stack_trace(); +} diff --git a/fs/xfs/xfs_error.h b/fs/xfs/xfs_error.h new file mode 100644 index 000000000..05fc1d1cf --- /dev/null +++ b/fs/xfs/xfs_error.h @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_ERROR_H__ +#define __XFS_ERROR_H__ + +struct xfs_mount; + +void xfs_error_report(const char *tag, int level, struct xfs_mount *mp, + const char *filename, int linenum, xfs_failaddr_t failaddr); +void xfs_corruption_error(const char *tag, int level, struct xfs_mount *mp, + const void *buf, size_t bufsize, const char *filename, + int linenum, xfs_failaddr_t failaddr); +void xfs_buf_corruption_error(struct xfs_buf *bp, xfs_failaddr_t fa); +void xfs_buf_verifier_error(struct xfs_buf *bp, int error, const char *name, + const void *buf, size_t bufsz, xfs_failaddr_t failaddr); +void xfs_verifier_error(struct xfs_buf *bp, int error, xfs_failaddr_t failaddr); +void xfs_inode_verifier_error(struct xfs_inode *ip, int error, const char *name, + const void *buf, size_t bufsz, xfs_failaddr_t failaddr); + +#define XFS_ERROR_REPORT(e, lvl, mp) \ + xfs_error_report(e, lvl, mp, __FILE__, __LINE__, __return_address) +#define XFS_CORRUPTION_ERROR(e, lvl, mp, buf, bufsize) \ + xfs_corruption_error(e, lvl, mp, buf, bufsize, \ + __FILE__, __LINE__, __return_address) + +#define XFS_ERRLEVEL_OFF 0 +#define XFS_ERRLEVEL_LOW 1 +#define XFS_ERRLEVEL_HIGH 5 + +/* Dump 128 bytes of any corrupt buffer */ +#define XFS_CORRUPTION_DUMP_LEN (128) + +#ifdef DEBUG +int xfs_errortag_init(struct xfs_mount *mp); +void xfs_errortag_del(struct xfs_mount *mp); +bool xfs_errortag_test(struct xfs_mount *mp, const char *file, int line, + unsigned int error_tag); +#define XFS_TEST_ERROR(mp, tag) \ + xfs_errortag_test((mp), __FILE__, __LINE__, (tag)) +void xfs_errortag_delay(struct xfs_mount *mp, const char *file, int line, + unsigned int error_tag); +#define XFS_ERRORTAG_DELAY(mp, tag) \ + xfs_errortag_delay((mp), __FILE__, __LINE__, (tag)) +int xfs_errortag_add(struct xfs_mount *mp, unsigned int error_tag); +int xfs_errortag_add_name(struct xfs_mount *mp, const char *tag_name); +void xfs_errortag_copy(struct xfs_mount *dst_mp, struct xfs_mount *src_mp); +int xfs_errortag_clearall(struct xfs_mount *mp); +#else +#define xfs_errortag_init(mp) (0) +#define xfs_errortag_del(mp) +#define XFS_TEST_ERROR(mp, tag) (false) +#define XFS_ERRORTAG_DELAY(mp, tag) ((void)0) +#define xfs_errortag_add(mp, tag) (-ENOSYS) +#define xfs_errortag_copy(dst_mp, src_mp) ((void)0) +#define xfs_errortag_add_name(mp, tag_name) (-ENOSYS) +#define xfs_errortag_clearall(mp) (-ENOSYS) +#endif /* DEBUG */ + +/* + * XFS panic tags -- allow a call to xfs_alert_tag() be turned into + * a panic by setting fs.xfs.panic_mask in a sysctl. + */ +#define XFS_NO_PTAG 0u +#define XFS_PTAG_IFLUSH (1u << 0) +#define XFS_PTAG_LOGRES (1u << 1) +#define XFS_PTAG_AILDELETE (1u << 2) +#define XFS_PTAG_ERROR_REPORT (1u << 3) +#define XFS_PTAG_SHUTDOWN_CORRUPT (1u << 4) +#define XFS_PTAG_SHUTDOWN_IOERROR (1u << 5) +#define XFS_PTAG_SHUTDOWN_LOGERROR (1u << 6) +#define XFS_PTAG_FSBLOCK_ZERO (1u << 7) +#define XFS_PTAG_VERIFIER_ERROR (1u << 8) + +#define XFS_PTAG_MASK (XFS_PTAG_IFLUSH | \ + XFS_PTAG_LOGRES | \ + XFS_PTAG_AILDELETE | \ + XFS_PTAG_ERROR_REPORT | \ + XFS_PTAG_SHUTDOWN_CORRUPT | \ + XFS_PTAG_SHUTDOWN_IOERROR | \ + XFS_PTAG_SHUTDOWN_LOGERROR | \ + XFS_PTAG_FSBLOCK_ZERO | \ + XFS_PTAG_VERIFIER_ERROR) + +#define XFS_PTAG_STRINGS \ + { XFS_NO_PTAG, "none" }, \ + { XFS_PTAG_IFLUSH, "iflush" }, \ + { XFS_PTAG_LOGRES, "logres" }, \ + { XFS_PTAG_AILDELETE, "aildelete" }, \ + { XFS_PTAG_ERROR_REPORT , "error_report" }, \ + { XFS_PTAG_SHUTDOWN_CORRUPT, "corrupt" }, \ + { XFS_PTAG_SHUTDOWN_IOERROR, "ioerror" }, \ + { XFS_PTAG_SHUTDOWN_LOGERROR, "logerror" }, \ + { XFS_PTAG_FSBLOCK_ZERO, "fsb_zero" }, \ + { XFS_PTAG_VERIFIER_ERROR, "verifier" } + +#endif /* __XFS_ERROR_H__ */ diff --git a/fs/xfs/xfs_exchmaps_item.c b/fs/xfs/xfs_exchmaps_item.c new file mode 100644 index 000000000..c3745d33e --- /dev/null +++ b/fs/xfs/xfs_exchmaps_item.c @@ -0,0 +1,611 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_shared.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_trans_priv.h" +#include "xfs_exchmaps_item.h" +#include "xfs_exchmaps.h" +#include "xfs_log.h" +#include "xfs_bmap.h" +#include "xfs_icache.h" +#include "xfs_bmap_btree.h" +#include "xfs_trans_space.h" +#include "xfs_error.h" +#include "xfs_log_priv.h" +#include "xfs_log_recover.h" +#include "xfs_exchrange.h" +#include "xfs_trace.h" + +struct kmem_cache *xfs_xmi_cache; +struct kmem_cache *xfs_xmd_cache; + +static const struct xfs_item_ops xfs_xmi_item_ops; + +static inline struct xfs_xmi_log_item *XMI_ITEM(struct xfs_log_item *lip) +{ + return container_of(lip, struct xfs_xmi_log_item, xmi_item); +} + +STATIC void +xfs_xmi_item_free( + struct xfs_xmi_log_item *xmi_lip) +{ + kvfree(xmi_lip->xmi_item.li_lv_shadow); + kmem_cache_free(xfs_xmi_cache, xmi_lip); +} + +/* + * Freeing the XMI requires that we remove it from the AIL if it has already + * been placed there. However, the XMI may not yet have been placed in the AIL + * when called by xfs_xmi_release() from XMD processing due to the ordering of + * committed vs unpin operations in bulk insert operations. Hence the reference + * count to ensure only the last caller frees the XMI. + */ +STATIC void +xfs_xmi_release( + struct xfs_xmi_log_item *xmi_lip) +{ + ASSERT(atomic_read(&xmi_lip->xmi_refcount) > 0); + if (atomic_dec_and_test(&xmi_lip->xmi_refcount)) { + xfs_trans_ail_delete(&xmi_lip->xmi_item, 0); + xfs_xmi_item_free(xmi_lip); + } +} + + +STATIC void +xfs_xmi_item_size( + struct xfs_log_item *lip, + int *nvecs, + int *nbytes) +{ + *nvecs += 1; + *nbytes += sizeof(struct xfs_xmi_log_format); +} + +/* + * This is called to fill in the vector of log iovecs for the given xmi log + * item. We use only 1 iovec, and we point that at the xmi_log_format structure + * embedded in the xmi item. + */ +STATIC void +xfs_xmi_item_format( + struct xfs_log_item *lip, + struct xlog_format_buf *lfb) +{ + struct xfs_xmi_log_item *xmi_lip = XMI_ITEM(lip); + + xmi_lip->xmi_format.xmi_type = XFS_LI_XMI; + xmi_lip->xmi_format.xmi_size = 1; + + xlog_format_copy(lfb, XLOG_REG_TYPE_XMI_FORMAT, &xmi_lip->xmi_format, + sizeof(struct xfs_xmi_log_format)); +} + +/* + * The unpin operation is the last place an XMI is manipulated in the log. It + * is either inserted in the AIL or aborted in the event of a log I/O error. In + * either case, the XMI transaction has been successfully committed to make it + * this far. Therefore, we expect whoever committed the XMI to either construct + * and commit the XMD or drop the XMD's reference in the event of error. Simply + * drop the log's XMI reference now that the log is done with it. + */ +STATIC void +xfs_xmi_item_unpin( + struct xfs_log_item *lip, + int remove) +{ + struct xfs_xmi_log_item *xmi_lip = XMI_ITEM(lip); + + xfs_xmi_release(xmi_lip); +} + +/* + * The XMI has been either committed or aborted if the transaction has been + * cancelled. If the transaction was cancelled, an XMD isn't going to be + * constructed and thus we free the XMI here directly. + */ +STATIC void +xfs_xmi_item_release( + struct xfs_log_item *lip) +{ + xfs_xmi_release(XMI_ITEM(lip)); +} + +/* Allocate and initialize an xmi item. */ +STATIC struct xfs_xmi_log_item * +xfs_xmi_init( + struct xfs_mount *mp) + +{ + struct xfs_xmi_log_item *xmi_lip; + + xmi_lip = kmem_cache_zalloc(xfs_xmi_cache, GFP_KERNEL | __GFP_NOFAIL); + + xfs_log_item_init(mp, &xmi_lip->xmi_item, XFS_LI_XMI, &xfs_xmi_item_ops); + xmi_lip->xmi_format.xmi_id = (uintptr_t)(void *)xmi_lip; + atomic_set(&xmi_lip->xmi_refcount, 2); + + return xmi_lip; +} + +static inline struct xfs_xmd_log_item *XMD_ITEM(struct xfs_log_item *lip) +{ + return container_of(lip, struct xfs_xmd_log_item, xmd_item); +} + +STATIC void +xfs_xmd_item_size( + struct xfs_log_item *lip, + int *nvecs, + int *nbytes) +{ + *nvecs += 1; + *nbytes += sizeof(struct xfs_xmd_log_format); +} + +/* + * This is called to fill in the vector of log iovecs for the given xmd log + * item. We use only 1 iovec, and we point that at the xmd_log_format structure + * embedded in the xmd item. + */ +STATIC void +xfs_xmd_item_format( + struct xfs_log_item *lip, + struct xlog_format_buf *lfb) +{ + struct xfs_xmd_log_item *xmd_lip = XMD_ITEM(lip); + + xmd_lip->xmd_format.xmd_type = XFS_LI_XMD; + xmd_lip->xmd_format.xmd_size = 1; + + xlog_format_copy(lfb, XLOG_REG_TYPE_XMD_FORMAT, &xmd_lip->xmd_format, + sizeof(struct xfs_xmd_log_format)); +} + +/* + * The XMD is either committed or aborted if the transaction is cancelled. If + * the transaction is cancelled, drop our reference to the XMI and free the + * XMD. + */ +STATIC void +xfs_xmd_item_release( + struct xfs_log_item *lip) +{ + struct xfs_xmd_log_item *xmd_lip = XMD_ITEM(lip); + + xfs_xmi_release(xmd_lip->xmd_intent_log_item); + kvfree(xmd_lip->xmd_item.li_lv_shadow); + kmem_cache_free(xfs_xmd_cache, xmd_lip); +} + +static struct xfs_log_item * +xfs_xmd_item_intent( + struct xfs_log_item *lip) +{ + return &XMD_ITEM(lip)->xmd_intent_log_item->xmi_item; +} + +static const struct xfs_item_ops xfs_xmd_item_ops = { + .flags = XFS_ITEM_RELEASE_WHEN_COMMITTED | + XFS_ITEM_INTENT_DONE, + .iop_size = xfs_xmd_item_size, + .iop_format = xfs_xmd_item_format, + .iop_release = xfs_xmd_item_release, + .iop_intent = xfs_xmd_item_intent, +}; + +/* Log file mapping exchange information in the intent item. */ +STATIC struct xfs_log_item * +xfs_exchmaps_create_intent( + struct xfs_trans *tp, + struct list_head *items, + unsigned int count, + bool sort) +{ + struct xfs_xmi_log_item *xmi_lip; + struct xfs_exchmaps_intent *xmi; + struct xfs_xmi_log_format *xlf; + + ASSERT(count == 1); + + xmi = list_first_entry_or_null(items, struct xfs_exchmaps_intent, + xmi_list); + + xmi_lip = xfs_xmi_init(tp->t_mountp); + xlf = &xmi_lip->xmi_format; + + xlf->xmi_inode1 = I_INO(xmi->xmi_ip1); + xlf->xmi_igen1 = VFS_I(xmi->xmi_ip1)->i_generation; + xlf->xmi_inode2 = I_INO(xmi->xmi_ip2); + xlf->xmi_igen2 = VFS_I(xmi->xmi_ip2)->i_generation; + xlf->xmi_startoff1 = xmi->xmi_startoff1; + xlf->xmi_startoff2 = xmi->xmi_startoff2; + xlf->xmi_blockcount = xmi->xmi_blockcount; + xlf->xmi_isize1 = xmi->xmi_isize1; + xlf->xmi_isize2 = xmi->xmi_isize2; + xlf->xmi_flags = xmi->xmi_flags & XFS_EXCHMAPS_LOGGED_FLAGS; + + return &xmi_lip->xmi_item; +} + +STATIC struct xfs_log_item * +xfs_exchmaps_create_done( + struct xfs_trans *tp, + struct xfs_log_item *intent, + unsigned int count) +{ + struct xfs_xmi_log_item *xmi_lip = XMI_ITEM(intent); + struct xfs_xmd_log_item *xmd_lip; + + xmd_lip = kmem_cache_zalloc(xfs_xmd_cache, GFP_KERNEL | __GFP_NOFAIL); + xfs_log_item_init(tp->t_mountp, &xmd_lip->xmd_item, XFS_LI_XMD, + &xfs_xmd_item_ops); + xmd_lip->xmd_intent_log_item = xmi_lip; + xmd_lip->xmd_format.xmd_xmi_id = xmi_lip->xmi_format.xmi_id; + + return &xmd_lip->xmd_item; +} + +/* Add this deferred XMI to the transaction. */ +void +xfs_exchmaps_defer_add( + struct xfs_trans *tp, + struct xfs_exchmaps_intent *xmi) +{ + trace_xfs_exchmaps_defer(tp->t_mountp, xmi); + + xfs_defer_add(tp, &xmi->xmi_list, &xfs_exchmaps_defer_type); +} + +static inline struct xfs_exchmaps_intent *xmi_entry(const struct list_head *e) +{ + return list_entry(e, struct xfs_exchmaps_intent, xmi_list); +} + +/* Cancel a deferred file mapping exchange. */ +STATIC void +xfs_exchmaps_cancel_item( + struct list_head *item) +{ + struct xfs_exchmaps_intent *xmi = xmi_entry(item); + + kmem_cache_free(xfs_exchmaps_intent_cache, xmi); +} + +/* Process a deferred file mapping exchange. */ +STATIC int +xfs_exchmaps_finish_item( + struct xfs_trans *tp, + struct xfs_log_item *done, + struct list_head *item, + struct xfs_btree_cur **state) +{ + struct xfs_exchmaps_intent *xmi = xmi_entry(item); + int error; + + /* + * Exchange one more mappings between two files. If there's still more + * work to do, we want to requeue ourselves after all other pending + * deferred operations have finished. This includes all of the dfops + * that we queued directly as well as any new ones created in the + * process of finishing the others. Doing so prevents us from queuing + * a large number of XMI log items in kernel memory, which in turn + * prevents us from pinning the tail of the log (while logging those + * new XMI items) until the first XMI items can be processed. + */ + error = xfs_exchmaps_finish_one(tp, xmi); + if (error != -EAGAIN) + xfs_exchmaps_cancel_item(item); + return error; +} + +/* Abort all pending XMIs. */ +STATIC void +xfs_exchmaps_abort_intent( + struct xfs_log_item *intent) +{ + xfs_xmi_release(XMI_ITEM(intent)); +} + +/* Is this recovered XMI ok? */ +static inline bool +xfs_xmi_validate( + struct xfs_mount *mp, + struct xfs_xmi_log_item *xmi_lip) +{ + struct xfs_xmi_log_format *xlf = &xmi_lip->xmi_format; + + if (!xfs_has_exchange_range(mp)) + return false; + + if (xmi_lip->xmi_format.__pad != 0) + return false; + + if (xlf->xmi_flags & ~XFS_EXCHMAPS_LOGGED_FLAGS) + return false; + + if (!xfs_verify_ino(mp, xlf->xmi_inode1) || + !xfs_verify_ino(mp, xlf->xmi_inode2)) + return false; + + if (!xfs_verify_fileext(mp, xlf->xmi_startoff1, xlf->xmi_blockcount)) + return false; + + return xfs_verify_fileext(mp, xlf->xmi_startoff2, xlf->xmi_blockcount); +} + +/* + * Use the recovered log state to create a new request, estimate resource + * requirements, and create a new incore intent state. + */ +STATIC struct xfs_exchmaps_intent * +xfs_xmi_item_recover_intent( + struct xfs_mount *mp, + struct xfs_defer_pending *dfp, + const struct xfs_xmi_log_format *xlf, + struct xfs_exchmaps_req *req, + struct xfs_inode **ipp1, + struct xfs_inode **ipp2) +{ + struct xfs_inode *ip1, *ip2; + struct xfs_exchmaps_intent *xmi; + int error; + + /* + * Grab both inodes and set IRECOVERY to prevent trimming of post-eof + * mappings and freeing of unlinked inodes until we're totally done + * processing files. The ondisk format of this new log item contains + * file handle information, which is why recovery for other items do + * not check the inode generation number. + */ + error = xlog_recover_iget_handle(mp, xlf->xmi_inode1, xlf->xmi_igen1, + &ip1); + if (error) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, xlf, + sizeof(*xlf)); + return ERR_PTR(error); + } + + error = xlog_recover_iget_handle(mp, xlf->xmi_inode2, xlf->xmi_igen2, + &ip2); + if (error) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, xlf, + sizeof(*xlf)); + goto err_rele1; + } + + req->ip1 = ip1; + req->ip2 = ip2; + req->startoff1 = xlf->xmi_startoff1; + req->startoff2 = xlf->xmi_startoff2; + req->blockcount = xlf->xmi_blockcount; + req->flags = xlf->xmi_flags & XFS_EXCHMAPS_PARAMS; + + xfs_exchrange_ilock(NULL, ip1, ip2); + error = xfs_exchmaps_estimate(req); + xfs_exchrange_iunlock(ip1, ip2); + if (error) + goto err_rele2; + + *ipp1 = ip1; + *ipp2 = ip2; + xmi = xfs_exchmaps_init_intent(req); + xfs_defer_add_item(dfp, &xmi->xmi_list); + return xmi; + +err_rele2: + xfs_irele(ip2); +err_rele1: + xfs_irele(ip1); + req->ip2 = req->ip1 = NULL; + return ERR_PTR(error); +} + +/* Process a file mapping exchange item that was recovered from the log. */ +STATIC int +xfs_exchmaps_recover_work( + struct xfs_defer_pending *dfp, + struct list_head *capture_list) +{ + struct xfs_exchmaps_req req = { .flags = 0 }; + struct xfs_trans_res resv; + struct xfs_exchmaps_intent *xmi; + struct xfs_log_item *lip = dfp->dfp_intent; + struct xfs_xmi_log_item *xmi_lip = XMI_ITEM(lip); + struct xfs_mount *mp = lip->li_log->l_mp; + struct xfs_trans *tp; + struct xfs_inode *ip1, *ip2; + int error = 0; + + if (!xfs_xmi_validate(mp, xmi_lip)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + &xmi_lip->xmi_format, + sizeof(xmi_lip->xmi_format)); + return -EFSCORRUPTED; + } + + xmi = xfs_xmi_item_recover_intent(mp, dfp, &xmi_lip->xmi_format, &req, + &ip1, &ip2); + if (IS_ERR(xmi)) + return PTR_ERR(xmi); + + trace_xfs_exchmaps_recover(mp, xmi); + + resv = xlog_recover_resv(&M_RES(mp)->tr_write); + error = xfs_trans_alloc(mp, &resv, req.resblks, 0, 0, &tp); + if (error) + goto err_rele; + + xfs_exchrange_ilock(tp, ip1, ip2); + + xfs_exchmaps_ensure_reflink(tp, xmi); + xfs_exchmaps_upgrade_extent_counts(tp, xmi); + error = xlog_recover_finish_intent(tp, dfp); + if (error == -EFSCORRUPTED) + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + &xmi_lip->xmi_format, + sizeof(xmi_lip->xmi_format)); + if (error) + goto err_cancel; + + /* + * Commit transaction, which frees the transaction and saves the inodes + * for later replay activities. + */ + error = xfs_defer_ops_capture_and_commit(tp, capture_list); + goto err_unlock; + +err_cancel: + xfs_trans_cancel(tp); +err_unlock: + xfs_exchrange_iunlock(ip1, ip2); +err_rele: + xfs_irele(ip2); + xfs_irele(ip1); + return error; +} + +/* Relog an intent item to push the log tail forward. */ +static struct xfs_log_item * +xfs_exchmaps_relog_intent( + struct xfs_trans *tp, + struct xfs_log_item *intent, + struct xfs_log_item *done_item) +{ + struct xfs_xmi_log_item *xmi_lip; + struct xfs_xmi_log_format *old_xlf, *new_xlf; + + old_xlf = &XMI_ITEM(intent)->xmi_format; + + xmi_lip = xfs_xmi_init(tp->t_mountp); + new_xlf = &xmi_lip->xmi_format; + + new_xlf->xmi_inode1 = old_xlf->xmi_inode1; + new_xlf->xmi_inode2 = old_xlf->xmi_inode2; + new_xlf->xmi_igen1 = old_xlf->xmi_igen1; + new_xlf->xmi_igen2 = old_xlf->xmi_igen2; + new_xlf->xmi_startoff1 = old_xlf->xmi_startoff1; + new_xlf->xmi_startoff2 = old_xlf->xmi_startoff2; + new_xlf->xmi_blockcount = old_xlf->xmi_blockcount; + new_xlf->xmi_flags = old_xlf->xmi_flags; + new_xlf->xmi_isize1 = old_xlf->xmi_isize1; + new_xlf->xmi_isize2 = old_xlf->xmi_isize2; + + return &xmi_lip->xmi_item; +} + +const struct xfs_defer_op_type xfs_exchmaps_defer_type = { + .name = "exchmaps", + .max_items = 1, + .create_intent = xfs_exchmaps_create_intent, + .abort_intent = xfs_exchmaps_abort_intent, + .create_done = xfs_exchmaps_create_done, + .finish_item = xfs_exchmaps_finish_item, + .cancel_item = xfs_exchmaps_cancel_item, + .recover_work = xfs_exchmaps_recover_work, + .relog_intent = xfs_exchmaps_relog_intent, +}; + +STATIC bool +xfs_xmi_item_match( + struct xfs_log_item *lip, + uint64_t intent_id) +{ + return XMI_ITEM(lip)->xmi_format.xmi_id == intent_id; +} + +static const struct xfs_item_ops xfs_xmi_item_ops = { + .flags = XFS_ITEM_INTENT, + .iop_size = xfs_xmi_item_size, + .iop_format = xfs_xmi_item_format, + .iop_unpin = xfs_xmi_item_unpin, + .iop_release = xfs_xmi_item_release, + .iop_match = xfs_xmi_item_match, +}; + +/* + * This routine is called to create an in-core file mapping exchange item from + * the xmi format structure which was logged on disk. It allocates an in-core + * xmi, copies the exchange information from the format structure into it, and + * adds the xmi to the AIL with the given LSN. + */ +STATIC int +xlog_recover_xmi_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_mount *mp = log->l_mp; + struct xfs_xmi_log_item *xmi_lip; + struct xfs_xmi_log_format *xmi_formatp; + size_t len; + + len = sizeof(struct xfs_xmi_log_format); + if (item->ri_buf[0].iov_len != len) { + XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + return -EFSCORRUPTED; + } + + xmi_formatp = item->ri_buf[0].iov_base; + if (xmi_formatp->__pad != 0) { + XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + return -EFSCORRUPTED; + } + + xmi_lip = xfs_xmi_init(mp); + memcpy(&xmi_lip->xmi_format, xmi_formatp, len); + + xlog_recover_intent_item(log, &xmi_lip->xmi_item, lsn, + &xfs_exchmaps_defer_type); + return 0; +} + +const struct xlog_recover_item_ops xlog_xmi_item_ops = { + .item_type = XFS_LI_XMI, + .commit_pass2 = xlog_recover_xmi_commit_pass2, +}; + +/* + * This routine is called when an XMD format structure is found in a committed + * transaction in the log. Its purpose is to cancel the corresponding XMI if it + * was still in the log. To do this it searches the AIL for the XMI with an id + * equal to that in the XMD format structure. If we find it we drop the XMD + * reference, which removes the XMI from the AIL and frees it. + */ +STATIC int +xlog_recover_xmd_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_xmd_log_format *xmd_formatp; + + xmd_formatp = item->ri_buf[0].iov_base; + if (item->ri_buf[0].iov_len != sizeof(struct xfs_xmd_log_format)) { + XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + return -EFSCORRUPTED; + } + + xlog_recover_release_intent(log, XFS_LI_XMI, xmd_formatp->xmd_xmi_id); + return 0; +} + +const struct xlog_recover_item_ops xlog_xmd_item_ops = { + .item_type = XFS_LI_XMD, + .commit_pass2 = xlog_recover_xmd_commit_pass2, +}; diff --git a/fs/xfs/xfs_exchmaps_item.h b/fs/xfs/xfs_exchmaps_item.h new file mode 100644 index 000000000..efa368d25 --- /dev/null +++ b/fs/xfs/xfs_exchmaps_item.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_EXCHMAPS_ITEM_H__ +#define __XFS_EXCHMAPS_ITEM_H__ + +/* + * The file mapping exchange intent item helps us exchange multiple file + * mappings between two inode forks. It does this by tracking the range of + * file block offsets that still need to be exchanged, and relogs as progress + * happens. + * + * *I items should be recorded in the *first* of a series of rolled + * transactions, and the *D items should be recorded in the same transaction + * that records the associated bmbt updates. + * + * Should the system crash after the commit of the first transaction but + * before the commit of the final transaction in a series, log recovery will + * use the redo information recorded by the intent items to replay the + * rest of the mapping exchanges. + */ + +/* kernel only XMI/XMD definitions */ + +struct xfs_mount; +struct kmem_cache; + +/* + * This is the incore file mapping exchange intent log item. It is used to log + * the fact that we are exchanging mappings between two files. It is used in + * conjunction with the incore file mapping exchange done log item described + * below. + * + * These log items follow the same rules as struct xfs_efi_log_item; see the + * comments about that structure (in xfs_extfree_item.h) for more details. + */ +struct xfs_xmi_log_item { + struct xfs_log_item xmi_item; + atomic_t xmi_refcount; + struct xfs_xmi_log_format xmi_format; +}; + +/* + * This is the incore file mapping exchange done log item. It is used to log + * the fact that an exchange mentioned in an earlier xmi item have been + * performed. + */ +struct xfs_xmd_log_item { + struct xfs_log_item xmd_item; + struct xfs_xmi_log_item *xmd_intent_log_item; + struct xfs_xmd_log_format xmd_format; +}; + +extern struct kmem_cache *xfs_xmi_cache; +extern struct kmem_cache *xfs_xmd_cache; + +struct xfs_exchmaps_intent; + +void xfs_exchmaps_defer_add(struct xfs_trans *tp, + struct xfs_exchmaps_intent *xmi); + +#endif /* __XFS_EXCHMAPS_ITEM_H__ */ diff --git a/fs/xfs/xfs_exchrange.c b/fs/xfs/xfs_exchrange.c new file mode 100644 index 000000000..94965a6c2 --- /dev/null +++ b/fs/xfs/xfs_exchrange.c @@ -0,0 +1,923 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_quota.h" +#include "xfs_bmap_util.h" +#include "xfs_reflink.h" +#include "xfs_trace.h" +#include "xfs_exchrange.h" +#include "xfs_exchmaps.h" +#include "xfs_sb.h" +#include "xfs_icache.h" +#include "xfs_log.h" +#include "xfs_rtbitmap.h" +#include + +/* Lock (and optionally join) two inodes for a file range exchange. */ +void +xfs_exchrange_ilock( + struct xfs_trans *tp, + struct xfs_inode *ip1, + struct xfs_inode *ip2) +{ + if (ip1 != ip2) + xfs_lock_two_inodes(ip1, XFS_ILOCK_EXCL, + ip2, XFS_ILOCK_EXCL); + else + xfs_ilock(ip1, XFS_ILOCK_EXCL); + if (tp) { + xfs_trans_ijoin(tp, ip1, 0); + if (ip2 != ip1) + xfs_trans_ijoin(tp, ip2, 0); + } + +} + +/* Unlock two inodes after a file range exchange operation. */ +void +xfs_exchrange_iunlock( + struct xfs_inode *ip1, + struct xfs_inode *ip2) +{ + if (ip2 != ip1) + xfs_iunlock(ip2, XFS_ILOCK_EXCL); + xfs_iunlock(ip1, XFS_ILOCK_EXCL); +} + +/* + * Estimate the resource requirements to exchange file contents between the two + * files. The caller is required to hold the IOLOCK and the MMAPLOCK and to + * have flushed both inodes' pagecache and active direct-ios. + */ +int +xfs_exchrange_estimate( + struct xfs_exchmaps_req *req) +{ + int error; + + xfs_exchrange_ilock(NULL, req->ip1, req->ip2); + error = xfs_exchmaps_estimate(req); + xfs_exchrange_iunlock(req->ip1, req->ip2); + return error; +} + +/* + * Check that file2's metadata agree with the snapshot that we took for the + * range commit request. + * + * This should be called after the filesystem has locked /all/ inode metadata + * against modification. + */ +STATIC int +xfs_exchrange_check_freshness( + const struct xfs_exchrange *fxr, + struct xfs_inode *ip2) +{ + struct inode *inode2 = VFS_I(ip2); + struct timespec64 ctime = inode_get_ctime(inode2); + struct timespec64 mtime = inode_get_mtime(inode2); + + trace_xfs_exchrange_freshness(fxr, ip2); + + /* Check that file2 hasn't otherwise been modified. */ + if (fxr->file2_ino != inode2->i_ino || + fxr->file2_gen != inode2->i_generation || + !timespec64_equal(&fxr->file2_ctime, &ctime) || + !timespec64_equal(&fxr->file2_mtime, &mtime)) + return -EBUSY; + + return 0; +} + +#define QRETRY_IP1 (0x1) +#define QRETRY_IP2 (0x2) + +/* + * Obtain a quota reservation to make sure we don't hit EDQUOT. We can skip + * this if quota enforcement is disabled or if both inodes' dquots are the + * same. The qretry structure must be initialized to zeroes before the first + * call to this function. + */ +STATIC int +xfs_exchrange_reserve_quota( + struct xfs_trans *tp, + const struct xfs_exchmaps_req *req, + unsigned int *qretry) +{ + int64_t ddelta, rdelta; + int ip1_error = 0; + int error; + + ASSERT(!xfs_is_metadir_inode(req->ip1)); + ASSERT(!xfs_is_metadir_inode(req->ip2)); + + /* + * Don't bother with a quota reservation if we're not enforcing them + * or the two inodes have the same dquots. + */ + if (!XFS_IS_QUOTA_ON(tp->t_mountp) || req->ip1 == req->ip2 || + (req->ip1->i_udquot == req->ip2->i_udquot && + req->ip1->i_gdquot == req->ip2->i_gdquot && + req->ip1->i_pdquot == req->ip2->i_pdquot)) + return 0; + + *qretry = 0; + + /* + * For each file, compute the net gain in the number of regular blocks + * that will be mapped into that file and reserve that much quota. The + * quota counts must be able to absorb at least that much space. + */ + ddelta = req->ip2_bcount - req->ip1_bcount; + rdelta = req->ip2_rtbcount - req->ip1_rtbcount; + if (ddelta > 0 || rdelta > 0) { + error = xfs_trans_reserve_quota_nblks(tp, req->ip1, + ddelta > 0 ? ddelta : 0, + rdelta > 0 ? rdelta : 0, + false); + if (error == -EDQUOT || error == -ENOSPC) { + /* + * Save this error and see what happens if we try to + * reserve quota for ip2. Then report both. + */ + *qretry |= QRETRY_IP1; + ip1_error = error; + error = 0; + } + if (error) + return error; + } + if (ddelta < 0 || rdelta < 0) { + error = xfs_trans_reserve_quota_nblks(tp, req->ip2, + ddelta < 0 ? -ddelta : 0, + rdelta < 0 ? -rdelta : 0, + false); + if (error == -EDQUOT || error == -ENOSPC) + *qretry |= QRETRY_IP2; + if (error) + return error; + } + if (ip1_error) + return ip1_error; + + /* + * For each file, forcibly reserve the gross gain in mapped blocks so + * that we don't trip over any quota block reservation assertions. + * We must reserve the gross gain because the quota code subtracts from + * bcount the number of blocks that we unmap; it does not add that + * quantity back to the quota block reservation. + */ + error = xfs_trans_reserve_quota_nblks(tp, req->ip1, req->ip1_bcount, + req->ip1_rtbcount, true); + if (error) + return error; + + return xfs_trans_reserve_quota_nblks(tp, req->ip2, req->ip2_bcount, + req->ip2_rtbcount, true); +} + +/* Exchange the mappings (and hence the contents) of two files' forks. */ +STATIC int +xfs_exchrange_mappings( + const struct xfs_exchrange *fxr, + struct xfs_inode *ip1, + struct xfs_inode *ip2) +{ + struct xfs_mount *mp = ip1->i_mount; + struct xfs_exchmaps_req req = { + .ip1 = ip1, + .ip2 = ip2, + .startoff1 = XFS_B_TO_FSBT(mp, fxr->file1_offset), + .startoff2 = XFS_B_TO_FSBT(mp, fxr->file2_offset), + .blockcount = XFS_B_TO_FSB(mp, fxr->length), + }; + struct xfs_trans *tp; + unsigned int qretry; + bool retried = false; + int error; + + trace_xfs_exchrange_mappings(fxr, ip1, ip2); + + if (fxr->flags & XFS_EXCHANGE_RANGE_TO_EOF) + req.flags |= XFS_EXCHMAPS_SET_SIZES; + if (fxr->flags & XFS_EXCHANGE_RANGE_FILE1_WRITTEN) + req.flags |= XFS_EXCHMAPS_INO1_WRITTEN; + + /* + * Round the request length up to the nearest file allocation unit. + * The prep function already checked that the request offsets and + * length in @fxr are safe to round up. + */ + if (xfs_inode_has_bigrtalloc(ip2)) + req.blockcount = xfs_blen_roundup_rtx(mp, req.blockcount); + + error = xfs_exchrange_estimate(&req); + if (error) + return error; + +retry: + /* Allocate the transaction, lock the inodes, and join them. */ + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, req.resblks, 0, + XFS_TRANS_RES_FDBLKS, &tp); + if (error) + return error; + + xfs_exchrange_ilock(tp, ip1, ip2); + + trace_xfs_exchrange_before(ip2, 2); + trace_xfs_exchrange_before(ip1, 1); + + error = xfs_exchmaps_check_forks(mp, &req); + if (error) + goto out_trans_cancel; + + /* + * Reserve ourselves some quota if any of them are in enforcing mode. + * In theory we only need enough to satisfy the change in the number + * of blocks between the two ranges being remapped. + */ + error = xfs_exchrange_reserve_quota(tp, &req, &qretry); + if ((error == -EDQUOT || error == -ENOSPC) && !retried) { + xfs_trans_cancel(tp); + xfs_exchrange_iunlock(ip1, ip2); + if (qretry & QRETRY_IP1) + xfs_blockgc_free_quota(ip1, 0); + if (qretry & QRETRY_IP2) + xfs_blockgc_free_quota(ip2, 0); + retried = true; + goto retry; + } + if (error) + goto out_trans_cancel; + + /* If we got this far on a dry run, all parameters are ok. */ + if (fxr->flags & XFS_EXCHANGE_RANGE_DRY_RUN) + goto out_trans_cancel; + + /* Update the mtime and ctime of both files. */ + if (fxr->flags & __XFS_EXCHANGE_RANGE_UPD_CMTIME1) + xfs_trans_ichgtime(tp, ip1, XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG); + if (fxr->flags & __XFS_EXCHANGE_RANGE_UPD_CMTIME2) + xfs_trans_ichgtime(tp, ip2, XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG); + + xfs_exchange_mappings(tp, &req); + + /* + * Force the log to persist metadata updates if the caller or the + * administrator requires this. The generic prep function already + * flushed the relevant parts of the page cache. + */ + if (xfs_has_wsync(mp) || (fxr->flags & XFS_EXCHANGE_RANGE_DSYNC)) + xfs_trans_set_sync(tp); + + error = xfs_trans_commit(tp); + + trace_xfs_exchrange_after(ip2, 2); + trace_xfs_exchrange_after(ip1, 1); + + if (error) + goto out_unlock; + + /* + * If the caller wanted us to exchange the contents of two complete + * files of unequal length, exchange the incore sizes now. This should + * be safe because we flushed both files' page caches, exchanged all + * the mappings, and updated the ondisk sizes. + */ + if (fxr->flags & XFS_EXCHANGE_RANGE_TO_EOF) { + loff_t temp; + + temp = i_size_read(VFS_I(ip2)); + i_size_write(VFS_I(ip2), i_size_read(VFS_I(ip1))); + i_size_write(VFS_I(ip1), temp); + } + +out_unlock: + xfs_exchrange_iunlock(ip1, ip2); + return error; + +out_trans_cancel: + xfs_trans_cancel(tp); + goto out_unlock; +} + +/* + * Generic code for exchanging ranges of two files via XFS_IOC_EXCHANGE_RANGE. + * This part deals with struct file objects and byte ranges and does not deal + * with XFS-specific data structures such as xfs_inodes and block ranges. This + * separation may some day facilitate porting to another filesystem. + * + * The goal is to exchange fxr.length bytes starting at fxr.file1_offset in + * file1 with the same number of bytes starting at fxr.file2_offset in file2. + * Implementations must call xfs_exchange_range_prep to prepare the two + * files prior to taking locks; and they must update the inode change and mod + * times of both files as part of the metadata update. The timestamp update + * and freshness checks must be done atomically as part of the data exchange + * operation to ensure correctness of the freshness check. + * xfs_exchange_range_finish must be called after the operation completes + * successfully but before locks are dropped. + */ + +/* + * Performs necessary checks before doing a range exchange, having stabilized + * mutable inode attributes via i_rwsem. + */ +static inline int +xfs_exchange_range_checks( + struct xfs_exchrange *fxr, + unsigned int alloc_unit) +{ + struct inode *inode1 = file_inode(fxr->file1); + loff_t size1 = i_size_read(inode1); + struct inode *inode2 = file_inode(fxr->file2); + loff_t size2 = i_size_read(inode2); + uint64_t allocmask = alloc_unit - 1; + int64_t test_len; + uint64_t blen; + loff_t tmp; + int error; + + /* Don't touch certain kinds of inodes */ + if (IS_IMMUTABLE(inode1) || IS_IMMUTABLE(inode2)) + return -EPERM; + if (IS_SWAPFILE(inode1) || IS_SWAPFILE(inode2)) + return -ETXTBSY; + + /* Ranges cannot start after EOF. */ + if (fxr->file1_offset > size1 || fxr->file2_offset > size2) + return -EINVAL; + + if (fxr->flags & XFS_EXCHANGE_RANGE_TO_EOF) { + /* + * If the caller said to exchange to EOF, we set the length of + * the request large enough to cover everything to the end of + * both files. + */ + fxr->length = max_t(int64_t, size1 - fxr->file1_offset, + size2 - fxr->file2_offset); + } else { + /* + * Otherwise we require both ranges to end within EOF. + */ + if (fxr->file1_offset + fxr->length > size1 || + fxr->file2_offset + fxr->length > size2) + return -EINVAL; + } + + /* + * The start of both ranges must be aligned to the file allocation + * unit. + */ + if (!IS_ALIGNED(fxr->file1_offset, alloc_unit) || + !IS_ALIGNED(fxr->file2_offset, alloc_unit)) + return -EINVAL; + + /* Ensure offsets don't wrap. */ + if (check_add_overflow(fxr->file1_offset, fxr->length, &tmp) || + check_add_overflow(fxr->file2_offset, fxr->length, &tmp)) + return -EINVAL; + + /* + * Make sure we don't hit any file size limits. If we hit any size + * limits such that test_length was adjusted, we abort the whole + * operation. + */ + test_len = fxr->length; + error = generic_write_check_limits(fxr->file2, fxr->file2_offset, + &test_len); + if (error) + return error; + error = generic_write_check_limits(fxr->file1, fxr->file1_offset, + &test_len); + if (error) + return error; + if (test_len != fxr->length) + return -EINVAL; + + /* + * If the user wanted us to exchange up to the infile's EOF, round up + * to the next allocation unit boundary for this check. Do the same + * for the outfile. + * + * Otherwise, reject the range length if it's not aligned to an + * allocation unit. + */ + if (fxr->file1_offset + fxr->length == size1) + blen = ALIGN(size1, alloc_unit) - fxr->file1_offset; + else if (fxr->file2_offset + fxr->length == size2) + blen = ALIGN(size2, alloc_unit) - fxr->file2_offset; + else if (!IS_ALIGNED(fxr->length, alloc_unit)) + return -EINVAL; + else + blen = fxr->length; + + /* Don't allow overlapped exchanges within the same file. */ + if (inode1 == inode2 && + fxr->file2_offset + blen > fxr->file1_offset && + fxr->file1_offset + blen > fxr->file2_offset) + return -EINVAL; + + /* + * Ensure that we don't exchange a partial EOF block into the middle of + * another file. + */ + if ((fxr->length & allocmask) == 0) + return 0; + + blen = fxr->length; + if (fxr->file2_offset + blen < size2) + blen &= ~allocmask; + + if (fxr->file1_offset + blen < size1) + blen &= ~allocmask; + + return blen == fxr->length ? 0 : -EINVAL; +} + +/* + * Check that the two inodes are eligible for range exchanges, the ranges make + * sense, and then flush all dirty data. Caller must ensure that the inodes + * have been locked against any other modifications. + */ +static inline int +xfs_exchange_range_prep( + struct xfs_exchrange *fxr, + unsigned int alloc_unit) +{ + struct inode *inode1 = file_inode(fxr->file1); + struct inode *inode2 = file_inode(fxr->file2); + bool same_inode = (inode1 == inode2); + int error; + + /* Check that we don't violate system file offset limits. */ + error = xfs_exchange_range_checks(fxr, alloc_unit); + if (error || fxr->length == 0) + return error; + + /* Wait for the completion of any pending IOs on both files */ + inode_dio_wait(inode1); + if (!same_inode) + inode_dio_wait(inode2); + + error = filemap_write_and_wait_range(inode1->i_mapping, + fxr->file1_offset, + fxr->file1_offset + fxr->length - 1); + if (error) + return error; + + error = filemap_write_and_wait_range(inode2->i_mapping, + fxr->file2_offset, + fxr->file2_offset + fxr->length - 1); + if (error) + return error; + + /* + * If the files or inodes involved require synchronous writes, amend + * the request to force the filesystem to flush all data and metadata + * to disk after the operation completes. + */ + if (((fxr->file1->f_flags | fxr->file2->f_flags) & O_SYNC) || + IS_SYNC(inode1) || IS_SYNC(inode2)) + fxr->flags |= XFS_EXCHANGE_RANGE_DSYNC; + + return 0; +} + +/* + * Finish a range exchange operation, if it was successful. Caller must ensure + * that the inodes are still locked against any other modifications. + */ +static inline int +xfs_exchange_range_finish( + struct xfs_exchrange *fxr) +{ + int error; + + error = file_remove_privs(fxr->file1); + if (error) + return error; + if (file_inode(fxr->file1) == file_inode(fxr->file2)) + return 0; + + return file_remove_privs(fxr->file2); +} + +/* + * Check the alignment of an exchange request when the allocation unit size + * isn't a power of two. The generic file-level helpers use (fast) + * bitmask-based alignment checks, but here we have to use slow long division. + */ +static int +xfs_exchrange_check_rtalign( + const struct xfs_exchrange *fxr, + struct xfs_inode *ip1, + struct xfs_inode *ip2, + unsigned int alloc_unit) +{ + uint64_t length = fxr->length; + uint64_t blen; + loff_t size1, size2; + + size1 = i_size_read(VFS_I(ip1)); + size2 = i_size_read(VFS_I(ip2)); + + /* The start of both ranges must be aligned to a rt extent. */ + if (!isaligned_64(fxr->file1_offset, alloc_unit) || + !isaligned_64(fxr->file2_offset, alloc_unit)) + return -EINVAL; + + if (fxr->flags & XFS_EXCHANGE_RANGE_TO_EOF) + length = max_t(int64_t, size1 - fxr->file1_offset, + size2 - fxr->file2_offset); + + /* + * If the user wanted us to exchange up to the infile's EOF, round up + * to the next rt extent boundary for this check. Do the same for the + * outfile. + * + * Otherwise, reject the range length if it's not rt extent aligned. + * We already confirmed the starting offsets' rt extent block + * alignment. + */ + if (fxr->file1_offset + length == size1) + blen = roundup_64(size1, alloc_unit) - fxr->file1_offset; + else if (fxr->file2_offset + length == size2) + blen = roundup_64(size2, alloc_unit) - fxr->file2_offset; + else if (!isaligned_64(length, alloc_unit)) + return -EINVAL; + else + blen = length; + + /* Don't allow overlapped exchanges within the same file. */ + if (ip1 == ip2 && + fxr->file2_offset + blen > fxr->file1_offset && + fxr->file1_offset + blen > fxr->file2_offset) + return -EINVAL; + + /* + * Ensure that we don't exchange a partial EOF rt extent into the + * middle of another file. + */ + if (isaligned_64(length, alloc_unit)) + return 0; + + blen = length; + if (fxr->file2_offset + length < size2) + blen = rounddown_64(blen, alloc_unit); + + if (fxr->file1_offset + blen < size1) + blen = rounddown_64(blen, alloc_unit); + + return blen == length ? 0 : -EINVAL; +} + +/* Prepare two files to have their data exchanged. */ +STATIC int +xfs_exchrange_prep( + struct xfs_exchrange *fxr, + struct xfs_inode *ip1, + struct xfs_inode *ip2) +{ + struct xfs_mount *mp = ip2->i_mount; + unsigned int alloc_unit = xfs_inode_alloc_unitsize(ip2); + int error; + + trace_xfs_exchrange_prep(fxr, ip1, ip2); + + /* Verify both files are either real-time or non-realtime */ + if (XFS_IS_REALTIME_INODE(ip1) != XFS_IS_REALTIME_INODE(ip2)) + return -EINVAL; + + /* Check non-power of two alignment issues, if necessary. */ + if (!is_power_of_2(alloc_unit)) { + error = xfs_exchrange_check_rtalign(fxr, ip1, ip2, alloc_unit); + if (error) + return error; + + /* + * Do the generic file-level checks with the regular block + * alignment. + */ + alloc_unit = mp->m_sb.sb_blocksize; + } + + error = xfs_exchange_range_prep(fxr, alloc_unit); + if (error || fxr->length == 0) + return error; + + if (fxr->flags & __XFS_EXCHANGE_RANGE_CHECK_FRESH2) { + error = xfs_exchrange_check_freshness(fxr, ip2); + if (error) + return error; + } + + /* Attach dquots to both inodes before changing block maps. */ + error = xfs_qm_dqattach(ip2); + if (error) + return error; + error = xfs_qm_dqattach(ip1); + if (error) + return error; + + trace_xfs_exchrange_flush(fxr, ip1, ip2); + + /* Flush the relevant ranges of both files. */ + error = xfs_flush_unmap_range(ip2, fxr->file2_offset, fxr->length); + if (error) + return error; + error = xfs_flush_unmap_range(ip1, fxr->file1_offset, fxr->length); + if (error) + return error; + + /* + * Cancel CoW fork preallocations for the ranges of both files. The + * prep function should have flushed all the dirty data, so the only + * CoW mappings remaining should be speculative. + */ + if (xfs_inode_has_cow_data(ip1)) { + error = xfs_reflink_cancel_cow_range(ip1, fxr->file1_offset, + fxr->length, true); + if (error) + return error; + } + + if (xfs_inode_has_cow_data(ip2)) { + error = xfs_reflink_cancel_cow_range(ip2, fxr->file2_offset, + fxr->length, true); + if (error) + return error; + } + + return 0; +} + +/* + * Exchange contents of files. This is the binding between the generic + * file-level concepts and the XFS inode-specific implementation. + */ +STATIC int +xfs_exchrange_contents( + struct xfs_exchrange *fxr) +{ + struct inode *inode1 = file_inode(fxr->file1); + struct inode *inode2 = file_inode(fxr->file2); + struct xfs_inode *ip1 = XFS_I(inode1); + struct xfs_inode *ip2 = XFS_I(inode2); + struct xfs_mount *mp = ip1->i_mount; + int error; + + if (!xfs_has_exchange_range(mp)) + return -EOPNOTSUPP; + + if (fxr->flags & ~(XFS_EXCHANGE_RANGE_ALL_FLAGS | + XFS_EXCHANGE_RANGE_PRIV_FLAGS)) + return -EINVAL; + + if (xfs_is_shutdown(mp)) + return -EIO; + + /* Lock both files against IO */ + error = xfs_ilock2_io_mmap(ip1, ip2); + if (error) + goto out_err; + + /* Prepare and then exchange file contents. */ + error = xfs_exchrange_prep(fxr, ip1, ip2); + if (error) + goto out_unlock; + + error = xfs_exchrange_mappings(fxr, ip1, ip2); + if (error) + goto out_unlock; + + /* + * Finish the exchange by removing special file privileges like any + * other file write would do. This may involve turning on support for + * logged xattrs if either file has security capabilities. + */ + error = xfs_exchange_range_finish(fxr); + if (error) + goto out_unlock; + +out_unlock: + xfs_iunlock2_io_mmap(ip1, ip2); +out_err: + if (error) + trace_xfs_exchrange_error(ip2, error, _RET_IP_); + return error; +} + +/* Exchange parts of two files. */ +static int +xfs_exchange_range( + struct xfs_exchrange *fxr) +{ + struct inode *inode1 = file_inode(fxr->file1); + struct inode *inode2 = file_inode(fxr->file2); + loff_t check_len = fxr->length; + int ret; + + BUILD_BUG_ON(XFS_EXCHANGE_RANGE_ALL_FLAGS & + XFS_EXCHANGE_RANGE_PRIV_FLAGS); + + /* Both files must be on the same mount/filesystem. */ + if (fxr->file1->f_path.mnt != fxr->file2->f_path.mnt) + return -EXDEV; + + if (fxr->flags & ~(XFS_EXCHANGE_RANGE_ALL_FLAGS | + __XFS_EXCHANGE_RANGE_CHECK_FRESH2)) + return -EINVAL; + + /* Userspace requests only honored for regular files. */ + if (S_ISDIR(inode1->i_mode) || S_ISDIR(inode2->i_mode)) + return -EISDIR; + if (!S_ISREG(inode1->i_mode) || !S_ISREG(inode2->i_mode)) + return -EINVAL; + + /* Both files must be opened for read and write. */ + if (!(fxr->file1->f_mode & FMODE_READ) || + !(fxr->file1->f_mode & FMODE_WRITE) || + !(fxr->file2->f_mode & FMODE_READ) || + !(fxr->file2->f_mode & FMODE_WRITE)) + return -EBADF; + + /* Neither file can be opened append-only. */ + if ((fxr->file1->f_flags & O_APPEND) || + (fxr->file2->f_flags & O_APPEND)) + return -EBADF; + + /* + * If we're exchanging to EOF we can't calculate the length until taking + * the iolock. Pass a 0 length to remap_verify_area similar to the + * FICLONE and FICLONERANGE ioctls that support cloning to EOF as well. + */ + if (fxr->flags & XFS_EXCHANGE_RANGE_TO_EOF) + check_len = 0; + ret = remap_verify_area(fxr->file1, fxr->file1_offset, check_len, true); + if (ret) + return ret; + ret = remap_verify_area(fxr->file2, fxr->file2_offset, check_len, true); + if (ret) + return ret; + + /* Update cmtime if the fd/inode don't forbid it. */ + if (!(fxr->file1->f_mode & FMODE_NOCMTIME) && !IS_NOCMTIME(inode1)) + fxr->flags |= __XFS_EXCHANGE_RANGE_UPD_CMTIME1; + if (!(fxr->file2->f_mode & FMODE_NOCMTIME) && !IS_NOCMTIME(inode2)) + fxr->flags |= __XFS_EXCHANGE_RANGE_UPD_CMTIME2; + + file_start_write(fxr->file2); + ret = xfs_exchrange_contents(fxr); + file_end_write(fxr->file2); + if (ret) + return ret; + + fsnotify_modify(fxr->file1); + if (fxr->file2 != fxr->file1) + fsnotify_modify(fxr->file2); + return 0; +} + +/* Collect exchange-range arguments from userspace. */ +long +xfs_ioc_exchange_range( + struct file *file, + struct xfs_exchange_range __user *argp) +{ + struct xfs_exchrange fxr = { + .file2 = file, + }; + struct xfs_exchange_range args; + + if (copy_from_user(&args, argp, sizeof(args))) + return -EFAULT; + if (memchr_inv(&args.pad, 0, sizeof(args.pad))) + return -EINVAL; + if (args.flags & ~XFS_EXCHANGE_RANGE_ALL_FLAGS) + return -EINVAL; + + fxr.file1_offset = args.file1_offset; + fxr.file2_offset = args.file2_offset; + fxr.length = args.length; + fxr.flags = args.flags; + + CLASS(fd, file1)(args.file1_fd); + if (fd_empty(file1)) + return -EBADF; + fxr.file1 = fd_file(file1); + + return xfs_exchange_range(&fxr); +} + +/* Opaque freshness blob for XFS_IOC_COMMIT_RANGE */ +struct xfs_commit_range_fresh { + xfs_fsid_t fsid; /* m_fixedfsid */ + __u64 file2_ino; /* inode number */ + __s64 file2_mtime; /* modification time */ + __s64 file2_ctime; /* change time */ + __s32 file2_mtime_nsec; /* mod time, nsec */ + __s32 file2_ctime_nsec; /* change time, nsec */ + __u32 file2_gen; /* inode generation */ + __u32 magic; /* zero */ +}; +#define XCR_FRESH_MAGIC 0x444F524B /* DORK */ + +/* Set up a commitrange operation by sampling file2's write-related attrs */ +long +xfs_ioc_start_commit( + struct file *file, + struct xfs_commit_range __user *argp) +{ + struct xfs_commit_range args = { }; + struct kstat kstat = { }; + struct xfs_commit_range_fresh *kern_f; + struct xfs_commit_range_fresh __user *user_f; + struct inode *inode2 = file_inode(file); + struct xfs_inode *ip2 = XFS_I(inode2); + const unsigned int lockflags = XFS_IOLOCK_SHARED | + XFS_MMAPLOCK_SHARED | + XFS_ILOCK_SHARED; + + BUILD_BUG_ON(sizeof(struct xfs_commit_range_fresh) != + sizeof(args.file2_freshness)); + + kern_f = (struct xfs_commit_range_fresh *)&args.file2_freshness; + + memcpy(&kern_f->fsid, ip2->i_mount->m_fixedfsid, sizeof(xfs_fsid_t)); + + xfs_ilock(ip2, lockflags); + /* Force writing of a distinct ctime if any writes happen. */ + fill_mg_cmtime(&kstat, STATX_CTIME | STATX_MTIME, inode2); + kern_f->file2_ctime = kstat.ctime.tv_sec; + kern_f->file2_ctime_nsec = kstat.ctime.tv_nsec; + kern_f->file2_mtime = kstat.mtime.tv_sec; + kern_f->file2_mtime_nsec = kstat.mtime.tv_nsec; + kern_f->file2_ino = inode2->i_ino; + kern_f->file2_gen = inode2->i_generation; + kern_f->magic = XCR_FRESH_MAGIC; + xfs_iunlock(ip2, lockflags); + + user_f = (struct xfs_commit_range_fresh __user *)&argp->file2_freshness; + if (copy_to_user(user_f, kern_f, sizeof(*kern_f))) + return -EFAULT; + + return 0; +} + +/* + * Exchange file1 and file2 contents if file2 has not been written since the + * start commit operation. + */ +long +xfs_ioc_commit_range( + struct file *file, + struct xfs_commit_range __user *argp) +{ + struct xfs_exchrange fxr = { + .file2 = file, + }; + struct xfs_commit_range args; + struct xfs_commit_range_fresh *kern_f; + struct xfs_inode *ip2 = XFS_I(file_inode(file)); + struct xfs_mount *mp = ip2->i_mount; + + kern_f = (struct xfs_commit_range_fresh *)&args.file2_freshness; + + if (copy_from_user(&args, argp, sizeof(args))) + return -EFAULT; + if (args.flags & ~XFS_EXCHANGE_RANGE_ALL_FLAGS) + return -EINVAL; + if (kern_f->magic != XCR_FRESH_MAGIC) + return -EBUSY; + if (memcmp(&kern_f->fsid, mp->m_fixedfsid, sizeof(xfs_fsid_t))) + return -EBUSY; + + fxr.file1_offset = args.file1_offset; + fxr.file2_offset = args.file2_offset; + fxr.length = args.length; + fxr.flags = args.flags | __XFS_EXCHANGE_RANGE_CHECK_FRESH2; + fxr.file2_ino = kern_f->file2_ino; + fxr.file2_gen = kern_f->file2_gen; + fxr.file2_mtime.tv_sec = kern_f->file2_mtime; + fxr.file2_mtime.tv_nsec = kern_f->file2_mtime_nsec; + fxr.file2_ctime.tv_sec = kern_f->file2_ctime; + fxr.file2_ctime.tv_nsec = kern_f->file2_ctime_nsec; + + CLASS(fd, file1)(args.file1_fd); + if (fd_empty(file1)) + return -EBADF; + fxr.file1 = fd_file(file1); + + return xfs_exchange_range(&fxr); +} diff --git a/fs/xfs/xfs_exchrange.h b/fs/xfs/xfs_exchrange.h new file mode 100644 index 000000000..bc1298aba --- /dev/null +++ b/fs/xfs/xfs_exchrange.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2020-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_EXCHRANGE_H__ +#define __XFS_EXCHRANGE_H__ + +/* Update the mtime/cmtime of file1 and file2 */ +#define __XFS_EXCHANGE_RANGE_UPD_CMTIME1 (1ULL << 63) +#define __XFS_EXCHANGE_RANGE_UPD_CMTIME2 (1ULL << 62) + +/* Freshness check required */ +#define __XFS_EXCHANGE_RANGE_CHECK_FRESH2 (1ULL << 61) + +#define XFS_EXCHANGE_RANGE_PRIV_FLAGS (__XFS_EXCHANGE_RANGE_UPD_CMTIME1 | \ + __XFS_EXCHANGE_RANGE_UPD_CMTIME2 | \ + __XFS_EXCHANGE_RANGE_CHECK_FRESH2) + +struct xfs_exchrange { + struct file *file1; + struct file *file2; + + loff_t file1_offset; + loff_t file2_offset; + u64 length; + + u64 flags; /* XFS_EXCHANGE_RANGE flags */ + + /* file2 metadata for freshness checks */ + u64 file2_ino; + struct timespec64 file2_mtime; + struct timespec64 file2_ctime; + u32 file2_gen; +}; + +long xfs_ioc_exchange_range(struct file *file, + struct xfs_exchange_range __user *argp); +long xfs_ioc_start_commit(struct file *file, + struct xfs_commit_range __user *argp); +long xfs_ioc_commit_range(struct file *file, + struct xfs_commit_range __user *argp); + +struct xfs_exchmaps_req; + +void xfs_exchrange_ilock(struct xfs_trans *tp, struct xfs_inode *ip1, + struct xfs_inode *ip2); +void xfs_exchrange_iunlock(struct xfs_inode *ip1, struct xfs_inode *ip2); + +int xfs_exchrange_estimate(struct xfs_exchmaps_req *req); + +#endif /* __XFS_EXCHRANGE_H__ */ diff --git a/fs/xfs/xfs_export.c b/fs/xfs/xfs_export.c new file mode 100644 index 000000000..9afda52ed --- /dev/null +++ b/fs/xfs/xfs_export.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2004-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_dir2.h" +#include "xfs_export.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_inode_item.h" +#include "xfs_icache.h" +#include "xfs_pnfs.h" + +/* + * Note that we only accept fileids which are long enough rather than allow + * the parent generation number to default to zero. XFS considers zero a + * valid generation number not an invalid/wildcard value. + */ +static int xfs_fileid_length(int fileid_type) +{ + switch (fileid_type) { + case FILEID_INO32_GEN: + return 2; + case FILEID_INO32_GEN_PARENT: + return 4; + case FILEID_INO32_GEN | XFS_FILEID_TYPE_64FLAG: + return 3; + case FILEID_INO32_GEN_PARENT | XFS_FILEID_TYPE_64FLAG: + return 6; + } + return FILEID_INVALID; +} + +STATIC int +xfs_fs_encode_fh( + struct inode *inode, + __u32 *fh, + int *max_len, + struct inode *parent) +{ + struct xfs_mount *mp = XFS_M(inode->i_sb); + struct fid *fid = (struct fid *)fh; + struct xfs_fid64 *fid64 = (struct xfs_fid64 *)fh; + int fileid_type; + int len; + + /* Directories don't need their parent encoded, they have ".." */ + if (!parent) + fileid_type = FILEID_INO32_GEN; + else + fileid_type = FILEID_INO32_GEN_PARENT; + + /* + * If the filesystem may contain 64bit inode numbers, we need + * to use larger file handles that can represent them. + * + * While we only allocate inodes that do not fit into 32 bits any + * large enough filesystem may contain them, thus the slightly + * confusing looking conditional below. + */ + if (!xfs_has_small_inums(mp) || xfs_is_inode32(mp)) + fileid_type |= XFS_FILEID_TYPE_64FLAG; + + /* + * Only encode if there is enough space given. In practice + * this means we can't export a filesystem with 64bit inodes + * over NFSv2 with the subtree_check export option; the other + * seven combinations work. The real answer is "don't use v2". + */ + len = xfs_fileid_length(fileid_type); + if (*max_len < len) { + *max_len = len; + return FILEID_INVALID; + } + *max_len = len; + + switch (fileid_type) { + case FILEID_INO32_GEN_PARENT: + fid->i32.parent_ino = parent->i_ino; + fid->i32.parent_gen = parent->i_generation; + fallthrough; + case FILEID_INO32_GEN: + fid->i32.ino = inode->i_ino; + fid->i32.gen = inode->i_generation; + break; + case FILEID_INO32_GEN_PARENT | XFS_FILEID_TYPE_64FLAG: + fid64->parent_ino = parent->i_ino; + fid64->parent_gen = parent->i_generation; + fallthrough; + case FILEID_INO32_GEN | XFS_FILEID_TYPE_64FLAG: + fid64->ino = inode->i_ino; + fid64->gen = inode->i_generation; + break; + } + + return fileid_type; +} + +struct inode * +xfs_nfs_get_inode( + struct super_block *sb, + u64 ino, + u32 generation) +{ + xfs_mount_t *mp = XFS_M(sb); + xfs_inode_t *ip; + int error; + + /* + * NFS can sometimes send requests for ino 0. Fail them gracefully. + */ + if (ino == 0) + return ERR_PTR(-ESTALE); + + /* + * The XFS_IGET_UNTRUSTED means that an invalid inode number is just + * fine and not an indication of a corrupted filesystem as clients can + * send invalid file handles and we have to handle it gracefully.. + */ + error = xfs_iget(mp, NULL, ino, XFS_IGET_UNTRUSTED, 0, &ip); + if (error) { + + /* + * EINVAL means the inode cluster doesn't exist anymore. + * EFSCORRUPTED means the metadata pointing to the inode cluster + * or the inode cluster itself is corrupt. This implies the + * filehandle is stale, so we should translate it here. + * We don't use ESTALE directly down the chain to not + * confuse applications using bulkstat that expect EINVAL. + */ + switch (error) { + case -EINVAL: + case -ENOENT: + case -EFSCORRUPTED: + error = -ESTALE; + break; + default: + break; + } + return ERR_PTR(error); + } + + /* + * Reload the incore unlinked list to avoid failure in inodegc. + * Use an unlocked check here because unrecovered unlinked inodes + * should be somewhat rare. + */ + if (xfs_inode_unlinked_incomplete(ip)) { + error = xfs_inode_reload_unlinked(ip); + if (error) { + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); + xfs_irele(ip); + return ERR_PTR(error); + } + } + + if (VFS_I(ip)->i_generation != generation || IS_PRIVATE(VFS_I(ip))) { + xfs_irele(ip); + return ERR_PTR(-ESTALE); + } + + return VFS_I(ip); +} + +STATIC struct dentry * +xfs_fs_fh_to_dentry(struct super_block *sb, struct fid *fid, + int fh_len, int fileid_type) +{ + struct xfs_fid64 *fid64 = (struct xfs_fid64 *)fid; + struct inode *inode = NULL; + + if (fh_len < xfs_fileid_length(fileid_type)) + return NULL; + + switch (fileid_type) { + case FILEID_INO32_GEN_PARENT: + case FILEID_INO32_GEN: + inode = xfs_nfs_get_inode(sb, fid->i32.ino, fid->i32.gen); + break; + case FILEID_INO32_GEN_PARENT | XFS_FILEID_TYPE_64FLAG: + case FILEID_INO32_GEN | XFS_FILEID_TYPE_64FLAG: + inode = xfs_nfs_get_inode(sb, fid64->ino, fid64->gen); + break; + } + + return d_obtain_alias(inode); +} + +STATIC struct dentry * +xfs_fs_fh_to_parent(struct super_block *sb, struct fid *fid, + int fh_len, int fileid_type) +{ + struct xfs_fid64 *fid64 = (struct xfs_fid64 *)fid; + struct inode *inode = NULL; + + if (fh_len < xfs_fileid_length(fileid_type)) + return NULL; + + switch (fileid_type) { + case FILEID_INO32_GEN_PARENT: + inode = xfs_nfs_get_inode(sb, fid->i32.parent_ino, + fid->i32.parent_gen); + break; + case FILEID_INO32_GEN_PARENT | XFS_FILEID_TYPE_64FLAG: + inode = xfs_nfs_get_inode(sb, fid64->parent_ino, + fid64->parent_gen); + break; + } + + return d_obtain_alias(inode); +} + +STATIC struct dentry * +xfs_fs_get_parent( + struct dentry *child) +{ + int error; + struct xfs_inode *cip; + + error = xfs_lookup(XFS_I(d_inode(child)), &xfs_name_dotdot, &cip, NULL); + if (unlikely(error)) + return ERR_PTR(error); + + return d_obtain_alias(VFS_I(cip)); +} + +STATIC int +xfs_fs_nfs_commit_metadata( + struct inode *inode) +{ + return xfs_log_force_inode(XFS_I(inode)); +} + +const struct export_operations xfs_export_operations = { + .encode_fh = xfs_fs_encode_fh, + .fh_to_dentry = xfs_fs_fh_to_dentry, + .fh_to_parent = xfs_fs_fh_to_parent, + .get_parent = xfs_fs_get_parent, + .commit_metadata = xfs_fs_nfs_commit_metadata, +#ifdef CONFIG_EXPORTFS_BLOCK_OPS + .block_ops = &xfs_export_block_ops, +#endif +}; diff --git a/fs/xfs/xfs_export.h b/fs/xfs/xfs_export.h new file mode 100644 index 000000000..3cd85e890 --- /dev/null +++ b/fs/xfs/xfs_export.h @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_EXPORT_H__ +#define __XFS_EXPORT_H__ + +/* + * Common defines for code related to exporting XFS filesystems over NFS. + * + * The NFS fileid goes out on the wire as an array of + * 32bit unsigned ints in host order. There are 5 possible + * formats. + * + * (1) fileid_type=0x00 + * (no fileid data; handled by the generic code) + * + * (2) fileid_type=0x01 + * inode-num + * generation + * + * (3) fileid_type=0x02 + * inode-num + * generation + * parent-inode-num + * parent-generation + * + * (4) fileid_type=0x81 + * inode-num-lo32 + * inode-num-hi32 + * generation + * + * (5) fileid_type=0x82 + * inode-num-lo32 + * inode-num-hi32 + * generation + * parent-inode-num-lo32 + * parent-inode-num-hi32 + * parent-generation + * + * Note, the NFS filehandle also includes an fsid portion which + * may have an inode number in it. That number is hardcoded to + * 32bits and there is no way for XFS to intercept it. In + * practice this means when exporting an XFS filesystem with 64bit + * inodes you should either export the mountpoint (rather than + * a subdirectory) or use the "fsid" export option. + */ + +struct xfs_fid64 { + u64 ino; + u32 gen; + u64 parent_ino; + u32 parent_gen; +} __attribute__((packed)); + +/* This flag goes on the wire. Don't play with it. */ +#define XFS_FILEID_TYPE_64FLAG 0x80 /* NFS fileid has 64bit inodes */ + +struct inode *xfs_nfs_get_inode(struct super_block *sb, u64 ino, u32 gen); + +#endif /* __XFS_EXPORT_H__ */ diff --git a/fs/xfs/xfs_extent_busy.c b/fs/xfs/xfs_extent_busy.c new file mode 100644 index 000000000..41cf0605e --- /dev/null +++ b/fs/xfs/xfs_extent_busy.c @@ -0,0 +1,727 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc. + * Copyright (c) 2010 David Chinner. + * Copyright (c) 2011 Christoph Hellwig. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_shared.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_alloc.h" +#include "xfs_extent_busy.h" +#include "xfs_trace.h" +#include "xfs_trans.h" +#include "xfs_log.h" +#include "xfs_ag.h" +#include "xfs_rtgroup.h" + +struct xfs_extent_busy_tree { + spinlock_t eb_lock; + struct rb_root eb_tree; + unsigned int eb_gen; + wait_queue_head_t eb_wait; +}; + +static void +xfs_extent_busy_insert_list( + struct xfs_group *xg, + xfs_agblock_t bno, + xfs_extlen_t len, + unsigned int flags, + struct list_head *busy_list) +{ + struct xfs_extent_busy_tree *eb = xg->xg_busy_extents; + struct xfs_extent_busy *new; + struct xfs_extent_busy *busyp; + struct rb_node **rbp; + struct rb_node *parent = NULL; + + new = kzalloc_obj(struct xfs_extent_busy, GFP_KERNEL | __GFP_NOFAIL); + new->group = xfs_group_hold(xg); + new->bno = bno; + new->length = len; + INIT_LIST_HEAD(&new->list); + new->flags = flags; + + /* trace before insert to be able to see failed inserts */ + trace_xfs_extent_busy(xg, bno, len); + + spin_lock(&eb->eb_lock); + rbp = &eb->eb_tree.rb_node; + while (*rbp) { + parent = *rbp; + busyp = rb_entry(parent, struct xfs_extent_busy, rb_node); + + if (new->bno < busyp->bno) { + rbp = &(*rbp)->rb_left; + ASSERT(new->bno + new->length <= busyp->bno); + } else if (new->bno > busyp->bno) { + rbp = &(*rbp)->rb_right; + ASSERT(bno >= busyp->bno + busyp->length); + } else { + ASSERT(0); + } + } + + rb_link_node(&new->rb_node, parent, rbp); + rb_insert_color(&new->rb_node, &eb->eb_tree); + + /* always process discard lists in fifo order */ + list_add_tail(&new->list, busy_list); + spin_unlock(&eb->eb_lock); +} + +void +xfs_extent_busy_insert( + struct xfs_trans *tp, + struct xfs_group *xg, + xfs_agblock_t bno, + xfs_extlen_t len, + unsigned int flags) +{ + xfs_extent_busy_insert_list(xg, bno, len, flags, &tp->t_busy); +} + +void +xfs_extent_busy_insert_discard( + struct xfs_group *xg, + xfs_agblock_t bno, + xfs_extlen_t len, + struct list_head *busy_list) +{ + xfs_extent_busy_insert_list(xg, bno, len, XFS_EXTENT_BUSY_DISCARDED, + busy_list); +} + +/* + * Search for a busy extent within the range of the extent we are about to + * allocate. You need to be holding the busy extent tree lock when calling + * xfs_extent_busy_search(). This function returns 0 for no overlapping busy + * extent, -1 for an overlapping but not exact busy extent, and 1 for an exact + * match. This is done so that a non-zero return indicates an overlap that + * will require a synchronous transaction, but it can still be + * used to distinguish between a partial or exact match. + */ +int +xfs_extent_busy_search( + struct xfs_group *xg, + xfs_agblock_t bno, + xfs_extlen_t len) +{ + struct xfs_extent_busy_tree *eb = xg->xg_busy_extents; + struct rb_node *rbp; + struct xfs_extent_busy *busyp; + int match = 0; + + /* find closest start bno overlap */ + spin_lock(&eb->eb_lock); + rbp = eb->eb_tree.rb_node; + while (rbp) { + busyp = rb_entry(rbp, struct xfs_extent_busy, rb_node); + if (bno < busyp->bno) { + /* may overlap, but exact start block is lower */ + if (bno + len > busyp->bno) + match = -1; + rbp = rbp->rb_left; + } else if (bno > busyp->bno) { + /* may overlap, but exact start block is higher */ + if (bno < busyp->bno + busyp->length) + match = -1; + rbp = rbp->rb_right; + } else { + /* bno matches busyp, length determines exact match */ + match = (busyp->length == len) ? 1 : -1; + break; + } + } + spin_unlock(&eb->eb_lock); + return match; +} + +/* + * The found free extent [fbno, fend] overlaps part or all of the given busy + * extent. If the overlap covers the beginning, the end, or all of the busy + * extent, the overlapping portion can be made unbusy and used for the + * allocation. We can't split a busy extent because we can't modify a + * transaction/CIL context busy list, but we can update an entry's block + * number or length. + * + * Returns true if the extent can safely be reused, or false if the search + * needs to be restarted. + */ +STATIC bool +xfs_extent_busy_update_extent( + struct xfs_group *xg, + struct xfs_extent_busy *busyp, + xfs_agblock_t fbno, + xfs_extlen_t flen, + bool userdata) + __releases(&eb->eb_lock) + __acquires(&eb->eb_lock) +{ + struct xfs_extent_busy_tree *eb = xg->xg_busy_extents; + xfs_agblock_t fend = fbno + flen; + xfs_agblock_t bbno = busyp->bno; + xfs_agblock_t bend = bbno + busyp->length; + + /* + * This extent is currently being discarded. Give the thread + * performing the discard a chance to mark the extent unbusy + * and retry. + */ + if (busyp->flags & XFS_EXTENT_BUSY_DISCARDED) { + spin_unlock(&eb->eb_lock); + delay(1); + spin_lock(&eb->eb_lock); + return false; + } + + /* + * If there is a busy extent overlapping a user allocation, we have + * no choice but to force the log and retry the search. + * + * Fortunately this does not happen during normal operation, but + * only if the filesystem is very low on space and has to dip into + * the AGFL for normal allocations. + */ + if (userdata) + goto out_force_log; + + if (bbno < fbno && bend > fend) { + /* + * Case 1: + * bbno bend + * +BBBBBBBBBBBBBBBBB+ + * +---------+ + * fbno fend + */ + + /* + * We would have to split the busy extent to be able to track + * it correct, which we cannot do because we would have to + * modify the list of busy extents attached to the transaction + * or CIL context, which is immutable. + * + * Force out the log to clear the busy extent and retry the + * search. + */ + goto out_force_log; + } else if (bbno >= fbno && bend <= fend) { + /* + * Case 2: + * bbno bend + * +BBBBBBBBBBBBBBBBB+ + * +-----------------+ + * fbno fend + * + * Case 3: + * bbno bend + * +BBBBBBBBBBBBBBBBB+ + * +--------------------------+ + * fbno fend + * + * Case 4: + * bbno bend + * +BBBBBBBBBBBBBBBBB+ + * +--------------------------+ + * fbno fend + * + * Case 5: + * bbno bend + * +BBBBBBBBBBBBBBBBB+ + * +-----------------------------------+ + * fbno fend + * + */ + + /* + * The busy extent is fully covered by the extent we are + * allocating, and can simply be removed from the rbtree. + * However we cannot remove it from the immutable list + * tracking busy extents in the transaction or CIL context, + * so set the length to zero to mark it invalid. + * + * We also need to restart the busy extent search from the + * tree root, because erasing the node can rearrange the + * tree topology. + */ + rb_erase(&busyp->rb_node, &eb->eb_tree); + busyp->length = 0; + return false; + } else if (fend < bend) { + /* + * Case 6: + * bbno bend + * +BBBBBBBBBBBBBBBBB+ + * +---------+ + * fbno fend + * + * Case 7: + * bbno bend + * +BBBBBBBBBBBBBBBBB+ + * +------------------+ + * fbno fend + * + */ + busyp->bno = fend; + busyp->length = bend - fend; + } else if (bbno < fbno) { + /* + * Case 8: + * bbno bend + * +BBBBBBBBBBBBBBBBB+ + * +-------------+ + * fbno fend + * + * Case 9: + * bbno bend + * +BBBBBBBBBBBBBBBBB+ + * +----------------------+ + * fbno fend + */ + busyp->length = fbno - busyp->bno; + } else { + ASSERT(0); + } + + trace_xfs_extent_busy_reuse(xg, fbno, flen); + return true; + +out_force_log: + spin_unlock(&eb->eb_lock); + xfs_log_force(xg->xg_mount, XFS_LOG_SYNC); + trace_xfs_extent_busy_force(xg, fbno, flen); + spin_lock(&eb->eb_lock); + return false; +} + +/* + * For a given extent [fbno, flen], make sure we can reuse it safely. + */ +void +xfs_extent_busy_reuse( + struct xfs_group *xg, + xfs_agblock_t fbno, + xfs_extlen_t flen, + bool userdata) +{ + struct xfs_extent_busy_tree *eb = xg->xg_busy_extents; + struct rb_node *rbp; + + ASSERT(flen > 0); + spin_lock(&eb->eb_lock); +restart: + rbp = eb->eb_tree.rb_node; + while (rbp) { + struct xfs_extent_busy *busyp = + rb_entry(rbp, struct xfs_extent_busy, rb_node); + xfs_agblock_t bbno = busyp->bno; + xfs_agblock_t bend = bbno + busyp->length; + + if (fbno + flen <= bbno) { + rbp = rbp->rb_left; + continue; + } else if (fbno >= bend) { + rbp = rbp->rb_right; + continue; + } + + if (!xfs_extent_busy_update_extent(xg, busyp, fbno, flen, + userdata)) + goto restart; + } + spin_unlock(&eb->eb_lock); +} + +/* + * For a given extent [fbno, flen], search the busy extent list to find a + * subset of the extent that is not busy. If *rlen is smaller than + * args->minlen no suitable extent could be found, and the higher level + * code needs to force out the log and retry the allocation. + * + * Return the current busy generation for the group if the extent is busy. This + * value can be used to wait for at least one of the currently busy extents + * to be cleared. Note that the busy list is not guaranteed to be empty after + * the gen is woken. The state of a specific extent must always be confirmed + * with another call to xfs_extent_busy_trim() before it can be used. + */ +bool +xfs_extent_busy_trim( + struct xfs_group *xg, + xfs_extlen_t minlen, + xfs_extlen_t maxlen, + xfs_agblock_t *bno, + xfs_extlen_t *len, + unsigned *busy_gen) +{ + struct xfs_extent_busy_tree *eb = xg->xg_busy_extents; + xfs_agblock_t fbno; + xfs_extlen_t flen; + struct rb_node *rbp; + bool ret = false; + + ASSERT(*len > 0); + + spin_lock(&eb->eb_lock); + fbno = *bno; + flen = *len; + rbp = eb->eb_tree.rb_node; + while (rbp && flen >= minlen) { + struct xfs_extent_busy *busyp = + rb_entry(rbp, struct xfs_extent_busy, rb_node); + xfs_agblock_t fend = fbno + flen; + xfs_agblock_t bbno = busyp->bno; + xfs_agblock_t bend = bbno + busyp->length; + + if (fend <= bbno) { + rbp = rbp->rb_left; + continue; + } else if (fbno >= bend) { + rbp = rbp->rb_right; + continue; + } + + if (bbno <= fbno) { + /* start overlap */ + + /* + * Case 1: + * bbno bend + * +BBBBBBBBBBBBBBBBB+ + * +---------+ + * fbno fend + * + * Case 2: + * bbno bend + * +BBBBBBBBBBBBBBBBB+ + * +-------------+ + * fbno fend + * + * Case 3: + * bbno bend + * +BBBBBBBBBBBBBBBBB+ + * +-------------+ + * fbno fend + * + * Case 4: + * bbno bend + * +BBBBBBBBBBBBBBBBB+ + * +-----------------+ + * fbno fend + * + * No unbusy region in extent, return failure. + */ + if (fend <= bend) + goto fail; + + /* + * Case 5: + * bbno bend + * +BBBBBBBBBBBBBBBBB+ + * +----------------------+ + * fbno fend + * + * Case 6: + * bbno bend + * +BBBBBBBBBBBBBBBBB+ + * +--------------------------+ + * fbno fend + * + * Needs to be trimmed to: + * +-------+ + * fbno fend + */ + fbno = bend; + } else if (bend >= fend) { + /* end overlap */ + + /* + * Case 7: + * bbno bend + * +BBBBBBBBBBBBBBBBB+ + * +------------------+ + * fbno fend + * + * Case 8: + * bbno bend + * +BBBBBBBBBBBBBBBBB+ + * +--------------------------+ + * fbno fend + * + * Needs to be trimmed to: + * +-------+ + * fbno fend + */ + fend = bbno; + } else { + /* middle overlap */ + + /* + * Case 9: + * bbno bend + * +BBBBBBBBBBBBBBBBB+ + * +-----------------------------------+ + * fbno fend + * + * Can be trimmed to: + * +-------+ OR +-------+ + * fbno fend fbno fend + * + * Backward allocation leads to significant + * fragmentation of directories, which degrades + * directory performance, therefore we always want to + * choose the option that produces forward allocation + * patterns. + * Preferring the lower bno extent will make the next + * request use "fend" as the start of the next + * allocation; if the segment is no longer busy at + * that point, we'll get a contiguous allocation, but + * even if it is still busy, we will get a forward + * allocation. + * We try to avoid choosing the segment at "bend", + * because that can lead to the next allocation + * taking the segment at "fbno", which would be a + * backward allocation. We only use the segment at + * "fbno" if it is much larger than the current + * requested size, because in that case there's a + * good chance subsequent allocations will be + * contiguous. + */ + if (bbno - fbno >= maxlen) { + /* left candidate fits perfect */ + fend = bbno; + } else if (fend - bend >= maxlen * 4) { + /* right candidate has enough free space */ + fbno = bend; + } else if (bbno - fbno >= minlen) { + /* left candidate fits minimum requirement */ + fend = bbno; + } else { + goto fail; + } + } + + flen = fend - fbno; + } +out: + + if (fbno != *bno || flen != *len) { + trace_xfs_extent_busy_trim(xg, *bno, *len, fbno, flen); + *bno = fbno; + *len = flen; + *busy_gen = eb->eb_gen; + ret = true; + } + spin_unlock(&eb->eb_lock); + return ret; +fail: + /* + * Return a zero extent length as failure indications. All callers + * re-check if the trimmed extent satisfies the minlen requirement. + */ + flen = 0; + goto out; +} + +static bool +xfs_extent_busy_clear_one( + struct xfs_extent_busy *busyp, + bool do_discard) +{ + struct xfs_extent_busy_tree *eb = busyp->group->xg_busy_extents; + + if (busyp->length) { + if (do_discard && + !(busyp->flags & XFS_EXTENT_BUSY_SKIP_DISCARD)) { + busyp->flags = XFS_EXTENT_BUSY_DISCARDED; + return false; + } + trace_xfs_extent_busy_clear(busyp->group, busyp->bno, + busyp->length); + rb_erase(&busyp->rb_node, &eb->eb_tree); + } + + list_del_init(&busyp->list); + xfs_group_put(busyp->group); + kfree(busyp); + return true; +} + +/* + * Remove all extents on the passed in list from the busy extents tree. + * If do_discard is set skip extents that need to be discarded, and mark + * these as undergoing a discard operation instead. + */ +void +xfs_extent_busy_clear( + struct list_head *list, + bool do_discard) +{ + struct xfs_extent_busy *busyp, *next; + + busyp = list_first_entry_or_null(list, typeof(*busyp), list); + if (!busyp) + return; + + do { + struct xfs_group *xg = xfs_group_hold(busyp->group); + struct xfs_extent_busy_tree *eb = xg->xg_busy_extents; + bool wakeup = false; + + spin_lock(&eb->eb_lock); + do { + next = list_next_entry(busyp, list); + if (xfs_extent_busy_clear_one(busyp, do_discard)) + wakeup = true; + busyp = next; + } while (!list_entry_is_head(busyp, list, list) && + busyp->group == xg); + + if (wakeup) { + eb->eb_gen++; + wake_up_all(&eb->eb_wait); + } + spin_unlock(&eb->eb_lock); + xfs_group_put(xg); + } while (!list_entry_is_head(busyp, list, list)); +} + +/* + * Flush out all busy extents for this group. + * + * If the current transaction is holding busy extents, the caller may not want + * to wait for committed busy extents to resolve. If we are being told just to + * try a flush or progress has been made since we last skipped a busy extent, + * return immediately to allow the caller to try again. + * + * If we are freeing extents, we might actually be holding the only free extents + * in the transaction busy list and the log force won't resolve that situation. + * In this case, we must return -EAGAIN to avoid a deadlock by informing the + * caller it needs to commit the busy extents it holds before retrying the + * extent free operation. + */ +int +xfs_extent_busy_flush( + struct xfs_trans *tp, + struct xfs_group *xg, + unsigned busy_gen, + uint32_t alloc_flags) +{ + struct xfs_extent_busy_tree *eb = xg->xg_busy_extents; + DEFINE_WAIT (wait); + int error; + + error = xfs_log_force(tp->t_mountp, XFS_LOG_SYNC); + if (error) + return error; + + /* Avoid deadlocks on uncommitted busy extents. */ + if (!list_empty(&tp->t_busy)) { + if (alloc_flags & XFS_ALLOC_FLAG_TRYFLUSH) + return 0; + + if (busy_gen != READ_ONCE(eb->eb_gen)) + return 0; + + if (alloc_flags & XFS_ALLOC_FLAG_FREEING) + return -EAGAIN; + } + + /* Wait for committed busy extents to resolve. */ + do { + prepare_to_wait(&eb->eb_wait, &wait, TASK_KILLABLE); + if (busy_gen != READ_ONCE(eb->eb_gen)) + break; + schedule(); + } while (1); + + finish_wait(&eb->eb_wait, &wait); + return 0; +} + +static void +xfs_extent_busy_wait_group( + struct xfs_group *xg) +{ + DEFINE_WAIT (wait); + struct xfs_extent_busy_tree *eb = xg->xg_busy_extents; + + do { + prepare_to_wait(&eb->eb_wait, &wait, TASK_KILLABLE); + if (RB_EMPTY_ROOT(&eb->eb_tree)) + break; + schedule(); + } while (1); + finish_wait(&eb->eb_wait, &wait); +} + +void +xfs_extent_busy_wait_all( + struct xfs_mount *mp) +{ + struct xfs_perag *pag = NULL; + struct xfs_rtgroup *rtg = NULL; + + while ((pag = xfs_perag_next(mp, pag))) + xfs_extent_busy_wait_group(pag_group(pag)); + + if (xfs_has_rtgroups(mp) && !xfs_has_zoned(mp)) + while ((rtg = xfs_rtgroup_next(mp, rtg))) + xfs_extent_busy_wait_group(rtg_group(rtg)); +} + +/* + * Callback for list_sort to sort busy extents by the group they reside in. + */ +int +xfs_extent_busy_ag_cmp( + void *priv, + const struct list_head *l1, + const struct list_head *l2) +{ + struct xfs_extent_busy *b1 = + container_of(l1, struct xfs_extent_busy, list); + struct xfs_extent_busy *b2 = + container_of(l2, struct xfs_extent_busy, list); + s32 diff; + + diff = cmp_int(b1->group->xg_gno, b2->group->xg_gno); + if (!diff) + diff = cmp_int(b1->bno, b2->bno); + return diff; +} + +/* Are there any busy extents in this group? */ +bool +xfs_extent_busy_list_empty( + struct xfs_group *xg, + unsigned *busy_gen) +{ + struct xfs_extent_busy_tree *eb = xg->xg_busy_extents; + bool res; + + spin_lock(&eb->eb_lock); + res = RB_EMPTY_ROOT(&eb->eb_tree); + *busy_gen = READ_ONCE(eb->eb_gen); + spin_unlock(&eb->eb_lock); + return res; +} + +struct xfs_extent_busy_tree * +xfs_extent_busy_alloc(void) +{ + struct xfs_extent_busy_tree *eb; + + eb = kzalloc_obj(*eb); + if (!eb) + return NULL; + spin_lock_init(&eb->eb_lock); + init_waitqueue_head(&eb->eb_wait); + eb->eb_tree = RB_ROOT; + return eb; +} diff --git a/fs/xfs/xfs_extent_busy.h b/fs/xfs/xfs_extent_busy.h new file mode 100644 index 000000000..3e6e019b6 --- /dev/null +++ b/fs/xfs/xfs_extent_busy.h @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc. + * Copyright (c) 2010 David Chinner. + * Copyright (c) 2011 Christoph Hellwig. + * All Rights Reserved. + */ +#ifndef __XFS_EXTENT_BUSY_H__ +#define __XFS_EXTENT_BUSY_H__ + +struct xfs_group; +struct xfs_mount; +struct xfs_trans; + +/* + * Busy block/extent entry. Indexed by a rbtree in the group to mark blocks + * that have been freed but whose transactions aren't committed to disk yet. + */ +struct xfs_extent_busy { + struct rb_node rb_node; /* group by-bno indexed search tree */ + struct list_head list; /* transaction busy extent list */ + struct xfs_group *group; + xfs_agblock_t bno; + xfs_extlen_t length; + unsigned int flags; +#define XFS_EXTENT_BUSY_DISCARDED 0x01 /* undergoing a discard op. */ +#define XFS_EXTENT_BUSY_SKIP_DISCARD 0x02 /* do not discard */ +}; + +/* + * List used to track groups of related busy extents all the way through + * to discard completion. + */ +struct xfs_busy_extents { + struct list_head extent_list; + struct work_struct endio_work; + + /* + * Owner is the object containing the struct xfs_busy_extents to free + * once the busy extents have been processed. If only the + * xfs_busy_extents object needs freeing, then point this at itself. + */ + void *owner; +}; + +void xfs_extent_busy_insert(struct xfs_trans *tp, struct xfs_group *xg, + xfs_agblock_t bno, xfs_extlen_t len, unsigned int flags); +void xfs_extent_busy_insert_discard(struct xfs_group *xg, xfs_agblock_t bno, + xfs_extlen_t len, struct list_head *busy_list); +void xfs_extent_busy_clear(struct list_head *list, bool do_discard); +int xfs_extent_busy_search(struct xfs_group *xg, xfs_agblock_t bno, + xfs_extlen_t len); +void xfs_extent_busy_reuse(struct xfs_group *xg, xfs_agblock_t fbno, + xfs_extlen_t flen, bool userdata); +bool xfs_extent_busy_trim(struct xfs_group *xg, xfs_extlen_t minlen, + xfs_extlen_t maxlen, xfs_agblock_t *bno, xfs_extlen_t *len, + unsigned *busy_gen); +int xfs_extent_busy_flush(struct xfs_trans *tp, struct xfs_group *xg, + unsigned busy_gen, uint32_t alloc_flags); +void xfs_extent_busy_wait_all(struct xfs_mount *mp); +bool xfs_extent_busy_list_empty(struct xfs_group *xg, unsigned int *busy_gen); +struct xfs_extent_busy_tree *xfs_extent_busy_alloc(void); + +int xfs_extent_busy_ag_cmp(void *priv, const struct list_head *a, + const struct list_head *b); +static inline void xfs_extent_busy_sort(struct list_head *list) +{ + list_sort(NULL, list, xfs_extent_busy_ag_cmp); +} + +/* + * Zoned RTGs don't need to track busy extents, as the actual block freeing only + * happens by a zone reset, which forces out all transactions that touched the + * to be reset zone first. + */ +#define xfs_group_has_extent_busy(mp, type) \ + ((type) == XG_TYPE_AG || !xfs_has_zoned((mp))) + +#endif /* __XFS_EXTENT_BUSY_H__ */ diff --git a/fs/xfs/xfs_extfree_item.c b/fs/xfs/xfs_extfree_item.c new file mode 100644 index 000000000..2266d56e3 --- /dev/null +++ b/fs/xfs/xfs_extfree_item.c @@ -0,0 +1,1029 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2001,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_shared.h" +#include "xfs_mount.h" +#include "xfs_ag.h" +#include "xfs_defer.h" +#include "xfs_trans.h" +#include "xfs_trans_priv.h" +#include "xfs_extfree_item.h" +#include "xfs_log.h" +#include "xfs_btree.h" +#include "xfs_rmap.h" +#include "xfs_alloc.h" +#include "xfs_bmap.h" +#include "xfs_trace.h" +#include "xfs_error.h" +#include "xfs_log_priv.h" +#include "xfs_log_recover.h" +#include "xfs_rtalloc.h" +#include "xfs_inode.h" +#include "xfs_rtbitmap.h" +#include "xfs_rtgroup.h" +#include "xfs_zone_alloc.h" + +struct kmem_cache *xfs_efi_cache; +struct kmem_cache *xfs_efd_cache; + +static const struct xfs_item_ops xfs_efi_item_ops; + +static inline struct xfs_efi_log_item *EFI_ITEM(struct xfs_log_item *lip) +{ + return container_of(lip, struct xfs_efi_log_item, efi_item); +} + +STATIC void +xfs_efi_item_free( + struct xfs_efi_log_item *efip) +{ + kvfree(efip->efi_item.li_lv_shadow); + if (efip->efi_format.efi_nextents > XFS_EFI_MAX_FAST_EXTENTS) + kfree(efip); + else + kmem_cache_free(xfs_efi_cache, efip); +} + +/* + * Freeing the efi requires that we remove it from the AIL if it has already + * been placed there. However, the EFI may not yet have been placed in the AIL + * when called by xfs_efi_release() from EFD processing due to the ordering of + * committed vs unpin operations in bulk insert operations. Hence the reference + * count to ensure only the last caller frees the EFI. + */ +STATIC void +xfs_efi_release( + struct xfs_efi_log_item *efip) +{ + ASSERT(atomic_read(&efip->efi_refcount) > 0); + if (!atomic_dec_and_test(&efip->efi_refcount)) + return; + + xfs_trans_ail_delete(&efip->efi_item, 0); + xfs_efi_item_free(efip); +} + +STATIC void +xfs_efi_item_size( + struct xfs_log_item *lip, + int *nvecs, + int *nbytes) +{ + struct xfs_efi_log_item *efip = EFI_ITEM(lip); + + *nvecs += 1; + *nbytes += xfs_efi_log_format_sizeof(efip->efi_format.efi_nextents); +} + +unsigned int xfs_efi_log_space(unsigned int nr) +{ + return xlog_item_space(1, xfs_efi_log_format_sizeof(nr)); +} + +/* + * This is called to fill in the vector of log iovecs for the + * given efi log item. We use only 1 iovec, and we point that + * at the efi_log_format structure embedded in the efi item. + * It is at this point that we assert that all of the extent + * slots in the efi item have been filled. + */ +STATIC void +xfs_efi_item_format( + struct xfs_log_item *lip, + struct xlog_format_buf *lfb) +{ + struct xfs_efi_log_item *efip = EFI_ITEM(lip); + + ASSERT(atomic_read(&efip->efi_next_extent) == + efip->efi_format.efi_nextents); + ASSERT(lip->li_type == XFS_LI_EFI || lip->li_type == XFS_LI_EFI_RT); + + efip->efi_format.efi_type = lip->li_type; + efip->efi_format.efi_size = 1; + + xlog_format_copy(lfb, XLOG_REG_TYPE_EFI_FORMAT, &efip->efi_format, + xfs_efi_log_format_sizeof(efip->efi_format.efi_nextents)); +} + +/* + * The unpin operation is the last place an EFI is manipulated in the log. It is + * either inserted in the AIL or aborted in the event of a log I/O error. In + * either case, the EFI transaction has been successfully committed to make it + * this far. Therefore, we expect whoever committed the EFI to either construct + * and commit the EFD or drop the EFD's reference in the event of error. Simply + * drop the log's EFI reference now that the log is done with it. + */ +STATIC void +xfs_efi_item_unpin( + struct xfs_log_item *lip, + int remove) +{ + struct xfs_efi_log_item *efip = EFI_ITEM(lip); + xfs_efi_release(efip); +} + +/* + * The EFI has been either committed or aborted if the transaction has been + * cancelled. If the transaction was cancelled, an EFD isn't going to be + * constructed and thus we free the EFI here directly. + */ +STATIC void +xfs_efi_item_release( + struct xfs_log_item *lip) +{ + xfs_efi_release(EFI_ITEM(lip)); +} + +/* + * Allocate and initialize an efi item with the given number of extents. + */ +STATIC struct xfs_efi_log_item * +xfs_efi_init( + struct xfs_mount *mp, + unsigned short item_type, + uint nextents) +{ + struct xfs_efi_log_item *efip; + + ASSERT(item_type == XFS_LI_EFI || item_type == XFS_LI_EFI_RT); + ASSERT(nextents > 0); + + if (nextents > XFS_EFI_MAX_FAST_EXTENTS) { + efip = kzalloc(xfs_efi_log_item_sizeof(nextents), + GFP_KERNEL | __GFP_NOFAIL); + } else { + efip = kmem_cache_zalloc(xfs_efi_cache, + GFP_KERNEL | __GFP_NOFAIL); + } + + xfs_log_item_init(mp, &efip->efi_item, item_type, &xfs_efi_item_ops); + efip->efi_format.efi_nextents = nextents; + efip->efi_format.efi_id = (uintptr_t)(void *)efip; + atomic_set(&efip->efi_next_extent, 0); + atomic_set(&efip->efi_refcount, 2); + + return efip; +} + +/* + * Copy an EFI format buffer from the given buf, and into the destination + * EFI format structure. + * The given buffer can be in 32 bit or 64 bit form (which has different padding), + * one of which will be the native format for this kernel. + * It will handle the conversion of formats if necessary. + */ +STATIC int +xfs_efi_copy_format( + struct kvec *buf, + struct xfs_efi_log_format *dst_efi_fmt) +{ + struct xfs_efi_log_format *src_efi_fmt = buf->iov_base; + uint len, len32, len64, i; + + len = xfs_efi_log_format_sizeof(src_efi_fmt->efi_nextents); + len32 = xfs_efi_log_format32_sizeof(src_efi_fmt->efi_nextents); + len64 = xfs_efi_log_format64_sizeof(src_efi_fmt->efi_nextents); + + if (buf->iov_len == len) { + memcpy(dst_efi_fmt, src_efi_fmt, + offsetof(struct xfs_efi_log_format, efi_extents)); + for (i = 0; i < src_efi_fmt->efi_nextents; i++) + memcpy(&dst_efi_fmt->efi_extents[i], + &src_efi_fmt->efi_extents[i], + sizeof(struct xfs_extent)); + return 0; + } else if (buf->iov_len == len32) { + struct xfs_efi_log_format_32 *src_efi_fmt_32 = buf->iov_base; + + dst_efi_fmt->efi_type = src_efi_fmt_32->efi_type; + dst_efi_fmt->efi_size = src_efi_fmt_32->efi_size; + dst_efi_fmt->efi_nextents = src_efi_fmt_32->efi_nextents; + dst_efi_fmt->efi_id = src_efi_fmt_32->efi_id; + for (i = 0; i < dst_efi_fmt->efi_nextents; i++) { + dst_efi_fmt->efi_extents[i].ext_start = + src_efi_fmt_32->efi_extents[i].ext_start; + dst_efi_fmt->efi_extents[i].ext_len = + src_efi_fmt_32->efi_extents[i].ext_len; + } + return 0; + } else if (buf->iov_len == len64) { + struct xfs_efi_log_format_64 *src_efi_fmt_64 = buf->iov_base; + + dst_efi_fmt->efi_type = src_efi_fmt_64->efi_type; + dst_efi_fmt->efi_size = src_efi_fmt_64->efi_size; + dst_efi_fmt->efi_nextents = src_efi_fmt_64->efi_nextents; + dst_efi_fmt->efi_id = src_efi_fmt_64->efi_id; + for (i = 0; i < dst_efi_fmt->efi_nextents; i++) { + dst_efi_fmt->efi_extents[i].ext_start = + src_efi_fmt_64->efi_extents[i].ext_start; + dst_efi_fmt->efi_extents[i].ext_len = + src_efi_fmt_64->efi_extents[i].ext_len; + } + return 0; + } + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, NULL, buf->iov_base, + buf->iov_len); + return -EFSCORRUPTED; +} + +static inline struct xfs_efd_log_item *EFD_ITEM(struct xfs_log_item *lip) +{ + return container_of(lip, struct xfs_efd_log_item, efd_item); +} + +STATIC void +xfs_efd_item_free(struct xfs_efd_log_item *efdp) +{ + kvfree(efdp->efd_item.li_lv_shadow); + if (efdp->efd_format.efd_nextents > XFS_EFD_MAX_FAST_EXTENTS) + kfree(efdp); + else + kmem_cache_free(xfs_efd_cache, efdp); +} + +STATIC void +xfs_efd_item_size( + struct xfs_log_item *lip, + int *nvecs, + int *nbytes) +{ + struct xfs_efd_log_item *efdp = EFD_ITEM(lip); + + *nvecs += 1; + *nbytes += xfs_efd_log_format_sizeof(efdp->efd_format.efd_nextents); +} + +unsigned int xfs_efd_log_space(unsigned int nr) +{ + return xlog_item_space(1, xfs_efd_log_format_sizeof(nr)); +} + +/* + * This is called to fill in the vector of log iovecs for the + * given efd log item. We use only 1 iovec, and we point that + * at the efd_log_format structure embedded in the efd item. + * It is at this point that we assert that all of the extent + * slots in the efd item have been filled. + */ +STATIC void +xfs_efd_item_format( + struct xfs_log_item *lip, + struct xlog_format_buf *lfb) +{ + struct xfs_efd_log_item *efdp = EFD_ITEM(lip); + + ASSERT(efdp->efd_next_extent == efdp->efd_format.efd_nextents); + ASSERT(lip->li_type == XFS_LI_EFD || lip->li_type == XFS_LI_EFD_RT); + + efdp->efd_format.efd_type = lip->li_type; + efdp->efd_format.efd_size = 1; + + xlog_format_copy(lfb, XLOG_REG_TYPE_EFD_FORMAT, &efdp->efd_format, + xfs_efd_log_format_sizeof(efdp->efd_format.efd_nextents)); +} + +/* + * The EFD is either committed or aborted if the transaction is cancelled. If + * the transaction is cancelled, drop our reference to the EFI and free the EFD. + */ +STATIC void +xfs_efd_item_release( + struct xfs_log_item *lip) +{ + struct xfs_efd_log_item *efdp = EFD_ITEM(lip); + + xfs_efi_release(efdp->efd_efip); + xfs_efd_item_free(efdp); +} + +static struct xfs_log_item * +xfs_efd_item_intent( + struct xfs_log_item *lip) +{ + return &EFD_ITEM(lip)->efd_efip->efi_item; +} + +static const struct xfs_item_ops xfs_efd_item_ops = { + .flags = XFS_ITEM_RELEASE_WHEN_COMMITTED | + XFS_ITEM_INTENT_DONE, + .iop_size = xfs_efd_item_size, + .iop_format = xfs_efd_item_format, + .iop_release = xfs_efd_item_release, + .iop_intent = xfs_efd_item_intent, +}; + +static inline struct xfs_extent_free_item *xefi_entry(const struct list_head *e) +{ + return list_entry(e, struct xfs_extent_free_item, xefi_list); +} + +static inline bool +xfs_efi_item_isrt(const struct xfs_log_item *lip) +{ + ASSERT(lip->li_type == XFS_LI_EFI || lip->li_type == XFS_LI_EFI_RT); + + return lip->li_type == XFS_LI_EFI_RT; +} + +/* + * Fill the EFD with all extents from the EFI when we need to roll the + * transaction and continue with a new EFI. + * + * This simply copies all the extents in the EFI to the EFD rather than make + * assumptions about which extents in the EFI have already been processed. We + * currently keep the xefi list in the same order as the EFI extent list, but + * that may not always be the case. Copying everything avoids leaving a landmine + * were we fail to cancel all the extents in an EFI if the xefi list is + * processed in a different order to the extents in the EFI. + */ +static void +xfs_efd_from_efi( + struct xfs_efd_log_item *efdp) +{ + struct xfs_efi_log_item *efip = efdp->efd_efip; + uint i; + + ASSERT(efip->efi_format.efi_nextents > 0); + ASSERT(efdp->efd_next_extent < efip->efi_format.efi_nextents); + + for (i = 0; i < efip->efi_format.efi_nextents; i++) { + efdp->efd_format.efd_extents[i] = + efip->efi_format.efi_extents[i]; + } + efdp->efd_next_extent = efip->efi_format.efi_nextents; +} + +static void +xfs_efd_add_extent( + struct xfs_efd_log_item *efdp, + struct xfs_extent_free_item *xefi) +{ + struct xfs_extent *extp; + + ASSERT(efdp->efd_next_extent < efdp->efd_format.efd_nextents); + + extp = &efdp->efd_format.efd_extents[efdp->efd_next_extent]; + extp->ext_start = xefi->xefi_startblock; + extp->ext_len = xefi->xefi_blockcount; + + efdp->efd_next_extent++; +} + +/* Sort bmap items by AG. */ +static int +xfs_extent_free_diff_items( + void *priv, + const struct list_head *a, + const struct list_head *b) +{ + struct xfs_extent_free_item *ra = xefi_entry(a); + struct xfs_extent_free_item *rb = xefi_entry(b); + + return cmp_int(ra->xefi_group->xg_gno, rb->xefi_group->xg_gno); +} + +/* Log a free extent to the intent item. */ +STATIC void +xfs_extent_free_log_item( + struct xfs_trans *tp, + struct xfs_efi_log_item *efip, + struct xfs_extent_free_item *xefi) +{ + uint next_extent; + struct xfs_extent *extp; + + /* + * atomic_inc_return gives us the value after the increment; + * we want to use it as an array index so we need to subtract 1 from + * it. + */ + next_extent = atomic_inc_return(&efip->efi_next_extent) - 1; + ASSERT(next_extent < efip->efi_format.efi_nextents); + extp = &efip->efi_format.efi_extents[next_extent]; + extp->ext_start = xefi->xefi_startblock; + extp->ext_len = xefi->xefi_blockcount; +} + +static struct xfs_log_item * +__xfs_extent_free_create_intent( + struct xfs_trans *tp, + struct list_head *items, + unsigned int count, + bool sort, + unsigned short item_type) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_efi_log_item *efip; + struct xfs_extent_free_item *xefi; + + ASSERT(count > 0); + + efip = xfs_efi_init(mp, item_type, count); + if (sort) + list_sort(mp, items, xfs_extent_free_diff_items); + list_for_each_entry(xefi, items, xefi_list) + xfs_extent_free_log_item(tp, efip, xefi); + return &efip->efi_item; +} + +static struct xfs_log_item * +xfs_extent_free_create_intent( + struct xfs_trans *tp, + struct list_head *items, + unsigned int count, + bool sort) +{ + return __xfs_extent_free_create_intent(tp, items, count, sort, + XFS_LI_EFI); +} + +static inline unsigned short +xfs_efd_type_from_efi(const struct xfs_efi_log_item *efip) +{ + return xfs_efi_item_isrt(&efip->efi_item) ? XFS_LI_EFD_RT : XFS_LI_EFD; +} + +/* Get an EFD so we can process all the free extents. */ +static struct xfs_log_item * +xfs_extent_free_create_done( + struct xfs_trans *tp, + struct xfs_log_item *intent, + unsigned int count) +{ + struct xfs_efi_log_item *efip = EFI_ITEM(intent); + struct xfs_efd_log_item *efdp; + + ASSERT(count > 0); + + if (count > XFS_EFD_MAX_FAST_EXTENTS) { + efdp = kzalloc(xfs_efd_log_item_sizeof(count), + GFP_KERNEL | __GFP_NOFAIL); + } else { + efdp = kmem_cache_zalloc(xfs_efd_cache, + GFP_KERNEL | __GFP_NOFAIL); + } + + xfs_log_item_init(tp->t_mountp, &efdp->efd_item, + xfs_efd_type_from_efi(efip), &xfs_efd_item_ops); + efdp->efd_efip = efip; + efdp->efd_format.efd_nextents = count; + efdp->efd_format.efd_efi_id = efip->efi_format.efi_id; + + return &efdp->efd_item; +} + +static inline const struct xfs_defer_op_type * +xefi_ops( + struct xfs_extent_free_item *xefi) +{ + if (xfs_efi_is_realtime(xefi)) + return &xfs_rtextent_free_defer_type; + if (xefi->xefi_agresv == XFS_AG_RESV_AGFL) + return &xfs_agfl_free_defer_type; + return &xfs_extent_free_defer_type; +} + +/* Add this deferred EFI to the transaction. */ +void +xfs_extent_free_defer_add( + struct xfs_trans *tp, + struct xfs_extent_free_item *xefi, + struct xfs_defer_pending **dfpp) +{ + struct xfs_mount *mp = tp->t_mountp; + + xefi->xefi_group = xfs_group_intent_get(mp, xefi->xefi_startblock, + xfs_efi_is_realtime(xefi) ? XG_TYPE_RTG : XG_TYPE_AG); + + trace_xfs_extent_free_defer(mp, xefi); + *dfpp = xfs_defer_add(tp, &xefi->xefi_list, xefi_ops(xefi)); +} + +/* Cancel a free extent. */ +STATIC void +xfs_extent_free_cancel_item( + struct list_head *item) +{ + struct xfs_extent_free_item *xefi = xefi_entry(item); + + xfs_group_intent_put(xefi->xefi_group); + kmem_cache_free(xfs_extfree_item_cache, xefi); +} + +/* Process a free extent. */ +STATIC int +xfs_extent_free_finish_item( + struct xfs_trans *tp, + struct xfs_log_item *done, + struct list_head *item, + struct xfs_btree_cur **state) +{ + struct xfs_owner_info oinfo = { }; + struct xfs_extent_free_item *xefi = xefi_entry(item); + struct xfs_efd_log_item *efdp = EFD_ITEM(done); + struct xfs_mount *mp = tp->t_mountp; + xfs_agblock_t agbno; + int error = 0; + + agbno = XFS_FSB_TO_AGBNO(mp, xefi->xefi_startblock); + + oinfo.oi_owner = xefi->xefi_owner; + if (xefi->xefi_flags & XFS_EFI_ATTR_FORK) + oinfo.oi_flags |= XFS_OWNER_INFO_ATTR_FORK; + if (xefi->xefi_flags & XFS_EFI_BMBT_BLOCK) + oinfo.oi_flags |= XFS_OWNER_INFO_BMBT_BLOCK; + + trace_xfs_extent_free_deferred(mp, xefi); + + /* + * If we need a new transaction to make progress, the caller will log a + * new EFI with the current contents. It will also log an EFD to cancel + * the existing EFI, and so we need to copy all the unprocessed extents + * in this EFI to the EFD so this works correctly. + */ + if (!(xefi->xefi_flags & XFS_EFI_CANCELLED)) + error = __xfs_free_extent(tp, to_perag(xefi->xefi_group), agbno, + xefi->xefi_blockcount, &oinfo, xefi->xefi_agresv, + xefi->xefi_flags & XFS_EFI_SKIP_DISCARD); + if (error == -EAGAIN) { + xfs_efd_from_efi(efdp); + return error; + } + + xfs_efd_add_extent(efdp, xefi); + xfs_extent_free_cancel_item(item); + return error; +} + +/* Abort all pending EFIs. */ +STATIC void +xfs_extent_free_abort_intent( + struct xfs_log_item *intent) +{ + xfs_efi_release(EFI_ITEM(intent)); +} + +/* + * AGFL blocks are accounted differently in the reserve pools and are not + * inserted into the busy extent list. + */ +STATIC int +xfs_agfl_free_finish_item( + struct xfs_trans *tp, + struct xfs_log_item *done, + struct list_head *item, + struct xfs_btree_cur **state) +{ + struct xfs_owner_info oinfo = { }; + struct xfs_mount *mp = tp->t_mountp; + struct xfs_efd_log_item *efdp = EFD_ITEM(done); + struct xfs_extent_free_item *xefi = xefi_entry(item); + struct xfs_buf *agbp; + int error; + xfs_agblock_t agbno; + + ASSERT(xefi->xefi_blockcount == 1); + agbno = XFS_FSB_TO_AGBNO(mp, xefi->xefi_startblock); + oinfo.oi_owner = xefi->xefi_owner; + + trace_xfs_agfl_free_deferred(mp, xefi); + + error = xfs_alloc_read_agf(to_perag(xefi->xefi_group), tp, 0, &agbp); + if (!error) + error = xfs_free_ag_extent(tp, agbp, agbno, 1, &oinfo, + XFS_AG_RESV_AGFL); + + xfs_efd_add_extent(efdp, xefi); + xfs_extent_free_cancel_item(&xefi->xefi_list); + return error; +} + +/* Is this recovered EFI ok? */ +static inline bool +xfs_efi_validate_ext( + struct xfs_mount *mp, + bool isrt, + struct xfs_extent *extp) +{ + if (isrt) + return xfs_verify_rtbext(mp, extp->ext_start, extp->ext_len); + + return xfs_verify_fsbext(mp, extp->ext_start, extp->ext_len); +} + +static inline void +xfs_efi_recover_work( + struct xfs_mount *mp, + struct xfs_defer_pending *dfp, + bool isrt, + struct xfs_extent *extp) +{ + struct xfs_extent_free_item *xefi; + + xefi = kmem_cache_zalloc(xfs_extfree_item_cache, + GFP_KERNEL | __GFP_NOFAIL); + xefi->xefi_startblock = extp->ext_start; + xefi->xefi_blockcount = extp->ext_len; + xefi->xefi_agresv = XFS_AG_RESV_NONE; + xefi->xefi_owner = XFS_RMAP_OWN_UNKNOWN; + xefi->xefi_group = xfs_group_intent_get(mp, extp->ext_start, + isrt ? XG_TYPE_RTG : XG_TYPE_AG); + if (isrt) + xefi->xefi_flags |= XFS_EFI_REALTIME; + + xfs_defer_add_item(dfp, &xefi->xefi_list); +} + +/* + * Process an extent free intent item that was recovered from + * the log. We need to free the extents that it describes. + */ +STATIC int +xfs_extent_free_recover_work( + struct xfs_defer_pending *dfp, + struct list_head *capture_list) +{ + struct xfs_trans_res resv; + struct xfs_log_item *lip = dfp->dfp_intent; + struct xfs_efi_log_item *efip = EFI_ITEM(lip); + struct xfs_mount *mp = lip->li_log->l_mp; + struct xfs_trans *tp; + int i; + int error = 0; + bool isrt = xfs_efi_item_isrt(lip); + + /* + * First check the validity of the extents described by the EFI. If + * any are bad, then assume that all are bad and just toss the EFI. + * Mixing RT and non-RT extents in the same EFI item is not allowed. + */ + for (i = 0; i < efip->efi_format.efi_nextents; i++) { + if (!xfs_efi_validate_ext(mp, isrt, + &efip->efi_format.efi_extents[i])) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + &efip->efi_format, + sizeof(efip->efi_format)); + return -EFSCORRUPTED; + } + + xfs_efi_recover_work(mp, dfp, isrt, + &efip->efi_format.efi_extents[i]); + } + + resv = xlog_recover_resv(&M_RES(mp)->tr_itruncate); + error = xfs_trans_alloc(mp, &resv, 0, 0, 0, &tp); + if (error) + return error; + + error = xlog_recover_finish_intent(tp, dfp); + if (error == -EFSCORRUPTED) + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + &efip->efi_format, + sizeof(efip->efi_format)); + if (error) + goto abort_error; + + return xfs_defer_ops_capture_and_commit(tp, capture_list); + +abort_error: + xfs_trans_cancel(tp); + return error; +} + +/* Relog an intent item to push the log tail forward. */ +static struct xfs_log_item * +xfs_extent_free_relog_intent( + struct xfs_trans *tp, + struct xfs_log_item *intent, + struct xfs_log_item *done_item) +{ + struct xfs_efd_log_item *efdp = EFD_ITEM(done_item); + struct xfs_efi_log_item *efip; + struct xfs_extent *extp; + unsigned int count; + + count = EFI_ITEM(intent)->efi_format.efi_nextents; + extp = EFI_ITEM(intent)->efi_format.efi_extents; + + ASSERT(intent->li_type == XFS_LI_EFI || intent->li_type == XFS_LI_EFI_RT); + + efdp->efd_next_extent = count; + memcpy(efdp->efd_format.efd_extents, extp, count * sizeof(*extp)); + + efip = xfs_efi_init(tp->t_mountp, intent->li_type, count); + memcpy(efip->efi_format.efi_extents, extp, count * sizeof(*extp)); + atomic_set(&efip->efi_next_extent, count); + + return &efip->efi_item; +} + +const struct xfs_defer_op_type xfs_extent_free_defer_type = { + .name = "extent_free", + .max_items = XFS_EFI_MAX_FAST_EXTENTS, + .create_intent = xfs_extent_free_create_intent, + .abort_intent = xfs_extent_free_abort_intent, + .create_done = xfs_extent_free_create_done, + .finish_item = xfs_extent_free_finish_item, + .cancel_item = xfs_extent_free_cancel_item, + .recover_work = xfs_extent_free_recover_work, + .relog_intent = xfs_extent_free_relog_intent, +}; + +/* sub-type with special handling for AGFL deferred frees */ +const struct xfs_defer_op_type xfs_agfl_free_defer_type = { + .name = "agfl_free", + .max_items = XFS_EFI_MAX_FAST_EXTENTS, + .create_intent = xfs_extent_free_create_intent, + .abort_intent = xfs_extent_free_abort_intent, + .create_done = xfs_extent_free_create_done, + .finish_item = xfs_agfl_free_finish_item, + .cancel_item = xfs_extent_free_cancel_item, + .recover_work = xfs_extent_free_recover_work, + .relog_intent = xfs_extent_free_relog_intent, +}; + +#ifdef CONFIG_XFS_RT +/* Create a realtime extent freeing */ +static struct xfs_log_item * +xfs_rtextent_free_create_intent( + struct xfs_trans *tp, + struct list_head *items, + unsigned int count, + bool sort) +{ + return __xfs_extent_free_create_intent(tp, items, count, sort, + XFS_LI_EFI_RT); +} + +/* Process a free realtime extent. */ +STATIC int +xfs_rtextent_free_finish_item( + struct xfs_trans *tp, + struct xfs_log_item *done, + struct list_head *item, + struct xfs_btree_cur **state) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_extent_free_item *xefi = xefi_entry(item); + struct xfs_efd_log_item *efdp = EFD_ITEM(done); + struct xfs_rtgroup **rtgp = (struct xfs_rtgroup **)state; + int error = 0; + + trace_xfs_extent_free_deferred(mp, xefi); + + if (xefi->xefi_flags & XFS_EFI_CANCELLED) + goto done; + + if (*rtgp != to_rtg(xefi->xefi_group)) { + unsigned int lock_flags; + + if (xfs_has_zoned(mp)) + lock_flags = XFS_RTGLOCK_RMAP; + else + lock_flags = XFS_RTGLOCK_BITMAP; + + *rtgp = to_rtg(xefi->xefi_group); + xfs_rtgroup_lock(*rtgp, lock_flags); + xfs_rtgroup_trans_join(tp, *rtgp, lock_flags); + } + + if (xfs_has_zoned(mp)) { + error = xfs_zone_free_blocks(tp, *rtgp, xefi->xefi_startblock, + xefi->xefi_blockcount); + } else { + error = xfs_rtfree_blocks(tp, *rtgp, xefi->xefi_startblock, + xefi->xefi_blockcount); + } + + if (error == -EAGAIN) { + xfs_efd_from_efi(efdp); + return error; + } +done: + xfs_efd_add_extent(efdp, xefi); + xfs_extent_free_cancel_item(item); + return error; +} + +const struct xfs_defer_op_type xfs_rtextent_free_defer_type = { + .name = "rtextent_free", + .max_items = XFS_EFI_MAX_FAST_EXTENTS, + .create_intent = xfs_rtextent_free_create_intent, + .abort_intent = xfs_extent_free_abort_intent, + .create_done = xfs_extent_free_create_done, + .finish_item = xfs_rtextent_free_finish_item, + .cancel_item = xfs_extent_free_cancel_item, + .recover_work = xfs_extent_free_recover_work, + .relog_intent = xfs_extent_free_relog_intent, +}; +#else +const struct xfs_defer_op_type xfs_rtextent_free_defer_type = { + .name = "rtextent_free", +}; +#endif /* CONFIG_XFS_RT */ + +STATIC bool +xfs_efi_item_match( + struct xfs_log_item *lip, + uint64_t intent_id) +{ + return EFI_ITEM(lip)->efi_format.efi_id == intent_id; +} + +static const struct xfs_item_ops xfs_efi_item_ops = { + .flags = XFS_ITEM_INTENT, + .iop_size = xfs_efi_item_size, + .iop_format = xfs_efi_item_format, + .iop_unpin = xfs_efi_item_unpin, + .iop_release = xfs_efi_item_release, + .iop_match = xfs_efi_item_match, +}; + +/* + * This routine is called to create an in-core extent free intent + * item from the efi format structure which was logged on disk. + * It allocates an in-core efi, copies the extents from the format + * structure into it, and adds the efi to the AIL with the given + * LSN. + */ +STATIC int +xlog_recover_efi_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_mount *mp = log->l_mp; + struct xfs_efi_log_item *efip; + struct xfs_efi_log_format *efi_formatp; + int error; + + efi_formatp = item->ri_buf[0].iov_base; + + if (item->ri_buf[0].iov_len < xfs_efi_log_format_sizeof(0)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[0].iov_base, item->ri_buf[0].iov_len); + return -EFSCORRUPTED; + } + + efip = xfs_efi_init(mp, ITEM_TYPE(item), efi_formatp->efi_nextents); + error = xfs_efi_copy_format(&item->ri_buf[0], &efip->efi_format); + if (error) { + xfs_efi_item_free(efip); + return error; + } + atomic_set(&efip->efi_next_extent, efi_formatp->efi_nextents); + + xlog_recover_intent_item(log, &efip->efi_item, lsn, + &xfs_extent_free_defer_type); + return 0; +} + +const struct xlog_recover_item_ops xlog_efi_item_ops = { + .item_type = XFS_LI_EFI, + .commit_pass2 = xlog_recover_efi_commit_pass2, +}; + +#ifdef CONFIG_XFS_RT +STATIC int +xlog_recover_rtefi_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_mount *mp = log->l_mp; + struct xfs_efi_log_item *efip; + struct xfs_efi_log_format *efi_formatp; + int error; + + efi_formatp = item->ri_buf[0].iov_base; + + if (item->ri_buf[0].iov_len < xfs_efi_log_format_sizeof(0)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[0].iov_base, item->ri_buf[0].iov_len); + return -EFSCORRUPTED; + } + + efip = xfs_efi_init(mp, ITEM_TYPE(item), efi_formatp->efi_nextents); + error = xfs_efi_copy_format(&item->ri_buf[0], &efip->efi_format); + if (error) { + xfs_efi_item_free(efip); + return error; + } + atomic_set(&efip->efi_next_extent, efi_formatp->efi_nextents); + + xlog_recover_intent_item(log, &efip->efi_item, lsn, + &xfs_rtextent_free_defer_type); + return 0; +} +#else +STATIC int +xlog_recover_rtefi_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, log->l_mp, + item->ri_buf[0].iov_base, item->ri_buf[0].iov_len); + return -EFSCORRUPTED; +} +#endif + +const struct xlog_recover_item_ops xlog_rtefi_item_ops = { + .item_type = XFS_LI_EFI_RT, + .commit_pass2 = xlog_recover_rtefi_commit_pass2, +}; + +/* + * This routine is called when an EFD format structure is found in a committed + * transaction in the log. Its purpose is to cancel the corresponding EFI if it + * was still in the log. To do this it searches the AIL for the EFI with an id + * equal to that in the EFD format structure. If we find it we drop the EFD + * reference, which removes the EFI from the AIL and frees it. + */ +STATIC int +xlog_recover_efd_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_efd_log_format *efd_formatp; + int buflen = item->ri_buf[0].iov_len; + + efd_formatp = item->ri_buf[0].iov_base; + + if (buflen < sizeof(struct xfs_efd_log_format)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, log->l_mp, + efd_formatp, buflen); + return -EFSCORRUPTED; + } + + if (item->ri_buf[0].iov_len != xfs_efd_log_format32_sizeof( + efd_formatp->efd_nextents) && + item->ri_buf[0].iov_len != xfs_efd_log_format64_sizeof( + efd_formatp->efd_nextents)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, log->l_mp, + efd_formatp, buflen); + return -EFSCORRUPTED; + } + + xlog_recover_release_intent(log, XFS_LI_EFI, efd_formatp->efd_efi_id); + return 0; +} + +const struct xlog_recover_item_ops xlog_efd_item_ops = { + .item_type = XFS_LI_EFD, + .commit_pass2 = xlog_recover_efd_commit_pass2, +}; + +#ifdef CONFIG_XFS_RT +STATIC int +xlog_recover_rtefd_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_efd_log_format *efd_formatp; + int buflen = item->ri_buf[0].iov_len; + + efd_formatp = item->ri_buf[0].iov_base; + + if (buflen < sizeof(struct xfs_efd_log_format)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, log->l_mp, + efd_formatp, buflen); + return -EFSCORRUPTED; + } + + if (item->ri_buf[0].iov_len != xfs_efd_log_format32_sizeof( + efd_formatp->efd_nextents) && + item->ri_buf[0].iov_len != xfs_efd_log_format64_sizeof( + efd_formatp->efd_nextents)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, log->l_mp, + efd_formatp, buflen); + return -EFSCORRUPTED; + } + + xlog_recover_release_intent(log, XFS_LI_EFI_RT, + efd_formatp->efd_efi_id); + return 0; +} +#else +# define xlog_recover_rtefd_commit_pass2 xlog_recover_rtefi_commit_pass2 +#endif + +const struct xlog_recover_item_ops xlog_rtefd_item_ops = { + .item_type = XFS_LI_EFD_RT, + .commit_pass2 = xlog_recover_rtefd_commit_pass2, +}; diff --git a/fs/xfs/xfs_extfree_item.h b/fs/xfs/xfs_extfree_item.h new file mode 100644 index 000000000..af1b0331f --- /dev/null +++ b/fs/xfs/xfs_extfree_item.h @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_EXTFREE_ITEM_H__ +#define __XFS_EXTFREE_ITEM_H__ + +/* kernel only EFI/EFD definitions */ + +struct xfs_mount; +struct kmem_cache; + +/* + * Max number of extents in fast allocation path. + */ +#define XFS_EFI_MAX_FAST_EXTENTS 16 + +/* + * This is the "extent free intention" log item. It is used to log the fact + * that some extents need to be free. It is used in conjunction with the + * "extent free done" log item described below. + * + * The EFI is reference counted so that it is not freed prior to both the EFI + * and EFD being committed and unpinned. This ensures the EFI is inserted into + * the AIL even in the event of out of order EFI/EFD processing. In other words, + * an EFI is born with two references: + * + * 1.) an EFI held reference to track EFI AIL insertion + * 2.) an EFD held reference to track EFD commit + * + * On allocation, both references are the responsibility of the caller. Once the + * EFI is added to and dirtied in a transaction, ownership of reference one + * transfers to the transaction. The reference is dropped once the EFI is + * inserted to the AIL or in the event of failure along the way (e.g., commit + * failure, log I/O error, etc.). Note that the caller remains responsible for + * the EFD reference under all circumstances to this point. The caller has no + * means to detect failure once the transaction is committed, however. + * Therefore, an EFD is required after this point, even in the event of + * unrelated failure. + * + * Once an EFD is allocated and dirtied in a transaction, reference two + * transfers to the transaction. The EFD reference is dropped once it reaches + * the unpin handler. Similar to the EFI, the reference also drops in the event + * of commit failure or log I/O errors. Note that the EFD is not inserted in the + * AIL, so at this point both the EFI and EFD are freed. + */ +struct xfs_efi_log_item { + struct xfs_log_item efi_item; + atomic_t efi_refcount; + atomic_t efi_next_extent; + struct xfs_efi_log_format efi_format; +}; + +static inline size_t +xfs_efi_log_item_sizeof( + unsigned int nr) +{ + return offsetof(struct xfs_efi_log_item, efi_format) + + xfs_efi_log_format_sizeof(nr); +} + +/* + * This is the "extent free done" log item. It is used to log + * the fact that some extents earlier mentioned in an efi item + * have been freed. + */ +struct xfs_efd_log_item { + struct xfs_log_item efd_item; + struct xfs_efi_log_item *efd_efip; + uint efd_next_extent; + struct xfs_efd_log_format efd_format; +}; + +static inline size_t +xfs_efd_log_item_sizeof( + unsigned int nr) +{ + return offsetof(struct xfs_efd_log_item, efd_format) + + xfs_efd_log_format_sizeof(nr); +} + +/* + * Max number of extents in fast allocation path. + */ +#define XFS_EFD_MAX_FAST_EXTENTS 16 + +extern struct kmem_cache *xfs_efi_cache; +extern struct kmem_cache *xfs_efd_cache; + +struct xfs_extent_free_item; + +void xfs_extent_free_defer_add(struct xfs_trans *tp, + struct xfs_extent_free_item *xefi, + struct xfs_defer_pending **dfpp); + +unsigned int xfs_efi_log_space(unsigned int nr); +unsigned int xfs_efd_log_space(unsigned int nr); + +#endif /* __XFS_EXTFREE_ITEM_H__ */ diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c new file mode 100644 index 000000000..7bff07e31 --- /dev/null +++ b/fs/xfs/xfs_file.c @@ -0,0 +1,2212 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_inode_item.h" +#include "xfs_bmap.h" +#include "xfs_bmap_util.h" +#include "xfs_dir2.h" +#include "xfs_dir2_priv.h" +#include "xfs_ioctl.h" +#include "xfs_trace.h" +#include "xfs_log.h" +#include "xfs_icache.h" +#include "xfs_pnfs.h" +#include "xfs_iomap.h" +#include "xfs_reflink.h" +#include "xfs_file.h" +#include "xfs_aops.h" +#include "xfs_zone_alloc.h" +#include "xfs_error.h" +#include "xfs_errortag.h" + +#include +#include +#include +#include +#include +#include +#include + +static const struct vm_operations_struct xfs_file_vm_ops; + +/* + * Decide if the given file range is aligned to the size of the fundamental + * allocation unit for the file. + */ +bool +xfs_is_falloc_aligned( + struct xfs_inode *ip, + loff_t pos, + long long int len) +{ + unsigned int alloc_unit = xfs_inode_alloc_unitsize(ip); + + if (!is_power_of_2(alloc_unit)) + return isaligned_64(pos, alloc_unit) && + isaligned_64(len, alloc_unit); + + return !((pos | len) & (alloc_unit - 1)); +} + +/* + * Fsync operations on directories are much simpler than on regular files, + * as there is no file data to flush, and thus also no need for explicit + * cache flush operations, and there are no non-transaction metadata updates + * on directories either. + */ +STATIC int +xfs_dir_fsync( + struct file *file, + loff_t start, + loff_t end, + int datasync) +{ + struct xfs_inode *ip = XFS_I(file->f_mapping->host); + + trace_xfs_dir_fsync(ip); + return xfs_log_force_inode(ip); +} + +/* + * All metadata updates are logged, which means that we just have to push the + * journal to the required sequence number than holds the updates. We track + * datasync commits separately to full sync commits, and hence only need to + * select the correct sequence number for the log force here. + * + * We don't have to serialise against concurrent modifications, as we do not + * have to wait for modifications that have not yet completed. We define a + * transaction commit as completing when the commit sequence number is updated, + * hence if the sequence number has not updated, the sync operation has been + * run before the commit completed and we don't have to wait for it. + * + * If we have concurrent fsync/fdatasync() calls, the sequence numbers remain + * set on the log item until - at least - the journal flush completes. In + * reality, they are only cleared when the inode is fully unpinned (i.e. + * persistent in the journal and not dirty in the CIL), and so we rely on + * xfs_log_force_seq() either skipping sequences that have been persisted or + * waiting on sequences that are still in flight to correctly order concurrent + * sync operations. + */ +static int +xfs_fsync_flush_log( + struct xfs_inode *ip, + bool datasync, + int *log_flushed) +{ + struct xfs_inode_log_item *iip = ip->i_itemp; + xfs_csn_t seq = 0; + + spin_lock(&iip->ili_lock); + if (datasync) + seq = iip->ili_datasync_seq; + else + seq = iip->ili_commit_seq; + spin_unlock(&iip->ili_lock); + + if (!seq) + return 0; + + return xfs_log_force_seq(ip->i_mount, seq, XFS_LOG_SYNC, + log_flushed); +} + +STATIC int +xfs_file_fsync( + struct file *file, + loff_t start, + loff_t end, + int datasync) +{ + struct xfs_inode *ip = XFS_I(file->f_mapping->host); + struct xfs_mount *mp = ip->i_mount; + int error, err2; + int log_flushed = 0; + + trace_xfs_file_fsync(ip); + + error = file_write_and_wait_range(file, start, end); + if (error) + return error; + + if (xfs_is_shutdown(mp)) + return -EIO; + + xfs_iflags_clear(ip, XFS_ITRUNCATED); + + /* + * If we have an RT and/or log subvolume we need to make sure to flush + * the write cache the device used for file data first. This is to + * ensure newly written file data make it to disk before logging the new + * inode size in case of an extending write. + */ + if (XFS_IS_REALTIME_INODE(ip) && mp->m_rtdev_targp != mp->m_ddev_targp) + error = blkdev_issue_flush(mp->m_rtdev_targp->bt_bdev); + else if (mp->m_logdev_targp != mp->m_ddev_targp) + error = blkdev_issue_flush(mp->m_ddev_targp->bt_bdev); + + /* + * If the inode has a inode log item attached, it may need the journal + * flushed to persist any changes the log item might be tracking. + */ + if (ip->i_itemp) { + err2 = xfs_fsync_flush_log(ip, datasync, &log_flushed); + if (err2 && !error) + error = err2; + } + + /* + * If the log force was a no-op, we may still need to flush the + * file data target cache here. This can happen for fdatasync/O_DSYNC + * when no metadata needed to be committed. + * + * Use the inode's actual file data target rather than assuming the + * main data device. Realtime inodes with a separate realtime device + * are flushed before the log force, so this fallback only applies + * when the file data target is the same as the log target. + */ + if (!log_flushed) { + struct xfs_buftarg *file_targp = xfs_inode_buftarg(ip); + + if (mp->m_logdev_targp == file_targp) { + err2 = blkdev_issue_flush(file_targp->bt_bdev); + if (err2 && !error) + error = err2; + } + } + + return error; +} + +static int +xfs_ilock_iocb( + struct kiocb *iocb, + unsigned int lock_mode) +{ + struct xfs_inode *ip = XFS_I(file_inode(iocb->ki_filp)); + + if (iocb->ki_flags & IOCB_NOWAIT) { + if (!xfs_ilock_nowait(ip, lock_mode)) + return -EAGAIN; + } else { + xfs_ilock(ip, lock_mode); + } + + return 0; +} + +static int +xfs_ilock_iocb_for_write( + struct kiocb *iocb, + unsigned int *lock_mode) +{ + ssize_t ret; + struct xfs_inode *ip = XFS_I(file_inode(iocb->ki_filp)); + + ret = xfs_ilock_iocb(iocb, *lock_mode); + if (ret) + return ret; + + /* + * If a reflink remap is in progress we always need to take the iolock + * exclusively to wait for it to finish. + */ + if (*lock_mode == XFS_IOLOCK_SHARED && + xfs_iflags_test(ip, XFS_IREMAPPING)) { + xfs_iunlock(ip, *lock_mode); + *lock_mode = XFS_IOLOCK_EXCL; + return xfs_ilock_iocb(iocb, *lock_mode); + } + + return 0; +} + +/* + * Bounce buffering dio reads need a user context to copy back the data. + * Use an ioend to provide that. + */ +static void +xfs_dio_read_bounce_submit_io( + const struct iomap_iter *iter, + struct bio *bio, + loff_t file_offset) +{ + iomap_init_ioend(iter->inode, bio, file_offset, IOMAP_IOEND_DIRECT); + bio->bi_end_io = xfs_end_bio; + submit_bio(bio); +} + +static const struct iomap_dio_ops xfs_dio_read_bounce_ops = { + .submit_io = xfs_dio_read_bounce_submit_io, + .bio_set = &iomap_ioend_bioset, +}; + +STATIC ssize_t +xfs_file_dio_read( + struct kiocb *iocb, + struct iov_iter *to) +{ + struct xfs_inode *ip = XFS_I(file_inode(iocb->ki_filp)); + ssize_t ret; + + trace_xfs_file_direct_read(iocb, to); + + if (!iov_iter_count(to)) + return 0; /* skip atime */ + + file_accessed(iocb->ki_filp); + + ret = xfs_ilock_iocb(iocb, XFS_IOLOCK_SHARED); + if (ret) + return ret; + if (mapping_stable_writes(iocb->ki_filp->f_mapping)) { + ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, + &xfs_dio_read_bounce_ops, IOMAP_DIO_BOUNCE, + NULL, 0); + } else { + ret = iomap_dio_read_simple(iocb, to, xfs_read_iomap_begin); + if (ret == -ENOTBLK) + ret = iomap_dio_rw(iocb, to, &xfs_read_iomap_ops, NULL, + 0, NULL, 0); + } + xfs_iunlock(ip, XFS_IOLOCK_SHARED); + + return ret; +} + +static noinline ssize_t +xfs_file_dax_read( + struct kiocb *iocb, + struct iov_iter *to) +{ + struct xfs_inode *ip = XFS_I(iocb->ki_filp->f_mapping->host); + ssize_t ret = 0; + + trace_xfs_file_dax_read(iocb, to); + + if (!iov_iter_count(to)) + return 0; /* skip atime */ + + ret = xfs_ilock_iocb(iocb, XFS_IOLOCK_SHARED); + if (ret) + return ret; + ret = dax_iomap_rw(iocb, to, &xfs_read_iomap_ops); + xfs_iunlock(ip, XFS_IOLOCK_SHARED); + + file_accessed(iocb->ki_filp); + return ret; +} + +STATIC ssize_t +xfs_file_buffered_read( + struct kiocb *iocb, + struct iov_iter *to) +{ + struct xfs_inode *ip = XFS_I(file_inode(iocb->ki_filp)); + ssize_t ret; + + trace_xfs_file_buffered_read(iocb, to); + + ret = xfs_ilock_iocb(iocb, XFS_IOLOCK_SHARED); + if (ret) + return ret; + ret = generic_file_read_iter(iocb, to); + xfs_iunlock(ip, XFS_IOLOCK_SHARED); + + return ret; +} + +STATIC ssize_t +xfs_file_read_iter( + struct kiocb *iocb, + struct iov_iter *to) +{ + struct inode *inode = file_inode(iocb->ki_filp); + struct xfs_mount *mp = XFS_I(inode)->i_mount; + ssize_t ret = 0; + + XFS_STATS_INC(mp, xs_read_calls); + + if (xfs_is_shutdown(mp)) + return -EIO; + + if (IS_DAX(inode)) + ret = xfs_file_dax_read(iocb, to); + else if (iocb->ki_flags & IOCB_DIRECT) + ret = xfs_file_dio_read(iocb, to); + else + ret = xfs_file_buffered_read(iocb, to); + + if (ret > 0) + XFS_STATS_ADD(mp, xs_read_bytes, ret); + return ret; +} + +STATIC ssize_t +xfs_file_splice_read( + struct file *in, + loff_t *ppos, + struct pipe_inode_info *pipe, + size_t len, + unsigned int flags) +{ + struct inode *inode = file_inode(in); + struct xfs_inode *ip = XFS_I(inode); + struct xfs_mount *mp = ip->i_mount; + ssize_t ret = 0; + + XFS_STATS_INC(mp, xs_read_calls); + + if (xfs_is_shutdown(mp)) + return -EIO; + + trace_xfs_file_splice_read(ip, *ppos, len); + + xfs_ilock(ip, XFS_IOLOCK_SHARED); + ret = filemap_splice_read(in, ppos, pipe, len, flags); + xfs_iunlock(ip, XFS_IOLOCK_SHARED); + if (ret > 0) + XFS_STATS_ADD(mp, xs_read_bytes, ret); + return ret; +} + +/* + * Take care of zeroing post-EOF blocks when they might exist. + * + * Returns 0 if successfully, a negative error for a failure, or 1 if this + * function dropped the iolock and reacquired it exclusively and the caller + * needs to restart the write sanity checks. + */ +static ssize_t +xfs_file_write_zero_eof( + struct kiocb *iocb, + struct iov_iter *from, + unsigned int *iolock, + size_t count, + bool *drained_dio, + struct xfs_zone_alloc_ctx *ac) +{ + struct xfs_inode *ip = XFS_I(iocb->ki_filp->f_mapping->host); + loff_t isize; + int error; + + /* + * We need to serialise against EOF updates that occur in IO completions + * here. We want to make sure that nobody is changing the size while + * we do this check until we have placed an IO barrier (i.e. hold + * XFS_IOLOCK_EXCL) that prevents new IO from being dispatched. The + * spinlock effectively forms a memory barrier once we have + * XFS_IOLOCK_EXCL so we are guaranteed to see the latest EOF value and + * hence be able to correctly determine if we need to run zeroing. + */ + spin_lock(&ip->i_flags_lock); + isize = i_size_read(VFS_I(ip)); + if (iocb->ki_pos <= isize) { + spin_unlock(&ip->i_flags_lock); + return 0; + } + spin_unlock(&ip->i_flags_lock); + + if (iocb->ki_flags & IOCB_NOWAIT) + return -EAGAIN; + + if (!*drained_dio) { + /* + * If zeroing is needed and we are currently holding the iolock + * shared, we need to update it to exclusive which implies + * having to redo all checks before. + */ + if (*iolock == XFS_IOLOCK_SHARED) { + xfs_iunlock(ip, *iolock); + *iolock = XFS_IOLOCK_EXCL; + xfs_ilock(ip, *iolock); + iov_iter_reexpand(from, count); + } + + /* + * We now have an IO submission barrier in place, but AIO can do + * EOF updates during IO completion and hence we now need to + * wait for all of them to drain. Non-AIO DIO will have drained + * before we are given the XFS_IOLOCK_EXCL, and so for most + * cases this wait is a no-op. + */ + inode_dio_wait(VFS_I(ip)); + *drained_dio = true; + return 1; + } + + trace_xfs_zero_eof(ip, isize, iocb->ki_pos - isize); + + xfs_ilock(ip, XFS_MMAPLOCK_EXCL); + error = xfs_zero_range(ip, isize, iocb->ki_pos - isize, ac, NULL); + xfs_iunlock(ip, XFS_MMAPLOCK_EXCL); + + return error; +} + +/* + * Common pre-write limit and setup checks. + * + * Called with the iolock held either shared and exclusive according to + * @iolock, and returns with it held. Might upgrade the iolock to exclusive + * if called for a direct write beyond i_size. + */ +STATIC ssize_t +xfs_file_write_checks( + struct kiocb *iocb, + struct iov_iter *from, + unsigned int *iolock, + struct xfs_zone_alloc_ctx *ac) +{ + struct inode *inode = iocb->ki_filp->f_mapping->host; + size_t count = iov_iter_count(from); + bool drained_dio = false; + ssize_t error; + +restart: + error = generic_write_checks(iocb, from); + if (error <= 0) + return error; + + if (iocb->ki_flags & IOCB_NOWAIT) { + error = break_layout(inode, false); + if (error == -EWOULDBLOCK) + error = -EAGAIN; + } else { + error = xfs_break_layouts(inode, iolock, BREAK_WRITE); + } + + if (error) + return error; + + /* + * For changing security info in file_remove_privs() we need i_rwsem + * exclusively. + */ + if (*iolock == XFS_IOLOCK_SHARED && !IS_NOSEC(inode)) { + xfs_iunlock(XFS_I(inode), *iolock); + *iolock = XFS_IOLOCK_EXCL; + error = xfs_ilock_iocb(iocb, *iolock); + if (error) { + *iolock = 0; + return error; + } + goto restart; + } + + /* + * If the offset is beyond the size of the file, we need to zero all + * blocks that fall between the existing EOF and the start of this + * write. + * + * We can do an unlocked check for i_size here safely as I/O completion + * can only extend EOF. Truncate is locked out at this point, so the + * EOF can not move backwards, only forwards. Hence we only need to take + * the slow path when we are at or beyond the current EOF. + */ + if (iocb->ki_pos > i_size_read(inode)) { + error = xfs_file_write_zero_eof(iocb, from, iolock, count, + &drained_dio, ac); + if (error == 1) + goto restart; + if (error) + return error; + } + + return kiocb_modified(iocb); +} + +static ssize_t +xfs_zoned_write_space_reserve( + struct xfs_mount *mp, + struct kiocb *iocb, + struct iov_iter *from, + unsigned int flags, + struct xfs_zone_alloc_ctx *ac) +{ + loff_t count = iov_iter_count(from); + int error; + + if (iocb->ki_flags & IOCB_NOWAIT) + flags |= XFS_ZR_NOWAIT; + + /* + * Check the rlimit and LFS boundary first so that we don't over-reserve + * by possibly a lot. + * + * The generic write path will redo this check later, and it might have + * changed by then. If it got expanded we'll stick to our earlier + * smaller limit, and if it is decreased the new smaller limit will be + * used and our extra space reservation will be returned after finishing + * the write. + */ + error = generic_write_check_limits(iocb->ki_filp, iocb->ki_pos, &count); + if (error) + return error; + + /* + * Sloppily round up count to file system blocks. + * + * This will often reserve an extra block, but that avoids having to look + * at the start offset, which isn't stable for O_APPEND until taking the + * iolock. Also we need to reserve a block each for zeroing the old + * EOF block and the new start block if they are unaligned. + * + * Any remaining block will be returned after the write. + */ + return xfs_zoned_space_reserve(mp, XFS_B_TO_FSB(mp, count) + 1 + 2, + flags, ac); +} + +/* + * We need to lock the test/set EOF update as we can be racing with + * other IO completions here to update the EOF. Failing to serialise + * here can result in EOF moving backwards and Bad Things Happen when + * that occurs. + * + * As IO completion only ever extends EOF, we can do an unlocked check + * here to avoid taking the spinlock. If we land within the current EOF, + * then we do not need to do an extending update at all, and we don't + * need to take the lock to check this. If we race with an update moving + * EOF, then we'll either still be beyond EOF and need to take the lock, + * or we'll be within EOF and we don't need to take it at all. + */ +static int +xfs_dio_endio_set_isize( + struct inode *inode, + loff_t offset, + ssize_t size) +{ + struct xfs_inode *ip = XFS_I(inode); + + if (offset + size <= i_size_read(inode)) + return 0; + + spin_lock(&ip->i_flags_lock); + if (offset + size <= i_size_read(inode)) { + spin_unlock(&ip->i_flags_lock); + return 0; + } + + i_size_write(inode, offset + size); + spin_unlock(&ip->i_flags_lock); + + return xfs_setfilesize(ip, offset, size); +} + +static int +xfs_zoned_dio_write_end_io( + struct kiocb *iocb, + ssize_t size, + int error, + unsigned flags) +{ + struct inode *inode = file_inode(iocb->ki_filp); + struct xfs_inode *ip = XFS_I(inode); + unsigned int nofs_flag; + + ASSERT(!(flags & (IOMAP_DIO_UNWRITTEN | IOMAP_DIO_COW))); + + trace_xfs_end_io_direct_write(ip, iocb->ki_pos, size); + + if (xfs_is_shutdown(ip->i_mount)) + return -EIO; + + if (error || !size) + return error; + + XFS_STATS_ADD(ip->i_mount, xs_write_bytes, size); + + nofs_flag = memalloc_nofs_save(); + error = xfs_dio_endio_set_isize(inode, iocb->ki_pos, size); + memalloc_nofs_restore(nofs_flag); + + return error; +} + +static int +xfs_dio_write_end_io( + struct kiocb *iocb, + ssize_t size, + int error, + unsigned flags) +{ + struct inode *inode = file_inode(iocb->ki_filp); + struct xfs_inode *ip = XFS_I(inode); + loff_t offset = iocb->ki_pos; + unsigned int nofs_flag; + + ASSERT(!xfs_is_zoned_inode(ip)); + + trace_xfs_end_io_direct_write(ip, offset, size); + + if (xfs_is_shutdown(ip->i_mount)) + return -EIO; + + if (error) + return error; + if (!size) + return 0; + + /* + * Capture amount written on completion as we can't reliably account + * for it on submission. + */ + XFS_STATS_ADD(ip->i_mount, xs_write_bytes, size); + + /* + * We can allocate memory here while doing writeback on behalf of + * memory reclaim. To avoid memory allocation deadlocks set the + * task-wide nofs context for the following operations. + */ + nofs_flag = memalloc_nofs_save(); + + if (flags & IOMAP_DIO_COW) { + if (iocb->ki_flags & IOCB_ATOMIC) + error = xfs_reflink_end_atomic_cow(ip, offset, size); + else + error = xfs_reflink_end_cow(ip, offset, size); + if (error) + goto out; + } + + /* + * Unwritten conversion updates the in-core isize after extent + * conversion but before updating the on-disk size. Updating isize any + * earlier allows a racing dio read to find unwritten extents before + * they are converted. + */ + if (flags & IOMAP_DIO_UNWRITTEN) { + error = xfs_iomap_write_unwritten(ip, offset, size, true); + goto out; + } + + /* + * We need to update the in-core inode size here so that we don't end up + * with the on-disk inode size being outside the in-core inode size. We + * have no other method of updating EOF for AIO, so always do it here + * if necessary. + */ + error = xfs_dio_endio_set_isize(inode, offset, size); + +out: + memalloc_nofs_restore(nofs_flag); + return error; +} + +static const struct iomap_dio_ops xfs_dio_write_ops = { + .end_io = xfs_dio_write_end_io, +}; + +static void +xfs_dio_zoned_submit_io( + const struct iomap_iter *iter, + struct bio *bio, + loff_t file_offset) +{ + struct xfs_mount *mp = XFS_I(iter->inode)->i_mount; + struct xfs_zone_alloc_ctx *ac = iter->private; + xfs_filblks_t count_fsb; + struct iomap_ioend *ioend; + + count_fsb = XFS_B_TO_FSB(mp, bio->bi_iter.bi_size); + if (count_fsb > ac->reserved_blocks) { + xfs_err(mp, +"allocation (%lld) larger than reservation (%lld).", + count_fsb, ac->reserved_blocks); + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); + bio_io_error(bio); + return; + } + ac->reserved_blocks -= count_fsb; + + bio->bi_end_io = xfs_end_bio; + ioend = iomap_init_ioend(iter->inode, bio, file_offset, + IOMAP_IOEND_DIRECT); + xfs_zone_alloc_and_submit(ioend, &ac->open_zone); +} + +static const struct iomap_dio_ops xfs_dio_zoned_write_ops = { + .bio_set = &iomap_ioend_bioset, + .submit_io = xfs_dio_zoned_submit_io, + .end_io = xfs_zoned_dio_write_end_io, +}; + +/* + * Handle block aligned direct I/O writes. + */ +static noinline ssize_t +xfs_file_dio_write_aligned( + struct xfs_inode *ip, + struct kiocb *iocb, + struct iov_iter *from, + const struct iomap_ops *ops, + const struct iomap_dio_ops *dops, + struct xfs_zone_alloc_ctx *ac) +{ + unsigned int iolock = XFS_IOLOCK_SHARED; + unsigned int dio_flags = 0; + ssize_t ret; + + /* + * For always COW inodes, each bio must be aligned to the file system + * block size and not just the device sector size because we need to + * allocate a block-aligned amount of space for each write. + */ + if (xfs_is_always_cow_inode(ip)) + dio_flags |= IOMAP_DIO_FSBLOCK_ALIGNED; + + ret = xfs_ilock_iocb_for_write(iocb, &iolock); + if (ret) + return ret; + ret = xfs_file_write_checks(iocb, from, &iolock, ac); + if (ret) + goto out_unlock; + + /* + * We don't need to hold the IOLOCK exclusively across the IO, so demote + * the iolock back to shared if we had to take the exclusive lock in + * xfs_file_write_checks() for other reasons. + */ + if (iolock == XFS_IOLOCK_EXCL) { + xfs_ilock_demote(ip, XFS_IOLOCK_EXCL); + iolock = XFS_IOLOCK_SHARED; + } + if (mapping_stable_writes(iocb->ki_filp->f_mapping)) + dio_flags |= IOMAP_DIO_BOUNCE; + trace_xfs_file_direct_write(iocb, from); + ret = iomap_dio_rw(iocb, from, ops, dops, dio_flags, ac, 0); +out_unlock: + xfs_iunlock(ip, iolock); + return ret; +} + +/* + * Handle block aligned direct I/O writes to zoned devices. + */ +static noinline ssize_t +xfs_file_dio_write_zoned( + struct xfs_inode *ip, + struct kiocb *iocb, + struct iov_iter *from) +{ + struct xfs_zone_alloc_ctx ac = { }; + ssize_t ret; + + ret = xfs_zoned_write_space_reserve(ip->i_mount, iocb, from, 0, &ac); + if (ret < 0) + return ret; + ret = xfs_file_dio_write_aligned(ip, iocb, from, + &xfs_zoned_direct_write_iomap_ops, + &xfs_dio_zoned_write_ops, &ac); + xfs_zoned_space_unreserve(ip->i_mount, &ac); + return ret; +} + +/* + * Handle block atomic writes + * + * Two methods of atomic writes are supported: + * - REQ_ATOMIC-based, which would typically use some form of HW offload in the + * disk + * - COW-based, which uses a COW fork as a staging extent for data updates + * before atomically updating extent mappings for the range being written + * + */ +static noinline ssize_t +xfs_file_dio_write_atomic( + struct xfs_inode *ip, + struct kiocb *iocb, + struct iov_iter *from) +{ + unsigned int iolock = XFS_IOLOCK_SHARED; + ssize_t ret, ocount = iov_iter_count(from); + unsigned int dio_flags = 0; + const struct iomap_ops *dops; + + /* + * HW offload should be faster, so try that first if it is already + * known that the write length is not too large. + */ + if (ocount > xfs_inode_buftarg(ip)->bt_awu_max) + dops = &xfs_atomic_write_cow_iomap_ops; + else + dops = &xfs_direct_write_iomap_ops; + +retry: + ret = xfs_ilock_iocb_for_write(iocb, &iolock); + if (ret) + return ret; + + ret = xfs_file_write_checks(iocb, from, &iolock, NULL); + if (ret) + goto out_unlock; + + /* Demote similar to xfs_file_dio_write_aligned() */ + if (iolock == XFS_IOLOCK_EXCL) { + xfs_ilock_demote(ip, XFS_IOLOCK_EXCL); + iolock = XFS_IOLOCK_SHARED; + } + + trace_xfs_file_direct_write(iocb, from); + if (mapping_stable_writes(iocb->ki_filp->f_mapping)) + dio_flags |= IOMAP_DIO_BOUNCE; + ret = iomap_dio_rw(iocb, from, dops, &xfs_dio_write_ops, dio_flags, + NULL, 0); + + /* + * The retry mechanism is based on the ->iomap_next method returning + * -ENOPROTOOPT, which would be when the REQ_ATOMIC-based write is not + * possible. The REQ_ATOMIC-based method is typically not possible if + * the write spans multiple extents or the disk blocks are misaligned. + */ + if (ret == -ENOPROTOOPT && dops == &xfs_direct_write_iomap_ops) { + xfs_iunlock(ip, iolock); + dops = &xfs_atomic_write_cow_iomap_ops; + goto retry; + } + +out_unlock: + if (iolock) + xfs_iunlock(ip, iolock); + return ret; +} + +/* + * Handle block unaligned direct I/O writes + * + * In most cases direct I/O writes will be done holding IOLOCK_SHARED, allowing + * them to be done in parallel with reads and other direct I/O writes. However, + * if the I/O is not aligned to filesystem blocks, the direct I/O layer may need + * to do sub-block zeroing and that requires serialisation against other direct + * I/O to the same block. In this case we need to serialise the submission of + * the unaligned I/O so that we don't get racing block zeroing in the dio layer. + * In the case where sub-block zeroing is not required, we can do concurrent + * sub-block dios to the same block successfully. + * + * Optimistically submit the I/O using the shared lock first, but use the + * IOMAP_DIO_OVERWRITE_ONLY flag to tell the lower layers to return -EAGAIN + * if block allocation or partial block zeroing would be required. In that case + * we try again with the exclusive lock. + */ +static noinline ssize_t +xfs_file_dio_write_unaligned( + struct xfs_inode *ip, + struct kiocb *iocb, + struct iov_iter *from) +{ + size_t isize = i_size_read(VFS_I(ip)); + size_t count = iov_iter_count(from); + unsigned int iolock = XFS_IOLOCK_SHARED; + unsigned int flags = IOMAP_DIO_OVERWRITE_ONLY; + ssize_t ret; + + /* + * Extending writes need exclusivity because of the sub-block zeroing + * that the DIO code always does for partial tail blocks beyond EOF, so + * don't even bother trying the fast path in this case. + */ + if (iocb->ki_pos > isize || iocb->ki_pos + count >= isize) { + if (iocb->ki_flags & IOCB_NOWAIT) + return -EAGAIN; +retry_exclusive: + iolock = XFS_IOLOCK_EXCL; + flags = IOMAP_DIO_FORCE_WAIT; + } + + ret = xfs_ilock_iocb_for_write(iocb, &iolock); + if (ret) + return ret; + + /* + * We can't properly handle unaligned direct I/O to reflink files yet, + * as we can't unshare a partial block. + */ + if (xfs_is_cow_inode(ip)) { + trace_xfs_reflink_bounce_dio_write(iocb, from); + ret = -ENOTBLK; + goto out_unlock; + } + + ret = xfs_file_write_checks(iocb, from, &iolock, NULL); + if (ret) + goto out_unlock; + + /* + * If we are doing exclusive unaligned I/O, this must be the only I/O + * in-flight. Otherwise we risk data corruption due to unwritten extent + * conversions from the AIO end_io handler. Wait for all other I/O to + * drain first. + */ + if (flags & IOMAP_DIO_FORCE_WAIT) + inode_dio_wait(VFS_I(ip)); + + if (mapping_stable_writes(iocb->ki_filp->f_mapping)) + flags |= IOMAP_DIO_BOUNCE; + + trace_xfs_file_direct_write(iocb, from); + ret = iomap_dio_rw(iocb, from, &xfs_direct_write_iomap_ops, + &xfs_dio_write_ops, flags, NULL, 0); + + /* + * Retry unaligned I/O with exclusive blocking semantics if the DIO + * layer rejected it for mapping or locking reasons. If we are doing + * nonblocking user I/O, propagate the error. + */ + if (ret == -EAGAIN && !(iocb->ki_flags & IOCB_NOWAIT)) { + ASSERT(flags & IOMAP_DIO_OVERWRITE_ONLY); + xfs_iunlock(ip, iolock); + goto retry_exclusive; + } + +out_unlock: + if (iolock) + xfs_iunlock(ip, iolock); + return ret; +} + +static ssize_t +xfs_file_dio_write( + struct kiocb *iocb, + struct iov_iter *from) +{ + struct xfs_inode *ip = XFS_I(file_inode(iocb->ki_filp)); + struct xfs_buftarg *target = xfs_inode_buftarg(ip); + size_t count = iov_iter_count(from); + + /* direct I/O must be aligned to device logical sector size */ + if ((iocb->ki_pos | count) & target->bt_logical_sectormask) + return -EINVAL; + + if ((iocb->ki_pos | count) & ip->i_mount->m_blockmask) + return xfs_file_dio_write_unaligned(ip, iocb, from); + if (xfs_is_zoned_inode(ip)) + return xfs_file_dio_write_zoned(ip, iocb, from); + if (iocb->ki_flags & IOCB_ATOMIC) + return xfs_file_dio_write_atomic(ip, iocb, from); + return xfs_file_dio_write_aligned(ip, iocb, from, + &xfs_direct_write_iomap_ops, &xfs_dio_write_ops, NULL); +} + +static noinline ssize_t +xfs_file_dax_write( + struct kiocb *iocb, + struct iov_iter *from) +{ + struct inode *inode = iocb->ki_filp->f_mapping->host; + struct xfs_inode *ip = XFS_I(inode); + unsigned int iolock = XFS_IOLOCK_EXCL; + ssize_t ret, error = 0; + loff_t pos; + + ret = xfs_ilock_iocb(iocb, iolock); + if (ret) + return ret; + ret = xfs_file_write_checks(iocb, from, &iolock, NULL); + if (ret) + goto out; + + pos = iocb->ki_pos; + + trace_xfs_file_dax_write(iocb, from); + ret = dax_iomap_rw(iocb, from, &xfs_dax_write_iomap_ops); + if (ret > 0 && iocb->ki_pos > i_size_read(inode)) { + i_size_write(inode, iocb->ki_pos); + error = xfs_setfilesize(ip, pos, ret); + } +out: + if (iolock) + xfs_iunlock(ip, iolock); + if (error) + return error; + + if (ret > 0) { + XFS_STATS_ADD(ip->i_mount, xs_write_bytes, ret); + + /* Handle various SYNC-type writes */ + ret = generic_write_sync(iocb, ret); + } + return ret; +} + +STATIC ssize_t +xfs_file_buffered_write( + struct kiocb *iocb, + struct iov_iter *from) +{ + struct inode *inode = iocb->ki_filp->f_mapping->host; + struct xfs_inode *ip = XFS_I(inode); + ssize_t ret; + bool cleared_space = false; + unsigned int iolock; + +write_retry: + iolock = XFS_IOLOCK_EXCL; + ret = xfs_ilock_iocb(iocb, iolock); + if (ret) + return ret; + + ret = xfs_file_write_checks(iocb, from, &iolock, NULL); + if (ret) + goto out; + + trace_xfs_file_buffered_write(iocb, from); + ret = iomap_file_buffered_write(iocb, from, + &xfs_buffered_write_iomap_ops, &xfs_iomap_write_ops, + NULL); + + /* + * If we hit a space limit, try to free up some lingering preallocated + * space before returning an error. In the case of ENOSPC, first try to + * write back all dirty inodes to free up some of the excess reserved + * metadata space. This reduces the chances that the eofblocks scan + * waits on dirty mappings. Since xfs_flush_inodes() is serialized, this + * also behaves as a filter to prevent too many eofblocks scans from + * running at the same time. Use a synchronous scan to increase the + * effectiveness of the scan. + */ + if (ret == -EDQUOT && !cleared_space) { + xfs_iunlock(ip, iolock); + xfs_blockgc_free_quota(ip, XFS_ICWALK_FLAG_SYNC); + cleared_space = true; + goto write_retry; + } else if (ret == -ENOSPC && !cleared_space) { + struct xfs_icwalk icw = {0}; + + cleared_space = true; + xfs_flush_inodes(ip->i_mount); + + xfs_iunlock(ip, iolock); + icw.icw_flags = XFS_ICWALK_FLAG_SYNC; + xfs_blockgc_free_space(ip->i_mount, &icw); + goto write_retry; + } + +out: + if (iolock) + xfs_iunlock(ip, iolock); + + if (ret > 0) { + XFS_STATS_ADD(ip->i_mount, xs_write_bytes, ret); + /* Handle various SYNC-type writes */ + ret = generic_write_sync(iocb, ret); + } + return ret; +} + +STATIC ssize_t +xfs_file_buffered_write_zoned( + struct kiocb *iocb, + struct iov_iter *from) +{ + struct xfs_inode *ip = XFS_I(iocb->ki_filp->f_mapping->host); + struct xfs_mount *mp = ip->i_mount; + unsigned int iolock = XFS_IOLOCK_EXCL; + bool cleared_space = false; + struct xfs_zone_alloc_ctx ac = { }; + ssize_t ret; + + ret = xfs_zoned_write_space_reserve(mp, iocb, from, XFS_ZR_GREEDY, &ac); + if (ret < 0) + return ret; + + ret = xfs_ilock_iocb(iocb, iolock); + if (ret) + goto out_unreserve; + + ret = xfs_file_write_checks(iocb, from, &iolock, &ac); + if (ret) + goto out_unlock; + + /* + * Truncate the iter to the length that we were actually able to + * allocate blocks for. This needs to happen after + * xfs_file_write_checks, because that assigns ki_pos for O_APPEND + * writes. + */ + iov_iter_truncate(from, + XFS_FSB_TO_B(mp, ac.reserved_blocks) - + (iocb->ki_pos & mp->m_blockmask)); + if (!iov_iter_count(from)) + goto out_unlock; + +retry: + trace_xfs_file_buffered_write(iocb, from); + ret = iomap_file_buffered_write(iocb, from, + &xfs_buffered_write_iomap_ops, &xfs_iomap_write_ops, + &ac); + if (ret == -ENOSPC && !cleared_space) { + /* + * Kick off writeback to convert delalloc space and release the + * usually too pessimistic indirect block reservations. + */ + xfs_flush_inodes(mp); + cleared_space = true; + goto retry; + } + +out_unlock: + xfs_iunlock(ip, iolock); +out_unreserve: + xfs_zoned_space_unreserve(ip->i_mount, &ac); + if (ret > 0) { + XFS_STATS_ADD(mp, xs_write_bytes, ret); + ret = generic_write_sync(iocb, ret); + } + return ret; +} + +STATIC ssize_t +xfs_file_write_iter( + struct kiocb *iocb, + struct iov_iter *from) +{ + struct inode *inode = iocb->ki_filp->f_mapping->host; + struct xfs_inode *ip = XFS_I(inode); + ssize_t ret; + size_t ocount = iov_iter_count(from); + + XFS_STATS_INC(ip->i_mount, xs_write_calls); + + if (ocount == 0) + return 0; + + if (xfs_is_shutdown(ip->i_mount)) + return -EIO; + + if (iocb->ki_flags & IOCB_ATOMIC) { + if (ocount < xfs_get_atomic_write_min(ip)) + return -EINVAL; + + if (ocount > xfs_get_atomic_write_max(ip)) + return -EINVAL; + + ret = generic_atomic_write_valid(iocb, from); + if (ret) + return ret; + } + + if (IS_DAX(inode)) + return xfs_file_dax_write(iocb, from); + + if (iocb->ki_flags & IOCB_DIRECT) { + /* + * Allow a directio write to fall back to a buffered + * write *only* in the case that we're doing a reflink + * CoW. In all other directio scenarios we do not + * allow an operation to fall back to buffered mode. + */ + ret = xfs_file_dio_write(iocb, from); + if (ret != -ENOTBLK) + return ret; + } + + if (xfs_is_zoned_inode(ip)) + return xfs_file_buffered_write_zoned(iocb, from); + return xfs_file_buffered_write(iocb, from); +} + +/* Does this file, inode, or mount want synchronous writes? */ +static inline bool xfs_file_sync_writes(struct file *filp) +{ + struct xfs_inode *ip = XFS_I(file_inode(filp)); + + if (xfs_has_wsync(ip->i_mount)) + return true; + if (filp->f_flags & (__O_SYNC | O_DSYNC)) + return true; + if (IS_SYNC(file_inode(filp))) + return true; + + return false; +} + +static int +xfs_falloc_newsize( + struct file *file, + int mode, + loff_t offset, + loff_t len, + loff_t *new_size) +{ + struct inode *inode = file_inode(file); + + if ((mode & FALLOC_FL_KEEP_SIZE) || offset + len <= i_size_read(inode)) + return 0; + *new_size = offset + len; + return inode_newsize_ok(inode, *new_size); +} + +static int +xfs_falloc_setsize( + struct file *file, + loff_t new_size) +{ + struct iattr iattr = { + .ia_valid = ATTR_SIZE, + .ia_size = new_size, + }; + + if (!new_size) + return 0; + return xfs_vn_setattr_size(file_mnt_idmap(file), file_dentry(file), + &iattr); +} + +static int +xfs_falloc_collapse_range( + struct file *file, + loff_t offset, + loff_t len, + struct xfs_zone_alloc_ctx *ac) +{ + struct inode *inode = file_inode(file); + loff_t new_size = i_size_read(inode) - len; + int error; + + if (!xfs_is_falloc_aligned(XFS_I(inode), offset, len)) + return -EINVAL; + + /* + * There is no need to overlap collapse range with EOF, in which case it + * is effectively a truncate operation + */ + if (offset + len >= i_size_read(inode)) + return -EINVAL; + + error = xfs_collapse_file_space(XFS_I(inode), offset, len, ac); + if (error) + return error; + return xfs_falloc_setsize(file, new_size); +} + +static int +xfs_falloc_insert_range( + struct file *file, + loff_t offset, + loff_t len) +{ + struct inode *inode = file_inode(file); + loff_t isize = i_size_read(inode); + int error; + + if (!xfs_is_falloc_aligned(XFS_I(inode), offset, len)) + return -EINVAL; + + /* + * New inode size must not exceed ->s_maxbytes, accounting for + * possible signed overflow. + */ + if (inode->i_sb->s_maxbytes - isize < len) + return -EFBIG; + + /* Offset should be less than i_size */ + if (offset >= isize) + return -EINVAL; + + /* + * Let writeback clean up EOF folio state before we bump i_size. The + * insert flushes before it starts shifting and under certain + * circumstances we can write back blocks that should technically be + * considered post-eof (and thus should not be submitted for writeback). + * + * For example, a large, dirty folio that spans EOF and is backed by + * post-eof COW fork preallocation can cause block remap into the data + * fork. This shifts back out beyond EOF, but creates an expectedly + * written post-eof block. The insert is going to flush, unmap and + * cancel prealloc across this whole range, so flush EOF now before we + * bump i_size to provide consistent behavior. + */ + error = filemap_write_and_wait_range(inode->i_mapping, isize, isize); + if (error) + return error; + + error = xfs_falloc_setsize(file, isize + len); + if (error) + return error; + + /* + * Perform hole insertion now that the file size has been updated so + * that if we crash during the operation we don't leave shifted extents + * past EOF and hence losing access to the data that is contained within + * them. + */ + return xfs_insert_file_space(XFS_I(inode), offset, len); +} + +/* + * For various operations we need to zero up to one block at each end of + * the affected range. For zoned file systems this will require a space + * allocation, for which we need a reservation ahead of time. + */ +#define XFS_ZONED_ZERO_EDGE_SPACE_RES 2 + +/* + * Zero range implements a full zeroing mechanism but is only used in limited + * situations. It is more efficient to allocate unwritten extents than to + * perform zeroing here, so use an errortag to randomly force zeroing on DEBUG + * kernels for added test coverage. + * + * On zoned file systems, the error is already injected by + * xfs_file_zoned_fallocate, which then reserves the additional space needed. + * We only check for this extra space reservation here. + */ +static inline bool +xfs_falloc_force_zero( + struct xfs_inode *ip, + struct xfs_zone_alloc_ctx *ac) +{ + if (xfs_is_zoned_inode(ip)) { + if (ac->reserved_blocks > XFS_ZONED_ZERO_EDGE_SPACE_RES) { + ASSERT(IS_ENABLED(CONFIG_XFS_DEBUG)); + return true; + } + return false; + } + return XFS_TEST_ERROR(ip->i_mount, XFS_ERRTAG_FORCE_ZERO_RANGE); +} + +static int +xfs_falloc_write_zeroes( + struct file *file, + int mode, + loff_t offset, + loff_t len, + struct xfs_zone_alloc_ctx *ac) +{ + struct inode *inode = file_inode(file); + struct xfs_inode *ip = XFS_I(inode); + loff_t new_size = 0; + int error; + + /* + * XXX: There is an issue with bigrtalloc inodes where there can be blocks + * that are written after the EOF block. This breaks the promise of no + * written blocks past EOF. Return EOPNOTSUPP until it is fixed. + */ + if (xfs_is_always_cow_inode(ip) || xfs_inode_has_bigrtalloc(ip) || + !bdev_write_zeroes_unmap_sectors(xfs_inode_buftarg(ip)->bt_bdev)) + return -EOPNOTSUPP; + + error = xfs_falloc_newsize(file, mode, offset, len, &new_size); + if (error) + return error; + + /* + * + * |----------|----------|----------|----------|----------| + * ^ ^ ^ ^ ^ ^ + * | | | | | | + * | offset | | end | + * | | | | + * offset_rd offset_ru end_rd end_ru + * + * xfs_free_file_space() punches the aligned interior offset_ru -> end_rd + * to holes and byte-zeroes the in-range parts of the partial edge blocks, + * offset -> offset_ru and end_rd -> end. xfs_zero_range() only touches + * already-written blocks here; it skips holes and unwritten extents, so + * unallocated/unwritten edge blocks are left for the allocation below. + */ + error = xfs_free_file_space(ip, offset, len, ac); + if (error) + return error; + + /* + * Publish the new size while the punched range is still a hole, then + * fill it with written zeroes. Like the other fallocate modes we use + * xfs_falloc_setsize(), but it must run *before* we convert the range + * to written extents: xfs_setattr_size() zeroes [old EOF, new size) via + * xfs_zero_range(), which skips holes, so there is nothing to re-zero. + * It will also writeback partial EOF block before the on-disk size is + * logged. + * Note: extending the size before allocating means a failure below + * leaves the file larger with unallocated holes in the new range. + * That is safe as holes within i_size read back as zeroes and expose + * no stale data while the error is propagated to the caller. + */ + error = xfs_falloc_setsize(file, new_size); + if (error) + return error; + + /* + * Allocate written, zeroed extents across the range. xfs_alloc_file_space() + * rounds outward to block granularity: + * - holes (the punched interior and any unallocated edge block) are + * allocated and zeroed; + * - unwritten extents (including unwritten edge blocks) are converted to + * written and zeroed; + * - Already written edge blocks are skipped. The out-of-range bytes of + * a written edge block keep their data (offset_rd -> offset and + * end -> end_rd); their in-range bytes (offset -> offset_ru and + * end_ru -> end were already zeroed by xfs_free_file_space(). + */ + return xfs_alloc_file_space(ip, offset, len, + XFS_ALLOC_FILE_SPACE_WRITE_ZEROES); +} + +/* + * Punch a hole and prealloc the range. We use a hole punch rather than + * unwritten extent conversion for two reasons: + * + * 1.) Hole punch handles partial block zeroing for us. + * 2.) If prealloc returns ENOSPC, the file range is still zero-valued by + * virtue of the hole punch. + */ +static int +xfs_falloc_zero_range( + struct file *file, + int mode, + loff_t offset, + loff_t len, + struct xfs_zone_alloc_ctx *ac) +{ + struct inode *inode = file_inode(file); + struct xfs_inode *ip = XFS_I(inode); + unsigned int blksize = i_blocksize(inode); + loff_t new_size = 0; + int error; + + trace_xfs_zero_file_space(ip); + + error = xfs_falloc_newsize(file, mode, offset, len, &new_size); + if (error) + return error; + + if (xfs_falloc_force_zero(ip, ac)) { + error = xfs_zero_range(ip, offset, len, ac, NULL); + } else { + error = xfs_free_file_space(ip, offset, len, ac); + if (error) + return error; + + len = round_up(offset + len, blksize) - + round_down(offset, blksize); + offset = round_down(offset, blksize); + error = xfs_alloc_file_space(ip, offset, len, + XFS_ALLOC_FILE_SPACE_PREALLOC); + } + if (error) + return error; + return xfs_falloc_setsize(file, new_size); +} + +static int +xfs_falloc_unshare_range( + struct file *file, + int mode, + loff_t offset, + loff_t len) +{ + struct inode *inode = file_inode(file); + loff_t new_size = 0; + int error; + + error = xfs_falloc_newsize(file, mode, offset, len, &new_size); + if (error) + return error; + + error = xfs_reflink_unshare(XFS_I(inode), offset, len); + if (error) + return error; + + error = xfs_alloc_file_space(XFS_I(inode), offset, len, + XFS_ALLOC_FILE_SPACE_PREALLOC); + if (error) + return error; + return xfs_falloc_setsize(file, new_size); +} + +static int +xfs_falloc_allocate_range( + struct file *file, + int mode, + loff_t offset, + loff_t len) +{ + struct inode *inode = file_inode(file); + loff_t new_size = 0; + int error; + + /* + * If always_cow mode we can't use preallocations and thus should not + * create them. + */ + if (xfs_is_always_cow_inode(XFS_I(inode))) + return -EOPNOTSUPP; + + error = xfs_falloc_newsize(file, mode, offset, len, &new_size); + if (error) + return error; + + error = xfs_alloc_file_space(XFS_I(inode), offset, len, + XFS_ALLOC_FILE_SPACE_PREALLOC); + if (error) + return error; + return xfs_falloc_setsize(file, new_size); +} + +#define XFS_FALLOC_FL_SUPPORTED \ + (FALLOC_FL_ALLOCATE_RANGE | FALLOC_FL_KEEP_SIZE | \ + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_COLLAPSE_RANGE | \ + FALLOC_FL_ZERO_RANGE | FALLOC_FL_INSERT_RANGE | \ + FALLOC_FL_UNSHARE_RANGE | FALLOC_FL_WRITE_ZEROES) + +STATIC long +__xfs_file_fallocate( + struct file *file, + int mode, + loff_t offset, + loff_t len, + struct xfs_zone_alloc_ctx *ac) +{ + struct inode *inode = file_inode(file); + struct xfs_inode *ip = XFS_I(inode); + long error; + uint iolock = XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL; + + xfs_ilock(ip, iolock); + error = xfs_break_layouts(inode, &iolock, BREAK_UNMAP); + if (error) + goto out_unlock; + + /* + * Must wait for all AIO to complete before we continue as AIO can + * change the file size on completion without holding any locks we + * currently hold. We must do this first because AIO can update both + * the on disk and in memory inode sizes, and the operations that follow + * require the in-memory size to be fully up-to-date. + */ + inode_dio_wait(inode); + + error = file_modified(file); + if (error) + goto out_unlock; + + switch (mode & FALLOC_FL_MODE_MASK) { + case FALLOC_FL_PUNCH_HOLE: + error = xfs_free_file_space(ip, offset, len, ac); + break; + case FALLOC_FL_COLLAPSE_RANGE: + error = xfs_falloc_collapse_range(file, offset, len, ac); + break; + case FALLOC_FL_INSERT_RANGE: + error = xfs_falloc_insert_range(file, offset, len); + break; + case FALLOC_FL_ZERO_RANGE: + error = xfs_falloc_zero_range(file, mode, offset, len, ac); + break; + case FALLOC_FL_UNSHARE_RANGE: + error = xfs_falloc_unshare_range(file, mode, offset, len); + break; + case FALLOC_FL_ALLOCATE_RANGE: + error = xfs_falloc_allocate_range(file, mode, offset, len); + break; + case FALLOC_FL_WRITE_ZEROES: + error = xfs_falloc_write_zeroes(file, mode, offset, len, ac); + break; + default: + error = -EOPNOTSUPP; + break; + } + + if (!error && xfs_file_sync_writes(file)) + error = xfs_log_force_inode(ip); + +out_unlock: + xfs_iunlock(ip, iolock); + return error; +} + +static long +xfs_file_zoned_fallocate( + struct file *file, + int mode, + loff_t offset, + loff_t len) +{ + struct xfs_zone_alloc_ctx ac = { }; + struct xfs_inode *ip = XFS_I(file_inode(file)); + struct xfs_mount *mp = ip->i_mount; + xfs_filblks_t count_fsb; + int error; + + /* + * If full zeroing is forced by the error injection knob, we need a + * space reservation that covers the entire range. See the comment in + * xfs_zoned_write_space_reserve for the rationale for the calculation. + * Otherwise just reserve space for the two boundary blocks. + */ + count_fsb = XFS_ZONED_ZERO_EDGE_SPACE_RES; + if ((mode & FALLOC_FL_MODE_MASK) == FALLOC_FL_ZERO_RANGE && + XFS_TEST_ERROR(mp, XFS_ERRTAG_FORCE_ZERO_RANGE)) + count_fsb += XFS_B_TO_FSB(mp, len) + 1; + + error = xfs_zoned_space_reserve(mp, count_fsb, XFS_ZR_RESERVED, &ac); + if (error) + return error; + error = __xfs_file_fallocate(file, mode, offset, len, &ac); + xfs_zoned_space_unreserve(mp, &ac); + return error; +} + +static long +xfs_file_fallocate( + struct file *file, + int mode, + loff_t offset, + loff_t len) +{ + struct inode *inode = file_inode(file); + + if (!S_ISREG(inode->i_mode)) + return -EINVAL; + if (mode & ~XFS_FALLOC_FL_SUPPORTED) + return -EOPNOTSUPP; + + /* + * For zoned file systems, zeroing the first and last block of a hole + * punch requires allocating a new block to rewrite the remaining data + * and new zeroes out of place. Get a reservations for those before + * taking the iolock. Dip into the reserved pool because we are + * expected to be able to punch a hole even on a completely full + * file system. + */ + if (xfs_is_zoned_inode(XFS_I(inode)) && + (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE | + FALLOC_FL_COLLAPSE_RANGE))) + return xfs_file_zoned_fallocate(file, mode, offset, len); + return __xfs_file_fallocate(file, mode, offset, len, NULL); +} + +STATIC int +xfs_file_fadvise( + struct file *file, + loff_t start, + loff_t end, + int advice) +{ + struct xfs_inode *ip = XFS_I(file_inode(file)); + int ret; + int lockflags = 0; + + /* + * Operations creating pages in page cache need protection from hole + * punching and similar ops + */ + if (advice == POSIX_FADV_WILLNEED) { + lockflags = XFS_IOLOCK_SHARED; + xfs_ilock(ip, lockflags); + } + ret = generic_fadvise(file, start, end, advice); + if (lockflags) + xfs_iunlock(ip, lockflags); + return ret; +} + +STATIC loff_t +xfs_file_remap_range( + struct file *file_in, + loff_t pos_in, + struct file *file_out, + loff_t pos_out, + loff_t len, + unsigned int remap_flags) +{ + struct inode *inode_in = file_inode(file_in); + struct xfs_inode *src = XFS_I(inode_in); + struct inode *inode_out = file_inode(file_out); + struct xfs_inode *dest = XFS_I(inode_out); + struct xfs_mount *mp = src->i_mount; + loff_t remapped = 0; + xfs_extlen_t cowextsize; + int ret; + + if (remap_flags & ~(REMAP_FILE_DEDUP | REMAP_FILE_ADVISORY)) + return -EINVAL; + + if (!xfs_has_reflink(mp)) + return -EOPNOTSUPP; + + if (xfs_is_shutdown(mp)) + return -EIO; + + /* Prepare and then clone file data. */ + ret = xfs_reflink_remap_prep(file_in, pos_in, file_out, pos_out, + &len, remap_flags); + if (ret || len == 0) + return ret; + + trace_xfs_reflink_remap_range(src, pos_in, len, dest, pos_out); + + ret = xfs_reflink_remap_blocks(src, pos_in, dest, pos_out, len, + &remapped); + if (ret) + goto out_unlock; + + /* + * Carry the cowextsize hint from src to dest if we're sharing the + * entire source file to the entire destination file, the source file + * has a cowextsize hint, and the destination file does not. + */ + cowextsize = 0; + if (pos_in == 0 && len == i_size_read(inode_in) && + (src->i_diflags2 & XFS_DIFLAG2_COWEXTSIZE) && + pos_out == 0 && len >= i_size_read(inode_out) && + !(dest->i_diflags2 & XFS_DIFLAG2_COWEXTSIZE)) + cowextsize = src->i_cowextsize; + + ret = xfs_reflink_update_dest(dest, pos_out + len, cowextsize, + remap_flags); + if (ret) + goto out_unlock; + + if (xfs_file_sync_writes(file_in) || xfs_file_sync_writes(file_out)) + xfs_log_force_inode(dest); +out_unlock: + xfs_iunlock2_remapping(src, dest); + if (ret) + trace_xfs_reflink_remap_range_error(dest, ret, _RET_IP_); + /* + * If the caller did not set CAN_SHORTEN, then it is not prepared to + * handle partial results -- either the whole remap succeeds, or we + * must say why it did not. In this case, any error should be returned + * to the caller. + */ + if (ret && remapped < len && !(remap_flags & REMAP_FILE_CAN_SHORTEN)) + return ret; + return remapped > 0 ? remapped : ret; +} + +STATIC int +xfs_file_open( + struct inode *inode, + struct file *file) +{ + if (xfs_is_shutdown(XFS_M(inode->i_sb))) + return -EIO; + file->f_mode |= FMODE_NOWAIT | FMODE_CAN_ODIRECT; + if (xfs_get_atomic_write_min(XFS_I(inode)) > 0) + file->f_mode |= FMODE_CAN_ATOMIC_WRITE; + return generic_file_open(inode, file); +} + +STATIC int +xfs_dir_open( + struct inode *inode, + struct file *file) +{ + struct xfs_inode *ip = XFS_I(inode); + unsigned int mode; + int error; + + if (xfs_is_shutdown(ip->i_mount)) + return -EIO; + error = generic_file_open(inode, file); + if (error) + return error; + + /* + * If there are any blocks, read-ahead block 0 as we're almost + * certain to have the next operation be a read there. + */ + mode = xfs_ilock_data_map_shared(ip); + if (ip->i_df.if_nextents > 0) + error = xfs_dir3_data_readahead(ip, 0, 0); + xfs_iunlock(ip, mode); + return error; +} + +/* + * Don't bother propagating errors. We're just doing cleanup, and the caller + * ignores the return value anyway. + */ +STATIC int +xfs_file_release( + struct inode *inode, + struct file *file) +{ + struct xfs_inode *ip = XFS_I(inode); + struct xfs_mount *mp = ip->i_mount; + + /* + * If this is a read-only mount or the file system has been shut down, + * don't generate I/O. + */ + if (xfs_is_readonly(mp) || xfs_is_shutdown(mp)) + return 0; + + /* + * If we previously truncated this file and removed old data in the + * process, we want to initiate "early" writeout on the last close. + * This is an attempt to combat the notorious NULL files problem which + * is particularly noticeable from a truncate down, buffered (re-)write + * (delalloc), followed by a crash. What we are effectively doing here + * is significantly reducing the time window where we'd otherwise be + * exposed to that problem. + */ + if (xfs_iflags_test_and_clear(ip, XFS_ITRUNCATED)) { + xfs_iflags_clear(ip, XFS_EOFBLOCKS_RELEASED); + if (ip->i_delayed_blks > 0) + filemap_flush(inode->i_mapping); + } + + /* + * XFS aggressively preallocates post-EOF space to generate contiguous + * allocations for writers that append to the end of the file. + * + * To support workloads that close and reopen the file frequently, these + * preallocations usually persist after a close unless it is the first + * close for the inode. This is a tradeoff to generate tightly packed + * data layouts for unpacking tarballs or similar archives that write + * one file after another without going back to it while keeping the + * preallocation for files that have recurring open/write/close cycles. + * + * This heuristic is skipped for inodes with the append-only flag as + * that flag is rather pointless for inodes written only once. + * + * There is no point in freeing blocks here for open but unlinked files + * as they will be taken care of by the inactivation path soon. + * + * When releasing a read-only context, don't flush data or trim post-EOF + * blocks. This avoids open/read/close workloads from removing EOF + * blocks that other writers depend upon to reduce fragmentation. + * + * Inodes on the zoned RT device never have preallocations, so skip + * taking the locks below. + */ + if (!inode->i_nlink || + !(file->f_mode & FMODE_WRITE) || + (ip->i_diflags & XFS_DIFLAG_APPEND) || + xfs_is_zoned_inode(ip)) + return 0; + + /* + * If we can't get the iolock just skip truncating the blocks past EOF + * because we could deadlock with the mmap_lock otherwise. We'll get + * another chance to drop them once the last reference to the inode is + * dropped, so we'll never leak blocks permanently. + */ + if (!xfs_iflags_test(ip, XFS_EOFBLOCKS_RELEASED) && + xfs_ilock_nowait(ip, XFS_IOLOCK_EXCL)) { + if (xfs_can_free_eofblocks(ip) && + !xfs_iflags_test_and_set(ip, XFS_EOFBLOCKS_RELEASED)) + xfs_free_eofblocks(ip); + xfs_iunlock(ip, XFS_IOLOCK_EXCL); + } + + return 0; +} + +STATIC int +xfs_file_readdir( + struct file *file, + struct dir_context *ctx) +{ + struct inode *inode = file_inode(file); + xfs_inode_t *ip = XFS_I(inode); + size_t bufsize; + + /* + * The Linux API doesn't pass down the total size of the buffer + * we read into down to the filesystem. With the filldir concept + * it's not needed for correct information, but the XFS dir2 leaf + * code wants an estimate of the buffer size to calculate it's + * readahead window and size the buffers used for mapping to + * physical blocks. + * + * Try to give it an estimate that's good enough, maybe at some + * point we can change the ->readdir prototype to include the + * buffer size. For now we use the current glibc buffer size. + */ + bufsize = (size_t)min_t(loff_t, XFS_READDIR_BUFSIZE, ip->i_disk_size); + + return xfs_readdir(NULL, ip, ctx, bufsize); +} + +STATIC loff_t +xfs_file_llseek( + struct file *file, + loff_t offset, + int whence) +{ + struct inode *inode = file->f_mapping->host; + + if (xfs_is_shutdown(XFS_I(inode)->i_mount)) + return -EIO; + + switch (whence) { + default: + return generic_file_llseek(file, offset, whence); + case SEEK_HOLE: + offset = iomap_seek_hole(inode, offset, &xfs_seek_iomap_ops); + break; + case SEEK_DATA: + offset = iomap_seek_data(inode, offset, &xfs_seek_iomap_ops); + break; + } + + if (offset < 0) + return offset; + return vfs_setpos(file, offset, inode->i_sb->s_maxbytes); +} + +static inline vm_fault_t +xfs_dax_fault_locked( + struct vm_fault *vmf, + unsigned int order, + bool write_fault) +{ + vm_fault_t ret; + unsigned long pfn; + + if (!IS_ENABLED(CONFIG_FS_DAX)) { + ASSERT(0); + return VM_FAULT_SIGBUS; + } + ret = dax_iomap_fault(vmf, order, &pfn, NULL, + (write_fault && !vmf->cow_page) ? + &xfs_dax_write_iomap_ops : + &xfs_read_iomap_ops); + if (ret & VM_FAULT_NEEDDSYNC) + ret = dax_finish_sync_fault(vmf, order, pfn); + return ret; +} + +static vm_fault_t +xfs_dax_read_fault( + struct vm_fault *vmf, + unsigned int order) +{ + struct xfs_inode *ip = XFS_I(file_inode(vmf->vma->vm_file)); + vm_fault_t ret; + + trace_xfs_read_fault(ip, order); + + xfs_ilock(ip, XFS_MMAPLOCK_SHARED); + ret = xfs_dax_fault_locked(vmf, order, false); + xfs_iunlock(ip, XFS_MMAPLOCK_SHARED); + + return ret; +} + +/* + * Locking for serialisation of IO during page faults. This results in a lock + * ordering of: + * + * mmap_lock (MM) + * sb_start_pagefault(vfs, freeze) + * invalidate_lock (vfs/XFS_MMAPLOCK - truncate serialisation) + * page_lock (MM) + * i_lock (XFS - extent map serialisation) + */ +static vm_fault_t +__xfs_write_fault( + struct vm_fault *vmf, + unsigned int order, + struct xfs_zone_alloc_ctx *ac) +{ + struct inode *inode = file_inode(vmf->vma->vm_file); + struct xfs_inode *ip = XFS_I(inode); + unsigned int lock_mode = XFS_MMAPLOCK_SHARED; + vm_fault_t ret; + + trace_xfs_write_fault(ip, order); + + sb_start_pagefault(inode->i_sb); + file_update_time(vmf->vma->vm_file); + + /* + * Normally we only need the shared mmaplock, but if a reflink remap is + * in progress we take the exclusive lock to wait for the remap to + * finish before taking a write fault. + */ + xfs_ilock(ip, XFS_MMAPLOCK_SHARED); + if (xfs_iflags_test(ip, XFS_IREMAPPING)) { + xfs_iunlock(ip, XFS_MMAPLOCK_SHARED); + xfs_ilock(ip, XFS_MMAPLOCK_EXCL); + lock_mode = XFS_MMAPLOCK_EXCL; + } + + if (IS_DAX(inode)) + ret = xfs_dax_fault_locked(vmf, order, true); + else + ret = iomap_page_mkwrite(vmf, &xfs_buffered_write_iomap_ops, + ac); + xfs_iunlock(ip, lock_mode); + + sb_end_pagefault(inode->i_sb); + return ret; +} + +static vm_fault_t +xfs_write_fault_zoned( + struct vm_fault *vmf, + unsigned int order) +{ + struct xfs_inode *ip = XFS_I(file_inode(vmf->vma->vm_file)); + unsigned int len = folio_size(page_folio(vmf->page)); + struct xfs_zone_alloc_ctx ac = { }; + int error; + vm_fault_t ret; + + /* + * This could over-allocate as it doesn't check for truncation. + * + * But as the overallocation is limited to less than a folio and will be + * release instantly that's just fine. + */ + error = xfs_zoned_space_reserve(ip->i_mount, + XFS_B_TO_FSB(ip->i_mount, len), 0, &ac); + if (error < 0) + return vmf_fs_error(error); + ret = __xfs_write_fault(vmf, order, &ac); + xfs_zoned_space_unreserve(ip->i_mount, &ac); + return ret; +} + +static vm_fault_t +xfs_write_fault( + struct vm_fault *vmf, + unsigned int order) +{ + if (xfs_is_zoned_inode(XFS_I(file_inode(vmf->vma->vm_file)))) + return xfs_write_fault_zoned(vmf, order); + return __xfs_write_fault(vmf, order, NULL); +} + +static inline bool +xfs_is_write_fault( + struct vm_fault *vmf) +{ + return (vmf->flags & FAULT_FLAG_WRITE) && + (vmf->vma->vm_flags & VM_SHARED); +} + +static vm_fault_t +xfs_filemap_fault( + struct vm_fault *vmf) +{ + struct inode *inode = file_inode(vmf->vma->vm_file); + + /* DAX can shortcut the normal fault path on write faults! */ + if (IS_DAX(inode)) { + if (xfs_is_write_fault(vmf)) + return xfs_write_fault(vmf, 0); + return xfs_dax_read_fault(vmf, 0); + } + + trace_xfs_read_fault(XFS_I(inode), 0); + return filemap_fault(vmf); +} + +static vm_fault_t +xfs_filemap_huge_fault( + struct vm_fault *vmf, + unsigned int order) +{ + if (!IS_DAX(file_inode(vmf->vma->vm_file))) + return VM_FAULT_FALLBACK; + + /* DAX can shortcut the normal fault path on write faults! */ + if (xfs_is_write_fault(vmf)) + return xfs_write_fault(vmf, order); + return xfs_dax_read_fault(vmf, order); +} + +static vm_fault_t +xfs_filemap_page_mkwrite( + struct vm_fault *vmf) +{ + return xfs_write_fault(vmf, 0); +} + +/* + * pfn_mkwrite was originally intended to ensure we capture time stamp updates + * on write faults. In reality, it needs to serialise against truncate and + * prepare memory for writing so handle is as standard write fault. + */ +static vm_fault_t +xfs_filemap_pfn_mkwrite( + struct vm_fault *vmf) +{ + return xfs_write_fault(vmf, 0); +} + +static const struct vm_operations_struct xfs_file_vm_ops = { + .fault = xfs_filemap_fault, + .huge_fault = xfs_filemap_huge_fault, + .map_pages = filemap_map_pages, + .page_mkwrite = xfs_filemap_page_mkwrite, + .pfn_mkwrite = xfs_filemap_pfn_mkwrite, +}; + +STATIC int +xfs_file_mmap_prepare( + struct vm_area_desc *desc) +{ + struct file *file = desc->file; + struct inode *inode = file_inode(file); + struct xfs_buftarg *target = xfs_inode_buftarg(XFS_I(inode)); + + /* + * We don't support synchronous mappings for non-DAX files and + * for DAX files if underneath dax_device is not synchronous. + */ + if (!daxdev_mapping_supported(desc, file_inode(file), + target->bt_daxdev)) + return -EOPNOTSUPP; + + file_accessed(file); + desc->vm_ops = &xfs_file_vm_ops; + if (IS_DAX(inode)) + vma_desc_set_flags(desc, VMA_HUGEPAGE_BIT); + return 0; +} + +const struct file_operations xfs_file_operations = { + .llseek = xfs_file_llseek, + .read_iter = xfs_file_read_iter, + .write_iter = xfs_file_write_iter, + .splice_read = xfs_file_splice_read, + .splice_write = iter_file_splice_write, + .iopoll = iocb_bio_iopoll, + .unlocked_ioctl = xfs_file_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = xfs_file_compat_ioctl, +#endif + .mmap_prepare = xfs_file_mmap_prepare, + .open = xfs_file_open, + .release = xfs_file_release, + .fsync = xfs_file_fsync, + .get_unmapped_area = thp_get_unmapped_area, + .fallocate = xfs_file_fallocate, + .fadvise = xfs_file_fadvise, + .remap_file_range = xfs_file_remap_range, + .fop_flags = FOP_MMAP_SYNC | FOP_BUFFER_RASYNC | + FOP_BUFFER_WASYNC | FOP_DIO_PARALLEL_WRITE | + FOP_DONTCACHE, + .setlease = generic_setlease, +}; + +const struct file_operations xfs_dir_file_operations = { + .open = xfs_dir_open, + .read = generic_read_dir, + .iterate_shared = xfs_file_readdir, + .llseek = generic_file_llseek, + .unlocked_ioctl = xfs_file_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = xfs_file_compat_ioctl, +#endif + .fsync = xfs_dir_fsync, + .setlease = generic_setlease, +}; diff --git a/fs/xfs/xfs_file.h b/fs/xfs/xfs_file.h new file mode 100644 index 000000000..2ad91f755 --- /dev/null +++ b/fs/xfs/xfs_file.h @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_FILE_H__ +#define __XFS_FILE_H__ + +extern const struct file_operations xfs_file_operations; +extern const struct file_operations xfs_dir_file_operations; + +bool xfs_is_falloc_aligned(struct xfs_inode *ip, loff_t pos, + long long int len); + +#endif /* __XFS_FILE_H__ */ diff --git a/fs/xfs/xfs_filestream.c b/fs/xfs/xfs_filestream.c new file mode 100644 index 000000000..bb4685207 --- /dev/null +++ b/fs/xfs/xfs_filestream.c @@ -0,0 +1,396 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2006-2007 Silicon Graphics, Inc. + * Copyright (c) 2014 Christoph Hellwig. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_bmap.h" +#include "xfs_bmap_util.h" +#include "xfs_alloc.h" +#include "xfs_mru_cache.h" +#include "xfs_trace.h" +#include "xfs_ag.h" +#include "xfs_ag_resv.h" +#include "xfs_trans.h" +#include "xfs_filestream.h" + +struct xfs_fstrm_item { + struct xfs_mru_cache_elem mru; + struct xfs_perag *pag; /* AG in use for this directory */ +}; + +enum xfs_fstrm_alloc { + XFS_PICK_USERDATA = 1, + XFS_PICK_LOWSPACE = 2, +}; + +static void +xfs_fstrm_free_func( + void *data, + struct xfs_mru_cache_elem *mru) +{ + struct xfs_fstrm_item *item = + container_of(mru, struct xfs_fstrm_item, mru); + struct xfs_perag *pag = item->pag; + + trace_xfs_filestream_free(pag, mru->key); + atomic_dec(&pag->pagf_fstrms); + xfs_perag_rele(pag); + + kfree(item); +} + +/* + * Scan the AGs starting at start_agno looking for an AG that isn't in use and + * has at least minlen blocks free. If no AG is found to match the allocation + * requirements, pick the AG with the most free space in it. + */ +static int +xfs_filestream_pick_ag( + struct xfs_alloc_arg *args, + xfs_ino_t pino, + xfs_agnumber_t start_agno, + int flags, + xfs_extlen_t *longest) +{ + struct xfs_mount *mp = args->mp; + struct xfs_perag *pag; + struct xfs_perag *max_pag = NULL; + xfs_extlen_t minlen = *longest; + xfs_extlen_t minfree, maxfree = 0; + xfs_agnumber_t agno; + bool first_pass = true; + + /* 2% of an AG's blocks must be free for it to be chosen. */ + minfree = mp->m_sb.sb_agblocks / 50; + +restart: + for_each_perag_wrap(mp, start_agno, agno, pag) { + int err; + + trace_xfs_filestream_scan(pag, pino); + + *longest = 0; + err = xfs_bmap_longest_free_extent(pag, NULL, longest); + if (err) { + if (err == -EAGAIN) { + /* Couldn't lock the AGF, skip this AG. */ + err = 0; + continue; + } + xfs_perag_rele(pag); + if (max_pag) + xfs_perag_rele(max_pag); + return err; + } + + /* Keep track of the AG with the most free blocks. */ + if (pag->pagf_freeblks > maxfree) { + maxfree = pag->pagf_freeblks; + if (max_pag) + xfs_perag_rele(max_pag); + atomic_inc(&pag_group(pag)->xg_active_ref); + max_pag = pag; + } + + /* + * The AG reference count does two things: it enforces mutual + * exclusion when examining the suitability of an AG in this + * loop, and it guards against two filestreams being established + * in the same AG as each other. + */ + if (atomic_inc_return(&pag->pagf_fstrms) <= 1) { + if (((minlen && *longest >= minlen) || + (!minlen && pag->pagf_freeblks >= minfree)) && + (!xfs_perag_prefers_metadata(pag) || + !(flags & XFS_PICK_USERDATA) || + (flags & XFS_PICK_LOWSPACE))) { + /* Break out, retaining the reference on the AG. */ + if (max_pag) + xfs_perag_rele(max_pag); + goto done; + } + } + + /* Drop the reference on this AG, it's not usable. */ + atomic_dec(&pag->pagf_fstrms); + } + + /* + * Allow a second pass to give xfs_bmap_longest_free_extent() another + * attempt at locking AGFs that it might have skipped over before we + * fail. + */ + if (first_pass) { + first_pass = false; + goto restart; + } + + /* + * We must be low on data space, so run a final lowspace optimised + * selection pass if we haven't already. + */ + if (!(flags & XFS_PICK_LOWSPACE)) { + flags |= XFS_PICK_LOWSPACE; + goto restart; + } + + /* + * No unassociated AGs are available, so select the AG with the most + * free space, regardless of whether it's already in use by another + * filestream. It none suit, just use whatever AG we can grab. + */ + if (!max_pag) { + for_each_perag_wrap(args->mp, 0, start_agno, pag) { + max_pag = pag; + break; + } + + /* Bail if there are no AGs at all to select from. */ + if (!max_pag) + return -ENOSPC; + } + + pag = max_pag; + atomic_inc(&pag->pagf_fstrms); +done: + trace_xfs_filestream_pick(pag, pino); + args->pag = pag; + return 0; +} + +static struct xfs_inode * +xfs_filestream_get_parent( + struct xfs_inode *ip) +{ + struct inode *inode = VFS_I(ip), *dir = NULL; + struct dentry *dentry, *parent; + + dentry = d_find_alias(inode); + if (!dentry) + goto out; + + parent = dget_parent(dentry); + if (!parent) + goto out_dput; + + dir = igrab(d_inode(parent)); + dput(parent); + +out_dput: + dput(dentry); +out: + return dir ? XFS_I(dir) : NULL; +} + +/* + * Lookup the mru cache for an existing association. If one exists and we can + * use it, return with an active perag reference indicating that the allocation + * will proceed with that association. + * + * If we have no association, or we cannot use the current one and have to + * destroy it, return with longest = 0 to tell the caller to create a new + * association. + */ +static int +xfs_filestream_lookup_association( + struct xfs_bmalloca *ap, + struct xfs_alloc_arg *args, + xfs_ino_t pino, + xfs_extlen_t *longest) +{ + struct xfs_mount *mp = args->mp; + struct xfs_perag *pag; + struct xfs_mru_cache_elem *mru; + int error = 0; + + *longest = 0; + mru = xfs_mru_cache_lookup(mp->m_filestream, pino); + if (!mru) + return 0; + /* + * Grab the pag and take an extra active reference for the caller whilst + * the mru item cannot go away. This means we'll pin the perag with + * the reference we get here even if the filestreams association is torn + * down immediately after we mark the lookup as done. + */ + pag = container_of(mru, struct xfs_fstrm_item, mru)->pag; + atomic_inc(&pag_group(pag)->xg_active_ref); + xfs_mru_cache_done(mp->m_filestream); + + trace_xfs_filestream_lookup(pag, I_INO(ap->ip)); + + ap->blkno = xfs_agbno_to_fsb(pag, 0); + xfs_bmap_adjacent(ap); + + /* + * If there is very little free space before we start a filestreams + * allocation, we're almost guaranteed to fail to find a large enough + * free space available so just use the cached AG. + */ + if (ap->tp->t_flags & XFS_TRANS_LOWMODE) { + *longest = 1; + goto out_done; + } + + error = xfs_bmap_longest_free_extent(pag, args->tp, longest); + if (error == -EAGAIN) + error = 0; + if (error || *longest < args->maxlen) { + /* We aren't going to use this perag */ + *longest = 0; + xfs_perag_rele(pag); + return error; + } + +out_done: + args->pag = pag; + return 0; +} + +static int +xfs_filestream_create_association( + struct xfs_bmalloca *ap, + struct xfs_alloc_arg *args, + xfs_ino_t pino, + xfs_extlen_t *longest) +{ + struct xfs_mount *mp = args->mp; + struct xfs_mru_cache_elem *mru; + struct xfs_fstrm_item *item; + xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, pino); + int flags = 0; + int error; + + /* Changing parent AG association now, so remove the existing one. */ + mru = xfs_mru_cache_remove(mp->m_filestream, pino); + if (mru) { + struct xfs_fstrm_item *item = + container_of(mru, struct xfs_fstrm_item, mru); + + agno = (pag_agno(item->pag) + 1) % mp->m_sb.sb_agcount; + xfs_fstrm_free_func(mp, mru); + } else if (xfs_is_inode32(mp)) { + xfs_agnumber_t rotorstep = xfs_rotorstep; + + agno = (mp->m_agfrotor / rotorstep) % mp->m_sb.sb_agcount; + mp->m_agfrotor = (mp->m_agfrotor + 1) % + (mp->m_sb.sb_agcount * rotorstep); + } + + ap->blkno = XFS_AGB_TO_FSB(args->mp, agno, 0); + xfs_bmap_adjacent(ap); + + if (ap->datatype & XFS_ALLOC_USERDATA) + flags |= XFS_PICK_USERDATA; + if (ap->tp->t_flags & XFS_TRANS_LOWMODE) + flags |= XFS_PICK_LOWSPACE; + + *longest = ap->length; + error = xfs_filestream_pick_ag(args, pino, agno, flags, longest); + if (error) + return error; + + /* + * We are going to use this perag now, so create an assoication for it. + * xfs_filestream_pick_ag() has already bumped the perag fstrms counter + * for us, so all we need to do here is take another active reference to + * the perag for the cached association. + * + * If we fail to store the association, we do not need to return an + * error for this failure - as long as we return a referenced AG, the + * allocation can still go ahead just fine. + */ + item = kmalloc_obj(*item, GFP_KERNEL | __GFP_RETRY_MAYFAIL); + if (!item) + goto out_put_fstrms; + + atomic_inc(&pag_group(args->pag)->xg_active_ref); + item->pag = args->pag; + xfs_mru_cache_insert(mp->m_filestream, pino, &item->mru); + return 0; + +out_put_fstrms: + atomic_dec(&args->pag->pagf_fstrms); + return 0; +} + +/* + * Search for an allocation group with a single extent large enough for + * the request. First we look for an existing association and use that if it + * is found. Otherwise, we create a new association by selecting an AG that fits + * the allocation criteria. + * + * We return with a referenced perag in args->pag to indicate which AG we are + * allocating into or an error with no references held. + */ +int +xfs_filestream_select_ag( + struct xfs_bmalloca *ap, + struct xfs_alloc_arg *args, + xfs_extlen_t *longest) +{ + struct xfs_inode *pip; + xfs_ino_t ino = 0; + int error = 0; + + *longest = 0; + args->total = ap->total; + pip = xfs_filestream_get_parent(ap->ip); + if (pip) { + ino = I_INO(pip); + error = xfs_filestream_lookup_association(ap, args, ino, + longest); + xfs_irele(pip); + if (error) + return error; + if (*longest >= args->maxlen) + goto out_select; + if (ap->tp->t_flags & XFS_TRANS_LOWMODE) + goto out_select; + } + + error = xfs_filestream_create_association(ap, args, ino, longest); + if (error) + return error; + +out_select: + ap->blkno = xfs_agbno_to_fsb(args->pag, 0); + return 0; +} + +void +xfs_filestream_deassociate( + struct xfs_inode *ip) +{ + xfs_mru_cache_delete(ip->i_mount->m_filestream, I_INO(ip)); +} + +int +xfs_filestream_mount( + xfs_mount_t *mp) +{ + /* + * The filestream timer tunable is currently fixed within the range of + * one second to four minutes, with five seconds being the default. The + * group count is somewhat arbitrary, but it'd be nice to adhere to the + * timer tunable to within about 10 percent. This requires at least 10 + * groups. + */ + return xfs_mru_cache_create(&mp->m_filestream, mp, + xfs_fstrm_centisecs * 10, 10, xfs_fstrm_free_func); +} + +void +xfs_filestream_unmount( + xfs_mount_t *mp) +{ + xfs_mru_cache_destroy(mp->m_filestream); +} diff --git a/fs/xfs/xfs_filestream.h b/fs/xfs/xfs_filestream.h new file mode 100644 index 000000000..84149ed0e --- /dev/null +++ b/fs/xfs/xfs_filestream.h @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2006-2007 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_FILESTREAM_H__ +#define __XFS_FILESTREAM_H__ + +struct xfs_mount; +struct xfs_inode; +struct xfs_bmalloca; +struct xfs_alloc_arg; + +int xfs_filestream_mount(struct xfs_mount *mp); +void xfs_filestream_unmount(struct xfs_mount *mp); +void xfs_filestream_deassociate(struct xfs_inode *ip); +int xfs_filestream_select_ag(struct xfs_bmalloca *ap, + struct xfs_alloc_arg *args, xfs_extlen_t *blen); + +static inline int +xfs_inode_is_filestream( + struct xfs_inode *ip) +{ + return xfs_has_filestreams(ip->i_mount) || + (ip->i_diflags & XFS_DIFLAG_FILESTREAM); +} + +#endif /* __XFS_FILESTREAM_H__ */ diff --git a/fs/xfs/xfs_fsmap.c b/fs/xfs/xfs_fsmap.c new file mode 100644 index 000000000..b6a3bc9f1 --- /dev/null +++ b/fs/xfs/xfs_fsmap.c @@ -0,0 +1,1428 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2017 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_btree.h" +#include "xfs_rmap_btree.h" +#include "xfs_trace.h" +#include "xfs_rmap.h" +#include "xfs_alloc.h" +#include "xfs_bit.h" +#include +#include "xfs_fsmap.h" +#include "xfs_refcount.h" +#include "xfs_refcount_btree.h" +#include "xfs_alloc_btree.h" +#include "xfs_rtbitmap.h" +#include "xfs_ag.h" +#include "xfs_rtgroup.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_rtrefcount_btree.h" + +/* Convert an xfs_fsmap to an fsmap. */ +static void +xfs_fsmap_from_internal( + struct fsmap *dest, + struct xfs_fsmap *src) +{ + dest->fmr_device = src->fmr_device; + dest->fmr_flags = src->fmr_flags; + dest->fmr_physical = BBTOB(src->fmr_physical); + dest->fmr_owner = src->fmr_owner; + dest->fmr_offset = BBTOB(src->fmr_offset); + dest->fmr_length = BBTOB(src->fmr_length); + dest->fmr_reserved[0] = 0; + dest->fmr_reserved[1] = 0; + dest->fmr_reserved[2] = 0; +} + +/* Convert an fsmap to an xfs_fsmap. */ +static void +xfs_fsmap_to_internal( + struct xfs_fsmap *dest, + struct fsmap *src) +{ + dest->fmr_device = src->fmr_device; + dest->fmr_flags = src->fmr_flags; + dest->fmr_physical = BTOBBT(src->fmr_physical); + dest->fmr_owner = src->fmr_owner; + dest->fmr_offset = BTOBBT(src->fmr_offset); + dest->fmr_length = BTOBBT(src->fmr_length); +} + +/* Convert an fsmap owner into an rmapbt owner. */ +static int +xfs_fsmap_owner_to_rmap( + struct xfs_rmap_irec *dest, + const struct xfs_fsmap *src) +{ + if (!(src->fmr_flags & FMR_OF_SPECIAL_OWNER)) { + dest->rm_owner = src->fmr_owner; + return 0; + } + + switch (src->fmr_owner) { + case 0: /* "lowest owner id possible" */ + case -1ULL: /* "highest owner id possible" */ + dest->rm_owner = src->fmr_owner; + break; + case XFS_FMR_OWN_FREE: + dest->rm_owner = XFS_RMAP_OWN_NULL; + break; + case XFS_FMR_OWN_UNKNOWN: + dest->rm_owner = XFS_RMAP_OWN_UNKNOWN; + break; + case XFS_FMR_OWN_FS: + dest->rm_owner = XFS_RMAP_OWN_FS; + break; + case XFS_FMR_OWN_LOG: + dest->rm_owner = XFS_RMAP_OWN_LOG; + break; + case XFS_FMR_OWN_AG: + dest->rm_owner = XFS_RMAP_OWN_AG; + break; + case XFS_FMR_OWN_INOBT: + dest->rm_owner = XFS_RMAP_OWN_INOBT; + break; + case XFS_FMR_OWN_INODES: + dest->rm_owner = XFS_RMAP_OWN_INODES; + break; + case XFS_FMR_OWN_REFC: + dest->rm_owner = XFS_RMAP_OWN_REFC; + break; + case XFS_FMR_OWN_COW: + dest->rm_owner = XFS_RMAP_OWN_COW; + break; + case XFS_FMR_OWN_DEFECTIVE: /* not implemented */ + /* fall through */ + default: + return -EINVAL; + } + return 0; +} + +/* Convert an rmapbt owner into an fsmap owner. */ +static int +xfs_fsmap_owner_from_frec( + struct xfs_fsmap *dest, + const struct xfs_fsmap_irec *frec) +{ + dest->fmr_flags = 0; + if (!XFS_RMAP_NON_INODE_OWNER(frec->owner)) { + dest->fmr_owner = frec->owner; + return 0; + } + dest->fmr_flags |= FMR_OF_SPECIAL_OWNER; + + switch (frec->owner) { + case XFS_RMAP_OWN_FS: + dest->fmr_owner = XFS_FMR_OWN_FS; + break; + case XFS_RMAP_OWN_LOG: + dest->fmr_owner = XFS_FMR_OWN_LOG; + break; + case XFS_RMAP_OWN_AG: + dest->fmr_owner = XFS_FMR_OWN_AG; + break; + case XFS_RMAP_OWN_INOBT: + dest->fmr_owner = XFS_FMR_OWN_INOBT; + break; + case XFS_RMAP_OWN_INODES: + dest->fmr_owner = XFS_FMR_OWN_INODES; + break; + case XFS_RMAP_OWN_REFC: + dest->fmr_owner = XFS_FMR_OWN_REFC; + break; + case XFS_RMAP_OWN_COW: + dest->fmr_owner = XFS_FMR_OWN_COW; + break; + case XFS_RMAP_OWN_NULL: /* "free" */ + dest->fmr_owner = XFS_FMR_OWN_FREE; + break; + default: + ASSERT(0); + return -EFSCORRUPTED; + } + return 0; +} + +/* getfsmap query state */ +struct xfs_getfsmap_info { + struct xfs_fsmap_head *head; + struct fsmap *fsmap_recs; /* mapping records */ + struct xfs_buf *agf_bp; /* AGF, for refcount queries */ + struct xfs_group *group; /* group info, if applicable */ + xfs_daddr_t next_daddr; /* next daddr we expect */ + /* daddr of low fsmap key when we're using the rtbitmap */ + xfs_daddr_t low_daddr; + /* daddr of high fsmap key, or the last daddr on the device */ + xfs_daddr_t end_daddr; + u64 missing_owner; /* owner of holes */ + u32 dev; /* device id */ + /* + * Low rmap key for the query. If low.rm_blockcount is nonzero, this + * is the second (or later) call to retrieve the recordset in pieces. + * xfs_getfsmap_rec_before_start will compare all records retrieved + * by the rmapbt query to filter out any records that start before + * the last record. + */ + struct xfs_rmap_irec low; + struct xfs_rmap_irec high; /* high rmap key */ + bool last; /* last extent? */ +}; + +/* Associate a device with a getfsmap handler. */ +struct xfs_getfsmap_dev { + u32 dev; + int (*fn)(struct xfs_trans *tp, + const struct xfs_fsmap *keys, + struct xfs_getfsmap_info *info); + sector_t nr_sectors; +}; + +/* Compare two getfsmap device handlers. */ +static int +xfs_getfsmap_dev_compare( + const void *p1, + const void *p2) +{ + const struct xfs_getfsmap_dev *d1 = p1; + const struct xfs_getfsmap_dev *d2 = p2; + + return d1->dev - d2->dev; +} + +/* Decide if this mapping is shared. */ +STATIC int +xfs_getfsmap_is_shared( + struct xfs_trans *tp, + struct xfs_getfsmap_info *info, + const struct xfs_fsmap_irec *frec, + bool *stat) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_btree_cur *cur; + xfs_agblock_t fbno; + xfs_extlen_t flen = 0; + int error; + + *stat = false; + if (!xfs_has_reflink(mp) || !info->group) + return 0; + + if (info->group->xg_type == XG_TYPE_RTG) + cur = xfs_rtrefcountbt_init_cursor(tp, to_rtg(info->group)); + else + cur = xfs_refcountbt_init_cursor(mp, tp, info->agf_bp, + to_perag(info->group)); + + /* Are there any shared blocks here? */ + error = xfs_refcount_find_shared(cur, frec->rec_key, + XFS_BB_TO_FSBT(mp, frec->len_daddr), &fbno, &flen, + false); + + xfs_btree_del_cursor(cur, error); + if (error) + return error; + + *stat = flen > 0; + return 0; +} + +static inline void +xfs_getfsmap_format( + struct xfs_mount *mp, + struct xfs_fsmap *xfm, + struct xfs_getfsmap_info *info) +{ + struct fsmap *rec; + + trace_xfs_getfsmap_mapping(mp, xfm); + + rec = &info->fsmap_recs[info->head->fmh_entries++]; + xfs_fsmap_from_internal(rec, xfm); +} + +static inline bool +xfs_getfsmap_frec_before_start( + struct xfs_getfsmap_info *info, + const struct xfs_fsmap_irec *frec) +{ + if (info->low_daddr != XFS_BUF_DADDR_NULL) + return frec->start_daddr < info->low_daddr; + if (info->low.rm_blockcount) { + struct xfs_rmap_irec rec = { + .rm_startblock = frec->rec_key, + .rm_owner = frec->owner, + .rm_flags = frec->rm_flags, + }; + + return xfs_rmap_compare(&rec, &info->low) < 0; + } + + return false; +} + +/* + * Format a reverse mapping for getfsmap, having translated rm_startblock + * into the appropriate daddr units. Pass in a nonzero @len_daddr if the + * length could be larger than rm_blockcount in struct xfs_rmap_irec. + */ +STATIC int +xfs_getfsmap_helper( + struct xfs_trans *tp, + struct xfs_getfsmap_info *info, + const struct xfs_fsmap_irec *frec) +{ + struct xfs_fsmap fmr; + struct xfs_mount *mp = tp->t_mountp; + bool shared; + int error = 0; + + if (fatal_signal_pending(current)) + return -EINTR; + + /* + * Filter out records that start before our startpoint, if the + * caller requested that. + */ + if (xfs_getfsmap_frec_before_start(info, frec)) + goto out; + + /* Are we just counting mappings? */ + if (info->head->fmh_count == 0) { + if (info->head->fmh_entries == UINT_MAX) + return -ECANCELED; + + if (frec->start_daddr > info->next_daddr) + info->head->fmh_entries++; + + if (info->last) + return 0; + + info->head->fmh_entries++; + goto out; + } + + /* + * If the record starts past the last physical block we saw, + * then we've found a gap. Report the gap as being owned by + * whatever the caller specified is the missing owner. + */ + if (frec->start_daddr > info->next_daddr) { + if (info->head->fmh_entries >= info->head->fmh_count) + return -ECANCELED; + + fmr.fmr_device = info->dev; + fmr.fmr_physical = info->next_daddr; + fmr.fmr_owner = info->missing_owner; + fmr.fmr_offset = 0; + fmr.fmr_length = frec->start_daddr - info->next_daddr; + fmr.fmr_flags = FMR_OF_SPECIAL_OWNER; + xfs_getfsmap_format(mp, &fmr, info); + } + + if (info->last) + goto out; + + /* Fill out the extent we found */ + if (info->head->fmh_entries >= info->head->fmh_count) + return -ECANCELED; + + trace_xfs_fsmap_mapping(mp, info->dev, + info->group ? info->group->xg_gno : NULLAGNUMBER, + frec); + + fmr.fmr_device = info->dev; + fmr.fmr_physical = frec->start_daddr; + error = xfs_fsmap_owner_from_frec(&fmr, frec); + if (error) + return error; + fmr.fmr_offset = XFS_FSB_TO_BB(mp, frec->offset); + fmr.fmr_length = frec->len_daddr; + if (frec->rm_flags & XFS_RMAP_UNWRITTEN) + fmr.fmr_flags |= FMR_OF_PREALLOC; + if (frec->rm_flags & XFS_RMAP_ATTR_FORK) + fmr.fmr_flags |= FMR_OF_ATTR_FORK; + if (frec->rm_flags & XFS_RMAP_BMBT_BLOCK) + fmr.fmr_flags |= FMR_OF_EXTENT_MAP; + if (fmr.fmr_flags == 0) { + error = xfs_getfsmap_is_shared(tp, info, frec, &shared); + if (error) + return error; + if (shared) + fmr.fmr_flags |= FMR_OF_SHARED; + } + + xfs_getfsmap_format(mp, &fmr, info); +out: + info->next_daddr = max(info->next_daddr, + frec->start_daddr + frec->len_daddr); + return 0; +} + +static inline int +xfs_getfsmap_group_helper( + struct xfs_getfsmap_info *info, + struct xfs_trans *tp, + struct xfs_group *xg, + xfs_agblock_t startblock, + xfs_extlen_t blockcount, + struct xfs_fsmap_irec *frec) +{ + /* + * For an info->last query, we're looking for a gap between the last + * mapping emitted and the high key specified by userspace. If the + * user's query spans less than 1 fsblock, then info->high and + * info->low will have the same rm_startblock, which causes rec_daddr + * and next_daddr to be the same. Therefore, use the end_daddr that + * we calculated from userspace's high key to synthesize the record. + * Note that if the btree query found a mapping, there won't be a gap. + */ + if (info->last) + frec->start_daddr = info->end_daddr + 1; + else + frec->start_daddr = xfs_gbno_to_daddr(xg, startblock); + + frec->len_daddr = XFS_FSB_TO_BB(xg->xg_mount, blockcount); + return xfs_getfsmap_helper(tp, info, frec); +} + +/* Transform a rmapbt irec into a fsmap */ +STATIC int +xfs_getfsmap_rmapbt_helper( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv) +{ + struct xfs_fsmap_irec frec = { + .owner = rec->rm_owner, + .offset = rec->rm_offset, + .rm_flags = rec->rm_flags, + .rec_key = rec->rm_startblock, + }; + struct xfs_getfsmap_info *info = priv; + + return xfs_getfsmap_group_helper(info, cur->bc_tp, cur->bc_group, + rec->rm_startblock, rec->rm_blockcount, &frec); +} + +/* Transform a bnobt irec into a fsmap */ +STATIC int +xfs_getfsmap_datadev_bnobt_helper( + struct xfs_btree_cur *cur, + const struct xfs_alloc_rec_incore *rec, + void *priv) +{ + struct xfs_fsmap_irec frec = { + .owner = XFS_RMAP_OWN_NULL, /* "free" */ + .rec_key = rec->ar_startblock, + }; + struct xfs_getfsmap_info *info = priv; + + return xfs_getfsmap_group_helper(info, cur->bc_tp, cur->bc_group, + rec->ar_startblock, rec->ar_blockcount, &frec); +} + +/* Set rmap flags based on the getfsmap flags */ +static void +xfs_getfsmap_set_irec_flags( + struct xfs_rmap_irec *irec, + const struct xfs_fsmap *fmr) +{ + irec->rm_flags = 0; + if (fmr->fmr_flags & FMR_OF_ATTR_FORK) + irec->rm_flags |= XFS_RMAP_ATTR_FORK; + if (fmr->fmr_flags & FMR_OF_EXTENT_MAP) + irec->rm_flags |= XFS_RMAP_BMBT_BLOCK; + if (fmr->fmr_flags & FMR_OF_PREALLOC) + irec->rm_flags |= XFS_RMAP_UNWRITTEN; +} + +static inline bool +rmap_not_shareable(struct xfs_mount *mp, const struct xfs_rmap_irec *r) +{ + if (!xfs_has_reflink(mp)) + return true; + if (XFS_RMAP_NON_INODE_OWNER(r->rm_owner)) + return true; + if (r->rm_flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK | + XFS_RMAP_UNWRITTEN)) + return true; + return false; +} + +/* Execute a getfsmap query against the regular data device. */ +STATIC int +__xfs_getfsmap_datadev( + struct xfs_trans *tp, + const struct xfs_fsmap *keys, + struct xfs_getfsmap_info *info, + int (*query_fn)(struct xfs_trans *, + struct xfs_getfsmap_info *, + struct xfs_btree_cur **, + void *), + void *priv) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_perag *pag = NULL; + struct xfs_btree_cur *bt_cur = NULL; + xfs_fsblock_t start_fsb; + xfs_fsblock_t end_fsb; + xfs_agnumber_t start_ag, end_ag; + uint64_t eofs; + int error = 0; + + eofs = XFS_FSB_TO_BB(mp, mp->m_sb.sb_dblocks); + if (keys[0].fmr_physical >= eofs) + return 0; + start_fsb = XFS_DADDR_TO_FSB(mp, keys[0].fmr_physical); + end_fsb = XFS_DADDR_TO_FSB(mp, min(eofs - 1, keys[1].fmr_physical)); + + /* + * Convert the fsmap low/high keys to AG based keys. Initialize + * low to the fsmap low key and max out the high key to the end + * of the AG. + */ + info->low.rm_offset = XFS_BB_TO_FSBT(mp, keys[0].fmr_offset); + error = xfs_fsmap_owner_to_rmap(&info->low, &keys[0]); + if (error) + return error; + info->low.rm_blockcount = XFS_BB_TO_FSBT(mp, keys[0].fmr_length); + xfs_getfsmap_set_irec_flags(&info->low, &keys[0]); + + /* Adjust the low key if we are continuing from where we left off. */ + if (info->low.rm_blockcount == 0) { + /* No previous record from which to continue */ + } else if (rmap_not_shareable(mp, &info->low)) { + /* Last record seen was an unshareable extent */ + info->low.rm_owner = 0; + info->low.rm_offset = 0; + + start_fsb += info->low.rm_blockcount; + if (XFS_FSB_TO_DADDR(mp, start_fsb) >= eofs) + return 0; + } else { + /* Last record seen was a shareable file data extent */ + info->low.rm_offset += info->low.rm_blockcount; + } + info->low.rm_startblock = XFS_FSB_TO_AGBNO(mp, start_fsb); + + info->high.rm_startblock = -1U; + info->high.rm_owner = ULLONG_MAX; + info->high.rm_offset = ULLONG_MAX; + info->high.rm_blockcount = 0; + info->high.rm_flags = XFS_RMAP_KEY_FLAGS | XFS_RMAP_REC_FLAGS; + + start_ag = XFS_FSB_TO_AGNO(mp, start_fsb); + end_ag = XFS_FSB_TO_AGNO(mp, end_fsb); + + while ((pag = xfs_perag_next_range(mp, pag, start_ag, end_ag))) { + /* + * Set the AG high key from the fsmap high key if this + * is the last AG that we're querying. + */ + info->group = pag_group(pag); + if (pag_agno(pag) == end_ag) { + info->high.rm_startblock = XFS_FSB_TO_AGBNO(mp, + end_fsb); + info->high.rm_offset = XFS_BB_TO_FSBT(mp, + keys[1].fmr_offset); + error = xfs_fsmap_owner_to_rmap(&info->high, &keys[1]); + if (error) + break; + xfs_getfsmap_set_irec_flags(&info->high, &keys[1]); + } + + if (bt_cur) { + xfs_btree_del_cursor(bt_cur, XFS_BTREE_NOERROR); + bt_cur = NULL; + xfs_trans_brelse(tp, info->agf_bp); + info->agf_bp = NULL; + } + + error = xfs_alloc_read_agf(pag, tp, 0, &info->agf_bp); + if (error) + break; + + trace_xfs_fsmap_low_group_key(mp, info->dev, pag_agno(pag), + &info->low); + trace_xfs_fsmap_high_group_key(mp, info->dev, pag_agno(pag), + &info->high); + + error = query_fn(tp, info, &bt_cur, priv); + if (error) + break; + + /* + * Set the AG low key to the start of the AG prior to + * moving on to the next AG. + */ + if (pag_agno(pag) == start_ag) + memset(&info->low, 0, sizeof(info->low)); + + /* + * If this is the last AG, report any gap at the end of it + * before we drop the reference to the perag when the loop + * terminates. + */ + if (pag_agno(pag) == end_ag) { + info->last = true; + error = query_fn(tp, info, &bt_cur, priv); + if (error) + break; + } + info->group = NULL; + } + + if (bt_cur) + xfs_btree_del_cursor(bt_cur, error < 0 ? XFS_BTREE_ERROR : + XFS_BTREE_NOERROR); + if (info->agf_bp) { + xfs_trans_brelse(tp, info->agf_bp); + info->agf_bp = NULL; + } + if (info->group) { + xfs_perag_rele(pag); + info->group = NULL; + } else if (pag) { + /* loop termination case */ + xfs_perag_rele(pag); + } + + return error; +} + +/* Actually query the rmap btree. */ +STATIC int +xfs_getfsmap_datadev_rmapbt_query( + struct xfs_trans *tp, + struct xfs_getfsmap_info *info, + struct xfs_btree_cur **curpp, + void *priv) +{ + /* Report any gap at the end of the last AG. */ + if (info->last) + return xfs_getfsmap_rmapbt_helper(*curpp, &info->high, info); + + /* Allocate cursor for this AG and query_range it. */ + *curpp = xfs_rmapbt_init_cursor(tp->t_mountp, tp, info->agf_bp, + to_perag(info->group)); + return xfs_rmap_query_range(*curpp, &info->low, &info->high, + xfs_getfsmap_rmapbt_helper, info); +} + +/* Execute a getfsmap query against the regular data device rmapbt. */ +STATIC int +xfs_getfsmap_datadev_rmapbt( + struct xfs_trans *tp, + const struct xfs_fsmap *keys, + struct xfs_getfsmap_info *info) +{ + info->missing_owner = XFS_FMR_OWN_FREE; + return __xfs_getfsmap_datadev(tp, keys, info, + xfs_getfsmap_datadev_rmapbt_query, NULL); +} + +/* Actually query the bno btree. */ +STATIC int +xfs_getfsmap_datadev_bnobt_query( + struct xfs_trans *tp, + struct xfs_getfsmap_info *info, + struct xfs_btree_cur **curpp, + void *priv) +{ + struct xfs_alloc_rec_incore *key = priv; + + /* Report any gap at the end of the last AG. */ + if (info->last) + return xfs_getfsmap_datadev_bnobt_helper(*curpp, &key[1], info); + + /* Allocate cursor for this AG and query_range it. */ + *curpp = xfs_bnobt_init_cursor(tp->t_mountp, tp, info->agf_bp, + to_perag(info->group)); + key->ar_startblock = info->low.rm_startblock; + key[1].ar_startblock = info->high.rm_startblock; + return xfs_alloc_query_range(*curpp, key, &key[1], + xfs_getfsmap_datadev_bnobt_helper, info); +} + +/* Execute a getfsmap query against the regular data device's bnobt. */ +STATIC int +xfs_getfsmap_datadev_bnobt( + struct xfs_trans *tp, + const struct xfs_fsmap *keys, + struct xfs_getfsmap_info *info) +{ + struct xfs_alloc_rec_incore akeys[2]; + + memset(akeys, 0, sizeof(akeys)); + info->missing_owner = XFS_FMR_OWN_UNKNOWN; + return __xfs_getfsmap_datadev(tp, keys, info, + xfs_getfsmap_datadev_bnobt_query, &akeys[0]); +} + +/* Execute a getfsmap query against the log device. */ +STATIC int +xfs_getfsmap_logdev( + struct xfs_trans *tp, + const struct xfs_fsmap *keys, + struct xfs_getfsmap_info *info) +{ + struct xfs_fsmap_irec frec = { + .start_daddr = 0, + .rec_key = 0, + .owner = XFS_RMAP_OWN_LOG, + }; + struct xfs_mount *mp = tp->t_mountp; + xfs_fsblock_t start_fsb, end_fsb; + uint64_t eofs; + + eofs = XFS_FSB_TO_BB(mp, mp->m_sb.sb_logblocks); + if (keys[0].fmr_physical >= eofs) + return 0; + start_fsb = XFS_BB_TO_FSBT(mp, + keys[0].fmr_physical + keys[0].fmr_length); + end_fsb = XFS_BB_TO_FSB(mp, min(eofs - 1, keys[1].fmr_physical)); + + /* Adjust the low key if we are continuing from where we left off. */ + if (keys[0].fmr_length > 0) + info->low_daddr = XFS_FSB_TO_BB(mp, start_fsb); + + trace_xfs_fsmap_low_linear_key(mp, info->dev, start_fsb); + trace_xfs_fsmap_high_linear_key(mp, info->dev, end_fsb); + + if (start_fsb > 0) + return 0; + + /* Fabricate an rmap entry for the external log device. */ + frec.len_daddr = XFS_FSB_TO_BB(mp, mp->m_sb.sb_logblocks); + return xfs_getfsmap_helper(tp, info, &frec); +} + +#ifdef CONFIG_XFS_RT +/* Transform a rtbitmap "record" into a fsmap */ +STATIC int +xfs_getfsmap_rtdev_rtbitmap_helper( + struct xfs_rtgroup *rtg, + struct xfs_trans *tp, + const struct xfs_rtalloc_rec *rec, + void *priv) +{ + struct xfs_fsmap_irec frec = { + .owner = XFS_RMAP_OWN_NULL, /* "free" */ + }; + struct xfs_mount *mp = rtg_mount(rtg); + struct xfs_getfsmap_info *info = priv; + xfs_rtblock_t start_rtb = + xfs_rtx_to_rtb(rtg, rec->ar_startext); + uint64_t rtbcount = + xfs_rtbxlen_to_blen(mp, rec->ar_extcount); + + /* + * For an info->last query, we're looking for a gap between the last + * mapping emitted and the high key specified by userspace. If the + * user's query spans less than 1 fsblock, then info->high and + * info->low will have the same rm_startblock, which causes rec_daddr + * and next_daddr to be the same. Therefore, use the end_daddr that + * we calculated from userspace's high key to synthesize the record. + * Note that if the btree query found a mapping, there won't be a gap. + */ + if (info->last) + frec.start_daddr = info->end_daddr + 1; + else + frec.start_daddr = xfs_rtb_to_daddr(mp, start_rtb); + + frec.len_daddr = XFS_FSB_TO_BB(mp, rtbcount); + return xfs_getfsmap_helper(tp, info, &frec); +} + +/* Execute a getfsmap query against the realtime device rtbitmap. */ +STATIC int +xfs_getfsmap_rtdev_rtbitmap( + struct xfs_trans *tp, + const struct xfs_fsmap *keys, + struct xfs_getfsmap_info *info) +{ + struct xfs_mount *mp = tp->t_mountp; + xfs_rtblock_t start_rtbno, end_rtbno; + xfs_rtxnum_t start_rtx, end_rtx; + xfs_rgnumber_t start_rgno, end_rgno; + struct xfs_rtgroup *rtg = NULL; + uint64_t eofs; + int error; + + eofs = XFS_FSB_TO_BB(mp, mp->m_sb.sb_rblocks); + if (keys[0].fmr_physical >= eofs) + return 0; + + info->missing_owner = XFS_FMR_OWN_UNKNOWN; + + /* Adjust the low key if we are continuing from where we left off. */ + start_rtbno = xfs_daddr_to_rtb(mp, + keys[0].fmr_physical + keys[0].fmr_length); + if (keys[0].fmr_length > 0) { + info->low_daddr = xfs_rtb_to_daddr(mp, start_rtbno); + if (info->low_daddr >= eofs) + return 0; + } + start_rtx = xfs_rtb_to_rtx(mp, start_rtbno); + start_rgno = xfs_rtb_to_rgno(mp, start_rtbno); + + end_rtbno = xfs_daddr_to_rtb(mp, min(eofs - 1, keys[1].fmr_physical)); + end_rgno = xfs_rtb_to_rgno(mp, end_rtbno); + + trace_xfs_fsmap_low_linear_key(mp, info->dev, start_rtbno); + trace_xfs_fsmap_high_linear_key(mp, info->dev, end_rtbno); + + end_rtx = -1ULL; + + while ((rtg = xfs_rtgroup_next_range(mp, rtg, start_rgno, end_rgno))) { + if (rtg_rgno(rtg) == end_rgno) + end_rtx = xfs_rtb_to_rtx(mp, + end_rtbno + mp->m_sb.sb_rextsize - 1); + + info->group = rtg_group(rtg); + xfs_rtgroup_lock(rtg, XFS_RTGLOCK_BITMAP_SHARED); + error = xfs_rtalloc_query_range(rtg, tp, start_rtx, end_rtx, + xfs_getfsmap_rtdev_rtbitmap_helper, info); + if (error) + break; + + /* + * Report any gaps at the end of the rtbitmap by simulating a + * zero-length free extent starting at the rtx after the end + * of the query range. + */ + if (rtg_rgno(rtg) == end_rgno) { + struct xfs_rtalloc_rec ahigh = { + .ar_startext = min(end_rtx + 1, + rtg->rtg_extents), + }; + + info->last = true; + error = xfs_getfsmap_rtdev_rtbitmap_helper(rtg, tp, + &ahigh, info); + if (error) + break; + } + + xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_BITMAP_SHARED); + info->group = NULL; + start_rtx = 0; + } + + /* loop termination case */ + if (rtg) { + if (info->group) { + xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_BITMAP_SHARED); + info->group = NULL; + } + xfs_rtgroup_rele(rtg); + } + + return error; +} + +/* Transform a realtime rmapbt record into a fsmap */ +STATIC int +xfs_getfsmap_rtdev_rmapbt_helper( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *priv) +{ + struct xfs_fsmap_irec frec = { + .owner = rec->rm_owner, + .offset = rec->rm_offset, + .rm_flags = rec->rm_flags, + .rec_key = rec->rm_startblock, + }; + struct xfs_getfsmap_info *info = priv; + + return xfs_getfsmap_group_helper(info, cur->bc_tp, cur->bc_group, + rec->rm_startblock, rec->rm_blockcount, &frec); +} + +/* Actually query the rtrmap btree. */ +STATIC int +xfs_getfsmap_rtdev_rmapbt_query( + struct xfs_trans *tp, + struct xfs_getfsmap_info *info, + struct xfs_btree_cur **curpp) +{ + struct xfs_rtgroup *rtg = to_rtg(info->group); + + /* Query the rtrmapbt */ + xfs_rtgroup_lock(rtg, XFS_RTGLOCK_RMAP | XFS_RTGLOCK_REFCOUNT); + *curpp = xfs_rtrmapbt_init_cursor(tp, rtg); + return xfs_rmap_query_range(*curpp, &info->low, &info->high, + xfs_getfsmap_rtdev_rmapbt_helper, info); +} + +/* Execute a getfsmap query against the realtime device rmapbt. */ +STATIC int +xfs_getfsmap_rtdev_rmapbt( + struct xfs_trans *tp, + const struct xfs_fsmap *keys, + struct xfs_getfsmap_info *info) +{ + struct xfs_fsmap key0 = *keys; /* struct copy */ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_rtgroup *rtg = NULL; + struct xfs_btree_cur *bt_cur = NULL; + xfs_daddr_t rtstart_daddr; + xfs_rtblock_t start_rtb; + xfs_rtblock_t end_rtb; + xfs_rgnumber_t start_rg, end_rg; + uint64_t eofs; + int error = 0; + + eofs = XFS_FSB_TO_BB(mp, mp->m_sb.sb_rtstart + mp->m_sb.sb_rblocks); + if (key0.fmr_physical >= eofs) + return 0; + + /* + * On zoned filesystems with an internal rt volume, the volume comes + * immediately after the end of the data volume. However, the + * xfs_rtblock_t address space is relative to the start of the data + * device, which means that the first @rtstart fsblocks do not actually + * point anywhere. If a fsmap query comes in with the low key starting + * below @rtstart, report it as "owned by filesystem". + */ + rtstart_daddr = XFS_FSB_TO_BB(mp, mp->m_sb.sb_rtstart); + if (xfs_has_zoned(mp) && key0.fmr_physical < rtstart_daddr) { + struct xfs_fsmap_irec frec = { + .owner = XFS_RMAP_OWN_FS, + .len_daddr = rtstart_daddr, + }; + + /* + * Adjust the start of the query range if we're picking up from + * a previous round, and only emit the record if we haven't + * already gone past. + */ + key0.fmr_physical += key0.fmr_length; + if (key0.fmr_physical < rtstart_daddr) { + error = xfs_getfsmap_helper(tp, info, &frec); + if (error) + return error; + + key0.fmr_physical = rtstart_daddr; + } + + /* Zero the other fields to avoid further adjustments. */ + key0.fmr_owner = 0; + key0.fmr_offset = 0; + key0.fmr_length = 0; + } + + start_rtb = xfs_daddr_to_rtb(mp, key0.fmr_physical); + end_rtb = xfs_daddr_to_rtb(mp, min(eofs - 1, keys[1].fmr_physical)); + info->missing_owner = XFS_FMR_OWN_FREE; + + /* + * Convert the fsmap low/high keys to rtgroup based keys. Initialize + * low to the fsmap low key and max out the high key to the end + * of the rtgroup. + */ + info->low.rm_offset = XFS_BB_TO_FSBT(mp, key0.fmr_offset); + error = xfs_fsmap_owner_to_rmap(&info->low, &key0); + if (error) + return error; + info->low.rm_blockcount = XFS_BB_TO_FSBT(mp, key0.fmr_length); + xfs_getfsmap_set_irec_flags(&info->low, &key0); + + /* Adjust the low key if we are continuing from where we left off. */ + if (info->low.rm_blockcount == 0) { + /* No previous record from which to continue */ + } else if (rmap_not_shareable(mp, &info->low)) { + /* Last record seen was an unshareable extent */ + info->low.rm_owner = 0; + info->low.rm_offset = 0; + + start_rtb += info->low.rm_blockcount; + if (xfs_rtb_to_daddr(mp, start_rtb) >= eofs) + return 0; + } else { + /* Last record seen was a shareable file data extent */ + info->low.rm_offset += info->low.rm_blockcount; + } + info->low.rm_startblock = xfs_rtb_to_rgbno(mp, start_rtb); + + info->high.rm_startblock = -1U; + info->high.rm_owner = ULLONG_MAX; + info->high.rm_offset = ULLONG_MAX; + info->high.rm_blockcount = 0; + info->high.rm_flags = XFS_RMAP_KEY_FLAGS | XFS_RMAP_REC_FLAGS; + + start_rg = xfs_rtb_to_rgno(mp, start_rtb); + end_rg = xfs_rtb_to_rgno(mp, end_rtb); + + while ((rtg = xfs_rtgroup_next_range(mp, rtg, start_rg, end_rg))) { + /* + * Set the rtgroup high key from the fsmap high key if this + * is the last rtgroup that we're querying. + */ + info->group = rtg_group(rtg); + if (rtg_rgno(rtg) == end_rg) { + info->high.rm_startblock = + xfs_rtb_to_rgbno(mp, end_rtb); + info->high.rm_offset = + XFS_BB_TO_FSBT(mp, keys[1].fmr_offset); + error = xfs_fsmap_owner_to_rmap(&info->high, &keys[1]); + if (error) + break; + xfs_getfsmap_set_irec_flags(&info->high, &keys[1]); + } + + if (bt_cur) { + xfs_rtgroup_unlock(to_rtg(bt_cur->bc_group), + XFS_RTGLOCK_RMAP | + XFS_RTGLOCK_REFCOUNT); + xfs_btree_del_cursor(bt_cur, XFS_BTREE_NOERROR); + bt_cur = NULL; + } + + trace_xfs_fsmap_low_group_key(mp, info->dev, rtg_rgno(rtg), + &info->low); + trace_xfs_fsmap_high_group_key(mp, info->dev, rtg_rgno(rtg), + &info->high); + + error = xfs_getfsmap_rtdev_rmapbt_query(tp, info, &bt_cur); + if (error) + break; + + /* + * Set the rtgroup low key to the start of the rtgroup prior to + * moving on to the next rtgroup. + */ + if (rtg_rgno(rtg) == start_rg) + memset(&info->low, 0, sizeof(info->low)); + + /* + * If this is the last rtgroup, report any gap at the end of it + * before we drop the reference to the perag when the loop + * terminates. + */ + if (rtg_rgno(rtg) == end_rg) { + info->last = true; + error = xfs_getfsmap_rtdev_rmapbt_helper(bt_cur, + &info->high, info); + if (error) + break; + } + info->group = NULL; + } + + if (bt_cur) { + xfs_rtgroup_unlock(to_rtg(bt_cur->bc_group), + XFS_RTGLOCK_RMAP | XFS_RTGLOCK_REFCOUNT); + xfs_btree_del_cursor(bt_cur, error < 0 ? XFS_BTREE_ERROR : + XFS_BTREE_NOERROR); + } + + /* loop termination case */ + if (rtg) { + info->group = NULL; + xfs_rtgroup_rele(rtg); + } + + return error; +} +#endif /* CONFIG_XFS_RT */ + +static uint32_t +xfs_getfsmap_device( + struct xfs_mount *mp, + enum xfs_device dev) +{ + if (mp->m_sb.sb_rtstart) + return dev; + + switch (dev) { + case XFS_DEV_DATA: + return new_encode_dev(mp->m_ddev_targp->bt_dev); + case XFS_DEV_LOG: + return new_encode_dev(mp->m_logdev_targp->bt_dev); + case XFS_DEV_RT: + if (!mp->m_rtdev_targp) + break; + return new_encode_dev(mp->m_rtdev_targp->bt_dev); + } + + return -1; +} + +/* Do we recognize the device? */ +STATIC bool +xfs_getfsmap_is_valid_device( + struct xfs_mount *mp, + struct xfs_fsmap *fm) +{ + return fm->fmr_device == 0 || + fm->fmr_device == UINT_MAX || + fm->fmr_device == xfs_getfsmap_device(mp, XFS_DEV_DATA) || + fm->fmr_device == xfs_getfsmap_device(mp, XFS_DEV_LOG) || + (mp->m_rtdev_targp && + fm->fmr_device == xfs_getfsmap_device(mp, XFS_DEV_RT)); +} + +/* Ensure that the low key is less than the high key. */ +STATIC bool +xfs_getfsmap_check_keys( + struct xfs_fsmap *low_key, + struct xfs_fsmap *high_key) +{ + if (low_key->fmr_flags & (FMR_OF_SPECIAL_OWNER | FMR_OF_EXTENT_MAP)) { + if (low_key->fmr_offset) + return false; + } + if (high_key->fmr_flags != -1U && + (high_key->fmr_flags & (FMR_OF_SPECIAL_OWNER | + FMR_OF_EXTENT_MAP))) { + if (high_key->fmr_offset && high_key->fmr_offset != -1ULL) + return false; + } + if (high_key->fmr_length && high_key->fmr_length != -1ULL) + return false; + + if (low_key->fmr_device > high_key->fmr_device) + return false; + if (low_key->fmr_device < high_key->fmr_device) + return true; + + if (low_key->fmr_physical > high_key->fmr_physical) + return false; + if (low_key->fmr_physical < high_key->fmr_physical) + return true; + + if (low_key->fmr_owner > high_key->fmr_owner) + return false; + if (low_key->fmr_owner < high_key->fmr_owner) + return true; + + if (low_key->fmr_offset > high_key->fmr_offset) + return false; + if (low_key->fmr_offset < high_key->fmr_offset) + return true; + + return false; +} + +/* + * There are only two devices if we didn't configure RT devices at build time. + */ +#ifdef CONFIG_XFS_RT +#define XFS_GETFSMAP_DEVS 3 +#else +#define XFS_GETFSMAP_DEVS 2 +#endif /* CONFIG_XFS_RT */ + +/* + * Get filesystem's extents as described in head, and format for output. Fills + * in the supplied records array until there are no more reverse mappings to + * return or head.fmh_entries == head.fmh_count. In the second case, this + * function returns -ECANCELED to indicate that more records would have been + * returned. + * + * Key to Confusion + * ---------------- + * There are multiple levels of keys and counters at work here: + * xfs_fsmap_head.fmh_keys -- low and high fsmap keys passed in; + * these reflect fs-wide sector addrs. + * dkeys -- fmh_keys used to query each device; + * these are fmh_keys but w/ the low key + * bumped up by fmr_length. + * xfs_getfsmap_info.next_daddr -- next disk addr we expect to see; this + * is how we detect gaps in the fsmap + records and report them. + * xfs_getfsmap_info.low/high -- per-AG low/high keys computed from + * dkeys; used to query the metadata. + */ +STATIC int +xfs_getfsmap( + struct xfs_mount *mp, + struct xfs_fsmap_head *head, + struct fsmap *fsmap_recs) +{ + struct xfs_trans *tp = NULL; + struct xfs_fsmap dkeys[2]; /* per-dev keys */ + struct xfs_getfsmap_dev handlers[XFS_GETFSMAP_DEVS]; + struct xfs_getfsmap_info info = { + .fsmap_recs = fsmap_recs, + .head = head, + }; + bool use_rmap; + int i; + int error = 0; + + if (head->fmh_iflags & ~FMH_IF_VALID) + return -EINVAL; + if (!xfs_getfsmap_is_valid_device(mp, &head->fmh_keys[0]) || + !xfs_getfsmap_is_valid_device(mp, &head->fmh_keys[1])) + return -EINVAL; + if (!xfs_getfsmap_check_keys(&head->fmh_keys[0], &head->fmh_keys[1])) + return -EINVAL; + + use_rmap = xfs_has_rmapbt(mp) && + has_capability_noaudit(current, CAP_SYS_ADMIN); + head->fmh_entries = 0; + + /* Set up our device handlers. */ + memset(handlers, 0, sizeof(handlers)); + handlers[0].nr_sectors = XFS_FSB_TO_BB(mp, mp->m_sb.sb_dblocks); + handlers[0].dev = xfs_getfsmap_device(mp, XFS_DEV_DATA); + if (use_rmap) + handlers[0].fn = xfs_getfsmap_datadev_rmapbt; + else + handlers[0].fn = xfs_getfsmap_datadev_bnobt; + if (mp->m_logdev_targp != mp->m_ddev_targp) { + handlers[1].nr_sectors = XFS_FSB_TO_BB(mp, + mp->m_sb.sb_logblocks); + handlers[1].dev = xfs_getfsmap_device(mp, XFS_DEV_LOG); + handlers[1].fn = xfs_getfsmap_logdev; + } +#ifdef CONFIG_XFS_RT + /* + * For zoned file systems there is no rtbitmap, so only support fsmap + * if the callers is privileged enough to use the full rmap version. + */ + if (mp->m_rtdev_targp && (use_rmap || !xfs_has_zoned(mp))) { + handlers[2].nr_sectors = XFS_FSB_TO_BB(mp, mp->m_sb.sb_rblocks); + handlers[2].dev = xfs_getfsmap_device(mp, XFS_DEV_RT); + if (use_rmap) + handlers[2].fn = xfs_getfsmap_rtdev_rmapbt; + else + handlers[2].fn = xfs_getfsmap_rtdev_rtbitmap; + } +#endif /* CONFIG_XFS_RT */ + + xfs_sort(handlers, XFS_GETFSMAP_DEVS, sizeof(struct xfs_getfsmap_dev), + xfs_getfsmap_dev_compare); + + /* + * To continue where we left off, we allow userspace to use the + * last mapping from a previous call as the low key of the next. + * This is identified by a non-zero length in the low key. We + * have to increment the low key in this scenario to ensure we + * don't return the same mapping again, and instead return the + * very next mapping. + * + * If the low key mapping refers to file data, the same physical + * blocks could be mapped to several other files/offsets. + * According to rmapbt record ordering, the minimal next + * possible record for the block range is the next starting + * offset in the same inode. Therefore, each fsmap backend bumps + * the file offset to continue the search appropriately. For + * all other low key mapping types (attr blocks, metadata), each + * fsmap backend bumps the physical offset as there can be no + * other mapping for the same physical block range. + */ + dkeys[0] = head->fmh_keys[0]; + memset(&dkeys[1], 0xFF, sizeof(struct xfs_fsmap)); + + info.next_daddr = head->fmh_keys[0].fmr_physical + + head->fmh_keys[0].fmr_length; + + /* For each device we support... */ + for (i = 0; i < XFS_GETFSMAP_DEVS; i++) { + /* Is this device within the range the user asked for? */ + if (!handlers[i].fn) + continue; + if (head->fmh_keys[0].fmr_device > handlers[i].dev) + continue; + if (head->fmh_keys[1].fmr_device < handlers[i].dev) + break; + + /* + * If this device number matches the high key, we have to pass + * the high key to the handler to limit the query results, and + * set the end_daddr so that we can synthesize records at the + * end of the query range or device. + */ + if (handlers[i].dev == head->fmh_keys[1].fmr_device) { + dkeys[1] = head->fmh_keys[1]; + info.end_daddr = min(handlers[i].nr_sectors - 1, + dkeys[1].fmr_physical); + } else { + info.end_daddr = handlers[i].nr_sectors - 1; + } + + /* + * If the device number exceeds the low key, zero out the low + * key so that we get everything from the beginning. + */ + if (handlers[i].dev > head->fmh_keys[0].fmr_device) + memset(&dkeys[0], 0, sizeof(struct xfs_fsmap)); + + /* + * Grab an empty transaction so that we can use its recursive + * buffer locking abilities to detect cycles in the rmapbt + * without deadlocking. + */ + tp = xfs_trans_alloc_empty(mp); + + info.dev = handlers[i].dev; + info.last = false; + info.group = NULL; + info.low_daddr = XFS_BUF_DADDR_NULL; + info.low.rm_blockcount = 0; + error = handlers[i].fn(tp, dkeys, &info); + if (error) + break; + xfs_trans_cancel(tp); + tp = NULL; + info.next_daddr = 0; + } + + if (tp) + xfs_trans_cancel(tp); + + /* + * For internal RT device we need to report different synthetic devices + * for a single physical device, and thus can't report the actual dev_t. + */ + if (!mp->m_sb.sb_rtstart) + head->fmh_oflags = FMH_OF_DEV_T; + return error; +} + +int +xfs_ioc_getfsmap( + struct xfs_inode *ip, + struct fsmap_head __user *arg) +{ + struct xfs_fsmap_head xhead = {0}; + struct fsmap_head head; + struct fsmap *recs; + unsigned int count; + __u32 last_flags = 0; + bool done = false; + int error; + + if (copy_from_user(&head, arg, sizeof(struct fsmap_head))) + return -EFAULT; + if (memchr_inv(head.fmh_reserved, 0, sizeof(head.fmh_reserved)) || + memchr_inv(head.fmh_keys[0].fmr_reserved, 0, + sizeof(head.fmh_keys[0].fmr_reserved)) || + memchr_inv(head.fmh_keys[1].fmr_reserved, 0, + sizeof(head.fmh_keys[1].fmr_reserved))) + return -EINVAL; + + /* + * Use an internal memory buffer so that we don't have to copy fsmap + * data to userspace while holding locks. Start by trying to allocate + * up to 128k for the buffer, but fall back to a single page if needed. + */ + count = min_t(unsigned int, head.fmh_count, + 131072 / sizeof(struct fsmap)); + recs = kvzalloc_objs(struct fsmap, count); + if (!recs) { + count = min_t(unsigned int, head.fmh_count, + PAGE_SIZE / sizeof(struct fsmap)); + recs = kvzalloc_objs(struct fsmap, count); + if (!recs) + return -ENOMEM; + } + + xhead.fmh_iflags = head.fmh_iflags; + xfs_fsmap_to_internal(&xhead.fmh_keys[0], &head.fmh_keys[0]); + xfs_fsmap_to_internal(&xhead.fmh_keys[1], &head.fmh_keys[1]); + + trace_xfs_getfsmap_low_key(ip->i_mount, &xhead.fmh_keys[0]); + trace_xfs_getfsmap_high_key(ip->i_mount, &xhead.fmh_keys[1]); + + head.fmh_entries = 0; + do { + struct fsmap __user *user_recs; + struct fsmap *last_rec; + + user_recs = &arg->fmh_recs[head.fmh_entries]; + xhead.fmh_entries = 0; + xhead.fmh_count = min_t(unsigned int, count, + head.fmh_count - head.fmh_entries); + + /* Run query, record how many entries we got. */ + error = xfs_getfsmap(ip->i_mount, &xhead, recs); + switch (error) { + case 0: + /* + * There are no more records in the result set. Copy + * whatever we got to userspace and break out. + */ + done = true; + break; + case -ECANCELED: + /* + * The internal memory buffer is full. Copy whatever + * records we got to userspace and go again if we have + * not yet filled the userspace buffer. + */ + error = 0; + break; + default: + goto out_free; + } + head.fmh_entries += xhead.fmh_entries; + head.fmh_oflags = xhead.fmh_oflags; + + /* + * If the caller wanted a record count or there aren't any + * new records to return, we're done. + */ + if (head.fmh_count == 0 || xhead.fmh_entries == 0) + break; + + /* Copy all the records we got out to userspace. */ + if (copy_to_user(user_recs, recs, + xhead.fmh_entries * sizeof(struct fsmap))) { + error = -EFAULT; + goto out_free; + } + + /* Remember the last record flags we copied to userspace. */ + last_rec = &recs[xhead.fmh_entries - 1]; + last_flags = last_rec->fmr_flags; + + /* Set up the low key for the next iteration. */ + xfs_fsmap_to_internal(&xhead.fmh_keys[0], last_rec); + trace_xfs_getfsmap_low_key(ip->i_mount, &xhead.fmh_keys[0]); + } while (!done && head.fmh_entries < head.fmh_count); + + /* + * If there are no more records in the query result set and we're not + * in counting mode, mark the last record returned with the LAST flag. + */ + if (done && head.fmh_count > 0 && head.fmh_entries > 0) { + struct fsmap __user *user_rec; + + last_flags |= FMR_OF_LAST; + user_rec = &arg->fmh_recs[head.fmh_entries - 1]; + + if (copy_to_user(&user_rec->fmr_flags, &last_flags, + sizeof(last_flags))) { + error = -EFAULT; + goto out_free; + } + } + + /* copy back header */ + if (copy_to_user(arg, &head, sizeof(struct fsmap_head))) { + error = -EFAULT; + goto out_free; + } + +out_free: + kvfree(recs); + return error; +} diff --git a/fs/xfs/xfs_fsmap.h b/fs/xfs/xfs_fsmap.h new file mode 100644 index 000000000..06e492fd4 --- /dev/null +++ b/fs/xfs/xfs_fsmap.h @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2017 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_FSMAP_H__ +#define __XFS_FSMAP_H__ + +struct fsmap; +struct fsmap_head; + +/* internal fsmap representation */ +struct xfs_fsmap { + dev_t fmr_device; /* device id */ + uint32_t fmr_flags; /* mapping flags */ + uint64_t fmr_physical; /* device offset of segment */ + uint64_t fmr_owner; /* owner id */ + xfs_fileoff_t fmr_offset; /* file offset of segment */ + xfs_filblks_t fmr_length; /* length of segment, blocks */ +}; + +struct xfs_fsmap_head { + uint32_t fmh_iflags; /* control flags */ + uint32_t fmh_oflags; /* output flags */ + unsigned int fmh_count; /* # of entries in array incl. input */ + unsigned int fmh_entries; /* # of entries filled in (output). */ + + struct xfs_fsmap fmh_keys[2]; /* low and high keys */ +}; + +/* internal fsmap record format */ +struct xfs_fsmap_irec { + xfs_daddr_t start_daddr; + xfs_daddr_t len_daddr; + uint64_t owner; /* extent owner */ + uint64_t offset; /* offset within the owner */ + unsigned int rm_flags; /* rmap state flags */ + + /* + * rmapbt startblock corresponding to start_daddr, if the record came + * from an rmap btree. + */ + xfs_agblock_t rec_key; +}; + +int xfs_ioc_getfsmap(struct xfs_inode *ip, struct fsmap_head __user *arg); + +#endif /* __XFS_FSMAP_H__ */ diff --git a/fs/xfs/xfs_fsops.c b/fs/xfs/xfs_fsops.c new file mode 100644 index 000000000..21114bb6d --- /dev/null +++ b/fs/xfs/xfs_fsops.c @@ -0,0 +1,590 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_sb.h" +#include "xfs_mount.h" +#include "xfs_trans.h" +#include "xfs_error.h" +#include "xfs_alloc.h" +#include "xfs_fsops.h" +#include "xfs_trans_space.h" +#include "xfs_log.h" +#include "xfs_log_priv.h" +#include "xfs_ag.h" +#include "xfs_ag_resv.h" +#include "xfs_trace.h" +#include "xfs_rtalloc.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_rtrefcount_btree.h" +#include "xfs_metafile.h" +#include "xfs_healthmon.h" + +#include + +/* + * Write new AG headers to disk. Non-transactional, but need to be + * written and completed prior to the growfs transaction being logged. + * To do this, we use a delayed write buffer list and wait for + * submission and IO completion of the list as a whole. This allows the + * IO subsystem to merge all the AG headers in a single AG into a single + * IO and hide most of the latency of the IO from us. + * + * This also means that if we get an error whilst building the buffer + * list to write, we can cancel the entire list without having written + * anything. + */ +static int +xfs_resizefs_init_new_ags( + struct xfs_trans *tp, + struct aghdr_init_data *id, + xfs_agnumber_t oagcount, + xfs_agnumber_t nagcount, + xfs_rfsblock_t delta, + struct xfs_perag *last_pag, + bool *lastag_extended) +{ + struct xfs_mount *mp = tp->t_mountp; + xfs_rfsblock_t nb = mp->m_sb.sb_dblocks + delta; + int error; + + *lastag_extended = false; + + INIT_LIST_HEAD(&id->buffer_list); + for (id->agno = nagcount - 1; + id->agno >= oagcount; + id->agno--, delta -= id->agsize) { + + if (id->agno == nagcount - 1) + id->agsize = nb - (id->agno * + (xfs_rfsblock_t)mp->m_sb.sb_agblocks); + else + id->agsize = mp->m_sb.sb_agblocks; + + error = xfs_ag_init_headers(mp, id); + if (error) { + xfs_buf_delwri_cancel(&id->buffer_list); + return error; + } + } + + error = xfs_buf_delwri_submit(&id->buffer_list); + if (error) + return error; + + if (delta) { + *lastag_extended = true; + error = xfs_ag_extend_space(last_pag, tp, delta); + } + return error; +} + +/* + * growfs operations + */ +static int +xfs_growfs_data_private( + struct xfs_mount *mp, /* mount point for filesystem */ + struct xfs_growfs_data *in) /* growfs data input struct */ +{ + xfs_agnumber_t oagcount = mp->m_sb.sb_agcount; + xfs_rfsblock_t nb = in->newblocks; + struct xfs_buf *bp; + int error; + xfs_agnumber_t nagcount; + xfs_agnumber_t nagimax = 0; + int64_t delta; + bool lastag_extended = false; + struct xfs_trans *tp; + struct aghdr_init_data id = {}; + struct xfs_perag *last_pag; + + error = xfs_sb_validate_fsb_count(&mp->m_sb, nb); + if (error) + return error; + + if (nb > mp->m_sb.sb_dblocks) { + error = xfs_buf_read_uncached(mp->m_ddev_targp, + XFS_FSB_TO_BB(mp, nb) - XFS_FSS_TO_BB(mp, 1), + XFS_FSS_TO_BB(mp, 1), &bp, NULL); + if (error) + return error; + xfs_buf_relse(bp); + } + + /* Make sure the new fs size won't cause problems with the log. */ + error = xfs_growfs_check_rtgeom(mp, nb, mp->m_sb.sb_rblocks, + mp->m_sb.sb_rextsize); + if (error) + return error; + + nagcount = xfs_growfs_compute_agcount(mp, &nb); + delta = nb - mp->m_sb.sb_dblocks; + + /* + * Reject filesystems with a single AG because they are not + * supported, and reject a shrink operation that would cause a + * filesystem to become unsupported. + */ + if (delta < 0 && nagcount < 2) + return -EINVAL; + + /* No work to do */ + if (delta == 0) + return 0; + + /* TODO: shrinking the entire AGs hasn't yet completed */ + if (nagcount < oagcount) + return -EINVAL; + + /* allocate the new per-ag structures */ + error = xfs_initialize_perag(mp, oagcount, nagcount, nb, &nagimax); + if (error) + return error; + + if (delta > 0) + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_growdata, + XFS_GROWFS_SPACE_RES(mp), 0, XFS_TRANS_RESERVE, + &tp); + else + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_growdata, -delta, 0, + 0, &tp); + if (error) + goto out_free_unused_perag; + + last_pag = xfs_perag_get(mp, oagcount - 1); + if (delta > 0) { + error = xfs_resizefs_init_new_ags(tp, &id, oagcount, nagcount, + delta, last_pag, &lastag_extended); + } else { + xfs_warn_experimental(mp, XFS_EXPERIMENTAL_SHRINK); + error = xfs_ag_shrink_space(last_pag, &tp, -delta); + } + xfs_perag_put(last_pag); + if (error) + goto out_trans_cancel; + + /* + * Update changed superblock fields transactionally. These are not + * seen by the rest of the world until the transaction commit applies + * them atomically to the superblock. + */ + if (nagcount > oagcount) + xfs_trans_mod_sb(tp, XFS_TRANS_SB_AGCOUNT, nagcount - oagcount); + if (delta) + xfs_trans_mod_sb(tp, XFS_TRANS_SB_DBLOCKS, delta); + if (id.nfree) + xfs_trans_mod_sb(tp, XFS_TRANS_SB_FDBLOCKS, id.nfree); + + /* + * Sync sb counters now to reflect the updated values. This is + * particularly important for shrink because the write verifier + * will fail if sb_fdblocks is ever larger than sb_dblocks. + */ + if (xfs_has_lazysbcount(mp)) + xfs_log_sb(tp); + + xfs_trans_set_sync(tp); + error = xfs_trans_commit(tp); + if (error) + return error; + + /* New allocation groups fully initialized, so update mount struct */ + if (nagimax) + mp->m_maxagi = nagimax; + xfs_set_low_space_thresholds(mp); + mp->m_alloc_set_aside = xfs_alloc_set_aside(mp); + + if (delta > 0) { + /* + * If we expanded the last AG, free the per-AG reservation + * so we can reinitialize it with the new size. + */ + if (lastag_extended) { + struct xfs_perag *pag; + + pag = xfs_perag_get(mp, id.agno); + xfs_ag_resv_free(pag); + xfs_perag_put(pag); + } + /* + * Reserve AG metadata blocks. ENOSPC here does not mean there + * was a growfs failure, just that there still isn't space for + * new user data after the grow has been run. + */ + error = xfs_fs_reserve_ag_blocks(mp); + if (error == -ENOSPC) + error = 0; + + /* Compute new maxlevels for rt btrees. */ + xfs_rtrmapbt_compute_maxlevels(mp); + xfs_rtrefcountbt_compute_maxlevels(mp); + } + + return error; + +out_trans_cancel: + xfs_trans_cancel(tp); +out_free_unused_perag: + if (nagcount > oagcount) + xfs_free_perag_range(mp, oagcount, nagcount); + return error; +} + +static int +xfs_growfs_log_private( + struct xfs_mount *mp, /* mount point for filesystem */ + struct xfs_growfs_log *in) /* growfs log input struct */ +{ + xfs_extlen_t nb; + + nb = in->newblocks; + if (nb < XFS_MIN_LOG_BLOCKS || nb < XFS_B_TO_FSB(mp, XFS_MIN_LOG_BYTES)) + return -EINVAL; + if (nb == mp->m_sb.sb_logblocks && + in->isint == (mp->m_sb.sb_logstart != 0)) + return -EINVAL; + /* + * Moving the log is hard, need new interfaces to sync + * the log first, hold off all activity while moving it. + * Can have shorter or longer log in the same space, + * or transform internal to external log or vice versa. + */ + return -ENOSYS; +} + +static int +xfs_growfs_imaxpct( + struct xfs_mount *mp, + __u32 imaxpct) +{ + struct xfs_trans *tp; + int dpct; + int error; + + if (imaxpct > 100) + return -EINVAL; + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_growdata, + XFS_GROWFS_SPACE_RES(mp), 0, XFS_TRANS_RESERVE, &tp); + if (error) + return error; + + dpct = imaxpct - mp->m_sb.sb_imax_pct; + xfs_trans_mod_sb(tp, XFS_TRANS_SB_IMAXPCT, dpct); + xfs_trans_set_sync(tp); + return xfs_trans_commit(tp); +} + +/* + * protected versions of growfs function acquire and release locks on the mount + * point - exported through ioctls: XFS_IOC_FSGROWFSDATA, XFS_IOC_FSGROWFSLOG, + * XFS_IOC_FSGROWFSRT + */ +int +xfs_growfs_data( + struct xfs_mount *mp, + struct xfs_growfs_data *in) +{ + int error; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (!mutex_trylock(&mp->m_growlock)) + return -EWOULDBLOCK; + + /* we can't grow the data section when an internal RT section exists */ + if (in->newblocks != mp->m_sb.sb_dblocks && mp->m_sb.sb_rtstart) { + error = -EINVAL; + goto out_unlock; + } + + /* update imaxpct separately to the physical grow of the filesystem */ + if (in->imaxpct != mp->m_sb.sb_imax_pct) { + error = xfs_growfs_imaxpct(mp, in->imaxpct); + if (error) + goto out_unlock; + } + + if (in->newblocks != mp->m_sb.sb_dblocks) { + error = xfs_growfs_data_private(mp, in); + if (error) + goto out_unlock; + } + + /* Post growfs calculations needed to reflect new state in operations */ + if (mp->m_sb.sb_imax_pct) { + uint64_t icount = mp->m_sb.sb_dblocks * mp->m_sb.sb_imax_pct; + do_div(icount, 100); + M_IGEO(mp)->maxicount = XFS_FSB_TO_INO(mp, icount); + } else + M_IGEO(mp)->maxicount = 0; + + /* Update secondary superblocks now the physical grow has completed */ + error = xfs_update_secondary_sbs(mp); + + /* + * Increment the generation unconditionally, after trying to update the + * secondary superblocks, as the new size is live already at this point. + */ + mp->m_generation++; +out_unlock: + mutex_unlock(&mp->m_growlock); + return error; +} + +int +xfs_growfs_log( + xfs_mount_t *mp, + struct xfs_growfs_log *in) +{ + int error; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (!mutex_trylock(&mp->m_growlock)) + return -EWOULDBLOCK; + error = xfs_growfs_log_private(mp, in); + mutex_unlock(&mp->m_growlock); + return error; +} + +/* + * Reserve the requested number of blocks if available. Otherwise return + * as many as possible to satisfy the request. The actual number + * reserved are returned in outval. + */ +int +xfs_reserve_blocks( + struct xfs_mount *mp, + enum xfs_free_counter ctr, + uint64_t request) +{ + int64_t lcounter, delta; + int64_t fdblks_delta = 0; + int64_t free; + int error = 0; + + ASSERT(ctr < XC_FREE_NR); + + /* + * With per-cpu counters, this becomes an interesting problem. we need + * to work out if we are freeing or allocation blocks first, then we can + * do the modification as necessary. + * + * We do this under the m_sb_lock so that if we are near ENOSPC, we will + * hold out any changes while we work out what to do. This means that + * the amount of free space can change while we do this, so we need to + * retry if we end up trying to reserve more space than is available. + */ + spin_lock(&mp->m_sb_lock); + + /* + * If our previous reservation was larger than the current value, + * then move any unused blocks back to the free pool. Modify the resblks + * counters directly since we shouldn't have any problems unreserving + * space. + */ + if (mp->m_free[ctr].res_total > request) { + lcounter = mp->m_free[ctr].res_avail - request; + if (lcounter > 0) { /* release unused blocks */ + fdblks_delta = lcounter; + mp->m_free[ctr].res_avail -= lcounter; + } + mp->m_free[ctr].res_total = request; + if (fdblks_delta) { + spin_unlock(&mp->m_sb_lock); + xfs_add_freecounter(mp, ctr, fdblks_delta); + spin_lock(&mp->m_sb_lock); + } + + goto out; + } + + /* + * If the request is larger than the current reservation, reserve the + * blocks before we update the reserve counters. Sample m_free and + * perform a partial reservation if the request exceeds free space. + * + * The code below estimates how many blocks it can request from + * fdblocks to stash in the reserve pool. This is a classic TOCTOU + * race since fdblocks updates are not always coordinated via + * m_sb_lock. Set the reserve size even if there's not enough free + * space to fill it because mod_fdblocks will refill an undersized + * reserve when it can. + */ + free = xfs_sum_freecounter_raw(mp, ctr) - + xfs_freecounter_unavailable(mp, ctr); + delta = request - mp->m_free[ctr].res_total; + mp->m_free[ctr].res_total = request; + if (delta > 0 && free > 0) { + /* + * We'll either succeed in getting space from the free block + * count or we'll get an ENOSPC. Don't set the reserved flag + * here - we don't want to reserve the extra reserve blocks + * from the reserve. + * + * The desired reserve size can change after we drop the lock. + * Use mod_fdblocks to put the space into the reserve or into + * fdblocks as appropriate. + */ + fdblks_delta = min(free, delta); + spin_unlock(&mp->m_sb_lock); + error = xfs_dec_freecounter(mp, ctr, fdblks_delta, 0); + if (!error) + xfs_add_freecounter(mp, ctr, fdblks_delta); + spin_lock(&mp->m_sb_lock); + } +out: + spin_unlock(&mp->m_sb_lock); + return error; +} + +int +xfs_fs_goingdown( + xfs_mount_t *mp, + uint32_t inflags) +{ + switch (inflags) { + case XFS_FSOP_GOING_FLAGS_DEFAULT: { + if (!bdev_freeze(mp->m_super->s_bdev)) { + xfs_force_shutdown(mp, SHUTDOWN_FORCE_UMOUNT); + bdev_thaw(mp->m_super->s_bdev); + } + break; + } + case XFS_FSOP_GOING_FLAGS_LOGFLUSH: + xfs_force_shutdown(mp, SHUTDOWN_FORCE_UMOUNT); + break; + case XFS_FSOP_GOING_FLAGS_NOLOGFLUSH: + xfs_force_shutdown(mp, + SHUTDOWN_FORCE_UMOUNT | SHUTDOWN_LOG_IO_ERROR); + break; + default: + return -EINVAL; + } + + return 0; +} + +/* + * Force a shutdown of the filesystem instantly while keeping the filesystem + * consistent. We don't do an unmount here; just shutdown the shop, make sure + * that absolutely nothing persistent happens to this filesystem after this + * point. + * + * The shutdown state change is atomic, resulting in the first and only the + * first shutdown call processing the shutdown. This means we only shutdown the + * log once as it requires, and we don't spam the logs when multiple concurrent + * shutdowns race to set the shutdown flags. + */ +void +xfs_do_force_shutdown( + struct xfs_mount *mp, + uint32_t flags, + char *fname, + int lnnum) +{ + int tag; + const char *why; + + + if (xfs_set_shutdown(mp)) { + xlog_shutdown_wait(mp->m_log); + return; + } + if (mp->m_sb_bp) + xfs_buf_set_uptodate(mp->m_sb_bp); + + if (flags & SHUTDOWN_FORCE_UMOUNT) + xfs_alert(mp, "User initiated shutdown received."); + + if (xlog_force_shutdown(mp->m_log, flags)) { + tag = XFS_PTAG_SHUTDOWN_LOGERROR; + why = "Log I/O Error"; + } else if (flags & SHUTDOWN_CORRUPT_INCORE) { + tag = XFS_PTAG_SHUTDOWN_CORRUPT; + why = "Corruption of in-memory data"; + } else if (flags & SHUTDOWN_CORRUPT_ONDISK) { + tag = XFS_PTAG_SHUTDOWN_CORRUPT; + why = "Corruption of on-disk metadata"; + } else if (flags & SHUTDOWN_DEVICE_REMOVED) { + tag = XFS_PTAG_SHUTDOWN_IOERROR; + why = "Block device removal"; + } else { + tag = XFS_PTAG_SHUTDOWN_IOERROR; + why = "Metadata I/O Error"; + } + + trace_xfs_force_shutdown(mp, tag, flags, fname, lnnum); + + xfs_alert_tag(mp, tag, +"%s (0x%x) detected at %pS (%s:%d). Shutting down filesystem.", + why, flags, __return_address, fname, lnnum); + xfs_alert(mp, + "Please unmount the filesystem and rectify the problem(s)"); + if (xfs_error_level >= XFS_ERRLEVEL_HIGH) + xfs_stack_trace(); + + fserror_report_shutdown(mp->m_super, GFP_KERNEL); + xfs_healthmon_report_shutdown(mp, flags); +} + +/* + * Reserve free space for per-AG metadata. + */ +int +xfs_fs_reserve_ag_blocks( + struct xfs_mount *mp) +{ + struct xfs_perag *pag = NULL; + int error = 0; + int err2; + + mp->m_finobt_nores = false; + while ((pag = xfs_perag_next(mp, pag))) { + err2 = xfs_ag_resv_init(pag, NULL); + if (err2 && !error) + error = err2; + } + + if (error && error != -ENOSPC) { + xfs_warn(mp, + "Error %d reserving per-AG metadata reserve pool.", error); + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); + return error; + } + + err2 = xfs_metafile_resv_init(mp); + if (err2 && err2 != -ENOSPC) { + xfs_warn(mp, + "Error %d reserving realtime metadata reserve pool.", err2); + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); + + if (!error) + error = err2; + } + + return error; +} + +/* + * Free space reserved for per-AG metadata. + */ +void +xfs_fs_unreserve_ag_blocks( + struct xfs_mount *mp) +{ + struct xfs_perag *pag = NULL; + + xfs_metafile_resv_free(mp); + while ((pag = xfs_perag_next(mp, pag))) + xfs_ag_resv_free(pag); +} diff --git a/fs/xfs/xfs_fsops.h b/fs/xfs/xfs_fsops.h new file mode 100644 index 000000000..9d23c361e --- /dev/null +++ b/fs/xfs/xfs_fsops.h @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2001,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_FSOPS_H__ +#define __XFS_FSOPS_H__ + +int xfs_growfs_data(struct xfs_mount *mp, struct xfs_growfs_data *in); +int xfs_growfs_log(struct xfs_mount *mp, struct xfs_growfs_log *in); +int xfs_reserve_blocks(struct xfs_mount *mp, enum xfs_free_counter cnt, + uint64_t request); +int xfs_fs_goingdown(struct xfs_mount *mp, uint32_t inflags); + +int xfs_fs_reserve_ag_blocks(struct xfs_mount *mp); +void xfs_fs_unreserve_ag_blocks(struct xfs_mount *mp); + +#endif /* __XFS_FSOPS_H__ */ diff --git a/fs/xfs/xfs_globals.c b/fs/xfs/xfs_globals.c new file mode 100644 index 000000000..60efe8246 --- /dev/null +++ b/fs/xfs/xfs_globals.c @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_error.h" + +/* + * Tunable XFS parameters. xfs_params is required even when CONFIG_SYSCTL=n, + * other XFS code uses these values. Times are measured in centisecs (i.e. + * 100ths of a second) with the exception of blockgc_timer, which is measured + * in seconds. + */ +xfs_param_t xfs_params = { + /* MIN DFLT MAX */ + .panic_mask = { 0, 0, XFS_PTAG_MASK}, + .error_level = { 0, 3, 11 }, + .syncd_timer = { 1*100, 30*100, 7200*100}, + .stats_clear = { 0, 0, 1 }, + .inherit_sync = { 0, 1, 1 }, + .inherit_nodump = { 0, 1, 1 }, + .inherit_noatim = { 0, 1, 1 }, + .inherit_nosym = { 0, 0, 1 }, + .rotorstep = { 1, 1, 255 }, + .inherit_nodfrg = { 0, 1, 1 }, + .fstrm_timer = { 1, 30*100, 3600*100}, + .blockgc_timer = { 1, 300, 3600*24}, +}; + +struct xfs_globals xfs_globals = { + .log_recovery_delay = 0, /* no delay by default */ + .mount_delay = 0, /* no delay by default */ +#ifdef XFS_ASSERT_FATAL + .bug_on_assert = true, /* assert failures BUG() */ +#else + .bug_on_assert = false, /* assert failures WARN() */ +#endif +#ifdef DEBUG + .pwork_threads = -1, /* automatic thread detection */ + .larp = false, /* log attribute replay */ +#endif + + /* + * Leave this many record slots empty when bulk loading btrees. By + * default we load new btree leaf blocks 75% full. + */ + .bload_leaf_slack = -1, + + /* + * Leave this many key/ptr slots empty when bulk loading btrees. By + * default we load new btree node blocks 75% full. + */ + .bload_node_slack = -1, +}; diff --git a/fs/xfs/xfs_handle.c b/fs/xfs/xfs_handle.c new file mode 100644 index 000000000..0689cade8 --- /dev/null +++ b/fs/xfs/xfs_handle.c @@ -0,0 +1,922 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * Copyright (c) 2022-2024 Oracle. + * All rights reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_shared.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_bmap_btree.h" +#include "xfs_inode.h" +#include "xfs_error.h" +#include "xfs_trace.h" +#include "xfs_trans.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_attr.h" +#include "xfs_ioctl.h" +#include "xfs_parent.h" +#include "xfs_handle.h" +#include "xfs_health.h" +#include "xfs_icache.h" +#include "xfs_export.h" +#include "xfs_xattr.h" +#include "xfs_acl.h" + +#include + +static inline size_t +xfs_filehandle_fid_len(void) +{ + struct xfs_handle *handle = NULL; + + return sizeof(struct xfs_fid) - sizeof(handle->ha_fid.fid_len); +} + +static inline size_t +xfs_filehandle_init( + struct xfs_mount *mp, + xfs_ino_t ino, + uint32_t gen, + struct xfs_handle *handle) +{ + memcpy(&handle->ha_fsid, mp->m_fixedfsid, sizeof(struct xfs_fsid)); + + handle->ha_fid.fid_len = xfs_filehandle_fid_len(); + handle->ha_fid.fid_pad = 0; + handle->ha_fid.fid_gen = gen; + handle->ha_fid.fid_ino = ino; + + return sizeof(struct xfs_handle); +} + +static inline size_t +xfs_fshandle_init( + struct xfs_mount *mp, + struct xfs_handle *handle) +{ + memcpy(&handle->ha_fsid, mp->m_fixedfsid, sizeof(struct xfs_fsid)); + memset(&handle->ha_fid, 0, sizeof(handle->ha_fid)); + + return sizeof(struct xfs_fsid); +} + +/* + * xfs_find_handle maps from userspace xfs_fsop_handlereq structure to + * a file or fs handle. + * + * XFS_IOC_PATH_TO_FSHANDLE + * returns fs handle for a mount point or path within that mount point + * XFS_IOC_FD_TO_HANDLE + * returns full handle for a FD opened in user space + * XFS_IOC_PATH_TO_HANDLE + * returns full handle for a path + */ +int +xfs_find_handle( + unsigned int cmd, + xfs_fsop_handlereq_t *hreq) +{ + int hsize; + xfs_handle_t handle; + struct inode *inode; + struct path path; + int error; + struct xfs_inode *ip; + + if (cmd == XFS_IOC_FD_TO_HANDLE) { + CLASS(fd, f)(hreq->fd); + + if (fd_empty(f)) + return -EBADF; + path = fd_file(f)->f_path; + path_get(&path); + } else { + error = user_path_at(AT_FDCWD, hreq->path, 0, &path); + if (error) + return error; + } + inode = d_inode(path.dentry); + ip = XFS_I(inode); + + /* + * We can only generate handles for inodes residing on a XFS filesystem, + * and only for regular files, directories or symbolic links. + */ + error = -EINVAL; + if (inode->i_sb->s_magic != XFS_SB_MAGIC) + goto out_put; + + error = -EBADF; + if (!S_ISREG(inode->i_mode) && + !S_ISDIR(inode->i_mode) && + !S_ISLNK(inode->i_mode)) + goto out_put; + + + memcpy(&handle.ha_fsid, ip->i_mount->m_fixedfsid, sizeof(xfs_fsid_t)); + + if (cmd == XFS_IOC_PATH_TO_FSHANDLE) + hsize = xfs_fshandle_init(ip->i_mount, &handle); + else + hsize = xfs_filehandle_init(ip->i_mount, inode->i_ino, + inode->i_generation, &handle); + + error = -EFAULT; + if (copy_to_user(hreq->ohandle, &handle, hsize) || + copy_to_user(hreq->ohandlen, &hsize, sizeof(__s32))) + goto out_put; + + error = 0; + + out_put: + path_put(&path); + return error; +} + +/* + * No need to do permission checks on the various pathname components + * as the handle operations are privileged. + */ +STATIC int +xfs_handle_acceptable( + void *context, + struct dentry *dentry) +{ + return 1; +} + +/* Convert handle already copied to kernel space into a dentry. */ +static struct dentry * +xfs_khandle_to_dentry( + struct file *file, + struct xfs_handle *handle) +{ + struct xfs_fid64 fid = { + .ino = handle->ha_fid.fid_ino, + .gen = handle->ha_fid.fid_gen, + }; + + /* + * Only allow handle opens under a directory. + */ + if (!S_ISDIR(file_inode(file)->i_mode)) + return ERR_PTR(-ENOTDIR); + + if (handle->ha_fid.fid_len != xfs_filehandle_fid_len()) + return ERR_PTR(-EINVAL); + + return exportfs_decode_fh(file->f_path.mnt, (struct fid *)&fid, 3, + FILEID_INO32_GEN | XFS_FILEID_TYPE_64FLAG, + xfs_handle_acceptable, NULL); +} + +/* Convert handle already copied to kernel space into an xfs_inode. */ +static struct xfs_inode * +xfs_khandle_to_inode( + struct file *file, + struct xfs_handle *handle) +{ + struct xfs_inode *ip = XFS_I(file_inode(file)); + struct xfs_mount *mp = ip->i_mount; + struct inode *inode; + + if (!S_ISDIR(VFS_I(ip)->i_mode)) + return ERR_PTR(-ENOTDIR); + + if (handle->ha_fid.fid_len != xfs_filehandle_fid_len()) + return ERR_PTR(-EINVAL); + + inode = xfs_nfs_get_inode(mp->m_super, handle->ha_fid.fid_ino, + handle->ha_fid.fid_gen); + if (IS_ERR(inode)) + return ERR_CAST(inode); + + return XFS_I(inode); +} + +/* + * Convert userspace handle data into a dentry. + */ +struct dentry * +xfs_handle_to_dentry( + struct file *parfilp, + void __user *uhandle, + u32 hlen) +{ + xfs_handle_t handle; + + if (hlen != sizeof(xfs_handle_t)) + return ERR_PTR(-EINVAL); + if (copy_from_user(&handle, uhandle, hlen)) + return ERR_PTR(-EFAULT); + + return xfs_khandle_to_dentry(parfilp, &handle); +} + +STATIC struct dentry * +xfs_handlereq_to_dentry( + struct file *parfilp, + xfs_fsop_handlereq_t *hreq) +{ + return xfs_handle_to_dentry(parfilp, hreq->ihandle, hreq->ihandlen); +} + +int +xfs_open_by_handle( + struct file *parfilp, + xfs_fsop_handlereq_t *hreq) +{ + const struct cred *cred = current_cred(); + int permflag; + struct inode *inode; + struct dentry *dentry; + fmode_t fmode; + struct path path __free(path_put) = {}; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + dentry = xfs_handlereq_to_dentry(parfilp, hreq); + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + inode = d_inode(dentry); + path.dentry = dentry; + + /* Restrict xfs_open_by_handle to directories & regular files. */ + if (!(S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode))) + return -EPERM; + +#if BITS_PER_LONG != 32 + hreq->oflags |= O_LARGEFILE; +#endif + + permflag = hreq->oflags; + fmode = OPEN_FMODE(permflag); + if ((!(permflag & O_APPEND) || (permflag & O_TRUNC)) && + (fmode & FMODE_WRITE) && IS_APPEND(inode)) + return -EPERM; + + if ((fmode & FMODE_WRITE) && IS_IMMUTABLE(inode)) + return -EPERM; + + /* Can't write directories. */ + if (S_ISDIR(inode->i_mode) && (fmode & FMODE_WRITE)) + return -EISDIR; + + path.mnt = mntget(parfilp->f_path.mnt); + + FD_PREPARE(fdf, 0, dentry_open(&path, hreq->oflags, cred)); + if (fdf.err) + return fdf.err; + + if (S_ISREG(inode->i_mode)) { + struct file *filp = fd_prepare_file(fdf); + + filp->f_flags |= O_NOATIME; + filp->f_mode |= FMODE_NOCMTIME; + } + + return fd_publish(fdf); +} + +int +xfs_readlink_by_handle( + struct file *parfilp, + xfs_fsop_handlereq_t *hreq) +{ + struct dentry *dentry; + __u32 olen; + int error; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + dentry = xfs_handlereq_to_dentry(parfilp, hreq); + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + + /* Restrict this handle operation to symlinks only. */ + if (!d_is_symlink(dentry)) { + error = -EINVAL; + goto out_dput; + } + + if (copy_from_user(&olen, hreq->ohandlen, sizeof(__u32))) { + error = -EFAULT; + goto out_dput; + } + + error = vfs_readlink(dentry, hreq->ohandle, olen); + + out_dput: + dput(dentry); + return error; +} + +/* + * Format an attribute and copy it out to the user's buffer. + * Take care to check values and protect against them changing later, + * we may be reading them directly out of a user buffer. + */ +static void +xfs_ioc_attr_put_listent( + struct xfs_attr_list_context *context, + int flags, + unsigned char *name, + int namelen, + void *value, + int valuelen) +{ + struct xfs_attrlist *alist = context->buffer; + struct xfs_attrlist_ent *aep; + int arraytop; + + ASSERT(!context->seen_enough); + ASSERT(context->count >= 0); + ASSERT(context->count < (ATTR_MAX_VALUELEN/8)); + ASSERT(context->firstu >= sizeof(*alist)); + ASSERT(context->firstu <= context->bufsize); + + /* + * Only list entries in the right namespace. + */ + if (context->attr_filter != (flags & XFS_ATTR_NSP_ONDISK_MASK)) + return; + + arraytop = sizeof(*alist) + + context->count * sizeof(alist->al_offset[0]); + + /* decrement by the actual bytes used by the attr */ + context->firstu -= round_up(offsetof(struct xfs_attrlist_ent, a_name) + + namelen + 1, sizeof(uint32_t)); + if (context->firstu < arraytop) { + trace_xfs_attr_list_full(context); + alist->al_more = 1; + context->seen_enough = 1; + return; + } + + aep = context->buffer + context->firstu; + aep->a_valuelen = valuelen; + memcpy(aep->a_name, name, namelen); + aep->a_name[namelen] = 0; + alist->al_offset[context->count++] = context->firstu; + alist->al_count = context->count; + trace_xfs_attr_list_add(context); +} + +static unsigned int +xfs_attr_filter( + u32 ioc_flags) +{ + if (ioc_flags & XFS_IOC_ATTR_ROOT) + return XFS_ATTR_ROOT; + if (ioc_flags & XFS_IOC_ATTR_SECURE) + return XFS_ATTR_SECURE; + return 0; +} + +static inline enum xfs_attr_update +xfs_xattr_flags( + u32 ioc_flags, + void *value) +{ + if (!value) + return XFS_ATTRUPDATE_REMOVE; + if (ioc_flags & XFS_IOC_ATTR_CREATE) + return XFS_ATTRUPDATE_CREATE; + if (ioc_flags & XFS_IOC_ATTR_REPLACE) + return XFS_ATTRUPDATE_REPLACE; + return XFS_ATTRUPDATE_UPSERT; +} + +int +xfs_ioc_attr_list( + struct xfs_inode *dp, + void __user *ubuf, + size_t bufsize, + int flags, + struct xfs_attrlist_cursor __user *ucursor) +{ + struct xfs_attr_list_context context = { }; + struct xfs_attrlist *alist; + void *buffer; + int error; + + if (bufsize < sizeof(struct xfs_attrlist) || + bufsize > XFS_XATTR_LIST_MAX) + return -EINVAL; + + /* + * Reject flags, only allow namespaces. + */ + if (flags & ~(XFS_IOC_ATTR_ROOT | XFS_IOC_ATTR_SECURE)) + return -EINVAL; + if (flags == (XFS_IOC_ATTR_ROOT | XFS_IOC_ATTR_SECURE)) + return -EINVAL; + + /* + * Validate the cursor. + */ + if (copy_from_user(&context.cursor, ucursor, sizeof(context.cursor))) + return -EFAULT; + if (context.cursor.pad1 || context.cursor.pad2) + return -EINVAL; + if (!context.cursor.initted && + (context.cursor.hashval || context.cursor.blkno || + context.cursor.offset)) + return -EINVAL; + + buffer = kvzalloc(bufsize, GFP_KERNEL); + if (!buffer) + return -ENOMEM; + + /* + * Initialize the output buffer. + */ + context.dp = dp; + context.resynch = 1; + context.attr_filter = xfs_attr_filter(flags); + context.bufsize = round_down(bufsize, sizeof(uint32_t)); + context.buffer = buffer; + context.firstu = context.bufsize; + context.put_listent = xfs_ioc_attr_put_listent; + + alist = context.buffer; + alist->al_count = 0; + alist->al_more = 0; + alist->al_offset[0] = context.bufsize; + + error = xfs_attr_list(&context); + if (error) + goto out_free; + + if (copy_to_user(ubuf, buffer, bufsize) || + copy_to_user(ucursor, &context.cursor, sizeof(context.cursor))) + error = -EFAULT; +out_free: + kvfree(buffer); + return error; +} + +int +xfs_attrlist_by_handle( + struct file *parfilp, + struct xfs_fsop_attrlist_handlereq __user *p) +{ + struct xfs_fsop_attrlist_handlereq al_hreq; + struct dentry *dentry; + int error = -ENOMEM; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (copy_from_user(&al_hreq, p, sizeof(al_hreq))) + return -EFAULT; + + dentry = xfs_handlereq_to_dentry(parfilp, &al_hreq.hreq); + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + + error = xfs_ioc_attr_list(XFS_I(d_inode(dentry)), al_hreq.buffer, + al_hreq.buflen, al_hreq.flags, &p->pos); + dput(dentry); + return error; +} + +static int +xfs_attrmulti_attr_get( + struct inode *inode, + unsigned char *name, + unsigned char __user *ubuf, + uint32_t *len, + uint32_t flags) +{ + struct xfs_da_args args = { + .dp = XFS_I(inode), + .attr_filter = xfs_attr_filter(flags), + .name = name, + .namelen = strlen(name), + .valuelen = *len, + }; + int error; + + if (*len > XFS_XATTR_SIZE_MAX) + return -EINVAL; + + error = xfs_attr_get(&args); + if (error) + goto out_kfree; + + *len = args.valuelen; + if (copy_to_user(ubuf, args.value, args.valuelen)) + error = -EFAULT; + +out_kfree: + kvfree(args.value); + return error; +} + +static int +xfs_attrmulti_attr_set( + struct inode *inode, + unsigned char *name, + const unsigned char __user *ubuf, + uint32_t len, + uint32_t flags) +{ + struct xfs_da_args args = { + .dp = XFS_I(inode), + .attr_filter = xfs_attr_filter(flags), + .name = name, + .namelen = strlen(name), + }; + int error; + + if (IS_IMMUTABLE(inode) || IS_APPEND(inode)) + return -EPERM; + + if (ubuf) { + if (len > XFS_XATTR_SIZE_MAX) + return -EINVAL; + args.value = memdup_user(ubuf, len); + if (IS_ERR(args.value)) + return PTR_ERR(args.value); + args.valuelen = len; + } + + error = xfs_attr_change(&args, xfs_xattr_flags(flags, args.value)); + if (!error && (flags & XFS_IOC_ATTR_ROOT)) + xfs_forget_acl(inode, name); + kfree(args.value); + return error; +} + +int +xfs_ioc_attrmulti_one( + struct file *parfilp, + struct inode *inode, + uint32_t opcode, + void __user *uname, + void __user *value, + uint32_t *len, + uint32_t flags) +{ + unsigned char *name; + int error; + + if ((flags & XFS_IOC_ATTR_ROOT) && (flags & XFS_IOC_ATTR_SECURE)) + return -EINVAL; + + name = strndup_user(uname, MAXNAMELEN); + if (IS_ERR(name)) + return PTR_ERR(name); + + switch (opcode) { + case ATTR_OP_GET: + error = xfs_attrmulti_attr_get(inode, name, value, len, flags); + break; + case ATTR_OP_REMOVE: + value = NULL; + *len = 0; + fallthrough; + case ATTR_OP_SET: + error = mnt_want_write_file(parfilp); + if (error) + break; + error = xfs_attrmulti_attr_set(inode, name, value, *len, flags); + mnt_drop_write_file(parfilp); + break; + default: + error = -EINVAL; + break; + } + + kfree(name); + return error; +} + +int +xfs_attrmulti_by_handle( + struct file *parfilp, + void __user *arg) +{ + int error; + xfs_attr_multiop_t *ops; + xfs_fsop_attrmulti_handlereq_t am_hreq; + struct dentry *dentry; + unsigned int i, size; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (copy_from_user(&am_hreq, arg, sizeof(xfs_fsop_attrmulti_handlereq_t))) + return -EFAULT; + + /* overflow check */ + if (am_hreq.opcount >= INT_MAX / sizeof(xfs_attr_multiop_t)) + return -E2BIG; + + dentry = xfs_handlereq_to_dentry(parfilp, &am_hreq.hreq); + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + + error = -E2BIG; + size = am_hreq.opcount * sizeof(xfs_attr_multiop_t); + if (!size || size > 16 * PAGE_SIZE) + goto out_dput; + + ops = memdup_user(am_hreq.ops, size); + if (IS_ERR(ops)) { + error = PTR_ERR(ops); + goto out_dput; + } + + error = 0; + for (i = 0; i < am_hreq.opcount; i++) { + ops[i].am_error = xfs_ioc_attrmulti_one(parfilp, + d_inode(dentry), ops[i].am_opcode, + ops[i].am_attrname, ops[i].am_attrvalue, + &ops[i].am_length, ops[i].am_flags); + } + + if (copy_to_user(am_hreq.ops, ops, size)) + error = -EFAULT; + + kfree(ops); + out_dput: + dput(dentry); + return error; +} + +struct xfs_getparents_ctx { + struct xfs_attr_list_context context; + struct xfs_getparents_by_handle gph; + + /* File to target */ + struct xfs_inode *ip; + + /* Internal buffer where we format records */ + void *krecords; + + /* Last record filled out */ + struct xfs_getparents_rec *lastrec; + + unsigned int count; +}; + +static inline unsigned int +xfs_getparents_rec_sizeof( + unsigned int namelen) +{ + return round_up(sizeof(struct xfs_getparents_rec) + namelen + 1, + sizeof(uint64_t)); +} + +static void +xfs_getparents_put_listent( + struct xfs_attr_list_context *context, + int flags, + unsigned char *name, + int namelen, + void *value, + int valuelen) +{ + struct xfs_getparents_ctx *gpx = + container_of(context, struct xfs_getparents_ctx, context); + struct xfs_inode *ip = context->dp; + struct xfs_mount *mp = ip->i_mount; + struct xfs_getparents *gp = &gpx->gph.gph_request; + struct xfs_getparents_rec *gpr = gpx->krecords + context->firstu; + unsigned short reclen = + xfs_getparents_rec_sizeof(namelen); + xfs_ino_t ino; + uint32_t gen; + int error; + + if (!(flags & XFS_ATTR_PARENT)) + return; + + error = xfs_parent_from_attr(mp, flags, name, namelen, value, valuelen, + &ino, &gen); + if (error) { + xfs_inode_mark_sick(ip, XFS_SICK_INO_PARENT); + context->seen_enough = -EFSCORRUPTED; + return; + } + + /* + * We found a parent pointer, but we've filled up the buffer. Signal + * to the caller that we did /not/ reach the end of the parent pointer + * recordset. + */ + if (context->firstu > context->bufsize - reclen) { + context->seen_enough = 1; + return; + } + + /* Format the parent pointer directly into the caller buffer. */ + gpr->gpr_reclen = reclen; + xfs_filehandle_init(mp, ino, gen, &gpr->gpr_parent); + memcpy(gpr->gpr_name, name, namelen); + gpr->gpr_name[namelen] = 0; + + trace_xfs_getparents_put_listent(ip, gp, context, gpr); + + context->firstu += reclen; + gpx->count++; + gpx->lastrec = gpr; +} + +/* Expand the last record to fill the rest of the caller's buffer. */ +static inline void +xfs_getparents_expand_lastrec( + struct xfs_getparents_ctx *gpx) +{ + struct xfs_getparents *gp = &gpx->gph.gph_request; + struct xfs_getparents_rec *gpr = gpx->lastrec; + + if (!gpx->lastrec) + gpr = gpx->krecords; + + gpr->gpr_reclen = gp->gp_bufsize - ((void *)gpr - gpx->krecords); + + trace_xfs_getparents_expand_lastrec(gpx->ip, gp, &gpx->context, gpr); +} + +/* Retrieve the parent pointers for a given inode. */ +STATIC int +xfs_getparents( + struct xfs_getparents_ctx *gpx) +{ + struct xfs_getparents *gp = &gpx->gph.gph_request; + struct xfs_inode *ip = gpx->ip; + struct xfs_mount *mp = ip->i_mount; + size_t bufsize; + int error; + + /* Check size of buffer requested by user */ + if (gp->gp_bufsize > XFS_XATTR_LIST_MAX) + return -ENOMEM; + if (gp->gp_bufsize < xfs_getparents_rec_sizeof(1)) + return -EINVAL; + + if (gp->gp_iflags & ~XFS_GETPARENTS_IFLAGS_ALL) + return -EINVAL; + if (gp->gp_reserved) + return -EINVAL; + + bufsize = round_down(gp->gp_bufsize, sizeof(uint64_t)); + gpx->krecords = kvzalloc(bufsize, GFP_KERNEL); + if (!gpx->krecords) { + bufsize = min(bufsize, PAGE_SIZE); + gpx->krecords = kvzalloc(bufsize, GFP_KERNEL); + if (!gpx->krecords) + return -ENOMEM; + } + + gpx->context.dp = ip; + gpx->context.resynch = 1; + gpx->context.put_listent = xfs_getparents_put_listent; + gpx->context.bufsize = bufsize; + /* firstu is used to track the bytes filled in the buffer */ + gpx->context.firstu = 0; + + /* Copy the cursor provided by caller */ + memcpy(&gpx->context.cursor, &gp->gp_cursor, + sizeof(struct xfs_attrlist_cursor)); + gpx->count = 0; + gp->gp_oflags = 0; + + trace_xfs_getparents_begin(ip, gp, &gpx->context.cursor); + + error = xfs_attr_list(&gpx->context); + if (error) + goto out_free_buf; + if (gpx->context.seen_enough < 0) { + error = gpx->context.seen_enough; + goto out_free_buf; + } + xfs_getparents_expand_lastrec(gpx); + + /* Update the caller with the current cursor position */ + memcpy(&gp->gp_cursor, &gpx->context.cursor, + sizeof(struct xfs_attrlist_cursor)); + + /* Is this the root directory? */ + if (I_INO(ip) == mp->m_sb.sb_rootino) + gp->gp_oflags |= XFS_GETPARENTS_OFLAG_ROOT; + + if (gpx->context.seen_enough == 0) { + /* + * If we did not run out of buffer space, then we reached the + * end of the pptr recordset, so set the DONE flag. + */ + gp->gp_oflags |= XFS_GETPARENTS_OFLAG_DONE; + } else if (gpx->count == 0) { + /* + * If we ran out of buffer space before copying any parent + * pointers at all, the caller's buffer was too short. Tell + * userspace that, erm, the message is too long. + */ + error = -EMSGSIZE; + goto out_free_buf; + } + + trace_xfs_getparents_end(ip, gp, &gpx->context.cursor); + + ASSERT(gpx->context.firstu <= gpx->gph.gph_request.gp_bufsize); + + /* Copy the records to userspace. */ + if (copy_to_user(u64_to_user_ptr(gpx->gph.gph_request.gp_buffer), + gpx->krecords, gpx->context.firstu)) + error = -EFAULT; + +out_free_buf: + kvfree(gpx->krecords); + gpx->krecords = NULL; + return error; +} + +/* Retrieve the parents of this file and pass them back to userspace. */ +int +xfs_ioc_getparents( + struct file *file, + struct xfs_getparents __user *ureq) +{ + struct xfs_getparents_ctx gpx = { + .ip = XFS_I(file_inode(file)), + }; + struct xfs_getparents *kreq = &gpx.gph.gph_request; + struct xfs_mount *mp = gpx.ip->i_mount; + int error; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (!xfs_has_parent(mp)) + return -EOPNOTSUPP; + if (copy_from_user(kreq, ureq, sizeof(*kreq))) + return -EFAULT; + + error = xfs_getparents(&gpx); + if (error) + return error; + + if (copy_to_user(ureq, kreq, sizeof(*kreq))) + return -EFAULT; + + return 0; +} + +/* Retrieve the parents of this file handle and pass them back to userspace. */ +int +xfs_ioc_getparents_by_handle( + struct file *file, + struct xfs_getparents_by_handle __user *ureq) +{ + struct xfs_getparents_ctx gpx = { }; + struct xfs_inode *ip = XFS_I(file_inode(file)); + struct xfs_mount *mp = ip->i_mount; + struct xfs_getparents_by_handle *kreq = &gpx.gph; + struct xfs_handle *handle = &kreq->gph_handle; + int error; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (!xfs_has_parent(mp)) + return -EOPNOTSUPP; + if (copy_from_user(kreq, ureq, sizeof(*kreq))) + return -EFAULT; + + /* + * We don't use exportfs_decode_fh because it does too much work here. + * If the handle refers to a directory, the exportfs code will walk + * upwards through the directory tree to connect the dentries to the + * root directory dentry. For GETPARENTS we don't care about that + * because we're not actually going to open a file descriptor; we only + * want to open an inode and read its parent pointers. + * + * Note that xfs_scrub uses GETPARENTS to log that it will try to fix a + * corrupted file's metadata. For this usecase we would really rather + * userspace single-step the path reconstruction to avoid loops or + * other strange things if the directory tree is corrupt. + */ + gpx.ip = xfs_khandle_to_inode(file, handle); + if (IS_ERR(gpx.ip)) + return PTR_ERR(gpx.ip); + + error = xfs_getparents(&gpx); + if (error) + goto out_rele; + + if (copy_to_user(ureq, kreq, sizeof(*kreq))) + error = -EFAULT; + +out_rele: + xfs_irele(gpx.ip); + return error; +} diff --git a/fs/xfs/xfs_handle.h b/fs/xfs/xfs_handle.h new file mode 100644 index 000000000..6799a86d8 --- /dev/null +++ b/fs/xfs/xfs_handle.h @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * Copyright (c) 2022-2024 Oracle. + * All rights reserved. + */ +#ifndef __XFS_HANDLE_H__ +#define __XFS_HANDLE_H__ + +int xfs_attrlist_by_handle(struct file *parfilp, + struct xfs_fsop_attrlist_handlereq __user *p); +int xfs_attrmulti_by_handle(struct file *parfilp, void __user *arg); + +int xfs_find_handle(unsigned int cmd, struct xfs_fsop_handlereq *hreq); +int xfs_open_by_handle(struct file *parfilp, struct xfs_fsop_handlereq *hreq); +int xfs_readlink_by_handle(struct file *parfilp, + struct xfs_fsop_handlereq *hreq); + +int xfs_ioc_attrmulti_one(struct file *parfilp, struct inode *inode, + uint32_t opcode, void __user *uname, void __user *value, + uint32_t *len, uint32_t flags); +int xfs_ioc_attr_list(struct xfs_inode *dp, void __user *ubuf, + size_t bufsize, int flags, + struct xfs_attrlist_cursor __user *ucursor); + +struct dentry *xfs_handle_to_dentry(struct file *parfilp, void __user *uhandle, + u32 hlen); + +int xfs_ioc_getparents(struct file *file, struct xfs_getparents __user *arg); +int xfs_ioc_getparents_by_handle(struct file *file, + struct xfs_getparents_by_handle __user *arg); + +#endif /* __XFS_HANDLE_H__ */ diff --git a/fs/xfs/xfs_health.c b/fs/xfs/xfs_health.c new file mode 100644 index 000000000..239b843e8 --- /dev/null +++ b/fs/xfs/xfs_health.c @@ -0,0 +1,744 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2019 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_trace.h" +#include "xfs_health.h" +#include "xfs_ag.h" +#include "xfs_btree.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_quota_defs.h" +#include "xfs_rtgroup.h" +#include "xfs_healthmon.h" + +#include + +static void +xfs_health_unmount_group( + struct xfs_group *xg, + bool *warn) +{ + unsigned int sick = 0; + unsigned int checked = 0; + + xfs_group_measure_sickness(xg, &sick, &checked); + if (sick) { + trace_xfs_group_unfixed_corruption(xg, sick); + *warn = true; + } +} + +/* + * Warn about metadata corruption that we detected but haven't fixed, and + * make sure we're not sitting on anything that would get in the way of + * recovery. + */ +void +xfs_health_unmount( + struct xfs_mount *mp) +{ + struct xfs_perag *pag = NULL; + struct xfs_rtgroup *rtg = NULL; + unsigned int sick = 0; + unsigned int checked = 0; + bool warn = false; + + if (xfs_is_shutdown(mp)) + return; + + /* Measure AG corruption levels. */ + while ((pag = xfs_perag_next(mp, pag))) + xfs_health_unmount_group(pag_group(pag), &warn); + + /* Measure realtime group corruption levels. */ + while ((rtg = xfs_rtgroup_next(mp, rtg))) + xfs_health_unmount_group(rtg_group(rtg), &warn); + + /* + * Measure fs corruption and keep the sample around for the warning. + * See the note below for why we exempt FS_COUNTERS. + */ + xfs_fs_measure_sickness(mp, &sick, &checked); + if (sick & ~XFS_SICK_FS_COUNTERS) { + trace_xfs_fs_unfixed_corruption(mp, sick); + warn = true; + } + + if (warn) { + xfs_warn(mp, +"Uncorrected metadata errors detected; please run xfs_repair."); + + /* + * We discovered uncorrected metadata problems at some point + * during this filesystem mount and have advised the + * administrator to run repair once the unmount completes. + * + * However, we must be careful -- when FSCOUNTERS are flagged + * unhealthy, the unmount procedure omits writing the clean + * unmount record to the log so that the next mount will run + * recovery and recompute the summary counters. In other + * words, we leave a dirty log to get the counters fixed. + * + * Unfortunately, xfs_repair cannot recover dirty logs, so if + * there were filesystem problems, FSCOUNTERS was flagged, and + * the administrator takes our advice to run xfs_repair, + * they'll have to zap the log before repairing structures. + * We don't really want to encourage this, so we mark the + * FSCOUNTERS healthy so that a subsequent repair run won't see + * a dirty log. + */ + if (sick & XFS_SICK_FS_COUNTERS) + xfs_fs_mark_healthy(mp, XFS_SICK_FS_COUNTERS); + } +} + +/* Mark unhealthy per-fs metadata. */ +void +xfs_fs_mark_sick( + struct xfs_mount *mp, + unsigned int mask) +{ + unsigned int old_mask; + + ASSERT(!(mask & ~XFS_SICK_FS_ALL)); + trace_xfs_fs_mark_sick(mp, mask); + + spin_lock(&mp->m_sb_lock); + old_mask = mp->m_fs_sick; + mp->m_fs_sick |= mask; + spin_unlock(&mp->m_sb_lock); + + fserror_report_metadata(mp->m_super, -EFSCORRUPTED, GFP_NOFS); + if (mask) + xfs_healthmon_report_fs(mp, XFS_HEALTHMON_SICK, old_mask, mask); +} + +/* Mark per-fs metadata as having been checked and found unhealthy by fsck. */ +void +xfs_fs_mark_corrupt( + struct xfs_mount *mp, + unsigned int mask) +{ + unsigned int old_mask; + + ASSERT(!(mask & ~XFS_SICK_FS_ALL)); + trace_xfs_fs_mark_corrupt(mp, mask); + + spin_lock(&mp->m_sb_lock); + old_mask = mp->m_fs_sick; + mp->m_fs_sick |= mask; + mp->m_fs_checked |= mask; + spin_unlock(&mp->m_sb_lock); + + fserror_report_metadata(mp->m_super, -EFSCORRUPTED, GFP_NOFS); + if (mask) + xfs_healthmon_report_fs(mp, XFS_HEALTHMON_CORRUPT, old_mask, + mask); +} + +/* Mark a per-fs metadata healed. */ +void +xfs_fs_mark_healthy( + struct xfs_mount *mp, + unsigned int mask) +{ + unsigned int old_mask; + + ASSERT(!(mask & ~XFS_SICK_FS_ALL)); + trace_xfs_fs_mark_healthy(mp, mask); + + spin_lock(&mp->m_sb_lock); + old_mask = mp->m_fs_sick; + mp->m_fs_sick &= ~mask; + if (!(mp->m_fs_sick & XFS_SICK_FS_PRIMARY)) + mp->m_fs_sick &= ~XFS_SICK_FS_SECONDARY; + mp->m_fs_checked |= mask; + spin_unlock(&mp->m_sb_lock); + + if (mask) + xfs_healthmon_report_fs(mp, XFS_HEALTHMON_HEALTHY, old_mask, + mask); +} + +/* Sample which per-fs metadata are unhealthy. */ +void +xfs_fs_measure_sickness( + struct xfs_mount *mp, + unsigned int *sick, + unsigned int *checked) +{ + spin_lock(&mp->m_sb_lock); + *sick = mp->m_fs_sick; + *checked = mp->m_fs_checked; + spin_unlock(&mp->m_sb_lock); +} + +/* Mark unhealthy per-ag metadata given a raw AG number. */ +void +xfs_agno_mark_sick( + struct xfs_mount *mp, + xfs_agnumber_t agno, + unsigned int mask) +{ + struct xfs_perag *pag = xfs_perag_get(mp, agno); + + /* per-ag structure not set up yet? */ + if (!pag) + return; + + xfs_ag_mark_sick(pag, mask); + xfs_perag_put(pag); +} + +static inline void +xfs_group_check_mask( + struct xfs_group *xg, + unsigned int mask) +{ + if (xg->xg_type == XG_TYPE_AG) + ASSERT(!(mask & ~XFS_SICK_AG_ALL)); + else + ASSERT(!(mask & ~XFS_SICK_RG_ALL)); +} + +/* Mark unhealthy per-ag metadata. */ +void +xfs_group_mark_sick( + struct xfs_group *xg, + unsigned int mask) +{ + unsigned int old_mask; + + xfs_group_check_mask(xg, mask); + trace_xfs_group_mark_sick(xg, mask); + + spin_lock(&xg->xg_state_lock); + old_mask = xg->xg_sick; + xg->xg_sick |= mask; + spin_unlock(&xg->xg_state_lock); + + fserror_report_metadata(xg->xg_mount->m_super, -EFSCORRUPTED, GFP_NOFS); + if (mask) + xfs_healthmon_report_group(xg, XFS_HEALTHMON_SICK, old_mask, + mask); +} + +/* + * Mark per-group metadata as having been checked and found unhealthy by fsck. + */ +void +xfs_group_mark_corrupt( + struct xfs_group *xg, + unsigned int mask) +{ + unsigned int old_mask; + + xfs_group_check_mask(xg, mask); + trace_xfs_group_mark_corrupt(xg, mask); + + spin_lock(&xg->xg_state_lock); + old_mask = xg->xg_sick; + xg->xg_sick |= mask; + xg->xg_checked |= mask; + spin_unlock(&xg->xg_state_lock); + + fserror_report_metadata(xg->xg_mount->m_super, -EFSCORRUPTED, GFP_NOFS); + if (mask) + xfs_healthmon_report_group(xg, XFS_HEALTHMON_CORRUPT, old_mask, + mask); +} + +/* + * Mark per-group metadata ok. + */ +void +xfs_group_mark_healthy( + struct xfs_group *xg, + unsigned int mask) +{ + unsigned int old_mask; + + xfs_group_check_mask(xg, mask); + trace_xfs_group_mark_healthy(xg, mask); + + spin_lock(&xg->xg_state_lock); + old_mask = xg->xg_sick; + xg->xg_sick &= ~mask; + if (!(xg->xg_sick & XFS_SICK_AG_PRIMARY)) + xg->xg_sick &= ~XFS_SICK_AG_SECONDARY; + xg->xg_checked |= mask; + spin_unlock(&xg->xg_state_lock); + + if (mask) + xfs_healthmon_report_group(xg, XFS_HEALTHMON_HEALTHY, old_mask, + mask); +} + +/* Sample which per-ag metadata are unhealthy. */ +void +xfs_group_measure_sickness( + struct xfs_group *xg, + unsigned int *sick, + unsigned int *checked) +{ + spin_lock(&xg->xg_state_lock); + *sick = xg->xg_sick; + *checked = xg->xg_checked; + spin_unlock(&xg->xg_state_lock); +} + +/* Mark unhealthy per-rtgroup metadata given a raw rt group number. */ +void +xfs_rgno_mark_sick( + struct xfs_mount *mp, + xfs_rgnumber_t rgno, + unsigned int mask) +{ + struct xfs_rtgroup *rtg = xfs_rtgroup_get(mp, rgno); + + /* per-rtgroup structure not set up yet? */ + if (!rtg) + return; + + xfs_group_mark_sick(rtg_group(rtg), mask); + xfs_rtgroup_put(rtg); +} + +static inline void xfs_inode_report_fserror(struct xfs_inode *ip) +{ + /* + * Do not report inodes being constructed or freed, or metadata inodes, + * to fsnotify. + */ + if (xfs_iflags_test(ip, XFS_INEW | XFS_IRECLAIM) || + xfs_is_internal_inode(ip)) { + fserror_report_metadata(ip->i_mount->m_super, -EFSCORRUPTED, + GFP_NOFS); + return; + } + + fserror_report_file_metadata(VFS_I(ip), -EFSCORRUPTED, GFP_NOFS); +} + +/* Mark the unhealthy parts of an inode. */ +void +xfs_inode_mark_sick( + struct xfs_inode *ip, + unsigned int mask) +{ + unsigned int old_mask; + + ASSERT(!(mask & ~XFS_SICK_INO_ALL)); + trace_xfs_inode_mark_sick(ip, mask); + + spin_lock(&ip->i_flags_lock); + old_mask = ip->i_sick; + ip->i_sick |= mask; + spin_unlock(&ip->i_flags_lock); + + /* + * Keep this inode around so we don't lose the sickness report. Scrub + * grabs inodes with DONTCACHE assuming that most inode are ok, which + * is not the case here. + */ + spin_lock(&VFS_I(ip)->i_lock); + inode_state_clear(VFS_I(ip), I_DONTCACHE); + spin_unlock(&VFS_I(ip)->i_lock); + + xfs_inode_report_fserror(ip); + if (mask) + xfs_healthmon_report_inode(ip, XFS_HEALTHMON_SICK, old_mask, + mask); +} + +/* Mark inode metadata as having been checked and found unhealthy by fsck. */ +void +xfs_inode_mark_corrupt( + struct xfs_inode *ip, + unsigned int mask) +{ + unsigned int old_mask; + + ASSERT(!(mask & ~XFS_SICK_INO_ALL)); + trace_xfs_inode_mark_corrupt(ip, mask); + + spin_lock(&ip->i_flags_lock); + old_mask = ip->i_sick; + ip->i_sick |= mask; + ip->i_checked |= mask; + spin_unlock(&ip->i_flags_lock); + + /* + * Keep this inode around so we don't lose the sickness report. Scrub + * grabs inodes with DONTCACHE assuming that most inode are ok, which + * is not the case here. + */ + spin_lock(&VFS_I(ip)->i_lock); + inode_state_clear(VFS_I(ip), I_DONTCACHE); + spin_unlock(&VFS_I(ip)->i_lock); + + xfs_inode_report_fserror(ip); + if (mask) + xfs_healthmon_report_inode(ip, XFS_HEALTHMON_CORRUPT, old_mask, + mask); +} + +/* Mark parts of an inode healed. */ +void +xfs_inode_mark_healthy( + struct xfs_inode *ip, + unsigned int mask) +{ + unsigned int old_mask; + + ASSERT(!(mask & ~XFS_SICK_INO_ALL)); + trace_xfs_inode_mark_healthy(ip, mask); + + spin_lock(&ip->i_flags_lock); + old_mask = ip->i_sick; + ip->i_sick &= ~mask; + if (!(ip->i_sick & XFS_SICK_INO_PRIMARY)) + ip->i_sick &= ~XFS_SICK_INO_SECONDARY; + ip->i_checked |= mask; + spin_unlock(&ip->i_flags_lock); + + if (mask) + xfs_healthmon_report_inode(ip, XFS_HEALTHMON_HEALTHY, old_mask, + mask); +} + +/* Sample which parts of an inode are unhealthy. */ +void +xfs_inode_measure_sickness( + struct xfs_inode *ip, + unsigned int *sick, + unsigned int *checked) +{ + spin_lock(&ip->i_flags_lock); + *sick = ip->i_sick; + *checked = ip->i_checked; + spin_unlock(&ip->i_flags_lock); +} + +/* Mappings between internal sick masks and ioctl sick masks. */ + +struct ioctl_sick_map { + unsigned int sick_mask; + unsigned int ioctl_mask; +}; + +#define for_each_sick_map(map, m) \ + for ((m) = (map); (m) < (map) + ARRAY_SIZE(map); (m)++) + +static const struct ioctl_sick_map fs_map[] = { + { XFS_SICK_FS_COUNTERS, XFS_FSOP_GEOM_SICK_COUNTERS}, + { XFS_SICK_FS_UQUOTA, XFS_FSOP_GEOM_SICK_UQUOTA }, + { XFS_SICK_FS_GQUOTA, XFS_FSOP_GEOM_SICK_GQUOTA }, + { XFS_SICK_FS_PQUOTA, XFS_FSOP_GEOM_SICK_PQUOTA }, + { XFS_SICK_FS_QUOTACHECK, XFS_FSOP_GEOM_SICK_QUOTACHECK }, + { XFS_SICK_FS_NLINKS, XFS_FSOP_GEOM_SICK_NLINKS }, + { XFS_SICK_FS_METADIR, XFS_FSOP_GEOM_SICK_METADIR }, + { XFS_SICK_FS_METAPATH, XFS_FSOP_GEOM_SICK_METAPATH }, +}; + +static const struct ioctl_sick_map rt_map[] = { + { XFS_SICK_RG_BITMAP, XFS_FSOP_GEOM_SICK_RT_BITMAP }, + { XFS_SICK_RG_SUMMARY, XFS_FSOP_GEOM_SICK_RT_SUMMARY }, +}; + +static inline void +xfgeo_health_tick( + struct xfs_fsop_geom *geo, + unsigned int sick, + unsigned int checked, + const struct ioctl_sick_map *m) +{ + if (checked & m->sick_mask) + geo->checked |= m->ioctl_mask; + if (sick & m->sick_mask) + geo->sick |= m->ioctl_mask; +} + +/* Fill out fs geometry health info. */ +void +xfs_fsop_geom_health( + struct xfs_mount *mp, + struct xfs_fsop_geom *geo) +{ + struct xfs_rtgroup *rtg = NULL; + const struct ioctl_sick_map *m; + unsigned int sick; + unsigned int checked; + + geo->sick = 0; + geo->checked = 0; + + xfs_fs_measure_sickness(mp, &sick, &checked); + for_each_sick_map(fs_map, m) + xfgeo_health_tick(geo, sick, checked, m); + + while ((rtg = xfs_rtgroup_next(mp, rtg))) { + xfs_group_measure_sickness(rtg_group(rtg), &sick, &checked); + for_each_sick_map(rt_map, m) + xfgeo_health_tick(geo, sick, checked, m); + } +} + +/* + * Translate XFS_SICK_FS_* into XFS_FSOP_GEOM_SICK_* except for the rt free + * space codes, which are sent via the rtgroup events. + */ +unsigned int +xfs_healthmon_fs_mask( + unsigned int sick_mask) +{ + const struct ioctl_sick_map *m; + unsigned int ioctl_mask = 0; + + for_each_sick_map(fs_map, m) { + if (sick_mask & m->sick_mask) + ioctl_mask |= m->ioctl_mask; + } + + return ioctl_mask; +} + +static const struct ioctl_sick_map ag_map[] = { + { XFS_SICK_AG_SB, XFS_AG_GEOM_SICK_SB }, + { XFS_SICK_AG_AGF, XFS_AG_GEOM_SICK_AGF }, + { XFS_SICK_AG_AGFL, XFS_AG_GEOM_SICK_AGFL }, + { XFS_SICK_AG_AGI, XFS_AG_GEOM_SICK_AGI }, + { XFS_SICK_AG_BNOBT, XFS_AG_GEOM_SICK_BNOBT }, + { XFS_SICK_AG_CNTBT, XFS_AG_GEOM_SICK_CNTBT }, + { XFS_SICK_AG_INOBT, XFS_AG_GEOM_SICK_INOBT }, + { XFS_SICK_AG_FINOBT, XFS_AG_GEOM_SICK_FINOBT }, + { XFS_SICK_AG_RMAPBT, XFS_AG_GEOM_SICK_RMAPBT }, + { XFS_SICK_AG_REFCNTBT, XFS_AG_GEOM_SICK_REFCNTBT }, + { XFS_SICK_AG_INODES, XFS_AG_GEOM_SICK_INODES }, +}; + +/* Fill out ag geometry health info. */ +void +xfs_ag_geom_health( + struct xfs_perag *pag, + struct xfs_ag_geometry *ageo) +{ + const struct ioctl_sick_map *m; + unsigned int sick; + unsigned int checked; + + ageo->ag_sick = 0; + ageo->ag_checked = 0; + + xfs_group_measure_sickness(pag_group(pag), &sick, &checked); + for_each_sick_map(ag_map, m) { + if (checked & m->sick_mask) + ageo->ag_checked |= m->ioctl_mask; + if (sick & m->sick_mask) + ageo->ag_sick |= m->ioctl_mask; + } +} + +/* Translate XFS_SICK_AG_* into XFS_AG_GEOM_SICK_*. */ +unsigned int +xfs_healthmon_perag_mask( + unsigned int sick_mask) +{ + const struct ioctl_sick_map *m; + unsigned int ioctl_mask = 0; + + for_each_sick_map(ag_map, m) { + if (sick_mask & m->sick_mask) + ioctl_mask |= m->ioctl_mask; + } + + return ioctl_mask; +} + +static const struct ioctl_sick_map rtgroup_map[] = { + { XFS_SICK_RG_SUPER, XFS_RTGROUP_GEOM_SICK_SUPER }, + { XFS_SICK_RG_BITMAP, XFS_RTGROUP_GEOM_SICK_BITMAP }, + { XFS_SICK_RG_SUMMARY, XFS_RTGROUP_GEOM_SICK_SUMMARY }, + { XFS_SICK_RG_RMAPBT, XFS_RTGROUP_GEOM_SICK_RMAPBT }, + { XFS_SICK_RG_REFCNTBT, XFS_RTGROUP_GEOM_SICK_REFCNTBT }, +}; + +/* Fill out rtgroup geometry health info. */ +void +xfs_rtgroup_geom_health( + struct xfs_rtgroup *rtg, + struct xfs_rtgroup_geometry *rgeo) +{ + const struct ioctl_sick_map *m; + unsigned int sick; + unsigned int checked; + + rgeo->rg_sick = 0; + rgeo->rg_checked = 0; + + xfs_group_measure_sickness(rtg_group(rtg), &sick, &checked); + for_each_sick_map(rtgroup_map, m) { + if (checked & m->sick_mask) + rgeo->rg_checked |= m->ioctl_mask; + if (sick & m->sick_mask) + rgeo->rg_sick |= m->ioctl_mask; + } +} + +/* Translate XFS_SICK_RG_* into XFS_RTGROUP_GEOM_SICK_*. */ +unsigned int +xfs_healthmon_rtgroup_mask( + unsigned int sick_mask) +{ + const struct ioctl_sick_map *m; + unsigned int ioctl_mask = 0; + + for_each_sick_map(rtgroup_map, m) { + if (sick_mask & m->sick_mask) + ioctl_mask |= m->ioctl_mask; + } + + return ioctl_mask; +} + +static const struct ioctl_sick_map ino_map[] = { + { XFS_SICK_INO_CORE, XFS_BS_SICK_INODE }, + { XFS_SICK_INO_BMBTD, XFS_BS_SICK_BMBTD }, + { XFS_SICK_INO_BMBTA, XFS_BS_SICK_BMBTA }, + { XFS_SICK_INO_BMBTC, XFS_BS_SICK_BMBTC }, + { XFS_SICK_INO_DIR, XFS_BS_SICK_DIR }, + { XFS_SICK_INO_XATTR, XFS_BS_SICK_XATTR }, + { XFS_SICK_INO_SYMLINK, XFS_BS_SICK_SYMLINK }, + { XFS_SICK_INO_PARENT, XFS_BS_SICK_PARENT }, + { XFS_SICK_INO_BMBTD_ZAPPED, XFS_BS_SICK_BMBTD }, + { XFS_SICK_INO_BMBTA_ZAPPED, XFS_BS_SICK_BMBTA }, + { XFS_SICK_INO_DIR_ZAPPED, XFS_BS_SICK_DIR }, + { XFS_SICK_INO_SYMLINK_ZAPPED, XFS_BS_SICK_SYMLINK }, + { XFS_SICK_INO_DIRTREE, XFS_BS_SICK_DIRTREE }, +}; + +/* Fill out bulkstat health info. */ +void +xfs_bulkstat_health( + struct xfs_inode *ip, + struct xfs_bulkstat *bs) +{ + const struct ioctl_sick_map *m; + unsigned int sick; + unsigned int checked; + + bs->bs_sick = 0; + bs->bs_checked = 0; + + xfs_inode_measure_sickness(ip, &sick, &checked); + for_each_sick_map(ino_map, m) { + if (checked & m->sick_mask) + bs->bs_checked |= m->ioctl_mask; + if (sick & m->sick_mask) + bs->bs_sick |= m->ioctl_mask; + } +} + +/* Translate XFS_SICK_INO_* into XFS_BS_SICK_*. */ +unsigned int +xfs_healthmon_inode_mask( + unsigned int sick_mask) +{ + const struct ioctl_sick_map *m; + unsigned int ioctl_mask = 0; + + for_each_sick_map(ino_map, m) { + if (sick_mask & m->sick_mask) + ioctl_mask |= m->ioctl_mask; + } + + return ioctl_mask; +} + +/* Mark a block mapping sick. */ +void +xfs_bmap_mark_sick( + struct xfs_inode *ip, + int whichfork) +{ + unsigned int mask; + + switch (whichfork) { + case XFS_DATA_FORK: + mask = XFS_SICK_INO_BMBTD; + break; + case XFS_ATTR_FORK: + mask = XFS_SICK_INO_BMBTA; + break; + case XFS_COW_FORK: + mask = XFS_SICK_INO_BMBTC; + break; + default: + ASSERT(0); + return; + } + + xfs_inode_mark_sick(ip, mask); +} + +/* Record observations of btree corruption with the health tracking system. */ +void +xfs_btree_mark_sick( + struct xfs_btree_cur *cur) +{ + if (xfs_btree_is_bmap(cur->bc_ops)) { + xfs_bmap_mark_sick(cur->bc_ino.ip, cur->bc_ino.whichfork); + /* no health state tracking for ephemeral btrees */ + } else if (cur->bc_ops->type != XFS_BTREE_TYPE_MEM) { + ASSERT(cur->bc_group); + ASSERT(cur->bc_ops->sick_mask); + xfs_group_mark_sick(cur->bc_group, cur->bc_ops->sick_mask); + } +} + +/* + * Record observations of dir/attr btree corruption with the health tracking + * system. + */ +void +xfs_dirattr_mark_sick( + struct xfs_inode *ip, + int whichfork) +{ + unsigned int mask; + + switch (whichfork) { + case XFS_DATA_FORK: + mask = XFS_SICK_INO_DIR; + break; + case XFS_ATTR_FORK: + mask = XFS_SICK_INO_XATTR; + break; + default: + ASSERT(0); + return; + } + + xfs_inode_mark_sick(ip, mask); +} + +/* + * Record observations of dir/attr btree corruption with the health tracking + * system. + */ +void +xfs_da_mark_sick( + struct xfs_da_args *args) +{ + xfs_dirattr_mark_sick(args->dp, args->whichfork); +} diff --git a/fs/xfs/xfs_healthmon.c b/fs/xfs/xfs_healthmon.c new file mode 100644 index 000000000..4521ffdab --- /dev/null +++ b/fs/xfs/xfs_healthmon.c @@ -0,0 +1,1260 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2024-2026 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_trace.h" +#include "xfs_ag.h" +#include "xfs_btree.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_quota_defs.h" +#include "xfs_rtgroup.h" +#include "xfs_health.h" +#include "xfs_healthmon.h" +#include "xfs_fsops.h" +#include "xfs_notify_failure.h" +#include "xfs_file.h" +#include "xfs_ioctl.h" + +#include +#include +#include +#include + +/* + * Live Health Monitoring + * ====================== + * + * Autonomous self-healing of XFS filesystems requires a means for the kernel + * to send filesystem health events to a monitoring daemon in userspace. To + * accomplish this, we establish a thread_with_file kthread object to handle + * translating internal events about filesystem health into a format that can + * be parsed easily by userspace. When those internal events occur, the core + * filesystem code calls this health monitor to convey the events to userspace. + * Userspace reads events from the file descriptor returned by the ioctl. + * + * The healthmon abstraction has a weak reference to the host filesystem mount + * so that the queueing and processing of the events do not pin the mount and + * cannot slow down the main filesystem. The healthmon object can exist past + * the end of the filesystem mount. + */ + +/* sign of a detached health monitor */ +#define DETACHED_MOUNT_COOKIE ((uintptr_t)0) + +/* Constrain the number of event objects that can build up in memory. */ +#define XFS_HEALTHMON_MAX_EVENTS (SZ_32K / \ + sizeof(struct xfs_healthmon_event)) + +/* Constrain the size of the output buffer for read_iter. */ +#define XFS_HEALTHMON_MAX_OUTBUF SZ_64K + +/* spinlock for atomically updating xfs_mount <-> xfs_healthmon pointers */ +static DEFINE_SPINLOCK(xfs_healthmon_lock); + +/* Grab a reference to the healthmon object for a given mount, if any. */ +static struct xfs_healthmon * +xfs_healthmon_get( + struct xfs_mount *mp) +{ + struct xfs_healthmon *hm; + + rcu_read_lock(); + hm = rcu_dereference(mp->m_healthmon); + if (hm && !refcount_inc_not_zero(&hm->ref)) + hm = NULL; + rcu_read_unlock(); + + return hm; +} + +/* + * Release the reference to a healthmon object. If there are no more holders, + * free the health monitor after an RCU grace period to eliminate possibility + * of races with xfs_healthmon_get. + */ +static void +xfs_healthmon_put( + struct xfs_healthmon *hm) +{ + if (refcount_dec_and_test(&hm->ref)) { + struct xfs_healthmon_event *event; + struct xfs_healthmon_event *next = hm->first_event; + + while ((event = next) != NULL) { + trace_xfs_healthmon_drop(hm, event); + next = event->next; + kfree(event); + } + + kfree(hm->unmount_event); + kfree(hm->buffer); + mutex_destroy(&hm->lock); + kfree_rcu_mightsleep(hm); + } +} + +/* Attach a health monitor to an xfs_mount. Only one allowed at a time. */ +STATIC int +xfs_healthmon_attach( + struct xfs_mount *mp, + struct xfs_healthmon *hm) +{ + spin_lock(&xfs_healthmon_lock); + if (rcu_access_pointer(mp->m_healthmon) != NULL) { + spin_unlock(&xfs_healthmon_lock); + return -EEXIST; + } + + refcount_inc(&hm->ref); + rcu_assign_pointer(mp->m_healthmon, hm); + hm->mount_cookie = (uintptr_t)mp->m_super; + spin_unlock(&xfs_healthmon_lock); + + return 0; +} + +/* Detach a xfs mount from a specific healthmon instance. */ +STATIC void +xfs_healthmon_detach( + struct xfs_healthmon *hm) +{ + struct xfs_mount *mp; + + spin_lock(&xfs_healthmon_lock); + if (hm->mount_cookie == DETACHED_MOUNT_COOKIE) { + spin_unlock(&xfs_healthmon_lock); + return; + } + + mp = XFS_M((struct super_block *)hm->mount_cookie); + rcu_assign_pointer(mp->m_healthmon, NULL); + hm->mount_cookie = DETACHED_MOUNT_COOKIE; + spin_unlock(&xfs_healthmon_lock); + + /* + * Wake up any readers that might remain. This can happen if unmount + * races with the healthmon fd owner entering ->read_iter, having + * already emptied the event queue. + * + * In the ->release case there shouldn't be any readers because the + * only users of the waiter are read and poll. + */ + wake_up_all(&hm->wait); + + trace_xfs_healthmon_detach(hm); + xfs_healthmon_put(hm); +} + +static inline void xfs_healthmon_bump_events(struct xfs_healthmon *hm) +{ + hm->events++; + hm->total_events++; +} + +static inline void xfs_healthmon_bump_lost(struct xfs_healthmon *hm) +{ + hm->lost_prev_event++; + hm->total_lost++; +} + +/* + * If possible, merge a new event into an existing event. Returns whether or + * not it merged anything. + */ +static bool +xfs_healthmon_merge_events( + struct xfs_healthmon_event *existing, + const struct xfs_healthmon_event *new) +{ + if (!existing) + return false; + + /* type and domain must match to merge events */ + if (existing->type != new->type || + existing->domain != new->domain) + return false; + + switch (existing->type) { + case XFS_HEALTHMON_RUNNING: + case XFS_HEALTHMON_UNMOUNT: + /* should only ever be one of these events anyway */ + return false; + + case XFS_HEALTHMON_LOST: + existing->lostcount += new->lostcount; + return true; + + case XFS_HEALTHMON_SICK: + case XFS_HEALTHMON_CORRUPT: + case XFS_HEALTHMON_HEALTHY: + switch (existing->domain) { + case XFS_HEALTHMON_FS: + existing->fsmask |= new->fsmask; + return true; + case XFS_HEALTHMON_AG: + case XFS_HEALTHMON_RTGROUP: + if (existing->group == new->group){ + existing->grpmask |= new->grpmask; + return true; + } + return false; + case XFS_HEALTHMON_INODE: + if (existing->ino == new->ino && + existing->gen == new->gen) { + existing->imask |= new->imask; + return true; + } + return false; + default: + ASSERT(0); + return false; + } + return false; + + case XFS_HEALTHMON_SHUTDOWN: + /* yes, we can race to shutdown */ + existing->flags |= new->flags; + return true; + + case XFS_HEALTHMON_MEDIA_ERROR: + /* physically adjacent errors can merge */ + if (existing->daddr + existing->bbcount == new->daddr) { + existing->bbcount += new->bbcount; + return true; + } + if (new->daddr + new->bbcount == existing->daddr) { + existing->daddr = new->daddr; + existing->bbcount += new->bbcount; + return true; + } + return false; + + case XFS_HEALTHMON_BUFREAD: + case XFS_HEALTHMON_BUFWRITE: + case XFS_HEALTHMON_DIOREAD: + case XFS_HEALTHMON_DIOWRITE: + case XFS_HEALTHMON_DATALOST: + /* logically adjacent file ranges can merge */ + if (existing->fino != new->fino || existing->fgen != new->fgen) + return false; + + if (existing->fpos + existing->flen == new->fpos) { + existing->flen += new->flen; + return true; + } + + if (new->fpos + new->flen == existing->fpos) { + existing->fpos = new->fpos; + existing->flen += new->flen; + return true; + } + return false; + } + + return false; +} + +/* Insert an event onto the start of the queue. */ +static inline void +__xfs_healthmon_insert( + struct xfs_healthmon *hm, + struct xfs_healthmon_event *event) +{ + struct timespec64 now; + + ktime_get_coarse_real_ts64(&now); + event->time_ns = (now.tv_sec * NSEC_PER_SEC) + now.tv_nsec; + + event->next = hm->first_event; + if (!hm->first_event) + hm->first_event = event; + if (!hm->last_event) + hm->last_event = event; + xfs_healthmon_bump_events(hm); + wake_up(&hm->wait); + + trace_xfs_healthmon_insert(hm, event); +} + +/* Push an event onto the end of the queue. */ +static inline void +__xfs_healthmon_push( + struct xfs_healthmon *hm, + struct xfs_healthmon_event *event) +{ + struct timespec64 now; + + ktime_get_coarse_real_ts64(&now); + event->time_ns = (now.tv_sec * NSEC_PER_SEC) + now.tv_nsec; + + if (!hm->first_event) + hm->first_event = event; + if (hm->last_event) + hm->last_event->next = event; + hm->last_event = event; + event->next = NULL; + xfs_healthmon_bump_events(hm); + wake_up(&hm->wait); + + trace_xfs_healthmon_push(hm, event); +} + +/* Deal with any previously lost events */ +static int +xfs_healthmon_clear_lost_prev( + struct xfs_healthmon *hm) +{ + struct xfs_healthmon_event lost_event = { + .type = XFS_HEALTHMON_LOST, + .domain = XFS_HEALTHMON_MOUNT, + .lostcount = hm->lost_prev_event, + }; + struct xfs_healthmon_event *event = NULL; + + if (xfs_healthmon_merge_events(hm->last_event, &lost_event)) { + trace_xfs_healthmon_merge(hm, hm->last_event); + wake_up(&hm->wait); + goto cleared; + } + + if (hm->events < XFS_HEALTHMON_MAX_EVENTS) + event = kmemdup(&lost_event, sizeof(struct xfs_healthmon_event), + GFP_NOFS); + if (!event) + return -ENOMEM; + + __xfs_healthmon_push(hm, event); +cleared: + hm->lost_prev_event = 0; + return 0; +} + +/* + * Push an event onto the end of the list after dealing with lost events and + * possibly full queues. + */ +STATIC int +xfs_healthmon_push( + struct xfs_healthmon *hm, + const struct xfs_healthmon_event *template) +{ + struct xfs_healthmon_event *event = NULL; + int error = 0; + + /* + * Locklessly check if the health monitor has already detached from the + * mount. If so, ignore the event. If we race with deactivation, + * we'll queue the event but never send it. + */ + if (hm->mount_cookie == DETACHED_MOUNT_COOKIE) + return -ESHUTDOWN; + + mutex_lock(&hm->lock); + + /* Report previously lost events before we do anything else */ + if (hm->lost_prev_event) { + error = xfs_healthmon_clear_lost_prev(hm); + if (error) + goto out_unlock; + } + + /* Try to merge with the newest event */ + if (xfs_healthmon_merge_events(hm->last_event, template)) { + trace_xfs_healthmon_merge(hm, hm->last_event); + wake_up(&hm->wait); + goto out_unlock; + } + + /* Only create a heap event object if we're not already at capacity. */ + if (hm->events < XFS_HEALTHMON_MAX_EVENTS) + event = kmemdup(template, sizeof(struct xfs_healthmon_event), + GFP_NOFS); + if (!event) { + /* No memory means we lose the event */ + trace_xfs_healthmon_lost_event(hm); + xfs_healthmon_bump_lost(hm); + error = -ENOMEM; + goto out_unlock; + } + + __xfs_healthmon_push(hm, event); + +out_unlock: + mutex_unlock(&hm->lock); + return error; +} + +/* + * Report that the filesystem is being unmounted, then detach the xfs mount + * from this healthmon instance. + */ +void +xfs_healthmon_unmount( + struct xfs_mount *mp) +{ + struct xfs_healthmon *hm = xfs_healthmon_get(mp); + + if (!hm) + return; + + trace_xfs_healthmon_report_unmount(hm); + + /* + * Insert the unmount notification at the start of the event queue so + * that userspace knows the filesystem went away as soon as possible. + * There's nothing actionable for userspace after an unmount. Once + * we've inserted the unmount event, hm no longer owns that event. + */ + __xfs_healthmon_insert(hm, hm->unmount_event); + hm->unmount_event = NULL; + + xfs_healthmon_detach(hm); + xfs_healthmon_put(hm); +} + +/* Compute the reporting mask for non-unmount metadata health events. */ +static inline unsigned int +metadata_event_mask( + struct xfs_healthmon *hm, + enum xfs_healthmon_type type, + unsigned int old_mask, + unsigned int new_mask) +{ + /* If we want all events, return all events. */ + if (hm->verbose) + return new_mask; + + switch (type) { + case XFS_HEALTHMON_SICK: + /* Always report runtime corruptions */ + return new_mask; + case XFS_HEALTHMON_CORRUPT: + /* Only report new fsck errors */ + return new_mask & ~old_mask; + case XFS_HEALTHMON_HEALTHY: + /* Only report healthy metadata that got fixed */ + return new_mask & old_mask; + default: + ASSERT(0); + break; + } + + return 0; +} + +/* Report XFS_FS_SICK_* events to healthmon */ +void +xfs_healthmon_report_fs( + struct xfs_mount *mp, + enum xfs_healthmon_type type, + unsigned int old_mask, + unsigned int new_mask) +{ + struct xfs_healthmon_event event = { + .type = type, + .domain = XFS_HEALTHMON_FS, + }; + struct xfs_healthmon *hm = xfs_healthmon_get(mp); + + if (!hm) + return; + + event.fsmask = metadata_event_mask(hm, type, old_mask, new_mask) & + ~XFS_SICK_FS_SECONDARY; + trace_xfs_healthmon_report_fs(hm, old_mask, new_mask, &event); + + if (event.fsmask) + xfs_healthmon_push(hm, &event); + + xfs_healthmon_put(hm); +} + +/* Report XFS_SICK_(AG|RG)* flags to healthmon */ +void +xfs_healthmon_report_group( + struct xfs_group *xg, + enum xfs_healthmon_type type, + unsigned int old_mask, + unsigned int new_mask) +{ + struct xfs_healthmon_event event = { + .type = type, + .group = xg->xg_gno, + }; + struct xfs_healthmon *hm = xfs_healthmon_get(xg->xg_mount); + + if (!hm) + return; + + switch (xg->xg_type) { + case XG_TYPE_RTG: + event.domain = XFS_HEALTHMON_RTGROUP; + event.grpmask = metadata_event_mask(hm, type, old_mask, + new_mask) & + ~XFS_SICK_RG_SECONDARY; + break; + case XG_TYPE_AG: + event.domain = XFS_HEALTHMON_AG; + event.grpmask = metadata_event_mask(hm, type, old_mask, + new_mask) & + ~XFS_SICK_AG_SECONDARY; + break; + default: + ASSERT(0); + break; + } + + trace_xfs_healthmon_report_group(hm, old_mask, new_mask, &event); + + if (event.grpmask) + xfs_healthmon_push(hm, &event); + + xfs_healthmon_put(hm); +} + +/* Report XFS_SICK_INO_* flags to healthmon */ +void +xfs_healthmon_report_inode( + struct xfs_inode *ip, + enum xfs_healthmon_type type, + unsigned int old_mask, + unsigned int new_mask) +{ + struct xfs_healthmon_event event = { + .type = type, + .domain = XFS_HEALTHMON_INODE, + .ino = I_INO(ip), + .gen = VFS_I(ip)->i_generation, + }; + struct xfs_healthmon *hm = xfs_healthmon_get(ip->i_mount); + + if (!hm) + return; + + event.imask = metadata_event_mask(hm, type, old_mask, new_mask) & + ~XFS_SICK_INO_SECONDARY; + trace_xfs_healthmon_report_inode(hm, old_mask, event.imask, &event); + + if (event.imask) + xfs_healthmon_push(hm, &event); + + xfs_healthmon_put(hm); +} + +/* Add a shutdown event to the reporting queue. */ +void +xfs_healthmon_report_shutdown( + struct xfs_mount *mp, + uint32_t flags) +{ + struct xfs_healthmon_event event = { + .type = XFS_HEALTHMON_SHUTDOWN, + .domain = XFS_HEALTHMON_MOUNT, + .flags = flags, + }; + struct xfs_healthmon *hm = xfs_healthmon_get(mp); + + if (!hm) + return; + + trace_xfs_healthmon_report_shutdown(hm, flags); + + xfs_healthmon_push(hm, &event); + xfs_healthmon_put(hm); +} + +static inline enum xfs_healthmon_domain +media_error_domain( + enum xfs_device fdev) +{ + switch (fdev) { + case XFS_DEV_DATA: + return XFS_HEALTHMON_DATADEV; + case XFS_DEV_LOG: + return XFS_HEALTHMON_LOGDEV; + case XFS_DEV_RT: + return XFS_HEALTHMON_RTDEV; + } + + ASSERT(0); + return 0; +} + +/* Add a media error event to the reporting queue. */ +void +xfs_healthmon_report_media( + struct xfs_mount *mp, + enum xfs_device fdev, + xfs_daddr_t daddr, + uint64_t bbcount) +{ + struct xfs_healthmon_event event = { + .type = XFS_HEALTHMON_MEDIA_ERROR, + .domain = media_error_domain(fdev), + .daddr = daddr, + .bbcount = bbcount, + }; + struct xfs_healthmon *hm = xfs_healthmon_get(mp); + + if (!hm) + return; + + trace_xfs_healthmon_report_media(hm, fdev, &event); + + xfs_healthmon_push(hm, &event); + xfs_healthmon_put(hm); +} + +static inline enum xfs_healthmon_type file_ioerr_type(enum fserror_type action) +{ + switch (action) { + case FSERR_BUFFERED_READ: + return XFS_HEALTHMON_BUFREAD; + case FSERR_BUFFERED_WRITE: + return XFS_HEALTHMON_BUFWRITE; + case FSERR_DIRECTIO_READ: + return XFS_HEALTHMON_DIOREAD; + case FSERR_DIRECTIO_WRITE: + return XFS_HEALTHMON_DIOWRITE; + case FSERR_DATA_LOST: + return XFS_HEALTHMON_DATALOST; + case FSERR_METADATA: + /* filtered out by xfs_fs_report_error */ + break; + } + + ASSERT(0); + return -1; +} + +/* Add a file io error event to the reporting queue. */ +void +xfs_healthmon_report_file_ioerror( + struct xfs_inode *ip, + const struct fserror_event *p) +{ + struct xfs_healthmon_event event = { + .type = file_ioerr_type(p->type), + .domain = XFS_HEALTHMON_FILERANGE, + .fino = I_INO(ip), + .fgen = VFS_I(ip)->i_generation, + .fpos = p->pos, + .flen = p->len, + /* send positive error number to userspace */ + .error = -p->error, + }; + struct xfs_healthmon *hm = xfs_healthmon_get(ip->i_mount); + + if (!hm) + return; + + trace_xfs_healthmon_report_file_ioerror(hm, p); + + xfs_healthmon_push(hm, &event); + xfs_healthmon_put(hm); +} + +static inline void +xfs_healthmon_reset_outbuf( + struct xfs_healthmon *hm) +{ + hm->buftail = 0; + hm->bufhead = 0; +} + +struct flags_map { + unsigned int in_mask; + unsigned int out_mask; +}; + +static const struct flags_map shutdown_map[] = { + { SHUTDOWN_META_IO_ERROR, XFS_HEALTH_SHUTDOWN_META_IO_ERROR }, + { SHUTDOWN_LOG_IO_ERROR, XFS_HEALTH_SHUTDOWN_LOG_IO_ERROR }, + { SHUTDOWN_FORCE_UMOUNT, XFS_HEALTH_SHUTDOWN_FORCE_UMOUNT }, + { SHUTDOWN_CORRUPT_INCORE, XFS_HEALTH_SHUTDOWN_CORRUPT_INCORE }, + { SHUTDOWN_CORRUPT_ONDISK, XFS_HEALTH_SHUTDOWN_CORRUPT_ONDISK }, + { SHUTDOWN_DEVICE_REMOVED, XFS_HEALTH_SHUTDOWN_DEVICE_REMOVED }, +}; + +static inline unsigned int +__map_flags( + const struct flags_map *map, + size_t array_len, + unsigned int flags) +{ + const struct flags_map *m; + unsigned int ret = 0; + + for (m = map; m < map + array_len; m++) { + if (flags & m->in_mask) + ret |= m->out_mask; + } + + return ret; +} + +#define map_flags(map, flags) __map_flags((map), ARRAY_SIZE(map), (flags)) + +static inline unsigned int shutdown_mask(unsigned int in) +{ + return map_flags(shutdown_map, in); +} + +static const unsigned int domain_map[] = { + [XFS_HEALTHMON_MOUNT] = XFS_HEALTH_MONITOR_DOMAIN_MOUNT, + [XFS_HEALTHMON_FS] = XFS_HEALTH_MONITOR_DOMAIN_FS, + [XFS_HEALTHMON_AG] = XFS_HEALTH_MONITOR_DOMAIN_AG, + [XFS_HEALTHMON_INODE] = XFS_HEALTH_MONITOR_DOMAIN_INODE, + [XFS_HEALTHMON_RTGROUP] = XFS_HEALTH_MONITOR_DOMAIN_RTGROUP, + [XFS_HEALTHMON_DATADEV] = XFS_HEALTH_MONITOR_DOMAIN_DATADEV, + [XFS_HEALTHMON_RTDEV] = XFS_HEALTH_MONITOR_DOMAIN_RTDEV, + [XFS_HEALTHMON_LOGDEV] = XFS_HEALTH_MONITOR_DOMAIN_LOGDEV, + [XFS_HEALTHMON_FILERANGE] = XFS_HEALTH_MONITOR_DOMAIN_FILERANGE, +}; + +static const unsigned int type_map[] = { + [XFS_HEALTHMON_RUNNING] = XFS_HEALTH_MONITOR_TYPE_RUNNING, + [XFS_HEALTHMON_LOST] = XFS_HEALTH_MONITOR_TYPE_LOST, + [XFS_HEALTHMON_SICK] = XFS_HEALTH_MONITOR_TYPE_SICK, + [XFS_HEALTHMON_CORRUPT] = XFS_HEALTH_MONITOR_TYPE_CORRUPT, + [XFS_HEALTHMON_HEALTHY] = XFS_HEALTH_MONITOR_TYPE_HEALTHY, + [XFS_HEALTHMON_UNMOUNT] = XFS_HEALTH_MONITOR_TYPE_UNMOUNT, + [XFS_HEALTHMON_SHUTDOWN] = XFS_HEALTH_MONITOR_TYPE_SHUTDOWN, + [XFS_HEALTHMON_MEDIA_ERROR] = XFS_HEALTH_MONITOR_TYPE_MEDIA_ERROR, + [XFS_HEALTHMON_BUFREAD] = XFS_HEALTH_MONITOR_TYPE_BUFREAD, + [XFS_HEALTHMON_BUFWRITE] = XFS_HEALTH_MONITOR_TYPE_BUFWRITE, + [XFS_HEALTHMON_DIOREAD] = XFS_HEALTH_MONITOR_TYPE_DIOREAD, + [XFS_HEALTHMON_DIOWRITE] = XFS_HEALTH_MONITOR_TYPE_DIOWRITE, + [XFS_HEALTHMON_DATALOST] = XFS_HEALTH_MONITOR_TYPE_DATALOST, +}; + +/* Render event as a V0 structure */ +STATIC int +xfs_healthmon_format_v0( + struct xfs_healthmon *hm, + const struct xfs_healthmon_event *event) +{ + struct xfs_health_monitor_event hme = { + .time_ns = event->time_ns, + }; + + trace_xfs_healthmon_format(hm, event); + + if (event->domain < 0 || event->domain >= ARRAY_SIZE(domain_map) || + event->type < 0 || event->type >= ARRAY_SIZE(type_map)) + return -EFSCORRUPTED; + + hme.domain = domain_map[event->domain]; + hme.type = type_map[event->type]; + + /* fill in the event-specific details */ + switch (event->domain) { + case XFS_HEALTHMON_MOUNT: + switch (event->type) { + case XFS_HEALTHMON_LOST: + hme.e.lost.count = event->lostcount; + break; + case XFS_HEALTHMON_SHUTDOWN: + hme.e.shutdown.reasons = shutdown_mask(event->flags); + break; + default: + break; + } + break; + case XFS_HEALTHMON_FS: + hme.e.fs.mask = xfs_healthmon_fs_mask(event->fsmask); + break; + case XFS_HEALTHMON_RTGROUP: + hme.e.group.mask = xfs_healthmon_rtgroup_mask(event->grpmask); + hme.e.group.gno = event->group; + break; + case XFS_HEALTHMON_AG: + hme.e.group.mask = xfs_healthmon_perag_mask(event->grpmask); + hme.e.group.gno = event->group; + break; + case XFS_HEALTHMON_INODE: + hme.e.inode.mask = xfs_healthmon_inode_mask(event->imask); + hme.e.inode.ino = event->ino; + hme.e.inode.gen = event->gen; + break; + case XFS_HEALTHMON_DATADEV: + case XFS_HEALTHMON_LOGDEV: + case XFS_HEALTHMON_RTDEV: + hme.e.media.daddr = event->daddr; + hme.e.media.bbcount = event->bbcount; + break; + case XFS_HEALTHMON_FILERANGE: + hme.e.filerange.ino = event->fino; + hme.e.filerange.gen = event->fgen; + hme.e.filerange.pos = event->fpos; + hme.e.filerange.len = event->flen; + hme.e.filerange.error = abs(event->error); + break; + default: + break; + } + + ASSERT(hm->bufhead + sizeof(hme) <= hm->bufsize); + + /* copy formatted object to the outbuf */ + if (hm->bufhead + sizeof(hme) <= hm->bufsize) { + memcpy(hm->buffer + hm->bufhead, &hme, sizeof(hme)); + hm->bufhead += sizeof(hme); + } + + return 0; +} + +/* How many bytes are waiting in the outbuf to be copied? */ +static inline size_t +xfs_healthmon_outbuf_bytes( + struct xfs_healthmon *hm) +{ + if (hm->bufhead > hm->buftail) + return hm->bufhead - hm->buftail; + return 0; +} + +/* + * Do we have something for userspace to read? This can mean unmount events, + * events pending in the queue, or pending bytes in the outbuf. + */ +static inline bool +xfs_healthmon_has_eventdata( + struct xfs_healthmon *hm) +{ + /* + * If the health monitor is already detached from the xfs_mount, we + * want reads to return 0 bytes even if there are no events, because + * userspace interprets that as EOF. If we race with deactivation, + * read_iter will take the necessary locks to discover that there are + * no events to send. + */ + if (hm->mount_cookie == DETACHED_MOUNT_COOKIE) + return true; + + /* + * Either there are events waiting to be formatted into the buffer, or + * there's unread bytes in the buffer. + */ + return hm->events > 0 || xfs_healthmon_outbuf_bytes(hm) > 0; +} + +/* Try to copy the rest of the outbuf to the iov iter. */ +STATIC ssize_t +xfs_healthmon_copybuf( + struct xfs_healthmon *hm, + struct iov_iter *to) +{ + size_t to_copy; + size_t w = 0; + + trace_xfs_healthmon_copybuf(hm, to); + + to_copy = xfs_healthmon_outbuf_bytes(hm); + if (to_copy) { + w = copy_to_iter(hm->buffer + hm->buftail, to_copy, to); + if (!w) + return -EFAULT; + + hm->buftail += w; + } + + /* + * Nothing left to copy? Reset the output buffer cursors to the start + * since there's no live data in the buffer. + */ + if (xfs_healthmon_outbuf_bytes(hm) == 0) + xfs_healthmon_reset_outbuf(hm); + return w; +} + +/* + * Return a health monitoring event for formatting into the output buffer if + * there's enough space in the outbuf and an event waiting for us. Caller + * must hold i_rwsem on the healthmon file. + */ +static inline struct xfs_healthmon_event * +xfs_healthmon_format_pop( + struct xfs_healthmon *hm) +{ + struct xfs_healthmon_event *event; + + if (hm->bufhead + sizeof(*event) > hm->bufsize) + return NULL; + + mutex_lock(&hm->lock); + event = hm->first_event; + if (event) { + if (hm->last_event == event) + hm->last_event = NULL; + hm->first_event = event->next; + hm->events--; + + trace_xfs_healthmon_pop(hm, event); + } + mutex_unlock(&hm->lock); + return event; +} + +/* Allocate formatting buffer */ +STATIC int +xfs_healthmon_alloc_outbuf( + struct xfs_healthmon *hm, + size_t user_bufsize) +{ + void *outbuf; + size_t bufsize = + min(XFS_HEALTHMON_MAX_OUTBUF, max(PAGE_SIZE, user_bufsize)); + + outbuf = kzalloc(bufsize, GFP_KERNEL); + if (!outbuf) { + if (bufsize == PAGE_SIZE) + return -ENOMEM; + + bufsize = PAGE_SIZE; + outbuf = kzalloc(bufsize, GFP_KERNEL); + if (!outbuf) + return -ENOMEM; + } + + hm->buffer = outbuf; + hm->bufsize = bufsize; + hm->bufhead = 0; + hm->buftail = 0; + + return 0; +} + +/* + * Convey queued event data to userspace. First copy any remaining bytes in + * the outbuf, then format the oldest event into the outbuf and copy that too. + */ +STATIC ssize_t +xfs_healthmon_read_iter( + struct kiocb *iocb, + struct iov_iter *to) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct xfs_healthmon *hm = file->private_data; + struct xfs_healthmon_event *event; + size_t copied = 0; + ssize_t ret = 0; + + if (file->f_flags & O_NONBLOCK) { + if (!xfs_healthmon_has_eventdata(hm) || !inode_trylock(inode)) + return -EAGAIN; + } else { + ret = wait_event_interruptible(hm->wait, + xfs_healthmon_has_eventdata(hm)); + if (ret) + return ret; + + inode_lock(inode); + } + + if (hm->bufsize == 0) { + ret = xfs_healthmon_alloc_outbuf(hm, iov_iter_count(to)); + if (ret) + goto out_unlock; + } + + trace_xfs_healthmon_read_start(hm); + + /* + * If there's anything left in the output buffer, copy that before + * formatting more events. + */ + ret = xfs_healthmon_copybuf(hm, to); + if (ret < 0) + goto out_unlock; + copied += ret; + + while (iov_iter_count(to) > 0) { + /* Format the next events into the outbuf until it's full. */ + while ((event = xfs_healthmon_format_pop(hm)) != NULL) { + ret = xfs_healthmon_format_v0(hm, event); + kfree(event); + if (ret) + goto out_unlock; + } + + /* Copy anything formatted into outbuf to userspace */ + ret = xfs_healthmon_copybuf(hm, to); + if (ret <= 0) + break; + + copied += ret; + } + +out_unlock: + trace_xfs_healthmon_read_finish(hm); + inode_unlock(inode); + return copied ?: ret; +} + +/* Poll for available events. */ +STATIC __poll_t +xfs_healthmon_poll( + struct file *file, + struct poll_table_struct *wait) +{ + struct xfs_healthmon *hm = file->private_data; + __poll_t mask = 0; + + poll_wait(file, &hm->wait, wait); + + if (xfs_healthmon_has_eventdata(hm)) + mask |= EPOLLIN; + return mask; +} + +/* Free the health monitoring information. */ +STATIC int +xfs_healthmon_release( + struct inode *inode, + struct file *file) +{ + struct xfs_healthmon *hm = file->private_data; + + trace_xfs_healthmon_release(hm); + + /* + * We might be closing the healthmon file before the filesystem + * unmounts, because userspace processes can terminate at any time and + * for any reason. Null out xfs_mount::m_healthmon so that another + * process can create another health monitor file. + */ + xfs_healthmon_detach(hm); + xfs_healthmon_put(hm); + return 0; +} + +/* Validate ioctl parameters. */ +static inline bool +xfs_healthmon_validate( + const struct xfs_health_monitor *hmo) +{ + if (hmo->flags & ~XFS_HEALTH_MONITOR_ALL) + return false; + if (hmo->format != XFS_HEALTH_MONITOR_FMT_V0) + return false; + if (memchr_inv(&hmo->pad, 0, sizeof(hmo->pad))) + return false; + return true; +} + +/* Emit some data about the health monitoring fd. */ +static void +xfs_healthmon_show_fdinfo( + struct seq_file *m, + struct file *file) +{ + struct xfs_healthmon *hm = file->private_data; + + mutex_lock(&hm->lock); + seq_printf(m, "state:\t%s\ndev:\t%d:%d\nformat:\tv0\nevents:\t%llu\nlost:\t%llu\n", + hm->mount_cookie == DETACHED_MOUNT_COOKIE ? + "dead" : "alive", + MAJOR(hm->dev), MINOR(hm->dev), + hm->total_events, + hm->total_lost); + mutex_unlock(&hm->lock); +} + +/* Reconfigure the health monitor. */ +STATIC long +xfs_healthmon_reconfigure( + struct file *file, + unsigned int cmd, + void __user *arg) +{ + struct xfs_health_monitor hmo; + struct xfs_healthmon *hm = file->private_data; + + if (copy_from_user(&hmo, arg, sizeof(hmo))) + return -EFAULT; + + if (!xfs_healthmon_validate(&hmo)) + return -EINVAL; + + mutex_lock(&hm->lock); + hm->verbose = !!(hmo.flags & XFS_HEALTH_MONITOR_VERBOSE); + mutex_unlock(&hm->lock); + + return 0; +} + +/* Does the fd point to the same filesystem as the one we're monitoring? */ +STATIC long +xfs_healthmon_file_on_monitored_fs( + struct file *file, + unsigned int cmd, + void __user *arg) +{ + struct xfs_health_file_on_monitored_fs hms; + struct xfs_healthmon *hm = file->private_data; + struct inode *hms_inode; + + if (copy_from_user(&hms, arg, sizeof(hms))) + return -EFAULT; + + if (hms.flags) + return -EINVAL; + + CLASS(fd, hms_fd)(hms.fd); + if (fd_empty(hms_fd)) + return -EBADF; + + hms_inode = file_inode(fd_file(hms_fd)); + mutex_lock(&hm->lock); + if (hm->mount_cookie != (uintptr_t)hms_inode->i_sb) { + mutex_unlock(&hm->lock); + return -ESTALE; + } + + mutex_unlock(&hm->lock); + return 0; +} + +/* Handle ioctls for the health monitoring thread. */ +STATIC long +xfs_healthmon_ioctl( + struct file *file, + unsigned int cmd, + unsigned long p) +{ + void __user *arg = (void __user *)p; + + switch (cmd) { + case XFS_IOC_HEALTH_MONITOR: + return xfs_healthmon_reconfigure(file, cmd, arg); + case XFS_IOC_HEALTH_FD_ON_MONITORED_FS: + return xfs_healthmon_file_on_monitored_fs(file, cmd, arg); + default: + break; + } + + return -ENOTTY; +} + +static const struct file_operations xfs_healthmon_fops = { + .owner = THIS_MODULE, + .show_fdinfo = xfs_healthmon_show_fdinfo, + .read_iter = xfs_healthmon_read_iter, + .poll = xfs_healthmon_poll, + .release = xfs_healthmon_release, + .unlocked_ioctl = xfs_healthmon_ioctl, +}; + +/* + * Create a health monitoring file. Returns an index to the fd table or a + * negative errno. + */ +long +xfs_ioc_health_monitor( + struct file *file, + struct xfs_health_monitor __user *arg) +{ + struct xfs_health_monitor hmo; + struct xfs_healthmon_event *running_event; + struct xfs_healthmon *hm; + struct xfs_inode *ip = XFS_I(file_inode(file)); + struct xfs_mount *mp = ip->i_mount; + int ret; + + /* + * The only intended user of the health monitoring system should be the + * xfs_healer daemon running on behalf of the whole filesystem in the + * initial user namespace. IOWs, we don't allow unprivileged userspace + * (they can use fsnotify) nor do we allow containers. + */ + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (I_INO(ip) != mp->m_sb.sb_rootino) + return -EPERM; + if (current_user_ns() != &init_user_ns) + return -EPERM; + + if (copy_from_user(&hmo, arg, sizeof(hmo))) + return -EFAULT; + + if (!xfs_healthmon_validate(&hmo)) + return -EINVAL; + + hm = kzalloc_obj(*hm); + if (!hm) + return -ENOMEM; + hm->dev = mp->m_super->s_dev; + refcount_set(&hm->ref, 1); + + mutex_init(&hm->lock); + init_waitqueue_head(&hm->wait); + + if (hmo.flags & XFS_HEALTH_MONITOR_VERBOSE) + hm->verbose = true; + + /* Queue up the first event that lets the client know we're running. */ + running_event = kzalloc_obj(struct xfs_healthmon_event, GFP_NOFS); + if (!running_event) { + ret = -ENOMEM; + goto out_hm; + } + running_event->type = XFS_HEALTHMON_RUNNING; + running_event->domain = XFS_HEALTHMON_MOUNT; + __xfs_healthmon_insert(hm, running_event); + + /* + * Preallocate the unmount event so that we can't fail to notify the + * filesystem later. This is key for triggering fast exit of the + * xfs_healer daemon. + */ + hm->unmount_event = kzalloc_obj(struct xfs_healthmon_event, GFP_NOFS); + if (!hm->unmount_event) { + ret = -ENOMEM; + goto out_hm; + } + hm->unmount_event->type = XFS_HEALTHMON_UNMOUNT; + hm->unmount_event->domain = XFS_HEALTHMON_MOUNT; + + /* + * Try to attach this health monitor to the xfs_mount. The monitor is + * considered live and will receive events if this succeeds. + */ + ret = xfs_healthmon_attach(mp, hm); + if (ret) + goto out_hm; + + /* + * Create the anonymous file and install a fd for it. If it succeeds, + * the file owns hm and can go away at any time, so we must not access + * it again. This must go last because we can't undo a fd table + * installation. + */ + ret = anon_inode_getfd("xfs_healthmon", &xfs_healthmon_fops, hm, + O_CLOEXEC | O_RDONLY); + if (ret < 0) + goto out_mp; + + trace_xfs_healthmon_create(mp->m_super->s_dev, hmo.flags, hmo.format); + + return ret; + +out_mp: + xfs_healthmon_detach(hm); +out_hm: + ASSERT(refcount_read(&hm->ref) == 1); + xfs_healthmon_put(hm); + return ret; +} diff --git a/fs/xfs/xfs_healthmon.h b/fs/xfs/xfs_healthmon.h new file mode 100644 index 000000000..0e9365070 --- /dev/null +++ b/fs/xfs/xfs_healthmon.h @@ -0,0 +1,184 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2024-2026 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_HEALTHMON_H__ +#define __XFS_HEALTHMON_H__ + +struct xfs_healthmon { + /* + * Weak reference to the xfs filesystem that is being monitored. It + * will be set to zero when the filesystem detaches from the monitor. + * Do not dereference this pointer. + */ + uintptr_t mount_cookie; + + /* + * Device number of the filesystem being monitored. This is for + * consistent tracing even after unmount. + */ + dev_t dev; + + /* + * Reference count of this structure. The open healthmon fd holds one + * ref, the xfs_mount holds another ref if it points to this object, + * and running event handlers hold their own refs. + */ + refcount_t ref; + + /* lock for event list and event counters */ + struct mutex lock; + + /* list of event objects */ + struct xfs_healthmon_event *first_event; + struct xfs_healthmon_event *last_event; + + /* preallocated event for unmount */ + struct xfs_healthmon_event *unmount_event; + + /* number of events in the list */ + unsigned int events; + + /* do we want all events? */ + bool verbose:1; + + /* waiter so read/poll can sleep until the arrival of events */ + struct wait_queue_head wait; + + /* + * Buffer for formatting events for a read_iter call. Events are + * formatted into the buffer at bufhead, and buftail determines where + * to start a copy_iter to get those events to userspace. All buffer + * fields are protected by inode_lock. + */ + char *buffer; + size_t bufsize; + size_t bufhead; + size_t buftail; + + /* did we lose previous events? */ + unsigned long long lost_prev_event; + + /* total counts of events observed and lost events */ + unsigned long long total_events; + unsigned long long total_lost; +}; + +void xfs_healthmon_unmount(struct xfs_mount *mp); + +enum xfs_healthmon_type { + XFS_HEALTHMON_RUNNING, /* monitor running */ + XFS_HEALTHMON_LOST, /* message lost */ + XFS_HEALTHMON_UNMOUNT, /* filesystem is unmounting */ + + /* filesystem shutdown */ + XFS_HEALTHMON_SHUTDOWN, + + /* metadata health events */ + XFS_HEALTHMON_SICK, /* runtime corruption observed */ + XFS_HEALTHMON_CORRUPT, /* fsck reported corruption */ + XFS_HEALTHMON_HEALTHY, /* fsck reported healthy structure */ + + /* media errors */ + XFS_HEALTHMON_MEDIA_ERROR, + + /* file range events */ + XFS_HEALTHMON_BUFREAD, + XFS_HEALTHMON_BUFWRITE, + XFS_HEALTHMON_DIOREAD, + XFS_HEALTHMON_DIOWRITE, + XFS_HEALTHMON_DATALOST, +}; + +enum xfs_healthmon_domain { + XFS_HEALTHMON_MOUNT, /* affects the whole fs */ + + /* metadata health events */ + XFS_HEALTHMON_FS, /* main filesystem metadata */ + XFS_HEALTHMON_AG, /* allocation group metadata */ + XFS_HEALTHMON_INODE, /* inode metadata */ + XFS_HEALTHMON_RTGROUP, /* realtime group metadata */ + + /* media errors */ + XFS_HEALTHMON_DATADEV, + XFS_HEALTHMON_RTDEV, + XFS_HEALTHMON_LOGDEV, + + /* file range events */ + XFS_HEALTHMON_FILERANGE, +}; + +struct xfs_healthmon_event { + struct xfs_healthmon_event *next; + + enum xfs_healthmon_type type; + enum xfs_healthmon_domain domain; + + uint64_t time_ns; + + union { + /* lost events */ + struct { + uint64_t lostcount; + }; + /* fs/rt metadata */ + struct { + /* XFS_SICK_* flags */ + unsigned int fsmask; + }; + /* ag/rtgroup metadata */ + struct { + /* XFS_SICK_(AG|RG)* flags */ + unsigned int grpmask; + unsigned int group; + }; + /* inode metadata */ + struct { + /* XFS_SICK_INO_* flags */ + unsigned int imask; + uint32_t gen; + xfs_ino_t ino; + }; + /* shutdown */ + struct { + unsigned int flags; + }; + /* media errors */ + struct { + xfs_daddr_t daddr; + uint64_t bbcount; + }; + /* file range events */ + struct { + xfs_ino_t fino; + loff_t fpos; + uint64_t flen; + uint32_t fgen; + int error; + }; + }; +}; + +void xfs_healthmon_report_fs(struct xfs_mount *mp, + enum xfs_healthmon_type type, unsigned int old_mask, + unsigned int new_mask); +void xfs_healthmon_report_group(struct xfs_group *xg, + enum xfs_healthmon_type type, unsigned int old_mask, + unsigned int new_mask); +void xfs_healthmon_report_inode(struct xfs_inode *ip, + enum xfs_healthmon_type type, unsigned int old_mask, + unsigned int new_mask); + +void xfs_healthmon_report_shutdown(struct xfs_mount *mp, uint32_t flags); + +void xfs_healthmon_report_media(struct xfs_mount *mp, enum xfs_device fdev, + xfs_daddr_t daddr, uint64_t bbcount); + +void xfs_healthmon_report_file_ioerror(struct xfs_inode *ip, + const struct fserror_event *p); + +long xfs_ioc_health_monitor(struct file *file, + struct xfs_health_monitor __user *arg); + +#endif /* __XFS_HEALTHMON_H__ */ diff --git a/fs/xfs/xfs_hooks.c b/fs/xfs/xfs_hooks.c new file mode 100644 index 000000000..a09109e69 --- /dev/null +++ b/fs/xfs/xfs_hooks.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2022-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_ag.h" +#include "xfs_trace.h" + +/* Initialize a notifier chain. */ +void +xfs_hooks_init( + struct xfs_hooks *chain) +{ + BLOCKING_INIT_NOTIFIER_HEAD(&chain->head); +} + +/* Make it so a function gets called whenever we hit a certain hook point. */ +int +xfs_hooks_add( + struct xfs_hooks *chain, + struct xfs_hook *hook) +{ + ASSERT(hook->nb.notifier_call != NULL); + BUILD_BUG_ON(offsetof(struct xfs_hook, nb) != 0); + + return blocking_notifier_chain_register(&chain->head, &hook->nb); +} + +/* Remove a previously installed hook. */ +void +xfs_hooks_del( + struct xfs_hooks *chain, + struct xfs_hook *hook) +{ + blocking_notifier_chain_unregister(&chain->head, &hook->nb); +} + +/* Call a hook. Returns the NOTIFY_* value returned by the last hook. */ +int +xfs_hooks_call( + struct xfs_hooks *chain, + unsigned long val, + void *priv) +{ + return blocking_notifier_call_chain(&chain->head, val, priv); +} diff --git a/fs/xfs/xfs_hooks.h b/fs/xfs/xfs_hooks.h new file mode 100644 index 000000000..60b8a5831 --- /dev/null +++ b/fs/xfs/xfs_hooks.h @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2022-2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef XFS_HOOKS_H_ +#define XFS_HOOKS_H_ + +#ifdef CONFIG_XFS_LIVE_HOOKS +struct xfs_hooks { + struct blocking_notifier_head head; +}; + +/* + * If jump labels are enabled in Kconfig, the static key uses nop sleds and + * code patching to eliminate the overhead of taking the rwsem in + * blocking_notifier_call_chain when there are no hooks configured. If not, + * the static key per-call overhead is an atomic read. Most arches that can + * handle XFS also support jump labels. + * + * Note: Patching the kernel code requires taking the cpu hotplug lock. Other + * parts of the kernel allocate memory with that lock held, which means that + * XFS callers cannot hold any locks that might be used by memory reclaim or + * writeback when calling the static_branch_{inc,dec} functions. + */ +# define DEFINE_STATIC_XFS_HOOK_SWITCH(name) \ + static DEFINE_STATIC_KEY_FALSE(name) +# define xfs_hooks_switch_on(name) static_branch_inc(name) +# define xfs_hooks_switch_off(name) static_branch_dec(name) +# define xfs_hooks_switched_on(name) static_branch_unlikely(name) + +struct xfs_hook { + /* This must come at the start of the structure. */ + struct notifier_block nb; +}; + +typedef int (*xfs_hook_fn_t)(struct xfs_hook *hook, unsigned long action, + void *data); + +void xfs_hooks_init(struct xfs_hooks *chain); +int xfs_hooks_add(struct xfs_hooks *chain, struct xfs_hook *hook); +void xfs_hooks_del(struct xfs_hooks *chain, struct xfs_hook *hook); +int xfs_hooks_call(struct xfs_hooks *chain, unsigned long action, + void *priv); + +static inline void xfs_hook_setup(struct xfs_hook *hook, notifier_fn_t fn) +{ + hook->nb.notifier_call = fn; + hook->nb.priority = 0; +} + +#else + +struct xfs_hooks { /* empty */ }; + +# define DEFINE_STATIC_XFS_HOOK_SWITCH(name) +# define xfs_hooks_switch_on(name) ((void)0) +# define xfs_hooks_switch_off(name) ((void)0) +# define xfs_hooks_switched_on(name) (false) + +# define xfs_hooks_init(chain) ((void)0) +# define xfs_hooks_call(chain, val, priv) (NOTIFY_DONE) +#endif + +#endif /* XFS_HOOKS_H_ */ diff --git a/fs/xfs/xfs_icache.c b/fs/xfs/xfs_icache.c new file mode 100644 index 000000000..9d8dd30bd --- /dev/null +++ b/fs/xfs/xfs_icache.c @@ -0,0 +1,2356 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_trans_priv.h" +#include "xfs_inode_item.h" +#include "xfs_quota.h" +#include "xfs_trace.h" +#include "xfs_icache.h" +#include "xfs_bmap_util.h" +#include "xfs_dquot_item.h" +#include "xfs_dquot.h" +#include "xfs_reflink.h" +#include "xfs_ialloc.h" +#include "xfs_ag.h" +#include "xfs_log_priv.h" +#include "xfs_health.h" +#include "xfs_da_format.h" +#include "xfs_dir2.h" +#include "xfs_metafile.h" + +#include + +/* Radix tree tags for incore inode tree. */ + +/* inode is to be reclaimed */ +#define XFS_ICI_RECLAIM_TAG 0 +/* Inode has speculative preallocations (posteof or cow) to clean. */ +#define XFS_ICI_BLOCKGC_TAG 1 + +/* + * The goal for walking incore inodes. These can correspond with incore inode + * radix tree tags when convenient. Avoid existing XFS_IWALK namespace. + */ +enum xfs_icwalk_goal { + /* Goals directly associated with tagged inodes. */ + XFS_ICWALK_BLOCKGC = XFS_ICI_BLOCKGC_TAG, + XFS_ICWALK_RECLAIM = XFS_ICI_RECLAIM_TAG, +}; + +static int xfs_icwalk(struct xfs_mount *mp, + enum xfs_icwalk_goal goal, struct xfs_icwalk *icw); +static int xfs_icwalk_ag(struct xfs_perag *pag, + enum xfs_icwalk_goal goal, struct xfs_icwalk *icw); + +/* + * Private inode cache walk flags for struct xfs_icwalk. Must not + * coincide with XFS_ICWALK_FLAGS_VALID. + */ + +/* Stop scanning after icw_scan_limit inodes. */ +#define XFS_ICWALK_FLAG_SCAN_LIMIT (1U << 28) + +#define XFS_ICWALK_FLAG_RECLAIM_SICK (1U << 27) +#define XFS_ICWALK_FLAG_UNION (1U << 26) /* union filter algorithm */ + +#define XFS_ICWALK_PRIVATE_FLAGS (XFS_ICWALK_FLAG_SCAN_LIMIT | \ + XFS_ICWALK_FLAG_RECLAIM_SICK | \ + XFS_ICWALK_FLAG_UNION) + +/* Marks for the perag xarray */ +#define XFS_PERAG_RECLAIM_MARK XA_MARK_0 +#define XFS_PERAG_BLOCKGC_MARK XA_MARK_1 + +static inline xa_mark_t ici_tag_to_mark(unsigned int tag) +{ + if (tag == XFS_ICI_RECLAIM_TAG) + return XFS_PERAG_RECLAIM_MARK; + ASSERT(tag == XFS_ICI_BLOCKGC_TAG); + return XFS_PERAG_BLOCKGC_MARK; +} + +/* + * Allocate and initialise an xfs_inode. + */ +struct xfs_inode * +xfs_inode_alloc( + struct xfs_mount *mp, + xfs_ino_t ino) +{ + struct xfs_inode *ip; + + /* + * XXX: If this didn't occur in transactions, we could drop GFP_NOFAIL + * and return NULL here on ENOMEM. + */ + ip = alloc_inode_sb(mp->m_super, xfs_inode_cache, GFP_KERNEL | __GFP_NOFAIL); + + if (inode_init_always(mp->m_super, VFS_I(ip))) { + kmem_cache_free(xfs_inode_cache, ip); + return NULL; + } + + VFS_I(ip)->i_ino = ino; + /* VFS doesn't initialise i_mode! */ + VFS_I(ip)->i_mode = 0; + mapping_set_folio_min_order(VFS_I(ip)->i_mapping, + M_IGEO(mp)->min_folio_order); + + XFS_STATS_INC(mp, xs_inodes_active); + ASSERT(atomic_read(&ip->i_pincount) == 0); + + /* initialise the xfs inode */ + ip->i_mount = mp; + memset(&ip->i_imap, 0, sizeof(struct xfs_imap)); + ip->i_cowfp = NULL; + memset(&ip->i_af, 0, sizeof(ip->i_af)); + ip->i_af.if_format = XFS_DINODE_FMT_EXTENTS; + memset(&ip->i_df, 0, sizeof(ip->i_df)); + ip->i_flags = 0; + ip->i_delayed_blks = 0; + ip->i_diflags2 = mp->m_ino_geo.new_diflags2; + ip->i_nblocks = 0; + ip->i_forkoff = 0; + ip->i_sick = 0; + ip->i_checked = 0; + INIT_WORK(&ip->i_ioend_work, xfs_end_io); + INIT_LIST_HEAD(&ip->i_ioend_list); + spin_lock_init(&ip->i_ioend_lock); + ip->i_next_unlinked = NULLAGINO; + ip->i_prev_unlinked = 0; + + return ip; +} + +STATIC void +xfs_inode_free_callback( + struct rcu_head *head) +{ + struct inode *inode = container_of(head, struct inode, i_rcu); + struct xfs_inode *ip = XFS_I(inode); + + switch (VFS_I(ip)->i_mode & S_IFMT) { + case S_IFREG: + case S_IFDIR: + case S_IFLNK: + xfs_idestroy_fork(&ip->i_df); + break; + } + + xfs_ifork_zap_attr(ip); + + if (ip->i_cowfp) { + xfs_idestroy_fork(ip->i_cowfp); + kmem_cache_free(xfs_ifork_cache, ip->i_cowfp); + } + if (ip->i_itemp) { + ASSERT(!test_bit(XFS_LI_IN_AIL, + &ip->i_itemp->ili_item.li_flags)); + xfs_inode_item_destroy(ip); + } + + kmem_cache_free(xfs_inode_cache, ip); +} + +static void +__xfs_inode_free( + struct xfs_inode *ip) +{ + /* asserts to verify all state is correct here */ + ASSERT(atomic_read(&ip->i_pincount) == 0); + ASSERT(!ip->i_itemp || list_empty(&ip->i_itemp->ili_item.li_bio_list)); + if (xfs_is_metadir_inode(ip)) + XFS_STATS_DEC(ip->i_mount, xs_inodes_meta); + else + XFS_STATS_DEC(ip->i_mount, xs_inodes_active); + + call_rcu(&VFS_I(ip)->i_rcu, xfs_inode_free_callback); +} + +void +xfs_inode_free( + struct xfs_inode *ip) +{ + ASSERT(!xfs_iflags_test(ip, XFS_IFLUSHING)); + + /* + * Because we use RCU freeing we need to ensure the inode always + * appears to be reclaimed with an invalid inode number when in the + * free state. The ip->i_flags_lock provides the barrier against lookup + * races. + */ + spin_lock(&ip->i_flags_lock); + ip->i_flags = XFS_IRECLAIM; + VFS_I(ip)->i_ino = 0; + spin_unlock(&ip->i_flags_lock); + + __xfs_inode_free(ip); +} + +/* + * Queue background inode reclaim work if there are reclaimable inodes and there + * isn't reclaim work already scheduled or in progress. + */ +static void +xfs_reclaim_work_queue( + struct xfs_mount *mp) +{ + + rcu_read_lock(); + if (xfs_group_marked(mp, XG_TYPE_AG, XFS_PERAG_RECLAIM_MARK)) { + queue_delayed_work(mp->m_reclaim_workqueue, &mp->m_reclaim_work, + msecs_to_jiffies(xfs_syncd_centisecs / 6 * 10)); + } + rcu_read_unlock(); +} + +/* + * Background scanning to trim preallocated space. This is queued based on the + * 'speculative_prealloc_lifetime' tunable (5m by default). + */ +static inline void +xfs_blockgc_queue( + struct xfs_perag *pag) +{ + struct xfs_mount *mp = pag_mount(pag); + + if (!xfs_is_blockgc_enabled(mp)) + return; + + rcu_read_lock(); + if (radix_tree_tagged(&pag->pag_ici_root, XFS_ICI_BLOCKGC_TAG)) + queue_delayed_work(mp->m_blockgc_wq, &pag->pag_blockgc_work, + secs_to_jiffies(xfs_blockgc_secs)); + rcu_read_unlock(); +} + +/* Set a tag on both the AG incore inode tree and the AG radix tree. */ +static void +xfs_perag_set_inode_tag( + struct xfs_perag *pag, + xfs_agino_t agino, + unsigned int tag) +{ + bool was_tagged; + + lockdep_assert_held(&pag->pag_ici_lock); + + was_tagged = radix_tree_tagged(&pag->pag_ici_root, tag); + radix_tree_tag_set(&pag->pag_ici_root, agino, tag); + + if (tag == XFS_ICI_RECLAIM_TAG) + pag->pag_ici_reclaimable++; + + if (was_tagged) + return; + + /* propagate the tag up into the pag xarray tree */ + xfs_group_set_mark(pag_group(pag), ici_tag_to_mark(tag)); + + /* start background work */ + switch (tag) { + case XFS_ICI_RECLAIM_TAG: + xfs_reclaim_work_queue(pag_mount(pag)); + break; + case XFS_ICI_BLOCKGC_TAG: + xfs_blockgc_queue(pag); + break; + } + + trace_xfs_perag_set_inode_tag(pag, _RET_IP_); +} + +/* Clear a tag on both the AG incore inode tree and the AG radix tree. */ +static void +xfs_perag_clear_inode_tag( + struct xfs_perag *pag, + xfs_agino_t agino, + unsigned int tag) +{ + lockdep_assert_held(&pag->pag_ici_lock); + + /* + * Reclaim can signal (with a null agino) that it cleared its own tag + * by removing the inode from the radix tree. + */ + if (agino != NULLAGINO) + radix_tree_tag_clear(&pag->pag_ici_root, agino, tag); + else + ASSERT(tag == XFS_ICI_RECLAIM_TAG); + + if (tag == XFS_ICI_RECLAIM_TAG) + pag->pag_ici_reclaimable--; + + if (radix_tree_tagged(&pag->pag_ici_root, tag)) + return; + + /* clear the tag from the pag xarray */ + xfs_group_clear_mark(pag_group(pag), ici_tag_to_mark(tag)); + trace_xfs_perag_clear_inode_tag(pag, _RET_IP_); +} + +/* + * Find the next AG after @pag, or the first AG if @pag is NULL. + */ +static struct xfs_perag * +xfs_perag_grab_next_tag( + struct xfs_mount *mp, + struct xfs_perag *pag, + int tag) +{ + return to_perag(xfs_group_grab_next_mark(mp, + pag ? pag_group(pag) : NULL, + ici_tag_to_mark(tag), XG_TYPE_AG)); +} + +/* + * When we recycle a reclaimable inode, we need to re-initialise the VFS inode + * part of the structure. This is made more complex by the fact we store + * information about the on-disk values in the VFS inode and so we can't just + * overwrite the values unconditionally. Hence we save the parameters we + * need to retain across reinitialisation, and rewrite them into the VFS inode + * after reinitialisation even if it fails. + */ +static int +xfs_reinit_inode( + struct xfs_mount *mp, + struct inode *inode) +{ + int error; + u64 ino = inode->i_ino; + uint32_t nlink = inode->i_nlink; + uint32_t generation = inode->i_generation; + uint64_t version = inode_peek_iversion(inode); + umode_t mode = inode->i_mode; + dev_t dev = inode->i_rdev; + kuid_t uid = inode->i_uid; + kgid_t gid = inode->i_gid; + unsigned long state = inode_state_read_once(inode); + + error = inode_init_always(mp->m_super, inode); + + inode->i_ino = ino; + set_nlink(inode, nlink); + inode->i_generation = generation; + inode_set_iversion_queried(inode, version); + inode->i_mode = mode; + inode->i_rdev = dev; + inode->i_uid = uid; + inode->i_gid = gid; + inode_state_assign_raw(inode, state); + mapping_set_folio_min_order(inode->i_mapping, + M_IGEO(mp)->min_folio_order); + return error; +} + +/* + * Carefully nudge an inode whose VFS state has been torn down back into a + * usable state. Drops the i_flags_lock and the rcu read lock. + */ +static int +xfs_iget_recycle( + struct xfs_perag *pag, + struct xfs_inode *ip) +{ + struct xfs_mount *mp = ip->i_mount; + struct inode *inode = VFS_I(ip); + int error; + + trace_xfs_iget_recycle(ip); + + ASSERT(!rwsem_is_locked(&inode->i_rwsem)); + error = xfs_reinit_inode(mp, inode); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + if (error) { + /* + * Re-initializing the inode failed, and we are in deep + * trouble. Try to re-add it to the reclaim list. + */ + rcu_read_lock(); + spin_lock(&ip->i_flags_lock); + ip->i_flags &= ~(XFS_INEW | XFS_IRECLAIM); + ASSERT(ip->i_flags & XFS_IRECLAIMABLE); + spin_unlock(&ip->i_flags_lock); + rcu_read_unlock(); + + trace_xfs_iget_recycle_fail(ip); + return error; + } + + spin_lock(&pag->pag_ici_lock); + spin_lock(&ip->i_flags_lock); + + /* + * Clear the per-lifetime state in the inode as we are now effectively + * a new inode and need to return to the initial state before reuse + * occurs. + */ + ip->i_flags &= ~XFS_IRECLAIM_RESET_FLAGS; + ip->i_flags |= XFS_INEW; + xfs_perag_clear_inode_tag(pag, XFS_INODE_TO_AGINO(ip), + XFS_ICI_RECLAIM_TAG); + inode_state_assign_raw(inode, I_NEW); + spin_unlock(&ip->i_flags_lock); + spin_unlock(&pag->pag_ici_lock); + + return 0; +} + +/* + * If we are allocating a new inode, then check what was returned is + * actually a free, empty inode. If we are not allocating an inode, + * then check we didn't find a free inode. + * + * Returns: + * 0 if the inode free state matches the lookup context + * -ENOENT if the inode is free and we are not allocating + * -EFSCORRUPTED if there is any state mismatch at all + */ +static int +xfs_iget_check_free_state( + struct xfs_inode *ip, + int flags) +{ + if (flags & XFS_IGET_CREATE) { + /* should be a free inode */ + if (VFS_I(ip)->i_mode != 0) { + xfs_warn(ip->i_mount, +"Corruption detected! Free inode 0x%llx not marked free! (mode 0x%x)", + I_INO(ip), VFS_I(ip)->i_mode); + xfs_agno_mark_sick(ip->i_mount, XFS_INODE_TO_AGNO(ip), + XFS_SICK_AG_INOBT); + return -EFSCORRUPTED; + } + + if (ip->i_nblocks != 0) { + xfs_warn(ip->i_mount, +"Corruption detected! Free inode 0x%llx has blocks allocated!", + I_INO(ip)); + xfs_agno_mark_sick(ip->i_mount, XFS_INODE_TO_AGNO(ip), + XFS_SICK_AG_INOBT); + return -EFSCORRUPTED; + } + return 0; + } + + /* should be an allocated inode */ + if (VFS_I(ip)->i_mode == 0) + return -ENOENT; + + return 0; +} + +/* Make all pending inactivation work start immediately. */ +static bool +xfs_inodegc_queue_all( + struct xfs_mount *mp) +{ + struct xfs_inodegc *gc; + int cpu; + bool ret = false; + + for_each_cpu(cpu, &mp->m_inodegc_cpumask) { + gc = per_cpu_ptr(mp->m_inodegc, cpu); + if (!llist_empty(&gc->list)) { + mod_delayed_work_on(cpu, mp->m_inodegc_wq, &gc->work, 0); + ret = true; + } + } + + return ret; +} + +/* Wait for all queued work and collect errors */ +static int +xfs_inodegc_wait_all( + struct xfs_mount *mp) +{ + int cpu; + int error = 0; + + flush_workqueue(mp->m_inodegc_wq); + for_each_cpu(cpu, &mp->m_inodegc_cpumask) { + struct xfs_inodegc *gc; + + gc = per_cpu_ptr(mp->m_inodegc, cpu); + if (gc->error && !error) + error = gc->error; + gc->error = 0; + } + + return error; +} + +/* + * Check the validity of the inode we just found it the cache + */ +static int +xfs_iget_cache_hit( + struct xfs_perag *pag, + struct xfs_inode *ip, + xfs_ino_t ino, + int flags, + int lock_flags) __releases(RCU) +{ + struct inode *inode = VFS_I(ip); + struct xfs_mount *mp = ip->i_mount; + int error; + + /* + * check for re-use of an inode within an RCU grace period due to the + * radix tree nodes not being updated yet. We monitor for this by + * setting the inode number to zero before freeing the inode structure. + * If the inode has been reallocated and set up, then the inode number + * will not match, so check for that, too. + */ + spin_lock(&ip->i_flags_lock); + if (I_INO(ip) != ino) + goto out_skip; + + /* + * If we are racing with another cache hit that is currently + * instantiating this inode or currently recycling it out of + * reclaimable state, wait for the initialisation to complete + * before continuing. + * + * If we're racing with the inactivation worker we also want to wait. + * If we're creating a new file, it's possible that the worker + * previously marked the inode as free on disk but hasn't finished + * updating the incore state yet. The AGI buffer will be dirty and + * locked to the icreate transaction, so a synchronous push of the + * inodegc workers would result in deadlock. For a regular iget, the + * worker is running already, so we might as well wait. + * + * XXX(hch): eventually we should do something equivalent to + * wait_on_inode to wait for these flags to be cleared + * instead of polling for it. + */ + if (ip->i_flags & (XFS_INEW | XFS_IRECLAIM | XFS_INACTIVATING)) + goto out_skip; + + if (ip->i_flags & XFS_NEED_INACTIVE) { + /* Unlinked inodes cannot be re-grabbed. */ + if (VFS_I(ip)->i_nlink == 0) { + error = -ENOENT; + goto out_error; + } + goto out_inodegc_flush; + } + + /* + * Check the inode free state is valid. This also detects lookup + * racing with unlinks. + */ + error = xfs_iget_check_free_state(ip, flags); + if (error) + goto out_error; + + /* Skip inodes that have no vfs state. */ + if ((flags & XFS_IGET_INCORE) && + (ip->i_flags & XFS_IRECLAIMABLE)) + goto out_skip; + + /* The inode fits the selection criteria; process it. */ + if (ip->i_flags & XFS_IRECLAIMABLE) { + /* + * We need to make it look like the inode is being reclaimed to + * prevent the actual reclaim workers from stomping over us + * while we recycle the inode. We can't clear the radix tree + * tag yet as it requires pag_ici_lock to be held exclusive. + */ + if (!xfs_ilock_nowait(ip, XFS_ILOCK_EXCL)) + goto out_skip; + ip->i_flags |= XFS_IRECLAIM; + spin_unlock(&ip->i_flags_lock); + rcu_read_unlock(); + + error = xfs_iget_recycle(pag, ip); + if (error) + return error; + } else { + /* If the VFS inode is being torn down, pause and try again. */ + if (!igrab(inode)) + goto out_skip; + + /* We've got a live one. */ + spin_unlock(&ip->i_flags_lock); + rcu_read_unlock(); + trace_xfs_iget_hit(ip); + } + + if (lock_flags != 0) + xfs_ilock(ip, lock_flags); + + if (!(flags & XFS_IGET_INCORE)) + xfs_iflags_clear(ip, XFS_ISTALE); + XFS_STATS_INC(mp, xs_ig_found); + + return 0; + +out_skip: + trace_xfs_iget_skip(ip); + XFS_STATS_INC(mp, xs_ig_frecycle); + error = -EAGAIN; +out_error: + spin_unlock(&ip->i_flags_lock); + rcu_read_unlock(); + return error; + +out_inodegc_flush: + spin_unlock(&ip->i_flags_lock); + rcu_read_unlock(); + /* + * Do not wait for the workers, because the caller could hold an AGI + * buffer lock. We're just going to sleep in a loop anyway. + */ + if (xfs_is_inodegc_enabled(mp)) + xfs_inodegc_queue_all(mp); + return -EAGAIN; +} + +static int +xfs_iget_cache_miss( + struct xfs_mount *mp, + struct xfs_perag *pag, + xfs_trans_t *tp, + xfs_ino_t ino, + struct xfs_inode **ipp, + int flags, + int lock_flags) +{ + struct xfs_inode *ip; + int error; + xfs_agino_t agino = XFS_INO_TO_AGINO(mp, ino); + + ip = xfs_inode_alloc(mp, ino); + if (!ip) + return -ENOMEM; + + /* + * Set XFS_INEW as early as possible so that the health code won't pass + * the inode to the fserror code if the ondisk inode cannot be loaded. + * We're going to free the xfs_inode immediately if that happens, which + * would lead to UAF problems. + */ + xfs_iflags_set(ip, XFS_INEW); + + error = xfs_imap(pag, tp, I_INO(ip), &ip->i_imap, flags); + if (error) + goto out_destroy; + + /* + * For version 5 superblocks, if we are initialising a new inode, we + * simply build the new inode core with a random generation number. + * + * For version 4 (and older) superblocks, log recovery is dependent on + * the i_flushiter field being initialised from the current on-disk + * value and hence we must also read the inode off disk even when + * initializing new inodes. + */ + if (xfs_has_v3inodes(mp) && (flags & XFS_IGET_CREATE)) { + VFS_I(ip)->i_generation = get_random_u32(); + } else { + struct xfs_buf *bp; + + error = xfs_read_icluster(pag, tp, ip->i_imap.im_agbno, &bp); + if (error) + goto out_destroy; + + error = xfs_inode_from_disk(ip, + xfs_buf_offset(bp, ip->i_imap.im_boffset)); + if (!error) + xfs_buf_set_ref(bp, XFS_INO_REF); + else + xfs_inode_mark_sick(ip, XFS_SICK_INO_CORE); + xfs_trans_brelse(tp, bp); + + if (error) + goto out_destroy; + } + + trace_xfs_iget_miss(ip); + + /* + * Check the inode free state is valid. This also detects lookup + * racing with unlinks. + */ + error = xfs_iget_check_free_state(ip, flags); + if (error) + goto out_destroy; + + /* + * Preload the radix tree so we can insert safely under the + * write spinlock. Note that we cannot sleep inside the preload + * region. + */ + if (radix_tree_preload(GFP_KERNEL | __GFP_NOLOCKDEP)) { + error = -EAGAIN; + goto out_destroy; + } + + /* + * Because the inode hasn't been added to the radix-tree yet it can't + * be found by another thread, so we can do the non-sleeping lock here. + */ + if (lock_flags) { + if (!xfs_ilock_nowait(ip, lock_flags)) + BUG(); + } + + /* + * These values must be set before inserting the inode into the radix + * tree as the moment it is inserted a concurrent lookup (allowed by the + * RCU locking mechanism) can find it and that lookup must see that this + * is an inode currently under construction (i.e. that XFS_INEW is set). + * The ip->i_flags_lock that protects the XFS_INEW flag forms the + * memory barrier that ensures this detection works correctly at lookup + * time. + */ + if (flags & XFS_IGET_DONTCACHE) + d_mark_dontcache(VFS_I(ip)); + ip->i_udquot = NULL; + ip->i_gdquot = NULL; + ip->i_pdquot = NULL; + + /* insert the new inode */ + spin_lock(&pag->pag_ici_lock); + error = radix_tree_insert(&pag->pag_ici_root, agino, ip); + if (unlikely(error)) { + WARN_ON(error != -EEXIST); + XFS_STATS_INC(mp, xs_ig_dup); + error = -EAGAIN; + goto out_preload_end; + } + spin_unlock(&pag->pag_ici_lock); + radix_tree_preload_end(); + + *ipp = ip; + return 0; + +out_preload_end: + spin_unlock(&pag->pag_ici_lock); + radix_tree_preload_end(); + if (lock_flags) + xfs_iunlock(ip, lock_flags); +out_destroy: + __destroy_inode(VFS_I(ip)); + xfs_inode_free(ip); + return error; +} + +/* + * Look up an inode by number in the given file system. The inode is looked up + * in the cache held in each AG. If the inode is found in the cache, initialise + * the vfs inode if necessary. + * + * If it is not in core, read it in from the file system's device, add it to the + * cache and initialise the vfs inode. + * + * The inode is locked according to the value of the lock_flags parameter. + * Inode lookup is only done during metadata operations and not as part of the + * data IO path. Hence we only allow locking of the XFS_ILOCK during lookup. + */ +int +xfs_iget( + struct xfs_mount *mp, + struct xfs_trans *tp, + xfs_ino_t ino, + uint flags, + uint lock_flags, + struct xfs_inode **ipp) +{ + struct xfs_inode *ip; + struct xfs_perag *pag; + xfs_agino_t agino; + int error; + + ASSERT((lock_flags & (XFS_IOLOCK_EXCL | XFS_IOLOCK_SHARED)) == 0); + + /* reject inode numbers outside existing AGs */ + if (!xfs_verify_ino(mp, ino)) + return -EINVAL; + + XFS_STATS_INC(mp, xs_ig_attempts); + + /* get the perag structure and ensure that it's inode capable */ + pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ino)); + agino = XFS_INO_TO_AGINO(mp, ino); + +again: + error = 0; + rcu_read_lock(); + ip = radix_tree_lookup(&pag->pag_ici_root, agino); + + if (ip) { + error = xfs_iget_cache_hit(pag, ip, ino, flags, lock_flags); + if (error) + goto out_error_or_again; + } else { + rcu_read_unlock(); + if (flags & XFS_IGET_INCORE) { + error = -ENODATA; + goto out_error_or_again; + } + XFS_STATS_INC(mp, xs_ig_missed); + + error = xfs_iget_cache_miss(mp, pag, tp, ino, &ip, + flags, lock_flags); + if (error) + goto out_error_or_again; + } + xfs_perag_put(pag); + + *ipp = ip; + + /* + * If we have a real type for an on-disk inode, we can setup the inode + * now. If it's a new inode being created, xfs_init_new_inode will + * handle it. + */ + if (xfs_iflags_test(ip, XFS_INEW) && VFS_I(ip)->i_mode != 0) { + xfs_setup_inode(ip); + xfs_setup_iops(ip); + xfs_finish_inode_setup(ip); + } + return 0; + +out_error_or_again: + if (!(flags & (XFS_IGET_INCORE | XFS_IGET_NORETRY)) && + error == -EAGAIN) { + delay(1); + goto again; + } + xfs_perag_put(pag); + return error; +} + +/* + * Get a metadata inode. + * + * The metafile type must match the file mode exactly, and for files in the + * metadata directory tree, it must match the inode's metatype exactly. + */ +int +xfs_trans_metafile_iget( + struct xfs_trans *tp, + xfs_ino_t ino, + enum xfs_metafile_type metafile_type, + struct xfs_inode **ipp) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_inode *ip; + umode_t mode; + int error; + + error = xfs_iget(mp, tp, ino, 0, 0, &ip); + if (error == -EFSCORRUPTED || error == -EINVAL) + goto whine; + if (error) + return error; + + if (VFS_I(ip)->i_nlink == 0) + goto bad_rele; + + if (metafile_type == XFS_METAFILE_DIR) + mode = S_IFDIR; + else + mode = S_IFREG; + if (inode_wrong_type(VFS_I(ip), mode)) + goto bad_rele; + if (xfs_has_metadir(mp)) { + if (!xfs_is_metadir_inode(ip)) + goto bad_rele; + if (metafile_type != ip->i_metatype) + goto bad_rele; + } + + *ipp = ip; + return 0; +bad_rele: + xfs_irele(ip); +whine: + xfs_err(mp, "metadata inode 0x%llx type %u is corrupt", ino, + metafile_type); + xfs_fs_mark_sick(mp, XFS_SICK_FS_METADIR); + return -EFSCORRUPTED; +} + +/* Grab a metadata file if the caller doesn't already have a transaction. */ +int +xfs_metafile_iget( + struct xfs_mount *mp, + xfs_ino_t ino, + enum xfs_metafile_type metafile_type, + struct xfs_inode **ipp) +{ + struct xfs_trans *tp; + int error; + + tp = xfs_trans_alloc_empty(mp); + error = xfs_trans_metafile_iget(tp, ino, metafile_type, ipp); + xfs_trans_cancel(tp); + return error; +} + +/* + * Grab the inode for reclaim exclusively. + * + * We have found this inode via a lookup under RCU, so the inode may have + * already been freed, or it may be in the process of being recycled by + * xfs_iget(). In both cases, the inode will have XFS_IRECLAIM set. If the inode + * has been fully recycled by the time we get the i_flags_lock, XFS_IRECLAIMABLE + * will not be set. Hence we need to check for both these flag conditions to + * avoid inodes that are no longer reclaim candidates. + * + * Note: checking for other state flags here, under the i_flags_lock or not, is + * racy and should be avoided. Those races should be resolved only after we have + * ensured that we are able to reclaim this inode and the world can see that we + * are going to reclaim it. + * + * Return true if we grabbed it, false otherwise. + */ +static bool +xfs_reclaim_igrab( + struct xfs_inode *ip, + struct xfs_icwalk *icw) +{ + ASSERT(rcu_read_lock_held()); + + spin_lock(&ip->i_flags_lock); + if (!__xfs_iflags_test(ip, XFS_IRECLAIMABLE) || + __xfs_iflags_test(ip, XFS_IRECLAIM)) { + /* not a reclaim candidate. */ + spin_unlock(&ip->i_flags_lock); + return false; + } + + /* Don't reclaim a sick inode unless the caller asked for it. */ + if (ip->i_sick && + (!icw || !(icw->icw_flags & XFS_ICWALK_FLAG_RECLAIM_SICK))) { + spin_unlock(&ip->i_flags_lock); + return false; + } + + __xfs_iflags_set(ip, XFS_IRECLAIM); + spin_unlock(&ip->i_flags_lock); + return true; +} + +/* + * Inode reclaim is non-blocking, so the default action if progress cannot be + * made is to "requeue" the inode for reclaim by unlocking it and clearing the + * XFS_IRECLAIM flag. If we are in a shutdown state, we don't care about + * blocking anymore and hence we can wait for the inode to be able to reclaim + * it. + * + * We do no IO here - if callers require inodes to be cleaned they must push the + * AIL first to trigger writeback of dirty inodes. This enables writeback to be + * done in the background in a non-blocking manner, and enables memory reclaim + * to make progress without blocking. + */ +static void +xfs_reclaim_inode( + struct xfs_inode *ip, + struct xfs_perag *pag) +{ + xfs_ino_t ino = I_INO(ip); /* for radix_tree_delete */ + + if (!xfs_ilock_nowait(ip, XFS_ILOCK_EXCL)) + goto out; + if (xfs_iflags_test_and_set(ip, XFS_IFLUSHING)) + goto out_iunlock; + + /* + * Check for log shutdown because aborting the inode can move the log + * tail and corrupt in memory state. This is fine if the log is shut + * down, but if the log is still active and only the mount is shut down + * then the in-memory log tail movement caused by the abort can be + * incorrectly propagated to disk. + */ + if (xlog_is_shutdown(ip->i_mount->m_log)) { + xfs_iunpin_wait(ip); + /* + * Avoid a ABBA deadlock on the inode cluster buffer vs + * concurrent xfs_ifree_cluster() trying to mark the inode + * stale. We don't need the inode locked to run the flush abort + * code, but the flush abort needs to lock the cluster buffer. + */ + xfs_iunlock(ip, XFS_ILOCK_EXCL); + xfs_iflush_shutdown_abort(ip); + xfs_ilock(ip, XFS_ILOCK_EXCL); + goto reclaim; + } + if (xfs_ipincount(ip)) + goto out_clear_flush; + if (!xfs_inode_clean(ip)) + goto out_clear_flush; + + xfs_iflags_clear(ip, XFS_IFLUSHING); +reclaim: + trace_xfs_inode_reclaiming(ip); + + /* + * Because we use RCU freeing we need to ensure the inode always appears + * to be reclaimed with an invalid inode number when in the free state. + * We do this as early as possible under the ILOCK so that + * xfs_iflush_cluster() and xfs_ifree_cluster() can be guaranteed to + * detect races with us here. By doing this, we guarantee that once + * xfs_iflush_cluster() or xfs_ifree_cluster() has locked XFS_ILOCK that + * it will see either a valid inode that will serialise correctly, or it + * will see an invalid inode that it can skip. + */ + spin_lock(&ip->i_flags_lock); + ip->i_flags = XFS_IRECLAIM; + VFS_I(ip)->i_ino = 0; + ip->i_sick = 0; + ip->i_checked = 0; + spin_unlock(&ip->i_flags_lock); + + ASSERT(!ip->i_itemp || ip->i_itemp->ili_item.li_buf == NULL); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + + XFS_STATS_INC(ip->i_mount, xs_ig_reclaims); + /* + * Remove the inode from the per-AG radix tree. + * + * Because radix_tree_delete won't complain even if the item was never + * added to the tree assert that it's been there before to catch + * problems with the inode life time early on. + */ + spin_lock(&pag->pag_ici_lock); + if (!radix_tree_delete(&pag->pag_ici_root, + XFS_INO_TO_AGINO(ip->i_mount, ino))) + ASSERT(0); + xfs_perag_clear_inode_tag(pag, NULLAGINO, XFS_ICI_RECLAIM_TAG); + spin_unlock(&pag->pag_ici_lock); + + /* + * Here we do an (almost) spurious inode lock in order to coordinate + * with inode cache radix tree lookups. This is because the lookup + * can reference the inodes in the cache without taking references. + * + * We make that OK here by ensuring that we wait until the inode is + * unlocked after the lookup before we go ahead and free it. + */ + xfs_ilock(ip, XFS_ILOCK_EXCL); + ASSERT(!ip->i_udquot && !ip->i_gdquot && !ip->i_pdquot); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + ASSERT(xfs_inode_clean(ip)); + + __xfs_inode_free(ip); + return; + +out_clear_flush: + xfs_iflags_clear(ip, XFS_IFLUSHING); +out_iunlock: + xfs_iunlock(ip, XFS_ILOCK_EXCL); +out: + xfs_iflags_clear(ip, XFS_IRECLAIM); +} + +/* Reclaim sick inodes if we're unmounting or the fs went down. */ +static inline bool +xfs_want_reclaim_sick( + struct xfs_mount *mp) +{ + return xfs_is_unmounting(mp) || xfs_has_norecovery(mp) || + xfs_is_shutdown(mp); +} + +void +xfs_reclaim_inodes( + struct xfs_mount *mp) +{ + struct xfs_icwalk icw = { + .icw_flags = 0, + }; + + if (xfs_want_reclaim_sick(mp)) + icw.icw_flags |= XFS_ICWALK_FLAG_RECLAIM_SICK; + + while (xfs_group_marked(mp, XG_TYPE_AG, XFS_PERAG_RECLAIM_MARK)) { + xfs_ail_push_all_sync(mp->m_ail); + xfs_icwalk(mp, XFS_ICWALK_RECLAIM, &icw); + } +} + +/* + * The shrinker infrastructure determines how many inodes we should scan for + * reclaim. We want as many clean inodes ready to reclaim as possible, so we + * push the AIL here. We also want to proactively free up memory if we can to + * minimise the amount of work memory reclaim has to do so we kick the + * background reclaim if it isn't already scheduled. + */ +long +xfs_reclaim_inodes_nr( + struct xfs_mount *mp, + unsigned long nr_to_scan) +{ + struct xfs_icwalk icw = { + .icw_flags = XFS_ICWALK_FLAG_SCAN_LIMIT, + .icw_scan_limit = min_t(unsigned long, LONG_MAX, nr_to_scan), + }; + + if (xfs_want_reclaim_sick(mp)) + icw.icw_flags |= XFS_ICWALK_FLAG_RECLAIM_SICK; + + /* kick background reclaimer and push the AIL */ + xfs_reclaim_work_queue(mp); + xfs_ail_push_all(mp->m_ail); + + xfs_icwalk(mp, XFS_ICWALK_RECLAIM, &icw); + return 0; +} + +/* + * Return the number of reclaimable inodes in the filesystem for + * the shrinker to determine how much to reclaim. + */ +long +xfs_reclaim_inodes_count( + struct xfs_mount *mp) +{ + XA_STATE (xas, &mp->m_groups[XG_TYPE_AG].xa, 0); + long reclaimable = 0; + struct xfs_perag *pag; + + rcu_read_lock(); + xas_for_each_marked(&xas, pag, ULONG_MAX, XFS_PERAG_RECLAIM_MARK) { + trace_xfs_reclaim_inodes_count(pag, _THIS_IP_); + reclaimable += pag->pag_ici_reclaimable; + } + rcu_read_unlock(); + + return reclaimable; +} + +STATIC bool +xfs_icwalk_match_id( + struct xfs_inode *ip, + struct xfs_icwalk *icw) +{ + if ((icw->icw_flags & XFS_ICWALK_FLAG_UID) && + !uid_eq(VFS_I(ip)->i_uid, icw->icw_uid)) + return false; + + if ((icw->icw_flags & XFS_ICWALK_FLAG_GID) && + !gid_eq(VFS_I(ip)->i_gid, icw->icw_gid)) + return false; + + if ((icw->icw_flags & XFS_ICWALK_FLAG_PRID) && + ip->i_projid != icw->icw_prid) + return false; + + return true; +} + +/* + * A union-based inode filtering algorithm. Process the inode if any of the + * criteria match. This is for global/internal scans only. + */ +STATIC bool +xfs_icwalk_match_id_union( + struct xfs_inode *ip, + struct xfs_icwalk *icw) +{ + if ((icw->icw_flags & XFS_ICWALK_FLAG_UID) && + uid_eq(VFS_I(ip)->i_uid, icw->icw_uid)) + return true; + + if ((icw->icw_flags & XFS_ICWALK_FLAG_GID) && + gid_eq(VFS_I(ip)->i_gid, icw->icw_gid)) + return true; + + if ((icw->icw_flags & XFS_ICWALK_FLAG_PRID) && + ip->i_projid == icw->icw_prid) + return true; + + return false; +} + +/* + * Is this inode @ip eligible for eof/cow block reclamation, given some + * filtering parameters @icw? The inode is eligible if @icw is null or + * if the predicate functions match. + */ +static bool +xfs_icwalk_match( + struct xfs_inode *ip, + struct xfs_icwalk *icw) +{ + bool match; + + if (!icw) + return true; + + if (icw->icw_flags & XFS_ICWALK_FLAG_UNION) + match = xfs_icwalk_match_id_union(ip, icw); + else + match = xfs_icwalk_match_id(ip, icw); + if (!match) + return false; + + /* skip the inode if the file size is too small */ + if ((icw->icw_flags & XFS_ICWALK_FLAG_MINFILESIZE) && + XFS_ISIZE(ip) < icw->icw_min_file_size) + return false; + + return true; +} + +/* + * This is a fast pass over the inode cache to try to get reclaim moving on as + * many inodes as possible in a short period of time. It kicks itself every few + * seconds, as well as being kicked by the inode cache shrinker when memory + * goes low. + */ +void +xfs_reclaim_worker( + struct work_struct *work) +{ + struct xfs_mount *mp = container_of(to_delayed_work(work), + struct xfs_mount, m_reclaim_work); + + xfs_icwalk(mp, XFS_ICWALK_RECLAIM, NULL); + xfs_reclaim_work_queue(mp); +} + +STATIC int +xfs_inode_free_eofblocks( + struct xfs_inode *ip, + struct xfs_icwalk *icw, + unsigned int *lockflags) +{ + bool wait; + + wait = icw && (icw->icw_flags & XFS_ICWALK_FLAG_SYNC); + + if (!xfs_iflags_test(ip, XFS_IEOFBLOCKS)) + return 0; + + /* + * If the mapping is dirty the operation can block and wait for some + * time. Unless we are waiting, skip it. + */ + if (!wait && mapping_tagged(VFS_I(ip)->i_mapping, PAGECACHE_TAG_DIRTY)) + return 0; + + if (!xfs_icwalk_match(ip, icw)) + return 0; + + /* + * If the caller is waiting, return -EAGAIN to keep the background + * scanner moving and revisit the inode in a subsequent pass. + */ + if (!xfs_ilock_nowait(ip, XFS_IOLOCK_EXCL)) { + if (wait) + return -EAGAIN; + return 0; + } + *lockflags |= XFS_IOLOCK_EXCL; + + if (xfs_can_free_eofblocks(ip)) + return xfs_free_eofblocks(ip); + + /* inode could be preallocated */ + trace_xfs_inode_free_eofblocks_invalid(ip); + xfs_inode_clear_eofblocks_tag(ip); + return 0; +} + +static void +xfs_blockgc_set_iflag( + struct xfs_inode *ip, + unsigned long iflag) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_perag *pag; + + ASSERT((iflag & ~(XFS_IEOFBLOCKS | XFS_ICOWBLOCKS)) == 0); + + /* + * Don't bother locking the AG and looking up in the radix trees + * if we already know that we have the tag set. + */ + if (ip->i_flags & iflag) + return; + spin_lock(&ip->i_flags_lock); + ip->i_flags |= iflag; + spin_unlock(&ip->i_flags_lock); + + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(ip)); + spin_lock(&pag->pag_ici_lock); + + xfs_perag_set_inode_tag(pag, XFS_INODE_TO_AGINO(ip), + XFS_ICI_BLOCKGC_TAG); + + spin_unlock(&pag->pag_ici_lock); + xfs_perag_put(pag); +} + +void +xfs_inode_set_eofblocks_tag( + xfs_inode_t *ip) +{ + trace_xfs_inode_set_eofblocks_tag(ip); + return xfs_blockgc_set_iflag(ip, XFS_IEOFBLOCKS); +} + +static void +xfs_blockgc_clear_iflag( + struct xfs_inode *ip, + unsigned long iflag) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_perag *pag; + bool clear_tag; + + ASSERT((iflag & ~(XFS_IEOFBLOCKS | XFS_ICOWBLOCKS)) == 0); + + spin_lock(&ip->i_flags_lock); + ip->i_flags &= ~iflag; + clear_tag = (ip->i_flags & (XFS_IEOFBLOCKS | XFS_ICOWBLOCKS)) == 0; + spin_unlock(&ip->i_flags_lock); + + if (!clear_tag) + return; + + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(ip)); + spin_lock(&pag->pag_ici_lock); + + xfs_perag_clear_inode_tag(pag, XFS_INODE_TO_AGINO(ip), + XFS_ICI_BLOCKGC_TAG); + + spin_unlock(&pag->pag_ici_lock); + xfs_perag_put(pag); +} + +void +xfs_inode_clear_eofblocks_tag( + xfs_inode_t *ip) +{ + trace_xfs_inode_clear_eofblocks_tag(ip); + return xfs_blockgc_clear_iflag(ip, XFS_IEOFBLOCKS); +} + +/* + * Prepare to free COW fork blocks from an inode. + */ +static bool +xfs_prep_free_cowblocks( + struct xfs_inode *ip, + struct xfs_icwalk *icw) +{ + bool sync; + + sync = icw && (icw->icw_flags & XFS_ICWALK_FLAG_SYNC); + + /* + * Just clear the tag if we have an empty cow fork or none at all. It's + * possible the inode was fully unshared since it was originally tagged. + */ + if (!xfs_inode_has_cow_data(ip)) { + trace_xfs_inode_free_cowblocks_invalid(ip); + xfs_inode_clear_cowblocks_tag(ip); + return false; + } + + /* + * A cowblocks trim of an inode can have a significant effect on + * fragmentation even when a reasonable COW extent size hint is set. + * Therefore, we prefer to not process cowblocks unless they are clean + * and idle. We can never process a cowblocks inode that is dirty or has + * in-flight I/O under any circumstances, because outstanding writeback + * or dio expects targeted COW fork blocks exist through write + * completion where they can be remapped into the data fork. + * + * Therefore, the heuristic used here is to never process inodes + * currently opened for write from background (i.e. non-sync) scans. For + * sync scans, use the pagecache/dio state of the inode to ensure we + * never free COW fork blocks out from under pending I/O. + */ + if (!sync && inode_is_open_for_write(VFS_I(ip))) + return false; + return xfs_can_free_cowblocks(ip); +} + +/* + * Automatic CoW Reservation Freeing + * + * These functions automatically garbage collect leftover CoW reservations + * that were made on behalf of a cowextsize hint when we start to run out + * of quota or when the reservations sit around for too long. If the file + * has dirty pages or is undergoing writeback, its CoW reservations will + * be retained. + * + * The actual garbage collection piggybacks off the same code that runs + * the speculative EOF preallocation garbage collector. + */ +STATIC int +xfs_inode_free_cowblocks( + struct xfs_inode *ip, + struct xfs_icwalk *icw, + unsigned int *lockflags) +{ + bool wait; + int ret = 0; + + wait = icw && (icw->icw_flags & XFS_ICWALK_FLAG_SYNC); + + if (!xfs_iflags_test(ip, XFS_ICOWBLOCKS)) + return 0; + + if (!xfs_prep_free_cowblocks(ip, icw)) + return 0; + + if (!xfs_icwalk_match(ip, icw)) + return 0; + + /* + * If the caller is waiting, return -EAGAIN to keep the background + * scanner moving and revisit the inode in a subsequent pass. + */ + if (!(*lockflags & XFS_IOLOCK_EXCL) && + !xfs_ilock_nowait(ip, XFS_IOLOCK_EXCL)) { + if (wait) + return -EAGAIN; + return 0; + } + *lockflags |= XFS_IOLOCK_EXCL; + + if (!xfs_ilock_nowait(ip, XFS_MMAPLOCK_EXCL)) { + if (wait) + return -EAGAIN; + return 0; + } + *lockflags |= XFS_MMAPLOCK_EXCL; + + /* + * Check again, nobody else should be able to dirty blocks or change + * the reflink iflag now that we have the first two locks held. + */ + if (xfs_prep_free_cowblocks(ip, icw)) + ret = xfs_reflink_cancel_cow_range(ip, 0, NULLFILEOFF, false); + return ret; +} + +void +xfs_inode_set_cowblocks_tag( + xfs_inode_t *ip) +{ + trace_xfs_inode_set_cowblocks_tag(ip); + return xfs_blockgc_set_iflag(ip, XFS_ICOWBLOCKS); +} + +void +xfs_inode_clear_cowblocks_tag( + xfs_inode_t *ip) +{ + trace_xfs_inode_clear_cowblocks_tag(ip); + return xfs_blockgc_clear_iflag(ip, XFS_ICOWBLOCKS); +} + +/* Disable post-EOF and CoW block auto-reclamation. */ +void +xfs_blockgc_stop( + struct xfs_mount *mp) +{ + struct xfs_perag *pag = NULL; + + if (!xfs_clear_blockgc_enabled(mp)) + return; + + while ((pag = xfs_perag_next(mp, pag))) + cancel_delayed_work_sync(&pag->pag_blockgc_work); + trace_xfs_blockgc_stop(mp, __return_address); +} + +/* Enable post-EOF and CoW block auto-reclamation. */ +void +xfs_blockgc_start( + struct xfs_mount *mp) +{ + struct xfs_perag *pag = NULL; + + if (xfs_set_blockgc_enabled(mp)) + return; + + trace_xfs_blockgc_start(mp, __return_address); + while ((pag = xfs_perag_grab_next_tag(mp, pag, XFS_ICI_BLOCKGC_TAG))) + xfs_blockgc_queue(pag); +} + +/* Don't try to run block gc on an inode that's in any of these states. */ +#define XFS_BLOCKGC_NOGRAB_IFLAGS (XFS_INEW | \ + XFS_NEED_INACTIVE | \ + XFS_INACTIVATING | \ + XFS_IRECLAIMABLE | \ + XFS_IRECLAIM) +/* + * Decide if the given @ip is eligible for garbage collection of speculative + * preallocations, and grab it if so. Returns true if it's ready to go or + * false if we should just ignore it. + */ +static bool +xfs_blockgc_igrab( + struct xfs_inode *ip) +{ + struct inode *inode = VFS_I(ip); + + ASSERT(rcu_read_lock_held()); + + /* Check for stale RCU freed inode */ + spin_lock(&ip->i_flags_lock); + if (!I_INO(ip)) + goto out_unlock_noent; + + if (ip->i_flags & XFS_BLOCKGC_NOGRAB_IFLAGS) + goto out_unlock_noent; + spin_unlock(&ip->i_flags_lock); + + /* nothing to sync during shutdown */ + if (xfs_is_shutdown(ip->i_mount)) + return false; + + /* If we can't grab the inode, it must on it's way to reclaim. */ + if (!igrab(inode)) + return false; + + /* inode is valid */ + return true; + +out_unlock_noent: + spin_unlock(&ip->i_flags_lock); + return false; +} + +/* Scan one incore inode for block preallocations that we can remove. */ +static int +xfs_blockgc_scan_inode( + struct xfs_inode *ip, + struct xfs_icwalk *icw) +{ + unsigned int lockflags = 0; + int error; + + error = xfs_inode_free_eofblocks(ip, icw, &lockflags); + if (error) + goto unlock; + + error = xfs_inode_free_cowblocks(ip, icw, &lockflags); +unlock: + if (lockflags) + xfs_iunlock(ip, lockflags); + xfs_irele(ip); + return error; +} + +/* Background worker that trims preallocated space. */ +void +xfs_blockgc_worker( + struct work_struct *work) +{ + struct xfs_perag *pag = container_of(to_delayed_work(work), + struct xfs_perag, pag_blockgc_work); + struct xfs_mount *mp = pag_mount(pag); + int error; + + trace_xfs_blockgc_worker(mp, __return_address); + + error = xfs_icwalk_ag(pag, XFS_ICWALK_BLOCKGC, NULL); + if (error) + xfs_info(mp, "AG %u preallocation gc worker failed, err=%d", + pag_agno(pag), error); + xfs_blockgc_queue(pag); +} + +/* + * Try to free space in the filesystem by purging inactive inodes, eofblocks + * and cowblocks. + */ +int +xfs_blockgc_free_space( + struct xfs_mount *mp, + struct xfs_icwalk *icw) +{ + int error; + + trace_xfs_blockgc_free_space(mp, icw, _RET_IP_); + + error = xfs_icwalk(mp, XFS_ICWALK_BLOCKGC, icw); + if (error) + return error; + + return xfs_inodegc_flush(mp); +} + +/* + * Reclaim all the free space that we can by scheduling the background blockgc + * and inodegc workers immediately and waiting for them all to clear. + */ +int +xfs_blockgc_flush_all( + struct xfs_mount *mp) +{ + struct xfs_perag *pag = NULL; + + trace_xfs_blockgc_flush_all(mp, __return_address); + + /* + * For each blockgc worker, move its queue time up to now. If it wasn't + * queued, it will not be requeued. Then flush whatever is left. + */ + while ((pag = xfs_perag_grab_next_tag(mp, pag, XFS_ICI_BLOCKGC_TAG))) + mod_delayed_work(mp->m_blockgc_wq, &pag->pag_blockgc_work, 0); + + while ((pag = xfs_perag_grab_next_tag(mp, pag, XFS_ICI_BLOCKGC_TAG))) + flush_delayed_work(&pag->pag_blockgc_work); + + return xfs_inodegc_flush(mp); +} + +/* + * Run cow/eofblocks scans on the supplied dquots. We don't know exactly which + * quota caused an allocation failure, so we make a best effort by including + * each quota under low free space conditions (less than 1% free space) in the + * scan. + * + * Callers must not hold any inode's ILOCK. If requesting a synchronous scan + * (XFS_ICWALK_FLAG_SYNC), the caller also must not hold any inode's IOLOCK or + * MMAPLOCK. + */ +int +xfs_blockgc_free_dquots( + struct xfs_mount *mp, + struct xfs_dquot *udqp, + struct xfs_dquot *gdqp, + struct xfs_dquot *pdqp, + unsigned int iwalk_flags) +{ + struct xfs_icwalk icw = {0}; + bool do_work = false; + + if (!udqp && !gdqp && !pdqp) + return 0; + + /* + * Run a scan to free blocks using the union filter to cover all + * applicable quotas in a single scan. + */ + icw.icw_flags = XFS_ICWALK_FLAG_UNION | iwalk_flags; + + if (XFS_IS_UQUOTA_ENFORCED(mp) && udqp && xfs_dquot_lowsp(udqp)) { + icw.icw_uid = make_kuid(mp->m_super->s_user_ns, udqp->q_id); + icw.icw_flags |= XFS_ICWALK_FLAG_UID; + do_work = true; + } + + if (XFS_IS_UQUOTA_ENFORCED(mp) && gdqp && xfs_dquot_lowsp(gdqp)) { + icw.icw_gid = make_kgid(mp->m_super->s_user_ns, gdqp->q_id); + icw.icw_flags |= XFS_ICWALK_FLAG_GID; + do_work = true; + } + + if (XFS_IS_PQUOTA_ENFORCED(mp) && pdqp && xfs_dquot_lowsp(pdqp)) { + icw.icw_prid = pdqp->q_id; + icw.icw_flags |= XFS_ICWALK_FLAG_PRID; + do_work = true; + } + + if (!do_work) + return 0; + + return xfs_blockgc_free_space(mp, &icw); +} + +/* Run cow/eofblocks scans on the quotas attached to the inode. */ +int +xfs_blockgc_free_quota( + struct xfs_inode *ip, + unsigned int iwalk_flags) +{ + return xfs_blockgc_free_dquots(ip->i_mount, + xfs_inode_dquot(ip, XFS_DQTYPE_USER), + xfs_inode_dquot(ip, XFS_DQTYPE_GROUP), + xfs_inode_dquot(ip, XFS_DQTYPE_PROJ), iwalk_flags); +} + +/* XFS Inode Cache Walking Code */ + +/* + * The inode lookup is done in batches to keep the amount of lock traffic and + * radix tree lookups to a minimum. The batch size is a trade off between + * lookup reduction and stack usage. This is in the reclaim path, so we can't + * be too greedy. + */ +#define XFS_LOOKUP_BATCH 32 + + +/* + * Decide if we want to grab this inode in anticipation of doing work towards + * the goal. + */ +static inline bool +xfs_icwalk_igrab( + enum xfs_icwalk_goal goal, + struct xfs_inode *ip, + struct xfs_icwalk *icw) +{ + switch (goal) { + case XFS_ICWALK_BLOCKGC: + return xfs_blockgc_igrab(ip); + case XFS_ICWALK_RECLAIM: + return xfs_reclaim_igrab(ip, icw); + default: + return false; + } +} + +/* + * Process an inode. Each processing function must handle any state changes + * made by the icwalk igrab function. Return -EAGAIN to skip an inode. + */ +static inline int +xfs_icwalk_process_inode( + enum xfs_icwalk_goal goal, + struct xfs_inode *ip, + struct xfs_perag *pag, + struct xfs_icwalk *icw) +{ + int error = 0; + + switch (goal) { + case XFS_ICWALK_BLOCKGC: + error = xfs_blockgc_scan_inode(ip, icw); + break; + case XFS_ICWALK_RECLAIM: + xfs_reclaim_inode(ip, pag); + break; + } + return error; +} + +/* + * For a given per-AG structure @pag and a goal, grab qualifying inodes and + * process them in some manner. + */ +static int +xfs_icwalk_ag( + struct xfs_perag *pag, + enum xfs_icwalk_goal goal, + struct xfs_icwalk *icw) +{ + struct xfs_mount *mp = pag_mount(pag); + uint32_t first_index; + int last_error = 0; + int skipped; + bool done; + int nr_found; + +restart: + done = false; + skipped = 0; + if (goal == XFS_ICWALK_RECLAIM) + first_index = READ_ONCE(pag->pag_ici_reclaim_cursor); + else + first_index = 0; + nr_found = 0; + do { + struct xfs_inode *batch[XFS_LOOKUP_BATCH]; + int error = 0; + int i; + + rcu_read_lock(); + + nr_found = radix_tree_gang_lookup_tag(&pag->pag_ici_root, + (void **) batch, first_index, + XFS_LOOKUP_BATCH, goal); + if (!nr_found) { + done = true; + rcu_read_unlock(); + break; + } + + /* + * Grab the inodes before we drop the lock. if we found + * nothing, nr == 0 and the loop will be skipped. + */ + for (i = 0; i < nr_found; i++) { + struct xfs_inode *ip = batch[i]; + + if (done || !xfs_icwalk_igrab(goal, ip, icw)) + batch[i] = NULL; + + /* + * Update the index for the next lookup. Catch + * overflows into the next AG range which can occur if + * we have inodes in the last block of the AG and we + * are currently pointing to the last inode. + * + * Because we may see inodes that are from the wrong AG + * due to RCU freeing and reallocation, only update the + * index if it lies in this AG. It was a race that lead + * us to see this inode, so another lookup from the + * same index will not find it again. + */ + if (XFS_INODE_TO_AGNO(ip) != pag_agno(pag)) + continue; + first_index = XFS_INO_TO_AGINO(mp, I_INO(ip) + 1); + if (first_index < XFS_INODE_TO_AGINO(ip)) + done = true; + } + + /* unlock now we've grabbed the inodes. */ + rcu_read_unlock(); + + for (i = 0; i < nr_found; i++) { + if (!batch[i]) + continue; + error = xfs_icwalk_process_inode(goal, batch[i], pag, + icw); + if (error == -EAGAIN) { + skipped++; + continue; + } + if (error && last_error != -EFSCORRUPTED) + last_error = error; + } + + /* bail out if the filesystem is corrupted. */ + if (error == -EFSCORRUPTED) + break; + + cond_resched(); + + if (icw && (icw->icw_flags & XFS_ICWALK_FLAG_SCAN_LIMIT)) { + icw->icw_scan_limit -= XFS_LOOKUP_BATCH; + if (icw->icw_scan_limit <= 0) + break; + } + } while (nr_found && !done); + + if (goal == XFS_ICWALK_RECLAIM) { + if (done) + first_index = 0; + WRITE_ONCE(pag->pag_ici_reclaim_cursor, first_index); + } + + if (skipped) { + delay(1); + goto restart; + } + return last_error; +} + +/* Walk all incore inodes to achieve a given goal. */ +static int +xfs_icwalk( + struct xfs_mount *mp, + enum xfs_icwalk_goal goal, + struct xfs_icwalk *icw) +{ + struct xfs_perag *pag = NULL; + int error = 0; + int last_error = 0; + + while ((pag = xfs_perag_grab_next_tag(mp, pag, goal))) { + error = xfs_icwalk_ag(pag, goal, icw); + if (error) { + last_error = error; + if (error == -EFSCORRUPTED) { + xfs_perag_rele(pag); + break; + } + } + } + return last_error; + BUILD_BUG_ON(XFS_ICWALK_PRIVATE_FLAGS & XFS_ICWALK_FLAGS_VALID); +} + +#ifdef DEBUG +static void +xfs_check_delalloc( + struct xfs_inode *ip, + int whichfork) +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + struct xfs_bmbt_irec got; + struct xfs_iext_cursor icur; + + if (!ifp || !xfs_iext_lookup_extent(ip, ifp, 0, &icur, &got)) + return; + do { + if (isnullstartblock(got.br_startblock)) { + xfs_warn(ip->i_mount, + "ino %llx %s fork has delalloc extent at [0x%llx:0x%llx]", + I_INO(ip), + whichfork == XFS_DATA_FORK ? "data" : "cow", + got.br_startoff, got.br_blockcount); + } + } while (xfs_iext_next_extent(ifp, &icur, &got)); +} +#else +#define xfs_check_delalloc(ip, whichfork) do { } while (0) +#endif + +/* Schedule the inode for reclaim. */ +static void +xfs_inodegc_set_reclaimable( + struct xfs_inode *ip) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_perag *pag; + + if (!xfs_is_shutdown(mp) && ip->i_delayed_blks) { + xfs_check_delalloc(ip, XFS_DATA_FORK); + xfs_check_delalloc(ip, XFS_COW_FORK); + ASSERT(0); + } + + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(ip)); + spin_lock(&pag->pag_ici_lock); + spin_lock(&ip->i_flags_lock); + + trace_xfs_inode_set_reclaimable(ip); + ip->i_flags &= ~(XFS_NEED_INACTIVE | XFS_INACTIVATING); + ip->i_flags |= XFS_IRECLAIMABLE; + xfs_perag_set_inode_tag(pag, XFS_INODE_TO_AGINO(ip), + XFS_ICI_RECLAIM_TAG); + + spin_unlock(&ip->i_flags_lock); + spin_unlock(&pag->pag_ici_lock); + xfs_perag_put(pag); +} + +/* + * Free all speculative preallocations and possibly even the inode itself. + * This is the last chance to make changes to an otherwise unreferenced file + * before incore reclamation happens. + */ +static int +xfs_inodegc_inactivate( + struct xfs_inode *ip) +{ + int error; + + trace_xfs_inode_inactivating(ip); + error = xfs_inactive(ip); + xfs_inodegc_set_reclaimable(ip); + return error; + +} + +void +xfs_inodegc_worker( + struct work_struct *work) +{ + struct xfs_inodegc *gc = container_of(to_delayed_work(work), + struct xfs_inodegc, work); + struct llist_node *node = llist_del_all(&gc->list); + struct xfs_inode *ip, *n; + struct xfs_mount *mp = gc->mp; + unsigned int nofs_flag; + + /* + * Clear the cpu mask bit and ensure that we have seen the latest + * update of the gc structure associated with this CPU. This matches + * with the release semantics used when setting the cpumask bit in + * xfs_inodegc_queue. + */ + cpumask_clear_cpu(gc->cpu, &mp->m_inodegc_cpumask); + smp_mb__after_atomic(); + + WRITE_ONCE(gc->items, 0); + + if (!node) + return; + + /* + * We can allocate memory here while doing writeback on behalf of + * memory reclaim. To avoid memory allocation deadlocks set the + * task-wide nofs context for the following operations. + */ + nofs_flag = memalloc_nofs_save(); + + ip = llist_entry(node, struct xfs_inode, i_gclist); + trace_xfs_inodegc_worker(mp, READ_ONCE(gc->shrinker_hits)); + + WRITE_ONCE(gc->shrinker_hits, 0); + llist_for_each_entry_safe(ip, n, node, i_gclist) { + int error; + + xfs_iflags_set(ip, XFS_INACTIVATING); + error = xfs_inodegc_inactivate(ip); + if (error && !gc->error) + gc->error = error; + } + + memalloc_nofs_restore(nofs_flag); +} + +/* + * Expedite all pending inodegc work to run immediately. This does not wait for + * completion of the work. + */ +void +xfs_inodegc_push( + struct xfs_mount *mp) +{ + if (!xfs_is_inodegc_enabled(mp)) + return; + trace_xfs_inodegc_push(mp, __return_address); + xfs_inodegc_queue_all(mp); +} + +/* + * Force all currently queued inode inactivation work to run immediately and + * wait for the work to finish. + */ +int +xfs_inodegc_flush( + struct xfs_mount *mp) +{ + xfs_inodegc_push(mp); + trace_xfs_inodegc_flush(mp, __return_address); + return xfs_inodegc_wait_all(mp); +} + +/* + * Flush all the pending work and then disable the inode inactivation background + * workers and wait for them to stop. Caller must hold sb->s_umount to + * coordinate changes in the inodegc_enabled state. + */ +void +xfs_inodegc_stop( + struct xfs_mount *mp) +{ + bool rerun; + + if (!xfs_clear_inodegc_enabled(mp)) + return; + + /* + * Drain all pending inodegc work, including inodes that could be + * queued by racing xfs_inodegc_queue or xfs_inodegc_shrinker_scan + * threads that sample the inodegc state just prior to us clearing it. + * The inodegc flag state prevents new threads from queuing more + * inodes, so we queue pending work items and flush the workqueue until + * all inodegc lists are empty. IOWs, we cannot use drain_workqueue + * here because it does not allow other unserialized mechanisms to + * reschedule inodegc work while this draining is in progress. + */ + xfs_inodegc_queue_all(mp); + do { + flush_workqueue(mp->m_inodegc_wq); + rerun = xfs_inodegc_queue_all(mp); + } while (rerun); + + trace_xfs_inodegc_stop(mp, __return_address); +} + +/* + * Enable the inode inactivation background workers and schedule deferred inode + * inactivation work if there is any. Caller must hold sb->s_umount to + * coordinate changes in the inodegc_enabled state. + */ +void +xfs_inodegc_start( + struct xfs_mount *mp) +{ + if (xfs_set_inodegc_enabled(mp)) + return; + + trace_xfs_inodegc_start(mp, __return_address); + xfs_inodegc_queue_all(mp); +} + +#ifdef CONFIG_XFS_RT +static inline bool +xfs_inodegc_want_queue_rt_file( + struct xfs_inode *ip) +{ + struct xfs_mount *mp = ip->i_mount; + + if (!XFS_IS_REALTIME_INODE(ip) || xfs_has_zoned(mp)) + return false; + + if (xfs_compare_freecounter(mp, XC_FREE_RTEXTENTS, + mp->m_low_rtexts[XFS_LOWSP_5_PCNT], + XFS_FDBLOCKS_BATCH) < 0) + return true; + + return false; +} +#else +# define xfs_inodegc_want_queue_rt_file(ip) (false) +#endif /* CONFIG_XFS_RT */ + +/* + * Schedule the inactivation worker when: + * + * - We've accumulated more than one inode cluster buffer's worth of inodes. + * - There is less than 5% free space left. + * - Any of the quotas for this inode are near an enforcement limit. + */ +static inline bool +xfs_inodegc_want_queue_work( + struct xfs_inode *ip, + unsigned int items) +{ + struct xfs_mount *mp = ip->i_mount; + + if (items > mp->m_ino_geo.inodes_per_cluster) + return true; + + if (xfs_compare_freecounter(mp, XC_FREE_BLOCKS, + mp->m_low_space[XFS_LOWSP_5_PCNT], + XFS_FDBLOCKS_BATCH) < 0) + return true; + + if (xfs_inodegc_want_queue_rt_file(ip)) + return true; + + if (xfs_inode_near_dquot_enforcement(ip, XFS_DQTYPE_USER)) + return true; + + if (xfs_inode_near_dquot_enforcement(ip, XFS_DQTYPE_GROUP)) + return true; + + if (xfs_inode_near_dquot_enforcement(ip, XFS_DQTYPE_PROJ)) + return true; + + return false; +} + +/* + * Upper bound on the number of inodes in each AG that can be queued for + * inactivation at any given time, to avoid monopolizing the workqueue. + */ +#define XFS_INODEGC_MAX_BACKLOG (4 * XFS_INODES_PER_CHUNK) + +/* + * Make the frontend wait for inactivations when: + * + * - Memory shrinkers queued the inactivation worker and it hasn't finished. + * - The queue depth exceeds the maximum allowable percpu backlog. + * + * Note: If we are in a NOFS context here (e.g. current thread is running a + * transaction) the we don't want to block here as inodegc progress may require + * filesystem resources we hold to make progress and that could result in a + * deadlock. Hence we skip out of here if we are in a scoped NOFS context. + */ +static inline bool +xfs_inodegc_want_flush_work( + struct xfs_inode *ip, + unsigned int items, + unsigned int shrinker_hits) +{ + if (current->flags & PF_MEMALLOC_NOFS) + return false; + + if (shrinker_hits > 0) + return true; + + if (items > XFS_INODEGC_MAX_BACKLOG) + return true; + + return false; +} + +/* + * Queue a background inactivation worker if there are inodes that need to be + * inactivated and higher level xfs code hasn't disabled the background + * workers. + */ +static void +xfs_inodegc_queue( + struct xfs_inode *ip) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_inodegc *gc; + int items; + unsigned int shrinker_hits; + unsigned int cpu_nr; + unsigned long queue_delay = 1; + + trace_xfs_inode_set_need_inactive(ip); + spin_lock(&ip->i_flags_lock); + ip->i_flags |= XFS_NEED_INACTIVE; + spin_unlock(&ip->i_flags_lock); + + cpu_nr = get_cpu(); + gc = this_cpu_ptr(mp->m_inodegc); + llist_add(&ip->i_gclist, &gc->list); + items = READ_ONCE(gc->items); + WRITE_ONCE(gc->items, items + 1); + shrinker_hits = READ_ONCE(gc->shrinker_hits); + + /* + * Ensure the list add is always seen by anyone who finds the cpumask + * bit set. This effectively gives the cpumask bit set operation + * release ordering semantics. + */ + smp_mb__before_atomic(); + if (!cpumask_test_cpu(cpu_nr, &mp->m_inodegc_cpumask)) + cpumask_test_and_set_cpu(cpu_nr, &mp->m_inodegc_cpumask); + + /* + * We queue the work while holding the current CPU so that the work + * is scheduled to run on this CPU. + */ + if (!xfs_is_inodegc_enabled(mp)) { + put_cpu(); + return; + } + + if (xfs_inodegc_want_queue_work(ip, items)) + queue_delay = 0; + + trace_xfs_inodegc_queue(mp, __return_address); + mod_delayed_work_on(current_cpu(), mp->m_inodegc_wq, &gc->work, + queue_delay); + put_cpu(); + + if (xfs_inodegc_want_flush_work(ip, items, shrinker_hits)) { + trace_xfs_inodegc_throttle(mp, __return_address); + flush_delayed_work(&gc->work); + } +} + +/* + * We set the inode flag atomically with the radix tree tag. Once we get tag + * lookups on the radix tree, this inode flag can go away. + * + * We always use background reclaim here because even if the inode is clean, it + * still may be under IO and hence we have wait for IO completion to occur + * before we can reclaim the inode. The background reclaim path handles this + * more efficiently than we can here, so simply let background reclaim tear down + * all inodes. + */ +void +xfs_inode_mark_reclaimable( + struct xfs_inode *ip) +{ + struct xfs_mount *mp = ip->i_mount; + bool need_inactive; + + XFS_STATS_INC(mp, xs_inode_mark_reclaimable); + + /* + * We should never get here with any of the reclaim flags already set. + */ + ASSERT_ALWAYS(!xfs_iflags_test(ip, XFS_ALL_IRECLAIM_FLAGS)); + + need_inactive = xfs_inode_needs_inactive(ip); + if (need_inactive) { + xfs_inodegc_queue(ip); + return; + } + + /* Going straight to reclaim, so drop the dquots. */ + xfs_qm_dqdetach(ip); + xfs_inodegc_set_reclaimable(ip); +} + +/* + * Register a phony shrinker so that we can run background inodegc sooner when + * there's memory pressure. Inactivation does not itself free any memory but + * it does make inodes reclaimable, which eventually frees memory. + * + * The count function, seek value, and batch value are crafted to trigger the + * scan function during the second round of scanning. Hopefully this means + * that we reclaimed enough memory that initiating metadata transactions won't + * make things worse. + */ +#define XFS_INODEGC_SHRINKER_COUNT (1UL << DEF_PRIORITY) +#define XFS_INODEGC_SHRINKER_BATCH ((XFS_INODEGC_SHRINKER_COUNT / 2) + 1) + +static unsigned long +xfs_inodegc_shrinker_count( + struct shrinker *shrink, + struct shrink_control *sc) +{ + struct xfs_mount *mp = shrink->private_data; + struct xfs_inodegc *gc; + int cpu; + + if (!xfs_is_inodegc_enabled(mp)) + return 0; + + for_each_cpu(cpu, &mp->m_inodegc_cpumask) { + gc = per_cpu_ptr(mp->m_inodegc, cpu); + if (!llist_empty(&gc->list)) + return XFS_INODEGC_SHRINKER_COUNT; + } + + return 0; +} + +static unsigned long +xfs_inodegc_shrinker_scan( + struct shrinker *shrink, + struct shrink_control *sc) +{ + struct xfs_mount *mp = shrink->private_data; + struct xfs_inodegc *gc; + int cpu; + bool no_items = true; + + if (!xfs_is_inodegc_enabled(mp)) + return SHRINK_STOP; + + trace_xfs_inodegc_shrinker_scan(mp, sc, __return_address); + + for_each_cpu(cpu, &mp->m_inodegc_cpumask) { + gc = per_cpu_ptr(mp->m_inodegc, cpu); + if (!llist_empty(&gc->list)) { + unsigned int h = READ_ONCE(gc->shrinker_hits); + + WRITE_ONCE(gc->shrinker_hits, h + 1); + mod_delayed_work_on(cpu, mp->m_inodegc_wq, &gc->work, 0); + no_items = false; + } + } + + /* + * If there are no inodes to inactivate, we don't want the shrinker + * to think there's deferred work to call us back about. + */ + if (no_items) + return LONG_MAX; + + return SHRINK_STOP; +} + +/* Register a shrinker so we can accelerate inodegc and throttle queuing. */ +int +xfs_inodegc_register_shrinker( + struct xfs_mount *mp) +{ + mp->m_inodegc_shrinker = shrinker_alloc(SHRINKER_NONSLAB, + "xfs-inodegc:%s", + mp->m_super->s_id); + if (!mp->m_inodegc_shrinker) + return -ENOMEM; + + mp->m_inodegc_shrinker->count_objects = xfs_inodegc_shrinker_count; + mp->m_inodegc_shrinker->scan_objects = xfs_inodegc_shrinker_scan; + mp->m_inodegc_shrinker->seeks = 0; + mp->m_inodegc_shrinker->batch = XFS_INODEGC_SHRINKER_BATCH; + mp->m_inodegc_shrinker->private_data = mp; + + shrinker_register(mp->m_inodegc_shrinker); + + return 0; +} diff --git a/fs/xfs/xfs_icache.h b/fs/xfs/xfs_icache.h new file mode 100644 index 000000000..905944daf --- /dev/null +++ b/fs/xfs/xfs_icache.h @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2006 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef XFS_SYNC_H +#define XFS_SYNC_H 1 + +struct xfs_mount; +struct xfs_perag; + +struct xfs_icwalk { + __u32 icw_flags; + kuid_t icw_uid; + kgid_t icw_gid; + prid_t icw_prid; + __u64 icw_min_file_size; + long icw_scan_limit; +}; + +/* Flags that reflect xfs_fs_eofblocks functionality. */ +#define XFS_ICWALK_FLAG_SYNC (1U << 0) /* sync/wait mode scan */ +#define XFS_ICWALK_FLAG_UID (1U << 1) /* filter by uid */ +#define XFS_ICWALK_FLAG_GID (1U << 2) /* filter by gid */ +#define XFS_ICWALK_FLAG_PRID (1U << 3) /* filter by project id */ +#define XFS_ICWALK_FLAG_MINFILESIZE (1U << 4) /* filter by min file size */ + +#define XFS_ICWALK_FLAGS_VALID (XFS_ICWALK_FLAG_SYNC | \ + XFS_ICWALK_FLAG_UID | \ + XFS_ICWALK_FLAG_GID | \ + XFS_ICWALK_FLAG_PRID | \ + XFS_ICWALK_FLAG_MINFILESIZE) + +/* + * Flags for xfs_iget() + */ +#define XFS_IGET_CREATE (1U << 0) +#define XFS_IGET_UNTRUSTED (1U << 1) +#define XFS_IGET_DONTCACHE (1U << 2) +/* don't read from disk or reinit */ +#define XFS_IGET_INCORE (1U << 3) +/* Return -EAGAIN immediately if the inode is unavailable. */ +#define XFS_IGET_NORETRY (1U << 4) + +int xfs_iget(struct xfs_mount *mp, struct xfs_trans *tp, xfs_ino_t ino, + uint flags, uint lock_flags, xfs_inode_t **ipp); + +/* recovery needs direct inode allocation capability */ +struct xfs_inode * xfs_inode_alloc(struct xfs_mount *mp, xfs_ino_t ino); +void xfs_inode_free(struct xfs_inode *ip); + +void xfs_reclaim_worker(struct work_struct *work); + +void xfs_reclaim_inodes(struct xfs_mount *mp); +long xfs_reclaim_inodes_count(struct xfs_mount *mp); +long xfs_reclaim_inodes_nr(struct xfs_mount *mp, unsigned long nr_to_scan); + +void xfs_inode_mark_reclaimable(struct xfs_inode *ip); + +int xfs_blockgc_free_dquots(struct xfs_mount *mp, struct xfs_dquot *udqp, + struct xfs_dquot *gdqp, struct xfs_dquot *pdqp, + unsigned int iwalk_flags); +int xfs_blockgc_free_quota(struct xfs_inode *ip, unsigned int iwalk_flags); +int xfs_blockgc_free_space(struct xfs_mount *mp, struct xfs_icwalk *icm); +int xfs_blockgc_flush_all(struct xfs_mount *mp); + +void xfs_inode_set_eofblocks_tag(struct xfs_inode *ip); +void xfs_inode_clear_eofblocks_tag(struct xfs_inode *ip); + +void xfs_inode_set_cowblocks_tag(struct xfs_inode *ip); +void xfs_inode_clear_cowblocks_tag(struct xfs_inode *ip); + +void xfs_blockgc_worker(struct work_struct *work); +void xfs_blockgc_stop(struct xfs_mount *mp); +void xfs_blockgc_start(struct xfs_mount *mp); + +void xfs_inodegc_worker(struct work_struct *work); +void xfs_inodegc_push(struct xfs_mount *mp); +int xfs_inodegc_flush(struct xfs_mount *mp); +void xfs_inodegc_stop(struct xfs_mount *mp); +void xfs_inodegc_start(struct xfs_mount *mp); +int xfs_inodegc_register_shrinker(struct xfs_mount *mp); + +#endif diff --git a/fs/xfs/xfs_icreate_item.c b/fs/xfs/xfs_icreate_item.c new file mode 100644 index 000000000..95b0eba24 --- /dev/null +++ b/fs/xfs/xfs_icreate_item.c @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2008-2010, 2013 Dave Chinner + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_trans_priv.h" +#include "xfs_icreate_item.h" +#include "xfs_log.h" +#include "xfs_log_priv.h" +#include "xfs_log_recover.h" +#include "xfs_ialloc.h" +#include "xfs_trace.h" + +struct kmem_cache *xfs_icreate_cache; /* inode create item */ + +static inline struct xfs_icreate_item *ICR_ITEM(struct xfs_log_item *lip) +{ + return container_of(lip, struct xfs_icreate_item, ic_item); +} + +/* + * This returns the number of iovecs needed to log the given inode item. + * + * We only need one iovec for the icreate log structure. + */ +STATIC void +xfs_icreate_item_size( + struct xfs_log_item *lip, + int *nvecs, + int *nbytes) +{ + *nvecs += 1; + *nbytes += sizeof(struct xfs_icreate_log); +} + +/* + * This is called to fill in the vector of log iovecs for the + * given inode create log item. + */ +STATIC void +xfs_icreate_item_format( + struct xfs_log_item *lip, + struct xlog_format_buf *lfb) +{ + struct xfs_icreate_item *icp = ICR_ITEM(lip); + + xlog_format_copy(lfb, XLOG_REG_TYPE_ICREATE, &icp->ic_format, + sizeof(struct xfs_icreate_log)); +} + +STATIC void +xfs_icreate_item_release( + struct xfs_log_item *lip) +{ + kvfree(ICR_ITEM(lip)->ic_item.li_lv_shadow); + kmem_cache_free(xfs_icreate_cache, ICR_ITEM(lip)); +} + +static const struct xfs_item_ops xfs_icreate_item_ops = { + .flags = XFS_ITEM_RELEASE_WHEN_COMMITTED, + .iop_size = xfs_icreate_item_size, + .iop_format = xfs_icreate_item_format, + .iop_release = xfs_icreate_item_release, +}; + + +/* + * Initialize the inode log item for a newly allocated (in-core) inode. + * + * Inode extents can only reside within an AG. Hence specify the starting + * block for the inode chunk by offset within an AG as well as the + * length of the allocated extent. + * + * This joins the item to the transaction and marks it dirty so + * that we don't need a separate call to do this, nor does the + * caller need to know anything about the icreate item. + */ +void +xfs_icreate_log( + struct xfs_trans *tp, + xfs_agnumber_t agno, + xfs_agblock_t agbno, + unsigned int count, + unsigned int inode_size, + xfs_agblock_t length, + unsigned int generation) +{ + struct xfs_icreate_item *icp; + + icp = kmem_cache_zalloc(xfs_icreate_cache, GFP_KERNEL | __GFP_NOFAIL); + + xfs_log_item_init(tp->t_mountp, &icp->ic_item, XFS_LI_ICREATE, + &xfs_icreate_item_ops); + + icp->ic_format.icl_type = XFS_LI_ICREATE; + icp->ic_format.icl_size = 1; /* single vector */ + icp->ic_format.icl_ag = cpu_to_be32(agno); + icp->ic_format.icl_agbno = cpu_to_be32(agbno); + icp->ic_format.icl_count = cpu_to_be32(count); + icp->ic_format.icl_isize = cpu_to_be32(inode_size); + icp->ic_format.icl_length = cpu_to_be32(length); + icp->ic_format.icl_gen = cpu_to_be32(generation); + + xfs_trans_add_item(tp, &icp->ic_item); + tp->t_flags |= XFS_TRANS_DIRTY; + set_bit(XFS_LI_DIRTY, &icp->ic_item.li_flags); +} + +static enum xlog_recover_reorder +xlog_recover_icreate_reorder( + struct xlog_recover_item *item) +{ + /* + * Inode allocation buffers must be replayed before subsequent inode + * items try to modify those buffers. ICREATE items are the logical + * equivalent of logging a newly initialized inode buffer, so recover + * these at the same time that we recover logged buffers. + */ + return XLOG_REORDER_BUFFER_LIST; +} + +/* + * This routine is called when an inode create format structure is found in a + * committed transaction in the log. It's purpose is to initialise the inodes + * being allocated on disk. This requires us to get inode cluster buffers that + * match the range to be initialised, stamped with inode templates and written + * by delayed write so that subsequent modifications will hit the cached buffer + * and only need writing out at the end of recovery. + */ +STATIC int +xlog_recover_icreate_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_mount *mp = log->l_mp; + struct xfs_icreate_log *icl; + struct xfs_ino_geometry *igeo = M_IGEO(mp); + xfs_agnumber_t agno; + xfs_agblock_t agbno; + unsigned int count; + unsigned int isize; + xfs_agblock_t length; + int bb_per_cluster; + int cancel_count; + int nbufs; + int i; + + icl = (struct xfs_icreate_log *)item->ri_buf[0].iov_base; + if (icl->icl_type != XFS_LI_ICREATE) { + xfs_warn(log->l_mp, "xlog_recover_do_icreate_trans: bad type"); + return -EINVAL; + } + + if (icl->icl_size != 1) { + xfs_warn(log->l_mp, "xlog_recover_do_icreate_trans: bad icl size"); + return -EINVAL; + } + + agno = be32_to_cpu(icl->icl_ag); + if (agno >= mp->m_sb.sb_agcount) { + xfs_warn(log->l_mp, "xlog_recover_do_icreate_trans: bad agno"); + return -EINVAL; + } + agbno = be32_to_cpu(icl->icl_agbno); + if (!agbno || agbno == NULLAGBLOCK || agbno >= mp->m_sb.sb_agblocks) { + xfs_warn(log->l_mp, "xlog_recover_do_icreate_trans: bad agbno"); + return -EINVAL; + } + isize = be32_to_cpu(icl->icl_isize); + if (isize != mp->m_sb.sb_inodesize) { + xfs_warn(log->l_mp, "xlog_recover_do_icreate_trans: bad isize"); + return -EINVAL; + } + count = be32_to_cpu(icl->icl_count); + if (!count) { + xfs_warn(log->l_mp, "xlog_recover_do_icreate_trans: bad count"); + return -EINVAL; + } + length = be32_to_cpu(icl->icl_length); + if (!length || length >= mp->m_sb.sb_agblocks) { + xfs_warn(log->l_mp, "xlog_recover_do_icreate_trans: bad length"); + return -EINVAL; + } + + /* + * The inode chunk is either full or sparse and we only support + * m_ino_geo.ialloc_min_blks sized sparse allocations at this time. + */ + if (length != igeo->ialloc_blks && + length != igeo->ialloc_min_blks) { + xfs_warn(log->l_mp, + "%s: unsupported chunk length", __func__); + return -EINVAL; + } + + /* verify inode count is consistent with extent length */ + if ((count >> mp->m_sb.sb_inopblog) != length) { + xfs_warn(log->l_mp, + "%s: inconsistent inode count and chunk length", + __func__); + return -EINVAL; + } + + /* + * The icreate transaction can cover multiple cluster buffers and these + * buffers could have been freed and reused. Check the individual + * buffers for cancellation so we don't overwrite anything written after + * a cancellation. + */ + bb_per_cluster = XFS_FSB_TO_BB(mp, igeo->blocks_per_cluster); + nbufs = length / igeo->blocks_per_cluster; + for (i = 0, cancel_count = 0; i < nbufs; i++) { + xfs_daddr_t daddr; + + daddr = XFS_AGB_TO_DADDR(mp, agno, + agbno + i * igeo->blocks_per_cluster); + if (xlog_is_buffer_cancelled(log, daddr, bb_per_cluster)) + cancel_count++; + } + + /* + * We currently only use icreate for a single allocation at a time. This + * means we should expect either all or none of the buffers to be + * cancelled. Be conservative and skip replay if at least one buffer is + * cancelled, but warn the user that something is awry if the buffers + * are not consistent. + * + * XXX: This must be refined to only skip cancelled clusters once we use + * icreate for multiple chunk allocations. + */ + ASSERT(!cancel_count || cancel_count == nbufs); + if (cancel_count) { + if (cancel_count != nbufs) + xfs_warn(mp, + "WARNING: partial inode chunk cancellation, skipped icreate."); + trace_xfs_log_recover_icreate_cancel(log, icl); + return 0; + } + + trace_xfs_log_recover_icreate_recover(log, icl); + return xfs_ialloc_inode_init(mp, NULL, buffer_list, count, agno, agbno, + length, be32_to_cpu(icl->icl_gen)); +} + +const struct xlog_recover_item_ops xlog_icreate_item_ops = { + .item_type = XFS_LI_ICREATE, + .reorder = xlog_recover_icreate_reorder, + .commit_pass2 = xlog_recover_icreate_commit_pass2, +}; diff --git a/fs/xfs/xfs_icreate_item.h b/fs/xfs/xfs_icreate_item.h new file mode 100644 index 000000000..649928231 --- /dev/null +++ b/fs/xfs/xfs_icreate_item.h @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2008-2010, Dave Chinner + * All Rights Reserved. + */ +#ifndef XFS_ICREATE_ITEM_H +#define XFS_ICREATE_ITEM_H 1 + +/* in memory log item structure */ +struct xfs_icreate_item { + struct xfs_log_item ic_item; + struct xfs_icreate_log ic_format; +}; + +extern struct kmem_cache *xfs_icreate_cache; /* inode create item */ + +void xfs_icreate_log(struct xfs_trans *tp, xfs_agnumber_t agno, + xfs_agblock_t agbno, unsigned int count, + unsigned int inode_size, xfs_agblock_t length, + unsigned int generation); + +#endif /* XFS_ICREATE_ITEM_H */ diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c new file mode 100644 index 000000000..030a7c8f2 --- /dev/null +++ b/fs/xfs/xfs_inode.c @@ -0,0 +1,3077 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2006 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include + +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_inode.h" +#include "xfs_dir2.h" +#include "xfs_attr.h" +#include "xfs_bit.h" +#include "xfs_trans_space.h" +#include "xfs_trans.h" +#include "xfs_buf_item.h" +#include "xfs_inode_item.h" +#include "xfs_iunlink_item.h" +#include "xfs_ialloc.h" +#include "xfs_bmap.h" +#include "xfs_bmap_util.h" +#include "xfs_errortag.h" +#include "xfs_error.h" +#include "xfs_quota.h" +#include "xfs_filestream.h" +#include "xfs_trace.h" +#include "xfs_icache.h" +#include "xfs_symlink.h" +#include "xfs_trans_priv.h" +#include "xfs_log.h" +#include "xfs_bmap_btree.h" +#include "xfs_reflink.h" +#include "xfs_ag.h" +#include "xfs_log_priv.h" +#include "xfs_health.h" +#include "xfs_pnfs.h" +#include "xfs_parent.h" +#include "xfs_xattr.h" +#include "xfs_inode_util.h" +#include "xfs_metafile.h" + +struct kmem_cache *xfs_inode_cache; + +/* + * These two are wrapper routines around the xfs_ilock() routine used to + * centralize some grungy code. They are used in places that wish to lock the + * inode solely for reading the extents. The reason these places can't just + * call xfs_ilock(ip, XFS_ILOCK_SHARED) is that the inode lock also guards to + * bringing in of the extents from disk for a file in b-tree format. If the + * inode is in b-tree format, then we need to lock the inode exclusively until + * the extents are read in. Locking it exclusively all the time would limit + * our parallelism unnecessarily, though. What we do instead is check to see + * if the extents have been read in yet, and only lock the inode exclusively + * if they have not. + * + * The functions return a value which should be given to the corresponding + * xfs_iunlock() call. + */ +uint +xfs_ilock_data_map_shared( + struct xfs_inode *ip) +{ + uint lock_mode = XFS_ILOCK_SHARED; + + if (xfs_need_iread_extents(&ip->i_df)) + lock_mode = XFS_ILOCK_EXCL; + xfs_ilock(ip, lock_mode); + return lock_mode; +} + +uint +xfs_ilock_attr_map_shared( + struct xfs_inode *ip) +{ + uint lock_mode = XFS_ILOCK_SHARED; + + if (xfs_inode_has_attr_fork(ip) && xfs_need_iread_extents(&ip->i_af)) + lock_mode = XFS_ILOCK_EXCL; + xfs_ilock(ip, lock_mode); + return lock_mode; +} + +/* + * You can't set both SHARED and EXCL for the same lock, + * and only XFS_IOLOCK_SHARED, XFS_IOLOCK_EXCL, XFS_MMAPLOCK_SHARED, + * XFS_MMAPLOCK_EXCL, XFS_ILOCK_SHARED, XFS_ILOCK_EXCL are valid values + * to set in lock_flags. + */ +static inline void +xfs_lock_flags_assert( + uint lock_flags) +{ + ASSERT((lock_flags & (XFS_IOLOCK_SHARED | XFS_IOLOCK_EXCL)) != + (XFS_IOLOCK_SHARED | XFS_IOLOCK_EXCL)); + ASSERT((lock_flags & (XFS_MMAPLOCK_SHARED | XFS_MMAPLOCK_EXCL)) != + (XFS_MMAPLOCK_SHARED | XFS_MMAPLOCK_EXCL)); + ASSERT((lock_flags & (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)) != + (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)); + ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_SUBCLASS_MASK)) == 0); + ASSERT(lock_flags != 0); +} + +/* + * In addition to i_rwsem in the VFS inode, the xfs inode contains 2 + * multi-reader locks: invalidate_lock and the i_lock. This routine allows + * various combinations of the locks to be obtained. + * + * The 3 locks should always be ordered so that the IO lock is obtained first, + * the mmap lock second and the ilock last in order to prevent deadlock. + * + * Basic locking order: + * + * i_rwsem -> invalidate_lock -> page_lock -> i_ilock + * + * mmap_lock locking order: + * + * i_rwsem -> page lock -> mmap_lock + * mmap_lock -> invalidate_lock -> page_lock + * + * The difference in mmap_lock locking order mean that we cannot hold the + * invalidate_lock over syscall based read(2)/write(2) based IO. These IO paths + * can fault in pages during copy in/out (for buffered IO) or require the + * mmap_lock in get_user_pages() to map the user pages into the kernel address + * space for direct IO. Similarly the i_rwsem cannot be taken inside a page + * fault because page faults already hold the mmap_lock. + * + * Hence to serialise fully against both syscall and mmap based IO, we need to + * take both the i_rwsem and the invalidate_lock. These locks should *only* be + * both taken in places where we need to invalidate the page cache in a race + * free manner (e.g. truncate, hole punch and other extent manipulation + * functions). + */ +void +xfs_ilock( + xfs_inode_t *ip, + uint lock_flags) +{ + trace_xfs_ilock(ip, lock_flags, _RET_IP_); + + xfs_lock_flags_assert(lock_flags); + + if (lock_flags & XFS_IOLOCK_EXCL) { + down_write_nested(&VFS_I(ip)->i_rwsem, + XFS_IOLOCK_DEP(lock_flags)); + } else if (lock_flags & XFS_IOLOCK_SHARED) { + down_read_nested(&VFS_I(ip)->i_rwsem, + XFS_IOLOCK_DEP(lock_flags)); + } + + if (lock_flags & XFS_MMAPLOCK_EXCL) { + down_write_nested(&VFS_I(ip)->i_mapping->invalidate_lock, + XFS_MMAPLOCK_DEP(lock_flags)); + } else if (lock_flags & XFS_MMAPLOCK_SHARED) { + down_read_nested(&VFS_I(ip)->i_mapping->invalidate_lock, + XFS_MMAPLOCK_DEP(lock_flags)); + } + + if (lock_flags & XFS_ILOCK_EXCL) + down_write_nested(&ip->i_lock, XFS_ILOCK_DEP(lock_flags)); + else if (lock_flags & XFS_ILOCK_SHARED) + down_read_nested(&ip->i_lock, XFS_ILOCK_DEP(lock_flags)); +} + +/* + * This is just like xfs_ilock(), except that the caller + * is guaranteed not to sleep. It returns 1 if it gets + * the requested locks and 0 otherwise. If the IO lock is + * obtained but the inode lock cannot be, then the IO lock + * is dropped before returning. + * + * ip -- the inode being locked + * lock_flags -- this parameter indicates the inode's locks to be + * to be locked. See the comment for xfs_ilock() for a list + * of valid values. + */ +int +xfs_ilock_nowait( + xfs_inode_t *ip, + uint lock_flags) +{ + trace_xfs_ilock_nowait(ip, lock_flags, _RET_IP_); + + xfs_lock_flags_assert(lock_flags); + + if (lock_flags & XFS_IOLOCK_EXCL) { + if (!down_write_trylock(&VFS_I(ip)->i_rwsem)) + goto out; + } else if (lock_flags & XFS_IOLOCK_SHARED) { + if (!down_read_trylock(&VFS_I(ip)->i_rwsem)) + goto out; + } + + if (lock_flags & XFS_MMAPLOCK_EXCL) { + if (!down_write_trylock(&VFS_I(ip)->i_mapping->invalidate_lock)) + goto out_undo_iolock; + } else if (lock_flags & XFS_MMAPLOCK_SHARED) { + if (!down_read_trylock(&VFS_I(ip)->i_mapping->invalidate_lock)) + goto out_undo_iolock; + } + + if (lock_flags & XFS_ILOCK_EXCL) { + if (!down_write_trylock(&ip->i_lock)) + goto out_undo_mmaplock; + } else if (lock_flags & XFS_ILOCK_SHARED) { + if (!down_read_trylock(&ip->i_lock)) + goto out_undo_mmaplock; + } + return 1; + +out_undo_mmaplock: + if (lock_flags & XFS_MMAPLOCK_EXCL) + up_write(&VFS_I(ip)->i_mapping->invalidate_lock); + else if (lock_flags & XFS_MMAPLOCK_SHARED) + up_read(&VFS_I(ip)->i_mapping->invalidate_lock); +out_undo_iolock: + if (lock_flags & XFS_IOLOCK_EXCL) + up_write(&VFS_I(ip)->i_rwsem); + else if (lock_flags & XFS_IOLOCK_SHARED) + up_read(&VFS_I(ip)->i_rwsem); +out: + return 0; +} + +/* + * xfs_iunlock() is used to drop the inode locks acquired with + * xfs_ilock() and xfs_ilock_nowait(). The caller must pass + * in the flags given to xfs_ilock() or xfs_ilock_nowait() so + * that we know which locks to drop. + * + * ip -- the inode being unlocked + * lock_flags -- this parameter indicates the inode's locks to be + * to be unlocked. See the comment for xfs_ilock() for a list + * of valid values for this parameter. + * + */ +void +xfs_iunlock( + xfs_inode_t *ip, + uint lock_flags) +{ + xfs_lock_flags_assert(lock_flags); + + if (lock_flags & XFS_IOLOCK_EXCL) + up_write(&VFS_I(ip)->i_rwsem); + else if (lock_flags & XFS_IOLOCK_SHARED) + up_read(&VFS_I(ip)->i_rwsem); + + if (lock_flags & XFS_MMAPLOCK_EXCL) + up_write(&VFS_I(ip)->i_mapping->invalidate_lock); + else if (lock_flags & XFS_MMAPLOCK_SHARED) + up_read(&VFS_I(ip)->i_mapping->invalidate_lock); + + if (lock_flags & XFS_ILOCK_EXCL) + up_write(&ip->i_lock); + else if (lock_flags & XFS_ILOCK_SHARED) + up_read(&ip->i_lock); + + trace_xfs_iunlock(ip, lock_flags, _RET_IP_); +} + +/* + * give up write locks. the i/o lock cannot be held nested + * if it is being demoted. + */ +void +xfs_ilock_demote( + xfs_inode_t *ip, + uint lock_flags) +{ + ASSERT(lock_flags & (XFS_IOLOCK_EXCL|XFS_MMAPLOCK_EXCL|XFS_ILOCK_EXCL)); + ASSERT((lock_flags & + ~(XFS_IOLOCK_EXCL|XFS_MMAPLOCK_EXCL|XFS_ILOCK_EXCL)) == 0); + + if (lock_flags & XFS_ILOCK_EXCL) + downgrade_write(&ip->i_lock); + if (lock_flags & XFS_MMAPLOCK_EXCL) + downgrade_write(&VFS_I(ip)->i_mapping->invalidate_lock); + if (lock_flags & XFS_IOLOCK_EXCL) + downgrade_write(&VFS_I(ip)->i_rwsem); + + trace_xfs_ilock_demote(ip, lock_flags, _RET_IP_); +} + +void +xfs_assert_ilocked( + struct xfs_inode *ip, + uint lock_flags) +{ + /* + * Sometimes we assert the ILOCK is held exclusively, but we're in + * a workqueue, so lockdep doesn't know we're the owner. + */ + if (lock_flags & XFS_ILOCK_SHARED) + rwsem_assert_held(&ip->i_lock); + else if (lock_flags & XFS_ILOCK_EXCL) + rwsem_assert_held_write_nolockdep(&ip->i_lock); + + if (lock_flags & XFS_MMAPLOCK_SHARED) + rwsem_assert_held(&VFS_I(ip)->i_mapping->invalidate_lock); + else if (lock_flags & XFS_MMAPLOCK_EXCL) + rwsem_assert_held_write(&VFS_I(ip)->i_mapping->invalidate_lock); + + if (lock_flags & XFS_IOLOCK_SHARED) + rwsem_assert_held(&VFS_I(ip)->i_rwsem); + else if (lock_flags & XFS_IOLOCK_EXCL) + rwsem_assert_held_write(&VFS_I(ip)->i_rwsem); +} + +/* + * xfs_lockdep_subclass_ok() is only used in an ASSERT, so is only called when + * DEBUG or XFS_WARN is set. And MAX_LOCKDEP_SUBCLASSES is then only defined + * when CONFIG_LOCKDEP is set. Hence the complex define below to avoid build + * errors and warnings. + */ +#if (defined(DEBUG) || defined(XFS_WARN)) && defined(CONFIG_LOCKDEP) +static bool +xfs_lockdep_subclass_ok( + int subclass) +{ + return subclass < MAX_LOCKDEP_SUBCLASSES; +} +#else +#define xfs_lockdep_subclass_ok(subclass) (true) +#endif + +/* + * Bump the subclass so xfs_lock_inodes() acquires each lock with a different + * value. This can be called for any type of inode lock combination, including + * parent locking. Care must be taken to ensure we don't overrun the subclass + * storage fields in the class mask we build. + */ +static inline uint +xfs_lock_inumorder( + uint lock_mode, + uint subclass) +{ + uint class = 0; + + ASSERT(!(lock_mode & XFS_ILOCK_PARENT)); + ASSERT(xfs_lockdep_subclass_ok(subclass)); + + if (lock_mode & (XFS_IOLOCK_SHARED|XFS_IOLOCK_EXCL)) { + ASSERT(subclass <= XFS_IOLOCK_MAX_SUBCLASS); + class += subclass << XFS_IOLOCK_SHIFT; + } + + if (lock_mode & (XFS_MMAPLOCK_SHARED|XFS_MMAPLOCK_EXCL)) { + ASSERT(subclass <= XFS_MMAPLOCK_MAX_SUBCLASS); + class += subclass << XFS_MMAPLOCK_SHIFT; + } + + if (lock_mode & (XFS_ILOCK_SHARED|XFS_ILOCK_EXCL)) { + ASSERT(subclass <= XFS_ILOCK_MAX_SUBCLASS); + class += subclass << XFS_ILOCK_SHIFT; + } + + return (lock_mode & ~XFS_LOCK_SUBCLASS_MASK) | class; +} + +/* + * The following routine will lock n inodes in exclusive mode. We assume the + * caller calls us with the inodes in i_ino order. + * + * We need to detect deadlock where an inode that we lock is in the AIL and we + * start waiting for another inode that is locked by a thread in a long running + * transaction (such as truncate). This can result in deadlock since the long + * running trans might need to wait for the inode we just locked in order to + * push the tail and free space in the log. + * + * xfs_lock_inodes() can only be used to lock one type of lock at a time - + * the iolock, the mmaplock or the ilock, but not more than one at a time. If we + * lock more than one at a time, lockdep will report false positives saying we + * have violated locking orders. + */ +void +xfs_lock_inodes( + struct xfs_inode **ips, + int inodes, + uint lock_mode) +{ + int attempts = 0; + uint i; + int j; + bool try_lock; + struct xfs_log_item *lp; + + /* + * Currently supports between 2 and 5 inodes with exclusive locking. We + * support an arbitrary depth of locking here, but absolute limits on + * inodes depend on the type of locking and the limits placed by + * lockdep annotations in xfs_lock_inumorder. These are all checked by + * the asserts. + */ + ASSERT(ips && inodes >= 2 && inodes <= 5); + ASSERT(lock_mode & (XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL | + XFS_ILOCK_EXCL)); + ASSERT(!(lock_mode & (XFS_IOLOCK_SHARED | XFS_MMAPLOCK_SHARED | + XFS_ILOCK_SHARED))); + ASSERT(!(lock_mode & XFS_MMAPLOCK_EXCL) || + inodes <= XFS_MMAPLOCK_MAX_SUBCLASS + 1); + ASSERT(!(lock_mode & XFS_ILOCK_EXCL) || + inodes <= XFS_ILOCK_MAX_SUBCLASS + 1); + + if (lock_mode & XFS_IOLOCK_EXCL) { + ASSERT(!(lock_mode & (XFS_MMAPLOCK_EXCL | XFS_ILOCK_EXCL))); + } else if (lock_mode & XFS_MMAPLOCK_EXCL) + ASSERT(!(lock_mode & XFS_ILOCK_EXCL)); + +again: + try_lock = false; + i = 0; + for (; i < inodes; i++) { + ASSERT(ips[i]); + + if (i && (ips[i] == ips[i - 1])) /* Already locked */ + continue; + + /* + * If try_lock is not set yet, make sure all locked inodes are + * not in the AIL. If any are, set try_lock to be used later. + */ + if (!try_lock) { + for (j = (i - 1); j >= 0 && !try_lock; j--) { + lp = &ips[j]->i_itemp->ili_item; + if (lp && test_bit(XFS_LI_IN_AIL, &lp->li_flags)) + try_lock = true; + } + } + + /* + * If any of the previous locks we have locked is in the AIL, + * we must TRY to get the second and subsequent locks. If + * we can't get any, we must release all we have + * and try again. + */ + if (!try_lock) { + xfs_ilock(ips[i], xfs_lock_inumorder(lock_mode, i)); + continue; + } + + /* try_lock means we have an inode locked that is in the AIL. */ + ASSERT(i != 0); + if (xfs_ilock_nowait(ips[i], xfs_lock_inumorder(lock_mode, i))) + continue; + + /* + * Unlock all previous guys and try again. xfs_iunlock will try + * to push the tail if the inode is in the AIL. + */ + attempts++; + for (j = i - 1; j >= 0; j--) { + /* + * Check to see if we've already unlocked this one. Not + * the first one going back, and the inode ptr is the + * same. + */ + if (j != (i - 1) && ips[j] == ips[j + 1]) + continue; + + xfs_iunlock(ips[j], lock_mode); + } + + if ((attempts % 5) == 0) { + delay(1); /* Don't just spin the CPU */ + } + goto again; + } +} + +/* + * xfs_lock_two_inodes() can only be used to lock ilock. The iolock and + * mmaplock must be double-locked separately since we use i_rwsem and + * invalidate_lock for that. We now support taking one lock EXCL and the + * other SHARED. + */ +void +xfs_lock_two_inodes( + struct xfs_inode *ip0, + uint ip0_mode, + struct xfs_inode *ip1, + uint ip1_mode) +{ + int attempts = 0; + struct xfs_log_item *lp; + + ASSERT(hweight32(ip0_mode) == 1); + ASSERT(hweight32(ip1_mode) == 1); + ASSERT(!(ip0_mode & (XFS_IOLOCK_SHARED|XFS_IOLOCK_EXCL))); + ASSERT(!(ip1_mode & (XFS_IOLOCK_SHARED|XFS_IOLOCK_EXCL))); + ASSERT(!(ip0_mode & (XFS_MMAPLOCK_SHARED|XFS_MMAPLOCK_EXCL))); + ASSERT(!(ip1_mode & (XFS_MMAPLOCK_SHARED|XFS_MMAPLOCK_EXCL))); + ASSERT(I_INO(ip0) != I_INO(ip1)); + + if (I_INO(ip0) > I_INO(ip1)) { + swap(ip0, ip1); + swap(ip0_mode, ip1_mode); + } + + again: + xfs_ilock(ip0, xfs_lock_inumorder(ip0_mode, 0)); + + /* + * If the first lock we have locked is in the AIL, we must TRY to get + * the second lock. If we can't get it, we must release the first one + * and try again. + */ + lp = &ip0->i_itemp->ili_item; + if (lp && test_bit(XFS_LI_IN_AIL, &lp->li_flags)) { + if (!xfs_ilock_nowait(ip1, xfs_lock_inumorder(ip1_mode, 1))) { + xfs_iunlock(ip0, ip0_mode); + if ((++attempts % 5) == 0) + delay(1); /* Don't just spin the CPU */ + goto again; + } + } else { + xfs_ilock(ip1, xfs_lock_inumorder(ip1_mode, 1)); + } +} + +/* + * Lookups up an inode from "name". If ci_name is not NULL, then a CI match + * is allowed, otherwise it has to be an exact match. If a CI match is found, + * ci_name->name will point to a the actual name (caller must free) or + * will be set to NULL if an exact match is found. + */ +int +xfs_lookup( + struct xfs_inode *dp, + const struct xfs_name *name, + struct xfs_inode **ipp, + struct xfs_name *ci_name) +{ + xfs_ino_t inum; + int error; + + trace_xfs_lookup(dp, name); + + if (xfs_is_shutdown(dp->i_mount)) + return -EIO; + if (xfs_ifork_zapped(dp, XFS_DATA_FORK)) + return -EIO; + + error = xfs_dir_lookup(NULL, dp, name, &inum, ci_name); + if (error) + goto out_unlock; + + error = xfs_iget(dp->i_mount, NULL, inum, 0, 0, ipp); + if (error) + goto out_free_name; + + /* + * Fail if a directory entry in the regular directory tree points to + * a metadata file. + */ + if (XFS_IS_CORRUPT(dp->i_mount, xfs_is_metadir_inode(*ipp))) { + xfs_fs_mark_sick(dp->i_mount, XFS_SICK_FS_METADIR); + error = -EFSCORRUPTED; + goto out_irele; + } + + return 0; + +out_irele: + xfs_irele(*ipp); +out_free_name: + if (ci_name) + kfree(ci_name->name); +out_unlock: + *ipp = NULL; + return error; +} + +/* + * Initialise a newly allocated inode and return the in-core inode to the + * caller locked exclusively. + * + * Caller is responsible for unlocking the inode manually upon return + */ +int +xfs_icreate( + struct xfs_trans *tp, + xfs_ino_t ino, + const struct xfs_icreate_args *args, + struct xfs_inode **ipp) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_inode *ip = NULL; + int error; + + /* + * Get the in-core inode with the lock held exclusively to prevent + * others from looking at until we're done. + */ + error = xfs_iget(mp, tp, ino, XFS_IGET_CREATE, XFS_ILOCK_EXCL, &ip); + if (error) + return error; + + ASSERT(ip != NULL); + xfs_trans_ijoin(tp, ip, 0); + xfs_inode_init(tp, args, ip); + + /* now that we have an i_mode we can setup the inode structure */ + xfs_setup_inode(ip); + + *ipp = ip; + return 0; +} + +/* Return dquots for the ids that will be assigned to a new file. */ +int +xfs_icreate_dqalloc( + const struct xfs_icreate_args *args, + struct xfs_dquot **udqpp, + struct xfs_dquot **gdqpp, + struct xfs_dquot **pdqpp) +{ + struct inode *dir = VFS_I(args->pip); + kuid_t uid = GLOBAL_ROOT_UID; + kgid_t gid = GLOBAL_ROOT_GID; + prid_t prid = 0; + unsigned int flags = XFS_QMOPT_QUOTALL; + + if (args->idmap) { + /* + * The uid/gid computation code must match what the VFS uses to + * assign i_[ug]id. INHERIT adjusts the gid computation for + * setgid/grpid systems. + */ + uid = mapped_fsuid(args->idmap, i_user_ns(dir)); + gid = mapped_fsgid(args->idmap, i_user_ns(dir)); + prid = xfs_get_initial_prid(args->pip); + flags |= XFS_QMOPT_INHERIT; + } + + *udqpp = *gdqpp = *pdqpp = NULL; + + return xfs_qm_vop_dqalloc(args->pip, uid, gid, prid, flags, udqpp, + gdqpp, pdqpp); +} + +int +xfs_create( + const struct xfs_icreate_args *args, + struct xfs_name *name, + struct xfs_inode **ipp) +{ + struct xfs_inode *dp = args->pip; + struct xfs_dir_update du = { + .dp = dp, + .name = name, + }; + struct xfs_mount *mp = dp->i_mount; + struct xfs_trans *tp = NULL; + struct xfs_dquot *udqp; + struct xfs_dquot *gdqp; + struct xfs_dquot *pdqp; + struct xfs_trans_res *tres; + xfs_ino_t ino; + bool unlock_dp_on_error = false; + bool is_dir = S_ISDIR(args->mode); + uint resblks; + int error; + + trace_xfs_create(dp, name); + + if (xfs_is_shutdown(mp)) + return -EIO; + if (xfs_ifork_zapped(dp, XFS_DATA_FORK)) + return -EIO; + + /* Make sure that we have allocated dquot(s) on disk. */ + error = xfs_icreate_dqalloc(args, &udqp, &gdqp, &pdqp); + if (error) + return error; + + if (is_dir) { + resblks = xfs_mkdir_space_res(mp, name->len); + tres = &M_RES(mp)->tr_mkdir; + } else { + resblks = xfs_create_space_res(mp, name->len); + tres = &M_RES(mp)->tr_create; + } + + error = xfs_parent_start(mp, &du.ppargs); + if (error) + goto out_release_dquots; + + /* + * Initially assume that the file does not exist and + * reserve the resources for that case. If that is not + * the case we'll drop the one we have and get a more + * appropriate transaction later. + */ + error = xfs_trans_alloc_icreate(mp, tres, udqp, gdqp, pdqp, resblks, + &tp); + if (error) + goto out_parent; + + xfs_ilock(dp, XFS_ILOCK_EXCL | XFS_ILOCK_PARENT); + unlock_dp_on_error = true; + + /* + * A newly created regular or special file just has one directory + * entry pointing to them, but a directory also the "." entry + * pointing to itself. + */ + error = xfs_dialloc(&tp, args, &ino); + if (!error) + error = xfs_icreate(tp, ino, args, &du.ip); + if (error) + goto out_trans_cancel; + + /* + * Now we join the directory inode to the transaction. We do not do it + * earlier because xfs_dialloc might commit the previous transaction + * (and release all the locks). An error from here on will result in + * the transaction cancel unlocking dp so don't do it explicitly in the + * error path. + */ + xfs_trans_ijoin(tp, dp, 0); + + error = xfs_dir_create_child(tp, resblks, &du); + if (error) + goto out_trans_cancel; + + /* + * If this is a synchronous mount, make sure that the + * create transaction goes to disk before returning to + * the user. + */ + if (xfs_has_wsync(mp) || xfs_has_dirsync(mp)) + xfs_trans_set_sync(tp); + + /* + * Attach the dquot(s) to the inodes and modify them incore. + * These ids of the inode couldn't have changed since the new + * inode has been locked ever since it was created. + */ + xfs_qm_vop_create_dqattach(tp, du.ip, udqp, gdqp, pdqp); + + error = xfs_trans_commit(tp); + if (error) + goto out_release_inode; + + xfs_qm_dqrele(udqp); + xfs_qm_dqrele(gdqp); + xfs_qm_dqrele(pdqp); + + *ipp = du.ip; + xfs_iunlock(du.ip, XFS_ILOCK_EXCL); + xfs_iunlock(dp, XFS_ILOCK_EXCL); + xfs_parent_finish(mp, du.ppargs); + return 0; + + out_trans_cancel: + xfs_trans_cancel(tp); + out_release_inode: + /* + * Wait until after the current transaction is aborted to finish the + * setup of the inode and release the inode. This prevents recursive + * transactions and deadlocks from xfs_inactive. + */ + if (du.ip) { + xfs_iunlock(du.ip, XFS_ILOCK_EXCL); + xfs_finish_inode_setup(du.ip); + xfs_irele(du.ip); + } + out_parent: + xfs_parent_finish(mp, du.ppargs); + out_release_dquots: + xfs_qm_dqrele(udqp); + xfs_qm_dqrele(gdqp); + xfs_qm_dqrele(pdqp); + + if (unlock_dp_on_error) + xfs_iunlock(dp, XFS_ILOCK_EXCL); + return error; +} + +int +xfs_create_tmpfile( + const struct xfs_icreate_args *args, + struct xfs_inode **ipp) +{ + struct xfs_inode *dp = args->pip; + struct xfs_mount *mp = dp->i_mount; + struct xfs_inode *ip = NULL; + struct xfs_trans *tp = NULL; + struct xfs_dquot *udqp; + struct xfs_dquot *gdqp; + struct xfs_dquot *pdqp; + struct xfs_trans_res *tres; + xfs_ino_t ino; + uint resblks; + int error; + + ASSERT(args->flags & XFS_ICREATE_TMPFILE); + + if (xfs_is_shutdown(mp)) + return -EIO; + + /* Make sure that we have allocated dquot(s) on disk. */ + error = xfs_icreate_dqalloc(args, &udqp, &gdqp, &pdqp); + if (error) + return error; + + resblks = XFS_IALLOC_SPACE_RES(mp); + tres = &M_RES(mp)->tr_create_tmpfile; + + error = xfs_trans_alloc_icreate(mp, tres, udqp, gdqp, pdqp, resblks, + &tp); + if (error) + goto out_release_dquots; + + error = xfs_dialloc(&tp, args, &ino); + if (!error) + error = xfs_icreate(tp, ino, args, &ip); + if (error) + goto out_trans_cancel; + + if (xfs_has_wsync(mp)) + xfs_trans_set_sync(tp); + + /* + * Attach the dquot(s) to the inodes and modify them incore. + * These ids of the inode couldn't have changed since the new + * inode has been locked ever since it was created. + */ + xfs_qm_vop_create_dqattach(tp, ip, udqp, gdqp, pdqp); + + error = xfs_iunlink(tp, ip); + if (error) + goto out_trans_cancel; + + error = xfs_trans_commit(tp); + if (error) + goto out_release_inode; + + xfs_qm_dqrele(udqp); + xfs_qm_dqrele(gdqp); + xfs_qm_dqrele(pdqp); + + *ipp = ip; + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return 0; + + out_trans_cancel: + xfs_trans_cancel(tp); + out_release_inode: + /* + * Wait until after the current transaction is aborted to finish the + * setup of the inode and release the inode. This prevents recursive + * transactions and deadlocks from xfs_inactive. + */ + if (ip) { + xfs_iunlock(ip, XFS_ILOCK_EXCL); + xfs_finish_inode_setup(ip); + xfs_irele(ip); + } + out_release_dquots: + xfs_qm_dqrele(udqp); + xfs_qm_dqrele(gdqp); + xfs_qm_dqrele(pdqp); + + return error; +} + +static inline int +xfs_projid_differ( + struct xfs_inode *tdp, + struct xfs_inode *sip) +{ + /* + * If we are using project inheritance, we only allow hard link/renames + * creation in our tree when the project IDs are the same; else + * the tree quota mechanism could be circumvented. + */ + if (unlikely((tdp->i_diflags & XFS_DIFLAG_PROJINHERIT) && + tdp->i_projid != sip->i_projid)) { + /* + * Project quota setup skips special files which can + * leave inodes in a PROJINHERIT directory without a + * project ID set. We need to allow links to be made + * to these "project-less" inodes because userspace + * expects them to succeed after project ID setup, + * but everything else should be rejected. + */ + if (!special_file(VFS_I(sip)->i_mode) || + sip->i_projid != 0) { + return -EXDEV; + } + } + + return 0; +} + +int +xfs_link( + struct xfs_inode *tdp, + struct xfs_inode *sip, + struct xfs_name *target_name) +{ + struct xfs_dir_update du = { + .dp = tdp, + .name = target_name, + .ip = sip, + }; + struct xfs_mount *mp = tdp->i_mount; + struct xfs_trans *tp; + int error, nospace_error = 0; + int resblks; + + trace_xfs_link(tdp, target_name); + + ASSERT(!S_ISDIR(VFS_I(sip)->i_mode)); + + if (xfs_is_shutdown(mp)) + return -EIO; + if (xfs_ifork_zapped(tdp, XFS_DATA_FORK)) + return -EIO; + + error = xfs_qm_dqattach(sip); + if (error) + goto std_return; + + error = xfs_qm_dqattach(tdp); + if (error) + goto std_return; + + error = xfs_parent_start(mp, &du.ppargs); + if (error) + goto std_return; + + resblks = xfs_link_space_res(mp, target_name->len); + error = xfs_trans_alloc_dir(tdp, &M_RES(mp)->tr_link, sip, &resblks, + &tp, &nospace_error); + if (error) + goto out_parent; + + /* + * We don't allow reservationless or quotaless hardlinking when parent + * pointers are enabled because we can't back out if the xattrs must + * grow. + */ + if (du.ppargs && nospace_error) { + error = nospace_error; + goto error_return; + } + + error = xfs_projid_differ(tdp, sip); + if (error) + goto error_return; + + error = xfs_dir_add_child(tp, resblks, &du); + if (error) + goto error_return; + + /* + * If this is a synchronous mount, make sure that the + * link transaction goes to disk before returning to + * the user. + */ + if (xfs_has_wsync(mp) || xfs_has_dirsync(mp)) + xfs_trans_set_sync(tp); + + error = xfs_trans_commit(tp); + xfs_iunlock(tdp, XFS_ILOCK_EXCL); + xfs_iunlock(sip, XFS_ILOCK_EXCL); + xfs_parent_finish(mp, du.ppargs); + return error; + + error_return: + xfs_trans_cancel(tp); + xfs_iunlock(tdp, XFS_ILOCK_EXCL); + xfs_iunlock(sip, XFS_ILOCK_EXCL); + out_parent: + xfs_parent_finish(mp, du.ppargs); + std_return: + if (error == -ENOSPC && nospace_error) + error = nospace_error; + return error; +} + +/* Clear the reflink flag and the cowblocks tag if possible. */ +static void +xfs_itruncate_clear_reflink_flags( + struct xfs_inode *ip) +{ + struct xfs_ifork *dfork; + struct xfs_ifork *cfork; + + if (!xfs_is_reflink_inode(ip)) + return; + dfork = xfs_ifork_ptr(ip, XFS_DATA_FORK); + cfork = xfs_ifork_ptr(ip, XFS_COW_FORK); + if (dfork->if_bytes == 0 && cfork->if_bytes == 0) + ip->i_diflags2 &= ~XFS_DIFLAG2_REFLINK; + if (cfork->if_bytes == 0) + xfs_inode_clear_cowblocks_tag(ip); +} + +/* + * Free up the underlying blocks past new_size. The new size must be smaller + * than the current size. This routine can be used both for the attribute and + * data fork, and does not modify the inode size, which is left to the caller. + * + * The transaction passed to this routine must have made a permanent log + * reservation of at least XFS_ITRUNCATE_LOG_RES. This routine may commit the + * given transaction and start new ones, so make sure everything involved in + * the transaction is tidy before calling here. Some transaction will be + * returned to the caller to be committed. The incoming transaction must + * already include the inode, and both inode locks must be held exclusively. + * The inode must also be "held" within the transaction. On return the inode + * will be "held" within the returned transaction. This routine does NOT + * require any disk space to be reserved for it within the transaction. + * + * If we get an error, we must return with the inode locked and linked into the + * current transaction. This keeps things simple for the higher level code, + * because it always knows that the inode is locked and held in the transaction + * that returns to it whether errors occur or not. We don't mark the inode + * dirty on error so that transactions can be easily aborted if possible. + */ +int +xfs_itruncate_extents_flags( + struct xfs_trans **tpp, + struct xfs_inode *ip, + int whichfork, + xfs_fsize_t new_size, + int flags) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_trans *tp = *tpp; + xfs_fileoff_t first_unmap_block; + int error = 0; + + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + if (icount_read_once(VFS_I(ip))) + xfs_assert_ilocked(ip, XFS_IOLOCK_EXCL); + if (whichfork == XFS_DATA_FORK) + ASSERT(new_size <= XFS_ISIZE(ip)); + ASSERT(tp->t_flags & XFS_TRANS_PERM_LOG_RES); + ASSERT(ip->i_itemp != NULL); + ASSERT(ip->i_itemp->ili_lock_flags == 0); + ASSERT(!XFS_NOT_DQATTACHED(mp, ip)); + + trace_xfs_itruncate_extents_start(ip, new_size); + + flags |= xfs_bmapi_aflag(whichfork); + + /* + * Since it is possible for space to become allocated beyond + * the end of the file (in a crash where the space is allocated + * but the inode size is not yet updated), simply remove any + * blocks which show up between the new EOF and the maximum + * possible file size. + * + * We have to free all the blocks to the bmbt maximum offset, even if + * the page cache can't scale that far. + */ + first_unmap_block = XFS_B_TO_FSB(mp, (xfs_ufsize_t)new_size); + if (!xfs_verify_fileoff(mp, first_unmap_block)) { + WARN_ON_ONCE(first_unmap_block > XFS_MAX_FILEOFF); + return 0; + } + + error = xfs_bunmapi_range(&tp, ip, flags, first_unmap_block, + XFS_MAX_FILEOFF); + if (error) + goto out; + + if (whichfork == XFS_DATA_FORK) { + /* Remove all pending CoW reservations. */ + error = xfs_reflink_cancel_cow_blocks(ip, &tp, + first_unmap_block, XFS_MAX_FILEOFF, true); + if (error) + goto out; + + xfs_itruncate_clear_reflink_flags(ip); + } + + /* + * Always re-log the inode so that our permanent transaction can keep + * on rolling it forward in the log. + */ + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + + trace_xfs_itruncate_extents_end(ip, new_size); + +out: + *tpp = tp; + return error; +} + +/* + * Mark all the buffers attached to this directory stale. In theory we should + * never be freeing a directory with any blocks at all, but this covers the + * case where we've recovered a directory swap with a "temporary" directory + * created by online repair and now need to dump it. + */ +STATIC void +xfs_inactive_dir( + struct xfs_inode *dp) +{ + struct xfs_iext_cursor icur; + struct xfs_bmbt_irec got; + struct xfs_mount *mp = dp->i_mount; + struct xfs_da_geometry *geo = mp->m_dir_geo; + struct xfs_ifork *ifp = xfs_ifork_ptr(dp, XFS_DATA_FORK); + xfs_fileoff_t off; + + /* + * Invalidate each directory block. All directory blocks are of + * fsbcount length and alignment, so we only need to walk those same + * offsets. We hold the only reference to this inode, so we must wait + * for the buffer locks. + */ + for_each_xfs_iext(ifp, &icur, &got) { + for (off = round_up(got.br_startoff, geo->fsbcount); + off < got.br_startoff + got.br_blockcount; + off += geo->fsbcount) { + struct xfs_buf *bp = NULL; + xfs_fsblock_t fsbno; + int error; + + fsbno = (off - got.br_startoff) + got.br_startblock; + error = xfs_buf_incore(mp->m_ddev_targp, + XFS_FSB_TO_DADDR(mp, fsbno), + XFS_FSB_TO_BB(mp, geo->fsbcount), + XBF_LIVESCAN, &bp); + if (error) + continue; + + xfs_buf_stale(bp); + xfs_buf_relse(bp); + } + } +} + +/* + * xfs_inactive_truncate + * + * Called to perform a truncate when an inode becomes unlinked. + */ +STATIC int +xfs_inactive_truncate( + struct xfs_inode *ip) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_trans *tp; + int error; + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_itruncate, 0, 0, 0, &tp); + if (error) { + ASSERT(xfs_is_shutdown(mp)); + return error; + } + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, 0); + + /* + * Log the inode size first to prevent stale data exposure in the event + * of a system crash before the truncate completes. See the related + * comment in xfs_vn_setattr_size() for details. + */ + ip->i_disk_size = 0; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + + error = xfs_itruncate_extents(&tp, ip, XFS_DATA_FORK, 0); + if (error) + goto error_trans_cancel; + + ASSERT(ip->i_df.if_nextents == 0); + + error = xfs_trans_commit(tp); + if (error) + goto error_unlock; + + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return 0; + +error_trans_cancel: + xfs_trans_cancel(tp); +error_unlock: + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; +} + +/* + * xfs_inactive_ifree() + * + * Perform the inode free when an inode is unlinked. + */ +STATIC int +xfs_inactive_ifree( + struct xfs_inode *ip) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_trans *tp; + int error; + + /* + * We try to use a per-AG reservation for any block needed by the finobt + * tree, but as the finobt feature predates the per-AG reservation + * support a degraded file system might not have enough space for the + * reservation at mount time. In that case try to dip into the reserved + * pool and pray. + * + * Send a warning if the reservation does happen to fail, as the inode + * now remains allocated and sits on the unlinked list until the fs is + * repaired. + */ + if (unlikely(mp->m_finobt_nores)) { + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_ifree, + XFS_IFREE_SPACE_RES(mp), 0, XFS_TRANS_RESERVE, + &tp); + } else { + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_ifree, 0, 0, 0, &tp); + } + if (error) { + if (error == -ENOSPC) { + xfs_warn_ratelimited(mp, + "Failed to remove inode(s) from unlinked list. " + "Please free space, unmount and run xfs_repair."); + } else { + ASSERT(xfs_is_shutdown(mp)); + } + return error; + } + + /* + * We do not hold the inode locked across the entire rolling transaction + * here. We only need to hold it for the first transaction that + * xfs_ifree() builds, which may mark the inode XFS_ISTALE if the + * underlying cluster buffer is freed. Relogging an XFS_ISTALE inode + * here breaks the relationship between cluster buffer invalidation and + * stale inode invalidation on cluster buffer item journal commit + * completion, and can result in leaving dirty stale inodes hanging + * around in memory. + * + * We have no need for serialising this inode operation against other + * operations - we freed the inode and hence reallocation is required + * and that will serialise on reallocating the space the deferops need + * to free. Hence we can unlock the inode on the first commit of + * the transaction rather than roll it right through the deferops. This + * avoids relogging the XFS_ISTALE inode. + * + * We check that xfs_ifree() hasn't grown an internal transaction roll + * by asserting that the inode is still locked when it returns. + */ + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL); + + error = xfs_ifree(tp, ip); + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + if (error) { + /* + * If we fail to free the inode, shut down. The cancel + * might do that, we need to make sure. Otherwise the + * inode might be lost for a long time or forever. + */ + if (!xfs_is_shutdown(mp)) { + xfs_notice(mp, "%s: xfs_ifree returned error %d", + __func__, error); + xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR); + } + xfs_trans_cancel(tp); + return error; + } + + /* + * Credit the quota account(s). The inode is gone. + */ + xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_ICOUNT, -1); + + return xfs_trans_commit(tp); +} + +/* + * Returns true if we need to update the on-disk metadata before we can free + * the memory used by this inode. Updates include freeing post-eof + * preallocations; freeing COW staging extents; and marking the inode free in + * the inobt if it is on the unlinked list. + */ +bool +xfs_inode_needs_inactive( + struct xfs_inode *ip) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_ifork *cow_ifp = xfs_ifork_ptr(ip, XFS_COW_FORK); + + /* + * If the inode is already free, then there can be nothing + * to clean up here. + */ + if (VFS_I(ip)->i_mode == 0) + return false; + + /* + * If this is a read-only mount, don't do this (would generate I/O) + * unless we're in log recovery and cleaning the iunlinked list. + */ + if (xfs_is_readonly(mp) && !xlog_recovery_needed(mp->m_log)) + return false; + + /* If the log isn't running, push inodes straight to reclaim. */ + if (xfs_is_shutdown(mp) || xfs_has_norecovery(mp)) + return false; + + /* Metadata inodes require explicit resource cleanup. */ + if (xfs_is_internal_inode(ip)) + return false; + + /* Want to clean out the cow blocks if there are any. */ + if (cow_ifp && cow_ifp->if_bytes > 0) + return true; + + /* Unlinked files must be freed. */ + if (VFS_I(ip)->i_nlink == 0) + return true; + + /* + * This file isn't being freed, so check if there are post-eof blocks + * to free. + * + * Note: don't bother with iolock here since lockdep complains about + * acquiring it in reclaim context. We have the only reference to the + * inode at this point anyways. + */ + return xfs_can_free_eofblocks(ip); +} + +/* + * Save health status somewhere, if we're dumping an inode with uncorrected + * errors and online repair isn't running. + */ +static inline void +xfs_inactive_health( + struct xfs_inode *ip) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_perag *pag; + unsigned int sick; + unsigned int checked; + + xfs_inode_measure_sickness(ip, &sick, &checked); + if (!sick) + return; + + trace_xfs_inode_unfixed_corruption(ip, sick); + + if (sick & XFS_SICK_INO_FORGET) + return; + + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(ip)); + if (!pag) { + /* There had better still be a perag structure! */ + ASSERT(0); + return; + } + + xfs_ag_mark_sick(pag, XFS_SICK_AG_INODES); + xfs_perag_put(pag); +} + +/* + * xfs_inactive + * + * This is called when the vnode reference count for the vnode + * goes to zero. If the file has been unlinked, then it must + * now be truncated. Also, we clear all of the read-ahead state + * kept for the inode here since the file is now closed. + */ +int +xfs_inactive( + xfs_inode_t *ip) +{ + struct xfs_mount *mp = ip->i_mount; + int error = 0; + int truncate = 0; + + /* + * If the inode is already free, then there can be nothing + * to clean up here. + */ + if (VFS_I(ip)->i_mode == 0) { + ASSERT(ip->i_df.if_broot_bytes == 0); + goto out; + } + + /* + * If the filesystem has been shut down - for example a mount that + * failed after background inactivation was enabled - do not + * inactivate the inode. Inactivation modifies persistent metadata, + * its transactions cannot complete on a shut down mount, and the + * subsystems it relies on (e.g. quota, mp->m_quotainfo) may not be + * set up. The attached dquots are dropped at the out: label and the + * inode then goes straight to reclaim, the same way + * xfs_inode_needs_inactive() already declines to inactivate on a shut + * down mount at queue time. + */ + if (xfs_is_shutdown(mp)) + goto out; + + ASSERT(!xfs_iflags_test(ip, XFS_IRECOVERY)); + + xfs_inactive_health(ip); + + /* + * If this is a read-only mount, don't do this (would generate I/O) + * unless we're in log recovery and cleaning the iunlinked list. + */ + if (xfs_is_readonly(mp) && !xlog_recovery_needed(mp->m_log)) + goto out; + + /* Metadata inodes require explicit resource cleanup. */ + if (xfs_is_internal_inode(ip)) + goto out; + + /* Try to clean out the cow blocks if there are any. */ + if (xfs_inode_has_cow_data(ip)) { + error = xfs_reflink_cancel_cow_range(ip, 0, NULLFILEOFF, true); + if (error) + goto out; + } + + if (VFS_I(ip)->i_nlink != 0) { + /* + * Note: don't bother with iolock here since lockdep complains + * about acquiring it in reclaim context. We have the only + * reference to the inode at this point anyways. + */ + if (xfs_can_free_eofblocks(ip)) + error = xfs_free_eofblocks(ip); + + goto out; + } + + if (S_ISREG(VFS_I(ip)->i_mode) && + (ip->i_disk_size != 0 || XFS_ISIZE(ip) != 0 || + xfs_inode_has_filedata(ip))) + truncate = 1; + + if (xfs_iflags_test(ip, XFS_IQUOTAUNCHECKED)) { + /* + * If this inode is being inactivated during a quotacheck and + * has not yet been scanned by quotacheck, we /must/ remove + * the dquots from the inode before inactivation changes the + * block and inode counts. Most probably this is a result of + * reloading the incore iunlinked list to purge unrecovered + * unlinked inodes. + */ + xfs_qm_dqdetach(ip); + } else { + error = xfs_qm_dqattach(ip); + if (error) + goto out; + } + + if (S_ISDIR(VFS_I(ip)->i_mode) && ip->i_df.if_nextents > 0) { + xfs_inactive_dir(ip); + truncate = 1; + } + + if (S_ISLNK(VFS_I(ip)->i_mode)) + error = xfs_inactive_symlink(ip); + else if (truncate) + error = xfs_inactive_truncate(ip); + if (error) + goto out; + + /* + * If there are attributes associated with the file then blow them away + * now. The code calls a routine that recursively deconstructs the + * attribute fork. If also blows away the in-core attribute fork. + */ + if (xfs_inode_has_attr_fork(ip)) { + error = xfs_attr_inactive(ip); + if (error) + goto out; + } + + ASSERT(ip->i_forkoff == 0); + + /* + * Free the inode. + */ + error = xfs_inactive_ifree(ip); + +out: + /* + * We're done making metadata updates for this inode, so we can release + * the attached dquots. + */ + xfs_qm_dqdetach(ip); + return error; +} + +/* + * Find an inode on the unlinked list. This does not take references to the + * inode as we have existence guarantees by holding the AGI buffer lock and that + * only unlinked, referenced inodes can be on the unlinked inode list. If we + * don't find the inode in cache, then let the caller handle the situation. + */ +struct xfs_inode * +xfs_iunlink_lookup( + struct xfs_perag *pag, + xfs_agino_t agino) +{ + struct xfs_inode *ip; + + rcu_read_lock(); + ip = radix_tree_lookup(&pag->pag_ici_root, agino); + if (!ip) { + /* Caller can handle inode not being in memory. */ + rcu_read_unlock(); + return NULL; + } + + /* + * Inode in RCU freeing limbo should not happen. Warn about this and + * let the caller handle the failure. + */ + if (WARN_ON_ONCE(!I_INO(ip))) { + rcu_read_unlock(); + return NULL; + } + ASSERT(!xfs_iflags_test(ip, XFS_IRECLAIMABLE | XFS_IRECLAIM)); + rcu_read_unlock(); + return ip; +} + +/* + * Load the inode @next_agino into the cache and set its prev_unlinked pointer + * to @prev_agino. Caller must hold the AGI to synchronize with other changes + * to the unlinked list. + */ +int +xfs_iunlink_reload_next( + struct xfs_trans *tp, + struct xfs_buf *agibp, + xfs_agino_t prev_agino, + xfs_agino_t next_agino) +{ + struct xfs_perag *pag = agibp->b_pag; + struct xfs_mount *mp = pag_mount(pag); + struct xfs_inode *next_ip = NULL; + int error; + + ASSERT(next_agino != NULLAGINO); + +#ifdef DEBUG + rcu_read_lock(); + next_ip = radix_tree_lookup(&pag->pag_ici_root, next_agino); + ASSERT(next_ip == NULL); + rcu_read_unlock(); +#endif + + xfs_info_ratelimited(mp, + "Found unrecovered unlinked inode 0x%x in AG 0x%x. Initiating recovery.", + next_agino, pag_agno(pag)); + + /* + * Use an untrusted lookup just to be cautious in case the AGI has been + * corrupted and now points at a free inode. That shouldn't happen, + * but we'd rather shut down now since we're already running in a weird + * situation. + */ + error = xfs_iget(mp, tp, xfs_agino_to_ino(pag, next_agino), + XFS_IGET_UNTRUSTED, 0, &next_ip); + if (error) { + xfs_ag_mark_sick(pag, XFS_SICK_AG_AGI); + return error; + } + + /* If this is not an unlinked inode, something is very wrong. */ + if (VFS_I(next_ip)->i_nlink != 0) { + xfs_ag_mark_sick(pag, XFS_SICK_AG_AGI); + error = -EFSCORRUPTED; + goto rele; + } + + next_ip->i_prev_unlinked = prev_agino; + trace_xfs_iunlink_reload_next(next_ip); +rele: + ASSERT(!(inode_state_read_once(VFS_I(next_ip)) & I_DONTCACHE)); + if (xfs_is_quotacheck_running(mp) && next_ip) + xfs_iflags_set(next_ip, XFS_IQUOTAUNCHECKED); + xfs_irele(next_ip); + return error; +} + +/* + * Look up the inode number specified and if it is not already marked XFS_ISTALE + * mark it stale. We should only find clean inodes in this lookup that aren't + * already stale. + */ +static void +xfs_ifree_mark_inode_stale( + struct xfs_perag *pag, + struct xfs_inode *free_ip, + xfs_ino_t inum) +{ + struct xfs_mount *mp = pag_mount(pag); + struct xfs_inode_log_item *iip; + struct xfs_inode *ip; + +retry: + rcu_read_lock(); + ip = radix_tree_lookup(&pag->pag_ici_root, XFS_INO_TO_AGINO(mp, inum)); + + /* Inode not in memory, nothing to do */ + if (!ip) { + rcu_read_unlock(); + return; + } + + /* + * because this is an RCU protected lookup, we could find a recently + * freed or even reallocated inode during the lookup. We need to check + * under the i_flags_lock for a valid inode here. Skip it if it is not + * valid, the wrong inode or stale. + */ + spin_lock(&ip->i_flags_lock); + if (I_INO(ip) != inum || __xfs_iflags_test(ip, XFS_ISTALE)) + goto out_iflags_unlock; + + /* + * Don't try to lock/unlock the current inode, but we _cannot_ skip the + * other inodes that we did not find in the list attached to the buffer + * and are not already marked stale. If we can't lock it, back off and + * retry. + */ + if (ip != free_ip) { + if (!xfs_ilock_nowait(ip, XFS_ILOCK_EXCL)) { + spin_unlock(&ip->i_flags_lock); + rcu_read_unlock(); + delay(1); + goto retry; + } + } + ip->i_flags |= XFS_ISTALE; + + /* + * If the inode is flushing, it is already attached to the buffer. All + * we needed to do here is mark the inode stale so buffer IO completion + * will remove it from the AIL. + */ + iip = ip->i_itemp; + if (__xfs_iflags_test(ip, XFS_IFLUSHING)) { + ASSERT(!list_empty(&iip->ili_item.li_bio_list)); + ASSERT(iip->ili_last_fields || xlog_is_shutdown(mp->m_log)); + goto out_iunlock; + } + + /* + * Inodes not attached to the buffer can be released immediately. + * Everything else has to go through xfs_iflush_abort() on journal + * commit as the flock synchronises removal of the inode from the + * cluster buffer against inode reclaim. + */ + if (!iip || list_empty(&iip->ili_item.li_bio_list)) + goto out_iunlock; + + __xfs_iflags_set(ip, XFS_IFLUSHING); + spin_unlock(&ip->i_flags_lock); + rcu_read_unlock(); + + /* we have a dirty inode in memory that has not yet been flushed. */ + spin_lock(&iip->ili_lock); + iip->ili_last_fields = iip->ili_fields; + iip->ili_fields = 0; + spin_unlock(&iip->ili_lock); + ASSERT(iip->ili_last_fields); + + if (ip != free_ip) + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return; + +out_iunlock: + if (ip != free_ip) + xfs_iunlock(ip, XFS_ILOCK_EXCL); +out_iflags_unlock: + spin_unlock(&ip->i_flags_lock); + rcu_read_unlock(); +} + +/* + * A big issue when freeing the inode cluster is that we _cannot_ skip any + * inodes that are in memory - they all must be marked stale and attached to + * the cluster buffer. + */ +static int +xfs_ifree_cluster( + struct xfs_trans *tp, + struct xfs_perag *pag, + struct xfs_inode *free_ip, + struct xfs_icluster *xic) +{ + struct xfs_mount *mp = free_ip->i_mount; + struct xfs_ino_geometry *igeo = M_IGEO(mp); + struct xfs_buf *bp; + xfs_daddr_t blkno; + xfs_ino_t inum = xic->first_ino; + int nbufs; + int i, j; + int ioffset; + int error; + + nbufs = igeo->ialloc_blks / igeo->blocks_per_cluster; + + for (j = 0; j < nbufs; j++, inum += igeo->inodes_per_cluster) { + /* + * The allocation bitmap tells us which inodes of the chunk were + * physically allocated. Skip the cluster if an inode falls into + * a sparse region. + */ + ioffset = inum - xic->first_ino; + if ((xic->alloc & XFS_INOBT_MASK(ioffset)) == 0) { + ASSERT(ioffset % igeo->inodes_per_cluster == 0); + continue; + } + + blkno = XFS_AGB_TO_DADDR(mp, XFS_INO_TO_AGNO(mp, inum), + XFS_INO_TO_AGBNO(mp, inum)); + + /* + * We obtain and lock the backing buffer first in the process + * here to ensure dirty inodes attached to the buffer remain in + * the flushing state while we mark them stale. + * + * If we scan the in-memory inodes first, then buffer IO can + * complete before we get a lock on it, and hence we may fail + * to mark all the active inodes on the buffer stale. + */ + error = xfs_trans_get_buf(tp, mp->m_ddev_targp, blkno, + mp->m_bsize * igeo->blocks_per_cluster, 0, &bp); + if (error) + return error; + + /* + * This buffer may not have been correctly initialised as we + * didn't read it from disk. That's not important because we are + * only using to mark the buffer as stale in the log, and to + * attach stale cached inodes on it. + * + * For the inode that triggered the cluster freeing, this + * attachment may occur in xfs_inode_item_precommit() after we + * have marked this buffer stale. If this buffer was not in + * memory before xfs_ifree_cluster() started, it will not be + * marked uptodate and this will cause problems later in + * xfs_inode_item_precommit() when we trip over a (stale, !done) + * buffer to attached to the transaction. + * + * Hence we have to mark the buffer as XFS_DONE here. This is + * safe because we are also marking the buffer as XBF_STALE and + * XFS_BLI_STALE. That means it will never be dispatched for + * IO and it won't be unlocked until the cluster freeing has + * been committed to the journal and the buffer unpinned. If it + * is written, we want to know about it, and we want it to + * fail. We can acheive this by adding a write verifier to the + * buffer. + */ + xfs_buf_set_uptodate(bp); + bp->b_ops = &xfs_inode_buf_ops; + + /* + * Now we need to set all the cached clean inodes as XFS_ISTALE, + * too. This requires lookups, and will skip inodes that we've + * already marked XFS_ISTALE. + */ + for (i = 0; i < igeo->inodes_per_cluster; i++) + xfs_ifree_mark_inode_stale(pag, free_ip, inum + i); + + xfs_trans_stale_inode_buf(tp, bp); + xfs_trans_binval(tp, bp); + } + return 0; +} + +/* + * This is called to return an inode to the inode free list. The inode should + * already be truncated to 0 length and have no pages associated with it. This + * routine also assumes that the inode is already a part of the transaction. + * + * The on-disk copy of the inode will have been added to the list of unlinked + * inodes in the AGI. We need to remove the inode from that list atomically with + * respect to freeing it here. + */ +int +xfs_ifree( + struct xfs_trans *tp, + struct xfs_inode *ip) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_perag *pag; + struct xfs_icluster xic = { 0 }; + struct xfs_inode_log_item *iip = ip->i_itemp; + int error; + + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + ASSERT(VFS_I(ip)->i_nlink == 0); + ASSERT(ip->i_df.if_nextents == 0); + ASSERT(ip->i_disk_size == 0 || !S_ISREG(VFS_I(ip)->i_mode)); + ASSERT(ip->i_nblocks == 0); + + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(ip)); + + error = xfs_inode_uninit(tp, pag, ip, &xic); + if (error) + goto out; + + if (xfs_iflags_test(ip, XFS_IPRESERVE_DM_FIELDS)) + xfs_iflags_clear(ip, XFS_IPRESERVE_DM_FIELDS); + + /* Don't attempt to replay owner changes for a deleted inode */ + spin_lock(&iip->ili_lock); + iip->ili_fields &= ~(XFS_ILOG_AOWNER | XFS_ILOG_DOWNER); + spin_unlock(&iip->ili_lock); + + if (xic.deleted) + error = xfs_ifree_cluster(tp, pag, ip, &xic); +out: + xfs_perag_put(pag); + return error; +} + +/* + * This is called to unpin an inode. The caller must have the inode locked + * in at least shared mode so that the buffer cannot be subsequently pinned + * once someone is waiting for it to be unpinned. + */ +static void +xfs_iunpin( + struct xfs_inode *ip) +{ + struct xfs_inode_log_item *iip = ip->i_itemp; + xfs_csn_t seq = 0; + + trace_xfs_inode_unpin_nowait(ip, _RET_IP_); + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL | XFS_ILOCK_SHARED); + + spin_lock(&iip->ili_lock); + seq = iip->ili_commit_seq; + spin_unlock(&iip->ili_lock); + if (!seq) + return; + + /* Give the log a push to start the unpinning I/O */ + xfs_log_force_seq(ip->i_mount, seq, 0, NULL); + +} + +static void +__xfs_iunpin_wait( + struct xfs_inode *ip) +{ + wait_queue_head_t *wq = bit_waitqueue(&ip->i_flags, __XFS_IPINNED_BIT); + DEFINE_WAIT_BIT(wait, &ip->i_flags, __XFS_IPINNED_BIT); + + xfs_iunpin(ip); + + do { + prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE); + if (xfs_ipincount(ip)) + io_schedule(); + } while (xfs_ipincount(ip)); + finish_wait(wq, &wait.wq_entry); +} + +void +xfs_iunpin_wait( + struct xfs_inode *ip) +{ + if (xfs_ipincount(ip)) + __xfs_iunpin_wait(ip); +} + +/* + * Removing an inode from the namespace involves removing the directory entry + * and dropping the link count on the inode. Removing the directory entry can + * result in locking an AGF (directory blocks were freed) and removing a link + * count can result in placing the inode on an unlinked list which results in + * locking an AGI. + * + * The big problem here is that we have an ordering constraint on AGF and AGI + * locking - inode allocation locks the AGI, then can allocate a new extent for + * new inodes, locking the AGF after the AGI. Similarly, freeing the inode + * removes the inode from the unlinked list, requiring that we lock the AGI + * first, and then freeing the inode can result in an inode chunk being freed + * and hence freeing disk space requiring that we lock an AGF. + * + * Hence the ordering that is imposed by other parts of the code is AGI before + * AGF. This means we cannot remove the directory entry before we drop the inode + * reference count and put it on the unlinked list as this results in a lock + * order of AGF then AGI, and this can deadlock against inode allocation and + * freeing. Therefore we must drop the link counts before we remove the + * directory entry. + * + * This is still safe from a transactional point of view - it is not until we + * get to xfs_defer_finish() that we have the possibility of multiple + * transactions in this operation. Hence as long as we remove the directory + * entry and drop the link count in the first transaction of the remove + * operation, there are no transactional constraints on the ordering here. + */ +int +xfs_remove( + struct xfs_inode *dp, + struct xfs_name *name, + struct xfs_inode *ip) +{ + struct xfs_dir_update du = { + .dp = dp, + .name = name, + .ip = ip, + }; + struct xfs_mount *mp = dp->i_mount; + struct xfs_trans *tp = NULL; + int is_dir = S_ISDIR(VFS_I(ip)->i_mode); + int dontcare; + int error = 0; + uint resblks; + + trace_xfs_remove(dp, name); + + if (xfs_is_shutdown(mp)) + return -EIO; + if (xfs_ifork_zapped(dp, XFS_DATA_FORK)) + return -EIO; + + error = xfs_qm_dqattach(dp); + if (error) + goto std_return; + + error = xfs_qm_dqattach(ip); + if (error) + goto std_return; + + error = xfs_parent_start(mp, &du.ppargs); + if (error) + goto std_return; + + /* + * We try to get the real space reservation first, allowing for + * directory btree deletion(s) implying possible bmap insert(s). If we + * can't get the space reservation then we use 0 instead, and avoid the + * bmap btree insert(s) in the directory code by, if the bmap insert + * tries to happen, instead trimming the LAST block from the directory. + * + * Ignore EDQUOT and ENOSPC being returned via nospace_error because + * the directory code can handle a reservationless update and we don't + * want to prevent a user from trying to free space by deleting things. + */ + resblks = xfs_remove_space_res(mp, name->len); + error = xfs_trans_alloc_dir(dp, &M_RES(mp)->tr_remove, ip, &resblks, + &tp, &dontcare); + if (error) { + ASSERT(error != -ENOSPC); + goto out_parent; + } + + error = xfs_dir_remove_child(tp, resblks, &du); + if (error) + goto out_trans_cancel; + + /* + * If this is a synchronous mount, make sure that the + * remove transaction goes to disk before returning to + * the user. + */ + if (xfs_has_wsync(mp) || xfs_has_dirsync(mp)) + xfs_trans_set_sync(tp); + + error = xfs_trans_commit(tp); + if (error) + goto out_unlock; + + if (is_dir && xfs_inode_is_filestream(ip)) + xfs_filestream_deassociate(ip); + + xfs_iunlock(ip, XFS_ILOCK_EXCL); + xfs_iunlock(dp, XFS_ILOCK_EXCL); + xfs_parent_finish(mp, du.ppargs); + return 0; + + out_trans_cancel: + xfs_trans_cancel(tp); + out_unlock: + xfs_iunlock(ip, XFS_ILOCK_EXCL); + xfs_iunlock(dp, XFS_ILOCK_EXCL); + out_parent: + xfs_parent_finish(mp, du.ppargs); + std_return: + return error; +} + +static inline void +xfs_iunlock_rename( + struct xfs_inode **i_tab, + int num_inodes) +{ + int i; + + for (i = num_inodes - 1; i >= 0; i--) { + /* Skip duplicate inodes if src and target dps are the same */ + if (!i_tab[i] || (i > 0 && i_tab[i] == i_tab[i - 1])) + continue; + xfs_iunlock(i_tab[i], XFS_ILOCK_EXCL); + } +} + +/* + * Enter all inodes for a rename transaction into a sorted array. + */ +#define __XFS_SORT_INODES 5 +STATIC void +xfs_sort_for_rename( + struct xfs_inode *dp1, /* in: old (source) directory inode */ + struct xfs_inode *dp2, /* in: new (target) directory inode */ + struct xfs_inode *ip1, /* in: inode of old entry */ + struct xfs_inode *ip2, /* in: inode of new entry */ + struct xfs_inode *wip, /* in: whiteout inode */ + struct xfs_inode **i_tab,/* out: sorted array of inodes */ + int *num_inodes) /* in/out: inodes in array */ +{ + int i; + + ASSERT(*num_inodes == __XFS_SORT_INODES); + memset(i_tab, 0, *num_inodes * sizeof(struct xfs_inode *)); + + /* + * i_tab contains a list of pointers to inodes. We initialize + * the table here & we'll sort it. We will then use it to + * order the acquisition of the inode locks. + * + * Note that the table may contain duplicates. e.g., dp1 == dp2. + */ + i = 0; + i_tab[i++] = dp1; + i_tab[i++] = dp2; + i_tab[i++] = ip1; + if (ip2) + i_tab[i++] = ip2; + if (wip) + i_tab[i++] = wip; + *num_inodes = i; + + xfs_sort_inodes(i_tab, *num_inodes); +} + +void +xfs_sort_inodes( + struct xfs_inode **i_tab, + unsigned int num_inodes) +{ + int i, j; + + ASSERT(num_inodes <= __XFS_SORT_INODES); + + /* + * Sort the elements via bubble sort. (Remember, there are at + * most 5 elements to sort, so this is adequate.) + */ + for (i = 0; i < num_inodes; i++) { + for (j = 1; j < num_inodes; j++) { + if (I_INO(i_tab[j]) < I_INO(i_tab[j-1])) + swap(i_tab[j], i_tab[j - 1]); + } + } +} + +/* + * xfs_rename_alloc_whiteout() + * + * Return a referenced, unlinked, unlocked inode that can be used as a + * whiteout in a rename transaction. We use a tmpfile inode here so that if we + * crash between allocating the inode and linking it into the rename transaction + * recovery will free the inode and we won't leak it. + */ +static int +xfs_rename_alloc_whiteout( + struct mnt_idmap *idmap, + struct xfs_name *src_name, + struct xfs_inode *dp, + struct xfs_inode **wip) +{ + struct xfs_icreate_args args = { + .idmap = idmap, + .pip = dp, + .mode = S_IFCHR | WHITEOUT_MODE, + .flags = XFS_ICREATE_TMPFILE, + }; + struct xfs_inode *tmpfile; + struct qstr name; + int error; + + error = xfs_create_tmpfile(&args, &tmpfile); + if (error) + return error; + + name.name = src_name->name; + name.len = src_name->len; + error = xfs_inode_init_security(VFS_I(tmpfile), VFS_I(dp), &name); + if (error) { + xfs_finish_inode_setup(tmpfile); + xfs_irele(tmpfile); + return error; + } + + /* + * Prepare the tmpfile inode as if it were created through the VFS. + * Complete the inode setup and flag it as linkable. nlink is already + * zero, so we can skip the drop_nlink. + */ + xfs_setup_iops(tmpfile); + xfs_finish_inode_setup(tmpfile); + inode_state_set_raw(VFS_I(tmpfile), I_LINKABLE); + + *wip = tmpfile; + return 0; +} + +/* + * xfs_rename + */ +int +xfs_rename( + struct mnt_idmap *idmap, + struct xfs_inode *src_dp, + struct xfs_name *src_name, + struct xfs_inode *src_ip, + struct xfs_inode *target_dp, + struct xfs_name *target_name, + struct xfs_inode *target_ip, + unsigned int flags) +{ + struct xfs_dir_update du_src = { + .dp = src_dp, + .name = src_name, + .ip = src_ip, + }; + struct xfs_dir_update du_tgt = { + .dp = target_dp, + .name = target_name, + .ip = target_ip, + }; + struct xfs_dir_update du_wip = { }; + struct xfs_mount *mp = src_dp->i_mount; + struct xfs_trans *tp; + struct xfs_inode *inodes[__XFS_SORT_INODES]; + int i; + int num_inodes = __XFS_SORT_INODES; + bool new_parent = (src_dp != target_dp); + bool src_is_directory = S_ISDIR(VFS_I(src_ip)->i_mode); + int spaceres; + bool retried = false; + int error, nospace_error = 0; + + trace_xfs_rename(src_dp, target_dp, src_name, target_name); + + if ((flags & RENAME_EXCHANGE) && !target_ip) + return -EINVAL; + + /* + * If we are doing a whiteout operation, allocate the whiteout inode + * we will be placing at the target and ensure the type is set + * appropriately. + */ + if (flags & RENAME_WHITEOUT) { + error = xfs_rename_alloc_whiteout(idmap, src_name, target_dp, + &du_wip.ip); + if (error) + return error; + + /* setup target dirent info as whiteout */ + src_name->type = XFS_DIR3_FT_CHRDEV; + } + + xfs_sort_for_rename(src_dp, target_dp, src_ip, target_ip, du_wip.ip, + inodes, &num_inodes); + + error = xfs_parent_start(mp, &du_src.ppargs); + if (error) + goto out_release_wip; + + if (du_wip.ip) { + error = xfs_parent_start(mp, &du_wip.ppargs); + if (error) + goto out_src_ppargs; + } + + if (target_ip) { + error = xfs_parent_start(mp, &du_tgt.ppargs); + if (error) + goto out_wip_ppargs; + } + +retry: + nospace_error = 0; + spaceres = xfs_rename_space_res(mp, src_name->len, target_ip != NULL, + target_name->len, du_wip.ip != NULL); + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_rename, spaceres, 0, 0, &tp); + if (error == -ENOSPC) { + nospace_error = error; + spaceres = 0; + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_rename, 0, 0, 0, + &tp); + } + if (error) + goto out_tgt_ppargs; + + /* + * We don't allow reservationless renaming when parent pointers are + * enabled because we can't back out if the xattrs must grow. + */ + if (du_src.ppargs && nospace_error) { + error = nospace_error; + xfs_trans_cancel(tp); + goto out_tgt_ppargs; + } + + /* + * Attach the dquots to the inodes + */ + error = xfs_qm_vop_rename_dqattach(inodes); + if (error) { + xfs_trans_cancel(tp); + goto out_tgt_ppargs; + } + + /* + * Lock all the participating inodes. Depending upon whether + * the target_name exists in the target directory, and + * whether the target directory is the same as the source + * directory, we can lock from 2 to 5 inodes. + */ + xfs_lock_inodes(inodes, num_inodes, XFS_ILOCK_EXCL); + + /* + * Join all the inodes to the transaction. + */ + xfs_trans_ijoin(tp, src_dp, 0); + if (new_parent) + xfs_trans_ijoin(tp, target_dp, 0); + xfs_trans_ijoin(tp, src_ip, 0); + if (target_ip) + xfs_trans_ijoin(tp, target_ip, 0); + if (du_wip.ip) + xfs_trans_ijoin(tp, du_wip.ip, 0); + + error = xfs_projid_differ(target_dp, src_ip); + if (error) + goto out_trans_cancel; + + /* RENAME_EXCHANGE is unique from here on. */ + if (flags & RENAME_EXCHANGE) { + error = xfs_dir_exchange_children(tp, &du_src, &du_tgt, + spaceres); + if (error) + goto out_trans_cancel; + goto out_commit; + } + + /* + * Try to reserve quota to handle an expansion of the target directory. + * We'll allow the rename to continue in reservationless mode if we hit + * a space usage constraint. If we trigger reservationless mode, save + * the errno if there isn't any free space in the target directory. + */ + if (spaceres != 0) { + error = xfs_trans_reserve_quota_nblks(tp, target_dp, spaceres, + 0, false); + if (error == -EDQUOT || error == -ENOSPC) { + if (!retried) { + xfs_trans_cancel(tp); + xfs_iunlock_rename(inodes, num_inodes); + xfs_blockgc_free_quota(target_dp, 0); + retried = true; + goto retry; + } + + nospace_error = error; + spaceres = 0; + error = 0; + } + if (error) + goto out_trans_cancel; + } + + /* + * We don't allow quotaless renaming when parent pointers are enabled + * because we can't back out if the xattrs must grow. + */ + if (du_src.ppargs && nospace_error) { + error = nospace_error; + goto out_trans_cancel; + } + + /* + * Lock the AGI buffers we need to handle bumping the nlink of the + * whiteout inode off the unlinked list and to handle dropping the + * nlink of the target inode. Per locking order rules, do this in + * increasing AG order and before directory block allocation tries to + * grab AGFs because we grab AGIs before AGFs. + * + * The (vfs) caller must ensure that if src is a directory then + * target_ip is either null or an empty directory. + */ + for (i = 0; i < num_inodes && inodes[i] != NULL; i++) { + if (inodes[i] == du_wip.ip || + (inodes[i] == target_ip && + (VFS_I(target_ip)->i_nlink == 1 || src_is_directory))) { + struct xfs_perag *pag; + struct xfs_buf *bp; + + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(inodes[i])); + error = xfs_read_agi(pag, tp, 0, &bp); + xfs_perag_put(pag); + if (error) + goto out_trans_cancel; + } + } + + error = xfs_dir_rename_children(tp, &du_src, &du_tgt, spaceres, + &du_wip); + if (error) + goto out_trans_cancel; + + if (du_wip.ip) { + /* + * Now we have a real link, clear the "I'm a tmpfile" state + * flag from the inode so it doesn't accidentally get misused in + * future. + */ + inode_state_clear_raw(VFS_I(du_wip.ip), I_LINKABLE); + } + +out_commit: + /* + * If this is a synchronous mount, make sure that the rename + * transaction goes to disk before returning to the user. + */ + if (xfs_has_wsync(tp->t_mountp) || xfs_has_dirsync(tp->t_mountp)) + xfs_trans_set_sync(tp); + + error = xfs_trans_commit(tp); + nospace_error = 0; + goto out_unlock; + +out_trans_cancel: + xfs_trans_cancel(tp); +out_unlock: + xfs_iunlock_rename(inodes, num_inodes); +out_tgt_ppargs: + xfs_parent_finish(mp, du_tgt.ppargs); +out_wip_ppargs: + xfs_parent_finish(mp, du_wip.ppargs); +out_src_ppargs: + xfs_parent_finish(mp, du_src.ppargs); +out_release_wip: + if (du_wip.ip) + xfs_irele(du_wip.ip); + if (error == -ENOSPC && nospace_error) + error = nospace_error; + return error; +} + +static int +xfs_iflush( + struct xfs_inode *ip, + struct xfs_buf *bp) +{ + struct xfs_inode_log_item *iip = ip->i_itemp; + struct xfs_dinode *dip; + struct xfs_mount *mp = ip->i_mount; + int error; + + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL | XFS_ILOCK_SHARED); + ASSERT(xfs_iflags_test(ip, XFS_IFLUSHING)); + ASSERT(ip->i_df.if_format != XFS_DINODE_FMT_BTREE || + ip->i_df.if_nextents > XFS_IFORK_MAXEXT(ip, XFS_DATA_FORK)); + ASSERT(iip->ili_item.li_buf == bp); + + dip = xfs_buf_offset(bp, ip->i_imap.im_boffset); + + /* + * We don't flush the inode if any of the following checks fail, but we + * do still update the log item and attach to the backing buffer as if + * the flush happened. This is a formality to facilitate predictable + * error handling as the caller will shutdown and fail the buffer. + */ + error = -EFSCORRUPTED; + if (dip->di_magic != cpu_to_be16(XFS_DINODE_MAGIC) || + XFS_TEST_ERROR(mp, XFS_ERRTAG_IFLUSH_1)) { + xfs_alert_tag(mp, XFS_PTAG_IFLUSH, + "%s: Bad inode %llu magic number 0x%x, ptr "PTR_FMT, + __func__, I_INO(ip), be16_to_cpu(dip->di_magic), dip); + goto flush_out; + } + if (ip->i_df.if_format == XFS_DINODE_FMT_META_BTREE) { + if (!S_ISREG(VFS_I(ip)->i_mode) || + !(ip->i_diflags2 & XFS_DIFLAG2_METADATA)) { + xfs_alert_tag(mp, XFS_PTAG_IFLUSH, + "%s: Bad %s meta btree inode %Lu, ptr "PTR_FMT, + __func__, xfs_metafile_type_str(ip->i_metatype), + I_INO(ip), ip); + goto flush_out; + } + } else if (S_ISREG(VFS_I(ip)->i_mode)) { + if ((ip->i_df.if_format != XFS_DINODE_FMT_EXTENTS && + ip->i_df.if_format != XFS_DINODE_FMT_BTREE) || + XFS_TEST_ERROR(mp, XFS_ERRTAG_IFLUSH_3)) { + xfs_alert_tag(mp, XFS_PTAG_IFLUSH, + "%s: Bad regular inode %llu, ptr "PTR_FMT, + __func__, I_INO(ip), ip); + goto flush_out; + } + } else if (S_ISDIR(VFS_I(ip)->i_mode)) { + if ((ip->i_df.if_format != XFS_DINODE_FMT_EXTENTS && + ip->i_df.if_format != XFS_DINODE_FMT_BTREE && + ip->i_df.if_format != XFS_DINODE_FMT_LOCAL) || + XFS_TEST_ERROR(mp, XFS_ERRTAG_IFLUSH_4)) { + xfs_alert_tag(mp, XFS_PTAG_IFLUSH, + "%s: Bad directory inode %llu, ptr "PTR_FMT, + __func__, I_INO(ip), ip); + goto flush_out; + } + } + if (ip->i_df.if_nextents + xfs_ifork_nextents(&ip->i_af) > + ip->i_nblocks || XFS_TEST_ERROR(mp, XFS_ERRTAG_IFLUSH_5)) { + xfs_alert_tag(mp, XFS_PTAG_IFLUSH, + "%s: detected corrupt incore inode %llu, " + "total extents = %llu nblocks = %lld, ptr "PTR_FMT, + __func__, I_INO(ip), + ip->i_df.if_nextents + xfs_ifork_nextents(&ip->i_af), + ip->i_nblocks, ip); + goto flush_out; + } + if (ip->i_forkoff > mp->m_sb.sb_inodesize || + XFS_TEST_ERROR(mp, XFS_ERRTAG_IFLUSH_6)) { + xfs_alert_tag(mp, XFS_PTAG_IFLUSH, + "%s: bad inode %llu, forkoff 0x%x, ptr "PTR_FMT, + __func__, I_INO(ip), ip->i_forkoff, ip); + goto flush_out; + } + + if (xfs_inode_has_attr_fork(ip) && + ip->i_af.if_format == XFS_DINODE_FMT_META_BTREE) { + xfs_alert_tag(mp, XFS_PTAG_IFLUSH, + "%s: meta btree in inode %Lu attr fork, ptr "PTR_FMT, + __func__, I_INO(ip), ip); + goto flush_out; + } + + /* + * Inode item log recovery for v2 inodes are dependent on the flushiter + * count for correct sequencing. We bump the flush iteration count so + * we can detect flushes which postdate a log record during recovery. + * This is redundant as we now log every change and hence this can't + * happen but we need to still do it to ensure backwards compatibility + * with old kernels that predate logging all inode changes. + */ + if (!xfs_has_v3inodes(mp)) + ip->i_flushiter++; + + /* + * If there are inline format data / attr forks attached to this inode, + * make sure they are not corrupt. + */ + if (ip->i_df.if_format == XFS_DINODE_FMT_LOCAL && + xfs_ifork_verify_local_data(ip)) + goto flush_out; + if (xfs_inode_has_attr_fork(ip) && + ip->i_af.if_format == XFS_DINODE_FMT_LOCAL && + xfs_ifork_verify_local_attr(ip)) + goto flush_out; + + /* + * Copy the dirty parts of the inode into the on-disk inode. We always + * copy out the core of the inode, because if the inode is dirty at all + * the core must be. + */ + xfs_inode_to_disk(ip, dip, iip->ili_item.li_lsn); + + /* Wrap, we never let the log put out DI_MAX_FLUSH */ + if (!xfs_has_v3inodes(mp)) { + if (ip->i_flushiter == DI_MAX_FLUSH) + ip->i_flushiter = 0; + } + + xfs_iflush_fork(ip, dip, iip, XFS_DATA_FORK); + if (xfs_inode_has_attr_fork(ip)) + xfs_iflush_fork(ip, dip, iip, XFS_ATTR_FORK); + + /* + * We've recorded everything logged in the inode, so we'd like to clear + * the ili_fields bits so we don't log and flush things unnecessarily. + * However, we can't stop logging all this information until the data + * we've copied into the disk buffer is written to disk. If we did we + * might overwrite the copy of the inode in the log with all the data + * after re-logging only part of it, and in the face of a crash we + * wouldn't have all the data we need to recover. + * + * What we do is move the bits to the ili_last_fields field. When + * logging the inode, these bits are moved back to the ili_fields field. + * In the xfs_buf_inode_iodone() routine we clear ili_last_fields, since + * we know that the information those bits represent is permanently on + * disk. As long as the flush completes before the inode is logged + * again, then both ili_fields and ili_last_fields will be cleared. + */ + error = 0; +flush_out: + spin_lock(&iip->ili_lock); + iip->ili_last_fields = iip->ili_fields; + iip->ili_fields = 0; + set_bit(XFS_LI_FLUSHING, &iip->ili_item.li_flags); + spin_unlock(&iip->ili_lock); + + /* + * Store the current LSN of the inode so that we can tell whether the + * item has moved in the AIL from xfs_buf_inode_iodone(). + */ + xfs_trans_ail_copy_lsn(mp->m_ail, &iip->ili_flush_lsn, + &iip->ili_item.li_lsn); + + /* generate the checksum. */ + xfs_dinode_calc_crc(mp, dip); + if (error) + xfs_inode_mark_sick(ip, XFS_SICK_INO_CORE); + return error; +} + +/* + * Non-blocking flush of dirty inode metadata into the backing buffer. + * + * The caller must have a reference to the inode and hold the cluster buffer + * locked. The function will walk across all the inodes on the cluster buffer it + * can find and lock without blocking, and flush them to the cluster buffer. + * + * On successful flushing of at least one inode, the caller must write out the + * buffer and release it. If no inodes are flushed, -EAGAIN will be returned and + * the caller needs to release the buffer. On failure, the filesystem will be + * shut down, the buffer will have been unlocked and released, and EFSCORRUPTED + * will be returned. + */ +int +xfs_iflush_cluster( + struct xfs_buf *bp) +{ + struct xfs_mount *mp = bp->b_mount; + struct xfs_log_item *lip, *n; + struct xfs_inode *ip; + struct xfs_inode_log_item *iip; + int clcount = 0; + int error = 0; + + /* + * We must use the safe variant here as on shutdown xfs_iflush_abort() + * will remove itself from the list. + */ + list_for_each_entry_safe(lip, n, &bp->b_li_list, li_bio_list) { + iip = (struct xfs_inode_log_item *)lip; + ip = iip->ili_inode; + + /* + * Quick and dirty check to avoid locks if possible. + */ + if (__xfs_iflags_test(ip, XFS_IRECLAIM | XFS_IFLUSHING)) + continue; + if (xfs_ipincount(ip)) + continue; + + /* + * The inode is still attached to the buffer, which means it is + * dirty but reclaim might try to grab it. Check carefully for + * that, and grab the ilock while still holding the i_flags_lock + * to guarantee reclaim will not be able to reclaim this inode + * once we drop the i_flags_lock. + */ + spin_lock(&ip->i_flags_lock); + ASSERT(!__xfs_iflags_test(ip, XFS_ISTALE)); + if (__xfs_iflags_test(ip, XFS_IRECLAIM | XFS_IFLUSHING)) { + spin_unlock(&ip->i_flags_lock); + continue; + } + + /* + * ILOCK will pin the inode against reclaim and prevent + * concurrent transactions modifying the inode while we are + * flushing the inode. If we get the lock, set the flushing + * state before we drop the i_flags_lock. + */ + if (!xfs_ilock_nowait(ip, XFS_ILOCK_SHARED)) { + spin_unlock(&ip->i_flags_lock); + continue; + } + __xfs_iflags_set(ip, XFS_IFLUSHING); + spin_unlock(&ip->i_flags_lock); + + /* + * Abort flushing this inode if we are shut down because the + * inode may not currently be in the AIL. This can occur when + * log I/O failure unpins the inode without inserting into the + * AIL, leaving a dirty/unpinned inode attached to the buffer + * that otherwise looks like it should be flushed. + */ + if (xlog_is_shutdown(mp->m_log)) { + xfs_iunpin_wait(ip); + xfs_iflush_abort(ip); + xfs_iunlock(ip, XFS_ILOCK_SHARED); + error = -EIO; + continue; + } + + /* don't block waiting on a log force to unpin dirty inodes */ + if (xfs_ipincount(ip)) { + xfs_iflags_clear(ip, XFS_IFLUSHING); + xfs_iunlock(ip, XFS_ILOCK_SHARED); + continue; + } + + if (!xfs_inode_clean(ip)) + error = xfs_iflush(ip, bp); + else + xfs_iflags_clear(ip, XFS_IFLUSHING); + xfs_iunlock(ip, XFS_ILOCK_SHARED); + if (error) + break; + clcount++; + } + + if (error) { + /* + * Shutdown first so we kill the log before we release this + * buffer. If it is an INODE_ALLOC buffer and pins the tail + * of the log, failing it before the _log_ is shut down can + * result in the log tail being moved forward in the journal + * on disk because log writes can still be taking place. Hence + * unpinning the tail will allow the ICREATE intent to be + * removed from the log an recovery will fail with uninitialised + * inode cluster buffers. + */ + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); + xfs_buf_fail(bp); + return error; + } + + if (!clcount) + return -EAGAIN; + + XFS_STATS_INC(mp, xs_icluster_flushcnt); + XFS_STATS_ADD(mp, xs_icluster_flushinode, clcount); + return 0; + +} + +/* Release an inode. */ +void +xfs_irele( + struct xfs_inode *ip) +{ + trace_xfs_irele(ip, _RET_IP_); + iput(VFS_I(ip)); +} + +/* + * Ensure all commited transactions touching the inode are written to the log. + */ +int +xfs_log_force_inode( + struct xfs_inode *ip) +{ + struct xfs_inode_log_item *iip = ip->i_itemp; + xfs_csn_t seq = 0; + + if (!iip) + return 0; + + spin_lock(&iip->ili_lock); + seq = iip->ili_commit_seq; + spin_unlock(&iip->ili_lock); + + if (!seq) + return 0; + return xfs_log_force_seq(ip->i_mount, seq, XFS_LOG_SYNC, NULL); +} + +/* + * Grab the exclusive iolock for a data copy from src to dest, making sure to + * abide vfs locking order (lowest pointer value goes first) and breaking the + * layout leases before proceeding. The loop is needed because we cannot call + * the blocking break_layout() with the iolocks held, and therefore have to + * back out both locks. + */ +static int +xfs_iolock_two_inodes_and_break_layout( + struct inode *src, + struct inode *dest) +{ + int error; + + if (src > dest) + swap(src, dest); + +retry: + /* Wait to break both inodes' layouts before we start locking. */ + error = break_layout(src, true); + if (error) + return error; + if (src != dest) { + error = break_layout(dest, true); + if (error) + return error; + } + + /* Lock one inode and make sure nobody got in and leased it. */ + inode_lock(src); + error = break_layout(src, false); + if (error) { + inode_unlock(src); + if (error == -EWOULDBLOCK) + goto retry; + return error; + } + + if (src == dest) + return 0; + + /* Lock the other inode and make sure nobody got in and leased it. */ + inode_lock_nested(dest, I_MUTEX_NONDIR2); + error = break_layout(dest, false); + if (error) { + inode_unlock(src); + inode_unlock(dest); + if (error == -EWOULDBLOCK) + goto retry; + return error; + } + + return 0; +} + +static int +xfs_mmaplock_two_inodes_and_break_dax_layout( + struct xfs_inode *ip1, + struct xfs_inode *ip2) +{ + int error; + + if (I_INO(ip1) > I_INO(ip2)) + swap(ip1, ip2); + +again: + /* Lock the first inode */ + xfs_ilock(ip1, XFS_MMAPLOCK_EXCL); + error = xfs_break_dax_layouts(VFS_I(ip1)); + if (error) { + xfs_iunlock(ip1, XFS_MMAPLOCK_EXCL); + return error; + } + + if (ip1 == ip2) + return 0; + + /* Nested lock the second inode */ + xfs_ilock(ip2, xfs_lock_inumorder(XFS_MMAPLOCK_EXCL, 1)); + /* + * We cannot use xfs_break_dax_layouts() directly here because it may + * need to unlock & lock the XFS_MMAPLOCK_EXCL which is not suitable + * for this nested lock case. + */ + error = dax_break_layout(VFS_I(ip2), 0, -1, NULL); + if (error) { + xfs_iunlock(ip2, XFS_MMAPLOCK_EXCL); + xfs_iunlock(ip1, XFS_MMAPLOCK_EXCL); + goto again; + } + + return 0; +} + +/* + * Lock two inodes so that userspace cannot initiate I/O via file syscalls or + * mmap activity. + */ +int +xfs_ilock2_io_mmap( + struct xfs_inode *ip1, + struct xfs_inode *ip2) +{ + int ret; + + ret = xfs_iolock_two_inodes_and_break_layout(VFS_I(ip1), VFS_I(ip2)); + if (ret) + return ret; + + if (IS_DAX(VFS_I(ip1)) && IS_DAX(VFS_I(ip2))) { + ret = xfs_mmaplock_two_inodes_and_break_dax_layout(ip1, ip2); + if (ret) { + inode_unlock(VFS_I(ip2)); + if (ip1 != ip2) + inode_unlock(VFS_I(ip1)); + return ret; + } + } else + filemap_invalidate_lock_two(VFS_I(ip1)->i_mapping, + VFS_I(ip2)->i_mapping); + + return 0; +} + +/* Unlock both inodes to allow IO and mmap activity. */ +void +xfs_iunlock2_io_mmap( + struct xfs_inode *ip1, + struct xfs_inode *ip2) +{ + if (IS_DAX(VFS_I(ip1)) && IS_DAX(VFS_I(ip2))) { + xfs_iunlock(ip2, XFS_MMAPLOCK_EXCL); + if (ip1 != ip2) + xfs_iunlock(ip1, XFS_MMAPLOCK_EXCL); + } else + filemap_invalidate_unlock_two(VFS_I(ip1)->i_mapping, + VFS_I(ip2)->i_mapping); + + inode_unlock(VFS_I(ip2)); + if (ip1 != ip2) + inode_unlock(VFS_I(ip1)); +} + +/* Drop the MMAPLOCK and the IOLOCK after a remap completes. */ +void +xfs_iunlock2_remapping( + struct xfs_inode *ip1, + struct xfs_inode *ip2) +{ + xfs_iflags_clear(ip1, XFS_IREMAPPING); + + if (ip1 != ip2) + xfs_iunlock(ip1, XFS_MMAPLOCK_SHARED); + xfs_iunlock(ip2, XFS_MMAPLOCK_EXCL); + + if (ip1 != ip2) + inode_unlock_shared(VFS_I(ip1)); + inode_unlock(VFS_I(ip2)); +} + +/* + * Reload the incore inode list for this inode. Caller should ensure that + * the link count cannot change, either by taking ILOCK_SHARED or otherwise + * preventing other threads from executing. + */ +int +xfs_inode_reload_unlinked_bucket( + struct xfs_trans *tp, + struct xfs_inode *ip) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_buf *agibp; + struct xfs_agi *agi; + struct xfs_perag *pag; + xfs_agnumber_t agno = XFS_INODE_TO_AGNO(ip); + xfs_agino_t agino = XFS_INODE_TO_AGINO(ip); + xfs_agino_t prev_agino, next_agino; + unsigned int bucket; + bool foundit = false; + int error; + + /* Grab the first inode in the list */ + pag = xfs_perag_get(mp, agno); + error = xfs_ialloc_read_agi(pag, tp, 0, &agibp); + xfs_perag_put(pag); + if (error) + return error; + + /* + * We've taken ILOCK_SHARED and the AGI buffer lock to stabilize the + * incore unlinked list pointers for this inode. Check once more to + * see if we raced with anyone else to reload the unlinked list. + */ + if (!xfs_inode_unlinked_incomplete(ip)) { + foundit = true; + goto out_agibp; + } + + bucket = agino % XFS_AGI_UNLINKED_BUCKETS; + agi = agibp->b_addr; + + trace_xfs_inode_reload_unlinked_bucket(ip); + + xfs_info_ratelimited(mp, + "Found unrecovered unlinked inode 0x%x in AG 0x%x. Initiating list recovery.", + agino, agno); + + prev_agino = NULLAGINO; + next_agino = be32_to_cpu(agi->agi_unlinked[bucket]); + while (next_agino != NULLAGINO) { + struct xfs_inode *next_ip = NULL; + + /* Found this caller's inode, set its backlink. */ + if (next_agino == agino) { + next_ip = ip; + next_ip->i_prev_unlinked = prev_agino; + foundit = true; + goto next_inode; + } + + /* Try in-memory lookup first. */ + next_ip = xfs_iunlink_lookup(pag, next_agino); + if (next_ip) + goto next_inode; + + /* Inode not in memory, try reloading it. */ + error = xfs_iunlink_reload_next(tp, agibp, prev_agino, + next_agino); + if (error) + break; + + /* Grab the reloaded inode. */ + next_ip = xfs_iunlink_lookup(pag, next_agino); + if (!next_ip) { + /* No incore inode at all? We reloaded it... */ + ASSERT(next_ip != NULL); + error = -EFSCORRUPTED; + break; + } + +next_inode: + prev_agino = next_agino; + next_agino = next_ip->i_next_unlinked; + } + +out_agibp: + xfs_trans_brelse(tp, agibp); + /* Should have found this inode somewhere in the iunlinked bucket. */ + if (!error && !foundit) + error = -EFSCORRUPTED; + return error; +} + +/* Decide if this inode is missing its unlinked list and reload it. */ +int +xfs_inode_reload_unlinked( + struct xfs_inode *ip) +{ + struct xfs_trans *tp; + int error = 0; + + tp = xfs_trans_alloc_empty(ip->i_mount); + xfs_ilock(ip, XFS_ILOCK_SHARED); + if (xfs_inode_unlinked_incomplete(ip)) + error = xfs_inode_reload_unlinked_bucket(tp, ip); + xfs_iunlock(ip, XFS_ILOCK_SHARED); + xfs_trans_cancel(tp); + + return error; +} + +/* Has this inode fork been zapped by repair? */ +bool +xfs_ifork_zapped( + const struct xfs_inode *ip, + int whichfork) +{ + unsigned int datamask = 0; + + switch (whichfork) { + case XFS_DATA_FORK: + switch (ip->i_vnode.i_mode & S_IFMT) { + case S_IFDIR: + datamask = XFS_SICK_INO_DIR_ZAPPED; + break; + case S_IFLNK: + datamask = XFS_SICK_INO_SYMLINK_ZAPPED; + break; + } + return ip->i_sick & (XFS_SICK_INO_BMBTD_ZAPPED | datamask); + case XFS_ATTR_FORK: + return ip->i_sick & XFS_SICK_INO_BMBTA_ZAPPED; + default: + return false; + } +} + +/* Compute the number of data and realtime blocks used by a file. */ +void +xfs_inode_count_blocks( + struct xfs_trans *tp, + struct xfs_inode *ip, + xfs_filblks_t *dblocks, + xfs_filblks_t *rblocks) +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK); + + *rblocks = 0; + if (XFS_IS_REALTIME_INODE(ip)) + xfs_bmap_count_leaves(ifp, rblocks); + *dblocks = ip->i_nblocks - *rblocks; +} + +static void +xfs_wait_dax_page( + struct inode *inode) +{ + struct xfs_inode *ip = XFS_I(inode); + + xfs_iunlock(ip, XFS_MMAPLOCK_EXCL); + schedule(); + xfs_ilock(ip, XFS_MMAPLOCK_EXCL); +} + +int +xfs_break_dax_layouts( + struct inode *inode) +{ + xfs_assert_ilocked(XFS_I(inode), XFS_MMAPLOCK_EXCL); + + return dax_break_layout_inode(inode, xfs_wait_dax_page); +} + +int +xfs_break_layouts( + struct inode *inode, + uint *iolock, + enum layout_break_reason reason) +{ + bool retry; + int error; + + xfs_assert_ilocked(XFS_I(inode), XFS_IOLOCK_SHARED | XFS_IOLOCK_EXCL); + + do { + retry = false; + switch (reason) { + case BREAK_UNMAP: + error = xfs_break_dax_layouts(inode); + if (error) + break; + fallthrough; + case BREAK_WRITE: + error = xfs_break_leased_layouts(inode, iolock, &retry); + break; + default: + WARN_ON_ONCE(1); + error = -EINVAL; + } + } while (error == 0 && retry); + + return error; +} + +/* Returns the size of fundamental allocation unit for a file, in bytes. */ +unsigned int +xfs_inode_alloc_unitsize( + struct xfs_inode *ip) +{ + unsigned int blocks = 1; + + if (XFS_IS_REALTIME_INODE(ip)) + blocks = ip->i_mount->m_sb.sb_rextsize; + + return XFS_FSB_TO_B(ip->i_mount, blocks); +} + +/* Should we always be using copy on write for file writes? */ +bool +xfs_is_always_cow_inode( + const struct xfs_inode *ip) +{ + return xfs_is_zoned_inode(ip) || + (ip->i_mount->m_always_cow && xfs_has_reflink(ip->i_mount)); +} diff --git a/fs/xfs/xfs_inode.h b/fs/xfs/xfs_inode.h new file mode 100644 index 000000000..34c1038eb --- /dev/null +++ b/fs/xfs/xfs_inode.h @@ -0,0 +1,676 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2003,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_INODE_H__ +#define __XFS_INODE_H__ + +#include "xfs_inode_buf.h" +#include "xfs_inode_fork.h" +#include "xfs_inode_util.h" + +/* + * Kernel only inode definitions + */ +struct xfs_dinode; +struct xfs_inode; +struct xfs_buf; +struct xfs_bmbt_irec; +struct xfs_inode_log_item; +struct xfs_mount; +struct xfs_trans; +struct xfs_dquot; + +typedef struct xfs_inode { + /* Inode linking and identification information. */ + struct xfs_mount *i_mount; /* fs mount struct ptr */ + struct xfs_dquot *i_udquot; /* user dquot */ + struct xfs_dquot *i_gdquot; /* group dquot */ + struct xfs_dquot *i_pdquot; /* project dquot */ + + /* Inode location stuff */ + struct xfs_imap i_imap; /* location for xfs_imap() */ + + /* Extent information. */ + struct xfs_ifork *i_cowfp; /* copy on write extents */ + struct xfs_ifork i_df; /* data fork */ + struct xfs_ifork i_af; /* attribute fork */ + + /* Transaction and locking information. */ + struct xfs_inode_log_item *i_itemp; /* logging information */ + struct rw_semaphore i_lock; /* inode lock */ + atomic_t i_pincount; /* inode pin count */ + struct llist_node i_gclist; /* deferred inactivation list */ + + /* + * Bitsets of inode metadata that have been checked and/or are sick. + * Callers must hold i_flags_lock before accessing this field. + */ + uint16_t i_checked; + uint16_t i_sick; + + spinlock_t i_flags_lock; /* inode i_flags lock */ + /* Miscellaneous state. */ + unsigned long i_flags; /* see defined flags below */ + uint64_t i_delayed_blks; /* count of delay alloc blks */ + xfs_fsize_t i_disk_size; /* number of bytes in file */ + xfs_rfsblock_t i_nblocks; /* # of direct & btree blocks */ + prid_t i_projid; /* owner's project id */ + xfs_extlen_t i_extsize; /* basic/minimum extent size */ + /* + * i_used_blocks is used for zoned rtrmap inodes, + * i_cowextsize is used for other v3 inodes, + * i_flushiter for v1/2 inodes + */ + union { + uint32_t i_used_blocks; /* used blocks in RTG */ + xfs_extlen_t i_cowextsize; /* basic cow extent size */ + uint16_t i_flushiter; /* incremented on flush */ + }; + uint8_t i_forkoff; /* attr fork offset >> 3 */ + enum xfs_metafile_type i_metatype; /* XFS_METAFILE_* */ + uint16_t i_diflags; /* XFS_DIFLAG_... */ + uint64_t i_diflags2; /* XFS_DIFLAG2_... */ + struct timespec64 i_crtime; /* time created */ + + /* + * Unlinked list pointers. These point to the next and previous inodes + * in the AGI unlinked bucket list, respectively. These fields can + * only be updated with the AGI locked. + * + * i_next_unlinked caches di_next_unlinked. + */ + xfs_agino_t i_next_unlinked; + + /* + * If the inode is not on an unlinked list, this field is zero. If the + * inode is the first element in an unlinked list, this field is + * NULLAGINO. Otherwise, i_prev_unlinked points to the previous inode + * in the unlinked list. + */ + xfs_agino_t i_prev_unlinked; + + /* VFS inode */ + struct inode i_vnode; /* embedded VFS inode */ + + /* pending io completions */ + spinlock_t i_ioend_lock; + struct work_struct i_ioend_work; + struct list_head i_ioend_list; +} xfs_inode_t; + +static inline bool xfs_inode_on_unlinked_list(const struct xfs_inode *ip) +{ + return ip->i_prev_unlinked != 0; +} + +static inline bool xfs_inode_has_attr_fork(const struct xfs_inode *ip) +{ + return ip->i_forkoff > 0; +} + +static inline struct xfs_ifork * +xfs_ifork_ptr( + struct xfs_inode *ip, + int whichfork) +{ + switch (whichfork) { + case XFS_DATA_FORK: + return &ip->i_df; + case XFS_ATTR_FORK: + if (!xfs_inode_has_attr_fork(ip)) + return NULL; + return &ip->i_af; + case XFS_COW_FORK: + return ip->i_cowfp; + default: + ASSERT(0); + return NULL; + } +} + +static inline unsigned int xfs_inode_fork_boff(struct xfs_inode *ip) +{ + return ip->i_forkoff << 3; +} + +static inline unsigned int xfs_inode_data_fork_size(struct xfs_inode *ip) +{ + if (xfs_inode_has_attr_fork(ip)) + return xfs_inode_fork_boff(ip); + + return XFS_LITINO(ip->i_mount); +} + +static inline unsigned int xfs_inode_attr_fork_size(struct xfs_inode *ip) +{ + if (xfs_inode_has_attr_fork(ip)) + return XFS_LITINO(ip->i_mount) - xfs_inode_fork_boff(ip); + return 0; +} + +static inline unsigned int +xfs_inode_fork_size( + struct xfs_inode *ip, + int whichfork) +{ + switch (whichfork) { + case XFS_DATA_FORK: + return xfs_inode_data_fork_size(ip); + case XFS_ATTR_FORK: + return xfs_inode_attr_fork_size(ip); + default: + return 0; + } +} + +/* Convert from vfs inode to xfs inode */ +static inline struct xfs_inode *XFS_I(struct inode *inode) +{ + return container_of(inode, struct xfs_inode, i_vnode); +} + +/* convert from xfs inode to vfs inode */ +static inline struct inode *VFS_I(struct xfs_inode *ip) +{ + return &ip->i_vnode; +} + +/* convert from const xfs inode to const vfs inode */ +static inline const struct inode *VFS_IC(const struct xfs_inode *ip) +{ + return &ip->i_vnode; +} + +static inline xfs_ino_t I_INO(const struct xfs_inode *ip) +{ + return VFS_IC(ip)->i_ino; +} + +/* + * For regular files we only update the on-disk filesize when actually + * writing data back to disk. Until then only the copy in the VFS inode + * is uptodate. + */ +static inline xfs_fsize_t XFS_ISIZE(struct xfs_inode *ip) +{ + if (S_ISREG(VFS_I(ip)->i_mode)) + return i_size_read(VFS_I(ip)); + return ip->i_disk_size; +} + +/* + * If this I/O goes past the on-disk inode size update it unless it would + * be past the current in-core inode size. + */ +static inline xfs_fsize_t +xfs_new_eof(struct xfs_inode *ip, xfs_fsize_t new_size) +{ + xfs_fsize_t i_size = i_size_read(VFS_I(ip)); + + if (new_size > i_size || new_size < 0) + new_size = i_size; + return new_size > ip->i_disk_size ? new_size : 0; +} + +/* + * i_flags helper functions + */ +static inline void +__xfs_iflags_set(xfs_inode_t *ip, unsigned long flags) +{ + ip->i_flags |= flags; +} + +static inline void +xfs_iflags_set(xfs_inode_t *ip, unsigned long flags) +{ + spin_lock(&ip->i_flags_lock); + __xfs_iflags_set(ip, flags); + spin_unlock(&ip->i_flags_lock); +} + +static inline void +xfs_iflags_clear(xfs_inode_t *ip, unsigned long flags) +{ + spin_lock(&ip->i_flags_lock); + ip->i_flags &= ~flags; + spin_unlock(&ip->i_flags_lock); +} + +static inline int +__xfs_iflags_test(const struct xfs_inode *ip, unsigned long flags) +{ + return (ip->i_flags & flags); +} + +static inline int +xfs_iflags_test(xfs_inode_t *ip, unsigned long flags) +{ + int ret; + spin_lock(&ip->i_flags_lock); + ret = __xfs_iflags_test(ip, flags); + spin_unlock(&ip->i_flags_lock); + return ret; +} + +static inline int +xfs_iflags_test_and_clear(xfs_inode_t *ip, unsigned long flags) +{ + int ret; + + spin_lock(&ip->i_flags_lock); + ret = ip->i_flags & flags; + if (ret) + ip->i_flags &= ~flags; + spin_unlock(&ip->i_flags_lock); + return ret; +} + +static inline int +xfs_iflags_test_and_set(xfs_inode_t *ip, unsigned long flags) +{ + int ret; + + spin_lock(&ip->i_flags_lock); + ret = ip->i_flags & flags; + if (!ret) + ip->i_flags |= flags; + spin_unlock(&ip->i_flags_lock); + return ret; +} + +static inline bool xfs_is_reflink_inode(const struct xfs_inode *ip) +{ + return ip->i_diflags2 & XFS_DIFLAG2_REFLINK; +} + +static inline bool xfs_is_metadir_inode(const struct xfs_inode *ip) +{ + return ip->i_diflags2 & XFS_DIFLAG2_METADATA; +} + +static inline bool xfs_is_internal_inode(const struct xfs_inode *ip) +{ + struct xfs_mount *mp = ip->i_mount; + + /* Any file in the metadata directory tree is a metadata inode. */ + if (xfs_has_metadir(mp)) + return xfs_is_metadir_inode(ip); + + /* + * Before metadata directories, the only metadata inodes were the + * three quota files, the realtime bitmap, and the realtime summary. + */ + return I_INO(ip) == mp->m_sb.sb_rbmino || + I_INO(ip) == mp->m_sb.sb_rsumino || + xfs_is_quota_inode(&mp->m_sb, I_INO(ip)); +} + +static inline bool xfs_is_zoned_inode(const struct xfs_inode *ip) +{ + return xfs_has_zoned(ip->i_mount) && XFS_IS_REALTIME_INODE(ip); +} + +bool xfs_is_always_cow_inode(const struct xfs_inode *ip); + +static inline bool xfs_is_cow_inode(const struct xfs_inode *ip) +{ + return xfs_is_reflink_inode(ip) || xfs_is_always_cow_inode(ip); +} + +static inline bool xfs_inode_has_filedata(const struct xfs_inode *ip) +{ + return ip->i_df.if_nextents > 0 || ip->i_delayed_blks > 0; +} + +/* + * Check if an inode has any data in the COW fork. This might be often false + * even for inodes with the reflink flag when there is no pending COW operation. + */ +static inline bool xfs_inode_has_cow_data(const struct xfs_inode *ip) +{ + return ip->i_cowfp && ip->i_cowfp->if_bytes; +} + +static inline bool xfs_inode_has_bigtime(const struct xfs_inode *ip) +{ + return ip->i_diflags2 & XFS_DIFLAG2_BIGTIME; +} + +static inline bool xfs_inode_has_large_extent_counts(const struct xfs_inode *ip) +{ + return ip->i_diflags2 & XFS_DIFLAG2_NREXT64; +} + +/* + * Decide if this file is a realtime file whose data allocation unit is larger + * than a single filesystem block. + */ +static inline bool xfs_inode_has_bigrtalloc(const struct xfs_inode *ip) +{ + return XFS_IS_REALTIME_INODE(ip) && ip->i_mount->m_sb.sb_rextsize > 1; +} + +/* + * Return the buftarg used for data allocations on a given inode. + */ +#define xfs_inode_buftarg(ip) \ + (XFS_IS_REALTIME_INODE(ip) ? \ + (ip)->i_mount->m_rtdev_targp : (ip)->i_mount->m_ddev_targp) + +static inline bool xfs_inode_can_hw_atomic_write(const struct xfs_inode *ip) +{ + if (IS_DAX(VFS_IC(ip))) + return false; + + return xfs_inode_buftarg(ip)->bt_awu_max > 0; +} + +static inline bool xfs_inode_can_sw_atomic_write(const struct xfs_inode *ip) +{ + if (IS_DAX(VFS_IC(ip))) + return false; + + return xfs_can_sw_atomic_write(ip->i_mount); +} + +/* + * In-core inode flags. + */ +#define XFS_IRECLAIM (1 << 0) /* started reclaiming this inode */ +#define XFS_ISTALE (1 << 1) /* inode has been staled */ +#define XFS_IRECLAIMABLE (1 << 2) /* inode can be reclaimed */ +#define XFS_INEW (1 << 3) /* inode has just been allocated */ +#define XFS_IPRESERVE_DM_FIELDS (1 << 4) /* has legacy DMAPI fields set */ +#define XFS_ITRUNCATED (1 << 5) /* truncated down so flush-on-close */ +#define XFS_EOFBLOCKS_RELEASED (1 << 6) /* eofblocks were freed in ->release */ +#define XFS_IFLUSHING (1 << 7) /* inode is being flushed */ +#define __XFS_IPINNED_BIT 8 /* wakeup key for zero pin count */ +#define XFS_IPINNED (1 << __XFS_IPINNED_BIT) +#define XFS_IEOFBLOCKS (1 << 9) /* has the preallocblocks tag set */ +#define XFS_NEED_INACTIVE (1 << 10) /* see XFS_INACTIVATING below */ +/* + * If this unlinked inode is in the middle of recovery, don't let drop_inode + * truncate and free the inode. This can happen if we iget the inode during + * log recovery to replay a bmap operation on the inode. + */ +#define XFS_IRECOVERY (1 << 11) +#define XFS_ICOWBLOCKS (1 << 12)/* has the cowblocks tag set */ + +/* + * If we need to update on-disk metadata before this IRECLAIMABLE inode can be + * freed, then NEED_INACTIVE will be set. Once we start the updates, the + * INACTIVATING bit will be set to keep iget away from this inode. After the + * inactivation completes, both flags will be cleared and the inode is a + * plain old IRECLAIMABLE inode. + */ +#define XFS_INACTIVATING (1 << 13) + +/* Quotacheck is running but inode has not been added to quota counts. */ +#define XFS_IQUOTAUNCHECKED (1 << 14) + +/* + * Remap in progress. Callers that wish to update file data while + * holding a shared IOLOCK or MMAPLOCK must drop the lock and retake + * the lock in exclusive mode. Relocking the file will block until + * IREMAPPING is cleared. + */ +#define XFS_IREMAPPING (1U << 15) + +/* All inode state flags related to inode reclaim. */ +#define XFS_ALL_IRECLAIM_FLAGS (XFS_IRECLAIMABLE | \ + XFS_IRECLAIM | \ + XFS_NEED_INACTIVE | \ + XFS_INACTIVATING) + +/* + * Per-lifetime flags need to be reset when re-using a reclaimable inode during + * inode lookup. This prevents unintended behaviour on the new inode from + * ocurring. + */ +#define XFS_IRECLAIM_RESET_FLAGS \ + (XFS_IRECLAIMABLE | XFS_IRECLAIM | \ + XFS_EOFBLOCKS_RELEASED | XFS_ITRUNCATED | XFS_NEED_INACTIVE | \ + XFS_INACTIVATING | XFS_IQUOTAUNCHECKED) + +/* + * Flags for inode locking. + * Bit ranges: 1<<1 - 1<<16-1 -- iolock/ilock modes (bitfield) + * 1<<16 - 1<<32-1 -- lockdep annotation (integers) + */ +#define XFS_IOLOCK_EXCL (1u << 0) +#define XFS_IOLOCK_SHARED (1u << 1) +#define XFS_ILOCK_EXCL (1u << 2) +#define XFS_ILOCK_SHARED (1u << 3) +#define XFS_MMAPLOCK_EXCL (1u << 4) +#define XFS_MMAPLOCK_SHARED (1u << 5) + +#define XFS_LOCK_MASK (XFS_IOLOCK_EXCL | XFS_IOLOCK_SHARED \ + | XFS_ILOCK_EXCL | XFS_ILOCK_SHARED \ + | XFS_MMAPLOCK_EXCL | XFS_MMAPLOCK_SHARED) + +#define XFS_LOCK_FLAGS \ + { XFS_IOLOCK_EXCL, "IOLOCK_EXCL" }, \ + { XFS_IOLOCK_SHARED, "IOLOCK_SHARED" }, \ + { XFS_ILOCK_EXCL, "ILOCK_EXCL" }, \ + { XFS_ILOCK_SHARED, "ILOCK_SHARED" }, \ + { XFS_MMAPLOCK_EXCL, "MMAPLOCK_EXCL" }, \ + { XFS_MMAPLOCK_SHARED, "MMAPLOCK_SHARED" } + + +/* + * Flags for lockdep annotations. + * + * XFS_LOCK_PARENT - for directory operations that require locking a + * parent directory inode and a child entry inode. IOLOCK requires nesting, + * MMAPLOCK does not support this class, ILOCK requires a single subclass + * to differentiate parent from child. + * + * XFS_LOCK_RTBITMAP/XFS_LOCK_RTSUM - the realtime device bitmap and summary + * inodes do not participate in the normal lock order, and thus have their + * own subclasses. + * + * XFS_LOCK_INUMORDER - for locking several inodes at the some time + * with xfs_lock_inodes(). This flag is used as the starting subclass + * and each subsequent lock acquired will increment the subclass by one. + * However, MAX_LOCKDEP_SUBCLASSES == 8, which means we are greatly + * limited to the subclasses we can represent via nesting. We need at least + * 5 inodes nest depth for the ILOCK through rename, and we also have to support + * XFS_ILOCK_PARENT, which gives 6 subclasses. That's 6 of the 8 subclasses + * supported by lockdep. + * + * This also means we have to number the sub-classes in the lowest bits of + * the mask we keep, and we have to ensure we never exceed 3 bits of lockdep + * mask and we can't use bit-masking to build the subclasses. What a mess. + * + * Bit layout: + * + * Bit Lock Region + * 16-19 XFS_IOLOCK_SHIFT dependencies + * 20-23 XFS_MMAPLOCK_SHIFT dependencies + * 24-31 XFS_ILOCK_SHIFT dependencies + * + * IOLOCK values + * + * 0-3 subclass value + * 4-7 unused + * + * MMAPLOCK values + * + * 0-3 subclass value + * 4-7 unused + * + * ILOCK values + * 0-4 subclass values + * 5 PARENT subclass (not nestable) + * 6 unused + * 7 unused + * + */ +#define XFS_IOLOCK_SHIFT 16 +#define XFS_IOLOCK_MAX_SUBCLASS 3 +#define XFS_IOLOCK_DEP_MASK 0x000f0000u + +#define XFS_MMAPLOCK_SHIFT 20 +#define XFS_MMAPLOCK_NUMORDER 0 +#define XFS_MMAPLOCK_MAX_SUBCLASS 3 +#define XFS_MMAPLOCK_DEP_MASK 0x00f00000u + +#define XFS_ILOCK_SHIFT 24 +#define XFS_ILOCK_PARENT_VAL 5u +#define XFS_ILOCK_MAX_SUBCLASS (XFS_ILOCK_PARENT_VAL - 1) +#define XFS_ILOCK_DEP_MASK 0xff000000u +#define XFS_ILOCK_PARENT (XFS_ILOCK_PARENT_VAL << XFS_ILOCK_SHIFT) + +#define XFS_LOCK_SUBCLASS_MASK (XFS_IOLOCK_DEP_MASK | \ + XFS_MMAPLOCK_DEP_MASK | \ + XFS_ILOCK_DEP_MASK) + +#define XFS_IOLOCK_DEP(flags) (((flags) & XFS_IOLOCK_DEP_MASK) \ + >> XFS_IOLOCK_SHIFT) +#define XFS_MMAPLOCK_DEP(flags) (((flags) & XFS_MMAPLOCK_DEP_MASK) \ + >> XFS_MMAPLOCK_SHIFT) +#define XFS_ILOCK_DEP(flags) (((flags) & XFS_ILOCK_DEP_MASK) \ + >> XFS_ILOCK_SHIFT) + +/* + * Layouts are broken in the BREAK_WRITE case to ensure that + * layout-holders do not collide with local writes. Additionally, + * layouts are broken in the BREAK_UNMAP case to make sure the + * layout-holder has a consistent view of the file's extent map. While + * BREAK_WRITE breaks can be satisfied by recalling FL_LAYOUT leases, + * BREAK_UNMAP breaks additionally require waiting for busy dax-pages to + * go idle. + */ +enum layout_break_reason { + BREAK_WRITE, + BREAK_UNMAP, +}; + +/* + * For multiple groups support: if S_ISGID bit is set in the parent + * directory, group of new file is set to that of the parent, and + * new subdirectory gets S_ISGID bit from parent. + */ +#define XFS_INHERIT_GID(pip) \ + (xfs_has_grpid((pip)->i_mount) || (VFS_I(pip)->i_mode & S_ISGID)) + +int xfs_inactive(struct xfs_inode *ip); +int xfs_lookup(struct xfs_inode *dp, const struct xfs_name *name, + struct xfs_inode **ipp, struct xfs_name *ci_name); +int xfs_create(const struct xfs_icreate_args *iargs, + struct xfs_name *name, struct xfs_inode **ipp); +int xfs_create_tmpfile(const struct xfs_icreate_args *iargs, + struct xfs_inode **ipp); +int xfs_remove(struct xfs_inode *dp, struct xfs_name *name, + struct xfs_inode *ip); +int xfs_link(struct xfs_inode *tdp, struct xfs_inode *sip, + struct xfs_name *target_name); +int xfs_rename(struct mnt_idmap *idmap, + struct xfs_inode *src_dp, struct xfs_name *src_name, + struct xfs_inode *src_ip, struct xfs_inode *target_dp, + struct xfs_name *target_name, + struct xfs_inode *target_ip, unsigned int flags); + +void xfs_ilock(xfs_inode_t *, uint); +int xfs_ilock_nowait(xfs_inode_t *, uint); +void xfs_iunlock(xfs_inode_t *, uint); +void xfs_ilock_demote(xfs_inode_t *, uint); +void xfs_assert_ilocked(struct xfs_inode *, uint); +uint xfs_ilock_data_map_shared(struct xfs_inode *); +uint xfs_ilock_attr_map_shared(struct xfs_inode *); + +int xfs_ifree(struct xfs_trans *, struct xfs_inode *); +int xfs_itruncate_extents_flags(struct xfs_trans **, + struct xfs_inode *, int, xfs_fsize_t, int); +void xfs_iext_realloc(xfs_inode_t *, int, int); + +int xfs_log_force_inode(struct xfs_inode *ip); +void xfs_iunpin_wait(xfs_inode_t *); +#define xfs_ipincount(ip) ((unsigned int) atomic_read(&ip->i_pincount)) + +int xfs_iflush_cluster(struct xfs_buf *); +void xfs_lock_two_inodes(struct xfs_inode *ip0, uint ip0_mode, + struct xfs_inode *ip1, uint ip1_mode); + +int xfs_icreate(struct xfs_trans *tp, xfs_ino_t ino, + const struct xfs_icreate_args *args, struct xfs_inode **ipp); + +static inline int +xfs_itruncate_extents( + struct xfs_trans **tpp, + struct xfs_inode *ip, + int whichfork, + xfs_fsize_t new_size) +{ + return xfs_itruncate_extents_flags(tpp, ip, whichfork, new_size, 0); +} + +int xfs_break_dax_layouts(struct inode *inode); +int xfs_break_layouts(struct inode *inode, uint *iolock, + enum layout_break_reason reason); + +static inline void xfs_update_stable_writes(struct xfs_inode *ip) +{ + if (bdev_stable_writes(xfs_inode_buftarg(ip)->bt_bdev)) + mapping_set_stable_writes(VFS_I(ip)->i_mapping); + else + mapping_clear_stable_writes(VFS_I(ip)->i_mapping); +} + +/* + * When setting up a newly allocated inode, we need to call + * xfs_finish_inode_setup() once the inode is fully instantiated at + * the VFS level to prevent the rest of the world seeing the inode + * before we've completed instantiation. Otherwise we can do it + * the moment the inode lookup is complete. + */ +static inline void xfs_finish_inode_setup(struct xfs_inode *ip) +{ + xfs_iflags_clear(ip, XFS_INEW); + barrier(); + unlock_new_inode(VFS_I(ip)); +} + +void xfs_irele(struct xfs_inode *ip); + +extern struct kmem_cache *xfs_inode_cache; + +/* The default CoW extent size hint. */ +#define XFS_DEFAULT_COWEXTSZ_HINT 32 + +bool xfs_inode_needs_inactive(struct xfs_inode *ip); + +struct xfs_inode *xfs_iunlink_lookup(struct xfs_perag *pag, xfs_agino_t agino); +int xfs_iunlink_reload_next(struct xfs_trans *tp, struct xfs_buf *agibp, + xfs_agino_t prev_agino, xfs_agino_t next_agino); + +void xfs_end_io(struct work_struct *work); + +int xfs_ilock2_io_mmap(struct xfs_inode *ip1, struct xfs_inode *ip2); +void xfs_iunlock2_io_mmap(struct xfs_inode *ip1, struct xfs_inode *ip2); +void xfs_iunlock2_remapping(struct xfs_inode *ip1, struct xfs_inode *ip2); +void xfs_lock_inodes(struct xfs_inode **ips, int inodes, uint lock_mode); +void xfs_sort_inodes(struct xfs_inode **i_tab, unsigned int num_inodes); + +static inline bool +xfs_inode_unlinked_incomplete( + const struct xfs_inode *ip) +{ + return VFS_IC(ip)->i_nlink == 0 && !xfs_inode_on_unlinked_list(ip); +} +int xfs_inode_reload_unlinked_bucket(struct xfs_trans *tp, struct xfs_inode *ip); +int xfs_inode_reload_unlinked(struct xfs_inode *ip); + +bool xfs_ifork_zapped(const struct xfs_inode *ip, int whichfork); +void xfs_inode_count_blocks(struct xfs_trans *tp, struct xfs_inode *ip, + xfs_filblks_t *dblocks, xfs_filblks_t *rblocks); +unsigned int xfs_inode_alloc_unitsize(struct xfs_inode *ip); + +int xfs_icreate_dqalloc(const struct xfs_icreate_args *args, + struct xfs_dquot **udqpp, struct xfs_dquot **gdqpp, + struct xfs_dquot **pdqpp); + +#endif /* __XFS_INODE_H__ */ diff --git a/fs/xfs/xfs_inode_item.c b/fs/xfs/xfs_inode_item.c new file mode 100644 index 000000000..99d6ecccd --- /dev/null +++ b/fs/xfs/xfs_inode_item.c @@ -0,0 +1,1247 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_inode_item.h" +#include "xfs_trace.h" +#include "xfs_trans_priv.h" +#include "xfs_buf_item.h" +#include "xfs_log.h" +#include "xfs_log_priv.h" +#include "xfs_ag.h" +#include "xfs_error.h" +#include "xfs_rtbitmap.h" + +#include + +struct kmem_cache *xfs_ili_cache; /* inode log item */ + +static inline struct xfs_inode_log_item *INODE_ITEM(struct xfs_log_item *lip) +{ + return container_of(lip, struct xfs_inode_log_item, ili_item); +} + +static uint64_t +xfs_inode_item_sort( + struct xfs_log_item *lip) +{ + return I_INO(INODE_ITEM(lip)->ili_inode); +} + +#ifdef DEBUG_EXPENSIVE +static void +xfs_inode_item_precommit_check( + struct xfs_inode *ip) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_dinode *dip; + xfs_failaddr_t fa; + + dip = kzalloc(mp->m_sb.sb_inodesize, GFP_KERNEL | GFP_NOFS); + if (!dip) { + ASSERT(dip != NULL); + return; + } + + xfs_inode_to_disk(ip, dip, 0); + xfs_dinode_calc_crc(mp, dip); + fa = xfs_dinode_verify(mp, I_INO(ip), dip); + if (fa) { + xfs_inode_verifier_error(ip, -EFSCORRUPTED, __func__, dip, + sizeof(*dip), fa); + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); + ASSERT(fa == NULL); + } + kfree(dip); +} +#else +# define xfs_inode_item_precommit_check(ip) ((void)0) +#endif + +/* + * Prior to finally logging the inode, we have to ensure that all the + * per-modification inode state changes are applied. This includes VFS inode + * state updates, format conversions, verifier state synchronisation and + * ensuring the inode buffer remains in memory whilst the inode is dirty. + * + * We have to be careful when we grab the inode cluster buffer due to lock + * ordering constraints. The unlinked inode modifications (xfs_iunlink_item) + * require AGI -> inode cluster buffer lock order. The inode cluster buffer is + * not locked until ->precommit, so it happens after everything else has been + * modified. + * + * Further, we have AGI -> AGF lock ordering, and with O_TMPFILE handling we + * have AGI -> AGF -> iunlink item -> inode cluster buffer lock order. Hence we + * cannot safely lock the inode cluster buffer in xfs_trans_log_inode() because + * it can be called on a inode (e.g. via bumplink/droplink) before we take the + * AGF lock modifying directory blocks. + * + * Rather than force a complete rework of all the transactions to call + * xfs_trans_log_inode() once and once only at the end of every transaction, we + * move the pinning of the inode cluster buffer to a ->precommit operation. This + * matches how the xfs_iunlink_item locks the inode cluster buffer, and it + * ensures that the inode cluster buffer locking is always done last in a + * transaction. i.e. we ensure the lock order is always AGI -> AGF -> inode + * cluster buffer. + * + * If we return the inode number as the precommit sort key then we'll also + * guarantee that the order all inode cluster buffer locking is the same all the + * inodes and unlink items in the transaction. + */ +static int +xfs_inode_item_precommit( + struct xfs_trans *tp, + struct xfs_log_item *lip) +{ + struct xfs_inode_log_item *iip = INODE_ITEM(lip); + struct xfs_inode *ip = iip->ili_inode; + struct xfs_mount *mp = ip->i_mount; + struct inode *inode = VFS_I(ip); + unsigned int flags = iip->ili_dirty_flags; + + /* + * Don't bother with i_lock for the I_DIRTY_TIME check here, as races + * don't matter - we either will need an extra transaction in 24 hours + * to log the timestamps, or will clear already cleared fields in the + * worst case. + */ + if (inode_state_read_once(inode) & I_DIRTY_TIME) { + spin_lock(&inode->i_lock); + inode_state_clear(inode, I_DIRTY_TIME); + spin_unlock(&inode->i_lock); + } + + /* + * If we're updating the inode core or the timestamps and it's possible + * to upgrade this inode to bigtime format, do so now. + */ + if ((flags & (XFS_ILOG_CORE | XFS_ILOG_TIMESTAMP)) && + xfs_has_bigtime(mp) && !xfs_inode_has_bigtime(ip)) { + ip->i_diflags2 |= XFS_DIFLAG2_BIGTIME; + flags |= XFS_ILOG_CORE; + } + + /* + * Inode verifiers do not check that the extent size hints are an + * integer multiple of the rt extent size on a directory with + * rtinherit flags set. If we're logging a directory that is + * misconfigured in this way, clear the bad hints. + */ + if (ip->i_diflags & XFS_DIFLAG_RTINHERIT) { + if ((ip->i_diflags & XFS_DIFLAG_EXTSZINHERIT) && + xfs_extlen_to_rtxmod(mp, ip->i_extsize) > 0) { + ip->i_diflags &= ~(XFS_DIFLAG_EXTSIZE | + XFS_DIFLAG_EXTSZINHERIT); + ip->i_extsize = 0; + flags |= XFS_ILOG_CORE; + } + if ((ip->i_diflags2 & XFS_DIFLAG2_COWEXTSIZE) && + xfs_extlen_to_rtxmod(mp, ip->i_cowextsize) > 0) { + ip->i_diflags2 &= ~XFS_DIFLAG2_COWEXTSIZE; + ip->i_cowextsize = 0; + flags |= XFS_ILOG_CORE; + } + } + + spin_lock(&iip->ili_lock); + if (!iip->ili_item.li_buf) { + struct xfs_perag *pag; + struct xfs_buf *bp; + int error; + + /* + * We hold the ILOCK here, so this inode is not going to be + * flushed while we are here. Further, because there is no + * buffer attached to the item, we know that there is no IO in + * progress, so nothing will clear the ili_fields while we read + * in the buffer. Hence we can safely drop the spin lock and + * read the buffer knowing that the state will not change from + * here. + */ + spin_unlock(&iip->ili_lock); + pag = xfs_perag_get(mp, XFS_INODE_TO_AGNO(ip)); + error = xfs_read_icluster(pag, tp, ip->i_imap.im_agbno, &bp); + xfs_perag_put(pag); + if (error) + return error; + + /* + * We need an explicit buffer reference for the log item but + * don't want the buffer to remain attached to the transaction. + * Hold the buffer but release the transaction reference once + * we've attached the inode log item to the buffer log item + * list. + */ + xfs_buf_hold(bp); + spin_lock(&iip->ili_lock); + iip->ili_item.li_buf = bp; + bp->b_iodone = xfs_buf_inode_iodone; + list_add_tail(&iip->ili_item.li_bio_list, &bp->b_li_list); + xfs_trans_brelse(tp, bp); + } + + /* + * Store the dirty flags back into the inode item as this state is used + * later on in xfs_inode_item_committing() to determine whether the + * transaction is relevant to fsync state or not. + */ + iip->ili_dirty_flags = flags; + + /* + * Convert the flags on-disk fields that have been modified in the + * transaction so that ili_fields tracks the changes correctly. + */ + if (flags & XFS_ILOG_IVERSION) + flags = ((flags & ~XFS_ILOG_IVERSION) | XFS_ILOG_CORE); + + /* + * Always OR in the bits from the ili_last_fields field. This is to + * coordinate with the xfs_iflush() and xfs_buf_inode_iodone() routines + * in the eventual clearing of the ili_fields bits. See the big comment + * in xfs_iflush() for an explanation of this coordination mechanism. + */ + iip->ili_fields |= (flags | iip->ili_last_fields); + spin_unlock(&iip->ili_lock); + + xfs_inode_item_precommit_check(ip); + return 0; +} + +/* + * The logged size of an inode fork is always the current size of the inode + * fork. This means that when an inode fork is relogged, the size of the logged + * region is determined by the current state, not the combination of the + * previously logged state + the current state. This is different relogging + * behaviour to most other log items which will retain the size of the + * previously logged changes when smaller regions are relogged. + * + * Hence operations that remove data from the inode fork (e.g. shortform + * dir/attr remove, extent form extent removal, etc), the size of the relogged + * inode gets -smaller- rather than stays the same size as the previously logged + * size and this can result in the committing transaction reducing the amount of + * space being consumed by the CIL. + */ +STATIC void +xfs_inode_item_data_fork_size( + struct xfs_inode_log_item *iip, + int *nvecs, + int *nbytes) +{ + struct xfs_inode *ip = iip->ili_inode; + + switch (ip->i_df.if_format) { + case XFS_DINODE_FMT_EXTENTS: + if ((iip->ili_fields & XFS_ILOG_DEXT) && + ip->i_df.if_nextents > 0 && + ip->i_df.if_bytes > 0) { + /* worst case, doesn't subtract delalloc extents */ + *nbytes += xfs_inode_data_fork_size(ip); + *nvecs += 1; + } + break; + case XFS_DINODE_FMT_BTREE: + case XFS_DINODE_FMT_META_BTREE: + if ((iip->ili_fields & XFS_ILOG_DBROOT) && + ip->i_df.if_broot_bytes > 0) { + *nbytes += ip->i_df.if_broot_bytes; + *nvecs += 1; + } + break; + case XFS_DINODE_FMT_LOCAL: + if ((iip->ili_fields & XFS_ILOG_DDATA) && + ip->i_df.if_bytes > 0) { + *nbytes += xlog_calc_iovec_len(ip->i_df.if_bytes); + *nvecs += 1; + } + break; + + case XFS_DINODE_FMT_DEV: + break; + default: + ASSERT(0); + break; + } +} + +STATIC void +xfs_inode_item_attr_fork_size( + struct xfs_inode_log_item *iip, + int *nvecs, + int *nbytes) +{ + struct xfs_inode *ip = iip->ili_inode; + + switch (ip->i_af.if_format) { + case XFS_DINODE_FMT_EXTENTS: + if ((iip->ili_fields & XFS_ILOG_AEXT) && + ip->i_af.if_nextents > 0 && + ip->i_af.if_bytes > 0) { + /* worst case, doesn't subtract unused space */ + *nbytes += xfs_inode_attr_fork_size(ip); + *nvecs += 1; + } + break; + case XFS_DINODE_FMT_BTREE: + if ((iip->ili_fields & XFS_ILOG_ABROOT) && + ip->i_af.if_broot_bytes > 0) { + *nbytes += ip->i_af.if_broot_bytes; + *nvecs += 1; + } + break; + case XFS_DINODE_FMT_LOCAL: + if ((iip->ili_fields & XFS_ILOG_ADATA) && + ip->i_af.if_bytes > 0) { + *nbytes += xlog_calc_iovec_len(ip->i_af.if_bytes); + *nvecs += 1; + } + break; + default: + ASSERT(0); + break; + } +} + +/* + * This returns the number of iovecs needed to log the given inode item. + * + * We need one iovec for the inode log format structure, one for the + * inode core, and possibly one for the inode data/extents/b-tree root + * and one for the inode attribute data/extents/b-tree root. + */ +STATIC void +xfs_inode_item_size( + struct xfs_log_item *lip, + int *nvecs, + int *nbytes) +{ + struct xfs_inode_log_item *iip = INODE_ITEM(lip); + struct xfs_inode *ip = iip->ili_inode; + + *nvecs += 2; + *nbytes += sizeof(struct xfs_inode_log_format) + + xfs_log_dinode_size(ip->i_mount); + + xfs_inode_item_data_fork_size(iip, nvecs, nbytes); + if (xfs_inode_has_attr_fork(ip)) + xfs_inode_item_attr_fork_size(iip, nvecs, nbytes); +} + +STATIC void +xfs_inode_item_format_data_fork( + struct xfs_inode_log_item *iip, + struct xfs_inode_log_format *ilf, + struct xlog_format_buf *lfb) +{ + struct xfs_inode *ip = iip->ili_inode; + size_t data_bytes; + + switch (ip->i_df.if_format) { + case XFS_DINODE_FMT_EXTENTS: + iip->ili_fields &= + ~(XFS_ILOG_DDATA | XFS_ILOG_DBROOT | XFS_ILOG_DEV); + + if ((iip->ili_fields & XFS_ILOG_DEXT) && + ip->i_df.if_nextents > 0 && + ip->i_df.if_bytes > 0) { + struct xfs_bmbt_rec *p; + + ASSERT(xfs_iext_count(&ip->i_df) > 0); + + p = xlog_format_start(lfb, XLOG_REG_TYPE_IEXT); + data_bytes = xfs_iextents_copy(ip, p, XFS_DATA_FORK); + xlog_format_commit(lfb, data_bytes); + + ASSERT(data_bytes <= ip->i_df.if_bytes); + + ilf->ilf_dsize = data_bytes; + ilf->ilf_size++; + } else { + iip->ili_fields &= ~XFS_ILOG_DEXT; + } + break; + case XFS_DINODE_FMT_BTREE: + case XFS_DINODE_FMT_META_BTREE: + iip->ili_fields &= + ~(XFS_ILOG_DDATA | XFS_ILOG_DEXT | XFS_ILOG_DEV); + + if ((iip->ili_fields & XFS_ILOG_DBROOT) && + ip->i_df.if_broot_bytes > 0) { + ASSERT(ip->i_df.if_broot != NULL); + xlog_format_copy(lfb, XLOG_REG_TYPE_IBROOT, + ip->i_df.if_broot, + ip->i_df.if_broot_bytes); + ilf->ilf_dsize = ip->i_df.if_broot_bytes; + ilf->ilf_size++; + } else { + ASSERT(!(iip->ili_fields & + XFS_ILOG_DBROOT)); + iip->ili_fields &= ~XFS_ILOG_DBROOT; + } + break; + case XFS_DINODE_FMT_LOCAL: + iip->ili_fields &= + ~(XFS_ILOG_DEXT | XFS_ILOG_DBROOT | XFS_ILOG_DEV); + if ((iip->ili_fields & XFS_ILOG_DDATA) && + ip->i_df.if_bytes > 0) { + ASSERT(ip->i_df.if_data != NULL); + ASSERT(ip->i_disk_size > 0); + xlog_format_copy(lfb, XLOG_REG_TYPE_ILOCAL, + ip->i_df.if_data, + ip->i_df.if_bytes); + ilf->ilf_dsize = (unsigned)ip->i_df.if_bytes; + ilf->ilf_size++; + } else { + iip->ili_fields &= ~XFS_ILOG_DDATA; + } + break; + case XFS_DINODE_FMT_DEV: + iip->ili_fields &= + ~(XFS_ILOG_DDATA | XFS_ILOG_DBROOT | XFS_ILOG_DEXT); + if (iip->ili_fields & XFS_ILOG_DEV) + ilf->ilf_u.ilfu_rdev = sysv_encode_dev(VFS_I(ip)->i_rdev); + break; + default: + ASSERT(0); + break; + } +} + +STATIC void +xfs_inode_item_format_attr_fork( + struct xfs_inode_log_item *iip, + struct xfs_inode_log_format *ilf, + struct xlog_format_buf *lfb) +{ + struct xfs_inode *ip = iip->ili_inode; + size_t data_bytes; + + switch (ip->i_af.if_format) { + case XFS_DINODE_FMT_EXTENTS: + iip->ili_fields &= + ~(XFS_ILOG_ADATA | XFS_ILOG_ABROOT); + + if ((iip->ili_fields & XFS_ILOG_AEXT) && + ip->i_af.if_nextents > 0 && + ip->i_af.if_bytes > 0) { + struct xfs_bmbt_rec *p; + + ASSERT(xfs_iext_count(&ip->i_af) == + ip->i_af.if_nextents); + + p = xlog_format_start(lfb, XLOG_REG_TYPE_IATTR_EXT); + data_bytes = xfs_iextents_copy(ip, p, XFS_ATTR_FORK); + xlog_format_commit(lfb, data_bytes); + + ilf->ilf_asize = data_bytes; + ilf->ilf_size++; + } else { + iip->ili_fields &= ~XFS_ILOG_AEXT; + } + break; + case XFS_DINODE_FMT_BTREE: + iip->ili_fields &= + ~(XFS_ILOG_ADATA | XFS_ILOG_AEXT); + + if ((iip->ili_fields & XFS_ILOG_ABROOT) && + ip->i_af.if_broot_bytes > 0) { + ASSERT(ip->i_af.if_broot != NULL); + + xlog_format_copy(lfb, XLOG_REG_TYPE_IATTR_BROOT, + ip->i_af.if_broot, + ip->i_af.if_broot_bytes); + ilf->ilf_asize = ip->i_af.if_broot_bytes; + ilf->ilf_size++; + } else { + iip->ili_fields &= ~XFS_ILOG_ABROOT; + } + break; + case XFS_DINODE_FMT_LOCAL: + iip->ili_fields &= + ~(XFS_ILOG_AEXT | XFS_ILOG_ABROOT); + + if ((iip->ili_fields & XFS_ILOG_ADATA) && + ip->i_af.if_bytes > 0) { + ASSERT(ip->i_af.if_data != NULL); + xlog_format_copy(lfb, XLOG_REG_TYPE_IATTR_LOCAL, + ip->i_af.if_data, + ip->i_af.if_bytes); + ilf->ilf_asize = (unsigned)ip->i_af.if_bytes; + ilf->ilf_size++; + } else { + iip->ili_fields &= ~XFS_ILOG_ADATA; + } + break; + default: + ASSERT(0); + break; + } +} + +/* + * Convert an incore timestamp to a log timestamp. Note that the log format + * specifies host endian format! + */ +static inline xfs_log_timestamp_t +xfs_inode_to_log_dinode_ts( + struct xfs_inode *ip, + const struct timespec64 tv) +{ + struct xfs_log_legacy_timestamp *lits; + xfs_log_timestamp_t its; + + if (xfs_inode_has_bigtime(ip)) + return xfs_inode_encode_bigtime(tv); + + lits = (struct xfs_log_legacy_timestamp *)&its; + lits->t_sec = tv.tv_sec; + lits->t_nsec = tv.tv_nsec; + + return its; +} + +/* + * The legacy DMAPI fields are only present in the on-disk and in-log inodes, + * but not in the in-memory one. But we are guaranteed to have an inode buffer + * in memory when logging an inode, so we can just copy it from the on-disk + * inode to the in-log inode here so that recovery of file system with these + * fields set to non-zero values doesn't lose them. For all other cases we zero + * the fields. + */ +static void +xfs_copy_dm_fields_to_log_dinode( + struct xfs_inode *ip, + struct xfs_log_dinode *to) +{ + struct xfs_dinode *dip; + + dip = xfs_buf_offset(ip->i_itemp->ili_item.li_buf, + ip->i_imap.im_boffset); + + if (xfs_iflags_test(ip, XFS_IPRESERVE_DM_FIELDS)) { + to->di_dmevmask = be32_to_cpu(dip->di_dmevmask); + to->di_dmstate = be16_to_cpu(dip->di_dmstate); + } else { + to->di_dmevmask = 0; + to->di_dmstate = 0; + } +} + +static inline void +xfs_inode_to_log_dinode_iext_counters( + struct xfs_inode *ip, + struct xfs_log_dinode *to) +{ + if (xfs_inode_has_large_extent_counts(ip)) { + to->di_big_nextents = xfs_ifork_nextents(&ip->i_df); + to->di_big_anextents = xfs_ifork_nextents(&ip->i_af); + to->di_nrext64_pad = 0; + } else { + to->di_nextents = xfs_ifork_nextents(&ip->i_df); + to->di_anextents = xfs_ifork_nextents(&ip->i_af); + } +} + +static void +xfs_inode_to_log_dinode( + struct xfs_inode *ip, + struct xfs_log_dinode *to, + xfs_lsn_t lsn) +{ + struct inode *inode = VFS_I(ip); + + to->di_magic = XFS_DINODE_MAGIC; + to->di_format = xfs_ifork_format(&ip->i_df); + to->di_uid = i_uid_read(inode); + to->di_gid = i_gid_read(inode); + to->di_projid_lo = ip->i_projid & 0xffff; + to->di_projid_hi = ip->i_projid >> 16; + + to->di_atime = xfs_inode_to_log_dinode_ts(ip, inode_get_atime(inode)); + to->di_mtime = xfs_inode_to_log_dinode_ts(ip, inode_get_mtime(inode)); + to->di_ctime = xfs_inode_to_log_dinode_ts(ip, inode_get_ctime(inode)); + to->di_nlink = inode->i_nlink; + to->di_gen = inode->i_generation; + to->di_mode = inode->i_mode; + + to->di_size = ip->i_disk_size; + to->di_nblocks = ip->i_nblocks; + to->di_extsize = ip->i_extsize; + to->di_forkoff = ip->i_forkoff; + to->di_aformat = xfs_ifork_format(&ip->i_af); + to->di_flags = ip->i_diflags; + + xfs_copy_dm_fields_to_log_dinode(ip, to); + + /* log a dummy value to ensure log structure is fully initialised */ + to->di_next_unlinked = NULLAGINO; + + if (xfs_has_v3inodes(ip->i_mount)) { + to->di_version = 3; + to->di_changecount = inode_peek_iversion(inode); + to->di_crtime = xfs_inode_to_log_dinode_ts(ip, ip->i_crtime); + to->di_flags2 = ip->i_diflags2; + /* also covers the di_used_blocks union arm: */ + to->di_cowextsize = ip->i_cowextsize; + to->di_ino = I_INO(ip); + to->di_lsn = lsn; + memset(to->di_pad2, 0, sizeof(to->di_pad2)); + uuid_copy(&to->di_uuid, &ip->i_mount->m_sb.sb_meta_uuid); + to->di_v3_pad = 0; + + /* dummy value for initialisation */ + to->di_crc = 0; + + if (xfs_is_metadir_inode(ip)) + to->di_metatype = ip->i_metatype; + else + to->di_metatype = 0; + } else { + to->di_version = 2; + to->di_flushiter = ip->i_flushiter; + memset(to->di_v2_pad, 0, sizeof(to->di_v2_pad)); + to->di_metatype = 0; + } + + xfs_inode_to_log_dinode_iext_counters(ip, to); +} + +/* + * Format the inode core. Current timestamp data is only in the VFS inode + * fields, so we need to grab them from there. Hence rather than just copying + * the XFS inode core structure, format the fields directly into the iovec. + */ +static void +xfs_inode_item_format_core( + struct xfs_inode *ip, + struct xlog_format_buf *lfb) +{ + struct xfs_log_dinode *dic; + + dic = xlog_format_start(lfb, XLOG_REG_TYPE_ICORE); + xfs_inode_to_log_dinode(ip, dic, ip->i_itemp->ili_item.li_lsn); + xlog_format_commit(lfb, xfs_log_dinode_size(ip->i_mount)); +} + +/* + * This is called to fill in the vector of log iovecs for the given inode + * log item. It fills the first item with an inode log format structure, + * the second with the on-disk inode structure, and a possible third and/or + * fourth with the inode data/extents/b-tree root and inode attributes + * data/extents/b-tree root. + * + * Note: Always use the 64 bit inode log format structure so we don't + * leave an uninitialised hole in the format item on 64 bit systems. Log + * recovery on 32 bit systems handles this just fine, so there's no reason + * for not using an initialising the properly padded structure all the time. + */ +STATIC void +xfs_inode_item_format( + struct xfs_log_item *lip, + struct xlog_format_buf *lfb) +{ + struct xfs_inode_log_item *iip = INODE_ITEM(lip); + struct xfs_inode *ip = iip->ili_inode; + struct xfs_mount *mp = ip->i_mount; + struct xfs_inode_log_format *ilf; + + ilf = xlog_format_start(lfb, XLOG_REG_TYPE_IFORMAT); + ilf->ilf_type = XFS_LI_INODE; + ilf->ilf_ino = I_INO(ip); + ilf->ilf_blkno = XFS_AGB_TO_DADDR(mp, XFS_INODE_TO_AGNO(ip), + ip->i_imap.im_agbno); + ilf->ilf_len = XFS_FSB_TO_BB(mp, M_IGEO(mp)->blocks_per_cluster); + ilf->ilf_boffset = ip->i_imap.im_boffset; + ilf->ilf_fields = XFS_ILOG_CORE; + ilf->ilf_size = 2; /* format + core */ + + /* + * make sure we don't leak uninitialised data into the log in the case + * when we don't log every field in the inode. + */ + ilf->ilf_dsize = 0; + ilf->ilf_asize = 0; + ilf->ilf_pad = 0; + memset(&ilf->ilf_u, 0, sizeof(ilf->ilf_u)); + xlog_format_commit(lfb, sizeof(*ilf)); + + xfs_inode_item_format_core(ip, lfb); + xfs_inode_item_format_data_fork(iip, ilf, lfb); + if (xfs_inode_has_attr_fork(ip)) { + xfs_inode_item_format_attr_fork(iip, ilf, lfb); + } else { + iip->ili_fields &= + ~(XFS_ILOG_ADATA | XFS_ILOG_ABROOT | XFS_ILOG_AEXT); + } + + /* update the format with the exact fields we actually logged */ + ilf->ilf_fields |= (iip->ili_fields & ~XFS_ILOG_TIMESTAMP); +} + +/* + * This is called to pin the inode associated with the inode log + * item in memory so it cannot be written out. + */ +STATIC void +xfs_inode_item_pin( + struct xfs_log_item *lip) +{ + struct xfs_inode *ip = INODE_ITEM(lip)->ili_inode; + + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + ASSERT(lip->li_buf); + + trace_xfs_inode_pin(ip, _RET_IP_); + atomic_inc(&ip->i_pincount); +} + + +/* + * This is called to unpin the inode associated with the inode log + * item which was previously pinned with a call to xfs_inode_item_pin(). + * + * Also wake up anyone in xfs_iunpin_wait() if the count goes to 0. + * + * Note that unpin can race with inode cluster buffer freeing marking the buffer + * stale. In that case, flush completions are run from the buffer unpin call, + * which may happen before the inode is unpinned. If we lose the race, there + * will be no buffer attached to the log item, but the inode will be marked + * XFS_ISTALE. + */ +STATIC void +xfs_inode_item_unpin( + struct xfs_log_item *lip, + int remove) +{ + struct xfs_inode_log_item *iip = INODE_ITEM(lip); + struct xfs_inode *ip = iip->ili_inode; + + trace_xfs_inode_unpin(ip, _RET_IP_); + ASSERT(lip->li_buf || xfs_iflags_test(ip, XFS_ISTALE)); + ASSERT(atomic_read(&ip->i_pincount) > 0); + + /* + * If this is the last unpin, then the inode no longer needs a journal + * flush to persist it. Hence we can clear the commit sequence numbers + * as a fsync/fdatasync operation on the inode at this point is a no-op. + */ + if (atomic_dec_and_lock(&ip->i_pincount, &iip->ili_lock)) { + iip->ili_commit_seq = 0; + iip->ili_datasync_seq = 0; + spin_unlock(&iip->ili_lock); + wake_up_bit(&ip->i_flags, __XFS_IPINNED_BIT); + } +} + +STATIC uint +xfs_inode_item_push( + struct xfs_log_item *lip, + struct list_head *buffer_list) + __releases(&lip->li_ailp->ail_lock) + __acquires(&lip->li_ailp->ail_lock) +{ + struct xfs_inode_log_item *iip = INODE_ITEM(lip); + struct xfs_inode *ip = iip->ili_inode; + struct xfs_buf *bp = lip->li_buf; + struct xfs_ail *ailp = lip->li_ailp; + uint rval = XFS_ITEM_SUCCESS; + int error; + + if (!bp || (ip->i_flags & XFS_ISTALE)) { + /* + * Inode item/buffer is being aborted due to cluster + * buffer deletion. Trigger a log force to have that operation + * completed and items removed from the AIL before the next push + * attempt. + */ + trace_xfs_inode_push_stale(ip, _RET_IP_); + return XFS_ITEM_PINNED; + } + + if (xfs_ipincount(ip) > 0 || xfs_buf_ispinned(bp)) { + trace_xfs_inode_push_pinned(ip, _RET_IP_); + return XFS_ITEM_PINNED; + } + + if (xfs_iflags_test(ip, XFS_IFLUSHING)) + return XFS_ITEM_FLUSHING; + + if (!xfs_buf_trylock(bp)) + return XFS_ITEM_LOCKED; + + spin_unlock(&ailp->ail_lock); + + /* + * We need to hold a reference for flushing the cluster buffer as it may + * fail the buffer without IO submission. In which case, we better get a + * reference for that completion because otherwise we don't get a + * reference for IO until we queue the buffer for delwri submission. + */ + xfs_buf_hold(bp); + error = xfs_iflush_cluster(bp); + if (!error) { + if (!xfs_buf_delwri_queue(bp, buffer_list)) + rval = XFS_ITEM_FLUSHING; + xfs_buf_relse(bp); + } else { + /* + * Release the buffer if we were unable to flush anything. On + * any other error, the buffer has already been released. + */ + if (error == -EAGAIN) + xfs_buf_relse(bp); + rval = XFS_ITEM_LOCKED; + } + + /* + * The buffer no longer protects the log item from reclaim, so + * do not reference lip after this point. + */ + spin_lock(&ailp->ail_lock); + return rval; +} + +/* + * Unlock the inode associated with the inode log item. + */ +STATIC void +xfs_inode_item_release( + struct xfs_log_item *lip) +{ + struct xfs_inode_log_item *iip = INODE_ITEM(lip); + struct xfs_inode *ip = iip->ili_inode; + unsigned short lock_flags; + + ASSERT(ip->i_itemp != NULL); + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + + lock_flags = iip->ili_lock_flags; + iip->ili_lock_flags = 0; + if (lock_flags) + xfs_iunlock(ip, lock_flags); +} + +/* + * This is called to find out where the oldest active copy of the inode log + * item in the on disk log resides now that the last log write of it completed + * at the given lsn. Since we always re-log all dirty data in an inode, the + * latest copy in the on disk log is the only one that matters. Therefore, + * simply return the given lsn. + * + * If the inode has been marked stale because the cluster is being freed, we + * don't want to (re-)insert this inode into the AIL. There is a race condition + * where the cluster buffer may be unpinned before the inode is inserted into + * the AIL during transaction committed processing. If the buffer is unpinned + * before the inode item has been committed and inserted, then it is possible + * for the buffer to be written and IO completes before the inode is inserted + * into the AIL. In that case, we'd be inserting a clean, stale inode into the + * AIL which will never get removed. It will, however, get reclaimed which + * triggers an assert in xfs_inode_free() complaining about freein an inode + * still in the AIL. + * + * To avoid this, just unpin the inode directly and return a LSN of -1 so the + * transaction committed code knows that it does not need to do any further + * processing on the item. + */ +STATIC xfs_lsn_t +xfs_inode_item_committed( + struct xfs_log_item *lip, + xfs_lsn_t lsn) +{ + struct xfs_inode_log_item *iip = INODE_ITEM(lip); + struct xfs_inode *ip = iip->ili_inode; + + if (xfs_iflags_test(ip, XFS_ISTALE)) { + xfs_inode_item_unpin(lip, 0); + return -1; + } + return lsn; +} + +/* + * The modification is now complete, so before we unlock the inode we need to + * update the commit sequence numbers for data integrity journal flushes. We + * always record the commit sequence number (ili_commit_seq) so that anything + * that needs a full journal sync will capture all of this modification. + * + * We then + * check if the changes will impact a datasync (O_DSYNC) journal flush. If the + * changes will require a datasync flush, then we also record the sequence in + * ili_datasync_seq. + * + * These commit sequence numbers will get cleared atomically with the inode being + * unpinned (i.e. pin count goes to zero), and so it will only be set when the + * inode is dirty in the journal. This removes the need for checking if the + * inode is pinned to determine if a journal flush is necessary, and hence + * removes the need for holding the ILOCK_SHARED in xfs_file_fsync() to + * serialise pin counts against commit sequence number updates. + * + */ +STATIC void +xfs_inode_item_committing( + struct xfs_log_item *lip, + xfs_csn_t seq) +{ + struct xfs_inode_log_item *iip = INODE_ITEM(lip); + + spin_lock(&iip->ili_lock); + iip->ili_commit_seq = seq; + if (iip->ili_dirty_flags & ~(XFS_ILOG_IVERSION | XFS_ILOG_TIMESTAMP)) + iip->ili_datasync_seq = seq; + spin_unlock(&iip->ili_lock); + + /* + * Clear the per-transaction dirty flags now that we have finished + * recording the transaction's inode modifications in the CIL and are + * about to release and (maybe) unlock the inode. + */ + iip->ili_dirty_flags = 0; + + return xfs_inode_item_release(lip); +} + +static const struct xfs_item_ops xfs_inode_item_ops = { + .iop_sort = xfs_inode_item_sort, + .iop_precommit = xfs_inode_item_precommit, + .iop_size = xfs_inode_item_size, + .iop_format = xfs_inode_item_format, + .iop_pin = xfs_inode_item_pin, + .iop_unpin = xfs_inode_item_unpin, + .iop_release = xfs_inode_item_release, + .iop_committed = xfs_inode_item_committed, + .iop_push = xfs_inode_item_push, + .iop_committing = xfs_inode_item_committing, +}; + + +/* + * Initialize the inode log item for a newly allocated (in-core) inode. + */ +void +xfs_inode_item_init( + struct xfs_inode *ip, + struct xfs_mount *mp) +{ + struct xfs_inode_log_item *iip; + + ASSERT(ip->i_itemp == NULL); + iip = ip->i_itemp = kmem_cache_zalloc(xfs_ili_cache, + GFP_KERNEL | __GFP_NOFAIL); + + iip->ili_inode = ip; + spin_lock_init(&iip->ili_lock); + xfs_log_item_init(mp, &iip->ili_item, XFS_LI_INODE, + &xfs_inode_item_ops); +} + +/* + * Free the inode log item and any memory hanging off of it. + */ +void +xfs_inode_item_destroy( + struct xfs_inode *ip) +{ + struct xfs_inode_log_item *iip = ip->i_itemp; + + ASSERT(iip->ili_item.li_buf == NULL); + + ip->i_itemp = NULL; + kvfree(iip->ili_item.li_lv_shadow); + kmem_cache_free(xfs_ili_cache, iip); +} + + +/* + * We only want to pull the item from the AIL if it is actually there + * and its location in the log has not changed since we started the + * flush. Thus, we only bother if the inode's lsn has not changed. + */ +static void +xfs_iflush_ail_updates( + struct xfs_ail *ailp, + struct list_head *list) +{ + struct xfs_log_item *lip; + xfs_lsn_t tail_lsn = 0; + + /* this is an opencoded batch version of xfs_trans_ail_delete */ + spin_lock(&ailp->ail_lock); + list_for_each_entry(lip, list, li_bio_list) { + xfs_lsn_t lsn; + + clear_bit(XFS_LI_FAILED, &lip->li_flags); + if (INODE_ITEM(lip)->ili_flush_lsn != lip->li_lsn) + continue; + + /* + * dgc: Not sure how this happens, but it happens very + * occassionaly via generic/388. xfs_iflush_abort() also + * silently handles this same "under writeback but not in AIL at + * shutdown" condition via xfs_trans_ail_delete(). + */ + if (!test_bit(XFS_LI_IN_AIL, &lip->li_flags)) { + ASSERT(xlog_is_shutdown(lip->li_log)); + continue; + } + + lsn = xfs_ail_delete_one(ailp, lip); + if (!tail_lsn && lsn) + tail_lsn = lsn; + } + xfs_ail_update_finish(ailp, tail_lsn); +} + +/* + * Walk the list of inodes that have completed their IOs. If they are clean + * remove them from the list and dissociate them from the buffer. Buffers that + * are still dirty remain linked to the buffer and on the list. Caller must + * handle them appropriately. + */ +static void +xfs_iflush_finish( + struct xfs_buf *bp, + struct list_head *list) +{ + struct xfs_log_item *lip, *n; + + list_for_each_entry_safe(lip, n, list, li_bio_list) { + struct xfs_inode_log_item *iip = INODE_ITEM(lip); + bool drop_buffer = false; + + spin_lock(&iip->ili_lock); + + /* + * Remove the reference to the cluster buffer if the inode is + * clean in memory and drop the buffer reference once we've + * dropped the locks we hold. + */ + ASSERT(iip->ili_item.li_buf == bp); + if (!iip->ili_fields) { + iip->ili_item.li_buf = NULL; + list_del_init(&lip->li_bio_list); + drop_buffer = true; + } + iip->ili_last_fields = 0; + iip->ili_flush_lsn = 0; + clear_bit(XFS_LI_FLUSHING, &lip->li_flags); + spin_unlock(&iip->ili_lock); + xfs_iflags_clear(iip->ili_inode, XFS_IFLUSHING); + if (drop_buffer) + xfs_buf_rele(bp); + } +} + +/* + * Inode buffer IO completion routine. It is responsible for removing inodes + * attached to the buffer from the AIL if they have not been re-logged and + * completing the inode flush. + */ +void +xfs_buf_inode_iodone( + struct xfs_buf *bp) +{ + struct xfs_log_item *lip, *n; + LIST_HEAD(flushed_inodes); + LIST_HEAD(ail_updates); + + /* + * Pull the attached inodes from the buffer one at a time and take the + * appropriate action on them. + */ + list_for_each_entry_safe(lip, n, &bp->b_li_list, li_bio_list) { + struct xfs_inode_log_item *iip = INODE_ITEM(lip); + + if (xfs_iflags_test(iip->ili_inode, XFS_ISTALE)) { + xfs_iflush_abort(iip->ili_inode); + continue; + } + if (!iip->ili_last_fields) + continue; + + /* Do an unlocked check for needing the AIL lock. */ + if (iip->ili_flush_lsn == lip->li_lsn || + test_bit(XFS_LI_FAILED, &lip->li_flags)) + list_move_tail(&lip->li_bio_list, &ail_updates); + else + list_move_tail(&lip->li_bio_list, &flushed_inodes); + } + + if (!list_empty(&ail_updates)) { + xfs_iflush_ail_updates(bp->b_mount->m_ail, &ail_updates); + list_splice_tail(&ail_updates, &flushed_inodes); + } + + xfs_iflush_finish(bp, &flushed_inodes); + if (!list_empty(&flushed_inodes)) + list_splice_tail(&flushed_inodes, &bp->b_li_list); +} + +/* + * Clear the inode logging fields so no more flushes are attempted. If we are + * on a buffer list, it is now safe to remove it because the buffer is + * guaranteed to be locked. The caller will drop the reference to the buffer + * the log item held. + */ +static void +xfs_iflush_abort_clean( + struct xfs_inode_log_item *iip) +{ + iip->ili_last_fields = 0; + iip->ili_fields = 0; + iip->ili_flush_lsn = 0; + iip->ili_item.li_buf = NULL; + list_del_init(&iip->ili_item.li_bio_list); + clear_bit(XFS_LI_FLUSHING, &iip->ili_item.li_flags); +} + +/* + * Abort flushing the inode from a context holding the cluster buffer locked. + * + * This is the normal runtime method of aborting writeback of an inode that is + * attached to a cluster buffer. It occurs when the inode and the backing + * cluster buffer have been freed (i.e. inode is XFS_ISTALE), or when cluster + * flushing or buffer IO completion encounters a log shutdown situation. + * + * If we need to abort inode writeback and we don't already hold the buffer + * locked, call xfs_iflush_shutdown_abort() instead as this should only ever be + * necessary in a shutdown situation. + */ +void +xfs_iflush_abort( + struct xfs_inode *ip) +{ + struct xfs_inode_log_item *iip = ip->i_itemp; + struct xfs_buf *bp; + + if (!iip) { + /* clean inode, nothing to do */ + xfs_iflags_clear(ip, XFS_IFLUSHING); + return; + } + + /* + * Remove the inode item from the AIL before we clear its internal + * state. Whilst the inode is in the AIL, it should have a valid buffer + * pointer for push operations to access - it is only safe to remove the + * inode from the buffer once it has been removed from the AIL. + */ + xfs_trans_ail_delete(&iip->ili_item, 0); + + /* + * Grab the inode buffer so can we release the reference the inode log + * item holds on it. + */ + spin_lock(&iip->ili_lock); + bp = iip->ili_item.li_buf; + xfs_iflush_abort_clean(iip); + spin_unlock(&iip->ili_lock); + + xfs_iflags_clear(ip, XFS_IFLUSHING); + if (bp) + xfs_buf_rele(bp); +} + +/* + * Abort an inode flush in the case of a shutdown filesystem. This can be called + * from anywhere with just an inode reference and does not require holding the + * inode cluster buffer locked. If the inode is attached to a cluster buffer, + * it will grab and lock it safely, then abort the inode flush. + */ +void +xfs_iflush_shutdown_abort( + struct xfs_inode *ip) +{ + struct xfs_inode_log_item *iip = ip->i_itemp; + struct xfs_buf *bp; + + if (!iip) { + /* clean inode, nothing to do */ + xfs_iflags_clear(ip, XFS_IFLUSHING); + return; + } + + spin_lock(&iip->ili_lock); + bp = iip->ili_item.li_buf; + if (!bp) { + spin_unlock(&iip->ili_lock); + xfs_iflush_abort(ip); + return; + } + + /* + * We have to take a reference to the buffer so that it doesn't get + * freed when we drop the ili_lock and then wait to lock the buffer. + * We'll clean up the extra reference after we pick up the ili_lock + * again. + */ + xfs_buf_hold(bp); + spin_unlock(&iip->ili_lock); + xfs_buf_lock(bp); + + spin_lock(&iip->ili_lock); + if (!iip->ili_item.li_buf) { + /* + * Raced with another removal, hold the only reference + * to bp now. Inode should not be in the AIL now, so just clean + * up and return; + */ + ASSERT(list_empty(&iip->ili_item.li_bio_list)); + ASSERT(!test_bit(XFS_LI_IN_AIL, &iip->ili_item.li_flags)); + xfs_iflush_abort_clean(iip); + spin_unlock(&iip->ili_lock); + xfs_iflags_clear(ip, XFS_IFLUSHING); + xfs_buf_relse(bp); + return; + } + + /* + * Got two references to bp. The first will get dropped by + * xfs_iflush_abort() when the item is removed from the buffer list, but + * we can't drop our reference until _abort() returns because we have to + * unlock the buffer as well. Hence we abort and then unlock and release + * our reference to the buffer. + */ + ASSERT(iip->ili_item.li_buf == bp); + spin_unlock(&iip->ili_lock); + xfs_iflush_abort(ip); + xfs_buf_relse(bp); +} + + +/* + * convert an xfs_inode_log_format struct from the old 32 bit version + * (which can have different field alignments) to the native 64 bit version + */ +int +xfs_inode_item_format_convert( + struct kvec *buf, + struct xfs_inode_log_format *in_f) +{ + struct xfs_inode_log_format_32 *in_f32 = buf->iov_base; + + if (buf->iov_len != sizeof(*in_f32)) { + XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, NULL); + return -EFSCORRUPTED; + } + + in_f->ilf_type = in_f32->ilf_type; + in_f->ilf_size = in_f32->ilf_size; + in_f->ilf_fields = in_f32->ilf_fields; + in_f->ilf_asize = in_f32->ilf_asize; + in_f->ilf_dsize = in_f32->ilf_dsize; + in_f->ilf_ino = in_f32->ilf_ino; + memcpy(&in_f->ilf_u, &in_f32->ilf_u, sizeof(in_f->ilf_u)); + in_f->ilf_blkno = in_f32->ilf_blkno; + in_f->ilf_len = in_f32->ilf_len; + in_f->ilf_boffset = in_f32->ilf_boffset; + return 0; +} diff --git a/fs/xfs/xfs_inode_item.h b/fs/xfs/xfs_inode_item.h new file mode 100644 index 000000000..2ddcca417 --- /dev/null +++ b/fs/xfs/xfs_inode_item.h @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_INODE_ITEM_H__ +#define __XFS_INODE_ITEM_H__ + +/* kernel only definitions */ + +struct xfs_buf; +struct xfs_bmbt_rec; +struct xfs_inode; +struct xfs_mount; + +struct xfs_inode_log_item { + struct xfs_log_item ili_item; /* common portion */ + struct xfs_inode *ili_inode; /* inode ptr */ + unsigned short ili_lock_flags; /* inode lock flags */ + unsigned int ili_dirty_flags; /* dirty in current tx */ + /* + * The ili_lock protects the interactions between the dirty state and + * the flush state of the inode log item. This allows us to do atomic + * modifications of multiple state fields without having to hold a + * specific inode lock to serialise them. + * + * We need atomic changes between inode dirtying, inode flushing and + * inode completion, but these all hold different combinations of + * ILOCK and IFLUSHING and hence we need some other method of + * serialising updates to the flush state. + */ + spinlock_t ili_lock; /* flush state lock */ + unsigned int ili_last_fields; /* fields when flushed */ + unsigned int ili_fields; /* fields to be logged */ + xfs_lsn_t ili_flush_lsn; /* lsn at last flush */ + + /* + * We record the sequence number for every inode modification, as + * well as those that only require fdatasync operations for data + * integrity. This allows optimisation of the O_DSYNC/fdatasync path + * without needing to track what modifications the journal is currently + * carrying for the inode. These are protected by the above ili_lock. + */ + xfs_csn_t ili_commit_seq; /* last transaction commit */ + xfs_csn_t ili_datasync_seq; /* for datasync optimisation */ +}; + +static inline int xfs_inode_clean(struct xfs_inode *ip) +{ + return !ip->i_itemp || !(ip->i_itemp->ili_fields & XFS_ILOG_ALL); +} + +extern void xfs_inode_item_init(struct xfs_inode *, struct xfs_mount *); +extern void xfs_inode_item_destroy(struct xfs_inode *); +extern void xfs_iflush_abort(struct xfs_inode *); +extern void xfs_iflush_shutdown_abort(struct xfs_inode *); +int xfs_inode_item_format_convert(struct kvec *buf, + struct xfs_inode_log_format *in_f); + +extern struct kmem_cache *xfs_ili_cache; + +#endif /* __XFS_INODE_ITEM_H__ */ diff --git a/fs/xfs/xfs_inode_item_recover.c b/fs/xfs/xfs_inode_item_recover.c new file mode 100644 index 000000000..1d2319ad1 --- /dev/null +++ b/fs/xfs/xfs_inode_item_recover.c @@ -0,0 +1,603 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2006 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_inode_item.h" +#include "xfs_trace.h" +#include "xfs_trans_priv.h" +#include "xfs_buf_item.h" +#include "xfs_log.h" +#include "xfs_error.h" +#include "xfs_log_priv.h" +#include "xfs_log_recover.h" +#include "xfs_icache.h" +#include "xfs_bmap_btree.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_rtrefcount_btree.h" + +STATIC void +xlog_recover_inode_ra_pass2( + struct xlog *log, + struct xlog_recover_item *item) +{ + if (item->ri_buf[0].iov_len == sizeof(struct xfs_inode_log_format)) { + struct xfs_inode_log_format *ilfp = item->ri_buf[0].iov_base; + + xlog_buf_readahead(log, ilfp->ilf_blkno, ilfp->ilf_len, + &xfs_inode_buf_ra_ops); + } else { + struct xfs_inode_log_format_32 *ilfp = item->ri_buf[0].iov_base; + + xlog_buf_readahead(log, ilfp->ilf_blkno, ilfp->ilf_len, + &xfs_inode_buf_ra_ops); + } +} + +/* + * Inode fork owner changes + * + * If we have been told that we have to reparent the inode fork, it's because an + * extent swap operation on a CRC enabled filesystem has been done and we are + * replaying it. We need to walk the BMBT of the appropriate fork and change the + * owners of it. + * + * The complexity here is that we don't have an inode context to work with, so + * after we've replayed the inode we need to instantiate one. This is where the + * fun begins. + * + * We are in the middle of log recovery, so we can't run transactions. That + * means we cannot use cache coherent inode instantiation via xfs_iget(), as + * that will result in the corresponding iput() running the inode through + * xfs_inactive(). If we've just replayed an inode core that changes the link + * count to zero (i.e. it's been unlinked), then xfs_inactive() will run + * transactions (bad!). + * + * So, to avoid this, we instantiate an inode directly from the inode core we've + * just recovered. We have the buffer still locked, and all we really need to + * instantiate is the inode core and the forks being modified. We can do this + * manually, then run the inode btree owner change, and then tear down the + * xfs_inode without having to run any transactions at all. + * + * Also, because we don't have a transaction context available here but need to + * gather all the buffers we modify for writeback so we pass the buffer_list + * instead for the operation to use. + */ + +STATIC int +xfs_recover_inode_owner_change( + struct xfs_mount *mp, + struct xfs_dinode *dip, + struct xfs_inode_log_format *in_f, + struct list_head *buffer_list) +{ + struct xfs_inode *ip; + int error; + + ASSERT(in_f->ilf_fields & (XFS_ILOG_DOWNER|XFS_ILOG_AOWNER)); + + ip = xfs_inode_alloc(mp, in_f->ilf_ino); + if (!ip) + return -ENOMEM; + + /* instantiate the inode */ + ASSERT(dip->di_version >= 3); + + error = xfs_inode_from_disk(ip, dip); + if (error) + goto out_free_ip; + + if (in_f->ilf_fields & XFS_ILOG_DOWNER) { + ASSERT(in_f->ilf_fields & XFS_ILOG_DBROOT); + error = xfs_bmbt_change_owner(NULL, ip, XFS_DATA_FORK, + I_INO(ip), buffer_list); + if (error) + goto out_free_ip; + } + + if (in_f->ilf_fields & XFS_ILOG_AOWNER) { + ASSERT(in_f->ilf_fields & XFS_ILOG_ABROOT); + error = xfs_bmbt_change_owner(NULL, ip, XFS_ATTR_FORK, + I_INO(ip), buffer_list); + if (error) + goto out_free_ip; + } + +out_free_ip: + xfs_inode_free(ip); + return error; +} + +static inline bool xfs_log_dinode_has_bigtime(const struct xfs_log_dinode *ld) +{ + return ld->di_version >= 3 && + (ld->di_flags2 & XFS_DIFLAG2_BIGTIME); +} + +/* Convert a log timestamp to an ondisk timestamp. */ +static inline xfs_timestamp_t +xfs_log_dinode_to_disk_ts( + struct xfs_log_dinode *from, + const xfs_log_timestamp_t its) +{ + struct xfs_legacy_timestamp *lts; + struct xfs_log_legacy_timestamp *lits; + xfs_timestamp_t ts; + + if (xfs_log_dinode_has_bigtime(from)) + return cpu_to_be64(its); + + lts = (struct xfs_legacy_timestamp *)&ts; + lits = (struct xfs_log_legacy_timestamp *)&its; + lts->t_sec = cpu_to_be32(lits->t_sec); + lts->t_nsec = cpu_to_be32(lits->t_nsec); + + return ts; +} + +static inline bool xfs_log_dinode_has_large_extent_counts( + const struct xfs_log_dinode *ld) +{ + return ld->di_version >= 3 && + (ld->di_flags2 & XFS_DIFLAG2_NREXT64); +} + +static inline void +xfs_log_dinode_to_disk_iext_counters( + struct xfs_log_dinode *from, + struct xfs_dinode *to) +{ + if (xfs_log_dinode_has_large_extent_counts(from)) { + to->di_big_nextents = cpu_to_be64(from->di_big_nextents); + to->di_big_anextents = cpu_to_be32(from->di_big_anextents); + to->di_nrext64_pad = cpu_to_be16(from->di_nrext64_pad); + } else { + to->di_nextents = cpu_to_be32(from->di_nextents); + to->di_anextents = cpu_to_be16(from->di_anextents); + } + +} + +STATIC void +xfs_log_dinode_to_disk( + struct xfs_log_dinode *from, + struct xfs_dinode *to, + xfs_lsn_t lsn) +{ + to->di_magic = cpu_to_be16(from->di_magic); + to->di_mode = cpu_to_be16(from->di_mode); + to->di_version = from->di_version; + to->di_format = from->di_format; + to->di_metatype = cpu_to_be16(from->di_metatype); + to->di_uid = cpu_to_be32(from->di_uid); + to->di_gid = cpu_to_be32(from->di_gid); + to->di_nlink = cpu_to_be32(from->di_nlink); + to->di_projid_lo = cpu_to_be16(from->di_projid_lo); + to->di_projid_hi = cpu_to_be16(from->di_projid_hi); + + to->di_atime = xfs_log_dinode_to_disk_ts(from, from->di_atime); + to->di_mtime = xfs_log_dinode_to_disk_ts(from, from->di_mtime); + to->di_ctime = xfs_log_dinode_to_disk_ts(from, from->di_ctime); + + to->di_size = cpu_to_be64(from->di_size); + to->di_nblocks = cpu_to_be64(from->di_nblocks); + to->di_extsize = cpu_to_be32(from->di_extsize); + to->di_forkoff = from->di_forkoff; + to->di_aformat = from->di_aformat; + to->di_dmevmask = cpu_to_be32(from->di_dmevmask); + to->di_dmstate = cpu_to_be16(from->di_dmstate); + to->di_flags = cpu_to_be16(from->di_flags); + to->di_gen = cpu_to_be32(from->di_gen); + + if (from->di_version == 3) { + to->di_changecount = cpu_to_be64(from->di_changecount); + to->di_crtime = xfs_log_dinode_to_disk_ts(from, + from->di_crtime); + to->di_flags2 = cpu_to_be64(from->di_flags2); + /* also covers the di_used_blocks union arm: */ + to->di_cowextsize = cpu_to_be32(from->di_cowextsize); + to->di_ino = cpu_to_be64(from->di_ino); + to->di_lsn = cpu_to_be64(lsn); + memset(to->di_pad2, 0, sizeof(to->di_pad2)); + uuid_copy(&to->di_uuid, &from->di_uuid); + to->di_v3_pad = 0; + } else { + to->di_flushiter = cpu_to_be16(from->di_flushiter); + memset(to->di_v2_pad, 0, sizeof(to->di_v2_pad)); + } + + xfs_log_dinode_to_disk_iext_counters(from, to); +} + +STATIC int +xlog_dinode_verify_extent_counts( + struct xfs_mount *mp, + struct xfs_log_dinode *ldip) +{ + xfs_extnum_t nextents; + xfs_aextnum_t anextents; + + if (xfs_log_dinode_has_large_extent_counts(ldip)) { + if (!xfs_has_large_extent_counts(mp) || + (ldip->di_nrext64_pad != 0)) { + XFS_CORRUPTION_ERROR( + "Bad log dinode large extent count format", + XFS_ERRLEVEL_LOW, mp, ldip, sizeof(*ldip)); + xfs_alert(mp, + "Bad inode 0x%llx, large extent counts %d, padding 0x%x", + ldip->di_ino, xfs_has_large_extent_counts(mp), + ldip->di_nrext64_pad); + return -EFSCORRUPTED; + } + + nextents = ldip->di_big_nextents; + anextents = ldip->di_big_anextents; + } else { + if (ldip->di_version == 3 && ldip->di_v3_pad != 0) { + XFS_CORRUPTION_ERROR( + "Bad log dinode di_v3_pad", + XFS_ERRLEVEL_LOW, mp, ldip, sizeof(*ldip)); + xfs_alert(mp, + "Bad inode 0x%llx, di_v3_pad 0x%llx", + ldip->di_ino, ldip->di_v3_pad); + return -EFSCORRUPTED; + } + + nextents = ldip->di_nextents; + anextents = ldip->di_anextents; + } + + if (unlikely(nextents + anextents > ldip->di_nblocks)) { + XFS_CORRUPTION_ERROR("Bad log dinode extent counts", + XFS_ERRLEVEL_LOW, mp, ldip, sizeof(*ldip)); + xfs_alert(mp, + "Bad inode 0x%llx, large extent counts %d, nextents 0x%llx, anextents 0x%x, nblocks 0x%llx", + ldip->di_ino, xfs_has_large_extent_counts(mp), nextents, + anextents, ldip->di_nblocks); + return -EFSCORRUPTED; + } + + return 0; +} + +static inline int +xlog_recover_inode_dbroot( + struct xfs_mount *mp, + void *src, + unsigned int len, + struct xfs_dinode *dip) +{ + void *dfork = XFS_DFORK_DPTR(dip); + unsigned int dsize = XFS_DFORK_DSIZE(dip, mp); + + switch (dip->di_format) { + case XFS_DINODE_FMT_BTREE: + xfs_bmbt_to_bmdr(mp, src, len, dfork, dsize); + break; + case XFS_DINODE_FMT_META_BTREE: + switch (be16_to_cpu(dip->di_metatype)) { + case XFS_METAFILE_RTRMAP: + xfs_rtrmapbt_to_disk(mp, src, len, dfork, dsize); + return 0; + case XFS_METAFILE_RTREFCOUNT: + xfs_rtrefcountbt_to_disk(mp, src, len, dfork, dsize); + return 0; + default: + ASSERT(0); + return -EFSCORRUPTED; + } + break; + default: + ASSERT(0); + return -EFSCORRUPTED; + } + + return 0; +} + +STATIC int +xlog_recover_inode_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t current_lsn) +{ + struct xfs_inode_log_format *in_f; + struct xfs_mount *mp = log->l_mp; + struct xfs_buf *bp; + struct xfs_dinode *dip; + int len; + char *src; + char *dest; + int error; + int attr_index; + uint fields; + struct xfs_log_dinode *ldip; + uint isize; + int need_free = 0; + xfs_failaddr_t fa; + + if (item->ri_buf[0].iov_len == sizeof(struct xfs_inode_log_format)) { + in_f = item->ri_buf[0].iov_base; + } else { + in_f = kmalloc_obj(struct xfs_inode_log_format, + GFP_KERNEL | __GFP_NOFAIL); + need_free = 1; + error = xfs_inode_item_format_convert(&item->ri_buf[0], in_f); + if (error) + goto error; + } + + /* + * Inode buffers can be freed, look out for it, + * and do not replay the inode. + */ + if (xlog_is_buffer_cancelled(log, in_f->ilf_blkno, in_f->ilf_len)) { + error = 0; + trace_xfs_log_recover_inode_cancel(log, in_f); + goto error; + } + trace_xfs_log_recover_inode_recover(log, in_f); + + error = xfs_buf_read(mp->m_ddev_targp, in_f->ilf_blkno, in_f->ilf_len, + 0, &bp, &xfs_inode_buf_ops); + if (error) + goto error; + ASSERT(in_f->ilf_fields & XFS_ILOG_CORE); + dip = xfs_buf_offset(bp, in_f->ilf_boffset); + + /* + * Make sure the place we're flushing out to really looks + * like an inode! + */ + if (XFS_IS_CORRUPT(mp, !xfs_verify_magic16(bp, dip->di_magic))) { + xfs_alert(mp, + "%s: Bad inode magic number, dip = "PTR_FMT", dino bp = "PTR_FMT", ino = %lld", + __func__, dip, bp, in_f->ilf_ino); + error = -EFSCORRUPTED; + goto out_release; + } + ldip = item->ri_buf[1].iov_base; + if (XFS_IS_CORRUPT(mp, ldip->di_magic != XFS_DINODE_MAGIC)) { + xfs_alert(mp, + "%s: Bad inode log record, rec ptr "PTR_FMT", ino %lld", + __func__, item, in_f->ilf_ino); + error = -EFSCORRUPTED; + goto out_release; + } + + /* + * If the inode has an LSN in it, recover the inode only if the on-disk + * inode's LSN is older than the lsn of the transaction we are + * replaying. We can have multiple checkpoints with the same start LSN, + * so the current LSN being equal to the on-disk LSN doesn't necessarily + * mean that the on-disk inode is more recent than the change being + * replayed. + * + * We must check the current_lsn against the on-disk inode + * here because the we can't trust the log dinode to contain a valid LSN + * (see comment below before replaying the log dinode for details). + * + * Note: we still need to replay an owner change even though the inode + * is more recent than the transaction as there is no guarantee that all + * the btree blocks are more recent than this transaction, too. + */ + if (dip->di_version >= 3) { + xfs_lsn_t lsn = be64_to_cpu(dip->di_lsn); + + if (lsn && lsn != -1 && XFS_LSN_CMP(lsn, current_lsn) > 0) { + trace_xfs_log_recover_inode_skip(log, in_f); + error = 0; + goto out_owner_change; + } + } + + /* + * di_flushiter is only valid for v1/2 inodes. All changes for v3 inodes + * are transactional and if ordering is necessary we can determine that + * more accurately by the LSN field in the V3 inode core. Don't trust + * the inode versions we might be changing them here - use the + * superblock flag to determine whether we need to look at di_flushiter + * to skip replay when the on disk inode is newer than the log one + */ + if (!xfs_has_v3inodes(mp)) { + if (ldip->di_flushiter < be16_to_cpu(dip->di_flushiter)) { + /* + * Deal with the wrap case, DI_MAX_FLUSH is less + * than smaller numbers + */ + if (be16_to_cpu(dip->di_flushiter) == DI_MAX_FLUSH && + ldip->di_flushiter < (DI_MAX_FLUSH >> 1)) { + /* do nothing */ + } else { + trace_xfs_log_recover_inode_skip(log, in_f); + error = 0; + goto out_release; + } + } + + /* Take the opportunity to reset the flush iteration count */ + ldip->di_flushiter = 0; + } + + + if (unlikely(S_ISREG(ldip->di_mode))) { + if (ldip->di_format != XFS_DINODE_FMT_EXTENTS && + ldip->di_format != XFS_DINODE_FMT_BTREE && + ldip->di_format != XFS_DINODE_FMT_META_BTREE) { + XFS_CORRUPTION_ERROR( + "Bad log dinode data fork format for regular file", + XFS_ERRLEVEL_LOW, mp, ldip, sizeof(*ldip)); + xfs_alert(mp, + "Bad inode 0x%llx, data fork format 0x%x", + in_f->ilf_ino, ldip->di_format); + error = -EFSCORRUPTED; + goto out_release; + } + } else if (unlikely(S_ISDIR(ldip->di_mode))) { + if ((ldip->di_format != XFS_DINODE_FMT_EXTENTS) && + (ldip->di_format != XFS_DINODE_FMT_BTREE) && + (ldip->di_format != XFS_DINODE_FMT_LOCAL)) { + XFS_CORRUPTION_ERROR( + "Bad log dinode data fork format for directory", + XFS_ERRLEVEL_LOW, mp, ldip, sizeof(*ldip)); + xfs_alert(mp, + "Bad inode 0x%llx, data fork format 0x%x", + in_f->ilf_ino, ldip->di_format); + error = -EFSCORRUPTED; + goto out_release; + } + } + + error = xlog_dinode_verify_extent_counts(mp, ldip); + if (error) + goto out_release; + + if (unlikely(ldip->di_forkoff > mp->m_sb.sb_inodesize)) { + XFS_CORRUPTION_ERROR("Bad log dinode fork offset", + XFS_ERRLEVEL_LOW, mp, ldip, sizeof(*ldip)); + xfs_alert(mp, + "Bad inode 0x%llx, di_forkoff 0x%x", + in_f->ilf_ino, ldip->di_forkoff); + error = -EFSCORRUPTED; + goto out_release; + } + isize = xfs_log_dinode_size(mp); + if (unlikely(item->ri_buf[1].iov_len > isize)) { + XFS_CORRUPTION_ERROR("Bad log dinode size", XFS_ERRLEVEL_LOW, + mp, ldip, sizeof(*ldip)); + xfs_alert(mp, + "Bad inode 0x%llx log dinode size 0x%zx", + in_f->ilf_ino, item->ri_buf[1].iov_len); + error = -EFSCORRUPTED; + goto out_release; + } + + /* + * Recover the log dinode inode into the on disk inode. + * + * The LSN in the log dinode is garbage - it can be zero or reflect + * stale in-memory runtime state that isn't coherent with the changes + * logged in this transaction or the changes written to the on-disk + * inode. Hence we write the current lSN into the inode because that + * matches what xfs_iflush() would write inode the inode when flushing + * the changes in this transaction. + */ + xfs_log_dinode_to_disk(ldip, dip, current_lsn); + + fields = in_f->ilf_fields; + if (fields & XFS_ILOG_DEV) + xfs_dinode_put_rdev(dip, in_f->ilf_u.ilfu_rdev); + + if (in_f->ilf_size == 2) + goto out_owner_change; + len = item->ri_buf[2].iov_len; + src = item->ri_buf[2].iov_base; + ASSERT(in_f->ilf_size <= 4); + ASSERT((in_f->ilf_size == 3) || (fields & XFS_ILOG_AFORK)); + ASSERT(!(fields & XFS_ILOG_DFORK) || + (len == xlog_calc_iovec_len(in_f->ilf_dsize))); + + switch (fields & XFS_ILOG_DFORK) { + case XFS_ILOG_DDATA: + case XFS_ILOG_DEXT: + memcpy(XFS_DFORK_DPTR(dip), src, len); + break; + + case XFS_ILOG_DBROOT: + error = xlog_recover_inode_dbroot(mp, src, len, dip); + if (error) + goto out_release; + break; + + default: + /* + * There are no data fork flags set. + */ + ASSERT((fields & XFS_ILOG_DFORK) == 0); + break; + } + + /* + * If we logged any attribute data, recover it. There may or + * may not have been any other non-core data logged in this + * transaction. + */ + if (in_f->ilf_fields & XFS_ILOG_AFORK) { + if (in_f->ilf_fields & XFS_ILOG_DFORK) { + attr_index = 3; + } else { + attr_index = 2; + } + len = item->ri_buf[attr_index].iov_len; + src = item->ri_buf[attr_index].iov_base; + ASSERT(len == xlog_calc_iovec_len(in_f->ilf_asize)); + + switch (in_f->ilf_fields & XFS_ILOG_AFORK) { + case XFS_ILOG_ADATA: + case XFS_ILOG_AEXT: + dest = XFS_DFORK_APTR(dip); + ASSERT(len <= XFS_DFORK_ASIZE(dip, mp)); + memcpy(dest, src, len); + break; + + case XFS_ILOG_ABROOT: + dest = XFS_DFORK_APTR(dip); + xfs_bmbt_to_bmdr(mp, (struct xfs_btree_block *)src, + len, (struct xfs_bmdr_block *)dest, + XFS_DFORK_ASIZE(dip, mp)); + break; + + default: + xfs_warn(log->l_mp, "%s: Invalid flag", __func__); + ASSERT(0); + error = -EFSCORRUPTED; + goto out_release; + } + } + +out_owner_change: + /* Recover the swapext owner change unless inode has been deleted */ + if ((in_f->ilf_fields & (XFS_ILOG_DOWNER|XFS_ILOG_AOWNER)) && + (dip->di_mode != 0)) + error = xfs_recover_inode_owner_change(mp, dip, in_f, + buffer_list); + /* re-generate the checksum and validate the recovered inode. */ + xfs_dinode_calc_crc(log->l_mp, dip); + fa = xfs_dinode_verify(log->l_mp, in_f->ilf_ino, dip); + if (fa) { + XFS_CORRUPTION_ERROR( + "Bad dinode after recovery", + XFS_ERRLEVEL_LOW, mp, dip, sizeof(*dip)); + xfs_alert(mp, + "Metadata corruption detected at %pS, inode 0x%llx", + fa, in_f->ilf_ino); + error = -EFSCORRUPTED; + goto out_release; + } + + ASSERT(bp->b_mount == mp); + xfs_buf_delwri_queue(bp, buffer_list); + +out_release: + xfs_buf_relse(bp); +error: + if (need_free) + kfree(in_f); + return error; +} + +const struct xlog_recover_item_ops xlog_inode_item_ops = { + .item_type = XFS_LI_INODE, + .ra_pass2 = xlog_recover_inode_ra_pass2, + .commit_pass2 = xlog_recover_inode_commit_pass2, +}; diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c new file mode 100644 index 000000000..1b53701be --- /dev/null +++ b/fs/xfs/xfs_ioctl.c @@ -0,0 +1,1472 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_rtalloc.h" +#include "xfs_iwalk.h" +#include "xfs_itable.h" +#include "xfs_error.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_attr.h" +#include "xfs_bmap.h" +#include "xfs_bmap_util.h" +#include "xfs_fsops.h" +#include "xfs_discard.h" +#include "xfs_quota.h" +#include "xfs_trace.h" +#include "xfs_icache.h" +#include "xfs_trans.h" +#include "xfs_btree.h" +#include +#include "xfs_fsmap.h" +#include "scrub/xfs_scrub.h" +#include "xfs_sb.h" +#include "xfs_ag.h" +#include "xfs_health.h" +#include "xfs_reflink.h" +#include "xfs_ioctl.h" +#include "xfs_xattr.h" +#include "xfs_rtbitmap.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_file.h" +#include "xfs_exchrange.h" +#include "xfs_handle.h" +#include "xfs_rtgroup.h" +#include "xfs_healthmon.h" +#include "xfs_verify_media.h" +#include "xfs_zone_priv.h" +#include "xfs_zone_alloc.h" + +#include +#include + +/* Return 0 on success or positive error */ +int +xfs_fsbulkstat_one_fmt( + struct xfs_ibulk *breq, + const struct xfs_bulkstat *bstat) +{ + struct xfs_bstat bs1; + + xfs_bulkstat_to_bstat(breq->mp, &bs1, bstat); + if (copy_to_user(breq->ubuffer, &bs1, sizeof(bs1))) + return -EFAULT; + return xfs_ibulk_advance(breq, sizeof(struct xfs_bstat)); +} + +int +xfs_fsinumbers_fmt( + struct xfs_ibulk *breq, + const struct xfs_inumbers *igrp) +{ + struct xfs_inogrp ig1; + + xfs_inumbers_to_inogrp(&ig1, igrp); + if (copy_to_user(breq->ubuffer, &ig1, sizeof(struct xfs_inogrp))) + return -EFAULT; + return xfs_ibulk_advance(breq, sizeof(struct xfs_inogrp)); +} + +STATIC int +xfs_ioc_fsbulkstat( + struct file *file, + unsigned int cmd, + void __user *arg) +{ + struct xfs_mount *mp = XFS_I(file_inode(file))->i_mount; + struct xfs_fsop_bulkreq bulkreq; + struct xfs_ibulk breq = { + .mp = mp, + .idmap = file_mnt_idmap(file), + .ocount = 0, + }; + xfs_ino_t lastino; + int error; + + /* done = 1 if there are more stats to get and if bulkstat */ + /* should be called again (unused here, but used in dmapi) */ + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (xfs_is_shutdown(mp)) + return -EIO; + + if (copy_from_user(&bulkreq, arg, sizeof(struct xfs_fsop_bulkreq))) + return -EFAULT; + + if (copy_from_user(&lastino, bulkreq.lastip, sizeof(__s64))) + return -EFAULT; + + if (bulkreq.icount <= 0) + return -EINVAL; + + if (bulkreq.ubuffer == NULL) + return -EINVAL; + + breq.ubuffer = bulkreq.ubuffer; + breq.icount = bulkreq.icount; + + /* + * FSBULKSTAT_SINGLE expects that *lastip contains the inode number + * that we want to stat. However, FSINUMBERS and FSBULKSTAT expect + * that *lastip contains either zero or the number of the last inode to + * be examined by the previous call and return results starting with + * the next inode after that. The new bulk request back end functions + * take the inode to start with, so we have to compute the startino + * parameter from lastino to maintain correct function. lastino == 0 + * is a special case because it has traditionally meant "first inode + * in filesystem". + */ + if (cmd == XFS_IOC_FSINUMBERS) { + breq.startino = lastino ? lastino + 1 : 0; + error = xfs_inumbers(&breq, xfs_fsinumbers_fmt); + lastino = breq.startino - 1; + } else if (cmd == XFS_IOC_FSBULKSTAT_SINGLE) { + breq.startino = lastino; + breq.icount = 1; + error = xfs_bulkstat_one(&breq, xfs_fsbulkstat_one_fmt); + } else { /* XFS_IOC_FSBULKSTAT */ + breq.startino = lastino ? lastino + 1 : 0; + error = xfs_bulkstat(&breq, xfs_fsbulkstat_one_fmt); + lastino = breq.startino - 1; + } + + if (error) + return error; + + if (bulkreq.lastip != NULL && + copy_to_user(bulkreq.lastip, &lastino, sizeof(xfs_ino_t))) + return -EFAULT; + + if (bulkreq.ocount != NULL && + copy_to_user(bulkreq.ocount, &breq.ocount, sizeof(__s32))) + return -EFAULT; + + return 0; +} + +/* Return 0 on success or positive error */ +static int +xfs_bulkstat_fmt( + struct xfs_ibulk *breq, + const struct xfs_bulkstat *bstat) +{ + if (copy_to_user(breq->ubuffer, bstat, sizeof(struct xfs_bulkstat))) + return -EFAULT; + return xfs_ibulk_advance(breq, sizeof(struct xfs_bulkstat)); +} + +/* + * Check the incoming bulk request @hdr from userspace and initialize the + * internal @breq bulk request appropriately. Returns 0 if the bulk request + * should proceed; -ECANCELED if there's nothing to do; or the usual + * negative error code. + */ +static int +xfs_bulk_ireq_setup( + struct xfs_mount *mp, + const struct xfs_bulk_ireq *hdr, + struct xfs_ibulk *breq, + void __user *ubuffer) +{ + if (hdr->icount == 0 || + (hdr->flags & ~XFS_BULK_IREQ_FLAGS_ALL) || + memchr_inv(hdr->reserved, 0, sizeof(hdr->reserved))) + return -EINVAL; + + breq->startino = hdr->ino; + breq->ubuffer = ubuffer; + breq->icount = hdr->icount; + breq->ocount = 0; + breq->flags = 0; + + /* + * The @ino parameter is a special value, so we must look it up here. + * We're not allowed to have IREQ_AGNO, and we only return one inode + * worth of data. + */ + if (hdr->flags & XFS_BULK_IREQ_SPECIAL) { + if (hdr->flags & XFS_BULK_IREQ_AGNO) + return -EINVAL; + + switch (hdr->ino) { + case XFS_BULK_IREQ_SPECIAL_ROOT: + breq->startino = mp->m_sb.sb_rootino; + break; + default: + return -EINVAL; + } + breq->icount = 1; + } + + /* + * The IREQ_AGNO flag means that we only want results from a given AG. + * If @hdr->ino is zero, we start iterating in that AG. If @hdr->ino is + * beyond the specified AG then we return no results. + */ + if (hdr->flags & XFS_BULK_IREQ_AGNO) { + if (hdr->agno >= mp->m_sb.sb_agcount) + return -EINVAL; + + if (breq->startino == 0) + breq->startino = XFS_AGINO_TO_INO(mp, hdr->agno, 0); + else if (XFS_INO_TO_AGNO(mp, breq->startino) < hdr->agno) + return -EINVAL; + + breq->iwalk_flags |= XFS_IWALK_SAME_AG; + + /* Asking for an inode past the end of the AG? We're done! */ + if (XFS_INO_TO_AGNO(mp, breq->startino) > hdr->agno) + return -ECANCELED; + } else if (hdr->agno) + return -EINVAL; + + /* Asking for an inode past the end of the FS? We're done! */ + if (XFS_INO_TO_AGNO(mp, breq->startino) >= mp->m_sb.sb_agcount) + return -ECANCELED; + + if (hdr->flags & XFS_BULK_IREQ_NREXT64) + breq->flags |= XFS_IBULK_NREXT64; + + /* Caller wants to see metadata directories in bulkstat output. */ + if (hdr->flags & XFS_BULK_IREQ_METADIR) + breq->flags |= XFS_IBULK_METADIR; + + return 0; +} + +/* + * Update the userspace bulk request @hdr to reflect the end state of the + * internal bulk request @breq. + */ +static void +xfs_bulk_ireq_teardown( + struct xfs_bulk_ireq *hdr, + struct xfs_ibulk *breq) +{ + hdr->ino = breq->startino; + hdr->ocount = breq->ocount; +} + +/* Handle the v5 bulkstat ioctl. */ +STATIC int +xfs_ioc_bulkstat( + struct file *file, + unsigned int cmd, + struct xfs_bulkstat_req __user *arg) +{ + struct xfs_mount *mp = XFS_I(file_inode(file))->i_mount; + struct xfs_bulk_ireq hdr; + struct xfs_ibulk breq = { + .mp = mp, + .idmap = file_mnt_idmap(file), + }; + int error; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (xfs_is_shutdown(mp)) + return -EIO; + + if (copy_from_user(&hdr, &arg->hdr, sizeof(hdr))) + return -EFAULT; + + error = xfs_bulk_ireq_setup(mp, &hdr, &breq, arg->bulkstat); + if (error == -ECANCELED) + goto out_teardown; + if (error < 0) + return error; + + error = xfs_bulkstat(&breq, xfs_bulkstat_fmt); + if (error) + return error; + +out_teardown: + xfs_bulk_ireq_teardown(&hdr, &breq); + if (copy_to_user(&arg->hdr, &hdr, sizeof(hdr))) + return -EFAULT; + + return 0; +} + +STATIC int +xfs_inumbers_fmt( + struct xfs_ibulk *breq, + const struct xfs_inumbers *igrp) +{ + if (copy_to_user(breq->ubuffer, igrp, sizeof(struct xfs_inumbers))) + return -EFAULT; + return xfs_ibulk_advance(breq, sizeof(struct xfs_inumbers)); +} + +/* Handle the v5 inumbers ioctl. */ +STATIC int +xfs_ioc_inumbers( + struct xfs_mount *mp, + unsigned int cmd, + struct xfs_inumbers_req __user *arg) +{ + struct xfs_bulk_ireq hdr; + struct xfs_ibulk breq = { + .mp = mp, + }; + int error; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (xfs_is_shutdown(mp)) + return -EIO; + + if (copy_from_user(&hdr, &arg->hdr, sizeof(hdr))) + return -EFAULT; + + if (hdr.flags & XFS_BULK_IREQ_METADIR) + return -EINVAL; + + error = xfs_bulk_ireq_setup(mp, &hdr, &breq, arg->inumbers); + if (error == -ECANCELED) + goto out_teardown; + if (error < 0) + return error; + + error = xfs_inumbers(&breq, xfs_inumbers_fmt); + if (error) + return error; + +out_teardown: + xfs_bulk_ireq_teardown(&hdr, &breq); + if (copy_to_user(&arg->hdr, &hdr, sizeof(hdr))) + return -EFAULT; + + return 0; +} + +STATIC int +xfs_ioc_fsgeometry( + struct xfs_mount *mp, + void __user *arg, + int struct_version) +{ + struct xfs_fsop_geom fsgeo; + size_t len; + + xfs_fs_geometry(mp, &fsgeo, struct_version); + + if (struct_version <= 3) + len = sizeof(struct xfs_fsop_geom_v1); + else if (struct_version == 4) + len = sizeof(struct xfs_fsop_geom_v4); + else { + xfs_fsop_geom_health(mp, &fsgeo); + len = sizeof(fsgeo); + } + + if (copy_to_user(arg, &fsgeo, len)) + return -EFAULT; + return 0; +} + +STATIC int +xfs_ioc_ag_geometry( + struct xfs_mount *mp, + void __user *arg) +{ + struct xfs_perag *pag; + struct xfs_ag_geometry ageo; + int error; + + if (copy_from_user(&ageo, arg, sizeof(ageo))) + return -EFAULT; + if (ageo.ag_flags) + return -EINVAL; + if (memchr_inv(&ageo.ag_reserved, 0, sizeof(ageo.ag_reserved))) + return -EINVAL; + + pag = xfs_perag_get(mp, ageo.ag_number); + if (!pag) + return -EINVAL; + + error = xfs_ag_get_geometry(pag, &ageo); + xfs_perag_put(pag); + if (error) + return error; + + if (copy_to_user(arg, &ageo, sizeof(ageo))) + return -EFAULT; + return 0; +} + +static void +xfs_rtgroup_report_write_pointer( + struct xfs_rtgroup *rtg, + struct xfs_rtgroup_geometry *rgeo) +{ + xfs_rtgroup_lock(rtg, XFS_RTGLOCK_RMAP); + if (rtg->rtg_open_zone) { + rgeo->rg_writepointer = rtg->rtg_open_zone->oz_allocated; + } else { + xfs_rgblock_t highest_rgbno = xfs_rtrmap_highest_rgbno(rtg); + + if (highest_rgbno == NULLRGBLOCK) + rgeo->rg_writepointer = 0; + else + rgeo->rg_writepointer = highest_rgbno + 1; + } + xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_RMAP); + rgeo->rg_flags |= XFS_RTGROUP_GEOM_WRITEPOINTER; +} + +STATIC int +xfs_ioc_rtgroup_geometry( + struct xfs_mount *mp, + void __user *arg) +{ + struct xfs_rtgroup *rtg; + struct xfs_rtgroup_geometry rgeo; + int error; + + if (copy_from_user(&rgeo, arg, sizeof(rgeo))) + return -EFAULT; + if (rgeo.rg_flags) + return -EINVAL; + if (memchr_inv(&rgeo.rg_reserved, 0, sizeof(rgeo.rg_reserved))) + return -EINVAL; + if (!xfs_has_rtgroups(mp)) + return -EINVAL; + + rtg = xfs_rtgroup_get(mp, rgeo.rg_number); + if (!rtg) + return -EINVAL; + + error = xfs_rtgroup_get_geometry(rtg, &rgeo); + if (error) + goto out_put_rtg; + if (xfs_has_zoned(mp)) + xfs_rtgroup_report_write_pointer(rtg, &rgeo); + + if (copy_to_user(arg, &rgeo, sizeof(rgeo))) + error = -EFAULT; +out_put_rtg: + xfs_rtgroup_put(rtg); + return error; +} + +/* + * Linux extended inode flags interface. + */ + +static void +xfs_fill_fsxattr( + struct xfs_inode *ip, + int whichfork, + struct file_kattr *fa) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + + fileattr_fill_xflags(fa, xfs_ip2xflags(ip)); + + if (ip->i_diflags & XFS_DIFLAG_EXTSIZE) { + fa->fsx_extsize = XFS_FSB_TO_B(mp, ip->i_extsize); + } else if (ip->i_diflags & XFS_DIFLAG_EXTSZINHERIT) { + /* + * Don't let a misaligned extent size hint on a directory + * escape to userspace if it won't pass the setattr checks + * later. + */ + if ((ip->i_diflags & XFS_DIFLAG_RTINHERIT) && + xfs_extlen_to_rtxmod(mp, ip->i_extsize) > 0) { + fa->fsx_xflags &= ~(FS_XFLAG_EXTSIZE | + FS_XFLAG_EXTSZINHERIT); + fa->fsx_extsize = 0; + } else { + fa->fsx_extsize = XFS_FSB_TO_B(mp, ip->i_extsize); + } + } + + if (ip->i_diflags2 & XFS_DIFLAG2_COWEXTSIZE) { + /* + * Don't let a misaligned CoW extent size hint on a directory + * escape to userspace if it won't pass the setattr checks + * later. + */ + if ((ip->i_diflags & XFS_DIFLAG_RTINHERIT) && + ip->i_cowextsize % mp->m_sb.sb_rextsize > 0) { + fa->fsx_xflags &= ~FS_XFLAG_COWEXTSIZE; + fa->fsx_cowextsize = 0; + } else { + fa->fsx_cowextsize = XFS_FSB_TO_B(mp, ip->i_cowextsize); + } + } + + fa->fsx_projid = ip->i_projid; + if (ifp && !xfs_need_iread_extents(ifp)) + fa->fsx_nextents = xfs_iext_count(ifp); + else + fa->fsx_nextents = xfs_ifork_nextents(ifp); +} + +STATIC int +xfs_ioc_fsgetxattra( + xfs_inode_t *ip, + void __user *arg) +{ + struct file_kattr fa = {}; + + xfs_ilock(ip, XFS_ILOCK_SHARED); + xfs_fill_fsxattr(ip, XFS_ATTR_FORK, &fa); + xfs_iunlock(ip, XFS_ILOCK_SHARED); + + return copy_fsxattr_to_user(&fa, arg); +} + +int +xfs_fileattr_get( + struct dentry *dentry, + struct file_kattr *fa) +{ + struct xfs_inode *ip = XFS_I(d_inode(dentry)); + + xfs_ilock(ip, XFS_ILOCK_SHARED); + xfs_fill_fsxattr(ip, XFS_DATA_FORK, fa); + xfs_iunlock(ip, XFS_ILOCK_SHARED); + + return 0; +} + +static int +xfs_ioctl_setattr_xflags( + struct xfs_trans *tp, + struct xfs_inode *ip, + struct file_kattr *fa) +{ + struct xfs_mount *mp = ip->i_mount; + bool rtflag = (fa->fsx_xflags & FS_XFLAG_REALTIME); + uint64_t i_flags2; + + if (rtflag != XFS_IS_REALTIME_INODE(ip)) { + /* Can't change realtime flag if any extents are allocated. */ + if (xfs_inode_has_filedata(ip)) + return -EINVAL; + + /* + * If S_DAX is enabled on this file, we can only switch the + * device if both support fsdax. We can't update S_DAX because + * there might be other threads walking down the access paths. + */ + if (IS_DAX(VFS_I(ip)) && + (mp->m_ddev_targp->bt_daxdev == NULL || + (mp->m_rtdev_targp && + mp->m_rtdev_targp->bt_daxdev == NULL))) + return -EINVAL; + } + + if (rtflag) { + /* If realtime flag is set then must have realtime device */ + if (mp->m_sb.sb_rblocks == 0 || mp->m_sb.sb_rextsize == 0 || + xfs_extlen_to_rtxmod(mp, ip->i_extsize)) + return -EINVAL; + } + + /* diflags2 only valid for v3 inodes. */ + i_flags2 = xfs_flags2diflags2(ip, fa->fsx_xflags); + if (i_flags2 && !xfs_has_v3inodes(mp)) + return -EINVAL; + + ip->i_diflags = xfs_flags2diflags(ip, fa->fsx_xflags); + ip->i_diflags2 = i_flags2; + + xfs_diflags_to_iflags(ip, false); + + /* + * Make the stable writes flag match that of the device the inode + * resides on when flipping the RT flag. + */ + if (rtflag != XFS_IS_REALTIME_INODE(ip) && S_ISREG(VFS_I(ip)->i_mode)) + xfs_update_stable_writes(ip); + + xfs_trans_ichgtime(tp, ip, XFS_ICHGTIME_CHG); + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + XFS_STATS_INC(mp, xs_ig_attrchg); + return 0; +} + +static void +xfs_ioctl_setattr_prepare_dax( + struct xfs_inode *ip, + struct file_kattr *fa) +{ + struct xfs_mount *mp = ip->i_mount; + struct inode *inode = VFS_I(ip); + + if (S_ISDIR(inode->i_mode)) + return; + + if (xfs_has_dax_always(mp) || xfs_has_dax_never(mp)) + return; + + if (((fa->fsx_xflags & FS_XFLAG_DAX) && + !(ip->i_diflags2 & XFS_DIFLAG2_DAX)) || + (!(fa->fsx_xflags & FS_XFLAG_DAX) && + (ip->i_diflags2 & XFS_DIFLAG2_DAX))) + d_mark_dontcache(inode); +} + +/* + * Set up the transaction structure for the setattr operation, checking that we + * have permission to do so. On success, return a clean transaction and the + * inode locked exclusively ready for further operation specific checks. On + * failure, return an error without modifying or locking the inode. + */ +static struct xfs_trans * +xfs_ioctl_setattr_get_trans( + struct xfs_inode *ip, + struct xfs_dquot *pdqp) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_trans *tp; + int error = -EROFS; + + if (xfs_is_readonly(mp)) + goto out_error; + error = -EIO; + if (xfs_is_shutdown(mp)) + goto out_error; + + error = xfs_trans_alloc_ichange(ip, NULL, NULL, pdqp, + has_capability_noaudit(current, CAP_FOWNER), &tp); + if (error) + goto out_error; + + if (xfs_has_wsync(mp)) + xfs_trans_set_sync(tp); + + return tp; + +out_error: + return ERR_PTR(error); +} + +/* + * Validate a proposed extent size hint. For regular files, the hint can only + * be changed if no extents are allocated. + */ +static int +xfs_ioctl_setattr_check_extsize( + struct xfs_inode *ip, + struct file_kattr *fa) +{ + struct xfs_mount *mp = ip->i_mount; + xfs_failaddr_t failaddr; + uint16_t new_diflags; + + if (!fa->fsx_valid) + return 0; + + if (S_ISREG(VFS_I(ip)->i_mode) && xfs_inode_has_filedata(ip) && + XFS_FSB_TO_B(mp, ip->i_extsize) != fa->fsx_extsize) + return -EINVAL; + + if (fa->fsx_extsize & mp->m_blockmask) + return -EINVAL; + + new_diflags = xfs_flags2diflags(ip, fa->fsx_xflags); + + /* + * Inode verifiers do not check that the extent size hint is an integer + * multiple of the rt extent size on a directory with both rtinherit + * and extszinherit flags set. Don't let sysadmins misconfigure + * directories. + */ + if ((new_diflags & XFS_DIFLAG_RTINHERIT) && + (new_diflags & XFS_DIFLAG_EXTSZINHERIT)) { + unsigned int rtextsize_bytes; + + rtextsize_bytes = XFS_FSB_TO_B(mp, mp->m_sb.sb_rextsize); + if (fa->fsx_extsize % rtextsize_bytes) + return -EINVAL; + } + + failaddr = xfs_inode_validate_extsize(ip->i_mount, + XFS_B_TO_FSB(mp, fa->fsx_extsize), + VFS_I(ip)->i_mode, new_diflags); + return failaddr != NULL ? -EINVAL : 0; +} + +static int +xfs_ioctl_setattr_check_cowextsize( + struct xfs_inode *ip, + struct file_kattr *fa) +{ + struct xfs_mount *mp = ip->i_mount; + xfs_failaddr_t failaddr; + uint64_t new_diflags2; + uint16_t new_diflags; + + if (!fa->fsx_valid) + return 0; + + if (fa->fsx_cowextsize & mp->m_blockmask) + return -EINVAL; + + new_diflags = xfs_flags2diflags(ip, fa->fsx_xflags); + new_diflags2 = xfs_flags2diflags2(ip, fa->fsx_xflags); + + failaddr = xfs_inode_validate_cowextsize(ip->i_mount, + XFS_B_TO_FSB(mp, fa->fsx_cowextsize), + VFS_I(ip)->i_mode, new_diflags, new_diflags2); + return failaddr != NULL ? -EINVAL : 0; +} + +static int +xfs_ioctl_setattr_check_projid( + struct xfs_inode *ip, + struct file_kattr *fa) +{ + if (!fa->fsx_valid) + return 0; + + /* Disallow 32bit project ids if 32bit IDs are not enabled. */ + if (fa->fsx_projid > (uint16_t)-1 && + !xfs_has_projid32(ip->i_mount)) + return -EINVAL; + return 0; +} + +int +xfs_fileattr_set( + struct mnt_idmap *idmap, + struct dentry *dentry, + struct file_kattr *fa) +{ + struct xfs_inode *ip = XFS_I(d_inode(dentry)); + struct xfs_mount *mp = ip->i_mount; + struct xfs_trans *tp; + struct xfs_dquot *pdqp = NULL; + struct xfs_dquot *olddquot = NULL; + int error; + + trace_xfs_ioctl_setattr(ip); + + if (!fa->fsx_valid) { + unsigned int allowed = FS_IMMUTABLE_FL | FS_APPEND_FL | + FS_NOATIME_FL | FS_NODUMP_FL | + FS_SYNC_FL | FS_DAX_FL | + FS_PROJINHERIT_FL; + + /* + * FS_CASEFOLD_FL reflects the ASCIICI superblock feature, + * a read-only property. Accept it as a no-op so chattr's + * RMW round-trip succeeds; reject any attempt to enable + * it on a non-ASCIICI filesystem. xfs_flags2diflags() + * has no clause for CASEFOLD, so the bit is dropped from + * the on-disk diflags regardless. + */ + if (xfs_has_asciici(mp)) + allowed |= FS_CASEFOLD_FL; + + if (fa->flags & ~allowed) + return -EOPNOTSUPP; + } + + error = xfs_ioctl_setattr_check_projid(ip, fa); + if (error) + return error; + + /* + * If disk quotas is on, we make sure that the dquots do exist on disk, + * before we start any other transactions. Trying to do this later + * is messy. We don't care to take a readlock to look at the ids + * in inode here, because we can't hold it across the trans_reserve. + * If the IDs do change before we take the ilock, we're covered + * because the i_*dquot fields will get updated anyway. + */ + if (fa->fsx_valid && XFS_IS_QUOTA_ON(mp)) { + error = xfs_qm_vop_dqalloc(ip, VFS_I(ip)->i_uid, + VFS_I(ip)->i_gid, fa->fsx_projid, + XFS_QMOPT_PQUOTA, NULL, NULL, &pdqp); + if (error) + return error; + } + + xfs_ioctl_setattr_prepare_dax(ip, fa); + + tp = xfs_ioctl_setattr_get_trans(ip, pdqp); + if (IS_ERR(tp)) { + error = PTR_ERR(tp); + goto error_free_dquots; + } + + error = xfs_ioctl_setattr_check_extsize(ip, fa); + if (error) + goto error_trans_cancel; + + error = xfs_ioctl_setattr_check_cowextsize(ip, fa); + if (error) + goto error_trans_cancel; + + error = xfs_ioctl_setattr_xflags(tp, ip, fa); + if (error) + goto error_trans_cancel; + + if (!fa->fsx_valid) + goto skip_xattr; + /* + * Change file ownership. Must be the owner or privileged. CAP_FSETID + * overrides the following restrictions: + * + * The set-user-ID and set-group-ID bits of a file will be cleared upon + * successful return from chown() + */ + + if ((VFS_I(ip)->i_mode & (S_ISUID|S_ISGID)) && + !capable_wrt_inode_uidgid(idmap, VFS_I(ip), CAP_FSETID)) + VFS_I(ip)->i_mode &= ~(S_ISUID|S_ISGID); + + /* Change the ownerships and register project quota modifications */ + if (ip->i_projid != fa->fsx_projid) { + if (XFS_IS_PQUOTA_ON(mp)) { + olddquot = xfs_qm_vop_chown(tp, ip, + &ip->i_pdquot, pdqp); + } + ip->i_projid = fa->fsx_projid; + } + + /* + * Only set the extent size hint if we've already determined that the + * extent size hint should be set on the inode. If no extent size flags + * are set on the inode then unconditionally clear the extent size hint. + */ + if (ip->i_diflags & (XFS_DIFLAG_EXTSIZE | XFS_DIFLAG_EXTSZINHERIT)) + ip->i_extsize = XFS_B_TO_FSB(mp, fa->fsx_extsize); + else + ip->i_extsize = 0; + + if (xfs_has_v3inodes(mp)) { + if (ip->i_diflags2 & XFS_DIFLAG2_COWEXTSIZE) + ip->i_cowextsize = XFS_B_TO_FSB(mp, fa->fsx_cowextsize); + else + ip->i_cowextsize = 0; + } + +skip_xattr: + error = xfs_trans_commit(tp); + + /* + * Release any dquot(s) the inode had kept before chown. + */ + xfs_qm_dqrele(olddquot); + xfs_qm_dqrele(pdqp); + + return error; + +error_trans_cancel: + xfs_trans_cancel(tp); +error_free_dquots: + xfs_qm_dqrele(pdqp); + return error; +} + +static bool +xfs_getbmap_format( + struct kgetbmap *p, + struct getbmapx __user *u, + size_t recsize) +{ + if (put_user(p->bmv_offset, &u->bmv_offset) || + put_user(p->bmv_block, &u->bmv_block) || + put_user(p->bmv_length, &u->bmv_length) || + put_user(0, &u->bmv_count) || + put_user(0, &u->bmv_entries)) + return false; + if (recsize < sizeof(struct getbmapx)) + return true; + if (put_user(0, &u->bmv_iflags) || + put_user(p->bmv_oflags, &u->bmv_oflags) || + put_user(0, &u->bmv_unused1) || + put_user(0, &u->bmv_unused2)) + return false; + return true; +} + +STATIC int +xfs_ioc_getbmap( + struct file *file, + unsigned int cmd, + void __user *arg) +{ + struct getbmapx bmx = { 0 }; + struct kgetbmap *buf; + size_t recsize; + int error, i; + + switch (cmd) { + case XFS_IOC_GETBMAPA: + bmx.bmv_iflags = BMV_IF_ATTRFORK; + fallthrough; + case XFS_IOC_GETBMAP: + /* struct getbmap is a strict subset of struct getbmapx. */ + recsize = sizeof(struct getbmap); + break; + case XFS_IOC_GETBMAPX: + recsize = sizeof(struct getbmapx); + break; + default: + return -EINVAL; + } + + if (copy_from_user(&bmx, arg, recsize)) + return -EFAULT; + + if (bmx.bmv_count < 2) + return -EINVAL; + if (bmx.bmv_count >= INT_MAX / recsize) + return -ENOMEM; + + buf = kvzalloc_objs(*buf, bmx.bmv_count); + if (!buf) + return -ENOMEM; + + error = xfs_getbmap(XFS_I(file_inode(file)), &bmx, buf); + if (error) + goto out_free_buf; + + error = -EFAULT; + if (copy_to_user(arg, &bmx, recsize)) + goto out_free_buf; + arg += recsize; + + for (i = 0; i < bmx.bmv_entries; i++) { + if (!xfs_getbmap_format(buf + i, arg, recsize)) + goto out_free_buf; + arg += recsize; + } + + error = 0; +out_free_buf: + kvfree(buf); + return error; +} + +int +xfs_ioc_swapext( + xfs_swapext_t *sxp) +{ + xfs_inode_t *ip, *tip; + + /* Pull information for the target fd */ + CLASS(fd, f)((int)sxp->sx_fdtarget); + if (fd_empty(f)) + return -EINVAL; + + if (!(fd_file(f)->f_mode & FMODE_WRITE) || + !(fd_file(f)->f_mode & FMODE_READ) || + (fd_file(f)->f_flags & O_APPEND)) + return -EBADF; + + CLASS(fd, tmp)((int)sxp->sx_fdtmp); + if (fd_empty(tmp)) + return -EINVAL; + + if (!(fd_file(tmp)->f_mode & FMODE_WRITE) || + !(fd_file(tmp)->f_mode & FMODE_READ) || + (fd_file(tmp)->f_flags & O_APPEND)) + return -EBADF; + + if (IS_SWAPFILE(file_inode(fd_file(f))) || + IS_SWAPFILE(file_inode(fd_file(tmp)))) + return -EINVAL; + + /* + * We need to ensure that the fds passed in point to XFS inodes + * before we cast and access them as XFS structures as we have no + * control over what the user passes us here. + */ + if (fd_file(f)->f_op != &xfs_file_operations || + fd_file(tmp)->f_op != &xfs_file_operations) + return -EINVAL; + + ip = XFS_I(file_inode(fd_file(f))); + tip = XFS_I(file_inode(fd_file(tmp))); + + if (ip->i_mount != tip->i_mount) + return -EINVAL; + + if (I_INO(ip) == I_INO(tip)) + return -EINVAL; + + if (xfs_is_shutdown(ip->i_mount)) + return -EIO; + + return xfs_swap_extents(ip, tip, sxp); +} + +static int +xfs_ioc_getlabel( + struct xfs_mount *mp, + char __user *user_label) +{ + struct xfs_sb *sbp = &mp->m_sb; + char label[XFSLABEL_MAX + 1]; + + /* Paranoia */ + BUILD_BUG_ON(sizeof(sbp->sb_fname) > FSLABEL_MAX); + + /* 1 larger than sb_fname, so this ensures a trailing NUL char */ + spin_lock(&mp->m_sb_lock); + memtostr_pad(label, sbp->sb_fname); + spin_unlock(&mp->m_sb_lock); + + if (copy_to_user(user_label, label, sizeof(label))) + return -EFAULT; + return 0; +} + +static int +xfs_ioc_setlabel( + struct file *filp, + struct xfs_mount *mp, + char __user *newlabel) +{ + struct xfs_sb *sbp = &mp->m_sb; + char label[XFSLABEL_MAX + 1]; + size_t len; + int error; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + /* + * The generic ioctl allows up to FSLABEL_MAX chars, but XFS is much + * smaller, at 12 bytes. We copy one more to be sure we find the + * (required) NULL character to test the incoming label length. + * NB: The on disk label doesn't need to be null terminated. + */ + if (copy_from_user(label, newlabel, XFSLABEL_MAX + 1)) + return -EFAULT; + len = strnlen(label, XFSLABEL_MAX + 1); + if (len > sizeof(sbp->sb_fname)) + return -EINVAL; + + error = mnt_want_write_file(filp); + if (error) + return error; + + spin_lock(&mp->m_sb_lock); + memset(sbp->sb_fname, 0, sizeof(sbp->sb_fname)); + memcpy(sbp->sb_fname, label, len); + spin_unlock(&mp->m_sb_lock); + + /* + * Now we do several things to satisfy userspace. + * In addition to normal logging of the primary superblock, we also + * immediately write these changes to sector zero for the primary, then + * update all backup supers (as xfs_db does for a label change), then + * invalidate the block device page cache. This is so that any prior + * buffered reads from userspace (i.e. from blkid) are invalidated, + * and userspace will see the newly-written label. + */ + error = xfs_sync_sb_buf(mp, true); + if (error) + goto out; + /* + * growfs also updates backup supers so lock against that. + */ + mutex_lock(&mp->m_growlock); + error = xfs_update_secondary_sbs(mp); + mutex_unlock(&mp->m_growlock); + + invalidate_bdev(mp->m_ddev_targp->bt_bdev); + if (xfs_has_rtsb(mp) && mp->m_rtdev_targp) + invalidate_bdev(mp->m_rtdev_targp->bt_bdev); + +out: + mnt_drop_write_file(filp); + return error; +} + +static inline int +xfs_fs_eofblocks_from_user( + struct xfs_fs_eofblocks *src, + struct xfs_icwalk *dst) +{ + if (src->eof_version != XFS_EOFBLOCKS_VERSION) + return -EINVAL; + + if (src->eof_flags & ~XFS_EOF_FLAGS_VALID) + return -EINVAL; + + if (memchr_inv(&src->pad32, 0, sizeof(src->pad32)) || + memchr_inv(src->pad64, 0, sizeof(src->pad64))) + return -EINVAL; + + dst->icw_flags = 0; + if (src->eof_flags & XFS_EOF_FLAGS_SYNC) + dst->icw_flags |= XFS_ICWALK_FLAG_SYNC; + if (src->eof_flags & XFS_EOF_FLAGS_UID) + dst->icw_flags |= XFS_ICWALK_FLAG_UID; + if (src->eof_flags & XFS_EOF_FLAGS_GID) + dst->icw_flags |= XFS_ICWALK_FLAG_GID; + if (src->eof_flags & XFS_EOF_FLAGS_PRID) + dst->icw_flags |= XFS_ICWALK_FLAG_PRID; + if (src->eof_flags & XFS_EOF_FLAGS_MINFILESIZE) + dst->icw_flags |= XFS_ICWALK_FLAG_MINFILESIZE; + + dst->icw_prid = src->eof_prid; + dst->icw_min_file_size = src->eof_min_file_size; + + dst->icw_uid = INVALID_UID; + if (src->eof_flags & XFS_EOF_FLAGS_UID) { + dst->icw_uid = make_kuid(current_user_ns(), src->eof_uid); + if (!uid_valid(dst->icw_uid)) + return -EINVAL; + } + + dst->icw_gid = INVALID_GID; + if (src->eof_flags & XFS_EOF_FLAGS_GID) { + dst->icw_gid = make_kgid(current_user_ns(), src->eof_gid); + if (!gid_valid(dst->icw_gid)) + return -EINVAL; + } + return 0; +} + +static int +xfs_ioctl_getset_resblocks( + struct file *filp, + unsigned int cmd, + void __user *arg) +{ + struct xfs_mount *mp = XFS_I(file_inode(filp))->i_mount; + struct xfs_fsop_resblks fsop = { }; + int error; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (cmd == XFS_IOC_SET_RESBLKS) { + if (xfs_is_readonly(mp)) + return -EROFS; + + if (copy_from_user(&fsop, arg, sizeof(fsop))) + return -EFAULT; + + error = mnt_want_write_file(filp); + if (error) + return error; + error = xfs_reserve_blocks(mp, XC_FREE_BLOCKS, fsop.resblks); + mnt_drop_write_file(filp); + if (error) + return error; + } + + spin_lock(&mp->m_sb_lock); + fsop.resblks = mp->m_free[XC_FREE_BLOCKS].res_total; + fsop.resblks_avail = mp->m_free[XC_FREE_BLOCKS].res_avail; + spin_unlock(&mp->m_sb_lock); + + if (copy_to_user(arg, &fsop, sizeof(fsop))) + return -EFAULT; + return 0; +} + +static int +xfs_ioctl_fs_counts( + struct xfs_mount *mp, + struct xfs_fsop_counts __user *uarg) +{ + struct xfs_fsop_counts out = { + .allocino = percpu_counter_read_positive(&mp->m_icount), + .freeino = percpu_counter_read_positive(&mp->m_ifree), + .freedata = xfs_estimate_freecounter(mp, XC_FREE_BLOCKS) - + xfs_freecounter_unavailable(mp, XC_FREE_BLOCKS), + .freertx = xfs_estimate_freecounter(mp, XC_FREE_RTEXTENTS), + }; + + if (copy_to_user(uarg, &out, sizeof(out))) + return -EFAULT; + return 0; +} + +/* + * These long-unused ioctls were removed from the official ioctl API in 5.17, + * but retain these definitions so that we can log warnings about them. + */ +#define XFS_IOC_ALLOCSP _IOW ('X', 10, struct xfs_flock64) +#define XFS_IOC_FREESP _IOW ('X', 11, struct xfs_flock64) +#define XFS_IOC_ALLOCSP64 _IOW ('X', 36, struct xfs_flock64) +#define XFS_IOC_FREESP64 _IOW ('X', 37, struct xfs_flock64) + +/* + * Note: some of the ioctl's return positive numbers as a + * byte count indicating success, such as readlink_by_handle. + * So we don't "sign flip" like most other routines. This means + * true errors need to be returned as a negative value. + */ +long +xfs_file_ioctl( + struct file *filp, + unsigned int cmd, + unsigned long p) +{ + struct inode *inode = file_inode(filp); + struct xfs_inode *ip = XFS_I(inode); + struct xfs_mount *mp = ip->i_mount; + void __user *arg = (void __user *)p; + int error; + + trace_xfs_file_ioctl(ip); + + switch (cmd) { + case FITRIM: + return xfs_ioc_trim(mp, arg); + case FS_IOC_GETFSLABEL: + return xfs_ioc_getlabel(mp, arg); + case FS_IOC_SETFSLABEL: + return xfs_ioc_setlabel(filp, mp, arg); + case XFS_IOC_ALLOCSP: + case XFS_IOC_FREESP: + case XFS_IOC_ALLOCSP64: + case XFS_IOC_FREESP64: + xfs_warn_once(mp, + "%s should use fallocate; XFS_IOC_{ALLOC,FREE}SP ioctl unsupported", + current->comm); + return -ENOTTY; + case XFS_IOC_DIOINFO: { + struct kstat st; + struct dioattr da; + + error = vfs_getattr(&filp->f_path, &st, STATX_DIOALIGN, 0); + if (error) + return error; + + /* + * Some userspace directly feeds the return value to + * posix_memalign, which fails for values that are smaller than + * the pointer size. Round up the value to not break userspace. + */ + da.d_mem = roundup(st.dio_mem_align, sizeof(void *)); + da.d_miniosz = st.dio_offset_align; + da.d_maxiosz = INT_MAX & ~(da.d_miniosz - 1); + if (copy_to_user(arg, &da, sizeof(da))) + return -EFAULT; + return 0; + } + + case XFS_IOC_FSBULKSTAT_SINGLE: + case XFS_IOC_FSBULKSTAT: + case XFS_IOC_FSINUMBERS: + return xfs_ioc_fsbulkstat(filp, cmd, arg); + + case XFS_IOC_BULKSTAT: + return xfs_ioc_bulkstat(filp, cmd, arg); + case XFS_IOC_INUMBERS: + return xfs_ioc_inumbers(mp, cmd, arg); + + case XFS_IOC_FSGEOMETRY_V1: + return xfs_ioc_fsgeometry(mp, arg, 3); + case XFS_IOC_FSGEOMETRY_V4: + return xfs_ioc_fsgeometry(mp, arg, 4); + case XFS_IOC_FSGEOMETRY: + return xfs_ioc_fsgeometry(mp, arg, 5); + + case XFS_IOC_AG_GEOMETRY: + return xfs_ioc_ag_geometry(mp, arg); + case XFS_IOC_RTGROUP_GEOMETRY: + return xfs_ioc_rtgroup_geometry(mp, arg); + + case XFS_IOC_GETVERSION: + return put_user(inode->i_generation, (int __user *)arg); + + case XFS_IOC_FSGETXATTRA: + return xfs_ioc_fsgetxattra(ip, arg); + case XFS_IOC_GETPARENTS: + return xfs_ioc_getparents(filp, arg); + case XFS_IOC_GETPARENTS_BY_HANDLE: + return xfs_ioc_getparents_by_handle(filp, arg); + case XFS_IOC_GETBMAP: + case XFS_IOC_GETBMAPA: + case XFS_IOC_GETBMAPX: + return xfs_ioc_getbmap(filp, cmd, arg); + + case FS_IOC_GETFSMAP: + return xfs_ioc_getfsmap(ip, arg); + + case XFS_IOC_SCRUBV_METADATA: + return xfs_ioc_scrubv_metadata(filp, arg); + case XFS_IOC_SCRUB_METADATA: + return xfs_ioc_scrub_metadata(filp, arg); + + case XFS_IOC_FD_TO_HANDLE: + case XFS_IOC_PATH_TO_HANDLE: + case XFS_IOC_PATH_TO_FSHANDLE: { + xfs_fsop_handlereq_t hreq; + + if (copy_from_user(&hreq, arg, sizeof(hreq))) + return -EFAULT; + return xfs_find_handle(cmd, &hreq); + } + case XFS_IOC_OPEN_BY_HANDLE: { + xfs_fsop_handlereq_t hreq; + + if (copy_from_user(&hreq, arg, sizeof(xfs_fsop_handlereq_t))) + return -EFAULT; + return xfs_open_by_handle(filp, &hreq); + } + + case XFS_IOC_READLINK_BY_HANDLE: { + xfs_fsop_handlereq_t hreq; + + if (copy_from_user(&hreq, arg, sizeof(xfs_fsop_handlereq_t))) + return -EFAULT; + return xfs_readlink_by_handle(filp, &hreq); + } + case XFS_IOC_ATTRLIST_BY_HANDLE: + return xfs_attrlist_by_handle(filp, arg); + + case XFS_IOC_ATTRMULTI_BY_HANDLE: + return xfs_attrmulti_by_handle(filp, arg); + + case XFS_IOC_SWAPEXT: { + struct xfs_swapext sxp; + + if (copy_from_user(&sxp, arg, sizeof(xfs_swapext_t))) + return -EFAULT; + error = mnt_want_write_file(filp); + if (error) + return error; + error = xfs_ioc_swapext(&sxp); + mnt_drop_write_file(filp); + return error; + } + + case XFS_IOC_FSCOUNTS: + return xfs_ioctl_fs_counts(mp, arg); + + case XFS_IOC_SET_RESBLKS: + case XFS_IOC_GET_RESBLKS: + return xfs_ioctl_getset_resblocks(filp, cmd, arg); + + case XFS_IOC_FSGROWFSDATA: { + struct xfs_growfs_data in; + + if (copy_from_user(&in, arg, sizeof(in))) + return -EFAULT; + + error = mnt_want_write_file(filp); + if (error) + return error; + error = xfs_growfs_data(mp, &in); + mnt_drop_write_file(filp); + return error; + } + + case XFS_IOC_FSGROWFSLOG: { + struct xfs_growfs_log in; + + if (copy_from_user(&in, arg, sizeof(in))) + return -EFAULT; + + error = mnt_want_write_file(filp); + if (error) + return error; + error = xfs_growfs_log(mp, &in); + mnt_drop_write_file(filp); + return error; + } + + case XFS_IOC_FSGROWFSRT: { + xfs_growfs_rt_t in; + + if (copy_from_user(&in, arg, sizeof(in))) + return -EFAULT; + + error = mnt_want_write_file(filp); + if (error) + return error; + error = xfs_growfs_rt(mp, &in); + mnt_drop_write_file(filp); + return error; + } + + case XFS_IOC_GOINGDOWN: { + uint32_t in; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (get_user(in, (uint32_t __user *)arg)) + return -EFAULT; + + return xfs_fs_goingdown(mp, in); + } + + case XFS_IOC_ERROR_INJECTION: { + xfs_error_injection_t in; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (copy_from_user(&in, arg, sizeof(in))) + return -EFAULT; + + return xfs_errortag_add(mp, in.errtag); + } + + case XFS_IOC_ERROR_CLEARALL: + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + return xfs_errortag_clearall(mp); + + case XFS_IOC_FREE_EOFBLOCKS: { + struct xfs_fs_eofblocks eofb; + struct xfs_icwalk icw; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (xfs_is_readonly(mp)) + return -EROFS; + + if (copy_from_user(&eofb, arg, sizeof(eofb))) + return -EFAULT; + + error = xfs_fs_eofblocks_from_user(&eofb, &icw); + if (error) + return error; + + trace_xfs_ioc_free_eofblocks(mp, &icw, _RET_IP_); + + guard(super_write)(mp->m_super); + return xfs_blockgc_free_space(mp, &icw); + } + + case XFS_IOC_EXCHANGE_RANGE: + return xfs_ioc_exchange_range(filp, arg); + case XFS_IOC_START_COMMIT: + return xfs_ioc_start_commit(filp, arg); + case XFS_IOC_COMMIT_RANGE: + return xfs_ioc_commit_range(filp, arg); + + case XFS_IOC_HEALTH_MONITOR: + return xfs_ioc_health_monitor(filp, arg); + case XFS_IOC_VERIFY_MEDIA: + return xfs_ioc_verify_media(filp, arg); + + default: + return -ENOTTY; + } +} diff --git a/fs/xfs/xfs_ioctl.h b/fs/xfs/xfs_ioctl.h new file mode 100644 index 000000000..f5ed5cf9d --- /dev/null +++ b/fs/xfs/xfs_ioctl.h @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2008 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_IOCTL_H__ +#define __XFS_IOCTL_H__ + +struct xfs_bstat; +struct xfs_ibulk; +struct xfs_inogrp; + +int +xfs_ioc_swapext( + xfs_swapext_t *sxp); + +extern int +xfs_fileattr_get( + struct dentry *dentry, + struct file_kattr *fa); + +extern int +xfs_fileattr_set( + struct mnt_idmap *idmap, + struct dentry *dentry, + struct file_kattr *fa); + +extern long +xfs_file_ioctl( + struct file *filp, + unsigned int cmd, + unsigned long p); + +extern long +xfs_file_compat_ioctl( + struct file *file, + unsigned int cmd, + unsigned long arg); + +int xfs_fsbulkstat_one_fmt(struct xfs_ibulk *breq, + const struct xfs_bulkstat *bstat); +int xfs_fsinumbers_fmt(struct xfs_ibulk *breq, const struct xfs_inumbers *igrp); + +#endif diff --git a/fs/xfs/xfs_ioctl32.c b/fs/xfs/xfs_ioctl32.c new file mode 100644 index 000000000..c66e19244 --- /dev/null +++ b/fs/xfs/xfs_ioctl32.c @@ -0,0 +1,518 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2004-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include +#include +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_iwalk.h" +#include "xfs_itable.h" +#include "xfs_fsops.h" +#include "xfs_rtalloc.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_attr.h" +#include "xfs_ioctl.h" +#include "xfs_ioctl32.h" +#include "xfs_trace.h" +#include "xfs_sb.h" +#include "xfs_handle.h" + +#define _NATIVE_IOC(cmd, type) \ + _IOC(_IOC_DIR(cmd), _IOC_TYPE(cmd), _IOC_NR(cmd), sizeof(type)) + +#ifdef BROKEN_X86_ALIGNMENT +STATIC int +xfs_compat_ioc_fsgeometry_v1( + struct xfs_mount *mp, + compat_xfs_fsop_geom_v1_t __user *arg32) +{ + struct xfs_fsop_geom fsgeo; + + xfs_fs_geometry(mp, &fsgeo, 3); + /* The 32-bit variant simply has some padding at the end */ + if (copy_to_user(arg32, &fsgeo, sizeof(struct compat_xfs_fsop_geom_v1))) + return -EFAULT; + return 0; +} + +STATIC int +xfs_compat_growfs_data_copyin( + struct xfs_growfs_data *in, + compat_xfs_growfs_data_t __user *arg32) +{ + if (get_user(in->newblocks, &arg32->newblocks) || + get_user(in->imaxpct, &arg32->imaxpct)) + return -EFAULT; + return 0; +} + +STATIC int +xfs_compat_growfs_rt_copyin( + struct xfs_growfs_rt *in, + compat_xfs_growfs_rt_t __user *arg32) +{ + if (get_user(in->newblocks, &arg32->newblocks) || + get_user(in->extsize, &arg32->extsize)) + return -EFAULT; + return 0; +} + +STATIC int +xfs_fsinumbers_fmt_compat( + struct xfs_ibulk *breq, + const struct xfs_inumbers *ig) +{ + struct compat_xfs_inogrp __user *p32 = breq->ubuffer; + struct xfs_inogrp ig1; + struct xfs_inogrp *igrp = &ig1; + + xfs_inumbers_to_inogrp(&ig1, ig); + + if (put_user(igrp->xi_startino, &p32->xi_startino) || + put_user(igrp->xi_alloccount, &p32->xi_alloccount) || + put_user(igrp->xi_allocmask, &p32->xi_allocmask)) + return -EFAULT; + + return xfs_ibulk_advance(breq, sizeof(struct compat_xfs_inogrp)); +} + +#else +#define xfs_fsinumbers_fmt_compat xfs_fsinumbers_fmt +#endif /* BROKEN_X86_ALIGNMENT */ + +STATIC int +xfs_ioctl32_bstime_copyin( + xfs_bstime_t *bstime, + compat_xfs_bstime_t __user *bstime32) +{ + old_time32_t sec32; /* tv_sec differs on 64 vs. 32 */ + + if (get_user(sec32, &bstime32->tv_sec) || + get_user(bstime->tv_nsec, &bstime32->tv_nsec)) + return -EFAULT; + bstime->tv_sec = sec32; + return 0; +} + +/* + * struct xfs_bstat has differing alignment on intel, & bstime_t sizes + * everywhere + */ +STATIC int +xfs_ioctl32_bstat_copyin( + struct xfs_bstat *bstat, + struct compat_xfs_bstat __user *bstat32) +{ + if (get_user(bstat->bs_ino, &bstat32->bs_ino) || + get_user(bstat->bs_mode, &bstat32->bs_mode) || + get_user(bstat->bs_nlink, &bstat32->bs_nlink) || + get_user(bstat->bs_uid, &bstat32->bs_uid) || + get_user(bstat->bs_gid, &bstat32->bs_gid) || + get_user(bstat->bs_rdev, &bstat32->bs_rdev) || + get_user(bstat->bs_blksize, &bstat32->bs_blksize) || + get_user(bstat->bs_size, &bstat32->bs_size) || + xfs_ioctl32_bstime_copyin(&bstat->bs_atime, &bstat32->bs_atime) || + xfs_ioctl32_bstime_copyin(&bstat->bs_mtime, &bstat32->bs_mtime) || + xfs_ioctl32_bstime_copyin(&bstat->bs_ctime, &bstat32->bs_ctime) || + get_user(bstat->bs_blocks, &bstat32->bs_size) || + get_user(bstat->bs_xflags, &bstat32->bs_size) || + get_user(bstat->bs_extsize, &bstat32->bs_extsize) || + get_user(bstat->bs_extents, &bstat32->bs_extents) || + get_user(bstat->bs_gen, &bstat32->bs_gen) || + get_user(bstat->bs_projid_lo, &bstat32->bs_projid_lo) || + get_user(bstat->bs_projid_hi, &bstat32->bs_projid_hi) || + get_user(bstat->bs_forkoff, &bstat32->bs_forkoff) || + get_user(bstat->bs_dmevmask, &bstat32->bs_dmevmask) || + get_user(bstat->bs_dmstate, &bstat32->bs_dmstate) || + get_user(bstat->bs_aextents, &bstat32->bs_aextents)) + return -EFAULT; + return 0; +} + +/* XFS_IOC_FSBULKSTAT and friends */ + +STATIC int +xfs_bstime_store_compat( + compat_xfs_bstime_t __user *p32, + const xfs_bstime_t *p) +{ + __s32 sec32; + + sec32 = p->tv_sec; + if (put_user(sec32, &p32->tv_sec) || + put_user(p->tv_nsec, &p32->tv_nsec)) + return -EFAULT; + return 0; +} + +/* Return 0 on success or positive error (to xfs_bulkstat()) */ +STATIC int +xfs_fsbulkstat_one_fmt_compat( + struct xfs_ibulk *breq, + const struct xfs_bulkstat *bstat) +{ + struct compat_xfs_bstat __user *p32 = breq->ubuffer; + struct xfs_bstat bs1; + struct xfs_bstat *buffer = &bs1; + + xfs_bulkstat_to_bstat(breq->mp, &bs1, bstat); + + if (put_user(buffer->bs_ino, &p32->bs_ino) || + put_user(buffer->bs_mode, &p32->bs_mode) || + put_user(buffer->bs_nlink, &p32->bs_nlink) || + put_user(buffer->bs_uid, &p32->bs_uid) || + put_user(buffer->bs_gid, &p32->bs_gid) || + put_user(buffer->bs_rdev, &p32->bs_rdev) || + put_user(buffer->bs_blksize, &p32->bs_blksize) || + put_user(buffer->bs_size, &p32->bs_size) || + xfs_bstime_store_compat(&p32->bs_atime, &buffer->bs_atime) || + xfs_bstime_store_compat(&p32->bs_mtime, &buffer->bs_mtime) || + xfs_bstime_store_compat(&p32->bs_ctime, &buffer->bs_ctime) || + put_user(buffer->bs_blocks, &p32->bs_blocks) || + put_user(buffer->bs_xflags, &p32->bs_xflags) || + put_user(buffer->bs_extsize, &p32->bs_extsize) || + put_user(buffer->bs_extents, &p32->bs_extents) || + put_user(buffer->bs_gen, &p32->bs_gen) || + put_user(buffer->bs_projid, &p32->bs_projid) || + put_user(buffer->bs_projid_hi, &p32->bs_projid_hi) || + put_user(buffer->bs_forkoff, &p32->bs_forkoff) || + put_user(buffer->bs_dmevmask, &p32->bs_dmevmask) || + put_user(buffer->bs_dmstate, &p32->bs_dmstate) || + put_user(buffer->bs_aextents, &p32->bs_aextents)) + return -EFAULT; + + return xfs_ibulk_advance(breq, sizeof(struct compat_xfs_bstat)); +} + +/* copied from xfs_ioctl.c */ +STATIC int +xfs_compat_ioc_fsbulkstat( + struct file *file, + unsigned int cmd, + struct compat_xfs_fsop_bulkreq __user *p32) +{ + struct xfs_mount *mp = XFS_I(file_inode(file))->i_mount; + u32 addr; + struct xfs_fsop_bulkreq bulkreq; + struct xfs_ibulk breq = { + .mp = mp, + .idmap = file_mnt_idmap(file), + .ocount = 0, + }; + xfs_ino_t lastino; + int error; + + /* + * Output structure handling functions. Depending on the command, + * either the xfs_bstat and xfs_inogrp structures are written out + * to userpace memory via bulkreq.ubuffer. Normally the compat + * functions and structure size are the correct ones to use ... + */ + inumbers_fmt_pf inumbers_func = xfs_fsinumbers_fmt_compat; + bulkstat_one_fmt_pf bs_one_func = xfs_fsbulkstat_one_fmt_compat; + +#ifdef CONFIG_X86_X32_ABI + if (in_x32_syscall()) { + /* + * ... but on x32 the input xfs_fsop_bulkreq has pointers + * which must be handled in the "compat" (32-bit) way, while + * the xfs_bstat and xfs_inogrp structures follow native 64- + * bit layout convention. So adjust accordingly, otherwise + * the data written out in compat layout will not match what + * x32 userspace expects. + */ + inumbers_func = xfs_fsinumbers_fmt; + bs_one_func = xfs_fsbulkstat_one_fmt; + } +#endif + + /* done = 1 if there are more stats to get and if bulkstat */ + /* should be called again (unused here, but used in dmapi) */ + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (xfs_is_shutdown(mp)) + return -EIO; + + if (get_user(addr, &p32->lastip)) + return -EFAULT; + bulkreq.lastip = compat_ptr(addr); + if (get_user(bulkreq.icount, &p32->icount) || + get_user(addr, &p32->ubuffer)) + return -EFAULT; + bulkreq.ubuffer = compat_ptr(addr); + if (get_user(addr, &p32->ocount)) + return -EFAULT; + bulkreq.ocount = compat_ptr(addr); + + if (copy_from_user(&lastino, bulkreq.lastip, sizeof(__s64))) + return -EFAULT; + + if (bulkreq.icount <= 0) + return -EINVAL; + + if (bulkreq.ubuffer == NULL) + return -EINVAL; + + breq.ubuffer = bulkreq.ubuffer; + breq.icount = bulkreq.icount; + + /* + * FSBULKSTAT_SINGLE expects that *lastip contains the inode number + * that we want to stat. However, FSINUMBERS and FSBULKSTAT expect + * that *lastip contains either zero or the number of the last inode to + * be examined by the previous call and return results starting with + * the next inode after that. The new bulk request back end functions + * take the inode to start with, so we have to compute the startino + * parameter from lastino to maintain correct function. lastino == 0 + * is a special case because it has traditionally meant "first inode + * in filesystem". + */ + if (cmd == XFS_IOC_FSINUMBERS_32) { + breq.startino = lastino ? lastino + 1 : 0; + error = xfs_inumbers(&breq, inumbers_func); + lastino = breq.startino - 1; + } else if (cmd == XFS_IOC_FSBULKSTAT_SINGLE_32) { + breq.startino = lastino; + breq.icount = 1; + error = xfs_bulkstat_one(&breq, bs_one_func); + lastino = breq.startino; + } else if (cmd == XFS_IOC_FSBULKSTAT_32) { + breq.startino = lastino ? lastino + 1 : 0; + error = xfs_bulkstat(&breq, bs_one_func); + lastino = breq.startino - 1; + } else { + error = -EINVAL; + } + if (error) + return error; + + if (bulkreq.lastip != NULL && + copy_to_user(bulkreq.lastip, &lastino, sizeof(xfs_ino_t))) + return -EFAULT; + + if (bulkreq.ocount != NULL && + copy_to_user(bulkreq.ocount, &breq.ocount, sizeof(__s32))) + return -EFAULT; + + return 0; +} + +STATIC int +xfs_compat_handlereq_copyin( + xfs_fsop_handlereq_t *hreq, + compat_xfs_fsop_handlereq_t __user *arg32) +{ + compat_xfs_fsop_handlereq_t hreq32; + + if (copy_from_user(&hreq32, arg32, sizeof(compat_xfs_fsop_handlereq_t))) + return -EFAULT; + + hreq->fd = hreq32.fd; + hreq->path = compat_ptr(hreq32.path); + hreq->oflags = hreq32.oflags; + hreq->ihandle = compat_ptr(hreq32.ihandle); + hreq->ihandlen = hreq32.ihandlen; + hreq->ohandle = compat_ptr(hreq32.ohandle); + hreq->ohandlen = compat_ptr(hreq32.ohandlen); + + return 0; +} + +STATIC struct dentry * +xfs_compat_handlereq_to_dentry( + struct file *parfilp, + compat_xfs_fsop_handlereq_t *hreq) +{ + return xfs_handle_to_dentry(parfilp, + compat_ptr(hreq->ihandle), hreq->ihandlen); +} + +STATIC int +xfs_compat_attrlist_by_handle( + struct file *parfilp, + compat_xfs_fsop_attrlist_handlereq_t __user *p) +{ + compat_xfs_fsop_attrlist_handlereq_t al_hreq; + struct dentry *dentry; + int error; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (copy_from_user(&al_hreq, p, sizeof(al_hreq))) + return -EFAULT; + + dentry = xfs_compat_handlereq_to_dentry(parfilp, &al_hreq.hreq); + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + + error = xfs_ioc_attr_list(XFS_I(d_inode(dentry)), + compat_ptr(al_hreq.buffer), al_hreq.buflen, + al_hreq.flags, &p->pos); + dput(dentry); + return error; +} + +STATIC int +xfs_compat_attrmulti_by_handle( + struct file *parfilp, + void __user *arg) +{ + int error; + compat_xfs_attr_multiop_t *ops; + compat_xfs_fsop_attrmulti_handlereq_t am_hreq; + struct dentry *dentry; + unsigned int i, size; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (copy_from_user(&am_hreq, arg, + sizeof(compat_xfs_fsop_attrmulti_handlereq_t))) + return -EFAULT; + + /* overflow check */ + if (am_hreq.opcount >= INT_MAX / sizeof(compat_xfs_attr_multiop_t)) + return -E2BIG; + + dentry = xfs_compat_handlereq_to_dentry(parfilp, &am_hreq.hreq); + if (IS_ERR(dentry)) + return PTR_ERR(dentry); + + error = -E2BIG; + size = am_hreq.opcount * sizeof(compat_xfs_attr_multiop_t); + if (!size || size > 16 * PAGE_SIZE) + goto out_dput; + + ops = memdup_user(compat_ptr(am_hreq.ops), size); + if (IS_ERR(ops)) { + error = PTR_ERR(ops); + goto out_dput; + } + + error = 0; + for (i = 0; i < am_hreq.opcount; i++) { + ops[i].am_error = xfs_ioc_attrmulti_one(parfilp, + d_inode(dentry), ops[i].am_opcode, + compat_ptr(ops[i].am_attrname), + compat_ptr(ops[i].am_attrvalue), + &ops[i].am_length, ops[i].am_flags); + } + + if (copy_to_user(compat_ptr(am_hreq.ops), ops, size)) + error = -EFAULT; + + kfree(ops); + out_dput: + dput(dentry); + return error; +} + +long +xfs_file_compat_ioctl( + struct file *filp, + unsigned cmd, + unsigned long p) +{ + struct inode *inode = file_inode(filp); + struct xfs_inode *ip = XFS_I(inode); + void __user *arg = compat_ptr(p); + int error; + + trace_xfs_file_compat_ioctl(ip); + + switch (cmd) { +#if defined(BROKEN_X86_ALIGNMENT) + case XFS_IOC_FSGEOMETRY_V1_32: + return xfs_compat_ioc_fsgeometry_v1(ip->i_mount, arg); + case XFS_IOC_FSGROWFSDATA_32: { + struct xfs_growfs_data in; + + if (xfs_compat_growfs_data_copyin(&in, arg)) + return -EFAULT; + error = mnt_want_write_file(filp); + if (error) + return error; + error = xfs_growfs_data(ip->i_mount, &in); + mnt_drop_write_file(filp); + return error; + } + case XFS_IOC_FSGROWFSRT_32: { + struct xfs_growfs_rt in; + + if (xfs_compat_growfs_rt_copyin(&in, arg)) + return -EFAULT; + error = mnt_want_write_file(filp); + if (error) + return error; + error = xfs_growfs_rt(ip->i_mount, &in); + mnt_drop_write_file(filp); + return error; + } +#endif + /* long changes size, but xfs only copiese out 32 bits */ + case XFS_IOC_GETVERSION_32: + cmd = _NATIVE_IOC(cmd, long); + return xfs_file_ioctl(filp, cmd, p); + case XFS_IOC_SWAPEXT_32: { + struct xfs_swapext sxp; + struct compat_xfs_swapext __user *sxu = arg; + + /* Bulk copy in up to the sx_stat field, then copy bstat */ + if (copy_from_user(&sxp, sxu, + offsetof(struct xfs_swapext, sx_stat)) || + xfs_ioctl32_bstat_copyin(&sxp.sx_stat, &sxu->sx_stat)) + return -EFAULT; + error = mnt_want_write_file(filp); + if (error) + return error; + error = xfs_ioc_swapext(&sxp); + mnt_drop_write_file(filp); + return error; + } + case XFS_IOC_FSBULKSTAT_32: + case XFS_IOC_FSBULKSTAT_SINGLE_32: + case XFS_IOC_FSINUMBERS_32: + return xfs_compat_ioc_fsbulkstat(filp, cmd, arg); + case XFS_IOC_FD_TO_HANDLE_32: + case XFS_IOC_PATH_TO_HANDLE_32: + case XFS_IOC_PATH_TO_FSHANDLE_32: { + struct xfs_fsop_handlereq hreq; + + if (xfs_compat_handlereq_copyin(&hreq, arg)) + return -EFAULT; + cmd = _NATIVE_IOC(cmd, struct xfs_fsop_handlereq); + return xfs_find_handle(cmd, &hreq); + } + case XFS_IOC_OPEN_BY_HANDLE_32: { + struct xfs_fsop_handlereq hreq; + + if (xfs_compat_handlereq_copyin(&hreq, arg)) + return -EFAULT; + return xfs_open_by_handle(filp, &hreq); + } + case XFS_IOC_READLINK_BY_HANDLE_32: { + struct xfs_fsop_handlereq hreq; + + if (xfs_compat_handlereq_copyin(&hreq, arg)) + return -EFAULT; + return xfs_readlink_by_handle(filp, &hreq); + } + case XFS_IOC_ATTRLIST_BY_HANDLE_32: + return xfs_compat_attrlist_by_handle(filp, arg); + case XFS_IOC_ATTRMULTI_BY_HANDLE_32: + return xfs_compat_attrmulti_by_handle(filp, arg); + default: + /* try the native version */ + return xfs_file_ioctl(filp, cmd, (unsigned long)arg); + } +} diff --git a/fs/xfs/xfs_ioctl32.h b/fs/xfs/xfs_ioctl32.h new file mode 100644 index 000000000..052d0e888 --- /dev/null +++ b/fs/xfs/xfs_ioctl32.h @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2004-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_IOCTL32_H__ +#define __XFS_IOCTL32_H__ + +#include + +/* + * on 32-bit arches, ioctl argument structures may have different sizes + * and/or alignment. We define compat structures which match the + * 32-bit sizes/alignments here, and their associated ioctl numbers. + * + * xfs_ioctl32.c contains routines to copy these structures in and out. + */ + +/* stock kernel-level ioctls we support */ +#define XFS_IOC_GETVERSION_32 FS_IOC32_GETVERSION + +/* + * On intel, even if sizes match, alignment and/or padding may differ. + */ +#if defined(CONFIG_X86_64) +#define BROKEN_X86_ALIGNMENT +#define __compat_packed __attribute__((packed)) +#else +#define __compat_packed +#endif + +typedef struct compat_xfs_bstime { + old_time32_t tv_sec; /* seconds */ + __s32 tv_nsec; /* and nanoseconds */ +} compat_xfs_bstime_t; + +struct compat_xfs_bstat { + __u64 bs_ino; /* inode number */ + __u16 bs_mode; /* type and mode */ + __u16 bs_nlink; /* number of links */ + __u32 bs_uid; /* user id */ + __u32 bs_gid; /* group id */ + __u32 bs_rdev; /* device value */ + __s32 bs_blksize; /* block size */ + __s64 bs_size; /* file size */ + compat_xfs_bstime_t bs_atime; /* access time */ + compat_xfs_bstime_t bs_mtime; /* modify time */ + compat_xfs_bstime_t bs_ctime; /* inode change time */ + int64_t bs_blocks; /* number of blocks */ + __u32 bs_xflags; /* extended flags */ + __s32 bs_extsize; /* extent size */ + __s32 bs_extents; /* number of extents */ + __u32 bs_gen; /* generation count */ + __u16 bs_projid_lo; /* lower part of project id */ +#define bs_projid bs_projid_lo /* (previously just bs_projid) */ + __u16 bs_forkoff; /* inode fork offset in bytes */ + __u16 bs_projid_hi; /* high part of project id */ + unsigned char bs_pad[10]; /* pad space, unused */ + __u32 bs_dmevmask; /* DMIG event mask */ + __u16 bs_dmstate; /* DMIG state info */ + __u16 bs_aextents; /* attribute number of extents */ +} __compat_packed; + +struct compat_xfs_fsop_bulkreq { + compat_uptr_t lastip; /* last inode # pointer */ + __s32 icount; /* count of entries in buffer */ + compat_uptr_t ubuffer; /* user buffer for inode desc. */ + compat_uptr_t ocount; /* output count pointer */ +}; + +#define XFS_IOC_FSBULKSTAT_32 \ + _IOWR('X', 101, struct compat_xfs_fsop_bulkreq) +#define XFS_IOC_FSBULKSTAT_SINGLE_32 \ + _IOWR('X', 102, struct compat_xfs_fsop_bulkreq) +#define XFS_IOC_FSINUMBERS_32 \ + _IOWR('X', 103, struct compat_xfs_fsop_bulkreq) + +typedef struct compat_xfs_fsop_handlereq { + __u32 fd; /* fd for FD_TO_HANDLE */ + compat_uptr_t path; /* user pathname */ + __u32 oflags; /* open flags */ + compat_uptr_t ihandle; /* user supplied handle */ + __u32 ihandlen; /* user supplied length */ + compat_uptr_t ohandle; /* user buffer for handle */ + compat_uptr_t ohandlen; /* user buffer length */ +} compat_xfs_fsop_handlereq_t; + +#define XFS_IOC_PATH_TO_FSHANDLE_32 \ + _IOWR('X', 104, struct compat_xfs_fsop_handlereq) +#define XFS_IOC_PATH_TO_HANDLE_32 \ + _IOWR('X', 105, struct compat_xfs_fsop_handlereq) +#define XFS_IOC_FD_TO_HANDLE_32 \ + _IOWR('X', 106, struct compat_xfs_fsop_handlereq) +#define XFS_IOC_OPEN_BY_HANDLE_32 \ + _IOWR('X', 107, struct compat_xfs_fsop_handlereq) +#define XFS_IOC_READLINK_BY_HANDLE_32 \ + _IOWR('X', 108, struct compat_xfs_fsop_handlereq) + +/* The bstat field in the swapext struct needs translation */ +struct compat_xfs_swapext { + int64_t sx_version; /* version */ + int64_t sx_fdtarget; /* fd of target file */ + int64_t sx_fdtmp; /* fd of tmp file */ + xfs_off_t sx_offset; /* offset into file */ + xfs_off_t sx_length; /* leng from offset */ + char sx_pad[16]; /* pad space, unused */ + struct compat_xfs_bstat sx_stat; /* stat of target b4 copy */ +} __compat_packed; + +#define XFS_IOC_SWAPEXT_32 _IOWR('X', 109, struct compat_xfs_swapext) + +typedef struct compat_xfs_fsop_attrlist_handlereq { + struct compat_xfs_fsop_handlereq hreq; /* handle interface structure */ + struct xfs_attrlist_cursor pos; /* opaque cookie, list offset */ + __u32 flags; /* which namespace to use */ + __u32 buflen; /* length of buffer supplied */ + compat_uptr_t buffer; /* returned names */ +} __compat_packed compat_xfs_fsop_attrlist_handlereq_t; + +/* Note: actually this is read/write */ +#define XFS_IOC_ATTRLIST_BY_HANDLE_32 \ + _IOW('X', 122, struct compat_xfs_fsop_attrlist_handlereq) + +/* am_opcodes defined in xfs_fs.h */ +typedef struct compat_xfs_attr_multiop { + __u32 am_opcode; + __s32 am_error; + compat_uptr_t am_attrname; + compat_uptr_t am_attrvalue; + __u32 am_length; + __u32 am_flags; +} compat_xfs_attr_multiop_t; + +typedef struct compat_xfs_fsop_attrmulti_handlereq { + struct compat_xfs_fsop_handlereq hreq; /* handle interface structure */ + __u32 opcount;/* count of following multiop */ + /* ptr to compat_xfs_attr_multiop */ + compat_uptr_t ops; /* attr_multi data */ +} compat_xfs_fsop_attrmulti_handlereq_t; + +#define XFS_IOC_ATTRMULTI_BY_HANDLE_32 \ + _IOW('X', 123, struct compat_xfs_fsop_attrmulti_handlereq) + +#ifdef BROKEN_X86_ALIGNMENT +typedef struct compat_xfs_fsop_geom_v1 { + __u32 blocksize; /* filesystem (data) block size */ + __u32 rtextsize; /* realtime extent size */ + __u32 agblocks; /* fsblocks in an AG */ + __u32 agcount; /* number of allocation groups */ + __u32 logblocks; /* fsblocks in the log */ + __u32 sectsize; /* (data) sector size, bytes */ + __u32 inodesize; /* inode size in bytes */ + __u32 imaxpct; /* max allowed inode space(%) */ + __u64 datablocks; /* fsblocks in data subvolume */ + __u64 rtblocks; /* fsblocks in realtime subvol */ + __u64 rtextents; /* rt extents in realtime subvol*/ + __u64 logstart; /* starting fsblock of the log */ + unsigned char uuid[16]; /* unique id of the filesystem */ + __u32 sunit; /* stripe unit, fsblocks */ + __u32 swidth; /* stripe width, fsblocks */ + __s32 version; /* structure version */ + __u32 flags; /* superblock version flags */ + __u32 logsectsize; /* log sector size, bytes */ + __u32 rtsectsize; /* realtime sector size, bytes */ + __u32 dirblocksize; /* directory block size, bytes */ +} __attribute__((packed)) compat_xfs_fsop_geom_v1_t; + +#define XFS_IOC_FSGEOMETRY_V1_32 \ + _IOR('X', 100, struct compat_xfs_fsop_geom_v1) + +struct compat_xfs_inogrp { + __u64 xi_startino; /* starting inode number */ + __s32 xi_alloccount; /* # bits set in allocmask */ + __u64 xi_allocmask; /* mask of allocated inodes */ +} __attribute__((packed)); + +/* These growfs input structures have padding on the end, so must translate */ +typedef struct compat_xfs_growfs_data { + __u64 newblocks; /* new data subvol size, fsblocks */ + __u32 imaxpct; /* new inode space percentage limit */ +} __attribute__((packed)) compat_xfs_growfs_data_t; + +typedef struct compat_xfs_growfs_rt { + __u64 newblocks; /* new realtime size, fsblocks */ + __u32 extsize; /* new realtime extent size, fsblocks */ +} __attribute__((packed)) compat_xfs_growfs_rt_t; + +#define XFS_IOC_FSGROWFSDATA_32 _IOW('X', 110, struct compat_xfs_growfs_data) +#define XFS_IOC_FSGROWFSRT_32 _IOW('X', 112, struct compat_xfs_growfs_rt) + +#endif /* BROKEN_X86_ALIGNMENT */ + +#endif /* __XFS_IOCTL32_H__ */ diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c new file mode 100644 index 000000000..71c45be8c --- /dev/null +++ b/fs/xfs/xfs_iomap.c @@ -0,0 +1,2410 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2006 Silicon Graphics, Inc. + * Copyright (c) 2016-2018 Christoph Hellwig. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_btree.h" +#include "xfs_bmap_btree.h" +#include "xfs_bmap.h" +#include "xfs_bmap_util.h" +#include "xfs_errortag.h" +#include "xfs_error.h" +#include "xfs_trans.h" +#include "xfs_trans_space.h" +#include "xfs_inode_item.h" +#include "xfs_iomap.h" +#include "xfs_trace.h" +#include "xfs_quota.h" +#include "xfs_rtgroup.h" +#include "xfs_dquot_item.h" +#include "xfs_dquot.h" +#include "xfs_reflink.h" +#include "xfs_health.h" +#include "xfs_rtbitmap.h" +#include "xfs_icache.h" +#include "xfs_zone_alloc.h" + +#define XFS_ALLOC_ALIGN(mp, off) \ + (((off) >> mp->m_allocsize_log) << mp->m_allocsize_log) + +static int +xfs_alert_fsblock_zero( + xfs_inode_t *ip, + xfs_bmbt_irec_t *imap) +{ + xfs_alert_tag(ip->i_mount, XFS_PTAG_FSBLOCK_ZERO, + "Access to block zero in inode %llu " + "start_block: %llx start_off: %llx " + "blkcnt: %llx extent-state: %x", + (unsigned long long)I_INO(ip), + (unsigned long long)imap->br_startblock, + (unsigned long long)imap->br_startoff, + (unsigned long long)imap->br_blockcount, + imap->br_state); + xfs_bmap_mark_sick(ip, XFS_DATA_FORK); + return -EFSCORRUPTED; +} + +u64 +xfs_iomap_inode_sequence( + struct xfs_inode *ip, + u16 iomap_flags) +{ + u64 cookie = 0; + + if (iomap_flags & IOMAP_F_XATTR) + return READ_ONCE(ip->i_af.if_seq); + if ((iomap_flags & IOMAP_F_SHARED) && ip->i_cowfp) + cookie = (u64)READ_ONCE(ip->i_cowfp->if_seq) << 32; + return cookie | READ_ONCE(ip->i_df.if_seq); +} + +/* + * Check that the iomap passed to us is still valid for the given offset and + * length. + */ +static bool +xfs_iomap_valid( + struct inode *inode, + const struct iomap *iomap) +{ + struct xfs_inode *ip = XFS_I(inode); + + if (iomap->type == IOMAP_HOLE) + return true; + + if (iomap->validity_cookie != + xfs_iomap_inode_sequence(ip, iomap->flags)) { + trace_xfs_iomap_invalid(ip, iomap); + return false; + } + + XFS_ERRORTAG_DELAY(ip->i_mount, XFS_ERRTAG_WRITE_DELAY_MS); + return true; +} + +const struct iomap_write_ops xfs_iomap_write_ops = { + .iomap_valid = xfs_iomap_valid, +}; + +int +xfs_bmbt_to_iomap( + struct xfs_inode *ip, + struct iomap *iomap, + struct xfs_bmbt_irec *imap, + unsigned int mapping_flags, + u16 iomap_flags, + u64 sequence_cookie) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_buftarg *target = xfs_inode_buftarg(ip); + + if (unlikely(!xfs_valid_startblock(ip, imap->br_startblock))) { + xfs_bmap_mark_sick(ip, XFS_DATA_FORK); + return xfs_alert_fsblock_zero(ip, imap); + } + + iomap->flags = iomap_flags; + if (imap->br_startblock == HOLESTARTBLOCK) { + iomap->addr = IOMAP_NULL_ADDR; + iomap->type = IOMAP_HOLE; + } else if (imap->br_startblock == DELAYSTARTBLOCK || + isnullstartblock(imap->br_startblock)) { + iomap->addr = IOMAP_NULL_ADDR; + iomap->type = IOMAP_DELALLOC; + } else { + xfs_daddr_t daddr = xfs_fsb_to_db(ip, imap->br_startblock); + + iomap->addr = BBTOB(daddr); + if (mapping_flags & IOMAP_DAX) + iomap->addr += target->bt_dax_part_off; + + if (imap->br_state == XFS_EXT_UNWRITTEN) + iomap->type = IOMAP_UNWRITTEN; + else + iomap->type = IOMAP_MAPPED; + + /* + * Mark iomaps starting at the first sector of a RTG as merge + * boundary so that each I/O completions is contained to a + * single RTG. + */ + if (XFS_IS_REALTIME_INODE(ip) && xfs_has_rtgroups(mp) && + xfs_rtbno_is_group_start(mp, imap->br_startblock)) + iomap->flags |= IOMAP_F_BOUNDARY; + } + iomap->offset = XFS_FSB_TO_B(mp, imap->br_startoff); + iomap->length = XFS_FSB_TO_B(mp, imap->br_blockcount); + if (mapping_flags & IOMAP_DAX) { + iomap->dax_dev = target->bt_daxdev; + } else { + iomap->bdev = target->bt_bdev; + if (bdev_has_integrity_csum(iomap->bdev)) + iomap->flags |= IOMAP_F_INTEGRITY; + } + + /* + * If the inode is dirty for datasync purposes, let iomap know so it + * doesn't elide the IO completion journal flushes on O_DSYNC IO. + */ + if (ip->i_itemp) { + struct xfs_inode_log_item *iip = ip->i_itemp; + + spin_lock(&iip->ili_lock); + if (iip->ili_datasync_seq) + iomap->flags |= IOMAP_F_DIRTY; + spin_unlock(&iip->ili_lock); + } + + iomap->validity_cookie = sequence_cookie; + return 0; +} + +static void +xfs_hole_to_iomap( + struct xfs_inode *ip, + struct iomap *iomap, + xfs_fileoff_t offset_fsb, + xfs_fileoff_t end_fsb) +{ + struct xfs_buftarg *target = xfs_inode_buftarg(ip); + + iomap->addr = IOMAP_NULL_ADDR; + iomap->type = IOMAP_HOLE; + iomap->offset = XFS_FSB_TO_B(ip->i_mount, offset_fsb); + iomap->length = XFS_FSB_TO_B(ip->i_mount, end_fsb - offset_fsb); + iomap->bdev = target->bt_bdev; + iomap->dax_dev = target->bt_daxdev; +} + +static inline xfs_fileoff_t +xfs_iomap_end_fsb( + struct xfs_mount *mp, + loff_t offset, + loff_t count) +{ + ASSERT(offset <= mp->m_super->s_maxbytes); + return min(XFS_B_TO_FSB(mp, offset + count), + XFS_B_TO_FSB(mp, mp->m_super->s_maxbytes)); +} + +static xfs_extlen_t +xfs_eof_alignment( + struct xfs_inode *ip) +{ + struct xfs_mount *mp = ip->i_mount; + xfs_extlen_t align = 0; + + if (!XFS_IS_REALTIME_INODE(ip)) { + /* + * Round up the allocation request to a stripe unit + * (m_dalign) boundary if the file size is >= stripe unit + * size, and we are allocating past the allocation eof. + * + * If mounted with the "-o swalloc" option the alignment is + * increased from the strip unit size to the stripe width. + */ + if (mp->m_swidth && xfs_has_swalloc(mp)) + align = mp->m_swidth; + else if (mp->m_dalign) + align = mp->m_dalign; + + if (align && XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, align)) + align = 0; + } + + return align; +} + +/* + * Check if last_fsb is outside the last extent, and if so grow it to the next + * stripe unit boundary. + */ +xfs_fileoff_t +xfs_iomap_eof_align_last_fsb( + struct xfs_inode *ip, + xfs_fileoff_t end_fsb) +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK); + xfs_extlen_t extsz = xfs_get_extsz_hint(ip); + xfs_extlen_t align = xfs_eof_alignment(ip); + struct xfs_bmbt_irec irec; + struct xfs_iext_cursor icur; + + ASSERT(!xfs_need_iread_extents(ifp)); + + /* + * Always round up the allocation request to the extent hint boundary. + */ + if (extsz) { + if (align) + align = roundup_64(align, extsz); + else + align = extsz; + } + + if (align) { + xfs_fileoff_t aligned_end_fsb = roundup_64(end_fsb, align); + + xfs_iext_last(ifp, &icur); + if (!xfs_iext_get_extent(ifp, &icur, &irec) || + aligned_end_fsb >= irec.br_startoff + irec.br_blockcount) + return aligned_end_fsb; + } + + return end_fsb; +} + +int +xfs_iomap_write_direct( + struct xfs_inode *ip, + xfs_fileoff_t offset_fsb, + xfs_fileoff_t count_fsb, + unsigned int flags, + struct xfs_bmbt_irec *imap, + u64 *seq) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_trans *tp; + xfs_filblks_t resaligned; + int nimaps; + unsigned int dblocks, rblocks; + bool force = false; + int error; + int bmapi_flags = XFS_BMAPI_PREALLOC; + int nr_exts = XFS_IEXT_ADD_NOSPLIT_CNT; + + ASSERT(count_fsb > 0); + + resaligned = xfs_aligned_fsb_count(offset_fsb, count_fsb, + xfs_get_extsz_hint(ip)); + if (unlikely(XFS_IS_REALTIME_INODE(ip))) { + dblocks = XFS_DIOSTRAT_SPACE_RES(mp, 0); + rblocks = resaligned; + } else { + dblocks = XFS_DIOSTRAT_SPACE_RES(mp, resaligned); + rblocks = 0; + } + + error = xfs_qm_dqattach(ip); + if (error) + return error; + + /* + * For DAX, we do not allocate unwritten extents, but instead we zero + * the block before we commit the transaction. Ideally we'd like to do + * this outside the transaction context, but if we commit and then crash + * we may not have zeroed the blocks and this will be exposed on + * recovery of the allocation. Hence we must zero before commit. + * + * Further, if we are mapping unwritten extents here, we need to zero + * and convert them to written so that we don't need an unwritten extent + * callback for DAX. This also means that we need to be able to dip into + * the reserve block pool for bmbt block allocation if there is no space + * left but we need to do unwritten extent conversion. + */ + if (flags & IOMAP_DAX) { + bmapi_flags = XFS_BMAPI_CONVERT | XFS_BMAPI_ZERO; + if (imap->br_state == XFS_EXT_UNWRITTEN) { + force = true; + nr_exts = XFS_IEXT_WRITE_UNWRITTEN_CNT; + dblocks = XFS_DIOSTRAT_SPACE_RES(mp, 0) << 1; + } + } + + error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, dblocks, + rblocks, force, &tp); + if (error) + return error; + + error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, nr_exts); + if (error) + goto out_trans_cancel; + + /* + * From this point onwards we overwrite the imap pointer that the + * caller gave to us. + */ + nimaps = 1; + error = xfs_bmapi_write(tp, ip, offset_fsb, count_fsb, bmapi_flags, 0, + imap, &nimaps); + if (error) + goto out_trans_cancel; + + /* + * Complete the transaction + */ + error = xfs_trans_commit(tp); + if (error) + goto out_unlock; + + if (unlikely(!xfs_valid_startblock(ip, imap->br_startblock))) { + xfs_bmap_mark_sick(ip, XFS_DATA_FORK); + error = xfs_alert_fsblock_zero(ip, imap); + } + +out_unlock: + *seq = xfs_iomap_inode_sequence(ip, 0); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; + +out_trans_cancel: + xfs_trans_cancel(tp); + goto out_unlock; +} + +STATIC bool +xfs_quota_need_throttle( + struct xfs_inode *ip, + xfs_dqtype_t type, + xfs_fsblock_t alloc_blocks) +{ + struct xfs_dquot *dq = xfs_inode_dquot(ip, type); + struct xfs_dquot_res *res; + struct xfs_dquot_pre *pre; + + if (!dq || !xfs_this_quota_on(ip->i_mount, type)) + return false; + + if (XFS_IS_REALTIME_INODE(ip)) { + res = &dq->q_rtb; + pre = &dq->q_rtb_prealloc; + } else { + res = &dq->q_blk; + pre = &dq->q_blk_prealloc; + } + + /* no hi watermark, no throttle */ + if (!pre->q_prealloc_hi_wmark) + return false; + + /* under the lo watermark, no throttle */ + if (res->reserved + alloc_blocks < pre->q_prealloc_lo_wmark) + return false; + + return true; +} + +STATIC void +xfs_quota_calc_throttle( + struct xfs_inode *ip, + xfs_dqtype_t type, + xfs_fsblock_t *qblocks, + int *qshift, + int64_t *qfreesp) +{ + struct xfs_dquot *dq = xfs_inode_dquot(ip, type); + struct xfs_dquot_res *res; + struct xfs_dquot_pre *pre; + int64_t freesp; + int shift = 0; + + if (!dq) { + res = NULL; + pre = NULL; + } else if (XFS_IS_REALTIME_INODE(ip)) { + res = &dq->q_rtb; + pre = &dq->q_rtb_prealloc; + } else { + res = &dq->q_blk; + pre = &dq->q_blk_prealloc; + } + + /* no dq, or over hi wmark, squash the prealloc completely */ + if (!res || res->reserved >= pre->q_prealloc_hi_wmark) { + *qblocks = 0; + *qfreesp = 0; + return; + } + + freesp = pre->q_prealloc_hi_wmark - res->reserved; + if (freesp < pre->q_low_space[XFS_QLOWSP_5_PCNT]) { + shift = 2; + if (freesp < pre->q_low_space[XFS_QLOWSP_3_PCNT]) + shift += 2; + if (freesp < pre->q_low_space[XFS_QLOWSP_1_PCNT]) + shift += 2; + } + + if (freesp < *qfreesp) + *qfreesp = freesp; + + /* only overwrite the throttle values if we are more aggressive */ + if ((freesp >> shift) < (*qblocks >> *qshift)) { + *qblocks = freesp; + *qshift = shift; + } +} + +static int64_t +xfs_iomap_freesp( + struct xfs_mount *mp, + unsigned int idx, + uint64_t low_space[XFS_LOWSP_MAX], + int *shift) +{ + int64_t freesp; + + freesp = xfs_estimate_freecounter(mp, idx); + if (freesp < low_space[XFS_LOWSP_5_PCNT]) { + *shift = 2; + if (freesp < low_space[XFS_LOWSP_4_PCNT]) + (*shift)++; + if (freesp < low_space[XFS_LOWSP_3_PCNT]) + (*shift)++; + if (freesp < low_space[XFS_LOWSP_2_PCNT]) + (*shift)++; + if (freesp < low_space[XFS_LOWSP_1_PCNT]) + (*shift)++; + } + return freesp; +} + +/* + * If we don't have a user specified preallocation size, dynamically increase + * the preallocation size as the size of the file grows. Cap the maximum size + * at a single extent or less if the filesystem is near full. The closer the + * filesystem is to being full, the smaller the maximum preallocation. + */ +STATIC xfs_fsblock_t +xfs_iomap_prealloc_size( + struct xfs_inode *ip, + int whichfork, + loff_t offset, + loff_t count, + struct xfs_iext_cursor *icur) +{ + struct xfs_iext_cursor ncur = *icur; + struct xfs_bmbt_irec prev, got; + struct xfs_mount *mp = ip->i_mount; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); + int64_t freesp; + xfs_fsblock_t qblocks; + xfs_fsblock_t alloc_blocks = 0; + xfs_extlen_t plen; + int shift = 0; + int qshift = 0; + + /* + * As an exception we don't do any preallocation at all if the file is + * smaller than the minimum preallocation and we are using the default + * dynamic preallocation scheme, as it is likely this is the only write + * to the file that is going to be done. + */ + if (XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, mp->m_allocsize_blocks)) + return 0; + + /* + * Use the minimum preallocation size for small files or if we are + * writing right after a hole. + */ + if (XFS_ISIZE(ip) < XFS_FSB_TO_B(mp, mp->m_dalign) || + !xfs_iext_prev_extent(ifp, &ncur, &prev) || + prev.br_startoff + prev.br_blockcount < offset_fsb) + return mp->m_allocsize_blocks; + + /* + * Take the size of the preceding data extents as the basis for the + * preallocation size. Note that we don't care if the previous extents + * are written or not. + */ + plen = prev.br_blockcount; + while (xfs_iext_prev_extent(ifp, &ncur, &got)) { + if (plen > XFS_MAX_BMBT_EXTLEN / 2 || + isnullstartblock(got.br_startblock) || + got.br_startoff + got.br_blockcount != prev.br_startoff || + got.br_startblock + got.br_blockcount != prev.br_startblock) + break; + plen += got.br_blockcount; + prev = got; + } + + /* + * If the size of the extents is greater than half the maximum extent + * length, then use the current offset as the basis. This ensures that + * for large files the preallocation size always extends to + * XFS_BMBT_MAX_EXTLEN rather than falling short due to things like stripe + * unit/width alignment of real extents. + */ + alloc_blocks = plen * 2; + if (alloc_blocks > XFS_MAX_BMBT_EXTLEN) + alloc_blocks = XFS_B_TO_FSB(mp, offset); + qblocks = alloc_blocks; + + /* + * XFS_BMBT_MAX_EXTLEN is not a power of two value but we round the prealloc + * down to the nearest power of two value after throttling. To prevent + * the round down from unconditionally reducing the maximum supported + * prealloc size, we round up first, apply appropriate throttling, round + * down and cap the value to XFS_BMBT_MAX_EXTLEN. + */ + alloc_blocks = XFS_FILEOFF_MIN(roundup_pow_of_two(XFS_MAX_BMBT_EXTLEN), + alloc_blocks); + + if (unlikely(XFS_IS_REALTIME_INODE(ip))) + freesp = xfs_rtbxlen_to_blen(mp, + xfs_iomap_freesp(mp, XC_FREE_RTEXTENTS, + mp->m_low_rtexts, &shift)); + else + freesp = xfs_iomap_freesp(mp, XC_FREE_BLOCKS, mp->m_low_space, + &shift); + + /* + * Check each quota to cap the prealloc size, provide a shift value to + * throttle with and adjust amount of available space. + */ + if (xfs_quota_need_throttle(ip, XFS_DQTYPE_USER, alloc_blocks)) + xfs_quota_calc_throttle(ip, XFS_DQTYPE_USER, &qblocks, &qshift, + &freesp); + if (xfs_quota_need_throttle(ip, XFS_DQTYPE_GROUP, alloc_blocks)) + xfs_quota_calc_throttle(ip, XFS_DQTYPE_GROUP, &qblocks, &qshift, + &freesp); + if (xfs_quota_need_throttle(ip, XFS_DQTYPE_PROJ, alloc_blocks)) + xfs_quota_calc_throttle(ip, XFS_DQTYPE_PROJ, &qblocks, &qshift, + &freesp); + + /* + * The final prealloc size is set to the minimum of free space available + * in each of the quotas and the overall filesystem. + * + * The shift throttle value is set to the maximum value as determined by + * the global low free space values and per-quota low free space values. + */ + alloc_blocks = min(alloc_blocks, qblocks); + shift = max(shift, qshift); + + if (shift) + alloc_blocks >>= shift; + /* + * rounddown_pow_of_two() returns an undefined result if we pass in + * alloc_blocks = 0. + */ + if (alloc_blocks) + alloc_blocks = rounddown_pow_of_two(alloc_blocks); + if (alloc_blocks > XFS_MAX_BMBT_EXTLEN) + alloc_blocks = XFS_MAX_BMBT_EXTLEN; + + /* + * If we are still trying to allocate more space than is + * available, squash the prealloc hard. This can happen if we + * have a large file on a small filesystem and the above + * lowspace thresholds are smaller than XFS_BMBT_MAX_EXTLEN. + */ + while (alloc_blocks && alloc_blocks >= freesp) + alloc_blocks >>= 4; + if (alloc_blocks < mp->m_allocsize_blocks) + alloc_blocks = mp->m_allocsize_blocks; + trace_xfs_iomap_prealloc_size(ip, alloc_blocks, shift, + mp->m_allocsize_blocks); + return alloc_blocks; +} + +int +xfs_iomap_write_unwritten( + xfs_inode_t *ip, + xfs_off_t offset, + xfs_off_t count, + bool update_isize) +{ + xfs_mount_t *mp = ip->i_mount; + xfs_fileoff_t offset_fsb; + xfs_filblks_t count_fsb; + xfs_filblks_t numblks_fsb; + int nimaps; + xfs_trans_t *tp; + xfs_bmbt_irec_t imap; + struct inode *inode = VFS_I(ip); + xfs_fsize_t i_size; + uint resblks; + int error; + + trace_xfs_unwritten_convert(ip, offset, count); + + offset_fsb = XFS_B_TO_FSBT(mp, offset); + count_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)offset + count); + count_fsb = (xfs_filblks_t)(count_fsb - offset_fsb); + + /* + * Reserve enough blocks in this transaction for two complete extent + * btree splits. We may be converting the middle part of an unwritten + * extent and in this case we will insert two new extents in the btree + * each of which could cause a full split. + * + * This reservation amount will be used in the first call to + * xfs_bmbt_split() to select an AG with enough space to satisfy the + * rest of the operation. + */ + resblks = XFS_DIOSTRAT_SPACE_RES(mp, 0) << 1; + + /* Attach dquots so that bmbt splits are accounted correctly. */ + error = xfs_qm_dqattach(ip); + if (error) + return error; + + do { + /* + * Set up a transaction to convert the range of extents + * from unwritten to real. Do allocations in a loop until + * we have covered the range passed in. + * + * Note that we can't risk to recursing back into the filesystem + * here as we might be asked to write out the same inode that we + * complete here and might deadlock on the iolock. + */ + error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, resblks, + 0, true, &tp); + if (error) + return error; + + error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, + XFS_IEXT_WRITE_UNWRITTEN_CNT); + if (error) + goto error_on_bmapi_transaction; + + /* + * Modify the unwritten extent state of the buffer. + */ + nimaps = 1; + error = xfs_bmapi_write(tp, ip, offset_fsb, count_fsb, + XFS_BMAPI_CONVERT, resblks, &imap, + &nimaps); + if (error) + goto error_on_bmapi_transaction; + + /* + * Log the updated inode size as we go. We have to be careful + * to only log it up to the actual write offset if it is + * halfway into a block. + */ + i_size = XFS_FSB_TO_B(mp, offset_fsb + count_fsb); + if (i_size > offset + count) + i_size = offset + count; + if (update_isize && i_size > i_size_read(inode)) + i_size_write(inode, i_size); + i_size = xfs_new_eof(ip, i_size); + if (i_size) { + ip->i_disk_size = i_size; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + } + + error = xfs_trans_commit(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + if (error) + return error; + + if (unlikely(!xfs_valid_startblock(ip, imap.br_startblock))) { + xfs_bmap_mark_sick(ip, XFS_DATA_FORK); + return xfs_alert_fsblock_zero(ip, &imap); + } + + if ((numblks_fsb = imap.br_blockcount) == 0) { + /* + * The numblks_fsb value should always get + * smaller, otherwise the loop is stuck. + */ + ASSERT(imap.br_blockcount); + break; + } + offset_fsb += numblks_fsb; + count_fsb -= numblks_fsb; + } while (count_fsb > 0); + + return 0; + +error_on_bmapi_transaction: + xfs_trans_cancel(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; +} + +static inline bool +imap_needs_alloc( + struct inode *inode, + unsigned flags, + struct xfs_bmbt_irec *imap, + int nimaps) +{ + /* don't allocate blocks when just zeroing */ + if (flags & IOMAP_ZERO) + return false; + if (!nimaps || + imap->br_startblock == HOLESTARTBLOCK || + imap->br_startblock == DELAYSTARTBLOCK) + return true; + /* we convert unwritten extents before copying the data for DAX */ + if ((flags & IOMAP_DAX) && imap->br_state == XFS_EXT_UNWRITTEN) + return true; + return false; +} + +static inline bool +imap_needs_cow( + struct xfs_inode *ip, + unsigned int flags, + struct xfs_bmbt_irec *imap, + int nimaps) +{ + if (!xfs_is_cow_inode(ip)) + return false; + + /* when zeroing we don't have to COW holes or unwritten extents */ + if (flags & (IOMAP_UNSHARE | IOMAP_ZERO)) { + if (!nimaps || + imap->br_startblock == HOLESTARTBLOCK || + imap->br_state == XFS_EXT_UNWRITTEN) + return false; + } + + return true; +} + +/* + * Extents not yet cached requires exclusive access, don't block for + * IOMAP_NOWAIT. + * + * This is basically an opencoded xfs_ilock_data_map_shared() call, but with + * support for IOMAP_NOWAIT. + */ +static int +xfs_ilock_for_iomap( + struct xfs_inode *ip, + unsigned flags, + unsigned *lockmode) +{ + if (flags & IOMAP_NOWAIT) { + if (xfs_need_iread_extents(&ip->i_df)) + return -EAGAIN; + if (!xfs_ilock_nowait(ip, *lockmode)) + return -EAGAIN; + } else { + if (xfs_need_iread_extents(&ip->i_df)) + *lockmode = XFS_ILOCK_EXCL; + xfs_ilock(ip, *lockmode); + } + + return 0; +} + +/* + * Check that the imap we are going to return to the caller spans the entire + * range that the caller requested for the IO. + */ +static bool +imap_spans_range( + struct xfs_bmbt_irec *imap, + xfs_fileoff_t offset_fsb, + xfs_fileoff_t end_fsb) +{ + if (imap->br_startoff > offset_fsb) + return false; + if (imap->br_startoff + imap->br_blockcount < end_fsb) + return false; + return true; +} + +static bool +xfs_bmap_hw_atomic_write_possible( + struct xfs_inode *ip, + struct xfs_bmbt_irec *imap, + xfs_fileoff_t offset_fsb, + xfs_fileoff_t end_fsb) +{ + struct xfs_mount *mp = ip->i_mount; + xfs_fsize_t len = XFS_FSB_TO_B(mp, end_fsb - offset_fsb); + + /* + * atomic writes are required to be naturally aligned for disk blocks, + * which ensures that we adhere to block layer rules that we won't + * straddle any boundary or violate write alignment requirement. + */ + if (!IS_ALIGNED(imap->br_startblock, imap->br_blockcount)) + return false; + + /* + * Spanning multiple extents would mean that multiple BIOs would be + * issued, and so would lose atomicity required for REQ_ATOMIC-based + * atomics. + */ + if (!imap_spans_range(imap, offset_fsb, end_fsb)) + return false; + + /* + * The ->iomap_begin caller should ensure this, but check anyway. + */ + return len <= xfs_inode_buftarg(ip)->bt_awu_max; +} + +static int +xfs_direct_write_iomap_begin( + struct inode *inode, + loff_t offset, + loff_t length, + unsigned flags, + struct iomap *iomap, + struct iomap *srcmap) +{ + struct xfs_inode *ip = XFS_I(inode); + struct xfs_mount *mp = ip->i_mount; + struct xfs_bmbt_irec imap, cmap; + xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); + xfs_fileoff_t end_fsb = xfs_iomap_end_fsb(mp, offset, length); + xfs_fileoff_t orig_end_fsb = end_fsb; + int nimaps = 1, error = 0; + bool shared = false; + u16 iomap_flags = 0; + bool needs_alloc; + unsigned int lockmode; + u64 seq; + + ASSERT(flags & (IOMAP_WRITE | IOMAP_ZERO)); + + if (xfs_is_shutdown(mp)) + return -EIO; + + /* + * Writes that span EOF might trigger an IO size update on completion, + * so consider them to be dirty for the purposes of O_DSYNC even if + * there is no other metadata changes pending or have been made here. + */ + if (offset + length > i_size_read(inode)) + iomap_flags |= IOMAP_F_DIRTY; + + /* HW-offload atomics are always used in this path */ + if (flags & IOMAP_ATOMIC) + iomap_flags |= IOMAP_F_ATOMIC_BIO; + + /* + * COW writes may allocate delalloc space or convert unwritten COW + * extents, so we need to make sure to take the lock exclusively here. + */ + if (xfs_is_cow_inode(ip)) + lockmode = XFS_ILOCK_EXCL; + else + lockmode = XFS_ILOCK_SHARED; + +relock: + error = xfs_ilock_for_iomap(ip, flags, &lockmode); + if (error) + return error; + + /* + * The reflink iflag could have changed since the earlier unlocked + * check, check if it again and relock if needed. + */ + if (xfs_is_cow_inode(ip) && lockmode == XFS_ILOCK_SHARED) { + xfs_iunlock(ip, lockmode); + lockmode = XFS_ILOCK_EXCL; + goto relock; + } + + error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, &imap, + &nimaps, 0); + if (error) + goto out_unlock; + + if (imap_needs_cow(ip, flags, &imap, nimaps)) { + error = -EAGAIN; + if (flags & IOMAP_NOWAIT) + goto out_unlock; + + /* may drop and re-acquire the ilock */ + error = xfs_reflink_allocate_cow(ip, &imap, &cmap, &shared, + &lockmode, + (flags & IOMAP_DIRECT) || IS_DAX(inode)); + if (error) + goto out_unlock; + if (shared) { + if ((flags & IOMAP_ATOMIC) && + !xfs_bmap_hw_atomic_write_possible(ip, &cmap, + offset_fsb, end_fsb)) { + error = -ENOPROTOOPT; + goto out_unlock; + } + goto out_found_cow; + } + end_fsb = imap.br_startoff + imap.br_blockcount; + length = XFS_FSB_TO_B(mp, end_fsb) - offset; + } + + needs_alloc = imap_needs_alloc(inode, flags, &imap, nimaps); + + if (flags & IOMAP_ATOMIC) { + error = -ENOPROTOOPT; + /* + * If we allocate less than what is required for the write + * then we may end up with multiple extents, which means that + * REQ_ATOMIC-based cannot be used, so avoid this possibility. + */ + if (needs_alloc && orig_end_fsb - offset_fsb > 1) + goto out_unlock; + + if (!xfs_bmap_hw_atomic_write_possible(ip, &imap, offset_fsb, + orig_end_fsb)) + goto out_unlock; + } + + if (needs_alloc) + goto allocate_blocks; + + /* + * NOWAIT and OVERWRITE I/O needs to span the entire requested I/O with + * a single map so that we avoid partial IO failures due to the rest of + * the I/O range not covered by this map triggering an EAGAIN condition + * when it is subsequently mapped and aborting the I/O. + */ + if (flags & (IOMAP_NOWAIT | IOMAP_OVERWRITE_ONLY)) { + error = -EAGAIN; + if (!imap_spans_range(&imap, offset_fsb, end_fsb)) + goto out_unlock; + } + + /* + * For overwrite only I/O, we cannot convert unwritten extents without + * requiring sub-block zeroing. This can only be done under an + * exclusive IOLOCK, hence return -EAGAIN if this is not a written + * extent to tell the caller to try again. + */ + if (flags & IOMAP_OVERWRITE_ONLY) { + error = -EAGAIN; + if (imap.br_state != XFS_EXT_NORM && + ((offset | length) & mp->m_blockmask)) + goto out_unlock; + } + + seq = xfs_iomap_inode_sequence(ip, iomap_flags); + xfs_iunlock(ip, lockmode); + trace_xfs_iomap_found(ip, offset, length, XFS_DATA_FORK, &imap); + return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, iomap_flags, seq); + +allocate_blocks: + error = -EAGAIN; + if (flags & (IOMAP_NOWAIT | IOMAP_OVERWRITE_ONLY)) + goto out_unlock; + + /* + * We cap the maximum length we map to a sane size to keep the chunks + * of work done where somewhat symmetric with the work writeback does. + * This is a completely arbitrary number pulled out of thin air as a + * best guess for initial testing. + * + * Note that the values needs to be less than 32-bits wide until the + * lower level functions are updated. + */ + length = min_t(loff_t, length, 1024 * PAGE_SIZE); + end_fsb = xfs_iomap_end_fsb(mp, offset, length); + + if (offset + length > XFS_ISIZE(ip)) + end_fsb = xfs_iomap_eof_align_last_fsb(ip, end_fsb); + else if (nimaps && imap.br_startblock == HOLESTARTBLOCK) + end_fsb = min(end_fsb, imap.br_startoff + imap.br_blockcount); + xfs_iunlock(ip, lockmode); + + error = xfs_iomap_write_direct(ip, offset_fsb, end_fsb - offset_fsb, + flags, &imap, &seq); + if (error) + return error; + + trace_xfs_iomap_alloc(ip, offset, length, XFS_DATA_FORK, &imap); + return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, + iomap_flags | IOMAP_F_NEW, seq); + +out_found_cow: + length = XFS_FSB_TO_B(mp, cmap.br_startoff + cmap.br_blockcount); + trace_xfs_iomap_found(ip, offset, length - offset, XFS_COW_FORK, &cmap); + if (imap.br_startblock != HOLESTARTBLOCK) { + seq = xfs_iomap_inode_sequence(ip, 0); + error = xfs_bmbt_to_iomap(ip, srcmap, &imap, flags, 0, seq); + if (error) + goto out_unlock; + } + seq = xfs_iomap_inode_sequence(ip, IOMAP_F_SHARED); + xfs_iunlock(ip, lockmode); + return xfs_bmbt_to_iomap(ip, iomap, &cmap, flags, IOMAP_F_SHARED, seq); + +out_unlock: + if (lockmode) + xfs_iunlock(ip, lockmode); + return error; +} + +static DEFINE_IOMAP_ITER_NEXT(xfs_direct_write_iomap_next, + xfs_direct_write_iomap_begin); + +const struct iomap_ops xfs_direct_write_iomap_ops = { + .iomap_next = xfs_direct_write_iomap_next, +}; + +#ifdef CONFIG_XFS_RT +/* + * This is really simple. The space has already been reserved before taking the + * IOLOCK, the actual block allocation is done just before submitting the bio + * and only recorded in the extent map on I/O completion. + */ +static int +xfs_zoned_direct_write_iomap_begin( + struct inode *inode, + loff_t offset, + loff_t length, + unsigned flags, + struct iomap *iomap, + struct iomap *srcmap) +{ + struct xfs_inode *ip = XFS_I(inode); + int error; + + ASSERT(!(flags & IOMAP_OVERWRITE_ONLY)); + + /* + * Needs to be pushed down into the allocator so that only writes into + * a single zone can be supported. + */ + if (flags & IOMAP_NOWAIT) + return -EAGAIN; + + /* + * Ensure the extent list is in memory in so that we don't have to do + * read it from the I/O completion handler. + */ + if (xfs_need_iread_extents(&ip->i_df)) { + xfs_ilock(ip, XFS_ILOCK_EXCL); + error = xfs_iread_extents(NULL, ip, XFS_DATA_FORK); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + if (error) + return error; + } + + iomap->type = IOMAP_MAPPED; + iomap->flags = IOMAP_F_DIRTY; + iomap->bdev = ip->i_mount->m_rtdev_targp->bt_bdev; + iomap->offset = offset; + iomap->length = length; + iomap->flags = IOMAP_F_ANON_WRITE; + return 0; +} + +static DEFINE_IOMAP_ITER_NEXT(xfs_zoned_direct_write_iomap_next, + xfs_zoned_direct_write_iomap_begin); + +const struct iomap_ops xfs_zoned_direct_write_iomap_ops = { + .iomap_next = xfs_zoned_direct_write_iomap_next, +}; +#endif /* CONFIG_XFS_RT */ + +#ifdef DEBUG +static void +xfs_check_atomic_cow_conversion( + struct xfs_inode *ip, + xfs_fileoff_t offset_fsb, + xfs_filblks_t count_fsb, + const struct xfs_bmbt_irec *cmap) +{ + struct xfs_iext_cursor icur; + struct xfs_bmbt_irec cmap2 = { }; + + if (xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &cmap2)) + xfs_trim_extent(&cmap2, offset_fsb, count_fsb); + + ASSERT(cmap2.br_startoff == cmap->br_startoff); + ASSERT(cmap2.br_blockcount == cmap->br_blockcount); + ASSERT(cmap2.br_startblock == cmap->br_startblock); + ASSERT(cmap2.br_state == cmap->br_state); +} +#else +# define xfs_check_atomic_cow_conversion(...) ((void)0) +#endif + +static int +xfs_atomic_write_cow_iomap_begin( + struct inode *inode, + loff_t offset, + loff_t length, + unsigned flags, + struct iomap *iomap, + struct iomap *srcmap) +{ + struct xfs_inode *ip = XFS_I(inode); + struct xfs_mount *mp = ip->i_mount; + const xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); + const xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + length); + const xfs_filblks_t count_fsb = end_fsb - offset_fsb; + xfs_filblks_t hole_count_fsb; + int nmaps = 1; + xfs_filblks_t resaligned; + struct xfs_bmbt_irec cmap; + struct xfs_iext_cursor icur; + struct xfs_trans *tp; + unsigned int dblocks = 0, rblocks = 0; + int error; + u64 seq; + + ASSERT(flags & IOMAP_WRITE); + ASSERT(flags & IOMAP_DIRECT); + + if (xfs_is_shutdown(mp)) + return -EIO; + + if (!xfs_can_sw_atomic_write(mp)) { + ASSERT(xfs_can_sw_atomic_write(mp)); + return -EINVAL; + } + + /* blocks are always allocated in this path */ + if (flags & IOMAP_NOWAIT) + return -EAGAIN; + + trace_xfs_iomap_atomic_write_cow(ip, offset, length); +retry: + xfs_ilock(ip, XFS_ILOCK_EXCL); + + if (!ip->i_cowfp) { + ASSERT(!xfs_is_reflink_inode(ip)); + xfs_ifork_init_cow(ip); + } + + if (!xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &cmap)) + cmap.br_startoff = end_fsb; + if (cmap.br_startoff <= offset_fsb) { + if (isnullstartblock(cmap.br_startblock)) + goto convert_delay; + + /* + * cmap could extend outside the write range due to previous + * speculative preallocations. We must trim cmap to the write + * range because the cow fork treats written mappings to mean + * "write in progress". + */ + xfs_trim_extent(&cmap, offset_fsb, count_fsb); + goto found; + } + + hole_count_fsb = cmap.br_startoff - offset_fsb; + + resaligned = xfs_aligned_fsb_count(offset_fsb, hole_count_fsb, + xfs_get_cowextsz_hint(ip)); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + + if (XFS_IS_REALTIME_INODE(ip)) { + dblocks = XFS_DIOSTRAT_SPACE_RES(mp, 0); + rblocks = resaligned; + } else { + dblocks = XFS_DIOSTRAT_SPACE_RES(mp, resaligned); + rblocks = 0; + } + + error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, dblocks, + rblocks, false, &tp); + if (error) + return error; + + /* extent layout could have changed since the unlock, so check again */ + if (!xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &cmap)) + cmap.br_startoff = end_fsb; + if (cmap.br_startoff <= offset_fsb) { + xfs_trans_cancel(tp); + if (isnullstartblock(cmap.br_startblock)) + goto convert_delay; + xfs_trim_extent(&cmap, offset_fsb, count_fsb); + goto found; + } + + /* + * Allocate the entire reservation as unwritten blocks. + * + * Use XFS_BMAPI_EXTSZALIGN to hint at aligning new extents according to + * extszhint, such that there will be a greater chance that future + * atomic writes to that same range will be aligned (and don't require + * this COW-based method). + */ + error = xfs_bmapi_write(tp, ip, offset_fsb, hole_count_fsb, + XFS_BMAPI_COWFORK | XFS_BMAPI_PREALLOC | + XFS_BMAPI_EXTSZALIGN, 0, &cmap, &nmaps); + if (error) { + xfs_trans_cancel(tp); + goto out_unlock; + } + + xfs_inode_set_cowblocks_tag(ip); + error = xfs_trans_commit(tp); + if (error) + goto out_unlock; + + /* + * cmap could map more blocks than the range we passed into bmapi_write + * because of EXTSZALIGN or adjacent pre-existing unwritten mappings + * that were merged. Trim cmap to the original write range so that we + * don't convert more than we were asked to do for this write. + */ + xfs_trim_extent(&cmap, offset_fsb, count_fsb); + +found: + if (cmap.br_state != XFS_EXT_NORM) { + error = xfs_reflink_convert_cow_locked(ip, cmap.br_startoff, + cmap.br_blockcount); + if (error) + goto out_unlock; + cmap.br_state = XFS_EXT_NORM; + xfs_check_atomic_cow_conversion(ip, offset_fsb, count_fsb, + &cmap); + } + + trace_xfs_iomap_found(ip, offset, length, XFS_COW_FORK, &cmap); + seq = xfs_iomap_inode_sequence(ip, IOMAP_F_SHARED); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return xfs_bmbt_to_iomap(ip, iomap, &cmap, flags, IOMAP_F_SHARED, seq); + +convert_delay: + xfs_iunlock(ip, XFS_ILOCK_EXCL); + error = xfs_bmapi_convert_delalloc(ip, XFS_COW_FORK, offset, iomap, + NULL); + if (error) + return error; + + /* + * Try the lookup again, because the delalloc conversion might have + * turned the COW mapping into unwritten, but we need it to be in + * written state. + */ + goto retry; +out_unlock: + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; +} + +static DEFINE_IOMAP_ITER_NEXT(xfs_atomic_write_cow_iomap_next, + xfs_atomic_write_cow_iomap_begin); + +const struct iomap_ops xfs_atomic_write_cow_iomap_ops = { + .iomap_next = xfs_atomic_write_cow_iomap_next, +}; + +static int +xfs_dax_write_iomap_end( + struct inode *inode, + loff_t pos, + loff_t length, + ssize_t written, + unsigned flags, + struct iomap *iomap) +{ + struct xfs_inode *ip = XFS_I(inode); + + if (!xfs_is_cow_inode(ip)) + return 0; + + if (!written) + return xfs_reflink_cancel_cow_range(ip, pos, length, true); + + return xfs_reflink_end_cow(ip, pos, written); +} + +static DEFINE_IOMAP_ITER_NEXT_END(xfs_dax_write_iomap_next, + xfs_direct_write_iomap_begin, xfs_dax_write_iomap_end); + +const struct iomap_ops xfs_dax_write_iomap_ops = { + .iomap_next = xfs_dax_write_iomap_next, +}; + +/* + * Convert a hole to a delayed allocation. + */ +static void +xfs_bmap_add_extent_hole_delay( + struct xfs_inode *ip, /* incore inode pointer */ + int whichfork, + struct xfs_iext_cursor *icur, + struct xfs_bmbt_irec *new) /* new data to add to file extents */ +{ + struct xfs_ifork *ifp; /* inode fork pointer */ + xfs_bmbt_irec_t left; /* left neighbor extent entry */ + xfs_filblks_t newlen=0; /* new indirect size */ + xfs_filblks_t oldlen=0; /* old indirect size */ + xfs_bmbt_irec_t right; /* right neighbor extent entry */ + uint32_t state = xfs_bmap_fork_to_state(whichfork); + xfs_filblks_t temp; /* temp for indirect calculations */ + + ifp = xfs_ifork_ptr(ip, whichfork); + ASSERT(isnullstartblock(new->br_startblock)); + + /* + * Check and set flags if this segment has a left neighbor + */ + if (xfs_iext_peek_prev_extent(ifp, icur, &left)) { + state |= BMAP_LEFT_VALID; + if (isnullstartblock(left.br_startblock)) + state |= BMAP_LEFT_DELAY; + } + + /* + * Check and set flags if the current (right) segment exists. + * If it doesn't exist, we're converting the hole at end-of-file. + */ + if (xfs_iext_get_extent(ifp, icur, &right)) { + state |= BMAP_RIGHT_VALID; + if (isnullstartblock(right.br_startblock)) + state |= BMAP_RIGHT_DELAY; + } + + /* + * Set contiguity flags on the left and right neighbors. + * Don't let extents get too large, even if the pieces are contiguous. + */ + if ((state & BMAP_LEFT_VALID) && (state & BMAP_LEFT_DELAY) && + left.br_startoff + left.br_blockcount == new->br_startoff && + left.br_blockcount + new->br_blockcount <= XFS_MAX_BMBT_EXTLEN) + state |= BMAP_LEFT_CONTIG; + + if ((state & BMAP_RIGHT_VALID) && (state & BMAP_RIGHT_DELAY) && + new->br_startoff + new->br_blockcount == right.br_startoff && + new->br_blockcount + right.br_blockcount <= XFS_MAX_BMBT_EXTLEN && + (!(state & BMAP_LEFT_CONTIG) || + (left.br_blockcount + new->br_blockcount + + right.br_blockcount <= XFS_MAX_BMBT_EXTLEN))) + state |= BMAP_RIGHT_CONTIG; + + /* + * Switch out based on the contiguity flags. + */ + switch (state & (BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG)) { + case BMAP_LEFT_CONTIG | BMAP_RIGHT_CONTIG: + /* + * New allocation is contiguous with delayed allocations + * on the left and on the right. + * Merge all three into a single extent record. + */ + temp = left.br_blockcount + new->br_blockcount + + right.br_blockcount; + + oldlen = startblockval(left.br_startblock) + + startblockval(new->br_startblock) + + startblockval(right.br_startblock); + newlen = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(ip, temp), + oldlen); + left.br_startblock = nullstartblock(newlen); + left.br_blockcount = temp; + + xfs_iext_remove(ip, icur, state); + xfs_iext_prev(ifp, icur); + xfs_iext_update_extent(ip, state, icur, &left); + break; + + case BMAP_LEFT_CONTIG: + /* + * New allocation is contiguous with a delayed allocation + * on the left. + * Merge the new allocation with the left neighbor. + */ + temp = left.br_blockcount + new->br_blockcount; + + oldlen = startblockval(left.br_startblock) + + startblockval(new->br_startblock); + newlen = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(ip, temp), + oldlen); + left.br_blockcount = temp; + left.br_startblock = nullstartblock(newlen); + + xfs_iext_prev(ifp, icur); + xfs_iext_update_extent(ip, state, icur, &left); + break; + + case BMAP_RIGHT_CONTIG: + /* + * New allocation is contiguous with a delayed allocation + * on the right. + * Merge the new allocation with the right neighbor. + */ + temp = new->br_blockcount + right.br_blockcount; + oldlen = startblockval(new->br_startblock) + + startblockval(right.br_startblock); + newlen = XFS_FILBLKS_MIN(xfs_bmap_worst_indlen(ip, temp), + oldlen); + right.br_startoff = new->br_startoff; + right.br_startblock = nullstartblock(newlen); + right.br_blockcount = temp; + xfs_iext_update_extent(ip, state, icur, &right); + break; + + case 0: + /* + * New allocation is not contiguous with another + * delayed allocation. + * Insert a new entry. + */ + oldlen = newlen = 0; + xfs_iext_insert(ip, icur, new, state); + break; + } + if (oldlen != newlen) { + ASSERT(oldlen > newlen); + xfs_add_fdblocks(ip->i_mount, oldlen - newlen); + + /* + * Nothing to do for disk quota accounting here. + */ + xfs_mod_delalloc(ip, 0, (int64_t)newlen - oldlen); + } +} + +/* + * Add a delayed allocation extent to an inode. Blocks are reserved from the + * global pool and the extent inserted into the inode in-core extent tree. + * + * On entry, got refers to the first extent beyond the offset of the extent to + * allocate or eof is specified if no such extent exists. On return, got refers + * to the extent record that was inserted to the inode fork. + * + * Note that the allocated extent may have been merged with contiguous extents + * during insertion into the inode fork. Thus, got does not reflect the current + * state of the inode fork on return. If necessary, the caller can use lastx to + * look up the updated record in the inode fork. + */ +static int +xfs_bmapi_reserve_delalloc( + struct xfs_inode *ip, + int whichfork, + xfs_fileoff_t off, + xfs_filblks_t len, + xfs_filblks_t prealloc, + struct xfs_bmbt_irec *got, + struct xfs_iext_cursor *icur, + int eof) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, whichfork); + xfs_extlen_t alen; + xfs_extlen_t indlen; + uint64_t fdblocks; + int error; + xfs_fileoff_t aoff; + bool use_cowextszhint = + whichfork == XFS_COW_FORK && !prealloc; + +retry: + /* + * Cap the alloc length. Keep track of prealloc so we know whether to + * tag the inode before we return. + */ + aoff = off; + alen = XFS_FILBLKS_MIN(len + prealloc, XFS_MAX_BMBT_EXTLEN); + if (!eof) + alen = XFS_FILBLKS_MIN(alen, got->br_startoff - aoff); + if (prealloc && alen >= len) + prealloc = alen - len; + + /* + * If we're targetting the COW fork but aren't creating a speculative + * posteof preallocation, try to expand the reservation to align with + * the COW extent size hint if there's sufficient free space. + * + * Unlike the data fork, the CoW cancellation functions will free all + * the reservations at inactivation, so we don't require that every + * delalloc reservation have a dirty pagecache. + */ + if (use_cowextszhint) { + struct xfs_bmbt_irec prev; + xfs_extlen_t extsz = xfs_get_cowextsz_hint(ip); + + if (!xfs_iext_peek_prev_extent(ifp, icur, &prev)) + prev.br_startoff = NULLFILEOFF; + + error = xfs_bmap_extsize_align(mp, got, &prev, extsz, 0, eof, + 1, 0, &aoff, &alen); + ASSERT(!error); + } + + /* + * Make a transaction-less quota reservation for delayed allocation + * blocks. This number gets adjusted later. We return if we haven't + * allocated blocks already inside this loop. + */ + error = xfs_quota_reserve_blkres(ip, alen); + if (error) + goto out; + + /* + * Split changing sb for alen and indlen since they could be coming + * from different places. + */ + indlen = (xfs_extlen_t)xfs_bmap_worst_indlen(ip, alen); + ASSERT(indlen > 0); + + fdblocks = indlen; + if (XFS_IS_REALTIME_INODE(ip)) { + ASSERT(!xfs_is_zoned_inode(ip)); + error = xfs_dec_frextents(mp, xfs_blen_to_rtbxlen(mp, alen)); + if (error) + goto out_unreserve_quota; + } else { + fdblocks += alen; + } + + error = xfs_dec_fdblocks(mp, fdblocks, false); + if (error) + goto out_unreserve_frextents; + + ip->i_delayed_blks += alen; + xfs_mod_delalloc(ip, alen, indlen); + + got->br_startoff = aoff; + got->br_startblock = nullstartblock(indlen); + got->br_blockcount = alen; + got->br_state = XFS_EXT_NORM; + + xfs_bmap_add_extent_hole_delay(ip, whichfork, icur, got); + + /* + * Tag the inode if blocks were preallocated. Note that COW fork + * preallocation can occur at the start or end of the extent, even when + * prealloc == 0, so we must also check the aligned offset and length. + */ + if (whichfork == XFS_DATA_FORK && prealloc) + xfs_inode_set_eofblocks_tag(ip); + if (whichfork == XFS_COW_FORK && (prealloc || aoff < off || alen > len)) + xfs_inode_set_cowblocks_tag(ip); + + return 0; + +out_unreserve_frextents: + if (XFS_IS_REALTIME_INODE(ip)) + xfs_add_frextents(mp, xfs_blen_to_rtbxlen(mp, alen)); +out_unreserve_quota: + if (XFS_IS_QUOTA_ON(mp)) + xfs_quota_unreserve_blkres(ip, alen); +out: + if (error == -ENOSPC || error == -EDQUOT) { + trace_xfs_delalloc_enospc(ip, off, len); + + if (prealloc || use_cowextszhint) { + /* retry without any preallocation */ + use_cowextszhint = false; + prealloc = 0; + goto retry; + } + } + return error; +} + +static int +xfs_zoned_buffered_write_iomap_begin( + struct inode *inode, + loff_t offset, + loff_t count, + unsigned flags, + struct iomap *iomap, + struct iomap *srcmap) +{ + struct iomap_iter *iter = + container_of(iomap, struct iomap_iter, iomap); + struct address_space *mapping = inode->i_mapping; + struct xfs_zone_alloc_ctx *ac = iter->private; + struct xfs_inode *ip = XFS_I(inode); + struct xfs_mount *mp = ip->i_mount; + xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); + xfs_fileoff_t end_fsb = xfs_iomap_end_fsb(mp, offset, count); + u16 iomap_flags = IOMAP_F_SHARED; + unsigned int lockmode = XFS_ILOCK_EXCL; + xfs_filblks_t count_fsb; + xfs_extlen_t indlen; + struct xfs_bmbt_irec got; + struct xfs_iext_cursor icur; + int error = 0; + + ASSERT(!xfs_get_extsz_hint(ip)); + ASSERT(!(flags & IOMAP_UNSHARE)); + ASSERT(ac); + + if (xfs_is_shutdown(mp)) + return -EIO; + + error = xfs_qm_dqattach(ip); + if (error) + return error; + +restart: + error = xfs_ilock_for_iomap(ip, flags, &lockmode); + if (error) + return error; + + if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(&ip->i_df)) || + XFS_TEST_ERROR(mp, XFS_ERRTAG_BMAPIFORMAT)) { + xfs_bmap_mark_sick(ip, XFS_DATA_FORK); + error = -EFSCORRUPTED; + goto out_unlock; + } + + XFS_STATS_INC(mp, xs_blk_mapw); + + error = xfs_iread_extents(NULL, ip, XFS_DATA_FORK); + if (error) + goto out_unlock; + + /* + * For zeroing operations check if there is any data to zero first. + * + * For regular writes we always need to allocate new blocks, but need to + * provide the source mapping when the range is unaligned to support + * read-modify-write of the whole block in the page cache. + * + * In either case we need to limit the reported range to the boundaries + * of the source map in the data fork. + */ + if (!IS_ALIGNED(offset, mp->m_sb.sb_blocksize) || + !IS_ALIGNED(offset + count, mp->m_sb.sb_blocksize) || + (flags & IOMAP_ZERO)) { + struct xfs_bmbt_irec smap; + struct xfs_iext_cursor scur; + + if (!xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &scur, + &smap)) + smap.br_startoff = end_fsb; /* fake hole until EOF */ + if (smap.br_startoff > offset_fsb) { + end_fsb = min(end_fsb, smap.br_startoff); + } else { + end_fsb = min(end_fsb, + smap.br_startoff + smap.br_blockcount); + xfs_trim_extent(&smap, offset_fsb, + end_fsb - offset_fsb); + error = xfs_bmbt_to_iomap(ip, srcmap, &smap, flags, 0, + xfs_iomap_inode_sequence(ip, 0)); + if (error) + goto out_unlock; + } + } + + if (!ip->i_cowfp) + xfs_ifork_init_cow(ip); + + if (!xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &got)) + got.br_startoff = end_fsb; + if (got.br_startoff <= offset_fsb) { + trace_xfs_reflink_cow_found(ip, &got); + goto done; + } + + /* + * Cap the maximum length to keep the chunks of work done here somewhat + * symmetric with the work writeback does. + */ + end_fsb = min(end_fsb, got.br_startoff); + count_fsb = min3(end_fsb - offset_fsb, XFS_MAX_BMBT_EXTLEN, + XFS_B_TO_FSB(mp, 1024 * PAGE_SIZE)); + + /* + * When zeroing, don't allocate blocks for holes as they are already + * zeroes, but we need to ensure that no extents exist in both the data + * and COW fork to ensure this really is a hole. + * + * A window exists where we might observe a hole in both forks with + * valid data in cache. Writeback removes the COW fork blocks on + * submission but doesn't remap into the data fork until completion. If + * the data fork was previously a hole, we'll fail to zero. Until we + * find a way to avoid this transient state, check for dirty pagecache + * and flush to wait on blocks to land in the data fork. + */ + if ((flags & IOMAP_ZERO) && srcmap->type == IOMAP_HOLE) { + if (filemap_range_needs_writeback(mapping, offset, + offset + count - 1)) { + xfs_iunlock(ip, lockmode); + error = filemap_write_and_wait_range(mapping, offset, + offset + count - 1); + if (error) + return error; + goto restart; + } + + xfs_hole_to_iomap(ip, iomap, offset_fsb, end_fsb); + goto out_unlock; + } + + /* + * The block reservation is supposed to cover all blocks that the + * operation could possible write, but there is a nasty corner case + * where blocks could be stolen from underneath us: + * + * 1) while this thread iterates over a larger buffered write, + * 2) another thread is causing a write fault that calls into + * ->page_mkwrite in range this thread writes to, using up the + * delalloc reservation created by a previous call to this function. + * 3) another thread does direct I/O on the range that the write fault + * happened on, which causes writeback of the dirty data. + * 4) this then set the stale flag, which cuts the current iomap + * iteration short, causing the new call to ->iomap_begin that gets + * us here again, but now without a sufficient reservation. + * + * This is a very unusual I/O pattern, and nothing but generic/095 is + * known to hit it. There's not really much we can do here, so turn this + * into a short write. + */ + if (count_fsb > ac->reserved_blocks) { + xfs_warn_ratelimited(mp, +"Short write on ino 0x%llx comm %.20s due to three-way race with write fault and direct I/O", + I_INO(ip), current->comm); + count_fsb = ac->reserved_blocks; + if (!count_fsb) { + error = -EIO; + goto out_unlock; + } + } + + error = xfs_quota_reserve_blkres(ip, count_fsb); + if (error) + goto out_unlock; + + indlen = xfs_bmap_worst_indlen(ip, count_fsb); + error = xfs_dec_fdblocks(mp, indlen, false); + if (error) + goto out_unlock; + ip->i_delayed_blks += count_fsb; + xfs_mod_delalloc(ip, count_fsb, indlen); + + got.br_startoff = offset_fsb; + got.br_startblock = nullstartblock(indlen); + got.br_blockcount = count_fsb; + got.br_state = XFS_EXT_NORM; + xfs_bmap_add_extent_hole_delay(ip, XFS_COW_FORK, &icur, &got); + ac->reserved_blocks -= count_fsb; + iomap_flags |= IOMAP_F_NEW; + + trace_xfs_iomap_alloc(ip, offset, XFS_FSB_TO_B(mp, count_fsb), + XFS_COW_FORK, &got); +done: + error = xfs_bmbt_to_iomap(ip, iomap, &got, flags, iomap_flags, + xfs_iomap_inode_sequence(ip, IOMAP_F_SHARED)); +out_unlock: + xfs_iunlock(ip, lockmode); + return error; +} + +static int +xfs_buffered_write_iomap_begin( + struct inode *inode, + loff_t offset, + loff_t count, + unsigned flags, + struct iomap *iomap, + struct iomap *srcmap) +{ + struct iomap_iter *iter = container_of(iomap, struct iomap_iter, + iomap); + struct xfs_inode *ip = XFS_I(inode); + struct xfs_mount *mp = ip->i_mount; + xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); + xfs_fileoff_t end_fsb = xfs_iomap_end_fsb(mp, offset, count); + xfs_fileoff_t cow_fsb = NULLFILEOFF; + xfs_fileoff_t eof_fsb = XFS_B_TO_FSB(mp, XFS_ISIZE(ip)); + struct xfs_bmbt_irec imap, cmap; + struct xfs_iext_cursor icur, ccur; + xfs_fsblock_t prealloc_blocks = 0; + bool eof = false, cow_eof = false, shared = false; + int allocfork = XFS_DATA_FORK; + int error = 0; + unsigned int lockmode = XFS_ILOCK_EXCL; + unsigned int iomap_flags = 0; + u64 seq; + + if (xfs_is_shutdown(mp)) + return -EIO; + + if (xfs_is_zoned_inode(ip)) + return xfs_zoned_buffered_write_iomap_begin(inode, offset, + count, flags, iomap, srcmap); + + /* we can't use delayed allocations when using extent size hints */ + if (xfs_get_extsz_hint(ip)) + return xfs_direct_write_iomap_begin(inode, offset, count, + flags, iomap, srcmap); + + error = xfs_qm_dqattach(ip); + if (error) + return error; + + error = xfs_ilock_for_iomap(ip, flags, &lockmode); + if (error) + return error; + + if (XFS_IS_CORRUPT(mp, !xfs_ifork_has_extents(&ip->i_df)) || + XFS_TEST_ERROR(mp, XFS_ERRTAG_BMAPIFORMAT)) { + xfs_bmap_mark_sick(ip, XFS_DATA_FORK); + error = -EFSCORRUPTED; + goto out_unlock; + } + + XFS_STATS_INC(mp, xs_blk_mapw); + + error = xfs_iread_extents(NULL, ip, XFS_DATA_FORK); + if (error) + goto out_unlock; + + /* + * Search the data fork first to look up our source mapping. We always + * need the data fork map, as we have to return it to the iomap code so + * that the higher level write code can read data in to perform + * read-modify-write cycles for unaligned writes. + * + * Then search the COW fork extent list even if we did not find a data + * fork extent. This serves two purposes: first this implements the + * speculative preallocation using cowextsize, so that we also unshare + * block adjacent to shared blocks instead of just the shared blocks + * themselves. Second the lookup in the extent list is generally faster + * than going out to the shared extent tree. + */ + eof = !xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &icur, &imap); + if (eof) + imap.br_startoff = end_fsb; /* fake hole until the end */ + if (xfs_is_cow_inode(ip)) { + if (!ip->i_cowfp) { + ASSERT(!xfs_is_reflink_inode(ip)); + xfs_ifork_init_cow(ip); + } + cow_eof = !xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, + &ccur, &cmap); + if (!cow_eof) + cow_fsb = cmap.br_startoff; + } + + /* We never need to allocate blocks for unsharing a hole. */ + if ((flags & IOMAP_UNSHARE) && imap.br_startoff > offset_fsb) { + xfs_hole_to_iomap(ip, iomap, offset_fsb, imap.br_startoff); + goto out_unlock; + } + + /* + * We may need to zero over a hole in the data fork if it's fronted by + * COW blocks and dirty pagecache. Scan such file ranges for dirty + * cache and fill the iomap batch with folios that need zeroing. + */ + if ((flags & IOMAP_ZERO) && imap.br_startoff > offset_fsb) { + loff_t start, end; + unsigned int fbatch_count; + + imap.br_blockcount = imap.br_startoff - offset_fsb; + imap.br_startoff = offset_fsb; + imap.br_startblock = HOLESTARTBLOCK; + imap.br_state = XFS_EXT_NORM; + + if (cow_fsb == NULLFILEOFF) + goto found_imap; + if (cow_fsb > offset_fsb) { + xfs_trim_extent(&imap, offset_fsb, + cow_fsb - offset_fsb); + goto found_imap; + } + + /* no zeroing beyond eof, so split at the boundary */ + if (offset_fsb >= eof_fsb) + goto found_imap; + if (offset_fsb < eof_fsb && end_fsb > eof_fsb) + xfs_trim_extent(&imap, offset_fsb, + eof_fsb - offset_fsb); + + /* COW fork blocks overlap the hole */ + xfs_trim_extent(&imap, offset_fsb, + cmap.br_startoff + cmap.br_blockcount - offset_fsb); + start = XFS_FSB_TO_B(mp, imap.br_startoff); + end = XFS_FSB_TO_B(mp, imap.br_startoff + imap.br_blockcount); + fbatch_count = iomap_fill_dirty_folios(iter, &start, end, + &iomap_flags); + xfs_trim_extent(&imap, offset_fsb, + XFS_B_TO_FSB(mp, start) - offset_fsb); + + /* + * Report the COW mapping if we have folios to zero. Otherwise + * ignore the COW blocks as preallocation and report a hole. + */ + if (fbatch_count) { + xfs_trim_extent(&cmap, imap.br_startoff, + imap.br_blockcount); + imap.br_startoff = end_fsb; /* fake hole */ + goto found_cow; + } + goto found_imap; + } + + /* + * For zeroing, trim extents that extend beyond the EOF block. If a + * delalloc extent starts beyond the EOF block, convert it to an + * unwritten extent. + */ + if (flags & IOMAP_ZERO) { + if (isnullstartblock(imap.br_startblock) && + offset_fsb >= eof_fsb) + goto convert_delay; + if (offset_fsb < eof_fsb && end_fsb > eof_fsb) + end_fsb = eof_fsb; + + /* + * Look up dirty folios for unwritten mappings within EOF. + * Providing this bypasses the flush iomap uses to trigger + * extent conversion when unwritten mappings have dirty + * pagecache in need of zeroing. + * + * Trim the mapping to the end pos of the lookup, which in turn + * was trimmed to the end of the batch if it became full before + * the end of the mapping. + */ + if (imap.br_state == XFS_EXT_UNWRITTEN && + offset_fsb < eof_fsb) { + loff_t foffset = offset, fend; + + fend = offset + + min(count, XFS_FSB_TO_B(mp, imap.br_blockcount)); + iomap_fill_dirty_folios(iter, &foffset, fend, + &iomap_flags); + end_fsb = min_t(xfs_fileoff_t, end_fsb, + XFS_B_TO_FSB(mp, foffset)); + } + + xfs_trim_extent(&imap, offset_fsb, end_fsb - offset_fsb); + } + + /* + * Now that we've handled any operation specific special cases, at this + * point we can report a COW mapping if found. + */ + if (xfs_is_cow_inode(ip) && + !cow_eof && cmap.br_startoff <= offset_fsb) { + trace_xfs_reflink_cow_found(ip, &cmap); + goto found_cow; + } + + if (imap.br_startoff <= offset_fsb) { + /* + * For reflink files we may need a delalloc reservation when + * overwriting shared extents. This includes zeroing of + * existing extents that contain data. + */ + if (!xfs_is_cow_inode(ip) || + ((flags & IOMAP_ZERO) && imap.br_state != XFS_EXT_NORM)) { + trace_xfs_iomap_found(ip, offset, count, XFS_DATA_FORK, + &imap); + goto found_imap; + } + + xfs_trim_extent(&imap, offset_fsb, end_fsb - offset_fsb); + + /* Trim the mapping to the nearest shared extent boundary. */ + error = xfs_bmap_trim_cow(ip, &imap, &shared); + if (error) + goto out_unlock; + + /* Not shared? Just report the (potentially capped) extent. */ + if (!shared) { + trace_xfs_iomap_found(ip, offset, count, XFS_DATA_FORK, + &imap); + goto found_imap; + } + + /* + * Fork all the shared blocks from our write offset until the + * end of the extent. + */ + allocfork = XFS_COW_FORK; + end_fsb = imap.br_startoff + imap.br_blockcount; + } else { + /* + * We cap the maximum length we map here to MAX_WRITEBACK_PAGES + * pages to keep the chunks of work done where somewhat + * symmetric with the work writeback does. This is a completely + * arbitrary number pulled out of thin air. + * + * Note that the values needs to be less than 32-bits wide until + * the lower level functions are updated. + */ + count = min_t(loff_t, count, 1024 * PAGE_SIZE); + end_fsb = xfs_iomap_end_fsb(mp, offset, count); + + if (xfs_is_always_cow_inode(ip)) + allocfork = XFS_COW_FORK; + } + + if (eof && offset + count > XFS_ISIZE(ip)) { + /* + * Determine the initial size of the preallocation. + * We clean up any extra preallocation when the file is closed. + */ + if (xfs_has_allocsize(mp)) + prealloc_blocks = mp->m_allocsize_blocks; + else if (allocfork == XFS_DATA_FORK) + prealloc_blocks = xfs_iomap_prealloc_size(ip, allocfork, + offset, count, &icur); + else + prealloc_blocks = xfs_iomap_prealloc_size(ip, allocfork, + offset, count, &ccur); + if (prealloc_blocks) { + xfs_extlen_t align; + xfs_off_t end_offset; + xfs_fileoff_t p_end_fsb; + + end_offset = XFS_ALLOC_ALIGN(mp, offset + count - 1); + p_end_fsb = XFS_B_TO_FSBT(mp, end_offset) + + prealloc_blocks; + + align = xfs_eof_alignment(ip); + if (align) + p_end_fsb = roundup_64(p_end_fsb, align); + + p_end_fsb = min(p_end_fsb, + XFS_B_TO_FSB(mp, mp->m_super->s_maxbytes)); + ASSERT(p_end_fsb > offset_fsb); + prealloc_blocks = p_end_fsb - end_fsb; + } + } + + /* + * Flag newly allocated delalloc blocks with IOMAP_F_NEW so we punch + * them out if the write happens to fail. + */ + iomap_flags |= IOMAP_F_NEW; + if (allocfork == XFS_COW_FORK) { + error = xfs_bmapi_reserve_delalloc(ip, allocfork, offset_fsb, + end_fsb - offset_fsb, prealloc_blocks, &cmap, + &ccur, cow_eof); + if (error) + goto out_unlock; + + trace_xfs_iomap_alloc(ip, offset, count, allocfork, &cmap); + goto found_cow; + } + + error = xfs_bmapi_reserve_delalloc(ip, allocfork, offset_fsb, + end_fsb - offset_fsb, prealloc_blocks, &imap, &icur, + eof); + if (error) + goto out_unlock; + + trace_xfs_iomap_alloc(ip, offset, count, allocfork, &imap); +found_imap: + seq = xfs_iomap_inode_sequence(ip, iomap_flags); + xfs_iunlock(ip, lockmode); + return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, iomap_flags, seq); + +convert_delay: + xfs_iunlock(ip, lockmode); + truncate_pagecache(inode, offset); + error = xfs_bmapi_convert_delalloc(ip, XFS_DATA_FORK, offset, + iomap, NULL); + if (error) + return error; + + trace_xfs_iomap_alloc(ip, offset, count, XFS_DATA_FORK, &imap); + return 0; + +found_cow: + if (imap.br_startoff <= offset_fsb) { + error = xfs_bmbt_to_iomap(ip, srcmap, &imap, flags, 0, + xfs_iomap_inode_sequence(ip, 0)); + if (error) + goto out_unlock; + } else { + xfs_trim_extent(&cmap, offset_fsb, + imap.br_startoff - offset_fsb); + } + + iomap_flags |= IOMAP_F_SHARED; + seq = xfs_iomap_inode_sequence(ip, iomap_flags); + xfs_iunlock(ip, lockmode); + return xfs_bmbt_to_iomap(ip, iomap, &cmap, flags, iomap_flags, seq); + +out_unlock: + xfs_iunlock(ip, lockmode); + return error; +} + +static void +xfs_buffered_write_delalloc_punch( + struct inode *inode, + loff_t offset, + loff_t length, + struct iomap *iomap) +{ + struct iomap_iter *iter = + container_of(iomap, struct iomap_iter, iomap); + + xfs_bmap_punch_delalloc_range(XFS_I(inode), + (iomap->flags & IOMAP_F_SHARED) ? + XFS_COW_FORK : XFS_DATA_FORK, + offset, offset + length, iter->private); +} + +static int +xfs_buffered_write_iomap_end( + struct inode *inode, + loff_t offset, + loff_t length, + ssize_t written, + unsigned flags, + struct iomap *iomap) +{ + loff_t start_byte, end_byte; + + /* If we didn't reserve the blocks, we're not allowed to punch them. */ + if (iomap->type != IOMAP_DELALLOC || !(iomap->flags & IOMAP_F_NEW)) + return 0; + + /* + * iomap_page_mkwrite() will never fail in a way that requires delalloc + * extents that it allocated to be revoked. Hence never try to release + * them here. + */ + if (flags & IOMAP_FAULT) + return 0; + + /* Nothing to do if we've written the entire delalloc extent */ + start_byte = iomap_last_written_block(inode, offset, written); + end_byte = round_up(offset + length, i_blocksize(inode)); + if (start_byte >= end_byte) + return 0; + + /* For zeroing operations the callers already hold invalidate_lock. */ + if (flags & (IOMAP_UNSHARE | IOMAP_ZERO)) { + rwsem_assert_held_write(&inode->i_mapping->invalidate_lock); + iomap_write_delalloc_release(inode, start_byte, end_byte, flags, + iomap, xfs_buffered_write_delalloc_punch); + } else { + filemap_invalidate_lock(inode->i_mapping); + iomap_write_delalloc_release(inode, start_byte, end_byte, flags, + iomap, xfs_buffered_write_delalloc_punch); + filemap_invalidate_unlock(inode->i_mapping); + } + + return 0; +} + +static DEFINE_IOMAP_ITER_NEXT_END(xfs_buffered_write_iomap_next, + xfs_buffered_write_iomap_begin, xfs_buffered_write_iomap_end); + +const struct iomap_ops xfs_buffered_write_iomap_ops = { + .iomap_next = xfs_buffered_write_iomap_next, +}; + +int +xfs_read_iomap_begin( + struct inode *inode, + loff_t offset, + loff_t length, + unsigned flags, + struct iomap *iomap, + struct iomap *srcmap) +{ + struct xfs_inode *ip = XFS_I(inode); + struct xfs_mount *mp = ip->i_mount; + struct xfs_bmbt_irec imap; + xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); + xfs_fileoff_t end_fsb = xfs_iomap_end_fsb(mp, offset, length); + int nimaps = 1, error = 0; + bool shared = false; + unsigned int lockmode = XFS_ILOCK_SHARED; + u64 seq; + + ASSERT(!(flags & (IOMAP_WRITE | IOMAP_ZERO))); + + if (xfs_is_shutdown(mp)) + return -EIO; + + error = xfs_ilock_for_iomap(ip, flags, &lockmode); + if (error) + return error; + error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, &imap, + &nimaps, 0); + if (!error && ((flags & IOMAP_REPORT) || IS_DAX(inode))) + error = xfs_reflink_trim_around_shared(ip, &imap, &shared); + seq = xfs_iomap_inode_sequence(ip, shared ? IOMAP_F_SHARED : 0); + xfs_iunlock(ip, lockmode); + + if (error) + return error; + trace_xfs_iomap_found(ip, offset, length, XFS_DATA_FORK, &imap); + return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, + shared ? IOMAP_F_SHARED : 0, seq); +} + +static DEFINE_IOMAP_ITER_NEXT(xfs_read_iomap_next, xfs_read_iomap_begin); + +const struct iomap_ops xfs_read_iomap_ops = { + .iomap_next = xfs_read_iomap_next, +}; + +static int +xfs_seek_iomap_begin( + struct inode *inode, + loff_t offset, + loff_t length, + unsigned flags, + struct iomap *iomap, + struct iomap *srcmap) +{ + struct xfs_inode *ip = XFS_I(inode); + struct xfs_mount *mp = ip->i_mount; + xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); + xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + length); + xfs_fileoff_t cow_fsb = NULLFILEOFF, data_fsb = NULLFILEOFF; + struct xfs_iext_cursor icur; + struct xfs_bmbt_irec imap, cmap; + int error = 0; + unsigned lockmode; + u64 seq; + + if (xfs_is_shutdown(mp)) + return -EIO; + + lockmode = xfs_ilock_data_map_shared(ip); + error = xfs_iread_extents(NULL, ip, XFS_DATA_FORK); + if (error) + goto out_unlock; + + if (xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &icur, &imap)) { + /* + * If we found a data extent we are done. + */ + if (imap.br_startoff <= offset_fsb) + goto done; + data_fsb = imap.br_startoff; + } else { + /* + * Fake a hole until the end of the file. + */ + data_fsb = xfs_iomap_end_fsb(mp, offset, length); + } + + /* + * If a COW fork extent covers the hole, report it - capped to the next + * data fork extent: + */ + if (xfs_inode_has_cow_data(ip) && + xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &cmap)) + cow_fsb = cmap.br_startoff; + if (cow_fsb != NULLFILEOFF && cow_fsb <= offset_fsb) { + if (data_fsb < cow_fsb + cmap.br_blockcount) + end_fsb = min(end_fsb, data_fsb); + xfs_trim_extent(&cmap, offset_fsb, end_fsb - offset_fsb); + seq = xfs_iomap_inode_sequence(ip, IOMAP_F_SHARED); + error = xfs_bmbt_to_iomap(ip, iomap, &cmap, flags, + IOMAP_F_SHARED, seq); + /* + * This is a COW extent, so we must probe the page cache + * because there could be dirty page cache being backed + * by this extent. + */ + iomap->type = IOMAP_UNWRITTEN; + goto out_unlock; + } + + /* + * Else report a hole, capped to the next found data or COW extent. + */ + if (cow_fsb != NULLFILEOFF && cow_fsb < data_fsb) + imap.br_blockcount = cow_fsb - offset_fsb; + else + imap.br_blockcount = data_fsb - offset_fsb; + imap.br_startoff = offset_fsb; + imap.br_startblock = HOLESTARTBLOCK; + imap.br_state = XFS_EXT_NORM; +done: + seq = xfs_iomap_inode_sequence(ip, 0); + xfs_trim_extent(&imap, offset_fsb, end_fsb - offset_fsb); + error = xfs_bmbt_to_iomap(ip, iomap, &imap, flags, 0, seq); +out_unlock: + xfs_iunlock(ip, lockmode); + return error; +} + +static DEFINE_IOMAP_ITER_NEXT(xfs_seek_iomap_next, xfs_seek_iomap_begin); + +const struct iomap_ops xfs_seek_iomap_ops = { + .iomap_next = xfs_seek_iomap_next, +}; + +static int +xfs_xattr_iomap_begin( + struct inode *inode, + loff_t offset, + loff_t length, + unsigned flags, + struct iomap *iomap, + struct iomap *srcmap) +{ + struct xfs_inode *ip = XFS_I(inode); + struct xfs_mount *mp = ip->i_mount; + xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); + xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + length); + struct xfs_bmbt_irec imap; + int nimaps = 1, error = 0; + unsigned lockmode; + int seq; + + if (xfs_is_shutdown(mp)) + return -EIO; + + lockmode = xfs_ilock_attr_map_shared(ip); + + /* if there are no attribute fork or extents, return ENOENT */ + if (!xfs_inode_has_attr_fork(ip) || !ip->i_af.if_nextents) { + error = -ENOENT; + goto out_unlock; + } + + ASSERT(ip->i_af.if_format != XFS_DINODE_FMT_LOCAL); + error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, &imap, + &nimaps, XFS_BMAPI_ATTRFORK); +out_unlock: + + seq = xfs_iomap_inode_sequence(ip, IOMAP_F_XATTR); + xfs_iunlock(ip, lockmode); + + if (error) + return error; + ASSERT(nimaps); + return xfs_bmbt_to_iomap(ip, iomap, &imap, flags, IOMAP_F_XATTR, seq); +} + +static DEFINE_IOMAP_ITER_NEXT(xfs_xattr_iomap_next, xfs_xattr_iomap_begin); + +const struct iomap_ops xfs_xattr_iomap_ops = { + .iomap_next = xfs_xattr_iomap_next, +}; + +int +xfs_zero_range( + struct xfs_inode *ip, + loff_t pos, + loff_t len, + struct xfs_zone_alloc_ctx *ac, + bool *did_zero) +{ + struct inode *inode = VFS_I(ip); + + xfs_assert_ilocked(ip, XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL); + + if (IS_DAX(inode)) + return dax_zero_range(inode, pos, len, did_zero, + &xfs_dax_write_iomap_ops); + return iomap_zero_range(inode, pos, len, did_zero, + &xfs_buffered_write_iomap_ops, &xfs_iomap_write_ops, + ac); +} + +int +xfs_truncate_page( + struct xfs_inode *ip, + loff_t pos, + struct xfs_zone_alloc_ctx *ac, + bool *did_zero) +{ + struct inode *inode = VFS_I(ip); + + if (IS_DAX(inode)) + return dax_truncate_page(inode, pos, did_zero, + &xfs_dax_write_iomap_ops); + return iomap_truncate_page(inode, pos, did_zero, + &xfs_buffered_write_iomap_ops, &xfs_iomap_write_ops, + ac); +} diff --git a/fs/xfs/xfs_iomap.h b/fs/xfs/xfs_iomap.h new file mode 100644 index 000000000..cffcec532 --- /dev/null +++ b/fs/xfs/xfs_iomap.h @@ -0,0 +1,66 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2003-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_IOMAP_H__ +#define __XFS_IOMAP_H__ + +#include + +struct xfs_inode; +struct xfs_bmbt_irec; +struct xfs_zone_alloc_ctx; + +int xfs_iomap_write_direct(struct xfs_inode *ip, xfs_fileoff_t offset_fsb, + xfs_fileoff_t count_fsb, unsigned int flags, + struct xfs_bmbt_irec *imap, u64 *sequence); +int xfs_iomap_write_unwritten(struct xfs_inode *, xfs_off_t, xfs_off_t, bool); +xfs_fileoff_t xfs_iomap_eof_align_last_fsb(struct xfs_inode *ip, + xfs_fileoff_t end_fsb); + +u64 xfs_iomap_inode_sequence(struct xfs_inode *ip, u16 iomap_flags); +int xfs_bmbt_to_iomap(struct xfs_inode *ip, struct iomap *iomap, + struct xfs_bmbt_irec *imap, unsigned int mapping_flags, + u16 iomap_flags, u64 sequence_cookie); + +int xfs_zero_range(struct xfs_inode *ip, loff_t pos, loff_t len, + struct xfs_zone_alloc_ctx *ac, bool *did_zero); +int xfs_truncate_page(struct xfs_inode *ip, loff_t pos, + struct xfs_zone_alloc_ctx *ac, bool *did_zero); + +static inline xfs_filblks_t +xfs_aligned_fsb_count( + xfs_fileoff_t offset_fsb, + xfs_filblks_t count_fsb, + xfs_extlen_t extsz) +{ + if (extsz) { + xfs_extlen_t align; + + div_u64_rem(offset_fsb, extsz, &align); + if (align) + count_fsb += align; + div_u64_rem(count_fsb, extsz, &align); + if (align) + count_fsb += extsz - align; + } + + return count_fsb; +} + +int xfs_read_iomap_begin(struct inode *inode, loff_t offset, + loff_t length, unsigned flags, struct iomap *iomap, + struct iomap *srcmap); + +extern const struct iomap_ops xfs_buffered_write_iomap_ops; +extern const struct iomap_ops xfs_direct_write_iomap_ops; +extern const struct iomap_ops xfs_zoned_direct_write_iomap_ops; +extern const struct iomap_ops xfs_read_iomap_ops; +extern const struct iomap_ops xfs_seek_iomap_ops; +extern const struct iomap_ops xfs_xattr_iomap_ops; +extern const struct iomap_ops xfs_dax_write_iomap_ops; +extern const struct iomap_ops xfs_atomic_write_cow_iomap_ops; +extern const struct iomap_write_ops xfs_iomap_write_ops; + +#endif /* __XFS_IOMAP_H__*/ diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c new file mode 100644 index 000000000..4a3299abf --- /dev/null +++ b/fs/xfs/xfs_iops.c @@ -0,0 +1,1513 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_acl.h" +#include "xfs_quota.h" +#include "xfs_da_format.h" +#include "xfs_da_btree.h" +#include "xfs_attr.h" +#include "xfs_trans.h" +#include "xfs_trans_space.h" +#include "xfs_bmap_btree.h" +#include "xfs_trace.h" +#include "xfs_icache.h" +#include "xfs_symlink.h" +#include "xfs_dir2.h" +#include "xfs_iomap.h" +#include "xfs_error.h" +#include "xfs_ioctl.h" +#include "xfs_xattr.h" +#include "xfs_file.h" +#include "xfs_bmap.h" +#include "xfs_zone_alloc.h" + +#include +#include +#include +#include + +/* + * Directories have different lock order w.r.t. mmap_lock compared to regular + * files. This is due to readdir potentially triggering page faults on a user + * buffer inside filldir(), and this happens with the ilock on the directory + * held. For regular files, the lock order is the other way around - the + * mmap_lock is taken during the page fault, and then we lock the ilock to do + * block mapping. Hence we need a different class for the directory ilock so + * that lockdep can tell them apart. Directories in the metadata directory + * tree get a separate class so that lockdep reports will warn us if someone + * ever tries to lock regular directories after locking metadata directories. + */ +static struct lock_class_key xfs_nondir_ilock_class; +static struct lock_class_key xfs_dir_ilock_class; + +static int +xfs_initxattrs( + struct inode *inode, + const struct xattr *xattr_array, + void *fs_info) +{ + const struct xattr *xattr; + struct xfs_inode *ip = XFS_I(inode); + int error = 0; + + for (xattr = xattr_array; xattr->name != NULL; xattr++) { + struct xfs_da_args args = { + .dp = ip, + .attr_filter = XFS_ATTR_SECURE, + .name = xattr->name, + .namelen = strlen(xattr->name), + .value = xattr->value, + .valuelen = xattr->value_len, + }; + error = xfs_attr_change(&args, XFS_ATTRUPDATE_UPSERT); + if (error < 0) + break; + } + return error; +} + +/* + * Hook in SELinux. This is not quite correct yet, what we really need + * here (as we do for default ACLs) is a mechanism by which creation of + * these attrs can be journalled at inode creation time (along with the + * inode, of course, such that log replay can't cause these to be lost). + */ +int +xfs_inode_init_security( + struct inode *inode, + struct inode *dir, + const struct qstr *qstr) +{ + return security_inode_init_security(inode, dir, qstr, + &xfs_initxattrs, NULL); +} + +static void +xfs_dentry_to_name( + struct xfs_name *namep, + struct dentry *dentry) +{ + namep->name = dentry->d_name.name; + namep->len = dentry->d_name.len; + namep->type = XFS_DIR3_FT_UNKNOWN; +} + +static int +xfs_dentry_mode_to_name( + struct xfs_name *namep, + struct dentry *dentry, + int mode) +{ + namep->name = dentry->d_name.name; + namep->len = dentry->d_name.len; + namep->type = xfs_mode_to_ftype(mode); + + if (unlikely(namep->type == XFS_DIR3_FT_UNKNOWN)) + return -EFSCORRUPTED; + + return 0; +} + +STATIC void +xfs_cleanup_inode( + struct inode *dir, + struct inode *inode, + struct dentry *dentry) +{ + struct xfs_name teardown; + + /* Oh, the horror. + * If we can't add the ACL or we fail in + * xfs_inode_init_security we must back out. + * ENOSPC can hit here, among other things. + */ + xfs_dentry_to_name(&teardown, dentry); + + xfs_remove(XFS_I(dir), &teardown, XFS_I(inode)); +} + +/* + * Check to see if we are likely to need an extended attribute to be added to + * the inode we are about to allocate. This allows the attribute fork to be + * created during the inode allocation, reducing the number of transactions we + * need to do in this fast path. + * + * The security checks are optimistic, but not guaranteed. The two LSMs that + * require xattrs to be added here (selinux and smack) are also the only two + * LSMs that add a sb->s_security structure to the superblock. Hence if security + * is enabled and sb->s_security is set, we have a pretty good idea that we are + * going to be asked to add a security xattr immediately after allocating the + * xfs inode and instantiating the VFS inode. + */ +static inline bool +xfs_create_need_xattr( + struct inode *dir, + struct posix_acl *default_acl, + struct posix_acl *acl) +{ + if (acl) + return true; + if (default_acl) + return true; +#if IS_ENABLED(CONFIG_SECURITY) + if (dir->i_sb->s_security) + return true; +#endif + return false; +} + + +STATIC int +xfs_generic_create( + struct mnt_idmap *idmap, + struct inode *dir, + struct dentry *dentry, + umode_t mode, + dev_t rdev, + struct file *tmpfile) /* unnamed file */ +{ + struct xfs_icreate_args args = { + .idmap = idmap, + .pip = XFS_I(dir), + .rdev = rdev, + .mode = mode, + }; + struct inode *inode; + struct xfs_inode *ip = NULL; + struct posix_acl *default_acl, *acl; + struct xfs_name name; + int error; + + /* + * Irix uses Missed'em'V split, but doesn't want to see + * the upper 5 bits of (14bit) major. + */ + if (S_ISCHR(args.mode) || S_ISBLK(args.mode)) { + if (unlikely(!sysv_valid_dev(args.rdev) || + MAJOR(args.rdev) & ~0x1ff)) + return -EINVAL; + } else { + args.rdev = 0; + } + + error = posix_acl_create(dir, &args.mode, &default_acl, &acl); + if (error) + return error; + + /* Verify mode is valid also for tmpfile case */ + error = xfs_dentry_mode_to_name(&name, dentry, args.mode); + if (unlikely(error)) + goto out_free_acl; + + if (!tmpfile) { + if (xfs_create_need_xattr(dir, default_acl, acl)) + args.flags |= XFS_ICREATE_INIT_XATTRS; + + error = xfs_create(&args, &name, &ip); + } else { + args.flags |= XFS_ICREATE_TMPFILE; + + /* + * If this temporary file will not be linkable, don't bother + * creating an attr fork to receive a parent pointer. + */ + if (tmpfile->f_flags & O_EXCL) + args.flags |= XFS_ICREATE_UNLINKABLE; + + error = xfs_create_tmpfile(&args, &ip); + } + if (unlikely(error)) + goto out_free_acl; + + inode = VFS_I(ip); + + error = xfs_inode_init_security(inode, dir, &dentry->d_name); + if (unlikely(error)) + goto out_cleanup_inode; + + if (default_acl) { + error = __xfs_set_acl(inode, default_acl, ACL_TYPE_DEFAULT); + if (error) + goto out_cleanup_inode; + } + if (acl) { + error = __xfs_set_acl(inode, acl, ACL_TYPE_ACCESS); + if (error) + goto out_cleanup_inode; + } + + xfs_setup_iops(ip); + + if (tmpfile) { + /* + * The VFS requires that any inode fed to d_tmpfile must have + * nlink == 1 so that it can decrement the nlink in d_tmpfile. + * However, we created the temp file with nlink == 0 because + * we're not allowed to put an inode with nlink > 0 on the + * unlinked list. Therefore we have to set nlink to 1 so that + * d_tmpfile can immediately set it back to zero. + */ + set_nlink(inode, 1); + d_tmpfile(tmpfile, inode); + } else + d_instantiate(dentry, inode); + + xfs_finish_inode_setup(ip); + + out_free_acl: + posix_acl_release(default_acl); + posix_acl_release(acl); + return error; + + out_cleanup_inode: + xfs_finish_inode_setup(ip); + if (!tmpfile) + xfs_cleanup_inode(dir, inode, dentry); + xfs_irele(ip); + goto out_free_acl; +} + +STATIC int +xfs_vn_mknod( + struct mnt_idmap *idmap, + struct inode *dir, + struct dentry *dentry, + umode_t mode, + dev_t rdev) +{ + return xfs_generic_create(idmap, dir, dentry, mode, rdev, NULL); +} + +STATIC int +xfs_vn_create( + struct mnt_idmap *idmap, + struct inode *dir, + struct dentry *dentry, + umode_t mode) +{ + return xfs_generic_create(idmap, dir, dentry, mode, 0, NULL); +} + +STATIC struct dentry * +xfs_vn_mkdir( + struct mnt_idmap *idmap, + struct inode *dir, + struct dentry *dentry, + umode_t mode) +{ + return ERR_PTR(xfs_generic_create(idmap, dir, dentry, mode, 0, NULL)); +} + +STATIC struct dentry * +xfs_vn_lookup( + struct inode *dir, + struct dentry *dentry, + unsigned int flags) +{ + struct inode *inode; + struct xfs_inode *cip; + struct xfs_name name; + int error; + + if (dentry->d_name.len >= MAXNAMELEN) + return ERR_PTR(-ENAMETOOLONG); + + xfs_dentry_to_name(&name, dentry); + error = xfs_lookup(XFS_I(dir), &name, &cip, NULL); + if (likely(!error)) + inode = VFS_I(cip); + else if (likely(error == -ENOENT)) + inode = NULL; + else + inode = ERR_PTR(error); + return d_splice_alias(inode, dentry); +} + +STATIC struct dentry * +xfs_vn_ci_lookup( + struct inode *dir, + struct dentry *dentry, + unsigned int flags) +{ + struct xfs_inode *ip; + struct xfs_name xname; + struct xfs_name ci_name; + struct qstr dname; + int error; + + if (dentry->d_name.len >= MAXNAMELEN) + return ERR_PTR(-ENAMETOOLONG); + + xfs_dentry_to_name(&xname, dentry); + error = xfs_lookup(XFS_I(dir), &xname, &ip, &ci_name); + if (unlikely(error)) { + if (unlikely(error != -ENOENT)) + return ERR_PTR(error); + /* + * call d_add(dentry, NULL) here when d_drop_negative_children + * is called in xfs_vn_mknod (ie. allow negative dentries + * with CI filesystems). + */ + return NULL; + } + + /* if exact match, just splice and exit */ + if (!ci_name.name) + return d_splice_alias(VFS_I(ip), dentry); + + /* else case-insensitive match... */ + dname.name = ci_name.name; + dname.len = ci_name.len; + dentry = d_add_ci(dentry, VFS_I(ip), &dname); + kfree(ci_name.name); + return dentry; +} + +STATIC int +xfs_vn_link( + struct dentry *old_dentry, + struct inode *dir, + struct dentry *dentry) +{ + struct inode *inode = d_inode(old_dentry); + struct xfs_name name; + int error; + + error = xfs_dentry_mode_to_name(&name, dentry, inode->i_mode); + if (unlikely(error)) + return error; + + if (IS_PRIVATE(inode)) + return -EPERM; + + error = xfs_link(XFS_I(dir), XFS_I(inode), &name); + if (unlikely(error)) + return error; + + ihold(inode); + d_instantiate(dentry, inode); + return 0; +} + +STATIC int +xfs_vn_unlink( + struct inode *dir, + struct dentry *dentry) +{ + struct xfs_name name; + int error; + + xfs_dentry_to_name(&name, dentry); + + error = xfs_remove(XFS_I(dir), &name, XFS_I(d_inode(dentry))); + if (error) + return error; + + /* + * With unlink, the VFS makes the dentry "negative": no inode, + * but still hashed. This is incompatible with case-insensitive + * mode, so invalidate (unhash) the dentry in CI-mode. + */ + if (xfs_has_asciici(XFS_M(dir->i_sb))) + d_invalidate(dentry); + return 0; +} + +STATIC int +xfs_vn_symlink( + struct mnt_idmap *idmap, + struct inode *dir, + struct dentry *dentry, + const char *symname) +{ + struct inode *inode; + struct xfs_inode *cip = NULL; + struct xfs_name name; + int error; + umode_t mode = S_IFLNK | S_IRWXUGO; + + error = xfs_dentry_mode_to_name(&name, dentry, mode); + if (unlikely(error)) + goto out; + + error = xfs_symlink(idmap, XFS_I(dir), &name, symname, mode, &cip); + if (unlikely(error)) + goto out; + + inode = VFS_I(cip); + + error = xfs_inode_init_security(inode, dir, &dentry->d_name); + if (unlikely(error)) + goto out_cleanup_inode; + + xfs_setup_iops(cip); + + d_instantiate(dentry, inode); + xfs_finish_inode_setup(cip); + return 0; + + out_cleanup_inode: + xfs_finish_inode_setup(cip); + xfs_cleanup_inode(dir, inode, dentry); + xfs_irele(cip); + out: + return error; +} + +STATIC int +xfs_vn_rename( + struct mnt_idmap *idmap, + struct inode *odir, + struct dentry *odentry, + struct inode *ndir, + struct dentry *ndentry, + unsigned int flags) +{ + struct inode *new_inode = d_inode(ndentry); + int omode = 0; + int error; + struct xfs_name oname; + struct xfs_name nname; + + if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) + return -EINVAL; + + /* if we are exchanging files, we need to set i_mode of both files */ + if (flags & RENAME_EXCHANGE) + omode = d_inode(ndentry)->i_mode; + + error = xfs_dentry_mode_to_name(&oname, odentry, omode); + if (omode && unlikely(error)) + return error; + + error = xfs_dentry_mode_to_name(&nname, ndentry, + d_inode(odentry)->i_mode); + if (unlikely(error)) + return error; + + return xfs_rename(idmap, XFS_I(odir), &oname, + XFS_I(d_inode(odentry)), XFS_I(ndir), &nname, + new_inode ? XFS_I(new_inode) : NULL, flags); +} + +/* + * careful here - this function can get called recursively, so + * we need to be very careful about how much stack we use. + * uio is kmalloced for this reason... + */ +STATIC const char * +xfs_vn_get_link( + struct dentry *dentry, + struct inode *inode, + struct delayed_call *done) +{ + char *link; + int error = -ENOMEM; + + if (!dentry) + return ERR_PTR(-ECHILD); + + link = kmalloc(XFS_SYMLINK_MAXLEN+1, GFP_KERNEL); + if (!link) + goto out_err; + + error = xfs_readlink(XFS_I(d_inode(dentry)), link); + if (unlikely(error)) + goto out_kfree; + + set_delayed_call(done, kfree_link, link); + return link; + + out_kfree: + kfree(link); + out_err: + return ERR_PTR(error); +} + +static uint32_t +xfs_stat_blksize( + struct xfs_inode *ip) +{ + struct xfs_mount *mp = ip->i_mount; + + /* + * If the file blocks are being allocated from a realtime volume, then + * always return the realtime extent size. + */ + if (XFS_IS_REALTIME_INODE(ip)) + return XFS_FSB_TO_B(mp, xfs_get_extsz_hint(ip) ? : 1); + + /* + * Allow large block sizes to be reported to userspace programs if the + * "largeio" mount option is used. + * + * If compatibility mode is specified, simply return the basic unit of + * caching so that we don't get inefficient read/modify/write I/O from + * user apps. Otherwise.... + * + * If the underlying volume is a stripe, then return the stripe width in + * bytes as the recommended I/O size. It is not a stripe and we've set a + * default buffered I/O size, return that, otherwise return the compat + * default. + */ + if (xfs_has_large_iosize(mp)) { + if (mp->m_swidth) + return XFS_FSB_TO_B(mp, mp->m_swidth); + if (xfs_has_allocsize(mp)) + return 1U << mp->m_allocsize_log; + } + + return max_t(uint32_t, PAGE_SIZE, mp->m_sb.sb_blocksize); +} + +static void +xfs_report_dioalign( + struct xfs_inode *ip, + struct kstat *stat) +{ + struct xfs_buftarg *target = xfs_inode_buftarg(ip); + struct block_device *bdev = target->bt_bdev; + + stat->result_mask |= STATX_DIOALIGN | STATX_DIO_READ_ALIGN; + stat->dio_mem_align = bdev_dma_alignment(bdev) + 1; + + /* + * For COW inodes, we can only perform out of place writes of entire + * allocation units (blocks or RT extents). + * For writes smaller than the allocation unit, we must fall back to + * buffered I/O to perform read-modify-write cycles. At best this is + * highly inefficient; at worst it leads to page cache invalidation + * races. Tell applications to avoid this by reporting the larger write + * alignment in dio_offset_align, and the smaller read alignment in + * dio_read_offset_align. + */ + stat->dio_read_offset_align = bdev_logical_block_size(bdev); + if (xfs_is_cow_inode(ip)) + stat->dio_offset_align = xfs_inode_alloc_unitsize(ip); + else + stat->dio_offset_align = stat->dio_read_offset_align; +} + +unsigned int +xfs_get_atomic_write_min( + struct xfs_inode *ip) +{ + struct xfs_mount *mp = ip->i_mount; + + /* + * If we can complete an atomic write via atomic out of place writes, + * then advertise a minimum size of one fsblock. Without this + * mechanism, we can only guarantee atomic writes up to a single LBA. + * + * If out of place writes are not available, we can guarantee an atomic + * write of exactly one single fsblock if the bdev will make that + * guarantee for us. + */ + if (xfs_inode_can_hw_atomic_write(ip) || + xfs_inode_can_sw_atomic_write(ip)) + return mp->m_sb.sb_blocksize; + + return 0; +} + +unsigned int +xfs_get_atomic_write_max( + struct xfs_inode *ip) +{ + struct xfs_mount *mp = ip->i_mount; + + /* + * If out of place writes are not available, we can guarantee an atomic + * write of exactly one single fsblock if the bdev will make that + * guarantee for us. + */ + if (!xfs_inode_can_sw_atomic_write(ip)) { + if (xfs_inode_can_hw_atomic_write(ip)) + return mp->m_sb.sb_blocksize; + return 0; + } + + /* + * If we can complete an atomic write via atomic out of place writes, + * then advertise a maximum size of whatever we can complete through + * that means. Hardware support is reported via max_opt, not here. + */ + if (XFS_IS_REALTIME_INODE(ip)) + return XFS_FSB_TO_B(mp, mp->m_groups[XG_TYPE_RTG].awu_max); + return XFS_FSB_TO_B(mp, mp->m_groups[XG_TYPE_AG].awu_max); +} + +unsigned int +xfs_get_atomic_write_max_opt( + struct xfs_inode *ip) +{ + unsigned int awu_max = xfs_get_atomic_write_max(ip); + + /* if the max is 1x block, then just keep behaviour that opt is 0 */ + if (awu_max <= ip->i_mount->m_sb.sb_blocksize) + return 0; + + /* + * Advertise the maximum size of an atomic write that we can tell the + * block device to perform for us. In general the bdev limit will be + * less than our out of place write limit, but we don't want to exceed + * the awu_max. + */ + return min(awu_max, xfs_inode_buftarg(ip)->bt_awu_max); +} + +static void +xfs_report_atomic_write( + struct xfs_inode *ip, + struct kstat *stat) +{ + generic_fill_statx_atomic_writes(stat, + xfs_get_atomic_write_min(ip), + xfs_get_atomic_write_max(ip), + xfs_get_atomic_write_max_opt(ip)); +} + +STATIC int +xfs_vn_getattr( + struct mnt_idmap *idmap, + const struct path *path, + struct kstat *stat, + u32 request_mask, + unsigned int query_flags) +{ + struct inode *inode = d_inode(path->dentry); + struct xfs_inode *ip = XFS_I(inode); + struct xfs_mount *mp = ip->i_mount; + vfsuid_t vfsuid = i_uid_into_vfsuid(idmap, inode); + vfsgid_t vfsgid = i_gid_into_vfsgid(idmap, inode); + + trace_xfs_getattr(ip); + + if (xfs_is_shutdown(mp)) + return -EIO; + + stat->size = XFS_ISIZE(ip); + stat->dev = inode->i_sb->s_dev; + stat->mode = inode->i_mode; + stat->nlink = inode->i_nlink; + stat->uid = vfsuid_into_kuid(vfsuid); + stat->gid = vfsgid_into_kgid(vfsgid); + stat->ino = inode->i_ino; + stat->atime = inode_get_atime(inode); + + fill_mg_cmtime(stat, request_mask, inode); + + stat->blocks = XFS_FSB_TO_BB(mp, ip->i_nblocks + ip->i_delayed_blks); + + if (xfs_has_v3inodes(mp)) { + if (request_mask & STATX_BTIME) { + stat->result_mask |= STATX_BTIME; + stat->btime = ip->i_crtime; + } + } + + /* + * Note: If you add another clause to set an attribute flag, please + * update attributes_mask below. + */ + if (ip->i_diflags & XFS_DIFLAG_IMMUTABLE) + stat->attributes |= STATX_ATTR_IMMUTABLE; + if (ip->i_diflags & XFS_DIFLAG_APPEND) + stat->attributes |= STATX_ATTR_APPEND; + if (ip->i_diflags & XFS_DIFLAG_NODUMP) + stat->attributes |= STATX_ATTR_NODUMP; + + stat->attributes_mask |= (STATX_ATTR_IMMUTABLE | + STATX_ATTR_APPEND | + STATX_ATTR_NODUMP); + + switch (inode->i_mode & S_IFMT) { + case S_IFBLK: + case S_IFCHR: + stat->blksize = BLKDEV_IOSIZE; + stat->rdev = inode->i_rdev; + break; + case S_IFREG: + if (request_mask & (STATX_DIOALIGN | STATX_DIO_READ_ALIGN)) + xfs_report_dioalign(ip, stat); + if (request_mask & STATX_WRITE_ATOMIC) + xfs_report_atomic_write(ip, stat); + fallthrough; + default: + stat->blksize = xfs_stat_blksize(ip); + stat->rdev = 0; + break; + } + + return 0; +} + +static int +xfs_vn_change_ok( + struct mnt_idmap *idmap, + struct dentry *dentry, + struct iattr *iattr) +{ + struct xfs_mount *mp = XFS_I(d_inode(dentry))->i_mount; + + if (xfs_is_readonly(mp)) + return -EROFS; + + if (xfs_is_shutdown(mp)) + return -EIO; + + return setattr_prepare(idmap, dentry, iattr); +} + +/* + * Set non-size attributes of an inode. + * + * Caution: The caller of this function is responsible for calling + * setattr_prepare() or otherwise verifying the change is fine. + */ +static int +xfs_setattr_nonsize( + struct mnt_idmap *idmap, + struct dentry *dentry, + struct xfs_inode *ip, + struct iattr *iattr) +{ + xfs_mount_t *mp = ip->i_mount; + struct inode *inode = VFS_I(ip); + int mask = iattr->ia_valid; + xfs_trans_t *tp; + int error; + kuid_t uid = GLOBAL_ROOT_UID; + kgid_t gid = GLOBAL_ROOT_GID; + struct xfs_dquot *udqp = NULL, *gdqp = NULL; + struct xfs_dquot *old_udqp = NULL, *old_gdqp = NULL; + + ASSERT((mask & ATTR_SIZE) == 0); + + /* + * If disk quotas is on, we make sure that the dquots do exist on disk, + * before we start any other transactions. Trying to do this later + * is messy. We don't care to take a readlock to look at the ids + * in inode here, because we can't hold it across the trans_reserve. + * If the IDs do change before we take the ilock, we're covered + * because the i_*dquot fields will get updated anyway. + */ + if (XFS_IS_QUOTA_ON(mp) && (mask & (ATTR_UID|ATTR_GID))) { + uint qflags = 0; + + if ((mask & ATTR_UID) && XFS_IS_UQUOTA_ON(mp)) { + uid = from_vfsuid(idmap, i_user_ns(inode), + iattr->ia_vfsuid); + qflags |= XFS_QMOPT_UQUOTA; + } else { + uid = inode->i_uid; + } + if ((mask & ATTR_GID) && XFS_IS_GQUOTA_ON(mp)) { + gid = from_vfsgid(idmap, i_user_ns(inode), + iattr->ia_vfsgid); + qflags |= XFS_QMOPT_GQUOTA; + } else { + gid = inode->i_gid; + } + + /* + * We take a reference when we initialize udqp and gdqp, + * so it is important that we never blindly double trip on + * the same variable. See xfs_create() for an example. + */ + ASSERT(udqp == NULL); + ASSERT(gdqp == NULL); + error = xfs_qm_vop_dqalloc(ip, uid, gid, ip->i_projid, + qflags, &udqp, &gdqp, NULL); + if (error) + return error; + } + + error = xfs_trans_alloc_ichange(ip, udqp, gdqp, NULL, + has_capability_noaudit(current, CAP_FOWNER), &tp); + if (error) + goto out_dqrele; + + /* + * Register quota modifications in the transaction. Must be the owner + * or privileged. These IDs could have changed since we last looked at + * them. But, we're assured that if the ownership did change while we + * didn't have the inode locked, inode's dquot(s) would have changed + * also. + */ + if (XFS_IS_UQUOTA_ON(mp) && + i_uid_needs_update(idmap, iattr, inode)) { + ASSERT(udqp); + old_udqp = xfs_qm_vop_chown(tp, ip, &ip->i_udquot, udqp); + } + if (XFS_IS_GQUOTA_ON(mp) && + i_gid_needs_update(idmap, iattr, inode)) { + ASSERT(xfs_has_pquotino(mp) || !XFS_IS_PQUOTA_ON(mp)); + ASSERT(gdqp); + old_gdqp = xfs_qm_vop_chown(tp, ip, &ip->i_gdquot, gdqp); + } + + setattr_copy(idmap, inode, iattr); + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + + XFS_STATS_INC(mp, xs_ig_attrchg); + + if (xfs_has_wsync(mp)) + xfs_trans_set_sync(tp); + error = xfs_trans_commit(tp); + + /* + * Release any dquot(s) the inode had kept before chown. + */ + xfs_qm_dqrele(old_udqp); + xfs_qm_dqrele(old_gdqp); + xfs_qm_dqrele(udqp); + xfs_qm_dqrele(gdqp); + + if (error) + return error; + + /* + * XXX(hch): Updating the ACL entries is not atomic vs the i_mode + * update. We could avoid this with linked transactions + * and passing down the transaction pointer all the way + * to attr_set. No previous user of the generic + * Posix ACL code seems to care about this issue either. + */ + if (mask & ATTR_MODE) { + error = posix_acl_chmod(idmap, dentry, inode->i_mode); + if (error) + return error; + } + + return 0; + +out_dqrele: + xfs_qm_dqrele(udqp); + xfs_qm_dqrele(gdqp); + return error; +} + +/* + * Truncate file. Must have write permission and not be a directory. + */ +int +xfs_vn_setattr_size( + struct mnt_idmap *idmap, + struct dentry *dentry, + struct iattr *iattr) +{ + struct inode *inode = d_inode(dentry); + struct xfs_inode *ip = XFS_I(inode); + struct xfs_mount *mp = ip->i_mount; + xfs_off_t oldsize = inode->i_size; + xfs_off_t newsize = iattr->ia_size; + struct xfs_trans *tp; + int error; + uint lock_flags = 0; + uint resblks = 0; + bool did_zeroing = false; + struct xfs_zone_alloc_ctx ac = { }; + + xfs_assert_ilocked(ip, XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL); + ASSERT(S_ISREG(inode->i_mode)); + ASSERT((iattr->ia_valid & (ATTR_UID|ATTR_GID|ATTR_ATIME|ATTR_ATIME_SET| + ATTR_MTIME_SET|ATTR_TIMES_SET)) == 0); + + trace_xfs_setattr(ip); + + error = xfs_vn_change_ok(idmap, dentry, iattr); + if (error) + return error; + + /* + * Short circuit the truncate case for zero length files. + */ + if (newsize == 0 && oldsize == 0 && ip->i_df.if_nextents == 0) { + if (!(iattr->ia_valid & (ATTR_CTIME|ATTR_MTIME))) + return 0; + + /* + * Use the regular setattr path to update the timestamps. + */ + iattr->ia_valid &= ~ATTR_SIZE; + return xfs_setattr_nonsize(idmap, dentry, ip, iattr); + } + + /* + * Make sure that the dquots are attached to the inode. + */ + error = xfs_qm_dqattach(ip); + if (error) + return error; + + /* + * Wait for all direct I/O to complete. + */ + inode_dio_wait(inode); + + /* + * Normally xfs_zoned_space_reserve is supposed to be called outside the + * IOLOCK. For truncate we can't do that since ->setattr is called with + * it already held by the VFS. So for now chicken out and try to + * allocate space under it. + * + * To avoid deadlocks this means we can't block waiting for space, which + * can lead to spurious -ENOSPC if there are no directly available + * blocks. We mitigate this a bit by allowing zeroing to dip into the + * reserved pool, but eventually the VFS calling convention needs to + * change. + */ + if (xfs_is_zoned_inode(ip)) { + error = xfs_zoned_space_reserve(mp, 1, + XFS_ZR_NOWAIT | XFS_ZR_RESERVED, &ac); + if (error) { + if (error == -EAGAIN) + return -ENOSPC; + return error; + } + } + + /* + * File data changes must be complete before we start the transaction to + * modify the inode. This needs to be done before joining the inode to + * the transaction because the inode cannot be unlocked once it is a + * part of the transaction. + * + * Start with zeroing any data beyond EOF that we may expose on file + * extension, or zeroing out the rest of the block on a downward + * truncate. + */ + if (newsize > oldsize) { + trace_xfs_zero_eof(ip, oldsize, newsize - oldsize); + error = xfs_zero_range(ip, oldsize, newsize - oldsize, + &ac, &did_zeroing); + } else { + error = xfs_truncate_page(ip, newsize, &ac, &did_zeroing); + } + + if (xfs_is_zoned_inode(ip)) + xfs_zoned_space_unreserve(mp, &ac); + + if (error) + return error; + + /* + * We've already locked out new page faults, so now we can safely remove + * pages from the page cache knowing they won't get refaulted until we + * drop the XFS_MMAP_EXCL lock after the extent manipulations are + * complete. The truncate_setsize() call also cleans partial EOF page + * PTEs on extending truncates and hence ensures sub-page block size + * filesystems are correctly handled, too. + * + * We have to do all the page cache truncate work outside the + * transaction context as the "lock" order is page lock->log space + * reservation as defined by extent allocation in the writeback path. + * Hence a truncate can fail with ENOMEM from xfs_trans_alloc(), but + * having already truncated the in-memory version of the file (i.e. made + * user visible changes). There's not much we can do about this, except + * to hope that the caller sees ENOMEM and retries the truncate + * operation. + * + * And we update in-core i_size and truncate page cache beyond newsize + * before writeback the [i_disk_size, newsize] range, so we're + * guaranteed not to write stale data past the new EOF on truncate down. + */ + truncate_setsize(inode, newsize); + + /* + * We are going to log the inode size change in this transaction so + * any previous writes that are beyond the on disk EOF and the new + * EOF that have not been written out need to be written here. If we + * do not write the data out, we expose ourselves to the null files + * problem. Note that this includes any block zeroing we did above; + * otherwise those blocks may not be zeroed after a crash. + */ + if (did_zeroing || + (newsize > ip->i_disk_size && oldsize != ip->i_disk_size)) { + error = filemap_write_and_wait_range(VFS_I(ip)->i_mapping, + ip->i_disk_size, newsize - 1); + if (error) + return error; + } + + /* + * For realtime inode with more than one block rtextsize, we need the + * block reservation for bmap btree block allocations/splits that can + * happen since it could split the tail written extent and convert the + * right beyond EOF one to unwritten. + */ + if (xfs_inode_has_bigrtalloc(ip)) + resblks = XFS_DIOSTRAT_SPACE_RES(mp, 0); + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_itruncate, resblks, + 0, 0, &tp); + if (error) + return error; + + lock_flags |= XFS_ILOCK_EXCL; + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, 0); + + /* + * Only change the c/mtime if we are changing the size or we are + * explicitly asked to change it. This handles the semantic difference + * between truncate() and ftruncate() as implemented in the VFS. + * + * The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a + * special case where we need to update the times despite not having + * these flags set. For all other operations the VFS set these flags + * explicitly if it wants a timestamp update. + */ + if (newsize != oldsize && + !(iattr->ia_valid & (ATTR_CTIME | ATTR_MTIME))) { + iattr->ia_ctime = iattr->ia_mtime = + current_time(inode); + iattr->ia_valid |= ATTR_CTIME | ATTR_MTIME; + } + + /* + * The first thing we do is set the size to new_size permanently on + * disk. This way we don't have to worry about anyone ever being able + * to look at the data being freed even in the face of a crash. + * What we're getting around here is the case where we free a block, it + * is allocated to another file, it is written to, and then we crash. + * If the new data gets written to the file but the log buffers + * containing the free and reallocation don't, then we'd end up with + * garbage in the blocks being freed. As long as we make the new size + * permanent before actually freeing any blocks it doesn't matter if + * they get written to. + */ + ip->i_disk_size = newsize; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + + if (newsize <= oldsize) { + error = xfs_itruncate_extents(&tp, ip, XFS_DATA_FORK, newsize); + if (error) + goto out_trans_cancel; + + /* + * Truncated "down", so we're removing references to old data + * here - if we delay flushing for a long time, we expose + * ourselves unduly to the notorious NULL files problem. So, + * we mark this inode and flush it when the file is closed, + * and do not wait the usual (long) time for writeout. + */ + xfs_iflags_set(ip, XFS_ITRUNCATED); + + /* A truncate down always removes post-EOF blocks. */ + xfs_inode_clear_eofblocks_tag(ip); + } + + setattr_copy(idmap, inode, iattr); + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + + XFS_STATS_INC(mp, xs_ig_attrchg); + + if (xfs_has_wsync(mp)) + xfs_trans_set_sync(tp); + + error = xfs_trans_commit(tp); +out_unlock: + if (lock_flags) + xfs_iunlock(ip, lock_flags); + return error; + +out_trans_cancel: + xfs_trans_cancel(tp); + goto out_unlock; +} + +STATIC int +xfs_vn_setattr( + struct mnt_idmap *idmap, + struct dentry *dentry, + struct iattr *iattr) +{ + struct inode *inode = d_inode(dentry); + struct xfs_inode *ip = XFS_I(inode); + int error; + + if (iattr->ia_valid & ATTR_SIZE) { + uint iolock; + + xfs_ilock(ip, XFS_MMAPLOCK_EXCL); + iolock = XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL; + + error = xfs_break_layouts(inode, &iolock, BREAK_UNMAP); + if (error) { + xfs_iunlock(ip, XFS_MMAPLOCK_EXCL); + return error; + } + + error = xfs_vn_setattr_size(idmap, dentry, iattr); + xfs_iunlock(ip, XFS_MMAPLOCK_EXCL); + } else { + trace_xfs_setattr(ip); + + error = xfs_vn_change_ok(idmap, dentry, iattr); + if (!error) + error = xfs_setattr_nonsize(idmap, dentry, ip, iattr); + } + + return error; +} + +STATIC int +xfs_vn_update_time( + struct inode *inode, + enum fs_update_time type, + unsigned int flags) +{ + struct xfs_inode *ip = XFS_I(inode); + struct xfs_mount *mp = ip->i_mount; + int log_flags = XFS_ILOG_TIMESTAMP; + struct xfs_trans *tp; + int error; + + trace_xfs_update_time(ip); + + if (inode->i_sb->s_flags & SB_LAZYTIME) { + int dirty; + + dirty = inode_update_time(inode, type, flags); + if (dirty <= 0) + return dirty; + if (dirty == I_DIRTY_TIME) { + __mark_inode_dirty(inode, I_DIRTY_TIME); + return 0; + } + + /* Capture the iversion update that just occurred */ + log_flags |= XFS_ILOG_CORE; + } else { + if (flags & IOCB_NOWAIT) + return -EAGAIN; + } + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_fsyncts, 0, 0, 0, &tp); + if (error) + return error; + + xfs_ilock(ip, XFS_ILOCK_EXCL); + if (type == FS_UPD_ATIME) + inode_set_atime_to_ts(inode, current_time(inode)); + else + inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); + xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL); + xfs_trans_log_inode(tp, ip, log_flags); + return xfs_trans_commit(tp); +} + +static void +xfs_vn_sync_lazytime( + struct inode *inode) +{ + struct xfs_inode *ip = XFS_I(inode); + struct xfs_mount *mp = ip->i_mount; + struct xfs_trans *tp; + + if (xfs_trans_alloc(mp, &M_RES(mp)->tr_fsyncts, 0, 0, 0, &tp)) + return; + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL); + xfs_trans_log_inode(tp, ip, XFS_ILOG_TIMESTAMP); + xfs_trans_commit(tp); +} + +STATIC int +xfs_vn_fiemap( + struct inode *inode, + struct fiemap_extent_info *fieinfo, + u64 start, + u64 length) +{ + int error; + + xfs_ilock(XFS_I(inode), XFS_IOLOCK_SHARED); + if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) { + fieinfo->fi_flags &= ~FIEMAP_FLAG_XATTR; + error = iomap_fiemap(inode, fieinfo, start, length, + &xfs_xattr_iomap_ops); + } else { + error = iomap_fiemap(inode, fieinfo, start, length, + &xfs_read_iomap_ops); + } + xfs_iunlock(XFS_I(inode), XFS_IOLOCK_SHARED); + + return error; +} + +STATIC int +xfs_vn_tmpfile( + struct mnt_idmap *idmap, + struct inode *dir, + struct file *file, + umode_t mode) +{ + int err = xfs_generic_create(idmap, dir, file->f_path.dentry, mode, 0, file); + + return finish_open_simple(file, err); +} + +static const struct inode_operations xfs_inode_operations = { + .get_inode_acl = xfs_get_acl, + .set_acl = xfs_set_acl, + .getattr = xfs_vn_getattr, + .setattr = xfs_vn_setattr, + .listxattr = xfs_vn_listxattr, + .fiemap = xfs_vn_fiemap, + .update_time = xfs_vn_update_time, + .sync_lazytime = xfs_vn_sync_lazytime, + .fileattr_get = xfs_fileattr_get, + .fileattr_set = xfs_fileattr_set, +}; + +static const struct inode_operations xfs_dir_inode_operations = { + .create = xfs_vn_create, + .lookup = xfs_vn_lookup, + .link = xfs_vn_link, + .unlink = xfs_vn_unlink, + .symlink = xfs_vn_symlink, + .mkdir = xfs_vn_mkdir, + /* + * Yes, XFS uses the same method for rmdir and unlink. + * + * There are some subtile differences deeper in the code, + * but we use S_ISDIR to check for those. + */ + .rmdir = xfs_vn_unlink, + .mknod = xfs_vn_mknod, + .rename = xfs_vn_rename, + .get_inode_acl = xfs_get_acl, + .set_acl = xfs_set_acl, + .getattr = xfs_vn_getattr, + .setattr = xfs_vn_setattr, + .listxattr = xfs_vn_listxattr, + .update_time = xfs_vn_update_time, + .sync_lazytime = xfs_vn_sync_lazytime, + .tmpfile = xfs_vn_tmpfile, + .fileattr_get = xfs_fileattr_get, + .fileattr_set = xfs_fileattr_set, +}; + +static const struct inode_operations xfs_dir_ci_inode_operations = { + .create = xfs_vn_create, + .lookup = xfs_vn_ci_lookup, + .link = xfs_vn_link, + .unlink = xfs_vn_unlink, + .symlink = xfs_vn_symlink, + .mkdir = xfs_vn_mkdir, + /* + * Yes, XFS uses the same method for rmdir and unlink. + * + * There are some subtile differences deeper in the code, + * but we use S_ISDIR to check for those. + */ + .rmdir = xfs_vn_unlink, + .mknod = xfs_vn_mknod, + .rename = xfs_vn_rename, + .get_inode_acl = xfs_get_acl, + .set_acl = xfs_set_acl, + .getattr = xfs_vn_getattr, + .setattr = xfs_vn_setattr, + .listxattr = xfs_vn_listxattr, + .update_time = xfs_vn_update_time, + .sync_lazytime = xfs_vn_sync_lazytime, + .tmpfile = xfs_vn_tmpfile, + .fileattr_get = xfs_fileattr_get, + .fileattr_set = xfs_fileattr_set, +}; + +static const struct inode_operations xfs_symlink_inode_operations = { + .get_link = xfs_vn_get_link, + .getattr = xfs_vn_getattr, + .setattr = xfs_vn_setattr, + .listxattr = xfs_vn_listxattr, + .update_time = xfs_vn_update_time, + .sync_lazytime = xfs_vn_sync_lazytime, + .fileattr_get = xfs_fileattr_get, + .fileattr_set = xfs_fileattr_set, +}; + +/* Figure out if this file actually supports DAX. */ +static bool +xfs_inode_supports_dax( + struct xfs_inode *ip) +{ + struct xfs_mount *mp = ip->i_mount; + + /* Only supported on regular files. */ + if (!S_ISREG(VFS_I(ip)->i_mode)) + return false; + + /* Block size must match page size */ + if (mp->m_sb.sb_blocksize != PAGE_SIZE) + return false; + + /* Device has to support DAX too. */ + return xfs_inode_buftarg(ip)->bt_daxdev != NULL; +} + +static bool +xfs_inode_should_enable_dax( + struct xfs_inode *ip) +{ + if (!IS_ENABLED(CONFIG_FS_DAX)) + return false; + if (xfs_has_dax_never(ip->i_mount)) + return false; + if (!xfs_inode_supports_dax(ip)) + return false; + if (xfs_has_dax_always(ip->i_mount)) + return true; + if (ip->i_diflags2 & XFS_DIFLAG2_DAX) + return true; + return false; +} + +void +xfs_diflags_to_iflags( + struct xfs_inode *ip, + bool init) +{ + struct inode *inode = VFS_I(ip); + unsigned int xflags = xfs_ip2xflags(ip); + unsigned int flags = 0; + + ASSERT(!(IS_DAX(inode) && init)); + + if (xflags & FS_XFLAG_IMMUTABLE) + flags |= S_IMMUTABLE; + if (xflags & FS_XFLAG_APPEND) + flags |= S_APPEND; + if (xflags & FS_XFLAG_SYNC) + flags |= S_SYNC; + if (xflags & FS_XFLAG_NOATIME) + flags |= S_NOATIME; + if (init && xfs_inode_should_enable_dax(ip)) + flags |= S_DAX; + + /* + * S_DAX can only be set during inode initialization and is never set by + * the VFS, so we cannot mask off S_DAX in i_flags. + */ + inode->i_flags &= ~(S_IMMUTABLE | S_APPEND | S_SYNC | S_NOATIME); + inode->i_flags |= flags; +} + +/* + * Initialize the Linux inode. + * + * When reading existing inodes from disk this is called directly from xfs_iget, + * when creating a new inode it is called from xfs_init_new_inode after setting + * up the inode. These callers have different criteria for clearing XFS_INEW, so + * leave it up to the caller to deal with unlocking the inode appropriately. + */ +void +xfs_setup_inode( + struct xfs_inode *ip) +{ + struct inode *inode = &ip->i_vnode; + gfp_t gfp_mask; + bool is_meta = xfs_is_internal_inode(ip); + + inode_state_set_raw(inode, I_NEW); + + inode_sb_list_add(inode); + /* make the inode look hashed for the writeback code */ + inode_fake_hash(inode); + + i_size_write(inode, ip->i_disk_size); + xfs_diflags_to_iflags(ip, true); + + /* + * Mark our metadata files as private so that LSMs and the ACL code + * don't try to add their own metadata or reason about these files, + * and users cannot ever obtain file handles to them. + */ + if (is_meta) { + inode->i_flags |= S_PRIVATE; + inode->i_opflags &= ~IOP_XATTR; + } + + if (S_ISDIR(inode->i_mode)) { + /* + * We set the i_rwsem class here to avoid potential races with + * lockdep_annotate_inode_mutex_key() reinitialising the lock + * after a filehandle lookup has already found the inode in + * cache before it has been unlocked via unlock_new_inode(). + */ + lockdep_set_class(&inode->i_rwsem, + &inode->i_sb->s_type->i_mutex_dir_key); + lockdep_set_class(&ip->i_lock, &xfs_dir_ilock_class); + } else { + lockdep_set_class(&ip->i_lock, &xfs_nondir_ilock_class); + } + + /* + * Ensure all page cache allocations are done from GFP_NOFS context to + * prevent direct reclaim recursion back into the filesystem and blowing + * stacks or deadlocking. + */ + gfp_mask = mapping_gfp_mask(inode->i_mapping); + mapping_set_gfp_mask(inode->i_mapping, (gfp_mask & ~(__GFP_FS))); + + /* + * For real-time inodes update the stable write flags to that of the RT + * device instead of the data device. + */ + if (S_ISREG(inode->i_mode) && XFS_IS_REALTIME_INODE(ip)) + xfs_update_stable_writes(ip); + + /* + * If there is no attribute fork no ACL can exist on this inode, + * and it can't have any file capabilities attached to it either. + */ + if (!xfs_inode_has_attr_fork(ip)) { + inode_has_no_xattr(inode); + cache_no_acl(inode); + } +} + +void +xfs_setup_iops( + struct xfs_inode *ip) +{ + struct inode *inode = &ip->i_vnode; + + switch (inode->i_mode & S_IFMT) { + case S_IFREG: + inode->i_op = &xfs_inode_operations; + inode->i_fop = &xfs_file_operations; + if (IS_DAX(inode)) + inode->i_mapping->a_ops = &xfs_dax_aops; + else + inode->i_mapping->a_ops = &xfs_address_space_operations; + break; + case S_IFDIR: + if (xfs_has_asciici(XFS_M(inode->i_sb))) + inode->i_op = &xfs_dir_ci_inode_operations; + else + inode->i_op = &xfs_dir_inode_operations; + inode->i_fop = &xfs_dir_file_operations; + break; + case S_IFLNK: + inode->i_op = &xfs_symlink_inode_operations; + break; + default: + inode->i_op = &xfs_inode_operations; + init_special_inode(inode, inode->i_mode, inode->i_rdev); + break; + } +} diff --git a/fs/xfs/xfs_iops.h b/fs/xfs/xfs_iops.h new file mode 100644 index 000000000..0896f6b8b --- /dev/null +++ b/fs/xfs/xfs_iops.h @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2003,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_IOPS_H__ +#define __XFS_IOPS_H__ + +struct xfs_inode; + +extern ssize_t xfs_vn_listxattr(struct dentry *, char *data, size_t size); + +int xfs_vn_setattr_size(struct mnt_idmap *idmap, + struct dentry *dentry, struct iattr *vap); + +int xfs_inode_init_security(struct inode *inode, struct inode *dir, + const struct qstr *qstr); + +extern void xfs_setup_inode(struct xfs_inode *ip); +extern void xfs_setup_iops(struct xfs_inode *ip); +extern void xfs_diflags_to_iflags(struct xfs_inode *ip, bool init); +unsigned int xfs_get_atomic_write_min(struct xfs_inode *ip); +unsigned int xfs_get_atomic_write_max(struct xfs_inode *ip); +unsigned int xfs_get_atomic_write_max_opt(struct xfs_inode *ip); + +#endif /* __XFS_IOPS_H__ */ diff --git a/fs/xfs/xfs_itable.c b/fs/xfs/xfs_itable.c new file mode 100644 index 000000000..159295c63 --- /dev/null +++ b/fs/xfs/xfs_itable.c @@ -0,0 +1,484 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_btree.h" +#include "xfs_ialloc.h" +#include "xfs_ialloc_btree.h" +#include "xfs_iwalk.h" +#include "xfs_itable.h" +#include "xfs_error.h" +#include "xfs_icache.h" +#include "xfs_health.h" +#include "xfs_trans.h" + +/* + * Bulk Stat + * ========= + * + * Use the inode walking functions to fill out struct xfs_bulkstat for every + * allocated inode, then pass the stat information to some externally provided + * iteration function. + */ + +struct xfs_bstat_chunk { + bulkstat_one_fmt_pf formatter; + struct xfs_ibulk *breq; + struct xfs_bulkstat *buf; +}; + +static inline bool +want_metadir_file( + struct xfs_inode *ip, + struct xfs_ibulk *breq) +{ + return xfs_is_metadir_inode(ip) && (breq->flags & XFS_IBULK_METADIR); +} + +/* + * Fill out the bulkstat info for a single inode and report it somewhere. + * + * bc->breq->lastino is effectively the inode cursor as we walk through the + * filesystem. Therefore, we update it any time we need to move the cursor + * forward, regardless of whether or not we're sending any bstat information + * back to userspace. If the inode is internal metadata or, has been freed + * out from under us, we just simply keep going. + * + * However, if any other type of error happens we want to stop right where we + * are so that userspace will call back with exact number of the bad inode and + * we can send back an error code. + * + * Note that if the formatter tells us there's no space left in the buffer we + * move the cursor forward and abort the walk. + */ +STATIC int +xfs_bulkstat_one_int( + struct xfs_mount *mp, + struct mnt_idmap *idmap, + struct xfs_trans *tp, + xfs_ino_t ino, + struct xfs_bstat_chunk *bc) +{ + struct user_namespace *sb_userns = mp->m_super->s_user_ns; + struct xfs_inode *ip; /* incore inode pointer */ + struct inode *inode; + struct xfs_bulkstat *buf = bc->buf; + xfs_extnum_t nextents; + int error = -EINVAL; + vfsuid_t vfsuid; + vfsgid_t vfsgid; + + error = xfs_iget(mp, tp, ino, + (XFS_IGET_DONTCACHE | XFS_IGET_UNTRUSTED), + XFS_ILOCK_SHARED, &ip); + if (error == -ENOENT || error == -EINVAL) + goto out_advance; + if (error) + goto out; + + /* Reload the incore unlinked list to avoid failure in inodegc. */ + if (xfs_inode_unlinked_incomplete(ip)) { + error = xfs_inode_reload_unlinked_bucket(tp, ip); + if (error) { + xfs_iunlock(ip, XFS_ILOCK_SHARED); + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); + xfs_irele(ip); + return error; + } + } + + ASSERT(ip != NULL); + ASSERT(ip->i_imap.im_agbno != 0); + inode = VFS_I(ip); + vfsuid = i_uid_into_vfsuid(idmap, inode); + vfsgid = i_gid_into_vfsgid(idmap, inode); + + /* + * If caller wants files from the metadata directories, push out the + * bare minimum information for enabling scrub. + */ + if (want_metadir_file(ip, bc->breq)) { + memset(buf, 0, sizeof(*buf)); + buf->bs_ino = ino; + buf->bs_gen = inode->i_generation; + buf->bs_mode = inode->i_mode & S_IFMT; + xfs_bulkstat_health(ip, buf); + buf->bs_version = XFS_BULKSTAT_VERSION_V5; + xfs_iunlock(ip, XFS_ILOCK_SHARED); + xfs_irele(ip); + + error = bc->formatter(bc->breq, buf); + if (!error || error == -ECANCELED) + goto out_advance; + goto out; + } + + /* If this is a private inode, don't leak its details to userspace. */ + if (IS_PRIVATE(inode) || xfs_is_sb_inum(mp, ino)) { + xfs_iunlock(ip, XFS_ILOCK_SHARED); + xfs_irele(ip); + error = -EINVAL; + goto out_advance; + } + + /* xfs_iget returns the following without needing + * further change. + */ + buf->bs_projectid = ip->i_projid; + buf->bs_ino = ino; + buf->bs_uid = from_kuid(sb_userns, vfsuid_into_kuid(vfsuid)); + buf->bs_gid = from_kgid(sb_userns, vfsgid_into_kgid(vfsgid)); + buf->bs_size = ip->i_disk_size; + + buf->bs_nlink = inode->i_nlink; + buf->bs_atime = inode_get_atime_sec(inode); + buf->bs_atime_nsec = inode_get_atime_nsec(inode); + buf->bs_mtime = inode_get_mtime_sec(inode); + buf->bs_mtime_nsec = inode_get_mtime_nsec(inode); + buf->bs_ctime = inode_get_ctime_sec(inode); + buf->bs_ctime_nsec = inode_get_ctime_nsec(inode); + buf->bs_gen = inode->i_generation; + buf->bs_mode = inode->i_mode; + + buf->bs_xflags = xfs_ip2xflags(ip); + buf->bs_extsize_blks = ip->i_extsize; + + nextents = xfs_ifork_nextents(&ip->i_df); + if (!(bc->breq->flags & XFS_IBULK_NREXT64)) + buf->bs_extents = min(nextents, XFS_MAX_EXTCNT_DATA_FORK_SMALL); + else + buf->bs_extents64 = nextents; + + xfs_bulkstat_health(ip, buf); + buf->bs_aextents = xfs_ifork_nextents(&ip->i_af); + buf->bs_forkoff = xfs_inode_fork_boff(ip); + buf->bs_version = XFS_BULKSTAT_VERSION_V5; + + if (xfs_has_v3inodes(mp)) { + buf->bs_btime = ip->i_crtime.tv_sec; + buf->bs_btime_nsec = ip->i_crtime.tv_nsec; + if (ip->i_diflags2 & XFS_DIFLAG2_COWEXTSIZE) + buf->bs_cowextsize_blks = ip->i_cowextsize; + } + + switch (ip->i_df.if_format) { + case XFS_DINODE_FMT_DEV: + buf->bs_rdev = sysv_encode_dev(inode->i_rdev); + buf->bs_blksize = BLKDEV_IOSIZE; + buf->bs_blocks = 0; + break; + case XFS_DINODE_FMT_LOCAL: + buf->bs_rdev = 0; + buf->bs_blksize = mp->m_sb.sb_blocksize; + buf->bs_blocks = 0; + break; + case XFS_DINODE_FMT_EXTENTS: + case XFS_DINODE_FMT_BTREE: + buf->bs_rdev = 0; + buf->bs_blksize = mp->m_sb.sb_blocksize; + buf->bs_blocks = ip->i_nblocks + ip->i_delayed_blks; + break; + } + xfs_iunlock(ip, XFS_ILOCK_SHARED); + xfs_irele(ip); + + error = bc->formatter(bc->breq, buf); + if (error == -ECANCELED) + goto out_advance; + if (error) + goto out; + +out_advance: + /* + * Advance the cursor to the inode that comes after the one we just + * looked at. We want the caller to move along if the bulkstat + * information was copied successfully; if we tried to grab the inode + * but it's no longer allocated; or if it's internal metadata. + */ + bc->breq->startino = ino + 1; +out: + return error; +} + +/* Bulkstat a single inode. */ +int +xfs_bulkstat_one( + struct xfs_ibulk *breq, + bulkstat_one_fmt_pf formatter) +{ + struct xfs_bstat_chunk bc = { + .formatter = formatter, + .breq = breq, + }; + struct xfs_trans *tp; + int error; + + if (breq->idmap != &nop_mnt_idmap) { + xfs_warn_ratelimited(breq->mp, + "bulkstat not supported inside of idmapped mounts."); + return -EINVAL; + } + + ASSERT(breq->icount == 1); + + bc.buf = kzalloc_obj(struct xfs_bulkstat, + GFP_KERNEL | __GFP_RETRY_MAYFAIL); + if (!bc.buf) + return -ENOMEM; + + /* + * Grab an empty transaction so that we can use its recursive buffer + * locking abilities to detect cycles in the inobt without deadlocking. + */ + tp = xfs_trans_alloc_empty(breq->mp); + error = xfs_bulkstat_one_int(breq->mp, breq->idmap, tp, + breq->startino, &bc); + xfs_trans_cancel(tp); + kfree(bc.buf); + + /* + * If we reported one inode to userspace then we abort because we hit + * the end of the buffer. Don't leak that back to userspace. + */ + if (error == -ECANCELED) + error = 0; + + return error; +} + +static int +xfs_bulkstat_iwalk( + struct xfs_mount *mp, + struct xfs_trans *tp, + xfs_ino_t ino, + void *data) +{ + struct xfs_bstat_chunk *bc = data; + int error; + + error = xfs_bulkstat_one_int(mp, bc->breq->idmap, tp, ino, data); + /* bulkstat just skips over missing inodes */ + if (error == -ENOENT || error == -EINVAL) + return 0; + return error; +} + +/* + * Check the incoming lastino parameter. + * + * We allow any inode value that could map to physical space inside the + * filesystem because if there are no inodes there, bulkstat moves on to the + * next chunk. In other words, the magic agino value of zero takes us to the + * first chunk in the AG, and an agino value past the end of the AG takes us to + * the first chunk in the next AG. + * + * Therefore we can end early if the requested inode is beyond the end of the + * filesystem or doesn't map properly. + */ +static inline bool +xfs_bulkstat_already_done( + struct xfs_mount *mp, + xfs_ino_t startino) +{ + xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, startino); + xfs_agino_t agino = XFS_INO_TO_AGINO(mp, startino); + + return agno >= mp->m_sb.sb_agcount || + startino != XFS_AGINO_TO_INO(mp, agno, agino); +} + +/* Return stat information in bulk (by-inode) for the filesystem. */ +int +xfs_bulkstat( + struct xfs_ibulk *breq, + bulkstat_one_fmt_pf formatter) +{ + struct xfs_bstat_chunk bc = { + .formatter = formatter, + .breq = breq, + }; + struct xfs_trans *tp; + int error; + + if (breq->idmap != &nop_mnt_idmap) { + xfs_warn_ratelimited(breq->mp, + "bulkstat not supported inside of idmapped mounts."); + return -EINVAL; + } + if (xfs_bulkstat_already_done(breq->mp, breq->startino)) + return 0; + + bc.buf = kzalloc_obj(struct xfs_bulkstat, + GFP_KERNEL | __GFP_RETRY_MAYFAIL); + if (!bc.buf) + return -ENOMEM; + + /* + * Grab an empty transaction so that we can use its recursive buffer + * locking abilities to detect cycles in the inobt without deadlocking. + */ + tp = xfs_trans_alloc_empty(breq->mp); + error = xfs_iwalk(breq->mp, tp, breq->startino, breq->iwalk_flags, + xfs_bulkstat_iwalk, breq->icount, &bc); + xfs_trans_cancel(tp); + kfree(bc.buf); + + /* + * We found some inodes, so clear the error status and return them. + * The lastino pointer will point directly at the inode that triggered + * any error that occurred, so on the next call the error will be + * triggered again and propagated to userspace as there will be no + * formatted inodes in the buffer. + */ + if (breq->ocount > 0) + error = 0; + + return error; +} + +/* Convert bulkstat (v5) to bstat (v1). */ +void +xfs_bulkstat_to_bstat( + struct xfs_mount *mp, + struct xfs_bstat *bs1, + const struct xfs_bulkstat *bstat) +{ + /* memset is needed here because of padding holes in the structure. */ + memset(bs1, 0, sizeof(struct xfs_bstat)); + bs1->bs_ino = bstat->bs_ino; + bs1->bs_mode = bstat->bs_mode; + bs1->bs_nlink = bstat->bs_nlink; + bs1->bs_uid = bstat->bs_uid; + bs1->bs_gid = bstat->bs_gid; + bs1->bs_rdev = bstat->bs_rdev; + bs1->bs_blksize = bstat->bs_blksize; + bs1->bs_size = bstat->bs_size; + bs1->bs_atime.tv_sec = bstat->bs_atime; + bs1->bs_mtime.tv_sec = bstat->bs_mtime; + bs1->bs_ctime.tv_sec = bstat->bs_ctime; + bs1->bs_atime.tv_nsec = bstat->bs_atime_nsec; + bs1->bs_mtime.tv_nsec = bstat->bs_mtime_nsec; + bs1->bs_ctime.tv_nsec = bstat->bs_ctime_nsec; + bs1->bs_blocks = bstat->bs_blocks; + bs1->bs_xflags = bstat->bs_xflags; + bs1->bs_extsize = XFS_FSB_TO_B(mp, bstat->bs_extsize_blks); + bs1->bs_extents = bstat->bs_extents; + bs1->bs_gen = bstat->bs_gen; + bs1->bs_projid_lo = bstat->bs_projectid & 0xFFFF; + bs1->bs_forkoff = bstat->bs_forkoff; + bs1->bs_projid_hi = bstat->bs_projectid >> 16; + bs1->bs_sick = bstat->bs_sick; + bs1->bs_checked = bstat->bs_checked; + bs1->bs_cowextsize = XFS_FSB_TO_B(mp, bstat->bs_cowextsize_blks); + bs1->bs_dmevmask = 0; + bs1->bs_dmstate = 0; + bs1->bs_aextents = bstat->bs_aextents; +} + +struct xfs_inumbers_chunk { + inumbers_fmt_pf formatter; + struct xfs_ibulk *breq; +}; + +/* + * INUMBERS + * ======== + * This is how we export inode btree records to userspace, so that XFS tools + * can figure out where inodes are allocated. + */ + +/* + * Format the inode group structure and report it somewhere. + * + * Similar to xfs_bulkstat_one_int, lastino is the inode cursor as we walk + * through the filesystem so we move it forward unless there was a runtime + * error. If the formatter tells us the buffer is now full we also move the + * cursor forward and abort the walk. + */ +STATIC int +xfs_inumbers_walk( + struct xfs_mount *mp, + struct xfs_trans *tp, + xfs_agnumber_t agno, + const struct xfs_inobt_rec_incore *irec, + void *data) +{ + struct xfs_inumbers inogrp = { + .xi_startino = XFS_AGINO_TO_INO(mp, agno, irec->ir_startino), + .xi_alloccount = irec->ir_count - irec->ir_freecount, + .xi_allocmask = ~irec->ir_free, + .xi_version = XFS_INUMBERS_VERSION_V5, + }; + struct xfs_inumbers_chunk *ic = data; + int error; + + error = ic->formatter(ic->breq, &inogrp); + if (error && error != -ECANCELED) + return error; + + ic->breq->startino = XFS_AGINO_TO_INO(mp, agno, irec->ir_startino) + + XFS_INODES_PER_CHUNK; + return error; +} + +/* + * Return inode number table for the filesystem. + */ +int +xfs_inumbers( + struct xfs_ibulk *breq, + inumbers_fmt_pf formatter) +{ + struct xfs_inumbers_chunk ic = { + .formatter = formatter, + .breq = breq, + }; + struct xfs_trans *tp; + int error = 0; + + if (xfs_bulkstat_already_done(breq->mp, breq->startino)) + return 0; + + /* + * Grab an empty transaction so that we can use its recursive buffer + * locking abilities to detect cycles in the inobt without deadlocking. + */ + tp = xfs_trans_alloc_empty(breq->mp); + error = xfs_inobt_walk(breq->mp, tp, breq->startino, breq->iwalk_flags, + xfs_inumbers_walk, breq->icount, &ic); + xfs_trans_cancel(tp); + + /* + * We found some inode groups, so clear the error status and return + * them. The lastino pointer will point directly at the inode that + * triggered any error that occurred, so on the next call the error + * will be triggered again and propagated to userspace as there will be + * no formatted inode groups in the buffer. + */ + if (breq->ocount > 0) + error = 0; + + return error; +} + +/* Convert an inumbers (v5) struct to a inogrp (v1) struct. */ +void +xfs_inumbers_to_inogrp( + struct xfs_inogrp *ig1, + const struct xfs_inumbers *ig) +{ + /* memset is needed here because of padding holes in the structure. */ + memset(ig1, 0, sizeof(struct xfs_inogrp)); + ig1->xi_startino = ig->xi_startino; + ig1->xi_alloccount = ig->xi_alloccount; + ig1->xi_allocmask = ig->xi_allocmask; +} diff --git a/fs/xfs/xfs_itable.h b/fs/xfs/xfs_itable.h new file mode 100644 index 000000000..2d0612f14 --- /dev/null +++ b/fs/xfs/xfs_itable.h @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2001 Silicon Graphics, Inc. All Rights Reserved. + */ +#ifndef __XFS_ITABLE_H__ +#define __XFS_ITABLE_H__ + +/* In-memory representation of a userspace request for batch inode data. */ +struct xfs_ibulk { + struct xfs_mount *mp; + struct mnt_idmap *idmap; + void __user *ubuffer; /* user output buffer */ + xfs_ino_t startino; /* start with this inode */ + unsigned int icount; /* number of elements in ubuffer */ + unsigned int ocount; /* number of records returned */ + unsigned int flags; /* XFS_IBULK_FLAG_* */ + unsigned int iwalk_flags; /* XFS_IWALK_FLAG_* */ +}; + +/* Fill out the bs_extents64 field if set. */ +#define XFS_IBULK_NREXT64 (1U << 0) + +/* Signal that we can return metadata directories. */ +#define XFS_IBULK_METADIR (1U << 1) + +/* + * Advance the user buffer pointer by one record of the given size. If the + * buffer is now full, return the appropriate error code. + */ +static inline int +xfs_ibulk_advance( + struct xfs_ibulk *breq, + size_t bytes) +{ + char __user *b = breq->ubuffer; + + breq->ubuffer = b + bytes; + breq->ocount++; + return breq->ocount == breq->icount ? -ECANCELED : 0; +} + +/* + * Return stat information in bulk (by-inode) for the filesystem. + */ + +/* + * Return codes for the formatter function are 0 to continue iterating, and + * non-zero to stop iterating. Any non-zero value will be passed up to the + * bulkstat/inumbers caller. The special value -ECANCELED can be used to stop + * iteration, as neither bulkstat nor inumbers will ever generate that error + * code on their own. + */ + +typedef int (*bulkstat_one_fmt_pf)(struct xfs_ibulk *breq, + const struct xfs_bulkstat *bstat); + +int xfs_bulkstat_one(struct xfs_ibulk *breq, bulkstat_one_fmt_pf formatter); +int xfs_bulkstat(struct xfs_ibulk *breq, bulkstat_one_fmt_pf formatter); +void xfs_bulkstat_to_bstat(struct xfs_mount *mp, struct xfs_bstat *bs1, + const struct xfs_bulkstat *bstat); + +typedef int (*inumbers_fmt_pf)(struct xfs_ibulk *breq, + const struct xfs_inumbers *igrp); + +int xfs_inumbers(struct xfs_ibulk *breq, inumbers_fmt_pf formatter); +void xfs_inumbers_to_inogrp(struct xfs_inogrp *ig1, + const struct xfs_inumbers *ig); + +#endif /* __XFS_ITABLE_H__ */ diff --git a/fs/xfs/xfs_iunlink_item.c b/fs/xfs/xfs_iunlink_item.c new file mode 100644 index 000000000..d76d3f043 --- /dev/null +++ b/fs/xfs/xfs_iunlink_item.c @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020-2022, Red Hat, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_trans_priv.h" +#include "xfs_ag.h" +#include "xfs_iunlink_item.h" +#include "xfs_trace.h" +#include "xfs_error.h" + +struct kmem_cache *xfs_iunlink_cache; + +static inline struct xfs_iunlink_item *IUL_ITEM(struct xfs_log_item *lip) +{ + return container_of(lip, struct xfs_iunlink_item, item); +} + +static void +xfs_iunlink_item_release( + struct xfs_log_item *lip) +{ + struct xfs_iunlink_item *iup = IUL_ITEM(lip); + + xfs_perag_put(iup->pag); + kmem_cache_free(xfs_iunlink_cache, IUL_ITEM(lip)); +} + + +static uint64_t +xfs_iunlink_item_sort( + struct xfs_log_item *lip) +{ + return I_INO(IUL_ITEM(lip)->ip); +} + +/* + * Look up the inode cluster buffer and log the on-disk unlinked inode change + * we need to make. + */ +static int +xfs_iunlink_log_dinode( + struct xfs_trans *tp, + struct xfs_iunlink_item *iup) +{ + struct xfs_inode *ip = iup->ip; + struct xfs_dinode *dip; + struct xfs_buf *ibp; + xfs_agino_t old_ptr; + int offset; + int error; + + error = xfs_read_icluster(iup->pag, tp, ip->i_imap.im_agbno, &ibp); + if (error) + return error; + /* + * Don't log the unlinked field on stale buffers as this may be the + * transaction that frees the inode cluster and relogging the buffer + * here will incorrectly remove the stale state. + */ + if (ibp->b_flags & XBF_STALE) + goto out; + + dip = xfs_buf_offset(ibp, ip->i_imap.im_boffset); + + /* Make sure the old pointer isn't garbage. */ + old_ptr = be32_to_cpu(dip->di_next_unlinked); + if (old_ptr != iup->old_agino) { + xfs_inode_verifier_error(ip, -EFSCORRUPTED, __func__, dip, + sizeof(*dip), __this_address); + error = -EFSCORRUPTED; + goto out; + } + + trace_xfs_iunlink_update_dinode(iup, old_ptr); + + dip->di_next_unlinked = cpu_to_be32(iup->next_agino); + offset = ip->i_imap.im_boffset + + offsetof(struct xfs_dinode, di_next_unlinked); + + xfs_dinode_calc_crc(tp->t_mountp, dip); + xfs_trans_inode_buf(tp, ibp); + xfs_trans_log_buf(tp, ibp, offset, offset + sizeof(xfs_agino_t) - 1); + return 0; +out: + xfs_trans_brelse(tp, ibp); + return error; +} + +/* + * On precommit, we grab the inode cluster buffer for the inode number we were + * passed, then update the next unlinked field for that inode in the buffer and + * log the buffer. This ensures that the inode cluster buffer was logged in the + * correct order w.r.t. other inode cluster buffers. We can then remove the + * iunlink item from the transaction and release it as it is has now served it's + * purpose. + */ +static int +xfs_iunlink_item_precommit( + struct xfs_trans *tp, + struct xfs_log_item *lip) +{ + struct xfs_iunlink_item *iup = IUL_ITEM(lip); + int error; + + error = xfs_iunlink_log_dinode(tp, iup); + list_del(&lip->li_trans); + xfs_iunlink_item_release(lip); + return error; +} + +static const struct xfs_item_ops xfs_iunlink_item_ops = { + .iop_release = xfs_iunlink_item_release, + .iop_sort = xfs_iunlink_item_sort, + .iop_precommit = xfs_iunlink_item_precommit, +}; + + +/* + * Initialize the inode log item for a newly allocated (in-core) inode. + * + * Inode extents can only reside within an AG. Hence specify the starting + * block for the inode chunk by offset within an AG as well as the + * length of the allocated extent. + * + * This joins the item to the transaction and marks it dirty so + * that we don't need a separate call to do this, nor does the + * caller need to know anything about the iunlink item. + */ +int +xfs_iunlink_log_inode( + struct xfs_trans *tp, + struct xfs_inode *ip, + struct xfs_perag *pag, + xfs_agino_t next_agino) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_iunlink_item *iup; + + ASSERT(xfs_verify_agino_or_null(pag, next_agino)); + ASSERT(xfs_verify_agino_or_null(pag, ip->i_next_unlinked)); + + /* + * Since we're updating a linked list, we should never find that the + * current pointer is the same as the new value, unless we're + * terminating the list. + */ + if (ip->i_next_unlinked == next_agino) { + if (next_agino != NULLAGINO) + return -EFSCORRUPTED; + return 0; + } + + iup = kmem_cache_zalloc(xfs_iunlink_cache, GFP_KERNEL | __GFP_NOFAIL); + xfs_log_item_init(mp, &iup->item, XFS_LI_IUNLINK, + &xfs_iunlink_item_ops); + + iup->ip = ip; + iup->next_agino = next_agino; + iup->old_agino = ip->i_next_unlinked; + iup->pag = xfs_perag_hold(pag); + + xfs_trans_add_item(tp, &iup->item); + tp->t_flags |= XFS_TRANS_DIRTY; + set_bit(XFS_LI_DIRTY, &iup->item.li_flags); + return 0; +} + diff --git a/fs/xfs/xfs_iunlink_item.h b/fs/xfs/xfs_iunlink_item.h new file mode 100644 index 000000000..c793cdcac --- /dev/null +++ b/fs/xfs/xfs_iunlink_item.h @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020-2022, Red Hat, Inc. + * All Rights Reserved. + */ +#ifndef XFS_IUNLINK_ITEM_H +#define XFS_IUNLINK_ITEM_H 1 + +struct xfs_trans; +struct xfs_inode; +struct xfs_perag; + +/* in memory log item structure */ +struct xfs_iunlink_item { + struct xfs_log_item item; + struct xfs_inode *ip; + struct xfs_perag *pag; + xfs_agino_t next_agino; + xfs_agino_t old_agino; +}; + +extern struct kmem_cache *xfs_iunlink_cache; + +int xfs_iunlink_log_inode(struct xfs_trans *tp, struct xfs_inode *ip, + struct xfs_perag *pag, xfs_agino_t next_agino); + +#endif /* XFS_IUNLINK_ITEM_H */ diff --git a/fs/xfs/xfs_iwalk.c b/fs/xfs/xfs_iwalk.c new file mode 100644 index 000000000..96a4cb336 --- /dev/null +++ b/fs/xfs/xfs_iwalk.c @@ -0,0 +1,754 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2019 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_btree.h" +#include "xfs_ialloc.h" +#include "xfs_ialloc_btree.h" +#include "xfs_iwalk.h" +#include "xfs_error.h" +#include "xfs_trace.h" +#include "xfs_icache.h" +#include "xfs_health.h" +#include "xfs_trans.h" +#include "xfs_pwork.h" +#include "xfs_ag.h" +#include "xfs_bit.h" + +/* + * Walking Inodes in the Filesystem + * ================================ + * + * This iterator function walks a subset of filesystem inodes in increasing + * order from @startino until there are no more inodes. For each allocated + * inode it finds, it calls a walk function with the relevant inode number and + * a pointer to caller-provided data. The walk function can return the usual + * negative error code to stop the iteration; 0 to continue the iteration; or + * -ECANCELED to stop the iteration. This return value is returned to the + * caller. + * + * Internally, we allow the walk function to do anything, which means that we + * cannot maintain the inobt cursor or our lock on the AGI buffer. We + * therefore cache the inobt records in kernel memory and only call the walk + * function when our memory buffer is full. @nr_recs is the number of records + * that we've cached, and @sz_recs is the size of our cache. + * + * It is the responsibility of the walk function to ensure it accesses + * allocated inodes, as the inobt records may be stale by the time they are + * acted upon. + */ + +struct xfs_iwalk_ag { + /* parallel work control data; will be null if single threaded */ + struct xfs_pwork pwork; + + struct xfs_mount *mp; + struct xfs_trans *tp; + struct xfs_perag *pag; + + /* Where do we start the traversal? */ + xfs_ino_t startino; + + /* What was the last inode number we saw when iterating the inobt? */ + xfs_ino_t lastino; + + /* Array of inobt records we cache. */ + struct xfs_inobt_rec_incore *recs; + + /* Number of entries allocated for the @recs array. */ + unsigned int sz_recs; + + /* Number of entries in the @recs array that are in use. */ + unsigned int nr_recs; + + /* Inode walk function and data pointer. */ + xfs_iwalk_fn iwalk_fn; + xfs_inobt_walk_fn inobt_walk_fn; + void *data; + + /* + * Make it look like the inodes up to startino are free so that + * bulkstat can start its inode iteration at the correct place without + * needing to special case everywhere. + */ + unsigned int trim_start:1; + + /* Skip empty inobt records? */ + unsigned int skip_empty:1; + + /* Drop the (hopefully empty) transaction when calling iwalk_fn. */ + unsigned int drop_trans:1; +}; + +/* + * Loop over all clusters in a chunk for a given incore inode allocation btree + * record. Do a readahead if there are any allocated inodes in that cluster. + */ +STATIC void +xfs_iwalk_ichunk_ra( + struct xfs_mount *mp, + struct xfs_perag *pag, + struct xfs_inobt_rec_incore *irec) +{ + struct xfs_ino_geometry *igeo = M_IGEO(mp); + xfs_agblock_t agbno; + struct blk_plug plug; + int i; /* inode chunk index */ + + agbno = XFS_AGINO_TO_AGBNO(mp, irec->ir_startino); + + blk_start_plug(&plug); + for (i = 0; i < XFS_INODES_PER_CHUNK; i += igeo->inodes_per_cluster) { + xfs_inofree_t imask; + + imask = xfs_inobt_maskn(i, igeo->inodes_per_cluster); + if (imask & ~irec->ir_free) { + xfs_buf_readahead(mp->m_ddev_targp, + xfs_agbno_to_daddr(pag, agbno), + igeo->blocks_per_cluster * mp->m_bsize, + &xfs_inode_buf_ops); + } + agbno += igeo->blocks_per_cluster; + } + blk_finish_plug(&plug); +} + +/* + * Set the bits in @irec's free mask that correspond to the inodes before + * @agino so that we skip them. This is how we restart an inode walk that was + * interrupted in the middle of an inode record. + */ +STATIC void +xfs_iwalk_adjust_start( + xfs_agino_t agino, /* starting inode of chunk */ + struct xfs_inobt_rec_incore *irec) /* btree record */ +{ + int idx; /* index into inode chunk */ + + idx = agino - irec->ir_startino; + + irec->ir_free |= xfs_inobt_maskn(0, idx); + irec->ir_freecount = hweight64(irec->ir_free); +} + +/* Allocate memory for a walk. */ +STATIC int +xfs_iwalk_alloc( + struct xfs_iwalk_ag *iwag) +{ + size_t size; + + ASSERT(iwag->recs == NULL); + iwag->nr_recs = 0; + + /* Allocate a prefetch buffer for inobt records. */ + size = iwag->sz_recs * sizeof(struct xfs_inobt_rec_incore); + iwag->recs = kmalloc(size, GFP_KERNEL | __GFP_RETRY_MAYFAIL); + if (iwag->recs == NULL) + return -ENOMEM; + + return 0; +} + +/* Free memory we allocated for a walk. */ +STATIC void +xfs_iwalk_free( + struct xfs_iwalk_ag *iwag) +{ + kfree(iwag->recs); + iwag->recs = NULL; +} + +/* For each inuse inode in each cached inobt record, call our function. */ +STATIC int +xfs_iwalk_ag_recs( + struct xfs_iwalk_ag *iwag) +{ + struct xfs_mount *mp = iwag->mp; + struct xfs_trans *tp = iwag->tp; + struct xfs_perag *pag = iwag->pag; + unsigned int i, j; + int error; + + for (i = 0; i < iwag->nr_recs; i++) { + struct xfs_inobt_rec_incore *irec = &iwag->recs[i]; + + trace_xfs_iwalk_ag_rec(pag, irec); + + if (xfs_pwork_want_abort(&iwag->pwork)) + return 0; + + if (iwag->inobt_walk_fn) { + error = iwag->inobt_walk_fn(mp, tp, pag_agno(pag), irec, + iwag->data); + if (error) + return error; + } + + if (!iwag->iwalk_fn) + continue; + + for (j = 0; j < XFS_INODES_PER_CHUNK; j++) { + if (xfs_pwork_want_abort(&iwag->pwork)) + return 0; + + /* Skip if this inode is free */ + if (XFS_INOBT_MASK(j) & irec->ir_free) + continue; + + /* Otherwise call our function. */ + error = iwag->iwalk_fn(mp, tp, + xfs_agino_to_ino(pag, + irec->ir_startino + j), + iwag->data); + if (error) + return error; + } + } + + return 0; +} + +/* Delete cursor and let go of AGI. */ +static inline void +xfs_iwalk_del_inobt( + struct xfs_trans *tp, + struct xfs_btree_cur **curpp, + struct xfs_buf **agi_bpp, + int error) +{ + if (*curpp) { + xfs_btree_del_cursor(*curpp, error); + *curpp = NULL; + } + if (*agi_bpp) { + xfs_trans_brelse(tp, *agi_bpp); + *agi_bpp = NULL; + } +} + +/* + * Set ourselves up for walking inobt records starting from a given point in + * the filesystem. + * + * If caller passed in a nonzero start inode number, load the record from the + * inobt and make the record look like all the inodes before agino are free so + * that we skip them, and then move the cursor to the next inobt record. This + * is how we support starting an iwalk in the middle of an inode chunk. + * + * If the caller passed in a start number of zero, move the cursor to the first + * inobt record. + * + * The caller is responsible for cleaning up the cursor and buffer pointer + * regardless of the error status. + */ +STATIC int +xfs_iwalk_ag_start( + struct xfs_iwalk_ag *iwag, + xfs_agino_t agino, + struct xfs_btree_cur **curpp, + struct xfs_buf **agi_bpp, + int *has_more) +{ + struct xfs_mount *mp = iwag->mp; + struct xfs_trans *tp = iwag->tp; + struct xfs_perag *pag = iwag->pag; + struct xfs_inobt_rec_incore *irec; + int error; + + /* Set up a fresh cursor and empty the inobt cache. */ + iwag->nr_recs = 0; + error = xfs_ialloc_read_agi(pag, tp, 0, agi_bpp); + if (error) + return error; + *curpp = xfs_inobt_init_cursor(pag, tp, *agi_bpp); + + /* Starting at the beginning of the AG? That's easy! */ + if (agino == 0) + return xfs_inobt_lookup(*curpp, 0, XFS_LOOKUP_GE, has_more); + + /* + * Otherwise, we have to grab the inobt record where we left off, stuff + * the record into our cache, and then see if there are more records. + * We require a lookup cache of at least two elements so that the + * caller doesn't have to deal with tearing down the cursor to walk the + * records. + */ + error = xfs_inobt_lookup(*curpp, agino, XFS_LOOKUP_LE, has_more); + if (error) + return error; + + /* + * If the LE lookup at @agino yields no records, jump ahead to the + * inobt cursor increment to see if there are more records to process. + */ + if (!*has_more) + goto out_advance; + + /* Get the record, should always work */ + irec = &iwag->recs[iwag->nr_recs]; + error = xfs_inobt_get_rec(*curpp, irec, has_more); + if (error) + return error; + if (XFS_IS_CORRUPT(mp, *has_more != 1)) { + xfs_btree_mark_sick(*curpp); + return -EFSCORRUPTED; + } + + iwag->lastino = xfs_agino_to_ino(pag, + irec->ir_startino + XFS_INODES_PER_CHUNK - 1); + + /* + * If the LE lookup yielded an inobt record before the cursor position, + * skip it and see if there's another one after it. + */ + if (irec->ir_startino + XFS_INODES_PER_CHUNK <= agino) + goto out_advance; + + /* + * If agino fell in the middle of the inode record, make it look like + * the inodes up to agino are free so that we don't return them again. + */ + if (iwag->trim_start) + xfs_iwalk_adjust_start(agino, irec); + + /* + * The prefetch calculation is supposed to give us a large enough inobt + * record cache that grab_ichunk can stage a partial first record and + * the loop body can cache a record without having to check for cache + * space until after it reads an inobt record. + */ + iwag->nr_recs++; + ASSERT(iwag->nr_recs < iwag->sz_recs); + +out_advance: + return xfs_btree_increment(*curpp, 0, has_more); +} + +/* + * The inobt record cache is full, so preserve the inobt cursor state and + * run callbacks on the cached inobt records. When we're done, restore the + * cursor state to wherever the cursor would have been had the cache not been + * full (and therefore we could've just incremented the cursor) if *@has_more + * is true. On exit, *@has_more will indicate whether or not the caller should + * try for more inode records. + */ +STATIC int +xfs_iwalk_run_callbacks( + struct xfs_iwalk_ag *iwag, + struct xfs_btree_cur **curpp, + struct xfs_buf **agi_bpp, + int *has_more) +{ + struct xfs_mount *mp = iwag->mp; + xfs_agino_t next_agino; + int error; + + next_agino = XFS_INO_TO_AGINO(mp, iwag->lastino) + 1; + + ASSERT(iwag->nr_recs > 0); + + /* Delete cursor but remember the last record we cached... */ + xfs_iwalk_del_inobt(iwag->tp, curpp, agi_bpp, 0); + ASSERT(next_agino >= iwag->recs[iwag->nr_recs - 1].ir_startino + + XFS_INODES_PER_CHUNK); + + if (iwag->drop_trans) { + xfs_trans_cancel(iwag->tp); + iwag->tp = NULL; + } + + error = xfs_iwalk_ag_recs(iwag); + if (error) + return error; + + /* ...empty the cache... */ + iwag->nr_recs = 0; + + if (!has_more) + return 0; + + if (iwag->drop_trans) + iwag->tp = xfs_trans_alloc_empty(mp); + + /* ...and recreate the cursor just past where we left off. */ + error = xfs_ialloc_read_agi(iwag->pag, iwag->tp, 0, agi_bpp); + if (error) + return error; + *curpp = xfs_inobt_init_cursor(iwag->pag, iwag->tp, *agi_bpp); + return xfs_inobt_lookup(*curpp, next_agino, XFS_LOOKUP_GE, has_more); +} + +/* Walk all inodes in a single AG, from @iwag->startino to the end of the AG. */ +STATIC int +xfs_iwalk_ag( + struct xfs_iwalk_ag *iwag) +{ + struct xfs_mount *mp = iwag->mp; + struct xfs_perag *pag = iwag->pag; + struct xfs_buf *agi_bp = NULL; + struct xfs_btree_cur *cur = NULL; + xfs_agino_t agino; + int has_more; + int error = 0; + + /* Set up our cursor at the right place in the inode btree. */ + ASSERT(pag_agno(pag) == XFS_INO_TO_AGNO(mp, iwag->startino)); + agino = XFS_INO_TO_AGINO(mp, iwag->startino); + error = xfs_iwalk_ag_start(iwag, agino, &cur, &agi_bp, &has_more); + + while (!error && has_more) { + struct xfs_inobt_rec_incore *irec; + xfs_ino_t rec_fsino; + + cond_resched(); + if (xfs_pwork_want_abort(&iwag->pwork)) + goto out; + + /* Fetch the inobt record. */ + irec = &iwag->recs[iwag->nr_recs]; + error = xfs_inobt_get_rec(cur, irec, &has_more); + if (error || !has_more) + break; + + /* Make sure that we always move forward. */ + rec_fsino = xfs_agino_to_ino(pag, irec->ir_startino); + if (iwag->lastino != NULLFSINO && + XFS_IS_CORRUPT(mp, iwag->lastino >= rec_fsino)) { + xfs_btree_mark_sick(cur); + error = -EFSCORRUPTED; + goto out; + } + iwag->lastino = rec_fsino + XFS_INODES_PER_CHUNK - 1; + + /* No allocated inodes in this chunk; skip it. */ + if (iwag->skip_empty && irec->ir_freecount == irec->ir_count) { + error = xfs_btree_increment(cur, 0, &has_more); + if (error) + break; + continue; + } + + /* + * Start readahead for this inode chunk in anticipation of + * walking the inodes. + */ + if (iwag->iwalk_fn) + xfs_iwalk_ichunk_ra(mp, pag, irec); + + /* + * If there's space in the buffer for more records, increment + * the btree cursor and grab more. + */ + if (++iwag->nr_recs < iwag->sz_recs) { + error = xfs_btree_increment(cur, 0, &has_more); + if (error || !has_more) + break; + continue; + } + + /* + * Otherwise, we need to save cursor state and run the callback + * function on the cached records. The run_callbacks function + * is supposed to return a cursor pointing to the record where + * we would be if we had been able to increment like above. + */ + ASSERT(has_more); + error = xfs_iwalk_run_callbacks(iwag, &cur, &agi_bp, &has_more); + } + + if (iwag->nr_recs == 0 || error) + goto out; + + /* Walk the unprocessed records in the cache. */ + error = xfs_iwalk_run_callbacks(iwag, &cur, &agi_bp, &has_more); + +out: + xfs_iwalk_del_inobt(iwag->tp, &cur, &agi_bp, error); + return error; +} + +/* + * We experimentally determined that the reduction in ioctl call overhead + * diminishes when userspace asks for more than 2048 inodes, so we'll cap + * prefetch at this point. + */ +#define IWALK_MAX_INODE_PREFETCH (2048U) + +/* + * Given the number of inodes to prefetch, set the number of inobt records that + * we cache in memory, which controls the number of inodes we try to read + * ahead. Set the maximum if @inodes == 0. + */ +static inline unsigned int +xfs_iwalk_prefetch( + unsigned int inodes) +{ + unsigned int inobt_records; + + /* + * If the caller didn't tell us the number of inodes they wanted, + * assume the maximum prefetch possible for best performance. + * Otherwise, cap prefetch at that maximum so that we don't start an + * absurd amount of prefetch. + */ + if (inodes == 0) + inodes = IWALK_MAX_INODE_PREFETCH; + inodes = min(inodes, IWALK_MAX_INODE_PREFETCH); + + /* Round the inode count up to a full chunk. */ + inodes = round_up(inodes, XFS_INODES_PER_CHUNK); + + /* + * In order to convert the number of inodes to prefetch into an + * estimate of the number of inobt records to cache, we require a + * conversion factor that reflects our expectations of the average + * loading factor of an inode chunk. Based on data gathered, most + * (but not all) filesystems manage to keep the inode chunks totally + * full, so we'll underestimate slightly so that our readahead will + * still deliver the performance we want on aging filesystems: + * + * inobt = inodes / (INODES_PER_CHUNK * (4 / 5)); + * + * The funny math is to avoid integer division. + */ + inobt_records = (inodes * 5) / (4 * XFS_INODES_PER_CHUNK); + + /* + * Allocate enough space to prefetch at least two inobt records so that + * we can cache both the record where the iwalk started and the next + * record. This simplifies the AG inode walk loop setup code. + */ + return max(inobt_records, 2U); +} + +static int +xfs_iwalk_args( + struct xfs_iwalk_ag *iwag, + unsigned int flags) +{ + struct xfs_mount *mp = iwag->mp; + xfs_agnumber_t start_agno; + int error; + + start_agno = XFS_INO_TO_AGNO(iwag->mp, iwag->startino); + ASSERT(start_agno < iwag->mp->m_sb.sb_agcount); + ASSERT(!(flags & ~XFS_IWALK_FLAGS_ALL)); + + error = xfs_iwalk_alloc(iwag); + if (error) + return error; + + while ((iwag->pag = xfs_perag_next_from(mp, iwag->pag, start_agno))) { + error = xfs_iwalk_ag(iwag); + if (error || (flags & XFS_IWALK_SAME_AG)) { + xfs_perag_rele(iwag->pag); + break; + } + iwag->startino = + XFS_AGINO_TO_INO(mp, pag_agno(iwag->pag) + 1, 0); + } + + xfs_iwalk_free(iwag); + return error; +} + +/* + * Walk all inodes in the filesystem starting from @startino. The @iwalk_fn + * will be called for each allocated inode, being passed the inode's number and + * @data. @max_prefetch controls how many inobt records' worth of inodes we + * try to readahead. + */ +int +xfs_iwalk( + struct xfs_mount *mp, + struct xfs_trans *tp, + xfs_ino_t startino, + unsigned int flags, + xfs_iwalk_fn iwalk_fn, + unsigned int inode_records, + void *data) +{ + struct xfs_iwalk_ag iwag = { + .mp = mp, + .tp = tp, + .iwalk_fn = iwalk_fn, + .data = data, + .startino = startino, + .sz_recs = xfs_iwalk_prefetch(inode_records), + .trim_start = 1, + .skip_empty = 1, + .pwork = XFS_PWORK_SINGLE_THREADED, + .lastino = NULLFSINO, + }; + + return xfs_iwalk_args(&iwag, flags); +} + +/* Run per-thread iwalk work. */ +static int +xfs_iwalk_ag_work( + struct xfs_mount *mp, + struct xfs_pwork *pwork) +{ + struct xfs_iwalk_ag *iwag; + int error = 0; + + iwag = container_of(pwork, struct xfs_iwalk_ag, pwork); + if (xfs_pwork_want_abort(pwork)) + goto out; + + error = xfs_iwalk_alloc(iwag); + if (error) + goto out; + /* + * Grab an empty transaction so that we can use its recursive buffer + * locking abilities to detect cycles in the inobt without deadlocking. + */ + iwag->tp = xfs_trans_alloc_empty(mp); + iwag->drop_trans = 1; + + error = xfs_iwalk_ag(iwag); + if (iwag->tp) + xfs_trans_cancel(iwag->tp); + xfs_iwalk_free(iwag); +out: + xfs_perag_put(iwag->pag); + kfree(iwag); + return error; +} + +/* + * Walk all the inodes in the filesystem using multiple threads to process each + * AG. + */ +int +xfs_iwalk_threaded( + struct xfs_mount *mp, + xfs_ino_t startino, + unsigned int flags, + xfs_iwalk_fn iwalk_fn, + unsigned int inode_records, + bool polled, + void *data) +{ + xfs_agnumber_t start_agno = XFS_INO_TO_AGNO(mp, startino); + struct xfs_pwork_ctl pctl; + struct xfs_perag *pag = NULL; + int error; + + ASSERT(start_agno < mp->m_sb.sb_agcount); + ASSERT(!(flags & ~XFS_IWALK_FLAGS_ALL)); + + error = xfs_pwork_init(mp, &pctl, xfs_iwalk_ag_work, "xfs_iwalk"); + if (error) + return error; + + while ((pag = xfs_perag_next_from(mp, pag, start_agno))) { + struct xfs_iwalk_ag *iwag; + + if (xfs_pwork_ctl_want_abort(&pctl)) + break; + + iwag = kzalloc_obj(struct xfs_iwalk_ag, + GFP_KERNEL | __GFP_NOFAIL); + iwag->mp = mp; + + /* + * perag is being handed off to async work, so take a passive + * reference for the async work to release. + */ + iwag->pag = xfs_perag_hold(pag); + iwag->iwalk_fn = iwalk_fn; + iwag->data = data; + iwag->startino = startino; + iwag->sz_recs = xfs_iwalk_prefetch(inode_records); + iwag->lastino = NULLFSINO; + xfs_pwork_queue(&pctl, &iwag->pwork); + startino = XFS_AGINO_TO_INO(mp, pag_agno(pag) + 1, 0); + if (flags & XFS_IWALK_SAME_AG) + break; + } + if (pag) + xfs_perag_rele(pag); + if (polled) + xfs_pwork_poll(&pctl); + return xfs_pwork_destroy(&pctl); +} + +/* + * Allow callers to cache up to a page's worth of inobt records. This reflects + * the existing inumbers prefetching behavior. Since the inobt walk does not + * itself do anything with the inobt records, we can set a fairly high limit + * here. + */ +#define MAX_INOBT_WALK_PREFETCH \ + (PAGE_SIZE / sizeof(struct xfs_inobt_rec_incore)) + +/* + * Given the number of records that the user wanted, set the number of inobt + * records that we buffer in memory. Set the maximum if @inobt_records == 0. + */ +static inline unsigned int +xfs_inobt_walk_prefetch( + unsigned int inobt_records) +{ + /* + * If the caller didn't tell us the number of inobt records they + * wanted, assume the maximum prefetch possible for best performance. + */ + if (inobt_records == 0) + inobt_records = MAX_INOBT_WALK_PREFETCH; + + /* + * Allocate enough space to prefetch at least two inobt records so that + * we can cache both the record where the iwalk started and the next + * record. This simplifies the AG inode walk loop setup code. + */ + inobt_records = max(inobt_records, 2U); + + /* + * Cap prefetch at that maximum so that we don't use an absurd amount + * of memory. + */ + return min_t(unsigned int, inobt_records, MAX_INOBT_WALK_PREFETCH); +} + +/* + * Walk all inode btree records in the filesystem starting from @startino. The + * @inobt_walk_fn will be called for each btree record, being passed the incore + * record and @data. @max_prefetch controls how many inobt records we try to + * cache ahead of time. + */ +int +xfs_inobt_walk( + struct xfs_mount *mp, + struct xfs_trans *tp, + xfs_ino_t startino, + unsigned int flags, + xfs_inobt_walk_fn inobt_walk_fn, + unsigned int inobt_records, + void *data) +{ + struct xfs_iwalk_ag iwag = { + .mp = mp, + .tp = tp, + .inobt_walk_fn = inobt_walk_fn, + .data = data, + .startino = startino, + .sz_recs = xfs_inobt_walk_prefetch(inobt_records), + .pwork = XFS_PWORK_SINGLE_THREADED, + .lastino = NULLFSINO, + }; + + return xfs_iwalk_args(&iwag, flags); +} diff --git a/fs/xfs/xfs_iwalk.h b/fs/xfs/xfs_iwalk.h new file mode 100644 index 000000000..17a5a2c6d --- /dev/null +++ b/fs/xfs/xfs_iwalk.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2019 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_IWALK_H__ +#define __XFS_IWALK_H__ + +/* + * Return codes for the inode/inobt walk function are 0 to continue iterating, + * and non-zero to stop iterating. Any non-zero value will be passed up to the + * iwalk or inobt_walk caller. The special value -ECANCELED can be used to + * stop iteration, as neither iwalk nor inobt_walk will ever generate that + * error code on their own. + */ + +/* Walk all inodes in the filesystem starting from @startino. */ +typedef int (*xfs_iwalk_fn)(struct xfs_mount *mp, struct xfs_trans *tp, + xfs_ino_t ino, void *data); + +int xfs_iwalk(struct xfs_mount *mp, struct xfs_trans *tp, xfs_ino_t startino, + unsigned int flags, xfs_iwalk_fn iwalk_fn, + unsigned int inode_records, void *data); +int xfs_iwalk_threaded(struct xfs_mount *mp, xfs_ino_t startino, + unsigned int flags, xfs_iwalk_fn iwalk_fn, + unsigned int inode_records, bool poll, void *data); + +/* Only iterate within the same AG as @startino. */ +#define XFS_IWALK_SAME_AG (1U << 0) + +#define XFS_IWALK_FLAGS_ALL (XFS_IWALK_SAME_AG) + +/* Walk all inode btree records in the filesystem starting from @startino. */ +typedef int (*xfs_inobt_walk_fn)(struct xfs_mount *mp, struct xfs_trans *tp, + xfs_agnumber_t agno, + const struct xfs_inobt_rec_incore *irec, + void *data); + +int xfs_inobt_walk(struct xfs_mount *mp, struct xfs_trans *tp, + xfs_ino_t startino, unsigned int flags, + xfs_inobt_walk_fn inobt_walk_fn, unsigned int inobt_records, + void *data); + +#endif /* __XFS_IWALK_H__ */ diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c new file mode 100644 index 000000000..f807f8f4f --- /dev/null +++ b/fs/xfs/xfs_log.c @@ -0,0 +1,3477 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_errortag.h" +#include "xfs_error.h" +#include "xfs_trans.h" +#include "xfs_trans_priv.h" +#include "xfs_log.h" +#include "xfs_log_priv.h" +#include "xfs_trace.h" +#include "xfs_sysfs.h" +#include "xfs_sb.h" +#include "xfs_health.h" +#include "xfs_zone_alloc.h" + +struct xlog_write_data { + struct xlog_ticket *ticket; + struct xlog_in_core *iclog; + uint32_t bytes_left; + uint32_t record_cnt; + uint32_t data_cnt; + int log_offset; +}; + +struct kmem_cache *xfs_log_ticket_cache; + +/* Local miscellaneous function prototypes */ +STATIC struct xlog * +xlog_alloc_log( + struct xfs_mount *mp, + struct xfs_buftarg *log_target, + xfs_daddr_t blk_offset, + int num_bblks); +STATIC void +xlog_dealloc_log( + struct xlog *log); + +/* local state machine functions */ +STATIC void xlog_state_done_syncing( + struct xlog_in_core *iclog); +STATIC void xlog_state_do_callback( + struct xlog *log); +STATIC int +xlog_state_get_iclog_space( + struct xlog *log, + struct xlog_write_data *data); +STATIC void +xlog_sync( + struct xlog *log, + struct xlog_in_core *iclog, + struct xlog_ticket *ticket); +#if defined(DEBUG) +STATIC void +xlog_verify_iclog( + struct xlog *log, + struct xlog_in_core *iclog, + int count); +STATIC void +xlog_verify_tail_lsn( + struct xlog *log, + struct xlog_in_core *iclog); +#else +#define xlog_verify_iclog(a,b,c) +#define xlog_verify_tail_lsn(a,b) +#endif + +STATIC int +xlog_iclogs_empty( + struct xlog *log); + +static int +xfs_log_cover(struct xfs_mount *); + +static inline void +xlog_grant_sub_space( + struct xlog_grant_head *head, + int64_t bytes) +{ + atomic64_sub(bytes, &head->grant); +} + +static inline void +xlog_grant_add_space( + struct xlog_grant_head *head, + int64_t bytes) +{ + atomic64_add(bytes, &head->grant); +} + +static void +xlog_grant_head_init( + struct xlog_grant_head *head) +{ + atomic64_set(&head->grant, 0); + INIT_LIST_HEAD(&head->waiters); + spin_lock_init(&head->lock); +} + +void +xlog_grant_return_space( + struct xlog *log, + xfs_lsn_t old_head, + xfs_lsn_t new_head) +{ + int64_t diff = xlog_lsn_sub(log, new_head, old_head); + + xlog_grant_sub_space(&log->l_reserve_head, diff); + xlog_grant_sub_space(&log->l_write_head, diff); +} + +/* + * Return the space in the log between the tail and the head. In the case where + * we have overrun available reservation space, return 0. The memory barrier + * pairs with the smp_wmb() in xlog_cil_ail_insert() to ensure that grant head + * vs tail space updates are seen in the correct order and hence avoid + * transients as space is transferred from the grant heads to the AIL on commit + * completion. + */ +static uint64_t +xlog_grant_space_left( + struct xlog *log, + struct xlog_grant_head *head) +{ + int64_t free_bytes; + + smp_rmb(); /* paired with smp_wmb in xlog_cil_ail_insert() */ + free_bytes = log->l_logsize - READ_ONCE(log->l_tail_space) - + atomic64_read(&head->grant); + if (free_bytes > 0) + return free_bytes; + return 0; +} + +STATIC void +xlog_grant_head_wake_all( + struct xlog_grant_head *head) +{ + struct xlog_ticket *tic; + + spin_lock(&head->lock); + list_for_each_entry(tic, &head->waiters, t_queue) + wake_up_process(tic->t_task); + spin_unlock(&head->lock); +} + +static inline int +xlog_ticket_reservation( + struct xlog *log, + struct xlog_grant_head *head, + struct xlog_ticket *tic) +{ + if (head == &log->l_write_head) { + ASSERT(tic->t_flags & XLOG_TIC_PERM_RESERV); + return tic->t_unit_res; + } + + if (tic->t_flags & XLOG_TIC_PERM_RESERV) + return tic->t_unit_res * tic->t_cnt; + + return tic->t_unit_res; +} + +STATIC bool +xlog_grant_head_wake( + struct xlog *log, + struct xlog_grant_head *head, + int *free_bytes) +{ + struct xlog_ticket *tic; + int need_bytes; + + list_for_each_entry(tic, &head->waiters, t_queue) { + need_bytes = xlog_ticket_reservation(log, head, tic); + if (*free_bytes < need_bytes) + return false; + + *free_bytes -= need_bytes; + trace_xfs_log_grant_wake_up(log, tic); + wake_up_process(tic->t_task); + } + + return true; +} + +STATIC int +xlog_grant_head_wait( + struct xlog *log, + struct xlog_grant_head *head, + struct xlog_ticket *tic, + int need_bytes) __releases(&head->lock) + __acquires(&head->lock) +{ + list_add_tail(&tic->t_queue, &head->waiters); + + do { + if (xlog_is_shutdown(log)) + goto shutdown; + + __set_current_state(TASK_UNINTERRUPTIBLE); + spin_unlock(&head->lock); + + XFS_STATS_INC(log->l_mp, xs_sleep_logspace); + + /* Push on the AIL to free up all the log space. */ + xfs_ail_push_all(log->l_ailp); + + trace_xfs_log_grant_sleep(log, tic); + schedule(); + trace_xfs_log_grant_wake(log, tic); + + spin_lock(&head->lock); + if (xlog_is_shutdown(log)) + goto shutdown; + } while (xlog_grant_space_left(log, head) < need_bytes); + + list_del_init(&tic->t_queue); + return 0; +shutdown: + list_del_init(&tic->t_queue); + return -EIO; +} + +/* + * Atomically get the log space required for a log ticket. + * + * Once a ticket gets put onto head->waiters, it will only return after the + * needed reservation is satisfied. + * + * This function is structured so that it has a lock free fast path. This is + * necessary because every new transaction reservation will come through this + * path. Hence any lock will be globally hot if we take it unconditionally on + * every pass. + * + * As tickets are only ever moved on and off head->waiters under head->lock, we + * only need to take that lock if we are going to add the ticket to the queue + * and sleep. We can avoid taking the lock if the ticket was never added to + * head->waiters because the t_queue list head will be empty and we hold the + * only reference to it so it can safely be checked unlocked. + */ +STATIC int +xlog_grant_head_check( + struct xlog *log, + struct xlog_grant_head *head, + struct xlog_ticket *tic, + int *need_bytes) +{ + int free_bytes; + int error = 0; + + ASSERT(!xlog_in_recovery(log)); + + /* + * If there are other waiters on the queue then give them a chance at + * logspace before us. Wake up the first waiters, if we do not wake + * up all the waiters then go to sleep waiting for more free space, + * otherwise try to get some space for this transaction. + */ + *need_bytes = xlog_ticket_reservation(log, head, tic); + free_bytes = xlog_grant_space_left(log, head); + if (!list_empty_careful(&head->waiters)) { + spin_lock(&head->lock); + if (!xlog_grant_head_wake(log, head, &free_bytes) || + free_bytes < *need_bytes) { + error = xlog_grant_head_wait(log, head, tic, + *need_bytes); + } + spin_unlock(&head->lock); + } else if (free_bytes < *need_bytes) { + spin_lock(&head->lock); + error = xlog_grant_head_wait(log, head, tic, *need_bytes); + spin_unlock(&head->lock); + } + + return error; +} + +bool +xfs_log_writable( + struct xfs_mount *mp) +{ + /* + * Do not write to the log on norecovery mounts, if the data or log + * devices are read-only, or if the filesystem is shutdown. Read-only + * mounts allow internal writes for log recovery and unmount purposes, + * so don't restrict that case. + */ + if (xfs_has_norecovery(mp)) + return false; + if (xfs_readonly_buftarg(mp->m_ddev_targp)) + return false; + if (xfs_readonly_buftarg(mp->m_log->l_targ)) + return false; + if (xlog_is_shutdown(mp->m_log)) + return false; + return true; +} + +/* + * Replenish the byte reservation required by moving the grant write head. + */ +int +xfs_log_regrant( + struct xfs_mount *mp, + struct xlog_ticket *tic) +{ + struct xlog *log = mp->m_log; + int need_bytes; + int error = 0; + + if (xlog_is_shutdown(log)) + return -EIO; + + XFS_STATS_INC(mp, xs_try_logspace); + + /* + * This is a new transaction on the ticket, so we need to change the + * transaction ID so that the next transaction has a different TID in + * the log. Just add one to the existing tid so that we can see chains + * of rolling transactions in the log easily. + */ + tic->t_tid++; + tic->t_curr_res = tic->t_unit_res; + if (tic->t_cnt > 0) + return 0; + + trace_xfs_log_regrant(log, tic); + + error = xlog_grant_head_check(log, &log->l_write_head, tic, + &need_bytes); + if (error) + goto out_error; + + xlog_grant_add_space(&log->l_write_head, need_bytes); + trace_xfs_log_regrant_exit(log, tic); + return 0; + +out_error: + /* + * If we are failing, make sure the ticket doesn't have any current + * reservations. We don't want to add this back when the ticket/ + * transaction gets cancelled. + */ + tic->t_curr_res = 0; + tic->t_cnt = 0; /* ungrant will give back unit_res * t_cnt. */ + return error; +} + +/* + * Reserve log space and return a ticket corresponding to the reservation. + * + * Each reservation is going to reserve extra space for a log record header. + * When writes happen to the on-disk log, we don't subtract the length of the + * log record header from any reservation. By wasting space in each + * reservation, we prevent over allocation problems. + */ +int +xfs_log_reserve( + struct xfs_mount *mp, + int unit_bytes, + int cnt, + struct xlog_ticket **ticp, + bool permanent) +{ + struct xlog *log = mp->m_log; + struct xlog_ticket *tic; + int need_bytes; + int error = 0; + + if (xlog_is_shutdown(log)) + return -EIO; + + XFS_STATS_INC(mp, xs_try_logspace); + + ASSERT(*ticp == NULL); + tic = xlog_ticket_alloc(log, unit_bytes, cnt, permanent); + *ticp = tic; + trace_xfs_log_reserve(log, tic); + error = xlog_grant_head_check(log, &log->l_reserve_head, tic, + &need_bytes); + if (error) + goto out_error; + + xlog_grant_add_space(&log->l_reserve_head, need_bytes); + xlog_grant_add_space(&log->l_write_head, need_bytes); + trace_xfs_log_reserve_exit(log, tic); + return 0; + +out_error: + /* + * If we are failing, make sure the ticket doesn't have any current + * reservations. We don't want to add this back when the ticket/ + * transaction gets cancelled. + */ + tic->t_curr_res = 0; + tic->t_cnt = 0; /* ungrant will give back unit_res * t_cnt. */ + return error; +} + +/* + * Run all the pending iclog callbacks and wake log force waiters and iclog + * space waiters so they can process the newly set shutdown state. We really + * don't care what order we process callbacks here because the log is shut down + * and so state cannot change on disk anymore. However, we cannot wake waiters + * until the callbacks have been processed because we may be in unmount and + * we must ensure that all AIL operations the callbacks perform have completed + * before we tear down the AIL. + * + * We avoid processing actively referenced iclogs so that we don't run callbacks + * while the iclog owner might still be preparing the iclog for IO submssion. + * These will be caught by xlog_state_iclog_release() and call this function + * again to process any callbacks that may have been added to that iclog. + */ +static void +xlog_state_shutdown_callbacks( + struct xlog *log) +{ + struct xlog_in_core *iclog; + LIST_HEAD(cb_list); + + iclog = log->l_iclog; + do { + if (atomic_read(&iclog->ic_refcnt)) { + /* Reference holder will re-run iclog callbacks. */ + continue; + } + list_splice_init(&iclog->ic_callbacks, &cb_list); + spin_unlock(&log->l_icloglock); + + xlog_cil_process_committed(&cb_list); + + spin_lock(&log->l_icloglock); + wake_up_all(&iclog->ic_write_wait); + wake_up_all(&iclog->ic_force_wait); + } while ((iclog = iclog->ic_next) != log->l_iclog); + + wake_up_all(&log->l_flush_wait); +} + +/* + * Flush iclog to disk if this is the last reference to the given iclog and the + * it is in the WANT_SYNC state. + * + * If XLOG_ICL_NEED_FUA is already set on the iclog, we need to ensure that the + * log tail is updated correctly. NEED_FUA indicates that the iclog will be + * written to stable storage, and implies that a commit record is contained + * within the iclog. We need to ensure that the log tail does not move beyond + * the tail that the first commit record in the iclog ordered against, otherwise + * correct recovery of that checkpoint becomes dependent on future operations + * performed on this iclog. + * + * Hence if NEED_FUA is set and the current iclog tail lsn is empty, write the + * current tail into iclog. Once the iclog tail is set, future operations must + * not modify it, otherwise they potentially violate ordering constraints for + * the checkpoint commit that wrote the initial tail lsn value. The tail lsn in + * the iclog will get zeroed on activation of the iclog after sync, so we + * always capture the tail lsn on the iclog on the first NEED_FUA release + * regardless of the number of active reference counts on this iclog. + */ +int +xlog_state_release_iclog( + struct xlog *log, + struct xlog_in_core *iclog, + struct xlog_ticket *ticket) +{ + bool last_ref; + + lockdep_assert_held(&log->l_icloglock); + + trace_xlog_iclog_release(iclog, _RET_IP_); + /* + * Grabbing the current log tail needs to be atomic w.r.t. the writing + * of the tail LSN into the iclog so we guarantee that the log tail does + * not move between the first time we know that the iclog needs to be + * made stable and when we eventually submit it. + */ + if ((iclog->ic_state == XLOG_STATE_WANT_SYNC || + (iclog->ic_flags & XLOG_ICL_NEED_FUA)) && + !iclog->ic_header->h_tail_lsn) { + iclog->ic_header->h_tail_lsn = + cpu_to_be64(atomic64_read(&log->l_tail_lsn)); + } + + last_ref = atomic_dec_and_test(&iclog->ic_refcnt); + + if (xlog_is_shutdown(log)) { + /* + * If there are no more references to this iclog, process the + * pending iclog callbacks that were waiting on the release of + * this iclog. + */ + if (last_ref) + xlog_state_shutdown_callbacks(log); + return -EIO; + } + + if (!last_ref) + return 0; + + if (iclog->ic_state != XLOG_STATE_WANT_SYNC) { + ASSERT(iclog->ic_state == XLOG_STATE_ACTIVE); + return 0; + } + + iclog->ic_state = XLOG_STATE_SYNCING; + xlog_verify_tail_lsn(log, iclog); + trace_xlog_iclog_syncing(iclog, _RET_IP_); + + spin_unlock(&log->l_icloglock); + xlog_sync(log, iclog, ticket); + spin_lock(&log->l_icloglock); + return 0; +} + +/* + * Mount a log filesystem + * + * mp - ubiquitous xfs mount point structure + * log_target - buftarg of on-disk log device + * blk_offset - Start block # where block size is 512 bytes (BBSIZE) + * num_bblocks - Number of BBSIZE blocks in on-disk log + * + * Return error or zero. + */ +int +xfs_log_mount( + xfs_mount_t *mp, + struct xfs_buftarg *log_target, + xfs_daddr_t blk_offset, + int num_bblks) +{ + struct xlog *log; + int error = 0; + int min_logfsbs; + + if (!xfs_has_norecovery(mp)) { + xfs_notice(mp, "Mounting V%d Filesystem %pU", + XFS_SB_VERSION_NUM(&mp->m_sb), + &mp->m_sb.sb_uuid); + } else { + xfs_notice(mp, +"Mounting V%d filesystem %pU in no-recovery mode. Filesystem will be inconsistent.", + XFS_SB_VERSION_NUM(&mp->m_sb), + &mp->m_sb.sb_uuid); + ASSERT(xfs_is_readonly(mp)); + } + + log = xlog_alloc_log(mp, log_target, blk_offset, num_bblks); + if (IS_ERR(log)) { + error = PTR_ERR(log); + goto out; + } + mp->m_log = log; + + /* + * Now that we have set up the log and it's internal geometry + * parameters, we can validate the given log space and drop a critical + * message via syslog if the log size is too small. A log that is too + * small can lead to unexpected situations in transaction log space + * reservation stage. The superblock verifier has already validated all + * the other log geometry constraints, so we don't have to check those + * here. + * + * Note: For v4 filesystems, we can't just reject the mount if the + * validation fails. This would mean that people would have to + * downgrade their kernel just to remedy the situation as there is no + * way to grow the log (short of black magic surgery with xfs_db). + * + * We can, however, reject mounts for V5 format filesystems, as the + * mkfs binary being used to make the filesystem should never create a + * filesystem with a log that is too small. + */ + min_logfsbs = xfs_log_calc_minimum_size(mp); + if (mp->m_sb.sb_logblocks < min_logfsbs) { + xfs_warn(mp, + "Log size %d blocks too small, minimum size is %d blocks", + mp->m_sb.sb_logblocks, min_logfsbs); + + /* + * Log check errors are always fatal on v5; or whenever bad + * metadata leads to a crash. + */ + if (xfs_has_crc(mp)) { + xfs_crit(mp, "AAIEEE! Log failed size checks. Abort!"); + ASSERT(0); + error = -EINVAL; + goto out_free_log; + } + xfs_crit(mp, "Log size out of supported range."); + xfs_crit(mp, +"Continuing onwards, but if log hangs are experienced then please report this message in the bug report."); + } + + /* + * Initialize the AIL now we have a log. + */ + error = xfs_trans_ail_init(mp); + if (error) { + xfs_warn(mp, "AIL initialisation failed: error %d", error); + goto out_free_log; + } + log->l_ailp = mp->m_ail; + + /* + * skip log recovery on a norecovery mount. pretend it all + * just worked. + */ + if (!xfs_has_norecovery(mp)) { + error = xlog_recover(log); + if (error) { + xfs_warn(mp, "log mount/recovery failed: error %d", + error); + xlog_recover_cancel(log); + goto out_destroy_ail; + } + } + + error = xfs_sysfs_init(&log->l_kobj, &xfs_log_ktype, &mp->m_kobj, + "log"); + if (error) + goto out_destroy_ail; + + /* Normal transactions can now occur */ + clear_bit(XLOG_ACTIVE_RECOVERY, &log->l_opstate); + + /* + * Now the log has been fully initialised and we know were our + * space grant counters are, we can initialise the permanent ticket + * needed for delayed logging to work. + */ + xlog_cil_init_post_recovery(log); + + return 0; + +out_destroy_ail: + xfs_trans_ail_destroy(mp); +out_free_log: + xlog_dealloc_log(log); +out: + return error; +} + +/* + * Finish the recovery of the file system. This is separate from the + * xfs_log_mount() call, because it depends on the code in xfs_mountfs() to read + * in the root and real-time bitmap inodes between calling xfs_log_mount() and + * here. + * + * If we finish recovery successfully, start the background log work. If we are + * not doing recovery, then we have a RO filesystem and we don't need to start + * it. + */ +int +xfs_log_mount_finish( + struct xfs_mount *mp) +{ + struct xlog *log = mp->m_log; + int error = 0; + + if (xfs_has_norecovery(mp)) { + ASSERT(xfs_is_readonly(mp)); + return 0; + } + + /* + * During the second phase of log recovery, we need iget and + * iput to behave like they do for an active filesystem. + * xfs_fs_drop_inode needs to be able to prevent the deletion + * of inodes before we're done replaying log items on those + * inodes. Turn it off immediately after recovery finishes + * so that we don't leak the quota inodes if subsequent mount + * activities fail. + * + * We let all inodes involved in redo item processing end up on + * the LRU instead of being evicted immediately so that if we do + * something to an unlinked inode, the irele won't cause + * premature truncation and freeing of the inode, which results + * in log recovery failure. We have to evict the unreferenced + * lru inodes after clearing SB_ACTIVE because we don't + * otherwise clean up the lru if there's a subsequent failure in + * xfs_mountfs, which leads to us leaking the inodes if nothing + * else (e.g. quotacheck) references the inodes before the + * mount failure occurs. + */ + mp->m_super->s_flags |= SB_ACTIVE; + xfs_log_work_queue(mp); + if (xlog_recovery_needed(log)) + error = xlog_recover_finish(log); + mp->m_super->s_flags &= ~SB_ACTIVE; + evict_inodes(mp->m_super); + + /* + * Drain the buffer LRU after log recovery. This is required for v4 + * filesystems to avoid leaving around buffers with NULL verifier ops, + * but we do it unconditionally to make sure we're always in a clean + * cache state after mount. + * + * Don't push in the error case because the AIL may have pending intents + * that aren't removed until recovery is cancelled. + */ + if (xlog_recovery_needed(log)) { + if (!error) { + xfs_log_force(mp, XFS_LOG_SYNC); + xfs_ail_push_all_sync(mp->m_ail); + } + xfs_notice(mp, "Ending recovery (logdev: %s)", + mp->m_logname ? mp->m_logname : "internal"); + } else { + xfs_info(mp, "Ending clean mount"); + } + xfs_buftarg_drain(mp->m_ddev_targp); + + clear_bit(XLOG_RECOVERY_NEEDED, &log->l_opstate); + + /* Make sure the log is dead if we're returning failure. */ + ASSERT(!error || xlog_is_shutdown(log)); + + return error; +} + +/* + * The mount has failed. Cancel the recovery if it hasn't completed and destroy + * the log. + */ +void +xfs_log_mount_cancel( + struct xfs_mount *mp) +{ + xlog_recover_cancel(mp->m_log); + xfs_log_unmount(mp); +} + +/* + * Flush out the iclog to disk ensuring that device caches are flushed and + * the iclog hits stable storage before any completion waiters are woken. + */ +static inline int +xlog_force_iclog( + struct xlog_in_core *iclog) +{ + atomic_inc(&iclog->ic_refcnt); + iclog->ic_flags |= XLOG_ICL_NEED_FLUSH | XLOG_ICL_NEED_FUA; + if (iclog->ic_state == XLOG_STATE_ACTIVE) + xlog_state_switch_iclogs(iclog->ic_log, iclog, 0); + return xlog_state_release_iclog(iclog->ic_log, iclog, NULL); +} + +/* + * Cycle all the iclogbuf locks to make sure all log IO completion + * is done before we tear down these buffers. + */ +static void +xlog_wait_iclog_completion(struct xlog *log) +{ + int i; + struct xlog_in_core *iclog = log->l_iclog; + + for (i = 0; i < log->l_iclog_bufs; i++) { + down(&iclog->ic_sema); + up(&iclog->ic_sema); + iclog = iclog->ic_next; + } +} + +/* + * Wait for the iclog and all prior iclogs to be written disk as required by the + * log force state machine. Waiting on ic_force_wait ensures iclog completions + * have been ordered and callbacks run before we are woken here, hence + * guaranteeing that all the iclogs up to this one are on stable storage. + */ +int +xlog_wait_on_iclog( + struct xlog_in_core *iclog) + __releases(iclog->ic_log->l_icloglock) +{ + struct xlog *log = iclog->ic_log; + + trace_xlog_iclog_wait_on(iclog, _RET_IP_); + if (!xlog_is_shutdown(log) && + iclog->ic_state != XLOG_STATE_ACTIVE && + iclog->ic_state != XLOG_STATE_DIRTY) { + XFS_STATS_INC(log->l_mp, xs_log_force_sleep); + xlog_wait(&iclog->ic_force_wait, &log->l_icloglock); + } else { + spin_unlock(&log->l_icloglock); + } + + if (xlog_is_shutdown(log)) + return -EIO; + return 0; +} + +int +xlog_write_one_vec( + struct xlog *log, + struct xfs_cil_ctx *ctx, + struct xfs_log_iovec *reg, + struct xlog_ticket *ticket) +{ + struct xfs_log_vec lv = { + .lv_niovecs = 1, + .lv_iovecp = reg, + .lv_bytes = reg->i_len, + }; + LIST_HEAD (lv_chain); + + /* account for space used by record data */ + ticket->t_curr_res -= lv.lv_bytes; + + list_add(&lv.lv_list, &lv_chain); + return xlog_write(log, ctx, &lv_chain, ticket, lv.lv_bytes); +} + +/* + * Write out an unmount record using the ticket provided. We have to account for + * the data space used in the unmount ticket as this write is not done from a + * transaction context that has already done the accounting for us. + */ +static int +xlog_write_unmount_record( + struct xlog *log, + struct xlog_ticket *ticket) +{ + struct { + struct xlog_op_header ophdr; + struct xfs_unmount_log_format ulf; + } unmount_rec = { + .ophdr = { + .oh_clientid = XFS_LOG, + .oh_tid = cpu_to_be32(ticket->t_tid), + .oh_flags = XLOG_UNMOUNT_TRANS, + }, + .ulf = { + .magic = XLOG_UNMOUNT_TYPE, + }, + }; + struct xfs_log_iovec reg = { + .i_addr = &unmount_rec, + .i_len = sizeof(unmount_rec), + .i_type = XLOG_REG_TYPE_UNMOUNT, + }; + + return xlog_write_one_vec(log, NULL, ®, ticket); +} + +/* + * Mark the filesystem clean by writing an unmount record to the head of the + * log. + */ +static void +xlog_unmount_write( + struct xlog *log) +{ + struct xfs_mount *mp = log->l_mp; + struct xlog_in_core *iclog; + struct xlog_ticket *tic = NULL; + int error; + + error = xfs_log_reserve(mp, 600, 1, &tic, 0); + if (error) + goto out_err; + + error = xlog_write_unmount_record(log, tic); + /* + * At this point, we're umounting anyway, so there's no point in + * transitioning log state to shutdown. Just continue... + */ +out_err: + if (error) + xfs_alert(mp, "%s: unmount record failed", __func__); + + spin_lock(&log->l_icloglock); + iclog = log->l_iclog; + error = xlog_force_iclog(iclog); + xlog_wait_on_iclog(iclog); + + if (tic) { + trace_xfs_log_umount_write(log, tic); + xfs_log_ticket_ungrant(log, tic); + } +} + +static void +xfs_log_unmount_verify_iclog( + struct xlog *log) +{ + struct xlog_in_core *iclog = log->l_iclog; + + do { + ASSERT(iclog->ic_state == XLOG_STATE_ACTIVE); + ASSERT(iclog->ic_offset == 0); + } while ((iclog = iclog->ic_next) != log->l_iclog); +} + +/* + * Unmount record used to have a string "Unmount filesystem--" in the + * data section where the "Un" was really a magic number (XLOG_UNMOUNT_TYPE). + * We just write the magic number now since that particular field isn't + * currently architecture converted and "Unmount" is a bit foo. + * As far as I know, there weren't any dependencies on the old behaviour. + */ +static void +xfs_log_unmount_write( + struct xfs_mount *mp) +{ + struct xlog *log = mp->m_log; + + if (!xfs_log_writable(mp)) + return; + + xfs_log_force(mp, XFS_LOG_SYNC); + + if (xlog_is_shutdown(log)) + return; + + /* + * If we think the summary counters are bad, avoid writing the unmount + * record to force log recovery at next mount, after which the summary + * counters will be recalculated. Refer to xlog_check_unmount_rec for + * more details. + */ + if (xfs_fs_has_sickness(mp, XFS_SICK_FS_COUNTERS) || + XFS_TEST_ERROR(mp, XFS_ERRTAG_FORCE_SUMMARY_RECALC)) { + xfs_alert(mp, "%s: will fix summary counters at next mount", + __func__); + return; + } + + xfs_log_unmount_verify_iclog(log); + xlog_unmount_write(log); +} + +/* + * Empty the log for unmount/freeze. + * + * To do this, we first need to shut down the background log work so it is not + * trying to cover the log as we clean up. We then need to unpin all objects in + * the log so we can then flush them out. Once they have completed their IO and + * run the callbacks removing themselves from the AIL, we can cover the log. + */ +int +xfs_log_quiesce( + struct xfs_mount *mp) +{ + /* + * Clear log incompat features since we're quiescing the log. Report + * failures, though it's not fatal to have a higher log feature + * protection level than the log contents actually require. + */ + if (xfs_clear_incompat_log_features(mp)) { + int error; + + error = xfs_sync_sb(mp, false); + if (error) + xfs_warn(mp, + "Failed to clear log incompat features on quiesce"); + } + + cancel_delayed_work_sync(&mp->m_log->l_work); + xfs_log_force(mp, XFS_LOG_SYNC); + + /* + * The superblock buffer is uncached and while xfs_ail_push_all_sync() + * will push it, xfs_buftarg_wait() will not wait for it. Further, + * xfs_buf_iowait() cannot be used because it was pushed with the + * XBF_ASYNC flag set, so we need to use a lock/unlock pair to wait for + * the IO to complete. + */ + xfs_ail_push_all_sync(mp->m_ail); + xfs_buftarg_wait(mp->m_ddev_targp); + xfs_buf_lock(mp->m_sb_bp); + xfs_buf_unlock(mp->m_sb_bp); + + return xfs_log_cover(mp); +} + +void +xfs_log_clean( + struct xfs_mount *mp) +{ + xfs_log_quiesce(mp); + xfs_log_unmount_write(mp); +} + +/* + * Shut down and release the AIL and Log. + * + * During unmount, we need to ensure we flush all the dirty metadata objects + * from the AIL so that the log is empty before we write the unmount record to + * the log. Once this is done, we can tear down the AIL and the log. + */ +void +xfs_log_unmount( + struct xfs_mount *mp) +{ + xfs_log_clean(mp); + + /* + * If shutdown has come from iclog IO context, the log + * cleaning will have been skipped and so we need to wait + * for the iclog to complete shutdown processing before we + * tear anything down. + */ + xlog_wait_iclog_completion(mp->m_log); + + xfs_buftarg_drain(mp->m_ddev_targp); + + xfs_trans_ail_destroy(mp); + + xfs_sysfs_del(&mp->m_log->l_kobj); + + xlog_dealloc_log(mp->m_log); +} + +void +xfs_log_item_init( + struct xfs_mount *mp, + struct xfs_log_item *item, + int type, + const struct xfs_item_ops *ops) +{ + item->li_log = mp->m_log; + item->li_ailp = mp->m_ail; + item->li_type = type; + item->li_ops = ops; + item->li_lv = NULL; + + INIT_LIST_HEAD(&item->li_ail); + INIT_LIST_HEAD(&item->li_cil); + INIT_LIST_HEAD(&item->li_bio_list); + INIT_LIST_HEAD(&item->li_trans); +} + +/* + * Wake up processes waiting for log space after we have moved the log tail. + */ +void +xfs_log_space_wake( + struct xfs_mount *mp) +{ + struct xlog *log = mp->m_log; + int free_bytes; + + if (xlog_is_shutdown(log)) + return; + + if (!list_empty_careful(&log->l_write_head.waiters)) { + ASSERT(!xlog_in_recovery(log)); + + spin_lock(&log->l_write_head.lock); + free_bytes = xlog_grant_space_left(log, &log->l_write_head); + xlog_grant_head_wake(log, &log->l_write_head, &free_bytes); + spin_unlock(&log->l_write_head.lock); + } + + if (!list_empty_careful(&log->l_reserve_head.waiters)) { + ASSERT(!xlog_in_recovery(log)); + + spin_lock(&log->l_reserve_head.lock); + free_bytes = xlog_grant_space_left(log, &log->l_reserve_head); + xlog_grant_head_wake(log, &log->l_reserve_head, &free_bytes); + spin_unlock(&log->l_reserve_head.lock); + } +} + +/* + * Determine if we have a transaction that has gone to disk that needs to be + * covered. To begin the transition to the idle state firstly the log needs to + * be idle. That means the CIL, the AIL and the iclogs needs to be empty before + * we start attempting to cover the log. + * + * Only if we are then in a state where covering is needed, the caller is + * informed that dummy transactions are required to move the log into the idle + * state. + * + * If there are any items in the AIl or CIL, then we do not want to attempt to + * cover the log as we may be in a situation where there isn't log space + * available to run a dummy transaction and this can lead to deadlocks when the + * tail of the log is pinned by an item that is modified in the CIL. Hence + * there's no point in running a dummy transaction at this point because we + * can't start trying to idle the log until both the CIL and AIL are empty. + */ +static bool +xfs_log_need_covered( + struct xfs_mount *mp) +{ + struct xlog *log = mp->m_log; + bool needed = false; + + if (!xlog_cil_empty(log)) + return false; + + spin_lock(&log->l_icloglock); + switch (log->l_covered_state) { + case XLOG_STATE_COVER_DONE: + case XLOG_STATE_COVER_DONE2: + case XLOG_STATE_COVER_IDLE: + break; + case XLOG_STATE_COVER_NEED: + case XLOG_STATE_COVER_NEED2: + if (xfs_ail_min_lsn(log->l_ailp)) + break; + if (!xlog_iclogs_empty(log)) + break; + + needed = true; + if (log->l_covered_state == XLOG_STATE_COVER_NEED) + log->l_covered_state = XLOG_STATE_COVER_DONE; + else + log->l_covered_state = XLOG_STATE_COVER_DONE2; + break; + default: + needed = true; + break; + } + spin_unlock(&log->l_icloglock); + return needed; +} + +/* + * Explicitly cover the log. This is similar to background log covering but + * intended for usage in quiesce codepaths. The caller is responsible to ensure + * the log is idle and suitable for covering. The CIL, iclog buffers and AIL + * must all be empty. + */ +static int +xfs_log_cover( + struct xfs_mount *mp) +{ + int error = 0; + bool need_covered; + + if (!xlog_is_shutdown(mp->m_log)) { + ASSERT(xlog_cil_empty(mp->m_log)); + ASSERT(xlog_iclogs_empty(mp->m_log)); + ASSERT(!xfs_ail_min_lsn(mp->m_log->l_ailp)); + } + + if (!xfs_log_writable(mp)) + return 0; + + /* + * xfs_log_need_covered() is not idempotent because it progresses the + * state machine if the log requires covering. Therefore, we must call + * this function once and use the result until we've issued an sb sync. + * Do so first to make that abundantly clear. + * + * Fall into the covering sequence if the log needs covering or the + * mount has lazy superblock accounting to sync to disk. The sb sync + * used for covering accumulates the in-core counters, so covering + * handles this for us. + */ + need_covered = xfs_log_need_covered(mp); + if (!need_covered && !xfs_has_lazysbcount(mp)) + return 0; + + /* + * To cover the log, commit the superblock twice (at most) in + * independent checkpoints. The first serves as a reference for the + * tail pointer. The sync transaction and AIL push empties the AIL and + * updates the in-core tail to the LSN of the first checkpoint. The + * second commit updates the on-disk tail with the in-core LSN, + * covering the log. Push the AIL one more time to leave it empty, as + * we found it. + */ + do { + error = xfs_sync_sb(mp, true); + if (error) + break; + xfs_ail_push_all_sync(mp->m_ail); + } while (xfs_log_need_covered(mp)); + + return error; +} + +static void +xlog_ioend_work( + struct work_struct *work) +{ + struct xlog_in_core *iclog = + container_of(work, struct xlog_in_core, ic_end_io_work); + struct xlog *log = iclog->ic_log; + int error; + + error = blk_status_to_errno(iclog->ic_bio.bi_status); +#ifdef DEBUG + /* treat writes with injected CRC errors as failed */ + if (iclog->ic_fail_crc) + error = -EIO; +#endif + + /* + * Race to shutdown the filesystem if we see an error. + */ + if (error || XFS_TEST_ERROR(log->l_mp, XFS_ERRTAG_IODONE_IOERR)) { + xfs_alert(log->l_mp, "log I/O error %d", error); + xlog_force_shutdown(log, SHUTDOWN_LOG_IO_ERROR); + } + + xlog_state_done_syncing(iclog); + bio_uninit(&iclog->ic_bio); + + /* + * Drop the lock to signal that we are done. Nothing references the + * iclog after this, so an unmount waiting on this lock can now tear it + * down safely. As such, it is unsafe to reference the iclog after the + * unlock as we could race with it being freed. + */ + up(&iclog->ic_sema); +} + +/* + * Return size of each in-core log record buffer. + * + * All machines get 8 x 32kB buffers by default, unless tuned otherwise. + * + * If the filesystem blocksize is too large, we may need to choose a + * larger size since the directory code currently logs entire blocks. + */ +STATIC void +xlog_get_iclog_buffer_size( + struct xfs_mount *mp, + struct xlog *log) +{ + if (mp->m_logbufs <= 0) + mp->m_logbufs = XLOG_MAX_ICLOGS; + if (mp->m_logbsize <= 0) + mp->m_logbsize = XLOG_BIG_RECORD_BSIZE; + + log->l_iclog_bufs = mp->m_logbufs; + log->l_iclog_size = mp->m_logbsize; + + /* + * Combined size of the log record headers. The first 32k cycles + * are stored directly in the xlog_rec_header, the rest in the + * variable number of xlog_rec_ext_headers at its end. + */ + log->l_iclog_hsize = struct_size(log->l_iclog->ic_header, h_ext, + DIV_ROUND_UP(mp->m_logbsize, XLOG_HEADER_CYCLE_SIZE) - 1); +} + +void +xfs_log_work_queue( + struct xfs_mount *mp) +{ + queue_delayed_work(mp->m_sync_workqueue, &mp->m_log->l_work, + msecs_to_jiffies(xfs_syncd_centisecs * 10)); +} + +/* + * Clear the log incompat flags if we have the opportunity. + * + * This only happens if we're about to log the second dummy transaction as part + * of covering the log. + */ +static inline void +xlog_clear_incompat( + struct xlog *log) +{ + struct xfs_mount *mp = log->l_mp; + + if (!xfs_sb_has_incompat_log_feature(&mp->m_sb, + XFS_SB_FEAT_INCOMPAT_LOG_ALL)) + return; + + if (log->l_covered_state != XLOG_STATE_COVER_DONE2) + return; + + xfs_clear_incompat_log_features(mp); +} + +/* + * Every sync period we need to unpin all items in the AIL and push them to + * disk. If there is nothing dirty, then we might need to cover the log to + * indicate that the filesystem is idle. + */ +static void +xfs_log_worker( + struct work_struct *work) +{ + struct xlog *log = container_of(to_delayed_work(work), + struct xlog, l_work); + struct xfs_mount *mp = log->l_mp; + + /* dgc: errors ignored - not fatal and nowhere to report them */ + if (xfs_fs_writable(mp, SB_FREEZE_WRITE) && xfs_log_need_covered(mp)) { + /* + * Dump a transaction into the log that contains no real change. + * This is needed to stamp the current tail LSN into the log + * during the covering operation. + * + * We cannot use an inode here for this - that will push dirty + * state back up into the VFS and then periodic inode flushing + * will prevent log covering from making progress. Hence we + * synchronously log the superblock instead to ensure the + * superblock is immediately unpinned and can be written back. + */ + xlog_clear_incompat(log); + xfs_sync_sb(mp, true); + } else + xfs_log_force(mp, 0); + + /* start pushing all the metadata that is currently dirty */ + xfs_ail_push_all(mp->m_ail); + + /* queue us up again */ + xfs_log_work_queue(mp); +} + +/* + * This routine initializes some of the log structure for a given mount point. + * Its primary purpose is to fill in enough, so recovery can occur. However, + * some other stuff may be filled in too. + */ +STATIC struct xlog * +xlog_alloc_log( + struct xfs_mount *mp, + struct xfs_buftarg *log_target, + xfs_daddr_t blk_offset, + int num_bblks) +{ + struct xlog *log; + struct xlog_in_core **iclogp; + struct xlog_in_core *iclog, *prev_iclog = NULL; + int i; + int error = -ENOMEM; + uint log2_size = 0; + + log = kzalloc_obj(struct xlog, GFP_KERNEL | __GFP_RETRY_MAYFAIL); + if (!log) { + xfs_warn(mp, "Log allocation failed: No memory!"); + goto out; + } + + log->l_mp = mp; + log->l_targ = log_target; + log->l_logsize = BBTOB(num_bblks); + log->l_logBBstart = blk_offset; + log->l_logBBsize = num_bblks; + log->l_covered_state = XLOG_STATE_COVER_IDLE; + set_bit(XLOG_ACTIVE_RECOVERY, &log->l_opstate); + INIT_DELAYED_WORK(&log->l_work, xfs_log_worker); + INIT_LIST_HEAD(&log->r_dfops); + + log->l_prev_block = -1; + /* log->l_tail_lsn = 0x100000000LL; cycle = 1; current block = 0 */ + xlog_assign_atomic_lsn(&log->l_tail_lsn, 1, 0); + log->l_curr_cycle = 1; /* 0 is bad since this is initial value */ + + if (xfs_has_logv2(mp) && mp->m_sb.sb_logsunit > 1) + log->l_iclog_roundoff = mp->m_sb.sb_logsunit; + else if (mp->m_sb.sb_logsectsize > 0) + log->l_iclog_roundoff = mp->m_sb.sb_logsectsize; + else + log->l_iclog_roundoff = BBSIZE; + + xlog_grant_head_init(&log->l_reserve_head); + xlog_grant_head_init(&log->l_write_head); + + error = -EFSCORRUPTED; + if (xfs_has_sector(mp)) { + log2_size = mp->m_sb.sb_logsectlog; + if (log2_size < BBSHIFT) { + xfs_warn(mp, "Log sector size too small (0x%x < 0x%x)", + log2_size, BBSHIFT); + goto out_free_log; + } + + log2_size -= BBSHIFT; + if (log2_size > mp->m_sectbb_log) { + xfs_warn(mp, "Log sector size too large (0x%x > 0x%x)", + log2_size, mp->m_sectbb_log); + goto out_free_log; + } + + /* for larger sector sizes, must have v2 or external log */ + if (log2_size && log->l_logBBstart > 0 && + !xfs_has_logv2(mp)) { + xfs_warn(mp, + "log sector size (0x%x) invalid for configuration.", + log2_size); + goto out_free_log; + } + } + log->l_sectBBsize = 1 << log2_size; + + xlog_get_iclog_buffer_size(mp, log); + + spin_lock_init(&log->l_icloglock); + init_waitqueue_head(&log->l_flush_wait); + + iclogp = &log->l_iclog; + ASSERT(log->l_iclog_size >= 4096); + for (i = 0; i < log->l_iclog_bufs; i++) { + size_t bvec_size = howmany(log->l_iclog_size, PAGE_SIZE) * + sizeof(struct bio_vec); + + iclog = kzalloc(sizeof(*iclog) + bvec_size, + GFP_KERNEL | __GFP_RETRY_MAYFAIL); + if (!iclog) + goto out_free_iclog; + + *iclogp = iclog; + iclog->ic_prev = prev_iclog; + prev_iclog = iclog; + + iclog->ic_header = kvzalloc(log->l_iclog_size, + GFP_KERNEL | __GFP_RETRY_MAYFAIL); + if (!iclog->ic_header) + goto out_free_iclog; + iclog->ic_header->h_magicno = + cpu_to_be32(XLOG_HEADER_MAGIC_NUM); + iclog->ic_header->h_version = cpu_to_be32( + xfs_has_logv2(log->l_mp) ? 2 : 1); + iclog->ic_header->h_size = cpu_to_be32(log->l_iclog_size); + iclog->ic_header->h_fmt = cpu_to_be32(XLOG_FMT); + memcpy(&iclog->ic_header->h_fs_uuid, &mp->m_sb.sb_uuid, + sizeof(iclog->ic_header->h_fs_uuid)); + + iclog->ic_datap = (void *)iclog->ic_header + log->l_iclog_hsize; + iclog->ic_size = log->l_iclog_size - log->l_iclog_hsize; + iclog->ic_state = XLOG_STATE_ACTIVE; + iclog->ic_log = log; + atomic_set(&iclog->ic_refcnt, 0); + INIT_LIST_HEAD(&iclog->ic_callbacks); + + init_waitqueue_head(&iclog->ic_force_wait); + init_waitqueue_head(&iclog->ic_write_wait); + INIT_WORK(&iclog->ic_end_io_work, xlog_ioend_work); + sema_init(&iclog->ic_sema, 1); + + iclogp = &iclog->ic_next; + } + *iclogp = log->l_iclog; /* complete ring */ + log->l_iclog->ic_prev = prev_iclog; /* re-write 1st prev ptr */ + + log->l_ioend_workqueue = alloc_workqueue("xfs-log/%s", + XFS_WQFLAGS(WQ_FREEZABLE | WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_PERCPU), + 0, mp->m_super->s_id); + if (!log->l_ioend_workqueue) + goto out_free_iclog; + + error = xlog_cil_init(log); + if (error) + goto out_destroy_workqueue; + return log; + +out_destroy_workqueue: + destroy_workqueue(log->l_ioend_workqueue); +out_free_iclog: + for (iclog = log->l_iclog; iclog; iclog = prev_iclog) { + prev_iclog = iclog->ic_next; + kvfree(iclog->ic_header); + kfree(iclog); + if (prev_iclog == log->l_iclog) + break; + } +out_free_log: + kfree(log); +out: + return ERR_PTR(error); +} /* xlog_alloc_log */ + +/* + * Stamp cycle number in every block + */ +STATIC void +xlog_pack_data( + struct xlog *log, + struct xlog_in_core *iclog, + int roundoff) +{ + struct xlog_rec_header *rhead = iclog->ic_header; + __be32 cycle_lsn = CYCLE_LSN_DISK(rhead->h_lsn); + char *dp = iclog->ic_datap; + int i; + + for (i = 0; i < BTOBB(iclog->ic_offset + roundoff); i++) { + *xlog_cycle_data(rhead, i) = *(__be32 *)dp; + *(__be32 *)dp = cycle_lsn; + dp += BBSIZE; + } + + for (i = 0; i < (log->l_iclog_hsize >> BBSHIFT) - 1; i++) + rhead->h_ext[i].xh_cycle = cycle_lsn; +} + +/* + * Calculate the checksum for a log buffer. + * + * This is a little more complicated than it should be because the various + * headers and the actual data are non-contiguous. + */ +__le32 +xlog_cksum( + struct xlog *log, + struct xlog_rec_header *rhead, + char *dp, + unsigned int hdrsize, + unsigned int size) +{ + uint32_t crc; + + /* first generate the crc for the record header ... */ + crc = xfs_start_cksum_update((char *)rhead, hdrsize, + offsetof(struct xlog_rec_header, h_crc)); + + /* ... then for additional cycle data for v2 logs ... */ + if (xfs_has_logv2(log->l_mp)) { + int xheads, i; + + xheads = DIV_ROUND_UP(size, XLOG_HEADER_CYCLE_SIZE) - 1; + for (i = 0; i < xheads; i++) + crc = crc32c(crc, &rhead->h_ext[i], XLOG_REC_EXT_SIZE); + } + + /* ... and finally for the payload */ + crc = crc32c(crc, dp, size); + + return xfs_end_cksum(crc); +} + +static void +xlog_bio_end_io( + struct bio *bio) +{ + struct xlog_in_core *iclog = bio->bi_private; + + queue_work(iclog->ic_log->l_ioend_workqueue, + &iclog->ic_end_io_work); +} + +STATIC void +xlog_write_iclog( + struct xlog *log, + struct xlog_in_core *iclog, + uint64_t bno, + unsigned int count) +{ + ASSERT(bno < log->l_logBBsize); + trace_xlog_iclog_write(iclog, _RET_IP_); + + /* + * We lock the iclogbufs here so that we can serialise against I/O + * completion during unmount. We might be processing a shutdown + * triggered during unmount, and that can occur asynchronously to the + * unmount thread, and hence we need to ensure that completes before + * tearing down the iclogbufs. Hence we need to hold the buffer lock + * across the log IO to archieve that. + */ + down(&iclog->ic_sema); + if (xlog_is_shutdown(log)) { + /* + * It would seem logical to return EIO here, but we rely on + * the log state machine to propagate I/O errors instead of + * doing it here. We kick of the state machine and unlock + * the buffer manually, the code needs to be kept in sync + * with the I/O completion path. + */ + goto sync; + } + + /* + * We use REQ_SYNC | REQ_IDLE here to tell the block layer the are more + * IOs coming immediately after this one. This prevents the block layer + * writeback throttle from throttling log writes behind background + * metadata writeback and causing priority inversions. + */ + bio_init(&iclog->ic_bio, log->l_targ->bt_bdev, iclog->ic_bvec, + howmany(count, PAGE_SIZE), + REQ_OP_WRITE | REQ_META | REQ_SYNC | REQ_IDLE); + iclog->ic_bio.bi_iter.bi_sector = log->l_logBBstart + bno; + iclog->ic_bio.bi_end_io = xlog_bio_end_io; + iclog->ic_bio.bi_private = iclog; + + if (iclog->ic_flags & XLOG_ICL_NEED_FLUSH) { + iclog->ic_bio.bi_opf |= REQ_PREFLUSH; + /* + * For external log devices, we also need to flush the data + * device cache first to ensure all metadata writeback covered + * by the LSN in this iclog is on stable storage. This is slow, + * but it *must* complete before we issue the external log IO. + * + * If the flush fails, we cannot conclude that past metadata + * writeback from the log succeeded. Repeating the flush is + * not possible, hence we must shut down with log IO error to + * avoid shutdown re-entering this path and erroring out again. + */ + if (log->l_targ != log->l_mp->m_ddev_targp && + blkdev_issue_flush(log->l_mp->m_ddev_targp->bt_bdev)) + goto shutdown; + } + if (iclog->ic_flags & XLOG_ICL_NEED_FUA) + iclog->ic_bio.bi_opf |= REQ_FUA; + + iclog->ic_flags &= ~(XLOG_ICL_NEED_FLUSH | XLOG_ICL_NEED_FUA); + + if (is_vmalloc_addr(iclog->ic_header)) { + if (!bio_add_vmalloc(&iclog->ic_bio, iclog->ic_header, count)) + goto shutdown; + } else { + bio_add_virt_nofail(&iclog->ic_bio, iclog->ic_header, count); + } + + /* + * If this log buffer would straddle the end of the log we will have + * to split it up into two bios, so that we can continue at the start. + */ + if (bno + BTOBB(count) > log->l_logBBsize) { + struct bio *split; + + split = bio_split(&iclog->ic_bio, log->l_logBBsize - bno, + GFP_NOIO, &fs_bio_set); + bio_chain(split, &iclog->ic_bio); + submit_bio(split); + + /* restart at logical offset zero for the remainder */ + iclog->ic_bio.bi_iter.bi_sector = log->l_logBBstart; + } + + submit_bio(&iclog->ic_bio); + return; +shutdown: + xlog_force_shutdown(log, SHUTDOWN_LOG_IO_ERROR); +sync: + xlog_state_done_syncing(iclog); + up(&iclog->ic_sema); +} + +/* + * We need to bump cycle number for the part of the iclog that is + * written to the start of the log. Watch out for the header magic + * number case, though. + */ +static void +xlog_split_iclog( + struct xlog *log, + void *data, + uint64_t bno, + unsigned int count) +{ + unsigned int split_offset = BBTOB(log->l_logBBsize - bno); + unsigned int i; + + for (i = split_offset; i < count; i += BBSIZE) { + uint32_t cycle = get_unaligned_be32(data + i); + + if (++cycle == XLOG_HEADER_MAGIC_NUM) + cycle++; + put_unaligned_be32(cycle, data + i); + } +} + +static int +xlog_calc_iclog_size( + struct xlog *log, + struct xlog_in_core *iclog, + uint32_t *roundoff) +{ + uint32_t count_init, count; + + /* Add for LR header */ + count_init = log->l_iclog_hsize + iclog->ic_offset; + count = roundup(count_init, log->l_iclog_roundoff); + + *roundoff = count - count_init; + + ASSERT(count >= count_init); + ASSERT(*roundoff < log->l_iclog_roundoff); + return count; +} + +/* + * Flush out the in-core log (iclog) to the on-disk log in an asynchronous + * fashion. Previously, we should have moved the current iclog + * ptr in the log to point to the next available iclog. This allows further + * write to continue while this code syncs out an iclog ready to go. + * Before an in-core log can be written out, the data section must be scanned + * to save away the 1st word of each BBSIZE block into the header. We replace + * it with the current cycle count. Each BBSIZE block is tagged with the + * cycle count because there in an implicit assumption that drives will + * guarantee that entire 512 byte blocks get written at once. In other words, + * we can't have part of a 512 byte block written and part not written. By + * tagging each block, we will know which blocks are valid when recovering + * after an unclean shutdown. + * + * This routine is single threaded on the iclog. No other thread can be in + * this routine with the same iclog. Changing contents of iclog can there- + * fore be done without grabbing the state machine lock. Updating the global + * log will require grabbing the lock though. + * + * The entire log manager uses a logical block numbering scheme. Only + * xlog_write_iclog knows about the fact that the log may not start with + * block zero on a given device. + */ +STATIC void +xlog_sync( + struct xlog *log, + struct xlog_in_core *iclog, + struct xlog_ticket *ticket) +{ + unsigned int count; /* byte count of bwrite */ + unsigned int roundoff; /* roundoff to BB or stripe */ + uint64_t bno; + unsigned int size; + + ASSERT(atomic_read(&iclog->ic_refcnt) == 0); + trace_xlog_iclog_sync(iclog, _RET_IP_); + + count = xlog_calc_iclog_size(log, iclog, &roundoff); + + /* + * If we have a ticket, account for the roundoff via the ticket + * reservation to avoid touching the hot grant heads needlessly. + * Otherwise, we have to move grant heads directly. + */ + if (ticket) { + ticket->t_curr_res -= roundoff; + } else { + xlog_grant_add_space(&log->l_reserve_head, roundoff); + xlog_grant_add_space(&log->l_write_head, roundoff); + } + + /* put cycle number in every block */ + xlog_pack_data(log, iclog, roundoff); + + /* real byte length */ + size = iclog->ic_offset; + if (xfs_has_logv2(log->l_mp)) + size += roundoff; + iclog->ic_header->h_len = cpu_to_be32(size); + + XFS_STATS_INC(log->l_mp, xs_log_writes); + XFS_STATS_ADD(log->l_mp, xs_log_blocks, BTOBB(count)); + + bno = BLOCK_LSN(be64_to_cpu(iclog->ic_header->h_lsn)); + + /* Do we need to split this write into 2 parts? */ + if (bno + BTOBB(count) > log->l_logBBsize) + xlog_split_iclog(log, iclog->ic_header, bno, count); + + /* calculcate the checksum */ + iclog->ic_header->h_crc = xlog_cksum(log, iclog->ic_header, + iclog->ic_datap, XLOG_REC_SIZE, size); + /* + * Intentionally corrupt the log record CRC based on the error injection + * frequency, if defined. This facilitates testing log recovery in the + * event of torn writes. Hence, set the IOABORT state to abort the log + * write on I/O completion and shutdown the fs. The subsequent mount + * detects the bad CRC and attempts to recover. + */ +#ifdef DEBUG + if (XFS_TEST_ERROR(log->l_mp, XFS_ERRTAG_LOG_BAD_CRC)) { + iclog->ic_header->h_crc &= cpu_to_le32(0xAAAAAAAA); + iclog->ic_fail_crc = true; + xfs_warn(log->l_mp, + "Intentionally corrupted log record at LSN 0x%llx. Shutdown imminent.", + be64_to_cpu(iclog->ic_header->h_lsn)); + } +#endif + xlog_verify_iclog(log, iclog, count); + xlog_write_iclog(log, iclog, bno, count); +} + +/* + * Deallocate a log structure + */ +STATIC void +xlog_dealloc_log( + struct xlog *log) +{ + struct xlog_in_core *iclog, *next_iclog; + int i; + + /* + * Destroy the CIL after waiting for iclog IO completion because an + * iclog EIO error will try to shut down the log, which accesses the + * CIL to wake up the waiters. + */ + xlog_cil_destroy(log); + + iclog = log->l_iclog; + for (i = 0; i < log->l_iclog_bufs; i++) { + next_iclog = iclog->ic_next; + kvfree(iclog->ic_header); + kfree(iclog); + iclog = next_iclog; + } + + log->l_mp->m_log = NULL; + destroy_workqueue(log->l_ioend_workqueue); + kfree(log); +} + +/* + * Update counters atomically now that memcpy is done. + */ +static inline void +xlog_state_finish_copy( + struct xlog *log, + struct xlog_in_core *iclog, + int record_cnt, + int copy_bytes) +{ + lockdep_assert_held(&log->l_icloglock); + + be32_add_cpu(&iclog->ic_header->h_num_logops, record_cnt); + iclog->ic_offset += copy_bytes; +} + +/* + * print out info relating to regions written which consume + * the reservation + */ +void +xlog_print_tic_res( + struct xfs_mount *mp, + struct xlog_ticket *ticket) +{ + xfs_warn(mp, "ticket reservation summary:"); + xfs_warn(mp, " unit res = %d bytes", ticket->t_unit_res); + xfs_warn(mp, " current res = %d bytes", ticket->t_curr_res); + xfs_warn(mp, " original count = %d", ticket->t_ocnt); + xfs_warn(mp, " remaining count = %d", ticket->t_cnt); +} + +/* + * Print a summary of the transaction. + */ +void +xlog_print_trans( + struct xfs_trans *tp) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_log_item *lip; + + /* dump core transaction and ticket info */ + xfs_warn(mp, "transaction summary:"); + xfs_warn(mp, " log res = %d", tp->t_log_res); + xfs_warn(mp, " log count = %d", tp->t_log_count); + xfs_warn(mp, " flags = 0x%x", tp->t_flags); + + xlog_print_tic_res(mp, tp->t_ticket); + + /* dump each log item */ + list_for_each_entry(lip, &tp->t_items, li_trans) { + struct xfs_log_vec *lv = lip->li_lv; + struct xfs_log_iovec *vec; + int i; + + xfs_warn(mp, "log item: "); + xfs_warn(mp, " type = 0x%x", lip->li_type); + xfs_warn(mp, " flags = 0x%lx", lip->li_flags); + if (!lv) + continue; + xfs_warn(mp, " niovecs = %d", lv->lv_niovecs); + xfs_warn(mp, " alloc_size = %d", lv->lv_alloc_size); + xfs_warn(mp, " bytes = %d", lv->lv_bytes); + xfs_warn(mp, " buf used= %d", lv->lv_buf_used); + + /* dump each iovec for the log item */ + vec = lv->lv_iovecp; + for (i = 0; i < lv->lv_niovecs; i++) { + int dumplen = min(vec->i_len, 32); + + xfs_warn(mp, " iovec[%d]", i); + xfs_warn(mp, " type = 0x%x", vec->i_type); + xfs_warn(mp, " len = %d", vec->i_len); + xfs_warn(mp, " first %d bytes of iovec[%d]:", dumplen, i); + xfs_hex_dump(vec->i_addr, dumplen); + + vec++; + } + } +} + +static inline uint32_t xlog_write_space_left(struct xlog_write_data *data) +{ + return data->iclog->ic_size - data->log_offset; +} + +static void * +xlog_write_space_advance( + struct xlog_write_data *data, + unsigned int len) +{ + void *p = data->iclog->ic_datap + data->log_offset; + + ASSERT(xlog_write_space_left(data) >= len); + ASSERT(data->log_offset % sizeof(int32_t) == 0); + ASSERT(len % sizeof(int32_t) == 0); + + data->data_cnt += len; + data->log_offset += len; + data->bytes_left -= len; + return p; +} + +static inline void +xlog_write_iovec( + struct xlog_write_data *data, + void *buf, + uint32_t buf_len) +{ + memcpy(xlog_write_space_advance(data, buf_len), buf, buf_len); + data->record_cnt++; +} + +/* + * Write log vectors into a single iclog which is guaranteed by the caller + * to have enough space to write the entire log vector into. + */ +static void +xlog_write_full( + struct xfs_log_vec *lv, + struct xlog_write_data *data) +{ + int index; + + ASSERT(data->bytes_left <= xlog_write_space_left(data) || + data->iclog->ic_state == XLOG_STATE_WANT_SYNC); + + /* + * Ordered log vectors have no regions to write so this + * loop will naturally skip them. + */ + for (index = 0; index < lv->lv_niovecs; index++) { + struct xfs_log_iovec *reg = &lv->lv_iovecp[index]; + struct xlog_op_header *ophdr = reg->i_addr; + + ophdr->oh_tid = cpu_to_be32(data->ticket->t_tid); + xlog_write_iovec(data, reg->i_addr, reg->i_len); + } +} + +static int +xlog_write_get_more_iclog_space( + struct xlog_write_data *data) +{ + struct xlog *log = data->iclog->ic_log; + int error; + + spin_lock(&log->l_icloglock); + ASSERT(data->iclog->ic_state == XLOG_STATE_WANT_SYNC); + xlog_state_finish_copy(log, data->iclog, data->record_cnt, + data->data_cnt); + error = xlog_state_release_iclog(log, data->iclog, data->ticket); + spin_unlock(&log->l_icloglock); + if (error) + return error; + + error = xlog_state_get_iclog_space(log, data); + if (error) + return error; + data->record_cnt = 0; + data->data_cnt = 0; + return 0; +} + +/* + * Write log vectors into a single iclog which is smaller than the current chain + * length. We write until we cannot fit a full record into the remaining space + * and then stop. We return the log vector that is to be written that cannot + * wholly fit in the iclog. + */ +static int +xlog_write_partial( + struct xfs_log_vec *lv, + struct xlog_write_data *data) +{ + struct xlog_op_header *ophdr; + int index = 0; + uint32_t rlen; + int error; + + /* walk the logvec, copying until we run out of space in the iclog */ + for (index = 0; index < lv->lv_niovecs; index++) { + struct xfs_log_iovec *reg = &lv->lv_iovecp[index]; + uint32_t reg_offset = 0; + + /* + * The first region of a continuation must have a non-zero + * length otherwise log recovery will just skip over it and + * start recovering from the next opheader it finds. Because we + * mark the next opheader as a continuation, recovery will then + * incorrectly add the continuation to the previous region and + * that breaks stuff. + * + * Hence if there isn't space for region data after the + * opheader, then we need to start afresh with a new iclog. + */ + if (xlog_write_space_left(data) <= + sizeof(struct xlog_op_header)) { + error = xlog_write_get_more_iclog_space(data); + if (error) + return error; + } + + ophdr = reg->i_addr; + rlen = min_t(uint32_t, reg->i_len, xlog_write_space_left(data)); + + ophdr->oh_tid = cpu_to_be32(data->ticket->t_tid); + ophdr->oh_len = cpu_to_be32(rlen - sizeof(struct xlog_op_header)); + if (rlen != reg->i_len) + ophdr->oh_flags |= XLOG_CONTINUE_TRANS; + + xlog_write_iovec(data, reg->i_addr, rlen); + + /* If we wrote the whole region, move to the next. */ + if (rlen == reg->i_len) + continue; + + /* + * We now have a partially written iovec, but it can span + * multiple iclogs so we loop here. First we release the iclog + * we currently have, then we get a new iclog and add a new + * opheader. Then we continue copying from where we were until + * we either complete the iovec or fill the iclog. If we + * complete the iovec, then we increment the index and go right + * back to the top of the outer loop. if we fill the iclog, we + * run the inner loop again. + * + * This is complicated by the tail of a region using all the + * space in an iclog and hence requiring us to release the iclog + * and get a new one before returning to the outer loop. We must + * always guarantee that we exit this inner loop with at least + * space for log transaction opheaders left in the current + * iclog, hence we cannot just terminate the loop at the end + * of the of the continuation. So we loop while there is no + * space left in the current iclog, and check for the end of the + * continuation after getting a new iclog. + */ + do { + /* + * Ensure we include the continuation opheader in the + * space we need in the new iclog by adding that size + * to the length we require. This continuation opheader + * needs to be accounted to the ticket as the space it + * consumes hasn't been accounted to the lv we are + * writing. + */ + data->bytes_left += sizeof(struct xlog_op_header); + error = xlog_write_get_more_iclog_space(data); + if (error) + return error; + + ophdr = xlog_write_space_advance(data, + sizeof(struct xlog_op_header)); + ophdr->oh_tid = cpu_to_be32(data->ticket->t_tid); + ophdr->oh_clientid = XFS_TRANSACTION; + ophdr->oh_res2 = 0; + ophdr->oh_flags = XLOG_WAS_CONT_TRANS; + + data->ticket->t_curr_res -= + sizeof(struct xlog_op_header); + + /* + * If rlen fits in the iclog, then end the region + * continuation. Otherwise we're going around again. + */ + reg_offset += rlen; + rlen = reg->i_len - reg_offset; + if (rlen <= xlog_write_space_left(data)) + ophdr->oh_flags |= XLOG_END_TRANS; + else + ophdr->oh_flags |= XLOG_CONTINUE_TRANS; + + rlen = min_t(uint32_t, rlen, + xlog_write_space_left(data)); + ophdr->oh_len = cpu_to_be32(rlen); + + xlog_write_iovec(data, reg->i_addr + reg_offset, rlen); + } while (ophdr->oh_flags & XLOG_CONTINUE_TRANS); + } + + return 0; +} + +/* + * Write some region out to in-core log + * + * This will be called when writing externally provided regions or when + * writing out a commit record for a given transaction. + * + * General algorithm: + * 1. Find total length of this write. This may include adding to the + * lengths passed in. + * 2. Check whether we violate the tickets reservation. + * 3. While writing to this iclog + * A. Reserve as much space in this iclog as can get + * B. If this is first write, save away start lsn + * C. While writing this region: + * 1. If first write of transaction, write start record + * 2. Write log operation header (header per region) + * 3. Find out if we can fit entire region into this iclog + * 4. Potentially, verify destination memcpy ptr + * 5. Memcpy (partial) region + * 6. If partial copy, release iclog; otherwise, continue + * copying more regions into current iclog + * 4. Mark want sync bit (in simulation mode) + * 5. Release iclog for potential flush to on-disk log. + * + * ERRORS: + * 1. Panic if reservation is overrun. This should never happen since + * reservation amounts are generated internal to the filesystem. + * NOTES: + * 1. Tickets are single threaded data structures. + * 2. The XLOG_END_TRANS & XLOG_CONTINUE_TRANS flags are passed down to the + * syncing routine. When a single log_write region needs to span + * multiple in-core logs, the XLOG_CONTINUE_TRANS bit should be set + * on all log operation writes which don't contain the end of the + * region. The XLOG_END_TRANS bit is used for the in-core log + * operation which contains the end of the continued log_write region. + * 3. When xlog_state_get_iclog_space() grabs the rest of the current iclog, + * we don't really know exactly how much space will be used. As a result, + * we don't update ic_offset until the end when we know exactly how many + * bytes have been written out. + */ +int +xlog_write( + struct xlog *log, + struct xfs_cil_ctx *ctx, + struct list_head *lv_chain, + struct xlog_ticket *ticket, + uint32_t len) + +{ + struct xfs_log_vec *lv; + struct xlog_write_data data = { + .ticket = ticket, + .bytes_left = len, + }; + int error; + + if (ticket->t_curr_res < 0) { + xfs_alert_tag(log->l_mp, XFS_PTAG_LOGRES, + "ctx ticket reservation ran out. Need to up reservation"); + xlog_print_tic_res(log->l_mp, ticket); + xlog_force_shutdown(log, SHUTDOWN_LOG_IO_ERROR); + } + + error = xlog_state_get_iclog_space(log, &data); + if (error) + return error; + + ASSERT(xlog_write_space_left(&data) > 0); + + /* + * If we have a context pointer, pass it the first iclog we are + * writing to so it can record state needed for iclog write + * ordering. + */ + if (ctx) + xlog_cil_set_ctx_write_state(ctx, data.iclog); + + list_for_each_entry(lv, lv_chain, lv_list) { + /* + * If the entire log vec does not fit in the iclog, punt it to + * the partial copy loop which can handle this case. + */ + if (lv->lv_niovecs && + lv->lv_bytes > xlog_write_space_left(&data)) { + error = xlog_write_partial(lv, &data); + if (error) { + /* + * We have no iclog to release, so just return + * the error immediately. + */ + return error; + } + } else { + xlog_write_full(lv, &data); + } + } + ASSERT(data.bytes_left == 0); + + /* + * We've already been guaranteed that the last writes will fit inside + * the current iclog, and hence it will already have the space used by + * those writes accounted to it. Hence we do not need to update the + * iclog with the number of bytes written here. + */ + spin_lock(&log->l_icloglock); + xlog_state_finish_copy(log, data.iclog, data.record_cnt, 0); + error = xlog_state_release_iclog(log, data.iclog, ticket); + spin_unlock(&log->l_icloglock); + + return error; +} + +static void +xlog_state_activate_iclog( + struct xlog_in_core *iclog, + int *iclogs_changed) +{ + ASSERT(list_empty_careful(&iclog->ic_callbacks)); + trace_xlog_iclog_activate(iclog, _RET_IP_); + + /* + * If the number of ops in this iclog indicate it just contains the + * dummy transaction, we can change state into IDLE (the second time + * around). Otherwise we should change the state into NEED a dummy. + * We don't need to cover the dummy. + */ + if (*iclogs_changed == 0 && + iclog->ic_header->h_num_logops == cpu_to_be32(XLOG_COVER_OPS)) { + *iclogs_changed = 1; + } else { + /* + * We have two dirty iclogs so start over. This could also be + * num of ops indicating this is not the dummy going out. + */ + *iclogs_changed = 2; + } + + iclog->ic_state = XLOG_STATE_ACTIVE; + iclog->ic_offset = 0; + iclog->ic_header->h_num_logops = 0; + memset(iclog->ic_header->h_cycle_data, 0, + sizeof(iclog->ic_header->h_cycle_data)); + iclog->ic_header->h_lsn = 0; + iclog->ic_header->h_tail_lsn = 0; +} + +/* + * Loop through all iclogs and mark all iclogs currently marked DIRTY as + * ACTIVE after iclog I/O has completed. + */ +static void +xlog_state_activate_iclogs( + struct xlog *log, + int *iclogs_changed) +{ + struct xlog_in_core *iclog = log->l_iclog; + + do { + if (iclog->ic_state == XLOG_STATE_DIRTY) + xlog_state_activate_iclog(iclog, iclogs_changed); + /* + * The ordering of marking iclogs ACTIVE must be maintained, so + * an iclog doesn't become ACTIVE beyond one that is SYNCING. + */ + else if (iclog->ic_state != XLOG_STATE_ACTIVE) + break; + } while ((iclog = iclog->ic_next) != log->l_iclog); +} + +static int +xlog_covered_state( + int prev_state, + int iclogs_changed) +{ + /* + * We go to NEED for any non-covering writes. We go to NEED2 if we just + * wrote the first covering record (DONE). We go to IDLE if we just + * wrote the second covering record (DONE2) and remain in IDLE until a + * non-covering write occurs. + */ + switch (prev_state) { + case XLOG_STATE_COVER_IDLE: + if (iclogs_changed == 1) + return XLOG_STATE_COVER_IDLE; + fallthrough; + case XLOG_STATE_COVER_NEED: + case XLOG_STATE_COVER_NEED2: + break; + case XLOG_STATE_COVER_DONE: + if (iclogs_changed == 1) + return XLOG_STATE_COVER_NEED2; + break; + case XLOG_STATE_COVER_DONE2: + if (iclogs_changed == 1) + return XLOG_STATE_COVER_IDLE; + break; + default: + ASSERT(0); + } + + return XLOG_STATE_COVER_NEED; +} + +STATIC void +xlog_state_clean_iclog( + struct xlog *log, + struct xlog_in_core *dirty_iclog) +{ + int iclogs_changed = 0; + + trace_xlog_iclog_clean(dirty_iclog, _RET_IP_); + + dirty_iclog->ic_state = XLOG_STATE_DIRTY; + + xlog_state_activate_iclogs(log, &iclogs_changed); + wake_up_all(&dirty_iclog->ic_force_wait); + + if (iclogs_changed) { + log->l_covered_state = xlog_covered_state(log->l_covered_state, + iclogs_changed); + } +} + +STATIC xfs_lsn_t +xlog_get_lowest_lsn( + struct xlog *log) +{ + struct xlog_in_core *iclog = log->l_iclog; + xfs_lsn_t lowest_lsn = 0, lsn; + + do { + if (iclog->ic_state == XLOG_STATE_ACTIVE || + iclog->ic_state == XLOG_STATE_DIRTY) + continue; + + lsn = be64_to_cpu(iclog->ic_header->h_lsn); + if ((lsn && !lowest_lsn) || XFS_LSN_CMP(lsn, lowest_lsn) < 0) + lowest_lsn = lsn; + } while ((iclog = iclog->ic_next) != log->l_iclog); + + return lowest_lsn; +} + +/* + * Return true if we need to stop processing, false to continue to the next + * iclog. The caller will need to run callbacks if the iclog is returned in the + * XLOG_STATE_CALLBACK state. + */ +static bool +xlog_state_iodone_process_iclog( + struct xlog *log, + struct xlog_in_core *iclog) +{ + xfs_lsn_t lowest_lsn; + xfs_lsn_t header_lsn; + + switch (iclog->ic_state) { + case XLOG_STATE_ACTIVE: + case XLOG_STATE_DIRTY: + /* + * Skip all iclogs in the ACTIVE & DIRTY states: + */ + return false; + case XLOG_STATE_DONE_SYNC: + /* + * Now that we have an iclog that is in the DONE_SYNC state, do + * one more check here to see if we have chased our tail around. + * If this is not the lowest lsn iclog, then we will leave it + * for another completion to process. + */ + header_lsn = be64_to_cpu(iclog->ic_header->h_lsn); + lowest_lsn = xlog_get_lowest_lsn(log); + if (lowest_lsn && XFS_LSN_CMP(lowest_lsn, header_lsn) < 0) + return false; + /* + * If there are no callbacks on this iclog, we can mark it clean + * immediately and return. Otherwise we need to run the + * callbacks. + */ + if (list_empty(&iclog->ic_callbacks)) { + xlog_state_clean_iclog(log, iclog); + return false; + } + trace_xlog_iclog_callback(iclog, _RET_IP_); + iclog->ic_state = XLOG_STATE_CALLBACK; + return false; + default: + /* + * Can only perform callbacks in order. Since this iclog is not + * in the DONE_SYNC state, we skip the rest and just try to + * clean up. + */ + return true; + } +} + +/* + * Loop over all the iclogs, running attached callbacks on them. Return true if + * we ran any callbacks, indicating that we dropped the icloglock. We don't need + * to handle transient shutdown state here at all because + * xlog_state_shutdown_callbacks() will be run to do the necessary shutdown + * cleanup of the callbacks. + */ +static bool +xlog_state_do_iclog_callbacks( + struct xlog *log) + __releases(&log->l_icloglock) + __acquires(&log->l_icloglock) +{ + struct xlog_in_core *first_iclog = log->l_iclog; + struct xlog_in_core *iclog = first_iclog; + bool ran_callback = false; + + do { + LIST_HEAD(cb_list); + + if (xlog_state_iodone_process_iclog(log, iclog)) + break; + if (iclog->ic_state != XLOG_STATE_CALLBACK) { + iclog = iclog->ic_next; + continue; + } + list_splice_init(&iclog->ic_callbacks, &cb_list); + spin_unlock(&log->l_icloglock); + + trace_xlog_iclog_callbacks_start(iclog, _RET_IP_); + xlog_cil_process_committed(&cb_list); + trace_xlog_iclog_callbacks_done(iclog, _RET_IP_); + ran_callback = true; + + spin_lock(&log->l_icloglock); + xlog_state_clean_iclog(log, iclog); + iclog = iclog->ic_next; + } while (iclog != first_iclog); + + return ran_callback; +} + + +/* + * Loop running iclog completion callbacks until there are no more iclogs in a + * state that can run callbacks. + */ +STATIC void +xlog_state_do_callback( + struct xlog *log) +{ + int flushcnt = 0; + int repeats = 0; + + spin_lock(&log->l_icloglock); + while (xlog_state_do_iclog_callbacks(log)) { + if (xlog_is_shutdown(log)) + break; + + if (++repeats > 5000) { + flushcnt += repeats; + repeats = 0; + xfs_warn(log->l_mp, + "%s: possible infinite loop (%d iterations)", + __func__, flushcnt); + } + } + + if (log->l_iclog->ic_state == XLOG_STATE_ACTIVE) + wake_up_all(&log->l_flush_wait); + + spin_unlock(&log->l_icloglock); +} + + +/* + * Finish transitioning this iclog to the dirty state. + * + * Callbacks could take time, so they are done outside the scope of the + * global state machine log lock. + */ +STATIC void +xlog_state_done_syncing( + struct xlog_in_core *iclog) +{ + struct xlog *log = iclog->ic_log; + + spin_lock(&log->l_icloglock); + ASSERT(atomic_read(&iclog->ic_refcnt) == 0); + trace_xlog_iclog_sync_done(iclog, _RET_IP_); + + /* + * If we got an error, either on the first buffer, or in the case of + * split log writes, on the second, we shut down the file system and + * no iclogs should ever be attempted to be written to disk again. + */ + if (!xlog_is_shutdown(log)) { + ASSERT(iclog->ic_state == XLOG_STATE_SYNCING); + iclog->ic_state = XLOG_STATE_DONE_SYNC; + } + + /* + * Someone could be sleeping prior to writing out the next + * iclog buffer, we wake them all, one will get to do the + * I/O, the others get to wait for the result. + */ + wake_up_all(&iclog->ic_write_wait); + spin_unlock(&log->l_icloglock); + xlog_state_do_callback(log); +} + +/* + * If the head of the in-core log ring is not (ACTIVE or DIRTY), then we must + * sleep. We wait on the flush queue on the head iclog as that should be + * the first iclog to complete flushing. Hence if all iclogs are syncing, + * we will wait here and all new writes will sleep until a sync completes. + * + * The in-core logs are used in a circular fashion. They are not used + * out-of-order even when an iclog past the head is free. + * + * return: + * * log_offset where xlog_write() can start writing into the in-core + * log's data space. + * * in-core log pointer to which xlog_write() should write. + * * boolean indicating this is a continued write to an in-core log. + * If this is the last write, then the in-core log's offset field + * needs to be incremented, depending on the amount of data which + * is copied. + */ +STATIC int +xlog_state_get_iclog_space( + struct xlog *log, + struct xlog_write_data *data) +{ + int log_offset; + struct xlog_rec_header *head; + struct xlog_in_core *iclog; + +restart: + spin_lock(&log->l_icloglock); + if (xlog_is_shutdown(log)) { + spin_unlock(&log->l_icloglock); + return -EIO; + } + + iclog = log->l_iclog; + if (iclog->ic_state != XLOG_STATE_ACTIVE) { + XFS_STATS_INC(log->l_mp, xs_log_noiclogs); + + /* Wait for log writes to have flushed */ + xlog_wait(&log->l_flush_wait, &log->l_icloglock); + goto restart; + } + + head = iclog->ic_header; + + atomic_inc(&iclog->ic_refcnt); /* prevents sync */ + log_offset = iclog->ic_offset; + + trace_xlog_iclog_get_space(iclog, _RET_IP_); + + /* On the 1st write to an iclog, figure out lsn. This works + * if iclogs marked XLOG_STATE_WANT_SYNC always write out what they are + * committing to. If the offset is set, that's how many blocks + * must be written. + */ + if (log_offset == 0) { + data->ticket->t_curr_res -= log->l_iclog_hsize; + head->h_cycle = cpu_to_be32(log->l_curr_cycle); + head->h_lsn = cpu_to_be64( + xlog_assign_lsn(log->l_curr_cycle, log->l_curr_block)); + ASSERT(log->l_curr_block >= 0); + } + + /* If there is enough room to write everything, then do it. Otherwise, + * claim the rest of the region and make sure the XLOG_STATE_WANT_SYNC + * bit is on, so this will get flushed out. Don't update ic_offset + * until you know exactly how many bytes get copied. Therefore, wait + * until later to update ic_offset. + * + * xlog_write() algorithm assumes that at least 2 xlog_op_header's + * can fit into remaining data section. + */ + if (iclog->ic_size - iclog->ic_offset < + 2 * sizeof(struct xlog_op_header)) { + int error = 0; + + xlog_state_switch_iclogs(log, iclog, iclog->ic_size); + + /* + * If we are the only one writing to this iclog, sync it to + * disk. We need to do an atomic compare and decrement here to + * avoid racing with concurrent atomic_dec_and_lock() calls in + * xlog_state_release_iclog() when there is more than one + * reference to the iclog. + */ + if (!atomic_add_unless(&iclog->ic_refcnt, -1, 1)) + error = xlog_state_release_iclog(log, iclog, + data->ticket); + spin_unlock(&log->l_icloglock); + if (error) + return error; + goto restart; + } + + /* Do we have enough room to write the full amount in the remainder + * of this iclog? Or must we continue a write on the next iclog and + * mark this iclog as completely taken? In the case where we switch + * iclogs (to mark it taken), this particular iclog will release/sync + * to disk in xlog_write(). + */ + if (data->bytes_left <= iclog->ic_size - iclog->ic_offset) + iclog->ic_offset += data->bytes_left; + else + xlog_state_switch_iclogs(log, iclog, iclog->ic_size); + data->iclog = iclog; + + ASSERT(iclog->ic_offset <= iclog->ic_size); + spin_unlock(&log->l_icloglock); + + data->log_offset = log_offset; + return 0; +} + +/* + * The first cnt-1 times a ticket goes through here we don't need to move the + * grant write head because the permanent reservation has reserved cnt times the + * unit amount. Release part of current permanent unit reservation and reset + * current reservation to be one units worth. Also move grant reservation head + * forward. + */ +void +xfs_log_ticket_regrant( + struct xlog *log, + struct xlog_ticket *ticket) +{ + trace_xfs_log_ticket_regrant(log, ticket); + + if (ticket->t_cnt > 0) + ticket->t_cnt--; + + xlog_grant_sub_space(&log->l_reserve_head, ticket->t_curr_res); + xlog_grant_sub_space(&log->l_write_head, ticket->t_curr_res); + ticket->t_curr_res = ticket->t_unit_res; + + trace_xfs_log_ticket_regrant_sub(log, ticket); + + /* just return if we still have some of the pre-reserved space */ + if (!ticket->t_cnt) { + xlog_grant_add_space(&log->l_reserve_head, ticket->t_unit_res); + trace_xfs_log_ticket_regrant_exit(log, ticket); + } + + xfs_log_ticket_put(ticket); +} + +/* + * Give back the space left from a reservation. + * + * All the information we need to make a correct determination of space left + * is present. For non-permanent reservations, things are quite easy. The + * count should have been decremented to zero. We only need to deal with the + * space remaining in the current reservation part of the ticket. If the + * ticket contains a permanent reservation, there may be left over space which + * needs to be released. A count of N means that N-1 refills of the current + * reservation can be done before we need to ask for more space. The first + * one goes to fill up the first current reservation. Once we run out of + * space, the count will stay at zero and the only space remaining will be + * in the current reservation field. + */ +void +xfs_log_ticket_ungrant( + struct xlog *log, + struct xlog_ticket *ticket) +{ + int bytes; + + trace_xfs_log_ticket_ungrant(log, ticket); + + if (ticket->t_cnt > 0) + ticket->t_cnt--; + + trace_xfs_log_ticket_ungrant_sub(log, ticket); + + /* + * If this is a permanent reservation ticket, we may be able to free + * up more space based on the remaining count. + */ + bytes = ticket->t_curr_res; + if (ticket->t_cnt > 0) { + ASSERT(ticket->t_flags & XLOG_TIC_PERM_RESERV); + bytes += ticket->t_unit_res*ticket->t_cnt; + } + + xlog_grant_sub_space(&log->l_reserve_head, bytes); + xlog_grant_sub_space(&log->l_write_head, bytes); + + trace_xfs_log_ticket_ungrant_exit(log, ticket); + + xfs_log_space_wake(log->l_mp); + xfs_log_ticket_put(ticket); +} + +/* + * This routine will mark the current iclog in the ring as WANT_SYNC and move + * the current iclog pointer to the next iclog in the ring. + */ +void +xlog_state_switch_iclogs( + struct xlog *log, + struct xlog_in_core *iclog, + int eventual_size) +{ + ASSERT(iclog->ic_state == XLOG_STATE_ACTIVE); + assert_spin_locked(&log->l_icloglock); + trace_xlog_iclog_switch(iclog, _RET_IP_); + + if (!eventual_size) + eventual_size = iclog->ic_offset; + iclog->ic_state = XLOG_STATE_WANT_SYNC; + iclog->ic_header->h_prev_block = cpu_to_be32(log->l_prev_block); + log->l_prev_block = log->l_curr_block; + log->l_prev_cycle = log->l_curr_cycle; + + /* roll log?: ic_offset changed later */ + log->l_curr_block += BTOBB(eventual_size)+BTOBB(log->l_iclog_hsize); + + /* Round up to next log-sunit */ + if (log->l_iclog_roundoff > BBSIZE) { + uint32_t sunit_bb = BTOBB(log->l_iclog_roundoff); + log->l_curr_block = roundup(log->l_curr_block, sunit_bb); + } + + if (log->l_curr_block >= log->l_logBBsize) { + /* + * Rewind the current block before the cycle is bumped to make + * sure that the combined LSN never transiently moves forward + * when the log wraps to the next cycle. This is to support the + * unlocked sample of these fields from xlog_valid_lsn(). Most + * other cases should acquire l_icloglock. + */ + log->l_curr_block -= log->l_logBBsize; + ASSERT(log->l_curr_block >= 0); + smp_wmb(); + log->l_curr_cycle++; + if (log->l_curr_cycle == XLOG_HEADER_MAGIC_NUM) + log->l_curr_cycle++; + } + ASSERT(iclog == log->l_iclog); + log->l_iclog = iclog->ic_next; +} + +/* + * Force the iclog to disk and check if the iclog has been completed before + * xlog_force_iclog() returns. This can happen on synchronous (e.g. + * pmem) or fast async storage because we drop the icloglock to issue the IO. + * If completion has already occurred, tell the caller so that it can avoid an + * unnecessary wait on the iclog. + */ +static int +xlog_force_and_check_iclog( + struct xlog_in_core *iclog, + bool *completed) +{ + xfs_lsn_t lsn = be64_to_cpu(iclog->ic_header->h_lsn); + int error; + + *completed = false; + error = xlog_force_iclog(iclog); + if (error) + return error; + + /* + * If the iclog has already been completed and reused the header LSN + * will have been rewritten by completion + */ + if (be64_to_cpu(iclog->ic_header->h_lsn) != lsn) + *completed = true; + return 0; +} + +/* + * Write out all data in the in-core log as of this exact moment in time. + * + * Data may be written to the in-core log during this call. However, + * we don't guarantee this data will be written out. A change from past + * implementation means this routine will *not* write out zero length LRs. + * + * Basically, we try and perform an intelligent scan of the in-core logs. + * If we determine there is no flushable data, we just return. There is no + * flushable data if: + * + * 1. the current iclog is active and has no data; the previous iclog + * is in the active or dirty state. + * 2. the current iclog is dirty, and the previous iclog is in the + * active or dirty state. + * + * We may sleep if: + * + * 1. the current iclog is not in the active nor dirty state. + * 2. the current iclog dirty, and the previous iclog is not in the + * active nor dirty state. + * 3. the current iclog is active, and there is another thread writing + * to this particular iclog. + * 4. a) the current iclog is active and has no other writers + * b) when we return from flushing out this iclog, it is still + * not in the active nor dirty state. + */ +int +xfs_log_force( + struct xfs_mount *mp, + uint flags) +{ + struct xlog *log = mp->m_log; + struct xlog_in_core *iclog; + + XFS_STATS_INC(mp, xs_log_force); + trace_xfs_log_force(mp, 0, _RET_IP_); + + xlog_cil_force(log); + + spin_lock(&log->l_icloglock); + if (xlog_is_shutdown(log)) + goto out_error; + + iclog = log->l_iclog; + trace_xlog_iclog_force(iclog, _RET_IP_); + + if (iclog->ic_state == XLOG_STATE_DIRTY || + (iclog->ic_state == XLOG_STATE_ACTIVE && + atomic_read(&iclog->ic_refcnt) == 0 && iclog->ic_offset == 0)) { + /* + * If the head is dirty or (active and empty), then we need to + * look at the previous iclog. + * + * If the previous iclog is active or dirty we are done. There + * is nothing to sync out. Otherwise, we attach ourselves to the + * previous iclog and go to sleep. + */ + iclog = iclog->ic_prev; + } else if (iclog->ic_state == XLOG_STATE_ACTIVE) { + if (atomic_read(&iclog->ic_refcnt) == 0) { + /* We have exclusive access to this iclog. */ + bool completed; + + if (xlog_force_and_check_iclog(iclog, &completed)) + goto out_error; + + if (completed) + goto out_unlock; + } else { + /* + * Someone else is still writing to this iclog, so we + * need to ensure that when they release the iclog it + * gets synced immediately as we may be waiting on it. + */ + xlog_state_switch_iclogs(log, iclog, 0); + } + } + + /* + * The iclog we are about to wait on may contain the checkpoint pushed + * by the above xlog_cil_force() call, but it may not have been pushed + * to disk yet. Like the ACTIVE case above, we need to make sure caches + * are flushed when this iclog is written. + */ + if (iclog->ic_state == XLOG_STATE_WANT_SYNC) + iclog->ic_flags |= XLOG_ICL_NEED_FLUSH | XLOG_ICL_NEED_FUA; + + if (flags & XFS_LOG_SYNC) + return xlog_wait_on_iclog(iclog); +out_unlock: + spin_unlock(&log->l_icloglock); + return 0; +out_error: + spin_unlock(&log->l_icloglock); + return -EIO; +} + +/* + * Force the log to a specific LSN. + * + * If an iclog with that lsn can be found: + * If it is in the DIRTY state, just return. + * If it is in the ACTIVE state, move the in-core log into the WANT_SYNC + * state and go to sleep or return. + * If it is in any other state, go to sleep or return. + * + * Synchronous forces are implemented with a wait queue. All callers trying + * to force a given lsn to disk must wait on the queue attached to the + * specific in-core log. When given in-core log finally completes its write + * to disk, that thread will wake up all threads waiting on the queue. + */ +static int +xlog_force_lsn( + struct xlog *log, + xfs_lsn_t lsn, + uint flags, + int *log_flushed, + bool already_slept) +{ + struct xlog_in_core *iclog; + bool completed; + + spin_lock(&log->l_icloglock); + if (xlog_is_shutdown(log)) + goto out_error; + + iclog = log->l_iclog; + while (be64_to_cpu(iclog->ic_header->h_lsn) != lsn) { + trace_xlog_iclog_force_lsn(iclog, _RET_IP_); + iclog = iclog->ic_next; + if (iclog == log->l_iclog) + goto out_unlock; + } + + switch (iclog->ic_state) { + case XLOG_STATE_ACTIVE: + /* + * We sleep here if we haven't already slept (e.g. this is the + * first time we've looked at the correct iclog buf) and the + * buffer before us is going to be sync'ed. The reason for this + * is that if we are doing sync transactions here, by waiting + * for the previous I/O to complete, we can allow a few more + * transactions into this iclog before we close it down. + * + * Otherwise, we mark the buffer WANT_SYNC, and bump up the + * refcnt so we can release the log (which drops the ref count). + * The state switch keeps new transaction commits from using + * this buffer. When the current commits finish writing into + * the buffer, the refcount will drop to zero and the buffer + * will go out then. + */ + if (!already_slept && + (iclog->ic_prev->ic_state == XLOG_STATE_WANT_SYNC || + iclog->ic_prev->ic_state == XLOG_STATE_SYNCING)) { + xlog_wait(&iclog->ic_prev->ic_write_wait, + &log->l_icloglock); + return -EAGAIN; + } + if (xlog_force_and_check_iclog(iclog, &completed)) + goto out_error; + if (log_flushed) + *log_flushed = 1; + if (completed) + goto out_unlock; + break; + case XLOG_STATE_WANT_SYNC: + /* + * This iclog may contain the checkpoint pushed by the + * xlog_cil_force_seq() call, but there are other writers still + * accessing it so it hasn't been pushed to disk yet. Like the + * ACTIVE case above, we need to make sure caches are flushed + * when this iclog is written. + */ + iclog->ic_flags |= XLOG_ICL_NEED_FLUSH | XLOG_ICL_NEED_FUA; + break; + default: + /* + * The entire checkpoint was written by the CIL force and is on + * its way to disk already. It will be stable when it + * completes, so we don't need to manipulate caches here at all. + * We just need to wait for completion if necessary. + */ + break; + } + + if (flags & XFS_LOG_SYNC) + return xlog_wait_on_iclog(iclog); +out_unlock: + spin_unlock(&log->l_icloglock); + return 0; +out_error: + spin_unlock(&log->l_icloglock); + return -EIO; +} + +/* + * Force the log to a specific checkpoint sequence. + * + * First force the CIL so that all the required changes have been flushed to the + * iclogs. If the CIL force completed it will return a commit LSN that indicates + * the iclog that needs to be flushed to stable storage. If the caller needs + * a synchronous log force, we will wait on the iclog with the LSN returned by + * xlog_cil_force_seq() to be completed. + */ +int +xfs_log_force_seq( + struct xfs_mount *mp, + xfs_csn_t seq, + uint flags, + int *log_flushed) +{ + struct xlog *log = mp->m_log; + xfs_lsn_t lsn; + int ret; + ASSERT(seq != 0); + + XFS_STATS_INC(mp, xs_log_force); + trace_xfs_log_force(mp, seq, _RET_IP_); + + lsn = xlog_cil_force_seq(log, seq); + if (lsn == NULLCOMMITLSN) + return 0; + + ret = xlog_force_lsn(log, lsn, flags, log_flushed, false); + if (ret == -EAGAIN) { + XFS_STATS_INC(mp, xs_log_force_sleep); + ret = xlog_force_lsn(log, lsn, flags, log_flushed, true); + } + return ret; +} + +/* + * Free a used ticket when its refcount falls to zero. + */ +void +xfs_log_ticket_put( + struct xlog_ticket *ticket) +{ + ASSERT(atomic_read(&ticket->t_ref) > 0); + if (atomic_dec_and_test(&ticket->t_ref)) + kmem_cache_free(xfs_log_ticket_cache, ticket); +} + +struct xlog_ticket * +xfs_log_ticket_get( + struct xlog_ticket *ticket) +{ + ASSERT(atomic_read(&ticket->t_ref) > 0); + atomic_inc(&ticket->t_ref); + return ticket; +} + +/* + * Figure out the total log space unit (in bytes) that would be + * required for a log ticket. + */ +static int +xlog_calc_unit_res( + struct xlog *log, + int unit_bytes, + int *niclogs) +{ + int iclog_space; + uint num_headers; + + /* + * Permanent reservations have up to 'cnt'-1 active log operations + * in the log. A unit in this case is the amount of space for one + * of these log operations. Normal reservations have a cnt of 1 + * and their unit amount is the total amount of space required. + * + * The following lines of code account for non-transaction data + * which occupy space in the on-disk log. + * + * Normal form of a transaction is: + * ... + * and then there are LR hdrs, split-recs and roundoff at end of syncs. + * + * We need to account for all the leadup data and trailer data + * around the transaction data. + * And then we need to account for the worst case in terms of using + * more space. + * The worst case will happen if: + * - the placement of the transaction happens to be such that the + * roundoff is at its maximum + * - the transaction data is synced before the commit record is synced + * i.e. | + * Therefore the commit record is in its own Log Record. + * This can happen as the commit record is called with its + * own region to xlog_write(). + * This then means that in the worst case, roundoff can happen for + * the commit-rec as well. + * The commit-rec is smaller than padding in this scenario and so it is + * not added separately. + */ + + /* for trans header */ + unit_bytes += sizeof(struct xlog_op_header); + unit_bytes += sizeof(struct xfs_trans_header); + + /* for start-rec */ + unit_bytes += sizeof(struct xlog_op_header); + + /* + * for LR headers - the space for data in an iclog is the size minus + * the space used for the headers. If we use the iclog size, then we + * undercalculate the number of headers required. + * + * Furthermore - the addition of op headers for split-recs might + * increase the space required enough to require more log and op + * headers, so take that into account too. + * + * IMPORTANT: This reservation makes the assumption that if this + * transaction is the first in an iclog and hence has the LR headers + * accounted to it, then the remaining space in the iclog is + * exclusively for this transaction. i.e. if the transaction is larger + * than the iclog, it will be the only thing in that iclog. + * Fundamentally, this means we must pass the entire log vector to + * xlog_write to guarantee this. + */ + iclog_space = log->l_iclog_size - log->l_iclog_hsize; + num_headers = howmany(unit_bytes, iclog_space); + + /* for split-recs - ophdrs added when data split over LRs */ + unit_bytes += sizeof(struct xlog_op_header) * num_headers; + + /* add extra header reservations if we overrun */ + while (!num_headers || + howmany(unit_bytes, iclog_space) > num_headers) { + unit_bytes += sizeof(struct xlog_op_header); + num_headers++; + } + unit_bytes += log->l_iclog_hsize * num_headers; + + /* for commit-rec LR header - note: padding will subsume the ophdr */ + unit_bytes += log->l_iclog_hsize; + + /* roundoff padding for transaction data and one for commit record */ + unit_bytes += 2 * log->l_iclog_roundoff; + + if (niclogs) + *niclogs = num_headers; + return unit_bytes; +} + +int +xfs_log_calc_unit_res( + struct xfs_mount *mp, + int unit_bytes) +{ + return xlog_calc_unit_res(mp->m_log, unit_bytes, NULL); +} + +/* + * Allocate and initialise a new log ticket. + */ +struct xlog_ticket * +xlog_ticket_alloc( + struct xlog *log, + int unit_bytes, + int cnt, + bool permanent) +{ + struct xlog_ticket *tic; + int unit_res; + + tic = kmem_cache_zalloc(xfs_log_ticket_cache, + GFP_KERNEL | __GFP_NOFAIL); + + unit_res = xlog_calc_unit_res(log, unit_bytes, &tic->t_iclog_hdrs); + + atomic_set(&tic->t_ref, 1); + tic->t_task = current; + INIT_LIST_HEAD(&tic->t_queue); + tic->t_unit_res = unit_res; + tic->t_curr_res = unit_res; + tic->t_cnt = cnt; + tic->t_ocnt = cnt; + tic->t_tid = get_random_u32(); + if (permanent) + tic->t_flags |= XLOG_TIC_PERM_RESERV; + + return tic; +} + +#if defined(DEBUG) +static void +xlog_verify_dump_tail( + struct xlog *log, + struct xlog_in_core *iclog) +{ + xfs_alert(log->l_mp, +"ran out of log space tail 0x%llx/0x%llx, head lsn 0x%llx, head 0x%x/0x%x, prev head 0x%x/0x%x", + iclog ? be64_to_cpu(iclog->ic_header->h_tail_lsn) : -1, + atomic64_read(&log->l_tail_lsn), + log->l_ailp->ail_head_lsn, + log->l_curr_cycle, log->l_curr_block, + log->l_prev_cycle, log->l_prev_block); + xfs_alert(log->l_mp, +"write grant 0x%llx, reserve grant 0x%llx, tail_space 0x%llx, size 0x%x, iclog flags 0x%x", + atomic64_read(&log->l_write_head.grant), + atomic64_read(&log->l_reserve_head.grant), + log->l_tail_space, log->l_logsize, + iclog ? iclog->ic_flags : -1); +} + +/* Check if the new iclog will fit in the log. */ +STATIC void +xlog_verify_tail_lsn( + struct xlog *log, + struct xlog_in_core *iclog) +{ + xfs_lsn_t tail_lsn = be64_to_cpu(iclog->ic_header->h_tail_lsn); + int blocks; + + if (CYCLE_LSN(tail_lsn) == log->l_prev_cycle) { + blocks = log->l_logBBsize - + (log->l_prev_block - BLOCK_LSN(tail_lsn)); + if (blocks < BTOBB(iclog->ic_offset) + + BTOBB(log->l_iclog_hsize)) { + xfs_emerg(log->l_mp, + "%s: ran out of log space", __func__); + xlog_verify_dump_tail(log, iclog); + } + return; + } + + if (CYCLE_LSN(tail_lsn) + 1 != log->l_prev_cycle) { + xfs_emerg(log->l_mp, "%s: head has wrapped tail.", __func__); + xlog_verify_dump_tail(log, iclog); + return; + } + if (BLOCK_LSN(tail_lsn) == log->l_prev_block) { + xfs_emerg(log->l_mp, "%s: tail wrapped", __func__); + xlog_verify_dump_tail(log, iclog); + return; + } + + blocks = BLOCK_LSN(tail_lsn) - log->l_prev_block; + if (blocks < BTOBB(iclog->ic_offset) + 1) { + xfs_emerg(log->l_mp, "%s: ran out of iclog space", __func__); + xlog_verify_dump_tail(log, iclog); + } +} + +/* + * Perform a number of checks on the iclog before writing to disk. + * + * 1. Make sure the iclogs are still circular + * 2. Make sure we have a good magic number + * 3. Make sure we don't have magic numbers in the data + * 4. Check fields of each log operation header for: + * A. Valid client identifier + * B. tid ptr value falls in valid ptr space (user space code) + * C. Length in log record header is correct according to the + * individual operation headers within record. + * 5. When a bwrite will occur within 5 blocks of the front of the physical + * log, check the preceding blocks of the physical log to make sure all + * the cycle numbers agree with the current cycle number. + */ +STATIC void +xlog_verify_iclog( + struct xlog *log, + struct xlog_in_core *iclog, + int count) +{ + struct xlog_rec_header *rhead = iclog->ic_header; + struct xlog_in_core *icptr; + void *base_ptr, *ptr; + ptrdiff_t field_offset; + uint8_t clientid; + int len, i, op_len; + int idx; + + /* check validity of iclog pointers */ + spin_lock(&log->l_icloglock); + icptr = log->l_iclog; + for (i = 0; i < log->l_iclog_bufs; i++, icptr = icptr->ic_next) + ASSERT(icptr); + + if (icptr != log->l_iclog) + xfs_emerg(log->l_mp, "%s: corrupt iclog ring", __func__); + spin_unlock(&log->l_icloglock); + + /* check log magic numbers */ + if (rhead->h_magicno != cpu_to_be32(XLOG_HEADER_MAGIC_NUM)) + xfs_emerg(log->l_mp, "%s: invalid magic num", __func__); + + base_ptr = ptr = rhead; + for (ptr += BBSIZE; ptr < base_ptr + count; ptr += BBSIZE) { + if (*(__be32 *)ptr == cpu_to_be32(XLOG_HEADER_MAGIC_NUM)) + xfs_emerg(log->l_mp, "%s: unexpected magic num", + __func__); + } + + /* check fields */ + len = be32_to_cpu(rhead->h_num_logops); + base_ptr = ptr = iclog->ic_datap; + for (i = 0; i < len; i++) { + struct xlog_op_header *ophead = ptr; + void *p = &ophead->oh_clientid; + + /* clientid is only 1 byte */ + field_offset = p - base_ptr; + if (field_offset & 0x1ff) { + clientid = ophead->oh_clientid; + } else { + idx = BTOBBT((void *)&ophead->oh_clientid - iclog->ic_datap); + clientid = xlog_get_client_id(*xlog_cycle_data(rhead, idx)); + } + if (clientid != XFS_TRANSACTION && clientid != XFS_LOG) { + xfs_warn(log->l_mp, + "%s: op %d invalid clientid %d op "PTR_FMT" offset 0x%lx", + __func__, i, clientid, ophead, + (unsigned long)field_offset); + } + + /* check length */ + p = &ophead->oh_len; + field_offset = p - base_ptr; + if (field_offset & 0x1ff) { + op_len = be32_to_cpu(ophead->oh_len); + } else { + idx = BTOBBT((void *)&ophead->oh_len - iclog->ic_datap); + op_len = be32_to_cpu(*xlog_cycle_data(rhead, idx)); + } + ptr += sizeof(struct xlog_op_header) + op_len; + } +} +#endif + +/* + * Perform a forced shutdown on the log. + * + * This can be called from low level log code to trigger a shutdown, or from the + * high level mount shutdown code when the mount shuts down. + * + * Our main objectives here are to make sure that: + * a. if the shutdown was not due to a log IO error, flush the logs to + * disk. Anything modified after this is ignored. + * b. the log gets atomically marked 'XLOG_IO_ERROR' for all interested + * parties to find out. Nothing new gets queued after this is done. + * c. Tasks sleeping on log reservations, pinned objects and + * other resources get woken up. + * d. The mount is also marked as shut down so that log triggered shutdowns + * still behave the same as if they called xfs_forced_shutdown(). + * + * Return true if the shutdown cause was a log IO error and we actually shut the + * log down. + */ +bool +xlog_force_shutdown( + struct xlog *log, + uint32_t shutdown_flags) +{ + bool log_error = (shutdown_flags & SHUTDOWN_LOG_IO_ERROR); + + if (!log) + return false; + + /* + * Ensure that there is only ever one log shutdown being processed. + * If we allow the log force below on a second pass after shutting + * down the log, we risk deadlocking the CIL push as it may require + * locks on objects the current shutdown context holds (e.g. taking + * buffer locks to abort buffers on last unpin of buf log items). + */ + if (test_and_set_bit(XLOG_SHUTDOWN_STARTED, &log->l_opstate)) + return false; + + /* + * Flush all the completed transactions to disk before marking the log + * being shut down. We need to do this first as shutting down the log + * before the force will prevent the log force from flushing the iclogs + * to disk. + * + * When we are in recovery, there are no transactions to flush, and + * we don't want to touch the log because we don't want to perturb the + * current head/tail for future recovery attempts. Hence we need to + * avoid a log force in this case. + * + * If we are shutting down due to a log IO error, then we must avoid + * trying to write the log as that may just result in more IO errors and + * an endless shutdown/force loop. + */ + if (!log_error && !xlog_in_recovery(log)) + xfs_log_force(log->l_mp, XFS_LOG_SYNC); + + /* + * Atomically set the shutdown state. If the shutdown state is already + * set, there someone else is performing the shutdown and so we are done + * here. This should never happen because we should only ever get called + * once by the first shutdown caller. + * + * Much of the log state machine transitions assume that shutdown state + * cannot change once they hold the log->l_icloglock. Hence we need to + * hold that lock here, even though we use the atomic test_and_set_bit() + * operation to set the shutdown state. + */ + spin_lock(&log->l_icloglock); + if (test_and_set_bit(XLOG_IO_ERROR, &log->l_opstate)) { + spin_unlock(&log->l_icloglock); + ASSERT(0); + return false; + } + spin_unlock(&log->l_icloglock); + + /* + * If this log shutdown also sets the mount shutdown state, issue a + * shutdown warning message. + */ + if (!xfs_set_shutdown(log->l_mp)) { + xfs_alert_tag(log->l_mp, XFS_PTAG_SHUTDOWN_LOGERROR, +"Filesystem has been shut down due to log error (0x%x).", + shutdown_flags); + xfs_alert(log->l_mp, +"Please unmount the filesystem and rectify the problem(s)."); + if (xfs_error_level >= XFS_ERRLEVEL_HIGH) + xfs_stack_trace(); + } + + /* + * We don't want anybody waiting for log reservations after this. That + * means we have to wake up everybody queued up on reserveq as well as + * writeq. In addition, we make sure in xlog_{re}grant_log_space that + * we don't enqueue anything once the SHUTDOWN flag is set, and this + * action is protected by the grant locks. + */ + xlog_grant_head_wake_all(&log->l_reserve_head); + xlog_grant_head_wake_all(&log->l_write_head); + + /* + * Wake up everybody waiting on xfs_log_force. Wake the CIL push first + * as if the log writes were completed. The abort handling in the log + * item committed callback functions will do this again under lock to + * avoid races. + */ + spin_lock(&log->l_cilp->xc_push_lock); + wake_up_all(&log->l_cilp->xc_start_wait); + wake_up_all(&log->l_cilp->xc_commit_wait); + spin_unlock(&log->l_cilp->xc_push_lock); + + spin_lock(&log->l_icloglock); + xlog_state_shutdown_callbacks(log); + spin_unlock(&log->l_icloglock); + + wake_up_var(&log->l_opstate); + if (IS_ENABLED(CONFIG_XFS_RT) && xfs_has_zoned(log->l_mp)) + xfs_zoned_wake_all(log->l_mp); + + return log_error; +} + +STATIC int +xlog_iclogs_empty( + struct xlog *log) +{ + struct xlog_in_core *iclog = log->l_iclog; + + do { + /* endianness does not matter here, zero is zero in + * any language. + */ + if (iclog->ic_header->h_num_logops) + return 0; + iclog = iclog->ic_next; + } while (iclog != log->l_iclog); + + return 1; +} + +/* + * Verify that an LSN stamped into a piece of metadata is valid. This is + * intended for use in read verifiers on v5 superblocks. + */ +bool +xfs_log_check_lsn( + struct xfs_mount *mp, + xfs_lsn_t lsn) +{ + struct xlog *log = mp->m_log; + bool valid; + + /* + * norecovery mode skips mount-time log processing and unconditionally + * resets the in-core LSN. We can't validate in this mode, but + * modifications are not allowed anyways so just return true. + */ + if (xfs_has_norecovery(mp)) + return true; + + /* + * Some metadata LSNs are initialized to NULL (e.g., the agfl). This is + * handled by recovery and thus safe to ignore here. + */ + if (lsn == NULLCOMMITLSN) + return true; + + valid = xlog_valid_lsn(mp->m_log, lsn); + + /* warn the user about what's gone wrong before verifier failure */ + if (!valid) { + spin_lock(&log->l_icloglock); + xfs_warn(mp, +"Corruption warning: Metadata has LSN (%d:%d) ahead of current LSN (%d:%d). " +"Please unmount and run xfs_repair (>= v4.3) to resolve.", + CYCLE_LSN(lsn), BLOCK_LSN(lsn), + log->l_curr_cycle, log->l_curr_block); + spin_unlock(&log->l_icloglock); + } + + return valid; +} diff --git a/fs/xfs/xfs_log.h b/fs/xfs/xfs_log.h new file mode 100644 index 000000000..ca66429bf --- /dev/null +++ b/fs/xfs/xfs_log.h @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2003,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_LOG_H__ +#define __XFS_LOG_H__ + +struct xlog_format_buf; +struct xfs_cil_ctx; + +/* Region types for iovec's i_type */ +#define XLOG_REG_TYPE_BFORMAT 1 +#define XLOG_REG_TYPE_BCHUNK 2 +#define XLOG_REG_TYPE_EFI_FORMAT 3 +#define XLOG_REG_TYPE_EFD_FORMAT 4 +#define XLOG_REG_TYPE_IFORMAT 5 +#define XLOG_REG_TYPE_ICORE 6 +#define XLOG_REG_TYPE_IEXT 7 +#define XLOG_REG_TYPE_IBROOT 8 +#define XLOG_REG_TYPE_ILOCAL 9 +#define XLOG_REG_TYPE_IATTR_EXT 10 +#define XLOG_REG_TYPE_IATTR_BROOT 11 +#define XLOG_REG_TYPE_IATTR_LOCAL 12 +#define XLOG_REG_TYPE_QFORMAT 13 +#define XLOG_REG_TYPE_DQUOT 14 +#define XLOG_REG_TYPE_QUOTAOFF 15 +#define XLOG_REG_TYPE_LRHEADER 16 +#define XLOG_REG_TYPE_UNMOUNT 17 +#define XLOG_REG_TYPE_COMMIT 18 +#define XLOG_REG_TYPE_TRANSHDR 19 +#define XLOG_REG_TYPE_ICREATE 20 +#define XLOG_REG_TYPE_RUI_FORMAT 21 +#define XLOG_REG_TYPE_RUD_FORMAT 22 +#define XLOG_REG_TYPE_CUI_FORMAT 23 +#define XLOG_REG_TYPE_CUD_FORMAT 24 +#define XLOG_REG_TYPE_BUI_FORMAT 25 +#define XLOG_REG_TYPE_BUD_FORMAT 26 +#define XLOG_REG_TYPE_ATTRI_FORMAT 27 +#define XLOG_REG_TYPE_ATTRD_FORMAT 28 +#define XLOG_REG_TYPE_ATTR_NAME 29 +#define XLOG_REG_TYPE_ATTR_VALUE 30 +#define XLOG_REG_TYPE_XMI_FORMAT 31 +#define XLOG_REG_TYPE_XMD_FORMAT 32 +#define XLOG_REG_TYPE_ATTR_NEWNAME 33 +#define XLOG_REG_TYPE_ATTR_NEWVALUE 34 +#define XLOG_REG_TYPE_MAX 34 + +#define XFS_LOG_VEC_ORDERED (-1) + +/* + * Calculate the log iovec length for a given user buffer length. Intended to be + * used by ->iop_size implementations when sizing buffers of arbitrary + * alignments. + */ +static inline int +xlog_calc_iovec_len(int len) +{ + return roundup(len, sizeof(uint32_t)); +} + +void *xlog_format_start(struct xlog_format_buf *lfb, uint16_t type); +void xlog_format_commit(struct xlog_format_buf *lfb, unsigned int data_len); + +/* + * Copy the amount of data requested by the caller into a new log iovec. + */ +static inline void * +xlog_format_copy( + struct xlog_format_buf *lfb, + uint16_t type, + void *data, + unsigned int len) +{ + void *buf; + + buf = xlog_format_start(lfb, type); + memcpy(buf, data, len); + xlog_format_commit(lfb, len); + return buf; +} + +/* + * Flags to xfs_log_force() + * + * XFS_LOG_SYNC: Synchronous force in-core log to disk + */ +#define XFS_LOG_SYNC 0x1 + +/* Log manager interfaces */ +struct xfs_mount; +struct xlog_in_core; +struct xlog_ticket; +struct xfs_log_item; +struct xfs_item_ops; +struct xfs_trans; +struct xlog; + +int xfs_log_force(struct xfs_mount *mp, uint flags); +int xfs_log_force_seq(struct xfs_mount *mp, xfs_csn_t seq, uint flags, + int *log_forced); +int xfs_log_mount(struct xfs_mount *mp, + struct xfs_buftarg *log_target, + xfs_daddr_t start_block, + int num_bblocks); +int xfs_log_mount_finish(struct xfs_mount *mp); +void xfs_log_mount_cancel(struct xfs_mount *); +xfs_lsn_t xlog_assign_tail_lsn(struct xfs_mount *mp); +xfs_lsn_t xlog_assign_tail_lsn_locked(struct xfs_mount *mp); +void xfs_log_space_wake(struct xfs_mount *mp); +int xfs_log_reserve(struct xfs_mount *mp, int length, int count, + struct xlog_ticket **ticket, bool permanent); +int xfs_log_regrant(struct xfs_mount *mp, struct xlog_ticket *tic); +void xfs_log_unmount(struct xfs_mount *mp); +bool xfs_log_writable(struct xfs_mount *mp); + +struct xlog_ticket *xfs_log_ticket_get(struct xlog_ticket *ticket); +void xfs_log_ticket_put(struct xlog_ticket *ticket); + +void xlog_cil_process_committed(struct list_head *list); +bool xfs_log_item_in_current_chkpt(struct xfs_log_item *lip); + +void xfs_log_work_queue(struct xfs_mount *mp); +int xfs_log_quiesce(struct xfs_mount *mp); +void xfs_log_clean(struct xfs_mount *mp); +bool xfs_log_check_lsn(struct xfs_mount *, xfs_lsn_t); + +bool xlog_force_shutdown(struct xlog *log, uint32_t shutdown_flags); + +#endif /* __XFS_LOG_H__ */ diff --git a/fs/xfs/xfs_log_cil.c b/fs/xfs/xfs_log_cil.c new file mode 100644 index 000000000..639f875a8 --- /dev/null +++ b/fs/xfs/xfs_log_cil.c @@ -0,0 +1,2069 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2010 Red Hat, Inc. All Rights Reserved. + */ + +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_shared.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_extent_busy.h" +#include "xfs_trans.h" +#include "xfs_trans_priv.h" +#include "xfs_log.h" +#include "xfs_log_priv.h" +#include "xfs_trace.h" +#include "xfs_discard.h" + +/* + * Allocate a new ticket. Failing to get a new ticket makes it really hard to + * recover, so we don't allow failure here. Also, we allocate in a context that + * we don't want to be issuing transactions from, so we need to tell the + * allocation code this as well. + * + * We don't reserve any space for the ticket - we are going to steal whatever + * space we require from transactions as they commit. To ensure we reserve all + * the space required, we need to set the current reservation of the ticket to + * zero so that we know to steal the initial transaction overhead from the + * first transaction commit. + */ +static struct xlog_ticket * +xlog_cil_ticket_alloc( + struct xlog *log) +{ + struct xlog_ticket *tic; + + tic = xlog_ticket_alloc(log, 0, 1, 0); + + /* + * set the current reservation to zero so we know to steal the basic + * transaction overhead reservation from the first transaction commit. + */ + tic->t_curr_res = 0; + tic->t_iclog_hdrs = 0; + return tic; +} + +static inline void +xlog_cil_set_iclog_hdr_count(struct xfs_cil *cil) +{ + struct xlog *log = cil->xc_log; + + atomic_set(&cil->xc_iclog_hdrs, + (XLOG_CIL_BLOCKING_SPACE_LIMIT(log) / + (log->l_iclog_size - log->l_iclog_hsize))); +} + +/* + * Check if the current log item was first committed in this sequence. + * We can't rely on just the log item being in the CIL, we have to check + * the recorded commit sequence number. + * + * Note: for this to be used in a non-racy manner, it has to be called with + * CIL flushing locked out. As a result, it should only be used during the + * transaction commit process when deciding what to format into the item. + */ +static bool +xlog_item_in_current_chkpt( + struct xfs_cil *cil, + struct xfs_log_item *lip) +{ + if (test_bit(XLOG_CIL_EMPTY, &cil->xc_flags)) + return false; + + /* + * li_seq is written on the first commit of a log item to record the + * first checkpoint it is written to. Hence if it is different to the + * current sequence, we're in a new checkpoint. + */ + return lip->li_seq == READ_ONCE(cil->xc_current_sequence); +} + +bool +xfs_log_item_in_current_chkpt( + struct xfs_log_item *lip) +{ + return xlog_item_in_current_chkpt(lip->li_log->l_cilp, lip); +} + +/* + * Unavoidable forward declaration - xlog_cil_push_work() calls + * xlog_cil_ctx_alloc() itself. + */ +static void xlog_cil_push_work(struct work_struct *work); + +static struct xfs_cil_ctx * +xlog_cil_ctx_alloc(void) +{ + struct xfs_cil_ctx *ctx; + + ctx = kzalloc_obj(*ctx, GFP_KERNEL | __GFP_NOFAIL); + INIT_LIST_HEAD(&ctx->committing); + INIT_LIST_HEAD(&ctx->busy_extents.extent_list); + INIT_LIST_HEAD(&ctx->log_items); + INIT_LIST_HEAD(&ctx->lv_chain); + INIT_WORK(&ctx->push_work, xlog_cil_push_work); + return ctx; +} + +/* + * Aggregate the CIL per cpu structures into global counts, lists, etc and + * clear the percpu state ready for the next context to use. This is called + * from the push code with the context lock held exclusively, hence nothing else + * will be accessing or modifying the per-cpu counters. + */ +static void +xlog_cil_push_pcp_aggregate( + struct xfs_cil *cil, + struct xfs_cil_ctx *ctx) +{ + struct xlog_cil_pcp *cilpcp; + int cpu; + + for_each_cpu(cpu, &ctx->cil_pcpmask) { + cilpcp = per_cpu_ptr(cil->xc_pcp, cpu); + + ctx->ticket->t_curr_res += cilpcp->space_reserved; + cilpcp->space_reserved = 0; + + if (!list_empty(&cilpcp->busy_extents)) { + list_splice_init(&cilpcp->busy_extents, + &ctx->busy_extents.extent_list); + } + if (!list_empty(&cilpcp->log_items)) + list_splice_init(&cilpcp->log_items, &ctx->log_items); + + /* + * We're in the middle of switching cil contexts. Reset the + * counter we use to detect when the current context is nearing + * full. + */ + cilpcp->space_used = 0; + } +} + +/* + * Aggregate the CIL per-cpu space used counters into the global atomic value. + * This is called when the per-cpu counter aggregation will first pass the soft + * limit threshold so we can switch to atomic counter aggregation for accurate + * detection of hard limit traversal. + */ +static void +xlog_cil_insert_pcp_aggregate( + struct xfs_cil *cil, + struct xfs_cil_ctx *ctx) +{ + int cpu; + int count = 0; + + /* Trigger atomic updates then aggregate only for the first caller */ + if (!test_and_clear_bit(XLOG_CIL_PCP_SPACE, &cil->xc_flags)) + return; + + /* + * We can race with other cpus setting cil_pcpmask. However, we've + * atomically cleared PCP_SPACE which forces other threads to add to + * the global space used count. cil_pcpmask is a superset of cilpcp + * structures that could have a nonzero space_used. + */ + for_each_cpu(cpu, &ctx->cil_pcpmask) { + struct xlog_cil_pcp *cilpcp = per_cpu_ptr(cil->xc_pcp, cpu); + + count += xchg(&cilpcp->space_used, 0); + } + atomic_add(count, &ctx->space_used); +} + +static void +xlog_cil_ctx_switch( + struct xfs_cil *cil, + struct xfs_cil_ctx *ctx) +{ + xlog_cil_set_iclog_hdr_count(cil); + set_bit(XLOG_CIL_EMPTY, &cil->xc_flags); + set_bit(XLOG_CIL_PCP_SPACE, &cil->xc_flags); + ctx->sequence = ++cil->xc_current_sequence; + ctx->cil = cil; + cil->xc_ctx = ctx; +} + +/* + * After the first stage of log recovery is done, we know where the head and + * tail of the log are. We need this log initialisation done before we can + * initialise the first CIL checkpoint context. + * + * Here we allocate a log ticket to track space usage during a CIL push. This + * ticket is passed to xlog_write() directly so that we don't slowly leak log + * space by failing to account for space used by log headers and additional + * region headers for split regions. + */ +void +xlog_cil_init_post_recovery( + struct xlog *log) +{ + log->l_cilp->xc_ctx->ticket = xlog_cil_ticket_alloc(log); + log->l_cilp->xc_ctx->sequence = 1; + xlog_cil_set_iclog_hdr_count(log->l_cilp); +} + +static inline int +xlog_cil_iovec_space( + uint niovecs) +{ + return round_up((sizeof(struct xfs_log_vec) + + niovecs * sizeof(struct xfs_log_iovec)), + sizeof(uint64_t)); +} + +/* + * Allocate or pin log vector buffers for CIL insertion. + * + * The CIL currently uses disposable buffers for copying a snapshot of the + * modified items into the log during a push. The biggest problem with this is + * the requirement to allocate the disposable buffer during the commit if: + * a) does not exist; or + * b) it is too small + * + * If we do this allocation within xlog_cil_insert_format_items(), it is done + * under the xc_ctx_lock, which means that a CIL push cannot occur during + * the memory allocation. This means that we have a potential deadlock situation + * under low memory conditions when we have lots of dirty metadata pinned in + * the CIL and we need a CIL commit to occur to free memory. + * + * To avoid this, we need to move the memory allocation outside the + * xc_ctx_lock, but because the log vector buffers are disposable, that opens + * up a TOCTOU race condition w.r.t. the CIL committing and removing the log + * vector buffers between the check and the formatting of the item into the + * log vector buffer within the xc_ctx_lock. + * + * Because the log vector buffer needs to be unchanged during the CIL push + * process, we cannot share the buffer between the transaction commit (which + * modifies the buffer) and the CIL push context that is writing the changes + * into the log. This means skipping preallocation of buffer space is + * unreliable, but we most definitely do not want to be allocating and freeing + * buffers unnecessarily during commits when overwrites can be done safely. + * + * The simplest solution to this problem is to allocate a shadow buffer when a + * log item is committed for the second time, and then to only use this buffer + * if necessary. The buffer can remain attached to the log item until such time + * it is needed, and this is the buffer that is reallocated to match the size of + * the incoming modification. Then during the formatting of the item we can swap + * the active buffer with the new one if we can't reuse the existing buffer. We + * don't free the old buffer as it may be reused on the next modification if + * it's size is right, otherwise we'll free and reallocate it at that point. + * + * This function builds a vector for the changes in each log item in the + * transaction. It then works out the length of the buffer needed for each log + * item, allocates them and attaches the vector to the log item in preparation + * for the formatting step which occurs under the xc_ctx_lock. + * + * While this means the memory footprint goes up, it avoids the repeated + * alloc/free pattern that repeated modifications of an item would otherwise + * cause, and hence minimises the CPU overhead of such behaviour. + */ +static void +xlog_cil_alloc_shadow_bufs( + struct xlog *log, + struct xfs_trans *tp) +{ + struct xfs_log_item *lip; + + list_for_each_entry(lip, &tp->t_items, li_trans) { + struct xfs_log_vec *lv; + int niovecs = 0; + int nbytes = 0; + int alloc_size; + bool ordered = false; + + /* Skip items which aren't dirty in this transaction. */ + if (!test_bit(XFS_LI_DIRTY, &lip->li_flags)) + continue; + + /* get number of vecs and size of data to be stored */ + lip->li_ops->iop_size(lip, &niovecs, &nbytes); + + /* + * Ordered items need to be tracked but we do not wish to write + * them. We need a logvec to track the object, but we do not + * need an iovec or buffer to be allocated for copying data. + */ + if (niovecs == XFS_LOG_VEC_ORDERED) { + ordered = true; + niovecs = 0; + nbytes = 0; + } + + /* + * We 64-bit align the length of each iovec so that the start of + * the next one is naturally aligned. We'll need to account for + * that slack space here. + * + * We also add the xlog_op_header to each region when + * formatting, but that's not accounted to the size of the item + * at this point. Hence we'll need an addition number of bytes + * for each vector to hold an opheader. + * + * Then round nbytes up to 64-bit alignment so that the initial + * buffer alignment is easy to calculate and verify. + */ + nbytes = xlog_item_space(niovecs, nbytes); + + /* + * The data buffer needs to start 64-bit aligned, so round up + * that space to ensure we can align it appropriately and not + * overrun the buffer. + */ + alloc_size = nbytes + xlog_cil_iovec_space(niovecs); + + /* + * if we have no shadow buffer, or it is too small, we need to + * reallocate it. + */ + if (!lip->li_lv_shadow || + alloc_size > lip->li_lv_shadow->lv_alloc_size) { + /* + * We free and allocate here as a realloc would copy + * unnecessary data. We don't use kvzalloc() for the + * same reason - we don't need to zero the data area in + * the buffer, only the log vector header and the iovec + * storage. + */ + kvfree(lip->li_lv_shadow); + lv = xlog_kvmalloc(alloc_size); + + memset(lv, 0, xlog_cil_iovec_space(niovecs)); + + INIT_LIST_HEAD(&lv->lv_list); + lv->lv_item = lip; + lv->lv_alloc_size = alloc_size; + if (ordered) + lv->lv_buf_used = XFS_LOG_VEC_ORDERED; + else + lv->lv_iovecp = (struct xfs_log_iovec *)&lv[1]; + lip->li_lv_shadow = lv; + } else { + /* same or smaller, optimise common overwrite case */ + lv = lip->li_lv_shadow; + if (ordered) + lv->lv_buf_used = XFS_LOG_VEC_ORDERED; + else + lv->lv_buf_used = 0; + lv->lv_bytes = 0; + } + + /* Ensure the lv is set up according to ->iop_size */ + lv->lv_niovecs = niovecs; + + /* The allocated data region lies beyond the iovec region */ + lv->lv_buf = (char *)lv + xlog_cil_iovec_space(niovecs); + } + +} + +/* + * Prepare the log item for insertion into the CIL. Calculate the difference in + * log space it will consume, and if it is a new item pin it as well. + */ +STATIC void +xfs_cil_prepare_item( + struct xlog *log, + struct xfs_log_item *lip, + struct xfs_log_vec *lv, + int *diff_len) +{ + /* Account for the new LV being passed in */ + if (lv->lv_buf_used != XFS_LOG_VEC_ORDERED) + *diff_len += lv->lv_bytes; + + /* + * If there is no old LV, this is the first time we've seen the item in + * this CIL context and so we need to pin it. If we are replacing the + * old lv, then remove the space it accounts for and make it the shadow + * buffer for later freeing. In both cases we are now switching to the + * shadow buffer, so update the pointer to it appropriately. + */ + if (!lip->li_lv) { + if (lv->lv_item->li_ops->iop_pin) + lv->lv_item->li_ops->iop_pin(lv->lv_item); + lv->lv_item->li_lv_shadow = NULL; + } else if (lip->li_lv != lv) { + ASSERT(lv->lv_buf_used != XFS_LOG_VEC_ORDERED); + + *diff_len -= lip->li_lv->lv_bytes; + lv->lv_item->li_lv_shadow = lip->li_lv; + } + + /* attach new log vector to log item */ + lv->lv_item->li_lv = lv; + + /* + * If this is the first time the item is being committed to the + * CIL, store the sequence number on the log item so we can + * tell in future commits whether this is the first checkpoint + * the item is being committed into. + */ + if (!lv->lv_item->li_seq) + lv->lv_item->li_seq = log->l_cilp->xc_ctx->sequence; +} + +struct xlog_format_buf { + struct xfs_log_vec *lv; + unsigned int idx; +}; + +/* + * We need to make sure the buffer pointer returned is naturally aligned for the + * biggest basic data type we put into it. We have already accounted for this + * padding when sizing the buffer. + * + * However, this padding does not get written into the log, and hence we have to + * track the space used by the log vectors separately to prevent log space hangs + * due to inaccurate accounting (i.e. a leak) of the used log space through the + * CIL context ticket. + * + * We also add space for the xlog_op_header that describes this region in the + * log. This prepends the data region we return to the caller to copy their data + * into, so do all the static initialisation of the ophdr now. Because the ophdr + * is not 8 byte aligned, we have to be careful to ensure that we align the + * start of the buffer such that the region we return to the call is 8 byte + * aligned and packed against the tail of the ophdr. + */ +void * +xlog_format_start( + struct xlog_format_buf *lfb, + uint16_t type) +{ + struct xfs_log_vec *lv = lfb->lv; + struct xfs_log_iovec *vec = &lv->lv_iovecp[lfb->idx]; + struct xlog_op_header *oph; + uint32_t len; + void *buf; + + ASSERT(lfb->idx < lv->lv_niovecs); + + len = lv->lv_buf_used + sizeof(struct xlog_op_header); + if (!IS_ALIGNED(len, sizeof(uint64_t))) { + lv->lv_buf_used = round_up(len, sizeof(uint64_t)) - + sizeof(struct xlog_op_header); + } + + vec->i_type = type; + vec->i_addr = lv->lv_buf + lv->lv_buf_used; + + oph = vec->i_addr; + oph->oh_clientid = XFS_TRANSACTION; + oph->oh_res2 = 0; + oph->oh_flags = 0; + + buf = vec->i_addr + sizeof(struct xlog_op_header); + ASSERT(IS_ALIGNED((unsigned long)buf, sizeof(uint64_t))); + return buf; +} + +void +xlog_format_commit( + struct xlog_format_buf *lfb, + unsigned int data_len) +{ + struct xfs_log_vec *lv = lfb->lv; + struct xfs_log_iovec *vec = &lv->lv_iovecp[lfb->idx]; + struct xlog_op_header *oph = vec->i_addr; + int len; + + /* + * Always round up the length to the correct alignment so callers don't + * need to know anything about this log vec layout requirement. This + * means we have to zero the area the data to be written does not cover. + * This is complicated by fact the payload region is offset into the + * logvec region by the opheader that tracks the payload. + */ + len = xlog_calc_iovec_len(data_len); + if (len - data_len != 0) { + char *buf = vec->i_addr + sizeof(struct xlog_op_header); + + memset(buf + data_len, 0, len - data_len); + } + + /* + * The opheader tracks aligned payload length, whilst the logvec tracks + * the overall region length. + */ + oph->oh_len = cpu_to_be32(len); + + len += sizeof(struct xlog_op_header); + lv->lv_buf_used += len; + lv->lv_bytes += len; + vec->i_len = len; + + /* Catch buffer overruns */ + ASSERT((void *)lv->lv_buf + lv->lv_bytes <= + (void *)lv + lv->lv_alloc_size); + + lfb->idx++; +} + +/* + * Format log item into a flat buffers + * + * For delayed logging, we need to hold a formatted buffer containing all the + * changes on the log item. This enables us to relog the item in memory and + * write it out asynchronously without needing to relock the object that was + * modified at the time it gets written into the iclog. + * + * This function takes the prepared log vectors attached to each log item, and + * formats the changes into the log vector buffer. The buffer it uses is + * dependent on the current state of the vector in the CIL - the shadow lv is + * guaranteed to be large enough for the current modification, but we will only + * use that if we can't reuse the existing lv. If we can't reuse the existing + * lv, then simple swap it out for the shadow lv. We don't free it - that is + * done lazily either by th enext modification or the freeing of the log item. + * + * We don't set up region headers during this process; we simply copy the + * regions into the flat buffer. We can do this because we still have to do a + * formatting step to write the regions into the iclog buffer. Writing the + * ophdrs during the iclog write means that we can support splitting large + * regions across iclog boundares without needing a change in the format of the + * item/region encapsulation. + * + * Hence what we need to do now is change the rewrite the vector array to point + * to the copied region inside the buffer we just allocated. This allows us to + * format the regions into the iclog as though they are being formatted + * directly out of the objects themselves. + */ +static void +xlog_cil_insert_format_items( + struct xlog *log, + struct xfs_trans *tp, + int *diff_len) +{ + struct xfs_log_item *lip; + + /* Bail out if we didn't find a log item. */ + if (list_empty(&tp->t_items)) { + ASSERT(0); + return; + } + + list_for_each_entry(lip, &tp->t_items, li_trans) { + struct xfs_log_vec *lv = lip->li_lv; + struct xfs_log_vec *shadow = lip->li_lv_shadow; + struct xlog_format_buf lfb = { }; + + /* Skip items which aren't dirty in this transaction. */ + if (!test_bit(XFS_LI_DIRTY, &lip->li_flags)) + continue; + + /* + * The formatting size information is already attached to + * the shadow lv on the log item. + */ + if (shadow->lv_buf_used == XFS_LOG_VEC_ORDERED) { + if (!lv) { + lv = shadow; + lv->lv_item = lip; + } + ASSERT(shadow->lv_alloc_size == lv->lv_alloc_size); + xfs_cil_prepare_item(log, lip, lv, diff_len); + continue; + } + + /* Skip items that do not have any vectors for writing */ + if (!shadow->lv_niovecs) + continue; + + /* compare to existing item size */ + if (lv && shadow->lv_alloc_size <= lv->lv_alloc_size) { + /* same or smaller, optimise common overwrite case */ + + /* + * set the item up as though it is a new insertion so + * that the space reservation accounting is correct. + */ + *diff_len -= lv->lv_bytes; + + /* Ensure the lv is set up according to ->iop_size */ + lv->lv_niovecs = shadow->lv_niovecs; + + /* reset the lv buffer information for new formatting */ + lv->lv_buf_used = 0; + lv->lv_bytes = 0; + lv->lv_buf = (char *)lv + + xlog_cil_iovec_space(lv->lv_niovecs); + } else { + /* switch to shadow buffer! */ + lv = shadow; + lv->lv_item = lip; + } + + lfb.lv = lv; + ASSERT(IS_ALIGNED((unsigned long)lv->lv_buf, sizeof(uint64_t))); + lip->li_ops->iop_format(lip, &lfb); + xfs_cil_prepare_item(log, lip, lv, diff_len); + } +} + +/* + * The use of lockless waitqueue_active() requires that the caller has + * serialised itself against the wakeup call in xlog_cil_push_work(). That + * can be done by either holding the push lock or the context lock. + */ +static inline bool +xlog_cil_over_hard_limit( + struct xlog *log, + int32_t space_used) +{ + if (waitqueue_active(&log->l_cilp->xc_push_wait)) + return true; + if (space_used >= XLOG_CIL_BLOCKING_SPACE_LIMIT(log)) + return true; + return false; +} + +/* + * Insert the log items into the CIL and calculate the difference in space + * consumed by the item. Add the space to the checkpoint ticket and calculate + * if the change requires additional log metadata. If it does, take that space + * as well. Remove the amount of space we added to the checkpoint ticket from + * the current transaction ticket so that the accounting works out correctly. + */ +static void +xlog_cil_insert_items( + struct xlog *log, + struct xfs_trans *tp, + uint32_t released_space) +{ + struct xfs_cil *cil = log->l_cilp; + struct xfs_cil_ctx *ctx = cil->xc_ctx; + struct xfs_log_item *lip; + int len = 0; + int iovhdr_res = 0, split_res = 0, ctx_res = 0; + int space_used; + int order; + unsigned int cpu_nr; + struct xlog_cil_pcp *cilpcp; + + ASSERT(tp); + + /* + * We can do this safely because the context can't checkpoint until we + * are done so it doesn't matter exactly how we update the CIL. + */ + xlog_cil_insert_format_items(log, tp, &len); + + /* + * Subtract the space released by intent cancelation from the space we + * consumed so that we remove it from the CIL space and add it back to + * the current transaction reservation context. + */ + len -= released_space; + + /* + * Grab the per-cpu pointer for the CIL before we start any accounting. + * That ensures that we are running with pre-emption disabled and so we + * can't be scheduled away between split sample/update operations that + * are done without outside locking to serialise them. + */ + cpu_nr = get_cpu(); + cilpcp = this_cpu_ptr(cil->xc_pcp); + + /* Tell the future push that there was work added by this CPU. */ + if (!cpumask_test_cpu(cpu_nr, &ctx->cil_pcpmask)) + cpumask_test_and_set_cpu(cpu_nr, &ctx->cil_pcpmask); + + /* + * We need to take the CIL checkpoint unit reservation on the first + * commit into the CIL. Test the XLOG_CIL_EMPTY bit first so we don't + * unnecessarily do an atomic op in the fast path here. We can clear the + * XLOG_CIL_EMPTY bit as we are under the xc_ctx_lock here and that + * needs to be held exclusively to reset the XLOG_CIL_EMPTY bit. + */ + if (test_bit(XLOG_CIL_EMPTY, &cil->xc_flags) && + test_and_clear_bit(XLOG_CIL_EMPTY, &cil->xc_flags)) + ctx_res = ctx->ticket->t_unit_res; + + /* + * Check if we need to steal iclog headers. atomic_read() is not a + * locked atomic operation, so we can check the value before we do any + * real atomic ops in the fast path. If we've already taken the CIL unit + * reservation from this commit, we've already got one iclog header + * space reserved so we have to account for that otherwise we risk + * overrunning the reservation on this ticket. + * + * If the CIL is already at the hard limit, we might need more header + * space that originally reserved. So steal more header space from every + * commit that occurs once we are over the hard limit to ensure the CIL + * push won't run out of reservation space. + * + * This can steal more than we need, but that's OK. + * + * The cil->xc_ctx_lock provides the serialisation necessary for safely + * calling xlog_cil_over_hard_limit() in this context. + */ + space_used = atomic_read(&ctx->space_used) + cilpcp->space_used + len; + if (atomic_read(&cil->xc_iclog_hdrs) > 0 || + xlog_cil_over_hard_limit(log, space_used)) { + split_res = log->l_iclog_hsize + + sizeof(struct xlog_op_header); + if (ctx_res) + ctx_res += split_res * (tp->t_ticket->t_iclog_hdrs - 1); + else + ctx_res = split_res * tp->t_ticket->t_iclog_hdrs; + atomic_sub(tp->t_ticket->t_iclog_hdrs, &cil->xc_iclog_hdrs); + } + cilpcp->space_reserved += ctx_res; + + /* + * Accurately account when over the soft limit, otherwise fold the + * percpu count into the global count if over the per-cpu threshold. + */ + if (!test_bit(XLOG_CIL_PCP_SPACE, &cil->xc_flags)) { + atomic_add(len, &ctx->space_used); + } else if (cilpcp->space_used + len > + (XLOG_CIL_SPACE_LIMIT(log) / num_online_cpus())) { + space_used = atomic_add_return(cilpcp->space_used + len, + &ctx->space_used); + cilpcp->space_used = 0; + + /* + * If we just transitioned over the soft limit, we need to + * transition to the global atomic counter. + */ + if (space_used >= XLOG_CIL_SPACE_LIMIT(log)) + xlog_cil_insert_pcp_aggregate(cil, ctx); + } else { + cilpcp->space_used += len; + } + /* attach the transaction to the CIL if it has any busy extents */ + if (!list_empty(&tp->t_busy)) + list_splice_init(&tp->t_busy, &cilpcp->busy_extents); + + /* + * Now update the order of everything modified in the transaction + * and insert items into the CIL if they aren't already there. + * We do this here so we only need to take the CIL lock once during + * the transaction commit. + */ + order = atomic_inc_return(&ctx->order_id); + list_for_each_entry(lip, &tp->t_items, li_trans) { + /* Skip items which aren't dirty in this transaction. */ + if (!test_bit(XFS_LI_DIRTY, &lip->li_flags)) + continue; + + lip->li_order_id = order; + if (!list_empty(&lip->li_cil)) + continue; + list_add_tail(&lip->li_cil, &cilpcp->log_items); + } + put_cpu(); + + /* + * If we've overrun the reservation, dump the tx details before we move + * the log items. Shutdown is imminent... + */ + tp->t_ticket->t_curr_res -= ctx_res + len; + if (WARN_ON(tp->t_ticket->t_curr_res < 0)) { + xfs_warn(log->l_mp, "Transaction log reservation overrun:"); + xfs_warn(log->l_mp, + " log items: %d bytes (iov hdrs: %d bytes)", + len, iovhdr_res); + xfs_warn(log->l_mp, " split region headers: %d bytes", + split_res); + xfs_warn(log->l_mp, " ctx ticket: %d bytes", ctx_res); + xlog_print_trans(tp); + xlog_force_shutdown(log, SHUTDOWN_LOG_IO_ERROR); + } +} + +static inline void +xlog_cil_ail_insert_batch( + struct xfs_ail *ailp, + struct xfs_ail_cursor *cur, + struct xfs_log_item **log_items, + int nr_items, + xfs_lsn_t commit_lsn) +{ + int i; + + spin_lock(&ailp->ail_lock); + /* xfs_trans_ail_update_bulk drops ailp->ail_lock */ + xfs_trans_ail_update_bulk(ailp, cur, log_items, nr_items, commit_lsn); + + for (i = 0; i < nr_items; i++) { + struct xfs_log_item *lip = log_items[i]; + + if (lip->li_ops->iop_unpin) + lip->li_ops->iop_unpin(lip, 0); + } +} + +/* + * Take the checkpoint's log vector chain of items and insert the attached log + * items into the AIL. This uses bulk insertion techniques to minimise AIL lock + * traffic. + * + * The AIL tracks log items via the start record LSN of the checkpoint, + * not the commit record LSN. This is because we can pipeline multiple + * checkpoints, and so the start record of checkpoint N+1 can be + * written before the commit record of checkpoint N. i.e: + * + * start N commit N + * +-------------+------------+----------------+ + * start N+1 commit N+1 + * + * The tail of the log cannot be moved to the LSN of commit N when all + * the items of that checkpoint are written back, because then the + * start record for N+1 is no longer in the active portion of the log + * and recovery will fail/corrupt the filesystem. + * + * Hence when all the log items in checkpoint N are written back, the + * tail of the log most now only move as far forwards as the start LSN + * of checkpoint N+1. + * + * If we are called with the aborted flag set, it is because a log write during + * a CIL checkpoint commit has failed. In this case, all the items in the + * checkpoint have already gone through iop_committed and iop_committing, which + * means that checkpoint commit abort handling is treated exactly the same as an + * iclog write error even though we haven't started any IO yet. Hence in this + * case all we need to do is iop_committed processing, followed by an + * iop_unpin(aborted) call. + * + * The AIL cursor is used to optimise the insert process. If commit_lsn is not + * at the end of the AIL, the insert cursor avoids the need to walk the AIL to + * find the insertion point on every xfs_log_item_batch_insert() call. This + * saves a lot of needless list walking and is a net win, even though it + * slightly increases that amount of AIL lock traffic to set it up and tear it + * down. + */ +static void +xlog_cil_ail_insert( + struct xfs_cil_ctx *ctx, + bool aborted) +{ +#define LOG_ITEM_BATCH_SIZE 32 + struct xfs_ail *ailp = ctx->cil->xc_log->l_ailp; + struct xfs_log_item *log_items[LOG_ITEM_BATCH_SIZE]; + struct xfs_log_vec *lv; + struct xfs_ail_cursor cur; + xfs_lsn_t old_head; + int i = 0; + + /* + * Update the AIL head LSN with the commit record LSN of this + * checkpoint. As iclogs are always completed in order, this should + * always be the same (as iclogs can contain multiple commit records) or + * higher LSN than the current head. We do this before insertion of the + * items so that log space checks during insertion will reflect the + * space that this checkpoint has already consumed. We call + * xfs_ail_update_finish() so that tail space and space-based wakeups + * will be recalculated appropriately. + */ + ASSERT(XFS_LSN_CMP(ctx->commit_lsn, ailp->ail_head_lsn) >= 0 || + aborted); + spin_lock(&ailp->ail_lock); + xfs_trans_ail_cursor_last(ailp, &cur, ctx->start_lsn); + old_head = ailp->ail_head_lsn; + ailp->ail_head_lsn = ctx->commit_lsn; + /* xfs_ail_update_finish() drops the ail_lock */ + xfs_ail_update_finish(ailp, NULLCOMMITLSN); + + /* + * We move the AIL head forwards to account for the space used in the + * log before we remove that space from the grant heads. This prevents a + * transient condition where reservation space appears to become + * available on return, only for it to disappear again immediately as + * the AIL head update accounts in the log tail space. + */ + smp_wmb(); /* paired with smp_rmb in xlog_grant_space_left */ + xlog_grant_return_space(ailp->ail_log, old_head, ailp->ail_head_lsn); + + /* unpin all the log items */ + list_for_each_entry(lv, &ctx->lv_chain, lv_list) { + struct xfs_log_item *lip = lv->lv_item; + xfs_lsn_t item_lsn; + + if (aborted) { + trace_xlog_ail_insert_abort(lip); + set_bit(XFS_LI_ABORTED, &lip->li_flags); + } + + if (lip->li_ops->flags & XFS_ITEM_RELEASE_WHEN_COMMITTED) { + lip->li_ops->iop_release(lip); + continue; + } + + if (lip->li_ops->iop_committed) + item_lsn = lip->li_ops->iop_committed(lip, + ctx->start_lsn); + else + item_lsn = ctx->start_lsn; + + /* item_lsn of -1 means the item needs no further processing */ + if (XFS_LSN_CMP(item_lsn, (xfs_lsn_t)-1) == 0) + continue; + + /* + * if we are aborting the operation, no point in inserting the + * object into the AIL as we are in a shutdown situation. + */ + if (aborted) { + ASSERT(xlog_is_shutdown(ailp->ail_log)); + if (lip->li_ops->iop_unpin) + lip->li_ops->iop_unpin(lip, 1); + continue; + } + + if (item_lsn != ctx->start_lsn) { + + /* + * Not a bulk update option due to unusual item_lsn. + * Push into AIL immediately, rechecking the lsn once + * we have the ail lock. Then unpin the item. This does + * not affect the AIL cursor the bulk insert path is + * using. + */ + spin_lock(&ailp->ail_lock); + if (XFS_LSN_CMP(item_lsn, lip->li_lsn) > 0) + xfs_trans_ail_update(ailp, lip, item_lsn); + else + spin_unlock(&ailp->ail_lock); + if (lip->li_ops->iop_unpin) + lip->li_ops->iop_unpin(lip, 0); + continue; + } + + /* Item is a candidate for bulk AIL insert. */ + log_items[i++] = lv->lv_item; + if (i >= LOG_ITEM_BATCH_SIZE) { + xlog_cil_ail_insert_batch(ailp, &cur, log_items, + LOG_ITEM_BATCH_SIZE, ctx->start_lsn); + i = 0; + } + } + + /* make sure we insert the remainder! */ + if (i) + xlog_cil_ail_insert_batch(ailp, &cur, log_items, i, + ctx->start_lsn); + + spin_lock(&ailp->ail_lock); + xfs_trans_ail_cursor_done(&cur); + spin_unlock(&ailp->ail_lock); +} + +static void +xlog_cil_free_logvec( + struct list_head *lv_chain) +{ + struct xfs_log_vec *lv; + + while (!list_empty(lv_chain)) { + lv = list_first_entry(lv_chain, struct xfs_log_vec, lv_list); + list_del_init(&lv->lv_list); + kvfree(lv); + } +} + +/* + * Mark all items committed and clear busy extents. We free the log vector + * chains in a separate pass so that we unpin the log items as quickly as + * possible. + */ +static void +xlog_cil_committed( + struct xfs_cil_ctx *ctx) +{ + struct xfs_mount *mp = ctx->cil->xc_log->l_mp; + bool abort = xlog_is_shutdown(ctx->cil->xc_log); + + /* + * If the I/O failed, we're aborting the commit and already shutdown. + * Wake any commit waiters before aborting the log items so we don't + * block async log pushers on callbacks. Async log pushers explicitly do + * not wait on log force completion because they may be holding locks + * required to unpin items. + */ + if (abort) { + spin_lock(&ctx->cil->xc_push_lock); + wake_up_all(&ctx->cil->xc_start_wait); + wake_up_all(&ctx->cil->xc_commit_wait); + spin_unlock(&ctx->cil->xc_push_lock); + } + + xlog_cil_ail_insert(ctx, abort); + + xfs_extent_busy_sort(&ctx->busy_extents.extent_list); + xfs_extent_busy_clear(&ctx->busy_extents.extent_list, + xfs_has_discard(mp) && !abort); + + spin_lock(&ctx->cil->xc_push_lock); + list_del(&ctx->committing); + spin_unlock(&ctx->cil->xc_push_lock); + + xlog_cil_free_logvec(&ctx->lv_chain); + + if (!list_empty(&ctx->busy_extents.extent_list)) { + ctx->busy_extents.owner = ctx; + xfs_discard_extents(mp, &ctx->busy_extents); + return; + } + + kfree(ctx); +} + +void +xlog_cil_process_committed( + struct list_head *list) +{ + struct xfs_cil_ctx *ctx; + + while ((ctx = list_first_entry_or_null(list, + struct xfs_cil_ctx, iclog_entry))) { + list_del(&ctx->iclog_entry); + xlog_cil_committed(ctx); + } +} + +/* +* Record the LSN of the iclog we were just granted space to start writing into. +* If the context doesn't have a start_lsn recorded, then this iclog will +* contain the start record for the checkpoint. Otherwise this write contains +* the commit record for the checkpoint. +*/ +void +xlog_cil_set_ctx_write_state( + struct xfs_cil_ctx *ctx, + struct xlog_in_core *iclog) +{ + struct xfs_cil *cil = ctx->cil; + xfs_lsn_t lsn = be64_to_cpu(iclog->ic_header->h_lsn); + + ASSERT(!ctx->commit_lsn); + if (!ctx->start_lsn) { + spin_lock(&cil->xc_push_lock); + /* + * The LSN we need to pass to the log items on transaction + * commit is the LSN reported by the first log vector write, not + * the commit lsn. If we use the commit record lsn then we can + * move the grant write head beyond the tail LSN and overwrite + * it. + */ + ctx->start_lsn = lsn; + wake_up_all(&cil->xc_start_wait); + spin_unlock(&cil->xc_push_lock); + + /* + * Make sure the metadata we are about to overwrite in the log + * has been flushed to stable storage before this iclog is + * issued. + */ + spin_lock(&cil->xc_log->l_icloglock); + iclog->ic_flags |= XLOG_ICL_NEED_FLUSH; + spin_unlock(&cil->xc_log->l_icloglock); + return; + } + + /* + * Take a reference to the iclog for the context so that we still hold + * it when xlog_write is done and has released it. This means the + * context controls when the iclog is released for IO. + */ + atomic_inc(&iclog->ic_refcnt); + + /* + * xlog_state_get_iclog_space() guarantees there is enough space in the + * iclog for an entire commit record, so we can attach the context + * callbacks now. This needs to be done before we make the commit_lsn + * visible to waiters so that checkpoints with commit records in the + * same iclog order their IO completion callbacks in the same order that + * the commit records appear in the iclog. + */ + spin_lock(&cil->xc_log->l_icloglock); + list_add_tail(&ctx->iclog_entry, &iclog->ic_callbacks); + spin_unlock(&cil->xc_log->l_icloglock); + + /* + * Now we can record the commit LSN and wake anyone waiting for this + * sequence to have the ordered commit record assigned to a physical + * location in the log. + */ + spin_lock(&cil->xc_push_lock); + ctx->commit_iclog = iclog; + ctx->commit_lsn = lsn; + wake_up_all(&cil->xc_commit_wait); + spin_unlock(&cil->xc_push_lock); +} + + +/* + * Ensure that the order of log writes follows checkpoint sequence order. This + * relies on the context LSN being zero until the log write has guaranteed the + * LSN that the log write will start at via xlog_state_get_iclog_space(). + */ +enum _record_type { + _START_RECORD, + _COMMIT_RECORD, +}; + +static int +xlog_cil_order_write( + struct xfs_cil *cil, + xfs_csn_t sequence, + enum _record_type record) +{ + struct xfs_cil_ctx *ctx; + +restart: + spin_lock(&cil->xc_push_lock); + list_for_each_entry(ctx, &cil->xc_committing, committing) { + /* + * Avoid getting stuck in this loop because we were woken by the + * shutdown, but then went back to sleep once already in the + * shutdown state. + */ + if (xlog_is_shutdown(cil->xc_log)) { + spin_unlock(&cil->xc_push_lock); + return -EIO; + } + + /* + * Higher sequences will wait for this one so skip them. + * Don't wait for our own sequence, either. + */ + if (ctx->sequence >= sequence) + continue; + + /* Wait until the LSN for the record has been recorded. */ + switch (record) { + case _START_RECORD: + if (!ctx->start_lsn) { + xlog_wait(&cil->xc_start_wait, &cil->xc_push_lock); + goto restart; + } + break; + case _COMMIT_RECORD: + if (!ctx->commit_lsn) { + xlog_wait(&cil->xc_commit_wait, &cil->xc_push_lock); + goto restart; + } + break; + } + } + spin_unlock(&cil->xc_push_lock); + return 0; +} + +/* + * Write out the log vector change now attached to the CIL context. This will + * write a start record that needs to be strictly ordered in ascending CIL + * sequence order so that log recovery will always use in-order start LSNs when + * replaying checkpoints. + */ +static int +xlog_cil_write_chain( + struct xfs_cil_ctx *ctx, + uint32_t chain_len) +{ + struct xlog *log = ctx->cil->xc_log; + int error; + + error = xlog_cil_order_write(ctx->cil, ctx->sequence, _START_RECORD); + if (error) + return error; + return xlog_write(log, ctx, &ctx->lv_chain, ctx->ticket, chain_len); +} + +/* + * Write out the commit record of a checkpoint transaction to close off a + * running log write. These commit records are strictly ordered in ascending CIL + * sequence order so that log recovery will always replay the checkpoints in the + * correct order. + */ +static int +xlog_cil_write_commit_record( + struct xfs_cil_ctx *ctx) +{ + struct xlog *log = ctx->cil->xc_log; + struct xlog_op_header ophdr = { + .oh_clientid = XFS_TRANSACTION, + .oh_tid = cpu_to_be32(ctx->ticket->t_tid), + .oh_flags = XLOG_COMMIT_TRANS, + }; + struct xfs_log_iovec reg = { + .i_addr = &ophdr, + .i_len = sizeof(struct xlog_op_header), + .i_type = XLOG_REG_TYPE_COMMIT, + }; + int error; + + if (xlog_is_shutdown(log)) + return -EIO; + + error = xlog_cil_order_write(ctx->cil, ctx->sequence, _COMMIT_RECORD); + if (error) + return error; + error = xlog_write_one_vec(log, ctx, ®, ctx->ticket); + if (error) + xlog_force_shutdown(log, SHUTDOWN_LOG_IO_ERROR); + return error; +} + +struct xlog_cil_trans_hdr { + struct xlog_op_header oph[2]; + struct xfs_trans_header thdr; + struct xfs_log_iovec lhdr[2]; +}; + +/* + * Build a checkpoint transaction header to begin the journal transaction. We + * need to account for the space used by the transaction header here as it is + * not accounted for in xlog_write(). + * + * This is the only place we write a transaction header, so we also build the + * log opheaders that indicate the start of a log transaction and wrap the + * transaction header. We keep the start record in it's own log vector rather + * than compacting them into a single region as this ends up making the logic + * in xlog_write() for handling empty opheaders for start, commit and unmount + * records much simpler. + */ +static void +xlog_cil_build_trans_hdr( + struct xfs_cil_ctx *ctx, + struct xlog_cil_trans_hdr *hdr, + struct xfs_log_vec *lvhdr, + int num_iovecs) +{ + struct xlog_ticket *tic = ctx->ticket; + __be32 tid = cpu_to_be32(tic->t_tid); + + memset(hdr, 0, sizeof(*hdr)); + + /* Log start record */ + hdr->oph[0].oh_tid = tid; + hdr->oph[0].oh_clientid = XFS_TRANSACTION; + hdr->oph[0].oh_flags = XLOG_START_TRANS; + + /* log iovec region pointer */ + hdr->lhdr[0].i_addr = &hdr->oph[0]; + hdr->lhdr[0].i_len = sizeof(struct xlog_op_header); + hdr->lhdr[0].i_type = XLOG_REG_TYPE_LRHEADER; + + /* log opheader */ + hdr->oph[1].oh_tid = tid; + hdr->oph[1].oh_clientid = XFS_TRANSACTION; + hdr->oph[1].oh_len = cpu_to_be32(sizeof(struct xfs_trans_header)); + + /* transaction header in host byte order format */ + hdr->thdr.th_magic = XFS_TRANS_HEADER_MAGIC; + hdr->thdr.th_type = XFS_TRANS_CHECKPOINT; + hdr->thdr.th_tid = tic->t_tid; + hdr->thdr.th_num_items = num_iovecs; + + /* log iovec region pointer */ + hdr->lhdr[1].i_addr = &hdr->oph[1]; + hdr->lhdr[1].i_len = sizeof(struct xlog_op_header) + + sizeof(struct xfs_trans_header); + hdr->lhdr[1].i_type = XLOG_REG_TYPE_TRANSHDR; + + lvhdr->lv_niovecs = 2; + lvhdr->lv_iovecp = &hdr->lhdr[0]; + lvhdr->lv_bytes = hdr->lhdr[0].i_len + hdr->lhdr[1].i_len; + + tic->t_curr_res -= lvhdr->lv_bytes; +} + +/* + * CIL item reordering compare function. We want to order in ascending ID order, + * but we want to leave items with the same ID in the order they were added to + * the list. This is important for operations like reflink where we log 4 order + * dependent intents in a single transaction when we overwrite an existing + * shared extent with a new shared extent. i.e. BUI(unmap), CUI(drop), + * CUI (inc), BUI(remap)... + */ +static int +xlog_cil_order_cmp( + void *priv, + const struct list_head *a, + const struct list_head *b) +{ + struct xfs_log_vec *l1 = container_of(a, struct xfs_log_vec, lv_list); + struct xfs_log_vec *l2 = container_of(b, struct xfs_log_vec, lv_list); + + return l1->lv_order_id > l2->lv_order_id; +} + +/* + * Pull all the log vectors off the items in the CIL, and remove the items from + * the CIL. We don't need the CIL lock here because it's only needed on the + * transaction commit side which is currently locked out by the flush lock. + * + * If a log item is marked with a whiteout, we do not need to write it to the + * journal and so we just move them to the whiteout list for the caller to + * dispose of appropriately. + */ +static void +xlog_cil_build_lv_chain( + struct xfs_cil_ctx *ctx, + struct list_head *whiteouts, + uint32_t *num_iovecs, + uint32_t *num_bytes) +{ + while (!list_empty(&ctx->log_items)) { + struct xfs_log_item *item; + struct xfs_log_vec *lv; + + item = list_first_entry(&ctx->log_items, + struct xfs_log_item, li_cil); + + if (test_bit(XFS_LI_WHITEOUT, &item->li_flags)) { + list_move(&item->li_cil, whiteouts); + trace_xfs_cil_whiteout_skip(item); + continue; + } + + lv = item->li_lv; + lv->lv_order_id = item->li_order_id; + + /* we don't write ordered log vectors */ + if (lv->lv_buf_used != XFS_LOG_VEC_ORDERED) + *num_bytes += lv->lv_bytes; + *num_iovecs += lv->lv_niovecs; + list_add_tail(&lv->lv_list, &ctx->lv_chain); + + list_del_init(&item->li_cil); + item->li_order_id = 0; + item->li_lv = NULL; + } +} + +static void +xlog_cil_cleanup_whiteouts( + struct list_head *whiteouts) +{ + while (!list_empty(whiteouts)) { + struct xfs_log_item *item = list_first_entry(whiteouts, + struct xfs_log_item, li_cil); + list_del_init(&item->li_cil); + trace_xfs_cil_whiteout_unpin(item); + item->li_ops->iop_unpin(item, 1); + } +} + +/* + * Push the Committed Item List to the log. + * + * If the current sequence is the same as xc_push_seq we need to do a flush. If + * xc_push_seq is less than the current sequence, then it has already been + * flushed and we don't need to do anything - the caller will wait for it to + * complete if necessary. + * + * xc_push_seq is checked unlocked against the sequence number for a match. + * Hence we can allow log forces to run racily and not issue pushes for the + * same sequence twice. If we get a race between multiple pushes for the same + * sequence they will block on the first one and then abort, hence avoiding + * needless pushes. + * + * This runs from a workqueue so it does not inherent any specific memory + * allocation context. However, we do not want to block on memory reclaim + * recursing back into the filesystem because this push may have been triggered + * by memory reclaim itself. Hence we really need to run under full GFP_NOFS + * contraints here. + */ +static void +xlog_cil_push_work( + struct work_struct *work) +{ + unsigned int nofs_flags = memalloc_nofs_save(); + struct xfs_cil_ctx *ctx = + container_of(work, struct xfs_cil_ctx, push_work); + struct xfs_cil *cil = ctx->cil; + struct xlog *log = cil->xc_log; + struct xfs_cil_ctx *new_ctx; + int num_iovecs = 0; + int num_bytes = 0; + int error = 0; + struct xlog_cil_trans_hdr thdr; + struct xfs_log_vec lvhdr = {}; + xfs_csn_t push_seq; + bool push_commit_stable; + LIST_HEAD (whiteouts); + struct xlog_ticket *ticket; + + new_ctx = xlog_cil_ctx_alloc(); + new_ctx->ticket = xlog_cil_ticket_alloc(log); + + down_write(&cil->xc_ctx_lock); + + spin_lock(&cil->xc_push_lock); + push_seq = cil->xc_push_seq; + ASSERT(push_seq <= ctx->sequence); + push_commit_stable = cil->xc_push_commit_stable; + cil->xc_push_commit_stable = false; + + /* + * As we are about to switch to a new, empty CIL context, we no longer + * need to throttle tasks on CIL space overruns. Wake any waiters that + * the hard push throttle may have caught so they can start committing + * to the new context. The ctx->xc_push_lock provides the serialisation + * necessary for safely using the lockless waitqueue_active() check in + * this context. + */ + if (waitqueue_active(&cil->xc_push_wait)) + wake_up_all(&cil->xc_push_wait); + + xlog_cil_push_pcp_aggregate(cil, ctx); + + /* + * Check if we've anything to push. If there is nothing, then we don't + * move on to a new sequence number and so we have to be able to push + * this sequence again later. + */ + if (test_bit(XLOG_CIL_EMPTY, &cil->xc_flags)) { + cil->xc_push_seq = 0; + spin_unlock(&cil->xc_push_lock); + goto out_skip; + } + + + /* check for a previously pushed sequence */ + if (push_seq < ctx->sequence) { + spin_unlock(&cil->xc_push_lock); + goto out_skip; + } + + /* + * We are now going to push this context, so add it to the committing + * list before we do anything else. This ensures that anyone waiting on + * this push can easily detect the difference between a "push in + * progress" and "CIL is empty, nothing to do". + * + * IOWs, a wait loop can now check for: + * the current sequence not being found on the committing list; + * an empty CIL; and + * an unchanged sequence number + * to detect a push that had nothing to do and therefore does not need + * waiting on. If the CIL is not empty, we get put on the committing + * list before emptying the CIL and bumping the sequence number. Hence + * an empty CIL and an unchanged sequence number means we jumped out + * above after doing nothing. + * + * Hence the waiter will either find the commit sequence on the + * committing list or the sequence number will be unchanged and the CIL + * still dirty. In that latter case, the push has not yet started, and + * so the waiter will have to continue trying to check the CIL + * committing list until it is found. In extreme cases of delay, the + * sequence may fully commit between the attempts the wait makes to wait + * on the commit sequence. + */ + list_add(&ctx->committing, &cil->xc_committing); + spin_unlock(&cil->xc_push_lock); + + xlog_cil_build_lv_chain(ctx, &whiteouts, &num_iovecs, &num_bytes); + + /* + * Switch the contexts so we can drop the context lock and move out + * of a shared context. We can't just go straight to the commit record, + * though - we need to synchronise with previous and future commits so + * that the commit records are correctly ordered in the log to ensure + * that we process items during log IO completion in the correct order. + * + * For example, if we get an EFI in one checkpoint and the EFD in the + * next (e.g. due to log forces), we do not want the checkpoint with + * the EFD to be committed before the checkpoint with the EFI. Hence + * we must strictly order the commit records of the checkpoints so + * that: a) the checkpoint callbacks are attached to the iclogs in the + * correct order; and b) the checkpoints are replayed in correct order + * in log recovery. + * + * Hence we need to add this context to the committing context list so + * that higher sequences will wait for us to write out a commit record + * before they do. + * + * xfs_log_force_seq requires us to mirror the new sequence into the cil + * structure atomically with the addition of this sequence to the + * committing list. This also ensures that we can do unlocked checks + * against the current sequence in log forces without risking + * deferencing a freed context pointer. + */ + spin_lock(&cil->xc_push_lock); + xlog_cil_ctx_switch(cil, new_ctx); + spin_unlock(&cil->xc_push_lock); + up_write(&cil->xc_ctx_lock); + + /* + * Sort the log vector chain before we add the transaction headers. + * This ensures we always have the transaction headers at the start + * of the chain. + */ + list_sort(NULL, &ctx->lv_chain, xlog_cil_order_cmp); + + /* + * Build a checkpoint transaction header and write it to the log to + * begin the transaction. We need to account for the space used by the + * transaction header here as it is not accounted for in xlog_write(). + * Add the lvhdr to the head of the lv chain we pass to xlog_write() so + * it gets written into the iclog first. + */ + xlog_cil_build_trans_hdr(ctx, &thdr, &lvhdr, num_iovecs); + num_bytes += lvhdr.lv_bytes; + list_add(&lvhdr.lv_list, &ctx->lv_chain); + + /* + * Take the lvhdr back off the lv_chain immediately after calling + * xlog_cil_write_chain() as it should not be passed to log IO + * completion. + */ + error = xlog_cil_write_chain(ctx, num_bytes); + list_del(&lvhdr.lv_list); + if (error) + goto out_abort_free_ticket; + + error = xlog_cil_write_commit_record(ctx); + if (error) + goto out_abort_free_ticket; + + /* + * Grab the ticket from the ctx so we can ungrant it after releasing the + * commit_iclog. The ctx may be freed by the time we return from + * releasing the commit_iclog (i.e. checkpoint has been completed and + * callback run) so we can't reference the ctx after the call to + * xlog_state_release_iclog(). + */ + ticket = ctx->ticket; + + /* + * If the checkpoint spans multiple iclogs, wait for all previous iclogs + * to complete before we submit the commit_iclog. We can't use state + * checks for this - ACTIVE can be either a past completed iclog or a + * future iclog being filled, while WANT_SYNC through SYNC_DONE can be a + * past or future iclog awaiting IO or ordered IO completion to be run. + * In the latter case, if it's a future iclog and we wait on it, the we + * will hang because it won't get processed through to ic_force_wait + * wakeup until this commit_iclog is written to disk. Hence we use the + * iclog header lsn and compare it to the commit lsn to determine if we + * need to wait on iclogs or not. + */ + spin_lock(&log->l_icloglock); + if (ctx->start_lsn != ctx->commit_lsn) { + xfs_lsn_t plsn = be64_to_cpu( + ctx->commit_iclog->ic_prev->ic_header->h_lsn); + + if (plsn && XFS_LSN_CMP(plsn, ctx->commit_lsn) < 0) { + /* + * Waiting on ic_force_wait orders the completion of + * iclogs older than ic_prev. Hence we only need to wait + * on the most recent older iclog here. + */ + xlog_wait_on_iclog(ctx->commit_iclog->ic_prev); + spin_lock(&log->l_icloglock); + } + + /* + * We need to issue a pre-flush so that the ordering for this + * checkpoint is correctly preserved down to stable storage. + */ + ctx->commit_iclog->ic_flags |= XLOG_ICL_NEED_FLUSH; + } + + /* + * The commit iclog must be written to stable storage to guarantee + * journal IO vs metadata writeback IO is correctly ordered on stable + * storage. + * + * If the push caller needs the commit to be immediately stable and the + * commit_iclog is not yet marked as XLOG_STATE_WANT_SYNC to indicate it + * will be written when released, switch it's state to WANT_SYNC right + * now. + */ + ctx->commit_iclog->ic_flags |= XLOG_ICL_NEED_FUA; + if (push_commit_stable && + ctx->commit_iclog->ic_state == XLOG_STATE_ACTIVE) + xlog_state_switch_iclogs(log, ctx->commit_iclog, 0); + ticket = ctx->ticket; + xlog_state_release_iclog(log, ctx->commit_iclog, ticket); + + /* Not safe to reference ctx now! */ + + spin_unlock(&log->l_icloglock); + xlog_cil_cleanup_whiteouts(&whiteouts); + xfs_log_ticket_ungrant(log, ticket); + memalloc_nofs_restore(nofs_flags); + return; + +out_skip: + up_write(&cil->xc_ctx_lock); + xfs_log_ticket_put(new_ctx->ticket); + kfree(new_ctx); + memalloc_nofs_restore(nofs_flags); + return; + +out_abort_free_ticket: + ASSERT(xlog_is_shutdown(log)); + xlog_cil_cleanup_whiteouts(&whiteouts); + if (!ctx->commit_iclog) { + xfs_log_ticket_ungrant(log, ctx->ticket); + xlog_cil_committed(ctx); + memalloc_nofs_restore(nofs_flags); + return; + } + spin_lock(&log->l_icloglock); + ticket = ctx->ticket; + xlog_state_release_iclog(log, ctx->commit_iclog, ticket); + /* Not safe to reference ctx now! */ + spin_unlock(&log->l_icloglock); + xfs_log_ticket_ungrant(log, ticket); + memalloc_nofs_restore(nofs_flags); +} + +/* + * We need to push CIL every so often so we don't cache more than we can fit in + * the log. The limit really is that a checkpoint can't be more than half the + * log (the current checkpoint is not allowed to overwrite the previous + * checkpoint), but commit latency and memory usage limit this to a smaller + * size. + */ +static void +xlog_cil_push_background( + struct xlog *log) +{ + struct xfs_cil *cil = log->l_cilp; + int space_used = atomic_read(&cil->xc_ctx->space_used); + + /* + * The cil won't be empty because we are called while holding the + * context lock so whatever we added to the CIL will still be there. + */ + ASSERT(!test_bit(XLOG_CIL_EMPTY, &cil->xc_flags)); + + /* + * We are done if: + * - we haven't used up all the space available yet; or + * - we've already queued up a push; and + * - we're not over the hard limit; and + * - nothing has been over the hard limit. + * + * If so, we don't need to take the push lock as there's nothing to do. + */ + if (space_used < XLOG_CIL_SPACE_LIMIT(log) || + (cil->xc_push_seq == cil->xc_current_sequence && + space_used < XLOG_CIL_BLOCKING_SPACE_LIMIT(log) && + !waitqueue_active(&cil->xc_push_wait))) { + up_read(&cil->xc_ctx_lock); + return; + } + + spin_lock(&cil->xc_push_lock); + if (cil->xc_push_seq < cil->xc_current_sequence) { + cil->xc_push_seq = cil->xc_current_sequence; + queue_work(cil->xc_push_wq, &cil->xc_ctx->push_work); + } + + /* + * Drop the context lock now, we can't hold that if we need to sleep + * because we are over the blocking threshold. The push_lock is still + * held, so blocking threshold sleep/wakeup is still correctly + * serialised here. + */ + up_read(&cil->xc_ctx_lock); + + /* + * If we are well over the space limit, throttle the work that is being + * done until the push work on this context has begun. Enforce the hard + * throttle on all transaction commits once it has been activated, even + * if the committing transactions have resulted in the space usage + * dipping back down under the hard limit. + * + * The ctx->xc_push_lock provides the serialisation necessary for safely + * calling xlog_cil_over_hard_limit() in this context. + */ + if (xlog_cil_over_hard_limit(log, space_used)) { + trace_xfs_log_cil_wait(log, cil->xc_ctx->ticket); + ASSERT(space_used < log->l_logsize); + xlog_wait(&cil->xc_push_wait, &cil->xc_push_lock); + return; + } + + spin_unlock(&cil->xc_push_lock); + +} + +/* + * xlog_cil_push_now() is used to trigger an immediate CIL push to the sequence + * number that is passed. When it returns, the work will be queued for + * @push_seq, but it won't be completed. + * + * If the caller is performing a synchronous force, we will flush the workqueue + * to get previously queued work moving to minimise the wait time they will + * undergo waiting for all outstanding pushes to complete. The caller is + * expected to do the required waiting for push_seq to complete. + * + * If the caller is performing an async push, we need to ensure that the + * checkpoint is fully flushed out of the iclogs when we finish the push. If we + * don't do this, then the commit record may remain sitting in memory in an + * ACTIVE iclog. This then requires another full log force to push to disk, + * which defeats the purpose of having an async, non-blocking CIL force + * mechanism. Hence in this case we need to pass a flag to the push work to + * indicate it needs to flush the commit record itself. + */ +static void +xlog_cil_push_now( + struct xlog *log, + xfs_csn_t push_seq, + bool async) +{ + struct xfs_cil *cil = log->l_cilp; + + if (!cil) + return; + + ASSERT(push_seq && push_seq <= cil->xc_current_sequence); + + /* start on any pending background push to minimise wait time on it */ + if (!async) + flush_workqueue(cil->xc_push_wq); + + spin_lock(&cil->xc_push_lock); + + /* + * If this is an async flush request, we always need to set the + * xc_push_commit_stable flag even if something else has already queued + * a push. The flush caller is asking for the CIL to be on stable + * storage when the next push completes, so regardless of who has queued + * the push, the flush requires stable semantics from it. + */ + cil->xc_push_commit_stable = async; + + /* + * If the CIL is empty or we've already pushed the sequence then + * there's no more work that we need to do. + */ + if (test_bit(XLOG_CIL_EMPTY, &cil->xc_flags) || + push_seq <= cil->xc_push_seq) { + spin_unlock(&cil->xc_push_lock); + return; + } + + cil->xc_push_seq = push_seq; + queue_work(cil->xc_push_wq, &cil->xc_ctx->push_work); + spin_unlock(&cil->xc_push_lock); +} + +bool +xlog_cil_empty( + struct xlog *log) +{ + struct xfs_cil *cil = log->l_cilp; + bool empty = false; + + spin_lock(&cil->xc_push_lock); + if (test_bit(XLOG_CIL_EMPTY, &cil->xc_flags)) + empty = true; + spin_unlock(&cil->xc_push_lock); + return empty; +} + +/* + * If there are intent done items in this transaction and the related intent was + * committed in the current (same) CIL checkpoint, we don't need to write either + * the intent or intent done item to the journal as the change will be + * journalled atomically within this checkpoint. As we cannot remove items from + * the CIL here, mark the related intent with a whiteout so that the CIL push + * can remove it rather than writing it to the journal. Then remove the intent + * done item from the current transaction and release it so it doesn't get put + * into the CIL at all. + */ +static uint32_t +xlog_cil_process_intents( + struct xfs_cil *cil, + struct xfs_trans *tp) +{ + struct xfs_log_item *lip, *ilip, *next; + uint32_t len = 0; + + list_for_each_entry_safe(lip, next, &tp->t_items, li_trans) { + if (!(lip->li_ops->flags & XFS_ITEM_INTENT_DONE)) + continue; + + ilip = lip->li_ops->iop_intent(lip); + if (!ilip || !xlog_item_in_current_chkpt(cil, ilip)) + continue; + set_bit(XFS_LI_WHITEOUT, &ilip->li_flags); + trace_xfs_cil_whiteout_mark(ilip); + len += ilip->li_lv->lv_bytes; + kvfree(ilip->li_lv); + ilip->li_lv = NULL; + + xfs_trans_del_item(lip); + lip->li_ops->iop_release(lip); + } + return len; +} + +/* + * Commit a transaction with the given vector to the Committed Item List. + * + * To do this, we need to format the item, pin it in memory if required and + * account for the space used by the transaction. Once we have done that we + * need to release the unused reservation for the transaction, attach the + * transaction to the checkpoint context so we carry the busy extents through + * to checkpoint completion, and then unlock all the items in the transaction. + * + * Called with the context lock already held in read mode to lock out + * background commit, returns without it held once background commits are + * allowed again. + */ +void +xlog_cil_commit( + struct xlog *log, + struct xfs_trans *tp, + xfs_csn_t *commit_seq, + bool regrant) +{ + struct xfs_cil *cil = log->l_cilp; + struct xfs_log_item *lip, *next; + uint32_t released_space = 0; + + /* + * Do all necessary memory allocation before we lock the CIL. + * This ensures the allocation does not deadlock with a CIL + * push in memory reclaim (e.g. from kswapd). + */ + xlog_cil_alloc_shadow_bufs(log, tp); + + /* lock out background commit */ + down_read(&cil->xc_ctx_lock); + + if (tp->t_flags & XFS_TRANS_HAS_INTENT_DONE) + released_space = xlog_cil_process_intents(cil, tp); + + xlog_cil_insert_items(log, tp, released_space); + + if (regrant && !xlog_is_shutdown(log)) + xfs_log_ticket_regrant(log, tp->t_ticket); + else + xfs_log_ticket_ungrant(log, tp->t_ticket); + tp->t_ticket = NULL; + xfs_trans_unreserve_and_mod_sb(tp); + + /* + * Once all the items of the transaction have been copied to the CIL, + * the items can be unlocked and possibly freed. + * + * This needs to be done before we drop the CIL context lock because we + * have to update state in the log items and unlock them before they go + * to disk. If we don't, then the CIL checkpoint can race with us and + * we can run checkpoint completion before we've updated and unlocked + * the log items. This affects (at least) processing of stale buffers, + * inodes and EFIs. + */ + trace_xfs_trans_commit_items(tp, _RET_IP_); + list_for_each_entry_safe(lip, next, &tp->t_items, li_trans) { + xfs_trans_del_item(lip); + if (lip->li_ops->iop_committing) + lip->li_ops->iop_committing(lip, cil->xc_ctx->sequence); + } + if (commit_seq) + *commit_seq = cil->xc_ctx->sequence; + + /* xlog_cil_push_background() releases cil->xc_ctx_lock */ + xlog_cil_push_background(log); +} + +/* + * Flush the CIL to stable storage but don't wait for it to complete. This + * requires the CIL push to ensure the commit record for the push hits the disk, + * but otherwise is no different to a push done from a log force. + */ +void +xlog_cil_flush( + struct xlog *log) +{ + xfs_csn_t seq = log->l_cilp->xc_current_sequence; + + trace_xfs_log_force(log->l_mp, seq, _RET_IP_); + xlog_cil_push_now(log, seq, true); + + /* + * If the CIL is empty, make sure that any previous checkpoint that may + * still be in an active iclog is pushed to stable storage. + */ + if (test_bit(XLOG_CIL_EMPTY, &log->l_cilp->xc_flags)) + xfs_log_force(log->l_mp, 0); +} + +/* + * Conditionally push the CIL based on the sequence passed in. + * + * We only need to push if we haven't already pushed the sequence number given. + * Hence the only time we will trigger a push here is if the push sequence is + * the same as the current context. + * + * We return the current commit lsn to allow the callers to determine if a + * iclog flush is necessary following this call. + */ +xfs_lsn_t +xlog_cil_force_seq( + struct xlog *log, + xfs_csn_t sequence) +{ + struct xfs_cil *cil = log->l_cilp; + struct xfs_cil_ctx *ctx; + xfs_lsn_t commit_lsn = NULLCOMMITLSN; + + ASSERT(sequence <= cil->xc_current_sequence); + + if (!sequence) + sequence = cil->xc_current_sequence; + trace_xfs_log_force(log->l_mp, sequence, _RET_IP_); + + /* + * check to see if we need to force out the current context. + * xlog_cil_push() handles racing pushes for the same sequence, + * so no need to deal with it here. + */ +restart: + xlog_cil_push_now(log, sequence, false); + + /* + * See if we can find a previous sequence still committing. + * We need to wait for all previous sequence commits to complete + * before allowing the force of push_seq to go ahead. Hence block + * on commits for those as well. + */ + spin_lock(&cil->xc_push_lock); + list_for_each_entry(ctx, &cil->xc_committing, committing) { + /* + * Avoid getting stuck in this loop because we were woken by the + * shutdown, but then went back to sleep once already in the + * shutdown state. + */ + if (xlog_is_shutdown(log)) + goto out_shutdown; + if (ctx->sequence > sequence) + continue; + if (!ctx->commit_lsn) { + /* + * It is still being pushed! Wait for the push to + * complete, then start again from the beginning. + */ + XFS_STATS_INC(log->l_mp, xs_log_force_sleep); + xlog_wait(&cil->xc_commit_wait, &cil->xc_push_lock); + goto restart; + } + if (ctx->sequence != sequence) + continue; + /* found it! */ + commit_lsn = ctx->commit_lsn; + } + + /* + * The call to xlog_cil_push_now() executes the push in the background. + * Hence by the time we have got here it our sequence may not have been + * pushed yet. This is true if the current sequence still matches the + * push sequence after the above wait loop and the CIL still contains + * dirty objects. This is guaranteed by the push code first adding the + * context to the committing list before emptying the CIL. + * + * Hence if we don't find the context in the committing list and the + * current sequence number is unchanged then the CIL contents are + * significant. If the CIL is empty, if means there was nothing to push + * and that means there is nothing to wait for. If the CIL is not empty, + * it means we haven't yet started the push, because if it had started + * we would have found the context on the committing list. + */ + if (sequence == cil->xc_current_sequence && + !test_bit(XLOG_CIL_EMPTY, &cil->xc_flags)) { + spin_unlock(&cil->xc_push_lock); + goto restart; + } + + spin_unlock(&cil->xc_push_lock); + return commit_lsn; + + /* + * We detected a shutdown in progress. We need to trigger the log force + * to pass through it's iclog state machine error handling, even though + * we are already in a shutdown state. Hence we can't return + * NULLCOMMITLSN here as that has special meaning to log forces (i.e. + * LSN is already stable), so we return a zero LSN instead. + */ +out_shutdown: + spin_unlock(&cil->xc_push_lock); + return 0; +} + +/* + * Perform initial CIL structure initialisation. + */ +int +xlog_cil_init( + struct xlog *log) +{ + struct xfs_cil *cil; + struct xfs_cil_ctx *ctx; + struct xlog_cil_pcp *cilpcp; + int cpu; + + cil = kzalloc_obj(*cil, GFP_KERNEL | __GFP_RETRY_MAYFAIL); + if (!cil) + return -ENOMEM; + /* + * Limit the CIL pipeline depth to 4 concurrent works to bound the + * concurrency the log spinlocks will be exposed to. + */ + cil->xc_push_wq = alloc_workqueue("xfs-cil/%s", + XFS_WQFLAGS(WQ_FREEZABLE | WQ_MEM_RECLAIM | WQ_UNBOUND), + 4, log->l_mp->m_super->s_id); + if (!cil->xc_push_wq) + goto out_destroy_cil; + + cil->xc_log = log; + cil->xc_pcp = alloc_percpu(struct xlog_cil_pcp); + if (!cil->xc_pcp) + goto out_destroy_wq; + + for_each_possible_cpu(cpu) { + cilpcp = per_cpu_ptr(cil->xc_pcp, cpu); + INIT_LIST_HEAD(&cilpcp->busy_extents); + INIT_LIST_HEAD(&cilpcp->log_items); + } + + INIT_LIST_HEAD(&cil->xc_committing); + spin_lock_init(&cil->xc_push_lock); + init_waitqueue_head(&cil->xc_push_wait); + init_rwsem(&cil->xc_ctx_lock); + init_waitqueue_head(&cil->xc_start_wait); + init_waitqueue_head(&cil->xc_commit_wait); + log->l_cilp = cil; + + ctx = xlog_cil_ctx_alloc(); + xlog_cil_ctx_switch(cil, ctx); + return 0; + +out_destroy_wq: + destroy_workqueue(cil->xc_push_wq); +out_destroy_cil: + kfree(cil); + return -ENOMEM; +} + +void +xlog_cil_destroy( + struct xlog *log) +{ + struct xfs_cil *cil = log->l_cilp; + + if (cil->xc_ctx) { + if (cil->xc_ctx->ticket) + xfs_log_ticket_put(cil->xc_ctx->ticket); + kfree(cil->xc_ctx); + } + + ASSERT(test_bit(XLOG_CIL_EMPTY, &cil->xc_flags)); + free_percpu(cil->xc_pcp); + destroy_workqueue(cil->xc_push_wq); + kfree(cil); +} + diff --git a/fs/xfs/xfs_log_priv.h b/fs/xfs/xfs_log_priv.h new file mode 100644 index 000000000..cf1e4ce61 --- /dev/null +++ b/fs/xfs/xfs_log_priv.h @@ -0,0 +1,747 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2003,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_LOG_PRIV_H__ +#define __XFS_LOG_PRIV_H__ + +#include "xfs_extent_busy.h" /* for struct xfs_busy_extents */ + +struct xfs_buf; +struct xlog; +struct xlog_ticket; +struct xfs_mount; + +struct xfs_log_iovec { + void *i_addr;/* beginning address of region */ + int i_len; /* length in bytes of region */ + uint i_type; /* type of region */ +}; + +struct xfs_log_vec { + struct list_head lv_list; /* CIL lv chain ptrs */ + uint32_t lv_order_id; /* chain ordering info */ + int lv_niovecs; /* number of iovecs in lv */ + struct xfs_log_iovec *lv_iovecp; /* iovec array */ + struct xfs_log_item *lv_item; /* owner */ + char *lv_buf; /* formatted buffer */ + int lv_bytes; /* accounted space in buffer */ + int lv_buf_used; /* buffer space used so far */ + int lv_alloc_size; /* size of allocated lv */ +}; + +/* + * get client id from packed copy. + * + * this hack is here because the xlog_pack code copies four bytes + * of xlog_op_header containing the fields oh_clientid, oh_flags + * and oh_res2 into the packed copy. + * + * later on this four byte chunk is treated as an int and the + * client id is pulled out. + * + * this has endian issues, of course. + */ +static inline uint xlog_get_client_id(__be32 i) +{ + return be32_to_cpu(i) >> 24; +} + +/* + * In core log state + */ +enum xlog_iclog_state { + XLOG_STATE_ACTIVE, /* Current IC log being written to */ + XLOG_STATE_WANT_SYNC, /* Want to sync this iclog; no more writes */ + XLOG_STATE_SYNCING, /* This IC log is syncing */ + XLOG_STATE_DONE_SYNC, /* Done syncing to disk */ + XLOG_STATE_CALLBACK, /* Callback functions now */ + XLOG_STATE_DIRTY, /* Dirty IC log, not ready for ACTIVE status */ +}; + +#define XLOG_STATE_STRINGS \ + { XLOG_STATE_ACTIVE, "XLOG_STATE_ACTIVE" }, \ + { XLOG_STATE_WANT_SYNC, "XLOG_STATE_WANT_SYNC" }, \ + { XLOG_STATE_SYNCING, "XLOG_STATE_SYNCING" }, \ + { XLOG_STATE_DONE_SYNC, "XLOG_STATE_DONE_SYNC" }, \ + { XLOG_STATE_CALLBACK, "XLOG_STATE_CALLBACK" }, \ + { XLOG_STATE_DIRTY, "XLOG_STATE_DIRTY" } + +/* + * In core log flags + */ +#define XLOG_ICL_NEED_FLUSH (1u << 0) /* iclog needs REQ_PREFLUSH */ +#define XLOG_ICL_NEED_FUA (1u << 1) /* iclog needs REQ_FUA */ + +#define XLOG_ICL_STRINGS \ + { XLOG_ICL_NEED_FLUSH, "XLOG_ICL_NEED_FLUSH" }, \ + { XLOG_ICL_NEED_FUA, "XLOG_ICL_NEED_FUA" } + + +/* + * Log ticket flags + */ +#define XLOG_TIC_PERM_RESERV (1u << 0) /* permanent reservation */ + +#define XLOG_TIC_FLAGS \ + { XLOG_TIC_PERM_RESERV, "XLOG_TIC_PERM_RESERV" } + +/* + * Below are states for covering allocation transactions. + * By covering, we mean changing the h_tail_lsn in the last on-disk + * log write such that no allocation transactions will be re-done during + * recovery after a system crash. Recovery starts at the last on-disk + * log write. + * + * These states are used to insert dummy log entries to cover + * space allocation transactions which can undo non-transactional changes + * after a crash. Writes to a file with space + * already allocated do not result in any transactions. Allocations + * might include space beyond the EOF. So if we just push the EOF a + * little, the last transaction for the file could contain the wrong + * size. If there is no file system activity, after an allocation + * transaction, and the system crashes, the allocation transaction + * will get replayed and the file will be truncated. This could + * be hours/days/... after the allocation occurred. + * + * The fix for this is to do two dummy transactions when the + * system is idle. We need two dummy transaction because the h_tail_lsn + * in the log record header needs to point beyond the last possible + * non-dummy transaction. The first dummy changes the h_tail_lsn to + * the first transaction before the dummy. The second dummy causes + * h_tail_lsn to point to the first dummy. Recovery starts at h_tail_lsn. + * + * These dummy transactions get committed when everything + * is idle (after there has been some activity). + * + * There are 5 states used to control this. + * + * IDLE -- no logging has been done on the file system or + * we are done covering previous transactions. + * NEED -- logging has occurred and we need a dummy transaction + * when the log becomes idle. + * DONE -- we were in the NEED state and have committed a dummy + * transaction. + * NEED2 -- we detected that a dummy transaction has gone to the + * on disk log with no other transactions. + * DONE2 -- we committed a dummy transaction when in the NEED2 state. + * + * There are two places where we switch states: + * + * 1.) In xfs_sync, when we detect an idle log and are in NEED or NEED2. + * We commit the dummy transaction and switch to DONE or DONE2, + * respectively. In all other states, we don't do anything. + * + * 2.) When we finish writing the on-disk log (xlog_state_clean_log). + * + * No matter what state we are in, if this isn't the dummy + * transaction going out, the next state is NEED. + * So, if we aren't in the DONE or DONE2 states, the next state + * is NEED. We can't be finishing a write of the dummy record + * unless it was committed and the state switched to DONE or DONE2. + * + * If we are in the DONE state and this was a write of the + * dummy transaction, we move to NEED2. + * + * If we are in the DONE2 state and this was a write of the + * dummy transaction, we move to IDLE. + * + * + * Writing only one dummy transaction can get appended to + * one file space allocation. When this happens, the log recovery + * code replays the space allocation and a file could be truncated. + * This is why we have the NEED2 and DONE2 states before going idle. + */ + +#define XLOG_STATE_COVER_IDLE 0 +#define XLOG_STATE_COVER_NEED 1 +#define XLOG_STATE_COVER_DONE 2 +#define XLOG_STATE_COVER_NEED2 3 +#define XLOG_STATE_COVER_DONE2 4 + +#define XLOG_COVER_OPS 5 + +struct xlog_ticket { + struct list_head t_queue; /* reserve/write queue */ + struct task_struct *t_task; /* task that owns this ticket */ + xlog_tid_t t_tid; /* transaction identifier */ + atomic_t t_ref; /* ticket reference count */ + int t_curr_res; /* current reservation */ + int t_unit_res; /* unit reservation */ + char t_ocnt; /* original unit count */ + char t_cnt; /* current unit count */ + uint8_t t_flags; /* properties of reservation */ + int t_iclog_hdrs; /* iclog hdrs in t_curr_res */ +}; + +/* + * In-core log structure. + * + * - ic_forcewait is used to implement synchronous forcing of the iclog to disk. + * - ic_next is the pointer to the next iclog in the ring. + * - ic_log is a pointer back to the global log structure. + * - ic_size is the full size of the log buffer, minus the cycle headers. + * - ic_offset is the current number of bytes written to in this iclog. + * - ic_refcnt is bumped when someone is writing to the log. + * - ic_state is the state of the iclog. + * + * Because of cacheline contention on large machines, we need to separate + * various resources onto different cachelines. To start with, make the + * structure cacheline aligned. The following fields can be contended on + * by independent processes: + * + * - ic_callbacks + * - ic_refcnt + * - fields protected by the global l_icloglock + * + * so we need to ensure that these fields are located in separate cachelines. + * We'll put all the read-only and l_icloglock fields in the first cacheline, + * and move everything else out to subsequent cachelines. + */ +struct xlog_in_core { + wait_queue_head_t ic_force_wait; + wait_queue_head_t ic_write_wait; + struct xlog_in_core *ic_next; + struct xlog_in_core *ic_prev; + struct xlog *ic_log; + u32 ic_size; + u32 ic_offset; + enum xlog_iclog_state ic_state; + unsigned int ic_flags; + void *ic_datap; /* pointer to iclog data */ + struct list_head ic_callbacks; + + /* reference counts need their own cacheline */ + atomic_t ic_refcnt ____cacheline_aligned_in_smp; + struct xlog_rec_header *ic_header; +#ifdef DEBUG + bool ic_fail_crc : 1; +#endif + struct semaphore ic_sema; + struct work_struct ic_end_io_work; + struct bio ic_bio; + struct bio_vec ic_bvec[]; +}; + +/* + * The CIL context is used to aggregate per-transaction details as well be + * passed to the iclog for checkpoint post-commit processing. After being + * passed to the iclog, another context needs to be allocated for tracking the + * next set of transactions to be aggregated into a checkpoint. + */ +struct xfs_cil; + +struct xfs_cil_ctx { + struct xfs_cil *cil; + xfs_csn_t sequence; /* chkpt sequence # */ + xfs_lsn_t start_lsn; /* first LSN of chkpt commit */ + xfs_lsn_t commit_lsn; /* chkpt commit record lsn */ + struct xlog_in_core *commit_iclog; + struct xlog_ticket *ticket; /* chkpt ticket */ + atomic_t space_used; /* aggregate size of regions */ + struct xfs_busy_extents busy_extents; + struct list_head log_items; /* log items in chkpt */ + struct list_head lv_chain; /* logvecs being pushed */ + struct list_head iclog_entry; + struct list_head committing; /* ctx committing list */ + struct work_struct push_work; + atomic_t order_id; + + /* + * CPUs that could have added items to the percpu CIL data. Access is + * coordinated with xc_ctx_lock. + */ + struct cpumask cil_pcpmask; +}; + +/* + * Per-cpu CIL tracking items + */ +struct xlog_cil_pcp { + int32_t space_used; + uint32_t space_reserved; + struct list_head busy_extents; + struct list_head log_items; +}; + +/* + * Committed Item List structure + * + * This structure is used to track log items that have been committed but not + * yet written into the log. It is used only when the delayed logging mount + * option is enabled. + * + * This structure tracks the list of committing checkpoint contexts so + * we can avoid the problem of having to hold out new transactions during a + * flush until we have a the commit record LSN of the checkpoint. We can + * traverse the list of committing contexts in xlog_cil_push_lsn() to find a + * sequence match and extract the commit LSN directly from there. If the + * checkpoint is still in the process of committing, we can block waiting for + * the commit LSN to be determined as well. This should make synchronous + * operations almost as efficient as the old logging methods. + */ +struct xfs_cil { + struct xlog *xc_log; + unsigned long xc_flags; + atomic_t xc_iclog_hdrs; + struct workqueue_struct *xc_push_wq; + + struct rw_semaphore xc_ctx_lock ____cacheline_aligned_in_smp; + struct xfs_cil_ctx *xc_ctx; + + spinlock_t xc_push_lock ____cacheline_aligned_in_smp; + xfs_csn_t xc_push_seq; + bool xc_push_commit_stable; + struct list_head xc_committing; + wait_queue_head_t xc_commit_wait; + wait_queue_head_t xc_start_wait; + xfs_csn_t xc_current_sequence; + wait_queue_head_t xc_push_wait; /* background push throttle */ + + void __percpu *xc_pcp; /* percpu CIL structures */ +} ____cacheline_aligned_in_smp; + +/* xc_flags bit values */ +#define XLOG_CIL_EMPTY 1 +#define XLOG_CIL_PCP_SPACE 2 + +/* + * The amount of log space we allow the CIL to aggregate is difficult to size. + * Whatever we choose, we have to make sure we can get a reservation for the + * log space effectively, that it is large enough to capture sufficient + * relogging to reduce log buffer IO significantly, but it is not too large for + * the log or induces too much latency when writing out through the iclogs. We + * track both space consumed and the number of vectors in the checkpoint + * context, so we need to decide which to use for limiting. + * + * Every log buffer we write out during a push needs a header reserved, which + * is at least one sector and more for v2 logs. Hence we need a reservation of + * at least 512 bytes per 32k of log space just for the LR headers. That means + * 16KB of reservation per megabyte of delayed logging space we will consume, + * plus various headers. The number of headers will vary based on the num of + * io vectors, so limiting on a specific number of vectors is going to result + * in transactions of varying size. IOWs, it is more consistent to track and + * limit space consumed in the log rather than by the number of objects being + * logged in order to prevent checkpoint ticket overruns. + * + * Further, use of static reservations through the log grant mechanism is + * problematic. It introduces a lot of complexity (e.g. reserve grant vs write + * grant) and a significant deadlock potential because regranting write space + * can block on log pushes. Hence if we have to regrant log space during a log + * push, we can deadlock. + * + * However, we can avoid this by use of a dynamic "reservation stealing" + * technique during transaction commit whereby unused reservation space in the + * transaction ticket is transferred to the CIL ctx commit ticket to cover the + * space needed by the checkpoint transaction. This means that we never need to + * specifically reserve space for the CIL checkpoint transaction, nor do we + * need to regrant space once the checkpoint completes. This also means the + * checkpoint transaction ticket is specific to the checkpoint context, rather + * than the CIL itself. + * + * With dynamic reservations, we can effectively make up arbitrary limits for + * the checkpoint size so long as they don't violate any other size rules. + * Recovery imposes a rule that no transaction exceed half the log, so we are + * limited by that. Furthermore, the log transaction reservation subsystem + * tries to keep 25% of the log free, so we need to keep below that limit or we + * risk running out of free log space to start any new transactions. + * + * In order to keep background CIL push efficient, we only need to ensure the + * CIL is large enough to maintain sufficient in-memory relogging to avoid + * repeated physical writes of frequently modified metadata. If we allow the CIL + * to grow to a substantial fraction of the log, then we may be pinning hundreds + * of megabytes of metadata in memory until the CIL flushes. This can cause + * issues when we are running low on memory - pinned memory cannot be reclaimed, + * and the CIL consumes a lot of memory. Hence we need to set an upper physical + * size limit for the CIL that limits the maximum amount of memory pinned by the + * CIL but does not limit performance by reducing relogging efficiency + * significantly. + * + * As such, the CIL push threshold ends up being the smaller of two thresholds: + * - a threshold large enough that it allows CIL to be pushed and progress to be + * made without excessive blocking of incoming transaction commits. This is + * defined to be 12.5% of the log space - half the 25% push threshold of the + * AIL. + * - small enough that it doesn't pin excessive amounts of memory but maintains + * close to peak relogging efficiency. This is defined to be 16x the iclog + * buffer window (32MB) as measurements have shown this to be roughly the + * point of diminishing performance increases under highly concurrent + * modification workloads. + * + * To prevent the CIL from overflowing upper commit size bounds, we introduce a + * new threshold at which we block committing transactions until the background + * CIL commit commences and switches to a new context. While this is not a hard + * limit, it forces the process committing a transaction to the CIL to block and + * yeild the CPU, giving the CIL push work a chance to be scheduled and start + * work. This prevents a process running lots of transactions from overfilling + * the CIL because it is not yielding the CPU. We set the blocking limit at + * twice the background push space threshold so we keep in line with the AIL + * push thresholds. + * + * Note: this is not a -hard- limit as blocking is applied after the transaction + * is inserted into the CIL and the push has been triggered. It is largely a + * throttling mechanism that allows the CIL push to be scheduled and run. A hard + * limit will be difficult to implement without introducing global serialisation + * in the CIL commit fast path, and it's not at all clear that we actually need + * such hard limits given the ~7 years we've run without a hard limit before + * finding the first situation where a checkpoint size overflow actually + * occurred. Hence the simple throttle, and an ASSERT check to tell us that + * we've overrun the max size. + */ +#define XLOG_CIL_SPACE_LIMIT(log) \ + min_t(int, (log)->l_logsize >> 3, BBTOB(XLOG_TOTAL_REC_SHIFT(log)) << 4) + +#define XLOG_CIL_BLOCKING_SPACE_LIMIT(log) \ + (XLOG_CIL_SPACE_LIMIT(log) * 2) + +/* + * ticket grant locks, queues and accounting have their own cachlines + * as these are quite hot and can be operated on concurrently. + */ +struct xlog_grant_head { + spinlock_t lock ____cacheline_aligned_in_smp; + struct list_head waiters; + atomic64_t grant; +}; + +/* + * The reservation head lsn is not made up of a cycle number and block number. + * Instead, it uses a cycle number and byte number. Logs don't expect to + * overflow 31 bits worth of byte offset, so using a byte number will mean + * that round off problems won't occur when releasing partial reservations. + */ +struct xlog { + /* The following fields don't need locking */ + struct xfs_mount *l_mp; /* mount point */ + struct xfs_ail *l_ailp; /* AIL log is working with */ + struct xfs_cil *l_cilp; /* CIL log is working with */ + struct xfs_buftarg *l_targ; /* buftarg of log */ + struct workqueue_struct *l_ioend_workqueue; /* for I/O completions */ + struct delayed_work l_work; /* background flush work */ + long l_opstate; /* operational state */ + uint l_quotaoffs_flag; /* XFS_DQ_*, for QUOTAOFFs */ + struct list_head *l_buf_cancel_table; + struct list_head r_dfops; /* recovered log intent items */ + int l_iclog_hsize; /* size of iclog header */ + uint l_sectBBsize; /* sector size in BBs (2^n) */ + int l_iclog_size; /* size of log in bytes */ + int l_iclog_bufs; /* number of iclog buffers */ + xfs_daddr_t l_logBBstart; /* start block of log */ + int l_logsize; /* size of log in bytes */ + int l_logBBsize; /* size of log in BB chunks */ + + /* The following block of fields are changed while holding icloglock */ + wait_queue_head_t l_flush_wait ____cacheline_aligned_in_smp; + /* waiting for iclog flush */ + int l_covered_state;/* state of "covering disk + * log entries" */ + struct xlog_in_core *l_iclog; /* head log queue */ + spinlock_t l_icloglock; /* grab to change iclog state */ + int l_curr_cycle; /* Cycle number of log writes */ + int l_prev_cycle; /* Cycle number before last + * block increment */ + int l_curr_block; /* current logical log block */ + int l_prev_block; /* previous logical log block */ + + /* + * l_tail_lsn is atomic so it can be set and read without needing to + * hold specific locks. To avoid operations contending with other hot + * objects, it on a separate cacheline. + */ + /* lsn of 1st LR with unflushed * buffers */ + atomic64_t l_tail_lsn ____cacheline_aligned_in_smp; + + struct xlog_grant_head l_reserve_head; + struct xlog_grant_head l_write_head; + uint64_t l_tail_space; + + struct xfs_kobj l_kobj; + + /* log recovery lsn tracking (for buffer submission */ + xfs_lsn_t l_recovery_lsn; + + uint32_t l_iclog_roundoff;/* padding roundoff */ +}; + +/* + * Bits for operational state + */ +#define XLOG_ACTIVE_RECOVERY 0 /* in the middle of recovery */ +#define XLOG_RECOVERY_NEEDED 1 /* log was recovered */ +#define XLOG_IO_ERROR 2 /* log hit an I/O error, and being + shutdown */ +#define XLOG_TAIL_WARN 3 /* log tail verify warning issued */ +#define XLOG_SHUTDOWN_STARTED 4 /* xlog_force_shutdown() exclusion */ + +static inline bool +xlog_recovery_needed(struct xlog *log) +{ + return test_bit(XLOG_RECOVERY_NEEDED, &log->l_opstate); +} + +static inline bool +xlog_in_recovery(struct xlog *log) +{ + return test_bit(XLOG_ACTIVE_RECOVERY, &log->l_opstate); +} + +static inline bool +xlog_is_shutdown(struct xlog *log) +{ + return test_bit(XLOG_IO_ERROR, &log->l_opstate); +} + +/* + * Wait until the xlog_force_shutdown() has marked the log as shut down + * so xlog_is_shutdown() will always return true. + */ +static inline void +xlog_shutdown_wait( + struct xlog *log) +{ + wait_var_event(&log->l_opstate, xlog_is_shutdown(log)); +} + +/* common routines */ +extern int +xlog_recover( + struct xlog *log); +extern int +xlog_recover_finish( + struct xlog *log); +extern void +xlog_recover_cancel(struct xlog *); + +__le32 xlog_cksum(struct xlog *log, struct xlog_rec_header *rhead, + char *dp, unsigned int hdrsize, unsigned int size); + +extern struct kmem_cache *xfs_log_ticket_cache; +struct xlog_ticket *xlog_ticket_alloc(struct xlog *log, int unit_bytes, + int count, bool permanent); + +void xlog_print_tic_res(struct xfs_mount *mp, struct xlog_ticket *ticket); +void xlog_print_trans(struct xfs_trans *); +int xlog_write(struct xlog *log, struct xfs_cil_ctx *ctx, + struct list_head *lv_chain, struct xlog_ticket *tic, + uint32_t len); +int xlog_write_one_vec(struct xlog *log, struct xfs_cil_ctx *ctx, + struct xfs_log_iovec *reg, struct xlog_ticket *ticket); +void xfs_log_ticket_ungrant(struct xlog *log, struct xlog_ticket *ticket); +void xfs_log_ticket_regrant(struct xlog *log, struct xlog_ticket *ticket); + +void xlog_state_switch_iclogs(struct xlog *log, struct xlog_in_core *iclog, + int eventual_size); +int xlog_state_release_iclog(struct xlog *log, struct xlog_in_core *iclog, + struct xlog_ticket *ticket); + +/* + * When we crack an atomic LSN, we sample it first so that the value will not + * change while we are cracking it into the component values. This means we + * will always get consistent component values to work from. This should always + * be used to sample and crack LSNs that are stored and updated in atomic + * variables. + */ +static inline void +xlog_crack_atomic_lsn(atomic64_t *lsn, uint *cycle, uint *block) +{ + xfs_lsn_t val = atomic64_read(lsn); + + *cycle = CYCLE_LSN(val); + *block = BLOCK_LSN(val); +} + +/* + * Calculate and assign a value to an atomic LSN variable from component pieces. + */ +static inline void +xlog_assign_atomic_lsn(atomic64_t *lsn, uint cycle, uint block) +{ + atomic64_set(lsn, xlog_assign_lsn(cycle, block)); +} + +/* + * Committed Item List interfaces + */ +int xlog_cil_init(struct xlog *log); +void xlog_cil_init_post_recovery(struct xlog *log); +void xlog_cil_destroy(struct xlog *log); +bool xlog_cil_empty(struct xlog *log); +void xlog_cil_commit(struct xlog *log, struct xfs_trans *tp, + xfs_csn_t *commit_seq, bool regrant); +void xlog_cil_set_ctx_write_state(struct xfs_cil_ctx *ctx, + struct xlog_in_core *iclog); + + +/* + * CIL force routines + */ +void xlog_cil_flush(struct xlog *log); +xfs_lsn_t xlog_cil_force_seq(struct xlog *log, xfs_csn_t sequence); + +static inline void +xlog_cil_force(struct xlog *log) +{ + xlog_cil_force_seq(log, log->l_cilp->xc_current_sequence); +} + +/* + * Wrapper function for waiting on a wait queue serialised against wakeups + * by a spinlock. This matches the semantics of all the wait queues used in the + * log code. + */ +static inline void +xlog_wait( + struct wait_queue_head *wq, + struct spinlock *lock) + __releases(lock) +{ + DECLARE_WAITQUEUE(wait, current); + + add_wait_queue_exclusive(wq, &wait); + __set_current_state(TASK_UNINTERRUPTIBLE); + spin_unlock(lock); + schedule(); + remove_wait_queue(wq, &wait); +} + +int xlog_wait_on_iclog(struct xlog_in_core *iclog) + __releases(iclog->ic_log->l_icloglock); + +/* Calculate the distance between two LSNs in bytes */ +static inline uint64_t +xlog_lsn_sub( + struct xlog *log, + xfs_lsn_t high, + xfs_lsn_t low) +{ + uint32_t hi_cycle = CYCLE_LSN(high); + uint32_t hi_block = BLOCK_LSN(high); + uint32_t lo_cycle = CYCLE_LSN(low); + uint32_t lo_block = BLOCK_LSN(low); + + if (hi_cycle == lo_cycle) + return BBTOB(hi_block - lo_block); + ASSERT((hi_cycle == lo_cycle + 1) || xlog_is_shutdown(log)); + return (uint64_t)log->l_logsize - BBTOB(lo_block - hi_block); +} + +void xlog_grant_return_space(struct xlog *log, xfs_lsn_t old_head, + xfs_lsn_t new_head); + +/* + * The LSN is valid so long as it is behind the current LSN. If it isn't, this + * means that the next log record that includes this metadata could have a + * smaller LSN. In turn, this means that the modification in the log would not + * replay. + */ +static inline bool +xlog_valid_lsn( + struct xlog *log, + xfs_lsn_t lsn) +{ + int cur_cycle; + int cur_block; + bool valid = true; + + /* + * First, sample the current lsn without locking to avoid added + * contention from metadata I/O. The current cycle and block are updated + * (in xlog_state_switch_iclogs()) and read here in a particular order + * to avoid false negatives (e.g., thinking the metadata LSN is valid + * when it is not). + * + * The current block is always rewound before the cycle is bumped in + * xlog_state_switch_iclogs() to ensure the current LSN is never seen in + * a transiently forward state. Instead, we can see the LSN in a + * transiently behind state if we happen to race with a cycle wrap. + */ + cur_cycle = READ_ONCE(log->l_curr_cycle); + smp_rmb(); + cur_block = READ_ONCE(log->l_curr_block); + + if ((CYCLE_LSN(lsn) > cur_cycle) || + (CYCLE_LSN(lsn) == cur_cycle && BLOCK_LSN(lsn) > cur_block)) { + /* + * If the metadata LSN appears invalid, it's possible the check + * above raced with a wrap to the next log cycle. Grab the lock + * to check for sure. + */ + spin_lock(&log->l_icloglock); + cur_cycle = log->l_curr_cycle; + cur_block = log->l_curr_block; + spin_unlock(&log->l_icloglock); + + if ((CYCLE_LSN(lsn) > cur_cycle) || + (CYCLE_LSN(lsn) == cur_cycle && BLOCK_LSN(lsn) > cur_block)) + valid = false; + } + + return valid; +} + +/* + * Log vector and shadow buffers can be large, so we need to use kvmalloc() here + * to ensure success. Unfortunately, kvmalloc() only allows GFP_KERNEL contexts + * to fall back to vmalloc, so we can't actually do anything useful with gfp + * flags to control the kmalloc() behaviour within kvmalloc(). Hence kmalloc() + * will do direct reclaim and compaction in the slow path, both of which are + * horrendously expensive. We just want kmalloc to fail fast and fall back to + * vmalloc if it can't get something straight away from the free lists or + * buddy allocator. Hence we have to open code kvmalloc outselves here. + * + * This assumes that the caller uses memalloc_nofs_save task context here, so + * despite the use of GFP_KERNEL here, we are going to be doing GFP_NOFS + * allocations. This is actually the only way to make vmalloc() do GFP_NOFS + * allocations, so lets just all pretend this is a GFP_KERNEL context + * operation.... + */ +static inline void * +xlog_kvmalloc( + size_t buf_size) +{ + gfp_t flags = GFP_KERNEL; + void *p; + + flags &= ~__GFP_DIRECT_RECLAIM; + flags |= __GFP_NOWARN | __GFP_NORETRY; + do { + p = kmalloc(buf_size, flags); + if (!p) + p = vmalloc(buf_size); + } while (!p); + + return p; +} + +/* + * Given a count of iovecs and space for a log item, compute the space we need + * in the log to store that data plus the log headers. + */ +static inline unsigned int +xlog_item_space( + unsigned int niovecs, + unsigned int nbytes) +{ + nbytes += niovecs * (sizeof(uint64_t) + sizeof(struct xlog_op_header)); + return round_up(nbytes, sizeof(uint64_t)); +} + +/* + * Cycles over XLOG_CYCLE_DATA_SIZE overflow into the extended header that was + * added for v2 logs. Addressing for the cycles array there is off by one, + * because the first batch of cycles is in the original header. + */ +static inline __be32 *xlog_cycle_data(struct xlog_rec_header *rhead, unsigned i) +{ + if (i >= XLOG_CYCLE_DATA_SIZE) { + unsigned j = i / XLOG_CYCLE_DATA_SIZE; + unsigned k = i % XLOG_CYCLE_DATA_SIZE; + + return &rhead->h_ext[j - 1].xh_cycle_data[k]; + } + + return &rhead->h_cycle_data[i]; +} + +#endif /* __XFS_LOG_PRIV_H__ */ diff --git a/fs/xfs/xfs_log_recover.c b/fs/xfs/xfs_log_recover.c new file mode 100644 index 000000000..e7e495296 --- /dev/null +++ b/fs/xfs/xfs_log_recover.c @@ -0,0 +1,3587 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2006 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_sb.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_log.h" +#include "xfs_log_priv.h" +#include "xfs_log_recover.h" +#include "xfs_trans_priv.h" +#include "xfs_alloc.h" +#include "xfs_ialloc.h" +#include "xfs_trace.h" +#include "xfs_icache.h" +#include "xfs_error.h" +#include "xfs_buf_item.h" +#include "xfs_ag.h" +#include "xfs_quota.h" +#include "xfs_reflink.h" + +#define BLK_AVG(blk1, blk2) ((blk1+blk2) >> 1) + +STATIC int +xlog_find_zeroed( + struct xlog *, + xfs_daddr_t *); +STATIC int +xlog_clear_stale_blocks( + struct xlog *, + xfs_lsn_t); +STATIC int +xlog_do_recovery_pass( + struct xlog *, xfs_daddr_t, xfs_daddr_t, int, xfs_daddr_t *); + +/* + * Sector aligned buffer routines for buffer create/read/write/access + */ + +/* + * Verify the log-relative block number and length in basic blocks are valid for + * an operation involving the given XFS log buffer. Returns true if the fields + * are valid, false otherwise. + */ +static inline bool +xlog_verify_bno( + struct xlog *log, + xfs_daddr_t blk_no, + int bbcount) +{ + if (blk_no < 0 || blk_no >= log->l_logBBsize) + return false; + if (bbcount <= 0 || (blk_no + bbcount) > log->l_logBBsize) + return false; + return true; +} + +/* + * Allocate a buffer to hold log data. The buffer needs to be able to map to + * a range of nbblks basic blocks at any valid offset within the log. + */ +static char * +xlog_alloc_buffer( + struct xlog *log, + int nbblks) +{ + /* + * Pass log block 0 since we don't have an addr yet, buffer will be + * verified on read. + */ + if (XFS_IS_CORRUPT(log->l_mp, !xlog_verify_bno(log, 0, nbblks))) { + xfs_warn(log->l_mp, "Invalid block length (0x%x) for buffer", + nbblks); + return NULL; + } + + /* + * We do log I/O in units of log sectors (a power-of-2 multiple of the + * basic block size), so we round up the requested size to accommodate + * the basic blocks required for complete log sectors. + * + * In addition, the buffer may be used for a non-sector-aligned block + * offset, in which case an I/O of the requested size could extend + * beyond the end of the buffer. If the requested size is only 1 basic + * block it will never straddle a sector boundary, so this won't be an + * issue. Nor will this be a problem if the log I/O is done in basic + * blocks (sector size 1). But otherwise we extend the buffer by one + * extra log sector to ensure there's space to accommodate this + * possibility. + */ + if (nbblks > 1 && log->l_sectBBsize > 1) + nbblks += log->l_sectBBsize; + nbblks = round_up(nbblks, log->l_sectBBsize); + return kvzalloc(BBTOB(nbblks), GFP_KERNEL | __GFP_RETRY_MAYFAIL); +} + +/* + * Return the address of the start of the given block number's data + * in a log buffer. The buffer covers a log sector-aligned region. + */ +static inline unsigned int +xlog_align( + struct xlog *log, + xfs_daddr_t blk_no) +{ + return BBTOB(blk_no & ((xfs_daddr_t)log->l_sectBBsize - 1)); +} + +static int +xlog_do_io( + struct xlog *log, + xfs_daddr_t blk_no, + unsigned int nbblks, + char *data, + enum req_op op) +{ + int error; + + if (XFS_IS_CORRUPT(log->l_mp, !xlog_verify_bno(log, blk_no, nbblks))) { + xfs_warn(log->l_mp, + "Invalid log block/length (0x%llx, 0x%x) for buffer", + blk_no, nbblks); + return -EFSCORRUPTED; + } + + blk_no = round_down(blk_no, log->l_sectBBsize); + nbblks = round_up(nbblks, log->l_sectBBsize); + ASSERT(nbblks > 0); + + error = xfs_rw_bdev(log->l_targ->bt_bdev, log->l_logBBstart + blk_no, + BBTOB(nbblks), data, op); + if (error && !xlog_is_shutdown(log)) { + xfs_alert(log->l_mp, + "log recovery %s I/O error at daddr 0x%llx len %d error %d", + op == REQ_OP_WRITE ? "write" : "read", + blk_no, nbblks, error); + } + return error; +} + +STATIC int +xlog_bread_noalign( + struct xlog *log, + xfs_daddr_t blk_no, + int nbblks, + char *data) +{ + return xlog_do_io(log, blk_no, nbblks, data, REQ_OP_READ); +} + +STATIC int +xlog_bread( + struct xlog *log, + xfs_daddr_t blk_no, + int nbblks, + char *data, + char **offset) +{ + int error; + + error = xlog_do_io(log, blk_no, nbblks, data, REQ_OP_READ); + if (!error) + *offset = data + xlog_align(log, blk_no); + return error; +} + +STATIC int +xlog_bwrite( + struct xlog *log, + xfs_daddr_t blk_no, + int nbblks, + char *data) +{ + return xlog_do_io(log, blk_no, nbblks, data, REQ_OP_WRITE); +} + +#ifdef DEBUG +/* + * dump debug superblock and log record information + */ +STATIC void +xlog_header_check_dump( + struct xfs_mount *mp, + struct xlog_rec_header *head) +{ + xfs_debug(mp, "%s: SB : uuid = %pU, fmt = %d", + __func__, &mp->m_sb.sb_uuid, XLOG_FMT); + xfs_debug(mp, " log : uuid = %pU, fmt = %d", + &head->h_fs_uuid, be32_to_cpu(head->h_fmt)); +} +#else +#define xlog_header_check_dump(mp, head) +#endif + +/* + * check log record header for recovery + */ +STATIC int +xlog_header_check_recover( + struct xfs_mount *mp, + struct xlog_rec_header *head) +{ + ASSERT(head->h_magicno == cpu_to_be32(XLOG_HEADER_MAGIC_NUM)); + + /* + * IRIX doesn't write the h_fmt field and leaves it zeroed + * (XLOG_FMT_UNKNOWN). This stops us from trying to recover + * a dirty log created in IRIX. + */ + if (XFS_IS_CORRUPT(mp, head->h_fmt != cpu_to_be32(XLOG_FMT))) { + xfs_warn(mp, + "dirty log written in incompatible format - can't recover"); + xlog_header_check_dump(mp, head); + return -EFSCORRUPTED; + } + if (XFS_IS_CORRUPT(mp, !uuid_equal(&mp->m_sb.sb_uuid, + &head->h_fs_uuid))) { + xfs_warn(mp, + "dirty log entry has mismatched uuid - can't recover"); + xlog_header_check_dump(mp, head); + return -EFSCORRUPTED; + } + return 0; +} + +/* + * read the head block of the log and check the header + */ +STATIC int +xlog_header_check_mount( + struct xfs_mount *mp, + struct xlog_rec_header *head) +{ + ASSERT(head->h_magicno == cpu_to_be32(XLOG_HEADER_MAGIC_NUM)); + + if (uuid_is_null(&head->h_fs_uuid)) { + /* + * IRIX doesn't write the h_fs_uuid or h_fmt fields. If + * h_fs_uuid is null, we assume this log was last mounted + * by IRIX and continue. + */ + xfs_warn(mp, "null uuid in log - IRIX style log"); + } else if (XFS_IS_CORRUPT(mp, !uuid_equal(&mp->m_sb.sb_uuid, + &head->h_fs_uuid))) { + xfs_warn(mp, "log has mismatched uuid - can't recover"); + xlog_header_check_dump(mp, head); + return -EFSCORRUPTED; + } + return 0; +} + +/* + * This routine finds (to an approximation) the first block in the physical + * log which contains the given cycle. It uses a binary search algorithm. + * Note that the algorithm can not be perfect because the disk will not + * necessarily be perfect. + */ +STATIC int +xlog_find_cycle_start( + struct xlog *log, + char *buffer, + xfs_daddr_t first_blk, + xfs_daddr_t *last_blk, + uint cycle) +{ + char *offset; + xfs_daddr_t mid_blk; + xfs_daddr_t end_blk; + uint mid_cycle; + int error; + + end_blk = *last_blk; + mid_blk = BLK_AVG(first_blk, end_blk); + while (mid_blk != first_blk && mid_blk != end_blk) { + error = xlog_bread(log, mid_blk, 1, buffer, &offset); + if (error) + return error; + mid_cycle = xlog_get_cycle(offset); + if (mid_cycle == cycle) + end_blk = mid_blk; /* last_half_cycle == mid_cycle */ + else + first_blk = mid_blk; /* first_half_cycle == mid_cycle */ + mid_blk = BLK_AVG(first_blk, end_blk); + } + ASSERT((mid_blk == first_blk && mid_blk+1 == end_blk) || + (mid_blk == end_blk && mid_blk-1 == first_blk)); + + *last_blk = end_blk; + + return 0; +} + +/* + * Check that a range of blocks does not contain stop_on_cycle_no. + * Fill in *new_blk with the block offset where such a block is + * found, or with -1 (an invalid block number) if there is no such + * block in the range. The scan needs to occur from front to back + * and the pointer into the region must be updated since a later + * routine will need to perform another test. + */ +STATIC int +xlog_find_verify_cycle( + struct xlog *log, + xfs_daddr_t start_blk, + int nbblks, + uint stop_on_cycle_no, + xfs_daddr_t *new_blk) +{ + xfs_daddr_t i, j; + uint cycle; + char *buffer; + xfs_daddr_t bufblks; + char *buf = NULL; + int error = 0; + + /* + * Greedily allocate a buffer big enough to handle the full + * range of basic blocks we'll be examining. If that fails, + * try a smaller size. We need to be able to read at least + * a log sector, or we're out of luck. + */ + bufblks = roundup_pow_of_two(nbblks); + while (bufblks > log->l_logBBsize) + bufblks >>= 1; + while (!(buffer = xlog_alloc_buffer(log, bufblks))) { + bufblks >>= 1; + if (bufblks < log->l_sectBBsize) + return -ENOMEM; + } + + for (i = start_blk; i < start_blk + nbblks; i += bufblks) { + int bcount; + + bcount = min(bufblks, (start_blk + nbblks - i)); + + error = xlog_bread(log, i, bcount, buffer, &buf); + if (error) + goto out; + + for (j = 0; j < bcount; j++) { + cycle = xlog_get_cycle(buf); + if (cycle == stop_on_cycle_no) { + *new_blk = i+j; + goto out; + } + + buf += BBSIZE; + } + } + + *new_blk = -1; + +out: + kvfree(buffer); + return error; +} + +static inline int +xlog_logrec_hblks(struct xlog *log, struct xlog_rec_header *rh) +{ + if (xfs_has_logv2(log->l_mp)) { + int h_size = be32_to_cpu(rh->h_size); + + if ((be32_to_cpu(rh->h_version) & XLOG_VERSION_2) && + h_size > XLOG_HEADER_CYCLE_SIZE) + return DIV_ROUND_UP(h_size, XLOG_HEADER_CYCLE_SIZE); + } + return 1; +} + +/* + * Potentially backup over partial log record write. + * + * In the typical case, last_blk is the number of the block directly after + * a good log record. Therefore, we subtract one to get the block number + * of the last block in the given buffer. extra_bblks contains the number + * of blocks we would have read on a previous read. This happens when the + * last log record is split over the end of the physical log. + * + * extra_bblks is the number of blocks potentially verified on a previous + * call to this routine. + */ +STATIC int +xlog_find_verify_log_record( + struct xlog *log, + xfs_daddr_t start_blk, + xfs_daddr_t *last_blk, + int extra_bblks) +{ + xfs_daddr_t i; + char *buffer; + char *offset = NULL; + struct xlog_rec_header *head = NULL; + int error = 0; + int smallmem = 0; + int num_blks = *last_blk - start_blk; + int xhdrs; + + ASSERT(start_blk != 0 || *last_blk != start_blk); + + buffer = xlog_alloc_buffer(log, num_blks); + if (!buffer) { + buffer = xlog_alloc_buffer(log, 1); + if (!buffer) + return -ENOMEM; + smallmem = 1; + } else { + error = xlog_bread(log, start_blk, num_blks, buffer, &offset); + if (error) + goto out; + offset += ((num_blks - 1) << BBSHIFT); + } + + for (i = (*last_blk) - 1; i >= 0; i--) { + if (i < start_blk) { + /* valid log record not found */ + xfs_warn(log->l_mp, + "Log inconsistent (didn't find previous header)"); + ASSERT(0); + error = -EFSCORRUPTED; + goto out; + } + + if (smallmem) { + error = xlog_bread(log, i, 1, buffer, &offset); + if (error) + goto out; + } + + head = (struct xlog_rec_header *)offset; + + if (head->h_magicno == cpu_to_be32(XLOG_HEADER_MAGIC_NUM)) + break; + + if (!smallmem) + offset -= BBSIZE; + } + + /* + * We hit the beginning of the physical log & still no header. Return + * to caller. If caller can handle a return of -1, then this routine + * will be called again for the end of the physical log. + */ + if (i == -1) { + error = 1; + goto out; + } + + /* + * We have the final block of the good log (the first block + * of the log record _before_ the head. So we check the uuid. + */ + if ((error = xlog_header_check_mount(log->l_mp, head))) + goto out; + + /* + * We may have found a log record header before we expected one. + * last_blk will be the 1st block # with a given cycle #. We may end + * up reading an entire log record. In this case, we don't want to + * reset last_blk. Only when last_blk points in the middle of a log + * record do we update last_blk. + */ + xhdrs = xlog_logrec_hblks(log, head); + + if (*last_blk - i + extra_bblks != + BTOBB(be32_to_cpu(head->h_len)) + xhdrs) + *last_blk = i; + +out: + kvfree(buffer); + return error; +} + +/* + * Head is defined to be the point of the log where the next log write + * could go. This means that incomplete LR writes at the end are + * eliminated when calculating the head. We aren't guaranteed that previous + * LR have complete transactions. We only know that a cycle number of + * current cycle number -1 won't be present in the log if we start writing + * from our current block number. + * + * last_blk contains the block number of the first block with a given + * cycle number. + * + * Return: zero if normal, non-zero if error. + */ +STATIC int +xlog_find_head( + struct xlog *log, + xfs_daddr_t *return_head_blk) +{ + char *buffer; + char *offset; + xfs_daddr_t new_blk, first_blk, start_blk, last_blk, head_blk; + int num_scan_bblks; + uint first_half_cycle, last_half_cycle; + uint stop_on_cycle; + int error, log_bbnum = log->l_logBBsize; + + /* Is the end of the log device zeroed? */ + error = xlog_find_zeroed(log, &first_blk); + if (error < 0) { + xfs_warn(log->l_mp, "empty log check failed"); + return error; + } + if (error == 1) { + *return_head_blk = first_blk; + + /* Is the whole lot zeroed? */ + if (!first_blk) { + /* Linux XFS shouldn't generate totally zeroed logs - + * mkfs etc write a dummy unmount record to a fresh + * log so we can store the uuid in there + */ + xfs_warn(log->l_mp, "totally zeroed log"); + } + + return 0; + } + + first_blk = 0; /* get cycle # of 1st block */ + buffer = xlog_alloc_buffer(log, 1); + if (!buffer) + return -ENOMEM; + + error = xlog_bread(log, 0, 1, buffer, &offset); + if (error) + goto out_free_buffer; + + first_half_cycle = xlog_get_cycle(offset); + + last_blk = head_blk = log_bbnum - 1; /* get cycle # of last block */ + error = xlog_bread(log, last_blk, 1, buffer, &offset); + if (error) + goto out_free_buffer; + + last_half_cycle = xlog_get_cycle(offset); + ASSERT(last_half_cycle != 0); + + /* + * If the 1st half cycle number is equal to the last half cycle number, + * then the entire log is stamped with the same cycle number. In this + * case, head_blk can't be set to zero (which makes sense). The below + * math doesn't work out properly with head_blk equal to zero. Instead, + * we set it to log_bbnum which is an invalid block number, but this + * value makes the math correct. If head_blk doesn't changed through + * all the tests below, *head_blk is set to zero at the very end rather + * than log_bbnum. In a sense, log_bbnum and zero are the same block + * in a circular file. + */ + if (first_half_cycle == last_half_cycle) { + /* + * In this case we believe that the entire log should have + * cycle number last_half_cycle. We need to scan backwards + * from the end verifying that there are no holes still + * containing last_half_cycle - 1. If we find such a hole, + * then the start of that hole will be the new head. The + * simple case looks like + * x | x ... | x - 1 | x + * Another case that fits this picture would be + * x | x + 1 | x ... | x + * In this case the head really is somewhere at the end of the + * log, as one of the latest writes at the beginning was + * incomplete. + * One more case is + * x | x + 1 | x ... | x - 1 | x + * This is really the combination of the above two cases, and + * the head has to end up at the start of the x-1 hole at the + * end of the log. + * + * In the 256k log case, we will read from the beginning to the + * end of the log and search for cycle numbers equal to x-1. + * We don't worry about the x+1 blocks that we encounter, + * because we know that they cannot be the head since the log + * started with x. + */ + head_blk = log_bbnum; + stop_on_cycle = last_half_cycle - 1; + } else { + /* + * In this case we want to find the first block with cycle + * number matching last_half_cycle. We expect the log to be + * some variation on + * x + 1 ... | x ... | x + * The first block with cycle number x (last_half_cycle) will + * be where the new head belongs. First we do a binary search + * for the first occurrence of last_half_cycle. The binary + * search may not be totally accurate, so then we scan back + * from there looking for occurrences of last_half_cycle before + * us. If that backwards scan wraps around the beginning of + * the log, then we look for occurrences of last_half_cycle - 1 + * at the end of the log. The cases we're looking for look + * like + * v binary search stopped here + * x + 1 ... | x | x + 1 | x ... | x + * ^ but we want to locate this spot + * or + * <---------> less than scan distance + * x + 1 ... | x ... | x - 1 | x + * ^ we want to locate this spot + */ + stop_on_cycle = last_half_cycle; + error = xlog_find_cycle_start(log, buffer, first_blk, &head_blk, + last_half_cycle); + if (error) + goto out_free_buffer; + } + + /* + * Now validate the answer. Scan back some number of maximum possible + * blocks and make sure each one has the expected cycle number. The + * maximum is determined by the total possible amount of buffering + * in the in-core log. The following number can be made tighter if + * we actually look at the block size of the filesystem. + */ + num_scan_bblks = min_t(int, log_bbnum, XLOG_TOTAL_REC_SHIFT(log)); + if (head_blk >= num_scan_bblks) { + /* + * We are guaranteed that the entire check can be performed + * in one buffer. + */ + start_blk = head_blk - num_scan_bblks; + if ((error = xlog_find_verify_cycle(log, + start_blk, num_scan_bblks, + stop_on_cycle, &new_blk))) + goto out_free_buffer; + if (new_blk != -1) + head_blk = new_blk; + } else { /* need to read 2 parts of log */ + /* + * We are going to scan backwards in the log in two parts. + * First we scan the physical end of the log. In this part + * of the log, we are looking for blocks with cycle number + * last_half_cycle - 1. + * If we find one, then we know that the log starts there, as + * we've found a hole that didn't get written in going around + * the end of the physical log. The simple case for this is + * x + 1 ... | x ... | x - 1 | x + * <---------> less than scan distance + * If all of the blocks at the end of the log have cycle number + * last_half_cycle, then we check the blocks at the start of + * the log looking for occurrences of last_half_cycle. If we + * find one, then our current estimate for the location of the + * first occurrence of last_half_cycle is wrong and we move + * back to the hole we've found. This case looks like + * x + 1 ... | x | x + 1 | x ... + * ^ binary search stopped here + * Another case we need to handle that only occurs in 256k + * logs is + * x + 1 ... | x ... | x+1 | x ... + * ^ binary search stops here + * In a 256k log, the scan at the end of the log will see the + * x + 1 blocks. We need to skip past those since that is + * certainly not the head of the log. By searching for + * last_half_cycle-1 we accomplish that. + */ + ASSERT(head_blk <= INT_MAX && + (xfs_daddr_t) num_scan_bblks >= head_blk); + start_blk = log_bbnum - (num_scan_bblks - head_blk); + if ((error = xlog_find_verify_cycle(log, start_blk, + num_scan_bblks - (int)head_blk, + (stop_on_cycle - 1), &new_blk))) + goto out_free_buffer; + if (new_blk != -1) { + head_blk = new_blk; + goto validate_head; + } + + /* + * Scan beginning of log now. The last part of the physical + * log is good. This scan needs to verify that it doesn't find + * the last_half_cycle. + */ + start_blk = 0; + ASSERT(head_blk <= INT_MAX); + if ((error = xlog_find_verify_cycle(log, + start_blk, (int)head_blk, + stop_on_cycle, &new_blk))) + goto out_free_buffer; + if (new_blk != -1) + head_blk = new_blk; + } + +validate_head: + /* + * Now we need to make sure head_blk is not pointing to a block in + * the middle of a log record. + */ + num_scan_bblks = XLOG_REC_SHIFT(log); + if (head_blk >= num_scan_bblks) { + start_blk = head_blk - num_scan_bblks; /* don't read head_blk */ + + /* start ptr at last block ptr before head_blk */ + error = xlog_find_verify_log_record(log, start_blk, &head_blk, 0); + if (error == 1) + error = -EIO; + if (error) + goto out_free_buffer; + } else { + start_blk = 0; + ASSERT(head_blk <= INT_MAX); + error = xlog_find_verify_log_record(log, start_blk, &head_blk, 0); + if (error < 0) + goto out_free_buffer; + if (error == 1) { + /* We hit the beginning of the log during our search */ + start_blk = log_bbnum - (num_scan_bblks - head_blk); + new_blk = log_bbnum; + ASSERT(start_blk <= INT_MAX && + (xfs_daddr_t) log_bbnum-start_blk >= 0); + ASSERT(head_blk <= INT_MAX); + error = xlog_find_verify_log_record(log, start_blk, + &new_blk, (int)head_blk); + if (error == 1) + error = -EIO; + if (error) + goto out_free_buffer; + if (new_blk != log_bbnum) + head_blk = new_blk; + } else if (error) + goto out_free_buffer; + } + + kvfree(buffer); + if (head_blk == log_bbnum) + *return_head_blk = 0; + else + *return_head_blk = head_blk; + /* + * When returning here, we have a good block number. Bad block + * means that during a previous crash, we didn't have a clean break + * from cycle number N to cycle number N-1. In this case, we need + * to find the first block with cycle number N-1. + */ + return 0; + +out_free_buffer: + kvfree(buffer); + if (error) + xfs_warn(log->l_mp, "failed to find log head"); + return error; +} + +/* + * Seek backwards in the log for log record headers. + * + * Given a starting log block, walk backwards until we find the provided number + * of records or hit the provided tail block. The return value is the number of + * records encountered or a negative error code. The log block and buffer + * pointer of the last record seen are returned in rblk and rhead respectively. + */ +STATIC int +xlog_rseek_logrec_hdr( + struct xlog *log, + xfs_daddr_t head_blk, + xfs_daddr_t tail_blk, + int count, + char *buffer, + xfs_daddr_t *rblk, + struct xlog_rec_header **rhead, + bool *wrapped) +{ + int i; + int error; + int found = 0; + char *offset = NULL; + xfs_daddr_t end_blk; + + *wrapped = false; + + /* + * Walk backwards from the head block until we hit the tail or the first + * block in the log. + */ + end_blk = head_blk > tail_blk ? tail_blk : 0; + for (i = (int) head_blk - 1; i >= end_blk; i--) { + error = xlog_bread(log, i, 1, buffer, &offset); + if (error) + goto out_error; + + if (*(__be32 *) offset == cpu_to_be32(XLOG_HEADER_MAGIC_NUM)) { + *rblk = i; + *rhead = (struct xlog_rec_header *) offset; + if (++found == count) + break; + } + } + + /* + * If we haven't hit the tail block or the log record header count, + * start looking again from the end of the physical log. Note that + * callers can pass head == tail if the tail is not yet known. + */ + if (tail_blk >= head_blk && found != count) { + for (i = log->l_logBBsize - 1; i >= (int) tail_blk; i--) { + error = xlog_bread(log, i, 1, buffer, &offset); + if (error) + goto out_error; + + if (*(__be32 *)offset == + cpu_to_be32(XLOG_HEADER_MAGIC_NUM)) { + *wrapped = true; + *rblk = i; + *rhead = (struct xlog_rec_header *) offset; + if (++found == count) + break; + } + } + } + + return found; + +out_error: + return error; +} + +/* + * Seek forward in the log for log record headers. + * + * Given head and tail blocks, walk forward from the tail block until we find + * the provided number of records or hit the head block. The return value is the + * number of records encountered or a negative error code. The log block and + * buffer pointer of the last record seen are returned in rblk and rhead + * respectively. + */ +STATIC int +xlog_seek_logrec_hdr( + struct xlog *log, + xfs_daddr_t head_blk, + xfs_daddr_t tail_blk, + int count, + char *buffer, + xfs_daddr_t *rblk, + struct xlog_rec_header **rhead, + bool *wrapped) +{ + int i; + int error; + int found = 0; + char *offset = NULL; + xfs_daddr_t end_blk; + + *wrapped = false; + + /* + * Walk forward from the tail block until we hit the head or the last + * block in the log. + */ + end_blk = head_blk > tail_blk ? head_blk : log->l_logBBsize - 1; + for (i = (int) tail_blk; i <= end_blk; i++) { + error = xlog_bread(log, i, 1, buffer, &offset); + if (error) + goto out_error; + + if (*(__be32 *) offset == cpu_to_be32(XLOG_HEADER_MAGIC_NUM)) { + *rblk = i; + *rhead = (struct xlog_rec_header *) offset; + if (++found == count) + break; + } + } + + /* + * If we haven't hit the head block or the log record header count, + * start looking again from the start of the physical log. + */ + if (tail_blk > head_blk && found != count) { + for (i = 0; i < (int) head_blk; i++) { + error = xlog_bread(log, i, 1, buffer, &offset); + if (error) + goto out_error; + + if (*(__be32 *)offset == + cpu_to_be32(XLOG_HEADER_MAGIC_NUM)) { + *wrapped = true; + *rblk = i; + *rhead = (struct xlog_rec_header *) offset; + if (++found == count) + break; + } + } + } + + return found; + +out_error: + return error; +} + +/* + * Calculate distance from head to tail (i.e., unused space in the log). + */ +static inline int +xlog_tail_distance( + struct xlog *log, + xfs_daddr_t head_blk, + xfs_daddr_t tail_blk) +{ + if (head_blk < tail_blk) + return tail_blk - head_blk; + + return tail_blk + (log->l_logBBsize - head_blk); +} + +/* + * Verify the log tail. This is particularly important when torn or incomplete + * writes have been detected near the front of the log and the head has been + * walked back accordingly. + * + * We also have to handle the case where the tail was pinned and the head + * blocked behind the tail right before a crash. If the tail had been pushed + * immediately prior to the crash and the subsequent checkpoint was only + * partially written, it's possible it overwrote the last referenced tail in the + * log with garbage. This is not a coherency problem because the tail must have + * been pushed before it can be overwritten, but appears as log corruption to + * recovery because we have no way to know the tail was updated if the + * subsequent checkpoint didn't write successfully. + * + * Therefore, CRC check the log from tail to head. If a failure occurs and the + * offending record is within max iclog bufs from the head, walk the tail + * forward and retry until a valid tail is found or corruption is detected out + * of the range of a possible overwrite. + */ +STATIC int +xlog_verify_tail( + struct xlog *log, + xfs_daddr_t head_blk, + xfs_daddr_t *tail_blk, + int hsize) +{ + struct xlog_rec_header *thead; + char *buffer; + xfs_daddr_t first_bad; + int error = 0; + bool wrapped; + xfs_daddr_t tmp_tail; + xfs_daddr_t orig_tail = *tail_blk; + + buffer = xlog_alloc_buffer(log, 1); + if (!buffer) + return -ENOMEM; + + /* + * Make sure the tail points to a record (returns positive count on + * success). + */ + error = xlog_seek_logrec_hdr(log, head_blk, *tail_blk, 1, buffer, + &tmp_tail, &thead, &wrapped); + if (error < 0) + goto out; + if (*tail_blk != tmp_tail) + *tail_blk = tmp_tail; + + /* + * Run a CRC check from the tail to the head. We can't just check + * MAX_ICLOGS records past the tail because the tail may point to stale + * blocks cleared during the search for the head/tail. These blocks are + * overwritten with zero-length records and thus record count is not a + * reliable indicator of the iclog state before a crash. + */ + first_bad = 0; + error = xlog_do_recovery_pass(log, head_blk, *tail_blk, + XLOG_RECOVER_CRCPASS, &first_bad); + while ((error == -EFSBADCRC || error == -EFSCORRUPTED) && first_bad) { + int tail_distance; + + /* + * Is corruption within range of the head? If so, retry from + * the next record. Otherwise return an error. + */ + tail_distance = xlog_tail_distance(log, head_blk, first_bad); + if (tail_distance > BTOBB(XLOG_MAX_ICLOGS * hsize)) + break; + + /* skip to the next record; returns positive count on success */ + error = xlog_seek_logrec_hdr(log, head_blk, first_bad, 2, + buffer, &tmp_tail, &thead, &wrapped); + if (error < 0) + goto out; + + *tail_blk = tmp_tail; + first_bad = 0; + error = xlog_do_recovery_pass(log, head_blk, *tail_blk, + XLOG_RECOVER_CRCPASS, &first_bad); + } + + if (!error && *tail_blk != orig_tail) + xfs_warn(log->l_mp, + "Tail block (0x%llx) overwrite detected. Updated to 0x%llx", + orig_tail, *tail_blk); +out: + kvfree(buffer); + return error; +} + +/* + * Detect and trim torn writes from the head of the log. + * + * Storage without sector atomicity guarantees can result in torn writes in the + * log in the event of a crash. Our only means to detect this scenario is via + * CRC verification. While we can't always be certain that CRC verification + * failure is due to a torn write vs. an unrelated corruption, we do know that + * only a certain number (XLOG_MAX_ICLOGS) of log records can be written out at + * one time. Therefore, CRC verify up to XLOG_MAX_ICLOGS records at the head of + * the log and treat failures in this range as torn writes as a matter of + * policy. In the event of CRC failure, the head is walked back to the last good + * record in the log and the tail is updated from that record and verified. + */ +STATIC int +xlog_verify_head( + struct xlog *log, + xfs_daddr_t *head_blk, /* in/out: unverified head */ + xfs_daddr_t *tail_blk, /* out: tail block */ + char *buffer, + xfs_daddr_t *rhead_blk, /* start blk of last record */ + struct xlog_rec_header **rhead, /* ptr to last record */ + bool *wrapped) /* last rec. wraps phys. log */ +{ + struct xlog_rec_header *tmp_rhead; + char *tmp_buffer; + xfs_daddr_t first_bad = XFS_BUF_DADDR_NULL; + xfs_daddr_t tmp_rhead_blk; + int found; + int error; + bool tmp_wrapped; + + /* + * Check the head of the log for torn writes. Search backwards from the + * head until we hit the tail or the maximum number of log record I/Os + * that could have been in flight at one time. Use a temporary buffer so + * we don't trash the rhead/buffer pointers from the caller. + */ + tmp_buffer = xlog_alloc_buffer(log, 1); + if (!tmp_buffer) + return -ENOMEM; + error = xlog_rseek_logrec_hdr(log, *head_blk, *tail_blk, + XLOG_MAX_ICLOGS, tmp_buffer, + &tmp_rhead_blk, &tmp_rhead, &tmp_wrapped); + kvfree(tmp_buffer); + if (error < 0) + return error; + + /* + * Now run a CRC verification pass over the records starting at the + * block found above to the current head. If a CRC failure occurs, the + * log block of the first bad record is saved in first_bad. + */ + error = xlog_do_recovery_pass(log, *head_blk, tmp_rhead_blk, + XLOG_RECOVER_CRCPASS, &first_bad); + if ((error == -EFSBADCRC || error == -EFSCORRUPTED) && + first_bad != XFS_BUF_DADDR_NULL) { + /* + * We've hit a potential torn write. Reset the error and warn + * about it. + */ + error = 0; + xfs_warn(log->l_mp, +"Torn write (CRC failure) detected at log block 0x%llx. Truncating head block from 0x%llx.", + first_bad, *head_blk); + + /* + * Get the header block and buffer pointer for the last good + * record before the bad record. + * + * Note that xlog_find_tail() clears the blocks at the new head + * (i.e., the records with invalid CRC) if the cycle number + * matches the current cycle. + */ + found = xlog_rseek_logrec_hdr(log, first_bad, *tail_blk, 1, + buffer, rhead_blk, rhead, wrapped); + if (found < 0) + return found; + if (found == 0) /* XXX: right thing to do here? */ + return -EIO; + + /* + * Reset the head block to the starting block of the first bad + * log record and set the tail block based on the last good + * record. + * + * Bail out if the updated head/tail match as this indicates + * possible corruption outside of the acceptable + * (XLOG_MAX_ICLOGS) range. This is a job for xfs_repair... + */ + *head_blk = first_bad; + *tail_blk = BLOCK_LSN(be64_to_cpu((*rhead)->h_tail_lsn)); + if (*head_blk == *tail_blk) { + ASSERT(0); + return 0; + } + } + if (error) + return error; + + return xlog_verify_tail(log, *head_blk, tail_blk, + be32_to_cpu((*rhead)->h_size)); +} + +/* + * We need to make sure we handle log wrapping properly, so we can't use the + * calculated logbno directly. Make sure it wraps to the correct bno inside the + * log. + * + * The log is limited to 32 bit sizes, so we use the appropriate modulus + * operation here and cast it back to a 64 bit daddr on return. + */ +static inline xfs_daddr_t +xlog_wrap_logbno( + struct xlog *log, + xfs_daddr_t bno) +{ + int mod; + + div_s64_rem(bno, log->l_logBBsize, &mod); + return mod; +} + +/* + * Check whether the head of the log points to an unmount record. In other + * words, determine whether the log is clean. If so, update the in-core state + * appropriately. + */ +static int +xlog_check_unmount_rec( + struct xlog *log, + xfs_daddr_t *head_blk, + xfs_daddr_t *tail_blk, + struct xlog_rec_header *rhead, + xfs_daddr_t rhead_blk, + char *buffer, + bool *clean) +{ + struct xlog_op_header *op_head; + xfs_daddr_t umount_data_blk; + xfs_daddr_t after_umount_blk; + int hblks; + int error; + char *offset; + + *clean = false; + + /* + * Look for unmount record. If we find it, then we know there was a + * clean unmount. Since 'i' could be the last block in the physical + * log, we convert to a log block before comparing to the head_blk. + * + * Save the current tail lsn to use to pass to xlog_clear_stale_blocks() + * below. We won't want to clear the unmount record if there is one, so + * we pass the lsn of the unmount record rather than the block after it. + */ + hblks = xlog_logrec_hblks(log, rhead); + after_umount_blk = xlog_wrap_logbno(log, + rhead_blk + hblks + BTOBB(be32_to_cpu(rhead->h_len))); + + if (*head_blk == after_umount_blk && + be32_to_cpu(rhead->h_num_logops) == 1) { + umount_data_blk = xlog_wrap_logbno(log, rhead_blk + hblks); + error = xlog_bread(log, umount_data_blk, 1, buffer, &offset); + if (error) + return error; + + op_head = (struct xlog_op_header *)offset; + if (op_head->oh_flags & XLOG_UNMOUNT_TRANS) { + /* + * Set tail and last sync so that newly written log + * records will point recovery to after the current + * unmount record. + */ + xlog_assign_atomic_lsn(&log->l_tail_lsn, + log->l_curr_cycle, after_umount_blk); + log->l_ailp->ail_head_lsn = + atomic64_read(&log->l_tail_lsn); + *tail_blk = after_umount_blk; + + *clean = true; + } + } + + return 0; +} + +static void +xlog_set_state( + struct xlog *log, + xfs_daddr_t head_blk, + struct xlog_rec_header *rhead, + xfs_daddr_t rhead_blk, + bool bump_cycle) +{ + /* + * Reset log values according to the state of the log when we + * crashed. In the case where head_blk == 0, we bump curr_cycle + * one because the next write starts a new cycle rather than + * continuing the cycle of the last good log record. At this + * point we have guaranteed that all partial log records have been + * accounted for. Therefore, we know that the last good log record + * written was complete and ended exactly on the end boundary + * of the physical log. + */ + log->l_prev_block = rhead_blk; + log->l_curr_block = (int)head_blk; + log->l_curr_cycle = be32_to_cpu(rhead->h_cycle); + if (bump_cycle) + log->l_curr_cycle++; + atomic64_set(&log->l_tail_lsn, be64_to_cpu(rhead->h_tail_lsn)); + log->l_ailp->ail_head_lsn = be64_to_cpu(rhead->h_lsn); +} + +/* + * Find the sync block number or the tail of the log. + * + * This will be the block number of the last record to have its + * associated buffers synced to disk. Every log record header has + * a sync lsn embedded in it. LSNs hold block numbers, so it is easy + * to get a sync block number. The only concern is to figure out which + * log record header to believe. + * + * The following algorithm uses the log record header with the largest + * lsn. The entire log record does not need to be valid. We only care + * that the header is valid. + * + * We could speed up search by using current head_blk buffer, but it is not + * available. + */ +STATIC int +xlog_find_tail( + struct xlog *log, + xfs_daddr_t *head_blk, + xfs_daddr_t *tail_blk) +{ + struct xlog_rec_header *rhead; + char *offset = NULL; + char *buffer; + int error; + xfs_daddr_t rhead_blk; + xfs_lsn_t tail_lsn; + bool wrapped = false; + bool clean = false; + + /* + * Find previous log record + */ + if ((error = xlog_find_head(log, head_blk))) + return error; + ASSERT(*head_blk < INT_MAX); + + buffer = xlog_alloc_buffer(log, 1); + if (!buffer) + return -ENOMEM; + if (*head_blk == 0) { /* special case */ + error = xlog_bread(log, 0, 1, buffer, &offset); + if (error) + goto done; + + if (xlog_get_cycle(offset) == 0) { + *tail_blk = 0; + /* leave all other log inited values alone */ + goto done; + } + } + + /* + * Search backwards through the log looking for the log record header + * block. This wraps all the way back around to the head so something is + * seriously wrong if we can't find it. + */ + error = xlog_rseek_logrec_hdr(log, *head_blk, *head_blk, 1, buffer, + &rhead_blk, &rhead, &wrapped); + if (error < 0) + goto done; + if (!error) { + xfs_warn(log->l_mp, "%s: couldn't find sync record", __func__); + error = -EFSCORRUPTED; + goto done; + } + *tail_blk = BLOCK_LSN(be64_to_cpu(rhead->h_tail_lsn)); + + /* + * Set the log state based on the current head record. + */ + xlog_set_state(log, *head_blk, rhead, rhead_blk, wrapped); + tail_lsn = atomic64_read(&log->l_tail_lsn); + + /* + * Look for an unmount record at the head of the log. This sets the log + * state to determine whether recovery is necessary. + */ + error = xlog_check_unmount_rec(log, head_blk, tail_blk, rhead, + rhead_blk, buffer, &clean); + if (error) + goto done; + + /* + * Verify the log head if the log is not clean (e.g., we have anything + * but an unmount record at the head). This uses CRC verification to + * detect and trim torn writes. If discovered, CRC failures are + * considered torn writes and the log head is trimmed accordingly. + * + * Note that we can only run CRC verification when the log is dirty + * because there's no guarantee that the log data behind an unmount + * record is compatible with the current architecture. + */ + if (!clean) { + xfs_daddr_t orig_head = *head_blk; + + error = xlog_verify_head(log, head_blk, tail_blk, buffer, + &rhead_blk, &rhead, &wrapped); + if (error) + goto done; + + /* update in-core state again if the head changed */ + if (*head_blk != orig_head) { + xlog_set_state(log, *head_blk, rhead, rhead_blk, + wrapped); + tail_lsn = atomic64_read(&log->l_tail_lsn); + error = xlog_check_unmount_rec(log, head_blk, tail_blk, + rhead, rhead_blk, buffer, + &clean); + if (error) + goto done; + } + } + + /* + * Note that the unmount was clean. If the unmount was not clean, we + * need to know this to rebuild the superblock counters from the perag + * headers if we have a filesystem using non-persistent counters. + */ + if (clean) + xfs_set_clean(log->l_mp); + + /* + * Make sure that there are no blocks in front of the head + * with the same cycle number as the head. This can happen + * because we allow multiple outstanding log writes concurrently, + * and the later writes might make it out before earlier ones. + * + * We use the lsn from before modifying it so that we'll never + * overwrite the unmount record after a clean unmount. + * + * Do this only if we are going to recover the filesystem + * + * NOTE: This used to say "if (!readonly)" + * However on Linux, we can & do recover a read-only filesystem. + * We only skip recovery if NORECOVERY is specified on mount, + * in which case we would not be here. + * + * But... if the -device- itself is readonly, just skip this. + * We can't recover this device anyway, so it won't matter. + */ + if (!xfs_readonly_buftarg(log->l_targ)) + error = xlog_clear_stale_blocks(log, tail_lsn); + +done: + kvfree(buffer); + + if (error) + xfs_warn(log->l_mp, "failed to locate log tail"); + return error; +} + +/* + * Is the log zeroed at all? + * + * The last binary search should be changed to perform an X block read + * once X becomes small enough. You can then search linearly through + * the X blocks. This will cut down on the number of reads we need to do. + * + * If the log is partially zeroed, this routine will pass back the blkno + * of the first block with cycle number 0. It won't have a complete LR + * preceding it. + * + * Return: + * 0 => the log is completely written to + * 1 => use *blk_no as the first block of the log + * <0 => error has occurred + */ +STATIC int +xlog_find_zeroed( + struct xlog *log, + xfs_daddr_t *blk_no) +{ + char *buffer; + char *offset; + uint first_cycle, last_cycle; + xfs_daddr_t new_blk, last_blk, start_blk; + xfs_daddr_t num_scan_bblks; + int error, log_bbnum = log->l_logBBsize; + int ret = 1; + + *blk_no = 0; + + /* check totally zeroed log */ + buffer = xlog_alloc_buffer(log, 1); + if (!buffer) + return -ENOMEM; + error = xlog_bread(log, 0, 1, buffer, &offset); + if (error) + goto out_free_buffer; + + first_cycle = xlog_get_cycle(offset); + if (first_cycle == 0) { /* completely zeroed log */ + *blk_no = 0; + goto out_free_buffer; + } + + /* check partially zeroed log */ + error = xlog_bread(log, log_bbnum-1, 1, buffer, &offset); + if (error) + goto out_free_buffer; + + last_cycle = xlog_get_cycle(offset); + if (last_cycle != 0) { /* log completely written to */ + ret = 0; + goto out_free_buffer; + } + + /* we have a partially zeroed log */ + last_blk = log_bbnum-1; + error = xlog_find_cycle_start(log, buffer, 0, &last_blk, 0); + if (error) + goto out_free_buffer; + + /* + * Validate the answer. Because there is no way to guarantee that + * the entire log is made up of log records which are the same size, + * we scan over the defined maximum blocks. At this point, the maximum + * is not chosen to mean anything special. XXXmiken + */ + num_scan_bblks = XLOG_TOTAL_REC_SHIFT(log); + ASSERT(num_scan_bblks <= INT_MAX); + + if (last_blk < num_scan_bblks) + num_scan_bblks = last_blk; + start_blk = last_blk - num_scan_bblks; + + /* + * We search for any instances of cycle number 0 that occur before + * our current estimate of the head. What we're trying to detect is + * 1 ... | 0 | 1 | 0... + * ^ binary search ends here + */ + if ((error = xlog_find_verify_cycle(log, start_blk, + (int)num_scan_bblks, 0, &new_blk))) + goto out_free_buffer; + if (new_blk != -1) + last_blk = new_blk; + + /* + * Potentially backup over partial log record write. We don't need + * to search the end of the log because we know it is zero. + */ + error = xlog_find_verify_log_record(log, start_blk, &last_blk, 0); + if (error == 1) + error = -EIO; + if (error) + goto out_free_buffer; + + *blk_no = last_blk; +out_free_buffer: + kvfree(buffer); + if (error) + return error; + return ret; +} + +/* + * These are simple subroutines used by xlog_clear_stale_blocks() below + * to initialize a buffer full of empty log record headers and write + * them into the log. + */ +STATIC void +xlog_add_record( + struct xlog *log, + char *buf, + int cycle, + int block, + int tail_cycle, + int tail_block) +{ + struct xlog_rec_header *recp = (struct xlog_rec_header *)buf; + + memset(buf, 0, BBSIZE); + recp->h_magicno = cpu_to_be32(XLOG_HEADER_MAGIC_NUM); + recp->h_cycle = cpu_to_be32(cycle); + recp->h_version = cpu_to_be32( + xfs_has_logv2(log->l_mp) ? 2 : 1); + recp->h_lsn = cpu_to_be64(xlog_assign_lsn(cycle, block)); + recp->h_tail_lsn = cpu_to_be64(xlog_assign_lsn(tail_cycle, tail_block)); + recp->h_fmt = cpu_to_be32(XLOG_FMT); + memcpy(&recp->h_fs_uuid, &log->l_mp->m_sb.sb_uuid, sizeof(uuid_t)); +} + +STATIC int +xlog_write_log_records( + struct xlog *log, + int cycle, + int start_block, + int blocks, + int tail_cycle, + int tail_block) +{ + char *offset; + char *buffer; + int balign, ealign; + int sectbb = log->l_sectBBsize; + int end_block = start_block + blocks; + int bufblks; + int error = 0; + int i, j = 0; + + /* + * Greedily allocate a buffer big enough to handle the full + * range of basic blocks to be written. If that fails, try + * a smaller size. We need to be able to write at least a + * log sector, or we're out of luck. + */ + bufblks = roundup_pow_of_two(blocks); + while (bufblks > log->l_logBBsize) + bufblks >>= 1; + while (!(buffer = xlog_alloc_buffer(log, bufblks))) { + bufblks >>= 1; + if (bufblks < sectbb) + return -ENOMEM; + } + + /* We may need to do a read at the start to fill in part of + * the buffer in the starting sector not covered by the first + * write below. + */ + balign = round_down(start_block, sectbb); + if (balign != start_block) { + error = xlog_bread_noalign(log, start_block, 1, buffer); + if (error) + goto out_free_buffer; + + j = start_block - balign; + } + + for (i = start_block; i < end_block; i += bufblks) { + int bcount, endcount; + + bcount = min(bufblks, end_block - start_block); + endcount = bcount - j; + + /* We may need to do a read at the end to fill in part of + * the buffer in the final sector not covered by the write. + * If this is the same sector as the above read, skip it. + */ + ealign = round_down(end_block, sectbb); + if (j == 0 && (start_block + endcount > ealign)) { + error = xlog_bread_noalign(log, ealign, sectbb, + buffer + BBTOB(ealign - start_block)); + if (error) + break; + + } + + offset = buffer + xlog_align(log, start_block); + for (; j < endcount; j++) { + xlog_add_record(log, offset, cycle, i+j, + tail_cycle, tail_block); + offset += BBSIZE; + } + error = xlog_bwrite(log, start_block, endcount, buffer); + if (error) + break; + start_block += endcount; + j = 0; + } + +out_free_buffer: + kvfree(buffer); + return error; +} + +/* + * This routine is called to blow away any incomplete log writes out + * in front of the log head. We do this so that we won't become confused + * if we come up, write only a little bit more, and then crash again. + * If we leave the partial log records out there, this situation could + * cause us to think those partial writes are valid blocks since they + * have the current cycle number. We get rid of them by overwriting them + * with empty log records with the old cycle number rather than the + * current one. + * + * The tail lsn is passed in rather than taken from + * the log so that we will not write over the unmount record after a + * clean unmount in a 512 block log. Doing so would leave the log without + * any valid log records in it until a new one was written. If we crashed + * during that time we would not be able to recover. + */ +STATIC int +xlog_clear_stale_blocks( + struct xlog *log, + xfs_lsn_t tail_lsn) +{ + int tail_cycle, head_cycle; + int tail_block, head_block; + int tail_distance, max_distance; + int distance; + int error; + + tail_cycle = CYCLE_LSN(tail_lsn); + tail_block = BLOCK_LSN(tail_lsn); + head_cycle = log->l_curr_cycle; + head_block = log->l_curr_block; + + /* + * Figure out the distance between the new head of the log + * and the tail. We want to write over any blocks beyond the + * head that we may have written just before the crash, but + * we don't want to overwrite the tail of the log. + */ + if (head_cycle == tail_cycle) { + /* + * The tail is behind the head in the physical log, + * so the distance from the head to the tail is the + * distance from the head to the end of the log plus + * the distance from the beginning of the log to the + * tail. + */ + if (XFS_IS_CORRUPT(log->l_mp, + head_block < tail_block || + head_block >= log->l_logBBsize)) + return -EFSCORRUPTED; + tail_distance = tail_block + (log->l_logBBsize - head_block); + } else { + /* + * The head is behind the tail in the physical log, + * so the distance from the head to the tail is just + * the tail block minus the head block. + */ + if (XFS_IS_CORRUPT(log->l_mp, + head_block >= tail_block || + head_cycle != tail_cycle + 1)) + return -EFSCORRUPTED; + tail_distance = tail_block - head_block; + } + + /* + * If the head is right up against the tail, we can't clear + * anything. + */ + if (tail_distance <= 0) { + ASSERT(tail_distance == 0); + return 0; + } + + max_distance = XLOG_TOTAL_REC_SHIFT(log); + /* + * Take the smaller of the maximum amount of outstanding I/O + * we could have and the distance to the tail to clear out. + * We take the smaller so that we don't overwrite the tail and + * we don't waste all day writing from the head to the tail + * for no reason. + */ + max_distance = min(max_distance, tail_distance); + + if ((head_block + max_distance) <= log->l_logBBsize) { + /* + * We can stomp all the blocks we need to without + * wrapping around the end of the log. Just do it + * in a single write. Use the cycle number of the + * current cycle minus one so that the log will look like: + * n ... | n - 1 ... + */ + error = xlog_write_log_records(log, (head_cycle - 1), + head_block, max_distance, tail_cycle, + tail_block); + if (error) + return error; + } else { + /* + * We need to wrap around the end of the physical log in + * order to clear all the blocks. Do it in two separate + * I/Os. The first write should be from the head to the + * end of the physical log, and it should use the current + * cycle number minus one just like above. + */ + distance = log->l_logBBsize - head_block; + error = xlog_write_log_records(log, (head_cycle - 1), + head_block, distance, tail_cycle, + tail_block); + + if (error) + return error; + + /* + * Now write the blocks at the start of the physical log. + * This writes the remainder of the blocks we want to clear. + * It uses the current cycle number since we're now on the + * same cycle as the head so that we get: + * n ... n ... | n - 1 ... + * ^^^^^ blocks we're writing + */ + distance = max_distance - (log->l_logBBsize - head_block); + error = xlog_write_log_records(log, head_cycle, 0, distance, + tail_cycle, tail_block); + if (error) + return error; + } + + return 0; +} + +/* + * Release the recovered intent item in the AIL that matches the given intent + * type and intent id. + */ +void +xlog_recover_release_intent( + struct xlog *log, + unsigned short intent_type, + uint64_t intent_id) +{ + struct xfs_defer_pending *dfp, *n; + + list_for_each_entry_safe(dfp, n, &log->r_dfops, dfp_list) { + struct xfs_log_item *lip = dfp->dfp_intent; + + if (lip->li_type != intent_type) + continue; + if (!lip->li_ops->iop_match(lip, intent_id)) + continue; + + ASSERT(xlog_item_is_intent(lip)); + + xfs_defer_cancel_recovery(log->l_mp, dfp); + } +} + +int +xlog_recover_iget( + struct xfs_mount *mp, + xfs_ino_t ino, + struct xfs_inode **ipp) +{ + int error; + + error = xfs_iget(mp, NULL, ino, 0, 0, ipp); + if (error) + return error; + + error = xfs_qm_dqattach(*ipp); + if (error) { + xfs_irele(*ipp); + return error; + } + + if (VFS_I(*ipp)->i_nlink == 0) + xfs_iflags_set(*ipp, XFS_IRECOVERY); + + return 0; +} + +/* + * Get an inode so that we can recover a log operation. + * + * Log intent items that target inodes effectively contain a file handle. + * Check that the generation number matches the intent item like we do for + * other file handles. Log intent items defined after this validation weakness + * was identified must use this function. + */ +int +xlog_recover_iget_handle( + struct xfs_mount *mp, + xfs_ino_t ino, + uint32_t gen, + struct xfs_inode **ipp) +{ + struct xfs_inode *ip; + int error; + + error = xlog_recover_iget(mp, ino, &ip); + if (error) + return error; + + if (VFS_I(ip)->i_generation != gen) { + xfs_irele(ip); + return -EFSCORRUPTED; + } + + *ipp = ip; + return 0; +} + +/****************************************************************************** + * + * Log recover routines + * + ****************************************************************************** + */ +static const struct xlog_recover_item_ops *xlog_recover_item_ops[] = { + &xlog_buf_item_ops, + &xlog_inode_item_ops, + &xlog_dquot_item_ops, + &xlog_quotaoff_item_ops, + &xlog_icreate_item_ops, + &xlog_efi_item_ops, + &xlog_efd_item_ops, + &xlog_rui_item_ops, + &xlog_rud_item_ops, + &xlog_cui_item_ops, + &xlog_cud_item_ops, + &xlog_bui_item_ops, + &xlog_bud_item_ops, + &xlog_attri_item_ops, + &xlog_attrd_item_ops, + &xlog_xmi_item_ops, + &xlog_xmd_item_ops, + &xlog_rtefi_item_ops, + &xlog_rtefd_item_ops, + &xlog_rtrui_item_ops, + &xlog_rtrud_item_ops, + &xlog_rtcui_item_ops, + &xlog_rtcud_item_ops, +}; + +static const struct xlog_recover_item_ops * +xlog_find_item_ops( + struct xlog_recover_item *item) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(xlog_recover_item_ops); i++) + if (ITEM_TYPE(item) == xlog_recover_item_ops[i]->item_type) + return xlog_recover_item_ops[i]; + + return NULL; +} + +/* + * Sort the log items in the transaction. + * + * The ordering constraints are defined by the inode allocation and unlink + * behaviour. The rules are: + * + * 1. Every item is only logged once in a given transaction. Hence it + * represents the last logged state of the item. Hence ordering is + * dependent on the order in which operations need to be performed so + * required initial conditions are always met. + * + * 2. Cancelled buffers are recorded in pass 1 in a separate table and + * there's nothing to replay from them so we can simply cull them + * from the transaction. However, we can't do that until after we've + * replayed all the other items because they may be dependent on the + * cancelled buffer and replaying the cancelled buffer can remove it + * form the cancelled buffer table. Hence they have to be done last. + * + * 3. Inode allocation buffers must be replayed before inode items that + * read the buffer and replay changes into it. For filesystems using the + * ICREATE transactions, this means XFS_LI_ICREATE objects need to get + * treated the same as inode allocation buffers as they create and + * initialise the buffers directly. + * + * 4. Inode unlink buffers must be replayed after inode items are replayed. + * This ensures that inodes are completely flushed to the inode buffer + * in a "free" state before we remove the unlinked inode list pointer. + * + * Hence the ordering needs to be inode allocation buffers first, inode items + * second, inode unlink buffers third and cancelled buffers last. + * + * But there's a problem with that - we can't tell an inode allocation buffer + * apart from a regular buffer, so we can't separate them. We can, however, + * tell an inode unlink buffer from the others, and so we can separate them out + * from all the other buffers and move them to last. + * + * Hence, 4 lists, in order from head to tail: + * - buffer_list for all buffers except cancelled/inode unlink buffers + * - item_list for all non-buffer items + * - inode_buffer_list for inode unlink buffers + * - cancel_list for the cancelled buffers + * + * Note that we add objects to the tail of the lists so that first-to-last + * ordering is preserved within the lists. Adding objects to the head of the + * list means when we traverse from the head we walk them in last-to-first + * order. For cancelled buffers and inode unlink buffers this doesn't matter, + * but for all other items there may be specific ordering that we need to + * preserve. + */ +STATIC int +xlog_recover_reorder_trans( + struct xlog *log, + struct xlog_recover *trans, + int pass) +{ + struct xlog_recover_item *item, *n; + int error = 0; + LIST_HEAD(sort_list); + LIST_HEAD(cancel_list); + LIST_HEAD(buffer_list); + LIST_HEAD(inode_buffer_list); + LIST_HEAD(item_list); + + list_splice_init(&trans->r_itemq, &sort_list); + list_for_each_entry_safe(item, n, &sort_list, ri_list) { + enum xlog_recover_reorder fate = XLOG_REORDER_ITEM_LIST; + + /* a committed item with no regions has a NULL ri_buf[0] */ + if (!item->ri_cnt || !item->ri_buf) { + xfs_warn(log->l_mp, + "%s: committed log item has no regions", + __func__); + error = -EFSCORRUPTED; + break; + } + + item->ri_ops = xlog_find_item_ops(item); + if (!item->ri_ops) { + xfs_warn(log->l_mp, + "%s: unrecognized type of log operation (%d)", + __func__, ITEM_TYPE(item)); + error = -EFSCORRUPTED; + break; + } + + if (item->ri_ops->reorder) + fate = item->ri_ops->reorder(item); + + switch (fate) { + case XLOG_REORDER_BUFFER_LIST: + list_move_tail(&item->ri_list, &buffer_list); + break; + case XLOG_REORDER_CANCEL_LIST: + trace_xfs_log_recover_item_reorder_head(log, + trans, item, pass); + list_move(&item->ri_list, &cancel_list); + break; + case XLOG_REORDER_INODE_BUFFER_LIST: + list_move(&item->ri_list, &inode_buffer_list); + break; + case XLOG_REORDER_ITEM_LIST: + trace_xfs_log_recover_item_reorder_tail(log, + trans, item, pass); + list_move_tail(&item->ri_list, &item_list); + break; + } + } + + /* + * Return the remaining items back to the transaction item list so they + * can be freed in caller. This should only happen when we encounter + * an error. + */ + if (!list_empty(&sort_list)) { + ASSERT(error); + list_splice_init(&sort_list, &trans->r_itemq); + } + if (!list_empty(&buffer_list)) + list_splice(&buffer_list, &trans->r_itemq); + if (!list_empty(&item_list)) + list_splice_tail(&item_list, &trans->r_itemq); + if (!list_empty(&inode_buffer_list)) + list_splice_tail(&inode_buffer_list, &trans->r_itemq); + if (!list_empty(&cancel_list)) + list_splice_tail(&cancel_list, &trans->r_itemq); + return error; +} + +void +xlog_buf_readahead( + struct xlog *log, + xfs_daddr_t blkno, + uint len, + const struct xfs_buf_ops *ops) +{ + if (!xlog_is_buffer_cancelled(log, blkno, len)) + xfs_buf_readahead(log->l_mp->m_ddev_targp, blkno, len, ops); +} + +/* + * Create a deferred work structure for resuming and tracking the progress of a + * log intent item that was found during recovery. + */ +void +xlog_recover_intent_item( + struct xlog *log, + struct xfs_log_item *lip, + xfs_lsn_t lsn, + const struct xfs_defer_op_type *ops) +{ + ASSERT(xlog_item_is_intent(lip)); + + xfs_defer_start_recovery(lip, &log->r_dfops, ops); + + /* + * Insert the intent into the AIL directly and drop one reference so + * that finishing or canceling the work will drop the other. + */ + xfs_trans_ail_insert(log->l_ailp, lip, lsn); + lip->li_ops->iop_unpin(lip, 0); +} + +STATIC int +xlog_recover_items_pass2( + struct xlog *log, + struct xlog_recover *trans, + struct list_head *buffer_list, + struct list_head *item_list) +{ + struct xlog_recover_item *item; + int error = 0; + + list_for_each_entry(item, item_list, ri_list) { + trace_xfs_log_recover_item_recover(log, trans, item, + XLOG_RECOVER_PASS2); + + if (item->ri_ops->commit_pass2) + error = item->ri_ops->commit_pass2(log, buffer_list, + item, trans->r_lsn); + if (error) + return error; + } + + return error; +} + +/* + * Perform the transaction. + * + * If the transaction modifies a buffer or inode, do it now. Otherwise, + * EFIs and EFDs get queued up by adding entries into the AIL for them. + */ +STATIC int +xlog_recover_commit_trans( + struct xlog *log, + struct xlog_recover *trans, + int pass, + struct list_head *buffer_list) +{ + int error = 0; + int items_queued = 0; + struct xlog_recover_item *item; + struct xlog_recover_item *next; + LIST_HEAD (ra_list); + LIST_HEAD (done_list); + + #define XLOG_RECOVER_COMMIT_QUEUE_MAX 100 + + hlist_del_init(&trans->r_list); + + error = xlog_recover_reorder_trans(log, trans, pass); + if (error) + return error; + + list_for_each_entry_safe(item, next, &trans->r_itemq, ri_list) { + trace_xfs_log_recover_item_recover(log, trans, item, pass); + + switch (pass) { + case XLOG_RECOVER_PASS1: + if (item->ri_ops->commit_pass1) + error = item->ri_ops->commit_pass1(log, item); + break; + case XLOG_RECOVER_PASS2: + if (item->ri_ops->ra_pass2) + item->ri_ops->ra_pass2(log, item); + list_move_tail(&item->ri_list, &ra_list); + items_queued++; + if (items_queued >= XLOG_RECOVER_COMMIT_QUEUE_MAX) { + error = xlog_recover_items_pass2(log, trans, + buffer_list, &ra_list); + list_splice_tail_init(&ra_list, &done_list); + items_queued = 0; + } + + break; + default: + ASSERT(0); + } + + if (error) + goto out; + } + +out: + if (!list_empty(&ra_list)) { + if (!error) + error = xlog_recover_items_pass2(log, trans, + buffer_list, &ra_list); + list_splice_tail_init(&ra_list, &done_list); + } + + if (!list_empty(&done_list)) + list_splice_init(&done_list, &trans->r_itemq); + + return error; +} + +STATIC void +xlog_recover_add_item( + struct list_head *head) +{ + struct xlog_recover_item *item; + + item = kzalloc_obj(struct xlog_recover_item, GFP_KERNEL | __GFP_NOFAIL); + INIT_LIST_HEAD(&item->ri_list); + list_add_tail(&item->ri_list, head); +} + +STATIC int +xlog_recover_add_to_cont_trans( + struct xlog *log, + struct xlog_recover *trans, + char *dp, + int len) +{ + struct xlog_recover_item *item; + char *ptr, *old_ptr; + int old_len; + + /* + * If the transaction is empty, the header was split across this and the + * previous record. Copy the rest of the header. + */ + if (list_empty(&trans->r_itemq)) { + ASSERT(len <= sizeof(struct xfs_trans_header)); + if (len > sizeof(struct xfs_trans_header)) { + xfs_warn(log->l_mp, "%s: bad header length", __func__); + return -EFSCORRUPTED; + } + + xlog_recover_add_item(&trans->r_itemq); + ptr = (char *)&trans->r_theader + + sizeof(struct xfs_trans_header) - len; + memcpy(ptr, dp, len); + return 0; + } + + /* take the tail entry */ + item = list_entry(trans->r_itemq.prev, struct xlog_recover_item, + ri_list); + + old_ptr = item->ri_buf[item->ri_cnt-1].iov_base; + old_len = item->ri_buf[item->ri_cnt-1].iov_len; + + ptr = kvrealloc(old_ptr, len + old_len, GFP_KERNEL); + if (!ptr) + return -ENOMEM; + memcpy(&ptr[old_len], dp, len); + item->ri_buf[item->ri_cnt-1].iov_len += len; + item->ri_buf[item->ri_cnt-1].iov_base = ptr; + trace_xfs_log_recover_item_add_cont(log, trans, item, 0); + return 0; +} + +/* + * The next region to add is the start of a new region. It could be + * a whole region or it could be the first part of a new region. Because + * of this, the assumption here is that the type and size fields of all + * format structures fit into the first 32 bits of the structure. + * + * This works because all regions must be 32 bit aligned. Therefore, we + * either have both fields or we have neither field. In the case we have + * neither field, the data part of the region is zero length. We only have + * a log_op_header and can throw away the header since a new one will appear + * later. If we have at least 4 bytes, then we can determine how many regions + * will appear in the current log item. + */ +STATIC int +xlog_recover_add_to_trans( + struct xlog *log, + struct xlog_recover *trans, + char *dp, + int len) +{ + struct xfs_inode_log_format *in_f; /* any will do */ + struct xlog_recover_item *item; + char *ptr; + + if (!len) + return 0; + if (list_empty(&trans->r_itemq)) { + /* we need to catch log corruptions here */ + if (*(uint *)dp != XFS_TRANS_HEADER_MAGIC) { + xfs_warn(log->l_mp, "%s: bad header magic number", + __func__); + ASSERT(0); + return -EFSCORRUPTED; + } + + if (len > sizeof(struct xfs_trans_header)) { + xfs_warn(log->l_mp, "%s: bad header length", __func__); + ASSERT(0); + return -EFSCORRUPTED; + } + + /* + * The transaction header can be arbitrarily split across op + * records. If we don't have the whole thing here, copy what we + * do have and handle the rest in the next record. + */ + if (len == sizeof(struct xfs_trans_header)) + xlog_recover_add_item(&trans->r_itemq); + memcpy(&trans->r_theader, dp, len); + return 0; + } + + ptr = xlog_kvmalloc(len); + memcpy(ptr, dp, len); + in_f = (struct xfs_inode_log_format *)ptr; + + /* take the tail entry */ + item = list_entry(trans->r_itemq.prev, struct xlog_recover_item, + ri_list); + if (item->ri_total != 0 && + item->ri_total == item->ri_cnt) { + /* tail item is in use, get a new one */ + xlog_recover_add_item(&trans->r_itemq); + item = list_entry(trans->r_itemq.prev, + struct xlog_recover_item, ri_list); + } + + if (item->ri_total == 0) { /* first region to be added */ + if (in_f->ilf_size == 0 || + in_f->ilf_size > XLOG_MAX_REGIONS_IN_ITEM) { + xfs_warn(log->l_mp, + "bad number of regions (%d) in inode log format", + in_f->ilf_size); + ASSERT(0); + kvfree(ptr); + return -EFSCORRUPTED; + } + + item->ri_total = in_f->ilf_size; + item->ri_buf = kzalloc_objs(*item->ri_buf, item->ri_total, + GFP_KERNEL | __GFP_NOFAIL); + } + + if (item->ri_total <= item->ri_cnt) { + xfs_warn(log->l_mp, + "log item region count (%d) overflowed size (%d)", + item->ri_cnt, item->ri_total); + ASSERT(0); + kvfree(ptr); + return -EFSCORRUPTED; + } + + /* Description region is ri_buf[0] */ + item->ri_buf[item->ri_cnt].iov_base = ptr; + item->ri_buf[item->ri_cnt].iov_len = len; + item->ri_cnt++; + trace_xfs_log_recover_item_add(log, trans, item, 0); + return 0; +} + +/* + * Free up any resources allocated by the transaction + * + * Remember that EFIs, EFDs, and IUNLINKs are handled later. + */ +STATIC void +xlog_recover_free_trans( + struct xlog_recover *trans) +{ + struct xlog_recover_item *item, *n; + int i; + + hlist_del_init(&trans->r_list); + + list_for_each_entry_safe(item, n, &trans->r_itemq, ri_list) { + /* Free the regions in the item. */ + list_del(&item->ri_list); + for (i = 0; i < item->ri_cnt; i++) + kvfree(item->ri_buf[i].iov_base); + /* Free the item itself */ + kfree(item->ri_buf); + kfree(item); + } + /* Free the transaction recover structure */ + kfree(trans); +} + +/* + * On error or completion, trans is freed. + */ +STATIC int +xlog_recovery_process_trans( + struct xlog *log, + struct xlog_recover *trans, + char *dp, + unsigned int len, + unsigned int flags, + int pass, + struct list_head *buffer_list) +{ + int error = 0; + bool freeit = false; + + /* mask off ophdr transaction container flags */ + flags &= ~XLOG_END_TRANS; + if (flags & XLOG_WAS_CONT_TRANS) + flags &= ~XLOG_CONTINUE_TRANS; + + /* + * Callees must not free the trans structure. We'll decide if we need to + * free it or not based on the operation being done and it's result. + */ + switch (flags) { + /* expected flag values */ + case 0: + case XLOG_CONTINUE_TRANS: + error = xlog_recover_add_to_trans(log, trans, dp, len); + break; + case XLOG_WAS_CONT_TRANS: + error = xlog_recover_add_to_cont_trans(log, trans, dp, len); + break; + case XLOG_COMMIT_TRANS: + error = xlog_recover_commit_trans(log, trans, pass, + buffer_list); + /* success or fail, we are now done with this transaction. */ + freeit = true; + break; + + /* unexpected flag values */ + case XLOG_UNMOUNT_TRANS: + /* just skip trans */ + xfs_warn(log->l_mp, "%s: Unmount LR", __func__); + freeit = true; + break; + case XLOG_START_TRANS: + default: + xfs_warn(log->l_mp, "%s: bad flag 0x%x", __func__, flags); + ASSERT(0); + error = -EFSCORRUPTED; + break; + } + if (error || freeit) + xlog_recover_free_trans(trans); + return error; +} + +/* + * Lookup the transaction recovery structure associated with the ID in the + * current ophdr. If the transaction doesn't exist and the start flag is set in + * the ophdr, then allocate a new transaction for future ID matches to find. + * Either way, return what we found during the lookup - an existing transaction + * or nothing. + */ +STATIC struct xlog_recover * +xlog_recover_ophdr_to_trans( + struct hlist_head rhash[], + struct xlog_rec_header *rhead, + struct xlog_op_header *ohead) +{ + struct xlog_recover *trans; + xlog_tid_t tid; + struct hlist_head *rhp; + + tid = be32_to_cpu(ohead->oh_tid); + rhp = &rhash[XLOG_RHASH(tid)]; + hlist_for_each_entry(trans, rhp, r_list) { + if (trans->r_log_tid == tid) + return trans; + } + + /* + * skip over non-start transaction headers - we could be + * processing slack space before the next transaction starts + */ + if (!(ohead->oh_flags & XLOG_START_TRANS)) + return NULL; + + ASSERT(be32_to_cpu(ohead->oh_len) == 0); + + /* + * This is a new transaction so allocate a new recovery container to + * hold the recovery ops that will follow. + */ + trans = kzalloc_obj(struct xlog_recover, GFP_KERNEL | __GFP_NOFAIL); + trans->r_log_tid = tid; + trans->r_lsn = be64_to_cpu(rhead->h_lsn); + INIT_LIST_HEAD(&trans->r_itemq); + INIT_HLIST_NODE(&trans->r_list); + hlist_add_head(&trans->r_list, rhp); + + /* + * Nothing more to do for this ophdr. Items to be added to this new + * transaction will be in subsequent ophdr containers. + */ + return NULL; +} + +STATIC int +xlog_recover_process_ophdr( + struct xlog *log, + struct hlist_head rhash[], + struct xlog_rec_header *rhead, + struct xlog_op_header *ohead, + char *dp, + char *end, + int pass, + struct list_head *buffer_list) +{ + struct xlog_recover *trans; + unsigned int len; + int error; + + /* Do we understand who wrote this op? */ + if (ohead->oh_clientid != XFS_TRANSACTION && + ohead->oh_clientid != XFS_LOG) { + xfs_warn(log->l_mp, "%s: bad clientid 0x%x", + __func__, ohead->oh_clientid); + ASSERT(0); + return -EFSCORRUPTED; + } + + /* + * Check the ophdr contains all the data it is supposed to contain. + */ + len = be32_to_cpu(ohead->oh_len); + if (dp + len > end) { + xfs_warn(log->l_mp, "%s: bad length 0x%x", __func__, len); + WARN_ON(1); + return -EFSCORRUPTED; + } + + trans = xlog_recover_ophdr_to_trans(rhash, rhead, ohead); + if (!trans) { + /* nothing to do, so skip over this ophdr */ + return 0; + } + + /* + * The recovered buffer queue is drained only once we know that all + * recovery items for the current LSN have been processed. This is + * required because: + * + * - Buffer write submission updates the metadata LSN of the buffer. + * - Log recovery skips items with a metadata LSN >= the current LSN of + * the recovery item. + * - Separate recovery items against the same metadata buffer can share + * a current LSN. I.e., consider that the LSN of a recovery item is + * defined as the starting LSN of the first record in which its + * transaction appears, that a record can hold multiple transactions, + * and/or that a transaction can span multiple records. + * + * In other words, we are allowed to submit a buffer from log recovery + * once per current LSN. Otherwise, we may incorrectly skip recovery + * items and cause corruption. + * + * We don't know up front whether buffers are updated multiple times per + * LSN. Therefore, track the current LSN of each commit log record as it + * is processed and drain the queue when it changes. Use commit records + * because they are ordered correctly by the logging code. + */ + if (log->l_recovery_lsn != trans->r_lsn && + ohead->oh_flags & XLOG_COMMIT_TRANS) { + error = xfs_buf_delwri_submit(buffer_list); + if (error) + return error; + log->l_recovery_lsn = trans->r_lsn; + } + + return xlog_recovery_process_trans(log, trans, dp, len, + ohead->oh_flags, pass, buffer_list); +} + +/* + * There are two valid states of the r_state field. 0 indicates that the + * transaction structure is in a normal state. We have either seen the + * start of the transaction or the last operation we added was not a partial + * operation. If the last operation we added to the transaction was a + * partial operation, we need to mark r_state with XLOG_WAS_CONT_TRANS. + * + * NOTE: skip LRs with 0 data length. + */ +STATIC int +xlog_recover_process_data( + struct xlog *log, + struct hlist_head rhash[], + struct xlog_rec_header *rhead, + char *dp, + int pass, + struct list_head *buffer_list) +{ + struct xlog_op_header *ohead; + char *end; + int num_logops; + int error; + + end = dp + be32_to_cpu(rhead->h_len); + num_logops = be32_to_cpu(rhead->h_num_logops); + + /* check the log format matches our own - else we can't recover */ + if (xlog_header_check_recover(log->l_mp, rhead)) + return -EIO; + + trace_xfs_log_recover_record(log, rhead, pass); + while ((dp < end) && num_logops) { + + ohead = (struct xlog_op_header *)dp; + dp += sizeof(*ohead); + if (dp > end) { + xfs_warn(log->l_mp, "%s: op header overrun", __func__); + return -EFSCORRUPTED; + } + + /* errors will abort recovery */ + error = xlog_recover_process_ophdr(log, rhash, rhead, ohead, + dp, end, pass, buffer_list); + if (error) + return error; + + dp += be32_to_cpu(ohead->oh_len); + num_logops--; + } + return 0; +} + +/* Take all the collected deferred ops and finish them in order. */ +static int +xlog_finish_defer_ops( + struct xfs_mount *mp, + struct list_head *capture_list) +{ + struct xfs_defer_capture *dfc, *next; + struct xfs_trans *tp; + int error = 0; + + list_for_each_entry_safe(dfc, next, capture_list, dfc_list) { + struct xfs_trans_res resv; + struct xfs_defer_resources dres; + + /* + * Create a new transaction reservation from the captured + * information. Set logcount to 1 to force the new transaction + * to regrant every roll so that we can make forward progress + * in recovery no matter how full the log might be. + */ + resv.tr_logres = dfc->dfc_logres; + resv.tr_logcount = 1; + resv.tr_logflags = XFS_TRANS_PERM_LOG_RES; + + error = xfs_trans_alloc(mp, &resv, dfc->dfc_blkres, + dfc->dfc_rtxres, XFS_TRANS_RESERVE, &tp); + if (error) { + xlog_force_shutdown(mp->m_log, SHUTDOWN_LOG_IO_ERROR); + return error; + } + + /* + * Transfer to this new transaction all the dfops we captured + * from recovering a single intent item. + */ + list_del_init(&dfc->dfc_list); + xfs_defer_ops_continue(dfc, tp, &dres); + error = xfs_trans_commit(tp); + xfs_defer_resources_rele(&dres); + if (error) + return error; + } + + ASSERT(list_empty(capture_list)); + return 0; +} + +/* Release all the captured defer ops and capture structures in this list. */ +static void +xlog_abort_defer_ops( + struct xfs_mount *mp, + struct list_head *capture_list) +{ + struct xfs_defer_capture *dfc; + struct xfs_defer_capture *next; + + list_for_each_entry_safe(dfc, next, capture_list, dfc_list) { + list_del_init(&dfc->dfc_list); + xfs_defer_ops_capture_abort(mp, dfc); + } +} + +/* + * When this is called, all of the log intent items which did not have + * corresponding log done items should be in the AIL. What we do now is update + * the data structures associated with each one. + * + * Since we process the log intent items in normal transactions, they will be + * removed at some point after the commit. This prevents us from just walking + * down the list processing each one. We'll use a flag in the intent item to + * skip those that we've already processed and use the AIL iteration mechanism's + * generation count to try to speed this up at least a bit. + * + * When we start, we know that the intents are the only things in the AIL. As we + * process them, however, other items are added to the AIL. Hence we know we + * have started recovery on all the pending intents when we find an non-intent + * item in the AIL. + */ +STATIC int +xlog_recover_process_intents( + struct xlog *log) +{ + LIST_HEAD(capture_list); + struct xfs_defer_pending *dfp, *n; + int error = 0; +#if defined(DEBUG) || defined(XFS_WARN) + xfs_lsn_t last_lsn; + + last_lsn = xlog_assign_lsn(log->l_curr_cycle, log->l_curr_block); +#endif + + list_for_each_entry_safe(dfp, n, &log->r_dfops, dfp_list) { + ASSERT(xlog_item_is_intent(dfp->dfp_intent)); + + /* + * We should never see a redo item with a LSN higher than + * the last transaction we found in the log at the start + * of recovery. + */ + ASSERT(XFS_LSN_CMP(last_lsn, dfp->dfp_intent->li_lsn) >= 0); + + /* + * NOTE: If your intent processing routine can create more + * deferred ops, you /must/ attach them to the capture list in + * the recover routine or else those subsequent intents will be + * replayed in the wrong order! + * + * The recovery function can free the log item, so we must not + * access dfp->dfp_intent after it returns. It must dispose of + * @dfp if it returns 0. + */ + error = xfs_defer_finish_recovery(log->l_mp, dfp, + &capture_list); + if (error) + break; + } + if (error) + goto err; + + error = xlog_finish_defer_ops(log->l_mp, &capture_list); + if (error) + goto err; + + return 0; +err: + xlog_abort_defer_ops(log->l_mp, &capture_list); + return error; +} + +/* + * A cancel occurs when the mount has failed and we're bailing out. Release all + * pending log intent items that we haven't started recovery on so they don't + * pin the AIL. + */ +STATIC void +xlog_recover_cancel_intents( + struct xlog *log) +{ + struct xfs_defer_pending *dfp, *n; + + list_for_each_entry_safe(dfp, n, &log->r_dfops, dfp_list) { + ASSERT(xlog_item_is_intent(dfp->dfp_intent)); + + xfs_defer_cancel_recovery(log->l_mp, dfp); + } +} + +/* + * Transfer ownership of the recovered pending work to the recovery transaction + * and try to finish the work. If there is more work to be done, the dfp will + * remain attached to the transaction. If not, the dfp is freed. + */ +int +xlog_recover_finish_intent( + struct xfs_trans *tp, + struct xfs_defer_pending *dfp) +{ + int error; + + list_move(&dfp->dfp_list, &tp->t_dfops); + error = xfs_defer_finish_one(tp, dfp); + if (error == -EAGAIN) + return 0; + return error; +} + +/* + * This routine performs a transaction to null out a bad inode pointer + * in an agi unlinked inode hash bucket. + */ +STATIC void +xlog_recover_clear_agi_bucket( + struct xfs_perag *pag, + int bucket) +{ + struct xfs_mount *mp = pag_mount(pag); + struct xfs_trans *tp; + struct xfs_agi *agi; + struct xfs_buf *agibp; + int offset; + int error; + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_clearagi, 0, 0, 0, &tp); + if (error) + goto out_error; + + error = xfs_read_agi(pag, tp, 0, &agibp); + if (error) + goto out_abort; + + agi = agibp->b_addr; + agi->agi_unlinked[bucket] = cpu_to_be32(NULLAGINO); + offset = offsetof(xfs_agi_t, agi_unlinked) + + (sizeof(xfs_agino_t) * bucket); + xfs_trans_log_buf(tp, agibp, offset, + (offset + sizeof(xfs_agino_t) - 1)); + + error = xfs_trans_commit(tp); + if (error) + goto out_error; + return; + +out_abort: + xfs_trans_cancel(tp); +out_error: + xfs_warn(mp, "%s: failed to clear agi %d. Continuing.", __func__, + pag_agno(pag)); + return; +} + +static int +xlog_recover_iunlink_bucket( + struct xfs_perag *pag, + struct xfs_agi *agi, + int bucket) +{ + struct xfs_mount *mp = pag_mount(pag); + struct xfs_inode *prev_ip = NULL; + struct xfs_inode *ip; + xfs_agino_t prev_agino, agino; + int error = 0; + + agino = be32_to_cpu(agi->agi_unlinked[bucket]); + while (agino != NULLAGINO) { + error = xfs_iget(mp, NULL, xfs_agino_to_ino(pag, agino), 0, 0, + &ip); + if (error) + break; + + ASSERT(VFS_I(ip)->i_nlink == 0); + ASSERT(VFS_I(ip)->i_mode != 0); + xfs_iflags_clear(ip, XFS_IRECOVERY); + agino = ip->i_next_unlinked; + + if (prev_ip) { + ip->i_prev_unlinked = prev_agino; + xfs_irele(prev_ip); + + /* + * Ensure the inode is removed from the unlinked list + * before we continue so that it won't race with + * building the in-memory list here. This could be + * serialised with the agibp lock, but that just + * serialises via lockstepping and it's much simpler + * just to flush the inodegc queue and wait for it to + * complete. + */ + error = xfs_inodegc_flush(mp); + if (error) + break; + } + + prev_agino = agino; + prev_ip = ip; + } + + if (prev_ip) { + int error2; + + ip->i_prev_unlinked = prev_agino; + xfs_irele(prev_ip); + + error2 = xfs_inodegc_flush(mp); + if (error2 && !error) + return error2; + } + return error; +} + +/* + * Recover AGI unlinked lists + * + * This is called during recovery to process any inodes which we unlinked but + * not freed when the system crashed. These inodes will be on the lists in the + * AGI blocks. What we do here is scan all the AGIs and fully truncate and free + * any inodes found on the lists. Each inode is removed from the lists when it + * has been fully truncated and is freed. The freeing of the inode and its + * removal from the list must be atomic. + * + * If everything we touch in the agi processing loop is already in memory, this + * loop can hold the cpu for a long time. It runs without lock contention, + * memory allocation contention, the need wait for IO, etc, and so will run + * until we either run out of inodes to process, run low on memory or we run out + * of log space. + * + * This behaviour is bad for latency on single CPU and non-preemptible kernels, + * and can prevent other filesystem work (such as CIL pushes) from running. This + * can lead to deadlocks if the recovery process runs out of log reservation + * space. Hence we need to yield the CPU when there is other kernel work + * scheduled on this CPU to ensure other scheduled work can run without undue + * latency. + */ +static void +xlog_recover_iunlink_ag( + struct xfs_perag *pag) +{ + struct xfs_agi *agi; + struct xfs_buf *agibp; + int bucket; + int error; + + error = xfs_read_agi(pag, NULL, 0, &agibp); + if (error) { + /* + * AGI is b0rked. Don't process it. + * + * We should probably mark the filesystem as corrupt after we've + * recovered all the ag's we can.... + */ + return; + } + + /* + * Unlock the buffer so that it can be acquired in the normal course of + * the transaction to truncate and free each inode. Because we are not + * racing with anyone else here for the AGI buffer, we don't even need + * to hold it locked to read the initial unlinked bucket entries out of + * the buffer. We keep buffer reference though, so that it stays pinned + * in memory while we need the buffer. + */ + agi = agibp->b_addr; + xfs_buf_unlock(agibp); + + for (bucket = 0; bucket < XFS_AGI_UNLINKED_BUCKETS; bucket++) { + error = xlog_recover_iunlink_bucket(pag, agi, bucket); + if (error) { + /* + * Bucket is unrecoverable, so only a repair scan can + * free the remaining unlinked inodes. Just empty the + * bucket and remaining inodes on it unreferenced and + * unfreeable. + */ + xlog_recover_clear_agi_bucket(pag, bucket); + } + } + + xfs_buf_rele(agibp); +} + +static void +xlog_recover_process_iunlinks( + struct xlog *log) +{ + struct xfs_perag *pag = NULL; + + while ((pag = xfs_perag_next(log->l_mp, pag))) + xlog_recover_iunlink_ag(pag); +} + +STATIC void +xlog_unpack_data( + struct xlog_rec_header *rhead, + char *dp, + struct xlog *log) +{ + int i; + + for (i = 0; i < BTOBB(be32_to_cpu(rhead->h_len)); i++) { + *(__be32 *)dp = *xlog_cycle_data(rhead, i); + dp += BBSIZE; + } +} + +/* + * CRC check, unpack and process a log record. + */ +STATIC int +xlog_recover_process( + struct xlog *log, + struct hlist_head rhash[], + struct xlog_rec_header *rhead, + char *dp, + int pass, + struct list_head *buffer_list) +{ + __le32 expected_crc = rhead->h_crc, crc, other_crc; + + crc = xlog_cksum(log, rhead, dp, XLOG_REC_SIZE, + be32_to_cpu(rhead->h_len)); + + /* + * Look at the end of the struct xlog_rec_header definition in + * xfs_log_format.h for the glory details. + */ + if (expected_crc && crc != expected_crc) { + other_crc = xlog_cksum(log, rhead, dp, XLOG_REC_SIZE_OTHER, + be32_to_cpu(rhead->h_len)); + if (other_crc == expected_crc) { + xfs_notice_once(log->l_mp, + "Fixing up incorrect CRC due to padding."); + crc = other_crc; + } + } + + /* + * Nothing else to do if this is a CRC verification pass. Just return + * if this a record with a non-zero crc. Unfortunately, mkfs always + * sets expected_crc to 0 so we must consider this valid even on v5 + * supers. Otherwise, return EFSBADCRC on failure so the callers up the + * stack know precisely what failed. + */ + if (pass == XLOG_RECOVER_CRCPASS) { + if (expected_crc && crc != expected_crc) + return -EFSBADCRC; + return 0; + } + + /* + * We're in the normal recovery path. Issue a warning if and only if the + * CRC in the header is non-zero. This is an advisory warning and the + * zero CRC check prevents warnings from being emitted when upgrading + * the kernel from one that does not add CRCs by default. + */ + if (crc != expected_crc) { + if (expected_crc || xfs_has_crc(log->l_mp)) { + xfs_alert(log->l_mp, + "log record CRC mismatch: found 0x%x, expected 0x%x.", + le32_to_cpu(expected_crc), + le32_to_cpu(crc)); + xfs_hex_dump(dp, 32); + } + + /* + * If the filesystem is CRC enabled, this mismatch becomes a + * fatal log corruption failure. + */ + if (xfs_has_crc(log->l_mp)) { + XFS_ERROR_REPORT(__func__, XFS_ERRLEVEL_LOW, log->l_mp); + return -EFSCORRUPTED; + } + } + + xlog_unpack_data(rhead, dp, log); + + return xlog_recover_process_data(log, rhash, rhead, dp, pass, + buffer_list); +} + +STATIC int +xlog_valid_rec_header( + struct xlog *log, + struct xlog_rec_header *rhead, + xfs_daddr_t blkno, + int bufsize) +{ + struct xfs_mount *mp = log->l_mp; + u32 h_version = be32_to_cpu(rhead->h_version); + int hlen; + + if (XFS_IS_CORRUPT(mp, + rhead->h_magicno != cpu_to_be32(XLOG_HEADER_MAGIC_NUM))) + return -EFSCORRUPTED; + + /* + * The log version must match the superblock + */ + if (xfs_has_logv2(mp)) { + if (XFS_IS_CORRUPT(mp, h_version != XLOG_VERSION_2)) + return -EFSCORRUPTED; + } else { + if (XFS_IS_CORRUPT(mp, h_version != XLOG_VERSION_1)) + return -EFSCORRUPTED; + } + + /* + * LR body must have data (or it wouldn't have been written) + * and h_len must not be greater than LR buffer size. + */ + hlen = be32_to_cpu(rhead->h_len); + if (XFS_IS_CORRUPT(mp, hlen <= 0 || hlen > bufsize)) + return -EFSCORRUPTED; + + if (XFS_IS_CORRUPT(mp, blkno > log->l_logBBsize || blkno > INT_MAX)) + return -EFSCORRUPTED; + + return 0; +} + +/* + * Read the log from tail to head and process the log records found. + * Handle the two cases where the tail and head are in the same cycle + * and where the active portion of the log wraps around the end of + * the physical log separately. The pass parameter is passed through + * to the routines called to process the data and is not looked at + * here. + */ +STATIC int +xlog_do_recovery_pass( + struct xlog *log, + xfs_daddr_t head_blk, + xfs_daddr_t tail_blk, + int pass, + xfs_daddr_t *first_bad) /* out: first bad log rec */ +{ + struct xlog_rec_header *rhead; + xfs_daddr_t blk_no, rblk_no; + xfs_daddr_t rhead_blk; + char *offset; + char *hbp, *dbp; + int error = 0, h_size, h_len; + int error2 = 0; + int bblks, split_bblks; + int hblks = 1, split_hblks, wrapped_hblks; + int i; + struct hlist_head rhash[XLOG_RHASH_SIZE]; + LIST_HEAD (buffer_list); + + ASSERT(head_blk != tail_blk); + blk_no = rhead_blk = tail_blk; + + for (i = 0; i < XLOG_RHASH_SIZE; i++) + INIT_HLIST_HEAD(&rhash[i]); + + hbp = xlog_alloc_buffer(log, hblks); + if (!hbp) + return -ENOMEM; + + /* + * Read the header of the tail block and get the iclog buffer size from + * h_size. Use this to tell how many sectors make up the log header. + */ + if (xfs_has_logv2(log->l_mp)) { + /* + * When using variable length iclogs, read first sector of + * iclog header and extract the header size from it. Get a + * new hbp that is the correct size. + */ + error = xlog_bread(log, tail_blk, 1, hbp, &offset); + if (error) + goto bread_err1; + + rhead = (struct xlog_rec_header *)offset; + + /* + * xfsprogs has a bug where record length is based on lsunit but + * h_size (iclog size) is hardcoded to 32k. Now that we + * unconditionally CRC verify the unmount record, this means the + * log buffer can be too small for the record and cause an + * overrun. + * + * Detect this condition here. Use lsunit for the buffer size as + * long as this looks like the mkfs case. Otherwise, return an + * error to avoid a buffer overrun. + */ + h_size = be32_to_cpu(rhead->h_size); + h_len = be32_to_cpu(rhead->h_len); + if (h_len > h_size && h_len <= log->l_mp->m_logbsize && + rhead->h_num_logops == cpu_to_be32(1)) { + xfs_warn(log->l_mp, + "invalid iclog size (%d bytes), using lsunit (%d bytes)", + h_size, log->l_mp->m_logbsize); + h_size = log->l_mp->m_logbsize; + } + + error = xlog_valid_rec_header(log, rhead, tail_blk, h_size); + if (error) + goto bread_err1; + + /* + * This open codes xlog_logrec_hblks so that we can reuse the + * fixed up h_size value calculated above. Without that we'd + * still allocate the buffer based on the incorrect on-disk + * size. + */ + if (h_size > XLOG_HEADER_CYCLE_SIZE && + (rhead->h_version & cpu_to_be32(XLOG_VERSION_2))) { + hblks = DIV_ROUND_UP(h_size, XLOG_HEADER_CYCLE_SIZE); + if (hblks > 1) { + kvfree(hbp); + hbp = xlog_alloc_buffer(log, hblks); + if (!hbp) + return -ENOMEM; + } + } + } else { + ASSERT(log->l_sectBBsize == 1); + h_size = XLOG_BIG_RECORD_BSIZE; + } + + dbp = xlog_alloc_buffer(log, BTOBB(h_size)); + if (!dbp) { + kvfree(hbp); + return -ENOMEM; + } + + memset(rhash, 0, sizeof(rhash)); + if (tail_blk > head_blk) { + /* + * Perform recovery around the end of the physical log. + * When the head is not on the same cycle number as the tail, + * we can't do a sequential recovery. + */ + while (blk_no < log->l_logBBsize) { + /* + * Check for header wrapping around physical end-of-log + */ + offset = hbp; + split_hblks = 0; + wrapped_hblks = 0; + if (blk_no + hblks <= log->l_logBBsize) { + /* Read header in one read */ + error = xlog_bread(log, blk_no, hblks, hbp, + &offset); + if (error) + goto bread_err2; + } else { + /* This LR is split across physical log end */ + if (blk_no != log->l_logBBsize) { + /* some data before physical log end */ + ASSERT(blk_no <= INT_MAX); + split_hblks = log->l_logBBsize - (int)blk_no; + ASSERT(split_hblks > 0); + error = xlog_bread(log, blk_no, + split_hblks, hbp, + &offset); + if (error) + goto bread_err2; + } + + /* + * Note: this black magic still works with + * large sector sizes (non-512) only because: + * - we increased the buffer size originally + * by 1 sector giving us enough extra space + * for the second read; + * - the log start is guaranteed to be sector + * aligned; + * - we read the log end (LR header start) + * _first_, then the log start (LR header end) + * - order is important. + */ + wrapped_hblks = hblks - split_hblks; + error = xlog_bread_noalign(log, 0, + wrapped_hblks, + offset + BBTOB(split_hblks)); + if (error) + goto bread_err2; + } + rhead = (struct xlog_rec_header *)offset; + error = xlog_valid_rec_header(log, rhead, + split_hblks ? blk_no : 0, h_size); + if (error) + goto bread_err2; + + bblks = (int)BTOBB(be32_to_cpu(rhead->h_len)); + blk_no += hblks; + + /* + * Read the log record data in multiple reads if it + * wraps around the end of the log. Note that if the + * header already wrapped, blk_no could point past the + * end of the log. The record data is contiguous in + * that case. + */ + if (blk_no + bblks <= log->l_logBBsize || + blk_no >= log->l_logBBsize) { + rblk_no = xlog_wrap_logbno(log, blk_no); + error = xlog_bread(log, rblk_no, bblks, dbp, + &offset); + if (error) + goto bread_err2; + } else { + /* This log record is split across the + * physical end of log */ + offset = dbp; + split_bblks = 0; + if (blk_no != log->l_logBBsize) { + /* some data is before the physical + * end of log */ + ASSERT(!wrapped_hblks); + ASSERT(blk_no <= INT_MAX); + split_bblks = + log->l_logBBsize - (int)blk_no; + ASSERT(split_bblks > 0); + error = xlog_bread(log, blk_no, + split_bblks, dbp, + &offset); + if (error) + goto bread_err2; + } + + /* + * Note: this black magic still works with + * large sector sizes (non-512) only because: + * - we increased the buffer size originally + * by 1 sector giving us enough extra space + * for the second read; + * - the log start is guaranteed to be sector + * aligned; + * - we read the log end (LR header start) + * _first_, then the log start (LR header end) + * - order is important. + */ + error = xlog_bread_noalign(log, 0, + bblks - split_bblks, + offset + BBTOB(split_bblks)); + if (error) + goto bread_err2; + } + + error = xlog_recover_process(log, rhash, rhead, offset, + pass, &buffer_list); + if (error) + goto bread_err2; + + blk_no += bblks; + rhead_blk = blk_no; + } + + ASSERT(blk_no >= log->l_logBBsize); + blk_no -= log->l_logBBsize; + rhead_blk = blk_no; + } + + /* read first part of physical log */ + while (blk_no < head_blk) { + error = xlog_bread(log, blk_no, hblks, hbp, &offset); + if (error) + goto bread_err2; + + rhead = (struct xlog_rec_header *)offset; + error = xlog_valid_rec_header(log, rhead, blk_no, h_size); + if (error) + goto bread_err2; + + /* blocks in data section */ + bblks = (int)BTOBB(be32_to_cpu(rhead->h_len)); + error = xlog_bread(log, blk_no+hblks, bblks, dbp, + &offset); + if (error) + goto bread_err2; + + error = xlog_recover_process(log, rhash, rhead, offset, pass, + &buffer_list); + if (error) + goto bread_err2; + + blk_no += bblks + hblks; + rhead_blk = blk_no; + } + + bread_err2: + kvfree(dbp); + bread_err1: + kvfree(hbp); + + /* + * Submit buffers that have been dirtied by the last record recovered. + */ + if (!list_empty(&buffer_list)) { + if (error) { + /* + * If there has been an item recovery error then we + * cannot allow partial checkpoint writeback to + * occur. We might have multiple checkpoints with the + * same start LSN in this buffer list, and partial + * writeback of a checkpoint in this situation can + * prevent future recovery of all the changes in the + * checkpoints at this start LSN. + * + * Note: Shutting down the filesystem will result in the + * delwri submission marking all the buffers stale and + * completing them without doing any IO. + */ + xlog_force_shutdown(log, SHUTDOWN_LOG_IO_ERROR); + } + error2 = xfs_buf_delwri_submit(&buffer_list); + } + + if (error && first_bad) + *first_bad = rhead_blk; + + /* + * Transactions are freed at commit time but transactions without commit + * records on disk are never committed. Free any that may be left in the + * hash table. + */ + for (i = 0; i < XLOG_RHASH_SIZE; i++) { + struct hlist_node *tmp; + struct xlog_recover *trans; + + hlist_for_each_entry_safe(trans, tmp, &rhash[i], r_list) + xlog_recover_free_trans(trans); + } + + return error ? error : error2; +} + +/* + * Do the recovery of the log. We actually do this in two phases. + * The two passes are necessary in order to implement the function + * of cancelling a record written into the log. The first pass + * determines those things which have been cancelled, and the + * second pass replays log items normally except for those which + * have been cancelled. The handling of the replay and cancellations + * takes place in the log item type specific routines. + * + * The table of items which have cancel records in the log is allocated + * and freed at this level, since only here do we know when all of + * the log recovery has been completed. + */ +STATIC int +xlog_do_log_recovery( + struct xlog *log, + xfs_daddr_t head_blk, + xfs_daddr_t tail_blk) +{ + int error; + + ASSERT(head_blk != tail_blk); + + /* + * First do a pass to find all of the cancelled buf log items. + * Store them in the buf_cancel_table for use in the second pass. + */ + error = xlog_alloc_buf_cancel_table(log); + if (error) + return error; + + error = xlog_do_recovery_pass(log, head_blk, tail_blk, + XLOG_RECOVER_PASS1, NULL); + if (error != 0) + goto out_cancel; + + /* + * Then do a second pass to actually recover the items in the log. + * When it is complete free the table of buf cancel items. + */ + error = xlog_do_recovery_pass(log, head_blk, tail_blk, + XLOG_RECOVER_PASS2, NULL); + if (!error) + xlog_check_buf_cancel_table(log); +out_cancel: + xlog_free_buf_cancel_table(log); + return error; +} + +/* + * Do the actual recovery + */ +STATIC int +xlog_do_recover( + struct xlog *log, + xfs_daddr_t head_blk, + xfs_daddr_t tail_blk) +{ + struct xfs_mount *mp = log->l_mp; + struct xfs_buf *bp = mp->m_sb_bp; + struct xfs_sb *sbp = &mp->m_sb; + int error; + + trace_xfs_log_recover(log, head_blk, tail_blk); + + /* + * First replay the images in the log. + */ + error = xlog_do_log_recovery(log, head_blk, tail_blk); + if (error) + return error; + + if (xlog_is_shutdown(log)) + return -EIO; + + /* + * We now update the tail_lsn since much of the recovery has completed + * and there may be space available to use. If there were no extent or + * iunlinks, we can free up the entire log. This was set in + * xlog_find_tail to be the lsn of the last known good LR on disk. If + * there are extent frees or iunlinks they will have some entries in the + * AIL; so we look at the AIL to determine how to set the tail_lsn. + */ + xfs_ail_assign_tail_lsn(log->l_ailp); + + /* + * Now that we've finished replaying all buffer and inode updates, + * re-read the superblock and reverify it. + */ + xfs_buf_lock(bp); + xfs_buf_hold(bp); + error = _xfs_buf_read(bp); + if (error) { + if (!xlog_is_shutdown(log)) { + xfs_buf_ioerror_alert(bp, __this_address); + ASSERT(0); + } + xfs_buf_relse(bp); + return error; + } + + /* Convert superblock from on-disk format */ + xfs_sb_from_disk(sbp, bp->b_addr); + xfs_buf_relse(bp); + + /* re-initialise in-core superblock and geometry structures */ + mp->m_features |= xfs_sb_version_to_features(sbp); + xfs_reinit_percpu_counters(mp); + + /* Normal transactions can now occur */ + clear_bit(XLOG_ACTIVE_RECOVERY, &log->l_opstate); + return 0; +} + +/* + * Perform recovery and re-initialize some log variables in xlog_find_tail. + * + * Return error or zero. + */ +int +xlog_recover( + struct xlog *log) +{ + xfs_daddr_t head_blk, tail_blk; + int error; + + /* find the tail of the log */ + error = xlog_find_tail(log, &head_blk, &tail_blk); + if (error) + return error; + + /* + * The superblock was read before the log was available and thus the LSN + * could not be verified. Check the superblock LSN against the current + * LSN now that it's known. + */ + if (xfs_has_crc(log->l_mp) && + !xfs_log_check_lsn(log->l_mp, log->l_mp->m_sb.sb_lsn)) + return -EINVAL; + + if (tail_blk != head_blk) { + /* There used to be a comment here: + * + * disallow recovery on read-only mounts. note -- mount + * checks for ENOSPC and turns it into an intelligent + * error message. + * ...but this is no longer true. Now, unless you specify + * NORECOVERY (in which case this function would never be + * called), we just go ahead and recover. We do this all + * under the vfs layer, so we can get away with it unless + * the device itself is read-only, in which case we fail. + */ + if ((error = xfs_dev_is_read_only(log->l_mp, "recovery"))) { + return error; + } + + /* + * Version 5 superblock log feature mask validation. We know the + * log is dirty so check if there are any unknown log features + * in what we need to recover. If there are unknown features + * (e.g. unsupported transactions, then simply reject the + * attempt at recovery before touching anything. + */ + if (xfs_sb_is_v5(&log->l_mp->m_sb) && + xfs_sb_has_incompat_log_feature(&log->l_mp->m_sb, + XFS_SB_FEAT_INCOMPAT_LOG_UNKNOWN)) { + xfs_warn(log->l_mp, +"Superblock has unknown incompatible log features (0x%x) enabled.", + (log->l_mp->m_sb.sb_features_log_incompat & + XFS_SB_FEAT_INCOMPAT_LOG_UNKNOWN)); + xfs_warn(log->l_mp, +"The log can not be fully and/or safely recovered by this kernel."); + xfs_warn(log->l_mp, +"Please recover the log on a kernel that supports the unknown features."); + return -EINVAL; + } + + /* + * Delay log recovery if the debug hook is set. This is debug + * instrumentation to coordinate simulation of I/O failures with + * log recovery. + */ + if (xfs_globals.log_recovery_delay) { + xfs_notice(log->l_mp, + "Delaying log recovery for %d seconds.", + xfs_globals.log_recovery_delay); + msleep(xfs_globals.log_recovery_delay * 1000); + } + + xfs_notice(log->l_mp, "Starting recovery (logdev: %s)", + log->l_mp->m_logname ? log->l_mp->m_logname + : "internal"); + + error = xlog_do_recover(log, head_blk, tail_blk); + set_bit(XLOG_RECOVERY_NEEDED, &log->l_opstate); + } + return error; +} + +/* + * In the first part of recovery we replay inodes and buffers and build up the + * list of intents which need to be processed. Here we process the intents and + * clean up the on disk unlinked inode lists. This is separated from the first + * part of recovery so that the root and real-time bitmap inodes can be read in + * from disk in between the two stages. This is necessary so that we can free + * space in the real-time portion of the file system. + * + * We run this whole process under GFP_NOFS allocation context. We do a + * combination of non-transactional and transactional work, yet we really don't + * want to recurse into the filesystem from direct reclaim during any of this + * processing. This allows all the recovery code run here not to care about the + * memory allocation context it is running in. + */ +int +xlog_recover_finish( + struct xlog *log) +{ + unsigned int nofs_flags = memalloc_nofs_save(); + int error; + + error = xlog_recover_process_intents(log); + if (error) { + /* + * Cancel all the unprocessed intent items now so that we don't + * leave them pinned in the AIL. This can cause the AIL to + * livelock on the pinned item if anyone tries to push the AIL + * (inode reclaim does this) before we get around to + * xfs_log_mount_cancel. + */ + xlog_recover_cancel_intents(log); + xfs_alert(log->l_mp, "Failed to recover intents"); + xlog_force_shutdown(log, SHUTDOWN_LOG_IO_ERROR); + goto out_error; + } + + /* + * Sync the log to get all the intents out of the AIL. This isn't + * absolutely necessary, but it helps in case the unlink transactions + * would have problems pushing the intents out of the way. + */ + xfs_log_force(log->l_mp, XFS_LOG_SYNC); + + xlog_recover_process_iunlinks(log); + + /* + * Recover any CoW staging blocks that are still referenced by the + * ondisk refcount metadata. During mount there cannot be any live + * staging extents as we have not permitted any user modifications. + * Therefore, it is safe to free them all right now, even on a + * read-only mount. + */ + error = xfs_reflink_recover_cow(log->l_mp); + if (error) { + xfs_alert(log->l_mp, + "Failed to recover leftover CoW staging extents, err %d.", + error); + /* + * If we get an error here, make sure the log is shut down + * but return zero so that any log items committed since the + * end of intents processing can be pushed through the CIL + * and AIL. + */ + xlog_force_shutdown(log, SHUTDOWN_LOG_IO_ERROR); + error = 0; + goto out_error; + } + +out_error: + memalloc_nofs_restore(nofs_flags); + return error; +} + +void +xlog_recover_cancel( + struct xlog *log) +{ + if (xlog_recovery_needed(log)) + xlog_recover_cancel_intents(log); +} diff --git a/fs/xfs/xfs_message.c b/fs/xfs/xfs_message.c new file mode 100644 index 000000000..0243e509a --- /dev/null +++ b/fs/xfs/xfs_message.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2011 Red Hat, Inc. All Rights Reserved. + */ + +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_error.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" + +/* + * XFS logging functions + */ +static void +__xfs_printk( + const char *level, + const struct xfs_mount *mp, + struct va_format *vaf) +{ + if (mp && mp->m_super) { + printk("%sXFS (%s): %pV\n", level, mp->m_super->s_id, vaf); + return; + } + printk("%sXFS: %pV\n", level, vaf); +} + +void +xfs_printk_level( + const char *kern_level, + const struct xfs_mount *mp, + const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + int level; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + + __xfs_printk(kern_level, mp, &vaf); + + va_end(args); + + if (!kstrtoint(kern_level, 0, &level) && + level <= LOGLEVEL_ERR && + xfs_error_level >= XFS_ERRLEVEL_HIGH) + xfs_stack_trace(); +} + +void +_xfs_alert_tag( + const struct xfs_mount *mp, + uint32_t panic_tag, + const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + int do_panic = 0; + + if (xfs_panic_mask && (xfs_panic_mask & panic_tag)) { + xfs_alert(mp, "Transforming an alert into a BUG."); + do_panic = 1; + } + + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + __xfs_printk(KERN_ALERT, mp, &vaf); + va_end(args); + + BUG_ON(do_panic); +} + +void +asswarn( + struct xfs_mount *mp, + char *expr, + char *file, + int line) +{ + xfs_warn(mp, "Assertion failed: %s, file: %s, line: %d", + expr, file, line); + WARN_ON(1); +} + +void +assfail( + struct xfs_mount *mp, + char *expr, + char *file, + int line) +{ + xfs_emerg(mp, "Assertion failed: %s, file: %s, line: %d", + expr, file, line); + if (xfs_globals.bug_on_assert) + BUG(); + else + WARN_ON(1); +} + +void +xfs_hex_dump(const void *p, int length) +{ + print_hex_dump(KERN_ALERT, "", DUMP_PREFIX_OFFSET, 16, 1, p, length, 1); +} + +void +xfs_buf_alert_ratelimited( + struct xfs_buf *bp, + const char *rlmsg, + const char *fmt, + ...) +{ + struct xfs_mount *mp = bp->b_mount; + struct va_format vaf; + va_list args; + + /* use the more aggressive per-target rate limit for buffers */ + if (!___ratelimit(&bp->b_target->bt_ioerror_rl, rlmsg)) + return; + + va_start(args, fmt); + vaf.fmt = fmt; + vaf.va = &args; + __xfs_printk(KERN_ALERT, mp, &vaf); + va_end(args); +} + +void +xfs_warn_experimental( + struct xfs_mount *mp, + enum xfs_experimental_feat feat) +{ + static const struct { + const char *name; + long opstate; + } features[] = { + [XFS_EXPERIMENTAL_SHRINK] = { + .opstate = XFS_OPSTATE_WARNED_SHRINK, + .name = "online shrink", + }, + [XFS_EXPERIMENTAL_LARP] = { + .opstate = XFS_OPSTATE_WARNED_LARP, + .name = "logged extended attributes", + }, + }; + ASSERT(feat >= 0 && feat < XFS_EXPERIMENTAL_MAX); + BUILD_BUG_ON(ARRAY_SIZE(features) != XFS_EXPERIMENTAL_MAX); + + if (xfs_should_warn(mp, features[feat].opstate)) + xfs_warn(mp, + "EXPERIMENTAL %s feature enabled. Use at your own risk!", + features[feat].name); +} diff --git a/fs/xfs/xfs_message.h b/fs/xfs/xfs_message.h new file mode 100644 index 000000000..811b885f4 --- /dev/null +++ b/fs/xfs/xfs_message.h @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __XFS_MESSAGE_H +#define __XFS_MESSAGE_H 1 + +#include + +struct xfs_mount; + +extern __printf(3, 4) +void xfs_printk_level(const char *kern_level, const struct xfs_mount *mp, + const char *fmt, ...); + +#define xfs_printk_index_wrap(kern_level, mp, fmt, ...) \ +({ \ + printk_index_subsys_emit("%sXFS%s: ", kern_level, fmt); \ + xfs_printk_level(kern_level, mp, fmt, ##__VA_ARGS__); \ +}) +#define xfs_emerg(mp, fmt, ...) \ + xfs_printk_index_wrap(KERN_EMERG, mp, fmt, ##__VA_ARGS__) +#define xfs_alert(mp, fmt, ...) \ + xfs_printk_index_wrap(KERN_ALERT, mp, fmt, ##__VA_ARGS__) +#define xfs_crit(mp, fmt, ...) \ + xfs_printk_index_wrap(KERN_CRIT, mp, fmt, ##__VA_ARGS__) +#define xfs_err(mp, fmt, ...) \ + xfs_printk_index_wrap(KERN_ERR, mp, fmt, ##__VA_ARGS__) +#define xfs_warn(mp, fmt, ...) \ + xfs_printk_index_wrap(KERN_WARNING, mp, fmt, ##__VA_ARGS__) +#define xfs_notice(mp, fmt, ...) \ + xfs_printk_index_wrap(KERN_NOTICE, mp, fmt, ##__VA_ARGS__) +#define xfs_info(mp, fmt, ...) \ + xfs_printk_index_wrap(KERN_INFO, mp, fmt, ##__VA_ARGS__) +#ifdef DEBUG +#define xfs_debug(mp, fmt, ...) \ + xfs_printk_index_wrap(KERN_DEBUG, mp, fmt, ##__VA_ARGS__) +#else +#define xfs_debug(mp, fmt, ...) do {} while (0) +#endif + +#define xfs_alert_tag(mp, tag, fmt, ...) \ +({ \ + printk_index_subsys_emit("%sXFS%s: ", KERN_ALERT, fmt); \ + _xfs_alert_tag(mp, tag, fmt, ##__VA_ARGS__); \ +}) + +extern __printf(3, 4) +void _xfs_alert_tag(const struct xfs_mount *mp, uint32_t tag, + const char *fmt, ...); + +#define xfs_printk_ratelimited(func, dev, fmt, ...) \ +do { \ + static DEFINE_RATELIMIT_STATE(_rs, \ + DEFAULT_RATELIMIT_INTERVAL, \ + DEFAULT_RATELIMIT_BURST); \ + if (__ratelimit(&_rs)) \ + func(dev, fmt, ##__VA_ARGS__); \ +} while (0) + +#define xfs_printk_once(func, dev, fmt, ...) \ + DO_ONCE_LITE(func, dev, fmt, ##__VA_ARGS__) + +#define xfs_emerg_ratelimited(dev, fmt, ...) \ + xfs_printk_ratelimited(xfs_emerg, dev, fmt, ##__VA_ARGS__) +#define xfs_alert_ratelimited(dev, fmt, ...) \ + xfs_printk_ratelimited(xfs_alert, dev, fmt, ##__VA_ARGS__) +#define xfs_crit_ratelimited(dev, fmt, ...) \ + xfs_printk_ratelimited(xfs_crit, dev, fmt, ##__VA_ARGS__) +#define xfs_err_ratelimited(dev, fmt, ...) \ + xfs_printk_ratelimited(xfs_err, dev, fmt, ##__VA_ARGS__) +#define xfs_warn_ratelimited(dev, fmt, ...) \ + xfs_printk_ratelimited(xfs_warn, dev, fmt, ##__VA_ARGS__) +#define xfs_notice_ratelimited(dev, fmt, ...) \ + xfs_printk_ratelimited(xfs_notice, dev, fmt, ##__VA_ARGS__) +#define xfs_info_ratelimited(dev, fmt, ...) \ + xfs_printk_ratelimited(xfs_info, dev, fmt, ##__VA_ARGS__) +#define xfs_debug_ratelimited(dev, fmt, ...) \ + xfs_printk_ratelimited(xfs_debug, dev, fmt, ##__VA_ARGS__) + +#define xfs_warn_once(dev, fmt, ...) \ + xfs_printk_once(xfs_warn, dev, fmt, ##__VA_ARGS__) +#define xfs_notice_once(dev, fmt, ...) \ + xfs_printk_once(xfs_notice, dev, fmt, ##__VA_ARGS__) +#define xfs_info_once(dev, fmt, ...) \ + xfs_printk_once(xfs_info, dev, fmt, ##__VA_ARGS__) + +void assfail(struct xfs_mount *mp, char *expr, char *f, int l); +void asswarn(struct xfs_mount *mp, char *expr, char *f, int l); + +extern void xfs_hex_dump(const void *p, int length); + +void xfs_buf_alert_ratelimited(struct xfs_buf *bp, const char *rlmsg, + const char *fmt, ...); + +enum xfs_experimental_feat { + XFS_EXPERIMENTAL_SHRINK, + XFS_EXPERIMENTAL_LARP, + + XFS_EXPERIMENTAL_MAX, +}; +void xfs_warn_experimental(struct xfs_mount *mp, enum xfs_experimental_feat f); + +#endif /* __XFS_MESSAGE_H */ diff --git a/fs/xfs/xfs_mount.c b/fs/xfs/xfs_mount.c new file mode 100644 index 000000000..be90c7b03 --- /dev/null +++ b/fs/xfs/xfs_mount.c @@ -0,0 +1,1699 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_sb.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_dir2.h" +#include "xfs_ialloc.h" +#include "xfs_alloc.h" +#include "xfs_rtalloc.h" +#include "xfs_bmap.h" +#include "xfs_trans.h" +#include "xfs_trans_priv.h" +#include "xfs_log.h" +#include "xfs_log_priv.h" +#include "xfs_error.h" +#include "xfs_quota.h" +#include "xfs_fsops.h" +#include "xfs_icache.h" +#include "xfs_sysfs.h" +#include "xfs_rmap_btree.h" +#include "xfs_refcount_btree.h" +#include "xfs_reflink.h" +#include "xfs_extent_busy.h" +#include "xfs_health.h" +#include "xfs_trace.h" +#include "xfs_ag.h" +#include "xfs_rtbitmap.h" +#include "xfs_metafile.h" +#include "xfs_rtgroup.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_rtrefcount_btree.h" +#include "scrub/stats.h" +#include "xfs_zone_alloc.h" +#include "xfs_healthmon.h" + +static DEFINE_MUTEX(xfs_uuid_table_mutex); +static DEFINE_XARRAY_ALLOC(xfs_uuid_table); + +static uuid_t * +xfs_uuid_search( + uuid_t *new_uuid) +{ + unsigned long index = 0; + uuid_t *uuid; + + xa_for_each(&xfs_uuid_table, index, uuid) { + if (uuid_equal(uuid, new_uuid)) + return uuid; + } + return NULL; +} + +static void +xfs_uuid_delete( + uuid_t *uuid, + unsigned int index) +{ + ASSERT(uuid_equal(xa_load(&xfs_uuid_table, index), uuid)); + xa_erase(&xfs_uuid_table, index); +} + +void +xfs_uuid_table_free(void) +{ + ASSERT(xa_empty(&xfs_uuid_table)); + xa_destroy(&xfs_uuid_table); +} + +/* + * See if the UUID is unique among mounted XFS filesystems. + * Mount fails if UUID is nil or a FS with the same UUID is already mounted. + */ +STATIC int +xfs_uuid_mount( + struct xfs_mount *mp) +{ + uuid_t *uuid = &mp->m_sb.sb_uuid; + int ret; + + /* Publish UUID in struct super_block */ + super_set_uuid(mp->m_super, uuid->b, sizeof(*uuid)); + + if (xfs_has_nouuid(mp)) + return 0; + + if (uuid_is_null(uuid)) { + xfs_warn(mp, "Filesystem has null UUID - can't mount"); + return -EINVAL; + } + + mutex_lock(&xfs_uuid_table_mutex); + if (unlikely(xfs_uuid_search(uuid))) { + xfs_warn(mp, "Filesystem has duplicate UUID %pU - can't mount", + uuid); + mutex_unlock(&xfs_uuid_table_mutex); + return -EINVAL; + } + + ret = xa_alloc(&xfs_uuid_table, &mp->m_uuid_table_index, uuid, + xa_limit_32b, GFP_KERNEL); + mutex_unlock(&xfs_uuid_table_mutex); + return ret; +} + +STATIC void +xfs_uuid_unmount( + struct xfs_mount *mp) +{ + uuid_t *uuid = &mp->m_sb.sb_uuid; + + if (xfs_has_nouuid(mp)) + return; + + mutex_lock(&xfs_uuid_table_mutex); + xfs_uuid_delete(uuid, mp->m_uuid_table_index); + mutex_unlock(&xfs_uuid_table_mutex); +} + +/* + * Check size of device based on the (data/realtime) block count. + * Note: this check is used by the growfs code as well as mount. + */ +int +xfs_sb_validate_fsb_count( + xfs_sb_t *sbp, + uint64_t nblocks) +{ + uint64_t max_bytes; + + ASSERT(sbp->sb_blocklog >= BBSHIFT); + + if (check_shl_overflow(nblocks, sbp->sb_blocklog, &max_bytes)) + return -EFBIG; + + /* Limited by ULONG_MAX of page cache index */ + if (max_bytes >> PAGE_SHIFT > ULONG_MAX) + return -EFBIG; + return 0; +} + +/* + * xfs_readsb + * + * Does the initial read of the superblock. + */ +int +xfs_readsb( + struct xfs_mount *mp, + int flags) +{ + unsigned int sector_size; + struct xfs_buf *bp; + struct xfs_sb *sbp = &mp->m_sb; + int error; + int loud = !(flags & XFS_MFSI_QUIET); + const struct xfs_buf_ops *buf_ops; + + ASSERT(mp->m_sb_bp == NULL); + ASSERT(mp->m_ddev_targp != NULL); + + /* + * In the first pass, use the device sector size to just read enough + * of the superblock to extract the XFS sector size. + * + * The device sector size must be smaller than or equal to the XFS + * sector size and thus we can always read the superblock. Once we know + * the XFS sector size, re-read it and run the buffer verifier. + */ + sector_size = mp->m_ddev_targp->bt_logical_sectorsize; + buf_ops = NULL; + +reread: + error = xfs_buf_read_uncached(mp->m_ddev_targp, XFS_SB_DADDR, + BTOBB(sector_size), &bp, buf_ops); + if (error) { + if (loud) + xfs_warn(mp, "SB validate failed with error %d.", error); + /* bad CRC means corrupted metadata */ + if (error == -EFSBADCRC) + error = -EFSCORRUPTED; + return error; + } + + /* + * Initialize the mount structure from the superblock. + */ + xfs_sb_from_disk(sbp, bp->b_addr); + + /* + * If we haven't validated the superblock, do so now before we try + * to check the sector size and reread the superblock appropriately. + */ + if (sbp->sb_magicnum != XFS_SB_MAGIC) { + if (loud) + xfs_warn(mp, "Invalid superblock magic number"); + error = -EINVAL; + goto release_buf; + } + + /* + * We must be able to do sector-sized and sector-aligned IO. + */ + if (sector_size > sbp->sb_sectsize) { + if (loud) + xfs_warn(mp, "device supports %u byte sectors (not %u)", + sector_size, sbp->sb_sectsize); + error = -ENOSYS; + goto release_buf; + } + + if (buf_ops == NULL) { + /* + * Re-read the superblock so the buffer is correctly sized, + * and properly verified. + */ + xfs_buf_relse(bp); + sector_size = sbp->sb_sectsize; + buf_ops = loud ? &xfs_sb_buf_ops : &xfs_sb_quiet_buf_ops; + goto reread; + } + + mp->m_features |= xfs_sb_version_to_features(sbp); + xfs_reinit_percpu_counters(mp); + + /* + * If logged xattrs are enabled after log recovery finishes, then set + * the opstate so that log recovery will work properly. + */ + if (xfs_sb_version_haslogxattrs(&mp->m_sb)) + xfs_set_using_logged_xattrs(mp); + + /* no need to be quiet anymore, so reset the buf ops */ + bp->b_ops = &xfs_sb_buf_ops; + + /* + * Keep a pointer of the sb buffer around instead of caching it in the + * buffer cache because we access it frequently. + */ + mp->m_sb_bp = bp; + xfs_buf_unlock(bp); + return 0; + +release_buf: + xfs_buf_relse(bp); + return error; +} + +/* + * If the sunit/swidth change would move the precomputed root inode value, we + * must reject the ondisk change because repair will stumble over that. + * However, we allow the mount to proceed because we never rejected this + * combination before. Returns true to update the sb, false otherwise. + */ +static inline int +xfs_check_new_dalign( + struct xfs_mount *mp, + int new_dalign, + bool *update_sb) +{ + struct xfs_sb *sbp = &mp->m_sb; + xfs_ino_t calc_ino; + + calc_ino = xfs_ialloc_calc_rootino(mp, new_dalign); + trace_xfs_check_new_dalign(mp, new_dalign, calc_ino); + + if (sbp->sb_rootino == calc_ino) { + *update_sb = true; + return 0; + } + + xfs_warn(mp, +"Cannot change stripe alignment; would require moving root inode."); + + /* + * XXX: Next time we add a new incompat feature, this should start + * returning -EINVAL to fail the mount. Until then, spit out a warning + * that we're ignoring the administrator's instructions. + */ + xfs_warn(mp, "Skipping superblock stripe alignment update."); + *update_sb = false; + return 0; +} + +/* + * If we were provided with new sunit/swidth values as mount options, make sure + * that they pass basic alignment and superblock feature checks, and convert + * them into the same units (FSB) that everything else expects. This step + * /must/ be done before computing the inode geometry. + */ +STATIC int +xfs_validate_new_dalign( + struct xfs_mount *mp) +{ + if (mp->m_dalign == 0) + return 0; + + /* + * If stripe unit and stripe width are not multiples + * of the fs blocksize turn off alignment. + */ + if ((BBTOB(mp->m_dalign) & mp->m_blockmask) || + (BBTOB(mp->m_swidth) & mp->m_blockmask)) { + xfs_warn(mp, + "alignment check failed: sunit/swidth vs. blocksize(%d)", + mp->m_sb.sb_blocksize); + return -EINVAL; + } + + /* + * Convert the stripe unit and width to FSBs. + */ + mp->m_dalign = XFS_BB_TO_FSBT(mp, mp->m_dalign); + if (mp->m_dalign && (mp->m_sb.sb_agblocks % mp->m_dalign)) { + xfs_warn(mp, + "alignment check failed: sunit/swidth vs. agsize(%d)", + mp->m_sb.sb_agblocks); + return -EINVAL; + } + + if (!mp->m_dalign) { + xfs_warn(mp, + "alignment check failed: sunit(%d) less than bsize(%d)", + mp->m_dalign, mp->m_sb.sb_blocksize); + return -EINVAL; + } + + mp->m_swidth = XFS_BB_TO_FSBT(mp, mp->m_swidth); + + if (!xfs_has_dalign(mp)) { + xfs_warn(mp, +"cannot change alignment: superblock does not support data alignment"); + return -EINVAL; + } + + return 0; +} + +/* Update alignment values based on mount options and sb values. */ +STATIC int +xfs_update_alignment( + struct xfs_mount *mp) +{ + struct xfs_sb *sbp = &mp->m_sb; + + if (mp->m_dalign) { + bool update_sb; + int error; + + if (sbp->sb_unit == mp->m_dalign && + sbp->sb_width == mp->m_swidth) + return 0; + + error = xfs_check_new_dalign(mp, mp->m_dalign, &update_sb); + if (error || !update_sb) + return error; + + sbp->sb_unit = mp->m_dalign; + sbp->sb_width = mp->m_swidth; + mp->m_update_sb = true; + } else if (!xfs_has_noalign(mp) && xfs_has_dalign(mp)) { + mp->m_dalign = sbp->sb_unit; + mp->m_swidth = sbp->sb_width; + } + + return 0; +} + +/* + * precalculate the low space thresholds for dynamic speculative preallocation. + */ +void +xfs_set_low_space_thresholds( + struct xfs_mount *mp) +{ + uint64_t dblocks = mp->m_sb.sb_dblocks; + uint64_t rtexts = mp->m_sb.sb_rextents; + int i; + + do_div(dblocks, 100); + do_div(rtexts, 100); + + for (i = 0; i < XFS_LOWSP_MAX; i++) { + mp->m_low_space[i] = dblocks * (i + 1); + mp->m_low_rtexts[i] = rtexts * (i + 1); + } +} + +/* + * Check that the data (and log if separate) is an ok size. + */ +STATIC int +xfs_check_sizes( + struct xfs_mount *mp) +{ + struct xfs_buf *bp; + xfs_daddr_t d; + int error; + + d = (xfs_daddr_t)XFS_FSB_TO_BB(mp, mp->m_sb.sb_dblocks); + if (XFS_BB_TO_FSB(mp, d) != mp->m_sb.sb_dblocks) { + xfs_warn(mp, "filesystem size mismatch detected"); + return -EFBIG; + } + error = xfs_buf_read_uncached(mp->m_ddev_targp, + d - XFS_FSS_TO_BB(mp, 1), + XFS_FSS_TO_BB(mp, 1), &bp, NULL); + if (error) { + xfs_warn(mp, "last sector read failed"); + return error; + } + xfs_buf_relse(bp); + + if (mp->m_logdev_targp == mp->m_ddev_targp) + return 0; + + d = (xfs_daddr_t)XFS_FSB_TO_BB(mp, mp->m_sb.sb_logblocks); + if (XFS_BB_TO_FSB(mp, d) != mp->m_sb.sb_logblocks) { + xfs_warn(mp, "log size mismatch detected"); + return -EFBIG; + } + error = xfs_buf_read_uncached(mp->m_logdev_targp, + d - XFS_FSB_TO_BB(mp, 1), + XFS_FSB_TO_BB(mp, 1), &bp, NULL); + if (error) { + xfs_warn(mp, "log device read failed"); + return error; + } + xfs_buf_relse(bp); + return 0; +} + +/* + * Clear the quotaflags in memory and in the superblock. + */ +int +xfs_mount_reset_sbqflags( + struct xfs_mount *mp) +{ + mp->m_qflags = 0; + + /* It is OK to look at sb_qflags in the mount path without m_sb_lock. */ + if (mp->m_sb.sb_qflags == 0) + return 0; + spin_lock(&mp->m_sb_lock); + mp->m_sb.sb_qflags = 0; + spin_unlock(&mp->m_sb_lock); + + if (!xfs_fs_writable(mp, SB_FREEZE_WRITE)) + return 0; + + return xfs_sync_sb(mp, false); +} + +static const char *const xfs_free_pool_name[] = { + [XC_FREE_BLOCKS] = "free blocks", + [XC_FREE_RTEXTENTS] = "free rt extents", + [XC_FREE_RTAVAILABLE] = "available rt extents", +}; + +uint64_t +xfs_default_resblks( + struct xfs_mount *mp, + enum xfs_free_counter ctr) +{ + switch (ctr) { + case XC_FREE_BLOCKS: + /* + * Default to 5% or 8192 FSBs of space reserved, whichever is + * smaller. + * + * This is intended to cover concurrent allocation transactions + * when we initially hit ENOSPC. These each require a 4 block + * reservation. Hence by default we cover roughly 2000 + * concurrent allocation reservations. + */ + return min(div_u64(mp->m_sb.sb_dblocks, 20), 8192ULL); + case XC_FREE_RTEXTENTS: + case XC_FREE_RTAVAILABLE: + if (IS_ENABLED(CONFIG_XFS_RT) && xfs_has_zoned(mp)) + return xfs_zoned_default_resblks(mp, ctr); + return 0; + default: + ASSERT(0); + return 0; + } +} + +/* Ensure the summary counts are correct. */ +STATIC int +xfs_check_summary_counts( + struct xfs_mount *mp) +{ + int error = 0; + + /* + * The AG0 superblock verifier rejects in-progress filesystems, + * so we should never see the flag set this far into mounting. + */ + if (mp->m_sb.sb_inprogress) { + xfs_err(mp, "sb_inprogress set after log recovery??"); + WARN_ON(1); + return -EFSCORRUPTED; + } + + /* + * Now the log is mounted, we know if it was an unclean shutdown or + * not. If it was, with the first phase of recovery has completed, we + * have consistent AG blocks on disk. We have not recovered EFIs yet, + * but they are recovered transactionally in the second recovery phase + * later. + * + * If the log was clean when we mounted, we can check the summary + * counters. If any of them are obviously incorrect, we can recompute + * them from the AGF headers in the next step. + */ + if (xfs_is_clean(mp) && + (mp->m_sb.sb_fdblocks > mp->m_sb.sb_dblocks || + !xfs_verify_icount(mp, mp->m_sb.sb_icount) || + mp->m_sb.sb_ifree > mp->m_sb.sb_icount)) + xfs_fs_mark_sick(mp, XFS_SICK_FS_COUNTERS); + + /* + * We can safely re-initialise incore superblock counters from the + * per-ag data. These may not be correct if the filesystem was not + * cleanly unmounted, so we waited for recovery to finish before doing + * this. + * + * If the filesystem was cleanly unmounted or the previous check did + * not flag anything weird, then we can trust the values in the + * superblock to be correct and we don't need to do anything here. + * Otherwise, recalculate the summary counters. + */ + if ((xfs_has_lazysbcount(mp) && !xfs_is_clean(mp)) || + xfs_fs_has_sickness(mp, XFS_SICK_FS_COUNTERS)) { + error = xfs_initialize_perag_data(mp, mp->m_sb.sb_agcount); + if (error) + return error; + } + + /* + * Older kernels misused sb_frextents to reflect both incore + * reservations made by running transactions and the actual count of + * free rt extents in the ondisk metadata. Transactions committed + * during runtime can therefore contain a superblock update that + * undercounts the number of free rt extents tracked in the rt bitmap. + * A clean unmount record will have the correct frextents value since + * there can be no other transactions running at that point. + * + * If we're mounting the rt volume after recovering the log, recompute + * frextents from the rtbitmap file to fix the inconsistency. + */ + if (xfs_has_realtime(mp) && !xfs_has_zoned(mp) && !xfs_is_clean(mp)) { + error = xfs_rtalloc_reinit_frextents(mp); + if (error) + return error; + } + + return 0; +} + +static void +xfs_unmount_check( + struct xfs_mount *mp) +{ + if (xfs_is_shutdown(mp)) + return; + + if (percpu_counter_sum(&mp->m_ifree) > + percpu_counter_sum(&mp->m_icount)) { + xfs_alert(mp, "ifree/icount mismatch at unmount"); + xfs_fs_mark_sick(mp, XFS_SICK_FS_COUNTERS); + } +} + +/* + * Flush and reclaim dirty inodes in preparation for unmount. Inodes and + * internal inode structures can be sitting in the CIL and AIL at this point, + * so we need to unpin them, write them back and/or reclaim them before unmount + * can proceed. In other words, callers are required to have inactivated all + * inodes. + * + * An inode cluster that has been freed can have its buffer still pinned in + * memory because the transaction is still sitting in a iclog. The stale inodes + * on that buffer will be pinned to the buffer until the transaction hits the + * disk and the callbacks run. Pushing the AIL will skip the stale inodes and + * may never see the pinned buffer, so nothing will push out the iclog and + * unpin the buffer. + * + * Hence we need to force the log to unpin everything first. However, log + * forces don't wait for the discards they issue to complete, so we have to + * explicitly wait for them to complete here as well. + * + * Then we can tell the world we are unmounting so that error handling knows + * that the filesystem is going away and we should error out anything that we + * have been retrying in the background. This will prevent never-ending + * retries in AIL pushing from hanging the unmount. + * + * Stop inodegc and background reclaim before pushing the AIL so that they + * are not running while the AIL is being flushed. Then push the AIL to + * clean all the remaining dirty objects and reclaim the remaining inodes. + */ +static void +xfs_unmount_flush_inodes( + struct xfs_mount *mp) +{ + xfs_log_force(mp, XFS_LOG_SYNC); + xfs_extent_busy_wait_all(mp); + flush_workqueue(xfs_discard_wq); + + xfs_set_unmounting(mp); + + xfs_inodegc_stop(mp); + cancel_delayed_work_sync(&mp->m_reclaim_work); + xfs_ail_push_all_sync(mp->m_ail); + xfs_reclaim_inodes(mp); + xfs_health_unmount(mp); + xfs_healthmon_unmount(mp); +} + +static void +xfs_mount_setup_inode_geom( + struct xfs_mount *mp) +{ + struct xfs_ino_geometry *igeo = M_IGEO(mp); + + igeo->attr_fork_offset = xfs_bmap_compute_attr_offset(mp); + ASSERT(igeo->attr_fork_offset < XFS_LITINO(mp)); + + xfs_ialloc_setup_geometry(mp); +} + +/* Mount the metadata directory tree root. */ +STATIC int +xfs_mount_setup_metadir( + struct xfs_mount *mp) +{ + int error; + + /* Load the metadata directory root inode into memory. */ + error = xfs_metafile_iget(mp, mp->m_sb.sb_metadirino, XFS_METAFILE_DIR, + &mp->m_metadirip); + if (error) + xfs_warn(mp, "Failed to load metadir root directory, error %d", + error); + return error; +} + +/* Compute maximum possible height for per-AG btree types for this fs. */ +static inline void +xfs_agbtree_compute_maxlevels( + struct xfs_mount *mp) +{ + unsigned int levels; + + levels = max(mp->m_alloc_maxlevels, M_IGEO(mp)->inobt_maxlevels); + levels = max(levels, mp->m_rmap_maxlevels); + mp->m_agbtree_maxlevels = max(levels, mp->m_refc_maxlevels); +} + +/* Maximum atomic write IO size that the kernel allows. */ +static inline xfs_extlen_t xfs_calc_atomic_write_max(struct xfs_mount *mp) +{ + return rounddown_pow_of_two(XFS_B_TO_FSB(mp, MAX_RW_COUNT)); +} + +/* + * If the underlying device advertises atomic write support, limit the size of + * atomic writes to the greatest power-of-two factor of the group size so + * that every atomic write unit aligns with the start of every group. This is + * required so that the allocations for an atomic write will always be + * aligned compatibly with the alignment requirements of the storage. + * + * If the device doesn't advertise atomic writes, then there are no alignment + * restrictions and the largest out-of-place write we can do ourselves is the + * number of blocks that user files can allocate from any group. + */ +static xfs_extlen_t +xfs_calc_group_awu_max( + struct xfs_mount *mp, + enum xfs_group_type type) +{ + struct xfs_groups *g = &mp->m_groups[type]; + struct xfs_buftarg *btp = xfs_group_type_buftarg(mp, type); + + if (g->blocks == 0) + return 0; + if (btp && btp->bt_awu_min > 0) + return max_pow_of_two_factor(g->blocks); + return rounddown_pow_of_two(g->blocks); +} + +/* Compute the maximum atomic write unit size for each section. */ +static inline void +xfs_calc_atomic_write_unit_max( + struct xfs_mount *mp, + enum xfs_group_type type) +{ + struct xfs_groups *g = &mp->m_groups[type]; + + const xfs_extlen_t max_write = xfs_calc_atomic_write_max(mp); + const xfs_extlen_t max_ioend = xfs_reflink_max_atomic_cow(mp); + const xfs_extlen_t max_gsize = xfs_calc_group_awu_max(mp, type); + + g->awu_max = min3(max_write, max_ioend, max_gsize); + trace_xfs_calc_atomic_write_unit_max(mp, type, max_write, max_ioend, + max_gsize, g->awu_max); +} + +/* + * Try to set the atomic write maximum to a new value that we got from + * userspace via mount option. + */ +int +xfs_set_max_atomic_write_opt( + struct xfs_mount *mp, + unsigned long long new_max_bytes) +{ + const xfs_filblks_t new_max_fsbs = XFS_B_TO_FSBT(mp, new_max_bytes); + const xfs_extlen_t max_write = xfs_calc_atomic_write_max(mp); + const xfs_extlen_t max_group = + max(mp->m_groups[XG_TYPE_AG].blocks, + mp->m_groups[XG_TYPE_RTG].blocks); + const xfs_extlen_t max_group_write = + max(xfs_calc_group_awu_max(mp, XG_TYPE_AG), + xfs_calc_group_awu_max(mp, XG_TYPE_RTG)); + int error; + + if (new_max_bytes == 0) + goto set_limit; + + ASSERT(max_write <= U32_MAX); + + /* generic_atomic_write_valid enforces power of two length */ + if (!is_power_of_2(new_max_bytes)) { + xfs_warn(mp, + "max atomic write size of %llu bytes is not a power of 2", + new_max_bytes); + return -EINVAL; + } + + if (new_max_bytes & mp->m_blockmask) { + xfs_warn(mp, + "max atomic write size of %llu bytes not aligned with fsblock", + new_max_bytes); + return -EINVAL; + } + + if (new_max_fsbs > max_write) { + xfs_warn(mp, + "max atomic write size of %lluk cannot be larger than max write size %lluk", + new_max_bytes >> 10, + XFS_FSB_TO_B(mp, max_write) >> 10); + return -EINVAL; + } + + if (new_max_fsbs > max_group) { + xfs_warn(mp, + "max atomic write size of %lluk cannot be larger than allocation group size %lluk", + new_max_bytes >> 10, + XFS_FSB_TO_B(mp, max_group) >> 10); + return -EINVAL; + } + + if (new_max_fsbs > max_group_write) { + xfs_warn(mp, + "max atomic write size of %lluk cannot be larger than max allocation group write size %lluk", + new_max_bytes >> 10, + XFS_FSB_TO_B(mp, max_group_write) >> 10); + return -EINVAL; + } + + if (xfs_has_reflink(mp)) + goto set_limit; + + if (new_max_fsbs == 1) { + if (mp->m_ddev_targp->bt_awu_max || + (mp->m_rtdev_targp && mp->m_rtdev_targp->bt_awu_max)) { + } else { + xfs_warn(mp, + "cannot support atomic writes of size %lluk with no reflink or HW support", + new_max_bytes >> 10); + return -EINVAL; + } + } else { + xfs_warn(mp, + "cannot support atomic writes of size %lluk with no reflink support", + new_max_bytes >> 10); + return -EINVAL; + } + +set_limit: + error = xfs_calc_atomic_write_reservation(mp, new_max_fsbs); + if (error) { + xfs_warn(mp, + "cannot support completing atomic writes of %lluk", + new_max_bytes >> 10); + return error; + } + + xfs_calc_atomic_write_unit_max(mp, XG_TYPE_AG); + xfs_calc_atomic_write_unit_max(mp, XG_TYPE_RTG); + mp->m_awu_max_bytes = new_max_bytes; + return 0; +} + +/* Compute maximum possible height for realtime btree types for this fs. */ +static inline void +xfs_rtbtree_compute_maxlevels( + struct xfs_mount *mp) +{ + mp->m_rtbtree_maxlevels = max(mp->m_rtrmap_maxlevels, + mp->m_rtrefc_maxlevels); +} + +/* + * This function does the following on an initial mount of a file system: + * - reads the superblock from disk and init the mount struct + * - if we're a 32-bit kernel, do a size check on the superblock + * so we don't mount terabyte filesystems + * - init mount struct realtime fields + * - allocate inode hash table for fs + * - init directory manager + * - perform recovery and init the log manager + */ +int +xfs_mountfs( + struct xfs_mount *mp) +{ + struct xfs_sb *sbp = &(mp->m_sb); + struct xfs_inode *rip; + struct xfs_ino_geometry *igeo = M_IGEO(mp); + uint quotamount = 0; + uint quotaflags = 0; + int error = 0; + int i; + + xfs_sb_mount_common(mp, sbp); + + /* + * Check for a mismatched features2 values. Older kernels read & wrote + * into the wrong sb offset for sb_features2 on some platforms due to + * xfs_sb_t not being 64bit size aligned when sb_features2 was added, + * which made older superblock reading/writing routines swap it as a + * 64-bit value. + * + * For backwards compatibility, we make both slots equal. + * + * If we detect a mismatched field, we OR the set bits into the existing + * features2 field in case it has already been modified; we don't want + * to lose any features. We then update the bad location with the ORed + * value so that older kernels will see any features2 flags. The + * superblock writeback code ensures the new sb_features2 is copied to + * sb_bad_features2 before it is logged or written to disk. + */ + if (xfs_sb_has_mismatched_features2(sbp)) { + xfs_warn(mp, "correcting sb_features alignment problem"); + sbp->sb_features2 |= sbp->sb_bad_features2; + mp->m_update_sb = true; + } + + + /* always use v2 inodes by default now */ + if (!(mp->m_sb.sb_versionnum & XFS_SB_VERSION_NLINKBIT)) { + mp->m_sb.sb_versionnum |= XFS_SB_VERSION_NLINKBIT; + mp->m_features |= XFS_FEAT_NLINK; + mp->m_update_sb = true; + } + + /* + * If we were given new sunit/swidth options, do some basic validation + * checks and convert the incore dalign and swidth values to the + * same units (FSB) that everything else uses. This /must/ happen + * before computing the inode geometry. + */ + error = xfs_validate_new_dalign(mp); + if (error) + goto out; + + xfs_alloc_compute_maxlevels(mp); + xfs_bmap_compute_maxlevels(mp, XFS_DATA_FORK); + xfs_bmap_compute_maxlevels(mp, XFS_ATTR_FORK); + xfs_mount_setup_inode_geom(mp); + xfs_rmapbt_compute_maxlevels(mp); + xfs_rtrmapbt_compute_maxlevels(mp); + xfs_refcountbt_compute_maxlevels(mp); + xfs_rtrefcountbt_compute_maxlevels(mp); + + xfs_agbtree_compute_maxlevels(mp); + xfs_rtbtree_compute_maxlevels(mp); + + /* + * Check if sb_agblocks is aligned at stripe boundary. If sb_agblocks + * is NOT aligned turn off m_dalign since allocator alignment is within + * an ag, therefore ag has to be aligned at stripe boundary. Note that + * we must compute the free space and rmap btree geometry before doing + * this. + */ + error = xfs_update_alignment(mp); + if (error) + goto out; + + /* enable fail_at_unmount as default */ + mp->m_fail_unmount = true; + + error = xfs_mount_sysfs_init(mp); + if (error) + goto out_remove_scrub_stats; + + xchk_stats_register(mp->m_scrub_stats, mp->m_debugfs); + + error = xfs_errortag_init(mp); + if (error) + goto out_remove_sysfs; + + error = xfs_uuid_mount(mp); + if (error) + goto out_remove_errortag; + + /* + * Update the preferred write size based on the information from the + * on-disk superblock. + */ + mp->m_allocsize_log = + max_t(uint32_t, sbp->sb_blocklog, mp->m_allocsize_log); + mp->m_allocsize_blocks = 1U << (mp->m_allocsize_log - sbp->sb_blocklog); + + /* set the low space thresholds for dynamic preallocation */ + xfs_set_low_space_thresholds(mp); + + /* + * If enabled, sparse inode chunk alignment is expected to match the + * cluster size. Full inode chunk alignment must match the chunk size, + * but that is checked on sb read verification... + */ + if (xfs_has_sparseinodes(mp) && + mp->m_sb.sb_spino_align != + XFS_B_TO_FSBT(mp, igeo->inode_cluster_size_raw)) { + xfs_warn(mp, + "Sparse inode block alignment (%u) must match cluster size (%llu).", + mp->m_sb.sb_spino_align, + XFS_B_TO_FSBT(mp, igeo->inode_cluster_size_raw)); + error = -EINVAL; + goto out_remove_uuid; + } + + /* + * Check that the data (and log if separate) is an ok size. + */ + error = xfs_check_sizes(mp); + if (error) + goto out_remove_uuid; + + /* + * Initialize realtime fields in the mount structure + */ + error = xfs_rtmount_init(mp); + if (error) { + xfs_warn(mp, "RT mount failed"); + goto out_remove_uuid; + } + + /* + * Copies the low order bits of the timestamp and the randomly + * set "sequence" number out of a UUID. + */ + mp->m_fixedfsid[0] = + (get_unaligned_be16(&sbp->sb_uuid.b[8]) << 16) | + get_unaligned_be16(&sbp->sb_uuid.b[4]); + mp->m_fixedfsid[1] = get_unaligned_be32(&sbp->sb_uuid.b[0]); + + error = xfs_da_mount(mp); + if (error) { + xfs_warn(mp, "Failed dir/attr init: %d", error); + goto out_remove_uuid; + } + + /* + * Initialize the precomputed transaction reservations values. + */ + xfs_trans_init(mp); + + /* + * Allocate and initialize the per-ag data. + */ + error = xfs_initialize_perag(mp, 0, sbp->sb_agcount, + mp->m_sb.sb_dblocks, &mp->m_maxagi); + if (error) { + xfs_warn(mp, "Failed per-ag init: %d", error); + goto out_free_dir; + } + + error = xfs_initialize_rtgroups(mp, 0, sbp->sb_rgcount, + mp->m_sb.sb_rextents); + if (error) { + xfs_warn(mp, "Failed rtgroup init: %d", error); + goto out_free_perag; + } + + if (XFS_IS_CORRUPT(mp, !sbp->sb_logblocks)) { + xfs_warn(mp, "no log defined"); + error = -EFSCORRUPTED; + goto out_free_rtgroup; + } + + error = xfs_inodegc_register_shrinker(mp); + if (error) + goto out_fail_wait; + + /* + * If we're resuming quota status, pick up the preliminary qflags from + * the ondisk superblock so that we know if we should recover dquots. + */ + if (xfs_is_resuming_quotaon(mp)) + xfs_qm_resume_quotaon(mp); + + /* + * Log's mount-time initialization. The first part of recovery can place + * some items on the AIL, to be handled when recovery is finished or + * cancelled. + */ + error = xfs_log_mount(mp, mp->m_logdev_targp, + XFS_FSB_TO_DADDR(mp, sbp->sb_logstart), + XFS_FSB_TO_BB(mp, sbp->sb_logblocks)); + if (error) { + xfs_warn(mp, "log mount failed"); + goto out_inodegc_shrinker; + } + + /* + * If we're resuming quota status and recovered the log, re-sample the + * qflags from the ondisk superblock now that we've recovered it, just + * in case someone shut down enforcement just before a crash. + */ + if (xfs_clear_resuming_quotaon(mp) && xlog_recovery_needed(mp->m_log)) + xfs_qm_resume_quotaon(mp); + + /* + * If logged xattrs are still enabled after log recovery finishes, then + * they'll be available until unmount. Otherwise, turn them off. + */ + if (xfs_sb_version_haslogxattrs(&mp->m_sb)) + xfs_set_using_logged_xattrs(mp); + else + xfs_clear_using_logged_xattrs(mp); + + /* Enable background inode inactivation workers. */ + xfs_inodegc_start(mp); + xfs_blockgc_start(mp); + + if (xfs_has_metadir(mp)) { + error = xfs_mount_setup_metadir(mp); + if (error) + goto out_free_metadir; + } + + /* + * Get and sanity-check the root inode. + * Save the pointer to it in the mount structure. + */ + error = xfs_iget(mp, NULL, sbp->sb_rootino, XFS_IGET_UNTRUSTED, + XFS_ILOCK_EXCL, &rip); + if (error) { + xfs_warn(mp, + "Failed to read root inode 0x%llx, error %d", + sbp->sb_rootino, -error); + goto out_free_metadir; + } + + ASSERT(rip != NULL); + + if (XFS_IS_CORRUPT(mp, !S_ISDIR(VFS_I(rip)->i_mode))) { + xfs_warn(mp, "corrupted root inode %llu: not a directory", + (unsigned long long)I_INO(rip)); + xfs_iunlock(rip, XFS_ILOCK_EXCL); + error = -EFSCORRUPTED; + goto out_rele_rip; + } + mp->m_rootip = rip; /* save it */ + + xfs_iunlock(rip, XFS_ILOCK_EXCL); + + /* + * Initialize realtime inode pointers in the mount structure + */ + error = xfs_rtmount_inodes(mp); + if (error) { + /* + * Free up the root inode. + */ + xfs_warn(mp, "failed to read RT inodes"); + goto out_rele_rip; + } + + /* Make sure the summary counts are ok. */ + error = xfs_check_summary_counts(mp); + if (error) + goto out_rtunmount; + + /* + * If this is a read-only mount defer the superblock updates until + * the next remount into writeable mode. Otherwise we would never + * perform the update e.g. for the root filesystem. + */ + if (mp->m_update_sb && !xfs_is_readonly(mp)) { + error = xfs_sync_sb(mp, false); + if (error) { + xfs_warn(mp, "failed to write sb changes"); + goto out_rtunmount; + } + } + + /* + * Initialise the XFS quota management subsystem for this mount + */ + if (XFS_IS_QUOTA_ON(mp)) { + error = xfs_qm_newmount(mp, "amount, "aflags); + if (error) + goto out_rtunmount; + } else { + /* + * If a file system had quotas running earlier, but decided to + * mount without -o uquota/pquota/gquota options, revoke the + * quotachecked license. + */ + if (mp->m_sb.sb_qflags & XFS_ALL_QUOTA_ACCT) { + xfs_notice(mp, "resetting quota flags"); + error = xfs_mount_reset_sbqflags(mp); + if (error) + goto out_rtunmount; + } + } + + /* + * Finish recovering the file system. This part needed to be delayed + * until after the root and real-time bitmap inodes were consistently + * read in. Temporarily create per-AG space reservations for metadata + * btree shape changes because space freeing transactions (for inode + * inactivation) require the per-AG reservation in lieu of reserving + * blocks. + */ + error = xfs_fs_reserve_ag_blocks(mp); + if (error) { + if (error != -ENOSPC) + goto out_rtunmount; + xfs_warn(mp, +"ENOSPC reserving per-AG metadata pool, log recovery may fail."); + } + error = xfs_log_mount_finish(mp); + xfs_fs_unreserve_ag_blocks(mp); + if (error) { + xfs_warn(mp, "log mount finish failed"); + goto out_rtunmount; + } + + /* + * Now the log is fully replayed, we can transition to full read-only + * mode for read-only mounts. This will sync all the metadata and clean + * the log so that the recovery we just performed does not have to be + * replayed again on the next mount. + * + * We use the same quiesce mechanism as the rw->ro remount, as they are + * semantically identical operations. + */ + if (xfs_is_readonly(mp) && !xfs_has_norecovery(mp)) + xfs_log_clean(mp); + + if (xfs_has_zoned(mp)) { + error = xfs_mount_zones(mp); + if (error) + goto out_rtunmount; + } + + /* + * Complete the quota initialisation, post-log-replay component. + */ + if (quotamount) { + ASSERT(mp->m_qflags == 0); + mp->m_qflags = quotaflags; + + xfs_qm_mount_quotas(mp); + } + + /* + * Now we are mounted, reserve a small amount of unused space for + * privileged transactions. This is needed so that transaction + * space required for critical operations can dip into this pool + * when at ENOSPC. This is needed for operations like create with + * attr, unwritten extent conversion at ENOSPC, garbage collection + * etc. Data allocations are not allowed to use this reserved space. + * + * This may drive us straight to ENOSPC on mount, but that implies + * we were already there on the last unmount. Warn if this occurs. + */ + if (!xfs_is_readonly(mp)) { + for (i = 0; i < XC_FREE_NR; i++) { + error = xfs_reserve_blocks(mp, i, + xfs_default_resblks(mp, i)); + if (error) + xfs_warn(mp, +"Unable to allocate reserve blocks. Continuing without reserve pool for %s.", + xfs_free_pool_name[i]); + } + + /* Reserve AG blocks for future btree expansion. */ + error = xfs_fs_reserve_ag_blocks(mp); + if (error && error != -ENOSPC) + goto out_agresv; + + xfs_zone_gc_start(mp); + } + + /* + * Pre-calculate atomic write unit max. This involves computations + * derived from transaction reservations, so we must do this after the + * log is fully initialized. + */ + error = xfs_set_max_atomic_write_opt(mp, mp->m_awu_max_bytes); + if (error) + goto out_agresv; + + return 0; + + out_agresv: + xfs_fs_unreserve_ag_blocks(mp); + xfs_qm_unmount_quotas(mp); + if (xfs_has_zoned(mp)) + xfs_unmount_zones(mp); + out_rtunmount: + xfs_rtunmount_inodes(mp); + out_rele_rip: + xfs_irele(rip); + /* Clean out dquots that might be in memory after quotacheck. */ + xfs_qm_unmount(mp); + out_free_metadir: + if (mp->m_metadirip) + xfs_irele(mp->m_metadirip); + + /* + * The mount has failed. Mark the filesystem shut down so that any + * inodes still queued for background inactivation are dropped + * straight to reclaim instead of being inactivated: a failed mount + * must not write to the (possibly corrupt, only partially set up) + * persistent metadata, and parts of the mount it would need - e.g. + * the quota subsystem (mp->m_quotainfo) - may never have been + * initialised. + * + * Flush the queue so that those inodes are pulled down and reclaim + * can free them; with the fs shut down xfs_inodegc_inactivate() + * turns each one reclaimable without touching the on-disk structures. + */ + xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR); + xfs_inodegc_flush(mp); + + /* + * Flush all inode reclamation work and flush the log. + * We have to do this /after/ rtunmount and qm_unmount because those + * two will have scheduled delayed reclaim for the rt/quota inodes. + * + * This is slightly different from the unmountfs call sequence + * because we could be tearing down a partially set up mount. In + * particular, if log_mount_finish fails we bail out without calling + * qm_unmount_quotas and therefore rely on qm_unmount to release the + * quota inodes. + */ + xfs_unmount_flush_inodes(mp); + xfs_log_mount_cancel(mp); + out_inodegc_shrinker: + shrinker_free(mp->m_inodegc_shrinker); + out_fail_wait: + if (mp->m_logdev_targp && mp->m_logdev_targp != mp->m_ddev_targp) + xfs_buftarg_drain(mp->m_logdev_targp); + xfs_buftarg_drain(mp->m_ddev_targp); + out_free_rtgroup: + xfs_free_rtgroups(mp, 0, mp->m_sb.sb_rgcount); + out_free_perag: + xfs_free_perag_range(mp, 0, mp->m_sb.sb_agcount); + out_free_dir: + xfs_da_unmount(mp); + out_remove_uuid: + xfs_uuid_unmount(mp); + out_remove_errortag: + xfs_errortag_del(mp); + out_remove_sysfs: + xfs_mount_sysfs_del(mp); + out_remove_scrub_stats: + xchk_stats_unregister(mp->m_scrub_stats); + out: + return error; +} + +/* + * This flushes out the inodes,dquots and the superblock, unmounts the + * log and makes sure that incore structures are freed. + */ +void +xfs_unmountfs( + struct xfs_mount *mp) +{ + int error; + + /* + * Perform all on-disk metadata updates required to inactivate inodes + * that the VFS evicted earlier in the unmount process. Freeing inodes + * and discarding CoW fork preallocations can cause shape changes to + * the free inode and refcount btrees, respectively, so we must finish + * this before we discard the metadata space reservations. Metadata + * inodes and the root directory do not require inactivation. + */ + xfs_inodegc_flush(mp); + + xfs_blockgc_stop(mp); + if (!test_bit(XFS_OPSTATE_READONLY, &mp->m_opstate)) + xfs_zone_gc_stop(mp); + xfs_fs_unreserve_ag_blocks(mp); + xfs_qm_unmount_quotas(mp); + if (xfs_has_zoned(mp)) + xfs_unmount_zones(mp); + xfs_rtunmount_inodes(mp); + xfs_irele(mp->m_rootip); + if (mp->m_metadirip) + xfs_irele(mp->m_metadirip); + + xfs_unmount_flush_inodes(mp); + + xfs_qm_unmount(mp); + + /* + * Unreserve any blocks we have so that when we unmount we don't account + * the reserved free space as used. This is really only necessary for + * lazy superblock counting because it trusts the incore superblock + * counters to be absolutely correct on clean unmount. + * + * We don't bother correcting this elsewhere for lazy superblock + * counting because on mount of an unclean filesystem we reconstruct the + * correct counter value and this is irrelevant. + * + * For non-lazy counter filesystems, this doesn't matter at all because + * we only every apply deltas to the superblock and hence the incore + * value does not matter.... + */ + error = xfs_reserve_blocks(mp, XC_FREE_BLOCKS, 0); + if (error) + xfs_warn(mp, "Unable to free reserved block pool. " + "Freespace may not be correct on next mount."); + xfs_unmount_check(mp); + + /* + * Indicate that it's ok to clear log incompat bits before cleaning + * the log and writing the unmount record. + */ + xfs_set_done_with_log_incompat(mp); + xfs_log_unmount(mp); + xfs_da_unmount(mp); + xfs_uuid_unmount(mp); + +#if defined(DEBUG) + xfs_errortag_clearall(mp); +#endif + shrinker_free(mp->m_inodegc_shrinker); + xfs_free_rtgroups(mp, 0, mp->m_sb.sb_rgcount); + xfs_free_perag_range(mp, 0, mp->m_sb.sb_agcount); + xfs_errortag_del(mp); + xchk_stats_unregister(mp->m_scrub_stats); + xfs_mount_sysfs_del(mp); +} + +/* + * Determine whether modifications can proceed. The caller specifies the minimum + * freeze level for which modifications should not be allowed. This allows + * certain operations to proceed while the freeze sequence is in progress, if + * necessary. + */ +bool +xfs_fs_writable( + struct xfs_mount *mp, + int level) +{ + ASSERT(level > SB_UNFROZEN); + if ((mp->m_super->s_writers.frozen >= level) || + xfs_is_shutdown(mp) || xfs_is_readonly(mp)) + return false; + + return true; +} + +/* + * Estimate the amount of free space that is not available to userspace and is + * not explicitly reserved from the incore fdblocks. This includes: + * + * - The minimum number of blocks needed to support splitting a bmap btree + * - The blocks currently in use by the freespace btrees because they record + * the actual blocks that will fill per-AG metadata space reservations + */ +uint64_t +xfs_freecounter_unavailable( + struct xfs_mount *mp, + enum xfs_free_counter ctr) +{ + if (ctr != XC_FREE_BLOCKS) + return 0; + return mp->m_alloc_set_aside + atomic64_read(&mp->m_allocbt_blks); +} + +void +xfs_add_freecounter( + struct xfs_mount *mp, + enum xfs_free_counter ctr, + uint64_t delta) +{ + struct xfs_freecounter *counter = &mp->m_free[ctr]; + uint64_t res_used; + + /* + * If the reserve pool is depleted, put blocks back into it first. + * Most of the time the pool is full. + */ + if (likely(counter->res_avail == counter->res_total)) { + percpu_counter_add(&counter->count, delta); + return; + } + + spin_lock(&mp->m_sb_lock); + res_used = counter->res_total - counter->res_avail; + if (res_used > delta) { + counter->res_avail += delta; + } else { + delta -= res_used; + counter->res_avail = counter->res_total; + percpu_counter_add(&counter->count, delta); + } + spin_unlock(&mp->m_sb_lock); +} + + +/* Adjust in-core free blocks or RT extents. */ +int +xfs_dec_freecounter( + struct xfs_mount *mp, + enum xfs_free_counter ctr, + uint64_t delta, + bool rsvd) +{ + struct xfs_freecounter *counter = &mp->m_free[ctr]; + s32 batch; + + ASSERT(ctr < XC_FREE_NR); + + /* + * Taking blocks away, need to be more accurate the closer we + * are to zero. + * + * If the counter has a value of less than 2 * max batch size, + * then make everything serialise as we are real close to + * ENOSPC. + */ + if (__percpu_counter_compare(&counter->count, 2 * XFS_FDBLOCKS_BATCH, + XFS_FDBLOCKS_BATCH) < 0) + batch = 1; + else + batch = XFS_FDBLOCKS_BATCH; + + /* + * Set aside allocbt blocks because these blocks are tracked as free + * space but not available for allocation. Technically this means that a + * single reservation cannot consume all remaining free space, but the + * ratio of allocbt blocks to usable free blocks should be rather small. + * The tradeoff without this is that filesystems that maintain high + * perag block reservations can over reserve physical block availability + * and fail physical allocation, which leads to much more serious + * problems (i.e. transaction abort, pagecache discards, etc.) than + * slightly premature -ENOSPC. + */ + percpu_counter_add_batch(&counter->count, -((int64_t)delta), batch); + if (__percpu_counter_compare(&counter->count, + xfs_freecounter_unavailable(mp, ctr), + XFS_FDBLOCKS_BATCH) < 0) { + /* + * Lock up the sb for dipping into reserves before releasing the + * space that took us to ENOSPC. + */ + spin_lock(&mp->m_sb_lock); + percpu_counter_add(&counter->count, delta); + if (!rsvd) + goto fdblocks_enospc; + if (delta > counter->res_avail) { + if (ctr == XC_FREE_BLOCKS) + xfs_warn_once(mp, +"Reserve blocks depleted! Consider increasing reserve pool size."); + goto fdblocks_enospc; + } + counter->res_avail -= delta; + trace_xfs_freecounter_reserved(mp, ctr, delta, _RET_IP_); + spin_unlock(&mp->m_sb_lock); + } + + /* we had space! */ + return 0; + +fdblocks_enospc: + trace_xfs_freecounter_enospc(mp, ctr, delta, _RET_IP_); + spin_unlock(&mp->m_sb_lock); + return -ENOSPC; +} + +/* + * Used to free the superblock along various error paths. + */ +void +xfs_freesb( + struct xfs_mount *mp) +{ + struct xfs_buf *bp = mp->m_sb_bp; + + xfs_buf_lock(bp); + mp->m_sb_bp = NULL; + xfs_buf_relse(bp); +} + +/* + * If the underlying (data/log/rt) device is readonly, there are some + * operations that cannot proceed. + */ +int +xfs_dev_is_read_only( + struct xfs_mount *mp, + char *message) +{ + if (xfs_readonly_buftarg(mp->m_ddev_targp) || + xfs_readonly_buftarg(mp->m_logdev_targp) || + (mp->m_rtdev_targp && xfs_readonly_buftarg(mp->m_rtdev_targp))) { + xfs_notice(mp, "%s required on read-only device.", message); + xfs_notice(mp, "write access unavailable, cannot proceed."); + return -EROFS; + } + return 0; +} + +/* Force the summary counters to be recalculated at next mount. */ +void +xfs_force_summary_recalc( + struct xfs_mount *mp) +{ + if (!xfs_has_lazysbcount(mp)) + return; + + xfs_fs_mark_sick(mp, XFS_SICK_FS_COUNTERS); +} + +/* + * Enable a log incompat feature flag in the primary superblock. The caller + * cannot have any other transactions in progress. + */ +int +xfs_add_incompat_log_feature( + struct xfs_mount *mp, + uint32_t feature) +{ + struct xfs_dsb *dsb; + int error; + + ASSERT(hweight32(feature) == 1); + ASSERT(!(feature & XFS_SB_FEAT_INCOMPAT_LOG_UNKNOWN)); + + /* + * Force the log to disk and kick the background AIL thread to reduce + * the chances that the bwrite will stall waiting for the AIL to unpin + * the primary superblock buffer. This isn't a data integrity + * operation, so we don't need a synchronous push. + */ + error = xfs_log_force(mp, XFS_LOG_SYNC); + if (error) + return error; + xfs_ail_push_all(mp->m_ail); + + /* + * Lock the primary superblock buffer to serialize all callers that + * are trying to set feature bits. + */ + xfs_buf_lock(mp->m_sb_bp); + xfs_buf_hold(mp->m_sb_bp); + + if (xfs_is_shutdown(mp)) { + error = -EIO; + goto rele; + } + + if (xfs_sb_has_incompat_log_feature(&mp->m_sb, feature)) + goto rele; + + /* + * Write the primary superblock to disk immediately, because we need + * the log_incompat bit to be set in the primary super now to protect + * the log items that we're going to commit later. + */ + dsb = mp->m_sb_bp->b_addr; + xfs_sb_to_disk(dsb, &mp->m_sb); + dsb->sb_features_log_incompat |= cpu_to_be32(feature); + error = xfs_bwrite(mp->m_sb_bp); + if (error) + goto shutdown; + + /* + * Add the feature bits to the incore superblock before we unlock the + * buffer. + */ + xfs_sb_add_incompat_log_features(&mp->m_sb, feature); + xfs_buf_relse(mp->m_sb_bp); + + /* Log the superblock to disk. */ + return xfs_sync_sb(mp, false); +shutdown: + xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR); +rele: + xfs_buf_relse(mp->m_sb_bp); + return error; +} + +/* + * Clear all the log incompat flags from the superblock. + * + * The caller cannot be in a transaction, must ensure that the log does not + * contain any log items protected by any log incompat bit, and must ensure + * that there are no other threads that depend on the state of the log incompat + * feature flags in the primary super. + * + * Returns true if the superblock is dirty. + */ +bool +xfs_clear_incompat_log_features( + struct xfs_mount *mp) +{ + bool ret = false; + + if (!xfs_has_crc(mp) || + !xfs_sb_has_incompat_log_feature(&mp->m_sb, + XFS_SB_FEAT_INCOMPAT_LOG_ALL) || + xfs_is_shutdown(mp) || + !xfs_is_done_with_log_incompat(mp)) + return false; + + /* + * Update the incore superblock. We synchronize on the primary super + * buffer lock to be consistent with the add function, though at least + * in theory this shouldn't be necessary. + */ + xfs_buf_lock(mp->m_sb_bp); + xfs_buf_hold(mp->m_sb_bp); + + if (xfs_sb_has_incompat_log_feature(&mp->m_sb, + XFS_SB_FEAT_INCOMPAT_LOG_ALL)) { + xfs_sb_remove_incompat_log_features(&mp->m_sb); + ret = true; + } + + xfs_buf_relse(mp->m_sb_bp); + return ret; +} + +/* + * Update the in-core delayed block counter. + * + * We prefer to update the counter without having to take a spinlock for every + * counter update (i.e. batching). Each change to delayed allocation + * reservations can change can easily exceed the default percpu counter + * batching, so we use a larger batch factor here. + * + * Note that we don't currently have any callers requiring fast summation + * (e.g. percpu_counter_read) so we can use a big batch value here. + */ +#define XFS_DELALLOC_BATCH (4096) +void +xfs_mod_delalloc( + struct xfs_inode *ip, + int64_t data_delta, + int64_t ind_delta) +{ + struct xfs_mount *mp = ip->i_mount; + + if (XFS_IS_REALTIME_INODE(ip)) { + percpu_counter_add_batch(&mp->m_delalloc_rtextents, + xfs_blen_to_rtbxlen(mp, data_delta), + XFS_DELALLOC_BATCH); + if (!ind_delta) + return; + data_delta = 0; + } + percpu_counter_add_batch(&mp->m_delalloc_blks, data_delta + ind_delta, + XFS_DELALLOC_BATCH); +} diff --git a/fs/xfs/xfs_mount.h b/fs/xfs/xfs_mount.h new file mode 100644 index 000000000..216a38a35 --- /dev/null +++ b/fs/xfs/xfs_mount.h @@ -0,0 +1,833 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_MOUNT_H__ +#define __XFS_MOUNT_H__ + +struct xlog; +struct xfs_inode; +struct xfs_mru_cache; +struct xfs_ail; +struct xfs_quotainfo; +struct xfs_da_geometry; +struct xfs_perag; +struct xfs_healthmon; + +/* dynamic preallocation free space thresholds, 5% down to 1% */ +enum { + XFS_LOWSP_1_PCNT = 0, + XFS_LOWSP_2_PCNT, + XFS_LOWSP_3_PCNT, + XFS_LOWSP_4_PCNT, + XFS_LOWSP_5_PCNT, + XFS_LOWSP_MAX, +}; + +/* + * Error Configuration + * + * Error classes define the subsystem the configuration belongs to. + * Error numbers define the errors that are configurable. + */ +enum { + XFS_ERR_METADATA, + XFS_ERR_CLASS_MAX, +}; +enum { + XFS_ERR_DEFAULT, + XFS_ERR_EIO, + XFS_ERR_ENOSPC, + XFS_ERR_ENODEV, + XFS_ERR_ERRNO_MAX, +}; + +#define XFS_ERR_RETRY_FOREVER -1 + +/* + * Although retry_timeout is in jiffies which is normally an unsigned long, + * we limit the retry timeout to 86400 seconds, or one day. So even a + * signed 32-bit long is sufficient for a HZ value up to 24855. Making it + * signed lets us store the special "-1" value, meaning retry forever. + */ +struct xfs_error_cfg { + struct xfs_kobj kobj; + int max_retries; + long retry_timeout; /* in jiffies, -1 = infinite */ +}; + +/* + * Per-cpu deferred inode inactivation GC lists. + */ +struct xfs_inodegc { + struct xfs_mount *mp; + struct llist_head list; + struct delayed_work work; + int error; + + /* approximate count of inodes in the list */ + unsigned int items; + unsigned int shrinker_hits; + unsigned int cpu; +}; + +/* + * Container for each type of groups, used to look up individual groups and + * describes the geometry. + */ +struct xfs_groups { + struct xarray xa; + + /* + * Maximum capacity of the group in FSBs. + * + * Each group is laid out densely in the daddr space. For the + * degenerate case of a pre-rtgroups filesystem, the incore rtgroup + * pretends to have a zero-block and zero-blklog rtgroup. + */ + uint32_t blocks; + + /* + * Log(2) of the logical size of each group. + * + * Compared to the blocks field above this is rounded up to the next + * power of two, and thus lays out the xfs_fsblock_t/xfs_rtblock_t + * space sparsely with a hole from blocks to (1 << blklog) at the end + * of each group. + */ + uint8_t blklog; + + /* + * Zoned devices can have gaps beyond the usable capacity of a zone and + * the end in the LBA/daddr address space. In other words, the hardware + * equivalent to the RT groups already takes care of the power of 2 + * alignment for us. In this case the sparse FSB/RTB address space maps + * 1:1 to the device address space. + */ + bool has_daddr_gaps; + + /* + * Mask to extract the group-relative block number from a FSB. + * For a pre-rtgroups filesystem we pretend to have one very large + * rtgroup, so this mask must be 64-bit. + */ + uint64_t blkmask; + + /* + * Start of the first group in the device. This is used to support a + * RT device following the data device on the same block device for + * SMR hard drives. + */ + xfs_fsblock_t start_fsb; + + /* + * Maximum length of an atomic write for files stored in this + * collection of allocation groups, in fsblocks. + */ + xfs_extlen_t awu_max; +}; + +struct xfs_freecounter { + /* free blocks for general use: */ + struct percpu_counter count; + + /* total reserved blocks: */ + uint64_t res_total; + + /* available reserved blocks: */ + uint64_t res_avail; + + /* reserved blks @ remount,ro: */ + uint64_t res_saved; +}; + +/* + * The struct xfsmount layout is optimised to separate read-mostly variables + * from variables that are frequently modified. We put the read-mostly variables + * first, then place all the other variables at the end. + * + * Typically, read-mostly variables are those that are set at mount time and + * never changed again, or only change rarely as a result of things like sysfs + * knobs being tweaked. + */ +typedef struct xfs_mount { + struct xfs_sb m_sb; /* copy of fs superblock */ + struct super_block *m_super; + struct xfs_ail *m_ail; /* fs active log item list */ + struct xfs_buf *m_sb_bp; /* buffer for superblock */ + struct xfs_buf *m_rtsb_bp; /* realtime superblock */ + char *m_rtname; /* realtime device name */ + char *m_logname; /* external log device name */ + struct xfs_da_geometry *m_dir_geo; /* directory block geometry */ + struct xfs_da_geometry *m_attr_geo; /* attribute block geometry */ + struct xlog *m_log; /* log specific stuff */ + struct xfs_inode *m_rootip; /* pointer to root directory */ + struct xfs_inode *m_metadirip; /* ptr to metadata directory */ + struct xfs_inode *m_rtdirip; /* ptr to realtime metadir */ + struct xfs_quotainfo *m_quotainfo; /* disk quota information */ + struct xfs_buftarg *m_ddev_targp; /* data device */ + struct xfs_buftarg *m_logdev_targp;/* log device */ + struct xfs_buftarg *m_rtdev_targp; /* rt device */ + void __percpu *m_inodegc; /* percpu inodegc structures */ + struct xfs_mru_cache *m_filestream; /* per-mount filestream data */ + struct workqueue_struct *m_buf_workqueue; + struct workqueue_struct *m_unwritten_workqueue; + struct workqueue_struct *m_reclaim_workqueue; + struct workqueue_struct *m_sync_workqueue; + struct workqueue_struct *m_blockgc_wq; + struct workqueue_struct *m_inodegc_wq; + + int m_bsize; /* fs logical block size */ + uint8_t m_blkbit_log; /* blocklog + NBBY */ + uint8_t m_blkbb_log; /* blocklog - BBSHIFT */ + uint8_t m_agno_log; /* log #ag's */ + uint8_t m_sectbb_log; /* sectlog - BBSHIFT */ + int8_t m_rtxblklog; /* log2 of rextsize, if possible */ + + uint m_blockmask; /* sb_blocksize-1 */ + uint m_blockwsize; /* sb_blocksize in words */ + /* number of rt extents per rt bitmap block if rtgroups enabled */ + unsigned int m_rtx_per_rbmblock; + uint m_alloc_mxr[2]; /* max alloc btree records */ + uint m_alloc_mnr[2]; /* min alloc btree records */ + uint m_bmap_dmxr[2]; /* max bmap btree records */ + uint m_bmap_dmnr[2]; /* min bmap btree records */ + uint m_rmap_mxr[2]; /* max rmap btree records */ + uint m_rmap_mnr[2]; /* min rmap btree records */ + uint m_rtrmap_mxr[2]; /* max rtrmap btree records */ + uint m_rtrmap_mnr[2]; /* min rtrmap btree records */ + uint m_refc_mxr[2]; /* max refc btree records */ + uint m_refc_mnr[2]; /* min refc btree records */ + uint m_rtrefc_mxr[2]; /* max rtrefc btree records */ + uint m_rtrefc_mnr[2]; /* min rtrefc btree records */ + uint m_alloc_maxlevels; /* max alloc btree levels */ + uint m_bm_maxlevels[2]; /* max bmap btree levels */ + uint m_rmap_maxlevels; /* max rmap btree levels */ + uint m_rtrmap_maxlevels; /* max rtrmap btree level */ + uint m_refc_maxlevels; /* max refcount btree level */ + uint m_rtrefc_maxlevels; /* max rtrefc btree level */ + unsigned int m_agbtree_maxlevels; /* max level of all AG btrees */ + unsigned int m_rtbtree_maxlevels; /* max level of all rt btrees */ + xfs_extlen_t m_ag_prealloc_blocks; /* reserved ag blocks */ + uint m_alloc_set_aside; /* space we can't use */ + uint m_ag_max_usable; /* max space per AG */ + int m_dalign; /* stripe unit */ + int m_swidth; /* stripe width */ + xfs_agnumber_t m_maxagi; /* highest inode alloc group */ + uint m_allocsize_log;/* min write size log bytes */ + uint m_allocsize_blocks; /* min write size blocks */ + int m_logbufs; /* number of log buffers */ + int m_logbsize; /* size of each log buffer */ + unsigned int m_rsumlevels; /* rt summary levels */ + xfs_filblks_t m_rsumblocks; /* size of rt summary, FSBs */ + int m_fixedfsid[2]; /* unchanged for life of FS */ + uint m_qflags; /* quota status flags */ + uint64_t m_features; /* active filesystem features */ + uint64_t m_low_space[XFS_LOWSP_MAX]; + uint64_t m_low_rtexts[XFS_LOWSP_MAX]; + uint64_t m_rtxblkmask; /* rt extent block mask */ + struct xfs_ino_geometry m_ino_geo; /* inode geometry */ + struct xfs_trans_resv m_resv; /* precomputed res values */ + /* low free space thresholds */ + unsigned long m_opstate; /* dynamic state flags */ + bool m_always_cow; + bool m_fail_unmount; + bool m_finobt_nores; /* no per-AG finobt resv. */ + bool m_update_sb; /* sb needs update in mount */ + unsigned int m_max_open_zones; + unsigned int m_zonegc_low_space; + + /* max_atomic_write mount option value */ + unsigned long long m_awu_max_bytes; + + /* + * Bitsets of per-fs metadata that have been checked and/or are sick. + * Callers must hold m_sb_lock to access these two fields. + */ + uint8_t m_fs_checked; + uint8_t m_fs_sick; + /* + * Bitsets of rt metadata that have been checked and/or are sick. + * Callers must hold m_sb_lock to access this field. + */ + uint8_t m_rt_checked; + uint8_t m_rt_sick; + + /* + * End of read-mostly variables. Frequently written variables and locks + * should be placed below this comment from now on. The first variable + * here is marked as cacheline aligned so they it is separated from + * the read-mostly variables. + */ + + spinlock_t ____cacheline_aligned m_sb_lock; /* sb counter lock */ + struct percpu_counter m_icount; /* allocated inodes counter */ + struct percpu_counter m_ifree; /* free inodes counter */ + + struct xfs_freecounter m_free[XC_FREE_NR]; + + /* + * Count of data device blocks reserved for delayed allocations, + * including indlen blocks. Does not include allocated CoW staging + * extents or anything related to the rt device. + */ + struct percpu_counter m_delalloc_blks; + + /* + * RT version of the above. + */ + struct percpu_counter m_delalloc_rtextents; + + /* + * Global count of allocation btree blocks in use across all AGs. Only + * used when perag reservation is enabled. Helps prevent block + * reservation from attempting to reserve allocation btree blocks. + */ + atomic64_t m_allocbt_blks; + + struct xfs_groups m_groups[XG_TYPE_MAX]; + struct delayed_work m_reclaim_work; /* background inode reclaim */ + struct xfs_zone_info *m_zone_info; /* zone allocator information */ + struct dentry *m_debugfs; /* debugfs parent */ + struct xfs_kobj m_kobj; + struct xfs_kobj m_error_kobj; + struct xfs_kobj m_error_meta_kobj; + struct xfs_error_cfg m_error_cfg[XFS_ERR_CLASS_MAX][XFS_ERR_ERRNO_MAX]; + struct xstats m_stats; /* per-fs stats */ +#ifdef CONFIG_XFS_ONLINE_SCRUB_STATS + struct xchk_stats *m_scrub_stats; +#endif + struct xfs_kobj m_zoned_kobj; + xfs_agnumber_t m_agfrotor; /* last ag where space found */ + atomic_t m_agirotor; /* last ag dir inode alloced */ + atomic_t m_rtgrotor; /* last rtgroup rtpicked */ + + struct mutex m_metafile_resv_lock; + uint64_t m_metafile_resv_target; + uint64_t m_metafile_resv_used; + uint64_t m_metafile_resv_avail; + + /* Memory shrinker to throttle and reprioritize inodegc */ + struct shrinker *m_inodegc_shrinker; + /* + * Workqueue item so that we can coalesce multiple inode flush attempts + * into a single flush. + */ + struct work_struct m_flush_inodes_work; + + /* + * Generation of the filesysyem layout. This is incremented by each + * growfs, and used by the pNFS server to ensure the client updates + * its view of the block device once it gets a layout that might + * reference the newly added blocks. Does not need to be persistent + * as long as we only allow file system size increments, but if we + * ever support shrinks it would have to be persisted in addition + * to various other kinds of pain inflicted on the pNFS server. + */ + uint32_t m_generation; + struct mutex m_growlock; /* growfs mutex */ + +#ifdef DEBUG + /* + * Frequency with which errors are injected. Replaces xfs_etest; the + * value stored in here is the inverse of the frequency with which the + * error triggers. 1 = always, 2 = half the time, etc. + */ + unsigned int *m_errortag; + struct xfs_kobj m_errortag_kobj; +#endif + + /* cpus that have inodes queued for inactivation */ + struct cpumask m_inodegc_cpumask; + + /* Hook to feed dirent updates to an active online repair. */ + struct xfs_hooks m_dir_update_hooks; + + /* Private data referring to a health monitor object. */ + struct xfs_healthmon __rcu *m_healthmon; + + /* Index of uuid record in the uuid xarray. */ + unsigned int m_uuid_table_index; + + /* + * Old io_pages/ra_pages valued in the main bdev BDI, and our initial + * calculated values. + */ + unsigned long m_old_io_pages, m_initial_io_pages; + unsigned long m_old_ra_pages, m_initial_ra_pages; +} xfs_mount_t; + +#define M_IGEO(mp) (&(mp)->m_ino_geo) + +/* + * Flags for m_features. + * + * These are all the active features in the filesystem, regardless of how + * they are configured. + */ +#define XFS_FEAT_ATTR (1ULL << 0) /* xattrs present in fs */ +#define XFS_FEAT_NLINK (1ULL << 1) /* 32 bit link counts */ +#define XFS_FEAT_QUOTA (1ULL << 2) /* quota active */ +#define XFS_FEAT_ALIGN (1ULL << 3) /* inode alignment */ +#define XFS_FEAT_DALIGN (1ULL << 4) /* data alignment */ +#define XFS_FEAT_LOGV2 (1ULL << 5) /* version 2 logs */ +#define XFS_FEAT_SECTOR (1ULL << 6) /* sector size > 512 bytes */ +#define XFS_FEAT_EXTFLG (1ULL << 7) /* unwritten extents */ +#define XFS_FEAT_ASCIICI (1ULL << 8) /* ASCII only case-insens. */ +#define XFS_FEAT_LAZYSBCOUNT (1ULL << 9) /* Superblk counters */ +#define XFS_FEAT_PARENT (1ULL << 11) /* parent pointers */ +#define XFS_FEAT_PROJID32 (1ULL << 12) /* 32 bit project id */ +#define XFS_FEAT_CRC (1ULL << 13) /* metadata CRCs */ +#define XFS_FEAT_V3INODES (1ULL << 14) /* Version 3 inodes */ +#define XFS_FEAT_PQUOTINO (1ULL << 15) /* non-shared proj/grp quotas */ +#define XFS_FEAT_FTYPE (1ULL << 16) /* inode type in dir */ +#define XFS_FEAT_FINOBT (1ULL << 17) /* free inode btree */ +#define XFS_FEAT_RMAPBT (1ULL << 18) /* reverse map btree */ +#define XFS_FEAT_REFLINK (1ULL << 19) /* reflinked files */ +#define XFS_FEAT_SPINODES (1ULL << 20) /* sparse inode chunks */ +#define XFS_FEAT_META_UUID (1ULL << 21) /* metadata UUID */ +#define XFS_FEAT_REALTIME (1ULL << 22) /* realtime device present */ +#define XFS_FEAT_INOBTCNT (1ULL << 23) /* inobt block counts */ +#define XFS_FEAT_BIGTIME (1ULL << 24) /* large timestamps */ +#define XFS_FEAT_NEEDSREPAIR (1ULL << 25) /* needs xfs_repair */ +#define XFS_FEAT_NREXT64 (1ULL << 26) /* large extent counters */ +#define XFS_FEAT_EXCHANGE_RANGE (1ULL << 27) /* exchange range */ +#define XFS_FEAT_METADIR (1ULL << 28) /* metadata directory tree */ +#define XFS_FEAT_ZONED (1ULL << 29) /* zoned RT device */ + +/* Mount features */ +#define XFS_FEAT_NOLIFETIME (1ULL << 47) /* disable lifetime hints */ +#define XFS_FEAT_NOALIGN (1ULL << 49) /* ignore alignment */ +#define XFS_FEAT_ALLOCSIZE (1ULL << 50) /* user specified allocation size */ +#define XFS_FEAT_LARGE_IOSIZE (1ULL << 51) /* report large preferred + * I/O size in stat() */ +#define XFS_FEAT_WSYNC (1ULL << 52) /* synchronous metadata ops */ +#define XFS_FEAT_DIRSYNC (1ULL << 53) /* synchronous directory ops */ +#define XFS_FEAT_DISCARD (1ULL << 54) /* discard unused blocks */ +#define XFS_FEAT_GRPID (1ULL << 55) /* group-ID assigned from directory */ +#define XFS_FEAT_SMALL_INUMS (1ULL << 56) /* user wants 32bit inodes */ +#define XFS_FEAT_SWALLOC (1ULL << 58) /* stripe width allocation */ +#define XFS_FEAT_FILESTREAMS (1ULL << 59) /* use filestreams allocator */ +#define XFS_FEAT_DAX_ALWAYS (1ULL << 60) /* DAX always enabled */ +#define XFS_FEAT_DAX_NEVER (1ULL << 61) /* DAX never enabled */ +#define XFS_FEAT_NORECOVERY (1ULL << 62) /* no recovery - dirty fs */ +#define XFS_FEAT_NOUUID (1ULL << 63) /* ignore uuid during mount */ + +#define __XFS_HAS_FEAT(name, NAME) \ +static inline bool xfs_has_ ## name (const struct xfs_mount *mp) \ +{ \ + return mp->m_features & XFS_FEAT_ ## NAME; \ +} + +/* Some features can be added dynamically so they need a set wrapper, too. */ +#define __XFS_ADD_FEAT(name, NAME) \ + __XFS_HAS_FEAT(name, NAME); \ +static inline void xfs_add_ ## name (struct xfs_mount *mp) \ +{ \ + mp->m_features |= XFS_FEAT_ ## NAME; \ + xfs_sb_version_add ## name(&mp->m_sb); \ +} + +/* Superblock features */ +__XFS_ADD_FEAT(attr, ATTR) +__XFS_HAS_FEAT(nlink, NLINK) +__XFS_ADD_FEAT(quota, QUOTA) +__XFS_HAS_FEAT(dalign, DALIGN) +__XFS_HAS_FEAT(sector, SECTOR) +__XFS_HAS_FEAT(asciici, ASCIICI) +__XFS_HAS_FEAT(parent, PARENT) +__XFS_HAS_FEAT(ftype, FTYPE) +__XFS_HAS_FEAT(finobt, FINOBT) +__XFS_HAS_FEAT(rmapbt, RMAPBT) +__XFS_HAS_FEAT(reflink, REFLINK) +__XFS_HAS_FEAT(sparseinodes, SPINODES) +__XFS_HAS_FEAT(metauuid, META_UUID) +__XFS_HAS_FEAT(realtime, REALTIME) +__XFS_HAS_FEAT(inobtcounts, INOBTCNT) +__XFS_HAS_FEAT(bigtime, BIGTIME) +__XFS_HAS_FEAT(needsrepair, NEEDSREPAIR) +__XFS_HAS_FEAT(large_extent_counts, NREXT64) +__XFS_HAS_FEAT(exchange_range, EXCHANGE_RANGE) +__XFS_HAS_FEAT(metadir, METADIR) +__XFS_HAS_FEAT(zoned, ZONED) +__XFS_HAS_FEAT(nolifetime, NOLIFETIME) + +static inline bool xfs_has_rtgroups(const struct xfs_mount *mp) +{ + /* all metadir file systems also allow rtgroups */ + return xfs_has_metadir(mp); +} + +static inline bool xfs_has_rtsb(const struct xfs_mount *mp) +{ + /* all rtgroups filesystems with an rt section have an rtsb */ + return xfs_has_rtgroups(mp) && + xfs_has_realtime(mp) && + !xfs_has_zoned(mp); +} + +static inline bool xfs_has_rtrmapbt(const struct xfs_mount *mp) +{ + return xfs_has_rtgroups(mp) && xfs_has_realtime(mp) && + xfs_has_rmapbt(mp); +} + +static inline bool xfs_has_rtreflink(const struct xfs_mount *mp) +{ + return xfs_has_metadir(mp) && xfs_has_realtime(mp) && + xfs_has_reflink(mp); +} + +static inline bool xfs_has_nonzoned(const struct xfs_mount *mp) +{ + return !xfs_has_zoned(mp); +} + +static inline bool xfs_can_sw_atomic_write(struct xfs_mount *mp) +{ + return xfs_has_reflink(mp); +} + +/* + * Some features are always on for v5 file systems, allow the compiler to + * eliminiate dead code when building without v4 support. + */ +#define __XFS_HAS_V4_FEAT(name, NAME) \ +static inline bool xfs_has_ ## name (struct xfs_mount *mp) \ +{ \ + return !IS_ENABLED(CONFIG_XFS_SUPPORT_V4) || \ + (mp->m_features & XFS_FEAT_ ## NAME); \ +} + +#define __XFS_ADD_V4_FEAT(name, NAME) \ + __XFS_HAS_V4_FEAT(name, NAME); \ +static inline void xfs_add_ ## name (struct xfs_mount *mp) \ +{ \ + if (IS_ENABLED(CONFIG_XFS_SUPPORT_V4)) { \ + mp->m_features |= XFS_FEAT_ ## NAME; \ + xfs_sb_version_add ## name(&mp->m_sb); \ + } \ +} + +__XFS_HAS_V4_FEAT(align, ALIGN) +__XFS_HAS_V4_FEAT(logv2, LOGV2) +__XFS_HAS_V4_FEAT(extflg, EXTFLG) +__XFS_HAS_V4_FEAT(lazysbcount, LAZYSBCOUNT) +__XFS_ADD_V4_FEAT(projid32, PROJID32) +__XFS_HAS_V4_FEAT(v3inodes, V3INODES) +__XFS_HAS_V4_FEAT(crc, CRC) +__XFS_HAS_V4_FEAT(pquotino, PQUOTINO) + +static inline void xfs_add_attr2(struct xfs_mount *mp) +{ + if (IS_ENABLED(CONFIG_XFS_SUPPORT_V4)) + xfs_sb_version_addattr2(&mp->m_sb); +} + +/* + * Mount features + * + * These do not change dynamically - features that can come and go, such as 32 + * bit inodes and read-only state, are kept as operational state rather than + * features. + */ +__XFS_HAS_FEAT(noalign, NOALIGN) +__XFS_HAS_FEAT(allocsize, ALLOCSIZE) +__XFS_HAS_FEAT(large_iosize, LARGE_IOSIZE) +__XFS_HAS_FEAT(wsync, WSYNC) +__XFS_HAS_FEAT(dirsync, DIRSYNC) +__XFS_HAS_FEAT(discard, DISCARD) +__XFS_HAS_FEAT(grpid, GRPID) +__XFS_HAS_FEAT(small_inums, SMALL_INUMS) +__XFS_HAS_FEAT(swalloc, SWALLOC) +__XFS_HAS_FEAT(filestreams, FILESTREAMS) +__XFS_HAS_FEAT(dax_always, DAX_ALWAYS) +__XFS_HAS_FEAT(dax_never, DAX_NEVER) +__XFS_HAS_FEAT(norecovery, NORECOVERY) +__XFS_HAS_FEAT(nouuid, NOUUID) + +/* + * Operational mount state flags + * + * Use these with atomic bit ops only! + */ +#define XFS_OPSTATE_UNMOUNTING 0 /* filesystem is unmounting */ +#define XFS_OPSTATE_CLEAN 1 /* mount was clean */ +#define XFS_OPSTATE_SHUTDOWN 2 /* stop all fs operations */ +#define XFS_OPSTATE_INODE32 3 /* inode32 allocator active */ +#define XFS_OPSTATE_READONLY 4 /* read-only fs */ + +/* + * If set, inactivation worker threads will be scheduled to process queued + * inodegc work. If not, queued inodes remain in memory waiting to be + * processed. + */ +#define XFS_OPSTATE_INODEGC_ENABLED 5 +/* + * If set, background speculative prealloc gc worker threads will be scheduled + * to process queued blockgc work. If not, inodes retain their preallocations + * until explicitly deleted. + */ +#define XFS_OPSTATE_BLOCKGC_ENABLED 6 + +/* Kernel has logged a warning about shrink being used on this fs. */ +#define XFS_OPSTATE_WARNED_SHRINK 9 +/* Kernel has logged a warning about logged xattr updates being used. */ +#define XFS_OPSTATE_WARNED_LARP 10 +/* Mount time quotacheck is running */ +#define XFS_OPSTATE_QUOTACHECK_RUNNING 11 +/* Do we want to clear log incompat flags? */ +#define XFS_OPSTATE_UNSET_LOG_INCOMPAT 12 +/* Filesystem can use logged extended attributes */ +#define XFS_OPSTATE_USE_LARP 13 +/* Kernel has logged a warning about blocksize > pagesize on this fs. */ +#define XFS_OPSTATE_WARNED_LBS 14 +/* Kernel has logged a warning about metadata dirs being used on this fs. */ +#define XFS_OPSTATE_WARNED_METADIR 17 +/* Filesystem should use qflags to determine quotaon status */ +#define XFS_OPSTATE_RESUMING_QUOTAON 18 +/* (Zoned) GC is in progress */ +#define XFS_OPSTATE_ZONEGC_RUNNING 20 + +#define __XFS_IS_OPSTATE(name, NAME) \ +static inline bool xfs_is_ ## name (struct xfs_mount *mp) \ +{ \ + return test_bit(XFS_OPSTATE_ ## NAME, &mp->m_opstate); \ +} \ +static inline bool xfs_clear_ ## name (struct xfs_mount *mp) \ +{ \ + return test_and_clear_bit(XFS_OPSTATE_ ## NAME, &mp->m_opstate); \ +} \ +static inline bool xfs_set_ ## name (struct xfs_mount *mp) \ +{ \ + return test_and_set_bit(XFS_OPSTATE_ ## NAME, &mp->m_opstate); \ +} + +__XFS_IS_OPSTATE(unmounting, UNMOUNTING) +__XFS_IS_OPSTATE(clean, CLEAN) +__XFS_IS_OPSTATE(shutdown, SHUTDOWN) +__XFS_IS_OPSTATE(inode32, INODE32) +__XFS_IS_OPSTATE(readonly, READONLY) +__XFS_IS_OPSTATE(inodegc_enabled, INODEGC_ENABLED) +__XFS_IS_OPSTATE(blockgc_enabled, BLOCKGC_ENABLED) +#ifdef CONFIG_XFS_QUOTA +__XFS_IS_OPSTATE(quotacheck_running, QUOTACHECK_RUNNING) +__XFS_IS_OPSTATE(resuming_quotaon, RESUMING_QUOTAON) +#else +static inline bool xfs_is_quotacheck_running(struct xfs_mount *mp) +{ + return false; +} +static inline bool xfs_is_resuming_quotaon(struct xfs_mount *mp) +{ + return false; +} +static inline void xfs_set_resuming_quotaon(struct xfs_mount *m) +{ +} +static inline bool xfs_clear_resuming_quotaon(struct xfs_mount *mp) +{ + return false; +} +#endif /* CONFIG_XFS_QUOTA */ +__XFS_IS_OPSTATE(done_with_log_incompat, UNSET_LOG_INCOMPAT) +__XFS_IS_OPSTATE(using_logged_xattrs, USE_LARP) +__XFS_IS_OPSTATE(zonegc_running, ZONEGC_RUNNING) + +static inline bool +xfs_should_warn(struct xfs_mount *mp, long nr) +{ + return !test_and_set_bit(nr, &mp->m_opstate); +} + +#define XFS_OPSTATE_STRINGS \ + { (1UL << XFS_OPSTATE_UNMOUNTING), "unmounting" }, \ + { (1UL << XFS_OPSTATE_CLEAN), "clean" }, \ + { (1UL << XFS_OPSTATE_SHUTDOWN), "shutdown" }, \ + { (1UL << XFS_OPSTATE_INODE32), "inode32" }, \ + { (1UL << XFS_OPSTATE_READONLY), "read_only" }, \ + { (1UL << XFS_OPSTATE_INODEGC_ENABLED), "inodegc" }, \ + { (1UL << XFS_OPSTATE_BLOCKGC_ENABLED), "blockgc" }, \ + { (1UL << XFS_OPSTATE_WARNED_SHRINK), "wshrink" }, \ + { (1UL << XFS_OPSTATE_WARNED_LARP), "wlarp" }, \ + { (1UL << XFS_OPSTATE_QUOTACHECK_RUNNING), "quotacheck" }, \ + { (1UL << XFS_OPSTATE_UNSET_LOG_INCOMPAT), "unset_log_incompat" }, \ + { (1UL << XFS_OPSTATE_USE_LARP), "logged_xattrs" } + +/* + * Max and min values for mount-option defined I/O + * preallocation sizes. + */ +#define XFS_MAX_IO_LOG 30 /* 1G */ +#define XFS_MIN_IO_LOG PAGE_SHIFT + +void xfs_do_force_shutdown(struct xfs_mount *mp, uint32_t flags, char *fname, + int lnnum); +#define xfs_force_shutdown(m,f) \ + xfs_do_force_shutdown(m, f, __FILE__, __LINE__) + +#define SHUTDOWN_META_IO_ERROR (1u << 0) /* write attempt to metadata failed */ +#define SHUTDOWN_LOG_IO_ERROR (1u << 1) /* write attempt to the log failed */ +#define SHUTDOWN_FORCE_UMOUNT (1u << 2) /* shutdown from a forced unmount */ +#define SHUTDOWN_CORRUPT_INCORE (1u << 3) /* corrupt in-memory structures */ +#define SHUTDOWN_CORRUPT_ONDISK (1u << 4) /* corrupt metadata on device */ +#define SHUTDOWN_DEVICE_REMOVED (1u << 5) /* device removed underneath us */ + +#define XFS_SHUTDOWN_STRINGS \ + { SHUTDOWN_META_IO_ERROR, "metadata_io" }, \ + { SHUTDOWN_LOG_IO_ERROR, "log_io" }, \ + { SHUTDOWN_FORCE_UMOUNT, "force_umount" }, \ + { SHUTDOWN_CORRUPT_INCORE, "corruption" }, \ + { SHUTDOWN_DEVICE_REMOVED, "device_removed" } + +/* + * Flags for xfs_mountfs + */ +#define XFS_MFSI_QUIET 0x40 /* Be silent if mount errors found */ + +static inline xfs_agnumber_t +xfs_daddr_to_agno(struct xfs_mount *mp, xfs_daddr_t d) +{ + xfs_rfsblock_t ld = XFS_BB_TO_FSBT(mp, d); + do_div(ld, mp->m_sb.sb_agblocks); + return (xfs_agnumber_t) ld; +} + +static inline xfs_agblock_t +xfs_daddr_to_agbno(struct xfs_mount *mp, xfs_daddr_t d) +{ + xfs_rfsblock_t ld = XFS_BB_TO_FSBT(mp, d); + return (xfs_agblock_t) do_div(ld, mp->m_sb.sb_agblocks); +} + +extern void xfs_uuid_table_free(void); +uint64_t xfs_default_resblks(struct xfs_mount *mp, + enum xfs_free_counter ctr); +extern int xfs_mountfs(xfs_mount_t *mp); +extern void xfs_unmountfs(xfs_mount_t *); + +/* + * Deltas for the block count can vary from 1 to very large, but lock contention + * only occurs on frequent small block count updates such as in the delayed + * allocation path for buffered writes (page a time updates). Hence we set + * a large batch count (1024) to minimise global counter updates except when + * we get near to ENOSPC and we have to be very accurate with our updates. + */ +#define XFS_FDBLOCKS_BATCH 1024 + +uint64_t xfs_freecounter_unavailable(struct xfs_mount *mp, + enum xfs_free_counter ctr); + +/* + * Sum up the freecount, but never return negative values. + */ +static inline s64 xfs_sum_freecounter(struct xfs_mount *mp, + enum xfs_free_counter ctr) +{ + return percpu_counter_sum_positive(&mp->m_free[ctr].count); +} + +/* + * Same as above, but does return negative values. Mostly useful for + * special cases like repair and tracing. + */ +static inline s64 xfs_sum_freecounter_raw(struct xfs_mount *mp, + enum xfs_free_counter ctr) +{ + return percpu_counter_sum(&mp->m_free[ctr].count); +} + +/* + * This just provides and estimate without the cpu-local updates, use + * xfs_sum_freecounter for the exact value. + */ +static inline s64 xfs_estimate_freecounter(struct xfs_mount *mp, + enum xfs_free_counter ctr) +{ + return percpu_counter_read_positive(&mp->m_free[ctr].count); +} + +static inline int xfs_compare_freecounter(struct xfs_mount *mp, + enum xfs_free_counter ctr, s64 rhs, s32 batch) +{ + return __percpu_counter_compare(&mp->m_free[ctr].count, rhs, batch); +} + +static inline void xfs_set_freecounter(struct xfs_mount *mp, + enum xfs_free_counter ctr, uint64_t val) +{ + percpu_counter_set(&mp->m_free[ctr].count, val); +} + +int xfs_dec_freecounter(struct xfs_mount *mp, enum xfs_free_counter ctr, + uint64_t delta, bool rsvd); +void xfs_add_freecounter(struct xfs_mount *mp, enum xfs_free_counter ctr, + uint64_t delta); + +static inline int xfs_dec_fdblocks(struct xfs_mount *mp, uint64_t delta, + bool reserved) +{ + return xfs_dec_freecounter(mp, XC_FREE_BLOCKS, delta, reserved); +} + +static inline void xfs_add_fdblocks(struct xfs_mount *mp, uint64_t delta) +{ + xfs_add_freecounter(mp, XC_FREE_BLOCKS, delta); +} + +static inline int xfs_dec_frextents(struct xfs_mount *mp, uint64_t delta) +{ + return xfs_dec_freecounter(mp, XC_FREE_RTEXTENTS, delta, false); +} + +static inline void xfs_add_frextents(struct xfs_mount *mp, uint64_t delta) +{ + xfs_add_freecounter(mp, XC_FREE_RTEXTENTS, delta); +} + +extern int xfs_readsb(xfs_mount_t *, int); +extern void xfs_freesb(xfs_mount_t *); +extern bool xfs_fs_writable(struct xfs_mount *mp, int level); +extern int xfs_sb_validate_fsb_count(struct xfs_sb *, uint64_t); + +extern int xfs_dev_is_read_only(struct xfs_mount *, char *); + +extern void xfs_set_low_space_thresholds(struct xfs_mount *); + +int xfs_zero_extent(struct xfs_inode *ip, xfs_fsblock_t start_fsb, + xfs_off_t count_fsb); + +struct xfs_error_cfg * xfs_error_get_cfg(struct xfs_mount *mp, + int error_class, int error); +void xfs_force_summary_recalc(struct xfs_mount *mp); +int xfs_add_incompat_log_feature(struct xfs_mount *mp, uint32_t feature); +bool xfs_clear_incompat_log_features(struct xfs_mount *mp); +void xfs_mod_delalloc(struct xfs_inode *ip, int64_t data_delta, + int64_t ind_delta); +static inline void xfs_mod_sb_delalloc(struct xfs_mount *mp, int64_t delta) +{ + percpu_counter_add(&mp->m_delalloc_blks, delta); +} + +int xfs_set_max_atomic_write_opt(struct xfs_mount *mp, + unsigned long long new_max_bytes); + +static inline struct xfs_buftarg * +xfs_group_type_buftarg( + struct xfs_mount *mp, + enum xfs_group_type type) +{ + switch (type) { + case XG_TYPE_AG: + return mp->m_ddev_targp; + case XG_TYPE_RTG: + return mp->m_rtdev_targp; + default: + ASSERT(0); + break; + } + return NULL; +} + +#endif /* __XFS_MOUNT_H__ */ diff --git a/fs/xfs/xfs_mru_cache.c b/fs/xfs/xfs_mru_cache.c new file mode 100644 index 000000000..d61ec8cb1 --- /dev/null +++ b/fs/xfs/xfs_mru_cache.c @@ -0,0 +1,541 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2006-2007 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_mru_cache.h" + +/* + * The MRU Cache data structure consists of a data store, an array of lists and + * a lock to protect its internal state. At initialisation time, the client + * supplies an element lifetime in milliseconds and a group count, as well as a + * function pointer to call when deleting elements. A data structure for + * queueing up work in the form of timed callbacks is also included. + * + * The group count controls how many lists are created, and thereby how finely + * the elements are grouped in time. When reaping occurs, all the elements in + * all the lists whose time has expired are deleted. + * + * To give an example of how this works in practice, consider a client that + * initialises an MRU Cache with a lifetime of ten seconds and a group count of + * five. Five internal lists will be created, each representing a two second + * period in time. When the first element is added, time zero for the data + * structure is initialised to the current time. + * + * All the elements added in the first two seconds are appended to the first + * list. Elements added in the third second go into the second list, and so on. + * If an element is accessed at any point, it is removed from its list and + * inserted at the head of the current most-recently-used list. + * + * The reaper function will have nothing to do until at least twelve seconds + * have elapsed since the first element was added. The reason for this is that + * if it were called at t=11s, there could be elements in the first list that + * have only been inactive for nine seconds, so it still does nothing. If it is + * called anywhere between t=12 and t=14 seconds, it will delete all the + * elements that remain in the first list. It's therefore possible for elements + * to remain in the data store even after they've been inactive for up to + * (t + t/g) seconds, where t is the inactive element lifetime and g is the + * number of groups. + * + * The above example assumes that the reaper function gets called at least once + * every (t/g) seconds. If it is called less frequently, unused elements will + * accumulate in the reap list until the reaper function is eventually called. + * The current implementation uses work queue callbacks to carefully time the + * reaper function calls, so this should happen rarely, if at all. + * + * From a design perspective, the primary reason for the choice of a list array + * representing discrete time intervals is that it's only practical to reap + * expired elements in groups of some appreciable size. This automatically + * introduces a granularity to element lifetimes, so there's no point storing an + * individual timeout with each element that specifies a more precise reap time. + * The bonus is a saving of sizeof(long) bytes of memory per element stored. + * + * The elements could have been stored in just one list, but an array of + * counters or pointers would need to be maintained to allow them to be divided + * up into discrete time groups. More critically, the process of touching or + * removing an element would involve walking large portions of the entire list, + * which would have a detrimental effect on performance. The additional memory + * requirement for the array of list heads is minimal. + * + * When an element is touched or deleted, it needs to be removed from its + * current list. Doubly linked lists are used to make the list maintenance + * portion of these operations O(1). Since reaper timing can be imprecise, + * inserts and lookups can occur when there are no free lists available. When + * this happens, all the elements on the LRU list need to be migrated to the end + * of the reap list. To keep the list maintenance portion of these operations + * O(1) also, list tails need to be accessible without walking the entire list. + * This is the reason why doubly linked list heads are used. + */ + +/* + * An MRU Cache is a dynamic data structure that stores its elements in a way + * that allows efficient lookups, but also groups them into discrete time + * intervals based on insertion time. This allows elements to be efficiently + * and automatically reaped after a fixed period of inactivity. + * + * When a client data pointer is stored in the MRU Cache it needs to be added to + * both the data store and to one of the lists. It must also be possible to + * access each of these entries via the other, i.e. to: + * + * a) Walk a list, removing the corresponding data store entry for each item. + * b) Look up a data store entry, then access its list entry directly. + * + * To achieve both of these goals, each entry must contain both a list entry and + * a key, in addition to the user's data pointer. Note that it's not a good + * idea to have the client embed one of these structures at the top of their own + * data structure, because inserting the same item more than once would most + * likely result in a loop in one of the lists. That's a sure-fire recipe for + * an infinite loop in the code. + */ +struct xfs_mru_cache { + struct radix_tree_root store; /* Core storage data structure. */ + struct list_head *lists; /* Array of lists, one per grp. */ + struct list_head reap_list; /* Elements overdue for reaping. */ + spinlock_t lock; /* Lock to protect this struct. */ + unsigned int grp_count; /* Number of discrete groups. */ + unsigned int grp_time; /* Time period spanned by grps. */ + unsigned int lru_grp; /* Group containing time zero. */ + unsigned long time_zero; /* Time first element was added. */ + xfs_mru_cache_free_func_t free_func; /* Function pointer for freeing. */ + struct delayed_work work; /* Workqueue data for reaping. */ + unsigned int queued; /* work has been queued */ + void *data; +}; + +static struct workqueue_struct *xfs_mru_reap_wq; + +/* + * When inserting, destroying or reaping, it's first necessary to update the + * lists relative to a particular time. In the case of destroying, that time + * will be well in the future to ensure that all items are moved to the reap + * list. In all other cases though, the time will be the current time. + * + * This function enters a loop, moving the contents of the LRU list to the reap + * list again and again until either a) the lists are all empty, or b) time zero + * has been advanced sufficiently to be within the immediate element lifetime. + * + * Case a) above is detected by counting how many groups are migrated and + * stopping when they've all been moved. Case b) is detected by monitoring the + * time_zero field, which is updated as each group is migrated. + * + * The return value is the earliest time that more migration could be needed, or + * zero if there's no need to schedule more work because the lists are empty. + */ +STATIC unsigned long +_xfs_mru_cache_migrate( + struct xfs_mru_cache *mru, + unsigned long now) +{ + unsigned int grp; + unsigned int migrated = 0; + struct list_head *lru_list; + + /* Nothing to do if the data store is empty. */ + if (!mru->time_zero) + return 0; + + /* While time zero is older than the time spanned by all the lists. */ + while (mru->time_zero <= now - mru->grp_count * mru->grp_time) { + + /* + * If the LRU list isn't empty, migrate its elements to the tail + * of the reap list. + */ + lru_list = mru->lists + mru->lru_grp; + if (!list_empty(lru_list)) + list_splice_init(lru_list, mru->reap_list.prev); + + /* + * Advance the LRU group number, freeing the old LRU list to + * become the new MRU list; advance time zero accordingly. + */ + mru->lru_grp = (mru->lru_grp + 1) % mru->grp_count; + mru->time_zero += mru->grp_time; + + /* + * If reaping is so far behind that all the elements on all the + * lists have been migrated to the reap list, it's now empty. + */ + if (++migrated == mru->grp_count) { + mru->lru_grp = 0; + mru->time_zero = 0; + return 0; + } + } + + /* Find the first non-empty list from the LRU end. */ + for (grp = 0; grp < mru->grp_count; grp++) { + + /* Check the grp'th list from the LRU end. */ + lru_list = mru->lists + ((mru->lru_grp + grp) % mru->grp_count); + if (!list_empty(lru_list)) + return mru->time_zero + + (mru->grp_count + grp) * mru->grp_time; + } + + /* All the lists must be empty. */ + mru->lru_grp = 0; + mru->time_zero = 0; + return 0; +} + +/* + * When inserting or doing a lookup, an element needs to be inserted into the + * MRU list. The lists must be migrated first to ensure that they're + * up-to-date, otherwise the new element could be given a shorter lifetime in + * the cache than it should. + */ +STATIC void +_xfs_mru_cache_list_insert( + struct xfs_mru_cache *mru, + struct xfs_mru_cache_elem *elem) +{ + unsigned int grp = 0; + unsigned long now = jiffies; + + /* + * If the data store is empty, initialise time zero, leave grp set to + * zero and start the work queue timer if necessary. Otherwise, set grp + * to the number of group times that have elapsed since time zero. + */ + if (!_xfs_mru_cache_migrate(mru, now)) { + mru->time_zero = now; + if (!mru->queued) { + mru->queued = 1; + queue_delayed_work(xfs_mru_reap_wq, &mru->work, + mru->grp_count * mru->grp_time); + } + } else { + grp = (now - mru->time_zero) / mru->grp_time; + grp = (mru->lru_grp + grp) % mru->grp_count; + } + + /* Insert the element at the tail of the corresponding list. */ + list_add_tail(&elem->list_node, mru->lists + grp); +} + +/* + * When destroying or reaping, all the elements that were migrated to the reap + * list need to be deleted. For each element this involves removing it from the + * data store, removing it from the reap list, calling the client's free + * function and deleting the element from the element cache. + * + * We get called holding the mru->lock, which we drop and then reacquire. + * Sparse need special help with this to tell it we know what we are doing. + */ +STATIC void +_xfs_mru_cache_clear_reap_list( + struct xfs_mru_cache *mru) + __releases(mru->lock) __acquires(mru->lock) +{ + struct xfs_mru_cache_elem *elem, *next; + LIST_HEAD(tmp); + + list_for_each_entry_safe(elem, next, &mru->reap_list, list_node) { + + /* Remove the element from the data store. */ + radix_tree_delete(&mru->store, elem->key); + + /* + * remove to temp list so it can be freed without + * needing to hold the lock + */ + list_move(&elem->list_node, &tmp); + } + spin_unlock(&mru->lock); + + list_for_each_entry_safe(elem, next, &tmp, list_node) { + list_del_init(&elem->list_node); + mru->free_func(mru->data, elem); + } + + spin_lock(&mru->lock); +} + +/* + * We fire the reap timer every group expiry interval so + * we always have a reaper ready to run. This makes shutdown + * and flushing of the reaper easy to do. Hence we need to + * keep when the next reap must occur so we can determine + * at each interval whether there is anything we need to do. + */ +STATIC void +_xfs_mru_cache_reap( + struct work_struct *work) +{ + struct xfs_mru_cache *mru = + container_of(work, struct xfs_mru_cache, work.work); + unsigned long now, next; + + ASSERT(mru && mru->lists); + if (!mru || !mru->lists) + return; + + spin_lock(&mru->lock); + next = _xfs_mru_cache_migrate(mru, jiffies); + _xfs_mru_cache_clear_reap_list(mru); + + mru->queued = next; + if ((mru->queued > 0)) { + now = jiffies; + if (next <= now) + next = 0; + else + next -= now; + queue_delayed_work(xfs_mru_reap_wq, &mru->work, next); + } + + spin_unlock(&mru->lock); +} + +int +xfs_mru_cache_init(void) +{ + xfs_mru_reap_wq = alloc_workqueue("xfs_mru_cache", + XFS_WQFLAGS(WQ_MEM_RECLAIM | WQ_FREEZABLE | WQ_PERCPU), + 1); + if (!xfs_mru_reap_wq) + return -ENOMEM; + return 0; +} + +void +xfs_mru_cache_uninit(void) +{ + destroy_workqueue(xfs_mru_reap_wq); +} + +/* + * To initialise a struct xfs_mru_cache pointer, call xfs_mru_cache_create() + * with the address of the pointer, a lifetime value in milliseconds, a group + * count and a free function to use when deleting elements. This function + * returns 0 if the initialisation was successful. + */ +int +xfs_mru_cache_create( + struct xfs_mru_cache **mrup, + void *data, + unsigned int lifetime_ms, + unsigned int grp_count, + xfs_mru_cache_free_func_t free_func) +{ + struct xfs_mru_cache *mru = NULL; + int grp; + unsigned int grp_time; + + if (mrup) + *mrup = NULL; + + if (!mrup || !grp_count || !lifetime_ms || !free_func) + return -EINVAL; + + if (!(grp_time = msecs_to_jiffies(lifetime_ms) / grp_count)) + return -EINVAL; + + mru = kzalloc_obj(*mru, GFP_KERNEL | __GFP_NOFAIL); + if (!mru) + return -ENOMEM; + + /* An extra list is needed to avoid reaping up to a grp_time early. */ + mru->grp_count = grp_count + 1; + mru->lists = kzalloc(mru->grp_count * sizeof(*mru->lists), + GFP_KERNEL | __GFP_NOFAIL); + if (!mru->lists) { + kfree(mru); + return -ENOMEM; + } + + for (grp = 0; grp < mru->grp_count; grp++) + INIT_LIST_HEAD(mru->lists + grp); + + /* + * We use GFP_KERNEL radix tree preload and do inserts under a + * spinlock so GFP_ATOMIC is appropriate for the radix tree itself. + */ + INIT_RADIX_TREE(&mru->store, GFP_ATOMIC); + INIT_LIST_HEAD(&mru->reap_list); + spin_lock_init(&mru->lock); + INIT_DELAYED_WORK(&mru->work, _xfs_mru_cache_reap); + + mru->grp_time = grp_time; + mru->free_func = free_func; + mru->data = data; + *mrup = mru; + return 0; +} + +/* + * Call xfs_mru_cache_flush() to flush out all cached entries, calling their + * free functions as they're deleted. When this function returns, the caller is + * guaranteed that all the free functions for all the elements have finished + * executing and the reaper is not running. + */ +static void +xfs_mru_cache_flush( + struct xfs_mru_cache *mru) +{ + if (!mru || !mru->lists) + return; + + spin_lock(&mru->lock); + if (mru->queued) { + spin_unlock(&mru->lock); + cancel_delayed_work_sync(&mru->work); + spin_lock(&mru->lock); + } + + _xfs_mru_cache_migrate(mru, jiffies + mru->grp_count * mru->grp_time); + _xfs_mru_cache_clear_reap_list(mru); + + spin_unlock(&mru->lock); +} + +void +xfs_mru_cache_destroy( + struct xfs_mru_cache *mru) +{ + if (!mru || !mru->lists) + return; + + xfs_mru_cache_flush(mru); + + kfree(mru->lists); + kfree(mru); +} + +/* + * To insert an element, call xfs_mru_cache_insert() with the data store, the + * element's key and the client data pointer. This function returns 0 on + * success or ENOMEM if memory for the data element couldn't be allocated. + * + * The passed in elem is freed through the per-cache free_func on failure. + */ +int +xfs_mru_cache_insert( + struct xfs_mru_cache *mru, + unsigned long key, + struct xfs_mru_cache_elem *elem) +{ + int error = -EINVAL; + + error = -ENOMEM; + if (radix_tree_preload(GFP_KERNEL)) + goto out_free; + + INIT_LIST_HEAD(&elem->list_node); + elem->key = key; + + spin_lock(&mru->lock); + error = radix_tree_insert(&mru->store, key, elem); + radix_tree_preload_end(); + if (!error) + _xfs_mru_cache_list_insert(mru, elem); + spin_unlock(&mru->lock); + + if (error) + goto out_free; + return 0; + +out_free: + mru->free_func(mru->data, elem); + return error; +} + +/* + * To remove an element without calling the free function, call + * xfs_mru_cache_remove() with the data store and the element's key. On success + * the client data pointer for the removed element is returned, otherwise this + * function will return a NULL pointer. + */ +struct xfs_mru_cache_elem * +xfs_mru_cache_remove( + struct xfs_mru_cache *mru, + unsigned long key) +{ + struct xfs_mru_cache_elem *elem; + + ASSERT(mru && mru->lists); + if (!mru || !mru->lists) + return NULL; + + spin_lock(&mru->lock); + elem = radix_tree_delete(&mru->store, key); + if (elem) + list_del(&elem->list_node); + spin_unlock(&mru->lock); + + return elem; +} + +/* + * To remove and element and call the free function, call xfs_mru_cache_delete() + * with the data store and the element's key. + */ +void +xfs_mru_cache_delete( + struct xfs_mru_cache *mru, + unsigned long key) +{ + struct xfs_mru_cache_elem *elem; + + elem = xfs_mru_cache_remove(mru, key); + if (elem) + mru->free_func(mru->data, elem); +} + +/* + * To look up an element using its key, call xfs_mru_cache_lookup() with the + * data store and the element's key. If found, the element will be moved to the + * head of the MRU list to indicate that it's been touched. + * + * The internal data structures are protected by a spinlock that is STILL HELD + * when this function returns. Call xfs_mru_cache_done() to release it. Note + * that it is not safe to call any function that might sleep in the interim. + * + * The implementation could have used reference counting to avoid this + * restriction, but since most clients simply want to get, set or test a member + * of the returned data structure, the extra per-element memory isn't warranted. + * + * If the element isn't found, this function returns NULL and the spinlock is + * released. xfs_mru_cache_done() should NOT be called when this occurs. + * + * Because sparse isn't smart enough to know about conditional lock return + * status, we need to help it get it right by annotating the path that does + * not release the lock. + */ +struct xfs_mru_cache_elem * +xfs_mru_cache_lookup( + struct xfs_mru_cache *mru, + unsigned long key) +{ + struct xfs_mru_cache_elem *elem; + + ASSERT(mru && mru->lists); + if (!mru || !mru->lists) + return NULL; + + spin_lock(&mru->lock); + elem = radix_tree_lookup(&mru->store, key); + if (elem) { + list_del(&elem->list_node); + _xfs_mru_cache_list_insert(mru, elem); + __release(mru_lock); /* help sparse not be stupid */ + } else + spin_unlock(&mru->lock); + + return elem; +} + +/* + * To release the internal data structure spinlock after having performed an + * xfs_mru_cache_lookup() or an xfs_mru_cache_peek(), call xfs_mru_cache_done() + * with the data store pointer. + */ +void +xfs_mru_cache_done( + struct xfs_mru_cache *mru) + __releases(mru->lock) +{ + spin_unlock(&mru->lock); +} diff --git a/fs/xfs/xfs_mru_cache.h b/fs/xfs/xfs_mru_cache.h new file mode 100644 index 000000000..f1fde1ecf --- /dev/null +++ b/fs/xfs/xfs_mru_cache.h @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2006-2007 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_MRU_CACHE_H__ +#define __XFS_MRU_CACHE_H__ + +struct xfs_mru_cache; + +struct xfs_mru_cache_elem { + struct list_head list_node; + unsigned long key; +}; + +/* Function pointer type for callback to free a client's data pointer. */ +typedef void (*xfs_mru_cache_free_func_t)(void *, struct xfs_mru_cache_elem *); + +int xfs_mru_cache_init(void); +void xfs_mru_cache_uninit(void); +int xfs_mru_cache_create(struct xfs_mru_cache **mrup, void *data, + unsigned int lifetime_ms, unsigned int grp_count, + xfs_mru_cache_free_func_t free_func); +void xfs_mru_cache_destroy(struct xfs_mru_cache *mru); +int xfs_mru_cache_insert(struct xfs_mru_cache *mru, unsigned long key, + struct xfs_mru_cache_elem *elem); +struct xfs_mru_cache_elem * +xfs_mru_cache_remove(struct xfs_mru_cache *mru, unsigned long key); +void xfs_mru_cache_delete(struct xfs_mru_cache *mru, unsigned long key); +struct xfs_mru_cache_elem * +xfs_mru_cache_lookup(struct xfs_mru_cache *mru, unsigned long key); +void xfs_mru_cache_done(struct xfs_mru_cache *mru); + +#endif /* __XFS_MRU_CACHE_H__ */ diff --git a/fs/xfs/xfs_notify_failure.c b/fs/xfs/xfs_notify_failure.c new file mode 100644 index 000000000..b994ff15d --- /dev/null +++ b/fs/xfs/xfs_notify_failure.c @@ -0,0 +1,397 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2022 Fujitsu. All Rights Reserved. + */ + +#include "xfs_platform.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_alloc.h" +#include "xfs_bit.h" +#include "xfs_btree.h" +#include "xfs_inode.h" +#include "xfs_icache.h" +#include "xfs_rmap.h" +#include "xfs_rmap_btree.h" +#include "xfs_rtalloc.h" +#include "xfs_trans.h" +#include "xfs_ag.h" +#include "xfs_notify_failure.h" +#include "xfs_rtgroup.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_healthmon.h" + +#include +#include +#include +#include + +struct xfs_failure_info { + xfs_agblock_t startblock; + xfs_extlen_t blockcount; + int mf_flags; + bool want_shutdown; +}; + +static pgoff_t +xfs_failure_pgoff( + struct xfs_mount *mp, + const struct xfs_rmap_irec *rec, + const struct xfs_failure_info *notify) +{ + loff_t pos = XFS_FSB_TO_B(mp, rec->rm_offset); + + if (notify->startblock > rec->rm_startblock) + pos += XFS_FSB_TO_B(mp, + notify->startblock - rec->rm_startblock); + return pos >> PAGE_SHIFT; +} + +static unsigned long +xfs_failure_pgcnt( + struct xfs_mount *mp, + const struct xfs_rmap_irec *rec, + const struct xfs_failure_info *notify) +{ + xfs_agblock_t end_rec; + xfs_agblock_t end_notify; + xfs_agblock_t start_cross; + xfs_agblock_t end_cross; + + start_cross = max(rec->rm_startblock, notify->startblock); + + end_rec = rec->rm_startblock + rec->rm_blockcount; + end_notify = notify->startblock + notify->blockcount; + end_cross = min(end_rec, end_notify); + + return XFS_FSB_TO_B(mp, end_cross - start_cross) >> PAGE_SHIFT; +} + +static int +xfs_dax_failure_fn( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *data) +{ + struct xfs_mount *mp = cur->bc_mp; + struct xfs_inode *ip; + struct xfs_failure_info *notify = data; + struct address_space *mapping; + pgoff_t pgoff; + unsigned long pgcnt; + int error = 0; + + if (XFS_RMAP_NON_INODE_OWNER(rec->rm_owner) || + (rec->rm_flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK))) { + /* Continue the query because this isn't a failure. */ + if (notify->mf_flags & MF_MEM_PRE_REMOVE) + return 0; + notify->want_shutdown = true; + return 0; + } + + /* Get files that incore, filter out others that are not in use. */ + error = xfs_iget(mp, cur->bc_tp, rec->rm_owner, XFS_IGET_INCORE, + 0, &ip); + /* Continue the rmap query if the inode isn't incore */ + if (error == -ENODATA) + return 0; + if (error) { + notify->want_shutdown = true; + return 0; + } + + mapping = VFS_I(ip)->i_mapping; + pgoff = xfs_failure_pgoff(mp, rec, notify); + pgcnt = xfs_failure_pgcnt(mp, rec, notify); + + /* Continue the rmap query if the inode isn't a dax file. */ + if (dax_mapping(mapping)) + error = mf_dax_kill_procs(mapping, pgoff, pgcnt, + notify->mf_flags); + + /* Invalidate the cache in dax pages. */ + if (notify->mf_flags & MF_MEM_PRE_REMOVE) + invalidate_inode_pages2_range(mapping, pgoff, + pgoff + pgcnt - 1); + + fserror_report_data_lost(VFS_I(ip), (u64)pgoff << PAGE_SHIFT, + (u64)pgcnt << PAGE_SHIFT, GFP_NOFS); + + xfs_irele(ip); + return error; +} + +static int +xfs_dax_notify_failure_freeze( + struct xfs_mount *mp) +{ + struct super_block *sb = mp->m_super; + int error; + + error = freeze_super(sb, FREEZE_HOLDER_KERNEL, NULL); + if (error) + xfs_emerg(mp, "already frozen by kernel, err=%d", error); + + return error; +} + +static void +xfs_dax_notify_failure_thaw( + struct xfs_mount *mp, + bool kernel_frozen) +{ + struct super_block *sb = mp->m_super; + int error; + + if (kernel_frozen) { + error = thaw_super(sb, FREEZE_HOLDER_KERNEL, NULL); + if (error) + xfs_emerg(mp, "still frozen after notify failure, err=%d", + error); + } + + /* + * Also thaw userspace call anyway because the device is about to be + * removed immediately. + */ + thaw_super(sb, FREEZE_HOLDER_USERSPACE, NULL); +} + +static int +xfs_dax_translate_range( + struct xfs_buftarg *btp, + u64 offset, + u64 len, + xfs_daddr_t *daddr, + uint64_t *bblen) +{ + u64 dev_start = btp->bt_dax_part_off; + u64 dev_len = BBTOB(btp->bt_nr_sectors); + u64 dev_end = dev_start + dev_len - 1; + + /* Notify failure on the whole device. */ + if (offset == 0 && len == U64_MAX) { + offset = dev_start; + len = dev_len; + } + + /* Ignore the range out of filesystem area */ + if (offset + len - 1 < dev_start) + return -ENXIO; + if (offset > dev_end) + return -ENXIO; + + /* Calculate the real range when it touches the boundary */ + if (offset > dev_start) + offset -= dev_start; + else { + len -= dev_start - offset; + offset = 0; + } + if (offset + len - 1 > dev_end) + len = dev_end - offset + 1; + + *daddr = BTOBB(offset); + *bblen = BTOBB(len); + return 0; +} + +static int +xfs_dax_notify_logdev_failure( + struct xfs_mount *mp, + u64 offset, + u64 len, + int mf_flags) +{ + xfs_daddr_t daddr; + uint64_t bblen; + int error; + + /* + * Return ENXIO instead of shutting down the filesystem if the failed + * region is beyond the end of the log. + */ + error = xfs_dax_translate_range(mp->m_logdev_targp, + offset, len, &daddr, &bblen); + if (error) + return error; + + xfs_healthmon_report_media(mp, XFS_DEV_LOG, daddr, bblen); + + /* + * In the pre-remove case the failure notification is attempting to + * trigger a force unmount. The expectation is that the device is + * still present, but its removal is in progress and can not be + * cancelled, proceed with accessing the log device. + */ + if (mf_flags & MF_MEM_PRE_REMOVE) + return 0; + + xfs_err(mp, "ondisk log corrupt, shutting down fs!"); + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_ONDISK); + return -EFSCORRUPTED; +} + +static int +xfs_dax_notify_dev_failure( + struct xfs_mount *mp, + u64 offset, + u64 len, + int mf_flags, + enum xfs_group_type type) +{ + struct xfs_failure_info notify = { .mf_flags = mf_flags }; + struct xfs_trans *tp = NULL; + struct xfs_btree_cur *cur = NULL; + int error = 0; + bool kernel_frozen = false; + uint32_t start_gno, end_gno; + xfs_fsblock_t start_bno, end_bno; + xfs_daddr_t daddr; + uint64_t bblen; + struct xfs_group *xg = NULL; + + error = xfs_dax_translate_range(xfs_group_type_buftarg(mp, type), + offset, len, &daddr, &bblen); + if (error) + return error; + + xfs_healthmon_report_media(mp, + type == XG_TYPE_RTG ? XFS_DEV_RT : XFS_DEV_DATA, + daddr, bblen); + + if (!xfs_has_rmapbt(mp)) { + xfs_debug(mp, "notify_failure() needs rmapbt enabled!"); + return -EOPNOTSUPP; + } + + if (type == XG_TYPE_RTG) { + start_bno = xfs_daddr_to_rtb(mp, daddr); + end_bno = xfs_daddr_to_rtb(mp, daddr + bblen - 1); + } else { + start_bno = XFS_DADDR_TO_FSB(mp, daddr); + end_bno = XFS_DADDR_TO_FSB(mp, daddr + bblen - 1); + } + + if (mf_flags & MF_MEM_PRE_REMOVE) { + xfs_info(mp, "Device is about to be removed!"); + /* + * Freeze fs to prevent new mappings from being created. + * - Keep going on if others already hold the kernel forzen. + * - Keep going on if other errors too because this device is + * starting to fail. + * - If kernel frozen state is hold successfully here, thaw it + * here as well at the end. + */ + kernel_frozen = xfs_dax_notify_failure_freeze(mp) == 0; + } + + tp = xfs_trans_alloc_empty(mp); + start_gno = xfs_fsb_to_gno(mp, start_bno, type); + end_gno = xfs_fsb_to_gno(mp, end_bno, type); + while ((xg = xfs_group_next_range(mp, xg, start_gno, end_gno, type))) { + struct xfs_buf *agf_bp = NULL; + struct xfs_rtgroup *rtg = NULL; + struct xfs_rmap_irec ri_low = { }; + struct xfs_rmap_irec ri_high; + + if (type == XG_TYPE_AG) { + struct xfs_perag *pag = to_perag(xg); + + error = xfs_alloc_read_agf(pag, tp, 0, &agf_bp); + if (error) { + xfs_perag_rele(pag); + break; + } + + cur = xfs_rmapbt_init_cursor(mp, tp, agf_bp, pag); + } else { + rtg = to_rtg(xg); + xfs_rtgroup_lock(rtg, XFS_RTGLOCK_RMAP); + cur = xfs_rtrmapbt_init_cursor(tp, rtg); + } + + /* + * Set the rmap range from ri_low to ri_high, which represents + * a [start, end] where we looking for the files or metadata. + */ + memset(&ri_high, 0xFF, sizeof(ri_high)); + if (xg->xg_gno == start_gno) + ri_low.rm_startblock = + xfs_fsb_to_gbno(mp, start_bno, type); + if (xg->xg_gno == end_gno) + ri_high.rm_startblock = + xfs_fsb_to_gbno(mp, end_bno, type); + + notify.startblock = ri_low.rm_startblock; + notify.blockcount = min(xg->xg_block_count, + ri_high.rm_startblock + 1) - + ri_low.rm_startblock; + + error = xfs_rmap_query_range(cur, &ri_low, &ri_high, + xfs_dax_failure_fn, ¬ify); + xfs_btree_del_cursor(cur, error); + if (agf_bp) + xfs_trans_brelse(tp, agf_bp); + if (rtg) + xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_RMAP); + if (error) { + xfs_group_rele(xg); + break; + } + } + + xfs_trans_cancel(tp); + + /* + * Shutdown fs from a force umount in pre-remove case which won't fail, + * so errors can be ignored. Otherwise, shutdown the filesystem with + * CORRUPT flag if error occurred or notify.want_shutdown was set during + * RMAP querying. + */ + if (mf_flags & MF_MEM_PRE_REMOVE) + xfs_force_shutdown(mp, SHUTDOWN_FORCE_UMOUNT); + else if (error || notify.want_shutdown) { + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_ONDISK); + if (!error) + error = -EFSCORRUPTED; + } + + /* Thaw the fs if it has been frozen before. */ + if (mf_flags & MF_MEM_PRE_REMOVE) + xfs_dax_notify_failure_thaw(mp, kernel_frozen); + + return error; +} + +static int +xfs_dax_notify_failure( + struct dax_device *dax_dev, + u64 offset, + u64 len, + int mf_flags) +{ + struct xfs_mount *mp = dax_holder(dax_dev); + + if (!(mp->m_super->s_flags & SB_BORN)) { + xfs_warn(mp, "filesystem is not ready for notify_failure()!"); + return -EIO; + } + + if (mp->m_logdev_targp != mp->m_ddev_targp && + mp->m_logdev_targp->bt_daxdev == dax_dev) { + return xfs_dax_notify_logdev_failure(mp, offset, len, mf_flags); + } + + return xfs_dax_notify_dev_failure(mp, offset, len, mf_flags, + (mp->m_rtdev_targp && mp->m_rtdev_targp->bt_daxdev == dax_dev) ? + XG_TYPE_RTG : XG_TYPE_AG); +} + +const struct dax_holder_operations xfs_dax_holder_operations = { + .notify_failure = xfs_dax_notify_failure, +}; diff --git a/fs/xfs/xfs_notify_failure.h b/fs/xfs/xfs_notify_failure.h new file mode 100644 index 000000000..8d08ec29d --- /dev/null +++ b/fs/xfs/xfs_notify_failure.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2024 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_NOTIFY_FAILURE_H__ +#define __XFS_NOTIFY_FAILURE_H__ + +extern const struct dax_holder_operations xfs_dax_holder_operations; + +#endif /* __XFS_NOTIFY_FAILURE_H__ */ diff --git a/fs/xfs/xfs_platform.h b/fs/xfs/xfs_platform.h new file mode 100644 index 000000000..59a33c60e --- /dev/null +++ b/fs/xfs/xfs_platform.h @@ -0,0 +1,303 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef _XFS_PLATFORM_H +#define _XFS_PLATFORM_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_XFS_DEBUG +#define DEBUG 1 +#endif + +#ifdef CONFIG_XFS_DEBUG_EXPENSIVE +#define DEBUG_EXPENSIVE 1 +#endif + +#ifdef CONFIG_XFS_ASSERT_FATAL +#define XFS_ASSERT_FATAL 1 +#endif + +#ifdef CONFIG_XFS_WARN +#define XFS_WARN 1 +#endif + +/* + * Kernel specific type declarations for XFS + */ +typedef __s64 xfs_off_t; /* type */ +typedef unsigned long long xfs_ino_t; /* type */ +typedef __s64 xfs_daddr_t; /* type */ +typedef __u32 xfs_dev_t; +typedef __u32 xfs_nlink_t; + +#include "xfs_types.h" +#include "xfs_fs.h" +#include "xfs_stats.h" +#include "xfs_sysctl.h" +#include "xfs_iops.h" +#include "xfs_aops.h" +#include "xfs_super.h" +#include "xfs_cksum.h" +#include "xfs_buf.h" +#include "xfs_message.h" +#include "xfs_drain.h" +#include "xfs_hooks.h" + +#ifdef __BIG_ENDIAN +#define XFS_NATIVE_HOST 1 +#else +#undef XFS_NATIVE_HOST +#endif + +#define xfs_panic_mask xfs_params.panic_mask.val +#define xfs_error_level xfs_params.error_level.val +#define xfs_syncd_centisecs xfs_params.syncd_timer.val +#define xfs_stats_clear xfs_params.stats_clear.val +#define xfs_inherit_sync xfs_params.inherit_sync.val +#define xfs_inherit_nodump xfs_params.inherit_nodump.val +#define xfs_inherit_noatime xfs_params.inherit_noatim.val +#define xfs_inherit_nosymlinks xfs_params.inherit_nosym.val +#define xfs_rotorstep xfs_params.rotorstep.val +#define xfs_inherit_nodefrag xfs_params.inherit_nodfrg.val +#define xfs_fstrm_centisecs xfs_params.fstrm_timer.val +#define xfs_blockgc_secs xfs_params.blockgc_timer.val + +#define current_cpu() (raw_smp_processor_id()) +#define current_set_flags_nested(sp, f) \ + (*(sp) = current->flags, current->flags |= (f)) +#define current_restore_flags_nested(sp, f) \ + (current->flags = ((current->flags & ~(f)) | (*(sp) & (f)))) + +#define NBBY 8 /* number of bits per byte */ + +/* + * Size of block device i/o is parameterized here. + * Currently the system supports page-sized i/o. + */ +#define BLKDEV_IOSHIFT PAGE_SHIFT +#define BLKDEV_IOSIZE (1<i_diflags & XFS_DIFLAG_REALTIME) && \ + (ip)->i_mount->m_rtdev_targp) +#define XFS_IS_REALTIME_MOUNT(mp) ((mp)->m_rtdev_targp ? 1 : 0) +#else +#define XFS_IS_REALTIME_INODE(ip) (0) +#define XFS_IS_REALTIME_MOUNT(mp) (0) +#endif + +/* + * Starting in Linux 4.15, the %p (raw pointer value) printk modifier + * prints a hashed version of the pointer to avoid leaking kernel + * pointers into dmesg. If we're trying to debug the kernel we want the + * raw values, so override this behavior as best we can. + */ +#ifdef DEBUG +# define PTR_FMT "%px" +#else +# define PTR_FMT "%p" +#endif + +/* + * Helper for IO routines to grab backing pages from allocated kernel memory. + */ +static inline struct page * +kmem_to_page(void *addr) +{ + if (is_vmalloc_addr(addr)) + return vmalloc_to_page(addr); + return virt_to_page(addr); +} + +#endif /* _XFS_PLATFORM_H */ diff --git a/fs/xfs/xfs_pnfs.c b/fs/xfs/xfs_pnfs.c new file mode 100644 index 000000000..f8535ecde --- /dev/null +++ b/fs/xfs/xfs_pnfs.c @@ -0,0 +1,357 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2014 Christoph Hellwig. + */ +#include "xfs_platform.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_bmap.h" +#include "xfs_iomap.h" +#include "xfs_pnfs.h" +#include + +/* + * Ensure that we do not have any outstanding pNFS layouts that can be used by + * clients to directly read from or write to this inode. This must be called + * before every operation that can remove blocks from the extent map. + * Additionally we call it during the write operation, where aren't concerned + * about exposing unallocated blocks but just want to provide basic + * synchronization between a local writer and pNFS clients. mmap writes would + * also benefit from this sort of synchronization, but due to the tricky locking + * rules in the page fault path we don't bother. + */ +int +xfs_break_leased_layouts( + struct inode *inode, + uint *iolock, + bool *did_unlock) +{ + struct xfs_inode *ip = XFS_I(inode); + int error; + + while ((error = break_layout(inode, false)) == -EWOULDBLOCK) { + xfs_iunlock(ip, *iolock); + *did_unlock = true; + error = break_layout(inode, true); + *iolock &= ~XFS_IOLOCK_SHARED; + *iolock |= XFS_IOLOCK_EXCL; + xfs_ilock(ip, *iolock); + } + + return error; +} + +static expfs_block_layouts_t +xfs_fs_layouts_supported( + struct super_block *sb) +{ + expfs_block_layouts_t supported = EXPFS_BLOCK_IN_BAND_ID; + + if (exportfs_bdev_supports_out_of_band_id(sb->s_bdev)) + supported |= EXPFS_BLOCK_OUT_OF_BAND_ID; + return supported; +} + +/* + * Get a unique ID including its location so that the client can identify + * the exported device. + */ +static int +xfs_fs_get_uuid( + struct super_block *sb, + u8 *buf, + u32 *len, + u64 *offset) +{ + struct xfs_mount *mp = XFS_M(sb); + + if (*len < sizeof(uuid_t)) + return -EINVAL; + + memcpy(buf, &mp->m_sb.sb_uuid, sizeof(uuid_t)); + *len = sizeof(uuid_t); + *offset = offsetof(struct xfs_dsb, sb_uuid); + return 0; +} + +/* + * We cannot use file based VFS helpers such as file_modified() to update + * inode state as we modify the data/metadata in the inode here. Hence we have + * to open code the timestamp updates and SUID/SGID stripping. We also need + * to set the inode prealloc flag to ensure that the extents we allocate are not + * removed if the inode is reclaimed from memory before xfs_fs_block_commit() + * is from the client to indicate that data has been written and the file size + * can be extended. + */ +static int +xfs_fs_map_update_inode( + struct xfs_inode *ip) +{ + struct xfs_trans *tp; + int error; + + error = xfs_trans_alloc(ip->i_mount, &M_RES(ip->i_mount)->tr_writeid, + 0, 0, 0, &tp); + if (error) + return error; + + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL); + + VFS_I(ip)->i_mode &= ~S_ISUID; + if (VFS_I(ip)->i_mode & S_IXGRP) + VFS_I(ip)->i_mode &= ~S_ISGID; + xfs_trans_ichgtime(tp, ip, XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG); + ip->i_diflags |= XFS_DIFLAG_PREALLOC; + + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + return xfs_trans_commit(tp); +} + +/* + * Get a layout for the pNFS client. + */ +static int +xfs_fs_map_blocks( + struct inode *inode, + loff_t offset, + u64 length, + struct iomap *iomap, + bool write, + u32 *device_generation) +{ + struct xfs_inode *ip = XFS_I(inode); + struct xfs_mount *mp = ip->i_mount; + struct xfs_bmbt_irec imap; + xfs_fileoff_t offset_fsb, end_fsb; + loff_t limit; + int nimaps = 1; + uint lock_flags; + int error = 0; + u64 seq; + + if (xfs_is_shutdown(mp)) + return -EIO; + + /* + * We can't export inodes residing on the realtime device. The realtime + * device doesn't have a UUID to identify it, so the client has no way + * to find it. + */ + if (XFS_IS_REALTIME_INODE(ip)) + return -ENXIO; + + /* + * The pNFS block layout spec actually supports reflink like + * functionality, but the Linux pNFS server doesn't implement it yet. + */ + if (xfs_is_reflink_inode(ip)) + return -ENXIO; + + /* + * Lock out any other I/O before we flush and invalidate the pagecache, + * and then hand out a layout to the remote system. This is very + * similar to direct I/O, except that the synchronization is much more + * complicated. See the comment near xfs_break_leased_layouts + * for a detailed explanation. + */ + xfs_ilock(ip, XFS_IOLOCK_EXCL); + + error = -EINVAL; + limit = mp->m_super->s_maxbytes; + if (!write) + limit = max(limit, round_up(i_size_read(inode), + inode->i_sb->s_blocksize)); + if (offset > limit) + goto out_unlock; + if (offset > limit - length) + length = limit - offset; + + error = filemap_write_and_wait(inode->i_mapping); + if (error) + goto out_unlock; + error = invalidate_inode_pages2(inode->i_mapping); + if (WARN_ON_ONCE(error)) + goto out_unlock; + + end_fsb = XFS_B_TO_FSB(mp, (xfs_ufsize_t)offset + length); + offset_fsb = XFS_B_TO_FSBT(mp, offset); + + lock_flags = xfs_ilock_data_map_shared(ip); + /* request mappings for the specified range only */ + error = xfs_bmapi_read(ip, offset_fsb, end_fsb - offset_fsb, + &imap, &nimaps, 0); + if (error) { + xfs_iunlock(ip, lock_flags); + goto out_unlock; + } + seq = xfs_iomap_inode_sequence(ip, 0); + + ASSERT(!nimaps || imap.br_startblock != DELAYSTARTBLOCK); + + if (write && (!nimaps || imap.br_startblock == HOLESTARTBLOCK)) { + if (offset + length > XFS_ISIZE(ip)) + end_fsb = xfs_iomap_eof_align_last_fsb(ip, end_fsb); + else if (nimaps && imap.br_startblock == HOLESTARTBLOCK) + end_fsb = min(end_fsb, imap.br_startoff + + imap.br_blockcount); + xfs_iunlock(ip, lock_flags); + + error = xfs_iomap_write_direct(ip, offset_fsb, + end_fsb - offset_fsb, 0, &imap, &seq); + if (error) + goto out_unlock; + + /* + * Ensure the next transaction is committed synchronously so + * that the blocks allocated and handed out to the client are + * guaranteed to be present even after a server crash. + */ + error = xfs_fs_map_update_inode(ip); + if (!error) + error = xfs_log_force_inode(ip); + if (error) + goto out_unlock; + + } else { + xfs_iunlock(ip, lock_flags); + } + xfs_iunlock(ip, XFS_IOLOCK_EXCL); + + error = xfs_bmbt_to_iomap(ip, iomap, &imap, 0, 0, seq); + *device_generation = mp->m_generation; + return error; +out_unlock: + xfs_iunlock(ip, XFS_IOLOCK_EXCL); + return error; +} + +/* + * Ensure the size update falls into a valid allocated block. + */ +static int +xfs_pnfs_validate_isize( + struct xfs_inode *ip, + xfs_off_t isize) +{ + struct xfs_bmbt_irec imap; + int nimaps = 1; + int error = 0; + + xfs_ilock(ip, XFS_ILOCK_SHARED); + error = xfs_bmapi_read(ip, XFS_B_TO_FSBT(ip->i_mount, isize - 1), 1, + &imap, &nimaps, 0); + xfs_iunlock(ip, XFS_ILOCK_SHARED); + if (error) + return error; + + if (imap.br_startblock == HOLESTARTBLOCK || + imap.br_startblock == DELAYSTARTBLOCK || + imap.br_state == XFS_EXT_UNWRITTEN) + return -EIO; + return 0; +} + +/* + * Make sure the blocks described by maps are stable on disk. This includes + * converting any unwritten extents, flushing the disk cache and updating the + * time stamps. + * + * Note that we rely on the caller to always send us a timestamp update so that + * we always commit a transaction here. If that stops being true we will have + * to manually flush the cache here similar to what the fsync code path does + * for datasyncs on files that have no dirty metadata. + */ +static int +xfs_fs_commit_blocks( + struct inode *inode, + struct iomap *maps, + int nr_maps, + loff_t new_size) +{ + struct xfs_inode *ip = XFS_I(inode); + struct xfs_mount *mp = ip->i_mount; + struct xfs_trans *tp; + struct timespec64 now; + bool update_isize = false; + int error, i; + loff_t size; + + xfs_ilock(ip, XFS_IOLOCK_EXCL); + + size = i_size_read(inode); + if (new_size > size) { + update_isize = true; + size = new_size; + } + + for (i = 0; i < nr_maps; i++) { + u64 start, length, end; + + start = maps[i].offset; + if (start > size) + continue; + + end = start + maps[i].length; + if (end > size) + end = size; + + length = end - start; + if (!length) + continue; + + /* + * Make sure reads through the pagecache see the new data. + */ + error = invalidate_inode_pages2_range(inode->i_mapping, + start >> PAGE_SHIFT, + (end - 1) >> PAGE_SHIFT); + WARN_ON_ONCE(error); + + error = xfs_iomap_write_unwritten(ip, start, length, false); + if (error) + goto out_drop_iolock; + } + + if (update_isize) { + error = xfs_pnfs_validate_isize(ip, size); + if (error) + goto out_drop_iolock; + } + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_ichange, 0, 0, 0, &tp); + if (error) + goto out_drop_iolock; + + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL); + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + + now = inode_set_ctime_current(inode); + inode_set_atime_to_ts(inode, now); + inode_set_mtime_to_ts(inode, now); + + if (update_isize) { + i_size_write(inode, new_size); + ip->i_disk_size = new_size; + } + + xfs_trans_set_sync(tp); + error = xfs_trans_commit(tp); + +out_drop_iolock: + xfs_iunlock(ip, XFS_IOLOCK_EXCL); + return error; +} + +const struct exportfs_block_ops xfs_export_block_ops = { + .layouts_supported = xfs_fs_layouts_supported, + .get_uuid = xfs_fs_get_uuid, + .map_blocks = xfs_fs_map_blocks, + .commit_blocks = xfs_fs_commit_blocks, +}; diff --git a/fs/xfs/xfs_pnfs.h b/fs/xfs/xfs_pnfs.h new file mode 100644 index 000000000..bf43b2009 --- /dev/null +++ b/fs/xfs/xfs_pnfs.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _XFS_PNFS_H +#define _XFS_PNFS_H 1 + +#include + +#ifdef CONFIG_EXPORTFS_BLOCK_OPS +int xfs_break_leased_layouts(struct inode *inode, uint *iolock, + bool *did_unlock); +#else +static inline int +xfs_break_leased_layouts(struct inode *inode, uint *iolock, bool *did_unlock) +{ + return 0; +} +#endif /* CONFIG_EXPORTFS_BLOCK_OPS */ + +extern const struct exportfs_block_ops xfs_export_block_ops; + +#endif /* _XFS_PNFS_H */ diff --git a/fs/xfs/xfs_pwork.c b/fs/xfs/xfs_pwork.c new file mode 100644 index 000000000..7c79ab0db --- /dev/null +++ b/fs/xfs/xfs_pwork.c @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2019 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_trace.h" +#include "xfs_sysctl.h" +#include "xfs_pwork.h" +#include + +/* + * Parallel Work Queue + * =================== + * + * Abstract away the details of running a large and "obviously" parallelizable + * task across multiple CPUs. Callers initialize the pwork control object with + * a desired level of parallelization and a work function. Next, they embed + * struct xfs_pwork in whatever structure they use to pass work context to a + * worker thread and queue that pwork. The work function will be passed the + * pwork item when it is run (from process context) and any returned error will + * be recorded in xfs_pwork_ctl.error. Work functions should check for errors + * and abort if necessary; the non-zeroness of xfs_pwork_ctl.error does not + * stop workqueue item processing. + * + * This is the rough equivalent of the xfsprogs workqueue code, though we can't + * reuse that name here. + */ + +/* Invoke our caller's function. */ +static void +xfs_pwork_work( + struct work_struct *work) +{ + struct xfs_pwork *pwork; + struct xfs_pwork_ctl *pctl; + int error; + + pwork = container_of(work, struct xfs_pwork, work); + pctl = pwork->pctl; + error = pctl->work_fn(pctl->mp, pwork); + if (error && !pctl->error) + pctl->error = error; + if (atomic_dec_and_test(&pctl->nr_work)) + wake_up(&pctl->poll_wait); +} + +/* + * Set up control data for parallel work. @work_fn is the function that will + * be called. @tag will be written into the kernel threads. @nr_threads is + * the level of parallelism desired, or 0 for no limit. + */ +int +xfs_pwork_init( + struct xfs_mount *mp, + struct xfs_pwork_ctl *pctl, + xfs_pwork_work_fn work_fn, + const char *tag) +{ + unsigned int nr_threads = 0; + +#ifdef DEBUG + if (xfs_globals.pwork_threads >= 0) + nr_threads = xfs_globals.pwork_threads; +#endif + trace_xfs_pwork_init(mp, nr_threads, current->pid); + + pctl->wq = alloc_workqueue("%s-%d", + WQ_UNBOUND | WQ_SYSFS | WQ_FREEZABLE, nr_threads, tag, + current->pid); + if (!pctl->wq) + return -ENOMEM; + pctl->work_fn = work_fn; + pctl->error = 0; + pctl->mp = mp; + atomic_set(&pctl->nr_work, 0); + init_waitqueue_head(&pctl->poll_wait); + + return 0; +} + +/* Queue some parallel work. */ +void +xfs_pwork_queue( + struct xfs_pwork_ctl *pctl, + struct xfs_pwork *pwork) +{ + INIT_WORK(&pwork->work, xfs_pwork_work); + pwork->pctl = pctl; + atomic_inc(&pctl->nr_work); + queue_work(pctl->wq, &pwork->work); +} + +/* Wait for the work to finish and tear down the control structure. */ +int +xfs_pwork_destroy( + struct xfs_pwork_ctl *pctl) +{ + destroy_workqueue(pctl->wq); + pctl->wq = NULL; + return pctl->error; +} + +/* + * Wait for the work to finish by polling completion status and touch the soft + * lockup watchdog. This is for callers such as mount which hold locks. + */ +void +xfs_pwork_poll( + struct xfs_pwork_ctl *pctl) +{ + while (wait_event_timeout(pctl->poll_wait, + atomic_read(&pctl->nr_work) == 0, HZ) == 0) + touch_softlockup_watchdog(); +} diff --git a/fs/xfs/xfs_pwork.h b/fs/xfs/xfs_pwork.h new file mode 100644 index 000000000..c0ef81fc8 --- /dev/null +++ b/fs/xfs/xfs_pwork.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2019 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_PWORK_H__ +#define __XFS_PWORK_H__ + +struct xfs_pwork; +struct xfs_mount; + +typedef int (*xfs_pwork_work_fn)(struct xfs_mount *mp, struct xfs_pwork *pwork); + +/* + * Parallel work coordination structure. + */ +struct xfs_pwork_ctl { + struct workqueue_struct *wq; + struct xfs_mount *mp; + xfs_pwork_work_fn work_fn; + struct wait_queue_head poll_wait; + atomic_t nr_work; + int error; +}; + +/* + * Embed this parallel work control item inside your own work structure, + * then queue work with it. + */ +struct xfs_pwork { + struct work_struct work; + struct xfs_pwork_ctl *pctl; +}; + +#define XFS_PWORK_SINGLE_THREADED { .pctl = NULL } + +/* Have we been told to abort? */ +static inline bool +xfs_pwork_ctl_want_abort( + struct xfs_pwork_ctl *pctl) +{ + return pctl && pctl->error; +} + +/* Have we been told to abort? */ +static inline bool +xfs_pwork_want_abort( + struct xfs_pwork *pwork) +{ + return xfs_pwork_ctl_want_abort(pwork->pctl); +} + +int xfs_pwork_init(struct xfs_mount *mp, struct xfs_pwork_ctl *pctl, + xfs_pwork_work_fn work_fn, const char *tag); +void xfs_pwork_queue(struct xfs_pwork_ctl *pctl, struct xfs_pwork *pwork); +int xfs_pwork_destroy(struct xfs_pwork_ctl *pctl); +void xfs_pwork_poll(struct xfs_pwork_ctl *pctl); + +#endif /* __XFS_PWORK_H__ */ diff --git a/fs/xfs/xfs_qm.c b/fs/xfs/xfs_qm.c new file mode 100644 index 000000000..99a82107b --- /dev/null +++ b/fs/xfs/xfs_qm.c @@ -0,0 +1,2137 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_sb.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_iwalk.h" +#include "xfs_quota.h" +#include "xfs_bmap.h" +#include "xfs_bmap_util.h" +#include "xfs_trans.h" +#include "xfs_trans_space.h" +#include "xfs_qm.h" +#include "xfs_trace.h" +#include "xfs_icache.h" +#include "xfs_error.h" +#include "xfs_ag.h" +#include "xfs_ialloc.h" +#include "xfs_log_priv.h" +#include "xfs_health.h" +#include "xfs_da_format.h" +#include "xfs_metafile.h" +#include "xfs_rtgroup.h" + +/* + * The global quota manager. There is only one of these for the entire + * system, _not_ one per file system. XQM keeps track of the overall + * quota functionality, including maintaining the freelist and hash + * tables of dquots. + */ +STATIC int xfs_qm_init_quotainos(struct xfs_mount *mp); +STATIC int xfs_qm_init_quotainfo(struct xfs_mount *mp); + +STATIC void xfs_qm_dqfree_one(struct xfs_dquot *dqp); +/* + * We use the batch lookup interface to iterate over the dquots as it + * currently is the only interface into the radix tree code that allows + * fuzzy lookups instead of exact matches. Holding the lock over multiple + * operations is fine as all callers are used either during mount/umount + * or quotaoff. + */ +#define XFS_DQ_LOOKUP_BATCH 32 + +STATIC int +xfs_qm_dquot_walk( + struct xfs_mount *mp, + xfs_dqtype_t type, + int (*execute)(struct xfs_dquot *dqp, void *data), + void *data) +{ + struct xfs_quotainfo *qi = mp->m_quotainfo; + struct radix_tree_root *tree = xfs_dquot_tree(qi, type); + uint32_t next_index; + int last_error = 0; + int skipped; + int nr_found; + +restart: + skipped = 0; + next_index = 0; + nr_found = 0; + + while (1) { + struct xfs_dquot *batch[XFS_DQ_LOOKUP_BATCH]; + int error; + int i; + + mutex_lock(&qi->qi_tree_lock); + nr_found = radix_tree_gang_lookup(tree, (void **)batch, + next_index, XFS_DQ_LOOKUP_BATCH); + if (!nr_found) { + mutex_unlock(&qi->qi_tree_lock); + break; + } + + for (i = 0; i < nr_found; i++) { + struct xfs_dquot *dqp = batch[i]; + + next_index = dqp->q_id + 1; + + error = execute(batch[i], data); + if (error == -EAGAIN) { + skipped++; + continue; + } + if (error && last_error != -EFSCORRUPTED) + last_error = error; + } + + mutex_unlock(&qi->qi_tree_lock); + + /* bail out if the filesystem is corrupted. */ + if (last_error == -EFSCORRUPTED) { + skipped = 0; + break; + } + /* we're done if id overflows back to zero */ + if (!next_index) + break; + } + + if (skipped) { + delay(1); + goto restart; + } + + return last_error; +} + + +/* + * Purge a dquot from all tracking data structures and free it. + */ +STATIC int +xfs_qm_dqpurge( + struct xfs_dquot *dqp, + void *data) +{ + struct xfs_quotainfo *qi = dqp->q_mount->m_quotainfo; + + spin_lock(&dqp->q_lockref.lock); + if (dqp->q_lockref.count > 0 || lockref_is_dead(&dqp->q_lockref)) { + spin_unlock(&dqp->q_lockref.lock); + return -EAGAIN; + } + lockref_mark_dead(&dqp->q_lockref); + spin_unlock(&dqp->q_lockref.lock); + + mutex_lock(&dqp->q_qlock); + xfs_qm_dqunpin_wait(dqp); + xfs_dqflock(dqp); + + /* + * If we are turning this type of quotas off, we don't care + * about the dirty metadata sitting in this dquot. OTOH, if + * we're unmounting, we do care, so we flush it and wait. + */ + if (XFS_DQ_IS_DIRTY(dqp)) { + struct xfs_buf *bp = NULL; + int error; + + /* + * We don't care about getting disk errors here. We need + * to purge this dquot anyway, so we go ahead regardless. + */ + error = xfs_dquot_use_attached_buf(dqp, &bp); + if (error == -EAGAIN) { + /* resurrect the refcount from the dead. */ + dqp->q_lockref.count = 0; + goto out_funlock; + } + if (!bp) + goto out_funlock; + + /* + * dqflush completes dqflock on error, and the bwrite ioend + * does it on success. + */ + error = xfs_qm_dqflush(dqp, bp); + if (!error) + error = xfs_bwrite(bp); + xfs_buf_relse(bp); + xfs_dqflock(dqp); + } + xfs_dquot_detach_buf(dqp); + +out_funlock: + ASSERT(atomic_read(&dqp->q_pincount) == 0); + ASSERT(xlog_is_shutdown(dqp->q_logitem.qli_item.li_log) || + !test_bit(XFS_LI_IN_AIL, &dqp->q_logitem.qli_item.li_flags)); + + xfs_dqfunlock(dqp); + mutex_unlock(&dqp->q_qlock); + + radix_tree_delete(xfs_dquot_tree(qi, xfs_dquot_type(dqp)), dqp->q_id); + qi->qi_dquots--; + + /* + * We move dquots to the freelist as soon as their reference count + * hits zero, so it really should be on the freelist here. + */ + ASSERT(!list_empty(&dqp->q_lru)); + list_lru_del_obj(&qi->qi_lru, &dqp->q_lru); + XFS_STATS_DEC(dqp->q_mount, xs_qm_dquot_unused); + + xfs_qm_dqdestroy(dqp); + return 0; +} + +/* + * Purge the dquot cache. + */ +static void +xfs_qm_dqpurge_all( + struct xfs_mount *mp) +{ + xfs_qm_dquot_walk(mp, XFS_DQTYPE_USER, xfs_qm_dqpurge, NULL); + xfs_qm_dquot_walk(mp, XFS_DQTYPE_GROUP, xfs_qm_dqpurge, NULL); + xfs_qm_dquot_walk(mp, XFS_DQTYPE_PROJ, xfs_qm_dqpurge, NULL); +} + +/* + * Just destroy the quotainfo structure. + */ +void +xfs_qm_unmount( + struct xfs_mount *mp) +{ + if (mp->m_quotainfo) { + xfs_qm_dqpurge_all(mp); + xfs_qm_destroy_quotainfo(mp); + } +} + +static void +xfs_qm_unmount_rt( + struct xfs_mount *mp) +{ + struct xfs_rtgroup *rtg = xfs_rtgroup_grab(mp, 0); + + if (!rtg) + return; + if (rtg_bitmap(rtg)) + xfs_qm_dqdetach(rtg_bitmap(rtg)); + if (rtg_summary(rtg)) + xfs_qm_dqdetach(rtg_summary(rtg)); + xfs_rtgroup_rele(rtg); +} + +STATIC void +xfs_qm_destroy_quotainos( + struct xfs_quotainfo *qi) +{ + if (qi->qi_uquotaip) { + xfs_irele(qi->qi_uquotaip); + qi->qi_uquotaip = NULL; /* paranoia */ + } + if (qi->qi_gquotaip) { + xfs_irele(qi->qi_gquotaip); + qi->qi_gquotaip = NULL; + } + if (qi->qi_pquotaip) { + xfs_irele(qi->qi_pquotaip); + qi->qi_pquotaip = NULL; + } + if (qi->qi_dirip) { + xfs_irele(qi->qi_dirip); + qi->qi_dirip = NULL; + } +} + +/* + * Called from the vfsops layer. + */ +void +xfs_qm_unmount_quotas( + xfs_mount_t *mp) +{ + /* + * Release the dquots that root inode, et al might be holding, + * before we flush quotas and blow away the quotainfo structure. + */ + ASSERT(mp->m_rootip); + xfs_qm_dqdetach(mp->m_rootip); + + /* + * For pre-RTG file systems, the RT inodes have quotas attached, + * detach them now. + */ + if (!xfs_has_rtgroups(mp)) + xfs_qm_unmount_rt(mp); + + /* + * Release the quota inodes. + */ + if (mp->m_quotainfo) + xfs_qm_destroy_quotainos(mp->m_quotainfo); +} + +static bool +xfs_qm_need_dqattach( + struct xfs_inode *ip) +{ + struct xfs_mount *mp = ip->i_mount; + + if (!XFS_IS_QUOTA_ON(mp)) + return false; + if (!XFS_NOT_DQATTACHED(mp, ip)) + return false; + if (xfs_is_quota_inode(&mp->m_sb, I_INO(ip))) + return false; + if (xfs_is_metadir_inode(ip)) + return false; + return true; +} + +/* + * Given a locked inode, attach dquot(s) to it, taking U/G/P-QUOTAON + * into account. + * If @doalloc is true, the dquot(s) will be allocated if needed. + * Inode may get unlocked and relocked in here, and the caller must deal with + * the consequences. + */ +int +xfs_qm_dqattach_locked( + xfs_inode_t *ip, + bool doalloc) +{ + xfs_mount_t *mp = ip->i_mount; + int error = 0; + + if (!xfs_qm_need_dqattach(ip)) + return 0; + + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + ASSERT(!xfs_is_metadir_inode(ip)); + + if (XFS_IS_UQUOTA_ON(mp) && !ip->i_udquot) { + error = xfs_qm_dqget_inode(ip, XFS_DQTYPE_USER, + doalloc, &ip->i_udquot); + if (error) + goto done; + ASSERT(ip->i_udquot); + } + + if (XFS_IS_GQUOTA_ON(mp) && !ip->i_gdquot) { + error = xfs_qm_dqget_inode(ip, XFS_DQTYPE_GROUP, + doalloc, &ip->i_gdquot); + if (error) + goto done; + ASSERT(ip->i_gdquot); + } + + if (XFS_IS_PQUOTA_ON(mp) && !ip->i_pdquot) { + error = xfs_qm_dqget_inode(ip, XFS_DQTYPE_PROJ, + doalloc, &ip->i_pdquot); + if (error) + goto done; + ASSERT(ip->i_pdquot); + } + +done: + /* + * Don't worry about the dquots that we may have attached before any + * error - they'll get detached later if it has not already been done. + */ + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + return error; +} + +int +xfs_qm_dqattach( + struct xfs_inode *ip) +{ + int error; + + if (!xfs_qm_need_dqattach(ip)) + return 0; + + xfs_ilock(ip, XFS_ILOCK_EXCL); + error = xfs_qm_dqattach_locked(ip, false); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + + return error; +} + +/* + * Release dquots (and their references) if any. + * The inode should be locked EXCL except when this's called by + * xfs_ireclaim. + */ +void +xfs_qm_dqdetach( + xfs_inode_t *ip) +{ + if (xfs_is_metadir_inode(ip)) + return; + if (!(ip->i_udquot || ip->i_gdquot || ip->i_pdquot)) + return; + + trace_xfs_dquot_dqdetach(ip); + + ASSERT(!xfs_is_quota_inode(&ip->i_mount->m_sb, I_INO(ip))); + if (ip->i_udquot) { + xfs_qm_dqrele(ip->i_udquot); + ip->i_udquot = NULL; + } + if (ip->i_gdquot) { + xfs_qm_dqrele(ip->i_gdquot); + ip->i_gdquot = NULL; + } + if (ip->i_pdquot) { + xfs_qm_dqrele(ip->i_pdquot); + ip->i_pdquot = NULL; + } +} + +struct xfs_qm_isolate { + struct list_head buffers; + struct list_head dispose; +}; + +static enum lru_status +xfs_qm_dquot_isolate( + struct list_head *item, + struct list_lru_one *lru, + void *arg) + __releases(&lru->lock) __acquires(&lru->lock) +{ + struct xfs_dquot *dqp = container_of(item, + struct xfs_dquot, q_lru); + struct xfs_qm_isolate *isol = arg; + enum lru_status ret = LRU_SKIP; + + if (!spin_trylock(&dqp->q_lockref.lock)) + goto out_miss_busy; + + /* + * If something else is freeing this dquot and hasn't yet removed it + * from the LRU, leave it for the freeing task to complete the freeing + * process rather than risk it being free from under us here. + */ + if (lockref_is_dead(&dqp->q_lockref)) + goto out_miss_unlock; + + /* + * If the dquot is pinned or dirty, rotate it to the end of the LRU to + * give some time for it to be cleaned before we try to isolate it + * again. + */ + ret = LRU_ROTATE; + if (XFS_DQ_IS_DIRTY(dqp) || atomic_read(&dqp->q_pincount) > 0) + goto out_miss_unlock; + + /* + * This dquot has acquired a reference in the meantime remove it from + * the freelist and try again. + */ + if (dqp->q_lockref.count) { + spin_unlock(&dqp->q_lockref.lock); + XFS_STATS_INC(dqp->q_mount, xs_qm_dqwants); + + trace_xfs_dqreclaim_want(dqp); + list_lru_isolate(lru, &dqp->q_lru); + XFS_STATS_DEC(dqp->q_mount, xs_qm_dquot_unused); + return LRU_REMOVED; + } + + /* + * The dquot may still be under IO, in which case the flush lock will be + * held. If we can't get the flush lock now, just skip over the dquot as + * if it was dirty. + */ + if (!xfs_dqflock_nowait(dqp)) + goto out_miss_unlock; + + ASSERT(!XFS_DQ_IS_DIRTY(dqp)); + xfs_dquot_detach_buf(dqp); + xfs_dqfunlock(dqp); + + /* + * Prevent lookups now that we are past the point of no return. + */ + lockref_mark_dead(&dqp->q_lockref); + spin_unlock(&dqp->q_lockref.lock); + + list_lru_isolate_move(lru, &dqp->q_lru, &isol->dispose); + XFS_STATS_DEC(dqp->q_mount, xs_qm_dquot_unused); + trace_xfs_dqreclaim_done(dqp); + XFS_STATS_INC(dqp->q_mount, xs_qm_dqreclaims); + return LRU_REMOVED; + +out_miss_unlock: + spin_unlock(&dqp->q_lockref.lock); +out_miss_busy: + trace_xfs_dqreclaim_busy(dqp); + XFS_STATS_INC(dqp->q_mount, xs_qm_dqreclaim_misses); + return ret; +} + +static unsigned long +xfs_qm_shrink_scan( + struct shrinker *shrink, + struct shrink_control *sc) +{ + struct xfs_quotainfo *qi = shrink->private_data; + struct xfs_qm_isolate isol; + unsigned long freed; + int error; + + if ((sc->gfp_mask & (__GFP_FS|__GFP_DIRECT_RECLAIM)) != (__GFP_FS|__GFP_DIRECT_RECLAIM)) + return 0; + + INIT_LIST_HEAD(&isol.buffers); + INIT_LIST_HEAD(&isol.dispose); + + freed = list_lru_shrink_walk(&qi->qi_lru, sc, + xfs_qm_dquot_isolate, &isol); + + error = xfs_buf_delwri_submit(&isol.buffers); + if (error) + xfs_warn(NULL, "%s: dquot reclaim failed", __func__); + + while (!list_empty(&isol.dispose)) { + struct xfs_dquot *dqp; + + dqp = list_first_entry(&isol.dispose, struct xfs_dquot, q_lru); + list_del_init(&dqp->q_lru); + xfs_qm_dqfree_one(dqp); + } + + return freed; +} + +static unsigned long +xfs_qm_shrink_count( + struct shrinker *shrink, + struct shrink_control *sc) +{ + struct xfs_quotainfo *qi = shrink->private_data; + + return list_lru_shrink_count(&qi->qi_lru, sc); +} + +STATIC void +xfs_qm_set_defquota( + struct xfs_mount *mp, + xfs_dqtype_t type, + struct xfs_quotainfo *qinf) +{ + struct xfs_dquot *dqp; + struct xfs_def_quota *defq; + int error; + + error = xfs_qm_dqget_uncached(mp, 0, type, &dqp); + if (error) + return; + + defq = xfs_get_defquota(qinf, xfs_dquot_type(dqp)); + + /* + * Timers and warnings have been already set, let's just set the + * default limits for this quota type + */ + defq->blk.hard = dqp->q_blk.hardlimit; + defq->blk.soft = dqp->q_blk.softlimit; + defq->ino.hard = dqp->q_ino.hardlimit; + defq->ino.soft = dqp->q_ino.softlimit; + defq->rtb.hard = dqp->q_rtb.hardlimit; + defq->rtb.soft = dqp->q_rtb.softlimit; + xfs_qm_dqdestroy(dqp); +} + +/* Initialize quota time limits from the root dquot. */ +static void +xfs_qm_init_timelimits( + struct xfs_mount *mp, + xfs_dqtype_t type) +{ + struct xfs_quotainfo *qinf = mp->m_quotainfo; + struct xfs_def_quota *defq; + struct xfs_dquot *dqp; + int error; + + defq = xfs_get_defquota(qinf, type); + + defq->blk.time = XFS_QM_BTIMELIMIT; + defq->ino.time = XFS_QM_ITIMELIMIT; + defq->rtb.time = XFS_QM_RTBTIMELIMIT; + + /* + * We try to get the limits from the superuser's limits fields. + * This is quite hacky, but it is standard quota practice. + * + * Since we may not have done a quotacheck by this point, just read + * the dquot without attaching it to any hashtables or lists. + */ + error = xfs_qm_dqget_uncached(mp, 0, type, &dqp); + if (error) + return; + + /* + * The warnings and timers set the grace period given to + * a user or group before he or she can not perform any + * more writing. If it is zero, a default is used. + */ + if (dqp->q_blk.timer) + defq->blk.time = dqp->q_blk.timer; + if (dqp->q_ino.timer) + defq->ino.time = dqp->q_ino.timer; + if (dqp->q_rtb.timer) + defq->rtb.time = dqp->q_rtb.timer; + + xfs_qm_dqdestroy(dqp); +} + +static int +xfs_qm_load_metadir_qinos( + struct xfs_mount *mp, + struct xfs_quotainfo *qi) +{ + struct xfs_trans *tp; + int error; + + tp = xfs_trans_alloc_empty(mp); + error = xfs_dqinode_load_parent(tp, &qi->qi_dirip); + if (error == -ENOENT) { + /* no quota dir directory, but we'll create one later */ + error = 0; + goto out_trans; + } + if (error) + goto out_trans; + + if (XFS_IS_UQUOTA_ON(mp)) { + error = xfs_dqinode_load(tp, qi->qi_dirip, XFS_DQTYPE_USER, + &qi->qi_uquotaip); + if (error && error != -ENOENT) + goto out_trans; + } + + if (XFS_IS_GQUOTA_ON(mp)) { + error = xfs_dqinode_load(tp, qi->qi_dirip, XFS_DQTYPE_GROUP, + &qi->qi_gquotaip); + if (error && error != -ENOENT) + goto out_trans; + } + + if (XFS_IS_PQUOTA_ON(mp)) { + error = xfs_dqinode_load(tp, qi->qi_dirip, XFS_DQTYPE_PROJ, + &qi->qi_pquotaip); + if (error && error != -ENOENT) + goto out_trans; + } + + error = 0; +out_trans: + xfs_trans_cancel(tp); + return error; +} + +/* Create quota inodes in the metadata directory tree. */ +STATIC int +xfs_qm_create_metadir_qinos( + struct xfs_mount *mp, + struct xfs_quotainfo *qi) +{ + int error; + + if (!qi->qi_dirip) { + error = xfs_dqinode_mkdir_parent(mp, &qi->qi_dirip); + if (error && error != -EEXIST) + return error; + /* + * If the /quotas dirent points to an inode that isn't + * loadable, qi_dirip will be NULL but mkdir_parent will return + * -EEXIST. In this case the metadir is corrupt, so bail out. + */ + if (XFS_IS_CORRUPT(mp, qi->qi_dirip == NULL)) + return -EFSCORRUPTED; + } + + if (XFS_IS_UQUOTA_ON(mp) && !qi->qi_uquotaip) { + error = xfs_dqinode_metadir_create(qi->qi_dirip, + XFS_DQTYPE_USER, &qi->qi_uquotaip); + if (error) + return error; + } + + if (XFS_IS_GQUOTA_ON(mp) && !qi->qi_gquotaip) { + error = xfs_dqinode_metadir_create(qi->qi_dirip, + XFS_DQTYPE_GROUP, &qi->qi_gquotaip); + if (error) + return error; + } + + if (XFS_IS_PQUOTA_ON(mp) && !qi->qi_pquotaip) { + error = xfs_dqinode_metadir_create(qi->qi_dirip, + XFS_DQTYPE_PROJ, &qi->qi_pquotaip); + if (error) + return error; + } + + return 0; +} + +/* + * Add QUOTABIT to sb_versionnum and initialize qflags in preparation for + * creating quota files on a metadir filesystem. + */ +STATIC int +xfs_qm_prep_metadir_sb( + struct xfs_mount *mp) +{ + struct xfs_trans *tp; + int error; + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_sb, 0, 0, 0, &tp); + if (error) + return error; + + spin_lock(&mp->m_sb_lock); + + xfs_add_quota(mp); + + /* qflags will get updated fully _after_ quotacheck */ + mp->m_sb.sb_qflags = mp->m_qflags & XFS_ALL_QUOTA_ACCT; + + spin_unlock(&mp->m_sb_lock); + xfs_log_sb(tp); + + return xfs_trans_commit(tp); +} + +/* + * Load existing quota inodes or create them. Since this is a V5 filesystem, + * we don't have to deal with the grp/prjquota switcheroo thing from V4. + */ +STATIC int +xfs_qm_init_metadir_qinos( + struct xfs_mount *mp) +{ + struct xfs_quotainfo *qi = mp->m_quotainfo; + int error; + + if (!xfs_has_quota(mp)) { + error = xfs_qm_prep_metadir_sb(mp); + if (error) + return error; + } + + error = xfs_qm_load_metadir_qinos(mp, qi); + if (error) + goto out_err; + + error = xfs_qm_create_metadir_qinos(mp, qi); + if (error) + goto out_err; + + /* The only user of the quota dir inode is online fsck */ +#if !IS_ENABLED(CONFIG_XFS_ONLINE_SCRUB) + xfs_irele(qi->qi_dirip); + qi->qi_dirip = NULL; +#endif + return 0; +out_err: + xfs_qm_destroy_quotainos(mp->m_quotainfo); + return error; +} + +/* + * This initializes all the quota information that's kept in the + * mount structure + */ +STATIC int +xfs_qm_init_quotainfo( + struct xfs_mount *mp) +{ + struct xfs_quotainfo *qinf; + int error; + + ASSERT(XFS_IS_QUOTA_ON(mp)); + + qinf = mp->m_quotainfo = kzalloc_obj(struct xfs_quotainfo, + GFP_KERNEL | __GFP_NOFAIL); + + error = list_lru_init(&qinf->qi_lru); + if (error) + goto out_free_qinf; + + /* + * See if quotainodes are setup, and if not, allocate them, + * and change the superblock accordingly. + */ + if (xfs_has_metadir(mp)) + error = xfs_qm_init_metadir_qinos(mp); + else + error = xfs_qm_init_quotainos(mp); + if (error) + goto out_free_lru; + + INIT_RADIX_TREE(&qinf->qi_uquota_tree, GFP_KERNEL); + INIT_RADIX_TREE(&qinf->qi_gquota_tree, GFP_KERNEL); + INIT_RADIX_TREE(&qinf->qi_pquota_tree, GFP_KERNEL); + mutex_init(&qinf->qi_tree_lock); + + /* mutex used to serialize quotaoffs */ + mutex_init(&qinf->qi_quotaofflock); + + /* Precalc some constants */ + qinf->qi_dqchunklen = XFS_FSB_TO_BB(mp, XFS_DQUOT_CLUSTER_SIZE_FSB); + qinf->qi_dqperchunk = xfs_calc_dquots_per_chunk(qinf->qi_dqchunklen); + if (xfs_has_bigtime(mp)) { + qinf->qi_expiry_min = + xfs_dq_bigtime_to_unix(XFS_DQ_BIGTIME_EXPIRY_MIN); + qinf->qi_expiry_max = + xfs_dq_bigtime_to_unix(XFS_DQ_BIGTIME_EXPIRY_MAX); + } else { + qinf->qi_expiry_min = XFS_DQ_LEGACY_EXPIRY_MIN; + qinf->qi_expiry_max = XFS_DQ_LEGACY_EXPIRY_MAX; + } + trace_xfs_quota_expiry_range(mp, qinf->qi_expiry_min, + qinf->qi_expiry_max); + + mp->m_qflags |= (mp->m_sb.sb_qflags & XFS_ALL_QUOTA_CHKD); + + xfs_qm_init_timelimits(mp, XFS_DQTYPE_USER); + xfs_qm_init_timelimits(mp, XFS_DQTYPE_GROUP); + xfs_qm_init_timelimits(mp, XFS_DQTYPE_PROJ); + + if (XFS_IS_UQUOTA_ON(mp)) + xfs_qm_set_defquota(mp, XFS_DQTYPE_USER, qinf); + if (XFS_IS_GQUOTA_ON(mp)) + xfs_qm_set_defquota(mp, XFS_DQTYPE_GROUP, qinf); + if (XFS_IS_PQUOTA_ON(mp)) + xfs_qm_set_defquota(mp, XFS_DQTYPE_PROJ, qinf); + + qinf->qi_shrinker = shrinker_alloc(SHRINKER_NUMA_AWARE, "xfs-qm:%s", + mp->m_super->s_id); + if (!qinf->qi_shrinker) { + error = -ENOMEM; + goto out_free_inos; + } + + qinf->qi_shrinker->count_objects = xfs_qm_shrink_count; + qinf->qi_shrinker->scan_objects = xfs_qm_shrink_scan; + qinf->qi_shrinker->private_data = qinf; + + shrinker_register(qinf->qi_shrinker); + + xfs_hooks_init(&qinf->qi_mod_ino_dqtrx_hooks); + xfs_hooks_init(&qinf->qi_apply_dqtrx_hooks); + + return 0; + +out_free_inos: + mutex_destroy(&qinf->qi_quotaofflock); + mutex_destroy(&qinf->qi_tree_lock); + xfs_qm_destroy_quotainos(qinf); +out_free_lru: + list_lru_destroy(&qinf->qi_lru); +out_free_qinf: + kfree(qinf); + mp->m_quotainfo = NULL; + return error; +} + +/* + * Gets called when unmounting a filesystem or when all quotas get + * turned off. + * This purges the quota inodes, destroys locks and frees itself. + */ +void +xfs_qm_destroy_quotainfo( + struct xfs_mount *mp) +{ + struct xfs_quotainfo *qi; + + qi = mp->m_quotainfo; + ASSERT(qi != NULL); + + shrinker_free(qi->qi_shrinker); + list_lru_destroy(&qi->qi_lru); + xfs_qm_destroy_quotainos(qi); + mutex_destroy(&qi->qi_tree_lock); + mutex_destroy(&qi->qi_quotaofflock); + kfree(qi); + mp->m_quotainfo = NULL; +} + +static inline enum xfs_metafile_type +xfs_qm_metafile_type( + unsigned int flags) +{ + if (flags & XFS_QMOPT_UQUOTA) + return XFS_METAFILE_USRQUOTA; + else if (flags & XFS_QMOPT_GQUOTA) + return XFS_METAFILE_GRPQUOTA; + return XFS_METAFILE_PRJQUOTA; +} + +/* + * Create an inode and return with a reference already taken, but unlocked + * This is how we create quota inodes + */ +STATIC int +xfs_qm_qino_alloc( + struct xfs_mount *mp, + struct xfs_inode **ipp, + unsigned int flags) +{ + struct xfs_trans *tp; + enum xfs_metafile_type metafile_type = xfs_qm_metafile_type(flags); + int error; + bool need_alloc = true; + + *ipp = NULL; + /* + * With superblock that doesn't have separate pquotino, we + * share an inode between gquota and pquota. If the on-disk + * superblock has GQUOTA and the filesystem is now mounted + * with PQUOTA, just use sb_gquotino for sb_pquotino and + * vice-versa. + */ + if (!xfs_has_pquotino(mp) && + (flags & (XFS_QMOPT_PQUOTA|XFS_QMOPT_GQUOTA))) { + xfs_ino_t ino = NULLFSINO; + + if ((flags & XFS_QMOPT_PQUOTA) && + (mp->m_sb.sb_gquotino != NULLFSINO)) { + ino = mp->m_sb.sb_gquotino; + if (XFS_IS_CORRUPT(mp, + mp->m_sb.sb_pquotino != NULLFSINO)) { + xfs_fs_mark_sick(mp, XFS_SICK_FS_PQUOTA); + return -EFSCORRUPTED; + } + } else if ((flags & XFS_QMOPT_GQUOTA) && + (mp->m_sb.sb_pquotino != NULLFSINO)) { + ino = mp->m_sb.sb_pquotino; + if (XFS_IS_CORRUPT(mp, + mp->m_sb.sb_gquotino != NULLFSINO)) { + xfs_fs_mark_sick(mp, XFS_SICK_FS_GQUOTA); + return -EFSCORRUPTED; + } + } + if (ino != NULLFSINO) { + error = xfs_metafile_iget(mp, ino, metafile_type, ipp); + if (error) + return error; + + mp->m_sb.sb_gquotino = NULLFSINO; + mp->m_sb.sb_pquotino = NULLFSINO; + need_alloc = false; + } + } + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_create, + need_alloc ? XFS_QM_QINOCREATE_SPACE_RES(mp) : 0, + 0, 0, &tp); + if (error) + return error; + + if (need_alloc) { + struct xfs_icreate_args args = { + .mode = S_IFREG, + .flags = XFS_ICREATE_UNLINKABLE, + }; + xfs_ino_t ino; + + error = xfs_dialloc(&tp, &args, &ino); + if (!error) + error = xfs_icreate(tp, ino, &args, ipp); + if (error) { + xfs_trans_cancel(tp); + return error; + } + if (xfs_has_metadir(mp)) + xfs_metafile_set_iflag(tp, *ipp, metafile_type); + } + + /* + * Make the changes in the superblock, and log those too. + * sbfields arg may contain fields other than *QUOTINO; + * VERSIONNUM for example. + */ + spin_lock(&mp->m_sb_lock); + if (flags & XFS_QMOPT_SBVERSION) { + ASSERT(!xfs_has_quota(mp)); + + xfs_add_quota(mp); + mp->m_sb.sb_uquotino = NULLFSINO; + mp->m_sb.sb_gquotino = NULLFSINO; + mp->m_sb.sb_pquotino = NULLFSINO; + + /* qflags will get updated fully _after_ quotacheck */ + mp->m_sb.sb_qflags = mp->m_qflags & XFS_ALL_QUOTA_ACCT; + } + if (flags & XFS_QMOPT_UQUOTA) + mp->m_sb.sb_uquotino = I_INO(*ipp); + else if (flags & XFS_QMOPT_GQUOTA) + mp->m_sb.sb_gquotino = I_INO(*ipp); + else + mp->m_sb.sb_pquotino = I_INO(*ipp); + spin_unlock(&mp->m_sb_lock); + xfs_log_sb(tp); + + error = xfs_trans_commit(tp); + if (error) { + ASSERT(xfs_is_shutdown(mp)); + xfs_alert(mp, "%s failed (error %d)!", __func__, error); + } + if (need_alloc) { + xfs_iunlock(*ipp, XFS_ILOCK_EXCL); + xfs_finish_inode_setup(*ipp); + } + return error; +} + + +STATIC void +xfs_qm_reset_dqcounts( + struct xfs_mount *mp, + struct xfs_buf *bp, + xfs_dqid_t id, + xfs_dqtype_t type) +{ + struct xfs_dqblk *dqb; + int j; + + trace_xfs_reset_dqcounts(bp, _RET_IP_); + + /* + * Reset all counters and timers. They'll be + * started afresh by xfs_qm_quotacheck. + */ +#ifdef DEBUG + j = (int)XFS_FSB_TO_B(mp, XFS_DQUOT_CLUSTER_SIZE_FSB) / + sizeof(struct xfs_dqblk); + ASSERT(mp->m_quotainfo->qi_dqperchunk == j); +#endif + dqb = bp->b_addr; + for (j = 0; j < mp->m_quotainfo->qi_dqperchunk; j++) { + struct xfs_disk_dquot *ddq; + + ddq = (struct xfs_disk_dquot *)&dqb[j]; + + /* + * Do a sanity check, and if needed, repair the dqblk. Don't + * output any warnings because it's perfectly possible to + * find uninitialised dquot blks. See comment in + * xfs_dquot_verify. + */ + if (xfs_dqblk_verify(mp, &dqb[j], id + j) || + (dqb[j].dd_diskdq.d_type & XFS_DQTYPE_REC_MASK) != type) + xfs_dqblk_repair(mp, &dqb[j], id + j, type); + + /* + * Reset type in case we are reusing group quota file for + * project quotas or vice versa + */ + ddq->d_type = type; + ddq->d_bcount = 0; + ddq->d_icount = 0; + ddq->d_rtbcount = 0; + + /* + * dquot id 0 stores the default grace period and the maximum + * warning limit that were set by the administrator, so we + * should not reset them. + */ + if (ddq->d_id != 0) { + ddq->d_btimer = 0; + ddq->d_itimer = 0; + ddq->d_rtbtimer = 0; + ddq->d_bwarns = 0; + ddq->d_iwarns = 0; + ddq->d_rtbwarns = 0; + if (xfs_has_bigtime(mp)) + ddq->d_type |= XFS_DQTYPE_BIGTIME; + } + + if (xfs_has_crc(mp)) { + xfs_update_cksum((char *)&dqb[j], + sizeof(struct xfs_dqblk), + XFS_DQUOT_CRC_OFF); + } + } +} + +STATIC int +xfs_qm_reset_dqcounts_all( + struct xfs_mount *mp, + xfs_dqid_t firstid, + xfs_fsblock_t bno, + xfs_filblks_t blkcnt, + xfs_dqtype_t type, + struct list_head *buffer_list) +{ + struct xfs_buf *bp; + int error = 0; + + ASSERT(blkcnt > 0); + + /* + * Blkcnt arg can be a very big number, and might even be + * larger than the log itself. So, we have to break it up into + * manageable-sized transactions. + * Note that we don't start a permanent transaction here; we might + * not be able to get a log reservation for the whole thing up front, + * and we don't really care to either, because we just discard + * everything if we were to crash in the middle of this loop. + */ + while (blkcnt--) { + error = xfs_trans_read_buf(mp, NULL, mp->m_ddev_targp, + XFS_FSB_TO_DADDR(mp, bno), + mp->m_quotainfo->qi_dqchunklen, 0, &bp, + &xfs_dquot_buf_ops); + + /* + * CRC and validation errors will return a EFSCORRUPTED here. If + * this occurs, re-read without CRC validation so that we can + * repair the damage via xfs_qm_reset_dqcounts(). This process + * will leave a trace in the log indicating corruption has + * been detected. + */ + if (error == -EFSCORRUPTED) { + error = xfs_trans_read_buf(mp, NULL, mp->m_ddev_targp, + XFS_FSB_TO_DADDR(mp, bno), + mp->m_quotainfo->qi_dqchunklen, 0, &bp, + NULL); + } + + if (error) + break; + + /* + * A corrupt buffer might not have a verifier attached, so + * make sure we have the correct one attached before writeback + * occurs. + */ + bp->b_ops = &xfs_dquot_buf_ops; + xfs_qm_reset_dqcounts(mp, bp, firstid, type); + xfs_buf_delwri_queue(bp, buffer_list); + xfs_buf_relse(bp); + + /* goto the next block. */ + bno++; + firstid += mp->m_quotainfo->qi_dqperchunk; + } + + return error; +} + +/* + * Iterate over all allocated dquot blocks in this quota inode, zeroing all + * counters for every chunk of dquots that we find. + */ +STATIC int +xfs_qm_reset_dqcounts_buf( + struct xfs_mount *mp, + struct xfs_inode *qip, + xfs_dqtype_t type, + struct list_head *buffer_list) +{ + struct xfs_bmbt_irec *map; + int i, nmaps; /* number of map entries */ + int error; /* return value */ + xfs_fileoff_t lblkno; + xfs_filblks_t maxlblkcnt; + xfs_dqid_t firstid; + xfs_fsblock_t rablkno; + xfs_filblks_t rablkcnt; + + error = 0; + /* + * This looks racy, but we can't keep an inode lock across a + * trans_reserve. But, this gets called during quotacheck, and that + * happens only at mount time which is single threaded. + */ + if (qip->i_nblocks == 0) + return 0; + + map = kmalloc(XFS_DQITER_MAP_SIZE * sizeof(*map), + GFP_KERNEL | __GFP_NOFAIL); + + lblkno = 0; + maxlblkcnt = XFS_B_TO_FSB(mp, mp->m_super->s_maxbytes); + do { + uint lock_mode; + + nmaps = XFS_DQITER_MAP_SIZE; + /* + * We aren't changing the inode itself. Just changing + * some of its data. No new blocks are added here, and + * the inode is never added to the transaction. + */ + lock_mode = xfs_ilock_data_map_shared(qip); + error = xfs_bmapi_read(qip, lblkno, maxlblkcnt - lblkno, + map, &nmaps, 0); + xfs_iunlock(qip, lock_mode); + if (error) + break; + + ASSERT(nmaps <= XFS_DQITER_MAP_SIZE); + for (i = 0; i < nmaps; i++) { + ASSERT(map[i].br_startblock != DELAYSTARTBLOCK); + ASSERT(map[i].br_blockcount); + + + lblkno += map[i].br_blockcount; + + if (map[i].br_startblock == HOLESTARTBLOCK) + continue; + + firstid = (xfs_dqid_t) map[i].br_startoff * + mp->m_quotainfo->qi_dqperchunk; + /* + * Do a read-ahead on the next extent. + */ + if ((i+1 < nmaps) && + (map[i+1].br_startblock != HOLESTARTBLOCK)) { + rablkcnt = map[i+1].br_blockcount; + rablkno = map[i+1].br_startblock; + while (rablkcnt--) { + xfs_buf_readahead(mp->m_ddev_targp, + XFS_FSB_TO_DADDR(mp, rablkno), + mp->m_quotainfo->qi_dqchunklen, + &xfs_dquot_buf_ops); + rablkno++; + } + } + /* + * Iterate thru all the blks in the extent and + * reset the counters of all the dquots inside them. + */ + error = xfs_qm_reset_dqcounts_all(mp, firstid, + map[i].br_startblock, + map[i].br_blockcount, + type, buffer_list); + if (error) + goto out; + } + } while (nmaps > 0); + +out: + kfree(map); + return error; +} + +/* + * Called by dqusage_adjust in doing a quotacheck. + * + * Given the inode, and a dquot id this updates both the incore dqout as well + * as the buffer copy. This is so that once the quotacheck is done, we can + * just log all the buffers, as opposed to logging numerous updates to + * individual dquots. + */ +STATIC int +xfs_qm_quotacheck_dqadjust( + struct xfs_inode *ip, + xfs_dqtype_t type, + xfs_qcnt_t nblks, + xfs_qcnt_t rtblks) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_dquot *dqp; + xfs_dqid_t id; + int error; + + id = xfs_qm_id_for_quotatype(ip, type); + error = xfs_qm_dqget(mp, id, type, true, &dqp); + if (error) { + /* + * Shouldn't be able to turn off quotas here. + */ + ASSERT(error != -ESRCH); + ASSERT(error != -ENOENT); + return error; + } + + mutex_lock(&dqp->q_qlock); + error = xfs_dquot_attach_buf(NULL, dqp); + if (error) + goto out_unlock; + + trace_xfs_dqadjust(dqp); + + /* + * Adjust the inode count and the block count to reflect this inode's + * resource usage. + */ + dqp->q_ino.count++; + dqp->q_ino.reserved++; + if (nblks) { + dqp->q_blk.count += nblks; + dqp->q_blk.reserved += nblks; + } + if (rtblks) { + dqp->q_rtb.count += rtblks; + dqp->q_rtb.reserved += rtblks; + } + + /* + * Set default limits, adjust timers (since we changed usages) + * + * There are no timers for the default values set in the root dquot. + */ + if (dqp->q_id) { + xfs_qm_adjust_dqlimits(dqp); + xfs_qm_adjust_dqtimers(dqp); + } + + dqp->q_flags |= XFS_DQFLAG_DIRTY; +out_unlock: + mutex_unlock(&dqp->q_qlock); + xfs_qm_dqrele(dqp); + return error; +} + +/* + * callback routine supplied to bulkstat(). Given an inumber, find its + * dquots and update them to account for resources taken by that inode. + */ +/* ARGSUSED */ +STATIC int +xfs_qm_dqusage_adjust( + struct xfs_mount *mp, + struct xfs_trans *tp, + xfs_ino_t ino, + void *data) +{ + struct xfs_inode *ip; + xfs_filblks_t nblks, rtblks; + unsigned int lock_mode; + int error; + + ASSERT(XFS_IS_QUOTA_ON(mp)); + + /* + * rootino must have its resources accounted for, not so with the quota + * inodes. + */ + if (xfs_is_quota_inode(&mp->m_sb, ino)) + return 0; + + /* + * We don't _need_ to take the ilock EXCL here because quotacheck runs + * at mount time and therefore nobody will be racing chown/chproj. + */ + error = xfs_iget(mp, tp, ino, XFS_IGET_DONTCACHE, 0, &ip); + if (error == -EINVAL || error == -ENOENT) + return 0; + if (error) + return error; + + /* + * Reload the incore unlinked list to avoid failure in inodegc. + * Use an unlocked check here because unrecovered unlinked inodes + * should be somewhat rare. + */ + if (xfs_inode_unlinked_incomplete(ip)) { + error = xfs_inode_reload_unlinked(ip); + if (error) { + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); + goto error0; + } + } + + /* Metadata directory files are not accounted to user-visible quotas. */ + if (xfs_is_metadir_inode(ip)) + goto error0; + + ASSERT(ip->i_delayed_blks == 0); + + lock_mode = xfs_ilock_data_map_shared(ip); + if (XFS_IS_REALTIME_INODE(ip)) { + error = xfs_iread_extents(tp, ip, XFS_DATA_FORK); + if (error) { + xfs_iunlock(ip, lock_mode); + goto error0; + } + } + xfs_inode_count_blocks(tp, ip, &nblks, &rtblks); + xfs_iflags_clear(ip, XFS_IQUOTAUNCHECKED); + xfs_iunlock(ip, lock_mode); + + /* + * Add the (disk blocks and inode) resources occupied by this + * inode to its dquots. We do this adjustment in the incore dquot, + * and also copy the changes to its buffer. + * We don't care about putting these changes in a transaction + * envelope because if we crash in the middle of a 'quotacheck' + * we have to start from the beginning anyway. + * Once we're done, we'll log all the dquot bufs. + * + * The *QUOTA_ON checks below may look pretty racy, but quotachecks + * and quotaoffs don't race. (Quotachecks happen at mount time only). + */ + if (XFS_IS_UQUOTA_ON(mp)) { + error = xfs_qm_quotacheck_dqadjust(ip, XFS_DQTYPE_USER, nblks, + rtblks); + if (error) + goto error0; + } + + if (XFS_IS_GQUOTA_ON(mp)) { + error = xfs_qm_quotacheck_dqadjust(ip, XFS_DQTYPE_GROUP, nblks, + rtblks); + if (error) + goto error0; + } + + if (XFS_IS_PQUOTA_ON(mp)) { + error = xfs_qm_quotacheck_dqadjust(ip, XFS_DQTYPE_PROJ, nblks, + rtblks); + if (error) + goto error0; + } + +error0: + xfs_irele(ip); + return error; +} + +STATIC int +xfs_qm_flush_one( + struct xfs_dquot *dqp, + void *data) +{ + struct list_head *buffer_list = data; + struct xfs_buf *bp = NULL; + int error = 0; + + if (!lockref_get_not_dead(&dqp->q_lockref)) + return 0; + + mutex_lock(&dqp->q_qlock); + if (!XFS_DQ_IS_DIRTY(dqp)) + goto out_unlock; + + xfs_qm_dqunpin_wait(dqp); + xfs_dqflock(dqp); + + error = xfs_dquot_use_attached_buf(dqp, &bp); + if (error) + goto out_unlock; + if (!bp) { + error = -EFSCORRUPTED; + goto out_unlock; + } + + error = xfs_qm_dqflush(dqp, bp); + if (!error) + xfs_buf_delwri_queue(bp, buffer_list); + xfs_buf_relse(bp); +out_unlock: + mutex_unlock(&dqp->q_qlock); + xfs_qm_dqrele(dqp); + return error; +} + +/* + * Walk thru all the filesystem inodes and construct a consistent view + * of the disk quota world. If the quotacheck fails, disable quotas. + */ +STATIC int +xfs_qm_quotacheck( + xfs_mount_t *mp) +{ + int error, error2; + uint flags; + LIST_HEAD (buffer_list); + struct xfs_inode *uip = mp->m_quotainfo->qi_uquotaip; + struct xfs_inode *gip = mp->m_quotainfo->qi_gquotaip; + struct xfs_inode *pip = mp->m_quotainfo->qi_pquotaip; + + flags = 0; + + ASSERT(uip || gip || pip); + ASSERT(XFS_IS_QUOTA_ON(mp)); + + xfs_notice(mp, "Quotacheck needed: Please wait."); + + /* + * First we go thru all the dquots on disk, USR and GRP/PRJ, and reset + * their counters to zero. We need a clean slate. + * We don't log our changes till later. + */ + if (uip) { + error = xfs_qm_reset_dqcounts_buf(mp, uip, XFS_DQTYPE_USER, + &buffer_list); + if (error) + goto error_return; + flags |= XFS_UQUOTA_CHKD; + } + + if (gip) { + error = xfs_qm_reset_dqcounts_buf(mp, gip, XFS_DQTYPE_GROUP, + &buffer_list); + if (error) + goto error_return; + flags |= XFS_GQUOTA_CHKD; + } + + if (pip) { + error = xfs_qm_reset_dqcounts_buf(mp, pip, XFS_DQTYPE_PROJ, + &buffer_list); + if (error) + goto error_return; + flags |= XFS_PQUOTA_CHKD; + } + + xfs_set_quotacheck_running(mp); + error = xfs_iwalk_threaded(mp, 0, 0, xfs_qm_dqusage_adjust, 0, true, + NULL); + xfs_clear_quotacheck_running(mp); + + /* + * On error, the inode walk may have partially populated the dquot + * caches. We must purge them before disabling quota and tearing down + * the quotainfo, or else the dquots will leak. + */ + if (error) + goto error_purge; + + /* + * We've made all the changes that we need to make incore. Flush them + * down to disk buffers if everything was updated successfully. + */ + if (XFS_IS_UQUOTA_ON(mp)) { + error = xfs_qm_dquot_walk(mp, XFS_DQTYPE_USER, xfs_qm_flush_one, + &buffer_list); + } + if (XFS_IS_GQUOTA_ON(mp)) { + error2 = xfs_qm_dquot_walk(mp, XFS_DQTYPE_GROUP, xfs_qm_flush_one, + &buffer_list); + if (!error) + error = error2; + } + if (XFS_IS_PQUOTA_ON(mp)) { + error2 = xfs_qm_dquot_walk(mp, XFS_DQTYPE_PROJ, xfs_qm_flush_one, + &buffer_list); + if (!error) + error = error2; + } + + error2 = xfs_buf_delwri_submit(&buffer_list); + if (!error) + error = error2; + + /* + * We can get this error if we couldn't do a dquot allocation inside + * xfs_qm_dqusage_adjust (via bulkstat). We don't care about the + * dirty dquots that might be cached, we just want to get rid of them + * and turn quotaoff. The dquots won't be attached to any of the inodes + * at this point (because we intentionally didn't in dqget_noattach). + */ + if (error) + goto error_purge; + + /* + * If one type of quotas is off, then it will lose its + * quotachecked status, since we won't be doing accounting for + * that type anymore. + */ + mp->m_qflags &= ~XFS_ALL_QUOTA_CHKD; + mp->m_qflags |= flags; + +error_return: + xfs_buf_delwri_cancel(&buffer_list); + + if (error) { + xfs_warn(mp, + "Quotacheck: Unsuccessful (Error %d): Disabling quotas.", + error); + /* + * We must turn off quotas. + */ + ASSERT(mp->m_quotainfo != NULL); + xfs_qm_destroy_quotainfo(mp); + if (xfs_mount_reset_sbqflags(mp)) { + xfs_warn(mp, + "Quotacheck: Failed to reset quota flags."); + } + xfs_fs_mark_sick(mp, XFS_SICK_FS_QUOTACHECK); + } else { + xfs_notice(mp, "Quotacheck: Done."); + xfs_fs_mark_healthy(mp, XFS_SICK_FS_QUOTACHECK); + } + + return error; + +error_purge: + /* + * On error, we may have inodes queued for inactivation. This may try + * to attach dquots to the inode before running cleanup operations on + * the inode and this can race with the xfs_qm_destroy_quotainfo() call + * below that frees mp->m_quotainfo. To avoid this race, flush all the + * pending inodegc operations before we purge the dquots from memory, + * ensuring that background inactivation is idle whilst we turn off + * quotas. + */ + xfs_inodegc_flush(mp); + xfs_qm_dqpurge_all(mp); + goto error_return; + +} + +/* + * This is called from xfs_mountfs to start quotas and initialize all + * necessary data structures like quotainfo. This is also responsible for + * running a quotacheck as necessary. We are guaranteed that the superblock + * is consistently read in at this point. + * + * If we fail here, the mount will continue with quota turned off. We don't + * need to inidicate success or failure at all. + */ +void +xfs_qm_mount_quotas( + struct xfs_mount *mp) +{ + int error = 0; + uint sbf; + + /* + * If quotas on realtime volumes is not supported, disable quotas + * immediately. We only support rtquota if rtgroups are enabled to + * avoid problems with older kernels. + */ + if (mp->m_sb.sb_rextents && + (!xfs_has_rtgroups(mp) || xfs_has_zoned(mp))) { + xfs_notice(mp, "Cannot turn on quotas for realtime filesystem"); + mp->m_qflags = 0; + goto write_changes; + } + + ASSERT(XFS_IS_QUOTA_ON(mp)); + + /* + * Allocate the quotainfo structure inside the mount struct, and + * create quotainode(s), and change/rev superblock if necessary. + */ + error = xfs_qm_init_quotainfo(mp); + if (error) { + /* + * We must turn off quotas. + */ + ASSERT(mp->m_quotainfo == NULL); + mp->m_qflags = 0; + goto write_changes; + } + /* + * If any of the quotas are not consistent, do a quotacheck. + */ + if (XFS_QM_NEED_QUOTACHECK(mp)) { + error = xfs_qm_quotacheck(mp); + if (error) { + /* Quotacheck failed and disabled quotas. */ + return; + } + } + /* + * If one type of quotas is off, then it will lose its + * quotachecked status, since we won't be doing accounting for + * that type anymore. + */ + if (!XFS_IS_UQUOTA_ON(mp)) + mp->m_qflags &= ~XFS_UQUOTA_CHKD; + if (!XFS_IS_GQUOTA_ON(mp)) + mp->m_qflags &= ~XFS_GQUOTA_CHKD; + if (!XFS_IS_PQUOTA_ON(mp)) + mp->m_qflags &= ~XFS_PQUOTA_CHKD; + + write_changes: + /* + * We actually don't have to acquire the m_sb_lock at all. + * This can only be called from mount, and that's single threaded. XXX + */ + spin_lock(&mp->m_sb_lock); + sbf = mp->m_sb.sb_qflags; + mp->m_sb.sb_qflags = mp->m_qflags & XFS_MOUNT_QUOTA_ALL; + spin_unlock(&mp->m_sb_lock); + + if (sbf != (mp->m_qflags & XFS_MOUNT_QUOTA_ALL)) { + if (xfs_sync_sb(mp, false)) { + /* + * We could only have been turning quotas off. + * We aren't in very good shape actually because + * the incore structures are convinced that quotas are + * off, but the on disk superblock doesn't know that ! + */ + ASSERT(!(XFS_IS_QUOTA_ON(mp))); + xfs_alert(mp, "%s: Superblock update failed!", + __func__); + } + } + + if (error) { + xfs_warn(mp, "Failed to initialize disk quotas, err %d.", error); + return; + } +} + +/* + * Load the inode for a given type of quota, assuming that the sb fields have + * been sorted out. This is not true when switching quota types on a V4 + * filesystem, so do not use this function for that. + * + * Returns -ENOENT if the quota inode field is NULLFSINO; 0 and an inode on + * success; or a negative errno. + */ +int +xfs_qm_qino_load( + struct xfs_mount *mp, + xfs_dqtype_t type, + struct xfs_inode **ipp) +{ + struct xfs_trans *tp; + struct xfs_inode *dp = NULL; + int error; + + tp = xfs_trans_alloc_empty(mp); + if (xfs_has_metadir(mp)) { + error = xfs_dqinode_load_parent(tp, &dp); + if (error) + goto out_cancel; + } + + error = xfs_dqinode_load(tp, dp, type, ipp); + if (dp) + xfs_irele(dp); +out_cancel: + xfs_trans_cancel(tp); + return error; +} + +/* + * This is called after the superblock has been read in and we're ready to + * iget the quota inodes. + */ +STATIC int +xfs_qm_init_quotainos( + xfs_mount_t *mp) +{ + struct xfs_inode *uip = NULL; + struct xfs_inode *gip = NULL; + struct xfs_inode *pip = NULL; + int error; + uint flags = 0; + + ASSERT(mp->m_quotainfo); + + /* + * Get the uquota and gquota inodes + */ + if (xfs_has_quota(mp)) { + if (XFS_IS_UQUOTA_ON(mp) && + mp->m_sb.sb_uquotino != NULLFSINO) { + ASSERT(mp->m_sb.sb_uquotino > 0); + error = xfs_qm_qino_load(mp, XFS_DQTYPE_USER, &uip); + if (error) + return error; + } + if (XFS_IS_GQUOTA_ON(mp) && + mp->m_sb.sb_gquotino != NULLFSINO) { + ASSERT(mp->m_sb.sb_gquotino > 0); + error = xfs_qm_qino_load(mp, XFS_DQTYPE_GROUP, &gip); + if (error) + goto error_rele; + } + if (XFS_IS_PQUOTA_ON(mp) && + mp->m_sb.sb_pquotino != NULLFSINO) { + ASSERT(mp->m_sb.sb_pquotino > 0); + error = xfs_qm_qino_load(mp, XFS_DQTYPE_PROJ, &pip); + if (error) + goto error_rele; + } + } else { + flags |= XFS_QMOPT_SBVERSION; + } + + /* + * Create the three inodes, if they don't exist already. The changes + * made above will get added to a transaction and logged in one of + * the qino_alloc calls below. If the device is readonly, + * temporarily switch to read-write to do this. + */ + if (XFS_IS_UQUOTA_ON(mp) && uip == NULL) { + error = xfs_qm_qino_alloc(mp, &uip, + flags | XFS_QMOPT_UQUOTA); + if (error) + goto error_rele; + + flags &= ~XFS_QMOPT_SBVERSION; + } + if (XFS_IS_GQUOTA_ON(mp) && gip == NULL) { + error = xfs_qm_qino_alloc(mp, &gip, + flags | XFS_QMOPT_GQUOTA); + if (error) + goto error_rele; + + flags &= ~XFS_QMOPT_SBVERSION; + } + if (XFS_IS_PQUOTA_ON(mp) && pip == NULL) { + error = xfs_qm_qino_alloc(mp, &pip, + flags | XFS_QMOPT_PQUOTA); + if (error) + goto error_rele; + } + + mp->m_quotainfo->qi_uquotaip = uip; + mp->m_quotainfo->qi_gquotaip = gip; + mp->m_quotainfo->qi_pquotaip = pip; + + return 0; + +error_rele: + if (uip) + xfs_irele(uip); + if (gip) + xfs_irele(gip); + if (pip) + xfs_irele(pip); + return error; +} + +STATIC void +xfs_qm_dqfree_one( + struct xfs_dquot *dqp) +{ + struct xfs_mount *mp = dqp->q_mount; + struct xfs_quotainfo *qi = mp->m_quotainfo; + + mutex_lock(&qi->qi_tree_lock); + radix_tree_delete(xfs_dquot_tree(qi, xfs_dquot_type(dqp)), dqp->q_id); + + qi->qi_dquots--; + mutex_unlock(&qi->qi_tree_lock); + + xfs_qm_dqdestroy(dqp); +} + +/* --------------- utility functions for vnodeops ---------------- */ + + +/* + * Given an inode, a uid, gid and prid make sure that we have + * allocated relevant dquot(s) on disk, and that we won't exceed inode + * quotas by creating this file. + * This also attaches dquot(s) to the given inode after locking it, + * and returns the dquots corresponding to the uid and/or gid. + * + * in : inode (unlocked) + * out : udquot, gdquot with references taken and unlocked + */ +int +xfs_qm_vop_dqalloc( + struct xfs_inode *ip, + kuid_t uid, + kgid_t gid, + prid_t prid, + uint flags, + struct xfs_dquot **O_udqpp, + struct xfs_dquot **O_gdqpp, + struct xfs_dquot **O_pdqpp) +{ + struct xfs_mount *mp = ip->i_mount; + struct inode *inode = VFS_I(ip); + struct user_namespace *user_ns = inode->i_sb->s_user_ns; + struct xfs_dquot *uq = NULL; + struct xfs_dquot *gq = NULL; + struct xfs_dquot *pq = NULL; + int error; + + if (!XFS_IS_QUOTA_ON(mp)) + return 0; + + ASSERT(!xfs_is_metadir_inode(ip)); + + if ((flags & XFS_QMOPT_INHERIT) && XFS_INHERIT_GID(ip)) + gid = inode->i_gid; + + /* + * Attach the dquot(s) to this inode, doing a dquot allocation + * if necessary. The dquot(s) will not be locked. + */ + if (XFS_NOT_DQATTACHED(mp, ip)) { + xfs_ilock(ip, XFS_ILOCK_EXCL); + error = xfs_qm_dqattach_locked(ip, true); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + if (error) + return error; + } + + if ((flags & XFS_QMOPT_UQUOTA) && XFS_IS_UQUOTA_ON(mp)) { + ASSERT(O_udqpp); + if (!uid_eq(inode->i_uid, uid)) { + error = xfs_qm_dqget(mp, from_kuid(user_ns, uid), + XFS_DQTYPE_USER, true, &uq); + if (error) { + ASSERT(error != -ENOENT); + return error; + } + } else { + /* + * Take an extra reference, because we'll return + * this to caller + */ + ASSERT(ip->i_udquot); + uq = xfs_qm_dqhold(ip->i_udquot); + } + } + if ((flags & XFS_QMOPT_GQUOTA) && XFS_IS_GQUOTA_ON(mp)) { + ASSERT(O_gdqpp); + if (!gid_eq(inode->i_gid, gid)) { + error = xfs_qm_dqget(mp, from_kgid(user_ns, gid), + XFS_DQTYPE_GROUP, true, &gq); + if (error) { + ASSERT(error != -ENOENT); + goto error_rele; + } + } else { + ASSERT(ip->i_gdquot); + gq = xfs_qm_dqhold(ip->i_gdquot); + } + } + if ((flags & XFS_QMOPT_PQUOTA) && XFS_IS_PQUOTA_ON(mp)) { + ASSERT(O_pdqpp); + if (ip->i_projid != prid) { + error = xfs_qm_dqget(mp, prid, + XFS_DQTYPE_PROJ, true, &pq); + if (error) { + ASSERT(error != -ENOENT); + goto error_rele; + } + } else { + ASSERT(ip->i_pdquot); + pq = xfs_qm_dqhold(ip->i_pdquot); + } + } + trace_xfs_dquot_dqalloc(ip); + + if (O_udqpp) + *O_udqpp = uq; + else + xfs_qm_dqrele(uq); + if (O_gdqpp) + *O_gdqpp = gq; + else + xfs_qm_dqrele(gq); + if (O_pdqpp) + *O_pdqpp = pq; + else + xfs_qm_dqrele(pq); + return 0; + +error_rele: + xfs_qm_dqrele(gq); + xfs_qm_dqrele(uq); + return error; +} + +/* + * Actually transfer ownership, and do dquot modifications. + * These were already reserved. + */ +struct xfs_dquot * +xfs_qm_vop_chown( + struct xfs_trans *tp, + struct xfs_inode *ip, + struct xfs_dquot **IO_olddq, + struct xfs_dquot *newdq) +{ + struct xfs_dquot *prevdq; + xfs_filblks_t dblocks, rblocks; + bool isrt = XFS_IS_REALTIME_INODE(ip); + + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + ASSERT(XFS_IS_QUOTA_ON(ip->i_mount)); + ASSERT(!xfs_is_metadir_inode(ip)); + + /* old dquot */ + prevdq = *IO_olddq; + ASSERT(prevdq); + ASSERT(prevdq != newdq); + + xfs_inode_count_blocks(tp, ip, &dblocks, &rblocks); + + xfs_trans_mod_ino_dquot(tp, ip, prevdq, XFS_TRANS_DQ_BCOUNT, + -(xfs_qcnt_t)dblocks); + xfs_trans_mod_ino_dquot(tp, ip, prevdq, XFS_TRANS_DQ_RTBCOUNT, + -(xfs_qcnt_t)rblocks); + xfs_trans_mod_ino_dquot(tp, ip, prevdq, XFS_TRANS_DQ_ICOUNT, -1); + + /* the sparkling new dquot */ + xfs_trans_mod_ino_dquot(tp, ip, newdq, XFS_TRANS_DQ_BCOUNT, dblocks); + xfs_trans_mod_ino_dquot(tp, ip, newdq, XFS_TRANS_DQ_RTBCOUNT, rblocks); + xfs_trans_mod_ino_dquot(tp, ip, newdq, XFS_TRANS_DQ_ICOUNT, 1); + + /* + * Back when we made quota reservations for the chown, we reserved the + * ondisk blocks + delalloc blocks with the new dquot. Now that we've + * switched the dquots, decrease the new dquot's block reservation + * (having already bumped up the real counter) so that we don't have + * any reservation to give back when we commit. + */ + xfs_trans_mod_dquot(tp, newdq, + isrt ? XFS_TRANS_DQ_RES_RTBLKS : XFS_TRANS_DQ_RES_BLKS, + -ip->i_delayed_blks); + + /* + * Give the incore reservation for delalloc blocks back to the old + * dquot. We don't normally handle delalloc quota reservations + * transactionally, so just lock the dquot and subtract from the + * reservation. Dirty the transaction because it's too late to turn + * back now. + */ + tp->t_flags |= XFS_TRANS_DIRTY; + mutex_lock(&prevdq->q_qlock); + if (isrt) { + ASSERT(prevdq->q_rtb.reserved >= ip->i_delayed_blks); + prevdq->q_rtb.reserved -= ip->i_delayed_blks; + } else { + ASSERT(prevdq->q_blk.reserved >= ip->i_delayed_blks); + prevdq->q_blk.reserved -= ip->i_delayed_blks; + } + mutex_unlock(&prevdq->q_qlock); + + /* + * Take an extra reference, because the inode is going to keep + * this dquot pointer even after the trans_commit. + */ + *IO_olddq = xfs_qm_dqhold(newdq); + + return prevdq; +} + +int +xfs_qm_vop_rename_dqattach( + struct xfs_inode **i_tab) +{ + struct xfs_mount *mp = i_tab[0]->i_mount; + int i; + + if (!XFS_IS_QUOTA_ON(mp)) + return 0; + + for (i = 0; (i < 4 && i_tab[i]); i++) { + struct xfs_inode *ip = i_tab[i]; + int error; + + /* + * Watch out for duplicate entries in the table. + */ + if (i == 0 || ip != i_tab[i-1]) { + if (XFS_NOT_DQATTACHED(mp, ip)) { + error = xfs_qm_dqattach(ip); + if (error) + return error; + } + } + } + return 0; +} + +void +xfs_qm_vop_create_dqattach( + struct xfs_trans *tp, + struct xfs_inode *ip, + struct xfs_dquot *udqp, + struct xfs_dquot *gdqp, + struct xfs_dquot *pdqp) +{ + struct xfs_mount *mp = tp->t_mountp; + + if (!XFS_IS_QUOTA_ON(mp)) + return; + + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + ASSERT(!xfs_is_metadir_inode(ip)); + + if (udqp && XFS_IS_UQUOTA_ON(mp)) { + ASSERT(ip->i_udquot == NULL); + ASSERT(i_uid_read(VFS_I(ip)) == udqp->q_id); + + ip->i_udquot = xfs_qm_dqhold(udqp); + } + if (gdqp && XFS_IS_GQUOTA_ON(mp)) { + ASSERT(ip->i_gdquot == NULL); + ASSERT(i_gid_read(VFS_I(ip)) == gdqp->q_id); + + ip->i_gdquot = xfs_qm_dqhold(gdqp); + } + if (pdqp && XFS_IS_PQUOTA_ON(mp)) { + ASSERT(ip->i_pdquot == NULL); + ASSERT(ip->i_projid == pdqp->q_id); + + ip->i_pdquot = xfs_qm_dqhold(pdqp); + } + + xfs_trans_mod_dquot_byino(tp, ip, XFS_TRANS_DQ_ICOUNT, 1); +} + +/* Decide if this inode's dquot is near an enforcement boundary. */ +bool +xfs_inode_near_dquot_enforcement( + struct xfs_inode *ip, + xfs_dqtype_t type) +{ + struct xfs_dquot *dqp; + struct xfs_dquot_res *res; + struct xfs_dquot_pre *pre; + int64_t freesp; + + /* We only care for quotas that are enabled and enforced. */ + dqp = xfs_inode_dquot(ip, type); + if (!dqp || !xfs_dquot_is_enforced(dqp)) + return false; + + if (xfs_dquot_res_over_limits(&dqp->q_ino) || + xfs_dquot_res_over_limits(&dqp->q_blk) || + xfs_dquot_res_over_limits(&dqp->q_rtb)) + return true; + + if (XFS_IS_REALTIME_INODE(ip)) { + res = &dqp->q_rtb; + pre = &dqp->q_rtb_prealloc; + } else { + res = &dqp->q_blk; + pre = &dqp->q_blk_prealloc; + } + + /* For space on the data device, check the various thresholds. */ + if (!pre->q_prealloc_hi_wmark) + return false; + + if (res->reserved < pre->q_prealloc_lo_wmark) + return false; + + if (res->reserved >= pre->q_prealloc_hi_wmark) + return true; + + freesp = pre->q_prealloc_hi_wmark - res->reserved; + if (freesp < pre->q_low_space[XFS_QLOWSP_5_PCNT]) + return true; + + return false; +} diff --git a/fs/xfs/xfs_qm.h b/fs/xfs/xfs_qm.h new file mode 100644 index 000000000..e88ed6ad0 --- /dev/null +++ b/fs/xfs/xfs_qm.h @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_QM_H__ +#define __XFS_QM_H__ + +#include "xfs_dquot_item.h" +#include "xfs_dquot.h" + +struct xfs_inode; + +extern struct kmem_cache *xfs_dqtrx_cache; + +/* + * Number of bmaps that we ask from bmapi when doing a quotacheck. + * We make this restriction to keep the memory usage to a minimum. + */ +#define XFS_DQITER_MAP_SIZE 10 + +#define XFS_IS_DQUOT_UNINITIALIZED(dqp) ( \ + !dqp->q_blk.hardlimit && \ + !dqp->q_blk.softlimit && \ + !dqp->q_rtb.hardlimit && \ + !dqp->q_rtb.softlimit && \ + !dqp->q_ino.hardlimit && \ + !dqp->q_ino.softlimit && \ + !dqp->q_blk.count && \ + !dqp->q_rtb.count && \ + !dqp->q_ino.count) + +struct xfs_quota_limits { + xfs_qcnt_t hard; /* default hard limit */ + xfs_qcnt_t soft; /* default soft limit */ + time64_t time; /* limit for timers */ +}; + +/* Defaults for each quota type: time limits, warn limits, usage limits */ +struct xfs_def_quota { + struct xfs_quota_limits blk; + struct xfs_quota_limits ino; + struct xfs_quota_limits rtb; +}; + +/* + * Various quota information for individual filesystems. + * The mount structure keeps a pointer to this. + */ +struct xfs_quotainfo { + struct radix_tree_root qi_uquota_tree; + struct radix_tree_root qi_gquota_tree; + struct radix_tree_root qi_pquota_tree; + struct mutex qi_tree_lock; + struct xfs_inode *qi_uquotaip; /* user quota inode */ + struct xfs_inode *qi_gquotaip; /* group quota inode */ + struct xfs_inode *qi_pquotaip; /* project quota inode */ + struct xfs_inode *qi_dirip; /* quota metadir */ + struct list_lru qi_lru; + uint64_t qi_dquots; + struct mutex qi_quotaofflock;/* to serialize quotaoff */ + xfs_filblks_t qi_dqchunklen; /* # BBs in a chunk of dqs */ + uint qi_dqperchunk; /* # ondisk dq in above chunk */ + struct xfs_def_quota qi_usr_default; + struct xfs_def_quota qi_grp_default; + struct xfs_def_quota qi_prj_default; + struct shrinker *qi_shrinker; + + /* Minimum and maximum quota expiration timestamp values. */ + time64_t qi_expiry_min; + time64_t qi_expiry_max; + + /* Hook to feed quota counter updates to an active online repair. */ + struct xfs_hooks qi_mod_ino_dqtrx_hooks; + struct xfs_hooks qi_apply_dqtrx_hooks; +}; + +static inline struct radix_tree_root * +xfs_dquot_tree( + struct xfs_quotainfo *qi, + xfs_dqtype_t type) +{ + switch (type) { + case XFS_DQTYPE_USER: + return &qi->qi_uquota_tree; + case XFS_DQTYPE_GROUP: + return &qi->qi_gquota_tree; + case XFS_DQTYPE_PROJ: + return &qi->qi_pquota_tree; + default: + ASSERT(0); + } + return NULL; +} + +static inline struct xfs_inode * +xfs_quota_inode(struct xfs_mount *mp, xfs_dqtype_t type) +{ + switch (type) { + case XFS_DQTYPE_USER: + return mp->m_quotainfo->qi_uquotaip; + case XFS_DQTYPE_GROUP: + return mp->m_quotainfo->qi_gquotaip; + case XFS_DQTYPE_PROJ: + return mp->m_quotainfo->qi_pquotaip; + default: + ASSERT(0); + } + return NULL; +} + +/* + * Parameters for tracking dqtrx changes on behalf of an inode. The hook + * function arg parameter is the field being updated. + */ +struct xfs_mod_ino_dqtrx_params { + uintptr_t tx_id; + xfs_ino_t ino; + xfs_dqtype_t q_type; + xfs_dqid_t q_id; + int64_t delta; +}; + +extern void xfs_trans_mod_dquot(struct xfs_trans *tp, struct xfs_dquot *dqp, + uint field, int64_t delta); +extern void xfs_trans_dqjoin(struct xfs_trans *, struct xfs_dquot *); +extern void xfs_trans_log_dquot(struct xfs_trans *, struct xfs_dquot *); + +/* + * We keep the usr, grp, and prj dquots separately so that locking will be + * easier to do at commit time. All transactions that we know of at this point + * affect no more than two dquots of one type. Hence, the TRANS_MAXDQS value. + */ +enum { + XFS_QM_TRANS_USR = 0, + XFS_QM_TRANS_GRP, + XFS_QM_TRANS_PRJ, + XFS_QM_TRANS_DQTYPES +}; +#define XFS_QM_TRANS_MAXDQS 5 +struct xfs_dquot_acct { + struct xfs_dqtrx dqs[XFS_QM_TRANS_DQTYPES][XFS_QM_TRANS_MAXDQS]; +}; + +/* + * Users are allowed to have a usage exceeding their softlimit for + * a period this long. + */ +#define XFS_QM_BTIMELIMIT (7 * 24*60*60) /* 1 week */ +#define XFS_QM_RTBTIMELIMIT (7 * 24*60*60) /* 1 week */ +#define XFS_QM_ITIMELIMIT (7 * 24*60*60) /* 1 week */ + +extern void xfs_qm_destroy_quotainfo(struct xfs_mount *); + +/* quota ops */ +extern int xfs_qm_scall_trunc_qfiles(struct xfs_mount *, uint); +extern int xfs_qm_scall_getquota(struct xfs_mount *mp, + xfs_dqid_t id, + xfs_dqtype_t type, + struct qc_dqblk *dst); +extern int xfs_qm_scall_getquota_next(struct xfs_mount *mp, + xfs_dqid_t *id, + xfs_dqtype_t type, + struct qc_dqblk *dst); +extern int xfs_qm_scall_setqlim(struct xfs_mount *mp, + xfs_dqid_t id, + xfs_dqtype_t type, + struct qc_dqblk *newlim); +extern int xfs_qm_scall_quotaon(struct xfs_mount *, uint); +extern int xfs_qm_scall_quotaoff(struct xfs_mount *, uint); + +static inline struct xfs_def_quota * +xfs_get_defquota(struct xfs_quotainfo *qi, xfs_dqtype_t type) +{ + switch (type) { + case XFS_DQTYPE_USER: + return &qi->qi_usr_default; + case XFS_DQTYPE_GROUP: + return &qi->qi_grp_default; + case XFS_DQTYPE_PROJ: + return &qi->qi_prj_default; + default: + ASSERT(0); + return NULL; + } +} + +int xfs_qm_qino_load(struct xfs_mount *mp, xfs_dqtype_t type, + struct xfs_inode **ipp); + +#endif /* __XFS_QM_H__ */ diff --git a/fs/xfs/xfs_qm_bhv.c b/fs/xfs/xfs_qm_bhv.c new file mode 100644 index 000000000..a094b8252 --- /dev/null +++ b/fs/xfs/xfs_qm_bhv.c @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2006 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_quota.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_qm.h" + + +STATIC void +xfs_fill_statvfs_from_dquot( + struct kstatfs *statp, + struct xfs_inode *ip, + struct xfs_dquot *dqp) +{ + struct xfs_dquot_res *blkres = &dqp->q_blk; + uint64_t limit; + + if (XFS_IS_REALTIME_MOUNT(ip->i_mount) && + (ip->i_diflags & (XFS_DIFLAG_RTINHERIT | XFS_DIFLAG_REALTIME))) + blkres = &dqp->q_rtb; + + limit = blkres->softlimit ? + blkres->softlimit : + blkres->hardlimit; + if (limit) { + uint64_t remaining = 0; + + if (limit > blkres->reserved) + remaining = limit - blkres->reserved; + + statp->f_blocks = min(statp->f_blocks, limit); + statp->f_bfree = min(statp->f_bfree, remaining); + } + + limit = dqp->q_ino.softlimit ? + dqp->q_ino.softlimit : + dqp->q_ino.hardlimit; + if (limit) { + uint64_t remaining = 0; + + if (limit > dqp->q_ino.reserved) + remaining = limit - dqp->q_ino.reserved; + + statp->f_files = min(statp->f_files, limit); + statp->f_ffree = min(statp->f_ffree, remaining); + } +} + + +/* + * Directory tree accounting is implemented using project quotas, where + * the project identifier is inherited from parent directories. + * A statvfs (df, etc.) of a directory that is using project quota should + * return a statvfs of the project, not the entire filesystem. + * This makes such trees appear as if they are filesystems in themselves. + */ +void +xfs_qm_statvfs( + struct xfs_inode *ip, + struct kstatfs *statp) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_dquot *dqp; + + if (!xfs_qm_dqget(mp, ip->i_projid, XFS_DQTYPE_PROJ, false, &dqp)) { + mutex_lock(&dqp->q_qlock); + xfs_fill_statvfs_from_dquot(statp, ip, dqp); + mutex_unlock(&dqp->q_qlock); + xfs_qm_dqrele(dqp); + } +} + +STATIC int +xfs_qm_validate_state_change( + struct xfs_mount *mp, + uint uqd, + uint gqd, + uint pqd) +{ + int state; + + /* Is quota state changing? */ + state = ((uqd && !XFS_IS_UQUOTA_ON(mp)) || + (!uqd && XFS_IS_UQUOTA_ON(mp)) || + (gqd && !XFS_IS_GQUOTA_ON(mp)) || + (!gqd && XFS_IS_GQUOTA_ON(mp)) || + (pqd && !XFS_IS_PQUOTA_ON(mp)) || + (!pqd && XFS_IS_PQUOTA_ON(mp))); + + return state && + (xfs_dev_is_read_only(mp, "changing quota state") || + xfs_has_norecovery(mp)); +} + +int +xfs_qm_newmount( + xfs_mount_t *mp, + uint *needquotamount, + uint *quotaflags) +{ + uint quotaondisk; + uint uquotaondisk = 0, gquotaondisk = 0, pquotaondisk = 0; + + quotaondisk = xfs_has_quota(mp) && + (mp->m_sb.sb_qflags & XFS_ALL_QUOTA_ACCT); + + if (quotaondisk) { + uquotaondisk = mp->m_sb.sb_qflags & XFS_UQUOTA_ACCT; + pquotaondisk = mp->m_sb.sb_qflags & XFS_PQUOTA_ACCT; + gquotaondisk = mp->m_sb.sb_qflags & XFS_GQUOTA_ACCT; + } + + /* + * If the device itself is read-only and/or in norecovery + * mode, we can't allow the user to change the state of + * quota on the mount - this would generate a transaction + * on the ro device, which would lead to an I/O error and + * shutdown. + */ + + if (xfs_qm_validate_state_change(mp, uquotaondisk, + gquotaondisk, pquotaondisk)) { + + if (xfs_has_metadir(mp)) + xfs_warn(mp, + "metadir enabled, please mount without any quota mount options"); + else + xfs_warn(mp, "please mount with%s%s%s%s.", + (!quotaondisk ? "out quota" : ""), + (uquotaondisk ? " usrquota" : ""), + (gquotaondisk ? " grpquota" : ""), + (pquotaondisk ? " prjquota" : "")); + return -EPERM; + } + + if (XFS_IS_QUOTA_ON(mp) || quotaondisk) { + /* + * Call mount_quotas at this point only if we won't have to do + * a quotacheck. + */ + if (quotaondisk && !XFS_QM_NEED_QUOTACHECK(mp)) { + /* + * If an error occurred, qm_mount_quotas code + * has already disabled quotas. So, just finish + * mounting, and get on with the boring life + * without disk quotas. + */ + xfs_qm_mount_quotas(mp); + } else { + /* + * Clear the quota flags, but remember them. This + * is so that the quota code doesn't get invoked + * before we're ready. This can happen when an + * inode goes inactive and wants to free blocks, + * or via xfs_log_mount_finish. + */ + *needquotamount = true; + *quotaflags = mp->m_qflags; + mp->m_qflags = 0; + } + } + + return 0; +} + +/* + * If the sysadmin didn't provide any quota mount options, restore the quota + * accounting and enforcement state from the ondisk superblock. Only do this + * for metadir filesystems because this is a behavior change. + */ +void +xfs_qm_resume_quotaon( + struct xfs_mount *mp) +{ + if (!xfs_has_metadir(mp)) + return; + if (xfs_has_norecovery(mp)) + return; + + mp->m_qflags = mp->m_sb.sb_qflags & (XFS_ALL_QUOTA_ACCT | + XFS_ALL_QUOTA_ENFD); +} diff --git a/fs/xfs/xfs_qm_syscalls.c b/fs/xfs/xfs_qm_syscalls.c new file mode 100644 index 000000000..21a784986 --- /dev/null +++ b/fs/xfs/xfs_qm_syscalls.c @@ -0,0 +1,546 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ + + +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_sb.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_quota.h" +#include "xfs_qm.h" +#include "xfs_icache.h" + +int +xfs_qm_scall_quotaoff( + xfs_mount_t *mp, + uint flags) +{ + /* + * No file system can have quotas enabled on disk but not in core. + * Note that quota utilities (like quotaoff) _expect_ + * errno == -EEXIST here. + */ + if ((mp->m_qflags & flags) == 0) + return -EEXIST; + + /* + * We do not support actually turning off quota accounting any more. + * Just log a warning and ignore the accounting related flags. + */ + if (flags & XFS_ALL_QUOTA_ACCT) + xfs_info(mp, "disabling of quota accounting not supported."); + + mutex_lock(&mp->m_quotainfo->qi_quotaofflock); + mp->m_qflags &= ~(flags & XFS_ALL_QUOTA_ENFD); + spin_lock(&mp->m_sb_lock); + mp->m_sb.sb_qflags = mp->m_qflags; + spin_unlock(&mp->m_sb_lock); + mutex_unlock(&mp->m_quotainfo->qi_quotaofflock); + + /* XXX what to do if error ? Revert back to old vals incore ? */ + return xfs_sync_sb(mp, false); +} + +STATIC int +xfs_qm_scall_trunc_qfile( + struct xfs_mount *mp, + xfs_dqtype_t type) +{ + struct xfs_inode *ip; + struct xfs_trans *tp; + int error; + + error = xfs_qm_qino_load(mp, type, &ip); + if (error == -ENOENT) + return 0; + if (error) + return error; + + xfs_ilock(ip, XFS_IOLOCK_EXCL); + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_itruncate, 0, 0, 0, &tp); + if (error) { + xfs_iunlock(ip, XFS_IOLOCK_EXCL); + goto out_put; + } + + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, 0); + + ip->i_disk_size = 0; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + + error = xfs_itruncate_extents(&tp, ip, XFS_DATA_FORK, 0); + if (error) { + xfs_trans_cancel(tp); + goto out_unlock; + } + + ASSERT(ip->i_df.if_nextents == 0); + + xfs_trans_ichgtime(tp, ip, XFS_ICHGTIME_MOD | XFS_ICHGTIME_CHG); + error = xfs_trans_commit(tp); + +out_unlock: + xfs_iunlock(ip, XFS_ILOCK_EXCL | XFS_IOLOCK_EXCL); +out_put: + xfs_irele(ip); + return error; +} + +int +xfs_qm_scall_trunc_qfiles( + xfs_mount_t *mp, + uint flags) +{ + int error = -EINVAL; + + if (!xfs_has_quota(mp) || flags == 0 || + (flags & ~XFS_QMOPT_QUOTALL)) { + xfs_debug(mp, "%s: flags=%x m_qflags=%x", + __func__, flags, mp->m_qflags); + return -EINVAL; + } + + if (flags & XFS_QMOPT_UQUOTA) { + error = xfs_qm_scall_trunc_qfile(mp, XFS_DQTYPE_USER); + if (error) + return error; + } + if (flags & XFS_QMOPT_GQUOTA) { + error = xfs_qm_scall_trunc_qfile(mp, XFS_DQTYPE_GROUP); + if (error) + return error; + } + if (flags & XFS_QMOPT_PQUOTA) + error = xfs_qm_scall_trunc_qfile(mp, XFS_DQTYPE_PROJ); + + return error; +} + +/* + * Switch on (a given) quota enforcement for a filesystem. This takes + * effect immediately. + * (Switching on quota accounting must be done at mount time.) + */ +int +xfs_qm_scall_quotaon( + xfs_mount_t *mp, + uint flags) +{ + int error; + uint qf; + + /* + * Switching on quota accounting must be done at mount time, + * only consider quota enforcement stuff here. + */ + flags &= XFS_ALL_QUOTA_ENFD; + + if (flags == 0) { + xfs_debug(mp, "%s: zero flags, m_qflags=%x", + __func__, mp->m_qflags); + return -EINVAL; + } + + /* + * Can't enforce without accounting. We check the superblock + * qflags here instead of m_qflags because rootfs can have + * quota acct on ondisk without m_qflags' knowing. + */ + if (((mp->m_sb.sb_qflags & XFS_UQUOTA_ACCT) == 0 && + (flags & XFS_UQUOTA_ENFD)) || + ((mp->m_sb.sb_qflags & XFS_GQUOTA_ACCT) == 0 && + (flags & XFS_GQUOTA_ENFD)) || + ((mp->m_sb.sb_qflags & XFS_PQUOTA_ACCT) == 0 && + (flags & XFS_PQUOTA_ENFD))) { + xfs_debug(mp, + "%s: Can't enforce without acct, flags=%x sbflags=%x", + __func__, flags, mp->m_sb.sb_qflags); + return -EINVAL; + } + /* + * If everything's up to-date incore, then don't waste time. + */ + if ((mp->m_qflags & flags) == flags) + return -EEXIST; + + /* + * Change sb_qflags on disk but not incore mp->qflags + * if this is the root filesystem. + */ + spin_lock(&mp->m_sb_lock); + qf = mp->m_sb.sb_qflags; + mp->m_sb.sb_qflags = qf | flags; + spin_unlock(&mp->m_sb_lock); + + /* + * There's nothing to change if it's the same. + */ + if ((qf & flags) == flags) + return -EEXIST; + + error = xfs_sync_sb(mp, false); + if (error) + return error; + /* + * If we aren't trying to switch on quota enforcement, we are done. + */ + if (((mp->m_sb.sb_qflags & XFS_UQUOTA_ACCT) != + (mp->m_qflags & XFS_UQUOTA_ACCT)) || + ((mp->m_sb.sb_qflags & XFS_PQUOTA_ACCT) != + (mp->m_qflags & XFS_PQUOTA_ACCT)) || + ((mp->m_sb.sb_qflags & XFS_GQUOTA_ACCT) != + (mp->m_qflags & XFS_GQUOTA_ACCT))) + return 0; + + if (!XFS_IS_QUOTA_ON(mp)) + return -ESRCH; + + /* + * Switch on quota enforcement in core. + */ + mutex_lock(&mp->m_quotainfo->qi_quotaofflock); + mp->m_qflags |= (flags & XFS_ALL_QUOTA_ENFD); + mutex_unlock(&mp->m_quotainfo->qi_quotaofflock); + + return 0; +} + +#define XFS_QC_MASK (QC_LIMIT_MASK | QC_TIMER_MASK) + +/* + * Adjust limits of this quota, and the defaults if passed in. Returns true + * if the new limits made sense and were applied, false otherwise. + */ +static inline bool +xfs_setqlim_limits( + struct xfs_mount *mp, + struct xfs_dquot_res *res, + struct xfs_quota_limits *qlim, + xfs_qcnt_t hard, + xfs_qcnt_t soft, + const char *tag) +{ + /* The hard limit can't be less than the soft limit. */ + if (hard != 0 && hard < soft) { + xfs_debug(mp, "%shard %lld < %ssoft %lld", tag, hard, tag, + soft); + return false; + } + + res->hardlimit = hard; + res->softlimit = soft; + if (qlim) { + qlim->hard = hard; + qlim->soft = soft; + } + + return true; +} + +static inline void +xfs_setqlim_timer( + struct xfs_mount *mp, + struct xfs_dquot_res *res, + struct xfs_quota_limits *qlim, + s64 timer) +{ + if (qlim) { + /* Set the length of the default grace period. */ + res->timer = xfs_dquot_set_grace_period(timer); + qlim->time = res->timer; + } else { + /* Set the grace period expiration on a quota. */ + res->timer = xfs_dquot_set_timeout(mp, timer); + } +} + +/* + * Adjust quota limits, and start/stop timers accordingly. + */ +int +xfs_qm_scall_setqlim( + struct xfs_mount *mp, + xfs_dqid_t id, + xfs_dqtype_t type, + struct qc_dqblk *newlim) +{ + struct xfs_quotainfo *q = mp->m_quotainfo; + struct xfs_dquot *dqp; + struct xfs_trans *tp; + struct xfs_def_quota *defq; + struct xfs_dquot_res *res; + struct xfs_quota_limits *qlim; + int error; + xfs_qcnt_t hard, soft; + + if (newlim->d_fieldmask & ~XFS_QC_MASK) + return -EINVAL; + if ((newlim->d_fieldmask & XFS_QC_MASK) == 0) + return 0; + + /* + * Get the dquot (locked) before we start, as we need to do a + * transaction to allocate it if it doesn't exist. Once we have the + * dquot, unlock it so we can start the next transaction safely. We hold + * a reference to the dquot, so it's safe to do this unlock/lock without + * it being reclaimed in the mean time. + */ + error = xfs_qm_dqget(mp, id, type, true, &dqp); + if (error) { + ASSERT(error != -ENOENT); + return error; + } + + defq = xfs_get_defquota(q, xfs_dquot_type(dqp)); + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_qm_setqlim, 0, 0, 0, &tp); + if (error) + goto out_rele; + + mutex_lock(&dqp->q_qlock); + xfs_trans_dqjoin(tp, dqp); + + /* + * Update quota limits, warnings, and timers, and the defaults + * if we're touching id == 0. + * + * Make sure that hardlimits are >= soft limits before changing. + * + * Update warnings counter(s) if requested. + * + * Timelimits for the super user set the relative time the other users + * can be over quota for this file system. If it is zero a default is + * used. Ditto for the default soft and hard limit values (already + * done, above), and for warnings. + * + * For other IDs, userspace can bump out the grace period if over + * the soft limit. + */ + + /* Blocks on the data device. */ + hard = (newlim->d_fieldmask & QC_SPC_HARD) ? + (xfs_qcnt_t) XFS_B_TO_FSB(mp, newlim->d_spc_hardlimit) : + dqp->q_blk.hardlimit; + soft = (newlim->d_fieldmask & QC_SPC_SOFT) ? + (xfs_qcnt_t) XFS_B_TO_FSB(mp, newlim->d_spc_softlimit) : + dqp->q_blk.softlimit; + res = &dqp->q_blk; + qlim = id == 0 ? &defq->blk : NULL; + + if (xfs_setqlim_limits(mp, res, qlim, hard, soft, "blk")) + xfs_dquot_set_prealloc_limits(dqp); + if (newlim->d_fieldmask & QC_SPC_TIMER) + xfs_setqlim_timer(mp, res, qlim, newlim->d_spc_timer); + + /* Blocks on the realtime device. */ + hard = (newlim->d_fieldmask & QC_RT_SPC_HARD) ? + (xfs_qcnt_t) XFS_B_TO_FSB(mp, newlim->d_rt_spc_hardlimit) : + dqp->q_rtb.hardlimit; + soft = (newlim->d_fieldmask & QC_RT_SPC_SOFT) ? + (xfs_qcnt_t) XFS_B_TO_FSB(mp, newlim->d_rt_spc_softlimit) : + dqp->q_rtb.softlimit; + res = &dqp->q_rtb; + qlim = id == 0 ? &defq->rtb : NULL; + + xfs_setqlim_limits(mp, res, qlim, hard, soft, "rtb"); + if (newlim->d_fieldmask & QC_RT_SPC_TIMER) + xfs_setqlim_timer(mp, res, qlim, newlim->d_rt_spc_timer); + + /* Inodes */ + hard = (newlim->d_fieldmask & QC_INO_HARD) ? + (xfs_qcnt_t) newlim->d_ino_hardlimit : + dqp->q_ino.hardlimit; + soft = (newlim->d_fieldmask & QC_INO_SOFT) ? + (xfs_qcnt_t) newlim->d_ino_softlimit : + dqp->q_ino.softlimit; + res = &dqp->q_ino; + qlim = id == 0 ? &defq->ino : NULL; + + xfs_setqlim_limits(mp, res, qlim, hard, soft, "ino"); + if (newlim->d_fieldmask & QC_INO_TIMER) + xfs_setqlim_timer(mp, res, qlim, newlim->d_ino_timer); + + if (id != 0) { + /* + * If the user is now over quota, start the timelimit. + * The user will not be 'warned'. + * Note that we keep the timers ticking, whether enforcement + * is on or off. We don't really want to bother with iterating + * over all ondisk dquots and turning the timers on/off. + */ + xfs_qm_adjust_dqtimers(dqp); + } + dqp->q_flags |= XFS_DQFLAG_DIRTY; + xfs_trans_log_dquot(tp, dqp); + + error = xfs_trans_commit(tp); + +out_rele: + xfs_qm_dqrele(dqp); + return error; +} + +/* + * Fill out the default quota limits for an ID that has no dquot on disk. + * Returns 0 if default limits are configured + * and were filled in, -ENOENT otherwise. + */ +static int +xfs_qm_scall_getquota_fill_defaults( + struct xfs_mount *mp, + xfs_dqtype_t type, + struct qc_dqblk *dst) +{ + struct xfs_def_quota *defq; + + defq = xfs_get_defquota(mp->m_quotainfo, type); + + if (!defq->blk.soft && !defq->blk.hard && + !defq->ino.soft && !defq->ino.hard && + !defq->rtb.soft && !defq->rtb.hard) { + return -ENOENT; + } + + memset(dst, 0, sizeof(*dst)); + dst->d_spc_softlimit = XFS_FSB_TO_B(mp, defq->blk.soft); + dst->d_spc_hardlimit = XFS_FSB_TO_B(mp, defq->blk.hard); + dst->d_ino_softlimit = defq->ino.soft; + dst->d_ino_hardlimit = defq->ino.hard; + dst->d_rt_spc_softlimit = XFS_FSB_TO_B(mp, defq->rtb.soft); + dst->d_rt_spc_hardlimit = XFS_FSB_TO_B(mp, defq->rtb.hard); + + return 0; +} + +/* Fill out the quota context. */ +static void +xfs_qm_scall_getquota_fill_qc( + struct xfs_mount *mp, + xfs_dqtype_t type, + const struct xfs_dquot *dqp, + struct qc_dqblk *dst) +{ + memset(dst, 0, sizeof(*dst)); + dst->d_spc_hardlimit = XFS_FSB_TO_B(mp, dqp->q_blk.hardlimit); + dst->d_spc_softlimit = XFS_FSB_TO_B(mp, dqp->q_blk.softlimit); + dst->d_ino_hardlimit = dqp->q_ino.hardlimit; + dst->d_ino_softlimit = dqp->q_ino.softlimit; + dst->d_space = XFS_FSB_TO_B(mp, dqp->q_blk.reserved); + dst->d_ino_count = dqp->q_ino.reserved; + dst->d_spc_timer = dqp->q_blk.timer; + dst->d_ino_timer = dqp->q_ino.timer; + dst->d_ino_warns = 0; + dst->d_spc_warns = 0; + dst->d_rt_spc_hardlimit = XFS_FSB_TO_B(mp, dqp->q_rtb.hardlimit); + dst->d_rt_spc_softlimit = XFS_FSB_TO_B(mp, dqp->q_rtb.softlimit); + dst->d_rt_space = XFS_FSB_TO_B(mp, dqp->q_rtb.reserved); + dst->d_rt_spc_timer = dqp->q_rtb.timer; + dst->d_rt_spc_warns = 0; + + /* + * Internally, we don't reset all the timers when quota enforcement + * gets turned off. No need to confuse the user level code, + * so return zeroes in that case. + */ + if (!xfs_dquot_is_enforced(dqp)) { + dst->d_spc_timer = 0; + dst->d_ino_timer = 0; + dst->d_rt_spc_timer = 0; + } +} + +/* Return the quota information for the dquot matching id. */ +int +xfs_qm_scall_getquota( + struct xfs_mount *mp, + xfs_dqid_t id, + xfs_dqtype_t type, + struct qc_dqblk *dst) +{ + struct xfs_dquot *dqp; + int error; + + /* + * Expedite pending inodegc work at the start of a quota reporting + * scan but don't block waiting for it to complete. + */ + if (id == 0) + xfs_inodegc_push(mp); + + /* + * Try to get the dquot. We don't want it allocated on disk, so don't + * set doalloc. If it doesn't exist, we'll get ENOENT back. + */ + error = xfs_qm_dqget(mp, id, type, false, &dqp); + if (error) { + /* + * If there is no dquot on disk and default limits are + * configured, return them with zero usage so that + * unprivileged users can see what limits apply to them. + */ + if (error == -ENOENT && id != 0 && + !xfs_qm_scall_getquota_fill_defaults(mp, type, dst)) + return 0; + return error; + } + + /* + * If everything's NULL, this dquot doesn't quite exist as far as + * our utility programs are concerned. + */ + mutex_lock(&dqp->q_qlock); + if (XFS_IS_DQUOT_UNINITIALIZED(dqp)) { + error = -ENOENT; + goto out_put; + } + + xfs_qm_scall_getquota_fill_qc(mp, type, dqp, dst); + +out_put: + mutex_unlock(&dqp->q_qlock); + xfs_qm_dqrele(dqp); + return error; +} + +/* + * Return the quota information for the first initialized dquot whose id + * is at least as high as id. + */ +int +xfs_qm_scall_getquota_next( + struct xfs_mount *mp, + xfs_dqid_t *id, + xfs_dqtype_t type, + struct qc_dqblk *dst) +{ + struct xfs_dquot *dqp; + int error; + + /* Flush inodegc work at the start of a quota reporting scan. */ + if (*id == 0) + xfs_inodegc_push(mp); + + error = xfs_qm_dqget_next(mp, *id, type, &dqp); + if (error) + return error; + + /* Fill in the ID we actually read from disk */ + *id = dqp->q_id; + + xfs_qm_scall_getquota_fill_qc(mp, type, dqp, dst); + mutex_unlock(&dqp->q_qlock); + + xfs_qm_dqrele(dqp); + return error; +} diff --git a/fs/xfs/xfs_quota.h b/fs/xfs/xfs_quota.h new file mode 100644 index 000000000..105e6eb57 --- /dev/null +++ b/fs/xfs/xfs_quota.h @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_QUOTA_H__ +#define __XFS_QUOTA_H__ + +#include "xfs_quota_defs.h" + +/* + * Kernel only quota definitions and functions + */ + +struct xfs_trans; +struct xfs_buf; + +/* + * This check is done typically without holding the inode lock; + * that may seem racy, but it is harmless in the context that it is used. + * The inode cannot go inactive as long a reference is kept, and + * therefore if dquot(s) were attached, they'll stay consistent. + * If, for example, the ownership of the inode changes while + * we didn't have the inode locked, the appropriate dquot(s) will be + * attached atomically. + */ +#define XFS_NOT_DQATTACHED(mp, ip) \ + ((XFS_IS_UQUOTA_ON(mp) && (ip)->i_udquot == NULL) || \ + (XFS_IS_GQUOTA_ON(mp) && (ip)->i_gdquot == NULL) || \ + (XFS_IS_PQUOTA_ON(mp) && (ip)->i_pdquot == NULL)) + +#define XFS_QM_NEED_QUOTACHECK(mp) \ + ((XFS_IS_UQUOTA_ON(mp) && \ + (mp->m_sb.sb_qflags & XFS_UQUOTA_CHKD) == 0) || \ + (XFS_IS_GQUOTA_ON(mp) && \ + (mp->m_sb.sb_qflags & XFS_GQUOTA_CHKD) == 0) || \ + (XFS_IS_PQUOTA_ON(mp) && \ + (mp->m_sb.sb_qflags & XFS_PQUOTA_CHKD) == 0)) + +static inline uint +xfs_quota_chkd_flag( + xfs_dqtype_t type) +{ + switch (type) { + case XFS_DQTYPE_USER: + return XFS_UQUOTA_CHKD; + case XFS_DQTYPE_GROUP: + return XFS_GQUOTA_CHKD; + case XFS_DQTYPE_PROJ: + return XFS_PQUOTA_CHKD; + default: + return 0; + } +} + +/* + * The structure kept inside the xfs_trans_t keep track of dquot changes + * within a transaction and apply them later. + */ +struct xfs_dqtrx { + struct xfs_dquot *qt_dquot; /* the dquot this refers to */ + + uint64_t qt_blk_res; /* blks reserved on a dquot */ + int64_t qt_bcount_delta; /* dquot blk count changes */ + int64_t qt_delbcnt_delta; /* delayed dquot blk count changes */ + + uint64_t qt_rtblk_res; /* # blks reserved on a dquot */ + uint64_t qt_rtblk_res_used;/* # blks used from reservation */ + int64_t qt_rtbcount_delta;/* dquot realtime blk changes */ + int64_t qt_delrtb_delta; /* delayed RT blk count changes */ + + uint64_t qt_ino_res; /* inode reserved on a dquot */ + uint64_t qt_ino_res_used; /* inodes used from the reservation */ + int64_t qt_icount_delta; /* dquot inode count changes */ +}; + +enum xfs_apply_dqtrx_type { + XFS_APPLY_DQTRX_COMMIT = 0, + XFS_APPLY_DQTRX_UNRESERVE, +}; + +/* + * Parameters for applying dqtrx changes to a dquot. The hook function arg + * parameter is enum xfs_apply_dqtrx_type. + */ +struct xfs_apply_dqtrx_params { + uintptr_t tx_id; + xfs_ino_t ino; + xfs_dqtype_t q_type; + xfs_dqid_t q_id; +}; + +#ifdef CONFIG_XFS_QUOTA +extern void xfs_trans_dup_dqinfo(struct xfs_trans *, struct xfs_trans *); +extern void xfs_trans_free_dqinfo(struct xfs_trans *); +extern void xfs_trans_mod_dquot_byino(struct xfs_trans *, struct xfs_inode *, + uint, int64_t); +extern void xfs_trans_apply_dquot_deltas(struct xfs_trans *); +void xfs_trans_unreserve_and_mod_dquots(struct xfs_trans *tp, + bool already_locked); +int xfs_trans_reserve_quota_nblks(struct xfs_trans *tp, struct xfs_inode *ip, + int64_t dblocks, int64_t rblocks, bool force); +extern int xfs_trans_reserve_quota_bydquots(struct xfs_trans *, + struct xfs_mount *, struct xfs_dquot *, + struct xfs_dquot *, struct xfs_dquot *, int64_t, long, uint); +int xfs_trans_reserve_quota_icreate(struct xfs_trans *tp, + struct xfs_dquot *udqp, struct xfs_dquot *gdqp, + struct xfs_dquot *pdqp, int64_t dblocks); + +extern int xfs_qm_vop_dqalloc(struct xfs_inode *, kuid_t, kgid_t, + prid_t, uint, struct xfs_dquot **, struct xfs_dquot **, + struct xfs_dquot **); +extern void xfs_qm_vop_create_dqattach(struct xfs_trans *, struct xfs_inode *, + struct xfs_dquot *, struct xfs_dquot *, struct xfs_dquot *); +extern int xfs_qm_vop_rename_dqattach(struct xfs_inode **); +extern struct xfs_dquot *xfs_qm_vop_chown(struct xfs_trans *, + struct xfs_inode *, struct xfs_dquot **, struct xfs_dquot *); +extern int xfs_qm_dqattach(struct xfs_inode *); +extern int xfs_qm_dqattach_locked(struct xfs_inode *ip, bool doalloc); +extern void xfs_qm_dqdetach(struct xfs_inode *); +extern void xfs_qm_dqrele(struct xfs_dquot *); +extern void xfs_qm_statvfs(struct xfs_inode *, struct kstatfs *); +extern int xfs_qm_newmount(struct xfs_mount *, uint *, uint *); +void xfs_qm_resume_quotaon(struct xfs_mount *mp); +extern void xfs_qm_mount_quotas(struct xfs_mount *); +extern void xfs_qm_unmount(struct xfs_mount *); +extern void xfs_qm_unmount_quotas(struct xfs_mount *); +bool xfs_inode_near_dquot_enforcement(struct xfs_inode *ip, xfs_dqtype_t type); +int xfs_quota_reserve_blkres(struct xfs_inode *ip, int64_t blocks); + +# ifdef CONFIG_XFS_LIVE_HOOKS +void xfs_trans_mod_ino_dquot(struct xfs_trans *tp, struct xfs_inode *ip, + struct xfs_dquot *dqp, unsigned int field, int64_t delta); + +struct xfs_quotainfo; + +struct xfs_dqtrx_hook { + struct xfs_hook mod_hook; + struct xfs_hook apply_hook; +}; + +void xfs_dqtrx_hook_disable(void); +void xfs_dqtrx_hook_enable(void); + +int xfs_dqtrx_hook_add(struct xfs_quotainfo *qi, struct xfs_dqtrx_hook *hook); +void xfs_dqtrx_hook_del(struct xfs_quotainfo *qi, struct xfs_dqtrx_hook *hook); +void xfs_dqtrx_hook_setup(struct xfs_dqtrx_hook *hook, notifier_fn_t mod_fn, + notifier_fn_t apply_fn); +# else +# define xfs_trans_mod_ino_dquot(tp, ip, dqp, field, delta) \ + xfs_trans_mod_dquot((tp), (dqp), (field), (delta)) +# endif /* CONFIG_XFS_LIVE_HOOKS */ + +#else +static inline int +xfs_qm_vop_dqalloc(struct xfs_inode *ip, kuid_t kuid, kgid_t kgid, + prid_t prid, uint flags, struct xfs_dquot **udqp, + struct xfs_dquot **gdqp, struct xfs_dquot **pdqp) +{ + *udqp = NULL; + *gdqp = NULL; + *pdqp = NULL; + return 0; +} +#define xfs_trans_dup_dqinfo(tp, tp2) +#define xfs_trans_free_dqinfo(tp) +static inline void xfs_trans_mod_dquot_byino(struct xfs_trans *tp, + struct xfs_inode *ip, uint field, int64_t delta) +{ +} +#define xfs_trans_apply_dquot_deltas(tp) +#define xfs_trans_unreserve_and_mod_dquots(tp, a) +static inline int xfs_trans_reserve_quota_nblks(struct xfs_trans *tp, + struct xfs_inode *ip, int64_t dblocks, int64_t rblocks, + bool force) +{ + return 0; +} +static inline int xfs_trans_reserve_quota_bydquots(struct xfs_trans *tp, + struct xfs_mount *mp, struct xfs_dquot *udqp, + struct xfs_dquot *gdqp, struct xfs_dquot *pdqp, + int64_t nblks, long nions, uint flags) +{ + return 0; +} + +static inline int +xfs_trans_reserve_quota_icreate(struct xfs_trans *tp, struct xfs_dquot *udqp, + struct xfs_dquot *gdqp, struct xfs_dquot *pdqp, int64_t dblocks) +{ + return 0; +} + +#define xfs_qm_vop_create_dqattach(tp, ip, u, g, p) +#define xfs_qm_vop_rename_dqattach(it) (0) +#define xfs_qm_vop_chown(tp, ip, old, new) (NULL) +#define xfs_qm_dqattach(ip) (0) +#define xfs_qm_dqattach_locked(ip, fl) (0) +#define xfs_qm_dqdetach(ip) +#define xfs_qm_dqrele(d) do { (d) = (d); } while(0) +#define xfs_qm_statvfs(ip, s) do { } while(0) +#define xfs_qm_newmount(mp, a, b) (0) +#define xfs_qm_resume_quotaon(mp) ((void)0) +#define xfs_qm_mount_quotas(mp) +#define xfs_qm_unmount(mp) +#define xfs_qm_unmount_quotas(mp) +#define xfs_inode_near_dquot_enforcement(ip, type) (false) + +static inline int xfs_quota_reserve_blkres(struct xfs_inode *ip, int64_t blocks) +{ + return 0; +} + +# ifdef CONFIG_XFS_LIVE_HOOKS +# define xfs_dqtrx_hook_enable() ((void)0) +# define xfs_dqtrx_hook_disable() ((void)0) +# endif /* CONFIG_XFS_LIVE_HOOKS */ + +#endif /* CONFIG_XFS_QUOTA */ + +static inline void +xfs_quota_unreserve_blkres(struct xfs_inode *ip, uint64_t blocks) +{ + /* don't return an error as unreserving quotas can't fail */ + xfs_quota_reserve_blkres(ip, -(int64_t)blocks); +} + +extern int xfs_mount_reset_sbqflags(struct xfs_mount *); + +#endif /* __XFS_QUOTA_H__ */ diff --git a/fs/xfs/xfs_quotaops.c b/fs/xfs/xfs_quotaops.c new file mode 100644 index 000000000..81f739e3a --- /dev/null +++ b/fs/xfs/xfs_quotaops.c @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2008, Christoph Hellwig + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_quota.h" +#include "xfs_trans.h" +#include "xfs_icache.h" +#include "xfs_qm.h" + + +static int +xfs_qm_fill_state( + struct qc_type_state *tstate, + struct xfs_mount *mp, + xfs_dqtype_t type) +{ + struct xfs_inode *ip; + struct xfs_def_quota *defq; + int error; + + error = xfs_qm_qino_load(mp, type, &ip); + if (error) { + tstate->ino = NULLFSINO; + return error != -ENOENT ? error : 0; + } + + defq = xfs_get_defquota(mp->m_quotainfo, type); + + tstate->ino = I_INO(ip); + tstate->flags |= QCI_SYSFILE; + tstate->blocks = ip->i_nblocks; + tstate->nextents = ip->i_df.if_nextents; + tstate->spc_timelimit = (u32)defq->blk.time; + tstate->ino_timelimit = (u32)defq->ino.time; + tstate->rt_spc_timelimit = (u32)defq->rtb.time; + tstate->spc_warnlimit = 0; + tstate->ino_warnlimit = 0; + tstate->rt_spc_warnlimit = 0; + xfs_irele(ip); + + return 0; +} + +/* + * Return quota status information, such as enforcements, quota file inode + * numbers etc. + */ +static int +xfs_fs_get_quota_state( + struct super_block *sb, + struct qc_state *state) +{ + struct xfs_mount *mp = XFS_M(sb); + struct xfs_quotainfo *q = mp->m_quotainfo; + int error; + + memset(state, 0, sizeof(*state)); + if (!XFS_IS_QUOTA_ON(mp)) + return 0; + state->s_incoredqs = min_t(uint64_t, q->qi_dquots, UINT_MAX); + if (XFS_IS_UQUOTA_ON(mp)) + state->s_state[USRQUOTA].flags |= QCI_ACCT_ENABLED; + if (XFS_IS_UQUOTA_ENFORCED(mp)) + state->s_state[USRQUOTA].flags |= QCI_LIMITS_ENFORCED; + if (XFS_IS_GQUOTA_ON(mp)) + state->s_state[GRPQUOTA].flags |= QCI_ACCT_ENABLED; + if (XFS_IS_GQUOTA_ENFORCED(mp)) + state->s_state[GRPQUOTA].flags |= QCI_LIMITS_ENFORCED; + if (XFS_IS_PQUOTA_ON(mp)) + state->s_state[PRJQUOTA].flags |= QCI_ACCT_ENABLED; + if (XFS_IS_PQUOTA_ENFORCED(mp)) + state->s_state[PRJQUOTA].flags |= QCI_LIMITS_ENFORCED; + + error = xfs_qm_fill_state(&state->s_state[USRQUOTA], mp, + XFS_DQTYPE_USER); + if (error) + return error; + error = xfs_qm_fill_state(&state->s_state[GRPQUOTA], mp, + XFS_DQTYPE_GROUP); + if (error) + return error; + error = xfs_qm_fill_state(&state->s_state[PRJQUOTA], mp, + XFS_DQTYPE_PROJ); + if (error) + return error; + return 0; +} + +STATIC xfs_dqtype_t +xfs_quota_type(int type) +{ + switch (type) { + case USRQUOTA: + return XFS_DQTYPE_USER; + case GRPQUOTA: + return XFS_DQTYPE_GROUP; + default: + return XFS_DQTYPE_PROJ; + } +} + +#define XFS_QC_SETINFO_MASK (QC_TIMER_MASK) + +/* + * Adjust quota timers & warnings + */ +static int +xfs_fs_set_info( + struct super_block *sb, + int type, + struct qc_info *info) +{ + struct xfs_mount *mp = XFS_M(sb); + struct qc_dqblk newlim; + + if (sb_rdonly(sb)) + return -EROFS; + if (!XFS_IS_QUOTA_ON(mp)) + return -ENOSYS; + if (info->i_fieldmask & ~XFS_QC_SETINFO_MASK) + return -EINVAL; + if ((info->i_fieldmask & XFS_QC_SETINFO_MASK) == 0) + return 0; + + newlim.d_fieldmask = info->i_fieldmask; + newlim.d_spc_timer = info->i_spc_timelimit; + newlim.d_ino_timer = info->i_ino_timelimit; + newlim.d_rt_spc_timer = info->i_rt_spc_timelimit; + newlim.d_ino_warns = info->i_ino_warnlimit; + newlim.d_spc_warns = info->i_spc_warnlimit; + newlim.d_rt_spc_warns = info->i_rt_spc_warnlimit; + + return xfs_qm_scall_setqlim(mp, 0, xfs_quota_type(type), &newlim); +} + +static unsigned int +xfs_quota_flags(unsigned int uflags) +{ + unsigned int flags = 0; + + if (uflags & FS_QUOTA_UDQ_ACCT) + flags |= XFS_UQUOTA_ACCT; + if (uflags & FS_QUOTA_PDQ_ACCT) + flags |= XFS_PQUOTA_ACCT; + if (uflags & FS_QUOTA_GDQ_ACCT) + flags |= XFS_GQUOTA_ACCT; + if (uflags & FS_QUOTA_UDQ_ENFD) + flags |= XFS_UQUOTA_ENFD; + if (uflags & FS_QUOTA_GDQ_ENFD) + flags |= XFS_GQUOTA_ENFD; + if (uflags & FS_QUOTA_PDQ_ENFD) + flags |= XFS_PQUOTA_ENFD; + + return flags; +} + +STATIC int +xfs_quota_enable( + struct super_block *sb, + unsigned int uflags) +{ + struct xfs_mount *mp = XFS_M(sb); + + if (sb_rdonly(sb)) + return -EROFS; + if (!XFS_IS_QUOTA_ON(mp)) + return -ENOSYS; + + return xfs_qm_scall_quotaon(mp, xfs_quota_flags(uflags)); +} + +STATIC int +xfs_quota_disable( + struct super_block *sb, + unsigned int uflags) +{ + struct xfs_mount *mp = XFS_M(sb); + + if (sb_rdonly(sb)) + return -EROFS; + if (!XFS_IS_QUOTA_ON(mp)) + return -ENOSYS; + + return xfs_qm_scall_quotaoff(mp, xfs_quota_flags(uflags)); +} + +STATIC int +xfs_fs_rm_xquota( + struct super_block *sb, + unsigned int uflags) +{ + struct xfs_mount *mp = XFS_M(sb); + unsigned int flags = 0; + + if (sb_rdonly(sb)) + return -EROFS; + + if (XFS_IS_QUOTA_ON(mp)) + return -EINVAL; + + if (uflags & ~(FS_USER_QUOTA | FS_GROUP_QUOTA | FS_PROJ_QUOTA)) + return -EINVAL; + + if (uflags & FS_USER_QUOTA) + flags |= XFS_QMOPT_UQUOTA; + if (uflags & FS_GROUP_QUOTA) + flags |= XFS_QMOPT_GQUOTA; + if (uflags & FS_PROJ_QUOTA) + flags |= XFS_QMOPT_PQUOTA; + + return xfs_qm_scall_trunc_qfiles(mp, flags); +} + +STATIC int +xfs_fs_get_dqblk( + struct super_block *sb, + struct kqid qid, + struct qc_dqblk *qdq) +{ + struct xfs_mount *mp = XFS_M(sb); + xfs_dqid_t id; + + if (!XFS_IS_QUOTA_ON(mp)) + return -ENOSYS; + + id = from_kqid(&init_user_ns, qid); + return xfs_qm_scall_getquota(mp, id, xfs_quota_type(qid.type), qdq); +} + +/* Return quota info for active quota >= this qid */ +STATIC int +xfs_fs_get_nextdqblk( + struct super_block *sb, + struct kqid *qid, + struct qc_dqblk *qdq) +{ + int ret; + struct xfs_mount *mp = XFS_M(sb); + xfs_dqid_t id; + + if (!XFS_IS_QUOTA_ON(mp)) + return -ENOSYS; + + id = from_kqid(&init_user_ns, *qid); + ret = xfs_qm_scall_getquota_next(mp, &id, xfs_quota_type(qid->type), + qdq); + if (ret) + return ret; + + /* ID may be different, so convert back what we got */ + *qid = make_kqid(current_user_ns(), qid->type, id); + return 0; +} + +STATIC int +xfs_fs_set_dqblk( + struct super_block *sb, + struct kqid qid, + struct qc_dqblk *qdq) +{ + struct xfs_mount *mp = XFS_M(sb); + + if (sb_rdonly(sb)) + return -EROFS; + if (!XFS_IS_QUOTA_ON(mp)) + return -ENOSYS; + + return xfs_qm_scall_setqlim(mp, from_kqid(&init_user_ns, qid), + xfs_quota_type(qid.type), qdq); +} + +const struct quotactl_ops xfs_quotactl_operations = { + .get_state = xfs_fs_get_quota_state, + .set_info = xfs_fs_set_info, + .quota_enable = xfs_quota_enable, + .quota_disable = xfs_quota_disable, + .rm_xquota = xfs_fs_rm_xquota, + .get_dqblk = xfs_fs_get_dqblk, + .get_nextdqblk = xfs_fs_get_nextdqblk, + .set_dqblk = xfs_fs_set_dqblk, +}; diff --git a/fs/xfs/xfs_refcount_item.c b/fs/xfs/xfs_refcount_item.c new file mode 100644 index 000000000..8bccf89a7 --- /dev/null +++ b/fs/xfs/xfs_refcount_item.c @@ -0,0 +1,862 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2016 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_shared.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_trans.h" +#include "xfs_trans_priv.h" +#include "xfs_refcount_item.h" +#include "xfs_log.h" +#include "xfs_refcount.h" +#include "xfs_error.h" +#include "xfs_log_priv.h" +#include "xfs_log_recover.h" +#include "xfs_ag.h" +#include "xfs_btree.h" +#include "xfs_trace.h" +#include "xfs_rtgroup.h" + +struct kmem_cache *xfs_cui_cache; +struct kmem_cache *xfs_cud_cache; + +static const struct xfs_item_ops xfs_cui_item_ops; + +static inline struct xfs_cui_log_item *CUI_ITEM(struct xfs_log_item *lip) +{ + return container_of(lip, struct xfs_cui_log_item, cui_item); +} + +STATIC void +xfs_cui_item_free( + struct xfs_cui_log_item *cuip) +{ + kvfree(cuip->cui_item.li_lv_shadow); + if (cuip->cui_format.cui_nextents > XFS_CUI_MAX_FAST_EXTENTS) + kfree(cuip); + else + kmem_cache_free(xfs_cui_cache, cuip); +} + +/* + * Freeing the CUI requires that we remove it from the AIL if it has already + * been placed there. However, the CUI may not yet have been placed in the AIL + * when called by xfs_cui_release() from CUD processing due to the ordering of + * committed vs unpin operations in bulk insert operations. Hence the reference + * count to ensure only the last caller frees the CUI. + */ +STATIC void +xfs_cui_release( + struct xfs_cui_log_item *cuip) +{ + ASSERT(atomic_read(&cuip->cui_refcount) > 0); + if (!atomic_dec_and_test(&cuip->cui_refcount)) + return; + + xfs_trans_ail_delete(&cuip->cui_item, 0); + xfs_cui_item_free(cuip); +} + + +STATIC void +xfs_cui_item_size( + struct xfs_log_item *lip, + int *nvecs, + int *nbytes) +{ + struct xfs_cui_log_item *cuip = CUI_ITEM(lip); + + *nvecs += 1; + *nbytes += xfs_cui_log_format_sizeof(cuip->cui_format.cui_nextents); +} + +unsigned int xfs_cui_log_space(unsigned int nr) +{ + return xlog_item_space(1, xfs_cui_log_format_sizeof(nr)); +} + +/* + * This is called to fill in the vector of log iovecs for the + * given cui log item. We use only 1 iovec, and we point that + * at the cui_log_format structure embedded in the cui item. + * It is at this point that we assert that all of the extent + * slots in the cui item have been filled. + */ +STATIC void +xfs_cui_item_format( + struct xfs_log_item *lip, + struct xlog_format_buf *lfb) +{ + struct xfs_cui_log_item *cuip = CUI_ITEM(lip); + + ASSERT(atomic_read(&cuip->cui_next_extent) == + cuip->cui_format.cui_nextents); + ASSERT(lip->li_type == XFS_LI_CUI || lip->li_type == XFS_LI_CUI_RT); + + cuip->cui_format.cui_type = lip->li_type; + cuip->cui_format.cui_size = 1; + + xlog_format_copy(lfb, XLOG_REG_TYPE_CUI_FORMAT, &cuip->cui_format, + xfs_cui_log_format_sizeof(cuip->cui_format.cui_nextents)); +} + +/* + * The unpin operation is the last place an CUI is manipulated in the log. It is + * either inserted in the AIL or aborted in the event of a log I/O error. In + * either case, the CUI transaction has been successfully committed to make it + * this far. Therefore, we expect whoever committed the CUI to either construct + * and commit the CUD or drop the CUD's reference in the event of error. Simply + * drop the log's CUI reference now that the log is done with it. + */ +STATIC void +xfs_cui_item_unpin( + struct xfs_log_item *lip, + int remove) +{ + struct xfs_cui_log_item *cuip = CUI_ITEM(lip); + + xfs_cui_release(cuip); +} + +/* + * The CUI has been either committed or aborted if the transaction has been + * cancelled. If the transaction was cancelled, an CUD isn't going to be + * constructed and thus we free the CUI here directly. + */ +STATIC void +xfs_cui_item_release( + struct xfs_log_item *lip) +{ + xfs_cui_release(CUI_ITEM(lip)); +} + +/* + * Allocate and initialize an cui item with the given number of extents. + */ +STATIC struct xfs_cui_log_item * +xfs_cui_init( + struct xfs_mount *mp, + unsigned short item_type, + uint nextents) +{ + struct xfs_cui_log_item *cuip; + + ASSERT(nextents > 0); + ASSERT(item_type == XFS_LI_CUI || item_type == XFS_LI_CUI_RT); + + if (nextents > XFS_CUI_MAX_FAST_EXTENTS) + cuip = kzalloc(xfs_cui_log_item_sizeof(nextents), + GFP_KERNEL | __GFP_NOFAIL); + else + cuip = kmem_cache_zalloc(xfs_cui_cache, + GFP_KERNEL | __GFP_NOFAIL); + + xfs_log_item_init(mp, &cuip->cui_item, item_type, &xfs_cui_item_ops); + cuip->cui_format.cui_nextents = nextents; + cuip->cui_format.cui_id = (uintptr_t)(void *)cuip; + atomic_set(&cuip->cui_next_extent, 0); + atomic_set(&cuip->cui_refcount, 2); + + return cuip; +} + +static inline struct xfs_cud_log_item *CUD_ITEM(struct xfs_log_item *lip) +{ + return container_of(lip, struct xfs_cud_log_item, cud_item); +} + +STATIC void +xfs_cud_item_size( + struct xfs_log_item *lip, + int *nvecs, + int *nbytes) +{ + *nvecs += 1; + *nbytes += sizeof(struct xfs_cud_log_format); +} + +unsigned int xfs_cud_log_space(void) +{ + return xlog_item_space(1, sizeof(struct xfs_cud_log_format)); +} + +/* + * This is called to fill in the vector of log iovecs for the + * given cud log item. We use only 1 iovec, and we point that + * at the cud_log_format structure embedded in the cud item. + * It is at this point that we assert that all of the extent + * slots in the cud item have been filled. + */ +STATIC void +xfs_cud_item_format( + struct xfs_log_item *lip, + struct xlog_format_buf *lfb) +{ + struct xfs_cud_log_item *cudp = CUD_ITEM(lip); + + ASSERT(lip->li_type == XFS_LI_CUD || lip->li_type == XFS_LI_CUD_RT); + + cudp->cud_format.cud_type = lip->li_type; + cudp->cud_format.cud_size = 1; + + xlog_format_copy(lfb, XLOG_REG_TYPE_CUD_FORMAT, &cudp->cud_format, + sizeof(struct xfs_cud_log_format)); +} + +/* + * The CUD is either committed or aborted if the transaction is cancelled. If + * the transaction is cancelled, drop our reference to the CUI and free the + * CUD. + */ +STATIC void +xfs_cud_item_release( + struct xfs_log_item *lip) +{ + struct xfs_cud_log_item *cudp = CUD_ITEM(lip); + + xfs_cui_release(cudp->cud_cuip); + kvfree(cudp->cud_item.li_lv_shadow); + kmem_cache_free(xfs_cud_cache, cudp); +} + +static struct xfs_log_item * +xfs_cud_item_intent( + struct xfs_log_item *lip) +{ + return &CUD_ITEM(lip)->cud_cuip->cui_item; +} + +static const struct xfs_item_ops xfs_cud_item_ops = { + .flags = XFS_ITEM_RELEASE_WHEN_COMMITTED | + XFS_ITEM_INTENT_DONE, + .iop_size = xfs_cud_item_size, + .iop_format = xfs_cud_item_format, + .iop_release = xfs_cud_item_release, + .iop_intent = xfs_cud_item_intent, +}; + +static inline struct xfs_refcount_intent *ci_entry(const struct list_head *e) +{ + return list_entry(e, struct xfs_refcount_intent, ri_list); +} + +static inline bool +xfs_cui_item_isrt(const struct xfs_log_item *lip) +{ + ASSERT(lip->li_type == XFS_LI_CUI || lip->li_type == XFS_LI_CUI_RT); + + return lip->li_type == XFS_LI_CUI_RT; +} + +/* Sort refcount intents by AG. */ +static int +xfs_refcount_update_diff_items( + void *priv, + const struct list_head *a, + const struct list_head *b) +{ + struct xfs_refcount_intent *ra = ci_entry(a); + struct xfs_refcount_intent *rb = ci_entry(b); + + return cmp_int(ra->ri_group->xg_gno, rb->ri_group->xg_gno); +} + +/* Log refcount updates in the intent item. */ +STATIC void +xfs_refcount_update_log_item( + struct xfs_trans *tp, + struct xfs_cui_log_item *cuip, + struct xfs_refcount_intent *ri) +{ + uint next_extent; + struct xfs_phys_extent *pmap; + + /* + * atomic_inc_return gives us the value after the increment; + * we want to use it as an array index so we need to subtract 1 from + * it. + */ + next_extent = atomic_inc_return(&cuip->cui_next_extent) - 1; + ASSERT(next_extent < cuip->cui_format.cui_nextents); + pmap = &cuip->cui_format.cui_extents[next_extent]; + pmap->pe_startblock = ri->ri_startblock; + pmap->pe_len = ri->ri_blockcount; + + pmap->pe_flags = 0; + switch (ri->ri_type) { + case XFS_REFCOUNT_INCREASE: + case XFS_REFCOUNT_DECREASE: + case XFS_REFCOUNT_ALLOC_COW: + case XFS_REFCOUNT_FREE_COW: + pmap->pe_flags |= ri->ri_type; + break; + default: + ASSERT(0); + } +} + +static struct xfs_log_item * +__xfs_refcount_update_create_intent( + struct xfs_trans *tp, + struct list_head *items, + unsigned int count, + bool sort, + unsigned short item_type) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_cui_log_item *cuip; + struct xfs_refcount_intent *ri; + + ASSERT(count > 0); + + cuip = xfs_cui_init(mp, item_type, count); + if (sort) + list_sort(mp, items, xfs_refcount_update_diff_items); + list_for_each_entry(ri, items, ri_list) + xfs_refcount_update_log_item(tp, cuip, ri); + return &cuip->cui_item; +} + +static struct xfs_log_item * +xfs_refcount_update_create_intent( + struct xfs_trans *tp, + struct list_head *items, + unsigned int count, + bool sort) +{ + return __xfs_refcount_update_create_intent(tp, items, count, sort, + XFS_LI_CUI); +} + +static inline unsigned short +xfs_cud_type_from_cui(const struct xfs_cui_log_item *cuip) +{ + return xfs_cui_item_isrt(&cuip->cui_item) ? XFS_LI_CUD_RT : XFS_LI_CUD; +} + +/* Get an CUD so we can process all the deferred refcount updates. */ +static struct xfs_log_item * +xfs_refcount_update_create_done( + struct xfs_trans *tp, + struct xfs_log_item *intent, + unsigned int count) +{ + struct xfs_cui_log_item *cuip = CUI_ITEM(intent); + struct xfs_cud_log_item *cudp; + + cudp = kmem_cache_zalloc(xfs_cud_cache, GFP_KERNEL | __GFP_NOFAIL); + xfs_log_item_init(tp->t_mountp, &cudp->cud_item, + xfs_cud_type_from_cui(cuip), &xfs_cud_item_ops); + cudp->cud_cuip = cuip; + cudp->cud_format.cud_cui_id = cuip->cui_format.cui_id; + + return &cudp->cud_item; +} + +/* Add this deferred CUI to the transaction. */ +void +xfs_refcount_defer_add( + struct xfs_trans *tp, + struct xfs_refcount_intent *ri) +{ + struct xfs_mount *mp = tp->t_mountp; + + /* + * Deferred refcount updates for the realtime and data sections must + * use separate transactions to finish deferred work because updates to + * realtime metadata files can lock AGFs to allocate btree blocks and + * we don't want that mixing with the AGF locks taken to finish data + * section updates. + */ + ri->ri_group = xfs_group_intent_get(mp, ri->ri_startblock, + ri->ri_realtime ? XG_TYPE_RTG : XG_TYPE_AG); + + trace_xfs_refcount_defer(mp, ri); + xfs_defer_add(tp, &ri->ri_list, ri->ri_realtime ? + &xfs_rtrefcount_update_defer_type : + &xfs_refcount_update_defer_type); +} + +/* Cancel a deferred refcount update. */ +STATIC void +xfs_refcount_update_cancel_item( + struct list_head *item) +{ + struct xfs_refcount_intent *ri = ci_entry(item); + + xfs_group_intent_put(ri->ri_group); + kmem_cache_free(xfs_refcount_intent_cache, ri); +} + +/* Process a deferred refcount update. */ +STATIC int +xfs_refcount_update_finish_item( + struct xfs_trans *tp, + struct xfs_log_item *done, + struct list_head *item, + struct xfs_btree_cur **state) +{ + struct xfs_refcount_intent *ri = ci_entry(item); + int error; + + /* Did we run out of reservation? Requeue what we didn't finish. */ + error = xfs_refcount_finish_one(tp, ri, state); + if (!error && ri->ri_blockcount > 0) { + ASSERT(ri->ri_type == XFS_REFCOUNT_INCREASE || + ri->ri_type == XFS_REFCOUNT_DECREASE); + return -EAGAIN; + } + + xfs_refcount_update_cancel_item(item); + return error; +} + +/* Clean up after calling xfs_refcount_finish_one. */ +STATIC void +xfs_refcount_finish_one_cleanup( + struct xfs_trans *tp, + struct xfs_btree_cur *rcur, + int error) +{ + struct xfs_buf *agbp; + + if (rcur == NULL) + return; + agbp = rcur->bc_ag.agbp; + xfs_btree_del_cursor(rcur, error); + if (error && agbp) + xfs_trans_brelse(tp, agbp); +} + +/* Abort all pending CUIs. */ +STATIC void +xfs_refcount_update_abort_intent( + struct xfs_log_item *intent) +{ + xfs_cui_release(CUI_ITEM(intent)); +} + +/* Is this recovered CUI ok? */ +static inline bool +xfs_cui_validate_phys( + struct xfs_mount *mp, + bool isrt, + struct xfs_phys_extent *pmap) +{ + if (!xfs_has_reflink(mp)) + return false; + + if (pmap->pe_flags & ~XFS_REFCOUNT_EXTENT_FLAGS) + return false; + + switch (pmap->pe_flags & XFS_REFCOUNT_EXTENT_TYPE_MASK) { + case XFS_REFCOUNT_INCREASE: + case XFS_REFCOUNT_DECREASE: + case XFS_REFCOUNT_ALLOC_COW: + case XFS_REFCOUNT_FREE_COW: + break; + default: + return false; + } + + if (isrt) + return xfs_verify_rtbext(mp, pmap->pe_startblock, pmap->pe_len); + + return xfs_verify_fsbext(mp, pmap->pe_startblock, pmap->pe_len); +} + +static inline void +xfs_cui_recover_work( + struct xfs_mount *mp, + struct xfs_defer_pending *dfp, + bool isrt, + struct xfs_phys_extent *pmap) +{ + struct xfs_refcount_intent *ri; + + ri = kmem_cache_alloc(xfs_refcount_intent_cache, + GFP_KERNEL | __GFP_NOFAIL); + ri->ri_type = pmap->pe_flags & XFS_REFCOUNT_EXTENT_TYPE_MASK; + ri->ri_startblock = pmap->pe_startblock; + ri->ri_blockcount = pmap->pe_len; + ri->ri_group = xfs_group_intent_get(mp, pmap->pe_startblock, + isrt ? XG_TYPE_RTG : XG_TYPE_AG); + ri->ri_realtime = isrt; + + xfs_defer_add_item(dfp, &ri->ri_list); +} + +/* + * Process a refcount update intent item that was recovered from the log. + * We need to update the refcountbt. + */ +STATIC int +xfs_refcount_recover_work( + struct xfs_defer_pending *dfp, + struct list_head *capture_list) +{ + struct xfs_trans_res resv; + struct xfs_log_item *lip = dfp->dfp_intent; + struct xfs_cui_log_item *cuip = CUI_ITEM(lip); + struct xfs_trans *tp; + struct xfs_mount *mp = lip->li_log->l_mp; + bool isrt = xfs_cui_item_isrt(lip); + int i; + int error = 0; + + /* + * First check the validity of the extents described by the + * CUI. If any are bad, then assume that all are bad and + * just toss the CUI. + */ + for (i = 0; i < cuip->cui_format.cui_nextents; i++) { + if (!xfs_cui_validate_phys(mp, isrt, + &cuip->cui_format.cui_extents[i])) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + &cuip->cui_format, + sizeof(cuip->cui_format)); + return -EFSCORRUPTED; + } + + xfs_cui_recover_work(mp, dfp, isrt, + &cuip->cui_format.cui_extents[i]); + } + + /* + * Under normal operation, refcount updates are deferred, so we + * wouldn't be adding them directly to a transaction. All + * refcount updates manage reservation usage internally and + * dynamically by deferring work that won't fit in the + * transaction. Normally, any work that needs to be deferred + * gets attached to the same defer_ops that scheduled the + * refcount update. However, we're in log recovery here, so we + * use the passed in defer_ops and to finish up any work that + * doesn't fit. We need to reserve enough blocks to handle a + * full btree split on either end of the refcount range. + */ + resv = xlog_recover_resv(&M_RES(mp)->tr_itruncate); + error = xfs_trans_alloc(mp, &resv, mp->m_refc_maxlevels * 2, 0, + XFS_TRANS_RESERVE, &tp); + if (error) + return error; + + error = xlog_recover_finish_intent(tp, dfp); + if (error == -EFSCORRUPTED) + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + &cuip->cui_format, + sizeof(cuip->cui_format)); + if (error) + goto abort_error; + + return xfs_defer_ops_capture_and_commit(tp, capture_list); + +abort_error: + xfs_trans_cancel(tp); + return error; +} + +/* Relog an intent item to push the log tail forward. */ +static struct xfs_log_item * +xfs_refcount_relog_intent( + struct xfs_trans *tp, + struct xfs_log_item *intent, + struct xfs_log_item *done_item) +{ + struct xfs_cui_log_item *cuip; + struct xfs_phys_extent *pmap; + unsigned int count; + + ASSERT(intent->li_type == XFS_LI_CUI || + intent->li_type == XFS_LI_CUI_RT); + + count = CUI_ITEM(intent)->cui_format.cui_nextents; + pmap = CUI_ITEM(intent)->cui_format.cui_extents; + + cuip = xfs_cui_init(tp->t_mountp, intent->li_type, count); + memcpy(cuip->cui_format.cui_extents, pmap, count * sizeof(*pmap)); + atomic_set(&cuip->cui_next_extent, count); + + return &cuip->cui_item; +} + +const struct xfs_defer_op_type xfs_refcount_update_defer_type = { + .name = "refcount", + .max_items = XFS_CUI_MAX_FAST_EXTENTS, + .create_intent = xfs_refcount_update_create_intent, + .abort_intent = xfs_refcount_update_abort_intent, + .create_done = xfs_refcount_update_create_done, + .finish_item = xfs_refcount_update_finish_item, + .finish_cleanup = xfs_refcount_finish_one_cleanup, + .cancel_item = xfs_refcount_update_cancel_item, + .recover_work = xfs_refcount_recover_work, + .relog_intent = xfs_refcount_relog_intent, +}; + +#ifdef CONFIG_XFS_RT +static struct xfs_log_item * +xfs_rtrefcount_update_create_intent( + struct xfs_trans *tp, + struct list_head *items, + unsigned int count, + bool sort) +{ + return __xfs_refcount_update_create_intent(tp, items, count, sort, + XFS_LI_CUI_RT); +} + +/* Process a deferred realtime refcount update. */ +STATIC int +xfs_rtrefcount_update_finish_item( + struct xfs_trans *tp, + struct xfs_log_item *done, + struct list_head *item, + struct xfs_btree_cur **state) +{ + struct xfs_refcount_intent *ri = ci_entry(item); + int error; + + error = xfs_rtrefcount_finish_one(tp, ri, state); + + /* Did we run out of reservation? Requeue what we didn't finish. */ + if (!error && ri->ri_blockcount > 0) { + ASSERT(ri->ri_type == XFS_REFCOUNT_INCREASE || + ri->ri_type == XFS_REFCOUNT_DECREASE); + return -EAGAIN; + } + + xfs_refcount_update_cancel_item(item); + return error; +} + +/* Clean up after calling xfs_rtrefcount_finish_one. */ +STATIC void +xfs_rtrefcount_finish_one_cleanup( + struct xfs_trans *tp, + struct xfs_btree_cur *rcur, + int error) +{ + if (rcur) + xfs_btree_del_cursor(rcur, error); +} + +const struct xfs_defer_op_type xfs_rtrefcount_update_defer_type = { + .name = "rtrefcount", + .max_items = XFS_CUI_MAX_FAST_EXTENTS, + .create_intent = xfs_rtrefcount_update_create_intent, + .abort_intent = xfs_refcount_update_abort_intent, + .create_done = xfs_refcount_update_create_done, + .finish_item = xfs_rtrefcount_update_finish_item, + .finish_cleanup = xfs_rtrefcount_finish_one_cleanup, + .cancel_item = xfs_refcount_update_cancel_item, + .recover_work = xfs_refcount_recover_work, + .relog_intent = xfs_refcount_relog_intent, +}; +#else +const struct xfs_defer_op_type xfs_rtrefcount_update_defer_type = { + .name = "rtrefcount", +}; +#endif /* CONFIG_XFS_RT */ + +STATIC bool +xfs_cui_item_match( + struct xfs_log_item *lip, + uint64_t intent_id) +{ + return CUI_ITEM(lip)->cui_format.cui_id == intent_id; +} + +static const struct xfs_item_ops xfs_cui_item_ops = { + .flags = XFS_ITEM_INTENT, + .iop_size = xfs_cui_item_size, + .iop_format = xfs_cui_item_format, + .iop_unpin = xfs_cui_item_unpin, + .iop_release = xfs_cui_item_release, + .iop_match = xfs_cui_item_match, +}; + +static inline void +xfs_cui_copy_format( + struct xfs_cui_log_format *dst, + const struct xfs_cui_log_format *src) +{ + unsigned int i; + + memcpy(dst, src, offsetof(struct xfs_cui_log_format, cui_extents)); + + for (i = 0; i < src->cui_nextents; i++) + memcpy(&dst->cui_extents[i], &src->cui_extents[i], + sizeof(struct xfs_phys_extent)); +} + +/* + * This routine is called to create an in-core extent refcount update + * item from the cui format structure which was logged on disk. + * It allocates an in-core cui, copies the extents from the format + * structure into it, and adds the cui to the AIL with the given + * LSN. + */ +STATIC int +xlog_recover_cui_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_mount *mp = log->l_mp; + struct xfs_cui_log_item *cuip; + struct xfs_cui_log_format *cui_formatp; + size_t len; + + cui_formatp = item->ri_buf[0].iov_base; + + if (item->ri_buf[0].iov_len < xfs_cui_log_format_sizeof(0)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[0].iov_base, item->ri_buf[0].iov_len); + return -EFSCORRUPTED; + } + + len = xfs_cui_log_format_sizeof(cui_formatp->cui_nextents); + if (item->ri_buf[0].iov_len != len) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[0].iov_base, item->ri_buf[0].iov_len); + return -EFSCORRUPTED; + } + + cuip = xfs_cui_init(mp, ITEM_TYPE(item), cui_formatp->cui_nextents); + xfs_cui_copy_format(&cuip->cui_format, cui_formatp); + atomic_set(&cuip->cui_next_extent, cui_formatp->cui_nextents); + + xlog_recover_intent_item(log, &cuip->cui_item, lsn, + &xfs_refcount_update_defer_type); + return 0; +} + +const struct xlog_recover_item_ops xlog_cui_item_ops = { + .item_type = XFS_LI_CUI, + .commit_pass2 = xlog_recover_cui_commit_pass2, +}; + +#ifdef CONFIG_XFS_RT +STATIC int +xlog_recover_rtcui_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_mount *mp = log->l_mp; + struct xfs_cui_log_item *cuip; + struct xfs_cui_log_format *cui_formatp; + size_t len; + + cui_formatp = item->ri_buf[0].iov_base; + + if (item->ri_buf[0].iov_len < xfs_cui_log_format_sizeof(0)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[0].iov_base, item->ri_buf[0].iov_len); + return -EFSCORRUPTED; + } + + len = xfs_cui_log_format_sizeof(cui_formatp->cui_nextents); + if (item->ri_buf[0].iov_len != len) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[0].iov_base, item->ri_buf[0].iov_len); + return -EFSCORRUPTED; + } + + cuip = xfs_cui_init(mp, ITEM_TYPE(item), cui_formatp->cui_nextents); + xfs_cui_copy_format(&cuip->cui_format, cui_formatp); + atomic_set(&cuip->cui_next_extent, cui_formatp->cui_nextents); + + xlog_recover_intent_item(log, &cuip->cui_item, lsn, + &xfs_rtrefcount_update_defer_type); + return 0; +} +#else +STATIC int +xlog_recover_rtcui_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, log->l_mp, + item->ri_buf[0].iov_base, item->ri_buf[0].iov_len); + return -EFSCORRUPTED; +} +#endif + +const struct xlog_recover_item_ops xlog_rtcui_item_ops = { + .item_type = XFS_LI_CUI_RT, + .commit_pass2 = xlog_recover_rtcui_commit_pass2, +}; + +/* + * This routine is called when an CUD format structure is found in a committed + * transaction in the log. Its purpose is to cancel the corresponding CUI if it + * was still in the log. To do this it searches the AIL for the CUI with an id + * equal to that in the CUD format structure. If we find it we drop the CUD + * reference, which removes the CUI from the AIL and frees it. + */ +STATIC int +xlog_recover_cud_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_cud_log_format *cud_formatp; + + cud_formatp = item->ri_buf[0].iov_base; + if (item->ri_buf[0].iov_len != sizeof(struct xfs_cud_log_format)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, log->l_mp, + item->ri_buf[0].iov_base, item->ri_buf[0].iov_len); + return -EFSCORRUPTED; + } + + xlog_recover_release_intent(log, XFS_LI_CUI, cud_formatp->cud_cui_id); + return 0; +} + +const struct xlog_recover_item_ops xlog_cud_item_ops = { + .item_type = XFS_LI_CUD, + .commit_pass2 = xlog_recover_cud_commit_pass2, +}; + +#ifdef CONFIG_XFS_RT +STATIC int +xlog_recover_rtcud_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_cud_log_format *cud_formatp; + + cud_formatp = item->ri_buf[0].iov_base; + if (item->ri_buf[0].iov_len != sizeof(struct xfs_cud_log_format)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, log->l_mp, + item->ri_buf[0].iov_base, item->ri_buf[0].iov_len); + return -EFSCORRUPTED; + } + + xlog_recover_release_intent(log, XFS_LI_CUI_RT, + cud_formatp->cud_cui_id); + return 0; +} +#else +# define xlog_recover_rtcud_commit_pass2 xlog_recover_rtcui_commit_pass2 +#endif + +const struct xlog_recover_item_ops xlog_rtcud_item_ops = { + .item_type = XFS_LI_CUD_RT, + .commit_pass2 = xlog_recover_rtcud_commit_pass2, +}; diff --git a/fs/xfs/xfs_refcount_item.h b/fs/xfs/xfs_refcount_item.h new file mode 100644 index 000000000..0fc3f4933 --- /dev/null +++ b/fs/xfs/xfs_refcount_item.h @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2016 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_REFCOUNT_ITEM_H__ +#define __XFS_REFCOUNT_ITEM_H__ + +/* + * There are (currently) two pairs of refcount btree redo item types: + * increase and decrease. The log items for these are CUI (refcount + * update intent) and CUD (refcount update done). The redo item type + * is encoded in the flags field of each xfs_map_extent. + * + * *I items should be recorded in the *first* of a series of rolled + * transactions, and the *D items should be recorded in the same + * transaction that records the associated refcountbt updates. + * + * Should the system crash after the commit of the first transaction + * but before the commit of the final transaction in a series, log + * recovery will use the redo information recorded by the intent items + * to replay the refcountbt metadata updates. + */ + +/* kernel only CUI/CUD definitions */ + +struct xfs_mount; +struct kmem_cache; + +/* + * Max number of extents in fast allocation path. + */ +#define XFS_CUI_MAX_FAST_EXTENTS 16 + +/* + * This is the "refcount update intent" log item. It is used to log + * the fact that some reverse mappings need to change. It is used in + * conjunction with the "refcount update done" log item described + * below. + * + * These log items follow the same rules as struct xfs_efi_log_item; + * see the comments about that structure (in xfs_extfree_item.h) for + * more details. + */ +struct xfs_cui_log_item { + struct xfs_log_item cui_item; + atomic_t cui_refcount; + atomic_t cui_next_extent; + struct xfs_cui_log_format cui_format; +}; + +static inline size_t +xfs_cui_log_item_sizeof( + unsigned int nr) +{ + return offsetof(struct xfs_cui_log_item, cui_format) + + xfs_cui_log_format_sizeof(nr); +} + +/* + * This is the "refcount update done" log item. It is used to log the + * fact that some refcountbt updates mentioned in an earlier cui item + * have been performed. + */ +struct xfs_cud_log_item { + struct xfs_log_item cud_item; + struct xfs_cui_log_item *cud_cuip; + struct xfs_cud_log_format cud_format; +}; + +extern struct kmem_cache *xfs_cui_cache; +extern struct kmem_cache *xfs_cud_cache; + +struct xfs_refcount_intent; + +void xfs_refcount_defer_add(struct xfs_trans *tp, + struct xfs_refcount_intent *ri); + +unsigned int xfs_cui_log_space(unsigned int nr); +unsigned int xfs_cud_log_space(void); + +#endif /* __XFS_REFCOUNT_ITEM_H__ */ diff --git a/fs/xfs/xfs_reflink.c b/fs/xfs/xfs_reflink.c new file mode 100644 index 000000000..480136136 --- /dev/null +++ b/fs/xfs/xfs_reflink.c @@ -0,0 +1,1964 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2016 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_bmap.h" +#include "xfs_bmap_util.h" +#include "xfs_trace.h" +#include "xfs_icache.h" +#include "xfs_btree.h" +#include "xfs_refcount_btree.h" +#include "xfs_refcount.h" +#include "xfs_bmap_btree.h" +#include "xfs_trans_space.h" +#include "xfs_bit.h" +#include "xfs_alloc.h" +#include "xfs_quota.h" +#include "xfs_reflink.h" +#include "xfs_iomap.h" +#include "xfs_ag.h" +#include "xfs_ag_resv.h" +#include "xfs_health.h" +#include "xfs_rtrefcount_btree.h" +#include "xfs_rtalloc.h" +#include "xfs_rtgroup.h" +#include "xfs_metafile.h" + +/* + * Copy on Write of Shared Blocks + * + * XFS must preserve "the usual" file semantics even when two files share + * the same physical blocks. This means that a write to one file must not + * alter the blocks in a different file; the way that we'll do that is + * through the use of a copy-on-write mechanism. At a high level, that + * means that when we want to write to a shared block, we allocate a new + * block, write the data to the new block, and if that succeeds we map the + * new block into the file. + * + * XFS provides a "delayed allocation" mechanism that defers the allocation + * of disk blocks to dirty-but-not-yet-mapped file blocks as long as + * possible. This reduces fragmentation by enabling the filesystem to ask + * for bigger chunks less often, which is exactly what we want for CoW. + * + * The delalloc mechanism begins when the kernel wants to make a block + * writable (write_begin or page_mkwrite). If the offset is not mapped, we + * create a delalloc mapping, which is a regular in-core extent, but without + * a real startblock. (For delalloc mappings, the startblock encodes both + * a flag that this is a delalloc mapping, and a worst-case estimate of how + * many blocks might be required to put the mapping into the BMBT.) delalloc + * mappings are a reservation against the free space in the filesystem; + * adjacent mappings can also be combined into fewer larger mappings. + * + * As an optimization, the CoW extent size hint (cowextsz) creates + * outsized aligned delalloc reservations in the hope of landing out of + * order nearby CoW writes in a single extent on disk, thereby reducing + * fragmentation and improving future performance. + * + * D: --RRRRRRSSSRRRRRRRR--- (data fork) + * C: ------DDDDDDD--------- (CoW fork) + * + * When dirty pages are being written out (typically in writepage), the + * delalloc reservations are converted into unwritten mappings by + * allocating blocks and replacing the delalloc mapping with real ones. + * A delalloc mapping can be replaced by several unwritten ones if the + * free space is fragmented. + * + * D: --RRRRRRSSSRRRRRRRR--- + * C: ------UUUUUUU--------- + * + * We want to adapt the delalloc mechanism for copy-on-write, since the + * write paths are similar. The first two steps (creating the reservation + * and allocating the blocks) are exactly the same as delalloc except that + * the mappings must be stored in a separate CoW fork because we do not want + * to disturb the mapping in the data fork until we're sure that the write + * succeeded. IO completion in this case is the process of removing the old + * mapping from the data fork and moving the new mapping from the CoW fork to + * the data fork. This will be discussed shortly. + * + * For now, unaligned directio writes will be bounced back to the page cache. + * Block-aligned directio writes will use the same mechanism as buffered + * writes. + * + * Just prior to submitting the actual disk write requests, we convert + * the extents representing the range of the file actually being written + * (as opposed to extra pieces created for the cowextsize hint) to real + * extents. This will become important in the next step: + * + * D: --RRRRRRSSSRRRRRRRR--- + * C: ------UUrrUUU--------- + * + * CoW remapping must be done after the data block write completes, + * because we don't want to destroy the old data fork map until we're sure + * the new block has been written. Since the new mappings are kept in a + * separate fork, we can simply iterate these mappings to find the ones + * that cover the file blocks that we just CoW'd. For each extent, simply + * unmap the corresponding range in the data fork, map the new range into + * the data fork, and remove the extent from the CoW fork. Because of + * the presence of the cowextsize hint, however, we must be careful + * only to remap the blocks that we've actually written out -- we must + * never remap delalloc reservations nor CoW staging blocks that have + * yet to be written. This corresponds exactly to the real extents in + * the CoW fork: + * + * D: --RRRRRRrrSRRRRRRRR--- + * C: ------UU--UUU--------- + * + * Since the remapping operation can be applied to an arbitrary file + * range, we record the need for the remap step as a flag in the ioend + * instead of declaring a new IO type. This is required for direct io + * because we only have ioend for the whole dio, and we have to be able to + * remember the presence of unwritten blocks and CoW blocks with a single + * ioend structure. Better yet, the more ground we can cover with one + * ioend, the better. + */ + +/* + * Given a file mapping for the data device, find the lowest-numbered run of + * shared blocks within that mapping and return it in shared_offset/shared_len. + * The offset is relative to the start of irec. + * + * If find_end_of_shared is true, return the longest contiguous extent of shared + * blocks. If there are no shared extents, shared_offset and shared_len will be + * set to 0; + */ +static int +xfs_reflink_find_shared( + struct xfs_mount *mp, + struct xfs_trans *tp, + const struct xfs_bmbt_irec *irec, + xfs_extlen_t *shared_offset, + xfs_extlen_t *shared_len, + bool find_end_of_shared) +{ + struct xfs_buf *agbp; + struct xfs_perag *pag; + struct xfs_btree_cur *cur; + int error; + xfs_agblock_t orig_bno, found_bno; + + pag = xfs_perag_get(mp, XFS_FSB_TO_AGNO(mp, irec->br_startblock)); + orig_bno = XFS_FSB_TO_AGBNO(mp, irec->br_startblock); + + error = xfs_alloc_read_agf(pag, tp, 0, &agbp); + if (error) + goto out; + + cur = xfs_refcountbt_init_cursor(mp, tp, agbp, pag); + error = xfs_refcount_find_shared(cur, orig_bno, irec->br_blockcount, + &found_bno, shared_len, find_end_of_shared); + xfs_btree_del_cursor(cur, error); + xfs_trans_brelse(tp, agbp); + + if (!error && *shared_len) + *shared_offset = found_bno - orig_bno; +out: + xfs_perag_put(pag); + return error; +} + +/* + * Given a file mapping for the rt device, find the lowest-numbered run of + * shared blocks within that mapping and return it in shared_offset/shared_len. + * The offset is relative to the start of irec. + * + * If find_end_of_shared is true, return the longest contiguous extent of shared + * blocks. If there are no shared extents, shared_offset and shared_len will be + * set to 0; + */ +static int +xfs_reflink_find_rtshared( + struct xfs_mount *mp, + struct xfs_trans *tp, + const struct xfs_bmbt_irec *irec, + xfs_extlen_t *shared_offset, + xfs_extlen_t *shared_len, + bool find_end_of_shared) +{ + struct xfs_rtgroup *rtg; + struct xfs_btree_cur *cur; + xfs_rgblock_t orig_bno; + xfs_agblock_t found_bno; + int error; + + BUILD_BUG_ON(NULLRGBLOCK != NULLAGBLOCK); + + /* + * Note: this uses the not quite correct xfs_agblock_t type because + * xfs_refcount_find_shared is shared between the RT and data device + * refcount code. + */ + orig_bno = xfs_rtb_to_rgbno(mp, irec->br_startblock); + rtg = xfs_rtgroup_get(mp, xfs_rtb_to_rgno(mp, irec->br_startblock)); + + xfs_rtgroup_lock(rtg, XFS_RTGLOCK_REFCOUNT); + cur = xfs_rtrefcountbt_init_cursor(tp, rtg); + error = xfs_refcount_find_shared(cur, orig_bno, irec->br_blockcount, + &found_bno, shared_len, find_end_of_shared); + xfs_btree_del_cursor(cur, error); + xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_REFCOUNT); + xfs_rtgroup_put(rtg); + + if (!error && *shared_len) + *shared_offset = found_bno - orig_bno; + return error; +} + +/* + * Trim the mapping to the next block where there's a change in the + * shared/unshared status. More specifically, this means that we + * find the lowest-numbered extent of shared blocks that coincides with + * the given block mapping. If the shared extent overlaps the start of + * the mapping, trim the mapping to the end of the shared extent. If + * the shared region intersects the mapping, trim the mapping to the + * start of the shared extent. If there are no shared regions that + * overlap, just return the original extent. + */ +int +xfs_reflink_trim_around_shared( + struct xfs_inode *ip, + struct xfs_bmbt_irec *irec, + bool *shared) +{ + struct xfs_mount *mp = ip->i_mount; + xfs_extlen_t shared_offset, shared_len; + int error = 0; + + /* Holes, unwritten, and delalloc extents cannot be shared */ + if (!xfs_is_reflink_inode(ip) || !xfs_bmap_is_written_extent(irec)) { + *shared = false; + return 0; + } + + trace_xfs_reflink_trim_around_shared(ip, irec); + + if (XFS_IS_REALTIME_INODE(ip)) + error = xfs_reflink_find_rtshared(mp, NULL, irec, + &shared_offset, &shared_len, true); + else + error = xfs_reflink_find_shared(mp, NULL, irec, + &shared_offset, &shared_len, true); + if (error) + return error; + + if (!shared_len) { + /* No shared blocks at all. */ + *shared = false; + } else if (!shared_offset) { + /* + * The start of this mapping points to shared space. Truncate + * the mapping at the end of the shared region so that a + * subsequent iteration starts at the start of the unshared + * region. + */ + irec->br_blockcount = shared_len; + *shared = true; + } else { + /* + * There's a shared region that doesn't start at the beginning + * of the mapping. Truncate the mapping at the start of the + * shared extent so that a subsequent iteration starts at the + * start of the shared region. + */ + irec->br_blockcount = shared_offset; + *shared = false; + } + return 0; +} + +int +xfs_bmap_trim_cow( + struct xfs_inode *ip, + struct xfs_bmbt_irec *imap, + bool *shared) +{ + /* We can't update any real extents in always COW mode. */ + if (xfs_is_always_cow_inode(ip) && + !isnullstartblock(imap->br_startblock)) { + *shared = true; + return 0; + } + + /* Trim the mapping to the nearest shared extent boundary. */ + return xfs_reflink_trim_around_shared(ip, imap, shared); +} + +int +xfs_reflink_convert_cow_locked( + struct xfs_inode *ip, + xfs_fileoff_t offset_fsb, + xfs_filblks_t count_fsb) +{ + struct xfs_iext_cursor icur; + struct xfs_bmbt_irec got; + struct xfs_btree_cur *dummy_cur = NULL; + int dummy_logflags; + int error = 0; + + if (!xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &got)) + return 0; + + do { + if (got.br_startoff >= offset_fsb + count_fsb) + break; + if (got.br_state == XFS_EXT_NORM) + continue; + if (WARN_ON_ONCE(isnullstartblock(got.br_startblock))) + return -EIO; + + xfs_trim_extent(&got, offset_fsb, count_fsb); + if (!got.br_blockcount) + continue; + + got.br_state = XFS_EXT_NORM; + error = xfs_bmap_add_extent_unwritten_real(NULL, ip, + XFS_COW_FORK, &icur, &dummy_cur, &got, + &dummy_logflags); + if (error) + return error; + } while (xfs_iext_next_extent(ip->i_cowfp, &icur, &got)); + + return error; +} + +/* Convert all of the unwritten CoW extents in a file's range to real ones. */ +int +xfs_reflink_convert_cow( + struct xfs_inode *ip, + xfs_off_t offset, + xfs_off_t count) +{ + struct xfs_mount *mp = ip->i_mount; + xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset); + xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + count); + xfs_filblks_t count_fsb = end_fsb - offset_fsb; + int error; + + ASSERT(count != 0); + + xfs_ilock(ip, XFS_ILOCK_EXCL); + error = xfs_reflink_convert_cow_locked(ip, offset_fsb, count_fsb); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; +} + +/* + * Find the extent that maps the given range in the COW fork. Even if the extent + * is not shared we might have a preallocation for it in the COW fork. If so we + * use it that rather than trigger a new allocation. + */ +static int +xfs_find_trim_cow_extent( + struct xfs_inode *ip, + struct xfs_bmbt_irec *imap, + struct xfs_bmbt_irec *cmap, + bool *shared, + bool *found) +{ + xfs_fileoff_t offset_fsb = imap->br_startoff; + xfs_filblks_t count_fsb = imap->br_blockcount; + struct xfs_iext_cursor icur; + + *found = false; + + /* + * If we don't find an overlapping extent, trim the range we need to + * allocate to fit the hole we found. + */ + if (!xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, cmap)) + cmap->br_startoff = offset_fsb + count_fsb; + if (cmap->br_startoff > offset_fsb) { + xfs_trim_extent(imap, imap->br_startoff, + cmap->br_startoff - imap->br_startoff); + return xfs_bmap_trim_cow(ip, imap, shared); + } + + *shared = true; + if (isnullstartblock(cmap->br_startblock)) { + xfs_trim_extent(imap, cmap->br_startoff, cmap->br_blockcount); + return 0; + } + + /* real extent found - no need to allocate */ + xfs_trim_extent(cmap, offset_fsb, count_fsb); + *found = true; + return 0; +} + +static int +xfs_reflink_convert_unwritten( + struct xfs_inode *ip, + struct xfs_bmbt_irec *imap, + struct xfs_bmbt_irec *cmap, + bool convert_now) +{ + xfs_fileoff_t offset_fsb = imap->br_startoff; + xfs_filblks_t count_fsb = imap->br_blockcount; + int error; + + /* + * cmap might larger than imap due to cowextsize hint. + */ + xfs_trim_extent(cmap, offset_fsb, count_fsb); + + /* + * COW fork extents are supposed to remain unwritten until we're ready + * to initiate a disk write. For direct I/O we are going to write the + * data and need the conversion, but for buffered writes we're done. + */ + if (!convert_now || cmap->br_state == XFS_EXT_NORM) + return 0; + + trace_xfs_reflink_convert_cow(ip, cmap); + + error = xfs_reflink_convert_cow_locked(ip, offset_fsb, count_fsb); + if (!error) + cmap->br_state = XFS_EXT_NORM; + + return error; +} + +static int +xfs_reflink_fill_cow_hole( + struct xfs_inode *ip, + struct xfs_bmbt_irec *imap, + struct xfs_bmbt_irec *cmap, + bool *shared, + uint *lockmode, + bool convert_now) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_trans *tp; + xfs_filblks_t resaligned; + unsigned int seq_before = READ_ONCE(ip->i_df.if_seq); + unsigned int dblocks = 0, rblocks = 0; + int nimaps; + int error; + bool found; + + resaligned = xfs_aligned_fsb_count(imap->br_startoff, + imap->br_blockcount, xfs_get_cowextsz_hint(ip)); + if (XFS_IS_REALTIME_INODE(ip)) { + dblocks = XFS_DIOSTRAT_SPACE_RES(mp, 0); + rblocks = resaligned; + } else { + dblocks = XFS_DIOSTRAT_SPACE_RES(mp, resaligned); + rblocks = 0; + } + + xfs_iunlock(ip, *lockmode); + *lockmode = 0; + + error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, dblocks, + rblocks, false, &tp); + if (error) + return error; + + *lockmode = XFS_ILOCK_EXCL; + + /* + * The data fork mapping may have changed while we dropped the ILOCK + * (a racing O_DIRECT writer under IOLOCK_SHARED can complete a full + * CoW cycle including xfs_reflink_end_cow(), which remaps this offset + * and drops the refcount of the old shared block). Re-read it so the + * shared-status recheck below and the caller's in-place iomap both + * operate on the current mapping rather than a stale physical block. + */ + if (seq_before != READ_ONCE(ip->i_df.if_seq)) { + nimaps = 1; + error = xfs_bmapi_read(ip, imap->br_startoff, + imap->br_blockcount, imap, &nimaps, 0); + if (error) + goto out_trans_cancel; + } + + error = xfs_find_trim_cow_extent(ip, imap, cmap, shared, &found); + if (error || !*shared) + goto out_trans_cancel; + + if (found) { + xfs_trans_cancel(tp); + goto convert; + } + + /* Allocate the entire reservation as unwritten blocks. */ + nimaps = 1; + error = xfs_bmapi_write(tp, ip, imap->br_startoff, imap->br_blockcount, + XFS_BMAPI_COWFORK | XFS_BMAPI_PREALLOC, 0, cmap, + &nimaps); + if (error) + goto out_trans_cancel; + + xfs_inode_set_cowblocks_tag(ip); + error = xfs_trans_commit(tp); + if (error) + return error; + +convert: + return xfs_reflink_convert_unwritten(ip, imap, cmap, convert_now); + +out_trans_cancel: + xfs_trans_cancel(tp); + return error; +} + +static int +xfs_reflink_fill_delalloc( + struct xfs_inode *ip, + struct xfs_bmbt_irec *imap, + struct xfs_bmbt_irec *cmap, + bool *shared, + uint *lockmode, + bool convert_now) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_trans *tp; + int nimaps; + int error; + bool found; + + do { + unsigned int seq_before = READ_ONCE(ip->i_df.if_seq); + + xfs_iunlock(ip, *lockmode); + *lockmode = 0; + + error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, 0, 0, + false, &tp); + if (error) + return error; + + *lockmode = XFS_ILOCK_EXCL; + + /* + * The data fork mapping may have changed while we dropped the + * ILOCK (a racing O_DIRECT writer under IOLOCK_SHARED can + * complete a full CoW cycle including xfs_reflink_end_cow(), + * which remaps this offset and drops the refcount of the old + * shared block). Re-read it so the shared-status recheck + * below and the caller's in-place iomap both operate on the + * current mapping rather than a stale physical block. + */ + if (seq_before != READ_ONCE(ip->i_df.if_seq)) { + nimaps = 1; + error = xfs_bmapi_read(ip, imap->br_startoff, + imap->br_blockcount, imap, &nimaps, 0); + if (error) + goto out_trans_cancel; + } + + error = xfs_find_trim_cow_extent(ip, imap, cmap, shared, + &found); + if (error || !*shared) + goto out_trans_cancel; + + if (found) { + xfs_trans_cancel(tp); + break; + } + + ASSERT(isnullstartblock(cmap->br_startblock) || + cmap->br_startblock == DELAYSTARTBLOCK); + + /* + * Replace delalloc reservation with an unwritten extent. + */ + nimaps = 1; + error = xfs_bmapi_write(tp, ip, cmap->br_startoff, + cmap->br_blockcount, + XFS_BMAPI_COWFORK | XFS_BMAPI_PREALLOC, 0, + cmap, &nimaps); + if (error) + goto out_trans_cancel; + + xfs_inode_set_cowblocks_tag(ip); + error = xfs_trans_commit(tp); + if (error) + return error; + } while (cmap->br_startoff + cmap->br_blockcount <= imap->br_startoff); + + return xfs_reflink_convert_unwritten(ip, imap, cmap, convert_now); + +out_trans_cancel: + xfs_trans_cancel(tp); + return error; +} + +/* Allocate all CoW reservations covering a range of blocks in a file. */ +int +xfs_reflink_allocate_cow( + struct xfs_inode *ip, + struct xfs_bmbt_irec *imap, + struct xfs_bmbt_irec *cmap, + bool *shared, + uint *lockmode, + bool convert_now) +{ + int error; + bool found; + + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + if (!ip->i_cowfp) { + ASSERT(!xfs_is_reflink_inode(ip)); + xfs_ifork_init_cow(ip); + } + + error = xfs_find_trim_cow_extent(ip, imap, cmap, shared, &found); + if (error || !*shared) + return error; + + /* CoW fork has a real extent */ + if (found) + return xfs_reflink_convert_unwritten(ip, imap, cmap, + convert_now); + + /* + * CoW fork does not have an extent and data extent is shared. + * Allocate a real extent in the CoW fork. + */ + if (cmap->br_startoff > imap->br_startoff) + return xfs_reflink_fill_cow_hole(ip, imap, cmap, shared, + lockmode, convert_now); + + /* + * CoW fork has a delalloc reservation. Replace it with a real extent. + * There may or may not be a data fork mapping. + */ + if (isnullstartblock(cmap->br_startblock) || + cmap->br_startblock == DELAYSTARTBLOCK) + return xfs_reflink_fill_delalloc(ip, imap, cmap, shared, + lockmode, convert_now); + + /* Shouldn't get here. */ + ASSERT(0); + return -EFSCORRUPTED; +} + +/* + * Cancel CoW reservations for some block range of an inode. + * + * If cancel_real is true this function cancels all COW fork extents for the + * inode; if cancel_real is false, real extents are not cleared. + * + * Caller must have already joined the inode to the current transaction. The + * inode will be joined to the transaction returned to the caller. + */ +int +xfs_reflink_cancel_cow_blocks( + struct xfs_inode *ip, + struct xfs_trans **tpp, + xfs_fileoff_t offset_fsb, + xfs_fileoff_t end_fsb, + bool cancel_real) +{ + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_COW_FORK); + struct xfs_bmbt_irec got, del; + struct xfs_iext_cursor icur; + bool isrt = XFS_IS_REALTIME_INODE(ip); + int error = 0; + + if (!xfs_inode_has_cow_data(ip)) + return 0; + if (!xfs_iext_lookup_extent_before(ip, ifp, &end_fsb, &icur, &got)) + return 0; + + /* Walk backwards until we're out of the I/O range... */ + while (got.br_startoff + got.br_blockcount > offset_fsb) { + del = got; + xfs_trim_extent(&del, offset_fsb, end_fsb - offset_fsb); + + /* Extent delete may have bumped ext forward */ + if (!del.br_blockcount) { + xfs_iext_prev(ifp, &icur); + goto next_extent; + } + + trace_xfs_reflink_cancel_cow(ip, &del); + + if (isnullstartblock(del.br_startblock)) { + xfs_bmap_del_extent_delay(ip, XFS_COW_FORK, &icur, &got, + &del, 0); + } else if (del.br_state == XFS_EXT_UNWRITTEN || cancel_real) { + ASSERT((*tpp)->t_highest_agno == NULLAGNUMBER); + + /* Free the CoW orphan record. */ + xfs_refcount_free_cow_extent(*tpp, isrt, + del.br_startblock, del.br_blockcount); + + error = xfs_free_extent_later(*tpp, del.br_startblock, + del.br_blockcount, NULL, + XFS_AG_RESV_NONE, + isrt ? XFS_FREE_EXTENT_REALTIME : 0); + if (error) + break; + + /* Roll the transaction */ + error = xfs_defer_finish(tpp); + if (error) + break; + + /* Remove the mapping from the CoW fork. */ + xfs_bmap_del_extent_cow(ip, &icur, &got, &del); + + /* Remove the quota reservation */ + xfs_quota_unreserve_blkres(ip, del.br_blockcount); + } else { + /* Didn't do anything, push cursor back. */ + xfs_iext_prev(ifp, &icur); + } +next_extent: + if (!xfs_iext_get_extent(ifp, &icur, &got)) + break; + } + + /* clear tag if cow fork is emptied */ + if (!ifp->if_bytes) + xfs_inode_clear_cowblocks_tag(ip); + return error; +} + +/* + * Cancel CoW reservations for some byte range of an inode. + * + * If cancel_real is true this function cancels all COW fork extents for the + * inode; if cancel_real is false, real extents are not cleared. + */ +int +xfs_reflink_cancel_cow_range( + struct xfs_inode *ip, + xfs_off_t offset, + xfs_off_t count, + bool cancel_real) +{ + struct xfs_trans *tp; + xfs_fileoff_t offset_fsb; + xfs_fileoff_t end_fsb; + int error; + + trace_xfs_reflink_cancel_cow_range(ip, offset, count); + ASSERT(ip->i_cowfp); + + offset_fsb = XFS_B_TO_FSBT(ip->i_mount, offset); + if (count == NULLFILEOFF) + end_fsb = NULLFILEOFF; + else + end_fsb = XFS_B_TO_FSB(ip->i_mount, offset + count); + + /* Start a rolling transaction to remove the mappings */ + error = xfs_trans_alloc(ip->i_mount, &M_RES(ip->i_mount)->tr_write, + 0, 0, 0, &tp); + if (error) + goto out; + + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, 0); + + /* Scrape out the old CoW reservations */ + error = xfs_reflink_cancel_cow_blocks(ip, &tp, offset_fsb, end_fsb, + cancel_real); + if (error) + goto out_cancel; + + error = xfs_trans_commit(tp); + + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; + +out_cancel: + xfs_trans_cancel(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); +out: + trace_xfs_reflink_cancel_cow_range_error(ip, error, _RET_IP_); + return error; +} + +#ifdef CONFIG_XFS_QUOTA +/* + * Update quota accounting for a remapping operation. When we're remapping + * something from the CoW fork to the data fork, we must update the quota + * accounting for delayed allocations. For remapping from the data fork to the + * data fork, use regular block accounting. + */ +static inline void +xfs_reflink_update_quota( + struct xfs_trans *tp, + struct xfs_inode *ip, + bool is_cow, + int64_t blocks) +{ + unsigned int qflag; + + if (XFS_IS_REALTIME_INODE(ip)) { + qflag = is_cow ? XFS_TRANS_DQ_DELRTBCOUNT : + XFS_TRANS_DQ_RTBCOUNT; + } else { + qflag = is_cow ? XFS_TRANS_DQ_DELBCOUNT : + XFS_TRANS_DQ_BCOUNT; + } + xfs_trans_mod_dquot_byino(tp, ip, qflag, blocks); +} +#else +# define xfs_reflink_update_quota(tp, ip, is_cow, blocks) ((void)0) +#endif + +/* + * Remap part of the CoW fork into the data fork. + * + * We aim to remap the range starting at @offset_fsb and ending at @end_fsb + * into the data fork; this function will remap what it can (at the end of the + * range) and update @end_fsb appropriately. Each remap gets its own + * transaction because we can end up merging and splitting bmbt blocks for + * every remap operation and we'd like to keep the block reservation + * requirements as low as possible. + */ +STATIC int +xfs_reflink_end_cow_extent_locked( + struct xfs_trans *tp, + struct xfs_inode *ip, + xfs_fileoff_t *offset_fsb, + xfs_fileoff_t end_fsb) +{ + struct xfs_iext_cursor icur; + struct xfs_bmbt_irec got, del, data; + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_COW_FORK); + int nmaps; + bool isrt = XFS_IS_REALTIME_INODE(ip); + int error; + + /* + * In case of racing, overlapping AIO writes no COW extents might be + * left by the time I/O completes for the loser of the race. In that + * case we are done. + */ + if (!xfs_iext_lookup_extent(ip, ifp, *offset_fsb, &icur, &got) || + got.br_startoff >= end_fsb) { + *offset_fsb = end_fsb; + return 0; + } + + /* + * Only remap real extents that contain data. With AIO, speculative + * preallocations can leak into the range we are called upon, and we + * need to skip them. Preserve @got for the eventual CoW fork + * deletion; from now on @del represents the mapping that we're + * actually remapping. + */ + while (!xfs_bmap_is_written_extent(&got)) { + if (!xfs_iext_next_extent(ifp, &icur, &got) || + got.br_startoff >= end_fsb) { + *offset_fsb = end_fsb; + return 0; + } + } + del = got; + xfs_trim_extent(&del, *offset_fsb, end_fsb - *offset_fsb); + + error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, + XFS_IEXT_REFLINK_END_COW_CNT); + if (error) + return error; + + /* Grab the corresponding mapping in the data fork. */ + nmaps = 1; + error = xfs_bmapi_read(ip, del.br_startoff, del.br_blockcount, &data, + &nmaps, 0); + if (error) + return error; + + /* We can only remap the smaller of the two extent sizes. */ + data.br_blockcount = min(data.br_blockcount, del.br_blockcount); + del.br_blockcount = data.br_blockcount; + + trace_xfs_reflink_cow_remap_from(ip, &del); + trace_xfs_reflink_cow_remap_to(ip, &data); + + if (xfs_bmap_is_real_extent(&data)) { + /* + * If the extent we're remapping is backed by storage (written + * or not), unmap the extent and drop its refcount. + */ + xfs_bmap_unmap_extent(tp, ip, XFS_DATA_FORK, &data); + xfs_refcount_decrease_extent(tp, isrt, &data); + xfs_reflink_update_quota(tp, ip, false, -data.br_blockcount); + } else if (data.br_startblock == DELAYSTARTBLOCK) { + int done; + + /* + * If the extent we're remapping is a delalloc reservation, + * we can use the regular bunmapi function to release the + * incore state. Dropping the delalloc reservation takes care + * of the quota reservation for us. + */ + error = xfs_bunmapi(NULL, ip, data.br_startoff, + data.br_blockcount, 0, 1, &done); + if (error) + return error; + ASSERT(done); + } + + /* Free the CoW orphan record. */ + xfs_refcount_free_cow_extent(tp, isrt, del.br_startblock, + del.br_blockcount); + + /* Map the new blocks into the data fork. */ + xfs_bmap_map_extent(tp, ip, XFS_DATA_FORK, &del); + + /* Charge this new data fork mapping to the on-disk quota. */ + xfs_reflink_update_quota(tp, ip, true, del.br_blockcount); + + /* Remove the mapping from the CoW fork. */ + xfs_bmap_del_extent_cow(ip, &icur, &got, &del); + + /* Update the caller about how much progress we made. */ + *offset_fsb = del.br_startoff + del.br_blockcount; + return 0; +} + +/* + * Remap part of the CoW fork into the data fork. + * + * We aim to remap the range starting at @offset_fsb and ending at @end_fsb + * into the data fork; this function will remap what it can (at the end of the + * range) and update @end_fsb appropriately. Each remap gets its own + * transaction because we can end up merging and splitting bmbt blocks for + * every remap operation and we'd like to keep the block reservation + * requirements as low as possible. + */ +STATIC int +xfs_reflink_end_cow_extent( + struct xfs_inode *ip, + xfs_fileoff_t *offset_fsb, + xfs_fileoff_t end_fsb) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_trans *tp; + unsigned int resblks; + int error; + + resblks = XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK); + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0, + XFS_TRANS_RESERVE, &tp); + if (error) + return error; + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, 0); + + error = xfs_reflink_end_cow_extent_locked(tp, ip, offset_fsb, end_fsb); + if (error) + xfs_trans_cancel(tp); + else + error = xfs_trans_commit(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; +} + +/* + * Remap parts of a file's data fork after a successful CoW. + */ +int +xfs_reflink_end_cow( + struct xfs_inode *ip, + xfs_off_t offset, + xfs_off_t count) +{ + xfs_fileoff_t offset_fsb; + xfs_fileoff_t end_fsb; + int error = 0; + + trace_xfs_reflink_end_cow(ip, offset, count); + + offset_fsb = XFS_B_TO_FSBT(ip->i_mount, offset); + end_fsb = XFS_B_TO_FSB(ip->i_mount, offset + count); + + /* + * Walk forwards until we've remapped the I/O range. The loop function + * repeatedly cycles the ILOCK to allocate one transaction per remapped + * extent. + * + * If we're being called by writeback then the folios will still + * have the writeback flag set, which prevents races with reflink + * remapping and truncate. Reflink remapping prevents races with + * writeback by taking the iolock and mmaplock before flushing + * the folios and remapping, which means there won't be any further + * writeback or page cache dirtying until the reflink completes. + * + * We should never have two threads issuing writeback for the same file + * region. There are also have post-eof checks in the writeback + * preparation code so that we don't bother writing out folios that are + * about to be truncated. + * + * If we're being called as part of directio write completion, the dio + * count is still elevated, which reflink and truncate will wait for. + * Reflink remapping takes the iolock and mmaplock and waits for + * pending dio to finish, which should prevent any directio until the + * remap completes. Multiple concurrent directio writes to the same + * region are handled by end_cow processing only occurring for the + * threads which succeed; the outcome of multiple overlapping direct + * writes is not well defined anyway. + * + * It's possible that a buffered write and a direct write could collide + * here (the buffered write stumbles in after the dio flushes and + * invalidates the page cache and immediately queues writeback), but we + * have never supported this 100%. If either disk write succeeds the + * blocks will be remapped. + */ + while (end_fsb > offset_fsb && !error) + error = xfs_reflink_end_cow_extent(ip, &offset_fsb, end_fsb); + + if (error) + trace_xfs_reflink_end_cow_error(ip, error, _RET_IP_); + return error; +} + +/* + * Fully remap all of the file's data fork at once, which is the critical part + * in achieving atomic behaviour. + * The regular CoW end path does not use function as to keep the block + * reservation per transaction as low as possible. + */ +int +xfs_reflink_end_atomic_cow( + struct xfs_inode *ip, + xfs_off_t offset, + xfs_off_t count) +{ + xfs_fileoff_t offset_fsb; + xfs_fileoff_t end_fsb; + int error = 0; + struct xfs_mount *mp = ip->i_mount; + struct xfs_trans *tp; + unsigned int resblks; + + trace_xfs_reflink_end_cow(ip, offset, count); + + offset_fsb = XFS_B_TO_FSBT(mp, offset); + end_fsb = XFS_B_TO_FSB(mp, offset + count); + + /* + * Each remapping operation could cause a btree split, so in the worst + * case that's one for each block. + */ + resblks = (end_fsb - offset_fsb) * + XFS_NEXTENTADD_SPACE_RES(mp, 1, XFS_DATA_FORK); + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_atomic_ioend, resblks, 0, + XFS_TRANS_RESERVE, &tp); + if (error) + return error; + + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, 0); + + while (end_fsb > offset_fsb && !error) { + error = xfs_reflink_end_cow_extent_locked(tp, ip, &offset_fsb, + end_fsb); + } + if (error) { + trace_xfs_reflink_end_cow_error(ip, error, _RET_IP_); + goto out_cancel; + } + error = xfs_trans_commit(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; +out_cancel: + xfs_trans_cancel(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; +} + +/* Compute the largest atomic write that we can complete through software. */ +xfs_extlen_t +xfs_reflink_max_atomic_cow( + struct xfs_mount *mp) +{ + /* We cannot do any atomic writes without out of place writes. */ + if (!xfs_can_sw_atomic_write(mp)) + return 0; + + /* + * Atomic write limits must always be a power-of-2, according to + * generic_atomic_write_valid. + */ + return rounddown_pow_of_two(xfs_calc_max_atomic_write_fsblocks(mp)); +} + +/* + * Free all CoW staging blocks that are still referenced by the ondisk refcount + * metadata. The ondisk metadata does not track which inode created the + * staging extent, so callers must ensure that there are no cached inodes with + * live CoW staging extents. + */ +int +xfs_reflink_recover_cow( + struct xfs_mount *mp) +{ + struct xfs_perag *pag = NULL; + struct xfs_rtgroup *rtg = NULL; + int error = 0; + + if (!xfs_has_reflink(mp)) + return 0; + + while ((pag = xfs_perag_next(mp, pag))) { + error = xfs_refcount_recover_cow_leftovers(pag_group(pag)); + if (error) { + xfs_perag_rele(pag); + return error; + } + } + + while ((rtg = xfs_rtgroup_next(mp, rtg))) { + error = xfs_refcount_recover_cow_leftovers(rtg_group(rtg)); + if (error) { + xfs_rtgroup_rele(rtg); + return error; + } + } + + return 0; +} + +/* + * Reflinking (Block) Ranges of Two Files Together + * + * First, ensure that the reflink flag is set on both inodes. The flag is an + * optimization to avoid unnecessary refcount btree lookups in the write path. + * + * Now we can iteratively remap the range of extents (and holes) in src to the + * corresponding ranges in dest. Let drange and srange denote the ranges of + * logical blocks in dest and src touched by the reflink operation. + * + * While the length of drange is greater than zero, + * - Read src's bmbt at the start of srange ("imap") + * - If imap doesn't exist, make imap appear to start at the end of srange + * with zero length. + * - If imap starts before srange, advance imap to start at srange. + * - If imap goes beyond srange, truncate imap to end at the end of srange. + * - Punch (imap start - srange start + imap len) blocks from dest at + * offset (drange start). + * - If imap points to a real range of pblks, + * > Increase the refcount of the imap's pblks + * > Map imap's pblks into dest at the offset + * (drange start + imap start - srange start) + * - Advance drange and srange by (imap start - srange start + imap len) + * + * Finally, if the reflink made dest longer, update both the in-core and + * on-disk file sizes. + * + * ASCII Art Demonstration: + * + * Let's say we want to reflink this source file: + * + * ----SSSSSSS-SSSSS----SSSSSS (src file) + * <--------------------> + * + * into this destination file: + * + * --DDDDDDDDDDDDDDDDDDD--DDD (dest file) + * <--------------------> + * '-' means a hole, and 'S' and 'D' are written blocks in the src and dest. + * Observe that the range has different logical offsets in either file. + * + * Consider that the first extent in the source file doesn't line up with our + * reflink range. Unmapping and remapping are separate operations, so we can + * unmap more blocks from the destination file than we remap. + * + * ----SSSSSSS-SSSSS----SSSSSS + * <-------> + * --DDDDD---------DDDDD--DDD + * <-------> + * + * Now remap the source extent into the destination file: + * + * ----SSSSSSS-SSSSS----SSSSSS + * <-------> + * --DDDDD--SSSSSSSDDDDD--DDD + * <-------> + * + * Do likewise with the second hole and extent in our range. Holes in the + * unmap range don't affect our operation. + * + * ----SSSSSSS-SSSSS----SSSSSS + * <----> + * --DDDDD--SSSSSSS-SSSSS-DDD + * <----> + * + * Finally, unmap and remap part of the third extent. This will increase the + * size of the destination file. + * + * ----SSSSSSS-SSSSS----SSSSSS + * <-----> + * --DDDDD--SSSSSSS-SSSSS----SSS + * <-----> + * + * Once we update the destination file's i_size, we're done. + */ + +/* + * Ensure the reflink bit is set in both inodes. + */ +STATIC int +xfs_reflink_set_inode_flag( + struct xfs_inode *src, + struct xfs_inode *dest) +{ + struct xfs_mount *mp = src->i_mount; + int error; + struct xfs_trans *tp; + + if (xfs_is_reflink_inode(src) && xfs_is_reflink_inode(dest)) + return 0; + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_ichange, 0, 0, 0, &tp); + if (error) + goto out_error; + + /* Lock both files against IO */ + if (I_INO(src) == I_INO(dest)) + xfs_ilock(src, XFS_ILOCK_EXCL); + else + xfs_lock_two_inodes(src, XFS_ILOCK_EXCL, dest, XFS_ILOCK_EXCL); + + if (!xfs_is_reflink_inode(src)) { + trace_xfs_reflink_set_inode_flag(src); + xfs_trans_ijoin(tp, src, XFS_ILOCK_EXCL); + src->i_diflags2 |= XFS_DIFLAG2_REFLINK; + xfs_trans_log_inode(tp, src, XFS_ILOG_CORE); + xfs_ifork_init_cow(src); + } else + xfs_iunlock(src, XFS_ILOCK_EXCL); + + if (I_INO(src) == I_INO(dest)) + goto commit_flags; + + if (!xfs_is_reflink_inode(dest)) { + trace_xfs_reflink_set_inode_flag(dest); + xfs_trans_ijoin(tp, dest, XFS_ILOCK_EXCL); + dest->i_diflags2 |= XFS_DIFLAG2_REFLINK; + xfs_trans_log_inode(tp, dest, XFS_ILOG_CORE); + xfs_ifork_init_cow(dest); + } else + xfs_iunlock(dest, XFS_ILOCK_EXCL); + +commit_flags: + error = xfs_trans_commit(tp); + if (error) + goto out_error; + return error; + +out_error: + trace_xfs_reflink_set_inode_flag_error(dest, error, _RET_IP_); + return error; +} + +/* + * Update destination inode size & cowextsize hint, if necessary. + */ +int +xfs_reflink_update_dest( + struct xfs_inode *dest, + xfs_off_t newlen, + xfs_extlen_t cowextsize, + unsigned int remap_flags) +{ + struct xfs_mount *mp = dest->i_mount; + struct xfs_trans *tp; + int error; + + if (newlen <= i_size_read(VFS_I(dest)) && cowextsize == 0) + return 0; + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_ichange, 0, 0, 0, &tp); + if (error) + goto out_error; + + xfs_ilock(dest, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, dest, XFS_ILOCK_EXCL); + + if (newlen > i_size_read(VFS_I(dest))) { + trace_xfs_reflink_update_inode_size(dest, newlen); + i_size_write(VFS_I(dest), newlen); + dest->i_disk_size = newlen; + } + + if (cowextsize) { + dest->i_cowextsize = cowextsize; + dest->i_diflags2 |= XFS_DIFLAG2_COWEXTSIZE; + } + + xfs_trans_log_inode(tp, dest, XFS_ILOG_CORE); + + error = xfs_trans_commit(tp); + if (error) + goto out_error; + return error; + +out_error: + trace_xfs_reflink_update_inode_size_error(dest, error, _RET_IP_); + return error; +} + +/* + * Do we have enough reserve in this AG to handle a reflink? The refcount + * btree already reserved all the space it needs, but the rmap btree can grow + * infinitely, so we won't allow more reflinks when the AG is down to the + * btree reserves. + */ +static int +xfs_reflink_ag_has_free_space( + struct xfs_mount *mp, + struct xfs_inode *ip, + xfs_fsblock_t fsb) +{ + struct xfs_perag *pag; + xfs_agnumber_t agno; + int error = 0; + + if (!xfs_has_rmapbt(mp)) + return 0; + if (XFS_IS_REALTIME_INODE(ip)) { + if (xfs_metafile_resv_critical(mp)) + return -ENOSPC; + return 0; + } + + agno = XFS_FSB_TO_AGNO(mp, fsb); + pag = xfs_perag_get(mp, agno); + if (xfs_ag_resv_critical(pag, XFS_AG_RESV_RMAPBT) || + xfs_ag_resv_critical(pag, XFS_AG_RESV_METADATA)) + error = -ENOSPC; + xfs_perag_put(pag); + return error; +} + +/* + * Remap the given extent into the file. The dmap blockcount will be set to + * the number of blocks that were actually remapped. + */ +STATIC int +xfs_reflink_remap_extent( + struct xfs_inode *ip, + struct xfs_bmbt_irec *dmap, + xfs_off_t new_isize) +{ + struct xfs_bmbt_irec smap; + struct xfs_mount *mp = ip->i_mount; + struct xfs_trans *tp; + xfs_off_t newlen; + int64_t qdelta = 0; + unsigned int dblocks, rblocks, resblks; + bool quota_reserved = true; + bool smap_real; + bool dmap_written = xfs_bmap_is_written_extent(dmap); + bool isrt = XFS_IS_REALTIME_INODE(ip); + int iext_delta = 0; + int nimaps; + int error; + + /* + * Start a rolling transaction to switch the mappings. + * + * Adding a written extent to the extent map can cause a bmbt split, + * and removing a mapped extent from the extent can cause a bmbt split. + * The two operations cannot both cause a split since they operate on + * the same index in the bmap btree, so we only need a reservation for + * one bmbt split if either thing is happening. However, we haven't + * locked the inode yet, so we reserve assuming this is the case. + * + * The first allocation call tries to reserve enough space to handle + * mapping dmap into a sparse part of the file plus the bmbt split. We + * haven't locked the inode or read the existing mapping yet, so we do + * not know for sure that we need the space. This should succeed most + * of the time. + * + * If the first attempt fails, try again but reserving only enough + * space to handle a bmbt split. This is the hard minimum requirement, + * and we revisit quota reservations later when we know more about what + * we're remapping. + */ + resblks = XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK); + if (XFS_IS_REALTIME_INODE(ip)) { + dblocks = resblks; + rblocks = dmap->br_blockcount; + } else { + dblocks = resblks + dmap->br_blockcount; + rblocks = 0; + } + error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, + dblocks, rblocks, false, &tp); + if (error == -EDQUOT || error == -ENOSPC) { + quota_reserved = false; + error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_write, + resblks, 0, false, &tp); + } + if (error) + goto out; + + /* + * Read what's currently mapped in the destination file into smap. + * If smap isn't a hole, we will have to remove it before we can add + * dmap to the destination file. + */ + nimaps = 1; + error = xfs_bmapi_read(ip, dmap->br_startoff, dmap->br_blockcount, + &smap, &nimaps, 0); + if (error) + goto out_cancel; + ASSERT(nimaps == 1 && smap.br_startoff == dmap->br_startoff); + smap_real = xfs_bmap_is_real_extent(&smap); + + /* + * We can only remap as many blocks as the smaller of the two extent + * maps, because we can only remap one extent at a time. + */ + dmap->br_blockcount = min(dmap->br_blockcount, smap.br_blockcount); + ASSERT(dmap->br_blockcount == smap.br_blockcount); + + trace_xfs_reflink_remap_extent_dest(ip, &smap); + + /* + * Two extents mapped to the same physical block must not have + * different states; that's filesystem corruption. Move on to the next + * extent if they're both holes or both the same physical extent. + */ + if (dmap->br_startblock == smap.br_startblock) { + if (dmap->br_state != smap.br_state) { + xfs_bmap_mark_sick(ip, XFS_DATA_FORK); + error = -EFSCORRUPTED; + } + goto out_cancel; + } + + /* If both extents are unwritten, leave them alone. */ + if (dmap->br_state == XFS_EXT_UNWRITTEN && + smap.br_state == XFS_EXT_UNWRITTEN) + goto out_cancel; + + /* No reflinking if the AG of the dest mapping is low on space. */ + if (dmap_written) { + error = xfs_reflink_ag_has_free_space(mp, ip, + dmap->br_startblock); + if (error) + goto out_cancel; + } + + /* + * Increase quota reservation if we think the quota block counter for + * this file could increase. + * + * If we are mapping a written extent into the file, we need to have + * enough quota block count reservation to handle the blocks in that + * extent. We log only the delta to the quota block counts, so if the + * extent we're unmapping also has blocks allocated to it, we don't + * need a quota reservation for the extent itself. + * + * Note that if we're replacing a delalloc reservation with a written + * extent, we have to take the full quota reservation because removing + * the delalloc reservation gives the block count back to the quota + * count. This is suboptimal, but the VFS flushed the dest range + * before we started. That should have removed all the delalloc + * reservations, but we code defensively. + * + * xfs_trans_alloc_inode above already tried to grab an even larger + * quota reservation, and kicked off a blockgc scan if it couldn't. + * If we can't get a potentially smaller quota reservation now, we're + * done. + */ + if (!quota_reserved && !smap_real && dmap_written) { + if (XFS_IS_REALTIME_INODE(ip)) { + dblocks = 0; + rblocks = dmap->br_blockcount; + } else { + dblocks = dmap->br_blockcount; + rblocks = 0; + } + error = xfs_trans_reserve_quota_nblks(tp, ip, dblocks, rblocks, + false); + if (error) + goto out_cancel; + } + + if (smap_real) + ++iext_delta; + + if (dmap_written) + ++iext_delta; + + error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, iext_delta); + if (error) + goto out_cancel; + + if (smap_real) { + /* + * If the extent we're unmapping is backed by storage (written + * or not), unmap the extent and drop its refcount. + */ + xfs_bmap_unmap_extent(tp, ip, XFS_DATA_FORK, &smap); + xfs_refcount_decrease_extent(tp, isrt, &smap); + qdelta -= smap.br_blockcount; + } else if (smap.br_startblock == DELAYSTARTBLOCK) { + int done; + + /* + * If the extent we're unmapping is a delalloc reservation, + * we can use the regular bunmapi function to release the + * incore state. Dropping the delalloc reservation takes care + * of the quota reservation for us. + */ + error = xfs_bunmapi(NULL, ip, smap.br_startoff, + smap.br_blockcount, 0, 1, &done); + if (error) + goto out_cancel; + ASSERT(done); + } + + /* + * If the extent we're sharing is backed by written storage, increase + * its refcount and map it into the file. + */ + if (dmap_written) { + xfs_refcount_increase_extent(tp, isrt, dmap); + xfs_bmap_map_extent(tp, ip, XFS_DATA_FORK, dmap); + qdelta += dmap->br_blockcount; + } + + xfs_reflink_update_quota(tp, ip, false, qdelta); + + /* Update dest isize if needed. */ + newlen = XFS_FSB_TO_B(mp, dmap->br_startoff + dmap->br_blockcount); + newlen = min_t(xfs_off_t, newlen, new_isize); + if (newlen > i_size_read(VFS_I(ip))) { + trace_xfs_reflink_update_inode_size(ip, newlen); + i_size_write(VFS_I(ip), newlen); + ip->i_disk_size = newlen; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + } + + /* Commit everything and unlock. */ + error = xfs_trans_commit(tp); + goto out_unlock; + +out_cancel: + xfs_trans_cancel(tp); +out_unlock: + xfs_iunlock(ip, XFS_ILOCK_EXCL); +out: + if (error) + trace_xfs_reflink_remap_extent_error(ip, error, _RET_IP_); + return error; +} + +/* Remap a range of one file to the other. */ +int +xfs_reflink_remap_blocks( + struct xfs_inode *src, + loff_t pos_in, + struct xfs_inode *dest, + loff_t pos_out, + loff_t remap_len, + loff_t *remapped) +{ + struct xfs_bmbt_irec imap; + struct xfs_mount *mp = src->i_mount; + xfs_fileoff_t srcoff = XFS_B_TO_FSBT(mp, pos_in); + xfs_fileoff_t destoff = XFS_B_TO_FSBT(mp, pos_out); + xfs_filblks_t len; + xfs_filblks_t remapped_len = 0; + xfs_off_t new_isize = pos_out + remap_len; + int nimaps; + int error = 0; + + len = min_t(xfs_filblks_t, XFS_B_TO_FSB(mp, remap_len), + XFS_MAX_FILEOFF); + + trace_xfs_reflink_remap_blocks(src, srcoff, len, dest, destoff); + + while (len > 0) { + unsigned int lock_mode; + + /* Read extent from the source file */ + nimaps = 1; + lock_mode = xfs_ilock_data_map_shared(src); + error = xfs_bmapi_read(src, srcoff, len, &imap, &nimaps, 0); + xfs_iunlock(src, lock_mode); + if (error) + break; + /* + * The caller supposedly flushed all dirty pages in the source + * file range, which means that writeback should have allocated + * or deleted all delalloc reservations in that range. If we + * find one, that's a good sign that something is seriously + * wrong here. + */ + ASSERT(nimaps == 1 && imap.br_startoff == srcoff); + if (imap.br_startblock == DELAYSTARTBLOCK) { + ASSERT(imap.br_startblock != DELAYSTARTBLOCK); + xfs_bmap_mark_sick(src, XFS_DATA_FORK); + error = -EFSCORRUPTED; + break; + } + + trace_xfs_reflink_remap_extent_src(src, &imap); + + /* Remap into the destination file at the given offset. */ + imap.br_startoff = destoff; + error = xfs_reflink_remap_extent(dest, &imap, new_isize); + if (error) + break; + + if (fatal_signal_pending(current)) { + error = -EINTR; + break; + } + + /* Advance drange/srange */ + srcoff += imap.br_blockcount; + destoff += imap.br_blockcount; + len -= imap.br_blockcount; + remapped_len += imap.br_blockcount; + cond_resched(); + } + + if (error) + trace_xfs_reflink_remap_blocks_error(dest, error, _RET_IP_); + *remapped = min_t(loff_t, remap_len, + XFS_FSB_TO_B(src->i_mount, remapped_len)); + return error; +} + +/* + * If we're reflinking to a point past the destination file's EOF, we must + * zero any speculative post-EOF preallocations that sit between the old EOF + * and the destination file offset. + */ +static int +xfs_reflink_zero_posteof( + struct xfs_inode *ip, + loff_t pos) +{ + loff_t isize = i_size_read(VFS_I(ip)); + + if (pos <= isize) + return 0; + + trace_xfs_zero_eof(ip, isize, pos - isize); + return xfs_zero_range(ip, isize, pos - isize, NULL, NULL); +} + +/* + * Prepare two files for range cloning. Upon a successful return both inodes + * will have the iolock and mmaplock held, the page cache of the out file will + * be truncated, and any leases on the out file will have been broken. This + * function borrows heavily from xfs_file_aio_write_checks. + * + * The VFS allows partial EOF blocks to "match" for dedupe even though it hasn't + * checked that the bytes beyond EOF physically match. Hence we cannot use the + * EOF block in the source dedupe range because it's not a complete block match, + * hence can introduce a corruption into the file that has it's block replaced. + * + * In similar fashion, the VFS file cloning also allows partial EOF blocks to be + * "block aligned" for the purposes of cloning entire files. However, if the + * source file range includes the EOF block and it lands within the existing EOF + * of the destination file, then we can expose stale data from beyond the source + * file EOF in the destination file. + * + * XFS doesn't support partial block sharing, so in both cases we have check + * these cases ourselves. For dedupe, we can simply round the length to dedupe + * down to the previous whole block and ignore the partial EOF block. While this + * means we can't dedupe the last block of a file, this is an acceptible + * tradeoff for simplicity on implementation. + * + * For cloning, we want to share the partial EOF block if it is also the new EOF + * block of the destination file. If the partial EOF block lies inside the + * existing destination EOF, then we have to abort the clone to avoid exposing + * stale data in the destination file. Hence we reject these clone attempts with + * -EINVAL in this case. + */ +int +xfs_reflink_remap_prep( + struct file *file_in, + loff_t pos_in, + struct file *file_out, + loff_t pos_out, + loff_t *len, + unsigned int remap_flags) +{ + struct inode *inode_in = file_inode(file_in); + struct xfs_inode *src = XFS_I(inode_in); + struct inode *inode_out = file_inode(file_out); + struct xfs_inode *dest = XFS_I(inode_out); + int ret; + + /* Lock both files against IO */ + ret = xfs_ilock2_io_mmap(src, dest); + if (ret) + return ret; + + /* Check file eligibility and prepare for block sharing. */ + ret = -EINVAL; + /* Can't reflink between data and rt volumes */ + if (XFS_IS_REALTIME_INODE(src) != XFS_IS_REALTIME_INODE(dest)) + goto out_unlock; + + /* Don't share DAX file data with non-DAX file. */ + if (IS_DAX(inode_in) != IS_DAX(inode_out)) + goto out_unlock; + + if (!IS_DAX(inode_in)) + ret = generic_remap_file_range_prep(file_in, pos_in, file_out, + pos_out, len, remap_flags); + else + ret = dax_remap_file_range_prep(file_in, pos_in, file_out, + pos_out, len, remap_flags, &xfs_read_iomap_ops); + if (ret || *len == 0) + goto out_unlock; + + /* Attach dquots to dest inode before changing block map */ + ret = xfs_qm_dqattach(dest); + if (ret) + goto out_unlock; + + /* + * Zero existing post-eof speculative preallocations in the destination + * file. + */ + ret = xfs_reflink_zero_posteof(dest, pos_out); + if (ret) + goto out_unlock; + + /* Set flags and remap blocks. */ + ret = xfs_reflink_set_inode_flag(src, dest); + if (ret) + goto out_unlock; + + /* + * If pos_out > EOF, we may have dirtied blocks between EOF and + * pos_out. In that case, we need to extend the flush and unmap to cover + * from EOF to the end of the copy length. + */ + if (pos_out > XFS_ISIZE(dest)) { + loff_t flen = *len + (pos_out - XFS_ISIZE(dest)); + ret = xfs_flush_unmap_range(dest, XFS_ISIZE(dest), flen); + } else { + ret = xfs_flush_unmap_range(dest, pos_out, *len); + } + if (ret) + goto out_unlock; + + xfs_iflags_set(src, XFS_IREMAPPING); + if (inode_in != inode_out) + xfs_ilock_demote(src, XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL); + + return 0; +out_unlock: + xfs_iunlock2_io_mmap(src, dest); + return ret; +} + +/* Does this inode need the reflink flag? */ +int +xfs_reflink_inode_has_shared_extents( + struct xfs_trans *tp, + struct xfs_inode *ip, + bool *has_shared) +{ + struct xfs_bmbt_irec got; + struct xfs_mount *mp = ip->i_mount; + struct xfs_ifork *ifp; + struct xfs_iext_cursor icur; + bool found; + int error; + + ifp = xfs_ifork_ptr(ip, XFS_DATA_FORK); + error = xfs_iread_extents(tp, ip, XFS_DATA_FORK); + if (error) + return error; + + *has_shared = false; + found = xfs_iext_lookup_extent(ip, ifp, 0, &icur, &got); + while (found) { + xfs_extlen_t shared_offset, shared_len; + + if (isnullstartblock(got.br_startblock) || + got.br_state != XFS_EXT_NORM) + goto next; + + if (XFS_IS_REALTIME_INODE(ip)) + error = xfs_reflink_find_rtshared(mp, tp, &got, + &shared_offset, &shared_len, false); + else + error = xfs_reflink_find_shared(mp, tp, &got, + &shared_offset, &shared_len, false); + if (error) + return error; + + /* Is there still a shared block here? */ + if (shared_len) { + *has_shared = true; + return 0; + } +next: + found = xfs_iext_next_extent(ifp, &icur, &got); + } + + return 0; +} + +/* + * Clear the inode reflink flag if there are no shared extents. + * + * The caller is responsible for joining the inode to the transaction passed in. + * The inode will be joined to the transaction that is returned to the caller. + */ +int +xfs_reflink_clear_inode_flag( + struct xfs_inode *ip, + struct xfs_trans **tpp) +{ + bool needs_flag; + int error = 0; + + ASSERT(xfs_is_reflink_inode(ip)); + + if (!xfs_can_free_cowblocks(ip)) + return 0; + + error = xfs_reflink_inode_has_shared_extents(*tpp, ip, &needs_flag); + if (error || needs_flag) + return error; + + /* + * We didn't find any shared blocks so turn off the reflink flag. + * First, get rid of any leftover CoW mappings. + */ + error = xfs_reflink_cancel_cow_blocks(ip, tpp, 0, XFS_MAX_FILEOFF, + true); + if (error) + return error; + + /* Clear the inode flag. */ + trace_xfs_reflink_unset_inode_flag(ip); + ip->i_diflags2 &= ~XFS_DIFLAG2_REFLINK; + xfs_inode_clear_cowblocks_tag(ip); + xfs_trans_log_inode(*tpp, ip, XFS_ILOG_CORE); + + return error; +} + +/* + * Clear the inode reflink flag if there are no shared extents and the size + * hasn't changed. + */ +STATIC int +xfs_reflink_try_clear_inode_flag( + struct xfs_inode *ip) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_trans *tp; + int error = 0; + + /* Start a rolling transaction to remove the mappings */ + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, 0, 0, 0, &tp); + if (error) + return error; + + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, 0); + + error = xfs_reflink_clear_inode_flag(ip, &tp); + if (error) + goto cancel; + + error = xfs_trans_commit(tp); + if (error) + goto out; + + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return 0; +cancel: + xfs_trans_cancel(tp); +out: + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; +} + +/* + * Pre-COW all shared blocks within a given byte range of a file and turn off + * the reflink flag if we unshare all of the file's blocks. + */ +int +xfs_reflink_unshare( + struct xfs_inode *ip, + xfs_off_t offset, + xfs_off_t len) +{ + struct inode *inode = VFS_I(ip); + int error; + + if (!xfs_is_reflink_inode(ip)) + return 0; + + trace_xfs_reflink_unshare(ip, offset, len); + + inode_dio_wait(inode); + + if (IS_DAX(inode)) + error = dax_file_unshare(inode, offset, len, + &xfs_dax_write_iomap_ops); + else + error = iomap_file_unshare(inode, offset, len, + &xfs_buffered_write_iomap_ops, + &xfs_iomap_write_ops); + if (error) + goto out; + + error = filemap_write_and_wait_range(inode->i_mapping, offset, + offset + len - 1); + if (error) + goto out; + + /* Turn off the reflink flag if possible. */ + error = xfs_reflink_try_clear_inode_flag(ip); + if (error) + goto out; + return 0; + +out: + trace_xfs_reflink_unshare_error(ip, error, _RET_IP_); + return error; +} + +/* + * Can we use reflink with this realtime extent size? Note that we don't check + * for rblocks > 0 here because this can be called as part of attaching a new + * rt section. + */ +bool +xfs_reflink_supports_rextsize( + struct xfs_mount *mp, + unsigned int rextsize) +{ + /* reflink on the realtime device requires rtgroups */ + if (!xfs_has_rtgroups(mp)) + return false; + + /* + * Reflink doesn't support rt extent size larger than a single fsblock + * because we would have to perform CoW-around for unaligned write + * requests to guarantee that we always remap entire rt extents. + */ + if (rextsize != 1) + return false; + + return true; +} diff --git a/fs/xfs/xfs_reflink.h b/fs/xfs/xfs_reflink.h new file mode 100644 index 000000000..9d1ed9bb0 --- /dev/null +++ b/fs/xfs/xfs_reflink.h @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2016 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_REFLINK_H +#define __XFS_REFLINK_H 1 + +/* + * Check whether it is safe to free COW fork blocks from an inode. It is unsafe + * to do so when an inode has dirty cache or I/O in-flight, even if no shared + * extents exist in the data fork, because outstanding I/O may target blocks + * that were speculatively allocated to the COW fork. + */ +static inline bool +xfs_can_free_cowblocks(struct xfs_inode *ip) +{ + struct inode *inode = VFS_I(ip); + + if ((inode_state_read_once(inode) & I_DIRTY_PAGES) || + mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY) || + mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK) || + atomic_read(&inode->i_dio_count)) + return false; + return true; +} + +int xfs_reflink_trim_around_shared(struct xfs_inode *ip, + struct xfs_bmbt_irec *irec, bool *shared); +int xfs_bmap_trim_cow(struct xfs_inode *ip, struct xfs_bmbt_irec *imap, + bool *shared); + +int xfs_reflink_allocate_cow(struct xfs_inode *ip, struct xfs_bmbt_irec *imap, + struct xfs_bmbt_irec *cmap, bool *shared, uint *lockmode, + bool convert_now); +extern int xfs_reflink_convert_cow(struct xfs_inode *ip, xfs_off_t offset, + xfs_off_t count); +int xfs_reflink_convert_cow_locked(struct xfs_inode *ip, + xfs_fileoff_t offset_fsb, xfs_filblks_t count_fsb); + +extern int xfs_reflink_cancel_cow_blocks(struct xfs_inode *ip, + struct xfs_trans **tpp, xfs_fileoff_t offset_fsb, + xfs_fileoff_t end_fsb, bool cancel_real); +extern int xfs_reflink_cancel_cow_range(struct xfs_inode *ip, xfs_off_t offset, + xfs_off_t count, bool cancel_real); +extern int xfs_reflink_end_cow(struct xfs_inode *ip, xfs_off_t offset, + xfs_off_t count); +int xfs_reflink_end_atomic_cow(struct xfs_inode *ip, xfs_off_t offset, + xfs_off_t count); +extern int xfs_reflink_recover_cow(struct xfs_mount *mp); +extern loff_t xfs_reflink_remap_range(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, loff_t len, + unsigned int remap_flags); +extern int xfs_reflink_inode_has_shared_extents(struct xfs_trans *tp, + struct xfs_inode *ip, bool *has_shared); +extern int xfs_reflink_clear_inode_flag(struct xfs_inode *ip, + struct xfs_trans **tpp); +extern int xfs_reflink_unshare(struct xfs_inode *ip, xfs_off_t offset, + xfs_off_t len); +extern int xfs_reflink_remap_prep(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, loff_t *len, + unsigned int remap_flags); +extern int xfs_reflink_remap_blocks(struct xfs_inode *src, loff_t pos_in, + struct xfs_inode *dest, loff_t pos_out, loff_t remap_len, + loff_t *remapped); +extern int xfs_reflink_update_dest(struct xfs_inode *dest, xfs_off_t newlen, + xfs_extlen_t cowextsize, unsigned int remap_flags); + +bool xfs_reflink_supports_rextsize(struct xfs_mount *mp, unsigned int rextsize); + +xfs_extlen_t xfs_reflink_max_atomic_cow(struct xfs_mount *mp); + +#endif /* __XFS_REFLINK_H */ diff --git a/fs/xfs/xfs_rmap_item.c b/fs/xfs/xfs_rmap_item.c new file mode 100644 index 000000000..2a3a73a85 --- /dev/null +++ b/fs/xfs/xfs_rmap_item.c @@ -0,0 +1,891 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2016 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_shared.h" +#include "xfs_mount.h" +#include "xfs_defer.h" +#include "xfs_trans.h" +#include "xfs_trans_priv.h" +#include "xfs_rmap_item.h" +#include "xfs_log.h" +#include "xfs_rmap.h" +#include "xfs_error.h" +#include "xfs_log_priv.h" +#include "xfs_log_recover.h" +#include "xfs_ag.h" +#include "xfs_btree.h" +#include "xfs_trace.h" +#include "xfs_rtgroup.h" + +struct kmem_cache *xfs_rui_cache; +struct kmem_cache *xfs_rud_cache; + +static const struct xfs_item_ops xfs_rui_item_ops; + +static inline struct xfs_rui_log_item *RUI_ITEM(struct xfs_log_item *lip) +{ + return container_of(lip, struct xfs_rui_log_item, rui_item); +} + +STATIC void +xfs_rui_item_free( + struct xfs_rui_log_item *ruip) +{ + kvfree(ruip->rui_item.li_lv_shadow); + if (ruip->rui_format.rui_nextents > XFS_RUI_MAX_FAST_EXTENTS) + kfree(ruip); + else + kmem_cache_free(xfs_rui_cache, ruip); +} + +/* + * Freeing the RUI requires that we remove it from the AIL if it has already + * been placed there. However, the RUI may not yet have been placed in the AIL + * when called by xfs_rui_release() from RUD processing due to the ordering of + * committed vs unpin operations in bulk insert operations. Hence the reference + * count to ensure only the last caller frees the RUI. + */ +STATIC void +xfs_rui_release( + struct xfs_rui_log_item *ruip) +{ + ASSERT(atomic_read(&ruip->rui_refcount) > 0); + if (!atomic_dec_and_test(&ruip->rui_refcount)) + return; + + xfs_trans_ail_delete(&ruip->rui_item, 0); + xfs_rui_item_free(ruip); +} + +STATIC void +xfs_rui_item_size( + struct xfs_log_item *lip, + int *nvecs, + int *nbytes) +{ + struct xfs_rui_log_item *ruip = RUI_ITEM(lip); + + *nvecs += 1; + *nbytes += xfs_rui_log_format_sizeof(ruip->rui_format.rui_nextents); +} + +unsigned int xfs_rui_log_space(unsigned int nr) +{ + return xlog_item_space(1, xfs_rui_log_format_sizeof(nr)); +} + +/* + * This is called to fill in the vector of log iovecs for the + * given rui log item. We use only 1 iovec, and we point that + * at the rui_log_format structure embedded in the rui item. + * It is at this point that we assert that all of the extent + * slots in the rui item have been filled. + */ +STATIC void +xfs_rui_item_format( + struct xfs_log_item *lip, + struct xlog_format_buf *lfb) +{ + struct xfs_rui_log_item *ruip = RUI_ITEM(lip); + + ASSERT(atomic_read(&ruip->rui_next_extent) == + ruip->rui_format.rui_nextents); + + ASSERT(lip->li_type == XFS_LI_RUI || lip->li_type == XFS_LI_RUI_RT); + + ruip->rui_format.rui_type = lip->li_type; + ruip->rui_format.rui_size = 1; + + xlog_format_copy(lfb, XLOG_REG_TYPE_RUI_FORMAT, &ruip->rui_format, + xfs_rui_log_format_sizeof(ruip->rui_format.rui_nextents)); +} + +/* + * The unpin operation is the last place an RUI is manipulated in the log. It is + * either inserted in the AIL or aborted in the event of a log I/O error. In + * either case, the RUI transaction has been successfully committed to make it + * this far. Therefore, we expect whoever committed the RUI to either construct + * and commit the RUD or drop the RUD's reference in the event of error. Simply + * drop the log's RUI reference now that the log is done with it. + */ +STATIC void +xfs_rui_item_unpin( + struct xfs_log_item *lip, + int remove) +{ + struct xfs_rui_log_item *ruip = RUI_ITEM(lip); + + xfs_rui_release(ruip); +} + +/* + * The RUI has been either committed or aborted if the transaction has been + * cancelled. If the transaction was cancelled, an RUD isn't going to be + * constructed and thus we free the RUI here directly. + */ +STATIC void +xfs_rui_item_release( + struct xfs_log_item *lip) +{ + xfs_rui_release(RUI_ITEM(lip)); +} + +/* + * Allocate and initialize an rui item with the given number of extents. + */ +STATIC struct xfs_rui_log_item * +xfs_rui_init( + struct xfs_mount *mp, + unsigned short item_type, + uint nextents) + +{ + struct xfs_rui_log_item *ruip; + + ASSERT(nextents > 0); + ASSERT(item_type == XFS_LI_RUI || item_type == XFS_LI_RUI_RT); + + if (nextents > XFS_RUI_MAX_FAST_EXTENTS) + ruip = kzalloc(xfs_rui_log_item_sizeof(nextents), + GFP_KERNEL | __GFP_NOFAIL); + else + ruip = kmem_cache_zalloc(xfs_rui_cache, + GFP_KERNEL | __GFP_NOFAIL); + + xfs_log_item_init(mp, &ruip->rui_item, item_type, &xfs_rui_item_ops); + ruip->rui_format.rui_nextents = nextents; + ruip->rui_format.rui_id = (uintptr_t)(void *)ruip; + atomic_set(&ruip->rui_next_extent, 0); + atomic_set(&ruip->rui_refcount, 2); + + return ruip; +} + +static inline struct xfs_rud_log_item *RUD_ITEM(struct xfs_log_item *lip) +{ + return container_of(lip, struct xfs_rud_log_item, rud_item); +} + +STATIC void +xfs_rud_item_size( + struct xfs_log_item *lip, + int *nvecs, + int *nbytes) +{ + *nvecs += 1; + *nbytes += sizeof(struct xfs_rud_log_format); +} + +unsigned int xfs_rud_log_space(void) +{ + return xlog_item_space(1, sizeof(struct xfs_rud_log_format)); +} + +/* + * This is called to fill in the vector of log iovecs for the + * given rud log item. We use only 1 iovec, and we point that + * at the rud_log_format structure embedded in the rud item. + * It is at this point that we assert that all of the extent + * slots in the rud item have been filled. + */ +STATIC void +xfs_rud_item_format( + struct xfs_log_item *lip, + struct xlog_format_buf *lfb) +{ + struct xfs_rud_log_item *rudp = RUD_ITEM(lip); + + ASSERT(lip->li_type == XFS_LI_RUD || lip->li_type == XFS_LI_RUD_RT); + + rudp->rud_format.rud_type = lip->li_type; + rudp->rud_format.rud_size = 1; + + xlog_format_copy(lfb, XLOG_REG_TYPE_RUD_FORMAT, &rudp->rud_format, + sizeof(struct xfs_rud_log_format)); +} + +/* + * The RUD is either committed or aborted if the transaction is cancelled. If + * the transaction is cancelled, drop our reference to the RUI and free the + * RUD. + */ +STATIC void +xfs_rud_item_release( + struct xfs_log_item *lip) +{ + struct xfs_rud_log_item *rudp = RUD_ITEM(lip); + + xfs_rui_release(rudp->rud_ruip); + kvfree(rudp->rud_item.li_lv_shadow); + kmem_cache_free(xfs_rud_cache, rudp); +} + +static struct xfs_log_item * +xfs_rud_item_intent( + struct xfs_log_item *lip) +{ + return &RUD_ITEM(lip)->rud_ruip->rui_item; +} + +static const struct xfs_item_ops xfs_rud_item_ops = { + .flags = XFS_ITEM_RELEASE_WHEN_COMMITTED | + XFS_ITEM_INTENT_DONE, + .iop_size = xfs_rud_item_size, + .iop_format = xfs_rud_item_format, + .iop_release = xfs_rud_item_release, + .iop_intent = xfs_rud_item_intent, +}; + +static inline struct xfs_rmap_intent *ri_entry(const struct list_head *e) +{ + return list_entry(e, struct xfs_rmap_intent, ri_list); +} + +static inline bool +xfs_rui_item_isrt(const struct xfs_log_item *lip) +{ + ASSERT(lip->li_type == XFS_LI_RUI || lip->li_type == XFS_LI_RUI_RT); + + return lip->li_type == XFS_LI_RUI_RT; +} + +/* Sort rmap intents by AG. */ +static int +xfs_rmap_update_diff_items( + void *priv, + const struct list_head *a, + const struct list_head *b) +{ + struct xfs_rmap_intent *ra = ri_entry(a); + struct xfs_rmap_intent *rb = ri_entry(b); + + return cmp_int(ra->ri_group->xg_gno, rb->ri_group->xg_gno); +} + +/* Log rmap updates in the intent item. */ +STATIC void +xfs_rmap_update_log_item( + struct xfs_trans *tp, + struct xfs_rui_log_item *ruip, + struct xfs_rmap_intent *ri) +{ + uint next_extent; + struct xfs_map_extent *map; + + /* + * atomic_inc_return gives us the value after the increment; + * we want to use it as an array index so we need to subtract 1 from + * it. + */ + next_extent = atomic_inc_return(&ruip->rui_next_extent) - 1; + ASSERT(next_extent < ruip->rui_format.rui_nextents); + map = &ruip->rui_format.rui_extents[next_extent]; + map->me_owner = ri->ri_owner; + map->me_startblock = ri->ri_bmap.br_startblock; + map->me_startoff = ri->ri_bmap.br_startoff; + map->me_len = ri->ri_bmap.br_blockcount; + + map->me_flags = 0; + if (ri->ri_bmap.br_state == XFS_EXT_UNWRITTEN) + map->me_flags |= XFS_RMAP_EXTENT_UNWRITTEN; + if (ri->ri_whichfork == XFS_ATTR_FORK) + map->me_flags |= XFS_RMAP_EXTENT_ATTR_FORK; + switch (ri->ri_type) { + case XFS_RMAP_MAP: + map->me_flags |= XFS_RMAP_EXTENT_MAP; + break; + case XFS_RMAP_MAP_SHARED: + map->me_flags |= XFS_RMAP_EXTENT_MAP_SHARED; + break; + case XFS_RMAP_UNMAP: + map->me_flags |= XFS_RMAP_EXTENT_UNMAP; + break; + case XFS_RMAP_UNMAP_SHARED: + map->me_flags |= XFS_RMAP_EXTENT_UNMAP_SHARED; + break; + case XFS_RMAP_CONVERT: + map->me_flags |= XFS_RMAP_EXTENT_CONVERT; + break; + case XFS_RMAP_CONVERT_SHARED: + map->me_flags |= XFS_RMAP_EXTENT_CONVERT_SHARED; + break; + case XFS_RMAP_ALLOC: + map->me_flags |= XFS_RMAP_EXTENT_ALLOC; + break; + case XFS_RMAP_FREE: + map->me_flags |= XFS_RMAP_EXTENT_FREE; + break; + default: + ASSERT(0); + } +} + +static struct xfs_log_item * +__xfs_rmap_update_create_intent( + struct xfs_trans *tp, + struct list_head *items, + unsigned int count, + bool sort, + unsigned short item_type) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_rui_log_item *ruip; + struct xfs_rmap_intent *ri; + + ASSERT(count > 0); + + ruip = xfs_rui_init(mp, item_type, count); + if (sort) + list_sort(mp, items, xfs_rmap_update_diff_items); + list_for_each_entry(ri, items, ri_list) + xfs_rmap_update_log_item(tp, ruip, ri); + return &ruip->rui_item; +} + +static struct xfs_log_item * +xfs_rmap_update_create_intent( + struct xfs_trans *tp, + struct list_head *items, + unsigned int count, + bool sort) +{ + return __xfs_rmap_update_create_intent(tp, items, count, sort, + XFS_LI_RUI); +} + +static inline unsigned short +xfs_rud_type_from_rui(const struct xfs_rui_log_item *ruip) +{ + return xfs_rui_item_isrt(&ruip->rui_item) ? XFS_LI_RUD_RT : XFS_LI_RUD; +} + +/* Get an RUD so we can process all the deferred rmap updates. */ +static struct xfs_log_item * +xfs_rmap_update_create_done( + struct xfs_trans *tp, + struct xfs_log_item *intent, + unsigned int count) +{ + struct xfs_rui_log_item *ruip = RUI_ITEM(intent); + struct xfs_rud_log_item *rudp; + + rudp = kmem_cache_zalloc(xfs_rud_cache, GFP_KERNEL | __GFP_NOFAIL); + xfs_log_item_init(tp->t_mountp, &rudp->rud_item, + xfs_rud_type_from_rui(ruip), &xfs_rud_item_ops); + rudp->rud_ruip = ruip; + rudp->rud_format.rud_rui_id = ruip->rui_format.rui_id; + + return &rudp->rud_item; +} + +/* Add this deferred RUI to the transaction. */ +void +xfs_rmap_defer_add( + struct xfs_trans *tp, + struct xfs_rmap_intent *ri) +{ + struct xfs_mount *mp = tp->t_mountp; + + /* + * Deferred rmap updates for the realtime and data sections must use + * separate transactions to finish deferred work because updates to + * realtime metadata files can lock AGFs to allocate btree blocks and + * we don't want that mixing with the AGF locks taken to finish data + * section updates. + */ + ri->ri_group = xfs_group_intent_get(mp, ri->ri_bmap.br_startblock, + ri->ri_realtime ? XG_TYPE_RTG : XG_TYPE_AG); + + trace_xfs_rmap_defer(mp, ri); + xfs_defer_add(tp, &ri->ri_list, ri->ri_realtime ? + &xfs_rtrmap_update_defer_type : + &xfs_rmap_update_defer_type); +} + +/* Cancel a deferred rmap update. */ +STATIC void +xfs_rmap_update_cancel_item( + struct list_head *item) +{ + struct xfs_rmap_intent *ri = ri_entry(item); + + xfs_group_intent_put(ri->ri_group); + kmem_cache_free(xfs_rmap_intent_cache, ri); +} + +/* Process a deferred rmap update. */ +STATIC int +xfs_rmap_update_finish_item( + struct xfs_trans *tp, + struct xfs_log_item *done, + struct list_head *item, + struct xfs_btree_cur **state) +{ + struct xfs_rmap_intent *ri = ri_entry(item); + int error; + + error = xfs_rmap_finish_one(tp, ri, state); + + xfs_rmap_update_cancel_item(item); + return error; +} + +/* Clean up after calling xfs_rmap_finish_one. */ +STATIC void +xfs_rmap_finish_one_cleanup( + struct xfs_trans *tp, + struct xfs_btree_cur *rcur, + int error) +{ + struct xfs_buf *agbp = NULL; + + if (rcur == NULL) + return; + agbp = rcur->bc_ag.agbp; + xfs_btree_del_cursor(rcur, error); + if (error && agbp) + xfs_trans_brelse(tp, agbp); +} + +/* Abort all pending RUIs. */ +STATIC void +xfs_rmap_update_abort_intent( + struct xfs_log_item *intent) +{ + xfs_rui_release(RUI_ITEM(intent)); +} + +/* Is this recovered RUI ok? */ +static inline bool +xfs_rui_validate_map( + struct xfs_mount *mp, + bool isrt, + struct xfs_map_extent *map) +{ + if (!xfs_has_rmapbt(mp)) + return false; + + if (map->me_flags & ~XFS_RMAP_EXTENT_FLAGS) + return false; + + switch (map->me_flags & XFS_RMAP_EXTENT_TYPE_MASK) { + case XFS_RMAP_EXTENT_MAP: + case XFS_RMAP_EXTENT_MAP_SHARED: + case XFS_RMAP_EXTENT_UNMAP: + case XFS_RMAP_EXTENT_UNMAP_SHARED: + case XFS_RMAP_EXTENT_CONVERT: + case XFS_RMAP_EXTENT_CONVERT_SHARED: + case XFS_RMAP_EXTENT_ALLOC: + case XFS_RMAP_EXTENT_FREE: + break; + default: + return false; + } + + if (!XFS_RMAP_NON_INODE_OWNER(map->me_owner) && + !xfs_verify_ino(mp, map->me_owner)) + return false; + + if (!xfs_verify_fileext(mp, map->me_startoff, map->me_len)) + return false; + + if (isrt) + return xfs_verify_rtbext(mp, map->me_startblock, map->me_len); + + return xfs_verify_fsbext(mp, map->me_startblock, map->me_len); +} + +static inline void +xfs_rui_recover_work( + struct xfs_mount *mp, + struct xfs_defer_pending *dfp, + bool isrt, + const struct xfs_map_extent *map) +{ + struct xfs_rmap_intent *ri; + + ri = kmem_cache_alloc(xfs_rmap_intent_cache, GFP_KERNEL | __GFP_NOFAIL); + + switch (map->me_flags & XFS_RMAP_EXTENT_TYPE_MASK) { + case XFS_RMAP_EXTENT_MAP: + ri->ri_type = XFS_RMAP_MAP; + break; + case XFS_RMAP_EXTENT_MAP_SHARED: + ri->ri_type = XFS_RMAP_MAP_SHARED; + break; + case XFS_RMAP_EXTENT_UNMAP: + ri->ri_type = XFS_RMAP_UNMAP; + break; + case XFS_RMAP_EXTENT_UNMAP_SHARED: + ri->ri_type = XFS_RMAP_UNMAP_SHARED; + break; + case XFS_RMAP_EXTENT_CONVERT: + ri->ri_type = XFS_RMAP_CONVERT; + break; + case XFS_RMAP_EXTENT_CONVERT_SHARED: + ri->ri_type = XFS_RMAP_CONVERT_SHARED; + break; + case XFS_RMAP_EXTENT_ALLOC: + ri->ri_type = XFS_RMAP_ALLOC; + break; + case XFS_RMAP_EXTENT_FREE: + ri->ri_type = XFS_RMAP_FREE; + break; + default: + ASSERT(0); + return; + } + + ri->ri_owner = map->me_owner; + ri->ri_whichfork = (map->me_flags & XFS_RMAP_EXTENT_ATTR_FORK) ? + XFS_ATTR_FORK : XFS_DATA_FORK; + ri->ri_bmap.br_startblock = map->me_startblock; + ri->ri_bmap.br_startoff = map->me_startoff; + ri->ri_bmap.br_blockcount = map->me_len; + ri->ri_bmap.br_state = (map->me_flags & XFS_RMAP_EXTENT_UNWRITTEN) ? + XFS_EXT_UNWRITTEN : XFS_EXT_NORM; + ri->ri_group = xfs_group_intent_get(mp, map->me_startblock, + isrt ? XG_TYPE_RTG : XG_TYPE_AG); + ri->ri_realtime = isrt; + + xfs_defer_add_item(dfp, &ri->ri_list); +} + +/* + * Process an rmap update intent item that was recovered from the log. + * We need to update the rmapbt. + */ +STATIC int +xfs_rmap_recover_work( + struct xfs_defer_pending *dfp, + struct list_head *capture_list) +{ + struct xfs_trans_res resv; + struct xfs_log_item *lip = dfp->dfp_intent; + struct xfs_rui_log_item *ruip = RUI_ITEM(lip); + struct xfs_trans *tp; + struct xfs_mount *mp = lip->li_log->l_mp; + bool isrt = xfs_rui_item_isrt(lip); + int i; + int error = 0; + + /* + * First check the validity of the extents described by the + * RUI. If any are bad, then assume that all are bad and + * just toss the RUI. + */ + for (i = 0; i < ruip->rui_format.rui_nextents; i++) { + if (!xfs_rui_validate_map(mp, isrt, + &ruip->rui_format.rui_extents[i])) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + &ruip->rui_format, + sizeof(ruip->rui_format)); + return -EFSCORRUPTED; + } + + xfs_rui_recover_work(mp, dfp, isrt, + &ruip->rui_format.rui_extents[i]); + } + + resv = xlog_recover_resv(&M_RES(mp)->tr_itruncate); + error = xfs_trans_alloc(mp, &resv, mp->m_rmap_maxlevels, 0, + XFS_TRANS_RESERVE, &tp); + if (error) + return error; + + error = xlog_recover_finish_intent(tp, dfp); + if (error == -EFSCORRUPTED) + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + &ruip->rui_format, + sizeof(ruip->rui_format)); + if (error) + goto abort_error; + + return xfs_defer_ops_capture_and_commit(tp, capture_list); + +abort_error: + xfs_trans_cancel(tp); + return error; +} + +/* Relog an intent item to push the log tail forward. */ +static struct xfs_log_item * +xfs_rmap_relog_intent( + struct xfs_trans *tp, + struct xfs_log_item *intent, + struct xfs_log_item *done_item) +{ + struct xfs_rui_log_item *ruip; + struct xfs_map_extent *map; + unsigned int count; + + ASSERT(intent->li_type == XFS_LI_RUI || + intent->li_type == XFS_LI_RUI_RT); + + count = RUI_ITEM(intent)->rui_format.rui_nextents; + map = RUI_ITEM(intent)->rui_format.rui_extents; + + ruip = xfs_rui_init(tp->t_mountp, intent->li_type, count); + memcpy(ruip->rui_format.rui_extents, map, count * sizeof(*map)); + atomic_set(&ruip->rui_next_extent, count); + + return &ruip->rui_item; +} + +const struct xfs_defer_op_type xfs_rmap_update_defer_type = { + .name = "rmap", + .max_items = XFS_RUI_MAX_FAST_EXTENTS, + .create_intent = xfs_rmap_update_create_intent, + .abort_intent = xfs_rmap_update_abort_intent, + .create_done = xfs_rmap_update_create_done, + .finish_item = xfs_rmap_update_finish_item, + .finish_cleanup = xfs_rmap_finish_one_cleanup, + .cancel_item = xfs_rmap_update_cancel_item, + .recover_work = xfs_rmap_recover_work, + .relog_intent = xfs_rmap_relog_intent, +}; + +#ifdef CONFIG_XFS_RT +static struct xfs_log_item * +xfs_rtrmap_update_create_intent( + struct xfs_trans *tp, + struct list_head *items, + unsigned int count, + bool sort) +{ + return __xfs_rmap_update_create_intent(tp, items, count, sort, + XFS_LI_RUI_RT); +} + +/* Clean up after calling xfs_rmap_finish_one. */ +STATIC void +xfs_rtrmap_finish_one_cleanup( + struct xfs_trans *tp, + struct xfs_btree_cur *rcur, + int error) +{ + if (rcur) + xfs_btree_del_cursor(rcur, error); +} + +const struct xfs_defer_op_type xfs_rtrmap_update_defer_type = { + .name = "rtrmap", + .max_items = XFS_RUI_MAX_FAST_EXTENTS, + .create_intent = xfs_rtrmap_update_create_intent, + .abort_intent = xfs_rmap_update_abort_intent, + .create_done = xfs_rmap_update_create_done, + .finish_item = xfs_rmap_update_finish_item, + .finish_cleanup = xfs_rtrmap_finish_one_cleanup, + .cancel_item = xfs_rmap_update_cancel_item, + .recover_work = xfs_rmap_recover_work, + .relog_intent = xfs_rmap_relog_intent, +}; +#else +const struct xfs_defer_op_type xfs_rtrmap_update_defer_type = { + .name = "rtrmap", +}; +#endif + +STATIC bool +xfs_rui_item_match( + struct xfs_log_item *lip, + uint64_t intent_id) +{ + return RUI_ITEM(lip)->rui_format.rui_id == intent_id; +} + +static const struct xfs_item_ops xfs_rui_item_ops = { + .flags = XFS_ITEM_INTENT, + .iop_size = xfs_rui_item_size, + .iop_format = xfs_rui_item_format, + .iop_unpin = xfs_rui_item_unpin, + .iop_release = xfs_rui_item_release, + .iop_match = xfs_rui_item_match, +}; + +static inline void +xfs_rui_copy_format( + struct xfs_rui_log_format *dst, + const struct xfs_rui_log_format *src) +{ + unsigned int i; + + memcpy(dst, src, offsetof(struct xfs_rui_log_format, rui_extents)); + + for (i = 0; i < src->rui_nextents; i++) + memcpy(&dst->rui_extents[i], &src->rui_extents[i], + sizeof(struct xfs_map_extent)); +} + +/* + * This routine is called to create an in-core extent rmap update + * item from the rui format structure which was logged on disk. + * It allocates an in-core rui, copies the extents from the format + * structure into it, and adds the rui to the AIL with the given + * LSN. + */ +STATIC int +xlog_recover_rui_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_mount *mp = log->l_mp; + struct xfs_rui_log_item *ruip; + struct xfs_rui_log_format *rui_formatp; + size_t len; + + rui_formatp = item->ri_buf[0].iov_base; + + if (item->ri_buf[0].iov_len < xfs_rui_log_format_sizeof(0)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[0].iov_base, item->ri_buf[0].iov_len); + return -EFSCORRUPTED; + } + + len = xfs_rui_log_format_sizeof(rui_formatp->rui_nextents); + if (item->ri_buf[0].iov_len != len) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[0].iov_base, item->ri_buf[0].iov_len); + return -EFSCORRUPTED; + } + + ruip = xfs_rui_init(mp, ITEM_TYPE(item), rui_formatp->rui_nextents); + xfs_rui_copy_format(&ruip->rui_format, rui_formatp); + atomic_set(&ruip->rui_next_extent, rui_formatp->rui_nextents); + + xlog_recover_intent_item(log, &ruip->rui_item, lsn, + &xfs_rmap_update_defer_type); + return 0; +} + +const struct xlog_recover_item_ops xlog_rui_item_ops = { + .item_type = XFS_LI_RUI, + .commit_pass2 = xlog_recover_rui_commit_pass2, +}; + +#ifdef CONFIG_XFS_RT +STATIC int +xlog_recover_rtrui_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_mount *mp = log->l_mp; + struct xfs_rui_log_item *ruip; + struct xfs_rui_log_format *rui_formatp; + size_t len; + + rui_formatp = item->ri_buf[0].iov_base; + + if (item->ri_buf[0].iov_len < xfs_rui_log_format_sizeof(0)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[0].iov_base, item->ri_buf[0].iov_len); + return -EFSCORRUPTED; + } + + len = xfs_rui_log_format_sizeof(rui_formatp->rui_nextents); + if (item->ri_buf[0].iov_len != len) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, + item->ri_buf[0].iov_base, item->ri_buf[0].iov_len); + return -EFSCORRUPTED; + } + + ruip = xfs_rui_init(mp, ITEM_TYPE(item), rui_formatp->rui_nextents); + xfs_rui_copy_format(&ruip->rui_format, rui_formatp); + atomic_set(&ruip->rui_next_extent, rui_formatp->rui_nextents); + + xlog_recover_intent_item(log, &ruip->rui_item, lsn, + &xfs_rtrmap_update_defer_type); + return 0; +} +#else +STATIC int +xlog_recover_rtrui_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, log->l_mp, + item->ri_buf[0].iov_base, item->ri_buf[0].iov_len); + return -EFSCORRUPTED; +} +#endif + +const struct xlog_recover_item_ops xlog_rtrui_item_ops = { + .item_type = XFS_LI_RUI_RT, + .commit_pass2 = xlog_recover_rtrui_commit_pass2, +}; + +/* + * This routine is called when an RUD format structure is found in a committed + * transaction in the log. Its purpose is to cancel the corresponding RUI if it + * was still in the log. To do this it searches the AIL for the RUI with an id + * equal to that in the RUD format structure. If we find it we drop the RUD + * reference, which removes the RUI from the AIL and frees it. + */ +STATIC int +xlog_recover_rud_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_rud_log_format *rud_formatp; + + rud_formatp = item->ri_buf[0].iov_base; + if (item->ri_buf[0].iov_len != sizeof(struct xfs_rud_log_format)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, log->l_mp, + rud_formatp, item->ri_buf[0].iov_len); + return -EFSCORRUPTED; + } + + xlog_recover_release_intent(log, XFS_LI_RUI, rud_formatp->rud_rui_id); + return 0; +} + +const struct xlog_recover_item_ops xlog_rud_item_ops = { + .item_type = XFS_LI_RUD, + .commit_pass2 = xlog_recover_rud_commit_pass2, +}; + +#ifdef CONFIG_XFS_RT +STATIC int +xlog_recover_rtrud_commit_pass2( + struct xlog *log, + struct list_head *buffer_list, + struct xlog_recover_item *item, + xfs_lsn_t lsn) +{ + struct xfs_rud_log_format *rud_formatp; + + rud_formatp = item->ri_buf[0].iov_base; + if (item->ri_buf[0].iov_len != sizeof(struct xfs_rud_log_format)) { + XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, log->l_mp, + rud_formatp, item->ri_buf[0].iov_len); + return -EFSCORRUPTED; + } + + xlog_recover_release_intent(log, XFS_LI_RUI_RT, + rud_formatp->rud_rui_id); + return 0; +} +#else +# define xlog_recover_rtrud_commit_pass2 xlog_recover_rtrui_commit_pass2 +#endif + +const struct xlog_recover_item_ops xlog_rtrud_item_ops = { + .item_type = XFS_LI_RUD_RT, + .commit_pass2 = xlog_recover_rtrud_commit_pass2, +}; diff --git a/fs/xfs/xfs_rmap_item.h b/fs/xfs/xfs_rmap_item.h new file mode 100644 index 000000000..3a99f0117 --- /dev/null +++ b/fs/xfs/xfs_rmap_item.h @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright (C) 2016 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_RMAP_ITEM_H__ +#define __XFS_RMAP_ITEM_H__ + +/* + * There are (currently) three pairs of rmap btree redo item types: map, unmap, + * and convert. The common abbreviations for these are RUI (rmap update + * intent) and RUD (rmap update done). The redo item type is encoded in the + * flags field of each xfs_map_extent. + * + * *I items should be recorded in the *first* of a series of rolled + * transactions, and the *D items should be recorded in the same transaction + * that records the associated rmapbt updates. Typically, the first + * transaction will record a bmbt update, followed by some number of + * transactions containing rmapbt updates, and finally transactions with any + * bnobt/cntbt updates. + * + * Should the system crash after the commit of the first transaction but + * before the commit of the final transaction in a series, log recovery will + * use the redo information recorded by the intent items to replay the + * (rmapbt/bnobt/cntbt) metadata updates in the non-first transaction. + */ + +/* kernel only RUI/RUD definitions */ + +struct xfs_mount; +struct kmem_cache; + +/* + * Max number of extents in fast allocation path. + */ +#define XFS_RUI_MAX_FAST_EXTENTS 16 + +/* + * This is the "rmap update intent" log item. It is used to log the fact that + * some reverse mappings need to change. It is used in conjunction with the + * "rmap update done" log item described below. + * + * These log items follow the same rules as struct xfs_efi_log_item; see the + * comments about that structure (in xfs_extfree_item.h) for more details. + */ +struct xfs_rui_log_item { + struct xfs_log_item rui_item; + atomic_t rui_refcount; + atomic_t rui_next_extent; + struct xfs_rui_log_format rui_format; +}; + +static inline size_t +xfs_rui_log_item_sizeof( + unsigned int nr) +{ + return offsetof(struct xfs_rui_log_item, rui_format) + + xfs_rui_log_format_sizeof(nr); +} + +/* + * This is the "rmap update done" log item. It is used to log the fact that + * some rmapbt updates mentioned in an earlier rui item have been performed. + */ +struct xfs_rud_log_item { + struct xfs_log_item rud_item; + struct xfs_rui_log_item *rud_ruip; + struct xfs_rud_log_format rud_format; +}; + +extern struct kmem_cache *xfs_rui_cache; +extern struct kmem_cache *xfs_rud_cache; + +struct xfs_rmap_intent; + +void xfs_rmap_defer_add(struct xfs_trans *tp, struct xfs_rmap_intent *ri); + +unsigned int xfs_rui_log_space(unsigned int nr); +unsigned int xfs_rud_log_space(void); + +#endif /* __XFS_RMAP_ITEM_H__ */ diff --git a/fs/xfs/xfs_rtalloc.c b/fs/xfs/xfs_rtalloc.c new file mode 100644 index 000000000..84efe5a8f --- /dev/null +++ b/fs/xfs/xfs_rtalloc.c @@ -0,0 +1,2223 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_alloc.h" +#include "xfs_bmap.h" +#include "xfs_bmap_btree.h" +#include "xfs_bmap_util.h" +#include "xfs_trans.h" +#include "xfs_trans_space.h" +#include "xfs_icache.h" +#include "xfs_rtalloc.h" +#include "xfs_sb.h" +#include "xfs_rtbitmap.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_quota.h" +#include "xfs_log_priv.h" +#include "xfs_health.h" +#include "xfs_da_format.h" +#include "xfs_metafile.h" +#include "xfs_rtgroup.h" +#include "xfs_error.h" +#include "xfs_trace.h" +#include "xfs_rtrefcount_btree.h" +#include "xfs_reflink.h" +#include "xfs_zone_alloc.h" + +/* + * Return whether there are any free extents in the size range given + * by low and high, for the bitmap block bbno. + */ +STATIC int +xfs_rtany_summary( + struct xfs_rtalloc_args *args, + int low, /* low log2 extent size */ + int high, /* high log2 extent size */ + xfs_fileoff_t bbno, /* bitmap block number */ + int *maxlog) /* out: max log2 extent size free */ +{ + uint8_t *rsum_cache = args->rtg->rtg_rsum_cache; + int error; + int log; /* loop counter, log2 of ext. size */ + xfs_suminfo_t sum; /* summary data */ + + /* There are no extents at levels >= rsum_cache[bbno]. */ + if (rsum_cache) { + high = min(high, rsum_cache[bbno] - 1); + if (low > high) { + *maxlog = -1; + return 0; + } + } + + /* + * Loop over logs of extent sizes. + */ + for (log = high; log >= low; log--) { + /* + * Get one summary datum. + */ + error = xfs_rtget_summary(args, log, bbno, &sum); + if (error) { + return error; + } + /* + * If there are any, return success. + */ + if (sum) { + *maxlog = log; + goto out; + } + } + /* + * Found nothing, return failure. + */ + *maxlog = -1; +out: + /* There were no extents at levels > log. */ + if (rsum_cache && log + 1 < rsum_cache[bbno]) + rsum_cache[bbno] = log + 1; + return 0; +} + +/* + * Copy and transform the summary file, given the old and new + * parameters in the mount structures. + */ +STATIC int +xfs_rtcopy_summary( + struct xfs_rtalloc_args *oargs, + struct xfs_rtalloc_args *nargs) +{ + xfs_fileoff_t bbno; /* bitmap block number */ + int error; + int log; /* summary level number (log length) */ + xfs_suminfo_t sum; /* summary data */ + + for (log = oargs->mp->m_rsumlevels - 1; log >= 0; log--) { + for (bbno = oargs->mp->m_sb.sb_rbmblocks - 1; + (xfs_srtblock_t)bbno >= 0; + bbno--) { + error = xfs_rtget_summary(oargs, log, bbno, &sum); + if (error) + goto out; + if (XFS_IS_CORRUPT(oargs->mp, sum < 0)) { + error = -EFSCORRUPTED; + goto out; + } + if (sum == 0) + continue; + error = xfs_rtmodify_summary(oargs, log, bbno, -sum); + if (error) + goto out; + error = xfs_rtmodify_summary(nargs, log, bbno, sum); + if (error) + goto out; + } + } + error = 0; +out: + xfs_rtbuf_cache_relse(oargs); + return error; +} +/* + * Mark an extent specified by start and len allocated. + * Updates all the summary information as well as the bitmap. + */ +STATIC int +xfs_rtallocate_range( + struct xfs_rtalloc_args *args, + xfs_rtxnum_t start, /* start rtext to allocate */ + xfs_rtxlen_t len) /* in/out: summary block number */ +{ + struct xfs_mount *mp = args->mp; + xfs_rtxnum_t end; /* end of the allocated rtext */ + int error; + xfs_rtxnum_t postblock = 0; /* first rtext allocated > end */ + xfs_rtxnum_t preblock = 0; /* first rtext allocated < start */ + + end = start + len - 1; + /* + * Assume we're allocating out of the middle of a free extent. + * We need to find the beginning and end of the extent so we can + * properly update the summary. + */ + error = xfs_rtfind_back(args, start, &preblock); + if (error) + return error; + + /* + * Find the next allocated block (end of free extent). + */ + error = xfs_rtfind_forw(args, end, args->rtg->rtg_extents - 1, + &postblock); + if (error) + return error; + + /* + * Decrement the summary information corresponding to the entire + * (old) free extent. + */ + error = xfs_rtmodify_summary(args, + xfs_highbit64(postblock + 1 - preblock), + xfs_rtx_to_rbmblock(mp, preblock), -1); + if (error) + return error; + + /* + * If there are blocks not being allocated at the front of the + * old extent, add summary data for them to be free. + */ + if (preblock < start) { + error = xfs_rtmodify_summary(args, + xfs_highbit64(start - preblock), + xfs_rtx_to_rbmblock(mp, preblock), 1); + if (error) + return error; + } + + /* + * If there are blocks not being allocated at the end of the + * old extent, add summary data for them to be free. + */ + if (postblock > end) { + error = xfs_rtmodify_summary(args, + xfs_highbit64(postblock - end), + xfs_rtx_to_rbmblock(mp, end + 1), 1); + if (error) + return error; + } + + /* + * Modify the bitmap to mark this extent allocated. + */ + return xfs_rtmodify_range(args, start, len, 0); +} + +/* Reduce @rtxlen until it is a multiple of @prod. */ +static inline xfs_rtxlen_t +xfs_rtalloc_align_len( + xfs_rtxlen_t rtxlen, + xfs_rtxlen_t prod) +{ + if (unlikely(prod > 1)) + return rounddown(rtxlen, prod); + return rtxlen; +} + +/* + * Make sure we don't run off the end of the rt volume. Be careful that + * adjusting maxlen downwards doesn't cause us to fail the alignment checks. + */ +static inline xfs_rtxlen_t +xfs_rtallocate_clamp_len( + struct xfs_rtgroup *rtg, + xfs_rtxnum_t startrtx, + xfs_rtxlen_t rtxlen, + xfs_rtxlen_t prod) +{ + xfs_rtxlen_t ret; + + ret = min(rtg->rtg_extents, startrtx + rtxlen) - startrtx; + return xfs_rtalloc_align_len(ret, prod); +} + +/* + * Attempt to allocate an extent minlen<=len<=maxlen starting from + * bitmap block bbno. If we don't get maxlen then use prod to trim + * the length, if given. Returns error; returns starting block in *rtx. + * The lengths are all in rtextents. + */ +STATIC int +xfs_rtallocate_extent_block( + struct xfs_rtalloc_args *args, + xfs_fileoff_t bbno, /* bitmap block number */ + xfs_rtxlen_t minlen, /* minimum length to allocate */ + xfs_rtxlen_t maxlen, /* maximum length to allocate */ + xfs_rtxlen_t *len, /* out: actual length allocated */ + xfs_rtxnum_t *nextp, /* out: next rtext to try */ + xfs_rtxlen_t prod, /* extent product factor */ + xfs_rtxnum_t *rtx) /* out: start rtext allocated */ +{ + struct xfs_mount *mp = args->mp; + xfs_rtxnum_t besti = -1; /* best rtext found so far */ + xfs_rtxnum_t end; /* last rtext in chunk */ + xfs_rtxnum_t i; /* current rtext trying */ + xfs_rtxnum_t next; /* next rtext to try */ + xfs_rtxlen_t scanlen; /* number of free rtx to look for */ + xfs_rtxlen_t bestlen = 0; /* best length found so far */ + int stat; /* status from internal calls */ + int error; + + /* + * Loop over all the extents starting in this bitmap block up to the + * end of the rt volume, looking for one that's long enough. + */ + end = min(args->rtg->rtg_extents, xfs_rbmblock_to_rtx(mp, bbno + 1)) - + 1; + for (i = xfs_rbmblock_to_rtx(mp, bbno); i <= end; i++) { + /* Make sure we don't scan off the end of the rt volume. */ + scanlen = xfs_rtallocate_clamp_len(args->rtg, i, maxlen, prod); + if (scanlen < minlen) + break; + + /* + * See if there's a free extent of scanlen starting at i. + * If it's not so then next will contain the first non-free. + */ + error = xfs_rtcheck_range(args, i, scanlen, 1, &next, &stat); + if (error) + return error; + if (stat) { + /* + * i to scanlen is all free, allocate and return that. + */ + *len = scanlen; + *rtx = i; + return 0; + } + + /* + * In the case where we have a variable-sized allocation + * request, figure out how big this free piece is, + * and if it's big enough for the minimum, and the best + * so far, remember it. + */ + if (minlen < maxlen) { + xfs_rtxnum_t thislen; /* this extent size */ + + thislen = next - i; + if (thislen >= minlen && thislen > bestlen) { + besti = i; + bestlen = thislen; + } + } + /* + * If not done yet, find the start of the next free space. + */ + if (next >= end) + break; + error = xfs_rtfind_forw(args, next, end, &i); + if (error) + return error; + } + + /* Searched the whole thing & didn't find a maxlen free extent. */ + if (besti == -1) + goto nospace; + + /* + * Ensure bestlen is a multiple of prod, but don't return a too-short + * extent. + */ + bestlen = xfs_rtalloc_align_len(bestlen, prod); + if (bestlen < minlen) + goto nospace; + + /* + * Pick besti for bestlen & return that. + */ + *len = bestlen; + *rtx = besti; + return 0; +nospace: + /* Allocation failed. Set *nextp to the next block to try. */ + *nextp = next; + return -ENOSPC; +} + +/* + * Allocate an extent of length minlen<=len<=maxlen, starting at block + * bno. If we don't get maxlen then use prod to trim the length, if given. + * Returns error; returns starting block in *rtx. + * The lengths are all in rtextents. + */ +STATIC int +xfs_rtallocate_extent_exact( + struct xfs_rtalloc_args *args, + xfs_rtxnum_t start, /* starting rtext number to allocate */ + xfs_rtxlen_t minlen, /* minimum length to allocate */ + xfs_rtxlen_t maxlen, /* maximum length to allocate */ + xfs_rtxlen_t *len, /* out: actual length allocated */ + xfs_rtxlen_t prod, /* extent product factor */ + xfs_rtxnum_t *rtx) /* out: start rtext allocated */ +{ + xfs_rtxnum_t next; /* next rtext to try (dummy) */ + xfs_rtxlen_t alloclen; /* candidate length */ + xfs_rtxlen_t scanlen; /* number of free rtx to look for */ + int isfree; /* extent is free */ + int error; + + ASSERT(minlen % prod == 0); + ASSERT(maxlen % prod == 0); + + /* Make sure we don't run off the end of the rt volume. */ + scanlen = xfs_rtallocate_clamp_len(args->rtg, start, maxlen, prod); + if (scanlen < minlen) + return -ENOSPC; + + /* Check if the range in question (for scanlen) is free. */ + error = xfs_rtcheck_range(args, start, scanlen, 1, &next, &isfree); + if (error) + return error; + + if (isfree) { + /* start to scanlen is all free; allocate it. */ + *len = scanlen; + *rtx = start; + return 0; + } + + /* + * If not, allocate what there is, if it's at least minlen. + */ + alloclen = next - start; + if (alloclen < minlen) + return -ENOSPC; + + /* Ensure alloclen is a multiple of prod. */ + alloclen = xfs_rtalloc_align_len(alloclen, prod); + if (alloclen < minlen) + return -ENOSPC; + + *len = alloclen; + *rtx = start; + return 0; +} + +/* + * Allocate an extent of length minlen<=len<=maxlen, starting as near + * to start as possible. If we don't get maxlen then use prod to trim + * the length, if given. The lengths are all in rtextents. + */ +STATIC int +xfs_rtallocate_extent_near( + struct xfs_rtalloc_args *args, + xfs_rtxnum_t start, /* starting rtext number to allocate */ + xfs_rtxlen_t minlen, /* minimum length to allocate */ + xfs_rtxlen_t maxlen, /* maximum length to allocate */ + xfs_rtxlen_t *len, /* out: actual length allocated */ + xfs_rtxlen_t prod, /* extent product factor */ + xfs_rtxnum_t *rtx) /* out: start rtext allocated */ +{ + struct xfs_mount *mp = args->mp; + int maxlog; /* max useful extent from summary */ + xfs_fileoff_t bbno; /* bitmap block number */ + int error; + int i; /* bitmap block offset (loop control) */ + int j; /* secondary loop control */ + int log2len; /* log2 of minlen */ + xfs_rtxnum_t n; /* next rtext to try */ + + ASSERT(minlen % prod == 0); + ASSERT(maxlen % prod == 0); + + /* + * If the block number given is off the end, silently set it to the last + * block. + */ + start = min(start, args->rtg->rtg_extents - 1); + + /* + * Try the exact allocation first. + */ + error = xfs_rtallocate_extent_exact(args, start, minlen, maxlen, len, + prod, rtx); + if (error != -ENOSPC) + return error; + + bbno = xfs_rtx_to_rbmblock(mp, start); + i = 0; + j = -1; + ASSERT(minlen != 0); + log2len = xfs_highbit32(minlen); + /* + * Loop over all bitmap blocks (bbno + i is current block). + */ + for (;;) { + /* + * Get summary information of extents of all useful levels + * starting in this bitmap block. + */ + error = xfs_rtany_summary(args, log2len, mp->m_rsumlevels - 1, + bbno + i, &maxlog); + if (error) + return error; + + /* + * If there are any useful extents starting here, try + * allocating one. + */ + if (maxlog >= 0) { + xfs_extlen_t maxavail = + min_t(xfs_rtblock_t, maxlen, + (1ULL << (maxlog + 1)) - 1); + /* + * On the positive side of the starting location. + */ + if (i >= 0) { + /* + * Try to allocate an extent starting in + * this block. + */ + error = xfs_rtallocate_extent_block(args, + bbno + i, minlen, maxavail, len, + &n, prod, rtx); + if (error != -ENOSPC) + return error; + } + /* + * On the negative side of the starting location. + */ + else { /* i < 0 */ + int maxblocks; + + /* + * Loop backwards to find the end of the extent + * we found in the realtime summary. + * + * maxblocks is the maximum possible number of + * bitmap blocks from the start of the extent + * to the end of the extent. + */ + if (maxlog == 0) + maxblocks = 0; + else if (maxlog < mp->m_blkbit_log) + maxblocks = 1; + else + maxblocks = 2 << (maxlog - mp->m_blkbit_log); + + /* + * We need to check bbno + i + maxblocks down to + * bbno + i. We already checked bbno down to + * bbno + j + 1, so we don't need to check those + * again. + */ + j = min(i + maxblocks, j); + for (; j >= i; j--) { + error = xfs_rtallocate_extent_block(args, + bbno + j, minlen, + maxavail, len, &n, prod, + rtx); + if (error != -ENOSPC) + return error; + } + } + } + /* + * Loop control. If we were on the positive side, and there's + * still more blocks on the negative side, go there. + */ + if (i > 0 && (int)bbno - i >= 0) + i = -i; + /* + * If positive, and no more negative, but there are more + * positive, go there. + */ + else if (i > 0 && (int)bbno + i < mp->m_sb.sb_rbmblocks - 1) + i++; + /* + * If negative or 0 (just started), and there are positive + * blocks to go, go there. The 0 case moves to block 1. + */ + else if (i <= 0 && (int)bbno - i < mp->m_sb.sb_rbmblocks - 1) + i = 1 - i; + /* + * If negative or 0 and there are more negative blocks, + * go there. + */ + else if (i <= 0 && (int)bbno + i > 0) + i--; + /* + * Must be done. Return failure. + */ + else + break; + } + return -ENOSPC; +} + +static int +xfs_rtalloc_sumlevel( + struct xfs_rtalloc_args *args, + int l, /* level number */ + xfs_rtxlen_t minlen, /* minimum length to allocate */ + xfs_rtxlen_t maxlen, /* maximum length to allocate */ + xfs_rtxlen_t prod, /* extent product factor */ + xfs_rtxlen_t *len, /* out: actual length allocated */ + xfs_rtxnum_t *rtx) /* out: start rtext allocated */ +{ + xfs_fileoff_t i; /* bitmap block number */ + int error; + + for (i = 0; i < args->mp->m_sb.sb_rbmblocks; i++) { + xfs_suminfo_t sum; /* summary information for extents */ + xfs_rtxnum_t n; /* next rtext to be tried */ + + error = xfs_rtget_summary(args, l, i, &sum); + if (error) + return error; + + /* + * Nothing there, on to the next block. + */ + if (!sum) + continue; + + /* + * Try allocating the extent. + */ + error = xfs_rtallocate_extent_block(args, i, minlen, maxlen, + len, &n, prod, rtx); + if (error != -ENOSPC) + return error; + + /* + * If the "next block to try" returned from the allocator is + * beyond the next bitmap block, skip to that bitmap block. + */ + if (xfs_rtx_to_rbmblock(args->mp, n) > i + 1) + i = xfs_rtx_to_rbmblock(args->mp, n) - 1; + } + + return -ENOSPC; +} + +/* + * Allocate an extent of length minlen<=len<=maxlen, with no position + * specified. If we don't get maxlen then use prod to trim + * the length, if given. The lengths are all in rtextents. + */ +static int +xfs_rtallocate_extent_size( + struct xfs_rtalloc_args *args, + xfs_rtxlen_t minlen, /* minimum length to allocate */ + xfs_rtxlen_t maxlen, /* maximum length to allocate */ + xfs_rtxlen_t *len, /* out: actual length allocated */ + xfs_rtxlen_t prod, /* extent product factor */ + xfs_rtxnum_t *rtx) /* out: start rtext allocated */ +{ + int error; + int l; /* level number (loop control) */ + + ASSERT(minlen % prod == 0); + ASSERT(maxlen % prod == 0); + ASSERT(maxlen != 0); + + /* + * Loop over all the levels starting with maxlen. + * + * At each level, look at all the bitmap blocks, to see if there are + * extents starting there that are long enough (>= maxlen). + * + * Note, only on the initial level can the allocation fail if the + * summary says there's an extent. + */ + for (l = xfs_highbit32(maxlen); l < args->mp->m_rsumlevels; l++) { + error = xfs_rtalloc_sumlevel(args, l, minlen, maxlen, prod, len, + rtx); + if (error != -ENOSPC) + return error; + } + + /* + * Didn't find any maxlen blocks. Try smaller ones, unless we are + * looking for a fixed size extent. + */ + if (minlen > --maxlen) + return -ENOSPC; + ASSERT(minlen != 0); + ASSERT(maxlen != 0); + + /* + * Loop over sizes, from maxlen down to minlen. + * + * This time, when we do the allocations, allow smaller ones to succeed, + * but make sure the specified minlen/maxlen are in the possible range + * for this summary level. + */ + for (l = xfs_highbit32(maxlen); l >= xfs_highbit32(minlen); l--) { + error = xfs_rtalloc_sumlevel(args, l, + max_t(xfs_rtxlen_t, minlen, 1 << l), + min_t(xfs_rtxlen_t, maxlen, (1 << (l + 1)) - 1), + prod, len, rtx); + if (error != -ENOSPC) + return error; + } + + return -ENOSPC; +} + +static void +xfs_rtunmount_rtg( + struct xfs_rtgroup *rtg) +{ + int i; + + for (i = 0; i < XFS_RTGI_MAX; i++) + xfs_rtginode_irele(&rtg->rtg_inodes[i]); + if (!xfs_has_zoned(rtg_mount(rtg))) + kvfree(rtg->rtg_rsum_cache); +} + +static int +xfs_alloc_rsum_cache( + struct xfs_rtgroup *rtg, + xfs_extlen_t rbmblocks) +{ + /* + * The rsum cache is initialized to the maximum value, which is + * trivially an upper bound on the maximum level with any free extents. + */ + rtg->rtg_rsum_cache = kvmalloc(rbmblocks, GFP_KERNEL); + if (!rtg->rtg_rsum_cache) + return -ENOMEM; + memset(rtg->rtg_rsum_cache, -1, rbmblocks); + return 0; +} + +/* + * If we changed the rt extent size (meaning there was no rt volume previously) + * and the root directory had EXTSZINHERIT and RTINHERIT set, it's possible + * that the extent size hint on the root directory is no longer congruent with + * the new rt extent size. Log the rootdir inode to fix this. + */ +static int +xfs_growfs_rt_fixup_extsize( + struct xfs_mount *mp) +{ + struct xfs_inode *ip = mp->m_rootip; + struct xfs_trans *tp; + int error = 0; + + xfs_ilock(ip, XFS_IOLOCK_EXCL); + if (!(ip->i_diflags & XFS_DIFLAG_RTINHERIT) || + !(ip->i_diflags & XFS_DIFLAG_EXTSZINHERIT)) + goto out_iolock; + + error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_ichange, 0, 0, false, + &tp); + if (error) + goto out_iolock; + + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + error = xfs_trans_commit(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + +out_iolock: + xfs_iunlock(ip, XFS_IOLOCK_EXCL); + return error; +} + +/* Ensure that the rtgroup metadata inode is loaded, creating it if neeeded. */ +static int +xfs_rtginode_ensure( + struct xfs_rtgroup *rtg, + enum xfs_rtg_inodes type) +{ + struct xfs_trans *tp; + int error; + + if (rtg->rtg_inodes[type]) + return 0; + + tp = xfs_trans_alloc_empty(rtg_mount(rtg)); + error = xfs_rtginode_load(rtg, type, tp); + xfs_trans_cancel(tp); + + if (error != -ENOENT) + return error; + return xfs_rtginode_create(rtg, type, true); +} + +static struct xfs_mount * +xfs_growfs_rt_alloc_fake_mount( + const struct xfs_mount *mp, + xfs_rfsblock_t rblocks, + xfs_agblock_t rextsize) +{ + struct xfs_mount *nmp; + + nmp = kmemdup(mp, sizeof(*mp), GFP_KERNEL); + if (!nmp) + return NULL; + xfs_mount_sb_set_rextsize(nmp, &nmp->m_sb, rextsize); + nmp->m_sb.sb_rblocks = rblocks; + nmp->m_sb.sb_rextents = xfs_blen_to_rtbxlen(nmp, nmp->m_sb.sb_rblocks); + nmp->m_sb.sb_rbmblocks = xfs_rtbitmap_blockcount(nmp); + nmp->m_sb.sb_rextslog = xfs_compute_rextslog(nmp->m_sb.sb_rextents); + if (xfs_has_rtgroups(nmp)) + nmp->m_sb.sb_rgcount = howmany_64(nmp->m_sb.sb_rextents, + nmp->m_sb.sb_rgextents); + else + nmp->m_sb.sb_rgcount = 1; + nmp->m_rsumblocks = xfs_rtsummary_blockcount(nmp, &nmp->m_rsumlevels); + + if (rblocks > 0) + nmp->m_features |= XFS_FEAT_REALTIME; + + /* recompute growfsrt reservation from new rsumsize */ + xfs_trans_resv_calc(nmp, &nmp->m_resv); + return nmp; +} + +/* Free all the new space and return the number of extents actually freed. */ +static int +xfs_growfs_rt_free_new( + struct xfs_rtgroup *rtg, + struct xfs_rtalloc_args *nargs, + xfs_rtbxlen_t *freed_rtx) +{ + struct xfs_mount *mp = rtg_mount(rtg); + xfs_rgnumber_t rgno = rtg_rgno(rtg); + xfs_rtxnum_t start_rtx = 0, end_rtx; + + if (rgno < mp->m_sb.sb_rgcount) + start_rtx = xfs_rtgroup_extents(mp, rgno); + end_rtx = xfs_rtgroup_extents(nargs->mp, rgno); + + /* + * Compute the first new extent that we want to free, being careful to + * skip past a realtime superblock at the start of the realtime volume. + */ + if (xfs_has_rtsb(nargs->mp) && rgno == 0 && start_rtx == 0) + start_rtx++; + *freed_rtx = end_rtx - start_rtx; + return xfs_rtfree_range(nargs, start_rtx, *freed_rtx); +} + +static xfs_rfsblock_t +xfs_growfs_rt_nrblocks( + struct xfs_rtgroup *rtg, + xfs_rfsblock_t nrblocks, + xfs_agblock_t rextsize, + xfs_fileoff_t bmbno) +{ + struct xfs_mount *mp = rtg_mount(rtg); + xfs_rfsblock_t step; + + step = (bmbno + 1) * mp->m_rtx_per_rbmblock * rextsize; + if (xfs_has_rtgroups(mp)) { + xfs_rfsblock_t rgblocks = mp->m_sb.sb_rgextents * rextsize; + + step = min(rgblocks, step) + rgblocks * rtg_rgno(rtg); + } + + return min(nrblocks, step); +} + +/* + * If the post-grow filesystem will have an rtsb; we're initializing the first + * rtgroup; and the filesystem didn't have a realtime section, write the rtsb + * now, and attach the rtsb buffer to the real mount. + */ +static int +xfs_growfs_rt_init_rtsb( + const struct xfs_rtalloc_args *nargs, + const struct xfs_rtgroup *rtg, + const struct xfs_rtalloc_args *args) +{ + struct xfs_mount *mp = args->mp; + struct xfs_buf *rtsb_bp; + int error; + + if (!xfs_has_rtsb(nargs->mp)) + return 0; + if (rtg_rgno(rtg) > 0) + return 0; + if (mp->m_sb.sb_rblocks) + return 0; + + error = xfs_buf_get_uncached(mp->m_rtdev_targp, XFS_FSB_TO_BB(mp, 1), + &rtsb_bp); + if (error) + return error; + + rtsb_bp->b_maps[0].bm_bn = XFS_RTSB_DADDR; + rtsb_bp->b_ops = &xfs_rtsb_buf_ops; + + xfs_update_rtsb(rtsb_bp, mp->m_sb_bp); + mp->m_rtsb_bp = rtsb_bp; + error = xfs_bwrite(rtsb_bp); + xfs_buf_unlock(rtsb_bp); + if (error) + return error; + + /* Initialize the rtrmap to reflect the rtsb. */ + if (rtg_rmap(args->rtg) != NULL) + error = xfs_rtrmapbt_init_rtsb(nargs->mp, args->rtg, args->tp); + + return error; +} + +static void +xfs_growfs_rt_sb_fields( + struct xfs_trans *tp, + const struct xfs_mount *nmp) +{ + struct xfs_mount *mp = tp->t_mountp; + + if (nmp->m_sb.sb_rextsize != mp->m_sb.sb_rextsize) + xfs_trans_mod_sb(tp, XFS_TRANS_SB_REXTSIZE, + nmp->m_sb.sb_rextsize - mp->m_sb.sb_rextsize); + if (nmp->m_sb.sb_rbmblocks != mp->m_sb.sb_rbmblocks) + xfs_trans_mod_sb(tp, XFS_TRANS_SB_RBMBLOCKS, + nmp->m_sb.sb_rbmblocks - mp->m_sb.sb_rbmblocks); + if (nmp->m_sb.sb_rblocks != mp->m_sb.sb_rblocks) + xfs_trans_mod_sb(tp, XFS_TRANS_SB_RBLOCKS, + nmp->m_sb.sb_rblocks - mp->m_sb.sb_rblocks); + if (nmp->m_sb.sb_rextents != mp->m_sb.sb_rextents) + xfs_trans_mod_sb(tp, XFS_TRANS_SB_REXTENTS, + nmp->m_sb.sb_rextents - mp->m_sb.sb_rextents); + if (nmp->m_sb.sb_rextslog != mp->m_sb.sb_rextslog) + xfs_trans_mod_sb(tp, XFS_TRANS_SB_REXTSLOG, + nmp->m_sb.sb_rextslog - mp->m_sb.sb_rextslog); + if (nmp->m_sb.sb_rgcount != mp->m_sb.sb_rgcount) + xfs_trans_mod_sb(tp, XFS_TRANS_SB_RGCOUNT, + nmp->m_sb.sb_rgcount - mp->m_sb.sb_rgcount); +} + +static int +xfs_growfs_rt_zoned( + struct xfs_rtgroup *rtg) +{ + struct xfs_mount *mp = rtg_mount(rtg); + struct xfs_mount *nmp; + struct xfs_trans *tp; + xfs_rtbxlen_t freed_rtx; + int error; + + /* + * Calculate new sb and mount fields for this round. Also ensure the + * rtg_extents value is uptodate as the rtbitmap code relies on it. + */ + nmp = xfs_growfs_rt_alloc_fake_mount(mp, + xfs_rtgs_to_rfsbs(mp, rtg_rgno(rtg) + 1), + mp->m_sb.sb_rextsize); + if (!nmp) + return -ENOMEM; + freed_rtx = nmp->m_sb.sb_rextents - mp->m_sb.sb_rextents; + + xfs_rtgroup_calc_geometry(nmp, rtg, rtg_rgno(rtg), + nmp->m_sb.sb_rgcount, nmp->m_sb.sb_rextents); + + error = xfs_trans_alloc(mp, &M_RES(nmp)->tr_growrtfree, 0, 0, 0, &tp); + if (error) + goto out_free; + + xfs_rtgroup_lock(rtg, XFS_RTGLOCK_RMAP); + xfs_rtgroup_trans_join(tp, rtg, XFS_RTGLOCK_RMAP); + + xfs_growfs_rt_sb_fields(tp, nmp); + xfs_trans_mod_sb(tp, XFS_TRANS_SB_FREXTENTS, freed_rtx); + + error = xfs_trans_commit(tp); + if (error) + goto out_free; + + /* + * Ensure the mount RT feature flag is now set, and compute new + * maxlevels for rt btrees. + */ + mp->m_features |= XFS_FEAT_REALTIME; + xfs_rtrmapbt_compute_maxlevels(mp); + xfs_rtrefcountbt_compute_maxlevels(mp); + + /* + * Finally add the newly added zone to the freelist and add the space + * to the available counter. The order is important here: only add + * the available space after the zones, as available space guarantees + * that zones to back it are available. + */ + xfs_zone_mark_free(rtg); + xfs_zoned_add_available(mp, freed_rtx); +out_free: + kfree(nmp); + return error; +} + +static int +xfs_growfs_rt_bmblock( + struct xfs_rtgroup *rtg, + xfs_rfsblock_t nrblocks, + xfs_agblock_t rextsize, + xfs_fileoff_t bmbno) +{ + struct xfs_mount *mp = rtg_mount(rtg); + struct xfs_inode *rbmip = rtg_bitmap(rtg); + struct xfs_inode *rsumip = rtg_summary(rtg); + struct xfs_rtalloc_args args = { + .mp = mp, + .rtg = rtg, + }; + struct xfs_rtalloc_args nargs = { + .rtg = rtg, + }; + struct xfs_mount *nmp; + xfs_rtbxlen_t freed_rtx; + int error; + + /* + * Calculate new sb and mount fields for this round. Also ensure the + * rtg_extents value is uptodate as the rtbitmap code relies on it. + */ + nmp = nargs.mp = xfs_growfs_rt_alloc_fake_mount(mp, + xfs_growfs_rt_nrblocks(rtg, nrblocks, rextsize, bmbno), + rextsize); + if (!nmp) + return -ENOMEM; + + xfs_rtgroup_calc_geometry(nmp, rtg, rtg_rgno(rtg), + nmp->m_sb.sb_rgcount, nmp->m_sb.sb_rextents); + + /* + * Recompute the growfsrt reservation from the new rsumsize, so that the + * transaction below use the new, potentially larger value. + * */ + xfs_trans_resv_calc(nmp, &nmp->m_resv); + error = xfs_trans_alloc(mp, &M_RES(nmp)->tr_growrtfree, 0, 0, 0, + &args.tp); + if (error) + goto out_free; + nargs.tp = args.tp; + + xfs_rtgroup_lock(args.rtg, XFS_RTGLOCK_BITMAP | XFS_RTGLOCK_RMAP); + xfs_rtgroup_trans_join(args.tp, args.rtg, + XFS_RTGLOCK_BITMAP | XFS_RTGLOCK_RMAP); + + /* + * Update the bitmap inode's size ondisk and incore. We need to update + * the incore size so that inode inactivation won't punch what it thinks + * are "posteof" blocks. + */ + rbmip->i_disk_size = nmp->m_sb.sb_rbmblocks * nmp->m_sb.sb_blocksize; + i_size_write(VFS_I(rbmip), rbmip->i_disk_size); + xfs_trans_log_inode(args.tp, rbmip, XFS_ILOG_CORE); + + /* + * Update the summary inode's size. We need to update the incore size + * so that inode inactivation won't punch what it thinks are "posteof" + * blocks. + */ + rsumip->i_disk_size = nmp->m_rsumblocks * nmp->m_sb.sb_blocksize; + i_size_write(VFS_I(rsumip), rsumip->i_disk_size); + xfs_trans_log_inode(args.tp, rsumip, XFS_ILOG_CORE); + + /* + * Copy summary data from old to new sizes when the real size (not + * block-aligned) changes. + */ + if (mp->m_sb.sb_rbmblocks != nmp->m_sb.sb_rbmblocks || + mp->m_rsumlevels != nmp->m_rsumlevels) { + error = xfs_rtcopy_summary(&args, &nargs); + if (error) + goto out_cancel; + } + + error = xfs_growfs_rt_init_rtsb(&nargs, rtg, &args); + if (error) + goto out_cancel; + + /* + * Update superblock fields. + */ + xfs_growfs_rt_sb_fields(args.tp, nmp); + + /* + * Free the new extent. + */ + error = xfs_growfs_rt_free_new(rtg, &nargs, &freed_rtx); + xfs_rtbuf_cache_relse(&nargs); + if (error) + goto out_cancel; + + /* + * Mark more blocks free in the superblock. + */ + xfs_trans_mod_sb(args.tp, XFS_TRANS_SB_FREXTENTS, freed_rtx); + + /* + * Update the calculated values in the real mount structure. + */ + mp->m_rsumlevels = nmp->m_rsumlevels; + mp->m_rsumblocks = nmp->m_rsumblocks; + + /* + * Recompute the growfsrt reservation from the new rsumsize. + */ + xfs_trans_resv_calc(mp, &mp->m_resv); + + /* + * Sync sb counters now to reflect the updated values. Lazy counters are + * not always updated and in order to avoid inconsistencies between + * frextents and rtextents, it is better to sync the counters. + */ + + if (xfs_has_lazysbcount(mp)) + xfs_log_sb(args.tp); + + error = xfs_trans_commit(args.tp); + if (error) + goto out_free; + + /* + * Ensure the mount RT feature flag is now set, and compute new + * maxlevels for rt btrees. + */ + mp->m_features |= XFS_FEAT_REALTIME; + xfs_rtrmapbt_compute_maxlevels(mp); + xfs_rtrefcountbt_compute_maxlevels(mp); + + kfree(nmp); + return 0; + +out_cancel: + xfs_trans_cancel(args.tp); +out_free: + kfree(nmp); + return error; +} + +static xfs_rtxnum_t +xfs_last_rtgroup_extents( + struct xfs_mount *mp) +{ + return mp->m_sb.sb_rextents - + ((xfs_rtxnum_t)(mp->m_sb.sb_rgcount - 1) * + mp->m_sb.sb_rgextents); +} + +/* + * This will return the bitmap block number (indexed at 0) that will be + * extended/modified. There are 2 cases here: + * 1. The size of the rtg is such that it is a multiple of + * xfs_rtbitmap_rtx_per_rbmblock() i.e, an integral number of bitmap blocks + * are completely filled up. In this case, we should return + * 1 + (the last used bitmap block number). + * 2. The size of the rtg is not an multiple of xfs_rtbitmap_rtx_per_rbmblock(). + * Here we will return the block number of last used block number. In this + * case, we will modify the last used bitmap block to extend the size of the + * rtgroup. + * + * This also deals with the case where there were no rtextents before. + */ +static xfs_fileoff_t +xfs_last_rt_bmblock_to_extend( + struct xfs_rtgroup *rtg) +{ + struct xfs_mount *mp = rtg_mount(rtg); + xfs_rgnumber_t rgno = rtg_rgno(rtg); + xfs_fileoff_t bmbno = 0; + unsigned int mod = 0; + + ASSERT(!mp->m_sb.sb_rgcount || rgno >= mp->m_sb.sb_rgcount - 1); + + if (mp->m_sb.sb_rgcount && rgno == mp->m_sb.sb_rgcount - 1) { + xfs_rtxnum_t nrext = xfs_last_rtgroup_extents(mp); + + /* Also fill up the previous block if not entirely full. */ + /* We are doing a -1 to convert it to a 0 based index */ + bmbno = xfs_rtbitmap_blockcount_len(mp, nrext) - 1; + div_u64_rem(nrext, xfs_rtbitmap_rtx_per_rbmblock(mp), &mod); + /* + * mod = 0 means that all the current blocks are full. So + * return the next block number to be used for the rtgroup + * growth. + */ + if (mod == 0) + bmbno++; + } + + return bmbno; +} + +/* + * Allocate space to the bitmap and summary files, as necessary. + */ +static int +xfs_growfs_rt_alloc_blocks( + struct xfs_rtgroup *rtg, + xfs_rfsblock_t nrblocks, + xfs_agblock_t rextsize, + xfs_extlen_t *nrbmblocks) +{ + struct xfs_mount *mp = rtg_mount(rtg); + struct xfs_inode *rbmip = rtg_bitmap(rtg); + struct xfs_inode *rsumip = rtg_summary(rtg); + xfs_extlen_t orbmblocks = 0; + xfs_extlen_t orsumblocks = 0; + struct xfs_mount *nmp; + int error = 0; + + nmp = xfs_growfs_rt_alloc_fake_mount(mp, nrblocks, rextsize); + if (!nmp) + return -ENOMEM; + *nrbmblocks = nmp->m_sb.sb_rbmblocks; + + if (xfs_has_rtgroups(mp)) { + /* + * For file systems with the rtgroups feature, the RT bitmap and + * summary are always fully allocated, which means that we never + * need to grow the existing files. + * + * But we have to be careful to only fill the bitmap until the + * end of the actually used range. + */ + if (rtg_rgno(rtg) == nmp->m_sb.sb_rgcount - 1) + *nrbmblocks = xfs_rtbitmap_blockcount_len(nmp, + xfs_last_rtgroup_extents(nmp)); + + if (mp->m_sb.sb_rgcount && + rtg_rgno(rtg) == mp->m_sb.sb_rgcount - 1) + goto out_free; + } else { + /* + * Get the old block counts for bitmap and summary inodes. + * These can't change since other growfs callers are locked out. + */ + orbmblocks = XFS_B_TO_FSB(mp, rbmip->i_disk_size); + orsumblocks = XFS_B_TO_FSB(mp, rsumip->i_disk_size); + } + + error = xfs_rtfile_initialize_blocks(rtg, XFS_RTGI_BITMAP, orbmblocks, + nmp->m_sb.sb_rbmblocks, NULL); + if (error) + goto out_free; + error = xfs_rtfile_initialize_blocks(rtg, XFS_RTGI_SUMMARY, orsumblocks, + nmp->m_rsumblocks, NULL); +out_free: + kfree(nmp); + return error; +} + +static int +xfs_growfs_rtg( + struct xfs_mount *mp, + xfs_rgnumber_t rgno, + xfs_rfsblock_t nrblocks, + xfs_agblock_t rextsize) +{ + uint8_t *old_rsum_cache = NULL; + xfs_extlen_t bmblocks; + xfs_fileoff_t bmbno; + struct xfs_rtgroup *rtg; + unsigned int i; + int error; + + rtg = xfs_rtgroup_grab(mp, rgno); + if (!rtg) + return -EINVAL; + + for (i = 0; i < XFS_RTGI_MAX; i++) { + error = xfs_rtginode_ensure(rtg, i); + if (error) + goto out_rele; + } + + if (xfs_has_zoned(mp)) { + error = xfs_growfs_rt_zoned(rtg); + goto out_rele; + } + + error = xfs_growfs_rt_alloc_blocks(rtg, nrblocks, rextsize, &bmblocks); + if (error) + goto out_rele; + + if (bmblocks != rtg_mount(rtg)->m_sb.sb_rbmblocks) { + old_rsum_cache = rtg->rtg_rsum_cache; + error = xfs_alloc_rsum_cache(rtg, bmblocks); + if (error) + goto out_rele; + } + + for (bmbno = xfs_last_rt_bmblock_to_extend(rtg); bmbno < bmblocks; + bmbno++) { + error = xfs_growfs_rt_bmblock(rtg, nrblocks, rextsize, bmbno); + if (error) + goto out_error; + } + + kvfree(old_rsum_cache); + goto out_rele; + +out_error: + /* + * Reset rtg_extents to the old value if adding more blocks failed. + */ + xfs_rtgroup_calc_geometry(mp, rtg, rtg_rgno(rtg), mp->m_sb.sb_rgcount, + mp->m_sb.sb_rextents); + if (old_rsum_cache) { + kvfree(rtg->rtg_rsum_cache); + rtg->rtg_rsum_cache = old_rsum_cache; + } +out_rele: + xfs_rtgroup_rele(rtg); + return error; +} + +int +xfs_growfs_check_rtgeom( + const struct xfs_mount *mp, + xfs_rfsblock_t dblocks, + xfs_rfsblock_t rblocks, + xfs_extlen_t rextsize) +{ + xfs_extlen_t min_logfsbs; + struct xfs_mount *nmp; + + nmp = xfs_growfs_rt_alloc_fake_mount(mp, rblocks, rextsize); + if (!nmp) + return -ENOMEM; + nmp->m_sb.sb_dblocks = dblocks; + + xfs_rtrmapbt_compute_maxlevels(nmp); + xfs_rtrefcountbt_compute_maxlevels(nmp); + xfs_trans_resv_calc(nmp, M_RES(nmp)); + + /* + * New summary size can't be more than half the size of the log. This + * prevents us from getting a log overflow, since we'll log basically + * the whole summary file at once. + */ + min_logfsbs = min_t(xfs_extlen_t, xfs_log_calc_minimum_size(nmp), + nmp->m_rsumblocks * 2); + + trace_xfs_growfs_check_rtgeom(mp, min_logfsbs); + + if (min_logfsbs > mp->m_sb.sb_logblocks) + goto out_inval; + + if (xfs_has_zoned(mp)) { + uint32_t gblocks = mp->m_groups[XG_TYPE_RTG].blocks; + uint32_t rem; + + if (rextsize != 1) + goto out_inval; + div_u64_rem(nmp->m_sb.sb_rblocks, gblocks, &rem); + if (rem) { + xfs_warn(mp, +"new RT volume size (%lld) not aligned to RT group size (%d)", + nmp->m_sb.sb_rblocks, gblocks); + goto out_inval; + } + } + + kfree(nmp); + return 0; +out_inval: + kfree(nmp); + return -EINVAL; +} + +/* + * Compute the new number of rt groups and ensure that /rtgroups exists. + * + * Changing the rtgroup size is not allowed (even if the rt volume hasn't yet + * been initialized) because the userspace ABI doesn't support it. + */ +static int +xfs_growfs_rt_prep_groups( + struct xfs_mount *mp, + xfs_rfsblock_t rblocks, + xfs_extlen_t rextsize, + xfs_rgnumber_t *new_rgcount) +{ + int error; + + *new_rgcount = howmany_64(rblocks, mp->m_sb.sb_rgextents * rextsize); + if (*new_rgcount > XFS_MAX_RGNUMBER) + return -EINVAL; + + /* Make sure the /rtgroups dir has been created */ + if (!mp->m_rtdirip) { + struct xfs_trans *tp; + + tp = xfs_trans_alloc_empty(mp); + error = xfs_rtginode_load_parent(tp); + xfs_trans_cancel(tp); + + if (error == -ENOENT) + error = xfs_rtginode_mkdir_parent(mp); + if (error) + return error; + } + + return 0; +} + +static bool +xfs_grow_last_rtg( + struct xfs_mount *mp) +{ + if (!xfs_has_rtgroups(mp)) + return true; + if (mp->m_sb.sb_rgcount == 0) + return false; + return xfs_rtgroup_extents(mp, mp->m_sb.sb_rgcount - 1) < + mp->m_sb.sb_rgextents; +} + +/* + * Read in the last block of the RT device to make sure it is accessible. + */ +static int +xfs_rt_check_size( + struct xfs_mount *mp, + xfs_rfsblock_t last_block) +{ + xfs_daddr_t daddr = XFS_FSB_TO_BB(mp, last_block); + struct xfs_buf *bp; + int error; + + if (XFS_BB_TO_FSB(mp, daddr) != last_block) { + xfs_warn(mp, "RT device size overflow: %llu != %llu", + XFS_BB_TO_FSB(mp, daddr), last_block); + return -EFBIG; + } + + error = xfs_buf_read_uncached(mp->m_rtdev_targp, + XFS_FSB_TO_BB(mp, mp->m_sb.sb_rtstart) + daddr, + XFS_FSB_TO_BB(mp, 1), &bp, NULL); + if (error) + xfs_warn(mp, "cannot read last RT device sector (%lld)", + last_block); + else + xfs_buf_relse(bp); + return error; +} + +/* + * Grow the realtime area of the filesystem. + */ +int +xfs_growfs_rt( + struct xfs_mount *mp, + struct xfs_growfs_rt *in) +{ + xfs_rgnumber_t old_rgcount = mp->m_sb.sb_rgcount; + xfs_rgnumber_t new_rgcount = 1; + xfs_rgnumber_t rgno; + xfs_agblock_t old_rextsize = mp->m_sb.sb_rextsize; + int error; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + /* Needs to have been mounted with an rt device. */ + if (!XFS_IS_REALTIME_MOUNT(mp)) + return -EINVAL; + + if (!mutex_trylock(&mp->m_growlock)) + return -EWOULDBLOCK; + + /* Shrink not supported. */ + error = -EINVAL; + if (in->newblocks <= mp->m_sb.sb_rblocks) + goto out_unlock; + /* Can only change rt extent size when adding rt volume. */ + if (mp->m_sb.sb_rblocks > 0 && in->extsize != mp->m_sb.sb_rextsize) + goto out_unlock; + + /* Range check the extent size. */ + if (XFS_FSB_TO_B(mp, in->extsize) > XFS_MAX_RTEXTSIZE || + XFS_FSB_TO_B(mp, in->extsize) < XFS_MIN_RTEXTSIZE) + goto out_unlock; + + /* Check for features supported only on rtgroups filesystems. */ + error = -EOPNOTSUPP; + if (!xfs_has_rtgroups(mp)) { + if (xfs_has_rmapbt(mp)) + goto out_unlock; + if (xfs_has_quota(mp)) + goto out_unlock; + if (xfs_has_reflink(mp)) + goto out_unlock; + } else if (xfs_has_reflink(mp) && + !xfs_reflink_supports_rextsize(mp, in->extsize)) + goto out_unlock; + + error = xfs_sb_validate_fsb_count(&mp->m_sb, in->newblocks); + if (error) + goto out_unlock; + + error = xfs_rt_check_size(mp, in->newblocks - 1); + if (error) + goto out_unlock; + + /* + * Calculate new parameters. These are the final values to be reached. + */ + error = -EINVAL; + if (in->newblocks < in->extsize) + goto out_unlock; + + /* Make sure the new fs size won't cause problems with the log. */ + error = xfs_growfs_check_rtgeom(mp, mp->m_sb.sb_dblocks, in->newblocks, + in->extsize); + if (error) + goto out_unlock; + + if (xfs_has_rtgroups(mp)) { + error = xfs_growfs_rt_prep_groups(mp, in->newblocks, + in->extsize, &new_rgcount); + if (error) + goto out_unlock; + } + + if (xfs_grow_last_rtg(mp)) { + error = xfs_growfs_rtg(mp, old_rgcount - 1, in->newblocks, + in->extsize); + if (error) + goto out_unlock; + } + + for (rgno = old_rgcount; rgno < new_rgcount; rgno++) { + xfs_rtbxlen_t rextents = div_u64(in->newblocks, in->extsize); + + error = xfs_rtgroup_alloc(mp, rgno, new_rgcount, rextents); + if (error) + goto out_unlock; + + error = xfs_growfs_rtg(mp, rgno, in->newblocks, in->extsize); + if (error) { + struct xfs_rtgroup *rtg; + + rtg = xfs_rtgroup_grab(mp, rgno); + if (!WARN_ON_ONCE(!rtg)) { + xfs_rtunmount_rtg(rtg); + xfs_rtgroup_rele(rtg); + xfs_rtgroup_free(mp, rgno); + } + break; + } + } + + if (!error && old_rextsize != in->extsize) + error = xfs_growfs_rt_fixup_extsize(mp); + + /* + * Update secondary superblocks now the physical grow has completed. + * + * Also do this in case of an error as we might have already + * successfully updated one or more RTGs and incremented sb_rgcount. + */ + if (!xfs_is_shutdown(mp)) { + int error2 = xfs_update_secondary_sbs(mp); + + if (!error) + error = error2; + + /* Reset the rt metadata btree space reservations. */ + error2 = xfs_metafile_resv_init(mp); + if (error2 && error2 != -ENOSPC) + error = error2; + } + +out_unlock: + mutex_unlock(&mp->m_growlock); + return error; +} + +/* Read the realtime superblock and attach it to the mount. */ +int +xfs_rtmount_readsb( + struct xfs_mount *mp) +{ + struct xfs_buf *bp; + int error; + + if (!xfs_has_rtsb(mp)) + return 0; + if (mp->m_sb.sb_rblocks == 0) + return 0; + if (mp->m_rtdev_targp == NULL) { + xfs_warn(mp, + "Filesystem has a realtime volume, use rtdev=device option"); + return -ENODEV; + } + + /* m_blkbb_log is not set up yet */ + error = xfs_buf_read_uncached(mp->m_rtdev_targp, XFS_RTSB_DADDR, + mp->m_sb.sb_blocksize >> BBSHIFT, &bp, + &xfs_rtsb_buf_ops); + if (error) { + xfs_warn(mp, "rt sb validate failed with error %d.", error); + /* bad CRC means corrupted metadata */ + if (error == -EFSBADCRC) + error = -EFSCORRUPTED; + return error; + } + + mp->m_rtsb_bp = bp; + xfs_buf_unlock(bp); + return 0; +} + +/* Detach the realtime superblock from the mount and free it. */ +void +xfs_rtmount_freesb( + struct xfs_mount *mp) +{ + struct xfs_buf *bp = mp->m_rtsb_bp; + + if (!bp) + return; + + xfs_buf_lock(bp); + mp->m_rtsb_bp = NULL; + xfs_buf_relse(bp); +} + +/* + * Initialize realtime fields in the mount structure. + */ +int /* error */ +xfs_rtmount_init( + struct xfs_mount *mp) /* file system mount structure */ +{ + if (mp->m_sb.sb_rblocks == 0) + return 0; + if (mp->m_rtdev_targp == NULL) { + xfs_warn(mp, + "Filesystem has a realtime volume, use rtdev=device option"); + return -ENODEV; + } + + mp->m_rsumblocks = xfs_rtsummary_blockcount(mp, &mp->m_rsumlevels); + + return xfs_rt_check_size(mp, mp->m_sb.sb_rblocks - 1); +} + +static int +xfs_rtalloc_count_frextent( + struct xfs_rtgroup *rtg, + struct xfs_trans *tp, + const struct xfs_rtalloc_rec *rec, + void *priv) +{ + uint64_t *valp = priv; + + *valp += rec->ar_extcount; + return 0; +} + +/* + * Reinitialize the number of free realtime extents from the realtime bitmap. + * Callers must ensure that there is no other activity in the filesystem. + */ +int +xfs_rtalloc_reinit_frextents( + struct xfs_mount *mp) +{ + uint64_t val = 0; + int error; + + struct xfs_rtgroup *rtg = NULL; + + while ((rtg = xfs_rtgroup_next(mp, rtg))) { + xfs_rtgroup_lock(rtg, XFS_RTGLOCK_BITMAP_SHARED); + error = xfs_rtalloc_query_all(rtg, NULL, + xfs_rtalloc_count_frextent, &val); + xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_BITMAP_SHARED); + if (error) { + xfs_rtgroup_rele(rtg); + return error; + } + } + + spin_lock(&mp->m_sb_lock); + mp->m_sb.sb_frextents = val; + spin_unlock(&mp->m_sb_lock); + xfs_set_freecounter(mp, XC_FREE_RTEXTENTS, mp->m_sb.sb_frextents); + return 0; +} + +/* + * Read in the bmbt of an rt metadata inode so that we never have to load them + * at runtime. This enables the use of shared ILOCKs for rtbitmap scans. Use + * an empty transaction to avoid deadlocking on loops in the bmbt. + */ +static inline int +xfs_rtmount_iread_extents( + struct xfs_trans *tp, + struct xfs_inode *ip) +{ + int error; + + xfs_ilock(ip, XFS_ILOCK_EXCL); + + error = xfs_iread_extents(tp, ip, XFS_DATA_FORK); + if (error) + goto out_unlock; + + if (xfs_inode_has_attr_fork(ip)) { + error = xfs_iread_extents(tp, ip, XFS_ATTR_FORK); + if (error) + goto out_unlock; + } + +out_unlock: + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; +} + +static int +xfs_rtmount_rtg( + struct xfs_mount *mp, + struct xfs_trans *tp, + struct xfs_rtgroup *rtg) +{ + int error, i; + + for (i = 0; i < XFS_RTGI_MAX; i++) { + error = xfs_rtginode_load(rtg, i, tp); + if (error) + return error; + + if (rtg->rtg_inodes[i]) { + error = xfs_rtmount_iread_extents(tp, + rtg->rtg_inodes[i]); + if (error) + return error; + } + } + + if (xfs_has_zoned(mp)) + return 0; + return xfs_alloc_rsum_cache(rtg, mp->m_sb.sb_rbmblocks); +} + +/* + * Get the bitmap and summary inodes and the summary cache into the mount + * structure at mount time. + */ +int +xfs_rtmount_inodes( + struct xfs_mount *mp) +{ + struct xfs_trans *tp; + struct xfs_rtgroup *rtg = NULL; + int error; + + tp = xfs_trans_alloc_empty(mp); + if (xfs_has_rtgroups(mp) && mp->m_sb.sb_rgcount > 0) { + error = xfs_rtginode_load_parent(tp); + if (error) + goto out_cancel; + } + + while ((rtg = xfs_rtgroup_next(mp, rtg))) { + error = xfs_rtmount_rtg(mp, tp, rtg); + if (error) { + xfs_rtgroup_rele(rtg); + xfs_rtunmount_inodes(mp); + break; + } + } + +out_cancel: + xfs_trans_cancel(tp); + return error; +} + +void +xfs_rtunmount_inodes( + struct xfs_mount *mp) +{ + struct xfs_rtgroup *rtg = NULL; + + while ((rtg = xfs_rtgroup_next(mp, rtg))) + xfs_rtunmount_rtg(rtg); + xfs_rtginode_irele(&mp->m_rtdirip); +} + +/* + * Pick an extent for allocation at the start of a new realtime file. + * Use the sequence number stored in the atime field of the bitmap inode. + * Translate this to a fraction of the rtextents, and return the product + * of rtextents and the fraction. + * The fraction sequence is 0, 1/2, 1/4, 3/4, 1/8, ..., 7/8, 1/16, ... + */ +static xfs_rtxnum_t +xfs_rtpick_extent( + struct xfs_rtgroup *rtg, + struct xfs_trans *tp, + xfs_rtxlen_t len) /* allocation length (rtextents) */ +{ + struct xfs_mount *mp = rtg_mount(rtg); + struct xfs_inode *rbmip = rtg_bitmap(rtg); + xfs_rtxnum_t b = 0; /* result rtext */ + int log2; /* log of sequence number */ + uint64_t resid; /* residual after log removed */ + uint64_t seq; /* sequence number of file creation */ + struct timespec64 ts; /* timespec in inode */ + + xfs_assert_ilocked(rbmip, XFS_ILOCK_EXCL); + + ts = inode_get_atime(VFS_I(rbmip)); + if (!(rbmip->i_diflags & XFS_DIFLAG_NEWRTBM)) { + rbmip->i_diflags |= XFS_DIFLAG_NEWRTBM; + seq = 0; + } else { + seq = ts.tv_sec; + } + log2 = xfs_highbit64(seq); + if (log2 != -1) { + resid = seq - (1ULL << log2); + b = (mp->m_sb.sb_rextents * ((resid << 1) + 1ULL)) >> + (log2 + 1); + if (b >= mp->m_sb.sb_rextents) + div64_u64_rem(b, mp->m_sb.sb_rextents, &b); + if (b + len > mp->m_sb.sb_rextents) + b = mp->m_sb.sb_rextents - len; + } + ts.tv_sec = seq + 1; + inode_set_atime_to_ts(VFS_I(rbmip), ts); + xfs_trans_log_inode(tp, rbmip, XFS_ILOG_CORE); + return b; +} + +static void +xfs_rtalloc_align_minmax( + xfs_rtxlen_t *raminlen, + xfs_rtxlen_t *ramaxlen, + xfs_rtxlen_t *prod) +{ + xfs_rtxlen_t newmaxlen = *ramaxlen; + xfs_rtxlen_t newminlen = *raminlen; + xfs_rtxlen_t slack; + + slack = newmaxlen % *prod; + if (slack) + newmaxlen -= slack; + slack = newminlen % *prod; + if (slack) + newminlen += *prod - slack; + + /* + * If adjusting for extent size hint alignment produces an invalid + * min/max len combination, go ahead without it. + */ + if (newmaxlen < newminlen) { + *prod = 1; + return; + } + *ramaxlen = newmaxlen; + *raminlen = newminlen; +} + +/* Given a free extent, find any part of it that isn't busy, if possible. */ +STATIC bool +xfs_rtalloc_check_busy( + struct xfs_rtalloc_args *args, + xfs_rtxnum_t start, + xfs_rtxlen_t minlen_rtx, + xfs_rtxlen_t maxlen_rtx, + xfs_rtxlen_t len_rtx, + xfs_rtxlen_t prod, + xfs_rtxnum_t rtx, + xfs_rtxlen_t *reslen, + xfs_rtxnum_t *resrtx, + unsigned *busy_gen) +{ + struct xfs_rtgroup *rtg = args->rtg; + struct xfs_mount *mp = rtg_mount(rtg); + xfs_agblock_t rgbno = xfs_rtx_to_rgbno(rtg, rtx); + xfs_rgblock_t min_rgbno = xfs_rtx_to_rgbno(rtg, start); + xfs_extlen_t minlen = xfs_rtxlen_to_extlen(mp, minlen_rtx); + xfs_extlen_t len = xfs_rtxlen_to_extlen(mp, len_rtx); + xfs_extlen_t diff; + bool busy; + + busy = xfs_extent_busy_trim(rtg_group(rtg), minlen, + xfs_rtxlen_to_extlen(mp, maxlen_rtx), &rgbno, &len, + busy_gen); + + /* + * If we have a largish extent that happens to start before min_rgbno, + * see if we can shift it into range... + */ + if (rgbno < min_rgbno && rgbno + len > min_rgbno) { + diff = min_rgbno - rgbno; + if (len > diff) { + rgbno += diff; + len -= diff; + } + } + + if (prod > 1 && len >= minlen) { + xfs_rgblock_t aligned_rgbno = roundup(rgbno, prod); + + diff = aligned_rgbno - rgbno; + + *resrtx = xfs_rgbno_to_rtx(mp, aligned_rgbno); + *reslen = xfs_extlen_to_rtxlen(mp, + diff >= len ? 0 : len - diff); + } else { + *resrtx = xfs_rgbno_to_rtx(mp, rgbno); + *reslen = xfs_extlen_to_rtxlen(mp, len); + } + + return busy; +} + +/* + * Adjust the given free extent so that it isn't busy, or flush the log and + * wait for the space to become unbusy. Only needed for rtgroups. + */ +STATIC int +xfs_rtallocate_adjust_for_busy( + struct xfs_rtalloc_args *args, + xfs_rtxnum_t start, + xfs_rtxlen_t minlen, + xfs_rtxlen_t maxlen, + xfs_rtxlen_t *len, + xfs_rtxlen_t prod, + xfs_rtxnum_t *rtx) +{ + xfs_rtxnum_t resrtx; + xfs_rtxlen_t reslen; + unsigned busy_gen; + bool busy; + int error; + +again: + busy = xfs_rtalloc_check_busy(args, start, minlen, maxlen, *len, prod, + *rtx, &reslen, &resrtx, &busy_gen); + if (!busy) + return 0; + + if (reslen < minlen || (start != 0 && resrtx != *rtx)) { + /* + * Enough of the extent was busy that we cannot satisfy the + * allocation, or this is a near allocation and the start of + * the extent is busy. Flush the log and wait for the busy + * situation to resolve. + */ + trace_xfs_rtalloc_extent_busy(args->rtg, start, minlen, maxlen, + *len, prod, *rtx, busy_gen); + + error = xfs_extent_busy_flush(args->tp, rtg_group(args->rtg), + busy_gen, 0); + if (error) + return error; + + goto again; + } + + /* Some of the free space wasn't busy, hand that back to the caller. */ + trace_xfs_rtalloc_extent_busy_trim(args->rtg, *rtx, *len, resrtx, + reslen); + *len = reslen; + *rtx = resrtx; + + return 0; +} + +static int +xfs_rtallocate_rtg( + struct xfs_trans *tp, + xfs_rgnumber_t rgno, + xfs_rtblock_t bno_hint, + xfs_rtxlen_t minlen, + xfs_rtxlen_t maxlen, + xfs_rtxlen_t prod, + bool wasdel, + bool initial_user_data, + bool *rtlocked, + xfs_rtblock_t *bno, + xfs_extlen_t *blen) +{ + struct xfs_rtalloc_args args = { + .mp = tp->t_mountp, + .tp = tp, + }; + xfs_rtxnum_t start = 0; + xfs_rtxnum_t rtx; + xfs_rtxlen_t len = 0; + int error = 0; + + args.rtg = xfs_rtgroup_grab(args.mp, rgno); + if (!args.rtg) + return -ENOSPC; + + /* + * We need to lock out modifications to both the RT bitmap and summary + * inodes for finding free space in xfs_rtallocate_extent_{near,size} + * and join the bitmap and summary inodes for the actual allocation + * down in xfs_rtallocate_range. + * + * For RTG-enabled file system we don't want to join the inodes to the + * transaction until we are committed to allocate to allocate from this + * RTG so that only one inode of each type is locked at a time. + * + * But for pre-RTG file systems we need to already to join the bitmap + * inode to the transaction for xfs_rtpick_extent, which bumps the + * sequence number in it, so we'll have to join the inode to the + * transaction early here. + * + * This is all a bit messy, but at least the mess is contained in + * this function. + */ + if (!*rtlocked) { + xfs_rtgroup_lock(args.rtg, XFS_RTGLOCK_BITMAP); + if (!xfs_has_rtgroups(args.mp)) + xfs_rtgroup_trans_join(tp, args.rtg, + XFS_RTGLOCK_BITMAP); + *rtlocked = true; + } + + /* + * For an allocation to an empty file at offset 0, pick an extent that + * will space things out in the rt area. + */ + if (bno_hint != NULLFSBLOCK) + start = xfs_rtb_to_rtx(args.mp, bno_hint); + else if (!xfs_has_rtgroups(args.mp) && initial_user_data) + start = xfs_rtpick_extent(args.rtg, tp, maxlen); + + if (start) { + error = xfs_rtallocate_extent_near(&args, start, minlen, maxlen, + &len, prod, &rtx); + /* + * If we can't allocate near a specific rt extent, try again + * without locality criteria. + */ + if (error == -ENOSPC) { + xfs_rtbuf_cache_relse(&args); + error = 0; + } + } + + if (!error) { + error = xfs_rtallocate_extent_size(&args, minlen, maxlen, &len, + prod, &rtx); + } + + if (error) { + if (xfs_has_rtgroups(args.mp)) + goto out_unlock; + goto out_release; + } + + if (xfs_has_rtgroups(args.mp)) { + error = xfs_rtallocate_adjust_for_busy(&args, start, minlen, + maxlen, &len, prod, &rtx); + if (error) + goto out_unlock; + + xfs_rtgroup_trans_join(tp, args.rtg, XFS_RTGLOCK_BITMAP); + } + + error = xfs_rtallocate_range(&args, rtx, len); + if (error) + goto out_release; + + xfs_trans_mod_sb(tp, wasdel ? + XFS_TRANS_SB_RES_FREXTENTS : XFS_TRANS_SB_FREXTENTS, + -(long)len); + *bno = xfs_rtx_to_rtb(args.rtg, rtx); + *blen = xfs_rtxlen_to_extlen(args.mp, len); + +out_release: + xfs_rtgroup_rele(args.rtg); + xfs_rtbuf_cache_relse(&args); + return error; +out_unlock: + xfs_rtgroup_unlock(args.rtg, XFS_RTGLOCK_BITMAP); + *rtlocked = false; + goto out_release; +} + +int +xfs_rtallocate_rtgs( + struct xfs_trans *tp, + xfs_fsblock_t bno_hint, + xfs_rtxlen_t minlen, + xfs_rtxlen_t maxlen, + xfs_rtxlen_t prod, + bool wasdel, + bool initial_user_data, + xfs_rtblock_t *bno, + xfs_extlen_t *blen) +{ + struct xfs_mount *mp = tp->t_mountp; + xfs_rgnumber_t start_rgno, rgno; + int error; + + /* + * For now this just blindly iterates over the RTGs for an initial + * allocation. We could try to keep an in-memory rtg_longest member + * to avoid the locking when just looking for big enough free space, + * but for now this keeps things simple. + */ + if (bno_hint != NULLFSBLOCK) + start_rgno = xfs_rtb_to_rgno(mp, bno_hint); + else + start_rgno = (atomic_inc_return(&mp->m_rtgrotor) - 1) % + mp->m_sb.sb_rgcount; + + rgno = start_rgno; + do { + bool rtlocked = false; + + error = xfs_rtallocate_rtg(tp, rgno, bno_hint, minlen, maxlen, + prod, wasdel, initial_user_data, &rtlocked, + bno, blen); + if (error != -ENOSPC) + return error; + ASSERT(!rtlocked); + + if (++rgno == mp->m_sb.sb_rgcount) + rgno = 0; + bno_hint = NULLFSBLOCK; + } while (rgno != start_rgno); + + return -ENOSPC; +} + +static int +xfs_rtallocate_align( + struct xfs_bmalloca *ap, + xfs_rtxlen_t *ralen, + xfs_rtxlen_t *raminlen, + xfs_rtxlen_t *prod, + bool *noalign) +{ + struct xfs_mount *mp = ap->ip->i_mount; + xfs_fileoff_t orig_offset = ap->offset; + xfs_extlen_t minlen = mp->m_sb.sb_rextsize; + xfs_extlen_t align; /* minimum allocation alignment */ + xfs_extlen_t mod; /* product factor for allocators */ + int error; + + if (*noalign) { + align = mp->m_sb.sb_rextsize; + } else { + if (ap->flags & XFS_BMAPI_COWFORK) + align = xfs_get_cowextsz_hint(ap->ip); + else + align = xfs_get_extsz_hint(ap->ip); + if (!align) + align = 1; + if (align == mp->m_sb.sb_rextsize) + *noalign = true; + } + + error = xfs_bmap_extsize_align(mp, &ap->got, &ap->prev, align, 1, + ap->eof, 0, ap->conv, &ap->offset, &ap->length); + if (error) + return error; + ASSERT(ap->length); + ASSERT(xfs_extlen_to_rtxmod(mp, ap->length) == 0); + + /* + * If we shifted the file offset downward to satisfy an extent size + * hint, increase minlen by that amount so that the allocator won't + * give us an allocation that's too short to cover at least one of the + * blocks that the caller asked for. + */ + if (ap->offset != orig_offset) + minlen += orig_offset - ap->offset; + + /* + * Set ralen to be the actual requested length in rtextents. + * + * If the old value was close enough to XFS_BMBT_MAX_EXTLEN that + * we rounded up to it, cut it back so it's valid again. + * Note that if it's a really large request (bigger than + * XFS_BMBT_MAX_EXTLEN), we don't hear about that number, and can't + * adjust the starting point to match it. + */ + *ralen = xfs_extlen_to_rtxlen(mp, min(ap->length, XFS_MAX_BMBT_EXTLEN)); + *raminlen = max_t(xfs_rtxlen_t, 1, xfs_extlen_to_rtxlen(mp, minlen)); + ASSERT(*raminlen > 0); + ASSERT(*raminlen <= *ralen); + + /* + * Only bother calculating a real prod factor if offset & length are + * perfectly aligned, otherwise it will just get us in trouble. + */ + div_u64_rem(ap->offset, align, &mod); + if (mod || ap->length % align) + *prod = 1; + else + *prod = xfs_extlen_to_rtxlen(mp, align); + + if (*prod > 1) + xfs_rtalloc_align_minmax(raminlen, ralen, prod); + return 0; +} + +int +xfs_bmap_rtalloc( + struct xfs_bmalloca *ap) +{ + xfs_fileoff_t orig_offset = ap->offset; + xfs_rtxlen_t prod = 0; /* product factor for allocators */ + xfs_rtxlen_t ralen = 0; /* realtime allocation length */ + xfs_rtblock_t bno_hint = NULLRTBLOCK; + xfs_extlen_t orig_length = ap->length; + xfs_rtxlen_t raminlen; + bool rtlocked = false; + bool noalign = false; + bool initial_user_data = + ap->datatype & XFS_ALLOC_INITIAL_USER_DATA; + int error; + + ASSERT(!xfs_has_zoned(ap->tp->t_mountp)); + +retry: + error = xfs_rtallocate_align(ap, &ralen, &raminlen, &prod, &noalign); + if (error) + return error; + + if (xfs_bmap_adjacent(ap)) + bno_hint = ap->blkno; + + if (xfs_has_rtgroups(ap->ip->i_mount)) { + error = xfs_rtallocate_rtgs(ap->tp, bno_hint, raminlen, ralen, + prod, ap->wasdel, initial_user_data, + &ap->blkno, &ap->length); + } else { + error = xfs_rtallocate_rtg(ap->tp, 0, bno_hint, raminlen, ralen, + prod, ap->wasdel, initial_user_data, + &rtlocked, &ap->blkno, &ap->length); + } + + if (error == -ENOSPC) { + if (!noalign) { + /* + * We previously enlarged the request length to try to + * satisfy an extent size hint. The allocator didn't + * return anything, so reset the parameters to the + * original values and try again without alignment + * criteria. + */ + ap->offset = orig_offset; + ap->length = orig_length; + noalign = true; + goto retry; + } + + ap->blkno = NULLFSBLOCK; + ap->length = 0; + return 0; + } + if (error) + return error; + + xfs_bmap_alloc_account(ap); + return 0; +} diff --git a/fs/xfs/xfs_rtalloc.h b/fs/xfs/xfs_rtalloc.h new file mode 100644 index 000000000..78a690b48 --- /dev/null +++ b/fs/xfs/xfs_rtalloc.h @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2003,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_RTALLOC_H__ +#define __XFS_RTALLOC_H__ + +/* kernel only definitions and functions */ + +struct xfs_mount; +struct xfs_trans; + +#ifdef CONFIG_XFS_RT +/* rtgroup superblock initialization */ +int xfs_rtmount_readsb(struct xfs_mount *mp); +void xfs_rtmount_freesb(struct xfs_mount *mp); + +/* + * Initialize realtime fields in the mount structure. + */ +int /* error */ +xfs_rtmount_init( + struct xfs_mount *mp); /* file system mount structure */ +void +xfs_rtunmount_inodes( + struct xfs_mount *mp); + +/* + * Get the bitmap and summary inodes into the mount structure + * at mount time. + */ +int /* error */ +xfs_rtmount_inodes( + struct xfs_mount *mp); /* file system mount structure */ + +/* + * Grow the realtime area of the filesystem. + */ +int +xfs_growfs_rt( + struct xfs_mount *mp, /* file system mount structure */ + xfs_growfs_rt_t *in); /* user supplied growfs struct */ + +int xfs_rtalloc_reinit_frextents(struct xfs_mount *mp); +int xfs_growfs_check_rtgeom(const struct xfs_mount *mp, xfs_rfsblock_t dblocks, + xfs_rfsblock_t rblocks, xfs_agblock_t rextsize); +#else +# define xfs_growfs_rt(mp,in) (-ENOSYS) +# define xfs_rtalloc_reinit_frextents(m) (0) +# define xfs_rtmount_readsb(mp) (0) +# define xfs_rtmount_freesb(mp) ((void)0) +static inline int /* error */ +xfs_rtmount_init( + xfs_mount_t *mp) /* file system mount structure */ +{ + if (mp->m_sb.sb_rblocks == 0) + return 0; + + xfs_warn(mp, "Not built with CONFIG_XFS_RT"); + return -ENOSYS; +} +# define xfs_rtmount_inodes(m) (((mp)->m_sb.sb_rblocks == 0)? 0 : (-ENOSYS)) +# define xfs_rtunmount_inodes(m) + +static inline int +xfs_growfs_check_rtgeom(const struct xfs_mount *mp, + xfs_rfsblock_t dblocks, xfs_rfsblock_t rblocks, + xfs_extlen_t rextsize) +{ + return 0; +} +#endif /* CONFIG_XFS_RT */ + +int xfs_rtallocate_rtgs(struct xfs_trans *tp, xfs_fsblock_t bno_hint, + xfs_rtxlen_t minlen, xfs_rtxlen_t maxlen, xfs_rtxlen_t prod, + bool wasdel, bool initial_user_data, xfs_rtblock_t *bno, + xfs_extlen_t *blen); + +#endif /* __XFS_RTALLOC_H__ */ diff --git a/fs/xfs/xfs_stats.c b/fs/xfs/xfs_stats.c new file mode 100644 index 000000000..c13d60073 --- /dev/null +++ b/fs/xfs/xfs_stats.c @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2003,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" + +struct xstats xfsstats; + +static int counter_val(struct xfsstats __percpu *stats, int idx) +{ + int val = 0, cpu; + + for_each_possible_cpu(cpu) + val += *(((__u32 *)per_cpu_ptr(stats, cpu) + idx)); + return val; +} + +int xfs_stats_format(struct xfsstats __percpu *stats, char *buf) +{ + int i, j; + int len = 0; + uint64_t xs_xstrat_bytes = 0; + uint64_t xs_write_bytes = 0; + uint64_t xs_read_bytes = 0; + uint64_t xs_defer_relog = 0; + uint64_t xs_gc_bytes = 0; + + static const struct xstats_entry { + char *desc; + int endpoint; + } xstats[] = { + { "extent_alloc", xfsstats_offset(xs_abt_lookup) }, + { "abt", xfsstats_offset(xs_blk_mapr) }, + { "blk_map", xfsstats_offset(xs_bmbt_lookup) }, + { "bmbt", xfsstats_offset(xs_dir_lookup) }, + { "dir", xfsstats_offset(xs_trans_sync) }, + { "trans", xfsstats_offset(xs_ig_attempts) }, + { "ig", xfsstats_offset(xs_log_writes) }, + { "log", xfsstats_offset(xs_try_logspace)}, + { "push_ail", xfsstats_offset(xs_xstrat_quick)}, + { "xstrat", xfsstats_offset(xs_write_calls) }, + { "rw", xfsstats_offset(xs_attr_get) }, + { "attr", xfsstats_offset(xs_iflush_count)}, + { "icluster", xfsstats_offset(xs_inodes_active) }, + { "vnodes", xfsstats_offset(xb_get) }, + { "buf", xfsstats_offset(xs_abtb_2) }, + { "abtb2", xfsstats_offset(xs_abtc_2) }, + { "abtc2", xfsstats_offset(xs_bmbt_2) }, + { "bmbt2", xfsstats_offset(xs_ibt_2) }, + { "ibt2", xfsstats_offset(xs_fibt_2) }, + { "fibt2", xfsstats_offset(xs_rmap_2) }, + { "rmapbt", xfsstats_offset(xs_refcbt_2) }, + { "refcntbt", xfsstats_offset(xs_rmap_mem_2) }, + { "rmapbt_mem", xfsstats_offset(xs_rcbag_2) }, + { "rcbagbt", xfsstats_offset(xs_rtrmap_2) }, + { "rtrmapbt", xfsstats_offset(xs_rtrmap_mem_2)}, + { "rtrmapbt_mem", xfsstats_offset(xs_rtrefcbt_2) }, + { "rtrefcntbt", xfsstats_offset(xs_qm_dqreclaims)}, + /* we print both series of quota information together */ + { "qm", xfsstats_offset(xs_gc_read_calls)}, + { "zoned", xfsstats_offset(xs_inodes_meta)}, + { "metafile", xfsstats_offset(xs_xstrat_bytes)}, + }; + + /* Loop over all stats groups */ + + for (i = j = 0; i < ARRAY_SIZE(xstats); i++) { + len += scnprintf(buf + len, PATH_MAX - len, "%s", + xstats[i].desc); + /* inner loop does each group */ + for (; j < xstats[i].endpoint; j++) + len += scnprintf(buf + len, PATH_MAX - len, " %u", + counter_val(stats, j)); + len += scnprintf(buf + len, PATH_MAX - len, "\n"); + } + /* extra precision counters */ + for_each_possible_cpu(i) { + xs_xstrat_bytes += per_cpu_ptr(stats, i)->s.xs_xstrat_bytes; + xs_write_bytes += per_cpu_ptr(stats, i)->s.xs_write_bytes; + xs_read_bytes += per_cpu_ptr(stats, i)->s.xs_read_bytes; + xs_defer_relog += per_cpu_ptr(stats, i)->s.xs_defer_relog; + xs_gc_bytes += per_cpu_ptr(stats, i)->s.xs_gc_bytes; + } + + len += scnprintf(buf + len, PATH_MAX-len, "xpc %llu %llu %llu\n", + xs_xstrat_bytes, xs_write_bytes, xs_read_bytes); + len += scnprintf(buf + len, PATH_MAX-len, "defer_relog %llu\n", + xs_defer_relog); + len += scnprintf(buf + len, PATH_MAX-len, "debug %u\n", +#if defined(DEBUG) + 1); +#else + 0); +#endif + len += scnprintf(buf + len, PATH_MAX-len, "gc xpc %llu\n", xs_gc_bytes); + + return len; +} + +void xfs_stats_clearall(struct xfsstats __percpu *stats) +{ + uint32_t xs_inodes_active, xs_inodes_meta; + int c; + + xfs_notice(NULL, "Clearing xfsstats"); + for_each_possible_cpu(c) { + preempt_disable(); + /* + * Save the active / meta inode counters, as they are stateful. + */ + xs_inodes_active = per_cpu_ptr(stats, c)->s.xs_inodes_active; + xs_inodes_meta = per_cpu_ptr(stats, c)->s.xs_inodes_meta; + memset(per_cpu_ptr(stats, c), 0, sizeof(*stats)); + per_cpu_ptr(stats, c)->s.xs_inodes_active = xs_inodes_active; + per_cpu_ptr(stats, c)->s.xs_inodes_meta = xs_inodes_meta; + preempt_enable(); + } +} + +#ifdef CONFIG_PROC_FS +/* legacy quota interfaces */ +#ifdef CONFIG_XFS_QUOTA + +#define XFSSTAT_START_XQMSTAT xfsstats_offset(xs_qm_dqreclaims) +#define XFSSTAT_END_XQMSTAT xfsstats_offset(xs_qm_dquot) + +static int xqm_proc_show(struct seq_file *m, void *v) +{ + /* maximum; incore; ratio free to inuse; freelist; rtquota */ + seq_printf(m, "%d\t%d\t%d\t%u\t%s\n", + 0, counter_val(xfsstats.xs_stats, XFSSTAT_END_XQMSTAT), + 0, counter_val(xfsstats.xs_stats, XFSSTAT_END_XQMSTAT + 1), + IS_ENABLED(CONFIG_XFS_RT) ? "rtquota" : "quota"); + return 0; +} + +/* legacy quota stats interface no 2 */ +static int xqmstat_proc_show(struct seq_file *m, void *v) +{ + int j; + + seq_puts(m, "qm"); + for (j = XFSSTAT_START_XQMSTAT; j < XFSSTAT_END_XQMSTAT; j++) + seq_printf(m, " %u", counter_val(xfsstats.xs_stats, j)); + seq_putc(m, '\n'); + return 0; +} +#endif /* CONFIG_XFS_QUOTA */ + +int +xfs_init_procfs(void) +{ + if (!proc_mkdir("fs/xfs", NULL)) + return -ENOMEM; + + if (!proc_symlink("fs/xfs/stat", NULL, + "/sys/fs/xfs/stats/stats")) + goto out; + +#ifdef CONFIG_XFS_QUOTA + if (!proc_create_single("fs/xfs/xqmstat", 0, NULL, xqmstat_proc_show)) + goto out; + if (!proc_create_single("fs/xfs/xqm", 0, NULL, xqm_proc_show)) + goto out; +#endif + return 0; + +out: + remove_proc_subtree("fs/xfs", NULL); + return -ENOMEM; +} + +void +xfs_cleanup_procfs(void) +{ + remove_proc_subtree("fs/xfs", NULL); +} +#endif /* CONFIG_PROC_FS */ diff --git a/fs/xfs/xfs_stats.h b/fs/xfs/xfs_stats.h new file mode 100644 index 000000000..57c32b86c --- /dev/null +++ b/fs/xfs/xfs_stats.h @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_STATS_H__ +#define __XFS_STATS_H__ + + +#include + +/* + * The btree stats arrays have fixed offsets for the different stats. We + * store the base index in the btree cursor via XFS_STATS_CALC_INDEX() and + * that allows us to use fixed offsets into the stats array for each btree + * stat. These index offsets are defined in the order they will be emitted + * in the stats files, so it is possible to add new btree stat types by + * appending to the enum list below. + */ +enum { + __XBTS_lookup = 0, + __XBTS_compare = 1, + __XBTS_insrec = 2, + __XBTS_delrec = 3, + __XBTS_newroot = 4, + __XBTS_killroot = 5, + __XBTS_increment = 6, + __XBTS_decrement = 7, + __XBTS_lshift = 8, + __XBTS_rshift = 9, + __XBTS_split = 10, + __XBTS_join = 11, + __XBTS_alloc = 12, + __XBTS_free = 13, + __XBTS_moves = 14, + + __XBTS_MAX = 15, +}; + +/* + * XFS global statistics + */ +struct __xfsstats { + uint32_t xs_allocx; + uint32_t xs_allocb; + uint32_t xs_freex; + uint32_t xs_freeb; + uint32_t xs_abt_lookup; + uint32_t xs_abt_compare; + uint32_t xs_abt_insrec; + uint32_t xs_abt_delrec; + uint32_t xs_blk_mapr; + uint32_t xs_blk_mapw; + uint32_t xs_blk_unmap; + uint32_t xs_add_exlist; + uint32_t xs_del_exlist; + uint32_t xs_look_exlist; + uint32_t xs_cmp_exlist; + uint32_t xs_bmbt_lookup; + uint32_t xs_bmbt_compare; + uint32_t xs_bmbt_insrec; + uint32_t xs_bmbt_delrec; + uint32_t xs_dir_lookup; + uint32_t xs_dir_create; + uint32_t xs_dir_remove; + uint32_t xs_dir_getdents; + uint32_t xs_trans_sync; + uint32_t xs_trans_async; + uint32_t xs_trans_empty; + uint32_t xs_ig_attempts; + uint32_t xs_ig_found; + uint32_t xs_ig_frecycle; + uint32_t xs_ig_missed; + uint32_t xs_ig_dup; + uint32_t xs_ig_reclaims; + uint32_t xs_ig_attrchg; + uint32_t xs_log_writes; + uint32_t xs_log_blocks; + uint32_t xs_log_noiclogs; + uint32_t xs_log_force; + uint32_t xs_log_force_sleep; + uint32_t xs_try_logspace; + uint32_t xs_sleep_logspace; + uint32_t xs_push_ail; + uint32_t xs_push_ail_success; + uint32_t xs_push_ail_pushbuf; + uint32_t xs_push_ail_pinned; + uint32_t xs_push_ail_locked; + uint32_t xs_push_ail_flushing; + uint32_t xs_push_ail_restarts; + uint32_t xs_push_ail_flush; + uint32_t xs_xstrat_quick; + uint32_t xs_xstrat_split; + uint32_t xs_write_calls; + uint32_t xs_read_calls; + uint32_t xs_attr_get; + uint32_t xs_attr_set; + uint32_t xs_attr_remove; + uint32_t xs_attr_list; + uint32_t xs_iflush_count; + uint32_t xs_icluster_flushcnt; + uint32_t xs_icluster_flushinode; + uint32_t xs_inodes_active; + uint32_t __unused_vn_alloc; + uint32_t __unused_vn_get; + uint32_t __unused_vn_hold; + uint32_t xs_inode_destroy; + uint32_t xs_inode_destroy2; /* same as xs_inode_destroy */ + uint32_t xs_inode_mark_reclaimable; + uint32_t __unused_vn_free; + uint32_t xb_get; + uint32_t xb_create; + uint32_t xb_get_locked; + uint32_t xb_get_locked_waited; + uint32_t xb_busy_locked; + uint32_t xb_miss_locked; + uint32_t xb_page_retries; + uint32_t xb_page_found; + uint32_t xb_get_read; +/* Version 2 btree counters */ + uint32_t xs_abtb_2[__XBTS_MAX]; + uint32_t xs_abtc_2[__XBTS_MAX]; + uint32_t xs_bmbt_2[__XBTS_MAX]; + uint32_t xs_ibt_2[__XBTS_MAX]; + uint32_t xs_fibt_2[__XBTS_MAX]; + uint32_t xs_rmap_2[__XBTS_MAX]; + uint32_t xs_refcbt_2[__XBTS_MAX]; + uint32_t xs_rmap_mem_2[__XBTS_MAX]; + uint32_t xs_rcbag_2[__XBTS_MAX]; + uint32_t xs_rtrmap_2[__XBTS_MAX]; + uint32_t xs_rtrmap_mem_2[__XBTS_MAX]; + uint32_t xs_rtrefcbt_2[__XBTS_MAX]; + uint32_t xs_qm_dqreclaims; + uint32_t xs_qm_dqreclaim_misses; + uint32_t xs_qm_dquot_dups; + uint32_t xs_qm_dqcachemisses; + uint32_t xs_qm_dqcachehits; + uint32_t xs_qm_dqwants; + uint32_t xs_qm_dquot; + uint32_t xs_qm_dquot_unused; +/* Zone GC counters */ + uint32_t xs_gc_read_calls; + uint32_t xs_gc_write_calls; + uint32_t xs_gc_zone_reset_calls; +/* Metafile counters */ + uint32_t xs_inodes_meta; +/* Extra precision counters */ + uint64_t xs_xstrat_bytes; + uint64_t xs_write_bytes; + uint64_t xs_read_bytes; + uint64_t xs_defer_relog; + uint64_t xs_gc_bytes; +}; + +#define xfsstats_offset(f) (offsetof(struct __xfsstats, f)/sizeof(uint32_t)) + +struct xfsstats { + union { + struct __xfsstats s; + uint32_t a[xfsstats_offset(xs_qm_dquot)]; + }; +}; + +/* + * simple wrapper for getting the array index of s struct member offset + */ +#define XFS_STATS_CALC_INDEX(member) \ + (offsetof(struct __xfsstats, member) / (int)sizeof(uint32_t)) + + +int xfs_stats_format(struct xfsstats __percpu *stats, char *buf); +void xfs_stats_clearall(struct xfsstats __percpu *stats); +extern struct xstats xfsstats; + +#define XFS_STATS_INC(mp, v) \ +do { \ + per_cpu_ptr(xfsstats.xs_stats, current_cpu())->s.v++; \ + per_cpu_ptr(mp->m_stats.xs_stats, current_cpu())->s.v++; \ +} while (0) + +#define XFS_STATS_DEC(mp, v) \ +do { \ + per_cpu_ptr(xfsstats.xs_stats, current_cpu())->s.v--; \ + per_cpu_ptr(mp->m_stats.xs_stats, current_cpu())->s.v--; \ +} while (0) + +#define XFS_STATS_ADD(mp, v, inc) \ +do { \ + per_cpu_ptr(xfsstats.xs_stats, current_cpu())->s.v += (inc); \ + per_cpu_ptr(mp->m_stats.xs_stats, current_cpu())->s.v += (inc); \ +} while (0) + +#define XFS_STATS_INC_OFF(mp, off) \ +do { \ + per_cpu_ptr(xfsstats.xs_stats, current_cpu())->a[off]++; \ + per_cpu_ptr(mp->m_stats.xs_stats, current_cpu())->a[off]++; \ +} while (0) + +#define XFS_STATS_DEC_OFF(mp, off) \ +do { \ + per_cpu_ptr(xfsstats.xs_stats, current_cpu())->a[off]; \ + per_cpu_ptr(mp->m_stats.xs_stats, current_cpu())->a[off]; \ +} while (0) + +#define XFS_STATS_ADD_OFF(mp, off, inc) \ +do { \ + per_cpu_ptr(xfsstats.xs_stats, current_cpu())->a[off] += (inc); \ + per_cpu_ptr(mp->m_stats.xs_stats, current_cpu())->a[off] += (inc); \ +} while (0) + +#if defined(CONFIG_PROC_FS) + +extern int xfs_init_procfs(void); +extern void xfs_cleanup_procfs(void); + + +#else /* !CONFIG_PROC_FS */ + +static inline int xfs_init_procfs(void) +{ + return 0; +} + +static inline void xfs_cleanup_procfs(void) +{ +} + +#endif /* !CONFIG_PROC_FS */ + +#endif /* __XFS_STATS_H__ */ diff --git a/fs/xfs/xfs_super.c b/fs/xfs/xfs_super.c new file mode 100644 index 000000000..4b2eeb778 --- /dev/null +++ b/fs/xfs/xfs_super.c @@ -0,0 +1,2774 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2006 Silicon Graphics, Inc. + * All Rights Reserved. + */ + +#include "xfs_platform.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_sb.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_btree.h" +#include "xfs_bmap.h" +#include "xfs_alloc.h" +#include "xfs_fsops.h" +#include "xfs_trans.h" +#include "xfs_buf_item.h" +#include "xfs_log.h" +#include "xfs_log_priv.h" +#include "xfs_dir2.h" +#include "xfs_extfree_item.h" +#include "xfs_mru_cache.h" +#include "xfs_inode_item.h" +#include "xfs_icache.h" +#include "xfs_trace.h" +#include "xfs_icreate_item.h" +#include "xfs_filestream.h" +#include "xfs_quota.h" +#include "xfs_sysfs.h" +#include "xfs_ondisk.h" +#include "xfs_rmap_item.h" +#include "xfs_refcount_item.h" +#include "xfs_bmap_item.h" +#include "xfs_reflink.h" +#include "xfs_pwork.h" +#include "xfs_ag.h" +#include "xfs_defer.h" +#include "xfs_attr_item.h" +#include "xfs_xattr.h" +#include "xfs_error.h" +#include "xfs_errortag.h" +#include "xfs_iunlink_item.h" +#include "xfs_dahash_test.h" +#include "xfs_rtbitmap.h" +#include "xfs_exchmaps_item.h" +#include "xfs_parent.h" +#include "xfs_rtalloc.h" +#include "xfs_zone_alloc.h" +#include "xfs_healthmon.h" +#include "scrub/stats.h" +#include "scrub/rcbag_btree.h" + +#include +#include +#include +#include + +static const struct super_operations xfs_super_operations; + +static struct dentry *xfs_debugfs; /* top-level xfs debugfs dir */ +static struct kset *xfs_kset; /* top-level xfs sysfs dir */ +#ifdef DEBUG +static struct xfs_kobj xfs_dbg_kobj; /* global debug sysfs attrs */ +#endif + +enum xfs_dax_mode { + XFS_DAX_INODE = 0, + XFS_DAX_ALWAYS = 1, + XFS_DAX_NEVER = 2, +}; + +/* Were quota mount options provided? Must use the upper 16 bits of qflags. */ +#define XFS_QFLAGS_MNTOPTS (1U << 31) + +static void +xfs_mount_set_dax_mode( + struct xfs_mount *mp, + enum xfs_dax_mode mode) +{ + switch (mode) { + case XFS_DAX_INODE: + mp->m_features &= ~(XFS_FEAT_DAX_ALWAYS | XFS_FEAT_DAX_NEVER); + break; + case XFS_DAX_ALWAYS: + mp->m_features |= XFS_FEAT_DAX_ALWAYS; + mp->m_features &= ~XFS_FEAT_DAX_NEVER; + break; + case XFS_DAX_NEVER: + mp->m_features |= XFS_FEAT_DAX_NEVER; + mp->m_features &= ~XFS_FEAT_DAX_ALWAYS; + break; + } +} + +static const struct constant_table dax_param_enums[] = { + {"inode", XFS_DAX_INODE }, + {"always", XFS_DAX_ALWAYS }, + {"never", XFS_DAX_NEVER }, + {} +}; + +/* + * Table driven mount option parser. + */ +enum { + Op_deprecated, Opt_logbufs, Opt_logbsize, Opt_logdev, Opt_rtdev, + Opt_wsync, Opt_noalign, Opt_swalloc, Opt_sunit, Opt_swidth, Opt_nouuid, + Opt_grpid, Opt_nogrpid, Opt_bsdgroups, Opt_sysvgroups, + Opt_allocsize, Opt_norecovery, Opt_inode64, Opt_inode32, + Opt_largeio, Opt_nolargeio, + Opt_filestreams, Opt_quota, Opt_noquota, Opt_usrquota, Opt_grpquota, + Opt_prjquota, Opt_uquota, Opt_gquota, Opt_pquota, + Opt_uqnoenforce, Opt_gqnoenforce, Opt_pqnoenforce, Opt_qnoenforce, + Opt_discard, Opt_nodiscard, Opt_dax, Opt_dax_enum, Opt_max_open_zones, + Opt_lifetime, Opt_nolifetime, Opt_max_atomic_write, Opt_errortag, +}; + +#define fsparam_dead(NAME) \ + __fsparam(NULL, (NAME), Op_deprecated, fs_param_deprecated, NULL) + +static const struct fs_parameter_spec xfs_fs_parameters[] = { + /* + * These mount options were supposed to be deprecated in September 2025 + * but the deprecation warning was buggy, so not all users were + * notified. The deprecation is now obnoxiously loud and postponed to + * September 2030. + */ + fsparam_dead("attr2"), + fsparam_dead("noattr2"), + fsparam_dead("ikeep"), + fsparam_dead("noikeep"), + + fsparam_u32("logbufs", Opt_logbufs), + fsparam_string("logbsize", Opt_logbsize), + fsparam_string("logdev", Opt_logdev), + fsparam_string("rtdev", Opt_rtdev), + fsparam_flag("wsync", Opt_wsync), + fsparam_flag("noalign", Opt_noalign), + fsparam_flag("swalloc", Opt_swalloc), + fsparam_u32("sunit", Opt_sunit), + fsparam_u32("swidth", Opt_swidth), + fsparam_flag("nouuid", Opt_nouuid), + fsparam_flag("grpid", Opt_grpid), + fsparam_flag("nogrpid", Opt_nogrpid), + fsparam_flag("bsdgroups", Opt_bsdgroups), + fsparam_flag("sysvgroups", Opt_sysvgroups), + fsparam_string("allocsize", Opt_allocsize), + fsparam_flag("norecovery", Opt_norecovery), + fsparam_flag("inode64", Opt_inode64), + fsparam_flag("inode32", Opt_inode32), + fsparam_flag("largeio", Opt_largeio), + fsparam_flag("nolargeio", Opt_nolargeio), + fsparam_flag("filestreams", Opt_filestreams), + fsparam_flag("quota", Opt_quota), + fsparam_flag("noquota", Opt_noquota), + fsparam_flag("usrquota", Opt_usrquota), + fsparam_flag("grpquota", Opt_grpquota), + fsparam_flag("prjquota", Opt_prjquota), + fsparam_flag("uquota", Opt_uquota), + fsparam_flag("gquota", Opt_gquota), + fsparam_flag("pquota", Opt_pquota), + fsparam_flag("uqnoenforce", Opt_uqnoenforce), + fsparam_flag("gqnoenforce", Opt_gqnoenforce), + fsparam_flag("pqnoenforce", Opt_pqnoenforce), + fsparam_flag("qnoenforce", Opt_qnoenforce), + fsparam_flag("discard", Opt_discard), + fsparam_flag("nodiscard", Opt_nodiscard), + fsparam_flag("dax", Opt_dax), + fsparam_enum("dax", Opt_dax_enum, dax_param_enums), + fsparam_u32("max_open_zones", Opt_max_open_zones), + fsparam_flag("lifetime", Opt_lifetime), + fsparam_flag("nolifetime", Opt_nolifetime), + fsparam_string("max_atomic_write", Opt_max_atomic_write), + fsparam_string("errortag", Opt_errortag), + {} +}; + +struct proc_xfs_info { + uint64_t flag; + char *str; +}; + +static int +xfs_fs_show_options( + struct seq_file *m, + struct dentry *root) +{ + static struct proc_xfs_info xfs_info_set[] = { + /* the few simple ones we can get from the mount struct */ + { XFS_FEAT_WSYNC, ",wsync" }, + { XFS_FEAT_NOALIGN, ",noalign" }, + { XFS_FEAT_SWALLOC, ",swalloc" }, + { XFS_FEAT_NOUUID, ",nouuid" }, + { XFS_FEAT_NORECOVERY, ",norecovery" }, + { XFS_FEAT_FILESTREAMS, ",filestreams" }, + { XFS_FEAT_GRPID, ",grpid" }, + { XFS_FEAT_DISCARD, ",discard" }, + { XFS_FEAT_LARGE_IOSIZE, ",largeio" }, + { XFS_FEAT_DAX_ALWAYS, ",dax=always" }, + { XFS_FEAT_DAX_NEVER, ",dax=never" }, + { XFS_FEAT_NOLIFETIME, ",nolifetime" }, + { 0, NULL } + }; + struct xfs_mount *mp = XFS_M(root->d_sb); + struct proc_xfs_info *xfs_infop; + + for (xfs_infop = xfs_info_set; xfs_infop->flag; xfs_infop++) { + if (mp->m_features & xfs_infop->flag) + seq_puts(m, xfs_infop->str); + } + + seq_printf(m, ",inode%d", xfs_has_small_inums(mp) ? 32 : 64); + + if (xfs_has_allocsize(mp)) + seq_printf(m, ",allocsize=%dk", + (1 << mp->m_allocsize_log) >> 10); + + if (mp->m_logbufs > 0) + seq_printf(m, ",logbufs=%d", mp->m_logbufs); + if (mp->m_logbsize > 0) + seq_printf(m, ",logbsize=%dk", mp->m_logbsize >> 10); + + if (mp->m_logname) + seq_show_option(m, "logdev", mp->m_logname); + if (mp->m_rtname) + seq_show_option(m, "rtdev", mp->m_rtname); + + if (mp->m_dalign > 0) + seq_printf(m, ",sunit=%d", + (int)XFS_FSB_TO_BB(mp, mp->m_dalign)); + if (mp->m_swidth > 0) + seq_printf(m, ",swidth=%d", + (int)XFS_FSB_TO_BB(mp, mp->m_swidth)); + + if (mp->m_qflags & XFS_UQUOTA_ENFD) + seq_puts(m, ",usrquota"); + else if (mp->m_qflags & XFS_UQUOTA_ACCT) + seq_puts(m, ",uqnoenforce"); + + if (mp->m_qflags & XFS_PQUOTA_ENFD) + seq_puts(m, ",prjquota"); + else if (mp->m_qflags & XFS_PQUOTA_ACCT) + seq_puts(m, ",pqnoenforce"); + + if (mp->m_qflags & XFS_GQUOTA_ENFD) + seq_puts(m, ",grpquota"); + else if (mp->m_qflags & XFS_GQUOTA_ACCT) + seq_puts(m, ",gqnoenforce"); + + if (!(mp->m_qflags & XFS_ALL_QUOTA_ACCT)) + seq_puts(m, ",noquota"); + + if (mp->m_max_open_zones) + seq_printf(m, ",max_open_zones=%u", mp->m_max_open_zones); + if (mp->m_awu_max_bytes) + seq_printf(m, ",max_atomic_write=%lluk", + mp->m_awu_max_bytes >> 10); + + return 0; +} + +static bool +xfs_set_inode_alloc_perag( + struct xfs_perag *pag, + xfs_ino_t ino, + xfs_agnumber_t max_metadata) +{ + if (!xfs_is_inode32(pag_mount(pag))) { + set_bit(XFS_AGSTATE_ALLOWS_INODES, &pag->pag_opstate); + clear_bit(XFS_AGSTATE_PREFERS_METADATA, &pag->pag_opstate); + return false; + } + + if (ino > XFS_MAXINUMBER_32) { + clear_bit(XFS_AGSTATE_ALLOWS_INODES, &pag->pag_opstate); + clear_bit(XFS_AGSTATE_PREFERS_METADATA, &pag->pag_opstate); + return false; + } + + set_bit(XFS_AGSTATE_ALLOWS_INODES, &pag->pag_opstate); + if (pag_agno(pag) < max_metadata) + set_bit(XFS_AGSTATE_PREFERS_METADATA, &pag->pag_opstate); + else + clear_bit(XFS_AGSTATE_PREFERS_METADATA, &pag->pag_opstate); + return true; +} + +/* + * Set parameters for inode allocation heuristics, taking into account + * filesystem size and inode32/inode64 mount options; i.e. specifically + * whether or not XFS_FEAT_SMALL_INUMS is set. + * + * Inode allocation patterns are altered only if inode32 is requested + * (XFS_FEAT_SMALL_INUMS), and the filesystem is sufficiently large. + * If altered, XFS_OPSTATE_INODE32 is set as well. + * + * An agcount independent of that in the mount structure is provided + * because in the growfs case, mp->m_sb.sb_agcount is not yet updated + * to the potentially higher ag count. + * + * Returns the maximum AG index which may contain inodes. + */ +xfs_agnumber_t +xfs_set_inode_alloc( + struct xfs_mount *mp, + xfs_agnumber_t agcount) +{ + xfs_agnumber_t index; + xfs_agnumber_t maxagi = 0; + xfs_sb_t *sbp = &mp->m_sb; + xfs_agnumber_t max_metadata; + xfs_agino_t agino; + xfs_ino_t ino; + + /* + * Calculate how much should be reserved for inodes to meet + * the max inode percentage. Used only for inode32. + */ + if (M_IGEO(mp)->maxicount) { + uint64_t icount; + + icount = sbp->sb_dblocks * sbp->sb_imax_pct; + do_div(icount, 100); + icount += sbp->sb_agblocks - 1; + do_div(icount, sbp->sb_agblocks); + max_metadata = icount; + } else { + max_metadata = agcount; + } + + /* Get the last possible inode in the filesystem */ + agino = XFS_AGB_TO_AGINO(mp, sbp->sb_agblocks - 1); + ino = XFS_AGINO_TO_INO(mp, agcount - 1, agino); + + /* + * If user asked for no more than 32-bit inodes, and the fs is + * sufficiently large, set XFS_OPSTATE_INODE32 if we must alter + * the allocator to accommodate the request. + */ + if (xfs_has_small_inums(mp) && ino > XFS_MAXINUMBER_32) + xfs_set_inode32(mp); + else + xfs_clear_inode32(mp); + + for (index = 0; index < agcount; index++) { + struct xfs_perag *pag; + + ino = XFS_AGINO_TO_INO(mp, index, agino); + + pag = xfs_perag_get(mp, index); + if (xfs_set_inode_alloc_perag(pag, ino, max_metadata)) + maxagi++; + xfs_perag_put(pag); + } + + return xfs_is_inode32(mp) ? maxagi : agcount; +} + +static int +xfs_setup_dax_always( + struct xfs_mount *mp) +{ + if (!mp->m_ddev_targp->bt_daxdev && + (!mp->m_rtdev_targp || !mp->m_rtdev_targp->bt_daxdev)) { + xfs_alert(mp, + "DAX unsupported by block device. Turning off DAX."); + goto disable_dax; + } + + if (mp->m_super->s_blocksize != PAGE_SIZE) { + xfs_alert(mp, + "DAX not supported for blocksize. Turning off DAX."); + goto disable_dax; + } + + if (xfs_has_reflink(mp) && + bdev_is_partition(mp->m_ddev_targp->bt_bdev)) { + xfs_alert(mp, + "DAX and reflink cannot work with multi-partitions!"); + return -EINVAL; + } + + return 0; + +disable_dax: + xfs_mount_set_dax_mode(mp, XFS_DAX_NEVER); + return 0; +} + +STATIC int +xfs_blkdev_get( + xfs_mount_t *mp, + const char *name, + struct file **bdev_filep) +{ + int error = 0; + blk_mode_t mode; + + mode = sb_open_mode(mp->m_super->s_flags); + *bdev_filep = fs_bdev_file_open_by_path(name, mode, + mp->m_super, mp->m_super); + if (IS_ERR(*bdev_filep)) { + error = PTR_ERR(*bdev_filep); + *bdev_filep = NULL; + xfs_warn(mp, "Invalid device [%s], error=%d", name, error); + } + + return error; +} + +STATIC void +xfs_shutdown_devices( + struct xfs_mount *mp) +{ + /* + * Udev is triggered whenever anyone closes a block device or unmounts + * a file systemm on a block device. + * The default udev rules invoke blkid to read the fs super and create + * symlinks to the bdev under /dev/disk. For this, it uses buffered + * reads through the page cache. + * + * xfs_db also uses buffered reads to examine metadata. There is no + * coordination between xfs_db and udev, which means that they can run + * concurrently. Note there is no coordination between the kernel and + * blkid either. + * + * On a system with 64k pages, the page cache can cache the superblock + * and the root inode (and hence the root directory) with the same 64k + * page. If udev spawns blkid after the mkfs and the system is busy + * enough that it is still running when xfs_db starts up, they'll both + * read from the same page in the pagecache. + * + * The unmount writes updated inode metadata to disk directly. The XFS + * buffer cache does not use the bdev pagecache, so it needs to + * invalidate that pagecache on unmount. If the above scenario occurs, + * the pagecache no longer reflects what's on disk, xfs_db reads the + * stale metadata, and fails to find /a. Most of the time this succeeds + * because closing a bdev invalidates the page cache, but when processes + * race, everyone loses. + */ + if (mp->m_logdev_targp && mp->m_logdev_targp != mp->m_ddev_targp) { + blkdev_issue_flush(mp->m_logdev_targp->bt_bdev); + invalidate_bdev(mp->m_logdev_targp->bt_bdev); + } + if (mp->m_rtdev_targp) { + blkdev_issue_flush(mp->m_rtdev_targp->bt_bdev); + invalidate_bdev(mp->m_rtdev_targp->bt_bdev); + } + blkdev_issue_flush(mp->m_ddev_targp->bt_bdev); + invalidate_bdev(mp->m_ddev_targp->bt_bdev); +} + +/* + * The file system configurations are: + * (1) device (partition) with data and internal log + * (2) logical volume with data and log subvolumes. + * (3) logical volume with data, log, and realtime subvolumes. + * + * We only have to handle opening the log and realtime volumes here if + * they are present. The data subvolume has already been opened by + * get_sb_bdev() and is stored in sb->s_bdev. + */ +STATIC int +xfs_open_devices( + struct xfs_mount *mp) +{ + struct super_block *sb = mp->m_super; + struct block_device *ddev = sb->s_bdev; + struct file *logdev_file = NULL, *rtdev_file = NULL; + int error; + + /* + * Open real time and log devices - order is important. + */ + if (mp->m_logname) { + error = xfs_blkdev_get(mp, mp->m_logname, &logdev_file); + if (error) + return error; + } + + if (mp->m_rtname) { + error = xfs_blkdev_get(mp, mp->m_rtname, &rtdev_file); + if (error) + goto out_close_logdev; + + if (file_bdev(rtdev_file) == ddev || + (logdev_file && + file_bdev(rtdev_file) == file_bdev(logdev_file))) { + xfs_warn(mp, + "Cannot mount filesystem with identical rtdev and ddev/logdev."); + error = -EINVAL; + goto out_close_rtdev; + } + } + + /* + * Setup xfs_mount buffer target pointers + */ + mp->m_ddev_targp = xfs_alloc_buftarg(mp, sb->s_bdev_file); + if (IS_ERR(mp->m_ddev_targp)) { + error = PTR_ERR(mp->m_ddev_targp); + mp->m_ddev_targp = NULL; + goto out_close_rtdev; + } + + if (rtdev_file) { + mp->m_rtdev_targp = xfs_alloc_buftarg(mp, rtdev_file); + if (IS_ERR(mp->m_rtdev_targp)) { + error = PTR_ERR(mp->m_rtdev_targp); + mp->m_rtdev_targp = NULL; + goto out_free_ddev_targ; + } + } + + if (logdev_file && file_bdev(logdev_file) != ddev) { + mp->m_logdev_targp = xfs_alloc_buftarg(mp, logdev_file); + if (IS_ERR(mp->m_logdev_targp)) { + error = PTR_ERR(mp->m_logdev_targp); + mp->m_logdev_targp = NULL; + goto out_free_rtdev_targ; + } + } else { + mp->m_logdev_targp = mp->m_ddev_targp; + /* Handle won't be used, drop it */ + if (logdev_file) + fs_bdev_file_release(logdev_file, mp->m_super); + } + + return 0; + + out_free_rtdev_targ: + if (mp->m_rtdev_targp) + xfs_free_buftarg(mp->m_rtdev_targp); + mp->m_rtdev_targp = NULL; + rtdev_file = NULL; /* released by xfs_free_buftarg() */ + out_free_ddev_targ: + xfs_free_buftarg(mp->m_ddev_targp); + mp->m_ddev_targp = NULL; + out_close_rtdev: + if (rtdev_file) + fs_bdev_file_release(rtdev_file, mp->m_super); + out_close_logdev: + if (logdev_file) + fs_bdev_file_release(logdev_file, mp->m_super); + return error; +} + +/* + * When using a RT device some or all data I/O is using the RT device, but + * the BDI is inherited from the main data device. When the underlying block + * device for the RT device has larger I/O sizes, the BDI settings might be + * incorrect, which is especially bad if the main device is a SSD and the + * RT device is a HDD, as the io_opt fixup in blk_apply_bdi_limits is missing + * for this case. + * + * Update the BDI values to the max of the data and RT device to cover our + * bases. + */ +static void +xfs_update_bdi_rahead( + struct xfs_mount *mp) +{ + struct backing_dev_info *rt_bdi = + mp->m_rtdev_targp->bt_bdev->bd_disk->bdi; + struct backing_dev_info *sb_bdi = mp->m_super->s_bdi; + + mp->m_old_io_pages = sb_bdi->io_pages; + mp->m_old_ra_pages = sb_bdi->ra_pages; + + sb_bdi->io_pages = mp->m_initial_io_pages = + max(sb_bdi->io_pages, rt_bdi->io_pages); + sb_bdi->ra_pages = mp->m_initial_ra_pages = + max(sb_bdi->ra_pages, rt_bdi->ra_pages); +} + +static void +xfs_restore_bdi_rahead( + struct xfs_mount *mp) +{ + struct backing_dev_info *sb_bdi = mp->m_super->s_bdi; + + if (sb_bdi->io_pages == mp->m_initial_io_pages) + sb_bdi->io_pages = mp->m_old_io_pages; + else + xfs_info(mp, "io_pages changed from %lu to %lu, not restoring.", + mp->m_initial_io_pages, sb_bdi->io_pages); + if (sb_bdi->ra_pages == mp->m_initial_ra_pages) + sb_bdi->ra_pages = mp->m_old_ra_pages; + else + xfs_info(mp, "ra_pages changed from %lu to %lu, not restoring.", + mp->m_initial_ra_pages, sb_bdi->ra_pages); +} + +/* + * Setup xfs_mount buffer target pointers based on superblock + */ +STATIC int +xfs_setup_devices( + struct xfs_mount *mp) +{ + int error; + + error = xfs_configure_buftarg(mp->m_ddev_targp, mp->m_sb.sb_sectsize, + mp->m_sb.sb_dblocks); + if (error) + return error; + + if (mp->m_logdev_targp && mp->m_logdev_targp != mp->m_ddev_targp) { + unsigned int log_sector_size = BBSIZE; + + if (xfs_has_sector(mp)) + log_sector_size = mp->m_sb.sb_logsectsize; + error = xfs_configure_buftarg(mp->m_logdev_targp, + log_sector_size, mp->m_sb.sb_logblocks); + if (error) + return error; + } + + if (mp->m_sb.sb_rtstart) { + if (mp->m_rtdev_targp) { + xfs_warn(mp, + "can't use internal and external rtdev at the same time"); + return -EINVAL; + } + mp->m_rtdev_targp = mp->m_ddev_targp; + } else if (mp->m_rtname) { + error = xfs_configure_buftarg(mp->m_rtdev_targp, + mp->m_sb.sb_sectsize, mp->m_sb.sb_rblocks); + if (error) + return error; + xfs_update_bdi_rahead(mp); + } + + return 0; +} + +STATIC int +xfs_init_mount_workqueues( + struct xfs_mount *mp) +{ + mp->m_buf_workqueue = alloc_workqueue("xfs-buf/%s", + XFS_WQFLAGS(WQ_FREEZABLE | WQ_MEM_RECLAIM | WQ_PERCPU), + 1, mp->m_super->s_id); + if (!mp->m_buf_workqueue) + goto out; + + mp->m_unwritten_workqueue = alloc_workqueue("xfs-conv/%s", + XFS_WQFLAGS(WQ_FREEZABLE | WQ_MEM_RECLAIM | WQ_PERCPU), + 0, mp->m_super->s_id); + if (!mp->m_unwritten_workqueue) + goto out_destroy_buf; + + mp->m_reclaim_workqueue = alloc_workqueue("xfs-reclaim/%s", + XFS_WQFLAGS(WQ_FREEZABLE | WQ_MEM_RECLAIM | WQ_PERCPU), + 0, mp->m_super->s_id); + if (!mp->m_reclaim_workqueue) + goto out_destroy_unwritten; + + mp->m_blockgc_wq = alloc_workqueue("xfs-blockgc/%s", + XFS_WQFLAGS(WQ_UNBOUND | WQ_FREEZABLE | WQ_MEM_RECLAIM), + 0, mp->m_super->s_id); + if (!mp->m_blockgc_wq) + goto out_destroy_reclaim; + + mp->m_inodegc_wq = alloc_workqueue("xfs-inodegc/%s", + XFS_WQFLAGS(WQ_FREEZABLE | WQ_MEM_RECLAIM | WQ_PERCPU), + 1, mp->m_super->s_id); + if (!mp->m_inodegc_wq) + goto out_destroy_blockgc; + + mp->m_sync_workqueue = alloc_workqueue("xfs-sync/%s", + XFS_WQFLAGS(WQ_FREEZABLE | WQ_PERCPU), 0, + mp->m_super->s_id); + if (!mp->m_sync_workqueue) + goto out_destroy_inodegc; + + return 0; + +out_destroy_inodegc: + destroy_workqueue(mp->m_inodegc_wq); +out_destroy_blockgc: + destroy_workqueue(mp->m_blockgc_wq); +out_destroy_reclaim: + destroy_workqueue(mp->m_reclaim_workqueue); +out_destroy_unwritten: + destroy_workqueue(mp->m_unwritten_workqueue); +out_destroy_buf: + destroy_workqueue(mp->m_buf_workqueue); +out: + return -ENOMEM; +} + +STATIC void +xfs_destroy_mount_workqueues( + struct xfs_mount *mp) +{ + destroy_workqueue(mp->m_sync_workqueue); + destroy_workqueue(mp->m_blockgc_wq); + destroy_workqueue(mp->m_inodegc_wq); + destroy_workqueue(mp->m_reclaim_workqueue); + destroy_workqueue(mp->m_unwritten_workqueue); + destroy_workqueue(mp->m_buf_workqueue); +} + +static void +xfs_flush_inodes_worker( + struct work_struct *work) +{ + struct xfs_mount *mp = container_of(work, struct xfs_mount, + m_flush_inodes_work); + struct super_block *sb = mp->m_super; + + if (down_read_trylock(&sb->s_umount)) { + sync_inodes_sb(sb); + up_read(&sb->s_umount); + } +} + +/* + * Flush all dirty data to disk. Must not be called while holding an XFS_ILOCK + * or a page lock. We use sync_inodes_sb() here to ensure we block while waiting + * for IO to complete so that we effectively throttle multiple callers to the + * rate at which IO is completing. + */ +void +xfs_flush_inodes( + struct xfs_mount *mp) +{ + /* + * If flush_work() returns true then that means we waited for a flush + * which was already in progress. Don't bother running another scan. + */ + if (flush_work(&mp->m_flush_inodes_work)) + return; + + queue_work(mp->m_sync_workqueue, &mp->m_flush_inodes_work); + flush_work(&mp->m_flush_inodes_work); +} + +/* Catch misguided souls that try to use this interface on XFS */ +STATIC struct inode * +xfs_fs_alloc_inode( + struct super_block *sb) +{ + BUG(); + return NULL; +} + +/* + * Now that the generic code is guaranteed not to be accessing + * the linux inode, we can inactivate and reclaim the inode. + */ +STATIC void +xfs_fs_destroy_inode( + struct inode *inode) +{ + struct xfs_inode *ip = XFS_I(inode); + + trace_xfs_destroy_inode(ip); + + ASSERT(!rwsem_is_locked(&inode->i_rwsem)); + XFS_STATS_INC(ip->i_mount, xs_inode_destroy); + XFS_STATS_INC(ip->i_mount, xs_inode_destroy2); + xfs_inode_mark_reclaimable(ip); +} + +/* + * Slab object creation initialisation for the XFS inode. + * This covers only the idempotent fields in the XFS inode; + * all other fields need to be initialised on allocation + * from the slab. This avoids the need to repeatedly initialise + * fields in the xfs inode that left in the initialise state + * when freeing the inode. + */ +STATIC void +xfs_fs_inode_init_once( + void *inode) +{ + struct xfs_inode *ip = inode; + + memset(ip, 0, sizeof(struct xfs_inode)); + + /* vfs inode */ + inode_init_once(VFS_I(ip)); + + /* xfs inode */ + atomic_set(&ip->i_pincount, 0); + spin_lock_init(&ip->i_flags_lock); + init_rwsem(&ip->i_lock); +} + +/* + * We do an unlocked check for XFS_IDONTCACHE here because we are already + * serialised against cache hits here via the inode->i_lock and igrab() in + * xfs_iget_cache_hit(). Hence a lookup that might clear this flag will not be + * racing with us, and it avoids needing to grab a spinlock here for every inode + * we drop the final reference on. + */ +STATIC int +xfs_fs_drop_inode( + struct inode *inode) +{ + struct xfs_inode *ip = XFS_I(inode); + + /* + * If this unlinked inode is in the middle of recovery, don't + * drop the inode just yet; log recovery will take care of + * that. See the comment for this inode flag. + */ + if (ip->i_flags & XFS_IRECOVERY) { + ASSERT(xlog_recovery_needed(ip->i_mount->m_log)); + return 0; + } + + return inode_generic_drop(inode); +} + +STATIC void +xfs_fs_evict_inode( + struct inode *inode) +{ + if (IS_DAX(inode)) + dax_break_layout_final(inode); + + truncate_inode_pages_final(&inode->i_data); + clear_inode(inode); + + if (IS_ENABLED(CONFIG_XFS_RT) && + S_ISREG(inode->i_mode) && inode->i_private) { + xfs_open_zone_put(inode->i_private); + inode->i_private = NULL; + } +} + +static void +xfs_mount_free( + struct xfs_mount *mp) +{ + if (mp->m_logdev_targp && mp->m_logdev_targp != mp->m_ddev_targp) + xfs_free_buftarg(mp->m_logdev_targp); + if (mp->m_rtdev_targp && mp->m_rtdev_targp != mp->m_ddev_targp) + xfs_free_buftarg(mp->m_rtdev_targp); + if (mp->m_ddev_targp) + xfs_free_buftarg(mp->m_ddev_targp); + + debugfs_remove(mp->m_debugfs); + kfree(mp->m_rtname); + kfree(mp->m_logname); +#ifdef DEBUG + kfree(mp->m_errortag); +#endif + kfree(mp); +} + +STATIC int +xfs_fs_sync_fs( + struct super_block *sb, + int wait) +{ + struct xfs_mount *mp = XFS_M(sb); + int error; + + trace_xfs_fs_sync_fs(mp, __return_address); + + /* + * Doing anything during the async pass would be counterproductive. + */ + if (!wait) + return 0; + + error = xfs_log_force(mp, XFS_LOG_SYNC); + if (error) + return error; + + /* + * If we are called with page faults frozen out, it means we are about + * to freeze the transaction subsystem. Take the opportunity to shut + * down inodegc because once SB_FREEZE_FS is set it's too late to + * prevent inactivation races with freeze. The fs doesn't get called + * again by the freezing process until after SB_FREEZE_FS has been set, + * so it's now or never. Same logic applies to speculative allocation + * garbage collection. + * + * We don't care if this is a normal syncfs call that does this or + * freeze that does this - we can run this multiple times without issue + * and we won't race with a restart because a restart can only occur + * when the state is either SB_FREEZE_FS or SB_FREEZE_COMPLETE. + */ + if (sb->s_writers.frozen == SB_FREEZE_PAGEFAULT) { + xfs_inodegc_stop(mp); + xfs_blockgc_stop(mp); + xfs_zone_gc_stop(mp); + } + + return 0; +} + +static xfs_extlen_t +xfs_internal_log_size( + struct xfs_mount *mp) +{ + if (!mp->m_sb.sb_logstart) + return 0; + return mp->m_sb.sb_logblocks; +} + +static void +xfs_statfs_data( + struct xfs_mount *mp, + struct kstatfs *st) +{ + int64_t fdblocks = + xfs_sum_freecounter(mp, XC_FREE_BLOCKS); + + /* make sure st->f_bfree does not underflow */ + st->f_bfree = max(0LL, + fdblocks - xfs_freecounter_unavailable(mp, XC_FREE_BLOCKS)); + + /* + * sb_dblocks can change during growfs, but nothing cares about reporting + * the old or new value during growfs. + */ + st->f_blocks = mp->m_sb.sb_dblocks - xfs_internal_log_size(mp); +} + +/* + * When stat(v)fs is called on a file with the realtime bit set or a directory + * with the rtinherit bit, report freespace information for the RT device + * instead of the main data device. + */ +static void +xfs_statfs_rt( + struct xfs_mount *mp, + struct kstatfs *st) +{ + st->f_bfree = xfs_rtbxlen_to_blen(mp, + xfs_sum_freecounter(mp, XC_FREE_RTEXTENTS)); + st->f_blocks = mp->m_sb.sb_rblocks - xfs_rtbxlen_to_blen(mp, + mp->m_free[XC_FREE_RTEXTENTS].res_total); +} + +static void +xfs_statfs_inodes( + struct xfs_mount *mp, + struct kstatfs *st) +{ + uint64_t icount = percpu_counter_sum(&mp->m_icount); + uint64_t ifree = percpu_counter_sum(&mp->m_ifree); + uint64_t fakeinos = XFS_FSB_TO_INO(mp, st->f_bfree); + + st->f_files = min(icount + fakeinos, (uint64_t)XFS_MAXINUMBER); + if (M_IGEO(mp)->maxicount) + st->f_files = min_t(typeof(st->f_files), st->f_files, + M_IGEO(mp)->maxicount); + + /* If sb_icount overshot maxicount, report actual allocation */ + st->f_files = max_t(typeof(st->f_files), st->f_files, + mp->m_sb.sb_icount); + + /* Make sure st->f_ffree does not underflow */ + st->f_ffree = max_t(int64_t, 0, st->f_files - (icount - ifree)); +} + +STATIC int +xfs_fs_statfs( + struct dentry *dentry, + struct kstatfs *st) +{ + struct xfs_mount *mp = XFS_M(dentry->d_sb); + struct xfs_inode *ip = XFS_I(d_inode(dentry)); + + /* + * Expedite background inodegc but don't wait. We do not want to block + * here waiting hours for a billion extent file to be truncated. + */ + xfs_inodegc_push(mp); + + st->f_type = XFS_SUPER_MAGIC; + st->f_namelen = MAXNAMELEN - 1; + st->f_bsize = mp->m_sb.sb_blocksize; + st->f_fsid = u64_to_fsid(huge_encode_dev(mp->m_ddev_targp->bt_dev)); + + xfs_statfs_data(mp, st); + xfs_statfs_inodes(mp, st); + + if (XFS_IS_REALTIME_MOUNT(mp) && + (ip->i_diflags & (XFS_DIFLAG_RTINHERIT | XFS_DIFLAG_REALTIME))) + xfs_statfs_rt(mp, st); + + if ((ip->i_diflags & XFS_DIFLAG_PROJINHERIT) && + ((mp->m_qflags & (XFS_PQUOTA_ACCT|XFS_PQUOTA_ENFD))) == + (XFS_PQUOTA_ACCT|XFS_PQUOTA_ENFD)) + xfs_qm_statvfs(ip, st); + + /* + * XFS does not distinguish between blocks available to privileged and + * unprivileged users. + */ + st->f_bavail = st->f_bfree; + return 0; +} + +STATIC void +xfs_save_resvblks( + struct xfs_mount *mp) +{ + enum xfs_free_counter i; + + for (i = 0; i < XC_FREE_NR; i++) { + mp->m_free[i].res_saved = mp->m_free[i].res_total; + xfs_reserve_blocks(mp, i, 0); + } +} + +STATIC void +xfs_restore_resvblks( + struct xfs_mount *mp) +{ + uint64_t resblks; + enum xfs_free_counter i; + + for (i = 0; i < XC_FREE_NR; i++) { + if (mp->m_free[i].res_saved) { + resblks = mp->m_free[i].res_saved; + mp->m_free[i].res_saved = 0; + } else + resblks = xfs_default_resblks(mp, i); + xfs_reserve_blocks(mp, i, resblks); + } +} + +/* + * Second stage of a freeze. The data is already frozen so we only + * need to take care of the metadata. Once that's done sync the superblock + * to the log to dirty it in case of a crash while frozen. This ensures that we + * will recover the unlinked inode lists on the next mount. + */ +STATIC int +xfs_fs_freeze( + struct super_block *sb) +{ + struct xfs_mount *mp = XFS_M(sb); + unsigned int flags; + int ret; + + /* + * The filesystem is now frozen far enough that memory reclaim + * cannot safely operate on the filesystem. Hence we need to + * set a GFP_NOFS context here to avoid recursion deadlocks. + */ + flags = memalloc_nofs_save(); + xfs_save_resvblks(mp); + ret = xfs_log_quiesce(mp); + memalloc_nofs_restore(flags); + + /* + * For read-write filesystems, we need to restart the inodegc on error + * because we stopped it at SB_FREEZE_PAGEFAULT level and a thaw is not + * going to be run to restart it now. We are at SB_FREEZE_FS level + * here, so we can restart safely without racing with a stop in + * xfs_fs_sync_fs(). + */ + if (ret && !xfs_is_readonly(mp)) { + xfs_blockgc_start(mp); + xfs_inodegc_start(mp); + xfs_zone_gc_start(mp); + } + + return ret; +} + +STATIC int +xfs_fs_unfreeze( + struct super_block *sb) +{ + struct xfs_mount *mp = XFS_M(sb); + + xfs_restore_resvblks(mp); + xfs_log_work_queue(mp); + + /* + * Don't reactivate the inodegc worker on a readonly filesystem because + * inodes are sent directly to reclaim. Don't reactivate the blockgc + * worker because there are no speculative preallocations on a readonly + * filesystem. + */ + if (!xfs_is_readonly(mp)) { + xfs_zone_gc_start(mp); + xfs_blockgc_start(mp); + xfs_inodegc_start(mp); + } + + return 0; +} + +/* + * This function fills in xfs_mount_t fields based on mount args. + * Note: the superblock _has_ now been read in. + */ +STATIC int +xfs_finish_flags( + struct xfs_mount *mp) +{ + /* Fail a mount where the logbuf is smaller than the log stripe */ + if (xfs_has_logv2(mp)) { + if (mp->m_logbsize <= 0 && + mp->m_sb.sb_logsunit > XLOG_BIG_RECORD_BSIZE) { + mp->m_logbsize = mp->m_sb.sb_logsunit; + } else if (mp->m_logbsize > 0 && + mp->m_logbsize < mp->m_sb.sb_logsunit) { + xfs_warn(mp, + "logbuf size must be greater than or equal to log stripe size"); + return -EINVAL; + } + } else { + /* Fail a mount if the logbuf is larger than 32K */ + if (mp->m_logbsize > XLOG_BIG_RECORD_BSIZE) { + xfs_warn(mp, + "logbuf size for version 1 logs must be 16K or 32K"); + return -EINVAL; + } + } + + /* + * prohibit r/w mounts of read-only filesystems + */ + if ((mp->m_sb.sb_flags & XFS_SBF_READONLY) && !xfs_is_readonly(mp)) { + xfs_warn(mp, + "cannot mount a read-only filesystem as read-write"); + return -EROFS; + } + + if ((mp->m_qflags & XFS_GQUOTA_ACCT) && + (mp->m_qflags & XFS_PQUOTA_ACCT) && + !xfs_has_pquotino(mp)) { + xfs_warn(mp, + "Super block does not support project and group quota together"); + return -EINVAL; + } + + if (!xfs_has_zoned(mp)) { + if (mp->m_max_open_zones) { + xfs_warn(mp, +"max_open_zones mount option only supported on zoned file systems."); + return -EINVAL; + } + if (mp->m_features & XFS_FEAT_NOLIFETIME) { + xfs_warn(mp, +"nolifetime mount option only supported on zoned file systems."); + return -EINVAL; + } + } + + return 0; +} + +static int +xfs_init_percpu_counters( + struct xfs_mount *mp) +{ + int error; + int i; + + error = percpu_counter_init(&mp->m_icount, 0, GFP_KERNEL); + if (error) + return -ENOMEM; + + error = percpu_counter_init(&mp->m_ifree, 0, GFP_KERNEL); + if (error) + goto free_icount; + + error = percpu_counter_init(&mp->m_delalloc_blks, 0, GFP_KERNEL); + if (error) + goto free_ifree; + + error = percpu_counter_init(&mp->m_delalloc_rtextents, 0, GFP_KERNEL); + if (error) + goto free_delalloc; + + for (i = 0; i < XC_FREE_NR; i++) { + error = percpu_counter_init(&mp->m_free[i].count, 0, + GFP_KERNEL); + if (error) + goto free_freecounters; + } + + return 0; + +free_freecounters: + while (--i >= 0) + percpu_counter_destroy(&mp->m_free[i].count); + percpu_counter_destroy(&mp->m_delalloc_rtextents); +free_delalloc: + percpu_counter_destroy(&mp->m_delalloc_blks); +free_ifree: + percpu_counter_destroy(&mp->m_ifree); +free_icount: + percpu_counter_destroy(&mp->m_icount); + return -ENOMEM; +} + +void +xfs_reinit_percpu_counters( + struct xfs_mount *mp) +{ + percpu_counter_set(&mp->m_icount, mp->m_sb.sb_icount); + percpu_counter_set(&mp->m_ifree, mp->m_sb.sb_ifree); + xfs_set_freecounter(mp, XC_FREE_BLOCKS, mp->m_sb.sb_fdblocks); + if (!xfs_has_zoned(mp)) + xfs_set_freecounter(mp, XC_FREE_RTEXTENTS, + mp->m_sb.sb_frextents); +} + +static void +xfs_destroy_percpu_counters( + struct xfs_mount *mp) +{ + enum xfs_free_counter i; + + for (i = 0; i < XC_FREE_NR; i++) + percpu_counter_destroy(&mp->m_free[i].count); + percpu_counter_destroy(&mp->m_icount); + percpu_counter_destroy(&mp->m_ifree); + ASSERT(xfs_is_shutdown(mp) || + percpu_counter_sum(&mp->m_delalloc_rtextents) == 0); + percpu_counter_destroy(&mp->m_delalloc_rtextents); + ASSERT(xfs_is_shutdown(mp) || + percpu_counter_sum(&mp->m_delalloc_blks) == 0); + percpu_counter_destroy(&mp->m_delalloc_blks); +} + +static int +xfs_inodegc_init_percpu( + struct xfs_mount *mp) +{ + struct xfs_inodegc *gc; + int cpu; + + mp->m_inodegc = alloc_percpu(struct xfs_inodegc); + if (!mp->m_inodegc) + return -ENOMEM; + + for_each_possible_cpu(cpu) { + gc = per_cpu_ptr(mp->m_inodegc, cpu); + gc->cpu = cpu; + gc->mp = mp; + init_llist_head(&gc->list); + gc->items = 0; + gc->error = 0; + INIT_DELAYED_WORK(&gc->work, xfs_inodegc_worker); + } + return 0; +} + +static void +xfs_inodegc_free_percpu( + struct xfs_mount *mp) +{ + if (!mp->m_inodegc) + return; + free_percpu(mp->m_inodegc); +} + +static void +xfs_fs_put_super( + struct super_block *sb) +{ + struct xfs_mount *mp = XFS_M(sb); + + xfs_notice(mp, "Unmounting Filesystem %pU", &mp->m_sb.sb_uuid); + xfs_filestream_unmount(mp); + xfs_unmountfs(mp); + + xfs_rtmount_freesb(mp); + xfs_freesb(mp); + xchk_mount_stats_free(mp); + free_percpu(mp->m_stats.xs_stats); + xfs_inodegc_free_percpu(mp); + xfs_destroy_percpu_counters(mp); + xfs_destroy_mount_workqueues(mp); + xfs_shutdown_devices(mp); +} + +static long +xfs_fs_nr_cached_objects( + struct super_block *sb, + struct shrink_control *sc) +{ + /* Paranoia: catch incorrect calls during mount setup or teardown */ + if (WARN_ON_ONCE(!sb->s_fs_info)) + return 0; + return xfs_reclaim_inodes_count(XFS_M(sb)); +} + +static long +xfs_fs_free_cached_objects( + struct super_block *sb, + struct shrink_control *sc) +{ + return xfs_reclaim_inodes_nr(XFS_M(sb), sc->nr_to_scan); +} + +static void +xfs_fs_shutdown( + struct super_block *sb) +{ + xfs_force_shutdown(XFS_M(sb), SHUTDOWN_DEVICE_REMOVED); +} + +static int +xfs_fs_show_stats( + struct seq_file *m, + struct dentry *root) +{ + struct xfs_mount *mp = XFS_M(root->d_sb); + + if (xfs_has_zoned(mp) && IS_ENABLED(CONFIG_XFS_RT)) + xfs_zoned_show_stats(m, mp); + return 0; +} + +static void +xfs_fs_report_error( + const struct fserror_event *event) +{ + /* healthmon already knows about non-inode and metadata errors */ + if (event->inode && event->type != FSERR_METADATA) + xfs_healthmon_report_file_ioerror(XFS_I(event->inode), event); +} + +static const struct super_operations xfs_super_operations = { + .alloc_inode = xfs_fs_alloc_inode, + .destroy_inode = xfs_fs_destroy_inode, + .drop_inode = xfs_fs_drop_inode, + .evict_inode = xfs_fs_evict_inode, + .put_super = xfs_fs_put_super, + .sync_fs = xfs_fs_sync_fs, + .freeze_fs = xfs_fs_freeze, + .unfreeze_fs = xfs_fs_unfreeze, + .statfs = xfs_fs_statfs, + .show_options = xfs_fs_show_options, + .nr_cached_objects = xfs_fs_nr_cached_objects, + .free_cached_objects = xfs_fs_free_cached_objects, + .shutdown = xfs_fs_shutdown, + .show_stats = xfs_fs_show_stats, + .report_error = xfs_fs_report_error, +}; + +static int +suffix_kstrtoint( + const char *s, + unsigned int base, + int *res) +{ + int last, shift_left_factor = 0, _res; + char *value; + int ret = 0; + + value = kstrdup(s, GFP_KERNEL); + if (!value) + return -ENOMEM; + + last = strlen(value) - 1; + if (value[last] == 'K' || value[last] == 'k') { + shift_left_factor = 10; + value[last] = '\0'; + } + if (value[last] == 'M' || value[last] == 'm') { + shift_left_factor = 20; + value[last] = '\0'; + } + if (value[last] == 'G' || value[last] == 'g') { + shift_left_factor = 30; + value[last] = '\0'; + } + + if (kstrtoint(value, base, &_res)) + ret = -EINVAL; + kfree(value); + *res = _res << shift_left_factor; + return ret; +} + +static int +suffix_kstrtoull( + const char *s, + unsigned int base, + unsigned long long *res) +{ + int last, shift_left_factor = 0; + unsigned long long _res; + char *value; + int ret = 0; + + value = kstrdup(s, GFP_KERNEL); + if (!value) + return -ENOMEM; + + last = strlen(value) - 1; + if (value[last] == 'K' || value[last] == 'k') { + shift_left_factor = 10; + value[last] = '\0'; + } + if (value[last] == 'M' || value[last] == 'm') { + shift_left_factor = 20; + value[last] = '\0'; + } + if (value[last] == 'G' || value[last] == 'g') { + shift_left_factor = 30; + value[last] = '\0'; + } + + if (kstrtoull(value, base, &_res)) + ret = -EINVAL; + kfree(value); + *res = _res << shift_left_factor; + return ret; +} + +static inline void +xfs_fs_warn_deprecated( + struct fs_context *fc, + struct fs_parameter *param) +{ + /* + * Always warn about someone passing in a deprecated mount option. + * Previously we wouldn't print the warning if we were reconfiguring + * and current mount point already had the flag set, but that was not + * the right thing to do. + * + * Many distributions mount the root filesystem with no options in the + * initramfs and rely on mount -a to remount the root fs with the + * options in fstab. However, the old behavior meant that there would + * never be a warning about deprecated mount options for the root fs in + * /etc/fstab. On a single-fs system, that means no warning at all. + * + * Compounding this problem are distribution scripts that copy + * /proc/mounts to fstab, which means that we can't remove mount + * options unless we're 100% sure they have only ever been advertised + * in /proc/mounts in response to explicitly provided mount options. + */ + xfs_warn(fc->s_fs_info, "%s mount option is deprecated.", param->key); +} + +/* + * Set mount state from a mount option. + * + * NOTE: mp->m_super is NULL here! + */ +static int +xfs_fs_parse_param( + struct fs_context *fc, + struct fs_parameter *param) +{ + struct xfs_mount *parsing_mp = fc->s_fs_info; + struct fs_parse_result result; + int size = 0; + int opt; + + BUILD_BUG_ON(XFS_QFLAGS_MNTOPTS & XFS_MOUNT_QUOTA_ALL); + + opt = fs_parse(fc, xfs_fs_parameters, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Op_deprecated: + xfs_fs_warn_deprecated(fc, param); + return 0; + case Opt_logbufs: + parsing_mp->m_logbufs = result.uint_32; + return 0; + case Opt_logbsize: + if (suffix_kstrtoint(param->string, 10, &parsing_mp->m_logbsize)) + return -EINVAL; + return 0; + case Opt_logdev: + kfree(parsing_mp->m_logname); + parsing_mp->m_logname = kstrdup(param->string, GFP_KERNEL); + if (!parsing_mp->m_logname) + return -ENOMEM; + return 0; + case Opt_rtdev: + kfree(parsing_mp->m_rtname); + parsing_mp->m_rtname = kstrdup(param->string, GFP_KERNEL); + if (!parsing_mp->m_rtname) + return -ENOMEM; + return 0; + case Opt_allocsize: + if (suffix_kstrtoint(param->string, 10, &size)) + return -EINVAL; + parsing_mp->m_allocsize_log = ffs(size) - 1; + parsing_mp->m_features |= XFS_FEAT_ALLOCSIZE; + return 0; + case Opt_grpid: + case Opt_bsdgroups: + parsing_mp->m_features |= XFS_FEAT_GRPID; + return 0; + case Opt_nogrpid: + case Opt_sysvgroups: + parsing_mp->m_features &= ~XFS_FEAT_GRPID; + return 0; + case Opt_wsync: + parsing_mp->m_features |= XFS_FEAT_WSYNC; + return 0; + case Opt_norecovery: + parsing_mp->m_features |= XFS_FEAT_NORECOVERY; + return 0; + case Opt_noalign: + parsing_mp->m_features |= XFS_FEAT_NOALIGN; + return 0; + case Opt_swalloc: + parsing_mp->m_features |= XFS_FEAT_SWALLOC; + return 0; + case Opt_sunit: + parsing_mp->m_dalign = result.uint_32; + return 0; + case Opt_swidth: + parsing_mp->m_swidth = result.uint_32; + return 0; + case Opt_inode32: + parsing_mp->m_features |= XFS_FEAT_SMALL_INUMS; + return 0; + case Opt_inode64: + parsing_mp->m_features &= ~XFS_FEAT_SMALL_INUMS; + return 0; + case Opt_nouuid: + parsing_mp->m_features |= XFS_FEAT_NOUUID; + return 0; + case Opt_largeio: + parsing_mp->m_features |= XFS_FEAT_LARGE_IOSIZE; + return 0; + case Opt_nolargeio: + parsing_mp->m_features &= ~XFS_FEAT_LARGE_IOSIZE; + return 0; + case Opt_filestreams: + parsing_mp->m_features |= XFS_FEAT_FILESTREAMS; + return 0; + case Opt_noquota: + parsing_mp->m_qflags &= ~XFS_ALL_QUOTA_ACCT; + parsing_mp->m_qflags &= ~XFS_ALL_QUOTA_ENFD; + parsing_mp->m_qflags |= XFS_QFLAGS_MNTOPTS; + return 0; + case Opt_quota: + case Opt_uquota: + case Opt_usrquota: + parsing_mp->m_qflags |= (XFS_UQUOTA_ACCT | XFS_UQUOTA_ENFD); + parsing_mp->m_qflags |= XFS_QFLAGS_MNTOPTS; + return 0; + case Opt_qnoenforce: + case Opt_uqnoenforce: + parsing_mp->m_qflags |= XFS_UQUOTA_ACCT; + parsing_mp->m_qflags &= ~XFS_UQUOTA_ENFD; + parsing_mp->m_qflags |= XFS_QFLAGS_MNTOPTS; + return 0; + case Opt_pquota: + case Opt_prjquota: + parsing_mp->m_qflags |= (XFS_PQUOTA_ACCT | XFS_PQUOTA_ENFD); + parsing_mp->m_qflags |= XFS_QFLAGS_MNTOPTS; + return 0; + case Opt_pqnoenforce: + parsing_mp->m_qflags |= XFS_PQUOTA_ACCT; + parsing_mp->m_qflags &= ~XFS_PQUOTA_ENFD; + parsing_mp->m_qflags |= XFS_QFLAGS_MNTOPTS; + return 0; + case Opt_gquota: + case Opt_grpquota: + parsing_mp->m_qflags |= (XFS_GQUOTA_ACCT | XFS_GQUOTA_ENFD); + parsing_mp->m_qflags |= XFS_QFLAGS_MNTOPTS; + return 0; + case Opt_gqnoenforce: + parsing_mp->m_qflags |= XFS_GQUOTA_ACCT; + parsing_mp->m_qflags &= ~XFS_GQUOTA_ENFD; + parsing_mp->m_qflags |= XFS_QFLAGS_MNTOPTS; + return 0; + case Opt_discard: + parsing_mp->m_features |= XFS_FEAT_DISCARD; + return 0; + case Opt_nodiscard: + parsing_mp->m_features &= ~XFS_FEAT_DISCARD; + return 0; +#ifdef CONFIG_FS_DAX + case Opt_dax: + xfs_mount_set_dax_mode(parsing_mp, XFS_DAX_ALWAYS); + return 0; + case Opt_dax_enum: + xfs_mount_set_dax_mode(parsing_mp, result.uint_32); + return 0; +#endif + case Opt_max_open_zones: + parsing_mp->m_max_open_zones = result.uint_32; + return 0; + case Opt_lifetime: + parsing_mp->m_features &= ~XFS_FEAT_NOLIFETIME; + return 0; + case Opt_nolifetime: + parsing_mp->m_features |= XFS_FEAT_NOLIFETIME; + return 0; + case Opt_max_atomic_write: + if (suffix_kstrtoull(param->string, 10, + &parsing_mp->m_awu_max_bytes)) { + xfs_warn(parsing_mp, + "max atomic write size must be positive integer"); + return -EINVAL; + } + return 0; + case Opt_errortag: + return xfs_errortag_add_name(parsing_mp, param->string); + default: + xfs_warn(parsing_mp, "unknown mount option [%s].", param->key); + return -EINVAL; + } + + return 0; +} + +static int +xfs_fs_validate_params( + struct xfs_mount *mp) +{ + /* No recovery flag requires a read-only mount */ + if (xfs_has_norecovery(mp) && !xfs_is_readonly(mp)) { + xfs_warn(mp, "no-recovery mounts must be read-only."); + return -EINVAL; + } + + if (xfs_has_noalign(mp) && (mp->m_dalign || mp->m_swidth)) { + xfs_warn(mp, + "sunit and swidth options incompatible with the noalign option"); + return -EINVAL; + } + + if (!IS_ENABLED(CONFIG_XFS_QUOTA) && + (mp->m_qflags & ~XFS_QFLAGS_MNTOPTS)) { + xfs_warn(mp, "quota support not available in this kernel."); + return -EINVAL; + } + + if ((mp->m_dalign && !mp->m_swidth) || + (!mp->m_dalign && mp->m_swidth)) { + xfs_warn(mp, "sunit and swidth must be specified together"); + return -EINVAL; + } + + if (mp->m_dalign && (mp->m_swidth % mp->m_dalign != 0)) { + xfs_warn(mp, + "stripe width (%d) must be a multiple of the stripe unit (%d)", + mp->m_swidth, mp->m_dalign); + return -EINVAL; + } + + if (mp->m_logbufs != -1 && + mp->m_logbufs != 0 && + (mp->m_logbufs < XLOG_MIN_ICLOGS || + mp->m_logbufs > XLOG_MAX_ICLOGS)) { + xfs_warn(mp, "invalid logbufs value: %d [not %d-%d]", + mp->m_logbufs, XLOG_MIN_ICLOGS, XLOG_MAX_ICLOGS); + return -EINVAL; + } + + if (mp->m_logbsize != -1 && + mp->m_logbsize != 0 && + (mp->m_logbsize < XLOG_MIN_RECORD_BSIZE || + mp->m_logbsize > XLOG_MAX_RECORD_BSIZE || + !is_power_of_2(mp->m_logbsize))) { + xfs_warn(mp, + "invalid logbufsize: %d [not 16k,32k,64k,128k or 256k]", + mp->m_logbsize); + return -EINVAL; + } + + if (xfs_has_allocsize(mp) && + (mp->m_allocsize_log > XFS_MAX_IO_LOG || + mp->m_allocsize_log < XFS_MIN_IO_LOG)) { + xfs_warn(mp, "invalid log iosize: %d [not %d-%d]", + mp->m_allocsize_log, XFS_MIN_IO_LOG, XFS_MAX_IO_LOG); + return -EINVAL; + } + + return 0; +} + +struct dentry * +xfs_debugfs_mkdir( + const char *name, + struct dentry *parent) +{ + struct dentry *child; + + /* Apparently we're expected to ignore error returns?? */ + child = debugfs_create_dir(name, parent); + if (IS_ERR(child)) + return NULL; + + return child; +} + +static int +xfs_fs_fill_super( + struct super_block *sb, + struct fs_context *fc) +{ + struct xfs_mount *mp = sb->s_fs_info; + struct inode *root; + int flags = 0, error; + + mp->m_super = sb; + + /* + * Copy VFS mount flags from the context now that all parameter parsing + * is guaranteed to have been completed by either the old mount API or + * the newer fsopen/fsconfig API. + */ + if (fc->sb_flags & SB_RDONLY) + xfs_set_readonly(mp); + if (fc->sb_flags & SB_DIRSYNC) + mp->m_features |= XFS_FEAT_DIRSYNC; + if (fc->sb_flags & SB_SYNCHRONOUS) + mp->m_features |= XFS_FEAT_WSYNC; + + error = xfs_fs_validate_params(mp); + if (error) + return error; + + if (!sb_min_blocksize(sb, BBSIZE)) { + xfs_err(mp, "unable to set blocksize"); + return -EINVAL; + } + sb->s_xattr = xfs_xattr_handlers; + sb->s_export_op = &xfs_export_operations; +#ifdef CONFIG_XFS_QUOTA + sb->s_qcop = &xfs_quotactl_operations; + sb->s_quota_types = QTYPE_MASK_USR | QTYPE_MASK_GRP | QTYPE_MASK_PRJ; +#endif + sb->s_op = &xfs_super_operations; + + /* + * Delay mount work if the debug hook is set. This is debug + * instrumention to coordinate simulation of xfs mount failures with + * VFS superblock operations + */ + if (xfs_globals.mount_delay) { + xfs_notice(mp, "Delaying mount for %d seconds.", + xfs_globals.mount_delay); + msleep(xfs_globals.mount_delay * 1000); + } + + if (fc->sb_flags & SB_SILENT) + flags |= XFS_MFSI_QUIET; + + error = xfs_open_devices(mp); + if (error) + return error; + + if (xfs_debugfs) { + mp->m_debugfs = xfs_debugfs_mkdir(mp->m_super->s_id, + xfs_debugfs); + } else { + mp->m_debugfs = NULL; + } + + error = xfs_init_mount_workqueues(mp); + if (error) + goto out_shutdown_devices; + + error = xfs_init_percpu_counters(mp); + if (error) + goto out_destroy_workqueues; + + error = xfs_inodegc_init_percpu(mp); + if (error) + goto out_destroy_counters; + + /* Allocate stats memory before we do operations that might use it */ + mp->m_stats.xs_stats = alloc_percpu(struct xfsstats); + if (!mp->m_stats.xs_stats) { + error = -ENOMEM; + goto out_destroy_inodegc; + } + + error = xchk_mount_stats_alloc(mp); + if (error) + goto out_free_stats; + + error = xfs_readsb(mp, flags); + if (error) + goto out_free_scrub_stats; + + error = xfs_finish_flags(mp); + if (error) + goto out_free_sb; + + error = xfs_setup_devices(mp); + if (error) + goto out_free_sb; + + /* + * V4 support is undergoing deprecation. + * + * Note: this has to use an open coded m_features check as xfs_has_crc + * always returns false for !CONFIG_XFS_SUPPORT_V4. + */ + if (!(mp->m_features & XFS_FEAT_CRC)) { + if (!IS_ENABLED(CONFIG_XFS_SUPPORT_V4)) { + xfs_warn(mp, + "Deprecated V4 format (crc=0) not supported by kernel."); + error = -EINVAL; + goto out_free_sb; + } + xfs_warn_once(mp, + "Deprecated V4 format (crc=0) will not be supported after September 2030."); + } + + /* ASCII case insensitivity is undergoing deprecation. */ + if (xfs_has_asciici(mp)) { +#ifdef CONFIG_XFS_SUPPORT_ASCII_CI + xfs_warn_once(mp, + "Deprecated ASCII case-insensitivity feature (ascii-ci=1) will not be supported after September 2030."); +#else + xfs_warn(mp, + "Deprecated ASCII case-insensitivity feature (ascii-ci=1) not supported by kernel."); + error = -EINVAL; + goto out_free_sb; +#endif + } + + /* + * Filesystem claims it needs repair, so refuse the mount unless + * norecovery is also specified, in which case the filesystem can + * be mounted with no risk of further damage. + */ + if (xfs_has_needsrepair(mp) && !xfs_has_norecovery(mp)) { + xfs_warn(mp, "Filesystem needs repair. Please run xfs_repair."); + error = -EFSCORRUPTED; + goto out_free_sb; + } + + /* + * Don't touch the filesystem if a user tool thinks it owns the primary + * superblock. mkfs doesn't clear the flag from secondary supers, so + * we don't check them at all. + */ + if (mp->m_sb.sb_inprogress) { + xfs_warn(mp, "Offline file system operation in progress!"); + error = -EFSCORRUPTED; + goto out_free_sb; + } + + if (mp->m_sb.sb_blocksize > PAGE_SIZE) { + size_t max_folio_size = mapping_max_folio_size_supported(); + + if (!xfs_has_crc(mp)) { + xfs_warn(mp, +"V4 Filesystem with blocksize %d bytes. Only pagesize (%ld) or less is supported.", + mp->m_sb.sb_blocksize, PAGE_SIZE); + error = -ENOSYS; + goto out_free_sb; + } + + if (mp->m_sb.sb_blocksize > max_folio_size) { + xfs_warn(mp, +"block size (%u bytes) not supported; Only block size (%zu) or less is supported", + mp->m_sb.sb_blocksize, max_folio_size); + error = -ENOSYS; + goto out_free_sb; + } + } + + /* Ensure this filesystem fits in the page cache limits */ + if (xfs_sb_validate_fsb_count(&mp->m_sb, mp->m_sb.sb_dblocks) || + xfs_sb_validate_fsb_count(&mp->m_sb, mp->m_sb.sb_rblocks)) { + xfs_warn(mp, + "file system too large to be mounted on this system."); + error = -EFBIG; + goto out_free_sb; + } + + /* + * XFS block mappings use 54 bits to store the logical block offset. + * This should suffice to handle the maximum file size that the VFS + * supports (currently 2^63 bytes on 64-bit and ULONG_MAX << PAGE_SHIFT + * bytes on 32-bit), but as XFS and VFS have gotten the s_maxbytes + * calculation wrong on 32-bit kernels in the past, we'll add a WARN_ON + * to check this assertion. + * + * Avoid integer overflow by comparing the maximum bmbt offset to the + * maximum pagecache offset in units of fs blocks. + */ + if (!xfs_verify_fileoff(mp, XFS_B_TO_FSBT(mp, MAX_LFS_FILESIZE))) { + xfs_warn(mp, +"MAX_LFS_FILESIZE block offset (%llu) exceeds extent map maximum (%llu)!", + XFS_B_TO_FSBT(mp, MAX_LFS_FILESIZE), + XFS_MAX_FILEOFF); + error = -EINVAL; + goto out_free_sb; + } + + error = xfs_rtmount_readsb(mp); + if (error) + goto out_free_sb; + + error = xfs_filestream_mount(mp); + if (error) + goto out_free_rtsb; + + /* + * we must configure the block size in the superblock before we run the + * full mount process as the mount process can lookup and cache inodes. + */ + sb->s_magic = XFS_SUPER_MAGIC; + sb->s_blocksize = mp->m_sb.sb_blocksize; + sb->s_blocksize_bits = ffs(sb->s_blocksize) - 1; + sb->s_maxbytes = MAX_LFS_FILESIZE; + sb->s_max_links = XFS_MAXLINK; + sb->s_time_gran = 1; + if (xfs_has_bigtime(mp)) { + sb->s_time_min = xfs_bigtime_to_unix(XFS_BIGTIME_TIME_MIN); + sb->s_time_max = xfs_bigtime_to_unix(XFS_BIGTIME_TIME_MAX); + } else { + sb->s_time_min = XFS_LEGACY_TIME_MIN; + sb->s_time_max = XFS_LEGACY_TIME_MAX; + } + trace_xfs_inode_timestamp_range(mp, sb->s_time_min, sb->s_time_max); + sb->s_iflags |= SB_I_CGROUPWB | SB_I_ALLOW_HSM; + + set_posix_acl_flag(sb); + + /* version 5 superblocks support inode version counters. */ + if (xfs_has_crc(mp)) + sb->s_flags |= SB_I_VERSION; + + if (xfs_has_dax_always(mp)) { + error = xfs_setup_dax_always(mp); + if (error) + goto out_filestream_unmount; + } + + if (xfs_has_discard(mp) && !bdev_max_discard_sectors(sb->s_bdev)) { + xfs_warn(mp, + "mounting with \"discard\" option, but the device does not support discard"); + mp->m_features &= ~XFS_FEAT_DISCARD; + } + + if (xfs_has_zoned(mp)) { + if (!xfs_has_metadir(mp)) { + xfs_alert(mp, + "metadir feature required for zoned realtime devices."); + error = -EINVAL; + goto out_filestream_unmount; + } + } + + if (xfs_has_reflink(mp)) { + if (xfs_has_realtime(mp) && + !xfs_reflink_supports_rextsize(mp, mp->m_sb.sb_rextsize)) { + xfs_alert(mp, + "reflink not compatible with realtime extent size %u!", + mp->m_sb.sb_rextsize); + error = -EINVAL; + goto out_filestream_unmount; + } + + if (xfs_has_zoned(mp)) { + xfs_alert(mp, + "reflink not compatible with zoned RT device!"); + error = -EINVAL; + goto out_filestream_unmount; + } + + if (xfs_globals.always_cow) { + xfs_info(mp, "using DEBUG-only always_cow mode."); + mp->m_always_cow = true; + } + } + + /* + * If no quota mount options were provided, maybe we'll try to pick + * up the quota accounting and enforcement flags from the ondisk sb. + */ + if (!(mp->m_qflags & XFS_QFLAGS_MNTOPTS)) + xfs_set_resuming_quotaon(mp); + mp->m_qflags &= ~XFS_QFLAGS_MNTOPTS; + + error = xfs_mountfs(mp); + if (error) + goto out_filestream_unmount; + + root = igrab(VFS_I(mp->m_rootip)); + if (!root) { + error = -ENOENT; + goto out_unmount; + } + sb->s_root = d_make_root(root); + if (!sb->s_root) { + error = -ENOMEM; + goto out_unmount; + } + + return 0; + + out_filestream_unmount: + xfs_filestream_unmount(mp); + out_free_rtsb: + xfs_rtmount_freesb(mp); + out_free_sb: + xfs_freesb(mp); + out_free_scrub_stats: + xchk_mount_stats_free(mp); + out_free_stats: + free_percpu(mp->m_stats.xs_stats); + out_destroy_inodegc: + xfs_inodegc_free_percpu(mp); + out_destroy_counters: + xfs_destroy_percpu_counters(mp); + out_destroy_workqueues: + xfs_destroy_mount_workqueues(mp); + out_shutdown_devices: + xfs_shutdown_devices(mp); + return error; + + out_unmount: + xfs_filestream_unmount(mp); + xfs_unmountfs(mp); + goto out_free_rtsb; +} + +static int +xfs_fs_get_tree( + struct fs_context *fc) +{ + return get_tree_bdev(fc, xfs_fs_fill_super); +} + +static int +xfs_remount_rw( + struct xfs_mount *mp) +{ + struct xfs_sb *sbp = &mp->m_sb; + int error; + + if (mp->m_logdev_targp && mp->m_logdev_targp != mp->m_ddev_targp && + xfs_readonly_buftarg(mp->m_logdev_targp)) { + xfs_warn(mp, + "ro->rw transition prohibited by read-only logdev"); + return -EACCES; + } + + if (mp->m_rtdev_targp && xfs_readonly_buftarg(mp->m_rtdev_targp)) { + xfs_warn(mp, + "ro->rw transition prohibited by read-only rtdev"); + return -EACCES; + } + + if (xfs_has_norecovery(mp)) { + xfs_warn(mp, + "ro->rw transition prohibited on norecovery mount"); + return -EINVAL; + } + + if (xfs_sb_is_v5(sbp) && + xfs_sb_has_ro_compat_feature(sbp, XFS_SB_FEAT_RO_COMPAT_UNKNOWN)) { + xfs_warn(mp, + "ro->rw transition prohibited on unknown (0x%x) ro-compat filesystem", + (sbp->sb_features_ro_compat & + XFS_SB_FEAT_RO_COMPAT_UNKNOWN)); + return -EINVAL; + } + + xfs_clear_readonly(mp); + + /* + * If this is the first remount to writeable state we might have some + * superblock changes to update. + */ + if (mp->m_update_sb) { + error = xfs_sync_sb(mp, false); + if (error) { + xfs_warn(mp, "failed to write sb changes"); + return error; + } + mp->m_update_sb = false; + } + + /* + * Fill out the reserve pool if it is empty. Use the stashed value if + * it is non-zero, otherwise go with the default. + */ + xfs_restore_resvblks(mp); + xfs_log_work_queue(mp); + xfs_blockgc_start(mp); + + /* Create the per-AG metadata reservation pool .*/ + error = xfs_fs_reserve_ag_blocks(mp); + if (error && error != -ENOSPC) + return error; + + /* Re-enable the background inode inactivation worker. */ + xfs_inodegc_start(mp); + + /* Restart zone reclaim */ + xfs_zone_gc_start(mp); + + return 0; +} + +static int +xfs_remount_ro( + struct xfs_mount *mp) +{ + struct xfs_icwalk icw = { + .icw_flags = XFS_ICWALK_FLAG_SYNC, + }; + int error; + + /* Flush all the dirty data to disk. */ + error = sync_filesystem(mp->m_super); + if (error) + return error; + + /* + * Cancel background eofb scanning so it cannot race with the final + * log force+buftarg wait and deadlock the remount. + */ + xfs_blockgc_stop(mp); + + /* + * Clear out all remaining COW staging extents and speculative post-EOF + * preallocations so that we don't leave inodes requiring inactivation + * cleanups during reclaim on a read-only mount. We must process every + * cached inode, so this requires a synchronous cache scan. + */ + error = xfs_blockgc_free_space(mp, &icw); + if (error) { + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); + return error; + } + + /* + * Stop the inodegc background worker. xfs_fs_reconfigure already + * flushed all pending inodegc work when it sync'd the filesystem. + * The VFS holds s_umount, so we know that inodes cannot enter + * xfs_fs_destroy_inode during a remount operation. In readonly mode + * we send inodes straight to reclaim, so no inodes will be queued. + */ + xfs_inodegc_stop(mp); + + /* Stop zone reclaim */ + xfs_zone_gc_stop(mp); + + /* Free the per-AG metadata reservation pool. */ + xfs_fs_unreserve_ag_blocks(mp); + + /* + * Before we sync the metadata, we need to free up the reserve block + * pool so that the used block count in the superblock on disk is + * correct at the end of the remount. Stash the current* reserve pool + * size so that if we get remounted rw, we can return it to the same + * size. + */ + xfs_save_resvblks(mp); + + xfs_log_clean(mp); + xfs_set_readonly(mp); + + return 0; +} + +/* + * Logically we would return an error here to prevent users from believing + * they might have changed mount options using remount which can't be changed. + * + * But unfortunately mount(8) adds all options from mtab and fstab to the mount + * arguments in some cases so we can't blindly reject options, but have to + * check for each specified option if it actually differs from the currently + * set option and only reject it if that's the case. + * + * Until that is implemented we return success for every remount request, and + * silently ignore all options that we can't actually change. + */ +static int +xfs_fs_reconfigure( + struct fs_context *fc) +{ + struct xfs_mount *mp = XFS_M(fc->root->d_sb); + struct xfs_mount *new_mp = fc->s_fs_info; + int flags = fc->sb_flags; + int error; + + new_mp->m_qflags &= ~XFS_QFLAGS_MNTOPTS; + + /* version 5 superblocks always support version counters. */ + if (xfs_has_crc(mp)) + fc->sb_flags |= SB_I_VERSION; + + error = xfs_fs_validate_params(new_mp); + if (error) + return error; + + xfs_errortag_copy(mp, new_mp); + + /* Validate new max_atomic_write option before making other changes */ + if (mp->m_awu_max_bytes != new_mp->m_awu_max_bytes) { + error = xfs_set_max_atomic_write_opt(mp, + new_mp->m_awu_max_bytes); + if (error) + return error; + } + + /* inode32 -> inode64 */ + if (xfs_has_small_inums(mp) && !xfs_has_small_inums(new_mp)) { + mp->m_features &= ~XFS_FEAT_SMALL_INUMS; + mp->m_maxagi = xfs_set_inode_alloc(mp, mp->m_sb.sb_agcount); + } + + /* inode64 -> inode32 */ + if (!xfs_has_small_inums(mp) && xfs_has_small_inums(new_mp)) { + mp->m_features |= XFS_FEAT_SMALL_INUMS; + mp->m_maxagi = xfs_set_inode_alloc(mp, mp->m_sb.sb_agcount); + } + + /* + * Now that mp has been modified according to the remount options, we + * do a final option validation with xfs_finish_flags() just like it is + * just like it is done during mount. We cannot use + * done during mount. We cannot use xfs_finish_flags() on new_mp as it + * contains only the user given options. + */ + error = xfs_finish_flags(mp); + if (error) + return error; + + /* ro -> rw */ + if (xfs_is_readonly(mp) && !(flags & SB_RDONLY)) { + error = xfs_remount_rw(mp); + if (error) + return error; + } + + /* rw -> ro */ + if (!xfs_is_readonly(mp) && (flags & SB_RDONLY)) { + error = xfs_remount_ro(mp); + if (error) + return error; + } + + return 0; +} + +static void +xfs_fs_free( + struct fs_context *fc) +{ + struct xfs_mount *mp = fc->s_fs_info; + + /* + * mp is stored in the fs_context when it is initialized. + * mp is transferred to the superblock on a successful mount, + * but if an error occurs before the transfer we have to free + * it here. + */ + if (mp) + xfs_mount_free(mp); +} + +static const struct fs_context_operations xfs_context_ops = { + .parse_param = xfs_fs_parse_param, + .get_tree = xfs_fs_get_tree, + .reconfigure = xfs_fs_reconfigure, + .free = xfs_fs_free, +}; + +/* + * WARNING: do not initialise any parameters in this function that depend on + * mount option parsing having already been performed as this can be called from + * fsopen() before any parameters have been set. + */ +static int +xfs_init_fs_context( + struct fs_context *fc) +{ + struct xfs_mount *mp; + int i; + + mp = kzalloc_obj(struct xfs_mount); + if (!mp) + return -ENOMEM; +#ifdef DEBUG + mp->m_errortag = kzalloc_objs(*mp->m_errortag, XFS_ERRTAG_MAX); + if (!mp->m_errortag) { + kfree(mp); + return -ENOMEM; + } +#endif + + spin_lock_init(&mp->m_sb_lock); + for (i = 0; i < XG_TYPE_MAX; i++) + xa_init(&mp->m_groups[i].xa); + mutex_init(&mp->m_growlock); + mutex_init(&mp->m_metafile_resv_lock); + INIT_WORK(&mp->m_flush_inodes_work, xfs_flush_inodes_worker); + INIT_DELAYED_WORK(&mp->m_reclaim_work, xfs_reclaim_worker); + mp->m_kobj.kobject.kset = xfs_kset; + /* + * We don't create the finobt per-ag space reservation until after log + * recovery, so we must set this to true so that an ifree transaction + * started during log recovery will not depend on space reservations + * for finobt expansion. + */ + mp->m_finobt_nores = true; + + /* + * These can be overridden by the mount option parsing. + */ + mp->m_logbufs = -1; + mp->m_logbsize = -1; + mp->m_allocsize_log = 16; /* 64k */ + + xfs_hooks_init(&mp->m_dir_update_hooks); + + fc->s_fs_info = mp; + fc->ops = &xfs_context_ops; + + return 0; +} + +static void +xfs_kill_sb( + struct super_block *sb) +{ + struct xfs_mount *mp = XFS_M(sb); + + if (mp->m_rtdev_targp && mp->m_rtdev_targp != mp->m_ddev_targp) + xfs_restore_bdi_rahead(mp); + kill_block_super(sb); + xfs_mount_free(mp); +} + +static struct file_system_type xfs_fs_type = { + .owner = THIS_MODULE, + .name = "xfs", + .init_fs_context = xfs_init_fs_context, + .parameters = xfs_fs_parameters, + .kill_sb = xfs_kill_sb, + .fs_flags = FS_REQUIRES_DEV | FS_ALLOW_IDMAP | FS_MGTIME | + FS_LBS, +}; +MODULE_ALIAS_FS("xfs"); + +STATIC int __init +xfs_init_caches(void) +{ + int error; + + xfs_buf_cache = kmem_cache_create("xfs_buf", sizeof(struct xfs_buf), 0, + SLAB_HWCACHE_ALIGN | + SLAB_RECLAIM_ACCOUNT, + NULL); + if (!xfs_buf_cache) + goto out; + + xfs_log_ticket_cache = kmem_cache_create("xfs_log_ticket", + sizeof(struct xlog_ticket), + 0, 0, NULL); + if (!xfs_log_ticket_cache) + goto out_destroy_buf_cache; + + error = xfs_btree_init_cur_caches(); + if (error) + goto out_destroy_log_ticket_cache; + + error = rcbagbt_init_cur_cache(); + if (error) + goto out_destroy_btree_cur_cache; + + error = xfs_defer_init_item_caches(); + if (error) + goto out_destroy_rcbagbt_cur_cache; + + xfs_da_state_cache = kmem_cache_create("xfs_da_state", + sizeof(struct xfs_da_state), + 0, 0, NULL); + if (!xfs_da_state_cache) + goto out_destroy_defer_item_cache; + + xfs_ifork_cache = kmem_cache_create("xfs_ifork", + sizeof(struct xfs_ifork), + 0, 0, NULL); + if (!xfs_ifork_cache) + goto out_destroy_da_state_cache; + + xfs_trans_cache = kmem_cache_create("xfs_trans", + sizeof(struct xfs_trans), + 0, 0, NULL); + if (!xfs_trans_cache) + goto out_destroy_ifork_cache; + + + /* + * The size of the cache-allocated buf log item is the maximum + * size possible under XFS. This wastes a little bit of memory, + * but it is much faster. + */ + xfs_buf_item_cache = kmem_cache_create("xfs_buf_item", + sizeof(struct xfs_buf_log_item), + 0, 0, NULL); + if (!xfs_buf_item_cache) + goto out_destroy_trans_cache; + + xfs_efd_cache = kmem_cache_create("xfs_efd_item", + xfs_efd_log_item_sizeof(XFS_EFD_MAX_FAST_EXTENTS), + 0, 0, NULL); + if (!xfs_efd_cache) + goto out_destroy_buf_item_cache; + + xfs_efi_cache = kmem_cache_create("xfs_efi_item", + xfs_efi_log_item_sizeof(XFS_EFI_MAX_FAST_EXTENTS), + 0, 0, NULL); + if (!xfs_efi_cache) + goto out_destroy_efd_cache; + + xfs_inode_cache = kmem_cache_create("xfs_inode", + sizeof(struct xfs_inode), 0, + (SLAB_HWCACHE_ALIGN | + SLAB_RECLAIM_ACCOUNT | + SLAB_ACCOUNT), + xfs_fs_inode_init_once); + if (!xfs_inode_cache) + goto out_destroy_efi_cache; + + xfs_ili_cache = kmem_cache_create("xfs_ili", + sizeof(struct xfs_inode_log_item), 0, + SLAB_RECLAIM_ACCOUNT, + NULL); + if (!xfs_ili_cache) + goto out_destroy_inode_cache; + + xfs_icreate_cache = kmem_cache_create("xfs_icr", + sizeof(struct xfs_icreate_item), + 0, 0, NULL); + if (!xfs_icreate_cache) + goto out_destroy_ili_cache; + + xfs_rud_cache = kmem_cache_create("xfs_rud_item", + sizeof(struct xfs_rud_log_item), + 0, 0, NULL); + if (!xfs_rud_cache) + goto out_destroy_icreate_cache; + + xfs_rui_cache = kmem_cache_create("xfs_rui_item", + xfs_rui_log_item_sizeof(XFS_RUI_MAX_FAST_EXTENTS), + 0, 0, NULL); + if (!xfs_rui_cache) + goto out_destroy_rud_cache; + + xfs_cud_cache = kmem_cache_create("xfs_cud_item", + sizeof(struct xfs_cud_log_item), + 0, 0, NULL); + if (!xfs_cud_cache) + goto out_destroy_rui_cache; + + xfs_cui_cache = kmem_cache_create("xfs_cui_item", + xfs_cui_log_item_sizeof(XFS_CUI_MAX_FAST_EXTENTS), + 0, 0, NULL); + if (!xfs_cui_cache) + goto out_destroy_cud_cache; + + xfs_bud_cache = kmem_cache_create("xfs_bud_item", + sizeof(struct xfs_bud_log_item), + 0, 0, NULL); + if (!xfs_bud_cache) + goto out_destroy_cui_cache; + + xfs_bui_cache = kmem_cache_create("xfs_bui_item", + xfs_bui_log_item_sizeof(XFS_BUI_MAX_FAST_EXTENTS), + 0, 0, NULL); + if (!xfs_bui_cache) + goto out_destroy_bud_cache; + + xfs_attrd_cache = kmem_cache_create("xfs_attrd_item", + sizeof(struct xfs_attrd_log_item), + 0, 0, NULL); + if (!xfs_attrd_cache) + goto out_destroy_bui_cache; + + xfs_attri_cache = kmem_cache_create("xfs_attri_item", + sizeof(struct xfs_attri_log_item), + 0, 0, NULL); + if (!xfs_attri_cache) + goto out_destroy_attrd_cache; + + xfs_iunlink_cache = kmem_cache_create("xfs_iul_item", + sizeof(struct xfs_iunlink_item), + 0, 0, NULL); + if (!xfs_iunlink_cache) + goto out_destroy_attri_cache; + + xfs_xmd_cache = kmem_cache_create("xfs_xmd_item", + sizeof(struct xfs_xmd_log_item), + 0, 0, NULL); + if (!xfs_xmd_cache) + goto out_destroy_iul_cache; + + xfs_xmi_cache = kmem_cache_create("xfs_xmi_item", + sizeof(struct xfs_xmi_log_item), + 0, 0, NULL); + if (!xfs_xmi_cache) + goto out_destroy_xmd_cache; + + xfs_parent_args_cache = kmem_cache_create("xfs_parent_args", + sizeof(struct xfs_parent_args), + 0, 0, NULL); + if (!xfs_parent_args_cache) + goto out_destroy_xmi_cache; + + return 0; + + out_destroy_xmi_cache: + kmem_cache_destroy(xfs_xmi_cache); + out_destroy_xmd_cache: + kmem_cache_destroy(xfs_xmd_cache); + out_destroy_iul_cache: + kmem_cache_destroy(xfs_iunlink_cache); + out_destroy_attri_cache: + kmem_cache_destroy(xfs_attri_cache); + out_destroy_attrd_cache: + kmem_cache_destroy(xfs_attrd_cache); + out_destroy_bui_cache: + kmem_cache_destroy(xfs_bui_cache); + out_destroy_bud_cache: + kmem_cache_destroy(xfs_bud_cache); + out_destroy_cui_cache: + kmem_cache_destroy(xfs_cui_cache); + out_destroy_cud_cache: + kmem_cache_destroy(xfs_cud_cache); + out_destroy_rui_cache: + kmem_cache_destroy(xfs_rui_cache); + out_destroy_rud_cache: + kmem_cache_destroy(xfs_rud_cache); + out_destroy_icreate_cache: + kmem_cache_destroy(xfs_icreate_cache); + out_destroy_ili_cache: + kmem_cache_destroy(xfs_ili_cache); + out_destroy_inode_cache: + kmem_cache_destroy(xfs_inode_cache); + out_destroy_efi_cache: + kmem_cache_destroy(xfs_efi_cache); + out_destroy_efd_cache: + kmem_cache_destroy(xfs_efd_cache); + out_destroy_buf_item_cache: + kmem_cache_destroy(xfs_buf_item_cache); + out_destroy_trans_cache: + kmem_cache_destroy(xfs_trans_cache); + out_destroy_ifork_cache: + kmem_cache_destroy(xfs_ifork_cache); + out_destroy_da_state_cache: + kmem_cache_destroy(xfs_da_state_cache); + out_destroy_defer_item_cache: + xfs_defer_destroy_item_caches(); + out_destroy_rcbagbt_cur_cache: + rcbagbt_destroy_cur_cache(); + out_destroy_btree_cur_cache: + xfs_btree_destroy_cur_caches(); + out_destroy_log_ticket_cache: + kmem_cache_destroy(xfs_log_ticket_cache); + out_destroy_buf_cache: + kmem_cache_destroy(xfs_buf_cache); + out: + return -ENOMEM; +} + +STATIC void +xfs_destroy_caches(void) +{ + /* + * Make sure all delayed rcu free are flushed before we + * destroy caches. + */ + rcu_barrier(); + kmem_cache_destroy(xfs_parent_args_cache); + kmem_cache_destroy(xfs_xmd_cache); + kmem_cache_destroy(xfs_xmi_cache); + kmem_cache_destroy(xfs_iunlink_cache); + kmem_cache_destroy(xfs_attri_cache); + kmem_cache_destroy(xfs_attrd_cache); + kmem_cache_destroy(xfs_bui_cache); + kmem_cache_destroy(xfs_bud_cache); + kmem_cache_destroy(xfs_cui_cache); + kmem_cache_destroy(xfs_cud_cache); + kmem_cache_destroy(xfs_rui_cache); + kmem_cache_destroy(xfs_rud_cache); + kmem_cache_destroy(xfs_icreate_cache); + kmem_cache_destroy(xfs_ili_cache); + kmem_cache_destroy(xfs_inode_cache); + kmem_cache_destroy(xfs_efi_cache); + kmem_cache_destroy(xfs_efd_cache); + kmem_cache_destroy(xfs_buf_item_cache); + kmem_cache_destroy(xfs_trans_cache); + kmem_cache_destroy(xfs_ifork_cache); + kmem_cache_destroy(xfs_da_state_cache); + xfs_defer_destroy_item_caches(); + rcbagbt_destroy_cur_cache(); + xfs_btree_destroy_cur_caches(); + kmem_cache_destroy(xfs_log_ticket_cache); + kmem_cache_destroy(xfs_buf_cache); +} + +STATIC int __init +xfs_init_workqueues(void) +{ + /* + * The allocation workqueue can be used in memory reclaim situations + * (writepage path), and parallelism is only limited by the number of + * AGs in all the filesystems mounted. Hence use the default large + * max_active value for this workqueue. + */ + xfs_alloc_wq = alloc_workqueue("xfsalloc", XFS_WQFLAGS(WQ_MEM_RECLAIM | WQ_FREEZABLE | WQ_PERCPU), + 0); + if (!xfs_alloc_wq) + return -ENOMEM; + + xfs_discard_wq = alloc_workqueue("xfsdiscard", XFS_WQFLAGS(WQ_UNBOUND), + 0); + if (!xfs_discard_wq) + goto out_free_alloc_wq; + + return 0; +out_free_alloc_wq: + destroy_workqueue(xfs_alloc_wq); + return -ENOMEM; +} + +STATIC void +xfs_destroy_workqueues(void) +{ + destroy_workqueue(xfs_discard_wq); + destroy_workqueue(xfs_alloc_wq); +} + +STATIC int __init +init_xfs_fs(void) +{ + int error; + + xfs_check_ondisk_structs(); + + error = xfs_dahash_test(); + if (error) + return error; + + printk(KERN_INFO XFS_VERSION_STRING " with " + XFS_BUILD_OPTIONS " enabled\n"); + + xfs_dir_startup(); + + error = xfs_init_caches(); + if (error) + goto out; + + error = xfs_init_workqueues(); + if (error) + goto out_destroy_caches; + + error = xfs_mru_cache_init(); + if (error) + goto out_destroy_wq; + + error = xfs_init_procfs(); + if (error) + goto out_mru_cache_uninit; + + error = xfs_sysctl_register(); + if (error) + goto out_cleanup_procfs; + + xfs_debugfs = xfs_debugfs_mkdir("xfs", NULL); + + xfs_kset = kset_create_and_add("xfs", NULL, fs_kobj); + if (!xfs_kset) { + error = -ENOMEM; + goto out_debugfs_unregister; + } + + xfsstats.xs_kobj.kobject.kset = xfs_kset; + + xfsstats.xs_stats = alloc_percpu(struct xfsstats); + if (!xfsstats.xs_stats) { + error = -ENOMEM; + goto out_kset_unregister; + } + + error = xfs_sysfs_init(&xfsstats.xs_kobj, &xfs_stats_ktype, NULL, + "stats"); + if (error) + goto out_free_stats; + + error = xchk_global_stats_setup(xfs_debugfs); + if (error) + goto out_remove_stats_kobj; + +#ifdef DEBUG + xfs_dbg_kobj.kobject.kset = xfs_kset; + error = xfs_sysfs_init(&xfs_dbg_kobj, &xfs_dbg_ktype, NULL, "debug"); + if (error) + goto out_remove_scrub_stats; +#endif + + error = xfs_qm_init(); + if (error) + goto out_remove_dbg_kobj; + + error = register_filesystem(&xfs_fs_type); + if (error) + goto out_qm_exit; + return 0; + + out_qm_exit: + xfs_qm_exit(); + out_remove_dbg_kobj: +#ifdef DEBUG + xfs_sysfs_del(&xfs_dbg_kobj); + out_remove_scrub_stats: +#endif + xchk_global_stats_teardown(); + out_remove_stats_kobj: + xfs_sysfs_del(&xfsstats.xs_kobj); + out_free_stats: + free_percpu(xfsstats.xs_stats); + out_kset_unregister: + kset_unregister(xfs_kset); + out_debugfs_unregister: + debugfs_remove(xfs_debugfs); + xfs_sysctl_unregister(); + out_cleanup_procfs: + xfs_cleanup_procfs(); + out_mru_cache_uninit: + xfs_mru_cache_uninit(); + out_destroy_wq: + xfs_destroy_workqueues(); + out_destroy_caches: + xfs_destroy_caches(); + out: + return error; +} + +STATIC void __exit +exit_xfs_fs(void) +{ + xfs_qm_exit(); + unregister_filesystem(&xfs_fs_type); +#ifdef DEBUG + xfs_sysfs_del(&xfs_dbg_kobj); +#endif + xchk_global_stats_teardown(); + xfs_sysfs_del(&xfsstats.xs_kobj); + free_percpu(xfsstats.xs_stats); + kset_unregister(xfs_kset); + debugfs_remove(xfs_debugfs); + xfs_sysctl_unregister(); + xfs_cleanup_procfs(); + xfs_mru_cache_uninit(); + xfs_destroy_workqueues(); + xfs_destroy_caches(); + xfs_uuid_table_free(); +} + +module_init(init_xfs_fs); +module_exit(exit_xfs_fs); + +MODULE_AUTHOR("Silicon Graphics, Inc."); +MODULE_DESCRIPTION(XFS_VERSION_STRING " with " XFS_BUILD_OPTIONS " enabled"); +MODULE_LICENSE("GPL"); diff --git a/fs/xfs/xfs_super.h b/fs/xfs/xfs_super.h new file mode 100644 index 000000000..c0e85c1e4 --- /dev/null +++ b/fs/xfs/xfs_super.h @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_SUPER_H__ +#define __XFS_SUPER_H__ + +#include + +#ifdef CONFIG_XFS_QUOTA +extern int xfs_qm_init(void); +extern void xfs_qm_exit(void); +# define XFS_QUOTA_STRING "quota, " +#else +# define xfs_qm_init() (0) +# define xfs_qm_exit() do { } while (0) +# define XFS_QUOTA_STRING +#endif + +#ifdef CONFIG_XFS_POSIX_ACL +# define XFS_ACL_STRING "ACLs, " +# define set_posix_acl_flag(sb) ((sb)->s_flags |= SB_POSIXACL) +#else +# define XFS_ACL_STRING +# define set_posix_acl_flag(sb) do { } while (0) +#endif + +#define XFS_SECURITY_STRING "security attributes, " + +#ifdef CONFIG_XFS_RT +# define XFS_REALTIME_STRING "realtime, " +#else +# define XFS_REALTIME_STRING +#endif + +#ifdef CONFIG_XFS_ONLINE_SCRUB +# define XFS_SCRUB_STRING "scrub, " +#else +# define XFS_SCRUB_STRING +#endif + +#ifdef CONFIG_XFS_ONLINE_REPAIR +# define XFS_REPAIR_STRING "repair, " +#else +# define XFS_REPAIR_STRING +#endif + +#ifdef CONFIG_XFS_WARN +# define XFS_WARN_STRING "verbose warnings, " +#else +# define XFS_WARN_STRING +#endif + +#ifdef CONFIG_XFS_ASSERT_FATAL +# define XFS_ASSERT_FATAL_STRING "fatal assert, " +#else +# define XFS_ASSERT_FATAL_STRING +#endif + +#ifdef DEBUG +# define XFS_DBG_STRING "debug" +#else +# define XFS_DBG_STRING "no debug" +#endif + +#define XFS_VERSION_STRING "SGI XFS" +#define XFS_BUILD_OPTIONS XFS_ACL_STRING \ + XFS_SECURITY_STRING \ + XFS_REALTIME_STRING \ + XFS_SCRUB_STRING \ + XFS_REPAIR_STRING \ + XFS_WARN_STRING \ + XFS_QUOTA_STRING \ + XFS_ASSERT_FATAL_STRING \ + XFS_DBG_STRING /* DBG must be last */ + +#ifdef DEBUG +# define XFS_WQFLAGS(wqflags) (WQ_SYSFS | (wqflags)) +#else +# define XFS_WQFLAGS(wqflags) (wqflags) +#endif + +struct xfs_inode; +struct xfs_mount; +struct xfs_buftarg; +struct block_device; + +extern void xfs_flush_inodes(struct xfs_mount *mp); +extern xfs_agnumber_t xfs_set_inode_alloc(struct xfs_mount *, + xfs_agnumber_t agcount); + +extern const struct export_operations xfs_export_operations; +extern const struct quotactl_ops xfs_quotactl_operations; + +extern void xfs_reinit_percpu_counters(struct xfs_mount *mp); + +extern struct workqueue_struct *xfs_discard_wq; + +#define XFS_M(sb) ((struct xfs_mount *)((sb)->s_fs_info)) + +struct dentry *xfs_debugfs_mkdir(const char *name, struct dentry *parent); + +#endif /* __XFS_SUPER_H__ */ diff --git a/fs/xfs/xfs_symlink.c b/fs/xfs/xfs_symlink.c new file mode 100644 index 000000000..5585ac7f4 --- /dev/null +++ b/fs/xfs/xfs_symlink.c @@ -0,0 +1,362 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2006 Silicon Graphics, Inc. + * Copyright (c) 2012-2013 Red Hat, Inc. + * All rights reserved. + */ +#include "xfs_platform.h" +#include "xfs_shared.h" +#include "xfs_fs.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_bit.h" +#include "xfs_mount.h" +#include "xfs_dir2.h" +#include "xfs_inode.h" +#include "xfs_bmap.h" +#include "xfs_bmap_btree.h" +#include "xfs_quota.h" +#include "xfs_symlink.h" +#include "xfs_trans_space.h" +#include "xfs_trace.h" +#include "xfs_trans.h" +#include "xfs_ialloc.h" +#include "xfs_error.h" +#include "xfs_health.h" +#include "xfs_symlink_remote.h" +#include "xfs_parent.h" +#include "xfs_defer.h" + +int +xfs_readlink( + struct xfs_inode *ip, + char *link) +{ + struct xfs_mount *mp = ip->i_mount; + xfs_fsize_t pathlen; + int error; + + trace_xfs_readlink(ip); + + if (xfs_is_shutdown(mp)) + return -EIO; + if (xfs_ifork_zapped(ip, XFS_DATA_FORK)) + return -EIO; + + xfs_ilock(ip, XFS_ILOCK_SHARED); + + pathlen = ip->i_disk_size; + if (!pathlen) + goto out_corrupt; + + if (pathlen < 0 || pathlen > XFS_SYMLINK_MAXLEN) { + xfs_alert(mp, "%s: inode (%llu) bad symlink length (%lld)", + __func__, (unsigned long long)I_INO(ip), + (long long) pathlen); + ASSERT(0); + goto out_corrupt; + } + + if (ip->i_df.if_format == XFS_DINODE_FMT_LOCAL) { + /* + * The VFS crashes on a NULL pointer, so return -EFSCORRUPTED + * if if_data is junk. + */ + if (XFS_IS_CORRUPT(ip->i_mount, !ip->i_df.if_data)) + goto out_corrupt; + + memcpy(link, ip->i_df.if_data, pathlen + 1); + error = 0; + } else { + error = xfs_symlink_remote_read(ip, link); + } + + xfs_iunlock(ip, XFS_ILOCK_SHARED); + return error; + out_corrupt: + xfs_iunlock(ip, XFS_ILOCK_SHARED); + xfs_inode_mark_sick(ip, XFS_SICK_INO_SYMLINK); + return -EFSCORRUPTED; +} + +int +xfs_symlink( + struct mnt_idmap *idmap, + struct xfs_inode *dp, + struct xfs_name *link_name, + const char *target_path, + umode_t mode, + struct xfs_inode **ipp) +{ + struct xfs_mount *mp = dp->i_mount; + struct xfs_icreate_args args = { + .idmap = idmap, + .pip = dp, + .mode = S_IFLNK | (mode & ~S_IFMT), + }; + struct xfs_dir_update du = { + .dp = dp, + .name = link_name, + }; + struct xfs_trans *tp = NULL; + int error = 0; + int pathlen; + bool unlock_dp_on_error = false; + xfs_filblks_t fs_blocks; + struct xfs_dquot *udqp; + struct xfs_dquot *gdqp; + struct xfs_dquot *pdqp; + uint resblks; + xfs_ino_t ino; + + *ipp = NULL; + + trace_xfs_symlink(dp, link_name); + + if (xfs_is_shutdown(mp)) + return -EIO; + + /* + * Check component lengths of the target path name. + */ + pathlen = strlen(target_path); + if (pathlen >= XFS_SYMLINK_MAXLEN) /* total string too long */ + return -ENAMETOOLONG; + ASSERT(pathlen > 0); + + /* Make sure that we have allocated dquot(s) on disk. */ + error = xfs_icreate_dqalloc(&args, &udqp, &gdqp, &pdqp); + if (error) + return error; + + /* + * The symlink will fit into the inode data fork? + * If there are no parent pointers, then there wont't be any attributes. + * So we get the whole variable part, and do not need to reserve extra + * blocks. Otherwise, we need to reserve the blocks. + */ + if (pathlen <= XFS_LITINO(mp) && !xfs_has_parent(mp)) + fs_blocks = 0; + else + fs_blocks = xfs_symlink_blocks(mp, pathlen); + resblks = xfs_symlink_space_res(mp, link_name->len, fs_blocks); + + error = xfs_parent_start(mp, &du.ppargs); + if (error) + goto out_release_dquots; + + error = xfs_trans_alloc_icreate(mp, &M_RES(mp)->tr_symlink, udqp, gdqp, + pdqp, resblks, &tp); + if (error) + goto out_parent; + + xfs_ilock(dp, XFS_ILOCK_EXCL | XFS_ILOCK_PARENT); + unlock_dp_on_error = true; + + /* + * Check whether the directory allows new symlinks or not. + */ + if (dp->i_diflags & XFS_DIFLAG_NOSYMLINKS) { + error = -EPERM; + goto out_trans_cancel; + } + + /* + * Allocate an inode for the symlink. + */ + error = xfs_dialloc(&tp, &args, &ino); + if (!error) + error = xfs_icreate(tp, ino, &args, &du.ip); + if (error) + goto out_trans_cancel; + + /* + * Now we join the directory inode to the transaction. We do not do it + * earlier because xfs_dir_ialloc might commit the previous transaction + * (and release all the locks). An error from here on will result in + * the transaction cancel unlocking dp so don't do it explicitly in the + * error path. + */ + xfs_trans_ijoin(tp, dp, 0); + + /* + * Also attach the dquot(s) to it, if applicable. + */ + xfs_qm_vop_create_dqattach(tp, du.ip, udqp, gdqp, pdqp); + + resblks -= XFS_IALLOC_SPACE_RES(mp); + error = xfs_symlink_write_target(tp, du.ip, I_INO(du.ip), target_path, + pathlen, fs_blocks, resblks); + if (error) + goto out_trans_cancel; + resblks -= fs_blocks; + i_size_write(VFS_I(du.ip), du.ip->i_disk_size); + + /* + * Create the directory entry for the symlink. + */ + error = xfs_dir_create_child(tp, resblks, &du); + if (error) + goto out_trans_cancel; + + /* + * If this is a synchronous mount, make sure that the + * symlink transaction goes to disk before returning to + * the user. + */ + if (xfs_has_wsync(mp) || xfs_has_dirsync(mp)) + xfs_trans_set_sync(tp); + + error = xfs_trans_commit(tp); + if (error) + goto out_release_inode; + + xfs_qm_dqrele(udqp); + xfs_qm_dqrele(gdqp); + xfs_qm_dqrele(pdqp); + + *ipp = du.ip; + xfs_iunlock(du.ip, XFS_ILOCK_EXCL); + xfs_iunlock(dp, XFS_ILOCK_EXCL); + xfs_parent_finish(mp, du.ppargs); + return 0; + +out_trans_cancel: + xfs_trans_cancel(tp); +out_release_inode: + /* + * Wait until after the current transaction is aborted to finish the + * setup of the inode and release the inode. This prevents recursive + * transactions and deadlocks from xfs_inactive. + */ + if (du.ip) { + xfs_iunlock(du.ip, XFS_ILOCK_EXCL); + xfs_finish_inode_setup(du.ip); + xfs_irele(du.ip); + } +out_parent: + xfs_parent_finish(mp, du.ppargs); +out_release_dquots: + xfs_qm_dqrele(udqp); + xfs_qm_dqrele(gdqp); + xfs_qm_dqrele(pdqp); + + if (unlock_dp_on_error) + xfs_iunlock(dp, XFS_ILOCK_EXCL); + return error; +} + +/* + * Free a symlink that has blocks associated with it. + * + * Note: zero length symlinks are not allowed to exist. When we set the size to + * zero, also change it to a regular file so that it does not get written to + * disk as a zero length symlink. The inode is on the unlinked list already, so + * userspace cannot find this inode anymore, so this change is not user visible + * but allows us to catch corrupt zero-length symlinks in the verifiers. + */ +STATIC int +xfs_inactive_symlink_rmt( + struct xfs_inode *ip) +{ + struct xfs_mount *mp = ip->i_mount; + struct xfs_trans *tp; + int error; + + ASSERT(!xfs_need_iread_extents(&ip->i_df)); + /* + * We're freeing a symlink that has some + * blocks allocated to it. Free the + * blocks here. We know that we've got + * either 1 or 2 extents and that we can + * free them all in one bunmapi call. + */ + ASSERT(ip->i_df.if_nextents > 0 && ip->i_df.if_nextents <= 2); + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_itruncate, 0, 0, 0, &tp); + if (error) + return error; + + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, 0); + + /* + * Lock the inode, fix the size, turn it into a regular file and join it + * to the transaction. Hold it so in the normal path, we still have it + * locked for the second transaction. In the error paths we need it + * held so the cancel won't rele it, see below. + */ + ip->i_disk_size = 0; + VFS_I(ip)->i_mode = (VFS_I(ip)->i_mode & ~S_IFMT) | S_IFREG; + xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE); + + error = xfs_symlink_remote_truncate(tp, ip); + if (error) + goto error_trans_cancel; + + error = xfs_trans_commit(tp); + if (error) { + ASSERT(xfs_is_shutdown(mp)); + goto error_unlock; + } + + /* + * Remove the memory for extent descriptions (just bookkeeping). + */ + if (ip->i_df.if_bytes) + xfs_idata_realloc(ip, -ip->i_df.if_bytes, XFS_DATA_FORK); + ASSERT(ip->i_df.if_bytes == 0); + + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return 0; + +error_trans_cancel: + xfs_trans_cancel(tp); +error_unlock: + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; +} + +/* + * xfs_inactive_symlink - free a symlink + */ +int +xfs_inactive_symlink( + struct xfs_inode *ip) +{ + struct xfs_mount *mp = ip->i_mount; + int pathlen; + + trace_xfs_inactive_symlink(ip); + + if (xfs_is_shutdown(mp)) + return -EIO; + + xfs_ilock(ip, XFS_ILOCK_EXCL); + pathlen = (int)ip->i_disk_size; + ASSERT(pathlen); + + if (pathlen <= 0 || pathlen > XFS_SYMLINK_MAXLEN) { + xfs_alert(mp, "%s: inode (0x%llx) bad symlink length (%d)", + __func__, (unsigned long long)I_INO(ip), pathlen); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + ASSERT(0); + xfs_inode_mark_sick(ip, XFS_SICK_INO_SYMLINK); + return -EFSCORRUPTED; + } + + /* + * Inline fork state gets removed by xfs_difree() so we have nothing to + * do here in that case. + */ + if (ip->i_df.if_format == XFS_DINODE_FMT_LOCAL) { + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return 0; + } + + xfs_iunlock(ip, XFS_ILOCK_EXCL); + + /* remove the remote symlink */ + return xfs_inactive_symlink_rmt(ip); +} diff --git a/fs/xfs/xfs_symlink.h b/fs/xfs/xfs_symlink.h new file mode 100644 index 000000000..0d29a50e6 --- /dev/null +++ b/fs/xfs/xfs_symlink.h @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2012 Red Hat, Inc. All rights reserved. + */ +#ifndef __XFS_SYMLINK_H +#define __XFS_SYMLINK_H 1 + +/* Kernel only symlink definitions */ + +int xfs_symlink(struct mnt_idmap *idmap, struct xfs_inode *dp, + struct xfs_name *link_name, const char *target_path, + umode_t mode, struct xfs_inode **ipp); +int xfs_readlink(struct xfs_inode *ip, char *link); +int xfs_inactive_symlink(struct xfs_inode *ip); + +#endif /* __XFS_SYMLINK_H */ diff --git a/fs/xfs/xfs_sysctl.c b/fs/xfs/xfs_sysctl.c new file mode 100644 index 000000000..7f32d282d --- /dev/null +++ b/fs/xfs/xfs_sysctl.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2001-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_error.h" + +static struct ctl_table_header *xfs_table_header; + +#ifdef CONFIG_PROC_FS +STATIC int +xfs_stats_clear_proc_handler( + const struct ctl_table *ctl, + int write, + void *buffer, + size_t *lenp, + loff_t *ppos) +{ + int ret, *valp = ctl->data; + + ret = proc_dointvec_minmax(ctl, write, buffer, lenp, ppos); + + if (!ret && write && *valp) { + xfs_stats_clearall(xfsstats.xs_stats); + xfs_stats_clear = 0; + } + + return ret; +} + +STATIC int +xfs_panic_mask_proc_handler( + const struct ctl_table *ctl, + int write, + void *buffer, + size_t *lenp, + loff_t *ppos) +{ + int ret, *valp = ctl->data; + + ret = proc_dointvec_minmax(ctl, write, buffer, lenp, ppos); + if (!ret && write) { + xfs_panic_mask = *valp; +#ifdef DEBUG + xfs_panic_mask |= (XFS_PTAG_SHUTDOWN_CORRUPT | XFS_PTAG_LOGRES); +#endif + } + return ret; +} +#endif /* CONFIG_PROC_FS */ + +static inline int +xfs_deprecated_dointvec_minmax( + const struct ctl_table *ctl, + int write, + void *buffer, + size_t *lenp, + loff_t *ppos) +{ + if (write) { + printk_ratelimited(KERN_WARNING + "XFS: %s sysctl option is deprecated.\n", + ctl->procname); + } + return proc_dointvec_minmax(ctl, write, buffer, lenp, ppos); +} + +static const struct ctl_table xfs_table[] = { + { + .procname = "panic_mask", + .data = &xfs_params.panic_mask.val, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = xfs_panic_mask_proc_handler, + .extra1 = &xfs_params.panic_mask.min, + .extra2 = &xfs_params.panic_mask.max + }, + + { + .procname = "error_level", + .data = &xfs_params.error_level.val, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = &xfs_params.error_level.min, + .extra2 = &xfs_params.error_level.max + }, + { + .procname = "xfssyncd_centisecs", + .data = &xfs_params.syncd_timer.val, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = &xfs_params.syncd_timer.min, + .extra2 = &xfs_params.syncd_timer.max + }, + { + .procname = "inherit_sync", + .data = &xfs_params.inherit_sync.val, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = &xfs_params.inherit_sync.min, + .extra2 = &xfs_params.inherit_sync.max + }, + { + .procname = "inherit_nodump", + .data = &xfs_params.inherit_nodump.val, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = &xfs_params.inherit_nodump.min, + .extra2 = &xfs_params.inherit_nodump.max + }, + { + .procname = "inherit_noatime", + .data = &xfs_params.inherit_noatim.val, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = &xfs_params.inherit_noatim.min, + .extra2 = &xfs_params.inherit_noatim.max + }, + { + .procname = "inherit_nosymlinks", + .data = &xfs_params.inherit_nosym.val, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = &xfs_params.inherit_nosym.min, + .extra2 = &xfs_params.inherit_nosym.max + }, + { + .procname = "rotorstep", + .data = &xfs_params.rotorstep.val, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = &xfs_params.rotorstep.min, + .extra2 = &xfs_params.rotorstep.max + }, + { + .procname = "inherit_nodefrag", + .data = &xfs_params.inherit_nodfrg.val, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = &xfs_params.inherit_nodfrg.min, + .extra2 = &xfs_params.inherit_nodfrg.max + }, + { + .procname = "filestream_centisecs", + .data = &xfs_params.fstrm_timer.val, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = &xfs_params.fstrm_timer.min, + .extra2 = &xfs_params.fstrm_timer.max, + }, + { + .procname = "speculative_prealloc_lifetime", + .data = &xfs_params.blockgc_timer.val, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = &xfs_params.blockgc_timer.min, + .extra2 = &xfs_params.blockgc_timer.max, + }, + /* please keep this the last entry */ +#ifdef CONFIG_PROC_FS + { + .procname = "stats_clear", + .data = &xfs_params.stats_clear.val, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = xfs_stats_clear_proc_handler, + .extra1 = &xfs_params.stats_clear.min, + .extra2 = &xfs_params.stats_clear.max + }, +#endif /* CONFIG_PROC_FS */ +}; + +int +xfs_sysctl_register(void) +{ + xfs_table_header = register_sysctl("fs/xfs", xfs_table); + if (!xfs_table_header) + return -ENOMEM; + return 0; +} + +void +xfs_sysctl_unregister(void) +{ + unregister_sysctl_table(xfs_table_header); +} diff --git a/fs/xfs/xfs_sysctl.h b/fs/xfs/xfs_sysctl.h new file mode 100644 index 000000000..ed9d89607 --- /dev/null +++ b/fs/xfs/xfs_sysctl.h @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2001-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_SYSCTL_H__ +#define __XFS_SYSCTL_H__ + +#include + +/* + * Tunable xfs parameters + */ + +typedef struct xfs_sysctl_val { + int min; + int val; + int max; +} xfs_sysctl_val_t; + +typedef struct xfs_param { + xfs_sysctl_val_t panic_mask; /* bitmask to cause panic on errors. */ + xfs_sysctl_val_t error_level; /* Degree of reporting for problems */ + xfs_sysctl_val_t syncd_timer; /* Interval between xfssyncd wakeups */ + xfs_sysctl_val_t stats_clear; /* Reset all XFS statistics to zero. */ + xfs_sysctl_val_t inherit_sync; /* Inherit the "sync" inode flag. */ + xfs_sysctl_val_t inherit_nodump;/* Inherit the "nodump" inode flag. */ + xfs_sysctl_val_t inherit_noatim;/* Inherit the "noatime" inode flag. */ + xfs_sysctl_val_t inherit_nosym; /* Inherit the "nosymlinks" flag. */ + xfs_sysctl_val_t rotorstep; /* inode32 AG rotoring control knob */ + xfs_sysctl_val_t inherit_nodfrg;/* Inherit the "nodefrag" inode flag. */ + xfs_sysctl_val_t fstrm_timer; /* Filestream dir-AG assoc'n timeout. */ + xfs_sysctl_val_t blockgc_timer; /* Interval between blockgc scans */ +} xfs_param_t; + +/* + * xfs_error_level: + * + * How much error reporting will be done when internal problems are + * encountered. These problems normally return an EFSCORRUPTED to their + * caller, with no other information reported. + * + * 0 No error reports + * 1 Report EFSCORRUPTED errors that will cause a filesystem shutdown + * 5 Report all EFSCORRUPTED errors (all of the above errors, plus any + * additional errors that are known to not cause shutdowns) + * + * xfs_panic_mask bit 0x8 turns the error reports into panics + */ + +enum { + /* XFS_REFCACHE_SIZE = 1 */ + /* XFS_REFCACHE_PURGE = 2 */ + /* XFS_RESTRICT_CHOWN = 3 */ + XFS_SGID_INHERIT = 4, + XFS_SYMLINK_MODE = 5, + XFS_PANIC_MASK = 6, + XFS_ERRLEVEL = 7, + XFS_SYNCD_TIMER = 8, + /* XFS_PROBE_DMAPI = 9 */ + /* XFS_PROBE_IOOPS = 10 */ + /* XFS_PROBE_QUOTA = 11 */ + XFS_STATS_CLEAR = 12, + XFS_INHERIT_SYNC = 13, + XFS_INHERIT_NODUMP = 14, + XFS_INHERIT_NOATIME = 15, + XFS_BUF_TIMER = 16, + XFS_BUF_AGE = 17, + /* XFS_IO_BYPASS = 18 */ + XFS_INHERIT_NOSYM = 19, + XFS_ROTORSTEP = 20, + XFS_INHERIT_NODFRG = 21, + XFS_FILESTREAM_TIMER = 22, +}; + +extern xfs_param_t xfs_params; + +struct xfs_globals { +#ifdef DEBUG + int pwork_threads; /* parallel workqueue threads */ + bool larp; /* log attribute replay */ +#endif + int bload_leaf_slack; /* btree bulk load leaf slack */ + int bload_node_slack; /* btree bulk load node slack */ + int log_recovery_delay; /* log recovery delay (secs) */ + int mount_delay; /* mount setup delay (secs) */ + bool bug_on_assert; /* BUG() the kernel on assert failure */ + bool always_cow; /* use COW fork for all overwrites */ +}; +extern struct xfs_globals xfs_globals; + +#ifdef CONFIG_SYSCTL +extern int xfs_sysctl_register(void); +extern void xfs_sysctl_unregister(void); +#else +# define xfs_sysctl_register() (0) +# define xfs_sysctl_unregister() do { } while (0) +#endif /* CONFIG_SYSCTL */ + +#endif /* __XFS_SYSCTL_H__ */ diff --git a/fs/xfs/xfs_sysfs.c b/fs/xfs/xfs_sysfs.c new file mode 100644 index 000000000..b62712187 --- /dev/null +++ b/fs/xfs/xfs_sysfs.c @@ -0,0 +1,898 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2014 Red Hat, Inc. + * All Rights Reserved. + */ + +#include "xfs_platform.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_sysfs.h" +#include "xfs_log.h" +#include "xfs_log_priv.h" +#include "xfs_mount.h" +#include "xfs_zone_priv.h" +#include "xfs_zones.h" +#include "xfs_zone_alloc.h" + +struct xfs_sysfs_attr { + struct attribute attr; + ssize_t (*show)(struct kobject *kobject, char *buf); + ssize_t (*store)(struct kobject *kobject, const char *buf, + size_t count); +}; + +static inline struct xfs_sysfs_attr * +to_attr(struct attribute *attr) +{ + return container_of(attr, struct xfs_sysfs_attr, attr); +} + +#define XFS_SYSFS_ATTR_RW(name) \ + static struct xfs_sysfs_attr xfs_sysfs_attr_##name = __ATTR_RW(name) +#define XFS_SYSFS_ATTR_RO(name) \ + static struct xfs_sysfs_attr xfs_sysfs_attr_##name = __ATTR_RO(name) +#define XFS_SYSFS_ATTR_WO(name) \ + static struct xfs_sysfs_attr xfs_sysfs_attr_##name = __ATTR_WO(name) + +#define ATTR_LIST(name) &xfs_sysfs_attr_##name.attr + +STATIC ssize_t +xfs_sysfs_object_show( + struct kobject *kobject, + struct attribute *attr, + char *buf) +{ + struct xfs_sysfs_attr *xfs_attr = to_attr(attr); + + return xfs_attr->show ? xfs_attr->show(kobject, buf) : 0; +} + +STATIC ssize_t +xfs_sysfs_object_store( + struct kobject *kobject, + struct attribute *attr, + const char *buf, + size_t count) +{ + struct xfs_sysfs_attr *xfs_attr = to_attr(attr); + + return xfs_attr->store ? xfs_attr->store(kobject, buf, count) : 0; +} + +static const struct sysfs_ops xfs_sysfs_ops = { + .show = xfs_sysfs_object_show, + .store = xfs_sysfs_object_store, +}; + +static struct attribute *xfs_mp_attrs[] = { + NULL, +}; +ATTRIBUTE_GROUPS(xfs_mp); + +static const struct kobj_type xfs_mp_ktype = { + .release = xfs_sysfs_release, + .sysfs_ops = &xfs_sysfs_ops, + .default_groups = xfs_mp_groups, +}; + +#ifdef DEBUG +/* debug */ + +STATIC ssize_t +bug_on_assert_store( + struct kobject *kobject, + const char *buf, + size_t count) +{ + int ret; + int val; + + ret = kstrtoint(buf, 0, &val); + if (ret) + return ret; + + if (val == 1) + xfs_globals.bug_on_assert = true; + else if (val == 0) + xfs_globals.bug_on_assert = false; + else + return -EINVAL; + + return count; +} + +STATIC ssize_t +bug_on_assert_show( + struct kobject *kobject, + char *buf) +{ + return sysfs_emit(buf, "%d\n", xfs_globals.bug_on_assert); +} +XFS_SYSFS_ATTR_RW(bug_on_assert); + +STATIC ssize_t +log_recovery_delay_store( + struct kobject *kobject, + const char *buf, + size_t count) +{ + int ret; + int val; + + ret = kstrtoint(buf, 0, &val); + if (ret) + return ret; + + if (val < 0 || val > 60) + return -EINVAL; + + xfs_globals.log_recovery_delay = val; + + return count; +} + +STATIC ssize_t +log_recovery_delay_show( + struct kobject *kobject, + char *buf) +{ + return sysfs_emit(buf, "%d\n", xfs_globals.log_recovery_delay); +} +XFS_SYSFS_ATTR_RW(log_recovery_delay); + +STATIC ssize_t +mount_delay_store( + struct kobject *kobject, + const char *buf, + size_t count) +{ + int ret; + int val; + + ret = kstrtoint(buf, 0, &val); + if (ret) + return ret; + + if (val < 0 || val > 60) + return -EINVAL; + + xfs_globals.mount_delay = val; + + return count; +} + +STATIC ssize_t +mount_delay_show( + struct kobject *kobject, + char *buf) +{ + return sysfs_emit(buf, "%d\n", xfs_globals.mount_delay); +} +XFS_SYSFS_ATTR_RW(mount_delay); + +static ssize_t +always_cow_store( + struct kobject *kobject, + const char *buf, + size_t count) +{ + ssize_t ret; + + ret = kstrtobool(buf, &xfs_globals.always_cow); + if (ret < 0) + return ret; + return count; +} + +static ssize_t +always_cow_show( + struct kobject *kobject, + char *buf) +{ + return sysfs_emit(buf, "%d\n", xfs_globals.always_cow); +} +XFS_SYSFS_ATTR_RW(always_cow); + +/* + * Override how many threads the parallel work queue is allowed to create. + * This has to be a debug-only global (instead of an errortag) because one of + * the main users of parallel workqueues is mount time quotacheck. + */ +STATIC ssize_t +pwork_threads_store( + struct kobject *kobject, + const char *buf, + size_t count) +{ + int ret; + int val; + + ret = kstrtoint(buf, 0, &val); + if (ret) + return ret; + + if (val < -1 || val > num_possible_cpus()) + return -EINVAL; + + xfs_globals.pwork_threads = val; + + return count; +} + +STATIC ssize_t +pwork_threads_show( + struct kobject *kobject, + char *buf) +{ + return sysfs_emit(buf, "%d\n", xfs_globals.pwork_threads); +} +XFS_SYSFS_ATTR_RW(pwork_threads); + +/* + * The "LARP" (Logged extended Attribute Recovery Persistence) debugging knob + * sets the XFS_DA_OP_LOGGED flag on all xfs_attr_set operations performed on + * V5 filesystems. As a result, the intermediate progress of all setxattr and + * removexattr operations are tracked via the log and can be restarted during + * recovery. This is useful for testing xattr recovery prior to merging of the + * parent pointer feature which requires it to maintain consistency, and may be + * enabled for userspace xattrs in the future. + */ +static ssize_t +larp_store( + struct kobject *kobject, + const char *buf, + size_t count) +{ + ssize_t ret; + + ret = kstrtobool(buf, &xfs_globals.larp); + if (ret < 0) + return ret; + return count; +} + +STATIC ssize_t +larp_show( + struct kobject *kobject, + char *buf) +{ + return snprintf(buf, PAGE_SIZE, "%d\n", xfs_globals.larp); +} +XFS_SYSFS_ATTR_RW(larp); + +STATIC ssize_t +bload_leaf_slack_store( + struct kobject *kobject, + const char *buf, + size_t count) +{ + int ret; + int val; + + ret = kstrtoint(buf, 0, &val); + if (ret) + return ret; + + xfs_globals.bload_leaf_slack = val; + return count; +} + +STATIC ssize_t +bload_leaf_slack_show( + struct kobject *kobject, + char *buf) +{ + return snprintf(buf, PAGE_SIZE, "%d\n", xfs_globals.bload_leaf_slack); +} +XFS_SYSFS_ATTR_RW(bload_leaf_slack); + +STATIC ssize_t +bload_node_slack_store( + struct kobject *kobject, + const char *buf, + size_t count) +{ + int ret; + int val; + + ret = kstrtoint(buf, 0, &val); + if (ret) + return ret; + + xfs_globals.bload_node_slack = val; + return count; +} + +STATIC ssize_t +bload_node_slack_show( + struct kobject *kobject, + char *buf) +{ + return snprintf(buf, PAGE_SIZE, "%d\n", xfs_globals.bload_node_slack); +} +XFS_SYSFS_ATTR_RW(bload_node_slack); + +static struct attribute *xfs_dbg_attrs[] = { + ATTR_LIST(bug_on_assert), + ATTR_LIST(log_recovery_delay), + ATTR_LIST(mount_delay), + ATTR_LIST(always_cow), + ATTR_LIST(pwork_threads), + ATTR_LIST(larp), + ATTR_LIST(bload_leaf_slack), + ATTR_LIST(bload_node_slack), + NULL, +}; +ATTRIBUTE_GROUPS(xfs_dbg); + +const struct kobj_type xfs_dbg_ktype = { + .release = xfs_sysfs_release, + .sysfs_ops = &xfs_sysfs_ops, + .default_groups = xfs_dbg_groups, +}; + +#endif /* DEBUG */ + +/* stats */ + +static inline struct xstats * +to_xstats(struct kobject *kobject) +{ + struct xfs_kobj *kobj = to_kobj(kobject); + + return container_of(kobj, struct xstats, xs_kobj); +} + +STATIC ssize_t +stats_show( + struct kobject *kobject, + char *buf) +{ + struct xstats *stats = to_xstats(kobject); + + return xfs_stats_format(stats->xs_stats, buf); +} +XFS_SYSFS_ATTR_RO(stats); + +STATIC ssize_t +stats_clear_store( + struct kobject *kobject, + const char *buf, + size_t count) +{ + int ret; + int val; + struct xstats *stats = to_xstats(kobject); + + ret = kstrtoint(buf, 0, &val); + if (ret) + return ret; + + if (val != 1) + return -EINVAL; + + xfs_stats_clearall(stats->xs_stats); + return count; +} +XFS_SYSFS_ATTR_WO(stats_clear); + +static struct attribute *xfs_stats_attrs[] = { + ATTR_LIST(stats), + ATTR_LIST(stats_clear), + NULL, +}; +ATTRIBUTE_GROUPS(xfs_stats); + +const struct kobj_type xfs_stats_ktype = { + .release = xfs_sysfs_release, + .sysfs_ops = &xfs_sysfs_ops, + .default_groups = xfs_stats_groups, +}; + +/* xlog */ + +static inline struct xlog * +to_xlog(struct kobject *kobject) +{ + struct xfs_kobj *kobj = to_kobj(kobject); + + return container_of(kobj, struct xlog, l_kobj); +} + +STATIC ssize_t +log_head_lsn_show( + struct kobject *kobject, + char *buf) +{ + int cycle; + int block; + struct xlog *log = to_xlog(kobject); + + spin_lock(&log->l_icloglock); + cycle = log->l_curr_cycle; + block = log->l_curr_block; + spin_unlock(&log->l_icloglock); + + return sysfs_emit(buf, "%d:%d\n", cycle, block); +} +XFS_SYSFS_ATTR_RO(log_head_lsn); + +STATIC ssize_t +log_tail_lsn_show( + struct kobject *kobject, + char *buf) +{ + int cycle; + int block; + struct xlog *log = to_xlog(kobject); + + xlog_crack_atomic_lsn(&log->l_tail_lsn, &cycle, &block); + return sysfs_emit(buf, "%d:%d\n", cycle, block); +} +XFS_SYSFS_ATTR_RO(log_tail_lsn); + +STATIC ssize_t +reserve_grant_head_bytes_show( + struct kobject *kobject, + char *buf) +{ + return sysfs_emit(buf, "%lld\n", + atomic64_read(&to_xlog(kobject)->l_reserve_head.grant)); +} +XFS_SYSFS_ATTR_RO(reserve_grant_head_bytes); + +STATIC ssize_t +write_grant_head_bytes_show( + struct kobject *kobject, + char *buf) +{ + return sysfs_emit(buf, "%lld\n", + atomic64_read(&to_xlog(kobject)->l_write_head.grant)); +} +XFS_SYSFS_ATTR_RO(write_grant_head_bytes); + +static struct attribute *xfs_log_attrs[] = { + ATTR_LIST(log_head_lsn), + ATTR_LIST(log_tail_lsn), + ATTR_LIST(reserve_grant_head_bytes), + ATTR_LIST(write_grant_head_bytes), + NULL, +}; +ATTRIBUTE_GROUPS(xfs_log); + +const struct kobj_type xfs_log_ktype = { + .release = xfs_sysfs_release, + .sysfs_ops = &xfs_sysfs_ops, + .default_groups = xfs_log_groups, +}; + +/* + * Metadata IO error configuration + * + * The sysfs structure here is: + * ...xfs//error/// + * + * where allows us to discriminate between data IO and metadata IO, + * and any other future type of IO (e.g. special inode or directory error + * handling) we care to support. + */ +static inline struct xfs_error_cfg * +to_error_cfg(struct kobject *kobject) +{ + struct xfs_kobj *kobj = to_kobj(kobject); + return container_of(kobj, struct xfs_error_cfg, kobj); +} + +static inline struct xfs_mount * +err_to_mp(struct kobject *kobject) +{ + struct xfs_kobj *kobj = to_kobj(kobject); + return container_of(kobj, struct xfs_mount, m_error_kobj); +} + +static ssize_t +max_retries_show( + struct kobject *kobject, + char *buf) +{ + int retries; + struct xfs_error_cfg *cfg = to_error_cfg(kobject); + + if (cfg->max_retries == XFS_ERR_RETRY_FOREVER) + retries = -1; + else + retries = cfg->max_retries; + + return sysfs_emit(buf, "%d\n", retries); +} + +static ssize_t +max_retries_store( + struct kobject *kobject, + const char *buf, + size_t count) +{ + struct xfs_error_cfg *cfg = to_error_cfg(kobject); + int ret; + int val; + + ret = kstrtoint(buf, 0, &val); + if (ret) + return ret; + + if (val < -1) + return -EINVAL; + + if (val == -1) + cfg->max_retries = XFS_ERR_RETRY_FOREVER; + else + cfg->max_retries = val; + return count; +} +XFS_SYSFS_ATTR_RW(max_retries); + +static ssize_t +retry_timeout_seconds_show( + struct kobject *kobject, + char *buf) +{ + int timeout; + struct xfs_error_cfg *cfg = to_error_cfg(kobject); + + if (cfg->retry_timeout == XFS_ERR_RETRY_FOREVER) + timeout = -1; + else + timeout = jiffies_to_msecs(cfg->retry_timeout) / MSEC_PER_SEC; + + return sysfs_emit(buf, "%d\n", timeout); +} + +static ssize_t +retry_timeout_seconds_store( + struct kobject *kobject, + const char *buf, + size_t count) +{ + struct xfs_error_cfg *cfg = to_error_cfg(kobject); + int ret; + int val; + + ret = kstrtoint(buf, 0, &val); + if (ret) + return ret; + + /* 1 day timeout maximum, -1 means infinite */ + if (val < -1 || val > 86400) + return -EINVAL; + + if (val == -1) + cfg->retry_timeout = XFS_ERR_RETRY_FOREVER; + else { + cfg->retry_timeout = secs_to_jiffies(val); + ASSERT(secs_to_jiffies(val) < LONG_MAX); + } + return count; +} +XFS_SYSFS_ATTR_RW(retry_timeout_seconds); + +static ssize_t +fail_at_unmount_show( + struct kobject *kobject, + char *buf) +{ + struct xfs_mount *mp = err_to_mp(kobject); + + return sysfs_emit(buf, "%d\n", mp->m_fail_unmount); +} + +static ssize_t +fail_at_unmount_store( + struct kobject *kobject, + const char *buf, + size_t count) +{ + struct xfs_mount *mp = err_to_mp(kobject); + int ret; + int val; + + ret = kstrtoint(buf, 0, &val); + if (ret) + return ret; + + if (val < 0 || val > 1) + return -EINVAL; + + mp->m_fail_unmount = val; + return count; +} +XFS_SYSFS_ATTR_RW(fail_at_unmount); + +static struct attribute *xfs_error_attrs[] = { + ATTR_LIST(max_retries), + ATTR_LIST(retry_timeout_seconds), + NULL, +}; +ATTRIBUTE_GROUPS(xfs_error); + +static const struct kobj_type xfs_error_cfg_ktype = { + .release = xfs_sysfs_release, + .sysfs_ops = &xfs_sysfs_ops, + .default_groups = xfs_error_groups, +}; + +static const struct kobj_type xfs_error_ktype = { + .release = xfs_sysfs_release, + .sysfs_ops = &xfs_sysfs_ops, +}; + +/* + * Error initialization tables. These need to be ordered in the same + * order as the enums used to index the array. All class init tables need to + * define a "default" behaviour as the first entry, all other entries can be + * empty. + */ +struct xfs_error_init { + char *name; + int max_retries; + int retry_timeout; /* in seconds */ +}; + +static const struct xfs_error_init xfs_error_meta_init[XFS_ERR_ERRNO_MAX] = { + { .name = "default", + .max_retries = XFS_ERR_RETRY_FOREVER, + .retry_timeout = XFS_ERR_RETRY_FOREVER, + }, + { .name = "EIO", + .max_retries = XFS_ERR_RETRY_FOREVER, + .retry_timeout = XFS_ERR_RETRY_FOREVER, + }, + { .name = "ENOSPC", + .max_retries = XFS_ERR_RETRY_FOREVER, + .retry_timeout = XFS_ERR_RETRY_FOREVER, + }, + { .name = "ENODEV", + .max_retries = 0, /* We can't recover from devices disappearing */ + .retry_timeout = 0, + }, +}; + +static int +xfs_error_sysfs_init_class( + struct xfs_mount *mp, + int class, + const char *parent_name, + struct xfs_kobj *parent_kobj, + const struct xfs_error_init init[]) +{ + struct xfs_error_cfg *cfg; + int error; + int i; + + ASSERT(class < XFS_ERR_CLASS_MAX); + + error = xfs_sysfs_init(parent_kobj, &xfs_error_ktype, + &mp->m_error_kobj, parent_name); + if (error) + return error; + + for (i = 0; i < XFS_ERR_ERRNO_MAX; i++) { + cfg = &mp->m_error_cfg[class][i]; + error = xfs_sysfs_init(&cfg->kobj, &xfs_error_cfg_ktype, + parent_kobj, init[i].name); + if (error) + goto out_error; + + cfg->max_retries = init[i].max_retries; + if (init[i].retry_timeout == XFS_ERR_RETRY_FOREVER) + cfg->retry_timeout = XFS_ERR_RETRY_FOREVER; + else + cfg->retry_timeout = + secs_to_jiffies(init[i].retry_timeout); + } + return 0; + +out_error: + /* unwind the entries that succeeded */ + for (i--; i >= 0; i--) { + cfg = &mp->m_error_cfg[class][i]; + xfs_sysfs_del(&cfg->kobj); + } + xfs_sysfs_del(parent_kobj); + return error; +} + +static inline struct xfs_mount *zoned_to_mp(struct kobject *kobj) +{ + return container_of(to_kobj(kobj), struct xfs_mount, m_zoned_kobj); +} + +static ssize_t +max_open_zones_show( + struct kobject *kobj, + char *buf) +{ + /* only report the open zones available for user data */ + return sysfs_emit(buf, "%u\n", + zoned_to_mp(kobj)->m_max_open_zones - XFS_OPEN_GC_ZONES); +} +XFS_SYSFS_ATTR_RO(max_open_zones); + +static ssize_t +nr_open_zones_show( + struct kobject *kobj, + char *buf) +{ + struct xfs_zone_info *zi = zoned_to_mp(kobj)->m_zone_info; + + return sysfs_emit(buf, "%u\n", READ_ONCE(zi->zi_nr_open_zones)); +} +XFS_SYSFS_ATTR_RO(nr_open_zones); + +static ssize_t +zonegc_low_space_store( + struct kobject *kobj, + const char *buf, + size_t count) +{ + struct xfs_mount *mp = zoned_to_mp(kobj); + int ret; + unsigned int val; + + ret = kstrtouint(buf, 0, &val); + if (ret) + return ret; + + if (val > 100) + return -EINVAL; + + if (mp->m_zonegc_low_space != val) { + mp->m_zonegc_low_space = val; + xfs_zone_gc_wakeup(mp); + } + + return count; +} + +static ssize_t +zonegc_low_space_show( + struct kobject *kobj, + char *buf) +{ + return sysfs_emit(buf, "%u\n", + zoned_to_mp(kobj)->m_zonegc_low_space); +} +XFS_SYSFS_ATTR_RW(zonegc_low_space); + +static struct attribute *xfs_zoned_attrs[] = { + ATTR_LIST(max_open_zones), + ATTR_LIST(nr_open_zones), + ATTR_LIST(zonegc_low_space), + NULL, +}; +ATTRIBUTE_GROUPS(xfs_zoned); + +static const struct kobj_type xfs_zoned_ktype = { + .release = xfs_sysfs_release, + .sysfs_ops = &xfs_sysfs_ops, + .default_groups = xfs_zoned_groups, +}; + +int +xfs_zoned_sysfs_init(struct xfs_mount *mp) +{ + if (!IS_ENABLED(CONFIG_XFS_RT) || !xfs_has_zoned(mp)) + return 0; + + return xfs_sysfs_init(&mp->m_zoned_kobj, &xfs_zoned_ktype, + &mp->m_kobj, "zoned"); +} + +void +xfs_zoned_sysfs_del(struct xfs_mount *mp) +{ + if (IS_ENABLED(CONFIG_XFS_RT) && xfs_has_zoned(mp)) + xfs_sysfs_del(&mp->m_zoned_kobj); +} + +int +xfs_mount_sysfs_init( + struct xfs_mount *mp) +{ + int error; + + super_set_sysfs_name_id(mp->m_super); + + /* .../xfs// */ + error = xfs_sysfs_init(&mp->m_kobj, &xfs_mp_ktype, + NULL, mp->m_super->s_id); + if (error) + return error; + + /* .../xfs//stats/ */ + error = xfs_sysfs_init(&mp->m_stats.xs_kobj, &xfs_stats_ktype, + &mp->m_kobj, "stats"); + if (error) + goto out_remove_fsdir; + + /* .../xfs//error/ */ + error = xfs_sysfs_init(&mp->m_error_kobj, &xfs_error_ktype, + &mp->m_kobj, "error"); + if (error) + goto out_remove_stats_dir; + + /* .../xfs//error/fail_at_unmount */ + error = sysfs_create_file(&mp->m_error_kobj.kobject, + ATTR_LIST(fail_at_unmount)); + + if (error) + goto out_remove_error_dir; + + /* .../xfs//error/metadata/ */ + error = xfs_error_sysfs_init_class(mp, XFS_ERR_METADATA, + "metadata", &mp->m_error_meta_kobj, + xfs_error_meta_init); + if (error) + goto out_remove_error_dir; + + return 0; + +out_remove_error_dir: + xfs_sysfs_del(&mp->m_error_kobj); +out_remove_stats_dir: + xfs_sysfs_del(&mp->m_stats.xs_kobj); +out_remove_fsdir: + xfs_sysfs_del(&mp->m_kobj); + return error; +} + +void +xfs_mount_sysfs_del( + struct xfs_mount *mp) +{ + struct xfs_error_cfg *cfg; + int i, j; + + for (i = 0; i < XFS_ERR_CLASS_MAX; i++) { + for (j = 0; j < XFS_ERR_ERRNO_MAX; j++) { + cfg = &mp->m_error_cfg[i][j]; + + xfs_sysfs_del(&cfg->kobj); + } + } + xfs_sysfs_del(&mp->m_error_meta_kobj); + xfs_sysfs_del(&mp->m_error_kobj); + xfs_sysfs_del(&mp->m_stats.xs_kobj); + xfs_sysfs_del(&mp->m_kobj); +} + +struct xfs_error_cfg * +xfs_error_get_cfg( + struct xfs_mount *mp, + int error_class, + int error) +{ + struct xfs_error_cfg *cfg; + + if (error < 0) + error = -error; + + switch (error) { + case EIO: + cfg = &mp->m_error_cfg[error_class][XFS_ERR_EIO]; + break; + case ENOSPC: + cfg = &mp->m_error_cfg[error_class][XFS_ERR_ENOSPC]; + break; + case ENODEV: + cfg = &mp->m_error_cfg[error_class][XFS_ERR_ENODEV]; + break; + default: + cfg = &mp->m_error_cfg[error_class][XFS_ERR_DEFAULT]; + break; + } + + return cfg; +} diff --git a/fs/xfs/xfs_sysfs.h b/fs/xfs/xfs_sysfs.h new file mode 100644 index 000000000..25e5f8fae --- /dev/null +++ b/fs/xfs/xfs_sysfs.h @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2014 Red Hat, Inc. + * All Rights Reserved. + */ + +#ifndef __XFS_SYSFS_H__ +#define __XFS_SYSFS_H__ + +extern const struct kobj_type xfs_dbg_ktype; /* debug */ +extern const struct kobj_type xfs_log_ktype; /* xlog */ +extern const struct kobj_type xfs_stats_ktype; /* stats */ + +static inline struct xfs_kobj * +to_kobj(struct kobject *kobject) +{ + return container_of(kobject, struct xfs_kobj, kobject); +} + +static inline void +xfs_sysfs_release(struct kobject *kobject) +{ + struct xfs_kobj *kobj = to_kobj(kobject); + complete(&kobj->complete); +} + +static inline int +xfs_sysfs_init( + struct xfs_kobj *kobj, + const struct kobj_type *ktype, + struct xfs_kobj *parent_kobj, + const char *name) +{ + struct kobject *parent; + int err; + + parent = parent_kobj ? &parent_kobj->kobject : NULL; + init_completion(&kobj->complete); + err = kobject_init_and_add(&kobj->kobject, ktype, parent, "%s", name); + if (err) + kobject_put(&kobj->kobject); + + return err; +} + +static inline void +xfs_sysfs_del( + struct xfs_kobj *kobj) +{ + kobject_del(&kobj->kobject); + kobject_put(&kobj->kobject); + wait_for_completion(&kobj->complete); +} + +int xfs_mount_sysfs_init(struct xfs_mount *mp); +int xfs_zoned_sysfs_init(struct xfs_mount *mp); +void xfs_zoned_sysfs_del(struct xfs_mount *mp); +void xfs_mount_sysfs_del(struct xfs_mount *mp); + +#endif /* __XFS_SYSFS_H__ */ diff --git a/fs/xfs/xfs_trace.c b/fs/xfs/xfs_trace.c new file mode 100644 index 000000000..912713a8a --- /dev/null +++ b/fs/xfs/xfs_trace.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2009, Christoph Hellwig + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_bit.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_group.h" +#include "xfs_defer.h" +#include "xfs_da_format.h" +#include "xfs_inode.h" +#include "xfs_btree.h" +#include "xfs_da_btree.h" +#include "xfs_alloc.h" +#include "xfs_bmap.h" +#include "xfs_attr.h" +#include "xfs_trans.h" +#include "xfs_log.h" +#include "xfs_log_priv.h" +#include "xfs_trans_priv.h" +#include "xfs_buf_item.h" +#include "xfs_quota.h" +#include "xfs_dquot_item.h" +#include "xfs_dquot.h" +#include "xfs_log_recover.h" +#include "xfs_filestream.h" +#include "xfs_fsmap.h" +#include "xfs_btree_staging.h" +#include "xfs_icache.h" +#include "xfs_iunlink_item.h" +#include "xfs_ag.h" +#include "xfs_ag_resv.h" +#include "xfs_error.h" +#include +#include "xfs_iomap.h" +#include "xfs_buf_mem.h" +#include "xfs_btree_mem.h" +#include "xfs_exchmaps.h" +#include "xfs_exchrange.h" +#include "xfs_parent.h" +#include "xfs_rmap.h" +#include "xfs_refcount.h" +#include "xfs_metafile.h" +#include "xfs_metadir.h" +#include "xfs_rtgroup.h" +#include "xfs_zone_alloc.h" +#include "xfs_zone_priv.h" +#include "xfs_health.h" +#include "xfs_healthmon.h" +#include "xfs_notify_failure.h" +#include "xfs_file.h" +#include + +/* + * We include this last to have the helpers above available for the trace + * event implementations. + */ +#define CREATE_TRACE_POINTS +#include "xfs_trace.h" diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h new file mode 100644 index 000000000..f333c938f --- /dev/null +++ b/fs/xfs/xfs_trace.h @@ -0,0 +1,6488 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2009, Christoph Hellwig + * All Rights Reserved. + * + * NOTE: none of these tracepoints shall be considered a stable kernel ABI + * as they can change at any time. + * + * Current conventions for printing numbers measuring specific units: + * + * agno: allocation group number + * + * agino: per-AG inode number + * ino: filesystem inode number + * + * agbno: per-AG block number in fs blocks + * rgbno: per-rtgroup block number in fs blocks + * startblock: physical block number for file mappings. This is either a + * segmented fsblock for data device mappings, or a rfsblock + * for realtime device mappings + * fsbcount: number of blocks in an extent, in fs blocks + * + * gbno: generic allocation group block number. This is an agbno for + * space in a per-AG or a rgbno for space in a realtime group. + * + * daddr: physical block number in 512b blocks + * bbcount: number of blocks in a physical extent, in 512b blocks + * + * rtx: physical rt extent number for extent mappings + * rtxcount: number of rt extents in an extent mapping + * + * owner: reverse-mapping owner, usually inodes + * + * fileoff: file offset, in fs blocks + * pos: file offset, in bytes + * bytecount: number of bytes + * + * dablk: directory or xattr block offset, in filesystem blocks + * + * disize: ondisk file size, in bytes + * isize: incore file size, in bytes + * + * forkoff: inode fork offset, in bytes + * + * ireccount: number of inode records + * + * Numbers describing space allocations (blocks, extents, inodes) should be + * formatted in hexadecimal. + */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM xfs + +#if !defined(_TRACE_XFS_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_XFS_H + +#include + +struct xfs_agf; +struct xfs_ail; +struct xfs_alloc_arg; +struct xfs_attr_list_context; +struct xfs_buf_log_item; +struct xfs_da_args; +struct xfs_da_node_entry; +struct xfs_dquot; +struct xfs_log_item; +struct xlog; +struct xlog_ticket; +struct xlog_recover; +struct xlog_recover_item; +struct xlog_rec_header; +struct xlog_in_core; +struct xfs_buf_log_format; +struct xfs_inode_log_format; +struct xfs_bmbt_irec; +struct xfs_btree_cur; +struct xfs_defer_op_type; +struct xfs_refcount_irec; +struct xfs_fsmap; +struct xfs_fsmap_irec; +struct xfs_group; +struct xfs_rmap_irec; +struct xfs_icreate_log; +struct xfs_iunlink_item; +struct xfs_owner_info; +struct xfs_trans_res; +struct xfs_inobt_rec_incore; +union xfs_btree_ptr; +struct xfs_dqtrx; +struct xfs_icwalk; +struct xfs_perag; +struct xfbtree; +struct xfs_btree_ops; +struct xfs_bmap_intent; +struct xfs_exchmaps_intent; +struct xfs_exchmaps_req; +struct xfs_exchrange; +struct xfs_getparents; +struct xfs_parent_irec; +struct xfs_attrlist_cursor_kern; +struct xfs_extent_free_item; +struct xfs_rmap_intent; +struct xfs_refcount_intent; +struct xfs_metadir_update; +struct xfs_rtgroup; +struct xfs_open_zone; +struct xfs_healthmon_event; +struct xfs_healthmon; +struct fserror_event; + +#define XFS_ATTR_FILTER_FLAGS \ + { XFS_ATTR_ROOT, "ROOT" }, \ + { XFS_ATTR_SECURE, "SECURE" }, \ + { XFS_ATTR_INCOMPLETE, "INCOMPLETE" }, \ + { XFS_ATTR_PARENT, "PARENT" } + +DECLARE_EVENT_CLASS(xfs_attr_list_class, + TP_PROTO(struct xfs_attr_list_context *ctx), + TP_ARGS(ctx), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(u32, hashval) + __field(u32, blkno) + __field(u32, offset) + __field(void *, buffer) + __field(int, bufsize) + __field(int, count) + __field(int, firstu) + __field(int, dupcnt) + __field(unsigned int, attr_filter) + ), + TP_fast_assign( + __entry->dev = VFS_I(ctx->dp)->i_sb->s_dev; + __entry->ino = I_INO(ctx->dp); + __entry->hashval = ctx->cursor.hashval; + __entry->blkno = ctx->cursor.blkno; + __entry->offset = ctx->cursor.offset; + __entry->buffer = ctx->buffer; + __entry->bufsize = ctx->bufsize; + __entry->count = ctx->count; + __entry->firstu = ctx->firstu; + __entry->attr_filter = ctx->attr_filter; + ), + TP_printk("dev %d:%d ino 0x%llx cursor h/b/o 0x%x/0x%x/%u dupcnt %u " + "buffer %p size %u count %u firstu %u filter %s", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->hashval, + __entry->blkno, + __entry->offset, + __entry->dupcnt, + __entry->buffer, + __entry->bufsize, + __entry->count, + __entry->firstu, + __print_flags(__entry->attr_filter, "|", + XFS_ATTR_FILTER_FLAGS) + ) +) + +#define DEFINE_ATTR_LIST_EVENT(name) \ +DEFINE_EVENT(xfs_attr_list_class, name, \ + TP_PROTO(struct xfs_attr_list_context *ctx), \ + TP_ARGS(ctx)) +DEFINE_ATTR_LIST_EVENT(xfs_attr_list_sf); +DEFINE_ATTR_LIST_EVENT(xfs_attr_list_sf_all); +DEFINE_ATTR_LIST_EVENT(xfs_attr_list_leaf); +DEFINE_ATTR_LIST_EVENT(xfs_attr_list_leaf_end); +DEFINE_ATTR_LIST_EVENT(xfs_attr_list_full); +DEFINE_ATTR_LIST_EVENT(xfs_attr_list_add); +DEFINE_ATTR_LIST_EVENT(xfs_attr_list_wrong_blk); +DEFINE_ATTR_LIST_EVENT(xfs_attr_list_notfound); +DEFINE_ATTR_LIST_EVENT(xfs_attr_leaf_list); +DEFINE_ATTR_LIST_EVENT(xfs_attr_node_list); + +TRACE_EVENT(xfs_calc_atomic_write_unit_max, + TP_PROTO(struct xfs_mount *mp, enum xfs_group_type type, + unsigned int max_write, unsigned int max_ioend, + unsigned int max_gsize, unsigned int awu_max), + TP_ARGS(mp, type, max_write, max_ioend, max_gsize, awu_max), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(enum xfs_group_type, type) + __field(unsigned int, max_write) + __field(unsigned int, max_ioend) + __field(unsigned int, max_gsize) + __field(unsigned int, awu_max) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->type = type; + __entry->max_write = max_write; + __entry->max_ioend = max_ioend; + __entry->max_gsize = max_gsize; + __entry->awu_max = awu_max; + ), + TP_printk("dev %d:%d %s max_write %u max_ioend %u max_gsize %u awu_max %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->max_write, + __entry->max_ioend, + __entry->max_gsize, + __entry->awu_max) +); + +TRACE_EVENT(xfs_calc_max_atomic_write_fsblocks, + TP_PROTO(struct xfs_mount *mp, unsigned int per_intent, + unsigned int step_size, unsigned int logres, + unsigned int blockcount), + TP_ARGS(mp, per_intent, step_size, logres, blockcount), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, per_intent) + __field(unsigned int, step_size) + __field(unsigned int, logres) + __field(unsigned int, blockcount) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->per_intent = per_intent; + __entry->step_size = step_size; + __entry->logres = logres; + __entry->blockcount = blockcount; + ), + TP_printk("dev %d:%d per_intent %u step_size %u logres %u blockcount %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->per_intent, + __entry->step_size, + __entry->logres, + __entry->blockcount) +); + +TRACE_EVENT(xfs_calc_max_atomic_write_log_geometry, + TP_PROTO(struct xfs_mount *mp, unsigned int per_intent, + unsigned int step_size, unsigned int blockcount, + unsigned int min_logblocks, unsigned int logres), + TP_ARGS(mp, per_intent, step_size, blockcount, min_logblocks, logres), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, per_intent) + __field(unsigned int, step_size) + __field(unsigned int, blockcount) + __field(unsigned int, min_logblocks) + __field(unsigned int, cur_logblocks) + __field(unsigned int, logres) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->per_intent = per_intent; + __entry->step_size = step_size; + __entry->blockcount = blockcount; + __entry->min_logblocks = min_logblocks; + __entry->cur_logblocks = mp->m_sb.sb_logblocks; + __entry->logres = logres; + ), + TP_printk("dev %d:%d per_intent %u step_size %u blockcount %u min_logblocks %u logblocks %u logres %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->per_intent, + __entry->step_size, + __entry->blockcount, + __entry->min_logblocks, + __entry->cur_logblocks, + __entry->logres) +); + +TRACE_EVENT(xlog_intent_recovery_failed, + TP_PROTO(struct xfs_mount *mp, const struct xfs_defer_op_type *ops, + int error), + TP_ARGS(mp, ops, error), + TP_STRUCT__entry( + __field(dev_t, dev) + __string(name, ops->name) + __field(int, error) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __assign_str(name); + __entry->error = error; + ), + TP_printk("dev %d:%d optype %s error %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __get_str(name), + __entry->error) +); + +DECLARE_EVENT_CLASS(xfs_perag_class, + TP_PROTO(const struct xfs_perag *pag, unsigned long caller_ip), + TP_ARGS(pag, caller_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(int, refcount) + __field(int, active_refcount) + __field(unsigned long, caller_ip) + ), + TP_fast_assign( + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->agno = pag_agno(pag); + __entry->refcount = atomic_read(&pag->pag_group.xg_ref); + __entry->active_refcount = + atomic_read(&pag->pag_group.xg_active_ref); + __entry->caller_ip = caller_ip; + ), + TP_printk("dev %d:%d agno 0x%x passive refs %d active refs %d caller %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->refcount, + __entry->active_refcount, + (char *)__entry->caller_ip) +); + +#define DEFINE_PERAG_REF_EVENT(name) \ +DEFINE_EVENT(xfs_perag_class, name, \ + TP_PROTO(const struct xfs_perag *pag, unsigned long caller_ip), \ + TP_ARGS(pag, caller_ip)) +DEFINE_PERAG_REF_EVENT(xfs_perag_set_inode_tag); +DEFINE_PERAG_REF_EVENT(xfs_perag_clear_inode_tag); +DEFINE_PERAG_REF_EVENT(xfs_reclaim_inodes_count); + +TRACE_DEFINE_ENUM(XG_TYPE_AG); +TRACE_DEFINE_ENUM(XG_TYPE_RTG); + +DECLARE_EVENT_CLASS(xfs_group_class, + TP_PROTO(struct xfs_group *xg, unsigned long caller_ip), + TP_ARGS(xg, caller_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(enum xfs_group_type, type) + __field(xfs_agnumber_t, agno) + __field(int, refcount) + __field(int, active_refcount) + __field(unsigned long, caller_ip) + ), + TP_fast_assign( + __entry->dev = xg->xg_mount->m_super->s_dev; + __entry->type = xg->xg_type; + __entry->agno = xg->xg_gno; + __entry->refcount = atomic_read(&xg->xg_ref); + __entry->active_refcount = atomic_read(&xg->xg_active_ref); + __entry->caller_ip = caller_ip; + ), + TP_printk("dev %d:%d %sno 0x%x passive refs %d active refs %d caller %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agno, + __entry->refcount, + __entry->active_refcount, + (char *)__entry->caller_ip) +); + +#define DEFINE_GROUP_REF_EVENT(name) \ +DEFINE_EVENT(xfs_group_class, name, \ + TP_PROTO(struct xfs_group *xg, unsigned long caller_ip), \ + TP_ARGS(xg, caller_ip)) +DEFINE_GROUP_REF_EVENT(xfs_group_get); +DEFINE_GROUP_REF_EVENT(xfs_group_hold); +DEFINE_GROUP_REF_EVENT(xfs_group_put); +DEFINE_GROUP_REF_EVENT(xfs_group_grab); +DEFINE_GROUP_REF_EVENT(xfs_group_grab_next_tag); +DEFINE_GROUP_REF_EVENT(xfs_group_rele); + +#ifdef CONFIG_XFS_RT +DECLARE_EVENT_CLASS(xfs_zone_class, + TP_PROTO(struct xfs_rtgroup *rtg), + TP_ARGS(rtg), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_rgnumber_t, rgno) + __field(xfs_rgblock_t, used) + __field(unsigned int, nr_open) + ), + TP_fast_assign( + struct xfs_mount *mp = rtg_mount(rtg); + + __entry->dev = mp->m_super->s_dev; + __entry->rgno = rtg_rgno(rtg); + __entry->used = rtg_rmap(rtg)->i_used_blocks; + __entry->nr_open = mp->m_zone_info->zi_nr_open_zones; + ), + TP_printk("dev %d:%d rgno 0x%x used 0x%x nr_open %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->rgno, + __entry->used, + __entry->nr_open) +); + +#define DEFINE_ZONE_EVENT(name) \ +DEFINE_EVENT(xfs_zone_class, name, \ + TP_PROTO(struct xfs_rtgroup *rtg), \ + TP_ARGS(rtg)) +DEFINE_ZONE_EVENT(xfs_zone_emptied); +DEFINE_ZONE_EVENT(xfs_zone_full); +DEFINE_ZONE_EVENT(xfs_zone_opened); +DEFINE_ZONE_EVENT(xfs_zone_reset); +DEFINE_ZONE_EVENT(xfs_zone_gc_target_opened); +DEFINE_ZONE_EVENT(xfs_zone_gc_target_stolen); + +TRACE_EVENT(xfs_zone_free_blocks, + TP_PROTO(struct xfs_rtgroup *rtg, xfs_rgblock_t rgbno, + xfs_extlen_t len), + TP_ARGS(rtg, rgbno, len), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_rgnumber_t, rgno) + __field(xfs_rgblock_t, used) + __field(xfs_rgblock_t, rgbno) + __field(xfs_extlen_t, len) + ), + TP_fast_assign( + __entry->dev = rtg_mount(rtg)->m_super->s_dev; + __entry->rgno = rtg_rgno(rtg); + __entry->used = rtg_rmap(rtg)->i_used_blocks; + __entry->rgbno = rgbno; + __entry->len = len; + ), + TP_printk("dev %d:%d rgno 0x%x used 0x%x rgbno 0x%x len 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->rgno, + __entry->used, + __entry->rgbno, + __entry->len) +); + +DECLARE_EVENT_CLASS(xfs_zone_alloc_class, + TP_PROTO(struct xfs_open_zone *oz, xfs_rgblock_t rgbno, + xfs_extlen_t len), + TP_ARGS(oz, rgbno, len), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_rgnumber_t, rgno) + __field(xfs_rgblock_t, used) + __field(xfs_rgblock_t, allocated) + __field(xfs_rgblock_t, written) + __field(xfs_rgblock_t, rgbno) + __field(xfs_extlen_t, len) + ), + TP_fast_assign( + __entry->dev = rtg_mount(oz->oz_rtg)->m_super->s_dev; + __entry->rgno = rtg_rgno(oz->oz_rtg); + __entry->used = rtg_rmap(oz->oz_rtg)->i_used_blocks; + __entry->allocated = oz->oz_allocated; + __entry->written = oz->oz_written; + __entry->rgbno = rgbno; + __entry->len = len; + ), + TP_printk("dev %d:%d rgno 0x%x used 0x%x alloced 0x%x written 0x%x rgbno 0x%x len 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->rgno, + __entry->used, + __entry->allocated, + __entry->written, + __entry->rgbno, + __entry->len) +); + +#define DEFINE_ZONE_ALLOC_EVENT(name) \ +DEFINE_EVENT(xfs_zone_alloc_class, name, \ + TP_PROTO(struct xfs_open_zone *oz, xfs_rgblock_t rgbno, \ + xfs_extlen_t len), \ + TP_ARGS(oz, rgbno, len)) +DEFINE_ZONE_ALLOC_EVENT(xfs_zone_record_blocks); +DEFINE_ZONE_ALLOC_EVENT(xfs_zone_skip_blocks); +DEFINE_ZONE_ALLOC_EVENT(xfs_zone_alloc_blocks); +DEFINE_ZONE_ALLOC_EVENT(xfs_zone_spurious_open); + +TRACE_EVENT(xfs_zone_gc_select_victim, + TP_PROTO(struct xfs_rtgroup *rtg, unsigned int bucket), + TP_ARGS(rtg, bucket), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_rgnumber_t, rgno) + __field(xfs_rgblock_t, used) + __field(unsigned int, bucket) + ), + TP_fast_assign( + __entry->dev = rtg_mount(rtg)->m_super->s_dev; + __entry->rgno = rtg_rgno(rtg); + __entry->used = rtg_rmap(rtg)->i_used_blocks; + __entry->bucket = bucket; + ), + TP_printk("dev %d:%d rgno 0x%x used 0x%x bucket %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->rgno, + __entry->used, + __entry->bucket) +); + +TRACE_EVENT(xfs_zones_mount, + TP_PROTO(struct xfs_mount *mp), + TP_ARGS(mp), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_rgnumber_t, rgcount) + __field(uint32_t, blocks) + __field(unsigned int, max_open_zones) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->rgcount = mp->m_sb.sb_rgcount; + __entry->blocks = mp->m_groups[XG_TYPE_RTG].blocks; + __entry->max_open_zones = mp->m_max_open_zones; + ), + TP_printk("dev %d:%d zoned %u blocks_per_zone %u, max_open %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->rgcount, + __entry->blocks, + __entry->max_open_zones) +); +#endif /* CONFIG_XFS_RT */ + +TRACE_EVENT(xfs_inodegc_worker, + TP_PROTO(struct xfs_mount *mp, unsigned int shrinker_hits), + TP_ARGS(mp, shrinker_hits), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, shrinker_hits) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->shrinker_hits = shrinker_hits; + ), + TP_printk("dev %d:%d shrinker_hits %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->shrinker_hits) +); + +DECLARE_EVENT_CLASS(xfs_fs_class, + TP_PROTO(struct xfs_mount *mp, void *caller_ip), + TP_ARGS(mp, caller_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned long long, mflags) + __field(unsigned long, opstate) + __field(unsigned long, sbflags) + __field(void *, caller_ip) + ), + TP_fast_assign( + if (mp) { + __entry->dev = mp->m_super->s_dev; + __entry->mflags = mp->m_features; + __entry->opstate = mp->m_opstate; + __entry->sbflags = mp->m_super->s_flags; + } + __entry->caller_ip = caller_ip; + ), + TP_printk("dev %d:%d m_features 0x%llx opstate (%s) s_flags 0x%lx caller %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->mflags, + __print_flags(__entry->opstate, "|", XFS_OPSTATE_STRINGS), + __entry->sbflags, + __entry->caller_ip) +); + +#define DEFINE_FS_EVENT(name) \ +DEFINE_EVENT(xfs_fs_class, name, \ + TP_PROTO(struct xfs_mount *mp, void *caller_ip), \ + TP_ARGS(mp, caller_ip)) +DEFINE_FS_EVENT(xfs_inodegc_flush); +DEFINE_FS_EVENT(xfs_inodegc_push); +DEFINE_FS_EVENT(xfs_inodegc_start); +DEFINE_FS_EVENT(xfs_inodegc_stop); +DEFINE_FS_EVENT(xfs_inodegc_queue); +DEFINE_FS_EVENT(xfs_inodegc_throttle); +DEFINE_FS_EVENT(xfs_fs_sync_fs); +DEFINE_FS_EVENT(xfs_blockgc_start); +DEFINE_FS_EVENT(xfs_blockgc_stop); +DEFINE_FS_EVENT(xfs_blockgc_worker); +DEFINE_FS_EVENT(xfs_blockgc_flush_all); + +TRACE_EVENT(xfs_inodegc_shrinker_scan, + TP_PROTO(struct xfs_mount *mp, struct shrink_control *sc, + void *caller_ip), + TP_ARGS(mp, sc, caller_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned long, nr_to_scan) + __field(void *, caller_ip) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->nr_to_scan = sc->nr_to_scan; + __entry->caller_ip = caller_ip; + ), + TP_printk("dev %d:%d nr_to_scan %lu caller %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->nr_to_scan, + __entry->caller_ip) +); + +DECLARE_EVENT_CLASS(xfs_ag_class, + TP_PROTO(const struct xfs_perag *pag), + TP_ARGS(pag), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + ), + TP_fast_assign( + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->agno = pag_agno(pag); + ), + TP_printk("dev %d:%d agno 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno) +); +#define DEFINE_AG_EVENT(name) \ +DEFINE_EVENT(xfs_ag_class, name, \ + TP_PROTO(const struct xfs_perag *pag), \ + TP_ARGS(pag)) + +DEFINE_AG_EVENT(xfs_read_agf); +DEFINE_AG_EVENT(xfs_alloc_read_agf); +DEFINE_AG_EVENT(xfs_read_agi); +DEFINE_AG_EVENT(xfs_ialloc_read_agi); + +TRACE_EVENT(xfs_attr_list_node_descend, + TP_PROTO(struct xfs_attr_list_context *ctx, + struct xfs_da_node_entry *btree), + TP_ARGS(ctx, btree), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(u32, hashval) + __field(u32, blkno) + __field(u32, offset) + __field(void *, buffer) + __field(int, bufsize) + __field(int, count) + __field(int, firstu) + __field(int, dupcnt) + __field(unsigned int, attr_filter) + __field(u32, bt_hashval) + __field(u32, bt_before) + ), + TP_fast_assign( + __entry->dev = VFS_I(ctx->dp)->i_sb->s_dev; + __entry->ino = I_INO(ctx->dp); + __entry->hashval = ctx->cursor.hashval; + __entry->blkno = ctx->cursor.blkno; + __entry->offset = ctx->cursor.offset; + __entry->buffer = ctx->buffer; + __entry->bufsize = ctx->bufsize; + __entry->count = ctx->count; + __entry->firstu = ctx->firstu; + __entry->attr_filter = ctx->attr_filter; + __entry->bt_hashval = be32_to_cpu(btree->hashval); + __entry->bt_before = be32_to_cpu(btree->before); + ), + TP_printk("dev %d:%d ino 0x%llx cursor h/b/o 0x%x/0x%x/%u dupcnt %u " + "buffer %p size %u count %u firstu %u filter %s " + "node hashval %u, node before %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->hashval, + __entry->blkno, + __entry->offset, + __entry->dupcnt, + __entry->buffer, + __entry->bufsize, + __entry->count, + __entry->firstu, + __print_flags(__entry->attr_filter, "|", + XFS_ATTR_FILTER_FLAGS), + __entry->bt_hashval, + __entry->bt_before) +); + +DECLARE_EVENT_CLASS(xfs_bmap_class, + TP_PROTO(struct xfs_inode *ip, struct xfs_iext_cursor *cur, int state, + unsigned long caller_ip), + TP_ARGS(ip, cur, state, caller_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(void *, leaf) + __field(int, pos) + __field(xfs_fileoff_t, startoff) + __field(xfs_fsblock_t, startblock) + __field(xfs_filblks_t, blockcount) + __field(xfs_exntst_t, state) + __field(int, bmap_state) + __field(unsigned long, caller_ip) + ), + TP_fast_assign( + struct xfs_ifork *ifp; + struct xfs_bmbt_irec r; + + ifp = xfs_iext_state_to_fork(ip, state); + xfs_iext_get_extent(ifp, cur, &r); + __entry->dev = VFS_I(ip)->i_sb->s_dev; + __entry->ino = I_INO(ip); + __entry->leaf = cur->leaf; + __entry->pos = cur->pos; + __entry->startoff = r.br_startoff; + __entry->startblock = r.br_startblock; + __entry->blockcount = r.br_blockcount; + __entry->state = r.br_state; + __entry->bmap_state = state; + __entry->caller_ip = caller_ip; + ), + TP_printk("dev %d:%d ino 0x%llx state %s cur %p/%d " + "fileoff 0x%llx startblock 0x%llx fsbcount 0x%llx flag %d caller %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __print_flags(__entry->bmap_state, "|", XFS_BMAP_EXT_FLAGS), + __entry->leaf, + __entry->pos, + __entry->startoff, + (int64_t)__entry->startblock, + __entry->blockcount, + __entry->state, + (char *)__entry->caller_ip) +) + +#define DEFINE_BMAP_EVENT(name) \ +DEFINE_EVENT(xfs_bmap_class, name, \ + TP_PROTO(struct xfs_inode *ip, struct xfs_iext_cursor *cur, int state, \ + unsigned long caller_ip), \ + TP_ARGS(ip, cur, state, caller_ip)) +DEFINE_BMAP_EVENT(xfs_iext_insert); +DEFINE_BMAP_EVENT(xfs_iext_remove); +DEFINE_BMAP_EVENT(xfs_bmap_pre_update); +DEFINE_BMAP_EVENT(xfs_bmap_post_update); +DEFINE_BMAP_EVENT(xfs_read_extent); +DEFINE_BMAP_EVENT(xfs_write_extent); + +DECLARE_EVENT_CLASS(xfs_buf_class, + TP_PROTO(struct xfs_buf *bp, unsigned long caller_ip), + TP_ARGS(bp, caller_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_daddr_t, bno) + __field(int, nblks) + __field(int, hold) + __field(int, pincount) + __field(unsigned, lockval) + __field(unsigned, flags) + __field(unsigned long, caller_ip) + __field(const void *, buf_ops) + ), + TP_fast_assign( + __entry->dev = bp->b_target->bt_dev; + __entry->bno = xfs_buf_daddr(bp); + __entry->nblks = bp->b_length; + __entry->hold = bp->b_lockref.count; + __entry->pincount = atomic_read(&bp->b_pin_count); + __entry->lockval = bp->b_sema.count; + __entry->flags = bp->b_flags; + __entry->caller_ip = caller_ip; + __entry->buf_ops = bp->b_ops; + ), + TP_printk("dev %d:%d daddr 0x%llx bbcount 0x%x hold %d pincount %d " + "lock %d flags %s bufops %pS caller %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->bno, + __entry->nblks, + __entry->hold, + __entry->pincount, + __entry->lockval, + __print_flags(__entry->flags, "|", XFS_BUF_FLAGS), + __entry->buf_ops, + (void *)__entry->caller_ip) +) + +#define DEFINE_BUF_EVENT(name) \ +DEFINE_EVENT(xfs_buf_class, name, \ + TP_PROTO(struct xfs_buf *bp, unsigned long caller_ip), \ + TP_ARGS(bp, caller_ip)) +DEFINE_BUF_EVENT(xfs_buf_init); +DEFINE_BUF_EVENT(xfs_buf_free); +DEFINE_BUF_EVENT(xfs_buf_hold); +DEFINE_BUF_EVENT(xfs_buf_rele); +DEFINE_BUF_EVENT(xfs_buf_iodone); +DEFINE_BUF_EVENT(xfs_buf_submit); +DEFINE_BUF_EVENT(xfs_buf_lock); +DEFINE_BUF_EVENT(xfs_buf_lock_done); +DEFINE_BUF_EVENT(xfs_buf_trylock_fail); +DEFINE_BUF_EVENT(xfs_buf_trylock); +DEFINE_BUF_EVENT(xfs_buf_unlock); +DEFINE_BUF_EVENT(xfs_buf_iowait); +DEFINE_BUF_EVENT(xfs_buf_iowait_done); +DEFINE_BUF_EVENT(xfs_buf_delwri_queue); +DEFINE_BUF_EVENT(xfs_buf_delwri_queued); +DEFINE_BUF_EVENT(xfs_buf_delwri_split); +DEFINE_BUF_EVENT(xfs_buf_get_uncached); +DEFINE_BUF_EVENT(xfs_buf_item_relse); +DEFINE_BUF_EVENT(xfs_buf_iodone_async); +DEFINE_BUF_EVENT(xfs_buf_error_relse); +DEFINE_BUF_EVENT(xfs_buf_drain_buftarg); +DEFINE_BUF_EVENT(xfs_trans_read_buf_shut); +DEFINE_BUF_EVENT(xfs_buf_backing_folio); +DEFINE_BUF_EVENT(xfs_buf_backing_kmem); +DEFINE_BUF_EVENT(xfs_buf_backing_vmalloc); +DEFINE_BUF_EVENT(xfs_buf_backing_fallback); +DEFINE_BUF_EVENT(xfs_buf_find); + +/* not really buffer traces, but the buf provides useful information */ +DEFINE_BUF_EVENT(xfs_btree_corrupt); +DEFINE_BUF_EVENT(xfs_reset_dqcounts); + +/* pass flags explicitly */ +DECLARE_EVENT_CLASS(xfs_buf_flags_class, + TP_PROTO(struct xfs_buf *bp, unsigned flags, unsigned long caller_ip), + TP_ARGS(bp, flags, caller_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_daddr_t, bno) + __field(unsigned int, length) + __field(int, hold) + __field(int, pincount) + __field(unsigned, lockval) + __field(unsigned, flags) + __field(unsigned long, caller_ip) + ), + TP_fast_assign( + __entry->dev = bp->b_target->bt_dev; + __entry->bno = xfs_buf_daddr(bp); + __entry->length = bp->b_length; + __entry->flags = flags; + __entry->hold = bp->b_lockref.count; + __entry->pincount = atomic_read(&bp->b_pin_count); + __entry->lockval = bp->b_sema.count; + __entry->caller_ip = caller_ip; + ), + TP_printk("dev %d:%d daddr 0x%llx bbcount 0x%x hold %d pincount %d " + "lock %d flags %s caller %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->bno, + __entry->length, + __entry->hold, + __entry->pincount, + __entry->lockval, + __print_flags(__entry->flags, "|", XFS_BUF_FLAGS), + (void *)__entry->caller_ip) +) + +#define DEFINE_BUF_FLAGS_EVENT(name) \ +DEFINE_EVENT(xfs_buf_flags_class, name, \ + TP_PROTO(struct xfs_buf *bp, unsigned flags, unsigned long caller_ip), \ + TP_ARGS(bp, flags, caller_ip)) +DEFINE_BUF_FLAGS_EVENT(xfs_buf_get); +DEFINE_BUF_FLAGS_EVENT(xfs_buf_read); +DEFINE_BUF_FLAGS_EVENT(xfs_buf_readahead); + +TRACE_EVENT(xfs_buf_ioerror, + TP_PROTO(struct xfs_buf *bp, int error, xfs_failaddr_t caller_ip), + TP_ARGS(bp, error, caller_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_daddr_t, bno) + __field(unsigned int, length) + __field(unsigned, flags) + __field(int, hold) + __field(int, pincount) + __field(unsigned, lockval) + __field(int, error) + __field(xfs_failaddr_t, caller_ip) + ), + TP_fast_assign( + __entry->dev = bp->b_target->bt_dev; + __entry->bno = xfs_buf_daddr(bp); + __entry->length = bp->b_length; + __entry->hold = bp->b_lockref.count; + __entry->pincount = atomic_read(&bp->b_pin_count); + __entry->lockval = bp->b_sema.count; + __entry->error = error; + __entry->flags = bp->b_flags; + __entry->caller_ip = caller_ip; + ), + TP_printk("dev %d:%d daddr 0x%llx bbcount 0x%x hold %d pincount %d " + "lock %d error %d flags %s caller %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->bno, + __entry->length, + __entry->hold, + __entry->pincount, + __entry->lockval, + __entry->error, + __print_flags(__entry->flags, "|", XFS_BUF_FLAGS), + (void *)__entry->caller_ip) +); + +DECLARE_EVENT_CLASS(xfs_buf_item_class, + TP_PROTO(struct xfs_buf_log_item *bip), + TP_ARGS(bip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_daddr_t, buf_bno) + __field(unsigned int, buf_len) + __field(int, buf_hold) + __field(int, buf_pincount) + __field(int, buf_lockval) + __field(unsigned, buf_flags) + __field(unsigned, bli_recur) + __field(int, bli_refcount) + __field(unsigned, bli_flags) + __field(unsigned long, li_flags) + ), + TP_fast_assign( + __entry->dev = bip->bli_buf->b_target->bt_dev; + __entry->bli_flags = bip->bli_flags; + __entry->bli_recur = bip->bli_recur; + __entry->bli_refcount = atomic_read(&bip->bli_refcount); + __entry->buf_bno = xfs_buf_daddr(bip->bli_buf); + __entry->buf_len = bip->bli_buf->b_length; + __entry->buf_flags = bip->bli_buf->b_flags; + __entry->buf_hold = bip->bli_buf->b_lockref.count; + __entry->buf_pincount = atomic_read(&bip->bli_buf->b_pin_count); + __entry->buf_lockval = bip->bli_buf->b_sema.count; + __entry->li_flags = bip->bli_item.li_flags; + ), + TP_printk("dev %d:%d daddr 0x%llx bbcount 0x%x hold %d pincount %d " + "lock %d flags %s recur %d refcount %d bliflags %s " + "liflags %s", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->buf_bno, + __entry->buf_len, + __entry->buf_hold, + __entry->buf_pincount, + __entry->buf_lockval, + __print_flags(__entry->buf_flags, "|", XFS_BUF_FLAGS), + __entry->bli_recur, + __entry->bli_refcount, + __print_flags(__entry->bli_flags, "|", XFS_BLI_FLAGS), + __print_flags(__entry->li_flags, "|", XFS_LI_FLAGS)) +) + +#define DEFINE_BUF_ITEM_EVENT(name) \ +DEFINE_EVENT(xfs_buf_item_class, name, \ + TP_PROTO(struct xfs_buf_log_item *bip), \ + TP_ARGS(bip)) +DEFINE_BUF_ITEM_EVENT(xfs_buf_item_size); +DEFINE_BUF_ITEM_EVENT(xfs_buf_item_size_ordered); +DEFINE_BUF_ITEM_EVENT(xfs_buf_item_size_stale); +DEFINE_BUF_ITEM_EVENT(xfs_buf_item_format); +DEFINE_BUF_ITEM_EVENT(xfs_buf_item_format_stale); +DEFINE_BUF_ITEM_EVENT(xfs_buf_item_ordered); +DEFINE_BUF_ITEM_EVENT(xfs_buf_item_pin); +DEFINE_BUF_ITEM_EVENT(xfs_buf_item_unpin); +DEFINE_BUF_ITEM_EVENT(xfs_buf_item_unpin_stale); +DEFINE_BUF_ITEM_EVENT(xfs_buf_item_release); +DEFINE_BUF_ITEM_EVENT(xfs_buf_item_committed); +DEFINE_BUF_ITEM_EVENT(xfs_buf_item_push); +DEFINE_BUF_ITEM_EVENT(xfs_trans_get_buf); +DEFINE_BUF_ITEM_EVENT(xfs_trans_get_buf_recur); +DEFINE_BUF_ITEM_EVENT(xfs_trans_getsb); +DEFINE_BUF_ITEM_EVENT(xfs_trans_getsb_recur); +DEFINE_BUF_ITEM_EVENT(xfs_trans_read_buf); +DEFINE_BUF_ITEM_EVENT(xfs_trans_read_buf_recur); +DEFINE_BUF_ITEM_EVENT(xfs_trans_log_buf); +DEFINE_BUF_ITEM_EVENT(xfs_trans_brelse); +DEFINE_BUF_ITEM_EVENT(xfs_trans_bdetach); +DEFINE_BUF_ITEM_EVENT(xfs_trans_bjoin); +DEFINE_BUF_ITEM_EVENT(xfs_trans_bhold); +DEFINE_BUF_ITEM_EVENT(xfs_trans_bhold_release); +DEFINE_BUF_ITEM_EVENT(xfs_trans_binval); + +DECLARE_EVENT_CLASS(xfs_filestream_class, + TP_PROTO(const struct xfs_perag *pag, xfs_ino_t ino), + TP_ARGS(pag, ino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_agnumber_t, agno) + __field(int, streams) + ), + TP_fast_assign( + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->ino = ino; + __entry->agno = pag_agno(pag); + __entry->streams = atomic_read(&pag->pagf_fstrms); + ), + TP_printk("dev %d:%d ino 0x%llx agno 0x%x streams %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->agno, + __entry->streams) +) +#define DEFINE_FILESTREAM_EVENT(name) \ +DEFINE_EVENT(xfs_filestream_class, name, \ + TP_PROTO(const struct xfs_perag *pag, xfs_ino_t ino), \ + TP_ARGS(pag, ino)) +DEFINE_FILESTREAM_EVENT(xfs_filestream_free); +DEFINE_FILESTREAM_EVENT(xfs_filestream_lookup); +DEFINE_FILESTREAM_EVENT(xfs_filestream_scan); + +TRACE_EVENT(xfs_filestream_pick, + TP_PROTO(const struct xfs_perag *pag, xfs_ino_t ino), + TP_ARGS(pag, ino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_agnumber_t, agno) + __field(int, streams) + __field(xfs_extlen_t, free) + ), + TP_fast_assign( + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->ino = ino; + __entry->agno = pag_agno(pag); + __entry->streams = atomic_read(&pag->pagf_fstrms); + __entry->free = pag->pagf_freeblks; + ), + TP_printk("dev %d:%d ino 0x%llx agno 0x%x streams %d free %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->agno, + __entry->streams, + __entry->free) +); + +DECLARE_EVENT_CLASS(xfs_lock_class, + TP_PROTO(struct xfs_inode *ip, unsigned lock_flags, + unsigned long caller_ip), + TP_ARGS(ip, lock_flags, caller_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(int, lock_flags) + __field(unsigned long, caller_ip) + ), + TP_fast_assign( + __entry->dev = VFS_I(ip)->i_sb->s_dev; + __entry->ino = I_INO(ip); + __entry->lock_flags = lock_flags; + __entry->caller_ip = caller_ip; + ), + TP_printk("dev %d:%d ino 0x%llx flags %s caller %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __print_flags(__entry->lock_flags, "|", XFS_LOCK_FLAGS), + (void *)__entry->caller_ip) +) + +#define DEFINE_LOCK_EVENT(name) \ +DEFINE_EVENT(xfs_lock_class, name, \ + TP_PROTO(struct xfs_inode *ip, unsigned lock_flags, \ + unsigned long caller_ip), \ + TP_ARGS(ip, lock_flags, caller_ip)) +DEFINE_LOCK_EVENT(xfs_ilock); +DEFINE_LOCK_EVENT(xfs_ilock_nowait); +DEFINE_LOCK_EVENT(xfs_ilock_demote); +DEFINE_LOCK_EVENT(xfs_iunlock); + +DECLARE_EVENT_CLASS(xfs_inode_class, + TP_PROTO(struct xfs_inode *ip), + TP_ARGS(ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(unsigned long, iflags) + ), + TP_fast_assign( + __entry->dev = VFS_I(ip)->i_sb->s_dev; + __entry->ino = I_INO(ip); + __entry->iflags = ip->i_flags; + ), + TP_printk("dev %d:%d ino 0x%llx iflags 0x%lx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->iflags) +) + +#define DEFINE_INODE_EVENT(name) \ +DEFINE_EVENT(xfs_inode_class, name, \ + TP_PROTO(struct xfs_inode *ip), \ + TP_ARGS(ip)) +DEFINE_INODE_EVENT(xfs_iget_skip); +DEFINE_INODE_EVENT(xfs_iget_recycle); +DEFINE_INODE_EVENT(xfs_iget_recycle_fail); +DEFINE_INODE_EVENT(xfs_iget_hit); +DEFINE_INODE_EVENT(xfs_iget_miss); + +DEFINE_INODE_EVENT(xfs_getattr); +DEFINE_INODE_EVENT(xfs_setattr); +DEFINE_INODE_EVENT(xfs_readlink); +DEFINE_INODE_EVENT(xfs_inactive_symlink); +DEFINE_INODE_EVENT(xfs_alloc_file_space); +DEFINE_INODE_EVENT(xfs_free_file_space); +DEFINE_INODE_EVENT(xfs_zero_file_space); +DEFINE_INODE_EVENT(xfs_collapse_file_space); +DEFINE_INODE_EVENT(xfs_insert_file_space); +DEFINE_INODE_EVENT(xfs_readdir); +#ifdef CONFIG_XFS_POSIX_ACL +DEFINE_INODE_EVENT(xfs_get_acl); +#endif +DEFINE_INODE_EVENT(xfs_vm_bmap); +DEFINE_INODE_EVENT(xfs_file_ioctl); +#ifdef CONFIG_COMPAT +DEFINE_INODE_EVENT(xfs_file_compat_ioctl); +#endif +DEFINE_INODE_EVENT(xfs_ioctl_setattr); +DEFINE_INODE_EVENT(xfs_dir_fsync); +DEFINE_INODE_EVENT(xfs_file_fsync); +DEFINE_INODE_EVENT(xfs_destroy_inode); +DEFINE_INODE_EVENT(xfs_update_time); + +DEFINE_INODE_EVENT(xfs_dquot_dqalloc); +DEFINE_INODE_EVENT(xfs_dquot_dqdetach); + +DEFINE_INODE_EVENT(xfs_inode_set_eofblocks_tag); +DEFINE_INODE_EVENT(xfs_inode_clear_eofblocks_tag); +DEFINE_INODE_EVENT(xfs_inode_free_eofblocks_invalid); +DEFINE_INODE_EVENT(xfs_inode_set_cowblocks_tag); +DEFINE_INODE_EVENT(xfs_inode_clear_cowblocks_tag); +DEFINE_INODE_EVENT(xfs_inode_free_cowblocks_invalid); +DEFINE_INODE_EVENT(xfs_inode_set_reclaimable); +DEFINE_INODE_EVENT(xfs_inode_reclaiming); +DEFINE_INODE_EVENT(xfs_inode_set_need_inactive); +DEFINE_INODE_EVENT(xfs_inode_inactivating); + +/* + * ftrace's __print_symbolic requires that all enum values be wrapped in the + * TRACE_DEFINE_ENUM macro so that the enum value can be encoded in the ftrace + * ring buffer. Somehow this was only worth mentioning in the ftrace sample + * code. + */ +TRACE_DEFINE_ENUM(XFS_REFC_DOMAIN_SHARED); +TRACE_DEFINE_ENUM(XFS_REFC_DOMAIN_COW); + +DECLARE_EVENT_CLASS(xfs_fault_class, + TP_PROTO(struct xfs_inode *ip, unsigned int order), + TP_ARGS(ip, order), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(unsigned int, order) + ), + TP_fast_assign( + __entry->dev = VFS_I(ip)->i_sb->s_dev; + __entry->ino = I_INO(ip); + __entry->order = order; + ), + TP_printk("dev %d:%d ino 0x%llx order %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->order) +) + +#define DEFINE_FAULT_EVENT(name) \ +DEFINE_EVENT(xfs_fault_class, name, \ + TP_PROTO(struct xfs_inode *ip, unsigned int order), \ + TP_ARGS(ip, order)) +DEFINE_FAULT_EVENT(xfs_read_fault); +DEFINE_FAULT_EVENT(xfs_write_fault); + +DECLARE_EVENT_CLASS(xfs_iref_class, + TP_PROTO(struct xfs_inode *ip, unsigned long caller_ip), + TP_ARGS(ip, caller_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(int, count) + __field(int, pincount) + __field(unsigned long, iflags) + __field(unsigned long, caller_ip) + ), + TP_fast_assign( + __entry->dev = VFS_I(ip)->i_sb->s_dev; + __entry->ino = I_INO(ip); + __entry->count = icount_read_once(VFS_I(ip)); + __entry->pincount = atomic_read(&ip->i_pincount); + __entry->iflags = ip->i_flags; + __entry->caller_ip = caller_ip; + ), + TP_printk("dev %d:%d ino 0x%llx count %d pincount %d iflags 0x%lx caller %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->count, + __entry->pincount, + __entry->iflags, + (char *)__entry->caller_ip) +) + +TRACE_EVENT(xfs_iomap_prealloc_size, + TP_PROTO(struct xfs_inode *ip, xfs_fsblock_t blocks, int shift, + unsigned int writeio_blocks), + TP_ARGS(ip, blocks, shift, writeio_blocks), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_fsblock_t, blocks) + __field(int, shift) + __field(unsigned int, writeio_blocks) + ), + TP_fast_assign( + __entry->dev = VFS_I(ip)->i_sb->s_dev; + __entry->ino = I_INO(ip); + __entry->blocks = blocks; + __entry->shift = shift; + __entry->writeio_blocks = writeio_blocks; + ), + TP_printk("dev %d:%d ino 0x%llx prealloc blocks %llu shift %d " + "m_allocsize_blocks %u", + MAJOR(__entry->dev), MINOR(__entry->dev), __entry->ino, + __entry->blocks, __entry->shift, __entry->writeio_blocks) +) + +TRACE_EVENT(xfs_irec_merge_pre, + TP_PROTO(const struct xfs_perag *pag, + const struct xfs_inobt_rec_incore *rec, + const struct xfs_inobt_rec_incore *nrec), + TP_ARGS(pag, rec, nrec), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agino_t, agino) + __field(uint16_t, holemask) + __field(xfs_agino_t, nagino) + __field(uint16_t, nholemask) + ), + TP_fast_assign( + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->agno = pag_agno(pag); + __entry->agino = rec->ir_startino; + __entry->holemask = rec->ir_holemask; + __entry->nagino = nrec->ir_startino; + __entry->nholemask = nrec->ir_holemask; + ), + TP_printk("dev %d:%d agno 0x%x agino 0x%x holemask 0x%x new_agino 0x%x new_holemask 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->agino, + __entry->holemask, + __entry->nagino, + __entry->nholemask) +) + +TRACE_EVENT(xfs_irec_merge_post, + TP_PROTO(const struct xfs_perag *pag, + const struct xfs_inobt_rec_incore *nrec), + TP_ARGS(pag, nrec), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agino_t, agino) + __field(uint16_t, holemask) + ), + TP_fast_assign( + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->agno = pag_agno(pag); + __entry->agino = nrec->ir_startino; + __entry->holemask = nrec->ir_holemask; + ), + TP_printk("dev %d:%d agno 0x%x agino 0x%x holemask 0x%x", + MAJOR(__entry->dev), + MINOR(__entry->dev), + __entry->agno, + __entry->agino, + __entry->holemask) +) + +#define DEFINE_IREF_EVENT(name) \ +DEFINE_EVENT(xfs_iref_class, name, \ + TP_PROTO(struct xfs_inode *ip, unsigned long caller_ip), \ + TP_ARGS(ip, caller_ip)) +DEFINE_IREF_EVENT(xfs_irele); +DEFINE_IREF_EVENT(xfs_inode_pin); +DEFINE_IREF_EVENT(xfs_inode_unpin); +DEFINE_IREF_EVENT(xfs_inode_unpin_nowait); +DEFINE_IREF_EVENT(xfs_inode_push_pinned); +DEFINE_IREF_EVENT(xfs_inode_push_stale); + +DECLARE_EVENT_CLASS(xfs_namespace_class, + TP_PROTO(struct xfs_inode *dp, const struct xfs_name *name), + TP_ARGS(dp, name), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, dp_ino) + __field(int, namelen) + __dynamic_array(char, name, name->len) + ), + TP_fast_assign( + __entry->dev = VFS_I(dp)->i_sb->s_dev; + __entry->dp_ino = I_INO(dp); + __entry->namelen = name->len; + memcpy(__get_str(name), name->name, name->len); + ), + TP_printk("dev %d:%d dp ino 0x%llx name %.*s", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->dp_ino, + __entry->namelen, + __get_str(name)) +) + +#define DEFINE_NAMESPACE_EVENT(name) \ +DEFINE_EVENT(xfs_namespace_class, name, \ + TP_PROTO(struct xfs_inode *dp, const struct xfs_name *name), \ + TP_ARGS(dp, name)) +DEFINE_NAMESPACE_EVENT(xfs_remove); +DEFINE_NAMESPACE_EVENT(xfs_link); +DEFINE_NAMESPACE_EVENT(xfs_lookup); +DEFINE_NAMESPACE_EVENT(xfs_create); +DEFINE_NAMESPACE_EVENT(xfs_symlink); + +TRACE_EVENT(xfs_rename, + TP_PROTO(struct xfs_inode *src_dp, struct xfs_inode *target_dp, + struct xfs_name *src_name, struct xfs_name *target_name), + TP_ARGS(src_dp, target_dp, src_name, target_name), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, src_dp_ino) + __field(xfs_ino_t, target_dp_ino) + __field(int, src_namelen) + __field(int, target_namelen) + __dynamic_array(char, src_name, src_name->len) + __dynamic_array(char, target_name, target_name->len) + ), + TP_fast_assign( + __entry->dev = VFS_I(src_dp)->i_sb->s_dev; + __entry->src_dp_ino = I_INO(src_dp); + __entry->target_dp_ino = I_INO(target_dp); + __entry->src_namelen = src_name->len; + __entry->target_namelen = target_name->len; + memcpy(__get_str(src_name), src_name->name, src_name->len); + memcpy(__get_str(target_name), target_name->name, + target_name->len); + ), + TP_printk("dev %d:%d src dp ino 0x%llx target dp ino 0x%llx" + " src name %.*s target name %.*s", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->src_dp_ino, + __entry->target_dp_ino, + __entry->src_namelen, + __get_str(src_name), + __entry->target_namelen, + __get_str(target_name)) +) + +DECLARE_EVENT_CLASS(xfs_dquot_class, + TP_PROTO(struct xfs_dquot *dqp), + TP_ARGS(dqp), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, id) + __field(xfs_dqtype_t, type) + __field(unsigned, flags) + __field(unsigned, nrefs) + __field(unsigned long long, res_bcount) + __field(unsigned long long, res_rtbcount) + __field(unsigned long long, res_icount) + + __field(unsigned long long, bcount) + __field(unsigned long long, rtbcount) + __field(unsigned long long, icount) + + __field(unsigned long long, blk_hardlimit) + __field(unsigned long long, blk_softlimit) + __field(unsigned long long, rtb_hardlimit) + __field(unsigned long long, rtb_softlimit) + __field(unsigned long long, ino_hardlimit) + __field(unsigned long long, ino_softlimit) + ), + TP_fast_assign( + __entry->dev = dqp->q_mount->m_super->s_dev; + __entry->id = dqp->q_id; + __entry->type = dqp->q_type; + __entry->flags = dqp->q_flags; + __entry->nrefs = data_race(dqp->q_lockref.count); + + __entry->res_bcount = dqp->q_blk.reserved; + __entry->res_rtbcount = dqp->q_rtb.reserved; + __entry->res_icount = dqp->q_ino.reserved; + + __entry->bcount = dqp->q_blk.count; + __entry->rtbcount = dqp->q_rtb.count; + __entry->icount = dqp->q_ino.count; + + __entry->blk_hardlimit = dqp->q_blk.hardlimit; + __entry->blk_softlimit = dqp->q_blk.softlimit; + __entry->rtb_hardlimit = dqp->q_rtb.hardlimit; + __entry->rtb_softlimit = dqp->q_rtb.softlimit; + __entry->ino_hardlimit = dqp->q_ino.hardlimit; + __entry->ino_softlimit = dqp->q_ino.softlimit; + ), + TP_printk("dev %d:%d id 0x%x type %s flags %s nrefs %u " + "res_bc 0x%llx res_rtbc 0x%llx res_ic 0x%llx " + "bcnt 0x%llx bhardlimit 0x%llx bsoftlimit 0x%llx " + "rtbcnt 0x%llx rtbhardlimit 0x%llx rtbsoftlimit 0x%llx " + "icnt 0x%llx ihardlimit 0x%llx isoftlimit 0x%llx]", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->id, + __print_flags(__entry->type, "|", XFS_DQTYPE_STRINGS), + __print_flags(__entry->flags, "|", XFS_DQFLAG_STRINGS), + __entry->nrefs, + __entry->res_bcount, + __entry->res_rtbcount, + __entry->res_icount, + __entry->bcount, + __entry->blk_hardlimit, + __entry->blk_softlimit, + __entry->rtbcount, + __entry->rtb_hardlimit, + __entry->rtb_softlimit, + __entry->icount, + __entry->ino_hardlimit, + __entry->ino_softlimit) +) + +#define DEFINE_DQUOT_EVENT(name) \ +DEFINE_EVENT(xfs_dquot_class, name, \ + TP_PROTO(struct xfs_dquot *dqp), \ + TP_ARGS(dqp)) +DEFINE_DQUOT_EVENT(xfs_dqadjust); +DEFINE_DQUOT_EVENT(xfs_dqreclaim_want); +DEFINE_DQUOT_EVENT(xfs_dqreclaim_busy); +DEFINE_DQUOT_EVENT(xfs_dqreclaim_done); +DEFINE_DQUOT_EVENT(xfs_dqattach_get); +DEFINE_DQUOT_EVENT(xfs_dqalloc); +DEFINE_DQUOT_EVENT(xfs_dqtobp_read); +DEFINE_DQUOT_EVENT(xfs_dqread); +DEFINE_DQUOT_EVENT(xfs_dqread_fail); +DEFINE_DQUOT_EVENT(xfs_dqget_hit); +DEFINE_DQUOT_EVENT(xfs_dqget_miss); +DEFINE_DQUOT_EVENT(xfs_dqget_freeing); +DEFINE_DQUOT_EVENT(xfs_dqget_dup); +DEFINE_DQUOT_EVENT(xfs_dqrele); +DEFINE_DQUOT_EVENT(xfs_dqrele_free); +DEFINE_DQUOT_EVENT(xfs_dqflush); +DEFINE_DQUOT_EVENT(xfs_dqflush_force); +DEFINE_DQUOT_EVENT(xfs_dqflush_done); +DEFINE_DQUOT_EVENT(xfs_trans_apply_dquot_deltas_before); +DEFINE_DQUOT_EVENT(xfs_trans_apply_dquot_deltas_after); + +TRACE_EVENT(xfs_trans_mod_dquot, + TP_PROTO(struct xfs_trans *tp, struct xfs_dquot *dqp, + unsigned int field, int64_t delta), + TP_ARGS(tp, dqp, field, delta), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_dqtype_t, type) + __field(unsigned int, flags) + __field(unsigned int, dqid) + __field(unsigned int, field) + __field(int64_t, delta) + ), + TP_fast_assign( + __entry->dev = tp->t_mountp->m_super->s_dev; + __entry->type = dqp->q_type; + __entry->flags = dqp->q_flags; + __entry->dqid = dqp->q_id; + __entry->field = field; + __entry->delta = delta; + ), + TP_printk("dev %d:%d dquot id 0x%x type %s flags %s field %s delta %lld", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->dqid, + __print_flags(__entry->type, "|", XFS_DQTYPE_STRINGS), + __print_flags(__entry->flags, "|", XFS_DQFLAG_STRINGS), + __print_flags(__entry->field, "|", XFS_QMOPT_FLAGS), + __entry->delta) +); + +DECLARE_EVENT_CLASS(xfs_dqtrx_class, + TP_PROTO(struct xfs_dqtrx *qtrx), + TP_ARGS(qtrx), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_dqtype_t, type) + __field(unsigned int, flags) + __field(u32, dqid) + + __field(uint64_t, blk_res) + __field(int64_t, bcount_delta) + __field(int64_t, delbcnt_delta) + + __field(uint64_t, rtblk_res) + __field(uint64_t, rtblk_res_used) + __field(int64_t, rtbcount_delta) + __field(int64_t, delrtb_delta) + + __field(uint64_t, ino_res) + __field(uint64_t, ino_res_used) + __field(int64_t, icount_delta) + ), + TP_fast_assign( + __entry->dev = qtrx->qt_dquot->q_mount->m_super->s_dev; + __entry->type = qtrx->qt_dquot->q_type; + __entry->flags = qtrx->qt_dquot->q_flags; + __entry->dqid = qtrx->qt_dquot->q_id; + + __entry->blk_res = qtrx->qt_blk_res; + __entry->bcount_delta = qtrx->qt_bcount_delta; + __entry->delbcnt_delta = qtrx->qt_delbcnt_delta; + + __entry->rtblk_res = qtrx->qt_rtblk_res; + __entry->rtblk_res_used = qtrx->qt_rtblk_res_used; + __entry->rtbcount_delta = qtrx->qt_rtbcount_delta; + __entry->delrtb_delta = qtrx->qt_delrtb_delta; + + __entry->ino_res = qtrx->qt_ino_res; + __entry->ino_res_used = qtrx->qt_ino_res_used; + __entry->icount_delta = qtrx->qt_icount_delta; + ), + TP_printk("dev %d:%d dquot id 0x%x type %s flags %s " + "blk_res %llu bcount_delta %lld delbcnt_delta %lld " + "rtblk_res %llu rtblk_res_used %llu rtbcount_delta %lld delrtb_delta %lld " + "ino_res %llu ino_res_used %llu icount_delta %lld", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->dqid, + __print_flags(__entry->type, "|", XFS_DQTYPE_STRINGS), + __print_flags(__entry->flags, "|", XFS_DQFLAG_STRINGS), + + __entry->blk_res, + __entry->bcount_delta, + __entry->delbcnt_delta, + + __entry->rtblk_res, + __entry->rtblk_res_used, + __entry->rtbcount_delta, + __entry->delrtb_delta, + + __entry->ino_res, + __entry->ino_res_used, + __entry->icount_delta) +) + +#define DEFINE_DQTRX_EVENT(name) \ +DEFINE_EVENT(xfs_dqtrx_class, name, \ + TP_PROTO(struct xfs_dqtrx *qtrx), \ + TP_ARGS(qtrx)) +DEFINE_DQTRX_EVENT(xfs_trans_apply_dquot_deltas); +DEFINE_DQTRX_EVENT(xfs_trans_mod_dquot_before); +DEFINE_DQTRX_EVENT(xfs_trans_mod_dquot_after); + +DECLARE_EVENT_CLASS(xfs_loggrant_class, + TP_PROTO(struct xlog *log, struct xlog_ticket *tic), + TP_ARGS(log, tic), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned long, tic) + __field(char, ocnt) + __field(char, cnt) + __field(int, curr_res) + __field(int, unit_res) + __field(unsigned int, flags) + __field(int, reserveq) + __field(int, writeq) + __field(uint64_t, grant_reserve_bytes) + __field(uint64_t, grant_write_bytes) + __field(uint64_t, tail_space) + __field(int, curr_cycle) + __field(int, curr_block) + __field(xfs_lsn_t, tail_lsn) + ), + TP_fast_assign( + __entry->dev = log->l_mp->m_super->s_dev; + __entry->tic = (unsigned long)tic; + __entry->ocnt = tic->t_ocnt; + __entry->cnt = tic->t_cnt; + __entry->curr_res = tic->t_curr_res; + __entry->unit_res = tic->t_unit_res; + __entry->flags = tic->t_flags; + __entry->reserveq = list_empty(&log->l_reserve_head.waiters); + __entry->writeq = list_empty(&log->l_write_head.waiters); + __entry->tail_space = READ_ONCE(log->l_tail_space); + __entry->grant_reserve_bytes = __entry->tail_space + + atomic64_read(&log->l_reserve_head.grant); + __entry->grant_write_bytes = __entry->tail_space + + atomic64_read(&log->l_write_head.grant); + __entry->curr_cycle = log->l_curr_cycle; + __entry->curr_block = log->l_curr_block; + __entry->tail_lsn = atomic64_read(&log->l_tail_lsn); + ), + TP_printk("dev %d:%d tic 0x%lx t_ocnt %u t_cnt %u t_curr_res %u " + "t_unit_res %u t_flags %s reserveq %s writeq %s " + "tail space %llu grant_reserve_bytes %llu " + "grant_write_bytes %llu curr_cycle %d curr_block %d " + "tail_cycle %d tail_block %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->tic, + __entry->ocnt, + __entry->cnt, + __entry->curr_res, + __entry->unit_res, + __print_flags(__entry->flags, "|", XLOG_TIC_FLAGS), + __entry->reserveq ? "empty" : "active", + __entry->writeq ? "empty" : "active", + __entry->tail_space, + __entry->grant_reserve_bytes, + __entry->grant_write_bytes, + __entry->curr_cycle, + __entry->curr_block, + CYCLE_LSN(__entry->tail_lsn), + BLOCK_LSN(__entry->tail_lsn) + ) +) + +#define DEFINE_LOGGRANT_EVENT(name) \ +DEFINE_EVENT(xfs_loggrant_class, name, \ + TP_PROTO(struct xlog *log, struct xlog_ticket *tic), \ + TP_ARGS(log, tic)) +DEFINE_LOGGRANT_EVENT(xfs_log_umount_write); +DEFINE_LOGGRANT_EVENT(xfs_log_grant_sleep); +DEFINE_LOGGRANT_EVENT(xfs_log_grant_wake); +DEFINE_LOGGRANT_EVENT(xfs_log_grant_wake_up); +DEFINE_LOGGRANT_EVENT(xfs_log_reserve); +DEFINE_LOGGRANT_EVENT(xfs_log_reserve_exit); +DEFINE_LOGGRANT_EVENT(xfs_log_regrant); +DEFINE_LOGGRANT_EVENT(xfs_log_regrant_exit); +DEFINE_LOGGRANT_EVENT(xfs_log_ticket_regrant); +DEFINE_LOGGRANT_EVENT(xfs_log_ticket_regrant_exit); +DEFINE_LOGGRANT_EVENT(xfs_log_ticket_regrant_sub); +DEFINE_LOGGRANT_EVENT(xfs_log_ticket_ungrant); +DEFINE_LOGGRANT_EVENT(xfs_log_ticket_ungrant_sub); +DEFINE_LOGGRANT_EVENT(xfs_log_ticket_ungrant_exit); +DEFINE_LOGGRANT_EVENT(xfs_log_cil_wait); + +DECLARE_EVENT_CLASS(xfs_log_item_class, + TP_PROTO(struct xfs_log_item *lip), + TP_ARGS(lip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(void *, lip) + __field(uint, type) + __field(unsigned long, flags) + __field(xfs_lsn_t, lsn) + ), + TP_fast_assign( + __entry->dev = lip->li_log->l_mp->m_super->s_dev; + __entry->lip = lip; + __entry->type = lip->li_type; + __entry->flags = lip->li_flags; + __entry->lsn = lip->li_lsn; + ), + TP_printk("dev %d:%d lip %p lsn %d/%d type %s flags %s", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->lip, + CYCLE_LSN(__entry->lsn), BLOCK_LSN(__entry->lsn), + __print_symbolic(__entry->type, XFS_LI_TYPE_DESC), + __print_flags(__entry->flags, "|", XFS_LI_FLAGS)) +) + +TRACE_EVENT(xfs_log_force, + TP_PROTO(struct xfs_mount *mp, xfs_lsn_t lsn, unsigned long caller_ip), + TP_ARGS(mp, lsn, caller_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_lsn_t, lsn) + __field(unsigned long, caller_ip) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->lsn = lsn; + __entry->caller_ip = caller_ip; + ), + TP_printk("dev %d:%d lsn 0x%llx caller %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->lsn, (void *)__entry->caller_ip) +) + +#define DEFINE_LOG_ITEM_EVENT(name) \ +DEFINE_EVENT(xfs_log_item_class, name, \ + TP_PROTO(struct xfs_log_item *lip), \ + TP_ARGS(lip)) +DEFINE_LOG_ITEM_EVENT(xfs_cil_whiteout_mark); +DEFINE_LOG_ITEM_EVENT(xfs_cil_whiteout_skip); +DEFINE_LOG_ITEM_EVENT(xfs_cil_whiteout_unpin); +DEFINE_LOG_ITEM_EVENT(xlog_ail_insert_abort); +DEFINE_LOG_ITEM_EVENT(xfs_trans_free_abort); + +DECLARE_EVENT_CLASS(xfs_ail_push_class, + TP_PROTO(struct xfs_ail *ailp, uint type, unsigned long flags, xfs_lsn_t lsn), + TP_ARGS(ailp, type, flags, lsn), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(uint, type) + __field(unsigned long, flags) + __field(xfs_lsn_t, lsn) + ), + TP_fast_assign( + __entry->dev = ailp->ail_log->l_mp->m_super->s_dev; + __entry->type = type; + __entry->flags = flags; + __entry->lsn = lsn; + ), + TP_printk("dev %d:%d lsn %d/%d type %s flags %s", + MAJOR(__entry->dev), MINOR(__entry->dev), + CYCLE_LSN(__entry->lsn), BLOCK_LSN(__entry->lsn), + __print_symbolic(__entry->type, XFS_LI_TYPE_DESC), + __print_flags(__entry->flags, "|", XFS_LI_FLAGS)) +) + +#define DEFINE_AIL_PUSH_EVENT(name) \ +DEFINE_EVENT(xfs_ail_push_class, name, \ + TP_PROTO(struct xfs_ail *ailp, uint type, unsigned long flags, xfs_lsn_t lsn), \ + TP_ARGS(ailp, type, flags, lsn)) +DEFINE_AIL_PUSH_EVENT(xfs_ail_push); +DEFINE_AIL_PUSH_EVENT(xfs_ail_pinned); +DEFINE_AIL_PUSH_EVENT(xfs_ail_locked); +DEFINE_AIL_PUSH_EVENT(xfs_ail_flushing); + +DECLARE_EVENT_CLASS(xfs_ail_class, + TP_PROTO(struct xfs_log_item *lip, xfs_lsn_t old_lsn, xfs_lsn_t new_lsn), + TP_ARGS(lip, old_lsn, new_lsn), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(void *, lip) + __field(uint, type) + __field(unsigned long, flags) + __field(xfs_lsn_t, old_lsn) + __field(xfs_lsn_t, new_lsn) + ), + TP_fast_assign( + __entry->dev = lip->li_log->l_mp->m_super->s_dev; + __entry->lip = lip; + __entry->type = lip->li_type; + __entry->flags = lip->li_flags; + __entry->old_lsn = old_lsn; + __entry->new_lsn = new_lsn; + ), + TP_printk("dev %d:%d lip %p old lsn %d/%d new lsn %d/%d type %s flags %s", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->lip, + CYCLE_LSN(__entry->old_lsn), BLOCK_LSN(__entry->old_lsn), + CYCLE_LSN(__entry->new_lsn), BLOCK_LSN(__entry->new_lsn), + __print_symbolic(__entry->type, XFS_LI_TYPE_DESC), + __print_flags(__entry->flags, "|", XFS_LI_FLAGS)) +) + +#define DEFINE_AIL_EVENT(name) \ +DEFINE_EVENT(xfs_ail_class, name, \ + TP_PROTO(struct xfs_log_item *lip, xfs_lsn_t old_lsn, xfs_lsn_t new_lsn), \ + TP_ARGS(lip, old_lsn, new_lsn)) +DEFINE_AIL_EVENT(xfs_ail_insert); +DEFINE_AIL_EVENT(xfs_ail_move); +DEFINE_AIL_EVENT(xfs_ail_delete); + +TRACE_EVENT(xfs_log_assign_tail_lsn, + TP_PROTO(struct xlog *log, xfs_lsn_t new_lsn), + TP_ARGS(log, new_lsn), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_lsn_t, new_lsn) + __field(xfs_lsn_t, old_lsn) + __field(xfs_lsn_t, head_lsn) + ), + TP_fast_assign( + __entry->dev = log->l_mp->m_super->s_dev; + __entry->new_lsn = new_lsn; + __entry->old_lsn = atomic64_read(&log->l_tail_lsn); + __entry->head_lsn = log->l_ailp->ail_head_lsn; + ), + TP_printk("dev %d:%d new tail lsn %d/%d, old lsn %d/%d, head lsn %d/%d", + MAJOR(__entry->dev), MINOR(__entry->dev), + CYCLE_LSN(__entry->new_lsn), BLOCK_LSN(__entry->new_lsn), + CYCLE_LSN(__entry->old_lsn), BLOCK_LSN(__entry->old_lsn), + CYCLE_LSN(__entry->head_lsn), BLOCK_LSN(__entry->head_lsn)) +) + +DECLARE_EVENT_CLASS(xfs_file_class, + TP_PROTO(struct kiocb *iocb, struct iov_iter *iter), + TP_ARGS(iocb, iter), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_fsize_t, size) + __field(loff_t, offset) + __field(size_t, count) + ), + TP_fast_assign( + __entry->dev = file_inode(iocb->ki_filp)->i_sb->s_dev; + __entry->ino = file_inode(iocb->ki_filp)->i_ino; + __entry->size = XFS_I(file_inode(iocb->ki_filp))->i_disk_size; + __entry->offset = iocb->ki_pos; + __entry->count = iov_iter_count(iter); + ), + TP_printk("dev %d:%d ino 0x%llx disize 0x%llx pos 0x%llx bytecount 0x%zx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->size, + __entry->offset, + __entry->count) +) + +#define DEFINE_RW_EVENT(name) \ +DEFINE_EVENT(xfs_file_class, name, \ + TP_PROTO(struct kiocb *iocb, struct iov_iter *iter), \ + TP_ARGS(iocb, iter)) +DEFINE_RW_EVENT(xfs_file_buffered_read); +DEFINE_RW_EVENT(xfs_file_direct_read); +DEFINE_RW_EVENT(xfs_file_dax_read); +DEFINE_RW_EVENT(xfs_file_buffered_write); +DEFINE_RW_EVENT(xfs_file_direct_write); +DEFINE_RW_EVENT(xfs_file_dax_write); +DEFINE_RW_EVENT(xfs_reflink_bounce_dio_write); + +TRACE_EVENT(xfs_iomap_atomic_write_cow, + TP_PROTO(struct xfs_inode *ip, xfs_off_t offset, ssize_t count), + TP_ARGS(ip, offset, count), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_off_t, offset) + __field(ssize_t, count) + ), + TP_fast_assign( + __entry->dev = VFS_I(ip)->i_sb->s_dev; + __entry->ino = I_INO(ip); + __entry->offset = offset; + __entry->count = count; + ), + TP_printk("dev %d:%d ino 0x%llx pos 0x%llx bytecount 0x%zx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->offset, + __entry->count) +) + +DECLARE_EVENT_CLASS(xfs_imap_class, + TP_PROTO(struct xfs_inode *ip, xfs_off_t offset, ssize_t count, + int whichfork, struct xfs_bmbt_irec *irec), + TP_ARGS(ip, offset, count, whichfork, irec), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(loff_t, size) + __field(loff_t, offset) + __field(size_t, count) + __field(int, whichfork) + __field(xfs_fileoff_t, startoff) + __field(xfs_fsblock_t, startblock) + __field(xfs_filblks_t, blockcount) + ), + TP_fast_assign( + __entry->dev = VFS_I(ip)->i_sb->s_dev; + __entry->ino = I_INO(ip); + __entry->size = ip->i_disk_size; + __entry->offset = offset; + __entry->count = count; + __entry->whichfork = whichfork; + __entry->startoff = irec ? irec->br_startoff : 0; + __entry->startblock = irec ? irec->br_startblock : 0; + __entry->blockcount = irec ? irec->br_blockcount : 0; + ), + TP_printk("dev %d:%d ino 0x%llx disize 0x%llx pos 0x%llx bytecount 0x%zx " + "fork %s startoff 0x%llx startblock 0x%llx fsbcount 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->size, + __entry->offset, + __entry->count, + __print_symbolic(__entry->whichfork, XFS_WHICHFORK_STRINGS), + __entry->startoff, + (int64_t)__entry->startblock, + __entry->blockcount) +) + +#define DEFINE_IMAP_EVENT(name) \ +DEFINE_EVENT(xfs_imap_class, name, \ + TP_PROTO(struct xfs_inode *ip, xfs_off_t offset, ssize_t count, \ + int whichfork, struct xfs_bmbt_irec *irec), \ + TP_ARGS(ip, offset, count, whichfork, irec)) +DEFINE_IMAP_EVENT(xfs_map_blocks_found); +DEFINE_IMAP_EVENT(xfs_map_blocks_alloc); +DEFINE_IMAP_EVENT(xfs_iomap_alloc); +DEFINE_IMAP_EVENT(xfs_iomap_found); + +DECLARE_EVENT_CLASS(xfs_simple_io_class, + TP_PROTO(struct xfs_inode *ip, xfs_off_t offset, ssize_t count), + TP_ARGS(ip, offset, count), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(loff_t, isize) + __field(loff_t, disize) + __field(loff_t, offset) + __field(size_t, count) + ), + TP_fast_assign( + __entry->dev = VFS_I(ip)->i_sb->s_dev; + __entry->ino = I_INO(ip); + __entry->isize = VFS_I(ip)->i_size; + __entry->disize = ip->i_disk_size; + __entry->offset = offset; + __entry->count = count; + ), + TP_printk("dev %d:%d ino 0x%llx isize 0x%llx disize 0x%llx " + "pos 0x%llx bytecount 0x%zx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->isize, + __entry->disize, + __entry->offset, + __entry->count) +); + +#define DEFINE_SIMPLE_IO_EVENT(name) \ +DEFINE_EVENT(xfs_simple_io_class, name, \ + TP_PROTO(struct xfs_inode *ip, xfs_off_t offset, ssize_t count), \ + TP_ARGS(ip, offset, count)) +DEFINE_SIMPLE_IO_EVENT(xfs_delalloc_enospc); +DEFINE_SIMPLE_IO_EVENT(xfs_unwritten_convert); +DEFINE_SIMPLE_IO_EVENT(xfs_setfilesize); +DEFINE_SIMPLE_IO_EVENT(xfs_zero_eof); +DEFINE_SIMPLE_IO_EVENT(xfs_end_io_direct_write); +DEFINE_SIMPLE_IO_EVENT(xfs_file_splice_read); +DEFINE_SIMPLE_IO_EVENT(xfs_zoned_map_blocks); + +DECLARE_EVENT_CLASS(xfs_itrunc_class, + TP_PROTO(struct xfs_inode *ip, xfs_fsize_t new_size), + TP_ARGS(ip, new_size), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_fsize_t, size) + __field(xfs_fsize_t, new_size) + ), + TP_fast_assign( + __entry->dev = VFS_I(ip)->i_sb->s_dev; + __entry->ino = I_INO(ip); + __entry->size = ip->i_disk_size; + __entry->new_size = new_size; + ), + TP_printk("dev %d:%d ino 0x%llx disize 0x%llx new_size 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->size, + __entry->new_size) +) + +#define DEFINE_ITRUNC_EVENT(name) \ +DEFINE_EVENT(xfs_itrunc_class, name, \ + TP_PROTO(struct xfs_inode *ip, xfs_fsize_t new_size), \ + TP_ARGS(ip, new_size)) +DEFINE_ITRUNC_EVENT(xfs_itruncate_extents_start); +DEFINE_ITRUNC_EVENT(xfs_itruncate_extents_end); + +TRACE_EVENT(xfs_bunmap, + TP_PROTO(struct xfs_inode *ip, xfs_fileoff_t fileoff, xfs_filblks_t len, + int flags, unsigned long caller_ip), + TP_ARGS(ip, fileoff, len, flags, caller_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_fsize_t, size) + __field(xfs_fileoff_t, fileoff) + __field(xfs_filblks_t, len) + __field(unsigned long, caller_ip) + __field(int, flags) + ), + TP_fast_assign( + __entry->dev = VFS_I(ip)->i_sb->s_dev; + __entry->ino = I_INO(ip); + __entry->size = ip->i_disk_size; + __entry->fileoff = fileoff; + __entry->len = len; + __entry->caller_ip = caller_ip; + __entry->flags = flags; + ), + TP_printk("dev %d:%d ino 0x%llx disize 0x%llx fileoff 0x%llx fsbcount 0x%llx " + "flags %s caller %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->size, + __entry->fileoff, + __entry->len, + __print_flags(__entry->flags, "|", XFS_BMAPI_FLAGS), + (void *)__entry->caller_ip) + +); + +DECLARE_EVENT_CLASS(xfs_extent_busy_class, + TP_PROTO(const struct xfs_group *xg, xfs_agblock_t agbno, + xfs_extlen_t len), + TP_ARGS(xg, agbno, len), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(enum xfs_group_type, type) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, agbno) + __field(xfs_extlen_t, len) + ), + TP_fast_assign( + __entry->dev = xg->xg_mount->m_super->s_dev; + __entry->type = xg->xg_type; + __entry->agno = xg->xg_gno; + __entry->agbno = agbno; + __entry->len = len; + ), + TP_printk("dev %d:%d %sno 0x%x %sbno 0x%x fsbcount 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agno, + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agbno, + __entry->len) +); +#define DEFINE_BUSY_EVENT(name) \ +DEFINE_EVENT(xfs_extent_busy_class, name, \ + TP_PROTO(const struct xfs_group *xg, xfs_agblock_t agbno, \ + xfs_extlen_t len), \ + TP_ARGS(xg, agbno, len)) +DEFINE_BUSY_EVENT(xfs_extent_busy); +DEFINE_BUSY_EVENT(xfs_extent_busy_force); +DEFINE_BUSY_EVENT(xfs_extent_busy_reuse); +DEFINE_BUSY_EVENT(xfs_extent_busy_clear); + +TRACE_EVENT(xfs_extent_busy_trim, + TP_PROTO(const struct xfs_group *xg, xfs_agblock_t agbno, + xfs_extlen_t len, xfs_agblock_t tbno, xfs_extlen_t tlen), + TP_ARGS(xg, agbno, len, tbno, tlen), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(enum xfs_group_type, type) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, agbno) + __field(xfs_extlen_t, len) + __field(xfs_agblock_t, tbno) + __field(xfs_extlen_t, tlen) + ), + TP_fast_assign( + __entry->dev = xg->xg_mount->m_super->s_dev; + __entry->type = xg->xg_type; + __entry->agno = xg->xg_gno; + __entry->agbno = agbno; + __entry->len = len; + __entry->tbno = tbno; + __entry->tlen = tlen; + ), + TP_printk("dev %d:%d %sno 0x%x %sbno 0x%x fsbcount 0x%x found_agbno 0x%x found_fsbcount 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agno, + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agbno, + __entry->len, + __entry->tbno, + __entry->tlen) +); + +#ifdef CONFIG_XFS_RT +TRACE_EVENT(xfs_rtalloc_extent_busy, + TP_PROTO(struct xfs_rtgroup *rtg, xfs_rtxnum_t start, + xfs_rtxlen_t minlen, xfs_rtxlen_t maxlen, + xfs_rtxlen_t len, xfs_rtxlen_t prod, xfs_rtxnum_t rtx, + unsigned busy_gen), + TP_ARGS(rtg, start, minlen, maxlen, len, prod, rtx, busy_gen), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_rgnumber_t, rgno) + __field(xfs_rtxnum_t, start) + __field(xfs_rtxlen_t, minlen) + __field(xfs_rtxlen_t, maxlen) + __field(xfs_rtxlen_t, mod) + __field(xfs_rtxlen_t, prod) + __field(xfs_rtxlen_t, len) + __field(xfs_rtxnum_t, rtx) + __field(unsigned, busy_gen) + ), + TP_fast_assign( + __entry->dev = rtg_mount(rtg)->m_super->s_dev; + __entry->rgno = rtg_rgno(rtg); + __entry->start = start; + __entry->minlen = minlen; + __entry->maxlen = maxlen; + __entry->prod = prod; + __entry->len = len; + __entry->rtx = rtx; + __entry->busy_gen = busy_gen; + ), + TP_printk("dev %d:%d rgno 0x%x startrtx 0x%llx minlen %u maxlen %u " + "prod %u len %u rtx 0%llx busy_gen 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->rgno, + __entry->start, + __entry->minlen, + __entry->maxlen, + __entry->prod, + __entry->len, + __entry->rtx, + __entry->busy_gen) +) + +TRACE_EVENT(xfs_rtalloc_extent_busy_trim, + TP_PROTO(struct xfs_rtgroup *rtg, xfs_rtxnum_t old_rtx, + xfs_rtxlen_t old_len, xfs_rtxnum_t new_rtx, + xfs_rtxlen_t new_len), + TP_ARGS(rtg, old_rtx, old_len, new_rtx, new_len), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_rgnumber_t, rgno) + __field(xfs_rtxnum_t, old_rtx) + __field(xfs_rtxnum_t, new_rtx) + __field(xfs_rtxlen_t, old_len) + __field(xfs_rtxlen_t, new_len) + ), + TP_fast_assign( + __entry->dev = rtg_mount(rtg)->m_super->s_dev; + __entry->rgno = rtg_rgno(rtg); + __entry->old_rtx = old_rtx; + __entry->old_len = old_len; + __entry->new_rtx = new_rtx; + __entry->new_len = new_len; + ), + TP_printk("dev %d:%d rgno 0x%x rtx 0x%llx rtxcount 0x%x -> rtx 0x%llx rtxcount 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->rgno, + __entry->old_rtx, + __entry->old_len, + __entry->new_rtx, + __entry->new_len) +); +#endif /* CONFIG_XFS_RT */ + +DECLARE_EVENT_CLASS(xfs_agf_class, + TP_PROTO(struct xfs_mount *mp, struct xfs_agf *agf, int flags, + unsigned long caller_ip), + TP_ARGS(mp, agf, flags, caller_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(int, flags) + __field(__u32, length) + __field(__u32, bno_root) + __field(__u32, cnt_root) + __field(__u32, bno_level) + __field(__u32, cnt_level) + __field(__u32, flfirst) + __field(__u32, fllast) + __field(__u32, flcount) + __field(__u32, freeblks) + __field(__u32, longest) + __field(unsigned long, caller_ip) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->agno = be32_to_cpu(agf->agf_seqno), + __entry->flags = flags; + __entry->length = be32_to_cpu(agf->agf_length), + __entry->bno_root = be32_to_cpu(agf->agf_bno_root), + __entry->cnt_root = be32_to_cpu(agf->agf_cnt_root), + __entry->bno_level = be32_to_cpu(agf->agf_bno_level), + __entry->cnt_level = be32_to_cpu(agf->agf_cnt_level), + __entry->flfirst = be32_to_cpu(agf->agf_flfirst), + __entry->fllast = be32_to_cpu(agf->agf_fllast), + __entry->flcount = be32_to_cpu(agf->agf_flcount), + __entry->freeblks = be32_to_cpu(agf->agf_freeblks), + __entry->longest = be32_to_cpu(agf->agf_longest); + __entry->caller_ip = caller_ip; + ), + TP_printk("dev %d:%d agno 0x%x flags %s length %u roots b %u c %u " + "levels b %u c %u flfirst %u fllast %u flcount %u " + "freeblks %u longest %u caller %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __print_flags(__entry->flags, "|", XFS_AGF_FLAGS), + __entry->length, + __entry->bno_root, + __entry->cnt_root, + __entry->bno_level, + __entry->cnt_level, + __entry->flfirst, + __entry->fllast, + __entry->flcount, + __entry->freeblks, + __entry->longest, + (void *)__entry->caller_ip) +); +#define DEFINE_AGF_EVENT(name) \ +DEFINE_EVENT(xfs_agf_class, name, \ + TP_PROTO(struct xfs_mount *mp, struct xfs_agf *agf, int flags, \ + unsigned long caller_ip), \ + TP_ARGS(mp, agf, flags, caller_ip)) +DEFINE_AGF_EVENT(xfs_agf); +DEFINE_AGF_EVENT(xfs_agfl_reset); + +TRACE_EVENT(xfs_free_extent, + TP_PROTO(const struct xfs_perag *pag, xfs_agblock_t agbno, + xfs_extlen_t len, enum xfs_ag_resv_type resv, int haveleft, + int haveright), + TP_ARGS(pag, agbno, len, resv, haveleft, haveright), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, agbno) + __field(xfs_extlen_t, len) + __field(int, resv) + __field(int, haveleft) + __field(int, haveright) + ), + TP_fast_assign( + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->agno = pag_agno(pag); + __entry->agbno = agbno; + __entry->len = len; + __entry->resv = resv; + __entry->haveleft = haveleft; + __entry->haveright = haveright; + ), + TP_printk("dev %d:%d agno 0x%x agbno 0x%x fsbcount 0x%x resv %d %s", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->agbno, + __entry->len, + __entry->resv, + __entry->haveleft ? + (__entry->haveright ? "both" : "left") : + (__entry->haveright ? "right" : "none")) + +); + +DECLARE_EVENT_CLASS(xfs_alloc_class, + TP_PROTO(struct xfs_alloc_arg *args), + TP_ARGS(args), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, agbno) + __field(xfs_extlen_t, minlen) + __field(xfs_extlen_t, maxlen) + __field(xfs_extlen_t, mod) + __field(xfs_extlen_t, prod) + __field(xfs_extlen_t, minleft) + __field(xfs_extlen_t, total) + __field(xfs_extlen_t, alignment) + __field(xfs_extlen_t, minalignslop) + __field(xfs_extlen_t, len) + __field(char, wasdel) + __field(char, wasfromfl) + __field(int, resv) + __field(int, datatype) + __field(xfs_agnumber_t, highest_agno) + ), + TP_fast_assign( + __entry->dev = args->mp->m_super->s_dev; + __entry->agno = args->agno; + __entry->agbno = args->agbno; + __entry->minlen = args->minlen; + __entry->maxlen = args->maxlen; + __entry->mod = args->mod; + __entry->prod = args->prod; + __entry->minleft = args->minleft; + __entry->total = args->total; + __entry->alignment = args->alignment; + __entry->minalignslop = args->minalignslop; + __entry->len = args->len; + __entry->wasdel = args->wasdel; + __entry->wasfromfl = args->wasfromfl; + __entry->resv = args->resv; + __entry->datatype = args->datatype; + __entry->highest_agno = args->tp->t_highest_agno; + ), + TP_printk("dev %d:%d agno 0x%x agbno 0x%x minlen %u maxlen %u mod %u " + "prod %u minleft %u total %u alignment %u minalignslop %u " + "len %u wasdel %d wasfromfl %d resv %d " + "datatype 0x%x highest_agno 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->agbno, + __entry->minlen, + __entry->maxlen, + __entry->mod, + __entry->prod, + __entry->minleft, + __entry->total, + __entry->alignment, + __entry->minalignslop, + __entry->len, + __entry->wasdel, + __entry->wasfromfl, + __entry->resv, + __entry->datatype, + __entry->highest_agno) +) + +#define DEFINE_ALLOC_EVENT(name) \ +DEFINE_EVENT(xfs_alloc_class, name, \ + TP_PROTO(struct xfs_alloc_arg *args), \ + TP_ARGS(args)) +DEFINE_ALLOC_EVENT(xfs_alloc_exact_done); +DEFINE_ALLOC_EVENT(xfs_alloc_exact_notfound); +DEFINE_ALLOC_EVENT(xfs_alloc_exact_error); +DEFINE_ALLOC_EVENT(xfs_alloc_near_first); +DEFINE_ALLOC_EVENT(xfs_alloc_cur); +DEFINE_ALLOC_EVENT(xfs_alloc_cur_right); +DEFINE_ALLOC_EVENT(xfs_alloc_cur_left); +DEFINE_ALLOC_EVENT(xfs_alloc_cur_lookup); +DEFINE_ALLOC_EVENT(xfs_alloc_cur_lookup_done); +DEFINE_ALLOC_EVENT(xfs_alloc_near_noentry); +DEFINE_ALLOC_EVENT(xfs_alloc_near_busy); +DEFINE_ALLOC_EVENT(xfs_alloc_size_neither); +DEFINE_ALLOC_EVENT(xfs_alloc_size_noentry); +DEFINE_ALLOC_EVENT(xfs_alloc_size_nominleft); +DEFINE_ALLOC_EVENT(xfs_alloc_size_done); +DEFINE_ALLOC_EVENT(xfs_alloc_size_error); +DEFINE_ALLOC_EVENT(xfs_alloc_size_busy); +DEFINE_ALLOC_EVENT(xfs_alloc_small_freelist); +DEFINE_ALLOC_EVENT(xfs_alloc_small_notenough); +DEFINE_ALLOC_EVENT(xfs_alloc_small_done); +DEFINE_ALLOC_EVENT(xfs_alloc_small_error); +DEFINE_ALLOC_EVENT(xfs_alloc_vextent_badargs); +DEFINE_ALLOC_EVENT(xfs_alloc_vextent_skip_deadlock); +DEFINE_ALLOC_EVENT(xfs_alloc_vextent_nofix); +DEFINE_ALLOC_EVENT(xfs_alloc_vextent_noagbp); +DEFINE_ALLOC_EVENT(xfs_alloc_vextent_loopfailed); +DEFINE_ALLOC_EVENT(xfs_alloc_vextent_allfailed); + +DEFINE_ALLOC_EVENT(xfs_alloc_vextent_this_ag); +DEFINE_ALLOC_EVENT(xfs_alloc_vextent_start_ag); +DEFINE_ALLOC_EVENT(xfs_alloc_vextent_first_ag); +DEFINE_ALLOC_EVENT(xfs_alloc_vextent_exact_bno); +DEFINE_ALLOC_EVENT(xfs_alloc_vextent_near_bno); +DEFINE_ALLOC_EVENT(xfs_alloc_vextent_finish); + +TRACE_EVENT(xfs_alloc_cur_check, + TP_PROTO(struct xfs_btree_cur *cur, xfs_agblock_t bno, + xfs_extlen_t len, xfs_extlen_t diff, bool new), + TP_ARGS(cur, bno, len, diff, new), + TP_STRUCT__entry( + __field(dev_t, dev) + __string(name, cur->bc_ops->name) + __field(xfs_agblock_t, bno) + __field(xfs_extlen_t, len) + __field(xfs_extlen_t, diff) + __field(bool, new) + ), + TP_fast_assign( + __entry->dev = cur->bc_mp->m_super->s_dev; + __assign_str(name); + __entry->bno = bno; + __entry->len = len; + __entry->diff = diff; + __entry->new = new; + ), + TP_printk("dev %d:%d %sbt agbno 0x%x fsbcount 0x%x diff 0x%x new %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __get_str(name), + __entry->bno, __entry->len, __entry->diff, __entry->new) +) + +DECLARE_EVENT_CLASS(xfs_da_class, + TP_PROTO(struct xfs_da_args *args), + TP_ARGS(args), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __dynamic_array(char, name, args->namelen) + __field(int, namelen) + __field(xfs_dahash_t, hashval) + __field(xfs_ino_t, inumber) + __field(uint32_t, op_flags) + __field(xfs_ino_t, owner) + ), + TP_fast_assign( + __entry->dev = VFS_I(args->dp)->i_sb->s_dev; + __entry->ino = I_INO(args->dp); + if (args->namelen) + memcpy(__get_str(name), args->name, args->namelen); + __entry->namelen = args->namelen; + __entry->hashval = args->hashval; + __entry->inumber = args->inumber; + __entry->op_flags = args->op_flags; + __entry->owner = args->owner; + ), + TP_printk("dev %d:%d ino 0x%llx name %.*s namelen %d hashval 0x%x " + "inumber 0x%llx op_flags %s owner 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->namelen, + __entry->namelen ? __get_str(name) : NULL, + __entry->namelen, + __entry->hashval, + __entry->inumber, + __print_flags(__entry->op_flags, "|", XFS_DA_OP_FLAGS), + __entry->owner) +) + +#define DEFINE_DIR2_EVENT(name) \ +DEFINE_EVENT(xfs_da_class, name, \ + TP_PROTO(struct xfs_da_args *args), \ + TP_ARGS(args)) +DEFINE_DIR2_EVENT(xfs_dir2_sf_addname); +DEFINE_DIR2_EVENT(xfs_dir2_sf_create); +DEFINE_DIR2_EVENT(xfs_dir2_sf_lookup); +DEFINE_DIR2_EVENT(xfs_dir2_sf_replace); +DEFINE_DIR2_EVENT(xfs_dir2_sf_removename); +DEFINE_DIR2_EVENT(xfs_dir2_sf_toino4); +DEFINE_DIR2_EVENT(xfs_dir2_sf_toino8); +DEFINE_DIR2_EVENT(xfs_dir2_sf_to_block); +DEFINE_DIR2_EVENT(xfs_dir2_block_addname); +DEFINE_DIR2_EVENT(xfs_dir2_block_lookup); +DEFINE_DIR2_EVENT(xfs_dir2_block_replace); +DEFINE_DIR2_EVENT(xfs_dir2_block_removename); +DEFINE_DIR2_EVENT(xfs_dir2_block_to_sf); +DEFINE_DIR2_EVENT(xfs_dir2_block_to_leaf); +DEFINE_DIR2_EVENT(xfs_dir2_leaf_addname); +DEFINE_DIR2_EVENT(xfs_dir2_leaf_lookup); +DEFINE_DIR2_EVENT(xfs_dir2_leaf_replace); +DEFINE_DIR2_EVENT(xfs_dir2_leaf_removename); +DEFINE_DIR2_EVENT(xfs_dir2_leaf_to_block); +DEFINE_DIR2_EVENT(xfs_dir2_leaf_to_node); +DEFINE_DIR2_EVENT(xfs_dir2_node_addname); +DEFINE_DIR2_EVENT(xfs_dir2_node_lookup); +DEFINE_DIR2_EVENT(xfs_dir2_node_replace); +DEFINE_DIR2_EVENT(xfs_dir2_node_removename); +DEFINE_DIR2_EVENT(xfs_dir2_node_to_leaf); + +DECLARE_EVENT_CLASS(xfs_attr_class, + TP_PROTO(struct xfs_da_args *args), + TP_ARGS(args), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __dynamic_array(char, name, args->namelen) + __field(int, namelen) + __field(int, valuelen) + __field(xfs_dahash_t, hashval) + __field(unsigned int, attr_filter) + __field(uint32_t, op_flags) + ), + TP_fast_assign( + __entry->dev = VFS_I(args->dp)->i_sb->s_dev; + __entry->ino = I_INO(args->dp); + if (args->namelen) + memcpy(__get_str(name), args->name, args->namelen); + __entry->namelen = args->namelen; + __entry->valuelen = args->valuelen; + __entry->hashval = args->hashval; + __entry->attr_filter = args->attr_filter; + __entry->op_flags = args->op_flags; + ), + TP_printk("dev %d:%d ino 0x%llx name %.*s namelen %d valuelen %d " + "hashval 0x%x filter %s op_flags %s", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->namelen, + __entry->namelen ? __get_str(name) : NULL, + __entry->namelen, + __entry->valuelen, + __entry->hashval, + __print_flags(__entry->attr_filter, "|", + XFS_ATTR_FILTER_FLAGS), + __print_flags(__entry->op_flags, "|", XFS_DA_OP_FLAGS)) +) + +#define DEFINE_ATTR_EVENT(name) \ +DEFINE_EVENT(xfs_attr_class, name, \ + TP_PROTO(struct xfs_da_args *args), \ + TP_ARGS(args)) +DEFINE_ATTR_EVENT(xfs_attr_sf_add); +DEFINE_ATTR_EVENT(xfs_attr_sf_addname); +DEFINE_ATTR_EVENT(xfs_attr_sf_create); +DEFINE_ATTR_EVENT(xfs_attr_sf_lookup); +DEFINE_ATTR_EVENT(xfs_attr_sf_remove); +DEFINE_ATTR_EVENT(xfs_attr_sf_replace); +DEFINE_ATTR_EVENT(xfs_attr_sf_to_leaf); + +DEFINE_ATTR_EVENT(xfs_attr_leaf_add); +DEFINE_ATTR_EVENT(xfs_attr_leaf_add_old); +DEFINE_ATTR_EVENT(xfs_attr_leaf_add_new); +DEFINE_ATTR_EVENT(xfs_attr_leaf_add_work); +DEFINE_ATTR_EVENT(xfs_attr_leaf_create); +DEFINE_ATTR_EVENT(xfs_attr_leaf_compact); +DEFINE_ATTR_EVENT(xfs_attr_leaf_get); +DEFINE_ATTR_EVENT(xfs_attr_leaf_lookup); +DEFINE_ATTR_EVENT(xfs_attr_leaf_replace); +DEFINE_ATTR_EVENT(xfs_attr_leaf_remove); +DEFINE_ATTR_EVENT(xfs_attr_leaf_removename); +DEFINE_ATTR_EVENT(xfs_attr_leaf_split); +DEFINE_ATTR_EVENT(xfs_attr_leaf_split_before); +DEFINE_ATTR_EVENT(xfs_attr_leaf_split_after); +DEFINE_ATTR_EVENT(xfs_attr_leaf_clearflag); +DEFINE_ATTR_EVENT(xfs_attr_leaf_setflag); +DEFINE_ATTR_EVENT(xfs_attr_leaf_flipflags); +DEFINE_ATTR_EVENT(xfs_attr_leaf_to_sf); +DEFINE_ATTR_EVENT(xfs_attr_leaf_to_node); +DEFINE_ATTR_EVENT(xfs_attr_leaf_rebalance); +DEFINE_ATTR_EVENT(xfs_attr_leaf_unbalance); +DEFINE_ATTR_EVENT(xfs_attr_leaf_toosmall); + +DEFINE_ATTR_EVENT(xfs_attr_node_addname); +DEFINE_ATTR_EVENT(xfs_attr_node_get); +DEFINE_ATTR_EVENT(xfs_attr_node_replace); + +DEFINE_ATTR_EVENT(xfs_attr_rmtval_get); + +#define DEFINE_DA_EVENT(name) \ +DEFINE_EVENT(xfs_da_class, name, \ + TP_PROTO(struct xfs_da_args *args), \ + TP_ARGS(args)) +DEFINE_DA_EVENT(xfs_da_split); +DEFINE_DA_EVENT(xfs_da_join); +DEFINE_DA_EVENT(xfs_da_link_before); +DEFINE_DA_EVENT(xfs_da_link_after); +DEFINE_DA_EVENT(xfs_da_unlink_back); +DEFINE_DA_EVENT(xfs_da_unlink_forward); +DEFINE_DA_EVENT(xfs_da_root_split); +DEFINE_DA_EVENT(xfs_da_root_join); +DEFINE_DA_EVENT(xfs_da_node_add); +DEFINE_DA_EVENT(xfs_da_node_create); +DEFINE_DA_EVENT(xfs_da_node_split); +DEFINE_DA_EVENT(xfs_da_node_remove); +DEFINE_DA_EVENT(xfs_da_node_rebalance); +DEFINE_DA_EVENT(xfs_da_node_unbalance); +DEFINE_DA_EVENT(xfs_da_node_toosmall); +DEFINE_DA_EVENT(xfs_da_swap_lastblock); +DEFINE_DA_EVENT(xfs_da_grow_inode); +DEFINE_DA_EVENT(xfs_da_shrink_inode); +DEFINE_DA_EVENT(xfs_da_fixhashpath); +DEFINE_DA_EVENT(xfs_da_path_shift); + +DECLARE_EVENT_CLASS(xfs_dir2_space_class, + TP_PROTO(struct xfs_da_args *args, int idx), + TP_ARGS(args, idx), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(uint32_t, op_flags) + __field(int, idx) + ), + TP_fast_assign( + __entry->dev = VFS_I(args->dp)->i_sb->s_dev; + __entry->ino = I_INO(args->dp); + __entry->op_flags = args->op_flags; + __entry->idx = idx; + ), + TP_printk("dev %d:%d ino 0x%llx op_flags %s index %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __print_flags(__entry->op_flags, "|", XFS_DA_OP_FLAGS), + __entry->idx) +) + +#define DEFINE_DIR2_SPACE_EVENT(name) \ +DEFINE_EVENT(xfs_dir2_space_class, name, \ + TP_PROTO(struct xfs_da_args *args, int idx), \ + TP_ARGS(args, idx)) +DEFINE_DIR2_SPACE_EVENT(xfs_dir2_leafn_add); +DEFINE_DIR2_SPACE_EVENT(xfs_dir2_leafn_remove); +DEFINE_DIR2_SPACE_EVENT(xfs_dir2_grow_inode); +DEFINE_DIR2_SPACE_EVENT(xfs_dir2_shrink_inode); + +TRACE_EVENT(xfs_dir2_leafn_moveents, + TP_PROTO(struct xfs_da_args *args, int src_idx, int dst_idx, int count), + TP_ARGS(args, src_idx, dst_idx, count), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(uint32_t, op_flags) + __field(int, src_idx) + __field(int, dst_idx) + __field(int, count) + ), + TP_fast_assign( + __entry->dev = VFS_I(args->dp)->i_sb->s_dev; + __entry->ino = I_INO(args->dp); + __entry->op_flags = args->op_flags; + __entry->src_idx = src_idx; + __entry->dst_idx = dst_idx; + __entry->count = count; + ), + TP_printk("dev %d:%d ino 0x%llx op_flags %s " + "src_idx %d dst_idx %d count %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __print_flags(__entry->op_flags, "|", XFS_DA_OP_FLAGS), + __entry->src_idx, + __entry->dst_idx, + __entry->count) +); + +#define XFS_SWAPEXT_INODES \ + { 0, "target" }, \ + { 1, "temp" } + +TRACE_DEFINE_ENUM(XFS_DINODE_FMT_DEV); +TRACE_DEFINE_ENUM(XFS_DINODE_FMT_LOCAL); +TRACE_DEFINE_ENUM(XFS_DINODE_FMT_EXTENTS); +TRACE_DEFINE_ENUM(XFS_DINODE_FMT_BTREE); +TRACE_DEFINE_ENUM(XFS_DINODE_FMT_UUID); +TRACE_DEFINE_ENUM(XFS_DINODE_FMT_META_BTREE); + +DECLARE_EVENT_CLASS(xfs_swap_extent_class, + TP_PROTO(struct xfs_inode *ip, int which), + TP_ARGS(ip, which), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(int, which) + __field(xfs_ino_t, ino) + __field(int, format) + __field(xfs_extnum_t, nex) + __field(int, broot_size) + __field(int, fork_off) + ), + TP_fast_assign( + __entry->dev = VFS_I(ip)->i_sb->s_dev; + __entry->which = which; + __entry->ino = I_INO(ip); + __entry->format = ip->i_df.if_format; + __entry->nex = ip->i_df.if_nextents; + __entry->broot_size = ip->i_df.if_broot_bytes; + __entry->fork_off = xfs_inode_fork_boff(ip); + ), + TP_printk("dev %d:%d ino 0x%llx (%s), %s format, num_extents %llu, " + "broot size %d, forkoff 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __print_symbolic(__entry->which, XFS_SWAPEXT_INODES), + __print_symbolic(__entry->format, XFS_INODE_FORMAT_STR), + __entry->nex, + __entry->broot_size, + __entry->fork_off) +) + +#define DEFINE_SWAPEXT_EVENT(name) \ +DEFINE_EVENT(xfs_swap_extent_class, name, \ + TP_PROTO(struct xfs_inode *ip, int which), \ + TP_ARGS(ip, which)) + +DEFINE_SWAPEXT_EVENT(xfs_swap_extent_before); +DEFINE_SWAPEXT_EVENT(xfs_swap_extent_after); + +TRACE_EVENT(xfs_log_recover, + TP_PROTO(struct xlog *log, xfs_daddr_t headblk, xfs_daddr_t tailblk), + TP_ARGS(log, headblk, tailblk), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_daddr_t, headblk) + __field(xfs_daddr_t, tailblk) + ), + TP_fast_assign( + __entry->dev = log->l_mp->m_super->s_dev; + __entry->headblk = headblk; + __entry->tailblk = tailblk; + ), + TP_printk("dev %d:%d headblk 0x%llx tailblk 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), __entry->headblk, + __entry->tailblk) +) + +TRACE_EVENT(xfs_log_recover_record, + TP_PROTO(struct xlog *log, struct xlog_rec_header *rhead, int pass), + TP_ARGS(log, rhead, pass), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_lsn_t, lsn) + __field(int, len) + __field(int, num_logops) + __field(int, pass) + ), + TP_fast_assign( + __entry->dev = log->l_mp->m_super->s_dev; + __entry->lsn = be64_to_cpu(rhead->h_lsn); + __entry->len = be32_to_cpu(rhead->h_len); + __entry->num_logops = be32_to_cpu(rhead->h_num_logops); + __entry->pass = pass; + ), + TP_printk("dev %d:%d lsn 0x%llx len 0x%x num_logops 0x%x pass %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->lsn, __entry->len, __entry->num_logops, + __entry->pass) +) + +DECLARE_EVENT_CLASS(xfs_log_recover_item_class, + TP_PROTO(struct xlog *log, struct xlog_recover *trans, + struct xlog_recover_item *item, int pass), + TP_ARGS(log, trans, item, pass), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned long, item) + __field(xlog_tid_t, tid) + __field(xfs_lsn_t, lsn) + __field(int, type) + __field(int, pass) + __field(int, count) + __field(int, total) + ), + TP_fast_assign( + __entry->dev = log->l_mp->m_super->s_dev; + __entry->item = (unsigned long)item; + __entry->tid = trans->r_log_tid; + __entry->lsn = trans->r_lsn; + __entry->type = ITEM_TYPE(item); + __entry->pass = pass; + __entry->count = item->ri_cnt; + __entry->total = item->ri_total; + ), + TP_printk("dev %d:%d tid 0x%x lsn 0x%llx, pass %d, item %p, " + "item type %s item region count/total %d/%d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->tid, + __entry->lsn, + __entry->pass, + (void *)__entry->item, + __print_symbolic(__entry->type, XFS_LI_TYPE_DESC), + __entry->count, + __entry->total) +) + +#define DEFINE_LOG_RECOVER_ITEM(name) \ +DEFINE_EVENT(xfs_log_recover_item_class, name, \ + TP_PROTO(struct xlog *log, struct xlog_recover *trans, \ + struct xlog_recover_item *item, int pass), \ + TP_ARGS(log, trans, item, pass)) + +DEFINE_LOG_RECOVER_ITEM(xfs_log_recover_item_add); +DEFINE_LOG_RECOVER_ITEM(xfs_log_recover_item_add_cont); +DEFINE_LOG_RECOVER_ITEM(xfs_log_recover_item_reorder_head); +DEFINE_LOG_RECOVER_ITEM(xfs_log_recover_item_reorder_tail); +DEFINE_LOG_RECOVER_ITEM(xfs_log_recover_item_recover); + +DECLARE_EVENT_CLASS(xfs_log_recover_buf_item_class, + TP_PROTO(struct xlog *log, struct xfs_buf_log_format *buf_f), + TP_ARGS(log, buf_f), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(int64_t, blkno) + __field(unsigned short, len) + __field(unsigned short, flags) + __field(unsigned short, size) + __field(unsigned int, map_size) + ), + TP_fast_assign( + __entry->dev = log->l_mp->m_super->s_dev; + __entry->blkno = buf_f->blf_blkno; + __entry->len = buf_f->blf_len; + __entry->flags = buf_f->blf_flags; + __entry->size = buf_f->blf_size; + __entry->map_size = buf_f->blf_map_size; + ), + TP_printk("dev %d:%d daddr 0x%llx, bbcount 0x%x, flags 0x%x, size %d, " + "map_size %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->blkno, + __entry->len, + __entry->flags, + __entry->size, + __entry->map_size) +) + +#define DEFINE_LOG_RECOVER_BUF_ITEM(name) \ +DEFINE_EVENT(xfs_log_recover_buf_item_class, name, \ + TP_PROTO(struct xlog *log, struct xfs_buf_log_format *buf_f), \ + TP_ARGS(log, buf_f)) + +DEFINE_LOG_RECOVER_BUF_ITEM(xfs_log_recover_buf_not_cancel); +DEFINE_LOG_RECOVER_BUF_ITEM(xfs_log_recover_buf_cancel); +DEFINE_LOG_RECOVER_BUF_ITEM(xfs_log_recover_buf_cancel_add); +DEFINE_LOG_RECOVER_BUF_ITEM(xfs_log_recover_buf_cancel_ref_inc); +DEFINE_LOG_RECOVER_BUF_ITEM(xfs_log_recover_buf_recover); +DEFINE_LOG_RECOVER_BUF_ITEM(xfs_log_recover_buf_skip); +DEFINE_LOG_RECOVER_BUF_ITEM(xfs_log_recover_buf_inode_buf); +DEFINE_LOG_RECOVER_BUF_ITEM(xfs_log_recover_buf_reg_buf); +DEFINE_LOG_RECOVER_BUF_ITEM(xfs_log_recover_buf_dquot_buf); + +DECLARE_EVENT_CLASS(xfs_log_recover_ino_item_class, + TP_PROTO(struct xlog *log, struct xfs_inode_log_format *in_f), + TP_ARGS(log, in_f), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(unsigned short, size) + __field(int, fields) + __field(unsigned short, asize) + __field(unsigned short, dsize) + __field(int64_t, blkno) + __field(int, len) + __field(int, boffset) + ), + TP_fast_assign( + __entry->dev = log->l_mp->m_super->s_dev; + __entry->ino = in_f->ilf_ino; + __entry->size = in_f->ilf_size; + __entry->fields = in_f->ilf_fields; + __entry->asize = in_f->ilf_asize; + __entry->dsize = in_f->ilf_dsize; + __entry->blkno = in_f->ilf_blkno; + __entry->len = in_f->ilf_len; + __entry->boffset = in_f->ilf_boffset; + ), + TP_printk("dev %d:%d ino 0x%llx, size %u, fields 0x%x, asize %d, " + "dsize %d, daddr 0x%llx, bbcount 0x%x, boffset %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->size, + __entry->fields, + __entry->asize, + __entry->dsize, + __entry->blkno, + __entry->len, + __entry->boffset) +) +#define DEFINE_LOG_RECOVER_INO_ITEM(name) \ +DEFINE_EVENT(xfs_log_recover_ino_item_class, name, \ + TP_PROTO(struct xlog *log, struct xfs_inode_log_format *in_f), \ + TP_ARGS(log, in_f)) + +DEFINE_LOG_RECOVER_INO_ITEM(xfs_log_recover_inode_recover); +DEFINE_LOG_RECOVER_INO_ITEM(xfs_log_recover_inode_cancel); +DEFINE_LOG_RECOVER_INO_ITEM(xfs_log_recover_inode_skip); + +DECLARE_EVENT_CLASS(xfs_log_recover_icreate_item_class, + TP_PROTO(struct xlog *log, struct xfs_icreate_log *in_f), + TP_ARGS(log, in_f), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, agbno) + __field(unsigned int, count) + __field(unsigned int, isize) + __field(xfs_agblock_t, length) + __field(unsigned int, gen) + ), + TP_fast_assign( + __entry->dev = log->l_mp->m_super->s_dev; + __entry->agno = be32_to_cpu(in_f->icl_ag); + __entry->agbno = be32_to_cpu(in_f->icl_agbno); + __entry->count = be32_to_cpu(in_f->icl_count); + __entry->isize = be32_to_cpu(in_f->icl_isize); + __entry->length = be32_to_cpu(in_f->icl_length); + __entry->gen = be32_to_cpu(in_f->icl_gen); + ), + TP_printk("dev %d:%d agno 0x%x agbno 0x%x fsbcount 0x%x ireccount %u isize %u gen 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->agbno, + __entry->length, + __entry->count, + __entry->isize, + __entry->gen) +) +#define DEFINE_LOG_RECOVER_ICREATE_ITEM(name) \ +DEFINE_EVENT(xfs_log_recover_icreate_item_class, name, \ + TP_PROTO(struct xlog *log, struct xfs_icreate_log *in_f), \ + TP_ARGS(log, in_f)) + +DEFINE_LOG_RECOVER_ICREATE_ITEM(xfs_log_recover_icreate_cancel); +DEFINE_LOG_RECOVER_ICREATE_ITEM(xfs_log_recover_icreate_recover); + +DECLARE_EVENT_CLASS(xfs_discard_class, + TP_PROTO(const struct xfs_group *xg, xfs_agblock_t agbno, + xfs_extlen_t len), + TP_ARGS(xg, agbno, len), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(enum xfs_group_type, type) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, agbno) + __field(xfs_extlen_t, len) + ), + TP_fast_assign( + __entry->dev = xg->xg_mount->m_super->s_dev; + __entry->type = xg->xg_type; + __entry->agno = xg->xg_gno; + __entry->agbno = agbno; + __entry->len = len; + ), + TP_printk("dev %d:%d %sno 0x%x gbno 0x%x fsbcount 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agno, + __entry->agbno, + __entry->len) +) + +#define DEFINE_DISCARD_EVENT(name) \ +DEFINE_EVENT(xfs_discard_class, name, \ + TP_PROTO(const struct xfs_group *xg, xfs_agblock_t agbno, \ + xfs_extlen_t len), \ + TP_ARGS(xg, agbno, len)) +DEFINE_DISCARD_EVENT(xfs_discard_extent); +DEFINE_DISCARD_EVENT(xfs_discard_toosmall); +DEFINE_DISCARD_EVENT(xfs_discard_exclude); +DEFINE_DISCARD_EVENT(xfs_discard_busy); + +DECLARE_EVENT_CLASS(xfs_rtdiscard_class, + TP_PROTO(struct xfs_mount *mp, + xfs_rtblock_t rtbno, xfs_rtblock_t len), + TP_ARGS(mp, rtbno, len), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_rtblock_t, rtbno) + __field(xfs_rtblock_t, len) + ), + TP_fast_assign( + __entry->dev = mp->m_rtdev_targp->bt_dev; + __entry->rtbno = rtbno; + __entry->len = len; + ), + TP_printk("dev %d:%d rtbno 0x%llx rtbcount 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->rtbno, + __entry->len) +) + +#define DEFINE_RTDISCARD_EVENT(name) \ +DEFINE_EVENT(xfs_rtdiscard_class, name, \ + TP_PROTO(struct xfs_mount *mp, \ + xfs_rtblock_t rtbno, xfs_rtblock_t len), \ + TP_ARGS(mp, rtbno, len)) +DEFINE_RTDISCARD_EVENT(xfs_discard_rtextent); +DEFINE_RTDISCARD_EVENT(xfs_discard_rttoosmall); + +DECLARE_EVENT_CLASS(xfs_btree_cur_class, + TP_PROTO(struct xfs_btree_cur *cur, int level, struct xfs_buf *bp), + TP_ARGS(cur, level, bp), + TP_STRUCT__entry( + __field(dev_t, dev) + __string(name, cur->bc_ops->name) + __field(int, level) + __field(int, nlevels) + __field(int, ptr) + __field(xfs_daddr_t, daddr) + ), + TP_fast_assign( + __entry->dev = cur->bc_mp->m_super->s_dev; + __assign_str(name); + __entry->level = level; + __entry->nlevels = cur->bc_nlevels; + __entry->ptr = cur->bc_levels[level].ptr; + __entry->daddr = bp ? xfs_buf_daddr(bp) : -1; + ), + TP_printk("dev %d:%d %sbt level %d/%d ptr %d daddr 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __get_str(name), + __entry->level, + __entry->nlevels, + __entry->ptr, + (unsigned long long)__entry->daddr) +) + +#define DEFINE_BTREE_CUR_EVENT(name) \ +DEFINE_EVENT(xfs_btree_cur_class, name, \ + TP_PROTO(struct xfs_btree_cur *cur, int level, struct xfs_buf *bp), \ + TP_ARGS(cur, level, bp)) +DEFINE_BTREE_CUR_EVENT(xfs_btree_updkeys); +DEFINE_BTREE_CUR_EVENT(xfs_btree_overlapped_query_range); + +TRACE_EVENT(xfs_btree_alloc_block, + TP_PROTO(struct xfs_btree_cur *cur, union xfs_btree_ptr *ptr, int stat, + int error), + TP_ARGS(cur, ptr, stat, error), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_ino_t, ino) + __string(name, cur->bc_ops->name) + __field(int, error) + __field(xfs_agblock_t, agbno) + ), + TP_fast_assign( + __entry->dev = cur->bc_mp->m_super->s_dev; + switch (cur->bc_ops->type) { + case XFS_BTREE_TYPE_INODE: + __entry->agno = 0; + __entry->ino = I_INO(cur->bc_ino.ip); + break; + case XFS_BTREE_TYPE_AG: + __entry->agno = cur->bc_group->xg_gno; + __entry->ino = 0; + break; + case XFS_BTREE_TYPE_MEM: + __entry->agno = 0; + __entry->ino = 0; + break; + } + __assign_str(name); + __entry->error = error; + if (!error && stat) { + if (cur->bc_ops->ptr_len == XFS_BTREE_LONG_PTR_LEN) { + xfs_fsblock_t fsb = be64_to_cpu(ptr->l); + + __entry->agno = XFS_FSB_TO_AGNO(cur->bc_mp, + fsb); + __entry->agbno = XFS_FSB_TO_AGBNO(cur->bc_mp, + fsb); + } else { + __entry->agbno = be32_to_cpu(ptr->s); + } + } else { + __entry->agbno = NULLAGBLOCK; + } + ), + TP_printk("dev %d:%d %sbt agno 0x%x ino 0x%llx agbno 0x%x error %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __get_str(name), + __entry->agno, + __entry->ino, + __entry->agbno, + __entry->error) +); + +TRACE_EVENT(xfs_btree_free_block, + TP_PROTO(struct xfs_btree_cur *cur, struct xfs_buf *bp), + TP_ARGS(cur, bp), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_ino_t, ino) + __string(name, cur->bc_ops->name) + __field(xfs_agblock_t, agbno) + ), + TP_fast_assign( + __entry->dev = cur->bc_mp->m_super->s_dev; + __entry->agno = xfs_daddr_to_agno(cur->bc_mp, + xfs_buf_daddr(bp)); + if (cur->bc_ops->type == XFS_BTREE_TYPE_INODE) + __entry->ino = I_INO(cur->bc_ino.ip); + else + __entry->ino = 0; + __assign_str(name); + __entry->agbno = xfs_daddr_to_agbno(cur->bc_mp, + xfs_buf_daddr(bp)); + ), + TP_printk("dev %d:%d %sbt agno 0x%x ino 0x%llx agbno 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __get_str(name), + __entry->agno, + __entry->ino, + __entry->agbno) +); + +/* deferred ops */ +struct xfs_defer_pending; + +DECLARE_EVENT_CLASS(xfs_defer_class, + TP_PROTO(struct xfs_trans *tp, unsigned long caller_ip), + TP_ARGS(tp, caller_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(struct xfs_trans *, tp) + __field(char, committed) + __field(unsigned long, caller_ip) + ), + TP_fast_assign( + __entry->dev = tp->t_mountp->m_super->s_dev; + __entry->tp = tp; + __entry->caller_ip = caller_ip; + ), + TP_printk("dev %d:%d tp %p caller %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->tp, + (char *)__entry->caller_ip) +) +#define DEFINE_DEFER_EVENT(name) \ +DEFINE_EVENT(xfs_defer_class, name, \ + TP_PROTO(struct xfs_trans *tp, unsigned long caller_ip), \ + TP_ARGS(tp, caller_ip)) + +DECLARE_EVENT_CLASS(xfs_defer_error_class, + TP_PROTO(struct xfs_trans *tp, int error), + TP_ARGS(tp, error), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(struct xfs_trans *, tp) + __field(char, committed) + __field(int, error) + ), + TP_fast_assign( + __entry->dev = tp->t_mountp->m_super->s_dev; + __entry->tp = tp; + __entry->error = error; + ), + TP_printk("dev %d:%d tp %p err %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->tp, + __entry->error) +) +#define DEFINE_DEFER_ERROR_EVENT(name) \ +DEFINE_EVENT(xfs_defer_error_class, name, \ + TP_PROTO(struct xfs_trans *tp, int error), \ + TP_ARGS(tp, error)) + +DECLARE_EVENT_CLASS(xfs_defer_pending_class, + TP_PROTO(struct xfs_mount *mp, struct xfs_defer_pending *dfp), + TP_ARGS(mp, dfp), + TP_STRUCT__entry( + __field(dev_t, dev) + __string(name, dfp->dfp_ops->name) + __field(void *, intent) + __field(unsigned int, flags) + __field(char, committed) + __field(int, nr) + ), + TP_fast_assign( + __entry->dev = mp ? mp->m_super->s_dev : 0; + __assign_str(name); + __entry->intent = dfp->dfp_intent; + __entry->flags = dfp->dfp_flags; + __entry->committed = dfp->dfp_done != NULL; + __entry->nr = dfp->dfp_count; + ), + TP_printk("dev %d:%d optype %s intent %p flags %s committed %d nr %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __get_str(name), + __entry->intent, + __print_flags(__entry->flags, "|", XFS_DEFER_PENDING_STRINGS), + __entry->committed, + __entry->nr) +) +#define DEFINE_DEFER_PENDING_EVENT(name) \ +DEFINE_EVENT(xfs_defer_pending_class, name, \ + TP_PROTO(struct xfs_mount *mp, struct xfs_defer_pending *dfp), \ + TP_ARGS(mp, dfp)) + +DEFINE_DEFER_EVENT(xfs_defer_cancel); +DEFINE_DEFER_EVENT(xfs_defer_trans_roll); +DEFINE_DEFER_EVENT(xfs_defer_trans_abort); +DEFINE_DEFER_EVENT(xfs_defer_finish); +DEFINE_DEFER_EVENT(xfs_defer_finish_done); + +DEFINE_DEFER_ERROR_EVENT(xfs_defer_trans_roll_error); +DEFINE_DEFER_ERROR_EVENT(xfs_defer_finish_error); + +DEFINE_DEFER_PENDING_EVENT(xfs_defer_create_intent); +DEFINE_DEFER_PENDING_EVENT(xfs_defer_cancel_list); +DEFINE_DEFER_PENDING_EVENT(xfs_defer_pending_finish); +DEFINE_DEFER_PENDING_EVENT(xfs_defer_pending_abort); +DEFINE_DEFER_PENDING_EVENT(xfs_defer_relog_intent); +DEFINE_DEFER_PENDING_EVENT(xfs_defer_isolate_paused); +DEFINE_DEFER_PENDING_EVENT(xfs_defer_item_pause); +DEFINE_DEFER_PENDING_EVENT(xfs_defer_item_unpause); + +DECLARE_EVENT_CLASS(xfs_free_extent_deferred_class, + TP_PROTO(struct xfs_mount *mp, struct xfs_extent_free_item *free), + TP_ARGS(mp, free), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(enum xfs_group_type, type) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, agbno) + __field(xfs_extlen_t, len) + __field(unsigned int, flags) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->type = free->xefi_group->xg_type; + __entry->agno = free->xefi_group->xg_gno; + __entry->agbno = xfs_fsb_to_gbno(mp, free->xefi_startblock, + free->xefi_group->xg_type); + __entry->len = free->xefi_blockcount; + __entry->flags = free->xefi_flags; + ), + TP_printk("dev %d:%d %sno 0x%x gbno 0x%x fsbcount 0x%x flags 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agno, + __entry->agbno, + __entry->len, + __entry->flags) +); +#define DEFINE_FREE_EXTENT_DEFERRED_EVENT(name) \ +DEFINE_EVENT(xfs_free_extent_deferred_class, name, \ + TP_PROTO(struct xfs_mount *mp, struct xfs_extent_free_item *free), \ + TP_ARGS(mp, free)) +DEFINE_FREE_EXTENT_DEFERRED_EVENT(xfs_agfl_free_deferred); +DEFINE_FREE_EXTENT_DEFERRED_EVENT(xfs_extent_free_defer); +DEFINE_FREE_EXTENT_DEFERRED_EVENT(xfs_extent_free_deferred); + +DECLARE_EVENT_CLASS(xfs_defer_pending_item_class, + TP_PROTO(struct xfs_mount *mp, struct xfs_defer_pending *dfp, + void *item), + TP_ARGS(mp, dfp, item), + TP_STRUCT__entry( + __field(dev_t, dev) + __string(name, dfp->dfp_ops->name) + __field(void *, intent) + __field(void *, item) + __field(char, committed) + __field(unsigned int, flags) + __field(int, nr) + ), + TP_fast_assign( + __entry->dev = mp ? mp->m_super->s_dev : 0; + __assign_str(name); + __entry->intent = dfp->dfp_intent; + __entry->item = item; + __entry->committed = dfp->dfp_done != NULL; + __entry->flags = dfp->dfp_flags; + __entry->nr = dfp->dfp_count; + ), + TP_printk("dev %d:%d optype %s intent %p item %p flags %s committed %d nr %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __get_str(name), + __entry->intent, + __entry->item, + __print_flags(__entry->flags, "|", XFS_DEFER_PENDING_STRINGS), + __entry->committed, + __entry->nr) +) +#define DEFINE_DEFER_PENDING_ITEM_EVENT(name) \ +DEFINE_EVENT(xfs_defer_pending_item_class, name, \ + TP_PROTO(struct xfs_mount *mp, struct xfs_defer_pending *dfp, \ + void *item), \ + TP_ARGS(mp, dfp, item)) + +DEFINE_DEFER_PENDING_ITEM_EVENT(xfs_defer_add_item); +DEFINE_DEFER_PENDING_ITEM_EVENT(xfs_defer_cancel_item); +DEFINE_DEFER_PENDING_ITEM_EVENT(xfs_defer_finish_item); + +/* rmap tracepoints */ +DECLARE_EVENT_CLASS(xfs_rmap_class, + TP_PROTO(struct xfs_btree_cur *cur, + xfs_agblock_t gbno, xfs_extlen_t len, bool unwritten, + const struct xfs_owner_info *oinfo), + TP_ARGS(cur, gbno, len, unwritten, oinfo), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(enum xfs_group_type, type) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, gbno) + __field(xfs_extlen_t, len) + __field(uint64_t, owner) + __field(uint64_t, offset) + __field(unsigned long, flags) + ), + TP_fast_assign( + __entry->dev = cur->bc_mp->m_super->s_dev; + __entry->type = cur->bc_group->xg_type; + __entry->agno = cur->bc_group->xg_gno; + __entry->gbno = gbno; + __entry->len = len; + __entry->owner = oinfo->oi_owner; + __entry->offset = oinfo->oi_offset; + __entry->flags = oinfo->oi_flags; + if (unwritten) + __entry->flags |= XFS_RMAP_UNWRITTEN; + ), + TP_printk("dev %d:%d %sno 0x%x gbno 0x%x fsbcount 0x%x owner 0x%llx fileoff 0x%llx flags 0x%lx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agno, + __entry->gbno, + __entry->len, + __entry->owner, + __entry->offset, + __entry->flags) +); +#define DEFINE_RMAP_EVENT(name) \ +DEFINE_EVENT(xfs_rmap_class, name, \ + TP_PROTO(struct xfs_btree_cur *cur, \ + xfs_agblock_t gbno, xfs_extlen_t len, bool unwritten, \ + const struct xfs_owner_info *oinfo), \ + TP_ARGS(cur, gbno, len, unwritten, oinfo)) + +/* btree cursor error/%ip tracepoint class */ +DECLARE_EVENT_CLASS(xfs_btree_error_class, + TP_PROTO(struct xfs_btree_cur *cur, int error, + unsigned long caller_ip), + TP_ARGS(cur, error, caller_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_ino_t, ino) + __field(int, error) + __field(unsigned long, caller_ip) + ), + TP_fast_assign( + __entry->dev = cur->bc_mp->m_super->s_dev; + switch (cur->bc_ops->type) { + case XFS_BTREE_TYPE_INODE: + __entry->agno = 0; + __entry->ino = I_INO(cur->bc_ino.ip); + break; + case XFS_BTREE_TYPE_AG: + __entry->agno = cur->bc_group->xg_gno; + __entry->ino = 0; + break; + case XFS_BTREE_TYPE_MEM: + __entry->agno = 0; + __entry->ino = 0; + break; + } + __entry->error = error; + __entry->caller_ip = caller_ip; + ), + TP_printk("dev %d:%d agno 0x%x ino 0x%llx error %d caller %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->ino, + __entry->error, + (char *)__entry->caller_ip) +); + +#define DEFINE_BTREE_ERROR_EVENT(name) \ +DEFINE_EVENT(xfs_btree_error_class, name, \ + TP_PROTO(struct xfs_btree_cur *cur, int error, \ + unsigned long caller_ip), \ + TP_ARGS(cur, error, caller_ip)) + +DEFINE_RMAP_EVENT(xfs_rmap_unmap); +DEFINE_RMAP_EVENT(xfs_rmap_unmap_done); +DEFINE_BTREE_ERROR_EVENT(xfs_rmap_unmap_error); +DEFINE_RMAP_EVENT(xfs_rmap_map); +DEFINE_RMAP_EVENT(xfs_rmap_map_done); +DEFINE_BTREE_ERROR_EVENT(xfs_rmap_map_error); +DEFINE_RMAP_EVENT(xfs_rmap_convert); +DEFINE_RMAP_EVENT(xfs_rmap_convert_done); +DEFINE_BTREE_ERROR_EVENT(xfs_rmap_convert_error); + +TRACE_EVENT(xfs_rmap_convert_state, + TP_PROTO(struct xfs_btree_cur *cur, int state, + unsigned long caller_ip), + TP_ARGS(cur, state, caller_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(enum xfs_group_type, type) + __field(xfs_agnumber_t, agno) + __field(int, state) + __field(unsigned long, caller_ip) + ), + TP_fast_assign( + __entry->dev = cur->bc_mp->m_super->s_dev; + __entry->type = cur->bc_group->xg_type; + __entry->agno = cur->bc_group->xg_gno; + __entry->state = state; + __entry->caller_ip = caller_ip; + ), + TP_printk("dev %d:%d %sno 0x%x state %d caller %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agno, + __entry->state, + (char *)__entry->caller_ip) +); + +DECLARE_EVENT_CLASS(xfs_rmapbt_class, + TP_PROTO(struct xfs_btree_cur *cur, + xfs_agblock_t gbno, xfs_extlen_t len, + uint64_t owner, uint64_t offset, unsigned int flags), + TP_ARGS(cur, gbno, len, owner, offset, flags), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(enum xfs_group_type, type) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, gbno) + __field(xfs_extlen_t, len) + __field(uint64_t, owner) + __field(uint64_t, offset) + __field(unsigned int, flags) + ), + TP_fast_assign( + __entry->dev = cur->bc_mp->m_super->s_dev; + __entry->type = cur->bc_group->xg_type; + __entry->agno = cur->bc_group->xg_gno; + __entry->gbno = gbno; + __entry->len = len; + __entry->owner = owner; + __entry->offset = offset; + __entry->flags = flags; + ), + TP_printk("dev %d:%d %sno 0x%x gbno 0x%x fsbcount 0x%x owner 0x%llx fileoff 0x%llx flags 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agno, + __entry->gbno, + __entry->len, + __entry->owner, + __entry->offset, + __entry->flags) +); +#define DEFINE_RMAPBT_EVENT(name) \ +DEFINE_EVENT(xfs_rmapbt_class, name, \ + TP_PROTO(struct xfs_btree_cur *cur, \ + xfs_agblock_t gbno, xfs_extlen_t len, \ + uint64_t owner, uint64_t offset, unsigned int flags), \ + TP_ARGS(cur, gbno, len, owner, offset, flags)) + +TRACE_DEFINE_ENUM(XFS_RMAP_MAP); +TRACE_DEFINE_ENUM(XFS_RMAP_MAP_SHARED); +TRACE_DEFINE_ENUM(XFS_RMAP_UNMAP); +TRACE_DEFINE_ENUM(XFS_RMAP_UNMAP_SHARED); +TRACE_DEFINE_ENUM(XFS_RMAP_CONVERT); +TRACE_DEFINE_ENUM(XFS_RMAP_CONVERT_SHARED); +TRACE_DEFINE_ENUM(XFS_RMAP_ALLOC); +TRACE_DEFINE_ENUM(XFS_RMAP_FREE); + +DECLARE_EVENT_CLASS(xfs_rmap_deferred_class, + TP_PROTO(struct xfs_mount *mp, struct xfs_rmap_intent *ri), + TP_ARGS(mp, ri), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned long long, owner) + __field(enum xfs_group_type, type) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, gbno) + __field(int, whichfork) + __field(xfs_fileoff_t, l_loff) + __field(xfs_filblks_t, l_len) + __field(xfs_exntst_t, l_state) + __field(int, op) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->type = ri->ri_group->xg_type; + __entry->agno = ri->ri_group->xg_gno; + __entry->gbno = xfs_fsb_to_gbno(mp, + ri->ri_bmap.br_startblock, + ri->ri_group->xg_type); + __entry->owner = ri->ri_owner; + __entry->whichfork = ri->ri_whichfork; + __entry->l_loff = ri->ri_bmap.br_startoff; + __entry->l_len = ri->ri_bmap.br_blockcount; + __entry->l_state = ri->ri_bmap.br_state; + __entry->op = ri->ri_type; + ), + TP_printk("dev %d:%d op %s %sno 0x%x gbno 0x%x owner 0x%llx %s fileoff 0x%llx fsbcount 0x%llx state %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->op, XFS_RMAP_INTENT_STRINGS), + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agno, + __entry->gbno, + __entry->owner, + __print_symbolic(__entry->whichfork, XFS_WHICHFORK_STRINGS), + __entry->l_loff, + __entry->l_len, + __entry->l_state) +); +#define DEFINE_RMAP_DEFERRED_EVENT(name) \ +DEFINE_EVENT(xfs_rmap_deferred_class, name, \ + TP_PROTO(struct xfs_mount *mp, struct xfs_rmap_intent *ri), \ + TP_ARGS(mp, ri)) +DEFINE_RMAP_DEFERRED_EVENT(xfs_rmap_defer); +DEFINE_RMAP_DEFERRED_EVENT(xfs_rmap_deferred); + +DEFINE_RMAPBT_EVENT(xfs_rmap_update); +DEFINE_RMAPBT_EVENT(xfs_rmap_insert); +DEFINE_RMAPBT_EVENT(xfs_rmap_delete); +DEFINE_BTREE_ERROR_EVENT(xfs_rmap_insert_error); +DEFINE_BTREE_ERROR_EVENT(xfs_rmap_delete_error); +DEFINE_BTREE_ERROR_EVENT(xfs_rmap_update_error); + +DEFINE_RMAPBT_EVENT(xfs_rmap_find_left_neighbor_candidate); +DEFINE_RMAPBT_EVENT(xfs_rmap_find_left_neighbor_query); +DEFINE_RMAPBT_EVENT(xfs_rmap_lookup_le_range_candidate); +DEFINE_RMAPBT_EVENT(xfs_rmap_lookup_le_range); +DEFINE_RMAPBT_EVENT(xfs_rmap_lookup_le_range_result); +DEFINE_RMAPBT_EVENT(xfs_rmap_find_right_neighbor_result); +DEFINE_RMAPBT_EVENT(xfs_rmap_find_left_neighbor_result); + +/* deferred bmbt updates */ +TRACE_DEFINE_ENUM(XFS_BMAP_MAP); +TRACE_DEFINE_ENUM(XFS_BMAP_UNMAP); + +DECLARE_EVENT_CLASS(xfs_bmap_deferred_class, + TP_PROTO(struct xfs_bmap_intent *bi), + TP_ARGS(bi), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(enum xfs_group_type, type) + __field(xfs_agnumber_t, agno) + __field(xfs_ino_t, ino) + __field(unsigned long long, gbno) + __field(int, whichfork) + __field(xfs_fileoff_t, l_loff) + __field(xfs_filblks_t, l_len) + __field(xfs_exntst_t, l_state) + __field(int, op) + ), + TP_fast_assign( + struct xfs_inode *ip = bi->bi_owner; + struct xfs_mount *mp = ip->i_mount; + + __entry->dev = mp->m_super->s_dev; + __entry->type = bi->bi_group->xg_type; + __entry->agno = bi->bi_group->xg_gno; + if (bi->bi_group->xg_type == XG_TYPE_RTG && + !xfs_has_rtgroups(mp)) { + /* + * Legacy rt filesystems do not have allocation groups + * ondisk. We emulate this incore with one gigantic + * rtgroup whose size can exceed a 32-bit block number. + * For this tracepoint, we report group 0 and a 64-bit + * group block number. + */ + __entry->gbno = bi->bi_bmap.br_startblock; + } else { + __entry->gbno = xfs_fsb_to_gbno(mp, + bi->bi_bmap.br_startblock, + bi->bi_group->xg_type); + } + __entry->ino = I_INO(ip); + __entry->whichfork = bi->bi_whichfork; + __entry->l_loff = bi->bi_bmap.br_startoff; + __entry->l_len = bi->bi_bmap.br_blockcount; + __entry->l_state = bi->bi_bmap.br_state; + __entry->op = bi->bi_type; + ), + TP_printk("dev %d:%d op %s ino 0x%llx %sno 0x%x gbno 0x%llx %s fileoff 0x%llx fsbcount 0x%llx state %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->op, XFS_BMAP_INTENT_STRINGS), + __entry->ino, + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agno, + __entry->gbno, + __print_symbolic(__entry->whichfork, XFS_WHICHFORK_STRINGS), + __entry->l_loff, + __entry->l_len, + __entry->l_state) +); +#define DEFINE_BMAP_DEFERRED_EVENT(name) \ +DEFINE_EVENT(xfs_bmap_deferred_class, name, \ + TP_PROTO(struct xfs_bmap_intent *bi), \ + TP_ARGS(bi)) +DEFINE_BMAP_DEFERRED_EVENT(xfs_bmap_defer); +DEFINE_BMAP_DEFERRED_EVENT(xfs_bmap_deferred); + +/* per-AG reservation */ +DECLARE_EVENT_CLASS(xfs_ag_resv_class, + TP_PROTO(struct xfs_perag *pag, enum xfs_ag_resv_type resv, + xfs_extlen_t len), + TP_ARGS(pag, resv, len), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(int, resv) + __field(xfs_extlen_t, freeblks) + __field(xfs_extlen_t, flcount) + __field(xfs_extlen_t, reserved) + __field(xfs_extlen_t, asked) + __field(xfs_extlen_t, len) + ), + TP_fast_assign( + struct xfs_ag_resv *r = xfs_perag_resv(pag, resv); + + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->agno = pag_agno(pag); + __entry->resv = resv; + __entry->freeblks = pag->pagf_freeblks; + __entry->flcount = pag->pagf_flcount; + __entry->reserved = r ? r->ar_reserved : 0; + __entry->asked = r ? r->ar_asked : 0; + __entry->len = len; + ), + TP_printk("dev %d:%d agno 0x%x resv %d freeblks %u flcount %u " + "resv %u ask %u len %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->resv, + __entry->freeblks, + __entry->flcount, + __entry->reserved, + __entry->asked, + __entry->len) +) +#define DEFINE_AG_RESV_EVENT(name) \ +DEFINE_EVENT(xfs_ag_resv_class, name, \ + TP_PROTO(struct xfs_perag *pag, enum xfs_ag_resv_type type, \ + xfs_extlen_t len), \ + TP_ARGS(pag, type, len)) + +/* per-AG reservation tracepoints */ +DEFINE_AG_RESV_EVENT(xfs_ag_resv_init); +DEFINE_AG_RESV_EVENT(xfs_ag_resv_free); +DEFINE_AG_RESV_EVENT(xfs_ag_resv_alloc_extent); +DEFINE_AG_RESV_EVENT(xfs_ag_resv_free_extent); +DEFINE_AG_RESV_EVENT(xfs_ag_resv_critical); +DEFINE_AG_RESV_EVENT(xfs_ag_resv_needed); + +TRACE_EVENT(xfs_ag_resv_init_error, + TP_PROTO(const struct xfs_perag *pag, int error, + unsigned long caller_ip), + TP_ARGS(pag, error, caller_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(int, error) + __field(unsigned long, caller_ip) + ), + TP_fast_assign( + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->agno = pag_agno(pag); + __entry->error = error; + __entry->caller_ip = caller_ip; + ), + TP_printk("dev %d:%d agno 0x%x error %d caller %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->error, + (char *)__entry->caller_ip) +); + +/* refcount tracepoint classes */ + +DECLARE_EVENT_CLASS(xfs_refcount_class, + TP_PROTO(struct xfs_btree_cur *cur, xfs_agblock_t gbno, + xfs_extlen_t len), + TP_ARGS(cur, gbno, len), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(enum xfs_group_type, type) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, gbno) + __field(xfs_extlen_t, len) + ), + TP_fast_assign( + __entry->dev = cur->bc_mp->m_super->s_dev; + __entry->type = cur->bc_group->xg_type; + __entry->agno = cur->bc_group->xg_gno; + __entry->gbno = gbno; + __entry->len = len; + ), + TP_printk("dev %d:%d %sno 0x%x gbno 0x%x fsbcount 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agno, + __entry->gbno, + __entry->len) +); +#define DEFINE_REFCOUNT_EVENT(name) \ +DEFINE_EVENT(xfs_refcount_class, name, \ + TP_PROTO(struct xfs_btree_cur *cur, xfs_agblock_t gbno, \ + xfs_extlen_t len), \ + TP_ARGS(cur, gbno, len)) + +TRACE_DEFINE_ENUM(XFS_LOOKUP_EQi); +TRACE_DEFINE_ENUM(XFS_LOOKUP_LEi); +TRACE_DEFINE_ENUM(XFS_LOOKUP_GEi); +TRACE_EVENT(xfs_refcount_lookup, + TP_PROTO(struct xfs_btree_cur *cur, xfs_agblock_t gbno, + xfs_lookup_t dir), + TP_ARGS(cur, gbno, dir), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(enum xfs_group_type, type) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, gbno) + __field(xfs_lookup_t, dir) + ), + TP_fast_assign( + __entry->dev = cur->bc_mp->m_super->s_dev; + __entry->type = cur->bc_group->xg_type; + __entry->agno = cur->bc_group->xg_gno; + __entry->gbno = gbno; + __entry->dir = dir; + ), + TP_printk("dev %d:%d %sno 0x%x gbno 0x%x cmp %s(%d)", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agno, + __entry->gbno, + __print_symbolic(__entry->dir, XFS_AG_BTREE_CMP_FORMAT_STR), + __entry->dir) +) + +/* single-rcext tracepoint class */ +DECLARE_EVENT_CLASS(xfs_refcount_extent_class, + TP_PROTO(struct xfs_btree_cur *cur, struct xfs_refcount_irec *irec), + TP_ARGS(cur, irec), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(enum xfs_group_type, type) + __field(xfs_agnumber_t, agno) + __field(enum xfs_refc_domain, domain) + __field(xfs_agblock_t, startblock) + __field(xfs_extlen_t, blockcount) + __field(xfs_nlink_t, refcount) + ), + TP_fast_assign( + __entry->dev = cur->bc_mp->m_super->s_dev; + __entry->type = cur->bc_group->xg_type; + __entry->agno = cur->bc_group->xg_gno; + __entry->domain = irec->rc_domain; + __entry->startblock = irec->rc_startblock; + __entry->blockcount = irec->rc_blockcount; + __entry->refcount = irec->rc_refcount; + ), + TP_printk("dev %d:%d %sno 0x%x dom %s gbno 0x%x fsbcount 0x%x refcount %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agno, + __print_symbolic(__entry->domain, XFS_REFC_DOMAIN_STRINGS), + __entry->startblock, + __entry->blockcount, + __entry->refcount) +) + +#define DEFINE_REFCOUNT_EXTENT_EVENT(name) \ +DEFINE_EVENT(xfs_refcount_extent_class, name, \ + TP_PROTO(struct xfs_btree_cur *cur, struct xfs_refcount_irec *irec), \ + TP_ARGS(cur, irec)) + +/* single-rcext and an agbno tracepoint class */ +DECLARE_EVENT_CLASS(xfs_refcount_extent_at_class, + TP_PROTO(struct xfs_btree_cur *cur, struct xfs_refcount_irec *irec, + xfs_agblock_t gbno), + TP_ARGS(cur, irec, gbno), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(enum xfs_group_type, type) + __field(xfs_agnumber_t, agno) + __field(enum xfs_refc_domain, domain) + __field(xfs_agblock_t, startblock) + __field(xfs_extlen_t, blockcount) + __field(xfs_nlink_t, refcount) + __field(xfs_agblock_t, gbno) + ), + TP_fast_assign( + __entry->dev = cur->bc_mp->m_super->s_dev; + __entry->type = cur->bc_group->xg_type; + __entry->agno = cur->bc_group->xg_gno; + __entry->domain = irec->rc_domain; + __entry->startblock = irec->rc_startblock; + __entry->blockcount = irec->rc_blockcount; + __entry->refcount = irec->rc_refcount; + __entry->gbno = gbno; + ), + TP_printk("dev %d:%d %sno 0x%x dom %s gbno 0x%x fsbcount 0x%x refcount %u @ gbno 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agno, + __print_symbolic(__entry->domain, XFS_REFC_DOMAIN_STRINGS), + __entry->startblock, + __entry->blockcount, + __entry->refcount, + __entry->gbno) +) + +#define DEFINE_REFCOUNT_EXTENT_AT_EVENT(name) \ +DEFINE_EVENT(xfs_refcount_extent_at_class, name, \ + TP_PROTO(struct xfs_btree_cur *cur, struct xfs_refcount_irec *irec, \ + xfs_agblock_t gbno), \ + TP_ARGS(cur, irec, gbno)) + +/* double-rcext tracepoint class */ +DECLARE_EVENT_CLASS(xfs_refcount_double_extent_class, + TP_PROTO(struct xfs_btree_cur *cur, struct xfs_refcount_irec *i1, + struct xfs_refcount_irec *i2), + TP_ARGS(cur, i1, i2), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(enum xfs_group_type, type) + __field(xfs_agnumber_t, agno) + __field(enum xfs_refc_domain, i1_domain) + __field(xfs_agblock_t, i1_startblock) + __field(xfs_extlen_t, i1_blockcount) + __field(xfs_nlink_t, i1_refcount) + __field(enum xfs_refc_domain, i2_domain) + __field(xfs_agblock_t, i2_startblock) + __field(xfs_extlen_t, i2_blockcount) + __field(xfs_nlink_t, i2_refcount) + ), + TP_fast_assign( + __entry->dev = cur->bc_mp->m_super->s_dev; + __entry->type = cur->bc_group->xg_type; + __entry->agno = cur->bc_group->xg_gno; + __entry->i1_domain = i1->rc_domain; + __entry->i1_startblock = i1->rc_startblock; + __entry->i1_blockcount = i1->rc_blockcount; + __entry->i1_refcount = i1->rc_refcount; + __entry->i2_domain = i2->rc_domain; + __entry->i2_startblock = i2->rc_startblock; + __entry->i2_blockcount = i2->rc_blockcount; + __entry->i2_refcount = i2->rc_refcount; + ), + TP_printk("dev %d:%d %sno 0x%x dom %s gbno 0x%x fsbcount 0x%x refcount %u -- " + "dom %s gbno 0x%x fsbcount 0x%x refcount %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agno, + __print_symbolic(__entry->i1_domain, XFS_REFC_DOMAIN_STRINGS), + __entry->i1_startblock, + __entry->i1_blockcount, + __entry->i1_refcount, + __print_symbolic(__entry->i2_domain, XFS_REFC_DOMAIN_STRINGS), + __entry->i2_startblock, + __entry->i2_blockcount, + __entry->i2_refcount) +) + +#define DEFINE_REFCOUNT_DOUBLE_EXTENT_EVENT(name) \ +DEFINE_EVENT(xfs_refcount_double_extent_class, name, \ + TP_PROTO(struct xfs_btree_cur *cur, struct xfs_refcount_irec *i1, \ + struct xfs_refcount_irec *i2), \ + TP_ARGS(cur, i1, i2)) + +/* double-rcext and an agbno tracepoint class */ +DECLARE_EVENT_CLASS(xfs_refcount_double_extent_at_class, + TP_PROTO(struct xfs_btree_cur *cur, struct xfs_refcount_irec *i1, + struct xfs_refcount_irec *i2, xfs_agblock_t gbno), + TP_ARGS(cur, i1, i2, gbno), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(enum xfs_group_type, type) + __field(xfs_agnumber_t, agno) + __field(enum xfs_refc_domain, i1_domain) + __field(xfs_agblock_t, i1_startblock) + __field(xfs_extlen_t, i1_blockcount) + __field(xfs_nlink_t, i1_refcount) + __field(enum xfs_refc_domain, i2_domain) + __field(xfs_agblock_t, i2_startblock) + __field(xfs_extlen_t, i2_blockcount) + __field(xfs_nlink_t, i2_refcount) + __field(xfs_agblock_t, gbno) + ), + TP_fast_assign( + __entry->dev = cur->bc_mp->m_super->s_dev; + __entry->type = cur->bc_group->xg_type; + __entry->agno = cur->bc_group->xg_gno; + __entry->i1_domain = i1->rc_domain; + __entry->i1_startblock = i1->rc_startblock; + __entry->i1_blockcount = i1->rc_blockcount; + __entry->i1_refcount = i1->rc_refcount; + __entry->i2_domain = i2->rc_domain; + __entry->i2_startblock = i2->rc_startblock; + __entry->i2_blockcount = i2->rc_blockcount; + __entry->i2_refcount = i2->rc_refcount; + __entry->gbno = gbno; + ), + TP_printk("dev %d:%d %sno 0x%x dom %s gbno 0x%x fsbcount 0x%x refcount %u -- " + "dom %s gbno 0x%x fsbcount 0x%x refcount %u @ gbno 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agno, + __print_symbolic(__entry->i1_domain, XFS_REFC_DOMAIN_STRINGS), + __entry->i1_startblock, + __entry->i1_blockcount, + __entry->i1_refcount, + __print_symbolic(__entry->i2_domain, XFS_REFC_DOMAIN_STRINGS), + __entry->i2_startblock, + __entry->i2_blockcount, + __entry->i2_refcount, + __entry->gbno) +) + +#define DEFINE_REFCOUNT_DOUBLE_EXTENT_AT_EVENT(name) \ +DEFINE_EVENT(xfs_refcount_double_extent_at_class, name, \ + TP_PROTO(struct xfs_btree_cur *cur, struct xfs_refcount_irec *i1, \ + struct xfs_refcount_irec *i2, xfs_agblock_t gbno), \ + TP_ARGS(cur, i1, i2, gbno)) + +/* triple-rcext tracepoint class */ +DECLARE_EVENT_CLASS(xfs_refcount_triple_extent_class, + TP_PROTO(struct xfs_btree_cur *cur, struct xfs_refcount_irec *i1, + struct xfs_refcount_irec *i2, struct xfs_refcount_irec *i3), + TP_ARGS(cur, i1, i2, i3), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(enum xfs_group_type, type) + __field(xfs_agnumber_t, agno) + __field(enum xfs_refc_domain, i1_domain) + __field(xfs_agblock_t, i1_startblock) + __field(xfs_extlen_t, i1_blockcount) + __field(xfs_nlink_t, i1_refcount) + __field(enum xfs_refc_domain, i2_domain) + __field(xfs_agblock_t, i2_startblock) + __field(xfs_extlen_t, i2_blockcount) + __field(xfs_nlink_t, i2_refcount) + __field(enum xfs_refc_domain, i3_domain) + __field(xfs_agblock_t, i3_startblock) + __field(xfs_extlen_t, i3_blockcount) + __field(xfs_nlink_t, i3_refcount) + ), + TP_fast_assign( + __entry->dev = cur->bc_mp->m_super->s_dev; + __entry->type = cur->bc_group->xg_type; + __entry->agno = cur->bc_group->xg_gno; + __entry->i1_domain = i1->rc_domain; + __entry->i1_startblock = i1->rc_startblock; + __entry->i1_blockcount = i1->rc_blockcount; + __entry->i1_refcount = i1->rc_refcount; + __entry->i2_domain = i2->rc_domain; + __entry->i2_startblock = i2->rc_startblock; + __entry->i2_blockcount = i2->rc_blockcount; + __entry->i2_refcount = i2->rc_refcount; + __entry->i3_domain = i3->rc_domain; + __entry->i3_startblock = i3->rc_startblock; + __entry->i3_blockcount = i3->rc_blockcount; + __entry->i3_refcount = i3->rc_refcount; + ), + TP_printk("dev %d:%d %sno 0x%x dom %s gbno 0x%x fsbcount 0x%x refcount %u -- " + "dom %s gbno 0x%x fsbcount 0x%x refcount %u -- " + "dom %s gbno 0x%x fsbcount 0x%x refcount %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agno, + __print_symbolic(__entry->i1_domain, XFS_REFC_DOMAIN_STRINGS), + __entry->i1_startblock, + __entry->i1_blockcount, + __entry->i1_refcount, + __print_symbolic(__entry->i2_domain, XFS_REFC_DOMAIN_STRINGS), + __entry->i2_startblock, + __entry->i2_blockcount, + __entry->i2_refcount, + __print_symbolic(__entry->i3_domain, XFS_REFC_DOMAIN_STRINGS), + __entry->i3_startblock, + __entry->i3_blockcount, + __entry->i3_refcount) +); + +#define DEFINE_REFCOUNT_TRIPLE_EXTENT_EVENT(name) \ +DEFINE_EVENT(xfs_refcount_triple_extent_class, name, \ + TP_PROTO(struct xfs_btree_cur *cur, struct xfs_refcount_irec *i1, \ + struct xfs_refcount_irec *i2, struct xfs_refcount_irec *i3), \ + TP_ARGS(cur, i1, i2, i3)) + +/* refcount btree tracepoints */ +DEFINE_REFCOUNT_EXTENT_EVENT(xfs_refcount_get); +DEFINE_REFCOUNT_EXTENT_EVENT(xfs_refcount_update); +DEFINE_REFCOUNT_EXTENT_EVENT(xfs_refcount_insert); +DEFINE_REFCOUNT_EXTENT_EVENT(xfs_refcount_delete); +DEFINE_BTREE_ERROR_EVENT(xfs_refcount_insert_error); +DEFINE_BTREE_ERROR_EVENT(xfs_refcount_delete_error); +DEFINE_BTREE_ERROR_EVENT(xfs_refcount_update_error); + +/* refcount adjustment tracepoints */ +DEFINE_REFCOUNT_EVENT(xfs_refcount_increase); +DEFINE_REFCOUNT_EVENT(xfs_refcount_decrease); +DEFINE_REFCOUNT_EVENT(xfs_refcount_cow_increase); +DEFINE_REFCOUNT_EVENT(xfs_refcount_cow_decrease); +DEFINE_REFCOUNT_TRIPLE_EXTENT_EVENT(xfs_refcount_merge_center_extents); +DEFINE_REFCOUNT_EXTENT_EVENT(xfs_refcount_modify_extent); +DEFINE_REFCOUNT_EXTENT_AT_EVENT(xfs_refcount_split_extent); +DEFINE_REFCOUNT_DOUBLE_EXTENT_EVENT(xfs_refcount_merge_left_extent); +DEFINE_REFCOUNT_DOUBLE_EXTENT_EVENT(xfs_refcount_merge_right_extent); +DEFINE_REFCOUNT_DOUBLE_EXTENT_AT_EVENT(xfs_refcount_find_left_extent); +DEFINE_REFCOUNT_DOUBLE_EXTENT_AT_EVENT(xfs_refcount_find_right_extent); +DEFINE_BTREE_ERROR_EVENT(xfs_refcount_adjust_error); +DEFINE_BTREE_ERROR_EVENT(xfs_refcount_adjust_cow_error); +DEFINE_BTREE_ERROR_EVENT(xfs_refcount_merge_center_extents_error); +DEFINE_BTREE_ERROR_EVENT(xfs_refcount_modify_extent_error); +DEFINE_BTREE_ERROR_EVENT(xfs_refcount_split_extent_error); +DEFINE_BTREE_ERROR_EVENT(xfs_refcount_merge_left_extent_error); +DEFINE_BTREE_ERROR_EVENT(xfs_refcount_merge_right_extent_error); +DEFINE_BTREE_ERROR_EVENT(xfs_refcount_find_left_extent_error); +DEFINE_BTREE_ERROR_EVENT(xfs_refcount_find_right_extent_error); + +/* reflink helpers */ +DEFINE_REFCOUNT_EVENT(xfs_refcount_find_shared); +DEFINE_REFCOUNT_EVENT(xfs_refcount_find_shared_result); +DEFINE_BTREE_ERROR_EVENT(xfs_refcount_find_shared_error); + +TRACE_DEFINE_ENUM(XFS_REFCOUNT_INCREASE); +TRACE_DEFINE_ENUM(XFS_REFCOUNT_DECREASE); +TRACE_DEFINE_ENUM(XFS_REFCOUNT_ALLOC_COW); +TRACE_DEFINE_ENUM(XFS_REFCOUNT_FREE_COW); + +DECLARE_EVENT_CLASS(xfs_refcount_deferred_class, + TP_PROTO(struct xfs_mount *mp, struct xfs_refcount_intent *refc), + TP_ARGS(mp, refc), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(enum xfs_group_type, type) + __field(xfs_agnumber_t, agno) + __field(int, op) + __field(xfs_agblock_t, gbno) + __field(xfs_extlen_t, len) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->type = refc->ri_group->xg_type; + __entry->agno = refc->ri_group->xg_gno; + __entry->op = refc->ri_type; + __entry->gbno = xfs_fsb_to_gbno(mp, refc->ri_startblock, + refc->ri_group->xg_type); + __entry->len = refc->ri_blockcount; + ), + TP_printk("dev %d:%d op %s %sno 0x%x gbno 0x%x fsbcount 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->op, XFS_REFCOUNT_INTENT_STRINGS), + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->agno, + __entry->gbno, + __entry->len) +); +#define DEFINE_REFCOUNT_DEFERRED_EVENT(name) \ +DEFINE_EVENT(xfs_refcount_deferred_class, name, \ + TP_PROTO(struct xfs_mount *mp, struct xfs_refcount_intent *refc), \ + TP_ARGS(mp, refc)) +DEFINE_REFCOUNT_DEFERRED_EVENT(xfs_refcount_defer); +DEFINE_REFCOUNT_DEFERRED_EVENT(xfs_refcount_deferred); +DEFINE_REFCOUNT_DEFERRED_EVENT(xfs_refcount_finish_one_leftover); + +/* simple inode-based error/%ip tracepoint class */ +DECLARE_EVENT_CLASS(xfs_inode_error_class, + TP_PROTO(struct xfs_inode *ip, int error, unsigned long caller_ip), + TP_ARGS(ip, error, caller_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(int, error) + __field(unsigned long, caller_ip) + ), + TP_fast_assign( + __entry->dev = VFS_I(ip)->i_sb->s_dev; + __entry->ino = I_INO(ip); + __entry->error = error; + __entry->caller_ip = caller_ip; + ), + TP_printk("dev %d:%d ino 0x%llx error %d caller %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->error, + (char *)__entry->caller_ip) +); + +#define DEFINE_INODE_ERROR_EVENT(name) \ +DEFINE_EVENT(xfs_inode_error_class, name, \ + TP_PROTO(struct xfs_inode *ip, int error, \ + unsigned long caller_ip), \ + TP_ARGS(ip, error, caller_ip)) + +/* reflink tracepoint classes */ + +/* two-file io tracepoint class */ +DECLARE_EVENT_CLASS(xfs_double_io_class, + TP_PROTO(struct xfs_inode *src, xfs_off_t soffset, xfs_off_t len, + struct xfs_inode *dest, xfs_off_t doffset), + TP_ARGS(src, soffset, len, dest, doffset), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, src_ino) + __field(loff_t, src_isize) + __field(loff_t, src_disize) + __field(loff_t, src_offset) + __field(long long, len) + __field(xfs_ino_t, dest_ino) + __field(loff_t, dest_isize) + __field(loff_t, dest_disize) + __field(loff_t, dest_offset) + ), + TP_fast_assign( + __entry->dev = VFS_I(src)->i_sb->s_dev; + __entry->src_ino = I_INO(src); + __entry->src_isize = VFS_I(src)->i_size; + __entry->src_disize = src->i_disk_size; + __entry->src_offset = soffset; + __entry->len = len; + __entry->dest_ino = I_INO(dest); + __entry->dest_isize = VFS_I(dest)->i_size; + __entry->dest_disize = dest->i_disk_size; + __entry->dest_offset = doffset; + ), + TP_printk("dev %d:%d bytecount 0x%llx " + "ino 0x%llx isize 0x%llx disize 0x%llx pos 0x%llx -> " + "ino 0x%llx isize 0x%llx disize 0x%llx pos 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->len, + __entry->src_ino, + __entry->src_isize, + __entry->src_disize, + __entry->src_offset, + __entry->dest_ino, + __entry->dest_isize, + __entry->dest_disize, + __entry->dest_offset) +) + +#define DEFINE_DOUBLE_IO_EVENT(name) \ +DEFINE_EVENT(xfs_double_io_class, name, \ + TP_PROTO(struct xfs_inode *src, xfs_off_t soffset, xfs_off_t len, \ + struct xfs_inode *dest, xfs_off_t doffset), \ + TP_ARGS(src, soffset, len, dest, doffset)) + +/* inode/irec events */ +DECLARE_EVENT_CLASS(xfs_inode_irec_class, + TP_PROTO(struct xfs_inode *ip, struct xfs_bmbt_irec *irec), + TP_ARGS(ip, irec), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_fileoff_t, lblk) + __field(xfs_extlen_t, len) + __field(xfs_fsblock_t, pblk) + __field(int, state) + ), + TP_fast_assign( + __entry->dev = VFS_I(ip)->i_sb->s_dev; + __entry->ino = I_INO(ip); + __entry->lblk = irec->br_startoff; + __entry->len = irec->br_blockcount; + __entry->pblk = irec->br_startblock; + __entry->state = irec->br_state; + ), + TP_printk("dev %d:%d ino 0x%llx fileoff 0x%llx fsbcount 0x%x startblock 0x%llx st %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->lblk, + __entry->len, + __entry->pblk, + __entry->state) +); +#define DEFINE_INODE_IREC_EVENT(name) \ +DEFINE_EVENT(xfs_inode_irec_class, name, \ + TP_PROTO(struct xfs_inode *ip, struct xfs_bmbt_irec *irec), \ + TP_ARGS(ip, irec)) + +/* inode iomap invalidation events */ +DECLARE_EVENT_CLASS(xfs_wb_invalid_class, + TP_PROTO(struct xfs_inode *ip, const struct iomap *iomap, unsigned int wpcseq, int whichfork), + TP_ARGS(ip, iomap, wpcseq, whichfork), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(u64, addr) + __field(loff_t, pos) + __field(u64, len) + __field(u16, type) + __field(u16, flags) + __field(u32, wpcseq) + __field(u32, forkseq) + ), + TP_fast_assign( + __entry->dev = VFS_I(ip)->i_sb->s_dev; + __entry->ino = I_INO(ip); + __entry->addr = iomap->addr; + __entry->pos = iomap->offset; + __entry->len = iomap->length; + __entry->type = iomap->type; + __entry->flags = iomap->flags; + __entry->wpcseq = wpcseq; + __entry->forkseq = READ_ONCE(xfs_ifork_ptr(ip, whichfork)->if_seq); + ), + TP_printk("dev %d:%d ino 0x%llx pos 0x%llx addr 0x%llx bytecount 0x%llx type 0x%x flags 0x%x wpcseq 0x%x forkseq 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->pos, + __entry->addr, + __entry->len, + __entry->type, + __entry->flags, + __entry->wpcseq, + __entry->forkseq) +); +#define DEFINE_WB_INVALID_EVENT(name) \ +DEFINE_EVENT(xfs_wb_invalid_class, name, \ + TP_PROTO(struct xfs_inode *ip, const struct iomap *iomap, unsigned int wpcseq, int whichfork), \ + TP_ARGS(ip, iomap, wpcseq, whichfork)) +DEFINE_WB_INVALID_EVENT(xfs_wb_cow_iomap_invalid); +DEFINE_WB_INVALID_EVENT(xfs_wb_data_iomap_invalid); + +DECLARE_EVENT_CLASS(xfs_iomap_invalid_class, + TP_PROTO(struct xfs_inode *ip, const struct iomap *iomap), + TP_ARGS(ip, iomap), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(u64, addr) + __field(loff_t, pos) + __field(u64, len) + __field(u64, validity_cookie) + __field(u64, inodeseq) + __field(u16, type) + __field(u16, flags) + ), + TP_fast_assign( + __entry->dev = VFS_I(ip)->i_sb->s_dev; + __entry->ino = I_INO(ip); + __entry->addr = iomap->addr; + __entry->pos = iomap->offset; + __entry->len = iomap->length; + __entry->validity_cookie = iomap->validity_cookie; + __entry->type = iomap->type; + __entry->flags = iomap->flags; + __entry->inodeseq = xfs_iomap_inode_sequence(ip, iomap->flags); + ), + TP_printk("dev %d:%d ino 0x%llx pos 0x%llx addr 0x%llx bytecount 0x%llx type 0x%x flags 0x%x validity_cookie 0x%llx inodeseq 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->pos, + __entry->addr, + __entry->len, + __entry->type, + __entry->flags, + __entry->validity_cookie, + __entry->inodeseq) +); +#define DEFINE_IOMAP_INVALID_EVENT(name) \ +DEFINE_EVENT(xfs_iomap_invalid_class, name, \ + TP_PROTO(struct xfs_inode *ip, const struct iomap *iomap), \ + TP_ARGS(ip, iomap)) +DEFINE_IOMAP_INVALID_EVENT(xfs_iomap_invalid); + +/* refcount/reflink tracepoint definitions */ + +/* reflink tracepoints */ +DEFINE_INODE_EVENT(xfs_reflink_set_inode_flag); +DEFINE_INODE_EVENT(xfs_reflink_unset_inode_flag); +DEFINE_ITRUNC_EVENT(xfs_reflink_update_inode_size); +TRACE_EVENT(xfs_reflink_remap_blocks, + TP_PROTO(struct xfs_inode *src, xfs_fileoff_t soffset, + xfs_filblks_t len, struct xfs_inode *dest, + xfs_fileoff_t doffset), + TP_ARGS(src, soffset, len, dest, doffset), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, src_ino) + __field(xfs_fileoff_t, src_lblk) + __field(xfs_filblks_t, len) + __field(xfs_ino_t, dest_ino) + __field(xfs_fileoff_t, dest_lblk) + ), + TP_fast_assign( + __entry->dev = VFS_I(src)->i_sb->s_dev; + __entry->src_ino = I_INO(src); + __entry->src_lblk = soffset; + __entry->len = len; + __entry->dest_ino = I_INO(dest); + __entry->dest_lblk = doffset; + ), + TP_printk("dev %d:%d fsbcount 0x%llx " + "ino 0x%llx fileoff 0x%llx -> ino 0x%llx fileoff 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->len, + __entry->src_ino, + __entry->src_lblk, + __entry->dest_ino, + __entry->dest_lblk) +); +DEFINE_DOUBLE_IO_EVENT(xfs_reflink_remap_range); +DEFINE_INODE_ERROR_EVENT(xfs_reflink_remap_range_error); +DEFINE_INODE_ERROR_EVENT(xfs_reflink_set_inode_flag_error); +DEFINE_INODE_ERROR_EVENT(xfs_reflink_update_inode_size_error); +DEFINE_INODE_ERROR_EVENT(xfs_reflink_remap_blocks_error); +DEFINE_INODE_ERROR_EVENT(xfs_reflink_remap_extent_error); +DEFINE_INODE_IREC_EVENT(xfs_reflink_remap_extent_src); +DEFINE_INODE_IREC_EVENT(xfs_reflink_remap_extent_dest); + +/* unshare tracepoints */ +DEFINE_SIMPLE_IO_EVENT(xfs_reflink_unshare); +DEFINE_INODE_ERROR_EVENT(xfs_reflink_unshare_error); + +/* copy on write */ +DEFINE_INODE_IREC_EVENT(xfs_reflink_trim_around_shared); +DEFINE_INODE_IREC_EVENT(xfs_reflink_cow_found); +DEFINE_INODE_IREC_EVENT(xfs_reflink_convert_cow); + +DEFINE_SIMPLE_IO_EVENT(xfs_reflink_cancel_cow_range); +DEFINE_SIMPLE_IO_EVENT(xfs_reflink_end_cow); +DEFINE_INODE_IREC_EVENT(xfs_reflink_cow_remap_from); +DEFINE_INODE_IREC_EVENT(xfs_reflink_cow_remap_to); +DEFINE_INODE_IREC_EVENT(xfs_reflink_cow_remap_skip); + +DEFINE_INODE_ERROR_EVENT(xfs_reflink_cancel_cow_range_error); +DEFINE_INODE_ERROR_EVENT(xfs_reflink_end_cow_error); + + +DEFINE_INODE_IREC_EVENT(xfs_reflink_cancel_cow); + +/* rmap swapext tracepoints */ +DEFINE_INODE_IREC_EVENT(xfs_swap_extent_rmap_remap); +DEFINE_INODE_IREC_EVENT(xfs_swap_extent_rmap_remap_piece); +DEFINE_INODE_ERROR_EVENT(xfs_swap_extent_rmap_error); + +/* fsmap traces */ +TRACE_EVENT(xfs_fsmap_mapping, + TP_PROTO(struct xfs_mount *mp, u32 keydev, xfs_agnumber_t agno, + const struct xfs_fsmap_irec *frec), + TP_ARGS(mp, keydev, agno, frec), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(dev_t, keydev) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, agbno) + __field(xfs_daddr_t, start_daddr) + __field(xfs_daddr_t, len_daddr) + __field(uint64_t, owner) + __field(uint64_t, offset) + __field(unsigned int, flags) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->keydev = new_decode_dev(keydev); + __entry->agno = agno; + __entry->agbno = frec->rec_key; + __entry->start_daddr = frec->start_daddr; + __entry->len_daddr = frec->len_daddr; + __entry->owner = frec->owner; + __entry->offset = frec->offset; + __entry->flags = frec->rm_flags; + ), + TP_printk("dev %d:%d keydev %d:%d agno 0x%x gbno 0x%x start_daddr 0x%llx len_daddr 0x%llx owner 0x%llx fileoff 0x%llx flags 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + MAJOR(__entry->keydev), MINOR(__entry->keydev), + __entry->agno, + __entry->agbno, + __entry->start_daddr, + __entry->len_daddr, + __entry->owner, + __entry->offset, + __entry->flags) +); + +DECLARE_EVENT_CLASS(xfs_fsmap_group_key_class, + TP_PROTO(struct xfs_mount *mp, u32 keydev, xfs_agnumber_t agno, + const struct xfs_rmap_irec *rmap), + TP_ARGS(mp, keydev, agno, rmap), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(dev_t, keydev) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, agbno) + __field(uint64_t, owner) + __field(uint64_t, offset) + __field(unsigned int, flags) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->keydev = new_decode_dev(keydev); + __entry->agno = agno; + __entry->agbno = rmap->rm_startblock; + __entry->owner = rmap->rm_owner; + __entry->offset = rmap->rm_offset; + __entry->flags = rmap->rm_flags; + ), + TP_printk("dev %d:%d keydev %d:%d agno 0x%x startblock 0x%x owner 0x%llx fileoff 0x%llx flags 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + MAJOR(__entry->keydev), MINOR(__entry->keydev), + __entry->agno, + __entry->agbno, + __entry->owner, + __entry->offset, + __entry->flags) +) +#define DEFINE_FSMAP_GROUP_KEY_EVENT(name) \ +DEFINE_EVENT(xfs_fsmap_group_key_class, name, \ + TP_PROTO(struct xfs_mount *mp, u32 keydev, xfs_agnumber_t agno, \ + const struct xfs_rmap_irec *rmap), \ + TP_ARGS(mp, keydev, agno, rmap)) +DEFINE_FSMAP_GROUP_KEY_EVENT(xfs_fsmap_low_group_key); +DEFINE_FSMAP_GROUP_KEY_EVENT(xfs_fsmap_high_group_key); + +DECLARE_EVENT_CLASS(xfs_fsmap_linear_key_class, + TP_PROTO(struct xfs_mount *mp, u32 keydev, xfs_fsblock_t bno), + TP_ARGS(mp, keydev, bno), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(dev_t, keydev) + __field(xfs_fsblock_t, bno) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->keydev = new_decode_dev(keydev); + __entry->bno = bno; + ), + TP_printk("dev %d:%d keydev %d:%d bno 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + MAJOR(__entry->keydev), MINOR(__entry->keydev), + __entry->bno) +) +#define DEFINE_FSMAP_LINEAR_KEY_EVENT(name) \ +DEFINE_EVENT(xfs_fsmap_linear_key_class, name, \ + TP_PROTO(struct xfs_mount *mp, u32 keydev, uint64_t bno), \ + TP_ARGS(mp, keydev, bno)) +DEFINE_FSMAP_LINEAR_KEY_EVENT(xfs_fsmap_low_linear_key); +DEFINE_FSMAP_LINEAR_KEY_EVENT(xfs_fsmap_high_linear_key); + +DECLARE_EVENT_CLASS(xfs_getfsmap_class, + TP_PROTO(struct xfs_mount *mp, struct xfs_fsmap *fsmap), + TP_ARGS(mp, fsmap), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(dev_t, keydev) + __field(xfs_daddr_t, block) + __field(xfs_daddr_t, len) + __field(uint64_t, owner) + __field(uint64_t, offset) + __field(uint64_t, flags) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->keydev = new_decode_dev(fsmap->fmr_device); + __entry->block = fsmap->fmr_physical; + __entry->len = fsmap->fmr_length; + __entry->owner = fsmap->fmr_owner; + __entry->offset = fsmap->fmr_offset; + __entry->flags = fsmap->fmr_flags; + ), + TP_printk("dev %d:%d keydev %d:%d daddr 0x%llx bbcount 0x%llx owner 0x%llx fileoff_daddr 0x%llx flags 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + MAJOR(__entry->keydev), MINOR(__entry->keydev), + __entry->block, + __entry->len, + __entry->owner, + __entry->offset, + __entry->flags) +) +#define DEFINE_GETFSMAP_EVENT(name) \ +DEFINE_EVENT(xfs_getfsmap_class, name, \ + TP_PROTO(struct xfs_mount *mp, struct xfs_fsmap *fsmap), \ + TP_ARGS(mp, fsmap)) +DEFINE_GETFSMAP_EVENT(xfs_getfsmap_low_key); +DEFINE_GETFSMAP_EVENT(xfs_getfsmap_high_key); +DEFINE_GETFSMAP_EVENT(xfs_getfsmap_mapping); + +DECLARE_EVENT_CLASS(xfs_trans_resv_class, + TP_PROTO(struct xfs_mount *mp, unsigned int type, + struct xfs_trans_res *res), + TP_ARGS(mp, type, res), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(int, type) + __field(uint, logres) + __field(int, logcount) + __field(int, logflags) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->type = type; + __entry->logres = res->tr_logres; + __entry->logcount = res->tr_logcount; + __entry->logflags = res->tr_logflags; + ), + TP_printk("dev %d:%d type %d logres %u logcount %d flags 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->type, + __entry->logres, + __entry->logcount, + __entry->logflags) +) + +#define DEFINE_TRANS_RESV_EVENT(name) \ +DEFINE_EVENT(xfs_trans_resv_class, name, \ + TP_PROTO(struct xfs_mount *mp, unsigned int type, \ + struct xfs_trans_res *res), \ + TP_ARGS(mp, type, res)) +DEFINE_TRANS_RESV_EVENT(xfs_trans_resv_calc); +DEFINE_TRANS_RESV_EVENT(xfs_trans_resv_calc_minlogsize); + +TRACE_EVENT(xfs_log_get_max_trans_res, + TP_PROTO(struct xfs_mount *mp, const struct xfs_trans_res *res), + TP_ARGS(mp, res), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(uint, logres) + __field(int, logcount) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->logres = res->tr_logres; + __entry->logcount = res->tr_logcount; + ), + TP_printk("dev %d:%d logres %u logcount %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->logres, + __entry->logcount) +); + +DECLARE_EVENT_CLASS(xfs_trans_class, + TP_PROTO(struct xfs_trans *tp, unsigned long caller_ip), + TP_ARGS(tp, caller_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(uint32_t, tid) + __field(uint32_t, flags) + __field(unsigned long, caller_ip) + ), + TP_fast_assign( + __entry->dev = tp->t_mountp->m_super->s_dev; + __entry->tid = 0; + if (tp->t_ticket) + __entry->tid = tp->t_ticket->t_tid; + __entry->flags = tp->t_flags; + __entry->caller_ip = caller_ip; + ), + TP_printk("dev %d:%d trans %x flags 0x%x caller %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->tid, + __entry->flags, + (char *)__entry->caller_ip) +) + +#define DEFINE_TRANS_EVENT(name) \ +DEFINE_EVENT(xfs_trans_class, name, \ + TP_PROTO(struct xfs_trans *tp, unsigned long caller_ip), \ + TP_ARGS(tp, caller_ip)) +DEFINE_TRANS_EVENT(xfs_trans_alloc); +DEFINE_TRANS_EVENT(xfs_trans_cancel); +DEFINE_TRANS_EVENT(xfs_trans_commit); +DEFINE_TRANS_EVENT(xfs_trans_dup); +DEFINE_TRANS_EVENT(xfs_trans_free); +DEFINE_TRANS_EVENT(xfs_trans_roll); +DEFINE_TRANS_EVENT(xfs_trans_add_item); +DEFINE_TRANS_EVENT(xfs_trans_commit_items); +DEFINE_TRANS_EVENT(xfs_trans_free_items); + +TRACE_EVENT(xfs_iunlink_update_bucket, + TP_PROTO(const struct xfs_perag *pag, unsigned int bucket, + xfs_agino_t old_ptr, xfs_agino_t new_ptr), + TP_ARGS(pag, bucket, old_ptr, new_ptr), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(unsigned int, bucket) + __field(xfs_agino_t, old_ptr) + __field(xfs_agino_t, new_ptr) + ), + TP_fast_assign( + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->agno = pag_agno(pag); + __entry->bucket = bucket; + __entry->old_ptr = old_ptr; + __entry->new_ptr = new_ptr; + ), + TP_printk("dev %d:%d agno 0x%x bucket %u old 0x%x new 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->bucket, + __entry->old_ptr, + __entry->new_ptr) +); + +TRACE_EVENT(xfs_iunlink_update_dinode, + TP_PROTO(const struct xfs_iunlink_item *iup, xfs_agino_t old_ptr), + TP_ARGS(iup, old_ptr), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agino_t, agino) + __field(xfs_agino_t, old_ptr) + __field(xfs_agino_t, new_ptr) + ), + TP_fast_assign( + __entry->dev = pag_mount(iup->pag)->m_super->s_dev; + __entry->agno = pag_agno(iup->pag); + __entry->agino = XFS_INODE_TO_AGINO(iup->ip); + __entry->old_ptr = old_ptr; + __entry->new_ptr = iup->next_agino; + ), + TP_printk("dev %d:%d agno 0x%x agino 0x%x old 0x%x new 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->agino, + __entry->old_ptr, + __entry->new_ptr) +); + +TRACE_EVENT(xfs_iunlink_reload_next, + TP_PROTO(struct xfs_inode *ip), + TP_ARGS(ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agino_t, agino) + __field(xfs_agino_t, prev_agino) + __field(xfs_agino_t, next_agino) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->agno = XFS_INODE_TO_AGNO(ip); + __entry->agino = XFS_INODE_TO_AGINO(ip); + __entry->prev_agino = ip->i_prev_unlinked; + __entry->next_agino = ip->i_next_unlinked; + ), + TP_printk("dev %d:%d agno 0x%x agino 0x%x prev_unlinked 0x%x next_unlinked 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->agino, + __entry->prev_agino, + __entry->next_agino) +); + +TRACE_EVENT(xfs_inode_reload_unlinked_bucket, + TP_PROTO(struct xfs_inode *ip), + TP_ARGS(ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agino_t, agino) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->agno = XFS_INODE_TO_AGNO(ip); + __entry->agino = XFS_INODE_TO_AGINO(ip); + ), + TP_printk("dev %d:%d agno 0x%x agino 0x%x bucket %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, + __entry->agino, + __entry->agino % XFS_AGI_UNLINKED_BUCKETS) +); + +DECLARE_EVENT_CLASS(xfs_ag_inode_class, + TP_PROTO(struct xfs_inode *ip), + TP_ARGS(ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agino_t, agino) + ), + TP_fast_assign( + __entry->dev = VFS_I(ip)->i_sb->s_dev; + __entry->agno = XFS_INODE_TO_AGNO(ip); + __entry->agino = XFS_INODE_TO_AGINO(ip); + ), + TP_printk("dev %d:%d agno 0x%x agino 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->agno, __entry->agino) +) + +#define DEFINE_AGINODE_EVENT(name) \ +DEFINE_EVENT(xfs_ag_inode_class, name, \ + TP_PROTO(struct xfs_inode *ip), \ + TP_ARGS(ip)) +DEFINE_AGINODE_EVENT(xfs_iunlink); +DEFINE_AGINODE_EVENT(xfs_iunlink_remove); + +DECLARE_EVENT_CLASS(xfs_fs_corrupt_class, + TP_PROTO(struct xfs_mount *mp, unsigned int flags), + TP_ARGS(mp, flags), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, flags) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->flags = flags; + ), + TP_printk("dev %d:%d flags 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->flags) +); +#define DEFINE_FS_CORRUPT_EVENT(name) \ +DEFINE_EVENT(xfs_fs_corrupt_class, name, \ + TP_PROTO(struct xfs_mount *mp, unsigned int flags), \ + TP_ARGS(mp, flags)) +DEFINE_FS_CORRUPT_EVENT(xfs_fs_mark_sick); +DEFINE_FS_CORRUPT_EVENT(xfs_fs_mark_corrupt); +DEFINE_FS_CORRUPT_EVENT(xfs_fs_mark_healthy); +DEFINE_FS_CORRUPT_EVENT(xfs_fs_unfixed_corruption); + +DECLARE_EVENT_CLASS(xfs_group_corrupt_class, + TP_PROTO(const struct xfs_group *xg, unsigned int flags), + TP_ARGS(xg, flags), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(enum xfs_group_type, type) + __field(uint32_t, index) + __field(unsigned int, flags) + ), + TP_fast_assign( + __entry->dev = xg->xg_mount->m_super->s_dev; + __entry->type = xg->xg_type; + __entry->index = xg->xg_gno; + __entry->flags = flags; + ), + TP_printk("dev %d:%d %sno 0x%x flags 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->index, __entry->flags) +); +#define DEFINE_GROUP_CORRUPT_EVENT(name) \ +DEFINE_EVENT(xfs_group_corrupt_class, name, \ + TP_PROTO(const struct xfs_group *xg, unsigned int flags), \ + TP_ARGS(xg, flags)) +DEFINE_GROUP_CORRUPT_EVENT(xfs_group_mark_sick); +DEFINE_GROUP_CORRUPT_EVENT(xfs_group_mark_corrupt); +DEFINE_GROUP_CORRUPT_EVENT(xfs_group_mark_healthy); +DEFINE_GROUP_CORRUPT_EVENT(xfs_group_unfixed_corruption); + +DECLARE_EVENT_CLASS(xfs_inode_corrupt_class, + TP_PROTO(struct xfs_inode *ip, unsigned int flags), + TP_ARGS(ip, flags), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(unsigned int, flags) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = I_INO(ip); + __entry->flags = flags; + ), + TP_printk("dev %d:%d ino 0x%llx flags 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, __entry->flags) +); +#define DEFINE_INODE_CORRUPT_EVENT(name) \ +DEFINE_EVENT(xfs_inode_corrupt_class, name, \ + TP_PROTO(struct xfs_inode *ip, unsigned int flags), \ + TP_ARGS(ip, flags)) +DEFINE_INODE_CORRUPT_EVENT(xfs_inode_mark_sick); +DEFINE_INODE_CORRUPT_EVENT(xfs_inode_mark_corrupt); +DEFINE_INODE_CORRUPT_EVENT(xfs_inode_mark_healthy); +DEFINE_INODE_CORRUPT_EVENT(xfs_inode_unfixed_corruption); + +TRACE_EVENT(xfs_iwalk_ag_rec, + TP_PROTO(const struct xfs_perag *pag, \ + struct xfs_inobt_rec_incore *irec), + TP_ARGS(pag, irec), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_agnumber_t, agno) + __field(xfs_agino_t, startino) + __field(uint64_t, freemask) + ), + TP_fast_assign( + __entry->dev = pag_mount(pag)->m_super->s_dev; + __entry->agno = pag_agno(pag); + __entry->startino = irec->ir_startino; + __entry->freemask = irec->ir_free; + ), + TP_printk("dev %d:%d agno 0x%x startino 0x%x freemask 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), __entry->agno, + __entry->startino, __entry->freemask) +) + +TRACE_EVENT(xfs_pwork_init, + TP_PROTO(struct xfs_mount *mp, unsigned int nr_threads, pid_t pid), + TP_ARGS(mp, nr_threads, pid), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, nr_threads) + __field(pid_t, pid) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->nr_threads = nr_threads; + __entry->pid = pid; + ), + TP_printk("dev %d:%d nr_threads %u pid %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->nr_threads, __entry->pid) +) + +TRACE_EVENT(xfs_check_new_dalign, + TP_PROTO(struct xfs_mount *mp, int new_dalign, xfs_ino_t calc_rootino), + TP_ARGS(mp, new_dalign, calc_rootino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(int, new_dalign) + __field(xfs_ino_t, sb_rootino) + __field(xfs_ino_t, calc_rootino) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->new_dalign = new_dalign; + __entry->sb_rootino = mp->m_sb.sb_rootino; + __entry->calc_rootino = calc_rootino; + ), + TP_printk("dev %d:%d new_dalign %d sb_rootino 0x%llx calc_rootino 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->new_dalign, __entry->sb_rootino, + __entry->calc_rootino) +) + +TRACE_EVENT(xfs_btree_commit_afakeroot, + TP_PROTO(struct xfs_btree_cur *cur), + TP_ARGS(cur), + TP_STRUCT__entry( + __field(dev_t, dev) + __string(name, cur->bc_ops->name) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, agbno) + __field(unsigned int, levels) + __field(unsigned int, blocks) + ), + TP_fast_assign( + __entry->dev = cur->bc_mp->m_super->s_dev; + __assign_str(name); + __entry->agno = cur->bc_group->xg_gno; + __entry->agbno = cur->bc_ag.afake->af_root; + __entry->levels = cur->bc_ag.afake->af_levels; + __entry->blocks = cur->bc_ag.afake->af_blocks; + ), + TP_printk("dev %d:%d %sbt agno 0x%x levels %u blocks %u root %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __get_str(name), + __entry->agno, + __entry->levels, + __entry->blocks, + __entry->agbno) +) + +TRACE_EVENT(xfs_btree_commit_ifakeroot, + TP_PROTO(struct xfs_btree_cur *cur), + TP_ARGS(cur), + TP_STRUCT__entry( + __field(dev_t, dev) + __string(name, cur->bc_ops->name) + __field(xfs_agnumber_t, agno) + __field(xfs_agino_t, agino) + __field(unsigned int, levels) + __field(unsigned int, blocks) + __field(int, whichfork) + ), + TP_fast_assign( + __entry->dev = cur->bc_mp->m_super->s_dev; + __assign_str(name); + __entry->agno = XFS_INODE_TO_AGNO(cur->bc_ino.ip); + __entry->agino = XFS_INODE_TO_AGINO(cur->bc_ino.ip); + __entry->levels = cur->bc_ino.ifake->if_levels; + __entry->blocks = cur->bc_ino.ifake->if_blocks; + __entry->whichfork = cur->bc_ino.whichfork; + ), + TP_printk("dev %d:%d %sbt agno 0x%x agino 0x%x whichfork %s levels %u blocks %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __get_str(name), + __entry->agno, + __entry->agino, + __print_symbolic(__entry->whichfork, XFS_WHICHFORK_STRINGS), + __entry->levels, + __entry->blocks) +) + +TRACE_EVENT(xfs_btree_bload_level_geometry, + TP_PROTO(struct xfs_btree_cur *cur, unsigned int level, + uint64_t nr_this_level, unsigned int nr_per_block, + unsigned int desired_npb, uint64_t blocks, + uint64_t blocks_with_extra), + TP_ARGS(cur, level, nr_this_level, nr_per_block, desired_npb, blocks, + blocks_with_extra), + TP_STRUCT__entry( + __field(dev_t, dev) + __string(name, cur->bc_ops->name) + __field(unsigned int, level) + __field(unsigned int, nlevels) + __field(uint64_t, nr_this_level) + __field(unsigned int, nr_per_block) + __field(unsigned int, desired_npb) + __field(unsigned long long, blocks) + __field(unsigned long long, blocks_with_extra) + ), + TP_fast_assign( + __entry->dev = cur->bc_mp->m_super->s_dev; + __assign_str(name); + __entry->level = level; + __entry->nlevels = cur->bc_nlevels; + __entry->nr_this_level = nr_this_level; + __entry->nr_per_block = nr_per_block; + __entry->desired_npb = desired_npb; + __entry->blocks = blocks; + __entry->blocks_with_extra = blocks_with_extra; + ), + TP_printk("dev %d:%d %sbt level %u/%u nr_this_level %llu nr_per_block %u desired_npb %u blocks %llu blocks_with_extra %llu", + MAJOR(__entry->dev), MINOR(__entry->dev), + __get_str(name), + __entry->level, + __entry->nlevels, + __entry->nr_this_level, + __entry->nr_per_block, + __entry->desired_npb, + __entry->blocks, + __entry->blocks_with_extra) +) + +TRACE_EVENT(xfs_btree_bload_block, + TP_PROTO(struct xfs_btree_cur *cur, unsigned int level, + uint64_t block_idx, uint64_t nr_blocks, + union xfs_btree_ptr *ptr, unsigned int nr_records), + TP_ARGS(cur, level, block_idx, nr_blocks, ptr, nr_records), + TP_STRUCT__entry( + __field(dev_t, dev) + __string(name, cur->bc_ops->name) + __field(unsigned int, level) + __field(unsigned long long, block_idx) + __field(unsigned long long, nr_blocks) + __field(xfs_agnumber_t, agno) + __field(xfs_agblock_t, agbno) + __field(unsigned int, nr_records) + ), + TP_fast_assign( + __entry->dev = cur->bc_mp->m_super->s_dev; + __assign_str(name); + __entry->level = level; + __entry->block_idx = block_idx; + __entry->nr_blocks = nr_blocks; + if (cur->bc_ops->ptr_len == XFS_BTREE_LONG_PTR_LEN) { + xfs_fsblock_t fsb = be64_to_cpu(ptr->l); + + __entry->agno = XFS_FSB_TO_AGNO(cur->bc_mp, fsb); + __entry->agbno = XFS_FSB_TO_AGBNO(cur->bc_mp, fsb); + } else { + __entry->agno = cur->bc_group->xg_gno; + __entry->agbno = be32_to_cpu(ptr->s); + } + __entry->nr_records = nr_records; + ), + TP_printk("dev %d:%d %sbt level %u block %llu/%llu agno 0x%x agbno 0x%x recs %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __get_str(name), + __entry->level, + __entry->block_idx, + __entry->nr_blocks, + __entry->agno, + __entry->agbno, + __entry->nr_records) +) + +DECLARE_EVENT_CLASS(xfs_timestamp_range_class, + TP_PROTO(struct xfs_mount *mp, time64_t min, time64_t max), + TP_ARGS(mp, min, max), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(long long, min) + __field(long long, max) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->min = min; + __entry->max = max; + ), + TP_printk("dev %d:%d min %lld max %lld", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->min, + __entry->max) +) + +#define DEFINE_TIMESTAMP_RANGE_EVENT(name) \ +DEFINE_EVENT(xfs_timestamp_range_class, name, \ + TP_PROTO(struct xfs_mount *mp, long long min, long long max), \ + TP_ARGS(mp, min, max)) +DEFINE_TIMESTAMP_RANGE_EVENT(xfs_inode_timestamp_range); +DEFINE_TIMESTAMP_RANGE_EVENT(xfs_quota_expiry_range); + +DECLARE_EVENT_CLASS(xfs_icwalk_class, + TP_PROTO(struct xfs_mount *mp, struct xfs_icwalk *icw, + unsigned long caller_ip), + TP_ARGS(mp, icw, caller_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(__u32, flags) + __field(uint32_t, uid) + __field(uint32_t, gid) + __field(prid_t, prid) + __field(__u64, min_file_size) + __field(long, scan_limit) + __field(unsigned long, caller_ip) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->flags = icw ? icw->icw_flags : 0; + __entry->uid = icw ? from_kuid(mp->m_super->s_user_ns, + icw->icw_uid) : 0; + __entry->gid = icw ? from_kgid(mp->m_super->s_user_ns, + icw->icw_gid) : 0; + __entry->prid = icw ? icw->icw_prid : 0; + __entry->min_file_size = icw ? icw->icw_min_file_size : 0; + __entry->scan_limit = icw ? icw->icw_scan_limit : 0; + __entry->caller_ip = caller_ip; + ), + TP_printk("dev %d:%d flags 0x%x uid %u gid %u prid %u minsize %llu scan_limit %ld caller %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->flags, + __entry->uid, + __entry->gid, + __entry->prid, + __entry->min_file_size, + __entry->scan_limit, + (char *)__entry->caller_ip) +); +#define DEFINE_ICWALK_EVENT(name) \ +DEFINE_EVENT(xfs_icwalk_class, name, \ + TP_PROTO(struct xfs_mount *mp, struct xfs_icwalk *icw, \ + unsigned long caller_ip), \ + TP_ARGS(mp, icw, caller_ip)) +DEFINE_ICWALK_EVENT(xfs_ioc_free_eofblocks); +DEFINE_ICWALK_EVENT(xfs_blockgc_free_space); + +TRACE_DEFINE_ENUM(XLOG_STATE_ACTIVE); +TRACE_DEFINE_ENUM(XLOG_STATE_WANT_SYNC); +TRACE_DEFINE_ENUM(XLOG_STATE_SYNCING); +TRACE_DEFINE_ENUM(XLOG_STATE_DONE_SYNC); +TRACE_DEFINE_ENUM(XLOG_STATE_CALLBACK); +TRACE_DEFINE_ENUM(XLOG_STATE_DIRTY); + +DECLARE_EVENT_CLASS(xlog_iclog_class, + TP_PROTO(struct xlog_in_core *iclog, unsigned long caller_ip), + TP_ARGS(iclog, caller_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(uint32_t, state) + __field(int32_t, refcount) + __field(uint32_t, offset) + __field(uint32_t, flags) + __field(unsigned long long, lsn) + __field(unsigned long, caller_ip) + ), + TP_fast_assign( + __entry->dev = iclog->ic_log->l_mp->m_super->s_dev; + __entry->state = iclog->ic_state; + __entry->refcount = atomic_read(&iclog->ic_refcnt); + __entry->offset = iclog->ic_offset; + __entry->flags = iclog->ic_flags; + __entry->lsn = be64_to_cpu(iclog->ic_header->h_lsn); + __entry->caller_ip = caller_ip; + ), + TP_printk("dev %d:%d state %s refcnt %d offset %u lsn 0x%llx flags %s caller %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->state, XLOG_STATE_STRINGS), + __entry->refcount, + __entry->offset, + __entry->lsn, + __print_flags(__entry->flags, "|", XLOG_ICL_STRINGS), + (char *)__entry->caller_ip) + +); + +#define DEFINE_ICLOG_EVENT(name) \ +DEFINE_EVENT(xlog_iclog_class, name, \ + TP_PROTO(struct xlog_in_core *iclog, unsigned long caller_ip), \ + TP_ARGS(iclog, caller_ip)) + +DEFINE_ICLOG_EVENT(xlog_iclog_activate); +DEFINE_ICLOG_EVENT(xlog_iclog_clean); +DEFINE_ICLOG_EVENT(xlog_iclog_callback); +DEFINE_ICLOG_EVENT(xlog_iclog_callbacks_start); +DEFINE_ICLOG_EVENT(xlog_iclog_callbacks_done); +DEFINE_ICLOG_EVENT(xlog_iclog_force); +DEFINE_ICLOG_EVENT(xlog_iclog_force_lsn); +DEFINE_ICLOG_EVENT(xlog_iclog_get_space); +DEFINE_ICLOG_EVENT(xlog_iclog_release); +DEFINE_ICLOG_EVENT(xlog_iclog_switch); +DEFINE_ICLOG_EVENT(xlog_iclog_sync); +DEFINE_ICLOG_EVENT(xlog_iclog_syncing); +DEFINE_ICLOG_EVENT(xlog_iclog_sync_done); +DEFINE_ICLOG_EVENT(xlog_iclog_wait_on); +DEFINE_ICLOG_EVENT(xlog_iclog_write); + +TRACE_DEFINE_ENUM(XFS_DAS_UNINIT); +TRACE_DEFINE_ENUM(XFS_DAS_SF_ADD); +TRACE_DEFINE_ENUM(XFS_DAS_SF_REMOVE); +TRACE_DEFINE_ENUM(XFS_DAS_LEAF_ADD); +TRACE_DEFINE_ENUM(XFS_DAS_LEAF_REMOVE); +TRACE_DEFINE_ENUM(XFS_DAS_NODE_ADD); +TRACE_DEFINE_ENUM(XFS_DAS_NODE_REMOVE); +TRACE_DEFINE_ENUM(XFS_DAS_LEAF_SET_RMT); +TRACE_DEFINE_ENUM(XFS_DAS_LEAF_ALLOC_RMT); +TRACE_DEFINE_ENUM(XFS_DAS_LEAF_REPLACE); +TRACE_DEFINE_ENUM(XFS_DAS_LEAF_REMOVE_OLD); +TRACE_DEFINE_ENUM(XFS_DAS_LEAF_REMOVE_RMT); +TRACE_DEFINE_ENUM(XFS_DAS_LEAF_REMOVE_ATTR); +TRACE_DEFINE_ENUM(XFS_DAS_NODE_SET_RMT); +TRACE_DEFINE_ENUM(XFS_DAS_NODE_ALLOC_RMT); +TRACE_DEFINE_ENUM(XFS_DAS_NODE_REPLACE); +TRACE_DEFINE_ENUM(XFS_DAS_NODE_REMOVE_OLD); +TRACE_DEFINE_ENUM(XFS_DAS_NODE_REMOVE_RMT); +TRACE_DEFINE_ENUM(XFS_DAS_NODE_REMOVE_ATTR); +TRACE_DEFINE_ENUM(XFS_DAS_DONE); + +DECLARE_EVENT_CLASS(xfs_das_state_class, + TP_PROTO(int das, struct xfs_inode *ip), + TP_ARGS(das, ip), + TP_STRUCT__entry( + __field(int, das) + __field(xfs_ino_t, ino) + ), + TP_fast_assign( + __entry->das = das; + __entry->ino = I_INO(ip); + ), + TP_printk("state change %s ino 0x%llx", + __print_symbolic(__entry->das, XFS_DAS_STRINGS), + __entry->ino) +) + +#define DEFINE_DAS_STATE_EVENT(name) \ +DEFINE_EVENT(xfs_das_state_class, name, \ + TP_PROTO(int das, struct xfs_inode *ip), \ + TP_ARGS(das, ip)) +DEFINE_DAS_STATE_EVENT(xfs_attr_sf_addname_return); +DEFINE_DAS_STATE_EVENT(xfs_attr_set_iter_return); +DEFINE_DAS_STATE_EVENT(xfs_attr_leaf_addname_return); +DEFINE_DAS_STATE_EVENT(xfs_attr_node_addname_return); +DEFINE_DAS_STATE_EVENT(xfs_attr_rmtval_alloc); +DEFINE_DAS_STATE_EVENT(xfs_attr_rmtval_remove_return); +DEFINE_DAS_STATE_EVENT(xfs_attr_defer_add); + + +TRACE_EVENT(xfs_force_shutdown, + TP_PROTO(struct xfs_mount *mp, int ptag, int flags, const char *fname, + int line_num), + TP_ARGS(mp, ptag, flags, fname, line_num), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(int, ptag) + __field(int, flags) + __string(fname, fname) + __field(int, line_num) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->ptag = ptag; + __entry->flags = flags; + __assign_str(fname); + __entry->line_num = line_num; + ), + TP_printk("dev %d:%d tag %s flags %s file %s line_num %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_flags(__entry->ptag, "|", XFS_PTAG_STRINGS), + __print_flags(__entry->flags, "|", XFS_SHUTDOWN_STRINGS), + __get_str(fname), + __entry->line_num) +); + +#ifdef CONFIG_XFS_DRAIN_INTENTS +DECLARE_EVENT_CLASS(xfs_group_intents_class, + TP_PROTO(const struct xfs_group *xg, void *caller_ip), + TP_ARGS(xg, caller_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(enum xfs_group_type, type) + __field(uint32_t, index) + __field(long, nr_intents) + __field(void *, caller_ip) + ), + TP_fast_assign( + __entry->dev = xg->xg_mount->m_super->s_dev; + __entry->type = xg->xg_type; + __entry->index = xg->xg_gno; + __entry->nr_intents = + atomic_read(&xg->xg_intents_drain.dr_count); + __entry->caller_ip = caller_ip; + ), + TP_printk("dev %d:%d %sno 0x%x intents %ld caller %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XG_TYPE_STRINGS), + __entry->index, + __entry->nr_intents, + __entry->caller_ip) +); + +#define DEFINE_GROUP_INTENTS_EVENT(name) \ +DEFINE_EVENT(xfs_group_intents_class, name, \ + TP_PROTO(const struct xfs_group *xg, void *caller_ip), \ + TP_ARGS(xg, caller_ip)) +DEFINE_GROUP_INTENTS_EVENT(xfs_group_intent_hold); +DEFINE_GROUP_INTENTS_EVENT(xfs_group_intent_rele); +DEFINE_GROUP_INTENTS_EVENT(xfs_group_wait_intents); + +#endif /* CONFIG_XFS_DRAIN_INTENTS */ + +#ifdef CONFIG_XFS_MEMORY_BUFS +TRACE_EVENT(xmbuf_create, + TP_PROTO(struct xfs_buftarg *btp), + TP_ARGS(btp), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned long, ino) + ), + TP_fast_assign( + struct file *file = btp->bt_file; + + __entry->dev = btp->bt_mount->m_super->s_dev; + __entry->ino = file_inode(file)->i_ino; + ), + TP_printk("dev %d:%d xmino 0x%lx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino) +); + +TRACE_EVENT(xmbuf_free, + TP_PROTO(struct xfs_buftarg *btp), + TP_ARGS(btp), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned long, ino) + __field(unsigned long long, bytes) + __field(loff_t, size) + ), + TP_fast_assign( + struct file *file = btp->bt_file; + struct inode *inode = file_inode(file); + + __entry->dev = btp->bt_mount->m_super->s_dev; + __entry->size = i_size_read(inode); + __entry->bytes = (inode->i_blocks << SECTOR_SHIFT) + inode->i_bytes; + __entry->ino = inode->i_ino; + ), + TP_printk("dev %d:%d xmino 0x%lx mem_bytes 0x%llx isize 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->bytes, + __entry->size) +); +#endif /* CONFIG_XFS_MEMORY_BUFS */ + +#ifdef CONFIG_XFS_BTREE_IN_MEM +TRACE_EVENT(xfbtree_init, + TP_PROTO(struct xfs_mount *mp, struct xfbtree *xfbt, + const struct xfs_btree_ops *ops), + TP_ARGS(mp, xfbt, ops), + TP_STRUCT__entry( + __field(const void *, btree_ops) + __field(unsigned long, xfino) + __field(unsigned int, leaf_mxr) + __field(unsigned int, leaf_mnr) + __field(unsigned int, node_mxr) + __field(unsigned int, node_mnr) + __field(unsigned long long, owner) + ), + TP_fast_assign( + __entry->btree_ops = ops; + __entry->xfino = file_inode(xfbt->target->bt_file)->i_ino; + __entry->leaf_mxr = xfbt->maxrecs[0]; + __entry->node_mxr = xfbt->maxrecs[1]; + __entry->leaf_mnr = xfbt->minrecs[0]; + __entry->node_mnr = xfbt->minrecs[1]; + __entry->owner = xfbt->owner; + ), + TP_printk("xfino 0x%lx btree_ops %pS owner 0x%llx leaf_mxr %u leaf_mnr %u node_mxr %u node_mnr %u", + __entry->xfino, + __entry->btree_ops, + __entry->owner, + __entry->leaf_mxr, + __entry->leaf_mnr, + __entry->node_mxr, + __entry->node_mnr) +); + +DECLARE_EVENT_CLASS(xfbtree_buf_class, + TP_PROTO(struct xfbtree *xfbt, struct xfs_buf *bp), + TP_ARGS(xfbt, bp), + TP_STRUCT__entry( + __field(unsigned long, xfino) + __field(xfs_daddr_t, bno) + __field(int, nblks) + __field(int, hold) + __field(int, pincount) + __field(unsigned int, lockval) + __field(unsigned int, flags) + ), + TP_fast_assign( + __entry->xfino = file_inode(xfbt->target->bt_file)->i_ino; + __entry->bno = xfs_buf_daddr(bp); + __entry->nblks = bp->b_length; + __entry->hold = bp->b_lockref.count; + __entry->pincount = atomic_read(&bp->b_pin_count); + __entry->lockval = bp->b_sema.count; + __entry->flags = bp->b_flags; + ), + TP_printk("xfino 0x%lx daddr 0x%llx bbcount 0x%x hold %d pincount %d lock %d flags %s", + __entry->xfino, + (unsigned long long)__entry->bno, + __entry->nblks, + __entry->hold, + __entry->pincount, + __entry->lockval, + __print_flags(__entry->flags, "|", XFS_BUF_FLAGS)) +) + +#define DEFINE_XFBTREE_BUF_EVENT(name) \ +DEFINE_EVENT(xfbtree_buf_class, name, \ + TP_PROTO(struct xfbtree *xfbt, struct xfs_buf *bp), \ + TP_ARGS(xfbt, bp)) +DEFINE_XFBTREE_BUF_EVENT(xfbtree_create_root_buf); +DEFINE_XFBTREE_BUF_EVENT(xfbtree_trans_commit_buf); +DEFINE_XFBTREE_BUF_EVENT(xfbtree_trans_cancel_buf); + +DECLARE_EVENT_CLASS(xfbtree_freesp_class, + TP_PROTO(struct xfbtree *xfbt, struct xfs_btree_cur *cur, + xfs_fileoff_t fileoff), + TP_ARGS(xfbt, cur, fileoff), + TP_STRUCT__entry( + __field(unsigned long, xfino) + __string(btname, cur->bc_ops->name) + __field(int, nlevels) + __field(xfs_fileoff_t, fileoff) + ), + TP_fast_assign( + __entry->xfino = file_inode(xfbt->target->bt_file)->i_ino; + __assign_str(btname); + __entry->nlevels = cur->bc_nlevels; + __entry->fileoff = fileoff; + ), + TP_printk("xfino 0x%lx %sbt nlevels %d fileoff 0x%llx", + __entry->xfino, + __get_str(btname), + __entry->nlevels, + (unsigned long long)__entry->fileoff) +) + +#define DEFINE_XFBTREE_FREESP_EVENT(name) \ +DEFINE_EVENT(xfbtree_freesp_class, name, \ + TP_PROTO(struct xfbtree *xfbt, struct xfs_btree_cur *cur, \ + xfs_fileoff_t fileoff), \ + TP_ARGS(xfbt, cur, fileoff)) +DEFINE_XFBTREE_FREESP_EVENT(xfbtree_alloc_block); +DEFINE_XFBTREE_FREESP_EVENT(xfbtree_free_block); +#endif /* CONFIG_XFS_BTREE_IN_MEM */ + +/* exchmaps tracepoints */ +#define XFS_EXCHMAPS_STRINGS \ + { XFS_EXCHMAPS_ATTR_FORK, "ATTRFORK" }, \ + { XFS_EXCHMAPS_SET_SIZES, "SETSIZES" }, \ + { XFS_EXCHMAPS_INO1_WRITTEN, "INO1_WRITTEN" }, \ + { XFS_EXCHMAPS_CLEAR_INO1_REFLINK, "CLEAR_INO1_REFLINK" }, \ + { XFS_EXCHMAPS_CLEAR_INO2_REFLINK, "CLEAR_INO2_REFLINK" }, \ + { __XFS_EXCHMAPS_INO2_SHORTFORM, "INO2_SF" } + +DEFINE_INODE_IREC_EVENT(xfs_exchmaps_mapping1_skip); +DEFINE_INODE_IREC_EVENT(xfs_exchmaps_mapping1); +DEFINE_INODE_IREC_EVENT(xfs_exchmaps_mapping2); +DEFINE_ITRUNC_EVENT(xfs_exchmaps_update_inode_size); + +#define XFS_EXCHRANGE_INODES \ + { 1, "file1" }, \ + { 2, "file2" } + +DECLARE_EVENT_CLASS(xfs_exchrange_inode_class, + TP_PROTO(struct xfs_inode *ip, int whichfile), + TP_ARGS(ip, whichfile), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(int, whichfile) + __field(xfs_ino_t, ino) + __field(int, format) + __field(xfs_extnum_t, nex) + __field(int, broot_size) + __field(int, fork_off) + ), + TP_fast_assign( + __entry->dev = VFS_I(ip)->i_sb->s_dev; + __entry->whichfile = whichfile; + __entry->ino = I_INO(ip); + __entry->format = ip->i_df.if_format; + __entry->nex = ip->i_df.if_nextents; + __entry->fork_off = xfs_inode_fork_boff(ip); + ), + TP_printk("dev %d:%d ino 0x%llx whichfile %s format %s num_extents %llu forkoff 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __print_symbolic(__entry->whichfile, XFS_EXCHRANGE_INODES), + __print_symbolic(__entry->format, XFS_INODE_FORMAT_STR), + __entry->nex, + __entry->fork_off) +) + +#define DEFINE_EXCHRANGE_INODE_EVENT(name) \ +DEFINE_EVENT(xfs_exchrange_inode_class, name, \ + TP_PROTO(struct xfs_inode *ip, int whichfile), \ + TP_ARGS(ip, whichfile)) + +DEFINE_EXCHRANGE_INODE_EVENT(xfs_exchrange_before); +DEFINE_EXCHRANGE_INODE_EVENT(xfs_exchrange_after); +DEFINE_INODE_ERROR_EVENT(xfs_exchrange_error); + +#define XFS_EXCHANGE_RANGE_FLAGS_STRS \ + { XFS_EXCHANGE_RANGE_TO_EOF, "TO_EOF" }, \ + { XFS_EXCHANGE_RANGE_DSYNC , "DSYNC" }, \ + { XFS_EXCHANGE_RANGE_DRY_RUN, "DRY_RUN" }, \ + { XFS_EXCHANGE_RANGE_FILE1_WRITTEN, "F1_WRITTEN" }, \ + { __XFS_EXCHANGE_RANGE_UPD_CMTIME1, "CMTIME1" }, \ + { __XFS_EXCHANGE_RANGE_UPD_CMTIME2, "CMTIME2" }, \ + { __XFS_EXCHANGE_RANGE_CHECK_FRESH2, "FRESH2" } + +/* file exchange-range tracepoint class */ +DECLARE_EVENT_CLASS(xfs_exchrange_class, + TP_PROTO(const struct xfs_exchrange *fxr, struct xfs_inode *ip1, + struct xfs_inode *ip2), + TP_ARGS(fxr, ip1, ip2), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ip1_ino) + __field(loff_t, ip1_isize) + __field(loff_t, ip1_disize) + __field(xfs_ino_t, ip2_ino) + __field(loff_t, ip2_isize) + __field(loff_t, ip2_disize) + + __field(loff_t, file1_offset) + __field(loff_t, file2_offset) + __field(unsigned long long, length) + __field(unsigned long long, flags) + ), + TP_fast_assign( + __entry->dev = VFS_I(ip1)->i_sb->s_dev; + __entry->ip1_ino = I_INO(ip1); + __entry->ip1_isize = VFS_I(ip1)->i_size; + __entry->ip1_disize = ip1->i_disk_size; + __entry->ip2_ino = I_INO(ip2); + __entry->ip2_isize = VFS_I(ip2)->i_size; + __entry->ip2_disize = ip2->i_disk_size; + + __entry->file1_offset = fxr->file1_offset; + __entry->file2_offset = fxr->file2_offset; + __entry->length = fxr->length; + __entry->flags = fxr->flags; + ), + TP_printk("dev %d:%d flags %s bytecount 0x%llx " + "ino1 0x%llx isize 0x%llx disize 0x%llx pos 0x%llx -> " + "ino2 0x%llx isize 0x%llx disize 0x%llx pos 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_flags_u64(__entry->flags, "|", XFS_EXCHANGE_RANGE_FLAGS_STRS), + __entry->length, + __entry->ip1_ino, + __entry->ip1_isize, + __entry->ip1_disize, + __entry->file1_offset, + __entry->ip2_ino, + __entry->ip2_isize, + __entry->ip2_disize, + __entry->file2_offset) +) + +#define DEFINE_EXCHRANGE_EVENT(name) \ +DEFINE_EVENT(xfs_exchrange_class, name, \ + TP_PROTO(const struct xfs_exchrange *fxr, struct xfs_inode *ip1, \ + struct xfs_inode *ip2), \ + TP_ARGS(fxr, ip1, ip2)) +DEFINE_EXCHRANGE_EVENT(xfs_exchrange_prep); +DEFINE_EXCHRANGE_EVENT(xfs_exchrange_flush); +DEFINE_EXCHRANGE_EVENT(xfs_exchrange_mappings); + +TRACE_EVENT(xfs_exchrange_freshness, + TP_PROTO(const struct xfs_exchrange *fxr, struct xfs_inode *ip2), + TP_ARGS(fxr, ip2), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ip2_ino) + __field(long long, ip2_mtime) + __field(long long, ip2_ctime) + __field(int, ip2_mtime_nsec) + __field(int, ip2_ctime_nsec) + + __field(xfs_ino_t, file2_ino) + __field(long long, file2_mtime) + __field(long long, file2_ctime) + __field(int, file2_mtime_nsec) + __field(int, file2_ctime_nsec) + ), + TP_fast_assign( + struct timespec64 ts64; + struct inode *inode2 = VFS_I(ip2); + + __entry->dev = inode2->i_sb->s_dev; + __entry->ip2_ino = I_INO(ip2); + + ts64 = inode_get_ctime(inode2); + __entry->ip2_ctime = ts64.tv_sec; + __entry->ip2_ctime_nsec = ts64.tv_nsec; + + ts64 = inode_get_mtime(inode2); + __entry->ip2_mtime = ts64.tv_sec; + __entry->ip2_mtime_nsec = ts64.tv_nsec; + + __entry->file2_ino = fxr->file2_ino; + __entry->file2_mtime = fxr->file2_mtime.tv_sec; + __entry->file2_ctime = fxr->file2_ctime.tv_sec; + __entry->file2_mtime_nsec = fxr->file2_mtime.tv_nsec; + __entry->file2_ctime_nsec = fxr->file2_ctime.tv_nsec; + ), + TP_printk("dev %d:%d " + "ino 0x%llx mtime %lld:%d ctime %lld:%d -> " + "file 0x%llx mtime %lld:%d ctime %lld:%d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ip2_ino, + __entry->ip2_mtime, + __entry->ip2_mtime_nsec, + __entry->ip2_ctime, + __entry->ip2_ctime_nsec, + __entry->file2_ino, + __entry->file2_mtime, + __entry->file2_mtime_nsec, + __entry->file2_ctime, + __entry->file2_ctime_nsec) +); + +TRACE_EVENT(xfs_exchmaps_overhead, + TP_PROTO(struct xfs_mount *mp, unsigned long long bmbt_blocks, + unsigned long long rmapbt_blocks), + TP_ARGS(mp, bmbt_blocks, rmapbt_blocks), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned long long, bmbt_blocks) + __field(unsigned long long, rmapbt_blocks) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->bmbt_blocks = bmbt_blocks; + __entry->rmapbt_blocks = rmapbt_blocks; + ), + TP_printk("dev %d:%d bmbt_blocks 0x%llx rmapbt_blocks 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->bmbt_blocks, + __entry->rmapbt_blocks) +); + +DECLARE_EVENT_CLASS(xfs_exchmaps_estimate_class, + TP_PROTO(const struct xfs_exchmaps_req *req), + TP_ARGS(req), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino1) + __field(xfs_ino_t, ino2) + __field(xfs_fileoff_t, startoff1) + __field(xfs_fileoff_t, startoff2) + __field(xfs_filblks_t, blockcount) + __field(uint64_t, flags) + __field(xfs_filblks_t, ip1_bcount) + __field(xfs_filblks_t, ip2_bcount) + __field(xfs_filblks_t, ip1_rtbcount) + __field(xfs_filblks_t, ip2_rtbcount) + __field(unsigned long long, resblks) + __field(unsigned long long, nr_exchanges) + ), + TP_fast_assign( + __entry->dev = req->ip1->i_mount->m_super->s_dev; + __entry->ino1 = I_INO(req->ip1); + __entry->ino2 = I_INO(req->ip2); + __entry->startoff1 = req->startoff1; + __entry->startoff2 = req->startoff2; + __entry->blockcount = req->blockcount; + __entry->flags = req->flags; + __entry->ip1_bcount = req->ip1_bcount; + __entry->ip2_bcount = req->ip2_bcount; + __entry->ip1_rtbcount = req->ip1_rtbcount; + __entry->ip2_rtbcount = req->ip2_rtbcount; + __entry->resblks = req->resblks; + __entry->nr_exchanges = req->nr_exchanges; + ), + TP_printk("dev %d:%d ino1 0x%llx fileoff1 0x%llx ino2 0x%llx fileoff2 0x%llx fsbcount 0x%llx flags (%s) bcount1 0x%llx rtbcount1 0x%llx bcount2 0x%llx rtbcount2 0x%llx resblks 0x%llx nr_exchanges %llu", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino1, __entry->startoff1, + __entry->ino2, __entry->startoff2, + __entry->blockcount, + __print_flags_u64(__entry->flags, "|", XFS_EXCHMAPS_STRINGS), + __entry->ip1_bcount, + __entry->ip1_rtbcount, + __entry->ip2_bcount, + __entry->ip2_rtbcount, + __entry->resblks, + __entry->nr_exchanges) +); + +#define DEFINE_EXCHMAPS_ESTIMATE_EVENT(name) \ +DEFINE_EVENT(xfs_exchmaps_estimate_class, name, \ + TP_PROTO(const struct xfs_exchmaps_req *req), \ + TP_ARGS(req)) +DEFINE_EXCHMAPS_ESTIMATE_EVENT(xfs_exchmaps_initial_estimate); +DEFINE_EXCHMAPS_ESTIMATE_EVENT(xfs_exchmaps_final_estimate); + +DECLARE_EVENT_CLASS(xfs_exchmaps_intent_class, + TP_PROTO(struct xfs_mount *mp, const struct xfs_exchmaps_intent *xmi), + TP_ARGS(mp, xmi), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino1) + __field(xfs_ino_t, ino2) + __field(uint64_t, flags) + __field(xfs_fileoff_t, startoff1) + __field(xfs_fileoff_t, startoff2) + __field(xfs_filblks_t, blockcount) + __field(xfs_fsize_t, isize1) + __field(xfs_fsize_t, isize2) + __field(xfs_fsize_t, new_isize1) + __field(xfs_fsize_t, new_isize2) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->ino1 = I_INO(xmi->xmi_ip1); + __entry->ino2 = I_INO(xmi->xmi_ip2); + __entry->flags = xmi->xmi_flags; + __entry->startoff1 = xmi->xmi_startoff1; + __entry->startoff2 = xmi->xmi_startoff2; + __entry->blockcount = xmi->xmi_blockcount; + __entry->isize1 = xmi->xmi_ip1->i_disk_size; + __entry->isize2 = xmi->xmi_ip2->i_disk_size; + __entry->new_isize1 = xmi->xmi_isize1; + __entry->new_isize2 = xmi->xmi_isize2; + ), + TP_printk("dev %d:%d ino1 0x%llx fileoff1 0x%llx ino2 0x%llx fileoff2 0x%llx fsbcount 0x%llx flags (%s) isize1 0x%llx newisize1 0x%llx isize2 0x%llx newisize2 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino1, __entry->startoff1, + __entry->ino2, __entry->startoff2, + __entry->blockcount, + __print_flags_u64(__entry->flags, "|", XFS_EXCHMAPS_STRINGS), + __entry->isize1, __entry->new_isize1, + __entry->isize2, __entry->new_isize2) +); + +#define DEFINE_EXCHMAPS_INTENT_EVENT(name) \ +DEFINE_EVENT(xfs_exchmaps_intent_class, name, \ + TP_PROTO(struct xfs_mount *mp, const struct xfs_exchmaps_intent *xmi), \ + TP_ARGS(mp, xmi)) +DEFINE_EXCHMAPS_INTENT_EVENT(xfs_exchmaps_defer); +DEFINE_EXCHMAPS_INTENT_EVENT(xfs_exchmaps_recover); + +TRACE_EVENT(xfs_exchmaps_delta_nextents_step, + TP_PROTO(struct xfs_mount *mp, + const struct xfs_bmbt_irec *left, + const struct xfs_bmbt_irec *curr, + const struct xfs_bmbt_irec *new, + const struct xfs_bmbt_irec *right, + int delta, unsigned int state), + TP_ARGS(mp, left, curr, new, right, delta, state), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_fileoff_t, loff) + __field(xfs_fsblock_t, lstart) + __field(xfs_filblks_t, lcount) + __field(xfs_fileoff_t, coff) + __field(xfs_fsblock_t, cstart) + __field(xfs_filblks_t, ccount) + __field(xfs_fileoff_t, noff) + __field(xfs_fsblock_t, nstart) + __field(xfs_filblks_t, ncount) + __field(xfs_fileoff_t, roff) + __field(xfs_fsblock_t, rstart) + __field(xfs_filblks_t, rcount) + __field(int, delta) + __field(unsigned int, state) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->loff = left->br_startoff; + __entry->lstart = left->br_startblock; + __entry->lcount = left->br_blockcount; + __entry->coff = curr->br_startoff; + __entry->cstart = curr->br_startblock; + __entry->ccount = curr->br_blockcount; + __entry->noff = new->br_startoff; + __entry->nstart = new->br_startblock; + __entry->ncount = new->br_blockcount; + __entry->roff = right->br_startoff; + __entry->rstart = right->br_startblock; + __entry->rcount = right->br_blockcount; + __entry->delta = delta; + __entry->state = state; + ), + TP_printk("dev %d:%d left 0x%llx:0x%llx:0x%llx; curr 0x%llx:0x%llx:0x%llx <- new 0x%llx:0x%llx:0x%llx; right 0x%llx:0x%llx:0x%llx delta %d state 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->loff, __entry->lstart, __entry->lcount, + __entry->coff, __entry->cstart, __entry->ccount, + __entry->noff, __entry->nstart, __entry->ncount, + __entry->roff, __entry->rstart, __entry->rcount, + __entry->delta, __entry->state) +); + +TRACE_EVENT(xfs_exchmaps_delta_nextents, + TP_PROTO(const struct xfs_exchmaps_req *req, int64_t d_nexts1, + int64_t d_nexts2), + TP_ARGS(req, d_nexts1, d_nexts2), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino1) + __field(xfs_ino_t, ino2) + __field(xfs_extnum_t, nexts1) + __field(xfs_extnum_t, nexts2) + __field(int64_t, d_nexts1) + __field(int64_t, d_nexts2) + ), + TP_fast_assign( + int whichfork = xfs_exchmaps_reqfork(req); + + __entry->dev = req->ip1->i_mount->m_super->s_dev; + __entry->ino1 = I_INO(req->ip1); + __entry->ino2 = I_INO(req->ip2); + __entry->nexts1 = xfs_ifork_ptr(req->ip1, whichfork)->if_nextents; + __entry->nexts2 = xfs_ifork_ptr(req->ip2, whichfork)->if_nextents; + __entry->d_nexts1 = d_nexts1; + __entry->d_nexts2 = d_nexts2; + ), + TP_printk("dev %d:%d ino1 0x%llx nexts %llu ino2 0x%llx nexts %llu delta1 %lld delta2 %lld", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino1, __entry->nexts1, + __entry->ino2, __entry->nexts2, + __entry->d_nexts1, __entry->d_nexts2) +); + +DECLARE_EVENT_CLASS(xfs_getparents_rec_class, + TP_PROTO(struct xfs_inode *ip, const struct xfs_getparents *ppi, + const struct xfs_attr_list_context *context, + const struct xfs_getparents_rec *pptr), + TP_ARGS(ip, ppi, context, pptr), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(unsigned int, firstu) + __field(unsigned short, reclen) + __field(unsigned int, bufsize) + __field(xfs_ino_t, parent_ino) + __field(unsigned int, parent_gen) + __string(name, pptr->gpr_name) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = I_INO(ip); + __entry->firstu = context->firstu; + __entry->reclen = pptr->gpr_reclen; + __entry->bufsize = ppi->gp_bufsize; + __entry->parent_ino = pptr->gpr_parent.ha_fid.fid_ino; + __entry->parent_gen = pptr->gpr_parent.ha_fid.fid_gen; + __assign_str(name); + ), + TP_printk("dev %d:%d ino 0x%llx firstu %u reclen %u bufsize %u parent_ino 0x%llx parent_gen 0x%x name '%s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->firstu, + __entry->reclen, + __entry->bufsize, + __entry->parent_ino, + __entry->parent_gen, + __get_str(name)) +) +#define DEFINE_XFS_GETPARENTS_REC_EVENT(name) \ +DEFINE_EVENT(xfs_getparents_rec_class, name, \ + TP_PROTO(struct xfs_inode *ip, const struct xfs_getparents *ppi, \ + const struct xfs_attr_list_context *context, \ + const struct xfs_getparents_rec *pptr), \ + TP_ARGS(ip, ppi, context, pptr)) +DEFINE_XFS_GETPARENTS_REC_EVENT(xfs_getparents_put_listent); +DEFINE_XFS_GETPARENTS_REC_EVENT(xfs_getparents_expand_lastrec); + +DECLARE_EVENT_CLASS(xfs_getparents_class, + TP_PROTO(struct xfs_inode *ip, const struct xfs_getparents *ppi, + const struct xfs_attrlist_cursor_kern *cur), + TP_ARGS(ip, ppi, cur), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(unsigned short, iflags) + __field(unsigned short, oflags) + __field(unsigned int, bufsize) + __field(unsigned int, hashval) + __field(unsigned int, blkno) + __field(unsigned int, offset) + __field(int, initted) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = I_INO(ip); + __entry->iflags = ppi->gp_iflags; + __entry->oflags = ppi->gp_oflags; + __entry->bufsize = ppi->gp_bufsize; + __entry->hashval = cur->hashval; + __entry->blkno = cur->blkno; + __entry->offset = cur->offset; + __entry->initted = cur->initted; + ), + TP_printk("dev %d:%d ino 0x%llx iflags 0x%x oflags 0x%x bufsize %u cur_init? %d hashval 0x%x blkno %u offset %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->iflags, + __entry->oflags, + __entry->bufsize, + __entry->initted, + __entry->hashval, + __entry->blkno, + __entry->offset) +) +#define DEFINE_XFS_GETPARENTS_EVENT(name) \ +DEFINE_EVENT(xfs_getparents_class, name, \ + TP_PROTO(struct xfs_inode *ip, const struct xfs_getparents *ppi, \ + const struct xfs_attrlist_cursor_kern *cur), \ + TP_ARGS(ip, ppi, cur)) +DEFINE_XFS_GETPARENTS_EVENT(xfs_getparents_begin); +DEFINE_XFS_GETPARENTS_EVENT(xfs_getparents_end); + +DECLARE_EVENT_CLASS(xfs_metadir_update_class, + TP_PROTO(const struct xfs_metadir_update *upd), + TP_ARGS(upd), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, dp_ino) + __field(xfs_ino_t, ino) + __string(fname, upd->path) + ), + TP_fast_assign( + __entry->dev = upd->dp->i_mount->m_super->s_dev; + __entry->dp_ino = I_INO(upd->dp); + __entry->ino = upd->ip ? I_INO(upd->ip) : NULLFSINO; + __assign_str(fname); + ), + TP_printk("dev %d:%d dp 0x%llx fname '%s' ino 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->dp_ino, + __get_str(fname), + __entry->ino) +) + +#define DEFINE_METADIR_UPDATE_EVENT(name) \ +DEFINE_EVENT(xfs_metadir_update_class, name, \ + TP_PROTO(const struct xfs_metadir_update *upd), \ + TP_ARGS(upd)) +DEFINE_METADIR_UPDATE_EVENT(xfs_metadir_start_create); +DEFINE_METADIR_UPDATE_EVENT(xfs_metadir_start_link); +DEFINE_METADIR_UPDATE_EVENT(xfs_metadir_commit); +DEFINE_METADIR_UPDATE_EVENT(xfs_metadir_cancel); +DEFINE_METADIR_UPDATE_EVENT(xfs_metadir_try_create); +DEFINE_METADIR_UPDATE_EVENT(xfs_metadir_create); +DEFINE_METADIR_UPDATE_EVENT(xfs_metadir_link); + +DECLARE_EVENT_CLASS(xfs_metadir_update_error_class, + TP_PROTO(const struct xfs_metadir_update *upd, int error), + TP_ARGS(upd, error), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, dp_ino) + __field(xfs_ino_t, ino) + __field(int, error) + __string(fname, upd->path) + ), + TP_fast_assign( + __entry->dev = upd->dp->i_mount->m_super->s_dev; + __entry->dp_ino = I_INO(upd->dp); + __entry->ino = upd->ip ? I_INO(upd->ip) : NULLFSINO; + __entry->error = error; + __assign_str(fname); + ), + TP_printk("dev %d:%d dp 0x%llx fname '%s' ino 0x%llx error %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->dp_ino, + __get_str(fname), + __entry->ino, + __entry->error) +) + +#define DEFINE_METADIR_UPDATE_ERROR_EVENT(name) \ +DEFINE_EVENT(xfs_metadir_update_error_class, name, \ + TP_PROTO(const struct xfs_metadir_update *upd, int error), \ + TP_ARGS(upd, error)) +DEFINE_METADIR_UPDATE_ERROR_EVENT(xfs_metadir_teardown); + +DECLARE_EVENT_CLASS(xfs_metadir_class, + TP_PROTO(struct xfs_inode *dp, struct xfs_name *name, + xfs_ino_t ino), + TP_ARGS(dp, name, ino), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, dp_ino) + __field(xfs_ino_t, ino) + __field(int, ftype) + __field(int, namelen) + __dynamic_array(char, name, name->len) + ), + TP_fast_assign( + __entry->dev = VFS_I(dp)->i_sb->s_dev; + __entry->dp_ino = I_INO(dp); + __entry->ino = ino, + __entry->ftype = name->type; + __entry->namelen = name->len; + memcpy(__get_str(name), name->name, name->len); + ), + TP_printk("dev %d:%d dir 0x%llx type %s name '%.*s' ino 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->dp_ino, + __print_symbolic(__entry->ftype, XFS_DIR3_FTYPE_STR), + __entry->namelen, + __get_str(name), + __entry->ino) +) + +#define DEFINE_METADIR_EVENT(name) \ +DEFINE_EVENT(xfs_metadir_class, name, \ + TP_PROTO(struct xfs_inode *dp, struct xfs_name *name, \ + xfs_ino_t ino), \ + TP_ARGS(dp, name, ino)) +DEFINE_METADIR_EVENT(xfs_metadir_lookup); + +/* metadata inode space reservations */ + +DECLARE_EVENT_CLASS(xfs_metafile_resv_class, + TP_PROTO(struct xfs_mount *mp, xfs_filblks_t len), + TP_ARGS(mp, len), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned long long, freeblks) + __field(unsigned long long, reserved) + __field(unsigned long long, asked) + __field(unsigned long long, used) + __field(unsigned long long, len) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->freeblks = xfs_sum_freecounter_raw(mp, XC_FREE_BLOCKS); + __entry->reserved = mp->m_metafile_resv_avail; + __entry->asked = mp->m_metafile_resv_target; + __entry->used = mp->m_metafile_resv_used; + __entry->len = len; + ), + TP_printk("dev %d:%d freeblks %llu resv %llu ask %llu used %llu len %llu", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->freeblks, + __entry->reserved, + __entry->asked, + __entry->used, + __entry->len) +) +#define DEFINE_METAFILE_RESV_EVENT(name) \ +DEFINE_EVENT(xfs_metafile_resv_class, name, \ + TP_PROTO(struct xfs_mount *mp, xfs_filblks_t len), \ + TP_ARGS(mp, len)) +DEFINE_METAFILE_RESV_EVENT(xfs_metafile_resv_init); +DEFINE_METAFILE_RESV_EVENT(xfs_metafile_resv_free); +DEFINE_METAFILE_RESV_EVENT(xfs_metafile_resv_alloc_space); +DEFINE_METAFILE_RESV_EVENT(xfs_metafile_resv_free_space); +DEFINE_METAFILE_RESV_EVENT(xfs_metafile_resv_critical); +DEFINE_METAFILE_RESV_EVENT(xfs_metafile_resv_init_error); + +#ifdef CONFIG_XFS_RT +TRACE_EVENT(xfs_growfs_check_rtgeom, + TP_PROTO(const struct xfs_mount *mp, unsigned int min_logfsbs), + TP_ARGS(mp, min_logfsbs), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, logblocks) + __field(unsigned int, min_logfsbs) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->logblocks = mp->m_sb.sb_logblocks; + __entry->min_logfsbs = min_logfsbs; + ), + TP_printk("dev %d:%d logblocks %u min_logfsbs %u", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->logblocks, + __entry->min_logfsbs) +); +#endif /* CONFIG_XFS_RT */ + +TRACE_DEFINE_ENUM(XC_FREE_BLOCKS); +TRACE_DEFINE_ENUM(XC_FREE_RTEXTENTS); +TRACE_DEFINE_ENUM(XC_FREE_RTAVAILABLE); + +DECLARE_EVENT_CLASS(xfs_freeblocks_resv_class, + TP_PROTO(struct xfs_mount *mp, enum xfs_free_counter ctr, + uint64_t delta, unsigned long caller_ip), + TP_ARGS(mp, ctr, delta, caller_ip), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(enum xfs_free_counter, ctr) + __field(uint64_t, delta) + __field(uint64_t, avail) + __field(uint64_t, total) + __field(unsigned long, caller_ip) + ), + TP_fast_assign( + __entry->dev = mp->m_super->s_dev; + __entry->ctr = ctr; + __entry->delta = delta; + __entry->avail = mp->m_free[ctr].res_avail; + __entry->total = mp->m_free[ctr].res_total; + __entry->caller_ip = caller_ip; + ), + TP_printk("dev %d:%d ctr %s delta %llu avail %llu total %llu caller %pS", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->ctr, XFS_FREECOUNTER_STR), + __entry->delta, + __entry->avail, + __entry->total, + (char *)__entry->caller_ip) +) +#define DEFINE_FREEBLOCKS_RESV_EVENT(name) \ +DEFINE_EVENT(xfs_freeblocks_resv_class, name, \ + TP_PROTO(struct xfs_mount *mp, enum xfs_free_counter ctr, \ + uint64_t delta, unsigned long caller_ip), \ + TP_ARGS(mp, ctr, delta, caller_ip)) +DEFINE_FREEBLOCKS_RESV_EVENT(xfs_freecounter_reserved); +DEFINE_FREEBLOCKS_RESV_EVENT(xfs_freecounter_enospc); + +TRACE_EVENT(xfs_healthmon_lost_event, + TP_PROTO(const struct xfs_healthmon *hm), + TP_ARGS(hm), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned long long, lost_prev) + ), + TP_fast_assign( + __entry->dev = hm->dev; + __entry->lost_prev = hm->lost_prev_event; + ), + TP_printk("dev %d:%d lost_prev %llu", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->lost_prev) +); + +#define XFS_HEALTHMON_FLAGS_STRINGS \ + { XFS_HEALTH_MONITOR_VERBOSE, "verbose" } +#define XFS_HEALTHMON_FMT_STRINGS \ + { XFS_HEALTH_MONITOR_FMT_V0, "v0" } + +TRACE_EVENT(xfs_healthmon_create, + TP_PROTO(dev_t dev, u64 flags, u8 format), + TP_ARGS(dev, flags, format), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u64, flags) + __field(u8, format) + ), + TP_fast_assign( + __entry->dev = dev; + __entry->flags = flags; + __entry->format = format; + ), + TP_printk("dev %d:%d flags %s format %s", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_flags(__entry->flags, "|", XFS_HEALTHMON_FLAGS_STRINGS), + __print_symbolic(__entry->format, XFS_HEALTHMON_FMT_STRINGS)) +); + +TRACE_EVENT(xfs_healthmon_copybuf, + TP_PROTO(const struct xfs_healthmon *hm, const struct iov_iter *iov), + TP_ARGS(hm, iov), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(size_t, bufsize) + __field(size_t, inpos) + __field(size_t, outpos) + __field(size_t, to_copy) + __field(size_t, iter_count) + ), + TP_fast_assign( + __entry->dev = hm->dev; + __entry->bufsize = hm->bufsize; + __entry->inpos = hm->bufhead; + __entry->outpos = hm->buftail; + if (hm->bufhead > hm->buftail) + __entry->to_copy = hm->bufhead - hm->buftail; + else + __entry->to_copy = 0; + __entry->iter_count = iov_iter_count(iov); + ), + TP_printk("dev %d:%d bufsize %zu in_pos %zu out_pos %zu to_copy %zu iter_count %zu", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->bufsize, + __entry->inpos, + __entry->outpos, + __entry->to_copy, + __entry->iter_count) +); + +DECLARE_EVENT_CLASS(xfs_healthmon_class, + TP_PROTO(const struct xfs_healthmon *hm), + TP_ARGS(hm), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, events) + __field(unsigned long long, lost_prev) + ), + TP_fast_assign( + __entry->dev = hm->dev; + __entry->events = hm->events; + __entry->lost_prev = hm->lost_prev_event; + ), + TP_printk("dev %d:%d events %u lost_prev? %llu", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->events, + __entry->lost_prev) +); +#define DEFINE_HEALTHMON_EVENT(name) \ +DEFINE_EVENT(xfs_healthmon_class, name, \ + TP_PROTO(const struct xfs_healthmon *hm), \ + TP_ARGS(hm)) +DEFINE_HEALTHMON_EVENT(xfs_healthmon_read_start); +DEFINE_HEALTHMON_EVENT(xfs_healthmon_read_finish); +DEFINE_HEALTHMON_EVENT(xfs_healthmon_release); +DEFINE_HEALTHMON_EVENT(xfs_healthmon_detach); +DEFINE_HEALTHMON_EVENT(xfs_healthmon_report_unmount); + +#define XFS_HEALTHMON_TYPE_STRINGS \ + { XFS_HEALTHMON_LOST, "lost" }, \ + { XFS_HEALTHMON_UNMOUNT, "unmount" }, \ + { XFS_HEALTHMON_SICK, "sick" }, \ + { XFS_HEALTHMON_CORRUPT, "corrupt" }, \ + { XFS_HEALTHMON_HEALTHY, "healthy" }, \ + { XFS_HEALTHMON_SHUTDOWN, "shutdown" } + +#define XFS_HEALTHMON_DOMAIN_STRINGS \ + { XFS_HEALTHMON_MOUNT, "mount" }, \ + { XFS_HEALTHMON_FS, "fs" }, \ + { XFS_HEALTHMON_AG, "ag" }, \ + { XFS_HEALTHMON_INODE, "inode" }, \ + { XFS_HEALTHMON_RTGROUP, "rtgroup" } + +TRACE_DEFINE_ENUM(XFS_HEALTHMON_LOST); +TRACE_DEFINE_ENUM(XFS_HEALTHMON_SHUTDOWN); +TRACE_DEFINE_ENUM(XFS_HEALTHMON_UNMOUNT); +TRACE_DEFINE_ENUM(XFS_HEALTHMON_SICK); +TRACE_DEFINE_ENUM(XFS_HEALTHMON_CORRUPT); +TRACE_DEFINE_ENUM(XFS_HEALTHMON_HEALTHY); + +TRACE_DEFINE_ENUM(XFS_HEALTHMON_MOUNT); +TRACE_DEFINE_ENUM(XFS_HEALTHMON_FS); +TRACE_DEFINE_ENUM(XFS_HEALTHMON_AG); +TRACE_DEFINE_ENUM(XFS_HEALTHMON_INODE); +TRACE_DEFINE_ENUM(XFS_HEALTHMON_RTGROUP); + +DECLARE_EVENT_CLASS(xfs_healthmon_event_class, + TP_PROTO(const struct xfs_healthmon *hm, + const struct xfs_healthmon_event *event), + TP_ARGS(hm, event), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, type) + __field(unsigned int, domain) + __field(unsigned int, mask) + __field(unsigned long long, ino) + __field(unsigned int, gen) + __field(unsigned int, group) + __field(unsigned long long, offset) + __field(unsigned long long, length) + __field(unsigned long long, lostcount) + ), + TP_fast_assign( + __entry->dev = hm->dev; + __entry->type = event->type; + __entry->domain = event->domain; + __entry->mask = 0; + __entry->group = 0; + __entry->ino = 0; + __entry->gen = 0; + __entry->offset = 0; + __entry->length = 0; + __entry->lostcount = 0; + switch (__entry->domain) { + case XFS_HEALTHMON_MOUNT: + switch (__entry->type) { + case XFS_HEALTHMON_SHUTDOWN: + __entry->mask = event->flags; + break; + case XFS_HEALTHMON_LOST: + __entry->lostcount = event->lostcount; + break; + } + break; + case XFS_HEALTHMON_FS: + __entry->mask = event->fsmask; + break; + case XFS_HEALTHMON_AG: + case XFS_HEALTHMON_RTGROUP: + __entry->mask = event->grpmask; + __entry->group = event->group; + break; + case XFS_HEALTHMON_INODE: + __entry->mask = event->imask; + __entry->ino = event->ino; + __entry->gen = event->gen; + break; + case XFS_HEALTHMON_DATADEV: + case XFS_HEALTHMON_LOGDEV: + case XFS_HEALTHMON_RTDEV: + __entry->offset = event->daddr; + __entry->length = event->bbcount; + break; + case XFS_HEALTHMON_FILERANGE: + __entry->ino = event->fino; + __entry->gen = event->fgen; + __entry->offset = event->fpos; + __entry->length = event->flen; + break; + } + ), + TP_printk("dev %d:%d type %s domain %s mask 0x%x ino 0x%llx gen 0x%x offset 0x%llx len 0x%llx group 0x%x lost %llu", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XFS_HEALTHMON_TYPE_STRINGS), + __print_symbolic(__entry->domain, XFS_HEALTHMON_DOMAIN_STRINGS), + __entry->mask, + __entry->ino, + __entry->gen, + __entry->offset, + __entry->length, + __entry->group, + __entry->lostcount) +); +#define DEFINE_HEALTHMONEVENT_EVENT(name) \ +DEFINE_EVENT(xfs_healthmon_event_class, name, \ + TP_PROTO(const struct xfs_healthmon *hm, \ + const struct xfs_healthmon_event *event), \ + TP_ARGS(hm, event)) +DEFINE_HEALTHMONEVENT_EVENT(xfs_healthmon_insert); +DEFINE_HEALTHMONEVENT_EVENT(xfs_healthmon_push); +DEFINE_HEALTHMONEVENT_EVENT(xfs_healthmon_pop); +DEFINE_HEALTHMONEVENT_EVENT(xfs_healthmon_format); +DEFINE_HEALTHMONEVENT_EVENT(xfs_healthmon_format_overflow); +DEFINE_HEALTHMONEVENT_EVENT(xfs_healthmon_drop); +DEFINE_HEALTHMONEVENT_EVENT(xfs_healthmon_merge); + +TRACE_EVENT(xfs_healthmon_report_fs, + TP_PROTO(const struct xfs_healthmon *hm, + unsigned int old_mask, unsigned int new_mask, + const struct xfs_healthmon_event *event), + TP_ARGS(hm, old_mask, new_mask, event), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, type) + __field(unsigned int, domain) + __field(unsigned int, old_mask) + __field(unsigned int, new_mask) + __field(unsigned int, fsmask) + ), + TP_fast_assign( + __entry->dev = hm->dev; + __entry->type = event->type; + __entry->domain = event->domain; + __entry->old_mask = old_mask; + __entry->new_mask = new_mask; + __entry->fsmask = event->fsmask; + ), + TP_printk("dev %d:%d type %s domain %s oldmask 0x%x newmask 0x%x fsmask 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XFS_HEALTHMON_TYPE_STRINGS), + __print_symbolic(__entry->domain, XFS_HEALTHMON_DOMAIN_STRINGS), + __entry->old_mask, + __entry->new_mask, + __entry->fsmask) +); + +TRACE_EVENT(xfs_healthmon_report_group, + TP_PROTO(const struct xfs_healthmon *hm, + unsigned int old_mask, unsigned int new_mask, + const struct xfs_healthmon_event *event), + TP_ARGS(hm, old_mask, new_mask, event), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, type) + __field(unsigned int, domain) + __field(unsigned int, old_mask) + __field(unsigned int, new_mask) + __field(unsigned int, grpmask) + __field(unsigned int, group) + ), + TP_fast_assign( + __entry->dev = hm->dev; + __entry->type = event->type; + __entry->domain = event->domain; + __entry->old_mask = old_mask; + __entry->new_mask = new_mask; + __entry->grpmask = event->grpmask; + __entry->group = event->group; + ), + TP_printk("dev %d:%d type %s domain %s oldmask 0x%x newmask 0x%x grpmask 0x%x group 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XFS_HEALTHMON_TYPE_STRINGS), + __print_symbolic(__entry->domain, XFS_HEALTHMON_DOMAIN_STRINGS), + __entry->old_mask, + __entry->new_mask, + __entry->grpmask, + __entry->group) +); + +TRACE_EVENT(xfs_healthmon_report_inode, + TP_PROTO(const struct xfs_healthmon *hm, + unsigned int old_mask, unsigned int new_mask, + const struct xfs_healthmon_event *event), + TP_ARGS(hm, old_mask, new_mask, event), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, type) + __field(unsigned int, domain) + __field(unsigned int, old_mask) + __field(unsigned int, new_mask) + __field(unsigned int, imask) + __field(unsigned long long, ino) + __field(unsigned int, gen) + ), + TP_fast_assign( + __entry->dev = hm->dev; + __entry->type = event->type; + __entry->domain = event->domain; + __entry->old_mask = old_mask; + __entry->new_mask = new_mask; + __entry->imask = event->imask; + __entry->ino = event->ino; + __entry->gen = event->gen; + ), + TP_printk("dev %d:%d type %s domain %s oldmask 0x%x newmask 0x%x imask 0x%x ino 0x%llx gen 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->type, XFS_HEALTHMON_TYPE_STRINGS), + __print_symbolic(__entry->domain, XFS_HEALTHMON_DOMAIN_STRINGS), + __entry->old_mask, + __entry->new_mask, + __entry->imask, + __entry->ino, + __entry->gen) +); + +TRACE_EVENT(xfs_healthmon_report_shutdown, + TP_PROTO(const struct xfs_healthmon *hm, uint32_t shutdown_flags), + TP_ARGS(hm, shutdown_flags), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(uint32_t, shutdown_flags) + ), + TP_fast_assign( + __entry->dev = hm->dev; + __entry->shutdown_flags = shutdown_flags; + ), + TP_printk("dev %d:%d shutdown_flags %s", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_flags(__entry->shutdown_flags, "|", XFS_SHUTDOWN_STRINGS)) +); + +#define XFS_DEVICE_STRINGS \ + { XFS_DEV_DATA, "datadev" }, \ + { XFS_DEV_RT, "rtdev" }, \ + { XFS_DEV_LOG, "logdev" } + +TRACE_DEFINE_ENUM(XFS_DEV_DATA); +TRACE_DEFINE_ENUM(XFS_DEV_RT); +TRACE_DEFINE_ENUM(XFS_DEV_LOG); + +TRACE_EVENT(xfs_healthmon_report_media, + TP_PROTO(const struct xfs_healthmon *hm, enum xfs_device fdev, + const struct xfs_healthmon_event *event), + TP_ARGS(hm, fdev, event), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, error_dev) + __field(uint64_t, daddr) + __field(uint64_t, bbcount) + ), + TP_fast_assign( + __entry->dev = hm->dev; + __entry->error_dev = fdev; + __entry->daddr = event->daddr; + __entry->bbcount = event->bbcount; + ), + TP_printk("dev %d:%d %s daddr 0x%llx bbcount 0x%llx", + MAJOR(__entry->dev), MINOR(__entry->dev), + __print_symbolic(__entry->error_dev, XFS_DEVICE_STRINGS), + __entry->daddr, + __entry->bbcount) +); + +#define FS_ERROR_STRINGS \ + { FSERR_BUFFERED_READ, "buffered_read" }, \ + { FSERR_BUFFERED_WRITE, "buffered_write" }, \ + { FSERR_DIRECTIO_READ, "directio_read" }, \ + { FSERR_DIRECTIO_WRITE, "directio_write" }, \ + { FSERR_DATA_LOST, "data_lost" }, \ + { FSERR_METADATA, "metadata" } + +TRACE_DEFINE_ENUM(FSERR_BUFFERED_READ); +TRACE_DEFINE_ENUM(FSERR_BUFFERED_WRITE); +TRACE_DEFINE_ENUM(FSERR_DIRECTIO_READ); +TRACE_DEFINE_ENUM(FSERR_DIRECTIO_WRITE); +TRACE_DEFINE_ENUM(FSERR_DATA_LOST); +TRACE_DEFINE_ENUM(FSERR_METADATA); + +TRACE_EVENT(xfs_healthmon_report_file_ioerror, + TP_PROTO(const struct xfs_healthmon *hm, + const struct fserror_event *p), + TP_ARGS(hm, p), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, type) + __field(unsigned long long, ino) + __field(unsigned int, gen) + __field(long long, pos) + __field(unsigned long long, len) + __field(int, error) + ), + TP_fast_assign( + __entry->dev = hm->dev; + __entry->type = p->type; + __entry->ino = p->inode->i_ino; + __entry->gen = p->inode->i_generation; + __entry->pos = p->pos; + __entry->len = p->len; + __entry->error = p->error; + ), + TP_printk("dev %d:%d ino 0x%llx gen 0x%x op %s pos 0x%llx bytecount 0x%llx error %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->gen, + __print_symbolic(__entry->type, FS_ERROR_STRINGS), + __entry->pos, + __entry->len, + __entry->error) +); + +TRACE_EVENT(xfs_verify_media, + TP_PROTO(const struct xfs_mount *mp, const struct xfs_verify_media *me, + dev_t fdev, xfs_daddr_t daddr, uint64_t bbcount, + const struct folio *folio), + TP_ARGS(mp, me, fdev, daddr, bbcount, folio), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(dev_t, fdev) + __field(xfs_daddr_t, start_daddr) + __field(xfs_daddr_t, end_daddr) + __field(unsigned int, flags) + __field(xfs_daddr_t, daddr) + __field(uint64_t, bbcount) + __field(unsigned int, bufsize) + ), + TP_fast_assign( + __entry->dev = mp->m_ddev_targp->bt_dev; + __entry->fdev = fdev; + __entry->start_daddr = me->me_start_daddr; + __entry->end_daddr = me->me_end_daddr; + __entry->flags = me->me_flags; + __entry->daddr = daddr; + __entry->bbcount = bbcount; + __entry->bufsize = folio_size(folio); + ), + TP_printk("dev %d:%d fdev %d:%d start_daddr 0x%llx end_daddr 0x%llx flags 0x%x daddr 0x%llx bbcount 0x%llx bufsize 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + MAJOR(__entry->fdev), MINOR(__entry->fdev), + __entry->start_daddr, + __entry->end_daddr, + __entry->flags, + __entry->daddr, + __entry->bbcount, + __entry->bufsize) +); + +TRACE_EVENT(xfs_verify_media_end, + TP_PROTO(const struct xfs_mount *mp, const struct xfs_verify_media *me, + dev_t fdev), + TP_ARGS(mp, me, fdev), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(dev_t, fdev) + __field(xfs_daddr_t, start_daddr) + __field(xfs_daddr_t, end_daddr) + __field(int, ioerror) + ), + TP_fast_assign( + __entry->dev = mp->m_ddev_targp->bt_dev; + __entry->fdev = fdev; + __entry->start_daddr = me->me_start_daddr; + __entry->end_daddr = me->me_end_daddr; + __entry->ioerror = me->me_ioerror; + ), + TP_printk("dev %d:%d fdev %d:%d start_daddr 0x%llx end_daddr 0x%llx ioerror %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + MAJOR(__entry->fdev), MINOR(__entry->fdev), + __entry->start_daddr, + __entry->end_daddr, + __entry->ioerror) +); + +TRACE_EVENT(xfs_verify_media_error, + TP_PROTO(const struct xfs_mount *mp, const struct xfs_verify_media *me, + dev_t fdev, xfs_daddr_t daddr, uint64_t bbcount, + blk_status_t status), + TP_ARGS(mp, me, fdev, daddr, bbcount, status), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(dev_t, fdev) + __field(xfs_daddr_t, start_daddr) + __field(xfs_daddr_t, end_daddr) + __field(unsigned int, flags) + __field(xfs_daddr_t, daddr) + __field(uint64_t, bbcount) + __field(int, error) + ), + TP_fast_assign( + __entry->dev = mp->m_ddev_targp->bt_dev; + __entry->fdev = fdev; + __entry->start_daddr = me->me_start_daddr; + __entry->end_daddr = me->me_end_daddr; + __entry->flags = me->me_flags; + __entry->daddr = daddr; + __entry->bbcount = bbcount; + __entry->error = blk_status_to_errno(status); + ), + TP_printk("dev %d:%d fdev %d:%d start_daddr 0x%llx end_daddr 0x%llx flags 0x%x daddr 0x%llx bbcount 0x%llx error %d", + MAJOR(__entry->dev), MINOR(__entry->dev), + MAJOR(__entry->fdev), MINOR(__entry->fdev), + __entry->start_daddr, + __entry->end_daddr, + __entry->flags, + __entry->daddr, + __entry->bbcount, + __entry->error) +); + +TRACE_EVENT(xfs_bmap_replace_cow_mapping, + TP_PROTO(struct xfs_inode *ip, const struct xfs_bmbt_irec *got, + const struct xfs_bmbt_irec *rep), + TP_ARGS(ip, got, rep), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(xfs_ino_t, ino) + __field(xfs_fsblock_t, startblock) + __field(xfs_fileoff_t, startoff) + __field(xfs_filblks_t, blockcount) + __field(xfs_exntst_t, state) + __field(xfs_fileoff_t, new_startoff) + __field(xfs_fsblock_t, new_startblock) + __field(xfs_extlen_t, new_blockcount) + __field(xfs_exntst_t, new_state) + ), + TP_fast_assign( + __entry->dev = ip->i_mount->m_super->s_dev; + __entry->ino = I_INO(ip); + __entry->startoff = got->br_startoff; + __entry->startblock = got->br_startblock; + __entry->blockcount = got->br_blockcount; + __entry->state = got->br_state; + __entry->new_startoff = rep->br_startoff; + __entry->new_startblock = rep->br_startblock; + __entry->new_blockcount = rep->br_blockcount; + __entry->new_state = rep->br_state; + ), + TP_printk("dev %d:%d ino 0x%llx startoff 0x%llx startblock 0x%llx fsbcount 0x%llx state 0x%x new_startoff 0x%llx new_startblock 0x%llx new_fsbcount 0x%x new_state 0x%x", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->ino, + __entry->startoff, + __entry->startblock, + __entry->blockcount, + __entry->state, + __entry->new_startoff, + __entry->new_startblock, + __entry->new_blockcount, + __entry->new_state) +); + +#endif /* _TRACE_XFS_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE xfs_trace +#include diff --git a/fs/xfs/xfs_trans.c b/fs/xfs/xfs_trans.c new file mode 100644 index 000000000..1b36cf12d --- /dev/null +++ b/fs/xfs/xfs_trans.c @@ -0,0 +1,1444 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2003,2005 Silicon Graphics, Inc. + * Copyright (C) 2010 Red Hat, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_extent_busy.h" +#include "xfs_quota.h" +#include "xfs_trans.h" +#include "xfs_trans_priv.h" +#include "xfs_log.h" +#include "xfs_log_priv.h" +#include "xfs_trace.h" +#include "xfs_error.h" +#include "xfs_defer.h" +#include "xfs_inode.h" +#include "xfs_dquot_item.h" +#include "xfs_dquot.h" +#include "xfs_icache.h" +#include "xfs_rtbitmap.h" +#include "xfs_rtgroup.h" +#include "xfs_sb.h" + +struct kmem_cache *xfs_trans_cache; + +#if defined(CONFIG_TRACEPOINTS) +static void +xfs_trans_trace_reservations( + struct xfs_mount *mp) +{ + struct xfs_trans_res *res; + struct xfs_trans_res *end_res; + int i; + + res = (struct xfs_trans_res *)M_RES(mp); + end_res = (struct xfs_trans_res *)(M_RES(mp) + 1); + for (i = 0; res < end_res; i++, res++) + trace_xfs_trans_resv_calc(mp, i, res); +} +#else +# define xfs_trans_trace_reservations(mp) +#endif + +/* + * Initialize the precomputed transaction reservation values + * in the mount structure. + */ +void +xfs_trans_init( + struct xfs_mount *mp) +{ + xfs_trans_resv_calc(mp, M_RES(mp)); + xfs_trans_trace_reservations(mp); +} + +/* + * Free the transaction structure. If there is more clean up + * to do when the structure is freed, add it here. + */ +STATIC void +xfs_trans_free( + struct xfs_trans *tp) +{ + xfs_extent_busy_sort(&tp->t_busy); + xfs_extent_busy_clear(&tp->t_busy, false); + + trace_xfs_trans_free(tp, _RET_IP_); + xfs_trans_clear_context(tp); + if (!(tp->t_flags & XFS_TRANS_NO_WRITECOUNT)) + sb_end_intwrite(tp->t_mountp->m_super); + xfs_trans_free_dqinfo(tp); + kmem_cache_free(xfs_trans_cache, tp); +} + +/* + * This is called to create a new transaction which will share the + * permanent log reservation of the given transaction. The remaining + * unused block and rt extent reservations are also inherited. This + * implies that the original transaction is no longer allowed to allocate + * blocks. Locks and log items, however, are no inherited. They must + * be added to the new transaction explicitly. + */ +STATIC struct xfs_trans * +xfs_trans_dup( + struct xfs_trans *tp) +{ + struct xfs_trans *ntp; + + trace_xfs_trans_dup(tp, _RET_IP_); + + ntp = kmem_cache_zalloc(xfs_trans_cache, GFP_KERNEL | __GFP_NOFAIL); + + /* + * Initialize the new transaction structure. + */ + ntp->t_mountp = tp->t_mountp; + INIT_LIST_HEAD(&ntp->t_items); + INIT_LIST_HEAD(&ntp->t_busy); + INIT_LIST_HEAD(&ntp->t_dfops); + ntp->t_highest_agno = NULLAGNUMBER; + + ASSERT(tp->t_flags & XFS_TRANS_PERM_LOG_RES); + ASSERT(tp->t_ticket != NULL); + + ntp->t_flags = XFS_TRANS_PERM_LOG_RES | + (tp->t_flags & XFS_TRANS_RESERVE) | + (tp->t_flags & XFS_TRANS_NO_WRITECOUNT) | + (tp->t_flags & XFS_TRANS_RES_FDBLKS); + /* We gave our writer reference to the new transaction */ + tp->t_flags |= XFS_TRANS_NO_WRITECOUNT; + ntp->t_ticket = xfs_log_ticket_get(tp->t_ticket); + + ASSERT(tp->t_blk_res >= tp->t_blk_res_used); + ntp->t_blk_res = tp->t_blk_res - tp->t_blk_res_used; + tp->t_blk_res = tp->t_blk_res_used; + + ntp->t_rtx_res = tp->t_rtx_res - tp->t_rtx_res_used; + tp->t_rtx_res = tp->t_rtx_res_used; + + /* move deferred ops over to the new tp */ + xfs_defer_move(ntp, tp); + + xfs_trans_dup_dqinfo(tp, ntp); + return ntp; +} + +/* + * This is called to reserve free disk blocks and log space for the given + * transaction before allocating any resources within the transaction. + * + * This will return ENOSPC if there are not enough blocks available. + * It will sleep waiting for available log space. + * + * This does not do quota reservations. That typically is done by the caller + * afterwards. + */ +static int +xfs_trans_reserve( + struct xfs_trans *tp, + struct xfs_trans_res *resp, + uint blocks, + uint rtextents) +{ + struct xfs_mount *mp = tp->t_mountp; + int error = 0; + bool rsvd = (tp->t_flags & XFS_TRANS_RESERVE) != 0; + + ASSERT(resp->tr_logres > 0); + + /* + * Attempt to reserve the needed disk blocks by decrementing the number + * needed from the number available. This will fail if the count would + * go below zero. + */ + if (blocks > 0) { + error = xfs_dec_fdblocks(mp, blocks, rsvd); + if (error != 0) + return -ENOSPC; + tp->t_blk_res += blocks; + } + + /* + * Reserve the log space needed for this transaction. + */ + if (resp->tr_logflags & XFS_TRANS_PERM_LOG_RES) + tp->t_flags |= XFS_TRANS_PERM_LOG_RES; + error = xfs_log_reserve(mp, resp->tr_logres, resp->tr_logcount, + &tp->t_ticket, (tp->t_flags & XFS_TRANS_PERM_LOG_RES)); + if (error) + goto undo_blocks; + + tp->t_log_res = resp->tr_logres; + tp->t_log_count = resp->tr_logcount; + + /* + * Attempt to reserve the needed realtime extents by decrementing the + * number needed from the number available. This will fail if the + * count would go below zero. + */ + if (rtextents > 0) { + error = xfs_dec_frextents(mp, rtextents); + if (error) { + error = -ENOSPC; + goto undo_log; + } + tp->t_rtx_res += rtextents; + } + + return 0; + +undo_log: + xfs_log_ticket_ungrant(mp->m_log, tp->t_ticket); + tp->t_ticket = NULL; + tp->t_log_res = 0; + tp->t_flags &= ~XFS_TRANS_PERM_LOG_RES; +undo_blocks: + if (blocks > 0) { + xfs_add_fdblocks(mp, blocks); + tp->t_blk_res = 0; + } + return error; +} + +static struct xfs_trans * +__xfs_trans_alloc( + struct xfs_mount *mp, + uint flags) +{ + struct xfs_trans *tp; + + ASSERT(!(flags & XFS_TRANS_RES_FDBLKS) || xfs_has_lazysbcount(mp)); + + tp = kmem_cache_zalloc(xfs_trans_cache, GFP_KERNEL | __GFP_NOFAIL); + if (!(flags & XFS_TRANS_NO_WRITECOUNT)) + sb_start_intwrite(mp->m_super); + xfs_trans_set_context(tp); + tp->t_flags = flags; + tp->t_mountp = mp; + INIT_LIST_HEAD(&tp->t_items); + INIT_LIST_HEAD(&tp->t_busy); + INIT_LIST_HEAD(&tp->t_dfops); + tp->t_highest_agno = NULLAGNUMBER; + return tp; +} + +int +xfs_trans_alloc( + struct xfs_mount *mp, + struct xfs_trans_res *resp, + uint blocks, + uint rtextents, + uint flags, + struct xfs_trans **tpp) +{ + struct xfs_trans *tp; + bool want_retry = true; + int error; + + ASSERT(resp->tr_logres > 0); + + /* + * Allocate the handle before we do our freeze accounting and setting up + * GFP_NOFS allocation context so that we avoid lockdep false positives + * by doing GFP_KERNEL allocations inside sb_start_intwrite(). + */ +retry: + tp = __xfs_trans_alloc(mp, flags); + WARN_ON(mp->m_super->s_writers.frozen == SB_FREEZE_COMPLETE); + error = xfs_trans_reserve(tp, resp, blocks, rtextents); + if (error == -ENOSPC && want_retry) { + xfs_trans_cancel(tp); + + /* + * We weren't able to reserve enough space for the transaction. + * Flush the other speculative space allocations to free space. + * Do not perform a synchronous scan because callers can hold + * other locks. + */ + error = xfs_blockgc_flush_all(mp); + if (error) + return error; + want_retry = false; + goto retry; + } + if (error) { + xfs_trans_cancel(tp); + return error; + } + + trace_xfs_trans_alloc(tp, _RET_IP_); + + *tpp = tp; + return 0; +} + +/* + * Create an empty transaction with no reservation. This is a defensive + * mechanism for routines that query metadata without actually modifying them -- + * if the metadata being queried is somehow cross-linked (think a btree block + * pointer that points higher in the tree), we risk deadlock. However, blocks + * grabbed as part of a transaction can be re-grabbed. The verifiers will + * notice the corrupt block and the operation will fail back to userspace + * without deadlocking. + * + * Note the zero-length reservation; this transaction MUST be cancelled without + * any dirty data. + * + * Callers should obtain freeze protection to avoid a conflict with fs freezing + * where we can be grabbing buffers at the same time that freeze is trying to + * drain the buffer LRU list. + */ +struct xfs_trans * +xfs_trans_alloc_empty( + struct xfs_mount *mp) +{ + return __xfs_trans_alloc(mp, XFS_TRANS_NO_WRITECOUNT); +} + +/* + * Record the indicated change to the given field for application + * to the file system's superblock when the transaction commits. + * For now, just store the change in the transaction structure. + * + * Mark the transaction structure to indicate that the superblock + * needs to be updated before committing. + * + * Because we may not be keeping track of allocated/free inodes and + * used filesystem blocks in the superblock, we do not mark the + * superblock dirty in this transaction if we modify these fields. + * We still need to update the transaction deltas so that they get + * applied to the incore superblock, but we don't want them to + * cause the superblock to get locked and logged if these are the + * only fields in the superblock that the transaction modifies. + */ +void +xfs_trans_mod_sb( + xfs_trans_t *tp, + uint field, + int64_t delta) +{ + uint32_t flags = (XFS_TRANS_DIRTY|XFS_TRANS_SB_DIRTY); + xfs_mount_t *mp = tp->t_mountp; + + switch (field) { + case XFS_TRANS_SB_ICOUNT: + tp->t_icount_delta += delta; + if (xfs_has_lazysbcount(mp)) + flags &= ~XFS_TRANS_SB_DIRTY; + break; + case XFS_TRANS_SB_IFREE: + tp->t_ifree_delta += delta; + if (xfs_has_lazysbcount(mp)) + flags &= ~XFS_TRANS_SB_DIRTY; + break; + case XFS_TRANS_SB_FDBLOCKS: + /* + * Track the number of blocks allocated in the transaction. + * Make sure it does not exceed the number reserved. If so, + * shutdown as this can lead to accounting inconsistency. + */ + if (delta < 0) { + tp->t_blk_res_used += (uint)-delta; + if (tp->t_blk_res_used > tp->t_blk_res) + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); + } else if (delta > 0 && (tp->t_flags & XFS_TRANS_RES_FDBLKS)) { + int64_t blkres_delta; + + /* + * Return freed blocks directly to the reservation + * instead of the global pool, being careful not to + * overflow the trans counter. This is used to preserve + * reservation across chains of transaction rolls that + * repeatedly free and allocate blocks. + */ + blkres_delta = min_t(int64_t, delta, + UINT_MAX - tp->t_blk_res); + tp->t_blk_res += blkres_delta; + delta -= blkres_delta; + } + tp->t_fdblocks_delta += delta; + if (xfs_has_lazysbcount(mp)) + flags &= ~XFS_TRANS_SB_DIRTY; + break; + case XFS_TRANS_SB_RES_FDBLOCKS: + /* + * The allocation has already been applied to the + * in-core superblock's counter. This should only + * be applied to the on-disk superblock. + */ + tp->t_res_fdblocks_delta += delta; + if (xfs_has_lazysbcount(mp)) + flags &= ~XFS_TRANS_SB_DIRTY; + break; + case XFS_TRANS_SB_FREXTENTS: + /* + * Track the number of blocks allocated in the + * transaction. Make sure it does not exceed the + * number reserved. + */ + if (delta < 0) { + tp->t_rtx_res_used += (uint)-delta; + ASSERT(tp->t_rtx_res_used <= tp->t_rtx_res); + } + tp->t_frextents_delta += delta; + if (xfs_has_rtgroups(mp)) + flags &= ~XFS_TRANS_SB_DIRTY; + break; + case XFS_TRANS_SB_RES_FREXTENTS: + /* + * The allocation has already been applied to the + * in-core superblock's counter. This should only + * be applied to the on-disk superblock. + */ + ASSERT(delta < 0); + tp->t_res_frextents_delta += delta; + if (xfs_has_rtgroups(mp)) + flags &= ~XFS_TRANS_SB_DIRTY; + break; + case XFS_TRANS_SB_DBLOCKS: + tp->t_dblocks_delta += delta; + break; + case XFS_TRANS_SB_AGCOUNT: + ASSERT(delta > 0); + tp->t_agcount_delta += delta; + break; + case XFS_TRANS_SB_IMAXPCT: + tp->t_imaxpct_delta += delta; + break; + case XFS_TRANS_SB_REXTSIZE: + tp->t_rextsize_delta += delta; + break; + case XFS_TRANS_SB_RBMBLOCKS: + tp->t_rbmblocks_delta += delta; + break; + case XFS_TRANS_SB_RBLOCKS: + tp->t_rblocks_delta += delta; + break; + case XFS_TRANS_SB_REXTENTS: + tp->t_rextents_delta += delta; + break; + case XFS_TRANS_SB_REXTSLOG: + tp->t_rextslog_delta += delta; + break; + case XFS_TRANS_SB_RGCOUNT: + ASSERT(delta > 0); + tp->t_rgcount_delta += delta; + break; + default: + ASSERT(0); + return; + } + + tp->t_flags |= flags; +} + +/* + * xfs_trans_apply_sb_deltas() is called from the commit code + * to bring the superblock buffer into the current transaction + * and modify it as requested by earlier calls to xfs_trans_mod_sb(). + * + * For now we just look at each field allowed to change and change + * it if necessary. + */ +STATIC void +xfs_trans_apply_sb_deltas( + struct xfs_trans *tp) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_buf *bp = xfs_trans_getsb(tp); + struct xfs_dsb *sbp = bp->b_addr; + int whole = 0; + + /* + * Only update the superblock counters if we are logging them + */ + if (!xfs_has_lazysbcount(mp)) { + if (tp->t_icount_delta) + be64_add_cpu(&sbp->sb_icount, tp->t_icount_delta); + if (tp->t_ifree_delta) + be64_add_cpu(&sbp->sb_ifree, tp->t_ifree_delta); + if (tp->t_fdblocks_delta) + be64_add_cpu(&sbp->sb_fdblocks, tp->t_fdblocks_delta); + if (tp->t_res_fdblocks_delta) + be64_add_cpu(&sbp->sb_fdblocks, tp->t_res_fdblocks_delta); + } + + /* + * sb_frextents was added to the lazy sb counters when the rt groups + * feature was introduced. This is possible because we know that all + * kernels supporting rtgroups will also recompute frextents from the + * realtime bitmap. + * + * For older file systems, updating frextents requires careful handling + * because we cannot rely on log recovery in older kernels to recompute + * the value from the rtbitmap. This means that the ondisk frextents + * must be consistent with the rtbitmap. + * + * Therefore, log the frextents change to the ondisk superblock and + * update the incore superblock so that future calls to xfs_log_sb + * write the correct value ondisk. + */ + if ((tp->t_frextents_delta || tp->t_res_frextents_delta) && + !xfs_has_rtgroups(mp)) { + int64_t rtxdelta; + + rtxdelta = tp->t_frextents_delta + tp->t_res_frextents_delta; + + spin_lock(&mp->m_sb_lock); + be64_add_cpu(&sbp->sb_frextents, rtxdelta); + mp->m_sb.sb_frextents += rtxdelta; + spin_unlock(&mp->m_sb_lock); + } + + if (tp->t_dblocks_delta) { + be64_add_cpu(&sbp->sb_dblocks, tp->t_dblocks_delta); + mp->m_ddev_targp->bt_nr_sectors += + XFS_FSB_TO_BB(mp, tp->t_dblocks_delta); + whole = 1; + } + if (tp->t_agcount_delta) { + be32_add_cpu(&sbp->sb_agcount, tp->t_agcount_delta); + whole = 1; + } + if (tp->t_imaxpct_delta) { + sbp->sb_imax_pct += tp->t_imaxpct_delta; + whole = 1; + } + if (tp->t_rextsize_delta) { + be32_add_cpu(&sbp->sb_rextsize, tp->t_rextsize_delta); + + /* + * Because the ondisk sb records rtgroup size in units of rt + * extents, any time we update the rt extent size we have to + * recompute the ondisk rtgroup block log. The incore values + * will be recomputed in xfs_trans_unreserve_and_mod_sb. + */ + if (xfs_has_rtgroups(mp)) { + sbp->sb_rgblklog = xfs_compute_rgblklog( + be32_to_cpu(sbp->sb_rgextents), + be32_to_cpu(sbp->sb_rextsize)); + } + whole = 1; + } + if (tp->t_rbmblocks_delta) { + be32_add_cpu(&sbp->sb_rbmblocks, tp->t_rbmblocks_delta); + whole = 1; + } + if (tp->t_rblocks_delta) { + be64_add_cpu(&sbp->sb_rblocks, tp->t_rblocks_delta); + mp->m_rtdev_targp->bt_nr_sectors += + XFS_FSB_TO_BB(mp, tp->t_rblocks_delta); + whole = 1; + } + if (tp->t_rextents_delta) { + be64_add_cpu(&sbp->sb_rextents, tp->t_rextents_delta); + whole = 1; + } + if (tp->t_rextslog_delta) { + sbp->sb_rextslog += tp->t_rextslog_delta; + whole = 1; + } + if (tp->t_rgcount_delta) { + be32_add_cpu(&sbp->sb_rgcount, tp->t_rgcount_delta); + whole = 1; + } + + xfs_trans_buf_set_type(tp, bp, XFS_BLFT_SB_BUF); + if (whole) + /* + * Log the whole thing, the fields are noncontiguous. + */ + xfs_trans_log_buf(tp, bp, 0, sizeof(struct xfs_dsb) - 1); + else + /* + * Since all the modifiable fields are contiguous, we + * can get away with this. + */ + xfs_trans_log_buf(tp, bp, offsetof(struct xfs_dsb, sb_icount), + offsetof(struct xfs_dsb, sb_frextents) + + sizeof(sbp->sb_frextents) - 1); +} + +/* + * xfs_trans_unreserve_and_mod_sb() is called to release unused reservations and + * apply superblock counter changes to the in-core superblock. The + * t_res_fdblocks_delta and t_res_frextents_delta fields are explicitly NOT + * applied to the in-core superblock. The idea is that that has already been + * done. + * + * If we are not logging superblock counters, then the inode allocated/free and + * used block counts are not updated in the on disk superblock. In this case, + * XFS_TRANS_SB_DIRTY will not be set when the transaction is updated but we + * still need to update the incore superblock with the changes. + * + * Deltas for the inode count are +/-64, hence we use a large batch size of 128 + * so we don't need to take the counter lock on every update. + */ +#define XFS_ICOUNT_BATCH 128 + +void +xfs_trans_unreserve_and_mod_sb( + struct xfs_trans *tp) +{ + struct xfs_mount *mp = tp->t_mountp; + int64_t blkdelta = tp->t_blk_res; + int64_t rtxdelta = tp->t_rtx_res; + int64_t idelta = 0; + int64_t ifreedelta = 0; + + /* + * Calculate the deltas. + * + * t_fdblocks_delta and t_frextents_delta can be positive or negative: + * + * - positive values indicate blocks freed in the transaction. + * - negative values indicate blocks allocated in the transaction + * + * Negative values can only happen if the transaction has a block + * reservation that covers the allocated block. The end result is + * that the calculated delta values must always be positive and we + * can only put back previous allocated or reserved blocks here. + */ + ASSERT(tp->t_blk_res || tp->t_fdblocks_delta >= 0); + if (xfs_has_lazysbcount(mp) || (tp->t_flags & XFS_TRANS_SB_DIRTY)) { + blkdelta += tp->t_fdblocks_delta; + ASSERT(blkdelta >= 0); + } + + ASSERT(tp->t_rtx_res || tp->t_frextents_delta >= 0); + if (xfs_has_rtgroups(mp) || (tp->t_flags & XFS_TRANS_SB_DIRTY)) { + rtxdelta += tp->t_frextents_delta; + ASSERT(rtxdelta >= 0); + } + + if (xfs_has_lazysbcount(mp) || (tp->t_flags & XFS_TRANS_SB_DIRTY)) { + idelta = tp->t_icount_delta; + ifreedelta = tp->t_ifree_delta; + } + + /* apply the per-cpu counters */ + if (blkdelta) + xfs_add_fdblocks(mp, blkdelta); + + if (idelta) + percpu_counter_add_batch(&mp->m_icount, idelta, + XFS_ICOUNT_BATCH); + + if (ifreedelta) + percpu_counter_add(&mp->m_ifree, ifreedelta); + + if (rtxdelta) + xfs_add_frextents(mp, rtxdelta); + + if (!(tp->t_flags & XFS_TRANS_SB_DIRTY)) + return; + + /* apply remaining deltas */ + spin_lock(&mp->m_sb_lock); + mp->m_sb.sb_fdblocks += tp->t_fdblocks_delta + tp->t_res_fdblocks_delta; + mp->m_sb.sb_icount += idelta; + mp->m_sb.sb_ifree += ifreedelta; + /* + * Do not touch sb_frextents here because it is handled in + * xfs_trans_apply_sb_deltas for file systems where it isn't a lazy + * counter anyway. + */ + mp->m_sb.sb_dblocks += tp->t_dblocks_delta; + mp->m_sb.sb_agcount += tp->t_agcount_delta; + mp->m_sb.sb_imax_pct += tp->t_imaxpct_delta; + if (tp->t_rextsize_delta) + xfs_mount_sb_set_rextsize(mp, &mp->m_sb, + mp->m_sb.sb_rextsize + tp->t_rextsize_delta); + mp->m_sb.sb_rbmblocks += tp->t_rbmblocks_delta; + mp->m_sb.sb_rblocks += tp->t_rblocks_delta; + mp->m_sb.sb_rextents += tp->t_rextents_delta; + mp->m_sb.sb_rextslog += tp->t_rextslog_delta; + mp->m_sb.sb_rgcount += tp->t_rgcount_delta; + spin_unlock(&mp->m_sb_lock); + + /* + * Debug checks outside of the spinlock so they don't lock up the + * machine if they fail. + */ + ASSERT(mp->m_sb.sb_imax_pct >= 0); + ASSERT(mp->m_sb.sb_rextslog >= 0); +} + +/* Add the given log item to the transaction's list of log items. */ +void +xfs_trans_add_item( + struct xfs_trans *tp, + struct xfs_log_item *lip) +{ + ASSERT(lip->li_log == tp->t_mountp->m_log); + ASSERT(lip->li_ailp == tp->t_mountp->m_ail); + ASSERT(list_empty(&lip->li_trans)); + ASSERT(!test_bit(XFS_LI_DIRTY, &lip->li_flags)); + + list_add_tail(&lip->li_trans, &tp->t_items); + trace_xfs_trans_add_item(tp, _RET_IP_); +} + +/* + * Unlink the log item from the transaction. the log item is no longer + * considered dirty in this transaction, as the linked transaction has + * finished, either by abort or commit completion. + */ +void +xfs_trans_del_item( + struct xfs_log_item *lip) +{ + clear_bit(XFS_LI_DIRTY, &lip->li_flags); + list_del_init(&lip->li_trans); +} + +/* Detach and unlock all of the items in a transaction */ +static void +xfs_trans_free_items( + struct xfs_trans *tp, + bool abort) +{ + struct xfs_log_item *lip, *next; + + trace_xfs_trans_free_items(tp, _RET_IP_); + + list_for_each_entry_safe(lip, next, &tp->t_items, li_trans) { + xfs_trans_del_item(lip); + if (abort) { + trace_xfs_trans_free_abort(lip); + set_bit(XFS_LI_ABORTED, &lip->li_flags); + } + if (lip->li_ops->iop_release) + lip->li_ops->iop_release(lip); + } +} + +/* + * Sort transaction items prior to running precommit operations. This will + * attempt to order the items such that they will always be locked in the same + * order. Items that have no sort function are moved to the end of the list + * and so are locked last. + * + * This may need refinement as different types of objects add sort functions. + * + * Function is more complex than it needs to be because we are comparing 64 bit + * values and the function only returns 32 bit values. + */ +static int +xfs_trans_precommit_sort( + void *unused_arg, + const struct list_head *a, + const struct list_head *b) +{ + struct xfs_log_item *lia = container_of(a, + struct xfs_log_item, li_trans); + struct xfs_log_item *lib = container_of(b, + struct xfs_log_item, li_trans); + int64_t diff; + + /* + * If both items are non-sortable, leave them alone. If only one is + * sortable, move the non-sortable item towards the end of the list. + */ + if (!lia->li_ops->iop_sort && !lib->li_ops->iop_sort) + return 0; + if (!lia->li_ops->iop_sort) + return 1; + if (!lib->li_ops->iop_sort) + return -1; + + diff = lia->li_ops->iop_sort(lia) - lib->li_ops->iop_sort(lib); + if (diff < 0) + return -1; + if (diff > 0) + return 1; + return 0; +} + +/* + * Run transaction precommit functions. + * + * If there is an error in any of the callouts, then stop immediately and + * trigger a shutdown to abort the transaction. There is no recovery possible + * from errors at this point as the transaction is dirty.... + */ +static int +xfs_trans_run_precommits( + struct xfs_trans *tp) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_log_item *lip, *n; + int error = 0; + + /* + * Sort the item list to avoid ABBA deadlocks with other transactions + * running precommit operations that lock multiple shared items such as + * inode cluster buffers. + */ + list_sort(NULL, &tp->t_items, xfs_trans_precommit_sort); + + /* + * Precommit operations can remove the log item from the transaction + * if the log item exists purely to delay modifications until they + * can be ordered against other operations. Hence we have to use + * list_for_each_entry_safe() here. + */ + list_for_each_entry_safe(lip, n, &tp->t_items, li_trans) { + if (!test_bit(XFS_LI_DIRTY, &lip->li_flags)) + continue; + if (lip->li_ops->iop_precommit) { + error = lip->li_ops->iop_precommit(tp, lip); + if (error) + break; + } + } + if (error) + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); + return error; +} + +/* + * Commit the given transaction to the log. + * + * XFS disk error handling mechanism is not based on a typical + * transaction abort mechanism. Logically after the filesystem + * gets marked 'SHUTDOWN', we can't let any new transactions + * be durable - ie. committed to disk - because some metadata might + * be inconsistent. In such cases, this returns an error, and the + * caller may assume that all locked objects joined to the transaction + * have already been unlocked as if the commit had succeeded. + * Do not reference the transaction structure after this call. + */ +static int +__xfs_trans_commit( + struct xfs_trans *tp, + bool regrant) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xlog *log = mp->m_log; + xfs_csn_t commit_seq = 0; + int error = 0; + int sync = tp->t_flags & XFS_TRANS_SYNC; + + trace_xfs_trans_commit(tp, _RET_IP_); + + /* + * Commit per-transaction changes that are not already tracked through + * log items. This can add dirty log items to the transaction. + */ + if (tp->t_flags & XFS_TRANS_SB_DIRTY) + xfs_trans_apply_sb_deltas(tp); + xfs_trans_apply_dquot_deltas(tp); + + error = xfs_trans_run_precommits(tp); + if (error) + goto out_unreserve; + + /* + * If there is nothing to be logged by the transaction, + * then unlock all of the items associated with the + * transaction and free the transaction structure. + * Also make sure to return any reserved blocks to + * the free pool. + */ + if (!(tp->t_flags & XFS_TRANS_DIRTY)) + goto out_unreserve; + + /* + * We must check against log shutdown here because we cannot abort log + * items and leave them dirty, inconsistent and unpinned in memory while + * the log is active. This leaves them open to being written back to + * disk, and that will lead to on-disk corruption. + */ + if (xlog_is_shutdown(log)) { + error = -EIO; + goto out_unreserve; + } + + ASSERT(tp->t_ticket != NULL); + + xlog_cil_commit(log, tp, &commit_seq, regrant); + + xfs_trans_free(tp); + + /* + * If the transaction needs to be synchronous, then force the + * log out now and wait for it. + */ + if (sync) { + error = xfs_log_force_seq(mp, commit_seq, XFS_LOG_SYNC, NULL); + XFS_STATS_INC(mp, xs_trans_sync); + } else { + XFS_STATS_INC(mp, xs_trans_async); + } + + return error; + +out_unreserve: + xfs_trans_unreserve_and_mod_sb(tp); + + /* + * It is indeed possible for the transaction to be not dirty but + * the dqinfo portion to be. All that means is that we have some + * (non-persistent) quota reservations that need to be unreserved. + */ + xfs_trans_unreserve_and_mod_dquots(tp, true); + if (tp->t_ticket) { + if (regrant && !xlog_is_shutdown(log)) + xfs_log_ticket_regrant(log, tp->t_ticket); + else + xfs_log_ticket_ungrant(log, tp->t_ticket); + tp->t_ticket = NULL; + } + xfs_trans_free_items(tp, !!error); + xfs_trans_free(tp); + + XFS_STATS_INC(mp, xs_trans_empty); + return error; +} + +int +xfs_trans_commit( + struct xfs_trans *tp) +{ + /* + * Finish deferred items on final commit. Only permanent transactions + * should ever have deferred ops. + */ + WARN_ON_ONCE(!list_empty(&tp->t_dfops) && + !(tp->t_flags & XFS_TRANS_PERM_LOG_RES)); + if (tp->t_flags & XFS_TRANS_PERM_LOG_RES) { + int error = xfs_defer_finish_noroll(&tp); + if (error) { + xfs_trans_cancel(tp); + return error; + } + } + + return __xfs_trans_commit(tp, false); +} + +/* + * Unlock all of the transaction's items and free the transaction. If the + * transaction is dirty, we must shut down the filesystem because there is no + * way to restore them to their previous state. + * + * If the transaction has made a log reservation, make sure to release it as + * well. + * + * This is a high level function (equivalent to xfs_trans_commit()) and so can + * be called after the transaction has effectively been aborted due to the mount + * being shut down. However, if the mount has not been shut down and the + * transaction is dirty we will shut the mount down and, in doing so, that + * guarantees that the log is shut down, too. Hence we don't need to be as + * careful with shutdown state and dirty items here as we need to be in + * xfs_trans_commit(). + */ +void +xfs_trans_cancel( + struct xfs_trans *tp) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xlog *log = mp->m_log; + bool dirty = (tp->t_flags & XFS_TRANS_DIRTY); + + trace_xfs_trans_cancel(tp, _RET_IP_); + + /* + * It's never valid to cancel a transaction with deferred ops attached, + * because the transaction is effectively dirty. Complain about this + * loudly before freeing the in-memory defer items and shutting down the + * filesystem. + */ + if (!list_empty(&tp->t_dfops)) { + ASSERT(tp->t_flags & XFS_TRANS_PERM_LOG_RES); + dirty = true; + xfs_defer_cancel(tp); + } + + /* + * See if the caller is relying on us to shut down the filesystem. We + * only want an error report if there isn't already a shutdown in + * progress, so we only need to check against the mount shutdown state + * here. + */ + if (dirty && !xfs_is_shutdown(mp)) { + XFS_ERROR_REPORT("xfs_trans_cancel", XFS_ERRLEVEL_LOW, mp); + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); + } +#ifdef DEBUG + /* Log items need to be consistent until the log is shut down. */ + if (!dirty && !xlog_is_shutdown(log)) { + struct xfs_log_item *lip; + + list_for_each_entry(lip, &tp->t_items, li_trans) + ASSERT(!xlog_item_is_intent_done(lip)); + } +#endif + xfs_trans_unreserve_and_mod_sb(tp); + xfs_trans_unreserve_and_mod_dquots(tp, false); + + if (tp->t_ticket) { + xfs_log_ticket_ungrant(log, tp->t_ticket); + tp->t_ticket = NULL; + } + + xfs_trans_free_items(tp, dirty); + xfs_trans_free(tp); +} + +/* + * Roll from one trans in the sequence of PERMANENT transactions to the next: + * permanent transactions are only flushed out when committed with + * xfs_trans_commit(), but we still want as soon as possible to let chunks of it + * go to the log. So we commit the chunk we've been working on and get a new + * transaction to continue. + */ +int +xfs_trans_roll( + struct xfs_trans **tpp) +{ + struct xfs_trans *tp = *tpp; + unsigned int log_res = tp->t_log_res; + unsigned int log_count = tp->t_log_count; + int error; + + trace_xfs_trans_roll(tp, _RET_IP_); + + ASSERT(log_res > 0); + + /* + * Copy the critical parameters from one trans to the next. + */ + *tpp = xfs_trans_dup(tp); + + /* + * Commit the current transaction. + * + * If this commit failed, then it'd just unlock those items that are not + * marked ihold. That also means that a filesystem shutdown is in + * progress. The caller takes the responsibility to cancel the + * duplicate transaction that gets returned. + */ + error = __xfs_trans_commit(tp, true); + + tp = *tpp; + /* + * __xfs_trans_commit cleared the NOFS flag by calling into + * xfs_trans_free. Set it again here before doing memory + * allocations. + */ + xfs_trans_set_context(tp); + + if (error) + return error; + + /* + * Reserve space in the log for the next transaction. + * + * This also pushes items in the AIL out to disk if they are taking up + * space at the tail of the log that we want to use. This requires that + * either nothing be locked across this call, or that anything that is + * locked be logged in the prior and the next transactions. + */ + error = xfs_log_regrant(tp->t_mountp, tp->t_ticket); + if (error) + return error; + tp->t_log_res = log_res; + tp->t_log_count = log_count; + return 0; +} + +/* + * Allocate an transaction, lock and join the inode to it, and reserve quota. + * + * The caller must ensure that the on-disk dquots attached to this inode have + * already been allocated and initialized. The caller is responsible for + * releasing ILOCK_EXCL if a new transaction is returned. + */ +int +xfs_trans_alloc_inode( + struct xfs_inode *ip, + struct xfs_trans_res *resv, + unsigned int dblocks, + unsigned int rblocks, + bool force, + struct xfs_trans **tpp) +{ + struct xfs_trans *tp; + struct xfs_mount *mp = ip->i_mount; + bool retried = false; + int error; + +retry: + error = xfs_trans_alloc(mp, resv, dblocks, + xfs_extlen_to_rtxlen(mp, rblocks), + force ? XFS_TRANS_RESERVE : 0, &tp); + if (error) + return error; + + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, 0); + + error = xfs_qm_dqattach_locked(ip, false); + if (error) { + /* Caller should have allocated the dquots! */ + ASSERT(error != -ENOENT); + goto out_cancel; + } + + error = xfs_trans_reserve_quota_nblks(tp, ip, dblocks, rblocks, force); + if ((error == -EDQUOT || error == -ENOSPC) && !retried) { + xfs_trans_cancel(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + xfs_blockgc_free_quota(ip, 0); + retried = true; + goto retry; + } + if (error) + goto out_cancel; + + *tpp = tp; + return 0; + +out_cancel: + xfs_trans_cancel(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; +} + +/* + * Try to reserve more blocks for a transaction. + * + * This is for callers that need to attach resources to a transaction, scan + * those resources to determine the space reservation requirements, and then + * modify the attached resources. In other words, online repair. This can + * fail due to ENOSPC, so the caller must be able to cancel the transaction + * without shutting down the fs. + */ +int +xfs_trans_reserve_more( + struct xfs_trans *tp, + unsigned int blocks, + unsigned int rtextents) +{ + bool rsvd = tp->t_flags & XFS_TRANS_RESERVE; + + if (blocks && xfs_dec_fdblocks(tp->t_mountp, blocks, rsvd)) + return -ENOSPC; + if (rtextents && xfs_dec_frextents(tp->t_mountp, rtextents)) { + if (blocks) + xfs_add_fdblocks(tp->t_mountp, blocks); + return -ENOSPC; + } + tp->t_blk_res += blocks; + tp->t_rtx_res += rtextents; + return 0; +} + +/* + * Try to reserve more blocks and file quota for a transaction. Same + * conditions of usage as xfs_trans_reserve_more. + */ +int +xfs_trans_reserve_more_inode( + struct xfs_trans *tp, + struct xfs_inode *ip, + unsigned int dblocks, + unsigned int rblocks, + bool force_quota) +{ + struct xfs_mount *mp = ip->i_mount; + unsigned int rtx = xfs_extlen_to_rtxlen(mp, rblocks); + int error; + + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + + error = xfs_trans_reserve_more(tp, dblocks, rtx); + if (error) + return error; + + if (!XFS_IS_QUOTA_ON(mp) || xfs_is_quota_inode(&mp->m_sb, I_INO(ip))) + return 0; + + if (tp->t_flags & XFS_TRANS_RESERVE) + force_quota = true; + + error = xfs_trans_reserve_quota_nblks(tp, ip, dblocks, rblocks, + force_quota); + if (!error) + return 0; + + /* Quota failed, give back the new reservation. */ + xfs_add_fdblocks(mp, dblocks); + tp->t_blk_res -= dblocks; + xfs_add_frextents(mp, rtx); + tp->t_rtx_res -= rtx; + return error; +} + +/* + * Allocate an transaction in preparation for inode creation by reserving quota + * against the given dquots. Callers are not required to hold any inode locks. + */ +int +xfs_trans_alloc_icreate( + struct xfs_mount *mp, + struct xfs_trans_res *resv, + struct xfs_dquot *udqp, + struct xfs_dquot *gdqp, + struct xfs_dquot *pdqp, + unsigned int dblocks, + struct xfs_trans **tpp) +{ + struct xfs_trans *tp; + bool retried = false; + bool flushed = false; + int error; + +retry: + error = xfs_trans_alloc(mp, resv, dblocks, 0, 0, &tp); + if (error == -ENOSPC && !flushed) { + /* + * Flush all delalloc blocks to reclaim space from speculative + * preallocation. This is similar to the quota retry below + * but targets FS-wide ENOSPC. + */ + xfs_flush_inodes(mp); + flushed = true; + goto retry; + } + if (error) + return error; + + error = xfs_trans_reserve_quota_icreate(tp, udqp, gdqp, pdqp, dblocks); + if ((error == -EDQUOT || error == -ENOSPC) && !retried) { + xfs_trans_cancel(tp); + xfs_blockgc_free_dquots(mp, udqp, gdqp, pdqp, 0); + retried = true; + goto retry; + } + if (error) { + xfs_trans_cancel(tp); + return error; + } + + *tpp = tp; + return 0; +} + +/* + * Allocate an transaction, lock and join the inode to it, and reserve quota + * in preparation for inode attribute changes that include uid, gid, or prid + * changes. + * + * The caller must ensure that the on-disk dquots attached to this inode have + * already been allocated and initialized. The ILOCK will be dropped when the + * transaction is committed or cancelled. + */ +int +xfs_trans_alloc_ichange( + struct xfs_inode *ip, + struct xfs_dquot *new_udqp, + struct xfs_dquot *new_gdqp, + struct xfs_dquot *new_pdqp, + bool force, + struct xfs_trans **tpp) +{ + struct xfs_trans *tp; + struct xfs_mount *mp = ip->i_mount; + struct xfs_dquot *udqp; + struct xfs_dquot *gdqp; + struct xfs_dquot *pdqp; + bool retried = false; + int error; + +retry: + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_ichange, 0, 0, 0, &tp); + if (error) + return error; + + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL); + + if (xfs_is_metadir_inode(ip)) + goto out; + + error = xfs_qm_dqattach_locked(ip, false); + if (error) { + /* Caller should have allocated the dquots! */ + ASSERT(error != -ENOENT); + goto out_cancel; + } + + /* + * For each quota type, skip quota reservations if the inode's dquots + * now match the ones that came from the caller, or the caller didn't + * pass one in. The inode's dquots can change if we drop the ILOCK to + * perform a blockgc scan, so we must preserve the caller's arguments. + */ + udqp = (new_udqp != ip->i_udquot) ? new_udqp : NULL; + gdqp = (new_gdqp != ip->i_gdquot) ? new_gdqp : NULL; + pdqp = (new_pdqp != ip->i_pdquot) ? new_pdqp : NULL; + if (udqp || gdqp || pdqp) { + xfs_filblks_t dblocks, rblocks; + unsigned int qflags = XFS_QMOPT_RES_REGBLKS; + bool isrt = XFS_IS_REALTIME_INODE(ip); + + if (force) + qflags |= XFS_QMOPT_FORCE_RES; + + if (isrt) { + error = xfs_iread_extents(tp, ip, XFS_DATA_FORK); + if (error) + goto out_cancel; + } + + xfs_inode_count_blocks(tp, ip, &dblocks, &rblocks); + + if (isrt) + rblocks += ip->i_delayed_blks; + else + dblocks += ip->i_delayed_blks; + + /* + * Reserve enough quota to handle blocks on disk and reserved + * for a delayed allocation. We'll actually transfer the + * delalloc reservation between dquots at chown time, even + * though that part is only semi-transactional. + */ + error = xfs_trans_reserve_quota_bydquots(tp, mp, udqp, gdqp, + pdqp, dblocks, 1, qflags); + if ((error == -EDQUOT || error == -ENOSPC) && !retried) { + xfs_trans_cancel(tp); + xfs_blockgc_free_dquots(mp, udqp, gdqp, pdqp, 0); + retried = true; + goto retry; + } + if (error) + goto out_cancel; + + /* Do the same for realtime. */ + qflags = XFS_QMOPT_RES_RTBLKS | (qflags & XFS_QMOPT_FORCE_RES); + error = xfs_trans_reserve_quota_bydquots(tp, mp, udqp, gdqp, + pdqp, rblocks, 0, qflags); + if ((error == -EDQUOT || error == -ENOSPC) && !retried) { + xfs_trans_cancel(tp); + xfs_blockgc_free_dquots(mp, udqp, gdqp, pdqp, 0); + retried = true; + goto retry; + } + if (error) + goto out_cancel; + } + +out: + *tpp = tp; + return 0; + +out_cancel: + xfs_trans_cancel(tp); + return error; +} + +/* + * Allocate an transaction, lock and join the directory and child inodes to it, + * and reserve quota for a directory update. If there isn't sufficient space, + * @dblocks will be set to zero for a reservationless directory update and + * @nospace_error will be set to a negative errno describing the space + * constraint we hit. + * + * The caller must ensure that the on-disk dquots attached to this inode have + * already been allocated and initialized. The ILOCKs will be dropped when the + * transaction is committed or cancelled. + * + * Caller is responsible for unlocking the inodes manually upon return + */ +int +xfs_trans_alloc_dir( + struct xfs_inode *dp, + struct xfs_trans_res *resv, + struct xfs_inode *ip, + unsigned int *dblocks, + struct xfs_trans **tpp, + int *nospace_error) +{ + struct xfs_trans *tp; + struct xfs_mount *mp = ip->i_mount; + unsigned int resblks; + bool retried = false; + int error; + +retry: + *nospace_error = 0; + resblks = *dblocks; + error = xfs_trans_alloc(mp, resv, resblks, 0, 0, &tp); + if (error == -ENOSPC) { + *nospace_error = error; + resblks = 0; + error = xfs_trans_alloc(mp, resv, resblks, 0, 0, &tp); + } + if (error) + return error; + + xfs_lock_two_inodes(dp, XFS_ILOCK_EXCL, ip, XFS_ILOCK_EXCL); + + xfs_trans_ijoin(tp, dp, 0); + xfs_trans_ijoin(tp, ip, 0); + + error = xfs_qm_dqattach_locked(dp, false); + if (error) { + /* Caller should have allocated the dquots! */ + ASSERT(error != -ENOENT); + goto out_cancel; + } + + error = xfs_qm_dqattach_locked(ip, false); + if (error) { + /* Caller should have allocated the dquots! */ + ASSERT(error != -ENOENT); + goto out_cancel; + } + + if (resblks == 0) + goto done; + + error = xfs_trans_reserve_quota_nblks(tp, dp, resblks, 0, false); + if (error == -EDQUOT || error == -ENOSPC) { + if (!retried) { + xfs_trans_cancel(tp); + xfs_iunlock(dp, XFS_ILOCK_EXCL); + if (dp != ip) + xfs_iunlock(ip, XFS_ILOCK_EXCL); + xfs_blockgc_free_quota(dp, 0); + retried = true; + goto retry; + } + + *nospace_error = error; + resblks = 0; + error = 0; + } + if (error) + goto out_cancel; + +done: + *tpp = tp; + *dblocks = resblks; + return 0; + +out_cancel: + xfs_trans_cancel(tp); + xfs_iunlock(dp, XFS_ILOCK_EXCL); + if (dp != ip) + xfs_iunlock(ip, XFS_ILOCK_EXCL); + return error; +} diff --git a/fs/xfs/xfs_trans.h b/fs/xfs/xfs_trans.h new file mode 100644 index 000000000..eb83c5dac --- /dev/null +++ b/fs/xfs/xfs_trans.h @@ -0,0 +1,283 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_TRANS_H__ +#define __XFS_TRANS_H__ + +/* kernel only transaction subsystem defines */ + +struct xlog; +struct xlog_format_buf; +struct xfs_buf; +struct xfs_buftarg; +struct xfs_efd_log_item; +struct xfs_efi_log_item; +struct xfs_inode; +struct xfs_item_ops; +struct xfs_mount; +struct xfs_trans; +struct xfs_trans_res; +struct xfs_dquot_acct; +struct xfs_rud_log_item; +struct xfs_rui_log_item; +struct xfs_btree_cur; +struct xfs_cui_log_item; +struct xfs_cud_log_item; +struct xfs_bui_log_item; +struct xfs_bud_log_item; + +struct xfs_log_item { + struct list_head li_ail; /* AIL pointers */ + struct list_head li_trans; /* transaction list */ + xfs_lsn_t li_lsn; /* last on-disk lsn */ + struct xlog *li_log; + struct xfs_ail *li_ailp; /* ptr to AIL */ + uint li_type; /* item type */ + unsigned long li_flags; /* misc flags */ + struct xfs_buf *li_buf; /* real buffer pointer */ + struct list_head li_bio_list; /* buffer item list */ + const struct xfs_item_ops *li_ops; /* function list */ + + /* delayed logging */ + struct list_head li_cil; /* CIL pointers */ + struct xfs_log_vec *li_lv; /* active log vector */ + struct xfs_log_vec *li_lv_shadow; /* standby vector */ + xfs_csn_t li_seq; /* CIL commit seq */ + uint32_t li_order_id; /* CIL commit order */ +}; + +/* + * li_flags use the (set/test/clear)_bit atomic interfaces because updates can + * race with each other and we don't want to have to use the AIL lock to + * serialise all updates. + */ +#define XFS_LI_IN_AIL 0 +#define XFS_LI_ABORTED 1 +#define XFS_LI_FAILED 2 +#define XFS_LI_DIRTY 3 +#define XFS_LI_WHITEOUT 4 +#define XFS_LI_FLUSHING 5 + +#define XFS_LI_FLAGS \ + { (1u << XFS_LI_IN_AIL), "IN_AIL" }, \ + { (1u << XFS_LI_ABORTED), "ABORTED" }, \ + { (1u << XFS_LI_FAILED), "FAILED" }, \ + { (1u << XFS_LI_DIRTY), "DIRTY" }, \ + { (1u << XFS_LI_WHITEOUT), "WHITEOUT" }, \ + { (1u << XFS_LI_FLUSHING), "FLUSHING" } + +struct xfs_item_ops { + unsigned flags; + void (*iop_size)(struct xfs_log_item *, int *, int *); + void (*iop_format)(struct xfs_log_item *lip, + struct xlog_format_buf *lfb); + void (*iop_pin)(struct xfs_log_item *); + void (*iop_unpin)(struct xfs_log_item *, int remove); + uint64_t (*iop_sort)(struct xfs_log_item *lip); + int (*iop_precommit)(struct xfs_trans *tp, struct xfs_log_item *lip); + void (*iop_committing)(struct xfs_log_item *lip, xfs_csn_t seq); + xfs_lsn_t (*iop_committed)(struct xfs_log_item *, xfs_lsn_t); + uint (*iop_push)(struct xfs_log_item *, struct list_head *); + void (*iop_release)(struct xfs_log_item *); + bool (*iop_match)(struct xfs_log_item *item, uint64_t id); + struct xfs_log_item *(*iop_intent)(struct xfs_log_item *intent_done); +}; + +/* + * Log item ops flags + */ +/* + * Release the log item when the journal commits instead of inserting into the + * AIL for writeback tracking and/or log tail pinning. + */ +#define XFS_ITEM_RELEASE_WHEN_COMMITTED (1 << 0) +#define XFS_ITEM_INTENT (1 << 1) +#define XFS_ITEM_INTENT_DONE (1 << 2) + +static inline bool +xlog_item_is_intent(struct xfs_log_item *lip) +{ + return lip->li_ops->flags & XFS_ITEM_INTENT; +} + +static inline bool +xlog_item_is_intent_done(struct xfs_log_item *lip) +{ + return lip->li_ops->flags & XFS_ITEM_INTENT_DONE; +} + +void xfs_log_item_init(struct xfs_mount *mp, struct xfs_log_item *item, + int type, const struct xfs_item_ops *ops); + +/* + * Return values for the iop_push() routines. + */ +#define XFS_ITEM_SUCCESS 0 +#define XFS_ITEM_PINNED 1 +#define XFS_ITEM_LOCKED 2 +#define XFS_ITEM_FLUSHING 3 + +/* + * This is the structure maintained for every active transaction. + */ +typedef struct xfs_trans { + unsigned int t_log_res; /* amt of log space resvd */ + unsigned int t_log_count; /* count for perm log res */ + unsigned int t_blk_res; /* # of blocks resvd */ + unsigned int t_blk_res_used; /* # of resvd blocks used */ + unsigned int t_rtx_res; /* # of rt extents resvd */ + unsigned int t_rtx_res_used; /* # of resvd rt extents used */ + unsigned int t_flags; /* misc flags */ + xfs_agnumber_t t_highest_agno; /* highest AGF locked */ + struct xlog_ticket *t_ticket; /* log mgr ticket */ + struct xfs_mount *t_mountp; /* ptr to fs mount struct */ + struct xfs_dquot_acct *t_dqinfo; /* acctg info for dquots */ + int64_t t_icount_delta; /* superblock icount change */ + int64_t t_ifree_delta; /* superblock ifree change */ + int64_t t_fdblocks_delta; /* superblock fdblocks chg */ + int64_t t_res_fdblocks_delta; /* on-disk only chg */ + int64_t t_frextents_delta;/* superblock freextents chg*/ + int64_t t_res_frextents_delta; /* on-disk only chg */ + int64_t t_dblocks_delta;/* superblock dblocks change */ + int64_t t_agcount_delta;/* superblock agcount change */ + int64_t t_imaxpct_delta;/* superblock imaxpct change */ + int64_t t_rextsize_delta;/* superblock rextsize chg */ + int64_t t_rbmblocks_delta;/* superblock rbmblocks chg */ + int64_t t_rblocks_delta;/* superblock rblocks change */ + int64_t t_rextents_delta;/* superblocks rextents chg */ + int64_t t_rextslog_delta;/* superblocks rextslog chg */ + int64_t t_rgcount_delta; /* realtime group count */ + struct list_head t_items; /* log item descriptors */ + struct list_head t_busy; /* list of busy extents */ + struct list_head t_dfops; /* deferred operations */ + unsigned long t_pflags; /* saved process flags state */ +} xfs_trans_t; + +/* + * XFS transaction mechanism exported interfaces that are + * actually macros. + */ +#define xfs_trans_set_sync(tp) ((tp)->t_flags |= XFS_TRANS_SYNC) + +/* + * XFS transaction mechanism exported interfaces. + */ +int xfs_trans_alloc(struct xfs_mount *mp, struct xfs_trans_res *resp, + uint blocks, uint rtextents, uint flags, + struct xfs_trans **tpp); +int xfs_trans_reserve_more(struct xfs_trans *tp, + unsigned int blocks, unsigned int rtextents); +struct xfs_trans *xfs_trans_alloc_empty(struct xfs_mount *mp); +void xfs_trans_mod_sb(xfs_trans_t *, uint, int64_t); + +int xfs_trans_get_buf_map(struct xfs_trans *tp, struct xfs_buftarg *target, + struct xfs_buf_map *map, int nmaps, xfs_buf_flags_t flags, + struct xfs_buf **bpp); + +static inline int +xfs_trans_get_buf( + struct xfs_trans *tp, + struct xfs_buftarg *target, + xfs_daddr_t blkno, + int numblks, + xfs_buf_flags_t flags, + struct xfs_buf **bpp) +{ + DEFINE_SINGLE_BUF_MAP(map, blkno, numblks); + return xfs_trans_get_buf_map(tp, target, &map, 1, flags, bpp); +} + +int xfs_trans_read_buf_map(struct xfs_mount *mp, + struct xfs_trans *tp, + struct xfs_buftarg *target, + struct xfs_buf_map *map, int nmaps, + xfs_buf_flags_t flags, + struct xfs_buf **bpp, + const struct xfs_buf_ops *ops); + +static inline int +xfs_trans_read_buf( + struct xfs_mount *mp, + struct xfs_trans *tp, + struct xfs_buftarg *target, + xfs_daddr_t blkno, + int numblks, + xfs_buf_flags_t flags, + struct xfs_buf **bpp, + const struct xfs_buf_ops *ops) +{ + DEFINE_SINGLE_BUF_MAP(map, blkno, numblks); + return xfs_trans_read_buf_map(mp, tp, target, &map, 1, + flags, bpp, ops); +} + +struct xfs_buf *xfs_trans_getsb(struct xfs_trans *); +struct xfs_buf *xfs_trans_getrtsb(struct xfs_trans *tp); + +void xfs_trans_brelse(xfs_trans_t *, struct xfs_buf *); +void xfs_trans_bjoin(xfs_trans_t *, struct xfs_buf *); +void xfs_trans_bdetach(struct xfs_trans *tp, struct xfs_buf *bp); +void xfs_trans_bhold(xfs_trans_t *, struct xfs_buf *); +void xfs_trans_bhold_release(xfs_trans_t *, struct xfs_buf *); +void xfs_trans_binval(xfs_trans_t *, struct xfs_buf *); +void xfs_trans_inode_buf(xfs_trans_t *, struct xfs_buf *); +void xfs_trans_stale_inode_buf(xfs_trans_t *, struct xfs_buf *); +bool xfs_trans_ordered_buf(xfs_trans_t *, struct xfs_buf *); +void xfs_trans_dquot_buf(xfs_trans_t *, struct xfs_buf *, uint); +void xfs_trans_inode_alloc_buf(xfs_trans_t *, struct xfs_buf *); +void xfs_trans_ijoin(struct xfs_trans *, struct xfs_inode *, uint); +void xfs_trans_log_buf(struct xfs_trans *, struct xfs_buf *, uint, + uint); +void xfs_trans_dirty_buf(struct xfs_trans *, struct xfs_buf *); +bool xfs_trans_buf_is_dirty(struct xfs_buf *bp); +void xfs_trans_log_inode(xfs_trans_t *, struct xfs_inode *, uint); + +int xfs_trans_commit(struct xfs_trans *); +int xfs_trans_roll(struct xfs_trans **); +int xfs_trans_roll_inode(struct xfs_trans **, struct xfs_inode *); +void xfs_trans_cancel(xfs_trans_t *); +int xfs_trans_ail_init(struct xfs_mount *); +void xfs_trans_ail_destroy(struct xfs_mount *); + +void xfs_trans_buf_set_type(struct xfs_trans *, struct xfs_buf *, + enum xfs_blft); +void xfs_trans_buf_copy_type(struct xfs_buf *dst_bp, + struct xfs_buf *src_bp); + +extern struct kmem_cache *xfs_trans_cache; + +struct xfs_dquot; + +int xfs_trans_alloc_inode(struct xfs_inode *ip, struct xfs_trans_res *resv, + unsigned int dblocks, unsigned int rblocks, bool force, + struct xfs_trans **tpp); +int xfs_trans_reserve_more_inode(struct xfs_trans *tp, struct xfs_inode *ip, + unsigned int dblocks, unsigned int rblocks, bool force_quota); +int xfs_trans_alloc_icreate(struct xfs_mount *mp, struct xfs_trans_res *resv, + struct xfs_dquot *udqp, struct xfs_dquot *gdqp, + struct xfs_dquot *pdqp, unsigned int dblocks, + struct xfs_trans **tpp); +int xfs_trans_alloc_ichange(struct xfs_inode *ip, struct xfs_dquot *udqp, + struct xfs_dquot *gdqp, struct xfs_dquot *pdqp, bool force, + struct xfs_trans **tpp); +int xfs_trans_alloc_dir(struct xfs_inode *dp, struct xfs_trans_res *resv, + struct xfs_inode *ip, unsigned int *dblocks, + struct xfs_trans **tpp, int *nospace_error); + +static inline void +xfs_trans_set_context( + struct xfs_trans *tp) +{ + tp->t_pflags = memalloc_nofs_save(); +} + +static inline void +xfs_trans_clear_context( + struct xfs_trans *tp) +{ + memalloc_nofs_restore(tp->t_pflags); +} + +#endif /* __XFS_TRANS_H__ */ diff --git a/fs/xfs/xfs_trans_ail.c b/fs/xfs/xfs_trans_ail.c new file mode 100644 index 000000000..99a9bf376 --- /dev/null +++ b/fs/xfs/xfs_trans_ail.c @@ -0,0 +1,980 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc. + * Copyright (c) 2008 Dave Chinner + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_trans.h" +#include "xfs_trans_priv.h" +#include "xfs_trace.h" +#include "xfs_errortag.h" +#include "xfs_error.h" +#include "xfs_log.h" +#include "xfs_log_priv.h" + +#ifdef DEBUG +/* + * Check that the list is sorted as it should be. + * + * Called with the ail lock held, but we don't want to assert fail with it + * held otherwise we'll lock everything up and won't be able to debug the + * cause. Hence we sample and check the state under the AIL lock and return if + * everything is fine, otherwise we drop the lock and run the ASSERT checks. + * Asserts may not be fatal, so pick the lock back up and continue onwards. + */ +STATIC void +xfs_ail_check( + struct xfs_ail *ailp, + struct xfs_log_item *lip) + __must_hold(&ailp->ail_lock) +{ + struct xfs_log_item *prev_lip; + struct xfs_log_item *next_lip; + xfs_lsn_t prev_lsn = NULLCOMMITLSN; + xfs_lsn_t next_lsn = NULLCOMMITLSN; + xfs_lsn_t lsn; + bool in_ail; + + + if (list_empty(&ailp->ail_head)) + return; + + /* + * Sample then check the next and previous entries are valid. + */ + in_ail = test_bit(XFS_LI_IN_AIL, &lip->li_flags); + prev_lip = list_entry(lip->li_ail.prev, struct xfs_log_item, li_ail); + if (&prev_lip->li_ail != &ailp->ail_head) + prev_lsn = prev_lip->li_lsn; + next_lip = list_entry(lip->li_ail.next, struct xfs_log_item, li_ail); + if (&next_lip->li_ail != &ailp->ail_head) + next_lsn = next_lip->li_lsn; + lsn = lip->li_lsn; + + if (in_ail && + (prev_lsn == NULLCOMMITLSN || XFS_LSN_CMP(prev_lsn, lsn) <= 0) && + (next_lsn == NULLCOMMITLSN || XFS_LSN_CMP(next_lsn, lsn) >= 0)) + return; + + spin_unlock(&ailp->ail_lock); + ASSERT(in_ail); + ASSERT(prev_lsn == NULLCOMMITLSN || XFS_LSN_CMP(prev_lsn, lsn) <= 0); + ASSERT(next_lsn == NULLCOMMITLSN || XFS_LSN_CMP(next_lsn, lsn) >= 0); + spin_lock(&ailp->ail_lock); +} +#else /* !DEBUG */ +#define xfs_ail_check(a,l) +#endif /* DEBUG */ + +/* + * Return a pointer to the last item in the AIL. If the AIL is empty, then + * return NULL. + */ +static struct xfs_log_item * +xfs_ail_max( + struct xfs_ail *ailp) +{ + if (list_empty(&ailp->ail_head)) + return NULL; + + return list_entry(ailp->ail_head.prev, struct xfs_log_item, li_ail); +} + +/* + * Return a pointer to the item which follows the given item in the AIL. If + * the given item is the last item in the list, then return NULL. + */ +static struct xfs_log_item * +xfs_ail_next( + struct xfs_ail *ailp, + struct xfs_log_item *lip) +{ + if (lip->li_ail.next == &ailp->ail_head) + return NULL; + + return list_first_entry(&lip->li_ail, struct xfs_log_item, li_ail); +} + +/* + * This is called by the log manager code to determine the LSN of the tail of + * the log. This is exactly the LSN of the first item in the AIL. If the AIL + * is empty, then this function returns 0. + * + * We need the AIL lock in order to get a coherent read of the lsn of the last + * item in the AIL. + */ +static xfs_lsn_t +__xfs_ail_min_lsn( + struct xfs_ail *ailp) +{ + struct xfs_log_item *lip = xfs_ail_min(ailp); + + if (lip) + return lip->li_lsn; + return 0; +} + +xfs_lsn_t +xfs_ail_min_lsn( + struct xfs_ail *ailp) +{ + xfs_lsn_t lsn; + + spin_lock(&ailp->ail_lock); + lsn = __xfs_ail_min_lsn(ailp); + spin_unlock(&ailp->ail_lock); + + return lsn; +} + +/* + * The cursor keeps track of where our current traversal is up to by tracking + * the next item in the list for us. However, for this to be safe, removing an + * object from the AIL needs to invalidate any cursor that points to it. hence + * the traversal cursor needs to be linked to the struct xfs_ail so that + * deletion can search all the active cursors for invalidation. + */ +STATIC void +xfs_trans_ail_cursor_init( + struct xfs_ail *ailp, + struct xfs_ail_cursor *cur) +{ + cur->item = NULL; + list_add_tail(&cur->list, &ailp->ail_cursors); +} + +/* + * Get the next item in the traversal and advance the cursor. If the cursor + * was invalidated (indicated by a lip of 1), restart the traversal. + */ +struct xfs_log_item * +xfs_trans_ail_cursor_next( + struct xfs_ail *ailp, + struct xfs_ail_cursor *cur) +{ + struct xfs_log_item *lip = cur->item; + + if ((uintptr_t)lip & 1) + lip = xfs_ail_min(ailp); + if (lip) + cur->item = xfs_ail_next(ailp, lip); + return lip; +} + +/* + * When the traversal is complete, we need to remove the cursor from the list + * of traversing cursors. + */ +void +xfs_trans_ail_cursor_done( + struct xfs_ail_cursor *cur) +{ + cur->item = NULL; + list_del_init(&cur->list); +} + +/* + * Invalidate any cursor that is pointing to this item. This is called when an + * item is removed from the AIL. Any cursor pointing to this object is now + * invalid and the traversal needs to be terminated so it doesn't reference a + * freed object. We set the low bit of the cursor item pointer so we can + * distinguish between an invalidation and the end of the list when getting the + * next item from the cursor. + */ +STATIC void +xfs_trans_ail_cursor_clear( + struct xfs_ail *ailp, + struct xfs_log_item *lip) +{ + struct xfs_ail_cursor *cur; + + list_for_each_entry(cur, &ailp->ail_cursors, list) { + if (cur->item == lip) + cur->item = (struct xfs_log_item *) + ((uintptr_t)cur->item | 1); + } +} + +/* + * Find the first item in the AIL with the given @lsn by searching in ascending + * LSN order and initialise the cursor to point to the next item for a + * ascending traversal. Pass a @lsn of zero to initialise the cursor to the + * first item in the AIL. Returns NULL if the list is empty. + */ +struct xfs_log_item * +xfs_trans_ail_cursor_first( + struct xfs_ail *ailp, + struct xfs_ail_cursor *cur, + xfs_lsn_t lsn) +{ + struct xfs_log_item *lip; + + xfs_trans_ail_cursor_init(ailp, cur); + + if (lsn == 0) { + lip = xfs_ail_min(ailp); + goto out; + } + + list_for_each_entry(lip, &ailp->ail_head, li_ail) { + if (XFS_LSN_CMP(lip->li_lsn, lsn) >= 0) + goto out; + } + return NULL; + +out: + if (lip) + cur->item = xfs_ail_next(ailp, lip); + return lip; +} + +static struct xfs_log_item * +__xfs_trans_ail_cursor_last( + struct xfs_ail *ailp, + xfs_lsn_t lsn) +{ + struct xfs_log_item *lip; + + list_for_each_entry_reverse(lip, &ailp->ail_head, li_ail) { + if (XFS_LSN_CMP(lip->li_lsn, lsn) <= 0) + return lip; + } + return NULL; +} + +/* + * Find the last item in the AIL with the given @lsn by searching in descending + * LSN order and initialise the cursor to point to that item. If there is no + * item with the value of @lsn, then it sets the cursor to the last item with an + * LSN lower than @lsn. Returns NULL if the list is empty. + */ +struct xfs_log_item * +xfs_trans_ail_cursor_last( + struct xfs_ail *ailp, + struct xfs_ail_cursor *cur, + xfs_lsn_t lsn) +{ + xfs_trans_ail_cursor_init(ailp, cur); + cur->item = __xfs_trans_ail_cursor_last(ailp, lsn); + return cur->item; +} + +/* + * Splice the log item list into the AIL at the given LSN. We splice to the + * tail of the given LSN to maintain insert order for push traversals. The + * cursor is optional, allowing repeated updates to the same LSN to avoid + * repeated traversals. This should not be called with an empty list. + */ +static void +xfs_ail_splice( + struct xfs_ail *ailp, + struct xfs_ail_cursor *cur, + struct list_head *list, + xfs_lsn_t lsn) +{ + struct xfs_log_item *lip; + + ASSERT(!list_empty(list)); + + /* + * Use the cursor to determine the insertion point if one is + * provided. If not, or if the one we got is not valid, + * find the place in the AIL where the items belong. + */ + lip = cur ? cur->item : NULL; + if (!lip || (uintptr_t)lip & 1) + lip = __xfs_trans_ail_cursor_last(ailp, lsn); + + /* + * If a cursor is provided, we know we're processing the AIL + * in lsn order, and future items to be spliced in will + * follow the last one being inserted now. Update the + * cursor to point to that last item, now while we have a + * reliable pointer to it. + */ + if (cur) + cur->item = list_entry(list->prev, struct xfs_log_item, li_ail); + + /* + * Finally perform the splice. Unless the AIL was empty, + * lip points to the item in the AIL _after_ which the new + * items should go. If lip is null the AIL was empty, so + * the new items go at the head of the AIL. + */ + if (lip) + list_splice(list, &lip->li_ail); + else + list_splice(list, &ailp->ail_head); +} + +/* + * Delete the given item from the AIL. + */ +static void +xfs_ail_delete( + struct xfs_ail *ailp, + struct xfs_log_item *lip) +{ + xfs_ail_check(ailp, lip); + list_del(&lip->li_ail); + xfs_trans_ail_cursor_clear(ailp, lip); +} + +/* + * Requeue a failed buffer for writeback. + * + * We clear the log item failed state here as well, but we have to be careful + * about reference counts because the only active reference counts on the buffer + * may be the failed log items. Hence if we clear the log item failed state + * before queuing the buffer for IO we can release all active references to + * the buffer and free it, leading to use after free problems in + * xfs_buf_delwri_queue. It makes no difference to the buffer or log items which + * order we process them in - the buffer is locked, and we own the buffer list + * so nothing on them is going to change while we are performing this action. + * + * Hence we can safely queue the buffer for IO before we clear the failed log + * item state, therefore always having an active reference to the buffer and + * avoiding the transient zero-reference state that leads to use-after-free. + */ +static inline int +xfsaild_resubmit_item( + struct xfs_log_item *lip, + struct list_head *buffer_list) +{ + struct xfs_buf *bp = lip->li_buf; + + if (!xfs_buf_trylock(bp)) + return XFS_ITEM_LOCKED; + + if (!xfs_buf_delwri_queue(bp, buffer_list)) { + xfs_buf_unlock(bp); + return XFS_ITEM_FLUSHING; + } + + /* protected by ail_lock */ + list_for_each_entry(lip, &bp->b_li_list, li_bio_list) + clear_bit(XFS_LI_FAILED, &lip->li_flags); + xfs_buf_unlock(bp); + return XFS_ITEM_SUCCESS; +} + +/* + * Push a single log item from the AIL. + * + * @lip may have been released and freed by the time this function returns, + * so callers must not dereference the log item afterwards. + */ +static inline uint +xfsaild_push_item( + struct xfs_ail *ailp, + struct xfs_log_item *lip) +{ + /* + * If log item pinning is enabled, skip the push and track the item as + * pinned. This can help induce head-behind-tail conditions. + */ + if (XFS_TEST_ERROR(ailp->ail_log->l_mp, XFS_ERRTAG_LOG_ITEM_PIN)) + return XFS_ITEM_PINNED; + + /* + * Consider the item pinned if a push callback is not defined so the + * caller will force the log. This should only happen for intent items + * as they are unpinned once the associated done item is committed to + * the on-disk log. + */ + if (!lip->li_ops->iop_push) + return XFS_ITEM_PINNED; + if (test_bit(XFS_LI_FAILED, &lip->li_flags)) + return xfsaild_resubmit_item(lip, &ailp->ail_buf_list); + return lip->li_ops->iop_push(lip, &ailp->ail_buf_list); +} + +/* + * Compute the LSN that we'd need to push the log tail towards in order to have + * at least 25% of the log space free. If the log free space already meets this + * threshold, this function returns the lowest LSN in the AIL to slowly keep + * writeback ticking over and the tail of the log moving forward. + */ +static xfs_lsn_t +xfs_ail_calc_push_target( + struct xfs_ail *ailp) +{ + struct xlog *log = ailp->ail_log; + struct xfs_log_item *lip; + xfs_lsn_t target_lsn; + xfs_lsn_t max_lsn; + xfs_lsn_t min_lsn; + int32_t free_bytes; + uint32_t target_block; + uint32_t target_cycle; + + lockdep_assert_held(&ailp->ail_lock); + + lip = xfs_ail_max(ailp); + if (!lip) + return NULLCOMMITLSN; + + max_lsn = lip->li_lsn; + min_lsn = __xfs_ail_min_lsn(ailp); + + /* + * If we are supposed to push all the items in the AIL, we want to push + * to the current head. We then clear the push flag so that we don't + * keep pushing newly queued items beyond where the push all command was + * run. If the push waiter wants to empty the ail, it should queue + * itself on the ail_empty wait queue. + */ + if (test_and_clear_bit(XFS_AIL_OPSTATE_PUSH_ALL, &ailp->ail_opstate)) + return max_lsn; + + /* If someone wants the AIL empty, keep pushing everything we have. */ + if (waitqueue_active(&ailp->ail_empty)) + return max_lsn; + + /* + * Background pushing - attempt to keep 25% of the log free and if we + * have that much free retain the existing target. + */ + free_bytes = log->l_logsize - xlog_lsn_sub(log, max_lsn, min_lsn); + if (free_bytes >= log->l_logsize >> 2) + return ailp->ail_target; + + target_cycle = CYCLE_LSN(min_lsn); + target_block = BLOCK_LSN(min_lsn) + (log->l_logBBsize >> 2); + if (target_block >= log->l_logBBsize) { + target_block -= log->l_logBBsize; + target_cycle += 1; + } + target_lsn = xlog_assign_lsn(target_cycle, target_block); + + /* Cap the target to the highest LSN known to be in the AIL. */ + if (XFS_LSN_CMP(target_lsn, max_lsn) > 0) + return max_lsn; + + /* If the existing target is higher than the new target, keep it. */ + if (XFS_LSN_CMP(ailp->ail_target, target_lsn) >= 0) + return ailp->ail_target; + return target_lsn; +} + +static void +xfsaild_process_logitem( + struct xfs_ail *ailp, + struct xfs_log_item *lip, + int *stuck, + int *flushing) +{ + struct xfs_mount *mp = ailp->ail_log->l_mp; + uint type = lip->li_type; + unsigned long flags = lip->li_flags; + xfs_lsn_t item_lsn = lip->li_lsn; + int lock_result; + + /* + * Note that iop_push may unlock and reacquire the AIL lock. We + * rely on the AIL cursor implementation to be able to deal with + * the dropped lock. + * + * The log item may have been freed by the push, so it must not + * be accessed or dereferenced below this line. + */ + lock_result = xfsaild_push_item(ailp, lip); + switch (lock_result) { + case XFS_ITEM_SUCCESS: + XFS_STATS_INC(mp, xs_push_ail_success); + trace_xfs_ail_push(ailp, type, flags, item_lsn); + + ailp->ail_last_pushed_lsn = item_lsn; + break; + + case XFS_ITEM_FLUSHING: + /* + * The item or its backing buffer is already being + * flushed. The typical reason for that is that an + * inode buffer is locked because we already pushed the + * updates to it as part of inode clustering. + * + * We do not want to stop flushing just because lots + * of items are already being flushed, but we need to + * re-try the flushing relatively soon if most of the + * AIL is being flushed. + */ + XFS_STATS_INC(mp, xs_push_ail_flushing); + trace_xfs_ail_flushing(ailp, type, flags, item_lsn); + + (*flushing)++; + ailp->ail_last_pushed_lsn = item_lsn; + break; + + case XFS_ITEM_PINNED: + XFS_STATS_INC(mp, xs_push_ail_pinned); + trace_xfs_ail_pinned(ailp, type, flags, item_lsn); + + (*stuck)++; + ailp->ail_log_flush++; + break; + case XFS_ITEM_LOCKED: + XFS_STATS_INC(mp, xs_push_ail_locked); + trace_xfs_ail_locked(ailp, type, flags, item_lsn); + + (*stuck)++; + break; + default: + ASSERT(0); + break; + } +} + +static long +xfsaild_push( + struct xfs_ail *ailp) +{ + struct xfs_mount *mp = ailp->ail_log->l_mp; + struct xfs_ail_cursor cur; + struct xfs_log_item *lip; + xfs_lsn_t lsn; + long tout; + int stuck = 0; + int flushing = 0; + int count = 0; + + /* + * If we encountered pinned items or did not finish writing out all + * buffers the last time we ran, force a background CIL push to get the + * items unpinned in the near future. We do not wait on the CIL push as + * that could stall us for seconds if there is enough background IO + * load. Stalling for that long when the tail of the log is pinned and + * needs flushing will hard stop the transaction subsystem when log + * space runs out. + */ + if (ailp->ail_log_flush && ailp->ail_last_pushed_lsn == 0 && + (!list_empty_careful(&ailp->ail_buf_list) || + xfs_ail_min_lsn(ailp))) { + ailp->ail_log_flush = 0; + + XFS_STATS_INC(mp, xs_push_ail_flush); + xlog_cil_flush(ailp->ail_log); + } + + spin_lock(&ailp->ail_lock); + WRITE_ONCE(ailp->ail_target, xfs_ail_calc_push_target(ailp)); + if (ailp->ail_target == NULLCOMMITLSN) + goto out_done; + + /* we're done if the AIL is empty or our push has reached the end */ + lip = xfs_trans_ail_cursor_first(ailp, &cur, ailp->ail_last_pushed_lsn); + if (!lip) + goto out_done_cursor; + + XFS_STATS_INC(mp, xs_push_ail); + + ASSERT(ailp->ail_target != NULLCOMMITLSN); + + lsn = lip->li_lsn; + while ((XFS_LSN_CMP(lip->li_lsn, ailp->ail_target) <= 0)) { + + if (test_bit(XFS_LI_FLUSHING, &lip->li_flags)) + goto next_item; + + xfsaild_process_logitem(ailp, lip, &stuck, &flushing); + count++; + + /* + * Are there too many items we can't do anything with? + * + * If we are skipping too many items because we can't flush + * them or they are already being flushed, we back off and + * given them time to complete whatever operation is being + * done. i.e. remove pressure from the AIL while we can't make + * progress so traversals don't slow down further inserts and + * removals to/from the AIL. + * + * The value of 100 is an arbitrary magic number based on + * observation. + */ + if (stuck > 100) + break; + +next_item: + lip = xfs_trans_ail_cursor_next(ailp, &cur); + if (lip == NULL) + break; + if (lip->li_lsn != lsn && count > 1000) + break; + lsn = lip->li_lsn; + } + +out_done_cursor: + xfs_trans_ail_cursor_done(&cur); +out_done: + spin_unlock(&ailp->ail_lock); + + if (xfs_buf_delwri_submit_nowait(&ailp->ail_buf_list)) + ailp->ail_log_flush++; + + if (!count || XFS_LSN_CMP(lsn, ailp->ail_target) >= 0) { + /* + * We reached the target or the AIL is empty, so wait a bit + * longer for I/O to complete and remove pushed items from the + * AIL before we start the next scan from the start of the AIL. + */ + tout = 50; + ailp->ail_last_pushed_lsn = 0; + } else if (((stuck + flushing) * 100) / count > 90) { + /* + * Either there is a lot of contention on the AIL or we are + * stuck due to operations in progress. "Stuck" in this case + * is defined as >90% of the items we tried to push were stuck. + * + * Backoff a bit more to allow some I/O to complete before + * restarting from the start of the AIL. This prevents us from + * spinning on the same items, and if they are pinned will all + * the restart to issue a log force to unpin the stuck items. + */ + tout = 20; + ailp->ail_last_pushed_lsn = 0; + } else { + /* + * Assume we have more work to do in a short while. + */ + tout = 0; + } + + return tout; +} + +static int +xfsaild( + void *data) +{ + struct xfs_ail *ailp = data; + long tout = 0; /* milliseconds */ + unsigned int noreclaim_flag; + + noreclaim_flag = memalloc_noreclaim_save(); + set_freezable(); + + while (1) { + /* + * Long waits of 50ms or more occur when we've run out of items + * to push, so we only want uninterruptible state if we're + * actually blocked on something. + */ + if (tout && tout <= 20) + set_current_state(TASK_KILLABLE|TASK_FREEZABLE); + else + set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE); + + /* + * Check kthread_should_stop() after we set the task state to + * guarantee that we either see the stop bit and exit or the + * task state is reset to runnable such that it's not scheduled + * out indefinitely and detects the stop bit at next iteration. + * A memory barrier is included in above task state set to + * serialize again kthread_stop(). + */ + if (kthread_should_stop()) { + __set_current_state(TASK_RUNNING); + + /* + * The caller forces out the AIL before stopping the + * thread in the common case, which means the delwri + * queue is drained. In the shutdown case, the queue may + * still hold relogged buffers that haven't been + * submitted because they were pinned since added to the + * queue. + * + * Log I/O error processing stales the underlying buffer + * and clears the delwri state, expecting the buf to be + * removed on the next submission attempt. That won't + * happen if we're shutting down, so this is the last + * opportunity to release such buffers from the queue. + */ + ASSERT(list_empty(&ailp->ail_buf_list) || + xlog_is_shutdown(ailp->ail_log)); + xfs_buf_delwri_cancel(&ailp->ail_buf_list); + break; + } + + /* Idle if the AIL is empty. */ + spin_lock(&ailp->ail_lock); + if (!xfs_ail_min(ailp) && list_empty(&ailp->ail_buf_list)) { + spin_unlock(&ailp->ail_lock); + schedule(); + tout = 0; + continue; + } + spin_unlock(&ailp->ail_lock); + + if (tout) + schedule_timeout(msecs_to_jiffies(tout)); + + __set_current_state(TASK_RUNNING); + + try_to_freeze(); + + tout = xfsaild_push(ailp); + } + + memalloc_noreclaim_restore(noreclaim_flag); + return 0; +} + +/* + * Push out all items in the AIL immediately and wait until the AIL is empty. + */ +void +xfs_ail_push_all_sync( + struct xfs_ail *ailp) +{ + DEFINE_WAIT(wait); + + spin_lock(&ailp->ail_lock); + while (xfs_ail_max(ailp) != NULL) { + prepare_to_wait(&ailp->ail_empty, &wait, TASK_UNINTERRUPTIBLE); + wake_up_process(ailp->ail_task); + spin_unlock(&ailp->ail_lock); + schedule(); + spin_lock(&ailp->ail_lock); + } + spin_unlock(&ailp->ail_lock); + + finish_wait(&ailp->ail_empty, &wait); +} + +void +__xfs_ail_assign_tail_lsn( + struct xfs_ail *ailp) +{ + struct xlog *log = ailp->ail_log; + xfs_lsn_t tail_lsn; + + assert_spin_locked(&ailp->ail_lock); + + if (xlog_is_shutdown(log)) + return; + + tail_lsn = __xfs_ail_min_lsn(ailp); + if (!tail_lsn) + tail_lsn = ailp->ail_head_lsn; + + WRITE_ONCE(log->l_tail_space, + xlog_lsn_sub(log, ailp->ail_head_lsn, tail_lsn)); + trace_xfs_log_assign_tail_lsn(log, tail_lsn); + atomic64_set(&log->l_tail_lsn, tail_lsn); +} + +/* + * Callers should pass the original tail lsn so that we can detect if the tail + * has moved as a result of the operation that was performed. If the caller + * needs to force a tail space update, it should pass NULLCOMMITLSN to bypass + * the "did the tail LSN change?" checks. If the caller wants to avoid a tail + * update (e.g. it knows the tail did not change) it should pass an @old_lsn of + * 0. + */ +void +xfs_ail_update_finish( + struct xfs_ail *ailp, + xfs_lsn_t old_lsn) __releases(ailp->ail_lock) +{ + struct xlog *log = ailp->ail_log; + + /* If the tail lsn hasn't changed, don't do updates or wakeups. */ + if (!old_lsn || old_lsn == __xfs_ail_min_lsn(ailp)) { + spin_unlock(&ailp->ail_lock); + return; + } + + __xfs_ail_assign_tail_lsn(ailp); + if (list_empty(&ailp->ail_head)) + wake_up_all(&ailp->ail_empty); + spin_unlock(&ailp->ail_lock); + xfs_log_space_wake(log->l_mp); +} + +/* + * xfs_trans_ail_update_bulk - bulk AIL insertion operation. + * + * @xfs_trans_ail_update_bulk takes an array of log items that all need to be + * positioned at the same LSN in the AIL. If an item is not in the AIL, it will + * be added. Otherwise, it will be repositioned by removing it and re-adding + * it to the AIL. + * + * If we move the first item in the AIL, update the log tail to match the new + * minimum LSN in the AIL. + * + * This function should be called with the AIL lock held. + * + * To optimise the insert operation, we add all items to a temporary list, then + * splice this list into the correct position in the AIL. + * + * Items that are already in the AIL are first deleted from their current + * location before being added to the temporary list. + * + * This avoids needing to do an insert operation on every item. + * + * The AIL lock is dropped by xfs_ail_update_finish() before returning to + * the caller. + */ +void +xfs_trans_ail_update_bulk( + struct xfs_ail *ailp, + struct xfs_ail_cursor *cur, + struct xfs_log_item **log_items, + int nr_items, + xfs_lsn_t lsn) __releases(ailp->ail_lock) +{ + struct xfs_log_item *mlip; + xfs_lsn_t tail_lsn = 0; + int i; + LIST_HEAD(tmp); + + ASSERT(nr_items > 0); /* Not required, but true. */ + mlip = xfs_ail_min(ailp); + + for (i = 0; i < nr_items; i++) { + struct xfs_log_item *lip = log_items[i]; + if (test_and_set_bit(XFS_LI_IN_AIL, &lip->li_flags)) { + /* check if we really need to move the item */ + if (XFS_LSN_CMP(lsn, lip->li_lsn) <= 0) + continue; + + trace_xfs_ail_move(lip, lip->li_lsn, lsn); + if (mlip == lip && !tail_lsn) + tail_lsn = lip->li_lsn; + + xfs_ail_delete(ailp, lip); + } else { + trace_xfs_ail_insert(lip, 0, lsn); + } + lip->li_lsn = lsn; + list_add_tail(&lip->li_ail, &tmp); + } + + if (!list_empty(&tmp)) + xfs_ail_splice(ailp, cur, &tmp, lsn); + + /* + * If this is the first insert, wake up the push daemon so it can + * actively scan for items to push. We also need to do a log tail + * LSN update to ensure that it is correctly tracked by the log, so + * set the tail_lsn to NULLCOMMITLSN so that xfs_ail_update_finish() + * will see that the tail lsn has changed and will update the tail + * appropriately. + */ + if (!mlip) { + wake_up_process(ailp->ail_task); + tail_lsn = NULLCOMMITLSN; + } + + xfs_ail_update_finish(ailp, tail_lsn); +} + +/* Insert a log item into the AIL. */ +void +xfs_trans_ail_insert( + struct xfs_ail *ailp, + struct xfs_log_item *lip, + xfs_lsn_t lsn) +{ + spin_lock(&ailp->ail_lock); + xfs_trans_ail_update_bulk(ailp, NULL, &lip, 1, lsn); +} + +/* + * Delete one log item from the AIL. + * + * If this item was at the tail of the AIL, return the LSN of the log item so + * that we can use it to check if the LSN of the tail of the log has moved + * when finishing up the AIL delete process in xfs_ail_update_finish(). + */ +xfs_lsn_t +xfs_ail_delete_one( + struct xfs_ail *ailp, + struct xfs_log_item *lip) +{ + struct xfs_log_item *mlip = xfs_ail_min(ailp); + xfs_lsn_t lsn = lip->li_lsn; + + trace_xfs_ail_delete(lip, mlip->li_lsn, lip->li_lsn); + xfs_ail_delete(ailp, lip); + clear_bit(XFS_LI_IN_AIL, &lip->li_flags); + lip->li_lsn = 0; + + if (mlip == lip) + return lsn; + return 0; +} + +void +xfs_trans_ail_delete( + struct xfs_log_item *lip, + int shutdown_type) +{ + struct xfs_ail *ailp = lip->li_ailp; + struct xlog *log = ailp->ail_log; + xfs_lsn_t tail_lsn; + + spin_lock(&ailp->ail_lock); + if (!test_bit(XFS_LI_IN_AIL, &lip->li_flags)) { + spin_unlock(&ailp->ail_lock); + if (shutdown_type && !xlog_is_shutdown(log)) { + xfs_alert_tag(log->l_mp, XFS_PTAG_AILDELETE, + "%s: attempting to delete a log item that is not in the AIL", + __func__); + xlog_force_shutdown(log, shutdown_type); + } + return; + } + + clear_bit(XFS_LI_FAILED, &lip->li_flags); + tail_lsn = xfs_ail_delete_one(ailp, lip); + xfs_ail_update_finish(ailp, tail_lsn); /* drops the AIL lock */ +} + +int +xfs_trans_ail_init( + xfs_mount_t *mp) +{ + struct xfs_ail *ailp; + + ailp = kzalloc_obj(struct xfs_ail, GFP_KERNEL | __GFP_RETRY_MAYFAIL); + if (!ailp) + return -ENOMEM; + + ailp->ail_log = mp->m_log; + INIT_LIST_HEAD(&ailp->ail_head); + INIT_LIST_HEAD(&ailp->ail_cursors); + spin_lock_init(&ailp->ail_lock); + INIT_LIST_HEAD(&ailp->ail_buf_list); + init_waitqueue_head(&ailp->ail_empty); + + ailp->ail_task = kthread_run(xfsaild, ailp, "xfsaild/%s", + mp->m_super->s_id); + if (IS_ERR(ailp->ail_task)) + goto out_free_ailp; + + mp->m_ail = ailp; + return 0; + +out_free_ailp: + kfree(ailp); + return -ENOMEM; +} + +void +xfs_trans_ail_destroy( + xfs_mount_t *mp) +{ + struct xfs_ail *ailp = mp->m_ail; + + kthread_stop(ailp->ail_task); + kfree(ailp); +} diff --git a/fs/xfs/xfs_trans_buf.c b/fs/xfs/xfs_trans_buf.c new file mode 100644 index 000000000..1e0258488 --- /dev/null +++ b/fs/xfs/xfs_trans_buf.c @@ -0,0 +1,793 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_trans.h" +#include "xfs_buf_item.h" +#include "xfs_trans_priv.h" +#include "xfs_trace.h" + +/* + * Check to see if a buffer matching the given parameters is already + * a part of the given transaction. + */ +STATIC struct xfs_buf * +xfs_trans_buf_item_match( + struct xfs_trans *tp, + struct xfs_buftarg *target, + struct xfs_buf_map *map, + int nmaps) +{ + struct xfs_log_item *lip; + struct xfs_buf_log_item *blip; + int len = 0; + int i; + + for (i = 0; i < nmaps; i++) + len += map[i].bm_len; + + list_for_each_entry(lip, &tp->t_items, li_trans) { + blip = (struct xfs_buf_log_item *)lip; + if (blip->bli_item.li_type == XFS_LI_BUF && + blip->bli_buf->b_target == target && + xfs_buf_daddr(blip->bli_buf) == map[0].bm_bn && + blip->bli_buf->b_length == len) { + ASSERT(blip->bli_buf->b_map_count == nmaps); + return blip->bli_buf; + } + } + + return NULL; +} + +/* + * Add the locked buffer to the transaction. + * + * The buffer must be locked, and it cannot be associated with any + * transaction. + * + * If the buffer does not yet have a buf log item associated with it, + * then allocate one for it. Then add the buf item to the transaction. + */ +STATIC void +_xfs_trans_bjoin( + struct xfs_trans *tp, + struct xfs_buf *bp, + int reset_recur) +{ + struct xfs_buf_log_item *bip; + + ASSERT(bp->b_transp == NULL); + + /* + * The xfs_buf_log_item pointer is stored in b_log_item. If + * it doesn't have one yet, then allocate one and initialize it. + * The checks to see if one is there are in xfs_buf_item_init(). + */ + xfs_buf_item_init(bp, tp->t_mountp); + bip = bp->b_log_item; + ASSERT(!(bip->bli_flags & XFS_BLI_STALE)); + ASSERT(!(bip->__bli_format.blf_flags & XFS_BLF_CANCEL)); + ASSERT(!(bip->bli_flags & XFS_BLI_LOGGED)); + if (reset_recur) + bip->bli_recur = 0; + + /* + * Take a reference for this transaction on the buf item. + */ + atomic_inc(&bip->bli_refcount); + + /* + * Attach the item to the transaction so we can find it in + * xfs_trans_get_buf() and friends. + */ + xfs_trans_add_item(tp, &bip->bli_item); + bp->b_transp = tp; + +} + +void +xfs_trans_bjoin( + struct xfs_trans *tp, + struct xfs_buf *bp) +{ + _xfs_trans_bjoin(tp, bp, 0); + trace_xfs_trans_bjoin(bp->b_log_item); +} + +/* + * Get and lock the buffer for the caller if it is not already + * locked within the given transaction. If it is already locked + * within the transaction, just increment its lock recursion count + * and return a pointer to it. + * + * If the transaction pointer is NULL, make this just a normal + * get_buf() call. + */ +int +xfs_trans_get_buf_map( + struct xfs_trans *tp, + struct xfs_buftarg *target, + struct xfs_buf_map *map, + int nmaps, + xfs_buf_flags_t flags, + struct xfs_buf **bpp) +{ + struct xfs_buf *bp; + struct xfs_buf_log_item *bip; + int error; + + *bpp = NULL; + if (!tp) + return xfs_buf_get_map(target, map, nmaps, flags, bpp); + + /* + * If we find the buffer in the cache with this transaction + * pointer in its b_fsprivate2 field, then we know we already + * have it locked. In this case we just increment the lock + * recursion count and return the buffer to the caller. + */ + bp = xfs_trans_buf_item_match(tp, target, map, nmaps); + if (bp != NULL) { + ASSERT(xfs_buf_islocked(bp)); + if (xfs_is_shutdown(tp->t_mountp)) { + xfs_buf_stale(bp); + xfs_buf_set_uptodate(bp); + } + + ASSERT(bp->b_transp == tp); + bip = bp->b_log_item; + ASSERT(bip != NULL); + ASSERT(atomic_read(&bip->bli_refcount) > 0); + bip->bli_recur++; + trace_xfs_trans_get_buf_recur(bip); + *bpp = bp; + return 0; + } + + error = xfs_buf_get_map(target, map, nmaps, flags, &bp); + if (error) + return error; + + ASSERT(!bp->b_error); + + _xfs_trans_bjoin(tp, bp, 1); + trace_xfs_trans_get_buf(bp->b_log_item); + *bpp = bp; + return 0; +} + +/* + * Get and lock the superblock buffer for the given transaction. + */ +static struct xfs_buf * +__xfs_trans_getsb( + struct xfs_trans *tp, + struct xfs_buf *bp) +{ + /* + * Just increment the lock recursion count if the buffer is already + * attached to this transaction. + */ + if (bp->b_transp == tp) { + struct xfs_buf_log_item *bip = bp->b_log_item; + + ASSERT(bip != NULL); + ASSERT(atomic_read(&bip->bli_refcount) > 0); + bip->bli_recur++; + + trace_xfs_trans_getsb_recur(bip); + } else { + xfs_buf_lock(bp); + xfs_buf_hold(bp); + _xfs_trans_bjoin(tp, bp, 1); + + trace_xfs_trans_getsb(bp->b_log_item); + } + + return bp; +} + +struct xfs_buf * +xfs_trans_getsb( + struct xfs_trans *tp) +{ + return __xfs_trans_getsb(tp, tp->t_mountp->m_sb_bp); +} + +struct xfs_buf * +xfs_trans_getrtsb( + struct xfs_trans *tp) +{ + if (!tp->t_mountp->m_rtsb_bp) + return NULL; + return __xfs_trans_getsb(tp, tp->t_mountp->m_rtsb_bp); +} + +/* + * Get and lock the buffer for the caller if it is not already + * locked within the given transaction. If it has not yet been + * read in, read it from disk. If it is already locked + * within the transaction and already read in, just increment its + * lock recursion count and return a pointer to it. + * + * If the transaction pointer is NULL, make this just a normal + * read_buf() call. + */ +int +xfs_trans_read_buf_map( + struct xfs_mount *mp, + struct xfs_trans *tp, + struct xfs_buftarg *target, + struct xfs_buf_map *map, + int nmaps, + xfs_buf_flags_t flags, + struct xfs_buf **bpp, + const struct xfs_buf_ops *ops) +{ + struct xfs_buf *bp = NULL; + int error; + + *bpp = NULL; + /* + * If we find the buffer in the cache with this transaction + * pointer in its b_fsprivate2 field, then we know we already + * have it locked. If it is already read in we just increment + * the lock recursion count and return the buffer to the caller. + * If the buffer is not yet read in, then we read it in, increment + * the lock recursion count, and return it to the caller. + */ + if (tp) + bp = xfs_trans_buf_item_match(tp, target, map, nmaps); + if (bp) { + ASSERT(xfs_buf_islocked(bp)); + ASSERT(bp->b_transp == tp); + ASSERT(!bp->b_error); + ASSERT(bp->b_flags & XBF_DONE); + ASSERT(atomic_read(&bp->b_log_item->bli_refcount) > 0); + ASSERT(bp->b_ops); + + /* + * We never locked this buf ourselves, so we shouldn't + * brelse it either. Just get out. + */ + if (xfs_is_shutdown(mp)) { + trace_xfs_trans_read_buf_shut(bp, _RET_IP_); + return -EIO; + } + + bp->b_log_item->bli_recur++; + trace_xfs_trans_read_buf_recur(bp->b_log_item); + *bpp = bp; + return 0; + } + + error = xfs_buf_read_map(target, map, nmaps, flags, &bp, ops, + __return_address); + switch (error) { + case 0: + break; + default: + if (tp && (tp->t_flags & XFS_TRANS_DIRTY)) + xfs_force_shutdown(tp->t_mountp, SHUTDOWN_META_IO_ERROR); + fallthrough; + case -ENOMEM: + case -EAGAIN: + return error; + } + + if (xfs_is_shutdown(mp)) { + xfs_buf_relse(bp); + trace_xfs_trans_read_buf_shut(bp, _RET_IP_); + return -EIO; + } + + if (tp) { + _xfs_trans_bjoin(tp, bp, 1); + trace_xfs_trans_read_buf(bp->b_log_item); + } + ASSERT(bp->b_ops != NULL || ops == NULL); + *bpp = bp; + return 0; + +} + +/* Has this buffer been dirtied by anyone? */ +bool +xfs_trans_buf_is_dirty( + struct xfs_buf *bp) +{ + struct xfs_buf_log_item *bip = bp->b_log_item; + + if (!bip) + return false; + ASSERT(bip->bli_item.li_type == XFS_LI_BUF); + return test_bit(XFS_LI_DIRTY, &bip->bli_item.li_flags); +} + +/* + * Release a buffer previously joined to the transaction. If the buffer is + * modified within this transaction, decrement the recursion count but do not + * release the buffer even if the count goes to 0. If the buffer is not modified + * within the transaction, decrement the recursion count and release the buffer + * if the recursion count goes to 0. + * + * If the buffer is to be released and it was not already dirty before this + * transaction began, then also free the buf_log_item associated with it. + * + * If the transaction pointer is NULL, this is a normal xfs_buf_relse() call. + */ +void +xfs_trans_brelse( + struct xfs_trans *tp, + struct xfs_buf *bp) +{ + struct xfs_buf_log_item *bip = bp->b_log_item; + + ASSERT(bp->b_transp == tp); + + if (!tp) { + xfs_buf_relse(bp); + return; + } + + trace_xfs_trans_brelse(bip); + ASSERT(bip->bli_item.li_type == XFS_LI_BUF); + ASSERT(atomic_read(&bip->bli_refcount) > 0); + + /* + * If the release is for a recursive lookup, then decrement the count + * and return. + */ + if (bip->bli_recur > 0) { + bip->bli_recur--; + return; + } + + /* + * If the buffer is invalidated or dirty in this transaction, we can't + * release it until we commit. + */ + if (test_bit(XFS_LI_DIRTY, &bip->bli_item.li_flags)) + return; + if (bip->bli_flags & XFS_BLI_STALE) + return; + + /* + * Unlink the log item from the transaction and clear the hold flag, if + * set. We wouldn't want the next user of the buffer to get confused. + */ + ASSERT(!(bip->bli_flags & XFS_BLI_LOGGED)); + xfs_trans_del_item(&bip->bli_item); + bip->bli_flags &= ~XFS_BLI_HOLD; + + /* drop the reference to the bli */ + xfs_buf_item_put(bip); + + bp->b_transp = NULL; + xfs_buf_relse(bp); +} + +/* + * Forcibly detach a buffer previously joined to the transaction. The caller + * will retain its locked reference to the buffer after this function returns. + * The buffer must be completely clean and must not be held to the transaction. + */ +void +xfs_trans_bdetach( + struct xfs_trans *tp, + struct xfs_buf *bp) +{ + struct xfs_buf_log_item *bip = bp->b_log_item; + + ASSERT(tp != NULL); + ASSERT(bp->b_transp == tp); + ASSERT(bip->bli_item.li_type == XFS_LI_BUF); + ASSERT(atomic_read(&bip->bli_refcount) > 0); + + trace_xfs_trans_bdetach(bip); + + /* + * Erase all recursion count, since we're removing this buffer from the + * transaction. + */ + bip->bli_recur = 0; + + /* + * The buffer must be completely clean. Specifically, it had better + * not be dirty, stale, logged, ordered, or held to the transaction. + */ + ASSERT(!test_bit(XFS_LI_DIRTY, &bip->bli_item.li_flags)); + ASSERT(!(bip->bli_flags & XFS_BLI_DIRTY)); + ASSERT(!(bip->bli_flags & XFS_BLI_HOLD)); + ASSERT(!(bip->bli_flags & XFS_BLI_LOGGED)); + ASSERT(!(bip->bli_flags & XFS_BLI_ORDERED)); + ASSERT(!(bip->bli_flags & XFS_BLI_STALE)); + + /* Unlink the log item from the transaction and drop the log item. */ + xfs_trans_del_item(&bip->bli_item); + xfs_buf_item_put(bip); + bp->b_transp = NULL; +} + +/* + * Mark the buffer as not needing to be unlocked when the buf item's + * iop_committing() routine is called. The buffer must already be locked + * and associated with the given transaction. + */ +/* ARGSUSED */ +void +xfs_trans_bhold( + xfs_trans_t *tp, + struct xfs_buf *bp) +{ + struct xfs_buf_log_item *bip = bp->b_log_item; + + ASSERT(bp->b_transp == tp); + ASSERT(bip != NULL); + ASSERT(!(bip->bli_flags & XFS_BLI_STALE)); + ASSERT(!(bip->__bli_format.blf_flags & XFS_BLF_CANCEL)); + ASSERT(atomic_read(&bip->bli_refcount) > 0); + + bip->bli_flags |= XFS_BLI_HOLD; + trace_xfs_trans_bhold(bip); +} + +/* + * Cancel the previous buffer hold request made on this buffer + * for this transaction. + */ +void +xfs_trans_bhold_release( + xfs_trans_t *tp, + struct xfs_buf *bp) +{ + struct xfs_buf_log_item *bip = bp->b_log_item; + + ASSERT(bp->b_transp == tp); + ASSERT(bip != NULL); + ASSERT(!(bip->bli_flags & XFS_BLI_STALE)); + ASSERT(!(bip->__bli_format.blf_flags & XFS_BLF_CANCEL)); + ASSERT(atomic_read(&bip->bli_refcount) > 0); + ASSERT(bip->bli_flags & XFS_BLI_HOLD); + + bip->bli_flags &= ~XFS_BLI_HOLD; + trace_xfs_trans_bhold_release(bip); +} + +/* + * Mark a buffer dirty in the transaction. + */ +void +xfs_trans_dirty_buf( + struct xfs_trans *tp, + struct xfs_buf *bp) +{ + struct xfs_buf_log_item *bip = bp->b_log_item; + + ASSERT(bp->b_transp == tp); + ASSERT(bip != NULL); + + /* + * Mark the buffer as needing to be written out eventually, + * and set its iodone function to remove the buffer's buf log + * item from the AIL and free it when the buffer is flushed + * to disk. + */ + xfs_buf_set_uptodate(bp); + + ASSERT(atomic_read(&bip->bli_refcount) > 0); + + /* + * If we invalidated the buffer within this transaction, then + * cancel the invalidation now that we're dirtying the buffer + * again. There are no races with the code in xfs_buf_item_unpin(), + * because we have a reference to the buffer this entire time. + */ + if (bip->bli_flags & XFS_BLI_STALE) { + bip->bli_flags &= ~XFS_BLI_STALE; + xfs_buf_clear_stale(bp); + bip->__bli_format.blf_flags &= ~XFS_BLF_CANCEL; + } + bip->bli_flags |= XFS_BLI_DIRTY | XFS_BLI_LOGGED; + + tp->t_flags |= XFS_TRANS_DIRTY; + set_bit(XFS_LI_DIRTY, &bip->bli_item.li_flags); +} + +/* + * This is called to mark bytes first through last inclusive of the given + * buffer as needing to be logged when the transaction is committed. + * The buffer must already be associated with the given transaction. + * + * First and last are numbers relative to the beginning of this buffer, + * so the first byte in the buffer is numbered 0 regardless of the + * value of b_blkno. + */ +void +xfs_trans_log_buf( + struct xfs_trans *tp, + struct xfs_buf *bp, + uint first, + uint last) +{ + struct xfs_buf_log_item *bip = bp->b_log_item; + + ASSERT(first <= last && last < BBTOB(bp->b_length)); + ASSERT(!(bip->bli_flags & XFS_BLI_ORDERED)); + + xfs_trans_dirty_buf(tp, bp); + + trace_xfs_trans_log_buf(bip); + xfs_buf_item_log(bip, first, last); +} + + +/* + * Invalidate a buffer that is being used within a transaction. + * + * Typically this is because the blocks in the buffer are being freed, so we + * need to prevent it from being written out when we're done. Allowing it + * to be written again might overwrite data in the free blocks if they are + * reallocated to a file. + * + * We prevent the buffer from being written out by marking it stale. We can't + * get rid of the buf log item at this point because the buffer may still be + * pinned by another transaction. If that is the case, then we'll wait until + * the buffer is committed to disk for the last time (we can tell by the ref + * count) and free it in xfs_buf_item_unpin(). Until that happens we will + * keep the buffer locked so that the buffer and buf log item are not reused. + * + * We also set the XFS_BLF_CANCEL flag in the buf log format structure and log + * the buf item. This will be used at recovery time to determine that copies + * of the buffer in the log before this should not be replayed. + * + * We mark the item descriptor and the transaction dirty so that we'll hold + * the buffer until after the commit. + * + * Since we're invalidating the buffer, we also clear the state about which + * parts of the buffer have been logged. We also clear the flag indicating + * that this is an inode buffer since the data in the buffer will no longer + * be valid. + * + * We set the stale bit in the buffer as well since we're getting rid of it. + */ +void +xfs_trans_binval( + xfs_trans_t *tp, + struct xfs_buf *bp) +{ + struct xfs_buf_log_item *bip = bp->b_log_item; + int i; + + ASSERT(bp->b_transp == tp); + ASSERT(bip != NULL); + ASSERT(atomic_read(&bip->bli_refcount) > 0); + + trace_xfs_trans_binval(bip); + + if (bip->bli_flags & XFS_BLI_STALE) { + /* + * If the buffer is already invalidated, then + * just return. + */ + ASSERT(bp->b_flags & XBF_STALE); + ASSERT(!(bip->bli_flags & (XFS_BLI_LOGGED | XFS_BLI_DIRTY))); + ASSERT(!(bip->__bli_format.blf_flags & XFS_BLF_INODE_BUF)); + ASSERT(!(bip->__bli_format.blf_flags & XFS_BLFT_MASK)); + ASSERT(bip->__bli_format.blf_flags & XFS_BLF_CANCEL); + ASSERT(test_bit(XFS_LI_DIRTY, &bip->bli_item.li_flags)); + ASSERT(tp->t_flags & XFS_TRANS_DIRTY); + return; + } + + xfs_buf_stale(bp); + + bip->bli_flags |= XFS_BLI_STALE; + bip->bli_flags &= ~(XFS_BLI_INODE_BUF | XFS_BLI_LOGGED | XFS_BLI_DIRTY); + bip->__bli_format.blf_flags &= ~XFS_BLF_INODE_BUF; + bip->__bli_format.blf_flags |= XFS_BLF_CANCEL; + bip->__bli_format.blf_flags &= ~XFS_BLFT_MASK; + for (i = 0; i < bip->bli_format_count; i++) { + memset(bip->bli_formats[i].blf_data_map, 0, + (bip->bli_formats[i].blf_map_size * sizeof(uint))); + } + set_bit(XFS_LI_DIRTY, &bip->bli_item.li_flags); + tp->t_flags |= XFS_TRANS_DIRTY; +} + +/* + * This call is used to indicate that the buffer contains on-disk inodes which + * must be handled specially during recovery. They require special handling + * because only the di_next_unlinked from the inodes in the buffer should be + * recovered. The rest of the data in the buffer is logged via the inodes + * themselves. + * + * All we do is set the XFS_BLI_INODE_BUF flag in the items flags so it can be + * transferred to the buffer's log format structure so that we'll know what to + * do at recovery time. + */ +void +xfs_trans_inode_buf( + xfs_trans_t *tp, + struct xfs_buf *bp) +{ + struct xfs_buf_log_item *bip = bp->b_log_item; + + ASSERT(bp->b_transp == tp); + ASSERT(bip != NULL); + ASSERT(atomic_read(&bip->bli_refcount) > 0); + + bip->bli_flags |= XFS_BLI_INODE_BUF; + bp->b_iodone = xfs_buf_inode_iodone; + xfs_trans_buf_set_type(tp, bp, XFS_BLFT_DINO_BUF); +} + +/* + * This call is used to indicate that the buffer is going to + * be staled and was an inode buffer. This means it gets + * special processing during unpin - where any inodes + * associated with the buffer should be removed from ail. + * There is also special processing during recovery, + * any replay of the inodes in the buffer needs to be + * prevented as the buffer may have been reused. + */ +void +xfs_trans_stale_inode_buf( + xfs_trans_t *tp, + struct xfs_buf *bp) +{ + struct xfs_buf_log_item *bip = bp->b_log_item; + + ASSERT(bp->b_transp == tp); + ASSERT(bip != NULL); + ASSERT(atomic_read(&bip->bli_refcount) > 0); + + bip->bli_flags |= XFS_BLI_STALE_INODE; + bp->b_iodone = xfs_buf_inode_iodone; + xfs_trans_buf_set_type(tp, bp, XFS_BLFT_DINO_BUF); +} + +/* + * Mark the buffer as being one which contains newly allocated + * inodes. We need to make sure that even if this buffer is + * relogged as an 'inode buf' we still recover all of the inode + * images in the face of a crash. This works in coordination with + * xfs_buf_item_committed() to ensure that the buffer remains in the + * AIL at its original location even after it has been relogged. + */ +/* ARGSUSED */ +void +xfs_trans_inode_alloc_buf( + xfs_trans_t *tp, + struct xfs_buf *bp) +{ + struct xfs_buf_log_item *bip = bp->b_log_item; + + ASSERT(bp->b_transp == tp); + ASSERT(bip != NULL); + ASSERT(atomic_read(&bip->bli_refcount) > 0); + + bip->bli_flags |= XFS_BLI_INODE_ALLOC_BUF; + bp->b_iodone = xfs_buf_inode_iodone; + xfs_trans_buf_set_type(tp, bp, XFS_BLFT_DINO_BUF); +} + +/* + * Mark the buffer as ordered for this transaction. This means that the contents + * of the buffer are not recorded in the transaction but it is tracked in the + * AIL as though it was. This allows us to record logical changes in + * transactions rather than the physical changes we make to the buffer without + * changing writeback ordering constraints of metadata buffers. + */ +bool +xfs_trans_ordered_buf( + struct xfs_trans *tp, + struct xfs_buf *bp) +{ + struct xfs_buf_log_item *bip = bp->b_log_item; + + ASSERT(bp->b_transp == tp); + ASSERT(bip != NULL); + ASSERT(atomic_read(&bip->bli_refcount) > 0); + + if (xfs_buf_item_dirty_format(bip)) + return false; + + bip->bli_flags |= XFS_BLI_ORDERED; + trace_xfs_buf_item_ordered(bip); + + /* + * We don't log a dirty range of an ordered buffer but it still needs + * to be marked dirty and that it has been logged. + */ + xfs_trans_dirty_buf(tp, bp); + return true; +} + +/* + * Set the type of the buffer for log recovery so that it can correctly identify + * and hence attach the correct buffer ops to the buffer after replay. + */ +void +xfs_trans_buf_set_type( + struct xfs_trans *tp, + struct xfs_buf *bp, + enum xfs_blft type) +{ + struct xfs_buf_log_item *bip = bp->b_log_item; + + if (!tp) + return; + + ASSERT(bp->b_transp == tp); + ASSERT(bip != NULL); + ASSERT(atomic_read(&bip->bli_refcount) > 0); + + xfs_blft_to_flags(&bip->__bli_format, type); +} + +void +xfs_trans_buf_copy_type( + struct xfs_buf *dst_bp, + struct xfs_buf *src_bp) +{ + struct xfs_buf_log_item *sbip = src_bp->b_log_item; + struct xfs_buf_log_item *dbip = dst_bp->b_log_item; + enum xfs_blft type; + + type = xfs_blft_from_flags(&sbip->__bli_format); + xfs_blft_to_flags(&dbip->__bli_format, type); +} + +/* + * Similar to xfs_trans_inode_buf(), this marks the buffer as a cluster of + * dquots. However, unlike in inode buffer recovery, dquot buffers get + * recovered in their entirety. (Hence, no XFS_BLI_DQUOT_ALLOC_BUF flag). + * The only thing that makes dquot buffers different from regular + * buffers is that we must not replay dquot bufs when recovering + * if a _corresponding_ quotaoff has happened. We also have to distinguish + * between usr dquot bufs and grp dquot bufs, because usr and grp quotas + * can be turned off independently. + */ +/* ARGSUSED */ +void +xfs_trans_dquot_buf( + xfs_trans_t *tp, + struct xfs_buf *bp, + uint type) +{ + struct xfs_buf_log_item *bip = bp->b_log_item; + + ASSERT(type == XFS_BLF_UDQUOT_BUF || + type == XFS_BLF_PDQUOT_BUF || + type == XFS_BLF_GDQUOT_BUF); + + bip->__bli_format.blf_flags |= type; + + switch (type) { + case XFS_BLF_UDQUOT_BUF: + type = XFS_BLFT_UDQUOT_BUF; + break; + case XFS_BLF_PDQUOT_BUF: + type = XFS_BLFT_PDQUOT_BUF; + break; + case XFS_BLF_GDQUOT_BUF: + type = XFS_BLFT_GDQUOT_BUF; + break; + default: + type = XFS_BLFT_UNKNOWN_BUF; + break; + } + + bp->b_iodone = xfs_buf_dquot_iodone; + xfs_trans_buf_set_type(tp, bp, type); +} diff --git a/fs/xfs/xfs_trans_dquot.c b/fs/xfs/xfs_trans_dquot.c new file mode 100644 index 000000000..1606c614f --- /dev/null +++ b/fs/xfs/xfs_trans_dquot.c @@ -0,0 +1,1065 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2002 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#include "xfs_platform.h" +#include "xfs_fs.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_trans.h" +#include "xfs_trans_priv.h" +#include "xfs_quota.h" +#include "xfs_qm.h" +#include "xfs_trace.h" +#include "xfs_error.h" +#include "xfs_health.h" + +STATIC void xfs_trans_alloc_dqinfo(xfs_trans_t *); + +/* + * Add the locked dquot to the transaction. + * The dquot must be locked, and it cannot be associated with any + * transaction. + */ +void +xfs_trans_dqjoin( + struct xfs_trans *tp, + struct xfs_dquot *dqp) +{ + ASSERT(XFS_DQ_IS_LOCKED(dqp)); + ASSERT(dqp->q_logitem.qli_dquot == dqp); + + /* + * Get a log_item_desc to point at the new item. + */ + xfs_trans_add_item(tp, &dqp->q_logitem.qli_item); +} + +/* + * This is called to mark the dquot as needing + * to be logged when the transaction is committed. The dquot must + * already be associated with the given transaction. + * Note that it marks the entire transaction as dirty. In the ordinary + * case, this gets called via xfs_trans_commit, after the transaction + * is already dirty. However, there's nothing stop this from getting + * called directly, as done by xfs_qm_scall_setqlim. Hence, the TRANS_DIRTY + * flag. + */ +void +xfs_trans_log_dquot( + struct xfs_trans *tp, + struct xfs_dquot *dqp) +{ + ASSERT(XFS_DQ_IS_LOCKED(dqp)); + + /* Upgrade the dquot to bigtime format if possible. */ + if (dqp->q_id != 0 && + xfs_has_bigtime(tp->t_mountp) && + !(dqp->q_type & XFS_DQTYPE_BIGTIME)) + dqp->q_type |= XFS_DQTYPE_BIGTIME; + + tp->t_flags |= XFS_TRANS_DIRTY; + set_bit(XFS_LI_DIRTY, &dqp->q_logitem.qli_item.li_flags); +} + +/* + * Carry forward whatever is left of the quota blk reservation to + * the spanky new transaction + */ +void +xfs_trans_dup_dqinfo( + struct xfs_trans *otp, + struct xfs_trans *ntp) +{ + struct xfs_dqtrx *oq, *nq; + int i, j; + struct xfs_dqtrx *oqa, *nqa; + uint64_t blk_res_used; + + if (!otp->t_dqinfo) + return; + + xfs_trans_alloc_dqinfo(ntp); + + for (j = 0; j < XFS_QM_TRANS_DQTYPES; j++) { + oqa = otp->t_dqinfo->dqs[j]; + nqa = ntp->t_dqinfo->dqs[j]; + for (i = 0; i < XFS_QM_TRANS_MAXDQS; i++) { + blk_res_used = 0; + + if (oqa[i].qt_dquot == NULL) + break; + oq = &oqa[i]; + nq = &nqa[i]; + + if (oq->qt_blk_res && oq->qt_bcount_delta > 0) + blk_res_used = oq->qt_bcount_delta; + + nq->qt_dquot = oq->qt_dquot; + nq->qt_bcount_delta = nq->qt_icount_delta = 0; + nq->qt_rtbcount_delta = 0; + + /* + * Transfer whatever is left of the reservations. + */ + nq->qt_blk_res = oq->qt_blk_res - blk_res_used; + oq->qt_blk_res = blk_res_used; + + nq->qt_rtblk_res = oq->qt_rtblk_res - + oq->qt_rtblk_res_used; + oq->qt_rtblk_res = oq->qt_rtblk_res_used; + + nq->qt_ino_res = oq->qt_ino_res - oq->qt_ino_res_used; + oq->qt_ino_res = oq->qt_ino_res_used; + + } + } +} + +#ifdef CONFIG_XFS_LIVE_HOOKS +/* + * Use a static key here to reduce the overhead of quota live updates. If the + * compiler supports jump labels, the static branch will be replaced by a nop + * sled when there are no hook users. Online fsck is currently the only + * caller, so this is a reasonable tradeoff. + * + * Note: Patching the kernel code requires taking the cpu hotplug lock. Other + * parts of the kernel allocate memory with that lock held, which means that + * XFS callers cannot hold any locks that might be used by memory reclaim or + * writeback when calling the static_branch_{inc,dec} functions. + */ +DEFINE_STATIC_XFS_HOOK_SWITCH(xfs_dqtrx_hooks_switch); + +void +xfs_dqtrx_hook_disable(void) +{ + xfs_hooks_switch_off(&xfs_dqtrx_hooks_switch); +} + +void +xfs_dqtrx_hook_enable(void) +{ + xfs_hooks_switch_on(&xfs_dqtrx_hooks_switch); +} + +/* Schedule a transactional dquot update on behalf of an inode. */ +void +xfs_trans_mod_ino_dquot( + struct xfs_trans *tp, + struct xfs_inode *ip, + struct xfs_dquot *dqp, + unsigned int field, + int64_t delta) +{ + if (xfs_is_metadir_inode(ip)) + return; + + xfs_trans_mod_dquot(tp, dqp, field, delta); + + if (xfs_hooks_switched_on(&xfs_dqtrx_hooks_switch)) { + struct xfs_mod_ino_dqtrx_params p = { + .tx_id = (uintptr_t)tp, + .ino = I_INO(ip), + .q_type = xfs_dquot_type(dqp), + .q_id = dqp->q_id, + .delta = delta + }; + struct xfs_quotainfo *qi = tp->t_mountp->m_quotainfo; + + xfs_hooks_call(&qi->qi_mod_ino_dqtrx_hooks, field, &p); + } +} + +/* Call the specified functions during a dquot counter update. */ +int +xfs_dqtrx_hook_add( + struct xfs_quotainfo *qi, + struct xfs_dqtrx_hook *hook) +{ + int error; + + /* + * Transactional dquot updates first call the mod hook when changes + * are attached to the transaction and then call the apply hook when + * those changes are committed (or canceled). + * + * The apply hook must be installed before the mod hook so that we + * never fail to catch the end of a quota update sequence. + */ + error = xfs_hooks_add(&qi->qi_apply_dqtrx_hooks, &hook->apply_hook); + if (error) + goto out; + + error = xfs_hooks_add(&qi->qi_mod_ino_dqtrx_hooks, &hook->mod_hook); + if (error) + goto out_apply; + + return 0; + +out_apply: + xfs_hooks_del(&qi->qi_apply_dqtrx_hooks, &hook->apply_hook); +out: + return error; +} + +/* Stop calling the specified function during a dquot counter update. */ +void +xfs_dqtrx_hook_del( + struct xfs_quotainfo *qi, + struct xfs_dqtrx_hook *hook) +{ + /* + * The mod hook must be removed before apply hook to avoid giving the + * hook consumer with an incomplete update. No hooks should be running + * after these functions return. + */ + xfs_hooks_del(&qi->qi_mod_ino_dqtrx_hooks, &hook->mod_hook); + xfs_hooks_del(&qi->qi_apply_dqtrx_hooks, &hook->apply_hook); +} + +/* Configure dquot update hook functions. */ +void +xfs_dqtrx_hook_setup( + struct xfs_dqtrx_hook *hook, + notifier_fn_t mod_fn, + notifier_fn_t apply_fn) +{ + xfs_hook_setup(&hook->mod_hook, mod_fn); + xfs_hook_setup(&hook->apply_hook, apply_fn); +} +#endif /* CONFIG_XFS_LIVE_HOOKS */ + +/* + * Wrap around mod_dquot to account for both user and group quotas. + */ +void +xfs_trans_mod_dquot_byino( + xfs_trans_t *tp, + xfs_inode_t *ip, + uint field, + int64_t delta) +{ + xfs_mount_t *mp = tp->t_mountp; + + if (!XFS_IS_QUOTA_ON(mp) || + xfs_is_quota_inode(&mp->m_sb, I_INO(ip)) || + xfs_is_metadir_inode(ip)) + return; + + if (XFS_IS_UQUOTA_ON(mp) && ip->i_udquot) + xfs_trans_mod_ino_dquot(tp, ip, ip->i_udquot, field, delta); + if (XFS_IS_GQUOTA_ON(mp) && ip->i_gdquot) + xfs_trans_mod_ino_dquot(tp, ip, ip->i_gdquot, field, delta); + if (XFS_IS_PQUOTA_ON(mp) && ip->i_pdquot) + xfs_trans_mod_ino_dquot(tp, ip, ip->i_pdquot, field, delta); +} + +STATIC struct xfs_dqtrx * +xfs_trans_get_dqtrx( + struct xfs_trans *tp, + struct xfs_dquot *dqp) +{ + int i; + struct xfs_dqtrx *qa; + + switch (xfs_dquot_type(dqp)) { + case XFS_DQTYPE_USER: + qa = tp->t_dqinfo->dqs[XFS_QM_TRANS_USR]; + break; + case XFS_DQTYPE_GROUP: + qa = tp->t_dqinfo->dqs[XFS_QM_TRANS_GRP]; + break; + case XFS_DQTYPE_PROJ: + qa = tp->t_dqinfo->dqs[XFS_QM_TRANS_PRJ]; + break; + default: + return NULL; + } + + for (i = 0; i < XFS_QM_TRANS_MAXDQS; i++) { + if (qa[i].qt_dquot == NULL || + qa[i].qt_dquot == dqp) + return &qa[i]; + } + + return NULL; +} + +/* + * Make the changes in the transaction structure. + * The moral equivalent to xfs_trans_mod_sb(). + * We don't touch any fields in the dquot, so we don't care + * if it's locked or not (most of the time it won't be). + */ +void +xfs_trans_mod_dquot( + struct xfs_trans *tp, + struct xfs_dquot *dqp, + uint field, + int64_t delta) +{ + struct xfs_dqtrx *qtrx; + + ASSERT(tp); + ASSERT(XFS_IS_QUOTA_ON(tp->t_mountp)); + qtrx = NULL; + + if (!delta) + return; + + if (tp->t_dqinfo == NULL) + xfs_trans_alloc_dqinfo(tp); + /* + * Find either the first free slot or the slot that belongs + * to this dquot. + */ + qtrx = xfs_trans_get_dqtrx(tp, dqp); + ASSERT(qtrx); + if (qtrx->qt_dquot == NULL) + qtrx->qt_dquot = dqp; + + trace_xfs_trans_mod_dquot_before(qtrx); + trace_xfs_trans_mod_dquot(tp, dqp, field, delta); + + switch (field) { + /* regular disk blk reservation */ + case XFS_TRANS_DQ_RES_BLKS: + qtrx->qt_blk_res += delta; + break; + + /* inode reservation */ + case XFS_TRANS_DQ_RES_INOS: + qtrx->qt_ino_res += delta; + break; + + /* disk blocks used. */ + case XFS_TRANS_DQ_BCOUNT: + qtrx->qt_bcount_delta += delta; + break; + + case XFS_TRANS_DQ_DELBCOUNT: + qtrx->qt_delbcnt_delta += delta; + break; + + /* Inode Count */ + case XFS_TRANS_DQ_ICOUNT: + if (qtrx->qt_ino_res && delta > 0) { + qtrx->qt_ino_res_used += delta; + ASSERT(qtrx->qt_ino_res >= qtrx->qt_ino_res_used); + } + qtrx->qt_icount_delta += delta; + break; + + /* rtblk reservation */ + case XFS_TRANS_DQ_RES_RTBLKS: + qtrx->qt_rtblk_res += delta; + break; + + /* rtblk count */ + case XFS_TRANS_DQ_RTBCOUNT: + if (qtrx->qt_rtblk_res && delta > 0) { + qtrx->qt_rtblk_res_used += delta; + ASSERT(qtrx->qt_rtblk_res >= qtrx->qt_rtblk_res_used); + } + qtrx->qt_rtbcount_delta += delta; + break; + + case XFS_TRANS_DQ_DELRTBCOUNT: + qtrx->qt_delrtb_delta += delta; + break; + + default: + ASSERT(0); + } + + trace_xfs_trans_mod_dquot_after(qtrx); +} + + +/* + * Given an array of dqtrx structures, lock all the dquots associated and join + * them to the transaction, provided they have been modified. + */ +STATIC void +xfs_trans_dqlockedjoin( + struct xfs_trans *tp, + struct xfs_dqtrx *q) +{ + unsigned int i; + ASSERT(q[0].qt_dquot != NULL); + if (q[1].qt_dquot == NULL) { + mutex_lock(&q[0].qt_dquot->q_qlock); + xfs_trans_dqjoin(tp, q[0].qt_dquot); + } else if (q[2].qt_dquot == NULL) { + xfs_dqlock2(q[0].qt_dquot, q[1].qt_dquot); + xfs_trans_dqjoin(tp, q[0].qt_dquot); + xfs_trans_dqjoin(tp, q[1].qt_dquot); + } else { + xfs_dqlockn(q); + for (i = 0; i < XFS_QM_TRANS_MAXDQS; i++) { + if (q[i].qt_dquot == NULL) + break; + xfs_trans_dqjoin(tp, q[i].qt_dquot); + } + } +} + +/* Apply dqtrx changes to the quota reservation counters. */ +static inline void +xfs_apply_quota_reservation_deltas( + struct xfs_dquot_res *res, + uint64_t reserved, + int64_t res_used, + int64_t count_delta) +{ + if (reserved != 0) { + /* + * Subtle math here: If reserved > res_used (the normal case), + * we're simply subtracting the unused transaction quota + * reservation from the dquot reservation. + * + * If, however, res_used > reserved, then we have allocated + * more quota blocks than were reserved for the transaction. + * We must add that excess to the dquot reservation since it + * tracks (usage + resv) and by definition we didn't reserve + * that excess. + */ + res->reserved -= abs(reserved - res_used); + } else if (count_delta != 0) { + /* + * These blks were never reserved, either inside a transaction + * or outside one (in a delayed allocation). Also, this isn't + * always a negative number since we sometimes deliberately + * skip quota reservations. + */ + res->reserved += count_delta; + } +} + +#ifdef CONFIG_XFS_LIVE_HOOKS +/* Call downstream hooks now that it's time to apply dquot deltas. */ +static inline void +xfs_trans_apply_dquot_deltas_hook( + struct xfs_trans *tp, + struct xfs_dquot *dqp) +{ + if (xfs_hooks_switched_on(&xfs_dqtrx_hooks_switch)) { + struct xfs_apply_dqtrx_params p = { + .tx_id = (uintptr_t)tp, + .q_type = xfs_dquot_type(dqp), + .q_id = dqp->q_id, + }; + struct xfs_quotainfo *qi = tp->t_mountp->m_quotainfo; + + xfs_hooks_call(&qi->qi_apply_dqtrx_hooks, + XFS_APPLY_DQTRX_COMMIT, &p); + } +} +#else +# define xfs_trans_apply_dquot_deltas_hook(tp, dqp) ((void)0) +#endif /* CONFIG_XFS_LIVE_HOOKS */ + +/* + * Called by xfs_trans_commit() and similar in spirit to + * xfs_trans_apply_sb_deltas(). + * Go thru all the dquots belonging to this transaction and modify the + * INCORE dquot to reflect the actual usages. + * Unreserve just the reservations done by this transaction. + * dquot is still left locked at exit. + */ +void +xfs_trans_apply_dquot_deltas( + struct xfs_trans *tp) +{ + int i, j; + struct xfs_dquot *dqp; + struct xfs_dqtrx *qtrx, *qa; + int64_t totalbdelta; + int64_t totalrtbdelta; + + if (!tp->t_dqinfo) + return; + + ASSERT(tp->t_dqinfo); + for (j = 0; j < XFS_QM_TRANS_DQTYPES; j++) { + qa = tp->t_dqinfo->dqs[j]; + if (qa[0].qt_dquot == NULL) + continue; + + /* + * Lock all of the dquots and join them to the transaction. + */ + xfs_trans_dqlockedjoin(tp, qa); + + for (i = 0; i < XFS_QM_TRANS_MAXDQS; i++) { + uint64_t blk_res_used; + + qtrx = &qa[i]; + /* + * The array of dquots is filled + * sequentially, not sparsely. + */ + if ((dqp = qtrx->qt_dquot) == NULL) + break; + + ASSERT(XFS_DQ_IS_LOCKED(dqp)); + + xfs_trans_apply_dquot_deltas_hook(tp, dqp); + + /* + * adjust the actual number of blocks used + */ + + /* + * The issue here is - sometimes we don't make a blkquota + * reservation intentionally to be fair to users + * (when the amount is small). On the other hand, + * delayed allocs do make reservations, but that's + * outside of a transaction, so we have no + * idea how much was really reserved. + * So, here we've accumulated delayed allocation blks and + * non-delay blks. The assumption is that the + * delayed ones are always reserved (outside of a + * transaction), and the others may or may not have + * quota reservations. + */ + totalbdelta = qtrx->qt_bcount_delta + + qtrx->qt_delbcnt_delta; + totalrtbdelta = qtrx->qt_rtbcount_delta + + qtrx->qt_delrtb_delta; + + if (totalbdelta != 0 || totalrtbdelta != 0 || + qtrx->qt_icount_delta != 0) { + trace_xfs_trans_apply_dquot_deltas_before(dqp); + trace_xfs_trans_apply_dquot_deltas(qtrx); + } + +#ifdef DEBUG + if (totalbdelta < 0) + ASSERT(dqp->q_blk.count >= -totalbdelta); + + if (totalrtbdelta < 0) + ASSERT(dqp->q_rtb.count >= -totalrtbdelta); + + if (qtrx->qt_icount_delta < 0) + ASSERT(dqp->q_ino.count >= -qtrx->qt_icount_delta); +#endif + if (totalbdelta) + dqp->q_blk.count += totalbdelta; + + if (qtrx->qt_icount_delta) + dqp->q_ino.count += qtrx->qt_icount_delta; + + if (totalrtbdelta) + dqp->q_rtb.count += totalrtbdelta; + + if (totalbdelta != 0 || totalrtbdelta != 0 || + qtrx->qt_icount_delta != 0) + trace_xfs_trans_apply_dquot_deltas_after(dqp); + + /* + * Get any default limits in use. + * Start/reset the timer(s) if needed. + */ + if (dqp->q_id) { + xfs_qm_adjust_dqlimits(dqp); + xfs_qm_adjust_dqtimers(dqp); + } + + dqp->q_flags |= XFS_DQFLAG_DIRTY; + /* + * add this to the list of items to get logged + */ + xfs_trans_log_dquot(tp, dqp); + /* + * Take off what's left of the original reservation. + * In case of delayed allocations, there's no + * reservation that a transaction structure knows of. + */ + blk_res_used = max_t(int64_t, 0, qtrx->qt_bcount_delta); + xfs_apply_quota_reservation_deltas(&dqp->q_blk, + qtrx->qt_blk_res, blk_res_used, + qtrx->qt_bcount_delta); + + /* + * Adjust the RT reservation. + */ + xfs_apply_quota_reservation_deltas(&dqp->q_rtb, + qtrx->qt_rtblk_res, + qtrx->qt_rtblk_res_used, + qtrx->qt_rtbcount_delta); + + /* + * Adjust the inode reservation. + */ + ASSERT(qtrx->qt_ino_res >= qtrx->qt_ino_res_used); + xfs_apply_quota_reservation_deltas(&dqp->q_ino, + qtrx->qt_ino_res, + qtrx->qt_ino_res_used, + qtrx->qt_icount_delta); + + ASSERT(dqp->q_blk.reserved >= dqp->q_blk.count); + ASSERT(dqp->q_ino.reserved >= dqp->q_ino.count); + ASSERT(dqp->q_rtb.reserved >= dqp->q_rtb.count); + + /* + * We've applied the count changes and given back + * whatever reservation we didn't use. Zero out the + * dqtrx fields. + */ + qtrx->qt_blk_res = 0; + qtrx->qt_bcount_delta = 0; + qtrx->qt_delbcnt_delta = 0; + + qtrx->qt_rtblk_res = 0; + qtrx->qt_rtblk_res_used = 0; + qtrx->qt_rtbcount_delta = 0; + qtrx->qt_delrtb_delta = 0; + + qtrx->qt_ino_res = 0; + qtrx->qt_ino_res_used = 0; + qtrx->qt_icount_delta = 0; + } + } +} + +#ifdef CONFIG_XFS_LIVE_HOOKS +/* Call downstream hooks now that it's time to cancel dquot deltas. */ +static inline void +xfs_trans_unreserve_and_mod_dquots_hook( + struct xfs_trans *tp, + struct xfs_dquot *dqp) +{ + if (xfs_hooks_switched_on(&xfs_dqtrx_hooks_switch)) { + struct xfs_apply_dqtrx_params p = { + .tx_id = (uintptr_t)tp, + .q_type = xfs_dquot_type(dqp), + .q_id = dqp->q_id, + }; + struct xfs_quotainfo *qi = tp->t_mountp->m_quotainfo; + + xfs_hooks_call(&qi->qi_apply_dqtrx_hooks, + XFS_APPLY_DQTRX_UNRESERVE, &p); + } +} +#else +# define xfs_trans_unreserve_and_mod_dquots_hook(tp, dqp) ((void)0) +#endif /* CONFIG_XFS_LIVE_HOOKS */ + +/* + * Release the reservations, and adjust the dquots accordingly. + * This is called only when the transaction is being aborted. If by + * any chance we have done dquot modifications incore (ie. deltas) already, + * we simply throw those away, since that's the expected behavior + * when a transaction is curtailed without a commit. + */ +void +xfs_trans_unreserve_and_mod_dquots( + struct xfs_trans *tp, + bool already_locked) +{ + int i, j; + struct xfs_dquot *dqp; + struct xfs_dqtrx *qtrx, *qa; + bool locked; + + if (!tp->t_dqinfo) + return; + + for (j = 0; j < XFS_QM_TRANS_DQTYPES; j++) { + qa = tp->t_dqinfo->dqs[j]; + + for (i = 0; i < XFS_QM_TRANS_MAXDQS; i++) { + qtrx = &qa[i]; + /* + * We assume that the array of dquots is filled + * sequentially, not sparsely. + */ + if ((dqp = qtrx->qt_dquot) == NULL) + break; + + xfs_trans_unreserve_and_mod_dquots_hook(tp, dqp); + + /* + * Unreserve the original reservation. We don't care + * about the number of blocks used field, or deltas. + * Also we don't bother to zero the fields. + */ + locked = already_locked; + if (qtrx->qt_blk_res) { + if (!locked) { + mutex_lock(&dqp->q_qlock); + locked = true; + } + dqp->q_blk.reserved -= + (xfs_qcnt_t)qtrx->qt_blk_res; + } + if (qtrx->qt_ino_res) { + if (!locked) { + mutex_lock(&dqp->q_qlock); + locked = true; + } + dqp->q_ino.reserved -= + (xfs_qcnt_t)qtrx->qt_ino_res; + } + + if (qtrx->qt_rtblk_res) { + if (!locked) { + mutex_lock(&dqp->q_qlock); + locked = true; + } + dqp->q_rtb.reserved -= + (xfs_qcnt_t)qtrx->qt_rtblk_res; + } + if (locked && !already_locked) + mutex_unlock(&dqp->q_qlock); + + } + } +} + +STATIC void +xfs_quota_warn( + struct xfs_mount *mp, + struct xfs_dquot *dqp, + int type) +{ + enum quota_type qtype; + + switch (xfs_dquot_type(dqp)) { + case XFS_DQTYPE_PROJ: + qtype = PRJQUOTA; + break; + case XFS_DQTYPE_USER: + qtype = USRQUOTA; + break; + case XFS_DQTYPE_GROUP: + qtype = GRPQUOTA; + break; + default: + return; + } + + quota_send_warning(make_kqid(&init_user_ns, qtype, dqp->q_id), + mp->m_super->s_dev, type); +} + +/* + * Decide if we can make an additional reservation against a quota resource. + * Returns an inode QUOTA_NL_ warning code and whether or not it's fatal. + * + * Note that we assume that the numeric difference between the inode and block + * warning codes will always be 3 since it's userspace ABI now, and will never + * decrease the quota reservation, so the *BELOW messages are irrelevant. + */ +static inline int +xfs_dqresv_check( + struct xfs_dquot_res *res, + struct xfs_quota_limits *qlim, + int64_t delta, + bool *fatal) +{ + xfs_qcnt_t hardlimit = res->hardlimit; + xfs_qcnt_t softlimit = res->softlimit; + xfs_qcnt_t total_count = res->reserved + delta; + + BUILD_BUG_ON(QUOTA_NL_BHARDWARN != QUOTA_NL_IHARDWARN + 3); + BUILD_BUG_ON(QUOTA_NL_BSOFTLONGWARN != QUOTA_NL_ISOFTLONGWARN + 3); + BUILD_BUG_ON(QUOTA_NL_BSOFTWARN != QUOTA_NL_ISOFTWARN + 3); + + *fatal = false; + if (delta <= 0) + return QUOTA_NL_NOWARN; + + if (!hardlimit) + hardlimit = qlim->hard; + if (!softlimit) + softlimit = qlim->soft; + + if (hardlimit && total_count > hardlimit) { + *fatal = true; + return QUOTA_NL_IHARDWARN; + } + + if (softlimit && total_count > softlimit) { + time64_t now = ktime_get_real_seconds(); + + if (res->timer != 0 && now > res->timer) { + *fatal = true; + return QUOTA_NL_ISOFTLONGWARN; + } + + return QUOTA_NL_ISOFTWARN; + } + + return QUOTA_NL_NOWARN; +} + +/* + * This reserves disk blocks and inodes against a dquot. + * Flags indicate if the dquot is to be locked here and also + * if the blk reservation is for RT or regular blocks. + * Sending in XFS_QMOPT_FORCE_RES flag skips the quota check. + */ +STATIC int +xfs_trans_dqresv( + struct xfs_trans *tp, + struct xfs_mount *mp, + struct xfs_dquot *dqp, + int64_t nblks, + long ninos, + uint flags) +{ + struct xfs_quotainfo *q = mp->m_quotainfo; + struct xfs_def_quota *defq; + struct xfs_dquot_res *blkres; + struct xfs_quota_limits *qlim; + + mutex_lock(&dqp->q_qlock); + + defq = xfs_get_defquota(q, xfs_dquot_type(dqp)); + + if (flags & XFS_TRANS_DQ_RES_BLKS) { + blkres = &dqp->q_blk; + qlim = &defq->blk; + } else { + blkres = &dqp->q_rtb; + qlim = &defq->rtb; + } + + if ((flags & XFS_QMOPT_FORCE_RES) == 0 && dqp->q_id && + xfs_dquot_is_enforced(dqp)) { + int quota_nl; + bool fatal; + + /* + * dquot is locked already. See if we'd go over the hardlimit + * or exceed the timelimit if we'd reserve resources. + */ + quota_nl = xfs_dqresv_check(blkres, qlim, nblks, &fatal); + if (quota_nl != QUOTA_NL_NOWARN) { + /* + * Quota block warning codes are 3 more than the inode + * codes, which we check above. + */ + xfs_quota_warn(mp, dqp, quota_nl + 3); + if (fatal) + goto error_return; + } + + quota_nl = xfs_dqresv_check(&dqp->q_ino, &defq->ino, ninos, + &fatal); + if (quota_nl != QUOTA_NL_NOWARN) { + xfs_quota_warn(mp, dqp, quota_nl); + if (fatal) + goto error_return; + } + } + + /* + * Change the reservation, but not the actual usage. + * Note that q_blk.reserved = q_blk.count + resv + */ + blkres->reserved += (xfs_qcnt_t)nblks; + dqp->q_ino.reserved += (xfs_qcnt_t)ninos; + + /* + * note the reservation amt in the trans struct too, + * so that the transaction knows how much was reserved by + * it against this particular dquot. + * We don't do this when we are reserving for a delayed allocation, + * because we don't have the luxury of a transaction envelope then. + */ + if (tp) { + ASSERT(flags & XFS_QMOPT_RESBLK_MASK); + xfs_trans_mod_dquot(tp, dqp, flags & XFS_QMOPT_RESBLK_MASK, + nblks); + xfs_trans_mod_dquot(tp, dqp, XFS_TRANS_DQ_RES_INOS, ninos); + } + + if (XFS_IS_CORRUPT(mp, dqp->q_blk.reserved < dqp->q_blk.count) || + XFS_IS_CORRUPT(mp, dqp->q_rtb.reserved < dqp->q_rtb.count) || + XFS_IS_CORRUPT(mp, dqp->q_ino.reserved < dqp->q_ino.count)) + goto error_corrupt; + + mutex_unlock(&dqp->q_qlock); + return 0; + +error_return: + mutex_unlock(&dqp->q_qlock); + if (xfs_dquot_type(dqp) == XFS_DQTYPE_PROJ) + return -ENOSPC; + return -EDQUOT; +error_corrupt: + mutex_unlock(&dqp->q_qlock); + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); + xfs_fs_mark_sick(mp, XFS_SICK_FS_QUOTACHECK); + return -EFSCORRUPTED; +} + + +/* + * Given dquot(s), make disk block and/or inode reservations against them. + * The fact that this does the reservation against user, group and + * project quotas is important, because this follows a all-or-nothing + * approach. + * + * flags = XFS_QMOPT_FORCE_RES evades limit enforcement. Used by chown. + * XFS_QMOPT_ENOSPC returns ENOSPC not EDQUOT. Used by pquota. + * XFS_TRANS_DQ_RES_BLKS reserves regular disk blocks + * XFS_TRANS_DQ_RES_RTBLKS reserves realtime disk blocks + * dquots are unlocked on return, if they were not locked by caller. + */ +int +xfs_trans_reserve_quota_bydquots( + struct xfs_trans *tp, + struct xfs_mount *mp, + struct xfs_dquot *udqp, + struct xfs_dquot *gdqp, + struct xfs_dquot *pdqp, + int64_t nblks, + long ninos, + uint flags) +{ + int error; + + if (!XFS_IS_QUOTA_ON(mp)) + return 0; + + ASSERT(flags & XFS_QMOPT_RESBLK_MASK); + + if (udqp) { + error = xfs_trans_dqresv(tp, mp, udqp, nblks, ninos, flags); + if (error) + return error; + } + + if (gdqp) { + error = xfs_trans_dqresv(tp, mp, gdqp, nblks, ninos, flags); + if (error) + goto unwind_usr; + } + + if (pdqp) { + error = xfs_trans_dqresv(tp, mp, pdqp, nblks, ninos, flags); + if (error) + goto unwind_grp; + } + + /* + * Didn't change anything critical, so, no need to log + */ + return 0; + +unwind_grp: + flags |= XFS_QMOPT_FORCE_RES; + if (gdqp) + xfs_trans_dqresv(tp, mp, gdqp, -nblks, -ninos, flags); +unwind_usr: + flags |= XFS_QMOPT_FORCE_RES; + if (udqp) + xfs_trans_dqresv(tp, mp, udqp, -nblks, -ninos, flags); + return error; +} + + +/* + * Lock the dquot and change the reservation if we can. + * This doesn't change the actual usage, just the reservation. + * The inode sent in is locked. + */ +int +xfs_trans_reserve_quota_nblks( + struct xfs_trans *tp, + struct xfs_inode *ip, + int64_t dblocks, + int64_t rblocks, + bool force) +{ + struct xfs_mount *mp = ip->i_mount; + unsigned int qflags = 0; + int error; + + if (!XFS_IS_QUOTA_ON(mp)) + return 0; + if (xfs_is_metadir_inode(ip)) + return 0; + + ASSERT(!xfs_is_quota_inode(&mp->m_sb, I_INO(ip))); + xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); + + if (force) + qflags |= XFS_QMOPT_FORCE_RES; + + /* Reserve data device quota against the inode's dquots. */ + error = xfs_trans_reserve_quota_bydquots(tp, mp, ip->i_udquot, + ip->i_gdquot, ip->i_pdquot, dblocks, 0, + XFS_QMOPT_RES_REGBLKS | qflags); + if (error) + return error; + + /* Do the same but for realtime blocks. */ + error = xfs_trans_reserve_quota_bydquots(tp, mp, ip->i_udquot, + ip->i_gdquot, ip->i_pdquot, rblocks, 0, + XFS_QMOPT_RES_RTBLKS | qflags); + if (error) { + xfs_trans_reserve_quota_bydquots(tp, mp, ip->i_udquot, + ip->i_gdquot, ip->i_pdquot, -dblocks, 0, + XFS_QMOPT_RES_REGBLKS); + return error; + } + + return 0; +} + +/* Change the quota reservations for an inode creation activity. */ +int +xfs_trans_reserve_quota_icreate( + struct xfs_trans *tp, + struct xfs_dquot *udqp, + struct xfs_dquot *gdqp, + struct xfs_dquot *pdqp, + int64_t dblocks) +{ + struct xfs_mount *mp = tp->t_mountp; + + if (!XFS_IS_QUOTA_ON(mp)) + return 0; + + return xfs_trans_reserve_quota_bydquots(tp, mp, udqp, gdqp, pdqp, + dblocks, 1, XFS_QMOPT_RES_REGBLKS); +} + +STATIC void +xfs_trans_alloc_dqinfo( + xfs_trans_t *tp) +{ + tp->t_dqinfo = kmem_cache_zalloc(xfs_dqtrx_cache, + GFP_KERNEL | __GFP_NOFAIL); +} + +void +xfs_trans_free_dqinfo( + xfs_trans_t *tp) +{ + if (!tp->t_dqinfo) + return; + kmem_cache_free(xfs_dqtrx_cache, tp->t_dqinfo); + tp->t_dqinfo = NULL; +} + +int +xfs_quota_reserve_blkres( + struct xfs_inode *ip, + int64_t blocks) +{ + if (XFS_IS_REALTIME_INODE(ip)) + return xfs_trans_reserve_quota_nblks(NULL, ip, 0, blocks, + false); + return xfs_trans_reserve_quota_nblks(NULL, ip, blocks, 0, false); +} diff --git a/fs/xfs/xfs_trans_priv.h b/fs/xfs/xfs_trans_priv.h new file mode 100644 index 000000000..f945f0450 --- /dev/null +++ b/fs/xfs/xfs_trans_priv.h @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000,2002,2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_TRANS_PRIV_H__ +#define __XFS_TRANS_PRIV_H__ + +struct xlog; +struct xfs_log_item; +struct xfs_mount; +struct xfs_trans; +struct xfs_ail; +struct xfs_log_vec; + + +void xfs_trans_init(struct xfs_mount *); +void xfs_trans_add_item(struct xfs_trans *, struct xfs_log_item *); +void xfs_trans_del_item(struct xfs_log_item *); +void xfs_trans_unreserve_and_mod_sb(struct xfs_trans *tp); + +/* + * AIL traversal cursor. + * + * Rather than using a generation number for detecting changes in the ail, use + * a cursor that is protected by the ail lock. The aild cursor exists in the + * struct xfs_ail, but other traversals can declare it on the stack and link it + * to the ail list. + * + * When an object is deleted from or moved int the AIL, the cursor list is + * searched to see if the object is a designated cursor item. If it is, it is + * deleted from the cursor so that the next time the cursor is used traversal + * will return to the start. + * + * This means a traversal colliding with a removal will cause a restart of the + * list scan, rather than any insertion or deletion anywhere in the list. The + * low bit of the item pointer is set if the cursor has been invalidated so + * that we can tell the difference between invalidation and reaching the end + * of the list to trigger traversal restarts. + */ +struct xfs_ail_cursor { + struct list_head list; + struct xfs_log_item *item; +}; + +/* + * Private AIL structures. + * + * Eventually we need to drive the locking in here as well. + */ +struct xfs_ail { + struct xlog *ail_log; + struct task_struct *ail_task; + struct list_head ail_head; + struct list_head ail_cursors; + spinlock_t ail_lock; + xfs_lsn_t ail_last_pushed_lsn; + xfs_lsn_t ail_head_lsn; + int ail_log_flush; + unsigned long ail_opstate; + struct list_head ail_buf_list; + wait_queue_head_t ail_empty; + xfs_lsn_t ail_target; +}; + +/* Push all items out of the AIL immediately. */ +#define XFS_AIL_OPSTATE_PUSH_ALL 0u + +/* + * From xfs_trans_ail.c + */ +void xfs_trans_ail_update_bulk(struct xfs_ail *ailp, + struct xfs_ail_cursor *cur, + struct xfs_log_item **log_items, int nr_items, + xfs_lsn_t lsn) __releases(ailp->ail_lock); +/* + * Return a pointer to the first item in the AIL. If the AIL is empty, then + * return NULL. + */ +static inline struct xfs_log_item * +xfs_ail_min( + struct xfs_ail *ailp) +{ + return list_first_entry_or_null(&ailp->ail_head, struct xfs_log_item, + li_ail); +} + +static inline void +xfs_trans_ail_update( + struct xfs_ail *ailp, + struct xfs_log_item *lip, + xfs_lsn_t lsn) __releases(ailp->ail_lock) +{ + xfs_trans_ail_update_bulk(ailp, NULL, &lip, 1, lsn); +} + +void xfs_trans_ail_insert(struct xfs_ail *ailp, struct xfs_log_item *lip, + xfs_lsn_t lsn); + +xfs_lsn_t xfs_ail_delete_one(struct xfs_ail *ailp, struct xfs_log_item *lip); +void xfs_ail_update_finish(struct xfs_ail *ailp, xfs_lsn_t old_lsn) + __releases(ailp->ail_lock); +void xfs_trans_ail_delete(struct xfs_log_item *lip, int shutdown_type); + +static inline void xfs_ail_push(struct xfs_ail *ailp) +{ + wake_up_process(ailp->ail_task); +} + +static inline void xfs_ail_push_all(struct xfs_ail *ailp) +{ + if (!test_and_set_bit(XFS_AIL_OPSTATE_PUSH_ALL, &ailp->ail_opstate)) + xfs_ail_push(ailp); +} + +static inline xfs_lsn_t xfs_ail_get_push_target(struct xfs_ail *ailp) +{ + return READ_ONCE(ailp->ail_target); +} + +void xfs_ail_push_all_sync(struct xfs_ail *ailp); +xfs_lsn_t xfs_ail_min_lsn(struct xfs_ail *ailp); + +struct xfs_log_item * xfs_trans_ail_cursor_first(struct xfs_ail *ailp, + struct xfs_ail_cursor *cur, + xfs_lsn_t lsn); +struct xfs_log_item * xfs_trans_ail_cursor_last(struct xfs_ail *ailp, + struct xfs_ail_cursor *cur, + xfs_lsn_t lsn); +struct xfs_log_item * xfs_trans_ail_cursor_next(struct xfs_ail *ailp, + struct xfs_ail_cursor *cur); +void xfs_trans_ail_cursor_done(struct xfs_ail_cursor *cur); + +void __xfs_ail_assign_tail_lsn(struct xfs_ail *ailp); + +static inline void +xfs_ail_assign_tail_lsn( + struct xfs_ail *ailp) +{ + + spin_lock(&ailp->ail_lock); + __xfs_ail_assign_tail_lsn(ailp); + spin_unlock(&ailp->ail_lock); +} + +#if BITS_PER_LONG != 64 +static inline void +xfs_trans_ail_copy_lsn( + struct xfs_ail *ailp, + xfs_lsn_t *dst, + xfs_lsn_t *src) +{ + ASSERT(sizeof(xfs_lsn_t) == 8); /* don't lock if it shrinks */ + spin_lock(&ailp->ail_lock); + *dst = *src; + spin_unlock(&ailp->ail_lock); +} +#else +static inline void +xfs_trans_ail_copy_lsn( + struct xfs_ail *ailp, + xfs_lsn_t *dst, + xfs_lsn_t *src) +{ + ASSERT(sizeof(xfs_lsn_t) == 8); + *dst = *src; +} +#endif + +#endif /* __XFS_TRANS_PRIV_H__ */ diff --git a/fs/xfs/xfs_verify_media.c b/fs/xfs/xfs_verify_media.c new file mode 100644 index 000000000..5ead3976d --- /dev/null +++ b/fs/xfs/xfs_verify_media.c @@ -0,0 +1,443 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (c) 2026 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#include "xfs_platform.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_bit.h" +#include "xfs_btree.h" +#include "xfs_inode.h" +#include "xfs_icache.h" +#include "xfs_trans.h" +#include "xfs_alloc.h" +#include "xfs_ag.h" +#include "xfs_rmap.h" +#include "xfs_rmap_btree.h" +#include "xfs_rtgroup.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_health.h" +#include "xfs_healthmon.h" +#include "xfs_trace.h" +#include "xfs_verify_media.h" + +#include + +struct xfs_group_data_lost { + xfs_agblock_t startblock; + xfs_extlen_t blockcount; +}; + +/* Report lost file data from rmap records */ +static int +xfs_verify_report_data_lost( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *rec, + void *data) +{ + struct xfs_mount *mp = cur->bc_mp; + struct xfs_inode *ip; + struct xfs_group_data_lost *lost = data; + xfs_fileoff_t fileoff = rec->rm_offset; + xfs_extlen_t blocks = rec->rm_blockcount; + const bool is_attr = + (rec->rm_flags & XFS_RMAP_ATTR_FORK); + const xfs_agblock_t lost_end = + lost->startblock + lost->blockcount; + const xfs_agblock_t rmap_end = + rec->rm_startblock + rec->rm_blockcount; + int error = 0; + + if (XFS_RMAP_NON_INODE_OWNER(rec->rm_owner)) + return 0; + + error = xfs_iget(mp, cur->bc_tp, rec->rm_owner, 0, 0, &ip); + if (error) + return 0; + + if (rec->rm_flags & XFS_RMAP_BMBT_BLOCK) { + xfs_bmap_mark_sick(ip, is_attr ? XFS_ATTR_FORK : XFS_DATA_FORK); + goto out_rele; + } + + if (is_attr) { + xfs_inode_mark_sick(ip, XFS_SICK_INO_XATTR); + goto out_rele; + } + + if (lost->startblock > rec->rm_startblock) { + fileoff += lost->startblock - rec->rm_startblock; + blocks -= lost->startblock - rec->rm_startblock; + } + if (rmap_end > lost_end) + blocks -= rmap_end - lost_end; + + fserror_report_data_lost(VFS_I(ip), XFS_FSB_TO_B(mp, fileoff), + XFS_FSB_TO_B(mp, blocks), GFP_NOFS); + +out_rele: + xfs_irele(ip); + return 0; +} + +/* Walk reverse mappings to look for all file data loss */ +static int +xfs_verify_report_losses( + struct xfs_mount *mp, + enum xfs_group_type type, + xfs_daddr_t daddr, + u64 bblen) +{ + struct xfs_group *xg = NULL; + struct xfs_trans *tp; + xfs_fsblock_t start_bno, end_bno; + uint32_t start_gno, end_gno; + int error; + + if (type == XG_TYPE_RTG) { + start_bno = xfs_daddr_to_rtb(mp, daddr); + end_bno = xfs_daddr_to_rtb(mp, daddr + bblen - 1); + } else { + start_bno = XFS_DADDR_TO_FSB(mp, daddr); + end_bno = XFS_DADDR_TO_FSB(mp, daddr + bblen - 1); + } + + tp = xfs_trans_alloc_empty(mp); + start_gno = xfs_fsb_to_gno(mp, start_bno, type); + end_gno = xfs_fsb_to_gno(mp, end_bno, type); + while ((xg = xfs_group_next_range(mp, xg, start_gno, end_gno, type))) { + struct xfs_buf *agf_bp = NULL; + struct xfs_rtgroup *rtg = NULL; + struct xfs_btree_cur *cur; + struct xfs_rmap_irec ri_low = { }; + struct xfs_rmap_irec ri_high; + struct xfs_group_data_lost lost; + + if (type == XG_TYPE_AG) { + struct xfs_perag *pag = to_perag(xg); + + error = xfs_alloc_read_agf(pag, tp, 0, &agf_bp); + if (error) { + xfs_perag_rele(pag); + break; + } + + cur = xfs_rmapbt_init_cursor(mp, tp, agf_bp, pag); + } else { + rtg = to_rtg(xg); + xfs_rtgroup_lock(rtg, XFS_RTGLOCK_RMAP); + cur = xfs_rtrmapbt_init_cursor(tp, rtg); + } + + /* + * Set the rmap range from ri_low to ri_high, which represents + * a [start, end] where we looking for the files or metadata. + */ + memset(&ri_high, 0xFF, sizeof(ri_high)); + if (xg->xg_gno == start_gno) + ri_low.rm_startblock = + xfs_fsb_to_gbno(mp, start_bno, type); + if (xg->xg_gno == end_gno) + ri_high.rm_startblock = + xfs_fsb_to_gbno(mp, end_bno, type); + + lost.startblock = ri_low.rm_startblock; + lost.blockcount = min(xg->xg_block_count, + ri_high.rm_startblock + 1) - + ri_low.rm_startblock; + + error = xfs_rmap_query_range(cur, &ri_low, &ri_high, + xfs_verify_report_data_lost, &lost); + xfs_btree_del_cursor(cur, error); + if (agf_bp) + xfs_trans_brelse(tp, agf_bp); + if (rtg) + xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_RMAP); + if (error) { + xfs_group_rele(xg); + break; + } + } + + xfs_trans_cancel(tp); + return 0; +} + +/* + * Compute the desired verify IO size. + * + * To minimize command overhead, we'd like to create bios that are 1MB, though + * we allow the user to ask for a smaller size. + */ +static unsigned int +xfs_verify_iosize( + const struct xfs_verify_media *me, + struct xfs_buftarg *btp, + uint64_t bbcount) +{ + unsigned int iosize = + min_not_zero(SZ_1M, me->me_max_io_size); + + BUILD_BUG_ON(BBSHIFT != SECTOR_SHIFT); + ASSERT(BBTOB(bbcount) >= btp->bt_logical_sectorsize); + + return clamp(iosize, btp->bt_logical_sectorsize, BBTOB(bbcount)); +} + +/* Allocate as much memory as we can get for verification buffer. */ +static struct folio * +xfs_verify_alloc_folio( + const unsigned int iosize) +{ + unsigned int order = get_order(iosize); + + while (order > 0) { + struct folio *folio = + folio_alloc(GFP_KERNEL | __GFP_NORETRY, order); + + if (folio) + return folio; + order--; + } + + return folio_alloc(GFP_KERNEL, 0); +} + +/* Report any kind of problem verifying media */ +static void +xfs_verify_media_error( + struct xfs_mount *mp, + struct xfs_verify_media *me, + struct xfs_buftarg *btp, + xfs_daddr_t daddr, + unsigned int bio_bbcount, + blk_status_t bio_status) +{ + trace_xfs_verify_media_error(mp, me, btp->bt_dev, daddr, bio_bbcount, + bio_status); + + /* + * Pass any error, I/O or otherwise, up to the caller if we didn't + * successfully verify any bytes at all. + */ + if (me->me_start_daddr == daddr) + me->me_ioerror = -blk_status_to_errno(bio_status); + + /* + * PI validation failures, medium errors, or general IO errors are + * treated as indicators of data loss. Everything else are (hopefully) + * transient errors and are not reported to healthmon or fsnotify. + */ + switch (bio_status) { + case BLK_STS_PROTECTION: + case BLK_STS_IOERR: + case BLK_STS_MEDIUM: + break; + default: + return; + } + + if (!(me->me_flags & XFS_VERIFY_MEDIA_REPORT)) + return; + + xfs_healthmon_report_media(mp, me->me_dev, daddr, bio_bbcount); + + if (!xfs_has_rmapbt(mp)) + return; + + switch (me->me_dev) { + case XFS_DEV_DATA: + xfs_verify_report_losses(mp, XG_TYPE_AG, daddr, bio_bbcount); + break; + case XFS_DEV_RT: + xfs_verify_report_losses(mp, XG_TYPE_RTG, daddr, bio_bbcount); + break; + } +} + +/* Verify the media of an xfs device by submitting read requests to the disk. */ +static int +xfs_verify_media( + struct xfs_mount *mp, + struct xfs_verify_media *me) +{ + struct xfs_buftarg *btp = NULL; + struct bio *bio; + struct folio *folio; + xfs_daddr_t daddr; + uint64_t bbcount; + int error = 0; + + me->me_ioerror = 0; + + switch (me->me_dev) { + case XFS_DEV_DATA: + btp = mp->m_ddev_targp; + break; + case XFS_DEV_LOG: + if (mp->m_logdev_targp != mp->m_ddev_targp) + btp = mp->m_logdev_targp; + break; + case XFS_DEV_RT: + btp = mp->m_rtdev_targp; + break; + } + if (!btp) + return -ENODEV; + + /* + * If the caller told us to verify beyond the end of the disk, tell the + * user exactly where that was. + */ + if (me->me_end_daddr > btp->bt_nr_sectors) + me->me_end_daddr = btp->bt_nr_sectors; + + /* start and end have to be aligned to the lba size */ + if (!IS_ALIGNED(BBTOB(me->me_start_daddr | me->me_end_daddr), + btp->bt_logical_sectorsize)) + return -EINVAL; + + /* + * end_daddr is the exclusive end of the range, so if start_daddr + * reaches there (or beyond), there's no work to be done. + */ + if (me->me_start_daddr >= me->me_end_daddr) + return 0; + + /* + * There are three ranges involved here: + * + * - [me->me_start_daddr, me->me_end_daddr) is the range that the + * user wants to verify. end_daddr can be beyond the end of the + * disk; we'll constrain it to the end if necessary. + * + * - [daddr, me->me_end_daddr) is the range that we have not yet + * verified. We update daddr after each successful read. + * me->me_start_daddr is set to daddr before returning. + * + * - [daddr, daddr + bio_bbcount) is the range that we're currently + * verifying. + */ + daddr = me->me_start_daddr; + bbcount = min_t(sector_t, me->me_end_daddr, btp->bt_nr_sectors) - + me->me_start_daddr; + + folio = xfs_verify_alloc_folio(xfs_verify_iosize(me, btp, bbcount)); + if (!folio) + return -ENOMEM; + + trace_xfs_verify_media(mp, me, btp->bt_dev, daddr, bbcount, folio); + + bio = bio_alloc(btp->bt_bdev, 1, REQ_OP_READ, GFP_KERNEL); + if (!bio) { + error = -ENOMEM; + goto out_folio; + } + + while (bbcount > 0) { + unsigned int bio_bbcount; + blk_status_t bio_status; + + bio_reset(bio, btp->bt_bdev, REQ_OP_READ); + bio->bi_iter.bi_sector = daddr; + bio_add_folio_nofail(bio, folio, + min(bbcount << SECTOR_SHIFT, folio_size(folio)), + 0); + + /* + * Save the length of the bio before we submit it, because we + * need the original daddr and length for reporting IO errors + * if the bio fails. + */ + bio_bbcount = bio->bi_iter.bi_size >> SECTOR_SHIFT; + submit_bio_wait(bio); + bio_status = bio->bi_status; + if (bio_status != BLK_STS_OK) { + xfs_verify_media_error(mp, me, btp, daddr, bio_bbcount, + bio_status); + error = 0; + break; + } + + daddr += bio_bbcount; + bbcount -= bio_bbcount; + + if (bbcount == 0) + break; + + if (me->me_rest_us) { + ktime_t expires; + + expires = ktime_add_ns(ktime_get(), + me->me_rest_us * 1000); + set_current_state(TASK_KILLABLE); + schedule_hrtimeout(&expires, HRTIMER_MODE_ABS); + } + + if (fatal_signal_pending(current)) { + error = -EINTR; + break; + } + + cond_resched(); + } + + bio_put(bio); +out_folio: + folio_put(folio); + + if (error) + return error; + + /* + * Advance start_daddr to the end of what we verified if there wasn't + * an operational error. + */ + me->me_start_daddr = daddr; + trace_xfs_verify_media_end(mp, me, btp->bt_dev); + return 0; +} + +int +xfs_ioc_verify_media( + struct file *file, + struct xfs_verify_media __user *arg) +{ + struct xfs_verify_media me; + struct xfs_inode *ip = XFS_I(file_inode(file)); + struct xfs_mount *mp = ip->i_mount; + int error; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + if (copy_from_user(&me, arg, sizeof(me))) + return -EFAULT; + + if (me.me_pad) + return -EINVAL; + if (me.me_flags & ~XFS_VERIFY_MEDIA_FLAGS) + return -EINVAL; + + switch (me.me_dev) { + case XFS_DEV_DATA: + case XFS_DEV_LOG: + case XFS_DEV_RT: + break; + default: + return -EINVAL; + } + + error = xfs_verify_media(mp, &me); + if (error) + return error; + + if (copy_to_user(arg, &me, sizeof(me))) + return -EFAULT; + + return 0; +} diff --git a/fs/xfs/xfs_verify_media.h b/fs/xfs/xfs_verify_media.h new file mode 100644 index 000000000..dc6eee9c8 --- /dev/null +++ b/fs/xfs/xfs_verify_media.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (c) 2026 Oracle. All Rights Reserved. + * Author: Darrick J. Wong + */ +#ifndef __XFS_VERIFY_MEDIA_H__ +#define __XFS_VERIFY_MEDIA_H__ + +struct xfs_verify_media; +int xfs_ioc_verify_media(struct file *file, + struct xfs_verify_media __user *arg); + +#endif /* __XFS_VERIFY_MEDIA_H__ */ diff --git a/fs/xfs/xfs_xattr.c b/fs/xfs/xfs_xattr.c new file mode 100644 index 000000000..1efe6c813 --- /dev/null +++ b/fs/xfs/xfs_xattr.c @@ -0,0 +1,347 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2008 Christoph Hellwig. + * Portions Copyright (C) 2000-2008 Silicon Graphics, Inc. + */ + +#include "xfs_platform.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_da_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_da_btree.h" +#include "xfs_attr.h" +#include "xfs_acl.h" +#include "xfs_log.h" +#include "xfs_xattr.h" +#include "xfs_quota.h" + +#include + +/* + * Get permission to use log-assisted atomic exchange of file extents. + * Callers must not be running any transactions or hold any ILOCKs. + */ +static inline int +xfs_attr_grab_log_assist( + struct xfs_mount *mp) +{ + int error = 0; + + /* xattr update log intent items are already enabled */ + if (xfs_is_using_logged_xattrs(mp)) + return 0; + + /* + * Check if the filesystem featureset is new enough to set this log + * incompat feature bit. Strictly speaking, the minimum requirement is + * a V5 filesystem for the superblock field, but we'll require rmap + * or reflink to avoid having to deal with really old kernels. + */ + if (!xfs_has_reflink(mp) && !xfs_has_rmapbt(mp)) + return -EOPNOTSUPP; + + /* Enable log-assisted xattrs. */ + error = xfs_add_incompat_log_feature(mp, + XFS_SB_FEAT_INCOMPAT_LOG_XATTRS); + if (error) + return error; + xfs_set_using_logged_xattrs(mp); + + xfs_warn_experimental(mp, XFS_EXPERIMENTAL_LARP); + + return 0; +} + +static inline bool +xfs_attr_want_log_assist( + struct xfs_mount *mp) +{ +#ifdef DEBUG + /* Logged xattrs require a V5 super for log_incompat */ + return xfs_has_crc(mp) && xfs_globals.larp; +#else + return false; +#endif +} + +/* + * Set or remove an xattr, having grabbed the appropriate logging resources + * prior to calling libxfs. Callers of this function are only required to + * initialize the inode, attr_filter, name, namelen, value, and valuelen fields + * of @args. + */ +int +xfs_attr_change( + struct xfs_da_args *args, + enum xfs_attr_update op) +{ + struct xfs_mount *mp = args->dp->i_mount; + int error; + + if (xfs_is_shutdown(mp)) + return -EIO; + + error = xfs_qm_dqattach(args->dp); + if (error) + return error; + + /* + * We have no control over the attribute names that userspace passes us + * to remove, so we have to allow the name lookup prior to attribute + * removal to fail as well. + */ + args->op_flags = XFS_DA_OP_OKNOENT; + + if (xfs_attr_want_log_assist(mp)) { + error = xfs_attr_grab_log_assist(mp); + if (error) + return error; + + args->op_flags |= XFS_DA_OP_LOGGED; + } + + args->owner = I_INO(args->dp); + args->geo = mp->m_attr_geo; + args->whichfork = XFS_ATTR_FORK; + xfs_attr_sethash(args); + + /* + * Some xattrs must be resistant to allocation failure at ENOSPC, e.g. + * creating an inode with ACLs or security attributes requires the + * allocation of the xattr holding that information to succeed. Hence + * we allow xattrs in the VFS TRUSTED, SYSTEM, POSIX_ACL and SECURITY + * (LSM xattr) namespaces to dip into the reserve block pool to allow + * manipulation of these xattrs when at ENOSPC. These VFS xattr + * namespaces translate to the XFS_ATTR_ROOT and XFS_ATTR_SECURE on-disk + * namespaces. + * + * For most of these cases, these special xattrs will fit in the inode + * itself and so consume no extra space or only require temporary extra + * space while an overwrite is being made. Hence the use of the reserved + * pool is largely to avoid the worst case reservation from preventing + * the xattr from being created at ENOSPC. + */ + return xfs_attr_set(args, op, + args->attr_filter & (XFS_ATTR_ROOT | XFS_ATTR_SECURE)); +} + + +static int +xfs_xattr_get(const struct xattr_handler *handler, struct dentry *unused, + struct inode *inode, const char *name, void *value, size_t size) +{ + struct xfs_da_args args = { + .dp = XFS_I(inode), + .attr_filter = handler->flags, + .name = name, + .namelen = strlen(name), + .value = value, + .valuelen = size, + }; + int error; + + if (xfs_ifork_zapped(XFS_I(inode), XFS_ATTR_FORK)) + return -EIO; + + error = xfs_attr_get(&args); + if (error) + return error; + return args.valuelen; +} + +static inline enum xfs_attr_update +xfs_xattr_flags_to_op( + int flags, + const void *value) +{ + if (!value) + return XFS_ATTRUPDATE_REMOVE; + if (flags & XATTR_CREATE) + return XFS_ATTRUPDATE_CREATE; + if (flags & XATTR_REPLACE) + return XFS_ATTRUPDATE_REPLACE; + return XFS_ATTRUPDATE_UPSERT; +} + +static int +xfs_xattr_set(const struct xattr_handler *handler, + struct mnt_idmap *idmap, struct dentry *unused, + struct inode *inode, const char *name, const void *value, + size_t size, int flags) +{ + struct xfs_da_args args = { + .dp = XFS_I(inode), + .attr_filter = handler->flags, + .name = name, + .namelen = strlen(name), + .value = (void *)value, + .valuelen = size, + }; + int error; + + error = xfs_attr_change(&args, xfs_xattr_flags_to_op(flags, value)); + if (!error && (handler->flags & XFS_ATTR_ROOT)) + xfs_forget_acl(inode, name); + return error; +} + +static const struct xattr_handler xfs_xattr_user_handler = { + .prefix = XATTR_USER_PREFIX, + .flags = 0, /* no flags implies user namespace */ + .get = xfs_xattr_get, + .set = xfs_xattr_set, +}; + +static const struct xattr_handler xfs_xattr_trusted_handler = { + .prefix = XATTR_TRUSTED_PREFIX, + .flags = XFS_ATTR_ROOT, + .get = xfs_xattr_get, + .set = xfs_xattr_set, +}; + +static const struct xattr_handler xfs_xattr_security_handler = { + .prefix = XATTR_SECURITY_PREFIX, + .flags = XFS_ATTR_SECURE, + .get = xfs_xattr_get, + .set = xfs_xattr_set, +}; + +const struct xattr_handler * const xfs_xattr_handlers[] = { + &xfs_xattr_user_handler, + &xfs_xattr_trusted_handler, + &xfs_xattr_security_handler, + NULL +}; + +static void +__xfs_xattr_put_listent( + struct xfs_attr_list_context *context, + char *prefix, + int prefix_len, + unsigned char *name, + int namelen) +{ + char *offset; + int arraytop; + + if (context->count < 0 || context->seen_enough) + return; + + if (!context->buffer) + goto compute_size; + + arraytop = context->count + prefix_len + namelen + 1; + if (arraytop > context->firstu) { + context->count = -1; /* insufficient space */ + context->seen_enough = 1; + return; + } + offset = context->buffer + context->count; + memcpy(offset, prefix, prefix_len); + offset += prefix_len; + memcpy(offset, (char *)name, namelen); /* real name */ + offset += namelen; + *offset = '\0'; + +compute_size: + context->count += prefix_len + namelen + 1; + return; +} + +static void +xfs_xattr_put_listent( + struct xfs_attr_list_context *context, + int flags, + unsigned char *name, + int namelen, + void *value, + int valuelen) +{ + char *prefix; + int prefix_len; + + ASSERT(context->count >= 0); + + /* Don't expose private xattr namespaces. */ + if (flags & XFS_ATTR_PRIVATE_NSP_MASK) + return; + + if (flags & XFS_ATTR_ROOT) { +#ifdef CONFIG_XFS_POSIX_ACL + if (namelen == SGI_ACL_FILE_SIZE && + strncmp(name, SGI_ACL_FILE, + SGI_ACL_FILE_SIZE) == 0) { + __xfs_xattr_put_listent( + context, XATTR_SYSTEM_PREFIX, + XATTR_SYSTEM_PREFIX_LEN, + XATTR_POSIX_ACL_ACCESS, + strlen(XATTR_POSIX_ACL_ACCESS)); + } else if (namelen == SGI_ACL_DEFAULT_SIZE && + strncmp(name, SGI_ACL_DEFAULT, + SGI_ACL_DEFAULT_SIZE) == 0) { + __xfs_xattr_put_listent( + context, XATTR_SYSTEM_PREFIX, + XATTR_SYSTEM_PREFIX_LEN, + XATTR_POSIX_ACL_DEFAULT, + strlen(XATTR_POSIX_ACL_DEFAULT)); + } +#endif + + /* + * Only show root namespace entries if we are actually allowed to + * see them. + */ + if (!capable(CAP_SYS_ADMIN)) + return; + + prefix = XATTR_TRUSTED_PREFIX; + prefix_len = XATTR_TRUSTED_PREFIX_LEN; + } else if (flags & XFS_ATTR_SECURE) { + prefix = XATTR_SECURITY_PREFIX; + prefix_len = XATTR_SECURITY_PREFIX_LEN; + } else { + prefix = XATTR_USER_PREFIX; + prefix_len = XATTR_USER_PREFIX_LEN; + } + + __xfs_xattr_put_listent(context, prefix, prefix_len, name, + namelen); + return; +} + +ssize_t +xfs_vn_listxattr( + struct dentry *dentry, + char *data, + size_t size) +{ + struct xfs_attr_list_context context; + struct inode *inode = d_inode(dentry); + int error; + + if (xfs_ifork_zapped(XFS_I(inode), XFS_ATTR_FORK)) + return -EIO; + + /* + * First read the regular on-disk attributes. + */ + memset(&context, 0, sizeof(context)); + context.dp = XFS_I(inode); + context.resynch = 1; + context.bufsize = size; + context.buffer = size ? data : NULL; + context.firstu = context.bufsize; + context.put_listent = xfs_xattr_put_listent; + + error = xfs_attr_list(&context); + if (error) + return error; + if (context.count < 0) + return -ERANGE; + + return context.count; +} diff --git a/fs/xfs/xfs_xattr.h b/fs/xfs/xfs_xattr.h new file mode 100644 index 000000000..c3eb858fb --- /dev/null +++ b/fs/xfs/xfs_xattr.h @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2000-2005 Silicon Graphics, Inc. + * All Rights Reserved. + */ +#ifndef __XFS_XATTR_H__ +#define __XFS_XATTR_H__ + +enum xfs_attr_update; +int xfs_attr_change(struct xfs_da_args *args, enum xfs_attr_update op); + +extern const struct xattr_handler * const xfs_xattr_handlers[]; + +#endif /* __XFS_XATTR_H__ */ diff --git a/fs/xfs/xfs_zone_alloc.c b/fs/xfs/xfs_zone_alloc.c new file mode 100644 index 000000000..7d13fa7ab --- /dev/null +++ b/fs/xfs/xfs_zone_alloc.c @@ -0,0 +1,1447 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2023-2025 Christoph Hellwig. + * Copyright (c) 2024-2025, Western Digital Corporation or its affiliates. + */ +#include "xfs_platform.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_error.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_iomap.h" +#include "xfs_trans.h" +#include "xfs_alloc.h" +#include "xfs_bmap.h" +#include "xfs_bmap_btree.h" +#include "xfs_trans_space.h" +#include "xfs_refcount.h" +#include "xfs_rtbitmap.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_zone_alloc.h" +#include "xfs_sysfs.h" +#include "xfs_zone_priv.h" +#include "xfs_zones.h" +#include "xfs_trace.h" +#include "xfs_mru_cache.h" + +static void +xfs_open_zone_free_rcu( + struct callback_head *cb) +{ + struct xfs_open_zone *oz = container_of(cb, typeof(*oz), oz_rcu); + + xfs_rtgroup_rele(oz->oz_rtg); + kfree(oz); +} + +void +xfs_open_zone_put( + struct xfs_open_zone *oz) +{ + if (atomic_dec_and_test(&oz->oz_ref)) + call_rcu(&oz->oz_rcu, xfs_open_zone_free_rcu); +} + +static inline uint32_t +xfs_zone_bucket( + struct xfs_mount *mp, + uint32_t used_blocks) +{ + return XFS_ZONE_USED_BUCKETS * used_blocks / + mp->m_groups[XG_TYPE_RTG].blocks; +} + +static inline void +xfs_zone_add_to_bucket( + struct xfs_zone_info *zi, + xfs_rgnumber_t rgno, + uint32_t to_bucket) +{ + __set_bit(rgno, zi->zi_used_bucket_bitmap[to_bucket]); + zi->zi_used_bucket_entries[to_bucket]++; +} + +static inline void +xfs_zone_remove_from_bucket( + struct xfs_zone_info *zi, + xfs_rgnumber_t rgno, + uint32_t from_bucket) +{ + __clear_bit(rgno, zi->zi_used_bucket_bitmap[from_bucket]); + zi->zi_used_bucket_entries[from_bucket]--; +} + +static void +xfs_zone_account_reclaimable( + struct xfs_rtgroup *rtg, + uint32_t freed) +{ + struct xfs_group *xg = rtg_group(rtg); + struct xfs_mount *mp = rtg_mount(rtg); + struct xfs_zone_info *zi = mp->m_zone_info; + uint32_t used = rtg_rmap(rtg)->i_used_blocks; + xfs_rgnumber_t rgno = rtg_rgno(rtg); + uint32_t from_bucket = xfs_zone_bucket(mp, used + freed); + uint32_t to_bucket = xfs_zone_bucket(mp, used); + bool was_full = (used + freed == rtg_blocks(rtg)); + + /* + * This can be called from log recovery, where the zone_info structure + * hasn't been allocated yet. Skip all work as xfs_mount_zones will + * add the zones to the right buckets before the file systems becomes + * active. + */ + if (!zi) + return; + + if (!used) { + /* + * The zone is now empty, remove it from the bottom bucket and + * trigger a reset. + */ + trace_xfs_zone_emptied(rtg); + + spin_lock(&zi->zi_used_buckets_lock); + if (!was_full) + xfs_zone_remove_from_bucket(zi, rgno, from_bucket); + spin_unlock(&zi->zi_used_buckets_lock); + + spin_lock(&zi->zi_reset_list_lock); + xg->xg_next_reset = zi->zi_reset_list; + zi->zi_reset_list = xg; + spin_unlock(&zi->zi_reset_list_lock); + + if (zi->zi_gc_thread) + wake_up_process(zi->zi_gc_thread); + } else if (was_full) { + /* + * The zone transitioned from full, mark it up as reclaimable + * and wake up GC which might be waiting for zones to reclaim. + */ + spin_lock(&zi->zi_used_buckets_lock); + xfs_zone_add_to_bucket(zi, rgno, to_bucket); + spin_unlock(&zi->zi_used_buckets_lock); + + if (zi->zi_gc_thread && xfs_zoned_need_gc(mp)) + wake_up_process(zi->zi_gc_thread); + } else if (to_bucket != from_bucket) { + /* + * Move the zone to a new bucket if it dropped below the + * threshold. + */ + spin_lock(&zi->zi_used_buckets_lock); + xfs_zone_add_to_bucket(zi, rgno, to_bucket); + xfs_zone_remove_from_bucket(zi, rgno, from_bucket); + spin_unlock(&zi->zi_used_buckets_lock); + } +} + +/* + * Check if we have any zones that can be reclaimed by looking at the entry + * counters for the zone buckets. + */ +bool +xfs_zoned_have_reclaimable( + struct xfs_zone_info *zi) +{ + int i; + + spin_lock(&zi->zi_used_buckets_lock); + for (i = 0; i < XFS_ZONE_USED_BUCKETS; i++) { + if (zi->zi_used_bucket_entries[i]) { + spin_unlock(&zi->zi_used_buckets_lock); + return true; + } + } + spin_unlock(&zi->zi_used_buckets_lock); + + return false; +} + +static void +xfs_open_zone_mark_full( + struct xfs_open_zone *oz) +{ + struct xfs_rtgroup *rtg = oz->oz_rtg; + struct xfs_mount *mp = rtg_mount(rtg); + struct xfs_zone_info *zi = mp->m_zone_info; + uint32_t used = rtg_rmap(rtg)->i_used_blocks; + + trace_xfs_zone_full(rtg); + + WRITE_ONCE(rtg->rtg_open_zone, NULL); + + spin_lock(&zi->zi_open_zones_lock); + if (oz->oz_is_gc) + zi->zi_nr_open_gc_zones--; + else + zi->zi_nr_open_zones--; + list_del_init(&oz->oz_entry); + spin_unlock(&zi->zi_open_zones_lock); + + if (oz->oz_is_gc) + wake_up_process(zi->zi_gc_thread); + else + wake_up_all(&zi->zi_zone_wait); + + if (used < rtg_blocks(rtg)) + xfs_zone_account_reclaimable(rtg, rtg_blocks(rtg) - used); + xfs_open_zone_put(oz); +} + +static inline void +xfs_zone_inc_written( + struct xfs_open_zone *oz, + xfs_filblks_t len) +{ + xfs_assert_ilocked(rtg_rmap(oz->oz_rtg), XFS_ILOCK_EXCL); + + oz->oz_written += len; + if (oz->oz_written == rtg_blocks(oz->oz_rtg)) + xfs_open_zone_mark_full(oz); +} + +/* + * Called for blocks that have been written to disk, but not actually linked to + * an inode, which can happen when garbage collection races with user data + * writes to a file. + */ +static void +xfs_zone_skip_blocks( + struct xfs_open_zone *oz, + xfs_filblks_t len) +{ + struct xfs_rtgroup *rtg = oz->oz_rtg; + + trace_xfs_zone_skip_blocks(oz, 0, len); + + xfs_rtgroup_lock(rtg, XFS_RTGLOCK_RMAP); + xfs_zone_inc_written(oz, len); + xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_RMAP); + + xfs_add_frextents(rtg_mount(rtg), len); +} + +static int +xfs_zoned_map_extent( + struct xfs_trans *tp, + struct xfs_inode *ip, + struct xfs_bmbt_irec *new, + struct xfs_open_zone *oz, + xfs_fsblock_t old_startblock) +{ + struct xfs_bmbt_irec data; + struct xfs_rtgroup *rtg = oz->oz_rtg; + struct xfs_inode *rmapip = rtg_rmap(rtg); + int nmaps = 1; + int error; + + /* Grab the corresponding mapping in the data fork. */ + error = xfs_bmapi_read(ip, new->br_startoff, new->br_blockcount, &data, + &nmaps, 0); + if (error) + return error; + + /* + * Cap the update to the existing extent in the data fork because we can + * only overwrite one extent at a time. + */ + ASSERT(new->br_blockcount >= data.br_blockcount); + new->br_blockcount = data.br_blockcount; + + /* + * If a data write raced with this GC write, keep the existing data in + * the data fork, mark our newly written GC extent as reclaimable, then + * move on to the next extent. + * + * Note that this can also happen when racing with operations that do + * not actually invalidate the data, but just move it to a different + * inode (XFS_IOC_EXCHANGE_RANGE), or to a different offset inside the + * inode (FALLOC_FL_COLLAPSE_RANGE / FALLOC_FL_INSERT_RANGE). If the + * data was just moved around, GC fails to free the zone, but the zone + * becomes a GC candidate again as soon as all previous GC I/O has + * finished and these blocks will be moved out eventually. + */ + if (old_startblock != NULLFSBLOCK && + old_startblock != data.br_startblock) + goto skip; + + trace_xfs_reflink_cow_remap_from(ip, new); + trace_xfs_reflink_cow_remap_to(ip, &data); + + error = xfs_iext_count_extend(tp, ip, XFS_DATA_FORK, + XFS_IEXT_REFLINK_END_COW_CNT); + if (error) + return error; + + if (data.br_startblock != HOLESTARTBLOCK) { + ASSERT(data.br_startblock != DELAYSTARTBLOCK); + ASSERT(!isnullstartblock(data.br_startblock)); + + xfs_bmap_unmap_extent(tp, ip, XFS_DATA_FORK, &data); + if (xfs_is_reflink_inode(ip)) { + xfs_refcount_decrease_extent(tp, true, &data); + } else { + error = xfs_free_extent_later(tp, data.br_startblock, + data.br_blockcount, NULL, + XFS_AG_RESV_NONE, + XFS_FREE_EXTENT_REALTIME); + if (error) + return error; + } + } + + trace_xfs_zone_record_blocks(oz, + xfs_rtb_to_rgbno(tp->t_mountp, new->br_startblock), + new->br_blockcount); + xfs_rtgroup_lock(rtg, XFS_RTGLOCK_RMAP); + xfs_rtgroup_trans_join(tp, rtg, XFS_RTGLOCK_RMAP); + rmapip->i_used_blocks += new->br_blockcount; + ASSERT(rmapip->i_used_blocks <= rtg_blocks(rtg)); + xfs_zone_inc_written(oz, new->br_blockcount); + xfs_trans_log_inode(tp, rmapip, XFS_ILOG_CORE); + + /* Map the new blocks into the data fork. */ + xfs_bmap_map_extent(tp, ip, XFS_DATA_FORK, new); + return 0; + +skip: + trace_xfs_reflink_cow_remap_skip(ip, new); + xfs_zone_skip_blocks(oz, new->br_blockcount); + return 0; +} + +int +xfs_zoned_end_io( + struct xfs_inode *ip, + xfs_off_t offset, + xfs_off_t count, + xfs_daddr_t daddr, + struct xfs_open_zone *oz, + xfs_fsblock_t old_startblock) +{ + struct xfs_mount *mp = ip->i_mount; + xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + count); + struct xfs_bmbt_irec new = { + .br_startoff = XFS_B_TO_FSBT(mp, offset), + .br_startblock = xfs_daddr_to_rtb(mp, daddr), + .br_state = XFS_EXT_NORM, + }; + unsigned int resblks = + XFS_EXTENTADD_SPACE_RES(mp, XFS_DATA_FORK); + struct xfs_trans *tp; + int error; + + if (xfs_is_shutdown(mp)) + return -EIO; + + while (new.br_startoff < end_fsb) { + new.br_blockcount = end_fsb - new.br_startoff; + + error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, resblks, 0, + XFS_TRANS_RESERVE | XFS_TRANS_RES_FDBLKS, &tp); + if (error) + return error; + xfs_ilock(ip, XFS_ILOCK_EXCL); + xfs_trans_ijoin(tp, ip, 0); + + error = xfs_zoned_map_extent(tp, ip, &new, oz, old_startblock); + if (error) + xfs_trans_cancel(tp); + else + error = xfs_trans_commit(tp); + xfs_iunlock(ip, XFS_ILOCK_EXCL); + if (error) + return error; + + new.br_startoff += new.br_blockcount; + new.br_startblock += new.br_blockcount; + if (old_startblock != NULLFSBLOCK) + old_startblock += new.br_blockcount; + } + + return 0; +} + +/* + * "Free" blocks allocated in a zone. + * + * Just decrement the used blocks counter and report the space as freed. + */ +int +xfs_zone_free_blocks( + struct xfs_trans *tp, + struct xfs_rtgroup *rtg, + xfs_fsblock_t fsbno, + xfs_filblks_t len) +{ + struct xfs_mount *mp = tp->t_mountp; + struct xfs_inode *rmapip = rtg_rmap(rtg); + + xfs_assert_ilocked(rmapip, XFS_ILOCK_EXCL); + + if (len > rmapip->i_used_blocks) { + xfs_err(mp, +"trying to free more blocks (%lld) than used counter (%u).", + len, rmapip->i_used_blocks); + ASSERT(len <= rmapip->i_used_blocks); + xfs_rtginode_mark_sick(rtg, XFS_RTGI_RMAP); + xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); + return -EFSCORRUPTED; + } + + trace_xfs_zone_free_blocks(rtg, xfs_rtb_to_rgbno(mp, fsbno), len); + + rmapip->i_used_blocks -= len; + /* + * Don't add open zones to the reclaimable buckets. The I/O completion + * for writing the last block will take care of accounting for already + * unused blocks instead. + */ + if (!READ_ONCE(rtg->rtg_open_zone)) + xfs_zone_account_reclaimable(rtg, len); + xfs_add_frextents(mp, len); + xfs_trans_log_inode(tp, rmapip, XFS_ILOG_CORE); + return 0; +} + +static struct xfs_open_zone * +xfs_init_open_zone( + struct xfs_rtgroup *rtg, + xfs_rgblock_t write_pointer, + enum rw_hint write_hint, + bool is_gc) +{ + struct xfs_open_zone *oz; + + oz = kzalloc_obj(*oz, GFP_NOFS | __GFP_NOFAIL); + spin_lock_init(&oz->oz_alloc_lock); + atomic_set(&oz->oz_ref, 1); + oz->oz_rtg = rtg; + oz->oz_allocated = write_pointer; + oz->oz_written = write_pointer; + oz->oz_write_hint = write_hint; + oz->oz_is_gc = is_gc; + + /* + * All dereferences of rtg->rtg_open_zone hold the ILOCK for the rmap + * inode, but we don't really want to take that here because we are + * under the zone_list_lock. Ensure the pointer is only set for a fully + * initialized open zone structure so that a racy lookup finding it is + * fine. + */ + WRITE_ONCE(rtg->rtg_open_zone, oz); + return oz; +} + +/* + * Find a completely free zone, open it, and return a reference. + */ +struct xfs_open_zone * +xfs_open_zone( + struct xfs_mount *mp, + enum rw_hint write_hint, + bool is_gc) +{ + struct xfs_zone_info *zi = mp->m_zone_info; + XA_STATE (xas, &mp->m_groups[XG_TYPE_RTG].xa, 0); + struct xfs_group *xg; + + /* + * Pick the free zone with lowest index. Zones in the beginning of the + * address space typically provides higher bandwidth than those at the + * end of the address space on HDDs. + */ + xas_lock(&xas); + xas_for_each_marked(&xas, xg, ULONG_MAX, XFS_RTG_FREE) + if (atomic_inc_not_zero(&xg->xg_active_ref)) + goto found; + xas_unlock(&xas); + return NULL; + +found: + xas_clear_mark(&xas, XFS_RTG_FREE); + atomic_dec(&zi->zi_nr_free_zones); + xas_unlock(&xas); + + set_current_state(TASK_RUNNING); + return xfs_init_open_zone(to_rtg(xg), 0, write_hint, is_gc); +} + +static struct xfs_open_zone * +xfs_try_open_zone( + struct xfs_mount *mp, + enum rw_hint write_hint) +{ + struct xfs_zone_info *zi = mp->m_zone_info; + struct xfs_open_zone *oz; + + if (zi->zi_nr_open_zones >= mp->m_max_open_zones - XFS_OPEN_GC_ZONES) + return NULL; + if (atomic_read(&zi->zi_nr_free_zones) < + XFS_GC_ZONES - XFS_OPEN_GC_ZONES) + return NULL; + + /* + * Increment the open zone count to reserve our slot before dropping + * zi_open_zones_lock. + */ + zi->zi_nr_open_zones++; + spin_unlock(&zi->zi_open_zones_lock); + oz = xfs_open_zone(mp, write_hint, false); + spin_lock(&zi->zi_open_zones_lock); + if (!oz) { + zi->zi_nr_open_zones--; + return NULL; + } + + atomic_inc(&oz->oz_ref); + list_add_tail(&oz->oz_entry, &zi->zi_open_zones); + + /* + * If this was the last free zone, other waiters might be waiting + * on us to write to it as well. + */ + wake_up_all(&zi->zi_zone_wait); + + if (xfs_zoned_need_gc(mp)) + wake_up_process(zi->zi_gc_thread); + + trace_xfs_zone_opened(oz->oz_rtg); + return oz; +} + +enum xfs_zone_alloc_score { + /* Any open zone will do it, we're desperate */ + XFS_ZONE_ALLOC_ANY = 0, + + /* It better fit somehow */ + XFS_ZONE_ALLOC_OK = 1, + + /* Only reuse a zone if it fits really well. */ + XFS_ZONE_ALLOC_GOOD = 2, +}; + +/* + * Life time hint co-location matrix. Fields not set default to 0 + * aka XFS_ZONE_ALLOC_ANY. + */ +static const unsigned int +xfs_zoned_hint_score[WRITE_LIFE_HINT_NR][WRITE_LIFE_HINT_NR] = { + [WRITE_LIFE_NOT_SET] = { + [WRITE_LIFE_NOT_SET] = XFS_ZONE_ALLOC_OK, + }, + [WRITE_LIFE_NONE] = { + [WRITE_LIFE_NONE] = XFS_ZONE_ALLOC_OK, + }, + [WRITE_LIFE_SHORT] = { + [WRITE_LIFE_SHORT] = XFS_ZONE_ALLOC_GOOD, + }, + [WRITE_LIFE_MEDIUM] = { + [WRITE_LIFE_MEDIUM] = XFS_ZONE_ALLOC_GOOD, + }, + [WRITE_LIFE_LONG] = { + [WRITE_LIFE_LONG] = XFS_ZONE_ALLOC_OK, + [WRITE_LIFE_EXTREME] = XFS_ZONE_ALLOC_OK, + }, + [WRITE_LIFE_EXTREME] = { + [WRITE_LIFE_LONG] = XFS_ZONE_ALLOC_OK, + [WRITE_LIFE_EXTREME] = XFS_ZONE_ALLOC_OK, + }, +}; + +static bool +xfs_try_use_zone( + struct xfs_zone_info *zi, + enum rw_hint file_hint, + struct xfs_open_zone *oz, + unsigned int goodness) +{ + if (oz->oz_is_gc) + return false; + + if (oz->oz_allocated == rtg_blocks(oz->oz_rtg)) + return false; + + if (xfs_zoned_hint_score[oz->oz_write_hint][file_hint] < goodness) + return false; + + if (!atomic_inc_not_zero(&oz->oz_ref)) + return false; + + /* + * If we have a hint set for the data, use that for the zone even if + * some data was written already without any hint set, but don't change + * the temperature after that as that would make little sense without + * tracking per-temperature class written block counts, which is + * probably overkill anyway. + */ + if (file_hint != WRITE_LIFE_NOT_SET && + oz->oz_write_hint == WRITE_LIFE_NOT_SET) + oz->oz_write_hint = file_hint; + + /* + * If we couldn't match by inode or life time we just pick the first + * zone with enough space above. For that we want the least busy zone + * for some definition of "least" busy. For now this simple LRU + * algorithm that rotates every zone to the end of the list will do it, + * even if it isn't exactly cache friendly. + */ + if (!list_is_last(&oz->oz_entry, &zi->zi_open_zones)) + list_move_tail(&oz->oz_entry, &zi->zi_open_zones); + return true; +} + +static struct xfs_open_zone * +xfs_select_open_zone_lru( + struct xfs_zone_info *zi, + enum rw_hint file_hint, + unsigned int goodness) +{ + struct xfs_open_zone *oz; + + lockdep_assert_held(&zi->zi_open_zones_lock); + + list_for_each_entry(oz, &zi->zi_open_zones, oz_entry) + if (xfs_try_use_zone(zi, file_hint, oz, goodness)) + return oz; + + cond_resched_lock(&zi->zi_open_zones_lock); + return NULL; +} + +static struct xfs_open_zone * +xfs_select_open_zone_mru( + struct xfs_zone_info *zi, + enum rw_hint file_hint) +{ + struct xfs_open_zone *oz; + + lockdep_assert_held(&zi->zi_open_zones_lock); + + list_for_each_entry_reverse(oz, &zi->zi_open_zones, oz_entry) + if (xfs_try_use_zone(zi, file_hint, oz, XFS_ZONE_ALLOC_OK)) + return oz; + + cond_resched_lock(&zi->zi_open_zones_lock); + return NULL; +} + +static inline enum rw_hint xfs_inode_write_hint(struct xfs_inode *ip) +{ + if (xfs_has_nolifetime(ip->i_mount)) + return WRITE_LIFE_NOT_SET; + return VFS_I(ip)->i_write_hint; +} + +/* + * Try to tightly pack small files that are written back after they were closed + * instead of trying to open new zones for them or spread them to the least + * recently used zone. This optimizes the data layout for workloads that untar + * or copy a lot of small files. Right now this does not separate multiple such + * streams. + */ +static inline bool xfs_zoned_pack_tight(struct xfs_inode *ip) +{ + struct xfs_mount *mp = ip->i_mount; + size_t zone_capacity = + XFS_FSB_TO_B(mp, mp->m_groups[XG_TYPE_RTG].blocks); + + /* + * Do not pack write files that are already using a full zone to avoid + * fragmentation. + */ + if (i_size_read(VFS_I(ip)) >= zone_capacity) + return false; + + return !inode_is_open_for_write(VFS_I(ip)) && + !(ip->i_diflags & XFS_DIFLAG_APPEND); +} + +static struct xfs_open_zone * +xfs_select_zone_nowait( + struct xfs_mount *mp, + enum rw_hint write_hint, + bool pack_tight) +{ + struct xfs_zone_info *zi = mp->m_zone_info; + struct xfs_open_zone *oz = NULL; + + if (xfs_is_shutdown(mp)) + return NULL; + + /* + * Try to fill up open zones with matching temperature if available. It + * is better to try to co-locate data when this is favorable, so we can + * activate empty zones when it is statistically better to separate + * data. + */ + spin_lock(&zi->zi_open_zones_lock); + oz = xfs_select_open_zone_lru(zi, write_hint, XFS_ZONE_ALLOC_GOOD); + if (oz) + goto out_unlock; + + if (pack_tight) { + oz = xfs_select_open_zone_mru(zi, write_hint); + if (oz) + goto out_unlock; + } + + /* + * See if we can open a new zone and use that so that data for different + * files is mixed as little as possible. + */ + oz = xfs_try_open_zone(mp, write_hint); + if (oz) + goto out_unlock; + + /* + * Try to find a zone that is an ok match to colocate data with. + */ + oz = xfs_select_open_zone_lru(zi, write_hint, XFS_ZONE_ALLOC_OK); + if (oz) + goto out_unlock; + + /* + * Pick the least recently used zone, regardless of hint match + */ + oz = xfs_select_open_zone_lru(zi, write_hint, XFS_ZONE_ALLOC_ANY); +out_unlock: + spin_unlock(&zi->zi_open_zones_lock); + return oz; +} + +static struct xfs_open_zone * +xfs_select_zone( + struct xfs_mount *mp, + enum rw_hint write_hint, + bool pack_tight) +{ + struct xfs_zone_info *zi = mp->m_zone_info; + DEFINE_WAIT (wait); + struct xfs_open_zone *oz; + + oz = xfs_select_zone_nowait(mp, write_hint, pack_tight); + if (oz) + return oz; + + for (;;) { + prepare_to_wait(&zi->zi_zone_wait, &wait, TASK_UNINTERRUPTIBLE); + oz = xfs_select_zone_nowait(mp, write_hint, pack_tight); + if (oz || xfs_is_shutdown(mp)) + break; + schedule(); + } + finish_wait(&zi->zi_zone_wait, &wait); + return oz; +} + +static unsigned int +xfs_zone_alloc_blocks( + struct xfs_open_zone *oz, + xfs_filblks_t count_fsb, + sector_t *sector, + bool *is_seq) +{ + struct xfs_rtgroup *rtg = oz->oz_rtg; + struct xfs_mount *mp = rtg_mount(rtg); + xfs_rgblock_t allocated; + + spin_lock(&oz->oz_alloc_lock); + count_fsb = min3(count_fsb, XFS_MAX_BMBT_EXTLEN, + (xfs_filblks_t)rtg_blocks(rtg) - oz->oz_allocated); + if (!count_fsb) { + spin_unlock(&oz->oz_alloc_lock); + return 0; + } + allocated = oz->oz_allocated; + oz->oz_allocated += count_fsb; + spin_unlock(&oz->oz_alloc_lock); + + trace_xfs_zone_alloc_blocks(oz, allocated, count_fsb); + + *sector = xfs_gbno_to_daddr(rtg_group(rtg), 0); + *is_seq = bdev_zone_is_seq(mp->m_rtdev_targp->bt_bdev, *sector); + if (!*is_seq) + *sector += XFS_FSB_TO_BB(mp, allocated); + return XFS_FSB_TO_B(mp, count_fsb); +} + +void +xfs_mark_rtg_boundary( + struct iomap_ioend *ioend) +{ + struct xfs_mount *mp = XFS_I(ioend->io_inode)->i_mount; + sector_t sector = ioend->io_bio.bi_iter.bi_sector; + + if (xfs_rtb_to_rgbno(mp, xfs_daddr_to_rtb(mp, sector)) == 0) + ioend->io_flags |= IOMAP_IOEND_BOUNDARY; +} + +/* + * Check if we have a cached last open zone available for the inode and + * if yes return a reference to it. + */ +static struct xfs_open_zone * +xfs_get_cached_zone( + struct xfs_inode *ip) +{ + struct xfs_open_zone *oz; + + rcu_read_lock(); + oz = VFS_I(ip)->i_private; + if (oz) { + /* + * GC only steals open zones at mount time, so no GC zones + * should end up in the cache. + */ + ASSERT(!oz->oz_is_gc); + if (!atomic_inc_not_zero(&oz->oz_ref)) + oz = NULL; + } + rcu_read_unlock(); + + return oz; +} + +/* + * Stash our zone in the inode so that is is reused for future allocations. + * + * The open_zone structure will be pinned until either the inode is freed or + * until the cached open zone is replaced with a different one because the + * current one was full when we tried to use it. This means we keep any + * open zone around forever as long as any inode that used it for the last + * write is cached, which slightly increases the memory use of cached inodes + * that were every written to, but significantly simplifies the cached zone + * lookup. Because the open_zone is clearly marked as full when all data + * in the underlying RTG was written, the caching is always safe. + */ +static void +xfs_set_cached_zone( + struct xfs_inode *ip, + struct xfs_open_zone *oz) +{ + struct xfs_open_zone *old_oz; + + atomic_inc(&oz->oz_ref); + old_oz = xchg(&VFS_I(ip)->i_private, oz); + if (old_oz) + xfs_open_zone_put(old_oz); +} + +static void +xfs_submit_zoned_bio( + struct iomap_ioend *ioend, + struct xfs_open_zone *oz, + bool is_seq) +{ + ioend->io_bio.bi_iter.bi_sector = ioend->io_sector; + ioend->io_private = oz; + atomic_inc(&oz->oz_ref); /* for xfs_zoned_end_io */ + + if (is_seq) { + ioend->io_bio.bi_opf &= ~REQ_OP_WRITE; + ioend->io_bio.bi_opf |= REQ_OP_ZONE_APPEND; + } else { + xfs_mark_rtg_boundary(ioend); + } + + submit_bio(&ioend->io_bio); +} + +void +xfs_zone_alloc_and_submit( + struct iomap_ioend *ioend, + struct xfs_open_zone **oz) +{ + struct xfs_inode *ip = XFS_I(ioend->io_inode); + struct xfs_mount *mp = ip->i_mount; + enum rw_hint write_hint = xfs_inode_write_hint(ip); + bool pack_tight = xfs_zoned_pack_tight(ip); + unsigned int alloc_len; + struct iomap_ioend *split; + bool is_seq; + + if (xfs_is_shutdown(mp)) + goto out_error; + + /* + * If we don't have a locally cached zone in this write context, see if + * the inode is still associated with a zone and use that if so. + */ + if (!*oz) + *oz = xfs_get_cached_zone(ip); + + if (!*oz) { +select_zone: + *oz = xfs_select_zone(mp, write_hint, pack_tight); + if (!*oz) + goto out_error; + xfs_set_cached_zone(ip, *oz); + } + + alloc_len = xfs_zone_alloc_blocks(*oz, XFS_B_TO_FSB(mp, ioend->io_size), + &ioend->io_sector, &is_seq); + if (!alloc_len) { + xfs_open_zone_put(*oz); + goto select_zone; + } + + while ((split = iomap_split_ioend(ioend, alloc_len, is_seq))) { + if (IS_ERR(split)) + goto out_split_error; + alloc_len -= split->io_bio.bi_iter.bi_size; + xfs_submit_zoned_bio(split, *oz, is_seq); + if (!alloc_len) { + xfs_open_zone_put(*oz); + goto select_zone; + } + } + + xfs_submit_zoned_bio(ioend, *oz, is_seq); + return; + +out_split_error: + ioend->io_bio.bi_status = errno_to_blk_status(PTR_ERR(split)); +out_error: + bio_io_error(&ioend->io_bio); +} + +/* + * Wake up all threads waiting for a zoned space allocation when the file system + * is shut down. + */ +void +xfs_zoned_wake_all( + struct xfs_mount *mp) +{ + /* + * Don't wake up if there is no m_zone_info. This is complicated by the + * fact that unmount can't atomically clear m_zone_info and thus we need + * to check SB_ACTIVE for that, but mount temporarily enables SB_ACTIVE + * during log recovery so we can't entirely rely on that either. + */ + if ((mp->m_super->s_flags & SB_ACTIVE) && mp->m_zone_info) + wake_up_all(&mp->m_zone_info->zi_zone_wait); +} + +/* + * Check if @rgbno in @rgb is a potentially valid block. It might still be + * unused, but that information is only found in the rmap. + */ +bool +xfs_zone_rgbno_is_valid( + struct xfs_rtgroup *rtg, + xfs_rgnumber_t rgbno) +{ + lockdep_assert_held(&rtg_rmap(rtg)->i_lock); + + if (rtg->rtg_open_zone) + return rgbno < rtg->rtg_open_zone->oz_allocated; + return !xa_get_mark(&rtg_mount(rtg)->m_groups[XG_TYPE_RTG].xa, + rtg_rgno(rtg), XFS_RTG_FREE); +} + +void +xfs_zone_mark_free( + struct xfs_rtgroup *rtg) +{ + xfs_group_set_mark(rtg_group(rtg), XFS_RTG_FREE); + atomic_inc(&rtg_mount(rtg)->m_zone_info->zi_nr_free_zones); +} + +static void +xfs_free_open_zones( + struct xfs_zone_info *zi) +{ + struct xfs_open_zone *oz; + + spin_lock(&zi->zi_open_zones_lock); + while ((oz = list_first_entry_or_null(&zi->zi_open_zones, + struct xfs_open_zone, oz_entry))) { + list_del(&oz->oz_entry); + xfs_open_zone_put(oz); + } + spin_unlock(&zi->zi_open_zones_lock); + + /* + * Wait for all open zones to be freed so that they drop the group + * references: + */ + rcu_barrier(); +} + +struct xfs_init_zones { + uint32_t zone_size; + uint32_t zone_capacity; + uint64_t available; + uint64_t reclaimable; +}; + +/* + * For sequential write required zones, we restart writing at the hardware write + * pointer returned by xfs_validate_blk_zone(). + * + * For conventional zones or conventional devices we have to query the rmap to + * find the highest recorded block and set the write pointer to the block after + * that. In case of a power loss this misses blocks where the data I/O has + * completed but not recorded in the rmap yet, and it also rewrites blocks if + * the most recently written ones got deleted again before unmount, but this is + * the best we can do without hardware support. + */ +static int +xfs_query_write_pointer( + struct xfs_init_zones *iz, + struct xfs_rtgroup *rtg, + xfs_rgblock_t *write_pointer) +{ + struct xfs_mount *mp = rtg_mount(rtg); + struct block_device *bdev = mp->m_rtdev_targp->bt_bdev; + sector_t start = xfs_gbno_to_daddr(&rtg->rtg_group, 0); + xfs_rgblock_t highest_rgbno; + struct blk_zone zone = {}; + int error; + + if (bdev_is_zoned(bdev)) { + error = blkdev_get_zone_info(bdev, start, &zone); + if (error) + return error; + if (zone.start != start) { + xfs_warn(mp, "mismatched zone start: 0x%llx/0x%llx.", + zone.start, start); + return -EFSCORRUPTED; + } + + if (!xfs_validate_blk_zone(mp, &zone, rtg_rgno(rtg), + iz->zone_size, iz->zone_capacity, + write_pointer)) + return -EFSCORRUPTED; + + /* + * Use the hardware write pointer returned by + * xfs_validate_blk_zone for sequential write required zones, + * else fall through to the rmap-based estimation below. + */ + if (zone.cond != BLK_ZONE_COND_NOT_WP) + return 0; + } + + xfs_rtgroup_lock(rtg, XFS_RTGLOCK_RMAP); + highest_rgbno = xfs_rtrmap_highest_rgbno(rtg); + xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_RMAP); + + if (highest_rgbno == NULLRGBLOCK) + *write_pointer = 0; + else + *write_pointer = highest_rgbno + 1; + return 0; +} + +static int +xfs_init_zone( + struct xfs_init_zones *iz, + struct xfs_rtgroup *rtg, + xfs_rgblock_t write_pointer) +{ + struct xfs_mount *mp = rtg_mount(rtg); + struct xfs_zone_info *zi = mp->m_zone_info; + uint32_t used = rtg_rmap(rtg)->i_used_blocks; + int error; + + if (write_pointer > rtg->rtg_extents) { + xfs_warn(mp, "zone %u has invalid write pointer (0x%x).", + rtg_rgno(rtg), write_pointer); + return -EFSCORRUPTED; + } + + if (used > rtg->rtg_extents) { + xfs_warn(mp, +"zone %u has used counter (0x%x) larger than zone capacity (0x%llx).", + rtg_rgno(rtg), used, rtg->rtg_extents); + return -EFSCORRUPTED; + } + + if (used > write_pointer) { + xfs_warn(mp, +"zone %u has used counter (0x%x) larger than write pointer (0x%x).", + rtg_rgno(rtg), used, write_pointer); + return -EFSCORRUPTED; + } + + if (write_pointer == 0 && used != 0) { + xfs_warn(mp, "empty zone %u has non-zero used counter (0x%x).", + rtg_rgno(rtg), used); + return -EFSCORRUPTED; + } + + /* + * If there are no used blocks, but the zone is not in empty state yet + * we lost power before the zoned reset. In that case finish the work + * here. + */ + if (write_pointer == rtg_blocks(rtg) && used == 0) { + error = xfs_zone_gc_reset_sync(rtg); + if (error) + return error; + write_pointer = 0; + } + + if (write_pointer == 0) { + /* zone is empty */ + xfs_zone_mark_free(rtg); + iz->available += rtg_blocks(rtg); + } else if (write_pointer < rtg_blocks(rtg)) { + /* zone is open */ + struct xfs_open_zone *oz; + + atomic_inc(&rtg_group(rtg)->xg_active_ref); + oz = xfs_init_open_zone(rtg, write_pointer, WRITE_LIFE_NOT_SET, + false); + list_add_tail(&oz->oz_entry, &zi->zi_open_zones); + zi->zi_nr_open_zones++; + + iz->available += (rtg_blocks(rtg) - write_pointer); + iz->reclaimable += write_pointer - used; + } else if (used < rtg_blocks(rtg)) { + /* zone fully written, but has freed blocks */ + xfs_zone_account_reclaimable(rtg, rtg_blocks(rtg) - used); + iz->reclaimable += (rtg_blocks(rtg) - used); + } + + return 0; +} + +/* + * Calculate the max open zone limit based on the of number of backing zones + * available. + */ +static inline uint32_t +xfs_max_open_zones( + struct xfs_mount *mp) +{ + unsigned int max_open, max_open_data_zones; + + /* + * We need two zones for every open data zone, one in reserve as we + * don't reclaim open zones. One data zone and its spare is included + * in XFS_MIN_ZONES to support at least one user data writer. + */ + max_open_data_zones = (mp->m_sb.sb_rgcount - XFS_MIN_ZONES) / 2 + 1; + max_open = max_open_data_zones + XFS_OPEN_GC_ZONES; + + /* + * Cap the max open limit to 1/4 of available space. Without this we'd + * run out of easy reclaim targets too quickly and storage devices don't + * handle huge numbers of concurrent write streams overly well. + */ + max_open = min(max_open, mp->m_sb.sb_rgcount / 4); + + return max(XFS_MIN_OPEN_ZONES, max_open); +} + +/* + * Normally we use the open zone limit that the device reports. If there is + * none let the user pick one from the command line. + * + * If the device doesn't report an open zone limit and there is no override, + * allow to hold about a quarter of the zones open. In theory we could allow + * all to be open, but at that point we run into GC deadlocks because we can't + * reclaim open zones. + * + * When used on conventional SSDs a lower open limit is advisable as we'll + * otherwise overwhelm the FTL just as much as a conventional block allocator. + * + * Note: To debug the open zone management code, force max_open to 1 here. + */ +static int +xfs_calc_open_zones( + struct xfs_mount *mp) +{ + struct block_device *bdev = mp->m_rtdev_targp->bt_bdev; + unsigned int bdev_open_zones = bdev_max_open_zones(bdev); + + if (!mp->m_max_open_zones) { + if (bdev_open_zones) + mp->m_max_open_zones = bdev_open_zones; + else + mp->m_max_open_zones = XFS_DEFAULT_MAX_OPEN_ZONES; + } + + if (mp->m_max_open_zones < XFS_MIN_OPEN_ZONES) { + xfs_notice(mp, "need at least %u open zones.", + XFS_MIN_OPEN_ZONES); + return -EIO; + } + + if (bdev_open_zones && bdev_open_zones < mp->m_max_open_zones) { + mp->m_max_open_zones = bdev_open_zones; + xfs_info(mp, "limiting open zones to %u due to hardware limit.", + bdev_open_zones); + } + + if (mp->m_max_open_zones > xfs_max_open_zones(mp)) { + mp->m_max_open_zones = xfs_max_open_zones(mp); + xfs_info(mp, +"limiting open zones to %u due to total zone count (%u)", + mp->m_max_open_zones, mp->m_sb.sb_rgcount); + } + + return 0; +} + +static unsigned long * +xfs_alloc_bucket_bitmap( + struct xfs_mount *mp) +{ + return kvmalloc_array(BITS_TO_LONGS(mp->m_sb.sb_rgcount), + sizeof(unsigned long), GFP_KERNEL | __GFP_ZERO); +} + +static struct xfs_zone_info * +xfs_alloc_zone_info( + struct xfs_mount *mp) +{ + struct xfs_zone_info *zi; + int i; + + zi = kzalloc_obj(*zi); + if (!zi) + return NULL; + INIT_LIST_HEAD(&zi->zi_open_zones); + INIT_LIST_HEAD(&zi->zi_reclaim_reservations); + spin_lock_init(&zi->zi_reset_list_lock); + spin_lock_init(&zi->zi_open_zones_lock); + spin_lock_init(&zi->zi_reservation_lock); + init_waitqueue_head(&zi->zi_zone_wait); + spin_lock_init(&zi->zi_used_buckets_lock); + for (i = 0; i < XFS_ZONE_USED_BUCKETS; i++) { + zi->zi_used_bucket_bitmap[i] = xfs_alloc_bucket_bitmap(mp); + if (!zi->zi_used_bucket_bitmap[i]) + goto out_free_bitmaps; + } + return zi; + +out_free_bitmaps: + while (--i >= 0) + kvfree(zi->zi_used_bucket_bitmap[i]); + kfree(zi); + return NULL; +} + +static void +xfs_free_zone_info( + struct xfs_zone_info *zi) +{ + int i; + + xfs_free_open_zones(zi); + for (i = 0; i < XFS_ZONE_USED_BUCKETS; i++) + kvfree(zi->zi_used_bucket_bitmap[i]); + kfree(zi); +} + +static int +xfs_report_zones( + struct xfs_mount *mp, + struct xfs_init_zones *iz) +{ + struct xfs_rtgroup *rtg = NULL; + + while ((rtg = xfs_rtgroup_next(mp, rtg))) { + xfs_rgblock_t write_pointer; + int error; + + error = xfs_query_write_pointer(iz, rtg, &write_pointer); + if (!error) + error = xfs_init_zone(iz, rtg, write_pointer); + if (error) { + xfs_rtgroup_rele(rtg); + return error; + } + } + + return 0; +} + +static inline bool +xfs_zone_is_conv( + struct xfs_rtgroup *rtg) +{ + return !bdev_zone_is_seq(rtg_mount(rtg)->m_rtdev_targp->bt_bdev, + xfs_gbno_to_daddr(rtg_group(rtg), 0)); +} + +static struct xfs_open_zone * +xfs_find_fullest_conventional_open_zone( + struct xfs_mount *mp) +{ + struct xfs_zone_info *zi = mp->m_zone_info; + struct xfs_open_zone *found = NULL, *oz; + + spin_lock(&zi->zi_open_zones_lock); + list_for_each_entry(oz, &zi->zi_open_zones, oz_entry) { + if (!xfs_zone_is_conv(oz->oz_rtg)) + continue; + if (!found || oz->oz_allocated > found->oz_allocated) + found = oz; + } + spin_unlock(&zi->zi_open_zones_lock); + + return found; +} + +/* + * Find the fullest conventional zones and remove them from the open zone pool + * until we are at the open zone limit. + * + * We can end up with spurious "open" zones when the last blocks in a fully + * written zone were invalidate as there is no write pointer for conventional + * zones. + * + * If we are still over the limit when there is no conventional open zone left, + * the user overrode the max open zones limit using the max_open_zones mount + * option we should fail. + */ +static int +xfs_finish_spurious_open_zones( + struct xfs_mount *mp, + struct xfs_init_zones *iz) +{ + struct xfs_zone_info *zi = mp->m_zone_info; + + while (zi->zi_nr_open_zones > mp->m_max_open_zones) { + struct xfs_open_zone *oz; + xfs_filblks_t adjust; + + oz = xfs_find_fullest_conventional_open_zone(mp); + if (!oz) { + xfs_err(mp, +"too many open zones for max_open_zones limit (%u/%u)", + zi->zi_nr_open_zones, mp->m_max_open_zones); + return -EINVAL; + } + + xfs_rtgroup_lock(oz->oz_rtg, XFS_RTGLOCK_RMAP); + adjust = rtg_blocks(oz->oz_rtg) - oz->oz_written; + trace_xfs_zone_spurious_open(oz, oz->oz_written, adjust); + oz->oz_written = rtg_blocks(oz->oz_rtg); + xfs_open_zone_mark_full(oz); + xfs_rtgroup_unlock(oz->oz_rtg, XFS_RTGLOCK_RMAP); + iz->available -= adjust; + iz->reclaimable += adjust; + } + + return 0; +} + +int +xfs_mount_zones( + struct xfs_mount *mp) +{ + struct xfs_init_zones iz = { + .zone_capacity = mp->m_groups[XG_TYPE_RTG].blocks, + .zone_size = xfs_rtgroup_raw_size(mp), + }; + int error; + + if (!mp->m_rtdev_targp) { + xfs_notice(mp, "RT device missing."); + return -EINVAL; + } + + if (!xfs_has_rtgroups(mp) || !xfs_has_rmapbt(mp)) { + xfs_notice(mp, "invalid flag combination."); + return -EFSCORRUPTED; + } + if (mp->m_sb.sb_rextsize != 1) { + xfs_notice(mp, "zoned file systems do not support rextsize."); + return -EFSCORRUPTED; + } + if (mp->m_sb.sb_rgcount < XFS_MIN_ZONES) { + xfs_notice(mp, +"zoned file systems need to have at least %u zones.", XFS_MIN_ZONES); + return -EFSCORRUPTED; + } + + error = xfs_calc_open_zones(mp); + if (error) + return error; + + mp->m_zone_info = xfs_alloc_zone_info(mp); + if (!mp->m_zone_info) + return -ENOMEM; + + error = xfs_report_zones(mp, &iz); + if (error) + goto out_free_zone_info; + + error = xfs_finish_spurious_open_zones(mp, &iz); + if (error) + goto out_free_zone_info; + + xfs_set_freecounter(mp, XC_FREE_RTAVAILABLE, iz.available); + xfs_set_freecounter(mp, XC_FREE_RTEXTENTS, + iz.available + iz.reclaimable); + + /* + * The writeback code switches between inodes regularly to provide + * fairness. The default lower bound is 4MiB, but for zoned file + * systems we want to increase that both to reduce seeks, but also more + * importantly so that workloads that writes files in a multiple of the + * zone size do not get fragmented and require garbage collection when + * they shouldn't. Increase is to the zone size capped by the max + * extent len. + * + * Note that because s_min_writeback_pages is a superblock field, this + * value also get applied to non-zoned files on the data device if + * there are any. On typical zoned setup all data is on the RT device + * because using the more efficient sequential write required zones + * is the reason for using the zone allocator, and either the RT device + * and the (meta)data device are on the same block device, or the + * (meta)data device is on a fast SSD while the data on the RT device + * is on a SMR HDD. In any combination of the above cases enforcing + * the higher min_writeback_pages for non-RT inodes is either a noop + * or beneficial. + */ + mp->m_super->s_min_writeback_pages = + XFS_FSB_TO_B(mp, min(iz.zone_capacity, XFS_MAX_BMBT_EXTLEN)) >> + PAGE_SHIFT; + + /* + * The user may configure GC to free up a percentage of unused blocks. + * By default this is 0. GC will always trigger at the minimum level + * for keeping max_open_zones available for data placement. + */ + mp->m_zonegc_low_space = 0; + + error = xfs_zone_gc_mount(mp); + if (error) + goto out_free_zone_info; + + error = xfs_zoned_sysfs_init(mp); + if (error) + goto out_zone_gc_unmount; + + xfs_info(mp, "%u zones of %u blocks (%u max open zones)", + mp->m_sb.sb_rgcount, iz.zone_capacity, mp->m_max_open_zones); + trace_xfs_zones_mount(mp); + return 0; + +out_zone_gc_unmount: + xfs_zone_gc_unmount(mp); +out_free_zone_info: + xfs_free_zone_info(mp->m_zone_info); + return error; +} + +void +xfs_unmount_zones( + struct xfs_mount *mp) +{ + xfs_zoned_sysfs_del(mp); + xfs_zone_gc_unmount(mp); + xfs_free_zone_info(mp->m_zone_info); +} diff --git a/fs/xfs/xfs_zone_alloc.h b/fs/xfs/xfs_zone_alloc.h new file mode 100644 index 000000000..abf8358bb --- /dev/null +++ b/fs/xfs/xfs_zone_alloc.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _XFS_ZONE_ALLOC_H +#define _XFS_ZONE_ALLOC_H + +struct iomap_ioend; +struct xfs_open_zone; + +struct xfs_zone_alloc_ctx { + struct xfs_open_zone *open_zone; + xfs_filblks_t reserved_blocks; +}; + +/* + * Grab any available space, even if it is less than what the caller asked for. + */ +#define XFS_ZR_GREEDY (1U << 0) +/* + * Only grab instantly available space, don't wait or GC. + */ +#define XFS_ZR_NOWAIT (1U << 1) +/* + * Dip into the reserved pool. + */ +#define XFS_ZR_RESERVED (1U << 2) + +int xfs_zoned_space_reserve(struct xfs_mount *mp, xfs_filblks_t count_fsb, + unsigned int flags, struct xfs_zone_alloc_ctx *ac); +void xfs_zoned_space_unreserve(struct xfs_mount *mp, + struct xfs_zone_alloc_ctx *ac); +void xfs_zoned_add_available(struct xfs_mount *mp, xfs_filblks_t count_fsb); + +void xfs_zone_alloc_and_submit(struct iomap_ioend *ioend, + struct xfs_open_zone **oz); +int xfs_zone_free_blocks(struct xfs_trans *tp, struct xfs_rtgroup *rtg, + xfs_fsblock_t fsbno, xfs_filblks_t len); +int xfs_zoned_end_io(struct xfs_inode *ip, xfs_off_t offset, xfs_off_t count, + xfs_daddr_t daddr, struct xfs_open_zone *oz, + xfs_fsblock_t old_startblock); +void xfs_open_zone_put(struct xfs_open_zone *oz); + +void xfs_zoned_wake_all(struct xfs_mount *mp); +bool xfs_zone_rgbno_is_valid(struct xfs_rtgroup *rtg, xfs_rgnumber_t rgbno); +void xfs_mark_rtg_boundary(struct iomap_ioend *ioend); + +void xfs_zone_mark_free(struct xfs_rtgroup *rtg); +uint64_t xfs_zoned_default_resblks(struct xfs_mount *mp, + enum xfs_free_counter ctr); +void xfs_zoned_show_stats(struct seq_file *m, struct xfs_mount *mp); + +#ifdef CONFIG_XFS_RT +int xfs_mount_zones(struct xfs_mount *mp); +void xfs_unmount_zones(struct xfs_mount *mp); +void xfs_zone_gc_start(struct xfs_mount *mp); +void xfs_zone_gc_stop(struct xfs_mount *mp); +void xfs_zone_gc_wakeup(struct xfs_mount *mp); +#else +static inline int xfs_mount_zones(struct xfs_mount *mp) +{ + return -EIO; +} +static inline void xfs_unmount_zones(struct xfs_mount *mp) +{ +} +static inline void xfs_zone_gc_start(struct xfs_mount *mp) +{ +} +static inline void xfs_zone_gc_stop(struct xfs_mount *mp) +{ +} +static inline void xfs_zone_gc_wakeup(struct xfs_mount *mp) +{ +} +#endif /* CONFIG_XFS_RT */ + +#endif /* _XFS_ZONE_ALLOC_H */ diff --git a/fs/xfs/xfs_zone_gc.c b/fs/xfs/xfs_zone_gc.c new file mode 100644 index 000000000..d0b85179a --- /dev/null +++ b/fs/xfs/xfs_zone_gc.c @@ -0,0 +1,1244 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2023-2025 Christoph Hellwig. + * Copyright (c) 2024-2025, Western Digital Corporation or its affiliates. + */ +#include "xfs_platform.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_log_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_btree.h" +#include "xfs_trans.h" +#include "xfs_icache.h" +#include "xfs_rmap.h" +#include "xfs_rtbitmap.h" +#include "xfs_rtrmap_btree.h" +#include "xfs_errortag.h" +#include "xfs_error.h" +#include "xfs_zone_alloc.h" +#include "xfs_zone_priv.h" +#include "xfs_zones.h" +#include "xfs_trace.h" + +/* + * Implement Garbage Collection (GC) of partially used zoned. + * + * To support the purely sequential writes in each zone, zoned XFS needs to be + * able to move data remaining in a zone out of it to reset the zone to prepare + * for writing to it again. + * + * This is done by the GC thread implemented in this file. To support that a + * number of zones (XFS_GC_ZONES) is reserved from the user visible capacity to + * write the garbage collected data into. + * + * Whenever the available space is below the chosen threshold, the GC thread + * looks for potential non-empty but not fully used zones that are worth + * reclaiming. Once found the rmap for the victim zone is queried, and after + * a bit of sorting to reduce fragmentation, the still live extents are read + * into memory and written to the GC target zone, and the bmap btree of the + * files is updated to point to the new location. To avoid taking the IOLOCK + * and MMAPLOCK for the entire GC process and thus affecting the latency of + * user reads and writes to the files, the GC writes are speculative and the + * I/O completion checks that no other writes happened for the affected regions + * before remapping. + * + * Once a zone does not contain any valid data, be that through GC or user + * block removal, it is queued for for a zone reset. The reset operation + * carefully ensures that the RT device cache is flushed and all transactions + * referencing the rmap have been committed to disk. + */ + +/* + * Size of each GC scratch allocation, and the number of buffers. + */ +#define XFS_GC_BUF_SIZE SZ_1M +#define XFS_GC_NR_BUFS 2 +static_assert(XFS_GC_NR_BUFS < BIO_MAX_VECS); + +/* + * Chunk that is read and written for each GC operation. + * + * Note that for writes to actual zoned devices, the chunk can be split when + * reaching the hardware limit. + */ +struct xfs_gc_bio { + struct xfs_zone_gc_data *data; + + /* + * Entry into the reading/writing/resetting list. Only accessed from + * the GC thread, so no locking needed. + */ + struct list_head entry; + + /* + * State of this gc_bio. Done means the current I/O completed. + * Set from the bio end I/O handler, read from the GC thread. + */ + enum { + XFS_GC_BIO_NEW, + XFS_GC_BIO_DONE, + } state; + + /* + * Pointer to the inode and byte range in the inode that this + * GC chunk is operating on. + */ + struct xfs_inode *ip; + loff_t offset; + unsigned int len; + + /* + * Existing startblock (in the zone to be freed) and newly assigned + * daddr in the zone GCed into. + */ + xfs_fsblock_t old_startblock; + xfs_daddr_t new_daddr; + + /* Are we writing to a sequential write required zone? */ + bool is_seq; + + /* Open Zone being written to */ + struct xfs_open_zone *oz; + + /* Realtime group currently being reclaimed */ + struct xfs_rtgroup *victim_rtg; + + /* Bio used for reads and writes, including the bvec used by it */ + struct bio bio; /* must be last */ +}; + +#define XFS_ZONE_GC_RECS 1024 + +/* iterator, needs to be reinitialized for each victim zone */ +struct xfs_zone_gc_iter { + struct xfs_rtgroup *victim_rtg; + unsigned int rec_count; + unsigned int rec_idx; + xfs_agblock_t next_startblock; + struct xfs_rmap_irec *recs; +}; + +/* + * Per-mount GC state. + */ +struct xfs_zone_gc_data { + struct xfs_mount *mp; + struct xfs_open_zone *oz; + + /* bioset used to allocate the gc_bios */ + struct bio_set bio_set; + + /* bioset used when writes need to be split to hardware limits */ + struct bio_set split_bio_set; + + /* + * Scratchpad to buffer GC data, organized as a ring buffer over + * discontiguous folios. scratch_head is where the buffer is filled, + * scratch_tail tracks the buffer space freed, and scratch_available + * counts the space available in the ring buffer between the head and + * the tail. + */ + struct folio *scratch_folios[XFS_GC_NR_BUFS]; + unsigned int scratch_size; + unsigned int scratch_available; + unsigned int scratch_head; + unsigned int scratch_tail; + + /* + * List of bios currently being read, written and reset. + * These lists are only accessed by the GC thread itself, and must only + * be processed in order. + */ + struct list_head reading; + struct list_head writing; + struct list_head resetting; + + /* + * Iterator for the victim zone. + */ + struct xfs_zone_gc_iter iter; +}; + +/* + * We aim to keep enough zones free in stock to fully use the open zone limit + * for data placement purposes. Additionally, the m_zonegc_low_space tunable + * can be set to make sure a fraction of the unused blocks are available for + * writing. + */ +bool +xfs_zoned_need_gc( + struct xfs_mount *mp) +{ + s64 available, free, threshold; + s32 remainder; + + /* If we have no reclaimable blocks, running GC is useless. */ + if (!xfs_zoned_have_reclaimable(mp->m_zone_info)) + return false; + + /* + * In order to avoid file fragmentation as much as possible, we should + * make sure that we can open enough zones. So trigger GC if the number + * of blocks immediately available for writes is lower than the total + * number of blocks from all possible open zones. + */ + available = xfs_estimate_freecounter(mp, XC_FREE_RTAVAILABLE); + if (available < + xfs_rtgs_to_rfsbs(mp, mp->m_max_open_zones - XFS_OPEN_GC_ZONES)) + return true; + + /* + * For cases where the user wants to be more aggressive with GC, + * the sysfs attribute zonegc_low_space may be set to a non zero value, + * to indicate that GC should try to maintain at least zonegc_low_space + * percent of the free space to be directly available for writing. Check + * this here. + */ + if (!mp->m_zonegc_low_space) + return false; + + free = xfs_estimate_freecounter(mp, XC_FREE_RTEXTENTS); + threshold = div_s64_rem(free, 100, &remainder); + threshold = threshold * mp->m_zonegc_low_space + + remainder * div_s64(mp->m_zonegc_low_space, 100); + + return available < threshold; +} + +static struct xfs_zone_gc_data * +xfs_zone_gc_data_alloc( + struct xfs_mount *mp) +{ + struct xfs_zone_gc_data *data; + int i; + + data = kzalloc_obj(*data); + if (!data) + return NULL; + data->iter.recs = kzalloc_objs(*data->iter.recs, XFS_ZONE_GC_RECS); + if (!data->iter.recs) + goto out_free_data; + + if (bioset_init(&data->bio_set, 16, offsetof(struct xfs_gc_bio, bio), + BIOSET_NEED_BVECS)) + goto out_free_recs; + if (bioset_init(&data->split_bio_set, 16, + offsetof(struct xfs_gc_bio, bio), 0)) + goto out_exit_bio_set; + for (i = 0; i < XFS_GC_NR_BUFS; i++) { + data->scratch_folios[i] = + folio_alloc(GFP_KERNEL, get_order(XFS_GC_BUF_SIZE)); + if (!data->scratch_folios[i]) + goto out_free_scratch; + } + data->scratch_size = XFS_GC_BUF_SIZE * XFS_GC_NR_BUFS; + data->scratch_available = data->scratch_size; + INIT_LIST_HEAD(&data->reading); + INIT_LIST_HEAD(&data->writing); + INIT_LIST_HEAD(&data->resetting); + data->mp = mp; + return data; + +out_free_scratch: + while (--i >= 0) + folio_put(data->scratch_folios[i]); + bioset_exit(&data->split_bio_set); +out_exit_bio_set: + bioset_exit(&data->bio_set); +out_free_recs: + kfree(data->iter.recs); +out_free_data: + kfree(data); + return NULL; +} + +static void +xfs_zone_gc_data_free( + struct xfs_zone_gc_data *data) +{ + int i; + + for (i = 0; i < XFS_GC_NR_BUFS; i++) + folio_put(data->scratch_folios[i]); + bioset_exit(&data->split_bio_set); + bioset_exit(&data->bio_set); + kfree(data->iter.recs); + kfree(data); +} + +static void +xfs_zone_gc_iter_init( + struct xfs_zone_gc_iter *iter, + struct xfs_rtgroup *victim_rtg) + +{ + iter->next_startblock = 0; + iter->rec_count = 0; + iter->rec_idx = 0; + iter->victim_rtg = victim_rtg; + atomic_inc(&victim_rtg->rtg_gccount); +} + +/* + * Query the rmap of the victim zone to gather the records to evacuate. + */ +static int +xfs_zone_gc_query_cb( + struct xfs_btree_cur *cur, + const struct xfs_rmap_irec *irec, + void *private) +{ + struct xfs_zone_gc_iter *iter = private; + + ASSERT(!XFS_RMAP_NON_INODE_OWNER(irec->rm_owner)); + ASSERT(!xfs_is_sb_inum(cur->bc_mp, irec->rm_owner)); + ASSERT(!(irec->rm_flags & (XFS_RMAP_ATTR_FORK | XFS_RMAP_BMBT_BLOCK))); + + iter->recs[iter->rec_count] = *irec; + if (++iter->rec_count == XFS_ZONE_GC_RECS) { + iter->next_startblock = + irec->rm_startblock + irec->rm_blockcount; + return 1; + } + return 0; +} + +static int +xfs_zone_gc_rmap_rec_cmp( + const void *a, + const void *b) +{ + const struct xfs_rmap_irec *reca = a; + const struct xfs_rmap_irec *recb = b; + int diff; + + diff = cmp_int(reca->rm_owner, recb->rm_owner); + if (diff) + return diff; + return cmp_int(reca->rm_offset, recb->rm_offset); +} + +static int +xfs_zone_gc_query( + struct xfs_mount *mp, + struct xfs_zone_gc_iter *iter) +{ + struct xfs_rtgroup *rtg = iter->victim_rtg; + struct xfs_rmap_irec ri_low = { }; + struct xfs_rmap_irec ri_high; + struct xfs_btree_cur *cur; + struct xfs_trans *tp; + int error; + + ASSERT(iter->next_startblock <= rtg_blocks(rtg)); + if (iter->next_startblock == rtg_blocks(rtg)) + goto done; + + ASSERT(iter->next_startblock < rtg_blocks(rtg)); + ri_low.rm_startblock = iter->next_startblock; + memset(&ri_high, 0xFF, sizeof(ri_high)); + + iter->rec_idx = 0; + iter->rec_count = 0; + + tp = xfs_trans_alloc_empty(mp); + xfs_rtgroup_lock(rtg, XFS_RTGLOCK_RMAP); + cur = xfs_rtrmapbt_init_cursor(tp, rtg); + error = xfs_rmap_query_range(cur, &ri_low, &ri_high, + xfs_zone_gc_query_cb, iter); + xfs_rtgroup_unlock(rtg, XFS_RTGLOCK_RMAP); + xfs_btree_del_cursor(cur, error < 0 ? error : 0); + xfs_trans_cancel(tp); + + if (error < 0) + return error; + + /* + * Sort the rmap records by inode number and increasing offset to + * defragment the mappings. + * + * This could be further enhanced by an even bigger look ahead window, + * but that's better left until we have better detection of changes to + * inode mapping to avoid the potential of GCing already dead data. + */ + sort(iter->recs, iter->rec_count, sizeof(iter->recs[0]), + xfs_zone_gc_rmap_rec_cmp, NULL); + + if (error == 0) { + /* + * We finished iterating through the zone. + */ + iter->next_startblock = rtg_blocks(rtg); + if (iter->rec_count == 0) + goto done; + } + + return 0; +done: + atomic_dec(&iter->victim_rtg->rtg_gccount); + xfs_rtgroup_rele(iter->victim_rtg); + iter->victim_rtg = NULL; + return 0; +} + +static bool +xfs_zone_gc_iter_irec( + struct xfs_mount *mp, + struct xfs_zone_gc_iter *iter, + struct xfs_rmap_irec *chunk_rec, + struct xfs_inode **ipp) +{ + struct xfs_rmap_irec *irec; + int error; + +retry: + if (iter->rec_idx == iter->rec_count) { + error = xfs_zone_gc_query(mp, iter); + if (error) + goto fail; + if (!iter->victim_rtg) + return false; + } + + irec = &iter->recs[iter->rec_idx]; + error = xfs_iget(mp, NULL, irec->rm_owner, + XFS_IGET_UNTRUSTED | XFS_IGET_DONTCACHE, 0, ipp); + if (error) { + /* + * If the inode was already deleted, skip over it. + */ + if (error == -ENOENT || error == -EINVAL) { + iter->rec_idx++; + goto retry; + } + goto fail; + } + + if (!S_ISREG(VFS_I(*ipp)->i_mode) || !XFS_IS_REALTIME_INODE(*ipp)) { + iter->rec_idx++; + xfs_irele(*ipp); + goto retry; + } + + *chunk_rec = *irec; + return true; + +fail: + xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR); + return false; +} + +static void +xfs_zone_gc_iter_advance( + struct xfs_zone_gc_iter *iter, + xfs_extlen_t count_fsb) +{ + struct xfs_rmap_irec *irec = &iter->recs[iter->rec_idx]; + + irec->rm_offset += count_fsb; + irec->rm_startblock += count_fsb; + irec->rm_blockcount -= count_fsb; + if (!irec->rm_blockcount) + iter->rec_idx++; +} + +static struct xfs_rtgroup * +xfs_zone_gc_pick_victim_from( + struct xfs_mount *mp, + uint32_t bucket) +{ + struct xfs_zone_info *zi = mp->m_zone_info; + uint32_t victim_used = U32_MAX; + struct xfs_rtgroup *victim_rtg = NULL; + uint32_t bit; + + if (!zi->zi_used_bucket_entries[bucket]) + return NULL; + + for_each_set_bit(bit, zi->zi_used_bucket_bitmap[bucket], + mp->m_sb.sb_rgcount) { + struct xfs_rtgroup *rtg = xfs_rtgroup_grab(mp, bit); + + if (!rtg) + continue; + + /* + * If the zone is already undergoing GC, don't pick it again. + * + * This prevents us from picking one of the zones for which we + * already submitted GC I/O, but for which the remapping hasn't + * concluded yet. This won't cause data corruption, but + * increases write amplification and slows down GC, so this is + * a bad thing. + */ + if (atomic_read(&rtg->rtg_gccount)) { + xfs_rtgroup_rele(rtg); + continue; + } + + /* skip zones that are just waiting for a reset */ + if (rtg_rmap(rtg)->i_used_blocks == 0 || + rtg_rmap(rtg)->i_used_blocks >= victim_used) { + xfs_rtgroup_rele(rtg); + continue; + } + + if (victim_rtg) + xfs_rtgroup_rele(victim_rtg); + victim_rtg = rtg; + victim_used = rtg_rmap(rtg)->i_used_blocks; + + /* + * Any zone that is less than 1 percent used is fair game for + * instant reclaim. All of these zones are in the last + * bucket, so avoid the expensive division for the zones + * in the other buckets. + */ + if (bucket == 0 && + rtg_rmap(rtg)->i_used_blocks < rtg_blocks(rtg) / 100) + break; + } + + return victim_rtg; +} + +/* + * Iterate through all zones marked as reclaimable and find a candidate to + * reclaim. + */ +static bool +xfs_zone_gc_select_victim( + struct xfs_zone_gc_data *data) +{ + struct xfs_zone_gc_iter *iter = &data->iter; + struct xfs_mount *mp = data->mp; + struct xfs_zone_info *zi = mp->m_zone_info; + struct xfs_rtgroup *victim_rtg = NULL; + unsigned int bucket; + + spin_lock(&zi->zi_used_buckets_lock); + for (bucket = 0; bucket < XFS_ZONE_USED_BUCKETS; bucket++) { + victim_rtg = xfs_zone_gc_pick_victim_from(mp, bucket); + if (victim_rtg) + break; + } + spin_unlock(&zi->zi_used_buckets_lock); + + if (!victim_rtg) + return false; + + trace_xfs_zone_gc_select_victim(victim_rtg, bucket); + xfs_zone_gc_iter_init(iter, victim_rtg); + return true; +} + +static int +xfs_zone_gc_steal_open_zone( + struct xfs_zone_gc_data *data) +{ + struct xfs_zone_info *zi = data->mp->m_zone_info; + struct xfs_open_zone *oz, *found = NULL; + + spin_lock(&zi->zi_open_zones_lock); + list_for_each_entry(oz, &zi->zi_open_zones, oz_entry) { + if (!found || oz->oz_allocated < found->oz_allocated) + found = oz; + } + if (!found) { + spin_unlock(&zi->zi_open_zones_lock); + return -EIO; + } + + trace_xfs_zone_gc_target_stolen(found->oz_rtg); + found->oz_is_gc = true; + zi->zi_nr_open_zones--; + zi->zi_nr_open_gc_zones++; + spin_unlock(&zi->zi_open_zones_lock); + + atomic_inc(&found->oz_ref); + data->oz = found; + return 0; +} + +/* + * Ensure we have a valid open zone to write to. + */ +static bool +xfs_zone_gc_select_target( + struct xfs_zone_gc_data *data) +{ + struct xfs_zone_info *zi = data->mp->m_zone_info; + + if (data->oz) { + /* + * If we have space available, just keep using the existing + * zone. + */ + if (data->oz->oz_allocated < rtg_blocks(data->oz->oz_rtg)) + return true; + + /* + * Wait for all writes to the current zone to finish before + * picking a new one. + */ + if (data->oz->oz_written < rtg_blocks(data->oz->oz_rtg)) + return false; + + xfs_open_zone_put(data->oz); + } + + /* + * Open a new zone when there is none currently in use. + */ + ASSERT(zi->zi_nr_open_zones <= + data->mp->m_max_open_zones - XFS_OPEN_GC_ZONES); + data->oz = xfs_open_zone(data->mp, WRITE_LIFE_NOT_SET, true); + if (!data->oz) + return false; + trace_xfs_zone_gc_target_opened(data->oz->oz_rtg); + atomic_inc(&data->oz->oz_ref); + spin_lock(&zi->zi_open_zones_lock); + zi->zi_nr_open_gc_zones++; + list_add_tail(&data->oz->oz_entry, &zi->zi_open_zones); + spin_unlock(&zi->zi_open_zones_lock); + return true; +} + +static void +xfs_zone_gc_end_io( + struct bio *bio) +{ + struct xfs_gc_bio *chunk = + container_of(bio, struct xfs_gc_bio, bio); + struct xfs_zone_gc_data *data = chunk->data; + + WRITE_ONCE(chunk->state, XFS_GC_BIO_DONE); + wake_up_process(data->mp->m_zone_info->zi_gc_thread); +} + +static bool +xfs_zone_gc_alloc_blocks( + struct xfs_zone_gc_data *data, + xfs_extlen_t *count_fsb, + xfs_daddr_t *daddr, + bool *is_seq) +{ + struct xfs_mount *mp = data->mp; + struct xfs_open_zone *oz = data->oz; + + *count_fsb = min(*count_fsb, XFS_B_TO_FSB(mp, data->scratch_available)); + + /* + * Directly allocate GC blocks from the reserved pool. + * + * If we'd take them from the normal pool we could be stealing blocks + * from a regular writer, which would then have to wait for GC and + * deadlock. + */ + spin_lock(&mp->m_sb_lock); + *count_fsb = min(*count_fsb, + rtg_blocks(oz->oz_rtg) - oz->oz_allocated); + *count_fsb = min3(*count_fsb, + mp->m_free[XC_FREE_RTEXTENTS].res_avail, + mp->m_free[XC_FREE_RTAVAILABLE].res_avail); + mp->m_free[XC_FREE_RTEXTENTS].res_avail -= *count_fsb; + mp->m_free[XC_FREE_RTAVAILABLE].res_avail -= *count_fsb; + spin_unlock(&mp->m_sb_lock); + + if (!*count_fsb) + return false; + + *daddr = xfs_gbno_to_daddr(rtg_group(oz->oz_rtg), 0); + *is_seq = bdev_zone_is_seq(mp->m_rtdev_targp->bt_bdev, *daddr); + if (!*is_seq) + *daddr += XFS_FSB_TO_BB(mp, oz->oz_allocated); + oz->oz_allocated += *count_fsb; + atomic_inc(&oz->oz_ref); + return true; +} + +static void +xfs_zone_gc_add_data( + struct xfs_gc_bio *chunk) +{ + struct xfs_zone_gc_data *data = chunk->data; + unsigned int len = chunk->len; + unsigned int off = data->scratch_head; + + do { + unsigned int this_off = off % XFS_GC_BUF_SIZE; + unsigned int this_len = min(len, XFS_GC_BUF_SIZE - this_off); + + bio_add_folio_nofail(&chunk->bio, + data->scratch_folios[off / XFS_GC_BUF_SIZE], + this_len, this_off); + len -= this_len; + off += this_len; + if (off == data->scratch_size) + off = 0; + } while (len); +} + +static bool +xfs_zone_gc_can_start_chunk( + struct xfs_zone_gc_data *data) +{ + + if (xfs_is_shutdown(data->mp)) + return false; + if (!data->scratch_available) + return false; + + if (!data->iter.victim_rtg) { + if (kthread_should_stop() || kthread_should_park()) + return false; + if (!xfs_zoned_need_gc(data->mp)) + return false; + if (!xfs_zone_gc_select_victim(data)) + return false; + } + + return xfs_zone_gc_select_target(data); +} + +static bool +xfs_zone_gc_start_chunk( + struct xfs_zone_gc_data *data) +{ + struct xfs_zone_gc_iter *iter = &data->iter; + struct xfs_mount *mp = data->mp; + struct block_device *bdev = mp->m_rtdev_targp->bt_bdev; + struct xfs_rmap_irec irec; + struct xfs_gc_bio *chunk; + struct xfs_inode *ip; + struct bio *bio; + xfs_daddr_t daddr; + bool is_seq; + + if (!xfs_zone_gc_can_start_chunk(data)) + return false; + + set_current_state(TASK_RUNNING); + if (!xfs_zone_gc_iter_irec(mp, iter, &irec, &ip)) + return false; + + if (!xfs_zone_gc_alloc_blocks(data, &irec.rm_blockcount, &daddr, + &is_seq)) { + xfs_irele(ip); + return false; + } + + /* + * Scratch allocation can wrap around to the same buffer again, + * provision an extra bvec for that case. + */ + bio = bio_alloc_bioset(bdev, XFS_GC_NR_BUFS + 1, REQ_OP_READ, GFP_NOFS, + &data->bio_set); + chunk = container_of(bio, struct xfs_gc_bio, bio); + chunk->ip = ip; + chunk->offset = XFS_FSB_TO_B(mp, irec.rm_offset); + chunk->len = XFS_FSB_TO_B(mp, irec.rm_blockcount); + chunk->old_startblock = + xfs_rgbno_to_rtb(iter->victim_rtg, irec.rm_startblock); + chunk->new_daddr = daddr; + chunk->is_seq = is_seq; + chunk->data = data; + chunk->oz = data->oz; + chunk->victim_rtg = iter->victim_rtg; + atomic_inc(&rtg_group(chunk->victim_rtg)->xg_active_ref); + atomic_inc(&chunk->victim_rtg->rtg_gccount); + + bio->bi_iter.bi_sector = xfs_rtb_to_daddr(mp, chunk->old_startblock); + bio->bi_end_io = xfs_zone_gc_end_io; + xfs_zone_gc_add_data(chunk); + data->scratch_head = + (data->scratch_head + chunk->len) % data->scratch_size; + data->scratch_available -= chunk->len; + + XFS_STATS_INC(mp, xs_gc_read_calls); + + WRITE_ONCE(chunk->state, XFS_GC_BIO_NEW); + list_add_tail(&chunk->entry, &data->reading); + xfs_zone_gc_iter_advance(iter, irec.rm_blockcount); + + submit_bio(bio); + return true; +} + +static void +xfs_zone_gc_free_chunk( + struct xfs_gc_bio *chunk) +{ + atomic_dec(&chunk->victim_rtg->rtg_gccount); + xfs_rtgroup_rele(chunk->victim_rtg); + list_del(&chunk->entry); + xfs_open_zone_put(chunk->oz); + xfs_irele(chunk->ip); + bio_put(&chunk->bio); +} + +static void +xfs_zone_gc_submit_write( + struct xfs_zone_gc_data *data, + struct xfs_gc_bio *chunk) +{ + if (chunk->is_seq) { + chunk->bio.bi_opf &= ~REQ_OP_WRITE; + chunk->bio.bi_opf |= REQ_OP_ZONE_APPEND; + } + chunk->bio.bi_iter.bi_sector = chunk->new_daddr; + chunk->bio.bi_end_io = xfs_zone_gc_end_io; + submit_bio(&chunk->bio); +} + +static struct xfs_gc_bio * +xfs_zone_gc_split_write( + struct xfs_zone_gc_data *data, + struct xfs_gc_bio *chunk) +{ + struct queue_limits *lim = + &bdev_get_queue(chunk->bio.bi_bdev)->limits; + struct xfs_gc_bio *split_chunk; + int split_sectors; + unsigned int split_len; + struct bio *split; + unsigned int nsegs; + + if (!chunk->is_seq) + return NULL; + + split_sectors = bio_split_rw_at(&chunk->bio, lim, &nsegs, + lim->max_zone_append_sectors << SECTOR_SHIFT); + if (split_sectors <= 0) + return NULL; + + /* ensure the split chunk is still block size aligned */ + split_sectors = ALIGN_DOWN(split_sectors << SECTOR_SHIFT, + data->mp->m_sb.sb_blocksize) >> SECTOR_SHIFT; + split_len = split_sectors << SECTOR_SHIFT; + + split = bio_split(&chunk->bio, split_sectors, GFP_NOFS, + &data->split_bio_set); + split_chunk = container_of(split, struct xfs_gc_bio, bio); + split_chunk->data = data; + ihold(VFS_I(chunk->ip)); + split_chunk->ip = chunk->ip; + split_chunk->is_seq = chunk->is_seq; + split_chunk->offset = chunk->offset; + split_chunk->len = split_len; + split_chunk->old_startblock = chunk->old_startblock; + split_chunk->new_daddr = chunk->new_daddr; + split_chunk->oz = chunk->oz; + atomic_inc(&chunk->oz->oz_ref); + + split_chunk->victim_rtg = chunk->victim_rtg; + atomic_inc(&rtg_group(chunk->victim_rtg)->xg_active_ref); + atomic_inc(&chunk->victim_rtg->rtg_gccount); + + chunk->offset += split_len; + chunk->len -= split_len; + chunk->old_startblock += XFS_B_TO_FSB(data->mp, split_len); + + /* add right before the original chunk */ + WRITE_ONCE(split_chunk->state, XFS_GC_BIO_NEW); + list_add_tail(&split_chunk->entry, &chunk->entry); + return split_chunk; +} + +static void +xfs_zone_gc_write_chunk( + struct xfs_gc_bio *chunk) +{ + struct xfs_zone_gc_data *data = chunk->data; + struct xfs_mount *mp = chunk->ip->i_mount; + struct xfs_gc_bio *split_chunk; + + if (chunk->bio.bi_status) + xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR); + if (xfs_is_shutdown(mp)) { + xfs_zone_gc_free_chunk(chunk); + return; + } + + XFS_STATS_INC(mp, xs_gc_write_calls); + XFS_STATS_ADD(mp, xs_gc_bytes, chunk->len); + + WRITE_ONCE(chunk->state, XFS_GC_BIO_NEW); + list_move_tail(&chunk->entry, &data->writing); + + bio_reuse(&chunk->bio, REQ_OP_WRITE); + while ((split_chunk = xfs_zone_gc_split_write(data, chunk))) + xfs_zone_gc_submit_write(data, split_chunk); + xfs_zone_gc_submit_write(data, chunk); +} + +static void +xfs_zone_gc_finish_chunk( + struct xfs_gc_bio *chunk) +{ + uint iolock = XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL; + struct xfs_zone_gc_data *data = chunk->data; + struct xfs_inode *ip = chunk->ip; + struct xfs_mount *mp = ip->i_mount; + int error; + + if (chunk->bio.bi_status) + xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR); + if (xfs_is_shutdown(mp)) { + xfs_zone_gc_free_chunk(chunk); + return; + } + + data->scratch_tail = + (data->scratch_tail + chunk->len) % data->scratch_size; + data->scratch_available += chunk->len; + + /* + * Cycle through the iolock and wait for direct I/O and layouts to + * ensure no one is reading from the old mapping before it goes away. + * + * Note that xfs_zoned_end_io() below checks that no other writer raced + * with us to update the mapping by checking that the old startblock + * didn't change. + */ + xfs_ilock(ip, iolock); + error = xfs_break_layouts(VFS_I(ip), &iolock, BREAK_UNMAP); + if (!error) + inode_dio_wait(VFS_I(ip)); + xfs_iunlock(ip, iolock); + if (error) + goto free; + + if (chunk->is_seq) + chunk->new_daddr = chunk->bio.bi_iter.bi_sector; + error = xfs_zoned_end_io(ip, chunk->offset, chunk->len, + chunk->new_daddr, chunk->oz, chunk->old_startblock); +free: + if (error) + xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR); + xfs_zone_gc_free_chunk(chunk); +} + +static void +xfs_zone_gc_finish_reset( + struct xfs_gc_bio *chunk) +{ + struct xfs_rtgroup *rtg = chunk->bio.bi_private; + struct xfs_mount *mp = rtg_mount(rtg); + struct xfs_zone_info *zi = mp->m_zone_info; + + if (chunk->bio.bi_status) { + xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR); + goto out; + } + + xfs_zone_mark_free(rtg); + xfs_zoned_add_available(mp, rtg_blocks(rtg)); + + wake_up_all(&zi->zi_zone_wait); +out: + list_del(&chunk->entry); + bio_put(&chunk->bio); +} + +static void +xfs_submit_zone_reset_bio( + struct bio *bio, + void *priv) +{ + struct xfs_rtgroup *rtg = priv; + struct xfs_mount *mp = rtg_mount(rtg); + + trace_xfs_zone_reset(rtg); + + ASSERT(rtg_rmap(rtg)->i_used_blocks == 0); + + if (XFS_TEST_ERROR(mp, XFS_ERRTAG_ZONE_RESET)) { + bio_io_error(bio); + return; + } + + XFS_STATS_INC(mp, xs_gc_zone_reset_calls); + + bio->bi_iter.bi_sector = xfs_gbno_to_daddr(rtg_group(rtg), 0); + if (!bdev_zone_is_seq(bio->bi_bdev, bio->bi_iter.bi_sector)) { + /* + * Also use the bio to drive the state machine when neither + * zone reset nor discard is supported to keep things simple. + */ + if (!bdev_max_discard_sectors(bio->bi_bdev)) { + bio_endio(bio); + return; + } + bio->bi_opf &= ~REQ_OP_ZONE_RESET; + bio->bi_opf |= REQ_OP_DISCARD; + bio->bi_iter.bi_size = XFS_FSB_TO_B(mp, rtg_blocks(rtg)); + } + + submit_bio(bio); +} + +int +xfs_zone_gc_reset_sync( + struct xfs_rtgroup *rtg) +{ + struct bio bio; + int error; + + bio_init(&bio, rtg_mount(rtg)->m_rtdev_targp->bt_bdev, NULL, 0, + REQ_OP_ZONE_RESET | REQ_SYNC); + bio_await(&bio, rtg, xfs_submit_zone_reset_bio); + error = blk_status_to_errno(bio.bi_status); + bio_uninit(&bio); + return error; +} + +static void +xfs_zone_gc_reset_zones( + struct xfs_zone_gc_data *data, + struct xfs_group *reset_list) +{ + struct xfs_group *next = reset_list; + + if (blkdev_issue_flush(data->mp->m_rtdev_targp->bt_bdev) < 0) { + xfs_force_shutdown(data->mp, SHUTDOWN_META_IO_ERROR); + return; + } + + do { + struct xfs_rtgroup *rtg = to_rtg(next); + struct xfs_gc_bio *chunk; + struct bio *bio; + + xfs_log_force_inode(rtg_rmap(rtg)); + + next = rtg_group(rtg)->xg_next_reset; + rtg_group(rtg)->xg_next_reset = NULL; + + bio = bio_alloc_bioset(rtg_mount(rtg)->m_rtdev_targp->bt_bdev, + 0, REQ_OP_ZONE_RESET, GFP_NOFS, &data->bio_set); + bio->bi_private = rtg; + bio->bi_end_io = xfs_zone_gc_end_io; + + chunk = container_of(bio, struct xfs_gc_bio, bio); + chunk->data = data; + WRITE_ONCE(chunk->state, XFS_GC_BIO_NEW); + list_add_tail(&chunk->entry, &data->resetting); + xfs_submit_zone_reset_bio(bio, rtg); + } while (next); +} + +/* + * Handle the work to read and write data for GC and to reset the zones, + * including handling all completions. + * + * Note that the order of the chunks is preserved so that we don't undo the + * optimal order established by xfs_zone_gc_query(). + */ +static void +xfs_zone_gc_handle_work( + struct xfs_zone_gc_data *data) +{ + struct xfs_zone_info *zi = data->mp->m_zone_info; + struct xfs_gc_bio *chunk, *next; + struct xfs_group *reset_list; + struct blk_plug plug; + + spin_lock(&zi->zi_reset_list_lock); + reset_list = zi->zi_reset_list; + zi->zi_reset_list = NULL; + spin_unlock(&zi->zi_reset_list_lock); + + if (reset_list) { + set_current_state(TASK_RUNNING); + xfs_zone_gc_reset_zones(data, reset_list); + } + + list_for_each_entry_safe(chunk, next, &data->resetting, entry) { + if (READ_ONCE(chunk->state) != XFS_GC_BIO_DONE) + break; + set_current_state(TASK_RUNNING); + xfs_zone_gc_finish_reset(chunk); + } + + list_for_each_entry_safe(chunk, next, &data->writing, entry) { + if (READ_ONCE(chunk->state) != XFS_GC_BIO_DONE) + break; + set_current_state(TASK_RUNNING); + xfs_zone_gc_finish_chunk(chunk); + } + + blk_start_plug(&plug); + list_for_each_entry_safe(chunk, next, &data->reading, entry) { + if (READ_ONCE(chunk->state) != XFS_GC_BIO_DONE) + break; + set_current_state(TASK_RUNNING); + xfs_zone_gc_write_chunk(chunk); + } + blk_finish_plug(&plug); + + blk_start_plug(&plug); + while (xfs_zone_gc_start_chunk(data)) + ; + blk_finish_plug(&plug); +} + +/* + * Note that the current GC algorithm would break reflinks and thus duplicate + * data that was shared by multiple owners before. Because of that reflinks + * are currently not supported on zoned file systems and can't be created or + * mounted. + */ +static int +xfs_zoned_gcd( + void *private) +{ + struct xfs_zone_gc_data *data = private; + struct xfs_mount *mp = data->mp; + struct xfs_zone_info *zi = mp->m_zone_info; + unsigned int nofs_flag; + + nofs_flag = memalloc_nofs_save(); + set_freezable(); + + for (;;) { + set_current_state(TASK_INTERRUPTIBLE | TASK_FREEZABLE); + xfs_set_zonegc_running(mp); + + xfs_zone_gc_handle_work(data); + + /* + * Only sleep if nothing set the state to running. Else check for + * work again as someone might have queued up more work and woken + * us in the meantime. + */ + if (get_current_state() == TASK_RUNNING) { + try_to_freeze(); + continue; + } + + if (list_empty(&data->reading) && + list_empty(&data->writing) && + list_empty(&data->resetting) && + !zi->zi_reset_list) { + xfs_clear_zonegc_running(mp); + xfs_zoned_resv_wake_all(mp); + + if (kthread_should_stop()) { + __set_current_state(TASK_RUNNING); + break; + } + + if (kthread_should_park()) { + __set_current_state(TASK_RUNNING); + kthread_parkme(); + continue; + } + } + + schedule(); + } + xfs_clear_zonegc_running(mp); + + if (data->oz) + xfs_open_zone_put(data->oz); + if (data->iter.victim_rtg) + xfs_rtgroup_rele(data->iter.victim_rtg); + + memalloc_nofs_restore(nofs_flag); + xfs_zone_gc_data_free(data); + return 0; +} + +void +xfs_zone_gc_start( + struct xfs_mount *mp) +{ + if (xfs_has_zoned(mp)) + kthread_unpark(mp->m_zone_info->zi_gc_thread); +} + +void +xfs_zone_gc_stop( + struct xfs_mount *mp) +{ + if (xfs_has_zoned(mp)) + kthread_park(mp->m_zone_info->zi_gc_thread); +} + +void +xfs_zone_gc_wakeup( + struct xfs_mount *mp) +{ + struct super_block *sb = mp->m_super; + + /* + * If we are unmounting the file system we must not try to + * wake gc as m_zone_info might have been freed already. + */ + if (down_read_trylock(&sb->s_umount)) { + if (!xfs_is_readonly(mp)) + wake_up_process(mp->m_zone_info->zi_gc_thread); + up_read(&sb->s_umount); + } +} + +int +xfs_zone_gc_mount( + struct xfs_mount *mp) +{ + struct xfs_zone_info *zi = mp->m_zone_info; + struct xfs_zone_gc_data *data; + int error; + + data = xfs_zone_gc_data_alloc(mp); + if (!data) + return -ENOMEM; + + /* + * If there are no free zones available for GC, or the number of open + * zones has reached the open zone limit, pick the open zone with + * the least used space to GC into. This should only happen after an + * unclean shutdown while GC was ongoing. Otherwise a GC zone will + * be selected from the free zone pool on demand. + */ + if (!xfs_group_marked(mp, XG_TYPE_RTG, XFS_RTG_FREE) || + zi->zi_nr_open_zones >= mp->m_max_open_zones) { + error = xfs_zone_gc_steal_open_zone(data); + if (error) { + xfs_warn(mp, "unable to steal an open zone for gc"); + goto out_free_gc_data; + } + } + + zi->zi_gc_thread = kthread_create(xfs_zoned_gcd, data, + "xfs-zone-gc/%s", mp->m_super->s_id); + if (IS_ERR(zi->zi_gc_thread)) { + xfs_warn(mp, "unable to create zone gc thread"); + error = PTR_ERR(zi->zi_gc_thread); + goto out_put_oz; + } + + /* xfs_zone_gc_start will unpark for rw mounts */ + kthread_park(zi->zi_gc_thread); + return 0; + +out_put_oz: + if (data->oz) + xfs_open_zone_put(data->oz); +out_free_gc_data: + xfs_zone_gc_data_free(data); + return error; +} + +void +xfs_zone_gc_unmount( + struct xfs_mount *mp) +{ + struct xfs_zone_info *zi = mp->m_zone_info; + + kthread_stop(zi->zi_gc_thread); +} diff --git a/fs/xfs/xfs_zone_info.c b/fs/xfs/xfs_zone_info.c new file mode 100644 index 000000000..47b475e21 --- /dev/null +++ b/fs/xfs/xfs_zone_info.c @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2023-2025 Christoph Hellwig. + * Copyright (c) 2024-2025, Western Digital Corporation or its affiliates. + */ +#include "xfs_platform.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_rtgroup.h" +#include "xfs_zone_alloc.h" +#include "xfs_zone_priv.h" + +static const char xfs_write_hint_shorthand[6][16] = { + "NOT_SET", "NONE", "SHORT", "MEDIUM", "LONG", "EXTREME"}; + +static inline const char * +xfs_write_hint_to_str( + uint8_t write_hint) +{ + if (write_hint > WRITE_LIFE_EXTREME) + return "UNKNOWN"; + return xfs_write_hint_shorthand[write_hint]; +} + +static void +xfs_show_open_zone( + struct seq_file *m, + struct xfs_open_zone *oz) +{ + seq_printf(m, "\t zone %d, wp %u, written %u, used %u, hint %s %s\n", + rtg_rgno(oz->oz_rtg), + oz->oz_allocated, oz->oz_written, + rtg_rmap(oz->oz_rtg)->i_used_blocks, + xfs_write_hint_to_str(oz->oz_write_hint), + oz->oz_is_gc ? "(GC)" : ""); +} + +static void +xfs_show_full_zone_used_distribution( + struct seq_file *m, + struct xfs_mount *mp) +{ + struct xfs_zone_info *zi = mp->m_zone_info; + unsigned int reclaimable = 0, full, i; + + spin_lock(&zi->zi_used_buckets_lock); + for (i = 0; i < XFS_ZONE_USED_BUCKETS; i++) { + unsigned int entries = zi->zi_used_bucket_entries[i]; + + seq_printf(m, "\t %2u..%2u%%: %u\n", + i * (100 / XFS_ZONE_USED_BUCKETS), + (i + 1) * (100 / XFS_ZONE_USED_BUCKETS) - 1, + entries); + reclaimable += entries; + } + spin_unlock(&zi->zi_used_buckets_lock); + + full = mp->m_sb.sb_rgcount; + full -= zi->zi_nr_open_zones; + full -= zi->zi_nr_open_gc_zones; + full -= atomic_read(&zi->zi_nr_free_zones); + full -= reclaimable; + + seq_printf(m, "\t 100%%: %u\n", full); +} + +void +xfs_zoned_show_stats( + struct seq_file *m, + struct xfs_mount *mp) +{ + struct xfs_zone_info *zi = mp->m_zone_info; + struct xfs_open_zone *oz; + + seq_puts(m, "\n"); + + seq_printf(m, "\tuser free RT blocks: %lld\n", + xfs_sum_freecounter(mp, XC_FREE_RTEXTENTS)); + seq_printf(m, "\treserved free RT blocks: %lld\n", + mp->m_free[XC_FREE_RTEXTENTS].res_avail); + seq_printf(m, "\tuser available RT blocks: %lld\n", + xfs_sum_freecounter(mp, XC_FREE_RTAVAILABLE)); + seq_printf(m, "\treserved available RT blocks: %lld\n", + mp->m_free[XC_FREE_RTAVAILABLE].res_avail); + seq_printf(m, "\tRT reservations required: %d\n", + !list_empty_careful(&zi->zi_reclaim_reservations)); + seq_printf(m, "\tRT GC required: %d\n", + xfs_zoned_need_gc(mp)); + + seq_printf(m, "\ttotal number of zones: %u\n", + mp->m_sb.sb_rgcount); + seq_printf(m, "\tfree zones: %d\n", atomic_read(&zi->zi_nr_free_zones)); + + spin_lock(&zi->zi_open_zones_lock); + seq_printf(m, "\tmax open zones: %u\n", + mp->m_max_open_zones); + seq_printf(m, "\tnr open zones: %u\n", + zi->zi_nr_open_zones); + seq_printf(m, "\tnr open GC zones: %u\n", + zi->zi_nr_open_gc_zones); + seq_puts(m, "\topen zones:\n"); + list_for_each_entry(oz, &zi->zi_open_zones, oz_entry) + xfs_show_open_zone(m, oz); + spin_unlock(&zi->zi_open_zones_lock); + seq_puts(m, "\tused blocks distribution (fully written zones):\n"); + xfs_show_full_zone_used_distribution(m, mp); +} diff --git a/fs/xfs/xfs_zone_priv.h b/fs/xfs/xfs_zone_priv.h new file mode 100644 index 000000000..fcb57506d --- /dev/null +++ b/fs/xfs/xfs_zone_priv.h @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _XFS_ZONE_PRIV_H +#define _XFS_ZONE_PRIV_H + +struct xfs_open_zone { + /* + * Entry in the open zone list and refcount. Protected by + * zi_open_zones_lock in struct xfs_zone_info. + */ + struct list_head oz_entry; + atomic_t oz_ref; + + /* + * oz_allocated is the amount of space already allocated out of the zone + * and is protected by oz_alloc_lock. + * + * For conventional zones it also is the offset of the next write. + */ + spinlock_t oz_alloc_lock; + xfs_rgblock_t oz_allocated; + + /* + * oz_written is the number of blocks for which we've received a write + * completion. oz_written must always be <= oz_allocated and is + * protected by the ILOCK of the rmap inode. + */ + xfs_rgblock_t oz_written; + + /* + * Write hint (data temperature) assigned to this zone, or + * WRITE_LIFE_NOT_SET if none was set. + */ + enum rw_hint oz_write_hint; + + /* Is this open zone used for garbage collection? */ + bool oz_is_gc; + + /* + * Pointer to the RT groups structure for this open zone. Constant over + * the life time of an open zone. + */ + struct xfs_rtgroup *oz_rtg; + + struct rcu_head oz_rcu; +}; + +/* + * Number of bitmap buckets to track reclaimable zones. There are 10 buckets + * so that each 10% of the usable capacity get their own bucket and GC can + * only has to walk the bitmaps of the lesser used zones if there are any. + */ +#define XFS_ZONE_USED_BUCKETS 10u + +struct xfs_zone_info { + /* + * List of pending space reservations: + */ + spinlock_t zi_reservation_lock; + struct list_head zi_reclaim_reservations; + + /* + * List and number of open zones: + */ + spinlock_t zi_open_zones_lock; + struct list_head zi_open_zones; + unsigned int zi_nr_open_zones; + unsigned int zi_nr_open_gc_zones; + + /* + * Free zone search cursor and number of free zones: + */ + atomic_t zi_nr_free_zones; + + /* + * Wait queue to wait for free zones or open zone resources to become + * available: + */ + wait_queue_head_t zi_zone_wait; + + /* + * Pointer to the GC thread. + */ + struct task_struct *zi_gc_thread; + + /* + * List of zones that need a reset: + */ + spinlock_t zi_reset_list_lock; + struct xfs_group *zi_reset_list; + + /* + * A set of bitmaps to bucket-sort reclaimable zones by used blocks to help + * garbage collection to quickly find the best candidate for reclaim. + */ + spinlock_t zi_used_buckets_lock; + unsigned int zi_used_bucket_entries[XFS_ZONE_USED_BUCKETS]; + unsigned long *zi_used_bucket_bitmap[XFS_ZONE_USED_BUCKETS]; + +}; + +struct xfs_open_zone *xfs_open_zone(struct xfs_mount *mp, + enum rw_hint write_hint, bool is_gc); + +int xfs_zone_gc_reset_sync(struct xfs_rtgroup *rtg); +bool xfs_zoned_need_gc(struct xfs_mount *mp); +bool xfs_zoned_have_reclaimable(struct xfs_zone_info *zi); +int xfs_zone_gc_mount(struct xfs_mount *mp); +void xfs_zone_gc_unmount(struct xfs_mount *mp); + +void xfs_zoned_resv_wake_all(struct xfs_mount *mp); + +#endif /* _XFS_ZONE_PRIV_H */ diff --git a/fs/xfs/xfs_zone_space_resv.c b/fs/xfs/xfs_zone_space_resv.c new file mode 100644 index 000000000..5c6e6ef62 --- /dev/null +++ b/fs/xfs/xfs_zone_space_resv.c @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2023-2025 Christoph Hellwig. + * Copyright (c) 2024-2025, Western Digital Corporation or its affiliates. + */ +#include "xfs_platform.h" +#include "xfs_shared.h" +#include "xfs_format.h" +#include "xfs_trans_resv.h" +#include "xfs_mount.h" +#include "xfs_inode.h" +#include "xfs_rtbitmap.h" +#include "xfs_icache.h" +#include "xfs_zone_alloc.h" +#include "xfs_zone_priv.h" +#include "xfs_zones.h" + +/* + * Note: the zoned allocator does not support a rtextsize > 1, so this code and + * the allocator itself uses file system blocks interchangeable with realtime + * extents without doing the otherwise required conversions. + */ + +/* + * Per-task space reservation. + * + * Tasks that need to wait for GC to free up space allocate one of these + * on-stack and adds it to the per-mount zi_reclaim_reservations lists. + * The GC thread will then wake the tasks in order when space becomes available. + */ +struct xfs_zone_reservation { + struct list_head entry; + struct task_struct *task; + xfs_filblks_t count_fsb; +}; + +/* + * Calculate the number of reserved blocks. + * + * XC_FREE_RTEXTENTS counts the user available capacity, to which the file + * system can be filled, while XC_FREE_RTAVAILABLE counts the blocks instantly + * available for writes without waiting for GC. + * + * For XC_FREE_RTAVAILABLE only the smaller reservation required for GC and + * block zeroing is excluded from the user capacity, while XC_FREE_RTEXTENTS + * is further restricted by at least one zone as well as the optional + * persistently reserved blocks. This allows the allocator to run more + * smoothly by not always triggering GC. + */ +uint64_t +xfs_zoned_default_resblks( + struct xfs_mount *mp, + enum xfs_free_counter ctr) +{ + switch (ctr) { + case XC_FREE_RTEXTENTS: + return xfs_rtgs_to_rfsbs(mp, XFS_RESERVED_ZONES) + + mp->m_sb.sb_rtreserved; + case XC_FREE_RTAVAILABLE: + return xfs_rtgs_to_rfsbs(mp, XFS_GC_ZONES); + default: + ASSERT(0); + return 0; + } +} + +void +xfs_zoned_resv_wake_all( + struct xfs_mount *mp) +{ + struct xfs_zone_info *zi = mp->m_zone_info; + struct xfs_zone_reservation *reservation; + + spin_lock(&zi->zi_reservation_lock); + list_for_each_entry(reservation, &zi->zi_reclaim_reservations, entry) + wake_up_process(reservation->task); + spin_unlock(&zi->zi_reservation_lock); +} + +void +xfs_zoned_add_available( + struct xfs_mount *mp, + xfs_filblks_t count_fsb) +{ + struct xfs_zone_info *zi = mp->m_zone_info; + struct xfs_zone_reservation *reservation; + + if (list_empty_careful(&zi->zi_reclaim_reservations)) { + xfs_add_freecounter(mp, XC_FREE_RTAVAILABLE, count_fsb); + return; + } + + spin_lock(&zi->zi_reservation_lock); + xfs_add_freecounter(mp, XC_FREE_RTAVAILABLE, count_fsb); + count_fsb = xfs_sum_freecounter(mp, XC_FREE_RTAVAILABLE); + list_for_each_entry(reservation, &zi->zi_reclaim_reservations, entry) { + if (reservation->count_fsb > count_fsb) + break; + wake_up_process(reservation->task); + count_fsb -= reservation->count_fsb; + + } + spin_unlock(&zi->zi_reservation_lock); +} + +static int +xfs_zoned_space_wait_error( + struct xfs_mount *mp) +{ + if (xfs_is_shutdown(mp)) + return -EIO; + if (fatal_signal_pending(current)) + return -EINTR; + return 0; +} + +static int +xfs_zoned_reserve_available( + struct xfs_mount *mp, + xfs_filblks_t count_fsb, + unsigned int flags) +{ + struct xfs_zone_info *zi = mp->m_zone_info; + struct xfs_zone_reservation reservation = { + .task = current, + .count_fsb = count_fsb, + }; + int error; + + /* + * If there are no waiters, try to directly grab the available blocks + * from the percpu counter. + * + * If the caller wants to dip into the reserved pool also bypass the + * wait list. This relies on the fact that we have a very graciously + * sized reserved pool that always has enough space. If the reserved + * allocations fail we're in trouble. + */ + if (likely(list_empty_careful(&zi->zi_reclaim_reservations) || + (flags & XFS_ZR_RESERVED))) { + error = xfs_dec_freecounter(mp, XC_FREE_RTAVAILABLE, count_fsb, + flags & XFS_ZR_RESERVED); + if (error != -ENOSPC) + return error; + } + + if (flags & XFS_ZR_NOWAIT) + return -EAGAIN; + + spin_lock(&zi->zi_reservation_lock); + list_add_tail(&reservation.entry, &zi->zi_reclaim_reservations); + while ((error = xfs_zoned_space_wait_error(mp)) == 0) { + set_current_state(TASK_KILLABLE); + + error = xfs_dec_freecounter(mp, XC_FREE_RTAVAILABLE, count_fsb, + flags & XFS_ZR_RESERVED); + if (error != -ENOSPC) + break; + + /* + * Make sure to start GC if it is not running already. As we + * check the rtavailable count when filling up zones, GC is + * normally already running at this point, but in some setups + * with very few zones we may completely run out of non- + * reserved blocks in between filling zones. + */ + if (!xfs_is_zonegc_running(mp)) + wake_up_process(zi->zi_gc_thread); + + /* + * If there is no reclaimable group left and we aren't still + * processing a pending GC request give up as we're fully out + * of space. + */ + if (!xfs_zoned_have_reclaimable(mp->m_zone_info) && + !xfs_is_zonegc_running(mp)) + break; + + spin_unlock(&zi->zi_reservation_lock); + schedule(); + spin_lock(&zi->zi_reservation_lock); + } + list_del(&reservation.entry); + spin_unlock(&zi->zi_reservation_lock); + + __set_current_state(TASK_RUNNING); + return error; +} + +/* + * Implement greedy space allocation for short writes by trying to grab all + * that is left after locking out other threads from trying to do the same. + * + * This isn't exactly optimal and can hopefully be replaced by a proper + * percpu_counter primitive one day. + */ +static int +xfs_zoned_reserve_extents_greedy( + struct xfs_mount *mp, + xfs_filblks_t *count_fsb, + unsigned int flags) +{ + struct xfs_zone_info *zi = mp->m_zone_info; + s64 len = *count_fsb; + int error = -ENOSPC; + + spin_lock(&zi->zi_reservation_lock); + len = min(len, xfs_sum_freecounter(mp, XC_FREE_RTEXTENTS)); + if (len > 0) { + *count_fsb = len; + error = xfs_dec_freecounter(mp, XC_FREE_RTEXTENTS, *count_fsb, + flags & XFS_ZR_RESERVED); + } + spin_unlock(&zi->zi_reservation_lock); + return error; +} + +int +xfs_zoned_space_reserve( + struct xfs_mount *mp, + xfs_filblks_t count_fsb, + unsigned int flags, + struct xfs_zone_alloc_ctx *ac) +{ + int error; + + ASSERT(ac->reserved_blocks == 0); + ASSERT(ac->open_zone == NULL); + + error = xfs_dec_freecounter(mp, XC_FREE_RTEXTENTS, count_fsb, + flags & XFS_ZR_RESERVED); + if (error == -ENOSPC && !(flags & XFS_ZR_NOWAIT)) { + xfs_inodegc_flush(mp); + error = xfs_dec_freecounter(mp, XC_FREE_RTEXTENTS, count_fsb, + flags & XFS_ZR_RESERVED); + } + if (error == -ENOSPC && (flags & XFS_ZR_GREEDY) && count_fsb > 1) + error = xfs_zoned_reserve_extents_greedy(mp, &count_fsb, flags); + if (error) + return error; + + error = xfs_zoned_reserve_available(mp, count_fsb, flags); + if (error) { + xfs_add_freecounter(mp, XC_FREE_RTEXTENTS, count_fsb); + return error; + } + ac->reserved_blocks = count_fsb; + return 0; +} + +void +xfs_zoned_space_unreserve( + struct xfs_mount *mp, + struct xfs_zone_alloc_ctx *ac) +{ + if (ac->reserved_blocks > 0) { + xfs_zoned_add_available(mp, ac->reserved_blocks); + xfs_add_freecounter(mp, XC_FREE_RTEXTENTS, ac->reserved_blocks); + } + if (ac->open_zone) + xfs_open_zone_put(ac->open_zone); +} diff --git a/fs/zonefs/Kconfig b/fs/zonefs/Kconfig new file mode 100644 index 000000000..827278f93 --- /dev/null +++ b/fs/zonefs/Kconfig @@ -0,0 +1,11 @@ +config ZONEFS_FS + tristate "zonefs filesystem support" + depends on BLOCK + depends on BLK_DEV_ZONED + select FS_IOMAP + select CRC32 + help + zonefs is a simple file system which exposes zones of a zoned block + device (e.g. host-managed or host-aware SMR disk drives) as files. + + If unsure, say N. diff --git a/fs/zonefs/Makefile b/fs/zonefs/Makefile new file mode 100644 index 000000000..645f7229d --- /dev/null +++ b/fs/zonefs/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +ccflags-y += -I$(src) + +obj-$(CONFIG_ZONEFS_FS) += zonefs.o + +zonefs-y := super.o file.o sysfs.o diff --git a/fs/zonefs/file.c b/fs/zonefs/file.c new file mode 100644 index 000000000..5b34849be --- /dev/null +++ b/fs/zonefs/file.c @@ -0,0 +1,878 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Simple file system for zoned block devices exposing zones as files. + * + * Copyright (C) 2022 Western Digital Corporation or its affiliates. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "zonefs.h" + +#include "trace.h" + +static int zonefs_read_iomap_begin(struct inode *inode, loff_t offset, + loff_t length, unsigned int flags, + struct iomap *iomap, struct iomap *srcmap) +{ + struct zonefs_inode_info *zi = ZONEFS_I(inode); + struct zonefs_zone *z = zonefs_inode_zone(inode); + struct super_block *sb = inode->i_sb; + loff_t isize; + + /* + * All blocks are always mapped below EOF. If reading past EOF, + * act as if there is a hole up to the file maximum size. + */ + mutex_lock(&zi->i_truncate_mutex); + iomap->bdev = inode->i_sb->s_bdev; + iomap->offset = ALIGN_DOWN(offset, sb->s_blocksize); + isize = i_size_read(inode); + if (iomap->offset >= isize) { + iomap->type = IOMAP_HOLE; + iomap->addr = IOMAP_NULL_ADDR; + iomap->length = length; + } else { + iomap->type = IOMAP_MAPPED; + iomap->addr = (z->z_sector << SECTOR_SHIFT) + iomap->offset; + iomap->length = isize - iomap->offset; + } + mutex_unlock(&zi->i_truncate_mutex); + + trace_zonefs_iomap_begin(inode, iomap); + + return 0; +} + +static DEFINE_IOMAP_ITER_NEXT(zonefs_read_iomap_next, zonefs_read_iomap_begin); + +static const struct iomap_ops zonefs_read_iomap_ops = { + .iomap_next = zonefs_read_iomap_next, +}; + +static int zonefs_write_iomap_begin(struct inode *inode, loff_t offset, + loff_t length, unsigned int flags, + struct iomap *iomap, struct iomap *srcmap) +{ + struct zonefs_inode_info *zi = ZONEFS_I(inode); + struct zonefs_zone *z = zonefs_inode_zone(inode); + struct super_block *sb = inode->i_sb; + loff_t isize; + + /* All write I/Os should always be within the file maximum size */ + if (WARN_ON_ONCE(offset + length > z->z_capacity)) + return -EIO; + + /* + * Sequential zones can only accept direct writes. This is already + * checked when writes are issued, so warn if we see a page writeback + * operation. + */ + if (WARN_ON_ONCE(zonefs_zone_is_seq(z) && !(flags & IOMAP_DIRECT))) + return -EIO; + + /* + * For conventional zones, all blocks are always mapped. For sequential + * zones, all blocks after always mapped below the inode size (zone + * write pointer) and unwritten beyond. + */ + mutex_lock(&zi->i_truncate_mutex); + iomap->bdev = inode->i_sb->s_bdev; + iomap->offset = ALIGN_DOWN(offset, sb->s_blocksize); + iomap->addr = (z->z_sector << SECTOR_SHIFT) + iomap->offset; + isize = i_size_read(inode); + if (iomap->offset >= isize) { + iomap->type = IOMAP_UNWRITTEN; + iomap->length = z->z_capacity - iomap->offset; + } else { + iomap->type = IOMAP_MAPPED; + iomap->length = isize - iomap->offset; + } + mutex_unlock(&zi->i_truncate_mutex); + + trace_zonefs_iomap_begin(inode, iomap); + + return 0; +} + +static DEFINE_IOMAP_ITER_NEXT(zonefs_write_iomap_next, + zonefs_write_iomap_begin); + +static const struct iomap_ops zonefs_write_iomap_ops = { + .iomap_next = zonefs_write_iomap_next, +}; + +static int zonefs_read_folio(struct file *unused, struct folio *folio) +{ + iomap_bio_read_folio(folio, &zonefs_read_iomap_ops); + return 0; +} + +static void zonefs_readahead(struct readahead_control *rac) +{ + iomap_bio_readahead(rac, &zonefs_read_iomap_ops); +} + +/* + * Map blocks for page writeback. This is used only on conventional zone files, + * which implies that the page range can only be within the fixed inode size. + */ +static ssize_t zonefs_writeback_range(struct iomap_writepage_ctx *wpc, + struct folio *folio, u64 offset, unsigned len, u64 end_pos) +{ + struct zonefs_zone *z = zonefs_inode_zone(wpc->inode); + + if (WARN_ON_ONCE(zonefs_zone_is_seq(z))) + return -EIO; + if (WARN_ON_ONCE(offset >= i_size_read(wpc->inode))) + return -EIO; + + /* If the mapping is already OK, nothing needs to be done */ + if (offset < wpc->iomap.offset || + offset >= wpc->iomap.offset + wpc->iomap.length) { + int error; + + error = zonefs_write_iomap_begin(wpc->inode, offset, + z->z_capacity - offset, IOMAP_WRITE, + &wpc->iomap, NULL); + if (error) + return error; + } + + return iomap_add_to_ioend(wpc, folio, offset, end_pos, len); +} + +static const struct iomap_writeback_ops zonefs_writeback_ops = { + .writeback_range = zonefs_writeback_range, + .writeback_submit = iomap_ioend_writeback_submit, +}; + +static int zonefs_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + struct iomap_writepage_ctx wpc = { + .inode = mapping->host, + .wbc = wbc, + .ops = &zonefs_writeback_ops, + }; + + return iomap_writepages(&wpc); +} + +static int zonefs_swap_activate(struct swap_info_struct *sis, + struct file *swap_file, sector_t *span) +{ + struct inode *inode = file_inode(swap_file); + + if (zonefs_inode_is_seq(inode)) { + zonefs_err(inode->i_sb, + "swap file: not a conventional zone file\n"); + return -EINVAL; + } + + return iomap_swapfile_activate(sis, swap_file, span, + &zonefs_read_iomap_ops); +} + +const struct address_space_operations zonefs_file_aops = { + .read_folio = zonefs_read_folio, + .readahead = zonefs_readahead, + .writepages = zonefs_writepages, + .dirty_folio = iomap_dirty_folio, + .release_folio = iomap_release_folio, + .invalidate_folio = iomap_invalidate_folio, + .migrate_folio = filemap_migrate_folio, + .is_partially_uptodate = iomap_is_partially_uptodate, + .error_remove_folio = generic_error_remove_folio, + .swap_activate = zonefs_swap_activate, +}; + +int zonefs_file_truncate(struct inode *inode, loff_t isize) +{ + struct zonefs_inode_info *zi = ZONEFS_I(inode); + struct zonefs_zone *z = zonefs_inode_zone(inode); + loff_t old_isize; + enum req_op op; + int ret = 0; + + /* + * Only sequential zone files can be truncated and truncation is allowed + * only down to a 0 size, which is equivalent to a zone reset, and to + * the maximum file size, which is equivalent to a zone finish. + */ + if (!zonefs_zone_is_seq(z)) + return -EPERM; + + if (!isize) + op = REQ_OP_ZONE_RESET; + else if (isize == z->z_capacity) + op = REQ_OP_ZONE_FINISH; + else + return -EPERM; + + inode_dio_wait(inode); + + /* Serialize against page faults */ + filemap_invalidate_lock(inode->i_mapping); + + /* Serialize against zonefs_iomap_begin() */ + mutex_lock(&zi->i_truncate_mutex); + + old_isize = i_size_read(inode); + if (isize == old_isize) + goto unlock; + + ret = zonefs_inode_zone_mgmt(inode, op); + if (ret) + goto unlock; + + /* + * If the mount option ZONEFS_MNTOPT_EXPLICIT_OPEN is set, + * take care of open zones. + */ + if (z->z_flags & ZONEFS_ZONE_OPEN) { + /* + * Truncating a zone to EMPTY or FULL is the equivalent of + * closing the zone. For a truncation to 0, we need to + * re-open the zone to ensure new writes can be processed. + * For a truncation to the maximum file size, the zone is + * closed and writes cannot be accepted anymore, so clear + * the open flag. + */ + if (!isize) + ret = zonefs_inode_zone_mgmt(inode, REQ_OP_ZONE_OPEN); + else + z->z_flags &= ~ZONEFS_ZONE_OPEN; + } + + zonefs_update_stats(inode, isize); + truncate_setsize(inode, isize); + z->z_wpoffset = isize; + zonefs_inode_account_active(inode); + +unlock: + mutex_unlock(&zi->i_truncate_mutex); + filemap_invalidate_unlock(inode->i_mapping); + + return ret; +} + +static int zonefs_file_fsync(struct file *file, loff_t start, loff_t end, + int datasync) +{ + struct inode *inode = file_inode(file); + int ret = 0; + + if (unlikely(IS_IMMUTABLE(inode))) + return -EPERM; + + /* + * Since only direct writes are allowed in sequential files, page cache + * flush is needed only for conventional zone files. + */ + if (zonefs_inode_is_cnv(inode)) + ret = file_write_and_wait_range(file, start, end); + if (!ret) + ret = blkdev_issue_flush(inode->i_sb->s_bdev); + + if (ret) + zonefs_io_error(inode, true); + + return ret; +} + +static vm_fault_t zonefs_filemap_page_mkwrite(struct vm_fault *vmf) +{ + struct inode *inode = file_inode(vmf->vma->vm_file); + vm_fault_t ret; + + if (unlikely(IS_IMMUTABLE(inode))) + return VM_FAULT_SIGBUS; + + /* + * Sanity check: only conventional zone files can have shared + * writeable mappings. + */ + if (zonefs_inode_is_seq(inode)) + return VM_FAULT_NOPAGE; + + sb_start_pagefault(inode->i_sb); + file_update_time(vmf->vma->vm_file); + + /* Serialize against truncates */ + filemap_invalidate_lock_shared(inode->i_mapping); + ret = iomap_page_mkwrite(vmf, &zonefs_write_iomap_ops, NULL); + filemap_invalidate_unlock_shared(inode->i_mapping); + + sb_end_pagefault(inode->i_sb); + return ret; +} + +static const struct vm_operations_struct zonefs_file_vm_ops = { + .fault = filemap_fault, + .map_pages = filemap_map_pages, + .page_mkwrite = zonefs_filemap_page_mkwrite, +}; + +static int zonefs_file_mmap_prepare(struct vm_area_desc *desc) +{ + struct file *file = desc->file; + + /* + * Conventional zones accept random writes, so their files can support + * shared writable mappings. For sequential zone files, only read + * mappings are possible since there are no guarantees for write + * ordering between msync() and page cache writeback. + */ + if (zonefs_inode_is_seq(file_inode(file)) && + vma_desc_test_all(desc, VMA_SHARED_BIT, VMA_MAYWRITE_BIT)) + return -EINVAL; + + file_accessed(file); + desc->vm_ops = &zonefs_file_vm_ops; + + return 0; +} + +static loff_t zonefs_file_llseek(struct file *file, loff_t offset, int whence) +{ + loff_t isize = i_size_read(file_inode(file)); + + /* + * Seeks are limited to below the zone size for conventional zones + * and below the zone write pointer for sequential zones. In both + * cases, this limit is the inode size. + */ + return generic_file_llseek_size(file, offset, whence, isize, isize); +} + +static int zonefs_file_write_dio_end_io(struct kiocb *iocb, ssize_t size, + int error, unsigned int flags) +{ + struct inode *inode = file_inode(iocb->ki_filp); + struct zonefs_inode_info *zi = ZONEFS_I(inode); + + if (error) { + /* + * For Sync IOs, error recovery is called from + * zonefs_file_dio_write(). + */ + if (!is_sync_kiocb(iocb)) + zonefs_io_error(inode, true); + return error; + } + + if (size && zonefs_inode_is_seq(inode)) { + /* + * Note that we may be seeing completions out of order, + * but that is not a problem since a write completed + * successfully necessarily means that all preceding writes + * were also successful. So we can safely increase the inode + * size to the write end location. + */ + mutex_lock(&zi->i_truncate_mutex); + if (i_size_read(inode) < iocb->ki_pos + size) { + zonefs_update_stats(inode, iocb->ki_pos + size); + zonefs_i_size_write(inode, iocb->ki_pos + size); + } + mutex_unlock(&zi->i_truncate_mutex); + } + + return 0; +} + +static const struct iomap_dio_ops zonefs_write_dio_ops = { + .end_io = zonefs_file_write_dio_end_io, +}; + +/* + * Do not exceed the LFS limits nor the file zone size. If pos is under the + * limit it becomes a short access. If it exceeds the limit, return -EFBIG. + */ +static loff_t zonefs_write_check_limits(struct file *file, loff_t pos, + loff_t count) +{ + struct inode *inode = file_inode(file); + struct zonefs_zone *z = zonefs_inode_zone(inode); + loff_t limit = rlimit(RLIMIT_FSIZE); + loff_t max_size = z->z_capacity; + + if (limit != RLIM_INFINITY) { + if (pos >= limit) { + send_sig(SIGXFSZ, current, 0); + return -EFBIG; + } + count = min(count, limit - pos); + } + + if (!(file->f_flags & O_LARGEFILE)) + max_size = min_t(loff_t, MAX_NON_LFS, max_size); + + if (unlikely(pos >= max_size)) + return -EFBIG; + + return min(count, max_size - pos); +} + +static ssize_t zonefs_write_checks(struct kiocb *iocb, struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file_inode(file); + struct zonefs_inode_info *zi = ZONEFS_I(inode); + struct zonefs_zone *z = zonefs_inode_zone(inode); + loff_t count; + + if (IS_SWAPFILE(inode)) + return -ETXTBSY; + + if (!iov_iter_count(from)) + return 0; + + if ((iocb->ki_flags & IOCB_NOWAIT) && !(iocb->ki_flags & IOCB_DIRECT)) + return -EINVAL; + + if (iocb->ki_flags & IOCB_APPEND) { + if (zonefs_zone_is_cnv(z)) + return -EINVAL; + mutex_lock(&zi->i_truncate_mutex); + iocb->ki_pos = z->z_wpoffset; + mutex_unlock(&zi->i_truncate_mutex); + } + + count = zonefs_write_check_limits(file, iocb->ki_pos, + iov_iter_count(from)); + if (count < 0) + return count; + + iov_iter_truncate(from, count); + return iov_iter_count(from); +} + +/* + * Handle direct writes. For sequential zone files, this is the only possible + * write path. For these files, check that the user is issuing writes + * sequentially from the end of the file. This code assumes that the block layer + * delivers write requests to the device in sequential order. This is always the + * case if a block IO scheduler implementing the ELEVATOR_F_ZBD_SEQ_WRITE + * elevator feature is being used (e.g. mq-deadline). The block layer always + * automatically select such an elevator for zoned block devices during the + * device initialization. + */ +static ssize_t zonefs_file_dio_write(struct kiocb *iocb, struct iov_iter *from) +{ + struct inode *inode = file_inode(iocb->ki_filp); + struct zonefs_inode_info *zi = ZONEFS_I(inode); + struct zonefs_zone *z = zonefs_inode_zone(inode); + struct super_block *sb = inode->i_sb; + ssize_t ret, count; + + /* + * For async direct IOs to sequential zone files, refuse IOCB_NOWAIT + * as this can cause write reordering (e.g. the first aio gets EAGAIN + * on the inode lock but the second goes through but is now unaligned). + */ + if (zonefs_zone_is_seq(z) && !is_sync_kiocb(iocb) && + (iocb->ki_flags & IOCB_NOWAIT)) + return -EOPNOTSUPP; + + if (iocb->ki_flags & IOCB_NOWAIT) { + if (!inode_trylock(inode)) + return -EAGAIN; + } else { + inode_lock(inode); + } + + count = zonefs_write_checks(iocb, from); + if (count <= 0) { + ret = count; + goto inode_unlock; + } + + if ((iocb->ki_pos | count) & (sb->s_blocksize - 1)) { + ret = -EINVAL; + goto inode_unlock; + } + + /* Enforce sequential writes (append only) in sequential zones */ + if (zonefs_zone_is_seq(z)) { + mutex_lock(&zi->i_truncate_mutex); + if (iocb->ki_pos != z->z_wpoffset) { + mutex_unlock(&zi->i_truncate_mutex); + ret = -EINVAL; + goto inode_unlock; + } + /* + * Advance the zone write pointer offset. This assumes that the + * IO will succeed, which is OK to do because we do not allow + * partial writes (IOMAP_DIO_PARTIAL is not set) and if the IO + * fails, the error path will correct the write pointer offset. + */ + z->z_wpoffset += count; + zonefs_inode_account_active(inode); + mutex_unlock(&zi->i_truncate_mutex); + } + + /* + * iomap_dio_rw() may return ENOTBLK if there was an issue with + * page invalidation. Overwrite that error code with EBUSY so that + * the user can make sense of the error. + */ + ret = iomap_dio_rw(iocb, from, &zonefs_write_iomap_ops, + &zonefs_write_dio_ops, 0, NULL, 0); + if (ret == -ENOTBLK) + ret = -EBUSY; + + /* + * For a failed IO or partial completion, trigger error recovery + * to update the zone write pointer offset to a correct value. + * For asynchronous IOs, zonefs_file_write_dio_end_io() may already + * have executed error recovery if the IO already completed when we + * reach here. However, we cannot know that and execute error recovery + * again (that will not change anything). + */ + if (zonefs_zone_is_seq(z)) { + if (ret > 0 && ret != count) + ret = -EIO; + if (ret < 0 && ret != -EIOCBQUEUED) + zonefs_io_error(inode, true); + } + +inode_unlock: + inode_unlock(inode); + + return ret; +} + +static ssize_t zonefs_file_buffered_write(struct kiocb *iocb, + struct iov_iter *from) +{ + struct inode *inode = file_inode(iocb->ki_filp); + ssize_t ret; + + /* + * Direct IO writes are mandatory for sequential zone files so that the + * write IO issuing order is preserved. + */ + if (zonefs_inode_is_seq(inode)) + return -EIO; + + if (iocb->ki_flags & IOCB_NOWAIT) { + if (!inode_trylock(inode)) + return -EAGAIN; + } else { + inode_lock(inode); + } + + ret = zonefs_write_checks(iocb, from); + if (ret <= 0) + goto inode_unlock; + + ret = iomap_file_buffered_write(iocb, from, &zonefs_write_iomap_ops, + NULL, NULL); + if (ret == -EIO) + zonefs_io_error(inode, true); + +inode_unlock: + inode_unlock(inode); + if (ret > 0) + ret = generic_write_sync(iocb, ret); + + return ret; +} + +static ssize_t zonefs_file_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + struct inode *inode = file_inode(iocb->ki_filp); + struct zonefs_zone *z = zonefs_inode_zone(inode); + + if (unlikely(IS_IMMUTABLE(inode))) + return -EPERM; + + if (sb_rdonly(inode->i_sb)) + return -EROFS; + + /* Write operations beyond the zone capacity are not allowed */ + if (iocb->ki_pos >= z->z_capacity) + return -EFBIG; + + if (iocb->ki_flags & IOCB_DIRECT) { + ssize_t ret = zonefs_file_dio_write(iocb, from); + + if (ret != -ENOTBLK) + return ret; + } + + return zonefs_file_buffered_write(iocb, from); +} + +static int zonefs_file_read_dio_end_io(struct kiocb *iocb, ssize_t size, + int error, unsigned int flags) +{ + if (error) { + zonefs_io_error(file_inode(iocb->ki_filp), false); + return error; + } + + return 0; +} + +static const struct iomap_dio_ops zonefs_read_dio_ops = { + .end_io = zonefs_file_read_dio_end_io, +}; + +static ssize_t zonefs_file_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct inode *inode = file_inode(iocb->ki_filp); + struct zonefs_inode_info *zi = ZONEFS_I(inode); + struct zonefs_zone *z = zonefs_inode_zone(inode); + struct super_block *sb = inode->i_sb; + loff_t isize; + ssize_t ret; + + /* Offline zones cannot be read */ + if (unlikely(IS_IMMUTABLE(inode) && !(inode->i_mode & 0777))) + return -EPERM; + + if (iocb->ki_pos >= z->z_capacity) + return 0; + + if (iocb->ki_flags & IOCB_NOWAIT) { + if (!inode_trylock_shared(inode)) + return -EAGAIN; + } else { + inode_lock_shared(inode); + } + + /* Limit read operations to written data */ + mutex_lock(&zi->i_truncate_mutex); + isize = i_size_read(inode); + if (iocb->ki_pos >= isize) { + mutex_unlock(&zi->i_truncate_mutex); + ret = 0; + goto inode_unlock; + } + iov_iter_truncate(to, isize - iocb->ki_pos); + mutex_unlock(&zi->i_truncate_mutex); + + if (iocb->ki_flags & IOCB_DIRECT) { + size_t count = iov_iter_count(to); + + if ((iocb->ki_pos | count) & (sb->s_blocksize - 1)) { + ret = -EINVAL; + goto inode_unlock; + } + file_accessed(iocb->ki_filp); + ret = iomap_dio_rw(iocb, to, &zonefs_read_iomap_ops, + &zonefs_read_dio_ops, 0, NULL, 0); + } else { + ret = generic_file_read_iter(iocb, to); + if (ret == -EIO) + zonefs_io_error(inode, false); + } + +inode_unlock: + inode_unlock_shared(inode); + + return ret; +} + +static ssize_t zonefs_file_splice_read(struct file *in, loff_t *ppos, + struct pipe_inode_info *pipe, + size_t len, unsigned int flags) +{ + struct inode *inode = file_inode(in); + struct zonefs_inode_info *zi = ZONEFS_I(inode); + struct zonefs_zone *z = zonefs_inode_zone(inode); + loff_t isize; + ssize_t ret = 0; + + /* Offline zones cannot be read */ + if (unlikely(IS_IMMUTABLE(inode) && !(inode->i_mode & 0777))) + return -EPERM; + + if (*ppos >= z->z_capacity) + return 0; + + inode_lock_shared(inode); + + /* Limit read operations to written data */ + mutex_lock(&zi->i_truncate_mutex); + isize = i_size_read(inode); + if (*ppos >= isize) + len = 0; + else + len = min_t(loff_t, len, isize - *ppos); + mutex_unlock(&zi->i_truncate_mutex); + + if (len > 0) { + ret = filemap_splice_read(in, ppos, pipe, len, flags); + if (ret == -EIO) + zonefs_io_error(inode, false); + } + + inode_unlock_shared(inode); + return ret; +} + +/* + * Write open accounting is done only for sequential files. + */ +static inline bool zonefs_seq_file_need_wro(struct inode *inode, + struct file *file) +{ + if (zonefs_inode_is_cnv(inode)) + return false; + + if (!(file->f_mode & FMODE_WRITE)) + return false; + + return true; +} + +static int zonefs_seq_file_write_open(struct inode *inode) +{ + struct zonefs_inode_info *zi = ZONEFS_I(inode); + struct zonefs_zone *z = zonefs_inode_zone(inode); + int ret = 0; + + mutex_lock(&zi->i_truncate_mutex); + + if (!zi->i_wr_refcnt) { + struct zonefs_sb_info *sbi = ZONEFS_SB(inode->i_sb); + unsigned int wro = atomic_inc_return(&sbi->s_wro_seq_files); + + if (sbi->s_mount_opts & ZONEFS_MNTOPT_EXPLICIT_OPEN) { + + if (sbi->s_max_wro_seq_files + && wro > sbi->s_max_wro_seq_files) { + atomic_dec(&sbi->s_wro_seq_files); + ret = -EBUSY; + goto unlock; + } + + if (i_size_read(inode) < z->z_capacity) { + ret = zonefs_inode_zone_mgmt(inode, + REQ_OP_ZONE_OPEN); + if (ret) { + atomic_dec(&sbi->s_wro_seq_files); + goto unlock; + } + z->z_flags |= ZONEFS_ZONE_OPEN; + zonefs_inode_account_active(inode); + } + } + } + + zi->i_wr_refcnt++; + +unlock: + mutex_unlock(&zi->i_truncate_mutex); + + return ret; +} + +static int zonefs_file_open(struct inode *inode, struct file *file) +{ + int ret; + + file->f_mode |= FMODE_CAN_ODIRECT; + ret = generic_file_open(inode, file); + if (ret) + return ret; + + if (zonefs_seq_file_need_wro(inode, file)) + return zonefs_seq_file_write_open(inode); + + return 0; +} + +static void zonefs_seq_file_write_close(struct inode *inode) +{ + struct zonefs_inode_info *zi = ZONEFS_I(inode); + struct zonefs_zone *z = zonefs_inode_zone(inode); + struct super_block *sb = inode->i_sb; + struct zonefs_sb_info *sbi = ZONEFS_SB(sb); + int ret = 0; + + mutex_lock(&zi->i_truncate_mutex); + + zi->i_wr_refcnt--; + if (zi->i_wr_refcnt) + goto unlock; + + /* + * The file zone may not be open anymore (e.g. the file was truncated to + * its maximum size or it was fully written). For this case, we only + * need to decrement the write open count. + */ + if (z->z_flags & ZONEFS_ZONE_OPEN) { + ret = zonefs_inode_zone_mgmt(inode, REQ_OP_ZONE_CLOSE); + if (ret) { + __zonefs_io_error(inode, false); + /* + * Leaving zones explicitly open may lead to a state + * where most zones cannot be written (zone resources + * exhausted). So take preventive action by remounting + * read-only. + */ + if (z->z_flags & ZONEFS_ZONE_OPEN && + !(sb->s_flags & SB_RDONLY)) { + zonefs_warn(sb, + "closing zone at %llu failed %d\n", + z->z_sector, ret); + zonefs_warn(sb, + "remounting filesystem read-only\n"); + sb->s_flags |= SB_RDONLY; + } + goto unlock; + } + + z->z_flags &= ~ZONEFS_ZONE_OPEN; + zonefs_inode_account_active(inode); + } + + atomic_dec(&sbi->s_wro_seq_files); + +unlock: + mutex_unlock(&zi->i_truncate_mutex); +} + +static int zonefs_file_release(struct inode *inode, struct file *file) +{ + /* + * If we explicitly open a zone we must close it again as well, but the + * zone management operation can fail (either due to an IO error or as + * the zone has gone offline or read-only). Make sure we don't fail the + * close(2) for user-space. + */ + if (zonefs_seq_file_need_wro(inode, file)) + zonefs_seq_file_write_close(inode); + + return 0; +} + +const struct file_operations zonefs_file_operations = { + .open = zonefs_file_open, + .release = zonefs_file_release, + .fsync = zonefs_file_fsync, + .mmap_prepare = zonefs_file_mmap_prepare, + .llseek = zonefs_file_llseek, + .read_iter = zonefs_file_read_iter, + .write_iter = zonefs_file_write_iter, + .splice_read = zonefs_file_splice_read, + .splice_write = iter_file_splice_write, + .iopoll = iocb_bio_iopoll, +}; diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c new file mode 100644 index 000000000..ff43d6d1e --- /dev/null +++ b/fs/zonefs/super.c @@ -0,0 +1,1476 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Simple file system for zoned block devices exposing zones as files. + * + * Copyright (C) 2019 Western Digital Corporation or its affiliates. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "zonefs.h" + +#define CREATE_TRACE_POINTS +#include "trace.h" + +/* + * Get the name of a zone group directory. + */ +static const char *zonefs_zgroup_name(enum zonefs_ztype ztype) +{ + switch (ztype) { + case ZONEFS_ZTYPE_CNV: + return "cnv"; + case ZONEFS_ZTYPE_SEQ: + return "seq"; + default: + WARN_ON_ONCE(1); + return "???"; + } +} + +/* + * Manage the active zone count. + */ +static void zonefs_account_active(struct super_block *sb, + struct zonefs_zone *z) +{ + struct zonefs_sb_info *sbi = ZONEFS_SB(sb); + + if (zonefs_zone_is_cnv(z)) + return; + + /* + * For zones that transitioned to the offline or readonly condition, + * we only need to clear the active state. + */ + if (z->z_flags & (ZONEFS_ZONE_OFFLINE | ZONEFS_ZONE_READONLY)) + goto out; + + /* + * If the zone is active, that is, if it is explicitly open or + * partially written, check if it was already accounted as active. + */ + if ((z->z_flags & ZONEFS_ZONE_OPEN) || + (z->z_wpoffset > 0 && z->z_wpoffset < z->z_capacity)) { + if (!(z->z_flags & ZONEFS_ZONE_ACTIVE)) { + z->z_flags |= ZONEFS_ZONE_ACTIVE; + atomic_inc(&sbi->s_active_seq_files); + } + return; + } + +out: + /* The zone is not active. If it was, update the active count */ + if (z->z_flags & ZONEFS_ZONE_ACTIVE) { + z->z_flags &= ~ZONEFS_ZONE_ACTIVE; + atomic_dec(&sbi->s_active_seq_files); + } +} + +/* + * Manage the active zone count. Called with zi->i_truncate_mutex held. + */ +void zonefs_inode_account_active(struct inode *inode) +{ + lockdep_assert_held(&ZONEFS_I(inode)->i_truncate_mutex); + + return zonefs_account_active(inode->i_sb, zonefs_inode_zone(inode)); +} + +/* + * Execute a zone management operation. + */ +static int zonefs_zone_mgmt(struct super_block *sb, + struct zonefs_zone *z, enum req_op op) +{ + int ret; + + /* + * With ZNS drives, closing an explicitly open zone that has not been + * written will change the zone state to "closed", that is, the zone + * will remain active. Since this can then cause failure of explicit + * open operation on other zones if the drive active zone resources + * are exceeded, make sure that the zone does not remain active by + * resetting it. + */ + if (op == REQ_OP_ZONE_CLOSE && !z->z_wpoffset) + op = REQ_OP_ZONE_RESET; + + trace_zonefs_zone_mgmt(sb, z, op); + ret = blkdev_zone_mgmt(sb->s_bdev, op, z->z_sector, + z->z_size >> SECTOR_SHIFT); + if (ret) { + zonefs_err(sb, + "Zone management operation %s at %llu failed %d\n", + blk_op_str(op), z->z_sector, ret); + return ret; + } + + return 0; +} + +int zonefs_inode_zone_mgmt(struct inode *inode, enum req_op op) +{ + lockdep_assert_held(&ZONEFS_I(inode)->i_truncate_mutex); + + return zonefs_zone_mgmt(inode->i_sb, zonefs_inode_zone(inode), op); +} + +void zonefs_i_size_write(struct inode *inode, loff_t isize) +{ + struct zonefs_zone *z = zonefs_inode_zone(inode); + + i_size_write(inode, isize); + + /* + * A full zone is no longer open/active and does not need + * explicit closing. + */ + if (isize >= z->z_capacity) { + struct zonefs_sb_info *sbi = ZONEFS_SB(inode->i_sb); + + if (z->z_flags & ZONEFS_ZONE_ACTIVE) + atomic_dec(&sbi->s_active_seq_files); + z->z_flags &= ~(ZONEFS_ZONE_OPEN | ZONEFS_ZONE_ACTIVE); + } +} + +void zonefs_update_stats(struct inode *inode, loff_t new_isize) +{ + struct super_block *sb = inode->i_sb; + struct zonefs_sb_info *sbi = ZONEFS_SB(sb); + loff_t old_isize = i_size_read(inode); + loff_t nr_blocks; + + if (new_isize == old_isize) + return; + + spin_lock(&sbi->s_lock); + + /* + * This may be called for an update after an IO error. + * So beware of the values seen. + */ + if (new_isize < old_isize) { + nr_blocks = (old_isize - new_isize) >> sb->s_blocksize_bits; + if (sbi->s_used_blocks > nr_blocks) + sbi->s_used_blocks -= nr_blocks; + else + sbi->s_used_blocks = 0; + } else { + sbi->s_used_blocks += + (new_isize - old_isize) >> sb->s_blocksize_bits; + if (sbi->s_used_blocks > sbi->s_blocks) + sbi->s_used_blocks = sbi->s_blocks; + } + + spin_unlock(&sbi->s_lock); +} + +/* + * Check a zone condition. Return the amount of written (and still readable) + * data in the zone. + */ +static loff_t zonefs_check_zone_condition(struct super_block *sb, + struct zonefs_zone *z, + struct blk_zone *zone) +{ + switch (zone->cond) { + case BLK_ZONE_COND_OFFLINE: + zonefs_warn(sb, "Zone %llu: offline zone\n", + z->z_sector); + z->z_flags |= ZONEFS_ZONE_OFFLINE; + return 0; + case BLK_ZONE_COND_READONLY: + /* + * The write pointer of read-only zones is invalid, so we cannot + * determine the zone wpoffset (inode size). We thus keep the + * zone wpoffset as is, which leads to an empty file + * (wpoffset == 0) on mount. For a runtime error, this keeps + * the inode size as it was when last updated so that the user + * can recover data. + */ + zonefs_warn(sb, "Zone %llu: read-only zone\n", + z->z_sector); + z->z_flags |= ZONEFS_ZONE_READONLY; + if (zonefs_zone_is_cnv(z)) + return z->z_capacity; + return z->z_wpoffset; + case BLK_ZONE_COND_FULL: + /* The write pointer of full zones is invalid. */ + return z->z_capacity; + default: + if (zonefs_zone_is_cnv(z)) + return z->z_capacity; + return (zone->wp - zone->start) << SECTOR_SHIFT; + } +} + +/* + * Check a zone condition and adjust its inode access permissions for + * offline and readonly zones. + */ +static void zonefs_inode_update_mode(struct inode *inode) +{ + struct zonefs_zone *z = zonefs_inode_zone(inode); + + if (z->z_flags & ZONEFS_ZONE_OFFLINE) { + /* Offline zones cannot be read nor written */ + inode->i_flags |= S_IMMUTABLE; + inode->i_mode &= ~0777; + } else if (z->z_flags & ZONEFS_ZONE_READONLY) { + /* Readonly zones cannot be written */ + inode->i_flags |= S_IMMUTABLE; + if (z->z_flags & ZONEFS_ZONE_INIT_MODE) + inode->i_mode &= ~0777; + else + inode->i_mode &= ~0222; + } + + z->z_flags &= ~ZONEFS_ZONE_INIT_MODE; + z->z_mode = inode->i_mode; +} + +static int zonefs_io_error_cb(struct blk_zone *zone, unsigned int idx, + void *data) +{ + struct blk_zone *z = data; + + *z = *zone; + return 0; +} + +static void zonefs_handle_io_error(struct inode *inode, struct blk_zone *zone, + bool write) +{ + struct zonefs_zone *z = zonefs_inode_zone(inode); + struct super_block *sb = inode->i_sb; + struct zonefs_sb_info *sbi = ZONEFS_SB(sb); + loff_t isize, data_size; + + /* + * Check the zone condition: if the zone is not "bad" (offline or + * read-only), read errors are simply signaled to the IO issuer as long + * as there is no inconsistency between the inode size and the amount of + * data written in the zone (data_size). + */ + data_size = zonefs_check_zone_condition(sb, z, zone); + isize = i_size_read(inode); + if (!(z->z_flags & (ZONEFS_ZONE_READONLY | ZONEFS_ZONE_OFFLINE)) && + !write && isize == data_size) + return; + + /* + * At this point, we detected either a bad zone or an inconsistency + * between the inode size and the amount of data written in the zone. + * For the latter case, the cause may be a write IO error or an external + * action on the device. Two error patterns exist: + * 1) The inode size is lower than the amount of data in the zone: + * a write operation partially failed and data was written at the end + * of the file. This can happen in the case of a large direct IO + * needing several BIOs and/or write requests to be processed. + * 2) The inode size is larger than the amount of data in the zone: + * this can happen with a deferred write error with the use of the + * device side write cache after getting successful write IO + * completions. Other possibilities are (a) an external corruption, + * e.g. an application reset the zone directly, or (b) the device + * has a serious problem (e.g. firmware bug). + * + * In all cases, warn about inode size inconsistency and handle the + * IO error according to the zone condition and to the mount options. + */ + if (isize != data_size) + zonefs_warn(sb, + "inode %llu: invalid size %lld (should be %lld)\n", + inode->i_ino, isize, data_size); + + /* + * First handle bad zones signaled by hardware. The mount options + * errors=zone-ro and errors=zone-offline result in changing the + * zone condition to read-only and offline respectively, as if the + * condition was signaled by the hardware. + */ + if ((z->z_flags & ZONEFS_ZONE_OFFLINE) || + (sbi->s_mount_opts & ZONEFS_MNTOPT_ERRORS_ZOL)) { + zonefs_warn(sb, "inode %llu: read/write access disabled\n", + inode->i_ino); + if (!(z->z_flags & ZONEFS_ZONE_OFFLINE)) + z->z_flags |= ZONEFS_ZONE_OFFLINE; + zonefs_inode_update_mode(inode); + data_size = 0; + } else if ((z->z_flags & ZONEFS_ZONE_READONLY) || + (sbi->s_mount_opts & ZONEFS_MNTOPT_ERRORS_ZRO)) { + zonefs_warn(sb, "inode %llu: write access disabled\n", + inode->i_ino); + if (!(z->z_flags & ZONEFS_ZONE_READONLY)) + z->z_flags |= ZONEFS_ZONE_READONLY; + zonefs_inode_update_mode(inode); + data_size = isize; + } else if (sbi->s_mount_opts & ZONEFS_MNTOPT_ERRORS_RO && + data_size > isize) { + /* Do not expose garbage data */ + data_size = isize; + } + + /* + * If the filesystem is mounted with the explicit-open mount option, we + * need to clear the ZONEFS_ZONE_OPEN flag if the zone transitioned to + * the read-only or offline condition, to avoid attempting an explicit + * close of the zone when the inode file is closed. + */ + if ((sbi->s_mount_opts & ZONEFS_MNTOPT_EXPLICIT_OPEN) && + (z->z_flags & (ZONEFS_ZONE_READONLY | ZONEFS_ZONE_OFFLINE))) + z->z_flags &= ~ZONEFS_ZONE_OPEN; + + /* + * If error=remount-ro was specified, any error result in remounting + * the volume as read-only. + */ + if ((sbi->s_mount_opts & ZONEFS_MNTOPT_ERRORS_RO) && !sb_rdonly(sb)) { + zonefs_warn(sb, "remounting filesystem read-only\n"); + sb->s_flags |= SB_RDONLY; + } + + /* + * Update block usage stats and the inode size to prevent access to + * invalid data. + */ + zonefs_update_stats(inode, data_size); + zonefs_i_size_write(inode, data_size); + z->z_wpoffset = data_size; + zonefs_inode_account_active(inode); +} + +/* + * When an file IO error occurs, check the file zone to see if there is a change + * in the zone condition (e.g. offline or read-only). For a failed write to a + * sequential zone, the zone write pointer position must also be checked to + * eventually correct the file size and zonefs inode write pointer offset + * (which can be out of sync with the drive due to partial write failures). + */ +void __zonefs_io_error(struct inode *inode, bool write) +{ + struct zonefs_zone *z = zonefs_inode_zone(inode); + struct super_block *sb = inode->i_sb; + unsigned int noio_flag; + struct blk_zone zone; + int ret; + + /* + * Conventional zone have no write pointer and cannot become read-only + * or offline. So simply fake a report for a single or aggregated zone + * and let zonefs_handle_io_error() correct the zone inode information + * according to the mount options. + */ + if (!zonefs_zone_is_seq(z)) { + zone.start = z->z_sector; + zone.len = z->z_size >> SECTOR_SHIFT; + zone.wp = zone.start + zone.len; + zone.type = BLK_ZONE_TYPE_CONVENTIONAL; + zone.cond = BLK_ZONE_COND_NOT_WP; + zone.capacity = zone.len; + goto handle_io_error; + } + + /* + * Memory allocations in blkdev_report_zones() can trigger a memory + * reclaim which may in turn cause a recursion into zonefs as well as + * struct request allocations for the same device. The former case may + * end up in a deadlock on the inode truncate mutex, while the latter + * may prevent IO forward progress. Executing the report zones under + * the GFP_NOIO context avoids both problems. + */ + noio_flag = memalloc_noio_save(); + ret = blkdev_report_zones(sb->s_bdev, z->z_sector, 1, + zonefs_io_error_cb, &zone); + memalloc_noio_restore(noio_flag); + + if (ret != 1) { + zonefs_err(sb, "Get inode %llu zone information failed %d\n", + inode->i_ino, ret); + zonefs_warn(sb, "remounting filesystem read-only\n"); + sb->s_flags |= SB_RDONLY; + return; + } + +handle_io_error: + zonefs_handle_io_error(inode, &zone, write); +} + +static struct kmem_cache *zonefs_inode_cachep; + +static struct inode *zonefs_alloc_inode(struct super_block *sb) +{ + struct zonefs_inode_info *zi; + + zi = alloc_inode_sb(sb, zonefs_inode_cachep, GFP_KERNEL); + if (!zi) + return NULL; + + inode_init_once(&zi->i_vnode); + mutex_init(&zi->i_truncate_mutex); + zi->i_wr_refcnt = 0; + + return &zi->i_vnode; +} + +static void zonefs_free_inode(struct inode *inode) +{ + kmem_cache_free(zonefs_inode_cachep, ZONEFS_I(inode)); +} + +/* + * File system stat. + */ +static int zonefs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct super_block *sb = dentry->d_sb; + struct zonefs_sb_info *sbi = ZONEFS_SB(sb); + enum zonefs_ztype t; + + buf->f_type = ZONEFS_MAGIC; + buf->f_bsize = sb->s_blocksize; + buf->f_namelen = ZONEFS_NAME_MAX; + + spin_lock(&sbi->s_lock); + + buf->f_blocks = sbi->s_blocks; + if (WARN_ON(sbi->s_used_blocks > sbi->s_blocks)) + buf->f_bfree = 0; + else + buf->f_bfree = buf->f_blocks - sbi->s_used_blocks; + buf->f_bavail = buf->f_bfree; + + for (t = 0; t < ZONEFS_ZTYPE_MAX; t++) { + if (sbi->s_zgroup[t].g_nr_zones) + buf->f_files += sbi->s_zgroup[t].g_nr_zones + 1; + } + buf->f_ffree = 0; + + spin_unlock(&sbi->s_lock); + + buf->f_fsid = uuid_to_fsid(sbi->s_uuid.b); + + return 0; +} + +enum { + Opt_errors, Opt_explicit_open, +}; + +struct zonefs_context { + unsigned long s_mount_opts; +}; + +static const struct constant_table zonefs_param_errors[] = { + {"remount-ro", ZONEFS_MNTOPT_ERRORS_RO}, + {"zone-ro", ZONEFS_MNTOPT_ERRORS_ZRO}, + {"zone-offline", ZONEFS_MNTOPT_ERRORS_ZOL}, + {"repair", ZONEFS_MNTOPT_ERRORS_REPAIR}, + {} +}; + +static const struct fs_parameter_spec zonefs_param_spec[] = { + fsparam_enum ("errors", Opt_errors, zonefs_param_errors), + fsparam_flag ("explicit-open", Opt_explicit_open), + {} +}; + +static int zonefs_parse_param(struct fs_context *fc, struct fs_parameter *param) +{ + struct zonefs_context *ctx = fc->fs_private; + struct fs_parse_result result; + int opt; + + opt = fs_parse(fc, zonefs_param_spec, param, &result); + if (opt < 0) + return opt; + + switch (opt) { + case Opt_errors: + ctx->s_mount_opts &= ~ZONEFS_MNTOPT_ERRORS_MASK; + ctx->s_mount_opts |= result.uint_32; + break; + case Opt_explicit_open: + ctx->s_mount_opts |= ZONEFS_MNTOPT_EXPLICIT_OPEN; + break; + default: + return -EINVAL; + } + + return 0; +} + +static int zonefs_show_options(struct seq_file *seq, struct dentry *root) +{ + struct zonefs_sb_info *sbi = ZONEFS_SB(root->d_sb); + + if (sbi->s_mount_opts & ZONEFS_MNTOPT_ERRORS_RO) + seq_puts(seq, ",errors=remount-ro"); + if (sbi->s_mount_opts & ZONEFS_MNTOPT_ERRORS_ZRO) + seq_puts(seq, ",errors=zone-ro"); + if (sbi->s_mount_opts & ZONEFS_MNTOPT_ERRORS_ZOL) + seq_puts(seq, ",errors=zone-offline"); + if (sbi->s_mount_opts & ZONEFS_MNTOPT_ERRORS_REPAIR) + seq_puts(seq, ",errors=repair"); + + return 0; +} + +static int zonefs_inode_setattr(struct mnt_idmap *idmap, + struct dentry *dentry, struct iattr *iattr) +{ + struct inode *inode = d_inode(dentry); + int ret; + + if (unlikely(IS_IMMUTABLE(inode))) + return -EPERM; + + ret = setattr_prepare(&nop_mnt_idmap, dentry, iattr); + if (ret) + return ret; + + /* + * Since files and directories cannot be created nor deleted, do not + * allow setting any write attributes on the sub-directories grouping + * files by zone type. + */ + if ((iattr->ia_valid & ATTR_MODE) && S_ISDIR(inode->i_mode) && + (iattr->ia_mode & 0222)) + return -EPERM; + + if (((iattr->ia_valid & ATTR_UID) && + !uid_eq(iattr->ia_uid, inode->i_uid)) || + ((iattr->ia_valid & ATTR_GID) && + !gid_eq(iattr->ia_gid, inode->i_gid))) { + ret = dquot_transfer(&nop_mnt_idmap, inode, iattr); + if (ret) + return ret; + } + + if (iattr->ia_valid & ATTR_SIZE) { + ret = zonefs_file_truncate(inode, iattr->ia_size); + if (ret) + return ret; + } + + setattr_copy(&nop_mnt_idmap, inode, iattr); + + if (S_ISREG(inode->i_mode)) { + struct zonefs_zone *z = zonefs_inode_zone(inode); + + z->z_mode = inode->i_mode; + z->z_uid = inode->i_uid; + z->z_gid = inode->i_gid; + } + + return 0; +} + +static const struct inode_operations zonefs_file_inode_operations = { + .setattr = zonefs_inode_setattr, +}; + +static long zonefs_fname_to_fno(const struct qstr *fname) +{ + const char *name = fname->name; + unsigned int len = fname->len; + long fno = 0, shift = 1; + const char *rname; + char c = *name; + unsigned int i; + + /* + * File names are always a base-10 number string without any + * leading 0s. + */ + if (!isdigit(c)) + return -ENOENT; + + if (len > 1 && c == '0') + return -ENOENT; + + if (len == 1) + return c - '0'; + + for (i = 0, rname = name + len - 1; i < len; i++, rname--) { + long digit; + + c = *rname; + if (!isdigit(c)) + return -ENOENT; + digit = (c - '0') * shift; + if (check_add_overflow(fno, digit, &fno)) + return -ENOENT; + shift *= 10; + } + + return fno; +} + +static struct inode *zonefs_get_file_inode(struct inode *dir, + struct dentry *dentry) +{ + struct zonefs_zone_group *zgroup = dir->i_private; + struct super_block *sb = dir->i_sb; + struct zonefs_sb_info *sbi = ZONEFS_SB(sb); + struct zonefs_zone *z; + struct inode *inode; + ino_t ino; + long fno; + + /* Get the file number from the file name */ + fno = zonefs_fname_to_fno(&dentry->d_name); + if (fno < 0) + return ERR_PTR(fno); + + if (!zgroup->g_nr_zones || fno >= zgroup->g_nr_zones) + return ERR_PTR(-ENOENT); + + z = &zgroup->g_zones[fno]; + ino = z->z_sector >> sbi->s_zone_sectors_shift; + inode = iget_locked(sb, ino); + if (!inode) + return ERR_PTR(-ENOMEM); + if (!(inode_state_read_once(inode) & I_NEW)) { + WARN_ON_ONCE(inode->i_private != z); + return inode; + } + + inode->i_ino = ino; + inode->i_mode = z->z_mode; + inode_set_mtime_to_ts(inode, + inode_set_atime_to_ts(inode, inode_set_ctime_to_ts(inode, inode_get_ctime(dir)))); + inode->i_uid = z->z_uid; + inode->i_gid = z->z_gid; + inode->i_size = z->z_wpoffset; + inode->i_blocks = z->z_capacity >> SECTOR_SHIFT; + inode->i_private = z; + + inode->i_op = &zonefs_file_inode_operations; + inode->i_fop = &zonefs_file_operations; + inode->i_mapping->a_ops = &zonefs_file_aops; + mapping_set_large_folios(inode->i_mapping); + + /* Update the inode access rights depending on the zone condition */ + zonefs_inode_update_mode(inode); + + unlock_new_inode(inode); + + return inode; +} + +static struct inode *zonefs_get_zgroup_inode(struct super_block *sb, + enum zonefs_ztype ztype) +{ + struct inode *root = d_inode(sb->s_root); + struct zonefs_sb_info *sbi = ZONEFS_SB(sb); + struct inode *inode; + ino_t ino = bdev_nr_zones(sb->s_bdev) + ztype + 1; + + inode = iget_locked(sb, ino); + if (!inode) + return ERR_PTR(-ENOMEM); + if (!(inode_state_read_once(inode) & I_NEW)) + return inode; + + inode->i_ino = ino; + inode_init_owner(&nop_mnt_idmap, inode, root, S_IFDIR | 0555); + inode->i_size = sbi->s_zgroup[ztype].g_nr_zones; + inode_set_mtime_to_ts(inode, + inode_set_atime_to_ts(inode, inode_set_ctime_to_ts(inode, inode_get_ctime(root)))); + inode->i_private = &sbi->s_zgroup[ztype]; + set_nlink(inode, 2); + + inode->i_op = &zonefs_dir_inode_operations; + inode->i_fop = &zonefs_dir_operations; + + unlock_new_inode(inode); + + return inode; +} + + +static struct inode *zonefs_get_dir_inode(struct inode *dir, + struct dentry *dentry) +{ + struct super_block *sb = dir->i_sb; + struct zonefs_sb_info *sbi = ZONEFS_SB(sb); + const char *name = dentry->d_name.name; + enum zonefs_ztype ztype; + + /* + * We only need to check for the "seq" directory and + * the "cnv" directory if we have conventional zones. + */ + if (dentry->d_name.len != 3) + return ERR_PTR(-ENOENT); + + for (ztype = 0; ztype < ZONEFS_ZTYPE_MAX; ztype++) { + if (sbi->s_zgroup[ztype].g_nr_zones && + memcmp(name, zonefs_zgroup_name(ztype), 3) == 0) + break; + } + if (ztype == ZONEFS_ZTYPE_MAX) + return ERR_PTR(-ENOENT); + + return zonefs_get_zgroup_inode(sb, ztype); +} + +static struct dentry *zonefs_lookup(struct inode *dir, struct dentry *dentry, + unsigned int flags) +{ + struct inode *inode; + + if (dentry->d_name.len > ZONEFS_NAME_MAX) + return ERR_PTR(-ENAMETOOLONG); + + if (dir == d_inode(dir->i_sb->s_root)) + inode = zonefs_get_dir_inode(dir, dentry); + else + inode = zonefs_get_file_inode(dir, dentry); + + return d_splice_alias(inode, dentry); +} + +static int zonefs_readdir_root(struct file *file, struct dir_context *ctx) +{ + struct inode *inode = file_inode(file); + struct super_block *sb = inode->i_sb; + struct zonefs_sb_info *sbi = ZONEFS_SB(sb); + enum zonefs_ztype ztype = ZONEFS_ZTYPE_CNV; + ino_t base_ino = bdev_nr_zones(sb->s_bdev) + 1; + + if (ctx->pos >= inode->i_size) + return 0; + + if (!dir_emit_dots(file, ctx)) + return 0; + + if (ctx->pos == 2) { + if (!sbi->s_zgroup[ZONEFS_ZTYPE_CNV].g_nr_zones) + ztype = ZONEFS_ZTYPE_SEQ; + + if (!dir_emit(ctx, zonefs_zgroup_name(ztype), 3, + base_ino + ztype, DT_DIR)) + return 0; + ctx->pos++; + } + + if (ctx->pos == 3 && ztype != ZONEFS_ZTYPE_SEQ) { + ztype = ZONEFS_ZTYPE_SEQ; + if (!dir_emit(ctx, zonefs_zgroup_name(ztype), 3, + base_ino + ztype, DT_DIR)) + return 0; + ctx->pos++; + } + + return 0; +} + +static int zonefs_readdir_zgroup(struct file *file, + struct dir_context *ctx) +{ + struct inode *inode = file_inode(file); + struct zonefs_zone_group *zgroup = inode->i_private; + struct super_block *sb = inode->i_sb; + struct zonefs_sb_info *sbi = ZONEFS_SB(sb); + struct zonefs_zone *z; + int fname_len; + char *fname; + ino_t ino; + int f; + + /* + * The size of zone group directories is equal to the number + * of zone files in the group and does note include the "." and + * ".." entries. Hence the "+ 2" here. + */ + if (ctx->pos >= inode->i_size + 2) + return 0; + + if (!dir_emit_dots(file, ctx)) + return 0; + + fname = kmalloc(ZONEFS_NAME_MAX, GFP_KERNEL); + if (!fname) + return -ENOMEM; + + for (f = ctx->pos - 2; f < zgroup->g_nr_zones; f++) { + z = &zgroup->g_zones[f]; + ino = z->z_sector >> sbi->s_zone_sectors_shift; + fname_len = snprintf(fname, ZONEFS_NAME_MAX - 1, "%u", f); + if (!dir_emit(ctx, fname, fname_len, ino, DT_REG)) + break; + ctx->pos++; + } + + kfree(fname); + + return 0; +} + +static int zonefs_readdir(struct file *file, struct dir_context *ctx) +{ + struct inode *inode = file_inode(file); + + if (inode == d_inode(inode->i_sb->s_root)) + return zonefs_readdir_root(file, ctx); + + return zonefs_readdir_zgroup(file, ctx); +} + +const struct inode_operations zonefs_dir_inode_operations = { + .lookup = zonefs_lookup, + .setattr = zonefs_inode_setattr, +}; + +const struct file_operations zonefs_dir_operations = { + .llseek = generic_file_llseek, + .read = generic_read_dir, + .iterate_shared = zonefs_readdir, +}; + +struct zonefs_zone_data { + struct super_block *sb; + unsigned int nr_zones[ZONEFS_ZTYPE_MAX]; + sector_t cnv_zone_start; + struct blk_zone *zones; +}; + +static int zonefs_get_zone_info_cb(struct blk_zone *zone, unsigned int idx, + void *data) +{ + struct zonefs_zone_data *zd = data; + struct super_block *sb = zd->sb; + struct zonefs_sb_info *sbi = ZONEFS_SB(sb); + + /* + * We do not care about the first zone: it contains the super block + * and not exposed as a file. + */ + if (!idx) + return 0; + + /* + * Count the number of zones that will be exposed as files. + * For sequential zones, we always have as many files as zones. + * FOr conventional zones, the number of files depends on if we have + * conventional zones aggregation enabled. + */ + switch (zone->type) { + case BLK_ZONE_TYPE_CONVENTIONAL: + if (sbi->s_features & ZONEFS_F_AGGRCNV) { + /* One file per set of contiguous conventional zones */ + if (!(sbi->s_zgroup[ZONEFS_ZTYPE_CNV].g_nr_zones) || + zone->start != zd->cnv_zone_start) + sbi->s_zgroup[ZONEFS_ZTYPE_CNV].g_nr_zones++; + zd->cnv_zone_start = zone->start + zone->len; + } else { + /* One file per zone */ + sbi->s_zgroup[ZONEFS_ZTYPE_CNV].g_nr_zones++; + } + break; + case BLK_ZONE_TYPE_SEQWRITE_REQ: + case BLK_ZONE_TYPE_SEQWRITE_PREF: + sbi->s_zgroup[ZONEFS_ZTYPE_SEQ].g_nr_zones++; + break; + default: + zonefs_err(zd->sb, "Unsupported zone type 0x%x\n", + zone->type); + return -EIO; + } + + memcpy(&zd->zones[idx], zone, sizeof(struct blk_zone)); + + return 0; +} + +static int zonefs_get_zone_info(struct zonefs_zone_data *zd) +{ + struct block_device *bdev = zd->sb->s_bdev; + int ret; + + zd->zones = kvzalloc_objs(struct blk_zone, bdev_nr_zones(bdev)); + if (!zd->zones) + return -ENOMEM; + + /* Get zones information from the device */ + ret = blkdev_report_zones(bdev, 0, BLK_ALL_ZONES, + zonefs_get_zone_info_cb, zd); + if (ret < 0) { + zonefs_err(zd->sb, "Zone report failed %d\n", ret); + return ret; + } + + if (ret != bdev_nr_zones(bdev)) { + zonefs_err(zd->sb, "Invalid zone report (%d/%u zones)\n", + ret, bdev_nr_zones(bdev)); + return -EIO; + } + + return 0; +} + +static inline void zonefs_free_zone_info(struct zonefs_zone_data *zd) +{ + kvfree(zd->zones); +} + +/* + * Create a zone group and populate it with zone files. + */ +static int zonefs_init_zgroup(struct super_block *sb, + struct zonefs_zone_data *zd, + enum zonefs_ztype ztype) +{ + struct zonefs_sb_info *sbi = ZONEFS_SB(sb); + struct zonefs_zone_group *zgroup = &sbi->s_zgroup[ztype]; + struct blk_zone *zone, *next, *end; + struct zonefs_zone *z; + unsigned int n = 0; + int ret; + + /* Allocate the zone group. If it is empty, we have nothing to do. */ + if (!zgroup->g_nr_zones) + return 0; + + zgroup->g_zones = kvzalloc_objs(struct zonefs_zone, zgroup->g_nr_zones); + if (!zgroup->g_zones) + return -ENOMEM; + + /* + * Initialize the zone groups using the device zone information. + * We always skip the first zone as it contains the super block + * and is not use to back a file. + */ + end = zd->zones + bdev_nr_zones(sb->s_bdev); + for (zone = &zd->zones[1]; zone < end; zone = next) { + + next = zone + 1; + if (zonefs_zone_type(zone) != ztype) + continue; + + if (WARN_ON_ONCE(n >= zgroup->g_nr_zones)) + return -EINVAL; + + /* + * For conventional zones, contiguous zones can be aggregated + * together to form larger files. Note that this overwrites the + * length of the first zone of the set of contiguous zones + * aggregated together. If one offline or read-only zone is + * found, assume that all zones aggregated have the same + * condition. + */ + if (ztype == ZONEFS_ZTYPE_CNV && + (sbi->s_features & ZONEFS_F_AGGRCNV)) { + for (; next < end; next++) { + if (zonefs_zone_type(next) != ztype) + break; + zone->len += next->len; + zone->capacity += next->capacity; + if (next->cond == BLK_ZONE_COND_READONLY && + zone->cond != BLK_ZONE_COND_OFFLINE) + zone->cond = BLK_ZONE_COND_READONLY; + else if (next->cond == BLK_ZONE_COND_OFFLINE) + zone->cond = BLK_ZONE_COND_OFFLINE; + } + } + + z = &zgroup->g_zones[n]; + if (ztype == ZONEFS_ZTYPE_CNV) + z->z_flags |= ZONEFS_ZONE_CNV; + z->z_sector = zone->start; + z->z_size = zone->len << SECTOR_SHIFT; + if (z->z_size > bdev_zone_sectors(sb->s_bdev) << SECTOR_SHIFT && + !(sbi->s_features & ZONEFS_F_AGGRCNV)) { + zonefs_err(sb, + "Invalid zone size %llu (device zone sectors %llu)\n", + z->z_size, + bdev_zone_sectors(sb->s_bdev) << SECTOR_SHIFT); + return -EINVAL; + } + + z->z_capacity = min_t(loff_t, MAX_LFS_FILESIZE, + zone->capacity << SECTOR_SHIFT); + z->z_wpoffset = zonefs_check_zone_condition(sb, z, zone); + + z->z_mode = S_IFREG | sbi->s_perm; + z->z_uid = sbi->s_uid; + z->z_gid = sbi->s_gid; + + /* + * Let zonefs_inode_update_mode() know that we will need + * special initialization of the inode mode the first time + * it is accessed. + */ + z->z_flags |= ZONEFS_ZONE_INIT_MODE; + + sb->s_maxbytes = max(z->z_capacity, sb->s_maxbytes); + sbi->s_blocks += z->z_capacity >> sb->s_blocksize_bits; + sbi->s_used_blocks += z->z_wpoffset >> sb->s_blocksize_bits; + + /* + * For sequential zones, make sure that any open zone is closed + * first to ensure that the initial number of open zones is 0, + * in sync with the open zone accounting done when the mount + * option ZONEFS_MNTOPT_EXPLICIT_OPEN is used. + */ + if (ztype == ZONEFS_ZTYPE_SEQ && + (zone->cond == BLK_ZONE_COND_IMP_OPEN || + zone->cond == BLK_ZONE_COND_EXP_OPEN)) { + ret = zonefs_zone_mgmt(sb, z, REQ_OP_ZONE_CLOSE); + if (ret) + return ret; + } + + zonefs_account_active(sb, z); + + n++; + } + + if (WARN_ON_ONCE(n != zgroup->g_nr_zones)) + return -EINVAL; + + zonefs_info(sb, "Zone group \"%s\" has %u file%s\n", + zonefs_zgroup_name(ztype), + zgroup->g_nr_zones, + str_plural(zgroup->g_nr_zones)); + + return 0; +} + +static void zonefs_free_zgroups(struct super_block *sb) +{ + struct zonefs_sb_info *sbi = ZONEFS_SB(sb); + enum zonefs_ztype ztype; + + if (!sbi) + return; + + for (ztype = 0; ztype < ZONEFS_ZTYPE_MAX; ztype++) { + kvfree(sbi->s_zgroup[ztype].g_zones); + sbi->s_zgroup[ztype].g_zones = NULL; + } +} + +/* + * Create a zone group and populate it with zone files. + */ +static int zonefs_init_zgroups(struct super_block *sb) +{ + struct zonefs_zone_data zd; + enum zonefs_ztype ztype; + int ret; + + /* First get the device zone information */ + memset(&zd, 0, sizeof(struct zonefs_zone_data)); + zd.sb = sb; + ret = zonefs_get_zone_info(&zd); + if (ret) + goto cleanup; + + /* Allocate and initialize the zone groups */ + for (ztype = 0; ztype < ZONEFS_ZTYPE_MAX; ztype++) { + ret = zonefs_init_zgroup(sb, &zd, ztype); + if (ret) { + zonefs_info(sb, + "Zone group \"%s\" initialization failed\n", + zonefs_zgroup_name(ztype)); + break; + } + } + +cleanup: + zonefs_free_zone_info(&zd); + if (ret) + zonefs_free_zgroups(sb); + + return ret; +} + +/* + * Read super block information from the device. + */ +static int zonefs_read_super(struct super_block *sb) +{ + struct zonefs_sb_info *sbi = ZONEFS_SB(sb); + struct zonefs_super *super; + u32 crc, stored_crc; + int ret; + + super = kmalloc(ZONEFS_SUPER_SIZE, GFP_KERNEL); + if (!super) + return -ENOMEM; + + ret = bdev_rw_virt(sb->s_bdev, 0, super, ZONEFS_SUPER_SIZE, + REQ_OP_READ); + if (ret) + goto free_super; + + ret = -EINVAL; + if (le32_to_cpu(super->s_magic) != ZONEFS_MAGIC) + goto free_super; + + stored_crc = le32_to_cpu(super->s_crc); + super->s_crc = 0; + crc = crc32(~0U, (unsigned char *)super, sizeof(struct zonefs_super)); + if (crc != stored_crc) { + zonefs_err(sb, "Invalid checksum (Expected 0x%08x, got 0x%08x)", + crc, stored_crc); + goto free_super; + } + + sbi->s_features = le64_to_cpu(super->s_features); + if (sbi->s_features & ~ZONEFS_F_DEFINED_FEATURES) { + zonefs_err(sb, "Unknown features set 0x%llx\n", + sbi->s_features); + goto free_super; + } + + if (sbi->s_features & ZONEFS_F_UID) { + sbi->s_uid = make_kuid(current_user_ns(), + le32_to_cpu(super->s_uid)); + if (!uid_valid(sbi->s_uid)) { + zonefs_err(sb, "Invalid UID feature\n"); + goto free_super; + } + } + + if (sbi->s_features & ZONEFS_F_GID) { + sbi->s_gid = make_kgid(current_user_ns(), + le32_to_cpu(super->s_gid)); + if (!gid_valid(sbi->s_gid)) { + zonefs_err(sb, "Invalid GID feature\n"); + goto free_super; + } + } + + if (sbi->s_features & ZONEFS_F_PERM) + sbi->s_perm = le32_to_cpu(super->s_perm); + + if (memchr_inv(super->s_reserved, 0, sizeof(super->s_reserved))) { + zonefs_err(sb, "Reserved area is being used\n"); + goto free_super; + } + + import_uuid(&sbi->s_uuid, super->s_uuid); + ret = 0; + +free_super: + kfree(super); + return ret; +} + +static const struct super_operations zonefs_sops = { + .alloc_inode = zonefs_alloc_inode, + .free_inode = zonefs_free_inode, + .statfs = zonefs_statfs, + .show_options = zonefs_show_options, +}; + +static int zonefs_get_zgroup_inodes(struct super_block *sb) +{ + struct zonefs_sb_info *sbi = ZONEFS_SB(sb); + struct inode *dir_inode; + enum zonefs_ztype ztype; + + for (ztype = 0; ztype < ZONEFS_ZTYPE_MAX; ztype++) { + if (!sbi->s_zgroup[ztype].g_nr_zones) + continue; + + dir_inode = zonefs_get_zgroup_inode(sb, ztype); + if (IS_ERR(dir_inode)) + return PTR_ERR(dir_inode); + + sbi->s_zgroup[ztype].g_inode = dir_inode; + } + + return 0; +} + +static void zonefs_release_zgroup_inodes(struct super_block *sb) +{ + struct zonefs_sb_info *sbi = ZONEFS_SB(sb); + enum zonefs_ztype ztype; + + if (!sbi) + return; + + for (ztype = 0; ztype < ZONEFS_ZTYPE_MAX; ztype++) { + if (sbi->s_zgroup[ztype].g_inode) { + iput(sbi->s_zgroup[ztype].g_inode); + sbi->s_zgroup[ztype].g_inode = NULL; + } + } +} + +/* + * Check that the device is zoned. If it is, get the list of zones and create + * sub-directories and files according to the device zone configuration and + * format options. + */ +static int zonefs_fill_super(struct super_block *sb, struct fs_context *fc) +{ + struct zonefs_sb_info *sbi; + struct zonefs_context *ctx = fc->fs_private; + struct inode *inode; + enum zonefs_ztype ztype; + int ret; + + if (!bdev_is_zoned(sb->s_bdev)) { + zonefs_err(sb, "Not a zoned block device\n"); + return -EINVAL; + } + + /* + * Initialize super block information: the maximum file size is updated + * when the zone files are created so that the format option + * ZONEFS_F_AGGRCNV which increases the maximum file size of a file + * beyond the zone size is taken into account. + */ + sbi = kzalloc_obj(*sbi); + if (!sbi) + return -ENOMEM; + + spin_lock_init(&sbi->s_lock); + sb->s_fs_info = sbi; + sb->s_magic = ZONEFS_MAGIC; + sb->s_maxbytes = 0; + sb->s_op = &zonefs_sops; + sb->s_time_gran = 1; + + /* + * The block size is set to the device zone write granularity to ensure + * that write operations are always aligned according to the device + * interface constraints. + */ + sb_set_blocksize(sb, bdev_zone_write_granularity(sb->s_bdev)); + sbi->s_zone_sectors_shift = ilog2(bdev_zone_sectors(sb->s_bdev)); + sbi->s_uid = GLOBAL_ROOT_UID; + sbi->s_gid = GLOBAL_ROOT_GID; + sbi->s_perm = 0640; + sbi->s_mount_opts = ctx->s_mount_opts; + + atomic_set(&sbi->s_wro_seq_files, 0); + sbi->s_max_wro_seq_files = bdev_max_open_zones(sb->s_bdev); + atomic_set(&sbi->s_active_seq_files, 0); + sbi->s_max_active_seq_files = bdev_max_active_zones(sb->s_bdev); + + ret = zonefs_read_super(sb); + if (ret) + return ret; + + zonefs_info(sb, "Mounting %u zones", bdev_nr_zones(sb->s_bdev)); + + if (!sbi->s_max_wro_seq_files && + !sbi->s_max_active_seq_files && + sbi->s_mount_opts & ZONEFS_MNTOPT_EXPLICIT_OPEN) { + zonefs_info(sb, + "No open and active zone limits. Ignoring explicit_open mount option\n"); + sbi->s_mount_opts &= ~ZONEFS_MNTOPT_EXPLICIT_OPEN; + } + + /* Initialize the zone groups */ + ret = zonefs_init_zgroups(sb); + if (ret) + goto cleanup; + + /* Create the root directory inode */ + ret = -ENOMEM; + inode = new_inode(sb); + if (!inode) + goto cleanup; + + inode->i_ino = bdev_nr_zones(sb->s_bdev); + inode->i_mode = S_IFDIR | 0555; + simple_inode_init_ts(inode); + inode->i_op = &zonefs_dir_inode_operations; + inode->i_fop = &zonefs_dir_operations; + inode->i_size = 2; + set_nlink(inode, 2); + for (ztype = 0; ztype < ZONEFS_ZTYPE_MAX; ztype++) { + if (sbi->s_zgroup[ztype].g_nr_zones) { + inc_nlink(inode); + inode->i_size++; + } + } + + sb->s_root = d_make_root(inode); + if (!sb->s_root) + goto cleanup; + + /* + * Take a reference on the zone groups directory inodes + * to keep them in the inode cache. + */ + ret = zonefs_get_zgroup_inodes(sb); + if (ret) + goto cleanup; + + ret = zonefs_sysfs_register(sb); + if (ret) + goto cleanup; + + return 0; + +cleanup: + zonefs_release_zgroup_inodes(sb); + zonefs_free_zgroups(sb); + + return ret; +} + +static void zonefs_kill_super(struct super_block *sb) +{ + struct zonefs_sb_info *sbi = ZONEFS_SB(sb); + + /* Release the reference on the zone group directory inodes */ + zonefs_release_zgroup_inodes(sb); + + kill_block_super(sb); + + zonefs_sysfs_unregister(sb); + zonefs_free_zgroups(sb); + kfree(sbi); +} + +static void zonefs_free_fc(struct fs_context *fc) +{ + struct zonefs_context *ctx = fc->fs_private; + + kfree(ctx); +} + +static int zonefs_get_tree(struct fs_context *fc) +{ + return get_tree_bdev(fc, zonefs_fill_super); +} + +static int zonefs_reconfigure(struct fs_context *fc) +{ + struct zonefs_context *ctx = fc->fs_private; + struct super_block *sb = fc->root->d_sb; + struct zonefs_sb_info *sbi = sb->s_fs_info; + + sync_filesystem(fc->root->d_sb); + /* Copy new options from ctx into sbi. */ + sbi->s_mount_opts = ctx->s_mount_opts; + + return 0; +} + +static const struct fs_context_operations zonefs_context_ops = { + .parse_param = zonefs_parse_param, + .get_tree = zonefs_get_tree, + .reconfigure = zonefs_reconfigure, + .free = zonefs_free_fc, +}; + +/* + * Set up the filesystem mount context. + */ +static int zonefs_init_fs_context(struct fs_context *fc) +{ + struct zonefs_context *ctx; + + ctx = kzalloc_obj(struct zonefs_context); + if (!ctx) + return -ENOMEM; + ctx->s_mount_opts = ZONEFS_MNTOPT_ERRORS_RO; + fc->ops = &zonefs_context_ops; + fc->fs_private = ctx; + + return 0; +} + +/* + * File system definition and registration. + */ +static struct file_system_type zonefs_type = { + .owner = THIS_MODULE, + .name = "zonefs", + .kill_sb = zonefs_kill_super, + .fs_flags = FS_REQUIRES_DEV, + .init_fs_context = zonefs_init_fs_context, + .parameters = zonefs_param_spec, +}; + +static int __init zonefs_init_inodecache(void) +{ + zonefs_inode_cachep = kmem_cache_create("zonefs_inode_cache", + sizeof(struct zonefs_inode_info), 0, + SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, + NULL); + if (zonefs_inode_cachep == NULL) + return -ENOMEM; + return 0; +} + +static void zonefs_destroy_inodecache(void) +{ + /* + * Make sure all delayed rcu free inodes are flushed before we + * destroy the inode cache. + */ + rcu_barrier(); + kmem_cache_destroy(zonefs_inode_cachep); +} + +static int __init zonefs_init(void) +{ + int ret; + + BUILD_BUG_ON(sizeof(struct zonefs_super) != ZONEFS_SUPER_SIZE); + + ret = zonefs_init_inodecache(); + if (ret) + return ret; + + ret = zonefs_sysfs_init(); + if (ret) + goto destroy_inodecache; + + ret = register_filesystem(&zonefs_type); + if (ret) + goto sysfs_exit; + + return 0; + +sysfs_exit: + zonefs_sysfs_exit(); +destroy_inodecache: + zonefs_destroy_inodecache(); + + return ret; +} + +static void __exit zonefs_exit(void) +{ + unregister_filesystem(&zonefs_type); + zonefs_sysfs_exit(); + zonefs_destroy_inodecache(); +} + +MODULE_AUTHOR("Damien Le Moal"); +MODULE_DESCRIPTION("Zone file system for zoned block devices"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS_FS("zonefs"); +module_init(zonefs_init); +module_exit(zonefs_exit); diff --git a/fs/zonefs/sysfs.c b/fs/zonefs/sysfs.c new file mode 100644 index 000000000..ff9a688f1 --- /dev/null +++ b/fs/zonefs/sysfs.c @@ -0,0 +1,135 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Simple file system for zoned block devices exposing zones as files. + * + * Copyright (C) 2022 Western Digital Corporation or its affiliates. + */ +#include +#include +#include + +#include "zonefs.h" + +struct zonefs_sysfs_attr { + struct attribute attr; + ssize_t (*show)(struct zonefs_sb_info *sbi, char *buf); +}; + +#define ZONEFS_SYSFS_ATTR_RO(name) \ +static struct zonefs_sysfs_attr zonefs_sysfs_attr_##name = __ATTR_RO(name) + +#define ATTR_LIST(name) &zonefs_sysfs_attr_##name.attr + +static ssize_t zonefs_sysfs_attr_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct zonefs_sb_info *sbi = + container_of(kobj, struct zonefs_sb_info, s_kobj); + struct zonefs_sysfs_attr *zonefs_attr = + container_of(attr, struct zonefs_sysfs_attr, attr); + + if (!zonefs_attr->show) + return 0; + + return zonefs_attr->show(sbi, buf); +} + +static ssize_t max_wro_seq_files_show(struct zonefs_sb_info *sbi, char *buf) +{ + return sysfs_emit(buf, "%u\n", sbi->s_max_wro_seq_files); +} +ZONEFS_SYSFS_ATTR_RO(max_wro_seq_files); + +static ssize_t nr_wro_seq_files_show(struct zonefs_sb_info *sbi, char *buf) +{ + return sysfs_emit(buf, "%d\n", atomic_read(&sbi->s_wro_seq_files)); +} +ZONEFS_SYSFS_ATTR_RO(nr_wro_seq_files); + +static ssize_t max_active_seq_files_show(struct zonefs_sb_info *sbi, char *buf) +{ + return sysfs_emit(buf, "%u\n", sbi->s_max_active_seq_files); +} +ZONEFS_SYSFS_ATTR_RO(max_active_seq_files); + +static ssize_t nr_active_seq_files_show(struct zonefs_sb_info *sbi, char *buf) +{ + return sysfs_emit(buf, "%d\n", atomic_read(&sbi->s_active_seq_files)); +} +ZONEFS_SYSFS_ATTR_RO(nr_active_seq_files); + +static struct attribute *zonefs_sysfs_attrs[] = { + ATTR_LIST(max_wro_seq_files), + ATTR_LIST(nr_wro_seq_files), + ATTR_LIST(max_active_seq_files), + ATTR_LIST(nr_active_seq_files), + NULL, +}; +ATTRIBUTE_GROUPS(zonefs_sysfs); + +static void zonefs_sysfs_sb_release(struct kobject *kobj) +{ + struct zonefs_sb_info *sbi = + container_of(kobj, struct zonefs_sb_info, s_kobj); + + complete(&sbi->s_kobj_unregister); +} + +static const struct sysfs_ops zonefs_sysfs_attr_ops = { + .show = zonefs_sysfs_attr_show, +}; + +static const struct kobj_type zonefs_sb_ktype = { + .default_groups = zonefs_sysfs_groups, + .sysfs_ops = &zonefs_sysfs_attr_ops, + .release = zonefs_sysfs_sb_release, +}; + +static struct kobject *zonefs_sysfs_root; + +int zonefs_sysfs_register(struct super_block *sb) +{ + struct zonefs_sb_info *sbi = ZONEFS_SB(sb); + int ret; + + super_set_sysfs_name_id(sb); + init_completion(&sbi->s_kobj_unregister); + ret = kobject_init_and_add(&sbi->s_kobj, &zonefs_sb_ktype, + zonefs_sysfs_root, "%s", sb->s_id); + if (ret) { + kobject_put(&sbi->s_kobj); + wait_for_completion(&sbi->s_kobj_unregister); + return ret; + } + + sbi->s_sysfs_registered = true; + + return 0; +} + +void zonefs_sysfs_unregister(struct super_block *sb) +{ + struct zonefs_sb_info *sbi = ZONEFS_SB(sb); + + if (!sbi || !sbi->s_sysfs_registered) + return; + + kobject_del(&sbi->s_kobj); + kobject_put(&sbi->s_kobj); + wait_for_completion(&sbi->s_kobj_unregister); +} + +int __init zonefs_sysfs_init(void) +{ + zonefs_sysfs_root = kobject_create_and_add("zonefs", fs_kobj); + if (!zonefs_sysfs_root) + return -ENOMEM; + + return 0; +} + +void zonefs_sysfs_exit(void) +{ + kobject_put(zonefs_sysfs_root); + zonefs_sysfs_root = NULL; +} diff --git a/fs/zonefs/trace.h b/fs/zonefs/trace.h new file mode 100644 index 000000000..7315a72ca --- /dev/null +++ b/fs/zonefs/trace.h @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * zonefs filesystem driver tracepoints. + * + * Copyright (C) 2021 Western Digital Corporation or its affiliates. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM zonefs + +#if !defined(_TRACE_ZONEFS_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_ZONEFS_H + +#include +#include +#include + +#include "zonefs.h" + +#define show_dev(dev) MAJOR(dev), MINOR(dev) + +TRACE_EVENT(zonefs_zone_mgmt, + TP_PROTO(struct super_block *sb, struct zonefs_zone *z, + enum req_op op), + TP_ARGS(sb, z, op), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u64, ino) + __field(enum req_op, op) + __field(sector_t, sector) + __field(sector_t, nr_sectors) + ), + TP_fast_assign( + __entry->dev = sb->s_dev; + __entry->ino = + z->z_sector >> ZONEFS_SB(sb)->s_zone_sectors_shift; + __entry->op = op; + __entry->sector = z->z_sector; + __entry->nr_sectors = z->z_size >> SECTOR_SHIFT; + ), + TP_printk("bdev=(%d,%d), ino=%llu op=%s, sector=%llu, nr_sectors=%llu", + show_dev(__entry->dev), __entry->ino, + blk_op_str(__entry->op), __entry->sector, + __entry->nr_sectors + ) +); + +TRACE_EVENT(zonefs_file_dio_append, + TP_PROTO(struct inode *inode, ssize_t size, ssize_t ret), + TP_ARGS(inode, size, ret), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u64, ino) + __field(sector_t, sector) + __field(ssize_t, size) + __field(loff_t, wpoffset) + __field(ssize_t, ret) + ), + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->sector = zonefs_inode_zone(inode)->z_sector; + __entry->size = size; + __entry->wpoffset = + zonefs_inode_zone(inode)->z_wpoffset; + __entry->ret = ret; + ), + TP_printk("bdev=(%d, %d), ino=%llu, sector=%llu, size=%zu, wpoffset=%llu, ret=%zu", + show_dev(__entry->dev), __entry->ino, + __entry->sector, __entry->size, __entry->wpoffset, + __entry->ret + ) +); + +TRACE_EVENT(zonefs_iomap_begin, + TP_PROTO(struct inode *inode, struct iomap *iomap), + TP_ARGS(inode, iomap), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u64, ino) + __field(u64, addr) + __field(loff_t, offset) + __field(u64, length) + ), + TP_fast_assign( + __entry->dev = inode->i_sb->s_dev; + __entry->ino = inode->i_ino; + __entry->addr = iomap->addr; + __entry->offset = iomap->offset; + __entry->length = iomap->length; + ), + TP_printk("bdev=(%d,%d), ino=%llu, addr=%llu, offset=%llu, length=%llu", + show_dev(__entry->dev), __entry->ino, + __entry->addr, __entry->offset, __entry->length + ) +); + +#endif /* _TRACE_ZONEFS_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace + +/* This part must be outside protection */ +#include diff --git a/fs/zonefs/zonefs.h b/fs/zonefs/zonefs.h new file mode 100644 index 000000000..817565224 --- /dev/null +++ b/fs/zonefs/zonefs.h @@ -0,0 +1,289 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Simple zone file system for zoned block devices. + * + * Copyright (C) 2019 Western Digital Corporation or its affiliates. + */ +#ifndef __ZONEFS_H__ +#define __ZONEFS_H__ + +#include +#include +#include +#include +#include +#include + +/* + * Maximum length of file names: this only needs to be large enough to fit + * the zone group directory names and a decimal zone number for file names. + * 16 characters is plenty. + */ +#define ZONEFS_NAME_MAX 16 + +/* + * Zone types: ZONEFS_ZTYPE_SEQ is used for all sequential zone types + * defined in linux/blkzoned.h, that is, BLK_ZONE_TYPE_SEQWRITE_REQ and + * BLK_ZONE_TYPE_SEQWRITE_PREF. + */ +enum zonefs_ztype { + ZONEFS_ZTYPE_CNV, + ZONEFS_ZTYPE_SEQ, + ZONEFS_ZTYPE_MAX, +}; + +static inline enum zonefs_ztype zonefs_zone_type(struct blk_zone *zone) +{ + if (zone->type == BLK_ZONE_TYPE_CONVENTIONAL) + return ZONEFS_ZTYPE_CNV; + return ZONEFS_ZTYPE_SEQ; +} + +#define ZONEFS_ZONE_INIT_MODE (1U << 0) +#define ZONEFS_ZONE_OPEN (1U << 1) +#define ZONEFS_ZONE_ACTIVE (1U << 2) +#define ZONEFS_ZONE_OFFLINE (1U << 3) +#define ZONEFS_ZONE_READONLY (1U << 4) +#define ZONEFS_ZONE_CNV (1U << 31) + +/* + * In-memory per-file inode zone data. + */ +struct zonefs_zone { + /* Zone state flags */ + unsigned int z_flags; + + /* Zone start sector (512B unit) */ + sector_t z_sector; + + /* Zone size (bytes) */ + loff_t z_size; + + /* Zone capacity (file maximum size, bytes) */ + loff_t z_capacity; + + /* Write pointer offset in the zone (sequential zones only, bytes) */ + loff_t z_wpoffset; + + /* Saved inode uid, gid and access rights */ + umode_t z_mode; + kuid_t z_uid; + kgid_t z_gid; +}; + +/* + * In memory zone group information: all zones of a group are exposed + * as files, one file per zone. + */ +struct zonefs_zone_group { + struct inode *g_inode; + unsigned int g_nr_zones; + struct zonefs_zone *g_zones; +}; + +/* + * In-memory inode data. + */ +struct zonefs_inode_info { + struct inode i_vnode; + + /* + * To serialise fully against both syscall and mmap based IO and + * sequential file truncation, two locks are used. For serializing + * zonefs_seq_file_truncate() against zonefs_iomap_begin(), that is, + * file truncate operations against block mapping, i_truncate_mutex is + * used. i_truncate_mutex also protects against concurrent accesses + * and changes to the inode private data, and in particular changes to + * a sequential file size on completion of direct IO writes. + * Serialization of mmap read IOs with truncate and syscall IO + * operations is done with invalidate_lock in addition to + * i_truncate_mutex. Only zonefs_seq_file_truncate() takes both lock + * (invalidate_lock first, i_truncate_mutex second). + */ + struct mutex i_truncate_mutex; + + /* guarded by i_truncate_mutex */ + unsigned int i_wr_refcnt; +}; + +static inline struct zonefs_inode_info *ZONEFS_I(struct inode *inode) +{ + return container_of(inode, struct zonefs_inode_info, i_vnode); +} + +static inline bool zonefs_zone_is_cnv(struct zonefs_zone *z) +{ + return z->z_flags & ZONEFS_ZONE_CNV; +} + +static inline bool zonefs_zone_is_seq(struct zonefs_zone *z) +{ + return !zonefs_zone_is_cnv(z); +} + +static inline struct zonefs_zone *zonefs_inode_zone(struct inode *inode) +{ + return inode->i_private; +} + +static inline bool zonefs_inode_is_cnv(struct inode *inode) +{ + return zonefs_zone_is_cnv(zonefs_inode_zone(inode)); +} + +static inline bool zonefs_inode_is_seq(struct inode *inode) +{ + return zonefs_zone_is_seq(zonefs_inode_zone(inode)); +} + +/* + * On-disk super block (block 0). + */ +#define ZONEFS_LABEL_LEN 64 +#define ZONEFS_UUID_SIZE 16 +#define ZONEFS_SUPER_SIZE 4096 + +struct zonefs_super { + + /* Magic number */ + __le32 s_magic; + + /* Checksum */ + __le32 s_crc; + + /* Volume label */ + char s_label[ZONEFS_LABEL_LEN]; + + /* 128-bit uuid */ + __u8 s_uuid[ZONEFS_UUID_SIZE]; + + /* Features */ + __le64 s_features; + + /* UID/GID to use for files */ + __le32 s_uid; + __le32 s_gid; + + /* File permissions */ + __le32 s_perm; + + /* Padding to ZONEFS_SUPER_SIZE bytes */ + __u8 s_reserved[3988]; + +} __packed; + +/* + * Feature flags: specified in the s_features field of the on-disk super + * block struct zonefs_super and in-memory in the s_feartures field of + * struct zonefs_sb_info. + */ +enum zonefs_features { + /* + * Aggregate contiguous conventional zones into a single file. + */ + ZONEFS_F_AGGRCNV = 1ULL << 0, + /* + * Use super block specified UID for files instead of default 0. + */ + ZONEFS_F_UID = 1ULL << 1, + /* + * Use super block specified GID for files instead of default 0. + */ + ZONEFS_F_GID = 1ULL << 2, + /* + * Use super block specified file permissions instead of default 640. + */ + ZONEFS_F_PERM = 1ULL << 3, +}; + +#define ZONEFS_F_DEFINED_FEATURES \ + (ZONEFS_F_AGGRCNV | ZONEFS_F_UID | ZONEFS_F_GID | ZONEFS_F_PERM) + +/* + * Mount options for zone write pointer error handling. + */ +#define ZONEFS_MNTOPT_ERRORS_RO (1 << 0) /* Make zone file readonly */ +#define ZONEFS_MNTOPT_ERRORS_ZRO (1 << 1) /* Make zone file offline */ +#define ZONEFS_MNTOPT_ERRORS_ZOL (1 << 2) /* Make zone file offline */ +#define ZONEFS_MNTOPT_ERRORS_REPAIR (1 << 3) /* Remount read-only */ +#define ZONEFS_MNTOPT_ERRORS_MASK \ + (ZONEFS_MNTOPT_ERRORS_RO | ZONEFS_MNTOPT_ERRORS_ZRO | \ + ZONEFS_MNTOPT_ERRORS_ZOL | ZONEFS_MNTOPT_ERRORS_REPAIR) +#define ZONEFS_MNTOPT_EXPLICIT_OPEN (1 << 4) /* Explicit open/close of zones on open/close */ + +/* + * In-memory Super block information. + */ +struct zonefs_sb_info { + + unsigned long s_mount_opts; + + spinlock_t s_lock; + + unsigned long long s_features; + kuid_t s_uid; + kgid_t s_gid; + umode_t s_perm; + uuid_t s_uuid; + unsigned int s_zone_sectors_shift; + + struct zonefs_zone_group s_zgroup[ZONEFS_ZTYPE_MAX]; + + loff_t s_blocks; + loff_t s_used_blocks; + + unsigned int s_max_wro_seq_files; + atomic_t s_wro_seq_files; + + unsigned int s_max_active_seq_files; + atomic_t s_active_seq_files; + + bool s_sysfs_registered; + struct kobject s_kobj; + struct completion s_kobj_unregister; +}; + +static inline struct zonefs_sb_info *ZONEFS_SB(struct super_block *sb) +{ + return sb->s_fs_info; +} + +#define zonefs_info(sb, format, args...) \ + pr_info("zonefs (%s): " format, sb->s_id, ## args) +#define zonefs_err(sb, format, args...) \ + pr_err("zonefs (%s) ERROR: " format, sb->s_id, ## args) +#define zonefs_warn(sb, format, args...) \ + pr_warn("zonefs (%s) WARNING: " format, sb->s_id, ## args) + +/* In super.c */ +void zonefs_inode_account_active(struct inode *inode); +int zonefs_inode_zone_mgmt(struct inode *inode, enum req_op op); +void zonefs_i_size_write(struct inode *inode, loff_t isize); +void zonefs_update_stats(struct inode *inode, loff_t new_isize); +void __zonefs_io_error(struct inode *inode, bool write); + +static inline void zonefs_io_error(struct inode *inode, bool write) +{ + struct zonefs_inode_info *zi = ZONEFS_I(inode); + + mutex_lock(&zi->i_truncate_mutex); + __zonefs_io_error(inode, write); + mutex_unlock(&zi->i_truncate_mutex); +} + +/* In super.c */ +extern const struct inode_operations zonefs_dir_inode_operations; +extern const struct file_operations zonefs_dir_operations; + +/* In file.c */ +extern const struct address_space_operations zonefs_file_aops; +extern const struct file_operations zonefs_file_operations; +int zonefs_file_truncate(struct inode *inode, loff_t isize); + +/* In sysfs.c */ +int zonefs_sysfs_register(struct super_block *sb); +void zonefs_sysfs_unregister(struct super_block *sb); +int zonefs_sysfs_init(void); +void zonefs_sysfs_exit(void); + +#endif -- cgit v1.3.1